diff --git a/.mailmap b/.mailmap
index 7fa9d41fbdaf..29ddeb1bf015 100644
--- a/.mailmap
+++ b/.mailmap
@@ -186,6 +186,9 @@ Uwe Kleine-König <ukleinek@informatik.uni-freiburg.de>
 Uwe Kleine-König <ukl@pengutronix.de>
 Uwe Kleine-König <Uwe.Kleine-Koenig@digi.com>
 Valdis Kletnieks <Valdis.Kletnieks@vt.edu>
+Vinod Koul <vkoul@kernel.org> <vinod.koul@intel.com>
+Vinod Koul <vkoul@kernel.org> <vinod.koul@linux.intel.com>
+Vinod Koul <vkoul@kernel.org> <vkoul@infradead.org>
 Viresh Kumar <vireshk@kernel.org> <viresh.kumar@st.com>
 Viresh Kumar <vireshk@kernel.org> <viresh.linux@gmail.com>
 Viresh Kumar <vireshk@kernel.org> <viresh.kumar2@arm.com>
diff --git a/Documentation/00-INDEX b/Documentation/00-INDEX
index 708dc4c166e4..2754fe83f0d4 100644
--- a/Documentation/00-INDEX
+++ b/Documentation/00-INDEX
@@ -64,8 +64,6 @@ auxdisplay/
 	- misc. LCD driver documentation (cfag12864b, ks0108).
 backlight/
 	- directory with info on controlling backlights in flat panel displays
-bcache.txt
-	- Block-layer cache on fast SSDs to improve slow (raid) I/O performance.
 block/
 	- info on the Block I/O (BIO) layer.
 blockdev/
@@ -78,18 +76,10 @@ bus-devices/
 	- directory with info on TI GPMC (General Purpose Memory Controller)
 bus-virt-phys-mapping.txt
 	- how to access I/O mapped memory from within device drivers.
-cachetlb.txt
-	- describes the cache/TLB flushing interfaces Linux uses.
 cdrom/
 	- directory with information on the CD-ROM drivers that Linux has.
 cgroup-v1/
 	- cgroups v1 features, including cpusets and memory controller.
-cgroup-v2.txt
-	- cgroups v2 features, including cpusets and memory controller.
-circular-buffers.txt
-	- how to make use of the existing circular buffer infrastructure
-clk.txt
-	- info on the common clock framework
 cma/
 	- Continuous Memory Area (CMA) debugfs interface.
 conf.py
diff --git a/Documentation/ABI/stable/sysfs-bus-vmbus b/Documentation/ABI/stable/sysfs-bus-vmbus
index 0c9d9dcd2151..3eaffbb2d468 100644
--- a/Documentation/ABI/stable/sysfs-bus-vmbus
+++ b/Documentation/ABI/stable/sysfs-bus-vmbus
@@ -1,25 +1,25 @@
-What:		/sys/bus/vmbus/devices/vmbus_*/id
+What:		/sys/bus/vmbus/devices/<UUID>/id
 Date:		Jul 2009
 KernelVersion:	2.6.31
 Contact:	K. Y. Srinivasan <kys@microsoft.com>
 Description:	The VMBus child_relid of the device's primary channel
 Users:		tools/hv/lsvmbus
 
-What:		/sys/bus/vmbus/devices/vmbus_*/class_id
+What:		/sys/bus/vmbus/devices/<UUID>/class_id
 Date:		Jul 2009
 KernelVersion:	2.6.31
 Contact:	K. Y. Srinivasan <kys@microsoft.com>
 Description:	The VMBus interface type GUID of the device
 Users:		tools/hv/lsvmbus
 
-What:		/sys/bus/vmbus/devices/vmbus_*/device_id
+What:		/sys/bus/vmbus/devices/<UUID>/device_id
 Date:		Jul 2009
 KernelVersion:	2.6.31
 Contact:	K. Y. Srinivasan <kys@microsoft.com>
 Description:	The VMBus interface instance GUID of the device
 Users:		tools/hv/lsvmbus
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channel_vp_mapping
+What:		/sys/bus/vmbus/devices/<UUID>/channel_vp_mapping
 Date:		Jul 2015
 KernelVersion:	4.2.0
 Contact:	K. Y. Srinivasan <kys@microsoft.com>
@@ -28,112 +28,112 @@ Description:	The mapping of which primary/sub channels are bound to which
 		Format: <channel's child_relid:the bound cpu's number>
 Users:		tools/hv/lsvmbus
 
-What:		/sys/bus/vmbus/devices/vmbus_*/device
+What:		/sys/bus/vmbus/devices/<UUID>/device
 Date:		Dec. 2015
 KernelVersion:	4.5
 Contact:	K. Y. Srinivasan <kys@microsoft.com>
 Description:	The 16 bit device ID of the device
 Users:		tools/hv/lsvmbus and user level RDMA libraries
 
-What:		/sys/bus/vmbus/devices/vmbus_*/vendor
+What:		/sys/bus/vmbus/devices/<UUID>/vendor
 Date:		Dec. 2015
 KernelVersion:	4.5
 Contact:	K. Y. Srinivasan <kys@microsoft.com>
 Description:	The 16 bit vendor ID of the device
 Users:		tools/hv/lsvmbus and user level RDMA libraries
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Directory for per-channel information
 		NN is the VMBUS relid associtated with the channel.
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/cpu
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/cpu
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	VCPU (sub)channel is affinitized to
 Users:		tools/hv/lsvmbus and other debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/cpu
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/cpu
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	VCPU (sub)channel is affinitized to
 Users:		tools/hv/lsvmbus and other debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/in_mask
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/in_mask
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Host to guest channel interrupt mask
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/latency
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/latency
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Channel signaling latency
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/out_mask
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/out_mask
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Guest to host channel interrupt mask
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/pending
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/pending
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Channel interrupt pending state
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/read_avail
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/read_avail
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Bytes available to read
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/write_avail
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/write_avail
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Bytes available to write
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/events
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/events
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Number of times we have signaled the host
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/interrupts
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/interrupts
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Number of times we have taken an interrupt (incoming)
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/subchannel_id
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/subchannel_id
 Date:		January. 2018
 KernelVersion:	4.16
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Subchannel ID associated with VMBUS channel
 Users:		Debugging tools and userspace drivers
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/monitor_id
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/monitor_id
 Date:		January. 2018
 KernelVersion:	4.16
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Monitor bit associated with channel
 Users:		Debugging tools and userspace drivers
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/ring
+What:		/sys/bus/vmbus/devices/<UUID>/channels/<N>/ring
 Date:		January. 2018
 KernelVersion:	4.16
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
diff --git a/Documentation/ABI/stable/sysfs-devices-node b/Documentation/ABI/stable/sysfs-devices-node
index 5b2d0f08867c..3e90e1f3bf0a 100644
--- a/Documentation/ABI/stable/sysfs-devices-node
+++ b/Documentation/ABI/stable/sysfs-devices-node
@@ -90,4 +90,4 @@ Date:		December 2009
 Contact:	Lee Schermerhorn <lee.schermerhorn@hp.com>
 Description:
 		The node's huge page size control/query attributes.
-		See Documentation/vm/hugetlbpage.txt
\ No newline at end of file
+		See Documentation/admin-guide/mm/hugetlbpage.rst
\ No newline at end of file
diff --git a/Documentation/ABI/testing/evm b/Documentation/ABI/testing/evm
index d12cb2eae9ee..201d10319fa1 100644
--- a/Documentation/ABI/testing/evm
+++ b/Documentation/ABI/testing/evm
@@ -57,3 +57,16 @@ Description:
 		dracut (via 97masterkey and 98integrity) and systemd (via
 		core/ima-setup) have support for loading keys at boot
 		time.
+
+What:		security/integrity/evm/evm_xattrs
+Date:		April 2018
+Contact:	Matthew Garrett <mjg59@google.com>
+Description:
+		Shows the set of extended attributes used to calculate or
+		validate the EVM signature, and allows additional attributes
+		to be added at runtime. Any signatures generated after
+		additional attributes are added (and on files posessing those
+		additional attributes) will only be valid if the same
+		additional attributes are configured on system boot. Writing
+		a single period (.) will lock the xattr list from any further
+		modification.
diff --git a/Documentation/ABI/testing/ima_policy b/Documentation/ABI/testing/ima_policy
index b8465e00ba5f..74c6702de74e 100644
--- a/Documentation/ABI/testing/ima_policy
+++ b/Documentation/ABI/testing/ima_policy
@@ -21,7 +21,7 @@ Description:
 			audit | hash | dont_hash
 		condition:= base | lsm  [option]
 			base:	[[func=] [mask=] [fsmagic=] [fsuuid=] [uid=]
-				[euid=] [fowner=]]
+				[euid=] [fowner=] [fsname=]]
 			lsm:	[[subj_user=] [subj_role=] [subj_type=]
 				 [obj_user=] [obj_role=] [obj_type=]]
 			option:	[[appraise_type=]] [permit_directio]
diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio
index 6a5f34b4d5b9..731146c3b138 100644
--- a/Documentation/ABI/testing/sysfs-bus-iio
+++ b/Documentation/ABI/testing/sysfs-bus-iio
@@ -190,6 +190,13 @@ Description:
 		but should match other such assignments on device).
 		Units after application of scale and offset are m/s^2.
 
+What:		/sys/bus/iio/devices/iio:deviceX/in_angl_raw
+KernelVersion:	4.17
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Angle of rotation. Units after application of scale and offset
+		are radians.
+
 What:		/sys/bus/iio/devices/iio:deviceX/in_anglvel_x_raw
 What:		/sys/bus/iio/devices/iio:deviceX/in_anglvel_y_raw
 What:		/sys/bus/iio/devices/iio:deviceX/in_anglvel_z_raw
@@ -297,6 +304,7 @@ What:		/sys/bus/iio/devices/iio:deviceX/in_pressure_offset
 What:		/sys/bus/iio/devices/iio:deviceX/in_humidityrelative_offset
 What:		/sys/bus/iio/devices/iio:deviceX/in_magn_offset
 What:		/sys/bus/iio/devices/iio:deviceX/in_rot_offset
+What:		/sys/bus/iio/devices/iio:deviceX/in_angl_offset
 KernelVersion:	2.6.35
 Contact:	linux-iio@vger.kernel.org
 Description:
@@ -350,6 +358,7 @@ What:		/sys/bus/iio/devices/iio:deviceX/in_humidityrelative_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_velocity_sqrt(x^2+y^2+z^2)_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_illuminance_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_countY_scale
+What:		/sys/bus/iio/devices/iio:deviceX/in_angl_scale
 KernelVersion:	2.6.35
 Contact:	linux-iio@vger.kernel.org
 Description:
diff --git a/Documentation/ABI/testing/sysfs-bus-rpmsg b/Documentation/ABI/testing/sysfs-bus-rpmsg
index 189e419a5a2d..990fcc420935 100644
--- a/Documentation/ABI/testing/sysfs-bus-rpmsg
+++ b/Documentation/ABI/testing/sysfs-bus-rpmsg
@@ -73,3 +73,23 @@ Description:
 		This sysfs entry tells us whether the channel is a local
 		server channel that is announced (values are either
 		true or false).
+
+What:		/sys/bus/rpmsg/devices/.../driver_override
+Date:		April 2018
+KernelVersion:	4.18
+Contact:	Bjorn Andersson <bjorn.andersson@linaro.org>
+Description:
+		Every rpmsg device is a communication channel with a remote
+		processor. Channels are identified by a textual name (see
+		/sys/bus/rpmsg/devices/.../name above) and have a local
+		("source") rpmsg address, and remote ("destination") rpmsg
+		address.
+
+		The listening entity (or client) which communicates with a
+		remote processor is referred as rpmsg driver. The rpmsg device
+		and rpmsg driver are matched based on rpmsg device name and
+		rpmsg driver ID table.
+
+		This sysfs entry allows the rpmsg driver for a rpmsg device
+		to be specified which will override standard OF, ID table
+		and name matching.
diff --git a/Documentation/ABI/testing/sysfs-bus-usb b/Documentation/ABI/testing/sysfs-bus-usb
index c702c78f24d8..c6e9b30f05b1 100644
--- a/Documentation/ABI/testing/sysfs-bus-usb
+++ b/Documentation/ABI/testing/sysfs-bus-usb
@@ -236,3 +236,21 @@ Description:
 		Supported values are 0 - 15.
 		More information on how besl values map to microseconds can be found in
 		USB 2.0 ECN Errata for Link Power Management, section 4.10)
+
+What:		/sys/bus/usb/devices/.../rx_lanes
+Date:		March 2018
+Contact:	Mathias Nyman <mathias.nyman@linux.intel.com>
+Description:
+		Number of rx lanes the device is using.
+		USB 3.2 adds Dual-lane support, 2 rx and 2 tx lanes over Type-C.
+		Inter-Chip SSIC devices support asymmetric lanes up to 4 lanes per
+		direction. Devices before USB 3.2 are single lane (rx_lanes = 1)
+
+What:		/sys/bus/usb/devices/.../tx_lanes
+Date:		March 2018
+Contact:	Mathias Nyman <mathias.nyman@linux.intel.com>
+Description:
+		Number of tx lanes the device is using.
+		USB 3.2 adds Dual-lane support, 2 rx and 2 tx -lanes over Type-C.
+		Inter-Chip SSIC devices support asymmetric lanes up to 4 lanes per
+		direction. Devices before USB 3.2 are single lane (tx_lanes = 1)
diff --git a/Documentation/ABI/testing/sysfs-class-bdi b/Documentation/ABI/testing/sysfs-class-bdi
index d773d5697cf5..3187a18af6da 100644
--- a/Documentation/ABI/testing/sysfs-class-bdi
+++ b/Documentation/ABI/testing/sysfs-class-bdi
@@ -53,3 +53,11 @@ stable_pages_required (read-only)
 
 	If set, the backing device requires that all pages comprising a write
 	request must not be changed until writeout is complete.
+
+strictlimit (read-write)
+
+	Forces per-BDI checks for the share of given device in the write-back
+	cache even before the global background dirty limit is reached. This
+	is useful in situations where the global limit is much higher than
+	affordable for given relatively slow (or untrusted) device. Turning
+	strictlimit on has no visible effect if max_ratio is equal to 100%.
diff --git a/Documentation/ABI/testing/sysfs-class-mtd b/Documentation/ABI/testing/sysfs-class-mtd
index f34e592301d1..3bc7c0a95c92 100644
--- a/Documentation/ABI/testing/sysfs-class-mtd
+++ b/Documentation/ABI/testing/sysfs-class-mtd
@@ -232,3 +232,11 @@ Description:
 		of the parent (another partition or a flash device) in bytes.
 		This attribute is absent on flash devices, so it can be used
 		to distinguish them from partitions.
+
+What:		/sys/class/mtd/mtdX/oobavail
+Date:		April 2018
+KernelVersion:	4.16
+Contact:	linux-mtd@lists.infradead.org
+Description:
+		Number of bytes available for a client to place data into
+		the out of band area.
diff --git a/Documentation/ABI/testing/sysfs-class-power b/Documentation/ABI/testing/sysfs-class-power
index f85ce9e327b9..5e23e22dce1b 100644
--- a/Documentation/ABI/testing/sysfs-class-power
+++ b/Documentation/ABI/testing/sysfs-class-power
@@ -1,3 +1,458 @@
+===== General Properties =====
+
+What:		/sys/class/power_supply/<supply_name>/manufacturer
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the name of the device manufacturer.
+
+		Access: Read
+		Valid values: Represented as string
+
+What:		/sys/class/power_supply/<supply_name>/model_name
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the name of the device model.
+
+		Access: Read
+		Valid values: Represented as string
+
+What:		/sys/class/power_supply/<supply_name>/serial_number
+Date:		January 2008
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the serial number of the device.
+
+		Access: Read
+		Valid values: Represented as string
+
+What:		/sys/class/power_supply/<supply_name>/type
+Date:		May 2010
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Describes the main type of the supply.
+
+		Access: Read
+		Valid values: "Battery", "UPS", "Mains", "USB"
+
+===== Battery Properties =====
+
+What:		/sys/class/power_supply/<supply_name>/capacity
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Fine grain representation of battery capacity.
+		Access: Read
+		Valid values: 0 - 100 (percent)
+
+What:		/sys/class/power_supply/<supply_name>/capacity_alert_max
+Date:		July 2012
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Maximum battery capacity trip-wire value where the supply will
+		notify user-space of the event. This is normally used for the
+		battery discharging scenario where user-space needs to know the
+		battery has dropped to an upper level so it can take
+		appropriate action (e.g. warning user that battery level is
+		low).
+
+		Access: Read, Write
+		Valid values: 0 - 100 (percent)
+
+What:		/sys/class/power_supply/<supply_name>/capacity_alert_min
+Date:		July 2012
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Minimum battery capacity trip-wire value where the supply will
+		notify user-space of the event. This is normally used for the
+		battery discharging scenario where user-space needs to know the
+		battery has dropped to a lower level so it can take
+		appropriate action (e.g. warning user that battery level is
+		critically low).
+
+		Access: Read, Write
+		Valid values: 0 - 100 (percent)
+
+What:		/sys/class/power_supply/<supply_name>/capacity_level
+Date:		June 2009
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Coarse representation of battery capacity.
+
+		Access: Read
+		Valid values: "Unknown", "Critical", "Low", "Normal", "High",
+			      "Full"
+
+What:		/sys/class/power_supply/<supply_name>/current_avg
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports an average IBAT current reading for the battery, over a
+		fixed period. Normally devices will provide a fixed interval in
+		which they average readings to smooth out the reported value.
+
+		Access: Read
+		Valid values: Represented in microamps
+
+What:		/sys/class/power_supply/<supply_name>/current_max
+Date:		October 2010
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the maximum IBAT current allowed into the battery.
+
+		Access: Read
+		Valid values: Represented in microamps
+
+What:		/sys/class/power_supply/<supply_name>/current_now
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports an instant, single IBAT current reading for the battery.
+		This value is not averaged/smoothed.
+
+		Access: Read
+		Valid values: Represented in microamps
+
+What:		/sys/class/power_supply/<supply_name>/charge_type
+Date:		July 2009
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Represents the type of charging currently being applied to the
+		battery.
+
+		Access: Read
+		Valid values: "Unknown", "N/A", "Trickle", "Fast"
+
+What:		/sys/class/power_supply/<supply_name>/charge_term_current
+Date:		July 2014
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the charging current value which is used to determine
+		when the battery is considered full and charging should end.
+
+		Access: Read
+		Valid values: Represented in microamps
+
+What:		/sys/class/power_supply/<supply_name>/health
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the health of the battery or battery side of charger
+		functionality.
+
+		Access: Read
+		Valid values: "Unknown", "Good", "Overheat", "Dead",
+			      "Over voltage", "Unspecified failure", "Cold",
+			      "Watchdog timer expire", "Safety timer expire"
+
+What:		/sys/class/power_supply/<supply_name>/precharge_current
+Date:		June 2017
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the charging current applied during pre-charging phase
+		for a battery charge cycle.
+
+		Access: Read
+		Valid values: Represented in microamps
+
+What:		/sys/class/power_supply/<supply_name>/present
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports whether a battery is present or not in the system.
+
+		Access: Read
+		Valid values:
+			0: Absent
+			1: Present
+
+What:		/sys/class/power_supply/<supply_name>/status
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Represents the charging status of the battery. Normally this
+		is read-only reporting although for some supplies this can be
+		used to enable/disable charging to the battery.
+
+		Access: Read, Write
+		Valid values: "Unknown", "Charging", "Discharging",
+			      "Not charging", "Full"
+
+What:		/sys/class/power_supply/<supply_name>/technology
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Describes the battery technology supported by the supply.
+
+		Access: Read
+		Valid values: "Unknown", "NiMH", "Li-ion", "Li-poly", "LiFe",
+			      "NiCd", "LiMn"
+
+What:		/sys/class/power_supply/<supply_name>/temp
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the current TBAT battery temperature reading.
+
+		Access: Read
+		Valid values: Represented in 1/10 Degrees Celsius
+
+What:		/sys/class/power_supply/<supply_name>/temp_alert_max
+Date:		July 2012
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Maximum TBAT temperature trip-wire value where the supply will
+		notify user-space of the event. This is normally used for the
+		battery charging scenario where user-space needs to know the
+		battery temperature has crossed an upper threshold so it can
+		take appropriate action (e.g. warning user that battery level is
+		critically high, and charging has stopped).
+
+		Access: Read
+		Valid values: Represented in 1/10 Degrees Celsius
+
+What:		/sys/class/power_supply/<supply_name>/temp_alert_min
+Date:		July 2012
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Minimum TBAT temperature trip-wire value where the supply will
+		notify user-space of the event. This is normally used for the
+		battery charging scenario where user-space needs to know the
+		battery temperature has crossed a lower threshold so it can take
+		appropriate action (e.g. warning user that battery level is
+		high, and charging current has been reduced accordingly to
+		remedy the situation).
+
+		Access: Read
+		Valid values: Represented in 1/10 Degrees Celsius
+
+What:		/sys/class/power_supply/<supply_name>/temp_max
+Date:		July 2014
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the maximum allowed TBAT battery temperature for
+		charging.
+
+		Access: Read
+		Valid values: Represented in 1/10 Degrees Celsius
+
+What:		/sys/class/power_supply/<supply_name>/temp_min
+Date:		July 2014
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the minimum allowed TBAT battery temperature for
+		charging.
+
+		Access: Read
+		Valid values: Represented in 1/10 Degrees Celsius
+
+What:		/sys/class/power_supply/<supply_name>/voltage_avg,
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports an average VBAT voltage reading for the battery, over a
+		fixed period. Normally devices will provide a fixed interval in
+		which they average readings to smooth out the reported value.
+
+		Access: Read
+		Valid values: Represented in microvolts
+
+What:		/sys/class/power_supply/<supply_name>/voltage_max,
+Date:		January 2008
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the maximum safe VBAT voltage permitted for the battery,
+		during charging.
+
+		Access: Read
+		Valid values: Represented in microvolts
+
+What:		/sys/class/power_supply/<supply_name>/voltage_min,
+Date:		January 2008
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the minimum safe VBAT voltage permitted for the battery,
+		during discharging.
+
+		Access: Read
+		Valid values: Represented in microvolts
+
+What:		/sys/class/power_supply/<supply_name>/voltage_now,
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports an instant, single VBAT voltage reading for the battery.
+		This value is not averaged/smoothed.
+
+		Access: Read
+		Valid values: Represented in microvolts
+
+===== USB Properties =====
+
+What: 		/sys/class/power_supply/<supply_name>/current_avg
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports an average IBUS current reading over a fixed period.
+		Normally devices will provide a fixed interval in which they
+		average readings to smooth out the reported value.
+
+		Access: Read
+		Valid values: Represented in microamps
+
+
+What: 		/sys/class/power_supply/<supply_name>/current_max
+Date:		October 2010
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the maximum IBUS current the supply can support.
+
+		Access: Read
+		Valid values: Represented in microamps
+
+What: 		/sys/class/power_supply/<supply_name>/current_now
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the IBUS current supplied now. This value is generally
+		read-only reporting, unless the 'online' state of the supply
+		is set to be programmable, in which case this value can be set
+		within the reported min/max range.
+
+		Access: Read, Write
+		Valid values: Represented in microamps
+
+What:		/sys/class/power_supply/<supply_name>/input_current_limit
+Date:		July 2014
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Details the incoming IBUS current limit currently set in the
+		supply. Normally this is configured based on the type of
+		connection made (e.g. A configured SDP should output a maximum
+		of 500mA so the input current limit is set to the same value).
+
+		Access: Read, Write
+		Valid values: Represented in microamps
+
+What:		/sys/class/power_supply/<supply_name>/online,
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Indicates if VBUS is present for the supply. When the supply is
+		online, and the supply allows it, then it's possible to switch
+		between online states (e.g. Fixed -> Programmable for a PD_PPS
+		USB supply so voltage and current can be controlled).
+
+		Access: Read, Write
+		Valid values:
+			0: Offline
+			1: Online Fixed - Fixed Voltage Supply
+			2: Online Programmable - Programmable Voltage Supply
+
+What:		/sys/class/power_supply/<supply_name>/temp
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the current supply temperature reading. This would
+		normally be the internal temperature of the device itself (e.g
+		TJUNC temperature of an IC)
+
+		Access: Read
+		Valid values: Represented in 1/10 Degrees Celsius
+
+What:		/sys/class/power_supply/<supply_name>/temp_alert_max
+Date:		July 2012
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Maximum supply temperature trip-wire value where the supply will
+		notify user-space of the event. This is normally used for the
+		charging scenario where user-space needs to know the supply
+		temperature has crossed an upper threshold so it can take
+		appropriate action (e.g. warning user that the supply
+		temperature is critically high, and charging has stopped to
+		remedy the situation).
+
+		Access: Read
+		Valid values: Represented in 1/10 Degrees Celsius
+
+What:		/sys/class/power_supply/<supply_name>/temp_alert_min
+Date:		July 2012
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Minimum supply temperature trip-wire value where the supply will
+		notify user-space of the event. This is normally used for the
+		charging scenario where user-space needs to know the supply
+		temperature has crossed a lower threshold so it can take
+		appropriate action (e.g. warning user that the supply
+		temperature is high, and charging current has been reduced
+		accordingly to remedy the situation).
+
+		Access: Read
+		Valid values: Represented in 1/10 Degrees Celsius
+
+What:		/sys/class/power_supply/<supply_name>/temp_max
+Date:		July 2014
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the maximum allowed supply temperature for operation.
+
+		Access: Read
+		Valid values: Represented in 1/10 Degrees Celsius
+
+What:		/sys/class/power_supply/<supply_name>/temp_min
+Date:		July 2014
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the mainimum allowed supply temperature for operation.
+
+		Access: Read
+		Valid values: Represented in 1/10 Degrees Celsius
+
+What: 		/sys/class/power_supply/<supply_name>/usb_type
+Date:		March 2018
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports what type of USB connection is currently active for
+		the supply, for example it can show if USB-PD capable source
+		is attached.
+
+		Access: Read-Only
+		Valid values: "Unknown", "SDP", "DCP", "CDP", "ACA", "C", "PD",
+			      "PD_DRP", "PD_PPS", "BrickID"
+
+What: 		/sys/class/power_supply/<supply_name>/voltage_max
+Date:		January 2008
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the maximum VBUS voltage the supply can support.
+
+		Access: Read
+		Valid values: Represented in microvolts
+
+What: 		/sys/class/power_supply/<supply_name>/voltage_min
+Date:		January 2008
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the minimum VBUS voltage the supply can support.
+
+		Access: Read
+		Valid values: Represented in microvolts
+
+What: 		/sys/class/power_supply/<supply_name>/voltage_now
+Date:		May 2007
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Reports the VBUS voltage supplied now. This value is generally
+		read-only reporting, unless the 'online' state of the supply
+		is set to be programmable, in which case this value can be set
+		within the reported min/max range.
+
+		Access: Read, Write
+		Valid values: Represented in microvolts
+
+===== Device Specific Properties =====
+
 What:		/sys/class/power/ds2760-battery.*/charge_now
 Date:		May 2010
 KernelVersion:	2.6.35
diff --git a/Documentation/ABI/testing/sysfs-class-rc b/Documentation/ABI/testing/sysfs-class-rc
index 8be1fd3760e0..6c0d6c8cb911 100644
--- a/Documentation/ABI/testing/sysfs-class-rc
+++ b/Documentation/ABI/testing/sysfs-class-rc
@@ -1,7 +1,7 @@
 What:		/sys/class/rc/
 Date:		Apr 2010
 KernelVersion:	2.6.35
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 Description:
 		The rc/ class sub-directory belongs to the Remote Controller
 		core and provides a sysfs interface for configuring infrared
@@ -10,7 +10,7 @@ Description:
 What:		/sys/class/rc/rcN/
 Date:		Apr 2010
 KernelVersion:	2.6.35
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 Description:
 		A /sys/class/rc/rcN directory is created for each remote
 		control receiver device where N is the number of the receiver.
@@ -18,7 +18,7 @@ Description:
 What:		/sys/class/rc/rcN/protocols
 Date:		Jun 2010
 KernelVersion:	2.6.36
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 Description:
 		Reading this file returns a list of available protocols,
 		something like:
@@ -36,7 +36,7 @@ Description:
 What:		/sys/class/rc/rcN/filter
 Date:		Jan 2014
 KernelVersion:	3.15
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 Description:
 		Sets the scancode filter expected value.
 		Use in combination with /sys/class/rc/rcN/filter_mask to set the
@@ -49,7 +49,7 @@ Description:
 What:		/sys/class/rc/rcN/filter_mask
 Date:		Jan 2014
 KernelVersion:	3.15
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 Description:
 		Sets the scancode filter mask of bits to compare.
 		Use in combination with /sys/class/rc/rcN/filter to set the bits
@@ -64,7 +64,7 @@ Description:
 What:		/sys/class/rc/rcN/wakeup_protocols
 Date:		Feb 2017
 KernelVersion:	4.11
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 Description:
 		Reading this file returns a list of available protocols to use
 		for the wakeup filter, something like:
@@ -83,7 +83,7 @@ Description:
 What:		/sys/class/rc/rcN/wakeup_filter
 Date:		Jan 2014
 KernelVersion:	3.15
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 Description:
 		Sets the scancode wakeup filter expected value.
 		Use in combination with /sys/class/rc/rcN/wakeup_filter_mask to
@@ -98,7 +98,7 @@ Description:
 What:		/sys/class/rc/rcN/wakeup_filter_mask
 Date:		Jan 2014
 KernelVersion:	3.15
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 Description:
 		Sets the scancode wakeup filter mask of bits to compare.
 		Use in combination with /sys/class/rc/rcN/wakeup_filter to set
diff --git a/Documentation/ABI/testing/sysfs-class-rc-nuvoton b/Documentation/ABI/testing/sysfs-class-rc-nuvoton
index 905bcdeedef2..d3abe45f8690 100644
--- a/Documentation/ABI/testing/sysfs-class-rc-nuvoton
+++ b/Documentation/ABI/testing/sysfs-class-rc-nuvoton
@@ -1,7 +1,7 @@
 What:		/sys/class/rc/rcN/wakeup_data
 Date:		Mar 2016
 KernelVersion:	4.6
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 Description:
 		Reading this file returns the stored CIR wakeup sequence.
 		It starts with a pulse, followed by a space, pulse etc.
diff --git a/Documentation/ABI/testing/sysfs-devices-edac b/Documentation/ABI/testing/sysfs-devices-edac
index 46ff929fd52a..256a9e990c0b 100644
--- a/Documentation/ABI/testing/sysfs-devices-edac
+++ b/Documentation/ABI/testing/sysfs-devices-edac
@@ -77,7 +77,7 @@ Description:	Read/Write attribute file that controls memory scrubbing.
 
 What:		/sys/devices/system/edac/mc/mc*/max_location
 Date:		April 2012
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 		linux-edac@vger.kernel.org
 Description:	This attribute file displays the information about the last
 		available memory slot in this memory controller. It is used by
@@ -85,7 +85,7 @@ Description:	This attribute file displays the information about the last
 
 What:		/sys/devices/system/edac/mc/mc*/(dimm|rank)*/size
 Date:		April 2012
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 		linux-edac@vger.kernel.org
 Description:	This attribute file will display the size of dimm or rank.
 		For dimm*/size, this is the size, in MB of the DIMM memory
@@ -96,14 +96,14 @@ Description:	This attribute file will display the size of dimm or rank.
 
 What:		/sys/devices/system/edac/mc/mc*/(dimm|rank)*/dimm_dev_type
 Date:		April 2012
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 		linux-edac@vger.kernel.org
 Description:	This attribute file will display what type of DRAM device is
 		being utilized on this DIMM (x1, x2, x4, x8, ...).
 
 What:		/sys/devices/system/edac/mc/mc*/(dimm|rank)*/dimm_edac_mode
 Date:		April 2012
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 		linux-edac@vger.kernel.org
 Description:	This attribute file will display what type of Error detection
 		and correction is being utilized. For example: S4ECD4ED would
@@ -111,7 +111,7 @@ Description:	This attribute file will display what type of Error detection
 
 What:		/sys/devices/system/edac/mc/mc*/(dimm|rank)*/dimm_label
 Date:		April 2012
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 		linux-edac@vger.kernel.org
 Description:	This control file allows this DIMM to have a label assigned
 		to it. With this label in the module, when errors occur
@@ -126,14 +126,14 @@ Description:	This control file allows this DIMM to have a label assigned
 
 What:		/sys/devices/system/edac/mc/mc*/(dimm|rank)*/dimm_location
 Date:		April 2012
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 		linux-edac@vger.kernel.org
 Description:	This attribute file will display the location (csrow/channel,
 		branch/channel/slot or channel/slot) of the dimm or rank.
 
 What:		/sys/devices/system/edac/mc/mc*/(dimm|rank)*/dimm_mem_type
 Date:		April 2012
-Contact:	Mauro Carvalho Chehab <m.chehab@samsung.com>
+Contact:	Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
 		linux-edac@vger.kernel.org
 Description:	This attribute file will display what type of memory is
 		currently on this csrow. Normally, either buffered or
diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs
index 540553c933b6..372b88f4e706 100644
--- a/Documentation/ABI/testing/sysfs-fs-f2fs
+++ b/Documentation/ABI/testing/sysfs-fs-f2fs
@@ -101,6 +101,7 @@ Date:		February 2015
 Contact:	"Jaegeuk Kim" <jaegeuk@kernel.org>
 Description:
 		 Controls the trimming rate in batch mode.
+		 <deprecated>
 
 What:		/sys/fs/f2fs/<disk>/cp_interval
 Date:		October 2015
diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-hugepages b/Documentation/ABI/testing/sysfs-kernel-mm-hugepages
index e21c00571cf4..fdaa2162fae1 100644
--- a/Documentation/ABI/testing/sysfs-kernel-mm-hugepages
+++ b/Documentation/ABI/testing/sysfs-kernel-mm-hugepages
@@ -12,4 +12,4 @@ Description:
 			free_hugepages
 			surplus_hugepages
 			resv_hugepages
-		See Documentation/vm/hugetlbpage.txt for details.
+		See Documentation/admin-guide/mm/hugetlbpage.rst for details.
diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-ksm b/Documentation/ABI/testing/sysfs-kernel-mm-ksm
index 73e653ee2481..dfc13244cda3 100644
--- a/Documentation/ABI/testing/sysfs-kernel-mm-ksm
+++ b/Documentation/ABI/testing/sysfs-kernel-mm-ksm
@@ -40,7 +40,7 @@ Description:	Kernel Samepage Merging daemon sysfs interface
 		sleep_millisecs: how many milliseconds ksm should sleep between
 		scans.
 
-		See Documentation/vm/ksm.txt for more information.
+		See Documentation/vm/ksm.rst for more information.
 
 What:		/sys/kernel/mm/ksm/merge_across_nodes
 Date:		January 2013
diff --git a/Documentation/ABI/testing/sysfs-kernel-slab b/Documentation/ABI/testing/sysfs-kernel-slab
index 2cc0a72b64be..29601d93a1c2 100644
--- a/Documentation/ABI/testing/sysfs-kernel-slab
+++ b/Documentation/ABI/testing/sysfs-kernel-slab
@@ -37,7 +37,7 @@ Description:
 		The alloc_calls file is read-only and lists the kernel code
 		locations from which allocations for this cache were performed.
 		The alloc_calls file only contains information if debugging is
-		enabled for that cache (see Documentation/vm/slub.txt).
+		enabled for that cache (see Documentation/vm/slub.rst).
 
 What:		/sys/kernel/slab/cache/alloc_fastpath
 Date:		February 2008
@@ -219,7 +219,7 @@ Contact:	Pekka Enberg <penberg@cs.helsinki.fi>,
 Description:
 		The free_calls file is read-only and lists the locations of
 		object frees if slab debugging is enabled (see
-		Documentation/vm/slub.txt).
+		Documentation/vm/slub.rst).
 
 What:		/sys/kernel/slab/cache/free_fastpath
 Date:		February 2008
diff --git a/Documentation/ABI/testing/sysfs-platform-ideapad-laptop b/Documentation/ABI/testing/sysfs-platform-ideapad-laptop
index 597a2f3d1efc..1b31be3f996a 100644
--- a/Documentation/ABI/testing/sysfs-platform-ideapad-laptop
+++ b/Documentation/ABI/testing/sysfs-platform-ideapad-laptop
@@ -25,3 +25,16 @@ Description:
 		Control touchpad mode.
 			* 1 -> Switched On
 			* 0 -> Switched Off
+
+What:		/sys/bus/pci/devices/<bdf>/<device>/VPC2004:00/fn_lock
+Date:		May 2018
+KernelVersion:	4.18
+Contact:	"Oleg Keri <ezhi99@gmail.com>"
+Description:
+		Control fn-lock mode.
+			* 1 -> Switched On
+			* 0 -> Switched Off
+
+		For example:
+		# echo "0" >	\
+		/sys/bus/pci/devices/0000:00:1f.0/PNP0C09:00/VPC2004:00/fn_lock
diff --git a/Documentation/PCI/pci-error-recovery.txt b/Documentation/PCI/pci-error-recovery.txt
index 0b6bb3ef449e..688b69121e82 100644
--- a/Documentation/PCI/pci-error-recovery.txt
+++ b/Documentation/PCI/pci-error-recovery.txt
@@ -110,7 +110,7 @@ The actual steps taken by a platform to recover from a PCI error
 event will be platform-dependent, but will follow the general
 sequence described below.
 
-STEP 0: Error Event
+STEP 0: Error Event: ERR_NONFATAL
 -------------------
 A PCI bus error is detected by the PCI hardware.  On powerpc, the slot
 is isolated, in that all I/O is blocked: all reads return 0xffffffff,
@@ -228,13 +228,7 @@ proceeds to either STEP3 (Link Reset) or to STEP 5 (Resume Operations).
 If any driver returned PCI_ERS_RESULT_NEED_RESET, then the platform
 proceeds to STEP 4 (Slot Reset)
 
-STEP 3: Link Reset
-------------------
-The platform resets the link.  This is a PCI-Express specific step
-and is done whenever a fatal error has been detected that can be
-"solved" by resetting the link.
-
-STEP 4: Slot Reset
+STEP 3: Slot Reset
 ------------------
 
 In response to a return value of PCI_ERS_RESULT_NEED_RESET, the
@@ -320,7 +314,7 @@ Failure).
 >>> However, it probably should.
 
 
-STEP 5: Resume Operations
+STEP 4: Resume Operations
 -------------------------
 The platform will call the resume() callback on all affected device
 drivers if all drivers on the segment have returned
@@ -332,7 +326,7 @@ a result code.
 At this point, if a new error happens, the platform will restart
 a new error recovery sequence.
 
-STEP 6: Permanent Failure
+STEP 5: Permanent Failure
 -------------------------
 A "permanent failure" has occurred, and the platform cannot recover
 the device.  The platform will call error_detected() with a
@@ -355,6 +349,27 @@ errors. See the discussion in powerpc/eeh-pci-error-recovery.txt
 for additional detail on real-life experience of the causes of
 software errors.
 
+STEP 0: Error Event: ERR_FATAL
+-------------------
+PCI bus error is detected by the PCI hardware. On powerpc, the slot is
+isolated, in that all I/O is blocked: all reads return 0xffffffff, all
+writes are ignored.
+
+STEP 1: Remove devices
+--------------------
+Platform removes the devices depending on the error agent, it could be
+this port for all subordinates or upstream component (likely downstream
+port)
+
+STEP 2: Reset link
+--------------------
+The platform resets the link.  This is a PCI-Express specific step and is
+done whenever a fatal error has been detected that can be "solved" by
+resetting the link.
+
+STEP 3: Re-enumerate the devices
+--------------------
+Initiates the re-enumeration.
 
 Conclusion; General Remarks
 ---------------------------
diff --git a/Documentation/RCU/Design/Data-Structures/Data-Structures.html b/Documentation/RCU/Design/Data-Structures/Data-Structures.html
index 6c06e10bd04b..f5120a00f511 100644
--- a/Documentation/RCU/Design/Data-Structures/Data-Structures.html
+++ b/Documentation/RCU/Design/Data-Structures/Data-Structures.html
@@ -380,31 +380,26 @@ and therefore need no protection.
 as follows:
 
 <pre>
-  1   unsigned long gpnum;
-  2   unsigned long completed;
+  1   unsigned long gp_seq;
 </pre>
 
 <p>RCU grace periods are numbered, and
-the <tt>-&gt;gpnum</tt> field contains the number of the grace
-period that started most recently.
-The <tt>-&gt;completed</tt> field contains the number of the
-grace period that completed most recently.
-If the two fields are equal, the RCU grace period that most recently
-started has already completed, and therefore the corresponding
-flavor of RCU is idle.
-If <tt>-&gt;gpnum</tt> is one greater than <tt>-&gt;completed</tt>,
-then <tt>-&gt;gpnum</tt> gives the number of the current RCU
-grace period, which has not yet completed.
-Any other combination of values indicates that something is broken.
-These two fields are protected by the root <tt>rcu_node</tt>'s
+the <tt>-&gt;gp_seq</tt> field contains the current grace-period
+sequence number.
+The bottom two bits are the state of the current grace period,
+which can be zero for not yet started or one for in progress.
+In other words, if the bottom two bits of <tt>-&gt;gp_seq</tt> are
+zero, the corresponding flavor of RCU is idle.
+Any other value in the bottom two bits indicates that something is broken.
+This field is protected by the root <tt>rcu_node</tt> structure's
 <tt>-&gt;lock</tt> field.
 
-</p><p>There are <tt>-&gt;gpnum</tt> and <tt>-&gt;completed</tt> fields
+</p><p>There are <tt>-&gt;gp_seq</tt> fields
 in the <tt>rcu_node</tt> and <tt>rcu_data</tt> structures
 as well.
 The fields in the <tt>rcu_state</tt> structure represent the
-most current values, and those of the other structures are compared
-in order to detect the start of a new grace period in a distributed
+most current value, and those of the other structures are compared
+in order to detect the beginnings and ends of grace periods in a distributed
 fashion.
 The values flow from <tt>rcu_state</tt> to <tt>rcu_node</tt>
 (down the tree from the root to the leaves) to <tt>rcu_data</tt>.
@@ -512,27 +507,47 @@ than to be heisenbugged out of existence.
 as follows:
 
 <pre>
-  1   unsigned long gpnum;
-  2   unsigned long completed;
+  1   unsigned long gp_seq;
+  2   unsigned long gp_seq_needed;
 </pre>
 
-<p>These fields are the counterparts of the fields of the same name in
-the <tt>rcu_state</tt> structure.
-They each may lag up to one behind their <tt>rcu_state</tt>
-counterparts.
-If a given <tt>rcu_node</tt> structure's <tt>-&gt;gpnum</tt> and
-<tt>-&gt;complete</tt> fields are equal, then this <tt>rcu_node</tt>
+<p>The <tt>rcu_node</tt> structures' <tt>-&gt;gp_seq</tt> fields are
+the counterparts of the field of the same name in the <tt>rcu_state</tt>
+structure.
+They each may lag up to one step behind their <tt>rcu_state</tt>
+counterpart.
+If the bottom two bits of a given <tt>rcu_node</tt> structure's
+<tt>-&gt;gp_seq</tt> field is zero, then this <tt>rcu_node</tt>
 structure believes that RCU is idle.
-Otherwise, as with the <tt>rcu_state</tt> structure,
-the <tt>-&gt;gpnum</tt> field will be one greater than the
-<tt>-&gt;complete</tt> fields, with <tt>-&gt;gpnum</tt>
-indicating which grace period this <tt>rcu_node</tt> believes
-is still being waited for.
+</p><p>The <tt>&gt;gp_seq</tt> field of each <tt>rcu_node</tt>
+structure is updated at the beginning and the end
+of each grace period.
+
+<p>The <tt>-&gt;gp_seq_needed</tt> fields record the
+furthest-in-the-future grace period request seen by the corresponding
+<tt>rcu_node</tt> structure.  The request is considered fulfilled when
+the value of the <tt>-&gt;gp_seq</tt> field equals or exceeds that of
+the <tt>-&gt;gp_seq_needed</tt> field.
 
-</p><p>The <tt>&gt;gpnum</tt> field of each <tt>rcu_node</tt>
-structure is updated at the beginning
-of each grace period, and the <tt>-&gt;completed</tt> fields are
-updated at the end of each grace period.
+<table>
+<tr><th>&nbsp;</th></tr>
+<tr><th align="left">Quick Quiz:</th></tr>
+<tr><td>
+	Suppose that this <tt>rcu_node</tt> structure doesn't see
+	a request for a very long time.
+	Won't wrapping of the <tt>-&gt;gp_seq</tt> field cause
+	problems?
+</td></tr>
+<tr><th align="left">Answer:</th></tr>
+<tr><td bgcolor="#ffffff"><font color="ffffff">
+	No, because if the <tt>-&gt;gp_seq_needed</tt> field lags behind the
+	<tt>-&gt;gp_seq</tt> field, the <tt>-&gt;gp_seq_needed</tt> field
+	will be updated at the end of the grace period.
+	Modulo-arithmetic comparisons therefore will always get the
+	correct answer, even with wrapping.
+</font></td></tr>
+<tr><td>&nbsp;</td></tr>
+</table>
 
 <h5>Quiescent-State Tracking</h5>
 
@@ -626,9 +641,8 @@ normal and expedited grace periods, respectively.
 	</ol>
 
 	<p><font color="ffffff">So the locking is absolutely required in
-	order to coordinate
-	clearing of the bits with the grace-period numbers in
-	<tt>-&gt;gpnum</tt> and <tt>-&gt;completed</tt>.
+	order to coordinate clearing of the bits with updating of the
+	grace-period sequence number in <tt>-&gt;gp_seq</tt>.
 </font></td></tr>
 <tr><td>&nbsp;</td></tr>
 </table>
@@ -1038,15 +1052,15 @@ out any <tt>rcu_data</tt> structure for which this flag is not set.
 as follows:
 
 <pre>
-  1   unsigned long completed;
-  2   unsigned long gpnum;
+  1   unsigned long gp_seq;
+  2   unsigned long gp_seq_needed;
   3   bool cpu_no_qs;
   4   bool core_needs_qs;
   5   bool gpwrap;
   6   unsigned long rcu_qs_ctr_snap;
 </pre>
 
-<p>The <tt>completed</tt> and <tt>gpnum</tt>
+<p>The <tt>-&gt;gp_seq</tt> and <tt>-&gt;gp_seq_needed</tt>
 fields are the counterparts of the fields of the same name
 in the <tt>rcu_state</tt> and <tt>rcu_node</tt> structures.
 They may each lag up to one behind their <tt>rcu_node</tt>
@@ -1054,15 +1068,9 @@ counterparts, but in <tt>CONFIG_NO_HZ_IDLE</tt> and
 <tt>CONFIG_NO_HZ_FULL</tt> kernels can lag
 arbitrarily far behind for CPUs in dyntick-idle mode (but these counters
 will catch up upon exit from dyntick-idle mode).
-If a given <tt>rcu_data</tt> structure's <tt>-&gt;gpnum</tt> and
-<tt>-&gt;complete</tt> fields are equal, then this <tt>rcu_data</tt>
+If the lower two bits of a given <tt>rcu_data</tt> structure's
+<tt>-&gt;gp_seq</tt> are zero, then this <tt>rcu_data</tt>
 structure believes that RCU is idle.
-Otherwise, as with the <tt>rcu_state</tt> and <tt>rcu_node</tt>
-structure,
-the <tt>-&gt;gpnum</tt> field will be one greater than the
-<tt>-&gt;complete</tt> fields, with <tt>-&gt;gpnum</tt>
-indicating which grace period this <tt>rcu_data</tt> believes
-is still being waited for.
 
 <table>
 <tr><th>&nbsp;</th></tr>
@@ -1070,13 +1078,13 @@ is still being waited for.
 <tr><td>
 	All this replication of the grace period numbers can only cause
 	massive confusion.
-	Why not just keep a global pair of counters and be done with it???
+	Why not just keep a global sequence number and be done with it???
 </td></tr>
 <tr><th align="left">Answer:</th></tr>
 <tr><td bgcolor="#ffffff"><font color="ffffff">
-	Because if there was only a single global pair of grace-period
+	Because if there was only a single global sequence
 	numbers, there would need to be a single global lock to allow
-	safely accessing and updating them.
+	safely accessing and updating it.
 	And if we are not going to have a single global lock, we need
 	to carefully manage the numbers on a per-node basis.
 	Recall from the answer to a previous Quick Quiz that the consequences
@@ -1091,8 +1099,8 @@ CPU has not yet passed through a quiescent state,
 while the <tt>-&gt;core_needs_qs</tt> flag indicates that the
 RCU core needs a quiescent state from the corresponding CPU.
 The <tt>-&gt;gpwrap</tt> field indicates that the corresponding
-CPU has remained idle for so long that the <tt>completed</tt>
-and <tt>gpnum</tt> counters are in danger of overflow, which
+CPU has remained idle for so long that the
+<tt>gp_seq</tt> counter is in danger of overflow, which
 will cause the CPU to disregard the values of its counters on
 its next exit from idle.
 Finally, the <tt>rcu_qs_ctr_snap</tt> field is used to detect
@@ -1130,10 +1138,10 @@ The CPU advances the callbacks in its <tt>rcu_data</tt> structure
 whenever it notices that another RCU grace period has completed.
 The CPU detects the completion of an RCU grace period by noticing
 that the value of its <tt>rcu_data</tt> structure's
-<tt>-&gt;completed</tt> field differs from that of its leaf
+<tt>-&gt;gp_seq</tt> field differs from that of its leaf
 <tt>rcu_node</tt> structure.
 Recall that each <tt>rcu_node</tt> structure's
-<tt>-&gt;completed</tt> field is updated at the end of each
+<tt>-&gt;gp_seq</tt> field is updated at the beginnings and ends of each
 grace period.
 
 <p>
diff --git a/Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.html b/Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.html
index 8651b0b4fd79..a346ce0116eb 100644
--- a/Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.html
+++ b/Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.html
@@ -357,7 +357,7 @@ parts, starting in this section with the various phases of
 grace-period initialization.
 
 <p>The first ordering-related grace-period initialization action is to
-increment the <tt>rcu_state</tt> structure's <tt>-&gt;gpnum</tt>
+advance the <tt>rcu_state</tt> structure's <tt>-&gt;gp_seq</tt>
 grace-period-number counter, as shown below:
 
 </p><p><img src="TreeRCU-gp-init-1.svg" alt="TreeRCU-gp-init-1.svg" width="75%">
@@ -388,7 +388,7 @@ its last CPU and if the next <tt>rcu_node</tt> structure has no online CPUs).
 
 <p>The final <tt>rcu_gp_init()</tt> pass through the <tt>rcu_node</tt>
 tree traverses breadth-first, setting each <tt>rcu_node</tt> structure's
-<tt>-&gt;gpnum</tt> field to the newly incremented value from the
+<tt>-&gt;gp_seq</tt> field to the newly advanced value from the
 <tt>rcu_state</tt> structure, as shown in the following diagram.
 
 </p><p><img src="TreeRCU-gp-init-3.svg" alt="TreeRCU-gp-init-1.svg" width="75%">
@@ -398,9 +398,9 @@ tree traverses breadth-first, setting each <tt>rcu_node</tt> structure's
 to notice that a new grace period has started, as described in the next
 section.
 But because the grace-period kthread started the grace period at the
-root (with the increment of the <tt>rcu_state</tt> structure's
-<tt>-&gt;gpnum</tt> field) before setting each leaf <tt>rcu_node</tt>
-structure's <tt>-&gt;gpnum</tt> field, each CPU's observation of
+root (with the advancing of the <tt>rcu_state</tt> structure's
+<tt>-&gt;gp_seq</tt> field) before setting each leaf <tt>rcu_node</tt>
+structure's <tt>-&gt;gp_seq</tt> field, each CPU's observation of
 the start of the grace period will happen after the actual start
 of the grace period.
 
@@ -466,7 +466,7 @@ section that the grace period must wait on.
 <tr><td>
 	But a RCU read-side critical section might have started
 	after the beginning of the grace period
-	(the <tt>-&gt;gpnum++</tt> from earlier), so why should
+	(the advancing of <tt>-&gt;gp_seq</tt> from earlier), so why should
 	the grace period wait on such a critical section?
 </td></tr>
 <tr><th align="left">Answer:</th></tr>
@@ -609,10 +609,8 @@ states outstanding from other CPUs.
 <h4><a name="Grace-Period Cleanup">Grace-Period Cleanup</a></h4>
 
 <p>Grace-period cleanup first scans the <tt>rcu_node</tt> tree
-breadth-first setting all the <tt>-&gt;completed</tt> fields equal
-to the number of the newly completed grace period, then it sets
-the <tt>rcu_state</tt> structure's <tt>-&gt;completed</tt> field,
-again to the number of the newly completed grace period.
+breadth-first advancing all the <tt>-&gt;gp_seq</tt> fields, then it
+advances the <tt>rcu_state</tt> structure's <tt>-&gt;gp_seq</tt> field.
 The ordering effects are shown below:
 
 </p><p><img src="TreeRCU-gp-cleanup.svg" alt="TreeRCU-gp-cleanup.svg" width="75%">
@@ -634,7 +632,7 @@ grace-period cleanup is complete, the next grace period can begin.
 	CPU has reported its quiescent state, but it may be some
 	milliseconds before RCU becomes aware of this.
 	The latest reasonable candidate is once the <tt>rcu_state</tt>
-	structure's <tt>-&gt;completed</tt> field has been updated,
+	structure's <tt>-&gt;gp_seq</tt> field has been updated,
 	but it is quite possible that some CPUs have already completed
 	phase two of their updates by that time.
 	In short, if you are going to work with RCU, you need to
@@ -647,7 +645,7 @@ grace-period cleanup is complete, the next grace period can begin.
 <h4><a name="Callback Invocation">Callback Invocation</a></h4>
 
 <p>Once a given CPU's leaf <tt>rcu_node</tt> structure's
-<tt>-&gt;completed</tt> field has been updated, that CPU can begin
+<tt>-&gt;gp_seq</tt> field has been updated, that CPU can begin
 invoking its RCU callbacks that were waiting for this grace period
 to end.
 These callbacks are identified by <tt>rcu_advance_cbs()</tt>,
diff --git a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-cleanup.svg b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-cleanup.svg
index 754f426b297a..bf84fbab27ee 100644
--- a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-cleanup.svg
+++ b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-cleanup.svg
@@ -384,11 +384,11 @@
      inkscape:window-height="1144"
      id="namedview208"
      showgrid="true"
-     inkscape:zoom="0.70710678"
-     inkscape:cx="617.89017"
-     inkscape:cy="542.52419"
-     inkscape:window-x="86"
-     inkscape:window-y="28"
+     inkscape:zoom="0.78716603"
+     inkscape:cx="513.06403"
+     inkscape:cy="623.1214"
+     inkscape:window-x="102"
+     inkscape:window-y="38"
      inkscape:window-maximized="0"
      inkscape:current-layer="g3188-3"
      fit-margin-top="5"
@@ -417,13 +417,15 @@
      id="g3188">
     <text
        xml:space="preserve"
-       x="3199.1516"
+       x="3145.9592"
        y="13255.592"
        font-style="normal"
        font-weight="bold"
        font-size="192"
        id="text202"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;font-family:Courier"><tspan
+         style="font-size:172.87567139px"
+         id="tspan3143">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
     <g
        id="g3107"
        transform="translate(947.90548,11584.029)">
@@ -502,13 +504,15 @@
     </g>
     <text
        xml:space="preserve"
-       x="5324.5371"
-       y="15414.598"
+       x="5264.4731"
+       y="15428.84"
        font-style="normal"
        font-weight="bold"
        font-size="192"
-       id="text202-753"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
+       id="text202-36-7"
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+         style="font-size:172.87567139px"
+         id="tspan3166-5">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
   </g>
   <g
      style="fill:none;stroke-width:0.025in"
@@ -547,15 +551,6 @@
        sodipodi:linespacing="125%"><tspan
          style="font-size:159.57754517px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;font-family:Liberation Sans;-inkscape-font-specification:Liberation Sans"
          id="tspan3104-6-5-6-0">Leaf</tspan></text>
-    <text
-       xml:space="preserve"
-       x="7479.5796"
-       y="17699.943"
-       font-style="normal"
-       font-weight="bold"
-       font-size="192"
-       id="text202-9"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
     <path
        sodipodi:nodetypes="cc"
        inkscape:connector-curvature="0"
@@ -566,15 +561,6 @@
        style="fill:none;stroke-width:0.025in"
        transform="translate(-737.93887,7732.6672)"
        id="g3188-3">
-      <text
-         xml:space="preserve"
-         x="3225.7478"
-         y="13175.802"
-         font-style="normal"
-         font-weight="bold"
-         font-size="192"
-         id="text202-60"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;font-family:Courier">rsp-&gt;completed =</text>
       <g
          id="g3107-62"
          transform="translate(947.90548,11584.029)">
@@ -607,15 +593,6 @@
          sodipodi:linespacing="125%"><tspan
            style="font-size:159.57754517px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;font-family:Liberation Sans;-inkscape-font-specification:Liberation Sans"
            id="tspan3104-6-5-7">Root</tspan></text>
-      <text
-         xml:space="preserve"
-         x="3225.7478"
-         y="13390.038"
-         font-style="normal"
-         font-weight="bold"
-         font-size="192"
-         id="text202-60-3"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">       rnp-&gt;completed</text>
       <flowRoot
          xml:space="preserve"
          id="flowRoot3356"
@@ -627,7 +604,18 @@
              height="63.63961"
              x="332.34018"
              y="681.87292" /></flowRegion><flowPara
-           id="flowPara3362" /></flowRoot>    </g>
+           id="flowPara3362" /></flowRoot>      <text
+         xml:space="preserve"
+         x="3156.6121"
+         y="13317.754"
+         font-style="normal"
+         font-weight="bold"
+         font-size="192"
+         id="text202-36-6"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+           style="font-size:172.87567139px"
+           id="tspan3166-0">rcu_seq_end(&amp;rsp-&gt;gp_seq)</tspan></text>
+    </g>
     <g
        style="fill:none;stroke-width:0.025in"
        transform="translate(-858.40227,7769.0342)"
@@ -859,6 +847,17 @@
        id="path3414-8-3-6-6"
        inkscape:connector-curvature="0"
        sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       x="7418.769"
+       y="17646.104"
+       font-style="normal"
+       font-weight="bold"
+       font-size="192"
+       id="text202-36-70"
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+         style="font-size:172.87567139px"
+         id="tspan3166-93">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
   </g>
   <g
      transform="translate(-1642.5377,-11611.245)"
@@ -887,13 +886,15 @@
     </g>
     <text
        xml:space="preserve"
-       x="5327.3057"
+       x="5274.1133"
        y="15428.84"
        font-style="normal"
        font-weight="bold"
        font-size="192"
        id="text202-36"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+         style="font-size:172.87567139px"
+         id="tspan3166">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
   </g>
   <g
      transform="translate(-151.71746,-11647.612)"
@@ -972,13 +973,15 @@
          id="tspan3104-6-5-6-0-92">Leaf</tspan></text>
     <text
        xml:space="preserve"
-       x="7486.4907"
-       y="17670.119"
+       x="7408.5918"
+       y="17619.504"
        font-style="normal"
        font-weight="bold"
        font-size="192"
-       id="text202-6"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
+       id="text202-36-2"
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+         style="font-size:172.87567139px"
+         id="tspan3166-9">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
   </g>
   <g
      transform="translate(-6817.1997,-11647.612)"
@@ -1019,13 +1022,15 @@
          id="tspan3104-6-5-6-0-1">Leaf</tspan></text>
     <text
        xml:space="preserve"
-       x="7474.1382"
-       y="17688.926"
+       x="7416.8003"
+       y="17619.504"
        font-style="normal"
        font-weight="bold"
        font-size="192"
-       id="text202-5"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
+       id="text202-36-3"
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+         style="font-size:172.87567139px"
+         id="tspan3166-56">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
   </g>
   <path
      style="fill:none;stroke:#000000;stroke-width:13.29812908px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow2Lend)"
@@ -1059,15 +1064,6 @@
      id="path3414-8-3-6"
      inkscape:connector-curvature="0"
      sodipodi:nodetypes="cc" />
-  <text
-     xml:space="preserve"
-     x="7318.9653"
-     y="6031.6353"
-     font-style="normal"
-     font-weight="bold"
-     font-size="192"
-     id="text202-2"
-     style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
   <g
      style="fill:none;stroke-width:0.025in"
      id="g4504-3-9"
@@ -1123,4 +1119,15 @@
      id="path3134-9-0-3-5"
      d="m 6875.6003,15833.906 1595.7755,0"
      style="fill:none;stroke:#969696;stroke-width:53.19251633;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;marker-end:url(#Arrow1Send-36)" />
+  <text
+     xml:space="preserve"
+     x="7275.2612"
+     y="5971.8916"
+     font-style="normal"
+     font-weight="bold"
+     font-size="192"
+     id="text202-36-1"
+     style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+       style="font-size:172.87567139px"
+       id="tspan3166-2">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
 </svg>
diff --git a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-init-1.svg b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-init-1.svg
index 0161262904ec..8c207550818f 100644
--- a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-init-1.svg
+++ b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-init-1.svg
@@ -272,13 +272,13 @@
      inkscape:window-height="1144"
      id="namedview208"
      showgrid="true"
-     inkscape:zoom="0.70710678"
-     inkscape:cx="617.89019"
-     inkscape:cy="636.57143"
-     inkscape:window-x="697"
+     inkscape:zoom="2.6330492"
+     inkscape:cx="524.82797"
+     inkscape:cy="519.31194"
+     inkscape:window-x="79"
      inkscape:window-y="28"
      inkscape:window-maximized="0"
-     inkscape:current-layer="svg2"
+     inkscape:current-layer="g3188"
      fit-margin-top="5"
      fit-margin-right="5"
      fit-margin-left="5"
@@ -305,13 +305,15 @@
      id="g3188">
     <text
        xml:space="preserve"
-       x="3305.5364"
+       x="3119.363"
        y="13255.592"
        font-style="normal"
        font-weight="bold"
        font-size="192"
        id="text202"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;font-family:Courier">rsp-&gt;gpnum++</text>
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;font-family:Courier"><tspan
+         style="font-size:172.87567139px"
+         id="tspan3071">rcu_seq_start(rsp-&gt;gp_seq)</tspan></text>
     <g
        id="g3107"
        transform="translate(947.90548,11584.029)">
diff --git a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-init-3.svg b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-init-3.svg
index de6ecc51b00e..d24d7d555dbc 100644
--- a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-init-3.svg
+++ b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp-init-3.svg
@@ -19,7 +19,7 @@
    id="svg2"
    version="1.1"
    inkscape:version="0.48.4 r9939"
-   sodipodi:docname="TreeRCU-gp-init-2.svg">
+   sodipodi:docname="TreeRCU-gp-init-3.svg">
   <metadata
      id="metadata212">
     <rdf:RDF>
@@ -257,18 +257,22 @@
      inkscape:window-width="1087"
      inkscape:window-height="1144"
      id="namedview208"
-     showgrid="false"
-     inkscape:zoom="0.70710678"
+     showgrid="true"
+     inkscape:zoom="0.68224756"
      inkscape:cx="617.89019"
      inkscape:cy="625.84293"
-     inkscape:window-x="697"
+     inkscape:window-x="54"
      inkscape:window-y="28"
      inkscape:window-maximized="0"
-     inkscape:current-layer="svg2"
+     inkscape:current-layer="g3153"
      fit-margin-top="5"
      fit-margin-right="5"
      fit-margin-left="5"
-     fit-margin-bottom="5" />
+     fit-margin-bottom="5">
+    <inkscape:grid
+       type="xygrid"
+       id="grid3090" />
+  </sodipodi:namedview>
   <path
      sodipodi:nodetypes="cccccccccccccccccccccccc"
      inkscape:connector-curvature="0"
@@ -281,13 +285,13 @@
      id="g3188">
     <text
        xml:space="preserve"
-       x="3305.5364"
+       x="3145.9592"
        y="13255.592"
        font-style="normal"
        font-weight="bold"
        font-size="192"
        id="text202"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
     <g
        id="g3107"
        transform="translate(947.90548,11584.029)">
@@ -366,13 +370,13 @@
     </g>
     <text
        xml:space="preserve"
-       x="5392.3345"
-       y="15407.104"
+       x="5253.6904"
+       y="15407.032"
        font-style="normal"
        font-weight="bold"
        font-size="192"
        id="text202-6"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
   </g>
   <g
      style="fill:none;stroke-width:0.025in"
@@ -413,13 +417,13 @@
          id="tspan3104-6-5-6-0">Leaf</tspan></text>
     <text
        xml:space="preserve"
-       x="7536.4883"
-       y="17640.934"
+       x="7415.4365"
+       y="17670.572"
        font-style="normal"
        font-weight="bold"
        font-size="192"
        id="text202-9"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
   </g>
   <g
      transform="translate(-1642.5375,-11610.962)"
@@ -448,13 +452,13 @@
     </g>
     <text
        xml:space="preserve"
-       x="5378.4146"
-       y="15436.927"
+       x="5258.0688"
+       y="15412.313"
        font-style="normal"
        font-weight="bold"
        font-size="192"
        id="text202-3"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
   </g>
   <g
      transform="translate(-151.71726,-11647.329)"
@@ -533,13 +537,13 @@
          id="tspan3104-6-5-6-0-92">Leaf</tspan></text>
     <text
        xml:space="preserve"
-       x="7520.1294"
-       y="17673.639"
+       x="7405.2607"
+       y="17670.572"
        font-style="normal"
        font-weight="bold"
        font-size="192"
        id="text202-35"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
   </g>
   <g
      transform="translate(-6817.1998,-11647.329)"
@@ -580,13 +584,13 @@
          id="tspan3104-6-5-6-0-1">Leaf</tspan></text>
     <text
        xml:space="preserve"
-       x="7521.4663"
-       y="17666.062"
+       x="7413.4688"
+       y="17670.566"
        font-style="normal"
        font-weight="bold"
        font-size="192"
        id="text202-75"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
   </g>
   <path
      style="fill:none;stroke:#000000;stroke-width:13.29812908px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow2Lend)"
@@ -622,11 +626,11 @@
      sodipodi:nodetypes="cc" />
   <text
      xml:space="preserve"
-     x="7370.856"
-     y="5997.5972"
+     x="7271.9297"
+     y="6023.2412"
      font-style="normal"
      font-weight="bold"
      font-size="192"
      id="text202-62"
-     style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+     style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
 </svg>
diff --git a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg
index b13b7b01bb3a..acd73c7ad0f4 100644
--- a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg
+++ b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg
@@ -1070,13 +1070,13 @@
      inkscape:window-height="1144"
      id="namedview208"
      showgrid="true"
-     inkscape:zoom="0.6004608"
-     inkscape:cx="826.65969"
-     inkscape:cy="483.3047"
-     inkscape:window-x="66"
-     inkscape:window-y="28"
+     inkscape:zoom="0.81932583"
+     inkscape:cx="840.45848"
+     inkscape:cy="5052.4242"
+     inkscape:window-x="787"
+     inkscape:window-y="24"
      inkscape:window-maximized="0"
-     inkscape:current-layer="svg2"
+     inkscape:current-layer="g4"
      fit-margin-top="5"
      fit-margin-right="5"
      fit-margin-left="5"
@@ -1543,15 +1543,6 @@
        style="fill:none;stroke-width:0.025in"
        transform="translate(1749.0282,658.72243)"
        id="g3188">
-      <text
-         xml:space="preserve"
-         x="3305.5364"
-         y="13255.592"
-         font-style="normal"
-         font-weight="bold"
-         font-size="192"
-         id="text202-5"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;font-family:Courier">rsp-&gt;gpnum++</text>
       <g
          id="g3107-62"
          transform="translate(947.90548,11584.029)">
@@ -1584,6 +1575,17 @@
          sodipodi:linespacing="125%"><tspan
            style="font-size:159.57754517px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;font-family:Liberation Sans;-inkscape-font-specification:Liberation Sans"
            id="tspan3104-6-5-7">Root</tspan></text>
+      <text
+         xml:space="preserve"
+         x="3137.9988"
+         y="13271.316"
+         font-style="normal"
+         font-weight="bold"
+         font-size="192"
+         id="text202-626"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+           style="font-size:172.87567139px"
+           id="tspan3071">rcu_seq_start(rsp-&gt;gp_seq)</tspan></text>
     </g>
     <rect
        ry="0"
@@ -2318,15 +2320,6 @@
        style="fill:none;stroke-width:0.025in"
        transform="translate(1739.0986,17188.625)"
        id="g3188-6">
-      <text
-         xml:space="preserve"
-         x="3305.5364"
-         y="13255.592"
-         font-style="normal"
-         font-weight="bold"
-         font-size="192"
-         id="text202-1"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
       <g
          id="g3107-5"
          transform="translate(947.90548,11584.029)">
@@ -2359,6 +2352,15 @@
          sodipodi:linespacing="125%"><tspan
            style="font-size:159.57754517px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;font-family:Liberation Sans;-inkscape-font-specification:Liberation Sans"
            id="tspan3104-6-5-1">Root</tspan></text>
+      <text
+         xml:space="preserve"
+         x="3147.9268"
+         y="13240.524"
+         font-style="normal"
+         font-weight="bold"
+         font-size="192"
+         id="text202-1"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
     </g>
     <g
        style="fill:none;stroke-width:0.025in"
@@ -2387,13 +2389,13 @@
       </g>
       <text
          xml:space="preserve"
-         x="5392.3345"
-         y="15407.104"
+         x="5263.1094"
+         y="15411.646"
          font-style="normal"
          font-weight="bold"
          font-size="192"
-         id="text202-6-7"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+         id="text202-92"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
     </g>
     <g
        style="fill:none;stroke-width:0.025in"
@@ -2434,13 +2436,13 @@
            id="tspan3104-6-5-6-0-94">Leaf</tspan></text>
       <text
          xml:space="preserve"
-         x="7536.4883"
-         y="17640.934"
+         x="7417.4053"
+         y="17655.502"
          font-style="normal"
          font-weight="bold"
          font-size="192"
-         id="text202-9"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+         id="text202-759"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
     </g>
     <g
        transform="translate(-2353.8462,17224.992)"
@@ -2469,13 +2471,13 @@
       </g>
       <text
          xml:space="preserve"
-         x="5378.4146"
-         y="15436.927"
+         x="5246.1548"
+         y="15411.648"
          font-style="normal"
          font-weight="bold"
          font-size="192"
-         id="text202-3"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+         id="text202-87"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
     </g>
     <g
        transform="translate(-863.02613,17188.625)"
@@ -2554,13 +2556,13 @@
            id="tspan3104-6-5-6-0-92-6">Leaf</tspan></text>
       <text
          xml:space="preserve"
-         x="7520.1294"
-         y="17673.639"
+         x="7433.8257"
+         y="17682.098"
          font-style="normal"
          font-weight="bold"
          font-size="192"
-         id="text202-35"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+         id="text202-2"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
     </g>
     <g
        transform="translate(-7528.5085,17188.625)"
@@ -2601,13 +2603,13 @@
            id="tspan3104-6-5-6-0-1-8">Leaf</tspan></text>
       <text
          xml:space="preserve"
-         x="7521.4663"
-         y="17666.062"
+         x="7415.4404"
+         y="17682.098"
          font-style="normal"
          font-weight="bold"
          font-size="192"
-         id="text202-75-1"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
+         id="text202-0"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
     </g>
     <path
        style="fill:none;stroke:#000000;stroke-width:13.29812813px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow2Lend)"
@@ -2641,15 +2643,6 @@
        id="path3414-8-3-6-4"
        inkscape:connector-curvature="0"
        sodipodi:nodetypes="cc" />
-    <text
-       xml:space="preserve"
-       x="6659.5469"
-       y="34833.551"
-       font-style="normal"
-       font-weight="bold"
-       font-size="192"
-       id="text202-62"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gpnum = rsp-&gt;gpnum</text>
     <path
        sodipodi:nodetypes="ccc"
        inkscape:connector-curvature="0"
@@ -3844,7 +3837,7 @@
          font-weight="bold"
          font-size="192"
          id="text202-6-6-5"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">rdp-&gt;gpnum</text>
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">rdp-&gt;gp_seq</text>
       <text
          xml:space="preserve"
          x="5035.4155"
@@ -4284,15 +4277,6 @@
        style="fill:none;stroke-width:0.025in"
        transform="translate(1874.038,53203.538)"
        id="g3188-7">
-      <text
-         xml:space="preserve"
-         x="3199.1516"
-         y="13255.592"
-         font-style="normal"
-         font-weight="bold"
-         font-size="192"
-         id="text202-82"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
       <g
          id="g3107-53"
          transform="translate(947.90548,11584.029)">
@@ -4325,6 +4309,17 @@
          sodipodi:linespacing="125%"><tspan
            style="font-size:159.57754517px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;font-family:Liberation Sans;-inkscape-font-specification:Liberation Sans"
            id="tspan3104-6-5-19">Root</tspan></text>
+      <text
+         xml:space="preserve"
+         x="3175.896"
+         y="13240.11"
+         font-style="normal"
+         font-weight="bold"
+         font-size="192"
+         id="text202-36-3"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+           style="font-size:172.87567139px"
+           id="tspan3166">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
     </g>
     <rect
        ry="0"
@@ -4371,13 +4366,15 @@
       </g>
       <text
          xml:space="preserve"
-         x="5324.5371"
-         y="15414.598"
+         x="5264.4829"
+         y="15411.231"
          font-style="normal"
          font-weight="bold"
          font-size="192"
-         id="text202-753"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
+         id="text202-36-7"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+           style="font-size:172.87567139px"
+           id="tspan3166-5">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
     </g>
     <g
        style="fill:none;stroke-width:0.025in"
@@ -4412,30 +4409,12 @@
        sodipodi:linespacing="125%"><tspan
          style="font-size:159.57754517px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;font-family:Liberation Sans;-inkscape-font-specification:Liberation Sans"
          id="tspan3104-6-5-6-0-4">Leaf</tspan></text>
-    <text
-       xml:space="preserve"
-       x="10084.225"
-       y="70903.312"
-       font-style="normal"
-       font-weight="bold"
-       font-size="192"
-       id="text202-9-0"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
     <path
        sodipodi:nodetypes="ccc"
        inkscape:connector-curvature="0"
        id="path3134-9-0-3-9"
        d="m 6315.6122,72629.054 -20.9533,8108.684 1648.968,0"
        style="fill:none;stroke:#969696;stroke-width:53.19251251;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;marker-end:url(#Arrow1Send)" />
-    <text
-       xml:space="preserve"
-       x="5092.4683"
-       y="74111.672"
-       font-style="normal"
-       font-weight="bold"
-       font-size="192"
-       id="text202-60"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">rsp-&gt;completed =</text>
     <g
        style="fill:none;stroke-width:0.025in"
        id="g3107-62-6"
@@ -4469,15 +4448,6 @@
        sodipodi:linespacing="125%"><tspan
          style="font-size:159.57754517px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;font-family:Liberation Sans;-inkscape-font-specification:Liberation Sans"
          id="tspan3104-6-5-7-7">Root</tspan></text>
-    <text
-       xml:space="preserve"
-       x="5092.4683"
-       y="74325.906"
-       font-style="normal"
-       font-weight="bold"
-       font-size="192"
-       id="text202-60-3"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">       rnp-&gt;completed</text>
     <g
        style="fill:none;stroke-width:0.025in"
        transform="translate(1746.2528,60972.572)"
@@ -4736,13 +4706,15 @@
       </g>
       <text
          xml:space="preserve"
-         x="5327.3057"
-         y="15428.84"
+         x="5274.1216"
+         y="15411.231"
          font-style="normal"
          font-weight="bold"
          font-size="192"
          id="text202-36"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+           style="font-size:172.87567139px"
+           id="tspan3166-6">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
     </g>
     <g
        transform="translate(-728.08545,53203.538)"
@@ -4821,13 +4793,15 @@
            id="tspan3104-6-5-6-0-92-5">Leaf</tspan></text>
       <text
          xml:space="preserve"
-         x="7486.4907"
-         y="17670.119"
+         x="7435.1987"
+         y="17708.281"
          font-style="normal"
          font-weight="bold"
          font-size="192"
-         id="text202-6-2"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
+         id="text202-36-9"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+           style="font-size:172.87567139px"
+           id="tspan3166-1">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
     </g>
     <g
        transform="translate(-7393.5687,53203.538)"
@@ -4868,13 +4842,15 @@
            id="tspan3104-6-5-6-0-1-5">Leaf</tspan></text>
       <text
          xml:space="preserve"
-         x="7474.1382"
-         y="17688.926"
+         x="7416.8125"
+         y="17708.281"
          font-style="normal"
          font-weight="bold"
          font-size="192"
-         id="text202-5-1"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
+         id="text202-36-35"
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+           style="font-size:172.87567139px"
+           id="tspan3166-62">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
     </g>
     <path
        style="fill:none;stroke:#000000;stroke-width:13.29812813px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow2Lend)"
@@ -4908,15 +4884,6 @@
        id="path3414-8-3-6-67"
        inkscape:connector-curvature="0"
        sodipodi:nodetypes="cc" />
-    <text
-       xml:space="preserve"
-       x="6742.6001"
-       y="70882.617"
-       font-style="normal"
-       font-weight="bold"
-       font-size="192"
-       id="text202-2"
-       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;completed = -&gt;gpnum</text>
     <g
        style="fill:none;stroke-width:0.025in"
        id="g4504-3-9-6"
@@ -5131,5 +5098,47 @@
        font-size="192"
        id="text202-7-9-6-6-7"
        style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">rcu_do_batch()</text>
+    <text
+       xml:space="preserve"
+       x="6698.9019"
+       y="70885.211"
+       font-style="normal"
+       font-weight="bold"
+       font-size="192"
+       id="text202-36-2"
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+         style="font-size:172.87567139px"
+         id="tspan3166-7">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
+    <text
+       xml:space="preserve"
+       x="10023.457"
+       y="70885.234"
+       font-style="normal"
+       font-weight="bold"
+       font-size="192"
+       id="text202-36-0"
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+         style="font-size:172.87567139px"
+         id="tspan3166-9">rcu_seq_end(&amp;rnp-&gt;gp_seq)</tspan></text>
+    <text
+       xml:space="preserve"
+       x="5023.3389"
+       y="74209.773"
+       font-style="normal"
+       font-weight="bold"
+       font-size="192"
+       id="text202-36-36"
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier"><tspan
+         style="font-size:172.87567139px"
+         id="tspan3166-0">rcu_seq_end(&amp;rsp-&gt;gp_seq)</tspan></text>
+    <text
+       xml:space="preserve"
+       x="6562.5884"
+       y="34870.727"
+       font-style="normal"
+       font-weight="bold"
+       font-size="192"
+       id="text202-3"
+       style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">-&gt;gp_seq = rsp-&gt;gp_seq</text>
   </g>
 </svg>
diff --git a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-qs.svg b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-qs.svg
index de3992f4cbe1..149bec2a4493 100644
--- a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-qs.svg
+++ b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-qs.svg
@@ -300,13 +300,13 @@
      inkscape:window-height="1144"
      id="namedview208"
      showgrid="true"
-     inkscape:zoom="0.70710678"
-     inkscape:cx="616.47598"
-     inkscape:cy="595.41964"
-     inkscape:window-x="813"
+     inkscape:zoom="0.96484375"
+     inkscape:cx="507.0191"
+     inkscape:cy="885.62207"
+     inkscape:window-x="47"
      inkscape:window-y="28"
      inkscape:window-maximized="0"
-     inkscape:current-layer="g4405"
+     inkscape:current-layer="g3115"
      fit-margin-top="5"
      fit-margin-right="5"
      fit-margin-left="5"
@@ -710,7 +710,7 @@
          font-weight="bold"
          font-size="192"
          id="text202-6-6"
-         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">rdp-&gt;gpnum</text>
+         style="font-size:192px;font-style:normal;font-weight:bold;text-anchor:start;fill:#000000;stroke-width:0.025in;font-family:Courier">rdp-&gt;gp_seq</text>
       <text
          xml:space="preserve"
          x="5035.4155"
diff --git a/Documentation/RCU/stallwarn.txt b/Documentation/RCU/stallwarn.txt
index 4259f95c3261..f99cf11b314b 100644
--- a/Documentation/RCU/stallwarn.txt
+++ b/Documentation/RCU/stallwarn.txt
@@ -172,7 +172,7 @@ it will print a message similar to the following:
 	INFO: rcu_sched detected stalls on CPUs/tasks:
 	2-...: (3 GPs behind) idle=06c/0/0 softirq=1453/1455 fqs=0
 	16-...: (0 ticks this GP) idle=81c/0/0 softirq=764/764 fqs=0
-	(detected by 32, t=2603 jiffies, g=7073, c=7072, q=625)
+	(detected by 32, t=2603 jiffies, g=7075, q=625)
 
 This message indicates that CPU 32 detected that CPUs 2 and 16 were both
 causing stalls, and that the stall was affecting RCU-sched.  This message
@@ -215,11 +215,10 @@ CPU since the last time that this CPU noted the beginning of a grace
 period.
 
 The "detected by" line indicates which CPU detected the stall (in this
-case, CPU 32), how many jiffies have elapsed since the start of the
-grace period (in this case 2603), the number of the last grace period
-to start and to complete (7073 and 7072, respectively), and an estimate
-of the total number of RCU callbacks queued across all CPUs (625 in
-this case).
+case, CPU 32), how many jiffies have elapsed since the start of the grace
+period (in this case 2603), the grace-period sequence number (7075), and
+an estimate of the total number of RCU callbacks queued across all CPUs
+(625 in this case).
 
 In kernels with CONFIG_RCU_FAST_NO_HZ, more information is printed
 for each CPU:
@@ -266,15 +265,16 @@ If the relevant grace-period kthread has been unable to run prior to
 the stall warning, as was the case in the "All QSes seen" line above,
 the following additional line is printed:
 
-	kthread starved for 23807 jiffies! g7073 c7072 f0x0 RCU_GP_WAIT_FQS(3) ->state=0x1
+	kthread starved for 23807 jiffies! g7075 f0x0 RCU_GP_WAIT_FQS(3) ->state=0x1 ->cpu=5
 
 Starving the grace-period kthreads of CPU time can of course result
 in RCU CPU stall warnings even when all CPUs and tasks have passed
-through the required quiescent states.  The "g" and "c" numbers flag the
-number of the last grace period started and completed, respectively,
-the "f" precedes the ->gp_flags command to the grace-period kthread,
-the "RCU_GP_WAIT_FQS" indicates that the kthread is waiting for a short
-timeout, and the "state" precedes value of the task_struct ->state field.
+through the required quiescent states.  The "g" number shows the current
+grace-period sequence number, the "f" precedes the ->gp_flags command
+to the grace-period kthread, the "RCU_GP_WAIT_FQS" indicates that the
+kthread is waiting for a short timeout, the "state" precedes value of the
+task_struct ->state field, and the "cpu" indicates that the grace-period
+kthread last ran on CPU 5.
 
 
 Multiple Warnings From One Stall
diff --git a/Documentation/RCU/whatisRCU.txt b/Documentation/RCU/whatisRCU.txt
index a27fbfb0efb8..65eb856526b7 100644
--- a/Documentation/RCU/whatisRCU.txt
+++ b/Documentation/RCU/whatisRCU.txt
@@ -1,3 +1,5 @@
+What is RCU?  --  "Read, Copy, Update"
+
 Please note that the "What is RCU?" LWN series is an excellent place
 to start learning about RCU:
 
diff --git a/Documentation/acpi/cppc_sysfs.txt b/Documentation/acpi/cppc_sysfs.txt
new file mode 100644
index 000000000000..f20fb445135d
--- /dev/null
+++ b/Documentation/acpi/cppc_sysfs.txt
@@ -0,0 +1,69 @@
+
+	Collaborative Processor Performance Control (CPPC)
+
+CPPC defined in the ACPI spec describes a mechanism for the OS to manage the
+performance of a logical processor on a contigious and abstract performance
+scale. CPPC exposes a set of registers to describe abstract performance scale,
+to request performance levels and to measure per-cpu delivered performance.
+
+For more details on CPPC please refer to the ACPI specification at:
+
+http://uefi.org/specifications
+
+Some of the CPPC registers are exposed via sysfs under:
+
+/sys/devices/system/cpu/cpuX/acpi_cppc/
+
+for each cpu X
+
+--------------------------------------------------------------------------------
+
+$ ls -lR  /sys/devices/system/cpu/cpu0/acpi_cppc/
+/sys/devices/system/cpu/cpu0/acpi_cppc/:
+total 0
+-r--r--r-- 1 root root 65536 Mar  5 19:38 feedback_ctrs
+-r--r--r-- 1 root root 65536 Mar  5 19:38 highest_perf
+-r--r--r-- 1 root root 65536 Mar  5 19:38 lowest_freq
+-r--r--r-- 1 root root 65536 Mar  5 19:38 lowest_nonlinear_perf
+-r--r--r-- 1 root root 65536 Mar  5 19:38 lowest_perf
+-r--r--r-- 1 root root 65536 Mar  5 19:38 nominal_freq
+-r--r--r-- 1 root root 65536 Mar  5 19:38 nominal_perf
+-r--r--r-- 1 root root 65536 Mar  5 19:38 reference_perf
+-r--r--r-- 1 root root 65536 Mar  5 19:38 wraparound_time
+
+--------------------------------------------------------------------------------
+
+* highest_perf : Highest performance of this processor (abstract scale).
+* nominal_perf : Highest sustained performance of this processor (abstract scale).
+* lowest_nonlinear_perf : Lowest performance of this processor with nonlinear
+  power savings (abstract scale).
+* lowest_perf : Lowest performance of this processor (abstract scale).
+
+* lowest_freq : CPU frequency corresponding to lowest_perf (in MHz).
+* nominal_freq : CPU frequency corresponding to nominal_perf (in MHz).
+  The above frequencies should only be used to report processor performance in
+  freqency instead of abstract scale. These values should not be used for any
+  functional decisions.
+
+* feedback_ctrs : Includes both Reference and delivered performance counter.
+  Reference counter ticks up proportional to processor's reference performance.
+  Delivered counter ticks up proportional to processor's delivered performance.
+* wraparound_time: Minimum time for the feedback counters to wraparound (seconds).
+* reference_perf : Performance level at which reference performance counter
+  accumulates (abstract scale).
+
+--------------------------------------------------------------------------------
+
+		Computing Average Delivered Performance
+
+Below describes the steps to compute the average performance delivered by taking
+two different snapshots of feedback counters at time T1 and T2.
+
+T1: Read feedback_ctrs as fbc_t1
+    Wait or run some workload
+T2: Read feedback_ctrs as fbc_t2
+
+delivered_counter_delta = fbc_t2[del] - fbc_t1[del]
+reference_counter_delta = fbc_t2[ref] - fbc_t1[ref]
+
+delivered_perf = (refernce_perf x delivered_counter_delta) / reference_counter_delta
diff --git a/Documentation/admin-guide/LSM/apparmor.rst b/Documentation/admin-guide/LSM/apparmor.rst
index 3e9734bd0e05..6cf81bbd7ce8 100644
--- a/Documentation/admin-guide/LSM/apparmor.rst
+++ b/Documentation/admin-guide/LSM/apparmor.rst
@@ -44,8 +44,8 @@ Links
 
 Mailing List - apparmor@lists.ubuntu.com
 
-Wiki - http://apparmor.wiki.kernel.org/
+Wiki - http://wiki.apparmor.net
 
-User space tools - https://launchpad.net/apparmor
+User space tools - https://gitlab.com/apparmor
 
-Kernel module - git://git.kernel.org/pub/scm/linux/kernel/git/jj/apparmor-dev.git
+Kernel module - git://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor
diff --git a/Documentation/admin-guide/bcache.rst b/Documentation/admin-guide/bcache.rst
new file mode 100644
index 000000000000..c0ce64d75bbf
--- /dev/null
+++ b/Documentation/admin-guide/bcache.rst
@@ -0,0 +1,649 @@
+============================
+A block layer cache (bcache)
+============================
+
+Say you've got a big slow raid 6, and an ssd or three. Wouldn't it be
+nice if you could use them as cache... Hence bcache.
+
+Wiki and git repositories are at:
+
+  - http://bcache.evilpiepirate.org
+  - http://evilpiepirate.org/git/linux-bcache.git
+  - http://evilpiepirate.org/git/bcache-tools.git
+
+It's designed around the performance characteristics of SSDs - it only allocates
+in erase block sized buckets, and it uses a hybrid btree/log to track cached
+extents (which can be anywhere from a single sector to the bucket size). It's
+designed to avoid random writes at all costs; it fills up an erase block
+sequentially, then issues a discard before reusing it.
+
+Both writethrough and writeback caching are supported. Writeback defaults to
+off, but can be switched on and off arbitrarily at runtime. Bcache goes to
+great lengths to protect your data - it reliably handles unclean shutdown. (It
+doesn't even have a notion of a clean shutdown; bcache simply doesn't return
+writes as completed until they're on stable storage).
+
+Writeback caching can use most of the cache for buffering writes - writing
+dirty data to the backing device is always done sequentially, scanning from the
+start to the end of the index.
+
+Since random IO is what SSDs excel at, there generally won't be much benefit
+to caching large sequential IO. Bcache detects sequential IO and skips it;
+it also keeps a rolling average of the IO sizes per task, and as long as the
+average is above the cutoff it will skip all IO from that task - instead of
+caching the first 512k after every seek. Backups and large file copies should
+thus entirely bypass the cache.
+
+In the event of a data IO error on the flash it will try to recover by reading
+from disk or invalidating cache entries.  For unrecoverable errors (meta data
+or dirty data), caching is automatically disabled; if dirty data was present
+in the cache it first disables writeback caching and waits for all dirty data
+to be flushed.
+
+Getting started:
+You'll need make-bcache from the bcache-tools repository. Both the cache device
+and backing device must be formatted before use::
+
+  make-bcache -B /dev/sdb
+  make-bcache -C /dev/sdc
+
+make-bcache has the ability to format multiple devices at the same time - if
+you format your backing devices and cache device at the same time, you won't
+have to manually attach::
+
+  make-bcache -B /dev/sda /dev/sdb -C /dev/sdc
+
+bcache-tools now ships udev rules, and bcache devices are known to the kernel
+immediately.  Without udev, you can manually register devices like this::
+
+  echo /dev/sdb > /sys/fs/bcache/register
+  echo /dev/sdc > /sys/fs/bcache/register
+
+Registering the backing device makes the bcache device show up in /dev; you can
+now format it and use it as normal. But the first time using a new bcache
+device, it'll be running in passthrough mode until you attach it to a cache.
+If you are thinking about using bcache later, it is recommended to setup all your
+slow devices as bcache backing devices without a cache, and you can choose to add
+a caching device later.
+See 'ATTACHING' section below.
+
+The devices show up as::
+
+  /dev/bcache<N>
+
+As well as (with udev)::
+
+  /dev/bcache/by-uuid/<uuid>
+  /dev/bcache/by-label/<label>
+
+To get started::
+
+  mkfs.ext4 /dev/bcache0
+  mount /dev/bcache0 /mnt
+
+You can control bcache devices through sysfs at /sys/block/bcache<N>/bcache .
+You can also control them through /sys/fs//bcache/<cset-uuid>/ .
+
+Cache devices are managed as sets; multiple caches per set isn't supported yet
+but will allow for mirroring of metadata and dirty data in the future. Your new
+cache set shows up as /sys/fs/bcache/<UUID>
+
+Attaching
+---------
+
+After your cache device and backing device are registered, the backing device
+must be attached to your cache set to enable caching. Attaching a backing
+device to a cache set is done thusly, with the UUID of the cache set in
+/sys/fs/bcache::
+
+  echo <CSET-UUID> > /sys/block/bcache0/bcache/attach
+
+This only has to be done once. The next time you reboot, just reregister all
+your bcache devices. If a backing device has data in a cache somewhere, the
+/dev/bcache<N> device won't be created until the cache shows up - particularly
+important if you have writeback caching turned on.
+
+If you're booting up and your cache device is gone and never coming back, you
+can force run the backing device::
+
+  echo 1 > /sys/block/sdb/bcache/running
+
+(You need to use /sys/block/sdb (or whatever your backing device is called), not
+/sys/block/bcache0, because bcache0 doesn't exist yet. If you're using a
+partition, the bcache directory would be at /sys/block/sdb/sdb2/bcache)
+
+The backing device will still use that cache set if it shows up in the future,
+but all the cached data will be invalidated. If there was dirty data in the
+cache, don't expect the filesystem to be recoverable - you will have massive
+filesystem corruption, though ext4's fsck does work miracles.
+
+Error Handling
+--------------
+
+Bcache tries to transparently handle IO errors to/from the cache device without
+affecting normal operation; if it sees too many errors (the threshold is
+configurable, and defaults to 0) it shuts down the cache device and switches all
+the backing devices to passthrough mode.
+
+ - For reads from the cache, if they error we just retry the read from the
+   backing device.
+
+ - For writethrough writes, if the write to the cache errors we just switch to
+   invalidating the data at that lba in the cache (i.e. the same thing we do for
+   a write that bypasses the cache)
+
+ - For writeback writes, we currently pass that error back up to the
+   filesystem/userspace. This could be improved - we could retry it as a write
+   that skips the cache so we don't have to error the write.
+
+ - When we detach, we first try to flush any dirty data (if we were running in
+   writeback mode). It currently doesn't do anything intelligent if it fails to
+   read some of the dirty data, though.
+
+
+Howto/cookbook
+--------------
+
+A) Starting a bcache with a missing caching device
+
+If registering the backing device doesn't help, it's already there, you just need
+to force it to run without the cache::
+
+	host:~# echo /dev/sdb1 > /sys/fs/bcache/register
+	[  119.844831] bcache: register_bcache() error opening /dev/sdb1: device already registered
+
+Next, you try to register your caching device if it's present. However
+if it's absent, or registration fails for some reason, you can still
+start your bcache without its cache, like so::
+
+	host:/sys/block/sdb/sdb1/bcache# echo 1 > running
+
+Note that this may cause data loss if you were running in writeback mode.
+
+
+B) Bcache does not find its cache::
+
+	host:/sys/block/md5/bcache# echo 0226553a-37cf-41d5-b3ce-8b1e944543a8 > attach
+	[ 1933.455082] bcache: bch_cached_dev_attach() Couldn't find uuid for md5 in set
+	[ 1933.478179] bcache: __cached_dev_store() Can't attach 0226553a-37cf-41d5-b3ce-8b1e944543a8
+	[ 1933.478179] : cache set not found
+
+In this case, the caching device was simply not registered at boot
+or disappeared and came back, and needs to be (re-)registered::
+
+	host:/sys/block/md5/bcache# echo /dev/sdh2 > /sys/fs/bcache/register
+
+
+C) Corrupt bcache crashes the kernel at device registration time:
+
+This should never happen.  If it does happen, then you have found a bug!
+Please report it to the bcache development list: linux-bcache@vger.kernel.org
+
+Be sure to provide as much information that you can including kernel dmesg
+output if available so that we may assist.
+
+
+D) Recovering data without bcache:
+
+If bcache is not available in the kernel, a filesystem on the backing
+device is still available at an 8KiB offset. So either via a loopdev
+of the backing device created with --offset 8K, or any value defined by
+--data-offset when you originally formatted bcache with `make-bcache`.
+
+For example::
+
+	losetup -o 8192 /dev/loop0 /dev/your_bcache_backing_dev
+
+This should present your unmodified backing device data in /dev/loop0
+
+If your cache is in writethrough mode, then you can safely discard the
+cache device without loosing data.
+
+
+E) Wiping a cache device
+
+::
+
+	host:~# wipefs -a /dev/sdh2
+	16 bytes were erased at offset 0x1018 (bcache)
+	they were: c6 85 73 f6 4e 1a 45 ca 82 65 f5 7f 48 ba 6d 81
+
+After you boot back with bcache enabled, you recreate the cache and attach it::
+
+	host:~# make-bcache -C /dev/sdh2
+	UUID:                   7be7e175-8f4c-4f99-94b2-9c904d227045
+	Set UUID:               5bc072a8-ab17-446d-9744-e247949913c1
+	version:                0
+	nbuckets:               106874
+	block_size:             1
+	bucket_size:            1024
+	nr_in_set:              1
+	nr_this_dev:            0
+	first_bucket:           1
+	[  650.511912] bcache: run_cache_set() invalidating existing data
+	[  650.549228] bcache: register_cache() registered cache device sdh2
+
+start backing device with missing cache::
+
+	host:/sys/block/md5/bcache# echo 1 > running
+
+attach new cache::
+
+	host:/sys/block/md5/bcache# echo 5bc072a8-ab17-446d-9744-e247949913c1 > attach
+	[  865.276616] bcache: bch_cached_dev_attach() Caching md5 as bcache0 on set 5bc072a8-ab17-446d-9744-e247949913c1
+
+
+F) Remove or replace a caching device::
+
+	host:/sys/block/sda/sda7/bcache# echo 1 > detach
+	[  695.872542] bcache: cached_dev_detach_finish() Caching disabled for sda7
+
+	host:~# wipefs -a /dev/nvme0n1p4
+	wipefs: error: /dev/nvme0n1p4: probing initialization failed: Device or resource busy
+	Ooops, it's disabled, but not unregistered, so it's still protected
+
+We need to go and unregister it::
+
+	host:/sys/fs/bcache/b7ba27a1-2398-4649-8ae3-0959f57ba128# ls -l cache0
+	lrwxrwxrwx 1 root root 0 Feb 25 18:33 cache0 -> ../../../devices/pci0000:00/0000:00:1d.0/0000:70:00.0/nvme/nvme0/nvme0n1/nvme0n1p4/bcache/
+	host:/sys/fs/bcache/b7ba27a1-2398-4649-8ae3-0959f57ba128# echo 1 > stop
+	kernel: [  917.041908] bcache: cache_set_free() Cache set b7ba27a1-2398-4649-8ae3-0959f57ba128 unregistered
+
+Now we can wipe it::
+
+	host:~# wipefs -a /dev/nvme0n1p4
+	/dev/nvme0n1p4: 16 bytes were erased at offset 0x00001018 (bcache): c6 85 73 f6 4e 1a 45 ca 82 65 f5 7f 48 ba 6d 81
+
+
+G) dm-crypt and bcache
+
+First setup bcache unencrypted and then install dmcrypt on top of
+/dev/bcache<N> This will work faster than if you dmcrypt both the backing
+and caching devices and then install bcache on top. [benchmarks?]
+
+
+H) Stop/free a registered bcache to wipe and/or recreate it
+
+Suppose that you need to free up all bcache references so that you can
+fdisk run and re-register a changed partition table, which won't work
+if there are any active backing or caching devices left on it:
+
+1) Is it present in /dev/bcache* ? (there are times where it won't be)
+
+   If so, it's easy::
+
+	host:/sys/block/bcache0/bcache# echo 1 > stop
+
+2) But if your backing device is gone, this won't work::
+
+	host:/sys/block/bcache0# cd bcache
+	bash: cd: bcache: No such file or directory
+
+   In this case, you may have to unregister the dmcrypt block device that
+   references this bcache to free it up::
+
+	host:~# dmsetup remove oldds1
+	bcache: bcache_device_free() bcache0 stopped
+	bcache: cache_set_free() Cache set 5bc072a8-ab17-446d-9744-e247949913c1 unregistered
+
+   This causes the backing bcache to be removed from /sys/fs/bcache and
+   then it can be reused.  This would be true of any block device stacking
+   where bcache is a lower device.
+
+3) In other cases, you can also look in /sys/fs/bcache/::
+
+	host:/sys/fs/bcache# ls -l */{cache?,bdev?}
+	lrwxrwxrwx 1 root root 0 Mar  5 09:39 0226553a-37cf-41d5-b3ce-8b1e944543a8/bdev1 -> ../../../devices/virtual/block/dm-1/bcache/
+	lrwxrwxrwx 1 root root 0 Mar  5 09:39 0226553a-37cf-41d5-b3ce-8b1e944543a8/cache0 -> ../../../devices/virtual/block/dm-4/bcache/
+	lrwxrwxrwx 1 root root 0 Mar  5 09:39 5bc072a8-ab17-446d-9744-e247949913c1/cache0 -> ../../../devices/pci0000:00/0000:00:01.0/0000:01:00.0/ata10/host9/target9:0:0/9:0:0:0/block/sdl/sdl2/bcache/
+
+   The device names will show which UUID is relevant, cd in that directory
+   and stop the cache::
+
+	host:/sys/fs/bcache/5bc072a8-ab17-446d-9744-e247949913c1# echo 1 > stop
+
+   This will free up bcache references and let you reuse the partition for
+   other purposes.
+
+
+
+Troubleshooting performance
+---------------------------
+
+Bcache has a bunch of config options and tunables. The defaults are intended to
+be reasonable for typical desktop and server workloads, but they're not what you
+want for getting the best possible numbers when benchmarking.
+
+ - Backing device alignment
+
+   The default metadata size in bcache is 8k.  If your backing device is
+   RAID based, then be sure to align this by a multiple of your stride
+   width using `make-bcache --data-offset`. If you intend to expand your
+   disk array in the future, then multiply a series of primes by your
+   raid stripe size to get the disk multiples that you would like.
+
+   For example:  If you have a 64k stripe size, then the following offset
+   would provide alignment for many common RAID5 data spindle counts::
+
+	64k * 2*2*2*3*3*5*7 bytes = 161280k
+
+   That space is wasted, but for only 157.5MB you can grow your RAID 5
+   volume to the following data-spindle counts without re-aligning::
+
+	3,4,5,6,7,8,9,10,12,14,15,18,20,21 ...
+
+ - Bad write performance
+
+   If write performance is not what you expected, you probably wanted to be
+   running in writeback mode, which isn't the default (not due to a lack of
+   maturity, but simply because in writeback mode you'll lose data if something
+   happens to your SSD)::
+
+	# echo writeback > /sys/block/bcache0/bcache/cache_mode
+
+ - Bad performance, or traffic not going to the SSD that you'd expect
+
+   By default, bcache doesn't cache everything. It tries to skip sequential IO -
+   because you really want to be caching the random IO, and if you copy a 10
+   gigabyte file you probably don't want that pushing 10 gigabytes of randomly
+   accessed data out of your cache.
+
+   But if you want to benchmark reads from cache, and you start out with fio
+   writing an 8 gigabyte test file - so you want to disable that::
+
+	# echo 0 > /sys/block/bcache0/bcache/sequential_cutoff
+
+   To set it back to the default (4 mb), do::
+
+	# echo 4M > /sys/block/bcache0/bcache/sequential_cutoff
+
+ - Traffic's still going to the spindle/still getting cache misses
+
+   In the real world, SSDs don't always keep up with disks - particularly with
+   slower SSDs, many disks being cached by one SSD, or mostly sequential IO. So
+   you want to avoid being bottlenecked by the SSD and having it slow everything
+   down.
+
+   To avoid that bcache tracks latency to the cache device, and gradually
+   throttles traffic if the latency exceeds a threshold (it does this by
+   cranking down the sequential bypass).
+
+   You can disable this if you need to by setting the thresholds to 0::
+
+	# echo 0 > /sys/fs/bcache/<cache set>/congested_read_threshold_us
+	# echo 0 > /sys/fs/bcache/<cache set>/congested_write_threshold_us
+
+   The default is 2000 us (2 milliseconds) for reads, and 20000 for writes.
+
+ - Still getting cache misses, of the same data
+
+   One last issue that sometimes trips people up is actually an old bug, due to
+   the way cache coherency is handled for cache misses. If a btree node is full,
+   a cache miss won't be able to insert a key for the new data and the data
+   won't be written to the cache.
+
+   In practice this isn't an issue because as soon as a write comes along it'll
+   cause the btree node to be split, and you need almost no write traffic for
+   this to not show up enough to be noticeable (especially since bcache's btree
+   nodes are huge and index large regions of the device). But when you're
+   benchmarking, if you're trying to warm the cache by reading a bunch of data
+   and there's no other traffic - that can be a problem.
+
+   Solution: warm the cache by doing writes, or use the testing branch (there's
+   a fix for the issue there).
+
+
+Sysfs - backing device
+----------------------
+
+Available at /sys/block/<bdev>/bcache, /sys/block/bcache*/bcache and
+(if attached) /sys/fs/bcache/<cset-uuid>/bdev*
+
+attach
+  Echo the UUID of a cache set to this file to enable caching.
+
+cache_mode
+  Can be one of either writethrough, writeback, writearound or none.
+
+clear_stats
+  Writing to this file resets the running total stats (not the day/hour/5 minute
+  decaying versions).
+
+detach
+  Write to this file to detach from a cache set. If there is dirty data in the
+  cache, it will be flushed first.
+
+dirty_data
+  Amount of dirty data for this backing device in the cache. Continuously
+  updated unlike the cache set's version, but may be slightly off.
+
+label
+  Name of underlying device.
+
+readahead
+  Size of readahead that should be performed.  Defaults to 0.  If set to e.g.
+  1M, it will round cache miss reads up to that size, but without overlapping
+  existing cache entries.
+
+running
+  1 if bcache is running (i.e. whether the /dev/bcache device exists, whether
+  it's in passthrough mode or caching).
+
+sequential_cutoff
+  A sequential IO will bypass the cache once it passes this threshold; the
+  most recent 128 IOs are tracked so sequential IO can be detected even when
+  it isn't all done at once.
+
+sequential_merge
+  If non zero, bcache keeps a list of the last 128 requests submitted to compare
+  against all new requests to determine which new requests are sequential
+  continuations of previous requests for the purpose of determining sequential
+  cutoff. This is necessary if the sequential cutoff value is greater than the
+  maximum acceptable sequential size for any single request.
+
+state
+  The backing device can be in one of four different states:
+
+  no cache: Has never been attached to a cache set.
+
+  clean: Part of a cache set, and there is no cached dirty data.
+
+  dirty: Part of a cache set, and there is cached dirty data.
+
+  inconsistent: The backing device was forcibly run by the user when there was
+  dirty data cached but the cache set was unavailable; whatever data was on the
+  backing device has likely been corrupted.
+
+stop
+  Write to this file to shut down the bcache device and close the backing
+  device.
+
+writeback_delay
+  When dirty data is written to the cache and it previously did not contain
+  any, waits some number of seconds before initiating writeback. Defaults to
+  30.
+
+writeback_percent
+  If nonzero, bcache tries to keep around this percentage of the cache dirty by
+  throttling background writeback and using a PD controller to smoothly adjust
+  the rate.
+
+writeback_rate
+  Rate in sectors per second - if writeback_percent is nonzero, background
+  writeback is throttled to this rate. Continuously adjusted by bcache but may
+  also be set by the user.
+
+writeback_running
+  If off, writeback of dirty data will not take place at all. Dirty data will
+  still be added to the cache until it is mostly full; only meant for
+  benchmarking. Defaults to on.
+
+Sysfs - backing device stats
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are directories with these numbers for a running total, as well as
+versions that decay over the past day, hour and 5 minutes; they're also
+aggregated in the cache set directory as well.
+
+bypassed
+  Amount of IO (both reads and writes) that has bypassed the cache
+
+cache_hits, cache_misses, cache_hit_ratio
+  Hits and misses are counted per individual IO as bcache sees them; a
+  partial hit is counted as a miss.
+
+cache_bypass_hits, cache_bypass_misses
+  Hits and misses for IO that is intended to skip the cache are still counted,
+  but broken out here.
+
+cache_miss_collisions
+  Counts instances where data was going to be inserted into the cache from a
+  cache miss, but raced with a write and data was already present (usually 0
+  since the synchronization for cache misses was rewritten)
+
+cache_readaheads
+  Count of times readahead occurred.
+
+Sysfs - cache set
+~~~~~~~~~~~~~~~~~
+
+Available at /sys/fs/bcache/<cset-uuid>
+
+average_key_size
+  Average data per key in the btree.
+
+bdev<0..n>
+  Symlink to each of the attached backing devices.
+
+block_size
+  Block size of the cache devices.
+
+btree_cache_size
+  Amount of memory currently used by the btree cache
+
+bucket_size
+  Size of buckets
+
+cache<0..n>
+  Symlink to each of the cache devices comprising this cache set.
+
+cache_available_percent
+  Percentage of cache device which doesn't contain dirty data, and could
+  potentially be used for writeback.  This doesn't mean this space isn't used
+  for clean cached data; the unused statistic (in priority_stats) is typically
+  much lower.
+
+clear_stats
+  Clears the statistics associated with this cache
+
+dirty_data
+  Amount of dirty data is in the cache (updated when garbage collection runs).
+
+flash_vol_create
+  Echoing a size to this file (in human readable units, k/M/G) creates a thinly
+  provisioned volume backed by the cache set.
+
+io_error_halflife, io_error_limit
+  These determines how many errors we accept before disabling the cache.
+  Each error is decayed by the half life (in # ios).  If the decaying count
+  reaches io_error_limit dirty data is written out and the cache is disabled.
+
+journal_delay_ms
+  Journal writes will delay for up to this many milliseconds, unless a cache
+  flush happens sooner. Defaults to 100.
+
+root_usage_percent
+  Percentage of the root btree node in use.  If this gets too high the node
+  will split, increasing the tree depth.
+
+stop
+  Write to this file to shut down the cache set - waits until all attached
+  backing devices have been shut down.
+
+tree_depth
+  Depth of the btree (A single node btree has depth 0).
+
+unregister
+  Detaches all backing devices and closes the cache devices; if dirty data is
+  present it will disable writeback caching and wait for it to be flushed.
+
+Sysfs - cache set internal
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This directory also exposes timings for a number of internal operations, with
+separate files for average duration, average frequency, last occurrence and max
+duration: garbage collection, btree read, btree node sorts and btree splits.
+
+active_journal_entries
+  Number of journal entries that are newer than the index.
+
+btree_nodes
+  Total nodes in the btree.
+
+btree_used_percent
+  Average fraction of btree in use.
+
+bset_tree_stats
+  Statistics about the auxiliary search trees
+
+btree_cache_max_chain
+  Longest chain in the btree node cache's hash table
+
+cache_read_races
+  Counts instances where while data was being read from the cache, the bucket
+  was reused and invalidated - i.e. where the pointer was stale after the read
+  completed. When this occurs the data is reread from the backing device.
+
+trigger_gc
+  Writing to this file forces garbage collection to run.
+
+Sysfs - Cache device
+~~~~~~~~~~~~~~~~~~~~
+
+Available at /sys/block/<cdev>/bcache
+
+block_size
+  Minimum granularity of writes - should match hardware sector size.
+
+btree_written
+  Sum of all btree writes, in (kilo/mega/giga) bytes
+
+bucket_size
+  Size of buckets
+
+cache_replacement_policy
+  One of either lru, fifo or random.
+
+discard
+  Boolean; if on a discard/TRIM will be issued to each bucket before it is
+  reused. Defaults to off, since SATA TRIM is an unqueued command (and thus
+  slow).
+
+freelist_percent
+  Size of the freelist as a percentage of nbuckets. Can be written to to
+  increase the number of buckets kept on the freelist, which lets you
+  artificially reduce the size of the cache at runtime. Mostly for testing
+  purposes (i.e. testing how different size caches affect your hit rate), but
+  since buckets are discarded when they move on to the freelist will also make
+  the SSD's garbage collection easier by effectively giving it more reserved
+  space.
+
+io_errors
+  Number of errors that have occurred, decayed by io_error_halflife.
+
+metadata_written
+  Sum of all non data writes (btree writes and all other metadata).
+
+nbuckets
+  Total buckets in this cache
+
+priority_stats
+  Statistics about how recently data in the cache has been accessed.
+  This can reveal your working set size.  Unused is the percentage of
+  the cache that doesn't contain any data.  Metadata is bcache's
+  metadata overhead.  Average is the average priority of cache buckets.
+  Next is a list of quantiles with the priority threshold of each.
+
+written
+  Sum of all data that has been written to the cache; comparison with
+  btree_written gives the amount of write inflation in bcache.
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
new file mode 100644
index 000000000000..2d4c4cf72008
--- /dev/null
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -0,0 +1,2114 @@
+================
+Control Group v2
+================
+
+:Date: October, 2015
+:Author: Tejun Heo <tj@kernel.org>
+
+This is the authoritative documentation on the design, interface and
+conventions of cgroup v2.  It describes all userland-visible aspects
+of cgroup including core and specific controller behaviors.  All
+future changes must be reflected in this document.  Documentation for
+v1 is available under Documentation/cgroup-v1/.
+
+.. CONTENTS
+
+   1. Introduction
+     1-1. Terminology
+     1-2. What is cgroup?
+   2. Basic Operations
+     2-1. Mounting
+     2-2. Organizing Processes and Threads
+       2-2-1. Processes
+       2-2-2. Threads
+     2-3. [Un]populated Notification
+     2-4. Controlling Controllers
+       2-4-1. Enabling and Disabling
+       2-4-2. Top-down Constraint
+       2-4-3. No Internal Process Constraint
+     2-5. Delegation
+       2-5-1. Model of Delegation
+       2-5-2. Delegation Containment
+     2-6. Guidelines
+       2-6-1. Organize Once and Control
+       2-6-2. Avoid Name Collisions
+   3. Resource Distribution Models
+     3-1. Weights
+     3-2. Limits
+     3-3. Protections
+     3-4. Allocations
+   4. Interface Files
+     4-1. Format
+     4-2. Conventions
+     4-3. Core Interface Files
+   5. Controllers
+     5-1. CPU
+       5-1-1. CPU Interface Files
+     5-2. Memory
+       5-2-1. Memory Interface Files
+       5-2-2. Usage Guidelines
+       5-2-3. Memory Ownership
+       5-2-4. OOM Killer
+     5-3. IO
+       5-3-1. IO Interface Files
+       5-3-2. Writeback
+     5-4. PID
+       5-4-1. PID Interface Files
+     5-5. Device
+     5-6. RDMA
+       5-6-1. RDMA Interface Files
+     5-7. Misc
+       5-7-1. perf_event
+     5-N. Non-normative information
+       5-N-1. CPU controller root cgroup process behaviour
+       5-N-2. IO controller root cgroup process behaviour
+   6. Namespace
+     6-1. Basics
+     6-2. The Root and Views
+     6-3. Migration and setns(2)
+     6-4. Interaction with Other Namespaces
+   P. Information on Kernel Programming
+     P-1. Filesystem Support for Writeback
+   D. Deprecated v1 Core Features
+   R. Issues with v1 and Rationales for v2
+     R-1. Multiple Hierarchies
+     R-2. Thread Granularity
+     R-3. Competition Between Inner Nodes and Threads
+     R-4. Other Interface Issues
+     R-5. Controller Issues and Remedies
+       R-5-1. Memory
+
+
+Introduction
+============
+
+Terminology
+-----------
+
+"cgroup" stands for "control group" and is never capitalized.  The
+singular form is used to designate the whole feature and also as a
+qualifier as in "cgroup controllers".  When explicitly referring to
+multiple individual control groups, the plural form "cgroups" is used.
+
+
+What is cgroup?
+---------------
+
+cgroup is a mechanism to organize processes hierarchically and
+distribute system resources along the hierarchy in a controlled and
+configurable manner.
+
+cgroup is largely composed of two parts - the core and controllers.
+cgroup core is primarily responsible for hierarchically organizing
+processes.  A cgroup controller is usually responsible for
+distributing a specific type of system resource along the hierarchy
+although there are utility controllers which serve purposes other than
+resource distribution.
+
+cgroups form a tree structure and every process in the system belongs
+to one and only one cgroup.  All threads of a process belong to the
+same cgroup.  On creation, all processes are put in the cgroup that
+the parent process belongs to at the time.  A process can be migrated
+to another cgroup.  Migration of a process doesn't affect already
+existing descendant processes.
+
+Following certain structural constraints, controllers may be enabled or
+disabled selectively on a cgroup.  All controller behaviors are
+hierarchical - if a controller is enabled on a cgroup, it affects all
+processes which belong to the cgroups consisting the inclusive
+sub-hierarchy of the cgroup.  When a controller is enabled on a nested
+cgroup, it always restricts the resource distribution further.  The
+restrictions set closer to the root in the hierarchy can not be
+overridden from further away.
+
+
+Basic Operations
+================
+
+Mounting
+--------
+
+Unlike v1, cgroup v2 has only single hierarchy.  The cgroup v2
+hierarchy can be mounted with the following mount command::
+
+  # mount -t cgroup2 none $MOUNT_POINT
+
+cgroup2 filesystem has the magic number 0x63677270 ("cgrp").  All
+controllers which support v2 and are not bound to a v1 hierarchy are
+automatically bound to the v2 hierarchy and show up at the root.
+Controllers which are not in active use in the v2 hierarchy can be
+bound to other hierarchies.  This allows mixing v2 hierarchy with the
+legacy v1 multiple hierarchies in a fully backward compatible way.
+
+A controller can be moved across hierarchies only after the controller
+is no longer referenced in its current hierarchy.  Because per-cgroup
+controller states are destroyed asynchronously and controllers may
+have lingering references, a controller may not show up immediately on
+the v2 hierarchy after the final umount of the previous hierarchy.
+Similarly, a controller should be fully disabled to be moved out of
+the unified hierarchy and it may take some time for the disabled
+controller to become available for other hierarchies; furthermore, due
+to inter-controller dependencies, other controllers may need to be
+disabled too.
+
+While useful for development and manual configurations, moving
+controllers dynamically between the v2 and other hierarchies is
+strongly discouraged for production use.  It is recommended to decide
+the hierarchies and controller associations before starting using the
+controllers after system boot.
+
+During transition to v2, system management software might still
+automount the v1 cgroup filesystem and so hijack all controllers
+during boot, before manual intervention is possible. To make testing
+and experimenting easier, the kernel parameter cgroup_no_v1= allows
+disabling controllers in v1 and make them always available in v2.
+
+cgroup v2 currently supports the following mount options.
+
+  nsdelegate
+
+	Consider cgroup namespaces as delegation boundaries.  This
+	option is system wide and can only be set on mount or modified
+	through remount from the init namespace.  The mount option is
+	ignored on non-init namespace mounts.  Please refer to the
+	Delegation section for details.
+
+
+Organizing Processes and Threads
+--------------------------------
+
+Processes
+~~~~~~~~~
+
+Initially, only the root cgroup exists to which all processes belong.
+A child cgroup can be created by creating a sub-directory::
+
+  # mkdir $CGROUP_NAME
+
+A given cgroup may have multiple child cgroups forming a tree
+structure.  Each cgroup has a read-writable interface file
+"cgroup.procs".  When read, it lists the PIDs of all processes which
+belong to the cgroup one-per-line.  The PIDs are not ordered and the
+same PID may show up more than once if the process got moved to
+another cgroup and then back or the PID got recycled while reading.
+
+A process can be migrated into a cgroup by writing its PID to the
+target cgroup's "cgroup.procs" file.  Only one process can be migrated
+on a single write(2) call.  If a process is composed of multiple
+threads, writing the PID of any thread migrates all threads of the
+process.
+
+When a process forks a child process, the new process is born into the
+cgroup that the forking process belongs to at the time of the
+operation.  After exit, a process stays associated with the cgroup
+that it belonged to at the time of exit until it's reaped; however, a
+zombie process does not appear in "cgroup.procs" and thus can't be
+moved to another cgroup.
+
+A cgroup which doesn't have any children or live processes can be
+destroyed by removing the directory.  Note that a cgroup which doesn't
+have any children and is associated only with zombie processes is
+considered empty and can be removed::
+
+  # rmdir $CGROUP_NAME
+
+"/proc/$PID/cgroup" lists a process's cgroup membership.  If legacy
+cgroup is in use in the system, this file may contain multiple lines,
+one for each hierarchy.  The entry for cgroup v2 is always in the
+format "0::$PATH"::
+
+  # cat /proc/842/cgroup
+  ...
+  0::/test-cgroup/test-cgroup-nested
+
+If the process becomes a zombie and the cgroup it was associated with
+is removed subsequently, " (deleted)" is appended to the path::
+
+  # cat /proc/842/cgroup
+  ...
+  0::/test-cgroup/test-cgroup-nested (deleted)
+
+
+Threads
+~~~~~~~
+
+cgroup v2 supports thread granularity for a subset of controllers to
+support use cases requiring hierarchical resource distribution across
+the threads of a group of processes.  By default, all threads of a
+process belong to the same cgroup, which also serves as the resource
+domain to host resource consumptions which are not specific to a
+process or thread.  The thread mode allows threads to be spread across
+a subtree while still maintaining the common resource domain for them.
+
+Controllers which support thread mode are called threaded controllers.
+The ones which don't are called domain controllers.
+
+Marking a cgroup threaded makes it join the resource domain of its
+parent as a threaded cgroup.  The parent may be another threaded
+cgroup whose resource domain is further up in the hierarchy.  The root
+of a threaded subtree, that is, the nearest ancestor which is not
+threaded, is called threaded domain or thread root interchangeably and
+serves as the resource domain for the entire subtree.
+
+Inside a threaded subtree, threads of a process can be put in
+different cgroups and are not subject to the no internal process
+constraint - threaded controllers can be enabled on non-leaf cgroups
+whether they have threads in them or not.
+
+As the threaded domain cgroup hosts all the domain resource
+consumptions of the subtree, it is considered to have internal
+resource consumptions whether there are processes in it or not and
+can't have populated child cgroups which aren't threaded.  Because the
+root cgroup is not subject to no internal process constraint, it can
+serve both as a threaded domain and a parent to domain cgroups.
+
+The current operation mode or type of the cgroup is shown in the
+"cgroup.type" file which indicates whether the cgroup is a normal
+domain, a domain which is serving as the domain of a threaded subtree,
+or a threaded cgroup.
+
+On creation, a cgroup is always a domain cgroup and can be made
+threaded by writing "threaded" to the "cgroup.type" file.  The
+operation is single direction::
+
+  # echo threaded > cgroup.type
+
+Once threaded, the cgroup can't be made a domain again.  To enable the
+thread mode, the following conditions must be met.
+
+- As the cgroup will join the parent's resource domain.  The parent
+  must either be a valid (threaded) domain or a threaded cgroup.
+
+- When the parent is an unthreaded domain, it must not have any domain
+  controllers enabled or populated domain children.  The root is
+  exempt from this requirement.
+
+Topology-wise, a cgroup can be in an invalid state.  Please consider
+the following topology::
+
+  A (threaded domain) - B (threaded) - C (domain, just created)
+
+C is created as a domain but isn't connected to a parent which can
+host child domains.  C can't be used until it is turned into a
+threaded cgroup.  "cgroup.type" file will report "domain (invalid)" in
+these cases.  Operations which fail due to invalid topology use
+EOPNOTSUPP as the errno.
+
+A domain cgroup is turned into a threaded domain when one of its child
+cgroup becomes threaded or threaded controllers are enabled in the
+"cgroup.subtree_control" file while there are processes in the cgroup.
+A threaded domain reverts to a normal domain when the conditions
+clear.
+
+When read, "cgroup.threads" contains the list of the thread IDs of all
+threads in the cgroup.  Except that the operations are per-thread
+instead of per-process, "cgroup.threads" has the same format and
+behaves the same way as "cgroup.procs".  While "cgroup.threads" can be
+written to in any cgroup, as it can only move threads inside the same
+threaded domain, its operations are confined inside each threaded
+subtree.
+
+The threaded domain cgroup serves as the resource domain for the whole
+subtree, and, while the threads can be scattered across the subtree,
+all the processes are considered to be in the threaded domain cgroup.
+"cgroup.procs" in a threaded domain cgroup contains the PIDs of all
+processes in the subtree and is not readable in the subtree proper.
+However, "cgroup.procs" can be written to from anywhere in the subtree
+to migrate all threads of the matching process to the cgroup.
+
+Only threaded controllers can be enabled in a threaded subtree.  When
+a threaded controller is enabled inside a threaded subtree, it only
+accounts for and controls resource consumptions associated with the
+threads in the cgroup and its descendants.  All consumptions which
+aren't tied to a specific thread belong to the threaded domain cgroup.
+
+Because a threaded subtree is exempt from no internal process
+constraint, a threaded controller must be able to handle competition
+between threads in a non-leaf cgroup and its child cgroups.  Each
+threaded controller defines how such competitions are handled.
+
+
+[Un]populated Notification
+--------------------------
+
+Each non-root cgroup has a "cgroup.events" file which contains
+"populated" field indicating whether the cgroup's sub-hierarchy has
+live processes in it.  Its value is 0 if there is no live process in
+the cgroup and its descendants; otherwise, 1.  poll and [id]notify
+events are triggered when the value changes.  This can be used, for
+example, to start a clean-up operation after all processes of a given
+sub-hierarchy have exited.  The populated state updates and
+notifications are recursive.  Consider the following sub-hierarchy
+where the numbers in the parentheses represent the numbers of processes
+in each cgroup::
+
+  A(4) - B(0) - C(1)
+              \ D(0)
+
+A, B and C's "populated" fields would be 1 while D's 0.  After the one
+process in C exits, B and C's "populated" fields would flip to "0" and
+file modified events will be generated on the "cgroup.events" files of
+both cgroups.
+
+
+Controlling Controllers
+-----------------------
+
+Enabling and Disabling
+~~~~~~~~~~~~~~~~~~~~~~
+
+Each cgroup has a "cgroup.controllers" file which lists all
+controllers available for the cgroup to enable::
+
+  # cat cgroup.controllers
+  cpu io memory
+
+No controller is enabled by default.  Controllers can be enabled and
+disabled by writing to the "cgroup.subtree_control" file::
+
+  # echo "+cpu +memory -io" > cgroup.subtree_control
+
+Only controllers which are listed in "cgroup.controllers" can be
+enabled.  When multiple operations are specified as above, either they
+all succeed or fail.  If multiple operations on the same controller
+are specified, the last one is effective.
+
+Enabling a controller in a cgroup indicates that the distribution of
+the target resource across its immediate children will be controlled.
+Consider the following sub-hierarchy.  The enabled controllers are
+listed in parentheses::
+
+  A(cpu,memory) - B(memory) - C()
+                            \ D()
+
+As A has "cpu" and "memory" enabled, A will control the distribution
+of CPU cycles and memory to its children, in this case, B.  As B has
+"memory" enabled but not "CPU", C and D will compete freely on CPU
+cycles but their division of memory available to B will be controlled.
+
+As a controller regulates the distribution of the target resource to
+the cgroup's children, enabling it creates the controller's interface
+files in the child cgroups.  In the above example, enabling "cpu" on B
+would create the "cpu." prefixed controller interface files in C and
+D.  Likewise, disabling "memory" from B would remove the "memory."
+prefixed controller interface files from C and D.  This means that the
+controller interface files - anything which doesn't start with
+"cgroup." are owned by the parent rather than the cgroup itself.
+
+
+Top-down Constraint
+~~~~~~~~~~~~~~~~~~~
+
+Resources are distributed top-down and a cgroup can further distribute
+a resource only if the resource has been distributed to it from the
+parent.  This means that all non-root "cgroup.subtree_control" files
+can only contain controllers which are enabled in the parent's
+"cgroup.subtree_control" file.  A controller can be enabled only if
+the parent has the controller enabled and a controller can't be
+disabled if one or more children have it enabled.
+
+
+No Internal Process Constraint
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Non-root cgroups can distribute domain resources to their children
+only when they don't have any processes of their own.  In other words,
+only domain cgroups which don't contain any processes can have domain
+controllers enabled in their "cgroup.subtree_control" files.
+
+This guarantees that, when a domain controller is looking at the part
+of the hierarchy which has it enabled, processes are always only on
+the leaves.  This rules out situations where child cgroups compete
+against internal processes of the parent.
+
+The root cgroup is exempt from this restriction.  Root contains
+processes and anonymous resource consumption which can't be associated
+with any other cgroups and requires special treatment from most
+controllers.  How resource consumption in the root cgroup is governed
+is up to each controller (for more information on this topic please
+refer to the Non-normative information section in the Controllers
+chapter).
+
+Note that the restriction doesn't get in the way if there is no
+enabled controller in the cgroup's "cgroup.subtree_control".  This is
+important as otherwise it wouldn't be possible to create children of a
+populated cgroup.  To control resource distribution of a cgroup, the
+cgroup must create children and transfer all its processes to the
+children before enabling controllers in its "cgroup.subtree_control"
+file.
+
+
+Delegation
+----------
+
+Model of Delegation
+~~~~~~~~~~~~~~~~~~~
+
+A cgroup can be delegated in two ways.  First, to a less privileged
+user by granting write access of the directory and its "cgroup.procs",
+"cgroup.threads" and "cgroup.subtree_control" files to the user.
+Second, if the "nsdelegate" mount option is set, automatically to a
+cgroup namespace on namespace creation.
+
+Because the resource control interface files in a given directory
+control the distribution of the parent's resources, the delegatee
+shouldn't be allowed to write to them.  For the first method, this is
+achieved by not granting access to these files.  For the second, the
+kernel rejects writes to all files other than "cgroup.procs" and
+"cgroup.subtree_control" on a namespace root from inside the
+namespace.
+
+The end results are equivalent for both delegation types.  Once
+delegated, the user can build sub-hierarchy under the directory,
+organize processes inside it as it sees fit and further distribute the
+resources it received from the parent.  The limits and other settings
+of all resource controllers are hierarchical and regardless of what
+happens in the delegated sub-hierarchy, nothing can escape the
+resource restrictions imposed by the parent.
+
+Currently, cgroup doesn't impose any restrictions on the number of
+cgroups in or nesting depth of a delegated sub-hierarchy; however,
+this may be limited explicitly in the future.
+
+
+Delegation Containment
+~~~~~~~~~~~~~~~~~~~~~~
+
+A delegated sub-hierarchy is contained in the sense that processes
+can't be moved into or out of the sub-hierarchy by the delegatee.
+
+For delegations to a less privileged user, this is achieved by
+requiring the following conditions for a process with a non-root euid
+to migrate a target process into a cgroup by writing its PID to the
+"cgroup.procs" file.
+
+- The writer must have write access to the "cgroup.procs" file.
+
+- The writer must have write access to the "cgroup.procs" file of the
+  common ancestor of the source and destination cgroups.
+
+The above two constraints ensure that while a delegatee may migrate
+processes around freely in the delegated sub-hierarchy it can't pull
+in from or push out to outside the sub-hierarchy.
+
+For an example, let's assume cgroups C0 and C1 have been delegated to
+user U0 who created C00, C01 under C0 and C10 under C1 as follows and
+all processes under C0 and C1 belong to U0::
+
+  ~~~~~~~~~~~~~ - C0 - C00
+  ~ cgroup    ~      \ C01
+  ~ hierarchy ~
+  ~~~~~~~~~~~~~ - C1 - C10
+
+Let's also say U0 wants to write the PID of a process which is
+currently in C10 into "C00/cgroup.procs".  U0 has write access to the
+file; however, the common ancestor of the source cgroup C10 and the
+destination cgroup C00 is above the points of delegation and U0 would
+not have write access to its "cgroup.procs" files and thus the write
+will be denied with -EACCES.
+
+For delegations to namespaces, containment is achieved by requiring
+that both the source and destination cgroups are reachable from the
+namespace of the process which is attempting the migration.  If either
+is not reachable, the migration is rejected with -ENOENT.
+
+
+Guidelines
+----------
+
+Organize Once and Control
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Migrating a process across cgroups is a relatively expensive operation
+and stateful resources such as memory are not moved together with the
+process.  This is an explicit design decision as there often exist
+inherent trade-offs between migration and various hot paths in terms
+of synchronization cost.
+
+As such, migrating processes across cgroups frequently as a means to
+apply different resource restrictions is discouraged.  A workload
+should be assigned to a cgroup according to the system's logical and
+resource structure once on start-up.  Dynamic adjustments to resource
+distribution can be made by changing controller configuration through
+the interface files.
+
+
+Avoid Name Collisions
+~~~~~~~~~~~~~~~~~~~~~
+
+Interface files for a cgroup and its children cgroups occupy the same
+directory and it is possible to create children cgroups which collide
+with interface files.
+
+All cgroup core interface files are prefixed with "cgroup." and each
+controller's interface files are prefixed with the controller name and
+a dot.  A controller's name is composed of lower case alphabets and
+'_'s but never begins with an '_' so it can be used as the prefix
+character for collision avoidance.  Also, interface file names won't
+start or end with terms which are often used in categorizing workloads
+such as job, service, slice, unit or workload.
+
+cgroup doesn't do anything to prevent name collisions and it's the
+user's responsibility to avoid them.
+
+
+Resource Distribution Models
+============================
+
+cgroup controllers implement several resource distribution schemes
+depending on the resource type and expected use cases.  This section
+describes major schemes in use along with their expected behaviors.
+
+
+Weights
+-------
+
+A parent's resource is distributed by adding up the weights of all
+active children and giving each the fraction matching the ratio of its
+weight against the sum.  As only children which can make use of the
+resource at the moment participate in the distribution, this is
+work-conserving.  Due to the dynamic nature, this model is usually
+used for stateless resources.
+
+All weights are in the range [1, 10000] with the default at 100.  This
+allows symmetric multiplicative biases in both directions at fine
+enough granularity while staying in the intuitive range.
+
+As long as the weight is in range, all configuration combinations are
+valid and there is no reason to reject configuration changes or
+process migrations.
+
+"cpu.weight" proportionally distributes CPU cycles to active children
+and is an example of this type.
+
+
+Limits
+------
+
+A child can only consume upto the configured amount of the resource.
+Limits can be over-committed - the sum of the limits of children can
+exceed the amount of resource available to the parent.
+
+Limits are in the range [0, max] and defaults to "max", which is noop.
+
+As limits can be over-committed, all configuration combinations are
+valid and there is no reason to reject configuration changes or
+process migrations.
+
+"io.max" limits the maximum BPS and/or IOPS that a cgroup can consume
+on an IO device and is an example of this type.
+
+
+Protections
+-----------
+
+A cgroup is protected to be allocated upto the configured amount of
+the resource if the usages of all its ancestors are under their
+protected levels.  Protections can be hard guarantees or best effort
+soft boundaries.  Protections can also be over-committed in which case
+only upto the amount available to the parent is protected among
+children.
+
+Protections are in the range [0, max] and defaults to 0, which is
+noop.
+
+As protections can be over-committed, all configuration combinations
+are valid and there is no reason to reject configuration changes or
+process migrations.
+
+"memory.low" implements best-effort memory protection and is an
+example of this type.
+
+
+Allocations
+-----------
+
+A cgroup is exclusively allocated a certain amount of a finite
+resource.  Allocations can't be over-committed - the sum of the
+allocations of children can not exceed the amount of resource
+available to the parent.
+
+Allocations are in the range [0, max] and defaults to 0, which is no
+resource.
+
+As allocations can't be over-committed, some configuration
+combinations are invalid and should be rejected.  Also, if the
+resource is mandatory for execution of processes, process migrations
+may be rejected.
+
+"cpu.rt.max" hard-allocates realtime slices and is an example of this
+type.
+
+
+Interface Files
+===============
+
+Format
+------
+
+All interface files should be in one of the following formats whenever
+possible::
+
+  New-line separated values
+  (when only one value can be written at once)
+
+	VAL0\n
+	VAL1\n
+	...
+
+  Space separated values
+  (when read-only or multiple values can be written at once)
+
+	VAL0 VAL1 ...\n
+
+  Flat keyed
+
+	KEY0 VAL0\n
+	KEY1 VAL1\n
+	...
+
+  Nested keyed
+
+	KEY0 SUB_KEY0=VAL00 SUB_KEY1=VAL01...
+	KEY1 SUB_KEY0=VAL10 SUB_KEY1=VAL11...
+	...
+
+For a writable file, the format for writing should generally match
+reading; however, controllers may allow omitting later fields or
+implement restricted shortcuts for most common use cases.
+
+For both flat and nested keyed files, only the values for a single key
+can be written at a time.  For nested keyed files, the sub key pairs
+may be specified in any order and not all pairs have to be specified.
+
+
+Conventions
+-----------
+
+- Settings for a single feature should be contained in a single file.
+
+- The root cgroup should be exempt from resource control and thus
+  shouldn't have resource control interface files.  Also,
+  informational files on the root cgroup which end up showing global
+  information available elsewhere shouldn't exist.
+
+- If a controller implements weight based resource distribution, its
+  interface file should be named "weight" and have the range [1,
+  10000] with 100 as the default.  The values are chosen to allow
+  enough and symmetric bias in both directions while keeping it
+  intuitive (the default is 100%).
+
+- If a controller implements an absolute resource guarantee and/or
+  limit, the interface files should be named "min" and "max"
+  respectively.  If a controller implements best effort resource
+  guarantee and/or limit, the interface files should be named "low"
+  and "high" respectively.
+
+  In the above four control files, the special token "max" should be
+  used to represent upward infinity for both reading and writing.
+
+- If a setting has a configurable default value and keyed specific
+  overrides, the default entry should be keyed with "default" and
+  appear as the first entry in the file.
+
+  The default value can be updated by writing either "default $VAL" or
+  "$VAL".
+
+  When writing to update a specific override, "default" can be used as
+  the value to indicate removal of the override.  Override entries
+  with "default" as the value must not appear when read.
+
+  For example, a setting which is keyed by major:minor device numbers
+  with integer values may look like the following::
+
+    # cat cgroup-example-interface-file
+    default 150
+    8:0 300
+
+  The default value can be updated by::
+
+    # echo 125 > cgroup-example-interface-file
+
+  or::
+
+    # echo "default 125" > cgroup-example-interface-file
+
+  An override can be set by::
+
+    # echo "8:16 170" > cgroup-example-interface-file
+
+  and cleared by::
+
+    # echo "8:0 default" > cgroup-example-interface-file
+    # cat cgroup-example-interface-file
+    default 125
+    8:16 170
+
+- For events which are not very high frequency, an interface file
+  "events" should be created which lists event key value pairs.
+  Whenever a notifiable event happens, file modified event should be
+  generated on the file.
+
+
+Core Interface Files
+--------------------
+
+All cgroup core files are prefixed with "cgroup."
+
+  cgroup.type
+
+	A read-write single value file which exists on non-root
+	cgroups.
+
+	When read, it indicates the current type of the cgroup, which
+	can be one of the following values.
+
+	- "domain" : A normal valid domain cgroup.
+
+	- "domain threaded" : A threaded domain cgroup which is
+          serving as the root of a threaded subtree.
+
+	- "domain invalid" : A cgroup which is in an invalid state.
+	  It can't be populated or have controllers enabled.  It may
+	  be allowed to become a threaded cgroup.
+
+	- "threaded" : A threaded cgroup which is a member of a
+          threaded subtree.
+
+	A cgroup can be turned into a threaded cgroup by writing
+	"threaded" to this file.
+
+  cgroup.procs
+	A read-write new-line separated values file which exists on
+	all cgroups.
+
+	When read, it lists the PIDs of all processes which belong to
+	the cgroup one-per-line.  The PIDs are not ordered and the
+	same PID may show up more than once if the process got moved
+	to another cgroup and then back or the PID got recycled while
+	reading.
+
+	A PID can be written to migrate the process associated with
+	the PID to the cgroup.  The writer should match all of the
+	following conditions.
+
+	- It must have write access to the "cgroup.procs" file.
+
+	- It must have write access to the "cgroup.procs" file of the
+	  common ancestor of the source and destination cgroups.
+
+	When delegating a sub-hierarchy, write access to this file
+	should be granted along with the containing directory.
+
+	In a threaded cgroup, reading this file fails with EOPNOTSUPP
+	as all the processes belong to the thread root.  Writing is
+	supported and moves every thread of the process to the cgroup.
+
+  cgroup.threads
+	A read-write new-line separated values file which exists on
+	all cgroups.
+
+	When read, it lists the TIDs of all threads which belong to
+	the cgroup one-per-line.  The TIDs are not ordered and the
+	same TID may show up more than once if the thread got moved to
+	another cgroup and then back or the TID got recycled while
+	reading.
+
+	A TID can be written to migrate the thread associated with the
+	TID to the cgroup.  The writer should match all of the
+	following conditions.
+
+	- It must have write access to the "cgroup.threads" file.
+
+	- The cgroup that the thread is currently in must be in the
+          same resource domain as the destination cgroup.
+
+	- It must have write access to the "cgroup.procs" file of the
+	  common ancestor of the source and destination cgroups.
+
+	When delegating a sub-hierarchy, write access to this file
+	should be granted along with the containing directory.
+
+  cgroup.controllers
+	A read-only space separated values file which exists on all
+	cgroups.
+
+	It shows space separated list of all controllers available to
+	the cgroup.  The controllers are not ordered.
+
+  cgroup.subtree_control
+	A read-write space separated values file which exists on all
+	cgroups.  Starts out empty.
+
+	When read, it shows space separated list of the controllers
+	which are enabled to control resource distribution from the
+	cgroup to its children.
+
+	Space separated list of controllers prefixed with '+' or '-'
+	can be written to enable or disable controllers.  A controller
+	name prefixed with '+' enables the controller and '-'
+	disables.  If a controller appears more than once on the list,
+	the last one is effective.  When multiple enable and disable
+	operations are specified, either all succeed or all fail.
+
+  cgroup.events
+	A read-only flat-keyed file which exists on non-root cgroups.
+	The following entries are defined.  Unless specified
+	otherwise, a value change in this file generates a file
+	modified event.
+
+	  populated
+		1 if the cgroup or its descendants contains any live
+		processes; otherwise, 0.
+
+  cgroup.max.descendants
+	A read-write single value files.  The default is "max".
+
+	Maximum allowed number of descent cgroups.
+	If the actual number of descendants is equal or larger,
+	an attempt to create a new cgroup in the hierarchy will fail.
+
+  cgroup.max.depth
+	A read-write single value files.  The default is "max".
+
+	Maximum allowed descent depth below the current cgroup.
+	If the actual descent depth is equal or larger,
+	an attempt to create a new child cgroup will fail.
+
+  cgroup.stat
+	A read-only flat-keyed file with the following entries:
+
+	  nr_descendants
+		Total number of visible descendant cgroups.
+
+	  nr_dying_descendants
+		Total number of dying descendant cgroups. A cgroup becomes
+		dying after being deleted by a user. The cgroup will remain
+		in dying state for some time undefined time (which can depend
+		on system load) before being completely destroyed.
+
+		A process can't enter a dying cgroup under any circumstances,
+		a dying cgroup can't revive.
+
+		A dying cgroup can consume system resources not exceeding
+		limits, which were active at the moment of cgroup deletion.
+
+
+Controllers
+===========
+
+CPU
+---
+
+The "cpu" controllers regulates distribution of CPU cycles.  This
+controller implements weight and absolute bandwidth limit models for
+normal scheduling policy and absolute bandwidth allocation model for
+realtime scheduling policy.
+
+WARNING: cgroup2 doesn't yet support control of realtime processes and
+the cpu controller can only be enabled when all RT processes are in
+the root cgroup.  Be aware that system management software may already
+have placed RT processes into nonroot cgroups during the system boot
+process, and these processes may need to be moved to the root cgroup
+before the cpu controller can be enabled.
+
+
+CPU Interface Files
+~~~~~~~~~~~~~~~~~~~
+
+All time durations are in microseconds.
+
+  cpu.stat
+	A read-only flat-keyed file which exists on non-root cgroups.
+	This file exists whether the controller is enabled or not.
+
+	It always reports the following three stats:
+
+	- usage_usec
+	- user_usec
+	- system_usec
+
+	and the following three when the controller is enabled:
+
+	- nr_periods
+	- nr_throttled
+	- throttled_usec
+
+  cpu.weight
+	A read-write single value file which exists on non-root
+	cgroups.  The default is "100".
+
+	The weight in the range [1, 10000].
+
+  cpu.weight.nice
+	A read-write single value file which exists on non-root
+	cgroups.  The default is "0".
+
+	The nice value is in the range [-20, 19].
+
+	This interface file is an alternative interface for
+	"cpu.weight" and allows reading and setting weight using the
+	same values used by nice(2).  Because the range is smaller and
+	granularity is coarser for the nice values, the read value is
+	the closest approximation of the current weight.
+
+  cpu.max
+	A read-write two value file which exists on non-root cgroups.
+	The default is "max 100000".
+
+	The maximum bandwidth limit.  It's in the following format::
+
+	  $MAX $PERIOD
+
+	which indicates that the group may consume upto $MAX in each
+	$PERIOD duration.  "max" for $MAX indicates no limit.  If only
+	one number is written, $MAX is updated.
+
+
+Memory
+------
+
+The "memory" controller regulates distribution of memory.  Memory is
+stateful and implements both limit and protection models.  Due to the
+intertwining between memory usage and reclaim pressure and the
+stateful nature of memory, the distribution model is relatively
+complex.
+
+While not completely water-tight, all major memory usages by a given
+cgroup are tracked so that the total memory consumption can be
+accounted and controlled to a reasonable extent.  Currently, the
+following types of memory usages are tracked.
+
+- Userland memory - page cache and anonymous memory.
+
+- Kernel data structures such as dentries and inodes.
+
+- TCP socket buffers.
+
+The above list may expand in the future for better coverage.
+
+
+Memory Interface Files
+~~~~~~~~~~~~~~~~~~~~~~
+
+All memory amounts are in bytes.  If a value which is not aligned to
+PAGE_SIZE is written, the value may be rounded up to the closest
+PAGE_SIZE multiple when read back.
+
+  memory.current
+	A read-only single value file which exists on non-root
+	cgroups.
+
+	The total amount of memory currently being used by the cgroup
+	and its descendants.
+
+  memory.min
+	A read-write single value file which exists on non-root
+	cgroups.  The default is "0".
+
+	Hard memory protection.  If the memory usage of a cgroup
+	is within its effective min boundary, the cgroup's memory
+	won't be reclaimed under any conditions. If there is no
+	unprotected reclaimable memory available, OOM killer
+	is invoked.
+
+       Effective min boundary is limited by memory.min values of
+	all ancestor cgroups. If there is memory.min overcommitment
+	(child cgroup or cgroups are requiring more protected memory
+	than parent will allow), then each child cgroup will get
+	the part of parent's protection proportional to its
+	actual memory usage below memory.min.
+
+	Putting more memory than generally available under this
+	protection is discouraged and may lead to constant OOMs.
+
+	If a memory cgroup is not populated with processes,
+	its memory.min is ignored.
+
+  memory.low
+	A read-write single value file which exists on non-root
+	cgroups.  The default is "0".
+
+	Best-effort memory protection.  If the memory usage of a
+	cgroup is within its effective low boundary, the cgroup's
+	memory won't be reclaimed unless memory can be reclaimed
+	from unprotected cgroups.
+
+	Effective low boundary is limited by memory.low values of
+	all ancestor cgroups. If there is memory.low overcommitment
+	(child cgroup or cgroups are requiring more protected memory
+	than parent will allow), then each child cgroup will get
+	the part of parent's protection proportional to its
+	actual memory usage below memory.low.
+
+	Putting more memory than generally available under this
+	protection is discouraged.
+
+  memory.high
+	A read-write single value file which exists on non-root
+	cgroups.  The default is "max".
+
+	Memory usage throttle limit.  This is the main mechanism to
+	control memory usage of a cgroup.  If a cgroup's usage goes
+	over the high boundary, the processes of the cgroup are
+	throttled and put under heavy reclaim pressure.
+
+	Going over the high limit never invokes the OOM killer and
+	under extreme conditions the limit may be breached.
+
+  memory.max
+	A read-write single value file which exists on non-root
+	cgroups.  The default is "max".
+
+	Memory usage hard limit.  This is the final protection
+	mechanism.  If a cgroup's memory usage reaches this limit and
+	can't be reduced, the OOM killer is invoked in the cgroup.
+	Under certain circumstances, the usage may go over the limit
+	temporarily.
+
+	This is the ultimate protection mechanism.  As long as the
+	high limit is used and monitored properly, this limit's
+	utility is limited to providing the final safety net.
+
+  memory.oom_group
+
+	A read-write single value file which exists on non-root
+	cgroups.  The default is "0".
+
+	If set, OOM killer will consider the memory cgroup as an
+	indivisible memory consumers and compare it with other memory
+	consumers by it's memory footprint.
+	If such memory cgroup is selected as an OOM victim, all
+	processes belonging to it or it's descendants will be killed.
+
+	This applies to system-wide OOM conditions and reaching
+	the hard memory limit of the cgroup and their ancestor.
+	If OOM condition happens in a descendant cgroup with it's own
+	memory limit, the memory cgroup can't be considered
+	as an OOM victim, and OOM killer will not kill all belonging
+	tasks.
+
+	Also, OOM killer respects the /proc/pid/oom_score_adj value -1000,
+	and will never kill the unkillable task, even if memory.oom_group
+	is set.
+
+	If cgroup-aware OOM killer is not enabled, ENOTSUPP error
+	is returned on attempt to access the file.
+
+  memory.events
+	A read-only flat-keyed file which exists on non-root cgroups.
+	The following entries are defined.  Unless specified
+	otherwise, a value change in this file generates a file
+	modified event.
+
+	  low
+		The number of times the cgroup is reclaimed due to
+		high memory pressure even though its usage is under
+		the low boundary.  This usually indicates that the low
+		boundary is over-committed.
+
+	  high
+		The number of times processes of the cgroup are
+		throttled and routed to perform direct memory reclaim
+		because the high memory boundary was exceeded.  For a
+		cgroup whose memory usage is capped by the high limit
+		rather than global memory pressure, this event's
+		occurrences are expected.
+
+	  max
+		The number of times the cgroup's memory usage was
+		about to go over the max boundary.  If direct reclaim
+		fails to bring it down, the cgroup goes to OOM state.
+
+	  oom
+		The number of time the cgroup's memory usage was
+		reached the limit and allocation was about to fail.
+
+		Depending on context result could be invocation of OOM
+		killer and retrying allocation or failing allocation.
+
+		Failed allocation in its turn could be returned into
+		userspace as -ENOMEM or silently ignored in cases like
+		disk readahead.  For now OOM in memory cgroup kills
+		tasks iff shortage has happened inside page fault.
+
+	  oom_kill
+		The number of processes belonging to this cgroup
+		killed by any kind of OOM killer.
+
+  memory.stat
+	A read-only flat-keyed file which exists on non-root cgroups.
+
+	This breaks down the cgroup's memory footprint into different
+	types of memory, type-specific details, and other information
+	on the state and past events of the memory management system.
+
+	All memory amounts are in bytes.
+
+	The entries are ordered to be human readable, and new entries
+	can show up in the middle. Don't rely on items remaining in a
+	fixed position; use the keys to look up specific values!
+
+	  anon
+		Amount of memory used in anonymous mappings such as
+		brk(), sbrk(), and mmap(MAP_ANONYMOUS)
+
+	  file
+		Amount of memory used to cache filesystem data,
+		including tmpfs and shared memory.
+
+	  kernel_stack
+		Amount of memory allocated to kernel stacks.
+
+	  slab
+		Amount of memory used for storing in-kernel data
+		structures.
+
+	  sock
+		Amount of memory used in network transmission buffers
+
+	  shmem
+		Amount of cached filesystem data that is swap-backed,
+		such as tmpfs, shm segments, shared anonymous mmap()s
+
+	  file_mapped
+		Amount of cached filesystem data mapped with mmap()
+
+	  file_dirty
+		Amount of cached filesystem data that was modified but
+		not yet written back to disk
+
+	  file_writeback
+		Amount of cached filesystem data that was modified and
+		is currently being written back to disk
+
+	  inactive_anon, active_anon, inactive_file, active_file, unevictable
+		Amount of memory, swap-backed and filesystem-backed,
+		on the internal memory management lists used by the
+		page reclaim algorithm
+
+	  slab_reclaimable
+		Part of "slab" that might be reclaimed, such as
+		dentries and inodes.
+
+	  slab_unreclaimable
+		Part of "slab" that cannot be reclaimed on memory
+		pressure.
+
+	  pgfault
+		Total number of page faults incurred
+
+	  pgmajfault
+		Number of major page faults incurred
+
+	  workingset_refault
+
+		Number of refaults of previously evicted pages
+
+	  workingset_activate
+
+		Number of refaulted pages that were immediately activated
+
+	  workingset_nodereclaim
+
+		Number of times a shadow node has been reclaimed
+
+	  pgrefill
+
+		Amount of scanned pages (in an active LRU list)
+
+	  pgscan
+
+		Amount of scanned pages (in an inactive LRU list)
+
+	  pgsteal
+
+		Amount of reclaimed pages
+
+	  pgactivate
+
+		Amount of pages moved to the active LRU list
+
+	  pgdeactivate
+
+		Amount of pages moved to the inactive LRU lis
+
+	  pglazyfree
+
+		Amount of pages postponed to be freed under memory pressure
+
+	  pglazyfreed
+
+		Amount of reclaimed lazyfree pages
+
+  memory.swap.current
+	A read-only single value file which exists on non-root
+	cgroups.
+
+	The total amount of swap currently being used by the cgroup
+	and its descendants.
+
+  memory.swap.max
+	A read-write single value file which exists on non-root
+	cgroups.  The default is "max".
+
+	Swap usage hard limit.  If a cgroup's swap usage reaches this
+	limit, anonymous memory of the cgroup will not be swapped out.
+
+  memory.swap.events
+	A read-only flat-keyed file which exists on non-root cgroups.
+	The following entries are defined.  Unless specified
+	otherwise, a value change in this file generates a file
+	modified event.
+
+	  max
+		The number of times the cgroup's swap usage was about
+		to go over the max boundary and swap allocation
+		failed.
+
+	  fail
+		The number of times swap allocation failed either
+		because of running out of swap system-wide or max
+		limit.
+
+	When reduced under the current usage, the existing swap
+	entries are reclaimed gradually and the swap usage may stay
+	higher than the limit for an extended period of time.  This
+	reduces the impact on the workload and memory management.
+
+
+Usage Guidelines
+~~~~~~~~~~~~~~~~
+
+"memory.high" is the main mechanism to control memory usage.
+Over-committing on high limit (sum of high limits > available memory)
+and letting global memory pressure to distribute memory according to
+usage is a viable strategy.
+
+Because breach of the high limit doesn't trigger the OOM killer but
+throttles the offending cgroup, a management agent has ample
+opportunities to monitor and take appropriate actions such as granting
+more memory or terminating the workload.
+
+Determining whether a cgroup has enough memory is not trivial as
+memory usage doesn't indicate whether the workload can benefit from
+more memory.  For example, a workload which writes data received from
+network to a file can use all available memory but can also operate as
+performant with a small amount of memory.  A measure of memory
+pressure - how much the workload is being impacted due to lack of
+memory - is necessary to determine whether a workload needs more
+memory; unfortunately, memory pressure monitoring mechanism isn't
+implemented yet.
+
+
+Memory Ownership
+~~~~~~~~~~~~~~~~
+
+A memory area is charged to the cgroup which instantiated it and stays
+charged to the cgroup until the area is released.  Migrating a process
+to a different cgroup doesn't move the memory usages that it
+instantiated while in the previous cgroup to the new cgroup.
+
+A memory area may be used by processes belonging to different cgroups.
+To which cgroup the area will be charged is in-deterministic; however,
+over time, the memory area is likely to end up in a cgroup which has
+enough memory allowance to avoid high reclaim pressure.
+
+If a cgroup sweeps a considerable amount of memory which is expected
+to be accessed repeatedly by other cgroups, it may make sense to use
+POSIX_FADV_DONTNEED to relinquish the ownership of memory areas
+belonging to the affected files to ensure correct memory ownership.
+
+OOM Killer
+~~~~~~~~~~
+
+Cgroup v2 memory controller implements a cgroup-aware OOM killer.
+It means that it treats cgroups as first class OOM entities.
+
+Cgroup-aware OOM logic is turned off by default and requires
+passing the "groupoom" option on mounting cgroupfs. It can also
+by remounting cgroupfs with the following command::
+
+  # mount -o remount,groupoom $MOUNT_POINT
+
+Under OOM conditions the memory controller tries to make the best
+choice of a victim, looking for a memory cgroup with the largest
+memory footprint, considering leaf cgroups and cgroups with the
+memory.oom_group option set, which are considered to be an indivisible
+memory consumers.
+
+By default, OOM killer will kill the biggest task in the selected
+memory cgroup. A user can change this behavior by enabling
+the per-cgroup memory.oom_group option. If set, it causes
+the OOM killer to kill all processes attached to the cgroup,
+except processes with oom_score_adj set to -1000.
+
+This affects both system- and cgroup-wide OOMs. For a cgroup-wide OOM
+the memory controller considers only cgroups belonging to the sub-tree
+of the OOM'ing cgroup.
+
+Leaf cgroups and cgroups with oom_group option set are compared based
+on their cumulative memory usage. The root cgroup is treated as a
+leaf memory cgroup as well, so it is compared with other leaf memory
+cgroups. Due to internal implementation restrictions the size of
+the root cgroup is the cumulative sum of oom_badness of all its tasks
+(in other words oom_score_adj of each task is obeyed). Relying on
+oom_score_adj (apart from OOM_SCORE_ADJ_MIN) can lead to over- or
+underestimation of the root cgroup consumption and it is therefore
+discouraged. This might change in the future, however.
+
+If there are no cgroups with the enabled memory controller,
+the OOM killer is using the "traditional" process-based approach.
+
+Please, note that memory charges are not migrating if tasks
+are moved between different memory cgroups. Moving tasks with
+significant memory footprint may affect OOM victim selection logic.
+If it's a case, please, consider creating a common ancestor for
+the source and destination memory cgroups and enabling oom_group
+on ancestor layer.
+
+
+IO
+--
+
+The "io" controller regulates the distribution of IO resources.  This
+controller implements both weight based and absolute bandwidth or IOPS
+limit distribution; however, weight based distribution is available
+only if cfq-iosched is in use and neither scheme is available for
+blk-mq devices.
+
+
+IO Interface Files
+~~~~~~~~~~~~~~~~~~
+
+  io.stat
+	A read-only nested-keyed file which exists on non-root
+	cgroups.
+
+	Lines are keyed by $MAJ:$MIN device numbers and not ordered.
+	The following nested keys are defined.
+
+	  ======	===================
+	  rbytes	Bytes read
+	  wbytes	Bytes written
+	  rios		Number of read IOs
+	  wios		Number of write IOs
+	  ======	===================
+
+	An example read output follows:
+
+	  8:16 rbytes=1459200 wbytes=314773504 rios=192 wios=353
+	  8:0 rbytes=90430464 wbytes=299008000 rios=8950 wios=1252
+
+  io.weight
+	A read-write flat-keyed file which exists on non-root cgroups.
+	The default is "default 100".
+
+	The first line is the default weight applied to devices
+	without specific override.  The rest are overrides keyed by
+	$MAJ:$MIN device numbers and not ordered.  The weights are in
+	the range [1, 10000] and specifies the relative amount IO time
+	the cgroup can use in relation to its siblings.
+
+	The default weight can be updated by writing either "default
+	$WEIGHT" or simply "$WEIGHT".  Overrides can be set by writing
+	"$MAJ:$MIN $WEIGHT" and unset by writing "$MAJ:$MIN default".
+
+	An example read output follows::
+
+	  default 100
+	  8:16 200
+	  8:0 50
+
+  io.max
+	A read-write nested-keyed file which exists on non-root
+	cgroups.
+
+	BPS and IOPS based IO limit.  Lines are keyed by $MAJ:$MIN
+	device numbers and not ordered.  The following nested keys are
+	defined.
+
+	  =====		==================================
+	  rbps		Max read bytes per second
+	  wbps		Max write bytes per second
+	  riops		Max read IO operations per second
+	  wiops		Max write IO operations per second
+	  =====		==================================
+
+	When writing, any number of nested key-value pairs can be
+	specified in any order.  "max" can be specified as the value
+	to remove a specific limit.  If the same key is specified
+	multiple times, the outcome is undefined.
+
+	BPS and IOPS are measured in each IO direction and IOs are
+	delayed if limit is reached.  Temporary bursts are allowed.
+
+	Setting read limit at 2M BPS and write at 120 IOPS for 8:16::
+
+	  echo "8:16 rbps=2097152 wiops=120" > io.max
+
+	Reading returns the following::
+
+	  8:16 rbps=2097152 wbps=max riops=max wiops=120
+
+	Write IOPS limit can be removed by writing the following::
+
+	  echo "8:16 wiops=max" > io.max
+
+	Reading now returns the following::
+
+	  8:16 rbps=2097152 wbps=max riops=max wiops=max
+
+
+Writeback
+~~~~~~~~~
+
+Page cache is dirtied through buffered writes and shared mmaps and
+written asynchronously to the backing filesystem by the writeback
+mechanism.  Writeback sits between the memory and IO domains and
+regulates the proportion of dirty memory by balancing dirtying and
+write IOs.
+
+The io controller, in conjunction with the memory controller,
+implements control of page cache writeback IOs.  The memory controller
+defines the memory domain that dirty memory ratio is calculated and
+maintained for and the io controller defines the io domain which
+writes out dirty pages for the memory domain.  Both system-wide and
+per-cgroup dirty memory states are examined and the more restrictive
+of the two is enforced.
+
+cgroup writeback requires explicit support from the underlying
+filesystem.  Currently, cgroup writeback is implemented on ext2, ext4
+and btrfs.  On other filesystems, all writeback IOs are attributed to
+the root cgroup.
+
+There are inherent differences in memory and writeback management
+which affects how cgroup ownership is tracked.  Memory is tracked per
+page while writeback per inode.  For the purpose of writeback, an
+inode is assigned to a cgroup and all IO requests to write dirty pages
+from the inode are attributed to that cgroup.
+
+As cgroup ownership for memory is tracked per page, there can be pages
+which are associated with different cgroups than the one the inode is
+associated with.  These are called foreign pages.  The writeback
+constantly keeps track of foreign pages and, if a particular foreign
+cgroup becomes the majority over a certain period of time, switches
+the ownership of the inode to that cgroup.
+
+While this model is enough for most use cases where a given inode is
+mostly dirtied by a single cgroup even when the main writing cgroup
+changes over time, use cases where multiple cgroups write to a single
+inode simultaneously are not supported well.  In such circumstances, a
+significant portion of IOs are likely to be attributed incorrectly.
+As memory controller assigns page ownership on the first use and
+doesn't update it until the page is released, even if writeback
+strictly follows page ownership, multiple cgroups dirtying overlapping
+areas wouldn't work as expected.  It's recommended to avoid such usage
+patterns.
+
+The sysctl knobs which affect writeback behavior are applied to cgroup
+writeback as follows.
+
+  vm.dirty_background_ratio, vm.dirty_ratio
+	These ratios apply the same to cgroup writeback with the
+	amount of available memory capped by limits imposed by the
+	memory controller and system-wide clean memory.
+
+  vm.dirty_background_bytes, vm.dirty_bytes
+	For cgroup writeback, this is calculated into ratio against
+	total available memory and applied the same way as
+	vm.dirty[_background]_ratio.
+
+
+PID
+---
+
+The process number controller is used to allow a cgroup to stop any
+new tasks from being fork()'d or clone()'d after a specified limit is
+reached.
+
+The number of tasks in a cgroup can be exhausted in ways which other
+controllers cannot prevent, thus warranting its own controller.  For
+example, a fork bomb is likely to exhaust the number of tasks before
+hitting memory restrictions.
+
+Note that PIDs used in this controller refer to TIDs, process IDs as
+used by the kernel.
+
+
+PID Interface Files
+~~~~~~~~~~~~~~~~~~~
+
+  pids.max
+	A read-write single value file which exists on non-root
+	cgroups.  The default is "max".
+
+	Hard limit of number of processes.
+
+  pids.current
+	A read-only single value file which exists on all cgroups.
+
+	The number of processes currently in the cgroup and its
+	descendants.
+
+Organisational operations are not blocked by cgroup policies, so it is
+possible to have pids.current > pids.max.  This can be done by either
+setting the limit to be smaller than pids.current, or attaching enough
+processes to the cgroup such that pids.current is larger than
+pids.max.  However, it is not possible to violate a cgroup PID policy
+through fork() or clone(). These will return -EAGAIN if the creation
+of a new process would cause a cgroup policy to be violated.
+
+
+Device controller
+-----------------
+
+Device controller manages access to device files. It includes both
+creation of new device files (using mknod), and access to the
+existing device files.
+
+Cgroup v2 device controller has no interface files and is implemented
+on top of cgroup BPF. To control access to device files, a user may
+create bpf programs of the BPF_CGROUP_DEVICE type and attach them
+to cgroups. On an attempt to access a device file, corresponding
+BPF programs will be executed, and depending on the return value
+the attempt will succeed or fail with -EPERM.
+
+A BPF_CGROUP_DEVICE program takes a pointer to the bpf_cgroup_dev_ctx
+structure, which describes the device access attempt: access type
+(mknod/read/write) and device (type, major and minor numbers).
+If the program returns 0, the attempt fails with -EPERM, otherwise
+it succeeds.
+
+An example of BPF_CGROUP_DEVICE program may be found in the kernel
+source tree in the tools/testing/selftests/bpf/dev_cgroup.c file.
+
+
+RDMA
+----
+
+The "rdma" controller regulates the distribution and accounting of
+of RDMA resources.
+
+RDMA Interface Files
+~~~~~~~~~~~~~~~~~~~~
+
+  rdma.max
+	A readwrite nested-keyed file that exists for all the cgroups
+	except root that describes current configured resource limit
+	for a RDMA/IB device.
+
+	Lines are keyed by device name and are not ordered.
+	Each line contains space separated resource name and its configured
+	limit that can be distributed.
+
+	The following nested keys are defined.
+
+	  ==========	=============================
+	  hca_handle	Maximum number of HCA Handles
+	  hca_object 	Maximum number of HCA Objects
+	  ==========	=============================
+
+	An example for mlx4 and ocrdma device follows::
+
+	  mlx4_0 hca_handle=2 hca_object=2000
+	  ocrdma1 hca_handle=3 hca_object=max
+
+  rdma.current
+	A read-only file that describes current resource usage.
+	It exists for all the cgroup except root.
+
+	An example for mlx4 and ocrdma device follows::
+
+	  mlx4_0 hca_handle=1 hca_object=20
+	  ocrdma1 hca_handle=1 hca_object=23
+
+
+Misc
+----
+
+perf_event
+~~~~~~~~~~
+
+perf_event controller, if not mounted on a legacy hierarchy, is
+automatically enabled on the v2 hierarchy so that perf events can
+always be filtered by cgroup v2 path.  The controller can still be
+moved to a legacy hierarchy after v2 hierarchy is populated.
+
+
+Non-normative information
+-------------------------
+
+This section contains information that isn't considered to be a part of
+the stable kernel API and so is subject to change.
+
+
+CPU controller root cgroup process behaviour
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When distributing CPU cycles in the root cgroup each thread in this
+cgroup is treated as if it was hosted in a separate child cgroup of the
+root cgroup. This child cgroup weight is dependent on its thread nice
+level.
+
+For details of this mapping see sched_prio_to_weight array in
+kernel/sched/core.c file (values from this array should be scaled
+appropriately so the neutral - nice 0 - value is 100 instead of 1024).
+
+
+IO controller root cgroup process behaviour
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Root cgroup processes are hosted in an implicit leaf child node.
+When distributing IO resources this implicit child node is taken into
+account as if it was a normal child cgroup of the root cgroup with a
+weight value of 200.
+
+
+Namespace
+=========
+
+Basics
+------
+
+cgroup namespace provides a mechanism to virtualize the view of the
+"/proc/$PID/cgroup" file and cgroup mounts.  The CLONE_NEWCGROUP clone
+flag can be used with clone(2) and unshare(2) to create a new cgroup
+namespace.  The process running inside the cgroup namespace will have
+its "/proc/$PID/cgroup" output restricted to cgroupns root.  The
+cgroupns root is the cgroup of the process at the time of creation of
+the cgroup namespace.
+
+Without cgroup namespace, the "/proc/$PID/cgroup" file shows the
+complete path of the cgroup of a process.  In a container setup where
+a set of cgroups and namespaces are intended to isolate processes the
+"/proc/$PID/cgroup" file may leak potential system level information
+to the isolated processes.  For Example::
+
+  # cat /proc/self/cgroup
+  0::/batchjobs/container_id1
+
+The path '/batchjobs/container_id1' can be considered as system-data
+and undesirable to expose to the isolated processes.  cgroup namespace
+can be used to restrict visibility of this path.  For example, before
+creating a cgroup namespace, one would see::
+
+  # ls -l /proc/self/ns/cgroup
+  lrwxrwxrwx 1 root root 0 2014-07-15 10:37 /proc/self/ns/cgroup -> cgroup:[4026531835]
+  # cat /proc/self/cgroup
+  0::/batchjobs/container_id1
+
+After unsharing a new namespace, the view changes::
+
+  # ls -l /proc/self/ns/cgroup
+  lrwxrwxrwx 1 root root 0 2014-07-15 10:35 /proc/self/ns/cgroup -> cgroup:[4026532183]
+  # cat /proc/self/cgroup
+  0::/
+
+When some thread from a multi-threaded process unshares its cgroup
+namespace, the new cgroupns gets applied to the entire process (all
+the threads).  This is natural for the v2 hierarchy; however, for the
+legacy hierarchies, this may be unexpected.
+
+A cgroup namespace is alive as long as there are processes inside or
+mounts pinning it.  When the last usage goes away, the cgroup
+namespace is destroyed.  The cgroupns root and the actual cgroups
+remain.
+
+
+The Root and Views
+------------------
+
+The 'cgroupns root' for a cgroup namespace is the cgroup in which the
+process calling unshare(2) is running.  For example, if a process in
+/batchjobs/container_id1 cgroup calls unshare, cgroup
+/batchjobs/container_id1 becomes the cgroupns root.  For the
+init_cgroup_ns, this is the real root ('/') cgroup.
+
+The cgroupns root cgroup does not change even if the namespace creator
+process later moves to a different cgroup::
+
+  # ~/unshare -c # unshare cgroupns in some cgroup
+  # cat /proc/self/cgroup
+  0::/
+  # mkdir sub_cgrp_1
+  # echo 0 > sub_cgrp_1/cgroup.procs
+  # cat /proc/self/cgroup
+  0::/sub_cgrp_1
+
+Each process gets its namespace-specific view of "/proc/$PID/cgroup"
+
+Processes running inside the cgroup namespace will be able to see
+cgroup paths (in /proc/self/cgroup) only inside their root cgroup.
+From within an unshared cgroupns::
+
+  # sleep 100000 &
+  [1] 7353
+  # echo 7353 > sub_cgrp_1/cgroup.procs
+  # cat /proc/7353/cgroup
+  0::/sub_cgrp_1
+
+From the initial cgroup namespace, the real cgroup path will be
+visible::
+
+  $ cat /proc/7353/cgroup
+  0::/batchjobs/container_id1/sub_cgrp_1
+
+From a sibling cgroup namespace (that is, a namespace rooted at a
+different cgroup), the cgroup path relative to its own cgroup
+namespace root will be shown.  For instance, if PID 7353's cgroup
+namespace root is at '/batchjobs/container_id2', then it will see::
+
+  # cat /proc/7353/cgroup
+  0::/../container_id2/sub_cgrp_1
+
+Note that the relative path always starts with '/' to indicate that
+its relative to the cgroup namespace root of the caller.
+
+
+Migration and setns(2)
+----------------------
+
+Processes inside a cgroup namespace can move into and out of the
+namespace root if they have proper access to external cgroups.  For
+example, from inside a namespace with cgroupns root at
+/batchjobs/container_id1, and assuming that the global hierarchy is
+still accessible inside cgroupns::
+
+  # cat /proc/7353/cgroup
+  0::/sub_cgrp_1
+  # echo 7353 > batchjobs/container_id2/cgroup.procs
+  # cat /proc/7353/cgroup
+  0::/../container_id2
+
+Note that this kind of setup is not encouraged.  A task inside cgroup
+namespace should only be exposed to its own cgroupns hierarchy.
+
+setns(2) to another cgroup namespace is allowed when:
+
+(a) the process has CAP_SYS_ADMIN against its current user namespace
+(b) the process has CAP_SYS_ADMIN against the target cgroup
+    namespace's userns
+
+No implicit cgroup changes happen with attaching to another cgroup
+namespace.  It is expected that the someone moves the attaching
+process under the target cgroup namespace root.
+
+
+Interaction with Other Namespaces
+---------------------------------
+
+Namespace specific cgroup hierarchy can be mounted by a process
+running inside a non-init cgroup namespace::
+
+  # mount -t cgroup2 none $MOUNT_POINT
+
+This will mount the unified cgroup hierarchy with cgroupns root as the
+filesystem root.  The process needs CAP_SYS_ADMIN against its user and
+mount namespaces.
+
+The virtualization of /proc/self/cgroup file combined with restricting
+the view of cgroup hierarchy by namespace-private cgroupfs mount
+provides a properly isolated cgroup view inside the container.
+
+
+Information on Kernel Programming
+=================================
+
+This section contains kernel programming information in the areas
+where interacting with cgroup is necessary.  cgroup core and
+controllers are not covered.
+
+
+Filesystem Support for Writeback
+--------------------------------
+
+A filesystem can support cgroup writeback by updating
+address_space_operations->writepage[s]() to annotate bio's using the
+following two functions.
+
+  wbc_init_bio(@wbc, @bio)
+	Should be called for each bio carrying writeback data and
+	associates the bio with the inode's owner cgroup.  Can be
+	called anytime between bio allocation and submission.
+
+  wbc_account_io(@wbc, @page, @bytes)
+	Should be called for each data segment being written out.
+	While this function doesn't care exactly when it's called
+	during the writeback session, it's the easiest and most
+	natural to call it as data segments are added to a bio.
+
+With writeback bio's annotated, cgroup support can be enabled per
+super_block by setting SB_I_CGROUPWB in ->s_iflags.  This allows for
+selective disabling of cgroup writeback support which is helpful when
+certain filesystem features, e.g. journaled data mode, are
+incompatible.
+
+wbc_init_bio() binds the specified bio to its cgroup.  Depending on
+the configuration, the bio may be executed at a lower priority and if
+the writeback session is holding shared resources, e.g. a journal
+entry, may lead to priority inversion.  There is no one easy solution
+for the problem.  Filesystems can try to work around specific problem
+cases by skipping wbc_init_bio() or using bio_associate_blkcg()
+directly.
+
+
+Deprecated v1 Core Features
+===========================
+
+- Multiple hierarchies including named ones are not supported.
+
+- All v1 mount options are not supported.
+
+- The "tasks" file is removed and "cgroup.procs" is not sorted.
+
+- "cgroup.clone_children" is removed.
+
+- /proc/cgroups is meaningless for v2.  Use "cgroup.controllers" file
+  at the root instead.
+
+
+Issues with v1 and Rationales for v2
+====================================
+
+Multiple Hierarchies
+--------------------
+
+cgroup v1 allowed an arbitrary number of hierarchies and each
+hierarchy could host any number of controllers.  While this seemed to
+provide a high level of flexibility, it wasn't useful in practice.
+
+For example, as there is only one instance of each controller, utility
+type controllers such as freezer which can be useful in all
+hierarchies could only be used in one.  The issue is exacerbated by
+the fact that controllers couldn't be moved to another hierarchy once
+hierarchies were populated.  Another issue was that all controllers
+bound to a hierarchy were forced to have exactly the same view of the
+hierarchy.  It wasn't possible to vary the granularity depending on
+the specific controller.
+
+In practice, these issues heavily limited which controllers could be
+put on the same hierarchy and most configurations resorted to putting
+each controller on its own hierarchy.  Only closely related ones, such
+as the cpu and cpuacct controllers, made sense to be put on the same
+hierarchy.  This often meant that userland ended up managing multiple
+similar hierarchies repeating the same steps on each hierarchy
+whenever a hierarchy management operation was necessary.
+
+Furthermore, support for multiple hierarchies came at a steep cost.
+It greatly complicated cgroup core implementation but more importantly
+the support for multiple hierarchies restricted how cgroup could be
+used in general and what controllers was able to do.
+
+There was no limit on how many hierarchies there might be, which meant
+that a thread's cgroup membership couldn't be described in finite
+length.  The key might contain any number of entries and was unlimited
+in length, which made it highly awkward to manipulate and led to
+addition of controllers which existed only to identify membership,
+which in turn exacerbated the original problem of proliferating number
+of hierarchies.
+
+Also, as a controller couldn't have any expectation regarding the
+topologies of hierarchies other controllers might be on, each
+controller had to assume that all other controllers were attached to
+completely orthogonal hierarchies.  This made it impossible, or at
+least very cumbersome, for controllers to cooperate with each other.
+
+In most use cases, putting controllers on hierarchies which are
+completely orthogonal to each other isn't necessary.  What usually is
+called for is the ability to have differing levels of granularity
+depending on the specific controller.  In other words, hierarchy may
+be collapsed from leaf towards root when viewed from specific
+controllers.  For example, a given configuration might not care about
+how memory is distributed beyond a certain level while still wanting
+to control how CPU cycles are distributed.
+
+
+Thread Granularity
+------------------
+
+cgroup v1 allowed threads of a process to belong to different cgroups.
+This didn't make sense for some controllers and those controllers
+ended up implementing different ways to ignore such situations but
+much more importantly it blurred the line between API exposed to
+individual applications and system management interface.
+
+Generally, in-process knowledge is available only to the process
+itself; thus, unlike service-level organization of processes,
+categorizing threads of a process requires active participation from
+the application which owns the target process.
+
+cgroup v1 had an ambiguously defined delegation model which got abused
+in combination with thread granularity.  cgroups were delegated to
+individual applications so that they can create and manage their own
+sub-hierarchies and control resource distributions along them.  This
+effectively raised cgroup to the status of a syscall-like API exposed
+to lay programs.
+
+First of all, cgroup has a fundamentally inadequate interface to be
+exposed this way.  For a process to access its own knobs, it has to
+extract the path on the target hierarchy from /proc/self/cgroup,
+construct the path by appending the name of the knob to the path, open
+and then read and/or write to it.  This is not only extremely clunky
+and unusual but also inherently racy.  There is no conventional way to
+define transaction across the required steps and nothing can guarantee
+that the process would actually be operating on its own sub-hierarchy.
+
+cgroup controllers implemented a number of knobs which would never be
+accepted as public APIs because they were just adding control knobs to
+system-management pseudo filesystem.  cgroup ended up with interface
+knobs which were not properly abstracted or refined and directly
+revealed kernel internal details.  These knobs got exposed to
+individual applications through the ill-defined delegation mechanism
+effectively abusing cgroup as a shortcut to implementing public APIs
+without going through the required scrutiny.
+
+This was painful for both userland and kernel.  Userland ended up with
+misbehaving and poorly abstracted interfaces and kernel exposing and
+locked into constructs inadvertently.
+
+
+Competition Between Inner Nodes and Threads
+-------------------------------------------
+
+cgroup v1 allowed threads to be in any cgroups which created an
+interesting problem where threads belonging to a parent cgroup and its
+children cgroups competed for resources.  This was nasty as two
+different types of entities competed and there was no obvious way to
+settle it.  Different controllers did different things.
+
+The cpu controller considered threads and cgroups as equivalents and
+mapped nice levels to cgroup weights.  This worked for some cases but
+fell flat when children wanted to be allocated specific ratios of CPU
+cycles and the number of internal threads fluctuated - the ratios
+constantly changed as the number of competing entities fluctuated.
+There also were other issues.  The mapping from nice level to weight
+wasn't obvious or universal, and there were various other knobs which
+simply weren't available for threads.
+
+The io controller implicitly created a hidden leaf node for each
+cgroup to host the threads.  The hidden leaf had its own copies of all
+the knobs with ``leaf_`` prefixed.  While this allowed equivalent
+control over internal threads, it was with serious drawbacks.  It
+always added an extra layer of nesting which wouldn't be necessary
+otherwise, made the interface messy and significantly complicated the
+implementation.
+
+The memory controller didn't have a way to control what happened
+between internal tasks and child cgroups and the behavior was not
+clearly defined.  There were attempts to add ad-hoc behaviors and
+knobs to tailor the behavior to specific workloads which would have
+led to problems extremely difficult to resolve in the long term.
+
+Multiple controllers struggled with internal tasks and came up with
+different ways to deal with it; unfortunately, all the approaches were
+severely flawed and, furthermore, the widely different behaviors
+made cgroup as a whole highly inconsistent.
+
+This clearly is a problem which needs to be addressed from cgroup core
+in a uniform way.
+
+
+Other Interface Issues
+----------------------
+
+cgroup v1 grew without oversight and developed a large number of
+idiosyncrasies and inconsistencies.  One issue on the cgroup core side
+was how an empty cgroup was notified - a userland helper binary was
+forked and executed for each event.  The event delivery wasn't
+recursive or delegatable.  The limitations of the mechanism also led
+to in-kernel event delivery filtering mechanism further complicating
+the interface.
+
+Controller interfaces were problematic too.  An extreme example is
+controllers completely ignoring hierarchical organization and treating
+all cgroups as if they were all located directly under the root
+cgroup.  Some controllers exposed a large amount of inconsistent
+implementation details to userland.
+
+There also was no consistency across controllers.  When a new cgroup
+was created, some controllers defaulted to not imposing extra
+restrictions while others disallowed any resource usage until
+explicitly configured.  Configuration knobs for the same type of
+control used widely differing naming schemes and formats.  Statistics
+and information knobs were named arbitrarily and used different
+formats and units even in the same controller.
+
+cgroup v2 establishes common conventions where appropriate and updates
+controllers so that they expose minimal and consistent interfaces.
+
+
+Controller Issues and Remedies
+------------------------------
+
+Memory
+~~~~~~
+
+The original lower boundary, the soft limit, is defined as a limit
+that is per default unset.  As a result, the set of cgroups that
+global reclaim prefers is opt-in, rather than opt-out.  The costs for
+optimizing these mostly negative lookups are so high that the
+implementation, despite its enormous size, does not even provide the
+basic desirable behavior.  First off, the soft limit has no
+hierarchical meaning.  All configured groups are organized in a global
+rbtree and treated like equal peers, regardless where they are located
+in the hierarchy.  This makes subtree delegation impossible.  Second,
+the soft limit reclaim pass is so aggressive that it not just
+introduces high allocation latencies into the system, but also impacts
+system performance due to overreclaim, to the point where the feature
+becomes self-defeating.
+
+The memory.low boundary on the other hand is a top-down allocated
+reserve.  A cgroup enjoys reclaim protection when it's within its low,
+which makes delegation of subtrees possible.
+
+The original high boundary, the hard limit, is defined as a strict
+limit that can not budge, even if the OOM killer has to be called.
+But this generally goes against the goal of making the most out of the
+available memory.  The memory consumption of workloads varies during
+runtime, and that requires users to overcommit.  But doing that with a
+strict upper limit requires either a fairly accurate prediction of the
+working set size or adding slack to the limit.  Since working set size
+estimation is hard and error prone, and getting it wrong results in
+OOM kills, most users tend to err on the side of a looser limit and
+end up wasting precious resources.
+
+The memory.high boundary on the other hand can be set much more
+conservatively.  When hit, it throttles allocations by forcing them
+into direct reclaim to work off the excess, but it never invokes the
+OOM killer.  As a result, a high boundary that is chosen too
+aggressively will not terminate the processes, but instead it will
+lead to gradual performance degradation.  The user can monitor this
+and make corrections until the minimal memory footprint that still
+gives acceptable performance is found.
+
+In extreme cases, with many concurrent allocations and a complete
+breakdown of reclaim progress within the group, the high boundary can
+be exceeded.  But even then it's mostly better to satisfy the
+allocation from the slack available in other groups or the rest of the
+system than killing the group.  Otherwise, memory.max is there to
+limit this type of spillover and ultimately contain buggy or even
+malicious applications.
+
+Setting the original memory.limit_in_bytes below the current usage was
+subject to a race condition, where concurrent charges could cause the
+limit setting to fail. memory.max on the other hand will first set the
+limit to prevent new charges, and then reclaim and OOM kill until the
+new limit is met - or the task writing to memory.max is killed.
+
+The combined memory+swap accounting and limiting is replaced by real
+control over swap space.
+
+The main argument for a combined memory+swap facility in the original
+cgroup design was that global or parental pressure would always be
+able to swap all anonymous memory of a child group, regardless of the
+child's own (possibly untrusted) configuration.  However, untrusted
+groups can sabotage swapping by other means - such as referencing its
+anonymous memory in a tight loop - and an admin can not assume full
+swappability when overcommitting untrusted jobs.
+
+For trusted jobs, on the other hand, a combined counter is not an
+intuitive userspace interface, and it flies in the face of the idea
+that cgroup controllers should account and limit specific physical
+resources.  Swap space is a resource like all others in the system,
+and that's why unified hierarchy allows distributing it separately.
diff --git a/Documentation/admin-guide/index.rst b/Documentation/admin-guide/index.rst
index 5bb9161dbe6a..48d70af11652 100644
--- a/Documentation/admin-guide/index.rst
+++ b/Documentation/admin-guide/index.rst
@@ -48,6 +48,7 @@ configure specific aspects of kernel behavior to your liking.
    :maxdepth: 1
 
    initrd
+   cgroup-v2
    serial-console
    braille-console
    parport
@@ -60,9 +61,11 @@ configure specific aspects of kernel behavior to your liking.
    mono
    java
    ras
+   bcache
    pm/index
    thunderbolt
    LSM/index
+   mm/index
 
 .. only::  subproject and html
 
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index f2040d46f095..4501e370af1f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -106,11 +106,11 @@
 			use by PCI
 			Format: <irq>,<irq>...
 
-	acpi_mask_gpe=  [HW,ACPI]
+	acpi_mask_gpe=	[HW,ACPI]
 			Due to the existence of _Lxx/_Exx, some GPEs triggered
 			by unsupported hardware/firmware features can result in
-                        GPE floodings that cannot be automatically disabled by
-                        the GPE dispatcher.
+			GPE floodings that cannot be automatically disabled by
+			the GPE dispatcher.
 			This facility can be used to prevent such uncontrolled
 			GPE floodings.
 			Format: <int>
@@ -472,10 +472,10 @@
 			for platform specific values (SB1, Loongson3 and
 			others).
 
-	ccw_timeout_log [S390]
+	ccw_timeout_log	[S390]
 			See Documentation/s390/CommonIO for details.
 
-	cgroup_disable= [KNL] Disable a particular controller
+	cgroup_disable=	[KNL] Disable a particular controller
 			Format: {name of the controller(s) to disable}
 			The effects of cgroup_disable=foo are:
 			- foo isn't auto-mounted if you mount all cgroups in
@@ -518,7 +518,7 @@
 			those clocks in any way. This parameter is useful for
 			debug and development, but should not be needed on a
 			platform with proper driver support.  For more
-			information, see Documentation/clk.txt.
+			information, see Documentation/driver-api/clk.rst.
 
 	clock=		[BUGS=X86-32, HW] gettimeofday clocksource override.
 			[Deprecated]
@@ -587,11 +587,6 @@
 			Sets the size of memory pool for coherent, atomic dma
 			allocations, by default set to 256K.
 
-	code_bytes	[X86] How many bytes of object code to print
-			in an oops report.
-			Range: 0 - 8192
-			Default: 64
-
 	com20020=	[HW,NET] ARCnet - COM20020 chipset
 			Format:
 			<io>[,<irq>[,<nodeID>[,<backplane>[,<ckp>[,<timeout>]]]]]
@@ -641,8 +636,8 @@
 		hvc<n>	Use the hypervisor console device <n>. This is for
 			both Xen and PowerPC hypervisors.
 
-                If the device connected to the port is not a TTY but a braille
-                device, prepend "brl," before the device type, for instance
+		If the device connected to the port is not a TTY but a braille
+		device, prepend "brl," before the device type, for instance
 			console=brl,ttyS0
 		For now, only VisioBraille is supported.
 
@@ -662,7 +657,7 @@
 
 	consoleblank=	[KNL] The console blank (screen saver) timeout in
 			seconds. A value of 0 disables the blank timer.
-                       Defaults to 0.
+			Defaults to 0.
 
 	coredump_filter=
 			[KNL] Change the default value for
@@ -730,7 +725,7 @@
 			or memory reserved is below 4G.
 
 	cryptomgr.notests
-                        [KNL] Disable crypto self-tests
+			[KNL] Disable crypto self-tests
 
 	cs89x0_dma=	[HW,NET]
 			Format: <dma>
@@ -746,7 +741,7 @@
 			Format: <port#>,<type>
 			See also Documentation/input/devices/joystick-parport.rst
 
-	ddebug_query=   [KNL,DYNAMIC_DEBUG] Enable debug messages at early boot
+	ddebug_query=	[KNL,DYNAMIC_DEBUG] Enable debug messages at early boot
 			time. See
 			Documentation/admin-guide/dynamic-debug-howto.rst for
 			details.  Deprecated, see dyndbg.
@@ -833,7 +828,7 @@
 			causing system reset or hang due to sending
 			INIT from AP to BSP.
 
-	disable_ddw     [PPC/PSERIES]
+	disable_ddw	[PPC/PSERIES]
 			Disable Dynamic DMA Window support. Use this if
 			to workaround buggy firmware.
 
@@ -1025,6 +1020,12 @@
 			address. The serial port must already be setup
 			and configured. Options are not yet supported.
 
+		qcom_geni,<addr>
+			Start an early, polled-mode console on a Qualcomm
+			Generic Interface (GENI) based serial port at the
+			specified address. The serial port must already be
+			setup and configured. Options are not yet supported.
+
 	earlyprintk=	[X86,SH,ARM,M68k,S390]
 			earlyprintk=vga
 			earlyprintk=efi
@@ -1188,7 +1189,7 @@
 			parameter will force ia64_sal_cache_flush to call
 			ia64_pal_cache_flush instead of SAL_CACHE_FLUSH.
 
-	forcepae [X86-32]
+	forcepae	[X86-32]
 			Forcefully enable Physical Address Extension (PAE).
 			Many Pentium M systems disable PAE but may have a
 			functionally usable PAE implementation.
@@ -1247,7 +1248,7 @@
 
 	gamma=		[HW,DRM]
 
-	gart_fix_e820=  [X86_64] disable the fix e820 for K8 GART
+	gart_fix_e820=	[X86_64] disable the fix e820 for K8 GART
 			Format: off | on
 			default: on
 
@@ -1341,23 +1342,32 @@
 			x86-64 are 2M (when the CPU supports "pse") and 1G
 			(when the CPU supports the "pdpe1gb" cpuinfo flag).
 
-	hvc_iucv=	[S390] Number of z/VM IUCV hypervisor console (HVC)
-			       terminal devices. Valid values: 0..8
-	hvc_iucv_allow=	[S390] Comma-separated list of z/VM user IDs.
-			       If specified, z/VM IUCV HVC accepts connections
-			       from listed z/VM user IDs only.
+	hung_task_panic=
+			[KNL] Should the hung task detector generate panics.
+			Format: <integer>
 
+			A nonzero value instructs the kernel to panic when a
+			hung task is detected. The default value is controlled
+			by the CONFIG_BOOTPARAM_HUNG_TASK_PANIC build-time
+			option. The value selected by this boot parameter can
+			be changed later by the kernel.hung_task_panic sysctl.
+
+	hvc_iucv=	[S390]	Number of z/VM IUCV hypervisor console (HVC)
+				terminal devices. Valid values: 0..8
+	hvc_iucv_allow=	[S390]	Comma-separated list of z/VM user IDs.
+				If specified, z/VM IUCV HVC accepts connections
+				from listed z/VM user IDs only.
 	keep_bootcon	[KNL]
 			Do not unregister boot console at start. This is only
 			useful for debugging when something happens in the window
 			between unregistering the boot console and initializing
 			the real console.
 
-	i2c_bus=	[HW] Override the default board specific I2C bus speed
-			     or register an additional I2C bus that is not
-			     registered from board initialization code.
-			     Format:
-			     <bus_id>,<clkrate>
+	i2c_bus=	[HW]	Override the default board specific I2C bus speed
+				or register an additional I2C bus that is not
+				registered from board initialization code.
+				Format:
+				<bus_id>,<clkrate>
 
 	i8042.debug	[HW] Toggle i8042 debug mode
 	i8042.unmask_kbd_data
@@ -1386,7 +1396,7 @@
 			Default: only on s2r transitions on x86; most other
 			architectures force reset to be always executed
 	i8042.unlock	[HW] Unlock (ignore) the keylock
-	i8042.kbdreset  [HW] Reset device connected to KBD port
+	i8042.kbdreset	[HW] Reset device connected to KBD port
 
 	i810=		[HW,DRM]
 
@@ -1548,13 +1558,13 @@
 			programs exec'd, files mmap'd for exec, and all files
 			opened for read by uid=0.
 
-	ima_template=   [IMA]
+	ima_template=	[IMA]
 			Select one of defined IMA measurements template formats.
 			Formats: { "ima" | "ima-ng" | "ima-sig" }
 			Default: "ima-ng"
 
 	ima_template_fmt=
-	                [IMA] Define a custom template format.
+			[IMA] Define a custom template format.
 			Format: { "field1|...|fieldN" }
 
 	ima.ahash_minsize= [IMA] Minimum file size for asynchronous hash usage
@@ -1597,7 +1607,7 @@
 	inport.irq=	[HW] Inport (ATI XL and Microsoft) busmouse driver
 			Format: <irq>
 
-	int_pln_enable  [x86] Enable power limit notification interrupt
+	int_pln_enable	[x86] Enable power limit notification interrupt
 
 	integrity_audit=[IMA]
 			Format: { "0" | "1" }
@@ -1650,39 +1660,39 @@
 			0	disables intel_idle and fall back on acpi_idle.
 			1 to 9	specify maximum depth of C-state.
 
-	intel_pstate=  [X86]
-		       disable
-		         Do not enable intel_pstate as the default
-		         scaling driver for the supported processors
-		       passive
-			 Use intel_pstate as a scaling driver, but configure it
-			 to work with generic cpufreq governors (instead of
-			 enabling its internal governor).  This mode cannot be
-			 used along with the hardware-managed P-states (HWP)
-			 feature.
-		       force
-			 Enable intel_pstate on systems that prohibit it by default
-			 in favor of acpi-cpufreq. Forcing the intel_pstate driver
-			 instead of acpi-cpufreq may disable platform features, such
-			 as thermal controls and power capping, that rely on ACPI
-			 P-States information being indicated to OSPM and therefore
-			 should be used with caution. This option does not work with
-			 processors that aren't supported by the intel_pstate driver
-			 or on platforms that use pcc-cpufreq instead of acpi-cpufreq.
-		       no_hwp
-		         Do not enable hardware P state control (HWP)
-			 if available.
-		hwp_only
-			Only load intel_pstate on systems which support
-			hardware P state control (HWP) if available.
-		support_acpi_ppc
-			Enforce ACPI _PPC performance limits. If the Fixed ACPI
-			Description Table, specifies preferred power management
-			profile as "Enterprise Server" or "Performance Server",
-			then this feature is turned on by default.
-		per_cpu_perf_limits
-			Allow per-logical-CPU P-State performance control limits using
-			cpufreq sysfs interface
+	intel_pstate=	[X86]
+			disable
+			  Do not enable intel_pstate as the default
+			  scaling driver for the supported processors
+			passive
+			  Use intel_pstate as a scaling driver, but configure it
+			  to work with generic cpufreq governors (instead of
+			  enabling its internal governor).  This mode cannot be
+			  used along with the hardware-managed P-states (HWP)
+			  feature.
+			force
+			  Enable intel_pstate on systems that prohibit it by default
+			  in favor of acpi-cpufreq. Forcing the intel_pstate driver
+			  instead of acpi-cpufreq may disable platform features, such
+			  as thermal controls and power capping, that rely on ACPI
+			  P-States information being indicated to OSPM and therefore
+			  should be used with caution. This option does not work with
+			  processors that aren't supported by the intel_pstate driver
+			  or on platforms that use pcc-cpufreq instead of acpi-cpufreq.
+			no_hwp
+			  Do not enable hardware P state control (HWP)
+			  if available.
+			hwp_only
+			  Only load intel_pstate on systems which support
+			  hardware P state control (HWP) if available.
+			support_acpi_ppc
+			  Enforce ACPI _PPC performance limits. If the Fixed ACPI
+			  Description Table, specifies preferred power management
+			  profile as "Enterprise Server" or "Performance Server",
+			  then this feature is turned on by default.
+			per_cpu_perf_limits
+			  Allow per-logical-CPU P-State performance control limits using
+			  cpufreq sysfs interface
 
 	intremap=	[X86-64, Intel-IOMMU]
 			on	enable Interrupt Remapping (default)
@@ -1705,7 +1715,6 @@
 		nopanic
 		merge
 		nomerge
-		forcesac
 		soft
 		pt		[x86, IA-64]
 		nobypass	[PPC/POWERNV]
@@ -2027,7 +2036,7 @@
 			* [no]ncqtrim: Turn off queued DSM TRIM.
 
 			* nohrst, nosrst, norst: suppress hard, soft
-                          and both resets.
+			  and both resets.
 
 			* rstonce: only attempt one reset during
 			  hot-unplug link recovery
@@ -2215,7 +2224,7 @@
 			[KNL,SH] Allow user to override the default size for
 			per-device physically contiguous DMA buffers.
 
-        memhp_default_state=online/offline
+	memhp_default_state=online/offline
 			[KNL] Set the initial state for the memory hotplug
 			onlining policy. If not specified, the default value is
 			set according to the
@@ -2765,7 +2774,7 @@
 			[X86,PV_OPS] Disable paravirtualized VMware scheduler
 			clock and use the default one.
 
-	no-steal-acc    [X86,KVM] Disable paravirtualized steal time accounting.
+	no-steal-acc	[X86,KVM] Disable paravirtualized steal time accounting.
 			steal time is computed, but won't influence scheduler
 			behaviour
 
@@ -2826,7 +2835,7 @@
 	notsc		[BUGS=X86-32] Disable Time Stamp Counter
 
 	nowatchdog	[KNL] Disable both lockup detectors, i.e.
-                        soft-lockup and NMI watchdog (hard-lockup).
+			soft-lockup and NMI watchdog (hard-lockup).
 
 	nowb		[ARM]
 
@@ -2846,7 +2855,7 @@
 			If the dependencies are under your control, you can
 			turn on cpu0_hotplug.
 
-	nps_mtm_hs_ctr= [KNL,ARC]
+	nps_mtm_hs_ctr=	[KNL,ARC]
 			This parameter sets the maximum duration, in
 			cycles, each HW thread of the CTOP can run
 			without interruptions, before HW switches it.
@@ -2987,7 +2996,7 @@
 
 	pci=option[,option...]	[PCI] various PCI subsystem options:
 		earlydump	[X86] dump PCI config space before the kernel
-			        changes anything
+				changes anything
 		off		[X86] don't probe for the PCI bus
 		bios		[X86-32] force use of PCI BIOS, don't access
 				the hardware directly. Use this if your machine
@@ -3075,7 +3084,7 @@
 				is enabled by default.  If you need to use this,
 				please report a bug.
 		nocrs		[X86] Ignore PCI host bridge windows from ACPI.
-			        If you need to use this, please report a bug.
+				If you need to use this, please report a bug.
 		routeirq	Do IRQ routing for all PCI devices.
 				This is normally done in pci_enable_device(),
 				so this option is a temporary workaround
@@ -3150,6 +3159,8 @@
 				on: Turn realloc on
 		realloc		same as realloc=on
 		noari		do not use PCIe ARI.
+		noats		[PCIE, Intel-IOMMU, AMD-IOMMU]
+				do not use PCIe ATS (and IOMMU device IOTLB).
 		pcie_scan_all	Scan all possible PCIe devices.  Otherwise we
 				only look for one device below a PCIe downstream
 				port.
@@ -3621,8 +3632,8 @@
 			Set time (s) after boot for CPU-hotplug testing.
 
 	rcutorture.onoff_interval= [KNL]
-			Set time (s) between CPU-hotplug operations, or
-			zero to disable CPU-hotplug testing.
+			Set time (jiffies) between CPU-hotplug operations,
+			or zero to disable CPU-hotplug testing.
 
 	rcutorture.shuffle_interval= [KNL]
 			Set task-shuffle interval (s).  Shuffling tasks
@@ -3918,7 +3929,7 @@
 			cache (risks via metadata attacks are mostly
 			unchanged). Debug options disable merging on their
 			own.
-			For more information see Documentation/vm/slub.txt.
+			For more information see Documentation/vm/slub.rst.
 
 	slab_max_order=	[MM, SLAB]
 			Determines the maximum allowed order for slabs.
@@ -3932,7 +3943,7 @@
 			slub_debug can create guard zones around objects and
 			may poison objects when not in use. Also tracks the
 			last alloc / free. For more information see
-			Documentation/vm/slub.txt.
+			Documentation/vm/slub.rst.
 
 	slub_memcg_sysfs=	[MM, SLUB]
 			Determines whether to enable sysfs directories for
@@ -3946,7 +3957,7 @@
 			Determines the maximum allowed order for slabs.
 			A high setting may cause OOMs due to memory
 			fragmentation. For more information see
-			Documentation/vm/slub.txt.
+			Documentation/vm/slub.rst.
 
 	slub_min_objects=	[MM, SLUB]
 			The minimum number of objects per slab. SLUB will
@@ -3955,12 +3966,12 @@
 			the number of objects indicated. The higher the number
 			of objects the smaller the overhead of tracking slabs
 			and the less frequently locks need to be acquired.
-			For more information see Documentation/vm/slub.txt.
+			For more information see Documentation/vm/slub.rst.
 
 	slub_min_order=	[MM, SLUB]
 			Determines the minimum page order for slabs. Must be
 			lower than slub_max_order.
-			For more information see Documentation/vm/slub.txt.
+			For more information see Documentation/vm/slub.rst.
 
 	slub_nomerge	[MM, SLUB]
 			Same with slab_nomerge. This is supported for legacy.
@@ -4358,7 +4369,8 @@
 			Format: [always|madvise|never]
 			Can be used to control the default behavior of the system
 			with respect to transparent hugepages.
-			See Documentation/vm/transhuge.txt for more details.
+			See Documentation/admin-guide/mm/transhuge.rst
+			for more details.
 
 	tsc=		Disable clocksource stability checks for TSC.
 			Format: <string>
@@ -4436,7 +4448,7 @@
 
 	usbcore.initial_descriptor_timeout=
 			[USB] Specifies timeout for the initial 64-byte
-                        USB_REQ_GET_DESCRIPTOR request in milliseconds
+			USB_REQ_GET_DESCRIPTOR request in milliseconds
 			(default 5000 = 5.0 seconds).
 
 	usbcore.nousb	[USB] Disable the USB subsystem
diff --git a/Documentation/admin-guide/mm/concepts.rst b/Documentation/admin-guide/mm/concepts.rst
new file mode 100644
index 000000000000..291699c810d4
--- /dev/null
+++ b/Documentation/admin-guide/mm/concepts.rst
@@ -0,0 +1,222 @@
+.. _mm_concepts:
+
+=================
+Concepts overview
+=================
+
+The memory management in Linux is complex system that evolved over the
+years and included more and more functionality to support variety of
+systems from MMU-less microcontrollers to supercomputers. The memory
+management for systems without MMU is called ``nommu`` and it
+definitely deserves a dedicated document, which hopefully will be
+eventually written. Yet, although some of the concepts are the same,
+here we assume that MMU is available and CPU can translate a virtual
+address to a physical address.
+
+.. contents:: :local:
+
+Virtual Memory Primer
+=====================
+
+The physical memory in a computer system is a limited resource and
+even for systems that support memory hotplug there is a hard limit on
+the amount of memory that can be installed. The physical memory is not
+necessary contiguous, it might be accessible as a set of distinct
+address ranges. Besides, different CPU architectures, and even
+different implementations of the same architecture have different view
+how these address ranges defined.
+
+All this makes dealing directly with physical memory quite complex and
+to avoid this complexity a concept of virtual memory was developed.
+
+The virtual memory abstracts the details of physical memory from the
+application software, allows to keep only needed information in the
+physical memory (demand paging) and provides a mechanism for the
+protection and controlled sharing of data between processes.
+
+With virtual memory, each and every memory access uses a virtual
+address. When the CPU decodes the an instruction that reads (or
+writes) from (or to) the system memory, it translates the `virtual`
+address encoded in that instruction to a `physical` address that the
+memory controller can understand.
+
+The physical system memory is divided into page frames, or pages. The
+size of each page is architecture specific. Some architectures allow
+selection of the page size from several supported values; this
+selection is performed at the kernel build time by setting an
+appropriate kernel configuration option.
+
+Each physical memory page can be mapped as one or more virtual
+pages. These mappings are described by page tables that allow
+translation from virtual address used by programs to real address in
+the physical memory. The page tables organized hierarchically.
+
+The tables at the lowest level of the hierarchy contain physical
+addresses of actual pages used by the software. The tables at higher
+levels contain physical addresses of the pages belonging to the lower
+levels. The pointer to the top level page table resides in a
+register. When the CPU performs the address translation, it uses this
+register to access the top level page table. The high bits of the
+virtual address are used to index an entry in the top level page
+table. That entry is then used to access the next level in the
+hierarchy with the next bits of the virtual address as the index to
+that level page table. The lowest bits in the virtual address define
+the offset inside the actual page.
+
+Huge Pages
+==========
+
+The address translation requires several memory accesses and memory
+accesses are slow relatively to CPU speed. To avoid spending precious
+processor cycles on the address translation, CPUs maintain a cache of
+such translations called Translation Lookaside Buffer (or
+TLB). Usually TLB is pretty scarce resource and applications with
+large memory working set will experience performance hit because of
+TLB misses.
+
+Many modern CPU architectures allow mapping of the memory pages
+directly by the higher levels in the page table. For instance, on x86,
+it is possible to map 2M and even 1G pages using entries in the second
+and the third level page tables. In Linux such pages are called
+`huge`. Usage of huge pages significantly reduces pressure on TLB,
+improves TLB hit-rate and thus improves overall system performance.
+
+There are two mechanisms in Linux that enable mapping of the physical
+memory with the huge pages. The first one is `HugeTLB filesystem`, or
+hugetlbfs. It is a pseudo filesystem that uses RAM as its backing
+store. For the files created in this filesystem the data resides in
+the memory and mapped using huge pages. The hugetlbfs is described at
+:ref:`Documentation/admin-guide/mm/hugetlbpage.rst <hugetlbpage>`.
+
+Another, more recent, mechanism that enables use of the huge pages is
+called `Transparent HugePages`, or THP. Unlike the hugetlbfs that
+requires users and/or system administrators to configure what parts of
+the system memory should and can be mapped by the huge pages, THP
+manages such mappings transparently to the user and hence the
+name. See
+:ref:`Documentation/admin-guide/mm/transhuge.rst <admin_guide_transhuge>`
+for more details about THP.
+
+Zones
+=====
+
+Often hardware poses restrictions on how different physical memory
+ranges can be accessed. In some cases, devices cannot perform DMA to
+all the addressable memory. In other cases, the size of the physical
+memory exceeds the maximal addressable size of virtual memory and
+special actions are required to access portions of the memory. Linux
+groups memory pages into `zones` according to their possible
+usage. For example, ZONE_DMA will contain memory that can be used by
+devices for DMA, ZONE_HIGHMEM will contain memory that is not
+permanently mapped into kernel's address space and ZONE_NORMAL will
+contain normally addressed pages.
+
+The actual layout of the memory zones is hardware dependent as not all
+architectures define all zones, and requirements for DMA are different
+for different platforms.
+
+Nodes
+=====
+
+Many multi-processor machines are NUMA - Non-Uniform Memory Access -
+systems. In such systems the memory is arranged into banks that have
+different access latency depending on the "distance" from the
+processor. Each bank is referred as `node` and for each node Linux
+constructs an independent memory management subsystem. A node has it's
+own set of zones, lists of free and used pages and various statistics
+counters. You can find more details about NUMA in
+:ref:`Documentation/vm/numa.rst <numa>` and in
+:ref:`Documentation/admin-guide/mm/numa_memory_policy.rst <numa_memory_policy>`.
+
+Page cache
+==========
+
+The physical memory is volatile and the common case for getting data
+into the memory is to read it from files. Whenever a file is read, the
+data is put into the `page cache` to avoid expensive disk access on
+the subsequent reads. Similarly, when one writes to a file, the data
+is placed in the page cache and eventually gets into the backing
+storage device. The written pages are marked as `dirty` and when Linux
+decides to reuse them for other purposes, it makes sure to synchronize
+the file contents on the device with the updated data.
+
+Anonymous Memory
+================
+
+The `anonymous memory` or `anonymous mappings` represent memory that
+is not backed by a filesystem. Such mappings are implicitly created
+for program's stack and heap or by explicit calls to mmap(2) system
+call. Usually, the anonymous mappings only define virtual memory areas
+that the program is allowed to access. The read accesses will result
+in creation of a page table entry that references a special physical
+page filled with zeroes. When the program performs a write, regular
+physical page will be allocated to hold the written data. The page
+will be marked dirty and if the kernel will decide to repurpose it,
+the dirty page will be swapped out.
+
+Reclaim
+=======
+
+Throughout the system lifetime, a physical page can be used for storing
+different types of data. It can be kernel internal data structures,
+DMA'able buffers for device drivers use, data read from a filesystem,
+memory allocated by user space processes etc.
+
+Depending on the page usage it is treated differently by the Linux
+memory management. The pages that can be freed at any time, either
+because they cache the data available elsewhere, for instance, on a
+hard disk, or because they can be swapped out, again, to the hard
+disk, are called `reclaimable`. The most notable categories of the
+reclaimable pages are page cache and anonymous memory.
+
+In most cases, the pages holding internal kernel data and used as DMA
+buffers cannot be repurposed, and they remain pinned until freed by
+their user. Such pages are called `unreclaimable`. However, in certain
+circumstances, even pages occupied with kernel data structures can be
+reclaimed. For instance, in-memory caches of filesystem metadata can
+be re-read from the storage device and therefore it is possible to
+discard them from the main memory when system is under memory
+pressure.
+
+The process of freeing the reclaimable physical memory pages and
+repurposing them is called (surprise!) `reclaim`. Linux can reclaim
+pages either asynchronously or synchronously, depending on the state
+of the system. When system is not loaded, most of the memory is free
+and allocation request will be satisfied immediately from the free
+pages supply. As the load increases, the amount of the free pages goes
+down and when it reaches a certain threshold (high watermark), an
+allocation request will awaken the ``kswapd`` daemon. It will
+asynchronously scan memory pages and either just free them if the data
+they contain is available elsewhere, or evict to the backing storage
+device (remember those dirty pages?). As memory usage increases even
+more and reaches another threshold - min watermark - an allocation
+will trigger the `direct reclaim`. In this case allocation is stalled
+until enough memory pages are reclaimed to satisfy the request.
+
+Compaction
+==========
+
+As the system runs, tasks allocate and free the memory and it becomes
+fragmented. Although with virtual memory it is possible to present
+scattered physical pages as virtually contiguous range, sometimes it is
+necessary to allocate large physically contiguous memory areas. Such
+need may arise, for instance, when a device driver requires large
+buffer for DMA, or when THP allocates a huge page. Memory `compaction`
+addresses the fragmentation issue. This mechanism moves occupied pages
+from the lower part of a memory zone to free pages in the upper part
+of the zone. When a compaction scan is finished free pages are grouped
+together at the beginning of the zone and allocations of large
+physically contiguous areas become possible.
+
+Like reclaim, the compaction may happen asynchronously in ``kcompactd``
+daemon or synchronously as a result of memory allocation request.
+
+OOM killer
+==========
+
+It may happen, that on a loaded machine memory will be exhausted. When
+the kernel detects that the system runs out of memory (OOM) it invokes
+`OOM killer`. Its mission is simple: all it has to do is to select a
+task to sacrifice for the sake of the overall system health. The
+selected task is killed in a hope that after it exits enough memory
+will be freed to continue normal operation.
diff --git a/Documentation/admin-guide/mm/hugetlbpage.rst b/Documentation/admin-guide/mm/hugetlbpage.rst
new file mode 100644
index 000000000000..1cc0bc78d10e
--- /dev/null
+++ b/Documentation/admin-guide/mm/hugetlbpage.rst
@@ -0,0 +1,382 @@
+.. _hugetlbpage:
+
+=============
+HugeTLB Pages
+=============
+
+Overview
+========
+
+The intent of this file is to give a brief summary of hugetlbpage support in
+the Linux kernel.  This support is built on top of multiple page size support
+that is provided by most modern architectures.  For example, x86 CPUs normally
+support 4K and 2M (1G if architecturally supported) page sizes, ia64
+architecture supports multiple page sizes 4K, 8K, 64K, 256K, 1M, 4M, 16M,
+256M and ppc64 supports 4K and 16M.  A TLB is a cache of virtual-to-physical
+translations.  Typically this is a very scarce resource on processor.
+Operating systems try to make best use of limited number of TLB resources.
+This optimization is more critical now as bigger and bigger physical memories
+(several GBs) are more readily available.
+
+Users can use the huge page support in Linux kernel by either using the mmap
+system call or standard SYSV shared memory system calls (shmget, shmat).
+
+First the Linux kernel needs to be built with the CONFIG_HUGETLBFS
+(present under "File systems") and CONFIG_HUGETLB_PAGE (selected
+automatically when CONFIG_HUGETLBFS is selected) configuration
+options.
+
+The ``/proc/meminfo`` file provides information about the total number of
+persistent hugetlb pages in the kernel's huge page pool.  It also displays
+default huge page size and information about the number of free, reserved
+and surplus huge pages in the pool of huge pages of default size.
+The huge page size is needed for generating the proper alignment and
+size of the arguments to system calls that map huge page regions.
+
+The output of ``cat /proc/meminfo`` will include lines like::
+
+	HugePages_Total: uuu
+	HugePages_Free:  vvv
+	HugePages_Rsvd:  www
+	HugePages_Surp:  xxx
+	Hugepagesize:    yyy kB
+	Hugetlb:         zzz kB
+
+where:
+
+HugePages_Total
+	is the size of the pool of huge pages.
+HugePages_Free
+	is the number of huge pages in the pool that are not yet
+        allocated.
+HugePages_Rsvd
+	is short for "reserved," and is the number of huge pages for
+        which a commitment to allocate from the pool has been made,
+        but no allocation has yet been made.  Reserved huge pages
+        guarantee that an application will be able to allocate a
+        huge page from the pool of huge pages at fault time.
+HugePages_Surp
+	is short for "surplus," and is the number of huge pages in
+        the pool above the value in ``/proc/sys/vm/nr_hugepages``. The
+        maximum number of surplus huge pages is controlled by
+        ``/proc/sys/vm/nr_overcommit_hugepages``.
+Hugepagesize
+	is the default hugepage size (in Kb).
+Hugetlb
+        is the total amount of memory (in kB), consumed by huge
+        pages of all sizes.
+        If huge pages of different sizes are in use, this number
+        will exceed HugePages_Total \* Hugepagesize. To get more
+        detailed information, please, refer to
+        ``/sys/kernel/mm/hugepages`` (described below).
+
+
+``/proc/filesystems`` should also show a filesystem of type "hugetlbfs"
+configured in the kernel.
+
+``/proc/sys/vm/nr_hugepages`` indicates the current number of "persistent" huge
+pages in the kernel's huge page pool.  "Persistent" huge pages will be
+returned to the huge page pool when freed by a task.  A user with root
+privileges can dynamically allocate more or free some persistent huge pages
+by increasing or decreasing the value of ``nr_hugepages``.
+
+Pages that are used as huge pages are reserved inside the kernel and cannot
+be used for other purposes.  Huge pages cannot be swapped out under
+memory pressure.
+
+Once a number of huge pages have been pre-allocated to the kernel huge page
+pool, a user with appropriate privilege can use either the mmap system call
+or shared memory system calls to use the huge pages.  See the discussion of
+:ref:`Using Huge Pages <using_huge_pages>`, below.
+
+The administrator can allocate persistent huge pages on the kernel boot
+command line by specifying the "hugepages=N" parameter, where 'N' = the
+number of huge pages requested.  This is the most reliable method of
+allocating huge pages as memory has not yet become fragmented.
+
+Some platforms support multiple huge page sizes.  To allocate huge pages
+of a specific size, one must precede the huge pages boot command parameters
+with a huge page size selection parameter "hugepagesz=<size>".  <size> must
+be specified in bytes with optional scale suffix [kKmMgG].  The default huge
+page size may be selected with the "default_hugepagesz=<size>" boot parameter.
+
+When multiple huge page sizes are supported, ``/proc/sys/vm/nr_hugepages``
+indicates the current number of pre-allocated huge pages of the default size.
+Thus, one can use the following command to dynamically allocate/deallocate
+default sized persistent huge pages::
+
+	echo 20 > /proc/sys/vm/nr_hugepages
+
+This command will try to adjust the number of default sized huge pages in the
+huge page pool to 20, allocating or freeing huge pages, as required.
+
+On a NUMA platform, the kernel will attempt to distribute the huge page pool
+over all the set of allowed nodes specified by the NUMA memory policy of the
+task that modifies ``nr_hugepages``. The default for the allowed nodes--when the
+task has default memory policy--is all on-line nodes with memory.  Allowed
+nodes with insufficient available, contiguous memory for a huge page will be
+silently skipped when allocating persistent huge pages.  See the
+:ref:`discussion below <mem_policy_and_hp_alloc>`
+of the interaction of task memory policy, cpusets and per node attributes
+with the allocation and freeing of persistent huge pages.
+
+The success or failure of huge page allocation depends on the amount of
+physically contiguous memory that is present in system at the time of the
+allocation attempt.  If the kernel is unable to allocate huge pages from
+some nodes in a NUMA system, it will attempt to make up the difference by
+allocating extra pages on other nodes with sufficient available contiguous
+memory, if any.
+
+System administrators may want to put this command in one of the local rc
+init files.  This will enable the kernel to allocate huge pages early in
+the boot process when the possibility of getting physical contiguous pages
+is still very high.  Administrators can verify the number of huge pages
+actually allocated by checking the sysctl or meminfo.  To check the per node
+distribution of huge pages in a NUMA system, use::
+
+	cat /sys/devices/system/node/node*/meminfo | fgrep Huge
+
+``/proc/sys/vm/nr_overcommit_hugepages`` specifies how large the pool of
+huge pages can grow, if more huge pages than ``/proc/sys/vm/nr_hugepages`` are
+requested by applications.  Writing any non-zero value into this file
+indicates that the hugetlb subsystem is allowed to try to obtain that
+number of "surplus" huge pages from the kernel's normal page pool, when the
+persistent huge page pool is exhausted. As these surplus huge pages become
+unused, they are freed back to the kernel's normal page pool.
+
+When increasing the huge page pool size via ``nr_hugepages``, any existing
+surplus pages will first be promoted to persistent huge pages.  Then, additional
+huge pages will be allocated, if necessary and if possible, to fulfill
+the new persistent huge page pool size.
+
+The administrator may shrink the pool of persistent huge pages for
+the default huge page size by setting the ``nr_hugepages`` sysctl to a
+smaller value.  The kernel will attempt to balance the freeing of huge pages
+across all nodes in the memory policy of the task modifying ``nr_hugepages``.
+Any free huge pages on the selected nodes will be freed back to the kernel's
+normal page pool.
+
+Caveat: Shrinking the persistent huge page pool via ``nr_hugepages`` such that
+it becomes less than the number of huge pages in use will convert the balance
+of the in-use huge pages to surplus huge pages.  This will occur even if
+the number of surplus pages would exceed the overcommit value.  As long as
+this condition holds--that is, until ``nr_hugepages+nr_overcommit_hugepages`` is
+increased sufficiently, or the surplus huge pages go out of use and are freed--
+no more surplus huge pages will be allowed to be allocated.
+
+With support for multiple huge page pools at run-time available, much of
+the huge page userspace interface in ``/proc/sys/vm`` has been duplicated in
+sysfs.
+The ``/proc`` interfaces discussed above have been retained for backwards
+compatibility. The root huge page control directory in sysfs is::
+
+	/sys/kernel/mm/hugepages
+
+For each huge page size supported by the running kernel, a subdirectory
+will exist, of the form::
+
+	hugepages-${size}kB
+
+Inside each of these directories, the same set of files will exist::
+
+	nr_hugepages
+	nr_hugepages_mempolicy
+	nr_overcommit_hugepages
+	free_hugepages
+	resv_hugepages
+	surplus_hugepages
+
+which function as described above for the default huge page-sized case.
+
+.. _mem_policy_and_hp_alloc:
+
+Interaction of Task Memory Policy with Huge Page Allocation/Freeing
+===================================================================
+
+Whether huge pages are allocated and freed via the ``/proc`` interface or
+the ``/sysfs`` interface using the ``nr_hugepages_mempolicy`` attribute, the
+NUMA nodes from which huge pages are allocated or freed are controlled by the
+NUMA memory policy of the task that modifies the ``nr_hugepages_mempolicy``
+sysctl or attribute.  When the ``nr_hugepages`` attribute is used, mempolicy
+is ignored.
+
+The recommended method to allocate or free huge pages to/from the kernel
+huge page pool, using the ``nr_hugepages`` example above, is::
+
+    numactl --interleave <node-list> echo 20 \
+				>/proc/sys/vm/nr_hugepages_mempolicy
+
+or, more succinctly::
+
+    numactl -m <node-list> echo 20 >/proc/sys/vm/nr_hugepages_mempolicy
+
+This will allocate or free ``abs(20 - nr_hugepages)`` to or from the nodes
+specified in <node-list>, depending on whether number of persistent huge pages
+is initially less than or greater than 20, respectively.  No huge pages will be
+allocated nor freed on any node not included in the specified <node-list>.
+
+When adjusting the persistent hugepage count via ``nr_hugepages_mempolicy``, any
+memory policy mode--bind, preferred, local or interleave--may be used.  The
+resulting effect on persistent huge page allocation is as follows:
+
+#. Regardless of mempolicy mode [see
+   :ref:`Documentation/admin-guide/mm/numa_memory_policy.rst <numa_memory_policy>`],
+   persistent huge pages will be distributed across the node or nodes
+   specified in the mempolicy as if "interleave" had been specified.
+   However, if a node in the policy does not contain sufficient contiguous
+   memory for a huge page, the allocation will not "fallback" to the nearest
+   neighbor node with sufficient contiguous memory.  To do this would cause
+   undesirable imbalance in the distribution of the huge page pool, or
+   possibly, allocation of persistent huge pages on nodes not allowed by
+   the task's memory policy.
+
+#. One or more nodes may be specified with the bind or interleave policy.
+   If more than one node is specified with the preferred policy, only the
+   lowest numeric id will be used.  Local policy will select the node where
+   the task is running at the time the nodes_allowed mask is constructed.
+   For local policy to be deterministic, the task must be bound to a cpu or
+   cpus in a single node.  Otherwise, the task could be migrated to some
+   other node at any time after launch and the resulting node will be
+   indeterminate.  Thus, local policy is not very useful for this purpose.
+   Any of the other mempolicy modes may be used to specify a single node.
+
+#. The nodes allowed mask will be derived from any non-default task mempolicy,
+   whether this policy was set explicitly by the task itself or one of its
+   ancestors, such as numactl.  This means that if the task is invoked from a
+   shell with non-default policy, that policy will be used.  One can specify a
+   node list of "all" with numactl --interleave or --membind [-m] to achieve
+   interleaving over all nodes in the system or cpuset.
+
+#. Any task mempolicy specified--e.g., using numactl--will be constrained by
+   the resource limits of any cpuset in which the task runs.  Thus, there will
+   be no way for a task with non-default policy running in a cpuset with a
+   subset of the system nodes to allocate huge pages outside the cpuset
+   without first moving to a cpuset that contains all of the desired nodes.
+
+#. Boot-time huge page allocation attempts to distribute the requested number
+   of huge pages over all on-lines nodes with memory.
+
+Per Node Hugepages Attributes
+=============================
+
+A subset of the contents of the root huge page control directory in sysfs,
+described above, will be replicated under each the system device of each
+NUMA node with memory in::
+
+	/sys/devices/system/node/node[0-9]*/hugepages/
+
+Under this directory, the subdirectory for each supported huge page size
+contains the following attribute files::
+
+	nr_hugepages
+	free_hugepages
+	surplus_hugepages
+
+The free\_' and surplus\_' attribute files are read-only.  They return the number
+of free and surplus [overcommitted] huge pages, respectively, on the parent
+node.
+
+The ``nr_hugepages`` attribute returns the total number of huge pages on the
+specified node.  When this attribute is written, the number of persistent huge
+pages on the parent node will be adjusted to the specified value, if sufficient
+resources exist, regardless of the task's mempolicy or cpuset constraints.
+
+Note that the number of overcommit and reserve pages remain global quantities,
+as we don't know until fault time, when the faulting task's mempolicy is
+applied, from which node the huge page allocation will be attempted.
+
+.. _using_huge_pages:
+
+Using Huge Pages
+================
+
+If the user applications are going to request huge pages using mmap system
+call, then it is required that system administrator mount a file system of
+type hugetlbfs::
+
+  mount -t hugetlbfs \
+	-o uid=<value>,gid=<value>,mode=<value>,pagesize=<value>,size=<value>,\
+	min_size=<value>,nr_inodes=<value> none /mnt/huge
+
+This command mounts a (pseudo) filesystem of type hugetlbfs on the directory
+``/mnt/huge``.  Any file created on ``/mnt/huge`` uses huge pages.
+
+The ``uid`` and ``gid`` options sets the owner and group of the root of the
+file system.  By default the ``uid`` and ``gid`` of the current process
+are taken.
+
+The ``mode`` option sets the mode of root of file system to value & 01777.
+This value is given in octal. By default the value 0755 is picked.
+
+If the platform supports multiple huge page sizes, the ``pagesize`` option can
+be used to specify the huge page size and associated pool. ``pagesize``
+is specified in bytes. If ``pagesize`` is not specified the platform's
+default huge page size and associated pool will be used.
+
+The ``size`` option sets the maximum value of memory (huge pages) allowed
+for that filesystem (``/mnt/huge``). The ``size`` option can be specified
+in bytes, or as a percentage of the specified huge page pool (``nr_hugepages``).
+The size is rounded down to HPAGE_SIZE boundary.
+
+The ``min_size`` option sets the minimum value of memory (huge pages) allowed
+for the filesystem. ``min_size`` can be specified in the same way as ``size``,
+either bytes or a percentage of the huge page pool.
+At mount time, the number of huge pages specified by ``min_size`` are reserved
+for use by the filesystem.
+If there are not enough free huge pages available, the mount will fail.
+As huge pages are allocated to the filesystem and freed, the reserve count
+is adjusted so that the sum of allocated and reserved huge pages is always
+at least ``min_size``.
+
+The option ``nr_inodes`` sets the maximum number of inodes that ``/mnt/huge``
+can use.
+
+If the ``size``, ``min_size`` or ``nr_inodes`` option is not provided on
+command line then no limits are set.
+
+For ``pagesize``, ``size``, ``min_size`` and ``nr_inodes`` options, you can
+use [G|g]/[M|m]/[K|k] to represent giga/mega/kilo.
+For example, size=2K has the same meaning as size=2048.
+
+While read system calls are supported on files that reside on hugetlb
+file systems, write system calls are not.
+
+Regular chown, chgrp, and chmod commands (with right permissions) could be
+used to change the file attributes on hugetlbfs.
+
+Also, it is important to note that no such mount command is required if
+applications are going to use only shmat/shmget system calls or mmap with
+MAP_HUGETLB.  For an example of how to use mmap with MAP_HUGETLB see
+:ref:`map_hugetlb <map_hugetlb>` below.
+
+Users who wish to use hugetlb memory via shared memory segment should be
+members of a supplementary group and system admin needs to configure that gid
+into ``/proc/sys/vm/hugetlb_shm_group``.  It is possible for same or different
+applications to use any combination of mmaps and shm* calls, though the mount of
+filesystem will be required for using mmap calls without MAP_HUGETLB.
+
+Syscalls that operate on memory backed by hugetlb pages only have their lengths
+aligned to the native page size of the processor; they will normally fail with
+errno set to EINVAL or exclude hugetlb pages that extend beyond the length if
+not hugepage aligned.  For example, munmap(2) will fail if memory is backed by
+a hugetlb page and the length is smaller than the hugepage size.
+
+
+Examples
+========
+
+.. _map_hugetlb:
+
+``map_hugetlb``
+	see tools/testing/selftests/vm/map_hugetlb.c
+
+``hugepage-shm``
+	see tools/testing/selftests/vm/hugepage-shm.c
+
+``hugepage-mmap``
+	see tools/testing/selftests/vm/hugepage-mmap.c
+
+The `libhugetlbfs`_  library provides a wide range of userspace tools
+to help with huge page usability, environment setup, and control.
+
+.. _libhugetlbfs: https://github.com/libhugetlbfs/libhugetlbfs
diff --git a/Documentation/admin-guide/mm/idle_page_tracking.rst b/Documentation/admin-guide/mm/idle_page_tracking.rst
new file mode 100644
index 000000000000..6f7b7ca1add3
--- /dev/null
+++ b/Documentation/admin-guide/mm/idle_page_tracking.rst
@@ -0,0 +1,116 @@
+.. _idle_page_tracking:
+
+==================
+Idle Page Tracking
+==================
+
+Motivation
+==========
+
+The idle page tracking feature allows to track which memory pages are being
+accessed by a workload and which are idle. This information can be useful for
+estimating the workload's working set size, which, in turn, can be taken into
+account when configuring the workload parameters, setting memory cgroup limits,
+or deciding where to place the workload within a compute cluster.
+
+It is enabled by CONFIG_IDLE_PAGE_TRACKING=y.
+
+.. _user_api:
+
+User API
+========
+
+The idle page tracking API is located at ``/sys/kernel/mm/page_idle``.
+Currently, it consists of the only read-write file,
+``/sys/kernel/mm/page_idle/bitmap``.
+
+The file implements a bitmap where each bit corresponds to a memory page. The
+bitmap is represented by an array of 8-byte integers, and the page at PFN #i is
+mapped to bit #i%64 of array element #i/64, byte order is native. When a bit is
+set, the corresponding page is idle.
+
+A page is considered idle if it has not been accessed since it was marked idle
+(for more details on what "accessed" actually means see the :ref:`Implementation
+Details <impl_details>` section).
+To mark a page idle one has to set the bit corresponding to
+the page by writing to the file. A value written to the file is OR-ed with the
+current bitmap value.
+
+Only accesses to user memory pages are tracked. These are pages mapped to a
+process address space, page cache and buffer pages, swap cache pages. For other
+page types (e.g. SLAB pages) an attempt to mark a page idle is silently ignored,
+and hence such pages are never reported idle.
+
+For huge pages the idle flag is set only on the head page, so one has to read
+``/proc/kpageflags`` in order to correctly count idle huge pages.
+
+Reading from or writing to ``/sys/kernel/mm/page_idle/bitmap`` will return
+-EINVAL if you are not starting the read/write on an 8-byte boundary, or
+if the size of the read/write is not a multiple of 8 bytes. Writing to
+this file beyond max PFN will return -ENXIO.
+
+That said, in order to estimate the amount of pages that are not used by a
+workload one should:
+
+ 1. Mark all the workload's pages as idle by setting corresponding bits in
+    ``/sys/kernel/mm/page_idle/bitmap``. The pages can be found by reading
+    ``/proc/pid/pagemap`` if the workload is represented by a process, or by
+    filtering out alien pages using ``/proc/kpagecgroup`` in case the workload
+    is placed in a memory cgroup.
+
+ 2. Wait until the workload accesses its working set.
+
+ 3. Read ``/sys/kernel/mm/page_idle/bitmap`` and count the number of bits set.
+    If one wants to ignore certain types of pages, e.g. mlocked pages since they
+    are not reclaimable, he or she can filter them out using
+    ``/proc/kpageflags``.
+
+See :ref:`Documentation/admin-guide/mm/pagemap.rst <pagemap>` for more
+information about ``/proc/pid/pagemap``, ``/proc/kpageflags``, and
+``/proc/kpagecgroup``.
+
+.. _impl_details:
+
+Implementation Details
+======================
+
+The kernel internally keeps track of accesses to user memory pages in order to
+reclaim unreferenced pages first on memory shortage conditions. A page is
+considered referenced if it has been recently accessed via a process address
+space, in which case one or more PTEs it is mapped to will have the Accessed bit
+set, or marked accessed explicitly by the kernel (see mark_page_accessed()). The
+latter happens when:
+
+ - a userspace process reads or writes a page using a system call (e.g. read(2)
+   or write(2))
+
+ - a page that is used for storing filesystem buffers is read or written,
+   because a process needs filesystem metadata stored in it (e.g. lists a
+   directory tree)
+
+ - a page is accessed by a device driver using get_user_pages()
+
+When a dirty page is written to swap or disk as a result of memory reclaim or
+exceeding the dirty memory limit, it is not marked referenced.
+
+The idle memory tracking feature adds a new page flag, the Idle flag. This flag
+is set manually, by writing to ``/sys/kernel/mm/page_idle/bitmap`` (see the
+:ref:`User API <user_api>`
+section), and cleared automatically whenever a page is referenced as defined
+above.
+
+When a page is marked idle, the Accessed bit must be cleared in all PTEs it is
+mapped to, otherwise we will not be able to detect accesses to the page coming
+from a process address space. To avoid interference with the reclaimer, which,
+as noted above, uses the Accessed bit to promote actively referenced pages, one
+more page flag is introduced, the Young flag. When the PTE Accessed bit is
+cleared as a result of setting or updating a page's Idle flag, the Young flag
+is set on the page. The reclaimer treats the Young flag as an extra PTE
+Accessed bit and therefore will consider such a page as referenced.
+
+Since the idle memory tracking feature is based on the memory reclaimer logic,
+it only works with pages that are on an LRU list, other pages are silently
+ignored. That means it will ignore a user memory page if it is isolated, but
+since there are usually not many of them, it should not affect the overall
+result noticeably. In order not to stall scanning of the idle page bitmap,
+locked pages may be skipped too.
diff --git a/Documentation/admin-guide/mm/index.rst b/Documentation/admin-guide/mm/index.rst
new file mode 100644
index 000000000000..ceead68c2df7
--- /dev/null
+++ b/Documentation/admin-guide/mm/index.rst
@@ -0,0 +1,36 @@
+=================
+Memory Management
+=================
+
+Linux memory management subsystem is responsible, as the name implies,
+for managing the memory in the system. This includes implemnetation of
+virtual memory and demand paging, memory allocation both for kernel
+internal structures and user space programms, mapping of files into
+processes address space and many other cool things.
+
+Linux memory management is a complex system with many configurable
+settings. Most of these settings are available via ``/proc``
+filesystem and can be quired and adjusted using ``sysctl``. These APIs
+are described in Documentation/sysctl/vm.txt and in `man 5 proc`_.
+
+.. _man 5 proc: http://man7.org/linux/man-pages/man5/proc.5.html
+
+Linux memory management has its own jargon and if you are not yet
+familiar with it, consider reading
+:ref:`Documentation/admin-guide/mm/concepts.rst <mm_concepts>`.
+
+Here we document in detail how to interact with various mechanisms in
+the Linux memory management.
+
+.. toctree::
+   :maxdepth: 1
+
+   concepts
+   hugetlbpage
+   idle_page_tracking
+   ksm
+   numa_memory_policy
+   pagemap
+   soft-dirty
+   transhuge
+   userfaultfd
diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst
new file mode 100644
index 000000000000..9303786632d1
--- /dev/null
+++ b/Documentation/admin-guide/mm/ksm.rst
@@ -0,0 +1,189 @@
+.. _admin_guide_ksm:
+
+=======================
+Kernel Samepage Merging
+=======================
+
+Overview
+========
+
+KSM is a memory-saving de-duplication feature, enabled by CONFIG_KSM=y,
+added to the Linux kernel in 2.6.32.  See ``mm/ksm.c`` for its implementation,
+and http://lwn.net/Articles/306704/ and http://lwn.net/Articles/330589/
+
+KSM was originally developed for use with KVM (where it was known as
+Kernel Shared Memory), to fit more virtual machines into physical memory,
+by sharing the data common between them.  But it can be useful to any
+application which generates many instances of the same data.
+
+The KSM daemon ksmd periodically scans those areas of user memory
+which have been registered with it, looking for pages of identical
+content which can be replaced by a single write-protected page (which
+is automatically copied if a process later wants to update its
+content). The amount of pages that KSM daemon scans in a single pass
+and the time between the passes are configured using :ref:`sysfs
+intraface <ksm_sysfs>`
+
+KSM only merges anonymous (private) pages, never pagecache (file) pages.
+KSM's merged pages were originally locked into kernel memory, but can now
+be swapped out just like other user pages (but sharing is broken when they
+are swapped back in: ksmd must rediscover their identity and merge again).
+
+Controlling KSM with madvise
+============================
+
+KSM only operates on those areas of address space which an application
+has advised to be likely candidates for merging, by using the madvise(2)
+system call::
+
+	int madvise(addr, length, MADV_MERGEABLE)
+
+The app may call
+
+::
+
+	int madvise(addr, length, MADV_UNMERGEABLE)
+
+to cancel that advice and restore unshared pages: whereupon KSM
+unmerges whatever it merged in that range.  Note: this unmerging call
+may suddenly require more memory than is available - possibly failing
+with EAGAIN, but more probably arousing the Out-Of-Memory killer.
+
+If KSM is not configured into the running kernel, madvise MADV_MERGEABLE
+and MADV_UNMERGEABLE simply fail with EINVAL.  If the running kernel was
+built with CONFIG_KSM=y, those calls will normally succeed: even if the
+the KSM daemon is not currently running, MADV_MERGEABLE still registers
+the range for whenever the KSM daemon is started; even if the range
+cannot contain any pages which KSM could actually merge; even if
+MADV_UNMERGEABLE is applied to a range which was never MADV_MERGEABLE.
+
+If a region of memory must be split into at least one new MADV_MERGEABLE
+or MADV_UNMERGEABLE region, the madvise may return ENOMEM if the process
+will exceed ``vm.max_map_count`` (see Documentation/sysctl/vm.txt).
+
+Like other madvise calls, they are intended for use on mapped areas of
+the user address space: they will report ENOMEM if the specified range
+includes unmapped gaps (though working on the intervening mapped areas),
+and might fail with EAGAIN if not enough memory for internal structures.
+
+Applications should be considerate in their use of MADV_MERGEABLE,
+restricting its use to areas likely to benefit.  KSM's scans may use a lot
+of processing power: some installations will disable KSM for that reason.
+
+.. _ksm_sysfs:
+
+KSM daemon sysfs interface
+==========================
+
+The KSM daemon is controlled by sysfs files in ``/sys/kernel/mm/ksm/``,
+readable by all but writable only by root:
+
+pages_to_scan
+        how many pages to scan before ksmd goes to sleep
+        e.g. ``echo 100 > /sys/kernel/mm/ksm/pages_to_scan``.
+
+        Default: 100 (chosen for demonstration purposes)
+
+sleep_millisecs
+        how many milliseconds ksmd should sleep before next scan
+        e.g. ``echo 20 > /sys/kernel/mm/ksm/sleep_millisecs``
+
+        Default: 20 (chosen for demonstration purposes)
+
+merge_across_nodes
+        specifies if pages from different NUMA nodes can be merged.
+        When set to 0, ksm merges only pages which physically reside
+        in the memory area of same NUMA node. That brings lower
+        latency to access of shared pages. Systems with more nodes, at
+        significant NUMA distances, are likely to benefit from the
+        lower latency of setting 0. Smaller systems, which need to
+        minimize memory usage, are likely to benefit from the greater
+        sharing of setting 1 (default). You may wish to compare how
+        your system performs under each setting, before deciding on
+        which to use. ``merge_across_nodes`` setting can be changed only
+        when there are no ksm shared pages in the system: set run 2 to
+        unmerge pages first, then to 1 after changing
+        ``merge_across_nodes``, to remerge according to the new setting.
+
+        Default: 1 (merging across nodes as in earlier releases)
+
+run
+        * set to 0 to stop ksmd from running but keep merged pages,
+        * set to 1 to run ksmd e.g. ``echo 1 > /sys/kernel/mm/ksm/run``,
+        * set to 2 to stop ksmd and unmerge all pages currently merged, but
+	  leave mergeable areas registered for next run.
+
+        Default: 0 (must be changed to 1 to activate KSM, except if
+        CONFIG_SYSFS is disabled)
+
+use_zero_pages
+        specifies whether empty pages (i.e. allocated pages that only
+        contain zeroes) should be treated specially.  When set to 1,
+        empty pages are merged with the kernel zero page(s) instead of
+        with each other as it would happen normally. This can improve
+        the performance on architectures with coloured zero pages,
+        depending on the workload. Care should be taken when enabling
+        this setting, as it can potentially degrade the performance of
+        KSM for some workloads, for example if the checksums of pages
+        candidate for merging match the checksum of an empty
+        page. This setting can be changed at any time, it is only
+        effective for pages merged after the change.
+
+        Default: 0 (normal KSM behaviour as in earlier releases)
+
+max_page_sharing
+        Maximum sharing allowed for each KSM page. This enforces a
+        deduplication limit to avoid high latency for virtual memory
+        operations that involve traversal of the virtual mappings that
+        share the KSM page. The minimum value is 2 as a newly created
+        KSM page will have at least two sharers. The higher this value
+        the faster KSM will merge the memory and the higher the
+        deduplication factor will be, but the slower the worst case
+        virtual mappings traversal could be for any given KSM
+        page. Slowing down this traversal means there will be higher
+        latency for certain virtual memory operations happening during
+        swapping, compaction, NUMA balancing and page migration, in
+        turn decreasing responsiveness for the caller of those virtual
+        memory operations. The scheduler latency of other tasks not
+        involved with the VM operations doing the virtual mappings
+        traversal is not affected by this parameter as these
+        traversals are always schedule friendly themselves.
+
+stable_node_chains_prune_millisecs
+        specifies how frequently KSM checks the metadata of the pages
+        that hit the deduplication limit for stale information.
+        Smaller milllisecs values will free up the KSM metadata with
+        lower latency, but they will make ksmd use more CPU during the
+        scan. It's a noop if not a single KSM page hit the
+        ``max_page_sharing`` yet.
+
+The effectiveness of KSM and MADV_MERGEABLE is shown in ``/sys/kernel/mm/ksm/``:
+
+pages_shared
+        how many shared pages are being used
+pages_sharing
+        how many more sites are sharing them i.e. how much saved
+pages_unshared
+        how many pages unique but repeatedly checked for merging
+pages_volatile
+        how many pages changing too fast to be placed in a tree
+full_scans
+        how many times all mergeable areas have been scanned
+stable_node_chains
+        the number of KSM pages that hit the ``max_page_sharing`` limit
+stable_node_dups
+        number of duplicated KSM pages
+
+A high ratio of ``pages_sharing`` to ``pages_shared`` indicates good
+sharing, but a high ratio of ``pages_unshared`` to ``pages_sharing``
+indicates wasted effort.  ``pages_volatile`` embraces several
+different kinds of activity, but a high proportion there would also
+indicate poor use of madvise MADV_MERGEABLE.
+
+The maximum possible ``pages_sharing/pages_shared`` ratio is limited by the
+``max_page_sharing`` tunable. To increase the ratio ``max_page_sharing`` must
+be increased accordingly.
+
+--
+Izik Eidus,
+Hugh Dickins, 17 Nov 2009
diff --git a/Documentation/admin-guide/mm/numa_memory_policy.rst b/Documentation/admin-guide/mm/numa_memory_policy.rst
new file mode 100644
index 000000000000..d78c5b315f72
--- /dev/null
+++ b/Documentation/admin-guide/mm/numa_memory_policy.rst
@@ -0,0 +1,495 @@
+.. _numa_memory_policy:
+
+==================
+NUMA Memory Policy
+==================
+
+What is NUMA Memory Policy?
+============================
+
+In the Linux kernel, "memory policy" determines from which node the kernel will
+allocate memory in a NUMA system or in an emulated NUMA system.  Linux has
+supported platforms with Non-Uniform Memory Access architectures since 2.4.?.
+The current memory policy support was added to Linux 2.6 around May 2004.  This
+document attempts to describe the concepts and APIs of the 2.6 memory policy
+support.
+
+Memory policies should not be confused with cpusets
+(``Documentation/cgroup-v1/cpusets.txt``)
+which is an administrative mechanism for restricting the nodes from which
+memory may be allocated by a set of processes. Memory policies are a
+programming interface that a NUMA-aware application can take advantage of.  When
+both cpusets and policies are applied to a task, the restrictions of the cpuset
+takes priority.  See :ref:`Memory Policies and cpusets <mem_pol_and_cpusets>`
+below for more details.
+
+Memory Policy Concepts
+======================
+
+Scope of Memory Policies
+------------------------
+
+The Linux kernel supports _scopes_ of memory policy, described here from
+most general to most specific:
+
+System Default Policy
+	this policy is "hard coded" into the kernel.  It is the policy
+	that governs all page allocations that aren't controlled by
+	one of the more specific policy scopes discussed below.  When
+	the system is "up and running", the system default policy will
+	use "local allocation" described below.  However, during boot
+	up, the system default policy will be set to interleave
+	allocations across all nodes with "sufficient" memory, so as
+	not to overload the initial boot node with boot-time
+	allocations.
+
+Task/Process Policy
+	this is an optional, per-task policy.  When defined for a
+	specific task, this policy controls all page allocations made
+	by or on behalf of the task that aren't controlled by a more
+	specific scope. If a task does not define a task policy, then
+	all page allocations that would have been controlled by the
+	task policy "fall back" to the System Default Policy.
+
+	The task policy applies to the entire address space of a task. Thus,
+	it is inheritable, and indeed is inherited, across both fork()
+	[clone() w/o the CLONE_VM flag] and exec*().  This allows a parent task
+	to establish the task policy for a child task exec()'d from an
+	executable image that has no awareness of memory policy.  See the
+	:ref:`Memory Policy APIs <memory_policy_apis>` section,
+	below, for an overview of the system call
+	that a task may use to set/change its task/process policy.
+
+	In a multi-threaded task, task policies apply only to the thread
+	[Linux kernel task] that installs the policy and any threads
+	subsequently created by that thread.  Any sibling threads existing
+	at the time a new task policy is installed retain their current
+	policy.
+
+	A task policy applies only to pages allocated after the policy is
+	installed.  Any pages already faulted in by the task when the task
+	changes its task policy remain where they were allocated based on
+	the policy at the time they were allocated.
+
+.. _vma_policy:
+
+VMA Policy
+	A "VMA" or "Virtual Memory Area" refers to a range of a task's
+	virtual address space.  A task may define a specific policy for a range
+	of its virtual address space.   See the
+	:ref:`Memory Policy APIs <memory_policy_apis>` section,
+	below, for an overview of the mbind() system call used to set a VMA
+	policy.
+
+	A VMA policy will govern the allocation of pages that back
+	this region of the address space.  Any regions of the task's
+	address space that don't have an explicit VMA policy will fall
+	back to the task policy, which may itself fall back to the
+	System Default Policy.
+
+	VMA policies have a few complicating details:
+
+	* VMA policy applies ONLY to anonymous pages.  These include
+	  pages allocated for anonymous segments, such as the task
+	  stack and heap, and any regions of the address space
+	  mmap()ed with the MAP_ANONYMOUS flag.  If a VMA policy is
+	  applied to a file mapping, it will be ignored if the mapping
+	  used the MAP_SHARED flag.  If the file mapping used the
+	  MAP_PRIVATE flag, the VMA policy will only be applied when
+	  an anonymous page is allocated on an attempt to write to the
+	  mapping-- i.e., at Copy-On-Write.
+
+	* VMA policies are shared between all tasks that share a
+	  virtual address space--a.k.a. threads--independent of when
+	  the policy is installed; and they are inherited across
+	  fork().  However, because VMA policies refer to a specific
+	  region of a task's address space, and because the address
+	  space is discarded and recreated on exec*(), VMA policies
+	  are NOT inheritable across exec().  Thus, only NUMA-aware
+	  applications may use VMA policies.
+
+	* A task may install a new VMA policy on a sub-range of a
+	  previously mmap()ed region.  When this happens, Linux splits
+	  the existing virtual memory area into 2 or 3 VMAs, each with
+	  it's own policy.
+
+	* By default, VMA policy applies only to pages allocated after
+	  the policy is installed.  Any pages already faulted into the
+	  VMA range remain where they were allocated based on the
+	  policy at the time they were allocated.  However, since
+	  2.6.16, Linux supports page migration via the mbind() system
+	  call, so that page contents can be moved to match a newly
+	  installed policy.
+
+Shared Policy
+	Conceptually, shared policies apply to "memory objects" mapped
+	shared into one or more tasks' distinct address spaces.  An
+	application installs shared policies the same way as VMA
+	policies--using the mbind() system call specifying a range of
+	virtual addresses that map the shared object.  However, unlike
+	VMA policies, which can be considered to be an attribute of a
+	range of a task's address space, shared policies apply
+	directly to the shared object.  Thus, all tasks that attach to
+	the object share the policy, and all pages allocated for the
+	shared object, by any task, will obey the shared policy.
+
+	As of 2.6.22, only shared memory segments, created by shmget() or
+	mmap(MAP_ANONYMOUS|MAP_SHARED), support shared policy.  When shared
+	policy support was added to Linux, the associated data structures were
+	added to hugetlbfs shmem segments.  At the time, hugetlbfs did not
+	support allocation at fault time--a.k.a lazy allocation--so hugetlbfs
+	shmem segments were never "hooked up" to the shared policy support.
+	Although hugetlbfs segments now support lazy allocation, their support
+	for shared policy has not been completed.
+
+	As mentioned above in :ref:`VMA policies <vma_policy>` section,
+	allocations of page cache pages for regular files mmap()ed
+	with MAP_SHARED ignore any VMA policy installed on the virtual
+	address range backed by the shared file mapping.  Rather,
+	shared page cache pages, including pages backing private
+	mappings that have not yet been written by the task, follow
+	task policy, if any, else System Default Policy.
+
+	The shared policy infrastructure supports different policies on subset
+	ranges of the shared object.  However, Linux still splits the VMA of
+	the task that installs the policy for each range of distinct policy.
+	Thus, different tasks that attach to a shared memory segment can have
+	different VMA configurations mapping that one shared object.  This
+	can be seen by examining the /proc/<pid>/numa_maps of tasks sharing
+	a shared memory region, when one task has installed shared policy on
+	one or more ranges of the region.
+
+Components of Memory Policies
+-----------------------------
+
+A NUMA memory policy consists of a "mode", optional mode flags, and
+an optional set of nodes.  The mode determines the behavior of the
+policy, the optional mode flags determine the behavior of the mode,
+and the optional set of nodes can be viewed as the arguments to the
+policy behavior.
+
+Internally, memory policies are implemented by a reference counted
+structure, struct mempolicy.  Details of this structure will be
+discussed in context, below, as required to explain the behavior.
+
+NUMA memory policy supports the following 4 behavioral modes:
+
+Default Mode--MPOL_DEFAULT
+	This mode is only used in the memory policy APIs.  Internally,
+	MPOL_DEFAULT is converted to the NULL memory policy in all
+	policy scopes.  Any existing non-default policy will simply be
+	removed when MPOL_DEFAULT is specified.  As a result,
+	MPOL_DEFAULT means "fall back to the next most specific policy
+	scope."
+
+	For example, a NULL or default task policy will fall back to the
+	system default policy.  A NULL or default vma policy will fall
+	back to the task policy.
+
+	When specified in one of the memory policy APIs, the Default mode
+	does not use the optional set of nodes.
+
+	It is an error for the set of nodes specified for this policy to
+	be non-empty.
+
+MPOL_BIND
+	This mode specifies that memory must come from the set of
+	nodes specified by the policy.  Memory will be allocated from
+	the node in the set with sufficient free memory that is
+	closest to the node where the allocation takes place.
+
+MPOL_PREFERRED
+	This mode specifies that the allocation should be attempted
+	from the single node specified in the policy.  If that
+	allocation fails, the kernel will search other nodes, in order
+	of increasing distance from the preferred node based on
+	information provided by the platform firmware.
+
+	Internally, the Preferred policy uses a single node--the
+	preferred_node member of struct mempolicy.  When the internal
+	mode flag MPOL_F_LOCAL is set, the preferred_node is ignored
+	and the policy is interpreted as local allocation.  "Local"
+	allocation policy can be viewed as a Preferred policy that
+	starts at the node containing the cpu where the allocation
+	takes place.
+
+	It is possible for the user to specify that local allocation
+	is always preferred by passing an empty nodemask with this
+	mode.  If an empty nodemask is passed, the policy cannot use
+	the MPOL_F_STATIC_NODES or MPOL_F_RELATIVE_NODES flags
+	described below.
+
+MPOL_INTERLEAVED
+	This mode specifies that page allocations be interleaved, on a
+	page granularity, across the nodes specified in the policy.
+	This mode also behaves slightly differently, based on the
+	context where it is used:
+
+	For allocation of anonymous pages and shared memory pages,
+	Interleave mode indexes the set of nodes specified by the
+	policy using the page offset of the faulting address into the
+	segment [VMA] containing the address modulo the number of
+	nodes specified by the policy.  It then attempts to allocate a
+	page, starting at the selected node, as if the node had been
+	specified by a Preferred policy or had been selected by a
+	local allocation.  That is, allocation will follow the per
+	node zonelist.
+
+	For allocation of page cache pages, Interleave mode indexes
+	the set of nodes specified by the policy using a node counter
+	maintained per task.  This counter wraps around to the lowest
+	specified node after it reaches the highest specified node.
+	This will tend to spread the pages out over the nodes
+	specified by the policy based on the order in which they are
+	allocated, rather than based on any page offset into an
+	address range or file.  During system boot up, the temporary
+	interleaved system default policy works in this mode.
+
+NUMA memory policy supports the following optional mode flags:
+
+MPOL_F_STATIC_NODES
+	This flag specifies that the nodemask passed by
+	the user should not be remapped if the task or VMA's set of allowed
+	nodes changes after the memory policy has been defined.
+
+	Without this flag, any time a mempolicy is rebound because of a
+	change in the set of allowed nodes, the node (Preferred) or
+	nodemask (Bind, Interleave) is remapped to the new set of
+	allowed nodes.  This may result in nodes being used that were
+	previously undesired.
+
+	With this flag, if the user-specified nodes overlap with the
+	nodes allowed by the task's cpuset, then the memory policy is
+	applied to their intersection.  If the two sets of nodes do not
+	overlap, the Default policy is used.
+
+	For example, consider a task that is attached to a cpuset with
+	mems 1-3 that sets an Interleave policy over the same set.  If
+	the cpuset's mems change to 3-5, the Interleave will now occur
+	over nodes 3, 4, and 5.  With this flag, however, since only node
+	3 is allowed from the user's nodemask, the "interleave" only
+	occurs over that node.  If no nodes from the user's nodemask are
+	now allowed, the Default behavior is used.
+
+	MPOL_F_STATIC_NODES cannot be combined with the
+	MPOL_F_RELATIVE_NODES flag.  It also cannot be used for
+	MPOL_PREFERRED policies that were created with an empty nodemask
+	(local allocation).
+
+MPOL_F_RELATIVE_NODES
+	This flag specifies that the nodemask passed
+	by the user will be mapped relative to the set of the task or VMA's
+	set of allowed nodes.  The kernel stores the user-passed nodemask,
+	and if the allowed nodes changes, then that original nodemask will
+	be remapped relative to the new set of allowed nodes.
+
+	Without this flag (and without MPOL_F_STATIC_NODES), anytime a
+	mempolicy is rebound because of a change in the set of allowed
+	nodes, the node (Preferred) or nodemask (Bind, Interleave) is
+	remapped to the new set of allowed nodes.  That remap may not
+	preserve the relative nature of the user's passed nodemask to its
+	set of allowed nodes upon successive rebinds: a nodemask of
+	1,3,5 may be remapped to 7-9 and then to 1-3 if the set of
+	allowed nodes is restored to its original state.
+
+	With this flag, the remap is done so that the node numbers from
+	the user's passed nodemask are relative to the set of allowed
+	nodes.  In other words, if nodes 0, 2, and 4 are set in the user's
+	nodemask, the policy will be effected over the first (and in the
+	Bind or Interleave case, the third and fifth) nodes in the set of
+	allowed nodes.  The nodemask passed by the user represents nodes
+	relative to task or VMA's set of allowed nodes.
+
+	If the user's nodemask includes nodes that are outside the range
+	of the new set of allowed nodes (for example, node 5 is set in
+	the user's nodemask when the set of allowed nodes is only 0-3),
+	then the remap wraps around to the beginning of the nodemask and,
+	if not already set, sets the node in the mempolicy nodemask.
+
+	For example, consider a task that is attached to a cpuset with
+	mems 2-5 that sets an Interleave policy over the same set with
+	MPOL_F_RELATIVE_NODES.  If the cpuset's mems change to 3-7, the
+	interleave now occurs over nodes 3,5-7.  If the cpuset's mems
+	then change to 0,2-3,5, then the interleave occurs over nodes
+	0,2-3,5.
+
+	Thanks to the consistent remapping, applications preparing
+	nodemasks to specify memory policies using this flag should
+	disregard their current, actual cpuset imposed memory placement
+	and prepare the nodemask as if they were always located on
+	memory nodes 0 to N-1, where N is the number of memory nodes the
+	policy is intended to manage.  Let the kernel then remap to the
+	set of memory nodes allowed by the task's cpuset, as that may
+	change over time.
+
+	MPOL_F_RELATIVE_NODES cannot be combined with the
+	MPOL_F_STATIC_NODES flag.  It also cannot be used for
+	MPOL_PREFERRED policies that were created with an empty nodemask
+	(local allocation).
+
+Memory Policy Reference Counting
+================================
+
+To resolve use/free races, struct mempolicy contains an atomic reference
+count field.  Internal interfaces, mpol_get()/mpol_put() increment and
+decrement this reference count, respectively.  mpol_put() will only free
+the structure back to the mempolicy kmem cache when the reference count
+goes to zero.
+
+When a new memory policy is allocated, its reference count is initialized
+to '1', representing the reference held by the task that is installing the
+new policy.  When a pointer to a memory policy structure is stored in another
+structure, another reference is added, as the task's reference will be dropped
+on completion of the policy installation.
+
+During run-time "usage" of the policy, we attempt to minimize atomic operations
+on the reference count, as this can lead to cache lines bouncing between cpus
+and NUMA nodes.  "Usage" here means one of the following:
+
+1) querying of the policy, either by the task itself [using the get_mempolicy()
+   API discussed below] or by another task using the /proc/<pid>/numa_maps
+   interface.
+
+2) examination of the policy to determine the policy mode and associated node
+   or node lists, if any, for page allocation.  This is considered a "hot
+   path".  Note that for MPOL_BIND, the "usage" extends across the entire
+   allocation process, which may sleep during page reclaimation, because the
+   BIND policy nodemask is used, by reference, to filter ineligible nodes.
+
+We can avoid taking an extra reference during the usages listed above as
+follows:
+
+1) we never need to get/free the system default policy as this is never
+   changed nor freed, once the system is up and running.
+
+2) for querying the policy, we do not need to take an extra reference on the
+   target task's task policy nor vma policies because we always acquire the
+   task's mm's mmap_sem for read during the query.  The set_mempolicy() and
+   mbind() APIs [see below] always acquire the mmap_sem for write when
+   installing or replacing task or vma policies.  Thus, there is no possibility
+   of a task or thread freeing a policy while another task or thread is
+   querying it.
+
+3) Page allocation usage of task or vma policy occurs in the fault path where
+   we hold them mmap_sem for read.  Again, because replacing the task or vma
+   policy requires that the mmap_sem be held for write, the policy can't be
+   freed out from under us while we're using it for page allocation.
+
+4) Shared policies require special consideration.  One task can replace a
+   shared memory policy while another task, with a distinct mmap_sem, is
+   querying or allocating a page based on the policy.  To resolve this
+   potential race, the shared policy infrastructure adds an extra reference
+   to the shared policy during lookup while holding a spin lock on the shared
+   policy management structure.  This requires that we drop this extra
+   reference when we're finished "using" the policy.  We must drop the
+   extra reference on shared policies in the same query/allocation paths
+   used for non-shared policies.  For this reason, shared policies are marked
+   as such, and the extra reference is dropped "conditionally"--i.e., only
+   for shared policies.
+
+   Because of this extra reference counting, and because we must lookup
+   shared policies in a tree structure under spinlock, shared policies are
+   more expensive to use in the page allocation path.  This is especially
+   true for shared policies on shared memory regions shared by tasks running
+   on different NUMA nodes.  This extra overhead can be avoided by always
+   falling back to task or system default policy for shared memory regions,
+   or by prefaulting the entire shared memory region into memory and locking
+   it down.  However, this might not be appropriate for all applications.
+
+.. _memory_policy_apis:
+
+Memory Policy APIs
+==================
+
+Linux supports 3 system calls for controlling memory policy.  These APIS
+always affect only the calling task, the calling task's address space, or
+some shared object mapped into the calling task's address space.
+
+.. note::
+   the headers that define these APIs and the parameter data types for
+   user space applications reside in a package that is not part of the
+   Linux kernel.  The kernel system call interfaces, with the 'sys\_'
+   prefix, are defined in <linux/syscalls.h>; the mode and flag
+   definitions are defined in <linux/mempolicy.h>.
+
+Set [Task] Memory Policy::
+
+	long set_mempolicy(int mode, const unsigned long *nmask,
+					unsigned long maxnode);
+
+Set's the calling task's "task/process memory policy" to mode
+specified by the 'mode' argument and the set of nodes defined by
+'nmask'.  'nmask' points to a bit mask of node ids containing at least
+'maxnode' ids.  Optional mode flags may be passed by combining the
+'mode' argument with the flag (for example: MPOL_INTERLEAVE |
+MPOL_F_STATIC_NODES).
+
+See the set_mempolicy(2) man page for more details
+
+
+Get [Task] Memory Policy or Related Information::
+
+	long get_mempolicy(int *mode,
+			   const unsigned long *nmask, unsigned long maxnode,
+			   void *addr, int flags);
+
+Queries the "task/process memory policy" of the calling task, or the
+policy or location of a specified virtual address, depending on the
+'flags' argument.
+
+See the get_mempolicy(2) man page for more details
+
+
+Install VMA/Shared Policy for a Range of Task's Address Space::
+
+	long mbind(void *start, unsigned long len, int mode,
+		   const unsigned long *nmask, unsigned long maxnode,
+		   unsigned flags);
+
+mbind() installs the policy specified by (mode, nmask, maxnodes) as a
+VMA policy for the range of the calling task's address space specified
+by the 'start' and 'len' arguments.  Additional actions may be
+requested via the 'flags' argument.
+
+See the mbind(2) man page for more details.
+
+Memory Policy Command Line Interface
+====================================
+
+Although not strictly part of the Linux implementation of memory policy,
+a command line tool, numactl(8), exists that allows one to:
+
++ set the task policy for a specified program via set_mempolicy(2), fork(2) and
+  exec(2)
+
++ set the shared policy for a shared memory segment via mbind(2)
+
+The numactl(8) tool is packaged with the run-time version of the library
+containing the memory policy system call wrappers.  Some distributions
+package the headers and compile-time libraries in a separate development
+package.
+
+.. _mem_pol_and_cpusets:
+
+Memory Policies and cpusets
+===========================
+
+Memory policies work within cpusets as described above.  For memory policies
+that require a node or set of nodes, the nodes are restricted to the set of
+nodes whose memories are allowed by the cpuset constraints.  If the nodemask
+specified for the policy contains nodes that are not allowed by the cpuset and
+MPOL_F_RELATIVE_NODES is not used, the intersection of the set of nodes
+specified for the policy and the set of nodes with memory is used.  If the
+result is the empty set, the policy is considered invalid and cannot be
+installed.  If MPOL_F_RELATIVE_NODES is used, the policy's nodes are mapped
+onto and folded into the task's set of allowed nodes as previously described.
+
+The interaction of memory policies and cpusets can be problematic when tasks
+in two cpusets share access to a memory region, such as shared memory segments
+created by shmget() of mmap() with the MAP_ANONYMOUS and MAP_SHARED flags, and
+any of the tasks install shared policy on the region, only nodes whose
+memories are allowed in both cpusets may be used in the policies.  Obtaining
+this information requires "stepping outside" the memory policy APIs to use the
+cpuset information and requires that one know in what cpusets other task might
+be attaching to the shared region.  Furthermore, if the cpusets' allowed
+memory sets are disjoint, "local" allocation is the only valid policy.
diff --git a/Documentation/admin-guide/mm/pagemap.rst b/Documentation/admin-guide/mm/pagemap.rst
new file mode 100644
index 000000000000..577af85beb41
--- /dev/null
+++ b/Documentation/admin-guide/mm/pagemap.rst
@@ -0,0 +1,201 @@
+.. _pagemap:
+
+=============================
+Examining Process Page Tables
+=============================
+
+pagemap is a new (as of 2.6.25) set of interfaces in the kernel that allow
+userspace programs to examine the page tables and related information by
+reading files in ``/proc``.
+
+There are four components to pagemap:
+
+ * ``/proc/pid/pagemap``.  This file lets a userspace process find out which
+   physical frame each virtual page is mapped to.  It contains one 64-bit
+   value for each virtual page, containing the following data (from
+   ``fs/proc/task_mmu.c``, above pagemap_read):
+
+    * Bits 0-54  page frame number (PFN) if present
+    * Bits 0-4   swap type if swapped
+    * Bits 5-54  swap offset if swapped
+    * Bit  55    pte is soft-dirty (see
+      :ref:`Documentation/admin-guide/mm/soft-dirty.rst <soft_dirty>`)
+    * Bit  56    page exclusively mapped (since 4.2)
+    * Bits 57-60 zero
+    * Bit  61    page is file-page or shared-anon (since 3.5)
+    * Bit  62    page swapped
+    * Bit  63    page present
+
+   Since Linux 4.0 only users with the CAP_SYS_ADMIN capability can get PFNs.
+   In 4.0 and 4.1 opens by unprivileged fail with -EPERM.  Starting from
+   4.2 the PFN field is zeroed if the user does not have CAP_SYS_ADMIN.
+   Reason: information about PFNs helps in exploiting Rowhammer vulnerability.
+
+   If the page is not present but in swap, then the PFN contains an
+   encoding of the swap file number and the page's offset into the
+   swap. Unmapped pages return a null PFN. This allows determining
+   precisely which pages are mapped (or in swap) and comparing mapped
+   pages between processes.
+
+   Efficient users of this interface will use ``/proc/pid/maps`` to
+   determine which areas of memory are actually mapped and llseek to
+   skip over unmapped regions.
+
+ * ``/proc/kpagecount``.  This file contains a 64-bit count of the number of
+   times each page is mapped, indexed by PFN.
+
+ * ``/proc/kpageflags``.  This file contains a 64-bit set of flags for each
+   page, indexed by PFN.
+
+   The flags are (from ``fs/proc/page.c``, above kpageflags_read):
+
+    0. LOCKED
+    1. ERROR
+    2. REFERENCED
+    3. UPTODATE
+    4. DIRTY
+    5. LRU
+    6. ACTIVE
+    7. SLAB
+    8. WRITEBACK
+    9. RECLAIM
+    10. BUDDY
+    11. MMAP
+    12. ANON
+    13. SWAPCACHE
+    14. SWAPBACKED
+    15. COMPOUND_HEAD
+    16. COMPOUND_TAIL
+    17. HUGE
+    18. UNEVICTABLE
+    19. HWPOISON
+    20. NOPAGE
+    21. KSM
+    22. THP
+    23. BALLOON
+    24. ZERO_PAGE
+    25. IDLE
+
+ * ``/proc/kpagecgroup``.  This file contains a 64-bit inode number of the
+   memory cgroup each page is charged to, indexed by PFN. Only available when
+   CONFIG_MEMCG is set.
+
+Short descriptions to the page flags
+====================================
+
+0 - LOCKED
+   page is being locked for exclusive access, e.g. by undergoing read/write IO
+7 - SLAB
+   page is managed by the SLAB/SLOB/SLUB/SLQB kernel memory allocator
+   When compound page is used, SLUB/SLQB will only set this flag on the head
+   page; SLOB will not flag it at all.
+10 - BUDDY
+    a free memory block managed by the buddy system allocator
+    The buddy system organizes free memory in blocks of various orders.
+    An order N block has 2^N physically contiguous pages, with the BUDDY flag
+    set for and _only_ for the first page.
+15 - COMPOUND_HEAD
+    A compound page with order N consists of 2^N physically contiguous pages.
+    A compound page with order 2 takes the form of "HTTT", where H donates its
+    head page and T donates its tail page(s).  The major consumers of compound
+    pages are hugeTLB pages
+    (:ref:`Documentation/admin-guide/mm/hugetlbpage.rst <hugetlbpage>`),
+    the SLUB etc.  memory allocators and various device drivers.
+    However in this interface, only huge/giga pages are made visible
+    to end users.
+16 - COMPOUND_TAIL
+    A compound page tail (see description above).
+17 - HUGE
+    this is an integral part of a HugeTLB page
+19 - HWPOISON
+    hardware detected memory corruption on this page: don't touch the data!
+20 - NOPAGE
+    no page frame exists at the requested address
+21 - KSM
+    identical memory pages dynamically shared between one or more processes
+22 - THP
+    contiguous pages which construct transparent hugepages
+23 - BALLOON
+    balloon compaction page
+24 - ZERO_PAGE
+    zero page for pfn_zero or huge_zero page
+25 - IDLE
+    page has not been accessed since it was marked idle (see
+    :ref:`Documentation/admin-guide/mm/idle_page_tracking.rst <idle_page_tracking>`).
+    Note that this flag may be stale in case the page was accessed via
+    a PTE. To make sure the flag is up-to-date one has to read
+    ``/sys/kernel/mm/page_idle/bitmap`` first.
+
+IO related page flags
+---------------------
+
+1 - ERROR
+   IO error occurred
+3 - UPTODATE
+   page has up-to-date data
+   ie. for file backed page: (in-memory data revision >= on-disk one)
+4 - DIRTY
+   page has been written to, hence contains new data
+   i.e. for file backed page: (in-memory data revision >  on-disk one)
+8 - WRITEBACK
+   page is being synced to disk
+
+LRU related page flags
+----------------------
+
+5 - LRU
+   page is in one of the LRU lists
+6 - ACTIVE
+   page is in the active LRU list
+18 - UNEVICTABLE
+   page is in the unevictable (non-)LRU list It is somehow pinned and
+   not a candidate for LRU page reclaims, e.g. ramfs pages,
+   shmctl(SHM_LOCK) and mlock() memory segments
+2 - REFERENCED
+   page has been referenced since last LRU list enqueue/requeue
+9 - RECLAIM
+   page will be reclaimed soon after its pageout IO completed
+11 - MMAP
+   a memory mapped page
+12 - ANON
+   a memory mapped page that is not part of a file
+13 - SWAPCACHE
+   page is mapped to swap space, i.e. has an associated swap entry
+14 - SWAPBACKED
+   page is backed by swap/RAM
+
+The page-types tool in the tools/vm directory can be used to query the
+above flags.
+
+Using pagemap to do something useful
+====================================
+
+The general procedure for using pagemap to find out about a process' memory
+usage goes like this:
+
+ 1. Read ``/proc/pid/maps`` to determine which parts of the memory space are
+    mapped to what.
+ 2. Select the maps you are interested in -- all of them, or a particular
+    library, or the stack or the heap, etc.
+ 3. Open ``/proc/pid/pagemap`` and seek to the pages you would like to examine.
+ 4. Read a u64 for each page from pagemap.
+ 5. Open ``/proc/kpagecount`` and/or ``/proc/kpageflags``.  For each PFN you
+    just read, seek to that entry in the file, and read the data you want.
+
+For example, to find the "unique set size" (USS), which is the amount of
+memory that a process is using that is not shared with any other process,
+you can go through every map in the process, find the PFNs, look those up
+in kpagecount, and tally up the number of pages that are only referenced
+once.
+
+Other notes
+===========
+
+Reading from any of the files will return -EINVAL if you are not starting
+the read on an 8-byte boundary (e.g., if you sought an odd number of bytes
+into the file), or if the size of the read is not a multiple of 8 bytes.
+
+Before Linux 3.11 pagemap bits 55-60 were used for "page-shift" (which is
+always 12 at most architectures). Since Linux 3.11 their meaning changes
+after first clear of soft-dirty bits. Since Linux 4.2 they are used for
+flags unconditionally.
diff --git a/Documentation/admin-guide/mm/soft-dirty.rst b/Documentation/admin-guide/mm/soft-dirty.rst
new file mode 100644
index 000000000000..cb0cfd6672fa
--- /dev/null
+++ b/Documentation/admin-guide/mm/soft-dirty.rst
@@ -0,0 +1,47 @@
+.. _soft_dirty:
+
+===============
+Soft-Dirty PTEs
+===============
+
+The soft-dirty is a bit on a PTE which helps to track which pages a task
+writes to. In order to do this tracking one should
+
+  1. Clear soft-dirty bits from the task's PTEs.
+
+     This is done by writing "4" into the ``/proc/PID/clear_refs`` file of the
+     task in question.
+
+  2. Wait some time.
+
+  3. Read soft-dirty bits from the PTEs.
+
+     This is done by reading from the ``/proc/PID/pagemap``. The bit 55 of the
+     64-bit qword is the soft-dirty one. If set, the respective PTE was
+     written to since step 1.
+
+
+Internally, to do this tracking, the writable bit is cleared from PTEs
+when the soft-dirty bit is cleared. So, after this, when the task tries to
+modify a page at some virtual address the #PF occurs and the kernel sets
+the soft-dirty bit on the respective PTE.
+
+Note, that although all the task's address space is marked as r/o after the
+soft-dirty bits clear, the #PF-s that occur after that are processed fast.
+This is so, since the pages are still mapped to physical memory, and thus all
+the kernel does is finds this fact out and puts both writable and soft-dirty
+bits on the PTE.
+
+While in most cases tracking memory changes by #PF-s is more than enough
+there is still a scenario when we can lose soft dirty bits -- a task
+unmaps a previously mapped memory region and then maps a new one at exactly
+the same place. When unmap is called, the kernel internally clears PTE values
+including soft dirty bits. To notify user space application about such
+memory region renewal the kernel always marks new memory regions (and
+expanded regions) as soft dirty.
+
+This feature is actively used by the checkpoint-restore project. You
+can find more details about it on http://criu.org
+
+
+-- Pavel Emelyanov, Apr 9, 2013
diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst
new file mode 100644
index 000000000000..7ab93a8404b9
--- /dev/null
+++ b/Documentation/admin-guide/mm/transhuge.rst
@@ -0,0 +1,418 @@
+.. _admin_guide_transhuge:
+
+============================
+Transparent Hugepage Support
+============================
+
+Objective
+=========
+
+Performance critical computing applications dealing with large memory
+working sets are already running on top of libhugetlbfs and in turn
+hugetlbfs. Transparent HugePage Support (THP) is an alternative mean of
+using huge pages for the backing of virtual memory with huge pages
+that supports the automatic promotion and demotion of page sizes and
+without the shortcomings of hugetlbfs.
+
+Currently THP only works for anonymous memory mappings and tmpfs/shmem.
+But in the future it can expand to other filesystems.
+
+.. note::
+   in the examples below we presume that the basic page size is 4K and
+   the huge page size is 2M, although the actual numbers may vary
+   depending on the CPU architecture.
+
+The reason applications are running faster is because of two
+factors. The first factor is almost completely irrelevant and it's not
+of significant interest because it'll also have the downside of
+requiring larger clear-page copy-page in page faults which is a
+potentially negative effect. The first factor consists in taking a
+single page fault for each 2M virtual region touched by userland (so
+reducing the enter/exit kernel frequency by a 512 times factor). This
+only matters the first time the memory is accessed for the lifetime of
+a memory mapping. The second long lasting and much more important
+factor will affect all subsequent accesses to the memory for the whole
+runtime of the application. The second factor consist of two
+components:
+
+1) the TLB miss will run faster (especially with virtualization using
+   nested pagetables but almost always also on bare metal without
+   virtualization)
+
+2) a single TLB entry will be mapping a much larger amount of virtual
+   memory in turn reducing the number of TLB misses. With
+   virtualization and nested pagetables the TLB can be mapped of
+   larger size only if both KVM and the Linux guest are using
+   hugepages but a significant speedup already happens if only one of
+   the two is using hugepages just because of the fact the TLB miss is
+   going to run faster.
+
+THP can be enabled system wide or restricted to certain tasks or even
+memory ranges inside task's address space. Unless THP is completely
+disabled, there is ``khugepaged`` daemon that scans memory and
+collapses sequences of basic pages into huge pages.
+
+The THP behaviour is controlled via :ref:`sysfs <thp_sysfs>`
+interface and using madivse(2) and prctl(2) system calls.
+
+Transparent Hugepage Support maximizes the usefulness of free memory
+if compared to the reservation approach of hugetlbfs by allowing all
+unused memory to be used as cache or other movable (or even unmovable
+entities). It doesn't require reservation to prevent hugepage
+allocation failures to be noticeable from userland. It allows paging
+and all other advanced VM features to be available on the
+hugepages. It requires no modifications for applications to take
+advantage of it.
+
+Applications however can be further optimized to take advantage of
+this feature, like for example they've been optimized before to avoid
+a flood of mmap system calls for every malloc(4k). Optimizing userland
+is by far not mandatory and khugepaged already can take care of long
+lived page allocations even for hugepage unaware applications that
+deals with large amounts of memory.
+
+In certain cases when hugepages are enabled system wide, application
+may end up allocating more memory resources. An application may mmap a
+large region but only touch 1 byte of it, in that case a 2M page might
+be allocated instead of a 4k page for no good. This is why it's
+possible to disable hugepages system-wide and to only have them inside
+MADV_HUGEPAGE madvise regions.
+
+Embedded systems should enable hugepages only inside madvise regions
+to eliminate any risk of wasting any precious byte of memory and to
+only run faster.
+
+Applications that gets a lot of benefit from hugepages and that don't
+risk to lose memory by using hugepages, should use
+madvise(MADV_HUGEPAGE) on their critical mmapped regions.
+
+.. _thp_sysfs:
+
+sysfs
+=====
+
+Global THP controls
+-------------------
+
+Transparent Hugepage Support for anonymous memory can be entirely disabled
+(mostly for debugging purposes) or only enabled inside MADV_HUGEPAGE
+regions (to avoid the risk of consuming more memory resources) or enabled
+system wide. This can be achieved with one of::
+
+	echo always >/sys/kernel/mm/transparent_hugepage/enabled
+	echo madvise >/sys/kernel/mm/transparent_hugepage/enabled
+	echo never >/sys/kernel/mm/transparent_hugepage/enabled
+
+It's also possible to limit defrag efforts in the VM to generate
+anonymous hugepages in case they're not immediately free to madvise
+regions or to never try to defrag memory and simply fallback to regular
+pages unless hugepages are immediately available. Clearly if we spend CPU
+time to defrag memory, we would expect to gain even more by the fact we
+use hugepages later instead of regular pages. This isn't always
+guaranteed, but it may be more likely in case the allocation is for a
+MADV_HUGEPAGE region.
+
+::
+
+	echo always >/sys/kernel/mm/transparent_hugepage/defrag
+	echo defer >/sys/kernel/mm/transparent_hugepage/defrag
+	echo defer+madvise >/sys/kernel/mm/transparent_hugepage/defrag
+	echo madvise >/sys/kernel/mm/transparent_hugepage/defrag
+	echo never >/sys/kernel/mm/transparent_hugepage/defrag
+
+always
+	means that an application requesting THP will stall on
+	allocation failure and directly reclaim pages and compact
+	memory in an effort to allocate a THP immediately. This may be
+	desirable for virtual machines that benefit heavily from THP
+	use and are willing to delay the VM start to utilise them.
+
+defer
+	means that an application will wake kswapd in the background
+	to reclaim pages and wake kcompactd to compact memory so that
+	THP is available in the near future. It's the responsibility
+	of khugepaged to then install the THP pages later.
+
+defer+madvise
+	will enter direct reclaim and compaction like ``always``, but
+	only for regions that have used madvise(MADV_HUGEPAGE); all
+	other regions will wake kswapd in the background to reclaim
+	pages and wake kcompactd to compact memory so that THP is
+	available in the near future.
+
+madvise
+	will enter direct reclaim like ``always`` but only for regions
+	that are have used madvise(MADV_HUGEPAGE). This is the default
+	behaviour.
+
+never
+	should be self-explanatory.
+
+By default kernel tries to use huge zero page on read page fault to
+anonymous mapping. It's possible to disable huge zero page by writing 0
+or enable it back by writing 1::
+
+	echo 0 >/sys/kernel/mm/transparent_hugepage/use_zero_page
+	echo 1 >/sys/kernel/mm/transparent_hugepage/use_zero_page
+
+Some userspace (such as a test program, or an optimized memory allocation
+library) may want to know the size (in bytes) of a transparent hugepage::
+
+	cat /sys/kernel/mm/transparent_hugepage/hpage_pmd_size
+
+khugepaged will be automatically started when
+transparent_hugepage/enabled is set to "always" or "madvise, and it'll
+be automatically shutdown if it's set to "never".
+
+Khugepaged controls
+-------------------
+
+khugepaged runs usually at low frequency so while one may not want to
+invoke defrag algorithms synchronously during the page faults, it
+should be worth invoking defrag at least in khugepaged. However it's
+also possible to disable defrag in khugepaged by writing 0 or enable
+defrag in khugepaged by writing 1::
+
+	echo 0 >/sys/kernel/mm/transparent_hugepage/khugepaged/defrag
+	echo 1 >/sys/kernel/mm/transparent_hugepage/khugepaged/defrag
+
+You can also control how many pages khugepaged should scan at each
+pass::
+
+	/sys/kernel/mm/transparent_hugepage/khugepaged/pages_to_scan
+
+and how many milliseconds to wait in khugepaged between each pass (you
+can set this to 0 to run khugepaged at 100% utilization of one core)::
+
+	/sys/kernel/mm/transparent_hugepage/khugepaged/scan_sleep_millisecs
+
+and how many milliseconds to wait in khugepaged if there's an hugepage
+allocation failure to throttle the next allocation attempt::
+
+	/sys/kernel/mm/transparent_hugepage/khugepaged/alloc_sleep_millisecs
+
+The khugepaged progress can be seen in the number of pages collapsed::
+
+	/sys/kernel/mm/transparent_hugepage/khugepaged/pages_collapsed
+
+for each pass::
+
+	/sys/kernel/mm/transparent_hugepage/khugepaged/full_scans
+
+``max_ptes_none`` specifies how many extra small pages (that are
+not already mapped) can be allocated when collapsing a group
+of small pages into one large page::
+
+	/sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_none
+
+A higher value leads to use additional memory for programs.
+A lower value leads to gain less thp performance. Value of
+max_ptes_none can waste cpu time very little, you can
+ignore it.
+
+``max_ptes_swap`` specifies how many pages can be brought in from
+swap when collapsing a group of pages into a transparent huge page::
+
+	/sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_swap
+
+A higher value can cause excessive swap IO and waste
+memory. A lower value can prevent THPs from being
+collapsed, resulting fewer pages being collapsed into
+THPs, and lower memory access performance.
+
+Boot parameter
+==============
+
+You can change the sysfs boot time defaults of Transparent Hugepage
+Support by passing the parameter ``transparent_hugepage=always`` or
+``transparent_hugepage=madvise`` or ``transparent_hugepage=never``
+to the kernel command line.
+
+Hugepages in tmpfs/shmem
+========================
+
+You can control hugepage allocation policy in tmpfs with mount option
+``huge=``. It can have following values:
+
+always
+    Attempt to allocate huge pages every time we need a new page;
+
+never
+    Do not allocate huge pages;
+
+within_size
+    Only allocate huge page if it will be fully within i_size.
+    Also respect fadvise()/madvise() hints;
+
+advise
+    Only allocate huge pages if requested with fadvise()/madvise();
+
+The default policy is ``never``.
+
+``mount -o remount,huge= /mountpoint`` works fine after mount: remounting
+``huge=never`` will not attempt to break up huge pages at all, just stop more
+from being allocated.
+
+There's also sysfs knob to control hugepage allocation policy for internal
+shmem mount: /sys/kernel/mm/transparent_hugepage/shmem_enabled. The mount
+is used for SysV SHM, memfds, shared anonymous mmaps (of /dev/zero or
+MAP_ANONYMOUS), GPU drivers' DRM objects, Ashmem.
+
+In addition to policies listed above, shmem_enabled allows two further
+values:
+
+deny
+    For use in emergencies, to force the huge option off from
+    all mounts;
+force
+    Force the huge option on for all - very useful for testing;
+
+Need of application restart
+===========================
+
+The transparent_hugepage/enabled values and tmpfs mount option only affect
+future behavior. So to make them effective you need to restart any
+application that could have been using hugepages. This also applies to the
+regions registered in khugepaged.
+
+Monitoring usage
+================
+
+The number of anonymous transparent huge pages currently used by the
+system is available by reading the AnonHugePages field in ``/proc/meminfo``.
+To identify what applications are using anonymous transparent huge pages,
+it is necessary to read ``/proc/PID/smaps`` and count the AnonHugePages fields
+for each mapping.
+
+The number of file transparent huge pages mapped to userspace is available
+by reading ShmemPmdMapped and ShmemHugePages fields in ``/proc/meminfo``.
+To identify what applications are mapping file transparent huge pages, it
+is necessary to read ``/proc/PID/smaps`` and count the FileHugeMapped fields
+for each mapping.
+
+Note that reading the smaps file is expensive and reading it
+frequently will incur overhead.
+
+There are a number of counters in ``/proc/vmstat`` that may be used to
+monitor how successfully the system is providing huge pages for use.
+
+thp_fault_alloc
+	is incremented every time a huge page is successfully
+	allocated to handle a page fault. This applies to both the
+	first time a page is faulted and for COW faults.
+
+thp_collapse_alloc
+	is incremented by khugepaged when it has found
+	a range of pages to collapse into one huge page and has
+	successfully allocated a new huge page to store the data.
+
+thp_fault_fallback
+	is incremented if a page fault fails to allocate
+	a huge page and instead falls back to using small pages.
+
+thp_collapse_alloc_failed
+	is incremented if khugepaged found a range
+	of pages that should be collapsed into one huge page but failed
+	the allocation.
+
+thp_file_alloc
+	is incremented every time a file huge page is successfully
+	allocated.
+
+thp_file_mapped
+	is incremented every time a file huge page is mapped into
+	user address space.
+
+thp_split_page
+	is incremented every time a huge page is split into base
+	pages. This can happen for a variety of reasons but a common
+	reason is that a huge page is old and is being reclaimed.
+	This action implies splitting all PMD the page mapped with.
+
+thp_split_page_failed
+	is incremented if kernel fails to split huge
+	page. This can happen if the page was pinned by somebody.
+
+thp_deferred_split_page
+	is incremented when a huge page is put onto split
+	queue. This happens when a huge page is partially unmapped and
+	splitting it would free up some memory. Pages on split queue are
+	going to be split under memory pressure.
+
+thp_split_pmd
+	is incremented every time a PMD split into table of PTEs.
+	This can happen, for instance, when application calls mprotect() or
+	munmap() on part of huge page. It doesn't split huge page, only
+	page table entry.
+
+thp_zero_page_alloc
+	is incremented every time a huge zero page is
+	successfully allocated. It includes allocations which where
+	dropped due race with other allocation. Note, it doesn't count
+	every map of the huge zero page, only its allocation.
+
+thp_zero_page_alloc_failed
+	is incremented if kernel fails to allocate
+	huge zero page and falls back to using small pages.
+
+thp_swpout
+	is incremented every time a huge page is swapout in one
+	piece without splitting.
+
+thp_swpout_fallback
+	is incremented if a huge page has to be split before swapout.
+	Usually because failed to allocate some continuous swap space
+	for the huge page.
+
+As the system ages, allocating huge pages may be expensive as the
+system uses memory compaction to copy data around memory to free a
+huge page for use. There are some counters in ``/proc/vmstat`` to help
+monitor this overhead.
+
+compact_stall
+	is incremented every time a process stalls to run
+	memory compaction so that a huge page is free for use.
+
+compact_success
+	is incremented if the system compacted memory and
+	freed a huge page for use.
+
+compact_fail
+	is incremented if the system tries to compact memory
+	but failed.
+
+compact_pages_moved
+	is incremented each time a page is moved. If
+	this value is increasing rapidly, it implies that the system
+	is copying a lot of data to satisfy the huge page allocation.
+	It is possible that the cost of copying exceeds any savings
+	from reduced TLB misses.
+
+compact_pagemigrate_failed
+	is incremented when the underlying mechanism
+	for moving a page failed.
+
+compact_blocks_moved
+	is incremented each time memory compaction examines
+	a huge page aligned range of pages.
+
+It is possible to establish how long the stalls were using the function
+tracer to record how long was spent in __alloc_pages_nodemask and
+using the mm_page_alloc tracepoint to identify which allocations were
+for huge pages.
+
+Optimizing the applications
+===========================
+
+To be guaranteed that the kernel will map a 2M page immediately in any
+memory region, the mmap region has to be hugepage naturally
+aligned. posix_memalign() can provide that guarantee.
+
+Hugetlbfs
+=========
+
+You can use hugetlbfs on a kernel that has transparent hugepage
+support enabled just fine as always. No difference can be noted in
+hugetlbfs other than there will be less overall fragmentation. All
+usual features belonging to hugetlbfs are preserved and
+unaffected. libhugetlbfs will also work fine as usual.
diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst
new file mode 100644
index 000000000000..5048cf661a8a
--- /dev/null
+++ b/Documentation/admin-guide/mm/userfaultfd.rst
@@ -0,0 +1,241 @@
+.. _userfaultfd:
+
+===========
+Userfaultfd
+===========
+
+Objective
+=========
+
+Userfaults allow the implementation of on-demand paging from userland
+and more generally they allow userland to take control of various
+memory page faults, something otherwise only the kernel code could do.
+
+For example userfaults allows a proper and more optimal implementation
+of the PROT_NONE+SIGSEGV trick.
+
+Design
+======
+
+Userfaults are delivered and resolved through the userfaultfd syscall.
+
+The userfaultfd (aside from registering and unregistering virtual
+memory ranges) provides two primary functionalities:
+
+1) read/POLLIN protocol to notify a userland thread of the faults
+   happening
+
+2) various UFFDIO_* ioctls that can manage the virtual memory regions
+   registered in the userfaultfd that allows userland to efficiently
+   resolve the userfaults it receives via 1) or to manage the virtual
+   memory in the background
+
+The real advantage of userfaults if compared to regular virtual memory
+management of mremap/mprotect is that the userfaults in all their
+operations never involve heavyweight structures like vmas (in fact the
+userfaultfd runtime load never takes the mmap_sem for writing).
+
+Vmas are not suitable for page- (or hugepage) granular fault tracking
+when dealing with virtual address spaces that could span
+Terabytes. Too many vmas would be needed for that.
+
+The userfaultfd once opened by invoking the syscall, can also be
+passed using unix domain sockets to a manager process, so the same
+manager process could handle the userfaults of a multitude of
+different processes without them being aware about what is going on
+(well of course unless they later try to use the userfaultfd
+themselves on the same region the manager is already tracking, which
+is a corner case that would currently return -EBUSY).
+
+API
+===
+
+When first opened the userfaultfd must be enabled invoking the
+UFFDIO_API ioctl specifying a uffdio_api.api value set to UFFD_API (or
+a later API version) which will specify the read/POLLIN protocol
+userland intends to speak on the UFFD and the uffdio_api.features
+userland requires. The UFFDIO_API ioctl if successful (i.e. if the
+requested uffdio_api.api is spoken also by the running kernel and the
+requested features are going to be enabled) will return into
+uffdio_api.features and uffdio_api.ioctls two 64bit bitmasks of
+respectively all the available features of the read(2) protocol and
+the generic ioctl available.
+
+The uffdio_api.features bitmask returned by the UFFDIO_API ioctl
+defines what memory types are supported by the userfaultfd and what
+events, except page fault notifications, may be generated.
+
+If the kernel supports registering userfaultfd ranges on hugetlbfs
+virtual memory areas, UFFD_FEATURE_MISSING_HUGETLBFS will be set in
+uffdio_api.features. Similarly, UFFD_FEATURE_MISSING_SHMEM will be
+set if the kernel supports registering userfaultfd ranges on shared
+memory (covering all shmem APIs, i.e. tmpfs, IPCSHM, /dev/zero
+MAP_SHARED, memfd_create, etc).
+
+The userland application that wants to use userfaultfd with hugetlbfs
+or shared memory need to set the corresponding flag in
+uffdio_api.features to enable those features.
+
+If the userland desires to receive notifications for events other than
+page faults, it has to verify that uffdio_api.features has appropriate
+UFFD_FEATURE_EVENT_* bits set. These events are described in more
+detail below in "Non-cooperative userfaultfd" section.
+
+Once the userfaultfd has been enabled the UFFDIO_REGISTER ioctl should
+be invoked (if present in the returned uffdio_api.ioctls bitmask) to
+register a memory range in the userfaultfd by setting the
+uffdio_register structure accordingly. The uffdio_register.mode
+bitmask will specify to the kernel which kind of faults to track for
+the range (UFFDIO_REGISTER_MODE_MISSING would track missing
+pages). The UFFDIO_REGISTER ioctl will return the
+uffdio_register.ioctls bitmask of ioctls that are suitable to resolve
+userfaults on the range registered. Not all ioctls will necessarily be
+supported for all memory types depending on the underlying virtual
+memory backend (anonymous memory vs tmpfs vs real filebacked
+mappings).
+
+Userland can use the uffdio_register.ioctls to manage the virtual
+address space in the background (to add or potentially also remove
+memory from the userfaultfd registered range). This means a userfault
+could be triggering just before userland maps in the background the
+user-faulted page.
+
+The primary ioctl to resolve userfaults is UFFDIO_COPY. That
+atomically copies a page into the userfault registered range and wakes
+up the blocked userfaults (unless uffdio_copy.mode &
+UFFDIO_COPY_MODE_DONTWAKE is set). Other ioctl works similarly to
+UFFDIO_COPY. They're atomic as in guaranteeing that nothing can see an
+half copied page since it'll keep userfaulting until the copy has
+finished.
+
+QEMU/KVM
+========
+
+QEMU/KVM is using the userfaultfd syscall to implement postcopy live
+migration. Postcopy live migration is one form of memory
+externalization consisting of a virtual machine running with part or
+all of its memory residing on a different node in the cloud. The
+userfaultfd abstraction is generic enough that not a single line of
+KVM kernel code had to be modified in order to add postcopy live
+migration to QEMU.
+
+Guest async page faults, FOLL_NOWAIT and all other GUP features work
+just fine in combination with userfaults. Userfaults trigger async
+page faults in the guest scheduler so those guest processes that
+aren't waiting for userfaults (i.e. network bound) can keep running in
+the guest vcpus.
+
+It is generally beneficial to run one pass of precopy live migration
+just before starting postcopy live migration, in order to avoid
+generating userfaults for readonly guest regions.
+
+The implementation of postcopy live migration currently uses one
+single bidirectional socket but in the future two different sockets
+will be used (to reduce the latency of the userfaults to the minimum
+possible without having to decrease /proc/sys/net/ipv4/tcp_wmem).
+
+The QEMU in the source node writes all pages that it knows are missing
+in the destination node, into the socket, and the migration thread of
+the QEMU running in the destination node runs UFFDIO_COPY|ZEROPAGE
+ioctls on the userfaultfd in order to map the received pages into the
+guest (UFFDIO_ZEROCOPY is used if the source page was a zero page).
+
+A different postcopy thread in the destination node listens with
+poll() to the userfaultfd in parallel. When a POLLIN event is
+generated after a userfault triggers, the postcopy thread read() from
+the userfaultfd and receives the fault address (or -EAGAIN in case the
+userfault was already resolved and waken by a UFFDIO_COPY|ZEROPAGE run
+by the parallel QEMU migration thread).
+
+After the QEMU postcopy thread (running in the destination node) gets
+the userfault address it writes the information about the missing page
+into the socket. The QEMU source node receives the information and
+roughly "seeks" to that page address and continues sending all
+remaining missing pages from that new page offset. Soon after that
+(just the time to flush the tcp_wmem queue through the network) the
+migration thread in the QEMU running in the destination node will
+receive the page that triggered the userfault and it'll map it as
+usual with the UFFDIO_COPY|ZEROPAGE (without actually knowing if it
+was spontaneously sent by the source or if it was an urgent page
+requested through a userfault).
+
+By the time the userfaults start, the QEMU in the destination node
+doesn't need to keep any per-page state bitmap relative to the live
+migration around and a single per-page bitmap has to be maintained in
+the QEMU running in the source node to know which pages are still
+missing in the destination node. The bitmap in the source node is
+checked to find which missing pages to send in round robin and we seek
+over it when receiving incoming userfaults. After sending each page of
+course the bitmap is updated accordingly. It's also useful to avoid
+sending the same page twice (in case the userfault is read by the
+postcopy thread just before UFFDIO_COPY|ZEROPAGE runs in the migration
+thread).
+
+Non-cooperative userfaultfd
+===========================
+
+When the userfaultfd is monitored by an external manager, the manager
+must be able to track changes in the process virtual memory
+layout. Userfaultfd can notify the manager about such changes using
+the same read(2) protocol as for the page fault notifications. The
+manager has to explicitly enable these events by setting appropriate
+bits in uffdio_api.features passed to UFFDIO_API ioctl:
+
+UFFD_FEATURE_EVENT_FORK
+	enable userfaultfd hooks for fork(). When this feature is
+	enabled, the userfaultfd context of the parent process is
+	duplicated into the newly created process. The manager
+	receives UFFD_EVENT_FORK with file descriptor of the new
+	userfaultfd context in the uffd_msg.fork.
+
+UFFD_FEATURE_EVENT_REMAP
+	enable notifications about mremap() calls. When the
+	non-cooperative process moves a virtual memory area to a
+	different location, the manager will receive
+	UFFD_EVENT_REMAP. The uffd_msg.remap will contain the old and
+	new addresses of the area and its original length.
+
+UFFD_FEATURE_EVENT_REMOVE
+	enable notifications about madvise(MADV_REMOVE) and
+	madvise(MADV_DONTNEED) calls. The event UFFD_EVENT_REMOVE will
+	be generated upon these calls to madvise. The uffd_msg.remove
+	will contain start and end addresses of the removed area.
+
+UFFD_FEATURE_EVENT_UNMAP
+	enable notifications about memory unmapping. The manager will
+	get UFFD_EVENT_UNMAP with uffd_msg.remove containing start and
+	end addresses of the unmapped area.
+
+Although the UFFD_FEATURE_EVENT_REMOVE and UFFD_FEATURE_EVENT_UNMAP
+are pretty similar, they quite differ in the action expected from the
+userfaultfd manager. In the former case, the virtual memory is
+removed, but the area is not, the area remains monitored by the
+userfaultfd, and if a page fault occurs in that area it will be
+delivered to the manager. The proper resolution for such page fault is
+to zeromap the faulting address. However, in the latter case, when an
+area is unmapped, either explicitly (with munmap() system call), or
+implicitly (e.g. during mremap()), the area is removed and in turn the
+userfaultfd context for such area disappears too and the manager will
+not get further userland page faults from the removed area. Still, the
+notification is required in order to prevent manager from using
+UFFDIO_COPY on the unmapped area.
+
+Unlike userland page faults which have to be synchronous and require
+explicit or implicit wakeup, all the events are delivered
+asynchronously and the non-cooperative process resumes execution as
+soon as manager executes read(). The userfaultfd manager should
+carefully synchronize calls to UFFDIO_COPY with the events
+processing. To aid the synchronization, the UFFDIO_COPY ioctl will
+return -ENOSPC when the monitored process exits at the time of
+UFFDIO_COPY, and -ENOENT, when the non-cooperative process has changed
+its virtual memory layout simultaneously with outstanding UFFDIO_COPY
+operation.
+
+The current asynchronous model of the event delivery is optimal for
+single threaded non-cooperative userfaultfd manager implementations. A
+synchronous event delivery model can be added later as a new
+userfaultfd feature to facilitate multithreading enhancements of the
+non cooperative manager, for example to allow UFFDIO_COPY ioctls to
+run in parallel to the event reception. Single threaded
+implementations should continue to use the current async event
+delivery model instead.
diff --git a/Documentation/admin-guide/ramoops.rst b/Documentation/admin-guide/ramoops.rst
index 4efd7ce77565..6dbcc5481000 100644
--- a/Documentation/admin-guide/ramoops.rst
+++ b/Documentation/admin-guide/ramoops.rst
@@ -61,7 +61,7 @@ Setting the ramoops parameters can be done in several different manners:
 	mem=128M ramoops.mem_address=0x8000000 ramoops.ecc=1
 
  B. Use Device Tree bindings, as described in
- ``Documentation/device-tree/bindings/reserved-memory/admin-guide/ramoops.rst``.
+ ``Documentation/devicetree/bindings/reserved-memory/ramoops.txt``.
  For example::
 
 	reserved-memory {
diff --git a/Documentation/arm/Marvell/README b/Documentation/arm/Marvell/README
index b5bb7f518840..56ada27c53be 100644
--- a/Documentation/arm/Marvell/README
+++ b/Documentation/arm/Marvell/README
@@ -302,19 +302,15 @@ Berlin family (Multimedia Solutions)
 	88DE3010, Armada 1000 (no Linux support)
 		Core:		Marvell PJ1 (ARMv5TE), Dual-core
 		Product Brief:	http://www.marvell.com.cn/digital-entertainment/assets/armada_1000_pb.pdf
-	88DE3005, Armada 1500-mini
 	88DE3005, Armada 1500 Mini
 		Design name:	BG2CD
 		Core:		ARM Cortex-A9, PL310 L2CC
-		Homepage:	http://www.marvell.com/multimedia-solutions/armada-1500-mini/
-        88DE3006, Armada 1500 Mini Plus
-                Design name:    BG2CDP
-                Core:           Dual Core ARM Cortex-A7
-                Homepage:       http://www.marvell.com/multimedia-solutions/armada-1500-mini-plus/
+	88DE3006, Armada 1500 Mini Plus
+		Design name:	BG2CDP
+		Core:		Dual Core ARM Cortex-A7
 	88DE3100, Armada 1500
 		Design name:	BG2
 		Core:		Marvell PJ4B-MP (ARMv7), Tauros3 L2CC
-		Product Brief:	http://www.marvell.com/digital-entertainment/armada-1500/assets/Marvell-ARMADA-1500-Product-Brief.pdf
 	88DE3114, Armada 1500 Pro
 		Design name:	BG2Q
 		Core:		Quad Core ARM Cortex-A9, PL310 L2CC
@@ -324,13 +320,16 @@ Berlin family (Multimedia Solutions)
 	88DE3218, ARMADA 1500 Ultra
 		Core:		ARM Cortex-A53
 
-  Homepage: http://www.marvell.com/multimedia-solutions/
+  Homepage: https://www.synaptics.com/products/multimedia-solutions
   Directory: arch/arm/mach-berlin
 
   Comments:
+
    * This line of SoCs is based on Marvell Sheeva or ARM Cortex CPUs
      with Synopsys DesignWare (IRQ, GPIO, Timers, ...) and PXA IP (SDHCI, USB, ETH, ...).
 
+   * The Berlin family was acquired by Synaptics from Marvell in 2017.
+
 CPU Cores
 ---------
 
diff --git a/Documentation/arm/OMAP/README b/Documentation/arm/OMAP/README
index 75645c45d14a..90c6c57d61e8 100644
--- a/Documentation/arm/OMAP/README
+++ b/Documentation/arm/OMAP/README
@@ -5,3 +5,7 @@ KERNEL		NEW DEPENDENCIES
 v4.3+		Update is needed for custom .config files to make sure
 		CONFIG_REGULATOR_PBIAS is enabled for MMC1 to work
 		properly.
+
+v4.18+		Update is needed for custom .config files to make sure
+		CONFIG_MMC_SDHCI_OMAP is enabled for all MMC instances
+		to work in DRA7 and K2G based boards.
diff --git a/Documentation/bcache.txt b/Documentation/bcache.txt
deleted file mode 100644
index c0ce64d75bbf..000000000000
--- a/Documentation/bcache.txt
+++ /dev/null
@@ -1,649 +0,0 @@
-============================
-A block layer cache (bcache)
-============================
-
-Say you've got a big slow raid 6, and an ssd or three. Wouldn't it be
-nice if you could use them as cache... Hence bcache.
-
-Wiki and git repositories are at:
-
-  - http://bcache.evilpiepirate.org
-  - http://evilpiepirate.org/git/linux-bcache.git
-  - http://evilpiepirate.org/git/bcache-tools.git
-
-It's designed around the performance characteristics of SSDs - it only allocates
-in erase block sized buckets, and it uses a hybrid btree/log to track cached
-extents (which can be anywhere from a single sector to the bucket size). It's
-designed to avoid random writes at all costs; it fills up an erase block
-sequentially, then issues a discard before reusing it.
-
-Both writethrough and writeback caching are supported. Writeback defaults to
-off, but can be switched on and off arbitrarily at runtime. Bcache goes to
-great lengths to protect your data - it reliably handles unclean shutdown. (It
-doesn't even have a notion of a clean shutdown; bcache simply doesn't return
-writes as completed until they're on stable storage).
-
-Writeback caching can use most of the cache for buffering writes - writing
-dirty data to the backing device is always done sequentially, scanning from the
-start to the end of the index.
-
-Since random IO is what SSDs excel at, there generally won't be much benefit
-to caching large sequential IO. Bcache detects sequential IO and skips it;
-it also keeps a rolling average of the IO sizes per task, and as long as the
-average is above the cutoff it will skip all IO from that task - instead of
-caching the first 512k after every seek. Backups and large file copies should
-thus entirely bypass the cache.
-
-In the event of a data IO error on the flash it will try to recover by reading
-from disk or invalidating cache entries.  For unrecoverable errors (meta data
-or dirty data), caching is automatically disabled; if dirty data was present
-in the cache it first disables writeback caching and waits for all dirty data
-to be flushed.
-
-Getting started:
-You'll need make-bcache from the bcache-tools repository. Both the cache device
-and backing device must be formatted before use::
-
-  make-bcache -B /dev/sdb
-  make-bcache -C /dev/sdc
-
-make-bcache has the ability to format multiple devices at the same time - if
-you format your backing devices and cache device at the same time, you won't
-have to manually attach::
-
-  make-bcache -B /dev/sda /dev/sdb -C /dev/sdc
-
-bcache-tools now ships udev rules, and bcache devices are known to the kernel
-immediately.  Without udev, you can manually register devices like this::
-
-  echo /dev/sdb > /sys/fs/bcache/register
-  echo /dev/sdc > /sys/fs/bcache/register
-
-Registering the backing device makes the bcache device show up in /dev; you can
-now format it and use it as normal. But the first time using a new bcache
-device, it'll be running in passthrough mode until you attach it to a cache.
-If you are thinking about using bcache later, it is recommended to setup all your
-slow devices as bcache backing devices without a cache, and you can choose to add
-a caching device later.
-See 'ATTACHING' section below.
-
-The devices show up as::
-
-  /dev/bcache<N>
-
-As well as (with udev)::
-
-  /dev/bcache/by-uuid/<uuid>
-  /dev/bcache/by-label/<label>
-
-To get started::
-
-  mkfs.ext4 /dev/bcache0
-  mount /dev/bcache0 /mnt
-
-You can control bcache devices through sysfs at /sys/block/bcache<N>/bcache .
-You can also control them through /sys/fs//bcache/<cset-uuid>/ .
-
-Cache devices are managed as sets; multiple caches per set isn't supported yet
-but will allow for mirroring of metadata and dirty data in the future. Your new
-cache set shows up as /sys/fs/bcache/<UUID>
-
-Attaching
----------
-
-After your cache device and backing device are registered, the backing device
-must be attached to your cache set to enable caching. Attaching a backing
-device to a cache set is done thusly, with the UUID of the cache set in
-/sys/fs/bcache::
-
-  echo <CSET-UUID> > /sys/block/bcache0/bcache/attach
-
-This only has to be done once. The next time you reboot, just reregister all
-your bcache devices. If a backing device has data in a cache somewhere, the
-/dev/bcache<N> device won't be created until the cache shows up - particularly
-important if you have writeback caching turned on.
-
-If you're booting up and your cache device is gone and never coming back, you
-can force run the backing device::
-
-  echo 1 > /sys/block/sdb/bcache/running
-
-(You need to use /sys/block/sdb (or whatever your backing device is called), not
-/sys/block/bcache0, because bcache0 doesn't exist yet. If you're using a
-partition, the bcache directory would be at /sys/block/sdb/sdb2/bcache)
-
-The backing device will still use that cache set if it shows up in the future,
-but all the cached data will be invalidated. If there was dirty data in the
-cache, don't expect the filesystem to be recoverable - you will have massive
-filesystem corruption, though ext4's fsck does work miracles.
-
-Error Handling
---------------
-
-Bcache tries to transparently handle IO errors to/from the cache device without
-affecting normal operation; if it sees too many errors (the threshold is
-configurable, and defaults to 0) it shuts down the cache device and switches all
-the backing devices to passthrough mode.
-
- - For reads from the cache, if they error we just retry the read from the
-   backing device.
-
- - For writethrough writes, if the write to the cache errors we just switch to
-   invalidating the data at that lba in the cache (i.e. the same thing we do for
-   a write that bypasses the cache)
-
- - For writeback writes, we currently pass that error back up to the
-   filesystem/userspace. This could be improved - we could retry it as a write
-   that skips the cache so we don't have to error the write.
-
- - When we detach, we first try to flush any dirty data (if we were running in
-   writeback mode). It currently doesn't do anything intelligent if it fails to
-   read some of the dirty data, though.
-
-
-Howto/cookbook
---------------
-
-A) Starting a bcache with a missing caching device
-
-If registering the backing device doesn't help, it's already there, you just need
-to force it to run without the cache::
-
-	host:~# echo /dev/sdb1 > /sys/fs/bcache/register
-	[  119.844831] bcache: register_bcache() error opening /dev/sdb1: device already registered
-
-Next, you try to register your caching device if it's present. However
-if it's absent, or registration fails for some reason, you can still
-start your bcache without its cache, like so::
-
-	host:/sys/block/sdb/sdb1/bcache# echo 1 > running
-
-Note that this may cause data loss if you were running in writeback mode.
-
-
-B) Bcache does not find its cache::
-
-	host:/sys/block/md5/bcache# echo 0226553a-37cf-41d5-b3ce-8b1e944543a8 > attach
-	[ 1933.455082] bcache: bch_cached_dev_attach() Couldn't find uuid for md5 in set
-	[ 1933.478179] bcache: __cached_dev_store() Can't attach 0226553a-37cf-41d5-b3ce-8b1e944543a8
-	[ 1933.478179] : cache set not found
-
-In this case, the caching device was simply not registered at boot
-or disappeared and came back, and needs to be (re-)registered::
-
-	host:/sys/block/md5/bcache# echo /dev/sdh2 > /sys/fs/bcache/register
-
-
-C) Corrupt bcache crashes the kernel at device registration time:
-
-This should never happen.  If it does happen, then you have found a bug!
-Please report it to the bcache development list: linux-bcache@vger.kernel.org
-
-Be sure to provide as much information that you can including kernel dmesg
-output if available so that we may assist.
-
-
-D) Recovering data without bcache:
-
-If bcache is not available in the kernel, a filesystem on the backing
-device is still available at an 8KiB offset. So either via a loopdev
-of the backing device created with --offset 8K, or any value defined by
---data-offset when you originally formatted bcache with `make-bcache`.
-
-For example::
-
-	losetup -o 8192 /dev/loop0 /dev/your_bcache_backing_dev
-
-This should present your unmodified backing device data in /dev/loop0
-
-If your cache is in writethrough mode, then you can safely discard the
-cache device without loosing data.
-
-
-E) Wiping a cache device
-
-::
-
-	host:~# wipefs -a /dev/sdh2
-	16 bytes were erased at offset 0x1018 (bcache)
-	they were: c6 85 73 f6 4e 1a 45 ca 82 65 f5 7f 48 ba 6d 81
-
-After you boot back with bcache enabled, you recreate the cache and attach it::
-
-	host:~# make-bcache -C /dev/sdh2
-	UUID:                   7be7e175-8f4c-4f99-94b2-9c904d227045
-	Set UUID:               5bc072a8-ab17-446d-9744-e247949913c1
-	version:                0
-	nbuckets:               106874
-	block_size:             1
-	bucket_size:            1024
-	nr_in_set:              1
-	nr_this_dev:            0
-	first_bucket:           1
-	[  650.511912] bcache: run_cache_set() invalidating existing data
-	[  650.549228] bcache: register_cache() registered cache device sdh2
-
-start backing device with missing cache::
-
-	host:/sys/block/md5/bcache# echo 1 > running
-
-attach new cache::
-
-	host:/sys/block/md5/bcache# echo 5bc072a8-ab17-446d-9744-e247949913c1 > attach
-	[  865.276616] bcache: bch_cached_dev_attach() Caching md5 as bcache0 on set 5bc072a8-ab17-446d-9744-e247949913c1
-
-
-F) Remove or replace a caching device::
-
-	host:/sys/block/sda/sda7/bcache# echo 1 > detach
-	[  695.872542] bcache: cached_dev_detach_finish() Caching disabled for sda7
-
-	host:~# wipefs -a /dev/nvme0n1p4
-	wipefs: error: /dev/nvme0n1p4: probing initialization failed: Device or resource busy
-	Ooops, it's disabled, but not unregistered, so it's still protected
-
-We need to go and unregister it::
-
-	host:/sys/fs/bcache/b7ba27a1-2398-4649-8ae3-0959f57ba128# ls -l cache0
-	lrwxrwxrwx 1 root root 0 Feb 25 18:33 cache0 -> ../../../devices/pci0000:00/0000:00:1d.0/0000:70:00.0/nvme/nvme0/nvme0n1/nvme0n1p4/bcache/
-	host:/sys/fs/bcache/b7ba27a1-2398-4649-8ae3-0959f57ba128# echo 1 > stop
-	kernel: [  917.041908] bcache: cache_set_free() Cache set b7ba27a1-2398-4649-8ae3-0959f57ba128 unregistered
-
-Now we can wipe it::
-
-	host:~# wipefs -a /dev/nvme0n1p4
-	/dev/nvme0n1p4: 16 bytes were erased at offset 0x00001018 (bcache): c6 85 73 f6 4e 1a 45 ca 82 65 f5 7f 48 ba 6d 81
-
-
-G) dm-crypt and bcache
-
-First setup bcache unencrypted and then install dmcrypt on top of
-/dev/bcache<N> This will work faster than if you dmcrypt both the backing
-and caching devices and then install bcache on top. [benchmarks?]
-
-
-H) Stop/free a registered bcache to wipe and/or recreate it
-
-Suppose that you need to free up all bcache references so that you can
-fdisk run and re-register a changed partition table, which won't work
-if there are any active backing or caching devices left on it:
-
-1) Is it present in /dev/bcache* ? (there are times where it won't be)
-
-   If so, it's easy::
-
-	host:/sys/block/bcache0/bcache# echo 1 > stop
-
-2) But if your backing device is gone, this won't work::
-
-	host:/sys/block/bcache0# cd bcache
-	bash: cd: bcache: No such file or directory
-
-   In this case, you may have to unregister the dmcrypt block device that
-   references this bcache to free it up::
-
-	host:~# dmsetup remove oldds1
-	bcache: bcache_device_free() bcache0 stopped
-	bcache: cache_set_free() Cache set 5bc072a8-ab17-446d-9744-e247949913c1 unregistered
-
-   This causes the backing bcache to be removed from /sys/fs/bcache and
-   then it can be reused.  This would be true of any block device stacking
-   where bcache is a lower device.
-
-3) In other cases, you can also look in /sys/fs/bcache/::
-
-	host:/sys/fs/bcache# ls -l */{cache?,bdev?}
-	lrwxrwxrwx 1 root root 0 Mar  5 09:39 0226553a-37cf-41d5-b3ce-8b1e944543a8/bdev1 -> ../../../devices/virtual/block/dm-1/bcache/
-	lrwxrwxrwx 1 root root 0 Mar  5 09:39 0226553a-37cf-41d5-b3ce-8b1e944543a8/cache0 -> ../../../devices/virtual/block/dm-4/bcache/
-	lrwxrwxrwx 1 root root 0 Mar  5 09:39 5bc072a8-ab17-446d-9744-e247949913c1/cache0 -> ../../../devices/pci0000:00/0000:00:01.0/0000:01:00.0/ata10/host9/target9:0:0/9:0:0:0/block/sdl/sdl2/bcache/
-
-   The device names will show which UUID is relevant, cd in that directory
-   and stop the cache::
-
-	host:/sys/fs/bcache/5bc072a8-ab17-446d-9744-e247949913c1# echo 1 > stop
-
-   This will free up bcache references and let you reuse the partition for
-   other purposes.
-
-
-
-Troubleshooting performance
----------------------------
-
-Bcache has a bunch of config options and tunables. The defaults are intended to
-be reasonable for typical desktop and server workloads, but they're not what you
-want for getting the best possible numbers when benchmarking.
-
- - Backing device alignment
-
-   The default metadata size in bcache is 8k.  If your backing device is
-   RAID based, then be sure to align this by a multiple of your stride
-   width using `make-bcache --data-offset`. If you intend to expand your
-   disk array in the future, then multiply a series of primes by your
-   raid stripe size to get the disk multiples that you would like.
-
-   For example:  If you have a 64k stripe size, then the following offset
-   would provide alignment for many common RAID5 data spindle counts::
-
-	64k * 2*2*2*3*3*5*7 bytes = 161280k
-
-   That space is wasted, but for only 157.5MB you can grow your RAID 5
-   volume to the following data-spindle counts without re-aligning::
-
-	3,4,5,6,7,8,9,10,12,14,15,18,20,21 ...
-
- - Bad write performance
-
-   If write performance is not what you expected, you probably wanted to be
-   running in writeback mode, which isn't the default (not due to a lack of
-   maturity, but simply because in writeback mode you'll lose data if something
-   happens to your SSD)::
-
-	# echo writeback > /sys/block/bcache0/bcache/cache_mode
-
- - Bad performance, or traffic not going to the SSD that you'd expect
-
-   By default, bcache doesn't cache everything. It tries to skip sequential IO -
-   because you really want to be caching the random IO, and if you copy a 10
-   gigabyte file you probably don't want that pushing 10 gigabytes of randomly
-   accessed data out of your cache.
-
-   But if you want to benchmark reads from cache, and you start out with fio
-   writing an 8 gigabyte test file - so you want to disable that::
-
-	# echo 0 > /sys/block/bcache0/bcache/sequential_cutoff
-
-   To set it back to the default (4 mb), do::
-
-	# echo 4M > /sys/block/bcache0/bcache/sequential_cutoff
-
- - Traffic's still going to the spindle/still getting cache misses
-
-   In the real world, SSDs don't always keep up with disks - particularly with
-   slower SSDs, many disks being cached by one SSD, or mostly sequential IO. So
-   you want to avoid being bottlenecked by the SSD and having it slow everything
-   down.
-
-   To avoid that bcache tracks latency to the cache device, and gradually
-   throttles traffic if the latency exceeds a threshold (it does this by
-   cranking down the sequential bypass).
-
-   You can disable this if you need to by setting the thresholds to 0::
-
-	# echo 0 > /sys/fs/bcache/<cache set>/congested_read_threshold_us
-	# echo 0 > /sys/fs/bcache/<cache set>/congested_write_threshold_us
-
-   The default is 2000 us (2 milliseconds) for reads, and 20000 for writes.
-
- - Still getting cache misses, of the same data
-
-   One last issue that sometimes trips people up is actually an old bug, due to
-   the way cache coherency is handled for cache misses. If a btree node is full,
-   a cache miss won't be able to insert a key for the new data and the data
-   won't be written to the cache.
-
-   In practice this isn't an issue because as soon as a write comes along it'll
-   cause the btree node to be split, and you need almost no write traffic for
-   this to not show up enough to be noticeable (especially since bcache's btree
-   nodes are huge and index large regions of the device). But when you're
-   benchmarking, if you're trying to warm the cache by reading a bunch of data
-   and there's no other traffic - that can be a problem.
-
-   Solution: warm the cache by doing writes, or use the testing branch (there's
-   a fix for the issue there).
-
-
-Sysfs - backing device
-----------------------
-
-Available at /sys/block/<bdev>/bcache, /sys/block/bcache*/bcache and
-(if attached) /sys/fs/bcache/<cset-uuid>/bdev*
-
-attach
-  Echo the UUID of a cache set to this file to enable caching.
-
-cache_mode
-  Can be one of either writethrough, writeback, writearound or none.
-
-clear_stats
-  Writing to this file resets the running total stats (not the day/hour/5 minute
-  decaying versions).
-
-detach
-  Write to this file to detach from a cache set. If there is dirty data in the
-  cache, it will be flushed first.
-
-dirty_data
-  Amount of dirty data for this backing device in the cache. Continuously
-  updated unlike the cache set's version, but may be slightly off.
-
-label
-  Name of underlying device.
-
-readahead
-  Size of readahead that should be performed.  Defaults to 0.  If set to e.g.
-  1M, it will round cache miss reads up to that size, but without overlapping
-  existing cache entries.
-
-running
-  1 if bcache is running (i.e. whether the /dev/bcache device exists, whether
-  it's in passthrough mode or caching).
-
-sequential_cutoff
-  A sequential IO will bypass the cache once it passes this threshold; the
-  most recent 128 IOs are tracked so sequential IO can be detected even when
-  it isn't all done at once.
-
-sequential_merge
-  If non zero, bcache keeps a list of the last 128 requests submitted to compare
-  against all new requests to determine which new requests are sequential
-  continuations of previous requests for the purpose of determining sequential
-  cutoff. This is necessary if the sequential cutoff value is greater than the
-  maximum acceptable sequential size for any single request.
-
-state
-  The backing device can be in one of four different states:
-
-  no cache: Has never been attached to a cache set.
-
-  clean: Part of a cache set, and there is no cached dirty data.
-
-  dirty: Part of a cache set, and there is cached dirty data.
-
-  inconsistent: The backing device was forcibly run by the user when there was
-  dirty data cached but the cache set was unavailable; whatever data was on the
-  backing device has likely been corrupted.
-
-stop
-  Write to this file to shut down the bcache device and close the backing
-  device.
-
-writeback_delay
-  When dirty data is written to the cache and it previously did not contain
-  any, waits some number of seconds before initiating writeback. Defaults to
-  30.
-
-writeback_percent
-  If nonzero, bcache tries to keep around this percentage of the cache dirty by
-  throttling background writeback and using a PD controller to smoothly adjust
-  the rate.
-
-writeback_rate
-  Rate in sectors per second - if writeback_percent is nonzero, background
-  writeback is throttled to this rate. Continuously adjusted by bcache but may
-  also be set by the user.
-
-writeback_running
-  If off, writeback of dirty data will not take place at all. Dirty data will
-  still be added to the cache until it is mostly full; only meant for
-  benchmarking. Defaults to on.
-
-Sysfs - backing device stats
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-There are directories with these numbers for a running total, as well as
-versions that decay over the past day, hour and 5 minutes; they're also
-aggregated in the cache set directory as well.
-
-bypassed
-  Amount of IO (both reads and writes) that has bypassed the cache
-
-cache_hits, cache_misses, cache_hit_ratio
-  Hits and misses are counted per individual IO as bcache sees them; a
-  partial hit is counted as a miss.
-
-cache_bypass_hits, cache_bypass_misses
-  Hits and misses for IO that is intended to skip the cache are still counted,
-  but broken out here.
-
-cache_miss_collisions
-  Counts instances where data was going to be inserted into the cache from a
-  cache miss, but raced with a write and data was already present (usually 0
-  since the synchronization for cache misses was rewritten)
-
-cache_readaheads
-  Count of times readahead occurred.
-
-Sysfs - cache set
-~~~~~~~~~~~~~~~~~
-
-Available at /sys/fs/bcache/<cset-uuid>
-
-average_key_size
-  Average data per key in the btree.
-
-bdev<0..n>
-  Symlink to each of the attached backing devices.
-
-block_size
-  Block size of the cache devices.
-
-btree_cache_size
-  Amount of memory currently used by the btree cache
-
-bucket_size
-  Size of buckets
-
-cache<0..n>
-  Symlink to each of the cache devices comprising this cache set.
-
-cache_available_percent
-  Percentage of cache device which doesn't contain dirty data, and could
-  potentially be used for writeback.  This doesn't mean this space isn't used
-  for clean cached data; the unused statistic (in priority_stats) is typically
-  much lower.
-
-clear_stats
-  Clears the statistics associated with this cache
-
-dirty_data
-  Amount of dirty data is in the cache (updated when garbage collection runs).
-
-flash_vol_create
-  Echoing a size to this file (in human readable units, k/M/G) creates a thinly
-  provisioned volume backed by the cache set.
-
-io_error_halflife, io_error_limit
-  These determines how many errors we accept before disabling the cache.
-  Each error is decayed by the half life (in # ios).  If the decaying count
-  reaches io_error_limit dirty data is written out and the cache is disabled.
-
-journal_delay_ms
-  Journal writes will delay for up to this many milliseconds, unless a cache
-  flush happens sooner. Defaults to 100.
-
-root_usage_percent
-  Percentage of the root btree node in use.  If this gets too high the node
-  will split, increasing the tree depth.
-
-stop
-  Write to this file to shut down the cache set - waits until all attached
-  backing devices have been shut down.
-
-tree_depth
-  Depth of the btree (A single node btree has depth 0).
-
-unregister
-  Detaches all backing devices and closes the cache devices; if dirty data is
-  present it will disable writeback caching and wait for it to be flushed.
-
-Sysfs - cache set internal
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This directory also exposes timings for a number of internal operations, with
-separate files for average duration, average frequency, last occurrence and max
-duration: garbage collection, btree read, btree node sorts and btree splits.
-
-active_journal_entries
-  Number of journal entries that are newer than the index.
-
-btree_nodes
-  Total nodes in the btree.
-
-btree_used_percent
-  Average fraction of btree in use.
-
-bset_tree_stats
-  Statistics about the auxiliary search trees
-
-btree_cache_max_chain
-  Longest chain in the btree node cache's hash table
-
-cache_read_races
-  Counts instances where while data was being read from the cache, the bucket
-  was reused and invalidated - i.e. where the pointer was stale after the read
-  completed. When this occurs the data is reread from the backing device.
-
-trigger_gc
-  Writing to this file forces garbage collection to run.
-
-Sysfs - Cache device
-~~~~~~~~~~~~~~~~~~~~
-
-Available at /sys/block/<cdev>/bcache
-
-block_size
-  Minimum granularity of writes - should match hardware sector size.
-
-btree_written
-  Sum of all btree writes, in (kilo/mega/giga) bytes
-
-bucket_size
-  Size of buckets
-
-cache_replacement_policy
-  One of either lru, fifo or random.
-
-discard
-  Boolean; if on a discard/TRIM will be issued to each bucket before it is
-  reused. Defaults to off, since SATA TRIM is an unqueued command (and thus
-  slow).
-
-freelist_percent
-  Size of the freelist as a percentage of nbuckets. Can be written to to
-  increase the number of buckets kept on the freelist, which lets you
-  artificially reduce the size of the cache at runtime. Mostly for testing
-  purposes (i.e. testing how different size caches affect your hit rate), but
-  since buckets are discarded when they move on to the freelist will also make
-  the SSD's garbage collection easier by effectively giving it more reserved
-  space.
-
-io_errors
-  Number of errors that have occurred, decayed by io_error_halflife.
-
-metadata_written
-  Sum of all non data writes (btree writes and all other metadata).
-
-nbuckets
-  Total buckets in this cache
-
-priority_stats
-  Statistics about how recently data in the cache has been accessed.
-  This can reveal your working set size.  Unused is the percentage of
-  the cache that doesn't contain any data.  Metadata is bcache's
-  metadata overhead.  Average is the average priority of cache buckets.
-  Next is a list of quantiles with the priority threshold of each.
-
-written
-  Sum of all data that has been written to the cache; comparison with
-  btree_written gives the amount of write inflation in bcache.
diff --git a/Documentation/block/cmdline-partition.txt b/Documentation/block/cmdline-partition.txt
index 525b9f6d7fb4..760a3f7c3ed4 100644
--- a/Documentation/block/cmdline-partition.txt
+++ b/Documentation/block/cmdline-partition.txt
@@ -1,7 +1,9 @@
 Embedded device command line partition parsing
 =====================================================================
 
-Support for reading the block device partition table from the command line.
+The "blkdevparts" command line option adds support for reading the
+block device partition table from the kernel command line.
+
 It is typically used for fixed block (eMMC) embedded devices.
 It has no MBR, so saves storage space. Bootloader can be easily accessed
 by absolute address of data on the block device.
@@ -14,22 +16,27 @@ blkdevparts=<blkdev-def>[;<blkdev-def>]
     <partdef> := <size>[@<offset>](part-name)
 
 <blkdev-id>
-    block device disk name, embedded device used fixed block device,
-    it's disk name also fixed. such as: mmcblk0, mmcblk1, mmcblk0boot0.
+    block device disk name. Embedded device uses fixed block device.
+    Its disk name is also fixed, such as: mmcblk0, mmcblk1, mmcblk0boot0.
 
 <size>
     partition size, in bytes, such as: 512, 1m, 1G.
+    size may contain an optional suffix of (upper or lower case):
+      K, M, G, T, P, E.
+    "-" is used to denote all remaining space.
 
 <offset>
     partition start address, in bytes.
+    offset may contain an optional suffix of (upper or lower case):
+      K, M, G, T, P, E.
 
 (part-name)
-    partition name, kernel send uevent with "PARTNAME". application can create
-    a link to block device partition with the name "PARTNAME".
-    user space application can access partition by partition name.
+    partition name. Kernel sends uevent with "PARTNAME". Application can
+    create a link to block device partition with the name "PARTNAME".
+    User space application can access partition by partition name.
 
 Example:
-    eMMC disk name is "mmcblk0" and "mmcblk0boot0"
+    eMMC disk names are "mmcblk0" and "mmcblk0boot0".
 
   bootargs:
     'blkdevparts=mmcblk0:1G(data0),1G(data1),-;mmcblk0boot0:1m(boot),-(kernel)'
diff --git a/Documentation/block/null_blk.txt b/Documentation/block/null_blk.txt
index 733927a7b501..07f147381f32 100644
--- a/Documentation/block/null_blk.txt
+++ b/Documentation/block/null_blk.txt
@@ -71,13 +71,16 @@ use_per_node_hctx=[0/1]: Default: 0
   1: The multi-queue block layer is instantiated with a hardware dispatch
      queue for each CPU node in the system.
 
-use_lightnvm=[0/1]: Default: 0
-  Register device with LightNVM. Requires blk-mq and CONFIG_NVM to be enabled.
-
 no_sched=[0/1]: Default: 0
   0: nullb* use default blk-mq io scheduler.
   1: nullb* doesn't use io scheduler.
 
+blocking=[0/1]: Default: 0
+  0: Register as a non-blocking blk-mq driver device.
+  1: Register as a blocking blk-mq driver device, null_blk will set
+     the BLK_MQ_F_BLOCKING flag, indicating that it sometimes/always
+     needs to block in its ->queue_rq() function.
+
 shared_tags=[0/1]: Default: 0
   0: Tag set is not shared.
   1: Tag set shared between devices for blk-mq. Only makes sense with
diff --git a/Documentation/blockdev/zram.txt b/Documentation/blockdev/zram.txt
index 257e65714c6a..875b2b56b87f 100644
--- a/Documentation/blockdev/zram.txt
+++ b/Documentation/blockdev/zram.txt
@@ -218,6 +218,7 @@ line of text and contains the following stats separated by whitespace:
  same_pages       the number of same element filled pages written to this disk.
                   No memory is allocated for such pages.
  pages_compacted  the number of pages freed during compaction
+ huge_pages	  the number of incompressible pages
 
 9) Deactivate:
 	swapoff /dev/zram0
@@ -242,5 +243,29 @@ to backing storage rather than keeping it in memory.
 User should set up backing device via /sys/block/zramX/backing_dev
 before disksize setting.
 
+= memory tracking
+
+With CONFIG_ZRAM_MEMORY_TRACKING, user can know information of the
+zram block. It could be useful to catch cold or incompressible
+pages of the process with*pagemap.
+If you enable the feature, you could see block state via
+/sys/kernel/debug/zram/zram0/block_state". The output is as follows,
+
+	  300    75.033841 .wh
+	  301    63.806904 s..
+	  302    63.806919 ..h
+
+First column is zram's block index.
+Second column is access time since the system was booted
+Third column is state of the block.
+(s: same page
+w: written page to backing store
+h: huge page)
+
+First line of above example says 300th block is accessed at 75.033841sec
+and the block's state is huge so it is written back to the backing
+storage. It's a debugging feature so anyone shouldn't rely on it to work
+properly.
+
 Nitin Gupta
 ngupta@vflare.org
diff --git a/Documentation/bpf/README.rst b/Documentation/bpf/README.rst
new file mode 100644
index 000000000000..b9a80c9e9392
--- /dev/null
+++ b/Documentation/bpf/README.rst
@@ -0,0 +1,36 @@
+=================
+BPF documentation
+=================
+
+This directory contains documentation for the BPF (Berkeley Packet
+Filter) facility, with a focus on the extended BPF version (eBPF).
+
+This kernel side documentation is still work in progress.  The main
+textual documentation is (for historical reasons) described in
+`Documentation/networking/filter.txt`_, which describe both classical
+and extended BPF instruction-set.
+The Cilium project also maintains a `BPF and XDP Reference Guide`_
+that goes into great technical depth about the BPF Architecture.
+
+The primary info for the bpf syscall is available in the `man-pages`_
+for `bpf(2)`_.
+
+
+
+Frequently asked questions (FAQ)
+================================
+
+Two sets of Questions and Answers (Q&A) are maintained.
+
+* QA for common questions about BPF see: bpf_design_QA_
+
+* QA for developers interacting with BPF subsystem: bpf_devel_QA_
+
+
+.. Links:
+.. _bpf_design_QA: bpf_design_QA.rst
+.. _bpf_devel_QA:  bpf_devel_QA.rst
+.. _Documentation/networking/filter.txt: ../networking/filter.txt
+.. _man-pages: https://www.kernel.org/doc/man-pages/
+.. _bpf(2): http://man7.org/linux/man-pages/man2/bpf.2.html
+.. _BPF and XDP Reference Guide: http://cilium.readthedocs.io/en/latest/bpf/
diff --git a/Documentation/bpf/bpf_design_QA.rst b/Documentation/bpf/bpf_design_QA.rst
new file mode 100644
index 000000000000..6780a6d81745
--- /dev/null
+++ b/Documentation/bpf/bpf_design_QA.rst
@@ -0,0 +1,221 @@
+==============
+BPF Design Q&A
+==============
+
+BPF extensibility and applicability to networking, tracing, security
+in the linux kernel and several user space implementations of BPF
+virtual machine led to a number of misunderstanding on what BPF actually is.
+This short QA is an attempt to address that and outline a direction
+of where BPF is heading long term.
+
+.. contents::
+    :local:
+    :depth: 3
+
+Questions and Answers
+=====================
+
+Q: Is BPF a generic instruction set similar to x64 and arm64?
+-------------------------------------------------------------
+A: NO.
+
+Q: Is BPF a generic virtual machine ?
+-------------------------------------
+A: NO.
+
+BPF is generic instruction set *with* C calling convention.
+-----------------------------------------------------------
+
+Q: Why C calling convention was chosen?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A: Because BPF programs are designed to run in the linux kernel
+which is written in C, hence BPF defines instruction set compatible
+with two most used architectures x64 and arm64 (and takes into
+consideration important quirks of other architectures) and
+defines calling convention that is compatible with C calling
+convention of the linux kernel on those architectures.
+
+Q: can multiple return values be supported in the future?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A: NO. BPF allows only register R0 to be used as return value.
+
+Q: can more than 5 function arguments be supported in the future?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A: NO. BPF calling convention only allows registers R1-R5 to be used
+as arguments. BPF is not a standalone instruction set.
+(unlike x64 ISA that allows msft, cdecl and other conventions)
+
+Q: can BPF programs access instruction pointer or return address?
+-----------------------------------------------------------------
+A: NO.
+
+Q: can BPF programs access stack pointer ?
+------------------------------------------
+A: NO.
+
+Only frame pointer (register R10) is accessible.
+From compiler point of view it's necessary to have stack pointer.
+For example LLVM defines register R11 as stack pointer in its
+BPF backend, but it makes sure that generated code never uses it.
+
+Q: Does C-calling convention diminishes possible use cases?
+-----------------------------------------------------------
+A: YES.
+
+BPF design forces addition of major functionality in the form
+of kernel helper functions and kernel objects like BPF maps with
+seamless interoperability between them. It lets kernel call into
+BPF programs and programs call kernel helpers with zero overhead.
+As all of them were native C code. That is particularly the case
+for JITed BPF programs that are indistinguishable from
+native kernel C code.
+
+Q: Does it mean that 'innovative' extensions to BPF code are disallowed?
+------------------------------------------------------------------------
+A: Soft yes.
+
+At least for now until BPF core has support for
+bpf-to-bpf calls, indirect calls, loops, global variables,
+jump tables, read only sections and all other normal constructs
+that C code can produce.
+
+Q: Can loops be supported in a safe way?
+----------------------------------------
+A: It's not clear yet.
+
+BPF developers are trying to find a way to
+support bounded loops where the verifier can guarantee that
+the program terminates in less than 4096 instructions.
+
+Instruction level questions
+---------------------------
+
+Q: LD_ABS and LD_IND instructions vs C code
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Q: How come LD_ABS and LD_IND instruction are present in BPF whereas
+C code cannot express them and has to use builtin intrinsics?
+
+A: This is artifact of compatibility with classic BPF. Modern
+networking code in BPF performs better without them.
+See 'direct packet access'.
+
+Q: BPF instructions mapping not one-to-one to native CPU
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Q: It seems not all BPF instructions are one-to-one to native CPU.
+For example why BPF_JNE and other compare and jumps are not cpu-like?
+
+A: This was necessary to avoid introducing flags into ISA which are
+impossible to make generic and efficient across CPU architectures.
+
+Q: why BPF_DIV instruction doesn't map to x64 div?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A: Because if we picked one-to-one relationship to x64 it would have made
+it more complicated to support on arm64 and other archs. Also it
+needs div-by-zero runtime check.
+
+Q: why there is no BPF_SDIV for signed divide operation?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A: Because it would be rarely used. llvm errors in such case and
+prints a suggestion to use unsigned divide instead
+
+Q: Why BPF has implicit prologue and epilogue?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A: Because architectures like sparc have register windows and in general
+there are enough subtle differences between architectures, so naive
+store return address into stack won't work. Another reason is BPF has
+to be safe from division by zero (and legacy exception path
+of LD_ABS insn). Those instructions need to invoke epilogue and
+return implicitly.
+
+Q: Why BPF_JLT and BPF_JLE instructions were not introduced in the beginning?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A: Because classic BPF didn't have them and BPF authors felt that compiler
+workaround would be acceptable. Turned out that programs lose performance
+due to lack of these compare instructions and they were added.
+These two instructions is a perfect example what kind of new BPF
+instructions are acceptable and can be added in the future.
+These two already had equivalent instructions in native CPUs.
+New instructions that don't have one-to-one mapping to HW instructions
+will not be accepted.
+
+Q: BPF 32-bit subregister requirements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Q: BPF 32-bit subregisters have a requirement to zero upper 32-bits of BPF
+registers which makes BPF inefficient virtual machine for 32-bit
+CPU architectures and 32-bit HW accelerators. Can true 32-bit registers
+be added to BPF in the future?
+
+A: NO. The first thing to improve performance on 32-bit archs is to teach
+LLVM to generate code that uses 32-bit subregisters. Then second step
+is to teach verifier to mark operations where zero-ing upper bits
+is unnecessary. Then JITs can take advantage of those markings and
+drastically reduce size of generated code and improve performance.
+
+Q: Does BPF have a stable ABI?
+------------------------------
+A: YES. BPF instructions, arguments to BPF programs, set of helper
+functions and their arguments, recognized return codes are all part
+of ABI. However when tracing programs are using bpf_probe_read() helper
+to walk kernel internal datastructures and compile with kernel
+internal headers these accesses can and will break with newer
+kernels. The union bpf_attr -> kern_version is checked at load time
+to prevent accidentally loading kprobe-based bpf programs written
+for a different kernel. Networking programs don't do kern_version check.
+
+Q: How much stack space a BPF program uses?
+-------------------------------------------
+A: Currently all program types are limited to 512 bytes of stack
+space, but the verifier computes the actual amount of stack used
+and both interpreter and most JITed code consume necessary amount.
+
+Q: Can BPF be offloaded to HW?
+------------------------------
+A: YES. BPF HW offload is supported by NFP driver.
+
+Q: Does classic BPF interpreter still exist?
+--------------------------------------------
+A: NO. Classic BPF programs are converted into extend BPF instructions.
+
+Q: Can BPF call arbitrary kernel functions?
+-------------------------------------------
+A: NO. BPF programs can only call a set of helper functions which
+is defined for every program type.
+
+Q: Can BPF overwrite arbitrary kernel memory?
+---------------------------------------------
+A: NO.
+
+Tracing bpf programs can *read* arbitrary memory with bpf_probe_read()
+and bpf_probe_read_str() helpers. Networking programs cannot read
+arbitrary memory, since they don't have access to these helpers.
+Programs can never read or write arbitrary memory directly.
+
+Q: Can BPF overwrite arbitrary user memory?
+-------------------------------------------
+A: Sort-of.
+
+Tracing BPF programs can overwrite the user memory
+of the current task with bpf_probe_write_user(). Every time such
+program is loaded the kernel will print warning message, so
+this helper is only useful for experiments and prototypes.
+Tracing BPF programs are root only.
+
+Q: bpf_trace_printk() helper warning
+------------------------------------
+Q: When bpf_trace_printk() helper is used the kernel prints nasty
+warning message. Why is that?
+
+A: This is done to nudge program authors into better interfaces when
+programs need to pass data to user space. Like bpf_perf_event_output()
+can be used to efficiently stream data via perf ring buffer.
+BPF maps can be used for asynchronous data sharing between kernel
+and user space. bpf_trace_printk() should only be used for debugging.
+
+Q: New functionality via kernel modules?
+----------------------------------------
+Q: Can BPF functionality such as new program or map types, new
+helpers, etc be added out of kernel module code?
+
+A: NO.
diff --git a/Documentation/bpf/bpf_design_QA.txt b/Documentation/bpf/bpf_design_QA.txt
deleted file mode 100644
index f3e458a0bb2f..000000000000
--- a/Documentation/bpf/bpf_design_QA.txt
+++ /dev/null
@@ -1,156 +0,0 @@
-BPF extensibility and applicability to networking, tracing, security
-in the linux kernel and several user space implementations of BPF
-virtual machine led to a number of misunderstanding on what BPF actually is.
-This short QA is an attempt to address that and outline a direction
-of where BPF is heading long term.
-
-Q: Is BPF a generic instruction set similar to x64 and arm64?
-A: NO.
-
-Q: Is BPF a generic virtual machine ?
-A: NO.
-
-BPF is generic instruction set _with_ C calling convention.
-
-Q: Why C calling convention was chosen?
-A: Because BPF programs are designed to run in the linux kernel
-   which is written in C, hence BPF defines instruction set compatible
-   with two most used architectures x64 and arm64 (and takes into
-   consideration important quirks of other architectures) and
-   defines calling convention that is compatible with C calling
-   convention of the linux kernel on those architectures.
-
-Q: can multiple return values be supported in the future?
-A: NO. BPF allows only register R0 to be used as return value.
-
-Q: can more than 5 function arguments be supported in the future?
-A: NO. BPF calling convention only allows registers R1-R5 to be used
-   as arguments. BPF is not a standalone instruction set.
-   (unlike x64 ISA that allows msft, cdecl and other conventions)
-
-Q: can BPF programs access instruction pointer or return address?
-A: NO.
-
-Q: can BPF programs access stack pointer ?
-A: NO. Only frame pointer (register R10) is accessible.
-   From compiler point of view it's necessary to have stack pointer.
-   For example LLVM defines register R11 as stack pointer in its
-   BPF backend, but it makes sure that generated code never uses it.
-
-Q: Does C-calling convention diminishes possible use cases?
-A: YES. BPF design forces addition of major functionality in the form
-   of kernel helper functions and kernel objects like BPF maps with
-   seamless interoperability between them. It lets kernel call into
-   BPF programs and programs call kernel helpers with zero overhead.
-   As all of them were native C code. That is particularly the case
-   for JITed BPF programs that are indistinguishable from
-   native kernel C code.
-
-Q: Does it mean that 'innovative' extensions to BPF code are disallowed?
-A: Soft yes. At least for now until BPF core has support for
-   bpf-to-bpf calls, indirect calls, loops, global variables,
-   jump tables, read only sections and all other normal constructs
-   that C code can produce.
-
-Q: Can loops be supported in a safe way?
-A: It's not clear yet. BPF developers are trying to find a way to
-   support bounded loops where the verifier can guarantee that
-   the program terminates in less than 4096 instructions.
-
-Q: How come LD_ABS and LD_IND instruction are present in BPF whereas
-   C code cannot express them and has to use builtin intrinsics?
-A: This is artifact of compatibility with classic BPF. Modern
-   networking code in BPF performs better without them.
-   See 'direct packet access'.
-
-Q: It seems not all BPF instructions are one-to-one to native CPU.
-   For example why BPF_JNE and other compare and jumps are not cpu-like?
-A: This was necessary to avoid introducing flags into ISA which are
-   impossible to make generic and efficient across CPU architectures.
-
-Q: why BPF_DIV instruction doesn't map to x64 div?
-A: Because if we picked one-to-one relationship to x64 it would have made
-   it more complicated to support on arm64 and other archs. Also it
-   needs div-by-zero runtime check.
-
-Q: why there is no BPF_SDIV for signed divide operation?
-A: Because it would be rarely used. llvm errors in such case and
-   prints a suggestion to use unsigned divide instead
-
-Q: Why BPF has implicit prologue and epilogue?
-A: Because architectures like sparc have register windows and in general
-   there are enough subtle differences between architectures, so naive
-   store return address into stack won't work. Another reason is BPF has
-   to be safe from division by zero (and legacy exception path
-   of LD_ABS insn). Those instructions need to invoke epilogue and
-   return implicitly.
-
-Q: Why BPF_JLT and BPF_JLE instructions were not introduced in the beginning?
-A: Because classic BPF didn't have them and BPF authors felt that compiler
-   workaround would be acceptable. Turned out that programs lose performance
-   due to lack of these compare instructions and they were added.
-   These two instructions is a perfect example what kind of new BPF
-   instructions are acceptable and can be added in the future.
-   These two already had equivalent instructions in native CPUs.
-   New instructions that don't have one-to-one mapping to HW instructions
-   will not be accepted.
-
-Q: BPF 32-bit subregisters have a requirement to zero upper 32-bits of BPF
-   registers which makes BPF inefficient virtual machine for 32-bit
-   CPU architectures and 32-bit HW accelerators. Can true 32-bit registers
-   be added to BPF in the future?
-A: NO. The first thing to improve performance on 32-bit archs is to teach
-   LLVM to generate code that uses 32-bit subregisters. Then second step
-   is to teach verifier to mark operations where zero-ing upper bits
-   is unnecessary. Then JITs can take advantage of those markings and
-   drastically reduce size of generated code and improve performance.
-
-Q: Does BPF have a stable ABI?
-A: YES. BPF instructions, arguments to BPF programs, set of helper
-   functions and their arguments, recognized return codes are all part
-   of ABI. However when tracing programs are using bpf_probe_read() helper
-   to walk kernel internal datastructures and compile with kernel
-   internal headers these accesses can and will break with newer
-   kernels. The union bpf_attr -> kern_version is checked at load time
-   to prevent accidentally loading kprobe-based bpf programs written
-   for a different kernel. Networking programs don't do kern_version check.
-
-Q: How much stack space a BPF program uses?
-A: Currently all program types are limited to 512 bytes of stack
-   space, but the verifier computes the actual amount of stack used
-   and both interpreter and most JITed code consume necessary amount.
-
-Q: Can BPF be offloaded to HW?
-A: YES. BPF HW offload is supported by NFP driver.
-
-Q: Does classic BPF interpreter still exist?
-A: NO. Classic BPF programs are converted into extend BPF instructions.
-
-Q: Can BPF call arbitrary kernel functions?
-A: NO. BPF programs can only call a set of helper functions which
-   is defined for every program type.
-
-Q: Can BPF overwrite arbitrary kernel memory?
-A: NO. Tracing bpf programs can _read_ arbitrary memory with bpf_probe_read()
-   and bpf_probe_read_str() helpers. Networking programs cannot read
-   arbitrary memory, since they don't have access to these helpers.
-   Programs can never read or write arbitrary memory directly.
-
-Q: Can BPF overwrite arbitrary user memory?
-A: Sort-of. Tracing BPF programs can overwrite the user memory
-   of the current task with bpf_probe_write_user(). Every time such
-   program is loaded the kernel will print warning message, so
-   this helper is only useful for experiments and prototypes.
-   Tracing BPF programs are root only.
-
-Q: When bpf_trace_printk() helper is used the kernel prints nasty
-   warning message. Why is that?
-A: This is done to nudge program authors into better interfaces when
-   programs need to pass data to user space. Like bpf_perf_event_output()
-   can be used to efficiently stream data via perf ring buffer.
-   BPF maps can be used for asynchronous data sharing between kernel
-   and user space. bpf_trace_printk() should only be used for debugging.
-
-Q: Can BPF functionality such as new program or map types, new
-   helpers, etc be added out of kernel module code?
-A: NO.
diff --git a/Documentation/bpf/bpf_devel_QA.rst b/Documentation/bpf/bpf_devel_QA.rst
new file mode 100644
index 000000000000..0e7c1d946e83
--- /dev/null
+++ b/Documentation/bpf/bpf_devel_QA.rst
@@ -0,0 +1,640 @@
+=================================
+HOWTO interact with BPF subsystem
+=================================
+
+This document provides information for the BPF subsystem about various
+workflows related to reporting bugs, submitting patches, and queueing
+patches for stable kernels.
+
+For general information about submitting patches, please refer to
+`Documentation/process/`_. This document only describes additional specifics
+related to BPF.
+
+.. contents::
+    :local:
+    :depth: 2
+
+Reporting bugs
+==============
+
+Q: How do I report bugs for BPF kernel code?
+--------------------------------------------
+A: Since all BPF kernel development as well as bpftool and iproute2 BPF
+loader development happens through the netdev kernel mailing list,
+please report any found issues around BPF to the following mailing
+list:
+
+ netdev@vger.kernel.org
+
+This may also include issues related to XDP, BPF tracing, etc.
+
+Given netdev has a high volume of traffic, please also add the BPF
+maintainers to Cc (from kernel MAINTAINERS_ file):
+
+* Alexei Starovoitov <ast@kernel.org>
+* Daniel Borkmann <daniel@iogearbox.net>
+
+In case a buggy commit has already been identified, make sure to keep
+the actual commit authors in Cc as well for the report. They can
+typically be identified through the kernel's git tree.
+
+**Please do NOT report BPF issues to bugzilla.kernel.org since it
+is a guarantee that the reported issue will be overlooked.**
+
+Submitting patches
+==================
+
+Q: To which mailing list do I need to submit my BPF patches?
+------------------------------------------------------------
+A: Please submit your BPF patches to the netdev kernel mailing list:
+
+ netdev@vger.kernel.org
+
+Historically, BPF came out of networking and has always been maintained
+by the kernel networking community. Although these days BPF touches
+many other subsystems as well, the patches are still routed mainly
+through the networking community.
+
+In case your patch has changes in various different subsystems (e.g.
+tracing, security, etc), make sure to Cc the related kernel mailing
+lists and maintainers from there as well, so they are able to review
+the changes and provide their Acked-by's to the patches.
+
+Q: Where can I find patches currently under discussion for BPF subsystem?
+-------------------------------------------------------------------------
+A: All patches that are Cc'ed to netdev are queued for review under netdev
+patchwork project:
+
+  http://patchwork.ozlabs.org/project/netdev/list/
+
+Those patches which target BPF, are assigned to a 'bpf' delegate for
+further processing from BPF maintainers. The current queue with
+patches under review can be found at:
+
+  https://patchwork.ozlabs.org/project/netdev/list/?delegate=77147
+
+Once the patches have been reviewed by the BPF community as a whole
+and approved by the BPF maintainers, their status in patchwork will be
+changed to 'Accepted' and the submitter will be notified by mail. This
+means that the patches look good from a BPF perspective and have been
+applied to one of the two BPF kernel trees.
+
+In case feedback from the community requires a respin of the patches,
+their status in patchwork will be set to 'Changes Requested', and purged
+from the current review queue. Likewise for cases where patches would
+get rejected or are not applicable to the BPF trees (but assigned to
+the 'bpf' delegate).
+
+Q: How do the changes make their way into Linux?
+------------------------------------------------
+A: There are two BPF kernel trees (git repositories). Once patches have
+been accepted by the BPF maintainers, they will be applied to one
+of the two BPF trees:
+
+ * https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/
+ * https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/
+
+The bpf tree itself is for fixes only, whereas bpf-next for features,
+cleanups or other kind of improvements ("next-like" content). This is
+analogous to net and net-next trees for networking. Both bpf and
+bpf-next will only have a master branch in order to simplify against
+which branch patches should get rebased to.
+
+Accumulated BPF patches in the bpf tree will regularly get pulled
+into the net kernel tree. Likewise, accumulated BPF patches accepted
+into the bpf-next tree will make their way into net-next tree. net and
+net-next are both run by David S. Miller. From there, they will go
+into the kernel mainline tree run by Linus Torvalds. To read up on the
+process of net and net-next being merged into the mainline tree, see
+the `netdev FAQ`_ under:
+
+ `Documentation/networking/netdev-FAQ.txt`_
+
+Occasionally, to prevent merge conflicts, we might send pull requests
+to other trees (e.g. tracing) with a small subset of the patches, but
+net and net-next are always the main trees targeted for integration.
+
+The pull requests will contain a high-level summary of the accumulated
+patches and can be searched on netdev kernel mailing list through the
+following subject lines (``yyyy-mm-dd`` is the date of the pull
+request)::
+
+  pull-request: bpf yyyy-mm-dd
+  pull-request: bpf-next yyyy-mm-dd
+
+Q: How do I indicate which tree (bpf vs. bpf-next) my patch should be applied to?
+---------------------------------------------------------------------------------
+
+A: The process is the very same as described in the `netdev FAQ`_, so
+please read up on it. The subject line must indicate whether the
+patch is a fix or rather "next-like" content in order to let the
+maintainers know whether it is targeted at bpf or bpf-next.
+
+For fixes eventually landing in bpf -> net tree, the subject must
+look like::
+
+  git format-patch --subject-prefix='PATCH bpf' start..finish
+
+For features/improvements/etc that should eventually land in
+bpf-next -> net-next, the subject must look like::
+
+  git format-patch --subject-prefix='PATCH bpf-next' start..finish
+
+If unsure whether the patch or patch series should go into bpf
+or net directly, or bpf-next or net-next directly, it is not a
+problem either if the subject line says net or net-next as target.
+It is eventually up to the maintainers to do the delegation of
+the patches.
+
+If it is clear that patches should go into bpf or bpf-next tree,
+please make sure to rebase the patches against those trees in
+order to reduce potential conflicts.
+
+In case the patch or patch series has to be reworked and sent out
+again in a second or later revision, it is also required to add a
+version number (``v2``, ``v3``, ...) into the subject prefix::
+
+  git format-patch --subject-prefix='PATCH net-next v2' start..finish
+
+When changes have been requested to the patch series, always send the
+whole patch series again with the feedback incorporated (never send
+individual diffs on top of the old series).
+
+Q: What does it mean when a patch gets applied to bpf or bpf-next tree?
+-----------------------------------------------------------------------
+A: It means that the patch looks good for mainline inclusion from
+a BPF point of view.
+
+Be aware that this is not a final verdict that the patch will
+automatically get accepted into net or net-next trees eventually:
+
+On the netdev kernel mailing list reviews can come in at any point
+in time. If discussions around a patch conclude that they cannot
+get included as-is, we will either apply a follow-up fix or drop
+them from the trees entirely. Therefore, we also reserve to rebase
+the trees when deemed necessary. After all, the purpose of the tree
+is to:
+
+i) accumulate and stage BPF patches for integration into trees
+   like net and net-next, and
+
+ii) run extensive BPF test suite and
+    workloads on the patches before they make their way any further.
+
+Once the BPF pull request was accepted by David S. Miller, then
+the patches end up in net or net-next tree, respectively, and
+make their way from there further into mainline. Again, see the
+`netdev FAQ`_ for additional information e.g. on how often they are
+merged to mainline.
+
+Q: How long do I need to wait for feedback on my BPF patches?
+-------------------------------------------------------------
+A: We try to keep the latency low. The usual time to feedback will
+be around 2 or 3 business days. It may vary depending on the
+complexity of changes and current patch load.
+
+Q: How often do you send pull requests to major kernel trees like net or net-next?
+----------------------------------------------------------------------------------
+
+A: Pull requests will be sent out rather often in order to not
+accumulate too many patches in bpf or bpf-next.
+
+As a rule of thumb, expect pull requests for each tree regularly
+at the end of the week. In some cases pull requests could additionally
+come also in the middle of the week depending on the current patch
+load or urgency.
+
+Q: Are patches applied to bpf-next when the merge window is open?
+-----------------------------------------------------------------
+A: For the time when the merge window is open, bpf-next will not be
+processed. This is roughly analogous to net-next patch processing,
+so feel free to read up on the `netdev FAQ`_ about further details.
+
+During those two weeks of merge window, we might ask you to resend
+your patch series once bpf-next is open again. Once Linus released
+a ``v*-rc1`` after the merge window, we continue processing of bpf-next.
+
+For non-subscribers to kernel mailing lists, there is also a status
+page run by David S. Miller on net-next that provides guidance:
+
+  http://vger.kernel.org/~davem/net-next.html
+
+Q: Verifier changes and test cases
+----------------------------------
+Q: I made a BPF verifier change, do I need to add test cases for
+BPF kernel selftests_?
+
+A: If the patch has changes to the behavior of the verifier, then yes,
+it is absolutely necessary to add test cases to the BPF kernel
+selftests_ suite. If they are not present and we think they are
+needed, then we might ask for them before accepting any changes.
+
+In particular, test_verifier.c is tracking a high number of BPF test
+cases, including a lot of corner cases that LLVM BPF back end may
+generate out of the restricted C code. Thus, adding test cases is
+absolutely crucial to make sure future changes do not accidentally
+affect prior use-cases. Thus, treat those test cases as: verifier
+behavior that is not tracked in test_verifier.c could potentially
+be subject to change.
+
+Q: samples/bpf preference vs selftests?
+---------------------------------------
+Q: When should I add code to `samples/bpf/`_ and when to BPF kernel
+selftests_ ?
+
+A: In general, we prefer additions to BPF kernel selftests_ rather than
+`samples/bpf/`_. The rationale is very simple: kernel selftests are
+regularly run by various bots to test for kernel regressions.
+
+The more test cases we add to BPF selftests, the better the coverage
+and the less likely it is that those could accidentally break. It is
+not that BPF kernel selftests cannot demo how a specific feature can
+be used.
+
+That said, `samples/bpf/`_ may be a good place for people to get started,
+so it might be advisable that simple demos of features could go into
+`samples/bpf/`_, but advanced functional and corner-case testing rather
+into kernel selftests.
+
+If your sample looks like a test case, then go for BPF kernel selftests
+instead!
+
+Q: When should I add code to the bpftool?
+-----------------------------------------
+A: The main purpose of bpftool (under tools/bpf/bpftool/) is to provide
+a central user space tool for debugging and introspection of BPF programs
+and maps that are active in the kernel. If UAPI changes related to BPF
+enable for dumping additional information of programs or maps, then
+bpftool should be extended as well to support dumping them.
+
+Q: When should I add code to iproute2's BPF loader?
+---------------------------------------------------
+A: For UAPI changes related to the XDP or tc layer (e.g. ``cls_bpf``),
+the convention is that those control-path related changes are added to
+iproute2's BPF loader as well from user space side. This is not only
+useful to have UAPI changes properly designed to be usable, but also
+to make those changes available to a wider user base of major
+downstream distributions.
+
+Q: Do you accept patches as well for iproute2's BPF loader?
+-----------------------------------------------------------
+A: Patches for the iproute2's BPF loader have to be sent to:
+
+  netdev@vger.kernel.org
+
+While those patches are not processed by the BPF kernel maintainers,
+please keep them in Cc as well, so they can be reviewed.
+
+The official git repository for iproute2 is run by Stephen Hemminger
+and can be found at:
+
+  https://git.kernel.org/pub/scm/linux/kernel/git/shemminger/iproute2.git/
+
+The patches need to have a subject prefix of '``[PATCH iproute2
+master]``' or '``[PATCH iproute2 net-next]``'. '``master``' or
+'``net-next``' describes the target branch where the patch should be
+applied to. Meaning, if kernel changes went into the net-next kernel
+tree, then the related iproute2 changes need to go into the iproute2
+net-next branch, otherwise they can be targeted at master branch. The
+iproute2 net-next branch will get merged into the master branch after
+the current iproute2 version from master has been released.
+
+Like BPF, the patches end up in patchwork under the netdev project and
+are delegated to 'shemminger' for further processing:
+
+  http://patchwork.ozlabs.org/project/netdev/list/?delegate=389
+
+Q: What is the minimum requirement before I submit my BPF patches?
+------------------------------------------------------------------
+A: When submitting patches, always take the time and properly test your
+patches *prior* to submission. Never rush them! If maintainers find
+that your patches have not been properly tested, it is a good way to
+get them grumpy. Testing patch submissions is a hard requirement!
+
+Note, fixes that go to bpf tree *must* have a ``Fixes:`` tag included.
+The same applies to fixes that target bpf-next, where the affected
+commit is in net-next (or in some cases bpf-next). The ``Fixes:`` tag is
+crucial in order to identify follow-up commits and tremendously helps
+for people having to do backporting, so it is a must have!
+
+We also don't accept patches with an empty commit message. Take your
+time and properly write up a high quality commit message, it is
+essential!
+
+Think about it this way: other developers looking at your code a month
+from now need to understand *why* a certain change has been done that
+way, and whether there have been flaws in the analysis or assumptions
+that the original author did. Thus providing a proper rationale and
+describing the use-case for the changes is a must.
+
+Patch submissions with >1 patch must have a cover letter which includes
+a high level description of the series. This high level summary will
+then be placed into the merge commit by the BPF maintainers such that
+it is also accessible from the git log for future reference.
+
+Q: Features changing BPF JIT and/or LLVM
+----------------------------------------
+Q: What do I need to consider when adding a new instruction or feature
+that would require BPF JIT and/or LLVM integration as well?
+
+A: We try hard to keep all BPF JITs up to date such that the same user
+experience can be guaranteed when running BPF programs on different
+architectures without having the program punt to the less efficient
+interpreter in case the in-kernel BPF JIT is enabled.
+
+If you are unable to implement or test the required JIT changes for
+certain architectures, please work together with the related BPF JIT
+developers in order to get the feature implemented in a timely manner.
+Please refer to the git log (``arch/*/net/``) to locate the necessary
+people for helping out.
+
+Also always make sure to add BPF test cases (e.g. test_bpf.c and
+test_verifier.c) for new instructions, so that they can receive
+broad test coverage and help run-time testing the various BPF JITs.
+
+In case of new BPF instructions, once the changes have been accepted
+into the Linux kernel, please implement support into LLVM's BPF back
+end. See LLVM_ section below for further information.
+
+Stable submission
+=================
+
+Q: I need a specific BPF commit in stable kernels. What should I do?
+--------------------------------------------------------------------
+A: In case you need a specific fix in stable kernels, first check whether
+the commit has already been applied in the related ``linux-*.y`` branches:
+
+  https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/
+
+If not the case, then drop an email to the BPF maintainers with the
+netdev kernel mailing list in Cc and ask for the fix to be queued up:
+
+  netdev@vger.kernel.org
+
+The process in general is the same as on netdev itself, see also the
+`netdev FAQ`_ document.
+
+Q: Do you also backport to kernels not currently maintained as stable?
+----------------------------------------------------------------------
+A: No. If you need a specific BPF commit in kernels that are currently not
+maintained by the stable maintainers, then you are on your own.
+
+The current stable and longterm stable kernels are all listed here:
+
+  https://www.kernel.org/
+
+Q: The BPF patch I am about to submit needs to go to stable as well
+-------------------------------------------------------------------
+What should I do?
+
+A: The same rules apply as with netdev patch submissions in general, see
+`netdev FAQ`_ under:
+
+  `Documentation/networking/netdev-FAQ.txt`_
+
+Never add "``Cc: stable@vger.kernel.org``" to the patch description, but
+ask the BPF maintainers to queue the patches instead. This can be done
+with a note, for example, under the ``---`` part of the patch which does
+not go into the git log. Alternatively, this can be done as a simple
+request by mail instead.
+
+Q: Queue stable patches
+-----------------------
+Q: Where do I find currently queued BPF patches that will be submitted
+to stable?
+
+A: Once patches that fix critical bugs got applied into the bpf tree, they
+are queued up for stable submission under:
+
+  http://patchwork.ozlabs.org/bundle/bpf/stable/?state=*
+
+They will be on hold there at minimum until the related commit made its
+way into the mainline kernel tree.
+
+After having been under broader exposure, the queued patches will be
+submitted by the BPF maintainers to the stable maintainers.
+
+Testing patches
+===============
+
+Q: How to run BPF selftests
+---------------------------
+A: After you have booted into the newly compiled kernel, navigate to
+the BPF selftests_ suite in order to test BPF functionality (current
+working directory points to the root of the cloned git tree)::
+
+  $ cd tools/testing/selftests/bpf/
+  $ make
+
+To run the verifier tests::
+
+  $ sudo ./test_verifier
+
+The verifier tests print out all the current checks being
+performed. The summary at the end of running all tests will dump
+information of test successes and failures::
+
+  Summary: 418 PASSED, 0 FAILED
+
+In order to run through all BPF selftests, the following command is
+needed::
+
+  $ sudo make run_tests
+
+See the kernels selftest `Documentation/dev-tools/kselftest.rst`_
+document for further documentation.
+
+Q: Which BPF kernel selftests version should I run my kernel against?
+---------------------------------------------------------------------
+A: If you run a kernel ``xyz``, then always run the BPF kernel selftests
+from that kernel ``xyz`` as well. Do not expect that the BPF selftest
+from the latest mainline tree will pass all the time.
+
+In particular, test_bpf.c and test_verifier.c have a large number of
+test cases and are constantly updated with new BPF test sequences, or
+existing ones are adapted to verifier changes e.g. due to verifier
+becoming smarter and being able to better track certain things.
+
+LLVM
+====
+
+Q: Where do I find LLVM with BPF support?
+-----------------------------------------
+A: The BPF back end for LLVM is upstream in LLVM since version 3.7.1.
+
+All major distributions these days ship LLVM with BPF back end enabled,
+so for the majority of use-cases it is not required to compile LLVM by
+hand anymore, just install the distribution provided package.
+
+LLVM's static compiler lists the supported targets through
+``llc --version``, make sure BPF targets are listed. Example::
+
+     $ llc --version
+     LLVM (http://llvm.org/):
+       LLVM version 6.0.0svn
+       Optimized build.
+       Default target: x86_64-unknown-linux-gnu
+       Host CPU: skylake
+
+       Registered Targets:
+         bpf    - BPF (host endian)
+         bpfeb  - BPF (big endian)
+         bpfel  - BPF (little endian)
+         x86    - 32-bit X86: Pentium-Pro and above
+         x86-64 - 64-bit X86: EM64T and AMD64
+
+For developers in order to utilize the latest features added to LLVM's
+BPF back end, it is advisable to run the latest LLVM releases. Support
+for new BPF kernel features such as additions to the BPF instruction
+set are often developed together.
+
+All LLVM releases can be found at: http://releases.llvm.org/
+
+Q: Got it, so how do I build LLVM manually anyway?
+--------------------------------------------------
+A: You need cmake and gcc-c++ as build requisites for LLVM. Once you have
+that set up, proceed with building the latest LLVM and clang version
+from the git repositories::
+
+     $ git clone http://llvm.org/git/llvm.git
+     $ cd llvm/tools
+     $ git clone --depth 1 http://llvm.org/git/clang.git
+     $ cd ..; mkdir build; cd build
+     $ cmake .. -DLLVM_TARGETS_TO_BUILD="BPF;X86" \
+                -DBUILD_SHARED_LIBS=OFF           \
+                -DCMAKE_BUILD_TYPE=Release        \
+                -DLLVM_BUILD_RUNTIME=OFF
+     $ make -j $(getconf _NPROCESSORS_ONLN)
+
+The built binaries can then be found in the build/bin/ directory, where
+you can point the PATH variable to.
+
+Q: Reporting LLVM BPF issues
+----------------------------
+Q: Should I notify BPF kernel maintainers about issues in LLVM's BPF code
+generation back end or about LLVM generated code that the verifier
+refuses to accept?
+
+A: Yes, please do!
+
+LLVM's BPF back end is a key piece of the whole BPF
+infrastructure and it ties deeply into verification of programs from the
+kernel side. Therefore, any issues on either side need to be investigated
+and fixed whenever necessary.
+
+Therefore, please make sure to bring them up at netdev kernel mailing
+list and Cc BPF maintainers for LLVM and kernel bits:
+
+* Yonghong Song <yhs@fb.com>
+* Alexei Starovoitov <ast@kernel.org>
+* Daniel Borkmann <daniel@iogearbox.net>
+
+LLVM also has an issue tracker where BPF related bugs can be found:
+
+  https://bugs.llvm.org/buglist.cgi?quicksearch=bpf
+
+However, it is better to reach out through mailing lists with having
+maintainers in Cc.
+
+Q: New BPF instruction for kernel and LLVM
+------------------------------------------
+Q: I have added a new BPF instruction to the kernel, how can I integrate
+it into LLVM?
+
+A: LLVM has a ``-mcpu`` selector for the BPF back end in order to allow
+the selection of BPF instruction set extensions. By default the
+``generic`` processor target is used, which is the base instruction set
+(v1) of BPF.
+
+LLVM has an option to select ``-mcpu=probe`` where it will probe the host
+kernel for supported BPF instruction set extensions and selects the
+optimal set automatically.
+
+For cross-compilation, a specific version can be select manually as well ::
+
+     $ llc -march bpf -mcpu=help
+     Available CPUs for this target:
+
+       generic - Select the generic processor.
+       probe   - Select the probe processor.
+       v1      - Select the v1 processor.
+       v2      - Select the v2 processor.
+     [...]
+
+Newly added BPF instructions to the Linux kernel need to follow the same
+scheme, bump the instruction set version and implement probing for the
+extensions such that ``-mcpu=probe`` users can benefit from the
+optimization transparently when upgrading their kernels.
+
+If you are unable to implement support for the newly added BPF instruction
+please reach out to BPF developers for help.
+
+By the way, the BPF kernel selftests run with ``-mcpu=probe`` for better
+test coverage.
+
+Q: clang flag for target bpf?
+-----------------------------
+Q: In some cases clang flag ``-target bpf`` is used but in other cases the
+default clang target, which matches the underlying architecture, is used.
+What is the difference and when I should use which?
+
+A: Although LLVM IR generation and optimization try to stay architecture
+independent, ``-target <arch>`` still has some impact on generated code:
+
+- BPF program may recursively include header file(s) with file scope
+  inline assembly codes. The default target can handle this well,
+  while ``bpf`` target may fail if bpf backend assembler does not
+  understand these assembly codes, which is true in most cases.
+
+- When compiled without ``-g``, additional elf sections, e.g.,
+  .eh_frame and .rela.eh_frame, may be present in the object file
+  with default target, but not with ``bpf`` target.
+
+- The default target may turn a C switch statement into a switch table
+  lookup and jump operation. Since the switch table is placed
+  in the global readonly section, the bpf program will fail to load.
+  The bpf target does not support switch table optimization.
+  The clang option ``-fno-jump-tables`` can be used to disable
+  switch table generation.
+
+- For clang ``-target bpf``, it is guaranteed that pointer or long /
+  unsigned long types will always have a width of 64 bit, no matter
+  whether underlying clang binary or default target (or kernel) is
+  32 bit. However, when native clang target is used, then it will
+  compile these types based on the underlying architecture's conventions,
+  meaning in case of 32 bit architecture, pointer or long / unsigned
+  long types e.g. in BPF context structure will have width of 32 bit
+  while the BPF LLVM back end still operates in 64 bit. The native
+  target is mostly needed in tracing for the case of walking ``pt_regs``
+  or other kernel structures where CPU's register width matters.
+  Otherwise, ``clang -target bpf`` is generally recommended.
+
+You should use default target when:
+
+- Your program includes a header file, e.g., ptrace.h, which eventually
+  pulls in some header files containing file scope host assembly codes.
+
+- You can add ``-fno-jump-tables`` to work around the switch table issue.
+
+Otherwise, you can use ``bpf`` target. Additionally, you *must* use bpf target
+when:
+
+- Your program uses data structures with pointer or long / unsigned long
+  types that interface with BPF helpers or context data structures. Access
+  into these structures is verified by the BPF verifier and may result
+  in verification failures if the native architecture is not aligned with
+  the BPF architecture, e.g. 64-bit. An example of this is
+  BPF_PROG_TYPE_SK_MSG require ``-target bpf``
+
+
+.. Links
+.. _Documentation/process/: https://www.kernel.org/doc/html/latest/process/
+.. _MAINTAINERS: ../../MAINTAINERS
+.. _Documentation/networking/netdev-FAQ.txt: ../networking/netdev-FAQ.txt
+.. _netdev FAQ: ../networking/netdev-FAQ.txt
+.. _samples/bpf/: ../../samples/bpf/
+.. _selftests: ../../tools/testing/selftests/bpf/
+.. _Documentation/dev-tools/kselftest.rst:
+   https://www.kernel.org/doc/html/latest/dev-tools/kselftest.html
+
+Happy BPF hacking!
diff --git a/Documentation/bpf/bpf_devel_QA.txt b/Documentation/bpf/bpf_devel_QA.txt
deleted file mode 100644
index da57601153a0..000000000000
--- a/Documentation/bpf/bpf_devel_QA.txt
+++ /dev/null
@@ -1,570 +0,0 @@
-This document provides information for the BPF subsystem about various
-workflows related to reporting bugs, submitting patches, and queueing
-patches for stable kernels.
-
-For general information about submitting patches, please refer to
-Documentation/process/. This document only describes additional specifics
-related to BPF.
-
-Reporting bugs:
----------------
-
-Q: How do I report bugs for BPF kernel code?
-
-A: Since all BPF kernel development as well as bpftool and iproute2 BPF
-   loader development happens through the netdev kernel mailing list,
-   please report any found issues around BPF to the following mailing
-   list:
-
-     netdev@vger.kernel.org
-
-   This may also include issues related to XDP, BPF tracing, etc.
-
-   Given netdev has a high volume of traffic, please also add the BPF
-   maintainers to Cc (from kernel MAINTAINERS file):
-
-     Alexei Starovoitov <ast@kernel.org>
-     Daniel Borkmann <daniel@iogearbox.net>
-
-   In case a buggy commit has already been identified, make sure to keep
-   the actual commit authors in Cc as well for the report. They can
-   typically be identified through the kernel's git tree.
-
-   Please do *not* report BPF issues to bugzilla.kernel.org since it
-   is a guarantee that the reported issue will be overlooked.
-
-Submitting patches:
--------------------
-
-Q: To which mailing list do I need to submit my BPF patches?
-
-A: Please submit your BPF patches to the netdev kernel mailing list:
-
-     netdev@vger.kernel.org
-
-   Historically, BPF came out of networking and has always been maintained
-   by the kernel networking community. Although these days BPF touches
-   many other subsystems as well, the patches are still routed mainly
-   through the networking community.
-
-   In case your patch has changes in various different subsystems (e.g.
-   tracing, security, etc), make sure to Cc the related kernel mailing
-   lists and maintainers from there as well, so they are able to review
-   the changes and provide their Acked-by's to the patches.
-
-Q: Where can I find patches currently under discussion for BPF subsystem?
-
-A: All patches that are Cc'ed to netdev are queued for review under netdev
-   patchwork project:
-
-     http://patchwork.ozlabs.org/project/netdev/list/
-
-   Those patches which target BPF, are assigned to a 'bpf' delegate for
-   further processing from BPF maintainers. The current queue with
-   patches under review can be found at:
-
-     https://patchwork.ozlabs.org/project/netdev/list/?delegate=77147
-
-   Once the patches have been reviewed by the BPF community as a whole
-   and approved by the BPF maintainers, their status in patchwork will be
-   changed to 'Accepted' and the submitter will be notified by mail. This
-   means that the patches look good from a BPF perspective and have been
-   applied to one of the two BPF kernel trees.
-
-   In case feedback from the community requires a respin of the patches,
-   their status in patchwork will be set to 'Changes Requested', and purged
-   from the current review queue. Likewise for cases where patches would
-   get rejected or are not applicable to the BPF trees (but assigned to
-   the 'bpf' delegate).
-
-Q: How do the changes make their way into Linux?
-
-A: There are two BPF kernel trees (git repositories). Once patches have
-   been accepted by the BPF maintainers, they will be applied to one
-   of the two BPF trees:
-
-     https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/
-     https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/
-
-   The bpf tree itself is for fixes only, whereas bpf-next for features,
-   cleanups or other kind of improvements ("next-like" content). This is
-   analogous to net and net-next trees for networking. Both bpf and
-   bpf-next will only have a master branch in order to simplify against
-   which branch patches should get rebased to.
-
-   Accumulated BPF patches in the bpf tree will regularly get pulled
-   into the net kernel tree. Likewise, accumulated BPF patches accepted
-   into the bpf-next tree will make their way into net-next tree. net and
-   net-next are both run by David S. Miller. From there, they will go
-   into the kernel mainline tree run by Linus Torvalds. To read up on the
-   process of net and net-next being merged into the mainline tree, see
-   the netdev FAQ under:
-
-     Documentation/networking/netdev-FAQ.txt
-
-   Occasionally, to prevent merge conflicts, we might send pull requests
-   to other trees (e.g. tracing) with a small subset of the patches, but
-   net and net-next are always the main trees targeted for integration.
-
-   The pull requests will contain a high-level summary of the accumulated
-   patches and can be searched on netdev kernel mailing list through the
-   following subject lines (yyyy-mm-dd is the date of the pull request):
-
-     pull-request: bpf yyyy-mm-dd
-     pull-request: bpf-next yyyy-mm-dd
-
-Q: How do I indicate which tree (bpf vs. bpf-next) my patch should be
-   applied to?
-
-A: The process is the very same as described in the netdev FAQ, so
-   please read up on it. The subject line must indicate whether the
-   patch is a fix or rather "next-like" content in order to let the
-   maintainers know whether it is targeted at bpf or bpf-next.
-
-   For fixes eventually landing in bpf -> net tree, the subject must
-   look like:
-
-     git format-patch --subject-prefix='PATCH bpf' start..finish
-
-   For features/improvements/etc that should eventually land in
-   bpf-next -> net-next, the subject must look like:
-
-     git format-patch --subject-prefix='PATCH bpf-next' start..finish
-
-   If unsure whether the patch or patch series should go into bpf
-   or net directly, or bpf-next or net-next directly, it is not a
-   problem either if the subject line says net or net-next as target.
-   It is eventually up to the maintainers to do the delegation of
-   the patches.
-
-   If it is clear that patches should go into bpf or bpf-next tree,
-   please make sure to rebase the patches against those trees in
-   order to reduce potential conflicts.
-
-   In case the patch or patch series has to be reworked and sent out
-   again in a second or later revision, it is also required to add a
-   version number (v2, v3, ...) into the subject prefix:
-
-     git format-patch --subject-prefix='PATCH net-next v2' start..finish
-
-   When changes have been requested to the patch series, always send the
-   whole patch series again with the feedback incorporated (never send
-   individual diffs on top of the old series).
-
-Q: What does it mean when a patch gets applied to bpf or bpf-next tree?
-
-A: It means that the patch looks good for mainline inclusion from
-   a BPF point of view.
-
-   Be aware that this is not a final verdict that the patch will
-   automatically get accepted into net or net-next trees eventually:
-
-   On the netdev kernel mailing list reviews can come in at any point
-   in time. If discussions around a patch conclude that they cannot
-   get included as-is, we will either apply a follow-up fix or drop
-   them from the trees entirely. Therefore, we also reserve to rebase
-   the trees when deemed necessary. After all, the purpose of the tree
-   is to i) accumulate and stage BPF patches for integration into trees
-   like net and net-next, and ii) run extensive BPF test suite and
-   workloads on the patches before they make their way any further.
-
-   Once the BPF pull request was accepted by David S. Miller, then
-   the patches end up in net or net-next tree, respectively, and
-   make their way from there further into mainline. Again, see the
-   netdev FAQ for additional information e.g. on how often they are
-   merged to mainline.
-
-Q: How long do I need to wait for feedback on my BPF patches?
-
-A: We try to keep the latency low. The usual time to feedback will
-   be around 2 or 3 business days. It may vary depending on the
-   complexity of changes and current patch load.
-
-Q: How often do you send pull requests to major kernel trees like
-   net or net-next?
-
-A: Pull requests will be sent out rather often in order to not
-   accumulate too many patches in bpf or bpf-next.
-
-   As a rule of thumb, expect pull requests for each tree regularly
-   at the end of the week. In some cases pull requests could additionally
-   come also in the middle of the week depending on the current patch
-   load or urgency.
-
-Q: Are patches applied to bpf-next when the merge window is open?
-
-A: For the time when the merge window is open, bpf-next will not be
-   processed. This is roughly analogous to net-next patch processing,
-   so feel free to read up on the netdev FAQ about further details.
-
-   During those two weeks of merge window, we might ask you to resend
-   your patch series once bpf-next is open again. Once Linus released
-   a v*-rc1 after the merge window, we continue processing of bpf-next.
-
-   For non-subscribers to kernel mailing lists, there is also a status
-   page run by David S. Miller on net-next that provides guidance:
-
-     http://vger.kernel.org/~davem/net-next.html
-
-Q: I made a BPF verifier change, do I need to add test cases for
-   BPF kernel selftests?
-
-A: If the patch has changes to the behavior of the verifier, then yes,
-   it is absolutely necessary to add test cases to the BPF kernel
-   selftests suite. If they are not present and we think they are
-   needed, then we might ask for them before accepting any changes.
-
-   In particular, test_verifier.c is tracking a high number of BPF test
-   cases, including a lot of corner cases that LLVM BPF back end may
-   generate out of the restricted C code. Thus, adding test cases is
-   absolutely crucial to make sure future changes do not accidentally
-   affect prior use-cases. Thus, treat those test cases as: verifier
-   behavior that is not tracked in test_verifier.c could potentially
-   be subject to change.
-
-Q: When should I add code to samples/bpf/ and when to BPF kernel
-   selftests?
-
-A: In general, we prefer additions to BPF kernel selftests rather than
-   samples/bpf/. The rationale is very simple: kernel selftests are
-   regularly run by various bots to test for kernel regressions.
-
-   The more test cases we add to BPF selftests, the better the coverage
-   and the less likely it is that those could accidentally break. It is
-   not that BPF kernel selftests cannot demo how a specific feature can
-   be used.
-
-   That said, samples/bpf/ may be a good place for people to get started,
-   so it might be advisable that simple demos of features could go into
-   samples/bpf/, but advanced functional and corner-case testing rather
-   into kernel selftests.
-
-   If your sample looks like a test case, then go for BPF kernel selftests
-   instead!
-
-Q: When should I add code to the bpftool?
-
-A: The main purpose of bpftool (under tools/bpf/bpftool/) is to provide
-   a central user space tool for debugging and introspection of BPF programs
-   and maps that are active in the kernel. If UAPI changes related to BPF
-   enable for dumping additional information of programs or maps, then
-   bpftool should be extended as well to support dumping them.
-
-Q: When should I add code to iproute2's BPF loader?
-
-A: For UAPI changes related to the XDP or tc layer (e.g. cls_bpf), the
-   convention is that those control-path related changes are added to
-   iproute2's BPF loader as well from user space side. This is not only
-   useful to have UAPI changes properly designed to be usable, but also
-   to make those changes available to a wider user base of major
-   downstream distributions.
-
-Q: Do you accept patches as well for iproute2's BPF loader?
-
-A: Patches for the iproute2's BPF loader have to be sent to:
-
-     netdev@vger.kernel.org
-
-   While those patches are not processed by the BPF kernel maintainers,
-   please keep them in Cc as well, so they can be reviewed.
-
-   The official git repository for iproute2 is run by Stephen Hemminger
-   and can be found at:
-
-     https://git.kernel.org/pub/scm/linux/kernel/git/shemminger/iproute2.git/
-
-   The patches need to have a subject prefix of '[PATCH iproute2 master]'
-   or '[PATCH iproute2 net-next]'. 'master' or 'net-next' describes the
-   target branch where the patch should be applied to. Meaning, if kernel
-   changes went into the net-next kernel tree, then the related iproute2
-   changes need to go into the iproute2 net-next branch, otherwise they
-   can be targeted at master branch. The iproute2 net-next branch will get
-   merged into the master branch after the current iproute2 version from
-   master has been released.
-
-   Like BPF, the patches end up in patchwork under the netdev project and
-   are delegated to 'shemminger' for further processing:
-
-     http://patchwork.ozlabs.org/project/netdev/list/?delegate=389
-
-Q: What is the minimum requirement before I submit my BPF patches?
-
-A: When submitting patches, always take the time and properly test your
-   patches *prior* to submission. Never rush them! If maintainers find
-   that your patches have not been properly tested, it is a good way to
-   get them grumpy. Testing patch submissions is a hard requirement!
-
-   Note, fixes that go to bpf tree *must* have a Fixes: tag included. The
-   same applies to fixes that target bpf-next, where the affected commit
-   is in net-next (or in some cases bpf-next). The Fixes: tag is crucial
-   in order to identify follow-up commits and tremendously helps for people
-   having to do backporting, so it is a must have!
-
-   We also don't accept patches with an empty commit message. Take your
-   time and properly write up a high quality commit message, it is
-   essential!
-
-   Think about it this way: other developers looking at your code a month
-   from now need to understand *why* a certain change has been done that
-   way, and whether there have been flaws in the analysis or assumptions
-   that the original author did. Thus providing a proper rationale and
-   describing the use-case for the changes is a must.
-
-   Patch submissions with >1 patch must have a cover letter which includes
-   a high level description of the series. This high level summary will
-   then be placed into the merge commit by the BPF maintainers such that
-   it is also accessible from the git log for future reference.
-
-Q: What do I need to consider when adding a new instruction or feature
-   that would require BPF JIT and/or LLVM integration as well?
-
-A: We try hard to keep all BPF JITs up to date such that the same user
-   experience can be guaranteed when running BPF programs on different
-   architectures without having the program punt to the less efficient
-   interpreter in case the in-kernel BPF JIT is enabled.
-
-   If you are unable to implement or test the required JIT changes for
-   certain architectures, please work together with the related BPF JIT
-   developers in order to get the feature implemented in a timely manner.
-   Please refer to the git log (arch/*/net/) to locate the necessary
-   people for helping out.
-
-   Also always make sure to add BPF test cases (e.g. test_bpf.c and
-   test_verifier.c) for new instructions, so that they can receive
-   broad test coverage and help run-time testing the various BPF JITs.
-
-   In case of new BPF instructions, once the changes have been accepted
-   into the Linux kernel, please implement support into LLVM's BPF back
-   end. See LLVM section below for further information.
-
-Stable submission:
-------------------
-
-Q: I need a specific BPF commit in stable kernels. What should I do?
-
-A: In case you need a specific fix in stable kernels, first check whether
-   the commit has already been applied in the related linux-*.y branches:
-
-     https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/
-
-   If not the case, then drop an email to the BPF maintainers with the
-   netdev kernel mailing list in Cc and ask for the fix to be queued up:
-
-     netdev@vger.kernel.org
-
-   The process in general is the same as on netdev itself, see also the
-   netdev FAQ document.
-
-Q: Do you also backport to kernels not currently maintained as stable?
-
-A: No. If you need a specific BPF commit in kernels that are currently not
-   maintained by the stable maintainers, then you are on your own.
-
-   The current stable and longterm stable kernels are all listed here:
-
-     https://www.kernel.org/
-
-Q: The BPF patch I am about to submit needs to go to stable as well. What
-   should I do?
-
-A: The same rules apply as with netdev patch submissions in general, see
-   netdev FAQ under:
-
-     Documentation/networking/netdev-FAQ.txt
-
-   Never add "Cc: stable@vger.kernel.org" to the patch description, but
-   ask the BPF maintainers to queue the patches instead. This can be done
-   with a note, for example, under the "---" part of the patch which does
-   not go into the git log. Alternatively, this can be done as a simple
-   request by mail instead.
-
-Q: Where do I find currently queued BPF patches that will be submitted
-   to stable?
-
-A: Once patches that fix critical bugs got applied into the bpf tree, they
-   are queued up for stable submission under:
-
-     http://patchwork.ozlabs.org/bundle/bpf/stable/?state=*
-
-   They will be on hold there at minimum until the related commit made its
-   way into the mainline kernel tree.
-
-   After having been under broader exposure, the queued patches will be
-   submitted by the BPF maintainers to the stable maintainers.
-
-Testing patches:
-----------------
-
-Q: Which BPF kernel selftests version should I run my kernel against?
-
-A: If you run a kernel xyz, then always run the BPF kernel selftests from
-   that kernel xyz as well. Do not expect that the BPF selftest from the
-   latest mainline tree will pass all the time.
-
-   In particular, test_bpf.c and test_verifier.c have a large number of
-   test cases and are constantly updated with new BPF test sequences, or
-   existing ones are adapted to verifier changes e.g. due to verifier
-   becoming smarter and being able to better track certain things.
-
-LLVM:
------
-
-Q: Where do I find LLVM with BPF support?
-
-A: The BPF back end for LLVM is upstream in LLVM since version 3.7.1.
-
-   All major distributions these days ship LLVM with BPF back end enabled,
-   so for the majority of use-cases it is not required to compile LLVM by
-   hand anymore, just install the distribution provided package.
-
-   LLVM's static compiler lists the supported targets through 'llc --version',
-   make sure BPF targets are listed. Example:
-
-     $ llc --version
-     LLVM (http://llvm.org/):
-       LLVM version 6.0.0svn
-       Optimized build.
-       Default target: x86_64-unknown-linux-gnu
-       Host CPU: skylake
-
-       Registered Targets:
-         bpf    - BPF (host endian)
-         bpfeb  - BPF (big endian)
-         bpfel  - BPF (little endian)
-         x86    - 32-bit X86: Pentium-Pro and above
-         x86-64 - 64-bit X86: EM64T and AMD64
-
-   For developers in order to utilize the latest features added to LLVM's
-   BPF back end, it is advisable to run the latest LLVM releases. Support
-   for new BPF kernel features such as additions to the BPF instruction
-   set are often developed together.
-
-   All LLVM releases can be found at: http://releases.llvm.org/
-
-Q: Got it, so how do I build LLVM manually anyway?
-
-A: You need cmake and gcc-c++ as build requisites for LLVM. Once you have
-   that set up, proceed with building the latest LLVM and clang version
-   from the git repositories:
-
-     $ git clone http://llvm.org/git/llvm.git
-     $ cd llvm/tools
-     $ git clone --depth 1 http://llvm.org/git/clang.git
-     $ cd ..; mkdir build; cd build
-     $ cmake .. -DLLVM_TARGETS_TO_BUILD="BPF;X86" \
-                -DBUILD_SHARED_LIBS=OFF           \
-                -DCMAKE_BUILD_TYPE=Release        \
-                -DLLVM_BUILD_RUNTIME=OFF
-     $ make -j $(getconf _NPROCESSORS_ONLN)
-
-   The built binaries can then be found in the build/bin/ directory, where
-   you can point the PATH variable to.
-
-Q: Should I notify BPF kernel maintainers about issues in LLVM's BPF code
-   generation back end or about LLVM generated code that the verifier
-   refuses to accept?
-
-A: Yes, please do! LLVM's BPF back end is a key piece of the whole BPF
-   infrastructure and it ties deeply into verification of programs from the
-   kernel side. Therefore, any issues on either side need to be investigated
-   and fixed whenever necessary.
-
-   Therefore, please make sure to bring them up at netdev kernel mailing
-   list and Cc BPF maintainers for LLVM and kernel bits:
-
-     Yonghong Song <yhs@fb.com>
-     Alexei Starovoitov <ast@kernel.org>
-     Daniel Borkmann <daniel@iogearbox.net>
-
-   LLVM also has an issue tracker where BPF related bugs can be found:
-
-     https://bugs.llvm.org/buglist.cgi?quicksearch=bpf
-
-   However, it is better to reach out through mailing lists with having
-   maintainers in Cc.
-
-Q: I have added a new BPF instruction to the kernel, how can I integrate
-   it into LLVM?
-
-A: LLVM has a -mcpu selector for the BPF back end in order to allow the
-   selection of BPF instruction set extensions. By default the 'generic'
-   processor target is used, which is the base instruction set (v1) of BPF.
-
-   LLVM has an option to select -mcpu=probe where it will probe the host
-   kernel for supported BPF instruction set extensions and selects the
-   optimal set automatically.
-
-   For cross-compilation, a specific version can be select manually as well.
-
-     $ llc -march bpf -mcpu=help
-     Available CPUs for this target:
-
-       generic - Select the generic processor.
-       probe   - Select the probe processor.
-       v1      - Select the v1 processor.
-       v2      - Select the v2 processor.
-     [...]
-
-   Newly added BPF instructions to the Linux kernel need to follow the same
-   scheme, bump the instruction set version and implement probing for the
-   extensions such that -mcpu=probe users can benefit from the optimization
-   transparently when upgrading their kernels.
-
-   If you are unable to implement support for the newly added BPF instruction
-   please reach out to BPF developers for help.
-
-   By the way, the BPF kernel selftests run with -mcpu=probe for better
-   test coverage.
-
-Q: In some cases clang flag "-target bpf" is used but in other cases the
-   default clang target, which matches the underlying architecture, is used.
-   What is the difference and when I should use which?
-
-A: Although LLVM IR generation and optimization try to stay architecture
-   independent, "-target <arch>" still has some impact on generated code:
-
-     - BPF program may recursively include header file(s) with file scope
-       inline assembly codes. The default target can handle this well,
-       while bpf target may fail if bpf backend assembler does not
-       understand these assembly codes, which is true in most cases.
-
-     - When compiled without -g, additional elf sections, e.g.,
-       .eh_frame and .rela.eh_frame, may be present in the object file
-       with default target, but not with bpf target.
-
-     - The default target may turn a C switch statement into a switch table
-       lookup and jump operation. Since the switch table is placed
-       in the global readonly section, the bpf program will fail to load.
-       The bpf target does not support switch table optimization.
-       The clang option "-fno-jump-tables" can be used to disable
-       switch table generation.
-
-     - For clang -target bpf, it is guaranteed that pointer or long /
-       unsigned long types will always have a width of 64 bit, no matter
-       whether underlying clang binary or default target (or kernel) is
-       32 bit. However, when native clang target is used, then it will
-       compile these types based on the underlying architecture's conventions,
-       meaning in case of 32 bit architecture, pointer or long / unsigned
-       long types e.g. in BPF context structure will have width of 32 bit
-       while the BPF LLVM back end still operates in 64 bit. The native
-       target is mostly needed in tracing for the case of walking pt_regs
-       or other kernel structures where CPU's register width matters.
-       Otherwise, clang -target bpf is generally recommended.
-
-   You should use default target when:
-
-     - Your program includes a header file, e.g., ptrace.h, which eventually
-       pulls in some header files containing file scope host assembly codes.
-     - You can add "-fno-jump-tables" to work around the switch table issue.
-
-   Otherwise, you can use bpf target. Additionally, you _must_ use bpf target
-   when:
-
-     - Your program uses data structures with pointer or long / unsigned long
-       types that interface with BPF helpers or context data structures. Access
-       into these structures is verified by the BPF verifier and may result
-       in verification failures if the native architecture is not aligned with
-       the BPF architecture, e.g. 64-bit. An example of this is
-       BPF_PROG_TYPE_SK_MSG require '-target bpf'
-
-Happy BPF hacking!
diff --git a/Documentation/cachetlb.txt b/Documentation/cachetlb.txt
deleted file mode 100644
index 6eb9d3f090cd..000000000000
--- a/Documentation/cachetlb.txt
+++ /dev/null
@@ -1,415 +0,0 @@
-==================================
-Cache and TLB Flushing Under Linux
-==================================
-
-:Author: David S. Miller <davem@redhat.com>
-
-This document describes the cache/tlb flushing interfaces called
-by the Linux VM subsystem.  It enumerates over each interface,
-describes its intended purpose, and what side effect is expected
-after the interface is invoked.
-
-The side effects described below are stated for a uniprocessor
-implementation, and what is to happen on that single processor.  The
-SMP cases are a simple extension, in that you just extend the
-definition such that the side effect for a particular interface occurs
-on all processors in the system.  Don't let this scare you into
-thinking SMP cache/tlb flushing must be so inefficient, this is in
-fact an area where many optimizations are possible.  For example,
-if it can be proven that a user address space has never executed
-on a cpu (see mm_cpumask()), one need not perform a flush
-for this address space on that cpu.
-
-First, the TLB flushing interfaces, since they are the simplest.  The
-"TLB" is abstracted under Linux as something the cpu uses to cache
-virtual-->physical address translations obtained from the software
-page tables.  Meaning that if the software page tables change, it is
-possible for stale translations to exist in this "TLB" cache.
-Therefore when software page table changes occur, the kernel will
-invoke one of the following flush methods _after_ the page table
-changes occur:
-
-1) ``void flush_tlb_all(void)``
-
-	The most severe flush of all.  After this interface runs,
-	any previous page table modification whatsoever will be
-	visible to the cpu.
-
-	This is usually invoked when the kernel page tables are
-	changed, since such translations are "global" in nature.
-
-2) ``void flush_tlb_mm(struct mm_struct *mm)``
-
-	This interface flushes an entire user address space from
-	the TLB.  After running, this interface must make sure that
-	any previous page table modifications for the address space
-	'mm' will be visible to the cpu.  That is, after running,
-	there will be no entries in the TLB for 'mm'.
-
-	This interface is used to handle whole address space
-	page table operations such as what happens during
-	fork, and exec.
-
-3) ``void flush_tlb_range(struct vm_area_struct *vma,
-   unsigned long start, unsigned long end)``
-
-	Here we are flushing a specific range of (user) virtual
-	address translations from the TLB.  After running, this
-	interface must make sure that any previous page table
-	modifications for the address space 'vma->vm_mm' in the range
-	'start' to 'end-1' will be visible to the cpu.  That is, after
-	running, there will be no entries in the TLB for 'mm' for
-	virtual addresses in the range 'start' to 'end-1'.
-
-	The "vma" is the backing store being used for the region.
-	Primarily, this is used for munmap() type operations.
-
-	The interface is provided in hopes that the port can find
-	a suitably efficient method for removing multiple page
-	sized translations from the TLB, instead of having the kernel
-	call flush_tlb_page (see below) for each entry which may be
-	modified.
-
-4) ``void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)``
-
-	This time we need to remove the PAGE_SIZE sized translation
-	from the TLB.  The 'vma' is the backing structure used by
-	Linux to keep track of mmap'd regions for a process, the
-	address space is available via vma->vm_mm.  Also, one may
-	test (vma->vm_flags & VM_EXEC) to see if this region is
-	executable (and thus could be in the 'instruction TLB' in
-	split-tlb type setups).
-
-	After running, this interface must make sure that any previous
-	page table modification for address space 'vma->vm_mm' for
-	user virtual address 'addr' will be visible to the cpu.  That
-	is, after running, there will be no entries in the TLB for
-	'vma->vm_mm' for virtual address 'addr'.
-
-	This is used primarily during fault processing.
-
-5) ``void update_mmu_cache(struct vm_area_struct *vma,
-   unsigned long address, pte_t *ptep)``
-
-	At the end of every page fault, this routine is invoked to
-	tell the architecture specific code that a translation
-	now exists at virtual address "address" for address space
-	"vma->vm_mm", in the software page tables.
-
-	A port may use this information in any way it so chooses.
-	For example, it could use this event to pre-load TLB
-	translations for software managed TLB configurations.
-	The sparc64 port currently does this.
-
-6) ``void tlb_migrate_finish(struct mm_struct *mm)``
-
-	This interface is called at the end of an explicit
-	process migration. This interface provides a hook
-	to allow a platform to update TLB or context-specific
-	information for the address space.
-
-	The ia64 sn2 platform is one example of a platform
-	that uses this interface.
-
-Next, we have the cache flushing interfaces.  In general, when Linux
-is changing an existing virtual-->physical mapping to a new value,
-the sequence will be in one of the following forms::
-
-	1) flush_cache_mm(mm);
-	   change_all_page_tables_of(mm);
-	   flush_tlb_mm(mm);
-
-	2) flush_cache_range(vma, start, end);
-	   change_range_of_page_tables(mm, start, end);
-	   flush_tlb_range(vma, start, end);
-
-	3) flush_cache_page(vma, addr, pfn);
-	   set_pte(pte_pointer, new_pte_val);
-	   flush_tlb_page(vma, addr);
-
-The cache level flush will always be first, because this allows
-us to properly handle systems whose caches are strict and require
-a virtual-->physical translation to exist for a virtual address
-when that virtual address is flushed from the cache.  The HyperSparc
-cpu is one such cpu with this attribute.
-
-The cache flushing routines below need only deal with cache flushing
-to the extent that it is necessary for a particular cpu.  Mostly,
-these routines must be implemented for cpus which have virtually
-indexed caches which must be flushed when virtual-->physical
-translations are changed or removed.  So, for example, the physically
-indexed physically tagged caches of IA32 processors have no need to
-implement these interfaces since the caches are fully synchronized
-and have no dependency on translation information.
-
-Here are the routines, one by one:
-
-1) ``void flush_cache_mm(struct mm_struct *mm)``
-
-	This interface flushes an entire user address space from
-	the caches.  That is, after running, there will be no cache
-	lines associated with 'mm'.
-
-	This interface is used to handle whole address space
-	page table operations such as what happens during exit and exec.
-
-2) ``void flush_cache_dup_mm(struct mm_struct *mm)``
-
-	This interface flushes an entire user address space from
-	the caches.  That is, after running, there will be no cache
-	lines associated with 'mm'.
-
-	This interface is used to handle whole address space
-	page table operations such as what happens during fork.
-
-	This option is separate from flush_cache_mm to allow some
-	optimizations for VIPT caches.
-
-3) ``void flush_cache_range(struct vm_area_struct *vma,
-   unsigned long start, unsigned long end)``
-
-	Here we are flushing a specific range of (user) virtual
-	addresses from the cache.  After running, there will be no
-	entries in the cache for 'vma->vm_mm' for virtual addresses in
-	the range 'start' to 'end-1'.
-
-	The "vma" is the backing store being used for the region.
-	Primarily, this is used for munmap() type operations.
-
-	The interface is provided in hopes that the port can find
-	a suitably efficient method for removing multiple page
-	sized regions from the cache, instead of having the kernel
-	call flush_cache_page (see below) for each entry which may be
-	modified.
-
-4) ``void flush_cache_page(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn)``
-
-	This time we need to remove a PAGE_SIZE sized range
-	from the cache.  The 'vma' is the backing structure used by
-	Linux to keep track of mmap'd regions for a process, the
-	address space is available via vma->vm_mm.  Also, one may
-	test (vma->vm_flags & VM_EXEC) to see if this region is
-	executable (and thus could be in the 'instruction cache' in
-	"Harvard" type cache layouts).
-
-	The 'pfn' indicates the physical page frame (shift this value
-	left by PAGE_SHIFT to get the physical address) that 'addr'
-	translates to.  It is this mapping which should be removed from
-	the cache.
-
-	After running, there will be no entries in the cache for
-	'vma->vm_mm' for virtual address 'addr' which translates
-	to 'pfn'.
-
-	This is used primarily during fault processing.
-
-5) ``void flush_cache_kmaps(void)``
-
-	This routine need only be implemented if the platform utilizes
-	highmem.  It will be called right before all of the kmaps
-	are invalidated.
-
-	After running, there will be no entries in the cache for
-	the kernel virtual address range PKMAP_ADDR(0) to
-	PKMAP_ADDR(LAST_PKMAP).
-
-	This routing should be implemented in asm/highmem.h
-
-6) ``void flush_cache_vmap(unsigned long start, unsigned long end)``
-   ``void flush_cache_vunmap(unsigned long start, unsigned long end)``
-
-	Here in these two interfaces we are flushing a specific range
-	of (kernel) virtual addresses from the cache.  After running,
-	there will be no entries in the cache for the kernel address
-	space for virtual addresses in the range 'start' to 'end-1'.
-
-	The first of these two routines is invoked after map_vm_area()
-	has installed the page table entries.  The second is invoked
-	before unmap_kernel_range() deletes the page table entries.
-
-There exists another whole class of cpu cache issues which currently
-require a whole different set of interfaces to handle properly.
-The biggest problem is that of virtual aliasing in the data cache
-of a processor.
-
-Is your port susceptible to virtual aliasing in its D-cache?
-Well, if your D-cache is virtually indexed, is larger in size than
-PAGE_SIZE, and does not prevent multiple cache lines for the same
-physical address from existing at once, you have this problem.
-
-If your D-cache has this problem, first define asm/shmparam.h SHMLBA
-properly, it should essentially be the size of your virtually
-addressed D-cache (or if the size is variable, the largest possible
-size).  This setting will force the SYSv IPC layer to only allow user
-processes to mmap shared memory at address which are a multiple of
-this value.
-
-.. note::
-
-  This does not fix shared mmaps, check out the sparc64 port for
-  one way to solve this (in particular SPARC_FLAG_MMAPSHARED).
-
-Next, you have to solve the D-cache aliasing issue for all
-other cases.  Please keep in mind that fact that, for a given page
-mapped into some user address space, there is always at least one more
-mapping, that of the kernel in its linear mapping starting at
-PAGE_OFFSET.  So immediately, once the first user maps a given
-physical page into its address space, by implication the D-cache
-aliasing problem has the potential to exist since the kernel already
-maps this page at its virtual address.
-
-  ``void copy_user_page(void *to, void *from, unsigned long addr, struct page *page)``
-  ``void clear_user_page(void *to, unsigned long addr, struct page *page)``
-
-	These two routines store data in user anonymous or COW
-	pages.  It allows a port to efficiently avoid D-cache alias
-	issues between userspace and the kernel.
-
-	For example, a port may temporarily map 'from' and 'to' to
-	kernel virtual addresses during the copy.  The virtual address
-	for these two pages is chosen in such a way that the kernel
-	load/store instructions happen to virtual addresses which are
-	of the same "color" as the user mapping of the page.  Sparc64
-	for example, uses this technique.
-
-	The 'addr' parameter tells the virtual address where the
-	user will ultimately have this page mapped, and the 'page'
-	parameter gives a pointer to the struct page of the target.
-
-	If D-cache aliasing is not an issue, these two routines may
-	simply call memcpy/memset directly and do nothing more.
-
-  ``void flush_dcache_page(struct page *page)``
-
-	Any time the kernel writes to a page cache page, _OR_
-	the kernel is about to read from a page cache page and
-	user space shared/writable mappings of this page potentially
-	exist, this routine is called.
-
-	.. note::
-
-	      This routine need only be called for page cache pages
-	      which can potentially ever be mapped into the address
-	      space of a user process.  So for example, VFS layer code
-	      handling vfs symlinks in the page cache need not call
-	      this interface at all.
-
-	The phrase "kernel writes to a page cache page" means,
-	specifically, that the kernel executes store instructions
-	that dirty data in that page at the page->virtual mapping
-	of that page.  It is important to flush here to handle
-	D-cache aliasing, to make sure these kernel stores are
-	visible to user space mappings of that page.
-
-	The corollary case is just as important, if there are users
-	which have shared+writable mappings of this file, we must make
-	sure that kernel reads of these pages will see the most recent
-	stores done by the user.
-
-	If D-cache aliasing is not an issue, this routine may
-	simply be defined as a nop on that architecture.
-
-        There is a bit set aside in page->flags (PG_arch_1) as
-	"architecture private".  The kernel guarantees that,
-	for pagecache pages, it will clear this bit when such
-	a page first enters the pagecache.
-
-	This allows these interfaces to be implemented much more
-	efficiently.  It allows one to "defer" (perhaps indefinitely)
-	the actual flush if there are currently no user processes
-	mapping this page.  See sparc64's flush_dcache_page and
-	update_mmu_cache implementations for an example of how to go
-	about doing this.
-
-	The idea is, first at flush_dcache_page() time, if
-	page->mapping->i_mmap is an empty tree, just mark the architecture
-	private page flag bit.  Later, in update_mmu_cache(), a check is
-	made of this flag bit, and if set the flush is done and the flag
-	bit is cleared.
-
-	.. important::
-
-			It is often important, if you defer the flush,
-			that the actual flush occurs on the same CPU
-			as did the cpu stores into the page to make it
-			dirty.  Again, see sparc64 for examples of how
-			to deal with this.
-
-  ``void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
-  unsigned long user_vaddr, void *dst, void *src, int len)``
-  ``void copy_from_user_page(struct vm_area_struct *vma, struct page *page,
-  unsigned long user_vaddr, void *dst, void *src, int len)``
-
-	When the kernel needs to copy arbitrary data in and out
-	of arbitrary user pages (f.e. for ptrace()) it will use
-	these two routines.
-
-	Any necessary cache flushing or other coherency operations
-	that need to occur should happen here.  If the processor's
-	instruction cache does not snoop cpu stores, it is very
-	likely that you will need to flush the instruction cache
-	for copy_to_user_page().
-
-  ``void flush_anon_page(struct vm_area_struct *vma, struct page *page,
-  unsigned long vmaddr)``
-
-  	When the kernel needs to access the contents of an anonymous
-	page, it calls this function (currently only
-	get_user_pages()).  Note: flush_dcache_page() deliberately
-	doesn't work for an anonymous page.  The default
-	implementation is a nop (and should remain so for all coherent
-	architectures).  For incoherent architectures, it should flush
-	the cache of the page at vmaddr.
-
-  ``void flush_kernel_dcache_page(struct page *page)``
-
-	When the kernel needs to modify a user page is has obtained
-	with kmap, it calls this function after all modifications are
-	complete (but before kunmapping it) to bring the underlying
-	page up to date.  It is assumed here that the user has no
-	incoherent cached copies (i.e. the original page was obtained
-	from a mechanism like get_user_pages()).  The default
-	implementation is a nop and should remain so on all coherent
-	architectures.  On incoherent architectures, this should flush
-	the kernel cache for page (using page_address(page)).
-
-
-  ``void flush_icache_range(unsigned long start, unsigned long end)``
-
-  	When the kernel stores into addresses that it will execute
-	out of (eg when loading modules), this function is called.
-
-	If the icache does not snoop stores then this routine will need
-	to flush it.
-
-  ``void flush_icache_page(struct vm_area_struct *vma, struct page *page)``
-
-	All the functionality of flush_icache_page can be implemented in
-	flush_dcache_page and update_mmu_cache. In the future, the hope
-	is to remove this interface completely.
-
-The final category of APIs is for I/O to deliberately aliased address
-ranges inside the kernel.  Such aliases are set up by use of the
-vmap/vmalloc API.  Since kernel I/O goes via physical pages, the I/O
-subsystem assumes that the user mapping and kernel offset mapping are
-the only aliases.  This isn't true for vmap aliases, so anything in
-the kernel trying to do I/O to vmap areas must manually manage
-coherency.  It must do this by flushing the vmap range before doing
-I/O and invalidating it after the I/O returns.
-
-  ``void flush_kernel_vmap_range(void *vaddr, int size)``
-
-       flushes the kernel cache for a given virtual address range in
-       the vmap area.  This is to make sure that any data the kernel
-       modified in the vmap range is made visible to the physical
-       page.  The design is to make this area safe to perform I/O on.
-       Note that this API does *not* also flush the offset map alias
-       of the area.
-
-  ``void invalidate_kernel_vmap_range(void *vaddr, int size) invalidates``
-
-       the cache for a given virtual address range in the vmap area
-       which prevents the processor from making the cache stale by
-       speculatively reading data while the I/O was occurring to the
-       physical pages.  This is only necessary for data reads into the
-       vmap area.
diff --git a/Documentation/cgroup-v2.txt b/Documentation/cgroup-v2.txt
deleted file mode 100644
index 74cdeaed9f7a..000000000000
--- a/Documentation/cgroup-v2.txt
+++ /dev/null
@@ -1,1998 +0,0 @@
-================
-Control Group v2
-================
-
-:Date: October, 2015
-:Author: Tejun Heo <tj@kernel.org>
-
-This is the authoritative documentation on the design, interface and
-conventions of cgroup v2.  It describes all userland-visible aspects
-of cgroup including core and specific controller behaviors.  All
-future changes must be reflected in this document.  Documentation for
-v1 is available under Documentation/cgroup-v1/.
-
-.. CONTENTS
-
-   1. Introduction
-     1-1. Terminology
-     1-2. What is cgroup?
-   2. Basic Operations
-     2-1. Mounting
-     2-2. Organizing Processes and Threads
-       2-2-1. Processes
-       2-2-2. Threads
-     2-3. [Un]populated Notification
-     2-4. Controlling Controllers
-       2-4-1. Enabling and Disabling
-       2-4-2. Top-down Constraint
-       2-4-3. No Internal Process Constraint
-     2-5. Delegation
-       2-5-1. Model of Delegation
-       2-5-2. Delegation Containment
-     2-6. Guidelines
-       2-6-1. Organize Once and Control
-       2-6-2. Avoid Name Collisions
-   3. Resource Distribution Models
-     3-1. Weights
-     3-2. Limits
-     3-3. Protections
-     3-4. Allocations
-   4. Interface Files
-     4-1. Format
-     4-2. Conventions
-     4-3. Core Interface Files
-   5. Controllers
-     5-1. CPU
-       5-1-1. CPU Interface Files
-     5-2. Memory
-       5-2-1. Memory Interface Files
-       5-2-2. Usage Guidelines
-       5-2-3. Memory Ownership
-     5-3. IO
-       5-3-1. IO Interface Files
-       5-3-2. Writeback
-     5-4. PID
-       5-4-1. PID Interface Files
-     5-5. Device
-     5-6. RDMA
-       5-6-1. RDMA Interface Files
-     5-7. Misc
-       5-7-1. perf_event
-     5-N. Non-normative information
-       5-N-1. CPU controller root cgroup process behaviour
-       5-N-2. IO controller root cgroup process behaviour
-   6. Namespace
-     6-1. Basics
-     6-2. The Root and Views
-     6-3. Migration and setns(2)
-     6-4. Interaction with Other Namespaces
-   P. Information on Kernel Programming
-     P-1. Filesystem Support for Writeback
-   D. Deprecated v1 Core Features
-   R. Issues with v1 and Rationales for v2
-     R-1. Multiple Hierarchies
-     R-2. Thread Granularity
-     R-3. Competition Between Inner Nodes and Threads
-     R-4. Other Interface Issues
-     R-5. Controller Issues and Remedies
-       R-5-1. Memory
-
-
-Introduction
-============
-
-Terminology
------------
-
-"cgroup" stands for "control group" and is never capitalized.  The
-singular form is used to designate the whole feature and also as a
-qualifier as in "cgroup controllers".  When explicitly referring to
-multiple individual control groups, the plural form "cgroups" is used.
-
-
-What is cgroup?
----------------
-
-cgroup is a mechanism to organize processes hierarchically and
-distribute system resources along the hierarchy in a controlled and
-configurable manner.
-
-cgroup is largely composed of two parts - the core and controllers.
-cgroup core is primarily responsible for hierarchically organizing
-processes.  A cgroup controller is usually responsible for
-distributing a specific type of system resource along the hierarchy
-although there are utility controllers which serve purposes other than
-resource distribution.
-
-cgroups form a tree structure and every process in the system belongs
-to one and only one cgroup.  All threads of a process belong to the
-same cgroup.  On creation, all processes are put in the cgroup that
-the parent process belongs to at the time.  A process can be migrated
-to another cgroup.  Migration of a process doesn't affect already
-existing descendant processes.
-
-Following certain structural constraints, controllers may be enabled or
-disabled selectively on a cgroup.  All controller behaviors are
-hierarchical - if a controller is enabled on a cgroup, it affects all
-processes which belong to the cgroups consisting the inclusive
-sub-hierarchy of the cgroup.  When a controller is enabled on a nested
-cgroup, it always restricts the resource distribution further.  The
-restrictions set closer to the root in the hierarchy can not be
-overridden from further away.
-
-
-Basic Operations
-================
-
-Mounting
---------
-
-Unlike v1, cgroup v2 has only single hierarchy.  The cgroup v2
-hierarchy can be mounted with the following mount command::
-
-  # mount -t cgroup2 none $MOUNT_POINT
-
-cgroup2 filesystem has the magic number 0x63677270 ("cgrp").  All
-controllers which support v2 and are not bound to a v1 hierarchy are
-automatically bound to the v2 hierarchy and show up at the root.
-Controllers which are not in active use in the v2 hierarchy can be
-bound to other hierarchies.  This allows mixing v2 hierarchy with the
-legacy v1 multiple hierarchies in a fully backward compatible way.
-
-A controller can be moved across hierarchies only after the controller
-is no longer referenced in its current hierarchy.  Because per-cgroup
-controller states are destroyed asynchronously and controllers may
-have lingering references, a controller may not show up immediately on
-the v2 hierarchy after the final umount of the previous hierarchy.
-Similarly, a controller should be fully disabled to be moved out of
-the unified hierarchy and it may take some time for the disabled
-controller to become available for other hierarchies; furthermore, due
-to inter-controller dependencies, other controllers may need to be
-disabled too.
-
-While useful for development and manual configurations, moving
-controllers dynamically between the v2 and other hierarchies is
-strongly discouraged for production use.  It is recommended to decide
-the hierarchies and controller associations before starting using the
-controllers after system boot.
-
-During transition to v2, system management software might still
-automount the v1 cgroup filesystem and so hijack all controllers
-during boot, before manual intervention is possible. To make testing
-and experimenting easier, the kernel parameter cgroup_no_v1= allows
-disabling controllers in v1 and make them always available in v2.
-
-cgroup v2 currently supports the following mount options.
-
-  nsdelegate
-
-	Consider cgroup namespaces as delegation boundaries.  This
-	option is system wide and can only be set on mount or modified
-	through remount from the init namespace.  The mount option is
-	ignored on non-init namespace mounts.  Please refer to the
-	Delegation section for details.
-
-
-Organizing Processes and Threads
---------------------------------
-
-Processes
-~~~~~~~~~
-
-Initially, only the root cgroup exists to which all processes belong.
-A child cgroup can be created by creating a sub-directory::
-
-  # mkdir $CGROUP_NAME
-
-A given cgroup may have multiple child cgroups forming a tree
-structure.  Each cgroup has a read-writable interface file
-"cgroup.procs".  When read, it lists the PIDs of all processes which
-belong to the cgroup one-per-line.  The PIDs are not ordered and the
-same PID may show up more than once if the process got moved to
-another cgroup and then back or the PID got recycled while reading.
-
-A process can be migrated into a cgroup by writing its PID to the
-target cgroup's "cgroup.procs" file.  Only one process can be migrated
-on a single write(2) call.  If a process is composed of multiple
-threads, writing the PID of any thread migrates all threads of the
-process.
-
-When a process forks a child process, the new process is born into the
-cgroup that the forking process belongs to at the time of the
-operation.  After exit, a process stays associated with the cgroup
-that it belonged to at the time of exit until it's reaped; however, a
-zombie process does not appear in "cgroup.procs" and thus can't be
-moved to another cgroup.
-
-A cgroup which doesn't have any children or live processes can be
-destroyed by removing the directory.  Note that a cgroup which doesn't
-have any children and is associated only with zombie processes is
-considered empty and can be removed::
-
-  # rmdir $CGROUP_NAME
-
-"/proc/$PID/cgroup" lists a process's cgroup membership.  If legacy
-cgroup is in use in the system, this file may contain multiple lines,
-one for each hierarchy.  The entry for cgroup v2 is always in the
-format "0::$PATH"::
-
-  # cat /proc/842/cgroup
-  ...
-  0::/test-cgroup/test-cgroup-nested
-
-If the process becomes a zombie and the cgroup it was associated with
-is removed subsequently, " (deleted)" is appended to the path::
-
-  # cat /proc/842/cgroup
-  ...
-  0::/test-cgroup/test-cgroup-nested (deleted)
-
-
-Threads
-~~~~~~~
-
-cgroup v2 supports thread granularity for a subset of controllers to
-support use cases requiring hierarchical resource distribution across
-the threads of a group of processes.  By default, all threads of a
-process belong to the same cgroup, which also serves as the resource
-domain to host resource consumptions which are not specific to a
-process or thread.  The thread mode allows threads to be spread across
-a subtree while still maintaining the common resource domain for them.
-
-Controllers which support thread mode are called threaded controllers.
-The ones which don't are called domain controllers.
-
-Marking a cgroup threaded makes it join the resource domain of its
-parent as a threaded cgroup.  The parent may be another threaded
-cgroup whose resource domain is further up in the hierarchy.  The root
-of a threaded subtree, that is, the nearest ancestor which is not
-threaded, is called threaded domain or thread root interchangeably and
-serves as the resource domain for the entire subtree.
-
-Inside a threaded subtree, threads of a process can be put in
-different cgroups and are not subject to the no internal process
-constraint - threaded controllers can be enabled on non-leaf cgroups
-whether they have threads in them or not.
-
-As the threaded domain cgroup hosts all the domain resource
-consumptions of the subtree, it is considered to have internal
-resource consumptions whether there are processes in it or not and
-can't have populated child cgroups which aren't threaded.  Because the
-root cgroup is not subject to no internal process constraint, it can
-serve both as a threaded domain and a parent to domain cgroups.
-
-The current operation mode or type of the cgroup is shown in the
-"cgroup.type" file which indicates whether the cgroup is a normal
-domain, a domain which is serving as the domain of a threaded subtree,
-or a threaded cgroup.
-
-On creation, a cgroup is always a domain cgroup and can be made
-threaded by writing "threaded" to the "cgroup.type" file.  The
-operation is single direction::
-
-  # echo threaded > cgroup.type
-
-Once threaded, the cgroup can't be made a domain again.  To enable the
-thread mode, the following conditions must be met.
-
-- As the cgroup will join the parent's resource domain.  The parent
-  must either be a valid (threaded) domain or a threaded cgroup.
-
-- When the parent is an unthreaded domain, it must not have any domain
-  controllers enabled or populated domain children.  The root is
-  exempt from this requirement.
-
-Topology-wise, a cgroup can be in an invalid state.  Please consider
-the following topology::
-
-  A (threaded domain) - B (threaded) - C (domain, just created)
-
-C is created as a domain but isn't connected to a parent which can
-host child domains.  C can't be used until it is turned into a
-threaded cgroup.  "cgroup.type" file will report "domain (invalid)" in
-these cases.  Operations which fail due to invalid topology use
-EOPNOTSUPP as the errno.
-
-A domain cgroup is turned into a threaded domain when one of its child
-cgroup becomes threaded or threaded controllers are enabled in the
-"cgroup.subtree_control" file while there are processes in the cgroup.
-A threaded domain reverts to a normal domain when the conditions
-clear.
-
-When read, "cgroup.threads" contains the list of the thread IDs of all
-threads in the cgroup.  Except that the operations are per-thread
-instead of per-process, "cgroup.threads" has the same format and
-behaves the same way as "cgroup.procs".  While "cgroup.threads" can be
-written to in any cgroup, as it can only move threads inside the same
-threaded domain, its operations are confined inside each threaded
-subtree.
-
-The threaded domain cgroup serves as the resource domain for the whole
-subtree, and, while the threads can be scattered across the subtree,
-all the processes are considered to be in the threaded domain cgroup.
-"cgroup.procs" in a threaded domain cgroup contains the PIDs of all
-processes in the subtree and is not readable in the subtree proper.
-However, "cgroup.procs" can be written to from anywhere in the subtree
-to migrate all threads of the matching process to the cgroup.
-
-Only threaded controllers can be enabled in a threaded subtree.  When
-a threaded controller is enabled inside a threaded subtree, it only
-accounts for and controls resource consumptions associated with the
-threads in the cgroup and its descendants.  All consumptions which
-aren't tied to a specific thread belong to the threaded domain cgroup.
-
-Because a threaded subtree is exempt from no internal process
-constraint, a threaded controller must be able to handle competition
-between threads in a non-leaf cgroup and its child cgroups.  Each
-threaded controller defines how such competitions are handled.
-
-
-[Un]populated Notification
---------------------------
-
-Each non-root cgroup has a "cgroup.events" file which contains
-"populated" field indicating whether the cgroup's sub-hierarchy has
-live processes in it.  Its value is 0 if there is no live process in
-the cgroup and its descendants; otherwise, 1.  poll and [id]notify
-events are triggered when the value changes.  This can be used, for
-example, to start a clean-up operation after all processes of a given
-sub-hierarchy have exited.  The populated state updates and
-notifications are recursive.  Consider the following sub-hierarchy
-where the numbers in the parentheses represent the numbers of processes
-in each cgroup::
-
-  A(4) - B(0) - C(1)
-              \ D(0)
-
-A, B and C's "populated" fields would be 1 while D's 0.  After the one
-process in C exits, B and C's "populated" fields would flip to "0" and
-file modified events will be generated on the "cgroup.events" files of
-both cgroups.
-
-
-Controlling Controllers
------------------------
-
-Enabling and Disabling
-~~~~~~~~~~~~~~~~~~~~~~
-
-Each cgroup has a "cgroup.controllers" file which lists all
-controllers available for the cgroup to enable::
-
-  # cat cgroup.controllers
-  cpu io memory
-
-No controller is enabled by default.  Controllers can be enabled and
-disabled by writing to the "cgroup.subtree_control" file::
-
-  # echo "+cpu +memory -io" > cgroup.subtree_control
-
-Only controllers which are listed in "cgroup.controllers" can be
-enabled.  When multiple operations are specified as above, either they
-all succeed or fail.  If multiple operations on the same controller
-are specified, the last one is effective.
-
-Enabling a controller in a cgroup indicates that the distribution of
-the target resource across its immediate children will be controlled.
-Consider the following sub-hierarchy.  The enabled controllers are
-listed in parentheses::
-
-  A(cpu,memory) - B(memory) - C()
-                            \ D()
-
-As A has "cpu" and "memory" enabled, A will control the distribution
-of CPU cycles and memory to its children, in this case, B.  As B has
-"memory" enabled but not "CPU", C and D will compete freely on CPU
-cycles but their division of memory available to B will be controlled.
-
-As a controller regulates the distribution of the target resource to
-the cgroup's children, enabling it creates the controller's interface
-files in the child cgroups.  In the above example, enabling "cpu" on B
-would create the "cpu." prefixed controller interface files in C and
-D.  Likewise, disabling "memory" from B would remove the "memory."
-prefixed controller interface files from C and D.  This means that the
-controller interface files - anything which doesn't start with
-"cgroup." are owned by the parent rather than the cgroup itself.
-
-
-Top-down Constraint
-~~~~~~~~~~~~~~~~~~~
-
-Resources are distributed top-down and a cgroup can further distribute
-a resource only if the resource has been distributed to it from the
-parent.  This means that all non-root "cgroup.subtree_control" files
-can only contain controllers which are enabled in the parent's
-"cgroup.subtree_control" file.  A controller can be enabled only if
-the parent has the controller enabled and a controller can't be
-disabled if one or more children have it enabled.
-
-
-No Internal Process Constraint
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Non-root cgroups can distribute domain resources to their children
-only when they don't have any processes of their own.  In other words,
-only domain cgroups which don't contain any processes can have domain
-controllers enabled in their "cgroup.subtree_control" files.
-
-This guarantees that, when a domain controller is looking at the part
-of the hierarchy which has it enabled, processes are always only on
-the leaves.  This rules out situations where child cgroups compete
-against internal processes of the parent.
-
-The root cgroup is exempt from this restriction.  Root contains
-processes and anonymous resource consumption which can't be associated
-with any other cgroups and requires special treatment from most
-controllers.  How resource consumption in the root cgroup is governed
-is up to each controller (for more information on this topic please
-refer to the Non-normative information section in the Controllers
-chapter).
-
-Note that the restriction doesn't get in the way if there is no
-enabled controller in the cgroup's "cgroup.subtree_control".  This is
-important as otherwise it wouldn't be possible to create children of a
-populated cgroup.  To control resource distribution of a cgroup, the
-cgroup must create children and transfer all its processes to the
-children before enabling controllers in its "cgroup.subtree_control"
-file.
-
-
-Delegation
-----------
-
-Model of Delegation
-~~~~~~~~~~~~~~~~~~~
-
-A cgroup can be delegated in two ways.  First, to a less privileged
-user by granting write access of the directory and its "cgroup.procs",
-"cgroup.threads" and "cgroup.subtree_control" files to the user.
-Second, if the "nsdelegate" mount option is set, automatically to a
-cgroup namespace on namespace creation.
-
-Because the resource control interface files in a given directory
-control the distribution of the parent's resources, the delegatee
-shouldn't be allowed to write to them.  For the first method, this is
-achieved by not granting access to these files.  For the second, the
-kernel rejects writes to all files other than "cgroup.procs" and
-"cgroup.subtree_control" on a namespace root from inside the
-namespace.
-
-The end results are equivalent for both delegation types.  Once
-delegated, the user can build sub-hierarchy under the directory,
-organize processes inside it as it sees fit and further distribute the
-resources it received from the parent.  The limits and other settings
-of all resource controllers are hierarchical and regardless of what
-happens in the delegated sub-hierarchy, nothing can escape the
-resource restrictions imposed by the parent.
-
-Currently, cgroup doesn't impose any restrictions on the number of
-cgroups in or nesting depth of a delegated sub-hierarchy; however,
-this may be limited explicitly in the future.
-
-
-Delegation Containment
-~~~~~~~~~~~~~~~~~~~~~~
-
-A delegated sub-hierarchy is contained in the sense that processes
-can't be moved into or out of the sub-hierarchy by the delegatee.
-
-For delegations to a less privileged user, this is achieved by
-requiring the following conditions for a process with a non-root euid
-to migrate a target process into a cgroup by writing its PID to the
-"cgroup.procs" file.
-
-- The writer must have write access to the "cgroup.procs" file.
-
-- The writer must have write access to the "cgroup.procs" file of the
-  common ancestor of the source and destination cgroups.
-
-The above two constraints ensure that while a delegatee may migrate
-processes around freely in the delegated sub-hierarchy it can't pull
-in from or push out to outside the sub-hierarchy.
-
-For an example, let's assume cgroups C0 and C1 have been delegated to
-user U0 who created C00, C01 under C0 and C10 under C1 as follows and
-all processes under C0 and C1 belong to U0::
-
-  ~~~~~~~~~~~~~ - C0 - C00
-  ~ cgroup    ~      \ C01
-  ~ hierarchy ~
-  ~~~~~~~~~~~~~ - C1 - C10
-
-Let's also say U0 wants to write the PID of a process which is
-currently in C10 into "C00/cgroup.procs".  U0 has write access to the
-file; however, the common ancestor of the source cgroup C10 and the
-destination cgroup C00 is above the points of delegation and U0 would
-not have write access to its "cgroup.procs" files and thus the write
-will be denied with -EACCES.
-
-For delegations to namespaces, containment is achieved by requiring
-that both the source and destination cgroups are reachable from the
-namespace of the process which is attempting the migration.  If either
-is not reachable, the migration is rejected with -ENOENT.
-
-
-Guidelines
-----------
-
-Organize Once and Control
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Migrating a process across cgroups is a relatively expensive operation
-and stateful resources such as memory are not moved together with the
-process.  This is an explicit design decision as there often exist
-inherent trade-offs between migration and various hot paths in terms
-of synchronization cost.
-
-As such, migrating processes across cgroups frequently as a means to
-apply different resource restrictions is discouraged.  A workload
-should be assigned to a cgroup according to the system's logical and
-resource structure once on start-up.  Dynamic adjustments to resource
-distribution can be made by changing controller configuration through
-the interface files.
-
-
-Avoid Name Collisions
-~~~~~~~~~~~~~~~~~~~~~
-
-Interface files for a cgroup and its children cgroups occupy the same
-directory and it is possible to create children cgroups which collide
-with interface files.
-
-All cgroup core interface files are prefixed with "cgroup." and each
-controller's interface files are prefixed with the controller name and
-a dot.  A controller's name is composed of lower case alphabets and
-'_'s but never begins with an '_' so it can be used as the prefix
-character for collision avoidance.  Also, interface file names won't
-start or end with terms which are often used in categorizing workloads
-such as job, service, slice, unit or workload.
-
-cgroup doesn't do anything to prevent name collisions and it's the
-user's responsibility to avoid them.
-
-
-Resource Distribution Models
-============================
-
-cgroup controllers implement several resource distribution schemes
-depending on the resource type and expected use cases.  This section
-describes major schemes in use along with their expected behaviors.
-
-
-Weights
--------
-
-A parent's resource is distributed by adding up the weights of all
-active children and giving each the fraction matching the ratio of its
-weight against the sum.  As only children which can make use of the
-resource at the moment participate in the distribution, this is
-work-conserving.  Due to the dynamic nature, this model is usually
-used for stateless resources.
-
-All weights are in the range [1, 10000] with the default at 100.  This
-allows symmetric multiplicative biases in both directions at fine
-enough granularity while staying in the intuitive range.
-
-As long as the weight is in range, all configuration combinations are
-valid and there is no reason to reject configuration changes or
-process migrations.
-
-"cpu.weight" proportionally distributes CPU cycles to active children
-and is an example of this type.
-
-
-Limits
-------
-
-A child can only consume upto the configured amount of the resource.
-Limits can be over-committed - the sum of the limits of children can
-exceed the amount of resource available to the parent.
-
-Limits are in the range [0, max] and defaults to "max", which is noop.
-
-As limits can be over-committed, all configuration combinations are
-valid and there is no reason to reject configuration changes or
-process migrations.
-
-"io.max" limits the maximum BPS and/or IOPS that a cgroup can consume
-on an IO device and is an example of this type.
-
-
-Protections
------------
-
-A cgroup is protected to be allocated upto the configured amount of
-the resource if the usages of all its ancestors are under their
-protected levels.  Protections can be hard guarantees or best effort
-soft boundaries.  Protections can also be over-committed in which case
-only upto the amount available to the parent is protected among
-children.
-
-Protections are in the range [0, max] and defaults to 0, which is
-noop.
-
-As protections can be over-committed, all configuration combinations
-are valid and there is no reason to reject configuration changes or
-process migrations.
-
-"memory.low" implements best-effort memory protection and is an
-example of this type.
-
-
-Allocations
------------
-
-A cgroup is exclusively allocated a certain amount of a finite
-resource.  Allocations can't be over-committed - the sum of the
-allocations of children can not exceed the amount of resource
-available to the parent.
-
-Allocations are in the range [0, max] and defaults to 0, which is no
-resource.
-
-As allocations can't be over-committed, some configuration
-combinations are invalid and should be rejected.  Also, if the
-resource is mandatory for execution of processes, process migrations
-may be rejected.
-
-"cpu.rt.max" hard-allocates realtime slices and is an example of this
-type.
-
-
-Interface Files
-===============
-
-Format
-------
-
-All interface files should be in one of the following formats whenever
-possible::
-
-  New-line separated values
-  (when only one value can be written at once)
-
-	VAL0\n
-	VAL1\n
-	...
-
-  Space separated values
-  (when read-only or multiple values can be written at once)
-
-	VAL0 VAL1 ...\n
-
-  Flat keyed
-
-	KEY0 VAL0\n
-	KEY1 VAL1\n
-	...
-
-  Nested keyed
-
-	KEY0 SUB_KEY0=VAL00 SUB_KEY1=VAL01...
-	KEY1 SUB_KEY0=VAL10 SUB_KEY1=VAL11...
-	...
-
-For a writable file, the format for writing should generally match
-reading; however, controllers may allow omitting later fields or
-implement restricted shortcuts for most common use cases.
-
-For both flat and nested keyed files, only the values for a single key
-can be written at a time.  For nested keyed files, the sub key pairs
-may be specified in any order and not all pairs have to be specified.
-
-
-Conventions
------------
-
-- Settings for a single feature should be contained in a single file.
-
-- The root cgroup should be exempt from resource control and thus
-  shouldn't have resource control interface files.  Also,
-  informational files on the root cgroup which end up showing global
-  information available elsewhere shouldn't exist.
-
-- If a controller implements weight based resource distribution, its
-  interface file should be named "weight" and have the range [1,
-  10000] with 100 as the default.  The values are chosen to allow
-  enough and symmetric bias in both directions while keeping it
-  intuitive (the default is 100%).
-
-- If a controller implements an absolute resource guarantee and/or
-  limit, the interface files should be named "min" and "max"
-  respectively.  If a controller implements best effort resource
-  guarantee and/or limit, the interface files should be named "low"
-  and "high" respectively.
-
-  In the above four control files, the special token "max" should be
-  used to represent upward infinity for both reading and writing.
-
-- If a setting has a configurable default value and keyed specific
-  overrides, the default entry should be keyed with "default" and
-  appear as the first entry in the file.
-
-  The default value can be updated by writing either "default $VAL" or
-  "$VAL".
-
-  When writing to update a specific override, "default" can be used as
-  the value to indicate removal of the override.  Override entries
-  with "default" as the value must not appear when read.
-
-  For example, a setting which is keyed by major:minor device numbers
-  with integer values may look like the following::
-
-    # cat cgroup-example-interface-file
-    default 150
-    8:0 300
-
-  The default value can be updated by::
-
-    # echo 125 > cgroup-example-interface-file
-
-  or::
-
-    # echo "default 125" > cgroup-example-interface-file
-
-  An override can be set by::
-
-    # echo "8:16 170" > cgroup-example-interface-file
-
-  and cleared by::
-
-    # echo "8:0 default" > cgroup-example-interface-file
-    # cat cgroup-example-interface-file
-    default 125
-    8:16 170
-
-- For events which are not very high frequency, an interface file
-  "events" should be created which lists event key value pairs.
-  Whenever a notifiable event happens, file modified event should be
-  generated on the file.
-
-
-Core Interface Files
---------------------
-
-All cgroup core files are prefixed with "cgroup."
-
-  cgroup.type
-
-	A read-write single value file which exists on non-root
-	cgroups.
-
-	When read, it indicates the current type of the cgroup, which
-	can be one of the following values.
-
-	- "domain" : A normal valid domain cgroup.
-
-	- "domain threaded" : A threaded domain cgroup which is
-          serving as the root of a threaded subtree.
-
-	- "domain invalid" : A cgroup which is in an invalid state.
-	  It can't be populated or have controllers enabled.  It may
-	  be allowed to become a threaded cgroup.
-
-	- "threaded" : A threaded cgroup which is a member of a
-          threaded subtree.
-
-	A cgroup can be turned into a threaded cgroup by writing
-	"threaded" to this file.
-
-  cgroup.procs
-	A read-write new-line separated values file which exists on
-	all cgroups.
-
-	When read, it lists the PIDs of all processes which belong to
-	the cgroup one-per-line.  The PIDs are not ordered and the
-	same PID may show up more than once if the process got moved
-	to another cgroup and then back or the PID got recycled while
-	reading.
-
-	A PID can be written to migrate the process associated with
-	the PID to the cgroup.  The writer should match all of the
-	following conditions.
-
-	- It must have write access to the "cgroup.procs" file.
-
-	- It must have write access to the "cgroup.procs" file of the
-	  common ancestor of the source and destination cgroups.
-
-	When delegating a sub-hierarchy, write access to this file
-	should be granted along with the containing directory.
-
-	In a threaded cgroup, reading this file fails with EOPNOTSUPP
-	as all the processes belong to the thread root.  Writing is
-	supported and moves every thread of the process to the cgroup.
-
-  cgroup.threads
-	A read-write new-line separated values file which exists on
-	all cgroups.
-
-	When read, it lists the TIDs of all threads which belong to
-	the cgroup one-per-line.  The TIDs are not ordered and the
-	same TID may show up more than once if the thread got moved to
-	another cgroup and then back or the TID got recycled while
-	reading.
-
-	A TID can be written to migrate the thread associated with the
-	TID to the cgroup.  The writer should match all of the
-	following conditions.
-
-	- It must have write access to the "cgroup.threads" file.
-
-	- The cgroup that the thread is currently in must be in the
-          same resource domain as the destination cgroup.
-
-	- It must have write access to the "cgroup.procs" file of the
-	  common ancestor of the source and destination cgroups.
-
-	When delegating a sub-hierarchy, write access to this file
-	should be granted along with the containing directory.
-
-  cgroup.controllers
-	A read-only space separated values file which exists on all
-	cgroups.
-
-	It shows space separated list of all controllers available to
-	the cgroup.  The controllers are not ordered.
-
-  cgroup.subtree_control
-	A read-write space separated values file which exists on all
-	cgroups.  Starts out empty.
-
-	When read, it shows space separated list of the controllers
-	which are enabled to control resource distribution from the
-	cgroup to its children.
-
-	Space separated list of controllers prefixed with '+' or '-'
-	can be written to enable or disable controllers.  A controller
-	name prefixed with '+' enables the controller and '-'
-	disables.  If a controller appears more than once on the list,
-	the last one is effective.  When multiple enable and disable
-	operations are specified, either all succeed or all fail.
-
-  cgroup.events
-	A read-only flat-keyed file which exists on non-root cgroups.
-	The following entries are defined.  Unless specified
-	otherwise, a value change in this file generates a file
-	modified event.
-
-	  populated
-		1 if the cgroup or its descendants contains any live
-		processes; otherwise, 0.
-
-  cgroup.max.descendants
-	A read-write single value files.  The default is "max".
-
-	Maximum allowed number of descent cgroups.
-	If the actual number of descendants is equal or larger,
-	an attempt to create a new cgroup in the hierarchy will fail.
-
-  cgroup.max.depth
-	A read-write single value files.  The default is "max".
-
-	Maximum allowed descent depth below the current cgroup.
-	If the actual descent depth is equal or larger,
-	an attempt to create a new child cgroup will fail.
-
-  cgroup.stat
-	A read-only flat-keyed file with the following entries:
-
-	  nr_descendants
-		Total number of visible descendant cgroups.
-
-	  nr_dying_descendants
-		Total number of dying descendant cgroups. A cgroup becomes
-		dying after being deleted by a user. The cgroup will remain
-		in dying state for some time undefined time (which can depend
-		on system load) before being completely destroyed.
-
-		A process can't enter a dying cgroup under any circumstances,
-		a dying cgroup can't revive.
-
-		A dying cgroup can consume system resources not exceeding
-		limits, which were active at the moment of cgroup deletion.
-
-
-Controllers
-===========
-
-CPU
----
-
-The "cpu" controllers regulates distribution of CPU cycles.  This
-controller implements weight and absolute bandwidth limit models for
-normal scheduling policy and absolute bandwidth allocation model for
-realtime scheduling policy.
-
-WARNING: cgroup2 doesn't yet support control of realtime processes and
-the cpu controller can only be enabled when all RT processes are in
-the root cgroup.  Be aware that system management software may already
-have placed RT processes into nonroot cgroups during the system boot
-process, and these processes may need to be moved to the root cgroup
-before the cpu controller can be enabled.
-
-
-CPU Interface Files
-~~~~~~~~~~~~~~~~~~~
-
-All time durations are in microseconds.
-
-  cpu.stat
-	A read-only flat-keyed file which exists on non-root cgroups.
-	This file exists whether the controller is enabled or not.
-
-	It always reports the following three stats:
-
-	- usage_usec
-	- user_usec
-	- system_usec
-
-	and the following three when the controller is enabled:
-
-	- nr_periods
-	- nr_throttled
-	- throttled_usec
-
-  cpu.weight
-	A read-write single value file which exists on non-root
-	cgroups.  The default is "100".
-
-	The weight in the range [1, 10000].
-
-  cpu.weight.nice
-	A read-write single value file which exists on non-root
-	cgroups.  The default is "0".
-
-	The nice value is in the range [-20, 19].
-
-	This interface file is an alternative interface for
-	"cpu.weight" and allows reading and setting weight using the
-	same values used by nice(2).  Because the range is smaller and
-	granularity is coarser for the nice values, the read value is
-	the closest approximation of the current weight.
-
-  cpu.max
-	A read-write two value file which exists on non-root cgroups.
-	The default is "max 100000".
-
-	The maximum bandwidth limit.  It's in the following format::
-
-	  $MAX $PERIOD
-
-	which indicates that the group may consume upto $MAX in each
-	$PERIOD duration.  "max" for $MAX indicates no limit.  If only
-	one number is written, $MAX is updated.
-
-
-Memory
-------
-
-The "memory" controller regulates distribution of memory.  Memory is
-stateful and implements both limit and protection models.  Due to the
-intertwining between memory usage and reclaim pressure and the
-stateful nature of memory, the distribution model is relatively
-complex.
-
-While not completely water-tight, all major memory usages by a given
-cgroup are tracked so that the total memory consumption can be
-accounted and controlled to a reasonable extent.  Currently, the
-following types of memory usages are tracked.
-
-- Userland memory - page cache and anonymous memory.
-
-- Kernel data structures such as dentries and inodes.
-
-- TCP socket buffers.
-
-The above list may expand in the future for better coverage.
-
-
-Memory Interface Files
-~~~~~~~~~~~~~~~~~~~~~~
-
-All memory amounts are in bytes.  If a value which is not aligned to
-PAGE_SIZE is written, the value may be rounded up to the closest
-PAGE_SIZE multiple when read back.
-
-  memory.current
-	A read-only single value file which exists on non-root
-	cgroups.
-
-	The total amount of memory currently being used by the cgroup
-	and its descendants.
-
-  memory.low
-	A read-write single value file which exists on non-root
-	cgroups.  The default is "0".
-
-	Best-effort memory protection.  If the memory usages of a
-	cgroup and all its ancestors are below their low boundaries,
-	the cgroup's memory won't be reclaimed unless memory can be
-	reclaimed from unprotected cgroups.
-
-	Putting more memory than generally available under this
-	protection is discouraged.
-
-  memory.high
-	A read-write single value file which exists on non-root
-	cgroups.  The default is "max".
-
-	Memory usage throttle limit.  This is the main mechanism to
-	control memory usage of a cgroup.  If a cgroup's usage goes
-	over the high boundary, the processes of the cgroup are
-	throttled and put under heavy reclaim pressure.
-
-	Going over the high limit never invokes the OOM killer and
-	under extreme conditions the limit may be breached.
-
-  memory.max
-	A read-write single value file which exists on non-root
-	cgroups.  The default is "max".
-
-	Memory usage hard limit.  This is the final protection
-	mechanism.  If a cgroup's memory usage reaches this limit and
-	can't be reduced, the OOM killer is invoked in the cgroup.
-	Under certain circumstances, the usage may go over the limit
-	temporarily.
-
-	This is the ultimate protection mechanism.  As long as the
-	high limit is used and monitored properly, this limit's
-	utility is limited to providing the final safety net.
-
-  memory.events
-	A read-only flat-keyed file which exists on non-root cgroups.
-	The following entries are defined.  Unless specified
-	otherwise, a value change in this file generates a file
-	modified event.
-
-	  low
-		The number of times the cgroup is reclaimed due to
-		high memory pressure even though its usage is under
-		the low boundary.  This usually indicates that the low
-		boundary is over-committed.
-
-	  high
-		The number of times processes of the cgroup are
-		throttled and routed to perform direct memory reclaim
-		because the high memory boundary was exceeded.  For a
-		cgroup whose memory usage is capped by the high limit
-		rather than global memory pressure, this event's
-		occurrences are expected.
-
-	  max
-		The number of times the cgroup's memory usage was
-		about to go over the max boundary.  If direct reclaim
-		fails to bring it down, the cgroup goes to OOM state.
-
-	  oom
-		The number of time the cgroup's memory usage was
-		reached the limit and allocation was about to fail.
-
-		Depending on context result could be invocation of OOM
-		killer and retrying allocation or failing allocation.
-
-		Failed allocation in its turn could be returned into
-		userspace as -ENOMEM or silently ignored in cases like
-		disk readahead.  For now OOM in memory cgroup kills
-		tasks iff shortage has happened inside page fault.
-
-	  oom_kill
-		The number of processes belonging to this cgroup
-		killed by any kind of OOM killer.
-
-  memory.stat
-	A read-only flat-keyed file which exists on non-root cgroups.
-
-	This breaks down the cgroup's memory footprint into different
-	types of memory, type-specific details, and other information
-	on the state and past events of the memory management system.
-
-	All memory amounts are in bytes.
-
-	The entries are ordered to be human readable, and new entries
-	can show up in the middle. Don't rely on items remaining in a
-	fixed position; use the keys to look up specific values!
-
-	  anon
-		Amount of memory used in anonymous mappings such as
-		brk(), sbrk(), and mmap(MAP_ANONYMOUS)
-
-	  file
-		Amount of memory used to cache filesystem data,
-		including tmpfs and shared memory.
-
-	  kernel_stack
-		Amount of memory allocated to kernel stacks.
-
-	  slab
-		Amount of memory used for storing in-kernel data
-		structures.
-
-	  sock
-		Amount of memory used in network transmission buffers
-
-	  shmem
-		Amount of cached filesystem data that is swap-backed,
-		such as tmpfs, shm segments, shared anonymous mmap()s
-
-	  file_mapped
-		Amount of cached filesystem data mapped with mmap()
-
-	  file_dirty
-		Amount of cached filesystem data that was modified but
-		not yet written back to disk
-
-	  file_writeback
-		Amount of cached filesystem data that was modified and
-		is currently being written back to disk
-
-	  inactive_anon, active_anon, inactive_file, active_file, unevictable
-		Amount of memory, swap-backed and filesystem-backed,
-		on the internal memory management lists used by the
-		page reclaim algorithm
-
-	  slab_reclaimable
-		Part of "slab" that might be reclaimed, such as
-		dentries and inodes.
-
-	  slab_unreclaimable
-		Part of "slab" that cannot be reclaimed on memory
-		pressure.
-
-	  pgfault
-		Total number of page faults incurred
-
-	  pgmajfault
-		Number of major page faults incurred
-
-	  workingset_refault
-
-		Number of refaults of previously evicted pages
-
-	  workingset_activate
-
-		Number of refaulted pages that were immediately activated
-
-	  workingset_nodereclaim
-
-		Number of times a shadow node has been reclaimed
-
-	  pgrefill
-
-		Amount of scanned pages (in an active LRU list)
-
-	  pgscan
-
-		Amount of scanned pages (in an inactive LRU list)
-
-	  pgsteal
-
-		Amount of reclaimed pages
-
-	  pgactivate
-
-		Amount of pages moved to the active LRU list
-
-	  pgdeactivate
-
-		Amount of pages moved to the inactive LRU lis
-
-	  pglazyfree
-
-		Amount of pages postponed to be freed under memory pressure
-
-	  pglazyfreed
-
-		Amount of reclaimed lazyfree pages
-
-  memory.swap.current
-	A read-only single value file which exists on non-root
-	cgroups.
-
-	The total amount of swap currently being used by the cgroup
-	and its descendants.
-
-  memory.swap.max
-	A read-write single value file which exists on non-root
-	cgroups.  The default is "max".
-
-	Swap usage hard limit.  If a cgroup's swap usage reaches this
-	limit, anonymous memory of the cgroup will not be swapped out.
-
-
-Usage Guidelines
-~~~~~~~~~~~~~~~~
-
-"memory.high" is the main mechanism to control memory usage.
-Over-committing on high limit (sum of high limits > available memory)
-and letting global memory pressure to distribute memory according to
-usage is a viable strategy.
-
-Because breach of the high limit doesn't trigger the OOM killer but
-throttles the offending cgroup, a management agent has ample
-opportunities to monitor and take appropriate actions such as granting
-more memory or terminating the workload.
-
-Determining whether a cgroup has enough memory is not trivial as
-memory usage doesn't indicate whether the workload can benefit from
-more memory.  For example, a workload which writes data received from
-network to a file can use all available memory but can also operate as
-performant with a small amount of memory.  A measure of memory
-pressure - how much the workload is being impacted due to lack of
-memory - is necessary to determine whether a workload needs more
-memory; unfortunately, memory pressure monitoring mechanism isn't
-implemented yet.
-
-
-Memory Ownership
-~~~~~~~~~~~~~~~~
-
-A memory area is charged to the cgroup which instantiated it and stays
-charged to the cgroup until the area is released.  Migrating a process
-to a different cgroup doesn't move the memory usages that it
-instantiated while in the previous cgroup to the new cgroup.
-
-A memory area may be used by processes belonging to different cgroups.
-To which cgroup the area will be charged is in-deterministic; however,
-over time, the memory area is likely to end up in a cgroup which has
-enough memory allowance to avoid high reclaim pressure.
-
-If a cgroup sweeps a considerable amount of memory which is expected
-to be accessed repeatedly by other cgroups, it may make sense to use
-POSIX_FADV_DONTNEED to relinquish the ownership of memory areas
-belonging to the affected files to ensure correct memory ownership.
-
-
-IO
---
-
-The "io" controller regulates the distribution of IO resources.  This
-controller implements both weight based and absolute bandwidth or IOPS
-limit distribution; however, weight based distribution is available
-only if cfq-iosched is in use and neither scheme is available for
-blk-mq devices.
-
-
-IO Interface Files
-~~~~~~~~~~~~~~~~~~
-
-  io.stat
-	A read-only nested-keyed file which exists on non-root
-	cgroups.
-
-	Lines are keyed by $MAJ:$MIN device numbers and not ordered.
-	The following nested keys are defined.
-
-	  ======	===================
-	  rbytes	Bytes read
-	  wbytes	Bytes written
-	  rios		Number of read IOs
-	  wios		Number of write IOs
-	  ======	===================
-
-	An example read output follows:
-
-	  8:16 rbytes=1459200 wbytes=314773504 rios=192 wios=353
-	  8:0 rbytes=90430464 wbytes=299008000 rios=8950 wios=1252
-
-  io.weight
-	A read-write flat-keyed file which exists on non-root cgroups.
-	The default is "default 100".
-
-	The first line is the default weight applied to devices
-	without specific override.  The rest are overrides keyed by
-	$MAJ:$MIN device numbers and not ordered.  The weights are in
-	the range [1, 10000] and specifies the relative amount IO time
-	the cgroup can use in relation to its siblings.
-
-	The default weight can be updated by writing either "default
-	$WEIGHT" or simply "$WEIGHT".  Overrides can be set by writing
-	"$MAJ:$MIN $WEIGHT" and unset by writing "$MAJ:$MIN default".
-
-	An example read output follows::
-
-	  default 100
-	  8:16 200
-	  8:0 50
-
-  io.max
-	A read-write nested-keyed file which exists on non-root
-	cgroups.
-
-	BPS and IOPS based IO limit.  Lines are keyed by $MAJ:$MIN
-	device numbers and not ordered.  The following nested keys are
-	defined.
-
-	  =====		==================================
-	  rbps		Max read bytes per second
-	  wbps		Max write bytes per second
-	  riops		Max read IO operations per second
-	  wiops		Max write IO operations per second
-	  =====		==================================
-
-	When writing, any number of nested key-value pairs can be
-	specified in any order.  "max" can be specified as the value
-	to remove a specific limit.  If the same key is specified
-	multiple times, the outcome is undefined.
-
-	BPS and IOPS are measured in each IO direction and IOs are
-	delayed if limit is reached.  Temporary bursts are allowed.
-
-	Setting read limit at 2M BPS and write at 120 IOPS for 8:16::
-
-	  echo "8:16 rbps=2097152 wiops=120" > io.max
-
-	Reading returns the following::
-
-	  8:16 rbps=2097152 wbps=max riops=max wiops=120
-
-	Write IOPS limit can be removed by writing the following::
-
-	  echo "8:16 wiops=max" > io.max
-
-	Reading now returns the following::
-
-	  8:16 rbps=2097152 wbps=max riops=max wiops=max
-
-
-Writeback
-~~~~~~~~~
-
-Page cache is dirtied through buffered writes and shared mmaps and
-written asynchronously to the backing filesystem by the writeback
-mechanism.  Writeback sits between the memory and IO domains and
-regulates the proportion of dirty memory by balancing dirtying and
-write IOs.
-
-The io controller, in conjunction with the memory controller,
-implements control of page cache writeback IOs.  The memory controller
-defines the memory domain that dirty memory ratio is calculated and
-maintained for and the io controller defines the io domain which
-writes out dirty pages for the memory domain.  Both system-wide and
-per-cgroup dirty memory states are examined and the more restrictive
-of the two is enforced.
-
-cgroup writeback requires explicit support from the underlying
-filesystem.  Currently, cgroup writeback is implemented on ext2, ext4
-and btrfs.  On other filesystems, all writeback IOs are attributed to
-the root cgroup.
-
-There are inherent differences in memory and writeback management
-which affects how cgroup ownership is tracked.  Memory is tracked per
-page while writeback per inode.  For the purpose of writeback, an
-inode is assigned to a cgroup and all IO requests to write dirty pages
-from the inode are attributed to that cgroup.
-
-As cgroup ownership for memory is tracked per page, there can be pages
-which are associated with different cgroups than the one the inode is
-associated with.  These are called foreign pages.  The writeback
-constantly keeps track of foreign pages and, if a particular foreign
-cgroup becomes the majority over a certain period of time, switches
-the ownership of the inode to that cgroup.
-
-While this model is enough for most use cases where a given inode is
-mostly dirtied by a single cgroup even when the main writing cgroup
-changes over time, use cases where multiple cgroups write to a single
-inode simultaneously are not supported well.  In such circumstances, a
-significant portion of IOs are likely to be attributed incorrectly.
-As memory controller assigns page ownership on the first use and
-doesn't update it until the page is released, even if writeback
-strictly follows page ownership, multiple cgroups dirtying overlapping
-areas wouldn't work as expected.  It's recommended to avoid such usage
-patterns.
-
-The sysctl knobs which affect writeback behavior are applied to cgroup
-writeback as follows.
-
-  vm.dirty_background_ratio, vm.dirty_ratio
-	These ratios apply the same to cgroup writeback with the
-	amount of available memory capped by limits imposed by the
-	memory controller and system-wide clean memory.
-
-  vm.dirty_background_bytes, vm.dirty_bytes
-	For cgroup writeback, this is calculated into ratio against
-	total available memory and applied the same way as
-	vm.dirty[_background]_ratio.
-
-
-PID
----
-
-The process number controller is used to allow a cgroup to stop any
-new tasks from being fork()'d or clone()'d after a specified limit is
-reached.
-
-The number of tasks in a cgroup can be exhausted in ways which other
-controllers cannot prevent, thus warranting its own controller.  For
-example, a fork bomb is likely to exhaust the number of tasks before
-hitting memory restrictions.
-
-Note that PIDs used in this controller refer to TIDs, process IDs as
-used by the kernel.
-
-
-PID Interface Files
-~~~~~~~~~~~~~~~~~~~
-
-  pids.max
-	A read-write single value file which exists on non-root
-	cgroups.  The default is "max".
-
-	Hard limit of number of processes.
-
-  pids.current
-	A read-only single value file which exists on all cgroups.
-
-	The number of processes currently in the cgroup and its
-	descendants.
-
-Organisational operations are not blocked by cgroup policies, so it is
-possible to have pids.current > pids.max.  This can be done by either
-setting the limit to be smaller than pids.current, or attaching enough
-processes to the cgroup such that pids.current is larger than
-pids.max.  However, it is not possible to violate a cgroup PID policy
-through fork() or clone(). These will return -EAGAIN if the creation
-of a new process would cause a cgroup policy to be violated.
-
-
-Device controller
------------------
-
-Device controller manages access to device files. It includes both
-creation of new device files (using mknod), and access to the
-existing device files.
-
-Cgroup v2 device controller has no interface files and is implemented
-on top of cgroup BPF. To control access to device files, a user may
-create bpf programs of the BPF_CGROUP_DEVICE type and attach them
-to cgroups. On an attempt to access a device file, corresponding
-BPF programs will be executed, and depending on the return value
-the attempt will succeed or fail with -EPERM.
-
-A BPF_CGROUP_DEVICE program takes a pointer to the bpf_cgroup_dev_ctx
-structure, which describes the device access attempt: access type
-(mknod/read/write) and device (type, major and minor numbers).
-If the program returns 0, the attempt fails with -EPERM, otherwise
-it succeeds.
-
-An example of BPF_CGROUP_DEVICE program may be found in the kernel
-source tree in the tools/testing/selftests/bpf/dev_cgroup.c file.
-
-
-RDMA
-----
-
-The "rdma" controller regulates the distribution and accounting of
-of RDMA resources.
-
-RDMA Interface Files
-~~~~~~~~~~~~~~~~~~~~
-
-  rdma.max
-	A readwrite nested-keyed file that exists for all the cgroups
-	except root that describes current configured resource limit
-	for a RDMA/IB device.
-
-	Lines are keyed by device name and are not ordered.
-	Each line contains space separated resource name and its configured
-	limit that can be distributed.
-
-	The following nested keys are defined.
-
-	  ==========	=============================
-	  hca_handle	Maximum number of HCA Handles
-	  hca_object 	Maximum number of HCA Objects
-	  ==========	=============================
-
-	An example for mlx4 and ocrdma device follows::
-
-	  mlx4_0 hca_handle=2 hca_object=2000
-	  ocrdma1 hca_handle=3 hca_object=max
-
-  rdma.current
-	A read-only file that describes current resource usage.
-	It exists for all the cgroup except root.
-
-	An example for mlx4 and ocrdma device follows::
-
-	  mlx4_0 hca_handle=1 hca_object=20
-	  ocrdma1 hca_handle=1 hca_object=23
-
-
-Misc
-----
-
-perf_event
-~~~~~~~~~~
-
-perf_event controller, if not mounted on a legacy hierarchy, is
-automatically enabled on the v2 hierarchy so that perf events can
-always be filtered by cgroup v2 path.  The controller can still be
-moved to a legacy hierarchy after v2 hierarchy is populated.
-
-
-Non-normative information
--------------------------
-
-This section contains information that isn't considered to be a part of
-the stable kernel API and so is subject to change.
-
-
-CPU controller root cgroup process behaviour
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-When distributing CPU cycles in the root cgroup each thread in this
-cgroup is treated as if it was hosted in a separate child cgroup of the
-root cgroup. This child cgroup weight is dependent on its thread nice
-level.
-
-For details of this mapping see sched_prio_to_weight array in
-kernel/sched/core.c file (values from this array should be scaled
-appropriately so the neutral - nice 0 - value is 100 instead of 1024).
-
-
-IO controller root cgroup process behaviour
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Root cgroup processes are hosted in an implicit leaf child node.
-When distributing IO resources this implicit child node is taken into
-account as if it was a normal child cgroup of the root cgroup with a
-weight value of 200.
-
-
-Namespace
-=========
-
-Basics
-------
-
-cgroup namespace provides a mechanism to virtualize the view of the
-"/proc/$PID/cgroup" file and cgroup mounts.  The CLONE_NEWCGROUP clone
-flag can be used with clone(2) and unshare(2) to create a new cgroup
-namespace.  The process running inside the cgroup namespace will have
-its "/proc/$PID/cgroup" output restricted to cgroupns root.  The
-cgroupns root is the cgroup of the process at the time of creation of
-the cgroup namespace.
-
-Without cgroup namespace, the "/proc/$PID/cgroup" file shows the
-complete path of the cgroup of a process.  In a container setup where
-a set of cgroups and namespaces are intended to isolate processes the
-"/proc/$PID/cgroup" file may leak potential system level information
-to the isolated processes.  For Example::
-
-  # cat /proc/self/cgroup
-  0::/batchjobs/container_id1
-
-The path '/batchjobs/container_id1' can be considered as system-data
-and undesirable to expose to the isolated processes.  cgroup namespace
-can be used to restrict visibility of this path.  For example, before
-creating a cgroup namespace, one would see::
-
-  # ls -l /proc/self/ns/cgroup
-  lrwxrwxrwx 1 root root 0 2014-07-15 10:37 /proc/self/ns/cgroup -> cgroup:[4026531835]
-  # cat /proc/self/cgroup
-  0::/batchjobs/container_id1
-
-After unsharing a new namespace, the view changes::
-
-  # ls -l /proc/self/ns/cgroup
-  lrwxrwxrwx 1 root root 0 2014-07-15 10:35 /proc/self/ns/cgroup -> cgroup:[4026532183]
-  # cat /proc/self/cgroup
-  0::/
-
-When some thread from a multi-threaded process unshares its cgroup
-namespace, the new cgroupns gets applied to the entire process (all
-the threads).  This is natural for the v2 hierarchy; however, for the
-legacy hierarchies, this may be unexpected.
-
-A cgroup namespace is alive as long as there are processes inside or
-mounts pinning it.  When the last usage goes away, the cgroup
-namespace is destroyed.  The cgroupns root and the actual cgroups
-remain.
-
-
-The Root and Views
-------------------
-
-The 'cgroupns root' for a cgroup namespace is the cgroup in which the
-process calling unshare(2) is running.  For example, if a process in
-/batchjobs/container_id1 cgroup calls unshare, cgroup
-/batchjobs/container_id1 becomes the cgroupns root.  For the
-init_cgroup_ns, this is the real root ('/') cgroup.
-
-The cgroupns root cgroup does not change even if the namespace creator
-process later moves to a different cgroup::
-
-  # ~/unshare -c # unshare cgroupns in some cgroup
-  # cat /proc/self/cgroup
-  0::/
-  # mkdir sub_cgrp_1
-  # echo 0 > sub_cgrp_1/cgroup.procs
-  # cat /proc/self/cgroup
-  0::/sub_cgrp_1
-
-Each process gets its namespace-specific view of "/proc/$PID/cgroup"
-
-Processes running inside the cgroup namespace will be able to see
-cgroup paths (in /proc/self/cgroup) only inside their root cgroup.
-From within an unshared cgroupns::
-
-  # sleep 100000 &
-  [1] 7353
-  # echo 7353 > sub_cgrp_1/cgroup.procs
-  # cat /proc/7353/cgroup
-  0::/sub_cgrp_1
-
-From the initial cgroup namespace, the real cgroup path will be
-visible::
-
-  $ cat /proc/7353/cgroup
-  0::/batchjobs/container_id1/sub_cgrp_1
-
-From a sibling cgroup namespace (that is, a namespace rooted at a
-different cgroup), the cgroup path relative to its own cgroup
-namespace root will be shown.  For instance, if PID 7353's cgroup
-namespace root is at '/batchjobs/container_id2', then it will see::
-
-  # cat /proc/7353/cgroup
-  0::/../container_id2/sub_cgrp_1
-
-Note that the relative path always starts with '/' to indicate that
-its relative to the cgroup namespace root of the caller.
-
-
-Migration and setns(2)
-----------------------
-
-Processes inside a cgroup namespace can move into and out of the
-namespace root if they have proper access to external cgroups.  For
-example, from inside a namespace with cgroupns root at
-/batchjobs/container_id1, and assuming that the global hierarchy is
-still accessible inside cgroupns::
-
-  # cat /proc/7353/cgroup
-  0::/sub_cgrp_1
-  # echo 7353 > batchjobs/container_id2/cgroup.procs
-  # cat /proc/7353/cgroup
-  0::/../container_id2
-
-Note that this kind of setup is not encouraged.  A task inside cgroup
-namespace should only be exposed to its own cgroupns hierarchy.
-
-setns(2) to another cgroup namespace is allowed when:
-
-(a) the process has CAP_SYS_ADMIN against its current user namespace
-(b) the process has CAP_SYS_ADMIN against the target cgroup
-    namespace's userns
-
-No implicit cgroup changes happen with attaching to another cgroup
-namespace.  It is expected that the someone moves the attaching
-process under the target cgroup namespace root.
-
-
-Interaction with Other Namespaces
----------------------------------
-
-Namespace specific cgroup hierarchy can be mounted by a process
-running inside a non-init cgroup namespace::
-
-  # mount -t cgroup2 none $MOUNT_POINT
-
-This will mount the unified cgroup hierarchy with cgroupns root as the
-filesystem root.  The process needs CAP_SYS_ADMIN against its user and
-mount namespaces.
-
-The virtualization of /proc/self/cgroup file combined with restricting
-the view of cgroup hierarchy by namespace-private cgroupfs mount
-provides a properly isolated cgroup view inside the container.
-
-
-Information on Kernel Programming
-=================================
-
-This section contains kernel programming information in the areas
-where interacting with cgroup is necessary.  cgroup core and
-controllers are not covered.
-
-
-Filesystem Support for Writeback
---------------------------------
-
-A filesystem can support cgroup writeback by updating
-address_space_operations->writepage[s]() to annotate bio's using the
-following two functions.
-
-  wbc_init_bio(@wbc, @bio)
-	Should be called for each bio carrying writeback data and
-	associates the bio with the inode's owner cgroup.  Can be
-	called anytime between bio allocation and submission.
-
-  wbc_account_io(@wbc, @page, @bytes)
-	Should be called for each data segment being written out.
-	While this function doesn't care exactly when it's called
-	during the writeback session, it's the easiest and most
-	natural to call it as data segments are added to a bio.
-
-With writeback bio's annotated, cgroup support can be enabled per
-super_block by setting SB_I_CGROUPWB in ->s_iflags.  This allows for
-selective disabling of cgroup writeback support which is helpful when
-certain filesystem features, e.g. journaled data mode, are
-incompatible.
-
-wbc_init_bio() binds the specified bio to its cgroup.  Depending on
-the configuration, the bio may be executed at a lower priority and if
-the writeback session is holding shared resources, e.g. a journal
-entry, may lead to priority inversion.  There is no one easy solution
-for the problem.  Filesystems can try to work around specific problem
-cases by skipping wbc_init_bio() or using bio_associate_blkcg()
-directly.
-
-
-Deprecated v1 Core Features
-===========================
-
-- Multiple hierarchies including named ones are not supported.
-
-- All v1 mount options are not supported.
-
-- The "tasks" file is removed and "cgroup.procs" is not sorted.
-
-- "cgroup.clone_children" is removed.
-
-- /proc/cgroups is meaningless for v2.  Use "cgroup.controllers" file
-  at the root instead.
-
-
-Issues with v1 and Rationales for v2
-====================================
-
-Multiple Hierarchies
---------------------
-
-cgroup v1 allowed an arbitrary number of hierarchies and each
-hierarchy could host any number of controllers.  While this seemed to
-provide a high level of flexibility, it wasn't useful in practice.
-
-For example, as there is only one instance of each controller, utility
-type controllers such as freezer which can be useful in all
-hierarchies could only be used in one.  The issue is exacerbated by
-the fact that controllers couldn't be moved to another hierarchy once
-hierarchies were populated.  Another issue was that all controllers
-bound to a hierarchy were forced to have exactly the same view of the
-hierarchy.  It wasn't possible to vary the granularity depending on
-the specific controller.
-
-In practice, these issues heavily limited which controllers could be
-put on the same hierarchy and most configurations resorted to putting
-each controller on its own hierarchy.  Only closely related ones, such
-as the cpu and cpuacct controllers, made sense to be put on the same
-hierarchy.  This often meant that userland ended up managing multiple
-similar hierarchies repeating the same steps on each hierarchy
-whenever a hierarchy management operation was necessary.
-
-Furthermore, support for multiple hierarchies came at a steep cost.
-It greatly complicated cgroup core implementation but more importantly
-the support for multiple hierarchies restricted how cgroup could be
-used in general and what controllers was able to do.
-
-There was no limit on how many hierarchies there might be, which meant
-that a thread's cgroup membership couldn't be described in finite
-length.  The key might contain any number of entries and was unlimited
-in length, which made it highly awkward to manipulate and led to
-addition of controllers which existed only to identify membership,
-which in turn exacerbated the original problem of proliferating number
-of hierarchies.
-
-Also, as a controller couldn't have any expectation regarding the
-topologies of hierarchies other controllers might be on, each
-controller had to assume that all other controllers were attached to
-completely orthogonal hierarchies.  This made it impossible, or at
-least very cumbersome, for controllers to cooperate with each other.
-
-In most use cases, putting controllers on hierarchies which are
-completely orthogonal to each other isn't necessary.  What usually is
-called for is the ability to have differing levels of granularity
-depending on the specific controller.  In other words, hierarchy may
-be collapsed from leaf towards root when viewed from specific
-controllers.  For example, a given configuration might not care about
-how memory is distributed beyond a certain level while still wanting
-to control how CPU cycles are distributed.
-
-
-Thread Granularity
-------------------
-
-cgroup v1 allowed threads of a process to belong to different cgroups.
-This didn't make sense for some controllers and those controllers
-ended up implementing different ways to ignore such situations but
-much more importantly it blurred the line between API exposed to
-individual applications and system management interface.
-
-Generally, in-process knowledge is available only to the process
-itself; thus, unlike service-level organization of processes,
-categorizing threads of a process requires active participation from
-the application which owns the target process.
-
-cgroup v1 had an ambiguously defined delegation model which got abused
-in combination with thread granularity.  cgroups were delegated to
-individual applications so that they can create and manage their own
-sub-hierarchies and control resource distributions along them.  This
-effectively raised cgroup to the status of a syscall-like API exposed
-to lay programs.
-
-First of all, cgroup has a fundamentally inadequate interface to be
-exposed this way.  For a process to access its own knobs, it has to
-extract the path on the target hierarchy from /proc/self/cgroup,
-construct the path by appending the name of the knob to the path, open
-and then read and/or write to it.  This is not only extremely clunky
-and unusual but also inherently racy.  There is no conventional way to
-define transaction across the required steps and nothing can guarantee
-that the process would actually be operating on its own sub-hierarchy.
-
-cgroup controllers implemented a number of knobs which would never be
-accepted as public APIs because they were just adding control knobs to
-system-management pseudo filesystem.  cgroup ended up with interface
-knobs which were not properly abstracted or refined and directly
-revealed kernel internal details.  These knobs got exposed to
-individual applications through the ill-defined delegation mechanism
-effectively abusing cgroup as a shortcut to implementing public APIs
-without going through the required scrutiny.
-
-This was painful for both userland and kernel.  Userland ended up with
-misbehaving and poorly abstracted interfaces and kernel exposing and
-locked into constructs inadvertently.
-
-
-Competition Between Inner Nodes and Threads
--------------------------------------------
-
-cgroup v1 allowed threads to be in any cgroups which created an
-interesting problem where threads belonging to a parent cgroup and its
-children cgroups competed for resources.  This was nasty as two
-different types of entities competed and there was no obvious way to
-settle it.  Different controllers did different things.
-
-The cpu controller considered threads and cgroups as equivalents and
-mapped nice levels to cgroup weights.  This worked for some cases but
-fell flat when children wanted to be allocated specific ratios of CPU
-cycles and the number of internal threads fluctuated - the ratios
-constantly changed as the number of competing entities fluctuated.
-There also were other issues.  The mapping from nice level to weight
-wasn't obvious or universal, and there were various other knobs which
-simply weren't available for threads.
-
-The io controller implicitly created a hidden leaf node for each
-cgroup to host the threads.  The hidden leaf had its own copies of all
-the knobs with ``leaf_`` prefixed.  While this allowed equivalent
-control over internal threads, it was with serious drawbacks.  It
-always added an extra layer of nesting which wouldn't be necessary
-otherwise, made the interface messy and significantly complicated the
-implementation.
-
-The memory controller didn't have a way to control what happened
-between internal tasks and child cgroups and the behavior was not
-clearly defined.  There were attempts to add ad-hoc behaviors and
-knobs to tailor the behavior to specific workloads which would have
-led to problems extremely difficult to resolve in the long term.
-
-Multiple controllers struggled with internal tasks and came up with
-different ways to deal with it; unfortunately, all the approaches were
-severely flawed and, furthermore, the widely different behaviors
-made cgroup as a whole highly inconsistent.
-
-This clearly is a problem which needs to be addressed from cgroup core
-in a uniform way.
-
-
-Other Interface Issues
-----------------------
-
-cgroup v1 grew without oversight and developed a large number of
-idiosyncrasies and inconsistencies.  One issue on the cgroup core side
-was how an empty cgroup was notified - a userland helper binary was
-forked and executed for each event.  The event delivery wasn't
-recursive or delegatable.  The limitations of the mechanism also led
-to in-kernel event delivery filtering mechanism further complicating
-the interface.
-
-Controller interfaces were problematic too.  An extreme example is
-controllers completely ignoring hierarchical organization and treating
-all cgroups as if they were all located directly under the root
-cgroup.  Some controllers exposed a large amount of inconsistent
-implementation details to userland.
-
-There also was no consistency across controllers.  When a new cgroup
-was created, some controllers defaulted to not imposing extra
-restrictions while others disallowed any resource usage until
-explicitly configured.  Configuration knobs for the same type of
-control used widely differing naming schemes and formats.  Statistics
-and information knobs were named arbitrarily and used different
-formats and units even in the same controller.
-
-cgroup v2 establishes common conventions where appropriate and updates
-controllers so that they expose minimal and consistent interfaces.
-
-
-Controller Issues and Remedies
-------------------------------
-
-Memory
-~~~~~~
-
-The original lower boundary, the soft limit, is defined as a limit
-that is per default unset.  As a result, the set of cgroups that
-global reclaim prefers is opt-in, rather than opt-out.  The costs for
-optimizing these mostly negative lookups are so high that the
-implementation, despite its enormous size, does not even provide the
-basic desirable behavior.  First off, the soft limit has no
-hierarchical meaning.  All configured groups are organized in a global
-rbtree and treated like equal peers, regardless where they are located
-in the hierarchy.  This makes subtree delegation impossible.  Second,
-the soft limit reclaim pass is so aggressive that it not just
-introduces high allocation latencies into the system, but also impacts
-system performance due to overreclaim, to the point where the feature
-becomes self-defeating.
-
-The memory.low boundary on the other hand is a top-down allocated
-reserve.  A cgroup enjoys reclaim protection when it and all its
-ancestors are below their low boundaries, which makes delegation of
-subtrees possible.  Secondly, new cgroups have no reserve per default
-and in the common case most cgroups are eligible for the preferred
-reclaim pass.  This allows the new low boundary to be efficiently
-implemented with just a minor addition to the generic reclaim code,
-without the need for out-of-band data structures and reclaim passes.
-Because the generic reclaim code considers all cgroups except for the
-ones running low in the preferred first reclaim pass, overreclaim of
-individual groups is eliminated as well, resulting in much better
-overall workload performance.
-
-The original high boundary, the hard limit, is defined as a strict
-limit that can not budge, even if the OOM killer has to be called.
-But this generally goes against the goal of making the most out of the
-available memory.  The memory consumption of workloads varies during
-runtime, and that requires users to overcommit.  But doing that with a
-strict upper limit requires either a fairly accurate prediction of the
-working set size or adding slack to the limit.  Since working set size
-estimation is hard and error prone, and getting it wrong results in
-OOM kills, most users tend to err on the side of a looser limit and
-end up wasting precious resources.
-
-The memory.high boundary on the other hand can be set much more
-conservatively.  When hit, it throttles allocations by forcing them
-into direct reclaim to work off the excess, but it never invokes the
-OOM killer.  As a result, a high boundary that is chosen too
-aggressively will not terminate the processes, but instead it will
-lead to gradual performance degradation.  The user can monitor this
-and make corrections until the minimal memory footprint that still
-gives acceptable performance is found.
-
-In extreme cases, with many concurrent allocations and a complete
-breakdown of reclaim progress within the group, the high boundary can
-be exceeded.  But even then it's mostly better to satisfy the
-allocation from the slack available in other groups or the rest of the
-system than killing the group.  Otherwise, memory.max is there to
-limit this type of spillover and ultimately contain buggy or even
-malicious applications.
-
-Setting the original memory.limit_in_bytes below the current usage was
-subject to a race condition, where concurrent charges could cause the
-limit setting to fail. memory.max on the other hand will first set the
-limit to prevent new charges, and then reclaim and OOM kill until the
-new limit is met - or the task writing to memory.max is killed.
-
-The combined memory+swap accounting and limiting is replaced by real
-control over swap space.
-
-The main argument for a combined memory+swap facility in the original
-cgroup design was that global or parental pressure would always be
-able to swap all anonymous memory of a child group, regardless of the
-child's own (possibly untrusted) configuration.  However, untrusted
-groups can sabotage swapping by other means - such as referencing its
-anonymous memory in a tight loop - and an admin can not assume full
-swappability when overcommitting untrusted jobs.
-
-For trusted jobs, on the other hand, a combined counter is not an
-intuitive userspace interface, and it flies in the face of the idea
-that cgroup controllers should account and limit specific physical
-resources.  Swap space is a resource like all others in the system,
-and that's why unified hierarchy allows distributing it separately.
diff --git a/Documentation/circular-buffers.txt b/Documentation/circular-buffers.txt
deleted file mode 100644
index 53e51caa3347..000000000000
--- a/Documentation/circular-buffers.txt
+++ /dev/null
@@ -1,237 +0,0 @@
-================
-Circular Buffers
-================
-
-:Author: David Howells <dhowells@redhat.com>
-:Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
-
-
-Linux provides a number of features that can be used to implement circular
-buffering.  There are two sets of such features:
-
- (1) Convenience functions for determining information about power-of-2 sized
-     buffers.
-
- (2) Memory barriers for when the producer and the consumer of objects in the
-     buffer don't want to share a lock.
-
-To use these facilities, as discussed below, there needs to be just one
-producer and just one consumer.  It is possible to handle multiple producers by
-serialising them, and to handle multiple consumers by serialising them.
-
-
-.. Contents:
-
- (*) What is a circular buffer?
-
- (*) Measuring power-of-2 buffers.
-
- (*) Using memory barriers with circular buffers.
-     - The producer.
-     - The consumer.
-
-
-
-What is a circular buffer?
-==========================
-
-First of all, what is a circular buffer?  A circular buffer is a buffer of
-fixed, finite size into which there are two indices:
-
- (1) A 'head' index - the point at which the producer inserts items into the
-     buffer.
-
- (2) A 'tail' index - the point at which the consumer finds the next item in
-     the buffer.
-
-Typically when the tail pointer is equal to the head pointer, the buffer is
-empty; and the buffer is full when the head pointer is one less than the tail
-pointer.
-
-The head index is incremented when items are added, and the tail index when
-items are removed.  The tail index should never jump the head index, and both
-indices should be wrapped to 0 when they reach the end of the buffer, thus
-allowing an infinite amount of data to flow through the buffer.
-
-Typically, items will all be of the same unit size, but this isn't strictly
-required to use the techniques below.  The indices can be increased by more
-than 1 if multiple items or variable-sized items are to be included in the
-buffer, provided that neither index overtakes the other.  The implementer must
-be careful, however, as a region more than one unit in size may wrap the end of
-the buffer and be broken into two segments.
-
-Measuring power-of-2 buffers
-============================
-
-Calculation of the occupancy or the remaining capacity of an arbitrarily sized
-circular buffer would normally be a slow operation, requiring the use of a
-modulus (divide) instruction.  However, if the buffer is of a power-of-2 size,
-then a much quicker bitwise-AND instruction can be used instead.
-
-Linux provides a set of macros for handling power-of-2 circular buffers.  These
-can be made use of by::
-
-	#include <linux/circ_buf.h>
-
-The macros are:
-
- (#) Measure the remaining capacity of a buffer::
-
-	CIRC_SPACE(head_index, tail_index, buffer_size);
-
-     This returns the amount of space left in the buffer[1] into which items
-     can be inserted.
-
-
- (#) Measure the maximum consecutive immediate space in a buffer::
-
-	CIRC_SPACE_TO_END(head_index, tail_index, buffer_size);
-
-     This returns the amount of consecutive space left in the buffer[1] into
-     which items can be immediately inserted without having to wrap back to the
-     beginning of the buffer.
-
-
- (#) Measure the occupancy of a buffer::
-
-	CIRC_CNT(head_index, tail_index, buffer_size);
-
-     This returns the number of items currently occupying a buffer[2].
-
-
- (#) Measure the non-wrapping occupancy of a buffer::
-
-	CIRC_CNT_TO_END(head_index, tail_index, buffer_size);
-
-     This returns the number of consecutive items[2] that can be extracted from
-     the buffer without having to wrap back to the beginning of the buffer.
-
-
-Each of these macros will nominally return a value between 0 and buffer_size-1,
-however:
-
- (1) CIRC_SPACE*() are intended to be used in the producer.  To the producer
-     they will return a lower bound as the producer controls the head index,
-     but the consumer may still be depleting the buffer on another CPU and
-     moving the tail index.
-
-     To the consumer it will show an upper bound as the producer may be busy
-     depleting the space.
-
- (2) CIRC_CNT*() are intended to be used in the consumer.  To the consumer they
-     will return a lower bound as the consumer controls the tail index, but the
-     producer may still be filling the buffer on another CPU and moving the
-     head index.
-
-     To the producer it will show an upper bound as the consumer may be busy
-     emptying the buffer.
-
- (3) To a third party, the order in which the writes to the indices by the
-     producer and consumer become visible cannot be guaranteed as they are
-     independent and may be made on different CPUs - so the result in such a
-     situation will merely be a guess, and may even be negative.
-
-Using memory barriers with circular buffers
-===========================================
-
-By using memory barriers in conjunction with circular buffers, you can avoid
-the need to:
-
- (1) use a single lock to govern access to both ends of the buffer, thus
-     allowing the buffer to be filled and emptied at the same time; and
-
- (2) use atomic counter operations.
-
-There are two sides to this: the producer that fills the buffer, and the
-consumer that empties it.  Only one thing should be filling a buffer at any one
-time, and only one thing should be emptying a buffer at any one time, but the
-two sides can operate simultaneously.
-
-
-The producer
-------------
-
-The producer will look something like this::
-
-	spin_lock(&producer_lock);
-
-	unsigned long head = buffer->head;
-	/* The spin_unlock() and next spin_lock() provide needed ordering. */
-	unsigned long tail = READ_ONCE(buffer->tail);
-
-	if (CIRC_SPACE(head, tail, buffer->size) >= 1) {
-		/* insert one item into the buffer */
-		struct item *item = buffer[head];
-
-		produce_item(item);
-
-		smp_store_release(buffer->head,
-				  (head + 1) & (buffer->size - 1));
-
-		/* wake_up() will make sure that the head is committed before
-		 * waking anyone up */
-		wake_up(consumer);
-	}
-
-	spin_unlock(&producer_lock);
-
-This will instruct the CPU that the contents of the new item must be written
-before the head index makes it available to the consumer and then instructs the
-CPU that the revised head index must be written before the consumer is woken.
-
-Note that wake_up() does not guarantee any sort of barrier unless something
-is actually awakened.  We therefore cannot rely on it for ordering.  However,
-there is always one element of the array left empty.  Therefore, the
-producer must produce two elements before it could possibly corrupt the
-element currently being read by the consumer.  Therefore, the unlock-lock
-pair between consecutive invocations of the consumer provides the necessary
-ordering between the read of the index indicating that the consumer has
-vacated a given element and the write by the producer to that same element.
-
-
-The Consumer
-------------
-
-The consumer will look something like this::
-
-	spin_lock(&consumer_lock);
-
-	/* Read index before reading contents at that index. */
-	unsigned long head = smp_load_acquire(buffer->head);
-	unsigned long tail = buffer->tail;
-
-	if (CIRC_CNT(head, tail, buffer->size) >= 1) {
-
-		/* extract one item from the buffer */
-		struct item *item = buffer[tail];
-
-		consume_item(item);
-
-		/* Finish reading descriptor before incrementing tail. */
-		smp_store_release(buffer->tail,
-				  (tail + 1) & (buffer->size - 1));
-	}
-
-	spin_unlock(&consumer_lock);
-
-This will instruct the CPU to make sure the index is up to date before reading
-the new item, and then it shall make sure the CPU has finished reading the item
-before it writes the new tail pointer, which will erase the item.
-
-Note the use of READ_ONCE() and smp_load_acquire() to read the
-opposition index.  This prevents the compiler from discarding and
-reloading its cached value.  This isn't strictly needed if you can
-be sure that the opposition index will _only_ be used the once.
-The smp_load_acquire() additionally forces the CPU to order against
-subsequent memory references.  Similarly, smp_store_release() is used
-in both algorithms to write the thread's index.  This documents the
-fact that we are writing to something that can be read concurrently,
-prevents the compiler from tearing the store, and enforces ordering
-against previous accesses.
-
-
-Further reading
-===============
-
-See also Documentation/memory-barriers.txt for a description of Linux's memory
-barrier facilities.
diff --git a/Documentation/clk.txt b/Documentation/clk.txt
deleted file mode 100644
index 511628bb3d3a..000000000000
--- a/Documentation/clk.txt
+++ /dev/null
@@ -1,307 +0,0 @@
-========================
-The Common Clk Framework
-========================
-
-:Author: Mike Turquette <mturquette@ti.com>
-
-This document endeavours to explain the common clk framework details,
-and how to port a platform over to this framework.  It is not yet a
-detailed explanation of the clock api in include/linux/clk.h, but
-perhaps someday it will include that information.
-
-Introduction and interface split
-================================
-
-The common clk framework is an interface to control the clock nodes
-available on various devices today.  This may come in the form of clock
-gating, rate adjustment, muxing or other operations.  This framework is
-enabled with the CONFIG_COMMON_CLK option.
-
-The interface itself is divided into two halves, each shielded from the
-details of its counterpart.  First is the common definition of struct
-clk which unifies the framework-level accounting and infrastructure that
-has traditionally been duplicated across a variety of platforms.  Second
-is a common implementation of the clk.h api, defined in
-drivers/clk/clk.c.  Finally there is struct clk_ops, whose operations
-are invoked by the clk api implementation.
-
-The second half of the interface is comprised of the hardware-specific
-callbacks registered with struct clk_ops and the corresponding
-hardware-specific structures needed to model a particular clock.  For
-the remainder of this document any reference to a callback in struct
-clk_ops, such as .enable or .set_rate, implies the hardware-specific
-implementation of that code.  Likewise, references to struct clk_foo
-serve as a convenient shorthand for the implementation of the
-hardware-specific bits for the hypothetical "foo" hardware.
-
-Tying the two halves of this interface together is struct clk_hw, which
-is defined in struct clk_foo and pointed to within struct clk_core.  This
-allows for easy navigation between the two discrete halves of the common
-clock interface.
-
-Common data structures and api
-==============================
-
-Below is the common struct clk_core definition from
-drivers/clk/clk.c, modified for brevity::
-
-	struct clk_core {
-		const char		*name;
-		const struct clk_ops	*ops;
-		struct clk_hw		*hw;
-		struct module		*owner;
-		struct clk_core		*parent;
-		const char		**parent_names;
-		struct clk_core		**parents;
-		u8			num_parents;
-		u8			new_parent_index;
-		...
-	};
-
-The members above make up the core of the clk tree topology.  The clk
-api itself defines several driver-facing functions which operate on
-struct clk.  That api is documented in include/linux/clk.h.
-
-Platforms and devices utilizing the common struct clk_core use the struct
-clk_ops pointer in struct clk_core to perform the hardware-specific parts of
-the operations defined in clk-provider.h::
-
-	struct clk_ops {
-		int		(*prepare)(struct clk_hw *hw);
-		void		(*unprepare)(struct clk_hw *hw);
-		int		(*is_prepared)(struct clk_hw *hw);
-		void		(*unprepare_unused)(struct clk_hw *hw);
-		int		(*enable)(struct clk_hw *hw);
-		void		(*disable)(struct clk_hw *hw);
-		int		(*is_enabled)(struct clk_hw *hw);
-		void		(*disable_unused)(struct clk_hw *hw);
-		unsigned long	(*recalc_rate)(struct clk_hw *hw,
-						unsigned long parent_rate);
-		long		(*round_rate)(struct clk_hw *hw,
-						unsigned long rate,
-						unsigned long *parent_rate);
-		int		(*determine_rate)(struct clk_hw *hw,
-						  struct clk_rate_request *req);
-		int		(*set_parent)(struct clk_hw *hw, u8 index);
-		u8		(*get_parent)(struct clk_hw *hw);
-		int		(*set_rate)(struct clk_hw *hw,
-					    unsigned long rate,
-					    unsigned long parent_rate);
-		int		(*set_rate_and_parent)(struct clk_hw *hw,
-					    unsigned long rate,
-					    unsigned long parent_rate,
-					    u8 index);
-		unsigned long	(*recalc_accuracy)(struct clk_hw *hw,
-						unsigned long parent_accuracy);
-		int		(*get_phase)(struct clk_hw *hw);
-		int		(*set_phase)(struct clk_hw *hw, int degrees);
-		void		(*init)(struct clk_hw *hw);
-		int		(*debug_init)(struct clk_hw *hw,
-					      struct dentry *dentry);
-	};
-
-Hardware clk implementations
-============================
-
-The strength of the common struct clk_core comes from its .ops and .hw pointers
-which abstract the details of struct clk from the hardware-specific bits, and
-vice versa.  To illustrate consider the simple gateable clk implementation in
-drivers/clk/clk-gate.c::
-
-	struct clk_gate {
-		struct clk_hw	hw;
-		void __iomem    *reg;
-		u8              bit_idx;
-		...
-	};
-
-struct clk_gate contains struct clk_hw hw as well as hardware-specific
-knowledge about which register and bit controls this clk's gating.
-Nothing about clock topology or accounting, such as enable_count or
-notifier_count, is needed here.  That is all handled by the common
-framework code and struct clk_core.
-
-Let's walk through enabling this clk from driver code::
-
-	struct clk *clk;
-	clk = clk_get(NULL, "my_gateable_clk");
-
-	clk_prepare(clk);
-	clk_enable(clk);
-
-The call graph for clk_enable is very simple::
-
-	clk_enable(clk);
-		clk->ops->enable(clk->hw);
-		[resolves to...]
-			clk_gate_enable(hw);
-			[resolves struct clk gate with to_clk_gate(hw)]
-				clk_gate_set_bit(gate);
-
-And the definition of clk_gate_set_bit::
-
-	static void clk_gate_set_bit(struct clk_gate *gate)
-	{
-		u32 reg;
-
-		reg = __raw_readl(gate->reg);
-		reg |= BIT(gate->bit_idx);
-		writel(reg, gate->reg);
-	}
-
-Note that to_clk_gate is defined as::
-
-	#define to_clk_gate(_hw) container_of(_hw, struct clk_gate, hw)
-
-This pattern of abstraction is used for every clock hardware
-representation.
-
-Supporting your own clk hardware
-================================
-
-When implementing support for a new type of clock it is only necessary to
-include the following header::
-
-	#include <linux/clk-provider.h>
-
-To construct a clk hardware structure for your platform you must define
-the following::
-
-	struct clk_foo {
-		struct clk_hw hw;
-		... hardware specific data goes here ...
-	};
-
-To take advantage of your data you'll need to support valid operations
-for your clk::
-
-	struct clk_ops clk_foo_ops {
-		.enable		= &clk_foo_enable;
-		.disable	= &clk_foo_disable;
-	};
-
-Implement the above functions using container_of::
-
-	#define to_clk_foo(_hw) container_of(_hw, struct clk_foo, hw)
-
-	int clk_foo_enable(struct clk_hw *hw)
-	{
-		struct clk_foo *foo;
-
-		foo = to_clk_foo(hw);
-
-		... perform magic on foo ...
-
-		return 0;
-	};
-
-Below is a matrix detailing which clk_ops are mandatory based upon the
-hardware capabilities of that clock.  A cell marked as "y" means
-mandatory, a cell marked as "n" implies that either including that
-callback is invalid or otherwise unnecessary.  Empty cells are either
-optional or must be evaluated on a case-by-case basis.
-
-.. table:: clock hardware characteristics
-
-   +----------------+------+-------------+---------------+-------------+------+
-   |                | gate | change rate | single parent | multiplexer | root |
-   +================+======+=============+===============+=============+======+
-   |.prepare        |      |             |               |             |      |
-   +----------------+------+-------------+---------------+-------------+------+
-   |.unprepare      |      |             |               |             |      |
-   +----------------+------+-------------+---------------+-------------+------+
-   +----------------+------+-------------+---------------+-------------+------+
-   |.enable         | y    |             |               |             |      |
-   +----------------+------+-------------+---------------+-------------+------+
-   |.disable        | y    |             |               |             |      |
-   +----------------+------+-------------+---------------+-------------+------+
-   |.is_enabled     | y    |             |               |             |      |
-   +----------------+------+-------------+---------------+-------------+------+
-   +----------------+------+-------------+---------------+-------------+------+
-   |.recalc_rate    |      | y           |               |             |      |
-   +----------------+------+-------------+---------------+-------------+------+
-   |.round_rate     |      | y [1]_      |               |             |      |
-   +----------------+------+-------------+---------------+-------------+------+
-   |.determine_rate |      | y [1]_      |               |             |      |
-   +----------------+------+-------------+---------------+-------------+------+
-   |.set_rate       |      | y           |               |             |      |
-   +----------------+------+-------------+---------------+-------------+------+
-   +----------------+------+-------------+---------------+-------------+------+
-   |.set_parent     |      |             | n             | y           | n    |
-   +----------------+------+-------------+---------------+-------------+------+
-   |.get_parent     |      |             | n             | y           | n    |
-   +----------------+------+-------------+---------------+-------------+------+
-   +----------------+------+-------------+---------------+-------------+------+
-   |.recalc_accuracy|      |             |               |             |      |
-   +----------------+------+-------------+---------------+-------------+------+
-   +----------------+------+-------------+---------------+-------------+------+
-   |.init           |      |             |               |             |      |
-   +----------------+------+-------------+---------------+-------------+------+
-
-.. [1] either one of round_rate or determine_rate is required.
-
-Finally, register your clock at run-time with a hardware-specific
-registration function.  This function simply populates struct clk_foo's
-data and then passes the common struct clk parameters to the framework
-with a call to::
-
-	clk_register(...)
-
-See the basic clock types in ``drivers/clk/clk-*.c`` for examples.
-
-Disabling clock gating of unused clocks
-=======================================
-
-Sometimes during development it can be useful to be able to bypass the
-default disabling of unused clocks. For example, if drivers aren't enabling
-clocks properly but rely on them being on from the bootloader, bypassing
-the disabling means that the driver will remain functional while the issues
-are sorted out.
-
-To bypass this disabling, include "clk_ignore_unused" in the bootargs to the
-kernel.
-
-Locking
-=======
-
-The common clock framework uses two global locks, the prepare lock and the
-enable lock.
-
-The enable lock is a spinlock and is held across calls to the .enable,
-.disable operations. Those operations are thus not allowed to sleep,
-and calls to the clk_enable(), clk_disable() API functions are allowed in
-atomic context.
-
-For clk_is_enabled() API, it is also designed to be allowed to be used in
-atomic context. However, it doesn't really make any sense to hold the enable
-lock in core, unless you want to do something else with the information of
-the enable state with that lock held. Otherwise, seeing if a clk is enabled is
-a one-shot read of the enabled state, which could just as easily change after
-the function returns because the lock is released. Thus the user of this API
-needs to handle synchronizing the read of the state with whatever they're
-using it for to make sure that the enable state doesn't change during that
-time.
-
-The prepare lock is a mutex and is held across calls to all other operations.
-All those operations are allowed to sleep, and calls to the corresponding API
-functions are not allowed in atomic context.
-
-This effectively divides operations in two groups from a locking perspective.
-
-Drivers don't need to manually protect resources shared between the operations
-of one group, regardless of whether those resources are shared by multiple
-clocks or not. However, access to resources that are shared between operations
-of the two groups needs to be protected by the drivers. An example of such a
-resource would be a register that controls both the clock rate and the clock
-enable/disable state.
-
-The clock framework is reentrant, in that a driver is allowed to call clock
-framework functions from within its implementation of clock operations. This
-can for instance cause a .set_rate operation of one clock being called from
-within the .set_rate operation of another clock. This case must be considered
-in the driver implementations, but the code flow is usually controlled by the
-driver in that case.
-
-Note that locking must also be considered when code outside of the common
-clock framework needs to access resources used by the clock operations. This
-is considered out of scope of this document.
diff --git a/Documentation/core-api/atomic_ops.rst b/Documentation/core-api/atomic_ops.rst
index fce929144ccd..2e7165f86f55 100644
--- a/Documentation/core-api/atomic_ops.rst
+++ b/Documentation/core-api/atomic_ops.rst
@@ -111,7 +111,6 @@ If the compiler can prove that do_something() does not store to the
 variable a, then the compiler is within its rights transforming this to
 the following::
 
-	tmp = a;
 	if (a > 0)
 		for (;;)
 			do_something();
@@ -119,7 +118,7 @@ the following::
 If you don't want the compiler to do this (and you probably don't), then
 you should use something like the following::
 
-	while (READ_ONCE(a) < 0)
+	while (READ_ONCE(a) > 0)
 		do_something();
 
 Alternatively, you could place a barrier() call in the loop.
@@ -467,10 +466,12 @@ Like the above, except that these routines return a boolean which
 indicates whether the changed bit was set _BEFORE_ the atomic bit
 operation.
 
-WARNING! It is incredibly important that the value be a boolean,
-ie. "0" or "1".  Do not try to be fancy and save a few instructions by
-declaring the above to return "long" and just returning something like
-"old_val & mask" because that will not work.
+
+.. warning::
+        It is incredibly important that the value be a boolean, ie. "0" or "1".
+        Do not try to be fancy and save a few instructions by declaring the
+        above to return "long" and just returning something like "old_val &
+        mask" because that will not work.
 
 For one thing, this return value gets truncated to int in many code
 paths using these interfaces, so on 64-bit if the bit is set in the
diff --git a/Documentation/core-api/cachetlb.rst b/Documentation/core-api/cachetlb.rst
new file mode 100644
index 000000000000..6eb9d3f090cd
--- /dev/null
+++ b/Documentation/core-api/cachetlb.rst
@@ -0,0 +1,415 @@
+==================================
+Cache and TLB Flushing Under Linux
+==================================
+
+:Author: David S. Miller <davem@redhat.com>
+
+This document describes the cache/tlb flushing interfaces called
+by the Linux VM subsystem.  It enumerates over each interface,
+describes its intended purpose, and what side effect is expected
+after the interface is invoked.
+
+The side effects described below are stated for a uniprocessor
+implementation, and what is to happen on that single processor.  The
+SMP cases are a simple extension, in that you just extend the
+definition such that the side effect for a particular interface occurs
+on all processors in the system.  Don't let this scare you into
+thinking SMP cache/tlb flushing must be so inefficient, this is in
+fact an area where many optimizations are possible.  For example,
+if it can be proven that a user address space has never executed
+on a cpu (see mm_cpumask()), one need not perform a flush
+for this address space on that cpu.
+
+First, the TLB flushing interfaces, since they are the simplest.  The
+"TLB" is abstracted under Linux as something the cpu uses to cache
+virtual-->physical address translations obtained from the software
+page tables.  Meaning that if the software page tables change, it is
+possible for stale translations to exist in this "TLB" cache.
+Therefore when software page table changes occur, the kernel will
+invoke one of the following flush methods _after_ the page table
+changes occur:
+
+1) ``void flush_tlb_all(void)``
+
+	The most severe flush of all.  After this interface runs,
+	any previous page table modification whatsoever will be
+	visible to the cpu.
+
+	This is usually invoked when the kernel page tables are
+	changed, since such translations are "global" in nature.
+
+2) ``void flush_tlb_mm(struct mm_struct *mm)``
+
+	This interface flushes an entire user address space from
+	the TLB.  After running, this interface must make sure that
+	any previous page table modifications for the address space
+	'mm' will be visible to the cpu.  That is, after running,
+	there will be no entries in the TLB for 'mm'.
+
+	This interface is used to handle whole address space
+	page table operations such as what happens during
+	fork, and exec.
+
+3) ``void flush_tlb_range(struct vm_area_struct *vma,
+   unsigned long start, unsigned long end)``
+
+	Here we are flushing a specific range of (user) virtual
+	address translations from the TLB.  After running, this
+	interface must make sure that any previous page table
+	modifications for the address space 'vma->vm_mm' in the range
+	'start' to 'end-1' will be visible to the cpu.  That is, after
+	running, there will be no entries in the TLB for 'mm' for
+	virtual addresses in the range 'start' to 'end-1'.
+
+	The "vma" is the backing store being used for the region.
+	Primarily, this is used for munmap() type operations.
+
+	The interface is provided in hopes that the port can find
+	a suitably efficient method for removing multiple page
+	sized translations from the TLB, instead of having the kernel
+	call flush_tlb_page (see below) for each entry which may be
+	modified.
+
+4) ``void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)``
+
+	This time we need to remove the PAGE_SIZE sized translation
+	from the TLB.  The 'vma' is the backing structure used by
+	Linux to keep track of mmap'd regions for a process, the
+	address space is available via vma->vm_mm.  Also, one may
+	test (vma->vm_flags & VM_EXEC) to see if this region is
+	executable (and thus could be in the 'instruction TLB' in
+	split-tlb type setups).
+
+	After running, this interface must make sure that any previous
+	page table modification for address space 'vma->vm_mm' for
+	user virtual address 'addr' will be visible to the cpu.  That
+	is, after running, there will be no entries in the TLB for
+	'vma->vm_mm' for virtual address 'addr'.
+
+	This is used primarily during fault processing.
+
+5) ``void update_mmu_cache(struct vm_area_struct *vma,
+   unsigned long address, pte_t *ptep)``
+
+	At the end of every page fault, this routine is invoked to
+	tell the architecture specific code that a translation
+	now exists at virtual address "address" for address space
+	"vma->vm_mm", in the software page tables.
+
+	A port may use this information in any way it so chooses.
+	For example, it could use this event to pre-load TLB
+	translations for software managed TLB configurations.
+	The sparc64 port currently does this.
+
+6) ``void tlb_migrate_finish(struct mm_struct *mm)``
+
+	This interface is called at the end of an explicit
+	process migration. This interface provides a hook
+	to allow a platform to update TLB or context-specific
+	information for the address space.
+
+	The ia64 sn2 platform is one example of a platform
+	that uses this interface.
+
+Next, we have the cache flushing interfaces.  In general, when Linux
+is changing an existing virtual-->physical mapping to a new value,
+the sequence will be in one of the following forms::
+
+	1) flush_cache_mm(mm);
+	   change_all_page_tables_of(mm);
+	   flush_tlb_mm(mm);
+
+	2) flush_cache_range(vma, start, end);
+	   change_range_of_page_tables(mm, start, end);
+	   flush_tlb_range(vma, start, end);
+
+	3) flush_cache_page(vma, addr, pfn);
+	   set_pte(pte_pointer, new_pte_val);
+	   flush_tlb_page(vma, addr);
+
+The cache level flush will always be first, because this allows
+us to properly handle systems whose caches are strict and require
+a virtual-->physical translation to exist for a virtual address
+when that virtual address is flushed from the cache.  The HyperSparc
+cpu is one such cpu with this attribute.
+
+The cache flushing routines below need only deal with cache flushing
+to the extent that it is necessary for a particular cpu.  Mostly,
+these routines must be implemented for cpus which have virtually
+indexed caches which must be flushed when virtual-->physical
+translations are changed or removed.  So, for example, the physically
+indexed physically tagged caches of IA32 processors have no need to
+implement these interfaces since the caches are fully synchronized
+and have no dependency on translation information.
+
+Here are the routines, one by one:
+
+1) ``void flush_cache_mm(struct mm_struct *mm)``
+
+	This interface flushes an entire user address space from
+	the caches.  That is, after running, there will be no cache
+	lines associated with 'mm'.
+
+	This interface is used to handle whole address space
+	page table operations such as what happens during exit and exec.
+
+2) ``void flush_cache_dup_mm(struct mm_struct *mm)``
+
+	This interface flushes an entire user address space from
+	the caches.  That is, after running, there will be no cache
+	lines associated with 'mm'.
+
+	This interface is used to handle whole address space
+	page table operations such as what happens during fork.
+
+	This option is separate from flush_cache_mm to allow some
+	optimizations for VIPT caches.
+
+3) ``void flush_cache_range(struct vm_area_struct *vma,
+   unsigned long start, unsigned long end)``
+
+	Here we are flushing a specific range of (user) virtual
+	addresses from the cache.  After running, there will be no
+	entries in the cache for 'vma->vm_mm' for virtual addresses in
+	the range 'start' to 'end-1'.
+
+	The "vma" is the backing store being used for the region.
+	Primarily, this is used for munmap() type operations.
+
+	The interface is provided in hopes that the port can find
+	a suitably efficient method for removing multiple page
+	sized regions from the cache, instead of having the kernel
+	call flush_cache_page (see below) for each entry which may be
+	modified.
+
+4) ``void flush_cache_page(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn)``
+
+	This time we need to remove a PAGE_SIZE sized range
+	from the cache.  The 'vma' is the backing structure used by
+	Linux to keep track of mmap'd regions for a process, the
+	address space is available via vma->vm_mm.  Also, one may
+	test (vma->vm_flags & VM_EXEC) to see if this region is
+	executable (and thus could be in the 'instruction cache' in
+	"Harvard" type cache layouts).
+
+	The 'pfn' indicates the physical page frame (shift this value
+	left by PAGE_SHIFT to get the physical address) that 'addr'
+	translates to.  It is this mapping which should be removed from
+	the cache.
+
+	After running, there will be no entries in the cache for
+	'vma->vm_mm' for virtual address 'addr' which translates
+	to 'pfn'.
+
+	This is used primarily during fault processing.
+
+5) ``void flush_cache_kmaps(void)``
+
+	This routine need only be implemented if the platform utilizes
+	highmem.  It will be called right before all of the kmaps
+	are invalidated.
+
+	After running, there will be no entries in the cache for
+	the kernel virtual address range PKMAP_ADDR(0) to
+	PKMAP_ADDR(LAST_PKMAP).
+
+	This routing should be implemented in asm/highmem.h
+
+6) ``void flush_cache_vmap(unsigned long start, unsigned long end)``
+   ``void flush_cache_vunmap(unsigned long start, unsigned long end)``
+
+	Here in these two interfaces we are flushing a specific range
+	of (kernel) virtual addresses from the cache.  After running,
+	there will be no entries in the cache for the kernel address
+	space for virtual addresses in the range 'start' to 'end-1'.
+
+	The first of these two routines is invoked after map_vm_area()
+	has installed the page table entries.  The second is invoked
+	before unmap_kernel_range() deletes the page table entries.
+
+There exists another whole class of cpu cache issues which currently
+require a whole different set of interfaces to handle properly.
+The biggest problem is that of virtual aliasing in the data cache
+of a processor.
+
+Is your port susceptible to virtual aliasing in its D-cache?
+Well, if your D-cache is virtually indexed, is larger in size than
+PAGE_SIZE, and does not prevent multiple cache lines for the same
+physical address from existing at once, you have this problem.
+
+If your D-cache has this problem, first define asm/shmparam.h SHMLBA
+properly, it should essentially be the size of your virtually
+addressed D-cache (or if the size is variable, the largest possible
+size).  This setting will force the SYSv IPC layer to only allow user
+processes to mmap shared memory at address which are a multiple of
+this value.
+
+.. note::
+
+  This does not fix shared mmaps, check out the sparc64 port for
+  one way to solve this (in particular SPARC_FLAG_MMAPSHARED).
+
+Next, you have to solve the D-cache aliasing issue for all
+other cases.  Please keep in mind that fact that, for a given page
+mapped into some user address space, there is always at least one more
+mapping, that of the kernel in its linear mapping starting at
+PAGE_OFFSET.  So immediately, once the first user maps a given
+physical page into its address space, by implication the D-cache
+aliasing problem has the potential to exist since the kernel already
+maps this page at its virtual address.
+
+  ``void copy_user_page(void *to, void *from, unsigned long addr, struct page *page)``
+  ``void clear_user_page(void *to, unsigned long addr, struct page *page)``
+
+	These two routines store data in user anonymous or COW
+	pages.  It allows a port to efficiently avoid D-cache alias
+	issues between userspace and the kernel.
+
+	For example, a port may temporarily map 'from' and 'to' to
+	kernel virtual addresses during the copy.  The virtual address
+	for these two pages is chosen in such a way that the kernel
+	load/store instructions happen to virtual addresses which are
+	of the same "color" as the user mapping of the page.  Sparc64
+	for example, uses this technique.
+
+	The 'addr' parameter tells the virtual address where the
+	user will ultimately have this page mapped, and the 'page'
+	parameter gives a pointer to the struct page of the target.
+
+	If D-cache aliasing is not an issue, these two routines may
+	simply call memcpy/memset directly and do nothing more.
+
+  ``void flush_dcache_page(struct page *page)``
+
+	Any time the kernel writes to a page cache page, _OR_
+	the kernel is about to read from a page cache page and
+	user space shared/writable mappings of this page potentially
+	exist, this routine is called.
+
+	.. note::
+
+	      This routine need only be called for page cache pages
+	      which can potentially ever be mapped into the address
+	      space of a user process.  So for example, VFS layer code
+	      handling vfs symlinks in the page cache need not call
+	      this interface at all.
+
+	The phrase "kernel writes to a page cache page" means,
+	specifically, that the kernel executes store instructions
+	that dirty data in that page at the page->virtual mapping
+	of that page.  It is important to flush here to handle
+	D-cache aliasing, to make sure these kernel stores are
+	visible to user space mappings of that page.
+
+	The corollary case is just as important, if there are users
+	which have shared+writable mappings of this file, we must make
+	sure that kernel reads of these pages will see the most recent
+	stores done by the user.
+
+	If D-cache aliasing is not an issue, this routine may
+	simply be defined as a nop on that architecture.
+
+        There is a bit set aside in page->flags (PG_arch_1) as
+	"architecture private".  The kernel guarantees that,
+	for pagecache pages, it will clear this bit when such
+	a page first enters the pagecache.
+
+	This allows these interfaces to be implemented much more
+	efficiently.  It allows one to "defer" (perhaps indefinitely)
+	the actual flush if there are currently no user processes
+	mapping this page.  See sparc64's flush_dcache_page and
+	update_mmu_cache implementations for an example of how to go
+	about doing this.
+
+	The idea is, first at flush_dcache_page() time, if
+	page->mapping->i_mmap is an empty tree, just mark the architecture
+	private page flag bit.  Later, in update_mmu_cache(), a check is
+	made of this flag bit, and if set the flush is done and the flag
+	bit is cleared.
+
+	.. important::
+
+			It is often important, if you defer the flush,
+			that the actual flush occurs on the same CPU
+			as did the cpu stores into the page to make it
+			dirty.  Again, see sparc64 for examples of how
+			to deal with this.
+
+  ``void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
+  unsigned long user_vaddr, void *dst, void *src, int len)``
+  ``void copy_from_user_page(struct vm_area_struct *vma, struct page *page,
+  unsigned long user_vaddr, void *dst, void *src, int len)``
+
+	When the kernel needs to copy arbitrary data in and out
+	of arbitrary user pages (f.e. for ptrace()) it will use
+	these two routines.
+
+	Any necessary cache flushing or other coherency operations
+	that need to occur should happen here.  If the processor's
+	instruction cache does not snoop cpu stores, it is very
+	likely that you will need to flush the instruction cache
+	for copy_to_user_page().
+
+  ``void flush_anon_page(struct vm_area_struct *vma, struct page *page,
+  unsigned long vmaddr)``
+
+  	When the kernel needs to access the contents of an anonymous
+	page, it calls this function (currently only
+	get_user_pages()).  Note: flush_dcache_page() deliberately
+	doesn't work for an anonymous page.  The default
+	implementation is a nop (and should remain so for all coherent
+	architectures).  For incoherent architectures, it should flush
+	the cache of the page at vmaddr.
+
+  ``void flush_kernel_dcache_page(struct page *page)``
+
+	When the kernel needs to modify a user page is has obtained
+	with kmap, it calls this function after all modifications are
+	complete (but before kunmapping it) to bring the underlying
+	page up to date.  It is assumed here that the user has no
+	incoherent cached copies (i.e. the original page was obtained
+	from a mechanism like get_user_pages()).  The default
+	implementation is a nop and should remain so on all coherent
+	architectures.  On incoherent architectures, this should flush
+	the kernel cache for page (using page_address(page)).
+
+
+  ``void flush_icache_range(unsigned long start, unsigned long end)``
+
+  	When the kernel stores into addresses that it will execute
+	out of (eg when loading modules), this function is called.
+
+	If the icache does not snoop stores then this routine will need
+	to flush it.
+
+  ``void flush_icache_page(struct vm_area_struct *vma, struct page *page)``
+
+	All the functionality of flush_icache_page can be implemented in
+	flush_dcache_page and update_mmu_cache. In the future, the hope
+	is to remove this interface completely.
+
+The final category of APIs is for I/O to deliberately aliased address
+ranges inside the kernel.  Such aliases are set up by use of the
+vmap/vmalloc API.  Since kernel I/O goes via physical pages, the I/O
+subsystem assumes that the user mapping and kernel offset mapping are
+the only aliases.  This isn't true for vmap aliases, so anything in
+the kernel trying to do I/O to vmap areas must manually manage
+coherency.  It must do this by flushing the vmap range before doing
+I/O and invalidating it after the I/O returns.
+
+  ``void flush_kernel_vmap_range(void *vaddr, int size)``
+
+       flushes the kernel cache for a given virtual address range in
+       the vmap area.  This is to make sure that any data the kernel
+       modified in the vmap range is made visible to the physical
+       page.  The design is to make this area safe to perform I/O on.
+       Note that this API does *not* also flush the offset map alias
+       of the area.
+
+  ``void invalidate_kernel_vmap_range(void *vaddr, int size) invalidates``
+
+       the cache for a given virtual address range in the vmap area
+       which prevents the processor from making the cache stale by
+       speculatively reading data while the I/O was occurring to the
+       physical pages.  This is only necessary for data reads into the
+       vmap area.
diff --git a/Documentation/core-api/circular-buffers.rst b/Documentation/core-api/circular-buffers.rst
new file mode 100644
index 000000000000..53e51caa3347
--- /dev/null
+++ b/Documentation/core-api/circular-buffers.rst
@@ -0,0 +1,237 @@
+================
+Circular Buffers
+================
+
+:Author: David Howells <dhowells@redhat.com>
+:Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
+
+
+Linux provides a number of features that can be used to implement circular
+buffering.  There are two sets of such features:
+
+ (1) Convenience functions for determining information about power-of-2 sized
+     buffers.
+
+ (2) Memory barriers for when the producer and the consumer of objects in the
+     buffer don't want to share a lock.
+
+To use these facilities, as discussed below, there needs to be just one
+producer and just one consumer.  It is possible to handle multiple producers by
+serialising them, and to handle multiple consumers by serialising them.
+
+
+.. Contents:
+
+ (*) What is a circular buffer?
+
+ (*) Measuring power-of-2 buffers.
+
+ (*) Using memory barriers with circular buffers.
+     - The producer.
+     - The consumer.
+
+
+
+What is a circular buffer?
+==========================
+
+First of all, what is a circular buffer?  A circular buffer is a buffer of
+fixed, finite size into which there are two indices:
+
+ (1) A 'head' index - the point at which the producer inserts items into the
+     buffer.
+
+ (2) A 'tail' index - the point at which the consumer finds the next item in
+     the buffer.
+
+Typically when the tail pointer is equal to the head pointer, the buffer is
+empty; and the buffer is full when the head pointer is one less than the tail
+pointer.
+
+The head index is incremented when items are added, and the tail index when
+items are removed.  The tail index should never jump the head index, and both
+indices should be wrapped to 0 when they reach the end of the buffer, thus
+allowing an infinite amount of data to flow through the buffer.
+
+Typically, items will all be of the same unit size, but this isn't strictly
+required to use the techniques below.  The indices can be increased by more
+than 1 if multiple items or variable-sized items are to be included in the
+buffer, provided that neither index overtakes the other.  The implementer must
+be careful, however, as a region more than one unit in size may wrap the end of
+the buffer and be broken into two segments.
+
+Measuring power-of-2 buffers
+============================
+
+Calculation of the occupancy or the remaining capacity of an arbitrarily sized
+circular buffer would normally be a slow operation, requiring the use of a
+modulus (divide) instruction.  However, if the buffer is of a power-of-2 size,
+then a much quicker bitwise-AND instruction can be used instead.
+
+Linux provides a set of macros for handling power-of-2 circular buffers.  These
+can be made use of by::
+
+	#include <linux/circ_buf.h>
+
+The macros are:
+
+ (#) Measure the remaining capacity of a buffer::
+
+	CIRC_SPACE(head_index, tail_index, buffer_size);
+
+     This returns the amount of space left in the buffer[1] into which items
+     can be inserted.
+
+
+ (#) Measure the maximum consecutive immediate space in a buffer::
+
+	CIRC_SPACE_TO_END(head_index, tail_index, buffer_size);
+
+     This returns the amount of consecutive space left in the buffer[1] into
+     which items can be immediately inserted without having to wrap back to the
+     beginning of the buffer.
+
+
+ (#) Measure the occupancy of a buffer::
+
+	CIRC_CNT(head_index, tail_index, buffer_size);
+
+     This returns the number of items currently occupying a buffer[2].
+
+
+ (#) Measure the non-wrapping occupancy of a buffer::
+
+	CIRC_CNT_TO_END(head_index, tail_index, buffer_size);
+
+     This returns the number of consecutive items[2] that can be extracted from
+     the buffer without having to wrap back to the beginning of the buffer.
+
+
+Each of these macros will nominally return a value between 0 and buffer_size-1,
+however:
+
+ (1) CIRC_SPACE*() are intended to be used in the producer.  To the producer
+     they will return a lower bound as the producer controls the head index,
+     but the consumer may still be depleting the buffer on another CPU and
+     moving the tail index.
+
+     To the consumer it will show an upper bound as the producer may be busy
+     depleting the space.
+
+ (2) CIRC_CNT*() are intended to be used in the consumer.  To the consumer they
+     will return a lower bound as the consumer controls the tail index, but the
+     producer may still be filling the buffer on another CPU and moving the
+     head index.
+
+     To the producer it will show an upper bound as the consumer may be busy
+     emptying the buffer.
+
+ (3) To a third party, the order in which the writes to the indices by the
+     producer and consumer become visible cannot be guaranteed as they are
+     independent and may be made on different CPUs - so the result in such a
+     situation will merely be a guess, and may even be negative.
+
+Using memory barriers with circular buffers
+===========================================
+
+By using memory barriers in conjunction with circular buffers, you can avoid
+the need to:
+
+ (1) use a single lock to govern access to both ends of the buffer, thus
+     allowing the buffer to be filled and emptied at the same time; and
+
+ (2) use atomic counter operations.
+
+There are two sides to this: the producer that fills the buffer, and the
+consumer that empties it.  Only one thing should be filling a buffer at any one
+time, and only one thing should be emptying a buffer at any one time, but the
+two sides can operate simultaneously.
+
+
+The producer
+------------
+
+The producer will look something like this::
+
+	spin_lock(&producer_lock);
+
+	unsigned long head = buffer->head;
+	/* The spin_unlock() and next spin_lock() provide needed ordering. */
+	unsigned long tail = READ_ONCE(buffer->tail);
+
+	if (CIRC_SPACE(head, tail, buffer->size) >= 1) {
+		/* insert one item into the buffer */
+		struct item *item = buffer[head];
+
+		produce_item(item);
+
+		smp_store_release(buffer->head,
+				  (head + 1) & (buffer->size - 1));
+
+		/* wake_up() will make sure that the head is committed before
+		 * waking anyone up */
+		wake_up(consumer);
+	}
+
+	spin_unlock(&producer_lock);
+
+This will instruct the CPU that the contents of the new item must be written
+before the head index makes it available to the consumer and then instructs the
+CPU that the revised head index must be written before the consumer is woken.
+
+Note that wake_up() does not guarantee any sort of barrier unless something
+is actually awakened.  We therefore cannot rely on it for ordering.  However,
+there is always one element of the array left empty.  Therefore, the
+producer must produce two elements before it could possibly corrupt the
+element currently being read by the consumer.  Therefore, the unlock-lock
+pair between consecutive invocations of the consumer provides the necessary
+ordering between the read of the index indicating that the consumer has
+vacated a given element and the write by the producer to that same element.
+
+
+The Consumer
+------------
+
+The consumer will look something like this::
+
+	spin_lock(&consumer_lock);
+
+	/* Read index before reading contents at that index. */
+	unsigned long head = smp_load_acquire(buffer->head);
+	unsigned long tail = buffer->tail;
+
+	if (CIRC_CNT(head, tail, buffer->size) >= 1) {
+
+		/* extract one item from the buffer */
+		struct item *item = buffer[tail];
+
+		consume_item(item);
+
+		/* Finish reading descriptor before incrementing tail. */
+		smp_store_release(buffer->tail,
+				  (tail + 1) & (buffer->size - 1));
+	}
+
+	spin_unlock(&consumer_lock);
+
+This will instruct the CPU to make sure the index is up to date before reading
+the new item, and then it shall make sure the CPU has finished reading the item
+before it writes the new tail pointer, which will erase the item.
+
+Note the use of READ_ONCE() and smp_load_acquire() to read the
+opposition index.  This prevents the compiler from discarding and
+reloading its cached value.  This isn't strictly needed if you can
+be sure that the opposition index will _only_ be used the once.
+The smp_load_acquire() additionally forces the CPU to order against
+subsequent memory references.  Similarly, smp_store_release() is used
+in both algorithms to write the thread's index.  This documents the
+fact that we are writing to something that can be read concurrently,
+prevents the compiler from tearing the store, and enforces ordering
+against previous accesses.
+
+
+Further reading
+===============
+
+See also Documentation/memory-barriers.txt for a description of Linux's memory
+barrier facilities.
diff --git a/Documentation/core-api/gfp_mask-from-fs-io.rst b/Documentation/core-api/gfp_mask-from-fs-io.rst
new file mode 100644
index 000000000000..e0df8f416582
--- /dev/null
+++ b/Documentation/core-api/gfp_mask-from-fs-io.rst
@@ -0,0 +1,66 @@
+=================================
+GFP masks used from FS/IO context
+=================================
+
+:Date: May, 2018
+:Author: Michal Hocko <mhocko@kernel.org>
+
+Introduction
+============
+
+Code paths in the filesystem and IO stacks must be careful when
+allocating memory to prevent recursion deadlocks caused by direct
+memory reclaim calling back into the FS or IO paths and blocking on
+already held resources (e.g. locks - most commonly those used for the
+transaction context).
+
+The traditional way to avoid this deadlock problem is to clear __GFP_FS
+respectively __GFP_IO (note the latter implies clearing the first as well) in
+the gfp mask when calling an allocator. GFP_NOFS respectively GFP_NOIO can be
+used as shortcut. It turned out though that above approach has led to
+abuses when the restricted gfp mask is used "just in case" without a
+deeper consideration which leads to problems because an excessive use
+of GFP_NOFS/GFP_NOIO can lead to memory over-reclaim or other memory
+reclaim issues.
+
+New API
+========
+
+Since 4.12 we do have a generic scope API for both NOFS and NOIO context
+``memalloc_nofs_save``, ``memalloc_nofs_restore`` respectively ``memalloc_noio_save``,
+``memalloc_noio_restore`` which allow to mark a scope to be a critical
+section from a filesystem or I/O point of view. Any allocation from that
+scope will inherently drop __GFP_FS respectively __GFP_IO from the given
+mask so no memory allocation can recurse back in the FS/IO.
+
+.. kernel-doc:: include/linux/sched/mm.h
+   :functions: memalloc_nofs_save memalloc_nofs_restore
+.. kernel-doc:: include/linux/sched/mm.h
+   :functions: memalloc_noio_save memalloc_noio_restore
+
+FS/IO code then simply calls the appropriate save function before
+any critical section with respect to the reclaim is started - e.g.
+lock shared with the reclaim context or when a transaction context
+nesting would be possible via reclaim. The restore function should be
+called when the critical section ends. All that ideally along with an
+explanation what is the reclaim context for easier maintenance.
+
+Please note that the proper pairing of save/restore functions
+allows nesting so it is safe to call ``memalloc_noio_save`` or
+``memalloc_noio_restore`` respectively from an existing NOIO or NOFS
+scope.
+
+What about __vmalloc(GFP_NOFS)
+==============================
+
+vmalloc doesn't support GFP_NOFS semantic because there are hardcoded
+GFP_KERNEL allocations deep inside the allocator which are quite non-trivial
+to fix up. That means that calling ``vmalloc`` with GFP_NOFS/GFP_NOIO is
+almost always a bug. The good news is that the NOFS/NOIO semantic can be
+achieved by the scope API.
+
+In the ideal world, upper layers should already mark dangerous contexts
+and so no special care is required and vmalloc should be called without
+any problems. Sometimes if the context is not really clear or there are
+layering violations then the recommended way around that is to wrap ``vmalloc``
+by the scope API with a comment explaining the problem.
diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst
index c670a8031786..f5a66b72f984 100644
--- a/Documentation/core-api/index.rst
+++ b/Documentation/core-api/index.rst
@@ -14,6 +14,7 @@ Core utilities
    kernel-api
    assoc_array
    atomic_ops
+   cachetlb
    refcount-vs-atomic
    cpu_hotplug
    idr
@@ -25,6 +26,8 @@ Core utilities
    genalloc
    errseq
    printk-formats
+   circular-buffers
+   gfp_mask-from-fs-io
 
 Interfaces for kernel debugging
 ===============================
diff --git a/Documentation/core-api/kernel-api.rst b/Documentation/core-api/kernel-api.rst
index 92f30006adae..8e44aea366c2 100644
--- a/Documentation/core-api/kernel-api.rst
+++ b/Documentation/core-api/kernel-api.rst
@@ -39,17 +39,17 @@ String Manipulation
 .. kernel-doc:: lib/string.c
    :export:
 
+Basic Kernel Library Functions
+==============================
+
+The Linux kernel provides more basic utility functions.
+
 Bit Operations
 --------------
 
 .. kernel-doc:: arch/x86/include/asm/bitops.h
    :internal:
 
-Basic Kernel Library Functions
-==============================
-
-The Linux kernel provides more basic utility functions.
-
 Bitmap Operations
 -----------------
 
@@ -80,6 +80,31 @@ Command-line Parsing
 .. kernel-doc:: lib/cmdline.c
    :export:
 
+Sorting
+-------
+
+.. kernel-doc:: lib/sort.c
+   :export:
+
+.. kernel-doc:: lib/list_sort.c
+   :export:
+
+Text Searching
+--------------
+
+.. kernel-doc:: lib/textsearch.c
+   :doc: ts_intro
+
+.. kernel-doc:: lib/textsearch.c
+   :export:
+
+.. kernel-doc:: include/linux/textsearch.h
+   :functions: textsearch_find textsearch_next \
+               textsearch_get_pattern textsearch_get_pattern_len
+
+CRC and Math Functions in Linux
+===============================
+
 CRC Functions
 -------------
 
@@ -103,9 +128,6 @@ CRC Functions
 .. kernel-doc:: lib/crc-itu-t.c
    :export:
 
-Math Functions in Linux
-=======================
-
 Base 2 log and power Functions
 ------------------------------
 
@@ -127,28 +149,6 @@ Division Functions
 .. kernel-doc:: lib/gcd.c
    :export:
 
-Sorting
--------
-
-.. kernel-doc:: lib/sort.c
-   :export:
-
-.. kernel-doc:: lib/list_sort.c
-   :export:
-
-Text Searching
---------------
-
-.. kernel-doc:: lib/textsearch.c
-   :doc: ts_intro
-
-.. kernel-doc:: lib/textsearch.c
-   :export:
-
-.. kernel-doc:: include/linux/textsearch.h
-   :functions: textsearch_find textsearch_next \
-               textsearch_get_pattern textsearch_get_pattern_len
-
 UUID/GUID
 ---------
 
diff --git a/Documentation/core-api/refcount-vs-atomic.rst b/Documentation/core-api/refcount-vs-atomic.rst
index 83351c258cdb..322851bada16 100644
--- a/Documentation/core-api/refcount-vs-atomic.rst
+++ b/Documentation/core-api/refcount-vs-atomic.rst
@@ -17,7 +17,7 @@ in order to help maintainers validate their code against the change in
 these memory ordering guarantees.
 
 The terms used through this document try to follow the formal LKMM defined in
-github.com/aparri/memory-model/blob/master/Documentation/explanation.txt
+tools/memory-model/Documentation/explanation.txt.
 
 memory-barriers.txt and atomic_t.txt provide more background to the
 memory ordering in general and for atomic operations specifically.
diff --git a/Documentation/crypto/index.rst b/Documentation/crypto/index.rst
index 94c4786f2573..c4ff5d791233 100644
--- a/Documentation/crypto/index.rst
+++ b/Documentation/crypto/index.rst
@@ -20,5 +20,6 @@ for cryptographic use cases, as well as programming examples.
    architecture
    devel-algos
    userspace-if
+   crypto_engine
    api
    api-samples
diff --git a/Documentation/dell_rbu.txt b/Documentation/dell_rbu.txt
index 0fdb6aa2704c..5d1ce7bcd04d 100644
--- a/Documentation/dell_rbu.txt
+++ b/Documentation/dell_rbu.txt
@@ -121,10 +121,7 @@ read back the image downloaded.
 
 .. note::
 
-   This driver requires a patch for firmware_class.c which has the modified
-   request_firmware_nowait function.
-
-   Also after updating the BIOS image a user mode application needs to execute
+   After updating the BIOS image a user mode application needs to execute
    code which sends the BIOS update request to the BIOS. So on the next reboot
    the BIOS knows about the new image downloaded and it updates itself.
    Also don't unload the rbu driver if the image has to be updated.
diff --git a/Documentation/dev-tools/kasan.rst b/Documentation/dev-tools/kasan.rst
index f7a18f274357..aabc8738b3d8 100644
--- a/Documentation/dev-tools/kasan.rst
+++ b/Documentation/dev-tools/kasan.rst
@@ -120,7 +120,7 @@ A typical out of bounds access report looks like this::
 
 The header of the report discribe what kind of bug happened and what kind of
 access caused it. It's followed by the description of the accessed slub object
-(see 'SLUB Debug output' section in Documentation/vm/slub.txt for details) and
+(see 'SLUB Debug output' section in Documentation/vm/slub.rst for details) and
 the description of the accessed memory page.
 
 In the last section the report shows memory state around the accessed address.
diff --git a/Documentation/dev-tools/kselftest.rst b/Documentation/dev-tools/kselftest.rst
index e80850eefe13..3bf371a938d0 100644
--- a/Documentation/dev-tools/kselftest.rst
+++ b/Documentation/dev-tools/kselftest.rst
@@ -151,6 +151,11 @@ Contributing new tests (details)
    TEST_FILES, TEST_GEN_FILES mean it is the file which is used by
    test.
 
+ * First use the headers inside the kernel source and/or git repo, and then the
+   system headers.  Headers for the kernel release as opposed to headers
+   installed by the distro on the system should be the primary focus to be able
+   to find regressions.
+
 Test Harness
 ============
 
diff --git a/Documentation/device-mapper/writecache.txt b/Documentation/device-mapper/writecache.txt
new file mode 100644
index 000000000000..4424fa2c67d7
--- /dev/null
+++ b/Documentation/device-mapper/writecache.txt
@@ -0,0 +1,68 @@
+The writecache target caches writes on persistent memory or on SSD. It
+doesn't cache reads because reads are supposed to be cached in page cache
+in normal RAM.
+
+When the device is constructed, the first sector should be zeroed or the
+first sector should contain valid superblock from previous invocation.
+
+Constructor parameters:
+1. type of the cache device - "p" or "s"
+	p - persistent memory
+	s - SSD
+2. the underlying device that will be cached
+3. the cache device
+4. block size (4096 is recommended; the maximum block size is the page
+   size)
+5. the number of optional parameters (the parameters with an argument
+   count as two)
+	high_watermark n	(default: 50)
+		start writeback when the number of used blocks reach this
+		watermark
+	low_watermark x		(default: 45)
+		stop writeback when the number of used blocks drops below
+		this watermark
+	writeback_jobs n	(default: unlimited)
+		limit the number of blocks that are in flight during
+		writeback. Setting this value reduces writeback
+		throughput, but it may improve latency of read requests
+	autocommit_blocks n	(default: 64 for pmem, 65536 for ssd)
+		when the application writes this amount of blocks without
+		issuing the FLUSH request, the blocks are automatically
+		commited
+	autocommit_time ms	(default: 1000)
+		autocommit time in milliseconds. The data is automatically
+		commited if this time passes and no FLUSH request is
+		received
+	fua			(by default on)
+		applicable only to persistent memory - use the FUA flag
+		when writing data from persistent memory back to the
+		underlying device
+	nofua
+		applicable only to persistent memory - don't use the FUA
+		flag when writing back data and send the FLUSH request
+		afterwards
+		- some underlying devices perform better with fua, some
+		  with nofua. The user should test it
+
+Status:
+1. error indicator - 0 if there was no error, otherwise error number
+2. the number of blocks
+3. the number of free blocks
+4. the number of blocks under writeback
+
+Messages:
+	flush
+		flush the cache device. The message returns successfully
+		if the cache device was flushed without an error
+	flush_on_suspend
+		flush the cache device on next suspend. Use this message
+		when you are going to remove the cache device. The proper
+		sequence for removing the cache device is:
+		1. send the "flush_on_suspend" message
+		2. load an inactive table with a linear target that maps
+		   to the underlying device
+		3. suspend the device
+		4. ask for status and verify that there are no errors
+		5. resume the device, so that it will use the linear
+		   target
+		6. the cache device is now inactive and it can be deleted
diff --git a/Documentation/devicetree/bindings/arm/altera/socfpga-eccmgr.txt b/Documentation/devicetree/bindings/arm/altera/socfpga-eccmgr.txt
deleted file mode 100644
index 4a1714f96bab..000000000000
--- a/Documentation/devicetree/bindings/arm/altera/socfpga-eccmgr.txt
+++ /dev/null
@@ -1,233 +0,0 @@
-Altera SoCFPGA ECC Manager
-This driver uses the EDAC framework to implement the SOCFPGA ECC Manager.
-The ECC Manager counts and corrects single bit errors and counts/handles
-double bit errors which are uncorrectable.
-
-Cyclone5 and Arria5 ECC Manager
-Required Properties:
-- compatible : Should be "altr,socfpga-ecc-manager"
-- #address-cells: must be 1
-- #size-cells: must be 1
-- ranges : standard definition, should translate from local addresses
-
-Subcomponents:
-
-L2 Cache ECC
-Required Properties:
-- compatible : Should be "altr,socfpga-l2-ecc"
-- reg : Address and size for ECC error interrupt clear registers.
-- interrupts : Should be single bit error interrupt, then double bit error
-	interrupt. Note the rising edge type.
-
-On Chip RAM ECC
-Required Properties:
-- compatible : Should be "altr,socfpga-ocram-ecc"
-- reg : Address and size for ECC error interrupt clear registers.
-- iram : phandle to On-Chip RAM definition.
-- interrupts : Should be single bit error interrupt, then double bit error
-	interrupt. Note the rising edge type.
-
-Example:
-
-	eccmgr: eccmgr@ffd08140 {
-		compatible = "altr,socfpga-ecc-manager";
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges;
-
-		l2-ecc@ffd08140 {
-			compatible = "altr,socfpga-l2-ecc";
-			reg = <0xffd08140 0x4>;
-			interrupts = <0 36 1>, <0 37 1>;
-		};
-
-		ocram-ecc@ffd08144 {
-			compatible = "altr,socfpga-ocram-ecc";
-			reg = <0xffd08144 0x4>;
-			iram = <&ocram>;
-			interrupts = <0 178 1>, <0 179 1>;
-		};
-	};
-
-Arria10 SoCFPGA ECC Manager
-The Arria10 SoC ECC Manager handles the IRQs for each peripheral
-in a shared register instead of individual IRQs like the Cyclone5
-and Arria5. Therefore the device tree is different as well.
-
-Required Properties:
-- compatible : Should be "altr,socfpga-a10-ecc-manager"
-- altr,sysgr-syscon : phandle to Arria10 System Manager Block
-	containing the ECC manager registers.
-- #address-cells: must be 1
-- #size-cells: must be 1
-- interrupts : Should be single bit error interrupt, then double bit error
-	interrupt.
-- interrupt-controller : boolean indicator that ECC Manager is an interrupt controller
-- #interrupt-cells : must be set to 2.
-- ranges : standard definition, should translate from local addresses
-
-Subcomponents:
-
-L2 Cache ECC
-Required Properties:
-- compatible : Should be "altr,socfpga-a10-l2-ecc"
-- reg : Address and size for ECC error interrupt clear registers.
-- interrupts : Should be single bit error interrupt, then double bit error
-	interrupt, in this order.
-
-On-Chip RAM ECC
-Required Properties:
-- compatible : Should be "altr,socfpga-a10-ocram-ecc"
-- reg        : Address and size for ECC block registers.
-- interrupts : Should be single bit error interrupt, then double bit error
-	interrupt, in this order.
-
-Ethernet FIFO ECC
-Required Properties:
-- compatible      : Should be "altr,socfpga-eth-mac-ecc"
-- reg             : Address and size for ECC block registers.
-- altr,ecc-parent : phandle to parent Ethernet node.
-- interrupts      : Should be single bit error interrupt, then double bit error
-	interrupt, in this order.
-
-NAND FIFO ECC
-Required Properties:
-- compatible      : Should be "altr,socfpga-nand-ecc"
-- reg             : Address and size for ECC block registers.
-- altr,ecc-parent : phandle to parent NAND node.
-- interrupts      : Should be single bit error interrupt, then double bit error
-	interrupt, in this order.
-
-DMA FIFO ECC
-Required Properties:
-- compatible      : Should be "altr,socfpga-dma-ecc"
-- reg             : Address and size for ECC block registers.
-- altr,ecc-parent : phandle to parent DMA node.
-- interrupts      : Should be single bit error interrupt, then double bit error
-	interrupt, in this order.
-
-USB FIFO ECC
-Required Properties:
-- compatible      : Should be "altr,socfpga-usb-ecc"
-- reg             : Address and size for ECC block registers.
-- altr,ecc-parent : phandle to parent USB node.
-- interrupts      : Should be single bit error interrupt, then double bit error
-	interrupt, in this order.
-
-QSPI FIFO ECC
-Required Properties:
-- compatible      : Should be "altr,socfpga-qspi-ecc"
-- reg             : Address and size for ECC block registers.
-- altr,ecc-parent : phandle to parent QSPI node.
-- interrupts      : Should be single bit error interrupt, then double bit error
-	interrupt, in this order.
-
-SDMMC FIFO ECC
-Required Properties:
-- compatible      : Should be "altr,socfpga-sdmmc-ecc"
-- reg             : Address and size for ECC block registers.
-- altr,ecc-parent : phandle to parent SD/MMC node.
-- interrupts      : Should be single bit error interrupt, then double bit error
-	interrupt, in this order for port A, and then single bit error interrupt,
-	then double bit error interrupt in this order for port B.
-
-Example:
-
-	eccmgr: eccmgr@ffd06000 {
-		compatible = "altr,socfpga-a10-ecc-manager";
-		altr,sysmgr-syscon = <&sysmgr>;
-		#address-cells = <1>;
-		#size-cells = <1>;
-		interrupts = <0 2 IRQ_TYPE_LEVEL_HIGH>,
-			     <0 0 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-controller;
-		#interrupt-cells = <2>;
-		ranges;
-
-		l2-ecc@ffd06010 {
-			compatible = "altr,socfpga-a10-l2-ecc";
-			reg = <0xffd06010 0x4>;
-			interrupts = <0 IRQ_TYPE_LEVEL_HIGH>,
-				     <32 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
-		ocram-ecc@ff8c3000 {
-			compatible = "altr,socfpga-a10-ocram-ecc";
-			reg = <0xff8c3000 0x90>;
-			interrupts = <1 IRQ_TYPE_LEVEL_HIGH>,
-				     <33 IRQ_TYPE_LEVEL_HIGH> ;
-		};
-
-		emac0-rx-ecc@ff8c0800 {
-			compatible = "altr,socfpga-eth-mac-ecc";
-			reg = <0xff8c0800 0x400>;
-			altr,ecc-parent = <&gmac0>;
-			interrupts = <4 IRQ_TYPE_LEVEL_HIGH>,
-				     <36 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
-		emac0-tx-ecc@ff8c0c00 {
-			compatible = "altr,socfpga-eth-mac-ecc";
-			reg = <0xff8c0c00 0x400>;
-			altr,ecc-parent = <&gmac0>;
-			interrupts = <5 IRQ_TYPE_LEVEL_HIGH>,
-				     <37 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
-		nand-buf-ecc@ff8c2000 {
-			compatible = "altr,socfpga-nand-ecc";
-			reg = <0xff8c2000 0x400>;
-			altr,ecc-parent = <&nand>;
-			interrupts = <11 IRQ_TYPE_LEVEL_HIGH>,
-				     <43 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
-		nand-rd-ecc@ff8c2400 {
-			compatible = "altr,socfpga-nand-ecc";
-			reg = <0xff8c2400 0x400>;
-			altr,ecc-parent = <&nand>;
-			interrupts = <13 IRQ_TYPE_LEVEL_HIGH>,
-				     <45 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
-		nand-wr-ecc@ff8c2800 {
-			compatible = "altr,socfpga-nand-ecc";
-			reg = <0xff8c2800 0x400>;
-			altr,ecc-parent = <&nand>;
-			interrupts = <12 IRQ_TYPE_LEVEL_HIGH>,
-				     <44 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
-		dma-ecc@ff8c8000 {
-			compatible = "altr,socfpga-dma-ecc";
-			reg = <0xff8c8000 0x400>;
-			altr,ecc-parent = <&pdma>;
-			interrupts = <10 IRQ_TYPE_LEVEL_HIGH>,
-				     <42 IRQ_TYPE_LEVEL_HIGH>;
-
-		usb0-ecc@ff8c8800 {
-			compatible = "altr,socfpga-usb-ecc";
-			reg = <0xff8c8800 0x400>;
-			altr,ecc-parent = <&usb0>;
-			interrupts = <2 IRQ_TYPE_LEVEL_HIGH>,
-				     <34 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
-		qspi-ecc@ff8c8400 {
-			compatible = "altr,socfpga-qspi-ecc";
-			reg = <0xff8c8400 0x400>;
-			altr,ecc-parent = <&qspi>;
-			interrupts = <14 IRQ_TYPE_LEVEL_HIGH>,
-				     <46 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
-		sdmmc-ecc@ff8c2c00 {
-			compatible = "altr,socfpga-sdmmc-ecc";
-			reg = <0xff8c2c00 0x400>;
-			altr,ecc-parent = <&mmc>;
-			interrupts = <15 IRQ_TYPE_LEVEL_HIGH>,
-				     <47 IRQ_TYPE_LEVEL_HIGH>,
-				     <16 IRQ_TYPE_LEVEL_HIGH>,
-				     <48 IRQ_TYPE_LEVEL_HIGH>;
-		};
-	};
diff --git a/Documentation/devicetree/bindings/arm/amlogic.txt b/Documentation/devicetree/bindings/arm/amlogic.txt
index f747f47922c5..69880560c0f0 100644
--- a/Documentation/devicetree/bindings/arm/amlogic.txt
+++ b/Documentation/devicetree/bindings/arm/amlogic.txt
@@ -25,6 +25,10 @@ Boards with the Amlogic Meson8b SoC shall have the following properties:
   Required root node property:
     compatible: "amlogic,meson8b";
 
+Boards with the Amlogic Meson8m2 SoC shall have the following properties:
+  Required root node property:
+    compatible: "amlogic,meson8m2";
+
 Boards with the Amlogic Meson GXBaby SoC shall have the following properties:
   Required root node property:
     compatible: "amlogic,meson-gxbb";
@@ -54,6 +58,8 @@ Board compatible values (alphabetically, grouped by SoC):
   - "hardkernel,odroid-c1" (Meson8b)
   - "tronfy,mxq" (Meson8b)
 
+  - "tronsmart,mxiii-plus" (Meson8m2)
+
   - "amlogic,p200" (Meson gxbb)
   - "amlogic,p201" (Meson gxbb)
   - "friendlyarm,nanopi-k2" (Meson gxbb)
diff --git a/Documentation/devicetree/bindings/arm/arch_timer.txt b/Documentation/devicetree/bindings/arm/arch_timer.txt
deleted file mode 100644
index 68301b77e854..000000000000
--- a/Documentation/devicetree/bindings/arm/arch_timer.txt
+++ /dev/null
@@ -1,112 +0,0 @@
-* ARM architected timer
-
-ARM cores may have a per-core architected timer, which provides per-cpu timers,
-or a memory mapped architected timer, which provides up to 8 frames with a
-physical and optional virtual timer per frame.
-
-The per-core architected timer is attached to a GIC to deliver its
-per-processor interrupts via PPIs. The memory mapped timer is attached to a GIC
-to deliver its interrupts via SPIs.
-
-** CP15 Timer node properties:
-
-- compatible : Should at least contain one of
-	"arm,armv7-timer"
-	"arm,armv8-timer"
-
-- interrupts : Interrupt list for secure, non-secure, virtual and
-  hypervisor timers, in that order.
-
-- clock-frequency : The frequency of the main counter, in Hz. Should be present
-  only where necessary to work around broken firmware which does not configure
-  CNTFRQ on all CPUs to a uniform correct value. Use of this property is
-  strongly discouraged; fix your firmware unless absolutely impossible.
-
-- always-on : a boolean property. If present, the timer is powered through an
-  always-on power domain, therefore it never loses context.
-
-- fsl,erratum-a008585 : A boolean property. Indicates the presence of
-  QorIQ erratum A-008585, which says that reading the counter is
-  unreliable unless the same value is returned by back-to-back reads.
-  This also affects writes to the tval register, due to the implicit
-  counter read.
-
-- hisilicon,erratum-161010101 : A boolean property. Indicates the
-  presence of Hisilicon erratum 161010101, which says that reading the
-  counters is unreliable in some cases, and reads may return a value 32
-  beyond the correct value. This also affects writes to the tval
-  registers, due to the implicit counter read.
-
-** Optional properties:
-
-- arm,cpu-registers-not-fw-configured : Firmware does not initialize
-  any of the generic timer CPU registers, which contain their
-  architecturally-defined reset values. Only supported for 32-bit
-  systems which follow the ARMv7 architected reset values.
-
-- arm,no-tick-in-suspend : The main counter does not tick when the system is in
-  low-power system suspend on some SoCs. This behavior does not match the
-  Architecture Reference Manual's specification that the system counter "must
-  be implemented in an always-on power domain."
-
-
-Example:
-
-	timer {
-		compatible = "arm,cortex-a15-timer",
-			     "arm,armv7-timer";
-		interrupts = <1 13 0xf08>,
-			     <1 14 0xf08>,
-			     <1 11 0xf08>,
-			     <1 10 0xf08>;
-		clock-frequency = <100000000>;
-	};
-
-** Memory mapped timer node properties:
-
-- compatible : Should at least contain "arm,armv7-timer-mem".
-
-- clock-frequency : The frequency of the main counter, in Hz. Should be present
-  only when firmware has not configured the MMIO CNTFRQ registers.
-
-- reg : The control frame base address.
-
-Note that #address-cells, #size-cells, and ranges shall be present to ensure
-the CPU can address a frame's registers.
-
-A timer node has up to 8 frame sub-nodes, each with the following properties:
-
-- frame-number: 0 to 7.
-
-- interrupts : Interrupt list for physical and virtual timers in that order.
-  The virtual timer interrupt is optional.
-
-- reg : The first and second view base addresses in that order. The second view
-  base address is optional.
-
-- status : "disabled" indicates the frame is not available for use. Optional.
-
-Example:
-
-	timer@f0000000 {
-		compatible = "arm,armv7-timer-mem";
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges;
-		reg = <0xf0000000 0x1000>;
-		clock-frequency = <50000000>;
-
-		frame@f0001000 {
-			frame-number = <0>
-			interrupts = <0 13 0x8>,
-				     <0 14 0x8>;
-			reg = <0xf0001000 0x1000>,
-			      <0xf0002000 0x1000>;
-		};
-
-		frame@f0003000 {
-			frame-number = <1>
-			interrupts = <0 15 0x8>;
-			reg = <0xf0003000 0x1000>;
-		};
-	};
diff --git a/Documentation/devicetree/bindings/arm/armv7m_systick.txt b/Documentation/devicetree/bindings/arm/armv7m_systick.txt
deleted file mode 100644
index 7cf4a24601eb..000000000000
--- a/Documentation/devicetree/bindings/arm/armv7m_systick.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-* ARMv7M System Timer
-
-ARMv7-M includes a system timer, known as SysTick. Current driver only
-implements the clocksource feature.
-
-Required properties:
-- compatible	  : Should be "arm,armv7m-systick"
-- reg		  : The address range of the timer
-
-Required clocking property, have to be one of:
-- clocks	  : The input clock of the timer
-- clock-frequency : The rate in HZ in input of the ARM SysTick
-
-Examples:
-
-systick: timer@e000e010 {
-	compatible = "arm,armv7m-systick";
-	reg = <0xe000e010 0x10>;
-	clocks = <&clk_systick>;
-};
-
-systick: timer@e000e010 {
-	compatible = "arm,armv7m-systick";
-	reg = <0xe000e010 0x10>;
-	clock-frequency = <90000000>;
-};
diff --git a/Documentation/devicetree/bindings/arm/bcm/brcm,bcm2835.txt b/Documentation/devicetree/bindings/arm/bcm/brcm,bcm2835.txt
index 3e3efa046ac5..1e3e29a545e2 100644
--- a/Documentation/devicetree/bindings/arm/bcm/brcm,bcm2835.txt
+++ b/Documentation/devicetree/bindings/arm/bcm/brcm,bcm2835.txt
@@ -34,6 +34,10 @@ Raspberry Pi 3 Model B
 Required root node properties:
 compatible = "raspberrypi,3-model-b", "brcm,bcm2837";
 
+Raspberry Pi 3 Model B+
+Required root node properties:
+compatible = "raspberrypi,3-model-b-plus", "brcm,bcm2837";
+
 Raspberry Pi Compute Module
 Required root node properties:
 compatible = "raspberrypi,compute-module", "brcm,bcm2835";
diff --git a/Documentation/devicetree/bindings/arm/global_timer.txt b/Documentation/devicetree/bindings/arm/global_timer.txt
deleted file mode 100644
index bdae3a818793..000000000000
--- a/Documentation/devicetree/bindings/arm/global_timer.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-
-* ARM Global Timer
-	Cortex-A9 are often associated with a per-core Global timer.
-
-** Timer node required properties:
-
-- compatible : should contain
-	     * "arm,cortex-a5-global-timer" for Cortex-A5 global timers.
-	     * "arm,cortex-a9-global-timer" for Cortex-A9 global
-	         timers or any compatible implementation. Note: driver
-	         supports versions r2p0 and above.
-
-- interrupts : One interrupt to each core
-
-- reg : Specify the base address and the size of the GT timer
-	register window.
-
-- clocks : Should be phandle to a clock.
-
-Example:
-
-	timer@2c000600 {
-		compatible = "arm,cortex-a9-global-timer";
-		reg = <0x2c000600 0x20>;
-		interrupts = <1 13 0xf01>;
-		clocks = <&arm_periph_clk>;
-	};
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,g3dsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,g3dsys.txt
new file mode 100644
index 000000000000..7de43bf41fdc
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,g3dsys.txt
@@ -0,0 +1,30 @@
+MediaTek g3dsys controller
+============================
+
+The MediaTek g3dsys controller provides various clocks and reset controller to
+the GPU.
+
+Required Properties:
+
+- compatible: Should be:
+	- "mediatek,mt2701-g3dsys", "syscon":
+		for MT2701 SoC
+	- "mediatek,mt7623-g3dsys", "mediatek,mt2701-g3dsys", "syscon":
+		for MT7623 SoC
+- #clock-cells: Must be 1
+- #reset-cells: Must be 1
+
+The g3dsys controller uses the common clk binding from
+Documentation/devicetree/bindings/clock/clock-bindings.txt
+The available clocks are defined in dt-bindings/clock/mt*-clk.h.
+
+Example:
+
+g3dsys: clock-controller@13000000 {
+	compatible = "mediatek,mt7623-g3dsys",
+		     "mediatek,mt2701-g3dsys",
+		     "syscon";
+	reg = <0 0x13000000 0 0x200>;
+	#clock-cells = <1>;
+	#reset-cells = <1>;
+};
diff --git a/Documentation/devicetree/bindings/arm/mrvl/timer.txt b/Documentation/devicetree/bindings/arm/mrvl/timer.txt
deleted file mode 100644
index 9a6e251462e7..000000000000
--- a/Documentation/devicetree/bindings/arm/mrvl/timer.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-* Marvell MMP Timer controller
-
-Required properties:
-- compatible : Should be "mrvl,mmp-timer".
-- reg : Address and length of the register set of timer controller.
-- interrupts : Should be the interrupt number.
-
-Example:
-	timer0: timer@d4014000 {
-		compatible = "mrvl,mmp-timer";
-		reg = <0xd4014000 0x100>;
-		interrupts = <13>;
-	};
diff --git a/Documentation/devicetree/bindings/arm/msm/timer.txt b/Documentation/devicetree/bindings/arm/msm/timer.txt
deleted file mode 100644
index 5e10c345548f..000000000000
--- a/Documentation/devicetree/bindings/arm/msm/timer.txt
+++ /dev/null
@@ -1,47 +0,0 @@
-* MSM Timer
-
-Properties:
-
-- compatible : Should at least contain "qcom,msm-timer". More specific
-               properties specify which subsystem the timers are paired with.
-
-               "qcom,kpss-timer" - krait subsystem
-               "qcom,scss-timer" - scorpion subsystem
-
-- interrupts : Interrupts for the debug timer, the first general purpose
-               timer, and optionally a second general purpose timer, and
-               optionally as well, 2 watchdog interrupts, in that order.
-
-- reg : Specifies the base address of the timer registers.
-
-- clocks: Reference to the parent clocks, one per output clock. The parents
-          must appear in the same order as the clock names.
-
-- clock-names: The name of the clocks as free-form strings. They should be in
-               the same order as the clocks.
-
-- clock-frequency : The frequency of the debug timer and the general purpose
-                    timer(s) in Hz in that order.
-
-Optional:
-
-- cpu-offset : per-cpu offset used when the timer is accessed without the
-               CPU remapping facilities. The offset is
-               cpu-offset + (0x10000 * cpu-nr).
-
-Example:
-
-       timer@200a000 {
-               compatible = "qcom,scss-timer", "qcom,msm-timer";
-               interrupts = <1 1 0x301>,
-                            <1 2 0x301>,
-                            <1 3 0x301>,
-                            <1 4 0x301>,
-                            <1 5 0x301>;
-               reg = <0x0200a000 0x100>;
-               clock-frequency = <19200000>,
-                                 <32768>;
-               clocks = <&sleep_clk>;
-               clock-names = "sleep";
-               cpu-offset = <0x40000>;
-       };
diff --git a/Documentation/devicetree/bindings/arm/omap/timer.txt b/Documentation/devicetree/bindings/arm/omap/timer.txt
deleted file mode 100644
index d02e27c764ec..000000000000
--- a/Documentation/devicetree/bindings/arm/omap/timer.txt
+++ /dev/null
@@ -1,44 +0,0 @@
-OMAP Timer bindings
-
-Required properties:
-- compatible:		Should be set to one of the below. Please note that
-			OMAP44xx devices have timer instances that are 100%
-			register compatible with OMAP3xxx devices as well as
-			newer timers that are not 100% register compatible.
-			So for OMAP44xx devices timer instances may use
-			different compatible strings.
-
-			ti,omap2420-timer (applicable to OMAP24xx devices)
-			ti,omap3430-timer (applicable to OMAP3xxx/44xx devices)
-			ti,omap4430-timer (applicable to OMAP44xx devices)
-			ti,omap5430-timer (applicable to OMAP543x devices)
-			ti,am335x-timer	(applicable to AM335x devices)
-			ti,am335x-timer-1ms (applicable to AM335x devices)
-
-- reg:			Contains timer register address range (base address and
-			length).
-- interrupts: 		Contains the interrupt information for the timer. The
-			format is being dependent on which interrupt controller
-			the OMAP device uses.
-- ti,hwmods:		Name of the hwmod associated to the timer, "timer<X>",
-			where <X> is the instance number of the timer from the
-			HW spec.
-
-Optional properties:
-- ti,timer-alwon:	Indicates the timer is in an alway-on power domain.
-- ti,timer-dsp:		Indicates the timer can interrupt the on-chip DSP in
-			addition to the ARM CPU.
-- ti,timer-pwm: 	Indicates the timer can generate a PWM output.
-- ti,timer-secure: 	Indicates the timer is reserved on a secure OMAP device
-			and therefore cannot be used by the kernel.
-
-Example:
-
-timer12: timer@48304000 {
-	compatible = "ti,omap3430-timer";
-	reg = <0x48304000 0x400>;
-	interrupts = <95>;
-	ti,hwmods = "timer12"
-	ti,timer-alwon;
-	ti,timer-secure;
-};
diff --git a/Documentation/devicetree/bindings/arm/samsung/exynos-adc.txt b/Documentation/devicetree/bindings/arm/samsung/exynos-adc.txt
deleted file mode 100644
index 6c49db7f8ad2..000000000000
--- a/Documentation/devicetree/bindings/arm/samsung/exynos-adc.txt
+++ /dev/null
@@ -1,103 +0,0 @@
-Samsung Exynos Analog to Digital Converter bindings
-
-The devicetree bindings are for the new ADC driver written for
-Exynos4 and upward SoCs from Samsung.
-
-New driver handles the following
-1. Supports ADC IF found on EXYNOS4412/EXYNOS5250
-   and future SoCs from Samsung
-2. Add ADC driver under iio/adc framework
-3. Also adds the Documentation for device tree bindings
-
-Required properties:
-- compatible:		Must be "samsung,exynos-adc-v1"
-				for exynos4412/5250 and s5pv210 controllers.
-			Must be "samsung,exynos-adc-v2" for
-				future controllers.
-			Must be "samsung,exynos3250-adc" for
-				controllers compatible with ADC of Exynos3250.
-			Must be "samsung,exynos7-adc" for
-				the ADC in Exynos7 and compatibles
-			Must be "samsung,s3c2410-adc" for
-				the ADC in s3c2410 and compatibles
-			Must be "samsung,s3c2416-adc" for
-				the ADC in s3c2416 and compatibles
-			Must be "samsung,s3c2440-adc" for
-				the ADC in s3c2440 and compatibles
-			Must be "samsung,s3c2443-adc" for
-				the ADC in s3c2443 and compatibles
-			Must be "samsung,s3c6410-adc" for
-				the ADC in s3c6410 and compatibles
-- reg:			List of ADC register address range
-			- The base address and range of ADC register
-			- The base address and range of ADC_PHY register (every
-			  SoC except for s3c24xx/s3c64xx ADC)
-- interrupts: 		Contains the interrupt information for the timer. The
-			format is being dependent on which interrupt controller
-			the Samsung device uses.
-- #io-channel-cells = <1>; As ADC has multiple outputs
-- clocks		From common clock bindings: handles to clocks specified
-			in "clock-names" property, in the same order.
-- clock-names		From common clock bindings: list of clock input names
-			used by ADC block:
-			- "adc" : ADC bus clock
-			- "sclk" : ADC special clock (only for Exynos3250 and
-				   compatible ADC block)
-- vdd-supply		VDD input supply.
-
-- samsung,syscon-phandle Contains the PMU system controller node
-			(To access the ADC_PHY register on Exynos5250/5420/5800/3250)
-Optional properties:
-- has-touchscreen:	If present, indicates that a touchscreen is
-			connected an usable.
-
-Note: child nodes can be added for auto probing from device tree.
-
-Example: adding device info in dtsi file
-
-adc: adc@12d10000 {
-	compatible = "samsung,exynos-adc-v1";
-	reg = <0x12D10000 0x100>;
-	interrupts = <0 106 0>;
-	#io-channel-cells = <1>;
-	io-channel-ranges;
-
-	clocks = <&clock 303>;
-	clock-names = "adc";
-
-	vdd-supply = <&buck5_reg>;
-	samsung,syscon-phandle = <&pmu_system_controller>;
-};
-
-Example: adding device info in dtsi file for Exynos3250 with additional sclk
-
-adc: adc@126c0000 {
-	compatible = "samsung,exynos3250-adc", "samsung,exynos-adc-v2;
-	reg = <0x126C0000 0x100>;
-	interrupts = <0 137 0>;
-	#io-channel-cells = <1>;
-	io-channel-ranges;
-
-	clocks = <&cmu CLK_TSADC>, <&cmu CLK_SCLK_TSADC>;
-	clock-names = "adc", "sclk";
-
-	vdd-supply = <&buck5_reg>;
-	samsung,syscon-phandle = <&pmu_system_controller>;
-};
-
-Example: Adding child nodes in dts file
-
-adc@12d10000 {
-
-	/* NTC thermistor is a hwmon device */
-	ncp15wb473@0 {
-		compatible = "murata,ncp15wb473";
-		pullup-uv = <1800000>;
-		pullup-ohm = <47000>;
-		pulldown-ohm = <0>;
-		io-channels = <&adc 4>;
-	};
-};
-
-Note: Does not apply to ADC driver under arch/arm/plat-samsung/
-Note: The child node can be added under the adc node or separately.
diff --git a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt
index 14510b215480..bdadc3da9556 100644
--- a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt
+++ b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt
@@ -21,8 +21,6 @@ Required root node properties:
 	- "samsung,smdk5420"	- for Exynos5420-based Samsung SMDK5420 eval board.
 	- "samsung,tm2"		- for Exynos5433-based Samsung TM2 board.
 	- "samsung,tm2e"	- for Exynos5433-based Samsung TM2E board.
-	- "samsung,sd5v1"	- for Exynos5440-based Samsung board.
-	- "samsung,ssdk5440"	- for Exynos5440-based Samsung board.
 
 * Other companies Exynos SoC based
   * FriendlyARM
diff --git a/Documentation/devicetree/bindings/arm/shmobile.txt b/Documentation/devicetree/bindings/arm/shmobile.txt
index d3d1df97834f..d8cf740132c6 100644
--- a/Documentation/devicetree/bindings/arm/shmobile.txt
+++ b/Documentation/devicetree/bindings/arm/shmobile.txt
@@ -21,6 +21,8 @@ SoCs:
     compatible = "renesas,r8a7744"
   - RZ/G1E (R8A77450)
     compatible = "renesas,r8a7745"
+  - RZ/G1C (R8A77470)
+    compatible = "renesas,r8a77470"
   - R-Car M1A (R8A77781)
     compatible = "renesas,r8a7778"
   - R-Car H1 (R8A77790)
@@ -45,6 +47,8 @@ SoCs:
     compatible = "renesas,r8a77970"
   - R-Car V3H (R8A77980)
     compatible = "renesas,r8a77980"
+  - R-Car E3 (R8A77990)
+    compatible = "renesas,r8a77990"
   - R-Car D3 (R8A77995)
     compatible = "renesas,r8a77995"
 
@@ -67,6 +71,8 @@ Boards:
     compatible = "renesas,draak", "renesas,r8a77995"
   - Eagle (RTP0RC77970SEB0010S)
     compatible = "renesas,eagle", "renesas,r8a77970"
+  - Ebisu (RTP0RC77990SEB0010S)
+    compatible = "renesas,ebisu", "renesas,r8a77990"
   - Genmai (RTK772100BC00000BR)
     compatible = "renesas,genmai", "renesas,r7s72100"
   - GR-Peach (X28A-M01-E/F)
@@ -78,6 +84,8 @@ Boards:
     compatible = "renesas,h3ulcb", "renesas,r8a7795"
   - Henninger
     compatible = "renesas,henninger", "renesas,r8a7791"
+  - iWave Systems RZ/G1C Single Board Computer (iW-RainboW-G23S)
+    compatible = "iwave,g23s", "renesas,r8a77470"
   - iWave Systems RZ/G1E SODIMM SOM Development Platform (iW-RainboW-G22D)
     compatible = "iwave,g22d", "iwave,g22m", "renesas,r8a7745"
   - iWave Systems RZ/G1E SODIMM System On Module (iW-RainboW-G22M-SM)
@@ -108,7 +116,7 @@ Boards:
     compatible = "renesas,salvator-x", "renesas,r8a7795"
   - Salvator-X (RTP0RC7796SIPB0011S)
     compatible = "renesas,salvator-x", "renesas,r8a7796"
-  - Salvator-X (RTP0RC7796SIPB0011S (M3N))
+  - Salvator-X (RTP0RC7796SIPB0011S (M3-N))
     compatible = "renesas,salvator-x", "renesas,r8a77965"
   - Salvator-XS (Salvator-X 2nd version, RTP0RC7795SIPB0012S)
     compatible = "renesas,salvator-xs", "renesas,r8a7795"
@@ -124,6 +132,8 @@ Boards:
     compatible = "renesas,sk-rzg1m", "renesas,r8a7743"
   - Stout (ADAS Starterkit, Y-R-CAR-ADAS-SKH2-BOARD)
     compatible = "renesas,stout", "renesas,r8a7790"
+  - V3HSK (Y-ASK-RCAR-V3H-WS10)
+    compatible = "renesas,v3hsk", "renesas,r8a77980"
   - V3MSK (Y-ASK-RCAR-V3M-WS10)
     compatible = "renesas,v3msk", "renesas,r8a77970"
   - Wheat (RTP0RC7792ASKB0000JE)
diff --git a/Documentation/devicetree/bindings/arm/spear-timer.txt b/Documentation/devicetree/bindings/arm/spear-timer.txt
deleted file mode 100644
index c0017221cf55..000000000000
--- a/Documentation/devicetree/bindings/arm/spear-timer.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-* SPEAr ARM Timer
-
-** Timer node required properties:
-
-- compatible : Should be:
-	"st,spear-timer"
-- reg: Address range of the timer registers
-- interrupt-parent: Should be the phandle for the interrupt controller
-  that services interrupts for this device
-- interrupt: Should contain the timer interrupt number
-
-Example:
-
-	timer@f0000000 {
-		compatible = "st,spear-timer";
-		reg = <0xf0000000 0x400>;
-		interrupts = <2>;
-	};
diff --git a/Documentation/devicetree/bindings/arm/stm32.txt b/Documentation/devicetree/bindings/arm/stm32.txt
deleted file mode 100644
index 6808ed9ddfd5..000000000000
--- a/Documentation/devicetree/bindings/arm/stm32.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-STMicroelectronics STM32 Platforms Device Tree Bindings
-
-Each device tree must specify which STM32 SoC it uses,
-using one of the following compatible strings:
-
-  st,stm32f429
-  st,stm32f469
-  st,stm32f746
-  st,stm32h743
-  st,stm32mp157
diff --git a/Documentation/devicetree/bindings/arm/stm32/stm32-syscon.txt b/Documentation/devicetree/bindings/arm/stm32/stm32-syscon.txt
new file mode 100644
index 000000000000..99980aee26e5
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/stm32/stm32-syscon.txt
@@ -0,0 +1,14 @@
+STMicroelectronics STM32 Platforms System Controller
+
+Properties:
+   - compatible : should contain two values. First value must be :
+                 - " st,stm32mp157-syscfg " - for stm32mp157 based SoCs,
+                 second value must be always "syscon".
+   - reg : offset and length of the register set.
+
+ Example:
+         syscfg: syscon@50020000 {
+                 compatible = "st,stm32mp157-syscfg", "syscon";
+                 reg = <0x50020000 0x400>;
+         };
+
diff --git a/Documentation/devicetree/bindings/arm/stm32/stm32.txt b/Documentation/devicetree/bindings/arm/stm32/stm32.txt
new file mode 100644
index 000000000000..6808ed9ddfd5
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/stm32/stm32.txt
@@ -0,0 +1,10 @@
+STMicroelectronics STM32 Platforms Device Tree Bindings
+
+Each device tree must specify which STM32 SoC it uses,
+using one of the following compatible strings:
+
+  st,stm32f429
+  st,stm32f469
+  st,stm32f746
+  st,stm32h743
+  st,stm32mp157
diff --git a/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra20-mc.txt b/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra20-mc.txt
deleted file mode 100644
index f9632bacbd04..000000000000
--- a/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra20-mc.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-NVIDIA Tegra20 MC(Memory Controller)
-
-Required properties:
-- compatible : "nvidia,tegra20-mc"
-- reg : Should contain 2 register ranges(address and length); see the
-  example below. Note that the MC registers are interleaved with the
-  GART registers, and hence must be represented as multiple ranges.
-- interrupts : Should contain MC General interrupt.
-
-Example:
-	memory-controller@7000f000 {
-		compatible = "nvidia,tegra20-mc";
-		reg = <0x7000f000 0x024
-		       0x7000f03c 0x3c4>;
-		interrupts = <0 77 0x04>;
-	};
diff --git a/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra30-mc.txt b/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra30-mc.txt
deleted file mode 100644
index bdf1a612422b..000000000000
--- a/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra30-mc.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-NVIDIA Tegra30 MC(Memory Controller)
-
-Required properties:
-- compatible : "nvidia,tegra30-mc"
-- reg : Should contain 4 register ranges(address and length); see the
-  example below. Note that the MC registers are interleaved with the
-  SMMU registers, and hence must be represented as multiple ranges.
-- interrupts : Should contain MC General interrupt.
-
-Example:
-	memory-controller {
-		compatible = "nvidia,tegra30-mc";
-		reg = <0x7000f000 0x010
-		       0x7000f03c 0x1b4
-		       0x7000f200 0x028
-		       0x7000f284 0x17c>;
-		interrupts = <0 77 0x04>;
-	};
diff --git a/Documentation/devicetree/bindings/arm/twd.txt b/Documentation/devicetree/bindings/arm/twd.txt
deleted file mode 100644
index 383ea19c2bf0..000000000000
--- a/Documentation/devicetree/bindings/arm/twd.txt
+++ /dev/null
@@ -1,53 +0,0 @@
-* ARM Timer Watchdog
-
-ARM 11MP, Cortex-A5 and Cortex-A9 are often associated with a per-core
-Timer-Watchdog (aka TWD), which provides both a per-cpu local timer
-and watchdog.
-
-The TWD is usually attached to a GIC to deliver its two per-processor
-interrupts.
-
-** Timer node required properties:
-
-- compatible : Should be one of:
-	"arm,cortex-a9-twd-timer"
-	"arm,cortex-a5-twd-timer"
-	"arm,arm11mp-twd-timer"
-
-- interrupts : One interrupt to each core
-
-- reg : Specify the base address and the size of the TWD timer
-	register window.
-
-Optional
-
-- always-on : a boolean property. If present, the timer is powered through
-  an always-on power domain, therefore it never loses context.
-
-Example:
-
-	twd-timer@2c000600 {
-		compatible = "arm,arm11mp-twd-timer"";
-		reg = <0x2c000600 0x20>;
-		interrupts = <1 13 0xf01>;
-	};
-
-** Watchdog node properties:
-
-- compatible : Should be one of:
-	"arm,cortex-a9-twd-wdt"
-	"arm,cortex-a5-twd-wdt"
-	"arm,arm11mp-twd-wdt"
-
-- interrupts : One interrupt to each core
-
-- reg : Specify the base address and the size of the TWD watchdog
-	register window.
-
-Example:
-
-	twd-watchdog@2c000620 {
-		compatible = "arm,arm11mp-twd-wdt";
-		reg = <0x2c000620 0x20>;
-		interrupts = <1 14 0xf01>;
-	};
diff --git a/Documentation/devicetree/bindings/arm/ux500/boards.txt b/Documentation/devicetree/bindings/arm/ux500/boards.txt
index 7334c24625fc..0fa429534f49 100644
--- a/Documentation/devicetree/bindings/arm/ux500/boards.txt
+++ b/Documentation/devicetree/bindings/arm/ux500/boards.txt
@@ -26,7 +26,7 @@ interrupt-controller:
 	see binding for interrupt-controller/arm,gic.txt
 
 timer:
-	see binding for arm/twd.txt
+	see binding for timer/arm,twd.txt
 
 clocks:
 	see binding for clocks/ux500.txt
diff --git a/Documentation/devicetree/bindings/arm/vt8500/via,vt8500-timer.txt b/Documentation/devicetree/bindings/arm/vt8500/via,vt8500-timer.txt
deleted file mode 100644
index 901c73f0d8ef..000000000000
--- a/Documentation/devicetree/bindings/arm/vt8500/via,vt8500-timer.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-VIA/Wondermedia VT8500 Timer
------------------------------------------------------
-
-Required properties:
-- compatible : "via,vt8500-timer"
-- reg : Should contain 1 register ranges(address and length)
-- interrupts : interrupt for the timer
-
-Example:
-
-	timer@d8130100 {
-		compatible = "via,vt8500-timer";
-		reg = <0xd8130100 0x28>;
-		interrupts = <36>;
-	};
diff --git a/Documentation/devicetree/bindings/bus/ti-sysc.txt b/Documentation/devicetree/bindings/bus/ti-sysc.txt
index 2957a9ae291f..d8ed5b780ed9 100644
--- a/Documentation/devicetree/bindings/bus/ti-sysc.txt
+++ b/Documentation/devicetree/bindings/bus/ti-sysc.txt
@@ -79,7 +79,11 @@ Optional properties:
 		mode as for example omap4 L4_CFG_CLKCTRL
 
 - clock-names	should contain at least "fck", and optionally also "ick"
-		depending on the SoC and the interconnect target module
+		depending on the SoC and the interconnect target module,
+		some interconnect target modules also need additional
+		optional clocks that can be specified as listed in TRM
+		for the related CLKCTRL register bits 8 to 15 such as
+		"dbclk" or "clk32k" depending on their role
 
 - ti,hwmods	optional TI interconnect module name to use legacy
 		hwmod platform data
diff --git a/Documentation/devicetree/bindings/c6x/timer64.txt b/Documentation/devicetree/bindings/c6x/timer64.txt
deleted file mode 100644
index 95911fe70224..000000000000
--- a/Documentation/devicetree/bindings/c6x/timer64.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-Timer64
--------
-
-The timer64 node describes C6X event timers.
-
-Required properties:
-
-- compatible: must be "ti,c64x+timer64"
-- reg: base address and size of register region
-- interrupt-parent: interrupt controller
-- interrupts: interrupt id
-
-Optional properties:
-
-- ti,dscr-dev-enable: Device ID used to enable timer IP through DSCR interface.
-
-- ti,core-mask: on multi-core SoCs, bitmask of cores allowed to use this timer.
-
-Example:
-	timer0: timer@25e0000 {
-		compatible = "ti,c64x+timer64";
-		ti,core-mask = < 0x01 >;
-		reg = <0x25e0000 0x40>;
-		interrupt-parent = <&megamod_pic>;
-		interrupts = < 16 >;
-	};
diff --git a/Documentation/devicetree/bindings/clock/actions,s900-cmu.txt b/Documentation/devicetree/bindings/clock/actions,s900-cmu.txt
new file mode 100644
index 000000000000..93e4fb827cd6
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/actions,s900-cmu.txt
@@ -0,0 +1,47 @@
+* Actions S900 Clock Management Unit (CMU)
+
+The Actions S900 clock management unit generates and supplies clock to various
+controllers within the SoC. The clock binding described here is applicable to
+S900 SoC.
+
+Required Properties:
+
+- compatible: should be "actions,s900-cmu"
+- reg: physical base address of the controller and length of memory mapped
+  region.
+- clocks: Reference to the parent clocks ("hosc", "losc")
+- #clock-cells: should be 1.
+
+Each clock is assigned an identifier, and client nodes can use this identifier
+to specify the clock which they consume.
+
+All available clocks are defined as preprocessor macros in
+dt-bindings/clock/actions,s900-cmu.h header and can be used in device
+tree sources.
+
+External clocks:
+
+The hosc clock used as input for the plls is generated outside the SoC. It is
+expected that it is defined using standard clock bindings as "hosc".
+
+Actions S900 CMU also requires one more clock:
+ - "losc" - internal low frequency oscillator
+
+Example: Clock Management Unit node:
+
+        cmu: clock-controller@e0160000 {
+                compatible = "actions,s900-cmu";
+                reg = <0x0 0xe0160000 0x0 0x1000>;
+                clocks = <&hosc>, <&losc>;
+                #clock-cells = <1>;
+        };
+
+Example: UART controller node that consumes clock generated by the clock
+management unit:
+
+        uart: serial@e012a000 {
+                compatible = "actions,s900-uart", "actions,owl-uart";
+                reg = <0x0 0xe012a000 0x0 0x2000>;
+                interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
+                clocks = <&cmu CLK_UART5>;
+        };
diff --git a/Documentation/devicetree/bindings/clock/amlogic,gxbb-aoclkc.txt b/Documentation/devicetree/bindings/clock/amlogic,gxbb-aoclkc.txt
index 786dc39ca904..3a880528030e 100644
--- a/Documentation/devicetree/bindings/clock/amlogic,gxbb-aoclkc.txt
+++ b/Documentation/devicetree/bindings/clock/amlogic,gxbb-aoclkc.txt
@@ -9,6 +9,7 @@ Required Properties:
 	- GXBB (S905) : "amlogic,meson-gxbb-aoclkc"
 	- GXL (S905X, S905D) : "amlogic,meson-gxl-aoclkc"
 	- GXM (S912) : "amlogic,meson-gxm-aoclkc"
+	- AXG (A113D, A113X) : "amlogic,meson-axg-aoclkc"
 	followed by the common "amlogic,meson-gx-aoclkc"
 
 - #clock-cells: should be 1.
diff --git a/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt b/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt
index e2b377ed6f91..e950599566a9 100644
--- a/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt
+++ b/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt
@@ -10,9 +10,6 @@ Required Properties:
 		"amlogic,gxl-clkc" for GXL and GXM SoC,
 		"amlogic,axg-clkc" for AXG SoC.
 
-- reg: physical base address of the clock controller and length of memory
-       mapped region.
-
 - #clock-cells: should be 1.
 
 Each clock is assigned an identifier and client nodes can use this identifier
@@ -20,13 +17,22 @@ to specify the clock which they consume. All available clocks are defined as
 preprocessor macros in the dt-bindings/clock/gxbb-clkc.h header and can be
 used in device tree sources.
 
+Parent node should have the following properties :
+- compatible: "syscon", "simple-mfd, and "amlogic,meson-gx-hhi-sysctrl" or
+              "amlogic,meson-axg-hhi-sysctrl"
+- reg: base address and size of the HHI system control register space.
+
 Example: Clock controller node:
 
-	clkc: clock-controller@c883c000 {
+sysctrl: system-controller@0 {
+	compatible = "amlogic,meson-gx-hhi-sysctrl", "syscon", "simple-mfd";
+	reg = <0 0 0 0x400>;
+
+	clkc: clock-controller {
 		#clock-cells = <1>;
 		compatible = "amlogic,gxbb-clkc";
-		reg = <0x0 0xc883c000 0x0 0x3db>;
 	};
+};
 
 Example: UART controller node that consumes the clock generated by the clock
   controller:
diff --git a/Documentation/devicetree/bindings/clock/nuvoton,npcm750-clk.txt b/Documentation/devicetree/bindings/clock/nuvoton,npcm750-clk.txt
new file mode 100644
index 000000000000..f82064546d11
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/nuvoton,npcm750-clk.txt
@@ -0,0 +1,100 @@
+* Nuvoton NPCM7XX Clock Controller
+
+Nuvoton Poleg BMC NPCM7XX contains an integrated clock controller, which
+generates and supplies clocks to all modules within the BMC.
+
+External clocks:
+
+There are six fixed clocks that are generated outside the BMC. All clocks are of
+a known fixed value that cannot be changed. clk_refclk, clk_mcbypck and
+clk_sysbypck are inputs to the clock controller.
+clk_rg1refck, clk_rg2refck and clk_xin are external clocks suppling the
+network. They are set on the device tree, but not used by the clock module. The
+network devices use them directly.
+Example can be found below.
+
+All available clocks are defined as preprocessor macros in:
+dt-bindings/clock/nuvoton,npcm7xx-clock.h
+and can be reused as DT sources.
+
+Required Properties of clock controller:
+
+	- compatible: "nuvoton,npcm750-clk" : for clock controller of Nuvoton
+		  Poleg BMC NPCM750
+
+	- reg: physical base address of the clock controller and length of
+		memory mapped region.
+
+	- #clock-cells: should be 1.
+
+Example: Clock controller node:
+
+	clk: clock-controller@f0801000 {
+		compatible = "nuvoton,npcm750-clk";
+		#clock-cells = <1>;
+		reg = <0xf0801000 0x1000>;
+		clock-names = "refclk", "sysbypck", "mcbypck";
+		clocks = <&clk_refclk>, <&clk_sysbypck>, <&clk_mcbypck>;
+	};
+
+Example: Required external clocks for network:
+
+	/* external reference clock */
+	clk_refclk: clk-refclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <25000000>;
+		clock-output-names = "refclk";
+	};
+
+	/* external reference clock for cpu. float in normal operation */
+	clk_sysbypck: clk-sysbypck {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <800000000>;
+		clock-output-names = "sysbypck";
+	};
+
+	/* external reference clock for MC. float in normal operation */
+	clk_mcbypck: clk-mcbypck {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <800000000>;
+		clock-output-names = "mcbypck";
+	};
+
+	 /* external clock signal rg1refck, supplied by the phy */
+	clk_rg1refck: clk-rg1refck {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <125000000>;
+		clock-output-names = "clk_rg1refck";
+	};
+
+	 /* external clock signal rg2refck, supplied by the phy */
+	clk_rg2refck: clk-rg2refck {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <125000000>;
+		clock-output-names = "clk_rg2refck";
+	};
+
+	clk_xin: clk-xin {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <50000000>;
+		clock-output-names = "clk_xin";
+	};
+
+
+Example: GMAC controller node that consumes two clocks: a generated clk by the
+clock controller and a fixed clock from DT (clk_rg1refck).
+
+	ethernet0: ethernet@f0802000 {
+		compatible = "snps,dwmac";
+		reg = <0xf0802000 0x2000>;
+		interrupts = <0 14 4>;
+		interrupt-names = "macirq";
+		clocks	= <&clk_rg1refck>, <&clk NPCM7XX_CLK_AHB>;
+		clock-names = "stmmaceth", "clk_gmac";
+	};
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc.txt b/Documentation/devicetree/bindings/clock/qcom,gcc.txt
index 551d03be9665..664ea1fd6c76 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc.txt
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc.txt
@@ -17,7 +17,9 @@ Required properties :
 			"qcom,gcc-msm8974pro-ac"
 			"qcom,gcc-msm8994"
 			"qcom,gcc-msm8996"
+			"qcom,gcc-msm8998"
 			"qcom,gcc-mdm9615"
+			"qcom,gcc-sdm845"
 
 - reg : shall contain base register location and length
 - #clock-cells : shall contain 1
diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmh-clk.txt b/Documentation/devicetree/bindings/clock/qcom,rpmh-clk.txt
new file mode 100644
index 000000000000..3c007653da31
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,rpmh-clk.txt
@@ -0,0 +1,22 @@
+Qualcomm Technologies, Inc. RPMh Clocks
+-------------------------------------------------------
+
+Resource Power Manager Hardened (RPMh) manages shared resources on
+some Qualcomm Technologies Inc. SoCs. It accepts clock requests from
+other hardware subsystems via RSC to control clocks.
+
+Required properties :
+- compatible : shall contain "qcom,sdm845-rpmh-clk"
+
+- #clock-cells : must contain 1
+
+Example :
+
+#include <dt-bindings/clock/qcom,rpmh.h>
+
+	&apps_rsc {
+		rpmhcc: clock-controller {
+			compatible = "qcom,sdm845-rpmh-clk";
+			#clock-cells = <1>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.txt b/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.txt
index 773a5226342f..db542abadb75 100644
--- a/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.txt
+++ b/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.txt
@@ -15,6 +15,7 @@ Required Properties:
   - compatible: Must be one of:
       - "renesas,r8a7743-cpg-mssr" for the r8a7743 SoC (RZ/G1M)
       - "renesas,r8a7745-cpg-mssr" for the r8a7745 SoC (RZ/G1E)
+      - "renesas,r8a77470-cpg-mssr" for the r8a77470 SoC (RZ/G1C)
       - "renesas,r8a7790-cpg-mssr" for the r8a7790 SoC (R-Car H2)
       - "renesas,r8a7791-cpg-mssr" for the r8a7791 SoC (R-Car M2-W)
       - "renesas,r8a7792-cpg-mssr" for the r8a7792 SoC (R-Car V2H)
@@ -25,6 +26,7 @@ Required Properties:
       - "renesas,r8a77965-cpg-mssr" for the r8a77965 SoC (R-Car M3-N)
       - "renesas,r8a77970-cpg-mssr" for the r8a77970 SoC (R-Car V3M)
       - "renesas,r8a77980-cpg-mssr" for the r8a77980 SoC (R-Car V3H)
+      - "renesas,r8a77990-cpg-mssr" for the r8a77990 SoC (R-Car E3)
       - "renesas,r8a77995-cpg-mssr" for the r8a77995 SoC (R-Car D3)
 
   - reg: Base address and length of the memory resource used by the CPG/MSSR
@@ -33,10 +35,12 @@ Required Properties:
   - clocks: References to external parent clocks, one entry for each entry in
     clock-names
   - clock-names: List of external parent clock names. Valid names are:
-      - "extal" (r8a7743, r8a7745, r8a7790, r8a7791, r8a7792, r8a7793, r8a7794,
-		 r8a7795, r8a7796, r8a77965, r8a77970, r8a77980, r8a77995)
+      - "extal" (r8a7743, r8a7745, r8a77470, r8a7790, r8a7791, r8a7792,
+		 r8a7793, r8a7794, r8a7795, r8a7796, r8a77965, r8a77970,
+		 r8a77980, r8a77990, r8a77995)
       - "extalr" (r8a7795, r8a7796, r8a77965, r8a77970, r8a77980)
-      - "usb_extal" (r8a7743, r8a7745, r8a7790, r8a7791, r8a7793, r8a7794)
+      - "usb_extal" (r8a7743, r8a7745, r8a77470, r8a7790, r8a7791, r8a7793,
+		     r8a7794)
 
   - #clock-cells: Must be 2
       - For CPG core clocks, the two clock specifier cells must be "CPG_CORE"
diff --git a/Documentation/devicetree/bindings/clock/rockchip.txt b/Documentation/devicetree/bindings/clock/rockchip.txt
deleted file mode 100644
index 22f6769e5d4a..000000000000
--- a/Documentation/devicetree/bindings/clock/rockchip.txt
+++ /dev/null
@@ -1,77 +0,0 @@
-Device Tree Clock bindings for arch-rockchip
-
-This binding uses the common clock binding[1].
-
-[1] Documentation/devicetree/bindings/clock/clock-bindings.txt
-
-== Gate clocks ==
-
-These bindings are deprecated!
-Please use the soc specific CRU bindings instead.
-
-The gate registers form a continuos block which makes the dt node
-structure a matter of taste, as either all gates can be put into
-one gate clock spanning all registers or they can be divided into
-the 10 individual gates containing 16 clocks each.
-The code supports both approaches.
-
-Required properties:
-- compatible : "rockchip,rk2928-gate-clk"
-- reg : shall be the control register address(es) for the clock.
-- #clock-cells : from common clock binding; shall be set to 1
-- clock-output-names : the corresponding gate names that the clock controls
-- clocks : should contain the parent clock for each individual gate,
-  therefore the number of clocks elements should match the number of
-  clock-output-names
-
-Example using multiple gate clocks:
-
-		clk_gates0: gate-clk@200000d0 {
-			compatible = "rockchip,rk2928-gate-clk";
-			reg = <0x200000d0 0x4>;
-			clocks = <&dummy>, <&dummy>,
-				 <&dummy>, <&dummy>,
-				 <&dummy>, <&dummy>,
-				 <&dummy>, <&dummy>,
-				 <&dummy>, <&dummy>,
-				 <&dummy>, <&dummy>,
-				 <&dummy>, <&dummy>,
-				 <&dummy>, <&dummy>;
-
-			clock-output-names =
-				"gate_core_periph", "gate_cpu_gpll",
-				"gate_ddrphy", "gate_aclk_cpu",
-				"gate_hclk_cpu", "gate_pclk_cpu",
-				"gate_atclk_cpu", "gate_i2s0",
-				"gate_i2s0_frac", "gate_i2s1",
-				"gate_i2s1_frac", "gate_i2s2",
-				"gate_i2s2_frac", "gate_spdif",
-				"gate_spdif_frac", "gate_testclk";
-
-			#clock-cells = <1>;
-		};
-
-		clk_gates1: gate-clk@200000d4 {
-			compatible = "rockchip,rk2928-gate-clk";
-			reg = <0x200000d4 0x4>;
-			clocks = <&xin24m>, <&xin24m>,
-				 <&xin24m>, <&dummy>,
-				 <&dummy>, <&xin24m>,
-				 <&xin24m>, <&dummy>,
-				 <&xin24m>, <&dummy>,
-				 <&xin24m>, <&dummy>,
-				 <&xin24m>, <&dummy>,
-				 <&xin24m>, <&dummy>;
-
-			clock-output-names =
-				"gate_timer0", "gate_timer1",
-				"gate_timer2", "gate_jtag",
-				"gate_aclk_lcdc1_src", "gate_otgphy0",
-				"gate_otgphy1", "gate_ddr_gpll",
-				"gate_uart0", "gate_frac_uart0",
-				"gate_uart1", "gate_frac_uart1",
-				"gate_uart2", "gate_frac_uart2",
-				"gate_uart3", "gate_frac_uart3";
-
-			#clock-cells = <1>;
-		};
diff --git a/Documentation/devicetree/bindings/clock/sunxi-ccu.txt b/Documentation/devicetree/bindings/clock/sunxi-ccu.txt
index 460ef27b1008..47d2e902ced4 100644
--- a/Documentation/devicetree/bindings/clock/sunxi-ccu.txt
+++ b/Documentation/devicetree/bindings/clock/sunxi-ccu.txt
@@ -21,6 +21,7 @@ Required properties :
 		- "allwinner,sun50i-a64-r-ccu"
 		- "allwinner,sun50i-h5-ccu"
 		- "allwinner,sun50i-h6-ccu"
+		- "allwinner,sun50i-h6-r-ccu"
 		- "nextthing,gr8-ccu"
 
 - reg: Must contain the registers base address and length
@@ -35,7 +36,7 @@ Required properties :
 For the main CCU on H6, one more clock is needed:
 - "iosc": the SoC's internal frequency oscillator
 
-For the PRCM CCUs on A83T/H3/A64, two more clocks are needed:
+For the PRCM CCUs on A83T/H3/A64/H6, two more clocks are needed:
 - "pll-periph": the SoC's peripheral PLL from the main CCU
 - "iosc": the SoC's internal frequency oscillator
 
diff --git a/Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt b/Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt
deleted file mode 100644
index a13fbdb4bd88..000000000000
--- a/Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-Exynos Pseudo Random Number Generator
-
-Required properties:
-
-- compatible  : One of:
-                - "samsung,exynos4-rng" for Exynos4210 and Exynos4412
-                - "samsung,exynos5250-prng" for Exynos5250+
-- reg         : Specifies base physical address and size of the registers map.
-- clocks      : Phandle to clock-controller plus clock-specifier pair.
-- clock-names : "secss" as a clock name.
-
-Example:
-
-	rng@10830400 {
-		compatible = "samsung,exynos4-rng";
-		reg = <0x10830400 0x200>;
-		clocks = <&clock CLK_SSS>;
-		clock-names = "secss";
-	};
diff --git a/Documentation/devicetree/bindings/display/bridge/adi,adv7511.txt b/Documentation/devicetree/bindings/display/bridge/adi,adv7511.txt
index 0047b1394c70..2c887536258c 100644
--- a/Documentation/devicetree/bindings/display/bridge/adi,adv7511.txt
+++ b/Documentation/devicetree/bindings/display/bridge/adi,adv7511.txt
@@ -14,7 +14,13 @@ Required properties:
 		"adi,adv7513"
 		"adi,adv7533"
 
-- reg: I2C slave address
+- reg: I2C slave addresses
+  The ADV7511 internal registers are split into four pages exposed through
+  different I2C addresses, creating four register maps. Each map has it own
+  I2C address and acts as a standard slave device on the I2C bus. The main
+  address is mandatory, others are optional and revert to defaults if not
+  specified.
+
 
 The ADV7511 supports a large number of input data formats that differ by their
 color depth, color format, clock mode, bit justification and random
@@ -70,6 +76,9 @@ Optional properties:
   rather than generate its own timings for HDMI output.
 - clocks: from common clock binding: reference to the CEC clock.
 - clock-names: from common clock binding: must be "cec".
+- reg-names : Names of maps with programmable addresses.
+	It can contain any map needing a non-default address.
+	Possible maps names are : "main", "edid", "cec", "packet"
 
 Required nodes:
 
@@ -88,7 +97,12 @@ Example
 
 	adv7511w: hdmi@39 {
 		compatible = "adi,adv7511w";
-		reg = <39>;
+		/*
+		 * The EDID page will be accessible on address 0x66 on the I2C
+		 * bus. All other maps continue to use their default addresses.
+		 */
+		reg = <0x39>, <0x66>;
+		reg-names = "main", "edid";
 		interrupt-parent = <&gpio3>;
 		interrupts = <29 IRQ_TYPE_EDGE_FALLING>;
 		clocks = <&cec_clock>;
diff --git a/Documentation/devicetree/bindings/display/bridge/cdns,dsi.txt b/Documentation/devicetree/bindings/display/bridge/cdns,dsi.txt
new file mode 100644
index 000000000000..f5725bb6c61c
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/bridge/cdns,dsi.txt
@@ -0,0 +1,133 @@
+Cadence DSI bridge
+==================
+
+The Cadence DSI bridge is a DPI to DSI bridge supporting up to 4 DSI lanes.
+
+Required properties:
+- compatible: should be set to "cdns,dsi".
+- reg: physical base address and length of the controller's registers.
+- interrupts: interrupt line connected to the DSI bridge.
+- clocks: DSI bridge clocks.
+- clock-names: must contain "dsi_p_clk" and "dsi_sys_clk".
+- phys: phandle link to the MIPI D-PHY controller.
+- phy-names: must contain "dphy".
+- #address-cells: must be set to 1.
+- #size-cells: must be set to 0.
+
+Optional properties:
+- resets: DSI reset lines.
+- reset-names: can contain "dsi_p_rst".
+
+Required subnodes:
+- ports: Ports as described in Documentation/devicetree/bindings/graph.txt.
+  2 ports are available:
+  * port 0: this port is only needed if some of your DSI devices are
+	    controlled through  an external bus like I2C or SPI. Can have at
+	    most 4 endpoints. The endpoint number is directly encoding the
+	    DSI virtual channel used by this device.
+  * port 1: represents the DPI input.
+  Other ports will be added later to support the new kind of inputs.
+
+- one subnode per DSI device connected on the DSI bus. Each DSI device should
+  contain a reg property encoding its virtual channel.
+
+Cadence DPHY
+============
+
+Cadence DPHY block.
+
+Required properties:
+- compatible: should be set to "cdns,dphy".
+- reg: physical base address and length of the DPHY registers.
+- clocks: DPHY reference clocks.
+- clock-names: must contain "psm" and "pll_ref".
+- #phy-cells: must be set to 0.
+
+
+Example:
+	dphy0: dphy@fd0e0000{
+		compatible = "cdns,dphy";
+		reg = <0x0 0xfd0e0000 0x0 0x1000>;
+		clocks = <&psm_clk>, <&pll_ref_clk>;
+		clock-names = "psm", "pll_ref";
+		#phy-cells = <0>;
+	};
+
+	dsi0: dsi@fd0c0000 {
+		compatible = "cdns,dsi";
+		reg = <0x0 0xfd0c0000 0x0 0x1000>;
+		clocks = <&pclk>, <&sysclk>;
+		clock-names = "dsi_p_clk", "dsi_sys_clk";
+		interrupts = <1>;
+		phys = <&dphy0>;
+		phy-names = "dphy";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@1 {
+				reg = <1>;
+				dsi0_dpi_input: endpoint {
+					remote-endpoint = <&xxx_dpi_output>;
+				};
+			};
+		};
+
+		panel: dsi-dev@0 {
+			compatible = "<vendor,panel>";
+			reg = <0>;
+		};
+	};
+
+or
+
+	dsi0: dsi@fd0c0000 {
+		compatible = "cdns,dsi";
+		reg = <0x0 0xfd0c0000 0x0 0x1000>;
+		clocks = <&pclk>, <&sysclk>;
+		clock-names = "dsi_p_clk", "dsi_sys_clk";
+		interrupts = <1>;
+		phys = <&dphy1>;
+		phy-names = "dphy";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				dsi0_output: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&dsi_panel_input>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				dsi0_dpi_input: endpoint {
+					remote-endpoint = <&xxx_dpi_output>;
+				};
+			};
+		};
+	};
+
+	i2c@xxx {
+		panel: panel@59 {
+			compatible = "<vendor,panel>";
+			reg = <0x59>;
+
+			port {
+				dsi_panel_input: endpoint {
+					remote-endpoint = <&dsi0_output>;
+				};
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.txt b/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.txt
index 3a72a103a18a..a41d280c3f9f 100644
--- a/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.txt
+++ b/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.txt
@@ -14,6 +14,7 @@ Required properties:
 - compatible : Shall contain one or more of
   - "renesas,r8a7795-hdmi" for R8A7795 (R-Car H3) compatible HDMI TX
   - "renesas,r8a7796-hdmi" for R8A7796 (R-Car M3-W) compatible HDMI TX
+  - "renesas,r8a77965-hdmi" for R8A77965 (R-Car M3-N) compatible HDMI TX
   - "renesas,rcar-gen3-hdmi" for the generic R-Car Gen3 compatible HDMI TX
 
     When compatible with generic versions, nodes must list the SoC-specific
diff --git a/Documentation/devicetree/bindings/display/bridge/tda998x.txt b/Documentation/devicetree/bindings/display/bridge/tda998x.txt
index 24cc2466185a..1a4eaca40d94 100644
--- a/Documentation/devicetree/bindings/display/bridge/tda998x.txt
+++ b/Documentation/devicetree/bindings/display/bridge/tda998x.txt
@@ -27,6 +27,9 @@ Optional properties:
 	in question is used. The implementation allows one or two DAIs. If two
 	DAIs are defined, they must be of different type.
 
+  - nxp,calib-gpios: calibration GPIO, which must correspond with the
+	gpio used for the TDA998x interrupt pin.
+
 [1] Documentation/sound/alsa/soc/DAI.txt
 [2] include/dt-bindings/display/tda998x.h
 
diff --git a/Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.txt b/Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.txt
new file mode 100644
index 000000000000..37f0c04d5a28
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.txt
@@ -0,0 +1,60 @@
+Thine Electronics THC63LVD1024 LVDS decoder
+-------------------------------------------
+
+The THC63LVD1024 is a dual link LVDS receiver designed to convert LVDS streams
+to parallel data outputs. The chip supports single/dual input/output modes,
+handling up to two LVDS input streams and up to two digital CMOS/TTL outputs.
+
+Single or dual operation mode, output data mapping and DDR output modes are
+configured through input signals and the chip does not expose any control bus.
+
+Required properties:
+- compatible: Shall be "thine,thc63lvd1024"
+- vcc-supply: Power supply for TTL output, TTL CLOCKOUT signal, LVDS input,
+  PPL and digital circuitry
+
+Optional properties:
+- powerdown-gpios: Power down GPIO signal, pin name "/PDWN". Active low
+- oe-gpios: Output enable GPIO signal, pin name "OE". Active high
+
+The THC63LVD1024 video port connections are modeled according
+to OF graph bindings specified by Documentation/devicetree/bindings/graph.txt
+
+Required video port nodes:
+- port@0: First LVDS input port
+- port@2: First digital CMOS/TTL parallel output
+
+Optional video port nodes:
+- port@1: Second LVDS input port
+- port@3: Second digital CMOS/TTL parallel output
+
+Example:
+--------
+
+	thc63lvd1024: lvds-decoder {
+		compatible = "thine,thc63lvd1024";
+
+		vcc-supply = <&reg_lvds_vcc>;
+		powerdown-gpios = <&gpio4 15 GPIO_ACTIVE_LOW>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				lvds_dec_in_0: endpoint {
+					remote-endpoint = <&lvds_out>;
+				};
+			};
+
+			port@2{
+				reg = <2>;
+
+				lvds_dec_out_2: endpoint {
+					remote-endpoint = <&adv7511_in>;
+				};
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/display/exynos/exynos5433-decon.txt b/Documentation/devicetree/bindings/display/exynos/exynos5433-decon.txt
index fc2588292a68..775193e1c641 100644
--- a/Documentation/devicetree/bindings/display/exynos/exynos5433-decon.txt
+++ b/Documentation/devicetree/bindings/display/exynos/exynos5433-decon.txt
@@ -19,7 +19,8 @@ Required properties:
 	  clock-names property.
 - clock-names: list of clock names sorted in the same order as the clocks
 	       property. Must contain "pclk", "aclk_decon", "aclk_smmu_decon0x",
-	       "aclk_xiu_decon0x", "pclk_smmu_decon0x", clk_decon_vclk",
+	       "aclk_xiu_decon0x", "pclk_smmu_decon0x", "aclk_smmu_decon1x",
+	       "aclk_xiu_decon1x", "pclk_smmu_decon1x", clk_decon_vclk",
 	       "sclk_decon_eclk"
 - ports: contains a port which is connected to mic node. address-cells and
 	 size-cells must 1 and 0, respectively.
@@ -34,10 +35,14 @@ decon: decon@13800000 {
 	clocks = <&cmu_disp CLK_ACLK_DECON>, <&cmu_disp CLK_ACLK_SMMU_DECON0X>,
 		<&cmu_disp CLK_ACLK_XIU_DECON0X>,
 		<&cmu_disp CLK_PCLK_SMMU_DECON0X>,
+		<&cmu_disp CLK_ACLK_SMMU_DECON1X>,
+		<&cmu_disp CLK_ACLK_XIU_DECON1X>,
+		<&cmu_disp CLK_PCLK_SMMU_DECON1X>,
 		<&cmu_disp CLK_SCLK_DECON_VCLK>,
 		<&cmu_disp CLK_SCLK_DECON_ECLK>;
 	clock-names = "aclk_decon", "aclk_smmu_decon0x", "aclk_xiu_decon0x",
-		"pclk_smmu_decon0x", "sclk_decon_vclk", "sclk_decon_eclk";
+		"pclk_smmu_decon0x", "aclk_smmu_decon1x", "aclk_xiu_decon1x",
+		"pclk_smmu_decon1x", "sclk_decon_vclk", "sclk_decon_eclk";
 	interrupt-names = "vsync", "lcd_sys";
 	interrupts = <0 202 0>, <0 203 0>;
 
diff --git a/Documentation/devicetree/bindings/display/renesas,du.txt b/Documentation/devicetree/bindings/display/renesas,du.txt
index c9cd17f99702..7c6854bd0a04 100644
--- a/Documentation/devicetree/bindings/display/renesas,du.txt
+++ b/Documentation/devicetree/bindings/display/renesas,du.txt
@@ -13,6 +13,7 @@ Required Properties:
     - "renesas,du-r8a7794" for R8A7794 (R-Car E2) compatible DU
     - "renesas,du-r8a7795" for R8A7795 (R-Car H3) compatible DU
     - "renesas,du-r8a7796" for R8A7796 (R-Car M3-W) compatible DU
+    - "renesas,du-r8a77965" for R8A77965 (R-Car M3-N) compatible DU
     - "renesas,du-r8a77970" for R8A77970 (R-Car V3M) compatible DU
     - "renesas,du-r8a77995" for R8A77995 (R-Car D3) compatible DU
 
@@ -47,20 +48,21 @@ bindings specified in Documentation/devicetree/bindings/graph.txt.
 The following table lists for each supported model the port number
 corresponding to each DU output.
 
-                      Port0          Port1          Port2          Port3
+                        Port0          Port1          Port2          Port3
 -----------------------------------------------------------------------------
- R8A7743 (RZ/G1M)     DPAD 0         LVDS 0         -              -
- R8A7745 (RZ/G1E)     DPAD 0         DPAD 1         -              -
- R8A7779 (R-Car H1)   DPAD 0         DPAD 1         -              -
- R8A7790 (R-Car H2)   DPAD 0         LVDS 0         LVDS 1         -
- R8A7791 (R-Car M2-W) DPAD 0         LVDS 0         -              -
- R8A7792 (R-Car V2H)  DPAD 0         DPAD 1         -              -
- R8A7793 (R-Car M2-N) DPAD 0         LVDS 0         -              -
- R8A7794 (R-Car E2)   DPAD 0         DPAD 1         -              -
- R8A7795 (R-Car H3)   DPAD 0         HDMI 0         HDMI 1         LVDS 0
- R8A7796 (R-Car M3-W) DPAD 0         HDMI 0         LVDS 0         -
- R8A77970 (R-Car V3M) DPAD 0         LVDS 0         -              -
- R8A77995 (R-Car D3)  DPAD 0         LVDS 0         LVDS 1         -
+ R8A7743 (RZ/G1M)       DPAD 0         LVDS 0         -              -
+ R8A7745 (RZ/G1E)       DPAD 0         DPAD 1         -              -
+ R8A7779 (R-Car H1)     DPAD 0         DPAD 1         -              -
+ R8A7790 (R-Car H2)     DPAD 0         LVDS 0         LVDS 1         -
+ R8A7791 (R-Car M2-W)   DPAD 0         LVDS 0         -              -
+ R8A7792 (R-Car V2H)    DPAD 0         DPAD 1         -              -
+ R8A7793 (R-Car M2-N)   DPAD 0         LVDS 0         -              -
+ R8A7794 (R-Car E2)     DPAD 0         DPAD 1         -              -
+ R8A7795 (R-Car H3)     DPAD 0         HDMI 0         HDMI 1         LVDS 0
+ R8A7796 (R-Car M3-W)   DPAD 0         HDMI 0         LVDS 0         -
+ R8A77965 (R-Car M3-N)  DPAD 0         HDMI 0         LVDS 0         -
+ R8A77970 (R-Car V3M)   DPAD 0         LVDS 0         -              -
+ R8A77995 (R-Car D3)    DPAD 0         LVDS 0         LVDS 1         -
 
 
 Example: R8A7795 (R-Car H3) ES2.0 DU
diff --git a/Documentation/devicetree/bindings/display/sunxi/sun6i-dsi.txt b/Documentation/devicetree/bindings/display/sunxi/sun6i-dsi.txt
new file mode 100644
index 000000000000..6a6cf5de08b0
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/sunxi/sun6i-dsi.txt
@@ -0,0 +1,93 @@
+Allwinner A31 DSI Encoder
+=========================
+
+The DSI pipeline consists of two separate blocks: the DSI controller
+itself, and its associated D-PHY.
+
+DSI Encoder
+-----------
+
+The DSI Encoder generates the DSI signal from the TCON's.
+
+Required properties:
+  - compatible: value must be one of:
+    * allwinner,sun6i-a31-mipi-dsi
+  - reg: base address and size of memory-mapped region
+  - interrupts: interrupt associated to this IP
+  - clocks: phandles to the clocks feeding the DSI encoder
+    * bus: the DSI interface clock
+    * mod: the DSI module clock
+  - clock-names: the clock names mentioned above
+  - phys: phandle to the D-PHY
+  - phy-names: must be "dphy"
+  - resets: phandle to the reset controller driving the encoder
+
+  - ports: A ports node with endpoint definitions as defined in
+    Documentation/devicetree/bindings/media/video-interfaces.txt. The
+    first port should be the input endpoint, usually coming from the
+    associated TCON.
+
+Any MIPI-DSI device attached to this should be described according to
+the bindings defined in ../mipi-dsi-bus.txt
+
+D-PHY
+-----
+
+Required properties:
+  - compatible: value must be one of:
+    * allwinner,sun6i-a31-mipi-dphy
+  - reg: base address and size of memory-mapped region
+  - clocks: phandles to the clocks feeding the DSI encoder
+    * bus: the DSI interface clock
+    * mod: the DSI module clock
+  - clock-names: the clock names mentioned above
+  - resets: phandle to the reset controller driving the encoder
+
+Example:
+
+dsi0: dsi@1ca0000 {
+	compatible = "allwinner,sun6i-a31-mipi-dsi";
+	reg = <0x01ca0000 0x1000>;
+	interrupts = <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
+	clocks = <&ccu CLK_BUS_MIPI_DSI>,
+		 <&ccu CLK_DSI_SCLK>;
+	clock-names = "bus", "mod";
+	resets = <&ccu RST_BUS_MIPI_DSI>;
+	phys = <&dphy0>;
+	phy-names = "dphy";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	panel@0 {
+		compatible = "bananapi,lhr050h41", "ilitek,ili9881c";
+		reg = <0>;
+		power-gpios = <&pio 1 7 GPIO_ACTIVE_HIGH>; /* PB07 */
+		reset-gpios = <&r_pio 0 5 GPIO_ACTIVE_LOW>; /* PL05 */
+		backlight = <&pwm_bl>;
+	};
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+
+			dsi0_in_tcon0: endpoint {
+				remote-endpoint = <&tcon0_out_dsi0>;
+			};
+		};
+	};
+};
+
+dphy0: d-phy@1ca1000 {
+	compatible = "allwinner,sun6i-a31-mipi-dphy";
+	reg = <0x01ca1000 0x1000>;
+	clocks = <&ccu CLK_BUS_MIPI_DSI>,
+		 <&ccu CLK_DSI_DPHY>;
+	clock-names = "bus", "mod";
+	resets = <&ccu RST_BUS_MIPI_DSI>;
+	#phy-cells = <0>;
+};
diff --git a/Documentation/devicetree/bindings/dma/k3dma.txt b/Documentation/devicetree/bindings/dma/k3dma.txt
index 23f8d712c3ce..4945aeac4dc4 100644
--- a/Documentation/devicetree/bindings/dma/k3dma.txt
+++ b/Documentation/devicetree/bindings/dma/k3dma.txt
@@ -23,7 +23,6 @@ Controller:
 			dma-requests = <27>;
 			interrupts = <0 12 4>;
 			clocks = <&pclk>;
-			status = "disable";
 		};
 
 Client:
diff --git a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt
index 61315eaa7660..b1ba639554c0 100644
--- a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt
+++ b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt
@@ -29,6 +29,7 @@ Required Properties:
 		- "renesas,dmac-r8a77965" (R-Car M3-N)
 		- "renesas,dmac-r8a77970" (R-Car V3M)
 		- "renesas,dmac-r8a77980" (R-Car V3H)
+		- "renesas,dmac-r8a77995" (R-Car D3)
 
 - reg: base address and length of the registers block for the DMAC
 
diff --git a/Documentation/devicetree/bindings/dma/renesas,usb-dmac.txt b/Documentation/devicetree/bindings/dma/renesas,usb-dmac.txt
index 9dc935e24e55..482e54362d3e 100644
--- a/Documentation/devicetree/bindings/dma/renesas,usb-dmac.txt
+++ b/Documentation/devicetree/bindings/dma/renesas,usb-dmac.txt
@@ -12,6 +12,8 @@ Required Properties:
 	  - "renesas,r8a7795-usb-dmac" (R-Car H3)
 	  - "renesas,r8a7796-usb-dmac" (R-Car M3-W)
 	  - "renesas,r8a77965-usb-dmac" (R-Car M3-N)
+	  - "renesas,r8a77990-usb-dmac" (R-Car E3)
+	  - "renesas,r8a77995-usb-dmac" (R-Car D3)
 - reg: base address and length of the registers block for the DMAC
 - interrupts: interrupt specifiers for the DMAC, one for each entry in
   interrupt-names.
diff --git a/Documentation/devicetree/bindings/dma/ti-edma.txt b/Documentation/devicetree/bindings/dma/ti-edma.txt
index 66026dcf53e1..3f15f6644527 100644
--- a/Documentation/devicetree/bindings/dma/ti-edma.txt
+++ b/Documentation/devicetree/bindings/dma/ti-edma.txt
@@ -190,7 +190,6 @@ mmc0: mmc@23000000 {
 	power-domains = <&k2g_pds 0xb>;
 	clocks = <&k2g_clks 0xb 1>, <&k2g_clks 0xb 2>;
 	clock-names = "fck", "mmchsdb_fck";
-	status = "disabled";
 };
 
 ------------------------------------------------------------------------------
diff --git a/Documentation/devicetree/bindings/edac/socfpga-eccmgr.txt b/Documentation/devicetree/bindings/edac/socfpga-eccmgr.txt
new file mode 100644
index 000000000000..5626560a6cfd
--- /dev/null
+++ b/Documentation/devicetree/bindings/edac/socfpga-eccmgr.txt
@@ -0,0 +1,268 @@
+Altera SoCFPGA ECC Manager
+This driver uses the EDAC framework to implement the SOCFPGA ECC Manager.
+The ECC Manager counts and corrects single bit errors and counts/handles
+double bit errors which are uncorrectable.
+
+Cyclone5 and Arria5 ECC Manager
+Required Properties:
+- compatible : Should be "altr,socfpga-ecc-manager"
+- #address-cells: must be 1
+- #size-cells: must be 1
+- ranges : standard definition, should translate from local addresses
+
+Subcomponents:
+
+L2 Cache ECC
+Required Properties:
+- compatible : Should be "altr,socfpga-l2-ecc"
+- reg : Address and size for ECC error interrupt clear registers.
+- interrupts : Should be single bit error interrupt, then double bit error
+	interrupt. Note the rising edge type.
+
+On Chip RAM ECC
+Required Properties:
+- compatible : Should be "altr,socfpga-ocram-ecc"
+- reg : Address and size for ECC error interrupt clear registers.
+- iram : phandle to On-Chip RAM definition.
+- interrupts : Should be single bit error interrupt, then double bit error
+	interrupt. Note the rising edge type.
+
+Example:
+
+	eccmgr: eccmgr@ffd08140 {
+		compatible = "altr,socfpga-ecc-manager";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		l2-ecc@ffd08140 {
+			compatible = "altr,socfpga-l2-ecc";
+			reg = <0xffd08140 0x4>;
+			interrupts = <0 36 1>, <0 37 1>;
+		};
+
+		ocram-ecc@ffd08144 {
+			compatible = "altr,socfpga-ocram-ecc";
+			reg = <0xffd08144 0x4>;
+			iram = <&ocram>;
+			interrupts = <0 178 1>, <0 179 1>;
+		};
+	};
+
+Arria10 SoCFPGA ECC Manager
+The Arria10 SoC ECC Manager handles the IRQs for each peripheral
+in a shared register instead of individual IRQs like the Cyclone5
+and Arria5. Therefore the device tree is different as well.
+
+Required Properties:
+- compatible : Should be "altr,socfpga-a10-ecc-manager"
+- altr,sysgr-syscon : phandle to Arria10 System Manager Block
+	containing the ECC manager registers.
+- #address-cells: must be 1
+- #size-cells: must be 1
+- interrupts : Should be single bit error interrupt, then double bit error
+	interrupt.
+- interrupt-controller : boolean indicator that ECC Manager is an interrupt controller
+- #interrupt-cells : must be set to 2.
+- ranges : standard definition, should translate from local addresses
+
+Subcomponents:
+
+L2 Cache ECC
+Required Properties:
+- compatible : Should be "altr,socfpga-a10-l2-ecc"
+- reg : Address and size for ECC error interrupt clear registers.
+- interrupts : Should be single bit error interrupt, then double bit error
+	interrupt, in this order.
+
+On-Chip RAM ECC
+Required Properties:
+- compatible : Should be "altr,socfpga-a10-ocram-ecc"
+- reg        : Address and size for ECC block registers.
+- interrupts : Should be single bit error interrupt, then double bit error
+	interrupt, in this order.
+
+Ethernet FIFO ECC
+Required Properties:
+- compatible      : Should be "altr,socfpga-eth-mac-ecc"
+- reg             : Address and size for ECC block registers.
+- altr,ecc-parent : phandle to parent Ethernet node.
+- interrupts      : Should be single bit error interrupt, then double bit error
+	interrupt, in this order.
+
+NAND FIFO ECC
+Required Properties:
+- compatible      : Should be "altr,socfpga-nand-ecc"
+- reg             : Address and size for ECC block registers.
+- altr,ecc-parent : phandle to parent NAND node.
+- interrupts      : Should be single bit error interrupt, then double bit error
+	interrupt, in this order.
+
+DMA FIFO ECC
+Required Properties:
+- compatible      : Should be "altr,socfpga-dma-ecc"
+- reg             : Address and size for ECC block registers.
+- altr,ecc-parent : phandle to parent DMA node.
+- interrupts      : Should be single bit error interrupt, then double bit error
+	interrupt, in this order.
+
+USB FIFO ECC
+Required Properties:
+- compatible      : Should be "altr,socfpga-usb-ecc"
+- reg             : Address and size for ECC block registers.
+- altr,ecc-parent : phandle to parent USB node.
+- interrupts      : Should be single bit error interrupt, then double bit error
+	interrupt, in this order.
+
+QSPI FIFO ECC
+Required Properties:
+- compatible      : Should be "altr,socfpga-qspi-ecc"
+- reg             : Address and size for ECC block registers.
+- altr,ecc-parent : phandle to parent QSPI node.
+- interrupts      : Should be single bit error interrupt, then double bit error
+	interrupt, in this order.
+
+SDMMC FIFO ECC
+Required Properties:
+- compatible      : Should be "altr,socfpga-sdmmc-ecc"
+- reg             : Address and size for ECC block registers.
+- altr,ecc-parent : phandle to parent SD/MMC node.
+- interrupts      : Should be single bit error interrupt, then double bit error
+	interrupt, in this order for port A, and then single bit error interrupt,
+	then double bit error interrupt in this order for port B.
+
+Example:
+
+	eccmgr: eccmgr@ffd06000 {
+		compatible = "altr,socfpga-a10-ecc-manager";
+		altr,sysmgr-syscon = <&sysmgr>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		interrupts = <0 2 IRQ_TYPE_LEVEL_HIGH>,
+			     <0 0 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ranges;
+
+		l2-ecc@ffd06010 {
+			compatible = "altr,socfpga-a10-l2-ecc";
+			reg = <0xffd06010 0x4>;
+			interrupts = <0 IRQ_TYPE_LEVEL_HIGH>,
+				     <32 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		ocram-ecc@ff8c3000 {
+			compatible = "altr,socfpga-a10-ocram-ecc";
+			reg = <0xff8c3000 0x90>;
+			interrupts = <1 IRQ_TYPE_LEVEL_HIGH>,
+				     <33 IRQ_TYPE_LEVEL_HIGH> ;
+		};
+
+		emac0-rx-ecc@ff8c0800 {
+			compatible = "altr,socfpga-eth-mac-ecc";
+			reg = <0xff8c0800 0x400>;
+			altr,ecc-parent = <&gmac0>;
+			interrupts = <4 IRQ_TYPE_LEVEL_HIGH>,
+				     <36 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		emac0-tx-ecc@ff8c0c00 {
+			compatible = "altr,socfpga-eth-mac-ecc";
+			reg = <0xff8c0c00 0x400>;
+			altr,ecc-parent = <&gmac0>;
+			interrupts = <5 IRQ_TYPE_LEVEL_HIGH>,
+				     <37 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		nand-buf-ecc@ff8c2000 {
+			compatible = "altr,socfpga-nand-ecc";
+			reg = <0xff8c2000 0x400>;
+			altr,ecc-parent = <&nand>;
+			interrupts = <11 IRQ_TYPE_LEVEL_HIGH>,
+				     <43 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		nand-rd-ecc@ff8c2400 {
+			compatible = "altr,socfpga-nand-ecc";
+			reg = <0xff8c2400 0x400>;
+			altr,ecc-parent = <&nand>;
+			interrupts = <13 IRQ_TYPE_LEVEL_HIGH>,
+				     <45 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		nand-wr-ecc@ff8c2800 {
+			compatible = "altr,socfpga-nand-ecc";
+			reg = <0xff8c2800 0x400>;
+			altr,ecc-parent = <&nand>;
+			interrupts = <12 IRQ_TYPE_LEVEL_HIGH>,
+				     <44 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		dma-ecc@ff8c8000 {
+			compatible = "altr,socfpga-dma-ecc";
+			reg = <0xff8c8000 0x400>;
+			altr,ecc-parent = <&pdma>;
+			interrupts = <10 IRQ_TYPE_LEVEL_HIGH>,
+				     <42 IRQ_TYPE_LEVEL_HIGH>;
+
+		usb0-ecc@ff8c8800 {
+			compatible = "altr,socfpga-usb-ecc";
+			reg = <0xff8c8800 0x400>;
+			altr,ecc-parent = <&usb0>;
+			interrupts = <2 IRQ_TYPE_LEVEL_HIGH>,
+				     <34 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		qspi-ecc@ff8c8400 {
+			compatible = "altr,socfpga-qspi-ecc";
+			reg = <0xff8c8400 0x400>;
+			altr,ecc-parent = <&qspi>;
+			interrupts = <14 IRQ_TYPE_LEVEL_HIGH>,
+				     <46 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		sdmmc-ecc@ff8c2c00 {
+			compatible = "altr,socfpga-sdmmc-ecc";
+			reg = <0xff8c2c00 0x400>;
+			altr,ecc-parent = <&mmc>;
+			interrupts = <15 IRQ_TYPE_LEVEL_HIGH>,
+				     <47 IRQ_TYPE_LEVEL_HIGH>,
+				     <16 IRQ_TYPE_LEVEL_HIGH>,
+				     <48 IRQ_TYPE_LEVEL_HIGH>;
+		};
+	};
+
+Stratix10 SoCFPGA ECC Manager
+The Stratix10 SoC ECC Manager handles the IRQs for each peripheral
+in a shared register similar to the Arria10. However, ECC requires
+access to registers that can only be read from Secure Monitor with
+SMC calls. Therefore the device tree is slightly different.
+
+Required Properties:
+- compatible : Should be "altr,socfpga-s10-ecc-manager"
+- interrupts : Should be single bit error interrupt, then double bit error
+	interrupt.
+- interrupt-controller : boolean indicator that ECC Manager is an interrupt controller
+- #interrupt-cells : must be set to 2.
+
+Subcomponents:
+
+SDRAM ECC
+Required Properties:
+- compatible : Should be "altr,sdram-edac-s10"
+- interrupts : Should be single bit error interrupt, then double bit error
+	interrupt, in this order.
+
+Example:
+
+	eccmgr {
+		compatible = "altr,socfpga-s10-ecc-manager";
+		interrupts = <0 15 4>, <0 95 4>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+
+		sdramedac {
+			compatible = "altr,sdram-edac-s10";
+			interrupts = <16 4>, <48 4>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/firmware/qcom,scm.txt b/Documentation/devicetree/bindings/firmware/qcom,scm.txt
index 7b40054be0d8..fcf6979c0b6d 100644
--- a/Documentation/devicetree/bindings/firmware/qcom,scm.txt
+++ b/Documentation/devicetree/bindings/firmware/qcom,scm.txt
@@ -11,9 +11,10 @@ Required properties:
  * "qcom,scm-msm8660" for MSM8660 platforms
  * "qcom,scm-msm8690" for MSM8690 platforms
  * "qcom,scm-msm8996" for MSM8996 platforms
+ * "qcom,scm-ipq4019" for IPQ4019 platforms
  * "qcom,scm" for later processors (MSM8916, APQ8084, MSM8974, etc)
 - clocks: One to three clocks may be required based on compatible.
- * No clock required for "qcom,scm-msm8996"
+ * No clock required for "qcom,scm-msm8996", "qcom,scm-ipq4019"
  * Only core clock required for "qcom,scm-apq8064", "qcom,scm-msm8660", and "qcom,scm-msm8960"
  * Core, iface, and bus clocks required for "qcom,scm"
 - clock-names: Must contain "core" for the core clock, "iface" for the interface
diff --git a/Documentation/devicetree/bindings/fpga/lattice-machxo2-spi.txt b/Documentation/devicetree/bindings/fpga/lattice-machxo2-spi.txt
new file mode 100644
index 000000000000..a8c362eb160c
--- /dev/null
+++ b/Documentation/devicetree/bindings/fpga/lattice-machxo2-spi.txt
@@ -0,0 +1,29 @@
+Lattice MachXO2 Slave SPI FPGA Manager
+
+Lattice MachXO2 FPGAs support a method of loading the bitstream over
+'slave SPI' interface.
+
+See 'MachXO2ProgrammingandConfigurationUsageGuide.pdf' on www.latticesemi.com
+
+Required properties:
+- compatible: should contain "lattice,machxo2-slave-spi"
+- reg: spi chip select of the FPGA
+
+Example for full FPGA configuration:
+
+	fpga-region0 {
+		compatible = "fpga-region";
+		fpga-mgr = <&fpga_mgr_spi>;
+		#address-cells = <0x1>;
+		#size-cells = <0x1>;
+	};
+
+	spi1: spi@2000 {
+        ...
+
+		fpga_mgr_spi: fpga-mgr@0 {
+			compatible = "lattice,machxo2-slave-spi";
+			spi-max-frequency = <8000000>;
+			reg = <0>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/gpio/gpio-pca953x.txt b/Documentation/devicetree/bindings/gpio/gpio-pca953x.txt
index d2a937682836..88f228665507 100644
--- a/Documentation/devicetree/bindings/gpio/gpio-pca953x.txt
+++ b/Documentation/devicetree/bindings/gpio/gpio-pca953x.txt
@@ -31,10 +31,15 @@ Required properties:
 	ti,tca9554
 	onnn,pca9654
 	exar,xra1202
+ - gpio-controller: if used as gpio expander.
+ - #gpio-cells: if used as gpio expander.
+ - interrupt-controller: if to be used as interrupt expander.
+ - #interrupt-cells: if to be used as interrupt expander.
 
 Optional properties:
  - reset-gpios: GPIO specification for the RESET input. This is an
 		active low signal to the PCA953x.
+ - vcc-supply:	power supply regulator.
 
 Example:
 
@@ -47,3 +52,32 @@ Example:
 		interrupt-parent = <&gpio3>;
 		interrupts = <23 IRQ_TYPE_LEVEL_LOW>;
 	};
+
+
+Example with Interrupts:
+
+
+	gpio99: gpio@22 {
+		compatible = "nxp,pcal6524";
+		reg = <0x22>;
+		interrupt-parent = <&gpio6>;
+		interrupts = <1 IRQ_TYPE_EDGE_FALLING>;	/* gpio6_161 */
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		vcc-supply = <&vdds_1v8_main>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-line-names =
+			"hdmi-ct-hpd", "hdmi.ls-oe", "p02", "p03", "vibra", "fault2", "p06", "p07",
+			"en-usb", "en-host1", "en-host2", "chg-int", "p14", "p15", "mic-int", "en-modem",
+			"shdn-hs-amp", "chg-status+red", "green", "blue", "en-esata", "fault1", "p26", "p27";
+	};
+
+	ts3a227@3b {
+		compatible = "ti,ts3a227e";
+		reg = <0x3b>;
+		interrupt-parent = <&gpio99>;
+		interrupts = <14 IRQ_TYPE_EDGE_RISING>;
+		ti,micbias = <0>;	/* 2.1V */
+	};
+
diff --git a/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt b/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt
index 9474138d776e..378f1322211e 100644
--- a/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt
+++ b/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt
@@ -5,6 +5,7 @@ Required Properties:
   - compatible: should contain one or more of the following:
     - "renesas,gpio-r8a7743": for R8A7743 (RZ/G1M) compatible GPIO controller.
     - "renesas,gpio-r8a7745": for R8A7745 (RZ/G1E) compatible GPIO controller.
+    - "renesas,gpio-r8a77470": for R8A77470 (RZ/G1C) compatible GPIO controller.
     - "renesas,gpio-r8a7778": for R8A7778 (R-Car M1) compatible GPIO controller.
     - "renesas,gpio-r8a7779": for R8A7779 (R-Car H1) compatible GPIO controller.
     - "renesas,gpio-r8a7790": for R8A7790 (R-Car H2) compatible GPIO controller.
@@ -14,7 +15,9 @@ Required Properties:
     - "renesas,gpio-r8a7794": for R8A7794 (R-Car E2) compatible GPIO controller.
     - "renesas,gpio-r8a7795": for R8A7795 (R-Car H3) compatible GPIO controller.
     - "renesas,gpio-r8a7796": for R8A7796 (R-Car M3-W) compatible GPIO controller.
+    - "renesas,gpio-r8a77965": for R8A77965 (R-Car M3-N) compatible GPIO controller.
     - "renesas,gpio-r8a77970": for R8A77970 (R-Car V3M) compatible GPIO controller.
+    - "renesas,gpio-r8a77990": for R8A77990 (R-Car E3) compatible GPIO controller.
     - "renesas,gpio-r8a77995": for R8A77995 (R-Car D3) compatible GPIO controller.
     - "renesas,rcar-gen1-gpio": for a generic R-Car Gen1 GPIO controller.
     - "renesas,rcar-gen2-gpio": for a generic R-Car Gen2 or RZ/G1 GPIO controller.
diff --git a/Documentation/devicetree/bindings/gpio/snps-dwapb-gpio.txt b/Documentation/devicetree/bindings/gpio/snps-dwapb-gpio.txt
index 4a75da7051bd..3c1118bc67f5 100644
--- a/Documentation/devicetree/bindings/gpio/snps-dwapb-gpio.txt
+++ b/Documentation/devicetree/bindings/gpio/snps-dwapb-gpio.txt
@@ -26,8 +26,13 @@ controller.
   the second encodes the triger flags encoded as described in
   Documentation/devicetree/bindings/interrupt-controller/interrupts.txt
 - interrupt-parent : The parent interrupt controller.
-- interrupts : The interrupt to the parent controller raised when GPIOs
-  generate the interrupts.
+- interrupts : The interrupts to the parent controller raised when GPIOs
+  generate the interrupts. If the controller provides one combined interrupt
+  for all GPIOs, specify a single interrupt. If the controller provides one
+  interrupt for each GPIO, provide a list of interrupts that correspond to each
+  of the GPIO pins. When specifying multiple interrupts, if any are unconnected,
+  use the interrupts-extended property to specify the interrupts and set the
+  interrupt controller handle for unused interrupts to 0.
 - snps,nr-gpios : The number of pins in the port, a single cell.
 - resets : Reset line for the controller.
 
diff --git a/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt
new file mode 100644
index 000000000000..c907aa8dd755
--- /dev/null
+++ b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt
@@ -0,0 +1,28 @@
+Broadcom V3D GPU
+
+Only the Broadcom V3D 3.x and newer GPUs are covered by this binding.
+For V3D 2.x, see brcm,bcm-vc4.txt.
+
+Required properties:
+- compatible:	Should be "brcm,7268-v3d" or "brcm,7278-v3d"
+- reg:		Physical base addresses and lengths of the register areas
+- reg-names:	Names for the register areas.  The "hub", "bridge", and "core0"
+		  register areas are always required.  The "gca" register area
+		  is required if the GCA cache controller is present.
+- interrupts:	The interrupt numbers.  The first interrupt is for the hub,
+		  while the following interrupts are for the cores.
+		  See bindings/interrupt-controller/interrupts.txt
+
+Optional properties:
+- clocks:	The core clock the unit runs on
+
+v3d {
+	compatible = "brcm,7268-v3d";
+	reg = <0xf1204000 0x100>,
+	      <0xf1200000 0x4000>,
+	      <0xf1208000 0x4000>,
+	      <0xf1204100 0x100>;
+	reg-names = "bridge", "hub", "core0", "gca";
+	interrupts = <0 78 4>,
+		     <0 77 4>;
+};
diff --git a/Documentation/devicetree/bindings/gpu/samsung-scaler.txt b/Documentation/devicetree/bindings/gpu/samsung-scaler.txt
new file mode 100644
index 000000000000..9c3d98105dfd
--- /dev/null
+++ b/Documentation/devicetree/bindings/gpu/samsung-scaler.txt
@@ -0,0 +1,27 @@
+* Samsung Exynos Image Scaler
+
+Required properties:
+  - compatible : value should be one of the following:
+	(a) "samsung,exynos5420-scaler" for Scaler IP in Exynos5420
+	(b) "samsung,exynos5433-scaler" for Scaler IP in Exynos5433
+
+  - reg : Physical base address of the IP registers and length of memory
+	  mapped region.
+
+  - interrupts : Interrupt specifier for scaler interrupt, according to format
+		 specific to interrupt parent.
+
+  - clocks : Clock specifier for scaler clock, according to generic clock
+	     bindings. (See Documentation/devicetree/bindings/clock/exynos*.txt)
+
+  - clock-names : Names of clocks. For exynos scaler, it should be "mscl"
+		  on 5420 and "pclk", "aclk" and "aclk_xiu" on 5433.
+
+Example:
+	scaler@12800000 {
+		compatible = "samsung,exynos5420-scaler";
+		reg = <0x12800000 0x1294>;
+		interrupts = <0 220 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&clock CLK_MSCL0>;
+		clock-names = "mscl";
+	};
diff --git a/Documentation/devicetree/bindings/hwmon/gpio-fan.txt b/Documentation/devicetree/bindings/hwmon/gpio-fan.txt
index 439a7430fc68..2becdcfdc840 100644
--- a/Documentation/devicetree/bindings/hwmon/gpio-fan.txt
+++ b/Documentation/devicetree/bindings/hwmon/gpio-fan.txt
@@ -11,7 +11,7 @@ Optional properties:
   must have the RPM values in ascending order.
 - alarm-gpios: This pin going active indicates something is wrong with
   the fan, and a udev event will be fired.
-- cooling-cells: If used as a cooling device, must be <2>
+- #cooling-cells: If used as a cooling device, must be <2>
   Also see: Documentation/devicetree/bindings/thermal/thermal.txt
   min and max states are derived from the speed-map of the fan.
 
diff --git a/Documentation/devicetree/bindings/hwmon/ltc2990.txt b/Documentation/devicetree/bindings/hwmon/ltc2990.txt
new file mode 100644
index 000000000000..f92f54029e84
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/ltc2990.txt
@@ -0,0 +1,36 @@
+ltc2990: Linear Technology LTC2990 power monitor
+
+Required properties:
+- compatible: Must be "lltc,ltc2990"
+- reg: I2C slave address
+- lltc,meas-mode:
+	An array of two integers for configuring the chip measurement mode.
+
+	The first integer defines the bits 2..0 in the control register. In all
+	cases the internal temperature and supply voltage are measured. In
+	addition the following input measurements are enabled per mode:
+
+		0: V1, V2, TR2
+		1: V1-V2, TR2
+		2: V1-V2, V3, V4
+		3: TR1, V3, V4
+		4: TR1, V3-V4
+		5: TR1, TR2
+		6: V1-V2, V3-V4
+		7: V1, V2, V3, V4
+
+	The second integer defines the bits 4..3 in the control register. This
+	allows a subset of the measurements to be enabled:
+
+		0: Internal temperature and supply voltage only
+		1: TR1, V1 or V1-V2 only per mode
+		2: TR2, V3 or V3-V4 only per mode
+		3: All measurements per mode
+
+Example:
+
+ltc2990@4c {
+	compatible = "lltc,ltc2990";
+	reg = <0x4c>;
+	lltc,meas-mode = <7 3>;	/* V1, V2, V3, V4 */
+};
diff --git a/Documentation/devicetree/bindings/i2c/i2c-davinci.txt b/Documentation/devicetree/bindings/i2c/i2c-davinci.txt
index 64e6e656c345..b745f3706120 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-davinci.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-davinci.txt
@@ -24,7 +24,7 @@ Recommended properties :
 - clock-frequency : desired I2C bus clock frequency in Hz.
 - ti,has-pfunc: boolean; if defined, it indicates that SoC supports PFUNC
 	registers. PFUNC registers allow to switch I2C pins to function as
-	GPIOs, so they can by toggled manually.
+	GPIOs, so they can be toggled manually.
 
 Example (enbw_cmc board):
 	i2c@1c22000 {
diff --git a/Documentation/devicetree/bindings/i2c/i2c-rcar.txt b/Documentation/devicetree/bindings/i2c/i2c-rcar.txt
index 4a7811ecd954..7ce8fae55537 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-rcar.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-rcar.txt
@@ -15,6 +15,7 @@ Required properties:
 	"renesas,i2c-r8a7796" if the device is a part of a R8A7796 SoC.
 	"renesas,i2c-r8a77965" if the device is a part of a R8A77965 SoC.
 	"renesas,i2c-r8a77970" if the device is a part of a R8A77970 SoC.
+	"renesas,i2c-r8a77980" if the device is a part of a R8A77980 SoC.
 	"renesas,i2c-r8a77995" if the device is a part of a R8A77995 SoC.
 	"renesas,rcar-gen1-i2c" for a generic R-Car Gen1 compatible device.
 	"renesas,rcar-gen2-i2c" for a generic R-Car Gen2 or RZ/G1 compatible
diff --git a/Documentation/devicetree/bindings/i2c/i2c-s3c2410.txt b/Documentation/devicetree/bindings/i2c/i2c-s3c2410.txt
index 89b3250f049b..66ae46d3bc2f 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-s3c2410.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-s3c2410.txt
@@ -8,9 +8,7 @@ Required properties:
       (b) "samsung, s3c2440-i2c", for i2c compatible with s3c2440 i2c.
       (c) "samsung, s3c2440-hdmiphy-i2c", for s3c2440-like i2c used
           inside HDMIPHY block found on several samsung SoCs
-      (d) "samsung, exynos5440-i2c", for s3c2440-like i2c used
-	   on EXYNOS5440 which does not need GPIO configuration.
-      (e) "samsung, exynos5-sata-phy-i2c", for s3c2440-like i2c used as
+      (d) "samsung, exynos5-sata-phy-i2c", for s3c2440-like i2c used as
 	   a host to SATA PHY controller on an internal bus.
   - reg: physical base address of the controller and length of memory mapped
     region.
diff --git a/Documentation/devicetree/bindings/iio/adc/amlogic,meson-saradc.txt b/Documentation/devicetree/bindings/iio/adc/amlogic,meson-saradc.txt
index 1e6ee3deb4fa..d1acd5ea2737 100644
--- a/Documentation/devicetree/bindings/iio/adc/amlogic,meson-saradc.txt
+++ b/Documentation/devicetree/bindings/iio/adc/amlogic,meson-saradc.txt
@@ -7,6 +7,7 @@ Required properties:
 			- "amlogic,meson-gxbb-saradc" for GXBB
 			- "amlogic,meson-gxl-saradc" for GXL
 			- "amlogic,meson-gxm-saradc" for GXM
+			- "amlogic,meson-axg-saradc" for AXG
 		along with the generic "amlogic,meson-saradc"
 - reg:		the physical base address and length of the registers
 - interrupts:	the interrupt indicating end of sampling
diff --git a/Documentation/devicetree/bindings/iio/adc/mcp320x.txt b/Documentation/devicetree/bindings/iio/adc/mcp320x.txt
index 7d64753df949..56373d643f76 100644
--- a/Documentation/devicetree/bindings/iio/adc/mcp320x.txt
+++ b/Documentation/devicetree/bindings/iio/adc/mcp320x.txt
@@ -49,7 +49,7 @@ Required properties:
 Examples:
 spi_controller {
 	mcp3x0x@0 {
-		compatible = "mcp3002";
+		compatible = "microchip,mcp3002";
 		reg = <0>;
 		spi-max-frequency = <1000000>;
 		vref-supply = <&vref_reg>;
diff --git a/Documentation/devicetree/bindings/iio/adc/samsung,exynos-adc.txt b/Documentation/devicetree/bindings/iio/adc/samsung,exynos-adc.txt
new file mode 100644
index 000000000000..6c49db7f8ad2
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/samsung,exynos-adc.txt
@@ -0,0 +1,103 @@
+Samsung Exynos Analog to Digital Converter bindings
+
+The devicetree bindings are for the new ADC driver written for
+Exynos4 and upward SoCs from Samsung.
+
+New driver handles the following
+1. Supports ADC IF found on EXYNOS4412/EXYNOS5250
+   and future SoCs from Samsung
+2. Add ADC driver under iio/adc framework
+3. Also adds the Documentation for device tree bindings
+
+Required properties:
+- compatible:		Must be "samsung,exynos-adc-v1"
+				for exynos4412/5250 and s5pv210 controllers.
+			Must be "samsung,exynos-adc-v2" for
+				future controllers.
+			Must be "samsung,exynos3250-adc" for
+				controllers compatible with ADC of Exynos3250.
+			Must be "samsung,exynos7-adc" for
+				the ADC in Exynos7 and compatibles
+			Must be "samsung,s3c2410-adc" for
+				the ADC in s3c2410 and compatibles
+			Must be "samsung,s3c2416-adc" for
+				the ADC in s3c2416 and compatibles
+			Must be "samsung,s3c2440-adc" for
+				the ADC in s3c2440 and compatibles
+			Must be "samsung,s3c2443-adc" for
+				the ADC in s3c2443 and compatibles
+			Must be "samsung,s3c6410-adc" for
+				the ADC in s3c6410 and compatibles
+- reg:			List of ADC register address range
+			- The base address and range of ADC register
+			- The base address and range of ADC_PHY register (every
+			  SoC except for s3c24xx/s3c64xx ADC)
+- interrupts: 		Contains the interrupt information for the timer. The
+			format is being dependent on which interrupt controller
+			the Samsung device uses.
+- #io-channel-cells = <1>; As ADC has multiple outputs
+- clocks		From common clock bindings: handles to clocks specified
+			in "clock-names" property, in the same order.
+- clock-names		From common clock bindings: list of clock input names
+			used by ADC block:
+			- "adc" : ADC bus clock
+			- "sclk" : ADC special clock (only for Exynos3250 and
+				   compatible ADC block)
+- vdd-supply		VDD input supply.
+
+- samsung,syscon-phandle Contains the PMU system controller node
+			(To access the ADC_PHY register on Exynos5250/5420/5800/3250)
+Optional properties:
+- has-touchscreen:	If present, indicates that a touchscreen is
+			connected an usable.
+
+Note: child nodes can be added for auto probing from device tree.
+
+Example: adding device info in dtsi file
+
+adc: adc@12d10000 {
+	compatible = "samsung,exynos-adc-v1";
+	reg = <0x12D10000 0x100>;
+	interrupts = <0 106 0>;
+	#io-channel-cells = <1>;
+	io-channel-ranges;
+
+	clocks = <&clock 303>;
+	clock-names = "adc";
+
+	vdd-supply = <&buck5_reg>;
+	samsung,syscon-phandle = <&pmu_system_controller>;
+};
+
+Example: adding device info in dtsi file for Exynos3250 with additional sclk
+
+adc: adc@126c0000 {
+	compatible = "samsung,exynos3250-adc", "samsung,exynos-adc-v2;
+	reg = <0x126C0000 0x100>;
+	interrupts = <0 137 0>;
+	#io-channel-cells = <1>;
+	io-channel-ranges;
+
+	clocks = <&cmu CLK_TSADC>, <&cmu CLK_SCLK_TSADC>;
+	clock-names = "adc", "sclk";
+
+	vdd-supply = <&buck5_reg>;
+	samsung,syscon-phandle = <&pmu_system_controller>;
+};
+
+Example: Adding child nodes in dts file
+
+adc@12d10000 {
+
+	/* NTC thermistor is a hwmon device */
+	ncp15wb473@0 {
+		compatible = "murata,ncp15wb473";
+		pullup-uv = <1800000>;
+		pullup-ohm = <47000>;
+		pulldown-ohm = <0>;
+		io-channels = <&adc 4>;
+	};
+};
+
+Note: Does not apply to ADC driver under arch/arm/plat-samsung/
+Note: The child node can be added under the adc node or separately.
diff --git a/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.txt b/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.txt
index e8bb8243e92c..f1ead43a1a95 100644
--- a/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.txt
+++ b/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.txt
@@ -24,8 +24,11 @@ Required properties:
 - compatible: Should be one of:
   "st,stm32f4-adc-core"
   "st,stm32h7-adc-core"
+  "st,stm32mp1-adc-core"
 - reg: Offset and length of the ADC block register set.
-- interrupts: Must contain the interrupt for ADC block.
+- interrupts: One or more interrupts for ADC block. Some parts like stm32f4
+  and stm32h7 share a common ADC interrupt line. stm32mp1 has two separate
+  interrupt lines, one for each ADC within ADC block.
 - clocks: Core can use up to two clocks, depending on part used:
   - "adc" clock: for the analog circuitry, common to all ADCs.
     It's required on stm32f4.
@@ -53,6 +56,7 @@ Required properties:
 - compatible: Should be one of:
   "st,stm32f4-adc"
   "st,stm32h7-adc"
+  "st,stm32mp1-adc"
 - reg: Offset of ADC instance in ADC block (e.g. may be 0x0, 0x100, 0x200).
 - clocks: Input clock private to this ADC instance. It's required only on
   stm32f4, that has per instance clock input for registers access.
diff --git a/Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.txt b/Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.txt
index ed7520d1d051..75ba25d062e1 100644
--- a/Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.txt
+++ b/Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.txt
@@ -8,14 +8,16 @@ It is mainly targeted for:
 - PDM microphones (audio digital microphone)
 
 It features up to 8 serial digital interfaces (SPI or Manchester) and
-up to 4 filters on stm32h7.
+up to 4 filters on stm32h7 or 6 filters on stm32mp1.
 
 Each child node match with a filter instance.
 
 Contents of a STM32 DFSDM root node:
 ------------------------------------
 Required properties:
-- compatible: Should be "st,stm32h7-dfsdm".
+- compatible: Should be one of:
+  "st,stm32h7-dfsdm"
+  "st,stm32mp1-dfsdm"
 - reg: Offset and length of the DFSDM block register set.
 - clocks: IP and serial interfaces clocking. Should be set according
 		to rcc clock ID and "clock-names".
@@ -45,6 +47,7 @@ Required properties:
 	"st,stm32-dfsdm-adc" for sigma delta ADCs
 	"st,stm32-dfsdm-dmic" for audio digital microphone.
 - reg: Specifies the DFSDM filter instance used.
+	Valid values are from 0 to 3 on stm32h7, 0 to 5 on stm32mp1.
 - interrupts: IRQ lines connected to each DFSDM filter instance.
 - st,adc-channels:	List of single-ended channels muxed for this ADC.
 			valid values:
diff --git a/Documentation/devicetree/bindings/iio/afe/current-sense-amplifier.txt b/Documentation/devicetree/bindings/iio/afe/current-sense-amplifier.txt
new file mode 100644
index 000000000000..821b61b8c542
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/afe/current-sense-amplifier.txt
@@ -0,0 +1,26 @@
+Current Sense Amplifier
+=======================
+
+When an io-channel measures the output voltage from a current sense
+amplifier, the interesting measurement is almost always the current
+through the sense resistor, not the voltage output. This binding
+describes such a current sense circuit.
+
+Required properties:
+- compatible : "current-sense-amplifier"
+- io-channels : Channel node of a voltage io-channel.
+- sense-resistor-micro-ohms : The sense resistance in microohms.
+
+Optional properties:
+- sense-gain-mult: Amplifier gain multiplier. The default is <1>.
+- sense-gain-div: Amplifier gain divider. The default is <1>.
+
+Example:
+
+sysi {
+	compatible = "current-sense-amplifier";
+	io-channels = <&tiadc 0>;
+
+	sense-resistor-micro-ohms = <20000>;
+	sense-gain-mul = <50>;
+};
diff --git a/Documentation/devicetree/bindings/iio/afe/current-sense-shunt.txt b/Documentation/devicetree/bindings/iio/afe/current-sense-shunt.txt
new file mode 100644
index 000000000000..0f67108a07b6
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/afe/current-sense-shunt.txt
@@ -0,0 +1,41 @@
+Current Sense Shunt
+===================
+
+When an io-channel measures the voltage over a current sense shunt,
+the interesting measurement is almost always the current through the
+shunt, not the voltage over it. This binding describes such a current
+sense circuit.
+
+Required properties:
+- compatible : "current-sense-shunt"
+- io-channels : Channel node of a voltage io-channel.
+- shunt-resistor-micro-ohms : The shunt resistance in microohms.
+
+Example:
+The system current is measured by measuring the voltage over a
+3.3 ohms shunt resistor.
+
+sysi {
+	compatible = "current-sense-shunt";
+	io-channels = <&tiadc 0>;
+
+	/* Divide the voltage by 3300000/1000000 (or 3.3) for the current. */
+	shunt-resistor-micro-ohms = <3300000>;
+};
+
+&i2c {
+	tiadc: adc@48 {
+		compatible = "ti,ads1015";
+		reg = <0x48>;
+		#io-channel-cells = <1>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		channel@0 { /* IN0,IN1 differential */
+			reg = <0>;
+			ti,gain = <1>;
+			ti,datarate = <4>;
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/iio/afe/voltage-divider.txt b/Documentation/devicetree/bindings/iio/afe/voltage-divider.txt
new file mode 100644
index 000000000000..b452a8406107
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/afe/voltage-divider.txt
@@ -0,0 +1,53 @@
+Voltage divider
+===============
+
+When an io-channel measures the midpoint of a voltage divider, the
+interesting voltage is often the voltage over the full resistance
+of the divider. This binding describes the voltage divider in such
+a curcuit.
+
+    Vin ----.
+            |
+         .-----.
+         |  R  |
+         '-----'
+            |
+            +---- Vout
+            |
+         .-----.
+         | Rout|
+         '-----'
+            |
+           GND
+
+Required properties:
+- compatible : "voltage-divider"
+- io-channels : Channel node of a voltage io-channel measuring Vout.
+- output-ohms : Resistance Rout over which the output voltage is measured.
+	        See full-ohms.
+- full-ohms : Resistance R + Rout for the full divider. The io-channel
+	      is scaled by the Rout / (R + Rout) quotient.
+
+Example:
+The system voltage is circa 12V, but divided down with a 22/222
+voltage divider (R = 200 Ohms, Rout = 22 Ohms) and fed to an ADC.
+
+sysv {
+	compatible = "voltage-divider";
+	io-channels = <&maxadc 1>;
+
+	/* Scale the system voltage by 22/222 to fit the ADC range. */
+	output-ohms = <22>;
+	full-ohms = <222>; /* 200 + 22 */
+};
+
+&spi {
+	maxadc: adc@0 {
+		compatible = "maxim,max1027";
+		reg = <0>;
+		#io-channel-cells = <1>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <15 IRQ_TYPE_EDGE_RISING>;
+		spi-max-frequency = <1000000>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/iio/dac/ltc2632.txt b/Documentation/devicetree/bindings/iio/dac/ltc2632.txt
index eb911e5a8ab4..e0d5fea33031 100644
--- a/Documentation/devicetree/bindings/iio/dac/ltc2632.txt
+++ b/Documentation/devicetree/bindings/iio/dac/ltc2632.txt
@@ -12,12 +12,26 @@ Required properties:
 Property rules described in Documentation/devicetree/bindings/spi/spi-bus.txt
 apply. In particular, "reg" and "spi-max-frequency" properties must be given.
 
+Optional properties:
+	- vref-supply: Phandle to the external reference voltage supply. This should
+	  only be set if there is an external reference voltage connected to the VREF
+	  pin. If the property is not set the internal reference is used.
+
 Example:
 
+	vref: regulator-vref {
+		compatible = "regulator-fixed";
+		regulator-name = "vref-ltc2632";
+		regulator-min-microvolt = <1250000>;
+		regulator-max-microvolt = <1250000>;
+		regulator-always-on;
+	};
+
 	spi_master {
 		dac: ltc2632@0 {
 			compatible = "lltc,ltc2632-l12";
 			reg = <0>; /* CS0 */
 			spi-max-frequency = <1000000>;
+			vref-supply = <&vref>; /* optional */
 		};
 	};
diff --git a/Documentation/devicetree/bindings/iio/dac/ti,dac5571.txt b/Documentation/devicetree/bindings/iio/dac/ti,dac5571.txt
new file mode 100644
index 000000000000..03af6b9a4d07
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/dac/ti,dac5571.txt
@@ -0,0 +1,24 @@
+* Texas Instruments DAC5571 Family
+
+Required properties:
+ - compatible: Should contain
+    "ti,dac5571"
+    "ti,dac6571"
+    "ti,dac7571"
+    "ti,dac5574"
+    "ti,dac6574"
+    "ti,dac7574"
+    "ti,dac5573"
+    "ti,dac6573"
+    "ti,dac7573"
+ - reg: Should contain the DAC I2C address
+
+Optional properties:
+ - vref-supply: The regulator supply for DAC reference voltage
+
+Example:
+dac@0 {
+	compatible = "ti,dac5571";
+	reg = <0x4C>;
+	vref-supply = <&vdd_supply>;
+};
diff --git a/Documentation/devicetree/bindings/iio/imu/inv_mpu6050.txt b/Documentation/devicetree/bindings/iio/imu/inv_mpu6050.txt
index 2b4514592f83..5f4777e8cc9e 100644
--- a/Documentation/devicetree/bindings/iio/imu/inv_mpu6050.txt
+++ b/Documentation/devicetree/bindings/iio/imu/inv_mpu6050.txt
@@ -8,10 +8,16 @@ Required properties:
  		"invensense,mpu6500"
 		"invensense,mpu9150"
 		"invensense,mpu9250"
+		"invensense,mpu9255"
 		"invensense,icm20608"
  - reg : the I2C address of the sensor
  - interrupt-parent : should be the phandle for the interrupt controller
- - interrupts : interrupt mapping for GPIO IRQ
+ - interrupts: interrupt mapping for IRQ. It should be configured with flags
+   IRQ_TYPE_LEVEL_HIGH, IRQ_TYPE_EDGE_RISING, IRQ_TYPE_LEVEL_LOW or
+   IRQ_TYPE_EDGE_FALLING.
+
+  Refer to interrupt-controller/interrupts.txt for generic interrupt client node
+  bindings.
 
 Optional properties:
  - mount-matrix: an optional 3x3 mounting rotation matrix
@@ -24,7 +30,7 @@ Example:
 		compatible = "invensense,mpu6050";
 		reg = <0x68>;
 		interrupt-parent = <&gpio1>;
-		interrupts = <18 1>;
+		interrupts = <18 IRQ_TYPE_EDGE_RISING>;
 		mount-matrix = "-0.984807753012208",  /* x0 */
 		               "0",                   /* y0 */
 		               "-0.173648177666930",  /* z0 */
@@ -41,7 +47,7 @@ Example:
 		compatible = "invensense,mpu9250";
 		reg = <0x68>;
 		interrupt-parent = <&gpio3>;
-		interrupts = <21 1>;
+		interrupts = <21 IRQ_TYPE_LEVEL_HIGH>;
 		i2c-gate {
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/Documentation/devicetree/bindings/iio/imu/st_lsm6dsx.txt b/Documentation/devicetree/bindings/iio/imu/st_lsm6dsx.txt
index 1ff1af799c76..ef8a8566c63f 100644
--- a/Documentation/devicetree/bindings/iio/imu/st_lsm6dsx.txt
+++ b/Documentation/devicetree/bindings/iio/imu/st_lsm6dsx.txt
@@ -6,6 +6,7 @@ Required properties:
   "st,lsm6ds3h"
   "st,lsm6dsl"
   "st,lsm6dsm"
+  "st,ism330dlc"
 - reg: i2c address of the sensor / spi cs line
 
 Optional properties:
diff --git a/Documentation/devicetree/bindings/iio/potentiostat/lmp91000.txt b/Documentation/devicetree/bindings/iio/potentiostat/lmp91000.txt
index b9b621e94cd7..e6d0c2eb345c 100644
--- a/Documentation/devicetree/bindings/iio/potentiostat/lmp91000.txt
+++ b/Documentation/devicetree/bindings/iio/potentiostat/lmp91000.txt
@@ -1,10 +1,13 @@
-* Texas Instruments LMP91000 potentiostat
+* Texas Instruments LMP91000 series of potentiostats
 
-http://www.ti.com/lit/ds/symlink/lmp91000.pdf
+LMP91000: http://www.ti.com/lit/ds/symlink/lmp91000.pdf
+LMP91002: http://www.ti.com/lit/ds/symlink/lmp91002.pdf
 
 Required properties:
 
-  - compatible: should be "ti,lmp91000"
+  - compatible: should be one of the following:
+                 "ti,lmp91000"
+                 "ti,lmp91002"
   - reg: the I2C address of the device
   - io-channels: the phandle of the iio provider
 
diff --git a/Documentation/devicetree/bindings/input/elan_i2c.txt b/Documentation/devicetree/bindings/input/elan_i2c.txt
index ee3242c4ba67..d80a83583238 100644
--- a/Documentation/devicetree/bindings/input/elan_i2c.txt
+++ b/Documentation/devicetree/bindings/input/elan_i2c.txt
@@ -14,6 +14,7 @@ Optional properties:
 - pinctrl-0: a phandle pointing to the pin settings for the device (see
   pinctrl binding [1]).
 - vcc-supply: a phandle for the regulator supplying 3.3V power.
+- elan,trackpoint: touchpad can support a trackpoint (boolean)
 
 [0]: Documentation/devicetree/bindings/interrupt-controller/interrupts.txt
 [1]: Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt
diff --git a/Documentation/devicetree/bindings/input/mtk-pmic-keys.txt b/Documentation/devicetree/bindings/input/mtk-pmic-keys.txt
new file mode 100644
index 000000000000..2888d07c2ef0
--- /dev/null
+++ b/Documentation/devicetree/bindings/input/mtk-pmic-keys.txt
@@ -0,0 +1,43 @@
+MediaTek MT6397/MT6323 PMIC Keys Device Driver
+
+There are two key functions provided by MT6397/MT6323 PMIC, pwrkey
+and homekey. The key functions are defined as the subnode of the function
+node provided by MT6397/MT6323 PMIC that is being defined as one kind
+of Muti-Function Device (MFD)
+
+For MT6397/MT6323 MFD bindings see:
+Documentation/devicetree/bindings/mfd/mt6397.txt
+
+Required properties:
+- compatible: "mediatek,mt6397-keys" or "mediatek,mt6323-keys"
+- linux,keycodes: See Documentation/devicetree/bindings/input/keys.txt
+
+Optional Properties:
+- wakeup-source: See Documentation/devicetree/bindings/power/wakeup-source.txt
+- mediatek,long-press-mode: Long press key shutdown setting, 1 for
+	pwrkey only, 2 for pwrkey/homekey together, others for disabled.
+- power-off-time-sec: See Documentation/devicetree/bindings/input/keys.txt
+
+Example:
+
+	pmic: mt6397 {
+		compatible = "mediatek,mt6397";
+
+		...
+
+		mt6397keys: mt6397keys {
+			compatible = "mediatek,mt6397-keys";
+			mediatek,long-press-mode = <1>;
+			power-off-time-sec = <0>;
+
+			power {
+				linux,keycodes = <116>;
+				wakeup-source;
+			};
+
+			home {
+				linux,keycodes = <114>;
+			};
+		};
+
+	};
diff --git a/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.txt b/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.txt
index a83f9a5734ca..89674ad8a097 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.txt
+++ b/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.txt
@@ -9,11 +9,12 @@ number of interrupt exposed depends on the SoC.
 
 Required properties:
 
-- compatible : must have "amlogic,meson8-gpio-intc” and either
-   “amlogic,meson8-gpio-intc” for meson8 SoCs (S802) or
-   “amlogic,meson8b-gpio-intc” for meson8b SoCs (S805) or
-   “amlogic,meson-gxbb-gpio-intc” for GXBB SoCs (S905) or
-   “amlogic,meson-gxl-gpio-intc” for GXL SoCs (S905X, S912)
+- compatible : must have "amlogic,meson8-gpio-intc" and either
+    "amlogic,meson8-gpio-intc" for meson8 SoCs (S802) or
+    "amlogic,meson8b-gpio-intc" for meson8b SoCs (S805) or
+    "amlogic,meson-gxbb-gpio-intc" for GXBB SoCs (S905) or
+    "amlogic,meson-gxl-gpio-intc" for GXL SoCs (S905X, S912)
+    "amlogic,meson-axg-gpio-intc" for AXG SoCs (A113D, A113X)
 - interrupt-parent : a phandle to the GIC the interrupts are routed to.
    Usually this is provided at the root level of the device tree as it is
    common to most of the SoC.
diff --git a/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.txt b/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.txt
index 0a57f2f4167d..3ea78c4ef887 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.txt
+++ b/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.txt
@@ -57,6 +57,20 @@ Optional
   occupied by the redistributors. Required if more than one such
   region is present.
 
+- msi-controller: Boolean property. Identifies the node as an MSI
+  controller. Only present if the Message Based Interrupt
+  functionnality is being exposed by the HW, and the mbi-ranges
+  property present.
+
+- mbi-ranges: A list of pairs <intid span>, where "intid" is the first
+  SPI of a range that can be used an MBI, and "span" the size of that
+  range. Multiple ranges can be provided. Requires "msi-controller" to
+  be set.
+
+- mbi-alias: Address property. Base address of an alias of the GICD
+  region containing only the {SET,CLR}SPI registers to be used if
+  isolation is required, and if supported by the HW.
+
 Sub-nodes:
 
 PPI affinity can be expressed as a single "ppi-partitions" node,
@@ -99,6 +113,9 @@ Examples:
 		      <0x0 0x2c020000 0 0x2000>;	// GICV
 		interrupts = <1 9 4>;
 
+		msi-controller;
+		mbi-ranges = <256 128>;
+
 		gic-its@2c200000 {
 			compatible = "arm,gic-v3-its";
 			msi-controller;
diff --git a/Documentation/devicetree/bindings/ipmi/npcm7xx-kcs-bmc.txt b/Documentation/devicetree/bindings/ipmi/npcm7xx-kcs-bmc.txt
new file mode 100644
index 000000000000..3538a214fff1
--- /dev/null
+++ b/Documentation/devicetree/bindings/ipmi/npcm7xx-kcs-bmc.txt
@@ -0,0 +1,39 @@
+* Nuvoton NPCM7xx KCS (Keyboard Controller Style) IPMI interface
+
+The Nuvoton SOCs (NPCM7xx) are commonly used as BMCs
+(Baseboard Management Controllers) and the KCS interface can be
+used to perform in-band IPMI communication with their host.
+
+Required properties:
+- compatible : should be one of
+    "nuvoton,npcm750-kcs-bmc"
+- interrupts : interrupt generated by the controller
+- kcs_chan : The KCS channel number in the controller
+
+Example:
+
+    lpc_kcs: lpc_kcs@f0007000 {
+        compatible = "nuvoton,npcm750-lpc-kcs", "simple-mfd", "syscon";
+        reg = <0xf0007000 0x40>;
+        reg-io-width = <1>;
+
+        #address-cells = <1>;
+        #size-cells = <1>;
+        ranges = <0x0 0xf0007000 0x40>;
+
+        kcs1: kcs1@0 {
+            compatible = "nuvoton,npcm750-kcs-bmc";
+            reg = <0x0 0x40>;
+            interrupts = <0 9 4>;
+            kcs_chan = <1>;
+            status = "disabled";
+        };
+
+        kcs2: kcs2@0 {
+            compatible = "nuvoton,npcm750-kcs-bmc";
+            reg = <0x0 0x40>;
+            interrupts = <0 9 4>;
+            kcs_chan = <2>;
+            status = "disabled";
+        };
+    };
\ No newline at end of file
diff --git a/Documentation/devicetree/bindings/leds/backlight/pwm-backlight.txt b/Documentation/devicetree/bindings/leds/backlight/pwm-backlight.txt
index 764db86d441a..310810906613 100644
--- a/Documentation/devicetree/bindings/leds/backlight/pwm-backlight.txt
+++ b/Documentation/devicetree/bindings/leds/backlight/pwm-backlight.txt
@@ -17,6 +17,10 @@ Optional properties:
                "pwms" property (see PWM binding[0])
   - enable-gpios: contains a single GPIO specifier for the GPIO which enables
                   and disables the backlight (see GPIO binding[1])
+  - post-pwm-on-delay-ms: Delay in ms between setting an initial (non-zero) PWM
+                          and enabling the backlight using GPIO.
+  - pwm-off-delay-ms: Delay in ms between disabling the backlight using GPIO
+                      and setting PWM value to 0.
 
 [0]: Documentation/devicetree/bindings/pwm/pwm.txt
 [1]: Documentation/devicetree/bindings/gpio/gpio.txt
@@ -32,4 +36,6 @@ Example:
 
 		power-supply = <&vdd_bl_reg>;
 		enable-gpios = <&gpio 58 0>;
+		post-pwm-on-delay-ms = <10>;
+		pwm-off-delay-ms = <10>;
 	};
diff --git a/Documentation/devicetree/bindings/leds/backlight/zii,rave-sp-backlight.txt b/Documentation/devicetree/bindings/leds/backlight/zii,rave-sp-backlight.txt
new file mode 100644
index 000000000000..ff5c92138650
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/backlight/zii,rave-sp-backlight.txt
@@ -0,0 +1,23 @@
+Zodiac Inflight Innovations RAVE Supervisory Processor Backlight Bindings
+
+RAVE SP backlight device is a "MFD cell" device corresponding to
+backlight functionality of RAVE Supervisory Processor. It is expected
+that its Device Tree node is specified as a child of the node
+corresponding to the parent RAVE SP device (as documented in
+Documentation/devicetree/bindings/mfd/zii,rave-sp.txt)
+
+Required properties:
+
+- compatible: Should be "zii,rave-sp-backlight"
+
+Example:
+
+	rave-sp {
+		compatible = "zii,rave-sp-rdu1";
+		current-speed = <38400>;
+
+		backlight {
+			compatible = "zii,rave-sp-backlight";
+		};
+	}
+
diff --git a/Documentation/devicetree/bindings/leds/leds-cr0014114.txt b/Documentation/devicetree/bindings/leds/leds-cr0014114.txt
new file mode 100644
index 000000000000..4255b19ad25c
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/leds-cr0014114.txt
@@ -0,0 +1,54 @@
+Crane Merchandising System - cr0014114 LED driver
+-------------------------------------------------
+
+This LED Board is widely used in vending machines produced
+by Crane Merchandising Systems.
+
+Required properties:
+- compatible: "crane,cr0014114"
+
+Property rules described in Documentation/devicetree/bindings/spi/spi-bus.txt
+apply. In particular, "reg" and "spi-max-frequency" properties must be given.
+
+LED sub-node properties:
+- label :
+	see Documentation/devicetree/bindings/leds/common.txt
+- linux,default-trigger : (optional)
+	see Documentation/devicetree/bindings/leds/common.txt
+
+Example
+-------
+
+led-controller@0 {
+	compatible = "crane,cr0014114";
+	reg = <0>;
+	spi-max-frequency = <50000>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	led@0 {
+		reg = <0>;
+		label = "red:coin";
+	};
+	led@1 {
+		reg = <1>;
+		label = "green:coin";
+	};
+	led@2 {
+		reg = <2>;
+		label = "blue:coin";
+	};
+	led@3 {
+		reg = <3>;
+		label = "red:bill";
+	};
+	led@4 {
+		reg = <4>;
+		label = "green:bill";
+	};
+	led@5 {
+		reg = <5>;
+		label = "blue:bill";
+	};
+	...
+};
diff --git a/Documentation/devicetree/bindings/leds/leds-lm3601x.txt b/Documentation/devicetree/bindings/leds/leds-lm3601x.txt
new file mode 100644
index 000000000000..a88b2c41e75b
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/leds-lm3601x.txt
@@ -0,0 +1,45 @@
+* Texas Instruments - lm3601x Single-LED Flash Driver
+
+The LM3601X are ultra-small LED flash drivers that
+provide a high level of adjustability.
+
+Required properties:
+	- compatible : Can be one of the following
+		"ti,lm36010"
+		"ti,lm36011"
+	- reg : I2C slave address
+	- #address-cells : 1
+	- #size-cells : 0
+
+Required child properties:
+	- reg : 0 - Indicates a IR mode
+		1 - Indicates a Torch (white LED) mode
+
+Required properties for flash LED child nodes:
+	See Documentation/devicetree/bindings/leds/common.txt
+	- flash-max-microamp : Range from 11mA - 1.5A
+	- flash-max-timeout-us : Range from 40ms - 1600ms
+	- led-max-microamp : Range from 2.4mA - 376mA
+
+Optional child properties:
+	- label : see Documentation/devicetree/bindings/leds/common.txt
+
+Example:
+led-controller@64 {
+	compatible = "ti,lm36010";
+	#address-cells = <1>;
+	#size-cells = <0>;
+	reg = <0x64>;
+
+	led@0 {
+		reg = <1>;
+		label = "white:torch";
+		led-max-microamp = <376000>;
+		flash-max-microamp = <1500000>;
+		flash-max-timeout-us = <1600000>;
+	};
+}
+
+For more product information please see the links below:
+http://www.ti.com/product/LM36010
+http://www.ti.com/product/LM36011
diff --git a/Documentation/devicetree/bindings/leds/leds-sc27xx-bltc.txt b/Documentation/devicetree/bindings/leds/leds-sc27xx-bltc.txt
new file mode 100644
index 000000000000..dddf84f9c7ea
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/leds-sc27xx-bltc.txt
@@ -0,0 +1,41 @@
+LEDs connected to Spreadtrum SC27XX PMIC breathing light controller
+
+The SC27xx breathing light controller supports to 3 outputs:
+red LED, green LED and blue LED. Each LED can work at normal
+PWM mode or breath light mode.
+
+Required properties:
+- compatible: Should be "sprd,sc2731-bltc".
+- #address-cells: Must be 1.
+- #size-cells: Must be 0.
+- reg: Specify the controller address.
+
+Required child properties:
+- reg: Port this LED is connected to.
+
+Optional child properties:
+- label: See Documentation/devicetree/bindings/leds/common.txt.
+
+Examples:
+
+led-controller@200 {
+	compatible = "sprd,sc2731-bltc";
+	#address-cells = <1>;
+	#size-cells = <0>;
+	reg = <0x200>;
+
+	led@0 {
+		label = "red";
+		reg = <0x0>;
+	};
+
+	led@1 {
+		label = "green";
+		reg = <0x1>;
+	};
+
+	led@2 {
+		label = "blue";
+		reg = <0x2>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/media/cdns,csi2rx.txt b/Documentation/devicetree/bindings/media/cdns,csi2rx.txt
new file mode 100644
index 000000000000..6b02a0657ad9
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/cdns,csi2rx.txt
@@ -0,0 +1,100 @@
+Cadence MIPI-CSI2 RX controller
+===============================
+
+The Cadence MIPI-CSI2 RX controller is a CSI-2 bridge supporting up to 4 CSI
+lanes in input, and 4 different pixel streams in output.
+
+Required properties:
+  - compatible: must be set to "cdns,csi2rx" and an SoC-specific compatible
+  - reg: base address and size of the memory mapped region
+  - clocks: phandles to the clocks driving the controller
+  - clock-names: must contain:
+    * sys_clk: main clock
+    * p_clk: register bank clock
+    * pixel_if[0-3]_clk: pixel stream output clock, one for each stream
+                         implemented in hardware, between 0 and 3
+
+Optional properties:
+  - phys: phandle to the external D-PHY, phy-names must be provided
+  - phy-names: must contain "dphy", if the implementation uses an
+               external D-PHY
+
+Required subnodes:
+  - ports: A ports node with one port child node per device input and output
+           port, in accordance with the video interface bindings defined in
+           Documentation/devicetree/bindings/media/video-interfaces.txt. The
+           port nodes are numbered as follows:
+
+           Port Description
+           -----------------------------
+           0    CSI-2 input
+           1    Stream 0 output
+           2    Stream 1 output
+           3    Stream 2 output
+           4    Stream 3 output
+
+           The stream output port nodes are optional if they are not
+           connected to anything at the hardware level or implemented
+           in the design.Since there is only one endpoint per port,
+           the endpoints are not numbered.
+
+
+Example:
+
+csi2rx: csi-bridge@0d060000 {
+	compatible = "cdns,csi2rx";
+	reg = <0x0d060000 0x1000>;
+	clocks = <&byteclock>, <&byteclock>
+		 <&coreclock>, <&coreclock>,
+		 <&coreclock>, <&coreclock>;
+	clock-names = "sys_clk", "p_clk",
+		      "pixel_if0_clk", "pixel_if1_clk",
+		      "pixel_if2_clk", "pixel_if3_clk";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+
+			csi2rx_in_sensor: endpoint {
+				remote-endpoint = <&sensor_out_csi2rx>;
+				clock-lanes = <0>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+
+			csi2rx_out_grabber0: endpoint {
+				remote-endpoint = <&grabber0_in_csi2rx>;
+			};
+		};
+
+		port@2 {
+			reg = <2>;
+
+			csi2rx_out_grabber1: endpoint {
+				remote-endpoint = <&grabber1_in_csi2rx>;
+			};
+		};
+
+		port@3 {
+			reg = <3>;
+
+			csi2rx_out_grabber2: endpoint {
+				remote-endpoint = <&grabber2_in_csi2rx>;
+			};
+		};
+
+		port@4 {
+			reg = <4>;
+
+			csi2rx_out_grabber3: endpoint {
+				remote-endpoint = <&grabber3_in_csi2rx>;
+			};
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/media/cdns,csi2tx.txt b/Documentation/devicetree/bindings/media/cdns,csi2tx.txt
new file mode 100644
index 000000000000..459c6e332f52
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/cdns,csi2tx.txt
@@ -0,0 +1,98 @@
+Cadence MIPI-CSI2 TX controller
+===============================
+
+The Cadence MIPI-CSI2 TX controller is a CSI-2 bridge supporting up to
+4 CSI lanes in output, and up to 4 different pixel streams in input.
+
+Required properties:
+  - compatible: must be set to "cdns,csi2tx"
+  - reg: base address and size of the memory mapped region
+  - clocks: phandles to the clocks driving the controller
+  - clock-names: must contain:
+    * esc_clk: escape mode clock
+    * p_clk: register bank clock
+    * pixel_if[0-3]_clk: pixel stream output clock, one for each stream
+                         implemented in hardware, between 0 and 3
+
+Optional properties
+  - phys: phandle to the D-PHY. If it is set, phy-names need to be set
+  - phy-names: must contain "dphy"
+
+Required subnodes:
+  - ports: A ports node with one port child node per device input and output
+           port, in accordance with the video interface bindings defined in
+           Documentation/devicetree/bindings/media/video-interfaces.txt. The
+           port nodes are numbered as follows.
+
+           Port Description
+           -----------------------------
+           0    CSI-2 output
+           1    Stream 0 input
+           2    Stream 1 input
+           3    Stream 2 input
+           4    Stream 3 input
+
+           The stream input port nodes are optional if they are not
+           connected to anything at the hardware level or implemented
+           in the design. Since there is only one endpoint per port,
+           the endpoints are not numbered.
+
+Example:
+
+csi2tx: csi-bridge@0d0e1000 {
+	compatible = "cdns,csi2tx";
+	reg = <0x0d0e1000 0x1000>;
+	clocks = <&byteclock>, <&byteclock>,
+		 <&coreclock>, <&coreclock>,
+		 <&coreclock>, <&coreclock>;
+	clock-names = "p_clk", "esc_clk",
+		      "pixel_if0_clk", "pixel_if1_clk",
+		      "pixel_if2_clk", "pixel_if3_clk";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+
+			csi2tx_out: endpoint {
+				remote-endpoint = <&remote_in>;
+				clock-lanes = <0>;
+				data-lanes = <1 2>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+
+			csi2tx_in_stream0: endpoint {
+				remote-endpoint = <&stream0_out>;
+			};
+		};
+
+		port@2 {
+			reg = <2>;
+
+			csi2tx_in_stream1: endpoint {
+				remote-endpoint = <&stream1_out>;
+			};
+		};
+
+		port@3 {
+			reg = <3>;
+
+			csi2tx_in_stream2: endpoint {
+				remote-endpoint = <&stream2_out>;
+			};
+		};
+
+		port@4 {
+			reg = <4>;
+
+			csi2tx_in_stream3: endpoint {
+				remote-endpoint = <&stream3_out>;
+			};
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/media/i2c/ov7251.txt b/Documentation/devicetree/bindings/media/i2c/ov7251.txt
new file mode 100644
index 000000000000..8281151f7493
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/i2c/ov7251.txt
@@ -0,0 +1,52 @@
+* Omnivision 1/7.5-Inch B&W VGA CMOS Digital Image Sensor
+
+The Omnivision OV7251 is a 1/7.5-Inch CMOS active pixel digital image sensor
+with an active array size of 640H x 480V. It is programmable through a serial
+I2C interface.
+
+Required Properties:
+- compatible: Value should be "ovti,ov7251".
+- clocks: Reference to the xclk clock.
+- clock-names: Should be "xclk".
+- clock-frequency: Frequency of the xclk clock.
+- enable-gpios: Chip enable GPIO. Polarity is GPIO_ACTIVE_HIGH. This corresponds
+  to the hardware pin XSHUTDOWN which is physically active low.
+- vdddo-supply: Chip digital IO regulator.
+- vdda-supply: Chip analog regulator.
+- vddd-supply: Chip digital core regulator.
+
+The device node shall contain one 'port' child node with a single 'endpoint'
+subnode for its digital output video port, in accordance with the video
+interface bindings defined in
+Documentation/devicetree/bindings/media/video-interfaces.txt.
+
+Example:
+
+	&i2c1 {
+		...
+
+		ov7251: camera-sensor@60 {
+			compatible = "ovti,ov7251";
+			reg = <0x60>;
+
+			enable-gpios = <&gpio1 6 GPIO_ACTIVE_HIGH>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&camera_bw_default>;
+
+			clocks = <&clks 200>;
+			clock-names = "xclk";
+			clock-frequency = <24000000>;
+
+			vdddo-supply = <&camera_dovdd_1v8>;
+			vdda-supply = <&camera_avdd_2v8>;
+			vddd-supply = <&camera_dvdd_1v2>;
+
+			port {
+				ov7251_ep: endpoint {
+					clock-lanes = <1>;
+					data-lanes = <0>;
+					remote-endpoint = <&csi0_ep>;
+				};
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/media/i2c/ov772x.txt b/Documentation/devicetree/bindings/media/i2c/ov772x.txt
new file mode 100644
index 000000000000..0b3ede5b8e6a
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/i2c/ov772x.txt
@@ -0,0 +1,40 @@
+* Omnivision OV7720/OV7725 CMOS sensor
+
+The Omnivision OV7720/OV7725 sensor supports multiple resolutions output,
+such as VGA, QVGA, and any size scaling down from CIF to 40x30. It also can
+support the YUV422, RGB565/555/444, GRB422 or raw RGB output formats.
+
+Required Properties:
+- compatible: shall be one of
+	"ovti,ov7720"
+	"ovti,ov7725"
+- clocks: reference to the xclk input clock.
+
+Optional Properties:
+- reset-gpios: reference to the GPIO connected to the RSTB pin which is
+  active low, if any.
+- powerdown-gpios: reference to the GPIO connected to the PWDN pin which is
+  active high, if any.
+
+The device node shall contain one 'port' child node with one child 'endpoint'
+subnode for its digital output video port, in accordance with the video
+interface bindings defined in Documentation/devicetree/bindings/media/
+video-interfaces.txt.
+
+Example:
+
+&i2c0 {
+	ov772x: camera@21 {
+		compatible = "ovti,ov7725";
+		reg = <0x21>;
+		reset-gpios = <&axi_gpio_0 0 GPIO_ACTIVE_LOW>;
+		powerdown-gpios = <&axi_gpio_0 1 GPIO_ACTIVE_LOW>;
+		clocks = <&xclk>;
+
+		port {
+			ov772x_0: endpoint {
+				remote-endpoint = <&vcap1_in0>;
+			};
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/media/i2c/panasonic,amg88xx.txt b/Documentation/devicetree/bindings/media/i2c/panasonic,amg88xx.txt
new file mode 100644
index 000000000000..4a3181a3dd7e
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/i2c/panasonic,amg88xx.txt
@@ -0,0 +1,19 @@
+* Panasonic AMG88xx
+
+The Panasonic family of AMG88xx Grid-Eye sensors allow recording
+8x8 10Hz video which consists of thermal datapoints
+
+Required Properties:
+ - compatible : Must be "panasonic,amg88xx"
+ - reg : i2c address of the device
+
+Example:
+
+	i2c0@1c22000 {
+		...
+		amg88xx@69 {
+			compatible = "panasonic,amg88xx";
+			reg = <0x69>;
+		};
+		...
+	};
diff --git a/Documentation/devicetree/bindings/media/rcar_vin.txt b/Documentation/devicetree/bindings/media/rcar_vin.txt
index 1ce7ff9449c5..a19517e1c669 100644
--- a/Documentation/devicetree/bindings/media/rcar_vin.txt
+++ b/Documentation/devicetree/bindings/media/rcar_vin.txt
@@ -2,20 +2,28 @@ Renesas R-Car Video Input driver (rcar_vin)
 -------------------------------------------
 
 The rcar_vin device provides video input capabilities for the Renesas R-Car
-family of devices. The current blocks are always slaves and suppot one input
-channel which can be either RGB, YUYV or BT656.
+family of devices.
+
+Each VIN instance has a single parallel input that supports RGB and YUV video,
+with both external synchronization and BT.656 synchronization for the latter.
+Depending on the instance the VIN input is connected to external SoC pins, or
+on Gen3 platforms to a CSI-2 receiver.
 
  - compatible: Must be one or more of the following
-   - "renesas,vin-r8a7795" for the R8A7795 device
-   - "renesas,vin-r8a7794" for the R8A7794 device
-   - "renesas,vin-r8a7793" for the R8A7793 device
-   - "renesas,vin-r8a7792" for the R8A7792 device
-   - "renesas,vin-r8a7791" for the R8A7791 device
-   - "renesas,vin-r8a7790" for the R8A7790 device
-   - "renesas,vin-r8a7779" for the R8A7779 device
+   - "renesas,vin-r8a7743" for the R8A7743 device
+   - "renesas,vin-r8a7745" for the R8A7745 device
    - "renesas,vin-r8a7778" for the R8A7778 device
-   - "renesas,rcar-gen2-vin" for a generic R-Car Gen2 compatible device.
-   - "renesas,rcar-gen3-vin" for a generic R-Car Gen3 compatible device.
+   - "renesas,vin-r8a7779" for the R8A7779 device
+   - "renesas,vin-r8a7790" for the R8A7790 device
+   - "renesas,vin-r8a7791" for the R8A7791 device
+   - "renesas,vin-r8a7792" for the R8A7792 device
+   - "renesas,vin-r8a7793" for the R8A7793 device
+   - "renesas,vin-r8a7794" for the R8A7794 device
+   - "renesas,vin-r8a7795" for the R8A7795 device
+   - "renesas,vin-r8a7796" for the R8A7796 device
+   - "renesas,vin-r8a77970" for the R8A77970 device
+   - "renesas,rcar-gen2-vin" for a generic R-Car Gen2 or RZ/G1 compatible
+     device.
 
    When compatible with the generic version nodes must list the
    SoC-specific version corresponding to the platform first
@@ -28,21 +36,38 @@ channel which can be either RGB, YUYV or BT656.
 Additionally, an alias named vinX will need to be created to specify
 which video input device this is.
 
-The per-board settings:
+The per-board settings Gen2 platforms:
  - port sub-node describing a single endpoint connected to the vin
    as described in video-interfaces.txt[1]. Only the first one will
    be considered as each vin interface has one input port.
 
-   These settings are used to work out video input format and widths
-   into the system.
+The per-board settings Gen3 platforms:
 
+Gen3 platforms can support both a single connected parallel input source
+from external SoC pins (port0) and/or multiple parallel input sources
+from local SoC CSI-2 receivers (port1) depending on SoC.
 
-Device node example
--------------------
+- renesas,id - ID number of the VIN, VINx in the documentation.
+- ports
+    - port 0 - sub-node describing a single endpoint connected to the VIN
+      from external SoC pins described in video-interfaces.txt[1].
+      Describing more then one endpoint in port 0 is invalid. Only VIN
+      instances that are connected to external pins should have port 0.
+    - port 1 - sub-nodes describing one or more endpoints connected to
+      the VIN from local SoC CSI-2 receivers. The endpoint numbers must
+      use the following schema.
 
-	aliases {
-	       vin0 = &vin0;
-	};
+        - Endpoint 0 - sub-node describing the endpoint connected to CSI20
+        - Endpoint 1 - sub-node describing the endpoint connected to CSI21
+        - Endpoint 2 - sub-node describing the endpoint connected to CSI40
+        - Endpoint 3 - sub-node describing the endpoint connected to CSI41
+
+Device node example for Gen2 platforms
+--------------------------------------
+
+        aliases {
+                vin0 = &vin0;
+        };
 
         vin0: vin@e6ef0000 {
                 compatible = "renesas,vin-r8a7790", "renesas,rcar-gen2-vin";
@@ -52,8 +77,8 @@ Device node example
                 status = "disabled";
         };
 
-Board setup example (vin1 composite video input)
-------------------------------------------------
+Board setup example for Gen2 platforms (vin1 composite video input)
+-------------------------------------------------------------------
 
 &i2c2   {
         status = "okay";
@@ -92,6 +117,77 @@ Board setup example (vin1 composite video input)
         };
 };
 
+Device node example for Gen3 platforms
+--------------------------------------
+
+        vin0: video@e6ef0000 {
+                compatible = "renesas,vin-r8a7795";
+                reg = <0 0xe6ef0000 0 0x1000>;
+                interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
+                clocks = <&cpg CPG_MOD 811>;
+                power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+                resets = <&cpg 811>;
+                renesas,id = <0>;
+
+                ports {
+                        #address-cells = <1>;
+                        #size-cells = <0>;
+
+                        port@1 {
+                                #address-cells = <1>;
+                                #size-cells = <0>;
+
+                                reg = <1>;
+
+                                vin0csi20: endpoint@0 {
+                                        reg = <0>;
+                                        remote-endpoint= <&csi20vin0>;
+                                };
+                                vin0csi21: endpoint@1 {
+                                        reg = <1>;
+                                        remote-endpoint= <&csi21vin0>;
+                                };
+                                vin0csi40: endpoint@2 {
+                                        reg = <2>;
+                                        remote-endpoint= <&csi40vin0>;
+                                };
+                        };
+                };
+        };
 
+        csi20: csi2@fea80000 {
+                compatible = "renesas,r8a7795-csi2";
+                reg = <0 0xfea80000 0 0x10000>;
+                interrupts = <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>;
+                clocks = <&cpg CPG_MOD 714>;
+                power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+                resets = <&cpg 714>;
+
+                ports {
+                        #address-cells = <1>;
+                        #size-cells = <0>;
+
+                        port@0 {
+                                reg = <0>;
+                                csi20_in: endpoint {
+                                        clock-lanes = <0>;
+                                        data-lanes = <1>;
+                                        remote-endpoint = <&adv7482_txb>;
+                                };
+                        };
+
+                        port@1 {
+                                #address-cells = <1>;
+                                #size-cells = <0>;
+
+                                reg = <1>;
+
+                                csi20vin0: endpoint@0 {
+                                        reg = <0>;
+                                        remote-endpoint = <&vin0csi20>;
+                                };
+                        };
+                };
+        };
 
 [1] video-interfaces.txt common video media interface
diff --git a/Documentation/devicetree/bindings/media/renesas,ceu.txt b/Documentation/devicetree/bindings/media/renesas,ceu.txt
index 3fc66dfb192c..8a7a616e9019 100644
--- a/Documentation/devicetree/bindings/media/renesas,ceu.txt
+++ b/Documentation/devicetree/bindings/media/renesas,ceu.txt
@@ -2,14 +2,15 @@ Renesas Capture Engine Unit (CEU)
 ----------------------------------------------
 
 The Capture Engine Unit is the image capture interface found in the Renesas
-SH Mobile and RZ SoCs.
+SH Mobile, R-Mobile and RZ SoCs.
 
 The interface supports a single parallel input with data bus width of 8 or 16
 bits.
 
 Required properties:
-- compatible: Shall be "renesas,r7s72100-ceu" for CEU units found in RZ/A1H
-  and RZ/A1M SoCs.
+- compatible: Shall be one of the following values:
+	"renesas,r7s72100-ceu" for CEU units found in RZ/A1H and RZ/A1M SoCs
+	"renesas,r8a7740-ceu" for CEU units found in R-Mobile A1 R8A7740 SoCs
 - reg: Registers address base and size.
 - interrupts: The interrupt specifier.
 
diff --git a/Documentation/devicetree/bindings/media/renesas,rcar-csi2.txt b/Documentation/devicetree/bindings/media/renesas,rcar-csi2.txt
new file mode 100644
index 000000000000..2d385b65b275
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/renesas,rcar-csi2.txt
@@ -0,0 +1,101 @@
+Renesas R-Car MIPI CSI-2
+------------------------
+
+The R-Car CSI-2 receiver device provides MIPI CSI-2 capabilities for the
+Renesas R-Car family of devices. It is used in conjunction with the
+R-Car VIN module, which provides the video capture capabilities.
+
+Mandatory properties
+--------------------
+ - compatible: Must be one or more of the following
+   - "renesas,r8a7795-csi2" for the R8A7795 device.
+   - "renesas,r8a7796-csi2" for the R8A7796 device.
+   - "renesas,r8a77965-csi2" for the R8A77965 device.
+   - "renesas,r8a77970-csi2" for the R8A77970 device.
+
+ - reg: the register base and size for the device registers
+ - interrupts: the interrupt for the device
+ - clocks: reference to the parent clock
+
+The device node shall contain two 'port' child nodes according to the
+bindings defined in Documentation/devicetree/bindings/media/
+video-interfaces.txt. port@0 shall connect to the CSI-2 source. port@1
+shall connect to all the R-Car VIN modules that have a hardware
+connection to the CSI-2 receiver.
+
+- port@0- Video source (mandatory)
+	- endpoint@0 - sub-node describing the endpoint that is the video source
+
+- port@1 - VIN instances (optional)
+	- One endpoint sub-node for every R-Car VIN instance which is connected
+	  to the R-Car CSI-2 receiver.
+
+Example:
+
+	csi20: csi2@fea80000 {
+		compatible = "renesas,r8a7796-csi2";
+		reg = <0 0xfea80000 0 0x10000>;
+		interrupts = <0 184 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cpg CPG_MOD 714>;
+		power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+		resets = <&cpg 714>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				reg = <0>;
+
+				csi20_in: endpoint@0 {
+					reg = <0>;
+					clock-lanes = <0>;
+					data-lanes = <1>;
+					remote-endpoint = <&adv7482_txb>;
+				};
+			};
+
+			port@1 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				reg = <1>;
+
+				csi20vin0: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&vin0csi20>;
+				};
+				csi20vin1: endpoint@1 {
+					reg = <1>;
+					remote-endpoint = <&vin1csi20>;
+				};
+				csi20vin2: endpoint@2 {
+					reg = <2>;
+					remote-endpoint = <&vin2csi20>;
+				};
+				csi20vin3: endpoint@3 {
+					reg = <3>;
+					remote-endpoint = <&vin3csi20>;
+				};
+				csi20vin4: endpoint@4 {
+					reg = <4>;
+					remote-endpoint = <&vin4csi20>;
+				};
+				csi20vin5: endpoint@5 {
+					reg = <5>;
+					remote-endpoint = <&vin5csi20>;
+				};
+				csi20vin6: endpoint@6 {
+					reg = <6>;
+					remote-endpoint = <&vin6csi20>;
+				};
+				csi20vin7: endpoint@7 {
+					reg = <7>;
+					remote-endpoint = <&vin7csi20>;
+				};
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra20-mc.txt b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra20-mc.txt
new file mode 100644
index 000000000000..7d60a50a4fa1
--- /dev/null
+++ b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra20-mc.txt
@@ -0,0 +1,26 @@
+NVIDIA Tegra20 MC(Memory Controller)
+
+Required properties:
+- compatible : "nvidia,tegra20-mc"
+- reg : Should contain 2 register ranges(address and length); see the
+  example below. Note that the MC registers are interleaved with the
+  GART registers, and hence must be represented as multiple ranges.
+- interrupts : Should contain MC General interrupt.
+- #reset-cells : Should be 1. This cell represents memory client module ID.
+  The assignments may be found in header file <dt-bindings/memory/tegra20-mc.h>
+  or in the TRM documentation.
+
+Example:
+	mc: memory-controller@7000f000 {
+		compatible = "nvidia,tegra20-mc";
+		reg = <0x7000f000 0x024
+		       0x7000f03c 0x3c4>;
+		interrupts = <0 77 0x04>;
+		#reset-cells = <1>;
+	};
+
+	video-codec@6001a000 {
+		compatible = "nvidia,tegra20-vde";
+		...
+		resets = <&mc TEGRA20_MC_RESET_VDE>;
+	};
diff --git a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra30-mc.txt b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra30-mc.txt
index 14968b048cd3..a878b5908a4d 100644
--- a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra30-mc.txt
+++ b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra30-mc.txt
@@ -12,6 +12,9 @@ Required properties:
 - clock-names: Must include the following entries:
   - mc: the module's clock input
 - interrupts: The interrupt outputs from the controller.
+- #reset-cells : Should be 1. This cell represents memory client module ID.
+  The assignments may be found in header file <dt-bindings/memory/tegra30-mc.h>
+  or in the TRM documentation.
 
 Required properties for Tegra30, Tegra114, Tegra124, Tegra132 and Tegra210:
 - #iommu-cells: Should be 1. The single cell of the IOMMU specifier defines
@@ -72,12 +75,14 @@ Example SoC include file:
 		interrupts = <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>;
 
 		#iommu-cells = <1>;
+		#reset-cells = <1>;
 	};
 
 	sdhci@700b0000 {
 		compatible = "nvidia,tegra124-sdhci";
 		...
 		iommus = <&mc TEGRA_SWGROUP_SDMMC1A>;
+		resets = <&mc TEGRA124_MC_RESET_SDMMC1>;
 	};
 };
 
diff --git a/Documentation/devicetree/bindings/mfd/axp20x.txt b/Documentation/devicetree/bindings/mfd/axp20x.txt
index 9455503b0299..d1762f3b30af 100644
--- a/Documentation/devicetree/bindings/mfd/axp20x.txt
+++ b/Documentation/devicetree/bindings/mfd/axp20x.txt
@@ -43,7 +43,7 @@ Optional properties:
 			  regulator to drive the OTG VBus, rather then
 			  as an input pin which signals whether the
 			  board is driving OTG VBus or not.
-			  (axp221 / axp223 / axp813 only)
+			  (axp221 / axp223 / axp803/ axp813 only)
 
 - x-powers,master-mode: Boolean (axp806 only). Set this when the PMIC is
 			wired for master mode. The default is slave mode.
@@ -132,6 +132,7 @@ FLDO2		: LDO		: fldoin-supply		: shared supply
 LDO_IO0		: LDO		: ips-supply		: GPIO 0
 LDO_IO1		: LDO		: ips-supply		: GPIO 1
 RTC_LDO		: LDO		: ips-supply		: always on
+DRIVEVBUS	: Enable output	: drivevbus-supply	: external regulator
 
 AXP806 regulators, type, and corresponding input supply names:
 
diff --git a/Documentation/devicetree/bindings/mfd/bd9571mwv.txt b/Documentation/devicetree/bindings/mfd/bd9571mwv.txt
index 9ab216a851d5..25d1f697eb25 100644
--- a/Documentation/devicetree/bindings/mfd/bd9571mwv.txt
+++ b/Documentation/devicetree/bindings/mfd/bd9571mwv.txt
@@ -25,6 +25,25 @@ Required properties:
 			    Each child node is defined using the standard
 			    binding for regulators.
 
+Optional properties:
+  - rohm,ddr-backup-power : Value to use for DDR-Backup Power (default 0).
+			    This is a bitmask that specifies which DDR power
+			    rails need to be kept powered when backup mode is
+			    entered, for system suspend:
+			      - bit 0: DDR0
+			      - bit 1: DDR1
+			      - bit 2: DDR0C
+			      - bit 3: DDR1C
+			    These bits match the KEEPON_DDR* bits in the
+			    documentation for the "BKUP Mode Cnt" register.
+  - rohm,rstbmode-level: The RSTB signal is configured for level mode, to
+			 accommodate a toggle power switch (the RSTBMODE pin is
+			 strapped low).
+  - rohm,rstbmode-pulse: The RSTB signal is configured for pulse mode, to
+			 accommodate a momentary power switch (the RSTBMODE pin
+			 is strapped high).
+			 The two properties above are mutually exclusive.
+
 Example:
 
 	pmic: pmic@30 {
@@ -36,6 +55,8 @@ Example:
 		#interrupt-cells = <2>;
 		gpio-controller;
 		#gpio-cells = <2>;
+		rohm,ddr-backup-power = <0xf>;
+		rohm,rstbmode-pulse;
 
 		regulators {
 			dvfs: dvfs {
diff --git a/Documentation/devicetree/bindings/mfd/motorola-cpcap.txt b/Documentation/devicetree/bindings/mfd/motorola-cpcap.txt
index 15bc885f9df4..c639705a98ef 100644
--- a/Documentation/devicetree/bindings/mfd/motorola-cpcap.txt
+++ b/Documentation/devicetree/bindings/mfd/motorola-cpcap.txt
@@ -12,6 +12,30 @@ Required properties:
 - spi-max-frequency	: Typically set to 3000000
 - spi-cs-high		: SPI chip select direction
 
+Optional subnodes:
+
+The sub-functions of CPCAP get their own node with their own compatible values,
+which are described in the following files:
+
+- ../power/supply/cpcap-battery.txt
+- ../power/supply/cpcap-charger.txt
+- ../regulator/cpcap-regulator.txt
+- ../phy/phy-cpcap-usb.txt
+- ../input/cpcap-pwrbutton.txt
+- ../rtc/cpcap-rtc.txt
+- ../leds/leds-cpcap.txt
+- ../iio/adc/cpcap-adc.txt
+
+The only exception is the audio codec. Instead of a compatible value its
+node must be named "audio-codec".
+
+Required properties for the audio-codec subnode:
+
+- #sound-dai-cells = <1>;
+
+The audio-codec provides two DAIs. The first one is connected to the
+Stereo HiFi DAC and the second one is connected to the Voice DAC.
+
 Example:
 
 &mcspi1 {
@@ -26,6 +50,24 @@ Example:
 		#size-cells = <0>;
 		spi-max-frequency = <3000000>;
 		spi-cs-high;
+
+		audio-codec {
+			#sound-dai-cells = <1>;
+
+			/* HiFi */
+			port@0 {
+				endpoint {
+					remote-endpoint = <&cpu_dai1>;
+				};
+			};
+
+			/* Voice */
+			port@1 {
+				endpoint {
+					remote-endpoint = <&cpu_dai2>;
+				};
+			};
+		};
 	};
 };
 
diff --git a/Documentation/devicetree/bindings/mfd/mt6397.txt b/Documentation/devicetree/bindings/mfd/mt6397.txt
index 522a3bbf1bac..d1df77f4d655 100644
--- a/Documentation/devicetree/bindings/mfd/mt6397.txt
+++ b/Documentation/devicetree/bindings/mfd/mt6397.txt
@@ -7,6 +7,7 @@ MT6397/MT6323 is a multifunction device with the following sub modules:
 - GPIO
 - Clock
 - LED
+- Keys
 
 It is interfaced to host controller using SPI interface by a proprietary hardware
 called PMIC wrapper or pwrap. MT6397/MT6323 MFD is a child device of pwrap.
@@ -40,6 +41,11 @@ Optional subnodes:
 		- compatible: "mediatek,mt6323-led"
 	see Documentation/devicetree/bindings/leds/leds-mt6323.txt
 
+- keys
+	Required properties:
+		- compatible: "mediatek,mt6397-keys" or "mediatek,mt6323-keys"
+	see Documentation/devicetree/bindings/input/mtk-pmic-keys.txt
+
 Example:
 	pwrap: pwrap@1000f000 {
 		compatible = "mediatek,mt8135-pwrap";
diff --git a/Documentation/devicetree/bindings/mips/lantiq/rcu.txt b/Documentation/devicetree/bindings/mips/lantiq/rcu.txt
index a086f1e1cdd7..7f0822b4beae 100644
--- a/Documentation/devicetree/bindings/mips/lantiq/rcu.txt
+++ b/Documentation/devicetree/bindings/mips/lantiq/rcu.txt
@@ -61,7 +61,6 @@ Example of the RCU bindings on a xRX200 SoC:
 		usb_phy0: usb2-phy@18 {
 			compatible = "lantiq,xrx200-usb2-phy";
 			reg = <0x18 4>, <0x38 4>;
-			status = "disabled";
 
 			resets = <&reset1 4 4>, <&reset0 4 4>;
 			reset-names = "phy", "ctrl";
@@ -71,7 +70,6 @@ Example of the RCU bindings on a xRX200 SoC:
 		usb_phy1: usb2-phy@34 {
 			compatible = "lantiq,xrx200-usb2-phy";
 			reg = <0x34 4>, <0x3C 4>;
-			status = "disabled";
 
 			resets = <&reset1 5 4>, <&reset0 4 4>;
 			reset-names = "phy", "ctrl";
diff --git a/Documentation/devicetree/bindings/mmc/amlogic,meson-gx.txt b/Documentation/devicetree/bindings/mmc/amlogic,meson-gx.txt
index 50bf611a4d2c..13e70409e8ac 100644
--- a/Documentation/devicetree/bindings/mmc/amlogic,meson-gx.txt
+++ b/Documentation/devicetree/bindings/mmc/amlogic,meson-gx.txt
@@ -12,6 +12,7 @@ Required properties:
   - "amlogic,meson-gxbb-mmc"
   - "amlogic,meson-gxl-mmc"
   - "amlogic,meson-gxm-mmc"
+  - "amlogic,meson-axg-mmc"
 - clocks     : A list of phandle + clock-specifier pairs for the clocks listed in clock-names.
 - clock-names: Should contain the following:
 	"core" - Main peripheral bus clock
@@ -19,6 +20,7 @@ Required properties:
 	"clkin1" - Other parent clock of internal mux
   The driver has an internal mux clock which switches between clkin0 and clkin1 depending on the
   clock rate requested by the MMC core.
+- resets     : phandle of the internal reset line
 
 Example:
 
@@ -29,4 +31,5 @@ Example:
 		clocks = <&clkc CLKID_SD_EMMC_A>, <&xtal>, <&clkc CLKID_FCLK_DIV2>;
 		clock-names = "core", "clkin0", "clkin1";
 		pinctrl-0 = <&emmc_pins>;
+		resets = <&reset RESET_SD_EMMC_A>;
 	};
diff --git a/Documentation/devicetree/bindings/mmc/bluefield-dw-mshc.txt b/Documentation/devicetree/bindings/mmc/bluefield-dw-mshc.txt
new file mode 100644
index 000000000000..b0f0999ea1a9
--- /dev/null
+++ b/Documentation/devicetree/bindings/mmc/bluefield-dw-mshc.txt
@@ -0,0 +1,29 @@
+* Mellanox Bluefield SoC specific extensions to the Synopsys Designware
+  Mobile Storage Host Controller
+
+Read synopsys-dw-mshc.txt for more details
+
+The Synopsys designware mobile storage host controller is used to interface
+a SoC with storage medium such as eMMC or SD/MMC cards. This file documents
+differences between the core Synopsys dw mshc controller properties described
+by synopsys-dw-mshc.txt and the properties used by the Mellanox Bluefield SoC
+specific extensions to the Synopsys Designware Mobile Storage Host Controller.
+
+Required Properties:
+
+* compatible: should be one of the following.
+  - "mellanox,bluefield-dw-mshc": for controllers with Mellanox Bluefield SoC
+    specific extensions.
+
+Example:
+
+	/* Mellanox Bluefield SoC MMC */
+	mmc@6008000 {
+		compatible = "mellanox,bluefield-dw-mshc";
+		reg = <0x6008000 0x400>;
+		interrupts = <32>;
+		fifo-depth = <0x100>;
+		clock-frequency = <24000000>;
+		bus-width = <8>;
+		cap-mmc-highspeed;
+	};
diff --git a/Documentation/devicetree/bindings/mmc/jz4740.txt b/Documentation/devicetree/bindings/mmc/jz4740.txt
new file mode 100644
index 000000000000..7cd8c432d7c8
--- /dev/null
+++ b/Documentation/devicetree/bindings/mmc/jz4740.txt
@@ -0,0 +1,38 @@
+* Ingenic JZ47xx MMC controllers
+
+This file documents the device tree properties used for the MMC controller in
+Ingenic JZ4740/JZ4780 SoCs. These are in addition to the core MMC properties
+described in mmc.txt.
+
+Required properties:
+- compatible: Should be one of the following:
+  - "ingenic,jz4740-mmc" for the JZ4740
+  - "ingenic,jz4780-mmc" for the JZ4780
+- reg: Should contain the MMC controller registers location and length.
+- interrupts: Should contain the interrupt specifier of the MMC controller.
+- clocks: Clock for the MMC controller.
+
+Optional properties:
+- dmas: List of DMA specifiers with the controller specific format
+        as described in the generic DMA client binding. A tx and rx
+        specifier is required.
+- dma-names: RX and TX  DMA request names.
+        Should be "rx" and "tx", in that order.
+
+For additional details on DMA client bindings see ../dma/dma.txt.
+
+Example:
+
+mmc0: mmc@13450000 {
+	compatible = "ingenic,jz4780-mmc";
+	reg = <0x13450000 0x1000>;
+
+	interrupt-parent = <&intc>;
+	interrupts = <37>;
+
+	clocks = <&cgu JZ4780_CLK_MSC0>;
+	clock-names = "mmc";
+
+	dmas = <&dma JZ4780_DMA_MSC0_RX 0xffffffff>, <&dma JZ4780_DMA_MSC0_TX 0xffffffff>;
+	dma-names = "rx", "tx";
+};
diff --git a/Documentation/devicetree/bindings/mmc/mmc.txt b/Documentation/devicetree/bindings/mmc/mmc.txt
index 467cd7b147ce..f5a0923b34ca 100644
--- a/Documentation/devicetree/bindings/mmc/mmc.txt
+++ b/Documentation/devicetree/bindings/mmc/mmc.txt
@@ -19,6 +19,8 @@ Optional properties:
 - wp-gpios: Specify GPIOs for write protection, see gpio binding
 - cd-inverted: when present, polarity on the CD line is inverted. See the note
   below for the case, when a GPIO is used for the CD line
+- cd-debounce-delay-ms: Set delay time before detecting card after card insert interrupt.
+  It's only valid when cd-gpios is present.
 - wp-inverted: when present, polarity on the WP line is inverted. See the note
   below for the case, when a GPIO is used for the WP line
 - disable-wp: When set no physical WP line is present. This property should
@@ -56,6 +58,10 @@ Optional properties:
 - fixed-emmc-driver-type: for non-removable eMMC, enforce this driver type.
   The value <n> is the driver type as specified in the eMMC specification
   (table 206 in spec version 5.1).
+- post-power-on-delay-ms : It was invented for MMC pwrseq-simple which could
+  be referred to mmc-pwrseq-simple.txt. But now it's reused as a tunable delay
+  waiting for I/O signalling and card power supply to be stable, regardless of
+  whether pwrseq-simple is used. Default to 10ms if no available.
 
 *NOTE* on CD and WP polarity. To use common for all SD/MMC host controllers line
 polarity properties, we have to fix the meaning of the "normal" and "inverted"
diff --git a/Documentation/devicetree/bindings/mmc/sdhci-omap.txt b/Documentation/devicetree/bindings/mmc/sdhci-omap.txt
index 51775a372c06..393848c2138e 100644
--- a/Documentation/devicetree/bindings/mmc/sdhci-omap.txt
+++ b/Documentation/devicetree/bindings/mmc/sdhci-omap.txt
@@ -4,7 +4,14 @@ Refer to mmc.txt for standard MMC bindings.
 
 Required properties:
 - compatible: Should be "ti,dra7-sdhci" for DRA7 and DRA72 controllers
+	      Should be "ti,k2g-sdhci" for K2G
 - ti,hwmods: Must be "mmc<n>", <n> is controller instance starting 1
+	     (Not required for K2G).
+- pinctrl-names: Should be subset of "default", "hs", "sdr12", "sdr25", "sdr50",
+		 "ddr50-rev11", "sdr104-rev11", "ddr50", "sdr104",
+		 "ddr_1_8v-rev11", "ddr_1_8v" or "ddr_3_3v", "hs200_1_8v-rev11",
+		 "hs200_1_8v",
+- pinctrl-<n> : Pinctrl states as described in bindings/pinctrl/pinctrl-bindings.txt
 
 Example:
 	mmc1: mmc@4809c000 {
diff --git a/Documentation/devicetree/bindings/mmc/tmio_mmc.txt b/Documentation/devicetree/bindings/mmc/tmio_mmc.txt
index 2d5287eeed95..839f469f4525 100644
--- a/Documentation/devicetree/bindings/mmc/tmio_mmc.txt
+++ b/Documentation/devicetree/bindings/mmc/tmio_mmc.txt
@@ -26,6 +26,8 @@ Required properties:
 		"renesas,sdhi-r8a7794" - SDHI IP on R8A7794 SoC
 		"renesas,sdhi-r8a7795" - SDHI IP on R8A7795 SoC
 		"renesas,sdhi-r8a7796" - SDHI IP on R8A7796 SoC
+		"renesas,sdhi-r8a77965" - SDHI IP on R8A77965 SoC
+		"renesas,sdhi-r8a77980" - SDHI IP on R8A77980 SoC
 		"renesas,sdhi-r8a77995" - SDHI IP on R8A77995 SoC
 		"renesas,sdhi-shmobile" - a generic sh-mobile SDHI controller
 		"renesas,rcar-gen1-sdhi" - a generic R-Car Gen1 SDHI controller
@@ -67,7 +69,6 @@ Example: R8A7790 (R-Car H2) SDHI controller nodes
 		max-frequency = <195000000>;
 		power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
 		resets = <&cpg 314>;
-		status = "disabled";
 	};
 
 	sdhi1: sd@ee120000 {
@@ -81,7 +82,6 @@ Example: R8A7790 (R-Car H2) SDHI controller nodes
 		max-frequency = <195000000>;
 		power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
 		resets = <&cpg 313>;
-		status = "disabled";
 	};
 
 	sdhi2: sd@ee140000 {
@@ -95,7 +95,6 @@ Example: R8A7790 (R-Car H2) SDHI controller nodes
 		max-frequency = <97500000>;
 		power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
 		resets = <&cpg 312>;
-		status = "disabled";
 	};
 
 	sdhi3: sd@ee160000 {
@@ -109,5 +108,4 @@ Example: R8A7790 (R-Car H2) SDHI controller nodes
 		max-frequency = <97500000>;
 		power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
 		resets = <&cpg 311>;
-		status = "disabled";
 	};
diff --git a/Documentation/devicetree/bindings/mtd/gpmi-nand.txt b/Documentation/devicetree/bindings/mtd/gpmi-nand.txt
index b289ef3c1b7e..393588385c6e 100644
--- a/Documentation/devicetree/bindings/mtd/gpmi-nand.txt
+++ b/Documentation/devicetree/bindings/mtd/gpmi-nand.txt
@@ -47,6 +47,11 @@ Optional properties:
                        partitions written from Linux with this feature
                        turned on may not be accessible by the BootROM
                        code.
+  - nand-ecc-strength: integer representing the number of bits to correct
+                       per ECC step. Needs to be a multiple of 2.
+  - nand-ecc-step-size: integer representing the number of data bytes
+                       that are covered by a single ECC step. The driver
+                       supports 512 and 1024.
 
 The device tree may optionally contain sub-nodes describing partitions of the
 address space. See partition.txt for more detail.
diff --git a/Documentation/devicetree/bindings/mtd/ibm,ndfc.txt b/Documentation/devicetree/bindings/mtd/ibm,ndfc.txt
new file mode 100644
index 000000000000..869f0b5f16e8
--- /dev/null
+++ b/Documentation/devicetree/bindings/mtd/ibm,ndfc.txt
@@ -0,0 +1,39 @@
+AMCC NDFC (NanD Flash Controller)
+
+Required properties:
+- compatible : "ibm,ndfc".
+- reg : should specify chip select and size used for the chip (0x2000).
+
+Optional properties:
+- ccr : NDFC config and control register value (default 0).
+- bank-settings : NDFC bank configuration register value (default 0).
+
+Notes:
+- partition(s) - follows the OF MTD standard for partitions
+
+Example:
+
+ndfc@1,0 {
+	compatible = "ibm,ndfc";
+	reg = <0x00000001 0x00000000 0x00002000>;
+	ccr = <0x00001000>;
+	bank-settings = <0x80002222>;
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	nand {
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		partition@0 {
+			label = "kernel";
+			reg = <0x00000000 0x00200000>;
+		};
+		partition@200000 {
+			label = "root";
+			reg = <0x00200000 0x03E00000>;
+		};
+	};
+};
+
+
diff --git a/Documentation/devicetree/bindings/mtd/mtk-nand.txt b/Documentation/devicetree/bindings/mtd/mtk-nand.txt
index 1c88526dedfc..4d3ec5e4ff8a 100644
--- a/Documentation/devicetree/bindings/mtd/mtk-nand.txt
+++ b/Documentation/devicetree/bindings/mtd/mtk-nand.txt
@@ -20,7 +20,6 @@ Required NFI properties:
 - interrupts:			Interrupts of NFI.
 - clocks:			NFI required clocks.
 - clock-names:			NFI clocks internal name.
-- status:			Disabled default. Then set "okay" by platform.
 - ecc-engine:			Required ECC Engine node.
 - #address-cells:		NAND chip index, should be 1.
 - #size-cells:			Should be 0.
@@ -34,7 +33,6 @@ Example:
 		clocks = <&pericfg CLK_PERI_NFI>,
 			 <&pericfg CLK_PERI_NFI_PAD>;
 		clock-names = "nfi_clk", "pad_clk";
-		status = "disabled";
 		ecc-engine = <&bch>;
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -50,14 +48,19 @@ Optional:
 - nand-on-flash-bbt:	Store BBT on NAND Flash.
 - nand-ecc-mode:	the NAND ecc mode (check driver for supported modes)
 - nand-ecc-step-size:	Number of data bytes covered by a single ECC step.
-			valid values: 512 and 1024.
+			valid values:
+			512 and 1024 on mt2701 and mt2712.
+			512 only on mt7622.
 			1024 is recommended for large page NANDs.
 - nand-ecc-strength:	Number of bits to correct per ECC step.
-			The valid values that the controller supports are: 4, 6,
-			8, 10, 12, 14, 16, 18, 20, 22, 24, 28, 32, 36, 40, 44,
-			48, 52, 56, 60.
+			The valid values that each controller supports:
+			mt2701: 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 28,
+				32, 36, 40, 44, 48, 52, 56, 60.
+			mt2712: 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 28,
+				32, 36, 40, 44, 48, 52, 56, 60, 68, 72, 80.
+			mt7622: 4, 6, 8, 10, 12, 14, 16.
 			The strength should be calculated as follows:
-			E = (S - F) * 8 / 14
+			E = (S - F) * 8 / B
 			S = O / (P / Q)
 				E :	nand-ecc-strength.
 				S :	spare size per sector.
@@ -66,6 +69,15 @@ Optional:
 				O :	oob size.
 				P :	page size.
 				Q :	nand-ecc-step-size.
+				B :	number of parity bits needed to correct
+					1 bitflip.
+					According to MTK NAND controller design,
+					this number depends on max ecc step size
+					that MTK NAND controller supports.
+					If max ecc step size supported is 1024,
+					then it should be always 14. And if max
+					ecc step size is 512, then it should be
+					always 13.
 			If the result does not match any one of the listed
 			choices above, please select the smaller valid value from
 			the list.
@@ -152,7 +164,6 @@ Required BCH properties:
 - interrupts:	Interrupts of ECC.
 - clocks:	ECC required clocks.
 - clock-names:	ECC clocks internal name.
-- status:	Disabled default. Then set "okay" by platform.
 
 Example:
 
@@ -162,5 +173,4 @@ Example:
 		interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_LOW>;
 		clocks = <&pericfg CLK_PERI_NFI_ECC>;
 		clock-names = "nfiecc_clk";
-		status = "disabled";
 	};
diff --git a/Documentation/devicetree/bindings/mtd/partition.txt b/Documentation/devicetree/bindings/mtd/partition.txt
index 36f3b769a626..a8f382642ba9 100644
--- a/Documentation/devicetree/bindings/mtd/partition.txt
+++ b/Documentation/devicetree/bindings/mtd/partition.txt
@@ -14,7 +14,7 @@ method is used for a given flash device. To describe the method there should be
 a subnode of the flash device that is named 'partitions'. It must have a
 'compatible' property, which is used to identify the method to use.
 
-We currently only document a binding for fixed layouts.
+Available bindings are listed in the "partitions" subdirectory.
 
 
 Fixed Partitions
diff --git a/Documentation/devicetree/bindings/mtd/partitions/brcm,bcm947xx-cfe-partitions.txt b/Documentation/devicetree/bindings/mtd/partitions/brcm,bcm947xx-cfe-partitions.txt
new file mode 100644
index 000000000000..1d61a029395e
--- /dev/null
+++ b/Documentation/devicetree/bindings/mtd/partitions/brcm,bcm947xx-cfe-partitions.txt
@@ -0,0 +1,42 @@
+Broadcom BCM47xx Partitions
+===========================
+
+Broadcom is one of hardware manufacturers providing SoCs (BCM47xx) used in
+home routers. Their BCM947xx boards using CFE bootloader have several partitions
+without any on-flash partition table. On some devices their sizes and/or
+meanings can also vary so fixed partitioning can't be used.
+
+Discovering partitions on these devices is possible thanks to having a special
+header and/or magic signature at the beginning of each of them. They are also
+block aligned which is important for determinig a size.
+
+Most of partitions use ASCII text based magic for determining a type. More
+complex partitions (like TRX with its HDR0 magic) may include extra header
+containing some details, including a length.
+
+A list of supported partitions includes:
+1) Bootloader with Broadcom's CFE (Common Firmware Environment)
+2) NVRAM with configuration/calibration data
+3) Device manufacturer's data with some default values (e.g. SSIDs)
+4) TRX firmware container which can hold up to 4 subpartitions
+5) Backup TRX firmware used after failed upgrade
+
+As mentioned earlier, role of some partitions may depend on extra configuration.
+For example both: main firmware and backup firmware use the same TRX format with
+the same header. To distinguish currently used firmware a CFE's environment
+variable "bootpartition" is used.
+
+
+Devices using Broadcom partitions described above should should have flash node
+with a subnode named "partitions" using following properties:
+
+Required properties:
+- compatible : (required) must be "brcm,bcm947xx-cfe-partitions"
+
+Example:
+
+flash@0 {
+	partitions {
+		compatible = "brcm,bcm947xx-cfe-partitions";
+	};
+};
diff --git a/Documentation/devicetree/bindings/mtd/sunxi-nand.txt b/Documentation/devicetree/bindings/mtd/sunxi-nand.txt
index 0734f03bf3d3..dcd5a5d80dc0 100644
--- a/Documentation/devicetree/bindings/mtd/sunxi-nand.txt
+++ b/Documentation/devicetree/bindings/mtd/sunxi-nand.txt
@@ -22,8 +22,6 @@ Optional properties:
 - reset : phandle + reset specifier pair
 - reset-names : must contain "ahb"
 - allwinner,rb : shall contain the native Ready/Busy ids.
- or
-- rb-gpios : shall contain the gpios used as R/B pins.
 - nand-ecc-mode : one of the supported ECC modes ("hw", "soft", "soft_bch" or
 		  "none")
 
diff --git a/Documentation/devicetree/bindings/net/dsa/dsa.txt b/Documentation/devicetree/bindings/net/dsa/dsa.txt
index cfe8f64eca4f..3ceeb8de1196 100644
--- a/Documentation/devicetree/bindings/net/dsa/dsa.txt
+++ b/Documentation/devicetree/bindings/net/dsa/dsa.txt
@@ -82,8 +82,6 @@ linked into one DSA cluster.
 
 	switch0: switch0@0 {
 		compatible = "marvell,mv88e6085";
-		#address-cells = <1>;
-		#size-cells = <0>;
 		reg = <0>;
 
 		dsa,member = <0 0>;
@@ -135,8 +133,6 @@ linked into one DSA cluster.
 
 	switch1: switch1@0 {
 		compatible = "marvell,mv88e6085";
-		#address-cells = <1>;
-		#size-cells = <0>;
 		reg = <0>;
 
 		dsa,member = <0 1>;
@@ -204,8 +200,6 @@ linked into one DSA cluster.
 
 	switch2: switch2@0 {
 		compatible = "marvell,mv88e6085";
-		#address-cells = <1>;
-		#size-cells = <0>;
 		reg = <0>;
 
 		dsa,member = <0 2>;
diff --git a/Documentation/devicetree/bindings/net/dsa/qca8k.txt b/Documentation/devicetree/bindings/net/dsa/qca8k.txt
index 9c67ee4890d7..bbcb255c3150 100644
--- a/Documentation/devicetree/bindings/net/dsa/qca8k.txt
+++ b/Documentation/devicetree/bindings/net/dsa/qca8k.txt
@@ -2,7 +2,10 @@
 
 Required properties:
 
-- compatible: should be "qca,qca8337"
+- compatible: should be one of:
+    "qca,qca8334"
+    "qca,qca8337"
+
 - #size-cells: must be 0
 - #address-cells: must be 1
 
@@ -14,6 +17,20 @@ port and PHY id, each subnode describing a port needs to have a valid phandle
 referencing the internal PHY connected to it. The CPU port of this switch is
 always port 0.
 
+A CPU port node has the following optional node:
+
+- fixed-link            : Fixed-link subnode describing a link to a non-MDIO
+                          managed entity. See
+                          Documentation/devicetree/bindings/net/fixed-link.txt
+                          for details.
+
+For QCA8K the 'fixed-link' sub-node supports only the following properties:
+
+- 'speed' (integer, mandatory), to indicate the link speed. Accepted
+  values are 10, 100 and 1000
+- 'full-duplex' (boolean, optional), to indicate that full duplex is
+  used. When absent, half duplex is assumed.
+
 Example:
 
 
@@ -53,6 +70,10 @@ Example:
 					label = "cpu";
 					ethernet = <&gmac1>;
 					phy-mode = "rgmii";
+					fixed-link {
+						speed = 1000;
+						full-duplex;
+					};
 				};
 
 				port@1 {
diff --git a/Documentation/devicetree/bindings/net/dwmac-sun8i.txt b/Documentation/devicetree/bindings/net/dwmac-sun8i.txt
index 3d6d5fa0c4d5..cfe724398a12 100644
--- a/Documentation/devicetree/bindings/net/dwmac-sun8i.txt
+++ b/Documentation/devicetree/bindings/net/dwmac-sun8i.txt
@@ -7,6 +7,7 @@ Required properties:
 - compatible: must be one of the following string:
 		"allwinner,sun8i-a83t-emac"
 		"allwinner,sun8i-h3-emac"
+		"allwinner,sun8i-r40-gmac"
 		"allwinner,sun8i-v3s-emac"
 		"allwinner,sun50i-a64-emac"
 - reg: address and length of the register for the device.
@@ -20,18 +21,18 @@ Required properties:
 - phy-handle: See ethernet.txt
 - #address-cells: shall be 1
 - #size-cells: shall be 0
-- syscon: A phandle to the syscon of the SoC with one of the following
- compatible string:
-  - allwinner,sun8i-h3-system-controller
-  - allwinner,sun8i-v3s-system-controller
-  - allwinner,sun50i-a64-system-controller
-  - allwinner,sun8i-a83t-system-controller
+- syscon: A phandle to the device containing the EMAC or GMAC clock register
 
 Optional properties:
-- allwinner,tx-delay-ps: TX clock delay chain value in ps. Range value is 0-700. Default is 0)
-- allwinner,rx-delay-ps: RX clock delay chain value in ps. Range value is 0-3100. Default is 0)
-Both delay properties need to be a multiple of 100. They control the delay for
-external PHY.
+- allwinner,tx-delay-ps: TX clock delay chain value in ps.
+			 Range is 0-700. Default is 0.
+			 Unavailable for allwinner,sun8i-r40-gmac
+- allwinner,rx-delay-ps: RX clock delay chain value in ps.
+			 Range is 0-3100. Default is 0.
+			 Range is 0-700 for allwinner,sun8i-r40-gmac
+Both delay properties need to be a multiple of 100. They control the
+clock delay for external RGMII PHY. They do not apply to the internal
+PHY or external non-RGMII PHYs.
 
 Optional properties for the following compatibles:
   - "allwinner,sun8i-h3-emac",
diff --git a/Documentation/devicetree/bindings/net/fsl-tsec-phy.txt b/Documentation/devicetree/bindings/net/fsl-tsec-phy.txt
index 79bf352e659c..047bdf7bdd2f 100644
--- a/Documentation/devicetree/bindings/net/fsl-tsec-phy.txt
+++ b/Documentation/devicetree/bindings/net/fsl-tsec-phy.txt
@@ -86,70 +86,4 @@ Example:
 
 * Gianfar PTP clock nodes
 
-General Properties:
-
-  - compatible   Should be "fsl,etsec-ptp"
-  - reg          Offset and length of the register set for the device
-  - interrupts   There should be at least two interrupts. Some devices
-                 have as many as four PTP related interrupts.
-
-Clock Properties:
-
-  - fsl,cksel        Timer reference clock source.
-  - fsl,tclk-period  Timer reference clock period in nanoseconds.
-  - fsl,tmr-prsc     Prescaler, divides the output clock.
-  - fsl,tmr-add      Frequency compensation value.
-  - fsl,tmr-fiper1   Fixed interval period pulse generator.
-  - fsl,tmr-fiper2   Fixed interval period pulse generator.
-  - fsl,max-adj      Maximum frequency adjustment in parts per billion.
-
-  These properties set the operational parameters for the PTP
-  clock. You must choose these carefully for the clock to work right.
-  Here is how to figure good values:
-
-  TimerOsc     = selected reference clock   MHz
-  tclk_period  = desired clock period       nanoseconds
-  NominalFreq  = 1000 / tclk_period         MHz
-  FreqDivRatio = TimerOsc / NominalFreq     (must be greater that 1.0)
-  tmr_add      = ceil(2^32 / FreqDivRatio)
-  OutputClock  = NominalFreq / tmr_prsc     MHz
-  PulseWidth   = 1 / OutputClock            microseconds
-  FiperFreq1   = desired frequency in Hz
-  FiperDiv1    = 1000000 * OutputClock / FiperFreq1
-  tmr_fiper1   = tmr_prsc * tclk_period * FiperDiv1 - tclk_period
-  max_adj      = 1000000000 * (FreqDivRatio - 1.0) - 1
-
-  The calculation for tmr_fiper2 is the same as for tmr_fiper1. The
-  driver expects that tmr_fiper1 will be correctly set to produce a 1
-  Pulse Per Second (PPS) signal, since this will be offered to the PPS
-  subsystem to synchronize the Linux clock.
-
-  Reference clock source is determined by the value, which is holded
-  in CKSEL bits in TMR_CTRL register. "fsl,cksel" property keeps the
-  value, which will be directly written in those bits, that is why,
-  according to reference manual, the next clock sources can be used:
-
-  <0> - external high precision timer reference clock (TSEC_TMR_CLK
-        input is used for this purpose);
-  <1> - eTSEC system clock;
-  <2> - eTSEC1 transmit clock;
-  <3> - RTC clock input.
-
-  When this attribute is not used, eTSEC system clock will serve as
-  IEEE 1588 timer reference clock.
-
-Example:
-
-	ptp_clock@24e00 {
-		compatible = "fsl,etsec-ptp";
-		reg = <0x24E00 0xB0>;
-		interrupts = <12 0x8 13 0x8>;
-		interrupt-parent = < &ipic >;
-		fsl,cksel       = <1>;
-		fsl,tclk-period = <10>;
-		fsl,tmr-prsc    = <100>;
-		fsl,tmr-add     = <0x999999A4>;
-		fsl,tmr-fiper1  = <0x3B9AC9F6>;
-		fsl,tmr-fiper2  = <0x00018696>;
-		fsl,max-adj     = <659999998>;
-	};
+Refer to Documentation/devicetree/bindings/ptp/ptp-qoriq.txt
diff --git a/Documentation/devicetree/bindings/net/ibm,emac.txt b/Documentation/devicetree/bindings/net/ibm,emac.txt
new file mode 100644
index 000000000000..44b842b6ca15
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/ibm,emac.txt
@@ -0,0 +1,206 @@
+    4xx/Axon EMAC ethernet nodes
+
+    The EMAC ethernet controller in IBM and AMCC 4xx chips, and also
+    the Axon bridge.  To operate this needs to interact with a this
+    special McMAL DMA controller, and sometimes an RGMII or ZMII
+    interface.  In addition to the nodes and properties described
+    below, the node for the OPB bus on which the EMAC sits must have a
+    correct clock-frequency property.
+
+      i) The EMAC node itself
+
+    Required properties:
+    - device_type       : "network"
+
+    - compatible        : compatible list, contains 2 entries, first is
+			  "ibm,emac-CHIP" where CHIP is the host ASIC (440gx,
+			  405gp, Axon) and second is either "ibm,emac" or
+			  "ibm,emac4".  For Axon, thus, we have: "ibm,emac-axon",
+			  "ibm,emac4"
+    - interrupts        : <interrupt mapping for EMAC IRQ and WOL IRQ>
+    - interrupt-parent  : optional, if needed for interrupt mapping
+    - reg               : <registers mapping>
+    - local-mac-address : 6 bytes, MAC address
+    - mal-device        : phandle of the associated McMAL node
+    - mal-tx-channel    : 1 cell, index of the tx channel on McMAL associated
+			  with this EMAC
+    - mal-rx-channel    : 1 cell, index of the rx channel on McMAL associated
+			  with this EMAC
+    - cell-index        : 1 cell, hardware index of the EMAC cell on a given
+			  ASIC (typically 0x0 and 0x1 for EMAC0 and EMAC1 on
+			  each Axon chip)
+    - max-frame-size    : 1 cell, maximum frame size supported in bytes
+    - rx-fifo-size      : 1 cell, Rx fifo size in bytes for 10 and 100 Mb/sec
+			  operations.
+			  For Axon, 2048
+    - tx-fifo-size      : 1 cell, Tx fifo size in bytes for 10 and 100 Mb/sec
+			  operations.
+			  For Axon, 2048.
+    - fifo-entry-size   : 1 cell, size of a fifo entry (used to calculate
+			  thresholds).
+			  For Axon, 0x00000010
+    - mal-burst-size    : 1 cell, MAL burst size (used to calculate thresholds)
+			  in bytes.
+			  For Axon, 0x00000100 (I think ...)
+    - phy-mode          : string, mode of operations of the PHY interface.
+			  Supported values are: "mii", "rmii", "smii", "rgmii",
+			  "tbi", "gmii", rtbi", "sgmii".
+			  For Axon on CAB, it is "rgmii"
+    - mdio-device       : 1 cell, required iff using shared MDIO registers
+			  (440EP).  phandle of the EMAC to use to drive the
+			  MDIO lines for the PHY used by this EMAC.
+    - zmii-device       : 1 cell, required iff connected to a ZMII.  phandle of
+			  the ZMII device node
+    - zmii-channel      : 1 cell, required iff connected to a ZMII.  Which ZMII
+			  channel or 0xffffffff if ZMII is only used for MDIO.
+    - rgmii-device      : 1 cell, required iff connected to an RGMII. phandle
+			  of the RGMII device node.
+			  For Axon: phandle of plb5/plb4/opb/rgmii
+    - rgmii-channel     : 1 cell, required iff connected to an RGMII.  Which
+			  RGMII channel is used by this EMAC.
+			  Fox Axon: present, whatever value is appropriate for each
+			  EMAC, that is the content of the current (bogus) "phy-port"
+			  property.
+
+    Optional properties:
+    - phy-address       : 1 cell, optional, MDIO address of the PHY. If absent,
+			  a search is performed.
+    - phy-map           : 1 cell, optional, bitmap of addresses to probe the PHY
+			  for, used if phy-address is absent. bit 0x00000001 is
+			  MDIO address 0.
+			  For Axon it can be absent, though my current driver
+			  doesn't handle phy-address yet so for now, keep
+			  0x00ffffff in it.
+    - phy-handle	: Used to describe configurations where a external PHY
+			  is used. Please refer to:
+			  Documentation/devicetree/bindings/net/ethernet.txt
+    - rx-fifo-size-gige : 1 cell, Rx fifo size in bytes for 1000 Mb/sec
+			  operations (if absent the value is the same as
+			  rx-fifo-size).  For Axon, either absent or 2048.
+    - tx-fifo-size-gige : 1 cell, Tx fifo size in bytes for 1000 Mb/sec
+			  operations (if absent the value is the same as
+			  tx-fifo-size). For Axon, either absent or 2048.
+    - tah-device        : 1 cell, optional. If connected to a TAH engine for
+			  offload, phandle of the TAH device node.
+    - tah-channel       : 1 cell, optional. If appropriate, channel used on the
+			  TAH engine.
+    - fixed-link	: Fixed-link subnode describing a link to a non-MDIO
+			  managed entity. See
+			  Documentation/devicetree/bindings/net/fixed-link.txt
+			  for details.
+    - mdio subnode	: When the EMAC has a phy connected to its local
+			  mdio, which us supported by the kernel's network
+			  PHY library in drivers/net/phy, there must be device
+			  tree subnode with the following required properties:
+				- #address-cells: Must be <1>.
+				- #size-cells: Must be <0>.
+
+			  For PHY definitions: Please refer to
+			  Documentation/devicetree/bindings/net/phy.txt and
+			  Documentation/devicetree/bindings/net/ethernet.txt
+
+    Examples:
+
+	EMAC0: ethernet@40000800 {
+		device_type = "network";
+		compatible = "ibm,emac-440gp", "ibm,emac";
+		interrupt-parent = <&UIC1>;
+		interrupts = <1c 4 1d 4>;
+		reg = <40000800 70>;
+		local-mac-address = [00 04 AC E3 1B 1E];
+		mal-device = <&MAL0>;
+		mal-tx-channel = <0 1>;
+		mal-rx-channel = <0>;
+		cell-index = <0>;
+		max-frame-size = <5dc>;
+		rx-fifo-size = <1000>;
+		tx-fifo-size = <800>;
+		phy-mode = "rmii";
+		phy-map = <00000001>;
+		zmii-device = <&ZMII0>;
+		zmii-channel = <0>;
+	};
+
+	EMAC1: ethernet@ef600c00 {
+		device_type = "network";
+		compatible = "ibm,emac-apm821xx", "ibm,emac4sync";
+		interrupt-parent = <&EMAC1>;
+		interrupts = <0 1>;
+		#interrupt-cells = <1>;
+		#address-cells = <0>;
+		#size-cells = <0>;
+		interrupt-map = <0 &UIC2 0x10 IRQ_TYPE_LEVEL_HIGH /* Status */
+				 1 &UIC2 0x14 IRQ_TYPE_LEVEL_HIGH /* Wake */>;
+		reg = <0xef600c00 0x000000c4>;
+		local-mac-address = [000000000000]; /* Filled in by U-Boot */
+		mal-device = <&MAL0>;
+		mal-tx-channel = <0>;
+		mal-rx-channel = <0>;
+		cell-index = <0>;
+		max-frame-size = <9000>;
+		rx-fifo-size = <16384>;
+		tx-fifo-size = <2048>;
+		fifo-entry-size = <10>;
+		phy-mode = "rgmii";
+		phy-handle = <&phy0>;
+		phy-map = <0x00000000>;
+		rgmii-device = <&RGMII0>;
+		rgmii-channel = <0>;
+		tah-device = <&TAH0>;
+		tah-channel = <0>;
+		has-inverted-stacr-oc;
+		has-new-stacr-staopc;
+
+	        mdio {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			phy0: ethernet-phy@0 {
+				compatible = "ethernet-phy-ieee802.3-c22";
+				reg = <0>;
+			};
+		};
+	};
+
+
+      ii) McMAL node
+
+    Required properties:
+    - device_type        : "dma-controller"
+    - compatible         : compatible list, containing 2 entries, first is
+			   "ibm,mcmal-CHIP" where CHIP is the host ASIC (like
+			   emac) and the second is either "ibm,mcmal" or
+			   "ibm,mcmal2".
+			   For Axon, "ibm,mcmal-axon","ibm,mcmal2"
+    - interrupts         : <interrupt mapping for the MAL interrupts sources:
+                           5 sources: tx_eob, rx_eob, serr, txde, rxde>.
+                           For Axon: This is _different_ from the current
+			   firmware.  We use the "delayed" interrupts for txeob
+			   and rxeob. Thus we end up with mapping those 5 MPIC
+			   interrupts, all level positive sensitive: 10, 11, 32,
+			   33, 34 (in decimal)
+    - dcr-reg            : < DCR registers range >
+    - dcr-parent         : if needed for dcr-reg
+    - num-tx-chans       : 1 cell, number of Tx channels
+    - num-rx-chans       : 1 cell, number of Rx channels
+
+      iii) ZMII node
+
+    Required properties:
+    - compatible         : compatible list, containing 2 entries, first is
+			   "ibm,zmii-CHIP" where CHIP is the host ASIC (like
+			   EMAC) and the second is "ibm,zmii".
+			   For Axon, there is no ZMII node.
+    - reg                : <registers mapping>
+
+      iv) RGMII node
+
+    Required properties:
+    - compatible         : compatible list, containing 2 entries, first is
+			   "ibm,rgmii-CHIP" where CHIP is the host ASIC (like
+			   EMAC) and the second is "ibm,rgmii".
+                           For Axon, "ibm,rgmii-axon","ibm,rgmii"
+    - reg                : <registers mapping>
+    - revision           : as provided by the RGMII new version register if
+			   available.
+			   For Axon: 0x0000012a
diff --git a/Documentation/devicetree/bindings/net/meson-dwmac.txt b/Documentation/devicetree/bindings/net/meson-dwmac.txt
index 61cada22ae6c..1321bb194ed9 100644
--- a/Documentation/devicetree/bindings/net/meson-dwmac.txt
+++ b/Documentation/devicetree/bindings/net/meson-dwmac.txt
@@ -11,6 +11,7 @@ Required properties on all platforms:
 			- "amlogic,meson8b-dwmac"
 			- "amlogic,meson8m2-dwmac"
 			- "amlogic,meson-gxbb-dwmac"
+			- "amlogic,meson-axg-dwmac"
 		Additionally "snps,dwmac" and any applicable more
 		detailed version number described in net/stmmac.txt
 		should be used.
diff --git a/Documentation/devicetree/bindings/net/microchip,lan78xx.txt b/Documentation/devicetree/bindings/net/microchip,lan78xx.txt
new file mode 100644
index 000000000000..76786a0f6d3d
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/microchip,lan78xx.txt
@@ -0,0 +1,54 @@
+Microchip LAN78xx Gigabit Ethernet controller
+
+The LAN78XX devices are usually configured by programming their OTP or with
+an external EEPROM, but some platforms (e.g. Raspberry Pi 3 B+) have neither.
+The Device Tree properties, if present, override the OTP and EEPROM.
+
+Required properties:
+- compatible: Should be one of "usb424,7800", "usb424,7801" or "usb424,7850".
+
+Optional properties:
+- local-mac-address:   see ethernet.txt
+- mac-address:         see ethernet.txt
+
+Optional properties of the embedded PHY:
+- microchip,led-modes: a 0..4 element vector, with each element configuring
+  the operating mode of an LED. Omitted LEDs are turned off. Allowed values
+  are defined in "include/dt-bindings/net/microchip-lan78xx.h".
+
+Example:
+
+/* Based on the configuration for a Raspberry Pi 3 B+ */
+&usb {
+	usb-port@1 {
+		compatible = "usb424,2514";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		usb-port@1 {
+			compatible = "usb424,2514";
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			ethernet: ethernet@1 {
+				compatible = "usb424,7800";
+				reg = <1>;
+				local-mac-address = [ 00 11 22 33 44 55 ];
+
+				mdio {
+					#address-cells = <0x1>;
+					#size-cells = <0x0>;
+					eth_phy: ethernet-phy@1 {
+						reg = <1>;
+						microchip,led-modes = <
+							LAN78XX_LINK_1000_ACTIVITY
+							LAN78XX_LINK_10_100_ACTIVITY
+						>;
+					};
+				};
+			};
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/net/mscc-miim.txt b/Documentation/devicetree/bindings/net/mscc-miim.txt
new file mode 100644
index 000000000000..7104679cf59d
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/mscc-miim.txt
@@ -0,0 +1,26 @@
+Microsemi MII Management Controller (MIIM) / MDIO
+=================================================
+
+Properties:
+- compatible: must be "mscc,ocelot-miim"
+- reg: The base address of the MDIO bus controller register bank. Optionally, a
+  second register bank can be defined if there is an associated reset register
+  for internal PHYs
+- #address-cells: Must be <1>.
+- #size-cells: Must be <0>.  MDIO addresses have no size component.
+- interrupts: interrupt specifier (refer to the interrupt binding)
+
+Typically an MDIO bus might have several children.
+
+Example:
+	mdio@107009c {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "mscc,ocelot-miim";
+		reg = <0x107009c 0x36>, <0x10700f0 0x8>;
+		interrupts = <14>;
+
+		phy0: ethernet-phy@0 {
+			reg = <0>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/net/mscc-ocelot.txt b/Documentation/devicetree/bindings/net/mscc-ocelot.txt
new file mode 100644
index 000000000000..0a84711abece
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/mscc-ocelot.txt
@@ -0,0 +1,82 @@
+Microsemi Ocelot network Switch
+===============================
+
+The Microsemi Ocelot network switch can be found on Microsemi SoCs (VSC7513,
+VSC7514)
+
+Required properties:
+- compatible: Should be "mscc,vsc7514-switch"
+- reg: Must contain an (offset, length) pair of the register set for each
+  entry in reg-names.
+- reg-names: Must include the following entries:
+  - "sys"
+  - "rew"
+  - "qs"
+  - "hsio"
+  - "qsys"
+  - "ana"
+  - "portX" with X from 0 to the number of last port index available on that
+    switch
+- interrupts: Should contain the switch interrupts for frame extraction and
+  frame injection
+- interrupt-names: should contain the interrupt names: "xtr", "inj"
+- ethernet-ports: A container for child nodes representing switch ports.
+
+The ethernet-ports container has the following properties
+
+Required properties:
+
+- #address-cells: Must be 1
+- #size-cells: Must be 0
+
+Each port node must have the following mandatory properties:
+- reg: Describes the port address in the switch
+
+Port nodes may also contain the following optional standardised
+properties, described in binding documents:
+
+- phy-handle: Phandle to a PHY on an MDIO bus. See
+  Documentation/devicetree/bindings/net/ethernet.txt for details.
+
+Example:
+
+	switch@1010000 {
+		compatible = "mscc,vsc7514-switch";
+		reg = <0x1010000 0x10000>,
+		      <0x1030000 0x10000>,
+		      <0x1080000 0x100>,
+		      <0x10d0000 0x10000>,
+		      <0x11e0000 0x100>,
+		      <0x11f0000 0x100>,
+		      <0x1200000 0x100>,
+		      <0x1210000 0x100>,
+		      <0x1220000 0x100>,
+		      <0x1230000 0x100>,
+		      <0x1240000 0x100>,
+		      <0x1250000 0x100>,
+		      <0x1260000 0x100>,
+		      <0x1270000 0x100>,
+		      <0x1280000 0x100>,
+		      <0x1800000 0x80000>,
+		      <0x1880000 0x10000>;
+		reg-names = "sys", "rew", "qs", "hsio", "port0",
+			    "port1", "port2", "port3", "port4", "port5",
+			    "port6", "port7", "port8", "port9", "port10",
+			    "qsys", "ana";
+		interrupts = <21 22>;
+		interrupt-names = "xtr", "inj";
+
+		ethernet-ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port0: port@0 {
+				reg = <0>;
+				phy-handle = <&phy0>;
+			};
+			port1: port@1 {
+				reg = <1>;
+				phy-handle = <&phy1>;
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/net/qualcomm-bluetooth.txt b/Documentation/devicetree/bindings/net/qualcomm-bluetooth.txt
new file mode 100644
index 000000000000..0ea18a53cc29
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/qualcomm-bluetooth.txt
@@ -0,0 +1,30 @@
+Qualcomm Bluetooth Chips
+---------------------
+
+This documents the binding structure and common properties for serial
+attached Qualcomm devices.
+
+Serial attached Qualcomm devices shall be a child node of the host UART
+device the slave device is attached to.
+
+Required properties:
+ - compatible: should contain one of the following:
+   * "qcom,qca6174-bt"
+
+Optional properties:
+ - enable-gpios: gpio specifier used to enable chip
+ - clocks: clock provided to the controller (SUSCLK_32KHZ)
+
+Example:
+
+serial@7570000 {
+	label = "BT-UART";
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,qca6174-bt";
+
+		enable-gpios = <&pm8994_gpios 19 GPIO_ACTIVE_HIGH>;
+		clocks = <&divclk4>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/net/sff,sfp.txt b/Documentation/devicetree/bindings/net/sff,sfp.txt
index 929591d52ed6..832139919f20 100644
--- a/Documentation/devicetree/bindings/net/sff,sfp.txt
+++ b/Documentation/devicetree/bindings/net/sff,sfp.txt
@@ -7,11 +7,11 @@ Required properties:
   "sff,sfp" for SFP modules
   "sff,sff" for soldered down SFF modules
 
-Optional Properties:
-
 - i2c-bus : phandle of an I2C bus controller for the SFP two wire serial
   interface
 
+Optional Properties:
+
 - mod-def0-gpios : GPIO phandle and a specifier of the MOD-DEF0 (AKA Mod_ABS)
   module presence input gpio signal, active (module absent) high. Must
   not be present for SFF modules
diff --git a/Documentation/devicetree/bindings/net/sh_eth.txt b/Documentation/devicetree/bindings/net/sh_eth.txt
index 5172799a7f1a..82a4cf2c145d 100644
--- a/Documentation/devicetree/bindings/net/sh_eth.txt
+++ b/Documentation/devicetree/bindings/net/sh_eth.txt
@@ -14,6 +14,7 @@ Required properties:
 	      "renesas,ether-r8a7791"  if the device is a part of R8A7791 SoC.
 	      "renesas,ether-r8a7793"  if the device is a part of R8A7793 SoC.
 	      "renesas,ether-r8a7794"  if the device is a part of R8A7794 SoC.
+	      "renesas,gether-r8a77980" if the device is a part of R8A77980 SoC.
 	      "renesas,ether-r7s72100" if the device is a part of R7S72100 SoC.
 	      "renesas,rcar-gen1-ether" for a generic R-Car Gen1 device.
 	      "renesas,rcar-gen2-ether" for a generic R-Car Gen2 or RZ/G1
diff --git a/Documentation/devicetree/bindings/net/socionext,uniphier-ave4.txt b/Documentation/devicetree/bindings/net/socionext,uniphier-ave4.txt
index 96398cc2982f..fc8f01718690 100644
--- a/Documentation/devicetree/bindings/net/socionext,uniphier-ave4.txt
+++ b/Documentation/devicetree/bindings/net/socionext,uniphier-ave4.txt
@@ -13,13 +13,25 @@ Required properties:
  - reg: Address where registers are mapped and size of region.
  - interrupts: Should contain the MAC interrupt.
  - phy-mode: See ethernet.txt in the same directory. Allow to choose
-	"rgmii", "rmii", or "mii" according to the PHY.
+	"rgmii", "rmii", "mii", or "internal" according to the PHY.
+	The acceptable mode is SoC-dependent.
  - phy-handle: Should point to the external phy device.
 	See ethernet.txt file in the same directory.
  - clocks: A phandle to the clock for the MAC.
+	For Pro4 SoC, that is "socionext,uniphier-pro4-ave4",
+	another MAC clock, GIO bus clock and PHY clock are also required.
+ - clock-names: Should contain
+	- "ether", "ether-gb", "gio", "ether-phy" for Pro4 SoC
+	- "ether" for others
+ - resets: A phandle to the reset control for the MAC. For Pro4 SoC,
+	GIO bus reset is also required.
+ - reset-names: Should contain
+	- "ether", "gio" for Pro4 SoC
+	- "ether" for others
+ - socionext,syscon-phy-mode: A phandle to syscon with one argument
+	that configures phy mode. The argument is the ID of MAC instance.
 
 Optional properties:
- - resets: A phandle to the reset control for the MAC.
  - local-mac-address: See ethernet.txt in the same directory.
 
 Required subnode:
@@ -34,8 +46,11 @@ Example:
 		interrupts = <0 66 4>;
 		phy-mode = "rgmii";
 		phy-handle = <&ethphy>;
+		clock-names = "ether";
 		clocks = <&sys_clk 6>;
+		reset-names = "ether";
 		resets = <&sys_rst 6>;
+		socionext,syscon-phy-mode = <&soc_glue 0>;
 		local-mac-address = [00 00 00 00 00 00];
 
 		mdio {
diff --git a/Documentation/devicetree/bindings/net/stm32-dwmac.txt b/Documentation/devicetree/bindings/net/stm32-dwmac.txt
index 489dbcb66c5a..1341012722aa 100644
--- a/Documentation/devicetree/bindings/net/stm32-dwmac.txt
+++ b/Documentation/devicetree/bindings/net/stm32-dwmac.txt
@@ -6,14 +6,28 @@ Please see stmmac.txt for the other unchanged properties.
 The device node has following properties.
 
 Required properties:
-- compatible:  Should be "st,stm32-dwmac" to select glue, and
+- compatible:  For MCU family should be "st,stm32-dwmac" to select glue, and
 	       "snps,dwmac-3.50a" to select IP version.
+	       For MPU family should be "st,stm32mp1-dwmac" to select
+	       glue, and "snps,dwmac-4.20a" to select IP version.
 - clocks: Must contain a phandle for each entry in clock-names.
 - clock-names: Should be "stmmaceth" for the host clock.
 	       Should be "mac-clk-tx" for the MAC TX clock.
 	       Should be "mac-clk-rx" for the MAC RX clock.
+	       For MPU family need to add also "ethstp" for power mode clock and,
+	                                       "syscfg-clk" for SYSCFG clock.
+- interrupt-names: Should contain a list of interrupt names corresponding to
+           the interrupts in the interrupts property, if available.
+		   Should be "macirq" for the main MAC IRQ
+		   Should be "eth_wake_irq" for the IT which wake up system
 - st,syscon : Should be phandle/offset pair. The phandle to the syscon node which
-	      encompases the glue register, and the offset of the control register.
+	       encompases the glue register, and the offset of the control register.
+
+Optional properties:
+- clock-names:     For MPU family "mac-clk-ck" for PHY without quartz
+- st,int-phyclk (boolean) :  valid only where PHY do not have quartz and need to be clock
+	           by RCC
+
 Example:
 
 	ethernet@40028000 {
diff --git a/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.txt b/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.txt
index 3d2a031217da..7fd4e8ce4149 100644
--- a/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.txt
+++ b/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.txt
@@ -4,6 +4,7 @@ Required properties:
 - compatible: Should be one of the following:
 	* "qcom,ath10k"
 	* "qcom,ipq4019-wifi"
+	* "qcom,wcn3990-wifi"
 
 PCI based devices uses compatible string "qcom,ath10k" and takes calibration
 data along with board specific data via "qcom,ath10k-calibration-data".
@@ -18,8 +19,12 @@ In general, entry "qcom,ath10k-pre-calibration-data" and
 "qcom,ath10k-calibration-data" conflict with each other and only one
 can be provided per device.
 
+SNOC based devices (i.e. wcn3990) uses compatible string "qcom,wcn3990-wifi".
+
 Optional properties:
 - reg: Address and length of the register set for the device.
+- reg-names: Must include the list of following reg names,
+	     "membase"
 - resets: Must contain an entry for each entry in reset-names.
           See ../reset/reseti.txt for details.
 - reset-names: Must include the list of following reset names,
@@ -49,6 +54,8 @@ Optional properties:
 				 hw versions.
 - qcom,ath10k-pre-calibration-data : pre calibration data as an array,
 				     the length can vary between hw versions.
+- <supply-name>-supply: handle to the regulator device tree node
+			   optional "supply-name" is "vdd-0.8-cx-mx".
 
 Example (to supply the calibration data alone):
 
@@ -119,3 +126,27 @@ wifi0: wifi@a000000 {
 	qcom,msi_base = <0x40>;
 	qcom,ath10k-pre-calibration-data = [ 01 02 03 ... ];
 };
+
+Example (to supply wcn3990 SoC wifi block details):
+
+wifi@18000000 {
+		compatible = "qcom,wcn3990-wifi";
+		reg = <0x18800000 0x800000>;
+		reg-names = "membase";
+		clocks = <&clock_gcc clk_aggre2_noc_clk>;
+		clock-names = "smmu_aggre2_noc_clk"
+		interrupts =
+			   <0 130 0 /* CE0 */ >,
+			   <0 131 0 /* CE1 */ >,
+			   <0 132 0 /* CE2 */ >,
+			   <0 133 0 /* CE3 */ >,
+			   <0 134 0 /* CE4 */ >,
+			   <0 135 0 /* CE5 */ >,
+			   <0 136 0 /* CE6 */ >,
+			   <0 137 0 /* CE7 */ >,
+			   <0 138 0 /* CE8 */ >,
+			   <0 139 0 /* CE9 */ >,
+			   <0 140 0 /* CE10 */ >,
+			   <0 141 0 /* CE11 */ >;
+		vdd-0.8-cx-mx-supply = <&pm8998_l5>;
+};
diff --git a/Documentation/devicetree/bindings/nios2/timer.txt b/Documentation/devicetree/bindings/nios2/timer.txt
deleted file mode 100644
index 904a5846d7ac..000000000000
--- a/Documentation/devicetree/bindings/nios2/timer.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-Altera Timer
-
-Required properties:
-
-- compatible : should be "altr,timer-1.0"
-- reg : Specifies base physical address and size of the registers.
-- interrupt-parent: phandle of the interrupt controller
-- interrupts : Should contain the timer interrupt number
-- clock-frequency : The frequency of the clock that drives the counter, in Hz.
-
-Example:
-
-timer {
-	compatible = "altr,timer-1.0";
-	reg = <0x00400000 0x00000020>;
-	interrupt-parent = <&cpu>;
-	interrupts = <11>;
-	clock-frequency = <125000000>;
-};
diff --git a/Documentation/devicetree/bindings/nvmem/allwinner,sunxi-sid.txt b/Documentation/devicetree/bindings/nvmem/allwinner,sunxi-sid.txt
index d69543701d5d..e319fe5e205a 100644
--- a/Documentation/devicetree/bindings/nvmem/allwinner,sunxi-sid.txt
+++ b/Documentation/devicetree/bindings/nvmem/allwinner,sunxi-sid.txt
@@ -4,6 +4,7 @@ Required properties:
 - compatible: Should be one of the following:
   "allwinner,sun4i-a10-sid"
   "allwinner,sun7i-a20-sid"
+  "allwinner,sun8i-a83t-sid"
   "allwinner,sun8i-h3-sid"
   "allwinner,sun50i-a64-sid"
 
diff --git a/Documentation/devicetree/bindings/nvmem/zii,rave-sp-eeprom.txt b/Documentation/devicetree/bindings/nvmem/zii,rave-sp-eeprom.txt
new file mode 100644
index 000000000000..d5e22fc67d66
--- /dev/null
+++ b/Documentation/devicetree/bindings/nvmem/zii,rave-sp-eeprom.txt
@@ -0,0 +1,40 @@
+Zodiac Inflight Innovations RAVE EEPROM Bindings
+
+RAVE SP EEPROM device is a "MFD cell" device exposing physical EEPROM
+attached to RAVE Supervisory Processor. It is expected that its Device
+Tree node is specified as a child of the node corresponding to the
+parent RAVE SP device (as documented in
+Documentation/devicetree/bindings/mfd/zii,rave-sp.txt)
+
+Required properties:
+
+- compatible: Should be "zii,rave-sp-eeprom"
+
+Optional properties:
+
+- zii,eeprom-name: Unique EEPROM identifier describing its function in the
+  system. Will be used as created NVMEM deivce's name.
+
+Data cells:
+
+Data cells are child nodes of eerpom node, bindings for which are
+documented in Documentation/bindings/nvmem/nvmem.txt
+
+Example:
+
+	rave-sp {
+		compatible = "zii,rave-sp-rdu1";
+		current-speed = <38400>;
+
+		eeprom@a4 {
+			compatible = "zii,rave-sp-eeprom";
+			reg = <0xa4 0x4000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			zii,eeprom-name = "main-eeprom";
+
+			wdt_timeout: wdt-timeout@81 {
+				reg = <0x81 2>;
+			};
+		};
+	}
diff --git a/Documentation/devicetree/bindings/opp/kryo-cpufreq.txt b/Documentation/devicetree/bindings/opp/kryo-cpufreq.txt
new file mode 100644
index 000000000000..c2127b96805a
--- /dev/null
+++ b/Documentation/devicetree/bindings/opp/kryo-cpufreq.txt
@@ -0,0 +1,680 @@
+Qualcomm Technologies, Inc. KRYO CPUFreq and OPP bindings
+===================================
+
+In Certain Qualcomm Technologies, Inc. SoCs like apq8096 and msm8996
+that have KRYO processors, the CPU ferequencies subset and voltage value
+of each OPP varies based on the silicon variant in use.
+Qualcomm Technologies, Inc. Process Voltage Scaling Tables
+defines the voltage and frequency value based on the msm-id in SMEM
+and speedbin blown in the efuse combination.
+The qcom-cpufreq-kryo driver reads the msm-id and efuse value from the SoC
+to provide the OPP framework with required information (existing HW bitmap).
+This is used to determine the voltage and frequency value for each OPP of
+operating-points-v2 table when it is parsed by the OPP framework.
+
+Required properties:
+--------------------
+In 'cpus' nodes:
+- operating-points-v2: Phandle to the operating-points-v2 table to use.
+
+In 'operating-points-v2' table:
+- compatible: Should be
+	- 'operating-points-v2-kryo-cpu' for apq8096 and msm8996.
+- nvmem-cells: A phandle pointing to a nvmem-cells node representing the
+		efuse registers that has information about the
+		speedbin that is used to select the right frequency/voltage
+		value pair.
+		Please refer the for nvmem-cells
+		bindings Documentation/devicetree/bindings/nvmem/nvmem.txt
+		and also examples below.
+
+In every OPP node:
+- opp-supported-hw: A single 32 bit bitmap value, representing compatible HW.
+		    Bitmap:
+			0:	MSM8996 V3, speedbin 0
+			1:	MSM8996 V3, speedbin 1
+			2:	MSM8996 V3, speedbin 2
+			3:	unused
+			4:	MSM8996 SG, speedbin 0
+			5:	MSM8996 SG, speedbin 1
+			6:	MSM8996 SG, speedbin 2
+			7-31:	unused
+
+Example 1:
+---------
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		CPU0: cpu@0 {
+			device_type = "cpu";
+			compatible = "qcom,kryo";
+			reg = <0x0 0x0>;
+			enable-method = "psci";
+			clocks = <&kryocc 0>;
+			cpu-supply = <&pm8994_s11_saw>;
+			operating-points-v2 = <&cluster0_opp>;
+			#cooling-cells = <2>;
+			next-level-cache = <&L2_0>;
+			L2_0: l2-cache {
+			      compatible = "cache";
+			      cache-level = <2>;
+			};
+		};
+
+		CPU1: cpu@1 {
+			device_type = "cpu";
+			compatible = "qcom,kryo";
+			reg = <0x0 0x1>;
+			enable-method = "psci";
+			clocks = <&kryocc 0>;
+			cpu-supply = <&pm8994_s11_saw>;
+			operating-points-v2 = <&cluster0_opp>;
+			#cooling-cells = <2>;
+			next-level-cache = <&L2_0>;
+		};
+
+		CPU2: cpu@100 {
+			device_type = "cpu";
+			compatible = "qcom,kryo";
+			reg = <0x0 0x100>;
+			enable-method = "psci";
+			clocks = <&kryocc 1>;
+			cpu-supply = <&pm8994_s11_saw>;
+			operating-points-v2 = <&cluster1_opp>;
+			#cooling-cells = <2>;
+			next-level-cache = <&L2_1>;
+			L2_1: l2-cache {
+			      compatible = "cache";
+			      cache-level = <2>;
+			};
+		};
+
+		CPU3: cpu@101 {
+			device_type = "cpu";
+			compatible = "qcom,kryo";
+			reg = <0x0 0x101>;
+			enable-method = "psci";
+			clocks = <&kryocc 1>;
+			cpu-supply = <&pm8994_s11_saw>;
+			operating-points-v2 = <&cluster1_opp>;
+			#cooling-cells = <2>;
+			next-level-cache = <&L2_1>;
+		};
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&CPU0>;
+				};
+
+				core1 {
+					cpu = <&CPU1>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&CPU2>;
+				};
+
+				core1 {
+					cpu = <&CPU3>;
+				};
+			};
+		};
+	};
+
+	cluster0_opp: opp_table0 {
+		compatible = "operating-points-v2-kryo-cpu";
+		nvmem-cells = <&speedbin_efuse>;
+		opp-shared;
+
+		opp-307200000 {
+			opp-hz = /bits/ 64 <307200000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x77>;
+			clock-latency-ns = <200000>;
+		};
+		opp-384000000 {
+			opp-hz = /bits/ 64 <384000000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-422400000 {
+			opp-hz = /bits/ 64 <422400000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-460800000 {
+			opp-hz = /bits/ 64 <460800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-480000000 {
+			opp-hz = /bits/ 64 <480000000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-537600000 {
+			opp-hz = /bits/ 64 <537600000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-556800000 {
+			opp-hz = /bits/ 64 <556800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-614400000 {
+			opp-hz = /bits/ 64 <614400000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-652800000 {
+			opp-hz = /bits/ 64 <652800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-691200000 {
+			opp-hz = /bits/ 64 <691200000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-729600000 {
+			opp-hz = /bits/ 64 <729600000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-768000000 {
+			opp-hz = /bits/ 64 <768000000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-844800000 {
+			opp-hz = /bits/ 64 <844800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x77>;
+			clock-latency-ns = <200000>;
+		};
+		opp-902400000 {
+			opp-hz = /bits/ 64 <902400000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-960000000 {
+			opp-hz = /bits/ 64 <960000000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-979200000 {
+			opp-hz = /bits/ 64 <979200000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1036800000 {
+			opp-hz = /bits/ 64 <1036800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1056000000 {
+			opp-hz = /bits/ 64 <1056000000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1113600000 {
+			opp-hz = /bits/ 64 <1113600000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1132800000 {
+			opp-hz = /bits/ 64 <1132800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1190400000 {
+			opp-hz = /bits/ 64 <1190400000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1209600000 {
+			opp-hz = /bits/ 64 <1209600000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1228800000 {
+			opp-hz = /bits/ 64 <1228800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1286400000 {
+			opp-hz = /bits/ 64 <1286400000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1324800000 {
+			opp-hz = /bits/ 64 <1324800000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x5>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1363200000 {
+			opp-hz = /bits/ 64 <1363200000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x72>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1401600000 {
+			opp-hz = /bits/ 64 <1401600000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x5>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1440000000 {
+			opp-hz = /bits/ 64 <1440000000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1478400000 {
+			opp-hz = /bits/ 64 <1478400000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x1>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1497600000 {
+			opp-hz = /bits/ 64 <1497600000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x4>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1516800000 {
+			opp-hz = /bits/ 64 <1516800000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1593600000 {
+			opp-hz = /bits/ 64 <1593600000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x71>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1996800000 {
+			opp-hz = /bits/ 64 <1996800000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x20>;
+			clock-latency-ns = <200000>;
+		};
+		opp-2188800000 {
+			opp-hz = /bits/ 64 <2188800000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x10>;
+			clock-latency-ns = <200000>;
+		};
+	};
+
+	cluster1_opp: opp_table1 {
+		compatible = "operating-points-v2-kryo-cpu";
+		nvmem-cells = <&speedbin_efuse>;
+		opp-shared;
+
+		opp-307200000 {
+			opp-hz = /bits/ 64 <307200000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x77>;
+			clock-latency-ns = <200000>;
+		};
+		opp-384000000 {
+			opp-hz = /bits/ 64 <384000000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-403200000 {
+			opp-hz = /bits/ 64 <403200000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-460800000 {
+			opp-hz = /bits/ 64 <460800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-480000000 {
+			opp-hz = /bits/ 64 <480000000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-537600000 {
+			opp-hz = /bits/ 64 <537600000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-556800000 {
+			opp-hz = /bits/ 64 <556800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-614400000 {
+			opp-hz = /bits/ 64 <614400000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-652800000 {
+			opp-hz = /bits/ 64 <652800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-691200000 {
+			opp-hz = /bits/ 64 <691200000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-729600000 {
+			opp-hz = /bits/ 64 <729600000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-748800000 {
+			opp-hz = /bits/ 64 <748800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-806400000 {
+			opp-hz = /bits/ 64 <806400000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-825600000 {
+			opp-hz = /bits/ 64 <825600000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-883200000 {
+			opp-hz = /bits/ 64 <883200000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-902400000 {
+			opp-hz = /bits/ 64 <902400000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-940800000 {
+			opp-hz = /bits/ 64 <940800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-979200000 {
+			opp-hz = /bits/ 64 <979200000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1036800000 {
+			opp-hz = /bits/ 64 <1036800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1056000000 {
+			opp-hz = /bits/ 64 <1056000000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1113600000 {
+			opp-hz = /bits/ 64 <1113600000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1132800000 {
+			opp-hz = /bits/ 64 <1132800000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1190400000 {
+			opp-hz = /bits/ 64 <1190400000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1209600000 {
+			opp-hz = /bits/ 64 <1209600000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1248000000 {
+			opp-hz = /bits/ 64 <1248000000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1286400000 {
+			opp-hz = /bits/ 64 <1286400000>;
+			opp-microvolt = <905000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1324800000 {
+			opp-hz = /bits/ 64 <1324800000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1363200000 {
+			opp-hz = /bits/ 64 <1363200000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1401600000 {
+			opp-hz = /bits/ 64 <1401600000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1440000000 {
+			opp-hz = /bits/ 64 <1440000000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1478400000 {
+			opp-hz = /bits/ 64 <1478400000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1516800000 {
+			opp-hz = /bits/ 64 <1516800000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1555200000 {
+			opp-hz = /bits/ 64 <1555200000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1593600000 {
+			opp-hz = /bits/ 64 <1593600000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1632000000 {
+			opp-hz = /bits/ 64 <1632000000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1670400000 {
+			opp-hz = /bits/ 64 <1670400000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1708800000 {
+			opp-hz = /bits/ 64 <1708800000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1747200000 {
+			opp-hz = /bits/ 64 <1747200000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x70>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1785600000 {
+			opp-hz = /bits/ 64 <1785600000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1804800000 {
+			opp-hz = /bits/ 64 <1804800000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x6>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1824000000 {
+			opp-hz = /bits/ 64 <1824000000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x71>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1900800000 {
+			opp-hz = /bits/ 64 <1900800000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x74>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1920000000 {
+			opp-hz = /bits/ 64 <1920000000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x1>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1977600000 {
+			opp-hz = /bits/ 64 <1977600000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x30>;
+			clock-latency-ns = <200000>;
+		};
+		opp-1996800000 {
+			opp-hz = /bits/ 64 <1996800000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x1>;
+			clock-latency-ns = <200000>;
+		};
+		opp-2054400000 {
+			opp-hz = /bits/ 64 <2054400000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x30>;
+			clock-latency-ns = <200000>;
+		};
+		opp-2073600000 {
+			opp-hz = /bits/ 64 <2073600000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x1>;
+			clock-latency-ns = <200000>;
+		};
+		opp-2150400000 {
+			opp-hz = /bits/ 64 <2150400000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x31>;
+			clock-latency-ns = <200000>;
+		};
+		opp-2246400000 {
+			opp-hz = /bits/ 64 <2246400000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x10>;
+			clock-latency-ns = <200000>;
+		};
+		opp-2342400000 {
+			opp-hz = /bits/ 64 <2342400000>;
+			opp-microvolt = <1140000 905000 1140000>;
+			opp-supported-hw = <0x10>;
+			clock-latency-ns = <200000>;
+		};
+	};
+
+....
+
+reserved-memory {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+....
+	smem_mem: smem-mem@86000000 {
+		reg = <0x0 0x86000000 0x0 0x200000>;
+		no-map;
+	};
+....
+};
+
+smem {
+	compatible = "qcom,smem";
+	memory-region = <&smem_mem>;
+	hwlocks = <&tcsr_mutex 3>;
+};
+
+soc {
+....
+	qfprom: qfprom@74000 {
+		compatible = "qcom,qfprom";
+		reg = <0x00074000 0x8ff>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		....
+		speedbin_efuse: speedbin@133 {
+			reg = <0x133 0x1>;
+			bits = <5 3>;
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/opp/opp.txt b/Documentation/devicetree/bindings/opp/opp.txt
index 4e4f30288c8b..c396c4c0af92 100644
--- a/Documentation/devicetree/bindings/opp/opp.txt
+++ b/Documentation/devicetree/bindings/opp/opp.txt
@@ -82,7 +82,10 @@ This defines voltage-current-frequency combinations along with other related
 properties.
 
 Required properties:
-- opp-hz: Frequency in Hz, expressed as a 64-bit big-endian integer.
+- opp-hz: Frequency in Hz, expressed as a 64-bit big-endian integer. This is a
+  required property for all device nodes but devices like power domains. The
+  power domain nodes must have another (implementation dependent) property which
+  uniquely identifies the OPP nodes.
 
 Optional properties:
 - opp-microvolt: voltage in micro Volts.
@@ -159,7 +162,7 @@ Optional properties:
 
 - status: Marks the node enabled/disabled.
 
-- required-opp: This contains phandle to an OPP node in another device's OPP
+- required-opps: This contains phandle to an OPP node in another device's OPP
   table. It may contain an array of phandles, where each phandle points to an
   OPP of a different device. It should not contain multiple phandles to the OPP
   nodes in the same OPP table. This specifies the minimum required OPP of the
diff --git a/Documentation/devicetree/bindings/pci/designware-pcie.txt b/Documentation/devicetree/bindings/pci/designware-pcie.txt
index 1da7ade3183c..c124f9bc11f3 100644
--- a/Documentation/devicetree/bindings/pci/designware-pcie.txt
+++ b/Documentation/devicetree/bindings/pci/designware-pcie.txt
@@ -1,7 +1,9 @@
 * Synopsys DesignWare PCIe interface
 
 Required properties:
-- compatible: should contain "snps,dw-pcie" to identify the core.
+- compatible:
+	"snps,dw-pcie" for RC mode;
+	"snps,dw-pcie-ep" for EP mode;
 - reg: Should contain the configuration address space.
 - reg-names: Must be "config" for the PCIe configuration space.
     (The old way of getting the configuration address space from "ranges"
@@ -41,11 +43,11 @@ EP mode:
 
 Example configuration:
 
-	pcie: pcie@dffff000 {
+	pcie: pcie@dfc00000 {
 		compatible = "snps,dw-pcie";
-		reg = <0xdffff000 0x1000>, /* Controller registers */
-		      <0xd0000000 0x2000>; /* PCI config space */
-		reg-names = "ctrlreg", "config";
+		reg = <0xdfc00000 0x0001000>, /* IP registers */
+		      <0xd0000000 0x0002000>; /* Configuration space */
+		reg-names = "dbi", "config";
 		#address-cells = <3>;
 		#size-cells = <2>;
 		device_type = "pci";
@@ -54,5 +56,15 @@ Example configuration:
 		interrupts = <25>, <24>;
 		#interrupt-cells = <1>;
 		num-lanes = <1>;
-		num-viewport = <3>;
+	};
+or
+	pcie: pcie@dfc00000 {
+		compatible = "snps,dw-pcie-ep";
+		reg = <0xdfc00000 0x0001000>, /* IP registers 1 */
+		      <0xdfc01000 0x0001000>, /* IP registers 2 */
+		      <0xd0000000 0x2000000>; /* Configuration space */
+		reg-names = "dbi", "dbi2", "addr_space";
+		num-ib-windows = <6>;
+		num-ob-windows = <2>;
+		num-lanes = <1>;
 	};
diff --git a/Documentation/devicetree/bindings/pci/mobiveil-pcie.txt b/Documentation/devicetree/bindings/pci/mobiveil-pcie.txt
new file mode 100644
index 000000000000..65038aa642e5
--- /dev/null
+++ b/Documentation/devicetree/bindings/pci/mobiveil-pcie.txt
@@ -0,0 +1,73 @@
+* Mobiveil AXI PCIe Root Port Bridge DT description
+
+Mobiveil's GPEX 4.0 is a PCIe Gen4 root port bridge IP. This configurable IP
+has up to 8 outbound and inbound windows for the address translation.
+
+Required properties:
+- #address-cells: Address representation for root ports, set to <3>
+- #size-cells: Size representation for root ports, set to <2>
+- #interrupt-cells: specifies the number of cells needed to encode an
+	interrupt source. The value must be 1.
+- compatible: Should contain "mbvl,gpex40-pcie"
+- reg: Should contain PCIe registers location and length
+	"config_axi_slave": PCIe controller registers
+	"csr_axi_slave"	  : Bridge config registers
+	"gpio_slave"	  : GPIO registers to control slot power
+	"apb_csr"	  : MSI registers
+
+- device_type: must be "pci"
+- apio-wins : number of requested apio outbound windows
+		default 2 outbound windows are configured -
+		1. Config window
+		2. Memory window
+- ppio-wins : number of requested ppio inbound windows
+		default 1 inbound memory window is configured.
+- bus-range: PCI bus numbers covered
+- interrupt-controller: identifies the node as an interrupt controller
+- #interrupt-cells: specifies the number of cells needed to encode an
+	interrupt source. The value must be 1.
+- interrupt-parent : phandle to the interrupt controller that
+		it is attached to, it should be set to gic to point to
+		ARM's Generic Interrupt Controller node in system DT.
+- interrupts: The interrupt line of the PCIe controller
+		last cell of this field is set to 4 to
+		denote it as IRQ_TYPE_LEVEL_HIGH type interrupt.
+- interrupt-map-mask,
+	interrupt-map: standard PCI properties to define the mapping of the
+	PCI interface to interrupt numbers.
+- ranges: ranges for the PCI memory regions (I/O space region is not
+	supported by hardware)
+	Please refer to the standard PCI bus binding document for a more
+	detailed explanation
+
+
+Example:
+++++++++
+	pcie0: pcie@a0000000 {
+		#address-cells = <3>;
+		#size-cells = <2>;
+		compatible = "mbvl,gpex40-pcie";
+		reg =	<0xa0000000 0x00001000>,
+			<0xb0000000 0x00010000>,
+			<0xff000000 0x00200000>,
+			<0xb0010000 0x00001000>;
+		reg-names =	"config_axi_slave",
+				"csr_axi_slave",
+				"gpio_slave",
+				"apb_csr";
+		device_type = "pci";
+		apio-wins = <2>;
+		ppio-wins = <1>;
+		bus-range = <0x00000000 0x000000ff>;
+		interrupt-controller;
+		interrupt-parent = <&gic>;
+		#interrupt-cells = <1>;
+		interrupts = < 0 89 4 >;
+		interrupt-map-mask = <0 0 0 7>;
+		interrupt-map = <0 0 0 0 &pci_express 0>,
+				<0 0 0 1 &pci_express 1>,
+				<0 0 0 2 &pci_express 2>,
+				<0 0 0 3 &pci_express 3>;
+		ranges = < 0x83000000 0 0x00000000 0xa8000000 0 0x8000000>;
+
+	};
diff --git a/Documentation/devicetree/bindings/pci/pci-armada8k.txt b/Documentation/devicetree/bindings/pci/pci-armada8k.txt
index c1e4c3d10a74..9e3fc15e1af8 100644
--- a/Documentation/devicetree/bindings/pci/pci-armada8k.txt
+++ b/Documentation/devicetree/bindings/pci/pci-armada8k.txt
@@ -12,7 +12,10 @@ Required properties:
    - "ctrl" for the control register region
    - "config" for the config space region
 - interrupts: Interrupt specifier for the PCIe controler
-- clocks: reference to the PCIe controller clock
+- clocks: reference to the PCIe controller clocks
+- clock-names: mandatory if there is a second clock, in this case the
+   name must be "core" for the first clock and "reg" for the second
+   one
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/pci/rcar-pci.txt b/Documentation/devicetree/bindings/pci/rcar-pci.txt
index 1fb614e615da..a5f7fc62d10e 100644
--- a/Documentation/devicetree/bindings/pci/rcar-pci.txt
+++ b/Documentation/devicetree/bindings/pci/rcar-pci.txt
@@ -8,6 +8,7 @@ compatible: "renesas,pcie-r8a7743" for the R8A7743 SoC;
 	    "renesas,pcie-r8a7793" for the R8A7793 SoC;
 	    "renesas,pcie-r8a7795" for the R8A7795 SoC;
 	    "renesas,pcie-r8a7796" for the R8A7796 SoC;
+	    "renesas,pcie-r8a77980" for the R8A77980 SoC;
 	    "renesas,pcie-rcar-gen2" for a generic R-Car Gen2 or
 				     RZ/G1 compatible device.
 	    "renesas,pcie-rcar-gen3" for a generic R-Car Gen3 compatible device.
@@ -32,6 +33,11 @@ compatible: "renesas,pcie-r8a7743" for the R8A7743 SoC;
 	and PCIe bus clocks.
 - clock-names: from common clock binding: should be "pcie" and "pcie_bus".
 
+Optional properties:
+- phys: from common PHY binding: PHY phandle and specifier (only make sense
+	for R-Car gen3 SoCs where the PCIe PHYs have their own register blocks).
+- phy-names: from common PHY binding: should be "pcie".
+
 Example:
 
 SoC-specific DT Entry:
diff --git a/Documentation/devicetree/bindings/pci/rockchip-pcie-ep.txt b/Documentation/devicetree/bindings/pci/rockchip-pcie-ep.txt
new file mode 100644
index 000000000000..778467307a93
--- /dev/null
+++ b/Documentation/devicetree/bindings/pci/rockchip-pcie-ep.txt
@@ -0,0 +1,62 @@
+* Rockchip AXI PCIe Endpoint Controller DT description
+
+Required properties:
+- compatible: Should contain "rockchip,rk3399-pcie-ep"
+- reg: Two register ranges as listed in the reg-names property
+- reg-names: Must include the following names
+	- "apb-base"
+	- "mem-base"
+- clocks: Must contain an entry for each entry in clock-names.
+		See ../clocks/clock-bindings.txt for details.
+- clock-names: Must include the following entries:
+	- "aclk"
+	- "aclk-perf"
+	- "hclk"
+	- "pm"
+- resets: Must contain seven entries for each entry in reset-names.
+	   See ../reset/reset.txt for details.
+- reset-names: Must include the following names
+	- "core"
+	- "mgmt"
+	- "mgmt-sticky"
+	- "pipe"
+	- "pm"
+	- "aclk"
+	- "pclk"
+- pinctrl-names : The pin control state names
+- pinctrl-0: The "default" pinctrl state
+- phys: Must contain an phandle to a PHY for each entry in phy-names.
+- phy-names: Must include 4 entries for all 4 lanes even if some of
+  them won't be used for your cases. Entries are of the form "pcie-phy-N":
+  where N ranges from 0 to 3.
+  (see example below and you MUST also refer to ../phy/rockchip-pcie-phy.txt
+  for changing the #phy-cells of phy node to support it)
+- rockchip,max-outbound-regions: Maximum number of outbound regions
+
+Optional Property:
+- num-lanes: number of lanes to use
+- max-functions: Maximum number of functions that can be configured (default 1).
+
+pcie0-ep: pcie@f8000000 {
+	compatible = "rockchip,rk3399-pcie-ep";
+	#address-cells = <3>;
+	#size-cells = <2>;
+	rockchip,max-outbound-regions = <16>;
+	clocks = <&cru ACLK_PCIE>, <&cru ACLK_PERF_PCIE>,
+		 <&cru PCLK_PCIE>, <&cru SCLK_PCIE_PM>;
+	clock-names = "aclk", "aclk-perf",
+		      "hclk", "pm";
+	max-functions = /bits/ 8 <8>;
+	num-lanes = <4>;
+	reg = <0x0 0xfd000000 0x0 0x1000000>, <0x0 0x80000000 0x0 0x20000>;
+	reg-names = "apb-base", "mem-base";
+	resets = <&cru SRST_PCIE_CORE>, <&cru SRST_PCIE_MGMT>,
+		 <&cru SRST_PCIE_MGMT_STICKY>, <&cru SRST_PCIE_PIPE> ,
+		 <&cru SRST_PCIE_PM>, <&cru SRST_P_PCIE>, <&cru SRST_A_PCIE>;
+	reset-names = "core", "mgmt", "mgmt-sticky", "pipe",
+		      "pm", "pclk", "aclk";
+	phys = <&pcie_phy 0>, <&pcie_phy 1>, <&pcie_phy 2>, <&pcie_phy 3>;
+	phy-names = "pcie-phy-0", "pcie-phy-1", "pcie-phy-2", "pcie-phy-3";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie_clkreq>;
+};
diff --git a/Documentation/devicetree/bindings/pci/rockchip-pcie-host.txt b/Documentation/devicetree/bindings/pci/rockchip-pcie-host.txt
new file mode 100644
index 000000000000..af34c65773fd
--- /dev/null
+++ b/Documentation/devicetree/bindings/pci/rockchip-pcie-host.txt
@@ -0,0 +1,135 @@
+* Rockchip AXI PCIe Root Port Bridge DT description
+
+Required properties:
+- #address-cells: Address representation for root ports, set to <3>
+- #size-cells: Size representation for root ports, set to <2>
+- #interrupt-cells: specifies the number of cells needed to encode an
+		interrupt source. The value must be 1.
+- compatible: Should contain "rockchip,rk3399-pcie"
+- reg: Two register ranges as listed in the reg-names property
+- reg-names: Must include the following names
+	- "axi-base"
+	- "apb-base"
+- clocks: Must contain an entry for each entry in clock-names.
+		See ../clocks/clock-bindings.txt for details.
+- clock-names: Must include the following entries:
+	- "aclk"
+	- "aclk-perf"
+	- "hclk"
+	- "pm"
+- msi-map: Maps a Requester ID to an MSI controller and associated
+	msi-specifier data. See ./pci-msi.txt
+- interrupts: Three interrupt entries must be specified.
+- interrupt-names: Must include the following names
+	- "sys"
+	- "legacy"
+	- "client"
+- resets: Must contain seven entries for each entry in reset-names.
+	   See ../reset/reset.txt for details.
+- reset-names: Must include the following names
+	- "core"
+	- "mgmt"
+	- "mgmt-sticky"
+	- "pipe"
+	- "pm"
+	- "aclk"
+	- "pclk"
+- pinctrl-names : The pin control state names
+- pinctrl-0: The "default" pinctrl state
+- #interrupt-cells: specifies the number of cells needed to encode an
+	interrupt source. The value must be 1.
+- interrupt-map-mask and interrupt-map: standard PCI properties
+
+Required properties for legacy PHY model (deprecated):
+- phys: From PHY bindings: Phandle for the Generic PHY for PCIe.
+- phy-names:  MUST be "pcie-phy".
+
+Required properties for per-lane PHY model (preferred):
+- phys: Must contain an phandle to a PHY for each entry in phy-names.
+- phy-names: Must include 4 entries for all 4 lanes even if some of
+  them won't be used for your cases. Entries are of the form "pcie-phy-N":
+  where N ranges from 0 to 3.
+  (see example below and you MUST also refer to ../phy/rockchip-pcie-phy.txt
+  for changing the #phy-cells of phy node to support it)
+
+Optional Property:
+- aspm-no-l0s: RC won't support ASPM L0s. This property is needed if
+	using 24MHz OSC for RC's PHY.
+- ep-gpios: contain the entry for pre-reset GPIO
+- num-lanes: number of lanes to use
+- vpcie12v-supply: The phandle to the 12v regulator to use for PCIe.
+- vpcie3v3-supply: The phandle to the 3.3v regulator to use for PCIe.
+- vpcie1v8-supply: The phandle to the 1.8v regulator to use for PCIe.
+- vpcie0v9-supply: The phandle to the 0.9v regulator to use for PCIe.
+
+*Interrupt controller child node*
+The core controller provides a single interrupt for legacy INTx. The PCIe node
+should contain an interrupt controller node as a target for the PCI
+'interrupt-map' property. This node represents the domain at which the four
+INTx interrupts are decoded and routed.
+
+
+Required properties for Interrupt controller child node:
+- interrupt-controller: identifies the node as an interrupt controller
+- #address-cells: specifies the number of cells needed to encode an
+	address. The value must be 0.
+- #interrupt-cells: specifies the number of cells needed to encode an
+	interrupt source. The value must be 1.
+
+Example:
+
+pcie0: pcie@f8000000 {
+	compatible = "rockchip,rk3399-pcie";
+	#address-cells = <3>;
+	#size-cells = <2>;
+	clocks = <&cru ACLK_PCIE>, <&cru ACLK_PERF_PCIE>,
+		 <&cru PCLK_PCIE>, <&cru SCLK_PCIE_PM>;
+	clock-names = "aclk", "aclk-perf",
+		      "hclk", "pm";
+	bus-range = <0x0 0x1>;
+	interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH 0>,
+		     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH 0>,
+		     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH 0>;
+	interrupt-names = "sys", "legacy", "client";
+	assigned-clocks = <&cru SCLK_PCIEPHY_REF>;
+	assigned-clock-parents = <&cru SCLK_PCIEPHY_REF100M>;
+	assigned-clock-rates = <100000000>;
+	ep-gpios = <&gpio3 13 GPIO_ACTIVE_HIGH>;
+	ranges = <0x83000000 0x0 0xfa000000 0x0 0xfa000000 0x0 0x600000
+		  0x81000000 0x0 0xfa600000 0x0 0xfa600000 0x0 0x100000>;
+	num-lanes = <4>;
+	msi-map = <0x0 &its 0x0 0x1000>;
+	reg = <0x0 0xf8000000 0x0 0x2000000>, <0x0 0xfd000000 0x0 0x1000000>;
+	reg-names = "axi-base", "apb-base";
+	resets = <&cru SRST_PCIE_CORE>, <&cru SRST_PCIE_MGMT>,
+		 <&cru SRST_PCIE_MGMT_STICKY>, <&cru SRST_PCIE_PIPE> ,
+		 <&cru SRST_PCIE_PM>, <&cru SRST_P_PCIE>, <&cru SRST_A_PCIE>;
+	reset-names = "core", "mgmt", "mgmt-sticky", "pipe",
+		      "pm", "pclk", "aclk";
+	/* deprecated legacy PHY model */
+	phys = <&pcie_phy>;
+	phy-names = "pcie-phy";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie_clkreq>;
+	#interrupt-cells = <1>;
+	interrupt-map-mask = <0 0 0 7>;
+	interrupt-map = <0 0 0 1 &pcie0_intc 0>,
+			<0 0 0 2 &pcie0_intc 1>,
+			<0 0 0 3 &pcie0_intc 2>,
+			<0 0 0 4 &pcie0_intc 3>;
+	pcie0_intc: interrupt-controller {
+		interrupt-controller;
+		#address-cells = <0>;
+		#interrupt-cells = <1>;
+	};
+};
+
+pcie0: pcie@f8000000 {
+	...
+
+	/* preferred per-lane PHY model */
+	phys = <&pcie_phy 0>, <&pcie_phy 1>, <&pcie_phy 2>, <&pcie_phy 3>;
+	phy-names = "pcie-phy-0", "pcie-phy-1", "pcie-phy-2", "pcie-phy-3";
+
+	...
+};
diff --git a/Documentation/devicetree/bindings/pci/rockchip-pcie.txt b/Documentation/devicetree/bindings/pci/rockchip-pcie.txt
deleted file mode 100644
index af34c65773fd..000000000000
--- a/Documentation/devicetree/bindings/pci/rockchip-pcie.txt
+++ /dev/null
@@ -1,135 +0,0 @@
-* Rockchip AXI PCIe Root Port Bridge DT description
-
-Required properties:
-- #address-cells: Address representation for root ports, set to <3>
-- #size-cells: Size representation for root ports, set to <2>
-- #interrupt-cells: specifies the number of cells needed to encode an
-		interrupt source. The value must be 1.
-- compatible: Should contain "rockchip,rk3399-pcie"
-- reg: Two register ranges as listed in the reg-names property
-- reg-names: Must include the following names
-	- "axi-base"
-	- "apb-base"
-- clocks: Must contain an entry for each entry in clock-names.
-		See ../clocks/clock-bindings.txt for details.
-- clock-names: Must include the following entries:
-	- "aclk"
-	- "aclk-perf"
-	- "hclk"
-	- "pm"
-- msi-map: Maps a Requester ID to an MSI controller and associated
-	msi-specifier data. See ./pci-msi.txt
-- interrupts: Three interrupt entries must be specified.
-- interrupt-names: Must include the following names
-	- "sys"
-	- "legacy"
-	- "client"
-- resets: Must contain seven entries for each entry in reset-names.
-	   See ../reset/reset.txt for details.
-- reset-names: Must include the following names
-	- "core"
-	- "mgmt"
-	- "mgmt-sticky"
-	- "pipe"
-	- "pm"
-	- "aclk"
-	- "pclk"
-- pinctrl-names : The pin control state names
-- pinctrl-0: The "default" pinctrl state
-- #interrupt-cells: specifies the number of cells needed to encode an
-	interrupt source. The value must be 1.
-- interrupt-map-mask and interrupt-map: standard PCI properties
-
-Required properties for legacy PHY model (deprecated):
-- phys: From PHY bindings: Phandle for the Generic PHY for PCIe.
-- phy-names:  MUST be "pcie-phy".
-
-Required properties for per-lane PHY model (preferred):
-- phys: Must contain an phandle to a PHY for each entry in phy-names.
-- phy-names: Must include 4 entries for all 4 lanes even if some of
-  them won't be used for your cases. Entries are of the form "pcie-phy-N":
-  where N ranges from 0 to 3.
-  (see example below and you MUST also refer to ../phy/rockchip-pcie-phy.txt
-  for changing the #phy-cells of phy node to support it)
-
-Optional Property:
-- aspm-no-l0s: RC won't support ASPM L0s. This property is needed if
-	using 24MHz OSC for RC's PHY.
-- ep-gpios: contain the entry for pre-reset GPIO
-- num-lanes: number of lanes to use
-- vpcie12v-supply: The phandle to the 12v regulator to use for PCIe.
-- vpcie3v3-supply: The phandle to the 3.3v regulator to use for PCIe.
-- vpcie1v8-supply: The phandle to the 1.8v regulator to use for PCIe.
-- vpcie0v9-supply: The phandle to the 0.9v regulator to use for PCIe.
-
-*Interrupt controller child node*
-The core controller provides a single interrupt for legacy INTx. The PCIe node
-should contain an interrupt controller node as a target for the PCI
-'interrupt-map' property. This node represents the domain at which the four
-INTx interrupts are decoded and routed.
-
-
-Required properties for Interrupt controller child node:
-- interrupt-controller: identifies the node as an interrupt controller
-- #address-cells: specifies the number of cells needed to encode an
-	address. The value must be 0.
-- #interrupt-cells: specifies the number of cells needed to encode an
-	interrupt source. The value must be 1.
-
-Example:
-
-pcie0: pcie@f8000000 {
-	compatible = "rockchip,rk3399-pcie";
-	#address-cells = <3>;
-	#size-cells = <2>;
-	clocks = <&cru ACLK_PCIE>, <&cru ACLK_PERF_PCIE>,
-		 <&cru PCLK_PCIE>, <&cru SCLK_PCIE_PM>;
-	clock-names = "aclk", "aclk-perf",
-		      "hclk", "pm";
-	bus-range = <0x0 0x1>;
-	interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH 0>,
-		     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH 0>,
-		     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH 0>;
-	interrupt-names = "sys", "legacy", "client";
-	assigned-clocks = <&cru SCLK_PCIEPHY_REF>;
-	assigned-clock-parents = <&cru SCLK_PCIEPHY_REF100M>;
-	assigned-clock-rates = <100000000>;
-	ep-gpios = <&gpio3 13 GPIO_ACTIVE_HIGH>;
-	ranges = <0x83000000 0x0 0xfa000000 0x0 0xfa000000 0x0 0x600000
-		  0x81000000 0x0 0xfa600000 0x0 0xfa600000 0x0 0x100000>;
-	num-lanes = <4>;
-	msi-map = <0x0 &its 0x0 0x1000>;
-	reg = <0x0 0xf8000000 0x0 0x2000000>, <0x0 0xfd000000 0x0 0x1000000>;
-	reg-names = "axi-base", "apb-base";
-	resets = <&cru SRST_PCIE_CORE>, <&cru SRST_PCIE_MGMT>,
-		 <&cru SRST_PCIE_MGMT_STICKY>, <&cru SRST_PCIE_PIPE> ,
-		 <&cru SRST_PCIE_PM>, <&cru SRST_P_PCIE>, <&cru SRST_A_PCIE>;
-	reset-names = "core", "mgmt", "mgmt-sticky", "pipe",
-		      "pm", "pclk", "aclk";
-	/* deprecated legacy PHY model */
-	phys = <&pcie_phy>;
-	phy-names = "pcie-phy";
-	pinctrl-names = "default";
-	pinctrl-0 = <&pcie_clkreq>;
-	#interrupt-cells = <1>;
-	interrupt-map-mask = <0 0 0 7>;
-	interrupt-map = <0 0 0 1 &pcie0_intc 0>,
-			<0 0 0 2 &pcie0_intc 1>,
-			<0 0 0 3 &pcie0_intc 2>,
-			<0 0 0 4 &pcie0_intc 3>;
-	pcie0_intc: interrupt-controller {
-		interrupt-controller;
-		#address-cells = <0>;
-		#interrupt-cells = <1>;
-	};
-};
-
-pcie0: pcie@f8000000 {
-	...
-
-	/* preferred per-lane PHY model */
-	phys = <&pcie_phy 0>, <&pcie_phy 1>, <&pcie_phy 2>, <&pcie_phy 3>;
-	phy-names = "pcie-phy-0", "pcie-phy-1", "pcie-phy-2", "pcie-phy-3";
-
-	...
-};
diff --git a/Documentation/devicetree/bindings/pci/xgene-pci.txt b/Documentation/devicetree/bindings/pci/xgene-pci.txt
index 6fd2decfa66c..92490330dc1c 100644
--- a/Documentation/devicetree/bindings/pci/xgene-pci.txt
+++ b/Documentation/devicetree/bindings/pci/xgene-pci.txt
@@ -25,8 +25,6 @@ Optional properties:
 
 Example:
 
-SoC-specific DT Entry:
-
 	pcie0: pcie@1f2b0000 {
 		status = "disabled";
 		device_type = "pci";
@@ -50,8 +48,3 @@ SoC-specific DT Entry:
 		clocks = <&pcie0clk 0>;
 	};
 
-
-Board-specific DT Entry:
-	&pcie0 {
-		status = "ok";
-	};
diff --git a/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt b/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt
new file mode 100644
index 000000000000..e7caefa0b9c2
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt
@@ -0,0 +1,109 @@
+MediaTek XS-PHY binding
+--------------------------
+
+The XS-PHY controller supports physical layer functionality for USB3.1
+GEN2 controller on MediaTek SoCs.
+
+Required properties (controller (parent) node):
+ - compatible	: should be "mediatek,<soc-model>-xsphy", "mediatek,xsphy",
+		  soc-model is the name of SoC, such as mt3611 etc;
+		  when using "mediatek,xsphy" compatible string, you need SoC specific
+		  ones in addition, one of:
+		  - "mediatek,mt3611-xsphy"
+
+ - #address-cells, #size-cells : should use the same values as the root node
+ - ranges: must be present
+
+Optional properties (controller (parent) node):
+ - reg		: offset and length of register shared by multiple U3 ports,
+		  exclude port's private register, if only U2 ports provided,
+		  shouldn't use the property.
+ - mediatek,src-ref-clk-mhz	: u32, frequency of reference clock for slew rate
+		  calibrate
+ - mediatek,src-coef	: u32, coefficient for slew rate calibrate, depends on
+		  SoC process
+
+Required nodes	: a sub-node is required for each port the controller
+		  provides. Address range information including the usual
+		  'reg' property is used inside these nodes to describe
+		  the controller's topology.
+
+Required properties (port (child) node):
+- reg		: address and length of the register set for the port.
+- clocks	: a list of phandle + clock-specifier pairs, one for each
+		  entry in clock-names
+- clock-names	: must contain
+		  "ref": 48M reference clock for HighSpeed analog phy; and 26M
+			reference clock for SuperSpeedPlus analog phy, sometimes is
+			24M, 25M or 27M, depended on platform.
+- #phy-cells	: should be 1
+		  cell after port phandle is phy type from:
+			- PHY_TYPE_USB2
+			- PHY_TYPE_USB3
+
+The following optional properties are only for debug or HQA test
+Optional properties (PHY_TYPE_USB2 port (child) node):
+- mediatek,eye-src	: u32, the value of slew rate calibrate
+- mediatek,eye-vrt	: u32, the selection of VRT reference voltage
+- mediatek,eye-term	: u32, the selection of HS_TX TERM reference voltage
+- mediatek,efuse-intr	: u32, the selection of Internal Resistor
+
+Optional properties (PHY_TYPE_USB3 port (child) node):
+- mediatek,efuse-intr	: u32, the selection of Internal Resistor
+- mediatek,efuse-tx-imp	: u32, the selection of TX Impedance
+- mediatek,efuse-rx-imp	: u32, the selection of RX Impedance
+
+Banks layout of xsphy
+-------------------------------------------------------------
+port        offset    bank
+u2 port0    0x0000    MISC
+            0x0100    FMREG
+            0x0300    U2PHY_COM
+u2 port1    0x1000    MISC
+            0x1100    FMREG
+            0x1300    U2PHY_COM
+u2 port2    0x2000    MISC
+            ...
+u31 common  0x3000    DIG_GLB
+            0x3100    PHYA_GLB
+u31 port0   0x3400    DIG_LN_TOP
+            0x3500    DIG_LN_TX0
+            0x3600    DIG_LN_RX0
+            0x3700    DIG_LN_DAIF
+            0x3800    PHYA_LN
+u31 port1   0x3a00    DIG_LN_TOP
+            0x3b00    DIG_LN_TX0
+            0x3c00    DIG_LN_RX0
+            0x3d00    DIG_LN_DAIF
+            0x3e00    PHYA_LN
+            ...
+
+DIG_GLB & PHYA_GLB are shared by U31 ports.
+
+Example:
+
+u3phy: usb-phy@11c40000 {
+	compatible = "mediatek,mt3611-xsphy", "mediatek,xsphy";
+	reg = <0 0x11c43000 0 0x0200>;
+	mediatek,src-ref-clk-mhz = <26>;
+	mediatek,src-coef = <17>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+
+	u2port0: usb-phy@11c40000 {
+		reg = <0 0x11c40000 0 0x0400>;
+		clocks = <&clk48m>;
+		clock-names = "ref";
+		mediatek,eye-src = <4>;
+		#phy-cells = <1>;
+	};
+
+	u3port0: usb-phy@11c43000 {
+		reg = <0 0x11c43400 0 0x0500>;
+		clocks = <&clk26m>;
+		clock-names = "ref";
+		mediatek,efuse-intr = <28>;
+		#phy-cells = <1>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/phy/qcom-qmp-phy.txt b/Documentation/devicetree/bindings/phy/qcom-qmp-phy.txt
index dcf1b8f691d5..266a1bb8bb6e 100644
--- a/Documentation/devicetree/bindings/phy/qcom-qmp-phy.txt
+++ b/Documentation/devicetree/bindings/phy/qcom-qmp-phy.txt
@@ -9,7 +9,8 @@ Required properties:
 	       "qcom,ipq8074-qmp-pcie-phy" for PCIe phy on IPQ8074
 	       "qcom,msm8996-qmp-pcie-phy" for 14nm PCIe phy on msm8996,
 	       "qcom,msm8996-qmp-usb3-phy" for 14nm USB3 phy on msm8996,
-	       "qcom,qmp-v3-usb3-phy" for USB3 QMP V3 phy.
+	       "qcom,sdm845-qmp-usb3-phy" for USB3 QMP V3 phy on sdm845,
+	       "qcom,sdm845-qmp-usb3-uni-phy" for USB3 QMP V3 UNI phy on sdm845.
 
  - reg: offset and length of register set for PHY's common serdes block.
 
diff --git a/Documentation/devicetree/bindings/phy/qcom-qusb2-phy.txt b/Documentation/devicetree/bindings/phy/qcom-qusb2-phy.txt
index 42c97426836e..03025d97998b 100644
--- a/Documentation/devicetree/bindings/phy/qcom-qusb2-phy.txt
+++ b/Documentation/devicetree/bindings/phy/qcom-qusb2-phy.txt
@@ -6,7 +6,7 @@ QUSB2 controller supports LS/FS/HS usb connectivity on Qualcomm chipsets.
 Required properties:
  - compatible: compatible list, contains
 	       "qcom,msm8996-qusb2-phy" for 14nm PHY on msm8996,
-	       "qcom,qusb2-v2-phy" for QUSB2 V2 PHY.
+	       "qcom,sdm845-qusb2-phy" for 10nm PHY on sdm845.
 
  - reg: offset and length of the PHY register set.
  - #phy-cells: must be 0.
@@ -27,6 +27,27 @@ Optional properties:
 		tuning parameter value for qusb2 phy.
 
  - qcom,tcsr-syscon: Phandle to TCSR syscon register region.
+ - qcom,imp-res-offset-value: It is a 6 bit value that specifies offset to be
+		added to PHY refgen RESCODE via IMP_CTRL1 register. It is a PHY
+		tuning parameter that may vary for different boards of same SOC.
+		This property is applicable to only QUSB2 v2 PHY (sdm845).
+ - qcom,hstx-trim-value: It is a 4 bit value that specifies tuning for HSTX
+		output current.
+		Possible range is - 15mA to 24mA (stepsize of 600 uA).
+		See dt-bindings/phy/phy-qcom-qusb2.h for applicable values.
+		This property is applicable to only QUSB2 v2 PHY (sdm845).
+		Default value is 22.2mA for sdm845.
+ - qcom,preemphasis-level: It is a 2 bit value that specifies pre-emphasis level.
+		Possible range is 0 to 15% (stepsize of 5%).
+		See dt-bindings/phy/phy-qcom-qusb2.h for applicable values.
+		This property is applicable to only QUSB2 v2 PHY (sdm845).
+		Default value is 10% for sdm845.
+- qcom,preemphasis-width: It is a 1 bit value that specifies how long the HSTX
+		pre-emphasis (specified using qcom,preemphasis-level) must be in
+		effect. Duration could be half-bit of full-bit.
+		See dt-bindings/phy/phy-qcom-qusb2.h for applicable values.
+		This property is applicable to only QUSB2 v2 PHY (sdm845).
+		Default value is full-bit width for sdm845.
 
 Example:
 	hsusb_phy: phy@7411000 {
diff --git a/Documentation/devicetree/bindings/pinctrl/actions,s900-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/actions,s900-pinctrl.txt
index fb87c7d74f2e..8fb5a53775e8 100644
--- a/Documentation/devicetree/bindings/pinctrl/actions,s900-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/actions,s900-pinctrl.txt
@@ -8,6 +8,17 @@ Required Properties:
 - reg:          Should contain the register base address and size of
                 the pin controller.
 - clocks:       phandle of the clock feeding the pin controller
+- gpio-controller: Marks the device node as a GPIO controller.
+- gpio-ranges: Specifies the mapping between gpio controller and
+               pin-controller pins.
+- #gpio-cells: Should be two. The first cell is the gpio pin number
+               and the second cell is used for optional parameters.
+- interrupt-controller: Marks the device node as an interrupt controller.
+- #interrupt-cells: Specifies the number of cells needed to encode an
+                    interrupt.  Shall be set to 2.  The first cell
+                    defines the interrupt number, the second encodes
+                    the trigger flags described in
+                    bindings/interrupt-controller/interrupts.txt
 
 Please refer to pinctrl-bindings.txt in this directory for details of the
 common pinctrl bindings used by client devices, including the meaning of the
@@ -164,6 +175,11 @@ Example:
                   compatible = "actions,s900-pinctrl";
                   reg = <0x0 0xe01b0000 0x0 0x1000>;
                   clocks = <&cmu CLK_GPIO>;
+                  gpio-controller;
+                  gpio-ranges = <&pinctrl 0 0 146>;
+                  #gpio-cells = <2>;
+                  interrupt-controller;
+                  #interrupt-cells = <2>;
 
                   uart2-default: uart2-default {
                           pinmux {
diff --git a/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt
index 64bc5c2a76da..258a4648ab81 100644
--- a/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt
@@ -28,6 +28,7 @@ Required properties:
   "allwinner,sun50i-a64-r-pinctrl"
   "allwinner,sun50i-h5-pinctrl"
   "allwinner,sun50i-h6-pinctrl"
+  "allwinner,sun50i-h6-r-pinctrl"
   "nextthing,gr8-pinctrl"
 
 - reg: Should contain the register physical address and length for the
diff --git a/Documentation/devicetree/bindings/pinctrl/brcm,bcm2835-gpio.txt b/Documentation/devicetree/bindings/pinctrl/brcm,bcm2835-gpio.txt
index 2569866c692f..3fac0a061bcc 100644
--- a/Documentation/devicetree/bindings/pinctrl/brcm,bcm2835-gpio.txt
+++ b/Documentation/devicetree/bindings/pinctrl/brcm,bcm2835-gpio.txt
@@ -36,6 +36,24 @@ listed. In other words, a subnode that lists only a mux function implies no
 information about any pull configuration. Similarly, a subnode that lists only
 a pul parameter implies no information about the mux function.
 
+The BCM2835 pin configuration and multiplexing supports the generic bindings.
+For details on each properties, you can refer to ./pinctrl-bindings.txt.
+
+Required sub-node properties:
+  - pins
+  - function
+
+Optional sub-node properties:
+  - bias-disable
+  - bias-pull-up
+  - bias-pull-down
+  - output-high
+  - output-low
+
+Legacy pin configuration and multiplexing binding:
+*** (Its use is deprecated, use generic multiplexing and configuration
+bindings instead)
+
 Required subnode-properties:
 - brcm,pins: An array of cells. Each cell contains the ID of a pin. Valid IDs
   are the integer GPIO IDs; 0==GPIO0, 1==GPIO1, ... 53==GPIO53.
diff --git a/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt
index 2c12f9789116..54ecb8ab7788 100644
--- a/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt
@@ -3,8 +3,10 @@
 Required properties for the root node:
  - compatible: one of "amlogic,meson8-cbus-pinctrl"
 		      "amlogic,meson8b-cbus-pinctrl"
+		      "amlogic,meson8m2-cbus-pinctrl"
 		      "amlogic,meson8-aobus-pinctrl"
 		      "amlogic,meson8b-aobus-pinctrl"
+		      "amlogic,meson8m2-aobus-pinctrl"
 		      "amlogic,meson-gxbb-periphs-pinctrl"
 		      "amlogic,meson-gxbb-aobus-pinctrl"
 		      "amlogic,meson-gxl-periphs-pinctrl"
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt
index a5a8322a31bd..a677145ae6d1 100644
--- a/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt
@@ -18,7 +18,9 @@ Required properties:
     removed.
 - #gpio-cells : Should be two.
   - first cell is the pin number
-  - second cell is used to specify flags. Flags are currently unused.
+  - second cell is used to specify flags as described in
+    'Documentation/devicetree/bindings/gpio/gpio.txt'. Allowed values defined by
+    'include/dt-bindings/gpio/gpio.h' (e.g. GPIO_ACTIVE_LOW).
 - gpio-controller : Marks the device node as a GPIO controller.
 - reg : For an address on its bus. I2C uses this a the I2C address of the chip.
         SPI uses this to specify the chipselect line which the chip is
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt7622.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt7622.txt
index f18ed99f6e14..def8fcad8941 100644
--- a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt7622.txt
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt7622.txt
@@ -9,6 +9,16 @@ Required properties for the root node:
  - #gpio-cells: Should be two. The first cell is the pin number and the
    second is the GPIO flags.
 
+Optional properties:
+- interrupt-controller  : Marks the device node as an interrupt controller
+
+If the property interrupt-controller is defined, following property is required
+- reg-names: A string describing the "reg" entries. Must contain "eint".
+- interrupts : The interrupt output from the controller.
+- #interrupt-cells: Should be two.
+- interrupt-parent: Phandle of the interrupt parent to which the external
+  GPIO interrupts are forwarded to.
+
 Please refer to pinctrl-bindings.txt in this directory for details of the
 common pinctrl bindings used by client devices, including the meaning of the
 phrase "pin configuration node".
diff --git a/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt
index 892d8fd7b700..abd8fbcf1e62 100644
--- a/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt
@@ -15,6 +15,7 @@ Required Properties:
     - "renesas,pfc-r8a7740": for R8A7740 (R-Mobile A1) compatible pin-controller.
     - "renesas,pfc-r8a7743": for R8A7743 (RZ/G1M) compatible pin-controller.
     - "renesas,pfc-r8a7745": for R8A7745 (RZ/G1E) compatible pin-controller.
+    - "renesas,pfc-r8a77470": for R8A77470 (RZ/G1C) compatible pin-controller.
     - "renesas,pfc-r8a7778": for R8A7778 (R-Car M1) compatible pin-controller.
     - "renesas,pfc-r8a7779": for R8A7779 (R-Car H1) compatible pin-controller.
     - "renesas,pfc-r8a7790": for R8A7790 (R-Car H2) compatible pin-controller.
@@ -27,6 +28,7 @@ Required Properties:
     - "renesas,pfc-r8a77965": for R8A77965 (R-Car M3-N) compatible pin-controller.
     - "renesas,pfc-r8a77970": for R8A77970 (R-Car V3M) compatible pin-controller.
     - "renesas,pfc-r8a77980": for R8A77980 (R-Car V3H) compatible pin-controller.
+    - "renesas,pfc-r8a77990": for R8A77990 (R-Car E3) compatible pin-controller.
     - "renesas,pfc-r8a77995": for R8A77995 (R-Car D3) compatible pin-controller.
     - "renesas,pfc-sh73a0": for SH73A0 (SH-Mobile AG5) compatible pin-controller.
 
diff --git a/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.txt
index a01a3b8a2363..0919db294c17 100644
--- a/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.txt
@@ -20,6 +20,7 @@ defined as gpio sub-nodes of the pinmux controller.
 
 Required properties for iomux controller:
   - compatible: should be
+		"rockchip,px30-pinctrl":    for Rockchip PX30
 		"rockchip,rv1108-pinctrl":  for Rockchip RV1108
 		"rockchip,rk2928-pinctrl":  for Rockchip RK2928
 		"rockchip,rk3066a-pinctrl": for Rockchip RK3066a
diff --git a/Documentation/devicetree/bindings/power/pd-samsung.txt b/Documentation/devicetree/bindings/power/pd-samsung.txt
index 549f7dee9b9d..92ef355e8f64 100644
--- a/Documentation/devicetree/bindings/power/pd-samsung.txt
+++ b/Documentation/devicetree/bindings/power/pd-samsung.txt
@@ -15,23 +15,13 @@ Required Properties:
 Optional Properties:
 - label: Human readable string with domain name. Will be visible in userspace
 	to let user to distinguish between multiple domains in SoC.
-- clocks: List of clock handles. The parent clocks of the input clocks to the
-	devices in this power domain are set to oscclk before power gating
-	and restored back after powering on a domain. This is required for
-	all domains which are powered on and off and not required for unused
-	domains.
-- clock-names: The following clocks can be specified:
-	- oscclk: Oscillator clock.
-	- clkN: Input clocks to the devices in this power domain. These clocks
-		will be reparented to oscclk before switching power domain off.
-		Their original parent will be brought back after turning on
-		the domain. Maximum of 4 clocks (N = 0 to 3) are supported.
-	- asbN: Clocks required by asynchronous bridges (ASB) present in
-		the power domain. These clock should be enabled during power
-		domain on/off operations.
 - power-domains: phandle pointing to the parent power domain, for more details
 		 see Documentation/devicetree/bindings/power/power_domain.txt
 
+Deprecated Properties:
+- clocks
+- clock-names
+
 Node of a device using power domains must have a power-domains property
 defined with a phandle to respective power domain.
 
@@ -47,8 +37,6 @@ Example:
 	mfc_pd: power-domain@10044060 {
 		compatible = "samsung,exynos4210-pd";
 		reg = <0x10044060 0x20>;
-		clocks = <&clock CLK_FIN_PLL>, <&clock CLK_MOUT_USER_ACLK333>;
-		clock-names = "oscclk", "clk0";
 		#power-domain-cells = <0>;
 		label = "MFC";
 	};
diff --git a/Documentation/devicetree/bindings/power/power_domain.txt b/Documentation/devicetree/bindings/power/power_domain.txt
index f3355313c020..4733f76cbe48 100644
--- a/Documentation/devicetree/bindings/power/power_domain.txt
+++ b/Documentation/devicetree/bindings/power/power_domain.txt
@@ -127,7 +127,7 @@ inside a PM domain with index 0 of a power controller represented by a node
 with the label "power".
 
 Optional properties:
-- required-opp: This contains phandle to an OPP node in another device's OPP
+- required-opps: This contains phandle to an OPP node in another device's OPP
   table. It may contain an array of phandles, where each phandle points to an
   OPP of a different device. It should not contain multiple phandles to the OPP
   nodes in the same OPP table. This specifies the minimum required OPP of the
@@ -175,14 +175,14 @@ Example:
 		compatible = "foo,i-leak-current";
 		reg = <0x12350000 0x1000>;
 		power-domains = <&power 0>;
-		required-opp = <&domain0_opp_0>;
+		required-opps = <&domain0_opp_0>;
 	};
 
 	leaky-device1@12350000 {
 		compatible = "foo,i-leak-current";
 		reg = <0x12350000 0x1000>;
 		power-domains = <&power 1>;
-		required-opp = <&domain1_opp_1>;
+		required-opps = <&domain1_opp_1>;
 	};
 
 [1]. Documentation/devicetree/bindings/power/domain-idle-state.txt
diff --git a/Documentation/devicetree/bindings/power/renesas,rcar-sysc.txt b/Documentation/devicetree/bindings/power/renesas,rcar-sysc.txt
index ab399e559257..180ae65be753 100644
--- a/Documentation/devicetree/bindings/power/renesas,rcar-sysc.txt
+++ b/Documentation/devicetree/bindings/power/renesas,rcar-sysc.txt
@@ -9,6 +9,7 @@ Required properties:
   - compatible: Must contain exactly one of the following:
       - "renesas,r8a7743-sysc" (RZ/G1M)
       - "renesas,r8a7745-sysc" (RZ/G1E)
+      - "renesas,r8a77470-sysc" (RZ/G1C)
       - "renesas,r8a7779-sysc" (R-Car H1)
       - "renesas,r8a7790-sysc" (R-Car H2)
       - "renesas,r8a7791-sysc" (R-Car M2-W)
@@ -20,6 +21,7 @@ Required properties:
       - "renesas,r8a77965-sysc" (R-Car M3-N)
       - "renesas,r8a77970-sysc" (R-Car V3M)
       - "renesas,r8a77980-sysc" (R-Car V3H)
+      - "renesas,r8a77990-sysc" (R-Car E3)
       - "renesas,r8a77995-sysc" (R-Car D3)
   - reg: Address start and address range for the device.
   - #power-domain-cells: Must be 1.
diff --git a/Documentation/devicetree/bindings/power/rockchip-io-domain.txt b/Documentation/devicetree/bindings/power/rockchip-io-domain.txt
index 4a4766e9c254..e66fd4eab71c 100644
--- a/Documentation/devicetree/bindings/power/rockchip-io-domain.txt
+++ b/Documentation/devicetree/bindings/power/rockchip-io-domain.txt
@@ -31,6 +31,8 @@ SoC is on the same page.
 
 Required properties:
 - compatible: should be one of:
+  - "rockchip,px30-io-voltage-domain" for px30
+  - "rockchip,px30-pmu-io-voltage-domain" for px30 pmu-domains
   - "rockchip,rk3188-io-voltage-domain" for rk3188
   - "rockchip,rk3228-io-voltage-domain" for rk3228
   - "rockchip,rk3288-io-voltage-domain" for rk3288
@@ -51,6 +53,19 @@ a phandle the relevant regulator.  All specified supplies must be able
 to report their voltage.  The IO Voltage Domain for any non-specified
 supplies will be not be touched.
 
+Possible supplies for PX30:
+- vccio6-supply: The supply connected to VCCIO6.
+- vccio1-supply: The supply connected to VCCIO1.
+- vccio2-supply: The supply connected to VCCIO2.
+- vccio3-supply: The supply connected to VCCIO3.
+- vccio4-supply: The supply connected to VCCIO4.
+- vccio5-supply: The supply connected to VCCIO5.
+- vccio-oscgpi-supply: The supply connected to VCCIO_OSCGPI.
+
+Possible supplies for PX30 pmu-domains:
+- pmuio1-supply: The supply connected to PMUIO1.
+- pmuio2-supply: The supply connected to PMUIO2.
+
 Possible supplies for rk3188:
 - ap0-supply:    The supply connected to AP0_VCC.
 - ap1-supply:    The supply connected to AP1_VCC.
diff --git a/Documentation/devicetree/bindings/power/supply/bq27xxx.txt b/Documentation/devicetree/bindings/power/supply/bq27xxx.txt
index 615c1cb6889f..37994fdb18ca 100644
--- a/Documentation/devicetree/bindings/power/supply/bq27xxx.txt
+++ b/Documentation/devicetree/bindings/power/supply/bq27xxx.txt
@@ -25,6 +25,7 @@ Required properties:
  * "ti,bq27545" - BQ27545
  * "ti,bq27421" - BQ27421
  * "ti,bq27425" - BQ27425
+ * "ti,bq27426" - BQ27426
  * "ti,bq27441" - BQ27441
  * "ti,bq27621" - BQ27621
 - reg: integer, I2C address of the fuel gauge.
diff --git a/Documentation/devicetree/bindings/powerpc/4xx/emac.txt b/Documentation/devicetree/bindings/powerpc/4xx/emac.txt
deleted file mode 100644
index 44b842b6ca15..000000000000
--- a/Documentation/devicetree/bindings/powerpc/4xx/emac.txt
+++ /dev/null
@@ -1,206 +0,0 @@
-    4xx/Axon EMAC ethernet nodes
-
-    The EMAC ethernet controller in IBM and AMCC 4xx chips, and also
-    the Axon bridge.  To operate this needs to interact with a this
-    special McMAL DMA controller, and sometimes an RGMII or ZMII
-    interface.  In addition to the nodes and properties described
-    below, the node for the OPB bus on which the EMAC sits must have a
-    correct clock-frequency property.
-
-      i) The EMAC node itself
-
-    Required properties:
-    - device_type       : "network"
-
-    - compatible        : compatible list, contains 2 entries, first is
-			  "ibm,emac-CHIP" where CHIP is the host ASIC (440gx,
-			  405gp, Axon) and second is either "ibm,emac" or
-			  "ibm,emac4".  For Axon, thus, we have: "ibm,emac-axon",
-			  "ibm,emac4"
-    - interrupts        : <interrupt mapping for EMAC IRQ and WOL IRQ>
-    - interrupt-parent  : optional, if needed for interrupt mapping
-    - reg               : <registers mapping>
-    - local-mac-address : 6 bytes, MAC address
-    - mal-device        : phandle of the associated McMAL node
-    - mal-tx-channel    : 1 cell, index of the tx channel on McMAL associated
-			  with this EMAC
-    - mal-rx-channel    : 1 cell, index of the rx channel on McMAL associated
-			  with this EMAC
-    - cell-index        : 1 cell, hardware index of the EMAC cell on a given
-			  ASIC (typically 0x0 and 0x1 for EMAC0 and EMAC1 on
-			  each Axon chip)
-    - max-frame-size    : 1 cell, maximum frame size supported in bytes
-    - rx-fifo-size      : 1 cell, Rx fifo size in bytes for 10 and 100 Mb/sec
-			  operations.
-			  For Axon, 2048
-    - tx-fifo-size      : 1 cell, Tx fifo size in bytes for 10 and 100 Mb/sec
-			  operations.
-			  For Axon, 2048.
-    - fifo-entry-size   : 1 cell, size of a fifo entry (used to calculate
-			  thresholds).
-			  For Axon, 0x00000010
-    - mal-burst-size    : 1 cell, MAL burst size (used to calculate thresholds)
-			  in bytes.
-			  For Axon, 0x00000100 (I think ...)
-    - phy-mode          : string, mode of operations of the PHY interface.
-			  Supported values are: "mii", "rmii", "smii", "rgmii",
-			  "tbi", "gmii", rtbi", "sgmii".
-			  For Axon on CAB, it is "rgmii"
-    - mdio-device       : 1 cell, required iff using shared MDIO registers
-			  (440EP).  phandle of the EMAC to use to drive the
-			  MDIO lines for the PHY used by this EMAC.
-    - zmii-device       : 1 cell, required iff connected to a ZMII.  phandle of
-			  the ZMII device node
-    - zmii-channel      : 1 cell, required iff connected to a ZMII.  Which ZMII
-			  channel or 0xffffffff if ZMII is only used for MDIO.
-    - rgmii-device      : 1 cell, required iff connected to an RGMII. phandle
-			  of the RGMII device node.
-			  For Axon: phandle of plb5/plb4/opb/rgmii
-    - rgmii-channel     : 1 cell, required iff connected to an RGMII.  Which
-			  RGMII channel is used by this EMAC.
-			  Fox Axon: present, whatever value is appropriate for each
-			  EMAC, that is the content of the current (bogus) "phy-port"
-			  property.
-
-    Optional properties:
-    - phy-address       : 1 cell, optional, MDIO address of the PHY. If absent,
-			  a search is performed.
-    - phy-map           : 1 cell, optional, bitmap of addresses to probe the PHY
-			  for, used if phy-address is absent. bit 0x00000001 is
-			  MDIO address 0.
-			  For Axon it can be absent, though my current driver
-			  doesn't handle phy-address yet so for now, keep
-			  0x00ffffff in it.
-    - phy-handle	: Used to describe configurations where a external PHY
-			  is used. Please refer to:
-			  Documentation/devicetree/bindings/net/ethernet.txt
-    - rx-fifo-size-gige : 1 cell, Rx fifo size in bytes for 1000 Mb/sec
-			  operations (if absent the value is the same as
-			  rx-fifo-size).  For Axon, either absent or 2048.
-    - tx-fifo-size-gige : 1 cell, Tx fifo size in bytes for 1000 Mb/sec
-			  operations (if absent the value is the same as
-			  tx-fifo-size). For Axon, either absent or 2048.
-    - tah-device        : 1 cell, optional. If connected to a TAH engine for
-			  offload, phandle of the TAH device node.
-    - tah-channel       : 1 cell, optional. If appropriate, channel used on the
-			  TAH engine.
-    - fixed-link	: Fixed-link subnode describing a link to a non-MDIO
-			  managed entity. See
-			  Documentation/devicetree/bindings/net/fixed-link.txt
-			  for details.
-    - mdio subnode	: When the EMAC has a phy connected to its local
-			  mdio, which us supported by the kernel's network
-			  PHY library in drivers/net/phy, there must be device
-			  tree subnode with the following required properties:
-				- #address-cells: Must be <1>.
-				- #size-cells: Must be <0>.
-
-			  For PHY definitions: Please refer to
-			  Documentation/devicetree/bindings/net/phy.txt and
-			  Documentation/devicetree/bindings/net/ethernet.txt
-
-    Examples:
-
-	EMAC0: ethernet@40000800 {
-		device_type = "network";
-		compatible = "ibm,emac-440gp", "ibm,emac";
-		interrupt-parent = <&UIC1>;
-		interrupts = <1c 4 1d 4>;
-		reg = <40000800 70>;
-		local-mac-address = [00 04 AC E3 1B 1E];
-		mal-device = <&MAL0>;
-		mal-tx-channel = <0 1>;
-		mal-rx-channel = <0>;
-		cell-index = <0>;
-		max-frame-size = <5dc>;
-		rx-fifo-size = <1000>;
-		tx-fifo-size = <800>;
-		phy-mode = "rmii";
-		phy-map = <00000001>;
-		zmii-device = <&ZMII0>;
-		zmii-channel = <0>;
-	};
-
-	EMAC1: ethernet@ef600c00 {
-		device_type = "network";
-		compatible = "ibm,emac-apm821xx", "ibm,emac4sync";
-		interrupt-parent = <&EMAC1>;
-		interrupts = <0 1>;
-		#interrupt-cells = <1>;
-		#address-cells = <0>;
-		#size-cells = <0>;
-		interrupt-map = <0 &UIC2 0x10 IRQ_TYPE_LEVEL_HIGH /* Status */
-				 1 &UIC2 0x14 IRQ_TYPE_LEVEL_HIGH /* Wake */>;
-		reg = <0xef600c00 0x000000c4>;
-		local-mac-address = [000000000000]; /* Filled in by U-Boot */
-		mal-device = <&MAL0>;
-		mal-tx-channel = <0>;
-		mal-rx-channel = <0>;
-		cell-index = <0>;
-		max-frame-size = <9000>;
-		rx-fifo-size = <16384>;
-		tx-fifo-size = <2048>;
-		fifo-entry-size = <10>;
-		phy-mode = "rgmii";
-		phy-handle = <&phy0>;
-		phy-map = <0x00000000>;
-		rgmii-device = <&RGMII0>;
-		rgmii-channel = <0>;
-		tah-device = <&TAH0>;
-		tah-channel = <0>;
-		has-inverted-stacr-oc;
-		has-new-stacr-staopc;
-
-	        mdio {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			phy0: ethernet-phy@0 {
-				compatible = "ethernet-phy-ieee802.3-c22";
-				reg = <0>;
-			};
-		};
-	};
-
-
-      ii) McMAL node
-
-    Required properties:
-    - device_type        : "dma-controller"
-    - compatible         : compatible list, containing 2 entries, first is
-			   "ibm,mcmal-CHIP" where CHIP is the host ASIC (like
-			   emac) and the second is either "ibm,mcmal" or
-			   "ibm,mcmal2".
-			   For Axon, "ibm,mcmal-axon","ibm,mcmal2"
-    - interrupts         : <interrupt mapping for the MAL interrupts sources:
-                           5 sources: tx_eob, rx_eob, serr, txde, rxde>.
-                           For Axon: This is _different_ from the current
-			   firmware.  We use the "delayed" interrupts for txeob
-			   and rxeob. Thus we end up with mapping those 5 MPIC
-			   interrupts, all level positive sensitive: 10, 11, 32,
-			   33, 34 (in decimal)
-    - dcr-reg            : < DCR registers range >
-    - dcr-parent         : if needed for dcr-reg
-    - num-tx-chans       : 1 cell, number of Tx channels
-    - num-rx-chans       : 1 cell, number of Rx channels
-
-      iii) ZMII node
-
-    Required properties:
-    - compatible         : compatible list, containing 2 entries, first is
-			   "ibm,zmii-CHIP" where CHIP is the host ASIC (like
-			   EMAC) and the second is "ibm,zmii".
-			   For Axon, there is no ZMII node.
-    - reg                : <registers mapping>
-
-      iv) RGMII node
-
-    Required properties:
-    - compatible         : compatible list, containing 2 entries, first is
-			   "ibm,rgmii-CHIP" where CHIP is the host ASIC (like
-			   EMAC) and the second is "ibm,rgmii".
-                           For Axon, "ibm,rgmii-axon","ibm,rgmii"
-    - reg                : <registers mapping>
-    - revision           : as provided by the RGMII new version register if
-			   available.
-			   For Axon: 0x0000012a
diff --git a/Documentation/devicetree/bindings/powerpc/4xx/ndfc.txt b/Documentation/devicetree/bindings/powerpc/4xx/ndfc.txt
deleted file mode 100644
index 869f0b5f16e8..000000000000
--- a/Documentation/devicetree/bindings/powerpc/4xx/ndfc.txt
+++ /dev/null
@@ -1,39 +0,0 @@
-AMCC NDFC (NanD Flash Controller)
-
-Required properties:
-- compatible : "ibm,ndfc".
-- reg : should specify chip select and size used for the chip (0x2000).
-
-Optional properties:
-- ccr : NDFC config and control register value (default 0).
-- bank-settings : NDFC bank configuration register value (default 0).
-
-Notes:
-- partition(s) - follows the OF MTD standard for partitions
-
-Example:
-
-ndfc@1,0 {
-	compatible = "ibm,ndfc";
-	reg = <0x00000001 0x00000000 0x00002000>;
-	ccr = <0x00001000>;
-	bank-settings = <0x80002222>;
-	#address-cells = <1>;
-	#size-cells = <1>;
-
-	nand {
-		#address-cells = <1>;
-		#size-cells = <1>;
-
-		partition@0 {
-			label = "kernel";
-			reg = <0x00000000 0x00200000>;
-		};
-		partition@200000 {
-			label = "root";
-			reg = <0x00200000 0x03E00000>;
-		};
-	};
-};
-
-
diff --git a/Documentation/devicetree/bindings/powerpc/fsl/gtm.txt b/Documentation/devicetree/bindings/powerpc/fsl/gtm.txt
deleted file mode 100644
index 9a33efded4bc..000000000000
--- a/Documentation/devicetree/bindings/powerpc/fsl/gtm.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-* Freescale General-purpose Timers Module
-
-Required properties:
-  - compatible : should be
-    "fsl,<chip>-gtm", "fsl,gtm" for SOC GTMs
-    "fsl,<chip>-qe-gtm", "fsl,qe-gtm", "fsl,gtm" for QE GTMs
-    "fsl,<chip>-cpm2-gtm", "fsl,cpm2-gtm", "fsl,gtm" for CPM2 GTMs
-  - reg : should contain gtm registers location and length (0x40).
-  - interrupts : should contain four interrupts.
-  - interrupt-parent : interrupt source phandle.
-  - clock-frequency : specifies the frequency driving the timer.
-
-Example:
-
-timer@500 {
-	compatible = "fsl,mpc8360-gtm", "fsl,gtm";
-	reg = <0x500 0x40>;
-	interrupts = <90 8 78 8 84 8 72 8>;
-	interrupt-parent = <&ipic>;
-	/* filled by u-boot */
-	clock-frequency = <0>;
-};
-
-timer@440 {
-	compatible = "fsl,mpc8360-qe-gtm", "fsl,qe-gtm", "fsl,gtm";
-	reg = <0x440 0x40>;
-	interrupts = <12 13 14 15>;
-	interrupt-parent = <&qeic>;
-	/* filled by u-boot */
-	clock-frequency = <0>;
-};
diff --git a/Documentation/devicetree/bindings/pps/pps-gpio.txt b/Documentation/devicetree/bindings/pps/pps-gpio.txt
index 0de23b793657..3683874832ae 100644
--- a/Documentation/devicetree/bindings/pps/pps-gpio.txt
+++ b/Documentation/devicetree/bindings/pps/pps-gpio.txt
@@ -20,5 +20,4 @@ Example:
 		assert-falling-edge;
 
 		compatible = "pps-gpio";
-		status = "okay";
 	};
diff --git a/Documentation/devicetree/bindings/ptp/ptp-qoriq.txt b/Documentation/devicetree/bindings/ptp/ptp-qoriq.txt
new file mode 100644
index 000000000000..0f569d8e73a3
--- /dev/null
+++ b/Documentation/devicetree/bindings/ptp/ptp-qoriq.txt
@@ -0,0 +1,69 @@
+* Freescale QorIQ 1588 timer based PTP clock
+
+General Properties:
+
+  - compatible   Should be "fsl,etsec-ptp"
+  - reg          Offset and length of the register set for the device
+  - interrupts   There should be at least two interrupts. Some devices
+                 have as many as four PTP related interrupts.
+
+Clock Properties:
+
+  - fsl,cksel        Timer reference clock source.
+  - fsl,tclk-period  Timer reference clock period in nanoseconds.
+  - fsl,tmr-prsc     Prescaler, divides the output clock.
+  - fsl,tmr-add      Frequency compensation value.
+  - fsl,tmr-fiper1   Fixed interval period pulse generator.
+  - fsl,tmr-fiper2   Fixed interval period pulse generator.
+  - fsl,max-adj      Maximum frequency adjustment in parts per billion.
+
+  These properties set the operational parameters for the PTP
+  clock. You must choose these carefully for the clock to work right.
+  Here is how to figure good values:
+
+  TimerOsc     = selected reference clock   MHz
+  tclk_period  = desired clock period       nanoseconds
+  NominalFreq  = 1000 / tclk_period         MHz
+  FreqDivRatio = TimerOsc / NominalFreq     (must be greater that 1.0)
+  tmr_add      = ceil(2^32 / FreqDivRatio)
+  OutputClock  = NominalFreq / tmr_prsc     MHz
+  PulseWidth   = 1 / OutputClock            microseconds
+  FiperFreq1   = desired frequency in Hz
+  FiperDiv1    = 1000000 * OutputClock / FiperFreq1
+  tmr_fiper1   = tmr_prsc * tclk_period * FiperDiv1 - tclk_period
+  max_adj      = 1000000000 * (FreqDivRatio - 1.0) - 1
+
+  The calculation for tmr_fiper2 is the same as for tmr_fiper1. The
+  driver expects that tmr_fiper1 will be correctly set to produce a 1
+  Pulse Per Second (PPS) signal, since this will be offered to the PPS
+  subsystem to synchronize the Linux clock.
+
+  Reference clock source is determined by the value, which is holded
+  in CKSEL bits in TMR_CTRL register. "fsl,cksel" property keeps the
+  value, which will be directly written in those bits, that is why,
+  according to reference manual, the next clock sources can be used:
+
+  <0> - external high precision timer reference clock (TSEC_TMR_CLK
+        input is used for this purpose);
+  <1> - eTSEC system clock;
+  <2> - eTSEC1 transmit clock;
+  <3> - RTC clock input.
+
+  When this attribute is not used, eTSEC system clock will serve as
+  IEEE 1588 timer reference clock.
+
+Example:
+
+	ptp_clock@24e00 {
+		compatible = "fsl,etsec-ptp";
+		reg = <0x24E00 0xB0>;
+		interrupts = <12 0x8 13 0x8>;
+		interrupt-parent = < &ipic >;
+		fsl,cksel       = <1>;
+		fsl,tclk-period = <10>;
+		fsl,tmr-prsc    = <100>;
+		fsl,tmr-add     = <0x999999A4>;
+		fsl,tmr-fiper1  = <0x3B9AC9F6>;
+		fsl,tmr-fiper2  = <0x00018696>;
+		fsl,max-adj     = <659999998>;
+	};
diff --git a/Documentation/devicetree/bindings/pwm/pwm-omap-dmtimer.txt b/Documentation/devicetree/bindings/pwm/pwm-omap-dmtimer.txt
index 2e53324fb720..5ccfcc82da08 100644
--- a/Documentation/devicetree/bindings/pwm/pwm-omap-dmtimer.txt
+++ b/Documentation/devicetree/bindings/pwm/pwm-omap-dmtimer.txt
@@ -2,7 +2,7 @@
 
 Required properties:
 - compatible: Shall contain "ti,omap-dmtimer-pwm".
-- ti,timers: phandle to PWM capable OMAP timer. See arm/omap/timer.txt for info
+- ti,timers: phandle to PWM capable OMAP timer. See timer/ti,timer.txt for info
   about these timers.
 - #pwm-cells: Should be 3. See pwm.txt in this directory for a description of
   the cells format.
diff --git a/Documentation/devicetree/bindings/regulator/pfuze100.txt b/Documentation/devicetree/bindings/regulator/pfuze100.txt
index c6dd3f5e485b..f0ada3b14d70 100644
--- a/Documentation/devicetree/bindings/regulator/pfuze100.txt
+++ b/Documentation/devicetree/bindings/regulator/pfuze100.txt
@@ -21,7 +21,7 @@ Each regulator is defined using the standard binding for regulators.
 
 Example 1: PFUZE100
 
-	pmic: pfuze100@8 {
+	pfuze100: pmic@8 {
 		compatible = "fsl,pfuze100";
 		reg = <0x08>;
 
@@ -122,7 +122,7 @@ Example 1: PFUZE100
 
 Example 2: PFUZE200
 
-	pmic: pfuze200@8 {
+	pfuze200: pmic@8 {
 		compatible = "fsl,pfuze200";
 		reg = <0x08>;
 
@@ -216,7 +216,7 @@ Example 2: PFUZE200
 
 Example 3: PFUZE3000
 
-	pmic: pfuze3000@8 {
+	pfuze3000: pmic@8 {
 		compatible = "fsl,pfuze3000";
 		reg = <0x08>;
 
diff --git a/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt b/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt
index 57d2c65899df..406f2e570c50 100644
--- a/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt
+++ b/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt
@@ -110,6 +110,11 @@ Qualcomm SPMI Regulators
 	Definition: Reference to regulator supplying the input pin, as
 		    described in the data sheet.
 
+- qcom,saw-reg:
+	Usage: optional
+	Value type: <phandle>
+	Description: Reference to syscon node defining the SAW registers.
+
 
 The regulator node houses sub-nodes for each regulator within the device. Each
 sub-node is identified using the node's name, with valid values listed for each
@@ -201,6 +206,17 @@ see regulator.txt - with additional custom properties described below:
 			2 = 0.55 uA
 			3 = 0.75 uA
 
+- qcom,saw-slave:
+	Usage: optional
+	Value type: <boo>
+	Description: SAW controlled gang slave. Will not be configured.
+
+- qcom,saw-leader:
+	Usage: optional
+	Value type: <boo>
+	Description: SAW controlled gang leader. Will be configured as
+		     SAW regulator.
+
 Example:
 
 	regulators {
@@ -221,3 +237,32 @@ Example:
 
 		....
 	};
+
+Example 2:
+
+	saw3: syscon@9A10000 {
+		compatible = "syscon";
+		reg = <0x9A10000 0x1000>;
+	};
+
+	...
+
+	spm-regulators {
+		compatible = "qcom,pm8994-regulators";
+		qcom,saw-reg = <&saw3>;
+		s8 {
+			qcom,saw-slave;
+		};
+		s9 {
+			qcom,saw-slave;
+		};
+		s10 {
+			qcom,saw-slave;
+		};
+		pm8994_s11_saw: s11 {
+			qcom,saw-leader;
+			regulator-always-on;
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <1140000>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/regulator/regulator.txt b/Documentation/devicetree/bindings/regulator/regulator.txt
index 2babe15b618d..a7cd36877bfe 100644
--- a/Documentation/devicetree/bindings/regulator/regulator.txt
+++ b/Documentation/devicetree/bindings/regulator/regulator.txt
@@ -59,6 +59,11 @@ Optional properties:
 - regulator-initial-mode: initial operating mode. The set of possible operating
   modes depends on the capabilities of every hardware so each device binding
   documentation explains which values the regulator supports.
+- regulator-allowed-modes: list of operating modes that software is allowed to
+  configure for the regulator at run-time.  Elements may be specified in any
+  order.  The set of possible operating modes depends on the capabilities of
+  every hardware so each device binding document explains which values the
+  regulator supports.
 - regulator-system-load: Load in uA present on regulator that is not captured by
   any consumer request.
 - regulator-pull-down: Enable pull down resistor when the regulator is disabled.
@@ -68,6 +73,11 @@ Optional properties:
 	0: Disable active discharge.
 	1: Enable active discharge.
 	Absence of this property will leave configuration to default.
+- regulator-coupled-with: Regulators with which the regulator
+  is coupled. The linkage is 2-way - all coupled regulators should be linked
+  with each other. A regulator should not be coupled with its supplier.
+- regulator-coupled-max-spread: Max spread between voltages of coupled regulators
+  in microvolts.
 
 Deprecated properties:
 - regulator-compatible: If a regulator chip contains multiple
diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.txt b/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.txt
new file mode 100644
index 000000000000..4edf3137d9f7
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.txt
@@ -0,0 +1,126 @@
+ROHM BD71837 Power Management Integrated Circuit (PMIC) regulator bindings
+
+BD71837MWV is a programmable Power Management
+IC (PMIC) for powering single-core, dual-core, and
+quad-core SoC’s such as NXP-i.MX 8M. It is optimized
+for low BOM cost and compact solution footprint. It
+integrates 8 Buck regulators and 7 LDO’s to provide all
+the power rails required by the SoC and the commonly
+used peripherals.
+
+Required properties:
+ - regulator-name: should be "buck1", ..., "buck8" and "ldo1", ..., "ldo7"
+
+List of regulators provided by this controller. BD71837 regulators node
+should be sub node of the BD71837 MFD node. See BD71837 MFD bindings at
+Documentation/devicetree/bindings/mfd/rohm,bd71837-pmic.txt
+Regulator nodes should be named to BUCK_<number> and LDO_<number>. The
+definition for each of these nodes is defined using the standard
+binding for regulators at
+Documentation/devicetree/bindings/regulator/regulator.txt.
+Note that if BD71837 starts at RUN state you probably want to use
+regulator-boot-on at least for BUCK6 and BUCK7 so that those are not
+disabled by driver at startup. LDO5 and LDO6 are supplied by those and
+if they are disabled at startup the voltage monitoring for LDO5/LDO6 will
+cause PMIC to reset.
+
+The valid names for regulator nodes are:
+BUCK1, BUCK2, BUCK3, BUCK4, BUCK5, BUCK6, BUCK7, BUCK8
+LDO1, LDO2, LDO3, LDO4, LDO5, LDO6, LDO7
+
+Optional properties:
+- Any optional property defined in bindings/regulator/regulator.txt
+
+Example:
+regulators {
+	buck1: BUCK1 {
+		regulator-name = "buck1";
+		regulator-min-microvolt = <700000>;
+		regulator-max-microvolt = <1300000>;
+		regulator-boot-on;
+		regulator-ramp-delay = <1250>;
+	};
+	buck2: BUCK2 {
+		regulator-name = "buck2";
+		regulator-min-microvolt = <700000>;
+		regulator-max-microvolt = <1300000>;
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-ramp-delay = <1250>;
+	};
+	buck3: BUCK3 {
+		regulator-name = "buck3";
+		regulator-min-microvolt = <700000>;
+		regulator-max-microvolt = <1300000>;
+		regulator-boot-on;
+	};
+	buck4: BUCK4 {
+		regulator-name = "buck4";
+		regulator-min-microvolt = <700000>;
+		regulator-max-microvolt = <1300000>;
+		regulator-boot-on;
+	};
+	buck5: BUCK5 {
+		regulator-name = "buck5";
+		regulator-min-microvolt = <700000>;
+		regulator-max-microvolt = <1350000>;
+		regulator-boot-on;
+	};
+	buck6: BUCK6 {
+		regulator-name = "buck6";
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+	};
+	buck7: BUCK7 {
+		regulator-name = "buck7";
+		regulator-min-microvolt = <1605000>;
+		regulator-max-microvolt = <1995000>;
+		regulator-boot-on;
+	};
+	buck8: BUCK8 {
+		regulator-name = "buck8";
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <1400000>;
+	};
+
+	ldo1: LDO1 {
+		regulator-name = "ldo1";
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+	};
+	ldo2: LDO2 {
+		regulator-name = "ldo2";
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-boot-on;
+	};
+	ldo3: LDO3 {
+		regulator-name = "ldo3";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+	};
+	ldo4: LDO4 {
+		regulator-name = "ldo4";
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <1800000>;
+	};
+	ldo5: LDO5 {
+		regulator-name = "ldo5";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+	};
+	ldo6: LDO6 {
+		regulator-name = "ldo6";
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <1800000>;
+	};
+	ldo7_reg: LDO7 {
+		regulator-name = "ldo7";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+	};
+};
+
+
diff --git a/Documentation/devicetree/bindings/regulator/sy8106a-regulator.txt b/Documentation/devicetree/bindings/regulator/sy8106a-regulator.txt
new file mode 100644
index 000000000000..39a8ca73f572
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/sy8106a-regulator.txt
@@ -0,0 +1,23 @@
+SY8106A Voltage regulator
+
+Required properties:
+- compatible: Must be "silergy,sy8106a"
+- reg: I2C slave address - must be <0x65>
+- silergy,fixed-microvolt - the voltage when I2C regulating is disabled (set
+  by external resistor like a fixed voltage)
+
+Any property defined as part of the core regulator binding, defined in
+./regulator.txt, can also be used.
+
+Example:
+
+	sy8106a {
+		compatible = "silergy,sy8106a";
+		reg = <0x65>;
+		regulator-name = "sy8106a-vdd";
+		silergy,fixed-microvolt = <1200000>;
+		regulator-min-microvolt = <1000000>;
+		regulator-max-microvolt = <1400000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt b/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt
index 00d3d58a102f..d90182425450 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt
@@ -11,6 +11,7 @@ on the Qualcomm Hexagon core.
 		    "qcom,msm8916-mss-pil",
 		    "qcom,msm8974-mss-pil"
 		    "qcom,msm8996-mss-pil"
+		    "qcom,sdm845-mss-pil"
 
 - reg:
 	Usage: required
diff --git a/Documentation/devicetree/bindings/reserved-memory/qcom,cmd-db.txt b/Documentation/devicetree/bindings/reserved-memory/qcom,cmd-db.txt
new file mode 100644
index 000000000000..68395530c0a5
--- /dev/null
+++ b/Documentation/devicetree/bindings/reserved-memory/qcom,cmd-db.txt
@@ -0,0 +1,37 @@
+Command DB
+---------
+
+Command DB is a database that provides a mapping between resource key and the
+resource address for a system resource managed by a remote processor. The data
+is stored in a shared memory region and is loaded by the remote processor.
+
+Some of the Qualcomm Technologies Inc SoC's have hardware accelerators for
+controlling shared resources. Depending on the board configuration the shared
+resource properties may change. These properties are dynamically probed by the
+remote processor and made available in the shared memory.
+
+The bindings for Command DB is specified in the reserved-memory section in
+devicetree. The devicetree representation of the command DB driver should be:
+
+Properties:
+- compatible:
+	Usage: required
+	Value type: <string>
+	Definition: Should be "qcom,cmd-db"
+
+- reg:
+	Usage: required
+	Value type: <prop encoded array>
+	Definition: The register address that points to the actual location of
+		    the Command DB in memory.
+
+Example:
+
+	reserved-memory {
+		[...]
+		reserved-memory@85fe0000 {
+			reg = <0x0 0x85fe0000 0x0 0x20000>;
+			compatible = "qcom,cmd-db";
+			no-map;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/reset/renesas,rst.txt b/Documentation/devicetree/bindings/reset/renesas,rst.txt
index 294a0dae106a..67e83b02e10b 100644
--- a/Documentation/devicetree/bindings/reset/renesas,rst.txt
+++ b/Documentation/devicetree/bindings/reset/renesas,rst.txt
@@ -17,6 +17,7 @@ Required properties:
 		Examples with soctypes are:
 		  - "renesas,r8a7743-rst" (RZ/G1M)
 		  - "renesas,r8a7745-rst" (RZ/G1E)
+		  - "renesas,r8a77470-rst" (RZ/G1C)
 		  - "renesas,r8a7778-reset-wdt" (R-Car M1A)
 		  - "renesas,r8a7779-reset-wdt" (R-Car H1)
 		  - "renesas,r8a7790-rst" (R-Car H2)
@@ -29,6 +30,7 @@ Required properties:
 		  - "renesas,r8a77965-rst" (R-Car M3-N)
 		  - "renesas,r8a77970-rst" (R-Car V3M)
 		  - "renesas,r8a77980-rst" (R-Car V3H)
+		  - "renesas,r8a77990-rst" (R-Car E3)
 		  - "renesas,r8a77995-rst" (R-Car D3)
   - reg: Address start and address range for the device.
 
diff --git a/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt b/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt
index 627b29531a32..aaac7975f61c 100644
--- a/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt
+++ b/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt
@@ -14,11 +14,16 @@ Optional properties:
 - clocks : phandle to clock-controller plus clock-specifier pair
 - clock-names : "ipsec" as a clock name
 
+Optional properties:
+
+- interrupts: specify the interrupt for the RNG block
+
 Example:
 
 rng {
-        compatible = "brcm,bcm2835-rng";
-        reg = <0x7e104000 0x10>;
+	compatible = "brcm,bcm2835-rng";
+	reg = <0x7e104000 0x10>;
+	interrupts = <2 29>;
 };
 
 rng@18033000 {
diff --git a/Documentation/devicetree/bindings/rng/samsung,exynos4-rng.txt b/Documentation/devicetree/bindings/rng/samsung,exynos4-rng.txt
new file mode 100644
index 000000000000..a13fbdb4bd88
--- /dev/null
+++ b/Documentation/devicetree/bindings/rng/samsung,exynos4-rng.txt
@@ -0,0 +1,19 @@
+Exynos Pseudo Random Number Generator
+
+Required properties:
+
+- compatible  : One of:
+                - "samsung,exynos4-rng" for Exynos4210 and Exynos4412
+                - "samsung,exynos5250-prng" for Exynos5250+
+- reg         : Specifies base physical address and size of the registers map.
+- clocks      : Phandle to clock-controller plus clock-specifier pair.
+- clock-names : "secss" as a clock name.
+
+Example:
+
+	rng@10830400 {
+		compatible = "samsung,exynos4-rng";
+		reg = <0x10830400 0x200>;
+		clocks = <&clock CLK_SSS>;
+		clock-names = "secss";
+	};
diff --git a/Documentation/devicetree/bindings/rng/sparc_sun_oracle_rng.txt b/Documentation/devicetree/bindings/rng/sparc_sun_oracle_rng.txt
new file mode 100644
index 000000000000..b0b211194c71
--- /dev/null
+++ b/Documentation/devicetree/bindings/rng/sparc_sun_oracle_rng.txt
@@ -0,0 +1,30 @@
+HWRNG support for the n2_rng driver
+
+Required properties:
+- reg		: base address to sample from
+- compatible	: should contain one of the following
+	RNG versions:
+	- 'SUNW,n2-rng' for Niagara 2 Platform (SUN UltraSPARC T2 CPU)
+	- 'SUNW,vf-rng' for Victoria Falls Platform (SUN UltraSPARC T2 Plus CPU)
+	- 'SUNW,kt-rng' for Rainbow/Yosemite Falls Platform (SUN SPARC T3/T4), (UltraSPARC KT/Niagara 3 - development names)
+	more recent systems (after Oracle acquisition of SUN)
+	- 'ORCL,m4-rng' for SPARC T5/M5
+	- 'ORCL,m7-rng' for SPARC T7/M7
+
+Examples:
+/* linux LDOM on SPARC T5-2 */
+Node 0xf029a4f4
+	.node:  f029a4f4
+	rng-#units:  00000002
+	compatible: 'ORCL,m4-rng'
+	reg:  0000000e
+	name: 'random-number-generator'
+
+/* solaris on SPARC M7-8 */
+Node 0xf028c08c
+	rng-#units:  00000003
+	compatible: 'ORCL,m7-rng'
+	reg:  0000000e
+	name:  'random-number-generator'
+
+PS: see as well prtconfs.git by DaveM
diff --git a/Documentation/devicetree/bindings/rtc/nxp,rtc-2123.txt b/Documentation/devicetree/bindings/rtc/nxp,rtc-2123.txt
index 5cbc0b145a61..811124a36d16 100644
--- a/Documentation/devicetree/bindings/rtc/nxp,rtc-2123.txt
+++ b/Documentation/devicetree/bindings/rtc/nxp,rtc-2123.txt
@@ -9,7 +9,7 @@ Optional properties:
 
 Example:
 
-rtc: nxp,rtc-pcf2123@3 {
+pcf2123: rtc@3 {
 	compatible = "nxp,rtc-pcf2123"
 	reg = <3>
 	spi-cs-high;
diff --git a/Documentation/devicetree/bindings/rtc/st,stm32-rtc.txt b/Documentation/devicetree/bindings/rtc/st,stm32-rtc.txt
index a66692a08ace..c920e2736991 100644
--- a/Documentation/devicetree/bindings/rtc/st,stm32-rtc.txt
+++ b/Documentation/devicetree/bindings/rtc/st,stm32-rtc.txt
@@ -1,23 +1,29 @@
 STM32 Real Time Clock
 
 Required properties:
-- compatible: can be either "st,stm32-rtc" or "st,stm32h7-rtc", depending on
-  the device is compatible with stm32(f4/f7) or stm32h7.
+- compatible: can be one of the following:
+  - "st,stm32-rtc" for devices compatible with stm32(f4/f7).
+  - "st,stm32h7-rtc" for devices compatible with stm32h7.
+  - "st,stm32mp1-rtc" for devices compatible with stm32mp1.
 - reg: address range of rtc register set.
 - clocks: can use up to two clocks, depending on part used:
   - "rtc_ck": RTC clock source.
-    It is required on stm32(f4/f7) and stm32h7.
   - "pclk": RTC APB interface clock.
     It is not present on stm32(f4/f7).
-    It is required on stm32h7.
+    It is required on stm32(h7/mp1).
 - clock-names: must be "rtc_ck" and "pclk".
-    It is required only on stm32h7.
+    It is required on stm32(h7/mp1).
 - interrupt-parent: phandle for the interrupt controller.
-- interrupts: rtc alarm interrupt.
-- st,syscfg: phandle for pwrcfg, mandatory to disable/enable backup domain
-  (RTC registers) write protection.
+    It is required on stm32(f4/f7/h7).
+- interrupts: rtc alarm interrupt. On stm32mp1, a second interrupt is required
+  for rtc alarm wakeup interrupt.
+- st,syscfg: phandle/offset/mask triplet. The phandle to pwrcfg used to
+  access control register at offset, and change the dbp (Disable Backup
+  Protection) bit represented by the mask, mandatory to disable/enable backup
+  domain (RTC registers) write protection.
+    It is required on stm32(f4/f7/h7).
 
-Optional properties (to override default rtc_ck parent clock):
+Optional properties (to override default rtc_ck parent clock on stm32(f4/f7/h7):
 - assigned-clocks: reference to the rtc_ck clock entry.
 - assigned-clock-parents: phandle of the new parent clock of rtc_ck.
 
@@ -31,7 +37,7 @@ Example:
 		assigned-clock-parents = <&rcc 1 CLK_LSE>;
 		interrupt-parent = <&exti>;
 		interrupts = <17 1>;
-		st,syscfg = <&pwrcfg>;
+		st,syscfg = <&pwrcfg 0x00 0x100>;
 	};
 
 	rtc: rtc@58004000 {
@@ -44,5 +50,14 @@ Example:
 		interrupt-parent = <&exti>;
 		interrupts = <17 1>;
 		interrupt-names = "alarm";
-		st,syscfg = <&pwrcfg>;
+		st,syscfg = <&pwrcfg 0x00 0x100>;
+	};
+
+	rtc: rtc@5c004000 {
+		compatible = "st,stm32mp1-rtc";
+		reg = <0x5c004000 0x400>;
+		clocks = <&rcc RTCAPB>, <&rcc RTC>;
+		clock-names = "pclk", "rtc_ck";
+		interrupts-extended = <&intc GIC_SPI 3 IRQ_TYPE_NONE>,
+				      <&exti 19 1>;
 	};
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,apr.txt b/Documentation/devicetree/bindings/soc/qcom/qcom,apr.txt
new file mode 100644
index 000000000000..bcc612cc7423
--- /dev/null
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,apr.txt
@@ -0,0 +1,84 @@
+Qualcomm APR (Asynchronous Packet Router) binding
+
+This binding describes the Qualcomm APR. APR is a IPC protocol for
+communication between Application processor and QDSP. APR is mainly
+used for audio/voice services on the QDSP.
+
+- compatible:
+	Usage: required
+	Value type: <stringlist>
+	Definition: must be "qcom,apr-v<VERSION-NUMBER>", example "qcom,apr-v2"
+
+- reg
+	Usage: required
+	Value type: <u32>
+	Definition: Destination processor ID.
+	Possible values are :
+			1 - APR simulator
+			2 - PC
+			3 - MODEM
+			4 - ADSP
+			5 - APPS
+			6 - MODEM2
+			7 - APPS2
+
+= APR SERVICES
+Each subnode of the APR node represents service tied to this apr. The name
+of the nodes are not important. The properties of these nodes are defined
+by the individual bindings for the specific service
+- All APR services MUST contain the following property:
+
+- reg
+	Usage: required
+	Value type: <u32>
+	Definition: APR Service ID
+	Possible values are :
+			3 - DSP Core Service
+			4 - Audio Front End Service.
+			5 - Voice Stream Manager Service.
+			6 - Voice processing manager.
+			7 - Audio Stream Manager Service.
+			8 - Audio Device Manager Service.
+			9 - Multimode voice manager.
+			10 - Core voice stream.
+			11 - Core voice processor.
+			12 - Ultrasound stream manager.
+			13 - Listen stream manager.
+
+= EXAMPLE
+The following example represents a QDSP based sound card on a MSM8996 device
+which uses apr as communication between Apps and QDSP.
+
+	apr@4 {
+		compatible = "qcom,apr-v2";
+		reg = <APR_DOMAIN_ADSP>;
+
+		q6core@3 {
+			compatible = "qcom,q6core";
+			reg = <APR_SVC_ADSP_CORE>;
+		};
+
+		q6afe@4 {
+			compatible = "qcom,q6afe";
+			reg = <APR_SVC_AFE>;
+
+			dais {
+				#sound-dai-cells = <1>;
+				hdmi@1 {
+					reg = <1>;
+				};
+			};
+		};
+
+		q6asm@7 {
+			compatible = "qcom,q6asm";
+			reg = <APR_SVC_ASM>;
+			...
+		};
+
+		q6adm@8 {
+			compatible = "qcom,q6adm";
+			reg = <APR_SVC_ADM>;
+			...
+		};
+	};
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.txt b/Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.txt
new file mode 100644
index 000000000000..d330c73de9a2
--- /dev/null
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.txt
@@ -0,0 +1,119 @@
+Qualcomm Technologies, Inc. GENI Serial Engine QUP Wrapper Controller
+
+Generic Interface (GENI) based Qualcomm Universal Peripheral (QUP) wrapper
+is a programmable module for supporting a wide range of serial interfaces
+like UART, SPI, I2C, I3C, etc. A single QUP module can provide upto 8 Serial
+Interfaces, using its internal Serial Engines. The GENI Serial Engine QUP
+Wrapper controller is modeled as a node with zero or more child nodes each
+representing a serial engine.
+
+Required properties:
+- compatible:		Must be "qcom,geni-se-qup".
+- reg:			Must contain QUP register address and length.
+- clock-names:		Must contain "m-ahb" and "s-ahb".
+- clocks:		AHB clocks needed by the device.
+
+Required properties if child node exists:
+- #address-cells: 	Must be <1> for Serial Engine Address
+- #size-cells: 		Must be <1> for Serial Engine Address Size
+- ranges: 		Must be present
+
+Properties for children:
+
+A GENI based QUP wrapper controller node can contain 0 or more child nodes
+representing serial devices.  These serial devices can be a QCOM UART, I2C
+controller, SPI controller, or some combination of aforementioned devices.
+Please refer below the child node definitions for the supported serial
+interface protocols.
+
+Qualcomm Technologies Inc. GENI Serial Engine based I2C Controller
+
+Required properties:
+- compatible:		Must be "qcom,geni-i2c".
+- reg: 			Must contain QUP register address and length.
+- interrupts: 		Must contain I2C interrupt.
+- clock-names: 		Must contain "se".
+- clocks: 		Serial engine core clock needed by the device.
+- #address-cells:	Must be <1> for I2C device address.
+- #size-cells:		Must be <0> as I2C addresses have no size component.
+
+Optional property:
+- clock-frequency:	Desired I2C bus clock frequency in Hz.
+			When missing default to 400000Hz.
+
+Child nodes should conform to I2C bus binding as described in i2c.txt.
+
+Qualcomm Technologies Inc. GENI Serial Engine based UART Controller
+
+Required properties:
+- compatible:		Must be "qcom,geni-debug-uart".
+- reg: 			Must contain UART register location and length.
+- interrupts: 		Must contain UART core interrupts.
+- clock-names:		Must contain "se".
+- clocks:		Serial engine core clock needed by the device.
+
+Qualcomm Technologies Inc. GENI Serial Engine based SPI Controller
+
+Required properties:
+- compatible:		Must contain "qcom,geni-spi".
+- reg:			Must contain SPI register location and length.
+- interrupts:		Must contain SPI controller interrupts.
+- clock-names:		Must contain "se".
+- clocks:		Serial engine core clock needed by the device.
+- spi-max-frequency:	Specifies maximum SPI clock frequency, units - Hz.
+- #address-cells:	Must be <1> to define a chip select address on
+			the SPI bus.
+- #size-cells:		Must be <0>.
+
+SPI slave nodes must be children of the SPI master node and conform to SPI bus
+binding as described in Documentation/devicetree/bindings/spi/spi-bus.txt.
+
+Example:
+	geniqup@8c0000 {
+		compatible = "qcom,geni-se-qup";
+		reg = <0x8c0000 0x6000>;
+		clock-names = "m-ahb", "s-ahb";
+		clocks = <&clock_gcc GCC_QUPV3_WRAP_0_M_AHB_CLK>,
+			<&clock_gcc GCC_QUPV3_WRAP_0_S_AHB_CLK>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		i2c0: i2c@a94000 {
+			compatible = "qcom,geni-i2c";
+			reg = <0xa94000 0x4000>;
+			interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "se";
+			clocks = <&clock_gcc GCC_QUPV3_WRAP0_S5_CLK>;
+			pinctrl-names = "default", "sleep";
+			pinctrl-0 = <&qup_1_i2c_5_active>;
+			pinctrl-1 = <&qup_1_i2c_5_sleep>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		uart0: serial@a88000 {
+			compatible = "qcom,geni-debug-uart";
+			reg = <0xa88000 0x7000>;
+			interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "se";
+			clocks = <&clock_gcc GCC_QUPV3_WRAP0_S0_CLK>;
+			pinctrl-names = "default", "sleep";
+			pinctrl-0 = <&qup_1_uart_3_active>;
+			pinctrl-1 = <&qup_1_uart_3_sleep>;
+		};
+
+		spi0: spi@a84000 {
+			compatible = "qcom,geni-spi";
+			reg = <0xa84000 0x4000>;
+			interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "se";
+			clocks = <&clock_gcc GCC_QUPV3_WRAP0_S0_CLK>;
+			pinctrl-names = "default", "sleep";
+			pinctrl-0 = <&qup_1_spi_2_active>;
+			pinctrl-1 = <&qup_1_spi_2_sleep>;
+			spi-max-frequency = <19200000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+	}
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.txt b/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.txt
index a48049ccf6d0..89e1cb9212f6 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.txt
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.txt
@@ -22,6 +22,7 @@ resources.
 		    "qcom,rpm-apq8084"
 		    "qcom,rpm-msm8916"
 		    "qcom,rpm-msm8974"
+		    "qcom,rpm-msm8998"
 
 - qcom,smd-channels:
 	Usage: required
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt b/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt
index ea1dc75ec9ea..234ae2256501 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt
@@ -22,9 +22,15 @@ The edge is described by the following properties:
 	Definition: should specify the IRQ used by the remote processor to
 		    signal this processor about communication related updates
 
-- qcom,ipc:
+- mboxes:
 	Usage: required
 	Value type: <prop-encoded-array>
+	Definition: reference to the associated doorbell in APCS, as described
+		    in mailbox/mailbox.txt
+
+- qcom,ipc:
+	Usage: required, unless mboxes is specified
+	Value type: <prop-encoded-array>
 	Definition: three entries specifying the outgoing ipc bit used for
 		    signaling the remote processor:
 		    - phandle to a syscon node representing the apcs registers
diff --git a/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt b/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt
index 301d2a9bc1b8..5d49d0a2ff29 100644
--- a/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt
+++ b/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt
@@ -5,6 +5,10 @@ powered up/down by software based on different application scenes to save power.
 
 Required properties for power domain controller:
 - compatible: Should be one of the following.
+	"rockchip,px30-power-controller" - for PX30 SoCs.
+	"rockchip,rk3036-power-controller" - for RK3036 SoCs.
+	"rockchip,rk3128-power-controller" - for RK3128 SoCs.
+	"rockchip,rk3228-power-controller" - for RK3228 SoCs.
 	"rockchip,rk3288-power-controller" - for RK3288 SoCs.
 	"rockchip,rk3328-power-controller" - for RK3328 SoCs.
 	"rockchip,rk3366-power-controller" - for RK3366 SoCs.
@@ -17,6 +21,10 @@ Required properties for power domain controller:
 
 Required properties for power domain sub nodes:
 - reg: index of the power domain, should use macros in:
+	"include/dt-bindings/power/px30-power.h" - for PX30 type power domain.
+	"include/dt-bindings/power/rk3036-power.h" - for RK3036 type power domain.
+	"include/dt-bindings/power/rk3128-power.h" - for RK3128 type power domain.
+	"include/dt-bindings/power/rk3228-power.h" - for RK3228 type power domain.
 	"include/dt-bindings/power/rk3288-power.h" - for RK3288 type power domain.
 	"include/dt-bindings/power/rk3328-power.h" - for RK3328 type power domain.
 	"include/dt-bindings/power/rk3366-power.h" - for RK3366 type power domain.
@@ -93,6 +101,10 @@ Node of a device using power domains must have a power-domains property,
 containing a phandle to the power device node and an index specifying which
 power domain to use.
 The index should use macros in:
+	"include/dt-bindings/power/px30-power.h" - for px30 type power domain.
+	"include/dt-bindings/power/rk3036-power.h" - for rk3036 type power domain.
+	"include/dt-bindings/power/rk3128-power.h" - for rk3128 type power domain.
+	"include/dt-bindings/power/rk3128-power.h" - for rk3228 type power domain.
 	"include/dt-bindings/power/rk3288-power.h" - for rk3288 type power domain.
 	"include/dt-bindings/power/rk3328-power.h" - for rk3328 type power domain.
 	"include/dt-bindings/power/rk3366-power.h" - for rk3366 type power domain.
diff --git a/Documentation/devicetree/bindings/soc/ti/keystone-navigator-qmss.txt b/Documentation/devicetree/bindings/soc/ti/keystone-navigator-qmss.txt
index 77cd42cc5f54..b025770eeb92 100644
--- a/Documentation/devicetree/bindings/soc/ti/keystone-navigator-qmss.txt
+++ b/Documentation/devicetree/bindings/soc/ti/keystone-navigator-qmss.txt
@@ -17,7 +17,8 @@ pool management.
 
 
 Required properties:
-- compatible	: Must be "ti,keystone-navigator-qmss";
+- compatible	: Must be "ti,keystone-navigator-qmss".
+		: Must be "ti,66ak2g-navss-qm" for QMSS on K2G SoC.
 - clocks	: phandle to the reference clock for this device.
 - queue-range	: <start number> total range of queue numbers for the device.
 - linkram0	: <address size> for internal link ram, where size is the total
@@ -39,6 +40,12 @@ Required properties:
 			  - Descriptor memory setup region.
 			  - Queue Management/Queue Proxy region for queue Push.
 			  - Queue Management/Queue Proxy region for queue Pop.
+
+For QMSS on K2G SoC, following QM reg indexes are used in that order
+			  - Queue Peek region.
+			  - Queue configuration region.
+			  - Queue Management/Queue Proxy region for queue Push/Pop.
+
 - queue-pools	: child node classifying the queue ranges into pools.
 		  Queue ranges are grouped into 3 type of pools:
 		  - qpend	    : pool of qpend(interruptible) queues
diff --git a/Documentation/devicetree/bindings/sound/adi,ssm2305.txt b/Documentation/devicetree/bindings/sound/adi,ssm2305.txt
new file mode 100644
index 000000000000..a9c9d83c8a30
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/adi,ssm2305.txt
@@ -0,0 +1,14 @@
+Analog Devices SSM2305 Speaker Amplifier
+========================================
+
+Required properties:
+  - compatible : "adi,ssm2305"
+  - shutdown-gpios : The gpio connected to the shutdown pin.
+                     The gpio signal is ACTIVE_LOW.
+
+Example:
+
+ssm2305: analog-amplifier {
+	compatible = "adi,ssm2305";
+	shutdown-gpios = <&gpio3 20 GPIO_ACTIVE_LOW>;
+};
diff --git a/Documentation/devicetree/bindings/sound/atmel-i2s.txt b/Documentation/devicetree/bindings/sound/atmel-i2s.txt
new file mode 100644
index 000000000000..735368b8a73f
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/atmel-i2s.txt
@@ -0,0 +1,47 @@
+* Atmel I2S controller
+
+Required properties:
+- compatible:     Should be "atmel,sama5d2-i2s".
+- reg:            Should be the physical base address of the controller and the
+                  length of memory mapped region.
+- interrupts:     Should contain the interrupt for the controller.
+- dmas:           Should be one per channel name listed in the dma-names property,
+                  as described in atmel-dma.txt and dma.txt files.
+- dma-names:      Two dmas have to be defined, "tx" and "rx".
+                  This IP also supports one shared channel for both rx and tx;
+                  if this mode is used, one "rx-tx" name must be used.
+- clocks:         Must contain an entry for each entry in clock-names.
+                  Please refer to clock-bindings.txt.
+- clock-names:    Should be one of each entry matching the clocks phandles list:
+                  - "pclk" (peripheral clock) Required.
+                  - "gclk" (generated clock) Optional (1).
+                  - "aclk" (Audio PLL clock) Optional (1).
+                  - "muxclk" (I2S mux clock) Optional (1).
+
+Optional properties:
+- pinctrl-0:      Should specify pin control groups used for this controller.
+- princtrl-names: Should contain only one value - "default".
+
+
+(1) : Only the peripheral clock is required. The generated clock, the Audio
+      PLL clock adn the I2S mux clock are optional and should only be set
+      together, when Master Mode is required.
+
+Example:
+
+	i2s@f8050000 {
+		compatible = "atmel,sama5d2-i2s";
+		reg = <0xf8050000 0x300>;
+		interrupts = <54 IRQ_TYPE_LEVEL_HIGH 7>;
+		dmas = <&dma0
+			(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) |
+			 AT91_XDMAC_DT_PERID(31))>,
+		       <&dma0
+			(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) |
+			 AT91_XDMAC_DT_PERID(32))>;
+		dma-names = "tx", "rx";
+		clocks = <&i2s0_clk>, <&i2s0_gclk>, <&audio_pll_pmc>, <&i2s0muxck>;
+		clock-names = "pclk", "gclk", "aclk", "muxclk";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_i2s0_default>;
+	};
diff --git a/Documentation/devicetree/bindings/sound/cs42xx8.txt b/Documentation/devicetree/bindings/sound/cs42xx8.txt
index f631fbca6284..8619a156d038 100644
--- a/Documentation/devicetree/bindings/sound/cs42xx8.txt
+++ b/Documentation/devicetree/bindings/sound/cs42xx8.txt
@@ -16,7 +16,7 @@ Required properties:
 
 Example:
 
-codec: cs42888@48 {
+cs42888: codec@48 {
 	compatible = "cirrus,cs42888";
 	reg = <0x48>;
 	clocks = <&codec_mclk 0>;
diff --git a/Documentation/devicetree/bindings/sound/fsl,asrc.txt b/Documentation/devicetree/bindings/sound/fsl,asrc.txt
index f5a14115b459..1d4d9f938689 100644
--- a/Documentation/devicetree/bindings/sound/fsl,asrc.txt
+++ b/Documentation/devicetree/bindings/sound/fsl,asrc.txt
@@ -31,14 +31,16 @@ Required properties:
 			  it. This property is optional depending on the SoC
 			  design.
 
-   - big-endian		: If this property is absent, the little endian mode
-			  will be in use as default. Otherwise, the big endian
-			  mode will be in use for all the device registers.
-
    - fsl,asrc-rate	: Defines a mutual sample rate used by DPCM Back Ends.
 
    - fsl,asrc-width	: Defines a mutual sample width used by DPCM Back Ends.
 
+Optional properties:
+
+   - big-endian		: If this property is absent, the little endian mode
+			  will be in use as default. Otherwise, the big endian
+			  mode will be in use for all the device registers.
+
 Example:
 
 asrc: asrc@2034000 {
diff --git a/Documentation/devicetree/bindings/sound/fsl,esai.txt b/Documentation/devicetree/bindings/sound/fsl,esai.txt
index cacd18bb9ba6..5b9914367610 100644
--- a/Documentation/devicetree/bindings/sound/fsl,esai.txt
+++ b/Documentation/devicetree/bindings/sound/fsl,esai.txt
@@ -42,6 +42,8 @@ Required properties:
 			  means all the settings for Receiving would be
 			  duplicated from Transmition related registers.
 
+Optional properties:
+
   - big-endian		: If this property is absent, the native endian mode
 			  will be in use as default, or the big endian mode
 			  will be in use for all the device registers.
diff --git a/Documentation/devicetree/bindings/sound/fsl,spdif.txt b/Documentation/devicetree/bindings/sound/fsl,spdif.txt
index 38cfa7573441..8b324f82a782 100644
--- a/Documentation/devicetree/bindings/sound/fsl,spdif.txt
+++ b/Documentation/devicetree/bindings/sound/fsl,spdif.txt
@@ -33,6 +33,8 @@ Required properties:
 			  it. This property is optional depending on the SoC
 			  design.
 
+Optional properties:
+
    - big-endian		: If this property is absent, the native endian mode
 			  will be in use as default, or the big endian mode
 			  will be in use for all the device registers.
diff --git a/Documentation/devicetree/bindings/sound/fsl-sai.txt b/Documentation/devicetree/bindings/sound/fsl-sai.txt
index 740b467adf7d..dd9e59738e08 100644
--- a/Documentation/devicetree/bindings/sound/fsl-sai.txt
+++ b/Documentation/devicetree/bindings/sound/fsl-sai.txt
@@ -28,9 +28,6 @@ Required properties:
 			  pinctrl-names. See ../pinctrl/pinctrl-bindings.txt
 			  for details of the property values.
 
-  - big-endian		: Boolean property, required if all the FTM_PWM
-			  registers are big-endian rather than little-endian.
-
   - lsb-first		: Configures whether the LSB or the MSB is transmitted
 			  first for the fifo data. If this property is absent,
 			  the MSB is transmitted first as default, or the LSB
@@ -48,6 +45,11 @@ Required properties:
 			  receive data by following their own bit clocks and
 			  frame sync clocks separately.
 
+Optional properties:
+
+  - big-endian		: Boolean property, required if all the SAI
+			  registers are big-endian rather than little-endian.
+
 Optional properties (for mx6ul):
 
   - fsl,sai-mclk-direction-output: This is a boolean property. If present,
diff --git a/Documentation/devicetree/bindings/sound/mt2701-afe-pcm.txt b/Documentation/devicetree/bindings/sound/mt2701-afe-pcm.txt
index e2f7f4951215..560762e0a168 100644
--- a/Documentation/devicetree/bindings/sound/mt2701-afe-pcm.txt
+++ b/Documentation/devicetree/bindings/sound/mt2701-afe-pcm.txt
@@ -1,7 +1,9 @@
 Mediatek AFE PCM controller for mt2701
 
 Required properties:
-- compatible = "mediatek,mt2701-audio";
+- compatible: should be one of the followings.
+	      - "mediatek,mt2701-audio"
+	      - "mediatek,mt7622-audio"
 - interrupts: should contain AFE and ASYS interrupts
 - interrupt-names: should be "afe" and "asys"
 - power-domains: should define the power domain
diff --git a/Documentation/devicetree/bindings/sound/mt6351.txt b/Documentation/devicetree/bindings/sound/mt6351.txt
new file mode 100644
index 000000000000..7fb2cb99245e
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/mt6351.txt
@@ -0,0 +1,16 @@
+Mediatek MT6351 Audio Codec
+
+The communication between MT6351 and SoC is through Mediatek PMIC wrapper.
+For more detail, please visit Mediatek PMIC wrapper documentation.
+
+Must be a child node of PMIC wrapper.
+
+Required properties:
+
+- compatible : "mediatek,mt6351-sound".
+
+Example:
+
+mt6351_snd {
+	compatible = "mediatek,mt6351-sound";
+};
diff --git a/Documentation/devicetree/bindings/sound/mt6797-afe-pcm.txt b/Documentation/devicetree/bindings/sound/mt6797-afe-pcm.txt
new file mode 100644
index 000000000000..0ae29de15bfd
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/mt6797-afe-pcm.txt
@@ -0,0 +1,42 @@
+Mediatek AFE PCM controller for mt6797
+
+Required properties:
+- compatible = "mediatek,mt6797-audio";
+- reg: register location and size
+- interrupts: should contain AFE interrupt
+- power-domains: should define the power domain
+- clocks: Must contain an entry for each entry in clock-names
+- clock-names: should have these clock names:
+		"infra_sys_audio_clk",
+		"infra_sys_audio_26m",
+		"mtkaif_26m_clk",
+		"top_mux_audio",
+		"top_mux_aud_intbus",
+		"top_sys_pll3_d4",
+		"top_sys_pll1_d4",
+		"top_clk26m_clk";
+
+Example:
+
+	afe: mt6797-afe-pcm@11220000  {
+		compatible = "mediatek,mt6797-audio";
+		reg = <0 0x11220000 0 0x1000>;
+		interrupts = <GIC_SPI 151 IRQ_TYPE_LEVEL_LOW>;
+		power-domains = <&scpsys MT6797_POWER_DOMAIN_AUDIO>;
+		clocks = <&infrasys CLK_INFRA_AUDIO>,
+			 <&infrasys CLK_INFRA_AUDIO_26M>,
+			 <&infrasys CLK_INFRA_AUDIO_26M_PAD_TOP>,
+			 <&topckgen CLK_TOP_MUX_AUDIO>,
+			 <&topckgen CLK_TOP_MUX_AUD_INTBUS>,
+			 <&topckgen CLK_TOP_SYSPLL3_D4>,
+			 <&topckgen CLK_TOP_SYSPLL1_D4>,
+			 <&clk26m>;
+		clock-names = "infra_sys_audio_clk",
+			      "infra_sys_audio_26m",
+			      "mtkaif_26m_clk",
+			      "top_mux_audio",
+			      "top_mux_aud_intbus",
+			      "top_sys_pll3_d4",
+			      "top_sys_pll1_d4",
+			      "top_clk26m_clk";
+	};
diff --git a/Documentation/devicetree/bindings/sound/mt6797-mt6351.txt b/Documentation/devicetree/bindings/sound/mt6797-mt6351.txt
new file mode 100644
index 000000000000..1d95a8840f19
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/mt6797-mt6351.txt
@@ -0,0 +1,14 @@
+MT6797 with MT6351 CODEC
+
+Required properties:
+- compatible: "mediatek,mt6797-mt6351-sound"
+- mediatek,platform: the phandle of MT6797 ASoC platform
+- mediatek,audio-codec: the phandles of MT6351 codec
+
+Example:
+
+	sound {
+		compatible = "mediatek,mt6797-mt6351-sound";
+		mediatek,audio-codec = <&mt6351_snd>;
+		mediatek,platform = <&afe>;
+	};
diff --git a/Documentation/devicetree/bindings/sound/qcom,apq8096.txt b/Documentation/devicetree/bindings/sound/qcom,apq8096.txt
new file mode 100644
index 000000000000..aa54e49fc8a2
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/qcom,apq8096.txt
@@ -0,0 +1,109 @@
+* Qualcomm Technologies APQ8096 ASoC sound card driver
+
+This binding describes the APQ8096 sound card, which uses qdsp for audio.
+
+- compatible:
+	Usage: required
+	Value type: <stringlist>
+	Definition: must be "qcom,apq8096-sndcard"
+
+- qcom,audio-routing:
+	Usage: Optional
+	Value type: <stringlist>
+	Definition:  A list of the connections between audio components.
+		  Each entry is a pair of strings, the first being the
+		  connection's sink, the second being the connection's
+		  source. Valid names could be power supplies, MicBias
+		  of codec and the jacks on the board:
+		  Valid names include:
+
+		Board Connectors:
+			"Headphone Left"
+			"Headphone Right"
+			"Earphone"
+			"Line Out1"
+			"Line Out2"
+			"Line Out3"
+			"Line Out4"
+			"Analog Mic1"
+			"Analog Mic2"
+			"Analog Mic3"
+			"Analog Mic4"
+			"Analog Mic5"
+			"Analog Mic6"
+			"Digital Mic2"
+			"Digital Mic3"
+
+		Audio pins and MicBias on WCD9335 Codec:
+			"MIC_BIAS1
+			"MIC_BIAS2"
+			"MIC_BIAS3"
+			"MIC_BIAS4"
+			"AMIC1"
+			"AMIC2"
+			"AMIC3"
+			"AMIC4"
+			"AMIC5"
+			"AMIC6"
+			"AMIC6"
+			"DMIC1"
+			"DMIC2"
+			"DMIC3"
+= dailinks
+Each subnode of sndcard represents either a dailink, and subnodes of each
+dailinks would be cpu/codec/platform dais.
+
+- link-name:
+	Usage: required
+	Value type: <string>
+	Definition: User friendly name for dai link
+
+= CPU, PLATFORM, CODEC dais subnodes
+- cpu:
+	Usage: required
+	Value type: <subnode>
+	Definition: cpu dai sub-node
+
+- codec:
+	Usage: Optional
+	Value type: <subnode>
+	Definition: codec dai sub-node
+
+- platform:
+	Usage: Optional
+	Value type: <subnode>
+	Definition: platform dai sub-node
+
+- sound-dai:
+	Usage: required
+	Value type: <phandle with arguments>
+	Definition: dai phandle/s and port of CPU/CODEC/PLATFORM node.
+
+Example:
+
+audio {
+	compatible = "qcom,apq8096-sndcard";
+	qcom,model = "DB820c";
+
+	mm1-dai-link {
+		link-name = "MultiMedia1";
+		cpu {
+			sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA1>;
+		};
+	};
+
+	hdmi-dai-link {
+		link-name = "HDMI Playback";
+		cpu {
+			sound-dai = <&q6afe HDMI_RX>;
+		};
+
+		platform {
+			sound-dai = <&q6adm>;
+		};
+
+		codec {
+			sound-dai = <&hdmi 0>;
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6adm.txt b/Documentation/devicetree/bindings/sound/qcom,q6adm.txt
new file mode 100644
index 000000000000..cb709e5dbc44
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/qcom,q6adm.txt
@@ -0,0 +1,33 @@
+Qualcomm Audio Device Manager (Q6ADM) binding
+
+Q6ADM is one of the APR audio service on Q6DSP.
+Please refer to qcom,apr.txt for details of the coommon apr service bindings
+used by the apr service device.
+
+- but must contain the following property:
+
+- compatible:
+	Usage: required
+	Value type: <stringlist>
+	Definition: must be "qcom,q6adm-v<MAJOR-NUMBER>.<MINOR-NUMBER>".
+		   Or "qcom,q6adm" where the version number can be queried
+		   from DSP.
+		   example "qcom,q6adm-v2.0"
+
+
+= ADM routing
+"routing" subnode of the ADM node represents adm routing specific configuration
+
+- #sound-dai-cells
+	Usage: required
+	Value type: <u32>
+	Definition: Must be 0
+
+= EXAMPLE
+q6adm@8 {
+	compatible = "qcom,q6adm";
+	reg = <APR_SVC_ADM>;
+	q6routing: routing {
+		#sound-dai-cells = <0>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6afe.txt b/Documentation/devicetree/bindings/sound/qcom,q6afe.txt
new file mode 100644
index 000000000000..bdbf87df8c0b
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/qcom,q6afe.txt
@@ -0,0 +1,172 @@
+Qualcomm Audio Front End (Q6AFE) binding
+
+AFE is one of the APR audio service on Q6DSP
+Please refer to qcom,apr.txt for details of the common apr service bindings
+used by all apr services. Must contain the following properties.
+
+- compatible:
+	Usage: required
+	Value type: <stringlist>
+	Definition: must be "qcom,q6afe-v<MAJOR-NUMBER>.<MINOR-NUMBER>"
+		  Or "qcom,q6afe" where the version number can be queried
+		  from DSP.
+		  example "qcom,q6afe"
+
+= AFE DAIs (Digial Audio Interface)
+"dais" subnode of the AFE node. It represents afe dais, each afe dai is a
+subnode of "dais" representing board specific dai setup.
+"dais" node should have following properties followed by dai children.
+
+- #sound-dai-cells
+	Usage: required
+	Value type: <u32>
+	Definition: Must be 1
+
+- #address-cells
+	Usage: required
+	Value type: <u32>
+	Definition: Must be 1
+
+- #size-cells
+	Usage: required
+	Value type: <u32>
+	Definition: Must be 0
+
+== AFE DAI is subnode of "dais" and represent a dai, it includes board specific
+configuration of each dai. Must contain the following properties.
+
+- reg
+	Usage: required
+	Value type: <u32>
+	Definition: Must be dai id
+
+- qcom,sd-lines
+	Usage: required for mi2s interface
+	Value type: <prop-encoded-array>
+	Definition: Must be list of serial data lines used by this dai.
+	should be one or more of the 1-4 sd lines.
+
+ - qcom,tdm-sync-mode:
+	Usage: required for tdm interface
+	Value type: <prop-encoded-array>
+	Definition: Synchronization mode.
+		0 - Short sync bit mode
+		1 - Long sync mode
+		2 - Short sync slot mode
+
+ - qcom,tdm-sync-src:
+	Usage: required for tdm interface
+	Value type: <prop-encoded-array>
+	Definition: Synchronization source.
+		0 - External source
+		1 - Internal source
+
+ - qcom,tdm-data-out:
+	Usage: required for tdm interface
+	Value type: <prop-encoded-array>
+	Definition: Data out signal to drive with other masters.
+		0 - Disable
+		1 - Enable
+
+ - qcom,tdm-invert-sync:
+	Usage: required for tdm interface
+	Value type: <prop-encoded-array>
+	Definition: Invert the sync.
+		0 - Normal
+		1 - Invert
+
+ - qcom,tdm-data-delay:
+	Usage: required for tdm interface
+	Value type: <prop-encoded-array>
+	Definition: Number of bit clock to delay data
+		with respect to sync edge.
+		0 - 0 bit clock cycle
+		1 - 1 bit clock cycle
+		2 - 2 bit clock cycle
+
+ - qcom,tdm-data-align:
+	Usage: required for tdm interface
+	Value type: <prop-encoded-array>
+	Definition: Indicate how data is packed
+		within the slot. For example, 32 slot width in case of
+		sample bit width is 24.
+		0 - MSB
+		1 - LSB
+
+= EXAMPLE
+
+q6afe@4 {
+	compatible = "qcom,q6afe";
+	reg = <APR_SVC_AFE>;
+
+	dais {
+		#sound-dai-cells = <1>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		hdmi@1 {
+			reg = <1>;
+		};
+
+		tdm@24 {
+			reg = <24>;
+			qcom,tdm-sync-mode = <1>:
+			qcom,tdm-sync-src = <1>;
+			qcom,tdm-data-out = <0>;
+			qcom,tdm-invert-sync = <1>;
+			qcom,tdm-data-delay = <1>;
+			qcom,tdm-data-align = <0>;
+
+		};
+
+		tdm@25 {
+			reg = <25>;
+			qcom,tdm-sync-mode = <1>:
+			qcom,tdm-sync-src = <1>;
+			qcom,tdm-data-out = <0>;
+			qcom,tdm-invert-sync = <1>;
+			qcom,tdm-data-delay <1>:
+			qcom,tdm-data-align = <0>;
+		};
+
+		prim-mi2s-rx@16 {
+			reg = <16>;
+			qcom,sd-lines = <1 3>;
+		};
+
+		prim-mi2s-tx@17 {
+			reg = <17>;
+			qcom,sd-lines = <2>;
+		};
+
+		sec-mi2s-rx@18 {
+			reg = <18>;
+			qcom,sd-lines = <1 4>;
+		};
+
+		sec-mi2s-tx@19 {
+			reg = <19>;
+			qcom,sd-lines = <2>;
+		};
+
+		tert-mi2s-rx@20 {
+			reg = <20>;
+			qcom,sd-lines = <2 4>;
+		};
+
+		tert-mi2s-tx@21 {
+			reg = <21>;
+			qcom,sd-lines = <1>;
+		};
+
+		quat-mi2s-rx@22 {
+			reg = <22>;
+			qcom,sd-lines = <1>;
+		};
+
+		quat-mi2s-tx@23 {
+			reg = <23>;
+			qcom,sd-lines = <2>;
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6asm.txt b/Documentation/devicetree/bindings/sound/qcom,q6asm.txt
new file mode 100644
index 000000000000..2178eb91146f
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/qcom,q6asm.txt
@@ -0,0 +1,33 @@
+Qualcomm Audio Stream Manager (Q6ASM) binding
+
+Q6ASM is one of the APR audio service on Q6DSP.
+Please refer to qcom,apr.txt for details of the common apr service bindings
+used by the apr service device.
+
+- but must contain the following property:
+
+- compatible:
+	Usage: required
+	Value type: <stringlist>
+	Definition: must be "qcom,q6asm-v<MAJOR-NUMBER>.<MINOR-NUMBER>".
+		    Or "qcom,q6asm" where the version number can be queried
+		    from DSP.
+		    example "qcom,q6asm-v2.0"
+
+= ASM DAIs (Digial Audio Interface)
+"dais" subnode of the ASM node represents dai specific configuration
+
+- #sound-dai-cells
+	Usage: required
+	Value type: <u32>
+	Definition: Must be 1
+
+= EXAMPLE
+
+q6asm@7 {
+	compatible = "qcom,q6asm";
+	reg = <APR_SVC_ASM>;
+	q6asmdai: dais {
+		#sound-dai-cells = <1>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6core.txt b/Documentation/devicetree/bindings/sound/qcom,q6core.txt
new file mode 100644
index 000000000000..7f36ff8bec18
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/qcom,q6core.txt
@@ -0,0 +1,21 @@
+Qualcomm ADSP Core service binding
+
+Q6CORE is one of the APR audio service on Q6DSP.
+Please refer to qcom,apr.txt for details of the common apr service bindings
+used by the apr service device.
+
+- but must contain the following property:
+
+- compatible:
+	Usage: required
+	Value type: <stringlist>
+	Definition: must be "qcom,q6core-v<MAJOR-NUMBER>.<MINOR-NUMBER>".
+		   Or "qcom,q6core" where the version number can be queried
+		   from DSP.
+		   example "qcom,q6core-v2.0"
+
+= EXAMPLE
+q6core@3 {
+	compatible = "qcom,q6core";
+	reg = <APR_SVC_ADSP_CORE>;
+};
diff --git a/Documentation/devicetree/bindings/sound/rt274.txt b/Documentation/devicetree/bindings/sound/rt274.txt
index e9a6178c78cf..791a1bd767b9 100644
--- a/Documentation/devicetree/bindings/sound/rt274.txt
+++ b/Documentation/devicetree/bindings/sound/rt274.txt
@@ -26,7 +26,7 @@ Pins on the device (for linking into audio routes) for RT274:
 
 Example:
 
-codec: rt274@1c {
+rt274: codec@1c {
 	compatible = "realtek,rt274";
 	reg = <0x1c>;
 	interrupts = <7 IRQ_TYPE_EDGE_FALLING>;
diff --git a/Documentation/devicetree/bindings/sound/rt5514.txt b/Documentation/devicetree/bindings/sound/rt5514.txt
index 4f33b0d96afe..b25ed08c7a5a 100644
--- a/Documentation/devicetree/bindings/sound/rt5514.txt
+++ b/Documentation/devicetree/bindings/sound/rt5514.txt
@@ -32,7 +32,7 @@ Pins on the device (for linking into audio routes) for I2C:
 
 Example:
 
-codec: rt5514@57 {
+rt5514: codec@57 {
 	compatible = "realtek,rt5514";
 	reg = <0x57>;
 };
diff --git a/Documentation/devicetree/bindings/sound/rt5616.txt b/Documentation/devicetree/bindings/sound/rt5616.txt
index e41085818559..540a4bf252e4 100644
--- a/Documentation/devicetree/bindings/sound/rt5616.txt
+++ b/Documentation/devicetree/bindings/sound/rt5616.txt
@@ -26,7 +26,7 @@ Pins on the device (for linking into audio routes) for RT5616:
 
 Example:
 
-codec: rt5616@1b {
+rt5616: codec@1b {
 	compatible = "realtek,rt5616";
 	reg = <0x1b>;
 };
diff --git a/Documentation/devicetree/bindings/sound/rt5640.txt b/Documentation/devicetree/bindings/sound/rt5640.txt
index 57fe64643050..e40e4893eed8 100644
--- a/Documentation/devicetree/bindings/sound/rt5640.txt
+++ b/Documentation/devicetree/bindings/sound/rt5640.txt
@@ -22,6 +22,41 @@ Optional properties:
 
 - realtek,ldo1-en-gpios : The GPIO that controls the CODEC's LDO1_EN pin.
 
+- realtek,dmic1-data-pin
+  0: dmic1 is not used
+  1: using IN1P pin as dmic1 data pin
+  2: using GPIO3 pin as dmic1 data pin
+
+- realtek,dmic2-data-pin
+  0: dmic2 is not used
+  1: using IN1N pin as dmic2 data pin
+  2: using GPIO4 pin as dmic2 data pin
+
+- realtek,jack-detect-source
+  u32. Valid values:
+  0: jack-detect is not used
+  1: Use GPIO1 for jack-detect
+  2: Use JD1_IN4P for jack-detect
+  3: Use JD2_IN4N for jack-detect
+  4: Use GPIO2 for jack-detect
+  5: Use GPIO3 for jack-detect
+  6: Use GPIO4 for jack-detect
+
+- realtek,jack-detect-not-inverted
+  bool. Normal jack-detect switches give an inverted signal, set this bool
+  in the rare case you've a jack-detect switch which is not inverted.
+
+- realtek,over-current-threshold-microamp
+  u32, micbias over-current detection threshold in µA, valid values are
+  600, 1500 and 2000µA.
+
+- realtek,over-current-scale-factor
+  u32, micbias over-current detection scale-factor, valid values are:
+  0: Scale current by 0.5
+  1: Scale current by 0.75
+  2: Scale current by 1.0
+  3: Scale current by 1.5
+
 Pins on the device (for linking into audio routes) for RT5639/RT5640:
 
   * DMIC1
diff --git a/Documentation/devicetree/bindings/sound/rt5645.txt b/Documentation/devicetree/bindings/sound/rt5645.txt
index 7cee1f518f59..a03f9a872a71 100644
--- a/Documentation/devicetree/bindings/sound/rt5645.txt
+++ b/Documentation/devicetree/bindings/sound/rt5645.txt
@@ -69,4 +69,4 @@ codec: rt5650@1a {
 	realtek,dmic-en = "true";
 	realtek,en-jd-func = "true";
 	realtek,jd-mode = <3>;
-};
\ No newline at end of file
+};
diff --git a/Documentation/devicetree/bindings/sound/rt5651.txt b/Documentation/devicetree/bindings/sound/rt5651.txt
index b85221864cec..a41199a5cd79 100644
--- a/Documentation/devicetree/bindings/sound/rt5651.txt
+++ b/Documentation/devicetree/bindings/sound/rt5651.txt
@@ -50,7 +50,7 @@ Pins on the device (for linking into audio routes) for RT5651:
 
 Example:
 
-codec: rt5651@1a {
+rt5651: codec@1a {
 	compatible = "realtek,rt5651";
 	reg = <0x1a>;
 	realtek,dmic-en = "true";
diff --git a/Documentation/devicetree/bindings/sound/rt5663.txt b/Documentation/devicetree/bindings/sound/rt5663.txt
index 497bcfc58b71..23386446c63d 100644
--- a/Documentation/devicetree/bindings/sound/rt5663.txt
+++ b/Documentation/devicetree/bindings/sound/rt5663.txt
@@ -47,7 +47,7 @@ Pins on the device (for linking into audio routes) for RT5663:
 
 Example:
 
-codec: rt5663@12 {
+rt5663: codec@12 {
 	compatible = "realtek,rt5663";
 	reg = <0x12>;
 	interrupts = <7 IRQ_TYPE_EDGE_FALLING>;
diff --git a/Documentation/devicetree/bindings/sound/rt5668.txt b/Documentation/devicetree/bindings/sound/rt5668.txt
new file mode 100644
index 000000000000..c88b96e7764b
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/rt5668.txt
@@ -0,0 +1,50 @@
+RT5668B audio CODEC
+
+This device supports I2C only.
+
+Required properties:
+
+- compatible : "realtek,rt5668b"
+
+- reg : The I2C address of the device.
+
+Optional properties:
+
+- interrupts : The CODEC's interrupt output.
+
+- realtek,dmic1-data-pin
+  0: dmic1 is not used
+  1: using GPIO2 pin as dmic1 data pin
+  2: using GPIO5 pin as dmic1 data pin
+
+- realtek,dmic1-clk-pin
+  0: using GPIO1 pin as dmic1 clock pin
+  1: using GPIO3 pin as dmic1 clock pin
+
+- realtek,jd-src
+  0: No JD is used
+  1: using JD1 as JD source
+
+- realtek,ldo1-en-gpios : The GPIO that controls the CODEC's LDO1_EN pin.
+
+Pins on the device (for linking into audio routes) for RT5668B:
+
+  * DMIC L1
+  * DMIC R1
+  * IN1P
+  * HPOL
+  * HPOR
+
+Example:
+
+rt5668 {
+	compatible = "realtek,rt5668b";
+	reg = <0x1a>;
+	interrupt-parent = <&gpio>;
+	interrupts = <TEGRA_GPIO(U, 6) GPIO_ACTIVE_HIGH>;
+	realtek,ldo1-en-gpios =
+		<&gpio TEGRA_GPIO(R, 2) GPIO_ACTIVE_HIGH>;
+	realtek,dmic1-data-pin = <1>;
+	realtek,dmic1-clk-pin = <1>;
+	realtek,jd-src = <1>;
+};
diff --git a/Documentation/devicetree/bindings/sound/sgtl5000.txt b/Documentation/devicetree/bindings/sound/sgtl5000.txt
index 9a36c7e2a143..0f214457476f 100644
--- a/Documentation/devicetree/bindings/sound/sgtl5000.txt
+++ b/Documentation/devicetree/bindings/sound/sgtl5000.txt
@@ -39,7 +39,7 @@ VDDIO		1.8V		2.5V		3.3V
 
 Example:
 
-codec: sgtl5000@a {
+sgtl5000: codec@a {
 	compatible = "fsl,sgtl5000";
 	reg = <0x0a>;
 	#sound-dai-cells = <0>;
diff --git a/Documentation/devicetree/bindings/sound/ti,tas6424.txt b/Documentation/devicetree/bindings/sound/ti,tas6424.txt
index 1c4ada0eef4e..eacb54f34188 100644
--- a/Documentation/devicetree/bindings/sound/ti,tas6424.txt
+++ b/Documentation/devicetree/bindings/sound/ti,tas6424.txt
@@ -6,6 +6,8 @@ Required properties:
 	- compatible: "ti,tas6424" - TAS6424
 	- reg: I2C slave address
 	- sound-dai-cells: must be equal to 0
+	- standby-gpios: GPIO used to shut the TAS6424 down.
+	- mute-gpios: GPIO used to mute all the outputs
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/sound/tscs42xx.txt b/Documentation/devicetree/bindings/sound/tscs42xx.txt
index 2ac2f0996697..7eea32e9d078 100644
--- a/Documentation/devicetree/bindings/sound/tscs42xx.txt
+++ b/Documentation/devicetree/bindings/sound/tscs42xx.txt
@@ -8,9 +8,15 @@ Required Properties:
 	- reg : 	<0x71> for analog mic
 			<0x69> for digital mic
 
+	- clock-names:	Must one of  the following "mclk1", "xtal", "mclk2"
+
+	- clocks:	phandle of the clock that provides the codec sysclk
+
 Example:
 
 wookie: codec@69 {
 	compatible = "tempo,tscs42A2";
 	reg = <0x69>;
+	clock-names = "xtal";
+	clocks = <&audio_xtal>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8510.txt b/Documentation/devicetree/bindings/sound/wm8510.txt
index fa1a32b85577..e6b6cc041f89 100644
--- a/Documentation/devicetree/bindings/sound/wm8510.txt
+++ b/Documentation/devicetree/bindings/sound/wm8510.txt
@@ -12,7 +12,7 @@ Required properties:
 
 Example:
 
-codec: wm8510@1a {
+wm8510: codec@1a {
 	compatible = "wlf,wm8510";
 	reg = <0x1a>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8523.txt b/Documentation/devicetree/bindings/sound/wm8523.txt
index 04746186b283..f3a6485f4b8a 100644
--- a/Documentation/devicetree/bindings/sound/wm8523.txt
+++ b/Documentation/devicetree/bindings/sound/wm8523.txt
@@ -10,7 +10,7 @@ Required properties:
 
 Example:
 
-codec: wm8523@1a {
+wm8523: codec@1a {
 	compatible = "wlf,wm8523";
 	reg = <0x1a>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8524.txt b/Documentation/devicetree/bindings/sound/wm8524.txt
index 0f0553563fc1..f6c0c263b135 100644
--- a/Documentation/devicetree/bindings/sound/wm8524.txt
+++ b/Documentation/devicetree/bindings/sound/wm8524.txt
@@ -10,7 +10,7 @@ Required properties:
 
 Example:
 
-codec: wm8524 {
+wm8524: codec {
 	compatible = "wlf,wm8524";
 	wlf,mute-gpios = <&gpio1 8 GPIO_ACTIVE_LOW>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8580.txt b/Documentation/devicetree/bindings/sound/wm8580.txt
index 78fce9b14954..ff3f9f5f2111 100644
--- a/Documentation/devicetree/bindings/sound/wm8580.txt
+++ b/Documentation/devicetree/bindings/sound/wm8580.txt
@@ -10,7 +10,7 @@ Required properties:
 
 Example:
 
-codec: wm8580@1a {
+wm8580: codec@1a {
 	compatible = "wlf,wm8580";
 	reg = <0x1a>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8711.txt b/Documentation/devicetree/bindings/sound/wm8711.txt
index 8ed9998cd23c..c30a1387c4bf 100644
--- a/Documentation/devicetree/bindings/sound/wm8711.txt
+++ b/Documentation/devicetree/bindings/sound/wm8711.txt
@@ -12,7 +12,7 @@ Required properties:
 
 Example:
 
-codec: wm8711@1a {
+wm8711: codec@1a {
 	compatible = "wlf,wm8711";
 	reg = <0x1a>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8728.txt b/Documentation/devicetree/bindings/sound/wm8728.txt
index a8b5c3668e60..a3608b4c78b9 100644
--- a/Documentation/devicetree/bindings/sound/wm8728.txt
+++ b/Documentation/devicetree/bindings/sound/wm8728.txt
@@ -12,7 +12,7 @@ Required properties:
 
 Example:
 
-codec: wm8728@1a {
+wm8728: codec@1a {
 	compatible = "wlf,wm8728";
 	reg = <0x1a>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8731.txt b/Documentation/devicetree/bindings/sound/wm8731.txt
index 236690e99b87..f660d9bb0e69 100644
--- a/Documentation/devicetree/bindings/sound/wm8731.txt
+++ b/Documentation/devicetree/bindings/sound/wm8731.txt
@@ -12,7 +12,7 @@ Required properties:
 
 Example:
 
-codec: wm8731@1a {
+wm8731: codec@1a {
 	compatible = "wlf,wm8731";
 	reg = <0x1a>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8737.txt b/Documentation/devicetree/bindings/sound/wm8737.txt
index 4bc2cea3b140..eda1ec6a7563 100644
--- a/Documentation/devicetree/bindings/sound/wm8737.txt
+++ b/Documentation/devicetree/bindings/sound/wm8737.txt
@@ -12,7 +12,7 @@ Required properties:
 
 Example:
 
-codec: wm8737@1a {
+wm8737: codec@1a {
 	compatible = "wlf,wm8737";
 	reg = <0x1a>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8741.txt b/Documentation/devicetree/bindings/sound/wm8741.txt
index a13315408719..b69e196c741c 100644
--- a/Documentation/devicetree/bindings/sound/wm8741.txt
+++ b/Documentation/devicetree/bindings/sound/wm8741.txt
@@ -21,7 +21,7 @@ Optional properties:
 
 Example:
 
-codec: wm8741@1a {
+wm8741: codec@1a {
 	compatible = "wlf,wm8741";
 	reg = <0x1a>;
 
diff --git a/Documentation/devicetree/bindings/sound/wm8750.txt b/Documentation/devicetree/bindings/sound/wm8750.txt
index 8db239fd5ecd..682f221f6f38 100644
--- a/Documentation/devicetree/bindings/sound/wm8750.txt
+++ b/Documentation/devicetree/bindings/sound/wm8750.txt
@@ -12,7 +12,7 @@ Required properties:
 
 Example:
 
-codec: wm8750@1a {
+wm8750: codec@1a {
 	compatible = "wlf,wm8750";
 	reg = <0x1a>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8753.txt b/Documentation/devicetree/bindings/sound/wm8753.txt
index 8eee61282105..eca9e5a825a9 100644
--- a/Documentation/devicetree/bindings/sound/wm8753.txt
+++ b/Documentation/devicetree/bindings/sound/wm8753.txt
@@ -34,7 +34,7 @@ Pins on the device (for linking into audio routes):
 
 Example:
 
-codec: wm8753@1a {
+wm8753: codec@1a {
 	compatible = "wlf,wm8753";
 	reg = <0x1a>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8770.txt b/Documentation/devicetree/bindings/sound/wm8770.txt
index 866e00ca150b..cac762a1105d 100644
--- a/Documentation/devicetree/bindings/sound/wm8770.txt
+++ b/Documentation/devicetree/bindings/sound/wm8770.txt
@@ -10,7 +10,7 @@ Required properties:
 
 Example:
 
-codec: wm8770@1 {
+wm8770: codec@1 {
 	compatible = "wlf,wm8770";
 	reg = <1>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8776.txt b/Documentation/devicetree/bindings/sound/wm8776.txt
index 3b9ca49abc2b..01173369c3ed 100644
--- a/Documentation/devicetree/bindings/sound/wm8776.txt
+++ b/Documentation/devicetree/bindings/sound/wm8776.txt
@@ -12,7 +12,7 @@ Required properties:
 
 Example:
 
-codec: wm8776@1a {
+wm8776: codec@1a {
 	compatible = "wlf,wm8776";
 	reg = <0x1a>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8804.txt b/Documentation/devicetree/bindings/sound/wm8804.txt
index 6fd124b16496..2c1641c17a91 100644
--- a/Documentation/devicetree/bindings/sound/wm8804.txt
+++ b/Documentation/devicetree/bindings/sound/wm8804.txt
@@ -19,7 +19,7 @@ Optional properties:
 
 Example:
 
-codec: wm8804@1a {
+wm8804: codec@1a {
 	compatible = "wlf,wm8804";
 	reg = <0x1a>;
 };
diff --git a/Documentation/devicetree/bindings/sound/wm8903.txt b/Documentation/devicetree/bindings/sound/wm8903.txt
index afc51caf1137..6371c2434afe 100644
--- a/Documentation/devicetree/bindings/sound/wm8903.txt
+++ b/Documentation/devicetree/bindings/sound/wm8903.txt
@@ -57,7 +57,7 @@ Pins on the device (for linking into audio routes):
 
 Example:
 
-codec: wm8903@1a {
+wm8903: codec@1a {
 	compatible = "wlf,wm8903";
 	reg = <0x1a>;
 	interrupts = < 347 >;
diff --git a/Documentation/devicetree/bindings/sound/wm8960.txt b/Documentation/devicetree/bindings/sound/wm8960.txt
index 2deb8a3da9c5..6d29ac3750ee 100644
--- a/Documentation/devicetree/bindings/sound/wm8960.txt
+++ b/Documentation/devicetree/bindings/sound/wm8960.txt
@@ -23,7 +23,7 @@ Optional properties:
 
 Example:
 
-codec: wm8960@1a {
+wm8960: codec@1a {
 	compatible = "wlf,wm8960";
 	reg = <0x1a>;
 
diff --git a/Documentation/devicetree/bindings/sound/wm8962.txt b/Documentation/devicetree/bindings/sound/wm8962.txt
index 7f82b59ec8f9..dcfa9a3369fd 100644
--- a/Documentation/devicetree/bindings/sound/wm8962.txt
+++ b/Documentation/devicetree/bindings/sound/wm8962.txt
@@ -24,7 +24,7 @@ Optional properties:
 
 Example:
 
-codec: wm8962@1a {
+wm8962: codec@1a {
 	compatible = "wlf,wm8962";
 	reg = <0x1a>;
 
diff --git a/Documentation/devicetree/bindings/sound/wm8994.txt b/Documentation/devicetree/bindings/sound/wm8994.txt
index 68c4e8d96bed..4a9dead1b7d3 100644
--- a/Documentation/devicetree/bindings/sound/wm8994.txt
+++ b/Documentation/devicetree/bindings/sound/wm8994.txt
@@ -59,7 +59,7 @@ Optional properties:
 
 Example:
 
-codec: wm8994@1a {
+wm8994: codec@1a {
 	compatible = "wlf,wm8994";
 	reg = <0x1a>;
 
diff --git a/Documentation/devicetree/bindings/sparc_sun_oracle_rng.txt b/Documentation/devicetree/bindings/sparc_sun_oracle_rng.txt
deleted file mode 100644
index b0b211194c71..000000000000
--- a/Documentation/devicetree/bindings/sparc_sun_oracle_rng.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-HWRNG support for the n2_rng driver
-
-Required properties:
-- reg		: base address to sample from
-- compatible	: should contain one of the following
-	RNG versions:
-	- 'SUNW,n2-rng' for Niagara 2 Platform (SUN UltraSPARC T2 CPU)
-	- 'SUNW,vf-rng' for Victoria Falls Platform (SUN UltraSPARC T2 Plus CPU)
-	- 'SUNW,kt-rng' for Rainbow/Yosemite Falls Platform (SUN SPARC T3/T4), (UltraSPARC KT/Niagara 3 - development names)
-	more recent systems (after Oracle acquisition of SUN)
-	- 'ORCL,m4-rng' for SPARC T5/M5
-	- 'ORCL,m7-rng' for SPARC T7/M7
-
-Examples:
-/* linux LDOM on SPARC T5-2 */
-Node 0xf029a4f4
-	.node:  f029a4f4
-	rng-#units:  00000002
-	compatible: 'ORCL,m4-rng'
-	reg:  0000000e
-	name: 'random-number-generator'
-
-/* solaris on SPARC M7-8 */
-Node 0xf028c08c
-	rng-#units:  00000003
-	compatible: 'ORCL,m7-rng'
-	reg:  0000000e
-	name:  'random-number-generator'
-
-PS: see as well prtconfs.git by DaveM
diff --git a/Documentation/devicetree/bindings/thermal/exynos-thermal.txt b/Documentation/devicetree/bindings/thermal/exynos-thermal.txt
index b957acff57aa..ad648d93d961 100644
--- a/Documentation/devicetree/bindings/thermal/exynos-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/exynos-thermal.txt
@@ -12,7 +12,6 @@
 	       "samsung,exynos5420-tmu-ext-triminfo" for TMU channels 2, 3 and 4
 			Exynos5420 (Must pass triminfo base and triminfo clock)
                "samsung,exynos5433-tmu"
-	       "samsung,exynos5440-tmu"
 	       "samsung,exynos7-tmu"
 - interrupt-parent : The phandle for the interrupt controller
 - reg : Address range of the thermal registers. For soc's which has multiple
@@ -68,18 +67,7 @@ Example 1):
 		#thermal-sensor-cells = <0>;
 	};
 
-Example 2):
-
-	tmuctrl_0: tmuctrl@160118 {
-		compatible = "samsung,exynos5440-tmu";
-		reg = <0x160118 0x230>, <0x160368 0x10>;
-		interrupts = <0 58 0>;
-		clocks = <&clock 21>;
-		clock-names = "tmu_apbif";
-		#thermal-sensor-cells = <0>;
-	};
-
-Example 3): (In case of Exynos5420 "with misplaced TRIMINFO register")
+Example 2): (In case of Exynos5420 "with misplaced TRIMINFO register")
 	tmu_cpu2: tmu@10068000 {
 		compatible = "samsung,exynos5420-tmu-ext-triminfo";
 		reg = <0x10068000 0x100>, <0x1006c000 0x4>;
diff --git a/Documentation/devicetree/bindings/thermal/imx-thermal.txt b/Documentation/devicetree/bindings/thermal/imx-thermal.txt
index 379eb763073e..823e4176eef8 100644
--- a/Documentation/devicetree/bindings/thermal/imx-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/imx-thermal.txt
@@ -1,8 +1,13 @@
 * Temperature Monitor (TEMPMON) on Freescale i.MX SoCs
 
 Required properties:
-- compatible : "fsl,imx6q-tempmon" for i.MX6Q, "fsl,imx6sx-tempmon" for i.MX6SX.
-  i.MX6SX has two more IRQs than i.MX6Q, one is IRQ_LOW and the other is IRQ_PANIC,
+- compatible : must be one of following:
+  - "fsl,imx6q-tempmon" for i.MX6Q,
+  - "fsl,imx6sx-tempmon" for i.MX6SX,
+  - "fsl,imx7d-tempmon" for i.MX7S/D.
+- interrupts : the interrupt output of the controller:
+  i.MX6Q has one IRQ which will be triggered when temperature is higher than high threshold,
+  i.MX6SX and i.MX7S/D have two more IRQs than i.MX6Q, one is IRQ_LOW and the other is IRQ_PANIC,
   when temperature is below than low threshold, IRQ_LOW will be triggered, when temperature
   is higher than panic threshold, system will auto reboot by SRC module.
 - fsl,tempmon : phandle pointer to system controller that contains TEMPMON
diff --git a/Documentation/devicetree/bindings/thermal/mediatek-thermal.txt b/Documentation/devicetree/bindings/thermal/mediatek-thermal.txt
index 0d73ea5e9c0c..41d6a443ad66 100644
--- a/Documentation/devicetree/bindings/thermal/mediatek-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/mediatek-thermal.txt
@@ -12,6 +12,7 @@ Required properties:
   - "mediatek,mt8173-thermal" : For MT8173 family of SoCs
   - "mediatek,mt2701-thermal" : For MT2701 family of SoCs
   - "mediatek,mt2712-thermal" : For MT2712 family of SoCs
+  - "mediatek,mt7622-thermal" : For MT7622 SoC
 - reg: Address range of the thermal controller
 - interrupts: IRQ for the thermal controller
 - clocks, clock-names: Clocks needed for the thermal controller. required
diff --git a/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.txt b/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.txt
index fdf5caa6229b..cfa154bb0fa7 100644
--- a/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.txt
@@ -9,6 +9,7 @@ Required properties:
 			  Examples with soctypes are:
 			    - "renesas,r8a7795-thermal" (R-Car H3)
 			    - "renesas,r8a7796-thermal" (R-Car M3-W)
+			    - "renesas,r8a77965-thermal" (R-Car M3-N)
 - reg			: Address ranges of the thermal registers. Each sensor
 			  needs one address range. Sorting must be done in
 			  increasing order according to datasheet, i.e.
@@ -18,7 +19,7 @@ Required properties:
 
 Optional properties:
 
-- interrupts           : interrupts routed to the TSC (3 for H3 and M3-W)
+- interrupts           : interrupts routed to the TSC (3 for H3, M3-W and M3-N)
 - power-domain		: Must contain a reference to the power domain. This
 			  property is mandatory if the thermal sensor instance
 			  is part of a controllable power domain.
@@ -27,9 +28,9 @@ Example:
 
 	tsc: thermal@e6198000 {
 		compatible = "renesas,r8a7795-thermal";
-		reg = <0 0xe6198000 0 0x68>,
-		      <0 0xe61a0000 0 0x5c>,
-		      <0 0xe61a8000 0 0x5c>;
+		reg = <0 0xe6198000 0 0x100>,
+		      <0 0xe61a0000 0 0x100>,
+		      <0 0xe61a8000 0 0x100>;
 		interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/Documentation/devicetree/bindings/thermal/uniphier-thermal.txt b/Documentation/devicetree/bindings/thermal/uniphier-thermal.txt
index 686c0b42ed3f..ceb92a95727a 100644
--- a/Documentation/devicetree/bindings/thermal/uniphier-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/uniphier-thermal.txt
@@ -8,6 +8,7 @@ Required properties:
 - compatible :
   - "socionext,uniphier-pxs2-thermal" : For UniPhier PXs2 SoC
   - "socionext,uniphier-ld20-thermal" : For UniPhier LD20 SoC
+  - "socionext,uniphier-pxs3-thermal" : For UniPhier PXs3 SoC
 - interrupts : IRQ for the temperature alarm
 - #thermal-sensor-cells : Should be 0. See ./thermal.txt for details.
 
diff --git a/Documentation/devicetree/bindings/timer/altr,timer-1.0.txt b/Documentation/devicetree/bindings/timer/altr,timer-1.0.txt
new file mode 100644
index 000000000000..904a5846d7ac
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/altr,timer-1.0.txt
@@ -0,0 +1,19 @@
+Altera Timer
+
+Required properties:
+
+- compatible : should be "altr,timer-1.0"
+- reg : Specifies base physical address and size of the registers.
+- interrupt-parent: phandle of the interrupt controller
+- interrupts : Should contain the timer interrupt number
+- clock-frequency : The frequency of the clock that drives the counter, in Hz.
+
+Example:
+
+timer {
+	compatible = "altr,timer-1.0";
+	reg = <0x00400000 0x00000020>;
+	interrupt-parent = <&cpu>;
+	interrupts = <11>;
+	clock-frequency = <125000000>;
+};
diff --git a/Documentation/devicetree/bindings/timer/arm,arch_timer.txt b/Documentation/devicetree/bindings/timer/arm,arch_timer.txt
new file mode 100644
index 000000000000..68301b77e854
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/arm,arch_timer.txt
@@ -0,0 +1,112 @@
+* ARM architected timer
+
+ARM cores may have a per-core architected timer, which provides per-cpu timers,
+or a memory mapped architected timer, which provides up to 8 frames with a
+physical and optional virtual timer per frame.
+
+The per-core architected timer is attached to a GIC to deliver its
+per-processor interrupts via PPIs. The memory mapped timer is attached to a GIC
+to deliver its interrupts via SPIs.
+
+** CP15 Timer node properties:
+
+- compatible : Should at least contain one of
+	"arm,armv7-timer"
+	"arm,armv8-timer"
+
+- interrupts : Interrupt list for secure, non-secure, virtual and
+  hypervisor timers, in that order.
+
+- clock-frequency : The frequency of the main counter, in Hz. Should be present
+  only where necessary to work around broken firmware which does not configure
+  CNTFRQ on all CPUs to a uniform correct value. Use of this property is
+  strongly discouraged; fix your firmware unless absolutely impossible.
+
+- always-on : a boolean property. If present, the timer is powered through an
+  always-on power domain, therefore it never loses context.
+
+- fsl,erratum-a008585 : A boolean property. Indicates the presence of
+  QorIQ erratum A-008585, which says that reading the counter is
+  unreliable unless the same value is returned by back-to-back reads.
+  This also affects writes to the tval register, due to the implicit
+  counter read.
+
+- hisilicon,erratum-161010101 : A boolean property. Indicates the
+  presence of Hisilicon erratum 161010101, which says that reading the
+  counters is unreliable in some cases, and reads may return a value 32
+  beyond the correct value. This also affects writes to the tval
+  registers, due to the implicit counter read.
+
+** Optional properties:
+
+- arm,cpu-registers-not-fw-configured : Firmware does not initialize
+  any of the generic timer CPU registers, which contain their
+  architecturally-defined reset values. Only supported for 32-bit
+  systems which follow the ARMv7 architected reset values.
+
+- arm,no-tick-in-suspend : The main counter does not tick when the system is in
+  low-power system suspend on some SoCs. This behavior does not match the
+  Architecture Reference Manual's specification that the system counter "must
+  be implemented in an always-on power domain."
+
+
+Example:
+
+	timer {
+		compatible = "arm,cortex-a15-timer",
+			     "arm,armv7-timer";
+		interrupts = <1 13 0xf08>,
+			     <1 14 0xf08>,
+			     <1 11 0xf08>,
+			     <1 10 0xf08>;
+		clock-frequency = <100000000>;
+	};
+
+** Memory mapped timer node properties:
+
+- compatible : Should at least contain "arm,armv7-timer-mem".
+
+- clock-frequency : The frequency of the main counter, in Hz. Should be present
+  only when firmware has not configured the MMIO CNTFRQ registers.
+
+- reg : The control frame base address.
+
+Note that #address-cells, #size-cells, and ranges shall be present to ensure
+the CPU can address a frame's registers.
+
+A timer node has up to 8 frame sub-nodes, each with the following properties:
+
+- frame-number: 0 to 7.
+
+- interrupts : Interrupt list for physical and virtual timers in that order.
+  The virtual timer interrupt is optional.
+
+- reg : The first and second view base addresses in that order. The second view
+  base address is optional.
+
+- status : "disabled" indicates the frame is not available for use. Optional.
+
+Example:
+
+	timer@f0000000 {
+		compatible = "arm,armv7-timer-mem";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+		reg = <0xf0000000 0x1000>;
+		clock-frequency = <50000000>;
+
+		frame@f0001000 {
+			frame-number = <0>
+			interrupts = <0 13 0x8>,
+				     <0 14 0x8>;
+			reg = <0xf0001000 0x1000>,
+			      <0xf0002000 0x1000>;
+		};
+
+		frame@f0003000 {
+			frame-number = <1>
+			interrupts = <0 15 0x8>;
+			reg = <0xf0003000 0x1000>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/timer/arm,armv7m-systick.txt b/Documentation/devicetree/bindings/timer/arm,armv7m-systick.txt
new file mode 100644
index 000000000000..7cf4a24601eb
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/arm,armv7m-systick.txt
@@ -0,0 +1,26 @@
+* ARMv7M System Timer
+
+ARMv7-M includes a system timer, known as SysTick. Current driver only
+implements the clocksource feature.
+
+Required properties:
+- compatible	  : Should be "arm,armv7m-systick"
+- reg		  : The address range of the timer
+
+Required clocking property, have to be one of:
+- clocks	  : The input clock of the timer
+- clock-frequency : The rate in HZ in input of the ARM SysTick
+
+Examples:
+
+systick: timer@e000e010 {
+	compatible = "arm,armv7m-systick";
+	reg = <0xe000e010 0x10>;
+	clocks = <&clk_systick>;
+};
+
+systick: timer@e000e010 {
+	compatible = "arm,armv7m-systick";
+	reg = <0xe000e010 0x10>;
+	clock-frequency = <90000000>;
+};
diff --git a/Documentation/devicetree/bindings/timer/arm,global_timer.txt b/Documentation/devicetree/bindings/timer/arm,global_timer.txt
new file mode 100644
index 000000000000..bdae3a818793
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/arm,global_timer.txt
@@ -0,0 +1,27 @@
+
+* ARM Global Timer
+	Cortex-A9 are often associated with a per-core Global timer.
+
+** Timer node required properties:
+
+- compatible : should contain
+	     * "arm,cortex-a5-global-timer" for Cortex-A5 global timers.
+	     * "arm,cortex-a9-global-timer" for Cortex-A9 global
+	         timers or any compatible implementation. Note: driver
+	         supports versions r2p0 and above.
+
+- interrupts : One interrupt to each core
+
+- reg : Specify the base address and the size of the GT timer
+	register window.
+
+- clocks : Should be phandle to a clock.
+
+Example:
+
+	timer@2c000600 {
+		compatible = "arm,cortex-a9-global-timer";
+		reg = <0x2c000600 0x20>;
+		interrupts = <1 13 0xf01>;
+		clocks = <&arm_periph_clk>;
+	};
diff --git a/Documentation/devicetree/bindings/timer/arm,twd.txt b/Documentation/devicetree/bindings/timer/arm,twd.txt
new file mode 100644
index 000000000000..383ea19c2bf0
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/arm,twd.txt
@@ -0,0 +1,53 @@
+* ARM Timer Watchdog
+
+ARM 11MP, Cortex-A5 and Cortex-A9 are often associated with a per-core
+Timer-Watchdog (aka TWD), which provides both a per-cpu local timer
+and watchdog.
+
+The TWD is usually attached to a GIC to deliver its two per-processor
+interrupts.
+
+** Timer node required properties:
+
+- compatible : Should be one of:
+	"arm,cortex-a9-twd-timer"
+	"arm,cortex-a5-twd-timer"
+	"arm,arm11mp-twd-timer"
+
+- interrupts : One interrupt to each core
+
+- reg : Specify the base address and the size of the TWD timer
+	register window.
+
+Optional
+
+- always-on : a boolean property. If present, the timer is powered through
+  an always-on power domain, therefore it never loses context.
+
+Example:
+
+	twd-timer@2c000600 {
+		compatible = "arm,arm11mp-twd-timer"";
+		reg = <0x2c000600 0x20>;
+		interrupts = <1 13 0xf01>;
+	};
+
+** Watchdog node properties:
+
+- compatible : Should be one of:
+	"arm,cortex-a9-twd-wdt"
+	"arm,cortex-a5-twd-wdt"
+	"arm,arm11mp-twd-wdt"
+
+- interrupts : One interrupt to each core
+
+- reg : Specify the base address and the size of the TWD watchdog
+	register window.
+
+Example:
+
+	twd-watchdog@2c000620 {
+		compatible = "arm,arm11mp-twd-wdt";
+		reg = <0x2c000620 0x20>;
+		interrupts = <1 14 0xf01>;
+	};
diff --git a/Documentation/devicetree/bindings/timer/fsl,gtm.txt b/Documentation/devicetree/bindings/timer/fsl,gtm.txt
new file mode 100644
index 000000000000..9a33efded4bc
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/fsl,gtm.txt
@@ -0,0 +1,31 @@
+* Freescale General-purpose Timers Module
+
+Required properties:
+  - compatible : should be
+    "fsl,<chip>-gtm", "fsl,gtm" for SOC GTMs
+    "fsl,<chip>-qe-gtm", "fsl,qe-gtm", "fsl,gtm" for QE GTMs
+    "fsl,<chip>-cpm2-gtm", "fsl,cpm2-gtm", "fsl,gtm" for CPM2 GTMs
+  - reg : should contain gtm registers location and length (0x40).
+  - interrupts : should contain four interrupts.
+  - interrupt-parent : interrupt source phandle.
+  - clock-frequency : specifies the frequency driving the timer.
+
+Example:
+
+timer@500 {
+	compatible = "fsl,mpc8360-gtm", "fsl,gtm";
+	reg = <0x500 0x40>;
+	interrupts = <90 8 78 8 84 8 72 8>;
+	interrupt-parent = <&ipic>;
+	/* filled by u-boot */
+	clock-frequency = <0>;
+};
+
+timer@440 {
+	compatible = "fsl,mpc8360-qe-gtm", "fsl,qe-gtm", "fsl,gtm";
+	reg = <0x440 0x40>;
+	interrupts = <12 13 14 15>;
+	interrupt-parent = <&qeic>;
+	/* filled by u-boot */
+	clock-frequency = <0>;
+};
diff --git a/Documentation/devicetree/bindings/timer/mrvl,mmp-timer.txt b/Documentation/devicetree/bindings/timer/mrvl,mmp-timer.txt
new file mode 100644
index 000000000000..9a6e251462e7
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/mrvl,mmp-timer.txt
@@ -0,0 +1,13 @@
+* Marvell MMP Timer controller
+
+Required properties:
+- compatible : Should be "mrvl,mmp-timer".
+- reg : Address and length of the register set of timer controller.
+- interrupts : Should be the interrupt number.
+
+Example:
+	timer0: timer@d4014000 {
+		compatible = "mrvl,mmp-timer";
+		reg = <0xd4014000 0x100>;
+		interrupts = <13>;
+	};
diff --git a/Documentation/devicetree/bindings/timer/qcom,msm-timer.txt b/Documentation/devicetree/bindings/timer/qcom,msm-timer.txt
new file mode 100644
index 000000000000..5e10c345548f
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/qcom,msm-timer.txt
@@ -0,0 +1,47 @@
+* MSM Timer
+
+Properties:
+
+- compatible : Should at least contain "qcom,msm-timer". More specific
+               properties specify which subsystem the timers are paired with.
+
+               "qcom,kpss-timer" - krait subsystem
+               "qcom,scss-timer" - scorpion subsystem
+
+- interrupts : Interrupts for the debug timer, the first general purpose
+               timer, and optionally a second general purpose timer, and
+               optionally as well, 2 watchdog interrupts, in that order.
+
+- reg : Specifies the base address of the timer registers.
+
+- clocks: Reference to the parent clocks, one per output clock. The parents
+          must appear in the same order as the clock names.
+
+- clock-names: The name of the clocks as free-form strings. They should be in
+               the same order as the clocks.
+
+- clock-frequency : The frequency of the debug timer and the general purpose
+                    timer(s) in Hz in that order.
+
+Optional:
+
+- cpu-offset : per-cpu offset used when the timer is accessed without the
+               CPU remapping facilities. The offset is
+               cpu-offset + (0x10000 * cpu-nr).
+
+Example:
+
+       timer@200a000 {
+               compatible = "qcom,scss-timer", "qcom,msm-timer";
+               interrupts = <1 1 0x301>,
+                            <1 2 0x301>,
+                            <1 3 0x301>,
+                            <1 4 0x301>,
+                            <1 5 0x301>;
+               reg = <0x0200a000 0x100>;
+               clock-frequency = <19200000>,
+                                 <32768>;
+               clocks = <&sleep_clk>;
+               clock-names = "sleep";
+               cpu-offset = <0x40000>;
+       };
diff --git a/Documentation/devicetree/bindings/timer/renesas,cmt.txt b/Documentation/devicetree/bindings/timer/renesas,cmt.txt
index d740989eb569..b40add2d9bb4 100644
--- a/Documentation/devicetree/bindings/timer/renesas,cmt.txt
+++ b/Documentation/devicetree/bindings/timer/renesas,cmt.txt
@@ -22,6 +22,10 @@ Required Properties:
 
     - "renesas,r8a73a4-cmt0" for the 32-bit CMT0 device included in r8a73a4.
     - "renesas,r8a73a4-cmt1" for the 48-bit CMT1 device included in r8a73a4.
+    - "renesas,r8a7743-cmt0" for the 32-bit CMT0 device included in r8a7743.
+    - "renesas,r8a7743-cmt1" for the 48-bit CMT1 device included in r8a7743.
+    - "renesas,r8a7745-cmt0" for the 32-bit CMT0 device included in r8a7745.
+    - "renesas,r8a7745-cmt1" for the 48-bit CMT1 device included in r8a7745.
     - "renesas,r8a7790-cmt0" for the 32-bit CMT0 device included in r8a7790.
     - "renesas,r8a7790-cmt1" for the 48-bit CMT1 device included in r8a7790.
     - "renesas,r8a7791-cmt0" for the 32-bit CMT0 device included in r8a7791.
@@ -31,10 +35,12 @@ Required Properties:
     - "renesas,r8a7794-cmt0" for the 32-bit CMT0 device included in r8a7794.
     - "renesas,r8a7794-cmt1" for the 48-bit CMT1 device included in r8a7794.
 
-    - "renesas,rcar-gen2-cmt0" for 32-bit CMT0 devices included in R-Car Gen2.
-    - "renesas,rcar-gen2-cmt1" for 48-bit CMT1 devices included in R-Car Gen2.
-		These are fallbacks for r8a73a4 and all the R-Car Gen2
-		entries	listed above.
+    - "renesas,rcar-gen2-cmt0" for 32-bit CMT0 devices included in R-Car Gen2
+		and RZ/G1.
+    - "renesas,rcar-gen2-cmt1" for 48-bit CMT1 devices included in R-Car Gen2
+		and RZ/G1.
+		These are fallbacks for r8a73a4, R-Car Gen2 and RZ/G1 entries
+		listed above.
 
   - reg: base address and length of the registers block for the timer module.
   - interrupts: interrupt-specifier for the timer, one per channel.
diff --git a/Documentation/devicetree/bindings/timer/st,spear-timer.txt b/Documentation/devicetree/bindings/timer/st,spear-timer.txt
new file mode 100644
index 000000000000..c0017221cf55
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/st,spear-timer.txt
@@ -0,0 +1,18 @@
+* SPEAr ARM Timer
+
+** Timer node required properties:
+
+- compatible : Should be:
+	"st,spear-timer"
+- reg: Address range of the timer registers
+- interrupt-parent: Should be the phandle for the interrupt controller
+  that services interrupts for this device
+- interrupt: Should contain the timer interrupt number
+
+Example:
+
+	timer@f0000000 {
+		compatible = "st,spear-timer";
+		reg = <0xf0000000 0x400>;
+		interrupts = <2>;
+	};
diff --git a/Documentation/devicetree/bindings/timer/ti,c64x+timer64.txt b/Documentation/devicetree/bindings/timer/ti,c64x+timer64.txt
new file mode 100644
index 000000000000..95911fe70224
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/ti,c64x+timer64.txt
@@ -0,0 +1,26 @@
+Timer64
+-------
+
+The timer64 node describes C6X event timers.
+
+Required properties:
+
+- compatible: must be "ti,c64x+timer64"
+- reg: base address and size of register region
+- interrupt-parent: interrupt controller
+- interrupts: interrupt id
+
+Optional properties:
+
+- ti,dscr-dev-enable: Device ID used to enable timer IP through DSCR interface.
+
+- ti,core-mask: on multi-core SoCs, bitmask of cores allowed to use this timer.
+
+Example:
+	timer0: timer@25e0000 {
+		compatible = "ti,c64x+timer64";
+		ti,core-mask = < 0x01 >;
+		reg = <0x25e0000 0x40>;
+		interrupt-parent = <&megamod_pic>;
+		interrupts = < 16 >;
+	};
diff --git a/Documentation/devicetree/bindings/timer/ti,timer.txt b/Documentation/devicetree/bindings/timer/ti,timer.txt
new file mode 100644
index 000000000000..d02e27c764ec
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/ti,timer.txt
@@ -0,0 +1,44 @@
+OMAP Timer bindings
+
+Required properties:
+- compatible:		Should be set to one of the below. Please note that
+			OMAP44xx devices have timer instances that are 100%
+			register compatible with OMAP3xxx devices as well as
+			newer timers that are not 100% register compatible.
+			So for OMAP44xx devices timer instances may use
+			different compatible strings.
+
+			ti,omap2420-timer (applicable to OMAP24xx devices)
+			ti,omap3430-timer (applicable to OMAP3xxx/44xx devices)
+			ti,omap4430-timer (applicable to OMAP44xx devices)
+			ti,omap5430-timer (applicable to OMAP543x devices)
+			ti,am335x-timer	(applicable to AM335x devices)
+			ti,am335x-timer-1ms (applicable to AM335x devices)
+
+- reg:			Contains timer register address range (base address and
+			length).
+- interrupts: 		Contains the interrupt information for the timer. The
+			format is being dependent on which interrupt controller
+			the OMAP device uses.
+- ti,hwmods:		Name of the hwmod associated to the timer, "timer<X>",
+			where <X> is the instance number of the timer from the
+			HW spec.
+
+Optional properties:
+- ti,timer-alwon:	Indicates the timer is in an alway-on power domain.
+- ti,timer-dsp:		Indicates the timer can interrupt the on-chip DSP in
+			addition to the ARM CPU.
+- ti,timer-pwm: 	Indicates the timer can generate a PWM output.
+- ti,timer-secure: 	Indicates the timer is reserved on a secure OMAP device
+			and therefore cannot be used by the kernel.
+
+Example:
+
+timer12: timer@48304000 {
+	compatible = "ti,omap3430-timer";
+	reg = <0x48304000 0x400>;
+	interrupts = <95>;
+	ti,hwmods = "timer12"
+	ti,timer-alwon;
+	ti,timer-secure;
+};
diff --git a/Documentation/devicetree/bindings/timer/via,vt8500-timer.txt b/Documentation/devicetree/bindings/timer/via,vt8500-timer.txt
new file mode 100644
index 000000000000..901c73f0d8ef
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/via,vt8500-timer.txt
@@ -0,0 +1,15 @@
+VIA/Wondermedia VT8500 Timer
+-----------------------------------------------------
+
+Required properties:
+- compatible : "via,vt8500-timer"
+- reg : Should contain 1 register ranges(address and length)
+- interrupts : interrupt for the timer
+
+Example:
+
+	timer@d8130100 {
+		compatible = "via,vt8500-timer";
+		reg = <0xd8130100 0x28>;
+		interrupts = <36>;
+	};
diff --git a/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.txt b/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.txt
index 0e03344e2e8b..2e9318151df7 100644
--- a/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.txt
+++ b/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.txt
@@ -76,6 +76,10 @@ Optional properties:
   needs to make sure it does not send more than 90%
   maximum_periodic_data_per_frame. The use case is multiple transactions, but
   less frame rate.
+- mux-controls: The mux control for toggling host/device output of this
+  controller. It's expected that a mux state of 0 indicates device mode and a
+  mux state of 1 indicates host mode.
+- mux-control-names: Shall be "usb_switch" if mux-controls is specified.
 
 i.mx specific properties
 - fsl,usbmisc: phandler of non-core register device, with one
@@ -102,4 +106,6 @@ Example:
 		rx-burst-size-dword = <0x10>;
 		extcon = <0>, <&usb_id>;
 		phy-clkgate-delay-us = <400>;
+		mux-controls = <&usb_switch>;
+		mux-control-names = "usb_switch";
 	};
diff --git a/Documentation/devicetree/bindings/usb/dwc3.txt b/Documentation/devicetree/bindings/usb/dwc3.txt
index 0dbd3083e7dd..7f13ebef06cb 100644
--- a/Documentation/devicetree/bindings/usb/dwc3.txt
+++ b/Documentation/devicetree/bindings/usb/dwc3.txt
@@ -7,6 +7,26 @@ Required properties:
  - compatible: must be "snps,dwc3"
  - reg : Address and length of the register set for the device
  - interrupts: Interrupts used by the dwc3 controller.
+ - clock-names: should contain "ref", "bus_early", "suspend"
+ - clocks: list of phandle and clock specifier pairs corresponding to
+           entries in the clock-names property.
+
+Exception for clocks:
+  clocks are optional if the parent node (i.e. glue-layer) is compatible to
+  one of the following:
+    "amlogic,meson-axg-dwc3"
+    "amlogic,meson-gxl-dwc3"
+    "cavium,octeon-7130-usb-uctl"
+    "qcom,dwc3"
+    "samsung,exynos5250-dwusb3"
+    "samsung,exynos7-dwusb3"
+    "sprd,sc9860-dwc3"
+    "st,stih407-dwc3"
+    "ti,am437x-dwc3"
+    "ti,dwc3"
+    "ti,keystone-dwc3"
+    "rockchip,rk3399-dwc3"
+    "xlnx,zynqmp-dwc3"
 
 Optional properties:
  - usb-phy : array of phandle for the PHY device.  The first element
@@ -15,6 +35,7 @@ Optional properties:
  - phys: from the *Generic PHY* bindings
  - phy-names: from the *Generic PHY* bindings; supported names are "usb2-phy"
 	or "usb3-phy".
+ - resets: a single pair of phandle and reset specifier
  - snps,usb3_lpm_capable: determines if platform is USB3 LPM capable
  - snps,disable_scramble_quirk: true when SW should disable data scrambling.
 	Only really useful for FPGA builds.
diff --git a/Documentation/devicetree/bindings/usb/fcs,fusb302.txt b/Documentation/devicetree/bindings/usb/fcs,fusb302.txt
index 472facfa5a71..6087dc7f209e 100644
--- a/Documentation/devicetree/bindings/usb/fcs,fusb302.txt
+++ b/Documentation/devicetree/bindings/usb/fcs,fusb302.txt
@@ -6,12 +6,6 @@ Required properties :
 - interrupts             : Interrupt specifier
 
 Optional properties :
-- fcs,max-sink-microvolt : Maximum voltage to negotiate when configured as sink
-- fcs,max-sink-microamp  : Maximum current to negotiate when configured as sink
-- fcs,max-sink-microwatt : Maximum power to negotiate when configured as sink
-			   If this is less then max-sink-microvolt *
-			   max-sink-microamp then the configured current will
-			   be clamped.
 - fcs,operating-sink-microwatt :
 			   Minimum amount of power accepted from a sink
 			   when negotiating
diff --git a/Documentation/devicetree/bindings/usb/hisilicon,histb-xhci.txt b/Documentation/devicetree/bindings/usb/hisilicon,histb-xhci.txt
new file mode 100644
index 000000000000..f4633496b122
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/hisilicon,histb-xhci.txt
@@ -0,0 +1,45 @@
+HiSilicon STB xHCI
+
+The device node for HiSilicon STB xHCI host controller
+
+Required properties:
+ - compatible: should be "hisilicon,hi3798cv200-xhci"
+ - reg: specifies physical base address and size of the registers
+ - interrupts : interrupt used by the controller
+ - clocks: a list of phandle + clock-specifier pairs, one for each
+	entry in clock-names
+ - clock-names: must contain
+	"bus": for bus clock
+	"utmi": for utmi clock
+	"pipe": for pipe clock
+	"suspend": for suspend clock
+ - resets: a list of phandle and reset specifier pairs as listed in
+	reset-names property.
+ - reset-names: must contain
+	"soft": for soft reset
+ - phys: a list of phandle + phy specifier pairs
+ - phy-names: must contain at least one of following:
+	"inno": for inno phy
+	"combo": for combo phy
+
+Optional properties:
+  - usb2-lpm-disable: indicate if we don't want to enable USB2 HW LPM
+  - usb3-lpm-capable: determines if platform is USB3 LPM capable
+  - imod-interval-ns: default interrupt moderation interval is 40000ns
+
+Example:
+
+xhci0: xchi@f98a0000 {
+	compatible = "hisilicon,hi3798cv200-xhci";
+	reg = <0xf98a0000 0x10000>;
+	interrupts = <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
+	clocks = <&crg HISTB_USB3_BUS_CLK>,
+		 <&crg HISTB_USB3_UTMI_CLK>,
+		 <&crg HISTB_USB3_PIPE_CLK>,
+		 <&crg HISTB_USB3_SUSPEND_CLK>;
+	clock-names = "bus", "utmi", "pipe", "suspend";
+	resets = <&crg 0xb0 12>;
+	reset-names = "soft";
+	phys = <&usb2_phy1_port1 0>, <&combphy0 PHY_TYPE_USB3>;
+	phy-names = "inno", "combo";
+};
diff --git a/Documentation/devicetree/bindings/usb/qcom,dwc3.txt b/Documentation/devicetree/bindings/usb/qcom,dwc3.txt
index bc8a2fa5d2bf..95afdcf3c337 100644
--- a/Documentation/devicetree/bindings/usb/qcom,dwc3.txt
+++ b/Documentation/devicetree/bindings/usb/qcom,dwc3.txt
@@ -1,54 +1,95 @@
 Qualcomm SuperSpeed DWC3 USB SoC controller
 
 Required properties:
-- compatible:	should contain "qcom,dwc3"
+- compatible:		Compatible list, contains
+			"qcom,dwc3"
+			"qcom,msm8996-dwc3" for msm8996 SOC.
+			"qcom,sdm845-dwc3" for sdm845 SOC.
+- reg:			Offset and length of register set for QSCRATCH wrapper
+- power-domains:	specifies a phandle to PM domain provider node
 - clocks:		A list of phandle + clock-specifier pairs for the
 				clocks listed in clock-names
-- clock-names:	Should contain the following:
+- clock-names:		Should contain the following:
   "core"		Master/Core clock, have to be >= 125 MHz for SS
 				operation and >= 60MHz for HS operation
+  "mock_utmi"		Mock utmi clock needed for ITP/SOF generation in
+				host mode. Its frequency should be 19.2MHz.
+  "sleep"		Sleep clock, used for wakeup when USB3 core goes
+				into low power mode (U3).
 
 Optional clocks:
-  "iface"		System bus AXI clock.  Not present on all platforms
-  "sleep"		Sleep clock, used when USB3 core goes into low
-				power mode (U3).
+  "iface"		System bus AXI clock.
+			Not present on "qcom,msm8996-dwc3" compatible.
+  "cfg_noc"		System Config NOC clock.
+			Not present on "qcom,msm8996-dwc3" compatible.
+- assigned-clocks:	Should be:
+				MOCK_UTMI_CLK
+				MASTER_CLK
+- assigned-clock-rates: Should be:
+                                19.2Mhz (192000000) for MOCK_UTMI_CLK
+                                >=125Mhz (125000000) for MASTER_CLK in SS mode
+                                >=60Mhz (60000000) for MASTER_CLK in HS mode
+
+Optional properties:
+- resets:		Phandle to reset control that resets core and wrapper.
+- interrupts:		specifies interrupts from controller wrapper used
+			to wakeup from low power/susepnd state.	Must contain
+			one or more entry for interrupt-names property
+- interrupt-names:	Must include the following entries:
+			- "hs_phy_irq": The interrupt that is asserted when a
+			   wakeup event is received on USB2 bus
+			- "ss_phy_irq": The interrupt that is asserted when a
+			   wakeup event is received on USB3 bus
+			- "dm_hs_phy_irq" and "dp_hs_phy_irq": Separate
+			   interrupts for any wakeup event on DM and DP lines
+- qcom,select-utmi-as-pipe-clk: if present, disable USB3 pipe_clk requirement.
+				Used when dwc3 operates without SSPHY and only
+				HS/FS/LS modes are supported.
 
 Required child node:
 A child node must exist to represent the core DWC3 IP block. The name of
 the node is not important. The content of the node is defined in dwc3.txt.
 
 Phy documentation is provided in the following places:
-Documentation/devicetree/bindings/phy/qcom-dwc3-usb-phy.txt
+Documentation/devicetree/bindings/phy/qcom-qmp-phy.txt   - USB3 QMP PHY
+Documentation/devicetree/bindings/phy/qcom-qusb2-phy.txt - USB2 QUSB2 PHY
 
 Example device nodes:
 
 		hs_phy: phy@100f8800 {
-			compatible = "qcom,dwc3-hs-usb-phy";
-			reg = <0x100f8800 0x30>;
-			clocks = <&gcc USB30_0_UTMI_CLK>;
-			clock-names = "ref";
-			#phy-cells = <0>;
-
+			compatible = "qcom,qusb2-v2-phy";
+			...
 		};
 
 		ss_phy: phy@100f8830 {
-			compatible = "qcom,dwc3-ss-usb-phy";
-			reg = <0x100f8830 0x30>;
-			clocks = <&gcc USB30_0_MASTER_CLK>;
-			clock-names = "ref";
-			#phy-cells = <0>;
-
+			compatible = "qcom,qmp-v3-usb3-phy";
+			...
 		};
 
-		usb3_0: usb30@0 {
+		usb3_0: usb30@a6f8800 {
 			compatible = "qcom,dwc3";
+			reg = <0xa6f8800 0x400>;
 			#address-cells = <1>;
 			#size-cells = <1>;
-			clocks = <&gcc USB30_0_MASTER_CLK>;
-			clock-names = "core";
-
 			ranges;
 
+			interrupts = <0 131 0>, <0 486 0>, <0 488 0>, <0 489 0>;
+			interrupt-names = "hs_phy_irq", "ss_phy_irq",
+				  "dm_hs_phy_irq", "dp_hs_phy_irq";
+
+			clocks = <&gcc GCC_USB30_PRIM_MASTER_CLK>,
+				<&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>,
+				<&gcc GCC_USB30_PRIM_SLEEP_CLK>;
+			clock-names = "core", "mock_utmi", "sleep";
+
+			assigned-clocks = <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>,
+					  <&gcc GCC_USB30_PRIM_MASTER_CLK>;
+			assigned-clock-rates = <19200000>, <133000000>;
+
+			resets = <&gcc GCC_USB30_PRIM_BCR>;
+			reset-names = "core_reset";
+			power-domains = <&gcc USB30_PRIM_GDSC>;
+			qcom,select-utmi-as-pipe-clk;
 
 			dwc3@10000000 {
 				compatible = "snps,dwc3";
diff --git a/Documentation/devicetree/bindings/usb/richtek,rt1711h.txt b/Documentation/devicetree/bindings/usb/richtek,rt1711h.txt
new file mode 100644
index 000000000000..09e847e92e5e
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/richtek,rt1711h.txt
@@ -0,0 +1,17 @@
+Richtek RT1711H TypeC PD Controller.
+
+Required properties:
+ - compatible : Must be "richtek,rt1711h".
+ - reg : Must be 0x4e, it's slave address of RT1711H.
+ - interrupt-parent : the phandle for the interrupt controller that
+   provides interrupts for this device.
+ - interrupts : <a b> where a is the interrupt number and b represents an
+   encoding of the sense and level information for the interrupt.
+
+Example :
+rt1711h@4e {
+	compatible = "richtek,rt1711h";
+	reg = <0x4e>;
+	interrupt-parent = <&gpio26>;
+	interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+};
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.txt b/Documentation/devicetree/bindings/vendor-prefixes.txt
index a38d8bfae19c..10828e094374 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.txt
+++ b/Documentation/devicetree/bindings/vendor-prefixes.txt
@@ -47,6 +47,7 @@ auvidea Auvidea GmbH
 avago	Avago Technologies
 avia	avia semiconductor
 avic	Shanghai AVIC Optoelectronics Co., Ltd.
+avnet	Avnet, Inc.
 axentia	Axentia Technologies AB
 axis	Axis Communications AB
 bananapi BIPAI KEJI LIMITED
@@ -56,6 +57,7 @@ bosch	Bosch Sensortec GmbH
 boundary	Boundary Devices Inc.
 brcm	Broadcom Corporation
 buffalo	Buffalo, Inc.
+bticino Bticino International
 calxeda	Calxeda
 capella	Capella Microsystems, Inc
 cascoda	Cascoda, Ltd.
@@ -75,6 +77,7 @@ cnxt	Conexant Systems, Inc.
 compulab	CompuLab Ltd.
 cortina	Cortina Systems, Inc.
 cosmic	Cosmic Circuits
+crane	Crane Connectivity Solutions
 creative	Creative Technology Ltd
 crystalfontz	Crystalfontz America, Inc.
 cubietech	Cubietech, Ltd.
@@ -185,6 +188,7 @@ khadas	Khadas
 kiebackpeter    Kieback & Peter GmbH
 kinetic Kinetic Technologies
 kingnovel	Kingnovel Technology Co., Ltd.
+koe	Kaohsiung Opto-Electronics Inc.
 kosagi	Sutajio Ko-Usagi PTE Ltd.
 kyo	Kyocera Corporation
 lacie	LaCie
@@ -199,11 +203,13 @@ linaro	Linaro Limited
 linksys	Belkin International, Inc. (Linksys)
 linux	Linux-specific binding
 lltc	Linear Technology Corporation
+logicpd	Logic PD, Inc.
 lsi	LSI Corp. (LSI Logic)
 lwn	Liebherr-Werk Nenzing GmbH
 macnica	Macnica Americas
 marvell	Marvell Technology Group Ltd.
 maxim	Maxim Integrated Products
+mbvl	Mobiveil Inc.
 mcube	mCube
 meas	Measurement Specialties
 mediatek	MediaTek Inc.
@@ -279,6 +285,7 @@ pine64	Pine64
 pixcir  PIXCIR MICROELECTRONICS Co., Ltd
 plathome	Plat'Home Co., Ltd.
 plda	PLDA
+portwell	Portwell Inc.
 poslab	Poslab Technology Co., Ltd.
 powervr	PowerVR (deprecated, use img)
 probox2	PROBOX2 (by W2COMP Co., Ltd.)
@@ -318,6 +325,7 @@ sgx	SGX Sensortech
 sharp	Sharp Corporation
 shimafuji	Shimafuji Electric, Inc.
 si-en	Si-En Technology Ltd.
+sifive	SiFive, Inc.
 sigma	Sigma Designs, Inc.
 sii	Seiko Instruments, Inc.
 sil	Silicon Image
@@ -393,6 +401,7 @@ vot	Vision Optical Technology Co., Ltd.
 wd	Western Digital Corp.
 wetek	WeTek Electronics, limited.
 wexler	Wexler
+wi2wi	Wi2Wi, Inc.
 winbond Winbond Electronics corp.
 winstar	Winstar Display Corp.
 wlf	Wolfson Microelectronics
diff --git a/Documentation/devicetree/bindings/watchdog/ingenic,jz4740-wdt.txt b/Documentation/devicetree/bindings/watchdog/ingenic,jz4740-wdt.txt
index cb44918f01a8..ce1cb72d5345 100644
--- a/Documentation/devicetree/bindings/watchdog/ingenic,jz4740-wdt.txt
+++ b/Documentation/devicetree/bindings/watchdog/ingenic,jz4740-wdt.txt
@@ -3,10 +3,15 @@ Ingenic Watchdog Timer (WDT) Controller for JZ4740 & JZ4780
 Required properties:
 compatible: "ingenic,jz4740-watchdog" or "ingenic,jz4780-watchdog"
 reg: Register address and length for watchdog registers
+clocks: phandle to the RTC clock
+clock-names: should be "rtc"
 
 Example:
 
 watchdog: jz4740-watchdog@10002000 {
 	compatible = "ingenic,jz4740-watchdog";
-	reg = <0x10002000 0x100>;
+	reg = <0x10002000 0x10>;
+
+	clocks = <&cgu JZ4740_CLK_RTC>;
+	clock-names = "rtc";
 };
diff --git a/Documentation/devicetree/bindings/watchdog/renesas-wdt.txt b/Documentation/devicetree/bindings/watchdog/renesas-wdt.txt
index 74b2f03c1515..f24d802b8056 100644
--- a/Documentation/devicetree/bindings/watchdog/renesas-wdt.txt
+++ b/Documentation/devicetree/bindings/watchdog/renesas-wdt.txt
@@ -1,18 +1,27 @@
 Renesas Watchdog Timer (WDT) Controller
 
 Required properties:
-- compatible : Should be "renesas,<soctype>-wdt", and
-	       "renesas,rcar-gen3-wdt" or "renesas,rza-wdt" as fallback.
+ - compatible : Must be "renesas,<soctype>-wdt", followed by a generic
+		fallback compatible string when compatible with the generic
+		version.
 	       Examples with soctypes are:
-	         - "renesas,r7s72100-wdt" (RZ/A1)
+		 - "renesas,r8a7743-wdt" (RZ/G1M)
+		 - "renesas,r8a7745-wdt" (RZ/G1E)
+	         - "renesas,r8a7790-wdt" (R-Car H2)
+	         - "renesas,r8a7791-wdt" (R-Car M2-W)
+	         - "renesas,r8a7792-wdt" (R-Car V2H)
+	         - "renesas,r8a7793-wdt" (R-Car M2-N)
+	         - "renesas,r8a7794-wdt" (R-Car E2)
 	         - "renesas,r8a7795-wdt" (R-Car H3)
 	         - "renesas,r8a7796-wdt" (R-Car M3-W)
+		 - "renesas,r8a77965-wdt" (R-Car M3-N)
 	         - "renesas,r8a77970-wdt" (R-Car V3M)
 	         - "renesas,r8a77995-wdt" (R-Car D3)
-
-  When compatible with the generic version, nodes must list the SoC-specific
-  version corresponding to the platform first, followed by the generic
-  version.
+	         - "renesas,r7s72100-wdt" (RZ/A1)
+		The generic compatible string must be:
+		 - "renesas,rza-wdt" for RZ/A
+		 - "renesas,rcar-gen2-wdt" for R-Car Gen2 and RZ/G
+		 - "renesas,rcar-gen3-wdt" for R-Car Gen3
 
 - reg : Should contain WDT registers location and length
 - clocks : the clock feeding the watchdog timer.
diff --git a/Documentation/driver-api/clk.rst b/Documentation/driver-api/clk.rst
new file mode 100644
index 000000000000..511628bb3d3a
--- /dev/null
+++ b/Documentation/driver-api/clk.rst
@@ -0,0 +1,307 @@
+========================
+The Common Clk Framework
+========================
+
+:Author: Mike Turquette <mturquette@ti.com>
+
+This document endeavours to explain the common clk framework details,
+and how to port a platform over to this framework.  It is not yet a
+detailed explanation of the clock api in include/linux/clk.h, but
+perhaps someday it will include that information.
+
+Introduction and interface split
+================================
+
+The common clk framework is an interface to control the clock nodes
+available on various devices today.  This may come in the form of clock
+gating, rate adjustment, muxing or other operations.  This framework is
+enabled with the CONFIG_COMMON_CLK option.
+
+The interface itself is divided into two halves, each shielded from the
+details of its counterpart.  First is the common definition of struct
+clk which unifies the framework-level accounting and infrastructure that
+has traditionally been duplicated across a variety of platforms.  Second
+is a common implementation of the clk.h api, defined in
+drivers/clk/clk.c.  Finally there is struct clk_ops, whose operations
+are invoked by the clk api implementation.
+
+The second half of the interface is comprised of the hardware-specific
+callbacks registered with struct clk_ops and the corresponding
+hardware-specific structures needed to model a particular clock.  For
+the remainder of this document any reference to a callback in struct
+clk_ops, such as .enable or .set_rate, implies the hardware-specific
+implementation of that code.  Likewise, references to struct clk_foo
+serve as a convenient shorthand for the implementation of the
+hardware-specific bits for the hypothetical "foo" hardware.
+
+Tying the two halves of this interface together is struct clk_hw, which
+is defined in struct clk_foo and pointed to within struct clk_core.  This
+allows for easy navigation between the two discrete halves of the common
+clock interface.
+
+Common data structures and api
+==============================
+
+Below is the common struct clk_core definition from
+drivers/clk/clk.c, modified for brevity::
+
+	struct clk_core {
+		const char		*name;
+		const struct clk_ops	*ops;
+		struct clk_hw		*hw;
+		struct module		*owner;
+		struct clk_core		*parent;
+		const char		**parent_names;
+		struct clk_core		**parents;
+		u8			num_parents;
+		u8			new_parent_index;
+		...
+	};
+
+The members above make up the core of the clk tree topology.  The clk
+api itself defines several driver-facing functions which operate on
+struct clk.  That api is documented in include/linux/clk.h.
+
+Platforms and devices utilizing the common struct clk_core use the struct
+clk_ops pointer in struct clk_core to perform the hardware-specific parts of
+the operations defined in clk-provider.h::
+
+	struct clk_ops {
+		int		(*prepare)(struct clk_hw *hw);
+		void		(*unprepare)(struct clk_hw *hw);
+		int		(*is_prepared)(struct clk_hw *hw);
+		void		(*unprepare_unused)(struct clk_hw *hw);
+		int		(*enable)(struct clk_hw *hw);
+		void		(*disable)(struct clk_hw *hw);
+		int		(*is_enabled)(struct clk_hw *hw);
+		void		(*disable_unused)(struct clk_hw *hw);
+		unsigned long	(*recalc_rate)(struct clk_hw *hw,
+						unsigned long parent_rate);
+		long		(*round_rate)(struct clk_hw *hw,
+						unsigned long rate,
+						unsigned long *parent_rate);
+		int		(*determine_rate)(struct clk_hw *hw,
+						  struct clk_rate_request *req);
+		int		(*set_parent)(struct clk_hw *hw, u8 index);
+		u8		(*get_parent)(struct clk_hw *hw);
+		int		(*set_rate)(struct clk_hw *hw,
+					    unsigned long rate,
+					    unsigned long parent_rate);
+		int		(*set_rate_and_parent)(struct clk_hw *hw,
+					    unsigned long rate,
+					    unsigned long parent_rate,
+					    u8 index);
+		unsigned long	(*recalc_accuracy)(struct clk_hw *hw,
+						unsigned long parent_accuracy);
+		int		(*get_phase)(struct clk_hw *hw);
+		int		(*set_phase)(struct clk_hw *hw, int degrees);
+		void		(*init)(struct clk_hw *hw);
+		int		(*debug_init)(struct clk_hw *hw,
+					      struct dentry *dentry);
+	};
+
+Hardware clk implementations
+============================
+
+The strength of the common struct clk_core comes from its .ops and .hw pointers
+which abstract the details of struct clk from the hardware-specific bits, and
+vice versa.  To illustrate consider the simple gateable clk implementation in
+drivers/clk/clk-gate.c::
+
+	struct clk_gate {
+		struct clk_hw	hw;
+		void __iomem    *reg;
+		u8              bit_idx;
+		...
+	};
+
+struct clk_gate contains struct clk_hw hw as well as hardware-specific
+knowledge about which register and bit controls this clk's gating.
+Nothing about clock topology or accounting, such as enable_count or
+notifier_count, is needed here.  That is all handled by the common
+framework code and struct clk_core.
+
+Let's walk through enabling this clk from driver code::
+
+	struct clk *clk;
+	clk = clk_get(NULL, "my_gateable_clk");
+
+	clk_prepare(clk);
+	clk_enable(clk);
+
+The call graph for clk_enable is very simple::
+
+	clk_enable(clk);
+		clk->ops->enable(clk->hw);
+		[resolves to...]
+			clk_gate_enable(hw);
+			[resolves struct clk gate with to_clk_gate(hw)]
+				clk_gate_set_bit(gate);
+
+And the definition of clk_gate_set_bit::
+
+	static void clk_gate_set_bit(struct clk_gate *gate)
+	{
+		u32 reg;
+
+		reg = __raw_readl(gate->reg);
+		reg |= BIT(gate->bit_idx);
+		writel(reg, gate->reg);
+	}
+
+Note that to_clk_gate is defined as::
+
+	#define to_clk_gate(_hw) container_of(_hw, struct clk_gate, hw)
+
+This pattern of abstraction is used for every clock hardware
+representation.
+
+Supporting your own clk hardware
+================================
+
+When implementing support for a new type of clock it is only necessary to
+include the following header::
+
+	#include <linux/clk-provider.h>
+
+To construct a clk hardware structure for your platform you must define
+the following::
+
+	struct clk_foo {
+		struct clk_hw hw;
+		... hardware specific data goes here ...
+	};
+
+To take advantage of your data you'll need to support valid operations
+for your clk::
+
+	struct clk_ops clk_foo_ops {
+		.enable		= &clk_foo_enable;
+		.disable	= &clk_foo_disable;
+	};
+
+Implement the above functions using container_of::
+
+	#define to_clk_foo(_hw) container_of(_hw, struct clk_foo, hw)
+
+	int clk_foo_enable(struct clk_hw *hw)
+	{
+		struct clk_foo *foo;
+
+		foo = to_clk_foo(hw);
+
+		... perform magic on foo ...
+
+		return 0;
+	};
+
+Below is a matrix detailing which clk_ops are mandatory based upon the
+hardware capabilities of that clock.  A cell marked as "y" means
+mandatory, a cell marked as "n" implies that either including that
+callback is invalid or otherwise unnecessary.  Empty cells are either
+optional or must be evaluated on a case-by-case basis.
+
+.. table:: clock hardware characteristics
+
+   +----------------+------+-------------+---------------+-------------+------+
+   |                | gate | change rate | single parent | multiplexer | root |
+   +================+======+=============+===============+=============+======+
+   |.prepare        |      |             |               |             |      |
+   +----------------+------+-------------+---------------+-------------+------+
+   |.unprepare      |      |             |               |             |      |
+   +----------------+------+-------------+---------------+-------------+------+
+   +----------------+------+-------------+---------------+-------------+------+
+   |.enable         | y    |             |               |             |      |
+   +----------------+------+-------------+---------------+-------------+------+
+   |.disable        | y    |             |               |             |      |
+   +----------------+------+-------------+---------------+-------------+------+
+   |.is_enabled     | y    |             |               |             |      |
+   +----------------+------+-------------+---------------+-------------+------+
+   +----------------+------+-------------+---------------+-------------+------+
+   |.recalc_rate    |      | y           |               |             |      |
+   +----------------+------+-------------+---------------+-------------+------+
+   |.round_rate     |      | y [1]_      |               |             |      |
+   +----------------+------+-------------+---------------+-------------+------+
+   |.determine_rate |      | y [1]_      |               |             |      |
+   +----------------+------+-------------+---------------+-------------+------+
+   |.set_rate       |      | y           |               |             |      |
+   +----------------+------+-------------+---------------+-------------+------+
+   +----------------+------+-------------+---------------+-------------+------+
+   |.set_parent     |      |             | n             | y           | n    |
+   +----------------+------+-------------+---------------+-------------+------+
+   |.get_parent     |      |             | n             | y           | n    |
+   +----------------+------+-------------+---------------+-------------+------+
+   +----------------+------+-------------+---------------+-------------+------+
+   |.recalc_accuracy|      |             |               |             |      |
+   +----------------+------+-------------+---------------+-------------+------+
+   +----------------+------+-------------+---------------+-------------+------+
+   |.init           |      |             |               |             |      |
+   +----------------+------+-------------+---------------+-------------+------+
+
+.. [1] either one of round_rate or determine_rate is required.
+
+Finally, register your clock at run-time with a hardware-specific
+registration function.  This function simply populates struct clk_foo's
+data and then passes the common struct clk parameters to the framework
+with a call to::
+
+	clk_register(...)
+
+See the basic clock types in ``drivers/clk/clk-*.c`` for examples.
+
+Disabling clock gating of unused clocks
+=======================================
+
+Sometimes during development it can be useful to be able to bypass the
+default disabling of unused clocks. For example, if drivers aren't enabling
+clocks properly but rely on them being on from the bootloader, bypassing
+the disabling means that the driver will remain functional while the issues
+are sorted out.
+
+To bypass this disabling, include "clk_ignore_unused" in the bootargs to the
+kernel.
+
+Locking
+=======
+
+The common clock framework uses two global locks, the prepare lock and the
+enable lock.
+
+The enable lock is a spinlock and is held across calls to the .enable,
+.disable operations. Those operations are thus not allowed to sleep,
+and calls to the clk_enable(), clk_disable() API functions are allowed in
+atomic context.
+
+For clk_is_enabled() API, it is also designed to be allowed to be used in
+atomic context. However, it doesn't really make any sense to hold the enable
+lock in core, unless you want to do something else with the information of
+the enable state with that lock held. Otherwise, seeing if a clk is enabled is
+a one-shot read of the enabled state, which could just as easily change after
+the function returns because the lock is released. Thus the user of this API
+needs to handle synchronizing the read of the state with whatever they're
+using it for to make sure that the enable state doesn't change during that
+time.
+
+The prepare lock is a mutex and is held across calls to all other operations.
+All those operations are allowed to sleep, and calls to the corresponding API
+functions are not allowed in atomic context.
+
+This effectively divides operations in two groups from a locking perspective.
+
+Drivers don't need to manually protect resources shared between the operations
+of one group, regardless of whether those resources are shared by multiple
+clocks or not. However, access to resources that are shared between operations
+of the two groups needs to be protected by the drivers. An example of such a
+resource would be a register that controls both the clock rate and the clock
+enable/disable state.
+
+The clock framework is reentrant, in that a driver is allowed to call clock
+framework functions from within its implementation of clock operations. This
+can for instance cause a .set_rate operation of one clock being called from
+within the .set_rate operation of another clock. This case must be considered
+in the driver implementations, but the code flow is usually controlled by the
+driver in that case.
+
+Note that locking must also be considered when code outside of the common
+clock framework needs to access resources used by the clock operations. This
+is considered out of scope of this document.
diff --git a/Documentation/driver-api/device_connection.rst b/Documentation/driver-api/device_connection.rst
index affbc5566ab0..ba364224c349 100644
--- a/Documentation/driver-api/device_connection.rst
+++ b/Documentation/driver-api/device_connection.rst
@@ -40,4 +40,4 @@ API
 ---
 
 .. kernel-doc:: drivers/base/devcon.c
-   : functions: device_connection_find_match device_connection_find device_connection_add device_connection_remove
+   :functions: device_connection_find_match device_connection_find device_connection_add device_connection_remove
diff --git a/Documentation/driver-api/firmware/fallback-mechanisms.rst b/Documentation/driver-api/firmware/fallback-mechanisms.rst
index f353783ae0be..d35fed65eae9 100644
--- a/Documentation/driver-api/firmware/fallback-mechanisms.rst
+++ b/Documentation/driver-api/firmware/fallback-mechanisms.rst
@@ -72,9 +72,12 @@ the firmware requested, and establishes it in the device hierarchy by
 associating the device used to make the request as the device's parent.
 The sysfs directory's file attributes are defined and controlled through
 the new device's class (firmware_class) and group (fw_dev_attr_groups).
-This is actually where the original firmware_class.c file name comes from,
-as originally the only firmware loading mechanism available was the
-mechanism we now use as a fallback mechanism.
+This is actually where the original firmware_class module name came from,
+given that originally the only firmware loading mechanism available was the
+mechanism we now use as a fallback mechanism, which registers a struct class
+firmware_class. Because the attributes exposed are part of the module name, the
+module name firmware_class cannot be renamed in the future, to ensure backward
+compatibility with old userspace.
 
 To load firmware using the sysfs interface we expose a loading indicator,
 and a file upload firmware into:
@@ -83,7 +86,7 @@ and a file upload firmware into:
   * /sys/$DEVPATH/data
 
 To upload firmware you will echo 1 onto the loading file to indicate
-you are loading firmware. You then cat the firmware into the data file,
+you are loading firmware. You then write the firmware into the data file,
 and you notify the kernel the firmware is ready by echo'ing 0 onto
 the loading file.
 
@@ -136,7 +139,8 @@ by kobject uevents. This is specially exacerbated due to the fact that most
 distributions today disable CONFIG_FW_LOADER_USER_HELPER_FALLBACK.
 
 Refer to do_firmware_uevent() for details of the kobject event variables
-setup. Variables passwdd with a kobject add event:
+setup. The variables currently passed to userspace with a "kobject add"
+event are:
 
 * FIRMWARE=firmware name
 * TIMEOUT=timeout value
diff --git a/Documentation/driver-api/firmware/firmware_cache.rst b/Documentation/driver-api/firmware/firmware_cache.rst
index 2210e5bfb332..c2e69d9c6bf1 100644
--- a/Documentation/driver-api/firmware/firmware_cache.rst
+++ b/Documentation/driver-api/firmware/firmware_cache.rst
@@ -29,8 +29,8 @@ Some implementation details about the firmware cache setup:
 * If an asynchronous call is used the firmware cache is only set up for a
   device if if the second argument (uevent) to request_firmware_nowait() is
   true. When uevent is true it requests that a kobject uevent be sent to
-  userspace for the firmware request. For details refer to the Fackback
-  mechanism documented below.
+  userspace for the firmware request through the sysfs fallback mechanism
+  if the firmware file is not found.
 
 * If the firmware cache is determined to be needed as per the above two
   criteria the firmware cache is setup by adding a devres entry for the
diff --git a/Documentation/driver-api/firmware/request_firmware.rst b/Documentation/driver-api/firmware/request_firmware.rst
index d5ec95a7195b..f62bdcbfed5b 100644
--- a/Documentation/driver-api/firmware/request_firmware.rst
+++ b/Documentation/driver-api/firmware/request_firmware.rst
@@ -20,6 +20,11 @@ request_firmware
 .. kernel-doc:: drivers/base/firmware_loader/main.c
    :functions: request_firmware
 
+firmware_request_nowarn
+-----------------------
+.. kernel-doc:: drivers/base/firmware_loader/main.c
+   :functions: firmware_request_nowarn
+
 request_firmware_direct
 -----------------------
 .. kernel-doc:: drivers/base/firmware_loader/main.c
diff --git a/Documentation/driver-api/fpga/fpga-bridge.rst b/Documentation/driver-api/fpga/fpga-bridge.rst
new file mode 100644
index 000000000000..2c2aaca894bf
--- /dev/null
+++ b/Documentation/driver-api/fpga/fpga-bridge.rst
@@ -0,0 +1,49 @@
+FPGA Bridge
+===========
+
+API to implement a new FPGA bridge
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. kernel-doc:: include/linux/fpga/fpga-bridge.h
+   :functions: fpga_bridge
+
+.. kernel-doc:: include/linux/fpga/fpga-bridge.h
+   :functions: fpga_bridge_ops
+
+.. kernel-doc:: drivers/fpga/fpga-bridge.c
+   :functions: fpga_bridge_create
+
+.. kernel-doc:: drivers/fpga/fpga-bridge.c
+   :functions: fpga_bridge_free
+
+.. kernel-doc:: drivers/fpga/fpga-bridge.c
+   :functions: fpga_bridge_register
+
+.. kernel-doc:: drivers/fpga/fpga-bridge.c
+   :functions: fpga_bridge_unregister
+
+API to control an FPGA bridge
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You probably won't need these directly.  FPGA regions should handle this.
+
+.. kernel-doc:: drivers/fpga/fpga-bridge.c
+   :functions: of_fpga_bridge_get
+
+.. kernel-doc:: drivers/fpga/fpga-bridge.c
+   :functions: fpga_bridge_get
+
+.. kernel-doc:: drivers/fpga/fpga-bridge.c
+   :functions: fpga_bridge_put
+
+.. kernel-doc:: drivers/fpga/fpga-bridge.c
+   :functions: fpga_bridge_get_to_list
+
+.. kernel-doc:: drivers/fpga/fpga-bridge.c
+   :functions: of_fpga_bridge_get_to_list
+
+.. kernel-doc:: drivers/fpga/fpga-bridge.c
+   :functions: fpga_bridge_enable
+
+.. kernel-doc:: drivers/fpga/fpga-bridge.c
+   :functions: fpga_bridge_disable
diff --git a/Documentation/driver-api/fpga/fpga-mgr.rst b/Documentation/driver-api/fpga/fpga-mgr.rst
new file mode 100644
index 000000000000..bcf2dd24e179
--- /dev/null
+++ b/Documentation/driver-api/fpga/fpga-mgr.rst
@@ -0,0 +1,220 @@
+FPGA Manager
+============
+
+Overview
+--------
+
+The FPGA manager core exports a set of functions for programming an FPGA with
+an image.  The API is manufacturer agnostic.  All manufacturer specifics are
+hidden away in a low level driver which registers a set of ops with the core.
+The FPGA image data itself is very manufacturer specific, but for our purposes
+it's just binary data.  The FPGA manager core won't parse it.
+
+The FPGA image to be programmed can be in a scatter gather list, a single
+contiguous buffer, or a firmware file.  Because allocating contiguous kernel
+memory for the buffer should be avoided, users are encouraged to use a scatter
+gather list instead if possible.
+
+The particulars for programming the image are presented in a structure (struct
+fpga_image_info).  This struct contains parameters such as pointers to the
+FPGA image as well as image-specific particulars such as whether the image was
+built for full or partial reconfiguration.
+
+How to support a new FPGA device
+--------------------------------
+
+To add another FPGA manager, write a driver that implements a set of ops.  The
+probe function calls fpga_mgr_register(), such as::
+
+	static const struct fpga_manager_ops socfpga_fpga_ops = {
+		.write_init = socfpga_fpga_ops_configure_init,
+		.write = socfpga_fpga_ops_configure_write,
+		.write_complete = socfpga_fpga_ops_configure_complete,
+		.state = socfpga_fpga_ops_state,
+	};
+
+	static int socfpga_fpga_probe(struct platform_device *pdev)
+	{
+		struct device *dev = &pdev->dev;
+		struct socfpga_fpga_priv *priv;
+		struct fpga_manager *mgr;
+		int ret;
+
+		priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+		if (!priv)
+			return -ENOMEM;
+
+		/*
+		 * do ioremaps, get interrupts, etc. and save
+		 * them in priv
+		 */
+
+		mgr = fpga_mgr_create(dev, "Altera SOCFPGA FPGA Manager",
+				      &socfpga_fpga_ops, priv);
+		if (!mgr)
+			return -ENOMEM;
+
+		platform_set_drvdata(pdev, mgr);
+
+		ret = fpga_mgr_register(mgr);
+		if (ret)
+			fpga_mgr_free(mgr);
+
+		return ret;
+	}
+
+	static int socfpga_fpga_remove(struct platform_device *pdev)
+	{
+		struct fpga_manager *mgr = platform_get_drvdata(pdev);
+
+		fpga_mgr_unregister(mgr);
+
+		return 0;
+	}
+
+
+The ops will implement whatever device specific register writes are needed to
+do the programming sequence for this particular FPGA.  These ops return 0 for
+success or negative error codes otherwise.
+
+The programming sequence is::
+ 1. .write_init
+ 2. .write or .write_sg (may be called once or multiple times)
+ 3. .write_complete
+
+The .write_init function will prepare the FPGA to receive the image data.  The
+buffer passed into .write_init will be atmost .initial_header_size bytes long,
+if the whole bitstream is not immediately available then the core code will
+buffer up at least this much before starting.
+
+The .write function writes a buffer to the FPGA. The buffer may be contain the
+whole FPGA image or may be a smaller chunk of an FPGA image.  In the latter
+case, this function is called multiple times for successive chunks. This interface
+is suitable for drivers which use PIO.
+
+The .write_sg version behaves the same as .write except the input is a sg_table
+scatter list. This interface is suitable for drivers which use DMA.
+
+The .write_complete function is called after all the image has been written
+to put the FPGA into operating mode.
+
+The ops include a .state function which will read the hardware FPGA manager and
+return a code of type enum fpga_mgr_states.  It doesn't result in a change in
+hardware state.
+
+How to write an image buffer to a supported FPGA
+------------------------------------------------
+
+Some sample code::
+
+	#include <linux/fpga/fpga-mgr.h>
+
+	struct fpga_manager *mgr;
+	struct fpga_image_info *info;
+	int ret;
+
+	/*
+	 * Get a reference to FPGA manager.  The manager is not locked, so you can
+	 * hold onto this reference without it preventing programming.
+	 *
+	 * This example uses the device node of the manager.  Alternatively, use
+	 * fpga_mgr_get(dev) instead if you have the device.
+	 */
+	mgr = of_fpga_mgr_get(mgr_node);
+
+	/* struct with information about the FPGA image to program. */
+	info = fpga_image_info_alloc(dev);
+
+	/* flags indicates whether to do full or partial reconfiguration */
+	info->flags = FPGA_MGR_PARTIAL_RECONFIG;
+
+	/*
+	 * At this point, indicate where the image is. This is pseudo-code; you're
+	 * going to use one of these three.
+	 */
+	if (image is in a scatter gather table) {
+
+		info->sgt = [your scatter gather table]
+
+	} else if (image is in a buffer) {
+
+		info->buf = [your image buffer]
+		info->count = [image buffer size]
+
+	} else if (image is in a firmware file) {
+
+		info->firmware_name = devm_kstrdup(dev, firmware_name, GFP_KERNEL);
+
+	}
+
+	/* Get exclusive control of FPGA manager */
+	ret = fpga_mgr_lock(mgr);
+
+	/* Load the buffer to the FPGA */
+	ret = fpga_mgr_buf_load(mgr, &info, buf, count);
+
+	/* Release the FPGA manager */
+	fpga_mgr_unlock(mgr);
+	fpga_mgr_put(mgr);
+
+	/* Deallocate the image info if you're done with it */
+	fpga_image_info_free(info);
+
+API for implementing a new FPGA Manager driver
+----------------------------------------------
+
+.. kernel-doc:: include/linux/fpga/fpga-mgr.h
+   :functions: fpga_manager
+
+.. kernel-doc:: include/linux/fpga/fpga-mgr.h
+   :functions: fpga_manager_ops
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: fpga_mgr_create
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: fpga_mgr_free
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: fpga_mgr_register
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: fpga_mgr_unregister
+
+API for programming a FPGA
+--------------------------
+
+.. kernel-doc:: include/linux/fpga/fpga-mgr.h
+   :functions: fpga_image_info
+
+.. kernel-doc:: include/linux/fpga/fpga-mgr.h
+   :functions: fpga_mgr_states
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: fpga_image_info_alloc
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: fpga_image_info_free
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: of_fpga_mgr_get
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: fpga_mgr_get
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: fpga_mgr_put
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: fpga_mgr_lock
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: fpga_mgr_unlock
+
+.. kernel-doc:: include/linux/fpga/fpga-mgr.h
+   :functions: fpga_mgr_states
+
+Note - use :c:func:`fpga_region_program_fpga()` instead of :c:func:`fpga_mgr_load()`
+
+.. kernel-doc:: drivers/fpga/fpga-mgr.c
+   :functions: fpga_mgr_load
diff --git a/Documentation/driver-api/fpga/fpga-region.rst b/Documentation/driver-api/fpga/fpga-region.rst
new file mode 100644
index 000000000000..f89e4a311722
--- /dev/null
+++ b/Documentation/driver-api/fpga/fpga-region.rst
@@ -0,0 +1,102 @@
+FPGA Region
+===========
+
+Overview
+--------
+
+This document is meant to be an brief overview of the FPGA region API usage.  A
+more conceptual look at regions can be found in the Device Tree binding
+document [#f1]_.
+
+For the purposes of this API document, let's just say that a region associates
+an FPGA Manager and a bridge (or bridges) with a reprogrammable region of an
+FPGA or the whole FPGA.  The API provides a way to register a region and to
+program a region.
+
+Currently the only layer above fpga-region.c in the kernel is the Device Tree
+support (of-fpga-region.c) described in [#f1]_.  The DT support layer uses regions
+to program the FPGA and then DT to handle enumeration.  The common region code
+is intended to be used by other schemes that have other ways of accomplishing
+enumeration after programming.
+
+An fpga-region can be set up to know the following things:
+
+ * which FPGA manager to use to do the programming
+
+ * which bridges to disable before programming and enable afterwards.
+
+Additional info needed to program the FPGA image is passed in the struct
+fpga_image_info including:
+
+ * pointers to the image as either a scatter-gather buffer, a contiguous
+   buffer, or the name of firmware file
+
+ * flags indicating specifics such as whether the image if for partial
+   reconfiguration.
+
+How to program a FPGA using a region
+------------------------------------
+
+First, allocate the info struct::
+
+	info = fpga_image_info_alloc(dev);
+	if (!info)
+		return -ENOMEM;
+
+Set flags as needed, i.e.::
+
+	info->flags |= FPGA_MGR_PARTIAL_RECONFIG;
+
+Point to your FPGA image, such as::
+
+	info->sgt = &sgt;
+
+Add info to region and do the programming::
+
+	region->info = info;
+	ret = fpga_region_program_fpga(region);
+
+:c:func:`fpga_region_program_fpga()` operates on info passed in the
+fpga_image_info (region->info).  This function will attempt to:
+
+ * lock the region's mutex
+ * lock the region's FPGA manager
+ * build a list of FPGA bridges if a method has been specified to do so
+ * disable the bridges
+ * program the FPGA
+ * re-enable the bridges
+ * release the locks
+
+Then you will want to enumerate whatever hardware has appeared in the FPGA.
+
+How to add a new FPGA region
+----------------------------
+
+An example of usage can be seen in the probe function of [#f2]_.
+
+.. [#f1] ../devicetree/bindings/fpga/fpga-region.txt
+.. [#f2] ../../drivers/fpga/of-fpga-region.c
+
+API to program a FGPA
+---------------------
+
+.. kernel-doc:: drivers/fpga/fpga-region.c
+   :functions: fpga_region_program_fpga
+
+API to add a new FPGA region
+----------------------------
+
+.. kernel-doc:: include/linux/fpga/fpga-region.h
+   :functions: fpga_region
+
+.. kernel-doc:: drivers/fpga/fpga-region.c
+   :functions: fpga_region_create
+
+.. kernel-doc:: drivers/fpga/fpga-region.c
+   :functions: fpga_region_free
+
+.. kernel-doc:: drivers/fpga/fpga-region.c
+   :functions: fpga_region_register
+
+.. kernel-doc:: drivers/fpga/fpga-region.c
+   :functions: fpga_region_unregister
diff --git a/Documentation/driver-api/fpga/index.rst b/Documentation/driver-api/fpga/index.rst
new file mode 100644
index 000000000000..c51e5ebd544a
--- /dev/null
+++ b/Documentation/driver-api/fpga/index.rst
@@ -0,0 +1,13 @@
+==============
+FPGA Subsystem
+==============
+
+:Author: Alan Tull
+
+.. toctree::
+   :maxdepth: 2
+
+   intro
+   fpga-mgr
+   fpga-bridge
+   fpga-region
diff --git a/Documentation/driver-api/fpga/intro.rst b/Documentation/driver-api/fpga/intro.rst
new file mode 100644
index 000000000000..51cd81dbb4dc
--- /dev/null
+++ b/Documentation/driver-api/fpga/intro.rst
@@ -0,0 +1,54 @@
+Introduction
+============
+
+The FPGA subsystem supports reprogramming FPGAs dynamically under
+Linux.  Some of the core intentions of the FPGA subsystems are:
+
+* The FPGA subsystem is vendor agnostic.
+
+* The FPGA subsystem separates upper layers (userspace interfaces and
+  enumeration) from lower layers that know how to program a specific
+  FPGA.
+
+* Code should not be shared between upper and lower layers.  This
+  should go without saying.  If that seems necessary, there's probably
+  framework functionality that that can be added that will benefit
+  other users.  Write the linux-fpga mailing list and maintainers and
+  seek out a solution that expands the framework for broad reuse.
+
+* Generally, when adding code, think of the future.  Plan for re-use.
+
+The framework in the kernel is divided into:
+
+FPGA Manager
+------------
+
+If you are adding a new FPGA or a new method of programming a FPGA,
+this is the subsystem for you.  Low level FPGA manager drivers contain
+the knowledge of how to program a specific device.  This subsystem
+includes the framework in fpga-mgr.c and the low level drivers that
+are registered with it.
+
+FPGA Bridge
+-----------
+
+FPGA Bridges prevent spurious signals from going out of a FPGA or a
+region of a FPGA during programming.  They are disabled before
+programming begins and re-enabled afterwards.  An FPGA bridge may be
+actual hard hardware that gates a bus to a cpu or a soft ("freeze")
+bridge in FPGA fabric that surrounds a partial reconfiguration region
+of an FPGA.  This subsystem includes fpga-bridge.c and the low level
+drivers that are registered with it.
+
+FPGA Region
+-----------
+
+If you are adding a new interface to the FPGA framework, add it on top
+of a FPGA region to allow the most reuse of your interface.
+
+The FPGA Region framework (fpga-region.c) associates managers and
+bridges as reconfigurable regions.  A region may refer to the whole
+FPGA in full reconfiguration or to a partial reconfiguration region.
+
+The Device Tree FPGA Region support (of-fpga-region.c) handles
+reprogramming FPGAs when device tree overlays are applied.
diff --git a/Documentation/driver-api/gpio/board.rst b/Documentation/driver-api/gpio/board.rst
index 25d62b2e9fd0..2c112553df84 100644
--- a/Documentation/driver-api/gpio/board.rst
+++ b/Documentation/driver-api/gpio/board.rst
@@ -177,3 +177,19 @@ mapping and is thus transparent to GPIO consumers.
 
 A set of functions such as gpiod_set_value() is available to work with
 the new descriptor-oriented interface.
+
+Boards using platform data can also hog GPIO lines by defining GPIO hog tables.
+
+.. code-block:: c
+
+        struct gpiod_hog gpio_hog_table[] = {
+                GPIO_HOG("gpio.0", 10, "foo", GPIO_ACTIVE_LOW, GPIOD_OUT_HIGH),
+                { }
+        };
+
+And the table can be added to the board code as follows::
+
+        gpiod_add_hogs(gpio_hog_table);
+
+The line will be hogged as soon as the gpiochip is created or - in case the
+chip was created earlier - when the hog table is registered.
diff --git a/Documentation/driver-api/gpio/driver.rst b/Documentation/driver-api/gpio/driver.rst
index 505ee906d7d9..cbe0242842d1 100644
--- a/Documentation/driver-api/gpio/driver.rst
+++ b/Documentation/driver-api/gpio/driver.rst
@@ -44,7 +44,7 @@ common to each controller of that type:
 
  - methods to establish GPIO line direction
  - methods used to access GPIO line values
- - method to set electrical configuration to a a given GPIO line
+ - method to set electrical configuration for a given GPIO line
  - method to return the IRQ number associated to a given GPIO line
  - flag saying whether calls to its methods may sleep
  - optional line names array to identify lines
@@ -143,7 +143,7 @@ resistor will make the line tend to high level unless one of the transistors on
 the rail actively pulls it down.
 
 The level on the line will go as high as the VDD on the pull-up resistor, which
-may be higher than the level supported by the transistor, achieveing a
+may be higher than the level supported by the transistor, achieving a
 level-shift to the higher VDD.
 
 Integrated electronics often have an output driver stage in the form of a CMOS
@@ -382,7 +382,7 @@ Real-Time compliance for GPIO IRQ chips
 
 Any provider of irqchips needs to be carefully tailored to support Real Time
 preemption. It is desirable that all irqchips in the GPIO subsystem keep this
-in mind and does the proper testing to assure they are real time-enabled.
+in mind and do the proper testing to assure they are real time-enabled.
 So, pay attention on above " RT_FULL:" notes, please.
 The following is a checklist to follow when preparing a driver for real
 time-compliance:
diff --git a/Documentation/driver-api/gpio/drivers-on-gpio.rst b/Documentation/driver-api/gpio/drivers-on-gpio.rst
index 7da0c1dd1f7a..f3a189320e11 100644
--- a/Documentation/driver-api/gpio/drivers-on-gpio.rst
+++ b/Documentation/driver-api/gpio/drivers-on-gpio.rst
@@ -85,6 +85,10 @@ hardware descriptions such as device tree or ACPI:
   any other serio bus to the system and makes it possible to connect drivers
   for e.g. keyboards and other PS/2 protocol based devices.
 
+- cec-gpio: drivers/media/platform/cec-gpio/ is used to interact with a CEC
+  Consumer Electronics Control bus using only GPIO. It is used to communicate
+  with devices on the HDMI bus.
+
 Apart from this there are special GPIO drivers in subsystems like MMC/SD to
 read card detect and write protect GPIO lines, and in the TTY serial subsystem
 to emulate MCTRL (modem control) signals CTS/RTS by using two GPIO lines. The
diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst
index 6d8352c0f354..6d9f2f9fe20e 100644
--- a/Documentation/driver-api/index.rst
+++ b/Documentation/driver-api/index.rst
@@ -17,7 +17,9 @@ available subsections can be seen below.
    basics
    infrastructure
    pm/index
+   clk
    device-io
+   device_connection
    dma-buf
    device_link
    message-based
@@ -34,6 +36,7 @@ available subsections can be seen below.
    edac
    scsi
    libata
+   target
    mtdnand
    miscellaneous
    w1
@@ -49,6 +52,7 @@ available subsections can be seen below.
    dmaengine/index
    slimbus
    soundwire/index
+   fpga/index
 
 .. only::  subproject and html
 
diff --git a/Documentation/driver-api/scsi.rst b/Documentation/driver-api/scsi.rst
index 31ad0fed6763..64b231d125e0 100644
--- a/Documentation/driver-api/scsi.rst
+++ b/Documentation/driver-api/scsi.rst
@@ -334,5 +334,5 @@ todo
 ~~~~
 
 Parallel (fast/wide/ultra) SCSI, USB, SATA, SAS, Fibre Channel,
-FireWire, ATAPI devices, Infiniband, I2O, iSCSI, Parallel ports,
+FireWire, ATAPI devices, Infiniband, I2O, Parallel ports,
 netlink...
diff --git a/Documentation/driver-api/soundwire/error_handling.rst b/Documentation/driver-api/soundwire/error_handling.rst
new file mode 100644
index 000000000000..aa3a0a23a066
--- /dev/null
+++ b/Documentation/driver-api/soundwire/error_handling.rst
@@ -0,0 +1,65 @@
+========================
+SoundWire Error Handling
+========================
+
+The SoundWire PHY was designed with care and errors on the bus are going to
+be very unlikely, and if they happen it should be limited to single bit
+errors. Examples of this design can be found in the synchronization
+mechanism (sync loss after two errors) and short CRCs used for the Bulk
+Register Access.
+
+The errors can be detected with multiple mechanisms:
+
+1. Bus clash or parity errors: This mechanism relies on low-level detectors
+   that are independent of the payload and usages, and they cover both control
+   and audio data. The current implementation only logs such errors.
+   Improvements could be invalidating an entire programming sequence and
+   restarting from a known position. In the case of such errors outside of a
+   control/command sequence, there is no concealment or recovery for audio
+   data enabled by the SoundWire protocol, the location of the error will also
+   impact its audibility (most-significant bits will be more impacted in PCM),
+   and after a number of such errors are detected the bus might be reset. Note
+   that bus clashes due to programming errors (two streams using the same bit
+   slots) or electrical issues during the transmit/receive transition cannot
+   be distinguished, although a recurring bus clash when audio is enabled is a
+   indication of a bus allocation issue. The interrupt mechanism can also help
+   identify Slaves which detected a Bus Clash or a Parity Error, but they may
+   not be responsible for the errors so resetting them individually is not a
+   viable recovery strategy.
+
+2. Command status: Each command is associated with a status, which only
+   covers transmission of the data between devices. The ACK status indicates
+   that the command was received and will be executed by the end of the
+   current frame. A NAK indicates that the command was in error and will not
+   be applied. In case of a bad programming (command sent to non-existent
+   Slave or to a non-implemented register) or electrical issue, no response
+   signals the command was ignored. Some Master implementations allow for a
+   command to be retransmitted several times.  If the retransmission fails,
+   backtracking and restarting the entire programming sequence might be a
+   solution. Alternatively some implementations might directly issue a bus
+   reset and re-enumerate all devices.
+
+3. Timeouts: In a number of cases such as ChannelPrepare or
+   ClockStopPrepare, the bus driver is supposed to poll a register field until
+   it transitions to a NotFinished value of zero. The MIPI SoundWire spec 1.1
+   does not define timeouts but the MIPI SoundWire DisCo document adds
+   recommendation on timeouts. If such configurations do not complete, the
+   driver will return a -ETIMEOUT. Such timeouts are symptoms of a faulty
+   Slave device and are likely impossible to recover from.
+
+Errors during global reconfiguration sequences are extremely difficult to
+handle:
+
+1. BankSwitch: An error during the last command issuing a BankSwitch is
+   difficult to backtrack from. Retransmitting the Bank Switch command may be
+   possible in a single segment setup, but this can lead to synchronization
+   problems when enabling multiple bus segments (a command with side effects
+   such as frame reconfiguration would be handled at different times). A global
+   hard-reset might be the best solution.
+
+Note that SoundWire does not provide a mechanism to detect illegal values
+written in valid registers. In a number of cases the standard even mentions
+that the Slave might behave in implementation-defined ways. The bus
+implementation does not provide a recovery mechanism for such errors, Slave
+or Master driver implementers are responsible for writing valid values in
+valid registers and implement additional range checking if needed.
diff --git a/Documentation/driver-api/soundwire/index.rst b/Documentation/driver-api/soundwire/index.rst
index 647e94654752..6db026028f27 100644
--- a/Documentation/driver-api/soundwire/index.rst
+++ b/Documentation/driver-api/soundwire/index.rst
@@ -6,6 +6,9 @@ SoundWire Documentation
    :maxdepth: 1
 
    summary
+   stream
+   error_handling
+   locking
 
 .. only::  subproject
 
diff --git a/Documentation/driver-api/soundwire/locking.rst b/Documentation/driver-api/soundwire/locking.rst
new file mode 100644
index 000000000000..253f73555255
--- /dev/null
+++ b/Documentation/driver-api/soundwire/locking.rst
@@ -0,0 +1,106 @@
+=================
+SoundWire Locking
+=================
+
+This document explains locking mechanism of the SoundWire Bus. Bus uses
+following locks in order to avoid race conditions in Bus operations on
+shared resources.
+
+  - Bus lock
+
+  - Message lock
+
+Bus lock
+========
+
+SoundWire Bus lock is a mutex and is part of Bus data structure
+(sdw_bus) which is used for every Bus instance. This lock is used to
+serialize each of the following operations(s) within SoundWire Bus instance.
+
+  - Addition and removal of Slave(s), changing Slave status.
+
+  - Prepare, Enable, Disable and De-prepare stream operations.
+
+  - Access of Stream data structure.
+
+Message lock
+============
+
+SoundWire message transfer lock. This mutex is part of
+Bus data structure (sdw_bus). This lock is used to serialize the message
+transfers (read/write) within a SoundWire Bus instance.
+
+Below examples show how locks are acquired.
+
+Example 1
+---------
+
+Message transfer.
+
+  1. For every message transfer
+
+     a. Acquire Message lock.
+
+     b. Transfer message (Read/Write) to Slave1 or broadcast message on
+        Bus in case of bank switch.
+
+     c. Release Message lock ::
+
+	+----------+                    +---------+
+	|          |                    |         |
+	|   Bus    |                    | Master  |
+	|          |                    | Driver  |
+	|          |                    |         |
+	+----+-----+                    +----+----+
+	     |                               |
+	     |     bus->ops->xfer_msg()      |
+	     <-------------------------------+   a. Acquire Message lock
+	     |                               |   b. Transfer message
+	     |                               |
+	     +------------------------------->   c. Release Message lock
+	     |    return success/error       |   d. Return success/error
+	     |                               |
+	     +                               +
+
+Example 2
+---------
+
+Prepare operation.
+
+  1. Acquire lock for Bus instance associated with Master 1.
+
+  2. For every message transfer in Prepare operation
+
+     a. Acquire Message lock.
+
+     b. Transfer message (Read/Write) to Slave1 or broadcast message on
+        Bus in case of bank switch.
+
+     c. Release Message lock.
+
+  3. Release lock for Bus instance associated with Master 1 ::
+
+	+----------+                    +---------+
+	|          |                    |         |
+	|   Bus    |                    | Master  |
+	|          |                    | Driver  |
+	|          |                    |         |
+	+----+-----+                    +----+----+
+	     |                               |
+	     |    sdw_prepare_stream()       |
+	     <-------------------------------+   1. Acquire bus lock
+	     |                               |   2. Perform stream prepare
+	     |                               |
+	     |                               |
+	     |     bus->ops->xfer_msg()      |
+	     <-------------------------------+   a. Acquire Message lock
+	     |                               |   b. Transfer message
+	     |                               |
+	     +------------------------------->   c. Release Message lock
+	     |    return success/error       |   d. Return success/error
+	     |                               |
+	     |                               |
+	     |    return success/error       |   3. Release bus lock
+	     +------------------------------->   4. Return success/error
+	     |                               |
+	     +                               +
diff --git a/Documentation/driver-api/soundwire/stream.rst b/Documentation/driver-api/soundwire/stream.rst
new file mode 100644
index 000000000000..29121aa55fb9
--- /dev/null
+++ b/Documentation/driver-api/soundwire/stream.rst
@@ -0,0 +1,372 @@
+=========================
+Audio Stream in SoundWire
+=========================
+
+An audio stream is a logical or virtual connection created between
+
+  (1) System memory buffer(s) and Codec(s)
+
+  (2) DSP memory buffer(s) and Codec(s)
+
+  (3) FIFO(s) and Codec(s)
+
+  (4) Codec(s) and Codec(s)
+
+which is typically driven by a DMA(s) channel through the data link. An
+audio stream contains one or more channels of data. All channels within
+stream must have same sample rate and same sample size.
+
+Assume a stream with two channels (Left & Right) is opened using SoundWire
+interface. Below are some ways a stream can be represented in SoundWire.
+
+Stream Sample in memory (System memory, DSP memory or FIFOs) ::
+
+	-------------------------
+	| L | R | L | R | L | R |
+	-------------------------
+
+Example 1: Stereo Stream with L and R channels is rendered from Master to
+Slave. Both Master and Slave is using single port. ::
+
+	+---------------+                    Clock Signal  +---------------+
+	|    Master     +----------------------------------+     Slave     |
+	|   Interface   |                                  |   Interface   |
+	|               |                                  |       1       |
+	|               |                     Data Signal  |               |
+	|    L  +  R    +----------------------------------+    L  +  R    |
+	|     (Data)    |     Data Direction               |     (Data)    |
+	+---------------+  +----------------------->       +---------------+
+
+
+Example 2: Stereo Stream with L and R channels is captured from Slave to
+Master. Both Master and Slave is using single port. ::
+
+
+	+---------------+                    Clock Signal  +---------------+
+	|    Master     +----------------------------------+     Slave     |
+	|   Interface   |                                  |   Interface   |
+	|               |                                  |       1       |
+	|               |                     Data Signal  |               |
+	|    L  +  R    +----------------------------------+    L  +  R    |
+	|     (Data)    |     Data Direction               |     (Data)    |
+	+---------------+  <-----------------------+       +---------------+
+
+
+Example 3: Stereo Stream with L and R channels is rendered by Master. Each
+of the L and R channel is received by two different Slaves. Master and both
+Slaves are using single port. ::
+
+	+---------------+                    Clock Signal  +---------------+
+	|    Master     +---------+------------------------+     Slave     |
+	|   Interface   |         |                        |   Interface   |
+	|               |         |                        |       1       |
+	|               |         |           Data Signal  |               |
+	|    L  +  R    +---+------------------------------+       L       |
+	|     (Data)    |   |     |    Data Direction      |     (Data)    |
+	+---------------+   |     |   +------------->      +---------------+
+	                    |     |
+	                    |     |
+	                    |     |                        +---------------+
+	                    |     +----------------------> |     Slave     |
+	                    |                              |   Interface   |
+	                    |                              |       2       |
+	                    |                              |               |
+	                    +----------------------------> |       R       |
+	                                                   |     (Data)    |
+	                                                   +---------------+
+
+
+Example 4: Stereo Stream with L and R channel is rendered by two different
+Ports of the Master and is received by only single Port of the Slave
+interface. ::
+
+	+--------------------+
+	|                    |
+	|     +--------------+                             +----------------+
+	|     |             ||                             |                |
+	|     |  Data Port  ||  L Channel                  |                |
+	|     |      1      |------------+                 |                |
+	|     |  L Channel  ||           |                 +-----+----+     |
+	|     |   (Data)    ||           |   L + R Channel ||    Data |     |
+	| Master  +----------+           | +---+---------> ||    Port |     |
+	| Interface          |           |                 ||     1   |     |
+	|     +--------------+           |                 ||         |     |
+	|     |             ||           |                 +----------+     |
+	|     |  Data Port  |------------+                 |                |
+	|     |      2      ||  R Channel                  |     Slave      |
+	|     |  R Channel  ||                             |   Interface    |
+	|     |   (Data)    ||                             |       1        |
+	|     +--------------+         Clock Signal        |     L  +  R    |
+	|                    +---------------------------> |      (Data)    |
+	+--------------------+                             |                |
+							   +----------------+
+
+SoundWire Stream Management flow
+================================
+
+Stream definitions
+------------------
+
+  (1) Current stream: This is classified as the stream on which operation has
+      to be performed like prepare, enable, disable, de-prepare etc.
+
+  (2) Active stream: This is classified as the stream which is already active
+      on Bus other than current stream. There can be multiple active streams
+      on the Bus.
+
+SoundWire Bus manages stream operations for each stream getting
+rendered/captured on the SoundWire Bus. This section explains Bus operations
+done for each of the stream allocated/released on Bus. Following are the
+stream states maintained by the Bus for each of the audio stream.
+
+
+SoundWire stream states
+-----------------------
+
+Below shows the SoundWire stream states and state transition diagram. ::
+
+	+-----------+     +------------+     +----------+     +----------+
+	| ALLOCATED +---->| CONFIGURED +---->| PREPARED +---->| ENABLED  |
+	|   STATE   |     |    STATE   |     |  STATE   |     |  STATE   |
+	+-----------+     +------------+     +----------+     +----+-----+
+	                                                           ^
+	                                                           |
+	                                                           |
+	                                                           v
+	         +----------+           +------------+        +----+-----+
+	         | RELEASED |<----------+ DEPREPARED |<-------+ DISABLED |
+	         |  STATE   |           |   STATE    |        |  STATE   |
+	         +----------+           +------------+        +----------+
+
+NOTE: State transition between prepare and deprepare is supported in Spec
+but not in the software (subsystem)
+
+NOTE2: Stream state transition checks need to be handled by caller
+framework, for example ALSA/ASoC. No checks for stream transition exist in
+SoundWire subsystem.
+
+Stream State Operations
+-----------------------
+
+Below section explains the operations done by the Bus on Master(s) and
+Slave(s) as part of stream state transitions.
+
+SDW_STREAM_ALLOCATED
+~~~~~~~~~~~~~~~~~~~~
+
+Allocation state for stream. This is the entry state
+of the stream. Operations performed before entering in this state:
+
+  (1) A stream runtime is allocated for the stream. This stream
+      runtime is used as a reference for all the operations performed
+      on the stream.
+
+  (2) The resources required for holding stream runtime information are
+      allocated and initialized. This holds all stream related information
+      such as stream type (PCM/PDM) and parameters, Master and Slave
+      interface associated with the stream, stream state etc.
+
+After all above operations are successful, stream state is set to
+``SDW_STREAM_ALLOCATED``.
+
+Bus implements below API for allocate a stream which needs to be called once
+per stream. From ASoC DPCM framework, this stream state maybe linked to
+.startup() operation.
+
+  .. code-block:: c
+  int sdw_alloc_stream(char * stream_name);
+
+
+SDW_STREAM_CONFIGURED
+~~~~~~~~~~~~~~~~~~~~~
+
+Configuration state of stream. Operations performed before entering in
+this state:
+
+  (1) The resources allocated for stream information in SDW_STREAM_ALLOCATED
+      state are updated here. This includes stream parameters, Master(s)
+      and Slave(s) runtime information associated with current stream.
+
+  (2) All the Master(s) and Slave(s) associated with current stream provide
+      the port information to Bus which includes port numbers allocated by
+      Master(s) and Slave(s) for current stream and their channel mask.
+
+After all above operations are successful, stream state is set to
+``SDW_STREAM_CONFIGURED``.
+
+Bus implements below APIs for CONFIG state which needs to be called by
+the respective Master(s) and Slave(s) associated with stream. These APIs can
+only be invoked once by respective Master(s) and Slave(s). From ASoC DPCM
+framework, this stream state is linked to .hw_params() operation.
+
+  .. code-block:: c
+  int sdw_stream_add_master(struct sdw_bus * bus,
+		struct sdw_stream_config * stream_config,
+		struct sdw_ports_config * ports_config,
+		struct sdw_stream_runtime * stream);
+
+  int sdw_stream_add_slave(struct sdw_slave * slave,
+		struct sdw_stream_config * stream_config,
+		struct sdw_ports_config * ports_config,
+		struct sdw_stream_runtime * stream);
+
+
+SDW_STREAM_PREPARED
+~~~~~~~~~~~~~~~~~~~
+
+Prepare state of stream. Operations performed before entering in this state:
+
+  (1) Bus parameters such as bandwidth, frame shape, clock frequency,
+      are computed based on current stream as well as already active
+      stream(s) on Bus. Re-computation is required to accommodate current
+      stream on the Bus.
+
+  (2) Transport and port parameters of all Master(s) and Slave(s) port(s) are
+      computed for the current as well as already active stream based on frame
+      shape and clock frequency computed in step 1.
+
+  (3) Computed Bus and transport parameters are programmed in Master(s) and
+      Slave(s) registers. The banked registers programming is done on the
+      alternate bank (bank currently unused). Port(s) are enabled for the
+      already active stream(s) on the alternate bank (bank currently unused).
+      This is done in order to not disrupt already active stream(s).
+
+  (4) Once all the values are programmed, Bus initiates switch to alternate
+      bank where all new values programmed gets into effect.
+
+  (5) Ports of Master(s) and Slave(s) for current stream are prepared by
+      programming PrepareCtrl register.
+
+After all above operations are successful, stream state is set to
+``SDW_STREAM_PREPARED``.
+
+Bus implements below API for PREPARE state which needs to be called once per
+stream. From ASoC DPCM framework, this stream state is linked to
+.prepare() operation.
+
+  .. code-block:: c
+  int sdw_prepare_stream(struct sdw_stream_runtime * stream);
+
+
+SDW_STREAM_ENABLED
+~~~~~~~~~~~~~~~~~~
+
+Enable state of stream. The data port(s) are enabled upon entering this state.
+Operations performed before entering in this state:
+
+  (1) All the values computed in SDW_STREAM_PREPARED state are programmed
+      in alternate bank (bank currently unused). It includes programming of
+      already active stream(s) as well.
+
+  (2) All the Master(s) and Slave(s) port(s) for the current stream are
+      enabled on alternate bank (bank currently unused) by programming
+      ChannelEn register.
+
+  (3) Once all the values are programmed, Bus initiates switch to alternate
+      bank where all new values programmed gets into effect and port(s)
+      associated with current stream are enabled.
+
+After all above operations are successful, stream state is set to
+``SDW_STREAM_ENABLED``.
+
+Bus implements below API for ENABLE state which needs to be called once per
+stream. From ASoC DPCM framework, this stream state is linked to
+.trigger() start operation.
+
+  .. code-block:: c
+  int sdw_enable_stream(struct sdw_stream_runtime * stream);
+
+SDW_STREAM_DISABLED
+~~~~~~~~~~~~~~~~~~~
+
+Disable state of stream. The data port(s) are disabled upon exiting this state.
+Operations performed before entering in this state:
+
+  (1) All the Master(s) and Slave(s) port(s) for the current stream are
+      disabled on alternate bank (bank currently unused) by programming
+      ChannelEn register.
+
+  (2) All the current configuration of Bus and active stream(s) are programmed
+      into alternate bank (bank currently unused).
+
+  (3) Once all the values are programmed, Bus initiates switch to alternate
+      bank where all new values programmed gets into effect and port(s) associated
+      with current stream are disabled.
+
+After all above operations are successful, stream state is set to
+``SDW_STREAM_DISABLED``.
+
+Bus implements below API for DISABLED state which needs to be called once
+per stream. From ASoC DPCM framework, this stream state is linked to
+.trigger() stop operation.
+
+  .. code-block:: c
+  int sdw_disable_stream(struct sdw_stream_runtime * stream);
+
+
+SDW_STREAM_DEPREPARED
+~~~~~~~~~~~~~~~~~~~~~
+
+De-prepare state of stream. Operations performed before entering in this
+state:
+
+  (1) All the port(s) of Master(s) and Slave(s) for current stream are
+      de-prepared by programming PrepareCtrl register.
+
+  (2) The payload bandwidth of current stream is reduced from the total
+      bandwidth requirement of bus and new parameters calculated and
+      applied by performing bank switch etc.
+
+After all above operations are successful, stream state is set to
+``SDW_STREAM_DEPREPARED``.
+
+Bus implements below API for DEPREPARED state which needs to be called once
+per stream. From ASoC DPCM framework, this stream state is linked to
+.trigger() stop operation.
+
+  .. code-block:: c
+  int sdw_deprepare_stream(struct sdw_stream_runtime * stream);
+
+
+SDW_STREAM_RELEASED
+~~~~~~~~~~~~~~~~~~~
+
+Release state of stream. Operations performed before entering in this state:
+
+  (1) Release port resources for all Master(s) and Slave(s) port(s)
+      associated with current stream.
+
+  (2) Release Master(s) and Slave(s) runtime resources associated with
+      current stream.
+
+  (3) Release stream runtime resources associated with current stream.
+
+After all above operations are successful, stream state is set to
+``SDW_STREAM_RELEASED``.
+
+Bus implements below APIs for RELEASE state which needs to be called by
+all the Master(s) and Slave(s) associated with stream. From ASoC DPCM
+framework, this stream state is linked to .hw_free() operation.
+
+  .. code-block:: c
+  int sdw_stream_remove_master(struct sdw_bus * bus,
+		struct sdw_stream_runtime * stream);
+  int sdw_stream_remove_slave(struct sdw_slave * slave,
+		struct sdw_stream_runtime * stream);
+
+
+The .shutdown() ASoC DPCM operation calls below Bus API to release
+stream assigned as part of ALLOCATED state.
+
+In .shutdown() the data structure maintaining stream state are freed up.
+
+  .. code-block:: c
+  void sdw_release_stream(struct sdw_stream_runtime * stream);
+
+Not Supported
+=============
+
+1. A single port with multiple channels supported cannot be used between two
+streams or across stream. For example a port with 4 channels cannot be used
+to handle 2 independent stereo streams even though it's possible in theory
+in SoundWire.
diff --git a/Documentation/driver-api/target.rst b/Documentation/driver-api/target.rst
new file mode 100644
index 000000000000..4363611dd86d
--- /dev/null
+++ b/Documentation/driver-api/target.rst
@@ -0,0 +1,64 @@
+=================================
+target and iSCSI Interfaces Guide
+=================================
+
+Introduction and Overview
+=========================
+
+TBD
+
+Target core device interfaces
+=============================
+
+.. kernel-doc:: drivers/target/target_core_device.c
+    :export:
+
+Target core transport interfaces
+================================
+
+.. kernel-doc:: drivers/target/target_core_transport.c
+    :export:
+
+Target-supported userspace I/O
+==============================
+
+.. kernel-doc:: drivers/target/target_core_user.c
+    :doc: Userspace I/O
+
+.. kernel-doc:: include/uapi/linux/target_core_user.h
+    :doc: Ring Design
+
+iSCSI helper functions
+======================
+
+.. kernel-doc:: drivers/scsi/libiscsi.c
+   :export:
+
+
+iSCSI boot information
+======================
+
+.. kernel-doc:: drivers/scsi/iscsi_boot_sysfs.c
+   :export:
+
+
+iSCSI transport class
+=====================
+
+The file drivers/scsi/scsi_transport_iscsi.c defines transport
+attributes for the iSCSI class, which sends SCSI packets over TCP/IP
+connections.
+
+.. kernel-doc:: drivers/scsi/scsi_transport_iscsi.c
+   :export:
+
+
+iSCSI TCP interfaces
+====================
+
+.. kernel-doc:: drivers/scsi/iscsi_tcp.c
+   :internal:
+
+.. kernel-doc:: drivers/scsi/libiscsi_tcp.c
+   :export:
+
diff --git a/Documentation/driver-api/uio-howto.rst b/Documentation/driver-api/uio-howto.rst
index 92056c20e070..fb2eb73be4a3 100644
--- a/Documentation/driver-api/uio-howto.rst
+++ b/Documentation/driver-api/uio-howto.rst
@@ -711,7 +711,8 @@ The vmbus device regions are mapped into uio device resources:
 
 If a subchannel is created by a request to host, then the uio_hv_generic
 device driver will create a sysfs binary file for the per-channel ring buffer.
-For example:
+For example::
+
 	/sys/bus/vmbus/devices/3811fe4d-0fa0-4b62-981a-74fc1084c757/channels/21/ring
 
 Further information
diff --git a/Documentation/driver-api/usb/dwc3.rst b/Documentation/driver-api/usb/dwc3.rst
index c3dc84a50ce5..8b36ff11cef9 100644
--- a/Documentation/driver-api/usb/dwc3.rst
+++ b/Documentation/driver-api/usb/dwc3.rst
@@ -674,9 +674,8 @@ operations, both of which can be traced. Format is::
   	__entry->flags & DWC3_EP_ENABLED ? 'E' : 'e',
   	__entry->flags & DWC3_EP_STALL ? 'S' : 's',
   	__entry->flags & DWC3_EP_WEDGE ? 'W' : 'w',
-  	__entry->flags & DWC3_EP_BUSY ? 'B' : 'b',
+  	__entry->flags & DWC3_EP_TRANSFER_STARTED ? 'B' : 'b',
   	__entry->flags & DWC3_EP_PENDING_REQUEST ? 'P' : 'p',
-  	__entry->flags & DWC3_EP_MISSED_ISOC ? 'M' : 'm',
   	__entry->flags & DWC3_EP_END_TRANSFER_PENDING ? 'E' : 'e',
   	__entry->direction ? '<' : '>'
   )
diff --git a/Documentation/driver-model/devres.txt b/Documentation/driver-model/devres.txt
index 7c1bb3d0c222..a3e8bceb5f19 100644
--- a/Documentation/driver-model/devres.txt
+++ b/Documentation/driver-model/devres.txt
@@ -344,6 +344,7 @@ MUX
   devm_mux_chip_alloc()
   devm_mux_chip_register()
   devm_mux_control_get()
+  devm_mux_control_get_optional()
 
 PER-CPU MEM
   devm_alloc_percpu()
diff --git a/Documentation/features/core/BPF-JIT/arch-support.txt b/Documentation/features/core/BPF-JIT/arch-support.txt
deleted file mode 100644
index 0b96b4e1e7d4..000000000000
--- a/Documentation/features/core/BPF-JIT/arch-support.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Feature name:          BPF-JIT
-#         Kconfig:       HAVE_BPF_JIT
-#         description:   arch supports BPF JIT optimizations
-#
-    -----------------------
-    |         arch |status|
-    -----------------------
-    |       alpha: | TODO |
-    |         arc: | TODO |
-    |         arm: |  ok  |
-    |       arm64: |  ok  |
-    |         c6x: | TODO |
-    |       h8300: | TODO |
-    |     hexagon: | TODO |
-    |        ia64: | TODO |
-    |        m68k: | TODO |
-    |  microblaze: | TODO |
-    |        mips: |  ok  |
-    |       nios2: | TODO |
-    |    openrisc: | TODO |
-    |      parisc: | TODO |
-    |     powerpc: |  ok  |
-    |        s390: |  ok  |
-    |          sh: | TODO |
-    |       sparc: |  ok  |
-    |          um: | TODO |
-    |   unicore32: | TODO |
-    |         x86: |  ok  |
-    |      xtensa: | TODO |
-    -----------------------
diff --git a/Documentation/features/core/cBPF-JIT/arch-support.txt b/Documentation/features/core/cBPF-JIT/arch-support.txt
new file mode 100644
index 000000000000..90459cdde314
--- /dev/null
+++ b/Documentation/features/core/cBPF-JIT/arch-support.txt
@@ -0,0 +1,33 @@
+#
+# Feature name:          cBPF-JIT
+#         Kconfig:       HAVE_CBPF_JIT
+#         description:   arch supports cBPF JIT optimizations
+#
+    -----------------------
+    |         arch |status|
+    -----------------------
+    |       alpha: | TODO |
+    |         arc: | TODO |
+    |         arm: | TODO |
+    |       arm64: | TODO |
+    |         c6x: | TODO |
+    |       h8300: | TODO |
+    |     hexagon: | TODO |
+    |        ia64: | TODO |
+    |        m68k: | TODO |
+    |  microblaze: | TODO |
+    |        mips: |  ok  |
+    |       nds32: | TODO |
+    |       nios2: | TODO |
+    |    openrisc: | TODO |
+    |      parisc: | TODO |
+    |     powerpc: |  ok  |
+    |       riscv: | TODO |
+    |        s390: | TODO |
+    |          sh: | TODO |
+    |       sparc: |  ok  |
+    |          um: | TODO |
+    |   unicore32: | TODO |
+    |         x86: | TODO |
+    |      xtensa: | TODO |
+    -----------------------
diff --git a/Documentation/features/core/eBPF-JIT/arch-support.txt b/Documentation/features/core/eBPF-JIT/arch-support.txt
new file mode 100644
index 000000000000..c90a0382fe66
--- /dev/null
+++ b/Documentation/features/core/eBPF-JIT/arch-support.txt
@@ -0,0 +1,33 @@
+#
+# Feature name:          eBPF-JIT
+#         Kconfig:       HAVE_EBPF_JIT
+#         description:   arch supports eBPF JIT optimizations
+#
+    -----------------------
+    |         arch |status|
+    -----------------------
+    |       alpha: | TODO |
+    |         arc: | TODO |
+    |         arm: |  ok  |
+    |       arm64: |  ok  |
+    |         c6x: | TODO |
+    |       h8300: | TODO |
+    |     hexagon: | TODO |
+    |        ia64: | TODO |
+    |        m68k: | TODO |
+    |  microblaze: | TODO |
+    |        mips: |  ok  |
+    |       nds32: | TODO |
+    |       nios2: | TODO |
+    |    openrisc: | TODO |
+    |      parisc: | TODO |
+    |     powerpc: |  ok  |
+    |       riscv: | TODO |
+    |        s390: |  ok  |
+    |          sh: | TODO |
+    |       sparc: |  ok  |
+    |          um: | TODO |
+    |   unicore32: | TODO |
+    |         x86: |  ok  |
+    |      xtensa: | TODO |
+    -----------------------
diff --git a/Documentation/features/core/generic-idle-thread/arch-support.txt b/Documentation/features/core/generic-idle-thread/arch-support.txt
index 372a2b18a617..0ef6acdb991c 100644
--- a/Documentation/features/core/generic-idle-thread/arch-support.txt
+++ b/Documentation/features/core/generic-idle-thread/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
-    |    openrisc: | TODO |
+    |    openrisc: |  ok  |
     |      parisc: |  ok  |
     |     powerpc: |  ok  |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: |  ok  |
     |       sparc: |  ok  |
diff --git a/Documentation/features/core/jump-labels/arch-support.txt b/Documentation/features/core/jump-labels/arch-support.txt
index ad97217b003b..27cbd63abfd2 100644
--- a/Documentation/features/core/jump-labels/arch-support.txt
+++ b/Documentation/features/core/jump-labels/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: |  ok  |
diff --git a/Documentation/features/core/tracehook/arch-support.txt b/Documentation/features/core/tracehook/arch-support.txt
index 36ee7bef5d18..f44c274e40ed 100644
--- a/Documentation/features/core/tracehook/arch-support.txt
+++ b/Documentation/features/core/tracehook/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: |  ok  |
     |       nios2: |  ok  |
     |    openrisc: |  ok  |
     |      parisc: |  ok  |
     |     powerpc: |  ok  |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: |  ok  |
     |       sparc: |  ok  |
diff --git a/Documentation/features/debug/KASAN/arch-support.txt b/Documentation/features/debug/KASAN/arch-support.txt
index f5c99fa576d3..282ecc8ea1da 100644
--- a/Documentation/features/debug/KASAN/arch-support.txt
+++ b/Documentation/features/debug/KASAN/arch-support.txt
@@ -17,15 +17,17 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: | TODO |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: | TODO |
     |       sparc: | TODO |
     |          um: | TODO |
     |   unicore32: | TODO |
-    |         x86: |  ok  | 64-bit only
+    |         x86: |  ok  |
     |      xtensa: |  ok  |
     -----------------------
diff --git a/Documentation/features/debug/gcov-profile-all/arch-support.txt b/Documentation/features/debug/gcov-profile-all/arch-support.txt
index 5170a9934843..01b2b3004e0a 100644
--- a/Documentation/features/debug/gcov-profile-all/arch-support.txt
+++ b/Documentation/features/debug/gcov-profile-all/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: |  ok  |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: |  ok  |
     |       sparc: | TODO |
diff --git a/Documentation/features/debug/kgdb/arch-support.txt b/Documentation/features/debug/kgdb/arch-support.txt
index 13b6e994ae1f..3b4dff22329f 100644
--- a/Documentation/features/debug/kgdb/arch-support.txt
+++ b/Documentation/features/debug/kgdb/arch-support.txt
@@ -11,16 +11,18 @@
     |         arm: |  ok  |
     |       arm64: |  ok  |
     |         c6x: | TODO |
-    |       h8300: | TODO |
+    |       h8300: |  ok  |
     |     hexagon: |  ok  |
     |        ia64: | TODO |
     |        m68k: | TODO |
     |  microblaze: |  ok  |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: |  ok  |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: |  ok  |
     |       sparc: |  ok  |
diff --git a/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt b/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt
index 419bb38820e7..7e963d0ae646 100644
--- a/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt
+++ b/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/Documentation/features/debug/kprobes/arch-support.txt b/Documentation/features/debug/kprobes/arch-support.txt
index 52b3ace0a030..4ada027faf16 100644
--- a/Documentation/features/debug/kprobes/arch-support.txt
+++ b/Documentation/features/debug/kprobes/arch-support.txt
@@ -9,7 +9,7 @@
     |       alpha: | TODO |
     |         arc: |  ok  |
     |         arm: |  ok  |
-    |       arm64: | TODO |
+    |       arm64: |  ok  |
     |         c6x: | TODO |
     |       h8300: | TODO |
     |     hexagon: | TODO |
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: |  ok  |
     |       sparc: |  ok  |
diff --git a/Documentation/features/debug/kretprobes/arch-support.txt b/Documentation/features/debug/kretprobes/arch-support.txt
index 180d24419518..044e13fcca5d 100644
--- a/Documentation/features/debug/kretprobes/arch-support.txt
+++ b/Documentation/features/debug/kretprobes/arch-support.txt
@@ -9,7 +9,7 @@
     |       alpha: | TODO |
     |         arc: |  ok  |
     |         arm: |  ok  |
-    |       arm64: | TODO |
+    |       arm64: |  ok  |
     |         c6x: | TODO |
     |       h8300: | TODO |
     |     hexagon: | TODO |
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: |  ok  |
     |       sparc: |  ok  |
diff --git a/Documentation/features/debug/optprobes/arch-support.txt b/Documentation/features/debug/optprobes/arch-support.txt
index 0a1241f45e41..dce7669c918f 100644
--- a/Documentation/features/debug/optprobes/arch-support.txt
+++ b/Documentation/features/debug/optprobes/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
-    |     powerpc: | TODO |
+    |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/Documentation/features/debug/stackprotector/arch-support.txt b/Documentation/features/debug/stackprotector/arch-support.txt
index 570019572383..74b89a9c8b3a 100644
--- a/Documentation/features/debug/stackprotector/arch-support.txt
+++ b/Documentation/features/debug/stackprotector/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: | TODO |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: |  ok  |
     |       sparc: | TODO |
diff --git a/Documentation/features/debug/uprobes/arch-support.txt b/Documentation/features/debug/uprobes/arch-support.txt
index 0b8d922eb799..1a3f9d3229bf 100644
--- a/Documentation/features/debug/uprobes/arch-support.txt
+++ b/Documentation/features/debug/uprobes/arch-support.txt
@@ -9,7 +9,7 @@
     |       alpha: | TODO |
     |         arc: | TODO |
     |         arm: |  ok  |
-    |       arm64: | TODO |
+    |       arm64: |  ok  |
     |         c6x: | TODO |
     |       h8300: | TODO |
     |     hexagon: | TODO |
@@ -17,13 +17,15 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: | TODO |
-    |       sparc: | TODO |
+    |       sparc: |  ok  |
     |          um: | TODO |
     |   unicore32: | TODO |
     |         x86: |  ok  |
diff --git a/Documentation/features/debug/user-ret-profiler/arch-support.txt b/Documentation/features/debug/user-ret-profiler/arch-support.txt
index 13852ae62e9e..1d78d1069a5f 100644
--- a/Documentation/features/debug/user-ret-profiler/arch-support.txt
+++ b/Documentation/features/debug/user-ret-profiler/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: | TODO |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/Documentation/features/io/dma-api-debug/arch-support.txt b/Documentation/features/io/dma-api-debug/arch-support.txt
deleted file mode 100644
index e438ed675623..000000000000
--- a/Documentation/features/io/dma-api-debug/arch-support.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Feature name:          dma-api-debug
-#         Kconfig:       HAVE_DMA_API_DEBUG
-#         description:   arch supports DMA debug facilities
-#
-    -----------------------
-    |         arch |status|
-    -----------------------
-    |       alpha: | TODO |
-    |         arc: | TODO |
-    |         arm: |  ok  |
-    |       arm64: |  ok  |
-    |         c6x: |  ok  |
-    |       h8300: | TODO |
-    |     hexagon: | TODO |
-    |        ia64: |  ok  |
-    |        m68k: | TODO |
-    |  microblaze: |  ok  |
-    |        mips: |  ok  |
-    |       nios2: | TODO |
-    |    openrisc: | TODO |
-    |      parisc: | TODO |
-    |     powerpc: |  ok  |
-    |        s390: |  ok  |
-    |          sh: |  ok  |
-    |       sparc: |  ok  |
-    |          um: | TODO |
-    |   unicore32: | TODO |
-    |         x86: |  ok  |
-    |      xtensa: |  ok  |
-    -----------------------
diff --git a/Documentation/features/io/dma-contiguous/arch-support.txt b/Documentation/features/io/dma-contiguous/arch-support.txt
index 47f64a433df0..30c072d2b67c 100644
--- a/Documentation/features/io/dma-contiguous/arch-support.txt
+++ b/Documentation/features/io/dma-contiguous/arch-support.txt
@@ -17,11 +17,13 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: | TODO |
-    |        s390: | TODO |
+    |       riscv: |  ok  |
+    |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: | TODO |
     |          um: | TODO |
diff --git a/Documentation/features/io/sg-chain/arch-support.txt b/Documentation/features/io/sg-chain/arch-support.txt
index 07f357fadbff..6554f0372c3f 100644
--- a/Documentation/features/io/sg-chain/arch-support.txt
+++ b/Documentation/features/io/sg-chain/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: |  ok  |
diff --git a/Documentation/features/lib/strncasecmp/arch-support.txt b/Documentation/features/lib/strncasecmp/arch-support.txt
deleted file mode 100644
index 4f3a6a0e4e68..000000000000
--- a/Documentation/features/lib/strncasecmp/arch-support.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Feature name:          strncasecmp
-#         Kconfig:       __HAVE_ARCH_STRNCASECMP
-#         description:   arch provides an optimized strncasecmp() function
-#
-    -----------------------
-    |         arch |status|
-    -----------------------
-    |       alpha: | TODO |
-    |         arc: | TODO |
-    |         arm: | TODO |
-    |       arm64: | TODO |
-    |         c6x: | TODO |
-    |       h8300: | TODO |
-    |     hexagon: | TODO |
-    |        ia64: | TODO |
-    |        m68k: | TODO |
-    |  microblaze: | TODO |
-    |        mips: | TODO |
-    |       nios2: | TODO |
-    |    openrisc: | TODO |
-    |      parisc: | TODO |
-    |     powerpc: | TODO |
-    |        s390: | TODO |
-    |          sh: | TODO |
-    |       sparc: | TODO |
-    |          um: | TODO |
-    |   unicore32: | TODO |
-    |         x86: | TODO |
-    |      xtensa: | TODO |
-    -----------------------
diff --git a/Documentation/features/locking/cmpxchg-local/arch-support.txt b/Documentation/features/locking/cmpxchg-local/arch-support.txt
index 482a0b09d1f8..51704a2dc8d1 100644
--- a/Documentation/features/locking/cmpxchg-local/arch-support.txt
+++ b/Documentation/features/locking/cmpxchg-local/arch-support.txt
@@ -9,7 +9,7 @@
     |       alpha: | TODO |
     |         arc: | TODO |
     |         arm: | TODO |
-    |       arm64: | TODO |
+    |       arm64: |  ok  |
     |         c6x: | TODO |
     |       h8300: | TODO |
     |     hexagon: | TODO |
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: | TODO |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/Documentation/features/locking/lockdep/arch-support.txt b/Documentation/features/locking/lockdep/arch-support.txt
index bb35c5ba6286..bd39c5edd460 100644
--- a/Documentation/features/locking/lockdep/arch-support.txt
+++ b/Documentation/features/locking/lockdep/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: |  ok  |
     |        mips: |  ok  |
+    |       nds32: |  ok  |
     |       nios2: | TODO |
-    |    openrisc: | TODO |
+    |    openrisc: |  ok  |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: |  ok  |
     |       sparc: |  ok  |
diff --git a/Documentation/features/locking/queued-rwlocks/arch-support.txt b/Documentation/features/locking/queued-rwlocks/arch-support.txt
index 627e9a6b2db9..da7aff3bee0b 100644
--- a/Documentation/features/locking/queued-rwlocks/arch-support.txt
+++ b/Documentation/features/locking/queued-rwlocks/arch-support.txt
@@ -9,21 +9,23 @@
     |       alpha: | TODO |
     |         arc: | TODO |
     |         arm: | TODO |
-    |       arm64: | TODO |
+    |       arm64: |  ok  |
     |         c6x: | TODO |
     |       h8300: | TODO |
     |     hexagon: | TODO |
     |        ia64: | TODO |
     |        m68k: | TODO |
     |  microblaze: | TODO |
-    |        mips: | TODO |
+    |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
-    |    openrisc: | TODO |
+    |    openrisc: |  ok  |
     |      parisc: | TODO |
     |     powerpc: | TODO |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: | TODO |
-    |       sparc: | TODO |
+    |       sparc: |  ok  |
     |          um: | TODO |
     |   unicore32: | TODO |
     |         x86: |  ok  |
diff --git a/Documentation/features/locking/queued-spinlocks/arch-support.txt b/Documentation/features/locking/queued-spinlocks/arch-support.txt
index 9edda216cdfb..478e9101322c 100644
--- a/Documentation/features/locking/queued-spinlocks/arch-support.txt
+++ b/Documentation/features/locking/queued-spinlocks/arch-support.txt
@@ -16,14 +16,16 @@
     |        ia64: | TODO |
     |        m68k: | TODO |
     |  microblaze: | TODO |
-    |        mips: | TODO |
+    |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
-    |    openrisc: | TODO |
+    |    openrisc: |  ok  |
     |      parisc: | TODO |
     |     powerpc: | TODO |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: | TODO |
-    |       sparc: | TODO |
+    |       sparc: |  ok  |
     |          um: | TODO |
     |   unicore32: | TODO |
     |         x86: |  ok  |
diff --git a/Documentation/features/locking/rwsem-optimized/arch-support.txt b/Documentation/features/locking/rwsem-optimized/arch-support.txt
index 8d9afb10b16e..e54b1f1a8091 100644
--- a/Documentation/features/locking/rwsem-optimized/arch-support.txt
+++ b/Documentation/features/locking/rwsem-optimized/arch-support.txt
@@ -1,6 +1,6 @@
 #
 # Feature name:          rwsem-optimized
-#         Kconfig:       Optimized asm/rwsem.h
+#         Kconfig:       !RWSEM_GENERIC_SPINLOCK
 #         description:   arch provides optimized rwsem APIs
 #
     -----------------------
@@ -8,8 +8,8 @@
     -----------------------
     |       alpha: |  ok  |
     |         arc: | TODO |
-    |         arm: | TODO |
-    |       arm64: | TODO |
+    |         arm: |  ok  |
+    |       arm64: |  ok  |
     |         c6x: | TODO |
     |       h8300: | TODO |
     |     hexagon: | TODO |
@@ -17,14 +17,16 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: | TODO |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: |  ok  |
     |       sparc: |  ok  |
-    |          um: | TODO |
+    |          um: |  ok  |
     |   unicore32: | TODO |
     |         x86: |  ok  |
     |      xtensa: |  ok  |
diff --git a/Documentation/features/perf/kprobes-event/arch-support.txt b/Documentation/features/perf/kprobes-event/arch-support.txt
index d01239ee34b3..7331402d1887 100644
--- a/Documentation/features/perf/kprobes-event/arch-support.txt
+++ b/Documentation/features/perf/kprobes-event/arch-support.txt
@@ -9,7 +9,7 @@
     |       alpha: | TODO |
     |         arc: | TODO |
     |         arm: |  ok  |
-    |       arm64: | TODO |
+    |       arm64: |  ok  |
     |         c6x: | TODO |
     |       h8300: | TODO |
     |     hexagon: |  ok  |
@@ -17,13 +17,15 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: |  ok  |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: |  ok  |
-    |       sparc: | TODO |
+    |       sparc: |  ok  |
     |          um: | TODO |
     |   unicore32: | TODO |
     |         x86: |  ok  |
diff --git a/Documentation/features/perf/perf-regs/arch-support.txt b/Documentation/features/perf/perf-regs/arch-support.txt
index 458faba5311a..53feeee6cdad 100644
--- a/Documentation/features/perf/perf-regs/arch-support.txt
+++ b/Documentation/features/perf/perf-regs/arch-support.txt
@@ -17,11 +17,13 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
-    |        s390: | TODO |
+    |       riscv: | TODO |
+    |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: | TODO |
     |          um: | TODO |
diff --git a/Documentation/features/perf/perf-stackdump/arch-support.txt b/Documentation/features/perf/perf-stackdump/arch-support.txt
index 545d01c69c88..16164348e0ea 100644
--- a/Documentation/features/perf/perf-stackdump/arch-support.txt
+++ b/Documentation/features/perf/perf-stackdump/arch-support.txt
@@ -17,11 +17,13 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
-    |        s390: | TODO |
+    |       riscv: | TODO |
+    |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: | TODO |
     |          um: | TODO |
diff --git a/Documentation/features/sched/membarrier-sync-core/arch-support.txt b/Documentation/features/sched/membarrier-sync-core/arch-support.txt
index 85a6c9d4571c..dbdf62907703 100644
--- a/Documentation/features/sched/membarrier-sync-core/arch-support.txt
+++ b/Documentation/features/sched/membarrier-sync-core/arch-support.txt
@@ -40,10 +40,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: | TODO |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/Documentation/features/sched/numa-balancing/arch-support.txt b/Documentation/features/sched/numa-balancing/arch-support.txt
index 347508863872..c68bb2c2cb62 100644
--- a/Documentation/features/sched/numa-balancing/arch-support.txt
+++ b/Documentation/features/sched/numa-balancing/arch-support.txt
@@ -9,7 +9,7 @@
     |       alpha: | TODO |
     |         arc: |  ..  |
     |         arm: |  ..  |
-    |       arm64: |  ..  |
+    |       arm64: |  ok  |
     |         c6x: |  ..  |
     |       h8300: |  ..  |
     |     hexagon: |  ..  |
@@ -17,11 +17,13 @@
     |        m68k: |  ..  |
     |  microblaze: |  ..  |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: |  ..  |
     |    openrisc: |  ..  |
     |      parisc: |  ..  |
     |     powerpc: |  ok  |
-    |        s390: |  ..  |
+    |       riscv: | TODO |
+    |        s390: |  ok  |
     |          sh: |  ..  |
     |       sparc: | TODO |
     |          um: |  ..  |
diff --git a/Documentation/features/scripts/features-refresh.sh b/Documentation/features/scripts/features-refresh.sh
new file mode 100755
index 000000000000..9e72d38a0720
--- /dev/null
+++ b/Documentation/features/scripts/features-refresh.sh
@@ -0,0 +1,98 @@
+#
+# Small script that refreshes the kernel feature support status in place.
+#
+
+for F_FILE in Documentation/features/*/*/arch-support.txt; do
+	F=$(grep "^#         Kconfig:" "$F_FILE" | cut -c26-)
+
+	#
+	# Each feature F is identified by a pair (O, K), where 'O' can
+	# be either the empty string (for 'nop') or "not" (the logical
+	# negation operator '!'); other operators are not supported.
+	#
+	O=""
+	K=$F
+	if [[ "$F" == !* ]]; then
+		O="not"
+		K=$(echo $F | sed -e 's/^!//g')
+	fi
+
+	#
+	# F := (O, K) is 'valid' iff there is a Kconfig file (for some
+	# arch) which contains K.
+	#
+	# Notice that this definition entails an 'asymmetry' between
+	# the case 'O = ""' and the case 'O = "not"'. E.g., F may be
+	# _invalid_ if:
+	#
+	# [case 'O = ""']
+	#   1) no arch provides support for F,
+	#   2) K does not exist (e.g., it was renamed/mis-typed);
+	#
+	# [case 'O = "not"']
+	#   3) all archs provide support for F,
+	#   4) as in (2).
+	#
+	# The rationale for adopting this definition (and, thus, for
+	# keeping the asymmetry) is:
+	#
+	#       We want to be able to 'detect' (2) (or (4)).
+	#
+	# (1) and (3) may further warn the developers about the fact
+	# that K can be removed.
+	#
+	F_VALID="false"
+	for ARCH_DIR in arch/*/; do
+		K_FILES=$(find $ARCH_DIR -name "Kconfig*")
+		K_GREP=$(grep "$K" $K_FILES)
+		if [ ! -z "$K_GREP" ]; then
+			F_VALID="true"
+			break
+		fi
+	done
+	if [ "$F_VALID" = "false" ]; then
+		printf "WARNING: '%s' is not a valid Kconfig\n" "$F"
+	fi
+
+	T_FILE="$F_FILE.tmp"
+	grep "^#" $F_FILE > $T_FILE
+	echo "    -----------------------" >> $T_FILE
+	echo "    |         arch |status|" >> $T_FILE
+	echo "    -----------------------" >> $T_FILE
+	for ARCH_DIR in arch/*/; do
+		ARCH=$(echo $ARCH_DIR | sed -e 's/arch//g' | sed -e 's/\///g')
+		K_FILES=$(find $ARCH_DIR -name "Kconfig*")
+		K_GREP=$(grep "$K" $K_FILES)
+		#
+		# Arch support status values for (O, K) are updated according
+		# to the following rules.
+		#
+		#   - ("", K) is 'supported by a given arch', if there is a
+		#     Kconfig file for that arch which contains K;
+		#
+		#   - ("not", K) is 'supported by a given arch', if there is
+		#     no Kconfig file for that arch which contains K;
+		#
+		#   - otherwise: preserve the previous status value (if any),
+		#                default to 'not yet supported'.
+		#
+		# Notice that, according these rules, invalid features may be
+		# updated/modified.
+		#
+		if [ "$O" = "" ] && [ ! -z "$K_GREP" ]; then
+			printf "    |%12s: |  ok  |\n" "$ARCH" >> $T_FILE
+		elif [ "$O" = "not" ] && [ -z "$K_GREP" ]; then
+			printf "    |%12s: |  ok  |\n" "$ARCH" >> $T_FILE
+		else
+			S=$(grep -v "^#" "$F_FILE" | grep " $ARCH:")
+			if [ ! -z "$S" ]; then
+				echo "$S" >> $T_FILE
+			else
+				printf "    |%12s: | TODO |\n" "$ARCH" \
+					>> $T_FILE
+			fi
+		fi
+	done
+	echo "    -----------------------" >> $T_FILE
+	mv $T_FILE $F_FILE
+done
diff --git a/Documentation/features/seccomp/seccomp-filter/arch-support.txt b/Documentation/features/seccomp/seccomp-filter/arch-support.txt
index e4fad58a05e5..d4271b493b41 100644
--- a/Documentation/features/seccomp/seccomp-filter/arch-support.txt
+++ b/Documentation/features/seccomp/seccomp-filter/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
-    |      parisc: | TODO |
-    |     powerpc: | TODO |
+    |      parisc: |  ok  |
+    |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/Documentation/features/time/arch-tick-broadcast/arch-support.txt b/Documentation/features/time/arch-tick-broadcast/arch-support.txt
index 8052904b25fc..83d9e68462bb 100644
--- a/Documentation/features/time/arch-tick-broadcast/arch-support.txt
+++ b/Documentation/features/time/arch-tick-broadcast/arch-support.txt
@@ -17,12 +17,14 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: | TODO |
-    |          sh: | TODO |
+    |          sh: |  ok  |
     |       sparc: | TODO |
     |          um: | TODO |
     |   unicore32: | TODO |
diff --git a/Documentation/features/time/clockevents/arch-support.txt b/Documentation/features/time/clockevents/arch-support.txt
index 7c76b946297e..3d4908fce6da 100644
--- a/Documentation/features/time/clockevents/arch-support.txt
+++ b/Documentation/features/time/clockevents/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: |  ok  |
     |  microblaze: |  ok  |
     |        mips: |  ok  |
+    |       nds32: |  ok  |
     |       nios2: |  ok  |
     |    openrisc: |  ok  |
-    |      parisc: | TODO |
+    |      parisc: |  ok  |
     |     powerpc: |  ok  |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: |  ok  |
     |       sparc: |  ok  |
diff --git a/Documentation/features/time/context-tracking/arch-support.txt b/Documentation/features/time/context-tracking/arch-support.txt
index 9433b3e523b3..c29974afffaa 100644
--- a/Documentation/features/time/context-tracking/arch-support.txt
+++ b/Documentation/features/time/context-tracking/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: | TODO |
     |       sparc: |  ok  |
diff --git a/Documentation/features/time/irq-time-acct/arch-support.txt b/Documentation/features/time/irq-time-acct/arch-support.txt
index 212dde0b578c..8d73c463ec27 100644
--- a/Documentation/features/time/irq-time-acct/arch-support.txt
+++ b/Documentation/features/time/irq-time-acct/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: |  ..  |
-    |     powerpc: |  ..  |
+    |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ..  |
     |          sh: | TODO |
     |       sparc: |  ..  |
diff --git a/Documentation/features/time/modern-timekeeping/arch-support.txt b/Documentation/features/time/modern-timekeeping/arch-support.txt
index 4074028f72f7..e7c6ea6b8fb3 100644
--- a/Documentation/features/time/modern-timekeeping/arch-support.txt
+++ b/Documentation/features/time/modern-timekeeping/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: |  ok  |
     |        mips: |  ok  |
+    |       nds32: |  ok  |
     |       nios2: |  ok  |
     |    openrisc: |  ok  |
     |      parisc: |  ok  |
     |     powerpc: |  ok  |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: |  ok  |
     |       sparc: |  ok  |
diff --git a/Documentation/features/time/virt-cpuacct/arch-support.txt b/Documentation/features/time/virt-cpuacct/arch-support.txt
index a394d8820517..4646457461cf 100644
--- a/Documentation/features/time/virt-cpuacct/arch-support.txt
+++ b/Documentation/features/time/virt-cpuacct/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: |  ok  |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: |  ok  |
diff --git a/Documentation/features/vm/ELF-ASLR/arch-support.txt b/Documentation/features/vm/ELF-ASLR/arch-support.txt
index 082f93d5b40e..1f71d090ff2c 100644
--- a/Documentation/features/vm/ELF-ASLR/arch-support.txt
+++ b/Documentation/features/vm/ELF-ASLR/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
-    |      parisc: | TODO |
+    |      parisc: |  ok  |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/Documentation/features/vm/PG_uncached/arch-support.txt b/Documentation/features/vm/PG_uncached/arch-support.txt
index 605e0abb756d..fbd5aa463b0a 100644
--- a/Documentation/features/vm/PG_uncached/arch-support.txt
+++ b/Documentation/features/vm/PG_uncached/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: | TODO |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/Documentation/features/vm/THP/arch-support.txt b/Documentation/features/vm/THP/arch-support.txt
index 7a8eb0bd5ca8..5d7ecc378f29 100644
--- a/Documentation/features/vm/THP/arch-support.txt
+++ b/Documentation/features/vm/THP/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: |  ..  |
     |  microblaze: |  ..  |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: |  ..  |
     |    openrisc: |  ..  |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: |  ..  |
     |       sparc: |  ok  |
diff --git a/Documentation/features/vm/TLB/arch-support.txt b/Documentation/features/vm/TLB/arch-support.txt
index 35fb99b2b3ea..f7af9678eb66 100644
--- a/Documentation/features/vm/TLB/arch-support.txt
+++ b/Documentation/features/vm/TLB/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: |  ..  |
     |  microblaze: |  ..  |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: |  ..  |
     |    openrisc: |  ..  |
     |      parisc: | TODO |
     |     powerpc: | TODO |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/Documentation/features/vm/huge-vmap/arch-support.txt b/Documentation/features/vm/huge-vmap/arch-support.txt
index ed8b943ad8fc..d0713ccc7117 100644
--- a/Documentation/features/vm/huge-vmap/arch-support.txt
+++ b/Documentation/features/vm/huge-vmap/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: | TODO |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: | TODO |
     |       sparc: | TODO |
diff --git a/Documentation/features/vm/ioremap_prot/arch-support.txt b/Documentation/features/vm/ioremap_prot/arch-support.txt
index 589947bdf0a8..8527601a3739 100644
--- a/Documentation/features/vm/ioremap_prot/arch-support.txt
+++ b/Documentation/features/vm/ioremap_prot/arch-support.txt
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: | TODO |
     |          sh: |  ok  |
     |       sparc: | TODO |
diff --git a/Documentation/features/vm/numa-memblock/arch-support.txt b/Documentation/features/vm/numa-memblock/arch-support.txt
index 8b8bea0318a0..1a988052cd24 100644
--- a/Documentation/features/vm/numa-memblock/arch-support.txt
+++ b/Documentation/features/vm/numa-memblock/arch-support.txt
@@ -9,7 +9,7 @@
     |       alpha: | TODO |
     |         arc: |  ..  |
     |         arm: |  ..  |
-    |       arm64: |  ..  |
+    |       arm64: |  ok  |
     |         c6x: |  ..  |
     |       h8300: |  ..  |
     |     hexagon: |  ..  |
@@ -17,10 +17,12 @@
     |        m68k: |  ..  |
     |  microblaze: |  ok  |
     |        mips: |  ok  |
+    |       nds32: | TODO |
     |       nios2: |  ..  |
     |    openrisc: |  ..  |
     |      parisc: |  ..  |
     |     powerpc: |  ok  |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: |  ok  |
     |       sparc: |  ok  |
diff --git a/Documentation/features/vm/pte_special/arch-support.txt b/Documentation/features/vm/pte_special/arch-support.txt
index 055004f467d2..a8378424bc98 100644
--- a/Documentation/features/vm/pte_special/arch-support.txt
+++ b/Documentation/features/vm/pte_special/arch-support.txt
@@ -1,6 +1,6 @@
 #
 # Feature name:          pte_special
-#         Kconfig:       __HAVE_ARCH_PTE_SPECIAL
+#         Kconfig:       ARCH_HAS_PTE_SPECIAL
 #         description:   arch supports the pte_special()/pte_mkspecial() VM APIs
 #
     -----------------------
@@ -17,10 +17,12 @@
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
+    |       nds32: | TODO |
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
     |     powerpc: |  ok  |
+    |       riscv: | TODO |
     |        s390: |  ok  |
     |          sh: |  ok  |
     |       sparc: |  ok  |
diff --git a/Documentation/filesystems/00-INDEX b/Documentation/filesystems/00-INDEX
index b7bd6c9009cc..a8bd4af7fbce 100644
--- a/Documentation/filesystems/00-INDEX
+++ b/Documentation/filesystems/00-INDEX
@@ -10,8 +10,8 @@ afs.txt
 	- info and examples for the distributed AFS (Andrew File System) fs.
 affs.txt
 	- info and mount options for the Amiga Fast File System.
-autofs4-mount-control.txt
-	- info on device control operations for autofs4 module.
+autofs-mount-control.txt
+	- info on device control operations for autofs module.
 automount-support.txt
 	- information about filesystem automount support.
 befs.txt
diff --git a/Documentation/filesystems/Locking b/Documentation/filesystems/Locking
index 75d2d57e2c44..92e97a3fbb12 100644
--- a/Documentation/filesystems/Locking
+++ b/Documentation/filesystems/Locking
@@ -21,8 +21,7 @@ prototypes:
 	char *(*d_dname)((struct dentry *dentry, char *buffer, int buflen);
 	struct vfsmount *(*d_automount)(struct path *path);
 	int (*d_manage)(const struct path *, bool);
-	struct dentry *(*d_real)(struct dentry *, const struct inode *,
-				 unsigned int, unsigned int);
+	struct dentry *(*d_real)(struct dentry *, const struct inode *);
 
 locking rules:
 		rename_lock	->d_lock	may block	rcu-walk
@@ -69,31 +68,31 @@ prototypes:
 
 locking rules:
 	all may block
-		i_mutex(inode)
-lookup:		yes
-create:		yes
-link:		yes (both)
-mknod:		yes
-symlink:	yes
-mkdir:		yes
-unlink:		yes (both)
-rmdir:		yes (both)	(see below)
-rename:	yes (all)	(see below)
+		i_rwsem(inode)
+lookup:		shared
+create:		exclusive
+link:		exclusive (both)
+mknod:		exclusive
+symlink:	exclusive
+mkdir:		exclusive
+unlink:		exclusive (both)
+rmdir:		exclusive (both)(see below)
+rename:		exclusive (all)	(see below)
 readlink:	no
 get_link:	no
-setattr:	yes
+setattr:	exclusive
 permission:	no (may not block if called in rcu-walk mode)
 get_acl:	no
 getattr:	no
 listxattr:	no
 fiemap:		no
 update_time:	no
-atomic_open:	yes
+atomic_open:	exclusive
 tmpfile:	no
 
 
-	Additionally, ->rmdir(), ->unlink() and ->rename() have ->i_mutex on
-victim.
+	Additionally, ->rmdir(), ->unlink() and ->rename() have ->i_rwsem
+	exclusive on victim.
 	cross-directory ->rename() has (per-superblock) ->s_vfs_rename_sem.
 
 See Documentation/filesystems/directory-locking for more detailed discussion
@@ -111,10 +110,10 @@ prototypes:
 
 locking rules:
 	all may block
-		i_mutex(inode)
+		i_rwsem(inode)
 list:		no
 get:		no
-set:		yes
+set:		exclusive
 
 --------------------------- super_operations ---------------------------
 prototypes:
@@ -217,14 +216,14 @@ prototypes:
 locking rules:
 	All except set_page_dirty and freepage may block
 
-			PageLocked(page)	i_mutex
+			PageLocked(page)	i_rwsem
 writepage:		yes, unlocks (see below)
 readpage:		yes, unlocks
 writepages:
 set_page_dirty		no
 readpages:
-write_begin:		locks the page		yes
-write_end:		yes, unlocks		yes
+write_begin:		locks the page		exclusive
+write_end:		yes, unlocks		exclusive
 bmap:
 invalidatepage:		yes
 releasepage:		yes
@@ -439,9 +438,13 @@ prototypes:
 	ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
 	ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
 	int (*iterate) (struct file *, struct dir_context *);
-	unsigned int (*poll) (struct file *, struct poll_table_struct *);
+	int (*iterate_shared) (struct file *, struct dir_context *);
+	__poll_t (*poll) (struct file *, struct poll_table_struct *);
+	struct wait_queue_head * (*get_poll_head)(struct file *, __poll_t);
+	__poll_t (*poll_mask) (struct file *, __poll_t);
 	long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
 	long (*compat_ioctl) (struct file *, unsigned int, unsigned long);
+	int (*pre_mmap) (struct file *, unsigned long, unsigned long);
 	int (*mmap) (struct file *, struct vm_area_struct *);
 	int (*open) (struct inode *, struct file *);
 	int (*flush) (struct file *);
@@ -470,7 +473,7 @@ prototypes:
 };
 
 locking rules:
-	All may block.
+	All except for ->poll_mask may block.
 
 ->llseek() locking has moved from llseek to the individual llseek
 implementations.  If your fs is not using generic_file_llseek, you
@@ -480,6 +483,10 @@ mutex or just to use i_size_read() instead.
 Note: this does not protect the file->f_pos against concurrent modifications
 since this is something the userspace has to take care about.
 
+->iterate() is called with i_rwsem exclusive.
+
+->iterate_shared() is called with i_rwsem at least shared.
+
 ->fasync() is responsible for maintaining the FASYNC bit in filp->f_flags.
 Most instances call fasync_helper(), which does that maintenance, so it's
 not normally something one needs to worry about.  Return values > 0 will be
@@ -498,6 +505,9 @@ in sys_read() and friends.
 the lease within the individual filesystem to record the result of the
 operation
 
+->poll_mask can be called with or without the waitqueue lock for the waitqueue
+returned from ->get_poll_head.
+
 --------------------------- dquot_operations -------------------------------
 prototypes:
 	int (*write_dquot) (struct dquot *);
diff --git a/Documentation/filesystems/autofs-mount-control.txt b/Documentation/filesystems/autofs-mount-control.txt
new file mode 100644
index 000000000000..6eba86e1ac72
--- /dev/null
+++ b/Documentation/filesystems/autofs-mount-control.txt
@@ -0,0 +1,407 @@
+
+Miscellaneous Device control operations for the autofs kernel module
+====================================================================
+
+The problem
+===========
+
+There is a problem with active restarts in autofs (that is to say
+restarting autofs when there are busy mounts).
+
+During normal operation autofs uses a file descriptor opened on the
+directory that is being managed in order to be able to issue control
+operations. Using a file descriptor gives ioctl operations access to
+autofs specific information stored in the super block. The operations
+are things such as setting an autofs mount catatonic, setting the
+expire timeout and requesting expire checks. As is explained below,
+certain types of autofs triggered mounts can end up covering an autofs
+mount itself which prevents us being able to use open(2) to obtain a
+file descriptor for these operations if we don't already have one open.
+
+Currently autofs uses "umount -l" (lazy umount) to clear active mounts
+at restart. While using lazy umount works for most cases, anything that
+needs to walk back up the mount tree to construct a path, such as
+getcwd(2) and the proc file system /proc/<pid>/cwd, no longer works
+because the point from which the path is constructed has been detached
+from the mount tree.
+
+The actual problem with autofs is that it can't reconnect to existing
+mounts. Immediately one thinks of just adding the ability to remount
+autofs file systems would solve it, but alas, that can't work. This is
+because autofs direct mounts and the implementation of "on demand mount
+and expire" of nested mount trees have the file system mounted directly
+on top of the mount trigger directory dentry.
+
+For example, there are two types of automount maps, direct (in the kernel
+module source you will see a third type called an offset, which is just
+a direct mount in disguise) and indirect.
+
+Here is a master map with direct and indirect map entries:
+
+/-      /etc/auto.direct
+/test   /etc/auto.indirect
+
+and the corresponding map files:
+
+/etc/auto.direct:
+
+/automount/dparse/g6  budgie:/autofs/export1
+/automount/dparse/g1  shark:/autofs/export1
+and so on.
+
+/etc/auto.indirect:
+
+g1    shark:/autofs/export1
+g6    budgie:/autofs/export1
+and so on.
+
+For the above indirect map an autofs file system is mounted on /test and
+mounts are triggered for each sub-directory key by the inode lookup
+operation. So we see a mount of shark:/autofs/export1 on /test/g1, for
+example.
+
+The way that direct mounts are handled is by making an autofs mount on
+each full path, such as /automount/dparse/g1, and using it as a mount
+trigger. So when we walk on the path we mount shark:/autofs/export1 "on
+top of this mount point". Since these are always directories we can
+use the follow_link inode operation to trigger the mount.
+
+But, each entry in direct and indirect maps can have offsets (making
+them multi-mount map entries).
+
+For example, an indirect mount map entry could also be:
+
+g1  \
+   /        shark:/autofs/export5/testing/test \
+   /s1      shark:/autofs/export/testing/test/s1 \
+   /s2      shark:/autofs/export5/testing/test/s2 \
+   /s1/ss1  shark:/autofs/export1 \
+   /s2/ss2  shark:/autofs/export2
+
+and a similarly a direct mount map entry could also be:
+
+/automount/dparse/g1 \
+    /       shark:/autofs/export5/testing/test \
+    /s1     shark:/autofs/export/testing/test/s1 \
+    /s2     shark:/autofs/export5/testing/test/s2 \
+    /s1/ss1 shark:/autofs/export2 \
+    /s2/ss2 shark:/autofs/export2
+
+One of the issues with version 4 of autofs was that, when mounting an
+entry with a large number of offsets, possibly with nesting, we needed
+to mount and umount all of the offsets as a single unit. Not really a
+problem, except for people with a large number of offsets in map entries.
+This mechanism is used for the well known "hosts" map and we have seen
+cases (in 2.4) where the available number of mounts are exhausted or
+where the number of privileged ports available is exhausted.
+
+In version 5 we mount only as we go down the tree of offsets and
+similarly for expiring them which resolves the above problem. There is
+somewhat more detail to the implementation but it isn't needed for the
+sake of the problem explanation. The one important detail is that these
+offsets are implemented using the same mechanism as the direct mounts
+above and so the mount points can be covered by a mount.
+
+The current autofs implementation uses an ioctl file descriptor opened
+on the mount point for control operations. The references held by the
+descriptor are accounted for in checks made to determine if a mount is
+in use and is also used to access autofs file system information held
+in the mount super block. So the use of a file handle needs to be
+retained.
+
+
+The Solution
+============
+
+To be able to restart autofs leaving existing direct, indirect and
+offset mounts in place we need to be able to obtain a file handle
+for these potentially covered autofs mount points. Rather than just
+implement an isolated operation it was decided to re-implement the
+existing ioctl interface and add new operations to provide this
+functionality.
+
+In addition, to be able to reconstruct a mount tree that has busy mounts,
+the uid and gid of the last user that triggered the mount needs to be
+available because these can be used as macro substitution variables in
+autofs maps. They are recorded at mount request time and an operation
+has been added to retrieve them.
+
+Since we're re-implementing the control interface, a couple of other
+problems with the existing interface have been addressed. First, when
+a mount or expire operation completes a status is returned to the
+kernel by either a "send ready" or a "send fail" operation. The
+"send fail" operation of the ioctl interface could only ever send
+ENOENT so the re-implementation allows user space to send an actual
+status. Another expensive operation in user space, for those using
+very large maps, is discovering if a mount is present. Usually this
+involves scanning /proc/mounts and since it needs to be done quite
+often it can introduce significant overhead when there are many entries
+in the mount table. An operation to lookup the mount status of a mount
+point dentry (covered or not) has also been added.
+
+Current kernel development policy recommends avoiding the use of the
+ioctl mechanism in favor of systems such as Netlink. An implementation
+using this system was attempted to evaluate its suitability and it was
+found to be inadequate, in this case. The Generic Netlink system was
+used for this as raw Netlink would lead to a significant increase in
+complexity. There's no question that the Generic Netlink system is an
+elegant solution for common case ioctl functions but it's not a complete
+replacement probably because its primary purpose in life is to be a
+message bus implementation rather than specifically an ioctl replacement.
+While it would be possible to work around this there is one concern
+that lead to the decision to not use it. This is that the autofs
+expire in the daemon has become far to complex because umount
+candidates are enumerated, almost for no other reason than to "count"
+the number of times to call the expire ioctl. This involves scanning
+the mount table which has proved to be a big overhead for users with
+large maps. The best way to improve this is try and get back to the
+way the expire was done long ago. That is, when an expire request is
+issued for a mount (file handle) we should continually call back to
+the daemon until we can't umount any more mounts, then return the
+appropriate status to the daemon. At the moment we just expire one
+mount at a time. A Generic Netlink implementation would exclude this
+possibility for future development due to the requirements of the
+message bus architecture.
+
+
+autofs Miscellaneous Device mount control interface
+====================================================
+
+The control interface is opening a device node, typically /dev/autofs.
+
+All the ioctls use a common structure to pass the needed parameter
+information and return operation results:
+
+struct autofs_dev_ioctl {
+	__u32 ver_major;
+	__u32 ver_minor;
+	__u32 size;             /* total size of data passed in
+				 * including this struct */
+	__s32 ioctlfd;          /* automount command fd */
+
+	/* Command parameters */
+	union {
+		struct args_protover		protover;
+		struct args_protosubver		protosubver;
+		struct args_openmount		openmount;
+		struct args_ready		ready;
+		struct args_fail		fail;
+		struct args_setpipefd		setpipefd;
+		struct args_timeout		timeout;
+		struct args_requester		requester;
+		struct args_expire		expire;
+		struct args_askumount		askumount;
+		struct args_ismountpoint	ismountpoint;
+	};
+
+	char path[0];
+};
+
+The ioctlfd field is a mount point file descriptor of an autofs mount
+point. It is returned by the open call and is used by all calls except
+the check for whether a given path is a mount point, where it may
+optionally be used to check a specific mount corresponding to a given
+mount point file descriptor, and when requesting the uid and gid of the
+last successful mount on a directory within the autofs file system.
+
+The union is used to communicate parameters and results of calls made
+as described below.
+
+The path field is used to pass a path where it is needed and the size field
+is used account for the increased structure length when translating the
+structure sent from user space.
+
+This structure can be initialized before setting specific fields by using
+the void function call init_autofs_dev_ioctl(struct autofs_dev_ioctl *).
+
+All of the ioctls perform a copy of this structure from user space to
+kernel space and return -EINVAL if the size parameter is smaller than
+the structure size itself, -ENOMEM if the kernel memory allocation fails
+or -EFAULT if the copy itself fails. Other checks include a version check
+of the compiled in user space version against the module version and a
+mismatch results in a -EINVAL return. If the size field is greater than
+the structure size then a path is assumed to be present and is checked to
+ensure it begins with a "/" and is NULL terminated, otherwise -EINVAL is
+returned. Following these checks, for all ioctl commands except
+AUTOFS_DEV_IOCTL_VERSION_CMD, AUTOFS_DEV_IOCTL_OPENMOUNT_CMD and
+AUTOFS_DEV_IOCTL_CLOSEMOUNT_CMD the ioctlfd is validated and if it is
+not a valid descriptor or doesn't correspond to an autofs mount point
+an error of -EBADF, -ENOTTY or -EINVAL (not an autofs descriptor) is
+returned.
+
+
+The ioctls
+==========
+
+An example of an implementation which uses this interface can be seen
+in autofs version 5.0.4 and later in file lib/dev-ioctl-lib.c of the
+distribution tar available for download from kernel.org in directory
+/pub/linux/daemons/autofs/v5.
+
+The device node ioctl operations implemented by this interface are:
+
+
+AUTOFS_DEV_IOCTL_VERSION
+------------------------
+
+Get the major and minor version of the autofs device ioctl kernel module
+implementation. It requires an initialized struct autofs_dev_ioctl as an
+input parameter and sets the version information in the passed in structure.
+It returns 0 on success or the error -EINVAL if a version mismatch is
+detected.
+
+
+AUTOFS_DEV_IOCTL_PROTOVER_CMD and AUTOFS_DEV_IOCTL_PROTOSUBVER_CMD
+------------------------------------------------------------------
+
+Get the major and minor version of the autofs protocol version understood
+by loaded module. This call requires an initialized struct autofs_dev_ioctl
+with the ioctlfd field set to a valid autofs mount point descriptor
+and sets the requested version number in version field of struct args_protover
+or sub_version field of struct args_protosubver. These commands return
+0 on success or one of the negative error codes if validation fails.
+
+
+AUTOFS_DEV_IOCTL_OPENMOUNT and AUTOFS_DEV_IOCTL_CLOSEMOUNT
+----------------------------------------------------------
+
+Obtain and release a file descriptor for an autofs managed mount point
+path. The open call requires an initialized struct autofs_dev_ioctl with
+the path field set and the size field adjusted appropriately as well
+as the devid field of struct args_openmount set to the device number of
+the autofs mount. The device number can be obtained from the mount options
+shown in /proc/mounts. The close call requires an initialized struct
+autofs_dev_ioct with the ioctlfd field set to the descriptor obtained
+from the open call. The release of the file descriptor can also be done
+with close(2) so any open descriptors will also be closed at process exit.
+The close call is included in the implemented operations largely for
+completeness and to provide for a consistent user space implementation.
+
+
+AUTOFS_DEV_IOCTL_READY_CMD and AUTOFS_DEV_IOCTL_FAIL_CMD
+--------------------------------------------------------
+
+Return mount and expire result status from user space to the kernel.
+Both of these calls require an initialized struct autofs_dev_ioctl
+with the ioctlfd field set to the descriptor obtained from the open
+call and the token field of struct args_ready or struct args_fail set
+to the wait queue token number, received by user space in the foregoing
+mount or expire request. The status field of struct args_fail is set to
+the errno of the operation. It is set to 0 on success.
+
+
+AUTOFS_DEV_IOCTL_SETPIPEFD_CMD
+------------------------------
+
+Set the pipe file descriptor used for kernel communication to the daemon.
+Normally this is set at mount time using an option but when reconnecting
+to a existing mount we need to use this to tell the autofs mount about
+the new kernel pipe descriptor. In order to protect mounts against
+incorrectly setting the pipe descriptor we also require that the autofs
+mount be catatonic (see next call).
+
+The call requires an initialized struct autofs_dev_ioctl with the
+ioctlfd field set to the descriptor obtained from the open call and
+the pipefd field of struct args_setpipefd set to descriptor of the pipe.
+On success the call also sets the process group id used to identify the
+controlling process (eg. the owning automount(8) daemon) to the process
+group of the caller.
+
+
+AUTOFS_DEV_IOCTL_CATATONIC_CMD
+------------------------------
+
+Make the autofs mount point catatonic. The autofs mount will no longer
+issue mount requests, the kernel communication pipe descriptor is released
+and any remaining waits in the queue released.
+
+The call requires an initialized struct autofs_dev_ioctl with the
+ioctlfd field set to the descriptor obtained from the open call.
+
+
+AUTOFS_DEV_IOCTL_TIMEOUT_CMD
+----------------------------
+
+Set the expire timeout for mounts within an autofs mount point.
+
+The call requires an initialized struct autofs_dev_ioctl with the
+ioctlfd field set to the descriptor obtained from the open call.
+
+
+AUTOFS_DEV_IOCTL_REQUESTER_CMD
+------------------------------
+
+Return the uid and gid of the last process to successfully trigger a the
+mount on the given path dentry.
+
+The call requires an initialized struct autofs_dev_ioctl with the path
+field set to the mount point in question and the size field adjusted
+appropriately. Upon return the uid field of struct args_requester contains
+the uid and gid field the gid.
+
+When reconstructing an autofs mount tree with active mounts we need to
+re-connect to mounts that may have used the original process uid and
+gid (or string variations of them) for mount lookups within the map entry.
+This call provides the ability to obtain this uid and gid so they may be
+used by user space for the mount map lookups.
+
+
+AUTOFS_DEV_IOCTL_EXPIRE_CMD
+---------------------------
+
+Issue an expire request to the kernel for an autofs mount. Typically
+this ioctl is called until no further expire candidates are found.
+
+The call requires an initialized struct autofs_dev_ioctl with the
+ioctlfd field set to the descriptor obtained from the open call. In
+addition an immediate expire, independent of the mount timeout, can be
+requested by setting the how field of struct args_expire to 1. If no
+expire candidates can be found the ioctl returns -1 with errno set to
+EAGAIN.
+
+This call causes the kernel module to check the mount corresponding
+to the given ioctlfd for mounts that can be expired, issues an expire
+request back to the daemon and waits for completion.
+
+AUTOFS_DEV_IOCTL_ASKUMOUNT_CMD
+------------------------------
+
+Checks if an autofs mount point is in use.
+
+The call requires an initialized struct autofs_dev_ioctl with the
+ioctlfd field set to the descriptor obtained from the open call and
+it returns the result in the may_umount field of struct args_askumount,
+1 for busy and 0 otherwise.
+
+
+AUTOFS_DEV_IOCTL_ISMOUNTPOINT_CMD
+---------------------------------
+
+Check if the given path is a mountpoint.
+
+The call requires an initialized struct autofs_dev_ioctl. There are two
+possible variations. Both use the path field set to the path of the mount
+point to check and the size field adjusted appropriately. One uses the
+ioctlfd field to identify a specific mount point to check while the other
+variation uses the path and optionally in.type field of struct args_ismountpoint
+set to an autofs mount type. The call returns 1 if this is a mount point
+and sets out.devid field to the device number of the mount and out.magic
+field to the relevant super block magic number (described below) or 0 if
+it isn't a mountpoint. In both cases the the device number (as returned
+by new_encode_dev()) is returned in out.devid field.
+
+If supplied with a file descriptor we're looking for a specific mount,
+not necessarily at the top of the mounted stack. In this case the path
+the descriptor corresponds to is considered a mountpoint if it is itself
+a mountpoint or contains a mount, such as a multi-mount without a root
+mount. In this case we return 1 if the descriptor corresponds to a mount
+point and and also returns the super magic of the covering mount if there
+is one or 0 if it isn't a mountpoint.
+
+If a path is supplied (and the ioctlfd field is set to -1) then the path
+is looked up and is checked to see if it is the root of a mount. If a
+type is also given we are looking for a particular autofs mount and if
+a match isn't found a fail is returned. If the the located path is the
+root of a mount 1 is returned along with the super magic of the mount
+or 0 otherwise.
+
diff --git a/Documentation/filesystems/autofs.txt b/Documentation/filesystems/autofs.txt
new file mode 100644
index 000000000000..373ad25852d3
--- /dev/null
+++ b/Documentation/filesystems/autofs.txt
@@ -0,0 +1,529 @@
+<head>
+<style> p { max-width:50em} ol, ul {max-width: 40em}</style>
+</head>
+
+autofs - how it works
+=====================
+
+Purpose
+-------
+
+The goal of autofs is to provide on-demand mounting and race free
+automatic unmounting of various other filesystems.  This provides two
+key advantages:
+
+1. There is no need to delay boot until all filesystems that
+   might be needed are mounted.  Processes that try to access those
+   slow filesystems might be delayed but other processes can
+   continue freely.  This is particularly important for
+   network filesystems (e.g. NFS) or filesystems stored on
+   media with a media-changing robot.
+
+2. The names and locations of filesystems can be stored in
+   a remote database and can change at any time.  The content
+   in that data base at the time of access will be used to provide
+   a target for the access.  The interpretation of names in the
+   filesystem can even be programmatic rather than database-backed,
+   allowing wildcards for example, and can vary based on the user who
+   first accessed a name.
+
+Context
+-------
+
+The "autofs" filesystem module is only one part of an autofs system.
+There also needs to be a user-space program which looks up names
+and mounts filesystems.  This will often be the "automount" program,
+though other tools including "systemd" can make use of "autofs".
+This document describes only the kernel module and the interactions
+required with any user-space program.  Subsequent text refers to this
+as the "automount daemon" or simply "the daemon".
+
+"autofs" is a Linux kernel module with provides the "autofs"
+filesystem type.  Several "autofs" filesystems can be mounted and they
+can each be managed separately, or all managed by the same daemon.
+
+Content
+-------
+
+An autofs filesystem can contain 3 sorts of objects: directories,
+symbolic links and mount traps.  Mount traps are directories with
+extra properties as described in the next section.
+
+Objects can only be created by the automount daemon: symlinks are
+created with a regular `symlink` system call, while directories and
+mount traps are created with `mkdir`.  The determination of whether a
+directory should be a mount trap or not is quite _ad hoc_, largely for
+historical reasons, and is determined in part by the
+*direct*/*indirect*/*offset* mount options, and the *maxproto* mount option.
+
+If neither the *direct* or *offset* mount options are given (so the
+mount is considered to be *indirect*), then the root directory is
+always a regular directory, otherwise it is a mount trap when it is
+empty and a regular directory when not empty.  Note that *direct* and
+*offset* are treated identically so a concise summary is that the root
+directory is a mount trap only if the filesystem is mounted *direct*
+and the root is empty.
+
+Directories created in the root directory are mount traps only if the
+filesystem is mounted *indirect* and they are empty.
+
+Directories further down the tree depend on the *maxproto* mount
+option and particularly whether it is less than five or not.
+When *maxproto* is five, no directories further down the
+tree are ever mount traps, they are always regular directories.  When
+the *maxproto* is four (or three), these directories are mount traps
+precisely when they are empty.
+
+So: non-empty (i.e. non-leaf) directories are never mount traps. Empty
+directories are sometimes mount traps, and sometimes not depending on
+where in the tree they are (root, top level, or lower), the *maxproto*,
+and whether the mount was *indirect* or not.
+
+Mount Traps
+---------------
+
+A core element of the implementation of autofs is the Mount Traps
+which are provided by the Linux VFS.  Any directory provided by a
+filesystem can be designated as a trap.  This involves two separate
+features that work together to allow autofs to do its job.
+
+**DCACHE_NEED_AUTOMOUNT**
+
+If a dentry has the DCACHE_NEED_AUTOMOUNT flag set (which gets set if
+the inode has S_AUTOMOUNT set, or can be set directly) then it is
+(potentially) a mount trap.  Any access to this directory beyond a
+"`stat`" will (normally) cause the `d_op->d_automount()` dentry operation
+to be called. The task of this method is to find the filesystem that
+should be mounted on the directory and to return it.  The VFS is
+responsible for actually mounting the root of this filesystem on the
+directory.
+
+autofs doesn't find the filesystem itself but sends a message to the
+automount daemon asking it to find and mount the filesystem.  The
+autofs `d_automount` method then waits for the daemon to report that
+everything is ready.  It will then return "`NULL`" indicating that the
+mount has already happened.  The VFS doesn't try to mount anything but
+follows down the mount that is already there.
+
+This functionality is sufficient for some users of mount traps such
+as NFS which creates traps so that mountpoints on the server can be
+reflected on the client.  However it is not sufficient for autofs.  As
+mounting onto a directory is considered to be "beyond a `stat`", the
+automount daemon would not be able to mount a filesystem on the 'trap'
+directory without some way to avoid getting caught in the trap.  For
+that purpose there is another flag.
+
+**DCACHE_MANAGE_TRANSIT**
+
+If a dentry has DCACHE_MANAGE_TRANSIT set then two very different but
+related behaviors are invoked, both using the `d_op->d_manage()`
+dentry operation.
+
+Firstly, before checking to see if any filesystem is mounted on the
+directory, d_manage() will be called with the `rcu_walk` parameter set
+to `false`.  It may return one of three things:
+
+-  A return value of zero indicates that there is nothing special
+   about this dentry and normal checks for mounts and automounts
+   should proceed.
+
+   autofs normally returns zero, but first waits for any
+   expiry (automatic unmounting of the mounted filesystem) to
+   complete.  This avoids races.
+
+-  A return value of `-EISDIR` tells the VFS to ignore any mounts
+   on the directory and to not consider calling `->d_automount()`.
+   This effectively disables the **DCACHE_NEED_AUTOMOUNT** flag
+   causing the directory not be a mount trap after all.
+
+   autofs returns this if it detects that the process performing the
+   lookup is the automount daemon and that the mount has been
+   requested but has not yet completed.  How it determines this is
+   discussed later.  This allows the automount daemon not to get
+   caught in the mount trap.
+
+   There is a subtlety here.  It is possible that a second autofs
+   filesystem can be mounted below the first and for both of them to
+   be managed by the same daemon.  For the daemon to be able to mount
+   something on the second it must be able to "walk" down past the
+   first.  This means that d_manage cannot *always* return -EISDIR for
+   the automount daemon.  It must only return it when a mount has
+   been requested, but has not yet completed.
+
+   `d_manage` also returns `-EISDIR` if the dentry shouldn't be a
+   mount trap, either because it is a symbolic link or because it is
+   not empty.
+
+-  Any other negative value is treated as an error and returned
+   to the caller.
+
+   autofs can return
+
+   - -ENOENT if the automount daemon failed to mount anything,
+   - -ENOMEM if it ran out of memory,
+   - -EINTR if a signal arrived while waiting for expiry to
+     complete
+   - or any other error sent down by the automount daemon.
+
+
+The second use case only occurs during an "RCU-walk" and so `rcu_walk`
+will be set.
+
+An RCU-walk is a fast and lightweight process for walking down a
+filename path (i.e. it is like running on tip-toes).  RCU-walk cannot
+cope with all situations so when it finds a difficulty it falls back
+to "REF-walk", which is slower but more robust.
+
+RCU-walk will never call `->d_automount`; the filesystems must already
+be mounted or RCU-walk cannot handle the path.
+To determine if a mount-trap is safe for RCU-walk mode it calls
+`->d_manage()` with `rcu_walk` set to `true`.
+
+In this case `d_manage()` must avoid blocking and should avoid taking
+spinlocks if at all possible.  Its sole purpose is to determine if it
+would be safe to follow down into any mounted directory and the only
+reason that it might not be is if an expiry of the mount is
+underway.
+
+In the `rcu_walk` case, `d_manage()` cannot return -EISDIR to tell the
+VFS that this is a directory that doesn't require d_automount.  If
+`rcu_walk` sees a dentry with DCACHE_NEED_AUTOMOUNT set but nothing
+mounted, it *will* fall back to REF-walk.  `d_manage()` cannot make the
+VFS remain in RCU-walk mode, but can only tell it to get out of
+RCU-walk mode by returning `-ECHILD`.
+
+So `d_manage()`, when called with `rcu_walk` set, should either return
+-ECHILD if there is any reason to believe it is unsafe to end the
+mounted filesystem, and otherwise should return 0.
+
+autofs will return `-ECHILD` if an expiry of the filesystem has been
+initiated or is being considered, otherwise it returns 0.
+
+
+Mountpoint expiry
+-----------------
+
+The VFS has a mechanism for automatically expiring unused mounts,
+much as it can expire any unused dentry information from the dcache.
+This is guided by the MNT_SHRINKABLE flag.  This only applies to
+mounts that were created by `d_automount()` returning a filesystem to be
+mounted.  As autofs doesn't return such a filesystem but leaves the
+mounting to the automount daemon, it must involve the automount daemon
+in unmounting as well.  This also means that autofs has more control
+of expiry.
+
+The VFS also supports "expiry" of mounts using the MNT_EXPIRE flag to
+the `umount` system call.  Unmounting with MNT_EXPIRE will fail unless
+a previous attempt had been made, and the filesystem has been inactive
+and untouched since that previous attempt.  autofs does not depend on
+this but has its own internal tracking of whether filesystems were
+recently used.  This allows individual names in the autofs directory
+to expire separately.
+
+With version 4 of the protocol, the automount daemon can try to
+unmount any filesystems mounted on the autofs filesystem or remove any
+symbolic links or empty directories any time it likes.  If the unmount
+or removal is successful the filesystem will be returned to the state
+it was before the mount or creation, so that any access of the name
+will trigger normal auto-mount processing.  In particlar, `rmdir` and
+`unlink` do not leave negative entries in the dcache as a normal
+filesystem would, so an attempt to access a recently-removed object is
+passed to autofs for handling.
+
+With version 5, this is not safe except for unmounting from top-level
+directories.  As lower-level directories are never mount traps, other
+processes will see an empty directory as soon as the filesystem is
+unmounted.  So it is generally safest to use the autofs expiry
+protocol described below.
+
+Normally the daemon only wants to remove entries which haven't been
+used for a while.  For this purpose autofs maintains a "`last_used`"
+time stamp on each directory or symlink.  For symlinks it genuinely
+does record the last time the symlink was "used" or followed to find
+out where it points to.  For directories the field is a slight
+misnomer.  It actually records the last time that autofs checked if
+the directory or one of its descendents was busy and found that it
+was.  This is just as useful and doesn't require updating the field so
+often.
+
+The daemon is able to ask autofs if anything is due to be expired,
+using an `ioctl` as discussed later.  For a *direct* mount, autofs
+considers if the entire mount-tree can be unmounted or not.  For an
+*indirect* mount, autofs considers each of the names in the top level
+directory to determine if any of those can be unmounted and cleaned
+up.
+
+There is an option with indirect mounts to consider each of the leaves
+that has been mounted on instead of considering the top-level names.
+This is intended for compatability with version 4 of autofs and should
+be considered as deprecated.
+
+When autofs considers a directory it checks the `last_used` time and
+compares it with the "timeout" value set when the filesystem was
+mounted, though this check is ignored in some cases. It also checks if
+the directory or anything below it is in use.  For symbolic links,
+only the `last_used` time is ever considered.
+
+If both appear to support expiring the directory or symlink, an action
+is taken.
+
+There are two ways to ask autofs to consider expiry.  The first is to
+use the **AUTOFS_IOC_EXPIRE** ioctl.  This only works for indirect
+mounts.  If it finds something in the root directory to expire it will
+return the name of that thing.  Once a name has been returned the
+automount daemon needs to unmount any filesystems mounted below the
+name normally.  As described above, this is unsafe for non-toplevel
+mounts in a version-5 autofs.  For this reason the current `automountd`
+does not use this ioctl.
+
+The second mechanism uses either the **AUTOFS_DEV_IOCTL_EXPIRE_CMD** or
+the **AUTOFS_IOC_EXPIRE_MULTI** ioctl.  This will work for both direct and
+indirect mounts.  If it selects an object to expire, it will notify
+the daemon using the notification mechanism described below.  This
+will block until the daemon acknowledges the expiry notification.
+This implies that the "`EXPIRE`" ioctl must be sent from a different
+thread than the one which handles notification.
+
+While the ioctl is blocking, the entry is marked as "expiring" and
+`d_manage` will block until the daemon affirms that the unmount has
+completed (together with removing any directories that might have been
+necessary), or has been aborted.
+
+Communicating with autofs: detecting the daemon
+-----------------------------------------------
+
+There are several forms of communication between the automount daemon
+and the filesystem.  As we have already seen, the daemon can create and
+remove directories and symlinks using normal filesystem operations.
+autofs knows whether a process requesting some operation is the daemon
+or not based on its process-group id number (see getpgid(1)).
+
+When an autofs filesystem is mounted the pgid of the mounting
+processes is recorded unless the "pgrp=" option is given, in which
+case that number is recorded instead.  Any request arriving from a
+process in that process group is considered to come from the daemon.
+If the daemon ever has to be stopped and restarted a new pgid can be
+provided through an ioctl as will be described below.
+
+Communicating with autofs: the event pipe
+-----------------------------------------
+
+When an autofs filesystem is mounted, the 'write' end of a pipe must
+be passed using the 'fd=' mount option.  autofs will write
+notification messages to this pipe for the daemon to respond to.
+For version 5, the format of the message is:
+
+        struct autofs_v5_packet {
+                int proto_version;                /* Protocol version */
+                int type;                        /* Type of packet */
+                autofs_wqt_t wait_queue_token;
+                __u32 dev;
+                __u64 ino;
+                __u32 uid;
+                __u32 gid;
+                __u32 pid;
+                __u32 tgid;
+                __u32 len;
+                char name[NAME_MAX+1];
+        };
+
+where the type is one of
+
+        autofs_ptype_missing_indirect
+        autofs_ptype_expire_indirect
+        autofs_ptype_missing_direct
+        autofs_ptype_expire_direct
+
+so messages can indicate that a name is missing (something tried to
+access it but it isn't there) or that it has been selected for expiry.
+
+The pipe will be set to "packet mode" (equivalent to passing
+`O_DIRECT`) to _pipe2(2)_ so that a read from the pipe will return at
+most one packet, and any unread portion of a packet will be discarded.
+
+The `wait_queue_token` is a unique number which can identify a
+particular request to be acknowledged.  When a message is sent over
+the pipe the affected dentry is marked as either "active" or
+"expiring" and other accesses to it block until the message is
+acknowledged using one of the ioctls below and the relevant
+`wait_queue_token`.
+
+Communicating with autofs: root directory ioctls
+------------------------------------------------
+
+The root directory of an autofs filesystem will respond to a number of
+ioctls.  The process issuing the ioctl must have the CAP_SYS_ADMIN
+capability, or must be the automount daemon.
+
+The available ioctl commands are:
+
+- **AUTOFS_IOC_READY**: a notification has been handled.  The argument
+    to the ioctl command is the "wait_queue_token" number
+    corresponding to the notification being acknowledged.
+- **AUTOFS_IOC_FAIL**: similar to above, but indicates failure with
+    the error code `ENOENT`.
+- **AUTOFS_IOC_CATATONIC**: Causes the autofs to enter "catatonic"
+    mode meaning that it stops sending notifications to the daemon.
+    This mode is also entered if a write to the pipe fails.
+- **AUTOFS_IOC_PROTOVER**:  This returns the protocol version in use.
+- **AUTOFS_IOC_PROTOSUBVER**: Returns the protocol sub-version which
+    is really a version number for the implementation.  It is
+    currently 2.
+- **AUTOFS_IOC_SETTIMEOUT**:  This passes a pointer to an unsigned
+    long.  The value is used to set the timeout for expiry, and
+    the current timeout value is stored back through the pointer.
+- **AUTOFS_IOC_ASKUMOUNT**:  Returns, in the pointed-to `int`, 1 if
+    the filesystem could be unmounted.  This is only a hint as
+    the situation could change at any instant.  This call can be
+    use to avoid a more expensive full unmount attempt.
+- **AUTOFS_IOC_EXPIRE**: as described above, this asks if there is
+    anything suitable to expire.  A pointer to a packet:
+
+        struct autofs_packet_expire_multi {
+                int proto_version;              /* Protocol version */
+                int type;                       /* Type of packet */
+                autofs_wqt_t wait_queue_token;
+                int len;
+                char name[NAME_MAX+1];
+        };
+
+     is required.  This is filled in with the name of something
+     that can be unmounted or removed.  If nothing can be expired,
+     `errno` is set to `EAGAIN`.  Even though a `wait_queue_token`
+     is present in the structure, no "wait queue" is established
+     and no acknowledgment is needed.
+- **AUTOFS_IOC_EXPIRE_MULTI**:  This is similar to
+     **AUTOFS_IOC_EXPIRE** except that it causes notification to be
+     sent to the daemon, and it blocks until the daemon acknowledges.
+     The argument is an integer which can contain two different flags.
+
+     **AUTOFS_EXP_IMMEDIATE** causes `last_used` time to be ignored
+     and objects are expired if the are not in use.
+
+     **AUTOFS_EXP_LEAVES** will select a leaf rather than a top-level
+     name to expire.  This is only safe when *maxproto* is 4.
+
+Communicating with autofs: char-device ioctls
+---------------------------------------------
+
+It is not always possible to open the root of an autofs filesystem,
+particularly a *direct* mounted filesystem.  If the automount daemon
+is restarted there is no way for it to regain control of existing
+mounts using any of the above communication channels.  To address this
+need there is a "miscellaneous" character device (major 10, minor 235)
+which can be used to communicate directly with the autofs filesystem.
+It requires CAP_SYS_ADMIN for access.
+
+The `ioctl`s that can be used on this device are described in a separate
+document `autofs-mount-control.txt`, and are summarized briefly here.
+Each ioctl is passed a pointer to an `autofs_dev_ioctl` structure:
+
+        struct autofs_dev_ioctl {
+                __u32 ver_major;
+                __u32 ver_minor;
+                __u32 size;             /* total size of data passed in
+                                         * including this struct */
+                __s32 ioctlfd;          /* automount command fd */
+
+		/* Command parameters */
+		union {
+			struct args_protover		protover;
+			struct args_protosubver		protosubver;
+			struct args_openmount		openmount;
+			struct args_ready		ready;
+			struct args_fail		fail;
+			struct args_setpipefd		setpipefd;
+			struct args_timeout		timeout;
+			struct args_requester		requester;
+			struct args_expire		expire;
+			struct args_askumount		askumount;
+			struct args_ismountpoint	ismountpoint;
+		};
+
+                char path[0];
+        };
+
+For the **OPEN_MOUNT** and **IS_MOUNTPOINT** commands, the target
+filesystem is identified by the `path`.  All other commands identify
+the filesystem by the `ioctlfd` which is a file descriptor open on the
+root, and which can be returned by **OPEN_MOUNT**.
+
+The `ver_major` and `ver_minor` are in/out parameters which check that
+the requested version is supported, and report the maximum version
+that the kernel module can support.
+
+Commands are:
+
+- **AUTOFS_DEV_IOCTL_VERSION_CMD**: does nothing, except validate and
+    set version numbers.
+- **AUTOFS_DEV_IOCTL_OPENMOUNT_CMD**: return an open file descriptor
+    on the root of an autofs filesystem.  The filesystem is identified
+    by name and device number, which is stored in `openmount.devid`.
+    Device numbers for existing filesystems can be found in
+    `/proc/self/mountinfo`.
+- **AUTOFS_DEV_IOCTL_CLOSEMOUNT_CMD**: same as `close(ioctlfd)`.
+- **AUTOFS_DEV_IOCTL_SETPIPEFD_CMD**: if the filesystem is in
+    catatonic mode, this can provide the write end of a new pipe
+    in `setpipefd.pipefd` to re-establish communication with a daemon.
+    The process group of the calling process is used to identify the
+    daemon.
+- **AUTOFS_DEV_IOCTL_REQUESTER_CMD**: `path` should be a
+    name within the filesystem that has been auto-mounted on.
+    On successful return, `requester.uid` and `requester.gid` will be
+    the UID and GID of the process which triggered that mount.
+- **AUTOFS_DEV_IOCTL_ISMOUNTPOINT_CMD**: Check if path is a
+    mountpoint of a particular type - see separate documentation for
+    details.
+- **AUTOFS_DEV_IOCTL_PROTOVER_CMD**:
+- **AUTOFS_DEV_IOCTL_PROTOSUBVER_CMD**:
+- **AUTOFS_DEV_IOCTL_READY_CMD**:
+- **AUTOFS_DEV_IOCTL_FAIL_CMD**:
+- **AUTOFS_DEV_IOCTL_CATATONIC_CMD**:
+- **AUTOFS_DEV_IOCTL_TIMEOUT_CMD**:
+- **AUTOFS_DEV_IOCTL_EXPIRE_CMD**:
+- **AUTOFS_DEV_IOCTL_ASKUMOUNT_CMD**:  These all have the same
+    function as the similarly named **AUTOFS_IOC** ioctls, except
+    that **FAIL** can be given an explicit error number in `fail.status`
+    instead of assuming `ENOENT`, and this **EXPIRE** command
+    corresponds to **AUTOFS_IOC_EXPIRE_MULTI**.
+
+Catatonic mode
+--------------
+
+As mentioned, an autofs mount can enter "catatonic" mode.  This
+happens if a write to the notification pipe fails, or if it is
+explicitly requested by an `ioctl`.
+
+When entering catatonic mode, the pipe is closed and any pending
+notifications are acknowledged with the error `ENOENT`.
+
+Once in catatonic mode attempts to access non-existing names will
+result in `ENOENT` while attempts to access existing directories will
+be treated in the same way as if they came from the daemon, so mount
+traps will not fire.
+
+When the filesystem is mounted a _uid_ and _gid_ can be given which
+set the ownership of directories and symbolic links.  When the
+filesystem is in catatonic mode, any process with a matching UID can
+create directories or symlinks in the root directory, but not in other
+directories.
+
+Catatonic mode can only be left via the
+**AUTOFS_DEV_IOCTL_OPENMOUNT_CMD** ioctl on the `/dev/autofs`.
+
+autofs, name spaces, and shared mounts
+--------------------------------------
+
+With bind mounts and name spaces it is possible for an autofs
+filesystem to appear at multiple places in one or more filesystem
+name spaces.  For this to work sensibly, the autofs filesystem should
+always be mounted "shared". e.g.
+
+> `mount --make-shared /autofs/mount/point`
+
+The automount daemon is only able to manage a single mount location for
+an autofs filesystem and if mounts on that are not 'shared', other
+locations will not behave as expected.  In particular access to those
+other locations will likely result in the `ELOOP` error
+
+> Too many levels of symbolic links
diff --git a/Documentation/filesystems/autofs4-mount-control.txt b/Documentation/filesystems/autofs4-mount-control.txt
deleted file mode 100644
index e5177cb31a04..000000000000
--- a/Documentation/filesystems/autofs4-mount-control.txt
+++ /dev/null
@@ -1,407 +0,0 @@
-
-Miscellaneous Device control operations for the autofs4 kernel module
-====================================================================
-
-The problem
-===========
-
-There is a problem with active restarts in autofs (that is to say
-restarting autofs when there are busy mounts).
-
-During normal operation autofs uses a file descriptor opened on the
-directory that is being managed in order to be able to issue control
-operations. Using a file descriptor gives ioctl operations access to
-autofs specific information stored in the super block. The operations
-are things such as setting an autofs mount catatonic, setting the
-expire timeout and requesting expire checks. As is explained below,
-certain types of autofs triggered mounts can end up covering an autofs
-mount itself which prevents us being able to use open(2) to obtain a
-file descriptor for these operations if we don't already have one open.
-
-Currently autofs uses "umount -l" (lazy umount) to clear active mounts
-at restart. While using lazy umount works for most cases, anything that
-needs to walk back up the mount tree to construct a path, such as
-getcwd(2) and the proc file system /proc/<pid>/cwd, no longer works
-because the point from which the path is constructed has been detached
-from the mount tree.
-
-The actual problem with autofs is that it can't reconnect to existing
-mounts. Immediately one thinks of just adding the ability to remount
-autofs file systems would solve it, but alas, that can't work. This is
-because autofs direct mounts and the implementation of "on demand mount
-and expire" of nested mount trees have the file system mounted directly
-on top of the mount trigger directory dentry.
-
-For example, there are two types of automount maps, direct (in the kernel
-module source you will see a third type called an offset, which is just
-a direct mount in disguise) and indirect.
-
-Here is a master map with direct and indirect map entries:
-
-/-      /etc/auto.direct
-/test   /etc/auto.indirect
-
-and the corresponding map files:
-
-/etc/auto.direct:
-
-/automount/dparse/g6  budgie:/autofs/export1
-/automount/dparse/g1  shark:/autofs/export1
-and so on.
-
-/etc/auto.indirect:
-
-g1    shark:/autofs/export1
-g6    budgie:/autofs/export1
-and so on.
-
-For the above indirect map an autofs file system is mounted on /test and
-mounts are triggered for each sub-directory key by the inode lookup
-operation. So we see a mount of shark:/autofs/export1 on /test/g1, for
-example.
-
-The way that direct mounts are handled is by making an autofs mount on
-each full path, such as /automount/dparse/g1, and using it as a mount
-trigger. So when we walk on the path we mount shark:/autofs/export1 "on
-top of this mount point". Since these are always directories we can
-use the follow_link inode operation to trigger the mount.
-
-But, each entry in direct and indirect maps can have offsets (making
-them multi-mount map entries).
-
-For example, an indirect mount map entry could also be:
-
-g1  \
-   /        shark:/autofs/export5/testing/test \
-   /s1      shark:/autofs/export/testing/test/s1 \
-   /s2      shark:/autofs/export5/testing/test/s2 \
-   /s1/ss1  shark:/autofs/export1 \
-   /s2/ss2  shark:/autofs/export2
-
-and a similarly a direct mount map entry could also be:
-
-/automount/dparse/g1 \
-    /       shark:/autofs/export5/testing/test \
-    /s1     shark:/autofs/export/testing/test/s1 \
-    /s2     shark:/autofs/export5/testing/test/s2 \
-    /s1/ss1 shark:/autofs/export2 \
-    /s2/ss2 shark:/autofs/export2
-
-One of the issues with version 4 of autofs was that, when mounting an
-entry with a large number of offsets, possibly with nesting, we needed
-to mount and umount all of the offsets as a single unit. Not really a
-problem, except for people with a large number of offsets in map entries.
-This mechanism is used for the well known "hosts" map and we have seen
-cases (in 2.4) where the available number of mounts are exhausted or
-where the number of privileged ports available is exhausted.
-
-In version 5 we mount only as we go down the tree of offsets and
-similarly for expiring them which resolves the above problem. There is
-somewhat more detail to the implementation but it isn't needed for the
-sake of the problem explanation. The one important detail is that these
-offsets are implemented using the same mechanism as the direct mounts
-above and so the mount points can be covered by a mount.
-
-The current autofs implementation uses an ioctl file descriptor opened
-on the mount point for control operations. The references held by the
-descriptor are accounted for in checks made to determine if a mount is
-in use and is also used to access autofs file system information held
-in the mount super block. So the use of a file handle needs to be
-retained.
-
-
-The Solution
-============
-
-To be able to restart autofs leaving existing direct, indirect and
-offset mounts in place we need to be able to obtain a file handle
-for these potentially covered autofs mount points. Rather than just
-implement an isolated operation it was decided to re-implement the
-existing ioctl interface and add new operations to provide this
-functionality.
-
-In addition, to be able to reconstruct a mount tree that has busy mounts,
-the uid and gid of the last user that triggered the mount needs to be
-available because these can be used as macro substitution variables in
-autofs maps. They are recorded at mount request time and an operation
-has been added to retrieve them.
-
-Since we're re-implementing the control interface, a couple of other
-problems with the existing interface have been addressed. First, when
-a mount or expire operation completes a status is returned to the
-kernel by either a "send ready" or a "send fail" operation. The
-"send fail" operation of the ioctl interface could only ever send
-ENOENT so the re-implementation allows user space to send an actual
-status. Another expensive operation in user space, for those using
-very large maps, is discovering if a mount is present. Usually this
-involves scanning /proc/mounts and since it needs to be done quite
-often it can introduce significant overhead when there are many entries
-in the mount table. An operation to lookup the mount status of a mount
-point dentry (covered or not) has also been added.
-
-Current kernel development policy recommends avoiding the use of the
-ioctl mechanism in favor of systems such as Netlink. An implementation
-using this system was attempted to evaluate its suitability and it was
-found to be inadequate, in this case. The Generic Netlink system was
-used for this as raw Netlink would lead to a significant increase in
-complexity. There's no question that the Generic Netlink system is an
-elegant solution for common case ioctl functions but it's not a complete
-replacement probably because its primary purpose in life is to be a
-message bus implementation rather than specifically an ioctl replacement.
-While it would be possible to work around this there is one concern
-that lead to the decision to not use it. This is that the autofs
-expire in the daemon has become far to complex because umount
-candidates are enumerated, almost for no other reason than to "count"
-the number of times to call the expire ioctl. This involves scanning
-the mount table which has proved to be a big overhead for users with
-large maps. The best way to improve this is try and get back to the
-way the expire was done long ago. That is, when an expire request is
-issued for a mount (file handle) we should continually call back to
-the daemon until we can't umount any more mounts, then return the
-appropriate status to the daemon. At the moment we just expire one
-mount at a time. A Generic Netlink implementation would exclude this
-possibility for future development due to the requirements of the
-message bus architecture.
-
-
-autofs4 Miscellaneous Device mount control interface
-====================================================
-
-The control interface is opening a device node, typically /dev/autofs.
-
-All the ioctls use a common structure to pass the needed parameter
-information and return operation results:
-
-struct autofs_dev_ioctl {
-	__u32 ver_major;
-	__u32 ver_minor;
-	__u32 size;             /* total size of data passed in
-				 * including this struct */
-	__s32 ioctlfd;          /* automount command fd */
-
-	/* Command parameters */
-	union {
-		struct args_protover		protover;
-		struct args_protosubver		protosubver;
-		struct args_openmount		openmount;
-		struct args_ready		ready;
-		struct args_fail		fail;
-		struct args_setpipefd		setpipefd;
-		struct args_timeout		timeout;
-		struct args_requester		requester;
-		struct args_expire		expire;
-		struct args_askumount		askumount;
-		struct args_ismountpoint	ismountpoint;
-	};
-
-	char path[0];
-};
-
-The ioctlfd field is a mount point file descriptor of an autofs mount
-point. It is returned by the open call and is used by all calls except
-the check for whether a given path is a mount point, where it may
-optionally be used to check a specific mount corresponding to a given
-mount point file descriptor, and when requesting the uid and gid of the
-last successful mount on a directory within the autofs file system.
-
-The union is used to communicate parameters and results of calls made
-as described below.
-
-The path field is used to pass a path where it is needed and the size field
-is used account for the increased structure length when translating the
-structure sent from user space.
-
-This structure can be initialized before setting specific fields by using
-the void function call init_autofs_dev_ioctl(struct autofs_dev_ioctl *).
-
-All of the ioctls perform a copy of this structure from user space to
-kernel space and return -EINVAL if the size parameter is smaller than
-the structure size itself, -ENOMEM if the kernel memory allocation fails
-or -EFAULT if the copy itself fails. Other checks include a version check
-of the compiled in user space version against the module version and a
-mismatch results in a -EINVAL return. If the size field is greater than
-the structure size then a path is assumed to be present and is checked to
-ensure it begins with a "/" and is NULL terminated, otherwise -EINVAL is
-returned. Following these checks, for all ioctl commands except
-AUTOFS_DEV_IOCTL_VERSION_CMD, AUTOFS_DEV_IOCTL_OPENMOUNT_CMD and
-AUTOFS_DEV_IOCTL_CLOSEMOUNT_CMD the ioctlfd is validated and if it is
-not a valid descriptor or doesn't correspond to an autofs mount point
-an error of -EBADF, -ENOTTY or -EINVAL (not an autofs descriptor) is
-returned.
-
-
-The ioctls
-==========
-
-An example of an implementation which uses this interface can be seen
-in autofs version 5.0.4 and later in file lib/dev-ioctl-lib.c of the
-distribution tar available for download from kernel.org in directory
-/pub/linux/daemons/autofs/v5.
-
-The device node ioctl operations implemented by this interface are:
-
-
-AUTOFS_DEV_IOCTL_VERSION
-------------------------
-
-Get the major and minor version of the autofs4 device ioctl kernel module
-implementation. It requires an initialized struct autofs_dev_ioctl as an
-input parameter and sets the version information in the passed in structure.
-It returns 0 on success or the error -EINVAL if a version mismatch is
-detected.
-
-
-AUTOFS_DEV_IOCTL_PROTOVER_CMD and AUTOFS_DEV_IOCTL_PROTOSUBVER_CMD
-------------------------------------------------------------------
-
-Get the major and minor version of the autofs4 protocol version understood
-by loaded module. This call requires an initialized struct autofs_dev_ioctl
-with the ioctlfd field set to a valid autofs mount point descriptor
-and sets the requested version number in version field of struct args_protover
-or sub_version field of struct args_protosubver. These commands return
-0 on success or one of the negative error codes if validation fails.
-
-
-AUTOFS_DEV_IOCTL_OPENMOUNT and AUTOFS_DEV_IOCTL_CLOSEMOUNT
-----------------------------------------------------------
-
-Obtain and release a file descriptor for an autofs managed mount point
-path. The open call requires an initialized struct autofs_dev_ioctl with
-the path field set and the size field adjusted appropriately as well
-as the devid field of struct args_openmount set to the device number of
-the autofs mount. The device number can be obtained from the mount options
-shown in /proc/mounts. The close call requires an initialized struct
-autofs_dev_ioct with the ioctlfd field set to the descriptor obtained
-from the open call. The release of the file descriptor can also be done
-with close(2) so any open descriptors will also be closed at process exit.
-The close call is included in the implemented operations largely for
-completeness and to provide for a consistent user space implementation.
-
-
-AUTOFS_DEV_IOCTL_READY_CMD and AUTOFS_DEV_IOCTL_FAIL_CMD
---------------------------------------------------------
-
-Return mount and expire result status from user space to the kernel.
-Both of these calls require an initialized struct autofs_dev_ioctl
-with the ioctlfd field set to the descriptor obtained from the open
-call and the token field of struct args_ready or struct args_fail set
-to the wait queue token number, received by user space in the foregoing
-mount or expire request. The status field of struct args_fail is set to
-the errno of the operation. It is set to 0 on success.
-
-
-AUTOFS_DEV_IOCTL_SETPIPEFD_CMD
-------------------------------
-
-Set the pipe file descriptor used for kernel communication to the daemon.
-Normally this is set at mount time using an option but when reconnecting
-to a existing mount we need to use this to tell the autofs mount about
-the new kernel pipe descriptor. In order to protect mounts against
-incorrectly setting the pipe descriptor we also require that the autofs
-mount be catatonic (see next call).
-
-The call requires an initialized struct autofs_dev_ioctl with the
-ioctlfd field set to the descriptor obtained from the open call and
-the pipefd field of struct args_setpipefd set to descriptor of the pipe.
-On success the call also sets the process group id used to identify the
-controlling process (eg. the owning automount(8) daemon) to the process
-group of the caller.
-
-
-AUTOFS_DEV_IOCTL_CATATONIC_CMD
-------------------------------
-
-Make the autofs mount point catatonic. The autofs mount will no longer
-issue mount requests, the kernel communication pipe descriptor is released
-and any remaining waits in the queue released.
-
-The call requires an initialized struct autofs_dev_ioctl with the
-ioctlfd field set to the descriptor obtained from the open call.
-
-
-AUTOFS_DEV_IOCTL_TIMEOUT_CMD
-----------------------------
-
-Set the expire timeout for mounts within an autofs mount point.
-
-The call requires an initialized struct autofs_dev_ioctl with the
-ioctlfd field set to the descriptor obtained from the open call.
-
-
-AUTOFS_DEV_IOCTL_REQUESTER_CMD
-------------------------------
-
-Return the uid and gid of the last process to successfully trigger a the
-mount on the given path dentry.
-
-The call requires an initialized struct autofs_dev_ioctl with the path
-field set to the mount point in question and the size field adjusted
-appropriately. Upon return the uid field of struct args_requester contains
-the uid and gid field the gid.
-
-When reconstructing an autofs mount tree with active mounts we need to
-re-connect to mounts that may have used the original process uid and
-gid (or string variations of them) for mount lookups within the map entry.
-This call provides the ability to obtain this uid and gid so they may be
-used by user space for the mount map lookups.
-
-
-AUTOFS_DEV_IOCTL_EXPIRE_CMD
----------------------------
-
-Issue an expire request to the kernel for an autofs mount. Typically
-this ioctl is called until no further expire candidates are found.
-
-The call requires an initialized struct autofs_dev_ioctl with the
-ioctlfd field set to the descriptor obtained from the open call. In
-addition an immediate expire, independent of the mount timeout, can be
-requested by setting the how field of struct args_expire to 1. If no
-expire candidates can be found the ioctl returns -1 with errno set to
-EAGAIN.
-
-This call causes the kernel module to check the mount corresponding
-to the given ioctlfd for mounts that can be expired, issues an expire
-request back to the daemon and waits for completion.
-
-AUTOFS_DEV_IOCTL_ASKUMOUNT_CMD
-------------------------------
-
-Checks if an autofs mount point is in use.
-
-The call requires an initialized struct autofs_dev_ioctl with the
-ioctlfd field set to the descriptor obtained from the open call and
-it returns the result in the may_umount field of struct args_askumount,
-1 for busy and 0 otherwise.
-
-
-AUTOFS_DEV_IOCTL_ISMOUNTPOINT_CMD
----------------------------------
-
-Check if the given path is a mountpoint.
-
-The call requires an initialized struct autofs_dev_ioctl. There are two
-possible variations. Both use the path field set to the path of the mount
-point to check and the size field adjusted appropriately. One uses the
-ioctlfd field to identify a specific mount point to check while the other
-variation uses the path and optionally in.type field of struct args_ismountpoint
-set to an autofs mount type. The call returns 1 if this is a mount point
-and sets out.devid field to the device number of the mount and out.magic
-field to the relevant super block magic number (described below) or 0 if
-it isn't a mountpoint. In both cases the the device number (as returned
-by new_encode_dev()) is returned in out.devid field.
-
-If supplied with a file descriptor we're looking for a specific mount,
-not necessarily at the top of the mounted stack. In this case the path
-the descriptor corresponds to is considered a mountpoint if it is itself
-a mountpoint or contains a mount, such as a multi-mount without a root
-mount. In this case we return 1 if the descriptor corresponds to a mount
-point and and also returns the super magic of the covering mount if there
-is one or 0 if it isn't a mountpoint.
-
-If a path is supplied (and the ioctlfd field is set to -1) then the path
-is looked up and is checked to see if it is the root of a mount. If a
-type is also given we are looking for a particular autofs mount and if
-a match isn't found a fail is returned. If the the located path is the
-root of a mount 1 is returned along with the super magic of the mount
-or 0 otherwise.
-
diff --git a/Documentation/filesystems/autofs4.txt b/Documentation/filesystems/autofs4.txt
deleted file mode 100644
index f10dd590f69f..000000000000
--- a/Documentation/filesystems/autofs4.txt
+++ /dev/null
@@ -1,529 +0,0 @@
-<head>
-<style> p { max-width:50em} ol, ul {max-width: 40em}</style>
-</head>
-
-autofs - how it works
-=====================
-
-Purpose
--------
-
-The goal of autofs is to provide on-demand mounting and race free
-automatic unmounting of various other filesystems.  This provides two
-key advantages:
-
-1. There is no need to delay boot until all filesystems that
-   might be needed are mounted.  Processes that try to access those
-   slow filesystems might be delayed but other processes can
-   continue freely.  This is particularly important for
-   network filesystems (e.g. NFS) or filesystems stored on
-   media with a media-changing robot.
-
-2. The names and locations of filesystems can be stored in
-   a remote database and can change at any time.  The content
-   in that data base at the time of access will be used to provide
-   a target for the access.  The interpretation of names in the
-   filesystem can even be programmatic rather than database-backed,
-   allowing wildcards for example, and can vary based on the user who
-   first accessed a name.
-
-Context
--------
-
-The "autofs4" filesystem module is only one part of an autofs system.
-There also needs to be a user-space program which looks up names
-and mounts filesystems.  This will often be the "automount" program,
-though other tools including "systemd" can make use of "autofs4".
-This document describes only the kernel module and the interactions
-required with any user-space program.  Subsequent text refers to this
-as the "automount daemon" or simply "the daemon".
-
-"autofs4" is a Linux kernel module with provides the "autofs"
-filesystem type.  Several "autofs" filesystems can be mounted and they
-can each be managed separately, or all managed by the same daemon.
-
-Content
--------
-
-An autofs filesystem can contain 3 sorts of objects: directories,
-symbolic links and mount traps.  Mount traps are directories with
-extra properties as described in the next section.
-
-Objects can only be created by the automount daemon: symlinks are
-created with a regular `symlink` system call, while directories and
-mount traps are created with `mkdir`.  The determination of whether a
-directory should be a mount trap or not is quite _ad hoc_, largely for
-historical reasons, and is determined in part by the
-*direct*/*indirect*/*offset* mount options, and the *maxproto* mount option.
-
-If neither the *direct* or *offset* mount options are given (so the
-mount is considered to be *indirect*), then the root directory is
-always a regular directory, otherwise it is a mount trap when it is
-empty and a regular directory when not empty.  Note that *direct* and
-*offset* are treated identically so a concise summary is that the root
-directory is a mount trap only if the filesystem is mounted *direct*
-and the root is empty.
-
-Directories created in the root directory are mount traps only if the
-filesystem is mounted *indirect* and they are empty.
-
-Directories further down the tree depend on the *maxproto* mount
-option and particularly whether it is less than five or not.
-When *maxproto* is five, no directories further down the
-tree are ever mount traps, they are always regular directories.  When
-the *maxproto* is four (or three), these directories are mount traps
-precisely when they are empty.
-
-So: non-empty (i.e. non-leaf) directories are never mount traps. Empty
-directories are sometimes mount traps, and sometimes not depending on
-where in the tree they are (root, top level, or lower), the *maxproto*,
-and whether the mount was *indirect* or not.
-
-Mount Traps
----------------
-
-A core element of the implementation of autofs is the Mount Traps
-which are provided by the Linux VFS.  Any directory provided by a
-filesystem can be designated as a trap.  This involves two separate
-features that work together to allow autofs to do its job.
-
-**DCACHE_NEED_AUTOMOUNT**
-
-If a dentry has the DCACHE_NEED_AUTOMOUNT flag set (which gets set if
-the inode has S_AUTOMOUNT set, or can be set directly) then it is
-(potentially) a mount trap.  Any access to this directory beyond a
-"`stat`" will (normally) cause the `d_op->d_automount()` dentry operation
-to be called. The task of this method is to find the filesystem that
-should be mounted on the directory and to return it.  The VFS is
-responsible for actually mounting the root of this filesystem on the
-directory.
-
-autofs doesn't find the filesystem itself but sends a message to the
-automount daemon asking it to find and mount the filesystem.  The
-autofs `d_automount` method then waits for the daemon to report that
-everything is ready.  It will then return "`NULL`" indicating that the
-mount has already happened.  The VFS doesn't try to mount anything but
-follows down the mount that is already there.
-
-This functionality is sufficient for some users of mount traps such
-as NFS which creates traps so that mountpoints on the server can be
-reflected on the client.  However it is not sufficient for autofs.  As
-mounting onto a directory is considered to be "beyond a `stat`", the
-automount daemon would not be able to mount a filesystem on the 'trap'
-directory without some way to avoid getting caught in the trap.  For
-that purpose there is another flag.
-
-**DCACHE_MANAGE_TRANSIT**
-
-If a dentry has DCACHE_MANAGE_TRANSIT set then two very different but
-related behaviors are invoked, both using the `d_op->d_manage()`
-dentry operation.
-
-Firstly, before checking to see if any filesystem is mounted on the
-directory, d_manage() will be called with the `rcu_walk` parameter set
-to `false`.  It may return one of three things:
-
--  A return value of zero indicates that there is nothing special
-   about this dentry and normal checks for mounts and automounts
-   should proceed.
-
-   autofs normally returns zero, but first waits for any
-   expiry (automatic unmounting of the mounted filesystem) to
-   complete.  This avoids races.
-
--  A return value of `-EISDIR` tells the VFS to ignore any mounts
-   on the directory and to not consider calling `->d_automount()`.
-   This effectively disables the **DCACHE_NEED_AUTOMOUNT** flag
-   causing the directory not be a mount trap after all.
-
-   autofs returns this if it detects that the process performing the
-   lookup is the automount daemon and that the mount has been
-   requested but has not yet completed.  How it determines this is
-   discussed later.  This allows the automount daemon not to get
-   caught in the mount trap.
-
-   There is a subtlety here.  It is possible that a second autofs
-   filesystem can be mounted below the first and for both of them to
-   be managed by the same daemon.  For the daemon to be able to mount
-   something on the second it must be able to "walk" down past the
-   first.  This means that d_manage cannot *always* return -EISDIR for
-   the automount daemon.  It must only return it when a mount has
-   been requested, but has not yet completed.
-
-   `d_manage` also returns `-EISDIR` if the dentry shouldn't be a
-   mount trap, either because it is a symbolic link or because it is
-   not empty.
-
--  Any other negative value is treated as an error and returned
-   to the caller.
-
-   autofs can return
-
-   - -ENOENT if the automount daemon failed to mount anything,
-   - -ENOMEM if it ran out of memory,
-   - -EINTR if a signal arrived while waiting for expiry to
-     complete
-   - or any other error sent down by the automount daemon.
-
-
-The second use case only occurs during an "RCU-walk" and so `rcu_walk`
-will be set.
-
-An RCU-walk is a fast and lightweight process for walking down a
-filename path (i.e. it is like running on tip-toes).  RCU-walk cannot
-cope with all situations so when it finds a difficulty it falls back
-to "REF-walk", which is slower but more robust.
-
-RCU-walk will never call `->d_automount`; the filesystems must already
-be mounted or RCU-walk cannot handle the path.
-To determine if a mount-trap is safe for RCU-walk mode it calls
-`->d_manage()` with `rcu_walk` set to `true`.
-
-In this case `d_manage()` must avoid blocking and should avoid taking
-spinlocks if at all possible.  Its sole purpose is to determine if it
-would be safe to follow down into any mounted directory and the only
-reason that it might not be is if an expiry of the mount is
-underway.
-
-In the `rcu_walk` case, `d_manage()` cannot return -EISDIR to tell the
-VFS that this is a directory that doesn't require d_automount.  If
-`rcu_walk` sees a dentry with DCACHE_NEED_AUTOMOUNT set but nothing
-mounted, it *will* fall back to REF-walk.  `d_manage()` cannot make the
-VFS remain in RCU-walk mode, but can only tell it to get out of
-RCU-walk mode by returning `-ECHILD`.
-
-So `d_manage()`, when called with `rcu_walk` set, should either return
--ECHILD if there is any reason to believe it is unsafe to end the
-mounted filesystem, and otherwise should return 0.
-
-autofs will return `-ECHILD` if an expiry of the filesystem has been
-initiated or is being considered, otherwise it returns 0.
-
-
-Mountpoint expiry
------------------
-
-The VFS has a mechanism for automatically expiring unused mounts,
-much as it can expire any unused dentry information from the dcache.
-This is guided by the MNT_SHRINKABLE flag.  This only applies to
-mounts that were created by `d_automount()` returning a filesystem to be
-mounted.  As autofs doesn't return such a filesystem but leaves the
-mounting to the automount daemon, it must involve the automount daemon
-in unmounting as well.  This also means that autofs has more control
-of expiry.
-
-The VFS also supports "expiry" of mounts using the MNT_EXPIRE flag to
-the `umount` system call.  Unmounting with MNT_EXPIRE will fail unless
-a previous attempt had been made, and the filesystem has been inactive
-and untouched since that previous attempt.  autofs4 does not depend on
-this but has its own internal tracking of whether filesystems were
-recently used.  This allows individual names in the autofs directory
-to expire separately.
-
-With version 4 of the protocol, the automount daemon can try to
-unmount any filesystems mounted on the autofs filesystem or remove any
-symbolic links or empty directories any time it likes.  If the unmount
-or removal is successful the filesystem will be returned to the state
-it was before the mount or creation, so that any access of the name
-will trigger normal auto-mount processing.  In particlar, `rmdir` and
-`unlink` do not leave negative entries in the dcache as a normal
-filesystem would, so an attempt to access a recently-removed object is
-passed to autofs for handling.
-
-With version 5, this is not safe except for unmounting from top-level
-directories.  As lower-level directories are never mount traps, other
-processes will see an empty directory as soon as the filesystem is
-unmounted.  So it is generally safest to use the autofs expiry
-protocol described below.
-
-Normally the daemon only wants to remove entries which haven't been
-used for a while.  For this purpose autofs maintains a "`last_used`"
-time stamp on each directory or symlink.  For symlinks it genuinely
-does record the last time the symlink was "used" or followed to find
-out where it points to.  For directories the field is a slight
-misnomer.  It actually records the last time that autofs checked if
-the directory or one of its descendents was busy and found that it
-was.  This is just as useful and doesn't require updating the field so
-often.
-
-The daemon is able to ask autofs if anything is due to be expired,
-using an `ioctl` as discussed later.  For a *direct* mount, autofs
-considers if the entire mount-tree can be unmounted or not.  For an
-*indirect* mount, autofs considers each of the names in the top level
-directory to determine if any of those can be unmounted and cleaned
-up.
-
-There is an option with indirect mounts to consider each of the leaves
-that has been mounted on instead of considering the top-level names.
-This is intended for compatability with version 4 of autofs and should
-be considered as deprecated.
-
-When autofs considers a directory it checks the `last_used` time and
-compares it with the "timeout" value set when the filesystem was
-mounted, though this check is ignored in some cases. It also checks if
-the directory or anything below it is in use.  For symbolic links,
-only the `last_used` time is ever considered.
-
-If both appear to support expiring the directory or symlink, an action
-is taken.
-
-There are two ways to ask autofs to consider expiry.  The first is to
-use the **AUTOFS_IOC_EXPIRE** ioctl.  This only works for indirect
-mounts.  If it finds something in the root directory to expire it will
-return the name of that thing.  Once a name has been returned the
-automount daemon needs to unmount any filesystems mounted below the
-name normally.  As described above, this is unsafe for non-toplevel
-mounts in a version-5 autofs.  For this reason the current `automountd`
-does not use this ioctl.
-
-The second mechanism uses either the **AUTOFS_DEV_IOCTL_EXPIRE_CMD** or
-the **AUTOFS_IOC_EXPIRE_MULTI** ioctl.  This will work for both direct and
-indirect mounts.  If it selects an object to expire, it will notify
-the daemon using the notification mechanism described below.  This
-will block until the daemon acknowledges the expiry notification.
-This implies that the "`EXPIRE`" ioctl must be sent from a different
-thread than the one which handles notification.
-
-While the ioctl is blocking, the entry is marked as "expiring" and
-`d_manage` will block until the daemon affirms that the unmount has
-completed (together with removing any directories that might have been
-necessary), or has been aborted.
-
-Communicating with autofs: detecting the daemon
------------------------------------------------
-
-There are several forms of communication between the automount daemon
-and the filesystem.  As we have already seen, the daemon can create and
-remove directories and symlinks using normal filesystem operations.
-autofs knows whether a process requesting some operation is the daemon
-or not based on its process-group id number (see getpgid(1)).
-
-When an autofs filesystem is mounted the pgid of the mounting
-processes is recorded unless the "pgrp=" option is given, in which
-case that number is recorded instead.  Any request arriving from a
-process in that process group is considered to come from the daemon.
-If the daemon ever has to be stopped and restarted a new pgid can be
-provided through an ioctl as will be described below.
-
-Communicating with autofs: the event pipe
------------------------------------------
-
-When an autofs filesystem is mounted, the 'write' end of a pipe must
-be passed using the 'fd=' mount option.  autofs will write
-notification messages to this pipe for the daemon to respond to.
-For version 5, the format of the message is:
-
-        struct autofs_v5_packet {
-                int proto_version;                /* Protocol version */
-                int type;                        /* Type of packet */
-                autofs_wqt_t wait_queue_token;
-                __u32 dev;
-                __u64 ino;
-                __u32 uid;
-                __u32 gid;
-                __u32 pid;
-                __u32 tgid;
-                __u32 len;
-                char name[NAME_MAX+1];
-        };
-
-where the type is one of
-
-        autofs_ptype_missing_indirect
-        autofs_ptype_expire_indirect
-        autofs_ptype_missing_direct
-        autofs_ptype_expire_direct
-
-so messages can indicate that a name is missing (something tried to
-access it but it isn't there) or that it has been selected for expiry.
-
-The pipe will be set to "packet mode" (equivalent to passing
-`O_DIRECT`) to _pipe2(2)_ so that a read from the pipe will return at
-most one packet, and any unread portion of a packet will be discarded.
-
-The `wait_queue_token` is a unique number which can identify a
-particular request to be acknowledged.  When a message is sent over
-the pipe the affected dentry is marked as either "active" or
-"expiring" and other accesses to it block until the message is
-acknowledged using one of the ioctls below and the relevant
-`wait_queue_token`.
-
-Communicating with autofs: root directory ioctls
-------------------------------------------------
-
-The root directory of an autofs filesystem will respond to a number of
-ioctls.  The process issuing the ioctl must have the CAP_SYS_ADMIN
-capability, or must be the automount daemon.
-
-The available ioctl commands are:
-
-- **AUTOFS_IOC_READY**: a notification has been handled.  The argument
-    to the ioctl command is the "wait_queue_token" number
-    corresponding to the notification being acknowledged.
-- **AUTOFS_IOC_FAIL**: similar to above, but indicates failure with
-    the error code `ENOENT`.
-- **AUTOFS_IOC_CATATONIC**: Causes the autofs to enter "catatonic"
-    mode meaning that it stops sending notifications to the daemon.
-    This mode is also entered if a write to the pipe fails.
-- **AUTOFS_IOC_PROTOVER**:  This returns the protocol version in use.
-- **AUTOFS_IOC_PROTOSUBVER**: Returns the protocol sub-version which
-    is really a version number for the implementation.  It is
-    currently 2.
-- **AUTOFS_IOC_SETTIMEOUT**:  This passes a pointer to an unsigned
-    long.  The value is used to set the timeout for expiry, and
-    the current timeout value is stored back through the pointer.
-- **AUTOFS_IOC_ASKUMOUNT**:  Returns, in the pointed-to `int`, 1 if
-    the filesystem could be unmounted.  This is only a hint as
-    the situation could change at any instant.  This call can be
-    use to avoid a more expensive full unmount attempt.
-- **AUTOFS_IOC_EXPIRE**: as described above, this asks if there is
-    anything suitable to expire.  A pointer to a packet:
-
-        struct autofs_packet_expire_multi {
-                int proto_version;              /* Protocol version */
-                int type;                       /* Type of packet */
-                autofs_wqt_t wait_queue_token;
-                int len;
-                char name[NAME_MAX+1];
-        };
-
-     is required.  This is filled in with the name of something
-     that can be unmounted or removed.  If nothing can be expired,
-     `errno` is set to `EAGAIN`.  Even though a `wait_queue_token`
-     is present in the structure, no "wait queue" is established
-     and no acknowledgment is needed.
-- **AUTOFS_IOC_EXPIRE_MULTI**:  This is similar to
-     **AUTOFS_IOC_EXPIRE** except that it causes notification to be
-     sent to the daemon, and it blocks until the daemon acknowledges.
-     The argument is an integer which can contain two different flags.
-
-     **AUTOFS_EXP_IMMEDIATE** causes `last_used` time to be ignored
-     and objects are expired if the are not in use.
-
-     **AUTOFS_EXP_LEAVES** will select a leaf rather than a top-level
-     name to expire.  This is only safe when *maxproto* is 4.
-
-Communicating with autofs: char-device ioctls
----------------------------------------------
-
-It is not always possible to open the root of an autofs filesystem,
-particularly a *direct* mounted filesystem.  If the automount daemon
-is restarted there is no way for it to regain control of existing
-mounts using any of the above communication channels.  To address this
-need there is a "miscellaneous" character device (major 10, minor 235)
-which can be used to communicate directly with the autofs filesystem.
-It requires CAP_SYS_ADMIN for access.
-
-The `ioctl`s that can be used on this device are described in a separate
-document `autofs4-mount-control.txt`, and are summarized briefly here.
-Each ioctl is passed a pointer to an `autofs_dev_ioctl` structure:
-
-        struct autofs_dev_ioctl {
-                __u32 ver_major;
-                __u32 ver_minor;
-                __u32 size;             /* total size of data passed in
-                                         * including this struct */
-                __s32 ioctlfd;          /* automount command fd */
-
-		/* Command parameters */
-		union {
-			struct args_protover		protover;
-			struct args_protosubver		protosubver;
-			struct args_openmount		openmount;
-			struct args_ready		ready;
-			struct args_fail		fail;
-			struct args_setpipefd		setpipefd;
-			struct args_timeout		timeout;
-			struct args_requester		requester;
-			struct args_expire		expire;
-			struct args_askumount		askumount;
-			struct args_ismountpoint	ismountpoint;
-		};
-
-                char path[0];
-        };
-
-For the **OPEN_MOUNT** and **IS_MOUNTPOINT** commands, the target
-filesystem is identified by the `path`.  All other commands identify
-the filesystem by the `ioctlfd` which is a file descriptor open on the
-root, and which can be returned by **OPEN_MOUNT**.
-
-The `ver_major` and `ver_minor` are in/out parameters which check that
-the requested version is supported, and report the maximum version
-that the kernel module can support.
-
-Commands are:
-
-- **AUTOFS_DEV_IOCTL_VERSION_CMD**: does nothing, except validate and
-    set version numbers.
-- **AUTOFS_DEV_IOCTL_OPENMOUNT_CMD**: return an open file descriptor
-    on the root of an autofs filesystem.  The filesystem is identified
-    by name and device number, which is stored in `openmount.devid`.
-    Device numbers for existing filesystems can be found in
-    `/proc/self/mountinfo`.
-- **AUTOFS_DEV_IOCTL_CLOSEMOUNT_CMD**: same as `close(ioctlfd)`.
-- **AUTOFS_DEV_IOCTL_SETPIPEFD_CMD**: if the filesystem is in
-    catatonic mode, this can provide the write end of a new pipe
-    in `setpipefd.pipefd` to re-establish communication with a daemon.
-    The process group of the calling process is used to identify the
-    daemon.
-- **AUTOFS_DEV_IOCTL_REQUESTER_CMD**: `path` should be a
-    name within the filesystem that has been auto-mounted on.
-    On successful return, `requester.uid` and `requester.gid` will be
-    the UID and GID of the process which triggered that mount.
-- **AUTOFS_DEV_IOCTL_ISMOUNTPOINT_CMD**: Check if path is a
-    mountpoint of a particular type - see separate documentation for
-    details.
-- **AUTOFS_DEV_IOCTL_PROTOVER_CMD**:
-- **AUTOFS_DEV_IOCTL_PROTOSUBVER_CMD**:
-- **AUTOFS_DEV_IOCTL_READY_CMD**:
-- **AUTOFS_DEV_IOCTL_FAIL_CMD**:
-- **AUTOFS_DEV_IOCTL_CATATONIC_CMD**:
-- **AUTOFS_DEV_IOCTL_TIMEOUT_CMD**:
-- **AUTOFS_DEV_IOCTL_EXPIRE_CMD**:
-- **AUTOFS_DEV_IOCTL_ASKUMOUNT_CMD**:  These all have the same
-    function as the similarly named **AUTOFS_IOC** ioctls, except
-    that **FAIL** can be given an explicit error number in `fail.status`
-    instead of assuming `ENOENT`, and this **EXPIRE** command
-    corresponds to **AUTOFS_IOC_EXPIRE_MULTI**.
-
-Catatonic mode
---------------
-
-As mentioned, an autofs mount can enter "catatonic" mode.  This
-happens if a write to the notification pipe fails, or if it is
-explicitly requested by an `ioctl`.
-
-When entering catatonic mode, the pipe is closed and any pending
-notifications are acknowledged with the error `ENOENT`.
-
-Once in catatonic mode attempts to access non-existing names will
-result in `ENOENT` while attempts to access existing directories will
-be treated in the same way as if they came from the daemon, so mount
-traps will not fire.
-
-When the filesystem is mounted a _uid_ and _gid_ can be given which
-set the ownership of directories and symbolic links.  When the
-filesystem is in catatonic mode, any process with a matching UID can
-create directories or symlinks in the root directory, but not in other
-directories.
-
-Catatonic mode can only be left via the
-**AUTOFS_DEV_IOCTL_OPENMOUNT_CMD** ioctl on the `/dev/autofs`.
-
-autofs, name spaces, and shared mounts
---------------------------------------
-
-With bind mounts and name spaces it is possible for an autofs
-filesystem to appear at multiple places in one or more filesystem
-name spaces.  For this to work sensibly, the autofs filesystem should
-always be mounted "shared". e.g.
-
-> `mount --make-shared /autofs/mount/point`
-
-The automount daemon is only able to manage a single mount location for
-an autofs filesystem and if mounts on that are not 'shared', other
-locations will not behave as expected.  In particular access to those
-other locations will likely result in the `ELOOP` error
-
-> Too many levels of symbolic links
diff --git a/Documentation/filesystems/automount-support.txt b/Documentation/filesystems/automount-support.txt
index 7eb762eb3136..b0afd3d55eaf 100644
--- a/Documentation/filesystems/automount-support.txt
+++ b/Documentation/filesystems/automount-support.txt
@@ -9,7 +9,7 @@ also be requested by userspace.
 IN-KERNEL AUTOMOUNTING
 ======================
 
-See section "Mount Traps" of  Documentation/filesystems/autofs4.txt
+See section "Mount Traps" of  Documentation/filesystems/autofs.txt
 
 Then from userspace, you can just do something like:
 
diff --git a/Documentation/filesystems/f2fs.txt b/Documentation/filesystems/f2fs.txt
index 12a147c9f87f..69f8de995739 100644
--- a/Documentation/filesystems/f2fs.txt
+++ b/Documentation/filesystems/f2fs.txt
@@ -182,13 +182,15 @@ whint_mode=%s          Control which write hints are passed down to block
                        passes down hints with its policy.
 alloc_mode=%s          Adjust block allocation policy, which supports "reuse"
                        and "default".
-fsync_mode=%s          Control the policy of fsync. Currently supports "posix"
-                       and "strict". In "posix" mode, which is default, fsync
-                       will follow POSIX semantics and does a light operation
-                       to improve the filesystem performance. In "strict" mode,
-                       fsync will be heavy and behaves in line with xfs, ext4
-                       and btrfs, where xfstest generic/342 will pass, but the
-                       performance will regress.
+fsync_mode=%s          Control the policy of fsync. Currently supports "posix",
+                       "strict", and "nobarrier". In "posix" mode, which is
+                       default, fsync will follow POSIX semantics and does a
+                       light operation to improve the filesystem performance.
+                       In "strict" mode, fsync will be heavy and behaves in line
+                       with xfs, ext4 and btrfs, where xfstest generic/342 will
+                       pass, but the performance will regress. "nobarrier" is
+                       based on "posix", but doesn't issue flush command for
+                       non-atomic files likewise "nobarrier" mount option.
 test_dummy_encryption  Enable dummy encryption, which provides a fake fscrypt
                        context. The fake fscrypt context is used by xfstests.
 
diff --git a/Documentation/filesystems/fscrypt.rst b/Documentation/filesystems/fscrypt.rst
index cfbc18f0d9c9..48b424de85bb 100644
--- a/Documentation/filesystems/fscrypt.rst
+++ b/Documentation/filesystems/fscrypt.rst
@@ -191,11 +191,21 @@ Currently, the following pairs of encryption modes are supported:
 
 - AES-256-XTS for contents and AES-256-CTS-CBC for filenames
 - AES-128-CBC for contents and AES-128-CTS-CBC for filenames
+- Speck128/256-XTS for contents and Speck128/256-CTS-CBC for filenames
 
 It is strongly recommended to use AES-256-XTS for contents encryption.
 AES-128-CBC was added only for low-powered embedded devices with
 crypto accelerators such as CAAM or CESA that do not support XTS.
 
+Similarly, Speck128/256 support was only added for older or low-end
+CPUs which cannot do AES fast enough -- especially ARM CPUs which have
+NEON instructions but not the Cryptography Extensions -- and for which
+it would not otherwise be feasible to use encryption at all.  It is
+not recommended to use Speck on CPUs that have AES instructions.
+Speck support is only available if it has been enabled in the crypto
+API via CONFIG_CRYPTO_SPECK.  Also, on ARM platforms, to get
+acceptable performance CONFIG_CRYPTO_SPECK_NEON must be enabled.
+
 New encryption modes can be added relatively easily, without changes
 to individual filesystems.  However, authenticated encryption (AE)
 modes are not currently supported because of the difficulty of dealing
diff --git a/Documentation/filesystems/fuse-io.txt b/Documentation/filesystems/fuse-io.txt
new file mode 100644
index 000000000000..07b8f73f100f
--- /dev/null
+++ b/Documentation/filesystems/fuse-io.txt
@@ -0,0 +1,38 @@
+Fuse supports the following I/O modes:
+
+- direct-io
+- cached
+  + write-through
+  + writeback-cache
+
+The direct-io mode can be selected with the FOPEN_DIRECT_IO flag in the
+FUSE_OPEN reply.
+
+In direct-io mode the page cache is completely bypassed for reads and writes.
+No read-ahead takes place. Shared mmap is disabled.
+
+In cached mode reads may be satisfied from the page cache, and data may be
+read-ahead by the kernel to fill the cache.  The cache is always kept consistent
+after any writes to the file.  All mmap modes are supported.
+
+The cached mode has two sub modes controlling how writes are handled.  The
+write-through mode is the default and is supported on all kernels.  The
+writeback-cache mode may be selected by the FUSE_WRITEBACK_CACHE flag in the
+FUSE_INIT reply.
+
+In write-through mode each write is immediately sent to userspace as one or more
+WRITE requests, as well as updating any cached pages (and caching previously
+uncached, but fully written pages).  No READ requests are ever sent for writes,
+so when an uncached page is partially written, the page is discarded.
+
+In writeback-cache mode (enabled by the FUSE_WRITEBACK_CACHE flag) writes go to
+the cache only, which means that the write(2) syscall can often complete very
+fast.  Dirty pages are written back implicitly (background writeback or page
+reclaim on memory pressure) or explicitly (invoked by close(2), fsync(2) and
+when the last ref to the file is being released on munmap(2)).  This mode
+assumes that all changes to the filesystem go through the FUSE kernel module
+(size and atime/ctime/mtime attributes are kept up-to-date by the kernel), so
+it's generally not suitable for network filesystems.  If a partial page is
+written, then the page needs to be first read from userspace.  This means, that
+even for files opened for O_WRONLY it is possible that READ requests will be
+generated by the kernel.
diff --git a/Documentation/filesystems/nfs/nfsroot.txt b/Documentation/filesystems/nfs/nfsroot.txt
index 5efae00f6c7f..d2963123eb1c 100644
--- a/Documentation/filesystems/nfs/nfsroot.txt
+++ b/Documentation/filesystems/nfs/nfsroot.txt
@@ -5,6 +5,7 @@ Written 1996 by Gero Kuhlmann <gero@gkminix.han.de>
 Updated 1997 by Martin Mares <mj@atrey.karlin.mff.cuni.cz>
 Updated 2006 by Nico Schottelius <nico-kernel-nfsroot@schottelius.org>
 Updated 2006 by Horms <horms@verge.net.au>
+Updated 2018 by Chris Novakovic <chris@chrisn.me.uk>
 
 
 
@@ -79,7 +80,7 @@ nfsroot=[<server-ip>:]<root-dir>[,<nfs-options>]
 
 
 ip=<client-ip>:<server-ip>:<gw-ip>:<netmask>:<hostname>:<device>:<autoconf>:
-   <dns0-ip>:<dns1-ip>
+   <dns0-ip>:<dns1-ip>:<ntp0-ip>
 
   This parameter tells the kernel how to configure IP addresses of devices
   and also how to set up the IP routing table. It was originally called
@@ -110,6 +111,9 @@ ip=<client-ip>:<server-ip>:<gw-ip>:<netmask>:<hostname>:<device>:<autoconf>:
 		will not be triggered if it is missing and NFS root is not
 		in operation.
 
+		Value is exported to /proc/net/pnp with the prefix "bootserver "
+		(see below).
+
 		Default: Determined using autoconfiguration.
 		         The address of the autoconfiguration server is used.
 
@@ -123,10 +127,13 @@ ip=<client-ip>:<server-ip>:<gw-ip>:<netmask>:<hostname>:<device>:<autoconf>:
 
 		Default:  Determined using autoconfiguration.
 
-  <hostname>	Name of the client. May be supplied by autoconfiguration,
-  		but its absence will not trigger autoconfiguration.
-		If specified and DHCP is used, the user provided hostname will
-		be carried in the DHCP request to hopefully update DNS record.
+  <hostname>	Name of the client. If a '.' character is present, anything
+		before the first '.' is used as the client's hostname, and anything
+		after it is used as its NIS domain name. May be supplied by
+		autoconfiguration, but its absence will not trigger autoconfiguration.
+		If specified and DHCP is used, the user-provided hostname (and NIS
+		domain name, if present) will be carried in the DHCP request; this
+		may cause a DNS record to be created or updated for the client.
 
   		Default: Client IP address is used in ASCII notation.
 
@@ -162,12 +169,55 @@ ip=<client-ip>:<server-ip>:<gw-ip>:<netmask>:<hostname>:<device>:<autoconf>:
 
                 Default: any
 
-  <dns0-ip>	IP address of first nameserver.
-		Value gets exported by /proc/net/pnp which is often linked
-		on embedded systems by /etc/resolv.conf.
+  <dns0-ip>	IP address of primary nameserver.
+		Value is exported to /proc/net/pnp with the prefix "nameserver "
+		(see below).
+
+		Default: None if not using autoconfiguration; determined
+		automatically if using autoconfiguration.
+
+  <dns1-ip>	IP address of secondary nameserver.
+		See <dns0-ip>.
+
+  <ntp0-ip>	IP address of a Network Time Protocol (NTP) server.
+		Value is exported to /proc/net/ipconfig/ntp_servers, but is
+		otherwise unused (see below).
+
+		Default: None if not using autoconfiguration; determined
+		automatically if using autoconfiguration.
+
+  After configuration (whether manual or automatic) is complete, two files
+  are created in the following format; lines are omitted if their respective
+  value is empty following configuration:
+
+  - /proc/net/pnp:
+
+	#PROTO: <DHCP|BOOTP|RARP|MANUAL>	(depending on configuration method)
+	domain <dns-domain>			(if autoconfigured, the DNS domain)
+	nameserver <dns0-ip>			(primary name server IP)
+	nameserver <dns1-ip>			(secondary name server IP)
+	nameserver <dns2-ip>			(tertiary name server IP)
+	bootserver <server-ip>			(NFS server IP)
+
+  - /proc/net/ipconfig/ntp_servers:
+
+	<ntp0-ip>				(NTP server IP)
+	<ntp1-ip>				(NTP server IP)
+	<ntp2-ip>				(NTP server IP)
+
+  <dns-domain> and <dns2-ip> (in /proc/net/pnp) and <ntp1-ip> and <ntp2-ip>
+  (in /proc/net/ipconfig/ntp_servers) are requested during autoconfiguration;
+  they cannot be specified as part of the "ip=" kernel command line parameter.
+
+  Because the "domain" and "nameserver" options are recognised by DNS
+  resolvers, /etc/resolv.conf is often linked to /proc/net/pnp on systems
+  that use an NFS root filesystem.
 
-  <dns1-ip>	IP address of second nameserver.
-		Same as above.
+  Note that the kernel will not synchronise the system time with any NTP
+  servers it discovers; this is the responsibility of a user space process
+  (e.g. an initrd/initramfs script that passes the IP addresses listed in
+  /proc/net/ipconfig/ntp_servers to an NTP client before mounting the real
+  root filesystem if it is on NFS).
 
 
 nfsrootdebug
diff --git a/Documentation/filesystems/overlayfs.txt b/Documentation/filesystems/overlayfs.txt
index 961b287ef323..79be4a77ca08 100644
--- a/Documentation/filesystems/overlayfs.txt
+++ b/Documentation/filesystems/overlayfs.txt
@@ -10,10 +10,6 @@ union-filesystems).  An overlay-filesystem tries to present a
 filesystem which is the result over overlaying one filesystem on top
 of the other.
 
-The result will inevitably fail to look exactly like a normal
-filesystem for various technical reasons.  The expectation is that
-many use cases will be able to ignore these differences.
-
 
 Overlay objects
 ---------------
@@ -266,6 +262,30 @@ rightmost one and going left.  In the above example lower1 will be the
 top, lower2 the middle and lower3 the bottom layer.
 
 
+Metadata only copy up
+--------------------
+
+When metadata only copy up feature is enabled, overlayfs will only copy
+up metadata (as opposed to whole file), when a metadata specific operation
+like chown/chmod is performed. Full file will be copied up later when
+file is opened for WRITE operation.
+
+In other words, this is delayed data copy up operation and data is copied
+up when there is a need to actually modify data.
+
+There are multiple ways to enable/disable this feature. A config option
+CONFIG_OVERLAY_FS_METACOPY can be set/unset to enable/disable this feature
+by default. Or one can enable/disable it at module load time with module
+parameter metacopy=on/off. Lastly, there is also a per mount option
+metacopy=on/off to enable/disable this feature per mount.
+
+Do not use metacopy=on with untrusted upper/lower directories. Otherwise
+it is possible that an attacker can create an handcrafted file with
+appropriate REDIRECT and METACOPY xattrs, and gain access to file on lower
+pointed by REDIRECT. This should not be possible on local system as setting
+"trusted." xattrs will require CAP_SYS_ADMIN. But it should be possible
+for untrusted layers like from a pen drive.
+
 Sharing and copying layers
 --------------------------
 
@@ -284,7 +304,7 @@ though it will not result in a crash or deadlock.
 Mounting an overlay using an upper layer path, where the upper layer path
 was previously used by another mounted overlay in combination with a
 different lower layer path, is allowed, unless the "inodes index" feature
-is enabled.
+or "metadata only copy up" feature is enabled.
 
 With the "inodes index" feature, on the first time mount, an NFS file
 handle of the lower layer root directory, along with the UUID of the lower
@@ -297,6 +317,10 @@ lower root origin, mount will fail with ESTALE.  An overlayfs mount with
 does not support NFS export, lower filesystem does not have a valid UUID or
 if the upper filesystem does not support extended attributes.
 
+For "metadata only copy up" feature there is no verification mechanism at
+mount time. So if same upper is mouted with different set of lower, mount
+probably will succeed but expect the unexpected later on. So don't do it.
+
 It is quite a common practice to copy overlay layers to a different
 directory tree on the same or different underlying filesystem, and even
 to a different machine.  With the "inodes index" feature, trying to mount
@@ -306,27 +330,49 @@ the copied layers will fail the verification of the lower root file handle.
 Non-standard behavior
 ---------------------
 
-The copy_up operation essentially creates a new, identical file and
-moves it over to the old name.  Any open files referring to this inode
-will access the old data.
+Overlayfs can now act as a POSIX compliant filesystem with the following
+features turned on:
+
+1) "redirect_dir"
+
+Enabled with the mount option or module option: "redirect_dir=on" or with
+the kernel config option CONFIG_OVERLAY_FS_REDIRECT_DIR=y.
+
+If this feature is disabled, then rename(2) on a lower or merged directory
+will fail with EXDEV ("Invalid cross-device link").
+
+2) "inode index"
+
+Enabled with the mount option or module option "index=on" or with the
+kernel config option CONFIG_OVERLAY_FS_INDEX=y.
+
+If this feature is disabled and a file with multiple hard links is copied
+up, then this will "break" the link.  Changes will not be propagated to
+other names referring to the same inode.
+
+3) "xino"
+
+Enabled with the mount option "xino=auto" or "xino=on", with the module
+option "xino_auto=on" or with the kernel config option
+CONFIG_OVERLAY_FS_XINO_AUTO=y.  Also implicitly enabled by using the same
+underlying filesystem for all layers making up the overlay.
 
-The new file may be on a different filesystem, so both st_dev and st_ino
-of the real file may change.  The values of st_dev and st_ino returned by
-stat(2) on an overlay object are often not the same as the real file
-stat(2) values to prevent the values from changing on copy_up.
+If this feature is disabled or the underlying filesystem doesn't have
+enough free bits in the inode number, then overlayfs will not be able to
+guarantee that the values of st_ino and st_dev returned by stat(2) and the
+value of d_ino returned by readdir(3) will act like on a normal filesystem.
+E.g. the value of st_dev may be different for two objects in the same
+overlay filesystem and the value of st_ino for directory objects may not be
+persistent and could change even while the overlay filesystem is mounted.
 
-Unless "xino" feature is enabled, when overlay layers are not all on the
-same underlying filesystem, the value of st_dev may be different for two
-non-directory objects in the same overlay filesystem and the value of
-st_ino for directory objects may be non persistent and could change even
-while the overlay filesystem is still mounted.
+4) "copy_up_shared"
 
-Unless "inode index" feature is enabled, if a file with multiple hard
-links is copied up, then this will "break" the link.  Changes will not be
-propagated to other names referring to the same inode.
+Enabled with the mount option or module option "copy_up_shared=on" or with
+the kernel config option CONFIG_OVERLAY_FS_COPY_UP_SHARED=y.
 
-Unless "redirect_dir" feature is enabled, rename(2) on a lower or merged
-directory will fail with EXDEV.
+If this feature is disabled, then a memory mapping created with MAP_SHARED
+might contain stale data if the file has been copied up and modified in the
+meantime.
 
 
 Changes to underlying filesystems
@@ -429,11 +475,12 @@ This verification may cause significant overhead in some cases.
 Testsuite
 ---------
 
-There's testsuite developed by David Howells at:
+There's a testsuite originally developed by David Howells and currently
+maintained by Amir Goldstein at:
 
-  git://git.infradead.org/users/dhowells/unionmount-testsuite.git
+  https://github.com/amir73il/unionmount-testsuite.git
 
 Run as root:
 
   # cd unionmount-testsuite
-  # ./run --ov
+  # ./run --ov --verify
diff --git a/Documentation/filesystems/path-lookup.md b/Documentation/filesystems/path-lookup.md
index 1933ef734e63..e2edd45c4bc0 100644
--- a/Documentation/filesystems/path-lookup.md
+++ b/Documentation/filesystems/path-lookup.md
@@ -460,7 +460,7 @@ this retry process in the next article.
 Automount points are locations in the filesystem where an attempt to
 lookup a name can trigger changes to how that lookup should be
 handled, in particular by mounting a filesystem there.  These are
-covered in greater detail in autofs4.txt in the Linux documentation
+covered in greater detail in autofs.txt in the Linux documentation
 tree, but a few notes specifically related to path lookup are in order
 here.
 
diff --git a/Documentation/filesystems/proc.txt b/Documentation/filesystems/proc.txt
index 2a84bb334894..520f6a84cf50 100644
--- a/Documentation/filesystems/proc.txt
+++ b/Documentation/filesystems/proc.txt
@@ -515,7 +515,8 @@ guarantees:
 
 The /proc/PID/clear_refs is used to reset the PG_Referenced and ACCESSED/YOUNG
 bits on both physical and virtual pages associated with a process, and the
-soft-dirty bit on pte (see Documentation/vm/soft-dirty.txt for details).
+soft-dirty bit on pte (see Documentation/admin-guide/mm/soft-dirty.rst
+for details).
 To clear the bits for all the pages associated with the process
     > echo 1 > /proc/PID/clear_refs
 
@@ -536,7 +537,8 @@ Any other value written to /proc/PID/clear_refs will have no effect.
 
 The /proc/pid/pagemap gives the PFN, which can be used to find the pageflags
 using /proc/kpageflags and number of times a page is mapped using
-/proc/kpagecount. For detailed explanation, see Documentation/vm/pagemap.txt.
+/proc/kpagecount. For detailed explanation, see
+Documentation/admin-guide/mm/pagemap.rst.
 
 The /proc/pid/numa_maps is an extension based on maps, showing the memory
 locality and binding policy, as well as the memory usage (in pages) of
@@ -564,7 +566,7 @@ address   policy    mapping details
 
 Where:
 "address" is the starting address for the mapping;
-"policy" reports the NUMA memory policy set for the mapping (see vm/numa_memory_policy.txt);
+"policy" reports the NUMA memory policy set for the mapping (see Documentation/admin-guide/mm/numa_memory_policy.rst);
 "mapping details" summarizes mapping data such as mapping type, page usage counters,
 node locality page counters (N0 == node0, N1 == node1, ...) and the kernel page
 size, in KB, that is backing the mapping up.
diff --git a/Documentation/filesystems/tmpfs.txt b/Documentation/filesystems/tmpfs.txt
index a85355cf85f4..d06e9a59a9f4 100644
--- a/Documentation/filesystems/tmpfs.txt
+++ b/Documentation/filesystems/tmpfs.txt
@@ -105,8 +105,9 @@ policy for the file will revert to "default" policy.
 NUMA memory allocation policies have optional flags that can be used in
 conjunction with their modes.  These optional flags can be specified
 when tmpfs is mounted by appending them to the mode before the NodeList.
-See Documentation/vm/numa_memory_policy.txt for a list of all available
-memory allocation policy mode flags and their effect on memory policy.
+See Documentation/admin-guide/mm/numa_memory_policy.rst for a list of
+all available memory allocation policy mode flags and their effect on
+memory policy.
 
 	=static		is equivalent to	MPOL_F_STATIC_NODES
 	=relative	is equivalent to	MPOL_F_RELATIVE_NODES
diff --git a/Documentation/filesystems/vfs.txt b/Documentation/filesystems/vfs.txt
index 5fd325df59e2..c9226a9dc02c 100644
--- a/Documentation/filesystems/vfs.txt
+++ b/Documentation/filesystems/vfs.txt
@@ -856,9 +856,12 @@ struct file_operations {
 	ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
 	ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
 	int (*iterate) (struct file *, struct dir_context *);
-	unsigned int (*poll) (struct file *, struct poll_table_struct *);
+	__poll_t (*poll) (struct file *, struct poll_table_struct *);
+	struct wait_queue_head * (*get_poll_head)(struct file *, __poll_t);
+	__poll_t (*poll_mask) (struct file *, __poll_t);
 	long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
 	long (*compat_ioctl) (struct file *, unsigned int, unsigned long);
+	int (*pre_mmap) (struct file *, unsigned long, unsigned long);
 	int (*mmap) (struct file *, struct vm_area_struct *);
 	int (*mremap)(struct file *, struct vm_area_struct *);
 	int (*open) (struct inode *, struct file *);
@@ -901,11 +904,24 @@ otherwise noted.
 	activity on this file and (optionally) go to sleep until there
 	is activity. Called by the select(2) and poll(2) system calls
 
+  get_poll_head: Returns the struct wait_queue_head that callers can
+  wait on.  Callers need to check the returned events using ->poll_mask
+  once woken.  Can return NULL to indicate polling is not supported,
+  or any error code using the ERR_PTR convention to indicate that a
+  grave error occured and ->poll_mask shall not be called.
+
+  poll_mask: return the mask of EPOLL* values describing the file descriptor
+  state.  Called either before going to sleep on the waitqueue returned by
+  get_poll_head, or after it has been woken.  If ->get_poll_head and
+  ->poll_mask are implemented ->poll does not need to be implement.
+
   unlocked_ioctl: called by the ioctl(2) system call.
 
   compat_ioctl: called by the ioctl(2) system call when 32 bit system calls
  	 are used on 64 bit kernels.
 
+  pre_mmap: called before mmap, without mmap_sem being held yet.
+
   mmap: called by the mmap(2) system call
 
   open: called by the VFS when an inode should be opened. When the VFS
@@ -987,8 +1003,7 @@ struct dentry_operations {
 	char *(*d_dname)(struct dentry *, char *, int);
 	struct vfsmount *(*d_automount)(struct path *);
 	int (*d_manage)(const struct path *, bool);
-	struct dentry *(*d_real)(struct dentry *, const struct inode *,
-				 unsigned int, unsigned int);
+	struct dentry *(*d_real)(struct dentry *, const struct inode *);
 };
 
   d_revalidate: called when the VFS needs to revalidate a dentry. This
@@ -1122,22 +1137,15 @@ struct dentry_operations {
 	dentry being transited from.
 
   d_real: overlay/union type filesystems implement this method to return one of
-	the underlying dentries hidden by the overlay.  It is used in three
+	the underlying dentries hidden by the overlay.  It is used in two
 	different modes:
 
-	Called from open it may need to copy-up the file depending on the
-	supplied open flags.  This mode is selected with a non-zero flags
-	argument.  In this mode the d_real method can return an error.
-
 	Called from file_dentry() it returns the real dentry matching the inode
 	argument.  The real dentry may be from a lower layer already copied up,
 	but still referenced from the file.  This mode is selected with a
-	non-NULL inode argument.  This will always succeed.
-
-	With NULL inode and zero flags the topmost real underlying dentry is
-	returned.  This will always succeed.
+	non-NULL inode argument.
 
-	This method is never called with both non-NULL inode and non-zero flags.
+	With NULL inode the topmost real underlying dentry is returned.
 
 Each dentry has a pointer to its parent dentry, as well as a hash list
 of child dentries. Child dentries are basically like files in a
diff --git a/Documentation/fpga/fpga-mgr.txt b/Documentation/fpga/fpga-mgr.txt
deleted file mode 100644
index cc6413ed6fc9..000000000000
--- a/Documentation/fpga/fpga-mgr.txt
+++ /dev/null
@@ -1,199 +0,0 @@
-FPGA Manager Core
-
-Alan Tull 2015
-
-Overview
-========
-
-The FPGA manager core exports a set of functions for programming an FPGA with
-an image.  The API is manufacturer agnostic.  All manufacturer specifics are
-hidden away in a low level driver which registers a set of ops with the core.
-The FPGA image data itself is very manufacturer specific, but for our purposes
-it's just binary data.  The FPGA manager core won't parse it.
-
-The FPGA image to be programmed can be in a scatter gather list, a single
-contiguous buffer, or a firmware file.  Because allocating contiguous kernel
-memory for the buffer should be avoided, users are encouraged to use a scatter
-gather list instead if possible.
-
-The particulars for programming the image are presented in a structure (struct
-fpga_image_info).  This struct contains parameters such as pointers to the
-FPGA image as well as image-specific particulars such as whether the image was
-built for full or partial reconfiguration.
-
-API Functions:
-==============
-
-To program the FPGA:
---------------------
-
-	int fpga_mgr_load(struct fpga_manager *mgr,
-			  struct fpga_image_info *info);
-
-Load the FPGA from an image which is indicated in the info.  If successful,
-the FPGA ends up in operating mode.  Return 0 on success or a negative error
-code.
-
-To allocate or free a struct fpga_image_info:
----------------------------------------------
-
-	struct fpga_image_info *fpga_image_info_alloc(struct device *dev);
-
-	void fpga_image_info_free(struct fpga_image_info *info);
-
-To get/put a reference to a FPGA manager:
------------------------------------------
-
-	struct fpga_manager *of_fpga_mgr_get(struct device_node *node);
-	struct fpga_manager *fpga_mgr_get(struct device *dev);
-	void fpga_mgr_put(struct fpga_manager *mgr);
-
-Given a DT node or device, get a reference to a FPGA manager.  This pointer
-can be saved until you are ready to program the FPGA.  fpga_mgr_put releases
-the reference.
-
-
-To get exclusive control of a FPGA manager:
--------------------------------------------
-
-	int fpga_mgr_lock(struct fpga_manager *mgr);
-	void fpga_mgr_unlock(struct fpga_manager *mgr);
-
-The user should call fpga_mgr_lock and verify that it returns 0 before
-attempting to program the FPGA.  Likewise, the user should call
-fpga_mgr_unlock when done programming the FPGA.
-
-
-To register or unregister the low level FPGA-specific driver:
--------------------------------------------------------------
-
-	int fpga_mgr_register(struct device *dev, const char *name,
-			      const struct fpga_manager_ops *mops,
-			      void *priv);
-
-	void fpga_mgr_unregister(struct device *dev);
-
-Use of these two functions is described below in "How To Support a new FPGA
-device."
-
-
-How to write an image buffer to a supported FPGA
-================================================
-#include <linux/fpga/fpga-mgr.h>
-
-struct fpga_manager *mgr;
-struct fpga_image_info *info;
-int ret;
-
-/*
- * Get a reference to FPGA manager.  The manager is not locked, so you can
- * hold onto this reference without it preventing programming.
- *
- * This example uses the device node of the manager.  Alternatively, use
- * fpga_mgr_get(dev) instead if you have the device.
- */
-mgr = of_fpga_mgr_get(mgr_node);
-
-/* struct with information about the FPGA image to program. */
-info = fpga_image_info_alloc(dev);
-
-/* flags indicates whether to do full or partial reconfiguration */
-info->flags = FPGA_MGR_PARTIAL_RECONFIG;
-
-/*
- * At this point, indicate where the image is. This is pseudo-code; you're
- * going to use one of these three.
- */
-if (image is in a scatter gather table) {
-
-	info->sgt = [your scatter gather table]
-
-} else if (image is in a buffer) {
-
-	info->buf = [your image buffer]
-	info->count = [image buffer size]
-
-} else if (image is in a firmware file) {
-
-	info->firmware_name = devm_kstrdup(dev, firmware_name, GFP_KERNEL);
-
-}
-
-/* Get exclusive control of FPGA manager */
-ret = fpga_mgr_lock(mgr);
-
-/* Load the buffer to the FPGA */
-ret = fpga_mgr_buf_load(mgr, &info, buf, count);
-
-/* Release the FPGA manager */
-fpga_mgr_unlock(mgr);
-fpga_mgr_put(mgr);
-
-/* Deallocate the image info if you're done with it */
-fpga_image_info_free(info);
-
-How to support a new FPGA device
-================================
-To add another FPGA manager, write a driver that implements a set of ops.  The
-probe function calls fpga_mgr_register(), such as:
-
-static const struct fpga_manager_ops socfpga_fpga_ops = {
-       .write_init = socfpga_fpga_ops_configure_init,
-       .write = socfpga_fpga_ops_configure_write,
-       .write_complete = socfpga_fpga_ops_configure_complete,
-       .state = socfpga_fpga_ops_state,
-};
-
-static int socfpga_fpga_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct socfpga_fpga_priv *priv;
-	int ret;
-
-	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	/* ... do ioremaps, get interrupts, etc. and save
-	   them in priv... */
-
-	return fpga_mgr_register(dev, "Altera SOCFPGA FPGA Manager",
-				 &socfpga_fpga_ops, priv);
-}
-
-static int socfpga_fpga_remove(struct platform_device *pdev)
-{
-	fpga_mgr_unregister(&pdev->dev);
-
-	return 0;
-}
-
-
-The ops will implement whatever device specific register writes are needed to
-do the programming sequence for this particular FPGA.  These ops return 0 for
-success or negative error codes otherwise.
-
-The programming sequence is:
- 1. .write_init
- 2. .write or .write_sg (may be called once or multiple times)
- 3. .write_complete
-
-The .write_init function will prepare the FPGA to receive the image data.  The
-buffer passed into .write_init will be atmost .initial_header_size bytes long,
-if the whole bitstream is not immediately available then the core code will
-buffer up at least this much before starting.
-
-The .write function writes a buffer to the FPGA. The buffer may be contain the
-whole FPGA image or may be a smaller chunk of an FPGA image.  In the latter
-case, this function is called multiple times for successive chunks. This interface
-is suitable for drivers which use PIO.
-
-The .write_sg version behaves the same as .write except the input is a sg_table
-scatter list. This interface is suitable for drivers which use DMA.
-
-The .write_complete function is called after all the image has been written
-to put the FPGA into operating mode.
-
-The ops include a .state function which will read the hardware FPGA manager and
-return a code of type enum fpga_mgr_states.  It doesn't result in a change in
-hardware state.
diff --git a/Documentation/fpga/fpga-region.txt b/Documentation/fpga/fpga-region.txt
deleted file mode 100644
index 139a02ba1ff6..000000000000
--- a/Documentation/fpga/fpga-region.txt
+++ /dev/null
@@ -1,95 +0,0 @@
-FPGA Regions
-
-Alan Tull 2017
-
-CONTENTS
- - Introduction
- - The FPGA region API
- - Usage example
-
-Introduction
-============
-
-This document is meant to be an brief overview of the FPGA region API usage.  A
-more conceptual look at regions can be found in [1].
-
-For the purposes of this API document, let's just say that a region associates
-an FPGA Manager and a bridge (or bridges) with a reprogrammable region of an
-FPGA or the whole FPGA.  The API provides a way to register a region and to
-program a region.
-
-Currently the only layer above fpga-region.c in the kernel is the Device Tree
-support (of-fpga-region.c) described in [1].  The DT support layer uses regions
-to program the FPGA and then DT to handle enumeration.  The common region code
-is intended to be used by other schemes that have other ways of accomplishing
-enumeration after programming.
-
-An fpga-region can be set up to know the following things:
-* which FPGA manager to use to do the programming
-* which bridges to disable before programming and enable afterwards.
-
-Additional info needed to program the FPGA image is passed in the struct
-fpga_image_info [2] including:
-* pointers to the image as either a scatter-gather buffer, a contiguous
-  buffer, or the name of firmware file
-* flags indicating specifics such as whether the image if for partial
-  reconfiguration.
-
-===================
-The FPGA region API
-===================
-
-To register or unregister a region:
------------------------------------
-
-	int fpga_region_register(struct device *dev,
-				 struct fpga_region *region);
-	int fpga_region_unregister(struct fpga_region *region);
-
-An example of usage can be seen in the probe function of [3]
-
-To program an FPGA:
--------------------
-	int fpga_region_program_fpga(struct fpga_region *region);
-
-This function operates on info passed in the fpga_image_info
-(region->info).
-
-This function will attempt to:
- * lock the region's mutex
- * lock the region's FPGA manager
- * build a list of FPGA bridges if a method has been specified to do so
- * disable the bridges
- * program the FPGA
- * re-enable the bridges
- * release the locks
-
-=============
-Usage example
-=============
-
-First, allocate the info struct:
-
-	info = fpga_image_info_alloc(dev);
-	if (!info)
-		return -ENOMEM;
-
-Set flags as needed, i.e.
-
-	info->flags |= FPGA_MGR_PARTIAL_RECONFIG;
-
-Point to your FPGA image, such as:
-
-	info->sgt = &sgt;
-
-Add info to region and do the programming:
-
-	region->info = info;
-	ret = fpga_region_program_fpga(region);
-
-Then enumerate whatever hardware has appeared in the FPGA.
-
---
-[1] ../devicetree/bindings/fpga/fpga-region.txt
-[2] ./fpga-mgr.txt
-[3] ../../drivers/fpga/of-fpga-region.c
diff --git a/Documentation/fpga/overview.txt b/Documentation/fpga/overview.txt
deleted file mode 100644
index 0f1236e7e675..000000000000
--- a/Documentation/fpga/overview.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-Linux kernel FPGA support
-
-Alan Tull 2017
-
-The main point of this project has been to separate the out the upper layers
-that know when to reprogram a FPGA from the lower layers that know how to
-reprogram a specific FPGA device.  The intention is to make this manufacturer
-agnostic, understanding that of course the FPGA images are very device specific
-themselves.
-
-The framework in the kernel includes:
-* low level FPGA manager drivers that know how to program a specific device
-* the fpga-mgr framework they are registered with
-* low level FPGA bridge drivers for hard/soft bridges which are intended to
-  be disable during FPGA programming
-* the fpga-bridge framework they are registered with
-* the fpga-region framework which associates and controls managers and bridges
-  as reconfigurable regions
-* the of-fpga-region support for reprogramming FPGAs when device tree overlays
-  are applied.
-
-I would encourage you the user to add code that creates FPGA regions rather
-that trying to control managers and bridges separately.
diff --git a/Documentation/gpu/drivers.rst b/Documentation/gpu/drivers.rst
index e8c84419a2a1..f982558fc25d 100644
--- a/Documentation/gpu/drivers.rst
+++ b/Documentation/gpu/drivers.rst
@@ -10,8 +10,10 @@ GPU Driver Documentation
    tegra
    tinydrm
    tve200
+   v3d
    vc4
    bridge/dw-hdmi
+   xen-front
 
 .. only::  subproject and html
 
diff --git a/Documentation/gpu/i915.rst b/Documentation/gpu/i915.rst
index 41dc881b00dc..055df45596c1 100644
--- a/Documentation/gpu/i915.rst
+++ b/Documentation/gpu/i915.rst
@@ -58,6 +58,12 @@ Intel GVT-g Host Support(vGPU device model)
 .. kernel-doc:: drivers/gpu/drm/i915/intel_gvt.c
    :internal:
 
+Workarounds
+-----------
+
+.. kernel-doc:: drivers/gpu/drm/i915/intel_workarounds.c
+   :doc: Hardware workarounds
+
 Display Hardware Handling
 =========================
 
@@ -249,6 +255,103 @@ Memory Management and Command Submission
 This sections covers all things related to the GEM implementation in the
 i915 driver.
 
+Intel GPU Basics
+----------------
+
+An Intel GPU has multiple engines. There are several engine types.
+
+- RCS engine is for rendering 3D and performing compute, this is named
+  `I915_EXEC_RENDER` in user space.
+- BCS is a blitting (copy) engine, this is named `I915_EXEC_BLT` in user
+  space.
+- VCS is a video encode and decode engine, this is named `I915_EXEC_BSD`
+  in user space
+- VECS is video enhancement engine, this is named `I915_EXEC_VEBOX` in user
+  space.
+- The enumeration `I915_EXEC_DEFAULT` does not refer to specific engine;
+  instead it is to be used by user space to specify a default rendering
+  engine (for 3D) that may or may not be the same as RCS.
+
+The Intel GPU family is a family of integrated GPU's using Unified
+Memory Access. For having the GPU "do work", user space will feed the
+GPU batch buffers via one of the ioctls `DRM_IOCTL_I915_GEM_EXECBUFFER2`
+or `DRM_IOCTL_I915_GEM_EXECBUFFER2_WR`. Most such batchbuffers will
+instruct the GPU to perform work (for example rendering) and that work
+needs memory from which to read and memory to which to write. All memory
+is encapsulated within GEM buffer objects (usually created with the ioctl
+`DRM_IOCTL_I915_GEM_CREATE`). An ioctl providing a batchbuffer for the GPU
+to create will also list all GEM buffer objects that the batchbuffer reads
+and/or writes. For implementation details of memory management see
+`GEM BO Management Implementation Details`_.
+
+The i915 driver allows user space to create a context via the ioctl
+`DRM_IOCTL_I915_GEM_CONTEXT_CREATE` which is identified by a 32-bit
+integer. Such a context should be viewed by user-space as -loosely-
+analogous to the idea of a CPU process of an operating system. The i915
+driver guarantees that commands issued to a fixed context are to be
+executed so that writes of a previously issued command are seen by
+reads of following commands. Actions issued between different contexts
+(even if from the same file descriptor) are NOT given that guarantee
+and the only way to synchronize across contexts (even from the same
+file descriptor) is through the use of fences. At least as far back as
+Gen4, also have that a context carries with it a GPU HW context;
+the HW context is essentially (most of atleast) the state of a GPU.
+In addition to the ordering guarantees, the kernel will restore GPU
+state via HW context when commands are issued to a context, this saves
+user space the need to restore (most of atleast) the GPU state at the
+start of each batchbuffer. The non-deprecated ioctls to submit batchbuffer
+work can pass that ID (in the lower bits of drm_i915_gem_execbuffer2::rsvd1)
+to identify what context to use with the command.
+
+The GPU has its own memory management and address space. The kernel
+driver maintains the memory translation table for the GPU. For older
+GPUs (i.e. those before Gen8), there is a single global such translation
+table, a global Graphics Translation Table (GTT). For newer generation
+GPUs each context has its own translation table, called Per-Process
+Graphics Translation Table (PPGTT). Of important note, is that although
+PPGTT is named per-process it is actually per context. When user space
+submits a batchbuffer, the kernel walks the list of GEM buffer objects
+used by the batchbuffer and guarantees that not only is the memory of
+each such GEM buffer object resident but it is also present in the
+(PP)GTT. If the GEM buffer object is not yet placed in the (PP)GTT,
+then it is given an address. Two consequences of this are: the kernel
+needs to edit the batchbuffer submitted to write the correct value of
+the GPU address when a GEM BO is assigned a GPU address and the kernel
+might evict a different GEM BO from the (PP)GTT to make address room
+for another GEM BO. Consequently, the ioctls submitting a batchbuffer
+for execution also include a list of all locations within buffers that
+refer to GPU-addresses so that the kernel can edit the buffer correctly.
+This process is dubbed relocation.
+
+GEM BO Management Implementation Details
+----------------------------------------
+
+.. kernel-doc:: drivers/gpu/drm/i915/i915_vma.h
+   :doc: Virtual Memory Address
+
+Buffer Object Eviction
+----------------------
+
+This section documents the interface functions for evicting buffer
+objects to make space available in the virtual gpu address spaces. Note
+that this is mostly orthogonal to shrinking buffer objects caches, which
+has the goal to make main memory (shared with the gpu through the
+unified memory architecture) available.
+
+.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_evict.c
+   :internal:
+
+Buffer Object Memory Shrinking
+------------------------------
+
+This section documents the interface function for shrinking memory usage
+of buffer object caches. Shrinking is used to make main memory
+available. Note that this is mostly orthogonal to evicting buffer
+objects, which has the goal to make space in gpu virtual address spaces.
+
+.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_shrinker.c
+   :internal:
+
 Batchbuffer Parsing
 -------------------
 
@@ -267,6 +370,12 @@ Batchbuffer Pools
 .. kernel-doc:: drivers/gpu/drm/i915/i915_gem_batch_pool.c
    :internal:
 
+User Batchbuffer Execution
+--------------------------
+
+.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_execbuffer.c
+   :doc: User command execution
+
 Logical Rings, Logical Ring Contexts and Execlists
 --------------------------------------------------
 
@@ -312,28 +421,14 @@ Object Tiling IOCTLs
 .. kernel-doc:: drivers/gpu/drm/i915/i915_gem_tiling.c
    :doc: buffer object tiling
 
-Buffer Object Eviction
-----------------------
-
-This section documents the interface functions for evicting buffer
-objects to make space available in the virtual gpu address spaces. Note
-that this is mostly orthogonal to shrinking buffer objects caches, which
-has the goal to make main memory (shared with the gpu through the
-unified memory architecture) available.
-
-.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_evict.c
-   :internal:
-
-Buffer Object Memory Shrinking
-------------------------------
+WOPCM
+=====
 
-This section documents the interface function for shrinking memory usage
-of buffer object caches. Shrinking is used to make main memory
-available. Note that this is mostly orthogonal to evicting buffer
-objects, which has the goal to make space in gpu virtual address spaces.
+WOPCM Layout
+------------
 
-.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_shrinker.c
-   :internal:
+.. kernel-doc:: drivers/gpu/drm/i915/intel_wopcm.c
+   :doc: WOPCM Layout
 
 GuC
 ===
@@ -359,6 +454,12 @@ GuC Firmware Layout
 .. kernel-doc:: drivers/gpu/drm/i915/intel_guc_fwif.h
    :doc: GuC Firmware Layout
 
+GuC Address Space
+-----------------
+
+.. kernel-doc:: drivers/gpu/drm/i915/intel_guc.c
+   :doc: GuC Address Space
+
 Tracing
 =======
 
diff --git a/Documentation/gpu/kms-properties.csv b/Documentation/gpu/kms-properties.csv
index 6b28b014cb7d..07ed22ea3bd6 100644
--- a/Documentation/gpu/kms-properties.csv
+++ b/Documentation/gpu/kms-properties.csv
@@ -98,5 +98,4 @@ radeon,DVI-I,“coherent”,RANGE,"Min=0, Max=1",Connector,TBD
 ,,"""underscan vborder""",RANGE,"Min=0, Max=128",Connector,TBD
 ,Audio,“audio”,ENUM,"{ ""off"", ""on"", ""auto"" }",Connector,TBD
 ,FMT Dithering,“dither”,ENUM,"{ ""off"", ""on"" }",Connector,TBD
-rcar-du,Generic,"""alpha""",RANGE,"Min=0, Max=255",Plane,TBD
 ,,"""colorkey""",RANGE,"Min=0, Max=0x01ffffff",Plane,TBD
diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst
index f4d0b3476d9c..a7c150d6b63f 100644
--- a/Documentation/gpu/todo.rst
+++ b/Documentation/gpu/todo.rst
@@ -212,6 +212,24 @@ probably use drm_fb_helper_fbdev_teardown().
 
 Contact: Maintainer of the driver you plan to convert
 
+Clean up mmap forwarding
+------------------------
+
+A lot of drivers forward gem mmap calls to dma-buf mmap for imported buffers.
+And also a lot of them forward dma-buf mmap to the gem mmap implementations.
+Would be great to refactor this all into a set of small common helpers.
+
+Contact: Daniel Vetter
+
+Put a reservation_object into drm_gem_object
+--------------------------------------------
+
+This would remove the need for the ->gem_prime_res_obj callback. It would also
+allow us to implement generic helpers for waiting for a bo, allowing for quite a
+bit of refactoring in the various wait ioctl implementations.
+
+Contact: Daniel Vetter
+
 idr_init_base()
 ---------------
 
diff --git a/Documentation/gpu/xen-front.rst b/Documentation/gpu/xen-front.rst
new file mode 100644
index 000000000000..d988da7d1983
--- /dev/null
+++ b/Documentation/gpu/xen-front.rst
@@ -0,0 +1,31 @@
+====================================================
+ drm/xen-front Xen para-virtualized frontend driver
+====================================================
+
+This frontend driver implements Xen para-virtualized display
+according to the display protocol described at
+include/xen/interface/io/displif.h
+
+Driver modes of operation in terms of display buffers used
+==========================================================
+
+.. kernel-doc:: drivers/gpu/drm/xen/xen_drm_front.h
+   :doc: Driver modes of operation in terms of display buffers used
+
+Buffers allocated by the frontend driver
+----------------------------------------
+
+.. kernel-doc:: drivers/gpu/drm/xen/xen_drm_front.h
+   :doc: Buffers allocated by the frontend driver
+
+Buffers allocated by the backend
+--------------------------------
+
+.. kernel-doc:: drivers/gpu/drm/xen/xen_drm_front.h
+   :doc: Buffers allocated by the backend
+
+Driver limitations
+==================
+
+.. kernel-doc:: drivers/gpu/drm/xen/xen_drm_front.h
+   :doc: Driver limitations
diff --git a/Documentation/hwmon/hwmon-kernel-api.txt b/Documentation/hwmon/hwmon-kernel-api.txt
index 53a806696c64..eb7a78aebb38 100644
--- a/Documentation/hwmon/hwmon-kernel-api.txt
+++ b/Documentation/hwmon/hwmon-kernel-api.txt
@@ -71,7 +71,8 @@ hwmon_device_register_with_info is the most comprehensive and preferred means
 to register a hardware monitoring device. It creates the standard sysfs
 attributes in the hardware monitoring core, letting the driver focus on reading
 from and writing to the chip instead of having to bother with sysfs attributes.
-Its parameters are described in more detail below.
+The parent device parameter cannot be NULL with non-NULL chip info. Its
+parameters are described in more detail below.
 
 devm_hwmon_device_register_with_info is similar to
 hwmon_device_register_with_info. However, it is device managed, meaning the
diff --git a/Documentation/hwmon/ltc2990 b/Documentation/hwmon/ltc2990
index c25211e90bdc..3ed68f676c0f 100644
--- a/Documentation/hwmon/ltc2990
+++ b/Documentation/hwmon/ltc2990
@@ -8,6 +8,7 @@ Supported chips:
     Datasheet: http://www.linear.com/product/ltc2990
 
 Author: Mike Looijmans <mike.looijmans@topic.nl>
+        Tom Levens <tom.levens@cern.ch>
 
 
 Description
@@ -16,10 +17,8 @@ Description
 LTC2990 is a Quad I2C Voltage, Current and Temperature Monitor.
 The chip's inputs can measure 4 voltages, or two inputs together (1+2 and 3+4)
 can be combined to measure a differential voltage, which is typically used to
-measure current through a series resistor, or a temperature.
-
-This driver currently uses the 2x differential mode only. In order to support
-other modes, the driver will need to be expanded.
+measure current through a series resistor, or a temperature with an external
+diode.
 
 
 Usage Notes
@@ -32,12 +31,19 @@ devices explicitly.
 Sysfs attributes
 ----------------
 
+in0_input     Voltage at Vcc pin in millivolt (range 2.5V to 5V)
+temp1_input   Internal chip temperature in millidegrees Celcius
+
+A subset of the following attributes are visible, depending on the measurement
+mode of the chip.
+
+in[1-4]_input Voltage at V[1-4] pin in millivolt
+temp2_input   External temperature sensor TR1 in millidegrees Celcius
+temp3_input   External temperature sensor TR2 in millidegrees Celcius
+curr1_input   Current in mA across V1-V2 assuming a 1mOhm sense resistor
+curr2_input   Current in mA across V3-V4 assuming a 1mOhm sense resistor
+
 The "curr*_input" measurements actually report the voltage drop across the
 input pins in microvolts. This is equivalent to the current through a 1mOhm
 sense resistor. Divide the reported value by the actual sense resistor value
 in mOhm to get the actual value.
-
-in0_input     Voltage at Vcc pin in millivolt (range 2.5V to 5V)
-temp1_input   Internal chip temperature in millidegrees Celcius
-curr1_input   Current in mA across v1-v2 assuming a 1mOhm sense resistor.
-curr2_input   Current in mA across v3-v4 assuming a 1mOhm sense resistor.
diff --git a/Documentation/i2c/busses/i2c-mlxcpld b/Documentation/i2c/busses/i2c-mlxcpld
index 4e46c440b38d..925904aa9b57 100644
--- a/Documentation/i2c/busses/i2c-mlxcpld
+++ b/Documentation/i2c/busses/i2c-mlxcpld
@@ -20,6 +20,10 @@ The next transaction types are supported:
  - Write Byte/Block.
 
 Registers:
+CPBLTY		0x0 - capability reg.
+			Bits [6:5] - transaction length. b01 - 72B is supported,
+			36B in other case.
+			Bit 7 - SMBus block read support.
 CTRL		0x1 - control reg.
 			Resets all the registers.
 HALF_CYC	0x4 - cycle reg.
diff --git a/Documentation/i2c/busses/i2c-ocores b/Documentation/i2c/busses/i2c-ocores
index c269aaa2f26a..4e713f4cdb2f 100644
--- a/Documentation/i2c/busses/i2c-ocores
+++ b/Documentation/i2c/busses/i2c-ocores
@@ -2,7 +2,7 @@ Kernel driver i2c-ocores
 
 Supported adapters:
   * OpenCores.org I2C controller by Richard Herveille (see datasheet link)
-    Datasheet: http://www.opencores.org/projects.cgi/web/i2c/overview
+    https://opencores.org/project/i2c/overview
 
 Author: Peter Korsgaard <jacmet@sunsite.dk>
 
@@ -18,7 +18,7 @@ Usage
 i2c-ocores uses the platform bus, so you need to provide a struct
 platform_device with the base address and interrupt number. The
 dev.platform_data of the device should also point to a struct
-ocores_i2c_platform_data (see linux/i2c-ocores.h) describing the
+ocores_i2c_platform_data (see linux/platform_data/i2c-ocores.h) describing the
 distance between registers and the input clock speed.
 There is also a possibility to attach a list of i2c_board_info which
 the i2c-ocores driver will add to the bus upon creation.
diff --git a/Documentation/i2c/muxes/i2c-mux-gpio b/Documentation/i2c/muxes/i2c-mux-gpio
index 7a8d7d261632..893ecdfe6e43 100644
--- a/Documentation/i2c/muxes/i2c-mux-gpio
+++ b/Documentation/i2c/muxes/i2c-mux-gpio
@@ -30,12 +30,12 @@ i2c-mux-gpio uses the platform bus, so you need to provide a struct
 platform_device with the platform_data pointing to a struct
 i2c_mux_gpio_platform_data with the I2C adapter number of the master
 bus, the number of bus segments to create and the GPIO pins used
-to control it. See include/linux/i2c-mux-gpio.h for details.
+to control it. See include/linux/platform_data/i2c-mux-gpio.h for details.
 
 E.G. something like this for a MUX providing 4 bus segments
 controlled through 3 GPIO pins:
 
-#include <linux/i2c-mux-gpio.h>
+#include <linux/platform_data/i2c-mux-gpio.h>
 #include <linux/platform_device.h>
 
 static const unsigned myboard_gpiomux_gpios[] = {
diff --git a/Documentation/index.rst b/Documentation/index.rst
index 3b99ab931d41..fdc585703498 100644
--- a/Documentation/index.rst
+++ b/Documentation/index.rst
@@ -45,7 +45,7 @@ the kernel interface as seen by application developers.
 .. toctree::
    :maxdepth: 2
 
-   userspace-api/index	      
+   userspace-api/index
 
 
 Introduction to kernel development
@@ -89,6 +89,7 @@ needed).
    sound/index
    crypto/index
    filesystems/index
+   vm/index
 
 Architecture-specific documentation
 -----------------------------------
diff --git a/Documentation/ioctl/botching-up-ioctls.txt b/Documentation/ioctl/botching-up-ioctls.txt
index d02cfb48901c..883fb034bd04 100644
--- a/Documentation/ioctl/botching-up-ioctls.txt
+++ b/Documentation/ioctl/botching-up-ioctls.txt
@@ -73,7 +73,9 @@ will have a second iteration or at least an extension for any given interface.
    future extensions is going right down the gutters since someone will submit
    an ioctl struct with random stack garbage in the yet unused parts. Which
    then bakes in the ABI that those fields can never be used for anything else
-   but garbage.
+   but garbage. This is also the reason why you must explicitly pad all
+   structures, even if you never use them in an array - the padding the compiler
+   might insert could contain garbage.
 
  * Have simple testcases for all of the above.
 
diff --git a/Documentation/ioctl/ioctl-number.txt b/Documentation/ioctl/ioctl-number.txt
index 7f7413e597f3..480c8609dc58 100644
--- a/Documentation/ioctl/ioctl-number.txt
+++ b/Documentation/ioctl/ioctl-number.txt
@@ -151,7 +151,7 @@ Code  Seq#(hex)	Include File		Comments
 'J'	00-1F	drivers/scsi/gdth_ioctl.h
 'K'	all	linux/kd.h
 'L'	00-1F	linux/loop.h		conflict!
-'L'	10-1F	drivers/scsi/mpt2sas/mpt2sas_ctl.h	conflict!
+'L'	10-1F	drivers/scsi/mpt3sas/mpt3sas_ctl.h	conflict!
 'L'	20-2F	linux/lightnvm.h
 'L'	E0-FF	linux/ppdd.h		encrypted disk device driver
 					<http://linux01.gwdg.de/~alatham/ppdd.html>
@@ -296,7 +296,8 @@ Code  Seq#(hex)	Include File		Comments
 0x90	00	drivers/cdrom/sbpcd.h
 0x92	00-0F	drivers/usb/mon/mon_bin.c
 0x93	60-7F	linux/auto_fs.h
-0x94	all	fs/btrfs/ioctl.h
+0x94	all	fs/btrfs/ioctl.h	Btrfs filesystem
+		and linux/fs.h		some lifted to vfs/generic
 0x97	00-7F	fs/ceph/ioctl.h		Ceph file system
 0x99	00-0F				537-Addinboard driver
 					<mailto:buk@buks.ipn.de>
@@ -327,6 +328,7 @@ Code  Seq#(hex)	Include File		Comments
 0xCA	80-BF	uapi/scsi/cxlflash_ioctl.h
 0xCB	00-1F	CBM serial IEC bus	in development:
 					<mailto:michael.klein@puffin.lb.shuttle.de>
+0xCC	00-0F	drivers/misc/ibmvmc.h    pseries VMC driver
 0xCD	01	linux/reiserfs_fs.h
 0xCF	02	fs/cifs/ioctl.c
 0xDB	00-0F	drivers/char/mwave/mwavepub.h
diff --git a/Documentation/kbuild/kconfig-language.txt b/Documentation/kbuild/kconfig-language.txt
index f5b9493f04ad..a4eb01843c04 100644
--- a/Documentation/kbuild/kconfig-language.txt
+++ b/Documentation/kbuild/kconfig-language.txt
@@ -198,14 +198,6 @@ applicable everywhere (see syntax).
     enables the third modular state for all config symbols.
     At most one symbol may have the "modules" option set.
 
-  - "env"=<value>
-    This imports the environment variable into Kconfig. It behaves like
-    a default, except that the value comes from the environment, this
-    also means that the behaviour when mixing it with normal defaults is
-    undefined at this point. The symbol is currently not exported back
-    to the build environment (if this is desired, it can be done via
-    another symbol).
-
   - "allnoconfig_y"
     This declares the symbol as one that should have the value y when
     using "allnoconfig". Used for symbols that hide other symbols.
@@ -481,6 +473,24 @@ config option to 'y' no matter the dependencies.
 The dependencies are moved to the symbol GENERIC_IOMAP and we avoid the
 situation where select forces a symbol equals to 'y'.
 
+Adding features that need compiler support
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are several features that need compiler support. The recommended way
+to describe the dependency on the compiler feature is to use "depends on"
+followed by a test macro.
+
+config CC_STACKPROTECTOR
+	bool "Stack Protector buffer overflow detection"
+	depends on $(cc-option,-fstack-protector)
+	...
+
+If you need to expose a compiler capability to makefiles and/or C source files,
+CC_HAS_ is the recommended prefix for the config option.
+
+config CC_HAS_STACKPROTECTOR_NONE
+	def_bool $(cc-option,-fno-stack-protector)
+
 Build as module only
 ~~~~~~~~~~~~~~~~~~~~
 To restrict a component build to module-only, qualify its config symbol
diff --git a/Documentation/kbuild/kconfig-macro-language.txt b/Documentation/kbuild/kconfig-macro-language.txt
new file mode 100644
index 000000000000..07da2ea68dce
--- /dev/null
+++ b/Documentation/kbuild/kconfig-macro-language.txt
@@ -0,0 +1,242 @@
+Concept
+-------
+
+The basic idea was inspired by Make. When we look at Make, we notice sort of
+two languages in one. One language describes dependency graphs consisting of
+targets and prerequisites. The other is a macro language for performing textual
+substitution.
+
+There is clear distinction between the two language stages. For example, you
+can write a makefile like follows:
+
+    APP := foo
+    SRC := foo.c
+    CC := gcc
+
+    $(APP): $(SRC)
+            $(CC) -o $(APP) $(SRC)
+
+The macro language replaces the variable references with their expanded form,
+and handles as if the source file were input like follows:
+
+    foo: foo.c
+            gcc -o foo foo.c
+
+Then, Make analyzes the dependency graph and determines the targets to be
+updated.
+
+The idea is quite similar in Kconfig - it is possible to describe a Kconfig
+file like this:
+
+    CC := gcc
+
+    config CC_HAS_FOO
+            def_bool $(shell, $(srctree)/scripts/gcc-check-foo.sh $(CC))
+
+The macro language in Kconfig processes the source file into the following
+intermediate:
+
+    config CC_HAS_FOO
+            def_bool y
+
+Then, Kconfig moves onto the evaluation stage to resolve inter-symbol
+dependency as explained in kconfig-language.txt.
+
+
+Variables
+---------
+
+Like in Make, a variable in Kconfig works as a macro variable.  A macro
+variable is expanded "in place" to yield a text string that may then be
+expanded further. To get the value of a variable, enclose the variable name in
+$( ). The parentheses are required even for single-letter variable names; $X is
+a syntax error. The curly brace form as in ${CC} is not supported either.
+
+There are two types of variables: simply expanded variables and recursively
+expanded variables.
+
+A simply expanded variable is defined using the := assignment operator. Its
+righthand side is expanded immediately upon reading the line from the Kconfig
+file.
+
+A recursively expanded variable is defined using the = assignment operator.
+Its righthand side is simply stored as the value of the variable without
+expanding it in any way. Instead, the expansion is performed when the variable
+is used.
+
+There is another type of assignment operator; += is used to append text to a
+variable. The righthand side of += is expanded immediately if the lefthand
+side was originally defined as a simple variable. Otherwise, its evaluation is
+deferred.
+
+The variable reference can take parameters, in the following form:
+
+  $(name,arg1,arg2,arg3)
+
+You can consider the parameterized reference as a function. (more precisely,
+"user-defined function" in contrast to "built-in function" listed below).
+
+Useful functions must be expanded when they are used since the same function is
+expanded differently if different parameters are passed. Hence, a user-defined
+function is defined using the = assignment operator. The parameters are
+referenced within the body definition with $(1), $(2), etc.
+
+In fact, recursively expanded variables and user-defined functions are the same
+internally. (In other words, "variable" is "function with zero argument".)
+When we say "variable" in a broad sense, it includes "user-defined function".
+
+
+Built-in functions
+------------------
+
+Like Make, Kconfig provides several built-in functions. Every function takes a
+particular number of arguments.
+
+In Make, every built-in function takes at least one argument. Kconfig allows
+zero argument for built-in functions, such as $(fileno), $(lineno). You could
+consider those as "built-in variable", but it is just a matter of how we call
+it after all. Let's say "built-in function" here to refer to natively supported
+functionality.
+
+Kconfig currently supports the following built-in functions.
+
+ - $(shell,command)
+
+  The "shell" function accepts a single argument that is expanded and passed
+  to a subshell for execution. The standard output of the command is then read
+  and returned as the value of the function. Every newline in the output is
+  replaced with a space. Any trailing newlines are deleted. The standard error
+  is not returned, nor is any program exit status.
+
+ - $(info,text)
+
+  The "info" function takes a single argument and prints it to stdout.
+  It evaluates to an empty string.
+
+ - $(warning-if,condition,text)
+
+  The "warning-if" function takes two arguments. If the condition part is "y",
+  the text part is sent to stderr. The text is prefixed with the name of the
+  current Kconfig file and the current line number.
+
+ - $(error-if,condition,text)
+
+  The "error-if" function is similar to "warning-if", but it terminates the
+  parsing immediately if the condition part is "y".
+
+ - $(filename)
+
+  The 'filename' takes no argument, and $(filename) is expanded to the file
+  name being parsed.
+
+ - $(lineno)
+
+  The 'lineno' takes no argument, and $(lineno) is expanded to the line number
+  being parsed.
+
+
+Make vs Kconfig
+---------------
+
+Kconfig adopts Make-like macro language, but the function call syntax is
+slightly different.
+
+A function call in Make looks like this:
+
+  $(func-name arg1,arg2,arg3)
+
+The function name and the first argument are separated by at least one
+whitespace. Then, leading whitespaces are trimmed from the first argument,
+while whitespaces in the other arguments are kept. You need to use a kind of
+trick to start the first parameter with spaces. For example, if you want
+to make "info" function print "  hello", you can write like follows:
+
+  empty :=
+  space := $(empty) $(empty)
+  $(info $(space)$(space)hello)
+
+Kconfig uses only commas for delimiters, and keeps all whitespaces in the
+function call. Some people prefer putting a space after each comma delimiter:
+
+  $(func-name, arg1, arg2, arg3)
+
+In this case, "func-name" will receive " arg1", " arg2", " arg3". The presence
+of leading spaces may matter depending on the function. The same applies to
+Make - for example, $(subst .c, .o, $(sources)) is a typical mistake; it
+replaces ".c" with " .o".
+
+In Make, a user-defined function is referenced by using a built-in function,
+'call', like this:
+
+    $(call my-func,arg1,arg2,arg3)
+
+Kconfig invokes user-defined functions and built-in functions in the same way.
+The omission of 'call' makes the syntax shorter.
+
+In Make, some functions treat commas verbatim instead of argument separators.
+For example, $(shell echo hello, world) runs the command "echo hello, world".
+Likewise, $(info hello, world) prints "hello, world" to stdout. You could say
+this is _useful_ inconsistency.
+
+In Kconfig, for simpler implementation and grammatical consistency, commas that
+appear in the $( ) context are always delimiters. It means
+
+  $(shell, echo hello, world)
+
+is an error because it is passing two parameters where the 'shell' function
+accepts only one. To pass commas in arguments, you can use the following trick:
+
+  comma := ,
+  $(shell, echo hello$(comma) world)
+
+
+Caveats
+-------
+
+A variable (or function) cannot be expanded across tokens. So, you cannot use
+a variable as a shorthand for an expression that consists of multiple tokens.
+The following works:
+
+    RANGE_MIN := 1
+    RANGE_MAX := 3
+
+    config FOO
+            int "foo"
+            range $(RANGE_MIN) $(RANGE_MAX)
+
+But, the following does not work:
+
+    RANGES := 1 3
+
+    config FOO
+            int "foo"
+            range $(RANGES)
+
+A variable cannot be expanded to any keyword in Kconfig.  The following does
+not work:
+
+    MY_TYPE := tristate
+
+    config FOO
+            $(MY_TYPE) "foo"
+            default y
+
+Obviously from the design, $(shell command) is expanded in the textual
+substitution phase. You cannot pass symbols to the 'shell' function.
+The following does not work as expected.
+
+    config ENDIAN_FLAG
+            string
+            default "-mbig-endian" if CPU_BIG_ENDIAN
+            default "-mlittle-endian" if CPU_LITTLE_ENDIAN
+
+    config CC_HAS_ENDIAN_FLAG
+            def_bool $(shell $(srctree)/scripts/gcc-check-flag ENDIAN_FLAG)
+
+Instead, you can do like follows so that any function call is statically
+expanded.
+
+    config CC_HAS_ENDIAN_FLAG
+            bool
+            default $(shell $(srctree)/scripts/gcc-check-flag -mbig-endian) if CPU_BIG_ENDIAN
+            default $(shell $(srctree)/scripts/gcc-check-flag -mlittle-endian) if CPU_LITTLE_ENDIAN
diff --git a/Documentation/laptops/thinkpad-acpi.txt b/Documentation/laptops/thinkpad-acpi.txt
index 00b6dfed573c..6cced88de6da 100644
--- a/Documentation/laptops/thinkpad-acpi.txt
+++ b/Documentation/laptops/thinkpad-acpi.txt
@@ -540,8 +540,10 @@ Events that are propagated by the driver to userspace:
 0x6021		ALARM: a sensor is too hot
 0x6022		ALARM: a sensor is extremely hot
 0x6030		System thermal table changed
+0x6032		Thermal Control command set completion  (DYTC, Windows)
 0x6040		Nvidia Optimus/AC adapter related (TO BE VERIFIED)
 0x60C0		X1 Yoga 2016, Tablet mode status changed
+0x60F0		Thermal Transformation changed (GMTS, Windows)
 
 Battery nearly empty alarms are a last resort attempt to get the
 operating system to hibernate or shutdown cleanly (0x2313), or shutdown
diff --git a/Documentation/livepatch/livepatch.txt b/Documentation/livepatch/livepatch.txt
index 1ae2de758c08..2d7ed09dbd59 100644
--- a/Documentation/livepatch/livepatch.txt
+++ b/Documentation/livepatch/livepatch.txt
@@ -429,30 +429,6 @@ See Documentation/ABI/testing/sysfs-kernel-livepatch for more details.
 
 The current Livepatch implementation has several limitations:
 
-
-  + The patch must not change the semantic of the patched functions.
-
-    The current implementation guarantees only that either the old
-    or the new function is called. The functions are patched one
-    by one. It means that the patch must _not_ change the semantic
-    of the function.
-
-
-  + Data structures can not be patched.
-
-    There is no support to version data structures or anyhow migrate
-    one structure into another. Also the simple consistency model does
-    not allow to switch more functions atomically.
-
-    Once there is more complex consistency mode, it will be possible to
-    use some workarounds. For example, it will be possible to use a hole
-    for a new member because the data structure is aligned. Or it will
-    be possible to use an existing member for something else.
-
-    There are no plans to add more generic support for modified structures
-    at the moment.
-
-
   + Only functions that can be traced could be patched.
 
     Livepatch is based on the dynamic ftrace. In particular, functions
diff --git a/Documentation/media/kapi/cec-core.rst b/Documentation/media/kapi/cec-core.rst
index a9f53f069a2d..1d989c544370 100644
--- a/Documentation/media/kapi/cec-core.rst
+++ b/Documentation/media/kapi/cec-core.rst
@@ -246,7 +246,10 @@ CEC_TX_STATUS_LOW_DRIVE:
 CEC_TX_STATUS_ERROR:
 	some unspecified error occurred: this can be one of ARB_LOST
 	or LOW_DRIVE if the hardware cannot differentiate or something
-	else entirely.
+	else entirely. Some hardware only supports OK and FAIL as the
+	result of a transmit, i.e. there is no way to differentiate
+	between the different possible errors. In that case map FAIL
+	to CEC_TX_STATUS_NACK and not to CEC_TX_STATUS_ERROR.
 
 CEC_TX_STATUS_MAX_RETRIES:
 	could not transmit the message after trying multiple times.
diff --git a/Documentation/media/uapi/cec/cec-ioc-receive.rst b/Documentation/media/uapi/cec/cec-ioc-receive.rst
index bdad4b197bcd..e964074cd15b 100644
--- a/Documentation/media/uapi/cec/cec-ioc-receive.rst
+++ b/Documentation/media/uapi/cec/cec-ioc-receive.rst
@@ -231,26 +231,32 @@ View On' messages from initiator 0xf ('Unregistered') to destination 0 ('TV').
       - ``CEC_TX_STATUS_OK``
       - 0x01
       - The message was transmitted successfully. This is mutually
-	exclusive with :ref:`CEC_TX_STATUS_MAX_RETRIES <CEC-TX-STATUS-MAX-RETRIES>`. Other bits can still
-	be set if earlier attempts met with failure before the transmit
-	was eventually successful.
+	exclusive with :ref:`CEC_TX_STATUS_MAX_RETRIES <CEC-TX-STATUS-MAX-RETRIES>`.
+	Other bits can still be set if earlier attempts met with failure before
+	the transmit was eventually successful.
     * .. _`CEC-TX-STATUS-ARB-LOST`:
 
       - ``CEC_TX_STATUS_ARB_LOST``
       - 0x02
-      - CEC line arbitration was lost.
+      - CEC line arbitration was lost, i.e. another transmit started at the
+        same time with a higher priority. Optional status, not all hardware
+	can detect this error condition.
     * .. _`CEC-TX-STATUS-NACK`:
 
       - ``CEC_TX_STATUS_NACK``
       - 0x04
-      - Message was not acknowledged.
+      - Message was not acknowledged. Note that some hardware cannot tell apart
+        a 'Not Acknowledged' status from other error conditions, i.e. the result
+	of a transmit is just OK or FAIL. In that case this status will be
+	returned when the transmit failed.
     * .. _`CEC-TX-STATUS-LOW-DRIVE`:
 
       - ``CEC_TX_STATUS_LOW_DRIVE``
       - 0x08
       - Low drive was detected on the CEC bus. This indicates that a
 	follower detected an error on the bus and requests a
-	retransmission.
+	retransmission. Optional status, not all hardware can detect this
+	error condition.
     * .. _`CEC-TX-STATUS-ERROR`:
 
       - ``CEC_TX_STATUS_ERROR``
@@ -258,14 +264,14 @@ View On' messages from initiator 0xf ('Unregistered') to destination 0 ('TV').
       - Some error occurred. This is used for any errors that do not fit
 	``CEC_TX_STATUS_ARB_LOST`` or ``CEC_TX_STATUS_LOW_DRIVE``, either because
 	the hardware could not tell which error occurred, or because the hardware
-	tested for other conditions besides those two.
+	tested for other conditions besides those two. Optional status.
     * .. _`CEC-TX-STATUS-MAX-RETRIES`:
 
       - ``CEC_TX_STATUS_MAX_RETRIES``
       - 0x20
       - The transmit failed after one or more retries. This status bit is
-	mutually exclusive with :ref:`CEC_TX_STATUS_OK <CEC-TX-STATUS-OK>`. Other bits can still
-	be set to explain which failures were seen.
+	mutually exclusive with :ref:`CEC_TX_STATUS_OK <CEC-TX-STATUS-OK>`.
+	Other bits can still be set to explain which failures were seen.
 
 
 .. tabularcolumns:: |p{5.6cm}|p{0.9cm}|p{11.0cm}|
diff --git a/Documentation/media/uapi/dvb/dvbapi.rst b/Documentation/media/uapi/dvb/dvbapi.rst
index 18c86b3a3af1..89ddca38626f 100644
--- a/Documentation/media/uapi/dvb/dvbapi.rst
+++ b/Documentation/media/uapi/dvb/dvbapi.rst
@@ -62,7 +62,7 @@ Authors:
 
  - Original author of the Digital TV API documentation.
 
-- Carvalho Chehab, Mauro <m.chehab@kernel.org>
+- Carvalho Chehab, Mauro <mchehab+samsung@kernel.org>
 
  - Ported document to Docbook XML, addition of DVBv5 API, documentation gaps fix.
 
diff --git a/Documentation/media/uapi/rc/lirc-dev-intro.rst b/Documentation/media/uapi/rc/lirc-dev-intro.rst
index 698e4f80270e..11516c8bff62 100644
--- a/Documentation/media/uapi/rc/lirc-dev-intro.rst
+++ b/Documentation/media/uapi/rc/lirc-dev-intro.rst
@@ -18,7 +18,7 @@ Example dmesg output upon a driver registering w/LIRC:
 .. code-block:: none
 
     $ dmesg |grep lirc_dev
-    rc rc0: lirc_dev: driver mceusb registered at minor = 0
+    rc rc0: lirc_dev: driver mceusb registered at minor = 0, raw IR receiver, raw IR transmitter
 
 What you should see for a chardev:
 
diff --git a/Documentation/media/uapi/rc/lirc-set-rec-timeout.rst b/Documentation/media/uapi/rc/lirc-set-rec-timeout.rst
index b3e16bbdbc90..a833a6a4c25a 100644
--- a/Documentation/media/uapi/rc/lirc-set-rec-timeout.rst
+++ b/Documentation/media/uapi/rc/lirc-set-rec-timeout.rst
@@ -1,19 +1,23 @@
 .. -*- coding: utf-8; mode: rst -*-
 
 .. _lirc_set_rec_timeout:
+.. _lirc_get_rec_timeout:
 
-**************************
-ioctl LIRC_SET_REC_TIMEOUT
-**************************
+***************************************************
+ioctl LIRC_GET_REC_TIMEOUT and LIRC_SET_REC_TIMEOUT
+***************************************************
 
 Name
 ====
 
-LIRC_SET_REC_TIMEOUT - sets the integer value for IR inactivity timeout.
+LIRC_GET_REC_TIMEOUT/LIRC_SET_REC_TIMEOUT - Get/set the integer value for IR inactivity timeout.
 
 Synopsis
 ========
 
+.. c:function:: int ioctl( int fd, LIRC_GET_REC_TIMEOUT, __u32 *timeout )
+    :name: LIRC_GET_REC_TIMEOUT
+
 .. c:function:: int ioctl( int fd, LIRC_SET_REC_TIMEOUT, __u32 *timeout )
     :name: LIRC_SET_REC_TIMEOUT
 
@@ -30,7 +34,7 @@ Arguments
 Description
 ===========
 
-Sets the integer value for IR inactivity timeout.
+Get and set the integer value for IR inactivity timeout.
 
 If supported by the hardware, setting it to 0  disables all hardware timeouts
 and data should be reported as soon as possible. If the exact value
diff --git a/Documentation/media/uapi/v4l/common.rst b/Documentation/media/uapi/v4l/common.rst
index 13f2ed3fc5a6..5f93e71122ef 100644
--- a/Documentation/media/uapi/v4l/common.rst
+++ b/Documentation/media/uapi/v4l/common.rst
@@ -41,6 +41,6 @@ applicable to all devices.
     extended-controls
     format
     planar-apis
-    crop
     selection-api
+    crop
     streaming-par
diff --git a/Documentation/media/uapi/v4l/crop.rst b/Documentation/media/uapi/v4l/crop.rst
index 182565b9ace4..45e8a895a320 100644
--- a/Documentation/media/uapi/v4l/crop.rst
+++ b/Documentation/media/uapi/v4l/crop.rst
@@ -2,9 +2,18 @@
 
 .. _crop:
 
-*************************************
-Image Cropping, Insertion and Scaling
-*************************************
+*****************************************************
+Image Cropping, Insertion and Scaling -- the CROP API
+*****************************************************
+
+.. note::
+
+   The CROP API is mostly superseded by the newer :ref:`SELECTION API
+   <selection-api>`. The new API should be preferred in most cases,
+   with the exception of pixel aspect ratio detection, which is
+   implemented by :ref:`VIDIOC_CROPCAP <VIDIOC_CROPCAP>` and has no
+   equivalent in the SELECTION API. See :ref:`selection-vs-crop` for a
+   comparison of the two APIs.
 
 Some video capture devices can sample a subsection of the picture and
 shrink or enlarge it to an image of arbitrary size. We call these
@@ -42,10 +51,9 @@ where applicable) will be fixed in this case.
 
 .. note::
 
-   All capture and output devices must support the
-   :ref:`VIDIOC_CROPCAP <VIDIOC_CROPCAP>` ioctl such that applications
-   can determine if scaling takes place.
-
+   All capture and output devices that support the CROP or SELECTION
+   API will also support the :ref:`VIDIOC_CROPCAP <VIDIOC_CROPCAP>`
+   ioctl.
 
 Cropping Structures
 ===================
diff --git a/Documentation/media/uapi/v4l/selection-api-002.rst b/Documentation/media/uapi/v4l/selection-api-002.rst
deleted file mode 100644
index 09ca93f91bf7..000000000000
--- a/Documentation/media/uapi/v4l/selection-api-002.rst
+++ /dev/null
@@ -1,28 +0,0 @@
-.. -*- coding: utf-8; mode: rst -*-
-
-************
-Introduction
-************
-
-Some video capture devices can sample a subsection of a picture and
-shrink or enlarge it to an image of arbitrary size. Next, the devices
-can insert the image into larger one. Some video output devices can crop
-part of an input image, scale it up or down and insert it at an
-arbitrary scan line and horizontal offset into a video signal. We call
-these abilities cropping, scaling and composing.
-
-On a video *capture* device the source is a video signal, and the
-cropping target determine the area actually sampled. The sink is an
-image stored in a memory buffer. The composing area specifies which part
-of the buffer is actually written to by the hardware.
-
-On a video *output* device the source is an image in a memory buffer,
-and the cropping target is a part of an image to be shown on a display.
-The sink is the display or the graphics screen. The application may
-select the part of display where the image should be displayed. The size
-and position of such a window is controlled by the compose target.
-
-Rectangles for all cropping and composing targets are defined even if
-the device does supports neither cropping nor composing. Their size and
-position will be fixed in such a case. If the device does not support
-scaling then the cropping and composing rectangles have the same size.
diff --git a/Documentation/media/uapi/v4l/selection-api-003.rst b/Documentation/media/uapi/v4l/selection-api-003.rst
deleted file mode 100644
index bf7e76dfbdf9..000000000000
--- a/Documentation/media/uapi/v4l/selection-api-003.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. -*- coding: utf-8; mode: rst -*-
-
-*****************
-Selection targets
-*****************
-
-
-.. _sel-targets-capture:
-
-.. kernel-figure:: selection.svg
-    :alt:   selection.svg
-    :align: center
-
-    Cropping and composing targets
-
-    Targets used by a cropping, composing and scaling process
-
-
-
-See :ref:`v4l2-selection-targets` for more information.
diff --git a/Documentation/media/uapi/v4l/selection-api-004.rst b/Documentation/media/uapi/v4l/selection-api-004.rst
deleted file mode 100644
index d782cd5b2117..000000000000
--- a/Documentation/media/uapi/v4l/selection-api-004.rst
+++ /dev/null
@@ -1,137 +0,0 @@
-.. -*- coding: utf-8; mode: rst -*-
-
-*************
-Configuration
-*************
-
-Applications can use the :ref:`selection API <VIDIOC_G_SELECTION>` to
-select an area in a video signal or a buffer, and to query for default
-settings and hardware limits.
-
-Video hardware can have various cropping, composing and scaling
-limitations. It may only scale up or down, support only discrete scaling
-factors, or have different scaling abilities in the horizontal and
-vertical directions. Also it may not support scaling at all. At the same
-time the cropping/composing rectangles may have to be aligned, and both
-the source and the sink may have arbitrary upper and lower size limits.
-Therefore, as usual, drivers are expected to adjust the requested
-parameters and return the actual values selected. An application can
-control the rounding behaviour using
-:ref:`constraint flags <v4l2-selection-flags>`.
-
-
-Configuration of video capture
-==============================
-
-See figure :ref:`sel-targets-capture` for examples of the selection
-targets available for a video capture device. It is recommended to
-configure the cropping targets before to the composing targets.
-
-The range of coordinates of the top left corner, width and height of
-areas that can be sampled is given by the ``V4L2_SEL_TGT_CROP_BOUNDS``
-target. It is recommended for the driver developers to put the top/left
-corner at position ``(0,0)``. The rectangle's coordinates are expressed
-in pixels.
-
-The top left corner, width and height of the source rectangle, that is
-the area actually sampled, is given by the ``V4L2_SEL_TGT_CROP`` target.
-It uses the same coordinate system as ``V4L2_SEL_TGT_CROP_BOUNDS``. The
-active cropping area must lie completely inside the capture boundaries.
-The driver may further adjust the requested size and/or position
-according to hardware limitations.
-
-Each capture device has a default source rectangle, given by the
-``V4L2_SEL_TGT_CROP_DEFAULT`` target. This rectangle shall over what the
-driver writer considers the complete picture. Drivers shall set the
-active crop rectangle to the default when the driver is first loaded,
-but not later.
-
-The composing targets refer to a memory buffer. The limits of composing
-coordinates are obtained using ``V4L2_SEL_TGT_COMPOSE_BOUNDS``. All
-coordinates are expressed in pixels. The rectangle's top/left corner
-must be located at position ``(0,0)``. The width and height are equal to
-the image size set by :ref:`VIDIOC_S_FMT <VIDIOC_G_FMT>`.
-
-The part of a buffer into which the image is inserted by the hardware is
-controlled by the ``V4L2_SEL_TGT_COMPOSE`` target. The rectangle's
-coordinates are also expressed in the same coordinate system as the
-bounds rectangle. The composing rectangle must lie completely inside
-bounds rectangle. The driver must adjust the composing rectangle to fit
-to the bounding limits. Moreover, the driver can perform other
-adjustments according to hardware limitations. The application can
-control rounding behaviour using
-:ref:`constraint flags <v4l2-selection-flags>`.
-
-For capture devices the default composing rectangle is queried using
-``V4L2_SEL_TGT_COMPOSE_DEFAULT``. It is usually equal to the bounding
-rectangle.
-
-The part of a buffer that is modified by the hardware is given by
-``V4L2_SEL_TGT_COMPOSE_PADDED``. It contains all pixels defined using
-``V4L2_SEL_TGT_COMPOSE`` plus all padding data modified by hardware
-during insertion process. All pixels outside this rectangle *must not*
-be changed by the hardware. The content of pixels that lie inside the
-padded area but outside active area is undefined. The application can
-use the padded and active rectangles to detect where the rubbish pixels
-are located and remove them if needed.
-
-
-Configuration of video output
-=============================
-
-For output devices targets and ioctls are used similarly to the video
-capture case. The *composing* rectangle refers to the insertion of an
-image into a video signal. The cropping rectangles refer to a memory
-buffer. It is recommended to configure the composing targets before to
-the cropping targets.
-
-The cropping targets refer to the memory buffer that contains an image
-to be inserted into a video signal or graphical screen. The limits of
-cropping coordinates are obtained using ``V4L2_SEL_TGT_CROP_BOUNDS``.
-All coordinates are expressed in pixels. The top/left corner is always
-point ``(0,0)``. The width and height is equal to the image size
-specified using :ref:`VIDIOC_S_FMT <VIDIOC_G_FMT>` ioctl.
-
-The top left corner, width and height of the source rectangle, that is
-the area from which image date are processed by the hardware, is given
-by the ``V4L2_SEL_TGT_CROP``. Its coordinates are expressed in in the
-same coordinate system as the bounds rectangle. The active cropping area
-must lie completely inside the crop boundaries and the driver may
-further adjust the requested size and/or position according to hardware
-limitations.
-
-For output devices the default cropping rectangle is queried using
-``V4L2_SEL_TGT_CROP_DEFAULT``. It is usually equal to the bounding
-rectangle.
-
-The part of a video signal or graphics display where the image is
-inserted by the hardware is controlled by ``V4L2_SEL_TGT_COMPOSE``
-target. The rectangle's coordinates are expressed in pixels. The
-composing rectangle must lie completely inside the bounds rectangle. The
-driver must adjust the area to fit to the bounding limits. Moreover, the
-driver can perform other adjustments according to hardware limitations.
-
-The device has a default composing rectangle, given by the
-``V4L2_SEL_TGT_COMPOSE_DEFAULT`` target. This rectangle shall cover what
-the driver writer considers the complete picture. It is recommended for
-the driver developers to put the top/left corner at position ``(0,0)``.
-Drivers shall set the active composing rectangle to the default one when
-the driver is first loaded.
-
-The devices may introduce additional content to video signal other than
-an image from memory buffers. It includes borders around an image.
-However, such a padded area is driver-dependent feature not covered by
-this document. Driver developers are encouraged to keep padded rectangle
-equal to active one. The padded target is accessed by the
-``V4L2_SEL_TGT_COMPOSE_PADDED`` identifier. It must contain all pixels
-from the ``V4L2_SEL_TGT_COMPOSE`` target.
-
-
-Scaling control
-===============
-
-An application can detect if scaling is performed by comparing the width
-and the height of rectangles obtained using ``V4L2_SEL_TGT_CROP`` and
-``V4L2_SEL_TGT_COMPOSE`` targets. If these are not equal then the
-scaling is applied. The application can compute the scaling ratios using
-these values.
diff --git a/Documentation/media/uapi/v4l/selection-api-005.rst b/Documentation/media/uapi/v4l/selection-api-005.rst
deleted file mode 100644
index 5b47a28ac6d7..000000000000
--- a/Documentation/media/uapi/v4l/selection-api-005.rst
+++ /dev/null
@@ -1,34 +0,0 @@
-.. -*- coding: utf-8; mode: rst -*-
-
-********************************
-Comparison with old cropping API
-********************************
-
-The selection API was introduced to cope with deficiencies of previous
-:ref:`API <crop>`, that was designed to control simple capture
-devices. Later the cropping API was adopted by video output drivers. The
-ioctls are used to select a part of the display were the video signal is
-inserted. It should be considered as an API abuse because the described
-operation is actually the composing. The selection API makes a clear
-distinction between composing and cropping operations by setting the
-appropriate targets. The V4L2 API lacks any support for composing to and
-cropping from an image inside a memory buffer. The application could
-configure a capture device to fill only a part of an image by abusing
-V4L2 API. Cropping a smaller image from a larger one is achieved by
-setting the field ``bytesperline`` at struct
-:c:type:`v4l2_pix_format`.
-Introducing an image offsets could be done by modifying field ``m_userptr``
-at struct
-:c:type:`v4l2_buffer` before calling
-:ref:`VIDIOC_QBUF`. Those operations should be avoided because they are not
-portable (endianness), and do not work for macroblock and Bayer formats
-and mmap buffers. The selection API deals with configuration of buffer
-cropping/composing in a clear, intuitive and portable way. Next, with
-the selection API the concepts of the padded target and constraints
-flags are introduced. Finally, struct :c:type:`v4l2_crop`
-and struct :c:type:`v4l2_cropcap` have no reserved
-fields. Therefore there is no way to extend their functionality. The new
-struct :c:type:`v4l2_selection` provides a lot of place
-for future extensions. Driver developers are encouraged to implement
-only selection API. The former cropping API would be simulated using the
-new one.
diff --git a/Documentation/media/uapi/v4l/selection-api-006.rst b/Documentation/media/uapi/v4l/selection-api-006.rst
deleted file mode 100644
index 67e0e9aed9e8..000000000000
--- a/Documentation/media/uapi/v4l/selection-api-006.rst
+++ /dev/null
@@ -1,84 +0,0 @@
-.. -*- coding: utf-8; mode: rst -*-
-
-********
-Examples
-********
-
-(A video capture device is assumed; change
-``V4L2_BUF_TYPE_VIDEO_CAPTURE`` for other devices; change target to
-``V4L2_SEL_TGT_COMPOSE_*`` family to configure composing area)
-
-Example: Resetting the cropping parameters
-==========================================
-
-.. code-block:: c
-
-	struct v4l2_selection sel = {
-	    .type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
-	    .target = V4L2_SEL_TGT_CROP_DEFAULT,
-	};
-	ret = ioctl(fd, VIDIOC_G_SELECTION, &sel);
-	if (ret)
-	    exit(-1);
-	sel.target = V4L2_SEL_TGT_CROP;
-	ret = ioctl(fd, VIDIOC_S_SELECTION, &sel);
-	if (ret)
-	    exit(-1);
-
-Setting a composing area on output of size of *at most* half of limit
-placed at a center of a display.
-
-Example: Simple downscaling
-===========================
-
-.. code-block:: c
-
-	struct v4l2_selection sel = {
-	    .type = V4L2_BUF_TYPE_VIDEO_OUTPUT,
-	    .target = V4L2_SEL_TGT_COMPOSE_BOUNDS,
-	};
-	struct v4l2_rect r;
-
-	ret = ioctl(fd, VIDIOC_G_SELECTION, &sel);
-	if (ret)
-	    exit(-1);
-	/* setting smaller compose rectangle */
-	r.width = sel.r.width / 2;
-	r.height = sel.r.height / 2;
-	r.left = sel.r.width / 4;
-	r.top = sel.r.height / 4;
-	sel.r = r;
-	sel.target = V4L2_SEL_TGT_COMPOSE;
-	sel.flags = V4L2_SEL_FLAG_LE;
-	ret = ioctl(fd, VIDIOC_S_SELECTION, &sel);
-	if (ret)
-	    exit(-1);
-
-A video output device is assumed; change ``V4L2_BUF_TYPE_VIDEO_OUTPUT``
-for other devices
-
-Example: Querying for scaling factors
-=====================================
-
-.. code-block:: c
-
-	struct v4l2_selection compose = {
-	    .type = V4L2_BUF_TYPE_VIDEO_OUTPUT,
-	    .target = V4L2_SEL_TGT_COMPOSE,
-	};
-	struct v4l2_selection crop = {
-	    .type = V4L2_BUF_TYPE_VIDEO_OUTPUT,
-	    .target = V4L2_SEL_TGT_CROP,
-	};
-	double hscale, vscale;
-
-	ret = ioctl(fd, VIDIOC_G_SELECTION, &compose);
-	if (ret)
-	    exit(-1);
-	ret = ioctl(fd, VIDIOC_G_SELECTION, &crop);
-	if (ret)
-	    exit(-1);
-
-	/* computing scaling factors */
-	hscale = (double)compose.r.width / crop.r.width;
-	vscale = (double)compose.r.height / crop.r.height;
diff --git a/Documentation/media/uapi/v4l/selection-api-configuration.rst b/Documentation/media/uapi/v4l/selection-api-configuration.rst
new file mode 100644
index 000000000000..0a4ddc2d71db
--- /dev/null
+++ b/Documentation/media/uapi/v4l/selection-api-configuration.rst
@@ -0,0 +1,137 @@
+.. -*- coding: utf-8; mode: rst -*-
+
+*************
+Configuration
+*************
+
+Applications can use the :ref:`selection API <VIDIOC_G_SELECTION>` to
+select an area in a video signal or a buffer, and to query for default
+settings and hardware limits.
+
+Video hardware can have various cropping, composing and scaling
+limitations. It may only scale up or down, support only discrete scaling
+factors, or have different scaling abilities in the horizontal and
+vertical directions. Also it may not support scaling at all. At the same
+time the cropping/composing rectangles may have to be aligned, and both
+the source and the sink may have arbitrary upper and lower size limits.
+Therefore, as usual, drivers are expected to adjust the requested
+parameters and return the actual values selected. An application can
+control the rounding behaviour using
+:ref:`constraint flags <v4l2-selection-flags>`.
+
+
+Configuration of video capture
+==============================
+
+See figure :ref:`sel-targets-capture` for examples of the selection
+targets available for a video capture device. It is recommended to
+configure the cropping targets before to the composing targets.
+
+The range of coordinates of the top left corner, width and height of
+areas that can be sampled is given by the ``V4L2_SEL_TGT_CROP_BOUNDS``
+target. It is recommended for the driver developers to put the top/left
+corner at position ``(0,0)``. The rectangle's coordinates are expressed
+in pixels.
+
+The top left corner, width and height of the source rectangle, that is
+the area actually sampled, is given by the ``V4L2_SEL_TGT_CROP`` target.
+It uses the same coordinate system as ``V4L2_SEL_TGT_CROP_BOUNDS``. The
+active cropping area must lie completely inside the capture boundaries.
+The driver may further adjust the requested size and/or position
+according to hardware limitations.
+
+Each capture device has a default source rectangle, given by the
+``V4L2_SEL_TGT_CROP_DEFAULT`` target. This rectangle shall cover what the
+driver writer considers the complete picture. Drivers shall set the
+active crop rectangle to the default when the driver is first loaded,
+but not later.
+
+The composing targets refer to a memory buffer. The limits of composing
+coordinates are obtained using ``V4L2_SEL_TGT_COMPOSE_BOUNDS``. All
+coordinates are expressed in pixels. The rectangle's top/left corner
+must be located at position ``(0,0)``. The width and height are equal to
+the image size set by :ref:`VIDIOC_S_FMT <VIDIOC_G_FMT>`.
+
+The part of a buffer into which the image is inserted by the hardware is
+controlled by the ``V4L2_SEL_TGT_COMPOSE`` target. The rectangle's
+coordinates are also expressed in the same coordinate system as the
+bounds rectangle. The composing rectangle must lie completely inside
+bounds rectangle. The driver must adjust the composing rectangle to fit
+to the bounding limits. Moreover, the driver can perform other
+adjustments according to hardware limitations. The application can
+control rounding behaviour using
+:ref:`constraint flags <v4l2-selection-flags>`.
+
+For capture devices the default composing rectangle is queried using
+``V4L2_SEL_TGT_COMPOSE_DEFAULT``. It is usually equal to the bounding
+rectangle.
+
+The part of a buffer that is modified by the hardware is given by
+``V4L2_SEL_TGT_COMPOSE_PADDED``. It contains all pixels defined using
+``V4L2_SEL_TGT_COMPOSE`` plus all padding data modified by hardware
+during insertion process. All pixels outside this rectangle *must not*
+be changed by the hardware. The content of pixels that lie inside the
+padded area but outside active area is undefined. The application can
+use the padded and active rectangles to detect where the rubbish pixels
+are located and remove them if needed.
+
+
+Configuration of video output
+=============================
+
+For output devices targets and ioctls are used similarly to the video
+capture case. The *composing* rectangle refers to the insertion of an
+image into a video signal. The cropping rectangles refer to a memory
+buffer. It is recommended to configure the composing targets before to
+the cropping targets.
+
+The cropping targets refer to the memory buffer that contains an image
+to be inserted into a video signal or graphical screen. The limits of
+cropping coordinates are obtained using ``V4L2_SEL_TGT_CROP_BOUNDS``.
+All coordinates are expressed in pixels. The top/left corner is always
+point ``(0,0)``. The width and height is equal to the image size
+specified using :ref:`VIDIOC_S_FMT <VIDIOC_G_FMT>` ioctl.
+
+The top left corner, width and height of the source rectangle, that is
+the area from which image date are processed by the hardware, is given
+by the ``V4L2_SEL_TGT_CROP``. Its coordinates are expressed in in the
+same coordinate system as the bounds rectangle. The active cropping area
+must lie completely inside the crop boundaries and the driver may
+further adjust the requested size and/or position according to hardware
+limitations.
+
+For output devices the default cropping rectangle is queried using
+``V4L2_SEL_TGT_CROP_DEFAULT``. It is usually equal to the bounding
+rectangle.
+
+The part of a video signal or graphics display where the image is
+inserted by the hardware is controlled by ``V4L2_SEL_TGT_COMPOSE``
+target. The rectangle's coordinates are expressed in pixels. The
+composing rectangle must lie completely inside the bounds rectangle. The
+driver must adjust the area to fit to the bounding limits. Moreover, the
+driver can perform other adjustments according to hardware limitations.
+
+The device has a default composing rectangle, given by the
+``V4L2_SEL_TGT_COMPOSE_DEFAULT`` target. This rectangle shall cover what
+the driver writer considers the complete picture. It is recommended for
+the driver developers to put the top/left corner at position ``(0,0)``.
+Drivers shall set the active composing rectangle to the default one when
+the driver is first loaded.
+
+The devices may introduce additional content to video signal other than
+an image from memory buffers. It includes borders around an image.
+However, such a padded area is driver-dependent feature not covered by
+this document. Driver developers are encouraged to keep padded rectangle
+equal to active one. The padded target is accessed by the
+``V4L2_SEL_TGT_COMPOSE_PADDED`` identifier. It must contain all pixels
+from the ``V4L2_SEL_TGT_COMPOSE`` target.
+
+
+Scaling control
+===============
+
+An application can detect if scaling is performed by comparing the width
+and the height of rectangles obtained using ``V4L2_SEL_TGT_CROP`` and
+``V4L2_SEL_TGT_COMPOSE`` targets. If these are not equal then the
+scaling is applied. The application can compute the scaling ratios using
+these values.
diff --git a/Documentation/media/uapi/v4l/selection-api-examples.rst b/Documentation/media/uapi/v4l/selection-api-examples.rst
new file mode 100644
index 000000000000..67e0e9aed9e8
--- /dev/null
+++ b/Documentation/media/uapi/v4l/selection-api-examples.rst
@@ -0,0 +1,84 @@
+.. -*- coding: utf-8; mode: rst -*-
+
+********
+Examples
+********
+
+(A video capture device is assumed; change
+``V4L2_BUF_TYPE_VIDEO_CAPTURE`` for other devices; change target to
+``V4L2_SEL_TGT_COMPOSE_*`` family to configure composing area)
+
+Example: Resetting the cropping parameters
+==========================================
+
+.. code-block:: c
+
+	struct v4l2_selection sel = {
+	    .type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
+	    .target = V4L2_SEL_TGT_CROP_DEFAULT,
+	};
+	ret = ioctl(fd, VIDIOC_G_SELECTION, &sel);
+	if (ret)
+	    exit(-1);
+	sel.target = V4L2_SEL_TGT_CROP;
+	ret = ioctl(fd, VIDIOC_S_SELECTION, &sel);
+	if (ret)
+	    exit(-1);
+
+Setting a composing area on output of size of *at most* half of limit
+placed at a center of a display.
+
+Example: Simple downscaling
+===========================
+
+.. code-block:: c
+
+	struct v4l2_selection sel = {
+	    .type = V4L2_BUF_TYPE_VIDEO_OUTPUT,
+	    .target = V4L2_SEL_TGT_COMPOSE_BOUNDS,
+	};
+	struct v4l2_rect r;
+
+	ret = ioctl(fd, VIDIOC_G_SELECTION, &sel);
+	if (ret)
+	    exit(-1);
+	/* setting smaller compose rectangle */
+	r.width = sel.r.width / 2;
+	r.height = sel.r.height / 2;
+	r.left = sel.r.width / 4;
+	r.top = sel.r.height / 4;
+	sel.r = r;
+	sel.target = V4L2_SEL_TGT_COMPOSE;
+	sel.flags = V4L2_SEL_FLAG_LE;
+	ret = ioctl(fd, VIDIOC_S_SELECTION, &sel);
+	if (ret)
+	    exit(-1);
+
+A video output device is assumed; change ``V4L2_BUF_TYPE_VIDEO_OUTPUT``
+for other devices
+
+Example: Querying for scaling factors
+=====================================
+
+.. code-block:: c
+
+	struct v4l2_selection compose = {
+	    .type = V4L2_BUF_TYPE_VIDEO_OUTPUT,
+	    .target = V4L2_SEL_TGT_COMPOSE,
+	};
+	struct v4l2_selection crop = {
+	    .type = V4L2_BUF_TYPE_VIDEO_OUTPUT,
+	    .target = V4L2_SEL_TGT_CROP,
+	};
+	double hscale, vscale;
+
+	ret = ioctl(fd, VIDIOC_G_SELECTION, &compose);
+	if (ret)
+	    exit(-1);
+	ret = ioctl(fd, VIDIOC_G_SELECTION, &crop);
+	if (ret)
+	    exit(-1);
+
+	/* computing scaling factors */
+	hscale = (double)compose.r.width / crop.r.width;
+	vscale = (double)compose.r.height / crop.r.height;
diff --git a/Documentation/media/uapi/v4l/selection-api-intro.rst b/Documentation/media/uapi/v4l/selection-api-intro.rst
new file mode 100644
index 000000000000..09ca93f91bf7
--- /dev/null
+++ b/Documentation/media/uapi/v4l/selection-api-intro.rst
@@ -0,0 +1,28 @@
+.. -*- coding: utf-8; mode: rst -*-
+
+************
+Introduction
+************
+
+Some video capture devices can sample a subsection of a picture and
+shrink or enlarge it to an image of arbitrary size. Next, the devices
+can insert the image into larger one. Some video output devices can crop
+part of an input image, scale it up or down and insert it at an
+arbitrary scan line and horizontal offset into a video signal. We call
+these abilities cropping, scaling and composing.
+
+On a video *capture* device the source is a video signal, and the
+cropping target determine the area actually sampled. The sink is an
+image stored in a memory buffer. The composing area specifies which part
+of the buffer is actually written to by the hardware.
+
+On a video *output* device the source is an image in a memory buffer,
+and the cropping target is a part of an image to be shown on a display.
+The sink is the display or the graphics screen. The application may
+select the part of display where the image should be displayed. The size
+and position of such a window is controlled by the compose target.
+
+Rectangles for all cropping and composing targets are defined even if
+the device does supports neither cropping nor composing. Their size and
+position will be fixed in such a case. If the device does not support
+scaling then the cropping and composing rectangles have the same size.
diff --git a/Documentation/media/uapi/v4l/selection-api-targets.rst b/Documentation/media/uapi/v4l/selection-api-targets.rst
new file mode 100644
index 000000000000..bf7e76dfbdf9
--- /dev/null
+++ b/Documentation/media/uapi/v4l/selection-api-targets.rst
@@ -0,0 +1,20 @@
+.. -*- coding: utf-8; mode: rst -*-
+
+*****************
+Selection targets
+*****************
+
+
+.. _sel-targets-capture:
+
+.. kernel-figure:: selection.svg
+    :alt:   selection.svg
+    :align: center
+
+    Cropping and composing targets
+
+    Targets used by a cropping, composing and scaling process
+
+
+
+See :ref:`v4l2-selection-targets` for more information.
diff --git a/Documentation/media/uapi/v4l/selection-api-vs-crop-api.rst b/Documentation/media/uapi/v4l/selection-api-vs-crop-api.rst
new file mode 100644
index 000000000000..e7455fb1e572
--- /dev/null
+++ b/Documentation/media/uapi/v4l/selection-api-vs-crop-api.rst
@@ -0,0 +1,39 @@
+.. -*- coding: utf-8; mode: rst -*-
+
+.. _selection-vs-crop:
+
+********************************
+Comparison with old cropping API
+********************************
+
+The selection API was introduced to cope with deficiencies of the
+older :ref:`CROP API <crop>`, that was designed to control simple
+capture devices. Later the cropping API was adopted by video output
+drivers. The ioctls are used to select a part of the display were the
+video signal is inserted. It should be considered as an API abuse
+because the described operation is actually the composing. The
+selection API makes a clear distinction between composing and cropping
+operations by setting the appropriate targets.
+
+The CROP API lacks any support for composing to and cropping from an
+image inside a memory buffer. The application could configure a
+capture device to fill only a part of an image by abusing V4L2
+API. Cropping a smaller image from a larger one is achieved by setting
+the field ``bytesperline`` at struct :c:type:`v4l2_pix_format`.
+Introducing an image offsets could be done by modifying field
+``m_userptr`` at struct :c:type:`v4l2_buffer` before calling
+:ref:`VIDIOC_QBUF <VIDIOC_QBUF>`. Those operations should be avoided
+because they are not portable (endianness), and do not work for
+macroblock and Bayer formats and mmap buffers.
+
+The selection API deals with configuration of buffer
+cropping/composing in a clear, intuitive and portable way. Next, with
+the selection API the concepts of the padded target and constraints
+flags are introduced. Finally, struct :c:type:`v4l2_crop` and struct
+:c:type:`v4l2_cropcap` have no reserved fields. Therefore there is no
+way to extend their functionality. The new struct
+:c:type:`v4l2_selection` provides a lot of place for future
+extensions.
+
+Driver developers are encouraged to implement only selection API. The
+former cropping API would be simulated using the new one.
diff --git a/Documentation/media/uapi/v4l/selection-api.rst b/Documentation/media/uapi/v4l/selection-api.rst
index 81ea52d785b9..390233f704a3 100644
--- a/Documentation/media/uapi/v4l/selection-api.rst
+++ b/Documentation/media/uapi/v4l/selection-api.rst
@@ -2,15 +2,15 @@
 
 .. _selection-api:
 
-API for cropping, composing and scaling
-=======================================
+Cropping, composing and scaling -- the SELECTION API
+====================================================
 
 
 .. toctree::
     :maxdepth: 1
 
-    selection-api-002
-    selection-api-003
-    selection-api-004
-    selection-api-005
-    selection-api-006
+    selection-api-intro.rst
+    selection-api-targets.rst
+    selection-api-configuration.rst
+    selection-api-vs-crop-api.rst
+    selection-api-examples.rst
diff --git a/Documentation/media/uapi/v4l/selection.svg b/Documentation/media/uapi/v4l/selection.svg
index a93e3b59786d..911062bd2844 100644
--- a/Documentation/media/uapi/v4l/selection.svg
+++ b/Documentation/media/uapi/v4l/selection.svg
@@ -1128,11 +1128,11 @@
    </text>
   </g>
   <text transform="matrix(.96106 0 0 1.0405 48.571 195.53)" x="2438.062" y="1368.429" enable-background="new" font-size="50" style="line-height:125%">
-   <tspan x="2438.062" y="1368.429">COMPOSE_BONDS</tspan>
+   <tspan x="2438.062" y="1368.429">COMPOSE_BOUNDS</tspan>
   </text>
   <g font-size="40">
    <text transform="translate(48.571 195.53)" x="8.082" y="1438.896" enable-background="new" style="line-height:125%">
-    <tspan x="8.082" y="1438.896" font-size="50">CROP_BONDS</tspan>
+    <tspan x="8.082" y="1438.896" font-size="50">CROP_BOUNDS</tspan>
    </text>
    <text transform="translate(48.571 195.53)" x="1455.443" y="-26.808" enable-background="new" style="line-height:125%">
     <tspan x="1455.443" y="-26.808" font-size="50">overscan area</tspan>
diff --git a/Documentation/media/uapi/v4l/v4l2.rst b/Documentation/media/uapi/v4l/v4l2.rst
index 2128717299b3..b89e5621ae69 100644
--- a/Documentation/media/uapi/v4l/v4l2.rst
+++ b/Documentation/media/uapi/v4l/v4l2.rst
@@ -45,7 +45,7 @@ Authors, in alphabetical order:
 
   - Subdev selections API.
 
-- Carvalho Chehab, Mauro <m.chehab@kernel.org>
+- Carvalho Chehab, Mauro <mchehab+samsung@kernel.org>
 
   - Documented libv4l, designed and added v4l2grab example, Remote Controller chapter.
 
diff --git a/Documentation/media/v4l-drivers/cx23885-cardlist.rst b/Documentation/media/v4l-drivers/cx23885-cardlist.rst
index 3129ef04ddd3..8c24df8e0423 100644
--- a/Documentation/media/v4l-drivers/cx23885-cardlist.rst
+++ b/Documentation/media/v4l-drivers/cx23885-cardlist.rst
@@ -186,7 +186,7 @@ cx23885 cards list
 
    * - 43
      - Hauppauge ImpactVCB-e
-     - 0070:7133
+     - 0070:7133, 0070:7137
 
    * - 44
      - DViCO FusionHDTV DVB-T Dual Express2
@@ -243,3 +243,19 @@ cx23885 cards list
    * - 57
      - Hauppauge WinTV-QuadHD-ATSC
      - 0070:6a18, 0070:6b18
+
+   * - 58
+     - Hauppauge WinTV-HVR-1265(161111)
+     - 0070:2a18
+
+   * - 59
+     - Hauppauge WinTV-Starburst2
+     - 0070:f02a
+
+   * - 60
+     - Hauppauge WinTV-QuadHD-DVB(885)
+     -
+
+   * - 61
+     - Hauppauge WinTV-QuadHD-ATSC(885)
+     -
diff --git a/Documentation/media/v4l-drivers/em28xx-cardlist.rst b/Documentation/media/v4l-drivers/em28xx-cardlist.rst
index ec938c08f43d..dfe882ca945f 100644
--- a/Documentation/media/v4l-drivers/em28xx-cardlist.rst
+++ b/Documentation/media/v4l-drivers/em28xx-cardlist.rst
@@ -391,7 +391,7 @@ EM28xx cards list
    * - 94
      - PCTV tripleStick (292e)
      - em28178
-     - 2013:025f, 2040:0264
+     - 2013:025f, 2013:0264, 2040:0264, 2040:8264, 2040:8268, 2040:8268
    * - 95
      - Leadtek VC100
      - em2861
@@ -411,12 +411,16 @@ EM28xx cards list
    * - 99
      - Hauppauge WinTV-dualHD DVB
      - em28174
-     - 2040:0265
+     - 2040:0265, 2040:8265
    * - 100
      - Hauppauge WinTV-dualHD 01595 ATSC/QAM
      - em28174
-     - 2040:026d
+     - 2040:026d, 2040:826d
    * - 101
      - Terratec Cinergy H6 rev. 2
      - em2884
      - 0ccd:10b2
+   * - 102
+     - :ZOLID HYBRID TV STICK
+     - em2882
+     -
diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt
index 6dafc8085acc..a02d6bbfc9d0 100644
--- a/Documentation/memory-barriers.txt
+++ b/Documentation/memory-barriers.txt
@@ -1920,9 +1920,6 @@ There are some more advanced barrier functions:
 		/* assign ownership */
 		desc->status = DEVICE_OWN;
 
-		/* force memory to sync before notifying device via MMIO */
-		wmb();
-
 		/* notify device of new descriptors */
 		writel(DESC_NOTIFY, doorbell);
 	}
@@ -1930,11 +1927,15 @@ There are some more advanced barrier functions:
      The dma_rmb() allows us guarantee the device has released ownership
      before we read the data from the descriptor, and the dma_wmb() allows
      us to guarantee the data is written to the descriptor before the device
-     can see it now has ownership.  The wmb() is needed to guarantee that the
-     cache coherent memory writes have completed before attempting a write to
-     the cache incoherent MMIO region.
+     can see it now has ownership.  Note that, when using writel(), a prior
+     wmb() is not needed to guarantee that the cache coherent memory writes
+     have completed before writing to the MMIO region.  The cheaper
+     writel_relaxed() does not provide this guarantee and must not be used
+     here.
 
-     See Documentation/DMA-API.txt for more information on consistent memory.
+     See the subsection "Kernel I/O barrier effects" for more information on
+     relaxed I/O accessors and the Documentation/DMA-API.txt file for more
+     information on consistent memory.
 
 
 MMIO WRITE BARRIER
@@ -2903,7 +2904,7 @@ is discarded from the CPU's cache and reloaded.  To deal with this, the
 appropriate part of the kernel must invalidate the overlapping bits of the
 cache on each CPU.
 
-See Documentation/cachetlb.txt for more information on cache management.
+See Documentation/core-api/cachetlb.rst for more information on cache management.
 
 
 CACHE COHERENCY VS MMIO
@@ -3083,7 +3084,7 @@ CIRCULAR BUFFERS
 Memory barriers can be used to implement circular buffering without the need
 of a lock to serialise the producer with the consumer.  See:
 
-	Documentation/circular-buffers.txt
+	Documentation/core-api/circular-buffers.rst
 
 for details.
 
diff --git a/Documentation/misc-devices/ibmvmc.rst b/Documentation/misc-devices/ibmvmc.rst
new file mode 100644
index 000000000000..46ded79554d4
--- /dev/null
+++ b/Documentation/misc-devices/ibmvmc.rst
@@ -0,0 +1,226 @@
+.. SPDX-License-Identifier: GPL-2.0+
+======================================================
+IBM Virtual Management Channel Kernel Driver (IBMVMC)
+======================================================
+
+:Authors:
+	Dave Engebretsen <engebret@us.ibm.com>,
+	Adam Reznechek <adreznec@linux.vnet.ibm.com>,
+	Steven Royer <seroyer@linux.vnet.ibm.com>,
+	Bryant G. Ly <bryantly@linux.vnet.ibm.com>,
+
+Introduction
+============
+
+Note: Knowledge of virtualization technology is required to understand
+this document.
+
+A good reference document would be:
+
+https://openpowerfoundation.org/wp-content/uploads/2016/05/LoPAPR_DRAFT_v11_24March2016_cmt1.pdf
+
+The Virtual Management Channel (VMC) is a logical device which provides an
+interface between the hypervisor and a management partition. This interface
+is like a message passing interface. This management partition is intended
+to provide an alternative to systems that use a Hardware Management
+Console (HMC) - based system management.
+
+The primary hardware management solution that is developed by IBM relies
+on an appliance server named the Hardware Management Console (HMC),
+packaged as an external tower or rack-mounted personal computer. In a
+Power Systems environment, a single HMC can manage multiple POWER
+processor-based systems.
+
+Management Application
+----------------------
+
+In the management partition, a management application exists which enables
+a system administrator to configure the system’s partitioning
+characteristics via a command line interface (CLI) or Representational
+State Transfer Application (REST API's).
+
+The management application runs on a Linux logical partition on a
+POWER8 or newer processor-based server that is virtualized by PowerVM.
+System configuration, maintenance, and control functions which
+traditionally require an HMC can be implemented in the management
+application using a combination of HMC to hypervisor interfaces and
+existing operating system methods. This tool provides a subset of the
+functions implemented by the HMC and enables basic partition configuration.
+The set of HMC to hypervisor messages supported by the management
+application component are passed to the hypervisor over a VMC interface,
+which is defined below.
+
+The VMC enables the management partition to provide basic partitioning
+functions:
+
+- Logical Partitioning Configuration
+- Start, and stop actions for individual partitions
+- Display of partition status
+- Management of virtual Ethernet
+- Management of virtual Storage
+- Basic system management
+
+Virtual Management Channel (VMC)
+--------------------------------
+
+A logical device, called the Virtual Management Channel (VMC), is defined
+for communicating between the management application and the hypervisor. It
+basically creates the pipes that enable virtualization management
+software. This device is presented to a designated management partition as
+a virtual device.
+
+This communication device uses Command/Response Queue (CRQ) and the
+Remote Direct Memory Access (RDMA) interfaces. A three-way handshake is
+defined that must take place to establish that both the hypervisor and
+management partition sides of the channel are running prior to
+sending/receiving any of the protocol messages.
+
+This driver also utilizes Transport Event CRQs. CRQ messages are sent
+when the hypervisor detects one of the peer partitions has abnormally
+terminated, or one side has called H_FREE_CRQ to close their CRQ.
+Two new classes of CRQ messages are introduced for the VMC device. VMC
+Administrative messages are used for each partition using the VMC to
+communicate capabilities to their partner. HMC Interface messages are used
+for the actual flow of HMC messages between the management partition and
+the hypervisor. As most HMC messages far exceed the size of a CRQ buffer,
+a virtual DMA (RMDA) of the HMC message data is done prior to each HMC
+Interface CRQ message. Only the management partition drives RDMA
+operations; hypervisors never directly cause the movement of message data.
+
+
+Terminology
+-----------
+RDMA
+        Remote Direct Memory Access is DMA transfer from the server to its
+        client or from the server to its partner partition. DMA refers
+        to both physical I/O to and from memory operations and to memory
+        to memory move operations.
+CRQ
+        Command/Response Queue a facility which is used to communicate
+        between partner partitions. Transport events which are signaled
+        from the hypervisor to partition are also reported in this queue.
+
+Example Management Partition VMC Driver Interface
+=================================================
+
+This section provides an example for the management application
+implementation where a device driver is used to interface to the VMC
+device. This driver consists of a new device, for example /dev/ibmvmc,
+which provides interfaces to open, close, read, write, and perform
+ioctl’s against the VMC device.
+
+VMC Interface Initialization
+----------------------------
+
+The device driver is responsible for initializing the VMC when the driver
+is loaded. It first creates and initializes the CRQ. Next, an exchange of
+VMC capabilities is performed to indicate the code version and number of
+resources available in both the management partition and the hypervisor.
+Finally, the hypervisor requests that the management partition create an
+initial pool of VMC buffers, one buffer for each possible HMC connection,
+which will be used for management application  session initialization.
+Prior to completion of this initialization sequence, the device returns
+EBUSY to open() calls. EIO is returned for all open() failures.
+
+::
+
+        Management Partition		Hypervisor
+                        CRQ INIT
+        ---------------------------------------->
+        	   CRQ INIT COMPLETE
+        <----------------------------------------
+        	      CAPABILITIES
+        ---------------------------------------->
+        	 CAPABILITIES RESPONSE
+        <----------------------------------------
+              ADD BUFFER (HMC IDX=0,1,..)         _
+        <----------------------------------------  |
+        	  ADD BUFFER RESPONSE              | - Perform # HMCs Iterations
+        ----------------------------------------> -
+
+VMC Interface Open
+------------------
+
+After the basic VMC channel has been initialized, an HMC session level
+connection can be established. The application layer performs an open() to
+the VMC device and executes an ioctl() against it, indicating the HMC ID
+(32 bytes of data) for this session. If the VMC device is in an invalid
+state, EIO will be returned for the ioctl(). The device driver creates a
+new HMC session value (ranging from 1 to 255) and HMC index value (starting
+at index 0 and ranging to 254) for this HMC ID. The driver then does an
+RDMA of the HMC ID to the hypervisor, and then sends an Interface Open
+message to the hypervisor to establish the session over the VMC. After the
+hypervisor receives this information, it sends Add Buffer messages to the
+management partition to seed an initial pool of buffers for the new HMC
+connection. Finally, the hypervisor sends an Interface Open Response
+message, to indicate that it is ready for normal runtime messaging. The
+following illustrates this VMC flow:
+
+::
+
+        Management Partition             Hypervisor
+        	      RDMA HMC ID
+        ---------------------------------------->
+        	    Interface Open
+        ---------------------------------------->
+        	      Add Buffer                  _
+        <----------------------------------------  |
+        	  Add Buffer Response              | - Perform N Iterations
+        ----------------------------------------> -
+        	Interface Open Response
+        <----------------------------------------
+
+VMC Interface Runtime
+---------------------
+
+During normal runtime, the management application and the hypervisor
+exchange HMC messages via the Signal VMC message and RDMA operations. When
+sending data to the hypervisor, the management application performs a
+write() to the VMC device, and the driver RDMA’s the data to the hypervisor
+and then sends a Signal Message. If a write() is attempted before VMC
+device buffers have been made available by the hypervisor, or no buffers
+are currently available, EBUSY is returned in response to the write(). A
+write() will return EIO for all other errors, such as an invalid device
+state. When the hypervisor sends a message to the management, the data is
+put into a VMC buffer and an Signal Message is sent to the VMC driver in
+the management partition. The driver RDMA’s the buffer into the partition
+and passes the data up to the appropriate management application via a
+read() to the VMC device. The read() request blocks if there is no buffer
+available to read. The management application may use select() to wait for
+the VMC device to become ready with data to read.
+
+::
+
+        Management Partition             Hypervisor
+        		MSG RDMA
+        ---------------------------------------->
+        		SIGNAL MSG
+        ---------------------------------------->
+        		SIGNAL MSG
+        <----------------------------------------
+        		MSG RDMA
+        <----------------------------------------
+
+VMC Interface Close
+-------------------
+
+HMC session level connections are closed by the management partition when
+the application layer performs a close() against the device. This action
+results in an Interface Close message flowing to the hypervisor, which
+causes the session to be terminated. The device driver must free any
+storage allocated for buffers for this HMC connection.
+
+::
+
+        Management Partition             Hypervisor
+        	     INTERFACE CLOSE
+        ---------------------------------------->
+                INTERFACE CLOSE RESPONSE
+        <----------------------------------------
+
+Additional Information
+======================
+
+For more information on the documentation for CRQ Messages, VMC Messages,
+HMC interface Buffers, and signal messages please refer to the Linux on
+Power Architecture Platform Reference. Section F.
diff --git a/Documentation/networking/af_xdp.rst b/Documentation/networking/af_xdp.rst
new file mode 100644
index 000000000000..91928d9ee4bf
--- /dev/null
+++ b/Documentation/networking/af_xdp.rst
@@ -0,0 +1,297 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+======
+AF_XDP
+======
+
+Overview
+========
+
+AF_XDP is an address family that is optimized for high performance
+packet processing.
+
+This document assumes that the reader is familiar with BPF and XDP. If
+not, the Cilium project has an excellent reference guide at
+http://cilium.readthedocs.io/en/doc-1.0/bpf/.
+
+Using the XDP_REDIRECT action from an XDP program, the program can
+redirect ingress frames to other XDP enabled netdevs, using the
+bpf_redirect_map() function. AF_XDP sockets enable the possibility for
+XDP programs to redirect frames to a memory buffer in a user-space
+application.
+
+An AF_XDP socket (XSK) is created with the normal socket()
+syscall. Associated with each XSK are two rings: the RX ring and the
+TX ring. A socket can receive packets on the RX ring and it can send
+packets on the TX ring. These rings are registered and sized with the
+setsockopts XDP_RX_RING and XDP_TX_RING, respectively. It is mandatory
+to have at least one of these rings for each socket. An RX or TX
+descriptor ring points to a data buffer in a memory area called a
+UMEM. RX and TX can share the same UMEM so that a packet does not have
+to be copied between RX and TX. Moreover, if a packet needs to be kept
+for a while due to a possible retransmit, the descriptor that points
+to that packet can be changed to point to another and reused right
+away. This again avoids copying data.
+
+The UMEM consists of a number of equally size frames and each frame
+has a unique frame id. A descriptor in one of the rings references a
+frame by referencing its frame id. The user space allocates memory for
+this UMEM using whatever means it feels is most appropriate (malloc,
+mmap, huge pages, etc). This memory area is then registered with the
+kernel using the new setsockopt XDP_UMEM_REG. The UMEM also has two
+rings: the FILL ring and the COMPLETION ring. The fill ring is used by
+the application to send down frame ids for the kernel to fill in with
+RX packet data. References to these frames will then appear in the RX
+ring once each packet has been received. The completion ring, on the
+other hand, contains frame ids that the kernel has transmitted
+completely and can now be used again by user space, for either TX or
+RX. Thus, the frame ids appearing in the completion ring are ids that
+were previously transmitted using the TX ring. In summary, the RX and
+FILL rings are used for the RX path and the TX and COMPLETION rings
+are used for the TX path.
+
+The socket is then finally bound with a bind() call to a device and a
+specific queue id on that device, and it is not until bind is
+completed that traffic starts to flow.
+
+The UMEM can be shared between processes, if desired. If a process
+wants to do this, it simply skips the registration of the UMEM and its
+corresponding two rings, sets the XDP_SHARED_UMEM flag in the bind
+call and submits the XSK of the process it would like to share UMEM
+with as well as its own newly created XSK socket. The new process will
+then receive frame id references in its own RX ring that point to this
+shared UMEM. Note that since the ring structures are single-consumer /
+single-producer (for performance reasons), the new process has to
+create its own socket with associated RX and TX rings, since it cannot
+share this with the other process. This is also the reason that there
+is only one set of FILL and COMPLETION rings per UMEM. It is the
+responsibility of a single process to handle the UMEM.
+
+How is then packets distributed from an XDP program to the XSKs? There
+is a BPF map called XSKMAP (or BPF_MAP_TYPE_XSKMAP in full). The
+user-space application can place an XSK at an arbitrary place in this
+map. The XDP program can then redirect a packet to a specific index in
+this map and at this point XDP validates that the XSK in that map was
+indeed bound to that device and ring number. If not, the packet is
+dropped. If the map is empty at that index, the packet is also
+dropped. This also means that it is currently mandatory to have an XDP
+program loaded (and one XSK in the XSKMAP) to be able to get any
+traffic to user space through the XSK.
+
+AF_XDP can operate in two different modes: XDP_SKB and XDP_DRV. If the
+driver does not have support for XDP, or XDP_SKB is explicitly chosen
+when loading the XDP program, XDP_SKB mode is employed that uses SKBs
+together with the generic XDP support and copies out the data to user
+space. A fallback mode that works for any network device. On the other
+hand, if the driver has support for XDP, it will be used by the AF_XDP
+code to provide better performance, but there is still a copy of the
+data into user space.
+
+Concepts
+========
+
+In order to use an AF_XDP socket, a number of associated objects need
+to be setup.
+
+Jonathan Corbet has also written an excellent article on LWN,
+"Accelerating networking with AF_XDP". It can be found at
+https://lwn.net/Articles/750845/.
+
+UMEM
+----
+
+UMEM is a region of virtual contiguous memory, divided into
+equal-sized frames. An UMEM is associated to a netdev and a specific
+queue id of that netdev. It is created and configured (frame size,
+frame headroom, start address and size) by using the XDP_UMEM_REG
+setsockopt system call. A UMEM is bound to a netdev and queue id, via
+the bind() system call.
+
+An AF_XDP is socket linked to a single UMEM, but one UMEM can have
+multiple AF_XDP sockets. To share an UMEM created via one socket A,
+the next socket B can do this by setting the XDP_SHARED_UMEM flag in
+struct sockaddr_xdp member sxdp_flags, and passing the file descriptor
+of A to struct sockaddr_xdp member sxdp_shared_umem_fd.
+
+The UMEM has two single-producer/single-consumer rings, that are used
+to transfer ownership of UMEM frames between the kernel and the
+user-space application.
+
+Rings
+-----
+
+There are a four different kind of rings: Fill, Completion, RX and
+TX. All rings are single-producer/single-consumer, so the user-space
+application need explicit synchronization of multiple
+processes/threads are reading/writing to them.
+
+The UMEM uses two rings: Fill and Completion. Each socket associated
+with the UMEM must have an RX queue, TX queue or both. Say, that there
+is a setup with four sockets (all doing TX and RX). Then there will be
+one Fill ring, one Completion ring, four TX rings and four RX rings.
+
+The rings are head(producer)/tail(consumer) based rings. A producer
+writes the data ring at the index pointed out by struct xdp_ring
+producer member, and increasing the producer index. A consumer reads
+the data ring at the index pointed out by struct xdp_ring consumer
+member, and increasing the consumer index.
+
+The rings are configured and created via the _RING setsockopt system
+calls and mmapped to user-space using the appropriate offset to mmap()
+(XDP_PGOFF_RX_RING, XDP_PGOFF_TX_RING, XDP_UMEM_PGOFF_FILL_RING and
+XDP_UMEM_PGOFF_COMPLETION_RING).
+
+The size of the rings need to be of size power of two.
+
+UMEM Fill Ring
+~~~~~~~~~~~~~~
+
+The Fill ring is used to transfer ownership of UMEM frames from
+user-space to kernel-space. The UMEM indicies are passed in the
+ring. As an example, if the UMEM is 64k and each frame is 4k, then the
+UMEM has 16 frames and can pass indicies between 0 and 15.
+
+Frames passed to the kernel are used for the ingress path (RX rings).
+
+The user application produces UMEM indicies to this ring.
+
+UMEM Completetion Ring
+~~~~~~~~~~~~~~~~~~~~~~
+
+The Completion Ring is used transfer ownership of UMEM frames from
+kernel-space to user-space. Just like the Fill ring, UMEM indicies are
+used.
+
+Frames passed from the kernel to user-space are frames that has been
+sent (TX ring) and can be used by user-space again.
+
+The user application consumes UMEM indicies from this ring.
+
+
+RX Ring
+~~~~~~~
+
+The RX ring is the receiving side of a socket. Each entry in the ring
+is a struct xdp_desc descriptor. The descriptor contains UMEM index
+(idx), the length of the data (len), the offset into the frame
+(offset).
+
+If no frames have been passed to kernel via the Fill ring, no
+descriptors will (or can) appear on the RX ring.
+
+The user application consumes struct xdp_desc descriptors from this
+ring.
+
+TX Ring
+~~~~~~~
+
+The TX ring is used to send frames. The struct xdp_desc descriptor is
+filled (index, length and offset) and passed into the ring.
+
+To start the transfer a sendmsg() system call is required. This might
+be relaxed in the future.
+
+The user application produces struct xdp_desc descriptors to this
+ring.
+
+XSKMAP / BPF_MAP_TYPE_XSKMAP
+----------------------------
+
+On XDP side there is a BPF map type BPF_MAP_TYPE_XSKMAP (XSKMAP) that
+is used in conjunction with bpf_redirect_map() to pass the ingress
+frame to a socket.
+
+The user application inserts the socket into the map, via the bpf()
+system call.
+
+Note that if an XDP program tries to redirect to a socket that does
+not match the queue configuration and netdev, the frame will be
+dropped. E.g. an AF_XDP socket is bound to netdev eth0 and
+queue 17. Only the XDP program executing for eth0 and queue 17 will
+successfully pass data to the socket. Please refer to the sample
+application (samples/bpf/) in for an example.
+
+Usage
+=====
+
+In order to use AF_XDP sockets there are two parts needed. The
+user-space application and the XDP program. For a complete setup and
+usage example, please refer to the sample application. The user-space
+side is xdpsock_user.c and the XDP side xdpsock_kern.c.
+
+Naive ring dequeue and enqueue could look like this::
+
+    // typedef struct xdp_rxtx_ring RING;
+    // typedef struct xdp_umem_ring RING;
+
+    // typedef struct xdp_desc RING_TYPE;
+    // typedef __u32 RING_TYPE;
+
+    int dequeue_one(RING *ring, RING_TYPE *item)
+    {
+        __u32 entries = ring->ptrs.producer - ring->ptrs.consumer;
+
+        if (entries == 0)
+            return -1;
+
+        // read-barrier!
+
+        *item = ring->desc[ring->ptrs.consumer & (RING_SIZE - 1)];
+        ring->ptrs.consumer++;
+        return 0;
+    }
+
+    int enqueue_one(RING *ring, const RING_TYPE *item)
+    {
+        u32 free_entries = RING_SIZE - (ring->ptrs.producer - ring->ptrs.consumer);
+
+        if (free_entries == 0)
+            return -1;
+
+        ring->desc[ring->ptrs.producer & (RING_SIZE - 1)] = *item;
+
+        // write-barrier!
+
+        ring->ptrs.producer++;
+        return 0;
+    }
+
+
+For a more optimized version, please refer to the sample application.
+
+Sample application
+==================
+
+There is a xdpsock benchmarking/test application included that
+demonstrates how to use AF_XDP sockets with both private and shared
+UMEMs. Say that you would like your UDP traffic from port 4242 to end
+up in queue 16, that we will enable AF_XDP on. Here, we use ethtool
+for this::
+
+      ethtool -N p3p2 rx-flow-hash udp4 fn
+      ethtool -N p3p2 flow-type udp4 src-port 4242 dst-port 4242 \
+          action 16
+
+Running the rxdrop benchmark in XDP_DRV mode can then be done
+using::
+
+      samples/bpf/xdpsock -i p3p2 -q 16 -r -N
+
+For XDP_SKB mode, use the switch "-S" instead of "-N" and all options
+can be displayed with "-h", as usual.
+
+Credits
+=======
+
+- Björn Töpel (AF_XDP core)
+- Magnus Karlsson (AF_XDP core)
+- Alexander Duyck
+- Alexei Starovoitov
+- Daniel Borkmann
+- Jesper Dangaard Brouer
+- John Fastabend
+- Jonathan Corbet (LWN coverage)
+- Michael S. Tsirkin
+- Qi Z Zhang
+- Willem de Bruijn
+
diff --git a/Documentation/networking/bonding.txt b/Documentation/networking/bonding.txt
index 9ba04c0bab8d..c13214d073a4 100644
--- a/Documentation/networking/bonding.txt
+++ b/Documentation/networking/bonding.txt
@@ -140,7 +140,7 @@ bonding module at load time, or are specified via sysfs.
 
 	Module options may be given as command line arguments to the
 insmod or modprobe command, but are usually specified in either the
-/etc/modrobe.d/*.conf configuration files, or in a distro-specific
+/etc/modprobe.d/*.conf configuration files, or in a distro-specific
 configuration file (some of which are detailed in the next section).
 
 	Details on bonding support for sysfs is provided in the
diff --git a/Documentation/networking/failover.rst b/Documentation/networking/failover.rst
new file mode 100644
index 000000000000..f0c8483cdbf5
--- /dev/null
+++ b/Documentation/networking/failover.rst
@@ -0,0 +1,18 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+========
+FAILOVER
+========
+
+Overview
+========
+
+The failover module provides a generic interface for paravirtual drivers
+to register a netdev and a set of ops with a failover instance. The ops
+are used as event handlers that get called to handle netdev register/
+unregister/link change/name change events on slave pci ethernet devices
+with the same mac address as the failover netdev.
+
+This enables paravirtual drivers to use a VF as an accelerated low latency
+datapath. It also allows live migration of VMs with direct attached VFs by
+failing over to the paravirtual datapath when the VF is unplugged.
diff --git a/Documentation/networking/filter.txt b/Documentation/networking/filter.txt
index fd55c7de9991..e6b4ebb2b243 100644
--- a/Documentation/networking/filter.txt
+++ b/Documentation/networking/filter.txt
@@ -483,6 +483,12 @@ Example output from dmesg:
 [ 3389.935851] JIT code: 00000030: 00 e8 28 94 ff e0 83 f8 01 75 07 b8 ff ff 00 00
 [ 3389.935852] JIT code: 00000040: eb 02 31 c0 c9 c3
 
+When CONFIG_BPF_JIT_ALWAYS_ON is enabled, bpf_jit_enable is permanently set to 1 and
+setting any other value than that will return in failure. This is even the case for
+setting bpf_jit_enable to 2, since dumping the final JIT image into the kernel log
+is discouraged and introspection through bpftool (under tools/bpf/bpftool/) is the
+generally recommended approach instead.
+
 In the kernel source tree under tools/bpf/, there's bpf_jit_disasm for
 generating disassembly out of the kernel log's hexdump:
 
@@ -1136,6 +1142,7 @@ into a register from memory, the register's top 56 bits are known zero, while
 the low 8 are unknown - which is represented as the tnum (0x0; 0xff).  If we
 then OR this with 0x40, we get (0x40; 0xbf), then if we add 1 we get (0x0;
 0x1ff), because of potential carries.
+
 Besides arithmetic, the register state can also be updated by conditional
 branches.  For instance, if a SCALAR_VALUE is compared > 8, in the 'true' branch
 it will have a umin_value (unsigned minimum value) of 9, whereas in the 'false'
@@ -1144,14 +1151,16 @@ BPF_JSGE) would instead update the signed minimum/maximum values.  Information
 from the signed and unsigned bounds can be combined; for instance if a value is
 first tested < 8 and then tested s> 4, the verifier will conclude that the value
 is also > 4 and s< 8, since the bounds prevent crossing the sign boundary.
+
 PTR_TO_PACKETs with a variable offset part have an 'id', which is common to all
 pointers sharing that same variable offset.  This is important for packet range
-checks: after adding some variable to a packet pointer, if you then copy it to
-another register and (say) add a constant 4, both registers will share the same
-'id' but one will have a fixed offset of +4.  Then if it is bounds-checked and
-found to be less than a PTR_TO_PACKET_END, the other register is now known to
-have a safe range of at least 4 bytes.  See 'Direct packet access', below, for
-more on PTR_TO_PACKET ranges.
+checks: after adding a variable to a packet pointer register A, if you then copy
+it to another register B and then add a constant 4 to A, both registers will
+share the same 'id' but the A will have a fixed offset of +4.  Then if A is
+bounds-checked and found to be less than a PTR_TO_PACKET_END, the register B is
+now known to have a safe range of at least 4 bytes.  See 'Direct packet access',
+below, for more on PTR_TO_PACKET ranges.
+
 The 'id' field is also used on PTR_TO_MAP_VALUE_OR_NULL, common to all copies of
 the pointer returned from a map lookup.  This means that when one copy is
 checked and found to be non-NULL, all copies can become PTR_TO_MAP_VALUEs.
diff --git a/Documentation/networking/index.rst b/Documentation/networking/index.rst
index f204eaff657d..cbd9bdd4a79e 100644
--- a/Documentation/networking/index.rst
+++ b/Documentation/networking/index.rst
@@ -6,6 +6,7 @@ Contents:
 .. toctree::
    :maxdepth: 2
 
+   af_xdp
    batman-adv
    can
    dpaa2/index
diff --git a/Documentation/networking/ip-sysctl.txt b/Documentation/networking/ip-sysctl.txt
index 35ffaa281b26..924bd51327b7 100644
--- a/Documentation/networking/ip-sysctl.txt
+++ b/Documentation/networking/ip-sysctl.txt
@@ -449,8 +449,10 @@ tcp_recovery - INTEGER
 	features.
 
 	RACK: 0x1 enables the RACK loss detection for fast detection of lost
-	      retransmissions and tail drops.
+	      retransmissions and tail drops. It also subsumes and disables
+	      RFC6675 recovery for SACK connections.
 	RACK: 0x2 makes RACK's reordering window static (min_rtt/4).
+	RACK: 0x4 disables RACK's DUPACK threshold heuristic
 
 	Default: 0x1
 
@@ -523,6 +525,19 @@ tcp_rmem - vector of 3 INTEGERs: min, default, max
 tcp_sack - BOOLEAN
 	Enable select acknowledgments (SACKS).
 
+tcp_comp_sack_delay_ns - LONG INTEGER
+	TCP tries to reduce number of SACK sent, using a timer
+	based on 5% of SRTT, capped by this sysctl, in nano seconds.
+	The default is 1ms, based on TSO autosizing period.
+
+	Default : 1,000,000 ns (1 ms)
+
+tcp_comp_sack_nr - INTEGER
+	Max numer of SACK that can be compressed.
+	Using 0 disables SACK compression.
+
+	Detault : 44
+
 tcp_slow_start_after_idle - BOOLEAN
 	If set, provide RFC2861 behavior and time out the congestion
 	window after an idle period.  An idle period is defined at
@@ -1428,6 +1443,19 @@ ip6frag_low_thresh - INTEGER
 ip6frag_time - INTEGER
 	Time in seconds to keep an IPv6 fragment in memory.
 
+IPv6 Segment Routing:
+
+seg6_flowlabel - INTEGER
+	Controls the behaviour of computing the flowlabel of outer
+	IPv6 header in case of SR T.encaps
+
+	-1 set flowlabel to zero.
+	0 copy flowlabel from Inner packet in case of Inner IPv6
+		(Set flowlabel to 0 in case IPv4/L2)
+	1 Compute the flowlabel using seg6_make_flowlabel()
+
+	Default is 0.
+
 conf/default/*:
 	Change the interface-specific default settings.
 
diff --git a/Documentation/networking/net_failover.rst b/Documentation/networking/net_failover.rst
new file mode 100644
index 000000000000..70ca2f5800c4
--- /dev/null
+++ b/Documentation/networking/net_failover.rst
@@ -0,0 +1,116 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+============
+NET_FAILOVER
+============
+
+Overview
+========
+
+The net_failover driver provides an automated failover mechanism via APIs
+to create and destroy a failover master netdev and mananges a primary and
+standby slave netdevs that get registered via the generic failover
+infrastructrure.
+
+The failover netdev acts a master device and controls 2 slave devices. The
+original paravirtual interface is registered as 'standby' slave netdev and
+a passthru/vf device with the same MAC gets registered as 'primary' slave
+netdev. Both 'standby' and 'failover' netdevs are associated with the same
+'pci' device. The user accesses the network interface via 'failover' netdev.
+The 'failover' netdev chooses 'primary' netdev as default for transmits when
+it is available with link up and running.
+
+This can be used by paravirtual drivers to enable an alternate low latency
+datapath. It also enables hypervisor controlled live migration of a VM with
+direct attached VF by failing over to the paravirtual datapath when the VF
+is unplugged.
+
+virtio-net accelerated datapath: STANDBY mode
+=============================================
+
+net_failover enables hypervisor controlled accelerated datapath to virtio-net
+enabled VMs in a transparent manner with no/minimal guest userspace chanages.
+
+To support this, the hypervisor needs to enable VIRTIO_NET_F_STANDBY
+feature on the virtio-net interface and assign the same MAC address to both
+virtio-net and VF interfaces.
+
+Here is an example XML snippet that shows such configuration.
+
+ <interface type='network'>
+   <mac address='52:54:00:00:12:53'/>
+   <source network='enp66s0f0_br'/>
+   <target dev='tap01'/>
+   <model type='virtio'/>
+   <driver name='vhost' queues='4'/>
+   <link state='down'/>
+   <address type='pci' domain='0x0000' bus='0x00' slot='0x0a' function='0x0'/>
+ </interface>
+ <interface type='hostdev' managed='yes'>
+   <mac address='52:54:00:00:12:53'/>
+   <source>
+     <address type='pci' domain='0x0000' bus='0x42' slot='0x02' function='0x5'/>
+   </source>
+   <address type='pci' domain='0x0000' bus='0x00' slot='0x0b' function='0x0'/>
+ </interface>
+
+Booting a VM with the above configuration will result in the following 3
+netdevs created in the VM.
+
+4: ens10: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
+    link/ether 52:54:00:00:12:53 brd ff:ff:ff:ff:ff:ff
+    inet 192.168.12.53/24 brd 192.168.12.255 scope global dynamic ens10
+       valid_lft 42482sec preferred_lft 42482sec
+    inet6 fe80::97d8:db2:8c10:b6d6/64 scope link
+       valid_lft forever preferred_lft forever
+5: ens10nsby: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master ens10 state UP group default qlen 1000
+    link/ether 52:54:00:00:12:53 brd ff:ff:ff:ff:ff:ff
+7: ens11: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ens10 state UP group default qlen 1000
+    link/ether 52:54:00:00:12:53 brd ff:ff:ff:ff:ff:ff
+
+ens10 is the 'failover' master netdev, ens10nsby and ens11 are the slave
+'standby' and 'primary' netdevs respectively.
+
+Live Migration of a VM with SR-IOV VF & virtio-net in STANDBY mode
+==================================================================
+
+net_failover also enables hypervisor controlled live migration to be supported
+with VMs that have direct attached SR-IOV VF devices by automatic failover to
+the paravirtual datapath when the VF is unplugged.
+
+Here is a sample script that shows the steps to initiate live migration on
+the source hypervisor.
+
+# cat vf_xml
+<interface type='hostdev' managed='yes'>
+  <mac address='52:54:00:00:12:53'/>
+  <source>
+    <address type='pci' domain='0x0000' bus='0x42' slot='0x02' function='0x5'/>
+  </source>
+  <address type='pci' domain='0x0000' bus='0x00' slot='0x0b' function='0x0'/>
+</interface>
+
+# Source Hypervisor
+#!/bin/bash
+
+DOMAIN=fedora27-tap01
+PF=enp66s0f0
+VF_NUM=5
+TAP_IF=tap01
+VF_XML=
+
+MAC=52:54:00:00:12:53
+ZERO_MAC=00:00:00:00:00:00
+
+virsh domif-setlink $DOMAIN $TAP_IF up
+bridge fdb del $MAC dev $PF master
+virsh detach-device $DOMAIN $VF_XML
+ip link set $PF vf $VF_NUM mac $ZERO_MAC
+
+virsh migrate --live $DOMAIN qemu+ssh://$REMOTE_HOST/system
+
+# Destination Hypervisor
+#!/bin/bash
+
+virsh attach-device $DOMAIN $VF_XML
+virsh domif-setlink $DOMAIN $TAP_IF down
diff --git a/Documentation/networking/netdev-features.txt b/Documentation/networking/netdev-features.txt
index c77f9d57eb91..c4a54c162547 100644
--- a/Documentation/networking/netdev-features.txt
+++ b/Documentation/networking/netdev-features.txt
@@ -113,6 +113,13 @@ whatever headers there might be.
 NETIF_F_TSO_ECN means that hardware can properly split packets with CWR bit
 set, be it TCPv4 (when NETIF_F_TSO is enabled) or TCPv6 (NETIF_F_TSO6).
 
+ * Transmit UDP segmentation offload
+
+NETIF_F_GSO_UDP_GSO_L4 accepts a single UDP header with a payload that exceeds
+gso_size. On segmentation, it segments the payload on gso_size boundaries and
+replicates the network and UDP headers (fixing up the last one if less than
+gso_size).
+
  * Transmit DMA from high memory
 
 On platforms where this is relevant, NETIF_F_HIGHDMA signals that
diff --git a/Documentation/process/2.Process.rst b/Documentation/process/2.Process.rst
index ce5561bb3f8e..a9c46dd0706b 100644
--- a/Documentation/process/2.Process.rst
+++ b/Documentation/process/2.Process.rst
@@ -18,17 +18,17 @@ major kernel release happening every two or three months.  The recent
 release history looks like this:
 
 	======  =================
-	2.6.38	March 14, 2011
-	2.6.37	January 4, 2011
-	2.6.36	October 20, 2010
-	2.6.35	August 1, 2010
-	2.6.34	May 15, 2010
-	2.6.33	February 24, 2010
+	4.11	April 30, 2017
+	4.12	July 2, 2017
+	4.13	September 3, 2017
+	4.14	November 12, 2017
+	4.15	January 28, 2018
+	4.16	April 1, 2018
 	======  =================
 
-Every 2.6.x release is a major kernel release with new features, internal
-API changes, and more.  A typical 2.6 release can contain nearly 10,000
-changesets with changes to several hundred thousand lines of code.  2.6 is
+Every 4.x release is a major kernel release with new features, internal
+API changes, and more.  A typical 4.x release contain about 13,000
+changesets with changes to several hundred thousand lines of code.  4.x is
 thus the leading edge of Linux kernel development; the kernel uses a
 rolling development model which is continually integrating major changes.
 
@@ -70,20 +70,19 @@ will get up to somewhere between -rc6 and -rc9 before the kernel is
 considered to be sufficiently stable and the final 2.6.x release is made.
 At that point the whole process starts over again.
 
-As an example, here is how the 2.6.38 development cycle went (all dates in
-2011):
+As an example, here is how the 4.16 development cycle went (all dates in
+2018):
 
 	==============  ===============================
-	January 4	2.6.37 stable release
-	January 18	2.6.38-rc1, merge window closes
-	January 21	2.6.38-rc2
-	February 1	2.6.38-rc3
-	February 7	2.6.38-rc4
-	February 15	2.6.38-rc5
-	February 21	2.6.38-rc6
-	March 1		2.6.38-rc7
-	March 7		2.6.38-rc8
-	March 14	2.6.38 stable release
+	January 28	4.15 stable release
+	February 11	4.16-rc1, merge window closes
+	February 18	4.16-rc2
+	February 25	4.16-rc3
+	March 4		4.16-rc4
+	March 11	4.16-rc5
+	March 18	4.16-rc6
+	March 25	4.16-rc7
+	April 1		4.17 stable release
 	==============  ===============================
 
 How do the developers decide when to close the development cycle and create
@@ -99,37 +98,42 @@ release is made.  In the real world, this kind of perfection is hard to
 achieve; there are just too many variables in a project of this size.
 There comes a point where delaying the final release just makes the problem
 worse; the pile of changes waiting for the next merge window will grow
-larger, creating even more regressions the next time around.  So most 2.6.x
+larger, creating even more regressions the next time around.  So most 4.x
 kernels go out with a handful of known regressions though, hopefully, none
 of them are serious.
 
 Once a stable release is made, its ongoing maintenance is passed off to the
 "stable team," currently consisting of Greg Kroah-Hartman.  The stable team
-will release occasional updates to the stable release using the 2.6.x.y
+will release occasional updates to the stable release using the 4.x.y
 numbering scheme.  To be considered for an update release, a patch must (1)
 fix a significant bug, and (2) already be merged into the mainline for the
 next development kernel.  Kernels will typically receive stable updates for
 a little more than one development cycle past their initial release.  So,
-for example, the 2.6.36 kernel's history looked like:
+for example, the 4.13 kernel's history looked like:
 
 	==============  ===============================
-	October 10	2.6.36 stable release
-	November 22	2.6.36.1
-	December 9	2.6.36.2
-	January 7	2.6.36.3
-	February 17	2.6.36.4
+	September 3 	4.13 stable release
+	September 13	4.13.1
+	September 20	4.13.2
+	September 27	4.13.3
+	October 5	4.13.4
+	October 12  	4.13.5
+	...		...
+	November 24	4.13.16
 	==============  ===============================
 
-2.6.36.4 was the final stable update for the 2.6.36 release.
+4.13.16 was the final stable update of the 4.13 release.
 
 Some kernels are designated "long term" kernels; they will receive support
 for a longer period.  As of this writing, the current long term kernels
 and their maintainers are:
 
-	======  ======================  ===========================
-	2.6.27	Willy Tarreau		(Deep-frozen stable kernel)
-	2.6.32	Greg Kroah-Hartman
-	2.6.35	Andi Kleen		(Embedded flag kernel)
+	======  ======================  ==============================
+	3.16	Ben Hutchings		(very long-term stable kernel)
+	4.1	Sasha Levin
+	4.4	Greg Kroah-Hartman	(very long-term stable kernel)
+	4.9	Greg Kroah-Hartman
+	4.14	Greg Kroah-Hartman
 	======  ======================  ===========================
 
 The selection of a kernel for long-term support is purely a matter of a
diff --git a/Documentation/process/5.Posting.rst b/Documentation/process/5.Posting.rst
index c209d70da66f..c418c5d6cae4 100644
--- a/Documentation/process/5.Posting.rst
+++ b/Documentation/process/5.Posting.rst
@@ -10,8 +10,8 @@ of conventions and procedures which are used in the posting of patches;
 following them will make life much easier for everybody involved.  This
 document will attempt to cover these expectations in reasonable detail;
 more information can also be found in the files process/submitting-patches.rst,
-process/submitting-drivers.rst, and process/submit-checklist.rst in the kernel documentation
-directory.
+process/submitting-drivers.rst, and process/submit-checklist.rst in the kernel
+documentation directory.
 
 
 When to post
@@ -198,8 +198,8 @@ pass it to diff with the "-X" option.
 
 The tags mentioned above are used to describe how various developers have
 been associated with the development of this patch.  They are described in
-detail in the process/submitting-patches.rst document; what follows here is a brief
-summary.  Each of these lines has the format:
+detail in the process/submitting-patches.rst document; what follows here is a
+brief summary.  Each of these lines has the format:
 
 ::
 
@@ -210,8 +210,8 @@ The tags in common use are:
  - Signed-off-by: this is a developer's certification that he or she has
    the right to submit the patch for inclusion into the kernel.  It is an
    agreement to the Developer's Certificate of Origin, the full text of
-   which can be found in Documentation/process/submitting-patches.rst.  Code without a
-   proper signoff cannot be merged into the mainline.
+   which can be found in Documentation/process/submitting-patches.rst.  Code
+   without a proper signoff cannot be merged into the mainline.
 
  - Co-developed-by: states that the patch was also created by another developer
    along with the original author.  This is useful at times when multiple
@@ -226,8 +226,8 @@ The tags in common use are:
    it to work.
 
  - Reviewed-by: the named developer has reviewed the patch for correctness;
-   see the reviewer's statement in Documentation/process/submitting-patches.rst for more
-   detail.
+   see the reviewer's statement in Documentation/process/submitting-patches.rst
+   for more detail.
 
  - Reported-by: names a user who reported a problem which is fixed by this
    patch; this tag is used to give credit to the (often underappreciated)
diff --git a/Documentation/process/index.rst b/Documentation/process/index.rst
index 1c9fe657ed01..37bd0628b6ee 100644
--- a/Documentation/process/index.rst
+++ b/Documentation/process/index.rst
@@ -52,6 +52,7 @@ lack of a better place.
    adding-syscalls
    magic-number
    volatile-considered-harmful
+   clang-format
 
 .. only::  subproject and html
 
diff --git a/Documentation/process/maintainer-pgp-guide.rst b/Documentation/process/maintainer-pgp-guide.rst
index b453561a7148..aff9b1a4d77b 100644
--- a/Documentation/process/maintainer-pgp-guide.rst
+++ b/Documentation/process/maintainer-pgp-guide.rst
@@ -219,7 +219,7 @@ Our goal is to protect your master key by moving it to offline media, so
 if you only have a combined **[SC]** key, then you should create a separate
 signing subkey::
 
-    $ gpg --quick-add-key [fpr] ed25519 sign
+    $ gpg --quick-addkey [fpr] ed25519 sign
 
 Remember to tell the keyservers about this change, so others can pull down
 your new subkey::
@@ -450,11 +450,18 @@ functionality.  There are several options available:
 others. If you want to use ECC keys, your best bet among commercially
 available devices is the Nitrokey Start.
 
+.. note::
+
+    If you are listed in MAINTAINERS or have an account at kernel.org,
+    you `qualify for a free Nitrokey Start`_ courtesy of The Linux
+    Foundation.
+
 .. _`Nitrokey Start`: https://shop.nitrokey.com/shop/product/nitrokey-start-6
 .. _`Nitrokey Pro`: https://shop.nitrokey.com/shop/product/nitrokey-pro-3
 .. _`Yubikey 4`: https://www.yubico.com/product/yubikey-4-series/
 .. _Gnuk: http://www.fsij.org/doc-gnuk/
 .. _`LWN has a good review`: https://lwn.net/Articles/736231/
+.. _`qualify for a free Nitrokey Start`: https://www.kernel.org/nitrokey-digital-tokens-for-kernel-developers.html
 
 Configure your smartcard device
 -------------------------------
@@ -482,7 +489,7 @@ there are no convenient command-line switches::
 You should set the user PIN (1), Admin PIN (3), and the Reset Code (4).
 Please make sure to record and store these in a safe place -- especially
 the Admin PIN and the Reset Code (which allows you to completely wipe
-the smartcard).  You so rarely need to use the Admin PIN, that you will
+the smartcard). You so rarely need to use the Admin PIN, that you will
 inevitably forget what it is if you do not record it.
 
 Getting back to the main card menu, you can also set other values (such
@@ -494,6 +501,12 @@ additionally leak information about your smartcard should you lose it.
     Despite having the name "PIN", neither the user PIN nor the admin
     PIN on the card need to be numbers.
 
+.. warning::
+
+    Some devices may require that you move the subkeys onto the device
+    before you can change the passphrase. Please check the documentation
+    provided by the device manufacturer.
+
 Move the subkeys to your smartcard
 ----------------------------------
 
@@ -655,6 +668,20 @@ want to import these changes back into your regular working directory::
     $ gpg --export | gpg --homedir ~/.gnupg --import
     $ unset GNUPGHOME
 
+Using gpg-agent over ssh
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can forward your gpg-agent over ssh if you need to sign tags or
+commits on a remote system. Please refer to the instructions provided
+on the GnuPG wiki:
+
+- `Agent Forwarding over SSH`_
+
+It works more smoothly if you can modify the sshd server settings on the
+remote end.
+
+.. _`Agent Forwarding over SSH`: https://wiki.gnupg.org/AgentForwarding
+
 
 Using PGP with Git
 ==================
@@ -692,6 +719,7 @@ should be used (``[fpr]`` is the fingerprint of your key)::
 tell git to always use it instead of the legacy ``gpg`` from version 1::
 
     $ git config --global gpg.program gpg2
+    $ git config --global gpgv.program gpgv2
 
 How to work with signed tags
 ----------------------------
@@ -731,6 +759,13 @@ If you are verifying someone else's git tag, then you will need to
 import their PGP key. Please refer to the
 ":ref:`verify_identities`" section below.
 
+.. note::
+
+    If you get "``gpg: Can't check signature: unknown pubkey
+    algorithm``" error, you need to tell git to use gpgv2 for
+    verification, so it properly processes signatures made by ECC keys.
+    See instructions at the start of this section.
+
 Configure git to always sign annotated tags
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst
index f7152ed565e5..908bb55be407 100644
--- a/Documentation/process/submitting-patches.rst
+++ b/Documentation/process/submitting-patches.rst
@@ -761,7 +761,7 @@ requests, especially from new, unknown developers.  If in doubt you can use
 the pull request as the cover letter for a normal posting of the patch
 series, giving the maintainer the option of using either.
 
-A pull request should have [GIT] or [PULL] in the subject line.  The
+A pull request should have [GIT PULL] in the subject line.  The
 request itself should include the repository name and the branch of
 interest on a single line; it should look something like::
 
diff --git a/Documentation/scheduler/sched-deadline.txt b/Documentation/scheduler/sched-deadline.txt
index 8ce78f82ae23..b14e03ff3528 100644
--- a/Documentation/scheduler/sched-deadline.txt
+++ b/Documentation/scheduler/sched-deadline.txt
@@ -49,7 +49,7 @@ CONTENTS
 2.1 Main algorithm
 ------------------
 
- SCHED_DEADLINE uses three parameters, named "runtime", "period", and
+ SCHED_DEADLINE [18] uses three parameters, named "runtime", "period", and
  "deadline", to schedule tasks. A SCHED_DEADLINE task should receive
  "runtime" microseconds of execution time every "period" microseconds, and
  these "runtime" microseconds are available within "deadline" microseconds
@@ -117,6 +117,10 @@ CONTENTS
          scheduling deadline = scheduling deadline + period
          remaining runtime = remaining runtime + runtime
 
+ The SCHED_FLAG_DL_OVERRUN flag in sched_attr's sched_flags field allows a task
+ to get informed about runtime overruns through the delivery of SIGXCPU
+ signals.
+
 
 2.2 Bandwidth reclaiming
 ------------------------
@@ -279,6 +283,19 @@ CONTENTS
     running_bw is incremented.
 
 
+2.3 Energy-aware scheduling
+------------------------
+
+ When cpufreq's schedutil governor is selected, SCHED_DEADLINE implements the
+ GRUB-PA [19] algorithm, reducing the CPU operating frequency to the minimum
+ value that still allows to meet the deadlines. This behavior is currently
+ implemented only for ARM architectures.
+
+ A particular care must be taken in case the time needed for changing frequency
+ is of the same order of magnitude of the reservation period. In such cases,
+ setting a fixed CPU frequency results in a lower amount of deadline misses.
+
+
 3. Scheduling Real-Time Tasks
 =============================
 
@@ -505,6 +522,12 @@ CONTENTS
   17 - L. Abeni, G. Lipari, A. Parri, Y. Sun, Multicore CPU reclaiming: parallel
        or sequential?. In Proceedings of the 31st Annual ACM Symposium on Applied
        Computing, 2016.
+  18 - J. Lelli, C. Scordino, L. Abeni, D. Faggioli, Deadline scheduling in the
+       Linux kernel, Software: Practice and Experience, 46(6): 821-839, June
+       2016.
+  19 - C. Scordino, L. Abeni, J. Lelli, Energy-Aware Real-Time Scheduling in
+       the Linux Kernel, 33rd ACM/SIGAPP Symposium On Applied Computing (SAC
+       2018), Pau, France, April 2018.
 
 
 4. Bandwidth management
diff --git a/Documentation/scsi/scsi_eh.txt b/Documentation/scsi/scsi_eh.txt
index 11e447bdb3a5..1b7436932a2b 100644
--- a/Documentation/scsi/scsi_eh.txt
+++ b/Documentation/scsi/scsi_eh.txt
@@ -82,24 +82,13 @@ function
  1. invokes optional hostt->eh_timed_out() callback.  Return value can
     be one of
 
-    - BLK_EH_HANDLED
-	This indicates that eh_timed_out() dealt with the timeout.
-	The command is passed back to the block layer and completed
-	via __blk_complete_requests().
-
-	*NOTE* After returning BLK_EH_HANDLED the SCSI layer is
-	assumed to be finished with the command, and no other
-	functions from the SCSI layer will be called. So this
-	should typically only be returned if the eh_timed_out()
-	handler raced with normal completion.
-
     - BLK_EH_RESET_TIMER
 	This indicates that more time is required to finish the
 	command.  Timer is restarted.  This action is counted as a
 	retry and only allowed scmd->allowed + 1(!) times.  Once the
-	limit is reached, action for BLK_EH_NOT_HANDLED is taken instead.
+	limit is reached, action for BLK_EH_DONE is taken instead.
 
-    - BLK_EH_NOT_HANDLED
+    - BLK_EH_DONE
         eh_timed_out() callback did not handle the command.
 	Step #2 is taken.
 
diff --git a/Documentation/security/index.rst b/Documentation/security/index.rst
index 298a94a33f05..85492bfca530 100644
--- a/Documentation/security/index.rst
+++ b/Documentation/security/index.rst
@@ -9,5 +9,7 @@ Security Documentation
    IMA-templates
    keys/index
    LSM
+   LSM-sctp
+   SELinux-sctp
    self-protection
    tpm/index
diff --git a/Documentation/sound/alsa-configuration.rst b/Documentation/sound/alsa-configuration.rst
index aed6b4fb8e46..4d83c1c0ca04 100644
--- a/Documentation/sound/alsa-configuration.rst
+++ b/Documentation/sound/alsa-configuration.rst
@@ -1062,7 +1062,7 @@ output (with ``--no-upload`` option) to kernel bugzilla or alsa-devel
 ML (see the section `Links and Addresses`_).
 
 ``power_save`` and ``power_save_controller`` options are for power-saving
-mode.  See powersave.txt for details.
+mode.  See powersave.rst for details.
 
 Note 2: If you get click noises on output, try the module option
 ``position_fix=1`` or ``2``.  ``position_fix=1`` will use the SD_LPIB
@@ -1133,7 +1133,7 @@ line_outs_monitor
 enable_monitor
     Enable Analog Out on Channel 63/64 by default.
 
-See hdspm.txt for details.
+See hdspm.rst for details.
 
 Module snd-ice1712
 ------------------
@@ -2224,6 +2224,13 @@ quirk_alias
     Quirk alias list, pass strings like ``0123abcd:5678beef``, which
     applies the existing quirk for the device 5678:beef to a new
     device 0123:abcd.
+use_vmalloc
+    Use vmalloc() for allocations of the PCM buffers (default: yes).
+    For architectures with non-coherent memory like ARM or MIPS, the
+    mmap access may give inconsistent results with vmalloc'ed
+    buffers.  If mmap is used on such architectures, turn off this
+    option, so that the DMA-coherent buffers are allocated and used
+    instead.
 
 This module supports multiple devices, autoprobe and hotplugging.
 
diff --git a/Documentation/sound/hd-audio/models.rst b/Documentation/sound/hd-audio/models.rst
index 1fee5a4f6660..7c2d37571af0 100644
--- a/Documentation/sound/hd-audio/models.rst
+++ b/Documentation/sound/hd-audio/models.rst
@@ -263,6 +263,8 @@ hp-dock
     HP dock support
 mute-led-gpio
     Mute LED control via GPIO
+hp-mic-fix
+    Fix for headset mic pin on HP boxes
 
 STAC9200
 ========
diff --git a/Documentation/sound/soc/codec.rst b/Documentation/sound/soc/codec.rst
index f87612b94812..8a9737eb7597 100644
--- a/Documentation/sound/soc/codec.rst
+++ b/Documentation/sound/soc/codec.rst
@@ -139,7 +139,7 @@ DAPM description
 ----------------
 The Dynamic Audio Power Management description describes the codec power
 components and their relationships and registers to the ASoC core.
-Please read dapm.txt for details of building the description.
+Please read dapm.rst for details of building the description.
 
 Please also see the examples in other codec drivers.
 
@@ -179,12 +179,12 @@ i.e.
 
   static int wm8974_mute(struct snd_soc_dai *dai, int mute)
   {
-	struct snd_soc_codec *codec = dai->codec;
-	u16 mute_reg = snd_soc_read(codec, WM8974_DAC) & 0xffbf;
+	struct snd_soc_component *component = dai->component;
+	u16 mute_reg = snd_soc_component_read32(component, WM8974_DAC) & 0xffbf;
 
 	if (mute)
-		snd_soc_write(codec, WM8974_DAC, mute_reg | 0x40);
+		snd_soc_component_write(component, WM8974_DAC, mute_reg | 0x40);
 	else
-		snd_soc_write(codec, WM8974_DAC, mute_reg);
+		snd_soc_component_write(component, WM8974_DAC, mute_reg);
 	return 0;
   }
diff --git a/Documentation/sound/soc/platform.rst b/Documentation/sound/soc/platform.rst
index d5574904d981..c1badea53d3d 100644
--- a/Documentation/sound/soc/platform.rst
+++ b/Documentation/sound/soc/platform.rst
@@ -23,30 +23,26 @@ The platform DMA driver optionally supports the following ALSA operations:-
   };
 
 The platform driver exports its DMA functionality via struct
-snd_soc_platform_driver:-
+snd_soc_component_driver:-
 ::
 
-  struct snd_soc_platform_driver {
-	char *name;
+  struct snd_soc_component_driver {
+	const char *name;
 
-	int (*probe)(struct platform_device *pdev);
-	int (*remove)(struct platform_device *pdev);
-	int (*suspend)(struct platform_device *pdev, struct snd_soc_cpu_dai *cpu_dai);
-	int (*resume)(struct platform_device *pdev, struct snd_soc_cpu_dai *cpu_dai);
+	...
+	int (*probe)(struct snd_soc_component *);
+	void (*remove)(struct snd_soc_component *);
+	int (*suspend)(struct snd_soc_component *);
+	int (*resume)(struct snd_soc_component *);
 
 	/* pcm creation and destruction */
-	int (*pcm_new)(struct snd_card *, struct snd_soc_codec_dai *, struct snd_pcm *);
+	int (*pcm_new)(struct snd_soc_pcm_runtime *);
 	void (*pcm_free)(struct snd_pcm *);
 
-	/*
-	 * For platform caused delay reporting.
-	 * Optional.
-	 */
-	snd_pcm_sframes_t (*delay)(struct snd_pcm_substream *,
-		struct snd_soc_dai *);
-
-	/* platform stream ops */
-	struct snd_pcm_ops *pcm_ops;
+	...
+	const struct snd_pcm_ops *ops;
+	const struct snd_compr_ops *compr_ops;
+	...
   };
 
 Please refer to the ALSA driver documentation for details of audio DMA.
@@ -66,7 +62,7 @@ Each SoC DAI driver must provide the following features:-
 4. SYSCLK configuration
 5. Suspend and resume (optional)
 
-Please see codec.txt for a description of items 1 - 4.
+Please see codec.rst for a description of items 1 - 4.
 
 
 SoC DSP Drivers
diff --git a/Documentation/sysctl/fs.txt b/Documentation/sysctl/fs.txt
index 6c00c1e2743f..819caf8ca05f 100644
--- a/Documentation/sysctl/fs.txt
+++ b/Documentation/sysctl/fs.txt
@@ -34,7 +34,9 @@ Currently, these files are in /proc/sys/fs:
 - overflowgid
 - pipe-user-pages-hard
 - pipe-user-pages-soft
+- protected_fifos
 - protected_hardlinks
+- protected_regular
 - protected_symlinks
 - suid_dumpable
 - super-max
@@ -182,6 +184,24 @@ applied.
 
 ==============================================================
 
+protected_fifos:
+
+The intent of this protection is to avoid unintentional writes to
+an attacker-controlled FIFO, where a program expected to create a regular
+file.
+
+When set to "0", writing to FIFOs is unrestricted.
+
+When set to "1" don't allow O_CREAT open on FIFOs that we don't own
+in world writable sticky directories, unless they are owned by the
+owner of the directory.
+
+When set to "2" it also applies to group writable sticky directories.
+
+This protection is based on the restrictions in Openwall.
+
+==============================================================
+
 protected_hardlinks:
 
 A long-standing class of security issues is the hardlink-based
@@ -202,6 +222,22 @@ This protection is based on the restrictions in Openwall and grsecurity.
 
 ==============================================================
 
+protected_regular:
+
+This protection is similar to protected_fifos, but it
+avoids writes to an attacker-controlled regular file, where a program
+expected to create one.
+
+When set to "0", writing to regular files is unrestricted.
+
+When set to "1" don't allow O_CREAT open on regular files that we
+don't own in world writable sticky directories, unless they are
+owned by the owner of the directory.
+
+When set to "2" it also applies to group writable sticky directories.
+
+==============================================================
+
 protected_symlinks:
 
 A long-standing class of security issues is the symlink-based
diff --git a/Documentation/sysctl/net.txt b/Documentation/sysctl/net.txt
index 5992602469d8..9ecde517728c 100644
--- a/Documentation/sysctl/net.txt
+++ b/Documentation/sysctl/net.txt
@@ -45,6 +45,7 @@ through bpf(2) and passing a verifier in the kernel, a JIT will then
 translate these BPF proglets into native CPU instructions. There are
 two flavors of JITs, the newer eBPF JIT currently supported on:
   - x86_64
+  - x86_32
   - arm64
   - arm32
   - ppc64
diff --git a/Documentation/sysctl/vm.txt b/Documentation/sysctl/vm.txt
index 17256f2ad919..697ef8c225df 100644
--- a/Documentation/sysctl/vm.txt
+++ b/Documentation/sysctl/vm.txt
@@ -515,7 +515,7 @@ nr_hugepages
 
 Change the minimum size of the hugepage pool.
 
-See Documentation/vm/hugetlbpage.txt
+See Documentation/admin-guide/mm/hugetlbpage.rst
 
 ==============================================================
 
@@ -524,7 +524,7 @@ nr_overcommit_hugepages
 Change the maximum size of the hugepage pool. The maximum is
 nr_hugepages + nr_overcommit_hugepages.
 
-See Documentation/vm/hugetlbpage.txt
+See Documentation/admin-guide/mm/hugetlbpage.rst
 
 ==============================================================
 
@@ -667,7 +667,7 @@ and don't use much of it.
 
 The default value is 0.
 
-See Documentation/vm/overcommit-accounting and
+See Documentation/vm/overcommit-accounting.rst and
 mm/mmap.c::__vm_enough_memory() for more information.
 
 ==============================================================
diff --git a/Documentation/trace/coresight-cpu-debug.txt b/Documentation/trace/coresight-cpu-debug.txt
index 2b9b51cd501e..89ab09e78e8d 100644
--- a/Documentation/trace/coresight-cpu-debug.txt
+++ b/Documentation/trace/coresight-cpu-debug.txt
@@ -177,11 +177,11 @@ Here is an example of the debugging output format:
 ARM external debug module:
 coresight-cpu-debug 850000.debug: CPU[0]:
 coresight-cpu-debug 850000.debug:  EDPRSR:  00000001 (Power:On DLK:Unlock)
-coresight-cpu-debug 850000.debug:  EDPCSR:  [<ffff00000808e9bc>] handle_IPI+0x174/0x1d8
+coresight-cpu-debug 850000.debug:  EDPCSR:  handle_IPI+0x174/0x1d8
 coresight-cpu-debug 850000.debug:  EDCIDSR: 00000000
 coresight-cpu-debug 850000.debug:  EDVIDSR: 90000000 (State:Non-secure Mode:EL1/0 Width:64bits VMID:0)
 coresight-cpu-debug 852000.debug: CPU[1]:
 coresight-cpu-debug 852000.debug:  EDPRSR:  00000001 (Power:On DLK:Unlock)
-coresight-cpu-debug 852000.debug:  EDPCSR:  [<ffff0000087fab34>] debug_notifier_call+0x23c/0x358
+coresight-cpu-debug 852000.debug:  EDPCSR:  debug_notifier_call+0x23c/0x358
 coresight-cpu-debug 852000.debug:  EDCIDSR: 00000000
 coresight-cpu-debug 852000.debug:  EDVIDSR: 90000000 (State:Non-secure Mode:EL1/0 Width:64bits VMID:0)
diff --git a/Documentation/trace/coresight.txt b/Documentation/trace/coresight.txt
index 6f0120c3a4f1..1d74ad0202b6 100644
--- a/Documentation/trace/coresight.txt
+++ b/Documentation/trace/coresight.txt
@@ -187,13 +187,19 @@ that can be performed on them (see "struct coresight_ops").  The
 specific to that component only.  "Implementation defined" customisations are
 expected to be accessed and controlled using those entries.
 
-Last but not least, "struct module *owner" is expected to be set to reflect
-the information carried in "THIS_MODULE".
 
 How to use the tracer modules
 -----------------------------
 
-Before trace collection can start, a coresight sink needs to be identify.
+There are two ways to use the Coresight framework: 1) using the perf cmd line
+tools and 2) interacting directly with the Coresight devices using the sysFS
+interface.  Preference is given to the former as using the sysFS interface
+requires a deep understanding of the Coresight HW.  The following sections
+provide details on using both methods.
+
+1) Using the sysFS interface:
+
+Before trace collection can start, a coresight sink needs to be identified.
 There is no limit on the amount of sinks (nor sources) that can be enabled at
 any given moment.  As a generic operation, all device pertaining to the sink
 class will have an "active" entry in sysfs:
@@ -298,42 +304,48 @@ Instruction     13570831        0x8026B584      E28DD00C        false   ADD
 Instruction     0       0x8026B588      E8BD8000        true    LDM      sp!,{pc}
 Timestamp                                       Timestamp: 17107041535
 
-How to use the STM module
--------------------------
+2) Using perf framework:
 
-Using the System Trace Macrocell module is the same as the tracers - the only
-difference is that clients are driving the trace capture rather
-than the program flow through the code.
+Coresight tracers are represented using the Perf framework's Performance
+Monitoring Unit (PMU) abstraction.  As such the perf framework takes charge of
+controlling when tracing gets enabled based on when the process of interest is
+scheduled.  When configured in a system, Coresight PMUs will be listed when
+queried by the perf command line tool:
 
-As with any other CoreSight component, specifics about the STM tracer can be
-found in sysfs with more information on each entry being found in [1]:
+	linaro@linaro-nano:~$ ./perf list pmu
 
-root@genericarmv8:~# ls /sys/bus/coresight/devices/20100000.stm
-enable_source   hwevent_select  port_enable     subsystem       uevent
-hwevent_enable  mgmt            port_select     traceid
-root@genericarmv8:~#
+		List of pre-defined events (to be used in -e):
 
-Like any other source a sink needs to be identified and the STM enabled before
-being used:
+		cs_etm//                                    [Kernel PMU event]
 
-root@genericarmv8:~# echo 1 > /sys/bus/coresight/devices/20010000.etf/enable_sink
-root@genericarmv8:~# echo 1 > /sys/bus/coresight/devices/20100000.stm/enable_source
+	linaro@linaro-nano:~$
 
-From there user space applications can request and use channels using the devfs
-interface provided for that purpose by the generic STM API:
+Regardless of the number of tracers available in a system (usually equal to the
+amount of processor cores), the "cs_etm" PMU will be listed only once.
 
-root@genericarmv8:~# ls -l /dev/20100000.stm
-crw-------    1 root     root       10,  61 Jan  3 18:11 /dev/20100000.stm
-root@genericarmv8:~#
+A Coresight PMU works the same way as any other PMU, i.e the name of the PMU is
+listed along with configuration options within forward slashes '/'.  Since a
+Coresight system will typically have more than one sink, the name of the sink to
+work with needs to be specified as an event option.  Names for sink to choose
+from are listed in sysFS under ($SYSFS)/bus/coresight/devices:
 
-Details on how to use the generic STM API can be found here [2].
+	root@linaro-nano:~# ls /sys/bus/coresight/devices/
+		20010000.etf   20040000.funnel  20100000.stm  22040000.etm
+		22140000.etm  230c0000.funnel  23240000.etm 20030000.tpiu
+		20070000.etr     20120000.replicator  220c0000.funnel
+		23040000.etm  23140000.etm     23340000.etm
 
-[1]. Documentation/ABI/testing/sysfs-bus-coresight-devices-stm
-[2]. Documentation/trace/stm.txt
+	root@linaro-nano:~# perf record -e cs_etm/@20070000.etr/u --per-thread program
 
+The syntax within the forward slashes '/' is important.  The '@' character
+tells the parser that a sink is about to be specified and that this is the sink
+to use for the trace session.
 
-Using perf tools
-----------------
+More information on the above and other example on how to use Coresight with
+the perf tools can be found in the "HOWTO.md" file of the openCSD gitHub
+repository [3].
+
+2.1) AutoFDO analysis using the perf tools:
 
 perf can be used to record and analyze trace of programs.
 
@@ -381,3 +393,38 @@ sort example is from the AutoFDO tutorial (https://gcc.gnu.org/wiki/AutoFDO/Tuto
 	$ taskset -c 2 ./sort_autofdo
 	Bubble sorting array of 30000 elements
 	5806 ms
+
+
+How to use the STM module
+-------------------------
+
+Using the System Trace Macrocell module is the same as the tracers - the only
+difference is that clients are driving the trace capture rather
+than the program flow through the code.
+
+As with any other CoreSight component, specifics about the STM tracer can be
+found in sysfs with more information on each entry being found in [1]:
+
+root@genericarmv8:~# ls /sys/bus/coresight/devices/20100000.stm
+enable_source   hwevent_select  port_enable     subsystem       uevent
+hwevent_enable  mgmt            port_select     traceid
+root@genericarmv8:~#
+
+Like any other source a sink needs to be identified and the STM enabled before
+being used:
+
+root@genericarmv8:~# echo 1 > /sys/bus/coresight/devices/20010000.etf/enable_sink
+root@genericarmv8:~# echo 1 > /sys/bus/coresight/devices/20100000.stm/enable_source
+
+From there user space applications can request and use channels using the devfs
+interface provided for that purpose by the generic STM API:
+
+root@genericarmv8:~# ls -l /dev/20100000.stm
+crw-------    1 root     root       10,  61 Jan  3 18:11 /dev/20100000.stm
+root@genericarmv8:~#
+
+Details on how to use the generic STM API can be found here [2].
+
+[1]. Documentation/ABI/testing/sysfs-bus-coresight-devices-stm
+[2]. Documentation/trace/stm.txt
+[3]. https://github.com/Linaro/perf-opencsd
diff --git a/Documentation/trace/ftrace-uses.rst b/Documentation/trace/ftrace-uses.rst
index 998a60a93015..00283b6dd101 100644
--- a/Documentation/trace/ftrace-uses.rst
+++ b/Documentation/trace/ftrace-uses.rst
@@ -12,7 +12,7 @@ Written for: 4.14
 Introduction
 ============
 
-The ftrace infrastructure was originially created to attach callbacks to the
+The ftrace infrastructure was originally created to attach callbacks to the
 beginning of functions in order to record and trace the flow of the kernel.
 But callbacks to the start of a function can have other use cases. Either
 for live kernel patching, or for security monitoring. This document describes
@@ -30,7 +30,7 @@ The ftrace context
   This requires extra care to what can be done inside a callback. A callback
   can be called outside the protective scope of RCU.
 
-The ftrace infrastructure has some protections agains recursions and RCU
+The ftrace infrastructure has some protections against recursions and RCU
 but one must still be very careful how they use the callbacks.
 
 
diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst
index 67d9c38e95eb..6b80ac4bbaae 100644
--- a/Documentation/trace/ftrace.rst
+++ b/Documentation/trace/ftrace.rst
@@ -224,6 +224,8 @@ of ftrace. Here is a list of some of the key files:
 	has a side effect of enabling or disabling specific functions
 	to be traced. Echoing names of functions into this file
 	will limit the trace to only those functions.
+	This influences the tracers "function" and "function_graph"
+	and thus also function profiling (see "function_profile_enabled").
 
 	The functions listed in "available_filter_functions" are what
 	can be written into this file.
@@ -265,6 +267,8 @@ of ftrace. Here is a list of some of the key files:
 	Functions listed in this file will cause the function graph
 	tracer to only trace these functions and the functions that
 	they call. (See the section "dynamic ftrace" for more details).
+	Note, set_ftrace_filter and set_ftrace_notrace still affects
+	what functions are being traced.
 
   set_graph_notrace:
 
@@ -277,7 +281,8 @@ of ftrace. Here is a list of some of the key files:
 
 	This lists the functions that ftrace has processed and can trace.
 	These are the function names that you can pass to
-	"set_ftrace_filter" or "set_ftrace_notrace".
+	"set_ftrace_filter", "set_ftrace_notrace",
+	"set_graph_function", or "set_graph_notrace".
 	(See the section "dynamic ftrace" below for more details.)
 
   dyn_ftrace_total_info:
diff --git a/Documentation/translations/ko_KR/memory-barriers.txt b/Documentation/translations/ko_KR/memory-barriers.txt
index 0a0930ab4156..921739d00f69 100644
--- a/Documentation/translations/ko_KR/memory-barriers.txt
+++ b/Documentation/translations/ko_KR/memory-barriers.txt
@@ -36,6 +36,9 @@ Documentation/memory-barriers.txt
 부분도 있고, 의도하진 않았지만 사람에 의해 쓰였다보니 불완전한 부분도 있습니다.
 이 문서는 리눅스에서 제공하는 다양한 메모리 배리어들을 사용하기 위한
 안내서입니다만, 뭔가 이상하다 싶으면 (그런게 많을 겁니다) 질문을 부탁드립니다.
+일부 이상한 점들은 공식적인 메모리 일관성 모델과 tools/memory-model/ 에 있는
+관련 문서를 참고해서 해결될 수 있을 겁니다.  그러나, 이 메모리 모델조차도 그
+관리자들의 의견의 집합으로 봐야지, 절대 옳은 예언자로 신봉해선 안될 겁니다.
 
 다시 말하지만, 이 문서는 리눅스가 하드웨어에 기대하는 사항에 대한 명세서가
 아닙니다.
@@ -77,7 +80,7 @@ Documentation/memory-barriers.txt
 
      - 메모리 배리어의 종류.
      - 메모리 배리어에 대해 가정해선 안될 것.
-     - 데이터 의존성 배리어.
+     - 데이터 의존성 배리어 (역사적).
      - 컨트롤 의존성.
      - SMP 배리어 짝맞추기.
      - 메모리 배리어 시퀀스의 예.
@@ -255,17 +258,20 @@ CPU 에게 기대할 수 있는 최소한의 보장사항 몇가지가 있습니
  (*) 어떤 CPU 든, 의존성이 존재하는 메모리 액세스들은 해당 CPU 자신에게
      있어서는 순서대로 메모리 시스템에 수행 요청됩니다. 즉, 다음에 대해서:
 
-	Q = READ_ONCE(P); smp_read_barrier_depends(); D = READ_ONCE(*Q);
+	Q = READ_ONCE(P); D = READ_ONCE(*Q);
 
      CPU 는 다음과 같은 메모리 오퍼레이션 시퀀스를 수행 요청합니다:
 
 	Q = LOAD P, D = LOAD *Q
 
-     그리고 그 시퀀스 내에서의 순서는 항상 지켜집니다.  대부분의 시스템에서
-     smp_read_barrier_depends() 는 아무일도 안하지만 DEC Alpha 에서는
-     명시적으로 사용되어야 합니다.  보통의 경우에는 smp_read_barrier_depends()
-     를 직접 사용하는 대신 rcu_dereference() 같은 것들을 사용해야 함을
-     알아두세요.
+     그리고 그 시퀀스 내에서의 순서는 항상 지켜집니다.  하지만, DEC Alpha 에서
+     READ_ONCE() 는 메모리 배리어 명령도 내게 되어 있어서, DEC Alpha CPU 는
+     다음과 같은 메모리 오퍼레이션들을 내놓게 됩니다:
+
+	Q = LOAD P, MEMORY_BARRIER, D = LOAD *Q, MEMORY_BARRIER
+
+     DEC Alpha 에서 수행되든 아니든, READ_ONCE() 는 컴파일러로부터의 악영향
+     또한 제거합니다.
 
  (*) 특정 CPU 내에서 겹치는 영역의 메모리에 행해지는 로드와 스토어 들은 해당
      CPU 안에서는 순서가 바뀌지 않은 것으로 보여집니다.  즉, 다음에 대해서:
@@ -421,8 +427,8 @@ CPU 에게 기대할 수 있는 최소한의 보장사항 몇가지가 있습니
      데이터 의존성 배리어는 읽기 배리어의 보다 완화된 형태입니다.  두개의 로드
      오퍼레이션이 있고 두번째 것이 첫번째 것의 결과에 의존하고 있을 때(예:
      두번째 로드가 참조할 주소를 첫번째 로드가 읽는 경우), 두번째 로드가 읽어올
-     데이터는 첫번째 로드에 의해 그 주소가 얻어지기 전에 업데이트 되어 있음을
-     보장하기 위해서 데이터 의존성 배리어가 필요할 수 있습니다.
+     데이터는 첫번째 로드에 의해 그 주소가 얻어진 뒤에 업데이트 됨을 보장하기
+     위해서 데이터 의존성 배리어가 필요할 수 있습니다.
 
      데이터 의존성 배리어는 상호 의존적인 로드 오퍼레이션들 사이의 부분적 순서
      세우기입니다; 스토어 오퍼레이션들이나 독립적인 로드들, 또는 중복되는
@@ -570,8 +576,14 @@ ACQUIRE 는 해당 오퍼레이션의 로드 부분에만 적용되고 RELEASE 
 	    Documentation/DMA-API.txt
 
 
-데이터 의존성 배리어
---------------------
+데이터 의존성 배리어 (역사적)
+-----------------------------
+
+리눅스 커널 v4.15 기준으로, smp_read_barrier_depends() 가 READ_ONCE() 에
+추가되었는데, 이는 이 섹션에 주의를 기울여야 하는 사람들은 DEC Alpha 아키텍쳐
+전용 코드를 만드는 사람들과 READ_ONCE() 자체를 만드는 사람들 뿐임을 의미합니다.
+그런 분들을 위해, 그리고 역사에 관심 있는 분들을 위해, 여기 데이터 의존성
+배리어에 대한 이야기를 적습니다.
 
 데이터 의존성 배리어의 사용에 있어 지켜야 하는 사항들은 약간 미묘하고, 데이터
 의존성 배리어가 사용되어야 하는 상황도 항상 명백하지는 않습니다.  설명을 위해
@@ -1787,7 +1799,7 @@ CPU 메모리 배리어
 	범용		mb()			smp_mb()
 	쓰기		wmb()			smp_wmb()
 	읽기		rmb()			smp_rmb()
-	데이터 의존성	read_barrier_depends()	smp_read_barrier_depends()
+	데이터 의존성				READ_ONCE()
 
 
 데이터 의존성 배리어를 제외한 모든 메모리 배리어는 컴파일러 배리어를
@@ -2796,8 +2808,9 @@ CPU 2 는 C/D 를 갖습니다)가 병렬로 연결되어 있는 시스템을 
 
 
 여기에 개입하기 위해선, 데이터 의존성 배리어나 읽기 배리어를 로드 오퍼레이션들
-사이에 넣어야 합니다.  이렇게 함으로써 캐시가 다음 요청을 처리하기 전에 일관성
-큐를 처리하도록 강제하게 됩니다.
+사이에 넣어야 합니다 (v4.15 부터는 READ_ONCE() 매크로에 의해 무조건적으로
+그렇게 됩니다).  이렇게 함으로써 캐시가 다음 요청을 처리하기 전에 일관성 큐를
+처리하도록 강제하게 됩니다.
 
 	CPU 1		CPU 2		COMMENT
 	===============	===============	=======================================
@@ -2826,7 +2839,10 @@ CPU 2 는 C/D 를 갖습니다)가 병렬로 연결되어 있는 시스템을 
 다른 CPU 들도 분할된 캐시를 가지고 있을 수 있지만, 그런 CPU 들은 평범한 메모리
 액세스를 위해서도 이 분할된 캐시들 사이의 조정을 해야만 합니다.  Alpha 는 가장
 약한 메모리 순서 시맨틱 (semantic) 을 선택함으로써 메모리 배리어가 명시적으로
-사용되지 않았을 때에는 그런 조정이 필요하지 않게 했습니다.
+사용되지 않았을 때에는 그런 조정이 필요하지 않게 했으며, 이는 Alpha 가 당시에
+더 높은 CPU 클락 속도를 가질 수 있게 했습니다.  하지만, (다시 말하건대, v4.15
+이후부터는) Alpha 아키텍쳐 전용 코드와 READ_ONCE() 매크로 내부에서를 제외하고는
+smp_read_barrier_depends() 가 사용되지 않아야 함을 알아두시기 바랍니다.
 
 
 캐시 일관성 VS DMA
@@ -2846,7 +2862,7 @@ CPU 의 캐시에서 RAM 으로 쓰여지는 더티 캐시 라인에 의해 덮
 문제를 해결하기 위해선, 커널의 적절한 부분에서 각 CPU 의 캐시 안의 문제가 되는
 비트들을 무효화 시켜야 합니다.
 
-캐시 관리에 대한 더 많은 정보를 위해선 Documentation/cachetlb.txt 를
+캐시 관리에 대한 더 많은 정보를 위해선 Documentation/core-api/cachetlb.rst 를
 참고하세요.
 
 
@@ -2988,7 +3004,9 @@ Alpha CPU 의 일부 버전은 분할된 데이터 캐시를 가지고 있어서
 메모리 일관성 시스템과 함께 두개의 캐시를 동기화 시켜서, 포인터 변경과 새로운
 데이터의 발견을 올바른 순서로 일어나게 하기 때문입니다.
 
-리눅스 커널의 메모리 배리어 모델은 Alpha 에 기초해서 정의되었습니다.
+리눅스 커널의 메모리 배리어 모델은 Alpha 에 기초해서 정의되었습니다만, v4.15
+부터는 리눅스 커널이 READ_ONCE() 내에 smp_read_barrier_depends() 를 추가해서
+Alpha 의 메모리 모델로의 영향력이 크게 줄어들긴 했습니다.
 
 위의 "캐시 일관성" 서브섹션을 참고하세요.
 
@@ -3023,7 +3041,7 @@ smp_mb() 가 아니라 virt_mb() 를 사용해야 합니다.
 동기화에 락을 사용하지 않고 구현하는데에 사용될 수 있습니다.  더 자세한 내용을
 위해선 다음을 참고하세요:
 
-	Documentation/circular-buffers.txt
+	Documentation/core-api/circular-buffers.rst
 
 
 =========
diff --git a/Documentation/userspace-api/seccomp_filter.rst b/Documentation/userspace-api/seccomp_filter.rst
index 099c412951d6..82a468bc7560 100644
--- a/Documentation/userspace-api/seccomp_filter.rst
+++ b/Documentation/userspace-api/seccomp_filter.rst
@@ -207,13 +207,6 @@ directory. Here's a description of each file in that directory:
 	to the file do not need to be in ordered form but reads from the file
 	will be ordered in the same way as the actions_avail sysctl.
 
-	It is important to note that the value of ``actions_logged`` does not
-	prevent certain actions from being logged when the audit subsystem is
-	configured to audit a task. If the action is not found in
-	``actions_logged`` list, the final decision on whether to audit the
-	action for that task is ultimately left up to the audit subsystem to
-	decide for all seccomp return values other than ``SECCOMP_RET_ALLOW``.
-
 	The ``allow`` string is not accepted in the ``actions_logged`` sysctl
 	as it is not possible to log ``SECCOMP_RET_ALLOW`` actions. Attempting
 	to write ``allow`` to the sysctl will result in an EINVAL being
diff --git a/Documentation/vfio-mediated-device.txt b/Documentation/vfio-mediated-device.txt
index 1b3950346532..c3f69bcaf96e 100644
--- a/Documentation/vfio-mediated-device.txt
+++ b/Documentation/vfio-mediated-device.txt
@@ -145,6 +145,11 @@ The functions in the mdev_parent_ops structure are as follows:
 * create: allocate basic resources in a driver for a mediated device
 * remove: free resources in a driver when a mediated device is destroyed
 
+(Note that mdev-core provides no implicit serialization of create/remove
+callbacks per mdev parent device, per mdev type, or any other categorization.
+Vendor drivers are expected to be fully asynchronous in this respect or
+provide their own internal resource protection.)
+
 The callbacks in the mdev_parent_ops structure are as follows:
 
 * open: open callback of mediated device
diff --git a/Documentation/vfio.txt b/Documentation/vfio.txt
index ef6a5111eaa1..f1a4d3c3ba0b 100644
--- a/Documentation/vfio.txt
+++ b/Documentation/vfio.txt
@@ -252,15 +252,14 @@ into VFIO core.  When devices are bound and unbound to the driver,
 the driver should call vfio_add_group_dev() and vfio_del_group_dev()
 respectively::
 
-	extern int vfio_add_group_dev(struct iommu_group *iommu_group,
-	                              struct device *dev,
+	extern int vfio_add_group_dev(struct device *dev,
 				      const struct vfio_device_ops *ops,
 				      void *device_data);
 
 	extern void *vfio_del_group_dev(struct device *dev);
 
 vfio_add_group_dev() indicates to the core to begin tracking the
-specified iommu_group and register the specified dev as owned by
+iommu_group of the specified dev and register the dev as owned by
 a VFIO bus driver.  The driver provides an ops structure for callbacks
 similar to a file operations structure::
 
diff --git a/Documentation/virtual/kvm/devices/arm-vgic-v3.txt b/Documentation/virtual/kvm/devices/arm-vgic-v3.txt
index 9293b45abdb9..2408ab720ef7 100644
--- a/Documentation/virtual/kvm/devices/arm-vgic-v3.txt
+++ b/Documentation/virtual/kvm/devices/arm-vgic-v3.txt
@@ -27,16 +27,42 @@ Groups:
       VCPU and all of the redistributor pages are contiguous.
       Only valid for KVM_DEV_TYPE_ARM_VGIC_V3.
       This address needs to be 64K aligned.
+
+    KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION (rw, 64-bit)
+      The attribute data pointed to by kvm_device_attr.addr is a __u64 value:
+      bits:     | 63   ....  52  |  51   ....   16 | 15 - 12  |11 - 0
+      values:   |     count      |       base      |  flags   | index
+      - index encodes the unique redistributor region index
+      - flags: reserved for future use, currently 0
+      - base field encodes bits [51:16] of the guest physical base address
+        of the first redistributor in the region.
+      - count encodes the number of redistributors in the region. Must be
+        greater than 0.
+      There are two 64K pages for each redistributor in the region and
+      redistributors are laid out contiguously within the region. Regions
+      are filled with redistributors in the index order. The sum of all
+      region count fields must be greater than or equal to the number of
+      VCPUs. Redistributor regions must be registered in the incremental
+      index order, starting from index 0.
+      The characteristics of a specific redistributor region can be read
+      by presetting the index field in the attr data.
+      Only valid for KVM_DEV_TYPE_ARM_VGIC_V3.
+
+  It is invalid to mix calls with KVM_VGIC_V3_ADDR_TYPE_REDIST and
+  KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION attributes.
+
   Errors:
     -E2BIG:  Address outside of addressable IPA range
-    -EINVAL: Incorrectly aligned address
+    -EINVAL: Incorrectly aligned address, bad redistributor region
+             count/index, mixed redistributor region attribute usage
     -EEXIST: Address already configured
+    -ENOENT: Attempt to read the characteristics of a non existing
+             redistributor region
     -ENXIO:  The group or attribute is unknown/unsupported for this device
              or hardware support is missing.
     -EFAULT: Invalid user pointer for attr->addr.
 
 
-
   KVM_DEV_ARM_VGIC_GRP_DIST_REGS
   KVM_DEV_ARM_VGIC_GRP_REDIST_REGS
   Attributes:
diff --git a/Documentation/vm/00-INDEX b/Documentation/vm/00-INDEX
index 0278f2c85efb..f4a4f3e884cf 100644
--- a/Documentation/vm/00-INDEX
+++ b/Documentation/vm/00-INDEX
@@ -1,62 +1,50 @@
 00-INDEX
 	- this file.
-active_mm.txt
+active_mm.rst
 	- An explanation from Linus about tsk->active_mm vs tsk->mm.
-balance
+balance.rst
 	- various information on memory balancing.
-cleancache.txt
+cleancache.rst
 	- Intro to cleancache and page-granularity victim cache.
-frontswap.txt
+frontswap.rst
 	- Outline frontswap, part of the transcendent memory frontend.
-highmem.txt
+highmem.rst
 	- Outline of highmem and common issues.
-hmm.txt
+hmm.rst
 	- Documentation of heterogeneous memory management
-hugetlbpage.txt
-	- a brief summary of hugetlbpage support in the Linux kernel.
-hugetlbfs_reserv.txt
+hugetlbfs_reserv.rst
 	- A brief overview of hugetlbfs reservation design/implementation.
-hwpoison.txt
+hwpoison.rst
 	- explains what hwpoison is
-idle_page_tracking.txt
-	- description of the idle page tracking feature.
-ksm.txt
+ksm.rst
 	- how to use the Kernel Samepage Merging feature.
-mmu_notifier.txt
+mmu_notifier.rst
 	- a note about clearing pte/pmd and mmu notifications
-numa
+numa.rst
 	- information about NUMA specific code in the Linux vm.
-numa_memory_policy.txt
-	- documentation of concepts and APIs of the 2.6 memory policy support.
-overcommit-accounting
+overcommit-accounting.rst
 	- description of the Linux kernels overcommit handling modes.
-page_frags
+page_frags.rst
 	- description of page fragments allocator
-page_migration
+page_migration.rst
 	- description of page migration in NUMA systems.
-pagemap.txt
-	- pagemap, from the userspace perspective
-page_owner.txt
+page_owner.rst
 	- tracking about who allocated each page
-remap_file_pages.txt
+remap_file_pages.rst
 	- a note about remap_file_pages() system call
-slub.txt
+slub.rst
 	- a short users guide for SLUB.
-soft-dirty.txt
-	- short explanation for soft-dirty PTEs
-split_page_table_lock
+split_page_table_lock.rst
 	- Separate per-table lock to improve scalability of the old page_table_lock.
-swap_numa.txt
+swap_numa.rst
 	- automatic binding of swap device to numa node
-transhuge.txt
+transhuge.rst
 	- Transparent Hugepage Support, alternative way of using hugepages.
-unevictable-lru.txt
+unevictable-lru.rst
 	- Unevictable LRU infrastructure
-userfaultfd.txt
-	- description of userfaultfd system call
 z3fold.txt
 	- outline of z3fold allocator for storing compressed pages
-zsmalloc.txt
+zsmalloc.rst
 	- outline of zsmalloc allocator for storing compressed pages
-zswap.txt
+zswap.rst
 	- Intro to compressed cache for swap pages
diff --git a/Documentation/vm/active_mm.rst b/Documentation/vm/active_mm.rst
new file mode 100644
index 000000000000..c84471b180f8
--- /dev/null
+++ b/Documentation/vm/active_mm.rst
@@ -0,0 +1,91 @@
+.. _active_mm:
+
+=========
+Active MM
+=========
+
+::
+
+ List:       linux-kernel
+ Subject:    Re: active_mm
+ From:       Linus Torvalds <torvalds () transmeta ! com>
+ Date:       1999-07-30 21:36:24
+
+ Cc'd to linux-kernel, because I don't write explanations all that often,
+ and when I do I feel better about more people reading them.
+
+ On Fri, 30 Jul 1999, David Mosberger wrote:
+ >
+ > Is there a brief description someplace on how "mm" vs. "active_mm" in
+ > the task_struct are supposed to be used?  (My apologies if this was
+ > discussed on the mailing lists---I just returned from vacation and
+ > wasn't able to follow linux-kernel for a while).
+
+ Basically, the new setup is:
+
+  - we have "real address spaces" and "anonymous address spaces". The
+    difference is that an anonymous address space doesn't care about the
+    user-level page tables at all, so when we do a context switch into an
+    anonymous address space we just leave the previous address space
+    active.
+
+    The obvious use for a "anonymous address space" is any thread that
+    doesn't need any user mappings - all kernel threads basically fall into
+    this category, but even "real" threads can temporarily say that for
+    some amount of time they are not going to be interested in user space,
+    and that the scheduler might as well try to avoid wasting time on
+    switching the VM state around. Currently only the old-style bdflush
+    sync does that.
+
+  - "tsk->mm" points to the "real address space". For an anonymous process,
+    tsk->mm will be NULL, for the logical reason that an anonymous process
+    really doesn't _have_ a real address space at all.
+
+  - however, we obviously need to keep track of which address space we
+    "stole" for such an anonymous user. For that, we have "tsk->active_mm",
+    which shows what the currently active address space is.
+
+    The rule is that for a process with a real address space (ie tsk->mm is
+    non-NULL) the active_mm obviously always has to be the same as the real
+    one.
+
+    For a anonymous process, tsk->mm == NULL, and tsk->active_mm is the
+    "borrowed" mm while the anonymous process is running. When the
+    anonymous process gets scheduled away, the borrowed address space is
+    returned and cleared.
+
+ To support all that, the "struct mm_struct" now has two counters: a
+ "mm_users" counter that is how many "real address space users" there are,
+ and a "mm_count" counter that is the number of "lazy" users (ie anonymous
+ users) plus one if there are any real users.
+
+ Usually there is at least one real user, but it could be that the real
+ user exited on another CPU while a lazy user was still active, so you do
+ actually get cases where you have a address space that is _only_ used by
+ lazy users. That is often a short-lived state, because once that thread
+ gets scheduled away in favour of a real thread, the "zombie" mm gets
+ released because "mm_users" becomes zero.
+
+ Also, a new rule is that _nobody_ ever has "init_mm" as a real MM any
+ more. "init_mm" should be considered just a "lazy context when no other
+ context is available", and in fact it is mainly used just at bootup when
+ no real VM has yet been created. So code that used to check
+
+ 	if (current->mm == &init_mm)
+
+ should generally just do
+
+ 	if (!current->mm)
+
+ instead (which makes more sense anyway - the test is basically one of "do
+ we have a user context", and is generally done by the page fault handler
+ and things like that).
+
+ Anyway, I put a pre-patch-2.3.13-1 on ftp.kernel.org just a moment ago,
+ because it slightly changes the interfaces to accommodate the alpha (who
+ would have thought it, but the alpha actually ends up having one of the
+ ugliest context switch codes - unlike the other architectures where the MM
+ and register state is separate, the alpha PALcode joins the two, and you
+ need to switch both together).
+
+ (From http://marc.info/?l=linux-kernel&m=93337278602211&w=2)
diff --git a/Documentation/vm/active_mm.txt b/Documentation/vm/active_mm.txt
deleted file mode 100644
index dbf45817405f..000000000000
--- a/Documentation/vm/active_mm.txt
+++ /dev/null
@@ -1,83 +0,0 @@
-List:       linux-kernel
-Subject:    Re: active_mm
-From:       Linus Torvalds <torvalds () transmeta ! com>
-Date:       1999-07-30 21:36:24
-
-Cc'd to linux-kernel, because I don't write explanations all that often,
-and when I do I feel better about more people reading them.
-
-On Fri, 30 Jul 1999, David Mosberger wrote:
->
-> Is there a brief description someplace on how "mm" vs. "active_mm" in
-> the task_struct are supposed to be used?  (My apologies if this was
-> discussed on the mailing lists---I just returned from vacation and
-> wasn't able to follow linux-kernel for a while).
-
-Basically, the new setup is:
-
- - we have "real address spaces" and "anonymous address spaces". The
-   difference is that an anonymous address space doesn't care about the
-   user-level page tables at all, so when we do a context switch into an
-   anonymous address space we just leave the previous address space
-   active.
-
-   The obvious use for a "anonymous address space" is any thread that
-   doesn't need any user mappings - all kernel threads basically fall into
-   this category, but even "real" threads can temporarily say that for
-   some amount of time they are not going to be interested in user space,
-   and that the scheduler might as well try to avoid wasting time on
-   switching the VM state around. Currently only the old-style bdflush
-   sync does that.
-
- - "tsk->mm" points to the "real address space". For an anonymous process,
-   tsk->mm will be NULL, for the logical reason that an anonymous process
-   really doesn't _have_ a real address space at all.
-
- - however, we obviously need to keep track of which address space we
-   "stole" for such an anonymous user. For that, we have "tsk->active_mm",
-   which shows what the currently active address space is.
-
-   The rule is that for a process with a real address space (ie tsk->mm is
-   non-NULL) the active_mm obviously always has to be the same as the real
-   one.
-
-   For a anonymous process, tsk->mm == NULL, and tsk->active_mm is the
-   "borrowed" mm while the anonymous process is running. When the
-   anonymous process gets scheduled away, the borrowed address space is
-   returned and cleared.
-
-To support all that, the "struct mm_struct" now has two counters: a
-"mm_users" counter that is how many "real address space users" there are,
-and a "mm_count" counter that is the number of "lazy" users (ie anonymous
-users) plus one if there are any real users.
-
-Usually there is at least one real user, but it could be that the real
-user exited on another CPU while a lazy user was still active, so you do
-actually get cases where you have a address space that is _only_ used by
-lazy users. That is often a short-lived state, because once that thread
-gets scheduled away in favour of a real thread, the "zombie" mm gets
-released because "mm_users" becomes zero.
-
-Also, a new rule is that _nobody_ ever has "init_mm" as a real MM any
-more. "init_mm" should be considered just a "lazy context when no other
-context is available", and in fact it is mainly used just at bootup when
-no real VM has yet been created. So code that used to check
-
-	if (current->mm == &init_mm)
-
-should generally just do
-
-	if (!current->mm)
-
-instead (which makes more sense anyway - the test is basically one of "do
-we have a user context", and is generally done by the page fault handler
-and things like that).
-
-Anyway, I put a pre-patch-2.3.13-1 on ftp.kernel.org just a moment ago,
-because it slightly changes the interfaces to accommodate the alpha (who
-would have thought it, but the alpha actually ends up having one of the
-ugliest context switch codes - unlike the other architectures where the MM
-and register state is separate, the alpha PALcode joins the two, and you
-need to switch both together).
-
-(From http://marc.info/?l=linux-kernel&m=93337278602211&w=2)
diff --git a/Documentation/vm/balance b/Documentation/vm/balance
deleted file mode 100644
index 964595481af6..000000000000
--- a/Documentation/vm/balance
+++ /dev/null
@@ -1,95 +0,0 @@
-Started Jan 2000 by Kanoj Sarcar <kanoj@sgi.com>
-
-Memory balancing is needed for !__GFP_ATOMIC and !__GFP_KSWAPD_RECLAIM as
-well as for non __GFP_IO allocations.
-
-The first reason why a caller may avoid reclaim is that the caller can not
-sleep due to holding a spinlock or is in interrupt context. The second may
-be that the caller is willing to fail the allocation without incurring the
-overhead of page reclaim. This may happen for opportunistic high-order
-allocation requests that have order-0 fallback options. In such cases,
-the caller may also wish to avoid waking kswapd.
-
-__GFP_IO allocation requests are made to prevent file system deadlocks.
-
-In the absence of non sleepable allocation requests, it seems detrimental
-to be doing balancing. Page reclamation can be kicked off lazily, that
-is, only when needed (aka zone free memory is 0), instead of making it
-a proactive process.
-
-That being said, the kernel should try to fulfill requests for direct
-mapped pages from the direct mapped pool, instead of falling back on
-the dma pool, so as to keep the dma pool filled for dma requests (atomic
-or not). A similar argument applies to highmem and direct mapped pages.
-OTOH, if there is a lot of free dma pages, it is preferable to satisfy
-regular memory requests by allocating one from the dma pool, instead
-of incurring the overhead of regular zone balancing.
-
-In 2.2, memory balancing/page reclamation would kick off only when the
-_total_ number of free pages fell below 1/64 th of total memory. With the
-right ratio of dma and regular memory, it is quite possible that balancing
-would not be done even when the dma zone was completely empty. 2.2 has
-been running production machines of varying memory sizes, and seems to be
-doing fine even with the presence of this problem. In 2.3, due to
-HIGHMEM, this problem is aggravated.
-
-In 2.3, zone balancing can be done in one of two ways: depending on the
-zone size (and possibly of the size of lower class zones), we can decide
-at init time how many free pages we should aim for while balancing any
-zone. The good part is, while balancing, we do not need to look at sizes
-of lower class zones, the bad part is, we might do too frequent balancing
-due to ignoring possibly lower usage in the lower class zones. Also,
-with a slight change in the allocation routine, it is possible to reduce
-the memclass() macro to be a simple equality.
-
-Another possible solution is that we balance only when the free memory
-of a zone _and_ all its lower class zones falls below 1/64th of the
-total memory in the zone and its lower class zones. This fixes the 2.2
-balancing problem, and stays as close to 2.2 behavior as possible. Also,
-the balancing algorithm works the same way on the various architectures,
-which have different numbers and types of zones. If we wanted to get
-fancy, we could assign different weights to free pages in different
-zones in the future.
-
-Note that if the size of the regular zone is huge compared to dma zone,
-it becomes less significant to consider the free dma pages while
-deciding whether to balance the regular zone. The first solution
-becomes more attractive then.
-
-The appended patch implements the second solution. It also "fixes" two
-problems: first, kswapd is woken up as in 2.2 on low memory conditions
-for non-sleepable allocations. Second, the HIGHMEM zone is also balanced,
-so as to give a fighting chance for replace_with_highmem() to get a
-HIGHMEM page, as well as to ensure that HIGHMEM allocations do not
-fall back into regular zone. This also makes sure that HIGHMEM pages
-are not leaked (for example, in situations where a HIGHMEM page is in 
-the swapcache but is not being used by anyone)
-
-kswapd also needs to know about the zones it should balance. kswapd is
-primarily needed in a situation where balancing can not be done, 
-probably because all allocation requests are coming from intr context
-and all process contexts are sleeping. For 2.3, kswapd does not really
-need to balance the highmem zone, since intr context does not request
-highmem pages. kswapd looks at the zone_wake_kswapd field in the zone
-structure to decide whether a zone needs balancing.
-
-Page stealing from process memory and shm is done if stealing the page would
-alleviate memory pressure on any zone in the page's node that has fallen below
-its watermark.
-
-watemark[WMARK_MIN/WMARK_LOW/WMARK_HIGH]/low_on_memory/zone_wake_kswapd: These
-are per-zone fields, used to determine when a zone needs to be balanced. When
-the number of pages falls below watermark[WMARK_MIN], the hysteric field
-low_on_memory gets set. This stays set till the number of free pages becomes
-watermark[WMARK_HIGH]. When low_on_memory is set, page allocation requests will
-try to free some pages in the zone (providing GFP_WAIT is set in the request).
-Orthogonal to this, is the decision to poke kswapd to free some zone pages.
-That decision is not hysteresis based, and is done when the number of free
-pages is below watermark[WMARK_LOW]; in which case zone_wake_kswapd is also set.
-
-
-(Good) Ideas that I have heard:
-1. Dynamic experience should influence balancing: number of failed requests
-for a zone can be tracked and fed into the balancing scheme (jalvo@mbay.net)
-2. Implement a replace_with_highmem()-like replace_with_regular() to preserve
-dma pages. (lkd@tantalophile.demon.co.uk)
diff --git a/Documentation/vm/balance.rst b/Documentation/vm/balance.rst
new file mode 100644
index 000000000000..6a1fadf3e173
--- /dev/null
+++ b/Documentation/vm/balance.rst
@@ -0,0 +1,102 @@
+.. _balance:
+
+================
+Memory Balancing
+================
+
+Started Jan 2000 by Kanoj Sarcar <kanoj@sgi.com>
+
+Memory balancing is needed for !__GFP_ATOMIC and !__GFP_KSWAPD_RECLAIM as
+well as for non __GFP_IO allocations.
+
+The first reason why a caller may avoid reclaim is that the caller can not
+sleep due to holding a spinlock or is in interrupt context. The second may
+be that the caller is willing to fail the allocation without incurring the
+overhead of page reclaim. This may happen for opportunistic high-order
+allocation requests that have order-0 fallback options. In such cases,
+the caller may also wish to avoid waking kswapd.
+
+__GFP_IO allocation requests are made to prevent file system deadlocks.
+
+In the absence of non sleepable allocation requests, it seems detrimental
+to be doing balancing. Page reclamation can be kicked off lazily, that
+is, only when needed (aka zone free memory is 0), instead of making it
+a proactive process.
+
+That being said, the kernel should try to fulfill requests for direct
+mapped pages from the direct mapped pool, instead of falling back on
+the dma pool, so as to keep the dma pool filled for dma requests (atomic
+or not). A similar argument applies to highmem and direct mapped pages.
+OTOH, if there is a lot of free dma pages, it is preferable to satisfy
+regular memory requests by allocating one from the dma pool, instead
+of incurring the overhead of regular zone balancing.
+
+In 2.2, memory balancing/page reclamation would kick off only when the
+_total_ number of free pages fell below 1/64 th of total memory. With the
+right ratio of dma and regular memory, it is quite possible that balancing
+would not be done even when the dma zone was completely empty. 2.2 has
+been running production machines of varying memory sizes, and seems to be
+doing fine even with the presence of this problem. In 2.3, due to
+HIGHMEM, this problem is aggravated.
+
+In 2.3, zone balancing can be done in one of two ways: depending on the
+zone size (and possibly of the size of lower class zones), we can decide
+at init time how many free pages we should aim for while balancing any
+zone. The good part is, while balancing, we do not need to look at sizes
+of lower class zones, the bad part is, we might do too frequent balancing
+due to ignoring possibly lower usage in the lower class zones. Also,
+with a slight change in the allocation routine, it is possible to reduce
+the memclass() macro to be a simple equality.
+
+Another possible solution is that we balance only when the free memory
+of a zone _and_ all its lower class zones falls below 1/64th of the
+total memory in the zone and its lower class zones. This fixes the 2.2
+balancing problem, and stays as close to 2.2 behavior as possible. Also,
+the balancing algorithm works the same way on the various architectures,
+which have different numbers and types of zones. If we wanted to get
+fancy, we could assign different weights to free pages in different
+zones in the future.
+
+Note that if the size of the regular zone is huge compared to dma zone,
+it becomes less significant to consider the free dma pages while
+deciding whether to balance the regular zone. The first solution
+becomes more attractive then.
+
+The appended patch implements the second solution. It also "fixes" two
+problems: first, kswapd is woken up as in 2.2 on low memory conditions
+for non-sleepable allocations. Second, the HIGHMEM zone is also balanced,
+so as to give a fighting chance for replace_with_highmem() to get a
+HIGHMEM page, as well as to ensure that HIGHMEM allocations do not
+fall back into regular zone. This also makes sure that HIGHMEM pages
+are not leaked (for example, in situations where a HIGHMEM page is in
+the swapcache but is not being used by anyone)
+
+kswapd also needs to know about the zones it should balance. kswapd is
+primarily needed in a situation where balancing can not be done,
+probably because all allocation requests are coming from intr context
+and all process contexts are sleeping. For 2.3, kswapd does not really
+need to balance the highmem zone, since intr context does not request
+highmem pages. kswapd looks at the zone_wake_kswapd field in the zone
+structure to decide whether a zone needs balancing.
+
+Page stealing from process memory and shm is done if stealing the page would
+alleviate memory pressure on any zone in the page's node that has fallen below
+its watermark.
+
+watemark[WMARK_MIN/WMARK_LOW/WMARK_HIGH]/low_on_memory/zone_wake_kswapd: These
+are per-zone fields, used to determine when a zone needs to be balanced. When
+the number of pages falls below watermark[WMARK_MIN], the hysteric field
+low_on_memory gets set. This stays set till the number of free pages becomes
+watermark[WMARK_HIGH]. When low_on_memory is set, page allocation requests will
+try to free some pages in the zone (providing GFP_WAIT is set in the request).
+Orthogonal to this, is the decision to poke kswapd to free some zone pages.
+That decision is not hysteresis based, and is done when the number of free
+pages is below watermark[WMARK_LOW]; in which case zone_wake_kswapd is also set.
+
+
+(Good) Ideas that I have heard:
+
+1. Dynamic experience should influence balancing: number of failed requests
+   for a zone can be tracked and fed into the balancing scheme (jalvo@mbay.net)
+2. Implement a replace_with_highmem()-like replace_with_regular() to preserve
+   dma pages. (lkd@tantalophile.demon.co.uk)
diff --git a/Documentation/vm/cleancache.rst b/Documentation/vm/cleancache.rst
new file mode 100644
index 000000000000..68cba9131c31
--- /dev/null
+++ b/Documentation/vm/cleancache.rst
@@ -0,0 +1,296 @@
+.. _cleancache:
+
+==========
+Cleancache
+==========
+
+Motivation
+==========
+
+Cleancache is a new optional feature provided by the VFS layer that
+potentially dramatically increases page cache effectiveness for
+many workloads in many environments at a negligible cost.
+
+Cleancache can be thought of as a page-granularity victim cache for clean
+pages that the kernel's pageframe replacement algorithm (PFRA) would like
+to keep around, but can't since there isn't enough memory.  So when the
+PFRA "evicts" a page, it first attempts to use cleancache code to
+put the data contained in that page into "transcendent memory", memory
+that is not directly accessible or addressable by the kernel and is
+of unknown and possibly time-varying size.
+
+Later, when a cleancache-enabled filesystem wishes to access a page
+in a file on disk, it first checks cleancache to see if it already
+contains it; if it does, the page of data is copied into the kernel
+and a disk access is avoided.
+
+Transcendent memory "drivers" for cleancache are currently implemented
+in Xen (using hypervisor memory) and zcache (using in-kernel compressed
+memory) and other implementations are in development.
+
+:ref:`FAQs <faq>` are included below.
+
+Implementation Overview
+=======================
+
+A cleancache "backend" that provides transcendent memory registers itself
+to the kernel's cleancache "frontend" by calling cleancache_register_ops,
+passing a pointer to a cleancache_ops structure with funcs set appropriately.
+The functions provided must conform to certain semantics as follows:
+
+Most important, cleancache is "ephemeral".  Pages which are copied into
+cleancache have an indefinite lifetime which is completely unknowable
+by the kernel and so may or may not still be in cleancache at any later time.
+Thus, as its name implies, cleancache is not suitable for dirty pages.
+Cleancache has complete discretion over what pages to preserve and what
+pages to discard and when.
+
+Mounting a cleancache-enabled filesystem should call "init_fs" to obtain a
+pool id which, if positive, must be saved in the filesystem's superblock;
+a negative return value indicates failure.  A "put_page" will copy a
+(presumably about-to-be-evicted) page into cleancache and associate it with
+the pool id, a file key, and a page index into the file.  (The combination
+of a pool id, a file key, and an index is sometimes called a "handle".)
+A "get_page" will copy the page, if found, from cleancache into kernel memory.
+An "invalidate_page" will ensure the page no longer is present in cleancache;
+an "invalidate_inode" will invalidate all pages associated with the specified
+file; and, when a filesystem is unmounted, an "invalidate_fs" will invalidate
+all pages in all files specified by the given pool id and also surrender
+the pool id.
+
+An "init_shared_fs", like init_fs, obtains a pool id but tells cleancache
+to treat the pool as shared using a 128-bit UUID as a key.  On systems
+that may run multiple kernels (such as hard partitioned or virtualized
+systems) that may share a clustered filesystem, and where cleancache
+may be shared among those kernels, calls to init_shared_fs that specify the
+same UUID will receive the same pool id, thus allowing the pages to
+be shared.  Note that any security requirements must be imposed outside
+of the kernel (e.g. by "tools" that control cleancache).  Or a
+cleancache implementation can simply disable shared_init by always
+returning a negative value.
+
+If a get_page is successful on a non-shared pool, the page is invalidated
+(thus making cleancache an "exclusive" cache).  On a shared pool, the page
+is NOT invalidated on a successful get_page so that it remains accessible to
+other sharers.  The kernel is responsible for ensuring coherency between
+cleancache (shared or not), the page cache, and the filesystem, using
+cleancache invalidate operations as required.
+
+Note that cleancache must enforce put-put-get coherency and get-get
+coherency.  For the former, if two puts are made to the same handle but
+with different data, say AAA by the first put and BBB by the second, a
+subsequent get can never return the stale data (AAA).  For get-get coherency,
+if a get for a given handle fails, subsequent gets for that handle will
+never succeed unless preceded by a successful put with that handle.
+
+Last, cleancache provides no SMP serialization guarantees; if two
+different Linux threads are simultaneously putting and invalidating a page
+with the same handle, the results are indeterminate.  Callers must
+lock the page to ensure serial behavior.
+
+Cleancache Performance Metrics
+==============================
+
+If properly configured, monitoring of cleancache is done via debugfs in
+the `/sys/kernel/debug/cleancache` directory.  The effectiveness of cleancache
+can be measured (across all filesystems) with:
+
+``succ_gets``
+	number of gets that were successful
+
+``failed_gets``
+	number of gets that failed
+
+``puts``
+	number of puts attempted (all "succeed")
+
+``invalidates``
+	number of invalidates attempted
+
+A backend implementation may provide additional metrics.
+
+.. _faq:
+
+FAQ
+===
+
+* Where's the value? (Andrew Morton)
+
+Cleancache provides a significant performance benefit to many workloads
+in many environments with negligible overhead by improving the
+effectiveness of the pagecache.  Clean pagecache pages are
+saved in transcendent memory (RAM that is otherwise not directly
+addressable to the kernel); fetching those pages later avoids "refaults"
+and thus disk reads.
+
+Cleancache (and its sister code "frontswap") provide interfaces for
+this transcendent memory (aka "tmem"), which conceptually lies between
+fast kernel-directly-addressable RAM and slower DMA/asynchronous devices.
+Disallowing direct kernel or userland reads/writes to tmem
+is ideal when data is transformed to a different form and size (such
+as with compression) or secretly moved (as might be useful for write-
+balancing for some RAM-like devices).  Evicted page-cache pages (and
+swap pages) are a great use for this kind of slower-than-RAM-but-much-
+faster-than-disk transcendent memory, and the cleancache (and frontswap)
+"page-object-oriented" specification provides a nice way to read and
+write -- and indirectly "name" -- the pages.
+
+In the virtual case, the whole point of virtualization is to statistically
+multiplex physical resources across the varying demands of multiple
+virtual machines.  This is really hard to do with RAM and efforts to
+do it well with no kernel change have essentially failed (except in some
+well-publicized special-case workloads).  Cleancache -- and frontswap --
+with a fairly small impact on the kernel, provide a huge amount
+of flexibility for more dynamic, flexible RAM multiplexing.
+Specifically, the Xen Transcendent Memory backend allows otherwise
+"fallow" hypervisor-owned RAM to not only be "time-shared" between multiple
+virtual machines, but the pages can be compressed and deduplicated to
+optimize RAM utilization.  And when guest OS's are induced to surrender
+underutilized RAM (e.g. with "self-ballooning"), page cache pages
+are the first to go, and cleancache allows those pages to be
+saved and reclaimed if overall host system memory conditions allow.
+
+And the identical interface used for cleancache can be used in
+physical systems as well.  The zcache driver acts as a memory-hungry
+device that stores pages of data in a compressed state.  And
+the proposed "RAMster" driver shares RAM across multiple physical
+systems.
+
+* Why does cleancache have its sticky fingers so deep inside the
+  filesystems and VFS? (Andrew Morton and Christoph Hellwig)
+
+The core hooks for cleancache in VFS are in most cases a single line
+and the minimum set are placed precisely where needed to maintain
+coherency (via cleancache_invalidate operations) between cleancache,
+the page cache, and disk.  All hooks compile into nothingness if
+cleancache is config'ed off and turn into a function-pointer-
+compare-to-NULL if config'ed on but no backend claims the ops
+functions, or to a compare-struct-element-to-negative if a
+backend claims the ops functions but a filesystem doesn't enable
+cleancache.
+
+Some filesystems are built entirely on top of VFS and the hooks
+in VFS are sufficient, so don't require an "init_fs" hook; the
+initial implementation of cleancache didn't provide this hook.
+But for some filesystems (such as btrfs), the VFS hooks are
+incomplete and one or more hooks in fs-specific code are required.
+And for some other filesystems, such as tmpfs, cleancache may
+be counterproductive.  So it seemed prudent to require a filesystem
+to "opt in" to use cleancache, which requires adding a hook in
+each filesystem.  Not all filesystems are supported by cleancache
+only because they haven't been tested.  The existing set should
+be sufficient to validate the concept, the opt-in approach means
+that untested filesystems are not affected, and the hooks in the
+existing filesystems should make it very easy to add more
+filesystems in the future.
+
+The total impact of the hooks to existing fs and mm files is only
+about 40 lines added (not counting comments and blank lines).
+
+* Why not make cleancache asynchronous and batched so it can more
+  easily interface with real devices with DMA instead of copying each
+  individual page? (Minchan Kim)
+
+The one-page-at-a-time copy semantics simplifies the implementation
+on both the frontend and backend and also allows the backend to
+do fancy things on-the-fly like page compression and
+page deduplication.  And since the data is "gone" (copied into/out
+of the pageframe) before the cleancache get/put call returns,
+a great deal of race conditions and potential coherency issues
+are avoided.  While the interface seems odd for a "real device"
+or for real kernel-addressable RAM, it makes perfect sense for
+transcendent memory.
+
+* Why is non-shared cleancache "exclusive"?  And where is the
+  page "invalidated" after a "get"? (Minchan Kim)
+
+The main reason is to free up space in transcendent memory and
+to avoid unnecessary cleancache_invalidate calls.  If you want inclusive,
+the page can be "put" immediately following the "get".  If
+put-after-get for inclusive becomes common, the interface could
+be easily extended to add a "get_no_invalidate" call.
+
+The invalidate is done by the cleancache backend implementation.
+
+* What's the performance impact?
+
+Performance analysis has been presented at OLS'09 and LCA'10.
+Briefly, performance gains can be significant on most workloads,
+especially when memory pressure is high (e.g. when RAM is
+overcommitted in a virtual workload); and because the hooks are
+invoked primarily in place of or in addition to a disk read/write,
+overhead is negligible even in worst case workloads.  Basically
+cleancache replaces I/O with memory-copy-CPU-overhead; on older
+single-core systems with slow memory-copy speeds, cleancache
+has little value, but in newer multicore machines, especially
+consolidated/virtualized machines, it has great value.
+
+* How do I add cleancache support for filesystem X? (Boaz Harrash)
+
+Filesystems that are well-behaved and conform to certain
+restrictions can utilize cleancache simply by making a call to
+cleancache_init_fs at mount time.  Unusual, misbehaving, or
+poorly layered filesystems must either add additional hooks
+and/or undergo extensive additional testing... or should just
+not enable the optional cleancache.
+
+Some points for a filesystem to consider:
+
+  - The FS should be block-device-based (e.g. a ram-based FS such
+    as tmpfs should not enable cleancache)
+  - To ensure coherency/correctness, the FS must ensure that all
+    file removal or truncation operations either go through VFS or
+    add hooks to do the equivalent cleancache "invalidate" operations
+  - To ensure coherency/correctness, either inode numbers must
+    be unique across the lifetime of the on-disk file OR the
+    FS must provide an "encode_fh" function.
+  - The FS must call the VFS superblock alloc and deactivate routines
+    or add hooks to do the equivalent cleancache calls done there.
+  - To maximize performance, all pages fetched from the FS should
+    go through the do_mpag_readpage routine or the FS should add
+    hooks to do the equivalent (cf. btrfs)
+  - Currently, the FS blocksize must be the same as PAGESIZE.  This
+    is not an architectural restriction, but no backends currently
+    support anything different.
+  - A clustered FS should invoke the "shared_init_fs" cleancache
+    hook to get best performance for some backends.
+
+* Why not use the KVA of the inode as the key? (Christoph Hellwig)
+
+If cleancache would use the inode virtual address instead of
+inode/filehandle, the pool id could be eliminated.  But, this
+won't work because cleancache retains pagecache data pages
+persistently even when the inode has been pruned from the
+inode unused list, and only invalidates the data page if the file
+gets removed/truncated.  So if cleancache used the inode kva,
+there would be potential coherency issues if/when the inode
+kva is reused for a different file.  Alternately, if cleancache
+invalidated the pages when the inode kva was freed, much of the value
+of cleancache would be lost because the cache of pages in cleanache
+is potentially much larger than the kernel pagecache and is most
+useful if the pages survive inode cache removal.
+
+* Why is a global variable required?
+
+The cleancache_enabled flag is checked in all of the frequently-used
+cleancache hooks.  The alternative is a function call to check a static
+variable. Since cleancache is enabled dynamically at runtime, systems
+that don't enable cleancache would suffer thousands (possibly
+tens-of-thousands) of unnecessary function calls per second.  So the
+global variable allows cleancache to be enabled by default at compile
+time, but have insignificant performance impact when cleancache remains
+disabled at runtime.
+
+* Does cleanache work with KVM?
+
+The memory model of KVM is sufficiently different that a cleancache
+backend may have less value for KVM.  This remains to be tested,
+especially in an overcommitted system.
+
+* Does cleancache work in userspace?  It sounds useful for
+  memory hungry caches like web browsers.  (Jamie Lokier)
+
+No plans yet, though we agree it sounds useful, at least for
+apps that bypass the page cache (e.g. O_DIRECT).
+
+Last updated: Dan Magenheimer, April 13 2011
diff --git a/Documentation/vm/cleancache.txt b/Documentation/vm/cleancache.txt
deleted file mode 100644
index e4b49df7a048..000000000000
--- a/Documentation/vm/cleancache.txt
+++ /dev/null
@@ -1,277 +0,0 @@
-MOTIVATION
-
-Cleancache is a new optional feature provided by the VFS layer that
-potentially dramatically increases page cache effectiveness for
-many workloads in many environments at a negligible cost.
-
-Cleancache can be thought of as a page-granularity victim cache for clean
-pages that the kernel's pageframe replacement algorithm (PFRA) would like
-to keep around, but can't since there isn't enough memory.  So when the
-PFRA "evicts" a page, it first attempts to use cleancache code to
-put the data contained in that page into "transcendent memory", memory
-that is not directly accessible or addressable by the kernel and is
-of unknown and possibly time-varying size.
-
-Later, when a cleancache-enabled filesystem wishes to access a page
-in a file on disk, it first checks cleancache to see if it already
-contains it; if it does, the page of data is copied into the kernel
-and a disk access is avoided.
-
-Transcendent memory "drivers" for cleancache are currently implemented
-in Xen (using hypervisor memory) and zcache (using in-kernel compressed
-memory) and other implementations are in development.
-
-FAQs are included below.
-
-IMPLEMENTATION OVERVIEW
-
-A cleancache "backend" that provides transcendent memory registers itself
-to the kernel's cleancache "frontend" by calling cleancache_register_ops,
-passing a pointer to a cleancache_ops structure with funcs set appropriately.
-The functions provided must conform to certain semantics as follows:
-
-Most important, cleancache is "ephemeral".  Pages which are copied into
-cleancache have an indefinite lifetime which is completely unknowable
-by the kernel and so may or may not still be in cleancache at any later time.
-Thus, as its name implies, cleancache is not suitable for dirty pages.
-Cleancache has complete discretion over what pages to preserve and what
-pages to discard and when.
-
-Mounting a cleancache-enabled filesystem should call "init_fs" to obtain a
-pool id which, if positive, must be saved in the filesystem's superblock;
-a negative return value indicates failure.  A "put_page" will copy a
-(presumably about-to-be-evicted) page into cleancache and associate it with
-the pool id, a file key, and a page index into the file.  (The combination
-of a pool id, a file key, and an index is sometimes called a "handle".)
-A "get_page" will copy the page, if found, from cleancache into kernel memory.
-An "invalidate_page" will ensure the page no longer is present in cleancache;
-an "invalidate_inode" will invalidate all pages associated with the specified
-file; and, when a filesystem is unmounted, an "invalidate_fs" will invalidate
-all pages in all files specified by the given pool id and also surrender
-the pool id.
-
-An "init_shared_fs", like init_fs, obtains a pool id but tells cleancache
-to treat the pool as shared using a 128-bit UUID as a key.  On systems
-that may run multiple kernels (such as hard partitioned or virtualized
-systems) that may share a clustered filesystem, and where cleancache
-may be shared among those kernels, calls to init_shared_fs that specify the
-same UUID will receive the same pool id, thus allowing the pages to
-be shared.  Note that any security requirements must be imposed outside
-of the kernel (e.g. by "tools" that control cleancache).  Or a
-cleancache implementation can simply disable shared_init by always
-returning a negative value.
-
-If a get_page is successful on a non-shared pool, the page is invalidated
-(thus making cleancache an "exclusive" cache).  On a shared pool, the page
-is NOT invalidated on a successful get_page so that it remains accessible to
-other sharers.  The kernel is responsible for ensuring coherency between
-cleancache (shared or not), the page cache, and the filesystem, using
-cleancache invalidate operations as required.
-
-Note that cleancache must enforce put-put-get coherency and get-get
-coherency.  For the former, if two puts are made to the same handle but
-with different data, say AAA by the first put and BBB by the second, a
-subsequent get can never return the stale data (AAA).  For get-get coherency,
-if a get for a given handle fails, subsequent gets for that handle will
-never succeed unless preceded by a successful put with that handle.
-
-Last, cleancache provides no SMP serialization guarantees; if two
-different Linux threads are simultaneously putting and invalidating a page
-with the same handle, the results are indeterminate.  Callers must
-lock the page to ensure serial behavior.
-
-CLEANCACHE PERFORMANCE METRICS
-
-If properly configured, monitoring of cleancache is done via debugfs in
-the /sys/kernel/debug/cleancache directory.  The effectiveness of cleancache
-can be measured (across all filesystems) with:
-
-succ_gets	- number of gets that were successful
-failed_gets	- number of gets that failed
-puts		- number of puts attempted (all "succeed")
-invalidates	- number of invalidates attempted
-
-A backend implementation may provide additional metrics.
-
-FAQ
-
-1) Where's the value? (Andrew Morton)
-
-Cleancache provides a significant performance benefit to many workloads
-in many environments with negligible overhead by improving the
-effectiveness of the pagecache.  Clean pagecache pages are
-saved in transcendent memory (RAM that is otherwise not directly
-addressable to the kernel); fetching those pages later avoids "refaults"
-and thus disk reads.
-
-Cleancache (and its sister code "frontswap") provide interfaces for
-this transcendent memory (aka "tmem"), which conceptually lies between
-fast kernel-directly-addressable RAM and slower DMA/asynchronous devices.
-Disallowing direct kernel or userland reads/writes to tmem
-is ideal when data is transformed to a different form and size (such
-as with compression) or secretly moved (as might be useful for write-
-balancing for some RAM-like devices).  Evicted page-cache pages (and
-swap pages) are a great use for this kind of slower-than-RAM-but-much-
-faster-than-disk transcendent memory, and the cleancache (and frontswap)
-"page-object-oriented" specification provides a nice way to read and
-write -- and indirectly "name" -- the pages.
-
-In the virtual case, the whole point of virtualization is to statistically
-multiplex physical resources across the varying demands of multiple
-virtual machines.  This is really hard to do with RAM and efforts to
-do it well with no kernel change have essentially failed (except in some
-well-publicized special-case workloads).  Cleancache -- and frontswap --
-with a fairly small impact on the kernel, provide a huge amount
-of flexibility for more dynamic, flexible RAM multiplexing.
-Specifically, the Xen Transcendent Memory backend allows otherwise
-"fallow" hypervisor-owned RAM to not only be "time-shared" between multiple
-virtual machines, but the pages can be compressed and deduplicated to
-optimize RAM utilization.  And when guest OS's are induced to surrender
-underutilized RAM (e.g. with "self-ballooning"), page cache pages
-are the first to go, and cleancache allows those pages to be
-saved and reclaimed if overall host system memory conditions allow.
-
-And the identical interface used for cleancache can be used in
-physical systems as well.  The zcache driver acts as a memory-hungry
-device that stores pages of data in a compressed state.  And
-the proposed "RAMster" driver shares RAM across multiple physical
-systems.
-
-2) Why does cleancache have its sticky fingers so deep inside the
-   filesystems and VFS? (Andrew Morton and Christoph Hellwig)
-
-The core hooks for cleancache in VFS are in most cases a single line
-and the minimum set are placed precisely where needed to maintain
-coherency (via cleancache_invalidate operations) between cleancache,
-the page cache, and disk.  All hooks compile into nothingness if
-cleancache is config'ed off and turn into a function-pointer-
-compare-to-NULL if config'ed on but no backend claims the ops
-functions, or to a compare-struct-element-to-negative if a
-backend claims the ops functions but a filesystem doesn't enable
-cleancache.
-
-Some filesystems are built entirely on top of VFS and the hooks
-in VFS are sufficient, so don't require an "init_fs" hook; the
-initial implementation of cleancache didn't provide this hook.
-But for some filesystems (such as btrfs), the VFS hooks are
-incomplete and one or more hooks in fs-specific code are required.
-And for some other filesystems, such as tmpfs, cleancache may
-be counterproductive.  So it seemed prudent to require a filesystem
-to "opt in" to use cleancache, which requires adding a hook in
-each filesystem.  Not all filesystems are supported by cleancache
-only because they haven't been tested.  The existing set should
-be sufficient to validate the concept, the opt-in approach means
-that untested filesystems are not affected, and the hooks in the
-existing filesystems should make it very easy to add more
-filesystems in the future.
-
-The total impact of the hooks to existing fs and mm files is only
-about 40 lines added (not counting comments and blank lines).
-
-3) Why not make cleancache asynchronous and batched so it can
-   more easily interface with real devices with DMA instead
-   of copying each individual page? (Minchan Kim)
-
-The one-page-at-a-time copy semantics simplifies the implementation
-on both the frontend and backend and also allows the backend to
-do fancy things on-the-fly like page compression and
-page deduplication.  And since the data is "gone" (copied into/out
-of the pageframe) before the cleancache get/put call returns,
-a great deal of race conditions and potential coherency issues
-are avoided.  While the interface seems odd for a "real device"
-or for real kernel-addressable RAM, it makes perfect sense for
-transcendent memory.
-
-4) Why is non-shared cleancache "exclusive"?  And where is the
-   page "invalidated" after a "get"? (Minchan Kim)
-
-The main reason is to free up space in transcendent memory and
-to avoid unnecessary cleancache_invalidate calls.  If you want inclusive,
-the page can be "put" immediately following the "get".  If
-put-after-get for inclusive becomes common, the interface could
-be easily extended to add a "get_no_invalidate" call.
-
-The invalidate is done by the cleancache backend implementation.
-
-5) What's the performance impact?
-
-Performance analysis has been presented at OLS'09 and LCA'10.
-Briefly, performance gains can be significant on most workloads,
-especially when memory pressure is high (e.g. when RAM is
-overcommitted in a virtual workload); and because the hooks are
-invoked primarily in place of or in addition to a disk read/write,
-overhead is negligible even in worst case workloads.  Basically
-cleancache replaces I/O with memory-copy-CPU-overhead; on older
-single-core systems with slow memory-copy speeds, cleancache
-has little value, but in newer multicore machines, especially
-consolidated/virtualized machines, it has great value.
-
-6) How do I add cleancache support for filesystem X? (Boaz Harrash)
-
-Filesystems that are well-behaved and conform to certain
-restrictions can utilize cleancache simply by making a call to
-cleancache_init_fs at mount time.  Unusual, misbehaving, or
-poorly layered filesystems must either add additional hooks
-and/or undergo extensive additional testing... or should just
-not enable the optional cleancache.
-
-Some points for a filesystem to consider:
-
-- The FS should be block-device-based (e.g. a ram-based FS such
-  as tmpfs should not enable cleancache)
-- To ensure coherency/correctness, the FS must ensure that all
-  file removal or truncation operations either go through VFS or
-  add hooks to do the equivalent cleancache "invalidate" operations
-- To ensure coherency/correctness, either inode numbers must
-  be unique across the lifetime of the on-disk file OR the
-  FS must provide an "encode_fh" function.
-- The FS must call the VFS superblock alloc and deactivate routines
-  or add hooks to do the equivalent cleancache calls done there.
-- To maximize performance, all pages fetched from the FS should
-  go through the do_mpag_readpage routine or the FS should add
-  hooks to do the equivalent (cf. btrfs)
-- Currently, the FS blocksize must be the same as PAGESIZE.  This
-  is not an architectural restriction, but no backends currently
-  support anything different.
-- A clustered FS should invoke the "shared_init_fs" cleancache
-  hook to get best performance for some backends.
-
-7) Why not use the KVA of the inode as the key? (Christoph Hellwig)
-
-If cleancache would use the inode virtual address instead of
-inode/filehandle, the pool id could be eliminated.  But, this
-won't work because cleancache retains pagecache data pages
-persistently even when the inode has been pruned from the
-inode unused list, and only invalidates the data page if the file
-gets removed/truncated.  So if cleancache used the inode kva,
-there would be potential coherency issues if/when the inode
-kva is reused for a different file.  Alternately, if cleancache
-invalidated the pages when the inode kva was freed, much of the value
-of cleancache would be lost because the cache of pages in cleanache
-is potentially much larger than the kernel pagecache and is most
-useful if the pages survive inode cache removal.
-
-8) Why is a global variable required?
-
-The cleancache_enabled flag is checked in all of the frequently-used
-cleancache hooks.  The alternative is a function call to check a static
-variable. Since cleancache is enabled dynamically at runtime, systems
-that don't enable cleancache would suffer thousands (possibly
-tens-of-thousands) of unnecessary function calls per second.  So the
-global variable allows cleancache to be enabled by default at compile
-time, but have insignificant performance impact when cleancache remains
-disabled at runtime.
-
-9) Does cleanache work with KVM?
-
-The memory model of KVM is sufficiently different that a cleancache
-backend may have less value for KVM.  This remains to be tested,
-especially in an overcommitted system.
-
-10) Does cleancache work in userspace?  It sounds useful for
-   memory hungry caches like web browsers.  (Jamie Lokier)
-
-No plans yet, though we agree it sounds useful, at least for
-apps that bypass the page cache (e.g. O_DIRECT).
-
-Last updated: Dan Magenheimer, April 13 2011
diff --git a/Documentation/vm/conf.py b/Documentation/vm/conf.py
new file mode 100644
index 000000000000..3b0b601af558
--- /dev/null
+++ b/Documentation/vm/conf.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8; mode: python -*-
+
+project = "Linux Memory Management Documentation"
+
+tags.add("subproject")
+
+latex_documents = [
+    ('index', 'memory-management.tex', project,
+     'The kernel development community', 'manual'),
+]
diff --git a/Documentation/vm/frontswap.rst b/Documentation/vm/frontswap.rst
new file mode 100644
index 000000000000..1979f430c1c5
--- /dev/null
+++ b/Documentation/vm/frontswap.rst
@@ -0,0 +1,293 @@
+.. _frontswap:
+
+=========
+Frontswap
+=========
+
+Frontswap provides a "transcendent memory" interface for swap pages.
+In some environments, dramatic performance savings may be obtained because
+swapped pages are saved in RAM (or a RAM-like device) instead of a swap disk.
+
+(Note, frontswap -- and :ref:`cleancache` (merged at 3.0) -- are the "frontends"
+and the only necessary changes to the core kernel for transcendent memory;
+all other supporting code -- the "backends" -- is implemented as drivers.
+See the LWN.net article `Transcendent memory in a nutshell`_
+for a detailed overview of frontswap and related kernel parts)
+
+.. _Transcendent memory in a nutshell: https://lwn.net/Articles/454795/
+
+Frontswap is so named because it can be thought of as the opposite of
+a "backing" store for a swap device.  The storage is assumed to be
+a synchronous concurrency-safe page-oriented "pseudo-RAM device" conforming
+to the requirements of transcendent memory (such as Xen's "tmem", or
+in-kernel compressed memory, aka "zcache", or future RAM-like devices);
+this pseudo-RAM device is not directly accessible or addressable by the
+kernel and is of unknown and possibly time-varying size.  The driver
+links itself to frontswap by calling frontswap_register_ops to set the
+frontswap_ops funcs appropriately and the functions it provides must
+conform to certain policies as follows:
+
+An "init" prepares the device to receive frontswap pages associated
+with the specified swap device number (aka "type").  A "store" will
+copy the page to transcendent memory and associate it with the type and
+offset associated with the page. A "load" will copy the page, if found,
+from transcendent memory into kernel memory, but will NOT remove the page
+from transcendent memory.  An "invalidate_page" will remove the page
+from transcendent memory and an "invalidate_area" will remove ALL pages
+associated with the swap type (e.g., like swapoff) and notify the "device"
+to refuse further stores with that swap type.
+
+Once a page is successfully stored, a matching load on the page will normally
+succeed.  So when the kernel finds itself in a situation where it needs
+to swap out a page, it first attempts to use frontswap.  If the store returns
+success, the data has been successfully saved to transcendent memory and
+a disk write and, if the data is later read back, a disk read are avoided.
+If a store returns failure, transcendent memory has rejected the data, and the
+page can be written to swap as usual.
+
+If a backend chooses, frontswap can be configured as a "writethrough
+cache" by calling frontswap_writethrough().  In this mode, the reduction
+in swap device writes is lost (and also a non-trivial performance advantage)
+in order to allow the backend to arbitrarily "reclaim" space used to
+store frontswap pages to more completely manage its memory usage.
+
+Note that if a page is stored and the page already exists in transcendent memory
+(a "duplicate" store), either the store succeeds and the data is overwritten,
+or the store fails AND the page is invalidated.  This ensures stale data may
+never be obtained from frontswap.
+
+If properly configured, monitoring of frontswap is done via debugfs in
+the `/sys/kernel/debug/frontswap` directory.  The effectiveness of
+frontswap can be measured (across all swap devices) with:
+
+``failed_stores``
+	how many store attempts have failed
+
+``loads``
+	how many loads were attempted (all should succeed)
+
+``succ_stores``
+	how many store attempts have succeeded
+
+``invalidates``
+	how many invalidates were attempted
+
+A backend implementation may provide additional metrics.
+
+FAQ
+===
+
+* Where's the value?
+
+When a workload starts swapping, performance falls through the floor.
+Frontswap significantly increases performance in many such workloads by
+providing a clean, dynamic interface to read and write swap pages to
+"transcendent memory" that is otherwise not directly addressable to the kernel.
+This interface is ideal when data is transformed to a different form
+and size (such as with compression) or secretly moved (as might be
+useful for write-balancing for some RAM-like devices).  Swap pages (and
+evicted page-cache pages) are a great use for this kind of slower-than-RAM-
+but-much-faster-than-disk "pseudo-RAM device" and the frontswap (and
+cleancache) interface to transcendent memory provides a nice way to read
+and write -- and indirectly "name" -- the pages.
+
+Frontswap -- and cleancache -- with a fairly small impact on the kernel,
+provides a huge amount of flexibility for more dynamic, flexible RAM
+utilization in various system configurations:
+
+In the single kernel case, aka "zcache", pages are compressed and
+stored in local memory, thus increasing the total anonymous pages
+that can be safely kept in RAM.  Zcache essentially trades off CPU
+cycles used in compression/decompression for better memory utilization.
+Benchmarks have shown little or no impact when memory pressure is
+low while providing a significant performance improvement (25%+)
+on some workloads under high memory pressure.
+
+"RAMster" builds on zcache by adding "peer-to-peer" transcendent memory
+support for clustered systems.  Frontswap pages are locally compressed
+as in zcache, but then "remotified" to another system's RAM.  This
+allows RAM to be dynamically load-balanced back-and-forth as needed,
+i.e. when system A is overcommitted, it can swap to system B, and
+vice versa.  RAMster can also be configured as a memory server so
+many servers in a cluster can swap, dynamically as needed, to a single
+server configured with a large amount of RAM... without pre-configuring
+how much of the RAM is available for each of the clients!
+
+In the virtual case, the whole point of virtualization is to statistically
+multiplex physical resources across the varying demands of multiple
+virtual machines.  This is really hard to do with RAM and efforts to do
+it well with no kernel changes have essentially failed (except in some
+well-publicized special-case workloads).
+Specifically, the Xen Transcendent Memory backend allows otherwise
+"fallow" hypervisor-owned RAM to not only be "time-shared" between multiple
+virtual machines, but the pages can be compressed and deduplicated to
+optimize RAM utilization.  And when guest OS's are induced to surrender
+underutilized RAM (e.g. with "selfballooning"), sudden unexpected
+memory pressure may result in swapping; frontswap allows those pages
+to be swapped to and from hypervisor RAM (if overall host system memory
+conditions allow), thus mitigating the potentially awful performance impact
+of unplanned swapping.
+
+A KVM implementation is underway and has been RFC'ed to lkml.  And,
+using frontswap, investigation is also underway on the use of NVM as
+a memory extension technology.
+
+* Sure there may be performance advantages in some situations, but
+  what's the space/time overhead of frontswap?
+
+If CONFIG_FRONTSWAP is disabled, every frontswap hook compiles into
+nothingness and the only overhead is a few extra bytes per swapon'ed
+swap device.  If CONFIG_FRONTSWAP is enabled but no frontswap "backend"
+registers, there is one extra global variable compared to zero for
+every swap page read or written.  If CONFIG_FRONTSWAP is enabled
+AND a frontswap backend registers AND the backend fails every "store"
+request (i.e. provides no memory despite claiming it might),
+CPU overhead is still negligible -- and since every frontswap fail
+precedes a swap page write-to-disk, the system is highly likely
+to be I/O bound and using a small fraction of a percent of a CPU
+will be irrelevant anyway.
+
+As for space, if CONFIG_FRONTSWAP is enabled AND a frontswap backend
+registers, one bit is allocated for every swap page for every swap
+device that is swapon'd.  This is added to the EIGHT bits (which
+was sixteen until about 2.6.34) that the kernel already allocates
+for every swap page for every swap device that is swapon'd.  (Hugh
+Dickins has observed that frontswap could probably steal one of
+the existing eight bits, but let's worry about that minor optimization
+later.)  For very large swap disks (which are rare) on a standard
+4K pagesize, this is 1MB per 32GB swap.
+
+When swap pages are stored in transcendent memory instead of written
+out to disk, there is a side effect that this may create more memory
+pressure that can potentially outweigh the other advantages.  A
+backend, such as zcache, must implement policies to carefully (but
+dynamically) manage memory limits to ensure this doesn't happen.
+
+* OK, how about a quick overview of what this frontswap patch does
+  in terms that a kernel hacker can grok?
+
+Let's assume that a frontswap "backend" has registered during
+kernel initialization; this registration indicates that this
+frontswap backend has access to some "memory" that is not directly
+accessible by the kernel.  Exactly how much memory it provides is
+entirely dynamic and random.
+
+Whenever a swap-device is swapon'd frontswap_init() is called,
+passing the swap device number (aka "type") as a parameter.
+This notifies frontswap to expect attempts to "store" swap pages
+associated with that number.
+
+Whenever the swap subsystem is readying a page to write to a swap
+device (c.f swap_writepage()), frontswap_store is called.  Frontswap
+consults with the frontswap backend and if the backend says it does NOT
+have room, frontswap_store returns -1 and the kernel swaps the page
+to the swap device as normal.  Note that the response from the frontswap
+backend is unpredictable to the kernel; it may choose to never accept a
+page, it could accept every ninth page, or it might accept every
+page.  But if the backend does accept a page, the data from the page
+has already been copied and associated with the type and offset,
+and the backend guarantees the persistence of the data.  In this case,
+frontswap sets a bit in the "frontswap_map" for the swap device
+corresponding to the page offset on the swap device to which it would
+otherwise have written the data.
+
+When the swap subsystem needs to swap-in a page (swap_readpage()),
+it first calls frontswap_load() which checks the frontswap_map to
+see if the page was earlier accepted by the frontswap backend.  If
+it was, the page of data is filled from the frontswap backend and
+the swap-in is complete.  If not, the normal swap-in code is
+executed to obtain the page of data from the real swap device.
+
+So every time the frontswap backend accepts a page, a swap device read
+and (potentially) a swap device write are replaced by a "frontswap backend
+store" and (possibly) a "frontswap backend loads", which are presumably much
+faster.
+
+* Can't frontswap be configured as a "special" swap device that is
+  just higher priority than any real swap device (e.g. like zswap,
+  or maybe swap-over-nbd/NFS)?
+
+No.  First, the existing swap subsystem doesn't allow for any kind of
+swap hierarchy.  Perhaps it could be rewritten to accommodate a hierarchy,
+but this would require fairly drastic changes.  Even if it were
+rewritten, the existing swap subsystem uses the block I/O layer which
+assumes a swap device is fixed size and any page in it is linearly
+addressable.  Frontswap barely touches the existing swap subsystem,
+and works around the constraints of the block I/O subsystem to provide
+a great deal of flexibility and dynamicity.
+
+For example, the acceptance of any swap page by the frontswap backend is
+entirely unpredictable. This is critical to the definition of frontswap
+backends because it grants completely dynamic discretion to the
+backend.  In zcache, one cannot know a priori how compressible a page is.
+"Poorly" compressible pages can be rejected, and "poorly" can itself be
+defined dynamically depending on current memory constraints.
+
+Further, frontswap is entirely synchronous whereas a real swap
+device is, by definition, asynchronous and uses block I/O.  The
+block I/O layer is not only unnecessary, but may perform "optimizations"
+that are inappropriate for a RAM-oriented device including delaying
+the write of some pages for a significant amount of time.  Synchrony is
+required to ensure the dynamicity of the backend and to avoid thorny race
+conditions that would unnecessarily and greatly complicate frontswap
+and/or the block I/O subsystem.  That said, only the initial "store"
+and "load" operations need be synchronous.  A separate asynchronous thread
+is free to manipulate the pages stored by frontswap.  For example,
+the "remotification" thread in RAMster uses standard asynchronous
+kernel sockets to move compressed frontswap pages to a remote machine.
+Similarly, a KVM guest-side implementation could do in-guest compression
+and use "batched" hypercalls.
+
+In a virtualized environment, the dynamicity allows the hypervisor
+(or host OS) to do "intelligent overcommit".  For example, it can
+choose to accept pages only until host-swapping might be imminent,
+then force guests to do their own swapping.
+
+There is a downside to the transcendent memory specifications for
+frontswap:  Since any "store" might fail, there must always be a real
+slot on a real swap device to swap the page.  Thus frontswap must be
+implemented as a "shadow" to every swapon'd device with the potential
+capability of holding every page that the swap device might have held
+and the possibility that it might hold no pages at all.  This means
+that frontswap cannot contain more pages than the total of swapon'd
+swap devices.  For example, if NO swap device is configured on some
+installation, frontswap is useless.  Swapless portable devices
+can still use frontswap but a backend for such devices must configure
+some kind of "ghost" swap device and ensure that it is never used.
+
+* Why this weird definition about "duplicate stores"?  If a page
+  has been previously successfully stored, can't it always be
+  successfully overwritten?
+
+Nearly always it can, but no, sometimes it cannot.  Consider an example
+where data is compressed and the original 4K page has been compressed
+to 1K.  Now an attempt is made to overwrite the page with data that
+is non-compressible and so would take the entire 4K.  But the backend
+has no more space.  In this case, the store must be rejected.  Whenever
+frontswap rejects a store that would overwrite, it also must invalidate
+the old data and ensure that it is no longer accessible.  Since the
+swap subsystem then writes the new data to the read swap device,
+this is the correct course of action to ensure coherency.
+
+* What is frontswap_shrink for?
+
+When the (non-frontswap) swap subsystem swaps out a page to a real
+swap device, that page is only taking up low-value pre-allocated disk
+space.  But if frontswap has placed a page in transcendent memory, that
+page may be taking up valuable real estate.  The frontswap_shrink
+routine allows code outside of the swap subsystem to force pages out
+of the memory managed by frontswap and back into kernel-addressable memory.
+For example, in RAMster, a "suction driver" thread will attempt
+to "repatriate" pages sent to a remote machine back to the local machine;
+this is driven using the frontswap_shrink mechanism when memory pressure
+subsides.
+
+* Why does the frontswap patch create the new include file swapfile.h?
+
+The frontswap code depends on some swap-subsystem-internal data
+structures that have, over the years, moved back and forth between
+static and global.  This seemed a reasonable compromise:  Define
+them as global but declare them in a new include file that isn't
+included by the large number of source files that include swap.h.
+
+Dan Magenheimer, last updated April 9, 2012
diff --git a/Documentation/vm/frontswap.txt b/Documentation/vm/frontswap.txt
deleted file mode 100644
index c71a019be600..000000000000
--- a/Documentation/vm/frontswap.txt
+++ /dev/null
@@ -1,278 +0,0 @@
-Frontswap provides a "transcendent memory" interface for swap pages.
-In some environments, dramatic performance savings may be obtained because
-swapped pages are saved in RAM (or a RAM-like device) instead of a swap disk.
-
-(Note, frontswap -- and cleancache (merged at 3.0) -- are the "frontends"
-and the only necessary changes to the core kernel for transcendent memory;
-all other supporting code -- the "backends" -- is implemented as drivers.
-See the LWN.net article "Transcendent memory in a nutshell" for a detailed
-overview of frontswap and related kernel parts:
-https://lwn.net/Articles/454795/ )
-
-Frontswap is so named because it can be thought of as the opposite of
-a "backing" store for a swap device.  The storage is assumed to be
-a synchronous concurrency-safe page-oriented "pseudo-RAM device" conforming
-to the requirements of transcendent memory (such as Xen's "tmem", or
-in-kernel compressed memory, aka "zcache", or future RAM-like devices);
-this pseudo-RAM device is not directly accessible or addressable by the
-kernel and is of unknown and possibly time-varying size.  The driver
-links itself to frontswap by calling frontswap_register_ops to set the
-frontswap_ops funcs appropriately and the functions it provides must
-conform to certain policies as follows:
-
-An "init" prepares the device to receive frontswap pages associated
-with the specified swap device number (aka "type").  A "store" will
-copy the page to transcendent memory and associate it with the type and
-offset associated with the page. A "load" will copy the page, if found,
-from transcendent memory into kernel memory, but will NOT remove the page
-from transcendent memory.  An "invalidate_page" will remove the page
-from transcendent memory and an "invalidate_area" will remove ALL pages
-associated with the swap type (e.g., like swapoff) and notify the "device"
-to refuse further stores with that swap type.
-
-Once a page is successfully stored, a matching load on the page will normally
-succeed.  So when the kernel finds itself in a situation where it needs
-to swap out a page, it first attempts to use frontswap.  If the store returns
-success, the data has been successfully saved to transcendent memory and
-a disk write and, if the data is later read back, a disk read are avoided.
-If a store returns failure, transcendent memory has rejected the data, and the
-page can be written to swap as usual.
-
-If a backend chooses, frontswap can be configured as a "writethrough
-cache" by calling frontswap_writethrough().  In this mode, the reduction
-in swap device writes is lost (and also a non-trivial performance advantage)
-in order to allow the backend to arbitrarily "reclaim" space used to
-store frontswap pages to more completely manage its memory usage.
-
-Note that if a page is stored and the page already exists in transcendent memory
-(a "duplicate" store), either the store succeeds and the data is overwritten,
-or the store fails AND the page is invalidated.  This ensures stale data may
-never be obtained from frontswap.
-
-If properly configured, monitoring of frontswap is done via debugfs in
-the /sys/kernel/debug/frontswap directory.  The effectiveness of
-frontswap can be measured (across all swap devices) with:
-
-failed_stores	- how many store attempts have failed
-loads		- how many loads were attempted (all should succeed)
-succ_stores	- how many store attempts have succeeded
-invalidates	- how many invalidates were attempted
-
-A backend implementation may provide additional metrics.
-
-FAQ
-
-1) Where's the value?
-
-When a workload starts swapping, performance falls through the floor.
-Frontswap significantly increases performance in many such workloads by
-providing a clean, dynamic interface to read and write swap pages to
-"transcendent memory" that is otherwise not directly addressable to the kernel.
-This interface is ideal when data is transformed to a different form
-and size (such as with compression) or secretly moved (as might be
-useful for write-balancing for some RAM-like devices).  Swap pages (and
-evicted page-cache pages) are a great use for this kind of slower-than-RAM-
-but-much-faster-than-disk "pseudo-RAM device" and the frontswap (and
-cleancache) interface to transcendent memory provides a nice way to read
-and write -- and indirectly "name" -- the pages.
-
-Frontswap -- and cleancache -- with a fairly small impact on the kernel,
-provides a huge amount of flexibility for more dynamic, flexible RAM
-utilization in various system configurations:
-
-In the single kernel case, aka "zcache", pages are compressed and
-stored in local memory, thus increasing the total anonymous pages
-that can be safely kept in RAM.  Zcache essentially trades off CPU
-cycles used in compression/decompression for better memory utilization.
-Benchmarks have shown little or no impact when memory pressure is
-low while providing a significant performance improvement (25%+)
-on some workloads under high memory pressure.
-
-"RAMster" builds on zcache by adding "peer-to-peer" transcendent memory
-support for clustered systems.  Frontswap pages are locally compressed
-as in zcache, but then "remotified" to another system's RAM.  This
-allows RAM to be dynamically load-balanced back-and-forth as needed,
-i.e. when system A is overcommitted, it can swap to system B, and
-vice versa.  RAMster can also be configured as a memory server so
-many servers in a cluster can swap, dynamically as needed, to a single
-server configured with a large amount of RAM... without pre-configuring
-how much of the RAM is available for each of the clients!
-
-In the virtual case, the whole point of virtualization is to statistically
-multiplex physical resources across the varying demands of multiple
-virtual machines.  This is really hard to do with RAM and efforts to do
-it well with no kernel changes have essentially failed (except in some
-well-publicized special-case workloads).
-Specifically, the Xen Transcendent Memory backend allows otherwise
-"fallow" hypervisor-owned RAM to not only be "time-shared" between multiple
-virtual machines, but the pages can be compressed and deduplicated to
-optimize RAM utilization.  And when guest OS's are induced to surrender
-underutilized RAM (e.g. with "selfballooning"), sudden unexpected
-memory pressure may result in swapping; frontswap allows those pages
-to be swapped to and from hypervisor RAM (if overall host system memory
-conditions allow), thus mitigating the potentially awful performance impact
-of unplanned swapping.
-
-A KVM implementation is underway and has been RFC'ed to lkml.  And,
-using frontswap, investigation is also underway on the use of NVM as
-a memory extension technology.
-
-2) Sure there may be performance advantages in some situations, but
-   what's the space/time overhead of frontswap?
-
-If CONFIG_FRONTSWAP is disabled, every frontswap hook compiles into
-nothingness and the only overhead is a few extra bytes per swapon'ed
-swap device.  If CONFIG_FRONTSWAP is enabled but no frontswap "backend"
-registers, there is one extra global variable compared to zero for
-every swap page read or written.  If CONFIG_FRONTSWAP is enabled
-AND a frontswap backend registers AND the backend fails every "store"
-request (i.e. provides no memory despite claiming it might),
-CPU overhead is still negligible -- and since every frontswap fail
-precedes a swap page write-to-disk, the system is highly likely
-to be I/O bound and using a small fraction of a percent of a CPU
-will be irrelevant anyway.
-
-As for space, if CONFIG_FRONTSWAP is enabled AND a frontswap backend
-registers, one bit is allocated for every swap page for every swap
-device that is swapon'd.  This is added to the EIGHT bits (which
-was sixteen until about 2.6.34) that the kernel already allocates
-for every swap page for every swap device that is swapon'd.  (Hugh
-Dickins has observed that frontswap could probably steal one of
-the existing eight bits, but let's worry about that minor optimization
-later.)  For very large swap disks (which are rare) on a standard
-4K pagesize, this is 1MB per 32GB swap.
-
-When swap pages are stored in transcendent memory instead of written
-out to disk, there is a side effect that this may create more memory
-pressure that can potentially outweigh the other advantages.  A
-backend, such as zcache, must implement policies to carefully (but
-dynamically) manage memory limits to ensure this doesn't happen.
-
-3) OK, how about a quick overview of what this frontswap patch does
-   in terms that a kernel hacker can grok?
-
-Let's assume that a frontswap "backend" has registered during
-kernel initialization; this registration indicates that this
-frontswap backend has access to some "memory" that is not directly
-accessible by the kernel.  Exactly how much memory it provides is
-entirely dynamic and random.
-
-Whenever a swap-device is swapon'd frontswap_init() is called,
-passing the swap device number (aka "type") as a parameter.
-This notifies frontswap to expect attempts to "store" swap pages
-associated with that number.
-
-Whenever the swap subsystem is readying a page to write to a swap
-device (c.f swap_writepage()), frontswap_store is called.  Frontswap
-consults with the frontswap backend and if the backend says it does NOT
-have room, frontswap_store returns -1 and the kernel swaps the page
-to the swap device as normal.  Note that the response from the frontswap
-backend is unpredictable to the kernel; it may choose to never accept a
-page, it could accept every ninth page, or it might accept every
-page.  But if the backend does accept a page, the data from the page
-has already been copied and associated with the type and offset,
-and the backend guarantees the persistence of the data.  In this case,
-frontswap sets a bit in the "frontswap_map" for the swap device
-corresponding to the page offset on the swap device to which it would
-otherwise have written the data.
-
-When the swap subsystem needs to swap-in a page (swap_readpage()),
-it first calls frontswap_load() which checks the frontswap_map to
-see if the page was earlier accepted by the frontswap backend.  If
-it was, the page of data is filled from the frontswap backend and
-the swap-in is complete.  If not, the normal swap-in code is
-executed to obtain the page of data from the real swap device.
-
-So every time the frontswap backend accepts a page, a swap device read
-and (potentially) a swap device write are replaced by a "frontswap backend
-store" and (possibly) a "frontswap backend loads", which are presumably much
-faster.
-
-4) Can't frontswap be configured as a "special" swap device that is
-   just higher priority than any real swap device (e.g. like zswap,
-   or maybe swap-over-nbd/NFS)?
-
-No.  First, the existing swap subsystem doesn't allow for any kind of
-swap hierarchy.  Perhaps it could be rewritten to accommodate a hierarchy,
-but this would require fairly drastic changes.  Even if it were
-rewritten, the existing swap subsystem uses the block I/O layer which
-assumes a swap device is fixed size and any page in it is linearly
-addressable.  Frontswap barely touches the existing swap subsystem,
-and works around the constraints of the block I/O subsystem to provide
-a great deal of flexibility and dynamicity.
-
-For example, the acceptance of any swap page by the frontswap backend is
-entirely unpredictable. This is critical to the definition of frontswap
-backends because it grants completely dynamic discretion to the
-backend.  In zcache, one cannot know a priori how compressible a page is.
-"Poorly" compressible pages can be rejected, and "poorly" can itself be
-defined dynamically depending on current memory constraints.
-
-Further, frontswap is entirely synchronous whereas a real swap
-device is, by definition, asynchronous and uses block I/O.  The
-block I/O layer is not only unnecessary, but may perform "optimizations"
-that are inappropriate for a RAM-oriented device including delaying
-the write of some pages for a significant amount of time.  Synchrony is
-required to ensure the dynamicity of the backend and to avoid thorny race
-conditions that would unnecessarily and greatly complicate frontswap
-and/or the block I/O subsystem.  That said, only the initial "store"
-and "load" operations need be synchronous.  A separate asynchronous thread
-is free to manipulate the pages stored by frontswap.  For example,
-the "remotification" thread in RAMster uses standard asynchronous
-kernel sockets to move compressed frontswap pages to a remote machine.
-Similarly, a KVM guest-side implementation could do in-guest compression
-and use "batched" hypercalls.
-
-In a virtualized environment, the dynamicity allows the hypervisor
-(or host OS) to do "intelligent overcommit".  For example, it can
-choose to accept pages only until host-swapping might be imminent,
-then force guests to do their own swapping.
-
-There is a downside to the transcendent memory specifications for
-frontswap:  Since any "store" might fail, there must always be a real
-slot on a real swap device to swap the page.  Thus frontswap must be
-implemented as a "shadow" to every swapon'd device with the potential
-capability of holding every page that the swap device might have held
-and the possibility that it might hold no pages at all.  This means
-that frontswap cannot contain more pages than the total of swapon'd
-swap devices.  For example, if NO swap device is configured on some
-installation, frontswap is useless.  Swapless portable devices
-can still use frontswap but a backend for such devices must configure
-some kind of "ghost" swap device and ensure that it is never used.
-
-5) Why this weird definition about "duplicate stores"?  If a page
-   has been previously successfully stored, can't it always be
-   successfully overwritten?
-
-Nearly always it can, but no, sometimes it cannot.  Consider an example
-where data is compressed and the original 4K page has been compressed
-to 1K.  Now an attempt is made to overwrite the page with data that
-is non-compressible and so would take the entire 4K.  But the backend
-has no more space.  In this case, the store must be rejected.  Whenever
-frontswap rejects a store that would overwrite, it also must invalidate
-the old data and ensure that it is no longer accessible.  Since the
-swap subsystem then writes the new data to the read swap device,
-this is the correct course of action to ensure coherency.
-
-6) What is frontswap_shrink for?
-
-When the (non-frontswap) swap subsystem swaps out a page to a real
-swap device, that page is only taking up low-value pre-allocated disk
-space.  But if frontswap has placed a page in transcendent memory, that
-page may be taking up valuable real estate.  The frontswap_shrink
-routine allows code outside of the swap subsystem to force pages out
-of the memory managed by frontswap and back into kernel-addressable memory.
-For example, in RAMster, a "suction driver" thread will attempt
-to "repatriate" pages sent to a remote machine back to the local machine;
-this is driven using the frontswap_shrink mechanism when memory pressure
-subsides.
-
-7) Why does the frontswap patch create the new include file swapfile.h?
-
-The frontswap code depends on some swap-subsystem-internal data
-structures that have, over the years, moved back and forth between
-static and global.  This seemed a reasonable compromise:  Define
-them as global but declare them in a new include file that isn't
-included by the large number of source files that include swap.h.
-
-Dan Magenheimer, last updated April 9, 2012
diff --git a/Documentation/vm/highmem.rst b/Documentation/vm/highmem.rst
new file mode 100644
index 000000000000..0f69a9fec34d
--- /dev/null
+++ b/Documentation/vm/highmem.rst
@@ -0,0 +1,147 @@
+.. _highmem:
+
+====================
+High Memory Handling
+====================
+
+By: Peter Zijlstra <a.p.zijlstra@chello.nl>
+
+.. contents:: :local:
+
+What Is High Memory?
+====================
+
+High memory (highmem) is used when the size of physical memory approaches or
+exceeds the maximum size of virtual memory.  At that point it becomes
+impossible for the kernel to keep all of the available physical memory mapped
+at all times.  This means the kernel needs to start using temporary mappings of
+the pieces of physical memory that it wants to access.
+
+The part of (physical) memory not covered by a permanent mapping is what we
+refer to as 'highmem'.  There are various architecture dependent constraints on
+where exactly that border lies.
+
+In the i386 arch, for example, we choose to map the kernel into every process's
+VM space so that we don't have to pay the full TLB invalidation costs for
+kernel entry/exit.  This means the available virtual memory space (4GiB on
+i386) has to be divided between user and kernel space.
+
+The traditional split for architectures using this approach is 3:1, 3GiB for
+userspace and the top 1GiB for kernel space::
+
+		+--------+ 0xffffffff
+		| Kernel |
+		+--------+ 0xc0000000
+		|        |
+		| User   |
+		|        |
+		+--------+ 0x00000000
+
+This means that the kernel can at most map 1GiB of physical memory at any one
+time, but because we need virtual address space for other things - including
+temporary maps to access the rest of the physical memory - the actual direct
+map will typically be less (usually around ~896MiB).
+
+Other architectures that have mm context tagged TLBs can have separate kernel
+and user maps.  Some hardware (like some ARMs), however, have limited virtual
+space when they use mm context tags.
+
+
+Temporary Virtual Mappings
+==========================
+
+The kernel contains several ways of creating temporary mappings:
+
+* vmap().  This can be used to make a long duration mapping of multiple
+  physical pages into a contiguous virtual space.  It needs global
+  synchronization to unmap.
+
+* kmap().  This permits a short duration mapping of a single page.  It needs
+  global synchronization, but is amortized somewhat.  It is also prone to
+  deadlocks when using in a nested fashion, and so it is not recommended for
+  new code.
+
+* kmap_atomic().  This permits a very short duration mapping of a single
+  page.  Since the mapping is restricted to the CPU that issued it, it
+  performs well, but the issuing task is therefore required to stay on that
+  CPU until it has finished, lest some other task displace its mappings.
+
+  kmap_atomic() may also be used by interrupt contexts, since it is does not
+  sleep and the caller may not sleep until after kunmap_atomic() is called.
+
+  It may be assumed that k[un]map_atomic() won't fail.
+
+
+Using kmap_atomic
+=================
+
+When and where to use kmap_atomic() is straightforward.  It is used when code
+wants to access the contents of a page that might be allocated from high memory
+(see __GFP_HIGHMEM), for example a page in the pagecache.  The API has two
+functions, and they can be used in a manner similar to the following::
+
+	/* Find the page of interest. */
+	struct page *page = find_get_page(mapping, offset);
+
+	/* Gain access to the contents of that page. */
+	void *vaddr = kmap_atomic(page);
+
+	/* Do something to the contents of that page. */
+	memset(vaddr, 0, PAGE_SIZE);
+
+	/* Unmap that page. */
+	kunmap_atomic(vaddr);
+
+Note that the kunmap_atomic() call takes the result of the kmap_atomic() call
+not the argument.
+
+If you need to map two pages because you want to copy from one page to
+another you need to keep the kmap_atomic calls strictly nested, like::
+
+	vaddr1 = kmap_atomic(page1);
+	vaddr2 = kmap_atomic(page2);
+
+	memcpy(vaddr1, vaddr2, PAGE_SIZE);
+
+	kunmap_atomic(vaddr2);
+	kunmap_atomic(vaddr1);
+
+
+Cost of Temporary Mappings
+==========================
+
+The cost of creating temporary mappings can be quite high.  The arch has to
+manipulate the kernel's page tables, the data TLB and/or the MMU's registers.
+
+If CONFIG_HIGHMEM is not set, then the kernel will try and create a mapping
+simply with a bit of arithmetic that will convert the page struct address into
+a pointer to the page contents rather than juggling mappings about.  In such a
+case, the unmap operation may be a null operation.
+
+If CONFIG_MMU is not set, then there can be no temporary mappings and no
+highmem.  In such a case, the arithmetic approach will also be used.
+
+
+i386 PAE
+========
+
+The i386 arch, under some circumstances, will permit you to stick up to 64GiB
+of RAM into your 32-bit machine.  This has a number of consequences:
+
+* Linux needs a page-frame structure for each page in the system and the
+  pageframes need to live in the permanent mapping, which means:
+
+* you can have 896M/sizeof(struct page) page-frames at most; with struct
+  page being 32-bytes that would end up being something in the order of 112G
+  worth of pages; the kernel, however, needs to store more than just
+  page-frames in that memory...
+
+* PAE makes your page tables larger - which slows the system down as more
+  data has to be accessed to traverse in TLB fills and the like.  One
+  advantage is that PAE has more PTE bits and can provide advanced features
+  like NX and PAT.
+
+The general recommendation is that you don't use more than 8GiB on a 32-bit
+machine - although more might work for you and your workload, you're pretty
+much on your own - don't expect kernel developers to really care much if things
+come apart.
diff --git a/Documentation/vm/highmem.txt b/Documentation/vm/highmem.txt
deleted file mode 100644
index 4324d24ffacd..000000000000
--- a/Documentation/vm/highmem.txt
+++ /dev/null
@@ -1,162 +0,0 @@
-
-			     ====================
-			     HIGH MEMORY HANDLING
-			     ====================
-
-By: Peter Zijlstra <a.p.zijlstra@chello.nl>
-
-Contents:
-
- (*) What is high memory?
-
- (*) Temporary virtual mappings.
-
- (*) Using kmap_atomic.
-
- (*) Cost of temporary mappings.
-
- (*) i386 PAE.
-
-
-====================
-WHAT IS HIGH MEMORY?
-====================
-
-High memory (highmem) is used when the size of physical memory approaches or
-exceeds the maximum size of virtual memory.  At that point it becomes
-impossible for the kernel to keep all of the available physical memory mapped
-at all times.  This means the kernel needs to start using temporary mappings of
-the pieces of physical memory that it wants to access.
-
-The part of (physical) memory not covered by a permanent mapping is what we
-refer to as 'highmem'.  There are various architecture dependent constraints on
-where exactly that border lies.
-
-In the i386 arch, for example, we choose to map the kernel into every process's
-VM space so that we don't have to pay the full TLB invalidation costs for
-kernel entry/exit.  This means the available virtual memory space (4GiB on
-i386) has to be divided between user and kernel space.
-
-The traditional split for architectures using this approach is 3:1, 3GiB for
-userspace and the top 1GiB for kernel space:
-
-		+--------+ 0xffffffff
-		| Kernel |
-		+--------+ 0xc0000000
-		|        |
-		| User   |
-		|        |
-		+--------+ 0x00000000
-
-This means that the kernel can at most map 1GiB of physical memory at any one
-time, but because we need virtual address space for other things - including
-temporary maps to access the rest of the physical memory - the actual direct
-map will typically be less (usually around ~896MiB).
-
-Other architectures that have mm context tagged TLBs can have separate kernel
-and user maps.  Some hardware (like some ARMs), however, have limited virtual
-space when they use mm context tags.
-
-
-==========================
-TEMPORARY VIRTUAL MAPPINGS
-==========================
-
-The kernel contains several ways of creating temporary mappings:
-
- (*) vmap().  This can be used to make a long duration mapping of multiple
-     physical pages into a contiguous virtual space.  It needs global
-     synchronization to unmap.
-
- (*) kmap().  This permits a short duration mapping of a single page.  It needs
-     global synchronization, but is amortized somewhat.  It is also prone to
-     deadlocks when using in a nested fashion, and so it is not recommended for
-     new code.
-
- (*) kmap_atomic().  This permits a very short duration mapping of a single
-     page.  Since the mapping is restricted to the CPU that issued it, it
-     performs well, but the issuing task is therefore required to stay on that
-     CPU until it has finished, lest some other task displace its mappings.
-
-     kmap_atomic() may also be used by interrupt contexts, since it is does not
-     sleep and the caller may not sleep until after kunmap_atomic() is called.
-
-     It may be assumed that k[un]map_atomic() won't fail.
-
-
-=================
-USING KMAP_ATOMIC
-=================
-
-When and where to use kmap_atomic() is straightforward.  It is used when code
-wants to access the contents of a page that might be allocated from high memory
-(see __GFP_HIGHMEM), for example a page in the pagecache.  The API has two
-functions, and they can be used in a manner similar to the following:
-
-	/* Find the page of interest. */
-	struct page *page = find_get_page(mapping, offset);
-
-	/* Gain access to the contents of that page. */
-	void *vaddr = kmap_atomic(page);
-
-	/* Do something to the contents of that page. */
-	memset(vaddr, 0, PAGE_SIZE);
-
-	/* Unmap that page. */
-	kunmap_atomic(vaddr);
-
-Note that the kunmap_atomic() call takes the result of the kmap_atomic() call
-not the argument.
-
-If you need to map two pages because you want to copy from one page to
-another you need to keep the kmap_atomic calls strictly nested, like:
-
-	vaddr1 = kmap_atomic(page1);
-	vaddr2 = kmap_atomic(page2);
-
-	memcpy(vaddr1, vaddr2, PAGE_SIZE);
-
-	kunmap_atomic(vaddr2);
-	kunmap_atomic(vaddr1);
-
-
-==========================
-COST OF TEMPORARY MAPPINGS
-==========================
-
-The cost of creating temporary mappings can be quite high.  The arch has to
-manipulate the kernel's page tables, the data TLB and/or the MMU's registers.
-
-If CONFIG_HIGHMEM is not set, then the kernel will try and create a mapping
-simply with a bit of arithmetic that will convert the page struct address into
-a pointer to the page contents rather than juggling mappings about.  In such a
-case, the unmap operation may be a null operation.
-
-If CONFIG_MMU is not set, then there can be no temporary mappings and no
-highmem.  In such a case, the arithmetic approach will also be used.
-
-
-========
-i386 PAE
-========
-
-The i386 arch, under some circumstances, will permit you to stick up to 64GiB
-of RAM into your 32-bit machine.  This has a number of consequences:
-
- (*) Linux needs a page-frame structure for each page in the system and the
-     pageframes need to live in the permanent mapping, which means:
-
- (*) you can have 896M/sizeof(struct page) page-frames at most; with struct
-     page being 32-bytes that would end up being something in the order of 112G
-     worth of pages; the kernel, however, needs to store more than just
-     page-frames in that memory...
-
- (*) PAE makes your page tables larger - which slows the system down as more
-     data has to be accessed to traverse in TLB fills and the like.  One
-     advantage is that PAE has more PTE bits and can provide advanced features
-     like NX and PAT.
-
-The general recommendation is that you don't use more than 8GiB on a 32-bit
-machine - although more might work for you and your workload, you're pretty
-much on your own - don't expect kernel developers to really care much if things
-come apart.
diff --git a/Documentation/vm/hmm.rst b/Documentation/vm/hmm.rst
new file mode 100644
index 000000000000..b6e0ae47709b
--- /dev/null
+++ b/Documentation/vm/hmm.rst
@@ -0,0 +1,385 @@
+.. hmm:
+
+=====================================
+Heterogeneous Memory Management (HMM)
+=====================================
+
+Provide infrastructure and helpers to integrate non-conventional memory (device
+memory like GPU on board memory) into regular kernel path, with the cornerstone
+of this being specialized struct page for such memory (see sections 5 to 7 of
+this document).
+
+HMM also provides optional helpers for SVM (Share Virtual Memory), i.e.,
+allowing a device to transparently access program address coherently with
+the CPU meaning that any valid pointer on the CPU is also a valid pointer
+for the device. This is becoming mandatory to simplify the use of advanced
+heterogeneous computing where GPU, DSP, or FPGA are used to perform various
+computations on behalf of a process.
+
+This document is divided as follows: in the first section I expose the problems
+related to using device specific memory allocators. In the second section, I
+expose the hardware limitations that are inherent to many platforms. The third
+section gives an overview of the HMM design. The fourth section explains how
+CPU page-table mirroring works and the purpose of HMM in this context. The
+fifth section deals with how device memory is represented inside the kernel.
+Finally, the last section presents a new migration helper that allows lever-
+aging the device DMA engine.
+
+.. contents:: :local:
+
+Problems of using a device specific memory allocator
+====================================================
+
+Devices with a large amount of on board memory (several gigabytes) like GPUs
+have historically managed their memory through dedicated driver specific APIs.
+This creates a disconnect between memory allocated and managed by a device
+driver and regular application memory (private anonymous, shared memory, or
+regular file backed memory). From here on I will refer to this aspect as split
+address space. I use shared address space to refer to the opposite situation:
+i.e., one in which any application memory region can be used by a device
+transparently.
+
+Split address space happens because device can only access memory allocated
+through device specific API. This implies that all memory objects in a program
+are not equal from the device point of view which complicates large programs
+that rely on a wide set of libraries.
+
+Concretely this means that code that wants to leverage devices like GPUs needs
+to copy object between generically allocated memory (malloc, mmap private, mmap
+share) and memory allocated through the device driver API (this still ends up
+with an mmap but of the device file).
+
+For flat data sets (array, grid, image, ...) this isn't too hard to achieve but
+complex data sets (list, tree, ...) are hard to get right. Duplicating a
+complex data set needs to re-map all the pointer relations between each of its
+elements. This is error prone and program gets harder to debug because of the
+duplicate data set and addresses.
+
+Split address space also means that libraries cannot transparently use data
+they are getting from the core program or another library and thus each library
+might have to duplicate its input data set using the device specific memory
+allocator. Large projects suffer from this and waste resources because of the
+various memory copies.
+
+Duplicating each library API to accept as input or output memory allocated by
+each device specific allocator is not a viable option. It would lead to a
+combinatorial explosion in the library entry points.
+
+Finally, with the advance of high level language constructs (in C++ but in
+other languages too) it is now possible for the compiler to leverage GPUs and
+other devices without programmer knowledge. Some compiler identified patterns
+are only do-able with a shared address space. It is also more reasonable to use
+a shared address space for all other patterns.
+
+
+I/O bus, device memory characteristics
+======================================
+
+I/O buses cripple shared address spaces due to a few limitations. Most I/O
+buses only allow basic memory access from device to main memory; even cache
+coherency is often optional. Access to device memory from CPU is even more
+limited. More often than not, it is not cache coherent.
+
+If we only consider the PCIE bus, then a device can access main memory (often
+through an IOMMU) and be cache coherent with the CPUs. However, it only allows
+a limited set of atomic operations from device on main memory. This is worse
+in the other direction: the CPU can only access a limited range of the device
+memory and cannot perform atomic operations on it. Thus device memory cannot
+be considered the same as regular memory from the kernel point of view.
+
+Another crippling factor is the limited bandwidth (~32GBytes/s with PCIE 4.0
+and 16 lanes). This is 33 times less than the fastest GPU memory (1 TBytes/s).
+The final limitation is latency. Access to main memory from the device has an
+order of magnitude higher latency than when the device accesses its own memory.
+
+Some platforms are developing new I/O buses or additions/modifications to PCIE
+to address some of these limitations (OpenCAPI, CCIX). They mainly allow two-
+way cache coherency between CPU and device and allow all atomic operations the
+architecture supports. Sadly, not all platforms are following this trend and
+some major architectures are left without hardware solutions to these problems.
+
+So for shared address space to make sense, not only must we allow devices to
+access any memory but we must also permit any memory to be migrated to device
+memory while device is using it (blocking CPU access while it happens).
+
+
+Shared address space and migration
+==================================
+
+HMM intends to provide two main features. First one is to share the address
+space by duplicating the CPU page table in the device page table so the same
+address points to the same physical memory for any valid main memory address in
+the process address space.
+
+To achieve this, HMM offers a set of helpers to populate the device page table
+while keeping track of CPU page table updates. Device page table updates are
+not as easy as CPU page table updates. To update the device page table, you must
+allocate a buffer (or use a pool of pre-allocated buffers) and write GPU
+specific commands in it to perform the update (unmap, cache invalidations, and
+flush, ...). This cannot be done through common code for all devices. Hence
+why HMM provides helpers to factor out everything that can be while leaving the
+hardware specific details to the device driver.
+
+The second mechanism HMM provides is a new kind of ZONE_DEVICE memory that
+allows allocating a struct page for each page of the device memory. Those pages
+are special because the CPU cannot map them. However, they allow migrating
+main memory to device memory using existing migration mechanisms and everything
+looks like a page is swapped out to disk from the CPU point of view. Using a
+struct page gives the easiest and cleanest integration with existing mm mech-
+anisms. Here again, HMM only provides helpers, first to hotplug new ZONE_DEVICE
+memory for the device memory and second to perform migration. Policy decisions
+of what and when to migrate things is left to the device driver.
+
+Note that any CPU access to a device page triggers a page fault and a migration
+back to main memory. For example, when a page backing a given CPU address A is
+migrated from a main memory page to a device page, then any CPU access to
+address A triggers a page fault and initiates a migration back to main memory.
+
+With these two features, HMM not only allows a device to mirror process address
+space and keeping both CPU and device page table synchronized, but also lever-
+ages device memory by migrating the part of the data set that is actively being
+used by the device.
+
+
+Address space mirroring implementation and API
+==============================================
+
+Address space mirroring's main objective is to allow duplication of a range of
+CPU page table into a device page table; HMM helps keep both synchronized. A
+device driver that wants to mirror a process address space must start with the
+registration of an hmm_mirror struct::
+
+ int hmm_mirror_register(struct hmm_mirror *mirror,
+                         struct mm_struct *mm);
+ int hmm_mirror_register_locked(struct hmm_mirror *mirror,
+                                struct mm_struct *mm);
+
+
+The locked variant is to be used when the driver is already holding mmap_sem
+of the mm in write mode. The mirror struct has a set of callbacks that are used
+to propagate CPU page tables::
+
+ struct hmm_mirror_ops {
+     /* sync_cpu_device_pagetables() - synchronize page tables
+      *
+      * @mirror: pointer to struct hmm_mirror
+      * @update_type: type of update that occurred to the CPU page table
+      * @start: virtual start address of the range to update
+      * @end: virtual end address of the range to update
+      *
+      * This callback ultimately originates from mmu_notifiers when the CPU
+      * page table is updated. The device driver must update its page table
+      * in response to this callback. The update argument tells what action
+      * to perform.
+      *
+      * The device driver must not return from this callback until the device
+      * page tables are completely updated (TLBs flushed, etc); this is a
+      * synchronous call.
+      */
+      void (*update)(struct hmm_mirror *mirror,
+                     enum hmm_update action,
+                     unsigned long start,
+                     unsigned long end);
+ };
+
+The device driver must perform the update action to the range (mark range
+read only, or fully unmap, ...). The device must be done with the update before
+the driver callback returns.
+
+When the device driver wants to populate a range of virtual addresses, it can
+use either::
+
+  int hmm_vma_get_pfns(struct vm_area_struct *vma,
+                      struct hmm_range *range,
+                      unsigned long start,
+                      unsigned long end,
+                      hmm_pfn_t *pfns);
+ int hmm_vma_fault(struct vm_area_struct *vma,
+                   struct hmm_range *range,
+                   unsigned long start,
+                   unsigned long end,
+                   hmm_pfn_t *pfns,
+                   bool write,
+                   bool block);
+
+The first one (hmm_vma_get_pfns()) will only fetch present CPU page table
+entries and will not trigger a page fault on missing or non-present entries.
+The second one does trigger a page fault on missing or read-only entry if the
+write parameter is true. Page faults use the generic mm page fault code path
+just like a CPU page fault.
+
+Both functions copy CPU page table entries into their pfns array argument. Each
+entry in that array corresponds to an address in the virtual range. HMM
+provides a set of flags to help the driver identify special CPU page table
+entries.
+
+Locking with the update() callback is the most important aspect the driver must
+respect in order to keep things properly synchronized. The usage pattern is::
+
+ int driver_populate_range(...)
+ {
+      struct hmm_range range;
+      ...
+ again:
+      ret = hmm_vma_get_pfns(vma, &range, start, end, pfns);
+      if (ret)
+          return ret;
+      take_lock(driver->update);
+      if (!hmm_vma_range_done(vma, &range)) {
+          release_lock(driver->update);
+          goto again;
+      }
+
+      // Use pfns array content to update device page table
+
+      release_lock(driver->update);
+      return 0;
+ }
+
+The driver->update lock is the same lock that the driver takes inside its
+update() callback. That lock must be held before hmm_vma_range_done() to avoid
+any race with a concurrent CPU page table update.
+
+HMM implements all this on top of the mmu_notifier API because we wanted a
+simpler API and also to be able to perform optimizations latter on like doing
+concurrent device updates in multi-devices scenario.
+
+HMM also serves as an impedance mismatch between how CPU page table updates
+are done (by CPU write to the page table and TLB flushes) and how devices
+update their own page table. Device updates are a multi-step process. First,
+appropriate commands are written to a buffer, then this buffer is scheduled for
+execution on the device. It is only once the device has executed commands in
+the buffer that the update is done. Creating and scheduling the update command
+buffer can happen concurrently for multiple devices. Waiting for each device to
+report commands as executed is serialized (there is no point in doing this
+concurrently).
+
+
+Represent and manage device memory from core kernel point of view
+=================================================================
+
+Several different designs were tried to support device memory. First one used
+a device specific data structure to keep information about migrated memory and
+HMM hooked itself in various places of mm code to handle any access to
+addresses that were backed by device memory. It turns out that this ended up
+replicating most of the fields of struct page and also needed many kernel code
+paths to be updated to understand this new kind of memory.
+
+Most kernel code paths never try to access the memory behind a page
+but only care about struct page contents. Because of this, HMM switched to
+directly using struct page for device memory which left most kernel code paths
+unaware of the difference. We only need to make sure that no one ever tries to
+map those pages from the CPU side.
+
+HMM provides a set of helpers to register and hotplug device memory as a new
+region needing a struct page. This is offered through a very simple API::
+
+ struct hmm_devmem *hmm_devmem_add(const struct hmm_devmem_ops *ops,
+                                   struct device *device,
+                                   unsigned long size);
+
+The hmm_devmem_ops is where most of the important things are::
+
+ struct hmm_devmem_ops {
+     void (*free)(struct hmm_devmem *devmem, struct page *page);
+     int (*fault)(struct hmm_devmem *devmem,
+                  struct vm_area_struct *vma,
+                  unsigned long addr,
+                  struct page *page,
+                  unsigned flags,
+                  pmd_t *pmdp);
+ };
+
+The first callback (free()) happens when the last reference on a device page is
+dropped. This means the device page is now free and no longer used by anyone.
+The second callback happens whenever the CPU tries to access a device page
+which it cannot do. This second callback must trigger a migration back to
+system memory.
+
+
+Migration to and from device memory
+===================================
+
+Because the CPU cannot access device memory, migration must use the device DMA
+engine to perform copy from and to device memory. For this we need a new
+migration helper::
+
+ int migrate_vma(const struct migrate_vma_ops *ops,
+                 struct vm_area_struct *vma,
+                 unsigned long mentries,
+                 unsigned long start,
+                 unsigned long end,
+                 unsigned long *src,
+                 unsigned long *dst,
+                 void *private);
+
+Unlike other migration functions it works on a range of virtual address, there
+are two reasons for that. First, device DMA copy has a high setup overhead cost
+and thus batching multiple pages is needed as otherwise the migration overhead
+makes the whole exercise pointless. The second reason is because the
+migration might be for a range of addresses the device is actively accessing.
+
+The migrate_vma_ops struct defines two callbacks. First one (alloc_and_copy())
+controls destination memory allocation and copy operation. Second one is there
+to allow the device driver to perform cleanup operations after migration::
+
+ struct migrate_vma_ops {
+     void (*alloc_and_copy)(struct vm_area_struct *vma,
+                            const unsigned long *src,
+                            unsigned long *dst,
+                            unsigned long start,
+                            unsigned long end,
+                            void *private);
+     void (*finalize_and_map)(struct vm_area_struct *vma,
+                              const unsigned long *src,
+                              const unsigned long *dst,
+                              unsigned long start,
+                              unsigned long end,
+                              void *private);
+ };
+
+It is important to stress that these migration helpers allow for holes in the
+virtual address range. Some pages in the range might not be migrated for all
+the usual reasons (page is pinned, page is locked, ...). This helper does not
+fail but just skips over those pages.
+
+The alloc_and_copy() might decide to not migrate all pages in the
+range (for reasons under the callback control). For those, the callback just
+has to leave the corresponding dst entry empty.
+
+Finally, the migration of the struct page might fail (for file backed page) for
+various reasons (failure to freeze reference, or update page cache, ...). If
+that happens, then the finalize_and_map() can catch any pages that were not
+migrated. Note those pages were still copied to a new page and thus we wasted
+bandwidth but this is considered as a rare event and a price that we are
+willing to pay to keep all the code simpler.
+
+
+Memory cgroup (memcg) and rss accounting
+========================================
+
+For now device memory is accounted as any regular page in rss counters (either
+anonymous if device page is used for anonymous, file if device page is used for
+file backed page or shmem if device page is used for shared memory). This is a
+deliberate choice to keep existing applications, that might start using device
+memory without knowing about it, running unimpacted.
+
+A drawback is that the OOM killer might kill an application using a lot of
+device memory and not a lot of regular system memory and thus not freeing much
+system memory. We want to gather more real world experience on how applications
+and system react under memory pressure in the presence of device memory before
+deciding to account device memory differently.
+
+
+Same decision was made for memory cgroup. Device memory pages are accounted
+against same memory cgroup a regular page would be accounted to. This does
+simplify migration to and from device memory. This also means that migration
+back from device memory to regular memory cannot fail because it would
+go above memory cgroup limit. We might revisit this choice latter on once we
+get more experience in how device memory is used and its impact on memory
+resource control.
+
+
+Note that device memory can never be pinned by device driver nor through GUP
+and thus such memory is always free upon process exit. Or when last reference
+is dropped in case of shared memory or file backed memory.
diff --git a/Documentation/vm/hmm.txt b/Documentation/vm/hmm.txt
deleted file mode 100644
index 2d1d6f69e91b..000000000000
--- a/Documentation/vm/hmm.txt
+++ /dev/null
@@ -1,396 +0,0 @@
-Heterogeneous Memory Management (HMM)
-
-Provide infrastructure and helpers to integrate non-conventional memory (device
-memory like GPU on board memory) into regular kernel path, with the cornerstone
-of this being specialized struct page for such memory (see sections 5 to 7 of
-this document).
-
-HMM also provides optional helpers for SVM (Share Virtual Memory), i.e.,
-allowing a device to transparently access program address coherently with the
-CPU meaning that any valid pointer on the CPU is also a valid pointer for the
-device. This is becoming mandatory to simplify the use of advanced hetero-
-geneous computing where GPU, DSP, or FPGA are used to perform various
-computations on behalf of a process.
-
-This document is divided as follows: in the first section I expose the problems
-related to using device specific memory allocators. In the second section, I
-expose the hardware limitations that are inherent to many platforms. The third
-section gives an overview of the HMM design. The fourth section explains how
-CPU page-table mirroring works and the purpose of HMM in this context. The
-fifth section deals with how device memory is represented inside the kernel.
-Finally, the last section presents a new migration helper that allows lever-
-aging the device DMA engine.
-
-
-1) Problems of using a device specific memory allocator:
-2) I/O bus, device memory characteristics
-3) Shared address space and migration
-4) Address space mirroring implementation and API
-5) Represent and manage device memory from core kernel point of view
-6) Migration to and from device memory
-7) Memory cgroup (memcg) and rss accounting
-
-
--------------------------------------------------------------------------------
-
-1) Problems of using a device specific memory allocator:
-
-Devices with a large amount of on board memory (several gigabytes) like GPUs
-have historically managed their memory through dedicated driver specific APIs.
-This creates a disconnect between memory allocated and managed by a device
-driver and regular application memory (private anonymous, shared memory, or
-regular file backed memory). From here on I will refer to this aspect as split
-address space. I use shared address space to refer to the opposite situation:
-i.e., one in which any application memory region can be used by a device
-transparently.
-
-Split address space happens because device can only access memory allocated
-through device specific API. This implies that all memory objects in a program
-are not equal from the device point of view which complicates large programs
-that rely on a wide set of libraries.
-
-Concretely this means that code that wants to leverage devices like GPUs needs
-to copy object between generically allocated memory (malloc, mmap private, mmap
-share) and memory allocated through the device driver API (this still ends up
-with an mmap but of the device file).
-
-For flat data sets (array, grid, image, ...) this isn't too hard to achieve but
-complex data sets (list, tree, ...) are hard to get right. Duplicating a
-complex data set needs to re-map all the pointer relations between each of its
-elements. This is error prone and program gets harder to debug because of the
-duplicate data set and addresses.
-
-Split address space also means that libraries cannot transparently use data
-they are getting from the core program or another library and thus each library
-might have to duplicate its input data set using the device specific memory
-allocator. Large projects suffer from this and waste resources because of the
-various memory copies.
-
-Duplicating each library API to accept as input or output memory allocated by
-each device specific allocator is not a viable option. It would lead to a
-combinatorial explosion in the library entry points.
-
-Finally, with the advance of high level language constructs (in C++ but in
-other languages too) it is now possible for the compiler to leverage GPUs and
-other devices without programmer knowledge. Some compiler identified patterns
-are only do-able with a shared address space. It is also more reasonable to use
-a shared address space for all other patterns.
-
-
--------------------------------------------------------------------------------
-
-2) I/O bus, device memory characteristics
-
-I/O buses cripple shared address spaces due to a few limitations. Most I/O
-buses only allow basic memory access from device to main memory; even cache
-coherency is often optional. Access to device memory from CPU is even more
-limited. More often than not, it is not cache coherent.
-
-If we only consider the PCIE bus, then a device can access main memory (often
-through an IOMMU) and be cache coherent with the CPUs. However, it only allows
-a limited set of atomic operations from device on main memory. This is worse
-in the other direction: the CPU can only access a limited range of the device
-memory and cannot perform atomic operations on it. Thus device memory cannot
-be considered the same as regular memory from the kernel point of view.
-
-Another crippling factor is the limited bandwidth (~32GBytes/s with PCIE 4.0
-and 16 lanes). This is 33 times less than the fastest GPU memory (1 TBytes/s).
-The final limitation is latency. Access to main memory from the device has an
-order of magnitude higher latency than when the device accesses its own memory.
-
-Some platforms are developing new I/O buses or additions/modifications to PCIE
-to address some of these limitations (OpenCAPI, CCIX). They mainly allow two-
-way cache coherency between CPU and device and allow all atomic operations the
-architecture supports. Sadly, not all platforms are following this trend and
-some major architectures are left without hardware solutions to these problems.
-
-So for shared address space to make sense, not only must we allow devices to
-access any memory but we must also permit any memory to be migrated to device
-memory while device is using it (blocking CPU access while it happens).
-
-
--------------------------------------------------------------------------------
-
-3) Shared address space and migration
-
-HMM intends to provide two main features. First one is to share the address
-space by duplicating the CPU page table in the device page table so the same
-address points to the same physical memory for any valid main memory address in
-the process address space.
-
-To achieve this, HMM offers a set of helpers to populate the device page table
-while keeping track of CPU page table updates. Device page table updates are
-not as easy as CPU page table updates. To update the device page table, you must
-allocate a buffer (or use a pool of pre-allocated buffers) and write GPU
-specific commands in it to perform the update (unmap, cache invalidations, and
-flush, ...). This cannot be done through common code for all devices. Hence
-why HMM provides helpers to factor out everything that can be while leaving the
-hardware specific details to the device driver.
-
-The second mechanism HMM provides is a new kind of ZONE_DEVICE memory that
-allows allocating a struct page for each page of the device memory. Those pages
-are special because the CPU cannot map them. However, they allow migrating
-main memory to device memory using existing migration mechanisms and everything
-looks like a page is swapped out to disk from the CPU point of view. Using a
-struct page gives the easiest and cleanest integration with existing mm mech-
-anisms. Here again, HMM only provides helpers, first to hotplug new ZONE_DEVICE
-memory for the device memory and second to perform migration. Policy decisions
-of what and when to migrate things is left to the device driver.
-
-Note that any CPU access to a device page triggers a page fault and a migration
-back to main memory. For example, when a page backing a given CPU address A is
-migrated from a main memory page to a device page, then any CPU access to
-address A triggers a page fault and initiates a migration back to main memory.
-
-With these two features, HMM not only allows a device to mirror process address
-space and keeping both CPU and device page table synchronized, but also lever-
-ages device memory by migrating the part of the data set that is actively being
-used by the device.
-
-
--------------------------------------------------------------------------------
-
-4) Address space mirroring implementation and API
-
-Address space mirroring's main objective is to allow duplication of a range of
-CPU page table into a device page table; HMM helps keep both synchronized. A
-device driver that wants to mirror a process address space must start with the
-registration of an hmm_mirror struct:
-
- int hmm_mirror_register(struct hmm_mirror *mirror,
-                         struct mm_struct *mm);
- int hmm_mirror_register_locked(struct hmm_mirror *mirror,
-                                struct mm_struct *mm);
-
-The locked variant is to be used when the driver is already holding mmap_sem
-of the mm in write mode. The mirror struct has a set of callbacks that are used
-to propagate CPU page tables:
-
- struct hmm_mirror_ops {
-     /* sync_cpu_device_pagetables() - synchronize page tables
-      *
-      * @mirror: pointer to struct hmm_mirror
-      * @update_type: type of update that occurred to the CPU page table
-      * @start: virtual start address of the range to update
-      * @end: virtual end address of the range to update
-      *
-      * This callback ultimately originates from mmu_notifiers when the CPU
-      * page table is updated. The device driver must update its page table
-      * in response to this callback. The update argument tells what action
-      * to perform.
-      *
-      * The device driver must not return from this callback until the device
-      * page tables are completely updated (TLBs flushed, etc); this is a
-      * synchronous call.
-      */
-      void (*update)(struct hmm_mirror *mirror,
-                     enum hmm_update action,
-                     unsigned long start,
-                     unsigned long end);
- };
-
-The device driver must perform the update action to the range (mark range
-read only, or fully unmap, ...). The device must be done with the update before
-the driver callback returns.
-
-
-When the device driver wants to populate a range of virtual addresses, it can
-use either:
- int hmm_vma_get_pfns(struct vm_area_struct *vma,
-                      struct hmm_range *range,
-                      unsigned long start,
-                      unsigned long end,
-                      hmm_pfn_t *pfns);
- int hmm_vma_fault(struct vm_area_struct *vma,
-                   struct hmm_range *range,
-                   unsigned long start,
-                   unsigned long end,
-                   hmm_pfn_t *pfns,
-                   bool write,
-                   bool block);
-
-The first one (hmm_vma_get_pfns()) will only fetch present CPU page table
-entries and will not trigger a page fault on missing or non-present entries.
-The second one does trigger a page fault on missing or read-only entry if the
-write parameter is true. Page faults use the generic mm page fault code path
-just like a CPU page fault.
-
-Both functions copy CPU page table entries into their pfns array argument. Each
-entry in that array corresponds to an address in the virtual range. HMM
-provides a set of flags to help the driver identify special CPU page table
-entries.
-
-Locking with the update() callback is the most important aspect the driver must
-respect in order to keep things properly synchronized. The usage pattern is:
-
- int driver_populate_range(...)
- {
-      struct hmm_range range;
-      ...
- again:
-      ret = hmm_vma_get_pfns(vma, &range, start, end, pfns);
-      if (ret)
-          return ret;
-      take_lock(driver->update);
-      if (!hmm_vma_range_done(vma, &range)) {
-          release_lock(driver->update);
-          goto again;
-      }
-
-      // Use pfns array content to update device page table
-
-      release_lock(driver->update);
-      return 0;
- }
-
-The driver->update lock is the same lock that the driver takes inside its
-update() callback. That lock must be held before hmm_vma_range_done() to avoid
-any race with a concurrent CPU page table update.
-
-HMM implements all this on top of the mmu_notifier API because we wanted a
-simpler API and also to be able to perform optimizations latter on like doing
-concurrent device updates in multi-devices scenario.
-
-HMM also serves as an impedance mismatch between how CPU page table updates
-are done (by CPU write to the page table and TLB flushes) and how devices
-update their own page table. Device updates are a multi-step process. First,
-appropriate commands are written to a buffer, then this buffer is scheduled for
-execution on the device. It is only once the device has executed commands in
-the buffer that the update is done. Creating and scheduling the update command
-buffer can happen concurrently for multiple devices. Waiting for each device to
-report commands as executed is serialized (there is no point in doing this
-concurrently).
-
-
--------------------------------------------------------------------------------
-
-5) Represent and manage device memory from core kernel point of view
-
-Several different designs were tried to support device memory. First one used
-a device specific data structure to keep information about migrated memory and
-HMM hooked itself in various places of mm code to handle any access to
-addresses that were backed by device memory. It turns out that this ended up
-replicating most of the fields of struct page and also needed many kernel code
-paths to be updated to understand this new kind of memory.
-
-Most kernel code paths never try to access the memory behind a page
-but only care about struct page contents. Because of this, HMM switched to
-directly using struct page for device memory which left most kernel code paths
-unaware of the difference. We only need to make sure that no one ever tries to
-map those pages from the CPU side.
-
-HMM provides a set of helpers to register and hotplug device memory as a new
-region needing a struct page. This is offered through a very simple API:
-
- struct hmm_devmem *hmm_devmem_add(const struct hmm_devmem_ops *ops,
-                                   struct device *device,
-                                   unsigned long size);
- void hmm_devmem_remove(struct hmm_devmem *devmem);
-
-The hmm_devmem_ops is where most of the important things are:
-
- struct hmm_devmem_ops {
-     void (*free)(struct hmm_devmem *devmem, struct page *page);
-     int (*fault)(struct hmm_devmem *devmem,
-                  struct vm_area_struct *vma,
-                  unsigned long addr,
-                  struct page *page,
-                  unsigned flags,
-                  pmd_t *pmdp);
- };
-
-The first callback (free()) happens when the last reference on a device page is
-dropped. This means the device page is now free and no longer used by anyone.
-The second callback happens whenever the CPU tries to access a device page
-which it cannot do. This second callback must trigger a migration back to
-system memory.
-
-
--------------------------------------------------------------------------------
-
-6) Migration to and from device memory
-
-Because the CPU cannot access device memory, migration must use the device DMA
-engine to perform copy from and to device memory. For this we need a new
-migration helper:
-
- int migrate_vma(const struct migrate_vma_ops *ops,
-                 struct vm_area_struct *vma,
-                 unsigned long mentries,
-                 unsigned long start,
-                 unsigned long end,
-                 unsigned long *src,
-                 unsigned long *dst,
-                 void *private);
-
-Unlike other migration functions it works on a range of virtual address, there
-are two reasons for that. First, device DMA copy has a high setup overhead cost
-and thus batching multiple pages is needed as otherwise the migration overhead
-makes the whole exercise pointless. The second reason is because the
-migration might be for a range of addresses the device is actively accessing.
-
-The migrate_vma_ops struct defines two callbacks. First one (alloc_and_copy())
-controls destination memory allocation and copy operation. Second one is there
-to allow the device driver to perform cleanup operations after migration.
-
- struct migrate_vma_ops {
-     void (*alloc_and_copy)(struct vm_area_struct *vma,
-                            const unsigned long *src,
-                            unsigned long *dst,
-                            unsigned long start,
-                            unsigned long end,
-                            void *private);
-     void (*finalize_and_map)(struct vm_area_struct *vma,
-                              const unsigned long *src,
-                              const unsigned long *dst,
-                              unsigned long start,
-                              unsigned long end,
-                              void *private);
- };
-
-It is important to stress that these migration helpers allow for holes in the
-virtual address range. Some pages in the range might not be migrated for all
-the usual reasons (page is pinned, page is locked, ...). This helper does not
-fail but just skips over those pages.
-
-The alloc_and_copy() might decide to not migrate all pages in the
-range (for reasons under the callback control). For those, the callback just
-has to leave the corresponding dst entry empty.
-
-Finally, the migration of the struct page might fail (for file backed page) for
-various reasons (failure to freeze reference, or update page cache, ...). If
-that happens, then the finalize_and_map() can catch any pages that were not
-migrated. Note those pages were still copied to a new page and thus we wasted
-bandwidth but this is considered as a rare event and a price that we are
-willing to pay to keep all the code simpler.
-
-
--------------------------------------------------------------------------------
-
-7) Memory cgroup (memcg) and rss accounting
-
-For now device memory is accounted as any regular page in rss counters (either
-anonymous if device page is used for anonymous, file if device page is used for
-file backed page or shmem if device page is used for shared memory). This is a
-deliberate choice to keep existing applications, that might start using device
-memory without knowing about it, running unimpacted.
-
-A drawback is that the OOM killer might kill an application using a lot of
-device memory and not a lot of regular system memory and thus not freeing much
-system memory. We want to gather more real world experience on how applications
-and system react under memory pressure in the presence of device memory before
-deciding to account device memory differently.
-
-
-Same decision was made for memory cgroup. Device memory pages are accounted
-against same memory cgroup a regular page would be accounted to. This does
-simplify migration to and from device memory. This also means that migration
-back from device memory to regular memory cannot fail because it would
-go above memory cgroup limit. We might revisit this choice latter on once we
-get more experience in how device memory is used and its impact on memory
-resource control.
-
-
-Note that device memory can never be pinned by device driver nor through GUP
-and thus such memory is always free upon process exit. Or when last reference
-is dropped in case of shared memory or file backed memory.
diff --git a/Documentation/vm/hugetlbfs_reserv.rst b/Documentation/vm/hugetlbfs_reserv.rst
new file mode 100644
index 000000000000..9d200762114f
--- /dev/null
+++ b/Documentation/vm/hugetlbfs_reserv.rst
@@ -0,0 +1,595 @@
+.. _hugetlbfs_reserve:
+
+=====================
+Hugetlbfs Reservation
+=====================
+
+Overview
+========
+
+Huge pages as described at :ref:`hugetlbpage` are typically
+preallocated for application use.  These huge pages are instantiated in a
+task's address space at page fault time if the VMA indicates huge pages are
+to be used.  If no huge page exists at page fault time, the task is sent
+a SIGBUS and often dies an unhappy death.  Shortly after huge page support
+was added, it was determined that it would be better to detect a shortage
+of huge pages at mmap() time.  The idea is that if there were not enough
+huge pages to cover the mapping, the mmap() would fail.  This was first
+done with a simple check in the code at mmap() time to determine if there
+were enough free huge pages to cover the mapping.  Like most things in the
+kernel, the code has evolved over time.  However, the basic idea was to
+'reserve' huge pages at mmap() time to ensure that huge pages would be
+available for page faults in that mapping.  The description below attempts to
+describe how huge page reserve processing is done in the v4.10 kernel.
+
+
+Audience
+========
+This description is primarily targeted at kernel developers who are modifying
+hugetlbfs code.
+
+
+The Data Structures
+===================
+
+resv_huge_pages
+	This is a global (per-hstate) count of reserved huge pages.  Reserved
+	huge pages are only available to the task which reserved them.
+	Therefore, the number of huge pages generally available is computed
+	as (``free_huge_pages - resv_huge_pages``).
+Reserve Map
+	A reserve map is described by the structure::
+
+		struct resv_map {
+			struct kref refs;
+			spinlock_t lock;
+			struct list_head regions;
+			long adds_in_progress;
+			struct list_head region_cache;
+			long region_cache_count;
+		};
+
+	There is one reserve map for each huge page mapping in the system.
+	The regions list within the resv_map describes the regions within
+	the mapping.  A region is described as::
+
+		struct file_region {
+			struct list_head link;
+			long from;
+			long to;
+		};
+
+	The 'from' and 'to' fields of the file region structure are huge page
+	indices into the mapping.  Depending on the type of mapping, a
+	region in the reserv_map may indicate reservations exist for the
+	range, or reservations do not exist.
+Flags for MAP_PRIVATE Reservations
+	These are stored in the bottom bits of the reservation map pointer.
+
+	``#define HPAGE_RESV_OWNER    (1UL << 0)``
+		Indicates this task is the owner of the reservations
+		associated with the mapping.
+	``#define HPAGE_RESV_UNMAPPED (1UL << 1)``
+		Indicates task originally mapping this range (and creating
+		reserves) has unmapped a page from this task (the child)
+		due to a failed COW.
+Page Flags
+	The PagePrivate page flag is used to indicate that a huge page
+	reservation must be restored when the huge page is freed.  More
+	details will be discussed in the "Freeing huge pages" section.
+
+
+Reservation Map Location (Private or Shared)
+============================================
+
+A huge page mapping or segment is either private or shared.  If private,
+it is typically only available to a single address space (task).  If shared,
+it can be mapped into multiple address spaces (tasks).  The location and
+semantics of the reservation map is significantly different for two types
+of mappings.  Location differences are:
+
+- For private mappings, the reservation map hangs off the the VMA structure.
+  Specifically, vma->vm_private_data.  This reserve map is created at the
+  time the mapping (mmap(MAP_PRIVATE)) is created.
+- For shared mappings, the reservation map hangs off the inode.  Specifically,
+  inode->i_mapping->private_data.  Since shared mappings are always backed
+  by files in the hugetlbfs filesystem, the hugetlbfs code ensures each inode
+  contains a reservation map.  As a result, the reservation map is allocated
+  when the inode is created.
+
+
+Creating Reservations
+=====================
+Reservations are created when a huge page backed shared memory segment is
+created (shmget(SHM_HUGETLB)) or a mapping is created via mmap(MAP_HUGETLB).
+These operations result in a call to the routine hugetlb_reserve_pages()::
+
+	int hugetlb_reserve_pages(struct inode *inode,
+				  long from, long to,
+				  struct vm_area_struct *vma,
+				  vm_flags_t vm_flags)
+
+The first thing hugetlb_reserve_pages() does is check for the NORESERVE
+flag was specified in either the shmget() or mmap() call.  If NORESERVE
+was specified, then this routine returns immediately as no reservation
+are desired.
+
+The arguments 'from' and 'to' are huge page indices into the mapping or
+underlying file.  For shmget(), 'from' is always 0 and 'to' corresponds to
+the length of the segment/mapping.  For mmap(), the offset argument could
+be used to specify the offset into the underlying file.  In such a case
+the 'from' and 'to' arguments have been adjusted by this offset.
+
+One of the big differences between PRIVATE and SHARED mappings is the way
+in which reservations are represented in the reservation map.
+
+- For shared mappings, an entry in the reservation map indicates a reservation
+  exists or did exist for the corresponding page.  As reservations are
+  consumed, the reservation map is not modified.
+- For private mappings, the lack of an entry in the reservation map indicates
+  a reservation exists for the corresponding page.  As reservations are
+  consumed, entries are added to the reservation map.  Therefore, the
+  reservation map can also be used to determine which reservations have
+  been consumed.
+
+For private mappings, hugetlb_reserve_pages() creates the reservation map and
+hangs it off the VMA structure.  In addition, the HPAGE_RESV_OWNER flag is set
+to indicate this VMA owns the reservations.
+
+The reservation map is consulted to determine how many huge page reservations
+are needed for the current mapping/segment.  For private mappings, this is
+always the value (to - from).  However, for shared mappings it is possible that some reservations may already exist within the range (to - from).  See the
+section :ref:`Reservation Map Modifications <resv_map_modifications>`
+for details on how this is accomplished.
+
+The mapping may be associated with a subpool.  If so, the subpool is consulted
+to ensure there is sufficient space for the mapping.  It is possible that the
+subpool has set aside reservations that can be used for the mapping.  See the
+section :ref:`Subpool Reservations <sub_pool_resv>` for more details.
+
+After consulting the reservation map and subpool, the number of needed new
+reservations is known.  The routine hugetlb_acct_memory() is called to check
+for and take the requested number of reservations.  hugetlb_acct_memory()
+calls into routines that potentially allocate and adjust surplus page counts.
+However, within those routines the code is simply checking to ensure there
+are enough free huge pages to accommodate the reservation.  If there are,
+the global reservation count resv_huge_pages is adjusted something like the
+following::
+
+	if (resv_needed <= (resv_huge_pages - free_huge_pages))
+		resv_huge_pages += resv_needed;
+
+Note that the global lock hugetlb_lock is held when checking and adjusting
+these counters.
+
+If there were enough free huge pages and the global count resv_huge_pages
+was adjusted, then the reservation map associated with the mapping is
+modified to reflect the reservations.  In the case of a shared mapping, a
+file_region will exist that includes the range 'from' 'to'.  For private
+mappings, no modifications are made to the reservation map as lack of an
+entry indicates a reservation exists.
+
+If hugetlb_reserve_pages() was successful, the global reservation count and
+reservation map associated with the mapping will be modified as required to
+ensure reservations exist for the range 'from' - 'to'.
+
+.. _consume_resv:
+
+Consuming Reservations/Allocating a Huge Page
+=============================================
+
+Reservations are consumed when huge pages associated with the reservations
+are allocated and instantiated in the corresponding mapping.  The allocation
+is performed within the routine alloc_huge_page()::
+
+	struct page *alloc_huge_page(struct vm_area_struct *vma,
+				     unsigned long addr, int avoid_reserve)
+
+alloc_huge_page is passed a VMA pointer and a virtual address, so it can
+consult the reservation map to determine if a reservation exists.  In addition,
+alloc_huge_page takes the argument avoid_reserve which indicates reserves
+should not be used even if it appears they have been set aside for the
+specified address.  The avoid_reserve argument is most often used in the case
+of Copy on Write and Page Migration where additional copies of an existing
+page are being allocated.
+
+The helper routine vma_needs_reservation() is called to determine if a
+reservation exists for the address within the mapping(vma).  See the section
+:ref:`Reservation Map Helper Routines <resv_map_helpers>` for detailed
+information on what this routine does.
+The value returned from vma_needs_reservation() is generally
+0 or 1.  0 if a reservation exists for the address, 1 if no reservation exists.
+If a reservation does not exist, and there is a subpool associated with the
+mapping the subpool is consulted to determine if it contains reservations.
+If the subpool contains reservations, one can be used for this allocation.
+However, in every case the avoid_reserve argument overrides the use of
+a reservation for the allocation.  After determining whether a reservation
+exists and can be used for the allocation, the routine dequeue_huge_page_vma()
+is called.  This routine takes two arguments related to reservations:
+
+- avoid_reserve, this is the same value/argument passed to alloc_huge_page()
+- chg, even though this argument is of type long only the values 0 or 1 are
+  passed to dequeue_huge_page_vma.  If the value is 0, it indicates a
+  reservation exists (see the section "Memory Policy and Reservations" for
+  possible issues).  If the value is 1, it indicates a reservation does not
+  exist and the page must be taken from the global free pool if possible.
+
+The free lists associated with the memory policy of the VMA are searched for
+a free page.  If a page is found, the value free_huge_pages is decremented
+when the page is removed from the free list.  If there was a reservation
+associated with the page, the following adjustments are made::
+
+	SetPagePrivate(page);	/* Indicates allocating this page consumed
+				 * a reservation, and if an error is
+				 * encountered such that the page must be
+				 * freed, the reservation will be restored. */
+	resv_huge_pages--;	/* Decrement the global reservation count */
+
+Note, if no huge page can be found that satisfies the VMA's memory policy
+an attempt will be made to allocate one using the buddy allocator.  This
+brings up the issue of surplus huge pages and overcommit which is beyond
+the scope reservations.  Even if a surplus page is allocated, the same
+reservation based adjustments as above will be made: SetPagePrivate(page) and
+resv_huge_pages--.
+
+After obtaining a new huge page, (page)->private is set to the value of
+the subpool associated with the page if it exists.  This will be used for
+subpool accounting when the page is freed.
+
+The routine vma_commit_reservation() is then called to adjust the reserve
+map based on the consumption of the reservation.  In general, this involves
+ensuring the page is represented within a file_region structure of the region
+map.  For shared mappings where the the reservation was present, an entry
+in the reserve map already existed so no change is made.  However, if there
+was no reservation in a shared mapping or this was a private mapping a new
+entry must be created.
+
+It is possible that the reserve map could have been changed between the call
+to vma_needs_reservation() at the beginning of alloc_huge_page() and the
+call to vma_commit_reservation() after the page was allocated.  This would
+be possible if hugetlb_reserve_pages was called for the same page in a shared
+mapping.  In such cases, the reservation count and subpool free page count
+will be off by one.  This rare condition can be identified by comparing the
+return value from vma_needs_reservation and vma_commit_reservation.  If such
+a race is detected, the subpool and global reserve counts are adjusted to
+compensate.  See the section
+:ref:`Reservation Map Helper Routines <resv_map_helpers>` for more
+information on these routines.
+
+
+Instantiate Huge Pages
+======================
+
+After huge page allocation, the page is typically added to the page tables
+of the allocating task.  Before this, pages in a shared mapping are added
+to the page cache and pages in private mappings are added to an anonymous
+reverse mapping.  In both cases, the PagePrivate flag is cleared.  Therefore,
+when a huge page that has been instantiated is freed no adjustment is made
+to the global reservation count (resv_huge_pages).
+
+
+Freeing Huge Pages
+==================
+
+Huge page freeing is performed by the routine free_huge_page().  This routine
+is the destructor for hugetlbfs compound pages.  As a result, it is only
+passed a pointer to the page struct.  When a huge page is freed, reservation
+accounting may need to be performed.  This would be the case if the page was
+associated with a subpool that contained reserves, or the page is being freed
+on an error path where a global reserve count must be restored.
+
+The page->private field points to any subpool associated with the page.
+If the PagePrivate flag is set, it indicates the global reserve count should
+be adjusted (see the section
+:ref:`Consuming Reservations/Allocating a Huge Page <consume_resv>`
+for information on how these are set).
+
+The routine first calls hugepage_subpool_put_pages() for the page.  If this
+routine returns a value of 0 (which does not equal the value passed 1) it
+indicates reserves are associated with the subpool, and this newly free page
+must be used to keep the number of subpool reserves above the minimum size.
+Therefore, the global resv_huge_pages counter is incremented in this case.
+
+If the PagePrivate flag was set in the page, the global resv_huge_pages counter
+will always be incremented.
+
+.. _sub_pool_resv:
+
+Subpool Reservations
+====================
+
+There is a struct hstate associated with each huge page size.  The hstate
+tracks all huge pages of the specified size.  A subpool represents a subset
+of pages within a hstate that is associated with a mounted hugetlbfs
+filesystem.
+
+When a hugetlbfs filesystem is mounted a min_size option can be specified
+which indicates the minimum number of huge pages required by the filesystem.
+If this option is specified, the number of huge pages corresponding to
+min_size are reserved for use by the filesystem.  This number is tracked in
+the min_hpages field of a struct hugepage_subpool.  At mount time,
+hugetlb_acct_memory(min_hpages) is called to reserve the specified number of
+huge pages.  If they can not be reserved, the mount fails.
+
+The routines hugepage_subpool_get/put_pages() are called when pages are
+obtained from or released back to a subpool.  They perform all subpool
+accounting, and track any reservations associated with the subpool.
+hugepage_subpool_get/put_pages are passed the number of huge pages by which
+to adjust the subpool 'used page' count (down for get, up for put).  Normally,
+they return the same value that was passed or an error if not enough pages
+exist in the subpool.
+
+However, if reserves are associated with the subpool a return value less
+than the passed value may be returned.  This return value indicates the
+number of additional global pool adjustments which must be made.  For example,
+suppose a subpool contains 3 reserved huge pages and someone asks for 5.
+The 3 reserved pages associated with the subpool can be used to satisfy part
+of the request.  But, 2 pages must be obtained from the global pools.  To
+relay this information to the caller, the value 2 is returned.  The caller
+is then responsible for attempting to obtain the additional two pages from
+the global pools.
+
+
+COW and Reservations
+====================
+
+Since shared mappings all point to and use the same underlying pages, the
+biggest reservation concern for COW is private mappings.  In this case,
+two tasks can be pointing at the same previously allocated page.  One task
+attempts to write to the page, so a new page must be allocated so that each
+task points to its own page.
+
+When the page was originally allocated, the reservation for that page was
+consumed.  When an attempt to allocate a new page is made as a result of
+COW, it is possible that no free huge pages are free and the allocation
+will fail.
+
+When the private mapping was originally created, the owner of the mapping
+was noted by setting the HPAGE_RESV_OWNER bit in the pointer to the reservation
+map of the owner.  Since the owner created the mapping, the owner owns all
+the reservations associated with the mapping.  Therefore, when a write fault
+occurs and there is no page available, different action is taken for the owner
+and non-owner of the reservation.
+
+In the case where the faulting task is not the owner, the fault will fail and
+the task will typically receive a SIGBUS.
+
+If the owner is the faulting task, we want it to succeed since it owned the
+original reservation.  To accomplish this, the page is unmapped from the
+non-owning task.  In this way, the only reference is from the owning task.
+In addition, the HPAGE_RESV_UNMAPPED bit is set in the reservation map pointer
+of the non-owning task.  The non-owning task may receive a SIGBUS if it later
+faults on a non-present page.  But, the original owner of the
+mapping/reservation will behave as expected.
+
+
+.. _resv_map_modifications:
+
+Reservation Map Modifications
+=============================
+
+The following low level routines are used to make modifications to a
+reservation map.  Typically, these routines are not called directly.  Rather,
+a reservation map helper routine is called which calls one of these low level
+routines.  These low level routines are fairly well documented in the source
+code (mm/hugetlb.c).  These routines are::
+
+	long region_chg(struct resv_map *resv, long f, long t);
+	long region_add(struct resv_map *resv, long f, long t);
+	void region_abort(struct resv_map *resv, long f, long t);
+	long region_count(struct resv_map *resv, long f, long t);
+
+Operations on the reservation map typically involve two operations:
+
+1) region_chg() is called to examine the reserve map and determine how
+   many pages in the specified range [f, t) are NOT currently represented.
+
+   The calling code performs global checks and allocations to determine if
+   there are enough huge pages for the operation to succeed.
+
+2)
+  a) If the operation can succeed, region_add() is called to actually modify
+     the reservation map for the same range [f, t) previously passed to
+     region_chg().
+  b) If the operation can not succeed, region_abort is called for the same
+     range [f, t) to abort the operation.
+
+Note that this is a two step process where region_add() and region_abort()
+are guaranteed to succeed after a prior call to region_chg() for the same
+range.  region_chg() is responsible for pre-allocating any data structures
+necessary to ensure the subsequent operations (specifically region_add()))
+will succeed.
+
+As mentioned above, region_chg() determines the number of pages in the range
+which are NOT currently represented in the map.  This number is returned to
+the caller.  region_add() returns the number of pages in the range added to
+the map.  In most cases, the return value of region_add() is the same as the
+return value of region_chg().  However, in the case of shared mappings it is
+possible for changes to the reservation map to be made between the calls to
+region_chg() and region_add().  In this case, the return value of region_add()
+will not match the return value of region_chg().  It is likely that in such
+cases global counts and subpool accounting will be incorrect and in need of
+adjustment.  It is the responsibility of the caller to check for this condition
+and make the appropriate adjustments.
+
+The routine region_del() is called to remove regions from a reservation map.
+It is typically called in the following situations:
+
+- When a file in the hugetlbfs filesystem is being removed, the inode will
+  be released and the reservation map freed.  Before freeing the reservation
+  map, all the individual file_region structures must be freed.  In this case
+  region_del is passed the range [0, LONG_MAX).
+- When a hugetlbfs file is being truncated.  In this case, all allocated pages
+  after the new file size must be freed.  In addition, any file_region entries
+  in the reservation map past the new end of file must be deleted.  In this
+  case, region_del is passed the range [new_end_of_file, LONG_MAX).
+- When a hole is being punched in a hugetlbfs file.  In this case, huge pages
+  are removed from the middle of the file one at a time.  As the pages are
+  removed, region_del() is called to remove the corresponding entry from the
+  reservation map.  In this case, region_del is passed the range
+  [page_idx, page_idx + 1).
+
+In every case, region_del() will return the number of pages removed from the
+reservation map.  In VERY rare cases, region_del() can fail.  This can only
+happen in the hole punch case where it has to split an existing file_region
+entry and can not allocate a new structure.  In this error case, region_del()
+will return -ENOMEM.  The problem here is that the reservation map will
+indicate that there is a reservation for the page.  However, the subpool and
+global reservation counts will not reflect the reservation.  To handle this
+situation, the routine hugetlb_fix_reserve_counts() is called to adjust the
+counters so that they correspond with the reservation map entry that could
+not be deleted.
+
+region_count() is called when unmapping a private huge page mapping.  In
+private mappings, the lack of a entry in the reservation map indicates that
+a reservation exists.  Therefore, by counting the number of entries in the
+reservation map we know how many reservations were consumed and how many are
+outstanding (outstanding = (end - start) - region_count(resv, start, end)).
+Since the mapping is going away, the subpool and global reservation counts
+are decremented by the number of outstanding reservations.
+
+.. _resv_map_helpers:
+
+Reservation Map Helper Routines
+===============================
+
+Several helper routines exist to query and modify the reservation maps.
+These routines are only interested with reservations for a specific huge
+page, so they just pass in an address instead of a range.  In addition,
+they pass in the associated VMA.  From the VMA, the type of mapping (private
+or shared) and the location of the reservation map (inode or VMA) can be
+determined.  These routines simply call the underlying routines described
+in the section "Reservation Map Modifications".  However, they do take into
+account the 'opposite' meaning of reservation map entries for private and
+shared mappings and hide this detail from the caller::
+
+	long vma_needs_reservation(struct hstate *h,
+				   struct vm_area_struct *vma,
+				   unsigned long addr)
+
+This routine calls region_chg() for the specified page.  If no reservation
+exists, 1 is returned.  If a reservation exists, 0 is returned::
+
+	long vma_commit_reservation(struct hstate *h,
+				    struct vm_area_struct *vma,
+				    unsigned long addr)
+
+This calls region_add() for the specified page.  As in the case of region_chg
+and region_add, this routine is to be called after a previous call to
+vma_needs_reservation.  It will add a reservation entry for the page.  It
+returns 1 if the reservation was added and 0 if not.  The return value should
+be compared with the return value of the previous call to
+vma_needs_reservation.  An unexpected difference indicates the reservation
+map was modified between calls::
+
+	void vma_end_reservation(struct hstate *h,
+				 struct vm_area_struct *vma,
+				 unsigned long addr)
+
+This calls region_abort() for the specified page.  As in the case of region_chg
+and region_abort, this routine is to be called after a previous call to
+vma_needs_reservation.  It will abort/end the in progress reservation add
+operation::
+
+	long vma_add_reservation(struct hstate *h,
+				 struct vm_area_struct *vma,
+				 unsigned long addr)
+
+This is a special wrapper routine to help facilitate reservation cleanup
+on error paths.  It is only called from the routine restore_reserve_on_error().
+This routine is used in conjunction with vma_needs_reservation in an attempt
+to add a reservation to the reservation map.  It takes into account the
+different reservation map semantics for private and shared mappings.  Hence,
+region_add is called for shared mappings (as an entry present in the map
+indicates a reservation), and region_del is called for private mappings (as
+the absence of an entry in the map indicates a reservation).  See the section
+"Reservation cleanup in error paths" for more information on what needs to
+be done on error paths.
+
+
+Reservation Cleanup in Error Paths
+==================================
+
+As mentioned in the section
+:ref:`Reservation Map Helper Routines <resv_map_helpers>`, reservation
+map modifications are performed in two steps.  First vma_needs_reservation
+is called before a page is allocated.  If the allocation is successful,
+then vma_commit_reservation is called.  If not, vma_end_reservation is called.
+Global and subpool reservation counts are adjusted based on success or failure
+of the operation and all is well.
+
+Additionally, after a huge page is instantiated the PagePrivate flag is
+cleared so that accounting when the page is ultimately freed is correct.
+
+However, there are several instances where errors are encountered after a huge
+page is allocated but before it is instantiated.  In this case, the page
+allocation has consumed the reservation and made the appropriate subpool,
+reservation map and global count adjustments.  If the page is freed at this
+time (before instantiation and clearing of PagePrivate), then free_huge_page
+will increment the global reservation count.  However, the reservation map
+indicates the reservation was consumed.  This resulting inconsistent state
+will cause the 'leak' of a reserved huge page.  The global reserve count will
+be  higher than it should and prevent allocation of a pre-allocated page.
+
+The routine restore_reserve_on_error() attempts to handle this situation.  It
+is fairly well documented.  The intention of this routine is to restore
+the reservation map to the way it was before the page allocation.   In this
+way, the state of the reservation map will correspond to the global reservation
+count after the page is freed.
+
+The routine restore_reserve_on_error itself may encounter errors while
+attempting to restore the reservation map entry.  In this case, it will
+simply clear the PagePrivate flag of the page.  In this way, the global
+reserve count will not be incremented when the page is freed.  However, the
+reservation map will continue to look as though the reservation was consumed.
+A page can still be allocated for the address, but it will not use a reserved
+page as originally intended.
+
+There is some code (most notably userfaultfd) which can not call
+restore_reserve_on_error.  In this case, it simply modifies the PagePrivate
+so that a reservation will not be leaked when the huge page is freed.
+
+
+Reservations and Memory Policy
+==============================
+Per-node huge page lists existed in struct hstate when git was first used
+to manage Linux code.  The concept of reservations was added some time later.
+When reservations were added, no attempt was made to take memory policy
+into account.  While cpusets are not exactly the same as memory policy, this
+comment in hugetlb_acct_memory sums up the interaction between reservations
+and cpusets/memory policy::
+
+	/*
+	 * When cpuset is configured, it breaks the strict hugetlb page
+	 * reservation as the accounting is done on a global variable. Such
+	 * reservation is completely rubbish in the presence of cpuset because
+	 * the reservation is not checked against page availability for the
+	 * current cpuset. Application can still potentially OOM'ed by kernel
+	 * with lack of free htlb page in cpuset that the task is in.
+	 * Attempt to enforce strict accounting with cpuset is almost
+	 * impossible (or too ugly) because cpuset is too fluid that
+	 * task or memory node can be dynamically moved between cpusets.
+	 *
+	 * The change of semantics for shared hugetlb mapping with cpuset is
+	 * undesirable. However, in order to preserve some of the semantics,
+	 * we fall back to check against current free page availability as
+	 * a best attempt and hopefully to minimize the impact of changing
+	 * semantics that cpuset has.
+	 */
+
+Huge page reservations were added to prevent unexpected page allocation
+failures (OOM) at page fault time.  However, if an application makes use
+of cpusets or memory policy there is no guarantee that huge pages will be
+available on the required nodes.  This is true even if there are a sufficient
+number of global reservations.
+
+Hugetlbfs regression testing
+============================
+
+The most complete set of hugetlb tests are in the libhugetlbfs repository.
+If you modify any hugetlb related code, use the libhugetlbfs test suite
+to check for regressions.  In addition, if you add any new hugetlb
+functionality, please add appropriate tests to libhugetlbfs.
+
+--
+Mike Kravetz, 7 April 2017
diff --git a/Documentation/vm/hugetlbfs_reserv.txt b/Documentation/vm/hugetlbfs_reserv.txt
deleted file mode 100644
index 9aca09a76bed..000000000000
--- a/Documentation/vm/hugetlbfs_reserv.txt
+++ /dev/null
@@ -1,529 +0,0 @@
-Hugetlbfs Reservation Overview
-------------------------------
-Huge pages as described at 'Documentation/vm/hugetlbpage.txt' are typically
-preallocated for application use.  These huge pages are instantiated in a
-task's address space at page fault time if the VMA indicates huge pages are
-to be used.  If no huge page exists at page fault time, the task is sent
-a SIGBUS and often dies an unhappy death.  Shortly after huge page support
-was added, it was determined that it would be better to detect a shortage
-of huge pages at mmap() time.  The idea is that if there were not enough
-huge pages to cover the mapping, the mmap() would fail.  This was first
-done with a simple check in the code at mmap() time to determine if there
-were enough free huge pages to cover the mapping.  Like most things in the
-kernel, the code has evolved over time.  However, the basic idea was to
-'reserve' huge pages at mmap() time to ensure that huge pages would be
-available for page faults in that mapping.  The description below attempts to
-describe how huge page reserve processing is done in the v4.10 kernel.
-
-
-Audience
---------
-This description is primarily targeted at kernel developers who are modifying
-hugetlbfs code.
-
-
-The Data Structures
--------------------
-resv_huge_pages
-	This is a global (per-hstate) count of reserved huge pages.  Reserved
-	huge pages are only available to the task which reserved them.
-	Therefore, the number of huge pages generally available is computed
-	as (free_huge_pages - resv_huge_pages).
-Reserve Map
-	A reserve map is described by the structure:
-	struct resv_map {
-		struct kref refs;
-		spinlock_t lock;
-		struct list_head regions;
-		long adds_in_progress;
-		struct list_head region_cache;
-		long region_cache_count;
-	};
-	There is one reserve map for each huge page mapping in the system.
-	The regions list within the resv_map describes the regions within
-	the mapping.  A region is described as:
-	struct file_region {
-		struct list_head link;
-		long from;
-		long to;
-	};
-	The 'from' and 'to' fields of the file region structure are huge page
-	indices into the mapping.  Depending on the type of mapping, a
-	region in the reserv_map may indicate reservations exist for the
-	range, or reservations do not exist.
-Flags for MAP_PRIVATE Reservations
-	These are stored in the bottom bits of the reservation map pointer.
-	#define HPAGE_RESV_OWNER    (1UL << 0) Indicates this task is the
-		owner of the reservations associated with the mapping.
-	#define HPAGE_RESV_UNMAPPED (1UL << 1) Indicates task originally
-		mapping this range (and creating reserves) has unmapped a
-		page from this task (the child) due to a failed COW.
-Page Flags
-	The PagePrivate page flag is used to indicate that a huge page
-	reservation must be restored when the huge page is freed.  More
-	details will be discussed in the "Freeing huge pages" section.
-
-
-Reservation Map Location (Private or Shared)
---------------------------------------------
-A huge page mapping or segment is either private or shared.  If private,
-it is typically only available to a single address space (task).  If shared,
-it can be mapped into multiple address spaces (tasks).  The location and
-semantics of the reservation map is significantly different for two types
-of mappings.  Location differences are:
-- For private mappings, the reservation map hangs off the the VMA structure.
-  Specifically, vma->vm_private_data.  This reserve map is created at the
-  time the mapping (mmap(MAP_PRIVATE)) is created.
-- For shared mappings, the reservation map hangs off the inode.  Specifically,
-  inode->i_mapping->private_data.  Since shared mappings are always backed
-  by files in the hugetlbfs filesystem, the hugetlbfs code ensures each inode
-  contains a reservation map.  As a result, the reservation map is allocated
-  when the inode is created.
-
-
-Creating Reservations
----------------------
-Reservations are created when a huge page backed shared memory segment is
-created (shmget(SHM_HUGETLB)) or a mapping is created via mmap(MAP_HUGETLB).
-These operations result in a call to the routine hugetlb_reserve_pages()
-
-int hugetlb_reserve_pages(struct inode *inode,
-					long from, long to,
-					struct vm_area_struct *vma,
-					vm_flags_t vm_flags)
-
-The first thing hugetlb_reserve_pages() does is check for the NORESERVE
-flag was specified in either the shmget() or mmap() call.  If NORESERVE
-was specified, then this routine returns immediately as no reservation
-are desired.
-
-The arguments 'from' and 'to' are huge page indices into the mapping or
-underlying file.  For shmget(), 'from' is always 0 and 'to' corresponds to
-the length of the segment/mapping.  For mmap(), the offset argument could
-be used to specify the offset into the underlying file.  In such a case
-the 'from' and 'to' arguments have been adjusted by this offset.
-
-One of the big differences between PRIVATE and SHARED mappings is the way
-in which reservations are represented in the reservation map.
-- For shared mappings, an entry in the reservation map indicates a reservation
-  exists or did exist for the corresponding page.  As reservations are
-  consumed, the reservation map is not modified.
-- For private mappings, the lack of an entry in the reservation map indicates
-  a reservation exists for the corresponding page.  As reservations are
-  consumed, entries are added to the reservation map.  Therefore, the
-  reservation map can also be used to determine which reservations have
-  been consumed.
-
-For private mappings, hugetlb_reserve_pages() creates the reservation map and
-hangs it off the VMA structure.  In addition, the HPAGE_RESV_OWNER flag is set
-to indicate this VMA owns the reservations.
-
-The reservation map is consulted to determine how many huge page reservations
-are needed for the current mapping/segment.  For private mappings, this is
-always the value (to - from).  However, for shared mappings it is possible that some reservations may already exist within the range (to - from).  See the
-section "Reservation Map Modifications" for details on how this is accomplished.
-
-The mapping may be associated with a subpool.  If so, the subpool is consulted
-to ensure there is sufficient space for the mapping.  It is possible that the
-subpool has set aside reservations that can be used for the mapping.  See the
-section "Subpool Reservations" for more details.
-
-After consulting the reservation map and subpool, the number of needed new
-reservations is known.  The routine hugetlb_acct_memory() is called to check
-for and take the requested number of reservations.  hugetlb_acct_memory()
-calls into routines that potentially allocate and adjust surplus page counts.
-However, within those routines the code is simply checking to ensure there
-are enough free huge pages to accommodate the reservation.  If there are,
-the global reservation count resv_huge_pages is adjusted something like the
-following.
-	if (resv_needed <= (resv_huge_pages - free_huge_pages))
-		resv_huge_pages += resv_needed;
-Note that the global lock hugetlb_lock is held when checking and adjusting
-these counters.
-
-If there were enough free huge pages and the global count resv_huge_pages
-was adjusted, then the reservation map associated with the mapping is
-modified to reflect the reservations.  In the case of a shared mapping, a
-file_region will exist that includes the range 'from' 'to'.  For private
-mappings, no modifications are made to the reservation map as lack of an
-entry indicates a reservation exists.
-
-If hugetlb_reserve_pages() was successful, the global reservation count and
-reservation map associated with the mapping will be modified as required to
-ensure reservations exist for the range 'from' - 'to'.
-
-
-Consuming Reservations/Allocating a Huge Page
----------------------------------------------
-Reservations are consumed when huge pages associated with the reservations
-are allocated and instantiated in the corresponding mapping.  The allocation
-is performed within the routine alloc_huge_page().
-struct page *alloc_huge_page(struct vm_area_struct *vma,
-                                    unsigned long addr, int avoid_reserve)
-alloc_huge_page is passed a VMA pointer and a virtual address, so it can
-consult the reservation map to determine if a reservation exists.  In addition,
-alloc_huge_page takes the argument avoid_reserve which indicates reserves
-should not be used even if it appears they have been set aside for the
-specified address.  The avoid_reserve argument is most often used in the case
-of Copy on Write and Page Migration where additional copies of an existing
-page are being allocated.
-
-The helper routine vma_needs_reservation() is called to determine if a
-reservation exists for the address within the mapping(vma).  See the section
-"Reservation Map Helper Routines" for detailed information on what this
-routine does.  The value returned from vma_needs_reservation() is generally
-0 or 1.  0 if a reservation exists for the address, 1 if no reservation exists.
-If a reservation does not exist, and there is a subpool associated with the
-mapping the subpool is consulted to determine if it contains reservations.
-If the subpool contains reservations, one can be used for this allocation.
-However, in every case the avoid_reserve argument overrides the use of
-a reservation for the allocation.  After determining whether a reservation
-exists and can be used for the allocation, the routine dequeue_huge_page_vma()
-is called.  This routine takes two arguments related to reservations:
-- avoid_reserve, this is the same value/argument passed to alloc_huge_page()
-- chg, even though this argument is of type long only the values 0 or 1 are
-  passed to dequeue_huge_page_vma.  If the value is 0, it indicates a
-  reservation exists (see the section "Memory Policy and Reservations" for
-  possible issues).  If the value is 1, it indicates a reservation does not
-  exist and the page must be taken from the global free pool if possible.
-The free lists associated with the memory policy of the VMA are searched for
-a free page.  If a page is found, the value free_huge_pages is decremented
-when the page is removed from the free list.  If there was a reservation
-associated with the page, the following adjustments are made:
-	SetPagePrivate(page);	/* Indicates allocating this page consumed
-				 * a reservation, and if an error is
-				 * encountered such that the page must be
-				 * freed, the reservation will be restored. */
-	resv_huge_pages--;	/* Decrement the global reservation count */
-Note, if no huge page can be found that satisfies the VMA's memory policy
-an attempt will be made to allocate one using the buddy allocator.  This
-brings up the issue of surplus huge pages and overcommit which is beyond
-the scope reservations.  Even if a surplus page is allocated, the same
-reservation based adjustments as above will be made: SetPagePrivate(page) and
-resv_huge_pages--.
-
-After obtaining a new huge page, (page)->private is set to the value of
-the subpool associated with the page if it exists.  This will be used for
-subpool accounting when the page is freed.
-
-The routine vma_commit_reservation() is then called to adjust the reserve
-map based on the consumption of the reservation.  In general, this involves
-ensuring the page is represented within a file_region structure of the region
-map.  For shared mappings where the the reservation was present, an entry
-in the reserve map already existed so no change is made.  However, if there
-was no reservation in a shared mapping or this was a private mapping a new
-entry must be created.
-
-It is possible that the reserve map could have been changed between the call
-to vma_needs_reservation() at the beginning of alloc_huge_page() and the
-call to vma_commit_reservation() after the page was allocated.  This would
-be possible if hugetlb_reserve_pages was called for the same page in a shared
-mapping.  In such cases, the reservation count and subpool free page count
-will be off by one.  This rare condition can be identified by comparing the
-return value from vma_needs_reservation and vma_commit_reservation.  If such
-a race is detected, the subpool and global reserve counts are adjusted to
-compensate.  See the section "Reservation Map Helper Routines" for more
-information on these routines.
-
-
-Instantiate Huge Pages
-----------------------
-After huge page allocation, the page is typically added to the page tables
-of the allocating task.  Before this, pages in a shared mapping are added
-to the page cache and pages in private mappings are added to an anonymous
-reverse mapping.  In both cases, the PagePrivate flag is cleared.  Therefore,
-when a huge page that has been instantiated is freed no adjustment is made
-to the global reservation count (resv_huge_pages).
-
-
-Freeing Huge Pages
-------------------
-Huge page freeing is performed by the routine free_huge_page().  This routine
-is the destructor for hugetlbfs compound pages.  As a result, it is only
-passed a pointer to the page struct.  When a huge page is freed, reservation
-accounting may need to be performed.  This would be the case if the page was
-associated with a subpool that contained reserves, or the page is being freed
-on an error path where a global reserve count must be restored.
-
-The page->private field points to any subpool associated with the page.
-If the PagePrivate flag is set, it indicates the global reserve count should
-be adjusted (see the section "Consuming Reservations/Allocating a Huge Page"
-for information on how these are set).
-
-The routine first calls hugepage_subpool_put_pages() for the page.  If this
-routine returns a value of 0 (which does not equal the value passed 1) it
-indicates reserves are associated with the subpool, and this newly free page
-must be used to keep the number of subpool reserves above the minimum size.
-Therefore, the global resv_huge_pages counter is incremented in this case.
-
-If the PagePrivate flag was set in the page, the global resv_huge_pages counter
-will always be incremented.
-
-
-Subpool Reservations
---------------------
-There is a struct hstate associated with each huge page size.  The hstate
-tracks all huge pages of the specified size.  A subpool represents a subset
-of pages within a hstate that is associated with a mounted hugetlbfs
-filesystem.
-
-When a hugetlbfs filesystem is mounted a min_size option can be specified
-which indicates the minimum number of huge pages required by the filesystem.
-If this option is specified, the number of huge pages corresponding to
-min_size are reserved for use by the filesystem.  This number is tracked in
-the min_hpages field of a struct hugepage_subpool.  At mount time,
-hugetlb_acct_memory(min_hpages) is called to reserve the specified number of
-huge pages.  If they can not be reserved, the mount fails.
-
-The routines hugepage_subpool_get/put_pages() are called when pages are
-obtained from or released back to a subpool.  They perform all subpool
-accounting, and track any reservations associated with the subpool.
-hugepage_subpool_get/put_pages are passed the number of huge pages by which
-to adjust the subpool 'used page' count (down for get, up for put).  Normally,
-they return the same value that was passed or an error if not enough pages
-exist in the subpool.
-
-However, if reserves are associated with the subpool a return value less
-than the passed value may be returned.  This return value indicates the
-number of additional global pool adjustments which must be made.  For example,
-suppose a subpool contains 3 reserved huge pages and someone asks for 5.
-The 3 reserved pages associated with the subpool can be used to satisfy part
-of the request.  But, 2 pages must be obtained from the global pools.  To
-relay this information to the caller, the value 2 is returned.  The caller
-is then responsible for attempting to obtain the additional two pages from
-the global pools.
-
-
-COW and Reservations
---------------------
-Since shared mappings all point to and use the same underlying pages, the
-biggest reservation concern for COW is private mappings.  In this case,
-two tasks can be pointing at the same previously allocated page.  One task
-attempts to write to the page, so a new page must be allocated so that each
-task points to its own page.
-
-When the page was originally allocated, the reservation for that page was
-consumed.  When an attempt to allocate a new page is made as a result of
-COW, it is possible that no free huge pages are free and the allocation
-will fail.
-
-When the private mapping was originally created, the owner of the mapping
-was noted by setting the HPAGE_RESV_OWNER bit in the pointer to the reservation
-map of the owner.  Since the owner created the mapping, the owner owns all
-the reservations associated with the mapping.  Therefore, when a write fault
-occurs and there is no page available, different action is taken for the owner
-and non-owner of the reservation.
-
-In the case where the faulting task is not the owner, the fault will fail and
-the task will typically receive a SIGBUS.
-
-If the owner is the faulting task, we want it to succeed since it owned the
-original reservation.  To accomplish this, the page is unmapped from the
-non-owning task.  In this way, the only reference is from the owning task.
-In addition, the HPAGE_RESV_UNMAPPED bit is set in the reservation map pointer
-of the non-owning task.  The non-owning task may receive a SIGBUS if it later
-faults on a non-present page.  But, the original owner of the
-mapping/reservation will behave as expected.
-
-
-Reservation Map Modifications
------------------------------
-The following low level routines are used to make modifications to a
-reservation map.  Typically, these routines are not called directly.  Rather,
-a reservation map helper routine is called which calls one of these low level
-routines.  These low level routines are fairly well documented in the source
-code (mm/hugetlb.c).  These routines are:
-long region_chg(struct resv_map *resv, long f, long t);
-long region_add(struct resv_map *resv, long f, long t);
-void region_abort(struct resv_map *resv, long f, long t);
-long region_count(struct resv_map *resv, long f, long t);
-
-Operations on the reservation map typically involve two operations:
-1) region_chg() is called to examine the reserve map and determine how
-   many pages in the specified range [f, t) are NOT currently represented.
-
-   The calling code performs global checks and allocations to determine if
-   there are enough huge pages for the operation to succeed.
-
-2a) If the operation can succeed, region_add() is called to actually modify
-    the reservation map for the same range [f, t) previously passed to
-    region_chg().
-2b) If the operation can not succeed, region_abort is called for the same range
-    [f, t) to abort the operation.
-
-Note that this is a two step process where region_add() and region_abort()
-are guaranteed to succeed after a prior call to region_chg() for the same
-range.  region_chg() is responsible for pre-allocating any data structures
-necessary to ensure the subsequent operations (specifically region_add()))
-will succeed.
-
-As mentioned above, region_chg() determines the number of pages in the range
-which are NOT currently represented in the map.  This number is returned to
-the caller.  region_add() returns the number of pages in the range added to
-the map.  In most cases, the return value of region_add() is the same as the
-return value of region_chg().  However, in the case of shared mappings it is
-possible for changes to the reservation map to be made between the calls to
-region_chg() and region_add().  In this case, the return value of region_add()
-will not match the return value of region_chg().  It is likely that in such
-cases global counts and subpool accounting will be incorrect and in need of
-adjustment.  It is the responsibility of the caller to check for this condition
-and make the appropriate adjustments.
-
-The routine region_del() is called to remove regions from a reservation map.
-It is typically called in the following situations:
-- When a file in the hugetlbfs filesystem is being removed, the inode will
-  be released and the reservation map freed.  Before freeing the reservation
-  map, all the individual file_region structures must be freed.  In this case
-  region_del is passed the range [0, LONG_MAX).
-- When a hugetlbfs file is being truncated.  In this case, all allocated pages
-  after the new file size must be freed.  In addition, any file_region entries
-  in the reservation map past the new end of file must be deleted.  In this
-  case, region_del is passed the range [new_end_of_file, LONG_MAX).
-- When a hole is being punched in a hugetlbfs file.  In this case, huge pages
-  are removed from the middle of the file one at a time.  As the pages are
-  removed, region_del() is called to remove the corresponding entry from the
-  reservation map.  In this case, region_del is passed the range
-  [page_idx, page_idx + 1).
-In every case, region_del() will return the number of pages removed from the
-reservation map.  In VERY rare cases, region_del() can fail.  This can only
-happen in the hole punch case where it has to split an existing file_region
-entry and can not allocate a new structure.  In this error case, region_del()
-will return -ENOMEM.  The problem here is that the reservation map will
-indicate that there is a reservation for the page.  However, the subpool and
-global reservation counts will not reflect the reservation.  To handle this
-situation, the routine hugetlb_fix_reserve_counts() is called to adjust the
-counters so that they correspond with the reservation map entry that could
-not be deleted.
-
-region_count() is called when unmapping a private huge page mapping.  In
-private mappings, the lack of a entry in the reservation map indicates that
-a reservation exists.  Therefore, by counting the number of entries in the
-reservation map we know how many reservations were consumed and how many are
-outstanding (outstanding = (end - start) - region_count(resv, start, end)).
-Since the mapping is going away, the subpool and global reservation counts
-are decremented by the number of outstanding reservations.
-
-
-Reservation Map Helper Routines
--------------------------------
-Several helper routines exist to query and modify the reservation maps.
-These routines are only interested with reservations for a specific huge
-page, so they just pass in an address instead of a range.  In addition,
-they pass in the associated VMA.  From the VMA, the type of mapping (private
-or shared) and the location of the reservation map (inode or VMA) can be
-determined.  These routines simply call the underlying routines described
-in the section "Reservation Map Modifications".  However, they do take into
-account the 'opposite' meaning of reservation map entries for private and
-shared mappings and hide this detail from the caller.
-
-long vma_needs_reservation(struct hstate *h,
-				struct vm_area_struct *vma, unsigned long addr)
-This routine calls region_chg() for the specified page.  If no reservation
-exists, 1 is returned.  If a reservation exists, 0 is returned.
-
-long vma_commit_reservation(struct hstate *h,
-				struct vm_area_struct *vma, unsigned long addr)
-This calls region_add() for the specified page.  As in the case of region_chg
-and region_add, this routine is to be called after a previous call to
-vma_needs_reservation.  It will add a reservation entry for the page.  It
-returns 1 if the reservation was added and 0 if not.  The return value should
-be compared with the return value of the previous call to
-vma_needs_reservation.  An unexpected difference indicates the reservation
-map was modified between calls.
-
-void vma_end_reservation(struct hstate *h,
-				struct vm_area_struct *vma, unsigned long addr)
-This calls region_abort() for the specified page.  As in the case of region_chg
-and region_abort, this routine is to be called after a previous call to
-vma_needs_reservation.  It will abort/end the in progress reservation add
-operation.
-
-long vma_add_reservation(struct hstate *h,
-				struct vm_area_struct *vma, unsigned long addr)
-This is a special wrapper routine to help facilitate reservation cleanup
-on error paths.  It is only called from the routine restore_reserve_on_error().
-This routine is used in conjunction with vma_needs_reservation in an attempt
-to add a reservation to the reservation map.  It takes into account the
-different reservation map semantics for private and shared mappings.  Hence,
-region_add is called for shared mappings (as an entry present in the map
-indicates a reservation), and region_del is called for private mappings (as
-the absence of an entry in the map indicates a reservation).  See the section
-"Reservation cleanup in error paths" for more information on what needs to
-be done on error paths.
-
-
-Reservation Cleanup in Error Paths
-----------------------------------
-As mentioned in the section "Reservation Map Helper Routines", reservation
-map modifications are performed in two steps.  First vma_needs_reservation
-is called before a page is allocated.  If the allocation is successful,
-then vma_commit_reservation is called.  If not, vma_end_reservation is called.
-Global and subpool reservation counts are adjusted based on success or failure
-of the operation and all is well.
-
-Additionally, after a huge page is instantiated the PagePrivate flag is
-cleared so that accounting when the page is ultimately freed is correct.
-
-However, there are several instances where errors are encountered after a huge
-page is allocated but before it is instantiated.  In this case, the page
-allocation has consumed the reservation and made the appropriate subpool,
-reservation map and global count adjustments.  If the page is freed at this
-time (before instantiation and clearing of PagePrivate), then free_huge_page
-will increment the global reservation count.  However, the reservation map
-indicates the reservation was consumed.  This resulting inconsistent state
-will cause the 'leak' of a reserved huge page.  The global reserve count will
-be  higher than it should and prevent allocation of a pre-allocated page.
-
-The routine restore_reserve_on_error() attempts to handle this situation.  It
-is fairly well documented.  The intention of this routine is to restore
-the reservation map to the way it was before the page allocation.   In this
-way, the state of the reservation map will correspond to the global reservation
-count after the page is freed.
-
-The routine restore_reserve_on_error itself may encounter errors while
-attempting to restore the reservation map entry.  In this case, it will
-simply clear the PagePrivate flag of the page.  In this way, the global
-reserve count will not be incremented when the page is freed.  However, the
-reservation map will continue to look as though the reservation was consumed.
-A page can still be allocated for the address, but it will not use a reserved
-page as originally intended.
-
-There is some code (most notably userfaultfd) which can not call
-restore_reserve_on_error.  In this case, it simply modifies the PagePrivate
-so that a reservation will not be leaked when the huge page is freed.
-
-
-Reservations and Memory Policy
-------------------------------
-Per-node huge page lists existed in struct hstate when git was first used
-to manage Linux code.  The concept of reservations was added some time later.
-When reservations were added, no attempt was made to take memory policy
-into account.  While cpusets are not exactly the same as memory policy, this
-comment in hugetlb_acct_memory sums up the interaction between reservations
-and cpusets/memory policy.
-	/*
-	 * When cpuset is configured, it breaks the strict hugetlb page
-	 * reservation as the accounting is done on a global variable. Such
-	 * reservation is completely rubbish in the presence of cpuset because
-	 * the reservation is not checked against page availability for the
-	 * current cpuset. Application can still potentially OOM'ed by kernel
-	 * with lack of free htlb page in cpuset that the task is in.
-	 * Attempt to enforce strict accounting with cpuset is almost
-	 * impossible (or too ugly) because cpuset is too fluid that
-	 * task or memory node can be dynamically moved between cpusets.
-	 *
-	 * The change of semantics for shared hugetlb mapping with cpuset is
-	 * undesirable. However, in order to preserve some of the semantics,
-	 * we fall back to check against current free page availability as
-	 * a best attempt and hopefully to minimize the impact of changing
-	 * semantics that cpuset has.
-	 */
-
-Huge page reservations were added to prevent unexpected page allocation
-failures (OOM) at page fault time.  However, if an application makes use
-of cpusets or memory policy there is no guarantee that huge pages will be
-available on the required nodes.  This is true even if there are a sufficient
-number of global reservations.
-
-
-Mike Kravetz, 7 April 2017
diff --git a/Documentation/vm/hugetlbpage.txt b/Documentation/vm/hugetlbpage.txt
deleted file mode 100644
index faf077d50d42..000000000000
--- a/Documentation/vm/hugetlbpage.txt
+++ /dev/null
@@ -1,351 +0,0 @@
-
-The intent of this file is to give a brief summary of hugetlbpage support in
-the Linux kernel.  This support is built on top of multiple page size support
-that is provided by most modern architectures.  For example, x86 CPUs normally
-support 4K and 2M (1G if architecturally supported) page sizes, ia64
-architecture supports multiple page sizes 4K, 8K, 64K, 256K, 1M, 4M, 16M,
-256M and ppc64 supports 4K and 16M.  A TLB is a cache of virtual-to-physical
-translations.  Typically this is a very scarce resource on processor.
-Operating systems try to make best use of limited number of TLB resources.
-This optimization is more critical now as bigger and bigger physical memories
-(several GBs) are more readily available.
-
-Users can use the huge page support in Linux kernel by either using the mmap
-system call or standard SYSV shared memory system calls (shmget, shmat).
-
-First the Linux kernel needs to be built with the CONFIG_HUGETLBFS
-(present under "File systems") and CONFIG_HUGETLB_PAGE (selected
-automatically when CONFIG_HUGETLBFS is selected) configuration
-options.
-
-The /proc/meminfo file provides information about the total number of
-persistent hugetlb pages in the kernel's huge page pool.  It also displays
-default huge page size and information about the number of free, reserved
-and surplus huge pages in the pool of huge pages of default size.
-The huge page size is needed for generating the proper alignment and
-size of the arguments to system calls that map huge page regions.
-
-The output of "cat /proc/meminfo" will include lines like:
-
-.....
-HugePages_Total: uuu
-HugePages_Free:  vvv
-HugePages_Rsvd:  www
-HugePages_Surp:  xxx
-Hugepagesize:    yyy kB
-Hugetlb:         zzz kB
-
-where:
-HugePages_Total is the size of the pool of huge pages.
-HugePages_Free  is the number of huge pages in the pool that are not yet
-                allocated.
-HugePages_Rsvd  is short for "reserved," and is the number of huge pages for
-                which a commitment to allocate from the pool has been made,
-                but no allocation has yet been made.  Reserved huge pages
-                guarantee that an application will be able to allocate a
-                huge page from the pool of huge pages at fault time.
-HugePages_Surp  is short for "surplus," and is the number of huge pages in
-                the pool above the value in /proc/sys/vm/nr_hugepages. The
-                maximum number of surplus huge pages is controlled by
-                /proc/sys/vm/nr_overcommit_hugepages.
-Hugepagesize    is the default hugepage size (in Kb).
-Hugetlb         is the total amount of memory (in kB), consumed by huge
-                pages of all sizes.
-                If huge pages of different sizes are in use, this number
-                will exceed HugePages_Total * Hugepagesize. To get more
-                detailed information, please, refer to
-                /sys/kernel/mm/hugepages (described below).
-
-
-/proc/filesystems should also show a filesystem of type "hugetlbfs" configured
-in the kernel.
-
-/proc/sys/vm/nr_hugepages indicates the current number of "persistent" huge
-pages in the kernel's huge page pool.  "Persistent" huge pages will be
-returned to the huge page pool when freed by a task.  A user with root
-privileges can dynamically allocate more or free some persistent huge pages
-by increasing or decreasing the value of 'nr_hugepages'.
-
-Pages that are used as huge pages are reserved inside the kernel and cannot
-be used for other purposes.  Huge pages cannot be swapped out under
-memory pressure.
-
-Once a number of huge pages have been pre-allocated to the kernel huge page
-pool, a user with appropriate privilege can use either the mmap system call
-or shared memory system calls to use the huge pages.  See the discussion of
-Using Huge Pages, below.
-
-The administrator can allocate persistent huge pages on the kernel boot
-command line by specifying the "hugepages=N" parameter, where 'N' = the
-number of huge pages requested.  This is the most reliable method of
-allocating huge pages as memory has not yet become fragmented.
-
-Some platforms support multiple huge page sizes.  To allocate huge pages
-of a specific size, one must precede the huge pages boot command parameters
-with a huge page size selection parameter "hugepagesz=<size>".  <size> must
-be specified in bytes with optional scale suffix [kKmMgG].  The default huge
-page size may be selected with the "default_hugepagesz=<size>" boot parameter.
-
-When multiple huge page sizes are supported, /proc/sys/vm/nr_hugepages
-indicates the current number of pre-allocated huge pages of the default size.
-Thus, one can use the following command to dynamically allocate/deallocate
-default sized persistent huge pages:
-
-	echo 20 > /proc/sys/vm/nr_hugepages
-
-This command will try to adjust the number of default sized huge pages in the
-huge page pool to 20, allocating or freeing huge pages, as required.
-
-On a NUMA platform, the kernel will attempt to distribute the huge page pool
-over all the set of allowed nodes specified by the NUMA memory policy of the
-task that modifies nr_hugepages.  The default for the allowed nodes--when the
-task has default memory policy--is all on-line nodes with memory.  Allowed
-nodes with insufficient available, contiguous memory for a huge page will be
-silently skipped when allocating persistent huge pages.  See the discussion
-below of the interaction of task memory policy, cpusets and per node attributes
-with the allocation and freeing of persistent huge pages.
-
-The success or failure of huge page allocation depends on the amount of
-physically contiguous memory that is present in system at the time of the
-allocation attempt.  If the kernel is unable to allocate huge pages from
-some nodes in a NUMA system, it will attempt to make up the difference by
-allocating extra pages on other nodes with sufficient available contiguous
-memory, if any.
-
-System administrators may want to put this command in one of the local rc
-init files.  This will enable the kernel to allocate huge pages early in
-the boot process when the possibility of getting physical contiguous pages
-is still very high.  Administrators can verify the number of huge pages
-actually allocated by checking the sysctl or meminfo.  To check the per node
-distribution of huge pages in a NUMA system, use:
-
-	cat /sys/devices/system/node/node*/meminfo | fgrep Huge
-
-/proc/sys/vm/nr_overcommit_hugepages specifies how large the pool of
-huge pages can grow, if more huge pages than /proc/sys/vm/nr_hugepages are
-requested by applications.  Writing any non-zero value into this file
-indicates that the hugetlb subsystem is allowed to try to obtain that
-number of "surplus" huge pages from the kernel's normal page pool, when the
-persistent huge page pool is exhausted. As these surplus huge pages become
-unused, they are freed back to the kernel's normal page pool.
-
-When increasing the huge page pool size via nr_hugepages, any existing surplus
-pages will first be promoted to persistent huge pages.  Then, additional
-huge pages will be allocated, if necessary and if possible, to fulfill
-the new persistent huge page pool size.
-
-The administrator may shrink the pool of persistent huge pages for
-the default huge page size by setting the nr_hugepages sysctl to a
-smaller value.  The kernel will attempt to balance the freeing of huge pages
-across all nodes in the memory policy of the task modifying nr_hugepages.
-Any free huge pages on the selected nodes will be freed back to the kernel's
-normal page pool.
-
-Caveat: Shrinking the persistent huge page pool via nr_hugepages such that
-it becomes less than the number of huge pages in use will convert the balance
-of the in-use huge pages to surplus huge pages.  This will occur even if
-the number of surplus pages it would exceed the overcommit value.  As long as
-this condition holds--that is, until nr_hugepages+nr_overcommit_hugepages is
-increased sufficiently, or the surplus huge pages go out of use and are freed--
-no more surplus huge pages will be allowed to be allocated.
-
-With support for multiple huge page pools at run-time available, much of
-the huge page userspace interface in /proc/sys/vm has been duplicated in sysfs.
-The /proc interfaces discussed above have been retained for backwards
-compatibility. The root huge page control directory in sysfs is:
-
-	/sys/kernel/mm/hugepages
-
-For each huge page size supported by the running kernel, a subdirectory
-will exist, of the form:
-
-	hugepages-${size}kB
-
-Inside each of these directories, the same set of files will exist:
-
-	nr_hugepages
-	nr_hugepages_mempolicy
-	nr_overcommit_hugepages
-	free_hugepages
-	resv_hugepages
-	surplus_hugepages
-
-which function as described above for the default huge page-sized case.
-
-
-Interaction of Task Memory Policy with Huge Page Allocation/Freeing
-===================================================================
-
-Whether huge pages are allocated and freed via the /proc interface or
-the /sysfs interface using the nr_hugepages_mempolicy attribute, the NUMA
-nodes from which huge pages are allocated or freed are controlled by the
-NUMA memory policy of the task that modifies the nr_hugepages_mempolicy
-sysctl or attribute.  When the nr_hugepages attribute is used, mempolicy
-is ignored.
-
-The recommended method to allocate or free huge pages to/from the kernel
-huge page pool, using the nr_hugepages example above, is:
-
-    numactl --interleave <node-list> echo 20 \
-				>/proc/sys/vm/nr_hugepages_mempolicy
-
-or, more succinctly:
-
-    numactl -m <node-list> echo 20 >/proc/sys/vm/nr_hugepages_mempolicy
-
-This will allocate or free abs(20 - nr_hugepages) to or from the nodes
-specified in <node-list>, depending on whether number of persistent huge pages
-is initially less than or greater than 20, respectively.  No huge pages will be
-allocated nor freed on any node not included in the specified <node-list>.
-
-When adjusting the persistent hugepage count via nr_hugepages_mempolicy, any
-memory policy mode--bind, preferred, local or interleave--may be used.  The
-resulting effect on persistent huge page allocation is as follows:
-
-1) Regardless of mempolicy mode [see Documentation/vm/numa_memory_policy.txt],
-   persistent huge pages will be distributed across the node or nodes
-   specified in the mempolicy as if "interleave" had been specified.
-   However, if a node in the policy does not contain sufficient contiguous
-   memory for a huge page, the allocation will not "fallback" to the nearest
-   neighbor node with sufficient contiguous memory.  To do this would cause
-   undesirable imbalance in the distribution of the huge page pool, or
-   possibly, allocation of persistent huge pages on nodes not allowed by
-   the task's memory policy.
-
-2) One or more nodes may be specified with the bind or interleave policy.
-   If more than one node is specified with the preferred policy, only the
-   lowest numeric id will be used.  Local policy will select the node where
-   the task is running at the time the nodes_allowed mask is constructed.
-   For local policy to be deterministic, the task must be bound to a cpu or
-   cpus in a single node.  Otherwise, the task could be migrated to some
-   other node at any time after launch and the resulting node will be
-   indeterminate.  Thus, local policy is not very useful for this purpose.
-   Any of the other mempolicy modes may be used to specify a single node.
-
-3) The nodes allowed mask will be derived from any non-default task mempolicy,
-   whether this policy was set explicitly by the task itself or one of its
-   ancestors, such as numactl.  This means that if the task is invoked from a
-   shell with non-default policy, that policy will be used.  One can specify a
-   node list of "all" with numactl --interleave or --membind [-m] to achieve
-   interleaving over all nodes in the system or cpuset.
-
-4) Any task mempolicy specified--e.g., using numactl--will be constrained by
-   the resource limits of any cpuset in which the task runs.  Thus, there will
-   be no way for a task with non-default policy running in a cpuset with a
-   subset of the system nodes to allocate huge pages outside the cpuset
-   without first moving to a cpuset that contains all of the desired nodes.
-
-5) Boot-time huge page allocation attempts to distribute the requested number
-   of huge pages over all on-lines nodes with memory.
-
-Per Node Hugepages Attributes
-=============================
-
-A subset of the contents of the root huge page control directory in sysfs,
-described above, will be replicated under each the system device of each
-NUMA node with memory in:
-
-	/sys/devices/system/node/node[0-9]*/hugepages/
-
-Under this directory, the subdirectory for each supported huge page size
-contains the following attribute files:
-
-	nr_hugepages
-	free_hugepages
-	surplus_hugepages
-
-The free_' and surplus_' attribute files are read-only.  They return the number
-of free and surplus [overcommitted] huge pages, respectively, on the parent
-node.
-
-The nr_hugepages attribute returns the total number of huge pages on the
-specified node.  When this attribute is written, the number of persistent huge
-pages on the parent node will be adjusted to the specified value, if sufficient
-resources exist, regardless of the task's mempolicy or cpuset constraints.
-
-Note that the number of overcommit and reserve pages remain global quantities,
-as we don't know until fault time, when the faulting task's mempolicy is
-applied, from which node the huge page allocation will be attempted.
-
-
-Using Huge Pages
-================
-
-If the user applications are going to request huge pages using mmap system
-call, then it is required that system administrator mount a file system of
-type hugetlbfs:
-
-  mount -t hugetlbfs \
-	-o uid=<value>,gid=<value>,mode=<value>,pagesize=<value>,size=<value>,\
-	min_size=<value>,nr_inodes=<value> none /mnt/huge
-
-This command mounts a (pseudo) filesystem of type hugetlbfs on the directory
-/mnt/huge.  Any files created on /mnt/huge uses huge pages.  The uid and gid
-options sets the owner and group of the root of the file system.  By default
-the uid and gid of the current process are taken.  The mode option sets the
-mode of root of file system to value & 01777.  This value is given in octal.
-By default the value 0755 is picked. If the platform supports multiple huge
-page sizes, the pagesize option can be used to specify the huge page size and
-associated pool.  pagesize is specified in bytes.  If pagesize is not specified
-the platform's default huge page size and associated pool will be used. The
-size option sets the maximum value of memory (huge pages) allowed for that
-filesystem (/mnt/huge).  The size option can be specified in bytes, or as a
-percentage of the specified huge page pool (nr_hugepages).  The size is
-rounded down to HPAGE_SIZE boundary.  The min_size option sets the minimum
-value of memory (huge pages) allowed for the filesystem.  min_size can be
-specified in the same way as size, either bytes or a percentage of the
-huge page pool.  At mount time, the number of huge pages specified by
-min_size are reserved for use by the filesystem.  If there are not enough
-free huge pages available, the mount will fail.  As huge pages are allocated
-to the filesystem and freed, the reserve count is adjusted so that the sum
-of allocated and reserved huge pages is always at least min_size.  The option
-nr_inodes sets the maximum number of inodes that /mnt/huge can use.  If the
-size, min_size or nr_inodes option is not provided on command line then
-no limits are set.  For pagesize, size, min_size and nr_inodes options, you
-can use [G|g]/[M|m]/[K|k] to represent giga/mega/kilo. For example, size=2K
-has the same meaning as size=2048.
-
-While read system calls are supported on files that reside on hugetlb
-file systems, write system calls are not.
-
-Regular chown, chgrp, and chmod commands (with right permissions) could be
-used to change the file attributes on hugetlbfs.
-
-Also, it is important to note that no such mount command is required if
-applications are going to use only shmat/shmget system calls or mmap with
-MAP_HUGETLB.  For an example of how to use mmap with MAP_HUGETLB see map_hugetlb
-below.
-
-Users who wish to use hugetlb memory via shared memory segment should be a
-member of a supplementary group and system admin needs to configure that gid
-into /proc/sys/vm/hugetlb_shm_group.  It is possible for same or different
-applications to use any combination of mmaps and shm* calls, though the mount of
-filesystem will be required for using mmap calls without MAP_HUGETLB.
-
-Syscalls that operate on memory backed by hugetlb pages only have their lengths
-aligned to the native page size of the processor; they will normally fail with
-errno set to EINVAL or exclude hugetlb pages that extend beyond the length if
-not hugepage aligned.  For example, munmap(2) will fail if memory is backed by
-a hugetlb page and the length is smaller than the hugepage size.
-
-
-Examples
-========
-
-1) map_hugetlb: see tools/testing/selftests/vm/map_hugetlb.c
-
-2) hugepage-shm:  see tools/testing/selftests/vm/hugepage-shm.c
-
-3) hugepage-mmap:  see tools/testing/selftests/vm/hugepage-mmap.c
-
-4) The libhugetlbfs (https://github.com/libhugetlbfs/libhugetlbfs) library
-   provides a wide range of userspace tools to help with huge page usability,
-   environment setup, and control.
-
-Kernel development regression testing
-=====================================
-
-The most complete set of hugetlb tests are in the libhugetlbfs repository.
-If you modify any hugetlb related code, use the libhugetlbfs test suite
-to check for regressions.  In addition, if you add any new hugetlb
-functionality, please add appropriate tests to libhugetlbfs.
diff --git a/Documentation/vm/hwpoison.rst b/Documentation/vm/hwpoison.rst
new file mode 100644
index 000000000000..09bd24a92784
--- /dev/null
+++ b/Documentation/vm/hwpoison.rst
@@ -0,0 +1,186 @@
+.. hwpoison:
+
+========
+hwpoison
+========
+
+What is hwpoison?
+=================
+
+Upcoming Intel CPUs have support for recovering from some memory errors
+(``MCA recovery``). This requires the OS to declare a page "poisoned",
+kill the processes associated with it and avoid using it in the future.
+
+This patchkit implements the necessary infrastructure in the VM.
+
+To quote the overview comment:
+
+ * High level machine check handler. Handles pages reported by the
+ * hardware as being corrupted usually due to a 2bit ECC memory or cache
+ * failure.
+ *
+ * This focusses on pages detected as corrupted in the background.
+ * When the current CPU tries to consume corruption the currently
+ * running process can just be killed directly instead. This implies
+ * that if the error cannot be handled for some reason it's safe to
+ * just ignore it because no corruption has been consumed yet. Instead
+ * when that happens another machine check will happen.
+ *
+ * Handles page cache pages in various states. The tricky part
+ * here is that we can access any page asynchronous to other VM
+ * users, because memory failures could happen anytime and anywhere,
+ * possibly violating some of their assumptions. This is why this code
+ * has to be extremely careful. Generally it tries to use normal locking
+ * rules, as in get the standard locks, even if that means the
+ * error handling takes potentially a long time.
+ *
+ * Some of the operations here are somewhat inefficient and have non
+ * linear algorithmic complexity, because the data structures have not
+ * been optimized for this case. This is in particular the case
+ * for the mapping from a vma to a process. Since this case is expected
+ * to be rare we hope we can get away with this.
+
+The code consists of a the high level handler in mm/memory-failure.c,
+a new page poison bit and various checks in the VM to handle poisoned
+pages.
+
+The main target right now is KVM guests, but it works for all kinds
+of applications. KVM support requires a recent qemu-kvm release.
+
+For the KVM use there was need for a new signal type so that
+KVM can inject the machine check into the guest with the proper
+address. This in theory allows other applications to handle
+memory failures too. The expection is that near all applications
+won't do that, but some very specialized ones might.
+
+Failure recovery modes
+======================
+
+There are two (actually three) modes memory failure recovery can be in:
+
+vm.memory_failure_recovery sysctl set to zero:
+	All memory failures cause a panic. Do not attempt recovery.
+	(on x86 this can be also affected by the tolerant level of the
+	MCE subsystem)
+
+early kill
+	(can be controlled globally and per process)
+	Send SIGBUS to the application as soon as the error is detected
+	This allows applications who can process memory errors in a gentle
+	way (e.g. drop affected object)
+	This is the mode used by KVM qemu.
+
+late kill
+	Send SIGBUS when the application runs into the corrupted page.
+	This is best for memory error unaware applications and default
+	Note some pages are always handled as late kill.
+
+User control
+============
+
+vm.memory_failure_recovery
+	See sysctl.txt
+
+vm.memory_failure_early_kill
+	Enable early kill mode globally
+
+PR_MCE_KILL
+	Set early/late kill mode/revert to system default
+
+	arg1: PR_MCE_KILL_CLEAR:
+		Revert to system default
+	arg1: PR_MCE_KILL_SET:
+		arg2 defines thread specific mode
+
+		PR_MCE_KILL_EARLY:
+			Early kill
+		PR_MCE_KILL_LATE:
+			Late kill
+		PR_MCE_KILL_DEFAULT
+			Use system global default
+
+	Note that if you want to have a dedicated thread which handles
+	the SIGBUS(BUS_MCEERR_AO) on behalf of the process, you should
+	call prctl(PR_MCE_KILL_EARLY) on the designated thread. Otherwise,
+	the SIGBUS is sent to the main thread.
+
+PR_MCE_KILL_GET
+	return current mode
+
+Testing
+=======
+
+* madvise(MADV_HWPOISON, ....) (as root) - Poison a page in the
+  process for testing
+
+* hwpoison-inject module through debugfs ``/sys/kernel/debug/hwpoison/``
+
+  corrupt-pfn
+	Inject hwpoison fault at PFN echoed into this file. This does
+	some early filtering to avoid corrupted unintended pages in test suites.
+
+  unpoison-pfn
+	Software-unpoison page at PFN echoed into this file. This way
+	a page can be reused again.  This only works for Linux
+	injected failures, not for real memory failures.
+
+  Note these injection interfaces are not stable and might change between
+  kernel versions
+
+  corrupt-filter-dev-major, corrupt-filter-dev-minor
+	Only handle memory failures to pages associated with the file
+	system defined by block device major/minor.  -1U is the
+	wildcard value.  This should be only used for testing with
+	artificial injection.
+
+  corrupt-filter-memcg
+	Limit injection to pages owned by memgroup. Specified by inode
+	number of the memcg.
+
+	Example::
+
+		mkdir /sys/fs/cgroup/mem/hwpoison
+
+	        usemem -m 100 -s 1000 &
+		echo `jobs -p` > /sys/fs/cgroup/mem/hwpoison/tasks
+
+		memcg_ino=$(ls -id /sys/fs/cgroup/mem/hwpoison | cut -f1 -d' ')
+		echo $memcg_ino > /debug/hwpoison/corrupt-filter-memcg
+
+		page-types -p `pidof init`   --hwpoison  # shall do nothing
+		page-types -p `pidof usemem` --hwpoison  # poison its pages
+
+  corrupt-filter-flags-mask, corrupt-filter-flags-value
+	When specified, only poison pages if ((page_flags & mask) ==
+	value).  This allows stress testing of many kinds of
+	pages. The page_flags are the same as in /proc/kpageflags. The
+	flag bits are defined in include/linux/kernel-page-flags.h and
+	documented in Documentation/admin-guide/mm/pagemap.rst
+
+* Architecture specific MCE injector
+
+  x86 has mce-inject, mce-test
+
+  Some portable hwpoison test programs in mce-test, see below.
+
+References
+==========
+
+http://halobates.de/mce-lc09-2.pdf
+	Overview presentation from LinuxCon 09
+
+git://git.kernel.org/pub/scm/utils/cpu/mce/mce-test.git
+	Test suite (hwpoison specific portable tests in tsrc)
+
+git://git.kernel.org/pub/scm/utils/cpu/mce/mce-inject.git
+	x86 specific injector
+
+
+Limitations
+===========
+- Not all page types are supported and never will. Most kernel internal
+  objects cannot be recovered, only LRU pages for now.
+- Right now hugepage support is missing.
+
+---
+Andi Kleen, Oct 2009
diff --git a/Documentation/vm/hwpoison.txt b/Documentation/vm/hwpoison.txt
deleted file mode 100644
index e912d7eee769..000000000000
--- a/Documentation/vm/hwpoison.txt
+++ /dev/null
@@ -1,187 +0,0 @@
-What is hwpoison?
-
-Upcoming Intel CPUs have support for recovering from some memory errors
-(``MCA recovery''). This requires the OS to declare a page "poisoned",
-kill the processes associated with it and avoid using it in the future.
-
-This patchkit implements the necessary infrastructure in the VM.
-
-To quote the overview comment:
-
- * High level machine check handler. Handles pages reported by the
- * hardware as being corrupted usually due to a 2bit ECC memory or cache
- * failure.
- *
- * This focusses on pages detected as corrupted in the background.
- * When the current CPU tries to consume corruption the currently
- * running process can just be killed directly instead. This implies
- * that if the error cannot be handled for some reason it's safe to
- * just ignore it because no corruption has been consumed yet. Instead
- * when that happens another machine check will happen.
- *
- * Handles page cache pages in various states. The tricky part
- * here is that we can access any page asynchronous to other VM
- * users, because memory failures could happen anytime and anywhere,
- * possibly violating some of their assumptions. This is why this code
- * has to be extremely careful. Generally it tries to use normal locking
- * rules, as in get the standard locks, even if that means the
- * error handling takes potentially a long time.
- *
- * Some of the operations here are somewhat inefficient and have non
- * linear algorithmic complexity, because the data structures have not
- * been optimized for this case. This is in particular the case
- * for the mapping from a vma to a process. Since this case is expected
- * to be rare we hope we can get away with this.
-
-The code consists of a the high level handler in mm/memory-failure.c,
-a new page poison bit and various checks in the VM to handle poisoned
-pages.
-
-The main target right now is KVM guests, but it works for all kinds
-of applications. KVM support requires a recent qemu-kvm release.
-
-For the KVM use there was need for a new signal type so that
-KVM can inject the machine check into the guest with the proper
-address. This in theory allows other applications to handle
-memory failures too. The expection is that near all applications
-won't do that, but some very specialized ones might.
-
----
-
-There are two (actually three) modi memory failure recovery can be in:
-
-vm.memory_failure_recovery sysctl set to zero:
-	All memory failures cause a panic. Do not attempt recovery.
-	(on x86 this can be also affected by the tolerant level of the
-	MCE subsystem)
-
-early kill
-	(can be controlled globally and per process)
-	Send SIGBUS to the application as soon as the error is detected
-	This allows applications who can process memory errors in a gentle
-	way (e.g. drop affected object)
-	This is the mode used by KVM qemu.
-
-late kill
-	Send SIGBUS when the application runs into the corrupted page.
-	This is best for memory error unaware applications and default
-	Note some pages are always handled as late kill.
-
----
-
-User control:
-
-vm.memory_failure_recovery
-	See sysctl.txt
-
-vm.memory_failure_early_kill
-	Enable early kill mode globally
-
-PR_MCE_KILL
-	Set early/late kill mode/revert to system default
-	arg1: PR_MCE_KILL_CLEAR: Revert to system default
-	arg1: PR_MCE_KILL_SET: arg2 defines thread specific mode
-		PR_MCE_KILL_EARLY: Early kill
-		PR_MCE_KILL_LATE:  Late kill
-		PR_MCE_KILL_DEFAULT: Use system global default
-	Note that if you want to have a dedicated thread which handles
-	the SIGBUS(BUS_MCEERR_AO) on behalf of the process, you should
-	call prctl(PR_MCE_KILL_EARLY) on the designated thread. Otherwise,
-	the SIGBUS is sent to the main thread.
-
-PR_MCE_KILL_GET
-	return current mode
-
-
----
-
-Testing:
-
-madvise(MADV_HWPOISON, ....)
-	(as root)
-	Poison a page in the process for testing
-
-
-hwpoison-inject module through debugfs
-
-/sys/kernel/debug/hwpoison/
-
-corrupt-pfn
-
-Inject hwpoison fault at PFN echoed into this file. This does
-some early filtering to avoid corrupted unintended pages in test suites.
-
-unpoison-pfn
-
-Software-unpoison page at PFN echoed into this file. This
-way a page can be reused again.
-This only works for Linux injected failures, not for real
-memory failures.
-
-Note these injection interfaces are not stable and might change between
-kernel versions
-
-corrupt-filter-dev-major
-corrupt-filter-dev-minor
-
-Only handle memory failures to pages associated with the file system defined
-by block device major/minor.  -1U is the wildcard value.
-This should be only used for testing with artificial injection.
-
-corrupt-filter-memcg
-
-Limit injection to pages owned by memgroup. Specified by inode number
-of the memcg.
-
-Example:
-        mkdir /sys/fs/cgroup/mem/hwpoison
-
-        usemem -m 100 -s 1000 &
-        echo `jobs -p` > /sys/fs/cgroup/mem/hwpoison/tasks
-
-        memcg_ino=$(ls -id /sys/fs/cgroup/mem/hwpoison | cut -f1 -d' ')
-        echo $memcg_ino > /debug/hwpoison/corrupt-filter-memcg
-
-        page-types -p `pidof init`   --hwpoison  # shall do nothing
-        page-types -p `pidof usemem` --hwpoison  # poison its pages
-
-corrupt-filter-flags-mask
-corrupt-filter-flags-value
-
-When specified, only poison pages if ((page_flags & mask) == value).
-This allows stress testing of many kinds of pages. The page_flags
-are the same as in /proc/kpageflags. The flag bits are defined in
-include/linux/kernel-page-flags.h and documented in
-Documentation/vm/pagemap.txt
-
-Architecture specific MCE injector
-
-x86 has mce-inject, mce-test
-
-Some portable hwpoison test programs in mce-test, see blow.
-
----
-
-References:
-
-http://halobates.de/mce-lc09-2.pdf
-	Overview presentation from LinuxCon 09
-
-git://git.kernel.org/pub/scm/utils/cpu/mce/mce-test.git
-	Test suite (hwpoison specific portable tests in tsrc)
-
-git://git.kernel.org/pub/scm/utils/cpu/mce/mce-inject.git
-	x86 specific injector
-
-
----
-
-Limitations:
-
-- Not all page types are supported and never will. Most kernel internal
-objects cannot be recovered, only LRU pages for now.
-- Right now hugepage support is missing.
-
----
-Andi Kleen, Oct 2009
-
diff --git a/Documentation/vm/idle_page_tracking.txt b/Documentation/vm/idle_page_tracking.txt
deleted file mode 100644
index 85dcc3bb85dc..000000000000
--- a/Documentation/vm/idle_page_tracking.txt
+++ /dev/null
@@ -1,98 +0,0 @@
-MOTIVATION
-
-The idle page tracking feature allows to track which memory pages are being
-accessed by a workload and which are idle. This information can be useful for
-estimating the workload's working set size, which, in turn, can be taken into
-account when configuring the workload parameters, setting memory cgroup limits,
-or deciding where to place the workload within a compute cluster.
-
-It is enabled by CONFIG_IDLE_PAGE_TRACKING=y.
-
-USER API
-
-The idle page tracking API is located at /sys/kernel/mm/page_idle. Currently,
-it consists of the only read-write file, /sys/kernel/mm/page_idle/bitmap.
-
-The file implements a bitmap where each bit corresponds to a memory page. The
-bitmap is represented by an array of 8-byte integers, and the page at PFN #i is
-mapped to bit #i%64 of array element #i/64, byte order is native. When a bit is
-set, the corresponding page is idle.
-
-A page is considered idle if it has not been accessed since it was marked idle
-(for more details on what "accessed" actually means see the IMPLEMENTATION
-DETAILS section). To mark a page idle one has to set the bit corresponding to
-the page by writing to the file. A value written to the file is OR-ed with the
-current bitmap value.
-
-Only accesses to user memory pages are tracked. These are pages mapped to a
-process address space, page cache and buffer pages, swap cache pages. For other
-page types (e.g. SLAB pages) an attempt to mark a page idle is silently ignored,
-and hence such pages are never reported idle.
-
-For huge pages the idle flag is set only on the head page, so one has to read
-/proc/kpageflags in order to correctly count idle huge pages.
-
-Reading from or writing to /sys/kernel/mm/page_idle/bitmap will return
--EINVAL if you are not starting the read/write on an 8-byte boundary, or
-if the size of the read/write is not a multiple of 8 bytes. Writing to
-this file beyond max PFN will return -ENXIO.
-
-That said, in order to estimate the amount of pages that are not used by a
-workload one should:
-
- 1. Mark all the workload's pages as idle by setting corresponding bits in
-    /sys/kernel/mm/page_idle/bitmap. The pages can be found by reading
-    /proc/pid/pagemap if the workload is represented by a process, or by
-    filtering out alien pages using /proc/kpagecgroup in case the workload is
-    placed in a memory cgroup.
-
- 2. Wait until the workload accesses its working set.
-
- 3. Read /sys/kernel/mm/page_idle/bitmap and count the number of bits set. If
-    one wants to ignore certain types of pages, e.g. mlocked pages since they
-    are not reclaimable, he or she can filter them out using /proc/kpageflags.
-
-See Documentation/vm/pagemap.txt for more information about /proc/pid/pagemap,
-/proc/kpageflags, and /proc/kpagecgroup.
-
-IMPLEMENTATION DETAILS
-
-The kernel internally keeps track of accesses to user memory pages in order to
-reclaim unreferenced pages first on memory shortage conditions. A page is
-considered referenced if it has been recently accessed via a process address
-space, in which case one or more PTEs it is mapped to will have the Accessed bit
-set, or marked accessed explicitly by the kernel (see mark_page_accessed()). The
-latter happens when:
-
- - a userspace process reads or writes a page using a system call (e.g. read(2)
-   or write(2))
-
- - a page that is used for storing filesystem buffers is read or written,
-   because a process needs filesystem metadata stored in it (e.g. lists a
-   directory tree)
-
- - a page is accessed by a device driver using get_user_pages()
-
-When a dirty page is written to swap or disk as a result of memory reclaim or
-exceeding the dirty memory limit, it is not marked referenced.
-
-The idle memory tracking feature adds a new page flag, the Idle flag. This flag
-is set manually, by writing to /sys/kernel/mm/page_idle/bitmap (see the USER API
-section), and cleared automatically whenever a page is referenced as defined
-above.
-
-When a page is marked idle, the Accessed bit must be cleared in all PTEs it is
-mapped to, otherwise we will not be able to detect accesses to the page coming
-from a process address space. To avoid interference with the reclaimer, which,
-as noted above, uses the Accessed bit to promote actively referenced pages, one
-more page flag is introduced, the Young flag. When the PTE Accessed bit is
-cleared as a result of setting or updating a page's Idle flag, the Young flag
-is set on the page. The reclaimer treats the Young flag as an extra PTE
-Accessed bit and therefore will consider such a page as referenced.
-
-Since the idle memory tracking feature is based on the memory reclaimer logic,
-it only works with pages that are on an LRU list, other pages are silently
-ignored. That means it will ignore a user memory page if it is isolated, but
-since there are usually not many of them, it should not affect the overall
-result noticeably. In order not to stall scanning of the idle page bitmap,
-locked pages may be skipped too.
diff --git a/Documentation/vm/index.rst b/Documentation/vm/index.rst
new file mode 100644
index 000000000000..c4ded22197ca
--- /dev/null
+++ b/Documentation/vm/index.rst
@@ -0,0 +1,50 @@
+=====================================
+Linux Memory Management Documentation
+=====================================
+
+This is a collection of documents about Linux memory management (mm) subsystem.
+
+User guides for MM features
+===========================
+
+The following documents provide guides for controlling and tuning
+various features of the Linux memory management
+
+.. toctree::
+   :maxdepth: 1
+
+   swap_numa
+   zswap
+
+Kernel developers MM documentation
+==================================
+
+The below documents describe MM internals with different level of
+details ranging from notes and mailing list responses to elaborate
+descriptions of data structures and algorithms.
+
+.. toctree::
+   :maxdepth: 1
+
+   active_mm
+   balance
+   cleancache
+   frontswap
+   highmem
+   hmm
+   hwpoison
+   hugetlbfs_reserv
+   ksm
+   mmu_notifier
+   numa
+   overcommit-accounting
+   page_migration
+   page_frags
+   page_owner
+   remap_file_pages
+   slub
+   split_page_table_lock
+   transhuge
+   unevictable-lru
+   z3fold
+   zsmalloc
diff --git a/Documentation/vm/ksm.rst b/Documentation/vm/ksm.rst
new file mode 100644
index 000000000000..d32016d9be2c
--- /dev/null
+++ b/Documentation/vm/ksm.rst
@@ -0,0 +1,87 @@
+.. _ksm:
+
+=======================
+Kernel Samepage Merging
+=======================
+
+KSM is a memory-saving de-duplication feature, enabled by CONFIG_KSM=y,
+added to the Linux kernel in 2.6.32.  See ``mm/ksm.c`` for its implementation,
+and http://lwn.net/Articles/306704/ and http://lwn.net/Articles/330589/
+
+The userspace interface of KSM is described in :ref:`Documentation/admin-guide/mm/ksm.rst <admin_guide_ksm>`
+
+Design
+======
+
+Overview
+--------
+
+.. kernel-doc:: mm/ksm.c
+   :DOC: Overview
+
+Reverse mapping
+---------------
+KSM maintains reverse mapping information for KSM pages in the stable
+tree.
+
+If a KSM page is shared between less than ``max_page_sharing`` VMAs,
+the node of the stable tree that represents such KSM page points to a
+list of :c:type:`struct rmap_item` and the ``page->mapping`` of the
+KSM page points to the stable tree node.
+
+When the sharing passes this threshold, KSM adds a second dimension to
+the stable tree. The tree node becomes a "chain" that links one or
+more "dups". Each "dup" keeps reverse mapping information for a KSM
+page with ``page->mapping`` pointing to that "dup".
+
+Every "chain" and all "dups" linked into a "chain" enforce the
+invariant that they represent the same write protected memory content,
+even if each "dup" will be pointed by a different KSM page copy of
+that content.
+
+This way the stable tree lookup computational complexity is unaffected
+if compared to an unlimited list of reverse mappings. It is still
+enforced that there cannot be KSM page content duplicates in the
+stable tree itself.
+
+The deduplication limit enforced by ``max_page_sharing`` is required
+to avoid the virtual memory rmap lists to grow too large. The rmap
+walk has O(N) complexity where N is the number of rmap_items
+(i.e. virtual mappings) that are sharing the page, which is in turn
+capped by ``max_page_sharing``. So this effectively spreads the linear
+O(N) computational complexity from rmap walk context over different
+KSM pages. The ksmd walk over the stable_node "chains" is also O(N),
+but N is the number of stable_node "dups", not the number of
+rmap_items, so it has not a significant impact on ksmd performance. In
+practice the best stable_node "dup" candidate will be kept and found
+at the head of the "dups" list.
+
+High values of ``max_page_sharing`` result in faster memory merging
+(because there will be fewer stable_node dups queued into the
+stable_node chain->hlist to check for pruning) and higher
+deduplication factor at the expense of slower worst case for rmap
+walks for any KSM page which can happen during swapping, compaction,
+NUMA balancing and page migration.
+
+The ``stable_node_dups/stable_node_chains`` ratio is also affected by the
+``max_page_sharing`` tunable, and an high ratio may indicate fragmentation
+in the stable_node dups, which could be solved by introducing
+fragmentation algorithms in ksmd which would refile rmap_items from
+one stable_node dup to another stable_node dup, in order to free up
+stable_node "dups" with few rmap_items in them, but that may increase
+the ksmd CPU usage and possibly slowdown the readonly computations on
+the KSM pages of the applications.
+
+The whole list of stable_node "dups" linked in the stable_node
+"chains" is scanned periodically in order to prune stale stable_nodes.
+The frequency of such scans is defined by
+``stable_node_chains_prune_millisecs`` sysfs tunable.
+
+Reference
+---------
+.. kernel-doc:: mm/ksm.c
+   :functions: mm_slot ksm_scan stable_node rmap_item
+
+--
+Izik Eidus,
+Hugh Dickins, 17 Nov 2009
diff --git a/Documentation/vm/ksm.txt b/Documentation/vm/ksm.txt
deleted file mode 100644
index 6686bd267dc9..000000000000
--- a/Documentation/vm/ksm.txt
+++ /dev/null
@@ -1,178 +0,0 @@
-How to use the Kernel Samepage Merging feature
-----------------------------------------------
-
-KSM is a memory-saving de-duplication feature, enabled by CONFIG_KSM=y,
-added to the Linux kernel in 2.6.32.  See mm/ksm.c for its implementation,
-and http://lwn.net/Articles/306704/ and http://lwn.net/Articles/330589/
-
-The KSM daemon ksmd periodically scans those areas of user memory which
-have been registered with it, looking for pages of identical content which
-can be replaced by a single write-protected page (which is automatically
-copied if a process later wants to update its content).
-
-KSM was originally developed for use with KVM (where it was known as
-Kernel Shared Memory), to fit more virtual machines into physical memory,
-by sharing the data common between them.  But it can be useful to any
-application which generates many instances of the same data.
-
-KSM only merges anonymous (private) pages, never pagecache (file) pages.
-KSM's merged pages were originally locked into kernel memory, but can now
-be swapped out just like other user pages (but sharing is broken when they
-are swapped back in: ksmd must rediscover their identity and merge again).
-
-KSM only operates on those areas of address space which an application
-has advised to be likely candidates for merging, by using the madvise(2)
-system call: int madvise(addr, length, MADV_MERGEABLE).
-
-The app may call int madvise(addr, length, MADV_UNMERGEABLE) to cancel
-that advice and restore unshared pages: whereupon KSM unmerges whatever
-it merged in that range.  Note: this unmerging call may suddenly require
-more memory than is available - possibly failing with EAGAIN, but more
-probably arousing the Out-Of-Memory killer.
-
-If KSM is not configured into the running kernel, madvise MADV_MERGEABLE
-and MADV_UNMERGEABLE simply fail with EINVAL.  If the running kernel was
-built with CONFIG_KSM=y, those calls will normally succeed: even if the
-the KSM daemon is not currently running, MADV_MERGEABLE still registers
-the range for whenever the KSM daemon is started; even if the range
-cannot contain any pages which KSM could actually merge; even if
-MADV_UNMERGEABLE is applied to a range which was never MADV_MERGEABLE.
-
-If a region of memory must be split into at least one new MADV_MERGEABLE
-or MADV_UNMERGEABLE region, the madvise may return ENOMEM if the process
-will exceed vm.max_map_count (see Documentation/sysctl/vm.txt).
-
-Like other madvise calls, they are intended for use on mapped areas of
-the user address space: they will report ENOMEM if the specified range
-includes unmapped gaps (though working on the intervening mapped areas),
-and might fail with EAGAIN if not enough memory for internal structures.
-
-Applications should be considerate in their use of MADV_MERGEABLE,
-restricting its use to areas likely to benefit.  KSM's scans may use a lot
-of processing power: some installations will disable KSM for that reason.
-
-The KSM daemon is controlled by sysfs files in /sys/kernel/mm/ksm/,
-readable by all but writable only by root:
-
-pages_to_scan    - how many present pages to scan before ksmd goes to sleep
-                   e.g. "echo 100 > /sys/kernel/mm/ksm/pages_to_scan"
-                   Default: 100 (chosen for demonstration purposes)
-
-sleep_millisecs  - how many milliseconds ksmd should sleep before next scan
-                   e.g. "echo 20 > /sys/kernel/mm/ksm/sleep_millisecs"
-                   Default: 20 (chosen for demonstration purposes)
-
-merge_across_nodes - specifies if pages from different numa nodes can be merged.
-                   When set to 0, ksm merges only pages which physically
-                   reside in the memory area of same NUMA node. That brings
-                   lower latency to access of shared pages. Systems with more
-                   nodes, at significant NUMA distances, are likely to benefit
-                   from the lower latency of setting 0. Smaller systems, which
-                   need to minimize memory usage, are likely to benefit from
-                   the greater sharing of setting 1 (default). You may wish to
-                   compare how your system performs under each setting, before
-                   deciding on which to use. merge_across_nodes setting can be
-                   changed only when there are no ksm shared pages in system:
-                   set run 2 to unmerge pages first, then to 1 after changing
-                   merge_across_nodes, to remerge according to the new setting.
-                   Default: 1 (merging across nodes as in earlier releases)
-
-run              - set 0 to stop ksmd from running but keep merged pages,
-                   set 1 to run ksmd e.g. "echo 1 > /sys/kernel/mm/ksm/run",
-                   set 2 to stop ksmd and unmerge all pages currently merged,
-                         but leave mergeable areas registered for next run
-                   Default: 0 (must be changed to 1 to activate KSM,
-                               except if CONFIG_SYSFS is disabled)
-
-use_zero_pages   - specifies whether empty pages (i.e. allocated pages
-                   that only contain zeroes) should be treated specially.
-                   When set to 1, empty pages are merged with the kernel
-                   zero page(s) instead of with each other as it would
-                   happen normally. This can improve the performance on
-                   architectures with coloured zero pages, depending on
-                   the workload. Care should be taken when enabling this
-                   setting, as it can potentially degrade the performance
-                   of KSM for some workloads, for example if the checksums
-                   of pages candidate for merging match the checksum of
-                   an empty page. This setting can be changed at any time,
-                   it is only effective for pages merged after the change.
-                   Default: 0 (normal KSM behaviour as in earlier releases)
-
-max_page_sharing - Maximum sharing allowed for each KSM page. This
-                   enforces a deduplication limit to avoid the virtual
-                   memory rmap lists to grow too large. The minimum
-                   value is 2 as a newly created KSM page will have at
-                   least two sharers. The rmap walk has O(N)
-                   complexity where N is the number of rmap_items
-                   (i.e. virtual mappings) that are sharing the page,
-                   which is in turn capped by max_page_sharing. So
-                   this effectively spread the the linear O(N)
-                   computational complexity from rmap walk context
-                   over different KSM pages. The ksmd walk over the
-                   stable_node "chains" is also O(N), but N is the
-                   number of stable_node "dups", not the number of
-                   rmap_items, so it has not a significant impact on
-                   ksmd performance. In practice the best stable_node
-                   "dup" candidate will be kept and found at the head
-                   of the "dups" list. The higher this value the
-                   faster KSM will merge the memory (because there
-                   will be fewer stable_node dups queued into the
-                   stable_node chain->hlist to check for pruning) and
-                   the higher the deduplication factor will be, but
-                   the slowest the worst case rmap walk could be for
-                   any given KSM page. Slowing down the rmap_walk
-                   means there will be higher latency for certain
-                   virtual memory operations happening during
-                   swapping, compaction, NUMA balancing and page
-                   migration, in turn decreasing responsiveness for
-                   the caller of those virtual memory operations. The
-                   scheduler latency of other tasks not involved with
-                   the VM operations doing the rmap walk is not
-                   affected by this parameter as the rmap walks are
-                   always schedule friendly themselves.
-
-stable_node_chains_prune_millisecs - How frequently to walk the whole
-                   list of stable_node "dups" linked in the
-                   stable_node "chains" in order to prune stale
-                   stable_nodes. Smaller milllisecs values will free
-                   up the KSM metadata with lower latency, but they
-                   will make ksmd use more CPU during the scan. This
-                   only applies to the stable_node chains so it's a
-                   noop if not a single KSM page hit the
-                   max_page_sharing yet (there would be no stable_node
-                   chains in such case).
-
-The effectiveness of KSM and MADV_MERGEABLE is shown in /sys/kernel/mm/ksm/:
-
-pages_shared     - how many shared pages are being used
-pages_sharing    - how many more sites are sharing them i.e. how much saved
-pages_unshared   - how many pages unique but repeatedly checked for merging
-pages_volatile   - how many pages changing too fast to be placed in a tree
-full_scans       - how many times all mergeable areas have been scanned
-
-stable_node_chains - number of stable node chains allocated, this is
-		     effectively the number of KSM pages that hit the
-		     max_page_sharing limit
-stable_node_dups   - number of stable node dups queued into the
-		     stable_node chains
-
-A high ratio of pages_sharing to pages_shared indicates good sharing, but
-a high ratio of pages_unshared to pages_sharing indicates wasted effort.
-pages_volatile embraces several different kinds of activity, but a high
-proportion there would also indicate poor use of madvise MADV_MERGEABLE.
-
-The maximum possible page_sharing/page_shared ratio is limited by the
-max_page_sharing tunable. To increase the ratio max_page_sharing must
-be increased accordingly.
-
-The stable_node_dups/stable_node_chains ratio is also affected by the
-max_page_sharing tunable, and an high ratio may indicate fragmentation
-in the stable_node dups, which could be solved by introducing
-fragmentation algorithms in ksmd which would refile rmap_items from
-one stable_node dup to another stable_node dup, in order to freeup
-stable_node "dups" with few rmap_items in them, but that may increase
-the ksmd CPU usage and possibly slowdown the readonly computations on
-the KSM pages of the applications.
-
-Izik Eidus,
-Hugh Dickins, 17 Nov 2009
diff --git a/Documentation/vm/mmu_notifier.rst b/Documentation/vm/mmu_notifier.rst
new file mode 100644
index 000000000000..47baa1cf28c5
--- /dev/null
+++ b/Documentation/vm/mmu_notifier.rst
@@ -0,0 +1,99 @@
+.. _mmu_notifier:
+
+When do you need to notify inside page table lock ?
+===================================================
+
+When clearing a pte/pmd we are given a choice to notify the event through
+(notify version of \*_clear_flush call mmu_notifier_invalidate_range) under
+the page table lock. But that notification is not necessary in all cases.
+
+For secondary TLB (non CPU TLB) like IOMMU TLB or device TLB (when device use
+thing like ATS/PASID to get the IOMMU to walk the CPU page table to access a
+process virtual address space). There is only 2 cases when you need to notify
+those secondary TLB while holding page table lock when clearing a pte/pmd:
+
+  A) page backing address is free before mmu_notifier_invalidate_range_end()
+  B) a page table entry is updated to point to a new page (COW, write fault
+     on zero page, __replace_page(), ...)
+
+Case A is obvious you do not want to take the risk for the device to write to
+a page that might now be used by some completely different task.
+
+Case B is more subtle. For correctness it requires the following sequence to
+happen:
+
+  - take page table lock
+  - clear page table entry and notify ([pmd/pte]p_huge_clear_flush_notify())
+  - set page table entry to point to new page
+
+If clearing the page table entry is not followed by a notify before setting
+the new pte/pmd value then you can break memory model like C11 or C++11 for
+the device.
+
+Consider the following scenario (device use a feature similar to ATS/PASID):
+
+Two address addrA and addrB such that \|addrA - addrB\| >= PAGE_SIZE we assume
+they are write protected for COW (other case of B apply too).
+
+::
+
+ [Time N] --------------------------------------------------------------------
+ CPU-thread-0  {try to write to addrA}
+ CPU-thread-1  {try to write to addrB}
+ CPU-thread-2  {}
+ CPU-thread-3  {}
+ DEV-thread-0  {read addrA and populate device TLB}
+ DEV-thread-2  {read addrB and populate device TLB}
+ [Time N+1] ------------------------------------------------------------------
+ CPU-thread-0  {COW_step0: {mmu_notifier_invalidate_range_start(addrA)}}
+ CPU-thread-1  {COW_step0: {mmu_notifier_invalidate_range_start(addrB)}}
+ CPU-thread-2  {}
+ CPU-thread-3  {}
+ DEV-thread-0  {}
+ DEV-thread-2  {}
+ [Time N+2] ------------------------------------------------------------------
+ CPU-thread-0  {COW_step1: {update page table to point to new page for addrA}}
+ CPU-thread-1  {COW_step1: {update page table to point to new page for addrB}}
+ CPU-thread-2  {}
+ CPU-thread-3  {}
+ DEV-thread-0  {}
+ DEV-thread-2  {}
+ [Time N+3] ------------------------------------------------------------------
+ CPU-thread-0  {preempted}
+ CPU-thread-1  {preempted}
+ CPU-thread-2  {write to addrA which is a write to new page}
+ CPU-thread-3  {}
+ DEV-thread-0  {}
+ DEV-thread-2  {}
+ [Time N+3] ------------------------------------------------------------------
+ CPU-thread-0  {preempted}
+ CPU-thread-1  {preempted}
+ CPU-thread-2  {}
+ CPU-thread-3  {write to addrB which is a write to new page}
+ DEV-thread-0  {}
+ DEV-thread-2  {}
+ [Time N+4] ------------------------------------------------------------------
+ CPU-thread-0  {preempted}
+ CPU-thread-1  {COW_step3: {mmu_notifier_invalidate_range_end(addrB)}}
+ CPU-thread-2  {}
+ CPU-thread-3  {}
+ DEV-thread-0  {}
+ DEV-thread-2  {}
+ [Time N+5] ------------------------------------------------------------------
+ CPU-thread-0  {preempted}
+ CPU-thread-1  {}
+ CPU-thread-2  {}
+ CPU-thread-3  {}
+ DEV-thread-0  {read addrA from old page}
+ DEV-thread-2  {read addrB from new page}
+
+So here because at time N+2 the clear page table entry was not pair with a
+notification to invalidate the secondary TLB, the device see the new value for
+addrB before seing the new value for addrA. This break total memory ordering
+for the device.
+
+When changing a pte to write protect or to point to a new write protected page
+with same content (KSM) it is fine to delay the mmu_notifier_invalidate_range
+call to mmu_notifier_invalidate_range_end() outside the page table lock. This
+is true even if the thread doing the page table update is preempted right after
+releasing page table lock but before call mmu_notifier_invalidate_range_end().
diff --git a/Documentation/vm/mmu_notifier.txt b/Documentation/vm/mmu_notifier.txt
deleted file mode 100644
index 23b462566bb7..000000000000
--- a/Documentation/vm/mmu_notifier.txt
+++ /dev/null
@@ -1,93 +0,0 @@
-When do you need to notify inside page table lock ?
-
-When clearing a pte/pmd we are given a choice to notify the event through
-(notify version of *_clear_flush call mmu_notifier_invalidate_range) under
-the page table lock. But that notification is not necessary in all cases.
-
-For secondary TLB (non CPU TLB) like IOMMU TLB or device TLB (when device use
-thing like ATS/PASID to get the IOMMU to walk the CPU page table to access a
-process virtual address space). There is only 2 cases when you need to notify
-those secondary TLB while holding page table lock when clearing a pte/pmd:
-
-  A) page backing address is free before mmu_notifier_invalidate_range_end()
-  B) a page table entry is updated to point to a new page (COW, write fault
-     on zero page, __replace_page(), ...)
-
-Case A is obvious you do not want to take the risk for the device to write to
-a page that might now be used by some completely different task.
-
-Case B is more subtle. For correctness it requires the following sequence to
-happen:
-  - take page table lock
-  - clear page table entry and notify ([pmd/pte]p_huge_clear_flush_notify())
-  - set page table entry to point to new page
-
-If clearing the page table entry is not followed by a notify before setting
-the new pte/pmd value then you can break memory model like C11 or C++11 for
-the device.
-
-Consider the following scenario (device use a feature similar to ATS/PASID):
-
-Two address addrA and addrB such that |addrA - addrB| >= PAGE_SIZE we assume
-they are write protected for COW (other case of B apply too).
-
-[Time N] --------------------------------------------------------------------
-CPU-thread-0  {try to write to addrA}
-CPU-thread-1  {try to write to addrB}
-CPU-thread-2  {}
-CPU-thread-3  {}
-DEV-thread-0  {read addrA and populate device TLB}
-DEV-thread-2  {read addrB and populate device TLB}
-[Time N+1] ------------------------------------------------------------------
-CPU-thread-0  {COW_step0: {mmu_notifier_invalidate_range_start(addrA)}}
-CPU-thread-1  {COW_step0: {mmu_notifier_invalidate_range_start(addrB)}}
-CPU-thread-2  {}
-CPU-thread-3  {}
-DEV-thread-0  {}
-DEV-thread-2  {}
-[Time N+2] ------------------------------------------------------------------
-CPU-thread-0  {COW_step1: {update page table to point to new page for addrA}}
-CPU-thread-1  {COW_step1: {update page table to point to new page for addrB}}
-CPU-thread-2  {}
-CPU-thread-3  {}
-DEV-thread-0  {}
-DEV-thread-2  {}
-[Time N+3] ------------------------------------------------------------------
-CPU-thread-0  {preempted}
-CPU-thread-1  {preempted}
-CPU-thread-2  {write to addrA which is a write to new page}
-CPU-thread-3  {}
-DEV-thread-0  {}
-DEV-thread-2  {}
-[Time N+3] ------------------------------------------------------------------
-CPU-thread-0  {preempted}
-CPU-thread-1  {preempted}
-CPU-thread-2  {}
-CPU-thread-3  {write to addrB which is a write to new page}
-DEV-thread-0  {}
-DEV-thread-2  {}
-[Time N+4] ------------------------------------------------------------------
-CPU-thread-0  {preempted}
-CPU-thread-1  {COW_step3: {mmu_notifier_invalidate_range_end(addrB)}}
-CPU-thread-2  {}
-CPU-thread-3  {}
-DEV-thread-0  {}
-DEV-thread-2  {}
-[Time N+5] ------------------------------------------------------------------
-CPU-thread-0  {preempted}
-CPU-thread-1  {}
-CPU-thread-2  {}
-CPU-thread-3  {}
-DEV-thread-0  {read addrA from old page}
-DEV-thread-2  {read addrB from new page}
-
-So here because at time N+2 the clear page table entry was not pair with a
-notification to invalidate the secondary TLB, the device see the new value for
-addrB before seing the new value for addrA. This break total memory ordering
-for the device.
-
-When changing a pte to write protect or to point to a new write protected page
-with same content (KSM) it is fine to delay the mmu_notifier_invalidate_range
-call to mmu_notifier_invalidate_range_end() outside the page table lock. This
-is true even if the thread doing the page table update is preempted right after
-releasing page table lock but before call mmu_notifier_invalidate_range_end().
diff --git a/Documentation/vm/numa b/Documentation/vm/numa
deleted file mode 100644
index a31b85b9bb88..000000000000
--- a/Documentation/vm/numa
+++ /dev/null
@@ -1,146 +0,0 @@
-Started Nov 1999 by Kanoj Sarcar <kanoj@sgi.com>
-
-What is NUMA?
-
-This question can be answered from a couple of perspectives:  the
-hardware view and the Linux software view.
-
-From the hardware perspective, a NUMA system is a computer platform that
-comprises multiple components or assemblies each of which may contain 0
-or more CPUs, local memory, and/or IO buses.  For brevity and to
-disambiguate the hardware view of these physical components/assemblies
-from the software abstraction thereof, we'll call the components/assemblies
-'cells' in this document.
-
-Each of the 'cells' may be viewed as an SMP [symmetric multi-processor] subset
-of the system--although some components necessary for a stand-alone SMP system
-may not be populated on any given cell.   The cells of the NUMA system are
-connected together with some sort of system interconnect--e.g., a crossbar or
-point-to-point link are common types of NUMA system interconnects.  Both of
-these types of interconnects can be aggregated to create NUMA platforms with
-cells at multiple distances from other cells.
-
-For Linux, the NUMA platforms of interest are primarily what is known as Cache
-Coherent NUMA or ccNUMA systems.   With ccNUMA systems, all memory is visible
-to and accessible from any CPU attached to any cell and cache coherency
-is handled in hardware by the processor caches and/or the system interconnect.
-
-Memory access time and effective memory bandwidth varies depending on how far
-away the cell containing the CPU or IO bus making the memory access is from the
-cell containing the target memory.  For example, access to memory by CPUs
-attached to the same cell will experience faster access times and higher
-bandwidths than accesses to memory on other, remote cells.  NUMA platforms
-can have cells at multiple remote distances from any given cell.
-
-Platform vendors don't build NUMA systems just to make software developers'
-lives interesting.  Rather, this architecture is a means to provide scalable
-memory bandwidth.  However, to achieve scalable memory bandwidth, system and
-application software must arrange for a large majority of the memory references
-[cache misses] to be to "local" memory--memory on the same cell, if any--or
-to the closest cell with memory.
-
-This leads to the Linux software view of a NUMA system:
-
-Linux divides the system's hardware resources into multiple software
-abstractions called "nodes".  Linux maps the nodes onto the physical cells
-of the hardware platform, abstracting away some of the details for some
-architectures.  As with physical cells, software nodes may contain 0 or more
-CPUs, memory and/or IO buses.  And, again, memory accesses to memory on
-"closer" nodes--nodes that map to closer cells--will generally experience
-faster access times and higher effective bandwidth than accesses to more
-remote cells.
-
-For some architectures, such as x86, Linux will "hide" any node representing a
-physical cell that has no memory attached, and reassign any CPUs attached to
-that cell to a node representing a cell that does have memory.  Thus, on
-these architectures, one cannot assume that all CPUs that Linux associates with
-a given node will see the same local memory access times and bandwidth.
-
-In addition, for some architectures, again x86 is an example, Linux supports
-the emulation of additional nodes.  For NUMA emulation, linux will carve up
-the existing nodes--or the system memory for non-NUMA platforms--into multiple
-nodes.  Each emulated node will manage a fraction of the underlying cells'
-physical memory.  NUMA emluation is useful for testing NUMA kernel and
-application features on non-NUMA platforms, and as a sort of memory resource
-management mechanism when used together with cpusets.
-[see Documentation/cgroup-v1/cpusets.txt]
-
-For each node with memory, Linux constructs an independent memory management
-subsystem, complete with its own free page lists, in-use page lists, usage
-statistics and locks to mediate access.  In addition, Linux constructs for
-each memory zone [one or more of DMA, DMA32, NORMAL, HIGH_MEMORY, MOVABLE],
-an ordered "zonelist".  A zonelist specifies the zones/nodes to visit when a
-selected zone/node cannot satisfy the allocation request.  This situation,
-when a zone has no available memory to satisfy a request, is called
-"overflow" or "fallback".
-
-Because some nodes contain multiple zones containing different types of
-memory, Linux must decide whether to order the zonelists such that allocations
-fall back to the same zone type on a different node, or to a different zone
-type on the same node.  This is an important consideration because some zones,
-such as DMA or DMA32, represent relatively scarce resources.  Linux chooses
-a default Node ordered zonelist. This means it tries to fallback to other zones
-from the same node before using remote nodes which are ordered by NUMA distance.
-
-By default, Linux will attempt to satisfy memory allocation requests from the
-node to which the CPU that executes the request is assigned.  Specifically,
-Linux will attempt to allocate from the first node in the appropriate zonelist
-for the node where the request originates.  This is called "local allocation."
-If the "local" node cannot satisfy the request, the kernel will examine other
-nodes' zones in the selected zonelist looking for the first zone in the list
-that can satisfy the request.
-
-Local allocation will tend to keep subsequent access to the allocated memory
-"local" to the underlying physical resources and off the system interconnect--
-as long as the task on whose behalf the kernel allocated some memory does not
-later migrate away from that memory.  The Linux scheduler is aware of the
-NUMA topology of the platform--embodied in the "scheduling domains" data
-structures [see Documentation/scheduler/sched-domains.txt]--and the scheduler
-attempts to minimize task migration to distant scheduling domains.  However,
-the scheduler does not take a task's NUMA footprint into account directly.
-Thus, under sufficient imbalance, tasks can migrate between nodes, remote
-from their initial node and kernel data structures.
-
-System administrators and application designers can restrict a task's migration
-to improve NUMA locality using various CPU affinity command line interfaces,
-such as taskset(1) and numactl(1), and program interfaces such as
-sched_setaffinity(2).  Further, one can modify the kernel's default local
-allocation behavior using Linux NUMA memory policy.
-[see Documentation/vm/numa_memory_policy.txt.]
-
-System administrators can restrict the CPUs and nodes' memories that a non-
-privileged user can specify in the scheduling or NUMA commands and functions
-using control groups and CPUsets.  [see Documentation/cgroup-v1/cpusets.txt]
-
-On architectures that do not hide memoryless nodes, Linux will include only
-zones [nodes] with memory in the zonelists.  This means that for a memoryless
-node the "local memory node"--the node of the first zone in CPU's node's
-zonelist--will not be the node itself.  Rather, it will be the node that the
-kernel selected as the nearest node with memory when it built the zonelists.
-So, default, local allocations will succeed with the kernel supplying the
-closest available memory.  This is a consequence of the same mechanism that
-allows such allocations to fallback to other nearby nodes when a node that
-does contain memory overflows.
-
-Some kernel allocations do not want or cannot tolerate this allocation fallback
-behavior.  Rather they want to be sure they get memory from the specified node
-or get notified that the node has no free memory.  This is usually the case when
-a subsystem allocates per CPU memory resources, for example.
-
-A typical model for making such an allocation is to obtain the node id of the
-node to which the "current CPU" is attached using one of the kernel's
-numa_node_id() or CPU_to_node() functions and then request memory from only
-the node id returned.  When such an allocation fails, the requesting subsystem
-may revert to its own fallback path.  The slab kernel memory allocator is an
-example of this.  Or, the subsystem may choose to disable or not to enable
-itself on allocation failure.  The kernel profiling subsystem is an example of
-this.
-
-If the architecture supports--does not hide--memoryless nodes, then CPUs
-attached to memoryless nodes would always incur the fallback path overhead
-or some subsystems would fail to initialize if they attempted to allocated
-memory exclusively from a node without memory.  To support such
-architectures transparently, kernel subsystems can use the numa_mem_id()
-or cpu_to_mem() function to locate the "local memory node" for the calling or
-specified CPU.  Again, this is the same node from which default, local page
-allocations will be attempted.
diff --git a/Documentation/vm/numa.rst b/Documentation/vm/numa.rst
new file mode 100644
index 000000000000..185d8a568168
--- /dev/null
+++ b/Documentation/vm/numa.rst
@@ -0,0 +1,150 @@
+.. _numa:
+
+Started Nov 1999 by Kanoj Sarcar <kanoj@sgi.com>
+
+=============
+What is NUMA?
+=============
+
+This question can be answered from a couple of perspectives:  the
+hardware view and the Linux software view.
+
+From the hardware perspective, a NUMA system is a computer platform that
+comprises multiple components or assemblies each of which may contain 0
+or more CPUs, local memory, and/or IO buses.  For brevity and to
+disambiguate the hardware view of these physical components/assemblies
+from the software abstraction thereof, we'll call the components/assemblies
+'cells' in this document.
+
+Each of the 'cells' may be viewed as an SMP [symmetric multi-processor] subset
+of the system--although some components necessary for a stand-alone SMP system
+may not be populated on any given cell.   The cells of the NUMA system are
+connected together with some sort of system interconnect--e.g., a crossbar or
+point-to-point link are common types of NUMA system interconnects.  Both of
+these types of interconnects can be aggregated to create NUMA platforms with
+cells at multiple distances from other cells.
+
+For Linux, the NUMA platforms of interest are primarily what is known as Cache
+Coherent NUMA or ccNUMA systems.   With ccNUMA systems, all memory is visible
+to and accessible from any CPU attached to any cell and cache coherency
+is handled in hardware by the processor caches and/or the system interconnect.
+
+Memory access time and effective memory bandwidth varies depending on how far
+away the cell containing the CPU or IO bus making the memory access is from the
+cell containing the target memory.  For example, access to memory by CPUs
+attached to the same cell will experience faster access times and higher
+bandwidths than accesses to memory on other, remote cells.  NUMA platforms
+can have cells at multiple remote distances from any given cell.
+
+Platform vendors don't build NUMA systems just to make software developers'
+lives interesting.  Rather, this architecture is a means to provide scalable
+memory bandwidth.  However, to achieve scalable memory bandwidth, system and
+application software must arrange for a large majority of the memory references
+[cache misses] to be to "local" memory--memory on the same cell, if any--or
+to the closest cell with memory.
+
+This leads to the Linux software view of a NUMA system:
+
+Linux divides the system's hardware resources into multiple software
+abstractions called "nodes".  Linux maps the nodes onto the physical cells
+of the hardware platform, abstracting away some of the details for some
+architectures.  As with physical cells, software nodes may contain 0 or more
+CPUs, memory and/or IO buses.  And, again, memory accesses to memory on
+"closer" nodes--nodes that map to closer cells--will generally experience
+faster access times and higher effective bandwidth than accesses to more
+remote cells.
+
+For some architectures, such as x86, Linux will "hide" any node representing a
+physical cell that has no memory attached, and reassign any CPUs attached to
+that cell to a node representing a cell that does have memory.  Thus, on
+these architectures, one cannot assume that all CPUs that Linux associates with
+a given node will see the same local memory access times and bandwidth.
+
+In addition, for some architectures, again x86 is an example, Linux supports
+the emulation of additional nodes.  For NUMA emulation, linux will carve up
+the existing nodes--or the system memory for non-NUMA platforms--into multiple
+nodes.  Each emulated node will manage a fraction of the underlying cells'
+physical memory.  NUMA emluation is useful for testing NUMA kernel and
+application features on non-NUMA platforms, and as a sort of memory resource
+management mechanism when used together with cpusets.
+[see Documentation/cgroup-v1/cpusets.txt]
+
+For each node with memory, Linux constructs an independent memory management
+subsystem, complete with its own free page lists, in-use page lists, usage
+statistics and locks to mediate access.  In addition, Linux constructs for
+each memory zone [one or more of DMA, DMA32, NORMAL, HIGH_MEMORY, MOVABLE],
+an ordered "zonelist".  A zonelist specifies the zones/nodes to visit when a
+selected zone/node cannot satisfy the allocation request.  This situation,
+when a zone has no available memory to satisfy a request, is called
+"overflow" or "fallback".
+
+Because some nodes contain multiple zones containing different types of
+memory, Linux must decide whether to order the zonelists such that allocations
+fall back to the same zone type on a different node, or to a different zone
+type on the same node.  This is an important consideration because some zones,
+such as DMA or DMA32, represent relatively scarce resources.  Linux chooses
+a default Node ordered zonelist. This means it tries to fallback to other zones
+from the same node before using remote nodes which are ordered by NUMA distance.
+
+By default, Linux will attempt to satisfy memory allocation requests from the
+node to which the CPU that executes the request is assigned.  Specifically,
+Linux will attempt to allocate from the first node in the appropriate zonelist
+for the node where the request originates.  This is called "local allocation."
+If the "local" node cannot satisfy the request, the kernel will examine other
+nodes' zones in the selected zonelist looking for the first zone in the list
+that can satisfy the request.
+
+Local allocation will tend to keep subsequent access to the allocated memory
+"local" to the underlying physical resources and off the system interconnect--
+as long as the task on whose behalf the kernel allocated some memory does not
+later migrate away from that memory.  The Linux scheduler is aware of the
+NUMA topology of the platform--embodied in the "scheduling domains" data
+structures [see Documentation/scheduler/sched-domains.txt]--and the scheduler
+attempts to minimize task migration to distant scheduling domains.  However,
+the scheduler does not take a task's NUMA footprint into account directly.
+Thus, under sufficient imbalance, tasks can migrate between nodes, remote
+from their initial node and kernel data structures.
+
+System administrators and application designers can restrict a task's migration
+to improve NUMA locality using various CPU affinity command line interfaces,
+such as taskset(1) and numactl(1), and program interfaces such as
+sched_setaffinity(2).  Further, one can modify the kernel's default local
+allocation behavior using Linux NUMA memory policy.
+[see Documentation/admin-guide/mm/numa_memory_policy.rst.]
+
+System administrators can restrict the CPUs and nodes' memories that a non-
+privileged user can specify in the scheduling or NUMA commands and functions
+using control groups and CPUsets.  [see Documentation/cgroup-v1/cpusets.txt]
+
+On architectures that do not hide memoryless nodes, Linux will include only
+zones [nodes] with memory in the zonelists.  This means that for a memoryless
+node the "local memory node"--the node of the first zone in CPU's node's
+zonelist--will not be the node itself.  Rather, it will be the node that the
+kernel selected as the nearest node with memory when it built the zonelists.
+So, default, local allocations will succeed with the kernel supplying the
+closest available memory.  This is a consequence of the same mechanism that
+allows such allocations to fallback to other nearby nodes when a node that
+does contain memory overflows.
+
+Some kernel allocations do not want or cannot tolerate this allocation fallback
+behavior.  Rather they want to be sure they get memory from the specified node
+or get notified that the node has no free memory.  This is usually the case when
+a subsystem allocates per CPU memory resources, for example.
+
+A typical model for making such an allocation is to obtain the node id of the
+node to which the "current CPU" is attached using one of the kernel's
+numa_node_id() or CPU_to_node() functions and then request memory from only
+the node id returned.  When such an allocation fails, the requesting subsystem
+may revert to its own fallback path.  The slab kernel memory allocator is an
+example of this.  Or, the subsystem may choose to disable or not to enable
+itself on allocation failure.  The kernel profiling subsystem is an example of
+this.
+
+If the architecture supports--does not hide--memoryless nodes, then CPUs
+attached to memoryless nodes would always incur the fallback path overhead
+or some subsystems would fail to initialize if they attempted to allocated
+memory exclusively from a node without memory.  To support such
+architectures transparently, kernel subsystems can use the numa_mem_id()
+or cpu_to_mem() function to locate the "local memory node" for the calling or
+specified CPU.  Again, this is the same node from which default, local page
+allocations will be attempted.
diff --git a/Documentation/vm/numa_memory_policy.txt b/Documentation/vm/numa_memory_policy.txt
deleted file mode 100644
index 622b927816e7..000000000000
--- a/Documentation/vm/numa_memory_policy.txt
+++ /dev/null
@@ -1,452 +0,0 @@
-
-What is Linux Memory Policy?
-
-In the Linux kernel, "memory policy" determines from which node the kernel will
-allocate memory in a NUMA system or in an emulated NUMA system.  Linux has
-supported platforms with Non-Uniform Memory Access architectures since 2.4.?.
-The current memory policy support was added to Linux 2.6 around May 2004.  This
-document attempts to describe the concepts and APIs of the 2.6 memory policy
-support.
-
-Memory policies should not be confused with cpusets
-(Documentation/cgroup-v1/cpusets.txt)
-which is an administrative mechanism for restricting the nodes from which
-memory may be allocated by a set of processes. Memory policies are a
-programming interface that a NUMA-aware application can take advantage of.  When
-both cpusets and policies are applied to a task, the restrictions of the cpuset
-takes priority.  See "MEMORY POLICIES AND CPUSETS" below for more details.
-
-MEMORY POLICY CONCEPTS
-
-Scope of Memory Policies
-
-The Linux kernel supports _scopes_ of memory policy, described here from
-most general to most specific:
-
-    System Default Policy:  this policy is "hard coded" into the kernel.  It
-    is the policy that governs all page allocations that aren't controlled
-    by one of the more specific policy scopes discussed below.  When the
-    system is "up and running", the system default policy will use "local
-    allocation" described below.  However, during boot up, the system
-    default policy will be set to interleave allocations across all nodes
-    with "sufficient" memory, so as not to overload the initial boot node
-    with boot-time allocations.
-
-    Task/Process Policy:  this is an optional, per-task policy.  When defined
-    for a specific task, this policy controls all page allocations made by or
-    on behalf of the task that aren't controlled by a more specific scope.
-    If a task does not define a task policy, then all page allocations that
-    would have been controlled by the task policy "fall back" to the System
-    Default Policy.
-
-	The task policy applies to the entire address space of a task. Thus,
-	it is inheritable, and indeed is inherited, across both fork()
-	[clone() w/o the CLONE_VM flag] and exec*().  This allows a parent task
-	to establish the task policy for a child task exec()'d from an
-	executable image that has no awareness of memory policy.  See the
-	MEMORY POLICY APIS section, below, for an overview of the system call
-	that a task may use to set/change its task/process policy.
-
-	In a multi-threaded task, task policies apply only to the thread
-	[Linux kernel task] that installs the policy and any threads
-	subsequently created by that thread.  Any sibling threads existing
-	at the time a new task policy is installed retain their current
-	policy.
-
-	A task policy applies only to pages allocated after the policy is
-	installed.  Any pages already faulted in by the task when the task
-	changes its task policy remain where they were allocated based on
-	the policy at the time they were allocated.
-
-    VMA Policy:  A "VMA" or "Virtual Memory Area" refers to a range of a task's
-    virtual address space.  A task may define a specific policy for a range
-    of its virtual address space.   See the MEMORY POLICIES APIS section,
-    below, for an overview of the mbind() system call used to set a VMA
-    policy.
-
-    A VMA policy will govern the allocation of pages that back this region of
-    the address space.  Any regions of the task's address space that don't
-    have an explicit VMA policy will fall back to the task policy, which may
-    itself fall back to the System Default Policy.
-
-    VMA policies have a few complicating details:
-
-	VMA policy applies ONLY to anonymous pages.  These include pages
-	allocated for anonymous segments, such as the task stack and heap, and
-	any regions of the address space mmap()ed with the MAP_ANONYMOUS flag.
-	If a VMA policy is applied to a file mapping, it will be ignored if
-	the mapping used the MAP_SHARED flag.  If the file mapping used the
-	MAP_PRIVATE flag, the VMA policy will only be applied when an
-	anonymous page is allocated on an attempt to write to the mapping--
-	i.e., at Copy-On-Write.
-
-	VMA policies are shared between all tasks that share a virtual address
-	space--a.k.a. threads--independent of when the policy is installed; and
-	they are inherited across fork().  However, because VMA policies refer
-	to a specific region of a task's address space, and because the address
-	space is discarded and recreated on exec*(), VMA policies are NOT
-	inheritable across exec().  Thus, only NUMA-aware applications may
-	use VMA policies.
-
-	A task may install a new VMA policy on a sub-range of a previously
-	mmap()ed region.  When this happens, Linux splits the existing virtual
-	memory area into 2 or 3 VMAs, each with it's own policy.
-
-	By default, VMA policy applies only to pages allocated after the policy
-	is installed.  Any pages already faulted into the VMA range remain
-	where they were allocated based on the policy at the time they were
-	allocated.  However, since 2.6.16, Linux supports page migration via
-	the mbind() system call, so that page contents can be moved to match
-	a newly installed policy.
-
-    Shared Policy:  Conceptually, shared policies apply to "memory objects"
-    mapped shared into one or more tasks' distinct address spaces.  An
-    application installs a shared policies the same way as VMA policies--using
-    the mbind() system call specifying a range of virtual addresses that map
-    the shared object.  However, unlike VMA policies, which can be considered
-    to be an attribute of a range of a task's address space, shared policies
-    apply directly to the shared object.  Thus, all tasks that attach to the
-    object share the policy, and all pages allocated for the shared object,
-    by any task, will obey the shared policy.
-
-	As of 2.6.22, only shared memory segments, created by shmget() or
-	mmap(MAP_ANONYMOUS|MAP_SHARED), support shared policy.  When shared
-	policy support was added to Linux, the associated data structures were
-	added to hugetlbfs shmem segments.  At the time, hugetlbfs did not
-	support allocation at fault time--a.k.a lazy allocation--so hugetlbfs
-	shmem segments were never "hooked up" to the shared policy support.
-	Although hugetlbfs segments now support lazy allocation, their support
-	for shared policy has not been completed.
-
-	As mentioned above [re: VMA policies], allocations of page cache
-	pages for regular files mmap()ed with MAP_SHARED ignore any VMA
-	policy installed on the virtual address range backed by the shared
-	file mapping.  Rather, shared page cache pages, including pages backing
-	private mappings that have not yet been written by the task, follow
-	task policy, if any, else System Default Policy.
-
-	The shared policy infrastructure supports different policies on subset
-	ranges of the shared object.  However, Linux still splits the VMA of
-	the task that installs the policy for each range of distinct policy.
-	Thus, different tasks that attach to a shared memory segment can have
-	different VMA configurations mapping that one shared object.  This
-	can be seen by examining the /proc/<pid>/numa_maps of tasks sharing
-	a shared memory region, when one task has installed shared policy on
-	one or more ranges of the region.
-
-Components of Memory Policies
-
-    A Linux memory policy consists of a "mode", optional mode flags, and an
-    optional set of nodes.  The mode determines the behavior of the policy,
-    the optional mode flags determine the behavior of the mode, and the
-    optional set of nodes can be viewed as the arguments to the policy
-    behavior.
-
-   Internally, memory policies are implemented by a reference counted
-   structure, struct mempolicy.  Details of this structure will be discussed
-   in context, below, as required to explain the behavior.
-
-   Linux memory policy supports the following 4 behavioral modes:
-
-	Default Mode--MPOL_DEFAULT:  This mode is only used in the memory
-	policy APIs.  Internally, MPOL_DEFAULT is converted to the NULL
-	memory policy in all policy scopes.  Any existing non-default policy
-	will simply be removed when MPOL_DEFAULT is specified.  As a result,
-	MPOL_DEFAULT means "fall back to the next most specific policy scope."
-
-	    For example, a NULL or default task policy will fall back to the
-	    system default policy.  A NULL or default vma policy will fall
-	    back to the task policy.
-
-	    When specified in one of the memory policy APIs, the Default mode
-	    does not use the optional set of nodes.
-
-	    It is an error for the set of nodes specified for this policy to
-	    be non-empty.
-
-	MPOL_BIND:  This mode specifies that memory must come from the
-	set of nodes specified by the policy.  Memory will be allocated from
-	the node in the set with sufficient free memory that is closest to
-	the node where the allocation takes place.
-
-	MPOL_PREFERRED:  This mode specifies that the allocation should be
-	attempted from the single node specified in the policy.  If that
-	allocation fails, the kernel will search other nodes, in order of
-	increasing distance from the preferred node based on information
-	provided by the platform firmware.
-
-	    Internally, the Preferred policy uses a single node--the
-	    preferred_node member of struct mempolicy.  When the internal
-	    mode flag MPOL_F_LOCAL is set, the preferred_node is ignored and
-	    the policy is interpreted as local allocation.  "Local" allocation
-	    policy can be viewed as a Preferred policy that starts at the node
-	    containing the cpu where the allocation takes place.
-
-	    It is possible for the user to specify that local allocation is
-	    always preferred by passing an empty nodemask with this mode.
-	    If an empty nodemask is passed, the policy cannot use the
-	    MPOL_F_STATIC_NODES or MPOL_F_RELATIVE_NODES flags described
-	    below.
-
-	MPOL_INTERLEAVED:  This mode specifies that page allocations be
-	interleaved, on a page granularity, across the nodes specified in
-	the policy.  This mode also behaves slightly differently, based on
-	the context where it is used:
-
-	    For allocation of anonymous pages and shared memory pages,
-	    Interleave mode indexes the set of nodes specified by the policy
-	    using the page offset of the faulting address into the segment
-	    [VMA] containing the address modulo the number of nodes specified
-	    by the policy.  It then attempts to allocate a page, starting at
-	    the selected node, as if the node had been specified by a Preferred
-	    policy or had been selected by a local allocation.  That is,
-	    allocation will follow the per node zonelist.
-
-	    For allocation of page cache pages, Interleave mode indexes the set
-	    of nodes specified by the policy using a node counter maintained
-	    per task.  This counter wraps around to the lowest specified node
-	    after it reaches the highest specified node.  This will tend to
-	    spread the pages out over the nodes specified by the policy based
-	    on the order in which they are allocated, rather than based on any
-	    page offset into an address range or file.  During system boot up,
-	    the temporary interleaved system default policy works in this
-	    mode.
-
-   Linux memory policy supports the following optional mode flags:
-
-	MPOL_F_STATIC_NODES:  This flag specifies that the nodemask passed by
-	the user should not be remapped if the task or VMA's set of allowed
-	nodes changes after the memory policy has been defined.
-
-	    Without this flag, anytime a mempolicy is rebound because of a
-	    change in the set of allowed nodes, the node (Preferred) or
-	    nodemask (Bind, Interleave) is remapped to the new set of
-	    allowed nodes.  This may result in nodes being used that were
-	    previously undesired.
-
-	    With this flag, if the user-specified nodes overlap with the
-	    nodes allowed by the task's cpuset, then the memory policy is
-	    applied to their intersection.  If the two sets of nodes do not
-	    overlap, the Default policy is used.
-
-	    For example, consider a task that is attached to a cpuset with
-	    mems 1-3 that sets an Interleave policy over the same set.  If
-	    the cpuset's mems change to 3-5, the Interleave will now occur
-	    over nodes 3, 4, and 5.  With this flag, however, since only node
-	    3 is allowed from the user's nodemask, the "interleave" only
-	    occurs over that node.  If no nodes from the user's nodemask are
-	    now allowed, the Default behavior is used.
-
-	    MPOL_F_STATIC_NODES cannot be combined with the
-	    MPOL_F_RELATIVE_NODES flag.  It also cannot be used for
-	    MPOL_PREFERRED policies that were created with an empty nodemask
-	    (local allocation).
-
-	MPOL_F_RELATIVE_NODES:  This flag specifies that the nodemask passed
-	by the user will be mapped relative to the set of the task or VMA's
-	set of allowed nodes.  The kernel stores the user-passed nodemask,
-	and if the allowed nodes changes, then that original nodemask will
-	be remapped relative to the new set of allowed nodes.
-
-	    Without this flag (and without MPOL_F_STATIC_NODES), anytime a
-	    mempolicy is rebound because of a change in the set of allowed
-	    nodes, the node (Preferred) or nodemask (Bind, Interleave) is
-	    remapped to the new set of allowed nodes.  That remap may not
-	    preserve the relative nature of the user's passed nodemask to its
-	    set of allowed nodes upon successive rebinds: a nodemask of
-	    1,3,5 may be remapped to 7-9 and then to 1-3 if the set of
-	    allowed nodes is restored to its original state.
-
-	    With this flag, the remap is done so that the node numbers from
-	    the user's passed nodemask are relative to the set of allowed
-	    nodes.  In other words, if nodes 0, 2, and 4 are set in the user's
-	    nodemask, the policy will be effected over the first (and in the
-	    Bind or Interleave case, the third and fifth) nodes in the set of
-	    allowed nodes.  The nodemask passed by the user represents nodes
-	    relative to task or VMA's set of allowed nodes.
-
-	    If the user's nodemask includes nodes that are outside the range
-	    of the new set of allowed nodes (for example, node 5 is set in
-	    the user's nodemask when the set of allowed nodes is only 0-3),
-	    then the remap wraps around to the beginning of the nodemask and,
-	    if not already set, sets the node in the mempolicy nodemask.
-
-	    For example, consider a task that is attached to a cpuset with
-	    mems 2-5 that sets an Interleave policy over the same set with
-	    MPOL_F_RELATIVE_NODES.  If the cpuset's mems change to 3-7, the
-	    interleave now occurs over nodes 3,5-7.  If the cpuset's mems
-	    then change to 0,2-3,5, then the interleave occurs over nodes
-	    0,2-3,5.
-
-	    Thanks to the consistent remapping, applications preparing
-	    nodemasks to specify memory policies using this flag should
-	    disregard their current, actual cpuset imposed memory placement
-	    and prepare the nodemask as if they were always located on
-	    memory nodes 0 to N-1, where N is the number of memory nodes the
-	    policy is intended to manage.  Let the kernel then remap to the
-	    set of memory nodes allowed by the task's cpuset, as that may
-	    change over time.
-
-	    MPOL_F_RELATIVE_NODES cannot be combined with the
-	    MPOL_F_STATIC_NODES flag.  It also cannot be used for
-	    MPOL_PREFERRED policies that were created with an empty nodemask
-	    (local allocation).
-
-MEMORY POLICY REFERENCE COUNTING
-
-To resolve use/free races, struct mempolicy contains an atomic reference
-count field.  Internal interfaces, mpol_get()/mpol_put() increment and
-decrement this reference count, respectively.  mpol_put() will only free
-the structure back to the mempolicy kmem cache when the reference count
-goes to zero.
-
-When a new memory policy is allocated, its reference count is initialized
-to '1', representing the reference held by the task that is installing the
-new policy.  When a pointer to a memory policy structure is stored in another
-structure, another reference is added, as the task's reference will be dropped
-on completion of the policy installation.
-
-During run-time "usage" of the policy, we attempt to minimize atomic operations
-on the reference count, as this can lead to cache lines bouncing between cpus
-and NUMA nodes.  "Usage" here means one of the following:
-
-1) querying of the policy, either by the task itself [using the get_mempolicy()
-   API discussed below] or by another task using the /proc/<pid>/numa_maps
-   interface.
-
-2) examination of the policy to determine the policy mode and associated node
-   or node lists, if any, for page allocation.  This is considered a "hot
-   path".  Note that for MPOL_BIND, the "usage" extends across the entire
-   allocation process, which may sleep during page reclaimation, because the
-   BIND policy nodemask is used, by reference, to filter ineligible nodes.
-
-We can avoid taking an extra reference during the usages listed above as
-follows:
-
-1) we never need to get/free the system default policy as this is never
-   changed nor freed, once the system is up and running.
-
-2) for querying the policy, we do not need to take an extra reference on the
-   target task's task policy nor vma policies because we always acquire the
-   task's mm's mmap_sem for read during the query.  The set_mempolicy() and
-   mbind() APIs [see below] always acquire the mmap_sem for write when
-   installing or replacing task or vma policies.  Thus, there is no possibility
-   of a task or thread freeing a policy while another task or thread is
-   querying it.
-
-3) Page allocation usage of task or vma policy occurs in the fault path where
-   we hold them mmap_sem for read.  Again, because replacing the task or vma
-   policy requires that the mmap_sem be held for write, the policy can't be
-   freed out from under us while we're using it for page allocation.
-
-4) Shared policies require special consideration.  One task can replace a
-   shared memory policy while another task, with a distinct mmap_sem, is
-   querying or allocating a page based on the policy.  To resolve this
-   potential race, the shared policy infrastructure adds an extra reference
-   to the shared policy during lookup while holding a spin lock on the shared
-   policy management structure.  This requires that we drop this extra
-   reference when we're finished "using" the policy.  We must drop the
-   extra reference on shared policies in the same query/allocation paths
-   used for non-shared policies.  For this reason, shared policies are marked
-   as such, and the extra reference is dropped "conditionally"--i.e., only
-   for shared policies.
-
-   Because of this extra reference counting, and because we must lookup
-   shared policies in a tree structure under spinlock, shared policies are
-   more expensive to use in the page allocation path.  This is especially
-   true for shared policies on shared memory regions shared by tasks running
-   on different NUMA nodes.  This extra overhead can be avoided by always
-   falling back to task or system default policy for shared memory regions,
-   or by prefaulting the entire shared memory region into memory and locking
-   it down.  However, this might not be appropriate for all applications.
-
-MEMORY POLICY APIs
-
-Linux supports 3 system calls for controlling memory policy.  These APIS
-always affect only the calling task, the calling task's address space, or
-some shared object mapped into the calling task's address space.
-
-	Note:  the headers that define these APIs and the parameter data types
-	for user space applications reside in a package that is not part of
-	the Linux kernel.  The kernel system call interfaces, with the 'sys_'
-	prefix, are defined in <linux/syscalls.h>; the mode and flag
-	definitions are defined in <linux/mempolicy.h>.
-
-Set [Task] Memory Policy:
-
-	long set_mempolicy(int mode, const unsigned long *nmask,
-					unsigned long maxnode);
-
-	Set's the calling task's "task/process memory policy" to mode
-	specified by the 'mode' argument and the set of nodes defined
-	by 'nmask'.  'nmask' points to a bit mask of node ids containing
-	at least 'maxnode' ids.  Optional mode flags may be passed by
-	combining the 'mode' argument with the flag (for example:
-	MPOL_INTERLEAVE | MPOL_F_STATIC_NODES).
-
-	See the set_mempolicy(2) man page for more details
-
-
-Get [Task] Memory Policy or Related Information
-
-	long get_mempolicy(int *mode,
-			   const unsigned long *nmask, unsigned long maxnode,
-			   void *addr, int flags);
-
-	Queries the "task/process memory policy" of the calling task, or
-	the policy or location of a specified virtual address, depending
-	on the 'flags' argument.
-
-	See the get_mempolicy(2) man page for more details
-
-
-Install VMA/Shared Policy for a Range of Task's Address Space
-
-	long mbind(void *start, unsigned long len, int mode,
-		   const unsigned long *nmask, unsigned long maxnode,
-		   unsigned flags);
-
-	mbind() installs the policy specified by (mode, nmask, maxnodes) as
-	a VMA policy for the range of the calling task's address space
-	specified by the 'start' and 'len' arguments.  Additional actions
-	may be requested via the 'flags' argument.
-
-	See the mbind(2) man page for more details.
-
-MEMORY POLICY COMMAND LINE INTERFACE
-
-Although not strictly part of the Linux implementation of memory policy,
-a command line tool, numactl(8), exists that allows one to:
-
-+ set the task policy for a specified program via set_mempolicy(2), fork(2) and
-  exec(2)
-
-+ set the shared policy for a shared memory segment via mbind(2)
-
-The numactl(8) tool is packaged with the run-time version of the library
-containing the memory policy system call wrappers.  Some distributions
-package the headers and compile-time libraries in a separate development
-package.
-
-
-MEMORY POLICIES AND CPUSETS
-
-Memory policies work within cpusets as described above.  For memory policies
-that require a node or set of nodes, the nodes are restricted to the set of
-nodes whose memories are allowed by the cpuset constraints.  If the nodemask
-specified for the policy contains nodes that are not allowed by the cpuset and
-MPOL_F_RELATIVE_NODES is not used, the intersection of the set of nodes
-specified for the policy and the set of nodes with memory is used.  If the
-result is the empty set, the policy is considered invalid and cannot be
-installed.  If MPOL_F_RELATIVE_NODES is used, the policy's nodes are mapped
-onto and folded into the task's set of allowed nodes as previously described.
-
-The interaction of memory policies and cpusets can be problematic when tasks
-in two cpusets share access to a memory region, such as shared memory segments
-created by shmget() of mmap() with the MAP_ANONYMOUS and MAP_SHARED flags, and
-any of the tasks install shared policy on the region, only nodes whose
-memories are allowed in both cpusets may be used in the policies.  Obtaining
-this information requires "stepping outside" the memory policy APIs to use the
-cpuset information and requires that one know in what cpusets other task might
-be attaching to the shared region.  Furthermore, if the cpusets' allowed
-memory sets are disjoint, "local" allocation is the only valid policy.
diff --git a/Documentation/vm/overcommit-accounting b/Documentation/vm/overcommit-accounting
deleted file mode 100644
index cbfaaa674118..000000000000
--- a/Documentation/vm/overcommit-accounting
+++ /dev/null
@@ -1,80 +0,0 @@
-The Linux kernel supports the following overcommit handling modes
-
-0	-	Heuristic overcommit handling. Obvious overcommits of
-		address space are refused. Used for a typical system. It
-		ensures a seriously wild allocation fails while allowing
-		overcommit to reduce swap usage.  root is allowed to 
-		allocate slightly more memory in this mode. This is the 
-		default.
-
-1	-	Always overcommit. Appropriate for some scientific
-		applications. Classic example is code using sparse arrays
-		and just relying on the virtual memory consisting almost
-		entirely of zero pages.
-
-2	-	Don't overcommit. The total address space commit
-		for the system is not permitted to exceed swap + a
-		configurable amount (default is 50%) of physical RAM.
-		Depending on the amount you use, in most situations
-		this means a process will not be killed while accessing
-		pages but will receive errors on memory allocation as
-		appropriate.
-
-		Useful for applications that want to guarantee their
-		memory allocations will be available in the future
-		without having to initialize every page.
-
-The overcommit policy is set via the sysctl `vm.overcommit_memory'.
-
-The overcommit amount can be set via `vm.overcommit_ratio' (percentage)
-or `vm.overcommit_kbytes' (absolute value).
-
-The current overcommit limit and amount committed are viewable in
-/proc/meminfo as CommitLimit and Committed_AS respectively.
-
-Gotchas
--------
-
-The C language stack growth does an implicit mremap. If you want absolute
-guarantees and run close to the edge you MUST mmap your stack for the 
-largest size you think you will need. For typical stack usage this does
-not matter much but it's a corner case if you really really care
-
-In mode 2 the MAP_NORESERVE flag is ignored. 
-
-
-How It Works
-------------
-
-The overcommit is based on the following rules
-
-For a file backed map
-	SHARED or READ-only	-	0 cost (the file is the map not swap)
-	PRIVATE WRITABLE	-	size of mapping per instance
-
-For an anonymous or /dev/zero map
-	SHARED			-	size of mapping
-	PRIVATE READ-only	-	0 cost (but of little use)
-	PRIVATE WRITABLE	-	size of mapping per instance
-
-Additional accounting
-	Pages made writable copies by mmap
-	shmfs memory drawn from the same pool
-
-Status
-------
-
-o	We account mmap memory mappings
-o	We account mprotect changes in commit
-o	We account mremap changes in size
-o	We account brk
-o	We account munmap
-o	We report the commit status in /proc
-o	Account and check on fork
-o	Review stack handling/building on exec
-o	SHMfs accounting
-o	Implement actual limit enforcement
-
-To Do
------
-o	Account ptrace pages (this is hard)
diff --git a/Documentation/vm/overcommit-accounting.rst b/Documentation/vm/overcommit-accounting.rst
new file mode 100644
index 000000000000..0dd54bbe4afa
--- /dev/null
+++ b/Documentation/vm/overcommit-accounting.rst
@@ -0,0 +1,87 @@
+.. _overcommit_accounting:
+
+=====================
+Overcommit Accounting
+=====================
+
+The Linux kernel supports the following overcommit handling modes
+
+0
+	Heuristic overcommit handling. Obvious overcommits of address
+	space are refused. Used for a typical system. It ensures a
+	seriously wild allocation fails while allowing overcommit to
+	reduce swap usage.  root is allowed to allocate slightly more
+	memory in this mode. This is the default.
+
+1
+	Always overcommit. Appropriate for some scientific
+	applications. Classic example is code using sparse arrays and
+	just relying on the virtual memory consisting almost entirely
+	of zero pages.
+
+2
+	Don't overcommit. The total address space commit for the
+	system is not permitted to exceed swap + a configurable amount
+	(default is 50%) of physical RAM.  Depending on the amount you
+	use, in most situations this means a process will not be
+	killed while accessing pages but will receive errors on memory
+	allocation as appropriate.
+
+	Useful for applications that want to guarantee their memory
+	allocations will be available in the future without having to
+	initialize every page.
+
+The overcommit policy is set via the sysctl ``vm.overcommit_memory``.
+
+The overcommit amount can be set via ``vm.overcommit_ratio`` (percentage)
+or ``vm.overcommit_kbytes`` (absolute value).
+
+The current overcommit limit and amount committed are viewable in
+``/proc/meminfo`` as CommitLimit and Committed_AS respectively.
+
+Gotchas
+=======
+
+The C language stack growth does an implicit mremap. If you want absolute
+guarantees and run close to the edge you MUST mmap your stack for the
+largest size you think you will need. For typical stack usage this does
+not matter much but it's a corner case if you really really care
+
+In mode 2 the MAP_NORESERVE flag is ignored.
+
+
+How It Works
+============
+
+The overcommit is based on the following rules
+
+For a file backed map
+	| SHARED or READ-only	-	0 cost (the file is the map not swap)
+	| PRIVATE WRITABLE	-	size of mapping per instance
+
+For an anonymous or ``/dev/zero`` map
+	| SHARED			-	size of mapping
+	| PRIVATE READ-only	-	0 cost (but of little use)
+	| PRIVATE WRITABLE	-	size of mapping per instance
+
+Additional accounting
+	| Pages made writable copies by mmap
+	| shmfs memory drawn from the same pool
+
+Status
+======
+
+*	We account mmap memory mappings
+*	We account mprotect changes in commit
+*	We account mremap changes in size
+*	We account brk
+*	We account munmap
+*	We report the commit status in /proc
+*	Account and check on fork
+*	Review stack handling/building on exec
+*	SHMfs accounting
+*	Implement actual limit enforcement
+
+To Do
+=====
+*	Account ptrace pages (this is hard)
diff --git a/Documentation/vm/page_frags b/Documentation/vm/page_frags
deleted file mode 100644
index a6714565dbf9..000000000000
--- a/Documentation/vm/page_frags
+++ /dev/null
@@ -1,42 +0,0 @@
-Page fragments
---------------
-
-A page fragment is an arbitrary-length arbitrary-offset area of memory
-which resides within a 0 or higher order compound page.  Multiple
-fragments within that page are individually refcounted, in the page's
-reference counter.
-
-The page_frag functions, page_frag_alloc and page_frag_free, provide a
-simple allocation framework for page fragments.  This is used by the
-network stack and network device drivers to provide a backing region of
-memory for use as either an sk_buff->head, or to be used in the "frags"
-portion of skb_shared_info.
-
-In order to make use of the page fragment APIs a backing page fragment
-cache is needed.  This provides a central point for the fragment allocation
-and tracks allows multiple calls to make use of a cached page.  The
-advantage to doing this is that multiple calls to get_page can be avoided
-which can be expensive at allocation time.  However due to the nature of
-this caching it is required that any calls to the cache be protected by
-either a per-cpu limitation, or a per-cpu limitation and forcing interrupts
-to be disabled when executing the fragment allocation.
-
-The network stack uses two separate caches per CPU to handle fragment
-allocation.  The netdev_alloc_cache is used by callers making use of the
-__netdev_alloc_frag and __netdev_alloc_skb calls.  The napi_alloc_cache is
-used by callers of the __napi_alloc_frag and __napi_alloc_skb calls.  The
-main difference between these two calls is the context in which they may be
-called.  The "netdev" prefixed functions are usable in any context as these
-functions will disable interrupts, while the "napi" prefixed functions are
-only usable within the softirq context.
-
-Many network device drivers use a similar methodology for allocating page
-fragments, but the page fragments are cached at the ring or descriptor
-level.  In order to enable these cases it is necessary to provide a generic
-way of tearing down a page cache.  For this reason __page_frag_cache_drain
-was implemented.  It allows for freeing multiple references from a single
-page via a single call.  The advantage to doing this is that it allows for
-cleaning up the multiple references that were added to a page in order to
-avoid calling get_page per allocation.
-
-Alexander Duyck, Nov 29, 2016.
diff --git a/Documentation/vm/page_frags.rst b/Documentation/vm/page_frags.rst
new file mode 100644
index 000000000000..637cc49d1b2f
--- /dev/null
+++ b/Documentation/vm/page_frags.rst
@@ -0,0 +1,45 @@
+.. _page_frags:
+
+==============
+Page fragments
+==============
+
+A page fragment is an arbitrary-length arbitrary-offset area of memory
+which resides within a 0 or higher order compound page.  Multiple
+fragments within that page are individually refcounted, in the page's
+reference counter.
+
+The page_frag functions, page_frag_alloc and page_frag_free, provide a
+simple allocation framework for page fragments.  This is used by the
+network stack and network device drivers to provide a backing region of
+memory for use as either an sk_buff->head, or to be used in the "frags"
+portion of skb_shared_info.
+
+In order to make use of the page fragment APIs a backing page fragment
+cache is needed.  This provides a central point for the fragment allocation
+and tracks allows multiple calls to make use of a cached page.  The
+advantage to doing this is that multiple calls to get_page can be avoided
+which can be expensive at allocation time.  However due to the nature of
+this caching it is required that any calls to the cache be protected by
+either a per-cpu limitation, or a per-cpu limitation and forcing interrupts
+to be disabled when executing the fragment allocation.
+
+The network stack uses two separate caches per CPU to handle fragment
+allocation.  The netdev_alloc_cache is used by callers making use of the
+__netdev_alloc_frag and __netdev_alloc_skb calls.  The napi_alloc_cache is
+used by callers of the __napi_alloc_frag and __napi_alloc_skb calls.  The
+main difference between these two calls is the context in which they may be
+called.  The "netdev" prefixed functions are usable in any context as these
+functions will disable interrupts, while the "napi" prefixed functions are
+only usable within the softirq context.
+
+Many network device drivers use a similar methodology for allocating page
+fragments, but the page fragments are cached at the ring or descriptor
+level.  In order to enable these cases it is necessary to provide a generic
+way of tearing down a page cache.  For this reason __page_frag_cache_drain
+was implemented.  It allows for freeing multiple references from a single
+page via a single call.  The advantage to doing this is that it allows for
+cleaning up the multiple references that were added to a page in order to
+avoid calling get_page per allocation.
+
+Alexander Duyck, Nov 29, 2016.
diff --git a/Documentation/vm/page_migration b/Documentation/vm/page_migration
deleted file mode 100644
index 496868072e24..000000000000
--- a/Documentation/vm/page_migration
+++ /dev/null
@@ -1,252 +0,0 @@
-Page migration
---------------
-
-Page migration allows the moving of the physical location of pages between
-nodes in a numa system while the process is running. This means that the
-virtual addresses that the process sees do not change. However, the
-system rearranges the physical location of those pages.
-
-The main intend of page migration is to reduce the latency of memory access
-by moving pages near to the processor where the process accessing that memory
-is running.
-
-Page migration allows a process to manually relocate the node on which its
-pages are located through the MF_MOVE and MF_MOVE_ALL options while setting
-a new memory policy via mbind(). The pages of process can also be relocated
-from another process using the sys_migrate_pages() function call. The
-migrate_pages function call takes two sets of nodes and moves pages of a
-process that are located on the from nodes to the destination nodes.
-Page migration functions are provided by the numactl package by Andi Kleen
-(a version later than 0.9.3 is required. Get it from
-ftp://oss.sgi.com/www/projects/libnuma/download/). numactl provides libnuma
-which provides an interface similar to other numa functionality for page
-migration.  cat /proc/<pid>/numa_maps allows an easy review of where the
-pages of a process are located. See also the numa_maps documentation in the
-proc(5) man page.
-
-Manual migration is useful if for example the scheduler has relocated
-a process to a processor on a distant node. A batch scheduler or an
-administrator may detect the situation and move the pages of the process
-nearer to the new processor. The kernel itself does only provide
-manual page migration support. Automatic page migration may be implemented
-through user space processes that move pages. A special function call
-"move_pages" allows the moving of individual pages within a process.
-A NUMA profiler may f.e. obtain a log showing frequent off node
-accesses and may use the result to move pages to more advantageous
-locations.
-
-Larger installations usually partition the system using cpusets into
-sections of nodes. Paul Jackson has equipped cpusets with the ability to
-move pages when a task is moved to another cpuset (See
-Documentation/cgroup-v1/cpusets.txt).
-Cpusets allows the automation of process locality. If a task is moved to
-a new cpuset then also all its pages are moved with it so that the
-performance of the process does not sink dramatically. Also the pages
-of processes in a cpuset are moved if the allowed memory nodes of a
-cpuset are changed.
-
-Page migration allows the preservation of the relative location of pages
-within a group of nodes for all migration techniques which will preserve a
-particular memory allocation pattern generated even after migrating a
-process. This is necessary in order to preserve the memory latencies.
-Processes will run with similar performance after migration.
-
-Page migration occurs in several steps. First a high level
-description for those trying to use migrate_pages() from the kernel
-(for userspace usage see the Andi Kleen's numactl package mentioned above)
-and then a low level description of how the low level details work.
-
-A. In kernel use of migrate_pages()
------------------------------------
-
-1. Remove pages from the LRU.
-
-   Lists of pages to be migrated are generated by scanning over
-   pages and moving them into lists. This is done by
-   calling isolate_lru_page().
-   Calling isolate_lru_page increases the references to the page
-   so that it cannot vanish while the page migration occurs.
-   It also prevents the swapper or other scans to encounter
-   the page.
-
-2. We need to have a function of type new_page_t that can be
-   passed to migrate_pages(). This function should figure out
-   how to allocate the correct new page given the old page.
-
-3. The migrate_pages() function is called which attempts
-   to do the migration. It will call the function to allocate
-   the new page for each page that is considered for
-   moving.
-
-B. How migrate_pages() works
-----------------------------
-
-migrate_pages() does several passes over its list of pages. A page is moved
-if all references to a page are removable at the time. The page has
-already been removed from the LRU via isolate_lru_page() and the refcount
-is increased so that the page cannot be freed while page migration occurs.
-
-Steps:
-
-1. Lock the page to be migrated
-
-2. Ensure that writeback is complete.
-
-3. Lock the new page that we want to move to. It is locked so that accesses to
-   this (not yet uptodate) page immediately lock while the move is in progress.
-
-4. All the page table references to the page are converted to migration
-   entries. This decreases the mapcount of a page. If the resulting
-   mapcount is not zero then we do not migrate the page. All user space
-   processes that attempt to access the page will now wait on the page lock.
-
-5. The i_pages lock is taken. This will cause all processes trying
-   to access the page via the mapping to block on the spinlock.
-
-6. The refcount of the page is examined and we back out if references remain
-   otherwise we know that we are the only one referencing this page.
-
-7. The radix tree is checked and if it does not contain the pointer to this
-   page then we back out because someone else modified the radix tree.
-
-8. The new page is prepped with some settings from the old page so that
-   accesses to the new page will discover a page with the correct settings.
-
-9. The radix tree is changed to point to the new page.
-
-10. The reference count of the old page is dropped because the address space
-    reference is gone. A reference to the new page is established because
-    the new page is referenced by the address space.
-
-11. The i_pages lock is dropped. With that lookups in the mapping
-    become possible again. Processes will move from spinning on the lock
-    to sleeping on the locked new page.
-
-12. The page contents are copied to the new page.
-
-13. The remaining page flags are copied to the new page.
-
-14. The old page flags are cleared to indicate that the page does
-    not provide any information anymore.
-
-15. Queued up writeback on the new page is triggered.
-
-16. If migration entries were page then replace them with real ptes. Doing
-    so will enable access for user space processes not already waiting for
-    the page lock.
-
-19. The page locks are dropped from the old and new page.
-    Processes waiting on the page lock will redo their page faults
-    and will reach the new page.
-
-20. The new page is moved to the LRU and can be scanned by the swapper
-    etc again.
-
-C. Non-LRU page migration
--------------------------
-
-Although original migration aimed for reducing the latency of memory access
-for NUMA, compaction who want to create high-order page is also main customer.
-
-Current problem of the implementation is that it is designed to migrate only
-*LRU* pages. However, there are potential non-lru pages which can be migrated
-in drivers, for example, zsmalloc, virtio-balloon pages.
-
-For virtio-balloon pages, some parts of migration code path have been hooked
-up and added virtio-balloon specific functions to intercept migration logics.
-It's too specific to a driver so other drivers who want to make their pages
-movable would have to add own specific hooks in migration path.
-
-To overclome the problem, VM supports non-LRU page migration which provides
-generic functions for non-LRU movable pages without driver specific hooks
-migration path.
-
-If a driver want to make own pages movable, it should define three functions
-which are function pointers of struct address_space_operations.
-
-1. bool (*isolate_page) (struct page *page, isolate_mode_t mode);
-
-What VM expects on isolate_page function of driver is to return *true*
-if driver isolates page successfully. On returing true, VM marks the page
-as PG_isolated so concurrent isolation in several CPUs skip the page
-for isolation. If a driver cannot isolate the page, it should return *false*.
-
-Once page is successfully isolated, VM uses page.lru fields so driver
-shouldn't expect to preserve values in that fields.
-
-2. int (*migratepage) (struct address_space *mapping,
-		struct page *newpage, struct page *oldpage, enum migrate_mode);
-
-After isolation, VM calls migratepage of driver with isolated page.
-The function of migratepage is to move content of the old page to new page
-and set up fields of struct page newpage. Keep in mind that you should
-indicate to the VM the oldpage is no longer movable via __ClearPageMovable()
-under page_lock if you migrated the oldpage successfully and returns
-MIGRATEPAGE_SUCCESS. If driver cannot migrate the page at the moment, driver
-can return -EAGAIN. On -EAGAIN, VM will retry page migration in a short time
-because VM interprets -EAGAIN as "temporal migration failure". On returning
-any error except -EAGAIN, VM will give up the page migration without retrying
-in this time.
-
-Driver shouldn't touch page.lru field VM using in the functions.
-
-3. void (*putback_page)(struct page *);
-
-If migration fails on isolated page, VM should return the isolated page
-to the driver so VM calls driver's putback_page with migration failed page.
-In this function, driver should put the isolated page back to the own data
-structure.
-
-4. non-lru movable page flags
-
-There are two page flags for supporting non-lru movable page.
-
-* PG_movable
-
-Driver should use the below function to make page movable under page_lock.
-
-	void __SetPageMovable(struct page *page, struct address_space *mapping)
-
-It needs argument of address_space for registering migration family functions
-which will be called by VM. Exactly speaking, PG_movable is not a real flag of
-struct page. Rather than, VM reuses page->mapping's lower bits to represent it.
-
-	#define PAGE_MAPPING_MOVABLE 0x2
-	page->mapping = page->mapping | PAGE_MAPPING_MOVABLE;
-
-so driver shouldn't access page->mapping directly. Instead, driver should
-use page_mapping which mask off the low two bits of page->mapping under
-page lock so it can get right struct address_space.
-
-For testing of non-lru movable page, VM supports __PageMovable function.
-However, it doesn't guarantee to identify non-lru movable page because
-page->mapping field is unified with other variables in struct page.
-As well, if driver releases the page after isolation by VM, page->mapping
-doesn't have stable value although it has PAGE_MAPPING_MOVABLE
-(Look at __ClearPageMovable). But __PageMovable is cheap to catch whether
-page is LRU or non-lru movable once the page has been isolated. Because
-LRU pages never can have PAGE_MAPPING_MOVABLE in page->mapping. It is also
-good for just peeking to test non-lru movable pages before more expensive
-checking with lock_page in pfn scanning to select victim.
-
-For guaranteeing non-lru movable page, VM provides PageMovable function.
-Unlike __PageMovable, PageMovable functions validates page->mapping and
-mapping->a_ops->isolate_page under lock_page. The lock_page prevents sudden
-destroying of page->mapping.
-
-Driver using __SetPageMovable should clear the flag via __ClearMovablePage
-under page_lock before the releasing the page.
-
-* PG_isolated
-
-To prevent concurrent isolation among several CPUs, VM marks isolated page
-as PG_isolated under lock_page. So if a CPU encounters PG_isolated non-lru
-movable page, it can skip it. Driver doesn't need to manipulate the flag
-because VM will set/clear it automatically. Keep in mind that if driver
-sees PG_isolated page, it means the page have been isolated by VM so it
-shouldn't touch page.lru field.
-PG_isolated is alias with PG_reclaim flag so driver shouldn't use the flag
-for own purpose.
-
-Christoph Lameter, May 8, 2006.
-Minchan Kim, Mar 28, 2016.
diff --git a/Documentation/vm/page_migration.rst b/Documentation/vm/page_migration.rst
new file mode 100644
index 000000000000..f68d61335abb
--- /dev/null
+++ b/Documentation/vm/page_migration.rst
@@ -0,0 +1,257 @@
+.. _page_migration:
+
+==============
+Page migration
+==============
+
+Page migration allows the moving of the physical location of pages between
+nodes in a numa system while the process is running. This means that the
+virtual addresses that the process sees do not change. However, the
+system rearranges the physical location of those pages.
+
+The main intend of page migration is to reduce the latency of memory access
+by moving pages near to the processor where the process accessing that memory
+is running.
+
+Page migration allows a process to manually relocate the node on which its
+pages are located through the MF_MOVE and MF_MOVE_ALL options while setting
+a new memory policy via mbind(). The pages of process can also be relocated
+from another process using the sys_migrate_pages() function call. The
+migrate_pages function call takes two sets of nodes and moves pages of a
+process that are located on the from nodes to the destination nodes.
+Page migration functions are provided by the numactl package by Andi Kleen
+(a version later than 0.9.3 is required. Get it from
+ftp://oss.sgi.com/www/projects/libnuma/download/). numactl provides libnuma
+which provides an interface similar to other numa functionality for page
+migration.  cat ``/proc/<pid>/numa_maps`` allows an easy review of where the
+pages of a process are located. See also the numa_maps documentation in the
+proc(5) man page.
+
+Manual migration is useful if for example the scheduler has relocated
+a process to a processor on a distant node. A batch scheduler or an
+administrator may detect the situation and move the pages of the process
+nearer to the new processor. The kernel itself does only provide
+manual page migration support. Automatic page migration may be implemented
+through user space processes that move pages. A special function call
+"move_pages" allows the moving of individual pages within a process.
+A NUMA profiler may f.e. obtain a log showing frequent off node
+accesses and may use the result to move pages to more advantageous
+locations.
+
+Larger installations usually partition the system using cpusets into
+sections of nodes. Paul Jackson has equipped cpusets with the ability to
+move pages when a task is moved to another cpuset (See
+Documentation/cgroup-v1/cpusets.txt).
+Cpusets allows the automation of process locality. If a task is moved to
+a new cpuset then also all its pages are moved with it so that the
+performance of the process does not sink dramatically. Also the pages
+of processes in a cpuset are moved if the allowed memory nodes of a
+cpuset are changed.
+
+Page migration allows the preservation of the relative location of pages
+within a group of nodes for all migration techniques which will preserve a
+particular memory allocation pattern generated even after migrating a
+process. This is necessary in order to preserve the memory latencies.
+Processes will run with similar performance after migration.
+
+Page migration occurs in several steps. First a high level
+description for those trying to use migrate_pages() from the kernel
+(for userspace usage see the Andi Kleen's numactl package mentioned above)
+and then a low level description of how the low level details work.
+
+In kernel use of migrate_pages()
+================================
+
+1. Remove pages from the LRU.
+
+   Lists of pages to be migrated are generated by scanning over
+   pages and moving them into lists. This is done by
+   calling isolate_lru_page().
+   Calling isolate_lru_page increases the references to the page
+   so that it cannot vanish while the page migration occurs.
+   It also prevents the swapper or other scans to encounter
+   the page.
+
+2. We need to have a function of type new_page_t that can be
+   passed to migrate_pages(). This function should figure out
+   how to allocate the correct new page given the old page.
+
+3. The migrate_pages() function is called which attempts
+   to do the migration. It will call the function to allocate
+   the new page for each page that is considered for
+   moving.
+
+How migrate_pages() works
+=========================
+
+migrate_pages() does several passes over its list of pages. A page is moved
+if all references to a page are removable at the time. The page has
+already been removed from the LRU via isolate_lru_page() and the refcount
+is increased so that the page cannot be freed while page migration occurs.
+
+Steps:
+
+1. Lock the page to be migrated
+
+2. Ensure that writeback is complete.
+
+3. Lock the new page that we want to move to. It is locked so that accesses to
+   this (not yet uptodate) page immediately lock while the move is in progress.
+
+4. All the page table references to the page are converted to migration
+   entries. This decreases the mapcount of a page. If the resulting
+   mapcount is not zero then we do not migrate the page. All user space
+   processes that attempt to access the page will now wait on the page lock.
+
+5. The i_pages lock is taken. This will cause all processes trying
+   to access the page via the mapping to block on the spinlock.
+
+6. The refcount of the page is examined and we back out if references remain
+   otherwise we know that we are the only one referencing this page.
+
+7. The radix tree is checked and if it does not contain the pointer to this
+   page then we back out because someone else modified the radix tree.
+
+8. The new page is prepped with some settings from the old page so that
+   accesses to the new page will discover a page with the correct settings.
+
+9. The radix tree is changed to point to the new page.
+
+10. The reference count of the old page is dropped because the address space
+    reference is gone. A reference to the new page is established because
+    the new page is referenced by the address space.
+
+11. The i_pages lock is dropped. With that lookups in the mapping
+    become possible again. Processes will move from spinning on the lock
+    to sleeping on the locked new page.
+
+12. The page contents are copied to the new page.
+
+13. The remaining page flags are copied to the new page.
+
+14. The old page flags are cleared to indicate that the page does
+    not provide any information anymore.
+
+15. Queued up writeback on the new page is triggered.
+
+16. If migration entries were page then replace them with real ptes. Doing
+    so will enable access for user space processes not already waiting for
+    the page lock.
+
+19. The page locks are dropped from the old and new page.
+    Processes waiting on the page lock will redo their page faults
+    and will reach the new page.
+
+20. The new page is moved to the LRU and can be scanned by the swapper
+    etc again.
+
+Non-LRU page migration
+======================
+
+Although original migration aimed for reducing the latency of memory access
+for NUMA, compaction who want to create high-order page is also main customer.
+
+Current problem of the implementation is that it is designed to migrate only
+*LRU* pages. However, there are potential non-lru pages which can be migrated
+in drivers, for example, zsmalloc, virtio-balloon pages.
+
+For virtio-balloon pages, some parts of migration code path have been hooked
+up and added virtio-balloon specific functions to intercept migration logics.
+It's too specific to a driver so other drivers who want to make their pages
+movable would have to add own specific hooks in migration path.
+
+To overclome the problem, VM supports non-LRU page migration which provides
+generic functions for non-LRU movable pages without driver specific hooks
+migration path.
+
+If a driver want to make own pages movable, it should define three functions
+which are function pointers of struct address_space_operations.
+
+1. ``bool (*isolate_page) (struct page *page, isolate_mode_t mode);``
+
+   What VM expects on isolate_page function of driver is to return *true*
+   if driver isolates page successfully. On returing true, VM marks the page
+   as PG_isolated so concurrent isolation in several CPUs skip the page
+   for isolation. If a driver cannot isolate the page, it should return *false*.
+
+   Once page is successfully isolated, VM uses page.lru fields so driver
+   shouldn't expect to preserve values in that fields.
+
+2. ``int (*migratepage) (struct address_space *mapping,``
+|	``struct page *newpage, struct page *oldpage, enum migrate_mode);``
+
+   After isolation, VM calls migratepage of driver with isolated page.
+   The function of migratepage is to move content of the old page to new page
+   and set up fields of struct page newpage. Keep in mind that you should
+   indicate to the VM the oldpage is no longer movable via __ClearPageMovable()
+   under page_lock if you migrated the oldpage successfully and returns
+   MIGRATEPAGE_SUCCESS. If driver cannot migrate the page at the moment, driver
+   can return -EAGAIN. On -EAGAIN, VM will retry page migration in a short time
+   because VM interprets -EAGAIN as "temporal migration failure". On returning
+   any error except -EAGAIN, VM will give up the page migration without retrying
+   in this time.
+
+   Driver shouldn't touch page.lru field VM using in the functions.
+
+3. ``void (*putback_page)(struct page *);``
+
+   If migration fails on isolated page, VM should return the isolated page
+   to the driver so VM calls driver's putback_page with migration failed page.
+   In this function, driver should put the isolated page back to the own data
+   structure.
+
+4. non-lru movable page flags
+
+   There are two page flags for supporting non-lru movable page.
+
+   * PG_movable
+
+     Driver should use the below function to make page movable under page_lock::
+
+	void __SetPageMovable(struct page *page, struct address_space *mapping)
+
+     It needs argument of address_space for registering migration
+     family functions which will be called by VM. Exactly speaking,
+     PG_movable is not a real flag of struct page. Rather than, VM
+     reuses page->mapping's lower bits to represent it.
+
+::
+	#define PAGE_MAPPING_MOVABLE 0x2
+	page->mapping = page->mapping | PAGE_MAPPING_MOVABLE;
+
+     so driver shouldn't access page->mapping directly. Instead, driver should
+     use page_mapping which mask off the low two bits of page->mapping under
+     page lock so it can get right struct address_space.
+
+     For testing of non-lru movable page, VM supports __PageMovable function.
+     However, it doesn't guarantee to identify non-lru movable page because
+     page->mapping field is unified with other variables in struct page.
+     As well, if driver releases the page after isolation by VM, page->mapping
+     doesn't have stable value although it has PAGE_MAPPING_MOVABLE
+     (Look at __ClearPageMovable). But __PageMovable is cheap to catch whether
+     page is LRU or non-lru movable once the page has been isolated. Because
+     LRU pages never can have PAGE_MAPPING_MOVABLE in page->mapping. It is also
+     good for just peeking to test non-lru movable pages before more expensive
+     checking with lock_page in pfn scanning to select victim.
+
+     For guaranteeing non-lru movable page, VM provides PageMovable function.
+     Unlike __PageMovable, PageMovable functions validates page->mapping and
+     mapping->a_ops->isolate_page under lock_page. The lock_page prevents sudden
+     destroying of page->mapping.
+
+     Driver using __SetPageMovable should clear the flag via __ClearMovablePage
+     under page_lock before the releasing the page.
+
+   * PG_isolated
+
+     To prevent concurrent isolation among several CPUs, VM marks isolated page
+     as PG_isolated under lock_page. So if a CPU encounters PG_isolated non-lru
+     movable page, it can skip it. Driver doesn't need to manipulate the flag
+     because VM will set/clear it automatically. Keep in mind that if driver
+     sees PG_isolated page, it means the page have been isolated by VM so it
+     shouldn't touch page.lru field.
+     PG_isolated is alias with PG_reclaim flag so driver shouldn't use the flag
+     for own purpose.
+
+Christoph Lameter, May 8, 2006.
+Minchan Kim, Mar 28, 2016.
diff --git a/Documentation/vm/page_owner.rst b/Documentation/vm/page_owner.rst
new file mode 100644
index 000000000000..0ed5ab8c7ab4
--- /dev/null
+++ b/Documentation/vm/page_owner.rst
@@ -0,0 +1,90 @@
+.. _page_owner:
+
+==================================================
+page owner: Tracking about who allocated each page
+==================================================
+
+Introduction
+============
+
+page owner is for the tracking about who allocated each page.
+It can be used to debug memory leak or to find a memory hogger.
+When allocation happens, information about allocation such as call stack
+and order of pages is stored into certain storage for each page.
+When we need to know about status of all pages, we can get and analyze
+this information.
+
+Although we already have tracepoint for tracing page allocation/free,
+using it for analyzing who allocate each page is rather complex. We need
+to enlarge the trace buffer for preventing overlapping until userspace
+program launched. And, launched program continually dump out the trace
+buffer for later analysis and it would change system behviour with more
+possibility rather than just keeping it in memory, so bad for debugging.
+
+page owner can also be used for various purposes. For example, accurate
+fragmentation statistics can be obtained through gfp flag information of
+each page. It is already implemented and activated if page owner is
+enabled. Other usages are more than welcome.
+
+page owner is disabled in default. So, if you'd like to use it, you need
+to add "page_owner=on" into your boot cmdline. If the kernel is built
+with page owner and page owner is disabled in runtime due to no enabling
+boot option, runtime overhead is marginal. If disabled in runtime, it
+doesn't require memory to store owner information, so there is no runtime
+memory overhead. And, page owner inserts just two unlikely branches into
+the page allocator hotpath and if not enabled, then allocation is done
+like as the kernel without page owner. These two unlikely branches should
+not affect to allocation performance, especially if the static keys jump
+label patching functionality is available. Following is the kernel's code
+size change due to this facility.
+
+- Without page owner::
+
+   text    data     bss     dec     hex filename
+   40662   1493     644   42799    a72f mm/page_alloc.o
+
+- With page owner::
+
+   text    data     bss     dec     hex filename
+   40892   1493     644   43029    a815 mm/page_alloc.o
+   1427      24       8    1459     5b3 mm/page_ext.o
+   2722      50       0    2772     ad4 mm/page_owner.o
+
+Although, roughly, 4 KB code is added in total, page_alloc.o increase by
+230 bytes and only half of it is in hotpath. Building the kernel with
+page owner and turning it on if needed would be great option to debug
+kernel memory problem.
+
+There is one notice that is caused by implementation detail. page owner
+stores information into the memory from struct page extension. This memory
+is initialized some time later than that page allocator starts in sparse
+memory system, so, until initialization, many pages can be allocated and
+they would have no owner information. To fix it up, these early allocated
+pages are investigated and marked as allocated in initialization phase.
+Although it doesn't mean that they have the right owner information,
+at least, we can tell whether the page is allocated or not,
+more accurately. On 2GB memory x86-64 VM box, 13343 early allocated pages
+are catched and marked, although they are mostly allocated from struct
+page extension feature. Anyway, after that, no page is left in
+un-tracking state.
+
+Usage
+=====
+
+1) Build user-space helper::
+
+	cd tools/vm
+	make page_owner_sort
+
+2) Enable page owner: add "page_owner=on" to boot cmdline.
+
+3) Do the job what you want to debug
+
+4) Analyze information from page owner::
+
+	cat /sys/kernel/debug/page_owner > page_owner_full.txt
+	grep -v ^PFN page_owner_full.txt > page_owner.txt
+	./page_owner_sort page_owner.txt sorted_page_owner.txt
+
+   See the result about who allocated each page
+   in the ``sorted_page_owner.txt``.
diff --git a/Documentation/vm/page_owner.txt b/Documentation/vm/page_owner.txt
deleted file mode 100644
index ffff1439076a..000000000000
--- a/Documentation/vm/page_owner.txt
+++ /dev/null
@@ -1,82 +0,0 @@
-page owner: Tracking about who allocated each page
------------------------------------------------------------
-
-* Introduction
-
-page owner is for the tracking about who allocated each page.
-It can be used to debug memory leak or to find a memory hogger.
-When allocation happens, information about allocation such as call stack
-and order of pages is stored into certain storage for each page.
-When we need to know about status of all pages, we can get and analyze
-this information.
-
-Although we already have tracepoint for tracing page allocation/free,
-using it for analyzing who allocate each page is rather complex. We need
-to enlarge the trace buffer for preventing overlapping until userspace
-program launched. And, launched program continually dump out the trace
-buffer for later analysis and it would change system behviour with more
-possibility rather than just keeping it in memory, so bad for debugging.
-
-page owner can also be used for various purposes. For example, accurate
-fragmentation statistics can be obtained through gfp flag information of
-each page. It is already implemented and activated if page owner is
-enabled. Other usages are more than welcome.
-
-page owner is disabled in default. So, if you'd like to use it, you need
-to add "page_owner=on" into your boot cmdline. If the kernel is built
-with page owner and page owner is disabled in runtime due to no enabling
-boot option, runtime overhead is marginal. If disabled in runtime, it
-doesn't require memory to store owner information, so there is no runtime
-memory overhead. And, page owner inserts just two unlikely branches into
-the page allocator hotpath and if not enabled, then allocation is done
-like as the kernel without page owner. These two unlikely branches should
-not affect to allocation performance, especially if the static keys jump
-label patching functionality is available. Following is the kernel's code
-size change due to this facility.
-
-- Without page owner
-   text    data     bss     dec     hex filename
-  40662    1493     644   42799    a72f mm/page_alloc.o
-
-- With page owner
-   text    data     bss     dec     hex filename
-  40892    1493     644   43029    a815 mm/page_alloc.o
-   1427      24       8    1459     5b3 mm/page_ext.o
-   2722      50       0    2772     ad4 mm/page_owner.o
-
-Although, roughly, 4 KB code is added in total, page_alloc.o increase by
-230 bytes and only half of it is in hotpath. Building the kernel with
-page owner and turning it on if needed would be great option to debug
-kernel memory problem.
-
-There is one notice that is caused by implementation detail. page owner
-stores information into the memory from struct page extension. This memory
-is initialized some time later than that page allocator starts in sparse
-memory system, so, until initialization, many pages can be allocated and
-they would have no owner information. To fix it up, these early allocated
-pages are investigated and marked as allocated in initialization phase.
-Although it doesn't mean that they have the right owner information,
-at least, we can tell whether the page is allocated or not,
-more accurately. On 2GB memory x86-64 VM box, 13343 early allocated pages
-are catched and marked, although they are mostly allocated from struct
-page extension feature. Anyway, after that, no page is left in
-un-tracking state.
-
-* Usage
-
-1) Build user-space helper
-	cd tools/vm
-	make page_owner_sort
-
-2) Enable page owner
-	Add "page_owner=on" to boot cmdline.
-
-3) Do the job what you want to debug
-
-4) Analyze information from page owner
-	cat /sys/kernel/debug/page_owner > page_owner_full.txt
-	grep -v ^PFN page_owner_full.txt > page_owner.txt
-	./page_owner_sort page_owner.txt sorted_page_owner.txt
-
-	See the result about who allocated each page
-	in the sorted_page_owner.txt.
diff --git a/Documentation/vm/pagemap.txt b/Documentation/vm/pagemap.txt
deleted file mode 100644
index eafcefa15261..000000000000
--- a/Documentation/vm/pagemap.txt
+++ /dev/null
@@ -1,183 +0,0 @@
-pagemap, from the userspace perspective
----------------------------------------
-
-pagemap is a new (as of 2.6.25) set of interfaces in the kernel that allow
-userspace programs to examine the page tables and related information by
-reading files in /proc.
-
-There are four components to pagemap:
-
- * /proc/pid/pagemap.  This file lets a userspace process find out which
-   physical frame each virtual page is mapped to.  It contains one 64-bit
-   value for each virtual page, containing the following data (from
-   fs/proc/task_mmu.c, above pagemap_read):
-
-    * Bits 0-54  page frame number (PFN) if present
-    * Bits 0-4   swap type if swapped
-    * Bits 5-54  swap offset if swapped
-    * Bit  55    pte is soft-dirty (see Documentation/vm/soft-dirty.txt)
-    * Bit  56    page exclusively mapped (since 4.2)
-    * Bits 57-60 zero
-    * Bit  61    page is file-page or shared-anon (since 3.5)
-    * Bit  62    page swapped
-    * Bit  63    page present
-
-   Since Linux 4.0 only users with the CAP_SYS_ADMIN capability can get PFNs.
-   In 4.0 and 4.1 opens by unprivileged fail with -EPERM.  Starting from
-   4.2 the PFN field is zeroed if the user does not have CAP_SYS_ADMIN.
-   Reason: information about PFNs helps in exploiting Rowhammer vulnerability.
-
-   If the page is not present but in swap, then the PFN contains an
-   encoding of the swap file number and the page's offset into the
-   swap. Unmapped pages return a null PFN. This allows determining
-   precisely which pages are mapped (or in swap) and comparing mapped
-   pages between processes.
-
-   Efficient users of this interface will use /proc/pid/maps to
-   determine which areas of memory are actually mapped and llseek to
-   skip over unmapped regions.
-
- * /proc/kpagecount.  This file contains a 64-bit count of the number of
-   times each page is mapped, indexed by PFN.
-
- * /proc/kpageflags.  This file contains a 64-bit set of flags for each
-   page, indexed by PFN.
-
-   The flags are (from fs/proc/page.c, above kpageflags_read):
-
-     0. LOCKED
-     1. ERROR
-     2. REFERENCED
-     3. UPTODATE
-     4. DIRTY
-     5. LRU
-     6. ACTIVE
-     7. SLAB
-     8. WRITEBACK
-     9. RECLAIM
-    10. BUDDY
-    11. MMAP
-    12. ANON
-    13. SWAPCACHE
-    14. SWAPBACKED
-    15. COMPOUND_HEAD
-    16. COMPOUND_TAIL
-    17. HUGE
-    18. UNEVICTABLE
-    19. HWPOISON
-    20. NOPAGE
-    21. KSM
-    22. THP
-    23. BALLOON
-    24. ZERO_PAGE
-    25. IDLE
-
- * /proc/kpagecgroup.  This file contains a 64-bit inode number of the
-   memory cgroup each page is charged to, indexed by PFN. Only available when
-   CONFIG_MEMCG is set.
-
-Short descriptions to the page flags:
-
- 0. LOCKED
-    page is being locked for exclusive access, eg. by undergoing read/write IO
-
- 7. SLAB
-    page is managed by the SLAB/SLOB/SLUB/SLQB kernel memory allocator
-    When compound page is used, SLUB/SLQB will only set this flag on the head
-    page; SLOB will not flag it at all.
-
-10. BUDDY
-    a free memory block managed by the buddy system allocator
-    The buddy system organizes free memory in blocks of various orders.
-    An order N block has 2^N physically contiguous pages, with the BUDDY flag
-    set for and _only_ for the first page.
-
-15. COMPOUND_HEAD
-16. COMPOUND_TAIL
-    A compound page with order N consists of 2^N physically contiguous pages.
-    A compound page with order 2 takes the form of "HTTT", where H donates its
-    head page and T donates its tail page(s).  The major consumers of compound
-    pages are hugeTLB pages (Documentation/vm/hugetlbpage.txt), the SLUB etc.
-    memory allocators and various device drivers. However in this interface,
-    only huge/giga pages are made visible to end users.
-17. HUGE
-    this is an integral part of a HugeTLB page
-
-19. HWPOISON
-    hardware detected memory corruption on this page: don't touch the data!
-
-20. NOPAGE
-    no page frame exists at the requested address
-
-21. KSM
-    identical memory pages dynamically shared between one or more processes
-
-22. THP
-    contiguous pages which construct transparent hugepages
-
-23. BALLOON
-    balloon compaction page
-
-24. ZERO_PAGE
-    zero page for pfn_zero or huge_zero page
-
-25. IDLE
-    page has not been accessed since it was marked idle (see
-    Documentation/vm/idle_page_tracking.txt). Note that this flag may be
-    stale in case the page was accessed via a PTE. To make sure the flag
-    is up-to-date one has to read /sys/kernel/mm/page_idle/bitmap first.
-
-    [IO related page flags]
- 1. ERROR     IO error occurred
- 3. UPTODATE  page has up-to-date data
-              ie. for file backed page: (in-memory data revision >= on-disk one)
- 4. DIRTY     page has been written to, hence contains new data
-              ie. for file backed page: (in-memory data revision >  on-disk one)
- 8. WRITEBACK page is being synced to disk
-
-    [LRU related page flags]
- 5. LRU         page is in one of the LRU lists
- 6. ACTIVE      page is in the active LRU list
-18. UNEVICTABLE page is in the unevictable (non-)LRU list
-                It is somehow pinned and not a candidate for LRU page reclaims,
-		eg. ramfs pages, shmctl(SHM_LOCK) and mlock() memory segments
- 2. REFERENCED  page has been referenced since last LRU list enqueue/requeue
- 9. RECLAIM     page will be reclaimed soon after its pageout IO completed
-11. MMAP        a memory mapped page
-12. ANON        a memory mapped page that is not part of a file
-13. SWAPCACHE   page is mapped to swap space, ie. has an associated swap entry
-14. SWAPBACKED  page is backed by swap/RAM
-
-The page-types tool in the tools/vm directory can be used to query the
-above flags.
-
-Using pagemap to do something useful:
-
-The general procedure for using pagemap to find out about a process' memory
-usage goes like this:
-
- 1. Read /proc/pid/maps to determine which parts of the memory space are
-    mapped to what.
- 2. Select the maps you are interested in -- all of them, or a particular
-    library, or the stack or the heap, etc.
- 3. Open /proc/pid/pagemap and seek to the pages you would like to examine.
- 4. Read a u64 for each page from pagemap.
- 5. Open /proc/kpagecount and/or /proc/kpageflags.  For each PFN you just
-    read, seek to that entry in the file, and read the data you want.
-
-For example, to find the "unique set size" (USS), which is the amount of
-memory that a process is using that is not shared with any other process,
-you can go through every map in the process, find the PFNs, look those up
-in kpagecount, and tally up the number of pages that are only referenced
-once.
-
-Other notes:
-
-Reading from any of the files will return -EINVAL if you are not starting
-the read on an 8-byte boundary (e.g., if you sought an odd number of bytes
-into the file), or if the size of the read is not a multiple of 8 bytes.
-
-Before Linux 3.11 pagemap bits 55-60 were used for "page-shift" (which is
-always 12 at most architectures). Since Linux 3.11 their meaning changes
-after first clear of soft-dirty bits. Since Linux 4.2 they are used for
-flags unconditionally.
diff --git a/Documentation/vm/remap_file_pages.rst b/Documentation/vm/remap_file_pages.rst
new file mode 100644
index 000000000000..7bef6718e3a9
--- /dev/null
+++ b/Documentation/vm/remap_file_pages.rst
@@ -0,0 +1,33 @@
+.. _remap_file_pages:
+
+==============================
+remap_file_pages() system call
+==============================
+
+The remap_file_pages() system call is used to create a nonlinear mapping,
+that is, a mapping in which the pages of the file are mapped into a
+nonsequential order in memory. The advantage of using remap_file_pages()
+over using repeated calls to mmap(2) is that the former approach does not
+require the kernel to create additional VMA (Virtual Memory Area) data
+structures.
+
+Supporting of nonlinear mapping requires significant amount of non-trivial
+code in kernel virtual memory subsystem including hot paths. Also to get
+nonlinear mapping work kernel need a way to distinguish normal page table
+entries from entries with file offset (pte_file). Kernel reserves flag in
+PTE for this purpose. PTE flags are scarce resource especially on some CPU
+architectures. It would be nice to free up the flag for other usage.
+
+Fortunately, there are not many users of remap_file_pages() in the wild.
+It's only known that one enterprise RDBMS implementation uses the syscall
+on 32-bit systems to map files bigger than can linearly fit into 32-bit
+virtual address space. This use-case is not critical anymore since 64-bit
+systems are widely available.
+
+The syscall is deprecated and replaced it with an emulation now. The
+emulation creates new VMAs instead of nonlinear mappings. It's going to
+work slower for rare users of remap_file_pages() but ABI is preserved.
+
+One side effect of emulation (apart from performance) is that user can hit
+vm.max_map_count limit more easily due to additional VMAs. See comment for
+DEFAULT_MAX_MAP_COUNT for more details on the limit.
diff --git a/Documentation/vm/remap_file_pages.txt b/Documentation/vm/remap_file_pages.txt
deleted file mode 100644
index f609142f406a..000000000000
--- a/Documentation/vm/remap_file_pages.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-The remap_file_pages() system call is used to create a nonlinear mapping,
-that is, a mapping in which the pages of the file are mapped into a
-nonsequential order in memory. The advantage of using remap_file_pages()
-over using repeated calls to mmap(2) is that the former approach does not
-require the kernel to create additional VMA (Virtual Memory Area) data
-structures.
-
-Supporting of nonlinear mapping requires significant amount of non-trivial
-code in kernel virtual memory subsystem including hot paths. Also to get
-nonlinear mapping work kernel need a way to distinguish normal page table
-entries from entries with file offset (pte_file). Kernel reserves flag in
-PTE for this purpose. PTE flags are scarce resource especially on some CPU
-architectures. It would be nice to free up the flag for other usage.
-
-Fortunately, there are not many users of remap_file_pages() in the wild.
-It's only known that one enterprise RDBMS implementation uses the syscall
-on 32-bit systems to map files bigger than can linearly fit into 32-bit
-virtual address space. This use-case is not critical anymore since 64-bit
-systems are widely available.
-
-The syscall is deprecated and replaced it with an emulation now. The
-emulation creates new VMAs instead of nonlinear mappings. It's going to
-work slower for rare users of remap_file_pages() but ABI is preserved.
-
-One side effect of emulation (apart from performance) is that user can hit
-vm.max_map_count limit more easily due to additional VMAs. See comment for
-DEFAULT_MAX_MAP_COUNT for more details on the limit.
diff --git a/Documentation/vm/slub.rst b/Documentation/vm/slub.rst
new file mode 100644
index 000000000000..3a775fd64e2d
--- /dev/null
+++ b/Documentation/vm/slub.rst
@@ -0,0 +1,361 @@
+.. _slub:
+
+==========================
+Short users guide for SLUB
+==========================
+
+The basic philosophy of SLUB is very different from SLAB. SLAB
+requires rebuilding the kernel to activate debug options for all
+slab caches. SLUB always includes full debugging but it is off by default.
+SLUB can enable debugging only for selected slabs in order to avoid
+an impact on overall system performance which may make a bug more
+difficult to find.
+
+In order to switch debugging on one can add an option ``slub_debug``
+to the kernel command line. That will enable full debugging for
+all slabs.
+
+Typically one would then use the ``slabinfo`` command to get statistical
+data and perform operation on the slabs. By default ``slabinfo`` only lists
+slabs that have data in them. See "slabinfo -h" for more options when
+running the command. ``slabinfo`` can be compiled with
+::
+
+	gcc -o slabinfo tools/vm/slabinfo.c
+
+Some of the modes of operation of ``slabinfo`` require that slub debugging
+be enabled on the command line. F.e. no tracking information will be
+available without debugging on and validation can only partially
+be performed if debugging was not switched on.
+
+Some more sophisticated uses of slub_debug:
+-------------------------------------------
+
+Parameters may be given to ``slub_debug``. If none is specified then full
+debugging is enabled. Format:
+
+slub_debug=<Debug-Options>
+	Enable options for all slabs
+slub_debug=<Debug-Options>,<slab name>
+	Enable options only for select slabs
+
+
+Possible debug options are::
+
+	F		Sanity checks on (enables SLAB_DEBUG_CONSISTENCY_CHECKS
+			Sorry SLAB legacy issues)
+	Z		Red zoning
+	P		Poisoning (object and padding)
+	U		User tracking (free and alloc)
+	T		Trace (please only use on single slabs)
+	A		Toggle failslab filter mark for the cache
+	O		Switch debugging off for caches that would have
+			caused higher minimum slab orders
+	-		Switch all debugging off (useful if the kernel is
+			configured with CONFIG_SLUB_DEBUG_ON)
+
+F.e. in order to boot just with sanity checks and red zoning one would specify::
+
+	slub_debug=FZ
+
+Trying to find an issue in the dentry cache? Try::
+
+	slub_debug=,dentry
+
+to only enable debugging on the dentry cache.
+
+Red zoning and tracking may realign the slab.  We can just apply sanity checks
+to the dentry cache with::
+
+	slub_debug=F,dentry
+
+Debugging options may require the minimum possible slab order to increase as
+a result of storing the metadata (for example, caches with PAGE_SIZE object
+sizes).  This has a higher liklihood of resulting in slab allocation errors
+in low memory situations or if there's high fragmentation of memory.  To
+switch off debugging for such caches by default, use::
+
+	slub_debug=O
+
+In case you forgot to enable debugging on the kernel command line: It is
+possible to enable debugging manually when the kernel is up. Look at the
+contents of::
+
+	/sys/kernel/slab/<slab name>/
+
+Look at the writable files. Writing 1 to them will enable the
+corresponding debug option. All options can be set on a slab that does
+not contain objects. If the slab already contains objects then sanity checks
+and tracing may only be enabled. The other options may cause the realignment
+of objects.
+
+Careful with tracing: It may spew out lots of information and never stop if
+used on the wrong slab.
+
+Slab merging
+============
+
+If no debug options are specified then SLUB may merge similar slabs together
+in order to reduce overhead and increase cache hotness of objects.
+``slabinfo -a`` displays which slabs were merged together.
+
+Slab validation
+===============
+
+SLUB can validate all object if the kernel was booted with slub_debug. In
+order to do so you must have the ``slabinfo`` tool. Then you can do
+::
+
+	slabinfo -v
+
+which will test all objects. Output will be generated to the syslog.
+
+This also works in a more limited way if boot was without slab debug.
+In that case ``slabinfo -v`` simply tests all reachable objects. Usually
+these are in the cpu slabs and the partial slabs. Full slabs are not
+tracked by SLUB in a non debug situation.
+
+Getting more performance
+========================
+
+To some degree SLUB's performance is limited by the need to take the
+list_lock once in a while to deal with partial slabs. That overhead is
+governed by the order of the allocation for each slab. The allocations
+can be influenced by kernel parameters:
+
+.. slub_min_objects=x		(default 4)
+.. slub_min_order=x		(default 0)
+.. slub_max_order=x		(default 3 (PAGE_ALLOC_COSTLY_ORDER))
+
+``slub_min_objects``
+	allows to specify how many objects must at least fit into one
+	slab in order for the allocation order to be acceptable.  In
+	general slub will be able to perform this number of
+	allocations on a slab without consulting centralized resources
+	(list_lock) where contention may occur.
+
+``slub_min_order``
+	specifies a minim order of slabs. A similar effect like
+	``slub_min_objects``.
+
+``slub_max_order``
+	specified the order at which ``slub_min_objects`` should no
+	longer be checked. This is useful to avoid SLUB trying to
+	generate super large order pages to fit ``slub_min_objects``
+	of a slab cache with large object sizes into one high order
+	page. Setting command line parameter
+	``debug_guardpage_minorder=N`` (N > 0), forces setting
+	``slub_max_order`` to 0, what cause minimum possible order of
+	slabs allocation.
+
+SLUB Debug output
+=================
+
+Here is a sample of slub debug output::
+
+ ====================================================================
+ BUG kmalloc-8: Redzone overwritten
+ --------------------------------------------------------------------
+
+ INFO: 0xc90f6d28-0xc90f6d2b. First byte 0x00 instead of 0xcc
+ INFO: Slab 0xc528c530 flags=0x400000c3 inuse=61 fp=0xc90f6d58
+ INFO: Object 0xc90f6d20 @offset=3360 fp=0xc90f6d58
+ INFO: Allocated in get_modalias+0x61/0xf5 age=53 cpu=1 pid=554
+
+ Bytes b4 0xc90f6d10:  00 00 00 00 00 00 00 00 5a 5a 5a 5a 5a 5a 5a 5a ........ZZZZZZZZ
+   Object 0xc90f6d20:  31 30 31 39 2e 30 30 35                         1019.005
+  Redzone 0xc90f6d28:  00 cc cc cc                                     .
+  Padding 0xc90f6d50:  5a 5a 5a 5a 5a 5a 5a 5a                         ZZZZZZZZ
+
+   [<c010523d>] dump_trace+0x63/0x1eb
+   [<c01053df>] show_trace_log_lvl+0x1a/0x2f
+   [<c010601d>] show_trace+0x12/0x14
+   [<c0106035>] dump_stack+0x16/0x18
+   [<c017e0fa>] object_err+0x143/0x14b
+   [<c017e2cc>] check_object+0x66/0x234
+   [<c017eb43>] __slab_free+0x239/0x384
+   [<c017f446>] kfree+0xa6/0xc6
+   [<c02e2335>] get_modalias+0xb9/0xf5
+   [<c02e23b7>] dmi_dev_uevent+0x27/0x3c
+   [<c027866a>] dev_uevent+0x1ad/0x1da
+   [<c0205024>] kobject_uevent_env+0x20a/0x45b
+   [<c020527f>] kobject_uevent+0xa/0xf
+   [<c02779f1>] store_uevent+0x4f/0x58
+   [<c027758e>] dev_attr_store+0x29/0x2f
+   [<c01bec4f>] sysfs_write_file+0x16e/0x19c
+   [<c0183ba7>] vfs_write+0xd1/0x15a
+   [<c01841d7>] sys_write+0x3d/0x72
+   [<c0104112>] sysenter_past_esp+0x5f/0x99
+   [<b7f7b410>] 0xb7f7b410
+   =======================
+
+ FIX kmalloc-8: Restoring Redzone 0xc90f6d28-0xc90f6d2b=0xcc
+
+If SLUB encounters a corrupted object (full detection requires the kernel
+to be booted with slub_debug) then the following output will be dumped
+into the syslog:
+
+1. Description of the problem encountered
+
+   This will be a message in the system log starting with::
+
+     ===============================================
+     BUG <slab cache affected>: <What went wrong>
+     -----------------------------------------------
+
+     INFO: <corruption start>-<corruption_end> <more info>
+     INFO: Slab <address> <slab information>
+     INFO: Object <address> <object information>
+     INFO: Allocated in <kernel function> age=<jiffies since alloc> cpu=<allocated by
+	cpu> pid=<pid of the process>
+     INFO: Freed in <kernel function> age=<jiffies since free> cpu=<freed by cpu>
+	pid=<pid of the process>
+
+   (Object allocation / free information is only available if SLAB_STORE_USER is
+   set for the slab. slub_debug sets that option)
+
+2. The object contents if an object was involved.
+
+   Various types of lines can follow the BUG SLUB line:
+
+   Bytes b4 <address> : <bytes>
+	Shows a few bytes before the object where the problem was detected.
+	Can be useful if the corruption does not stop with the start of the
+	object.
+
+   Object <address> : <bytes>
+	The bytes of the object. If the object is inactive then the bytes
+	typically contain poison values. Any non-poison value shows a
+	corruption by a write after free.
+
+   Redzone <address> : <bytes>
+	The Redzone following the object. The Redzone is used to detect
+	writes after the object. All bytes should always have the same
+	value. If there is any deviation then it is due to a write after
+	the object boundary.
+
+	(Redzone information is only available if SLAB_RED_ZONE is set.
+	slub_debug sets that option)
+
+   Padding <address> : <bytes>
+	Unused data to fill up the space in order to get the next object
+	properly aligned. In the debug case we make sure that there are
+	at least 4 bytes of padding. This allows the detection of writes
+	before the object.
+
+3. A stackdump
+
+   The stackdump describes the location where the error was detected. The cause
+   of the corruption is may be more likely found by looking at the function that
+   allocated or freed the object.
+
+4. Report on how the problem was dealt with in order to ensure the continued
+   operation of the system.
+
+   These are messages in the system log beginning with::
+
+	FIX <slab cache affected>: <corrective action taken>
+
+   In the above sample SLUB found that the Redzone of an active object has
+   been overwritten. Here a string of 8 characters was written into a slab that
+   has the length of 8 characters. However, a 8 character string needs a
+   terminating 0. That zero has overwritten the first byte of the Redzone field.
+   After reporting the details of the issue encountered the FIX SLUB message
+   tells us that SLUB has restored the Redzone to its proper value and then
+   system operations continue.
+
+Emergency operations
+====================
+
+Minimal debugging (sanity checks alone) can be enabled by booting with::
+
+	slub_debug=F
+
+This will be generally be enough to enable the resiliency features of slub
+which will keep the system running even if a bad kernel component will
+keep corrupting objects. This may be important for production systems.
+Performance will be impacted by the sanity checks and there will be a
+continual stream of error messages to the syslog but no additional memory
+will be used (unlike full debugging).
+
+No guarantees. The kernel component still needs to be fixed. Performance
+may be optimized further by locating the slab that experiences corruption
+and enabling debugging only for that cache
+
+I.e.::
+
+	slub_debug=F,dentry
+
+If the corruption occurs by writing after the end of the object then it
+may be advisable to enable a Redzone to avoid corrupting the beginning
+of other objects::
+
+	slub_debug=FZ,dentry
+
+Extended slabinfo mode and plotting
+===================================
+
+The ``slabinfo`` tool has a special 'extended' ('-X') mode that includes:
+ - Slabcache Totals
+ - Slabs sorted by size (up to -N <num> slabs, default 1)
+ - Slabs sorted by loss (up to -N <num> slabs, default 1)
+
+Additionally, in this mode ``slabinfo`` does not dynamically scale
+sizes (G/M/K) and reports everything in bytes (this functionality is
+also available to other slabinfo modes via '-B' option) which makes
+reporting more precise and accurate. Moreover, in some sense the `-X'
+mode also simplifies the analysis of slabs' behaviour, because its
+output can be plotted using the ``slabinfo-gnuplot.sh`` script. So it
+pushes the analysis from looking through the numbers (tons of numbers)
+to something easier -- visual analysis.
+
+To generate plots:
+
+a) collect slabinfo extended records, for example::
+
+	while [ 1 ]; do slabinfo -X >> FOO_STATS; sleep 1; done
+
+b) pass stats file(-s) to ``slabinfo-gnuplot.sh`` script::
+
+	slabinfo-gnuplot.sh FOO_STATS [FOO_STATS2 .. FOO_STATSN]
+
+   The ``slabinfo-gnuplot.sh`` script will pre-processes the collected records
+   and generates 3 png files (and 3 pre-processing cache files) per STATS
+   file:
+   - Slabcache Totals: FOO_STATS-totals.png
+   - Slabs sorted by size: FOO_STATS-slabs-by-size.png
+   - Slabs sorted by loss: FOO_STATS-slabs-by-loss.png
+
+Another use case, when ``slabinfo-gnuplot.sh`` can be useful, is when you
+need to compare slabs' behaviour "prior to" and "after" some code
+modification.  To help you out there, ``slabinfo-gnuplot.sh`` script
+can 'merge' the `Slabcache Totals` sections from different
+measurements. To visually compare N plots:
+
+a) Collect as many STATS1, STATS2, .. STATSN files as you need::
+
+	while [ 1 ]; do slabinfo -X >> STATS<X>; sleep 1; done
+
+b) Pre-process those STATS files::
+
+	slabinfo-gnuplot.sh STATS1 STATS2 .. STATSN
+
+c) Execute ``slabinfo-gnuplot.sh`` in '-t' mode, passing all of the
+   generated pre-processed \*-totals::
+
+	slabinfo-gnuplot.sh -t STATS1-totals STATS2-totals .. STATSN-totals
+
+   This will produce a single plot (png file).
+
+   Plots, expectedly, can be large so some fluctuations or small spikes
+   can go unnoticed. To deal with that, ``slabinfo-gnuplot.sh`` has two
+   options to 'zoom-in'/'zoom-out':
+
+   a) ``-s %d,%d`` -- overwrites the default image width and heigh
+   b) ``-r %d,%d`` -- specifies a range of samples to use (for example,
+      in ``slabinfo -X >> FOO_STATS; sleep 1;`` case, using a ``-r
+      40,60`` range will plot only samples collected between 40th and
+      60th seconds).
+
+Christoph Lameter, May 30, 2007
+Sergey Senozhatsky, October 23, 2015
diff --git a/Documentation/vm/slub.txt b/Documentation/vm/slub.txt
deleted file mode 100644
index 84652419bff2..000000000000
--- a/Documentation/vm/slub.txt
+++ /dev/null
@@ -1,342 +0,0 @@
-Short users guide for SLUB
---------------------------
-
-The basic philosophy of SLUB is very different from SLAB. SLAB
-requires rebuilding the kernel to activate debug options for all
-slab caches. SLUB always includes full debugging but it is off by default.
-SLUB can enable debugging only for selected slabs in order to avoid
-an impact on overall system performance which may make a bug more
-difficult to find.
-
-In order to switch debugging on one can add an option "slub_debug"
-to the kernel command line. That will enable full debugging for
-all slabs.
-
-Typically one would then use the "slabinfo" command to get statistical
-data and perform operation on the slabs. By default slabinfo only lists
-slabs that have data in them. See "slabinfo -h" for more options when
-running the command. slabinfo can be compiled with
-
-gcc -o slabinfo tools/vm/slabinfo.c
-
-Some of the modes of operation of slabinfo require that slub debugging
-be enabled on the command line. F.e. no tracking information will be
-available without debugging on and validation can only partially
-be performed if debugging was not switched on.
-
-Some more sophisticated uses of slub_debug:
--------------------------------------------
-
-Parameters may be given to slub_debug. If none is specified then full
-debugging is enabled. Format:
-
-slub_debug=<Debug-Options>       Enable options for all slabs
-slub_debug=<Debug-Options>,<slab name>
-				Enable options only for select slabs
-
-Possible debug options are
-	F		Sanity checks on (enables SLAB_DEBUG_CONSISTENCY_CHECKS
-			Sorry SLAB legacy issues)
-	Z		Red zoning
-	P		Poisoning (object and padding)
-	U		User tracking (free and alloc)
-	T		Trace (please only use on single slabs)
-	A		Toggle failslab filter mark for the cache
-	O		Switch debugging off for caches that would have
-			caused higher minimum slab orders
-	-		Switch all debugging off (useful if the kernel is
-			configured with CONFIG_SLUB_DEBUG_ON)
-
-F.e. in order to boot just with sanity checks and red zoning one would specify:
-
-	slub_debug=FZ
-
-Trying to find an issue in the dentry cache? Try
-
-	slub_debug=,dentry
-
-to only enable debugging on the dentry cache.
-
-Red zoning and tracking may realign the slab.  We can just apply sanity checks
-to the dentry cache with
-
-	slub_debug=F,dentry
-
-Debugging options may require the minimum possible slab order to increase as
-a result of storing the metadata (for example, caches with PAGE_SIZE object
-sizes).  This has a higher liklihood of resulting in slab allocation errors
-in low memory situations or if there's high fragmentation of memory.  To
-switch off debugging for such caches by default, use
-
-	slub_debug=O
-
-In case you forgot to enable debugging on the kernel command line: It is
-possible to enable debugging manually when the kernel is up. Look at the
-contents of:
-
-/sys/kernel/slab/<slab name>/
-
-Look at the writable files. Writing 1 to them will enable the
-corresponding debug option. All options can be set on a slab that does
-not contain objects. If the slab already contains objects then sanity checks
-and tracing may only be enabled. The other options may cause the realignment
-of objects.
-
-Careful with tracing: It may spew out lots of information and never stop if
-used on the wrong slab.
-
-Slab merging
-------------
-
-If no debug options are specified then SLUB may merge similar slabs together
-in order to reduce overhead and increase cache hotness of objects.
-slabinfo -a displays which slabs were merged together.
-
-Slab validation
----------------
-
-SLUB can validate all object if the kernel was booted with slub_debug. In
-order to do so you must have the slabinfo tool. Then you can do
-
-slabinfo -v
-
-which will test all objects. Output will be generated to the syslog.
-
-This also works in a more limited way if boot was without slab debug.
-In that case slabinfo -v simply tests all reachable objects. Usually
-these are in the cpu slabs and the partial slabs. Full slabs are not
-tracked by SLUB in a non debug situation.
-
-Getting more performance
-------------------------
-
-To some degree SLUB's performance is limited by the need to take the
-list_lock once in a while to deal with partial slabs. That overhead is
-governed by the order of the allocation for each slab. The allocations
-can be influenced by kernel parameters:
-
-slub_min_objects=x		(default 4)
-slub_min_order=x		(default 0)
-slub_max_order=x		(default 3 (PAGE_ALLOC_COSTLY_ORDER))
-
-slub_min_objects allows to specify how many objects must at least fit
-into one slab in order for the allocation order to be acceptable.
-In general slub will be able to perform this number of allocations
-on a slab without consulting centralized resources (list_lock) where
-contention may occur.
-
-slub_min_order specifies a minim order of slabs. A similar effect like
-slub_min_objects.
-
-slub_max_order specified the order at which slub_min_objects should no
-longer be checked. This is useful to avoid SLUB trying to generate
-super large order pages to fit slub_min_objects of a slab cache with
-large object sizes into one high order page. Setting command line
-parameter debug_guardpage_minorder=N (N > 0), forces setting
-slub_max_order to 0, what cause minimum possible order of slabs
-allocation.
-
-SLUB Debug output
------------------
-
-Here is a sample of slub debug output:
-
-====================================================================
-BUG kmalloc-8: Redzone overwritten
---------------------------------------------------------------------
-
-INFO: 0xc90f6d28-0xc90f6d2b. First byte 0x00 instead of 0xcc
-INFO: Slab 0xc528c530 flags=0x400000c3 inuse=61 fp=0xc90f6d58
-INFO: Object 0xc90f6d20 @offset=3360 fp=0xc90f6d58
-INFO: Allocated in get_modalias+0x61/0xf5 age=53 cpu=1 pid=554
-
-Bytes b4 0xc90f6d10:  00 00 00 00 00 00 00 00 5a 5a 5a 5a 5a 5a 5a 5a ........ZZZZZZZZ
-  Object 0xc90f6d20:  31 30 31 39 2e 30 30 35                         1019.005
- Redzone 0xc90f6d28:  00 cc cc cc                                     .
- Padding 0xc90f6d50:  5a 5a 5a 5a 5a 5a 5a 5a                         ZZZZZZZZ
-
-  [<c010523d>] dump_trace+0x63/0x1eb
-  [<c01053df>] show_trace_log_lvl+0x1a/0x2f
-  [<c010601d>] show_trace+0x12/0x14
-  [<c0106035>] dump_stack+0x16/0x18
-  [<c017e0fa>] object_err+0x143/0x14b
-  [<c017e2cc>] check_object+0x66/0x234
-  [<c017eb43>] __slab_free+0x239/0x384
-  [<c017f446>] kfree+0xa6/0xc6
-  [<c02e2335>] get_modalias+0xb9/0xf5
-  [<c02e23b7>] dmi_dev_uevent+0x27/0x3c
-  [<c027866a>] dev_uevent+0x1ad/0x1da
-  [<c0205024>] kobject_uevent_env+0x20a/0x45b
-  [<c020527f>] kobject_uevent+0xa/0xf
-  [<c02779f1>] store_uevent+0x4f/0x58
-  [<c027758e>] dev_attr_store+0x29/0x2f
-  [<c01bec4f>] sysfs_write_file+0x16e/0x19c
-  [<c0183ba7>] vfs_write+0xd1/0x15a
-  [<c01841d7>] sys_write+0x3d/0x72
-  [<c0104112>] sysenter_past_esp+0x5f/0x99
-  [<b7f7b410>] 0xb7f7b410
-  =======================
-
-FIX kmalloc-8: Restoring Redzone 0xc90f6d28-0xc90f6d2b=0xcc
-
-If SLUB encounters a corrupted object (full detection requires the kernel
-to be booted with slub_debug) then the following output will be dumped
-into the syslog:
-
-1. Description of the problem encountered
-
-This will be a message in the system log starting with
-
-===============================================
-BUG <slab cache affected>: <What went wrong>
------------------------------------------------
-
-INFO: <corruption start>-<corruption_end> <more info>
-INFO: Slab <address> <slab information>
-INFO: Object <address> <object information>
-INFO: Allocated in <kernel function> age=<jiffies since alloc> cpu=<allocated by
-	cpu> pid=<pid of the process>
-INFO: Freed in <kernel function> age=<jiffies since free> cpu=<freed by cpu>
-	 pid=<pid of the process>
-
-(Object allocation / free information is only available if SLAB_STORE_USER is
-set for the slab. slub_debug sets that option)
-
-2. The object contents if an object was involved.
-
-Various types of lines can follow the BUG SLUB line:
-
-Bytes b4 <address> : <bytes>
-	Shows a few bytes before the object where the problem was detected.
-	Can be useful if the corruption does not stop with the start of the
-	object.
-
-Object <address> : <bytes>
-	The bytes of the object. If the object is inactive then the bytes
-	typically contain poison values. Any non-poison value shows a
-	corruption by a write after free.
-
-Redzone <address> : <bytes>
-	The Redzone following the object. The Redzone is used to detect
-	writes after the object. All bytes should always have the same
-	value. If there is any deviation then it is due to a write after
-	the object boundary.
-
-	(Redzone information is only available if SLAB_RED_ZONE is set.
-	slub_debug sets that option)
-
-Padding <address> : <bytes>
-	Unused data to fill up the space in order to get the next object
-	properly aligned. In the debug case we make sure that there are
-	at least 4 bytes of padding. This allows the detection of writes
-	before the object.
-
-3. A stackdump
-
-The stackdump describes the location where the error was detected. The cause
-of the corruption is may be more likely found by looking at the function that
-allocated or freed the object.
-
-4. Report on how the problem was dealt with in order to ensure the continued
-operation of the system.
-
-These are messages in the system log beginning with
-
-FIX <slab cache affected>: <corrective action taken>
-
-In the above sample SLUB found that the Redzone of an active object has
-been overwritten. Here a string of 8 characters was written into a slab that
-has the length of 8 characters. However, a 8 character string needs a
-terminating 0. That zero has overwritten the first byte of the Redzone field.
-After reporting the details of the issue encountered the FIX SLUB message
-tells us that SLUB has restored the Redzone to its proper value and then
-system operations continue.
-
-Emergency operations:
----------------------
-
-Minimal debugging (sanity checks alone) can be enabled by booting with
-
-	slub_debug=F
-
-This will be generally be enough to enable the resiliency features of slub
-which will keep the system running even if a bad kernel component will
-keep corrupting objects. This may be important for production systems.
-Performance will be impacted by the sanity checks and there will be a
-continual stream of error messages to the syslog but no additional memory
-will be used (unlike full debugging).
-
-No guarantees. The kernel component still needs to be fixed. Performance
-may be optimized further by locating the slab that experiences corruption
-and enabling debugging only for that cache
-
-I.e.
-
-	slub_debug=F,dentry
-
-If the corruption occurs by writing after the end of the object then it
-may be advisable to enable a Redzone to avoid corrupting the beginning
-of other objects.
-
-	slub_debug=FZ,dentry
-
-Extended slabinfo mode and plotting
------------------------------------
-
-The slabinfo tool has a special 'extended' ('-X') mode that includes:
- - Slabcache Totals
- - Slabs sorted by size (up to -N <num> slabs, default 1)
- - Slabs sorted by loss (up to -N <num> slabs, default 1)
-
-Additionally, in this mode slabinfo does not dynamically scale sizes (G/M/K)
-and reports everything in bytes (this functionality is also available to
-other slabinfo modes via '-B' option) which makes reporting more precise and
-accurate. Moreover, in some sense the `-X' mode also simplifies the analysis
-of slabs' behaviour, because its output can be plotted using the
-slabinfo-gnuplot.sh script. So it pushes the analysis from looking through
-the numbers (tons of numbers) to something easier -- visual analysis.
-
-To generate plots:
-a) collect slabinfo extended records, for example:
-
-  while [ 1 ]; do slabinfo -X >> FOO_STATS; sleep 1; done
-
-b) pass stats file(-s) to slabinfo-gnuplot.sh script:
-  slabinfo-gnuplot.sh FOO_STATS [FOO_STATS2 .. FOO_STATSN]
-
-The slabinfo-gnuplot.sh script will pre-processes the collected records
-and generates 3 png files (and 3 pre-processing cache files) per STATS
-file:
- - Slabcache Totals: FOO_STATS-totals.png
- - Slabs sorted by size: FOO_STATS-slabs-by-size.png
- - Slabs sorted by loss: FOO_STATS-slabs-by-loss.png
-
-Another use case, when slabinfo-gnuplot can be useful, is when you need
-to compare slabs' behaviour "prior to" and "after" some code modification.
-To help you out there, slabinfo-gnuplot.sh script can 'merge' the
-`Slabcache Totals` sections from different measurements. To visually
-compare N plots:
-
-a) Collect as many STATS1, STATS2, .. STATSN files as you need
-  while [ 1 ]; do slabinfo -X >> STATS<X>; sleep 1; done
-
-b) Pre-process those STATS files
-  slabinfo-gnuplot.sh STATS1 STATS2 .. STATSN
-
-c) Execute slabinfo-gnuplot.sh in '-t' mode, passing all of the
-generated pre-processed *-totals
-  slabinfo-gnuplot.sh -t STATS1-totals STATS2-totals .. STATSN-totals
-
-This will produce a single plot (png file).
-
-Plots, expectedly, can be large so some fluctuations or small spikes
-can go unnoticed. To deal with that, `slabinfo-gnuplot.sh' has two
-options to 'zoom-in'/'zoom-out':
- a) -s %d,%d  overwrites the default image width and heigh
- b) -r %d,%d  specifies a range of samples to use (for example,
-              in `slabinfo -X >> FOO_STATS; sleep 1;' case, using
-              a "-r 40,60" range will plot only samples collected
-              between 40th and 60th seconds).
-
-Christoph Lameter, May 30, 2007
-Sergey Senozhatsky, October 23, 2015
diff --git a/Documentation/vm/soft-dirty.txt b/Documentation/vm/soft-dirty.txt
deleted file mode 100644
index 55684d11a1e8..000000000000
--- a/Documentation/vm/soft-dirty.txt
+++ /dev/null
@@ -1,43 +0,0 @@
-                            SOFT-DIRTY PTEs
-
-  The soft-dirty is a bit on a PTE which helps to track which pages a task
-writes to. In order to do this tracking one should
-
-  1. Clear soft-dirty bits from the task's PTEs.
-
-     This is done by writing "4" into the /proc/PID/clear_refs file of the
-     task in question.
-
-  2. Wait some time.
-
-  3. Read soft-dirty bits from the PTEs.
-
-     This is done by reading from the /proc/PID/pagemap. The bit 55 of the
-     64-bit qword is the soft-dirty one. If set, the respective PTE was
-     written to since step 1.
-
-
-  Internally, to do this tracking, the writable bit is cleared from PTEs
-when the soft-dirty bit is cleared. So, after this, when the task tries to
-modify a page at some virtual address the #PF occurs and the kernel sets
-the soft-dirty bit on the respective PTE.
-
-  Note, that although all the task's address space is marked as r/o after the
-soft-dirty bits clear, the #PF-s that occur after that are processed fast.
-This is so, since the pages are still mapped to physical memory, and thus all
-the kernel does is finds this fact out and puts both writable and soft-dirty
-bits on the PTE.
-
-  While in most cases tracking memory changes by #PF-s is more than enough
-there is still a scenario when we can lose soft dirty bits -- a task
-unmaps a previously mapped memory region and then maps a new one at exactly
-the same place. When unmap is called, the kernel internally clears PTE values
-including soft dirty bits. To notify user space application about such
-memory region renewal the kernel always marks new memory regions (and
-expanded regions) as soft dirty.
-
-  This feature is actively used by the checkpoint-restore project. You
-can find more details about it on http://criu.org
-
-
--- Pavel Emelyanov, Apr 9, 2013
diff --git a/Documentation/vm/split_page_table_lock b/Documentation/vm/split_page_table_lock
deleted file mode 100644
index 62842a857dab..000000000000
--- a/Documentation/vm/split_page_table_lock
+++ /dev/null
@@ -1,94 +0,0 @@
-Split page table lock
-=====================
-
-Originally, mm->page_table_lock spinlock protected all page tables of the
-mm_struct. But this approach leads to poor page fault scalability of
-multi-threaded applications due high contention on the lock. To improve
-scalability, split page table lock was introduced.
-
-With split page table lock we have separate per-table lock to serialize
-access to the table. At the moment we use split lock for PTE and PMD
-tables. Access to higher level tables protected by mm->page_table_lock.
-
-There are helpers to lock/unlock a table and other accessor functions:
- - pte_offset_map_lock()
-	maps pte and takes PTE table lock, returns pointer to the taken
-	lock;
- - pte_unmap_unlock()
-	unlocks and unmaps PTE table;
- - pte_alloc_map_lock()
-	allocates PTE table if needed and take the lock, returns pointer
-	to taken lock or NULL if allocation failed;
- - pte_lockptr()
-	returns pointer to PTE table lock;
- - pmd_lock()
-	takes PMD table lock, returns pointer to taken lock;
- - pmd_lockptr()
-	returns pointer to PMD table lock;
-
-Split page table lock for PTE tables is enabled compile-time if
-CONFIG_SPLIT_PTLOCK_CPUS (usually 4) is less or equal to NR_CPUS.
-If split lock is disabled, all tables guaded by mm->page_table_lock.
-
-Split page table lock for PMD tables is enabled, if it's enabled for PTE
-tables and the architecture supports it (see below).
-
-Hugetlb and split page table lock
----------------------------------
-
-Hugetlb can support several page sizes. We use split lock only for PMD
-level, but not for PUD.
-
-Hugetlb-specific helpers:
- - huge_pte_lock()
-	takes pmd split lock for PMD_SIZE page, mm->page_table_lock
-	otherwise;
- - huge_pte_lockptr()
-	returns pointer to table lock;
-
-Support of split page table lock by an architecture
----------------------------------------------------
-
-There's no need in special enabling of PTE split page table lock:
-everything required is done by pgtable_page_ctor() and pgtable_page_dtor(),
-which must be called on PTE table allocation / freeing.
-
-Make sure the architecture doesn't use slab allocator for page table
-allocation: slab uses page->slab_cache for its pages.
-This field shares storage with page->ptl.
-
-PMD split lock only makes sense if you have more than two page table
-levels.
-
-PMD split lock enabling requires pgtable_pmd_page_ctor() call on PMD table
-allocation and pgtable_pmd_page_dtor() on freeing.
-
-Allocation usually happens in pmd_alloc_one(), freeing in pmd_free() and
-pmd_free_tlb(), but make sure you cover all PMD table allocation / freeing
-paths: i.e X86_PAE preallocate few PMDs on pgd_alloc().
-
-With everything in place you can set CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK.
-
-NOTE: pgtable_page_ctor() and pgtable_pmd_page_ctor() can fail -- it must
-be handled properly.
-
-page->ptl
----------
-
-page->ptl is used to access split page table lock, where 'page' is struct
-page of page containing the table. It shares storage with page->private
-(and few other fields in union).
-
-To avoid increasing size of struct page and have best performance, we use a
-trick:
- - if spinlock_t fits into long, we use page->ptr as spinlock, so we
-   can avoid indirect access and save a cache line.
- - if size of spinlock_t is bigger then size of long, we use page->ptl as
-   pointer to spinlock_t and allocate it dynamically. This allows to use
-   split lock with enabled DEBUG_SPINLOCK or DEBUG_LOCK_ALLOC, but costs
-   one more cache line for indirect access;
-
-The spinlock_t allocated in pgtable_page_ctor() for PTE table and in
-pgtable_pmd_page_ctor() for PMD table.
-
-Please, never access page->ptl directly -- use appropriate helper.
diff --git a/Documentation/vm/split_page_table_lock.rst b/Documentation/vm/split_page_table_lock.rst
new file mode 100644
index 000000000000..889b00be469f
--- /dev/null
+++ b/Documentation/vm/split_page_table_lock.rst
@@ -0,0 +1,100 @@
+.. _split_page_table_lock:
+
+=====================
+Split page table lock
+=====================
+
+Originally, mm->page_table_lock spinlock protected all page tables of the
+mm_struct. But this approach leads to poor page fault scalability of
+multi-threaded applications due high contention on the lock. To improve
+scalability, split page table lock was introduced.
+
+With split page table lock we have separate per-table lock to serialize
+access to the table. At the moment we use split lock for PTE and PMD
+tables. Access to higher level tables protected by mm->page_table_lock.
+
+There are helpers to lock/unlock a table and other accessor functions:
+
+ - pte_offset_map_lock()
+	maps pte and takes PTE table lock, returns pointer to the taken
+	lock;
+ - pte_unmap_unlock()
+	unlocks and unmaps PTE table;
+ - pte_alloc_map_lock()
+	allocates PTE table if needed and take the lock, returns pointer
+	to taken lock or NULL if allocation failed;
+ - pte_lockptr()
+	returns pointer to PTE table lock;
+ - pmd_lock()
+	takes PMD table lock, returns pointer to taken lock;
+ - pmd_lockptr()
+	returns pointer to PMD table lock;
+
+Split page table lock for PTE tables is enabled compile-time if
+CONFIG_SPLIT_PTLOCK_CPUS (usually 4) is less or equal to NR_CPUS.
+If split lock is disabled, all tables guaded by mm->page_table_lock.
+
+Split page table lock for PMD tables is enabled, if it's enabled for PTE
+tables and the architecture supports it (see below).
+
+Hugetlb and split page table lock
+=================================
+
+Hugetlb can support several page sizes. We use split lock only for PMD
+level, but not for PUD.
+
+Hugetlb-specific helpers:
+
+ - huge_pte_lock()
+	takes pmd split lock for PMD_SIZE page, mm->page_table_lock
+	otherwise;
+ - huge_pte_lockptr()
+	returns pointer to table lock;
+
+Support of split page table lock by an architecture
+===================================================
+
+There's no need in special enabling of PTE split page table lock:
+everything required is done by pgtable_page_ctor() and pgtable_page_dtor(),
+which must be called on PTE table allocation / freeing.
+
+Make sure the architecture doesn't use slab allocator for page table
+allocation: slab uses page->slab_cache for its pages.
+This field shares storage with page->ptl.
+
+PMD split lock only makes sense if you have more than two page table
+levels.
+
+PMD split lock enabling requires pgtable_pmd_page_ctor() call on PMD table
+allocation and pgtable_pmd_page_dtor() on freeing.
+
+Allocation usually happens in pmd_alloc_one(), freeing in pmd_free() and
+pmd_free_tlb(), but make sure you cover all PMD table allocation / freeing
+paths: i.e X86_PAE preallocate few PMDs on pgd_alloc().
+
+With everything in place you can set CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK.
+
+NOTE: pgtable_page_ctor() and pgtable_pmd_page_ctor() can fail -- it must
+be handled properly.
+
+page->ptl
+=========
+
+page->ptl is used to access split page table lock, where 'page' is struct
+page of page containing the table. It shares storage with page->private
+(and few other fields in union).
+
+To avoid increasing size of struct page and have best performance, we use a
+trick:
+
+ - if spinlock_t fits into long, we use page->ptr as spinlock, so we
+   can avoid indirect access and save a cache line.
+ - if size of spinlock_t is bigger then size of long, we use page->ptl as
+   pointer to spinlock_t and allocate it dynamically. This allows to use
+   split lock with enabled DEBUG_SPINLOCK or DEBUG_LOCK_ALLOC, but costs
+   one more cache line for indirect access;
+
+The spinlock_t allocated in pgtable_page_ctor() for PTE table and in
+pgtable_pmd_page_ctor() for PMD table.
+
+Please, never access page->ptl directly -- use appropriate helper.
diff --git a/Documentation/vm/swap_numa.rst b/Documentation/vm/swap_numa.rst
new file mode 100644
index 000000000000..e0466f2db8fa
--- /dev/null
+++ b/Documentation/vm/swap_numa.rst
@@ -0,0 +1,80 @@
+.. _swap_numa:
+
+===========================================
+Automatically bind swap device to numa node
+===========================================
+
+If the system has more than one swap device and swap device has the node
+information, we can make use of this information to decide which swap
+device to use in get_swap_pages() to get better performance.
+
+
+How to use this feature
+=======================
+
+Swap device has priority and that decides the order of it to be used. To make
+use of automatically binding, there is no need to manipulate priority settings
+for swap devices. e.g. on a 2 node machine, assume 2 swap devices swapA and
+swapB, with swapA attached to node 0 and swapB attached to node 1, are going
+to be swapped on. Simply swapping them on by doing::
+
+	# swapon /dev/swapA
+	# swapon /dev/swapB
+
+Then node 0 will use the two swap devices in the order of swapA then swapB and
+node 1 will use the two swap devices in the order of swapB then swapA. Note
+that the order of them being swapped on doesn't matter.
+
+A more complex example on a 4 node machine. Assume 6 swap devices are going to
+be swapped on: swapA and swapB are attached to node 0, swapC is attached to
+node 1, swapD and swapE are attached to node 2 and swapF is attached to node3.
+The way to swap them on is the same as above::
+
+	# swapon /dev/swapA
+	# swapon /dev/swapB
+	# swapon /dev/swapC
+	# swapon /dev/swapD
+	# swapon /dev/swapE
+	# swapon /dev/swapF
+
+Then node 0 will use them in the order of::
+
+	swapA/swapB -> swapC -> swapD -> swapE -> swapF
+
+swapA and swapB will be used in a round robin mode before any other swap device.
+
+node 1 will use them in the order of::
+
+	swapC -> swapA -> swapB -> swapD -> swapE -> swapF
+
+node 2 will use them in the order of::
+
+	swapD/swapE -> swapA -> swapB -> swapC -> swapF
+
+Similaly, swapD and swapE will be used in a round robin mode before any
+other swap devices.
+
+node 3 will use them in the order of::
+
+	swapF -> swapA -> swapB -> swapC -> swapD -> swapE
+
+
+Implementation details
+======================
+
+The current code uses a priority based list, swap_avail_list, to decide
+which swap device to use and if multiple swap devices share the same
+priority, they are used round robin. This change here replaces the single
+global swap_avail_list with a per-numa-node list, i.e. for each numa node,
+it sees its own priority based list of available swap devices. Swap
+device's priority can be promoted on its matching node's swap_avail_list.
+
+The current swap device's priority is set as: user can set a >=0 value,
+or the system will pick one starting from -1 then downwards. The priority
+value in the swap_avail_list is the negated value of the swap device's
+due to plist being sorted from low to high. The new policy doesn't change
+the semantics for priority >=0 cases, the previous starting from -1 then
+downwards now becomes starting from -2 then downwards and -1 is reserved
+as the promoted value. So if multiple swap devices are attached to the same
+node, they will all be promoted to priority -1 on that node's plist and will
+be used round robin before any other swap devices.
diff --git a/Documentation/vm/swap_numa.txt b/Documentation/vm/swap_numa.txt
deleted file mode 100644
index d5960c9124f5..000000000000
--- a/Documentation/vm/swap_numa.txt
+++ /dev/null
@@ -1,69 +0,0 @@
-Automatically bind swap device to numa node
--------------------------------------------
-
-If the system has more than one swap device and swap device has the node
-information, we can make use of this information to decide which swap
-device to use in get_swap_pages() to get better performance.
-
-
-How to use this feature
------------------------
-
-Swap device has priority and that decides the order of it to be used. To make
-use of automatically binding, there is no need to manipulate priority settings
-for swap devices. e.g. on a 2 node machine, assume 2 swap devices swapA and
-swapB, with swapA attached to node 0 and swapB attached to node 1, are going
-to be swapped on. Simply swapping them on by doing:
-# swapon /dev/swapA
-# swapon /dev/swapB
-
-Then node 0 will use the two swap devices in the order of swapA then swapB and
-node 1 will use the two swap devices in the order of swapB then swapA. Note
-that the order of them being swapped on doesn't matter.
-
-A more complex example on a 4 node machine. Assume 6 swap devices are going to
-be swapped on: swapA and swapB are attached to node 0, swapC is attached to
-node 1, swapD and swapE are attached to node 2 and swapF is attached to node3.
-The way to swap them on is the same as above:
-# swapon /dev/swapA
-# swapon /dev/swapB
-# swapon /dev/swapC
-# swapon /dev/swapD
-# swapon /dev/swapE
-# swapon /dev/swapF
-
-Then node 0 will use them in the order of:
-swapA/swapB -> swapC -> swapD -> swapE -> swapF
-swapA and swapB will be used in a round robin mode before any other swap device.
-
-node 1 will use them in the order of:
-swapC -> swapA -> swapB -> swapD -> swapE -> swapF
-
-node 2 will use them in the order of:
-swapD/swapE -> swapA -> swapB -> swapC -> swapF
-Similaly, swapD and swapE will be used in a round robin mode before any
-other swap devices.
-
-node 3 will use them in the order of:
-swapF -> swapA -> swapB -> swapC -> swapD -> swapE
-
-
-Implementation details
-----------------------
-
-The current code uses a priority based list, swap_avail_list, to decide
-which swap device to use and if multiple swap devices share the same
-priority, they are used round robin. This change here replaces the single
-global swap_avail_list with a per-numa-node list, i.e. for each numa node,
-it sees its own priority based list of available swap devices. Swap
-device's priority can be promoted on its matching node's swap_avail_list.
-
-The current swap device's priority is set as: user can set a >=0 value,
-or the system will pick one starting from -1 then downwards. The priority
-value in the swap_avail_list is the negated value of the swap device's
-due to plist being sorted from low to high. The new policy doesn't change
-the semantics for priority >=0 cases, the previous starting from -1 then
-downwards now becomes starting from -2 then downwards and -1 is reserved
-as the promoted value. So if multiple swap devices are attached to the same
-node, they will all be promoted to priority -1 on that node's plist and will
-be used round robin before any other swap devices.
diff --git a/Documentation/vm/transhuge.rst b/Documentation/vm/transhuge.rst
new file mode 100644
index 000000000000..a8cf6809e36e
--- /dev/null
+++ b/Documentation/vm/transhuge.rst
@@ -0,0 +1,197 @@
+.. _transhuge:
+
+============================
+Transparent Hugepage Support
+============================
+
+This document describes design principles Transparent Hugepage (THP)
+Support and its interaction with other parts of the memory management.
+
+Design principles
+=================
+
+- "graceful fallback": mm components which don't have transparent hugepage
+  knowledge fall back to breaking huge pmd mapping into table of ptes and,
+  if necessary, split a transparent hugepage. Therefore these components
+  can continue working on the regular pages or regular pte mappings.
+
+- if a hugepage allocation fails because of memory fragmentation,
+  regular pages should be gracefully allocated instead and mixed in
+  the same vma without any failure or significant delay and without
+  userland noticing
+
+- if some task quits and more hugepages become available (either
+  immediately in the buddy or through the VM), guest physical memory
+  backed by regular pages should be relocated on hugepages
+  automatically (with khugepaged)
+
+- it doesn't require memory reservation and in turn it uses hugepages
+  whenever possible (the only possible reservation here is kernelcore=
+  to avoid unmovable pages to fragment all the memory but such a tweak
+  is not specific to transparent hugepage support and it's a generic
+  feature that applies to all dynamic high order allocations in the
+  kernel)
+
+get_user_pages and follow_page
+==============================
+
+get_user_pages and follow_page if run on a hugepage, will return the
+head or tail pages as usual (exactly as they would do on
+hugetlbfs). Most gup users will only care about the actual physical
+address of the page and its temporary pinning to release after the I/O
+is complete, so they won't ever notice the fact the page is huge. But
+if any driver is going to mangle over the page structure of the tail
+page (like for checking page->mapping or other bits that are relevant
+for the head page and not the tail page), it should be updated to jump
+to check head page instead. Taking reference on any head/tail page would
+prevent page from being split by anyone.
+
+.. note::
+   these aren't new constraints to the GUP API, and they match the
+   same constrains that applies to hugetlbfs too, so any driver capable
+   of handling GUP on hugetlbfs will also work fine on transparent
+   hugepage backed mappings.
+
+In case you can't handle compound pages if they're returned by
+follow_page, the FOLL_SPLIT bit can be specified as parameter to
+follow_page, so that it will split the hugepages before returning
+them. Migration for example passes FOLL_SPLIT as parameter to
+follow_page because it's not hugepage aware and in fact it can't work
+at all on hugetlbfs (but it instead works fine on transparent
+hugepages thanks to FOLL_SPLIT). migration simply can't deal with
+hugepages being returned (as it's not only checking the pfn of the
+page and pinning it during the copy but it pretends to migrate the
+memory in regular page sizes and with regular pte/pmd mappings).
+
+Graceful fallback
+=================
+
+Code walking pagetables but unaware about huge pmds can simply call
+split_huge_pmd(vma, pmd, addr) where the pmd is the one returned by
+pmd_offset. It's trivial to make the code transparent hugepage aware
+by just grepping for "pmd_offset" and adding split_huge_pmd where
+missing after pmd_offset returns the pmd. Thanks to the graceful
+fallback design, with a one liner change, you can avoid to write
+hundred if not thousand of lines of complex code to make your code
+hugepage aware.
+
+If you're not walking pagetables but you run into a physical hugepage
+but you can't handle it natively in your code, you can split it by
+calling split_huge_page(page). This is what the Linux VM does before
+it tries to swapout the hugepage for example. split_huge_page() can fail
+if the page is pinned and you must handle this correctly.
+
+Example to make mremap.c transparent hugepage aware with a one liner
+change::
+
+	diff --git a/mm/mremap.c b/mm/mremap.c
+	--- a/mm/mremap.c
+	+++ b/mm/mremap.c
+	@@ -41,6 +41,7 @@ static pmd_t *get_old_pmd(struct mm_stru
+			return NULL;
+
+		pmd = pmd_offset(pud, addr);
+	+	split_huge_pmd(vma, pmd, addr);
+		if (pmd_none_or_clear_bad(pmd))
+			return NULL;
+
+Locking in hugepage aware code
+==============================
+
+We want as much code as possible hugepage aware, as calling
+split_huge_page() or split_huge_pmd() has a cost.
+
+To make pagetable walks huge pmd aware, all you need to do is to call
+pmd_trans_huge() on the pmd returned by pmd_offset. You must hold the
+mmap_sem in read (or write) mode to be sure an huge pmd cannot be
+created from under you by khugepaged (khugepaged collapse_huge_page
+takes the mmap_sem in write mode in addition to the anon_vma lock). If
+pmd_trans_huge returns false, you just fallback in the old code
+paths. If instead pmd_trans_huge returns true, you have to take the
+page table lock (pmd_lock()) and re-run pmd_trans_huge. Taking the
+page table lock will prevent the huge pmd to be converted into a
+regular pmd from under you (split_huge_pmd can run in parallel to the
+pagetable walk). If the second pmd_trans_huge returns false, you
+should just drop the page table lock and fallback to the old code as
+before. Otherwise you can proceed to process the huge pmd and the
+hugepage natively. Once finished you can drop the page table lock.
+
+Refcounts and transparent huge pages
+====================================
+
+Refcounting on THP is mostly consistent with refcounting on other compound
+pages:
+
+  - get_page()/put_page() and GUP operate in head page's ->_refcount.
+
+  - ->_refcount in tail pages is always zero: get_page_unless_zero() never
+    succeed on tail pages.
+
+  - map/unmap of the pages with PTE entry increment/decrement ->_mapcount
+    on relevant sub-page of the compound page.
+
+  - map/unmap of the whole compound page accounted in compound_mapcount
+    (stored in first tail page). For file huge pages, we also increment
+    ->_mapcount of all sub-pages in order to have race-free detection of
+    last unmap of subpages.
+
+PageDoubleMap() indicates that the page is *possibly* mapped with PTEs.
+
+For anonymous pages PageDoubleMap() also indicates ->_mapcount in all
+subpages is offset up by one. This additional reference is required to
+get race-free detection of unmap of subpages when we have them mapped with
+both PMDs and PTEs.
+
+This is optimization required to lower overhead of per-subpage mapcount
+tracking. The alternative is alter ->_mapcount in all subpages on each
+map/unmap of the whole compound page.
+
+For anonymous pages, we set PG_double_map when a PMD of the page got split
+for the first time, but still have PMD mapping. The additional references
+go away with last compound_mapcount.
+
+File pages get PG_double_map set on first map of the page with PTE and
+goes away when the page gets evicted from page cache.
+
+split_huge_page internally has to distribute the refcounts in the head
+page to the tail pages before clearing all PG_head/tail bits from the page
+structures. It can be done easily for refcounts taken by page table
+entries. But we don't have enough information on how to distribute any
+additional pins (i.e. from get_user_pages). split_huge_page() fails any
+requests to split pinned huge page: it expects page count to be equal to
+sum of mapcount of all sub-pages plus one (split_huge_page caller must
+have reference for head page).
+
+split_huge_page uses migration entries to stabilize page->_refcount and
+page->_mapcount of anonymous pages. File pages just got unmapped.
+
+We safe against physical memory scanners too: the only legitimate way
+scanner can get reference to a page is get_page_unless_zero().
+
+All tail pages have zero ->_refcount until atomic_add(). This prevents the
+scanner from getting a reference to the tail page up to that point. After the
+atomic_add() we don't care about the ->_refcount value. We already known how
+many references should be uncharged from the head page.
+
+For head page get_page_unless_zero() will succeed and we don't mind. It's
+clear where reference should go after split: it will stay on head page.
+
+Note that split_huge_pmd() doesn't have any limitation on refcounting:
+pmd can be split at any point and never fails.
+
+Partial unmap and deferred_split_huge_page()
+============================================
+
+Unmapping part of THP (with munmap() or other way) is not going to free
+memory immediately. Instead, we detect that a subpage of THP is not in use
+in page_remove_rmap() and queue the THP for splitting if memory pressure
+comes. Splitting will free up unused subpages.
+
+Splitting the page right away is not an option due to locking context in
+the place where we can detect partial unmap. It's also might be
+counterproductive since in many cases partial unmap happens during exit(2) if
+a THP crosses a VMA boundary.
+
+Function deferred_split_huge_page() is used to queue page for splitting.
+The splitting itself will happen when we get memory pressure via shrinker
+interface.
diff --git a/Documentation/vm/transhuge.txt b/Documentation/vm/transhuge.txt
deleted file mode 100644
index 4dde03b44ad1..000000000000
--- a/Documentation/vm/transhuge.txt
+++ /dev/null
@@ -1,527 +0,0 @@
-= Transparent Hugepage Support =
-
-== Objective ==
-
-Performance critical computing applications dealing with large memory
-working sets are already running on top of libhugetlbfs and in turn
-hugetlbfs. Transparent Hugepage Support is an alternative means of
-using huge pages for the backing of virtual memory with huge pages
-that supports the automatic promotion and demotion of page sizes and
-without the shortcomings of hugetlbfs.
-
-Currently it only works for anonymous memory mappings and tmpfs/shmem.
-But in the future it can expand to other filesystems.
-
-The reason applications are running faster is because of two
-factors. The first factor is almost completely irrelevant and it's not
-of significant interest because it'll also have the downside of
-requiring larger clear-page copy-page in page faults which is a
-potentially negative effect. The first factor consists in taking a
-single page fault for each 2M virtual region touched by userland (so
-reducing the enter/exit kernel frequency by a 512 times factor). This
-only matters the first time the memory is accessed for the lifetime of
-a memory mapping. The second long lasting and much more important
-factor will affect all subsequent accesses to the memory for the whole
-runtime of the application. The second factor consist of two
-components: 1) the TLB miss will run faster (especially with
-virtualization using nested pagetables but almost always also on bare
-metal without virtualization) and 2) a single TLB entry will be
-mapping a much larger amount of virtual memory in turn reducing the
-number of TLB misses. With virtualization and nested pagetables the
-TLB can be mapped of larger size only if both KVM and the Linux guest
-are using hugepages but a significant speedup already happens if only
-one of the two is using hugepages just because of the fact the TLB
-miss is going to run faster.
-
-== Design ==
-
-- "graceful fallback": mm components which don't have transparent hugepage
-  knowledge fall back to breaking huge pmd mapping into table of ptes and,
-  if necessary, split a transparent hugepage. Therefore these components
-  can continue working on the regular pages or regular pte mappings.
-
-- if a hugepage allocation fails because of memory fragmentation,
-  regular pages should be gracefully allocated instead and mixed in
-  the same vma without any failure or significant delay and without
-  userland noticing
-
-- if some task quits and more hugepages become available (either
-  immediately in the buddy or through the VM), guest physical memory
-  backed by regular pages should be relocated on hugepages
-  automatically (with khugepaged)
-
-- it doesn't require memory reservation and in turn it uses hugepages
-  whenever possible (the only possible reservation here is kernelcore=
-  to avoid unmovable pages to fragment all the memory but such a tweak
-  is not specific to transparent hugepage support and it's a generic
-  feature that applies to all dynamic high order allocations in the
-  kernel)
-
-Transparent Hugepage Support maximizes the usefulness of free memory
-if compared to the reservation approach of hugetlbfs by allowing all
-unused memory to be used as cache or other movable (or even unmovable
-entities). It doesn't require reservation to prevent hugepage
-allocation failures to be noticeable from userland. It allows paging
-and all other advanced VM features to be available on the
-hugepages. It requires no modifications for applications to take
-advantage of it.
-
-Applications however can be further optimized to take advantage of
-this feature, like for example they've been optimized before to avoid
-a flood of mmap system calls for every malloc(4k). Optimizing userland
-is by far not mandatory and khugepaged already can take care of long
-lived page allocations even for hugepage unaware applications that
-deals with large amounts of memory.
-
-In certain cases when hugepages are enabled system wide, application
-may end up allocating more memory resources. An application may mmap a
-large region but only touch 1 byte of it, in that case a 2M page might
-be allocated instead of a 4k page for no good. This is why it's
-possible to disable hugepages system-wide and to only have them inside
-MADV_HUGEPAGE madvise regions.
-
-Embedded systems should enable hugepages only inside madvise regions
-to eliminate any risk of wasting any precious byte of memory and to
-only run faster.
-
-Applications that gets a lot of benefit from hugepages and that don't
-risk to lose memory by using hugepages, should use
-madvise(MADV_HUGEPAGE) on their critical mmapped regions.
-
-== sysfs ==
-
-Transparent Hugepage Support for anonymous memory can be entirely disabled
-(mostly for debugging purposes) or only enabled inside MADV_HUGEPAGE
-regions (to avoid the risk of consuming more memory resources) or enabled
-system wide. This can be achieved with one of:
-
-echo always >/sys/kernel/mm/transparent_hugepage/enabled
-echo madvise >/sys/kernel/mm/transparent_hugepage/enabled
-echo never >/sys/kernel/mm/transparent_hugepage/enabled
-
-It's also possible to limit defrag efforts in the VM to generate
-anonymous hugepages in case they're not immediately free to madvise
-regions or to never try to defrag memory and simply fallback to regular
-pages unless hugepages are immediately available. Clearly if we spend CPU
-time to defrag memory, we would expect to gain even more by the fact we
-use hugepages later instead of regular pages. This isn't always
-guaranteed, but it may be more likely in case the allocation is for a
-MADV_HUGEPAGE region.
-
-echo always >/sys/kernel/mm/transparent_hugepage/defrag
-echo defer >/sys/kernel/mm/transparent_hugepage/defrag
-echo defer+madvise >/sys/kernel/mm/transparent_hugepage/defrag
-echo madvise >/sys/kernel/mm/transparent_hugepage/defrag
-echo never >/sys/kernel/mm/transparent_hugepage/defrag
-
-"always" means that an application requesting THP will stall on allocation
-failure and directly reclaim pages and compact memory in an effort to
-allocate a THP immediately. This may be desirable for virtual machines
-that benefit heavily from THP use and are willing to delay the VM start
-to utilise them.
-
-"defer" means that an application will wake kswapd in the background
-to reclaim pages and wake kcompactd to compact memory so that THP is
-available in the near future. It's the responsibility of khugepaged
-to then install the THP pages later.
-
-"defer+madvise" will enter direct reclaim and compaction like "always", but
-only for regions that have used madvise(MADV_HUGEPAGE); all other regions
-will wake kswapd in the background to reclaim pages and wake kcompactd to
-compact memory so that THP is available in the near future.
-
-"madvise" will enter direct reclaim like "always" but only for regions
-that are have used madvise(MADV_HUGEPAGE). This is the default behaviour.
-
-"never" should be self-explanatory.
-
-By default kernel tries to use huge zero page on read page fault to
-anonymous mapping. It's possible to disable huge zero page by writing 0
-or enable it back by writing 1:
-
-echo 0 >/sys/kernel/mm/transparent_hugepage/use_zero_page
-echo 1 >/sys/kernel/mm/transparent_hugepage/use_zero_page
-
-Some userspace (such as a test program, or an optimized memory allocation
-library) may want to know the size (in bytes) of a transparent hugepage:
-
-cat /sys/kernel/mm/transparent_hugepage/hpage_pmd_size
-
-khugepaged will be automatically started when
-transparent_hugepage/enabled is set to "always" or "madvise, and it'll
-be automatically shutdown if it's set to "never".
-
-khugepaged runs usually at low frequency so while one may not want to
-invoke defrag algorithms synchronously during the page faults, it
-should be worth invoking defrag at least in khugepaged. However it's
-also possible to disable defrag in khugepaged by writing 0 or enable
-defrag in khugepaged by writing 1:
-
-echo 0 >/sys/kernel/mm/transparent_hugepage/khugepaged/defrag
-echo 1 >/sys/kernel/mm/transparent_hugepage/khugepaged/defrag
-
-You can also control how many pages khugepaged should scan at each
-pass:
-
-/sys/kernel/mm/transparent_hugepage/khugepaged/pages_to_scan
-
-and how many milliseconds to wait in khugepaged between each pass (you
-can set this to 0 to run khugepaged at 100% utilization of one core):
-
-/sys/kernel/mm/transparent_hugepage/khugepaged/scan_sleep_millisecs
-
-and how many milliseconds to wait in khugepaged if there's an hugepage
-allocation failure to throttle the next allocation attempt.
-
-/sys/kernel/mm/transparent_hugepage/khugepaged/alloc_sleep_millisecs
-
-The khugepaged progress can be seen in the number of pages collapsed:
-
-/sys/kernel/mm/transparent_hugepage/khugepaged/pages_collapsed
-
-for each pass:
-
-/sys/kernel/mm/transparent_hugepage/khugepaged/full_scans
-
-max_ptes_none specifies how many extra small pages (that are
-not already mapped) can be allocated when collapsing a group
-of small pages into one large page.
-
-/sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_none
-
-A higher value leads to use additional memory for programs.
-A lower value leads to gain less thp performance. Value of
-max_ptes_none can waste cpu time very little, you can
-ignore it.
-
-max_ptes_swap specifies how many pages can be brought in from
-swap when collapsing a group of pages into a transparent huge page.
-
-/sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_swap
-
-A higher value can cause excessive swap IO and waste
-memory. A lower value can prevent THPs from being
-collapsed, resulting fewer pages being collapsed into
-THPs, and lower memory access performance.
-
-== Boot parameter ==
-
-You can change the sysfs boot time defaults of Transparent Hugepage
-Support by passing the parameter "transparent_hugepage=always" or
-"transparent_hugepage=madvise" or "transparent_hugepage=never"
-(without "") to the kernel command line.
-
-== Hugepages in tmpfs/shmem ==
-
-You can control hugepage allocation policy in tmpfs with mount option
-"huge=". It can have following values:
-
-  - "always":
-    Attempt to allocate huge pages every time we need a new page;
-
-  - "never":
-    Do not allocate huge pages;
-
-  - "within_size":
-    Only allocate huge page if it will be fully within i_size.
-    Also respect fadvise()/madvise() hints;
-
-  - "advise:
-    Only allocate huge pages if requested with fadvise()/madvise();
-
-The default policy is "never".
-
-"mount -o remount,huge= /mountpoint" works fine after mount: remounting
-huge=never will not attempt to break up huge pages at all, just stop more
-from being allocated.
-
-There's also sysfs knob to control hugepage allocation policy for internal
-shmem mount: /sys/kernel/mm/transparent_hugepage/shmem_enabled. The mount
-is used for SysV SHM, memfds, shared anonymous mmaps (of /dev/zero or
-MAP_ANONYMOUS), GPU drivers' DRM objects, Ashmem.
-
-In addition to policies listed above, shmem_enabled allows two further
-values:
-
-  - "deny":
-    For use in emergencies, to force the huge option off from
-    all mounts;
-  - "force":
-    Force the huge option on for all - very useful for testing;
-
-== Need of application restart ==
-
-The transparent_hugepage/enabled values and tmpfs mount option only affect
-future behavior. So to make them effective you need to restart any
-application that could have been using hugepages. This also applies to the
-regions registered in khugepaged.
-
-== Monitoring usage ==
-
-The number of anonymous transparent huge pages currently used by the
-system is available by reading the AnonHugePages field in /proc/meminfo.
-To identify what applications are using anonymous transparent huge pages,
-it is necessary to read /proc/PID/smaps and count the AnonHugePages fields
-for each mapping.
-
-The number of file transparent huge pages mapped to userspace is available
-by reading ShmemPmdMapped and ShmemHugePages fields in /proc/meminfo.
-To identify what applications are mapping file transparent huge pages, it
-is necessary to read /proc/PID/smaps and count the FileHugeMapped fields
-for each mapping.
-
-Note that reading the smaps file is expensive and reading it
-frequently will incur overhead.
-
-There are a number of counters in /proc/vmstat that may be used to
-monitor how successfully the system is providing huge pages for use.
-
-thp_fault_alloc is incremented every time a huge page is successfully
-	allocated to handle a page fault. This applies to both the
-	first time a page is faulted and for COW faults.
-
-thp_collapse_alloc is incremented by khugepaged when it has found
-	a range of pages to collapse into one huge page and has
-	successfully allocated a new huge page to store the data.
-
-thp_fault_fallback is incremented if a page fault fails to allocate
-	a huge page and instead falls back to using small pages.
-
-thp_collapse_alloc_failed is incremented if khugepaged found a range
-	of pages that should be collapsed into one huge page but failed
-	the allocation.
-
-thp_file_alloc is incremented every time a file huge page is successfully
-	allocated.
-
-thp_file_mapped is incremented every time a file huge page is mapped into
-	user address space.
-
-thp_split_page is incremented every time a huge page is split into base
-	pages. This can happen for a variety of reasons but a common
-	reason is that a huge page is old and is being reclaimed.
-	This action implies splitting all PMD the page mapped with.
-
-thp_split_page_failed is incremented if kernel fails to split huge
-	page. This can happen if the page was pinned by somebody.
-
-thp_deferred_split_page is incremented when a huge page is put onto split
-	queue. This happens when a huge page is partially unmapped and
-	splitting it would free up some memory. Pages on split queue are
-	going to be split under memory pressure.
-
-thp_split_pmd is incremented every time a PMD split into table of PTEs.
-	This can happen, for instance, when application calls mprotect() or
-	munmap() on part of huge page. It doesn't split huge page, only
-	page table entry.
-
-thp_zero_page_alloc is incremented every time a huge zero page is
-	successfully allocated. It includes allocations which where
-	dropped due race with other allocation. Note, it doesn't count
-	every map of the huge zero page, only its allocation.
-
-thp_zero_page_alloc_failed is incremented if kernel fails to allocate
-	huge zero page and falls back to using small pages.
-
-As the system ages, allocating huge pages may be expensive as the
-system uses memory compaction to copy data around memory to free a
-huge page for use. There are some counters in /proc/vmstat to help
-monitor this overhead.
-
-compact_stall is incremented every time a process stalls to run
-	memory compaction so that a huge page is free for use.
-
-compact_success is incremented if the system compacted memory and
-	freed a huge page for use.
-
-compact_fail is incremented if the system tries to compact memory
-	but failed.
-
-compact_pages_moved is incremented each time a page is moved. If
-	this value is increasing rapidly, it implies that the system
-	is copying a lot of data to satisfy the huge page allocation.
-	It is possible that the cost of copying exceeds any savings
-	from reduced TLB misses.
-
-compact_pagemigrate_failed is incremented when the underlying mechanism
-	for moving a page failed.
-
-compact_blocks_moved is incremented each time memory compaction examines
-	a huge page aligned range of pages.
-
-It is possible to establish how long the stalls were using the function
-tracer to record how long was spent in __alloc_pages_nodemask and
-using the mm_page_alloc tracepoint to identify which allocations were
-for huge pages.
-
-== get_user_pages and follow_page ==
-
-get_user_pages and follow_page if run on a hugepage, will return the
-head or tail pages as usual (exactly as they would do on
-hugetlbfs). Most gup users will only care about the actual physical
-address of the page and its temporary pinning to release after the I/O
-is complete, so they won't ever notice the fact the page is huge. But
-if any driver is going to mangle over the page structure of the tail
-page (like for checking page->mapping or other bits that are relevant
-for the head page and not the tail page), it should be updated to jump
-to check head page instead. Taking reference on any head/tail page would
-prevent page from being split by anyone.
-
-NOTE: these aren't new constraints to the GUP API, and they match the
-same constrains that applies to hugetlbfs too, so any driver capable
-of handling GUP on hugetlbfs will also work fine on transparent
-hugepage backed mappings.
-
-In case you can't handle compound pages if they're returned by
-follow_page, the FOLL_SPLIT bit can be specified as parameter to
-follow_page, so that it will split the hugepages before returning
-them. Migration for example passes FOLL_SPLIT as parameter to
-follow_page because it's not hugepage aware and in fact it can't work
-at all on hugetlbfs (but it instead works fine on transparent
-hugepages thanks to FOLL_SPLIT). migration simply can't deal with
-hugepages being returned (as it's not only checking the pfn of the
-page and pinning it during the copy but it pretends to migrate the
-memory in regular page sizes and with regular pte/pmd mappings).
-
-== Optimizing the applications ==
-
-To be guaranteed that the kernel will map a 2M page immediately in any
-memory region, the mmap region has to be hugepage naturally
-aligned. posix_memalign() can provide that guarantee.
-
-== Hugetlbfs ==
-
-You can use hugetlbfs on a kernel that has transparent hugepage
-support enabled just fine as always. No difference can be noted in
-hugetlbfs other than there will be less overall fragmentation. All
-usual features belonging to hugetlbfs are preserved and
-unaffected. libhugetlbfs will also work fine as usual.
-
-== Graceful fallback ==
-
-Code walking pagetables but unaware about huge pmds can simply call
-split_huge_pmd(vma, pmd, addr) where the pmd is the one returned by
-pmd_offset. It's trivial to make the code transparent hugepage aware
-by just grepping for "pmd_offset" and adding split_huge_pmd where
-missing after pmd_offset returns the pmd. Thanks to the graceful
-fallback design, with a one liner change, you can avoid to write
-hundred if not thousand of lines of complex code to make your code
-hugepage aware.
-
-If you're not walking pagetables but you run into a physical hugepage
-but you can't handle it natively in your code, you can split it by
-calling split_huge_page(page). This is what the Linux VM does before
-it tries to swapout the hugepage for example. split_huge_page() can fail
-if the page is pinned and you must handle this correctly.
-
-Example to make mremap.c transparent hugepage aware with a one liner
-change:
-
-diff --git a/mm/mremap.c b/mm/mremap.c
---- a/mm/mremap.c
-+++ b/mm/mremap.c
-@@ -41,6 +41,7 @@ static pmd_t *get_old_pmd(struct mm_stru
-		return NULL;
-
-	pmd = pmd_offset(pud, addr);
-+	split_huge_pmd(vma, pmd, addr);
-	if (pmd_none_or_clear_bad(pmd))
-		return NULL;
-
-== Locking in hugepage aware code ==
-
-We want as much code as possible hugepage aware, as calling
-split_huge_page() or split_huge_pmd() has a cost.
-
-To make pagetable walks huge pmd aware, all you need to do is to call
-pmd_trans_huge() on the pmd returned by pmd_offset. You must hold the
-mmap_sem in read (or write) mode to be sure an huge pmd cannot be
-created from under you by khugepaged (khugepaged collapse_huge_page
-takes the mmap_sem in write mode in addition to the anon_vma lock). If
-pmd_trans_huge returns false, you just fallback in the old code
-paths. If instead pmd_trans_huge returns true, you have to take the
-page table lock (pmd_lock()) and re-run pmd_trans_huge. Taking the
-page table lock will prevent the huge pmd to be converted into a
-regular pmd from under you (split_huge_pmd can run in parallel to the
-pagetable walk). If the second pmd_trans_huge returns false, you
-should just drop the page table lock and fallback to the old code as
-before. Otherwise you can proceed to process the huge pmd and the
-hugepage natively. Once finished you can drop the page table lock.
-
-== Refcounts and transparent huge pages ==
-
-Refcounting on THP is mostly consistent with refcounting on other compound
-pages:
-
-  - get_page()/put_page() and GUP operate in head page's ->_refcount.
-
-  - ->_refcount in tail pages is always zero: get_page_unless_zero() never
-    succeed on tail pages.
-
-  - map/unmap of the pages with PTE entry increment/decrement ->_mapcount
-    on relevant sub-page of the compound page.
-
-  - map/unmap of the whole compound page accounted in compound_mapcount
-    (stored in first tail page). For file huge pages, we also increment
-    ->_mapcount of all sub-pages in order to have race-free detection of
-    last unmap of subpages.
-
-PageDoubleMap() indicates that the page is *possibly* mapped with PTEs.
-
-For anonymous pages PageDoubleMap() also indicates ->_mapcount in all
-subpages is offset up by one. This additional reference is required to
-get race-free detection of unmap of subpages when we have them mapped with
-both PMDs and PTEs.
-
-This is optimization required to lower overhead of per-subpage mapcount
-tracking. The alternative is alter ->_mapcount in all subpages on each
-map/unmap of the whole compound page.
-
-For anonymous pages, we set PG_double_map when a PMD of the page got split
-for the first time, but still have PMD mapping. The additional references
-go away with last compound_mapcount.
-
-File pages get PG_double_map set on first map of the page with PTE and
-goes away when the page gets evicted from page cache.
-
-split_huge_page internally has to distribute the refcounts in the head
-page to the tail pages before clearing all PG_head/tail bits from the page
-structures. It can be done easily for refcounts taken by page table
-entries. But we don't have enough information on how to distribute any
-additional pins (i.e. from get_user_pages). split_huge_page() fails any
-requests to split pinned huge page: it expects page count to be equal to
-sum of mapcount of all sub-pages plus one (split_huge_page caller must
-have reference for head page).
-
-split_huge_page uses migration entries to stabilize page->_refcount and
-page->_mapcount of anonymous pages. File pages just got unmapped.
-
-We safe against physical memory scanners too: the only legitimate way
-scanner can get reference to a page is get_page_unless_zero().
-
-All tail pages have zero ->_refcount until atomic_add(). This prevents the
-scanner from getting a reference to the tail page up to that point. After the
-atomic_add() we don't care about the ->_refcount value. We already known how
-many references should be uncharged from the head page.
-
-For head page get_page_unless_zero() will succeed and we don't mind. It's
-clear where reference should go after split: it will stay on head page.
-
-Note that split_huge_pmd() doesn't have any limitation on refcounting:
-pmd can be split at any point and never fails.
-
-== Partial unmap and deferred_split_huge_page() ==
-
-Unmapping part of THP (with munmap() or other way) is not going to free
-memory immediately. Instead, we detect that a subpage of THP is not in use
-in page_remove_rmap() and queue the THP for splitting if memory pressure
-comes. Splitting will free up unused subpages.
-
-Splitting the page right away is not an option due to locking context in
-the place where we can detect partial unmap. It's also might be
-counterproductive since in many cases partial unmap happens during exit(2) if
-a THP crosses a VMA boundary.
-
-Function deferred_split_huge_page() is used to queue page for splitting.
-The splitting itself will happen when we get memory pressure via shrinker
-interface.
diff --git a/Documentation/vm/unevictable-lru.rst b/Documentation/vm/unevictable-lru.rst
new file mode 100644
index 000000000000..fdd84cb8d511
--- /dev/null
+++ b/Documentation/vm/unevictable-lru.rst
@@ -0,0 +1,614 @@
+.. _unevictable_lru:
+
+==============================
+Unevictable LRU Infrastructure
+==============================
+
+.. contents:: :local:
+
+
+Introduction
+============
+
+This document describes the Linux memory manager's "Unevictable LRU"
+infrastructure and the use of this to manage several types of "unevictable"
+pages.
+
+The document attempts to provide the overall rationale behind this mechanism
+and the rationale for some of the design decisions that drove the
+implementation.  The latter design rationale is discussed in the context of an
+implementation description.  Admittedly, one can obtain the implementation
+details - the "what does it do?" - by reading the code.  One hopes that the
+descriptions below add value by provide the answer to "why does it do that?".
+
+
+
+The Unevictable LRU
+===================
+
+The Unevictable LRU facility adds an additional LRU list to track unevictable
+pages and to hide these pages from vmscan.  This mechanism is based on a patch
+by Larry Woodman of Red Hat to address several scalability problems with page
+reclaim in Linux.  The problems have been observed at customer sites on large
+memory x86_64 systems.
+
+To illustrate this with an example, a non-NUMA x86_64 platform with 128GB of
+main memory will have over 32 million 4k pages in a single zone.  When a large
+fraction of these pages are not evictable for any reason [see below], vmscan
+will spend a lot of time scanning the LRU lists looking for the small fraction
+of pages that are evictable.  This can result in a situation where all CPUs are
+spending 100% of their time in vmscan for hours or days on end, with the system
+completely unresponsive.
+
+The unevictable list addresses the following classes of unevictable pages:
+
+ * Those owned by ramfs.
+
+ * Those mapped into SHM_LOCK'd shared memory regions.
+
+ * Those mapped into VM_LOCKED [mlock()ed] VMAs.
+
+The infrastructure may also be able to handle other conditions that make pages
+unevictable, either by definition or by circumstance, in the future.
+
+
+The Unevictable Page List
+-------------------------
+
+The Unevictable LRU infrastructure consists of an additional, per-zone, LRU list
+called the "unevictable" list and an associated page flag, PG_unevictable, to
+indicate that the page is being managed on the unevictable list.
+
+The PG_unevictable flag is analogous to, and mutually exclusive with, the
+PG_active flag in that it indicates on which LRU list a page resides when
+PG_lru is set.
+
+The Unevictable LRU infrastructure maintains unevictable pages on an additional
+LRU list for a few reasons:
+
+ (1) We get to "treat unevictable pages just like we treat other pages in the
+     system - which means we get to use the same code to manipulate them, the
+     same code to isolate them (for migrate, etc.), the same code to keep track
+     of the statistics, etc..." [Rik van Riel]
+
+ (2) We want to be able to migrate unevictable pages between nodes for memory
+     defragmentation, workload management and memory hotplug.  The linux kernel
+     can only migrate pages that it can successfully isolate from the LRU
+     lists.  If we were to maintain pages elsewhere than on an LRU-like list,
+     where they can be found by isolate_lru_page(), we would prevent their
+     migration, unless we reworked migration code to find the unevictable pages
+     itself.
+
+
+The unevictable list does not differentiate between file-backed and anonymous,
+swap-backed pages.  This differentiation is only important while the pages are,
+in fact, evictable.
+
+The unevictable list benefits from the "arrayification" of the per-zone LRU
+lists and statistics originally proposed and posted by Christoph Lameter.
+
+The unevictable list does not use the LRU pagevec mechanism. Rather,
+unevictable pages are placed directly on the page's zone's unevictable list
+under the zone lru_lock.  This allows us to prevent the stranding of pages on
+the unevictable list when one task has the page isolated from the LRU and other
+tasks are changing the "evictability" state of the page.
+
+
+Memory Control Group Interaction
+--------------------------------
+
+The unevictable LRU facility interacts with the memory control group [aka
+memory controller; see Documentation/cgroup-v1/memory.txt] by extending the
+lru_list enum.
+
+The memory controller data structure automatically gets a per-zone unevictable
+list as a result of the "arrayification" of the per-zone LRU lists (one per
+lru_list enum element).  The memory controller tracks the movement of pages to
+and from the unevictable list.
+
+When a memory control group comes under memory pressure, the controller will
+not attempt to reclaim pages on the unevictable list.  This has a couple of
+effects:
+
+ (1) Because the pages are "hidden" from reclaim on the unevictable list, the
+     reclaim process can be more efficient, dealing only with pages that have a
+     chance of being reclaimed.
+
+ (2) On the other hand, if too many of the pages charged to the control group
+     are unevictable, the evictable portion of the working set of the tasks in
+     the control group may not fit into the available memory.  This can cause
+     the control group to thrash or to OOM-kill tasks.
+
+
+.. _mark_addr_space_unevict:
+
+Marking Address Spaces Unevictable
+----------------------------------
+
+For facilities such as ramfs none of the pages attached to the address space
+may be evicted.  To prevent eviction of any such pages, the AS_UNEVICTABLE
+address space flag is provided, and this can be manipulated by a filesystem
+using a number of wrapper functions:
+
+ * ``void mapping_set_unevictable(struct address_space *mapping);``
+
+	Mark the address space as being completely unevictable.
+
+ * ``void mapping_clear_unevictable(struct address_space *mapping);``
+
+	Mark the address space as being evictable.
+
+ * ``int mapping_unevictable(struct address_space *mapping);``
+
+	Query the address space, and return true if it is completely
+	unevictable.
+
+These are currently used in two places in the kernel:
+
+ (1) By ramfs to mark the address spaces of its inodes when they are created,
+     and this mark remains for the life of the inode.
+
+ (2) By SYSV SHM to mark SHM_LOCK'd address spaces until SHM_UNLOCK is called.
+
+     Note that SHM_LOCK is not required to page in the locked pages if they're
+     swapped out; the application must touch the pages manually if it wants to
+     ensure they're in memory.
+
+
+Detecting Unevictable Pages
+---------------------------
+
+The function page_evictable() in vmscan.c determines whether a page is
+evictable or not using the query function outlined above [see section
+:ref:`Marking address spaces unevictable <mark_addr_space_unevict>`]
+to check the AS_UNEVICTABLE flag.
+
+For address spaces that are so marked after being populated (as SHM regions
+might be), the lock action (eg: SHM_LOCK) can be lazy, and need not populate
+the page tables for the region as does, for example, mlock(), nor need it make
+any special effort to push any pages in the SHM_LOCK'd area to the unevictable
+list.  Instead, vmscan will do this if and when it encounters the pages during
+a reclamation scan.
+
+On an unlock action (such as SHM_UNLOCK), the unlocker (eg: shmctl()) must scan
+the pages in the region and "rescue" them from the unevictable list if no other
+condition is keeping them unevictable.  If an unevictable region is destroyed,
+the pages are also "rescued" from the unevictable list in the process of
+freeing them.
+
+page_evictable() also checks for mlocked pages by testing an additional page
+flag, PG_mlocked (as wrapped by PageMlocked()), which is set when a page is
+faulted into a VM_LOCKED vma, or found in a vma being VM_LOCKED.
+
+
+Vmscan's Handling of Unevictable Pages
+--------------------------------------
+
+If unevictable pages are culled in the fault path, or moved to the unevictable
+list at mlock() or mmap() time, vmscan will not encounter the pages until they
+have become evictable again (via munlock() for example) and have been "rescued"
+from the unevictable list.  However, there may be situations where we decide,
+for the sake of expediency, to leave a unevictable page on one of the regular
+active/inactive LRU lists for vmscan to deal with.  vmscan checks for such
+pages in all of the shrink_{active|inactive|page}_list() functions and will
+"cull" such pages that it encounters: that is, it diverts those pages to the
+unevictable list for the zone being scanned.
+
+There may be situations where a page is mapped into a VM_LOCKED VMA, but the
+page is not marked as PG_mlocked.  Such pages will make it all the way to
+shrink_page_list() where they will be detected when vmscan walks the reverse
+map in try_to_unmap().  If try_to_unmap() returns SWAP_MLOCK,
+shrink_page_list() will cull the page at that point.
+
+To "cull" an unevictable page, vmscan simply puts the page back on the LRU list
+using putback_lru_page() - the inverse operation to isolate_lru_page() - after
+dropping the page lock.  Because the condition which makes the page unevictable
+may change once the page is unlocked, putback_lru_page() will recheck the
+unevictable state of a page that it places on the unevictable list.  If the
+page has become unevictable, putback_lru_page() removes it from the list and
+retries, including the page_unevictable() test.  Because such a race is a rare
+event and movement of pages onto the unevictable list should be rare, these
+extra evictabilty checks should not occur in the majority of calls to
+putback_lru_page().
+
+
+MLOCKED Pages
+=============
+
+The unevictable page list is also useful for mlock(), in addition to ramfs and
+SYSV SHM.  Note that mlock() is only available in CONFIG_MMU=y situations; in
+NOMMU situations, all mappings are effectively mlocked.
+
+
+History
+-------
+
+The "Unevictable mlocked Pages" infrastructure is based on work originally
+posted by Nick Piggin in an RFC patch entitled "mm: mlocked pages off LRU".
+Nick posted his patch as an alternative to a patch posted by Christoph Lameter
+to achieve the same objective: hiding mlocked pages from vmscan.
+
+In Nick's patch, he used one of the struct page LRU list link fields as a count
+of VM_LOCKED VMAs that map the page.  This use of the link field for a count
+prevented the management of the pages on an LRU list, and thus mlocked pages
+were not migratable as isolate_lru_page() could not find them, and the LRU list
+link field was not available to the migration subsystem.
+
+Nick resolved this by putting mlocked pages back on the lru list before
+attempting to isolate them, thus abandoning the count of VM_LOCKED VMAs.  When
+Nick's patch was integrated with the Unevictable LRU work, the count was
+replaced by walking the reverse map to determine whether any VM_LOCKED VMAs
+mapped the page.  More on this below.
+
+
+Basic Management
+----------------
+
+mlocked pages - pages mapped into a VM_LOCKED VMA - are a class of unevictable
+pages.  When such a page has been "noticed" by the memory management subsystem,
+the page is marked with the PG_mlocked flag.  This can be manipulated using the
+PageMlocked() functions.
+
+A PG_mlocked page will be placed on the unevictable list when it is added to
+the LRU.  Such pages can be "noticed" by memory management in several places:
+
+ (1) in the mlock()/mlockall() system call handlers;
+
+ (2) in the mmap() system call handler when mmapping a region with the
+     MAP_LOCKED flag;
+
+ (3) mmapping a region in a task that has called mlockall() with the MCL_FUTURE
+     flag
+
+ (4) in the fault path, if mlocked pages are "culled" in the fault path,
+     and when a VM_LOCKED stack segment is expanded; or
+
+ (5) as mentioned above, in vmscan:shrink_page_list() when attempting to
+     reclaim a page in a VM_LOCKED VMA via try_to_unmap()
+
+all of which result in the VM_LOCKED flag being set for the VMA if it doesn't
+already have it set.
+
+mlocked pages become unlocked and rescued from the unevictable list when:
+
+ (1) mapped in a range unlocked via the munlock()/munlockall() system calls;
+
+ (2) munmap()'d out of the last VM_LOCKED VMA that maps the page, including
+     unmapping at task exit;
+
+ (3) when the page is truncated from the last VM_LOCKED VMA of an mmapped file;
+     or
+
+ (4) before a page is COW'd in a VM_LOCKED VMA.
+
+
+mlock()/mlockall() System Call Handling
+---------------------------------------
+
+Both [do\_]mlock() and [do\_]mlockall() system call handlers call mlock_fixup()
+for each VMA in the range specified by the call.  In the case of mlockall(),
+this is the entire active address space of the task.  Note that mlock_fixup()
+is used for both mlocking and munlocking a range of memory.  A call to mlock()
+an already VM_LOCKED VMA, or to munlock() a VMA that is not VM_LOCKED is
+treated as a no-op, and mlock_fixup() simply returns.
+
+If the VMA passes some filtering as described in "Filtering Special Vmas"
+below, mlock_fixup() will attempt to merge the VMA with its neighbors or split
+off a subset of the VMA if the range does not cover the entire VMA.  Once the
+VMA has been merged or split or neither, mlock_fixup() will call
+populate_vma_page_range() to fault in the pages via get_user_pages() and to
+mark the pages as mlocked via mlock_vma_page().
+
+Note that the VMA being mlocked might be mapped with PROT_NONE.  In this case,
+get_user_pages() will be unable to fault in the pages.  That's okay.  If pages
+do end up getting faulted into this VM_LOCKED VMA, we'll handle them in the
+fault path or in vmscan.
+
+Also note that a page returned by get_user_pages() could be truncated or
+migrated out from under us, while we're trying to mlock it.  To detect this,
+populate_vma_page_range() checks page_mapping() after acquiring the page lock.
+If the page is still associated with its mapping, we'll go ahead and call
+mlock_vma_page().  If the mapping is gone, we just unlock the page and move on.
+In the worst case, this will result in a page mapped in a VM_LOCKED VMA
+remaining on a normal LRU list without being PageMlocked().  Again, vmscan will
+detect and cull such pages.
+
+mlock_vma_page() will call TestSetPageMlocked() for each page returned by
+get_user_pages().  We use TestSetPageMlocked() because the page might already
+be mlocked by another task/VMA and we don't want to do extra work.  We
+especially do not want to count an mlocked page more than once in the
+statistics.  If the page was already mlocked, mlock_vma_page() need do nothing
+more.
+
+If the page was NOT already mlocked, mlock_vma_page() attempts to isolate the
+page from the LRU, as it is likely on the appropriate active or inactive list
+at that time.  If the isolate_lru_page() succeeds, mlock_vma_page() will put
+back the page - by calling putback_lru_page() - which will notice that the page
+is now mlocked and divert the page to the zone's unevictable list.  If
+mlock_vma_page() is unable to isolate the page from the LRU, vmscan will handle
+it later if and when it attempts to reclaim the page.
+
+
+Filtering Special VMAs
+----------------------
+
+mlock_fixup() filters several classes of "special" VMAs:
+
+1) VMAs with VM_IO or VM_PFNMAP set are skipped entirely.  The pages behind
+   these mappings are inherently pinned, so we don't need to mark them as
+   mlocked.  In any case, most of the pages have no struct page in which to so
+   mark the page.  Because of this, get_user_pages() will fail for these VMAs,
+   so there is no sense in attempting to visit them.
+
+2) VMAs mapping hugetlbfs page are already effectively pinned into memory.  We
+   neither need nor want to mlock() these pages.  However, to preserve the
+   prior behavior of mlock() - before the unevictable/mlock changes -
+   mlock_fixup() will call make_pages_present() in the hugetlbfs VMA range to
+   allocate the huge pages and populate the ptes.
+
+3) VMAs with VM_DONTEXPAND are generally userspace mappings of kernel pages,
+   such as the VDSO page, relay channel pages, etc. These pages
+   are inherently unevictable and are not managed on the LRU lists.
+   mlock_fixup() treats these VMAs the same as hugetlbfs VMAs.  It calls
+   make_pages_present() to populate the ptes.
+
+Note that for all of these special VMAs, mlock_fixup() does not set the
+VM_LOCKED flag.  Therefore, we won't have to deal with them later during
+munlock(), munmap() or task exit.  Neither does mlock_fixup() account these
+VMAs against the task's "locked_vm".
+
+.. _munlock_munlockall_handling:
+
+munlock()/munlockall() System Call Handling
+-------------------------------------------
+
+The munlock() and munlockall() system calls are handled by the same functions -
+do_mlock[all]() - as the mlock() and mlockall() system calls with the unlock vs
+lock operation indicated by an argument.  So, these system calls are also
+handled by mlock_fixup().  Again, if called for an already munlocked VMA,
+mlock_fixup() simply returns.  Because of the VMA filtering discussed above,
+VM_LOCKED will not be set in any "special" VMAs.  So, these VMAs will be
+ignored for munlock.
+
+If the VMA is VM_LOCKED, mlock_fixup() again attempts to merge or split off the
+specified range.  The range is then munlocked via the function
+populate_vma_page_range() - the same function used to mlock a VMA range -
+passing a flag to indicate that munlock() is being performed.
+
+Because the VMA access protections could have been changed to PROT_NONE after
+faulting in and mlocking pages, get_user_pages() was unreliable for visiting
+these pages for munlocking.  Because we don't want to leave pages mlocked,
+get_user_pages() was enhanced to accept a flag to ignore the permissions when
+fetching the pages - all of which should be resident as a result of previous
+mlocking.
+
+For munlock(), populate_vma_page_range() unlocks individual pages by calling
+munlock_vma_page().  munlock_vma_page() unconditionally clears the PG_mlocked
+flag using TestClearPageMlocked().  As with mlock_vma_page(),
+munlock_vma_page() use the Test*PageMlocked() function to handle the case where
+the page might have already been unlocked by another task.  If the page was
+mlocked, munlock_vma_page() updates that zone statistics for the number of
+mlocked pages.  Note, however, that at this point we haven't checked whether
+the page is mapped by other VM_LOCKED VMAs.
+
+We can't call try_to_munlock(), the function that walks the reverse map to
+check for other VM_LOCKED VMAs, without first isolating the page from the LRU.
+try_to_munlock() is a variant of try_to_unmap() and thus requires that the page
+not be on an LRU list [more on these below].  However, the call to
+isolate_lru_page() could fail, in which case we couldn't try_to_munlock().  So,
+we go ahead and clear PG_mlocked up front, as this might be the only chance we
+have.  If we can successfully isolate the page, we go ahead and
+try_to_munlock(), which will restore the PG_mlocked flag and update the zone
+page statistics if it finds another VMA holding the page mlocked.  If we fail
+to isolate the page, we'll have left a potentially mlocked page on the LRU.
+This is fine, because we'll catch it later if and if vmscan tries to reclaim
+the page.  This should be relatively rare.
+
+
+Migrating MLOCKED Pages
+-----------------------
+
+A page that is being migrated has been isolated from the LRU lists and is held
+locked across unmapping of the page, updating the page's address space entry
+and copying the contents and state, until the page table entry has been
+replaced with an entry that refers to the new page.  Linux supports migration
+of mlocked pages and other unevictable pages.  This involves simply moving the
+PG_mlocked and PG_unevictable states from the old page to the new page.
+
+Note that page migration can race with mlocking or munlocking of the same page.
+This has been discussed from the mlock/munlock perspective in the respective
+sections above.  Both processes (migration and m[un]locking) hold the page
+locked.  This provides the first level of synchronization.  Page migration
+zeros out the page_mapping of the old page before unlocking it, so m[un]lock
+can skip these pages by testing the page mapping under page lock.
+
+To complete page migration, we place the new and old pages back onto the LRU
+after dropping the page lock.  The "unneeded" page - old page on success, new
+page on failure - will be freed when the reference count held by the migration
+process is released.  To ensure that we don't strand pages on the unevictable
+list because of a race between munlock and migration, page migration uses the
+putback_lru_page() function to add migrated pages back to the LRU.
+
+
+Compacting MLOCKED Pages
+------------------------
+
+The unevictable LRU can be scanned for compactable regions and the default
+behavior is to do so.  /proc/sys/vm/compact_unevictable_allowed controls
+this behavior (see Documentation/sysctl/vm.txt).  Once scanning of the
+unevictable LRU is enabled, the work of compaction is mostly handled by
+the page migration code and the same work flow as described in MIGRATING
+MLOCKED PAGES will apply.
+
+MLOCKING Transparent Huge Pages
+-------------------------------
+
+A transparent huge page is represented by a single entry on an LRU list.
+Therefore, we can only make unevictable an entire compound page, not
+individual subpages.
+
+If a user tries to mlock() part of a huge page, we want the rest of the
+page to be reclaimable.
+
+We cannot just split the page on partial mlock() as split_huge_page() can
+fail and new intermittent failure mode for the syscall is undesirable.
+
+We handle this by keeping PTE-mapped huge pages on normal LRU lists: the
+PMD on border of VM_LOCKED VMA will be split into PTE table.
+
+This way the huge page is accessible for vmscan. Under memory pressure the
+page will be split, subpages which belong to VM_LOCKED VMAs will be moved
+to unevictable LRU and the rest can be reclaimed.
+
+See also comment in follow_trans_huge_pmd().
+
+mmap(MAP_LOCKED) System Call Handling
+-------------------------------------
+
+In addition the mlock()/mlockall() system calls, an application can request
+that a region of memory be mlocked supplying the MAP_LOCKED flag to the mmap()
+call. There is one important and subtle difference here, though. mmap() + mlock()
+will fail if the range cannot be faulted in (e.g. because mm_populate fails)
+and returns with ENOMEM while mmap(MAP_LOCKED) will not fail. The mmaped
+area will still have properties of the locked area - aka. pages will not get
+swapped out - but major page faults to fault memory in might still happen.
+
+Furthermore, any mmap() call or brk() call that expands the heap by a
+task that has previously called mlockall() with the MCL_FUTURE flag will result
+in the newly mapped memory being mlocked.  Before the unevictable/mlock
+changes, the kernel simply called make_pages_present() to allocate pages and
+populate the page table.
+
+To mlock a range of memory under the unevictable/mlock infrastructure, the
+mmap() handler and task address space expansion functions call
+populate_vma_page_range() specifying the vma and the address range to mlock.
+
+The callers of populate_vma_page_range() will have already added the memory range
+to be mlocked to the task's "locked_vm".  To account for filtered VMAs,
+populate_vma_page_range() returns the number of pages NOT mlocked.  All of the
+callers then subtract a non-negative return value from the task's locked_vm.  A
+negative return value represent an error - for example, from get_user_pages()
+attempting to fault in a VMA with PROT_NONE access.  In this case, we leave the
+memory range accounted as locked_vm, as the protections could be changed later
+and pages allocated into that region.
+
+
+munmap()/exit()/exec() System Call Handling
+-------------------------------------------
+
+When unmapping an mlocked region of memory, whether by an explicit call to
+munmap() or via an internal unmap from exit() or exec() processing, we must
+munlock the pages if we're removing the last VM_LOCKED VMA that maps the pages.
+Before the unevictable/mlock changes, mlocking did not mark the pages in any
+way, so unmapping them required no processing.
+
+To munlock a range of memory under the unevictable/mlock infrastructure, the
+munmap() handler and task address space call tear down function
+munlock_vma_pages_all().  The name reflects the observation that one always
+specifies the entire VMA range when munlock()ing during unmap of a region.
+Because of the VMA filtering when mlocking() regions, only "normal" VMAs that
+actually contain mlocked pages will be passed to munlock_vma_pages_all().
+
+munlock_vma_pages_all() clears the VM_LOCKED VMA flag and, like mlock_fixup()
+for the munlock case, calls __munlock_vma_pages_range() to walk the page table
+for the VMA's memory range and munlock_vma_page() each resident page mapped by
+the VMA.  This effectively munlocks the page, only if this is the last
+VM_LOCKED VMA that maps the page.
+
+
+try_to_unmap()
+--------------
+
+Pages can, of course, be mapped into multiple VMAs.  Some of these VMAs may
+have VM_LOCKED flag set.  It is possible for a page mapped into one or more
+VM_LOCKED VMAs not to have the PG_mlocked flag set and therefore reside on one
+of the active or inactive LRU lists.  This could happen if, for example, a task
+in the process of munlocking the page could not isolate the page from the LRU.
+As a result, vmscan/shrink_page_list() might encounter such a page as described
+in section "vmscan's handling of unevictable pages".  To handle this situation,
+try_to_unmap() checks for VM_LOCKED VMAs while it is walking a page's reverse
+map.
+
+try_to_unmap() is always called, by either vmscan for reclaim or for page
+migration, with the argument page locked and isolated from the LRU.  Separate
+functions handle anonymous and mapped file and KSM pages, as these types of
+pages have different reverse map lookup mechanisms, with different locking.
+In each case, whether rmap_walk_anon() or rmap_walk_file() or rmap_walk_ksm(),
+it will call try_to_unmap_one() for every VMA which might contain the page.
+
+When trying to reclaim, if try_to_unmap_one() finds the page in a VM_LOCKED
+VMA, it will then mlock the page via mlock_vma_page() instead of unmapping it,
+and return SWAP_MLOCK to indicate that the page is unevictable: and the scan
+stops there.
+
+mlock_vma_page() is called while holding the page table's lock (in addition
+to the page lock, and the rmap lock): to serialize against concurrent mlock or
+munlock or munmap system calls, mm teardown (munlock_vma_pages_all), reclaim,
+holepunching, and truncation of file pages and their anonymous COWed pages.
+
+
+try_to_munlock() Reverse Map Scan
+---------------------------------
+
+.. warning::
+   [!] TODO/FIXME: a better name might be page_mlocked() - analogous to the
+   page_referenced() reverse map walker.
+
+When munlock_vma_page() [see section :ref:`munlock()/munlockall() System Call
+Handling <munlock_munlockall_handling>` above] tries to munlock a
+page, it needs to determine whether or not the page is mapped by any
+VM_LOCKED VMA without actually attempting to unmap all PTEs from the
+page.  For this purpose, the unevictable/mlock infrastructure
+introduced a variant of try_to_unmap() called try_to_munlock().
+
+try_to_munlock() calls the same functions as try_to_unmap() for anonymous and
+mapped file and KSM pages with a flag argument specifying unlock versus unmap
+processing.  Again, these functions walk the respective reverse maps looking
+for VM_LOCKED VMAs.  When such a VMA is found, as in the try_to_unmap() case,
+the functions mlock the page via mlock_vma_page() and return SWAP_MLOCK.  This
+undoes the pre-clearing of the page's PG_mlocked done by munlock_vma_page.
+
+Note that try_to_munlock()'s reverse map walk must visit every VMA in a page's
+reverse map to determine that a page is NOT mapped into any VM_LOCKED VMA.
+However, the scan can terminate when it encounters a VM_LOCKED VMA.
+Although try_to_munlock() might be called a great many times when munlocking a
+large region or tearing down a large address space that has been mlocked via
+mlockall(), overall this is a fairly rare event.
+
+
+Page Reclaim in shrink_*_list()
+-------------------------------
+
+shrink_active_list() culls any obviously unevictable pages - i.e.
+!page_evictable(page) - diverting these to the unevictable list.
+However, shrink_active_list() only sees unevictable pages that made it onto the
+active/inactive lru lists.  Note that these pages do not have PageUnevictable
+set - otherwise they would be on the unevictable list and shrink_active_list
+would never see them.
+
+Some examples of these unevictable pages on the LRU lists are:
+
+ (1) ramfs pages that have been placed on the LRU lists when first allocated.
+
+ (2) SHM_LOCK'd shared memory pages.  shmctl(SHM_LOCK) does not attempt to
+     allocate or fault in the pages in the shared memory region.  This happens
+     when an application accesses the page the first time after SHM_LOCK'ing
+     the segment.
+
+ (3) mlocked pages that could not be isolated from the LRU and moved to the
+     unevictable list in mlock_vma_page().
+
+shrink_inactive_list() also diverts any unevictable pages that it finds on the
+inactive lists to the appropriate zone's unevictable list.
+
+shrink_inactive_list() should only see SHM_LOCK'd pages that became SHM_LOCK'd
+after shrink_active_list() had moved them to the inactive list, or pages mapped
+into VM_LOCKED VMAs that munlock_vma_page() couldn't isolate from the LRU to
+recheck via try_to_munlock().  shrink_inactive_list() won't notice the latter,
+but will pass on to shrink_page_list().
+
+shrink_page_list() again culls obviously unevictable pages that it could
+encounter for similar reason to shrink_inactive_list().  Pages mapped into
+VM_LOCKED VMAs but without PG_mlocked set will make it all the way to
+try_to_unmap().  shrink_page_list() will divert them to the unevictable list
+when try_to_unmap() returns SWAP_MLOCK, as discussed above.
diff --git a/Documentation/vm/unevictable-lru.txt b/Documentation/vm/unevictable-lru.txt
deleted file mode 100644
index e14718572476..000000000000
--- a/Documentation/vm/unevictable-lru.txt
+++ /dev/null
@@ -1,633 +0,0 @@
-			==============================
-			UNEVICTABLE LRU INFRASTRUCTURE
-			==============================
-
-========
-CONTENTS
-========
-
- (*) The Unevictable LRU
-
-     - The unevictable page list.
-     - Memory control group interaction.
-     - Marking address spaces unevictable.
-     - Detecting Unevictable Pages.
-     - vmscan's handling of unevictable pages.
-
- (*) mlock()'d pages.
-
-     - History.
-     - Basic management.
-     - mlock()/mlockall() system call handling.
-     - Filtering special vmas.
-     - munlock()/munlockall() system call handling.
-     - Migrating mlocked pages.
-     - Compacting mlocked pages.
-     - mmap(MAP_LOCKED) system call handling.
-     - munmap()/exit()/exec() system call handling.
-     - try_to_unmap().
-     - try_to_munlock() reverse map scan.
-     - Page reclaim in shrink_*_list().
-
-
-============
-INTRODUCTION
-============
-
-This document describes the Linux memory manager's "Unevictable LRU"
-infrastructure and the use of this to manage several types of "unevictable"
-pages.
-
-The document attempts to provide the overall rationale behind this mechanism
-and the rationale for some of the design decisions that drove the
-implementation.  The latter design rationale is discussed in the context of an
-implementation description.  Admittedly, one can obtain the implementation
-details - the "what does it do?" - by reading the code.  One hopes that the
-descriptions below add value by provide the answer to "why does it do that?".
-
-
-===================
-THE UNEVICTABLE LRU
-===================
-
-The Unevictable LRU facility adds an additional LRU list to track unevictable
-pages and to hide these pages from vmscan.  This mechanism is based on a patch
-by Larry Woodman of Red Hat to address several scalability problems with page
-reclaim in Linux.  The problems have been observed at customer sites on large
-memory x86_64 systems.
-
-To illustrate this with an example, a non-NUMA x86_64 platform with 128GB of
-main memory will have over 32 million 4k pages in a single zone.  When a large
-fraction of these pages are not evictable for any reason [see below], vmscan
-will spend a lot of time scanning the LRU lists looking for the small fraction
-of pages that are evictable.  This can result in a situation where all CPUs are
-spending 100% of their time in vmscan for hours or days on end, with the system
-completely unresponsive.
-
-The unevictable list addresses the following classes of unevictable pages:
-
- (*) Those owned by ramfs.
-
- (*) Those mapped into SHM_LOCK'd shared memory regions.
-
- (*) Those mapped into VM_LOCKED [mlock()ed] VMAs.
-
-The infrastructure may also be able to handle other conditions that make pages
-unevictable, either by definition or by circumstance, in the future.
-
-
-THE UNEVICTABLE PAGE LIST
--------------------------
-
-The Unevictable LRU infrastructure consists of an additional, per-zone, LRU list
-called the "unevictable" list and an associated page flag, PG_unevictable, to
-indicate that the page is being managed on the unevictable list.
-
-The PG_unevictable flag is analogous to, and mutually exclusive with, the
-PG_active flag in that it indicates on which LRU list a page resides when
-PG_lru is set.
-
-The Unevictable LRU infrastructure maintains unevictable pages on an additional
-LRU list for a few reasons:
-
- (1) We get to "treat unevictable pages just like we treat other pages in the
-     system - which means we get to use the same code to manipulate them, the
-     same code to isolate them (for migrate, etc.), the same code to keep track
-     of the statistics, etc..." [Rik van Riel]
-
- (2) We want to be able to migrate unevictable pages between nodes for memory
-     defragmentation, workload management and memory hotplug.  The linux kernel
-     can only migrate pages that it can successfully isolate from the LRU
-     lists.  If we were to maintain pages elsewhere than on an LRU-like list,
-     where they can be found by isolate_lru_page(), we would prevent their
-     migration, unless we reworked migration code to find the unevictable pages
-     itself.
-
-
-The unevictable list does not differentiate between file-backed and anonymous,
-swap-backed pages.  This differentiation is only important while the pages are,
-in fact, evictable.
-
-The unevictable list benefits from the "arrayification" of the per-zone LRU
-lists and statistics originally proposed and posted by Christoph Lameter.
-
-The unevictable list does not use the LRU pagevec mechanism. Rather,
-unevictable pages are placed directly on the page's zone's unevictable list
-under the zone lru_lock.  This allows us to prevent the stranding of pages on
-the unevictable list when one task has the page isolated from the LRU and other
-tasks are changing the "evictability" state of the page.
-
-
-MEMORY CONTROL GROUP INTERACTION
---------------------------------
-
-The unevictable LRU facility interacts with the memory control group [aka
-memory controller; see Documentation/cgroup-v1/memory.txt] by extending the
-lru_list enum.
-
-The memory controller data structure automatically gets a per-zone unevictable
-list as a result of the "arrayification" of the per-zone LRU lists (one per
-lru_list enum element).  The memory controller tracks the movement of pages to
-and from the unevictable list.
-
-When a memory control group comes under memory pressure, the controller will
-not attempt to reclaim pages on the unevictable list.  This has a couple of
-effects:
-
- (1) Because the pages are "hidden" from reclaim on the unevictable list, the
-     reclaim process can be more efficient, dealing only with pages that have a
-     chance of being reclaimed.
-
- (2) On the other hand, if too many of the pages charged to the control group
-     are unevictable, the evictable portion of the working set of the tasks in
-     the control group may not fit into the available memory.  This can cause
-     the control group to thrash or to OOM-kill tasks.
-
-
-MARKING ADDRESS SPACES UNEVICTABLE
-----------------------------------
-
-For facilities such as ramfs none of the pages attached to the address space
-may be evicted.  To prevent eviction of any such pages, the AS_UNEVICTABLE
-address space flag is provided, and this can be manipulated by a filesystem
-using a number of wrapper functions:
-
- (*) void mapping_set_unevictable(struct address_space *mapping);
-
-	Mark the address space as being completely unevictable.
-
- (*) void mapping_clear_unevictable(struct address_space *mapping);
-
-	Mark the address space as being evictable.
-
- (*) int mapping_unevictable(struct address_space *mapping);
-
-	Query the address space, and return true if it is completely
-	unevictable.
-
-These are currently used in two places in the kernel:
-
- (1) By ramfs to mark the address spaces of its inodes when they are created,
-     and this mark remains for the life of the inode.
-
- (2) By SYSV SHM to mark SHM_LOCK'd address spaces until SHM_UNLOCK is called.
-
-     Note that SHM_LOCK is not required to page in the locked pages if they're
-     swapped out; the application must touch the pages manually if it wants to
-     ensure they're in memory.
-
-
-DETECTING UNEVICTABLE PAGES
----------------------------
-
-The function page_evictable() in vmscan.c determines whether a page is
-evictable or not using the query function outlined above [see section "Marking
-address spaces unevictable"] to check the AS_UNEVICTABLE flag.
-
-For address spaces that are so marked after being populated (as SHM regions
-might be), the lock action (eg: SHM_LOCK) can be lazy, and need not populate
-the page tables for the region as does, for example, mlock(), nor need it make
-any special effort to push any pages in the SHM_LOCK'd area to the unevictable
-list.  Instead, vmscan will do this if and when it encounters the pages during
-a reclamation scan.
-
-On an unlock action (such as SHM_UNLOCK), the unlocker (eg: shmctl()) must scan
-the pages in the region and "rescue" them from the unevictable list if no other
-condition is keeping them unevictable.  If an unevictable region is destroyed,
-the pages are also "rescued" from the unevictable list in the process of
-freeing them.
-
-page_evictable() also checks for mlocked pages by testing an additional page
-flag, PG_mlocked (as wrapped by PageMlocked()), which is set when a page is
-faulted into a VM_LOCKED vma, or found in a vma being VM_LOCKED.
-
-
-VMSCAN'S HANDLING OF UNEVICTABLE PAGES
---------------------------------------
-
-If unevictable pages are culled in the fault path, or moved to the unevictable
-list at mlock() or mmap() time, vmscan will not encounter the pages until they
-have become evictable again (via munlock() for example) and have been "rescued"
-from the unevictable list.  However, there may be situations where we decide,
-for the sake of expediency, to leave a unevictable page on one of the regular
-active/inactive LRU lists for vmscan to deal with.  vmscan checks for such
-pages in all of the shrink_{active|inactive|page}_list() functions and will
-"cull" such pages that it encounters: that is, it diverts those pages to the
-unevictable list for the zone being scanned.
-
-There may be situations where a page is mapped into a VM_LOCKED VMA, but the
-page is not marked as PG_mlocked.  Such pages will make it all the way to
-shrink_page_list() where they will be detected when vmscan walks the reverse
-map in try_to_unmap().  If try_to_unmap() returns SWAP_MLOCK,
-shrink_page_list() will cull the page at that point.
-
-To "cull" an unevictable page, vmscan simply puts the page back on the LRU list
-using putback_lru_page() - the inverse operation to isolate_lru_page() - after
-dropping the page lock.  Because the condition which makes the page unevictable
-may change once the page is unlocked, putback_lru_page() will recheck the
-unevictable state of a page that it places on the unevictable list.  If the
-page has become unevictable, putback_lru_page() removes it from the list and
-retries, including the page_unevictable() test.  Because such a race is a rare
-event and movement of pages onto the unevictable list should be rare, these
-extra evictabilty checks should not occur in the majority of calls to
-putback_lru_page().
-
-
-=============
-MLOCKED PAGES
-=============
-
-The unevictable page list is also useful for mlock(), in addition to ramfs and
-SYSV SHM.  Note that mlock() is only available in CONFIG_MMU=y situations; in
-NOMMU situations, all mappings are effectively mlocked.
-
-
-HISTORY
--------
-
-The "Unevictable mlocked Pages" infrastructure is based on work originally
-posted by Nick Piggin in an RFC patch entitled "mm: mlocked pages off LRU".
-Nick posted his patch as an alternative to a patch posted by Christoph Lameter
-to achieve the same objective: hiding mlocked pages from vmscan.
-
-In Nick's patch, he used one of the struct page LRU list link fields as a count
-of VM_LOCKED VMAs that map the page.  This use of the link field for a count
-prevented the management of the pages on an LRU list, and thus mlocked pages
-were not migratable as isolate_lru_page() could not find them, and the LRU list
-link field was not available to the migration subsystem.
-
-Nick resolved this by putting mlocked pages back on the lru list before
-attempting to isolate them, thus abandoning the count of VM_LOCKED VMAs.  When
-Nick's patch was integrated with the Unevictable LRU work, the count was
-replaced by walking the reverse map to determine whether any VM_LOCKED VMAs
-mapped the page.  More on this below.
-
-
-BASIC MANAGEMENT
-----------------
-
-mlocked pages - pages mapped into a VM_LOCKED VMA - are a class of unevictable
-pages.  When such a page has been "noticed" by the memory management subsystem,
-the page is marked with the PG_mlocked flag.  This can be manipulated using the
-PageMlocked() functions.
-
-A PG_mlocked page will be placed on the unevictable list when it is added to
-the LRU.  Such pages can be "noticed" by memory management in several places:
-
- (1) in the mlock()/mlockall() system call handlers;
-
- (2) in the mmap() system call handler when mmapping a region with the
-     MAP_LOCKED flag;
-
- (3) mmapping a region in a task that has called mlockall() with the MCL_FUTURE
-     flag
-
- (4) in the fault path, if mlocked pages are "culled" in the fault path,
-     and when a VM_LOCKED stack segment is expanded; or
-
- (5) as mentioned above, in vmscan:shrink_page_list() when attempting to
-     reclaim a page in a VM_LOCKED VMA via try_to_unmap()
-
-all of which result in the VM_LOCKED flag being set for the VMA if it doesn't
-already have it set.
-
-mlocked pages become unlocked and rescued from the unevictable list when:
-
- (1) mapped in a range unlocked via the munlock()/munlockall() system calls;
-
- (2) munmap()'d out of the last VM_LOCKED VMA that maps the page, including
-     unmapping at task exit;
-
- (3) when the page is truncated from the last VM_LOCKED VMA of an mmapped file;
-     or
-
- (4) before a page is COW'd in a VM_LOCKED VMA.
-
-
-mlock()/mlockall() SYSTEM CALL HANDLING
----------------------------------------
-
-Both [do_]mlock() and [do_]mlockall() system call handlers call mlock_fixup()
-for each VMA in the range specified by the call.  In the case of mlockall(),
-this is the entire active address space of the task.  Note that mlock_fixup()
-is used for both mlocking and munlocking a range of memory.  A call to mlock()
-an already VM_LOCKED VMA, or to munlock() a VMA that is not VM_LOCKED is
-treated as a no-op, and mlock_fixup() simply returns.
-
-If the VMA passes some filtering as described in "Filtering Special Vmas"
-below, mlock_fixup() will attempt to merge the VMA with its neighbors or split
-off a subset of the VMA if the range does not cover the entire VMA.  Once the
-VMA has been merged or split or neither, mlock_fixup() will call
-populate_vma_page_range() to fault in the pages via get_user_pages() and to
-mark the pages as mlocked via mlock_vma_page().
-
-Note that the VMA being mlocked might be mapped with PROT_NONE.  In this case,
-get_user_pages() will be unable to fault in the pages.  That's okay.  If pages
-do end up getting faulted into this VM_LOCKED VMA, we'll handle them in the
-fault path or in vmscan.
-
-Also note that a page returned by get_user_pages() could be truncated or
-migrated out from under us, while we're trying to mlock it.  To detect this,
-populate_vma_page_range() checks page_mapping() after acquiring the page lock.
-If the page is still associated with its mapping, we'll go ahead and call
-mlock_vma_page().  If the mapping is gone, we just unlock the page and move on.
-In the worst case, this will result in a page mapped in a VM_LOCKED VMA
-remaining on a normal LRU list without being PageMlocked().  Again, vmscan will
-detect and cull such pages.
-
-mlock_vma_page() will call TestSetPageMlocked() for each page returned by
-get_user_pages().  We use TestSetPageMlocked() because the page might already
-be mlocked by another task/VMA and we don't want to do extra work.  We
-especially do not want to count an mlocked page more than once in the
-statistics.  If the page was already mlocked, mlock_vma_page() need do nothing
-more.
-
-If the page was NOT already mlocked, mlock_vma_page() attempts to isolate the
-page from the LRU, as it is likely on the appropriate active or inactive list
-at that time.  If the isolate_lru_page() succeeds, mlock_vma_page() will put
-back the page - by calling putback_lru_page() - which will notice that the page
-is now mlocked and divert the page to the zone's unevictable list.  If
-mlock_vma_page() is unable to isolate the page from the LRU, vmscan will handle
-it later if and when it attempts to reclaim the page.
-
-
-FILTERING SPECIAL VMAS
-----------------------
-
-mlock_fixup() filters several classes of "special" VMAs:
-
-1) VMAs with VM_IO or VM_PFNMAP set are skipped entirely.  The pages behind
-   these mappings are inherently pinned, so we don't need to mark them as
-   mlocked.  In any case, most of the pages have no struct page in which to so
-   mark the page.  Because of this, get_user_pages() will fail for these VMAs,
-   so there is no sense in attempting to visit them.
-
-2) VMAs mapping hugetlbfs page are already effectively pinned into memory.  We
-   neither need nor want to mlock() these pages.  However, to preserve the
-   prior behavior of mlock() - before the unevictable/mlock changes -
-   mlock_fixup() will call make_pages_present() in the hugetlbfs VMA range to
-   allocate the huge pages and populate the ptes.
-
-3) VMAs with VM_DONTEXPAND are generally userspace mappings of kernel pages,
-   such as the VDSO page, relay channel pages, etc. These pages
-   are inherently unevictable and are not managed on the LRU lists.
-   mlock_fixup() treats these VMAs the same as hugetlbfs VMAs.  It calls
-   make_pages_present() to populate the ptes.
-
-Note that for all of these special VMAs, mlock_fixup() does not set the
-VM_LOCKED flag.  Therefore, we won't have to deal with them later during
-munlock(), munmap() or task exit.  Neither does mlock_fixup() account these
-VMAs against the task's "locked_vm".
-
-
-munlock()/munlockall() SYSTEM CALL HANDLING
--------------------------------------------
-
-The munlock() and munlockall() system calls are handled by the same functions -
-do_mlock[all]() - as the mlock() and mlockall() system calls with the unlock vs
-lock operation indicated by an argument.  So, these system calls are also
-handled by mlock_fixup().  Again, if called for an already munlocked VMA,
-mlock_fixup() simply returns.  Because of the VMA filtering discussed above,
-VM_LOCKED will not be set in any "special" VMAs.  So, these VMAs will be
-ignored for munlock.
-
-If the VMA is VM_LOCKED, mlock_fixup() again attempts to merge or split off the
-specified range.  The range is then munlocked via the function
-populate_vma_page_range() - the same function used to mlock a VMA range -
-passing a flag to indicate that munlock() is being performed.
-
-Because the VMA access protections could have been changed to PROT_NONE after
-faulting in and mlocking pages, get_user_pages() was unreliable for visiting
-these pages for munlocking.  Because we don't want to leave pages mlocked,
-get_user_pages() was enhanced to accept a flag to ignore the permissions when
-fetching the pages - all of which should be resident as a result of previous
-mlocking.
-
-For munlock(), populate_vma_page_range() unlocks individual pages by calling
-munlock_vma_page().  munlock_vma_page() unconditionally clears the PG_mlocked
-flag using TestClearPageMlocked().  As with mlock_vma_page(),
-munlock_vma_page() use the Test*PageMlocked() function to handle the case where
-the page might have already been unlocked by another task.  If the page was
-mlocked, munlock_vma_page() updates that zone statistics for the number of
-mlocked pages.  Note, however, that at this point we haven't checked whether
-the page is mapped by other VM_LOCKED VMAs.
-
-We can't call try_to_munlock(), the function that walks the reverse map to
-check for other VM_LOCKED VMAs, without first isolating the page from the LRU.
-try_to_munlock() is a variant of try_to_unmap() and thus requires that the page
-not be on an LRU list [more on these below].  However, the call to
-isolate_lru_page() could fail, in which case we couldn't try_to_munlock().  So,
-we go ahead and clear PG_mlocked up front, as this might be the only chance we
-have.  If we can successfully isolate the page, we go ahead and
-try_to_munlock(), which will restore the PG_mlocked flag and update the zone
-page statistics if it finds another VMA holding the page mlocked.  If we fail
-to isolate the page, we'll have left a potentially mlocked page on the LRU.
-This is fine, because we'll catch it later if and if vmscan tries to reclaim
-the page.  This should be relatively rare.
-
-
-MIGRATING MLOCKED PAGES
------------------------
-
-A page that is being migrated has been isolated from the LRU lists and is held
-locked across unmapping of the page, updating the page's address space entry
-and copying the contents and state, until the page table entry has been
-replaced with an entry that refers to the new page.  Linux supports migration
-of mlocked pages and other unevictable pages.  This involves simply moving the
-PG_mlocked and PG_unevictable states from the old page to the new page.
-
-Note that page migration can race with mlocking or munlocking of the same page.
-This has been discussed from the mlock/munlock perspective in the respective
-sections above.  Both processes (migration and m[un]locking) hold the page
-locked.  This provides the first level of synchronization.  Page migration
-zeros out the page_mapping of the old page before unlocking it, so m[un]lock
-can skip these pages by testing the page mapping under page lock.
-
-To complete page migration, we place the new and old pages back onto the LRU
-after dropping the page lock.  The "unneeded" page - old page on success, new
-page on failure - will be freed when the reference count held by the migration
-process is released.  To ensure that we don't strand pages on the unevictable
-list because of a race between munlock and migration, page migration uses the
-putback_lru_page() function to add migrated pages back to the LRU.
-
-
-COMPACTING MLOCKED PAGES
-------------------------
-
-The unevictable LRU can be scanned for compactable regions and the default
-behavior is to do so.  /proc/sys/vm/compact_unevictable_allowed controls
-this behavior (see Documentation/sysctl/vm.txt).  Once scanning of the
-unevictable LRU is enabled, the work of compaction is mostly handled by
-the page migration code and the same work flow as described in MIGRATING
-MLOCKED PAGES will apply.
-
-MLOCKING TRANSPARENT HUGE PAGES
--------------------------------
-
-A transparent huge page is represented by a single entry on an LRU list.
-Therefore, we can only make unevictable an entire compound page, not
-individual subpages.
-
-If a user tries to mlock() part of a huge page, we want the rest of the
-page to be reclaimable.
-
-We cannot just split the page on partial mlock() as split_huge_page() can
-fail and new intermittent failure mode for the syscall is undesirable.
-
-We handle this by keeping PTE-mapped huge pages on normal LRU lists: the
-PMD on border of VM_LOCKED VMA will be split into PTE table.
-
-This way the huge page is accessible for vmscan. Under memory pressure the
-page will be split, subpages which belong to VM_LOCKED VMAs will be moved
-to unevictable LRU and the rest can be reclaimed.
-
-See also comment in follow_trans_huge_pmd().
-
-mmap(MAP_LOCKED) SYSTEM CALL HANDLING
--------------------------------------
-
-In addition the mlock()/mlockall() system calls, an application can request
-that a region of memory be mlocked supplying the MAP_LOCKED flag to the mmap()
-call. There is one important and subtle difference here, though. mmap() + mlock()
-will fail if the range cannot be faulted in (e.g. because mm_populate fails)
-and returns with ENOMEM while mmap(MAP_LOCKED) will not fail. The mmaped
-area will still have properties of the locked area - aka. pages will not get
-swapped out - but major page faults to fault memory in might still happen.
-
-Furthermore, any mmap() call or brk() call that expands the heap by a
-task that has previously called mlockall() with the MCL_FUTURE flag will result
-in the newly mapped memory being mlocked.  Before the unevictable/mlock
-changes, the kernel simply called make_pages_present() to allocate pages and
-populate the page table.
-
-To mlock a range of memory under the unevictable/mlock infrastructure, the
-mmap() handler and task address space expansion functions call
-populate_vma_page_range() specifying the vma and the address range to mlock.
-
-The callers of populate_vma_page_range() will have already added the memory range
-to be mlocked to the task's "locked_vm".  To account for filtered VMAs,
-populate_vma_page_range() returns the number of pages NOT mlocked.  All of the
-callers then subtract a non-negative return value from the task's locked_vm.  A
-negative return value represent an error - for example, from get_user_pages()
-attempting to fault in a VMA with PROT_NONE access.  In this case, we leave the
-memory range accounted as locked_vm, as the protections could be changed later
-and pages allocated into that region.
-
-
-munmap()/exit()/exec() SYSTEM CALL HANDLING
--------------------------------------------
-
-When unmapping an mlocked region of memory, whether by an explicit call to
-munmap() or via an internal unmap from exit() or exec() processing, we must
-munlock the pages if we're removing the last VM_LOCKED VMA that maps the pages.
-Before the unevictable/mlock changes, mlocking did not mark the pages in any
-way, so unmapping them required no processing.
-
-To munlock a range of memory under the unevictable/mlock infrastructure, the
-munmap() handler and task address space call tear down function
-munlock_vma_pages_all().  The name reflects the observation that one always
-specifies the entire VMA range when munlock()ing during unmap of a region.
-Because of the VMA filtering when mlocking() regions, only "normal" VMAs that
-actually contain mlocked pages will be passed to munlock_vma_pages_all().
-
-munlock_vma_pages_all() clears the VM_LOCKED VMA flag and, like mlock_fixup()
-for the munlock case, calls __munlock_vma_pages_range() to walk the page table
-for the VMA's memory range and munlock_vma_page() each resident page mapped by
-the VMA.  This effectively munlocks the page, only if this is the last
-VM_LOCKED VMA that maps the page.
-
-
-try_to_unmap()
---------------
-
-Pages can, of course, be mapped into multiple VMAs.  Some of these VMAs may
-have VM_LOCKED flag set.  It is possible for a page mapped into one or more
-VM_LOCKED VMAs not to have the PG_mlocked flag set and therefore reside on one
-of the active or inactive LRU lists.  This could happen if, for example, a task
-in the process of munlocking the page could not isolate the page from the LRU.
-As a result, vmscan/shrink_page_list() might encounter such a page as described
-in section "vmscan's handling of unevictable pages".  To handle this situation,
-try_to_unmap() checks for VM_LOCKED VMAs while it is walking a page's reverse
-map.
-
-try_to_unmap() is always called, by either vmscan for reclaim or for page
-migration, with the argument page locked and isolated from the LRU.  Separate
-functions handle anonymous and mapped file and KSM pages, as these types of
-pages have different reverse map lookup mechanisms, with different locking.
-In each case, whether rmap_walk_anon() or rmap_walk_file() or rmap_walk_ksm(),
-it will call try_to_unmap_one() for every VMA which might contain the page.
-
-When trying to reclaim, if try_to_unmap_one() finds the page in a VM_LOCKED
-VMA, it will then mlock the page via mlock_vma_page() instead of unmapping it,
-and return SWAP_MLOCK to indicate that the page is unevictable: and the scan
-stops there.
-
-mlock_vma_page() is called while holding the page table's lock (in addition
-to the page lock, and the rmap lock): to serialize against concurrent mlock or
-munlock or munmap system calls, mm teardown (munlock_vma_pages_all), reclaim,
-holepunching, and truncation of file pages and their anonymous COWed pages.
-
-
-try_to_munlock() REVERSE MAP SCAN
----------------------------------
-
- [!] TODO/FIXME: a better name might be page_mlocked() - analogous to the
-     page_referenced() reverse map walker.
-
-When munlock_vma_page() [see section "munlock()/munlockall() System Call
-Handling" above] tries to munlock a page, it needs to determine whether or not
-the page is mapped by any VM_LOCKED VMA without actually attempting to unmap
-all PTEs from the page.  For this purpose, the unevictable/mlock infrastructure
-introduced a variant of try_to_unmap() called try_to_munlock().
-
-try_to_munlock() calls the same functions as try_to_unmap() for anonymous and
-mapped file and KSM pages with a flag argument specifying unlock versus unmap
-processing.  Again, these functions walk the respective reverse maps looking
-for VM_LOCKED VMAs.  When such a VMA is found, as in the try_to_unmap() case,
-the functions mlock the page via mlock_vma_page() and return SWAP_MLOCK.  This
-undoes the pre-clearing of the page's PG_mlocked done by munlock_vma_page.
-
-Note that try_to_munlock()'s reverse map walk must visit every VMA in a page's
-reverse map to determine that a page is NOT mapped into any VM_LOCKED VMA.
-However, the scan can terminate when it encounters a VM_LOCKED VMA.
-Although try_to_munlock() might be called a great many times when munlocking a
-large region or tearing down a large address space that has been mlocked via
-mlockall(), overall this is a fairly rare event.
-
-
-PAGE RECLAIM IN shrink_*_list()
--------------------------------
-
-shrink_active_list() culls any obviously unevictable pages - i.e.
-!page_evictable(page) - diverting these to the unevictable list.
-However, shrink_active_list() only sees unevictable pages that made it onto the
-active/inactive lru lists.  Note that these pages do not have PageUnevictable
-set - otherwise they would be on the unevictable list and shrink_active_list
-would never see them.
-
-Some examples of these unevictable pages on the LRU lists are:
-
- (1) ramfs pages that have been placed on the LRU lists when first allocated.
-
- (2) SHM_LOCK'd shared memory pages.  shmctl(SHM_LOCK) does not attempt to
-     allocate or fault in the pages in the shared memory region.  This happens
-     when an application accesses the page the first time after SHM_LOCK'ing
-     the segment.
-
- (3) mlocked pages that could not be isolated from the LRU and moved to the
-     unevictable list in mlock_vma_page().
-
-shrink_inactive_list() also diverts any unevictable pages that it finds on the
-inactive lists to the appropriate zone's unevictable list.
-
-shrink_inactive_list() should only see SHM_LOCK'd pages that became SHM_LOCK'd
-after shrink_active_list() had moved them to the inactive list, or pages mapped
-into VM_LOCKED VMAs that munlock_vma_page() couldn't isolate from the LRU to
-recheck via try_to_munlock().  shrink_inactive_list() won't notice the latter,
-but will pass on to shrink_page_list().
-
-shrink_page_list() again culls obviously unevictable pages that it could
-encounter for similar reason to shrink_inactive_list().  Pages mapped into
-VM_LOCKED VMAs but without PG_mlocked set will make it all the way to
-try_to_unmap().  shrink_page_list() will divert them to the unevictable list
-when try_to_unmap() returns SWAP_MLOCK, as discussed above.
diff --git a/Documentation/vm/userfaultfd.txt b/Documentation/vm/userfaultfd.txt
deleted file mode 100644
index bb2f945f87ab..000000000000
--- a/Documentation/vm/userfaultfd.txt
+++ /dev/null
@@ -1,229 +0,0 @@
-= Userfaultfd =
-
-== Objective ==
-
-Userfaults allow the implementation of on-demand paging from userland
-and more generally they allow userland to take control of various
-memory page faults, something otherwise only the kernel code could do.
-
-For example userfaults allows a proper and more optimal implementation
-of the PROT_NONE+SIGSEGV trick.
-
-== Design ==
-
-Userfaults are delivered and resolved through the userfaultfd syscall.
-
-The userfaultfd (aside from registering and unregistering virtual
-memory ranges) provides two primary functionalities:
-
-1) read/POLLIN protocol to notify a userland thread of the faults
-   happening
-
-2) various UFFDIO_* ioctls that can manage the virtual memory regions
-   registered in the userfaultfd that allows userland to efficiently
-   resolve the userfaults it receives via 1) or to manage the virtual
-   memory in the background
-
-The real advantage of userfaults if compared to regular virtual memory
-management of mremap/mprotect is that the userfaults in all their
-operations never involve heavyweight structures like vmas (in fact the
-userfaultfd runtime load never takes the mmap_sem for writing).
-
-Vmas are not suitable for page- (or hugepage) granular fault tracking
-when dealing with virtual address spaces that could span
-Terabytes. Too many vmas would be needed for that.
-
-The userfaultfd once opened by invoking the syscall, can also be
-passed using unix domain sockets to a manager process, so the same
-manager process could handle the userfaults of a multitude of
-different processes without them being aware about what is going on
-(well of course unless they later try to use the userfaultfd
-themselves on the same region the manager is already tracking, which
-is a corner case that would currently return -EBUSY).
-
-== API ==
-
-When first opened the userfaultfd must be enabled invoking the
-UFFDIO_API ioctl specifying a uffdio_api.api value set to UFFD_API (or
-a later API version) which will specify the read/POLLIN protocol
-userland intends to speak on the UFFD and the uffdio_api.features
-userland requires. The UFFDIO_API ioctl if successful (i.e. if the
-requested uffdio_api.api is spoken also by the running kernel and the
-requested features are going to be enabled) will return into
-uffdio_api.features and uffdio_api.ioctls two 64bit bitmasks of
-respectively all the available features of the read(2) protocol and
-the generic ioctl available.
-
-The uffdio_api.features bitmask returned by the UFFDIO_API ioctl
-defines what memory types are supported by the userfaultfd and what
-events, except page fault notifications, may be generated.
-
-If the kernel supports registering userfaultfd ranges on hugetlbfs
-virtual memory areas, UFFD_FEATURE_MISSING_HUGETLBFS will be set in
-uffdio_api.features. Similarly, UFFD_FEATURE_MISSING_SHMEM will be
-set if the kernel supports registering userfaultfd ranges on shared
-memory (covering all shmem APIs, i.e. tmpfs, IPCSHM, /dev/zero
-MAP_SHARED, memfd_create, etc).
-
-The userland application that wants to use userfaultfd with hugetlbfs
-or shared memory need to set the corresponding flag in
-uffdio_api.features to enable those features.
-
-If the userland desires to receive notifications for events other than
-page faults, it has to verify that uffdio_api.features has appropriate
-UFFD_FEATURE_EVENT_* bits set. These events are described in more
-detail below in "Non-cooperative userfaultfd" section.
-
-Once the userfaultfd has been enabled the UFFDIO_REGISTER ioctl should
-be invoked (if present in the returned uffdio_api.ioctls bitmask) to
-register a memory range in the userfaultfd by setting the
-uffdio_register structure accordingly. The uffdio_register.mode
-bitmask will specify to the kernel which kind of faults to track for
-the range (UFFDIO_REGISTER_MODE_MISSING would track missing
-pages). The UFFDIO_REGISTER ioctl will return the
-uffdio_register.ioctls bitmask of ioctls that are suitable to resolve
-userfaults on the range registered. Not all ioctls will necessarily be
-supported for all memory types depending on the underlying virtual
-memory backend (anonymous memory vs tmpfs vs real filebacked
-mappings).
-
-Userland can use the uffdio_register.ioctls to manage the virtual
-address space in the background (to add or potentially also remove
-memory from the userfaultfd registered range). This means a userfault
-could be triggering just before userland maps in the background the
-user-faulted page.
-
-The primary ioctl to resolve userfaults is UFFDIO_COPY. That
-atomically copies a page into the userfault registered range and wakes
-up the blocked userfaults (unless uffdio_copy.mode &
-UFFDIO_COPY_MODE_DONTWAKE is set). Other ioctl works similarly to
-UFFDIO_COPY. They're atomic as in guaranteeing that nothing can see an
-half copied page since it'll keep userfaulting until the copy has
-finished.
-
-== QEMU/KVM ==
-
-QEMU/KVM is using the userfaultfd syscall to implement postcopy live
-migration. Postcopy live migration is one form of memory
-externalization consisting of a virtual machine running with part or
-all of its memory residing on a different node in the cloud. The
-userfaultfd abstraction is generic enough that not a single line of
-KVM kernel code had to be modified in order to add postcopy live
-migration to QEMU.
-
-Guest async page faults, FOLL_NOWAIT and all other GUP features work
-just fine in combination with userfaults. Userfaults trigger async
-page faults in the guest scheduler so those guest processes that
-aren't waiting for userfaults (i.e. network bound) can keep running in
-the guest vcpus.
-
-It is generally beneficial to run one pass of precopy live migration
-just before starting postcopy live migration, in order to avoid
-generating userfaults for readonly guest regions.
-
-The implementation of postcopy live migration currently uses one
-single bidirectional socket but in the future two different sockets
-will be used (to reduce the latency of the userfaults to the minimum
-possible without having to decrease /proc/sys/net/ipv4/tcp_wmem).
-
-The QEMU in the source node writes all pages that it knows are missing
-in the destination node, into the socket, and the migration thread of
-the QEMU running in the destination node runs UFFDIO_COPY|ZEROPAGE
-ioctls on the userfaultfd in order to map the received pages into the
-guest (UFFDIO_ZEROCOPY is used if the source page was a zero page).
-
-A different postcopy thread in the destination node listens with
-poll() to the userfaultfd in parallel. When a POLLIN event is
-generated after a userfault triggers, the postcopy thread read() from
-the userfaultfd and receives the fault address (or -EAGAIN in case the
-userfault was already resolved and waken by a UFFDIO_COPY|ZEROPAGE run
-by the parallel QEMU migration thread).
-
-After the QEMU postcopy thread (running in the destination node) gets
-the userfault address it writes the information about the missing page
-into the socket. The QEMU source node receives the information and
-roughly "seeks" to that page address and continues sending all
-remaining missing pages from that new page offset. Soon after that
-(just the time to flush the tcp_wmem queue through the network) the
-migration thread in the QEMU running in the destination node will
-receive the page that triggered the userfault and it'll map it as
-usual with the UFFDIO_COPY|ZEROPAGE (without actually knowing if it
-was spontaneously sent by the source or if it was an urgent page
-requested through a userfault).
-
-By the time the userfaults start, the QEMU in the destination node
-doesn't need to keep any per-page state bitmap relative to the live
-migration around and a single per-page bitmap has to be maintained in
-the QEMU running in the source node to know which pages are still
-missing in the destination node. The bitmap in the source node is
-checked to find which missing pages to send in round robin and we seek
-over it when receiving incoming userfaults. After sending each page of
-course the bitmap is updated accordingly. It's also useful to avoid
-sending the same page twice (in case the userfault is read by the
-postcopy thread just before UFFDIO_COPY|ZEROPAGE runs in the migration
-thread).
-
-== Non-cooperative userfaultfd ==
-
-When the userfaultfd is monitored by an external manager, the manager
-must be able to track changes in the process virtual memory
-layout. Userfaultfd can notify the manager about such changes using
-the same read(2) protocol as for the page fault notifications. The
-manager has to explicitly enable these events by setting appropriate
-bits in uffdio_api.features passed to UFFDIO_API ioctl:
-
-UFFD_FEATURE_EVENT_FORK - enable userfaultfd hooks for fork(). When
-this feature is enabled, the userfaultfd context of the parent process
-is duplicated into the newly created process. The manager receives
-UFFD_EVENT_FORK with file descriptor of the new userfaultfd context in
-the uffd_msg.fork.
-
-UFFD_FEATURE_EVENT_REMAP - enable notifications about mremap()
-calls. When the non-cooperative process moves a virtual memory area to
-a different location, the manager will receive UFFD_EVENT_REMAP. The
-uffd_msg.remap will contain the old and new addresses of the area and
-its original length.
-
-UFFD_FEATURE_EVENT_REMOVE - enable notifications about
-madvise(MADV_REMOVE) and madvise(MADV_DONTNEED) calls. The event
-UFFD_EVENT_REMOVE will be generated upon these calls to madvise. The
-uffd_msg.remove will contain start and end addresses of the removed
-area.
-
-UFFD_FEATURE_EVENT_UNMAP - enable notifications about memory
-unmapping. The manager will get UFFD_EVENT_UNMAP with uffd_msg.remove
-containing start and end addresses of the unmapped area.
-
-Although the UFFD_FEATURE_EVENT_REMOVE and UFFD_FEATURE_EVENT_UNMAP
-are pretty similar, they quite differ in the action expected from the
-userfaultfd manager. In the former case, the virtual memory is
-removed, but the area is not, the area remains monitored by the
-userfaultfd, and if a page fault occurs in that area it will be
-delivered to the manager. The proper resolution for such page fault is
-to zeromap the faulting address. However, in the latter case, when an
-area is unmapped, either explicitly (with munmap() system call), or
-implicitly (e.g. during mremap()), the area is removed and in turn the
-userfaultfd context for such area disappears too and the manager will
-not get further userland page faults from the removed area. Still, the
-notification is required in order to prevent manager from using
-UFFDIO_COPY on the unmapped area.
-
-Unlike userland page faults which have to be synchronous and require
-explicit or implicit wakeup, all the events are delivered
-asynchronously and the non-cooperative process resumes execution as
-soon as manager executes read(). The userfaultfd manager should
-carefully synchronize calls to UFFDIO_COPY with the events
-processing. To aid the synchronization, the UFFDIO_COPY ioctl will
-return -ENOSPC when the monitored process exits at the time of
-UFFDIO_COPY, and -ENOENT, when the non-cooperative process has changed
-its virtual memory layout simultaneously with outstanding UFFDIO_COPY
-operation.
-
-The current asynchronous model of the event delivery is optimal for
-single threaded non-cooperative userfaultfd manager implementations. A
-synchronous event delivery model can be added later as a new
-userfaultfd feature to facilitate multithreading enhancements of the
-non cooperative manager, for example to allow UFFDIO_COPY ioctls to
-run in parallel to the event reception. Single threaded
-implementations should continue to use the current async event
-delivery model instead.
diff --git a/Documentation/vm/z3fold.rst b/Documentation/vm/z3fold.rst
new file mode 100644
index 000000000000..224e3c61d686
--- /dev/null
+++ b/Documentation/vm/z3fold.rst
@@ -0,0 +1,30 @@
+.. _z3fold:
+
+======
+z3fold
+======
+
+z3fold is a special purpose allocator for storing compressed pages.
+It is designed to store up to three compressed pages per physical page.
+It is a zbud derivative which allows for higher compression
+ratio keeping the simplicity and determinism of its predecessor.
+
+The main differences between z3fold and zbud are:
+
+* unlike zbud, z3fold allows for up to PAGE_SIZE allocations
+* z3fold can hold up to 3 compressed pages in its page
+* z3fold doesn't export any API itself and is thus intended to be used
+  via the zpool API.
+
+To keep the determinism and simplicity, z3fold, just like zbud, always
+stores an integral number of compressed pages per page, but it can store
+up to 3 pages unlike zbud which can store at most 2. Therefore the
+compression ratio goes to around 2.7x while zbud's one is around 1.7x.
+
+Unlike zbud (but like zsmalloc for that matter) z3fold_alloc() does not
+return a dereferenceable pointer. Instead, it returns an unsigned long
+handle which encodes actual location of the allocated object.
+
+Keeping effective compression ratio close to zsmalloc's, z3fold doesn't
+depend on MMU enabled and provides more predictable reclaim behavior
+which makes it a better fit for small and response-critical systems.
diff --git a/Documentation/vm/z3fold.txt b/Documentation/vm/z3fold.txt
deleted file mode 100644
index 38e4dac810b6..000000000000
--- a/Documentation/vm/z3fold.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-z3fold
-------
-
-z3fold is a special purpose allocator for storing compressed pages.
-It is designed to store up to three compressed pages per physical page.
-It is a zbud derivative which allows for higher compression
-ratio keeping the simplicity and determinism of its predecessor.
-
-The main differences between z3fold and zbud are:
-* unlike zbud, z3fold allows for up to PAGE_SIZE allocations
-* z3fold can hold up to 3 compressed pages in its page
-* z3fold doesn't export any API itself and is thus intended to be used
-  via the zpool API.
-
-To keep the determinism and simplicity, z3fold, just like zbud, always
-stores an integral number of compressed pages per page, but it can store
-up to 3 pages unlike zbud which can store at most 2. Therefore the
-compression ratio goes to around 2.7x while zbud's one is around 1.7x.
-
-Unlike zbud (but like zsmalloc for that matter) z3fold_alloc() does not
-return a dereferenceable pointer. Instead, it returns an unsigned long
-handle which encodes actual location of the allocated object.
-
-Keeping effective compression ratio close to zsmalloc's, z3fold doesn't
-depend on MMU enabled and provides more predictable reclaim behavior
-which makes it a better fit for small and response-critical systems.
diff --git a/Documentation/vm/zsmalloc.rst b/Documentation/vm/zsmalloc.rst
new file mode 100644
index 000000000000..6e79893d6132
--- /dev/null
+++ b/Documentation/vm/zsmalloc.rst
@@ -0,0 +1,82 @@
+.. _zsmalloc:
+
+========
+zsmalloc
+========
+
+This allocator is designed for use with zram. Thus, the allocator is
+supposed to work well under low memory conditions. In particular, it
+never attempts higher order page allocation which is very likely to
+fail under memory pressure. On the other hand, if we just use single
+(0-order) pages, it would suffer from very high fragmentation --
+any object of size PAGE_SIZE/2 or larger would occupy an entire page.
+This was one of the major issues with its predecessor (xvmalloc).
+
+To overcome these issues, zsmalloc allocates a bunch of 0-order pages
+and links them together using various 'struct page' fields. These linked
+pages act as a single higher-order page i.e. an object can span 0-order
+page boundaries. The code refers to these linked pages as a single entity
+called zspage.
+
+For simplicity, zsmalloc can only allocate objects of size up to PAGE_SIZE
+since this satisfies the requirements of all its current users (in the
+worst case, page is incompressible and is thus stored "as-is" i.e. in
+uncompressed form). For allocation requests larger than this size, failure
+is returned (see zs_malloc).
+
+Additionally, zs_malloc() does not return a dereferenceable pointer.
+Instead, it returns an opaque handle (unsigned long) which encodes actual
+location of the allocated object. The reason for this indirection is that
+zsmalloc does not keep zspages permanently mapped since that would cause
+issues on 32-bit systems where the VA region for kernel space mappings
+is very small. So, before using the allocating memory, the object has to
+be mapped using zs_map_object() to get a usable pointer and subsequently
+unmapped using zs_unmap_object().
+
+stat
+====
+
+With CONFIG_ZSMALLOC_STAT, we could see zsmalloc internal information via
+``/sys/kernel/debug/zsmalloc/<user name>``. Here is a sample of stat output::
+
+ # cat /sys/kernel/debug/zsmalloc/zram0/classes
+
+ class  size almost_full almost_empty obj_allocated   obj_used pages_used pages_per_zspage
+    ...
+    ...
+     9   176           0            1           186        129          8                4
+    10   192           1            0          2880       2872        135                3
+    11   208           0            1           819        795         42                2
+    12   224           0            1           219        159         12                4
+    ...
+    ...
+
+
+class
+	index
+size
+	object size zspage stores
+almost_empty
+	the number of ZS_ALMOST_EMPTY zspages(see below)
+almost_full
+	the number of ZS_ALMOST_FULL zspages(see below)
+obj_allocated
+	the number of objects allocated
+obj_used
+	the number of objects allocated to the user
+pages_used
+	the number of pages allocated for the class
+pages_per_zspage
+	the number of 0-order pages to make a zspage
+
+We assign a zspage to ZS_ALMOST_EMPTY fullness group when n <= N / f, where
+
+* n = number of allocated objects
+* N = total number of objects zspage can store
+* f = fullness_threshold_frac(ie, 4 at the moment)
+
+Similarly, we assign zspage to:
+
+* ZS_ALMOST_FULL  when n > N / f
+* ZS_EMPTY        when n == 0
+* ZS_FULL         when n == N
diff --git a/Documentation/vm/zsmalloc.txt b/Documentation/vm/zsmalloc.txt
deleted file mode 100644
index 64ed63c4f69d..000000000000
--- a/Documentation/vm/zsmalloc.txt
+++ /dev/null
@@ -1,70 +0,0 @@
-zsmalloc
---------
-
-This allocator is designed for use with zram. Thus, the allocator is
-supposed to work well under low memory conditions. In particular, it
-never attempts higher order page allocation which is very likely to
-fail under memory pressure. On the other hand, if we just use single
-(0-order) pages, it would suffer from very high fragmentation --
-any object of size PAGE_SIZE/2 or larger would occupy an entire page.
-This was one of the major issues with its predecessor (xvmalloc).
-
-To overcome these issues, zsmalloc allocates a bunch of 0-order pages
-and links them together using various 'struct page' fields. These linked
-pages act as a single higher-order page i.e. an object can span 0-order
-page boundaries. The code refers to these linked pages as a single entity
-called zspage.
-
-For simplicity, zsmalloc can only allocate objects of size up to PAGE_SIZE
-since this satisfies the requirements of all its current users (in the
-worst case, page is incompressible and is thus stored "as-is" i.e. in
-uncompressed form). For allocation requests larger than this size, failure
-is returned (see zs_malloc).
-
-Additionally, zs_malloc() does not return a dereferenceable pointer.
-Instead, it returns an opaque handle (unsigned long) which encodes actual
-location of the allocated object. The reason for this indirection is that
-zsmalloc does not keep zspages permanently mapped since that would cause
-issues on 32-bit systems where the VA region for kernel space mappings
-is very small. So, before using the allocating memory, the object has to
-be mapped using zs_map_object() to get a usable pointer and subsequently
-unmapped using zs_unmap_object().
-
-stat
-----
-
-With CONFIG_ZSMALLOC_STAT, we could see zsmalloc internal information via
-/sys/kernel/debug/zsmalloc/<user name>. Here is a sample of stat output:
-
-# cat /sys/kernel/debug/zsmalloc/zram0/classes
-
- class  size almost_full almost_empty obj_allocated   obj_used pages_used pages_per_zspage
-    ..
-    ..
-     9   176           0            1           186        129          8                4
-    10   192           1            0          2880       2872        135                3
-    11   208           0            1           819        795         42                2
-    12   224           0            1           219        159         12                4
-    ..
-    ..
-
-
-class: index
-size: object size zspage stores
-almost_empty: the number of ZS_ALMOST_EMPTY zspages(see below)
-almost_full: the number of ZS_ALMOST_FULL zspages(see below)
-obj_allocated: the number of objects allocated
-obj_used: the number of objects allocated to the user
-pages_used: the number of pages allocated for the class
-pages_per_zspage: the number of 0-order pages to make a zspage
-
-We assign a zspage to ZS_ALMOST_EMPTY fullness group when:
-      n <= N / f, where
-n = number of allocated objects
-N = total number of objects zspage can store
-f = fullness_threshold_frac(ie, 4 at the moment)
-
-Similarly, we assign zspage to:
-      ZS_ALMOST_FULL  when n > N / f
-      ZS_EMPTY        when n == 0
-      ZS_FULL         when n == N
diff --git a/Documentation/vm/zswap.rst b/Documentation/vm/zswap.rst
new file mode 100644
index 000000000000..1444ecd40911
--- /dev/null
+++ b/Documentation/vm/zswap.rst
@@ -0,0 +1,135 @@
+.. _zswap:
+
+=====
+zswap
+=====
+
+Overview
+========
+
+Zswap is a lightweight compressed cache for swap pages. It takes pages that are
+in the process of being swapped out and attempts to compress them into a
+dynamically allocated RAM-based memory pool.  zswap basically trades CPU cycles
+for potentially reduced swap I/O.  This trade-off can also result in a
+significant performance improvement if reads from the compressed cache are
+faster than reads from a swap device.
+
+.. note::
+   Zswap is a new feature as of v3.11 and interacts heavily with memory
+   reclaim.  This interaction has not been fully explored on the large set of
+   potential configurations and workloads that exist.  For this reason, zswap
+   is a work in progress and should be considered experimental.
+
+   Some potential benefits:
+
+* Desktop/laptop users with limited RAM capacities can mitigate the
+  performance impact of swapping.
+* Overcommitted guests that share a common I/O resource can
+  dramatically reduce their swap I/O pressure, avoiding heavy handed I/O
+  throttling by the hypervisor. This allows more work to get done with less
+  impact to the guest workload and guests sharing the I/O subsystem
+* Users with SSDs as swap devices can extend the life of the device by
+  drastically reducing life-shortening writes.
+
+Zswap evicts pages from compressed cache on an LRU basis to the backing swap
+device when the compressed pool reaches its size limit.  This requirement had
+been identified in prior community discussions.
+
+Zswap is disabled by default but can be enabled at boot time by setting
+the ``enabled`` attribute to 1 at boot time. ie: ``zswap.enabled=1``.  Zswap
+can also be enabled and disabled at runtime using the sysfs interface.
+An example command to enable zswap at runtime, assuming sysfs is mounted
+at ``/sys``, is::
+
+	echo 1 > /sys/module/zswap/parameters/enabled
+
+When zswap is disabled at runtime it will stop storing pages that are
+being swapped out.  However, it will _not_ immediately write out or fault
+back into memory all of the pages stored in the compressed pool.  The
+pages stored in zswap will remain in the compressed pool until they are
+either invalidated or faulted back into memory.  In order to force all
+pages out of the compressed pool, a swapoff on the swap device(s) will
+fault back into memory all swapped out pages, including those in the
+compressed pool.
+
+Design
+======
+
+Zswap receives pages for compression through the Frontswap API and is able to
+evict pages from its own compressed pool on an LRU basis and write them back to
+the backing swap device in the case that the compressed pool is full.
+
+Zswap makes use of zpool for the managing the compressed memory pool.  Each
+allocation in zpool is not directly accessible by address.  Rather, a handle is
+returned by the allocation routine and that handle must be mapped before being
+accessed.  The compressed memory pool grows on demand and shrinks as compressed
+pages are freed.  The pool is not preallocated.  By default, a zpool
+of type zbud is created, but it can be selected at boot time by
+setting the ``zpool`` attribute, e.g. ``zswap.zpool=zbud``. It can
+also be changed at runtime using the sysfs ``zpool`` attribute, e.g.::
+
+	echo zbud > /sys/module/zswap/parameters/zpool
+
+The zbud type zpool allocates exactly 1 page to store 2 compressed pages, which
+means the compression ratio will always be 2:1 or worse (because of half-full
+zbud pages).  The zsmalloc type zpool has a more complex compressed page
+storage method, and it can achieve greater storage densities.  However,
+zsmalloc does not implement compressed page eviction, so once zswap fills it
+cannot evict the oldest page, it can only reject new pages.
+
+When a swap page is passed from frontswap to zswap, zswap maintains a mapping
+of the swap entry, a combination of the swap type and swap offset, to the zpool
+handle that references that compressed swap page.  This mapping is achieved
+with a red-black tree per swap type.  The swap offset is the search key for the
+tree nodes.
+
+During a page fault on a PTE that is a swap entry, frontswap calls the zswap
+load function to decompress the page into the page allocated by the page fault
+handler.
+
+Once there are no PTEs referencing a swap page stored in zswap (i.e. the count
+in the swap_map goes to 0) the swap code calls the zswap invalidate function,
+via frontswap, to free the compressed entry.
+
+Zswap seeks to be simple in its policies.  Sysfs attributes allow for one user
+controlled policy:
+
+* max_pool_percent - The maximum percentage of memory that the compressed
+  pool can occupy.
+
+The default compressor is lzo, but it can be selected at boot time by
+setting the ``compressor`` attribute, e.g. ``zswap.compressor=lzo``.
+It can also be changed at runtime using the sysfs "compressor"
+attribute, e.g.::
+
+	echo lzo > /sys/module/zswap/parameters/compressor
+
+When the zpool and/or compressor parameter is changed at runtime, any existing
+compressed pages are not modified; they are left in their own zpool.  When a
+request is made for a page in an old zpool, it is uncompressed using its
+original compressor.  Once all pages are removed from an old zpool, the zpool
+and its compressor are freed.
+
+Some of the pages in zswap are same-value filled pages (i.e. contents of the
+page have same value or repetitive pattern). These pages include zero-filled
+pages and they are handled differently. During store operation, a page is
+checked if it is a same-value filled page before compressing it. If true, the
+compressed length of the page is set to zero and the pattern or same-filled
+value is stored.
+
+Same-value filled pages identification feature is enabled by default and can be
+disabled at boot time by setting the ``same_filled_pages_enabled`` attribute
+to 0, e.g. ``zswap.same_filled_pages_enabled=0``. It can also be enabled and
+disabled at runtime using the sysfs ``same_filled_pages_enabled``
+attribute, e.g.::
+
+	echo 1 > /sys/module/zswap/parameters/same_filled_pages_enabled
+
+When zswap same-filled page identification is disabled at runtime, it will stop
+checking for the same-value filled pages during store operation. However, the
+existing pages which are marked as same-value filled pages remain stored
+unchanged in zswap until they are either loaded or invalidated.
+
+A debugfs interface is provided for various statistic about pool size, number
+of pages stored, same-value filled pages and various counters for the reasons
+pages are rejected.
diff --git a/Documentation/vm/zswap.txt b/Documentation/vm/zswap.txt
deleted file mode 100644
index 0b3a1148f9f0..000000000000
--- a/Documentation/vm/zswap.txt
+++ /dev/null
@@ -1,122 +0,0 @@
-Overview:
-
-Zswap is a lightweight compressed cache for swap pages. It takes pages that are
-in the process of being swapped out and attempts to compress them into a
-dynamically allocated RAM-based memory pool.  zswap basically trades CPU cycles
-for potentially reduced swap I/O.  This trade-off can also result in a
-significant performance improvement if reads from the compressed cache are
-faster than reads from a swap device.
-
-NOTE: Zswap is a new feature as of v3.11 and interacts heavily with memory
-reclaim.  This interaction has not been fully explored on the large set of
-potential configurations and workloads that exist.  For this reason, zswap
-is a work in progress and should be considered experimental.
-
-Some potential benefits:
-* Desktop/laptop users with limited RAM capacities can mitigate the
-    performance impact of swapping.
-* Overcommitted guests that share a common I/O resource can
-    dramatically reduce their swap I/O pressure, avoiding heavy handed I/O
-    throttling by the hypervisor. This allows more work to get done with less
-    impact to the guest workload and guests sharing the I/O subsystem
-* Users with SSDs as swap devices can extend the life of the device by
-    drastically reducing life-shortening writes.
-
-Zswap evicts pages from compressed cache on an LRU basis to the backing swap
-device when the compressed pool reaches its size limit.  This requirement had
-been identified in prior community discussions.
-
-Zswap is disabled by default but can be enabled at boot time by setting
-the "enabled" attribute to 1 at boot time. ie: zswap.enabled=1.  Zswap
-can also be enabled and disabled at runtime using the sysfs interface.
-An example command to enable zswap at runtime, assuming sysfs is mounted
-at /sys, is:
-
-echo 1 > /sys/module/zswap/parameters/enabled
-
-When zswap is disabled at runtime it will stop storing pages that are
-being swapped out.  However, it will _not_ immediately write out or fault
-back into memory all of the pages stored in the compressed pool.  The
-pages stored in zswap will remain in the compressed pool until they are
-either invalidated or faulted back into memory.  In order to force all
-pages out of the compressed pool, a swapoff on the swap device(s) will
-fault back into memory all swapped out pages, including those in the
-compressed pool.
-
-Design:
-
-Zswap receives pages for compression through the Frontswap API and is able to
-evict pages from its own compressed pool on an LRU basis and write them back to
-the backing swap device in the case that the compressed pool is full.
-
-Zswap makes use of zpool for the managing the compressed memory pool.  Each
-allocation in zpool is not directly accessible by address.  Rather, a handle is
-returned by the allocation routine and that handle must be mapped before being
-accessed.  The compressed memory pool grows on demand and shrinks as compressed
-pages are freed.  The pool is not preallocated.  By default, a zpool of type
-zbud is created, but it can be selected at boot time by setting the "zpool"
-attribute, e.g. zswap.zpool=zbud.  It can also be changed at runtime using the
-sysfs "zpool" attribute, e.g.
-
-echo zbud > /sys/module/zswap/parameters/zpool
-
-The zbud type zpool allocates exactly 1 page to store 2 compressed pages, which
-means the compression ratio will always be 2:1 or worse (because of half-full
-zbud pages).  The zsmalloc type zpool has a more complex compressed page
-storage method, and it can achieve greater storage densities.  However,
-zsmalloc does not implement compressed page eviction, so once zswap fills it
-cannot evict the oldest page, it can only reject new pages.
-
-When a swap page is passed from frontswap to zswap, zswap maintains a mapping
-of the swap entry, a combination of the swap type and swap offset, to the zpool
-handle that references that compressed swap page.  This mapping is achieved
-with a red-black tree per swap type.  The swap offset is the search key for the
-tree nodes.
-
-During a page fault on a PTE that is a swap entry, frontswap calls the zswap
-load function to decompress the page into the page allocated by the page fault
-handler.
-
-Once there are no PTEs referencing a swap page stored in zswap (i.e. the count
-in the swap_map goes to 0) the swap code calls the zswap invalidate function,
-via frontswap, to free the compressed entry.
-
-Zswap seeks to be simple in its policies.  Sysfs attributes allow for one user
-controlled policy:
-* max_pool_percent - The maximum percentage of memory that the compressed
-    pool can occupy.
-
-The default compressor is lzo, but it can be selected at boot time by setting
-the “compressor” attribute, e.g. zswap.compressor=lzo.  It can also be changed
-at runtime using the sysfs "compressor" attribute, e.g.
-
-echo lzo > /sys/module/zswap/parameters/compressor
-
-When the zpool and/or compressor parameter is changed at runtime, any existing
-compressed pages are not modified; they are left in their own zpool.  When a
-request is made for a page in an old zpool, it is uncompressed using its
-original compressor.  Once all pages are removed from an old zpool, the zpool
-and its compressor are freed.
-
-Some of the pages in zswap are same-value filled pages (i.e. contents of the
-page have same value or repetitive pattern). These pages include zero-filled
-pages and they are handled differently. During store operation, a page is
-checked if it is a same-value filled page before compressing it. If true, the
-compressed length of the page is set to zero and the pattern or same-filled
-value is stored.
-
-Same-value filled pages identification feature is enabled by default and can be
-disabled at boot time by setting the "same_filled_pages_enabled" attribute to 0,
-e.g. zswap.same_filled_pages_enabled=0. It can also be enabled and disabled at
-runtime using the sysfs "same_filled_pages_enabled" attribute, e.g.
-
-echo 1 > /sys/module/zswap/parameters/same_filled_pages_enabled
-
-When zswap same-filled page identification is disabled at runtime, it will stop
-checking for the same-value filled pages during store operation. However, the
-existing pages which are marked as same-value filled pages remain stored
-unchanged in zswap until they are either loaded or invalidated.
-
-A debugfs interface is provided for various statistic about pool size, number
-of pages stored, same-value filled pages and various counters for the reasons
-pages are rejected.
diff --git a/Documentation/x86/x86_64/boot-options.txt b/Documentation/x86/x86_64/boot-options.txt
index b297c48389b9..8d109ef67ab6 100644
--- a/Documentation/x86/x86_64/boot-options.txt
+++ b/Documentation/x86/x86_64/boot-options.txt
@@ -187,9 +187,9 @@ PCI
 
 IOMMU (input/output memory management unit)
 
- Currently four x86-64 PCI-DMA mapping implementations exist:
+ Multiple x86-64 PCI-DMA mapping implementations exist, for example:
 
-   1. <arch/x86_64/kernel/pci-nommu.c>: use no hardware/software IOMMU at all
+   1. <lib/dma-direct.c>: use no hardware/software IOMMU at all
       (e.g. because you have < 3 GB memory).
       Kernel boot message: "PCI-DMA: Disabling IOMMU"
 
@@ -208,7 +208,7 @@ IOMMU (input/output memory management unit)
       Kernel boot message: "PCI-DMA: Using Calgary IOMMU"
 
  iommu=[<size>][,noagp][,off][,force][,noforce][,leak[=<nr_of_leak_pages>]
-	[,memaper[=<order>]][,merge][,forcesac][,fullflush][,nomerge]
+	[,memaper[=<order>]][,merge][,fullflush][,nomerge]
 	[,noaperture][,calgary]
 
   General iommu options:
@@ -235,14 +235,7 @@ IOMMU (input/output memory management unit)
                        (experimental).
     nomerge            Don't do scatter-gather (SG) merging.
     noaperture         Ask the IOMMU not to touch the aperture for AGP.
-    forcesac           Force single-address cycle (SAC) mode for masks <40bits
-                       (experimental).
     noagp              Don't initialize the AGP driver and use full aperture.
-    allowdac           Allow double-address cycle (DAC) mode, i.e. DMA >4GB.
-                       DAC is used with 32-bit PCI to push a 64-bit address in
-                       two cycles. When off all DMA over >4GB is forced through
-                       an IOMMU or software bounce buffering.
-    nodac              Forbid DAC mode, i.e. DMA >4GB.
     panic              Always panic when IOMMU overflows.
     calgary            Use the Calgary IOMMU if it is available
 
diff --git a/Kconfig b/Kconfig
index 8c4c1cb0f9cd..a90d9f9e268b 100644
--- a/Kconfig
+++ b/Kconfig
@@ -3,10 +3,10 @@
 # For a description of the syntax of this configuration file,
 # see Documentation/kbuild/kconfig-language.txt.
 #
-mainmenu "Linux/$ARCH $KERNELVERSION Kernel Configuration"
+mainmenu "Linux/$(ARCH) $(KERNELVERSION) Kernel Configuration"
 
-config SRCARCH
-	string
-	option env="SRCARCH"
+comment "Compiler: $(CC_VERSION_TEXT)"
 
-source "arch/$SRCARCH/Kconfig"
+source "scripts/Kconfig.include"
+
+source "arch/$(SRCARCH)/Kconfig"
diff --git a/LICENSES/exceptions/Linux-syscall-note b/LICENSES/exceptions/Linux-syscall-note
index 6b60b61be4e9..9abdad71fafd 100644
--- a/LICENSES/exceptions/Linux-syscall-note
+++ b/LICENSES/exceptions/Linux-syscall-note
@@ -1,6 +1,6 @@
 SPDX-Exception-Identifier: Linux-syscall-note
 SPDX-URL: https://spdx.org/licenses/Linux-syscall-note.html
-SPDX-Licenses: GPL-2.0, GPL-2.0+, GPL-1.0+, LGPL-2.0, LGPL-2.0+, LGPL-2.1, LGPL-2.1+
+SPDX-Licenses: GPL-2.0, GPL-2.0+, GPL-1.0+, LGPL-2.0, LGPL-2.0+, LGPL-2.1, LGPL-2.1+, GPL-2.0-only, GPL-2.0-or-later
 Usage-Guide:
   This exception is used together with one of the above SPDX-Licenses
   to mark user space API (uapi) header files so they can be included
diff --git a/LICENSES/other/Apache-2.0 b/LICENSES/other/Apache-2.0
new file mode 100644
index 000000000000..7cd903f573e5
--- /dev/null
+++ b/LICENSES/other/Apache-2.0
@@ -0,0 +1,183 @@
+Valid-License-Identifier: Apache-2.0
+SPDX-URL: https://spdx.org/licenses/Apache-2.0.html
+Usage-Guide:
+  To use the Apache License version 2.0 put the following SPDX tag/value
+  pair into a comment according to the placement guidelines in the
+  licensing rules documentation:
+    SPDX-License-Identifier: Apache-2.0
+License-Text:
+
+Apache License
+
+Version 2.0, January 2004
+
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the
+copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other
+entities that control, are controlled by, or are under common control with
+that entity. For the purposes of this definition, "control" means (i) the
+power, direct or indirect, to cause the direction or management of such
+entity, whether by contract or otherwise, or (ii) ownership of fifty
+percent (50%) or more of the outstanding shares, or (iii) beneficial
+ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation source,
+and configuration files.
+
+"Object" form shall mean any form resulting from mechanical transformation
+or translation of a Source form, including but not limited to compiled
+object code, generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form,
+made available under the License, as indicated by a copyright notice that
+is included in or attached to the work (an example is provided in the
+Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form,
+that is based on (or derived from) the Work and for which the editorial
+revisions, annotations, elaborations, or other modifications represent, as
+a whole, an original work of authorship. For the purposes of this License,
+Derivative Works shall not include works that remain separable from, or
+merely link (or bind by name) to the interfaces of, the Work and Derivative
+Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original
+version of the Work and any modifications or additions to that Work or
+Derivative Works thereof, that is intentionally submitted to Licensor for
+inclusion in the Work by the copyright owner or by an individual or Legal
+Entity authorized to submit on behalf of the copyright owner. For the
+purposes of this definition, "submitted" means any form of electronic,
+verbal, or written communication sent to the Licensor or its
+representatives, including but not limited to communication on electronic
+mailing lists, source code control systems, and issue tracking systems that
+are managed by, or on behalf of, the Licensor for the purpose of discussing
+and improving the Work, but excluding communication that is conspicuously
+marked or otherwise designated in writing by the copyright owner as "Not a
+Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on
+behalf of whom a Contribution has been received by Licensor and
+subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this
+   License, each Contributor hereby grants to You a perpetual, worldwide,
+   non-exclusive, no-charge, royalty-free, irrevocable copyright license to
+   reproduce, prepare Derivative Works of, publicly display, publicly
+   perform, sublicense, and distribute the Work and such Derivative Works
+   in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this
+   License, each Contributor hereby grants to You a perpetual, worldwide,
+   non-exclusive, no-charge, royalty-free, irrevocable (except as stated in
+   this section) patent license to make, have made, use, offer to sell,
+   sell, import, and otherwise transfer the Work, where such license
+   applies only to those patent claims licensable by such Contributor that
+   are necessarily infringed by their Contribution(s) alone or by
+   combination of their Contribution(s) with the Work to which such
+   Contribution(s) was submitted. If You institute patent litigation
+   against any entity (including a cross-claim or counterclaim in a
+   lawsuit) alleging that the Work or a Contribution incorporated within
+   the Work constitutes direct or contributory patent infringement, then
+   any patent licenses granted to You under this License for that Work
+   shall terminate as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or
+   Derivative Works thereof in any medium, with or without modifications,
+   and in Source or Object form, provided that You meet the following
+   conditions:
+
+   a. You must give any other recipients of the Work or Derivative Works a
+      copy of this License; and
+
+   b. You must cause any modified files to carry prominent notices stating
+      that You changed the files; and
+
+   c. You must retain, in the Source form of any Derivative Works that You
+      distribute, all copyright, patent, trademark, and attribution notices
+      from the Source form of the Work, excluding those notices that do not
+      pertain to any part of the Derivative Works; and
+
+   d. If the Work includes a "NOTICE" text file as part of its
+      distribution, then any Derivative Works that You distribute must
+      include a readable copy of the attribution notices contained within
+      such NOTICE file, excluding those notices that do not pertain to any
+      part of the Derivative Works, in at least one of the following
+      places: within a NOTICE text file distributed as part of the
+      Derivative Works; within the Source form or documentation, if
+      provided along with the Derivative Works; or, within a display
+      generated by the Derivative Works, if and wherever such third-party
+      notices normally appear. The contents of the NOTICE file are for
+      informational purposes only and do not modify the License. You may
+      add Your own attribution notices within Derivative Works that You
+      distribute, alongside or as an addendum to the NOTICE text from the
+      Work, provided that such additional attribution notices cannot be
+      construed as modifying the License.
+
+    You may add Your own copyright statement to Your modifications and may
+    provide additional or different license terms and conditions for use,
+    reproduction, or distribution of Your modifications, or for any such
+    Derivative Works as a whole, provided Your use, reproduction, and
+    distribution of the Work otherwise complies with the conditions stated
+    in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise, any
+   Contribution intentionally submitted for inclusion in the Work by You to
+   the Licensor shall be under the terms and conditions of this License,
+   without any additional terms or conditions. Notwithstanding the above,
+   nothing herein shall supersede or modify the terms of any separate
+   license agreement you may have executed with Licensor regarding such
+   Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+   names, trademarks, service marks, or product names of the Licensor,
+   except as required for reasonable and customary use in describing the
+   origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to
+   in writing, Licensor provides the Work (and each Contributor provides
+   its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+   OF ANY KIND, either express or implied, including, without limitation,
+   any warranties or conditions of TITLE, NON-INFRINGEMENT,
+   MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely
+   responsible for determining the appropriateness of using or
+   redistributing the Work and assume any risks associated with Your
+   exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory, whether
+   in tort (including negligence), contract, or otherwise, unless required
+   by applicable law (such as deliberate and grossly negligent acts) or
+   agreed to in writing, shall any Contributor be liable to You for
+   damages, including any direct, indirect, special, incidental, or
+   consequential damages of any character arising as a result of this
+   License or out of the use or inability to use the Work (including but
+   not limited to damages for loss of goodwill, work stoppage, computer
+   failure or malfunction, or any and all other commercial damages or
+   losses), even if such Contributor has been advised of the possibility of
+   such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing the
+   Work or Derivative Works thereof, You may choose to offer, and charge a
+   fee for, acceptance of support, warranty, indemnity, or other liability
+   obligations and/or rights consistent with this License. However, in
+   accepting such obligations, You may act only on Your own behalf and on
+   Your sole responsibility, not on behalf of any other Contributor, and
+   only if You agree to indemnify, defend, and hold each Contributor
+   harmless for any liability incurred by, or claims asserted against, such
+   Contributor by reason of your accepting any such warranty or additional
+   liability.
+
+END OF TERMS AND CONDITIONS
diff --git a/LICENSES/other/CC-BY-SA-4.0 b/LICENSES/other/CC-BY-SA-4.0
new file mode 100644
index 000000000000..f9158e831e79
--- /dev/null
+++ b/LICENSES/other/CC-BY-SA-4.0
@@ -0,0 +1,397 @@
+Valid-License-Identifier: CC-BY-SA-4.0
+SPDX-URL: https://spdx.org/licenses/CC-BY-SA-4.0
+Usage-Guide:
+  To use the Creative Commons Attribution Share Alike 4.0 International
+  license put the following SPDX tag/value pair into a comment according to
+  the placement guidelines in the licensing rules documentation:
+    SPDX-License-Identifier: CC-BY-SA-4.0
+License-Text:
+
+Creative Commons Attribution-ShareAlike 4.0 International
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of Creative
+Commons public licenses does not create a lawyer-client or other
+relationship. Creative Commons makes its licenses and related information
+available on an "as-is" basis. Creative Commons gives no warranties
+regarding its licenses, any material licensed under their terms and
+conditions, or any related information. Creative Commons disclaims all
+liability for damages resulting from their use to the fullest extent
+possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share original
+works of authorship and other material subject to copyright and certain
+other rights specified in the public license below. The following
+considerations are for informational purposes only, are not exhaustive, and
+do not form part of our licenses.
+
+Considerations for licensors: Our public licenses are intended for use by
+those authorized to give the public permission to use material in ways
+otherwise restricted by copyright and certain other rights. Our licenses
+are irrevocable. Licensors should read and understand the terms and
+conditions of the license they choose before applying it. Licensors should
+also secure all rights necessary before applying our licenses so that the
+public can reuse the material as expected. Licensors should clearly mark
+any material not subject to the license. This includes other CC-licensed
+material, or material used under an exception or limitation to
+copyright. More considerations for licensors :
+wiki.creativecommons.org/Considerations_for_licensors
+
+Considerations for the public: By using one of our public licenses, a
+licensor grants the public permission to use the licensed material under
+specified terms and conditions. If the licensor's permission is not
+necessary for any reason - for example, because of any applicable exception
+or limitation to copyright - then that use is not regulated by the
+license. Our licenses grant only permissions under copyright and certain
+other rights that a licensor has authority to grant. Use of the licensed
+material may still be restricted for other reasons, including because
+others have copyright or other rights in the material. A licensor may make
+special requests, such as asking that all changes be marked or described.
+
+Although not required by our licenses, you are encouraged to respect those
+requests where reasonable. More considerations for the public :
+wiki.creativecommons.org/Considerations_for_licensees
+
+Creative Commons Attribution-ShareAlike 4.0 International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree to
+be bound by the terms and conditions of this Creative Commons
+Attribution-ShareAlike 4.0 International Public License ("Public
+License"). To the extent this Public License may be interpreted as a
+contract, You are granted the Licensed Rights in consideration of Your
+acceptance of these terms and conditions, and the Licensor grants You such
+rights in consideration of benefits the Licensor receives from making the
+Licensed Material available under these terms and conditions.
+
+Section 1 - Definitions.
+
+    a. Adapted Material means material subject to Copyright and Similar
+       Rights that is derived from or based upon the Licensed Material and
+       in which the Licensed Material is translated, altered, arranged,
+       transformed, or otherwise modified in a manner requiring permission
+       under the Copyright and Similar Rights held by the Licensor. For
+       purposes of this Public License, where the Licensed Material is a
+       musical work, performance, or sound recording, Adapted Material is
+       always produced where the Licensed Material is synched in timed
+       relation with a moving image.
+
+    b. Adapter's License means the license You apply to Your Copyright and
+       Similar Rights in Your contributions to Adapted Material in
+       accordance with the terms and conditions of this Public License.
+
+    c. BY-SA Compatible License means a license listed at
+       creativecommons.org/compatiblelicenses, approved by Creative Commons
+       as essentially the equivalent of this Public License.
+
+    d. Copyright and Similar Rights means copyright and/or similar rights
+       closely related to copyright including, without limitation,
+       performance, broadcast, sound recording, and Sui Generis Database
+       Rights, without regard to how the rights are labeled or
+       categorized. For purposes of this Public License, the rights
+       specified in Section 2(b)(1)-(2) are not Copyright and Similar
+       Rights.
+
+    e. Effective Technological Measures means those measures that, in the
+       absence of proper authority, may not be circumvented under laws
+       fulfilling obligations under Article 11 of the WIPO Copyright Treaty
+       adopted on December 20, 1996, and/or similar international
+       agreements.
+
+    f. Exceptions and Limitations means fair use, fair dealing, and/or any
+       other exception or limitation to Copyright and Similar Rights that
+       applies to Your use of the Licensed Material.
+
+    g. License Elements means the license attributes listed in the name of
+       a Creative Commons Public License. The License Elements of this
+       Public License are Attribution and ShareAlike.
+
+    h. Licensed Material means the artistic or literary work, database, or
+       other material to which the Licensor applied this Public License.
+
+    i. Licensed Rights means the rights granted to You subject to the terms
+       and conditions of this Public License, which are limited to all
+       Copyright and Similar Rights that apply to Your use of the Licensed
+       Material and that the Licensor has authority to license.
+
+    j. Licensor means the individual(s) or entity(ies) granting rights
+       under this Public License.
+
+    k. Share means to provide material to the public by any means or
+       process that requires permission under the Licensed Rights, such as
+       reproduction, public display, public performance, distribution,
+       dissemination, communication, or importation, and to make material
+       available to the public including in ways that members of the public
+       may access the material from a place and at a time individually
+       chosen by them.
+
+    l. Sui Generis Database Rights means rights other than copyright
+       resulting from Directive 96/9/EC of the European Parliament and of
+       the Council of 11 March 1996 on the legal protection of databases,
+       as amended and/or succeeded, as well as other essentially equivalent
+       rights anywhere in the world.  m. You means the individual or entity
+       exercising the Licensed Rights under this Public License. Your has a
+       corresponding meaning.
+
+Section 2 - Scope.
+
+    a. License grant.
+
+        1. Subject to the terms and conditions of this Public License, the
+           Licensor hereby grants You a worldwide, royalty-free,
+           non-sublicensable, non-exclusive, irrevocable license to
+           exercise the Licensed Rights in the Licensed Material to:
+
+            A. reproduce and Share the Licensed Material, in whole or in part; and
+
+            B. produce, reproduce, and Share Adapted Material.
+
+        2. Exceptions and Limitations. For the avoidance of doubt, where
+           Exceptions and Limitations apply to Your use, this Public
+           License does not apply, and You do not need to comply with its
+           terms and conditions.
+
+        3. Term. The term of this Public License is specified in Section 6(a).
+
+        4. Media and formats; technical modifications allowed. The Licensor
+           authorizes You to exercise the Licensed Rights in all media and
+           formats whether now known or hereafter created, and to make
+           technical modifications necessary to do so. The Licensor waives
+           and/or agrees not to assert any right or authority to forbid You
+           from making technical modifications necessary to exercise the
+           Licensed Rights, including technical modifications necessary to
+           circumvent Effective Technological Measures. For purposes of
+           this Public License, simply making modifications authorized by
+           this Section 2(a)(4) never produces Adapted Material.
+
+        5. Downstream recipients.
+
+            A. Offer from the Licensor - Licensed Material. Every recipient
+               of the Licensed Material automatically receives an offer
+               from the Licensor to exercise the Licensed Rights under the
+               terms and conditions of this Public License.
+
+            B. Additional offer from the Licensor - Adapted Material. Every
+               recipient of Adapted Material from You automatically
+               receives an offer from the Licensor to exercise the Licensed
+               Rights in the Adapted Material under the conditions of the
+               Adapter's License You apply.
+
+            C. No downstream restrictions. You may not offer or impose any
+               additional or different terms or conditions on, or apply any
+               Effective Technological Measures to, the Licensed Material
+               if doing so restricts exercise of the Licensed Rights by any
+               recipient of the Licensed Material.
+
+        6. No endorsement. Nothing in this Public License constitutes or
+           may be construed as permission to assert or imply that You are,
+           or that Your use of the Licensed Material is, connected with, or
+           sponsored, endorsed, or granted official status by, the Licensor
+           or others designated to receive attribution as provided in
+           Section 3(a)(1)(A)(i).
+
+    b. Other rights.
+
+        1. Moral rights, such as the right of integrity, are not licensed
+           under this Public License, nor are publicity, privacy, and/or
+           other similar personality rights; however, to the extent
+           possible, the Licensor waives and/or agrees not to assert any
+           such rights held by the Licensor to the limited extent necessary
+           to allow You to exercise the Licensed Rights, but not otherwise.
+
+        2. Patent and trademark rights are not licensed under this Public
+           License.
+
+        3. To the extent possible, the Licensor waives any right to collect
+           royalties from You for the exercise of the Licensed Rights,
+           whether directly or through a collecting society under any
+           voluntary or waivable statutory or compulsory licensing
+           scheme. In all other cases the Licensor expressly reserves any
+           right to collect such royalties.
+
+Section 3 - License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+    a. Attribution.
+
+        1. If You Share the Licensed Material (including in modified form),
+           You must:
+
+            A. retain the following if it is supplied by the Licensor with
+               the Licensed Material:
+
+                i. identification of the creator(s) of the Licensed
+                   Material and any others designated to receive
+                   attribution, in any reasonable manner requested by the
+                   Licensor (including by pseudonym if designated);
+
+                ii. a copyright notice;
+
+                iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of warranties;
+
+                v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable;
+
+            B. indicate if You modified the Licensed Material and retain an
+               indication of any previous modifications; and
+
+            C. indicate the Licensed Material is licensed under this Public
+            License, and include the text of, or the URI or hyperlink to,
+            this Public License.
+
+        2. You may satisfy the conditions in Section 3(a)(1) in any
+           reasonable manner based on the medium, means, and context in
+           which You Share the Licensed Material. For example, it may be
+           reasonable to satisfy the conditions by providing a URI or
+           hyperlink to a resource that includes the required information.
+
+        3. If requested by the Licensor, You must remove any of the
+           information required by Section 3(a)(1)(A) to the extent
+           reasonably practicable.  b. ShareAlike.In addition to the
+           conditions in Section 3(a), if You Share Adapted Material You
+           produce, the following conditions also apply.
+
+           1. The Adapter's License You apply must be a Creative Commons
+              license with the same License Elements, this version or
+              later, or a BY-SA Compatible License.
+
+           2. You must include the text of, or the URI or hyperlink to, the
+              Adapter's License You apply. You may satisfy this condition
+              in any reasonable manner based on the medium, means, and
+              context in which You Share Adapted Material.
+
+           3. You may not offer or impose any additional or different terms
+              or conditions on, or apply any Effective Technological
+              Measures to, Adapted Material that restrict exercise of the
+              rights granted under the Adapter's License You apply.
+
+Section 4 - Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that apply to
+Your use of the Licensed Material:
+
+    a. for the avoidance of doubt, Section 2(a)(1) grants You the right to
+       extract, reuse, reproduce, and Share all or a substantial portion of
+       the contents of the database;
+
+    b. if You include all or a substantial portion of the database contents
+       in a database in which You have Sui Generis Database Rights, then
+       the database in which You have Sui Generis Database Rights (but not
+       its individual contents) is Adapted Material, including for purposes
+       of Section 3(b); and
+
+    c. You must comply with the conditions in Section 3(a) if You Share all
+       or a substantial portion of the contents of the database.
+
+    For the avoidance of doubt, this Section 4 supplements and does not
+    replace Your obligations under this Public License where the Licensed
+    Rights include other Copyright and Similar Rights.
+
+Section 5 - Disclaimer of Warranties and Limitation of Liability.
+
+    a. Unless otherwise separately undertaken by the Licensor, to the
+       extent possible, the Licensor offers the Licensed Material as-is and
+       as-available, and makes no representations or warranties of any kind
+       concerning the Licensed Material, whether express, implied,
+       statutory, or other. This includes, without limitation, warranties
+       of title, merchantability, fitness for a particular purpose,
+       non-infringement, absence of latent or other defects, accuracy, or
+       the presence or absence of errors, whether or not known or
+       discoverable. Where disclaimers of warranties are not allowed in
+       full or in part, this disclaimer may not apply to You.
+
+    b. To the extent possible, in no event will the Licensor be liable to
+       You on any legal theory (including, without limitation, negligence)
+       or otherwise for any direct, special, indirect, incidental,
+       consequential, punitive, exemplary, or other losses, costs,
+       expenses, or damages arising out of this Public License or use of
+       the Licensed Material, even if the Licensor has been advised of the
+       possibility of such losses, costs, expenses, or damages. Where a
+       limitation of liability is not allowed in full or in part, this
+       limitation may not apply to You.
+
+    c. The disclaimer of warranties and limitation of liability provided
+       above shall be interpreted in a manner that, to the extent possible,
+       most closely approximates an absolute disclaimer and waiver of all
+       liability.
+
+Section 6 - Term and Termination.
+
+    a. This Public License applies for the term of the Copyright and
+       Similar Rights licensed here. However, if You fail to comply with
+       this Public License, then Your rights under this Public License
+       terminate automatically.
+
+    b. Where Your right to use the Licensed Material has terminated under
+       Section 6(a), it reinstates:
+
+        1. automatically as of the date the violation is cured, provided it
+           is cured within 30 days of Your discovery of the violation; or
+
+        2. upon express reinstatement by the Licensor.
+
+    c. For the avoidance of doubt, this Section 6(b) does not affect any
+       right the Licensor may have to seek remedies for Your violations of
+       this Public License.
+
+    d. For the avoidance of doubt, the Licensor may also offer the Licensed
+       Material under separate terms or conditions or stop distributing the
+       Licensed Material at any time; however, doing so will not terminate
+       this Public License.
+
+    e. Sections 1, 5, 6, 7, and 8 survive termination of this Public License.
+
+Section 7 - Other Terms and Conditions.
+
+    a. The Licensor shall not be bound by any additional or different terms
+       or conditions communicated by You unless expressly agreed.
+
+    b. Any arrangements, understandings, or agreements regarding the
+       Licensed Material not stated herein are separate from and
+       independent of the terms and conditions of this Public License.
+
+Section 8 - Interpretation.
+
+    a. For the avoidance of doubt, this Public License does not, and shall
+       not be interpreted to, reduce, limit, restrict, or impose conditions
+       on any use of the Licensed Material that could lawfully be made
+       without permission under this Public License.
+
+    b. To the extent possible, if any provision of this Public License is
+       deemed unenforceable, it shall be automatically reformed to the
+       minimum extent necessary to make it enforceable. If the provision
+       cannot be reformed, it shall be severed from this Public License
+       without affecting the enforceability of the remaining terms and
+       conditions.
+
+    c. No term or condition of this Public License will be waived and no
+       failure to comply consented to unless expressly agreed to by the
+       Licensor.
+
+    d. Nothing in this Public License constitutes or may be interpreted as
+       a limitation upon, or waiver of, any privileges and immunities that
+       apply to the Licensor or You, including from the legal processes of
+       any jurisdiction or authority.
+
+Creative Commons is not a party to its public licenses. Notwithstanding,
+Creative Commons may elect to apply one of its public licenses to material
+it publishes and in those instances will be considered the "Licensor." The
+text of the Creative Commons public licenses is dedicated to the public
+domain under the CC0 Public Domain Dedication. Except for the limited
+purpose of indicating that material is shared under a Creative Commons
+public license or as otherwise permitted by the Creative Commons policies
+published at creativecommons.org/policies, Creative Commons does not
+authorize the use of the trademark "Creative Commons" or any other
+trademark or logo of Creative Commons without its prior written consent
+including, without limitation, in connection with any unauthorized
+modifications to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For the
+avoidance of doubt, this paragraph does not form part of the public
+licenses.
+
+Creative Commons may be contacted at creativecommons.org.
diff --git a/LICENSES/other/CDDL-1.0 b/LICENSES/other/CDDL-1.0
new file mode 100644
index 000000000000..195a1687930a
--- /dev/null
+++ b/LICENSES/other/CDDL-1.0
@@ -0,0 +1,364 @@
+Valid-License-Identifier: CDDL-1.0
+SPDX-URL: https://spdx.org/licenses/CDDL-1.0.html
+Usage-Guide:
+  To use the Common Development and Distribution License 1.0 put the
+  following SPDX tag/value pair into a comment according to the placement
+  guidelines in the licensing rules documentation:
+    SPDX-License-Identifier: CDDL-1.0
+
+License-Text:
+
+COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)
+Version 1.0
+
+    1. Definitions.
+
+        1.1. "Contributor" means each individual or entity that creates or
+             contributes to the creation of Modifications.
+
+        1.2. "Contributor Version" means the combination of the Original
+	     Software, prior Modifications used by a Contributor (if any),
+	     and the Modifications made by that particular Contributor.
+
+        1.3. "Covered Software" means (a) the Original Software, or (b)
+             Modifications, or (c) the combination of files containing
+             Original Software with files containing Modifications, in each
+             case including portions thereof.
+
+	1.4. "Executable" means the Covered Software in any form other than
+             Source Code.
+
+        1.5. "Initial Developer" means the individual or entity that first
+             makes Original Software available under this License.
+
+        1.6. "Larger Work" means a work which combines Covered Software or
+             portions thereof with code not governed by the terms of this
+             License.
+
+        1.7. "License" means this document.
+
+        1.8. "Licensable" means having the right to grant, to the maximum
+             extent possible, whether at the time of the initial grant or
+             subsequently acquired, any and all of the rights conveyed herein.
+
+        1.9. "Modifications" means the Source Code and Executable form of
+             any of the following:
+
+            A. Any file that results from an addition to, deletion from or
+               modification of the contents of a file containing Original
+               Software or previous Modifications;
+
+            B. Any new file that contains any part of the Original Software
+               or previous Modification; or
+
+            C. Any new file that is contributed or otherwise made available
+               under the terms of this License.
+
+        1.10. "Original Software" means the Source Code and Executable form
+              of computer software code that is originally released under
+              this License.
+
+        1.11. "Patent Claims" means any patent claim(s), now owned or
+              hereafter acquired, including without limitation, method,
+              process, and apparatus claims, in any patent Licensable by
+              grantor.
+
+        1.12. "Source Code" means (a) the common form of computer software
+	      code in which modifications are made and (b) associated
+              documentation included in or with such code.
+
+        1.13. "You" (or "Your") means an individual or a legal entity
+              exercising rights under, and complying with all of the terms
+              of, this License. For legal entities, "You" includes any
+              entity which controls, is controlled by, or is under common
+              control with You. For purposes of this definition, "control"
+              means (a) the power, direct or indirect, to cause the
+              direction or management of such entity, whether by contract
+              or otherwise, or (b) ownership of more than fifty percent
+              (50%) of the outstanding shares or beneficial ownership of
+              such entity.
+
+    2. License Grants.
+        2.1. The Initial Developer Grant.
+
+        Conditioned upon Your compliance with Section 3.1 below and subject
+        to third party intellectual property claims, the Initial Developer
+        hereby grants You a world-wide, royalty-free, non-exclusive
+        license:
+
+            (a) under intellectual property rights (other than patent or
+                trademark) Licensable by Initial Developer, to use,
+                reproduce, modify, display, perform, sublicense and
+                distribute the Original Software (or portions thereof),
+                with or without Modifications, and/or as part of a Larger
+                Work; and
+
+            (b) under Patent Claims infringed by the making, using or
+                selling of Original Software, to make, have made, use,
+                practice, sell, and offer for sale, and/or otherwise
+                dispose of the Original Software (or portions thereof).
+
+            (c) The licenses granted in Sections 2.1(a) and (b) are
+                effective on the date Initial Developer first distributes
+                or otherwise makes the Original Software available to a
+                third party under the terms of this License.
+
+            (d) Notwithstanding Section 2.1(b) above, no patent license is
+                granted: (1) for code that You delete from the Original
+                Software, or (2) for infringements caused by: (i) the
+                modification of the Original Software, or (ii) the
+                combination of the Original Software with other software or
+                devices.
+
+        2.2. Contributor Grant.
+
+        Conditioned upon Your compliance with Section 3.1 below and subject
+        to third party intellectual property claims, each Contributor
+        hereby grants You a world-wide, royalty-free, non-exclusive
+        license:
+
+            (a) under intellectual property rights (other than patent or
+	        trademark) Licensable by Contributor to use, reproduce,
+	        modify, display, perform, sublicense and distribute the
+	        Modifications created by such Contributor (or portions
+	        thereof), either on an unmodified basis, with other
+	        Modifications, as Covered Software and/or as part of a
+	        Larger Work; and
+
+            (b) under Patent Claims infringed by the making, using, or
+                selling of Modifications made by that Contributor either
+                alone and/or in combination with its Contributor Version
+                (or portions of such combination), to make, use, sell,
+                offer for sale, have made, and/or otherwise dispose of: (1)
+                Modifications made by that Contributor (or portions
+                thereof); and (2) the combination of Modifications made by
+                that Contributor with its Contributor Version (or portions
+                of such combination).
+
+            (c) The licenses granted in Sections 2.2(a) and 2.2(b) are
+                effective on the date Contributor first distributes or
+                otherwise makes the Modifications available to a third
+                party.
+
+            (d) Notwithstanding Section 2.2(b) above, no patent license is
+                granted: (1) for any code that Contributor has deleted from
+                the Contributor Version; (2) for infringements caused by:
+                (i) third party modifications of Contributor Version, or
+                (ii) the combination of Modifications made by that
+                Contributor with other software (except as part of the
+                Contributor Version) or other devices; or (3) under Patent
+                Claims infringed by Covered Software in the absence of
+                Modifications made by that Contributor.
+
+    3. Distribution Obligations.
+        3.1. Availability of Source Code.
+
+        Any Covered Software that You distribute or otherwise make
+        available in Executable form must also be made available in Source
+        Code form and that Source Code form must be distributed only under
+        the terms of this License. You must include a copy of this License
+        with every copy of the Source Code form of the Covered Software You
+        distribute or otherwise make available. You must inform recipients
+        of any such Covered Software in Executable form as to how they can
+        obtain such Covered Software in Source Code form in a reasonable
+        manner on or through a medium customarily used for software
+        exchange.
+
+        3.2. Modifications.
+
+        The Modifications that You create or to which You contribute are
+        governed by the terms of this License. You represent that You
+        believe Your Modifications are Your original creation(s) and/or You
+        have sufficient rights to grant the rights conveyed by this
+        License.
+
+        3.3. Required Notices.
+
+        You must include a notice in each of Your Modifications that
+        identifies You as the Contributor of the Modification. You may not
+        remove or alter any copyright, patent or trademark notices
+        contained within the Covered Software, or any notices of licensing
+        or any descriptive text giving attribution to any Contributor or
+        the Initial Developer.
+
+        3.4. Application of Additional Terms.
+
+        You may not offer or impose any terms on any Covered Software in
+        Source Code form that alters or restricts the applicable version of
+        this License or the recipients' rights hereunder. You may choose to
+        offer, and to charge a fee for, warranty, support, indemnity or
+        liability obligations to one or more recipients of Covered
+        Software. However, you may do so only on Your own behalf, and not
+        on behalf of the Initial Developer or any Contributor. You must
+        make it absolutely clear that any such warranty, support, indemnity
+        or liability obligation is offered by You alone, and You hereby
+        agree to indemnify the Initial Developer and every Contributor for
+        any liability incurred by the Initial Developer or such Contributor
+        as a result of warranty, support, indemnity or liability terms You
+        offer.
+
+        3.5. Distribution of Executable Versions.
+
+        You may distribute the Executable form of the Covered Software
+        under the terms of this License or under the terms of a license of
+        Your choice, which may contain terms different from this License,
+        provided that You are in compliance with the terms of this License
+        and that the license for the Executable form does not attempt to
+        limit or alter the recipient's rights in the Source Code form from
+        the rights set forth in this License. If You distribute the Covered
+        Software in Executable form under a different license, You must
+        make it absolutely clear that any terms which differ from this
+        License are offered by You alone, not by the Initial Developer or
+        Contributor. You hereby agree to indemnify the Initial Developer
+        and every Contributor for any liability incurred by the Initial
+        Developer or such Contributor as a result of any such terms You
+        offer.
+
+        3.6. Larger Works.
+
+        You may create a Larger Work by combining Covered Software with
+        other code not governed by the terms of this License and distribute
+        the Larger Work as a single product. In such a case, You must make
+        sure the requirements of this License are fulfilled for the Covered
+        Software.
+
+    4. Versions of the License.
+        4.1. New Versions.
+
+        Sun Microsystems, Inc. is the initial license steward and may
+        publish revised and/or new versions of this License from time to
+        time. Each version will be given a distinguishing version
+        number. Except as provided in Section 4.3, no one other than the
+        license steward has the right to modify this License.
+
+        4.2. Effect of New Versions.
+
+        You may always continue to use, distribute or otherwise make the
+        Covered Software available under the terms of the version of the
+        License under which You originally received the Covered
+        Software. If the Initial Developer includes a notice in the
+        Original Software prohibiting it from being distributed or
+        otherwise made available under any subsequent version of the
+        License, You must distribute and make the Covered Software
+        available under the terms of the version of the License under which
+        You originally received the Covered Software. Otherwise, You may
+        also choose to use, distribute or otherwise make the Covered
+        Software available under the terms of any subsequent version of the
+        License published by the license steward.
+
+        4.3. Modified Versions.
+
+        When You are an Initial Developer and You want to create a new
+        license for Your Original Software, You may create and use a
+        modified version of this License if You: (a) rename the license and
+        remove any references to the name of the license steward (except to
+        note that the license differs from this License); and (b) otherwise
+        make it clear that the license contains terms which differ from
+        this License.
+
+    5. DISCLAIMER OF WARRANTY.
+
+    COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS,
+    WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
+    WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF
+    DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR
+    NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF
+    THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE
+    DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER
+    CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR
+    CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART
+    OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER
+    EXCEPT UNDER THIS DISCLAIMER.
+
+    6. TERMINATION.
+
+        6.1. This License and the rights granted hereunder will terminate
+        automatically if You fail to comply with terms herein and fail to
+        cure such breach within 30 days of becoming aware of the
+        breach. Provisions which, by their nature, must remain in effect
+        beyond the termination of this License shall survive.
+
+        6.2. If You assert a patent infringement claim (excluding
+        declaratory judgment actions) against Initial Developer or a
+        Contributor (the Initial Developer or Contributor against whom You
+        assert such claim is referred to as "Participant") alleging that
+        the Participant Software (meaning the Contributor Version where the
+        Participant is a Contributor or the Original Software where the
+        Participant is the Initial Developer) directly or indirectly
+        infringes any patent, then any and all rights granted directly or
+        indirectly to You by such Participant, the Initial Developer (if
+        the Initial Developer is not the Participant) and all Contributors
+        under Sections 2.1 and/or 2.2 of this License shall, upon 60 days
+        notice from Participant terminate prospectively and automatically
+        at the expiration of such 60 day notice period, unless if within
+        such 60 day period You withdraw Your claim with respect to the
+        Participant Software against such Participant either unilaterally
+        or pursuant to a written agreement with Participant.
+
+        6.3. In the event of termination under Sections 6.1 or 6.2 above,
+        all end user licenses that have been validly granted by You or any
+        distributor hereunder prior to termination (excluding licenses
+        granted to You by any distributor) shall survive termination.
+
+    7. LIMITATION OF LIABILITY.
+
+    UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT
+    (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL
+    DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED
+    SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY
+    PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
+    OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST
+    PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR
+    MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF
+    SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH
+    DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR
+    DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE
+    EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO
+    NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL
+    DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.
+
+    8. U.S. GOVERNMENT END USERS.
+
+    The Covered Software is a "commercial item," as that term is defined in
+    48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer
+    software" (as that term is defined at 48 C.F.R. $ 252.227-7014(a)(1))
+    and "commercial computer software documentation" as such terms are used
+    in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and
+    48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all
+    U.S. Government End Users acquire Covered Software with only those
+    rights set forth herein. This U.S. Government Rights clause is in lieu
+    of, and supersedes, any other FAR, DFAR, or other clause or provision
+    that addresses Government rights in computer software under this
+    License.
+
+    9. MISCELLANEOUS.
+
+    This License represents the complete agreement concerning subject
+    matter hereof. If any provision of this License is held to be
+    unenforceable, such provision shall be reformed only to the extent
+    necessary to make it enforceable. This License shall be governed by the
+    law of the jurisdiction specified in a notice contained within the
+    Original Software (except to the extent applicable law, if any,
+    provides otherwise), excluding such jurisdiction's conflict-of-law
+    provisions. Any litigation relating to this License shall be subject to
+    the jurisdiction of the courts located in the jurisdiction and venue
+    specified in a notice contained within the Original Software, with the
+    losing party responsible for costs, including, without limitation,
+    court costs and reasonable attorneys' fees and expenses. The
+    application of the United Nations Convention on Contracts for the
+    International Sale of Goods is expressly excluded. Any law or
+    regulation which provides that the language of a contract shall be
+    construed against the drafter shall not apply to this License. You
+    agree that You alone are responsible for compliance with the United
+    States export administration regulations (and the export control laws
+    and regulation of any other countries) when You use, distribute or
+    otherwise make available any Covered Software.
+
+    10. RESPONSIBILITY FOR CLAIMS.
+
+    As between Initial Developer and the Contributors, each party is
+    responsible for claims and damages arising, directly or indirectly, out
+    of its utilization of rights under this License and You agree to work
+    with Initial Developer and Contributors to distribute such
+    responsibility on an equitable basis. Nothing herein is intended or
+    shall be deemed to constitute any admission of liability.
diff --git a/LICENSES/other/Linux-OpenIB b/LICENSES/other/Linux-OpenIB
new file mode 100644
index 000000000000..1ad85f6b3a89
--- /dev/null
+++ b/LICENSES/other/Linux-OpenIB
@@ -0,0 +1,26 @@
+Valid-License-Identifier: Linux-OpenIB
+SPDX-URL: https://spdx.org/licenses/Linux-OpenIB.html
+Usage-Guide:
+  To use the Linux Kernel Variant of OpenIB.org license put the following
+  SPDX tag/value pair into a comment according to the placement guidelines
+  in the licensing rules documentation:
+    SPDX-License-Identifier: Linux-OpenIB
+License-Text:
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    - Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    - Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/LICENSES/other/X11 b/LICENSES/other/X11
new file mode 100644
index 000000000000..fe4353fd0000
--- /dev/null
+++ b/LICENSES/other/X11
@@ -0,0 +1,37 @@
+Valid-License-Identifier: X11
+SPDX-URL: https://spdx.org/licenses/X11.html
+Usage-Guide:
+  To use the X11 put the following SPDX tag/value pair into a comment
+  according to the placement guidelines in the licensing rules
+  documentation:
+    SPDX-License-Identifier: X11
+License-Text:
+
+
+X11 License
+
+Copyright (C) 1996 X Consortium
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Except as contained in this notice, the name of the X Consortium shall not
+be used in advertising or otherwise to promote the sale, use or other
+dealings in this Software without prior written authorization from the X
+Consortium.
+
+X Window System is a trademark of X Consortium, Inc.
diff --git a/LICENSES/preferred/GPL-2.0 b/LICENSES/preferred/GPL-2.0
index b8db91d3a1cb..ff0812fd89cc 100644
--- a/LICENSES/preferred/GPL-2.0
+++ b/LICENSES/preferred/GPL-2.0
@@ -1,5 +1,7 @@
 Valid-License-Identifier: GPL-2.0
+Valid-License-Identifier: GPL-2.0-only
 Valid-License-Identifier: GPL-2.0+
+Valid-License-Identifier: GPL-2.0-or-later
 SPDX-URL: https://spdx.org/licenses/GPL-2.0.html
 Usage-Guide:
   To use this license in source code, put one of the following SPDX
@@ -7,8 +9,12 @@ Usage-Guide:
   guidelines in the licensing rules documentation.
   For 'GNU General Public License (GPL) version 2 only' use:
     SPDX-License-Identifier: GPL-2.0
+  or
+    SPDX-License-Identifier: GPL-2.0-only
   For 'GNU General Public License (GPL) version 2 or any later version' use:
     SPDX-License-Identifier: GPL-2.0+
+  or
+    SPDX-License-Identifier: GPL-2.0-or-later
 License-Text:
 
 		    GNU GENERAL PUBLIC LICENSE
diff --git a/MAINTAINERS b/MAINTAINERS
index ca4afd68530c..939d99c7e05b 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -767,12 +767,14 @@ F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
 F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
 F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
 F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
 F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
 F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
 F:	drivers/gpu/drm/amd/amdkfd/
 F:	drivers/gpu/drm/amd/include/cik_structs.h
 F:	drivers/gpu/drm/amd/include/kgd_kfd_interface.h
 F:	drivers/gpu/drm/amd/include/vi_structs.h
+F:	drivers/gpu/drm/amd/include/v9_structs.h
 F:	include/uapi/linux/kfd_ioctl.h
 
 AMD SEATTLE DEVICE TREE SUPPORT
@@ -794,6 +796,14 @@ M:	Michael Hanselmann <linux-kernel@hansmi.ch>
 S:	Supported
 F:	drivers/macintosh/ams/
 
+ANALOG DEVICES INC AD5686 DRIVER
+M:	Stefan Popa <stefan.popa@analog.com>
+L:	linux-pm@vger.kernel.org
+W:	http://ez.analog.com/community/linux-device-drivers
+S:	Supported
+F:	drivers/iio/dac/ad5686*
+F:	drivers/iio/dac/ad5696*
+
 ANALOG DEVICES INC AD9389B DRIVER
 M:	Hans Verkuil <hans.verkuil@cisco.com>
 L:	linux-media@vger.kernel.org
@@ -1133,10 +1143,12 @@ F:	arch/arm/mach-actions/
 F:	arch/arm/boot/dts/owl-*
 F:	arch/arm64/boot/dts/actions/
 F:	drivers/clocksource/owl-*
+F:	drivers/pinctrl/actions/*
 F:	drivers/soc/actions/
 F:	include/dt-bindings/power/owl-*
 F:	include/linux/soc/actions/
 F:	Documentation/devicetree/bindings/arm/actions.txt
+F:	Documentation/devicetree/bindings/pinctrl/actions,s900-pinctrl.txt
 F:	Documentation/devicetree/bindings/power/actions,owl-sps.txt
 F:	Documentation/devicetree/bindings/timer/actions,owl-timer.txt
 
@@ -1817,7 +1829,7 @@ F:	drivers/spi/spi-qup.c
 F:	drivers/tty/serial/msm_serial.c
 F:	drivers/*/pm8???-*
 F:	drivers/mfd/ssbi.c
-F:	drivers/firmware/qcom_scm.c
+F:	drivers/firmware/qcom_scm*
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/agross/linux.git
 
 ARM/RADISYS ENP2611 MACHINE SUPPORT
@@ -2331,6 +2343,14 @@ S:	Maintained
 F:	drivers/gpio/gpio-ath79.c
 F:	Documentation/devicetree/bindings/gpio/gpio-ath79.txt
 
+ATHEROS 71XX/9XXX USB PHY DRIVER
+M:	Alban Bedel <albeu@free.fr>
+W:	https://github.com/AlbanBedel/linux
+T:	git git://github.com/AlbanBedel/linux
+S:	Maintained
+F:	drivers/phy/qualcomm/phy-ath79-usb.c
+F:	Documentation/devicetree/bindings/phy/phy-ath79-usb.txt
+
 ATHEROS ATH GENERIC UTILITIES
 M:	Kalle Valo <kvalo@codeaurora.org>
 L:	linux-wireless@vger.kernel.org
@@ -2588,6 +2608,7 @@ BACKLIGHT CLASS/SUBSYSTEM
 M:	Lee Jones <lee.jones@linaro.org>
 M:	Daniel Thompson <daniel.thompson@linaro.org>
 M:	Jingoo Han <jingoohan1@gmail.com>
+L:	dri-devel@lists.freedesktop.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git
 S:	Maintained
 F:	drivers/video/backlight/
@@ -2728,7 +2749,6 @@ F:	Documentation/networking/filter.txt
 F:	Documentation/bpf/
 F:	include/linux/bpf*
 F:	include/linux/filter.h
-F:	include/trace/events/bpf.h
 F:	include/trace/events/xdp.h
 F:	include/uapi/linux/bpf*
 F:	include/uapi/linux/filter.h
@@ -3136,6 +3156,13 @@ S:	Supported
 F:	Documentation/filesystems/caching/cachefiles.txt
 F:	fs/cachefiles/
 
+CADENCE MIPI-CSI2 BRIDGES
+M:	Maxime Ripard <maxime.ripard@bootlin.com>
+L:	linux-media@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/media/cdns,*.txt
+F:	drivers/media/platform/cadence/cdns-csi2*
+
 CADET FM/AM RADIO RECEIVER DRIVER
 M:	Hans Verkuil <hverkuil@xs4all.nl>
 L:	linux-media@vger.kernel.org
@@ -3431,6 +3458,12 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/input/touchscreen/chipone_icn8318.txt
 F:	drivers/input/touchscreen/chipone_icn8318.c
 
+CHIPONE ICN8505 I2C TOUCHSCREEN DRIVER
+M:	Hans de Goede <hdegoede@redhat.com>
+L:	linux-input@vger.kernel.org
+S:	Maintained
+F:	drivers/input/touchscreen/chipone_icn8505.c
+
 CHROME HARDWARE PLATFORM SUPPORT
 M:	Benson Leung <bleung@chromium.org>
 M:	Olof Johansson <olof@lixom.net>
@@ -3559,6 +3592,7 @@ F:	drivers/clk/
 X:	drivers/clk/clkdev.c
 F:	include/linux/clk-pr*
 F:	include/linux/clk/
+F:	include/linux/of_clk.h
 
 COMMON INTERNET FILE SYSTEM (CIFS)
 M:	Steve French <sfrench@samba.org>
@@ -4330,12 +4364,14 @@ W:	http://git.infradead.org/users/hch/dma-mapping.git
 S:	Supported
 F:	lib/dma-debug.c
 F:	lib/dma-direct.c
+F:	lib/dma-noncoherent.c
 F:	lib/dma-virt.c
 F:	drivers/base/dma-mapping.c
 F:	drivers/base/dma-coherent.c
 F:	include/asm-generic/dma-mapping.h
 F:	include/linux/dma-direct.h
 F:	include/linux/dma-mapping.h
+F:	include/linux/dma-noncoherent.h
 
 DME1737 HARDWARE MONITOR DRIVER
 M:	Juerg Haefliger <juergh@gmail.com>
@@ -4399,6 +4435,12 @@ L:	linux-kernel@vger.kernel.org
 S:	Maintained
 F:	drivers/staging/fsl-dpaa2/ethsw
 
+DPAA2 PTP CLOCK DRIVER
+M:	Yangbo Lu <yangbo.lu@nxp.com>
+L:	linux-kernel@vger.kernel.org
+S:	Maintained
+F:	drivers/staging/fsl-dpaa2/rtc
+
 DPT_I2O SCSI RAID DRIVER
 M:	Adaptec OEM Raid Solutions <aacraid@microsemi.com>
 L:	linux-scsi@vger.kernel.org
@@ -4669,7 +4711,7 @@ F:	Documentation/devicetree/bindings/display/exynos/
 
 DRM DRIVERS FOR FREESCALE DCU
 M:	Stefan Agner <stefan@agner.ch>
-M:	Alison Wang <alison.wang@freescale.com>
+M:	Alison Wang <alison.wang@nxp.com>
 L:	dri-devel@lists.freedesktop.org
 S:	Supported
 F:	drivers/gpu/drm/fsl-dcu/
@@ -4780,6 +4822,14 @@ S:	Maintained
 F:	drivers/gpu/drm/omapdrm/
 F:	Documentation/devicetree/bindings/display/ti/
 
+DRM DRIVERS FOR V3D
+M:	Eric Anholt <eric@anholt.net>
+S:	Supported
+F:	drivers/gpu/drm/v3d/
+F:	include/uapi/drm/v3d_drm.h
+F:	Documentation/devicetree/bindings/display/brcm,bcm-v3d.txt
+T:	git git://anongit.freedesktop.org/drm/drm-misc
+
 DRM DRIVERS FOR VC4
 M:	Eric Anholt <eric@anholt.net>
 T:	git git://github.com/anholt/linux
@@ -4826,6 +4876,15 @@ S:	Maintained
 F:	drivers/gpu/drm/tinydrm/
 F:	include/drm/tinydrm/
 
+DRM DRIVERS FOR XEN
+M:	Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+T:	git git://anongit.freedesktop.org/drm/drm-misc
+L:	dri-devel@lists.freedesktop.org
+L:	xen-devel@lists.xen.org
+S:	Supported
+F:	drivers/gpu/drm/xen/
+F:	Documentation/gpu/xen-front.rst
+
 DRM TTM SUBSYSTEM
 M:	Christian Koenig <christian.koenig@amd.com>
 M:	Roger He <Hongbo.He@amd.com>
@@ -5317,7 +5376,6 @@ F:	include/linux/*mdio*.h
 F:	include/linux/of_net.h
 F:	include/linux/phy.h
 F:	include/linux/phy_fixed.h
-F:	include/linux/platform_data/mdio-gpio.h
 F:	include/linux/platform_data/mdio-bcm-unimac.h
 F:	include/trace/events/mdio.h
 F:	include/uapi/linux/mdio.h
@@ -5413,6 +5471,14 @@ S:	Maintained
 F:	Documentation/hwmon/f71805f
 F:	drivers/hwmon/f71805f.c
 
+FAILOVER MODULE
+M:	Sridhar Samudrala <sridhar.samudrala@intel.com>
+L:	netdev@vger.kernel.org
+S:	Supported
+F:	net/core/failover.c
+F:	include/net/failover.h
+F:	Documentation/networking/failover.rst
+
 FANOTIFY
 M:	Jan Kara <jack@suse.cz>
 R:	Amir Goldstein <amir73il@gmail.com>
@@ -5565,6 +5631,7 @@ S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/atull/linux-fpga.git
 Q:	http://patchwork.kernel.org/project/linux-fpga/list/
 F:	Documentation/fpga/
+F:	Documentation/driver-api/fpga/
 F:	Documentation/devicetree/bindings/fpga/
 F:	drivers/fpga/
 F:	include/linux/fpga/
@@ -5877,14 +5944,14 @@ GENERIC GPIO I2C DRIVER
 M:	Haavard Skinnemoen <hskinnemoen@gmail.com>
 S:	Supported
 F:	drivers/i2c/busses/i2c-gpio.c
-F:	include/linux/i2c-gpio.h
+F:	include/linux/platform_data/i2c-gpio.h
 
 GENERIC GPIO I2C MULTIPLEXER DRIVER
 M:	Peter Korsgaard <peter.korsgaard@barco.com>
 L:	linux-i2c@vger.kernel.org
 S:	Supported
 F:	drivers/i2c/muxes/i2c-mux-gpio.c
-F:	include/linux/i2c-mux-gpio.h
+F:	include/linux/platform_data/i2c-mux-gpio.h
 F:	Documentation/i2c/muxes/i2c-mux-gpio
 
 GENERIC HDLC (WAN) DRIVERS
@@ -6217,6 +6284,7 @@ L:	linux-hwmon@vger.kernel.org
 W:	http://hwmon.wiki.kernel.org/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git
 S:	Maintained
+F:	Documentation/devicetree/bindings/hwmon/
 F:	Documentation/hwmon/
 F:	drivers/hwmon/
 F:	include/linux/hwmon*.h
@@ -6751,6 +6819,12 @@ L:	linux-scsi@vger.kernel.org
 S:	Supported
 F:	drivers/scsi/ibmvscsi/ibmvfc*
 
+IBM Power Virtual Management Channel Driver
+M:	Bryant G. Ly <bryantly@linux.vnet.ibm.com>
+M:	Steven Royer <seroyer@linux.vnet.ibm.com>
+S:	Supported
+F:	drivers/misc/ibmvmc.*
+
 IBM Power Virtual SCSI Device Drivers
 M:	Tyrel Datwyler <tyreld@linux.vnet.ibm.com>
 L:	linux-scsi@vger.kernel.org
@@ -6902,6 +6976,15 @@ F:	drivers/staging/iio/
 F:	include/linux/iio/
 F:	tools/iio/
 
+IIO UNIT CONVERTER
+M:	Peter Rosin <peda@axentia.se>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/iio/afe/current-sense-amplifier.txt
+F:	Documentation/devicetree/bindings/iio/afe/current-sense-shunt.txt
+F:	Documentation/devicetree/bindings/iio/afe/voltage-divider.txt
+F:	drivers/iio/afe/iio-rescale.c
+
 IKANOS/ADI EAGLE ADSL USB DRIVER
 M:	Matthieu Castet <castet.matthieu@free.fr>
 M:	Stanislaw Gruszka <stf_xl@wp.pl>
@@ -7034,14 +7117,13 @@ L:	linux-fbdev@vger.kernel.org
 S:	Maintained
 F:	drivers/video/fbdev/i810/
 
-INTEL ASoC BDW/HSW DRIVERS
+INTEL ASoC DRIVERS
+M:	Pierre-Louis Bossart <pierre-louis.bossart@linux.intel.com>
+M:	Liam Girdwood <liam.r.girdwood@linux.intel.com>
 M:	Jie Yang <yang.jie@linux.intel.com>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
 S:	Supported
-F:	sound/soc/intel/common/sst-dsp*
-F:	sound/soc/intel/common/sst-firmware.c
-F:	sound/soc/intel/boards/broadwell.c
-F:	sound/soc/intel/haswell/
+F:	sound/soc/intel/
 
 INTEL C600 SERIES SAS CONTROLLER DRIVER
 M:	Intel SCU Linux support <intel-linux-scu@intel.com>
@@ -7637,8 +7719,9 @@ M:	Masahiro Yamada <yamada.masahiro@socionext.com>
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-kbuild.git kconfig
 L:	linux-kbuild@vger.kernel.org
 S:	Maintained
-F:	Documentation/kbuild/kconfig-language.txt
+F:	Documentation/kbuild/kconfig*
 F:	scripts/kconfig/
+F:	scripts/Kconfig.include
 
 KDUMP
 M:	Dave Young <dyoung@redhat.com>
@@ -7657,11 +7740,11 @@ W:	https://linuxtv.org
 S:	Maintained
 F:	drivers/media/radio/radio-keene*
 
-KERNEL AUTOMOUNTER v4 (AUTOFS4)
+KERNEL AUTOMOUNTER
 M:	Ian Kent <raven@themaw.net>
 L:	autofs@vger.kernel.org
 S:	Maintained
-F:	fs/autofs4/
+F:	fs/autofs/
 
 KERNEL BUILD + files below scripts/ (unless maintained elsewhere)
 M:	Masahiro Yamada <yamada.masahiro@socionext.com>
@@ -8209,7 +8292,7 @@ F:	drivers/misc/lkdtm/*
 
 LINUX KERNEL MEMORY CONSISTENCY MODEL (LKMM)
 M:	Alan Stern <stern@rowland.harvard.edu>
-M:	Andrea Parri <parri.andrea@gmail.com>
+M:	Andrea Parri <andrea.parri@amarulasolutions.com>
 M:	Will Deacon <will.deacon@arm.com>
 M:	Peter Zijlstra <peterz@infradead.org>
 M:	Boqun Feng <boqun.feng@gmail.com>
@@ -8316,6 +8399,7 @@ F:	Documentation/admin-guide/LSM/LoadPin.rst
 LOCKING PRIMITIVES
 M:	Peter Zijlstra <peterz@infradead.org>
 M:	Ingo Molnar <mingo@redhat.com>
+M:	Will Deacon <will.deacon@arm.com>
 L:	linux-kernel@vger.kernel.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git locking/core
 S:	Maintained
@@ -8350,7 +8434,6 @@ L:	linux-scsi@vger.kernel.org
 W:	http://www.avagotech.com/support/
 S:	Supported
 F:	drivers/message/fusion/
-F:	drivers/scsi/mpt2sas/
 F:	drivers/scsi/mpt3sas/
 
 LSILOGIC/SYMBIOS/NCR 53C8XX and 53C1010 PCI-SCSI drivers
@@ -8472,6 +8555,7 @@ M:	Vivien Didelot <vivien.didelot@savoirfairelinux.com>
 L:	netdev@vger.kernel.org
 S:	Maintained
 F:	drivers/net/dsa/mv88e6xxx/
+F:	linux/platform_data/mv88e6xxx.h
 F:	Documentation/devicetree/bindings/net/dsa/marvell.txt
 
 MARVELL ARMADA DRM SUPPORT
@@ -8829,6 +8913,7 @@ L:	linux-media@vger.kernel.org
 L:	linux-renesas-soc@vger.kernel.org
 T:	git git://linuxtv.org/media_tree.git
 S:	Supported
+F:	Documentation/devicetree/bindings/media/renesas,rcar-csi2.txt
 F:	Documentation/devicetree/bindings/media/rcar_vin.txt
 F:	drivers/media/platform/rcar-vin/
 
@@ -8955,6 +9040,13 @@ L:	linux-wireless@vger.kernel.org
 S:	Maintained
 F:	drivers/net/wireless/mediatek/mt7601u/
 
+MEDIATEK NAND CONTROLLER DRIVER
+M:	Xiaolei Li <xiaolei.li@mediatek.com>
+L:	linux-mtd@lists.infradead.org
+S:	Maintained
+F:	drivers/mtd/nand/raw/mtk_*
+F:	Documentation/devicetree/bindings/mtd/mtk-nand.txt
+
 MEDIATEK RANDOM NUMBER GENERATOR SUPPORT
 M:	Sean Wang <sean.wang@mediatek.com>
 S:	Maintained
@@ -9025,12 +9117,14 @@ W:	http://www.mellanox.com
 Q:	http://patchwork.ozlabs.org/project/netdev/list/
 F:	drivers/net/ethernet/mellanox/mlx5/core/en_*
 
-MELLANOX ETHERNET INNOVA DRIVER
+MELLANOX ETHERNET INNOVA DRIVERS
 R:	Boris Pismenny <borisp@mellanox.com>
 L:	netdev@vger.kernel.org
 S:	Supported
 W:	http://www.mellanox.com
 Q:	http://patchwork.ozlabs.org/project/netdev/list/
+F:	drivers/net/ethernet/mellanox/mlx5/core/en_accel/*
+F:	drivers/net/ethernet/mellanox/mlx5/core/accel/*
 F:	drivers/net/ethernet/mellanox/mlx5/core/fpga/*
 F:	include/linux/mlx5/mlx5_ifc_fpga.h
 
@@ -9290,6 +9384,12 @@ F:	include/linux/cciss*.h
 F:	include/uapi/linux/cciss*.h
 F:	Documentation/scsi/smartpqi.txt
 
+MICROSEMI ETHERNET SWITCH DRIVER
+M:	Alexandre Belloni <alexandre.belloni@bootlin.com>
+L:	netdev@vger.kernel.org
+S:	Supported
+F:	drivers/net/ethernet/mscc/
+
 MICROSOFT SURFACE PRO 3 BUTTON DRIVER
 M:	Chen Yu <yu.c.chen@intel.com>
 L:	platform-driver-x86@vger.kernel.org
@@ -9403,6 +9503,13 @@ Q:	http://patchwork.linuxtv.org/project/linux-media/list/
 S:	Maintained
 F:	drivers/media/dvb-frontends/mn88473*
 
+PCI DRIVER FOR MOBIVEIL PCIE IP
+M:	Subrahmanya Lingappa <l.subrahmanya@mobiveil.co.in>
+L:	linux-pci@vger.kernel.org
+S:	Supported
+F:	Documentation/devicetree/bindings/pci/mobiveil-pcie.txt
+F:	drivers/pci/host/pcie-mobiveil.c
+
 MODULE SUPPORT
 M:	Jessica Yu <jeyu@kernel.org>
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/jeyu/linux.git modules-next
@@ -9584,6 +9691,7 @@ F:	drivers/net/ethernet/myricom/myri10ge/
 
 NAND FLASH SUBSYSTEM
 M:	Boris Brezillon <boris.brezillon@bootlin.com>
+M:	Miquel Raynal <miquel.raynal@bootlin.com>
 R:	Richard Weinberger <richard@nod.at>
 L:	linux-mtd@lists.infradead.org
 W:	http://www.linux-mtd.infradead.org/
@@ -9639,6 +9747,14 @@ S:	Maintained
 F:	Documentation/hwmon/nct6775
 F:	drivers/hwmon/nct6775.c
 
+NET_FAILOVER MODULE
+M:	Sridhar Samudrala <sridhar.samudrala@intel.com>
+L:	netdev@vger.kernel.org
+S:	Supported
+F:	driver/net/net_failover.c
+F:	include/net/net_failover.h
+F:	Documentation/networking/net_failover.rst
+
 NETEFFECT IWARP RNIC DRIVER (IW_NES)
 M:	Faisal Latif <faisal.latif@intel.com>
 L:	linux-rdma@vger.kernel.org
@@ -9700,7 +9816,7 @@ S:	Maintained
 F:	drivers/net/ethernet/netronome/
 
 NETWORK BLOCK DEVICE (NBD)
-M:	Josef Bacik <jbacik@fb.com>
+M:	Josef Bacik <josef@toxicpanda.com>
 S:	Maintained
 L:	linux-block@vger.kernel.org
 L:	nbd@other.debian.org
@@ -9833,6 +9949,7 @@ F:	net/netfilter/xt_CONNSECMARK.c
 F:	net/netfilter/xt_SECMARK.c
 
 NETWORKING [TLS]
+M:	Boris Pismenny <borisp@mellanox.com>
 M:	Aviad Yehezkel <aviadye@mellanox.com>
 M:	Dave Watson <davejwatson@fb.com>
 L:	netdev@vger.kernel.org
@@ -10246,7 +10363,7 @@ F:	arch/arm/mach-omap1/
 F:	arch/arm/plat-omap/
 F:	arch/arm/configs/omap1_defconfig
 F:	drivers/i2c/busses/i2c-omap.c
-F:	include/linux/i2c-omap.h
+F:	include/linux/platform_data/i2c-omap.h
 
 OMAP2+ SUPPORT
 M:	Tony Lindgren <tony@atomide.com>
@@ -10278,7 +10395,7 @@ F:	drivers/regulator/tps65218-regulator.c
 F:	drivers/regulator/tps65910-regulator.c
 F:	drivers/regulator/twl-regulator.c
 F:	drivers/regulator/twl6030-regulator.c
-F:	include/linux/i2c-omap.h
+F:	include/linux/platform_data/i2c-omap.h
 
 ONION OMEGA2+ BOARD
 M:	Harvey Hunt <harveyhuntnexus@gmail.com>
@@ -10355,6 +10472,7 @@ T:	git git://linuxtv.org/media_tree.git
 S:	Odd fixes
 F:	drivers/media/i2c/ov772x.c
 F:	include/media/i2c/ov772x.h
+F:	Documentation/devicetree/bindings/media/i2c/ov772x.txt
 
 OMNIVISION OV7740 SENSOR DRIVER
 M:	Wenyou Yang <wenyou.yang@microchip.com>
@@ -10405,12 +10523,14 @@ F:	drivers/infiniband/ulp/opa_vnic
 
 OPEN FIRMWARE AND DEVICE TREE OVERLAYS
 M:	Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+M:	Frank Rowand <frowand.list@gmail.com>
 L:	devicetree@vger.kernel.org
 S:	Maintained
 F:	Documentation/devicetree/dynamic-resolution-notes.txt
 F:	Documentation/devicetree/overlay-notes.txt
 F:	drivers/of/overlay.c
 F:	drivers/of/resolver.c
+K:	of_overlay_notifier_
 
 OPEN FIRMWARE AND FLATTENED DEVICE TREE
 M:	Rob Herring <robh+dt@kernel.org>
@@ -10722,9 +10842,9 @@ F:	Documentation/devicetree/bindings/pci/cdns,*.txt
 F:	drivers/pci/cadence/pcie-cadence*
 
 PCI DRIVER FOR FREESCALE LAYERSCAPE
-M:	Minghuan Lian <minghuan.Lian@freescale.com>
-M:	Mingkai Hu <mingkai.hu@freescale.com>
-M:	Roy Zang <tie-fei.zang@freescale.com>
+M:	Minghuan Lian <minghuan.Lian@nxp.com>
+M:	Mingkai Hu <mingkai.hu@nxp.com>
+M:	Roy Zang <roy.zang@nxp.com>
 L:	linuxppc-dev@lists.ozlabs.org
 L:	linux-pci@vger.kernel.org
 L:	linux-arm-kernel@lists.infradead.org
@@ -10950,8 +11070,8 @@ M:	Shawn Lin <shawn.lin@rock-chips.com>
 L:	linux-pci@vger.kernel.org
 L:	linux-rockchip@lists.infradead.org
 S:	Maintained
-F:	Documentation/devicetree/bindings/pci/rockchip-pcie.txt
-F:	drivers/pci/host/pcie-rockchip.c
+F:	Documentation/devicetree/bindings/pci/rockchip-pcie*
+F:	drivers/pci/host/pcie-rockchip*
 
 PCI DRIVER FOR V3 SEMICONDUCTOR V360EPC
 M:	Linus Walleij <linus.walleij@linaro.org>
@@ -11114,6 +11234,7 @@ L:	linux-mediatek@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 F:	Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt
 F:	Documentation/devicetree/bindings/pinctrl/pinctrl-mt7622.txt
+F:	drivers/pinctrl/mediatek/mtk-eint.*
 F:	drivers/pinctrl/mediatek/pinctrl-mtk-common.*
 F:	drivers/pinctrl/mediatek/pinctrl-mt2701.c
 F:	drivers/pinctrl/mediatek/pinctrl-mt7622.c
@@ -11252,6 +11373,7 @@ M:	Sebastian Reichel <sre@kernel.org>
 L:	linux-pm@vger.kernel.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git
 S:	Maintained
+F:	Documentation/ABI/testing/sysfs-class-power
 F:	Documentation/devicetree/bindings/power/supply/
 F:	include/linux/power_supply.h
 F:	drivers/power/supply/
@@ -11654,6 +11776,13 @@ F:	Documentation/devicetree/bindings/media/qcom,camss.txt
 F:	Documentation/media/v4l-drivers/qcom_camss.rst
 F:	drivers/media/platform/qcom/camss-8x16/
 
+QUALCOMM CPUFREQ DRIVER MSM8996/APQ8096
+M:  Ilia Lin <ilia.lin@gmail.com>
+L:  linux-pm@vger.kernel.org
+S:  Maintained
+F:  Documentation/devicetree/bindings/opp/kryo-cpufreq.txt
+F:  drivers/cpufreq/qcom-cpufreq-kryo.c
+
 QUALCOMM EMAC GIGABIT ETHERNET DRIVER
 M:	Timur Tabi <timur@codeaurora.org>
 L:	netdev@vger.kernel.org
@@ -11667,6 +11796,14 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/rkuo/linux-hexagon-kernel.g
 S:	Supported
 F:	arch/hexagon/
 
+QUALCOMM HIDMA DRIVER
+M:	Sinan Kaya <okaya@codeaurora.org>
+L:	linux-arm-kernel@lists.infradead.org
+L:	linux-arm-msm@vger.kernel.org
+L:	dmaengine@vger.kernel.org
+S:	Supported
+F:	drivers/dma/qcom/hidma*
+
 QUALCOMM IOMMU
 M:	Rob Clark <robdclark@gmail.com>
 L:	iommu@lists.linux-foundation.org
@@ -11878,7 +12015,7 @@ F:	include/linux/rtc.h
 F:	include/uapi/linux/rtc.h
 F:	include/linux/rtc/
 F:	include/linux/platform_data/rtc-*
-F:	tools/testing/selftests/timers/rtctest.c
+F:	tools/testing/selftests/rtc/
 
 REALTEK AUDIO CODECS
 M:	Bard Liao <bardliao@realtek.com>
@@ -13061,6 +13198,13 @@ S:	Maintained
 F:	drivers/ssb/
 F:	include/linux/ssb/
 
+SONY IMX258 SENSOR DRIVER
+M:	Sakari Ailus <sakari.ailus@linux.intel.com>
+L:	linux-media@vger.kernel.org
+T:	git git://linuxtv.org/media_tree.git
+S:	Maintained
+F:	drivers/media/i2c/imx258.c
+
 SONY IMX274 SENSOR DRIVER
 M:	Leon Luo <leonl@leopardimaging.com>
 L:	linux-media@vger.kernel.org
@@ -13105,7 +13249,7 @@ F:	include/uapi/sound/
 F:	sound/
 
 SOUND - COMPRESSED AUDIO
-M:	Vinod Koul <vinod.koul@intel.com>
+M:	Vinod Koul <vkoul@kernel.org>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git
 S:	Supported
@@ -13277,13 +13421,6 @@ L:	stable@vger.kernel.org
 S:	Supported
 F:	Documentation/process/stable-kernel-rules.rst
 
-STAGING - ATOMISP DRIVER
-M:	Alan Cox <alan@linux.intel.com>
-M:	Sakari Ailus <sakari.ailus@linux.intel.com>
-L:	linux-media@vger.kernel.org
-S:	Maintained
-F:	drivers/staging/media/atomisp/
-
 STAGING - COMEDI
 M:	Ian Abbott <abbotti@mev.co.uk>
 M:	H Hartley Sweeten <hsweeten@visionengravers.com>
@@ -13398,9 +13535,20 @@ T:	git git://linuxtv.org/media_tree.git
 S:	Maintained
 F:	drivers/media/usb/stk1160/
 
+STM32 TIMER/LPTIMER DRIVERS
+M:	Fabrice Gasnier <fabrice.gasnier@st.com>
+S:	Maintained
+F:	drivers/*/stm32-*timer*
+F:	drivers/pwm/pwm-stm32*
+F:	include/linux/*/stm32-*tim*
+F:	Documentation/ABI/testing/*timer-stm32
+F:	Documentation/devicetree/bindings/*/stm32-*timer
+F:	Documentation/devicetree/bindings/pwm/pwm-stm32*
+
 STMMAC ETHERNET DRIVER
 M:	Giuseppe Cavallaro <peppe.cavallaro@st.com>
 M:	Alexandre Torgue <alexandre.torgue@st.com>
+M:	Jose Abreu <joabreu@synopsys.com>
 L:	netdev@vger.kernel.org
 W:	http://www.stlinux.com
 S:	Supported
@@ -13477,6 +13625,12 @@ S:	Supported
 F:	net/switchdev/
 F:	include/net/switchdev.h
 
+SY8106A REGULATOR DRIVER
+M:	Icenowy Zheng <icenowy@aosc.io>
+S:	Maintained
+F:	drivers/regulator/sy8106a-regulator.c
+F:	Documentation/devicetree/bindings/regulator/sy8106a-regulator.txt
+
 SYNC FILE FRAMEWORK
 M:	Sumit Semwal <sumit.semwal@linaro.org>
 R:	Gustavo Padovan <gustavo@padovan.org>
@@ -14617,7 +14771,9 @@ M:	Woojung Huh <woojung.huh@microchip.com>
 M:	Microchip Linux Driver Support <UNGLinuxDriver@microchip.com>
 L:	netdev@vger.kernel.org
 S:	Maintained
+F:	Documentation/devicetree/bindings/net/microchip,lan78xx.txt
 F:	drivers/net/usb/lan78xx.*
+F:	include/dt-bindings/net/microchip-lan78xx.h
 
 USB MASS STORAGE DRIVER
 M:	Alan Stern <stern@rowland.harvard.edu>
@@ -14933,6 +15089,12 @@ L:	linux-media@vger.kernel.org
 S:	Maintained
 F:	drivers/media/platform/video-mux.c
 
+VIDEO I2C POLLING DRIVER
+M:	Matt Ranostay <matt.ranostay@konsulko.com>
+L:	linux-media@vger.kernel.org
+S:	Maintained
+F:	drivers/media/i2c/video-i2c.c
+
 VIDEOBUF2 FRAMEWORK
 M:	Pawel Osciak <pawel@osciak.com>
 M:	Marek Szyprowski <m.szyprowski@samsung.com>
@@ -15405,6 +15567,14 @@ T:	git git://linuxtv.org/media_tree.git
 S:	Maintained
 F:	drivers/media/tuners/tuner-xc2028.*
 
+XDP SOCKETS (AF_XDP)
+M:	Björn Töpel <bjorn.topel@intel.com>
+M:	Magnus Karlsson <magnus.karlsson@intel.com>
+L:	netdev@vger.kernel.org
+S:	Maintained
+F:	kernel/bpf/xskmap.c
+F:	net/xdp/
+
 XEN BLOCK SUBSYSTEM
 M:	Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
 M:	Roger Pau Monné <roger.pau@citrix.com>
@@ -15475,6 +15645,13 @@ S:	Supported
 F:	arch/x86/xen/*swiotlb*
 F:	drivers/xen/*swiotlb*
 
+XEN SOUND FRONTEND DRIVER
+M:	Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+L:	xen-devel@lists.xenproject.org (moderated for non-subscribers)
+L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
+S:	Supported
+F:	sound/xen/*
+
 XFS FILESYSTEM
 M:	Darrick J. Wong <darrick.wong@oracle.com>
 M:	linux-xfs@vger.kernel.org
@@ -15513,6 +15690,14 @@ L:	linux-kernel@vger.kernel.org
 S:	Supported
 F:	drivers/char/xillybus/
 
+XLP9XX I2C DRIVER
+M:	George Cherian <george.cherian@cavium.com>
+M:	Jan Glauber <jglauber@cavium.com>
+L:	linux-i2c@vger.kernel.org
+W:	http://www.cavium.com
+S:	Supported
+F:	drivers/i2c/busses/i2c-xlp9xx.c
+
 XRA1403 GPIO EXPANDER
 M:	Nandor Han <nandor.han@ge.com>
 M:	Semi Malinen <semi.malinen@ge.com>
@@ -15607,7 +15792,7 @@ L:	linux-media@vger.kernel.org
 W:	http://mjpeg.sourceforge.net/driver-zoran/
 T:	hg https://linuxtv.org/hg/v4l-dvb
 S:	Odd Fixes
-F:	drivers/media/pci/zoran/
+F:	drivers/staging/media/zoran/
 
 ZRAM COMPRESSED RAM BLOCK DEVICE DRVIER
 M:	Minchan Kim <minchan@kernel.org>
@@ -15631,7 +15816,7 @@ L:	linux-mm@kvack.org
 S:	Maintained
 F:	mm/zsmalloc.c
 F:	include/linux/zsmalloc.h
-F:	Documentation/vm/zsmalloc.txt
+F:	Documentation/vm/zsmalloc.rst
 
 ZSWAP COMPRESSED SWAP CACHING
 M:	Seth Jennings <sjenning@redhat.com>
diff --git a/Makefile b/Makefile
index 56ba070dfa09..eb4cf0f9566c 100644
--- a/Makefile
+++ b/Makefile
@@ -316,12 +316,9 @@ SUBARCH := $(shell uname -m | sed -e s/i.86/x86/ -e s/x86_64/x86/ \
 # CROSS_COMPILE can be set on the command line
 # make CROSS_COMPILE=ia64-linux-
 # Alternatively CROSS_COMPILE can be set in the environment.
-# A third alternative is to store a setting in .config so that plain
-# "make" in the configured kernel build directory always uses that.
 # Default value for CROSS_COMPILE is not to prefix executables
 # Note: Some architectures assign CROSS_COMPILE in their arch/*/Makefile
 ARCH		?= $(SUBARCH)
-CROSS_COMPILE	?= $(CONFIG_CROSS_COMPILE:"%"=%)
 
 # Architecture as present in compile.h
 UTS_MACHINE 	:= $(ARCH)
@@ -445,6 +442,8 @@ export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE
 export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL
 export KBUILD_ARFLAGS
 
+export CC_VERSION_TEXT := $(shell $(CC) --version | head -n 1)
+
 # When compiling out-of-tree modules, put MODVERDIR in the module
 # tree rather than in the kernel tree. The kernel tree might
 # even be read-only.
@@ -504,7 +503,7 @@ KBUILD_CFLAGS	+= $(call cc-option,-fno-PIE)
 KBUILD_AFLAGS	+= $(call cc-option,-fno-PIE)
 
 # check for 'asm goto'
-ifeq ($(call shell-cached,$(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC) $(KBUILD_CFLAGS)), y)
+ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC) $(KBUILD_CFLAGS)), y)
   CC_HAVE_ASM_GOTO := 1
   KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO
   KBUILD_AFLAGS += -DCC_HAVE_ASM_GOTO
@@ -627,7 +626,7 @@ all: vmlinux
 CFLAGS_GCOV	:= -fprofile-arcs -ftest-coverage \
 	$(call cc-option,-fno-tree-loop-im) \
 	$(call cc-disable-warning,maybe-uninitialized,)
-export CFLAGS_GCOV CFLAGS_KCOV
+export CFLAGS_GCOV
 
 # The arch Makefile can set ARCH_{CPP,A,C}FLAGS to override the default
 # values of the respective KBUILD_* variables
@@ -676,55 +675,11 @@ ifneq ($(CONFIG_FRAME_WARN),0)
 KBUILD_CFLAGS += $(call cc-option,-Wframe-larger-than=${CONFIG_FRAME_WARN})
 endif
 
-# This selects the stack protector compiler flag. Testing it is delayed
-# until after .config has been reprocessed, in the prepare-compiler-check
-# target.
-ifdef CONFIG_CC_STACKPROTECTOR_AUTO
-  stackp-flag := $(call cc-option,-fstack-protector-strong,$(call cc-option,-fstack-protector))
-  stackp-name := AUTO
-else
-ifdef CONFIG_CC_STACKPROTECTOR_REGULAR
-  stackp-flag := -fstack-protector
-  stackp-name := REGULAR
-else
-ifdef CONFIG_CC_STACKPROTECTOR_STRONG
-  stackp-flag := -fstack-protector-strong
-  stackp-name := STRONG
-else
-  # If either there is no stack protector for this architecture or
-  # CONFIG_CC_STACKPROTECTOR_NONE is selected, we're done, and $(stackp-name)
-  # is empty, skipping all remaining stack protector tests.
-  #
-  # Force off for distro compilers that enable stack protector by default.
-  KBUILD_CFLAGS += $(call cc-option, -fno-stack-protector)
-endif
-endif
-endif
-# Find arch-specific stack protector compiler sanity-checking script.
-ifdef stackp-name
-ifneq ($(stackp-flag),)
-  stackp-path := $(srctree)/scripts/gcc-$(SRCARCH)_$(BITS)-has-stack-protector.sh
-  stackp-check := $(wildcard $(stackp-path))
-  # If the wildcard test matches a test script, run it to check functionality.
-  ifdef stackp-check
-    ifneq ($(shell $(CONFIG_SHELL) $(stackp-check) $(CC) $(KBUILD_CPPFLAGS) $(biarch)),y)
-      stackp-broken := y
-    endif
-  endif
-  ifndef stackp-broken
-    # If the stack protector is functional, enable code that depends on it.
-    KBUILD_CPPFLAGS += -DCONFIG_CC_STACKPROTECTOR
-    # Either we've already detected the flag (for AUTO) or we'll fail the
-    # build in the prepare-compiler-check rule (for specific flag).
-    KBUILD_CFLAGS += $(stackp-flag)
-  else
-    # We have to make sure stack protector is unconditionally disabled if
-    # the compiler is broken (in case we're going to continue the build in
-    # AUTO mode).
-    KBUILD_CFLAGS += $(call cc-option, -fno-stack-protector)
-  endif
-endif
-endif
+stackp-flags-$(CONFIG_CC_HAS_STACKPROTECTOR_NONE) := -fno-stack-protector
+stackp-flags-$(CONFIG_CC_STACKPROTECTOR)          := -fstack-protector
+stackp-flags-$(CONFIG_CC_STACKPROTECTOR_STRONG)   := -fstack-protector-strong
+
+KBUILD_CFLAGS += $(stackp-flags-y)
 
 ifeq ($(cc-name),clang)
 KBUILD_CPPFLAGS += $(call cc-option,-Qunused-arguments,)
@@ -802,13 +757,12 @@ KBUILD_CFLAGS += $(call cc-option, -fno-inline-functions-called-once)
 endif
 
 ifdef CONFIG_LD_DEAD_CODE_DATA_ELIMINATION
-KBUILD_CFLAGS	+= $(call cc-option,-ffunction-sections,)
-KBUILD_CFLAGS	+= $(call cc-option,-fdata-sections,)
+KBUILD_CFLAGS_KERNEL	+= $(call cc-option,-ffunction-sections,)
+KBUILD_CFLAGS_KERNEL	+= $(call cc-option,-fdata-sections,)
 endif
 
 # arch Makefile may override CC so keep this after arch Makefile is included
-NOSTDINC_FLAGS += -nostdinc -isystem $(call shell-cached,$(CC) -print-file-name=include)
-CHECKFLAGS     += $(NOSTDINC_FLAGS)
+NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
 # warn about C99 declaration after statement
 KBUILD_CFLAGS += $(call cc-option,-Wdeclaration-after-statement,)
@@ -878,6 +832,9 @@ ifeq ($(CONFIG_STRIP_ASM_SYMS),y)
 LDFLAGS_vmlinux	+= $(call ld-option, -X,)
 endif
 
+# insure the checker run with the right endianness
+CHECKFLAGS += $(if $(CONFIG_CPU_BIG_ENDIAN),-mbig-endian,-mlittle-endian)
+
 # Default kernel image to build when no specific target is given.
 # KBUILD_IMAGE may be overruled on the command line or
 # set in the environment
@@ -1103,7 +1060,7 @@ endif
 # prepare2 creates a makefile if using a separate output directory.
 # From this point forward, .config has been reprocessed, so any rules
 # that need to depend on updated CONFIG_* values can be checked here.
-prepare2: prepare3 prepare-compiler-check outputmakefile asm-generic
+prepare2: prepare3 outputmakefile asm-generic
 
 prepare1: prepare2 $(version_h) $(autoksyms_h) include/generated/utsrelease.h \
                    include/config/auto.conf
@@ -1129,43 +1086,6 @@ uapi-asm-generic:
 PHONY += prepare-objtool
 prepare-objtool: $(objtool_target)
 
-# Check for CONFIG flags that require compiler support. Abort the build
-# after .config has been processed, but before the kernel build starts.
-#
-# For security-sensitive CONFIG options, we don't want to fallback and/or
-# silently change which compiler flags will be used, since that leads to
-# producing kernels with different security feature characteristics
-# depending on the compiler used. (For example, "But I selected
-# CC_STACKPROTECTOR_STRONG! Why did it build with _REGULAR?!")
-PHONY += prepare-compiler-check
-prepare-compiler-check: FORCE
-# Make sure compiler supports requested stack protector flag.
-ifdef stackp-name
-  # Warn about CONFIG_CC_STACKPROTECTOR_AUTO having found no option.
-  ifeq ($(stackp-flag),)
-	@echo CONFIG_CC_STACKPROTECTOR_$(stackp-name): \
-		  Compiler does not support any known stack-protector >&2
-  else
-  # Fail if specifically requested stack protector is missing.
-  ifeq ($(call cc-option, $(stackp-flag)),)
-	@echo Cannot use CONFIG_CC_STACKPROTECTOR_$(stackp-name): \
-		  $(stackp-flag) not supported by compiler >&2 && exit 1
-  endif
-  endif
-endif
-# Make sure compiler does not have buggy stack-protector support. If a
-# specific stack-protector was requested, fail the build, otherwise warn.
-ifdef stackp-broken
-  ifeq ($(stackp-name),AUTO)
-	@echo CONFIG_CC_STACKPROTECTOR_$(stackp-name): \
-                  $(stackp-flag) available but compiler is broken: disabling >&2
-  else
-	@echo Cannot use CONFIG_CC_STACKPROTECTOR_$(stackp-name): \
-                  $(stackp-flag) available but compiler is broken >&2 && exit 1
-  endif
-endif
-	@:
-
 # Generate some files
 # ---------------------------------------------------------------------------
 
@@ -1625,7 +1545,6 @@ clean: $(clean-dirs)
 		-o -name '*.asn1.[ch]' \
 		-o -name '*.symtypes' -o -name 'modules.order' \
 		-o -name modules.builtin -o -name '.tmp_*.o.*' \
-		-o -name .cache.mk \
 		-o -name '*.c.[012]*.*' \
 		-o -name '*.ll' \
 		-o -name '*.gcno' \) -type f -print | xargs rm -f
@@ -1763,7 +1682,7 @@ quiet_cmd_rmfiles = $(if $(wildcard $(rm-files)),CLEAN   $(wildcard $(rm-files))
 # Run depmod only if we have System.map and depmod is executable
 quiet_cmd_depmod = DEPMOD  $(KERNELRELEASE)
       cmd_depmod = $(CONFIG_SHELL) $(srctree)/scripts/depmod.sh $(DEPMOD) \
-                   $(KERNELRELEASE) "$(patsubst y,_,$(CONFIG_HAVE_UNDERSCORE_SYMBOL_PREFIX))"
+                   $(KERNELRELEASE)
 
 # Create temporary dir for module support files
 # clean it up only when building all modules
diff --git a/Next/SHA1s b/Next/SHA1s
new file mode 100644
index 000000000000..87154c2800c1
--- /dev/null
+++ b/Next/SHA1s
@@ -0,0 +1,280 @@
+Name		SHA1
+----		----
+origin		d60d61f36b8f8d3f67c3fe49ee35cfd54b3bc49b
+fixes		147a89bc71e7db40f011454a40add7ff2d10f8d8
+kbuild-current	b04e217704b7f879c6b91222b066983a44a7a09f
+arc-current	661e50bc853209e41a5c14a290ca4decc43cbfd1
+arm-current	92d44a42af81e850a038c38278ff4f434b2871df
+arm64-fixes	82034c23fcbc2389c73d97737f61fa2dd6526413
+m68k-current	ecd685580c8feb9a91c758257a66355a673594d9
+powerpc-fixes	faf37c44a105f3608115785f17cbbf3500f8bc71
+sparc		fff75eb2a08c2ac96404a2d79685668f3cf5a7a3
+fscrypt-current	ae64f9bd1d3621b5e60d7363bc20afb46aede215
+net		f5a4941aa6d190e676065e8f4ed35999f52a01c3
+bpf		c93552c443ebc63b14e26e46d2e76941c88e0d71
+ipsec		d9f92772e8ec388d070752ee8f187ef8fa18621f
+netfilter	312564269535892cc082bc80592150cd1f5e8ec3
+ipvs		2f99aa31cd7a5d667f17dd2924c884c3d2c621ac
+wireless-drivers	ab1068d6866e28bf6427ceaea681a381e5870a4a
+mac80211	312564269535892cc082bc80592150cd1f5e8ec3
+rdma-fixes	533d1daea8d8a389b37207ad7b50c4e750969231
+sound-current	009f8c90f571d87855914dbc20e6c0ea2a3b19ae
+sound-asoc-fixes	3e170d2c7bd4c9eef0bdb564f482da1777887364
+regmap-fixes	4b1b7043a286ffee6ab8cc0c6048ddf77e485df5
+regulator-fixes	16d005d82c847c20bf5985bf63f4947a4ced4077
+spi-fixes	2a5c57d435d9a342f8f8f57b0c60e2d79081488f
+pci-current	0cf22d6b317ce0103b7d5a47878aa5ef94240433
+driver-core.current	6da6c0db5316275015e8cc2959f12a17584aeb64
+tty.current	6da6c0db5316275015e8cc2959f12a17584aeb64
+usb.current	771c577c23bac90597c685971d7297ea00f99d11
+usb-gadget-fixes	6d08b06e67cd117f6992c46611dfb4ce267cd71e
+usb-serial-fixes	75bc37fefc4471e718ba8e651aa74673d4e0a9eb
+usb-chipidea-fixes	964728f9f407eca0b417fdf8e784b7a76979490c
+phy		60cc43fc888428bb2f18f08997432d426a243338
+staging.current	9d569b1cf7a2a385770835df15f77dde587b58a3
+char-misc.current	806e30873f0e74d9d41b0ef761bd4d3e55c7d510
+input-current	40f7090bb1b4ec327ea1e1402ff5783af5b35195
+crypto-current	bf4407f081f33466ef0b06e213c04264d356aa35
+ide		8e44e6600caa7b96a5b71ae36c8829db753c2d70
+vfio-fixes	834814e80268c818f354c8f402e0c6604ed75589
+kselftest-fixes	75bc37fefc4471e718ba8e651aa74673d4e0a9eb
+modules-fixes	be71eda5383faa663efdba9ef54a6b8255e3c7f0
+slave-dma-fixes	dbad41e7bb5f4b9949ff5ea1d76c20711f326308
+backlight-fixes	8ff5cbc6e3c5c2855245c014f3ba572ddfc5ab0f
+mtd-fixes	9f825e74d761c13b0cfaa5f65344d64ff970e252
+nand-fixes	ee02f73e04c0e690600f621a3a1d2245834af7fe
+spi-nor-fixes	7928b2cbe55b2a410a0f5c1f154610059c57b1b2
+mfd-fixes	11799564fc7eedff50801950090773928f867996
+v4l-dvb-fixes	b84c8fdf8e6c318ecdb3fdbdd4953d83113283ea
+reset-fixes	e6914365fd280fce303a89b8a8d4529af5a2e0f9
+mips-fixes	32795631e67e16141aa5e065c28ba03bf17abb90
+omap-fixes	41bd6adf3c5d3990d7a67639aff53d54483744fa
+kvm-fixes	696ca779a928d0e93d61c38ffc3a4d8914a9b9a0
+kvms390-fixes	f4a551b72358facbbe5714248dff78404272feee
+hwmon-fixes	771c577c23bac90597c685971d7297ea00f99d11
+nvdimm-fixes	75bc37fefc4471e718ba8e651aa74673d4e0a9eb
+btrfs-fixes	e97ce3cc988823bdeb515f8fca24e5a38a38a00d
+vfs-fixes	4faa99965e027cc057c5145ce45fa772caa04e8d
+dma-mapping-fixes	05e13bb57e6f181d7605f8608181c7e6fb7f591d
+drivers-x86-fixes	54940fa60ad3728c592f62dadb558165495a6938
+samsung-krzk-fixes	60cc43fc888428bb2f18f08997432d426a243338
+pinctrl-samsung-fixes	60cc43fc888428bb2f18f08997432d426a243338
+devicetree-fixes	f130307054a59ca21d2396f386be77ebd2e8ca96
+scsi-fixes	c9ddf73476ff4fffb7a87bd5107a0705bf2cf64b
+drm-intel-fixes	65b3bdc807ac7bd83f5b27bc2c29a3c631eed7dd
+mmc-fixes	3de06d5a1f05c11c94cbb68af14dbfa7fb81d78b
+rtc-fixes	60cc43fc888428bb2f18f08997432d426a243338
+drm-misc-fixes	c32048d9e93a5ab925d745396c63e7b912147f0a
+kbuild		57282f7da50c5972c4f4453ebeed67bf714d5265
+leaks		9f84a0f7f1a613335a9aaf0abd5858906752bf93
+uuid		c0020756315eebec58310aca42cf9fb73e1322eb
+dma-mapping	267d2e18a281c965229a3c3ccce51c471f9c4f8a
+asm-generic	a71e7c44ffb7baea0c0795824afc34cc0bc1a301
+arc		4fbd8d194f06c8a3fd2af1ce560ddb31f7ec8323
+arm		8de97027b2b4dd4676cbfcd27aefabfcc9e11721
+arm-perf	73acc0315cd72174729141856c6607d1c91419cb
+arm-soc		933d3cf3d4166fe687fc86325a4c7c54d82321f2
+actions		61862a89e6acb61e1193c3ad6d4eeab966c470ed
+alpine		7928b2cbe55b2a410a0f5c1f154610059c57b1b2
+amlogic		9999d2013e27be903e227bac2d2b4510954d959b
+aspeed		6534b2d0de3693b8508375c2f01951d6eb5c69be
+at91		20d3f36b0feaad5ab90ccff77e18c13939065cad
+bcm2835		5c483a0c319cdc7fa46676939da59df1fa8abb5a
+imx-mxs		452daaaf6021e7ff1d9d849493f8b7a740b206e4
+keystone	fb5d812c3cb33d0410cb6d3beebca670b3af097d
+mediatek	9378b342734132ddf6d95ed035e1124eacf762a1
+mvebu		78e5d8c26c70bf14e1b4d918e8d65209ffe12123
+omap		1b7e42c1dfe837642cfcc426776a3f98c6fb2d3e
+reset		d7bab65b1f57cdf2d81fc469cea6b2160a50e917
+qcom		509d20563e5d72c2b3a85cc6d05bee30006b8127
+realtek		2b6286eb7ab8b9a22f7238bdaad403badd4a93e9
+renesas		e800ff91fb6723f332fa837d8d11b3eab7aaad5b
+rockchip	42568f346f6545c3b80fb9fe09d5d2a13766db02
+samsung		bebc6082da0a9f5d47a1ea2edc099bf671058bd4
+samsung-krzk	fba97222728d6067f7d26da3136631a1cd4e44ac
+sunxi		e9736005439109315e474dfc59c734e724d3e071
+tegra		6de9ae597a799a1f753385a08ef0d48c6c683917
+arm64		159fd7b8d3d12b27593d4fe3f6ae1d8e14ea9d0b
+clk		5f016be1c548f9dadceb1f6b0fbb062f1205ebfb
+clk-samsung	e8cf870f253bdeadd4ce8a10e34655ef89ad57b1
+c6x		85fa2cc51104ee5b529ce85b12f4c8421888544f
+m68k		b2ab01132c13029c26b6218050be0639d250fffd
+m68knommu	082f55c459845088c3fee99c3a88ee117c148218
+microblaze	a7a9e2f0410ddf9ec7491fca0c91fcb4c5c30cbb
+mips		ea4d340cbea470247b19ee696e31f102bc64466a
+mips-james	21325631f395b1885a0979ac2eb9838ab2036526
+nds32		3c6b01addf6eaa4f364661935e2222c84dc0c1c4
+nios2		3d9644ef9a0f6c3ca0c1bd2aea1d82e7ea0a7f24
+openrisc	d56f3af9e801970d21c57621de3b42bc17eac152
+parisc-hd	8d40c754fbacc2432e53ab2b2a98b86beba251c0
+powerpc		d647b210ac738b401c7f824bbebdcbcedbe7cb6b
+powerpc-pkey	d86a4ba02fc6bea4670b2f59b208152a4a34c3c0
+fsl		c095ff93f901c1620b28dce4d813dd548bc5236b
+risc-v		67b8d5c7081221efa252e111cd52532ec6d4266f
+s390		89a0c0ec0d2e3ce0ee9caa00f60c0c26ccf11c21
+sparc-next	17dec0a949153d9ac00760ba2f5b78cb583e995f
+sh		ac21fc2dcb405cf250ad3f1228f64f64930d9211
+uml		e40238dedb484c8a19f8257e4ef5d77d038f9ad8
+xtensa		ae6063c83634cbef7f73ec8a3ab7a6adf5294716
+fscrypt		e1cc40e5d42acb1d99652babb17e6a5ee4247409
+befs		55d945e2e4aa148ef38bdfabb9569af78183a539
+btrfs		7c9a09f56a91c4ee5636703d3a95c72710917718
+btrfs-kdave	85b1781f82422db46598c7b96fb024d0c8e5491b
+ceph		fc218544fbc800d1c91348ec834cacfb257348f7
+cifs		5c5a41be89e87c3a9ce2fed3c818770ef12bd2bb
+configfs	6ace4f6bbcfde70278a33c11443ecda3ed0ec1b6
+ecryptfs	e86281e700cca8a773f9a572fa406adf2784ba5c
+ext3		20b67d6401f1fca9cc92ba7bdfb99b2c5c262e8f
+ext4		4f2f76f751433908364ccff82f437a57d0e6e9b7
+f2fs		cba608493d3d5d54da4b7dd67868f88c43dfbd35
+fuse		5ba24197b94d4070d8c2a17fa4944a55cc39ef03
+jfs		86313903430d08739e391ad51b4865dc3a2bf522
+nfs		036db8bd96374c66424f270f3370ddaf0adf7506
+nfs-anna	ffbfa2ce88f169aa4b5cc90065cdc1c3c8003c27
+nfsd		28771950c592482ee86cb1c3b661688aec3c0d7d
+orangefs	8d720dc81d4d3a0cbae0df8d7442e2c23620657a
+overlayfs	b0ee6e9a4c53ae7618bbd146f1c5b8399a3ece08
+ubifs		b5094b7f135be34630e3ea8a98fa215715d0f29d
+xfs		d25522f10cfa342a20867086a516c45c87b0877c
+file-locks	7a107c0f55a3b4c6f84a4323df5610360bde1684
+vfs		1e9d358032bca35a697646e87b857b1f1c171bff
+vfs-miklos	0eb8af4916a540c362a2950e5ab54eca32eb7d58
+printk		913a61197c1f3acb5f15c963d2a6924a30214f82
+pci		6bf16b354ddc03c3ee8a05175bab2278362fabc6
+pstore		e698aaf37f9fac419ac6c1867137cce83c90c80b
+hid		f1b6ae6701a772bba1150b5f1b8d43803f039be4
+i2c		bbfc642934a7e1e0e63474c6fbfa2de8e0eed8c1
+dmi		5cb06cfce4a8a15bda8c7bcf8012f6ce301cad0e
+hwmon-staging	853d5e4b9a48fa91a329123bac33b1cf30277a04
+jc_docs		a49d9c0ae46e149a22aefa8251d07dddd5611851
+v4l-dvb		a00031c159748f322f771f3c1d5ed944cba4bd30
+v4l-dvb-next	f10379aad39e9da8bc7d1822e251b5f0673067ef
+fbdev		dbeef4fe697bfc0c071a88fe565182ce9729a7ba
+pm		c7533d0fd82f8d3b10683c3238a4c6c330f2244d
+cpupower	f9652d5cae04eb5e85303c087f5842d320499c65
+idle		8a5776a5f49812d29fe4b2d0a2d71675c3facf3f
+opp		8a352fd8787cefcb19c25ca1390301f874797b9c
+thermal		e76a4386b2f57e187d617bef947feda9e84d422f
+thermal-soc	9efc58dfa14a200efd5c005dd25ca95c686c1d8a
+ieee1394	188775181bc05f29372b305ef96485840e351fde
+dlm		da3627c30d229fea1e070e984366f80a1c4d9166
+swiotlb		4855c92dbb7b3b85c23e88ab7ca04f99b9677b41
+rdma		f3ca0ab114e0de3bbad4c4a537d32fb57aa42f81
+net-next	37c9102f216c2ecd4296de1027fbfc4fa7d73877
+bpf-next	bcece5dc40b9e00d0a9a91fc0acc9a825c23362b
+ipsec-next	f03329627b46c28f0682040e7ddbfd346b78bd4e
+netfilter-next	e523452ac0e046d38fcfb7c16748ab8090bf98cd
+nfc-next	4d63adfe12dd9cb61ed8badb4d798955399048c2
+ipvs-next	90278871d4b0da39c84fc9aa4929b0809dc7cf3c
+wireless-drivers-next	76606886c9f433bb46ed9c8910cabce780b23f36
+bluetooth	1cd2fabf4bdcf95eda6a1bcebc4a0a965509da36
+mac80211-next	f03329627b46c28f0682040e7ddbfd346b78bd4e
+gfs2		9a38662ba4e2682f3f3e9f3ce02a243b837aa8c6
+mtd		34c819073c8b8b652ced18e53bd2b0332dbe29cf
+nand		782d1967d0479ffd59412b2f3179c8bb35f50ff6
+spi-nor		771ff17e82fac54686f149e311d4ea74a70255b1
+crypto		b268b3506d9910ca8238e92cb1dc51340574b2f2
+drm		568cf2e6aa0c762f14d2d0d481a006f93c63ab7a
+drm-panel	e4bac408b08437d190785ab1250cc676304ed171
+drm-intel	315852b422972e6ebb1dfddaadada09e46a2681a
+drm-tegra	6134534ca24f42043cacdd7108026803577f6c59
+drm-misc	fbecef131676c1d18e8e6b42c04e10dc49725e96
+drm-msm		64cb96587ba652878019c02ee5f6c9a341466c52
+hdlcd		f73e8b82531573a198a4d0e5bff0d3256cbbd1d8
+mali-dp		6e810eb508f4b937bc2a718bd4e5cd74cca55500
+sunxi-drm	7dafb83edd3271917458de0ae70586ac63098e2f
+imx-drm		d34515f6779cd89888bfe66762637e626d53c0f6
+etnaviv		f6ffbd4fc1a1caafe2ab840993b917fba5324598
+kconfig		bebc6082da0a9f5d47a1ea2edc099bf671058bd4
+regmap		16c66b8804fa4ec431486002a16ca68b5cdc719b
+sound		e46dcbb17d790008a12b1c18f6235d03d1dd635b
+sound-asoc	64ed10ff091fbbe85692a5b597c34c2f7d3129de
+modules		c554b89868015d86cd330d9cc10656c3756352a5
+input		22a844b853684a9f6088ede15114f8b286b49560
+block		04c4950d5b373ba712d928592e05e73510785bca
+lightnvm	1c6286f26301ad154a1c10fb6e68925493dd28ba
+device-mapper	1ea5c37566939dd6e208b9d092e067b324608260
+pcmcia		c3a5307873d4a47efd4373f4a50f450a17b5d9c1
+mmc		af6b8ff4bce4ea43e89236339f84fc0a26044c42
+kgdb		2cf2f0d5b91fd1b06a6ae260462fc7945ea84add
+md		5a409b4f56d50b212334f338cb8465d65550cd85
+mfd		d28b62520830b2d0bffa2d98e81afc9f5e537e8b
+backlight	cd4270258cbe9a427a6c1216e502bf18bdfc6b29
+battery		1f140ff467081212562327da6561edb6b3b3ed10
+regulator	16bf416e6d330f5ae9213552377508044299e9aa
+security	d19289133b1d3d7b11ab52216ea4ac4a7c86942b
+apparmor	430837708c219dfb25d148808078f15add2ea71c
+integrity	6f0911a666d1f99ff72e7848ddee36af7bbce050
+keys		1e684d3820d8f72d877c0adb521ae17c6fc9bc88
+selinux		d141136f523a3a6372d22981bdff7a8906f36fea
+tpmdd		3ab2011ea368ec3433ad49e1b9e1c7b70d2e65df
+watchdog	1c85ffc86a2b7b687b110b1427c7e701a453fc54
+iommu		1f568357117cafb48aafb10413cbd1567dc015b1
+dwmw2-iommu	d8a5b80568a9cb66810e75b182018e9edb68e8ff
+vfio		a823b3a1b88b7233bace3eeafdbcbd76a58a65dc
+trivial		75a24b822d383489badd343cab4cb171d09f93ef
+audit		5b71388663c0920848c0ee7de946970a2692b76d
+devicetree	399d9e0695775e17a6f100bd5f89a1ec6466e6a1
+mailbox		dfbc9c5841fc455f9b284319fdc01cb0ca5e6b2e
+spi		953083ffb6f294ebe48a2ded164aa53eb0bae877
+tip		760c3aa700294fdc6f0c5184e7e3530b1322d62e
+clockevents	6bd9549d8a7dd06087a06fa7a6df651b02d6f21e
+edac		345fb0a9a6340ba0b3b714d62f877fde88c8a755
+edac-amd	eaa3a1d46cfdbf1af50311e7a22f5d38c0418b56
+irqchip		868c4e07533c67f7b6525a4561d8f59dbfdd5e06
+ftrace		0c5a9acc8b4e878e761f735e144d4a7e4477d4e6
+rcu		09ad34c3289531b95e3d779a4c2cf1fc2be35301
+kvm		0ea3286e2df74c9ec2fadbf91170cd3edd14e3e5
+kvm-arm		e25028c8ded011d19f9a11164807507c94febc01
+kvm-mips	dc44abd6aad22411f7f9890e39fd4753dabf0d03
+kvm-ppc		acc9eb9305fecd958e2877c4e6cd3284d01c2e82
+kvms390		2c8180e885c1b2844a24dcaf4a675972b8ce8edc
+xen-tip		c78ad16670364834b52bfe435c61db48886572c8
+percpu		b3a5d111994450909158929560906f2c1c6c1d85
+workqueues	197f6accacdaf9a0cf4da3c4ac8dd788633c0e38
+drivers-x86	8b0695841da8a66e26ca9e7656c238d4c91d48d2
+chrome-platform	79a3d60300fc28def9b58dc30d86274b47052422
+hsi		70d52ba9e57eb93cfe916f7f4d14a09750890ffd
+leds		6d71021ab3b0bd6ff98270343353b03dfae5d550
+ipmi		048f7c3e352eeef50ed2c14dd89683f8a3af2f9b
+driver-core	0dda2bb6242361afd68332bf19bd67cd5981eb26
+usb		2c093cdb0a5e62d3303a759f4330e62b3e2f82ed
+usb-gadget	47265c067c0d129f3a0e94bc221293a780af9d78
+usb-serial	7041d9c3f01b365daa50340a5d2dce84a09ea813
+usb-chipidea-next	8bcf4c0bd7fbff64ec3fc70b5ecb0d1285e71eca
+phy-next	c1eb8f83845b6ca9e53c41c26a4dff57597d4969
+tty		4b4ecd9cb853c14913a3726cfcc60ccda1d2924a
+char-misc	d6edc07cca93b8669512501f25d03187ca531f63
+extcon		60cc43fc888428bb2f18f08997432d426a243338
+staging		7a5abc3d9699c88781ae12cf957a8c49f9bbb494
+mux		6a3d4baac84a624d7f4ae602254d022fea1eb5f2
+slave-dma	cece9f043feb7f608f23e0423eca6895ea7bfe3a
+cgroup		d8742e22902186e30c346b1ba881cb52942ae3e4
+scsi		7ce1b53b1b5d5326389eb04b906cd63a25eaaa37
+scsi-mkp	1b5c2cb196684f1418fe82257a1b0a8cb0aabc9d
+target-updates	1c130ae00b769a2e2df41bad3d6051ee8234b636
+target-bva	60cc43fc888428bb2f18f08997432d426a243338
+libata		1b1a8f3d0161a3b55d00a11f2917ad069b640d5d
+vhost		cfcde40e8ce3aab6f61b4998945ab19985ea852c
+rpmsg		454575aba6e49d7027ee1db78adac94e7b3b33a6
+gpio		84bf021ecee4e50b63ab5e5563a5202e69cb5623
+pinctrl		6c1b317ff0d82dcf937f9f3832faf2e94dc1056b
+pinctrl-samsung	7c24e71c8a6423dfa5a7f19b253726b23af0c507
+pwm		692099cdcf275df272936b31c8409f2a5a1f7239
+userns		4b245b90065bee3231ca41a8c7f017f55a212656
+ktest		6cd110a91f52197e3392809cd43466bfe2c524d0
+random		9e66317d3c92ddaab330c125dfe9d06eee268aff
+aio		2a8a98673c13cb2a61a6476153acf8344adfa992
+kselftest	fa32156921daa5c175228e2cac7679d50efd6c52
+y2038		5dc0b1529d21b54aad4098874e334a52027fd16d
+livepatching	d7ab8e67aa52e27837aefdc0d67af93ae0931abc
+coresight	ec1df1673899a60a1447b7ce23eae1d424ef4438
+rtc		b22e568f658f0242614e7483f714a889f6a5600d
+nvdimm		1dcdf20fcf9c714e9fd4701257a8598e48e338c8
+at24		73b0d922cd5be4219eb5528a961714d740291b0e
+ntb		5e083738f65d0fb292dcc737a195b532925c58bc
+kspp		45888b40d2a6221d46bb69959e2600ddba71cc1f
+init_task	e1e871aff3ded26348c631b1370e257d401cd22d
+cisco		5997aab0a11ea27ee8e520ecc551ed18fd3e8296
+akpm-current	06a5af82ba137c5b451e0cc017a4cbe9a2c6f72b
+akpm		7c2656ad4cbd3ff08a80f7c1224f168a6c85cfb6
diff --git a/Next/Trees b/Next/Trees
new file mode 100644
index 000000000000..82ddd0b1f111
--- /dev/null
+++ b/Next/Trees
@@ -0,0 +1,282 @@
+Trees included into this release:
+
+Name		Type	URL
+----		----	---
+origin		git	git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git#master
+fixes		git	git://git.kernel.org/pub/scm/linux/kernel/git/sfr/next-fixes.git#master
+kbuild-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-kbuild.git#fixes
+arc-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git#for-curr
+arm-current	git	git://git.armlinux.org.uk/~rmk/linux-arm.git#fixes
+arm64-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/fixes
+m68k-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-linus
+powerpc-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#fixes
+sparc		git	git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc.git#master
+fscrypt-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/tytso/fscrypt.git#for-stable
+net		git	git://git.kernel.org/pub/scm/linux/kernel/git/davem/net.git#master
+bpf		git	git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git#master
+ipsec		git	git://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git#master
+netfilter	git	git://git.kernel.org/pub/scm/linux/kernel/git/pablo/nf.git#master
+ipvs		git	git://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git#master
+wireless-drivers	git	git://git.kernel.org/pub/scm/linux/kernel/git/kvalo/wireless-drivers.git#master
+mac80211		git	git://git.kernel.org/pub/scm/linux/kernel/git/jberg/mac80211.git#master
+rdma-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-rc
+sound-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-linus
+sound-asoc-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-linus
+regmap-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-linus
+regulator-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-linus
+spi-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-linus
+pci-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci.git#for-linus
+driver-core.current	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git#driver-core-linus
+tty.current	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-linus
+usb.current	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-linus
+usb-gadget-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/balbi/usb.git#fixes
+usb-serial-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-linus
+usb-chipidea-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/usb.git#ci-for-usb-stable
+phy		git	git://git.kernel.org/pub/scm/linux/kernel/git/kishon/linux-phy.git#fixes
+staging.current	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-linus
+char-misc.current	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-linus
+input-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#for-linus
+crypto-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git#master
+ide		git	git://git.kernel.org/pub/scm/linux/kernel/git/davem/ide.git#master
+vfio-fixes	git	git://github.com/awilliam/linux-vfio.git#for-linus
+kselftest-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#fixes
+modules-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/jeyu/linux.git#modules-linus
+slave-dma-fixes	git	git://git.infradead.org/users/vkoul/slave-dma.git#fixes
+backlight-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-fixes
+mtd-fixes	git	git://git.infradead.org/linux-mtd.git#master
+nand-fixes	git	git://git.infradead.org/linux-mtd.git#nand/fixes
+spi-nor-fixes	git	git://git.infradead.org/linux-mtd.git#spi-nor/fixes
+mfd-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-fixes
+v4l-dvb-fixes	git	git://linuxtv.org/mchehab/media-next.git#fixes
+reset-fixes	git	https://git.pengutronix.de/git/pza/linux#reset/fixes
+mips-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/jhogan/mips.git#mips-fixes
+omap-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap.git#fixes
+kvm-fixes	git	git://git.kernel.org/pub/scm/virt/kvm/kvm.git#master
+kvms390-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#master
+hwmon-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon
+nvdimm-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-fixes
+btrfs-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#next-fixes
+vfs-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#fixes
+dma-mapping-fixes	git	git://git.infradead.org/users/hch/dma-mapping.git#for-linus
+drivers-x86-fixes	git	git://git.infradead.org/linux-platform-drivers-x86.git#fixes
+samsung-krzk-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#fixes
+pinctrl-samsung-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#pinctrl-fixes
+devicetree-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#dt/linus
+scsi-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#fixes
+drm-intel-fixes	git	git://anongit.freedesktop.org/drm-intel#for-linux-next-fixes
+mmc-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#fixes
+rtc-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-fixes
+drm-misc-fixes	git	git://anongit.freedesktop.org/drm/drm-misc#for-linux-next-fixes
+kbuild		git	git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-kbuild.git#for-next
+leaks		git	https://git.kernel.org/pub/scm/linux/kernel/git/tobin/leaks.git#leaks-next
+uuid		git	git://git.infradead.org/users/hch/uuid.git#for-next
+dma-mapping	git	git://git.infradead.org/users/hch/dma-mapping.git#for-next
+asm-generic	git	git://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic.git#master
+arc		git	git://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git#for-next
+arm		git	git://git.armlinux.org.uk/~rmk/linux-arm.git#for-next
+arm-perf		git	git://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git#for-next/perf
+arm-soc		git	git://git.kernel.org/pub/scm/linux/kernel/git/arm/arm-soc.git#for-next
+actions		git	git://git.kernel.org/pub/scm/linux/kernel/git/afaerber/linux-actions.git#for-next
+alpine		git	git://git.kernel.org/pub/scm/linux/kernel/git/atenart/linux.git#alpine/for-next
+amlogic		git	git://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-amlogic.git#for-next
+aspeed		git	git://git.kernel.org/pub/scm/linux/kernel/git/joel/aspeed.git#for-next
+at91		git	git://git.kernel.org/pub/scm/linux/kernel/git/nferre/linux-at91.git#at91-next
+bcm2835		git	git://github.com/anholt/linux.git#for-next
+imx-mxs		git	git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux.git#for-next
+keystone		git	git://git.kernel.org/pub/scm/linux/kernel/git/ssantosh/linux-keystone.git#next
+mediatek		git	git://git.kernel.org/pub/scm/linux/kernel/git/matthias.bgg/linux.git#for-next
+mvebu		git	git://git.infradead.org/linux-mvebu.git#for-next
+omap		git	git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap.git#for-next
+reset		git	https://git.pengutronix.de/git/pza/linux#reset/next
+qcom		git	git://git.kernel.org/pub/scm/linux/kernel/git/agross/linux.git#for-next
+realtek		git	git://git.kernel.org/pub/scm/linux/kernel/git/afaerber/linux-realtek.git#for-next
+renesas		git	git://git.kernel.org/pub/scm/linux/kernel/git/horms/renesas.git#next
+rockchip		git	git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git#for-next
+samsung		git	git://git.kernel.org/pub/scm/linux/kernel/git/kgene/linux-samsung.git#for-next
+samsung-krzk	git	git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#for-next
+sunxi		git	git://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git#sunxi/for-next
+tegra		git	git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git#for-next
+arm64		git	git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/core
+clk		git	git://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-next
+clk-samsung	git	git://git.kernel.org/pub/scm/linux/kernel/git/snawrocki/clk.git#for-next
+c6x		git	git://linux-c6x.org/git/projects/linux-c6x-upstreaming.git#for-linux-next
+m68k		git	git://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-next
+m68knommu	git	git://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git#for-next
+microblaze	git	git://git.monstr.eu/linux-2.6-microblaze.git#next
+mips		git	git://git.linux-mips.org/pub/scm/ralf/upstream-sfr#mips-next
+mips-james	git	git://git.kernel.org/pub/scm/linux/kernel/git/jhogan/mips.git#mips-next
+nds32		git	git://git.kernel.org/pub/scm/linux/kernel/git/greentime/linux.git#next
+nios2		git	git://git.kernel.org/pub/scm/linux/kernel/git/lftan/nios2.git#for-next
+openrisc		git	git://github.com/openrisc/linux.git#for-next
+parisc-hd	git	git://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git#for-next
+powerpc		git	git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#next
+powerpc-pkey	git	git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#topic/pkey
+fsl		git	git://git.kernel.org/pub/scm/linux/kernel/git/scottwood/linux.git#next
+risc-v		git	git://git.kernel.org/pub/scm/linux/kernel/git/palmer/riscv-linux.git#for-next
+s390		git	git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#features
+sparc-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc-next.git#master
+sh		git	git://git.libc.org/linux-sh#for-next
+uml		git	git://git.kernel.org/pub/scm/linux/kernel/git/rw/uml.git#linux-next
+xtensa		git	git://github.com/jcmvbkbc/linux-xtensa.git#xtensa-for-next
+fscrypt		git	git://git.kernel.org/pub/scm/linux/kernel/git/tytso/fscrypt.git#master
+befs		git	git://git.kernel.org/pub/scm/linux/kernel/git/luisbg/linux-befs.git#for-next
+btrfs		git	git://git.kernel.org/pub/scm/linux/kernel/git/mason/linux-btrfs.git#next
+btrfs-kdave	git	git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#for-next
+ceph		git	git://github.com/ceph/ceph-client.git#master
+cifs		git	git://git.samba.org/sfrench/cifs-2.6.git#for-next
+configfs		git	git://git.infradead.org/users/hch/configfs.git#for-next
+ecryptfs		git	git://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git#next
+ext3		git	git://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git#for_next
+ext4		git	git://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#dev
+f2fs		git	git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git#dev
+fuse		git	git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-next
+jfs		git	git://github.com/kleikamp/linux-shaggy.git#jfs-next
+nfs		git	git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git#linux-next
+nfs-anna		git	git://git.linux-nfs.org/projects/anna/linux-nfs.git#linux-next
+nfsd		git	git://git.linux-nfs.org/~bfields/linux.git#nfsd-next
+orangefs		git	git://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux#for-next
+overlayfs	git	git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/vfs.git#overlayfs-next
+ubifs		git	git://git.infradead.org/ubifs-2.6.git#linux-next
+xfs		git	git://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git#for-next
+file-locks	git	git://git.kernel.org/pub/scm/linux/kernel/git/jlayton/linux.git#locks-next
+vfs		git	git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#for-next
+vfs-miklos	git	git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/vfs.git#next
+printk		git	git://git.kernel.org/pub/scm/linux/kernel/git/pmladek/printk.git#for-next
+pci		git	git://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci.git#next
+pstore		git	git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/pstore
+hid		git	git://git.kernel.org/pub/scm/linux/kernel/git/jikos/hid.git#for-next
+i2c		git	git://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git#i2c/for-next
+dmi		quilt	http://jdelvare.nerim.net/devel/linux/jdelvare-dmi/
+hwmon-staging	git	git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon-next
+jc_docs		git	git://git.lwn.net/linux.git#docs-next
+v4l-dvb		git	git://linuxtv.org/media_tree.git#master
+v4l-dvb-next	git	git://linuxtv.org/mchehab/media-next.git#master
+fbdev		git	git://github.com/bzolnier/linux.git#fbdev-for-next
+pm		git	git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git#linux-next
+cpupower		git	git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git#cpupower
+idle		git	git://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git#next
+opp		git	git://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#opp/linux-next
+thermal		git	git://git.kernel.org/pub/scm/linux/kernel/git/rzhang/linux.git#next
+thermal-soc	git	git://git.kernel.org/pub/scm/linux/kernel/git/evalenti/linux-soc-thermal.git#next
+ieee1394		git	git://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git#for-next
+dlm		git	git://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git#next
+swiotlb		git	git://git.kernel.org/pub/scm/linux/kernel/git/konrad/swiotlb.git#linux-next
+rdma		git	git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-next
+net-next		git	git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next.git#master
+bpf-next		git	git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git#master
+ipsec-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git#master
+netfilter-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/pablo/nf-next.git#master
+nfc-next		git	git://git.kernel.org/pub/scm/linux/kernel/git/sameo/nfc-next.git#master
+ipvs-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git#master
+wireless-drivers-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/kvalo/wireless-drivers-next.git#master
+bluetooth	git	git://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git#master
+mac80211-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/jberg/mac80211-next.git#master
+gfs2		git	git://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git#for-next
+mtd		git	git://git.infradead.org/linux-mtd.git#mtd/next
+nand		git	git://git.infradead.org/linux-mtd.git#nand/next
+spi-nor		git	git://git.infradead.org/linux-mtd.git#spi-nor/next
+crypto		git	git://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git#master
+drm		git	git://git.freedesktop.org/git/drm/drm.git#drm-next
+drm-panel	git	git://anongit.freedesktop.org/tegra/linux.git#drm/panel/for-next
+drm-intel	git	git://anongit.freedesktop.org/drm-intel#for-linux-next
+drm-tegra	git	git://anongit.freedesktop.org/tegra/linux.git#drm/tegra/for-next
+drm-misc		git	git://anongit.freedesktop.org/drm/drm-misc#for-linux-next
+drm-msm		git	git://people.freedesktop.org/~robclark/linux#msm-next
+hdlcd		git	git://linux-arm.org/linux-ld#for-upstream/hdlcd
+mali-dp		git	git://linux-arm.org/linux-ld#for-upstream/mali-dp
+sunxi-drm	git	git://git.kernel.org/pub/scm/linux/kernel/git/mripard/linux.git#sunxi-drm/for-next
+imx-drm		git	https://git.pengutronix.de/git/pza/linux#imx-drm/next
+etnaviv		git	https://git.pengutronix.de/git/lst/linux#etnaviv/next
+kconfig		git	git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-kconfig.git#for-next
+regmap		git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-next
+sound		git	git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-next
+sound-asoc	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-next
+modules		git	git://git.kernel.org/pub/scm/linux/kernel/git/jeyu/linux.git#modules-next
+input		git	git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#next
+block		git	git://git.kernel.dk/linux-block.git#for-next
+lightnvm		git	git://github.com/OpenChannelSSD/linux.git#for-next
+device-mapper	git	git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git#for-next
+pcmcia		git	git://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git#pcmcia-next
+mmc		git	git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#next
+kgdb		git	git://git.kernel.org/pub/scm/linux/kernel/git/jwessel/kgdb.git#kgdb-next
+md		git	git://git.kernel.org/pub/scm/linux/kernel/git/shli/md.git#for-next
+mfd		git	git://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-next
+backlight	git	git://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-next
+battery		git	git://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#for-next
+regulator	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-next
+security		git	git://git.kernel.org/pub/scm/linux/kernel/git/jmorris/linux-security.git#next-testing
+apparmor		git	git://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor#apparmor-next
+integrity	git	git://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity#next-integrity
+keys		git	git://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git#keys-next
+selinux		git	git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git#next
+tpmdd		git	git://git.infradead.org/users/jjs/linux-tpmdd.git#next
+watchdog		git	git://www.linux-watchdog.org/linux-watchdog-next.git#master
+iommu		git	git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git#next
+dwmw2-iommu	git	git://git.infradead.org/iommu-2.6.git#master
+vfio		git	git://github.com/awilliam/linux-vfio.git#next
+trivial		git	git://git.kernel.org/pub/scm/linux/kernel/git/jikos/trivial.git#for-next
+audit		git	git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git#next
+devicetree	git	git://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#for-next
+mailbox		git	git://git.linaro.org/landing-teams/working/fujitsu/integration.git#mailbox-for-next
+spi		git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-next
+tip		git	git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#auto-latest
+clockevents	git	git://git.linaro.org/people/daniel.lezcano/linux.git#clockevents/next
+edac		git	git://git.kernel.org/pub/scm/linux/kernel/git/mchehab/linux-edac.git#linux_next
+edac-amd		git	git://git.kernel.org/pub/scm/linux/kernel/git/bp/bp.git#for-next
+irqchip		git	git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git#irq/irqchip-next
+ftrace		git	git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace.git#for-next
+rcu		git	git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git#rcu/next
+kvm		git	git://git.kernel.org/pub/scm/virt/kvm/kvm.git#linux-next
+kvm-arm		git	git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git#next
+kvm-mips		git	git://git.kernel.org/pub/scm/linux/kernel/git/jhogan/kvm-mips.git#next
+kvm-ppc		git	git://git.kernel.org/pub/scm/linux/kernel/git/paulus/powerpc.git#kvm-ppc-next
+kvms390		git	git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#next
+xen-tip		git	git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git#linux-next
+percpu		git	git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu.git#for-next
+workqueues	git	git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git#for-next
+drivers-x86	git	git://git.infradead.org/linux-platform-drivers-x86.git#for-next
+chrome-platform	git	git://git.kernel.org/pub/scm/linux/kernel/git/bleung/chrome-platform.git#for-next
+hsi		git	git://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git#for-next
+leds		git	git://git.kernel.org/pub/scm/linux/kernel/git/j.anaszewski/linux-leds.git#for-next
+ipmi		git	git://github.com/cminyard/linux-ipmi.git#for-next
+driver-core	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git#driver-core-next
+usb		git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-next
+usb-gadget	git	git://git.kernel.org/pub/scm/linux/kernel/git/balbi/usb.git#next
+usb-serial	git	git://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-next
+usb-chipidea-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/usb.git#ci-for-usb-next
+phy-next		git	git://git.kernel.org/pub/scm/linux/kernel/git/kishon/linux-phy.git#next
+tty		git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-next
+char-misc	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-next
+extcon		git	git://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git#extcon-next
+staging		git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-next
+mux		git	https://gitlab.com/peda-linux/mux.git#for-next
+slave-dma	git	git://git.infradead.org/users/vkoul/slave-dma.git#next
+cgroup		git	git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git#for-next
+scsi		git	git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git#for-next
+scsi-mkp		git	git://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#for-next
+target-updates	git	git://git.kernel.org/pub/scm/linux/kernel/git/nab/target-pending.git#for-next
+target-bva	git	git://git.kernel.org/pub/scm/linux/kernel/git/bvanassche/linux.git#for-next
+libata		git	git://git.kernel.org/pub/scm/linux/kernel/git/tj/libata.git#for-next
+vhost		git	git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git#linux-next
+rpmsg		git	git://github.com/andersson/remoteproc.git#for-next
+gpio		git	git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-gpio.git#for-next
+pinctrl		git	git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git#for-next
+pinctrl-samsung	git	git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#for-next
+pwm		git	git://git.kernel.org/pub/scm/linux/kernel/git/thierry.reding/linux-pwm.git#for-next
+userns		git	git://git.kernel.org/pub/scm/linux/kernel/git/ebiederm/user-namespace.git#for-next
+ktest		git	git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git#for-next
+random		git	git://git.kernel.org/pub/scm/linux/kernel/git/tytso/random.git#dev
+aio		git	git://git.kvack.org/~bcrl/aio-next.git#master
+kselftest	git	git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#next
+y2038		git	git://git.kernel.org/pub/scm/linux/kernel/git/arnd/playground.git#y2038
+livepatching	git	git://git.kernel.org/pub/scm/linux/kernel/git/jikos/livepatching#for-next
+coresight	git	git://git.linaro.org/kernel/coresight.git#next
+rtc		git	git://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-next
+nvdimm		git	git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-for-next
+at24		git	git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#at24/for-next
+ntb		git	https://github.com/jonmason/ntb.git#ntb-next
+kspp		git	git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/kspp
+init_task	git	git://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git#init_task
+cisco		git	https://github.com/daniel-walker/cisco-linux.git#for-next
+akpm-current	mmotm	http://www.ozlabs.org/~akpm/mmotm/
+akpm		mmotm	http://www.ozlabs.org/~akpm/mmotm/
diff --git a/Next/merge.log b/Next/merge.log
new file mode 100644
index 000000000000..f19eefd90cb0
--- /dev/null
+++ b/Next/merge.log
@@ -0,0 +1,15615 @@
+$ date -R
+Thu, 31 May 2018 07:35:35 +1000
+$ git checkout master
+Already on 'master'
+$ git reset --hard stable
+HEAD is now at 3d661e2a2d1c Merge tag 'trace-v4.17-rc4-3' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace
+Merging origin/master (d60d61f36b8f Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux)
+$ git merge origin/master
+Updating 3d661e2a2d1c..d60d61f36b8f
+Fast-forward
+ arch/s390/purgatory/Makefile         |  2 +-
+ drivers/input/mouse/elan_i2c_smbus.c | 22 +++++++++++-----------
+ drivers/input/mouse/synaptics.c      |  6 ++++++
+ drivers/s390/block/dasd.c            |  7 +++++--
+ fs/afs/security.c                    | 10 +++-------
+ fs/afs/vlclient.c                    | 19 ++++++++++---------
+ 6 files changed, 36 insertions(+), 30 deletions(-)
+Merging fixes/master (147a89bc71e7 Merge tag 'kconfig-v4.17' of git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-kbuild)
+$ git merge fixes/master
+Already up to date.
+Merging kbuild-current/fixes (b04e217704b7 Linux 4.17-rc7)
+$ git merge kbuild-current/fixes
+Already up to date.
+Merging arc-current/for-curr (661e50bc8532 Linux 4.16-rc4)
+$ git merge arc-current/for-curr
+Already up to date.
+Merging arm-current/fixes (92d44a42af81 ARM: fix kill( ,SIGFPE) breakage)
+$ git merge arm-current/fixes
+Already up to date.
+Merging arm64-fixes/for-next/fixes (82034c23fcbc arm64: Make sure permission updates happen for pmd/pud)
+$ git merge arm64-fixes/for-next/fixes
+Already up to date.
+Merging m68k-current/for-linus (ecd685580c8f m68k/mac: Remove bogus "FIXME" comment)
+$ git merge m68k-current/for-linus
+Already up to date.
+Merging powerpc-fixes/fixes (faf37c44a105 powerpc/64s: Clear PCR on boot)
+$ git merge powerpc-fixes/fixes
+Already up to date.
+Merging sparc/master (fff75eb2a08c Merge tag 'errseq-v4.17' of git://git.kernel.org/pub/scm/linux/kernel/git/jlayton/linux)
+$ git merge sparc/master
+Already up to date.
+Merging fscrypt-current/for-stable (ae64f9bd1d36 Linux 4.15-rc2)
+$ git merge fscrypt-current/for-stable
+Already up to date.
+Merging net/master (f5a4941aa6d1 vhost_net: flush batched heads before trying to busy polling)
+$ git merge net/master
+Merge made by the 'recursive' strategy.
+ drivers/atm/zatm.c                             |  4 +--
+ drivers/net/ethernet/emulex/benet/be_main.c    |  4 ++-
+ drivers/net/ethernet/mellanox/mlxsw/spectrum.c |  5 ++++
+ drivers/net/ethernet/socionext/netsec.c        |  4 +--
+ drivers/net/tun.c                              | 15 ++++++----
+ drivers/net/usb/qmi_wwan.c                     |  1 +
+ drivers/vhost/net.c                            | 37 ++++++++++++++++---------
+ net/bridge/netfilter/ebtables.c                |  3 +-
+ net/ipv6/seg6_iptunnel.c                       |  4 +--
+ net/netfilter/ipvs/ip_vs_ctl.c                 | 21 ++++++++++----
+ net/netfilter/nf_tables_api.c                  |  8 ++++--
+ net/netfilter/nf_tables_core.c                 |  4 +--
+ net/netfilter/nfnetlink_acct.c                 |  2 +-
+ net/netfilter/nfnetlink_cthelper.c             |  4 +--
+ net/netfilter/nft_ct.c                         | 20 ++++++++------
+ net/netfilter/nft_limit.c                      | 38 ++++++++++++++++----------
+ net/netfilter/nft_meta.c                       | 14 ++++++----
+ 17 files changed, 119 insertions(+), 69 deletions(-)
+Merging bpf/master (c93552c443eb bpf: properly enforce index mask to prevent out-of-bounds speculation)
+$ git merge bpf/master
+Already up to date.
+Merging ipsec/master (d9f92772e8ec xfrm6: avoid potential infinite loop in _decode_session6())
+$ git merge ipsec/master
+Merge made by the 'recursive' strategy.
+ net/ipv6/xfrm6_policy.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging netfilter/master (312564269535 net: netsec: reduce DMA mask to 40 bits)
+$ git merge netfilter/master
+Already up to date.
+Merging ipvs/master (2f99aa31cd7a netfilter: nf_tables: skip synchronize_rcu if transaction log is empty)
+$ git merge ipvs/master
+Already up to date.
+Merging wireless-drivers/master (ab1068d6866e iwlwifi: pcie: compare with number of IRQs requested for, not number of CPUs)
+$ git merge wireless-drivers/master
+Merge made by the 'recursive' strategy.
+ drivers/net/wireless/intel/iwlwifi/pcie/trans.c  | 10 +++++-----
+ drivers/net/wireless/ralink/rt2x00/rt2x00queue.c |  7 +++----
+ 2 files changed, 8 insertions(+), 9 deletions(-)
+Merging mac80211/master (312564269535 net: netsec: reduce DMA mask to 40 bits)
+$ git merge mac80211/master
+Already up to date.
+Merging rdma-fixes/for-rc (533d1daea8d8 IB: Revert "remove redundant INFINIBAND kconfig dependencies")
+$ git merge rdma-fixes/for-rc
+Merge made by the 'recursive' strategy.
+ drivers/infiniband/hw/bnxt_re/main.c       | 55 ++++++++++++++++-
+ drivers/infiniband/hw/bnxt_re/qplib_fp.c   | 94 +++++++++++++++++++-----------
+ drivers/infiniband/hw/bnxt_re/qplib_fp.h   |  3 +
+ drivers/infiniband/hw/bnxt_re/qplib_rcfw.c | 61 +++++++++++++------
+ drivers/infiniband/hw/bnxt_re/qplib_rcfw.h |  3 +
+ drivers/infiniband/ulp/srpt/Kconfig        |  2 +-
+ drivers/nvme/host/Kconfig                  |  2 +-
+ drivers/nvme/target/Kconfig                |  2 +-
+ drivers/staging/lustre/lnet/Kconfig        |  2 +-
+ fs/cifs/Kconfig                            |  2 +-
+ net/9p/Kconfig                             |  2 +-
+ net/rds/Kconfig                            |  2 +-
+ net/sunrpc/Kconfig                         |  2 +-
+ 13 files changed, 171 insertions(+), 61 deletions(-)
+Merging sound-current/for-linus (009f8c90f571 ALSA: hda - Fix runtime PM)
+$ git merge sound-current/for-linus
+Already up to date.
+Merging sound-asoc-fixes/for-linus (3e170d2c7bd4 Merge branch 'asoc-4.17' into asoc-linus)
+$ git merge sound-asoc-fixes/for-linus
+Merge made by the 'recursive' strategy.
+ .mailmap                                           |  3 ++
+ MAINTAINERS                                        | 11 +++--
+ include/sound/soc-dai.h                            |  4 +-
+ sound/soc/cirrus/edb93xx.c                         |  2 +-
+ sound/soc/cirrus/ep93xx-i2s.c                      | 26 ++++++-----
+ sound/soc/cirrus/snappercl15.c                     |  2 +-
+ sound/soc/codecs/cs35l35.c                         |  1 +
+ sound/soc/intel/boards/bytcr_rt5651.c              |  5 ++-
+ .../soc/mediatek/common/mtk-afe-platform-driver.c  |  4 +-
+ sound/soc/omap/n810.c                              | 10 ++---
+ sound/soc/soc-compress.c                           | 52 ++++++++++++----------
+ 11 files changed, 65 insertions(+), 55 deletions(-)
+Merging regmap-fixes/for-linus (4b1b7043a286 Merge branch 'regmap-4.17' into regmap-linus)
+$ git merge regmap-fixes/for-linus
+Merge made by the 'recursive' strategy.
+ drivers/base/regmap/regmap-mmio.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+Merging regulator-fixes/for-linus (16d005d82c84 Merge branch 'regulator-4.17' into regulator-linus)
+$ git merge regulator-fixes/for-linus
+Merge made by the 'recursive' strategy.
+ drivers/regulator/ltc3676.c | 20 +++++++++++++++++++-
+ drivers/regulator/max8998.c |  3 +--
+ 2 files changed, 20 insertions(+), 3 deletions(-)
+Merging spi-fixes/for-linus (2a5c57d435d9 Merge branch 'spi-4.17' into spi-linus)
+$ git merge spi-fixes/for-linus
+Merge made by the 'recursive' strategy.
+ drivers/spi/spi-sh-msiof.c | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+Merging pci-current/for-linus (0cf22d6b317c PCI: Add "PCIe" to pcie_print_link_status() messages)
+$ git merge pci-current/for-linus
+Already up to date.
+Merging driver-core.current/driver-core-linus (6da6c0db5316 Linux v4.17-rc3)
+$ git merge driver-core.current/driver-core-linus
+Already up to date.
+Merging tty.current/tty-linus (6da6c0db5316 Linux v4.17-rc3)
+$ git merge tty.current/tty-linus
+Already up to date.
+Merging usb.current/usb-linus (771c577c23ba Linux 4.17-rc6)
+$ git merge usb.current/usb-linus
+Already up to date.
+Merging usb-gadget-fixes/fixes (6d08b06e67cd Linux 4.17-rc2)
+$ git merge usb-gadget-fixes/fixes
+Already up to date.
+Merging usb-serial-fixes/usb-linus (75bc37fefc44 Linux 4.17-rc4)
+$ git merge usb-serial-fixes/usb-linus
+Already up to date.
+Merging usb-chipidea-fixes/ci-for-usb-stable (964728f9f407 USB: chipidea: msm: fix ulpi-node lookup)
+$ git merge usb-chipidea-fixes/ci-for-usb-stable
+Already up to date.
+Merging phy/fixes (60cc43fc8884 Linux 4.17-rc1)
+$ git merge phy/fixes
+Already up to date.
+Merging staging.current/staging-linus (9d569b1cf7a2 Merge tag 'iio-fixes-for-4.17a' of git://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio into staging-linus)
+$ git merge staging.current/staging-linus
+Merge made by the 'recursive' strategy.
+ drivers/iio/adc/Kconfig                            |  1 +
+ drivers/iio/adc/ad7793.c                           | 75 +++++++---------------
+ drivers/iio/adc/at91-sama5d2_adc.c                 | 41 ++++++++++--
+ drivers/iio/adc/stm32-dfsdm-adc.c                  | 17 ++++-
+ drivers/iio/buffer/industrialio-buffer-dma.c       |  2 +-
+ drivers/iio/buffer/kfifo_buf.c                     | 11 +++-
+ .../iio/common/hid-sensors/hid-sensor-trigger.c    |  8 +--
+ include/linux/iio/buffer_impl.h                    |  6 +-
+ 8 files changed, 93 insertions(+), 68 deletions(-)
+Merging char-misc.current/char-misc-linus (806e30873f0e hwtracing: stm: fix build error on some arches)
+$ git merge char-misc.current/char-misc-linus
+Merge made by the 'recursive' strategy.
+ drivers/hwtracing/intel_th/msu.c | 6 +++---
+ drivers/hwtracing/stm/core.c     | 7 ++++---
+ drivers/thunderbolt/icm.c        | 2 +-
+ 3 files changed, 8 insertions(+), 7 deletions(-)
+Merging input-current/for-linus (40f7090bb1b4 Input: elan_i2c_smbus - fix corrupted stack)
+$ git merge input-current/for-linus
+Already up to date.
+Merging crypto-current/master (bf4407f081f3 crypto: inside-secure - do not use memset on MMIO)
+$ git merge crypto-current/master
+Merge made by the 'recursive' strategy.
+ drivers/crypto/inside-secure/safexcel.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging ide/master (8e44e6600caa Merge branch 'KASAN-read_word_at_a_time')
+$ git merge ide/master
+Already up to date.
+Merging vfio-fixes/for-linus (834814e80268 Revert: "vfio-pci: Mask INTx if a device is not capabable of enabling it")
+$ git merge vfio-fixes/for-linus
+Already up to date.
+Merging kselftest-fixes/fixes (75bc37fefc44 Linux 4.17-rc4)
+$ git merge kselftest-fixes/fixes
+Already up to date.
+Merging modules-fixes/modules-linus (be71eda5383f module: Fix display of wrong module .text address)
+$ git merge modules-fixes/modules-linus
+Already up to date.
+Merging slave-dma-fixes/fixes (dbad41e7bb5f dmaengine: qcom: bam_dma: check if the runtime pm enabled)
+$ git merge slave-dma-fixes/fixes
+Already up to date.
+Merging backlight-fixes/for-backlight-fixes (8ff5cbc6e3c5 backlight: as3711_bl: fix device-tree node leaks)
+$ git merge backlight-fixes/for-backlight-fixes
+Merge made by the 'recursive' strategy.
+ drivers/video/backlight/as3711_bl.c   | 45 ++++++++++++++++++++++-------------
+ drivers/video/backlight/max8925_bl.c  |  4 +++-
+ drivers/video/backlight/tps65217_bl.c |  4 ++--
+ include/linux/mfd/as3711.h            |  4 ++--
+ 4 files changed, 36 insertions(+), 21 deletions(-)
+Merging mtd-fixes/master (9f825e74d761 mtd: rawnand: Fix return type of __DIVIDE() when called with 32-bit)
+$ git merge mtd-fixes/master
+Already up to date.
+Merging nand-fixes/nand/fixes (ee02f73e04c0 mtd: nand: atmel: Fix EDO mode check)
+$ git merge nand-fixes/nand/fixes
+Already up to date.
+Merging spi-nor-fixes/spi-nor/fixes (7928b2cbe55b Linux 4.16-rc1)
+$ git merge spi-nor-fixes/spi-nor/fixes
+Already up to date.
+Merging mfd-fixes/for-mfd-fixes (11799564fc7e mfd: cros_ec: Retry commands when EC is known to be busy)
+$ git merge mfd-fixes/for-mfd-fixes
+Already up to date.
+Merging v4l-dvb-fixes/fixes (b84c8fdf8e6c media: uvcvideo: Prevent setting unavailable flags)
+$ git merge v4l-dvb-fixes/fixes
+Merge made by the 'recursive' strategy.
+ drivers/media/usb/uvc/uvc_ctrl.c | 17 +++++++++--------
+ 1 file changed, 9 insertions(+), 8 deletions(-)
+Merging reset-fixes/reset/fixes (e6914365fd28 reset: uniphier: fix USB clock line for LD20)
+$ git merge reset-fixes/reset/fixes
+Already up to date.
+Merging mips-fixes/mips-fixes (32795631e67e MIPS: lantiq: gphy: Drop reboot/remove reset asserts)
+$ git merge mips-fixes/mips-fixes
+Merge made by the 'recursive' strategy.
+ drivers/soc/lantiq/gphy.c | 36 ------------------------------------
+ 1 file changed, 36 deletions(-)
+Merging omap-fixes/fixes (41bd6adf3c5d Revert "ARM: dts: logicpd-som-lv: Fix pinmux controller references")
+$ git merge omap-fixes/fixes
+Already up to date.
+Merging kvm-fixes/master (696ca779a928 KVM: x86: fix #UD address of failed Hyper-V hypercalls)
+$ git merge kvm-fixes/master
+Already up to date.
+Merging kvms390-fixes/master (f4a551b72358 KVM: s390: vsie: fix < 8k check for the itdba)
+$ git merge kvms390-fixes/master
+Already up to date.
+Merging hwmon-fixes/hwmon (771c577c23ba Linux 4.17-rc6)
+$ git merge hwmon-fixes/hwmon
+Already up to date.
+Merging nvdimm-fixes/libnvdimm-fixes (75bc37fefc44 Linux 4.17-rc4)
+$ git merge nvdimm-fixes/libnvdimm-fixes
+Already up to date.
+Merging btrfs-fixes/next-fixes (e97ce3cc9888 Merge branch 'misc-4.17' into next-fixes)
+$ git merge btrfs-fixes/next-fixes
+Already up to date!
+Merge made by the 'recursive' strategy.
+Merging vfs-fixes/fixes (4faa99965e02 fix io_destroy()/aio_complete() race)
+$ git merge vfs-fixes/fixes
+Merge made by the 'recursive' strategy.
+ fs/aio.c | 3 +--
+ 1 file changed, 1 insertion(+), 2 deletions(-)
+Merging dma-mapping-fixes/for-linus (05e13bb57e6f swiotlb: silent unwanted warning "buffer is full")
+$ git merge dma-mapping-fixes/for-linus
+Already up to date.
+Merging drivers-x86-fixes/fixes (54940fa60ad3 platform/x86: DELL_WMI use depends on instead of select for DELL_SMBIOS)
+$ git merge drivers-x86-fixes/fixes
+Already up to date.
+Merging samsung-krzk-fixes/fixes (60cc43fc8884 Linux 4.17-rc1)
+$ git merge samsung-krzk-fixes/fixes
+Already up to date.
+Merging pinctrl-samsung-fixes/pinctrl-fixes (60cc43fc8884 Linux 4.17-rc1)
+$ git merge pinctrl-samsung-fixes/pinctrl-fixes
+Already up to date.
+Merging devicetree-fixes/dt/linus (f130307054a5 dt-bindings: panel: lvds: Fix path to display timing bindings)
+$ git merge devicetree-fixes/dt/linus
+Already up to date.
+Merging scsi-fixes/fixes (c9ddf73476ff scsi: scsi_transport_srp: Fix shost to rport translation)
+$ git merge scsi-fixes/fixes
+Merge made by the 'recursive' strategy.
+ drivers/scsi/scsi_transport_srp.c | 22 ++++++++++++++++++++--
+ 1 file changed, 20 insertions(+), 2 deletions(-)
+Merging drm-intel-fixes/for-linux-next-fixes (65b3bdc807ac drm/i915/query: nospec expects no more than an unsigned long)
+$ git merge drm-intel-fixes/for-linux-next-fixes
+Merge made by the 'recursive' strategy.
+ drivers/gpu/drm/i915/i915_query.c | 15 +++++++++---
+ drivers/gpu/drm/i915/intel_lvds.c | 51 ++++++++++++++++++++++++++++++---------
+ 2 files changed, 51 insertions(+), 15 deletions(-)
+Merging mmc-fixes/fixes (3de06d5a1f05 mmc: sdhci-iproc: add SDHCI_QUIRK2_HOST_OFF_CARD_ON for cygnus)
+$ git merge mmc-fixes/fixes
+Already up to date.
+Merging rtc-fixes/rtc-fixes (60cc43fc8884 Linux 4.17-rc1)
+$ git merge rtc-fixes/rtc-fixes
+Already up to date.
+Merging drm-misc-fixes/for-linux-next-fixes (c32048d9e93a drm/bridge/synopsys: dw-hdmi: fix dw_hdmi_setup_rx_sense)
+$ git merge drm-misc-fixes/for-linux-next-fixes
+Merge made by the 'recursive' strategy.
+ drivers/gpu/drm/bridge/synopsys/dw-hdmi.c | 15 ++++-----------
+ drivers/gpu/drm/drm_dp_helper.c           |  1 +
+ drivers/gpu/drm/meson/meson_dw_hdmi.c     |  2 +-
+ drivers/gpu/drm/omapdrm/dss/sdi.c         |  5 ++++-
+ include/drm/bridge/dw_hdmi.h              |  2 +-
+ 5 files changed, 11 insertions(+), 14 deletions(-)
+Merging kbuild/for-next (57282f7da50c Merge branch 'kconfig' into for-next)
+$ git merge kbuild/for-next
+Removing scripts/kconfig/lxdialog/check-lxdialog.sh
+Removing scripts/kconfig/kxgettext.c
+Removing scripts/kconfig/check.sh
+Removing scripts/kconfig/POTFILES.in
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ Documentation/kbuild/kconfig-language.txt          |  26 +-
+ Documentation/kbuild/kconfig-macro-language.txt    | 242 +++++++++
+ Kconfig                                            |  10 +-
+ MAINTAINERS                                        |   3 +-
+ Makefile                                           | 115 +----
+ arch/Kconfig                                       |  70 +--
+ arch/openrisc/Makefile                             |   1 -
+ arch/parisc/Makefile                               |   2 +-
+ arch/powerpc/Kconfig                               |  19 +-
+ arch/powerpc/Makefile                              |  44 +-
+ arch/powerpc/boot/Makefile                         |  16 +-
+ arch/powerpc/include/asm/module.h                  |   2 +-
+ arch/powerpc/kernel/module_64.c                    |   4 +-
+ arch/powerpc/kernel/trace/ftrace.c                 |   6 +-
+ arch/powerpc/kernel/vdso32/Makefile                |  15 +-
+ arch/powerpc/kernel/vmlinux.lds.S                  |  22 +-
+ arch/powerpc/tools/gcc-check-mprofile-kernel.sh    |  13 +-
+ arch/s390/Makefile                                 |   2 +-
+ arch/sh/Kconfig                                    |   4 +-
+ arch/sparc/Kconfig                                 |   4 +-
+ arch/um/Kconfig.common                             |   4 -
+ arch/x86/Kconfig                                   |  15 +-
+ arch/x86/um/Kconfig                                |  10 +-
+ include/asm-generic/export.h                       |  34 +-
+ include/asm-generic/vmlinux.lds.h                  | 334 ++++++------
+ include/linux/export.h                             |  16 +-
+ init/Kconfig                                       |  67 ++-
+ kernel/gcov/Kconfig                                |  17 +-
+ kernel/gcov/Makefile                               |   2 -
+ lib/Kconfig.debug                                  |  11 +-
+ scripts/Kbuild.include                             | 102 +---
+ scripts/Kconfig.include                            |  30 ++
+ scripts/Makefile.build                             |   9 +-
+ scripts/Makefile.gcc-plugins                       |  91 +---
+ scripts/Makefile.kcov                              |  10 +-
+ scripts/adjust_autoksyms.sh                        |   3 -
+ scripts/basic/fixdep.c                             |   6 +-
+ scripts/checkpatch.pl                              |  10 -
+ scripts/clang-version.sh                           |  18 +-
+ scripts/coccinelle/locks/mini_lock.cocci           |   6 +-
+ scripts/coccinelle/null/deref_null.cocci           |  40 +-
+ scripts/depmod.sh                                  |  25 +-
+ scripts/gcc-plugins/Makefile                       |   1 +
+ scripts/gcc-x86_32-has-stack-protector.sh          |   7 +-
+ scripts/gcc-x86_64-has-stack-protector.sh          |   5 -
+ scripts/genksyms/genksyms.c                        |  11 +-
+ scripts/kallsyms.c                                 |  49 +-
+ scripts/kconfig/.gitignore                         |   4 -
+ scripts/kconfig/Makefile                           | 190 ++-----
+ scripts/kconfig/POTFILES.in                        |  12 -
+ scripts/kconfig/check.sh                           |  14 -
+ scripts/kconfig/conf.c                             |  51 +-
+ scripts/kconfig/confdata.c                         |  37 +-
+ scripts/kconfig/gconf-cfg.sh                       |  23 +
+ scripts/kconfig/gconf.c                            |  46 +-
+ scripts/kconfig/kconf_id.c                         |   1 -
+ scripts/kconfig/kxgettext.c                        | 235 ---------
+ scripts/kconfig/lkc.h                              |  19 +-
+ scripts/kconfig/lkc_proto.h                        |  15 +-
+ scripts/kconfig/lxdialog/check-lxdialog.sh         |  93 ----
+ scripts/kconfig/lxdialog/checklist.c               |   4 +-
+ scripts/kconfig/lxdialog/dialog.h                  |   8 +-
+ scripts/kconfig/lxdialog/inputbox.c                |   4 +-
+ scripts/kconfig/lxdialog/menubox.c                 |  10 +-
+ scripts/kconfig/lxdialog/textbox.c                 |   2 +-
+ scripts/kconfig/lxdialog/yesno.c                   |   4 +-
+ scripts/kconfig/mconf-cfg.sh                       |  44 ++
+ scripts/kconfig/mconf.c                            | 141 +++--
+ scripts/kconfig/menu.c                             |  23 +-
+ scripts/kconfig/nconf-cfg.sh                       |  44 ++
+ scripts/kconfig/nconf.c                            | 148 +++---
+ scripts/kconfig/nconf.h                            |   1 -
+ scripts/kconfig/preprocess.c                       | 572 +++++++++++++++++++++
+ scripts/kconfig/qconf-cfg.sh                       |  25 +
+ scripts/kconfig/qconf.cc                           | 104 ++--
+ scripts/kconfig/symbol.c                           | 109 ----
+ .../tests/no_write_if_dep_unmet/expected_config    |   2 +-
+ .../kconfig/tests/preprocess/builtin_func/Kconfig  |  27 +
+ .../tests/preprocess/builtin_func/__init__.py      |   9 +
+ .../tests/preprocess/builtin_func/expected_stderr  |   5 +
+ .../tests/preprocess/builtin_func/expected_stdout  |   1 +
+ .../tests/preprocess/circular_expansion/Kconfig    |   5 +
+ .../preprocess/circular_expansion/__init__.py      |  11 +
+ .../preprocess/circular_expansion/expected_stderr  |   1 +
+ scripts/kconfig/tests/preprocess/escape/Kconfig    |  44 ++
+ .../kconfig/tests/preprocess/escape/__init__.py    |   8 +
+ .../tests/preprocess/escape/expected_stderr        |  10 +
+ scripts/kconfig/tests/preprocess/variable/Kconfig  |  53 ++
+ .../kconfig/tests/preprocess/variable/__init__.py  |   8 +
+ .../tests/preprocess/variable/expected_stderr      |   9 +
+ scripts/kconfig/util.c                             |  33 +-
+ scripts/kconfig/zconf.l                            |  95 +++-
+ scripts/kconfig/zconf.y                            |  46 +-
+ scripts/link-vmlinux.sh                            |   4 -
+ scripts/mod/modpost.c                              |  97 ++--
+ scripts/package/mkdebian                           |  27 +-
+ scripts/recordmcount.c                             |   2 +-
+ scripts/recordmcount.h                             |   2 +-
+ scripts/recordmcount.pl                            |  18 +-
+ scripts/tags.sh                                    |  11 +-
+ 100 files changed, 2198 insertions(+), 1866 deletions(-)
+ create mode 100644 Documentation/kbuild/kconfig-macro-language.txt
+ create mode 100644 scripts/Kconfig.include
+ delete mode 100644 scripts/kconfig/POTFILES.in
+ delete mode 100755 scripts/kconfig/check.sh
+ create mode 100755 scripts/kconfig/gconf-cfg.sh
+ delete mode 100644 scripts/kconfig/kxgettext.c
+ delete mode 100755 scripts/kconfig/lxdialog/check-lxdialog.sh
+ create mode 100755 scripts/kconfig/mconf-cfg.sh
+ create mode 100644 scripts/kconfig/nconf-cfg.sh
+ create mode 100644 scripts/kconfig/preprocess.c
+ create mode 100755 scripts/kconfig/qconf-cfg.sh
+ create mode 100644 scripts/kconfig/tests/preprocess/builtin_func/Kconfig
+ create mode 100644 scripts/kconfig/tests/preprocess/builtin_func/__init__.py
+ create mode 100644 scripts/kconfig/tests/preprocess/builtin_func/expected_stderr
+ create mode 100644 scripts/kconfig/tests/preprocess/builtin_func/expected_stdout
+ create mode 100644 scripts/kconfig/tests/preprocess/circular_expansion/Kconfig
+ create mode 100644 scripts/kconfig/tests/preprocess/circular_expansion/__init__.py
+ create mode 100644 scripts/kconfig/tests/preprocess/circular_expansion/expected_stderr
+ create mode 100644 scripts/kconfig/tests/preprocess/escape/Kconfig
+ create mode 100644 scripts/kconfig/tests/preprocess/escape/__init__.py
+ create mode 100644 scripts/kconfig/tests/preprocess/escape/expected_stderr
+ create mode 100644 scripts/kconfig/tests/preprocess/variable/Kconfig
+ create mode 100644 scripts/kconfig/tests/preprocess/variable/__init__.py
+ create mode 100644 scripts/kconfig/tests/preprocess/variable/expected_stderr
+Merging leaks/leaks-next (9f84a0f7f1a6 leaking_addresses: check if file name contains address)
+$ git merge leaks/leaks-next
+Merge made by the 'recursive' strategy.
+Merging uuid/for-next (c0020756315e efi: switch to use new generic UUID API)
+$ git merge uuid/for-next
+Already up to date.
+Merging dma-mapping/for-next (267d2e18a281 nds32: use generic dma_noncoherent_ops)
+$ git merge dma-mapping/for-next
+Auto-merging net/core/dev.c
+Auto-merging mm/Kconfig
+Auto-merging lib/swiotlb.c
+Auto-merging lib/Kconfig.debug
+Auto-merging drivers/parisc/ccio-dma.c
+Auto-merging arch/x86/Kconfig
+Auto-merging arch/sparc/kernel/iommu-common.c
+Removing arch/sparc/kernel/dma.c
+Auto-merging arch/sparc/Kconfig
+Auto-merging arch/sh/mm/consistent.c
+Auto-merging arch/sh/Kconfig
+Auto-merging arch/powerpc/Kconfig
+Removing arch/nds32/include/asm/dma-mapping.h
+Auto-merging arch/nds32/include/asm/Kbuild
+Auto-merging arch/nds32/Kconfig
+CONFLICT (content): Merge conflict in arch/nds32/Kconfig
+Removing arch/c6x/kernel/dma.c
+Removing arch/c6x/include/asm/dma-mapping.h
+Auto-merging arch/arm/mm/dma-mapping.c
+Removing arch/arc/include/asm/dma-mapping.h
+Auto-merging arch/alpha/Kconfig
+Auto-merging arch/Kconfig
+Auto-merging MAINTAINERS
+Removing Documentation/features/io/dma-api-debug/arch-support.txt
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Resolved 'arch/nds32/Kconfig' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master 74302fb0dad3] Merge remote-tracking branch 'dma-mapping/for-next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/admin-guide/kernel-parameters.txt    |   1 -
+ .../features/io/dma-api-debug/arch-support.txt     |  31 ----
+ Documentation/x86/x86_64/boot-options.txt          |  13 +-
+ MAINTAINERS                                        |   2 +
+ arch/Kconfig                                       |   3 -
+ arch/alpha/Kconfig                                 |  14 +-
+ arch/alpha/include/asm/pci.h                       |   5 -
+ arch/arc/Kconfig                                   |  11 +-
+ arch/arc/include/asm/Kbuild                        |   1 +
+ arch/arc/include/asm/dma-mapping.h                 |  21 ---
+ arch/arc/include/asm/pci.h                         |   6 -
+ arch/arc/mm/dma.c                                  | 162 ++---------------
+ arch/arm/Kconfig                                   |  15 +-
+ arch/arm/include/asm/pci.h                         |   7 -
+ arch/arm/kernel/setup.c                            |   2 +-
+ arch/arm/mach-axxia/Kconfig                        |   1 -
+ arch/arm/mach-bcm/Kconfig                          |   1 -
+ arch/arm/mach-exynos/Kconfig                       |   1 -
+ arch/arm/mach-highbank/Kconfig                     |   1 -
+ arch/arm/mach-rockchip/Kconfig                     |   1 -
+ arch/arm/mach-shmobile/Kconfig                     |   1 -
+ arch/arm/mach-tegra/Kconfig                        |   1 -
+ arch/arm/mm/Kconfig                                |   7 +-
+ arch/arm/mm/dma-mapping-nommu.c                    |   9 -
+ arch/arm/mm/dma-mapping.c                          |   9 -
+ arch/arm64/Kconfig                                 |  22 +--
+ arch/arm64/include/asm/pci.h                       |   5 -
+ arch/arm64/mm/dma-mapping.c                        |  10 --
+ arch/c6x/Kconfig                                   |   4 +-
+ arch/c6x/include/asm/Kbuild                        |   1 +
+ arch/c6x/include/asm/dma-mapping.h                 |  28 ---
+ arch/c6x/include/asm/setup.h                       |   2 +
+ arch/c6x/kernel/Makefile                           |   2 +-
+ arch/c6x/kernel/dma.c                              | 149 ----------------
+ arch/c6x/mm/dma-coherent.c                         |  40 ++++-
+ arch/h8300/include/asm/pci.h                       |   2 -
+ arch/hexagon/Kconfig                               |   4 +-
+ arch/hexagon/kernel/dma.c                          |   1 -
+ arch/ia64/Kconfig                                  |  23 +--
+ arch/ia64/hp/common/sba_iommu.c                    |   3 -
+ arch/ia64/include/asm/pci.h                        |  17 --
+ arch/ia64/kernel/dma-mapping.c                     |  10 --
+ arch/ia64/kernel/setup.c                           |  12 --
+ arch/ia64/sn/kernel/io_common.c                    |   5 -
+ arch/m68k/include/asm/pci.h                        |   6 -
+ arch/microblaze/Kconfig                            |   1 -
+ arch/microblaze/include/asm/pci.h                  |   6 -
+ arch/microblaze/kernel/dma.c                       |  11 --
+ arch/mips/Kconfig                                  |  22 +--
+ arch/mips/cavium-octeon/Kconfig                    |  12 --
+ arch/mips/include/asm/pci.h                        |   7 -
+ arch/mips/loongson64/Kconfig                       |  15 --
+ arch/mips/mm/dma-default.c                         |  10 --
+ arch/mips/netlogic/Kconfig                         |   6 -
+ arch/nds32/Kconfig                                 |   3 +
+ arch/nds32/include/asm/Kbuild                      |   1 +
+ arch/nds32/include/asm/dma-mapping.h               |  14 --
+ arch/nds32/kernel/dma.c                            | 196 ++++++---------------
+ arch/openrisc/kernel/dma.c                         |  11 --
+ arch/parisc/Kconfig                                |   8 +-
+ arch/parisc/include/asm/pci.h                      |  23 ---
+ arch/parisc/kernel/setup.c                         |   5 -
+ arch/powerpc/Kconfig                               |  25 +--
+ arch/powerpc/include/asm/pci.h                     |  18 --
+ arch/powerpc/kernel/dma.c                          |   3 -
+ arch/powerpc/platforms/Kconfig.cputype             |   1 +
+ arch/riscv/Kconfig                                 |  44 ++---
+ arch/riscv/include/asm/dma-mapping.h               |  15 ++
+ arch/riscv/include/asm/pci.h                       |   3 -
+ arch/riscv/kernel/setup.c                          |   2 +
+ arch/s390/Kconfig                                  |  17 +-
+ arch/s390/include/asm/pci.h                        |   2 -
+ arch/s390/pci/pci_dma.c                            |  11 --
+ arch/sh/Kconfig                                    |  10 +-
+ arch/sh/include/asm/pci.h                          |   6 -
+ arch/sh/kernel/dma-nommu.c                         |   1 -
+ arch/sh/mm/consistent.c                            |   9 -
+ arch/sparc/Kconfig                                 |  18 +-
+ .../sparc/include/asm}/iommu-common.h              |   0
+ arch/sparc/include/asm/iommu_64.h                  |   2 +-
+ arch/sparc/include/asm/pci_32.h                    |   4 -
+ arch/sparc/include/asm/pci_64.h                    |   6 -
+ arch/sparc/kernel/Makefile                         |   4 +-
+ arch/sparc/kernel/dma.c                            |  13 --
+ {lib => arch/sparc/kernel}/iommu-common.c          |   5 +-
+ arch/sparc/kernel/iommu.c                          |   2 +-
+ arch/sparc/kernel/ldc.c                            |   2 +-
+ arch/sparc/kernel/pci_sun4v.c                      |   2 +-
+ arch/unicore32/Kconfig                             |   5 +-
+ arch/unicore32/mm/Kconfig                          |  11 --
+ arch/x86/Kconfig                                   |  36 +---
+ arch/x86/include/asm/dma-mapping.h                 |   5 +-
+ arch/x86/include/asm/pci.h                         |   3 -
+ arch/x86/kernel/pci-dma.c                          |  58 ++----
+ arch/xtensa/Kconfig                                |   1 -
+ arch/xtensa/include/asm/pci.h                      |   2 -
+ arch/xtensa/kernel/pci-dma.c                       |   9 -
+ drivers/amba/bus.c                                 |   5 +-
+ drivers/base/dma-mapping.c                         |  31 +---
+ drivers/base/platform.c                            |  18 +-
+ drivers/bcma/main.c                                |   2 +-
+ drivers/dma/qcom/hidma_mgmt.c                      |   2 +-
+ drivers/gpu/host1x/bus.c                           |   9 +-
+ drivers/ide/ide-dma.c                              |   2 -
+ drivers/ide/ide-lib.c                              |  26 ---
+ drivers/ide/ide-probe.c                            |   6 +-
+ drivers/iommu/Kconfig                              |   1 +
+ drivers/net/ethernet/sfc/efx.c                     |   5 +-
+ drivers/net/ethernet/sfc/falcon/efx.c              |   5 +-
+ drivers/of/device.c                                |   6 +-
+ drivers/of/of_reserved_mem.c                       |   2 +-
+ drivers/parisc/Kconfig                             |   5 -
+ drivers/parisc/ccio-dma.c                          |   2 -
+ drivers/parisc/sba_iommu.c                         |   2 -
+ drivers/pci/Kconfig                                |   4 -
+ drivers/pci/bus.c                                  |   4 +-
+ drivers/pci/pci-driver.c                           |  33 +++-
+ drivers/scsi/scsi_lib.c                            |  24 +--
+ include/asm-generic/dma-mapping.h                  |   9 +
+ include/asm-generic/pci.h                          |   8 -
+ include/linux/device.h                             |  11 +-
+ include/linux/dma-debug.h                          |   6 -
+ include/linux/dma-direct.h                         |   7 +-
+ include/linux/dma-mapping.h                        |  19 +-
+ include/linux/dma-noncoherent.h                    |  47 +++++
+ include/linux/ide.h                                |   2 -
+ include/linux/iommu-helper.h                       |  13 +-
+ include/linux/of_device.h                          |   8 +-
+ include/linux/pci.h                                |   2 +-
+ include/linux/platform_device.h                    |   2 +
+ lib/Kconfig                                        |  43 ++++-
+ lib/Kconfig.debug                                  |  19 +-
+ lib/Makefile                                       |   3 +-
+ lib/dma-debug.c                                    |  65 ++++---
+ lib/dma-direct.c                                   |  22 ++-
+ lib/dma-noncoherent.c                              | 102 +++++++++++
+ lib/iommu-helper.c                                 |  14 +-
+ lib/swiotlb.c                                      |  11 +-
+ mm/Kconfig                                         |   2 +-
+ net/core/dev.c                                     |  20 +--
+ tools/virtio/linux/dma-mapping.h                   |   2 -
+ 141 files changed, 617 insertions(+), 1392 deletions(-)
+ delete mode 100644 Documentation/features/io/dma-api-debug/arch-support.txt
+ delete mode 100644 arch/arc/include/asm/dma-mapping.h
+ delete mode 100644 arch/c6x/include/asm/dma-mapping.h
+ delete mode 100644 arch/c6x/kernel/dma.c
+ delete mode 100644 arch/nds32/include/asm/dma-mapping.h
+ create mode 100644 arch/riscv/include/asm/dma-mapping.h
+ rename {include/linux => arch/sparc/include/asm}/iommu-common.h (100%)
+ delete mode 100644 arch/sparc/kernel/dma.c
+ rename {lib => arch/sparc/kernel}/iommu-common.c (98%)
+ create mode 100644 include/linux/dma-noncoherent.h
+ create mode 100644 lib/dma-noncoherent.c
+Merging asm-generic/master (a71e7c44ffb7 io: change writeX_relaxed() to remove barriers)
+$ git merge asm-generic/master
+Already up to date.
+Merging arc/for-next (4fbd8d194f06 Linux 4.15-rc1)
+$ git merge arc/for-next
+Already up to date.
+Merging arm/for-next (8de97027b2b4 Merge branches 'fixes' and 'misc'; commit 'spectre^{/^ARM: KVM:}' into for-next)
+$ git merge arm/for-next
+Auto-merging drivers/amba/bus.c
+Auto-merging arch/arm/mm/dma-mapping.c
+Auto-merging arch/arm/mm/Kconfig
+Auto-merging arch/arm/include/asm/kvm_mmu.h
+Auto-merging arch/arm/include/asm/kvm_host.h
+Auto-merging arch/arm/Kconfig
+Merge made by the 'recursive' strategy.
+ arch/arm/Kconfig                   |   4 +-
+ arch/arm/Makefile                  |   2 +-
+ arch/arm/boot/compressed/Makefile  |   2 +-
+ arch/arm/include/asm/bugs.h        |   6 +-
+ arch/arm/include/asm/cp15.h        |   3 +
+ arch/arm/include/asm/cputype.h     |   8 +
+ arch/arm/include/asm/kgdb.h        |   2 +-
+ arch/arm/include/asm/kvm_asm.h     |   2 -
+ arch/arm/include/asm/kvm_host.h    |  14 +-
+ arch/arm/include/asm/kvm_mmu.h     |  23 ++-
+ arch/arm/include/asm/mpu.h         | 112 +++++++++-----
+ arch/arm/include/asm/proc-fns.h    |   4 +
+ arch/arm/include/asm/system_misc.h |  15 ++
+ arch/arm/include/asm/uaccess.h     |   2 +-
+ arch/arm/include/asm/v7m.h         |  14 +-
+ arch/arm/kernel/Makefile           |   1 +
+ arch/arm/kernel/asm-offsets.c      |   8 +-
+ arch/arm/kernel/bugs.c             |  18 +++
+ arch/arm/kernel/head-nommu.S       | 289 +++++++++++++++++++++++++++-------
+ arch/arm/kernel/smp.c              |   7 +-
+ arch/arm/kernel/suspend.c          |   2 +
+ arch/arm/kernel/vmlinux-xip.lds.S  |   4 +
+ arch/arm/kernel/vmlinux.lds.S      |   7 +
+ arch/arm/kernel/vmlinux.lds.h      |  16 +-
+ arch/arm/kvm/hyp/hyp-entry.S       | 112 +++++++++++++-
+ arch/arm/mm/Kconfig                |  23 +++
+ arch/arm/mm/Makefile               |   4 +-
+ arch/arm/mm/dma-mapping.c          |   2 +-
+ arch/arm/mm/fault.c                |   3 +
+ arch/arm/mm/nommu.c                |  32 ++++
+ arch/arm/mm/pmsa-v7.c              |  59 +++----
+ arch/arm/mm/pmsa-v8.c              | 307 +++++++++++++++++++++++++++++++++++++
+ arch/arm/mm/proc-macros.S          |   3 +-
+ arch/arm/mm/proc-v7-2level.S       |   6 -
+ arch/arm/mm/proc-v7-bugs.c         | 176 +++++++++++++++++++++
+ arch/arm/mm/proc-v7.S              | 154 +++++++++++++++----
+ drivers/amba/bus.c                 |   1 +
+ 37 files changed, 1248 insertions(+), 199 deletions(-)
+ create mode 100644 arch/arm/kernel/bugs.c
+ create mode 100644 arch/arm/mm/pmsa-v8.c
+ create mode 100644 arch/arm/mm/proc-v7-bugs.c
+Merging arm-perf/for-next/perf (73acc0315cd7 ARM: mcpm, perf/arm-cci: export mcpm_is_available)
+$ git merge arm-perf/for-next/perf
+Merge made by the 'recursive' strategy.
+ arch/arm/common/mcpm_entry.c        |  2 ++
+ arch/arm/kernel/perf_event_v6.c     |  4 +---
+ arch/arm/kernel/perf_event_v7.c     |  3 +--
+ arch/arm/kernel/perf_event_xscale.c |  6 ++---
+ arch/arm64/kernel/perf_event.c      |  3 +--
+ drivers/perf/Kconfig                | 36 ++++++++++++++--------------
+ drivers/perf/arm-cci.c              | 47 ++++++++++++++++++-------------------
+ drivers/perf/arm-ccn.c              | 22 ++++++++---------
+ drivers/perf/arm_pmu.c              |  2 +-
+ drivers/perf/arm_spe_pmu.c          |  6 ++---
+ include/linux/perf/arm_pmu.h        |  2 +-
+ 11 files changed, 64 insertions(+), 69 deletions(-)
+Merging arm-soc/for-next (933d3cf3d416 ARM: Document merges)
+$ git merge arm-soc/for-next
+Removing include/linux/platform_data/media/ir-rx51.h
+Removing drivers/memory/tegra20-mc.c
+Auto-merging arch/arm64/boot/dts/synaptics/berlin4ct.dtsi
+Removing arch/arm64/boot/dts/marvell/berlin4ct-stb.dts
+Removing arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts
+Removing arch/arm/mach-omap2/omap-pm.h
+Removing arch/arm/mach-omap2/omap-pm-noop.c
+Auto-merging arch/arm/mach-exynos/Kconfig
+CONFLICT (content): Merge conflict in arch/arm/mach-exynos/Kconfig
+Auto-merging arch/arm/include/asm/cputype.h
+CONFLICT (content): Merge conflict in arch/arm/include/asm/cputype.h
+Removing arch/arm/boot/dts/ste-ccu9540.dts
+Removing arch/arm/boot/dts/ste-ccu8540.dts
+Removing arch/arm/boot/dts/ste-ccu8540-pinctrl.dtsi
+Removing arch/arm/boot/dts/exynos5440.dtsi
+Removing arch/arm/boot/dts/exynos5440-trip-points.dtsi
+Removing arch/arm/boot/dts/exynos5440-tmu-sensor-conf.dtsi
+Removing arch/arm/boot/dts/exynos5440-ssdk5440.dts
+Removing arch/arm/boot/dts/exynos5440-sd5v1.dts
+Auto-merging MAINTAINERS
+Auto-merging Documentation/devicetree/bindings/memory-controllers/nvidia,tegra20-mc.txt
+Removing Documentation/devicetree/bindings/arm/tegra/nvidia,tegra30-mc.txt
+Resolved 'arch/arm/include/asm/cputype.h' using previous resolution.
+Resolved 'arch/arm/mach-exynos/Kconfig' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master 89d673762c36] Merge remote-tracking branch 'arm-soc/for-next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/arm/OMAP/README                      |    4 +
+ Documentation/devicetree/bindings/arm/amlogic.txt  |    6 +
+ .../devicetree/bindings/arm/bcm/brcm,bcm2835.txt   |    4 +
+ .../bindings/arm/samsung/samsung-boards.txt        |    2 -
+ Documentation/devicetree/bindings/arm/shmobile.txt |   10 +-
+ .../bindings/arm/tegra/nvidia,tegra30-mc.txt       |   18 -
+ Documentation/devicetree/bindings/bus/ti-sysc.txt  |    6 +-
+ .../bindings/clock/amlogic,gxbb-aoclkc.txt         |    1 +
+ .../bindings/clock/amlogic,gxbb-clkc.txt           |   16 +-
+ .../devicetree/bindings/firmware/qcom,scm.txt      |    3 +-
+ .../nvidia,tegra20-mc.txt                          |   12 +-
+ .../memory-controllers/nvidia,tegra30-mc.txt       |    5 +
+ .../devicetree/bindings/mmc/sdhci-omap.txt         |    7 +
+ .../devicetree/bindings/net/qualcomm-bluetooth.txt |   30 +
+ .../bindings/nvmem/allwinner,sunxi-sid.txt         |    1 +
+ .../devicetree/bindings/power/pd-samsung.txt       |   20 +-
+ .../bindings/reserved-memory/qcom,cmd-db.txt       |   37 +
+ .../devicetree/bindings/rng/brcm,bcm2835.txt       |    9 +-
+ .../devicetree/bindings/soc/qcom/qcom,geni-se.txt  |  119 +++
+ .../devicetree/bindings/soc/qcom/qcom,smd-rpm.txt  |    1 +
+ .../bindings/soc/rockchip/power_domain.txt         |   12 +
+ .../devicetree/bindings/timer/renesas,cmt.txt      |   14 +-
+ .../devicetree/bindings/vendor-prefixes.txt        |    2 +
+ MAINTAINERS                                        |    2 +-
+ arch/arm/arm-soc-for-next-contents.txt             |  203 ++++
+ arch/arm/boot/dts/Makefile                         |   40 +-
+ arch/arm/boot/dts/am335x-baltos-ir3220.dts         |    2 +-
+ arch/arm/boot/dts/am335x-baltos-ir5221.dts         |    2 +-
+ arch/arm/boot/dts/am335x-baltos.dtsi               |    2 +-
+ arch/arm/boot/dts/am335x-bone-common.dtsi          |    9 +-
+ arch/arm/boot/dts/am335x-boneblue.dts              |    2 +-
+ arch/arm/boot/dts/am335x-evm.dts                   |    9 +-
+ arch/arm/boot/dts/am335x-evmsk.dts                 |   12 +-
+ arch/arm/boot/dts/am335x-osd335x-common.dtsi       |  124 +++
+ arch/arm/boot/dts/am335x-pocketbeagle.dts          |  237 +++++
+ arch/arm/boot/dts/am3517-evm.dts                   |  233 +++-
+ arch/arm/boot/dts/am3517-som.dtsi                  |  142 +++
+ arch/arm/boot/dts/am437x-cm-t43.dts                |    2 +-
+ arch/arm/boot/dts/am437x-gp-evm.dts                |    2 +-
+ arch/arm/boot/dts/am437x-sk-evm.dts                |  115 ++
+ arch/arm/boot/dts/am43x-epos-evm.dts               |    2 +-
+ arch/arm/boot/dts/am571x-idk.dts                   |    5 +-
+ arch/arm/boot/dts/am572x-idk.dts                   |    5 +-
+ arch/arm/boot/dts/am574x-idk.dts                   |   20 +
+ arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi    |    4 +-
+ arch/arm/boot/dts/am57xx-beagle-x15.dts            |    3 +-
+ arch/arm/boot/dts/am57xx-idk-common.dtsi           |   14 +-
+ arch/arm/boot/dts/armada-370-db.dts                |   57 +-
+ arch/arm/boot/dts/armada-370-dlink-dns327l.dts     |  120 ++-
+ arch/arm/boot/dts/armada-370-mirabox.dts           |   51 +-
+ arch/arm/boot/dts/armada-370-netgear-rn102.dts     |   90 +-
+ arch/arm/boot/dts/armada-370-netgear-rn104.dts     |   90 +-
+ arch/arm/boot/dts/armada-370-rd.dts                |   52 +-
+ arch/arm/boot/dts/armada-370-seagate-nas-xbay.dtsi |   64 +-
+ arch/arm/boot/dts/armada-370-xp.dtsi               |    6 +-
+ arch/arm/boot/dts/armada-375-db.dts                |   50 +-
+ arch/arm/boot/dts/armada-375.dtsi                  |    6 +-
+ arch/arm/boot/dts/armada-385-db-ap.dts             |   69 +-
+ arch/arm/boot/dts/armada-385-linksys-caiman.dts    |  129 +--
+ arch/arm/boot/dts/armada-385-linksys-cobra.dts     |  129 +--
+ arch/arm/boot/dts/armada-385-linksys-rango.dts     |  141 +--
+ arch/arm/boot/dts/armada-385-linksys-shelby.dts    |  129 +--
+ arch/arm/boot/dts/armada-385-linksys.dtsi          |   16 +-
+ arch/arm/boot/dts/armada-388-db.dts                |   55 +-
+ arch/arm/boot/dts/armada-38x.dtsi                  |    6 +-
+ arch/arm/boot/dts/armada-390-db.dts                |   66 +-
+ arch/arm/boot/dts/armada-395-gp.dts                |   74 +-
+ arch/arm/boot/dts/armada-398-db.dts                |   60 +-
+ arch/arm/boot/dts/armada-39x.dtsi                  |    6 +-
+ arch/arm/boot/dts/armada-xp-98dx3236.dtsi          |   13 +
+ arch/arm/boot/dts/armada-xp-db-dxbc2.dts           |    2 +-
+ arch/arm/boot/dts/armada-xp-db-xc3-24g4xg.dts      |    2 +-
+ arch/arm/boot/dts/armada-xp-db.dts                 |    2 +-
+ arch/arm/boot/dts/armada-xp-gp.dts                 |    2 +-
+ arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts    |    2 +-
+ arch/arm/boot/dts/armada-xp-linksys-mamba.dts      |  156 +--
+ arch/arm/boot/dts/armada-xp-netgear-rn2120.dts     |   90 +-
+ arch/arm/boot/dts/aspeed-ast2500-evb.dts           |   18 +
+ arch/arm/boot/dts/aspeed-bmc-intel-s2600wf.dts     |  129 +++
+ arch/arm/boot/dts/aspeed-bmc-opp-lanyang.dts       |  325 ++++++
+ arch/arm/boot/dts/aspeed-bmc-opp-romulus.dts       |    6 +
+ arch/arm/boot/dts/aspeed-bmc-opp-witherspoon.dts   |   32 +
+ arch/arm/boot/dts/aspeed-bmc-opp-zaius.dts         |    6 +
+ arch/arm/boot/dts/aspeed-bmc-portwell-neptune.dts  |  159 +++
+ arch/arm/boot/dts/aspeed-g4.dtsi                   |   35 +
+ arch/arm/boot/dts/aspeed-g5.dtsi                   |   47 +
+ arch/arm/boot/dts/at91-sama5d2_xplained.dts        |    2 +-
+ arch/arm/boot/dts/at91-sama5d4ek.dts               |    4 +-
+ arch/arm/boot/dts/bcm2835-rpi-a-plus.dts           |    6 +
+ arch/arm/boot/dts/bcm2835-rpi-a.dts                |    6 +
+ arch/arm/boot/dts/bcm2835-rpi-b-plus.dts           |    6 +
+ arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts           |    6 +
+ arch/arm/boot/dts/bcm2835-rpi-b.dts                |    6 +
+ arch/arm/boot/dts/bcm2835-rpi.dtsi                 |    6 -
+ arch/arm/boot/dts/bcm2836-rpi-2-b.dts              |    6 +
+ arch/arm/boot/dts/bcm2837-rpi-3-b-plus.dts         |  108 ++
+ arch/arm/boot/dts/bcm2837-rpi-3-b.dts              |   26 +-
+ arch/arm/boot/dts/bcm283x-rpi-lan7515.dtsi         |   27 +
+ arch/arm/boot/dts/bcm283x.dtsi                     |    1 +
+ arch/arm/boot/dts/bcm4708-asus-rt-ac56u.dts        |   13 +-
+ arch/arm/boot/dts/bcm4708-asus-rt-ac68u.dts        |   13 +-
+ arch/arm/boot/dts/bcm4708-buffalo-wzr-1750dhp.dts  |    3 +-
+ arch/arm/boot/dts/bcm4708-linksys-ea6300-v1.dts    |    3 +-
+ arch/arm/boot/dts/bcm4708-luxul-xap-1510.dts       |    3 +-
+ arch/arm/boot/dts/bcm4708-luxul-xwc-1000.dts       |   15 +-
+ arch/arm/boot/dts/bcm4708-netgear-r6300-v2.dts     |   13 +-
+ arch/arm/boot/dts/bcm4708-smartrg-sr400ac.dts      |    3 +-
+ arch/arm/boot/dts/bcm47081-asus-rt-n18u.dts        |   13 +-
+ arch/arm/boot/dts/bcm47081-buffalo-wzr-600dhp2.dts |   13 +-
+ arch/arm/boot/dts/bcm47081-buffalo-wzr-900dhp.dts  |   13 +-
+ arch/arm/boot/dts/bcm47081-luxul-xap-1410.dts      |    3 +-
+ arch/arm/boot/dts/bcm47081-luxul-xwr-1200.dts      |    3 +-
+ arch/arm/boot/dts/bcm47081-tplink-archer-c5-v2.dts |    3 +-
+ arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts        |   13 +-
+ arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts  |    3 +-
+ arch/arm/boot/dts/bcm4709-linksys-ea9200.dts       |    3 +-
+ arch/arm/boot/dts/bcm4709-netgear-r7000.dts        |   13 +-
+ arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts  |    3 +-
+ arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts      |   25 +-
+ arch/arm/boot/dts/bcm47094-linksys-panamera.dts    |    3 +-
+ arch/arm/boot/dts/bcm47094-luxul-abr-4500.dts      |    3 +-
+ arch/arm/boot/dts/bcm47094-luxul-xap-1610.dts      |   57 +
+ arch/arm/boot/dts/bcm47094-luxul-xbr-4500.dts      |    3 +-
+ arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts      |    3 +-
+ arch/arm/boot/dts/bcm47094-luxul-xwr-3150-v1.dts   |   77 ++
+ arch/arm/boot/dts/bcm47094-netgear-r8500.dts       |    3 +-
+ arch/arm/boot/dts/bcm5301x-nand-cs0-bch1.dtsi      |    3 +-
+ arch/arm/boot/dts/bcm5301x-nand-cs0-bch4.dtsi      |    3 +-
+ arch/arm/boot/dts/bcm5301x-nand-cs0-bch8.dtsi      |    3 +-
+ arch/arm/boot/dts/bcm5301x-nand-cs0.dtsi           |    3 +-
+ arch/arm/boot/dts/berlin2-sony-nsz-gs7.dts         |   33 +-
+ arch/arm/boot/dts/berlin2.dtsi                     |   35 +-
+ arch/arm/boot/dts/berlin2cd-google-chromecast.dts  |   65 +-
+ arch/arm/boot/dts/berlin2cd-valve-steamlink.dts    |   79 ++
+ arch/arm/boot/dts/berlin2cd.dtsi                   |  167 ++-
+ arch/arm/boot/dts/berlin2q-marvell-dmp.dts         |   33 +-
+ arch/arm/boot/dts/berlin2q.dtsi                    |   65 +-
+ arch/arm/boot/dts/da850-evm.dts                    |  298 +++---
+ arch/arm/boot/dts/da850-lego-ev3.dts               |   83 +-
+ arch/arm/boot/dts/da850.dtsi                       |  154 +++
+ arch/arm/boot/dts/dm8148-t410.dts                  |    2 +-
+ arch/arm/boot/dts/dra7-evm-common.dtsi             |   15 +
+ arch/arm/boot/dts/dra7-evm.dts                     |   68 ++
+ arch/arm/boot/dts/dra7-mmc-iodelay.dtsi            |   19 +
+ arch/arm/boot/dts/dra7.dtsi                        |   32 +-
+ arch/arm/boot/dts/dra71-evm.dts                    |   17 +-
+ arch/arm/boot/dts/dra72-evm-common.dtsi            |   71 +-
+ arch/arm/boot/dts/dra72x-mmc-iodelay.dtsi          |   11 +
+ arch/arm/boot/dts/dra76-evm.dts                    |   34 +-
+ arch/arm/boot/dts/exynos-syscon-restart.dtsi       |   28 +-
+ arch/arm/boot/dts/exynos3250-rinato.dts            |    4 +
+ arch/arm/boot/dts/exynos3250.dtsi                  |    2 +-
+ arch/arm/boot/dts/exynos4.dtsi                     |    3 +-
+ arch/arm/boot/dts/exynos4210-origen.dts            |   34 +-
+ arch/arm/boot/dts/exynos4210-trats.dts             |    4 +-
+ arch/arm/boot/dts/exynos4412-galaxy-s3.dtsi        |   36 +-
+ arch/arm/boot/dts/exynos4412-midas.dtsi            |   86 +-
+ arch/arm/boot/dts/exynos4412-n710x.dts             |   16 +-
+ arch/arm/boot/dts/exynos4412-odroid-common.dtsi    |   33 +-
+ arch/arm/boot/dts/exynos4412-odroidu3.dts          |    6 +-
+ arch/arm/boot/dts/exynos4412-odroidx.dts           |    6 +-
+ arch/arm/boot/dts/exynos4412-origen.dts            |    2 +-
+ arch/arm/boot/dts/exynos4412.dtsi                  |    2 +-
+ arch/arm/boot/dts/exynos5.dtsi                     |    3 -
+ arch/arm/boot/dts/exynos5250.dtsi                  |    7 +-
+ arch/arm/boot/dts/exynos5410.dtsi                  |    1 +
+ arch/arm/boot/dts/exynos5420-peach-pit.dts         |    4 +-
+ arch/arm/boot/dts/exynos5420.dtsi                  |   87 +-
+ arch/arm/boot/dts/exynos5422-odroid-core.dtsi      |    2 +-
+ arch/arm/boot/dts/exynos5440-sd5v1.dts             |   42 -
+ arch/arm/boot/dts/exynos5440-ssdk5440.dts          |   81 --
+ arch/arm/boot/dts/exynos5440-tmu-sensor-conf.dtsi  |   20 -
+ arch/arm/boot/dts/exynos5440-trip-points.dtsi      |   21 -
+ arch/arm/boot/dts/exynos5440.dtsi                  |  355 -------
+ arch/arm/boot/dts/exynos5800-peach-pi.dts          |    4 +-
+ arch/arm/boot/dts/gemini-dlink-dir-685.dts         |   16 +-
+ arch/arm/boot/dts/gemini-dlink-dns-313.dts         |   19 +-
+ arch/arm/boot/dts/gemini-nas4220b.dts              |   20 +-
+ arch/arm/boot/dts/gemini-rut1xx.dts                |   12 +-
+ arch/arm/boot/dts/gemini-sq201.dts                 |   12 +-
+ arch/arm/boot/dts/gemini-wbd111.dts                |   17 +-
+ arch/arm/boot/dts/gemini-wbd222.dts                |   16 +-
+ arch/arm/boot/dts/gemini.dtsi                      |    2 -
+ arch/arm/boot/dts/imx1-ads.dts                     |   11 -
+ arch/arm/boot/dts/imx1.dtsi                        |   21 +-
+ arch/arm/boot/dts/imx23-evk.dts                    |   13 +-
+ arch/arm/boot/dts/imx23.dtsi                       |   13 +-
+ arch/arm/boot/dts/imx25-pdk.dts                    |   14 +-
+ arch/arm/boot/dts/imx25.dtsi                       |   16 +-
+ arch/arm/boot/dts/imx27-apf27.dts                  |   13 +-
+ arch/arm/boot/dts/imx27-pdk.dts                    |   13 +-
+ arch/arm/boot/dts/imx27.dtsi                       |   18 +-
+ arch/arm/boot/dts/imx28-cfa10049.dts               |    2 -
+ arch/arm/boot/dts/imx28-duckbill-2-enocean.dts     |    2 -
+ arch/arm/boot/dts/imx28-evk.dts                    |   13 +-
+ arch/arm/boot/dts/imx28-tx28.dts                   |   14 +-
+ arch/arm/boot/dts/imx28.dtsi                       |   13 +-
+ arch/arm/boot/dts/imx31.dtsi                       |   13 +-
+ arch/arm/boot/dts/imx35-pdk.dts                    |   15 +-
+ arch/arm/boot/dts/imx35.dtsi                       |   14 +-
+ arch/arm/boot/dts/imx50-evk.dts                    |   17 +-
+ arch/arm/boot/dts/imx50.dtsi                       |    3 -
+ arch/arm/boot/dts/imx51-babbage.dts                |   15 +-
+ arch/arm/boot/dts/imx51-zii-rdu1.dts               |    6 +-
+ arch/arm/boot/dts/imx51.dtsi                       |   18 +-
+ arch/arm/boot/dts/imx53-ard.dts                    |   11 +-
+ arch/arm/boot/dts/imx53-m53.dtsi                   |    2 -
+ arch/arm/boot/dts/imx53-ppd.dts                    |   12 +-
+ arch/arm/boot/dts/imx53-qsb-common.dtsi            |   15 +-
+ arch/arm/boot/dts/imx53-qsb.dts                    |   17 +-
+ arch/arm/boot/dts/imx53-qsrb.dts                   |   15 +-
+ arch/arm/boot/dts/imx53-smd.dts                    |   20 +-
+ arch/arm/boot/dts/imx53-tx53-x03x.dts              |    1 +
+ arch/arm/boot/dts/imx53-tx53.dtsi                  |   14 +-
+ arch/arm/boot/dts/imx53-voipac-dmm-668.dtsi        |    2 +-
+ arch/arm/boot/dts/imx53.dtsi                       |   11 +-
+ arch/arm/boot/dts/imx6dl-aristainetos2_4.dts       |    2 -
+ arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts       |   15 +-
+ arch/arm/boot/dts/imx6dl-mamoj.dts                 |  224 ++++
+ arch/arm/boot/dts/imx6dl-sabreauto.dts             |   10 +-
+ arch/arm/boot/dts/imx6dl-sabresd.dts               |   10 +-
+ arch/arm/boot/dts/imx6dl-udoo.dts                  |    6 +-
+ arch/arm/boot/dts/imx6dl-wandboard-revb1.dts       |    6 +-
+ arch/arm/boot/dts/imx6dl-wandboard-revd1.dts       |    6 +-
+ arch/arm/boot/dts/imx6dl-wandboard.dts             |    6 +-
+ arch/arm/boot/dts/imx6dl.dtsi                      |   12 +-
+ arch/arm/boot/dts/imx6q-b850v3.dts                 |    4 -
+ arch/arm/boot/dts/imx6q-ba16.dtsi                  |    7 +-
+ arch/arm/boot/dts/imx6q-bx50v3.dtsi                |   13 +-
+ arch/arm/boot/dts/imx6q-dhcom-pdk2.dts             |  151 +++
+ arch/arm/boot/dts/imx6q-dhcom-som.dtsi             |  476 +++++++++
+ arch/arm/boot/dts/imx6q-gk802.dts                  |    3 +-
+ arch/arm/boot/dts/imx6q-icore-mipi.dts             |   25 +
+ arch/arm/boot/dts/imx6q-icore-ofcap12.dts          |   31 +-
+ arch/arm/boot/dts/imx6q-kp-tpc.dts                 |   22 +
+ arch/arm/boot/dts/imx6q-kp.dtsi                    |  432 ++++++++
+ arch/arm/boot/dts/imx6q-novena.dts                 |    2 -
+ arch/arm/boot/dts/imx6q-pistachio.dts              |    2 +-
+ arch/arm/boot/dts/imx6q-sabreauto.dts              |   15 +-
+ arch/arm/boot/dts/imx6q-sabresd.dts                |   15 +-
+ arch/arm/boot/dts/imx6q-udoo.dts                   |    6 +-
+ arch/arm/boot/dts/imx6q-utilite-pro.dts            |    2 -
+ arch/arm/boot/dts/imx6q-var-dt6customboard.dts     |    2 -
+ arch/arm/boot/dts/imx6q-wandboard-revb1.dts        |    6 +-
+ arch/arm/boot/dts/imx6q-wandboard-revd1.dts        |    6 +-
+ arch/arm/boot/dts/imx6q-wandboard.dts              |    6 +-
+ arch/arm/boot/dts/imx6q.dtsi                       |   39 +-
+ arch/arm/boot/dts/imx6qdl-apalis.dtsi              |    2 -
+ arch/arm/boot/dts/imx6qdl-colibri.dtsi             |    2 -
+ arch/arm/boot/dts/imx6qdl-gw5904.dtsi              |    2 -
+ arch/arm/boot/dts/imx6qdl-hummingboard.dtsi        |   52 +-
+ arch/arm/boot/dts/imx6qdl-hummingboard2.dtsi       |   47 +-
+ arch/arm/boot/dts/imx6qdl-icore.dtsi               |   25 +-
+ arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi       |    2 +-
+ arch/arm/boot/dts/imx6qdl-sabreauto.dtsi           |   15 +-
+ arch/arm/boot/dts/imx6qdl-sabrelite.dtsi           |    3 -
+ arch/arm/boot/dts/imx6qdl-sabresd.dtsi             |   18 +-
+ arch/arm/boot/dts/imx6qdl-tx6-lcd.dtsi             |    1 -
+ arch/arm/boot/dts/imx6qdl-tx6-mb7.dtsi             |    3 -
+ arch/arm/boot/dts/imx6qdl-tx6.dtsi                 |    6 +-
+ arch/arm/boot/dts/imx6qdl-udoo.dtsi                |    6 +-
+ arch/arm/boot/dts/imx6qdl-wandboard-revb1.dtsi     |    1 -
+ arch/arm/boot/dts/imx6qdl-wandboard-revc1.dtsi     |    1 -
+ arch/arm/boot/dts/imx6qdl-wandboard-revd1.dtsi     |    1 -
+ arch/arm/boot/dts/imx6qdl-wandboard.dtsi           |   14 +-
+ arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi            |   24 +-
+ arch/arm/boot/dts/imx6qdl.dtsi                     |   70 +-
+ arch/arm/boot/dts/imx6qp-sabreauto.dts             |   44 +-
+ arch/arm/boot/dts/imx6qp-sabresd.dts               |   44 +-
+ arch/arm/boot/dts/imx6qp-wandboard-revd1.dts       |    6 +-
+ arch/arm/boot/dts/imx6qp-zii-rdu2.dts              |    5 +
+ arch/arm/boot/dts/imx6qp.dtsi                      |   44 +-
+ arch/arm/boot/dts/imx6sl-evk.dts                   |   10 +-
+ arch/arm/boot/dts/imx6sl.dtsi                      |   34 +-
+ arch/arm/boot/dts/imx6sx-nitrogen6sx.dts           |    4 +-
+ arch/arm/boot/dts/imx6sx-sabreauto.dts             |  427 ++++++--
+ arch/arm/boot/dts/imx6sx.dtsi                      |  101 +-
+ arch/arm/boot/dts/imx6ul-14x14-evk.dts             |   10 +-
+ arch/arm/boot/dts/imx6ul-isiot.dtsi                |    2 -
+ arch/arm/boot/dts/imx6ul-tx6ul-mainboard.dts       |    2 +-
+ arch/arm/boot/dts/imx6ul-tx6ul.dtsi                |    6 +-
+ arch/arm/boot/dts/imx6ul.dtsi                      |   51 +-
+ arch/arm/boot/dts/imx6ull-pinfunc.h                |    9 +
+ arch/arm/boot/dts/imx6ull.dtsi                     |    2 +
+ arch/arm/boot/dts/imx7d-cl-som-imx7.dts            |    2 +-
+ arch/arm/boot/dts/imx7d-nitrogen7.dts              |    6 +-
+ arch/arm/boot/dts/imx7d-pinfunc.h                  |    6 +-
+ arch/arm/boot/dts/imx7d-sdb-sht11.dts              |   44 +-
+ arch/arm/boot/dts/imx7d-sdb.dts                    |   70 +-
+ arch/arm/boot/dts/imx7d.dtsi                       |   71 +-
+ arch/arm/boot/dts/imx7s-warp.dts                   |    4 -
+ arch/arm/boot/dts/imx7s.dtsi                       |   88 +-
+ arch/arm/boot/dts/keystone-k2g-evm.dts             |   26 +
+ arch/arm/boot/dts/logicpd-som-lv.dtsi              |   34 +
+ arch/arm/boot/dts/logicpd-torpedo-som.dtsi         |   10 +
+ arch/arm/boot/dts/meson8.dtsi                      |   33 +-
+ arch/arm/boot/dts/meson8b-odroidc1.dts             |   68 +-
+ arch/arm/boot/dts/meson8b.dtsi                     |   24 +-
+ arch/arm/boot/dts/meson8m2-mxiii-plus.dts          |  244 +++++
+ arch/arm/boot/dts/meson8m2.dtsi                    |   54 +
+ arch/arm/boot/dts/mt2701-evb.dts                   |    9 +-
+ arch/arm/boot/dts/mt2701.dtsi                      |  197 ++--
+ arch/arm/boot/dts/mt6323.dtsi                      |   17 +-
+ arch/arm/boot/dts/mt6580-evbp1.dts                 |    9 +-
+ arch/arm/boot/dts/mt6580.dtsi                      |    9 +-
+ arch/arm/boot/dts/mt6589-aquaris5.dts              |   10 +-
+ arch/arm/boot/dts/mt6589.dtsi                      |   12 +-
+ arch/arm/boot/dts/mt6592-evb.dts                   |    9 +-
+ arch/arm/boot/dts/mt6592.dtsi                      |    9 +-
+ arch/arm/boot/dts/mt7623.dtsi                      |  537 ++++++++--
+ arch/arm/boot/dts/mt7623a-rfb-emmc.dts             |  291 +++++
+ arch/arm/boot/dts/mt7623a-rfb-nand.dts             |  337 ++++++
+ arch/arm/boot/dts/mt7623a.dtsi                     |   44 +
+ arch/arm/boot/dts/mt7623n-bananapi-bpi-r2.dts      |  279 +----
+ arch/arm/boot/dts/mt7623n-rfb-emmc.dts             |  326 ++++++
+ arch/arm/boot/dts/mt7623n-rfb-nand.dts             |   40 +-
+ arch/arm/boot/dts/mt7623n-rfb.dtsi                 |   12 +-
+ arch/arm/boot/dts/mt8127-moose.dts                 |    9 +-
+ arch/arm/boot/dts/mt8127.dtsi                      |    9 +-
+ arch/arm/boot/dts/mt8135-evbp1.dts                 |    9 +-
+ arch/arm/boot/dts/mt8135.dtsi                      |    9 +-
+ arch/arm/boot/dts/omap2420-n810.dts                |   63 +-
+ arch/arm/boot/dts/omap3-beagle-xm.dts              |   10 +
+ arch/arm/boot/dts/omap3-cm-t3x.dtsi                |    2 +-
+ arch/arm/boot/dts/omap3-devkit8000-common.dtsi     |    9 +-
+ arch/arm/boot/dts/omap3-devkit8000-lcd-common.dtsi |    5 +-
+ arch/arm/boot/dts/omap3-gta04.dtsi                 |   10 +
+ arch/arm/boot/dts/omap3-pandora-common.dtsi        |   12 +-
+ arch/arm/boot/dts/omap3-sb-t35.dtsi                |    2 +-
+ arch/arm/boot/dts/qcom-apq8064.dtsi                |   58 +-
+ arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi      |   10 +-
+ arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1-c1.dts    |   19 +
+ arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1-c3.dts    |    9 +
+ arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi      |  111 ++
+ arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts    |   64 ++
+ arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c2.dts    |   25 +
+ arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1.dtsi      |   75 ++
+ arch/arm/boot/dts/qcom-ipq4019.dtsi                |  178 +++-
+ arch/arm/boot/dts/qcom-msm8660.dtsi                |    2 +-
+ .../boot/dts/qcom-msm8974-sony-xperia-amami.dts    |  436 ++++++++
+ arch/arm/boot/dts/qcom-pm8941.dtsi                 |    6 +
+ arch/arm/boot/dts/rk3036.dtsi                      |    2 +
+ arch/arm/boot/dts/rk322x.dtsi                      |    8 +
+ arch/arm/boot/dts/rk3288-phycore-som.dtsi          |    1 +
+ arch/arm/boot/dts/rk3288-tinker.dts                |    4 +
+ arch/arm/boot/dts/rk3288-veyron-chromebook.dtsi    |    3 +-
+ arch/arm/boot/dts/rk3288-veyron-minnie.dts         |    2 +
+ arch/arm/boot/dts/rk3288.dtsi                      |   20 +-
+ arch/arm/boot/dts/s3c2416-smdk2416.dts             |    5 +-
+ arch/arm/boot/dts/s3c2416.dtsi                     |   11 +-
+ arch/arm/boot/dts/s3c24xx.dtsi                     |    4 +-
+ arch/arm/boot/dts/s3c6410-mini6410.dts             |    3 +-
+ arch/arm/boot/dts/s3c6410-smdk6410.dts             |    3 +-
+ arch/arm/boot/dts/s3c64xx.dtsi                     |    4 +-
+ arch/arm/boot/dts/ste-ccu8540-pinctrl.dtsi         |  196 ----
+ arch/arm/boot/dts/ste-ccu8540.dts                  |   94 --
+ arch/arm/boot/dts/ste-ccu9540.dts                  |   79 --
+ arch/arm/boot/dts/ste-snowball.dts                 |   10 +-
+ arch/arm/boot/dts/stih407-family.dtsi              |   52 +-
+ arch/arm/boot/dts/stih407-pinctrl.dtsi             |   10 +-
+ arch/arm/boot/dts/stih407.dtsi                     |    2 +-
+ arch/arm/boot/dts/stih410.dtsi                     |   18 +-
+ arch/arm/boot/dts/stihxxx-b2120.dtsi               |    4 +-
+ arch/arm/boot/dts/stm32f469-disco.dts              |   53 +-
+ arch/arm/boot/dts/stm32f469.dtsi                   |   19 +
+ arch/arm/boot/dts/stm32f746-disco.dts              |    8 +
+ arch/arm/boot/dts/stm32f746.dtsi                   |   36 +
+ arch/arm/boot/dts/stm32f769-disco.dts              |    8 +
+ arch/arm/boot/dts/stm32h743-pinctrl.dtsi           |   10 +
+ arch/arm/boot/dts/stm32h743.dtsi                   |   53 +
+ arch/arm/boot/dts/stm32h743i-eval.dts              |    8 +
+ arch/arm/boot/dts/stm32mp157-pinctrl.dtsi          |  151 ++-
+ arch/arm/boot/dts/stm32mp157c-ed1.dts              |   57 +-
+ arch/arm/boot/dts/stm32mp157c-ev1.dts              |   87 ++
+ arch/arm/boot/dts/stm32mp157c.dtsi                 |  720 ++++++++++++-
+ .../arm/boot/dts/sun7i-a20-olimex-som-evb-emmc.dts |   37 +
+ arch/arm/boot/dts/sun7i-a20-olimex-som204-evb.dts  |    3 +-
+ arch/arm/boot/dts/sun8i-a23-a33.dtsi               |   33 +
+ arch/arm/boot/dts/sun8i-a33.dtsi                   |   44 +
+ arch/arm/boot/dts/sun8i-a83t.dtsi                  |   64 ++
+ .../boot/dts/sun8i-h2-plus-libretech-all-h3-cc.dts |   13 +
+ arch/arm/boot/dts/sun8i-h2-plus-orangepi-zero.dts  |   21 +
+ arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dts |  206 +---
+ arch/arm/boot/dts/sun8i-h3-orangepi-one.dts        |   21 +
+ arch/arm/boot/dts/sun8i-h3-orangepi-pc.dts         |   28 +
+ arch/arm/boot/dts/sun8i-h3.dtsi                    |   32 +-
+ .../boot/dts/sun8i-r16-nintendo-nes-classic.dts    |   56 +
+ .../dts/sun8i-r16-nintendo-super-nes-classic.dts   |   11 +
+ arch/arm/boot/dts/sun8i-r40-bananapi-m2-ultra.dts  |   99 +-
+ arch/arm/boot/dts/sun8i-r40.dtsi                   |   34 +
+ arch/arm/boot/dts/sun8i-v40-bananapi-m2-berry.dts  |   10 +
+ arch/arm/boot/dts/sunxi-h3-h5.dtsi                 |   18 +
+ arch/arm/boot/dts/sunxi-libretech-all-h3-cc.dtsi   |  215 ++++
+ arch/arm/boot/dts/tegra114.dtsi                    |    5 +
+ arch/arm/boot/dts/tegra124-apalis-v1.2.dtsi        |    6 +-
+ arch/arm/boot/dts/tegra124-apalis.dtsi             |    6 +-
+ arch/arm/boot/dts/tegra30.dtsi                     |   14 +
+ arch/arm/boot/dts/uniphier-pro4.dtsi               |   10 +-
+ arch/arm/boot/dts/uniphier-pxs2.dtsi               |    3 +
+ arch/arm/boot/dts/vexpress-v2m-rs1.dtsi            |  712 +++++++------
+ arch/arm/boot/dts/vexpress-v2m.dtsi                |  710 +++++++------
+ arch/arm/boot/dts/vexpress-v2p-ca15-tc1.dts        |    3 +-
+ arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts         |    9 +-
+ arch/arm/boot/dts/vexpress-v2p-ca5s.dts            |    5 +-
+ arch/arm/boot/dts/vexpress-v2p-ca9.dts             |    5 +-
+ arch/arm/boot/dts/vf-colibri-eval-v3.dtsi          |    2 +-
+ arch/arm/boot/dts/vf610-zii-dev-rev-b.dts          |    6 -
+ arch/arm/boot/dts/vf610-zii-dev-rev-c.dts          |    4 -
+ arch/arm/boot/dts/vf610-zii-dev.dtsi               |    4 +
+ arch/arm/boot/dts/vfxxx.dtsi                       |    2 +-
+ arch/arm/common/Makefile                           |    1 +
+ arch/arm/common/secure_cntvoff.S                   |   32 +
+ arch/arm/configs/bcm2835_defconfig                 |    2 +
+ arch/arm/configs/davinci_all_defconfig             |    4 +-
+ arch/arm/configs/exynos_defconfig                  |    1 +
+ arch/arm/configs/imx_v6_v7_defconfig               |    9 +
+ arch/arm/configs/multi_v7_defconfig                |   12 +-
+ arch/arm/configs/shmobile_defconfig                |    3 +-
+ arch/arm/include/asm/cputype.h                     |   14 +-
+ arch/arm/include/asm/secure_cntvoff.h              |    8 +
+ arch/arm/include/debug/brcmstb.S                   |   21 +-
+ arch/arm/mach-berlin/Kconfig                       |    6 +-
+ arch/arm/mach-berlin/berlin.c                      |    5 +-
+ arch/arm/mach-berlin/headsmp.S                     |    5 +-
+ arch/arm/mach-berlin/platsmp.c                     |    6 +-
+ arch/arm/mach-davinci/aemif.c                      |    8 +-
+ arch/arm/mach-davinci/board-da830-evm.c            |    1 +
+ arch/arm/mach-davinci/board-da850-evm.c            |    1 +
+ arch/arm/mach-davinci/board-dm355-evm.c            |    1 +
+ arch/arm/mach-davinci/board-dm355-leopard.c        |    1 +
+ arch/arm/mach-davinci/board-dm365-evm.c            |    1 +
+ arch/arm/mach-davinci/board-dm644x-evm.c           |    3 +
+ arch/arm/mach-davinci/board-dm646x-evm.c           |    1 +
+ arch/arm/mach-davinci/board-mityomapl138.c         |    1 +
+ arch/arm/mach-davinci/board-neuros-osd2.c          |    3 +
+ arch/arm/mach-davinci/board-sffsdr.c               |    2 +
+ arch/arm/mach-davinci/davinci.h                    |    1 +
+ arch/arm/mach-davinci/dm644x.c                     |   13 +-
+ arch/arm/mach-exynos/Kconfig                       |   12 -
+ arch/arm/mach-exynos/common.h                      |   17 +-
+ arch/arm/mach-exynos/exynos.c                      |   37 +-
+ arch/arm/mach-exynos/include/mach/map.h            |    2 -
+ arch/arm/mach-exynos/platsmp.c                     |   27 +-
+ arch/arm/mach-exynos/pm.c                          |    4 +-
+ arch/arm/mach-exynos/suspend.c                     |    4 +-
+ arch/arm/mach-imx/Kconfig                          |    1 +
+ arch/arm/mach-imx/mach-mx31_3ds.c                  |   18 +-
+ arch/arm/mach-imx/mach-mx31lilly.c                 |   12 +-
+ arch/arm/mach-imx/mach-mx31lite.c                  |   16 +-
+ arch/arm/mach-imx/mach-mx31moboard.c               |   17 +-
+ arch/arm/mach-imx/mach-pca100.c                    |   13 +-
+ arch/arm/mach-imx/mach-pcm037.c                    |   13 +-
+ arch/arm/mach-imx/mach-pcm037_eet.c                |    5 +-
+ arch/arm/mach-imx/mach-pcm043.c                    |   13 +-
+ arch/arm/mach-imx/mach-vpr200.c                    |    9 +-
+ arch/arm/mach-meson/Kconfig                        |    9 +-
+ arch/arm/mach-meson/meson.c                        |    1 +
+ arch/arm/mach-omap1/board-ams-delta.c              |  107 +-
+ arch/arm/mach-omap1/board-osk.c                    |   10 +-
+ arch/arm/mach-omap2/Makefile                       |    1 -
+ arch/arm/mach-omap2/board-generic.c                |    2 -
+ arch/arm/mach-omap2/clockdomain.c                  |   73 ++
+ arch/arm/mach-omap2/clockdomain.h                  |    8 +
+ arch/arm/mach-omap2/cm33xx.c                       |   53 +
+ arch/arm/mach-omap2/cminst44xx.c                   |   43 +
+ arch/arm/mach-omap2/common.h                       |   18 +-
+ arch/arm/mach-omap2/control.c                      |  112 ++
+ arch/arm/mach-omap2/control.h                      |   61 ++
+ arch/arm/mach-omap2/display.c                      |   10 +-
+ arch/arm/mach-omap2/hsmmc.c                        |    1 -
+ arch/arm/mach-omap2/i2c.c                          |    1 -
+ arch/arm/mach-omap2/io.c                           |   70 +-
+ arch/arm/mach-omap2/omap-pm-noop.c                 |  176 ---
+ arch/arm/mach-omap2/omap-pm.h                      |  161 ---
+ arch/arm/mach-omap2/omap_device.c                  |   22 +-
+ arch/arm/mach-omap2/omap_hwmod.c                   |   21 +-
+ arch/arm/mach-omap2/omap_hwmod.h                   |    6 +-
+ .../mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c |    1 +
+ arch/arm/mach-omap2/omap_hwmod_3xxx_data.c         |    4 +
+ arch/arm/mach-omap2/omap_hwmod_43xx_data.c         |    1 +
+ arch/arm/mach-omap2/omap_hwmod_44xx_data.c         |    4 +
+ arch/arm/mach-omap2/omap_hwmod_54xx_data.c         |    3 +
+ arch/arm/mach-omap2/omap_hwmod_7xx_data.c          |    6 +
+ arch/arm/mach-omap2/omap_hwmod_81xx_data.c         |    1 +
+ arch/arm/mach-omap2/pdata-quirks.c                 |   15 -
+ arch/arm/mach-omap2/pm-debug.c                     |    5 -
+ arch/arm/mach-omap2/pm.c                           |   21 +-
+ arch/arm/mach-omap2/pm33xx-core.c                  |    4 +-
+ arch/arm/mach-omap2/pm44xx.c                       |   13 +
+ arch/arm/mach-omap2/powerdomain.c                  |   87 ++
+ arch/arm/mach-omap2/powerdomain.h                  |    7 +
+ arch/arm/mach-omap2/prm33xx.c                      |   31 +
+ arch/arm/mach-omap2/prm44xx.c                      |  104 ++
+ arch/arm/mach-omap2/timer.c                        |  100 +-
+ arch/arm/mach-s3c24xx/h1940-bluetooth.c            |    2 +-
+ arch/arm/mach-s3c24xx/mach-mini2440.c              |   10 +-
+ arch/arm/mach-shmobile/common.h                    |    1 -
+ arch/arm/mach-shmobile/headsmp-apmu.S              |   22 +-
+ arch/arm/mach-shmobile/setup-rcar-gen2.c           |    3 +-
+ arch/arm/mach-sunxi/Kconfig                        |    2 +-
+ arch/arm/mach-sunxi/Makefile                       |    2 +-
+ arch/arm/mach-sunxi/headsmp.S                      |   81 ++
+ arch/arm/mach-sunxi/mc_smp.c                       |  239 +++--
+ arch/arm/mach-sunxi/sunxi.c                        |   20 +-
+ arch/arm/mach-tegra/tegra.c                        |    4 +
+ arch/arm/mach-ux500/Kconfig                        |   53 +-
+ arch/arm/mach-ux500/cpu-db8500.c                   |   16 +-
+ arch/arm/mach-ux500/db8500-regs.h                  |    4 -
+ arch/arm/mm/cache-b15-rac.c                        |   30 +-
+ arch/arm/plat-omap/Kconfig                         |   10 -
+ arch/arm/plat-samsung/adc.c                        |    3 +-
+ arch/arm/plat-samsung/include/plat/map-s5p.h       |    4 -
+ arch/arm64/boot/dts/Makefile                       |    1 +
+ arch/arm64/boot/dts/allwinner/Makefile             |    5 +-
+ arch/arm64/boot/dts/allwinner/axp803.dtsi          |    5 +
+ .../boot/dts/allwinner/sun50i-a64-bananapi-m64.dts |   21 +
+ .../allwinner/sun50i-h5-libretech-all-h3-cc.dts    |   14 +
+ arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi       |    2 +-
+ .../boot/dts/allwinner/sun50i-h6-pine-h64.dts      |   10 +
+ arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi       |   70 +-
+ arch/arm64/boot/dts/amlogic/meson-axg-s400.dts     |  106 ++
+ arch/arm64/boot/dts/amlogic/meson-axg.dtsi         |  453 +++++++-
+ arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi        |   12 +-
+ arch/arm64/boot/dts/amlogic/meson-gxl.dtsi         |   16 +-
+ arch/arm64/boot/dts/arm/juno-base.dtsi             |   15 +-
+ arch/arm64/boot/dts/arm/juno-motherboard.dtsi      |   18 +-
+ arch/arm64/boot/dts/arm/juno-r1.dts                |    8 +-
+ arch/arm64/boot/dts/arm/juno-r2.dts                |    8 +-
+ arch/arm64/boot/dts/arm/juno.dts                   |    4 +-
+ arch/arm64/boot/dts/arm/rtsm_ve-aemv8a.dts         |    4 +-
+ arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi   |  453 ++++----
+ .../boot/dts/arm/vexpress-v2f-1xv7-ca53x2.dts      |    5 +-
+ arch/arm64/boot/dts/broadcom/Makefile              |    3 +-
+ .../boot/dts/broadcom/bcm2837-rpi-3-b-plus.dts     |    2 +
+ arch/arm64/boot/dts/exynos/exynos5433.dtsi         |   80 +-
+ arch/arm64/boot/dts/exynos/exynos7.dtsi            |   18 +-
+ arch/arm64/boot/dts/freescale/fsl-ls1012a.dtsi     |   10 +-
+ arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi |    2 +-
+ arch/arm64/boot/dts/hisilicon/hi3660.dtsi          |  147 +++
+ .../boot/dts/hisilicon/hi3798cv200-poplar.dts      |   38 +
+ arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi     |  205 +++-
+ arch/arm64/boot/dts/hisilicon/hip06-d03.dts        |    8 +
+ arch/arm64/boot/dts/hisilicon/hip06.dtsi           |   21 +
+ arch/arm64/boot/dts/hisilicon/hip07-d05.dts        |    4 +
+ arch/arm64/boot/dts/hisilicon/hip07.dtsi           |   14 +
+ arch/arm64/boot/dts/hisilicon/poplar-pinctrl.dtsi  |   98 ++
+ arch/arm64/boot/dts/marvell/Makefile               |    4 -
+ .../boot/dts/marvell/armada-3720-espressobin.dts   |   27 +
+ arch/arm64/boot/dts/marvell/armada-37xx.dtsi       |    6 +
+ arch/arm64/boot/dts/marvell/armada-7040-db.dts     |    5 +
+ arch/arm64/boot/dts/marvell/armada-8040-db.dts     |   10 +
+ arch/arm64/boot/dts/marvell/armada-8040-mcbin.dts  |   70 ++
+ arch/arm64/boot/dts/marvell/armada-cp110.dtsi      |    1 +
+ arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts      |   66 --
+ arch/arm64/boot/dts/marvell/berlin4ct-stb.dts      |   66 --
+ arch/arm64/boot/dts/mediatek/mt2712-pinfunc.h      | 1123 ++++++++++++++++++++
+ arch/arm64/boot/dts/mediatek/mt2712e.dtsi          |   46 +
+ arch/arm64/boot/dts/mediatek/mt7622-rfb1.dts       |   11 +-
+ arch/arm64/boot/dts/mediatek/mt7622.dtsi           |   99 ++
+ arch/arm64/boot/dts/qcom/Makefile                  |    1 +
+ arch/arm64/boot/dts/qcom/apq8096-db820c-pins.dtsi  |   26 +
+ .../boot/dts/qcom/apq8096-db820c-pmic-pins.dtsi    |   34 +-
+ arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi       |   63 +-
+ arch/arm64/boot/dts/qcom/ipq8074-hk01.dts          |   62 +-
+ arch/arm64/boot/dts/qcom/ipq8074.dtsi              |  313 +++++-
+ arch/arm64/boot/dts/qcom/msm8916.dtsi              |   18 +-
+ .../boot/dts/qcom/msm8992-bullhead-rev-101.dts     |   17 +
+ arch/arm64/boot/dts/qcom/msm8992-pins.dtsi         |   60 ++
+ arch/arm64/boot/dts/qcom/msm8992.dtsi              |   87 +-
+ arch/arm64/boot/dts/qcom/msm8994-smd-rpm.dtsi      |  276 +++++
+ arch/arm64/boot/dts/qcom/msm8996.dtsi              |  120 ++-
+ arch/arm64/boot/dts/qcom/sdm845-mtp.dts            |   15 +
+ arch/arm64/boot/dts/qcom/sdm845.dtsi               |  327 ++++++
+ arch/arm64/boot/dts/rockchip/rk3328.dtsi           |   10 +
+ arch/arm64/boot/dts/rockchip/rk3368.dtsi           |   10 +
+ arch/arm64/boot/dts/rockchip/rk3399-firefly.dts    |    8 +
+ arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts  |    4 +-
+ arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi       |    8 +-
+ .../arm64/boot/dts/rockchip/rk3399-puma-haikou.dts |   25 +
+ arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi      |    4 +
+ .../dts/rockchip/rk3399-sapphire-excavator.dts     |   12 +
+ arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi  |   23 +-
+ arch/arm64/boot/dts/rockchip/rk3399.dtsi           |   36 +-
+ arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi   |    5 +-
+ arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi   |    3 +
+ arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi   |    8 +-
+ arch/arm64/boot/dts/sprd/sc2731.dtsi               |   11 +
+ arch/arm64/boot/dts/sprd/sc9860.dtsi               |   32 +-
+ arch/arm64/boot/dts/sprd/whale2.dtsi               |   55 +-
+ arch/arm64/boot/dts/synaptics/Makefile             |    4 +
+ arch/arm64/boot/dts/synaptics/berlin4ct-dmp.dts    |   29 +
+ arch/arm64/boot/dts/synaptics/berlin4ct-stb.dts    |   29 +
+ .../boot/dts/{marvell => synaptics}/berlin4ct.dtsi |   39 +-
+ arch/arm64/configs/defconfig                       |   43 +-
+ drivers/bus/Kconfig                                |    1 -
+ drivers/bus/arm-cci.c                              |    2 -
+ drivers/bus/hisi_lpc.c                             |  159 +--
+ drivers/bus/ti-sysc.c                              |  388 ++++++-
+ drivers/cpufreq/scmi-cpufreq.c                     |    4 +-
+ drivers/firmware/arm_scmi/base.c                   |   43 +-
+ drivers/firmware/arm_scmi/bus.c                    |   22 +-
+ drivers/firmware/arm_scmi/clock.c                  |   24 +-
+ drivers/firmware/arm_scmi/common.h                 |   22 +-
+ drivers/firmware/arm_scmi/driver.c                 |  109 +-
+ drivers/firmware/arm_scmi/perf.c                   |   38 +-
+ drivers/firmware/arm_scmi/power.c                  |   16 +-
+ drivers/firmware/arm_scmi/sensors.c                |   20 +-
+ drivers/firmware/qcom_scm.c                        |    3 +
+ drivers/firmware/ti_sci.c                          |   10 +-
+ drivers/firmware/ti_sci.h                          |   30 +-
+ drivers/media/rc/ir-rx51.c                         |   17 +-
+ drivers/memory/Kconfig                             |   10 -
+ drivers/memory/Makefile                            |    1 -
+ drivers/memory/brcmstb_dpfe.c                      |   42 +-
+ drivers/memory/omap-gpmc.c                         |    4 +-
+ drivers/memory/tegra/Makefile                      |    1 +
+ drivers/memory/tegra/mc.c                          |  362 ++++++-
+ drivers/memory/tegra/mc.h                          |   22 +
+ drivers/memory/tegra/tegra114.c                    |   33 +
+ drivers/memory/tegra/tegra124.c                    |   48 +
+ drivers/memory/tegra/tegra20.c                     |  296 ++++++
+ drivers/memory/tegra/tegra210.c                    |   53 +-
+ drivers/memory/tegra/tegra30.c                     |   35 +
+ drivers/memory/tegra20-mc.c                        |  254 -----
+ drivers/memory/ti-aemif.c                          |   60 +-
+ drivers/mmc/host/sdhci-omap.c                      |   92 +-
+ drivers/mmc/host/sdhci.c                           |  148 ++-
+ drivers/mmc/host/sdhci.h                           |   15 +
+ drivers/mtd/nand/raw/davinci_nand.c                |    6 +-
+ drivers/of/platform.c                              |    1 +
+ drivers/reset/reset-uniphier.c                     |   13 +-
+ drivers/soc/Makefile                               |    2 +-
+ drivers/soc/imx/gpc.c                              |   18 +-
+ drivers/soc/imx/gpcv2.c                            |   22 +-
+ drivers/soc/mediatek/mtk-infracfg.c                |   46 +-
+ drivers/soc/mediatek/mtk-pmic-wrap.c               |   13 +-
+ drivers/soc/mediatek/mtk-scpsys.c                  |  167 ++-
+ drivers/soc/qcom/Kconfig                           |   18 +
+ drivers/soc/qcom/Makefile                          |    2 +
+ drivers/soc/qcom/cmd-db.c                          |  317 ++++++
+ drivers/soc/qcom/qcom-geni-se.c                    |  748 +++++++++++++
+ drivers/soc/qcom/qmi_interface.c                   |    5 +-
+ drivers/soc/qcom/smd-rpm.c                         |    1 +
+ drivers/soc/qcom/smem.c                            |   77 +-
+ drivers/soc/rockchip/pm_domains.c                  |  117 +-
+ drivers/soc/samsung/pm_domains.c                   |   90 +-
+ drivers/soc/ti/knav_qmss.h                         |   14 +-
+ drivers/soc/ti/knav_qmss_queue.c                   |   60 +-
+ include/dt-bindings/clock/axg-aoclkc.h             |   26 +
+ include/dt-bindings/clock/gxbb-clkc.h              |    2 +
+ include/dt-bindings/clock/meson8b-clkc.h           |    1 +
+ include/dt-bindings/memory/tegra114-mc.h           |   17 +
+ include/dt-bindings/memory/tegra124-mc.h           |   25 +
+ include/dt-bindings/memory/tegra20-mc.h            |   21 +
+ include/dt-bindings/memory/tegra210-mc.h           |   31 +
+ include/dt-bindings/memory/tegra30-mc.h            |   19 +
+ include/dt-bindings/power/px30-power.h             |   27 +
+ include/dt-bindings/power/rk3036-power.h           |   13 +
+ include/dt-bindings/power/rk3128-power.h           |   14 +
+ include/dt-bindings/power/rk3228-power.h           |   21 +
+ include/dt-bindings/reset/axg-aoclkc.h             |   20 +
+ include/linux/mmc/host.h                           |    4 +
+ include/linux/platform_data/media/ir-rx51.h        |    9 -
+ include/linux/platform_data/mtd-davinci.h          |   10 +
+ include/linux/platform_data/spi-imx.h              |   29 +-
+ include/linux/platform_data/ti-aemif.h             |   25 +
+ include/linux/platform_data/ti-sysc.h              |    1 +
+ include/linux/qcom-geni-se.h                       |  425 ++++++++
+ include/linux/regmap.h                             |    1 +
+ include/linux/scmi_protocol.h                      |   18 +-
+ include/linux/soc/qcom/smem.h                      |    2 +
+ include/linux/soc/ti/ti_sci_protocol.h             |   10 +-
+ include/soc/qcom/cmd-db.h                          |   45 +
+ include/soc/tegra/mc.h                             |   37 +-
+ sound/soc/omap/ams-delta.c                         |   38 +-
+ 675 files changed, 22595 insertions(+), 8088 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/arm/tegra/nvidia,tegra30-mc.txt
+ rename Documentation/devicetree/bindings/{arm/tegra => memory-controllers}/nvidia,tegra20-mc.txt (57%)
+ create mode 100644 Documentation/devicetree/bindings/net/qualcomm-bluetooth.txt
+ create mode 100644 Documentation/devicetree/bindings/reserved-memory/qcom,cmd-db.txt
+ create mode 100644 Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.txt
+ create mode 100644 arch/arm/arm-soc-for-next-contents.txt
+ create mode 100644 arch/arm/boot/dts/am335x-osd335x-common.dtsi
+ create mode 100644 arch/arm/boot/dts/am335x-pocketbeagle.dts
+ create mode 100644 arch/arm/boot/dts/am3517-som.dtsi
+ create mode 100644 arch/arm/boot/dts/aspeed-bmc-intel-s2600wf.dts
+ create mode 100644 arch/arm/boot/dts/aspeed-bmc-opp-lanyang.dts
+ create mode 100644 arch/arm/boot/dts/aspeed-bmc-portwell-neptune.dts
+ create mode 100644 arch/arm/boot/dts/bcm2837-rpi-3-b-plus.dts
+ create mode 100644 arch/arm/boot/dts/bcm283x-rpi-lan7515.dtsi
+ create mode 100644 arch/arm/boot/dts/bcm47094-luxul-xap-1610.dts
+ create mode 100644 arch/arm/boot/dts/bcm47094-luxul-xwr-3150-v1.dts
+ create mode 100644 arch/arm/boot/dts/berlin2cd-valve-steamlink.dts
+ create mode 100644 arch/arm/boot/dts/dra7-mmc-iodelay.dtsi
+ delete mode 100644 arch/arm/boot/dts/exynos5440-sd5v1.dts
+ delete mode 100644 arch/arm/boot/dts/exynos5440-ssdk5440.dts
+ delete mode 100644 arch/arm/boot/dts/exynos5440-tmu-sensor-conf.dtsi
+ delete mode 100644 arch/arm/boot/dts/exynos5440-trip-points.dtsi
+ delete mode 100644 arch/arm/boot/dts/exynos5440.dtsi
+ create mode 100644 arch/arm/boot/dts/imx6dl-mamoj.dts
+ create mode 100644 arch/arm/boot/dts/imx6q-dhcom-pdk2.dts
+ create mode 100644 arch/arm/boot/dts/imx6q-dhcom-som.dtsi
+ create mode 100644 arch/arm/boot/dts/imx6q-icore-mipi.dts
+ create mode 100644 arch/arm/boot/dts/imx6q-kp-tpc.dts
+ create mode 100644 arch/arm/boot/dts/imx6q-kp.dtsi
+ create mode 100644 arch/arm/boot/dts/meson8m2-mxiii-plus.dts
+ create mode 100644 arch/arm/boot/dts/meson8m2.dtsi
+ create mode 100644 arch/arm/boot/dts/mt7623a-rfb-emmc.dts
+ create mode 100644 arch/arm/boot/dts/mt7623a-rfb-nand.dts
+ create mode 100644 arch/arm/boot/dts/mt7623a.dtsi
+ create mode 100644 arch/arm/boot/dts/mt7623n-rfb-emmc.dts
+ create mode 100644 arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1-c1.dts
+ create mode 100644 arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1-c3.dts
+ create mode 100644 arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi
+ create mode 100644 arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts
+ create mode 100644 arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c2.dts
+ create mode 100644 arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1.dtsi
+ create mode 100644 arch/arm/boot/dts/qcom-msm8974-sony-xperia-amami.dts
+ delete mode 100644 arch/arm/boot/dts/ste-ccu8540-pinctrl.dtsi
+ delete mode 100644 arch/arm/boot/dts/ste-ccu8540.dts
+ delete mode 100644 arch/arm/boot/dts/ste-ccu9540.dts
+ create mode 100644 arch/arm/boot/dts/stm32f469.dtsi
+ create mode 100644 arch/arm/boot/dts/sun7i-a20-olimex-som-evb-emmc.dts
+ create mode 100644 arch/arm/boot/dts/sun8i-h2-plus-libretech-all-h3-cc.dts
+ create mode 100644 arch/arm/boot/dts/sun8i-r16-nintendo-nes-classic.dts
+ create mode 100644 arch/arm/boot/dts/sun8i-r16-nintendo-super-nes-classic.dts
+ create mode 100644 arch/arm/boot/dts/sunxi-libretech-all-h3-cc.dtsi
+ create mode 100644 arch/arm/common/secure_cntvoff.S
+ create mode 100644 arch/arm/include/asm/secure_cntvoff.h
+ delete mode 100644 arch/arm/mach-omap2/omap-pm-noop.c
+ delete mode 100644 arch/arm/mach-omap2/omap-pm.h
+ create mode 100644 arch/arm/mach-sunxi/headsmp.S
+ create mode 100644 arch/arm64/boot/dts/allwinner/sun50i-h5-libretech-all-h3-cc.dts
+ create mode 100644 arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b-plus.dts
+ create mode 100644 arch/arm64/boot/dts/hisilicon/poplar-pinctrl.dtsi
+ delete mode 100644 arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts
+ delete mode 100644 arch/arm64/boot/dts/marvell/berlin4ct-stb.dts
+ create mode 100644 arch/arm64/boot/dts/mediatek/mt2712-pinfunc.h
+ create mode 100644 arch/arm64/boot/dts/qcom/msm8994-smd-rpm.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/sdm845-mtp.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/sdm845.dtsi
+ create mode 100644 arch/arm64/boot/dts/synaptics/Makefile
+ create mode 100644 arch/arm64/boot/dts/synaptics/berlin4ct-dmp.dts
+ create mode 100644 arch/arm64/boot/dts/synaptics/berlin4ct-stb.dts
+ rename arch/arm64/boot/dts/{marvell => synaptics}/berlin4ct.dtsi (78%)
+ create mode 100644 drivers/memory/tegra/tegra20.c
+ delete mode 100644 drivers/memory/tegra20-mc.c
+ create mode 100644 drivers/soc/qcom/cmd-db.c
+ create mode 100644 drivers/soc/qcom/qcom-geni-se.c
+ create mode 100644 include/dt-bindings/clock/axg-aoclkc.h
+ create mode 100644 include/dt-bindings/memory/tegra20-mc.h
+ create mode 100644 include/dt-bindings/power/px30-power.h
+ create mode 100644 include/dt-bindings/power/rk3036-power.h
+ create mode 100644 include/dt-bindings/power/rk3128-power.h
+ create mode 100644 include/dt-bindings/power/rk3228-power.h
+ create mode 100644 include/dt-bindings/reset/axg-aoclkc.h
+ delete mode 100644 include/linux/platform_data/media/ir-rx51.h
+ create mode 100644 include/linux/qcom-geni-se.h
+ create mode 100644 include/soc/qcom/cmd-db.h
+Merging actions/for-next (61862a89e6ac Merge branch 'v4.16/drivers' into next)
+$ git merge actions/for-next
+Already up to date!
+Merge made by the 'recursive' strategy.
+Merging alpine/alpine/for-next (7928b2cbe55b Linux 4.16-rc1)
+$ git merge alpine/alpine/for-next
+Already up to date.
+Merging amlogic/for-next (9999d2013e27 Merge branch 'v4.18/defconfig' into tmp/aml-rebuild)
+$ git merge amlogic/for-next
+Merge made by the 'recursive' strategy.
+ arch/arm64/boot/dts/amlogic/meson-gxl-s905x-libretech-cc.dts | 3 ---
+ 1 file changed, 3 deletions(-)
+Merging aspeed/for-next (6534b2d0de36 Merge branch 'dt-for-v4.17' into for-next)
+$ git merge aspeed/for-next
+Already up to date!
+Merge made by the 'recursive' strategy.
+Merging at91/at91-next (20d3f36b0fea Merge tag 'at91-ab-4.17-dt2' into at91-next)
+$ git merge at91/at91-next
+Already up to date!
+Merge made by the 'recursive' strategy.
+Merging bcm2835/for-next (5c483a0c319c Merge branch anholt/bcm2835-defconfig-64-next into for-next)
+$ git merge bcm2835/for-next
+Already up to date!
+Merge made by the 'recursive' strategy.
+Merging imx-mxs/for-next (452daaaf6021 Merge branch 'imx/defconfig' into for-next)
+$ git merge imx-mxs/for-next
+Already up to date!
+Merge made by the 'recursive' strategy.
+Merging keystone/next (fb5d812c3cb3 Merge branch 'for_4.18/keystone-dts' into next)
+$ git merge keystone/next
+Already up to date!
+Merge made by the 'recursive' strategy.
+Merging mediatek/for-next (9378b3427341 Merge branch 'v4.17-next/soc' into for-next)
+$ git merge mediatek/for-next
+Already up to date!
+Merge made by the 'recursive' strategy.
+Merging mvebu/for-next (78e5d8c26c70 Merge branch 'mvebu/dt64' into mvebu/for-next)
+$ git merge mvebu/for-next
+Merge made by the 'recursive' strategy.
+ arch/arm/boot/dts/armada-385-synology-ds116.dts | 2 +-
+ arch/arm64/boot/dts/marvell/armada-cp110.dtsi   | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+Merging omap/for-next (1b7e42c1dfe8 Merge branch 'omap-for-v4.18/soc' into for-next)
+$ git merge omap/for-next
+Merge made by the 'recursive' strategy.
+ arch/arm/boot/dts/dra7.dtsi | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging reset/reset/next (d7bab65b1f57 reset: uniphier: add LD11/LD20 stream demux system reset control)
+$ git merge reset/reset/next
+Already up to date.
+Merging qcom/for-next (509d20563e5d firmware: qcom: scm: Fix crash in qcom_scm_call_atomic1())
+$ git merge qcom/for-next
+Merge made by the 'recursive' strategy.
+Merging realtek/for-next (2b6286eb7ab8 Merge branch 'v4.15/dt64' into next)
+$ git merge realtek/for-next
+Already up to date!
+Merge made by the 'recursive' strategy.
+Merging renesas/next (e800ff91fb67 Merge branches 'fixes-for-v4.17', 'arm64-dt-for-v4.18', 'arm64-defconfig-for-v4.18', 'defconfig-for-v4.18', 'dt-bindings-for-v4.18', 'dt-for-v4.18' and 'soc-for-v4.18' into next)
+$ git merge renesas/next
+Auto-merging arch/arm/mach-shmobile/setup-rcar-gen2.c
+Auto-merging arch/arm/mach-shmobile/Kconfig
+Auto-merging arch/arm/boot/dts/Makefile
+Auto-merging arch/arm/Makefile
+Auto-merging arch/arm/Kconfig
+Merge made by the 'recursive' strategy.
+ Documentation/devicetree/bindings/arm/shmobile.txt |    2 +
+ .../bindings/power/renesas,rcar-sysc.txt           |    2 +
+ .../devicetree/bindings/reset/renesas,rst.txt      |    2 +
+ arch/arm/Kconfig                                   |    2 +-
+ arch/arm/Kconfig.debug                             |   13 +-
+ arch/arm/Makefile                                  |    2 +-
+ arch/arm/boot/dts/Makefile                         |    1 +
+ arch/arm/boot/dts/emev2-kzm9d.dts                  |    3 -
+ arch/arm/boot/dts/emev2.dtsi                       |    5 +-
+ arch/arm/boot/dts/r7s72100.dtsi                    | 1038 ++++----
+ arch/arm/boot/dts/r8a73a4-ape6evm.dts              |    4 +-
+ arch/arm/boot/dts/r8a73a4.dtsi                     |   10 +-
+ arch/arm/boot/dts/r8a7740.dtsi                     |   18 +
+ arch/arm/boot/dts/r8a7743-iwg20m.dtsi              |    5 +
+ arch/arm/boot/dts/r8a7743.dtsi                     |   19 +-
+ arch/arm/boot/dts/r8a7745-iwg22m.dtsi              |    5 +
+ arch/arm/boot/dts/r8a7745.dtsi                     |   19 +-
+ arch/arm/boot/dts/r8a77470-iwg23s-sbc.dts          |   48 +
+ arch/arm/boot/dts/r8a77470.dtsi                    |  336 +++
+ arch/arm/boot/dts/r8a7790-lager.dts                |    8 +-
+ arch/arm/boot/dts/r8a7790.dtsi                     |   67 +-
+ arch/arm/boot/dts/r8a7791-koelsch.dts              |   11 +-
+ arch/arm/boot/dts/r8a7791-porter.dts               |    8 +-
+ arch/arm/boot/dts/r8a7791.dtsi                     |   37 +-
+ arch/arm/boot/dts/r8a7792-blanche.dts              |    5 +
+ arch/arm/boot/dts/r8a7792-wheat.dts                |   16 +-
+ arch/arm/boot/dts/r8a7792.dtsi                     |   19 +-
+ arch/arm/boot/dts/r8a7793-gose.dts                 |   11 +-
+ arch/arm/boot/dts/r8a7793.dtsi                     |   37 +-
+ arch/arm/boot/dts/r8a7794-alt.dts                  |   14 +-
+ arch/arm/boot/dts/r8a7794-silk.dts                 |    8 +-
+ arch/arm/boot/dts/r8a7794.dtsi                     |   28 +-
+ arch/arm/boot/dts/sh73a0.dtsi                      |    9 +-
+ arch/arm/mach-shmobile/Kconfig                     |   13 +
+ arch/arm/mach-shmobile/setup-rcar-gen2.c           |    2 +
+ arch/arm64/Kconfig.platforms                       |    6 +
+ arch/arm64/boot/dts/renesas/Makefile               |    3 +-
+ .../boot/dts/renesas/r8a7795-es1-salvator-x.dts    |   46 +
+ arch/arm64/boot/dts/renesas/r8a7795-es1.dtsi       |  144 +-
+ arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts |   46 +
+ .../arm64/boot/dts/renesas/r8a7795-salvator-xs.dts |   85 +
+ arch/arm64/boot/dts/renesas/r8a7795.dtsi           | 2499 +++++++++++---------
+ arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts |   28 +
+ .../arm64/boot/dts/renesas/r8a7796-salvator-xs.dts |   28 +
+ arch/arm64/boot/dts/renesas/r8a7796.dtsi           | 2395 +++++++++++--------
+ .../arm64/boot/dts/renesas/r8a77965-salvator-x.dts |   28 +
+ .../boot/dts/renesas/r8a77965-salvator-xs.dts      |   28 +
+ arch/arm64/boot/dts/renesas/r8a77965.dtsi          | 1601 +++++++++----
+ arch/arm64/boot/dts/renesas/r8a77970-eagle.dts     |  116 +
+ arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts     |  137 ++
+ arch/arm64/boot/dts/renesas/r8a77970.dtsi          |  661 ++++--
+ arch/arm64/boot/dts/renesas/r8a77980-condor.dts    |   81 +
+ arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts     |   60 +
+ arch/arm64/boot/dts/renesas/r8a77980.dtsi          |   99 +-
+ arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts     |   65 +
+ arch/arm64/boot/dts/renesas/r8a77990.dtsi          |  281 +++
+ arch/arm64/boot/dts/renesas/r8a77995-draak.dts     |    2 +-
+ arch/arm64/boot/dts/renesas/r8a77995.dtsi          |  730 +++---
+ arch/arm64/boot/dts/renesas/salvator-common.dtsi   |  187 +-
+ arch/arm64/boot/dts/renesas/ulcb.dtsi              |   37 +-
+ drivers/soc/renesas/Kconfig                        |   13 +-
+ drivers/soc/renesas/Makefile                       |    2 +
+ drivers/soc/renesas/r8a77470-sysc.c                |   29 +
+ drivers/soc/renesas/r8a77990-sysc.c                |   68 +
+ drivers/soc/renesas/r8a77995-sysc.c                |    3 +-
+ drivers/soc/renesas/rcar-rst.c                     |    2 +
+ drivers/soc/renesas/rcar-sysc.c                    |    6 +
+ drivers/soc/renesas/rcar-sysc.h                    |    2 +
+ drivers/soc/renesas/renesas-soc.c                  |   16 +
+ include/dt-bindings/power/r8a77470-sysc.h          |   22 +
+ include/dt-bindings/power/r8a77990-sysc.h          |   26 +
+ 71 files changed, 7779 insertions(+), 3630 deletions(-)
+ create mode 100644 arch/arm/boot/dts/r8a77470-iwg23s-sbc.dts
+ create mode 100644 arch/arm/boot/dts/r8a77470.dtsi
+ create mode 100644 arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts
+ create mode 100644 arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts
+ create mode 100644 arch/arm64/boot/dts/renesas/r8a77990.dtsi
+ create mode 100644 drivers/soc/renesas/r8a77470-sysc.c
+ create mode 100644 drivers/soc/renesas/r8a77990-sysc.c
+ create mode 100644 include/dt-bindings/power/r8a77470-sysc.h
+ create mode 100644 include/dt-bindings/power/r8a77990-sysc.h
+Merging rockchip/for-next (42568f346f65 Merge branch 'v4.18-clk/next' into for-next)
+$ git merge rockchip/for-next
+Removing drivers/clk/rockchip/clk-rockchip.c
+Removing Documentation/devicetree/bindings/clock/rockchip.txt
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/clock/rockchip.txt         | 77 -----------------
+ drivers/clk/rockchip/Makefile                      |  1 -
+ drivers/clk/rockchip/clk-rockchip.c                | 98 ----------------------
+ drivers/clk/rockchip/clk.c                         | 16 ++--
+ 4 files changed, 5 insertions(+), 187 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/clock/rockchip.txt
+ delete mode 100644 drivers/clk/rockchip/clk-rockchip.c
+Merging samsung/for-next (bebc6082da0a Linux 4.14)
+$ git merge samsung/for-next
+Already up to date.
+Merging samsung-krzk/for-next (fba97222728d Merge branch 'next/dt64' into for-next)
+$ git merge samsung-krzk/for-next
+Merge made by the 'recursive' strategy.
+Merging sunxi/sunxi/for-next (e97360054391 Merge branch 'sunxi/dt64-for-4.18' into sunxi/for-next)
+$ git merge sunxi/sunxi/for-next
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/clock/sunxi-ccu.txt        |   3 +-
+ arch/arm64/boot/dts/allwinner/Makefile             |   1 +
+ .../dts/allwinner/sun50i-a64-amarula-relic.dts     | 188 +++++++++++++++++++
+ .../boot/dts/allwinner/sun50i-a64-sopine.dtsi      |  12 ++
+ .../boot/dts/allwinner/sun50i-h5-orangepi-pc2.dts  |  12 ++
+ drivers/clk/Makefile                               |   2 +-
+ drivers/clk/sunxi-ng/Kconfig                       |   5 +
+ drivers/clk/sunxi-ng/Makefile                      |  40 ++--
+ drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c             | 207 +++++++++++++++++++++
+ drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h             |  19 ++
+ drivers/clk/sunxi-ng/ccu-sun8i-r40.c               |  72 +++++--
+ include/dt-bindings/clock/sun50i-h6-r-ccu.h        |  24 +++
+ include/dt-bindings/reset/sun50i-h6-r-ccu.h        |  17 ++
+ 13 files changed, 565 insertions(+), 37 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/allwinner/sun50i-a64-amarula-relic.dts
+ create mode 100644 drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c
+ create mode 100644 drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h
+ create mode 100644 include/dt-bindings/clock/sun50i-h6-r-ccu.h
+ create mode 100644 include/dt-bindings/reset/sun50i-h6-r-ccu.h
+Merging tegra/for-next (6de9ae597a79 Merge branch for-4.18/clk into for-next)
+$ git merge tegra/for-next
+Merge made by the 'recursive' strategy.
+ drivers/clk/tegra/clk-tegra114.c |  2 +-
+ drivers/clk/tegra/clk-tegra124.c |  2 +-
+ drivers/clk/tegra/clk-tegra20.c  | 52 ++++++++++++++++++++++++++++++++++++----
+ drivers/clk/tegra/clk-tegra210.c |  2 +-
+ drivers/clk/tegra/clk-tegra30.c  |  2 +-
+ drivers/clk/tegra/clk.c          |  5 ++--
+ drivers/clk/tegra/clk.h          |  2 +-
+ 7 files changed, 55 insertions(+), 12 deletions(-)
+Merging arm64/for-next/core (159fd7b8d3d1 arm64/sve: Write ZCR_EL1 on context switch only if changed)
+$ git merge arm64/for-next/core
+Auto-merging arch/arm64/mm/dma-mapping.c
+Auto-merging arch/arm64/include/asm/kvm_mmu.h
+Auto-merging arch/arm64/Kconfig
+Merge made by the 'recursive' strategy.
+ arch/arm64/Kconfig                    |   2 +
+ arch/arm64/include/asm/acpi.h         |   4 +
+ arch/arm64/include/asm/cache.h        |   6 +-
+ arch/arm64/include/asm/cmpxchg.h      |   4 +-
+ arch/arm64/include/asm/fpsimdmacros.h |  12 +-
+ arch/arm64/include/asm/kvm_mmu.h      |   1 -
+ arch/arm64/include/asm/topology.h     |   6 +-
+ arch/arm64/kernel/armv8_deprecated.c  |   3 +-
+ arch/arm64/kernel/cacheinfo.c         |  15 +-
+ arch/arm64/kernel/cpufeature.c        |   9 +-
+ arch/arm64/kernel/entry-fpsimd.S      |   2 +-
+ arch/arm64/kernel/fpsimd.c            |   1 -
+ arch/arm64/kernel/ptrace.c            |   2 -
+ arch/arm64/kernel/topology.c          | 107 +++++-
+ arch/arm64/kernel/vmlinux.lds.S       |  20 +-
+ arch/arm64/mm/dma-mapping.c           |   5 +
+ arch/riscv/kernel/cacheinfo.c         |   1 -
+ drivers/acpi/Kconfig                  |   3 +
+ drivers/acpi/Makefile                 |   1 +
+ drivers/acpi/pptt.c                   | 655 ++++++++++++++++++++++++++++++++++
+ drivers/acpi/tables.c                 |   2 +-
+ drivers/base/cacheinfo.c              | 157 ++++----
+ include/linux/acpi.h                  |   4 +
+ include/linux/cacheinfo.h             |  25 +-
+ 24 files changed, 907 insertions(+), 140 deletions(-)
+ create mode 100644 drivers/acpi/pptt.c
+Merging clk/clk-next (5f016be1c548 Merge remote-tracking branch 'clk/clk-davinci' into clk-next)
+$ git merge clk/clk-next
+Auto-merging drivers/soc/rockchip/pm_domains.c
+Auto-merging drivers/clk/Makefile
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ .../bindings/arm/mediatek/mediatek,g3dsys.txt      |   30 +
+ .../devicetree/bindings/clock/actions,s900-cmu.txt |   47 +
+ .../bindings/clock/nuvoton,npcm750-clk.txt         |  100 +
+ .../devicetree/bindings/clock/qcom,gcc.txt         |    2 +
+ .../devicetree/bindings/clock/qcom,rpmh-clk.txt    |   22 +
+ .../devicetree/bindings/clock/renesas,cpg-mssr.txt |   10 +-
+ MAINTAINERS                                        |    1 +
+ drivers/clk/Kconfig                                |    1 +
+ drivers/clk/Makefile                               |    2 +
+ drivers/clk/actions/Kconfig                        |   14 +
+ drivers/clk/actions/Makefile                       |   12 +
+ drivers/clk/actions/owl-common.c                   |   89 +
+ drivers/clk/actions/owl-common.h                   |   41 +
+ drivers/clk/actions/owl-composite.c                |  199 ++
+ drivers/clk/actions/owl-composite.h                |  124 +
+ drivers/clk/actions/owl-divider.c                  |   94 +
+ drivers/clk/actions/owl-divider.h                  |   75 +
+ drivers/clk/actions/owl-factor.c                   |  222 ++
+ drivers/clk/actions/owl-factor.h                   |   83 +
+ drivers/clk/actions/owl-fixed-factor.h             |   28 +
+ drivers/clk/actions/owl-gate.c                     |   77 +
+ drivers/clk/actions/owl-gate.h                     |   73 +
+ drivers/clk/actions/owl-mux.c                      |   60 +
+ drivers/clk/actions/owl-mux.h                      |   61 +
+ drivers/clk/actions/owl-pll.c                      |  194 ++
+ drivers/clk/actions/owl-pll.h                      |   92 +
+ drivers/clk/actions/owl-s900.c                     |  721 ++++
+ drivers/clk/at91/clk-pll.c                         |   13 +-
+ drivers/clk/clk-aspeed.c                           |   48 +-
+ drivers/clk/clk-bulk.c                             |    5 +-
+ drivers/clk/clk-npcm7xx.c                          |  656 ++++
+ drivers/clk/clk-stm32mp1.c                         |   12 +-
+ drivers/clk/clk.c                                  |   21 +-
+ drivers/clk/davinci/pll-da830.c                    |    5 +-
+ drivers/clk/davinci/pll-da850.c                    |   37 +-
+ drivers/clk/davinci/pll-dm355.c                    |   22 +-
+ drivers/clk/davinci/pll-dm365.c                    |    9 +-
+ drivers/clk/davinci/pll-dm644x.c                   |    9 +-
+ drivers/clk/davinci/pll-dm646x.c                   |   11 +-
+ drivers/clk/davinci/pll.c                          |  299 +-
+ drivers/clk/davinci/pll.h                          |   41 +-
+ drivers/clk/davinci/psc-da830.c                    |    3 +-
+ drivers/clk/davinci/psc-dm355.c                    |    7 +-
+ drivers/clk/davinci/psc-dm365.c                    |   22 +-
+ drivers/clk/davinci/psc-dm644x.c                   |    3 +-
+ drivers/clk/davinci/psc-dm646x.c                   |    3 +-
+ drivers/clk/davinci/psc.c                          |   72 +-
+ drivers/clk/davinci/psc.h                          |   12 +
+ drivers/clk/hisilicon/crg-hi3798cv200.c            |   17 +
+ drivers/clk/imx/clk-imx6sx.c                       |   10 +-
+ drivers/clk/imx/clk-imx6ul.c                       |   20 +-
+ drivers/clk/mediatek/Kconfig                       |    6 +
+ drivers/clk/mediatek/Makefile                      |    1 +
+ drivers/clk/mediatek/clk-mt2701-g3d.c              |   95 +
+ drivers/clk/mediatek/clk-mt2701.c                  |    8 +-
+ drivers/clk/meson/Kconfig                          |    8 +
+ drivers/clk/meson/Makefile                         |    3 +-
+ drivers/clk/meson/axg-aoclk.c                      |  164 +
+ drivers/clk/meson/axg-aoclk.h                      |   29 +
+ drivers/clk/meson/gxbb-aoclk.c                     |   96 +-
+ drivers/clk/meson/gxbb-aoclk.h                     |    5 +
+ drivers/clk/meson/gxbb.c                           |  114 +
+ drivers/clk/meson/gxbb.h                           |    6 +-
+ drivers/clk/meson/meson-aoclk.c                    |   81 +
+ drivers/clk/meson/meson-aoclk.h                    |   34 +
+ drivers/clk/meson/meson8b.c                        |   54 +
+ drivers/clk/meson/meson8b.h                        |    5 +-
+ drivers/clk/qcom/Kconfig                           |   17 +
+ drivers/clk/qcom/Makefile                          |    2 +
+ drivers/clk/qcom/clk-alpha-pll.c                   |  301 +-
+ drivers/clk/qcom/clk-alpha-pll.h                   |   18 +-
+ drivers/clk/qcom/clk-branch.c                      |    7 +-
+ drivers/clk/qcom/clk-branch.h                      |    1 +
+ drivers/clk/qcom/clk-rcg.h                         |   17 +-
+ drivers/clk/qcom/clk-rcg2.c                        |  168 +-
+ drivers/clk/qcom/gcc-msm8998.c                     | 2834 ++++++++++++++++
+ drivers/clk/qcom/gcc-sdm845.c                      | 3465 ++++++++++++++++++++
+ drivers/clk/qcom/gdsc.c                            |   94 +-
+ drivers/clk/qcom/gdsc.h                            |    5 +-
+ drivers/clk/renesas/Kconfig                        |   10 +
+ drivers/clk/renesas/Makefile                       |    2 +
+ drivers/clk/renesas/r8a7743-cpg-mssr.c             |    2 +-
+ drivers/clk/renesas/r8a7745-cpg-mssr.c             |    2 +-
+ drivers/clk/renesas/r8a77470-cpg-mssr.c            |  229 ++
+ drivers/clk/renesas/r8a7791-cpg-mssr.c             |    2 +-
+ drivers/clk/renesas/r8a7792-cpg-mssr.c             |    2 +-
+ drivers/clk/renesas/r8a7794-cpg-mssr.c             |    2 +-
+ drivers/clk/renesas/r8a77965-cpg-mssr.c            |    4 +
+ drivers/clk/renesas/r8a77980-cpg-mssr.c            |    2 +-
+ drivers/clk/renesas/r8a77990-cpg-mssr.c            |  289 ++
+ drivers/clk/renesas/rcar-gen2-cpg.c                |   24 +
+ drivers/clk/renesas/renesas-cpg-mssr.c             |   12 +
+ drivers/clk/renesas/renesas-cpg-mssr.h             |    2 +
+ drivers/clk/samsung/clk-s3c2410-dclk.c             |    6 +-
+ drivers/clk/socfpga/clk-s10.c                      |   40 +-
+ drivers/clk/spear/spear6xx_clock.c                 |    2 +-
+ drivers/clk/uniphier/clk-uniphier-sys.c            |    5 +
+ drivers/clocksource/timer-sp804.c                  |    3 +-
+ drivers/pinctrl/sunxi/pinctrl-sunxi.c              |    4 +-
+ drivers/soc/rockchip/pm_domains.c                  |    4 +-
+ drivers/soc/tegra/pmc.c                            |    3 +-
+ include/dt-bindings/clock/actions,s900-cmu.h       |  129 +
+ include/dt-bindings/clock/aspeed-clock.h           |    3 +-
+ include/dt-bindings/clock/histb-clock.h            |    8 +
+ include/dt-bindings/clock/imx6sx-clock.h           |    6 +-
+ include/dt-bindings/clock/imx6ul-clock.h           |   33 +-
+ include/dt-bindings/clock/mt2701-clk.h             |   20 +-
+ include/dt-bindings/clock/nuvoton,npcm7xx-clock.h  |   44 +
+ include/dt-bindings/clock/qcom,gcc-msm8998.h       |  208 ++
+ include/dt-bindings/clock/qcom,gcc-sdm845.h        |  239 ++
+ include/dt-bindings/clock/qcom,rpmh.h              |   24 +
+ include/dt-bindings/clock/r8a77470-cpg-mssr.h      |   36 +
+ include/dt-bindings/clock/r8a77990-cpg-mssr.h      |   62 +
+ include/dt-bindings/reset/mt2701-resets.h          |    3 +
+ include/linux/clk-provider.h                       |   16 +-
+ include/linux/clk/davinci.h                        |   40 +
+ include/linux/of_clk.h                             |   30 +
+ 117 files changed, 12592 insertions(+), 429 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/arm/mediatek/mediatek,g3dsys.txt
+ create mode 100644 Documentation/devicetree/bindings/clock/actions,s900-cmu.txt
+ create mode 100644 Documentation/devicetree/bindings/clock/nuvoton,npcm750-clk.txt
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,rpmh-clk.txt
+ create mode 100644 drivers/clk/actions/Kconfig
+ create mode 100644 drivers/clk/actions/Makefile
+ create mode 100644 drivers/clk/actions/owl-common.c
+ create mode 100644 drivers/clk/actions/owl-common.h
+ create mode 100644 drivers/clk/actions/owl-composite.c
+ create mode 100644 drivers/clk/actions/owl-composite.h
+ create mode 100644 drivers/clk/actions/owl-divider.c
+ create mode 100644 drivers/clk/actions/owl-divider.h
+ create mode 100644 drivers/clk/actions/owl-factor.c
+ create mode 100644 drivers/clk/actions/owl-factor.h
+ create mode 100644 drivers/clk/actions/owl-fixed-factor.h
+ create mode 100644 drivers/clk/actions/owl-gate.c
+ create mode 100644 drivers/clk/actions/owl-gate.h
+ create mode 100644 drivers/clk/actions/owl-mux.c
+ create mode 100644 drivers/clk/actions/owl-mux.h
+ create mode 100644 drivers/clk/actions/owl-pll.c
+ create mode 100644 drivers/clk/actions/owl-pll.h
+ create mode 100644 drivers/clk/actions/owl-s900.c
+ create mode 100644 drivers/clk/clk-npcm7xx.c
+ create mode 100644 drivers/clk/mediatek/clk-mt2701-g3d.c
+ create mode 100644 drivers/clk/meson/axg-aoclk.c
+ create mode 100644 drivers/clk/meson/axg-aoclk.h
+ create mode 100644 drivers/clk/meson/meson-aoclk.c
+ create mode 100644 drivers/clk/meson/meson-aoclk.h
+ create mode 100644 drivers/clk/qcom/gcc-msm8998.c
+ create mode 100644 drivers/clk/qcom/gcc-sdm845.c
+ create mode 100644 drivers/clk/renesas/r8a77470-cpg-mssr.c
+ create mode 100644 drivers/clk/renesas/r8a77990-cpg-mssr.c
+ create mode 100644 include/dt-bindings/clock/actions,s900-cmu.h
+ create mode 100644 include/dt-bindings/clock/nuvoton,npcm7xx-clock.h
+ create mode 100644 include/dt-bindings/clock/qcom,gcc-msm8998.h
+ create mode 100644 include/dt-bindings/clock/qcom,gcc-sdm845.h
+ create mode 100644 include/dt-bindings/clock/qcom,rpmh.h
+ create mode 100644 include/dt-bindings/clock/r8a77470-cpg-mssr.h
+ create mode 100644 include/dt-bindings/clock/r8a77990-cpg-mssr.h
+ create mode 100644 include/linux/clk/davinci.h
+ create mode 100644 include/linux/of_clk.h
+Merging clk-samsung/for-next (e8cf870f253b clk: samsung: simplify getting .drvdata)
+$ git merge clk-samsung/for-next
+Merge made by the 'recursive' strategy.
+Merging c6x/for-linux-next (85fa2cc51104 c6x: pass endianness info to sparse)
+$ git merge c6x/for-linux-next
+Already up to date.
+Merging m68k/for-next (b2ab01132c13 m68k: Set default dma mask for platform devices)
+$ git merge m68k/for-next
+Merge made by the 'recursive' strategy.
+ arch/m68k/68000/timers.c             |  4 +++-
+ arch/m68k/apollo/config.c            |  8 +++++---
+ arch/m68k/configs/amiga_defconfig    | 20 ++++++++++++--------
+ arch/m68k/configs/apollo_defconfig   | 20 ++++++++++++--------
+ arch/m68k/configs/atari_defconfig    | 20 ++++++++++++--------
+ arch/m68k/configs/bvme6000_defconfig | 20 ++++++++++++--------
+ arch/m68k/configs/hp300_defconfig    | 20 ++++++++++++--------
+ arch/m68k/configs/mac_defconfig      | 20 ++++++++++++--------
+ arch/m68k/configs/multi_defconfig    | 20 ++++++++++++--------
+ arch/m68k/configs/mvme147_defconfig  | 20 ++++++++++++--------
+ arch/m68k/configs/mvme16x_defconfig  | 20 ++++++++++++--------
+ arch/m68k/configs/q40_defconfig      | 20 ++++++++++++--------
+ arch/m68k/configs/sun3_defconfig     | 20 ++++++++++++--------
+ arch/m68k/configs/sun3x_defconfig    | 20 ++++++++++++--------
+ arch/m68k/include/asm/delay.h        | 11 ++++++++---
+ arch/m68k/include/asm/uaccess_mm.h   | 16 +++++++++-------
+ arch/m68k/kernel/dma.c               | 10 ++++++++++
+ arch/m68k/kernel/signal.c            | 23 ++++++++++++-----------
+ arch/m68k/kernel/time.c              | 25 ++++++++++++++-----------
+ arch/m68k/mac/config.c               |  2 +-
+ arch/m68k/mm/kmap.c                  |  3 ++-
+ arch/m68k/mvme147/config.c           |  4 +++-
+ arch/m68k/mvme16x/config.c           |  4 +++-
+ arch/m68k/sun3/intersil.c            |  8 +++++---
+ arch/m68k/sun3x/time.c               |  8 +++++---
+ drivers/zorro/zorro.c                |  3 +++
+ 26 files changed, 227 insertions(+), 142 deletions(-)
+Merging m68knommu/for-next (082f55c45984 m68k: fix ColdFire PCI config reads and writes)
+$ git merge m68knommu/for-next
+Auto-merging arch/m68k/mm/kmap.c
+Merge made by the 'recursive' strategy.
+ arch/m68k/Kconfig.bus               |   8 +-
+ arch/m68k/coldfire/pci.c            | 114 +---------------
+ arch/m68k/include/asm/atarihw.h     |   1 +
+ arch/m68k/include/asm/io.h          |  10 +-
+ arch/m68k/include/asm/io_mm.h       |  98 ++------------
+ arch/m68k/include/asm/io_no.h       | 253 +++++++++++++++---------------------
+ arch/m68k/include/asm/kmap.h        |  80 ++++++++++++
+ arch/m68k/include/asm/nubus.h       |   1 +
+ arch/m68k/include/asm/q40_master.h  |   2 +-
+ arch/m68k/include/asm/raw_io.h      |  14 --
+ arch/m68k/include/asm/vga.h         |   9 ++
+ arch/m68k/include/asm/virtconvert.h |   2 +
+ arch/m68k/include/asm/zorro.h       |   1 +
+ arch/m68k/mm/kmap.c                 |   8 ++
+ 14 files changed, 230 insertions(+), 371 deletions(-)
+ create mode 100644 arch/m68k/include/asm/kmap.h
+Merging microblaze/next (a7a9e2f0410d microblaze: dts: replace 'linux,stdout-path' with 'stdout-path')
+$ git merge microblaze/next
+Removing arch/microblaze/kernel/early_printk.c
+Removing arch/microblaze/include/asm/prom.h
+Auto-merging arch/microblaze/include/asm/pci.h
+Merge made by the 'recursive' strategy.
+ arch/microblaze/Kconfig.debug         |   8 --
+ arch/microblaze/boot/Makefile         |  10 +-
+ arch/microblaze/boot/dts/system.dts   |   2 +-
+ arch/microblaze/include/asm/cpuinfo.h |   2 +-
+ arch/microblaze/include/asm/pci.h     |   1 -
+ arch/microblaze/include/asm/prom.h    |  27 -----
+ arch/microblaze/include/asm/setup.h   |   5 -
+ arch/microblaze/kernel/Makefile       |   1 -
+ arch/microblaze/kernel/early_printk.c | 184 ----------------------------------
+ arch/microblaze/kernel/misc.S         |  35 -------
+ arch/microblaze/kernel/platform.c     |   1 -
+ arch/microblaze/kernel/prom.c         |  82 ---------------
+ arch/microblaze/kernel/setup.c        |  17 +---
+ arch/microblaze/pci/indirect_pci.c    |   1 -
+ 14 files changed, 12 insertions(+), 364 deletions(-)
+ delete mode 100644 arch/microblaze/include/asm/prom.h
+ delete mode 100644 arch/microblaze/kernel/early_printk.c
+Merging mips/mips-next (ea4d340cbea4 Merge branches '4.15-fixes', '4.16-features' and 'octeon-3-net-mips-bits' into mips-next)
+$ git merge mips/mips-next
+Auto-merging drivers/staging/octeon/ethernet-tx.c
+Auto-merging drivers/staging/octeon/ethernet-rx.c
+Auto-merging drivers/staging/octeon/ethernet-defines.h
+Auto-merging arch/mips/include/asm/mipsregs.h
+Auto-merging arch/mips/cavium-octeon/Kconfig
+Merge made by the 'recursive' strategy.
+ arch/mips/cavium-octeon/Kconfig                    |  27 +-
+ arch/mips/cavium-octeon/Makefile                   |   1 +
+ arch/mips/cavium-octeon/resource-mgr.c             | 351 +++++++++++++++++++++
+ arch/mips/cavium-octeon/setup.c                    |  22 +-
+ arch/mips/include/asm/barrier.h                    |   2 +-
+ .../asm/mach-cavium-octeon/kernel-entry-init.h     |  20 +-
+ arch/mips/include/asm/mipsregs.h                   |   2 +
+ arch/mips/include/asm/octeon/octeon.h              |  32 +-
+ arch/mips/include/asm/processor.h                  |   2 +-
+ arch/mips/kernel/octeon_switch.S                   |   2 -
+ arch/mips/mm/tlbex.c                               |  29 +-
+ drivers/staging/octeon/ethernet-defines.h          |   6 -
+ drivers/staging/octeon/ethernet-rx.c               |  25 +-
+ drivers/staging/octeon/ethernet-tx.c               |  85 ++---
+ 14 files changed, 473 insertions(+), 133 deletions(-)
+ create mode 100644 arch/mips/cavium-octeon/resource-mgr.c
+Merging mips-james/mips-next (21325631f395 MIPS: memset.S: Reinstate delay slot indentation)
+$ git merge mips-james/mips-next
+Auto-merging lib/Makefile
+Auto-merging lib/Kconfig
+Auto-merging arch/riscv/Kconfig
+Removing arch/mips/lib/ucmpdi2.c
+Removing arch/mips/lib/lshrdi3.c
+Removing arch/mips/lib/cmpdi2.c
+Removing arch/mips/lib/ashrdi3.c
+Removing arch/mips/lib/ashldi3.c
+Auto-merging arch/mips/include/asm/mipsregs.h
+Auto-merging arch/mips/boot/dts/xilfpga/Makefile
+CONFLICT (content): Merge conflict in arch/mips/boot/dts/xilfpga/Makefile
+Auto-merging arch/mips/Kconfig
+Resolved 'arch/mips/boot/dts/xilfpga/Makefile' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master 64c57715599c] Merge remote-tracking branch 'mips-james/mips-next'
+$ git diff -M --stat --summary HEAD^..
+ .../bindings/watchdog/ingenic,jz4740-wdt.txt       |  7 +-
+ arch/mips/Kconfig                                  |  5 ++
+ arch/mips/bcm47xx/board.c                          |  2 +
+ arch/mips/bcm47xx/buttons.c                        |  9 +++
+ arch/mips/bcm47xx/leds.c                           | 11 ++-
+ arch/mips/boot/compressed/Makefile                 | 11 ++-
+ arch/mips/boot/dts/brcm/Makefile                   |  2 +-
+ arch/mips/boot/dts/cavium-octeon/Makefile          |  2 +-
+ arch/mips/boot/dts/ingenic/Makefile                |  2 +-
+ arch/mips/boot/dts/ingenic/jz4740.dtsi             |  8 ++
+ arch/mips/boot/dts/ingenic/jz4780.dtsi             |  5 +-
+ arch/mips/boot/dts/lantiq/Makefile                 |  2 +-
+ arch/mips/boot/dts/mscc/Makefile                   |  2 +-
+ arch/mips/boot/dts/mscc/ocelot.dtsi                | 88 +++++++++++++++++++++
+ arch/mips/boot/dts/mscc/ocelot_pcb123.dts          | 20 +++++
+ arch/mips/boot/dts/mti/Makefile                    |  2 +-
+ arch/mips/boot/dts/netlogic/Makefile               |  2 +-
+ arch/mips/boot/dts/pic32/Makefile                  |  2 +-
+ arch/mips/boot/dts/ralink/Makefile                 |  2 +-
+ arch/mips/configs/qi_lb60_defconfig                |  2 +
+ arch/mips/dec/time.c                               | 12 +--
+ arch/mips/include/asm/cpu-features.h               |  7 ++
+ arch/mips/include/asm/cpu.h                        |  2 +
+ arch/mips/include/asm/mach-bcm47xx/bcm47xx_board.h |  1 +
+ arch/mips/include/asm/mach-jz4740/platform.h       |  1 -
+ arch/mips/include/asm/mc146818-time.h              |  4 +-
+ arch/mips/include/asm/mipsregs.h                   |  5 ++
+ arch/mips/include/asm/time.h                       |  9 ---
+ arch/mips/jz4740/platform.c                        | 16 ----
+ arch/mips/jz4740/reset.c                           | 31 --------
+ arch/mips/kernel/cpu-probe.c                       | 12 +++
+ arch/mips/kernel/perf_event_mipsxx.c               | 91 ++++++++++------------
+ arch/mips/kernel/time.c                            | 15 ----
+ arch/mips/kernel/vpe.c                             |  2 +-
+ arch/mips/lasat/ds1603.c                           | 11 ++-
+ arch/mips/lasat/sysctl.c                           | 12 ++-
+ arch/mips/lib/Makefile                             |  3 +-
+ arch/mips/lib/ashldi3.c                            | 30 -------
+ arch/mips/lib/ashrdi3.c                            | 32 --------
+ arch/mips/lib/cmpdi2.c                             | 28 -------
+ arch/mips/lib/lshrdi3.c                            | 30 -------
+ arch/mips/lib/memset.S                             | 28 +++----
+ arch/mips/lib/ucmpdi2.c                            | 22 ------
+ arch/mips/loongson64/common/time.c                 |  2 +-
+ arch/mips/mm/sc-debugfs.c                          |  9 +--
+ arch/mips/mti-malta/malta-time.c                   |  2 +-
+ arch/mips/oprofile/op_model_mipsxx.c               |  2 -
+ arch/mips/sibyte/swarm/rtc_m41t81.c                |  8 +-
+ arch/mips/sibyte/swarm/rtc_xicor1241.c             |  8 +-
+ arch/mips/sibyte/swarm/setup.c                     | 18 +++--
+ arch/mips/sni/time.c                               |  6 --
+ arch/riscv/Kconfig                                 |  6 +-
+ drivers/firmware/broadcom/bcm47xx_nvram.c          |  2 +-
+ drivers/watchdog/jz4740_wdt.c                      | 42 ++++------
+ lib/Kconfig                                        | 12 +--
+ lib/Makefile                                       | 12 +--
+ lib/ucmpdi2.c                                      |  2 +-
+ 57 files changed, 333 insertions(+), 388 deletions(-)
+ delete mode 100644 arch/mips/lib/ashldi3.c
+ delete mode 100644 arch/mips/lib/ashrdi3.c
+ delete mode 100644 arch/mips/lib/cmpdi2.c
+ delete mode 100644 arch/mips/lib/lshrdi3.c
+ delete mode 100644 arch/mips/lib/ucmpdi2.c
+Merging nds32/next (3c6b01addf6e nds32: Fix compiler warning, Wstringop-overflow, in vdso.c)
+$ git merge nds32/next
+Auto-merging arch/nds32/Kconfig
+CONFLICT (content): Merge conflict in arch/nds32/Kconfig
+Resolved 'arch/nds32/Kconfig' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master a54256fe9739] Merge remote-tracking branch 'nds32/next'
+$ git diff -M --stat --summary HEAD^..
+Merging nios2/for-next (3d9644ef9a0f nios2: Use read_persistent_clock64() instead of read_persistent_clock())
+$ git merge nios2/for-next
+Already up to date.
+Merging openrisc/for-next (d56f3af9e801 openrisc: remove unused __ARCH_HAVE_MMU define)
+$ git merge openrisc/for-next
+Already up to date.
+Merging parisc-hd/for-next (8d40c754fbac parisc: Drop struct sigaction from not exported header file)
+$ git merge parisc-hd/for-next
+Auto-merging arch/parisc/Makefile
+Merge made by the 'recursive' strategy.
+ arch/parisc/Makefile             | 4 ----
+ arch/parisc/include/asm/signal.h | 8 --------
+ 2 files changed, 12 deletions(-)
+Merging powerpc/next (d647b210ac73 powerpc: Add a missing include header)
+$ git merge powerpc/next
+Removing tools/testing/selftests/powerpc/context_switch/cp_abort.c
+Removing tools/testing/selftests/powerpc/context_switch/Makefile
+Removing tools/testing/selftests/powerpc/context_switch/.gitignore
+Auto-merging arch/powerpc/platforms/powernv/setup.c
+Auto-merging arch/powerpc/platforms/Kconfig.cputype
+Removing arch/powerpc/platforms/83xx/sbc834x.c
+Auto-merging arch/powerpc/lib/feature-fixups.c
+Auto-merging arch/powerpc/kvm/book3s_hv_rmhandlers.S
+Auto-merging arch/powerpc/kvm/book3s_hv.c
+Auto-merging arch/powerpc/kvm/book3s_64_mmu_radix.c
+Auto-merging arch/powerpc/kernel/trace/ftrace.c
+Auto-merging arch/powerpc/kernel/module_64.c
+CONFLICT (content): Merge conflict in arch/powerpc/kernel/module_64.c
+Auto-merging arch/powerpc/kernel/asm-offsets.c
+Auto-merging arch/powerpc/include/asm/paca.h
+Removing arch/powerpc/include/asm/nohash/64/pgtable-64k.h
+Auto-merging arch/powerpc/include/asm/module.h
+Auto-merging arch/powerpc/include/asm/ftrace.h
+CONFLICT (content): Merge conflict in arch/powerpc/include/asm/ftrace.h
+Removing arch/powerpc/configs/83xx/sbc834x_defconfig
+Removing arch/powerpc/boot/dts/sbc8349.dts
+Auto-merging arch/powerpc/Kconfig
+Resolved 'arch/powerpc/include/asm/ftrace.h' using previous resolution.
+Recorded preimage for 'arch/powerpc/kernel/module_64.c'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+Recorded resolution for 'arch/powerpc/kernel/module_64.c'.
+[master fd52f9eedaf1] Merge remote-tracking branch 'powerpc/next'
+$ git diff -M --stat --summary HEAD^..
+ arch/powerpc/Kconfig                               |   2 +
+ arch/powerpc/boot/dts/fsl/t104xqds.dtsi            |   2 +-
+ arch/powerpc/boot/dts/sbc8349.dts                  | 331 --------------------
+ arch/powerpc/configs/83xx/sbc834x_defconfig        |  74 -----
+ arch/powerpc/configs/powernv_defconfig             | 108 ++++---
+ arch/powerpc/configs/wii_defconfig                 |  14 +
+ arch/powerpc/include/asm/asm-prototypes.h          |  17 +-
+ arch/powerpc/include/asm/book3s/64/hash-4k.h       |   8 +-
+ arch/powerpc/include/asm/book3s/64/hash-64k.h      |   7 +
+ arch/powerpc/include/asm/book3s/64/hash.h          |  10 -
+ arch/powerpc/include/asm/book3s/64/mmu.h           |   7 +-
+ arch/powerpc/include/asm/book3s/64/pgalloc.h       |  46 +--
+ arch/powerpc/include/asm/book3s/64/pgtable.h       |  20 +-
+ arch/powerpc/include/asm/book3s/64/radix-4k.h      |   3 +
+ arch/powerpc/include/asm/book3s/64/radix-64k.h     |   4 +
+ arch/powerpc/include/asm/checksum.h                |   2 +-
+ arch/powerpc/include/asm/ftrace.h                  |  31 +-
+ arch/powerpc/include/asm/hugetlb.h                 |   2 +-
+ arch/powerpc/include/asm/io.h                      |  10 +
+ arch/powerpc/include/asm/mmu-book3e.h              |   6 -
+ arch/powerpc/include/asm/module.h                  |   3 +
+ arch/powerpc/include/asm/mpc52xx.h                 |   6 +-
+ arch/powerpc/include/asm/nohash/32/pgtable.h       |  34 +-
+ arch/powerpc/include/asm/nohash/32/pte-40x.h       |   3 -
+ arch/powerpc/include/asm/nohash/64/pgalloc.h       |  95 ++----
+ arch/powerpc/include/asm/nohash/64/pgtable-64k.h   |  57 ----
+ arch/powerpc/include/asm/nohash/64/pgtable.h       |  39 +--
+ arch/powerpc/include/asm/nohash/pgtable.h          |  58 +---
+ arch/powerpc/include/asm/nohash/pte-book3e.h       |   6 -
+ arch/powerpc/include/asm/opal-api.h                |   1 +
+ arch/powerpc/include/asm/opal.h                    |   2 +
+ arch/powerpc/include/asm/paca.h                    |   1 +
+ arch/powerpc/include/asm/page.h                    |   1 +
+ arch/powerpc/include/asm/pmac_pfunc.h              |   1 +
+ arch/powerpc/include/asm/ppc-opcode.h              |   1 +
+ arch/powerpc/include/asm/ppc_asm.h                 |   6 +-
+ arch/powerpc/include/asm/pte-common.h              |   8 -
+ arch/powerpc/include/asm/rheap.h                   |   3 +
+ arch/powerpc/include/asm/syscalls.h                |   2 +-
+ arch/powerpc/include/asm/systbl.h                  |   6 +-
+ arch/powerpc/include/asm/trace.h                   |   7 +-
+ arch/powerpc/include/asm/xmon.h                    |   2 +-
+ arch/powerpc/include/asm/xor.h                     |  12 +-
+ arch/powerpc/include/asm/xor_altivec.h             |  19 ++
+ arch/powerpc/kernel/asm-offsets.c                  |   1 +
+ arch/powerpc/kernel/btext.c                        |  10 +-
+ arch/powerpc/kernel/fadump.c                       |  40 ++-
+ arch/powerpc/kernel/head_8xx.S                     |   2 +-
+ arch/powerpc/kernel/hw_breakpoint.c                |   4 +-
+ arch/powerpc/kernel/irq.c                          |   2 +-
+ arch/powerpc/kernel/machine_kexec.c                |   2 +
+ arch/powerpc/kernel/machine_kexec_64.c             |   8 +-
+ arch/powerpc/kernel/misc.S                         |  19 +-
+ arch/powerpc/kernel/module_32.c                    |   4 +-
+ arch/powerpc/kernel/module_64.c                    |  43 ++-
+ arch/powerpc/kernel/pci_32.c                       |   7 +-
+ arch/powerpc/kernel/pci_64.c                       |   4 +-
+ arch/powerpc/kernel/ppc_save_regs.S                |   4 +
+ arch/powerpc/kernel/prom.c                         |  23 +-
+ arch/powerpc/kernel/prom_init.c                    | 116 +++----
+ arch/powerpc/kernel/ptrace.c                       |   1 +
+ arch/powerpc/kernel/rtas-proc.c                    |   2 +-
+ arch/powerpc/kernel/rtas.c                         |   3 +-
+ arch/powerpc/kernel/setup-common.c                 |   6 -
+ arch/powerpc/kernel/setup.h                        |   6 +
+ arch/powerpc/kernel/setup_64.c                     |   7 +
+ arch/powerpc/kernel/signal.h                       |   6 +-
+ arch/powerpc/kernel/signal_32.c                    |  40 ++-
+ arch/powerpc/kernel/signal_64.c                    |  15 +-
+ arch/powerpc/kernel/smp.c                          |  12 +
+ arch/powerpc/kernel/stacktrace.c                   | 119 ++++++-
+ arch/powerpc/kernel/sys_ppc32.c                    |   9 -
+ arch/powerpc/kernel/systbl.S                       |   2 +-
+ arch/powerpc/kernel/systbl_chk.c                   |   2 +-
+ arch/powerpc/kernel/systbl_chk.sh                  |   4 +-
+ arch/powerpc/kernel/tau_6xx.c                      |  15 +-
+ arch/powerpc/kernel/trace/ftrace.c                 | 212 ++++++++++---
+ arch/powerpc/kernel/trace/ftrace_32.S              |  20 --
+ arch/powerpc/kernel/trace/ftrace_64.S              |  29 --
+ arch/powerpc/kernel/trace/ftrace_64_mprofile.S     |  88 +++++-
+ arch/powerpc/kernel/trace/ftrace_64_pg.S           |   6 +-
+ arch/powerpc/kernel/vecemu.c                       |   1 +
+ arch/powerpc/kernel/watchdog.c                     |  32 +-
+ arch/powerpc/kvm/book3s_64_mmu_radix.c             |  36 ++-
+ arch/powerpc/kvm/book3s_hv.c                       |   4 +
+ arch/powerpc/kvm/book3s_hv_rmhandlers.S            |   3 +
+ arch/powerpc/lib/feature-fixups-test.S             |  42 ++-
+ arch/powerpc/lib/feature-fixups.c                  |  23 +-
+ arch/powerpc/lib/sstep.c                           |  11 +-
+ arch/powerpc/lib/xor_vmx_glue.c                    |   1 +
+ arch/powerpc/mm/fault.c                            |  76 +++--
+ arch/powerpc/mm/hash_utils_64.c                    |   9 +-
+ arch/powerpc/mm/hugetlbpage.c                      |   7 +
+ arch/powerpc/mm/mmu_context_book3s64.c             |  39 ++-
+ arch/powerpc/mm/pgtable-book3s64.c                 | 267 +++++++++++++++-
+ arch/powerpc/mm/pgtable-hash64.c                   |   8 +-
+ arch/powerpc/mm/pgtable-radix.c                    |   5 +-
+ arch/powerpc/mm/pgtable_64.c                       | 171 -----------
+ arch/powerpc/mm/ppc_mmu_32.c                       |   2 +-
+ arch/powerpc/mm/subpage-prot.c                     |   4 +-
+ arch/powerpc/mm/tlb_hash32.c                       |  10 +-
+ arch/powerpc/perf/imc-pmu.c                        |   4 +-
+ arch/powerpc/platforms/83xx/Kconfig                |   7 -
+ arch/powerpc/platforms/83xx/Makefile               |   1 -
+ arch/powerpc/platforms/83xx/sbc834x.c              |  73 -----
+ arch/powerpc/platforms/Kconfig.cputype             |   4 +
+ arch/powerpc/platforms/cell/spu_callbacks.c        |   2 +-
+ arch/powerpc/platforms/cell/spu_syscalls.c         |   3 +-
+ arch/powerpc/platforms/cell/spufs/file.c           |  33 +-
+ arch/powerpc/platforms/chrp/pci.c                  |  12 +-
+ arch/powerpc/platforms/chrp/setup.c                |  12 +-
+ arch/powerpc/platforms/chrp/time.c                 |   6 +-
+ arch/powerpc/platforms/embedded6xx/flipper-pic.c   |   8 -
+ arch/powerpc/platforms/embedded6xx/hlwd-pic.c      |   7 +-
+ arch/powerpc/platforms/powermac/bootx_init.c       |   4 +-
+ arch/powerpc/platforms/powermac/pci.c              |   4 +-
+ arch/powerpc/platforms/powermac/setup.c            |   9 +-
+ arch/powerpc/platforms/powermac/smp.c              |   1 -
+ arch/powerpc/platforms/powernv/memtrace.c          |   2 +-
+ arch/powerpc/platforms/powernv/npu-dma.c           |   5 +-
+ arch/powerpc/platforms/powernv/opal-irqchip.c      |   2 +-
+ arch/powerpc/platforms/powernv/opal-sensor.c       |  53 ++++
+ arch/powerpc/platforms/powernv/opal-wrappers.S     |   1 +
+ arch/powerpc/platforms/powernv/pci-ioda.c          |  45 ++-
+ arch/powerpc/platforms/powernv/setup.c             |  10 +-
+ arch/powerpc/platforms/pseries/hvCall_inst.c       |   2 +-
+ arch/powerpc/platforms/pseries/kexec.c             |   7 +-
+ arch/powerpc/platforms/pseries/lpar.c              |   3 +-
+ arch/powerpc/sysdev/xive/spapr.c                   |  88 +++++-
+ arch/powerpc/xmon/nonstdio.h                       |   8 +-
+ arch/powerpc/xmon/spu-dis.c                        |  18 +-
+ arch/powerpc/xmon/xmon.c                           | 234 +++++++-------
+ drivers/hwmon/ibmpowernv.c                         |   9 +-
+ drivers/macintosh/via-pmu.c                        |  18 +-
+ drivers/macintosh/windfarm_pm121.c                 |   2 +-
+ drivers/macintosh/windfarm_pm81.c                  |   2 +-
+ drivers/macintosh/windfarm_pm91.c                  |   2 +-
+ kernel/sys_ni.c                                    |   2 +-
+ kernel/trace/Kconfig                               |   6 +-
+ tools/testing/selftests/powerpc/Makefile           |   1 -
+ .../testing/selftests/powerpc/alignment/.gitignore |   1 +
+ .../selftests/powerpc/benchmarks/exec_target.c     |   7 +-
+ .../selftests/powerpc/context_switch/.gitignore    |   1 -
+ .../selftests/powerpc/context_switch/Makefile      |   5 -
+ .../selftests/powerpc/context_switch/cp_abort.c    | 110 -------
+ tools/testing/selftests/powerpc/ptrace/.gitignore  |   1 +
+ tools/testing/selftests/powerpc/ptrace/Makefile    |   2 +-
+ .../selftests/powerpc/ptrace/ptrace-hwbreak.c      | 342 +++++++++++++++++++++
+ tools/testing/selftests/powerpc/tm/.gitignore      |   1 +
+ 149 files changed, 2138 insertions(+), 1763 deletions(-)
+ delete mode 100644 arch/powerpc/boot/dts/sbc8349.dts
+ delete mode 100644 arch/powerpc/configs/83xx/sbc834x_defconfig
+ delete mode 100644 arch/powerpc/include/asm/nohash/64/pgtable-64k.h
+ create mode 100644 arch/powerpc/include/asm/xor_altivec.h
+ delete mode 100644 arch/powerpc/platforms/83xx/sbc834x.c
+ delete mode 100644 tools/testing/selftests/powerpc/context_switch/.gitignore
+ delete mode 100644 tools/testing/selftests/powerpc/context_switch/Makefile
+ delete mode 100644 tools/testing/selftests/powerpc/context_switch/cp_abort.c
+ create mode 100644 tools/testing/selftests/powerpc/ptrace/ptrace-hwbreak.c
+Merging powerpc-pkey/topic/pkey (d86a4ba02fc6 powerpc/pkeys: Drop private VM_PKEY definitions)
+$ git merge powerpc-pkey/topic/pkey
+Auto-merging include/linux/mm.h
+Auto-merging arch/x86/include/asm/pkeys.h
+Auto-merging arch/x86/include/asm/mmu_context.h
+Merge made by the 'recursive' strategy.
+ arch/powerpc/include/asm/mmu_context.h |  5 -----
+ arch/powerpc/include/asm/pkeys.h       | 13 -------------
+ arch/x86/include/asm/mmu_context.h     | 15 ---------------
+ arch/x86/include/asm/pkeys.h           | 13 +++++++++++++
+ arch/x86/kernel/setup.c                |  8 --------
+ fs/proc/task_mmu.c                     | 13 +++++++------
+ include/linux/mm.h                     | 14 ++++++++++----
+ include/linux/pkeys.h                  | 13 +++++++++++--
+ 8 files changed, 41 insertions(+), 53 deletions(-)
+Merging fsl/next (c095ff93f901 powerpc/sysdev: change CPM GPIO to platform_device)
+$ git merge fsl/next
+Already up to date.
+Merging risc-v/for-next (67b8d5c70812 Linux 4.17-rc5)
+$ git merge risc-v/for-next
+Already up to date.
+Merging s390/features (89a0c0ec0d2e s390/zcrypt: Fix CCA and EP11 CPRB processing failure memory leak.)
+$ git merge s390/features
+Auto-merging drivers/s390/block/dasd.c
+Auto-merging arch/s390/Kconfig
+Merge made by the 'recursive' strategy.
+ arch/s390/Kconfig                     |   4 ++
+ arch/s390/crypto/arch_random.c        | 103 ++++++++++++++++++++++++++++++++--
+ arch/s390/include/asm/archrandom.h    |  13 +----
+ arch/s390/include/asm/ccwdev.h        |   1 +
+ arch/s390/include/asm/ccwgroup.h      |  10 ++++
+ arch/s390/include/asm/cpu_mf.h        |   6 +-
+ arch/s390/include/asm/pnet.h          |  23 ++++++++
+ arch/s390/kernel/Makefile             |  27 +++++----
+ arch/s390/kernel/early.c              |  80 --------------------------
+ arch/s390/kernel/early_nobss.c        |  65 +++++++++++++++++++++
+ arch/s390/kernel/entry.h              |   1 +
+ arch/s390/kernel/head64.S             |   8 ++-
+ arch/s390/kernel/nospec-branch.c      |   4 +-
+ arch/s390/kernel/vmlinux.lds.S        |   4 +-
+ arch/s390/lib/Makefile                |   3 +
+ arch/s390/net/Makefile                |   1 +
+ arch/s390/net/pnet.c                  |  80 ++++++++++++++++++++++++++
+ arch/s390/scripts/Makefile.chkbss     |  14 +++++
+ drivers/s390/block/dasd.c             |  31 +++++-----
+ drivers/s390/char/Makefile            |   3 +
+ drivers/s390/cio/ccwgroup.c           |   6 ++
+ drivers/s390/cio/device_ops.c         |  31 ++++++++++
+ drivers/s390/crypto/ap_bus.h          |  17 ++++--
+ drivers/s390/crypto/zcrypt_api.c      |  15 ++++-
+ drivers/s390/crypto/zcrypt_msgtype6.c |  51 +++++++----------
+ 25 files changed, 435 insertions(+), 166 deletions(-)
+ create mode 100644 arch/s390/include/asm/pnet.h
+ create mode 100644 arch/s390/kernel/early_nobss.c
+ create mode 100644 arch/s390/net/pnet.c
+ create mode 100644 arch/s390/scripts/Makefile.chkbss
+Merging sparc-next/master (17dec0a94915 Merge branch 'userns-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiederm/user-namespace)
+$ git merge sparc-next/master
+Already up to date.
+Merging sh/for-next (ac21fc2dcb40 sh: switch to NO_BOOTMEM)
+$ git merge sh/for-next
+Already up to date.
+Merging uml/linux-next (e40238dedb48 Fix vector raw inintialization logic)
+$ git merge uml/linux-next
+Already up to date.
+Merging xtensa/xtensa-for-next (ae6063c83634 Merge branch 'xtensa-fixes' into xtensa-for-next)
+$ git merge xtensa/xtensa-for-next
+Auto-merging arch/xtensa/kernel/pci-dma.c
+Merge made by the 'recursive' strategy.
+ arch/xtensa/kernel/pci-dma.c | 10 ++++++++--
+ 1 file changed, 8 insertions(+), 2 deletions(-)
+Merging fscrypt/master (e1cc40e5d42a fscrypt: log the crypto algorithm implementations)
+$ git merge fscrypt/master
+Merge made by the 'recursive' strategy.
+ Documentation/filesystems/fscrypt.rst |  10 ++
+ fs/crypto/crypto.c                    |  47 ++++--
+ fs/crypto/fname.c                     |  32 ++--
+ fs/crypto/fscrypt_private.h           |  23 +--
+ fs/crypto/hooks.c                     |   5 +-
+ fs/crypto/keyinfo.c                   | 286 ++++++++++++++++++++--------------
+ fs/ext4/super.c                       |   8 +-
+ fs/f2fs/super.c                       |   8 +-
+ fs/ubifs/crypto.c                     |  10 +-
+ include/linux/fs.h                    |   4 +-
+ include/linux/fscrypt_notsupp.h       |  10 --
+ include/linux/fscrypt_supp.h          |  16 +-
+ include/uapi/linux/fs.h               |   2 +
+ 13 files changed, 248 insertions(+), 213 deletions(-)
+Merging befs/for-next (55d945e2e4aa fs: befs: btree: Fixed some coding standard issues)
+$ git merge befs/for-next
+Merge made by the 'recursive' strategy.
+ fs/befs/btree.c | 97 ++++++++++++++++++++++++++++++---------------------------
+ 1 file changed, 51 insertions(+), 46 deletions(-)
+Merging btrfs/next (7c9a09f56a91 btrfs: don't use async helpers for crcs when under IO limits)
+$ git merge btrfs/next
+Auto-merging kernel/cgroup/cgroup.c
+Auto-merging include/linux/fs.h
+Auto-merging include/linux/buffer_head.h
+Auto-merging include/linux/backing-dev.h
+Auto-merging fs/ext4/super.c
+Auto-merging fs/ext4/inode.c
+Auto-merging fs/ext2/super.c
+Auto-merging fs/ext2/inode.c
+Auto-merging fs/buffer.c
+Auto-merging fs/btrfs/super.c
+Auto-merging fs/btrfs/ioctl.c
+Auto-merging fs/btrfs/inode.c
+Auto-merging fs/btrfs/disk-io.c
+Auto-merging fs/btrfs/check-integrity.c
+Auto-merging fs/block_dev.c
+Auto-merging block/blk-cgroup.c
+Merge made by the 'recursive' strategy.
+ block/blk-cgroup.c          |  1 +
+ fs/block_dev.c              |  3 +--
+ fs/btrfs/check-integrity.c  |  2 +-
+ fs/btrfs/disk-io.c          |  4 ++++
+ fs/btrfs/inode.c            | 15 ++++++++++++---
+ fs/btrfs/ioctl.c            |  8 +++++++-
+ fs/btrfs/super.c            |  1 -
+ fs/buffer.c                 | 42 ++++++++++++++++++++++++++++++++++--------
+ fs/ext2/inode.c             |  3 ++-
+ fs/ext2/super.c             |  1 -
+ fs/ext4/inode.c             |  4 +++-
+ fs/ext4/super.c             |  2 --
+ include/linux/backing-dev.h |  2 +-
+ include/linux/buffer_head.h |  3 +++
+ include/linux/fs.h          |  3 ++-
+ kernel/cgroup/cgroup.c      |  1 +
+ 16 files changed, 72 insertions(+), 23 deletions(-)
+$ git am -3 ../patches/0001-f2fs-fixup-for-cgroup-writeback-change.patch
+Applying: f2fs: fixup for cgroup/writeback change
+$ git reset HEAD^
+Unstaged changes after reset:
+M	fs/f2fs/inode.c
+M	fs/f2fs/super.c
+$ git add -A .
+$ git commit -v -a --amend
+[master a2bbb91e5326] Merge remote-tracking branch 'btrfs/next'
+ Date: Thu May 31 09:59:05 2018 +1000
+Merging btrfs-kdave/for-next (85b1781f8242 Merge branch 'for-next-next-v4.18-20180529' into for-next-20180529)
+$ git merge btrfs-kdave/for-next
+Auto-merging fs/btrfs/super.c
+Auto-merging fs/btrfs/ioctl.c
+Auto-merging fs/btrfs/inode.c
+Auto-merging fs/btrfs/disk-io.c
+Merge made by the 'recursive' strategy.
+ fs/btrfs/btrfs_inode.h                 |   22 +-
+ fs/btrfs/compression.c                 |   28 +-
+ fs/btrfs/compression.h                 |    2 +
+ fs/btrfs/ctree.c                       |  123 +--
+ fs/btrfs/ctree.h                       |   78 +-
+ fs/btrfs/delayed-inode.c               |    9 +-
+ fs/btrfs/delayed-ref.c                 |  275 +++----
+ fs/btrfs/delayed-ref.h                 |    5 +-
+ fs/btrfs/dev-replace.c                 |  150 +++-
+ fs/btrfs/disk-io.c                     |  391 +++++----
+ fs/btrfs/extent-tree.c                 |  374 +++++----
+ fs/btrfs/extent_io.c                   |  213 +++--
+ fs/btrfs/extent_io.h                   |   20 +-
+ fs/btrfs/extent_map.c                  |    6 +-
+ fs/btrfs/extent_map.h                  |    3 +-
+ fs/btrfs/file.c                        |   56 +-
+ fs/btrfs/free-space-cache.c            |   28 +-
+ fs/btrfs/free-space-cache.h            |    3 +-
+ fs/btrfs/free-space-tree.c             |  192 +++--
+ fs/btrfs/free-space-tree.h             |    8 -
+ fs/btrfs/inode.c                       | 1366 +++++++++++++++-----------------
+ fs/btrfs/ioctl.c                       | 1245 ++++++++++++++++++++---------
+ fs/btrfs/locking.c                     |   34 +-
+ fs/btrfs/lzo.c                         |   85 +-
+ fs/btrfs/ordered-data.c                |  137 +---
+ fs/btrfs/ordered-data.h                |   20 +-
+ fs/btrfs/print-tree.c                  |   21 +
+ fs/btrfs/props.c                       |    2 +-
+ fs/btrfs/qgroup.c                      |   69 +-
+ fs/btrfs/relocation.c                  |   36 +-
+ fs/btrfs/scrub.c                       |    1 +
+ fs/btrfs/send.c                        |   46 +-
+ fs/btrfs/super.c                       |   11 +-
+ fs/btrfs/sysfs.c                       |   52 +-
+ fs/btrfs/sysfs.h                       |    4 +-
+ fs/btrfs/tests/btrfs-tests.c           |    4 +-
+ fs/btrfs/tests/btrfs-tests.h           |    6 +-
+ fs/btrfs/tests/extent-buffer-tests.c   |   56 +-
+ fs/btrfs/tests/extent-io-tests.c       |   75 +-
+ fs/btrfs/tests/extent-map-tests.c      |   90 ++-
+ fs/btrfs/tests/free-space-tests.c      |  177 +++--
+ fs/btrfs/tests/free-space-tree-tests.c |  129 +--
+ fs/btrfs/tests/inode-tests.c           |  312 ++++----
+ fs/btrfs/tests/qgroup-tests.c          |  100 +--
+ fs/btrfs/transaction.c                 |   15 +-
+ fs/btrfs/tree-log.c                    |  194 +----
+ fs/btrfs/uuid-tree.c                   |   10 +-
+ fs/btrfs/volumes.c                     |  660 ++++++++-------
+ fs/btrfs/volumes.h                     |   25 +-
+ fs/btrfs/zstd.c                        |   19 +-
+ include/trace/events/btrfs.h           |  324 ++++----
+ include/uapi/linux/btrfs.h             |  110 +++
+ 52 files changed, 4081 insertions(+), 3340 deletions(-)
+Merging ceph/master (fc218544fbc8 ceph: fix iov_iter issues in ceph_direct_read_write())
+$ git merge ceph/master
+Already up to date.
+Merging cifs/for-next (5c5a41be89e8 cifs: add debug output to show nocase mount option)
+$ git merge cifs/for-next
+Merge made by the 'recursive' strategy.
+ fs/cifs/Makefile       |   7 +-
+ fs/cifs/cifs_debug.c   |  43 ++++---
+ fs/cifs/cifs_debug.h   |   2 +-
+ fs/cifs/cifs_fs_sb.h   |   1 +
+ fs/cifs/cifsfs.c       |  48 ++++++-
+ fs/cifs/cifsfs.h       |   2 +-
+ fs/cifs/cifsglob.h     |  40 ++++--
+ fs/cifs/cifsproto.h    |  10 +-
+ fs/cifs/cifssmb.c      |   6 +
+ fs/cifs/connect.c      |  47 ++++++-
+ fs/cifs/inode.c        |  13 +-
+ fs/cifs/misc.c         |   5 +-
+ fs/cifs/netmisc.c      |   2 +-
+ fs/cifs/readdir.c      |   6 +-
+ fs/cifs/smb2inode.c    |  43 ++++---
+ fs/cifs/smb2maperror.c |   9 +-
+ fs/cifs/smb2misc.c     |  62 ++++-----
+ fs/cifs/smb2ops.c      | 124 ++++++++++++++----
+ fs/cifs/smb2pdu.c      | 148 +++++++++++++++------
+ fs/cifs/smb2pdu.h      |  19 ++-
+ fs/cifs/smb2proto.h    |   6 +-
+ fs/cifs/trace.c        |  18 +++
+ fs/cifs/trace.h        | 343 +++++++++++++++++++++++++++++++++++++++++++++++++
+ 23 files changed, 836 insertions(+), 168 deletions(-)
+ create mode 100644 fs/cifs/trace.c
+ create mode 100644 fs/cifs/trace.h
+Merging configfs/for-next (6ace4f6bbcfd RDMA/cma: make config_item_type const)
+$ git merge configfs/for-next
+Already up to date.
+Merging ecryptfs/next (e86281e700cc eCryptfs: don't pass up plaintext names when using filename encryption)
+$ git merge ecryptfs/next
+Already up to date.
+Merging ext3/for_next (20b67d6401f1 Merge fsnotify cleanups from Amir into for_next.)
+$ git merge ext3/for_next
+Auto-merging include/linux/fs.h
+Merge made by the 'recursive' strategy.
+ fs/inode.c                           |   2 +-
+ fs/mount.h                           |  11 +-
+ fs/notify/dnotify/dnotify.c          |  16 +--
+ fs/notify/fanotify/fanotify.c        |  46 +++----
+ fs/notify/fanotify/fanotify_user.c   |  98 +++++--------
+ fs/notify/fdinfo.c                   |  12 +-
+ fs/notify/fsnotify.c                 | 160 ++++++++++++---------
+ fs/notify/fsnotify.h                 |  14 +-
+ fs/notify/group.c                    |   2 +-
+ fs/notify/inotify/inotify.h          |   2 -
+ fs/notify/inotify/inotify_fsnotify.c |   6 +-
+ fs/notify/inotify/inotify_user.c     |  16 ++-
+ fs/notify/mark.c                     | 140 +++++++++----------
+ fs/udf/Kconfig                       |   6 +-
+ fs/udf/super.c                       |  12 +-
+ fs/udf/udfdecl.h                     |   3 +-
+ fs/udf/unicode.c                     | 260 +++++++++++++++++------------------
+ include/linux/fs.h                   |  20 ++-
+ include/linux/fsnotify_backend.h     | 109 ++++++++++++---
+ include/linux/fsnotify_obj.h         |  18 +++
+ kernel/audit_fsnotify.c              |   5 +-
+ kernel/audit_tree.c                  |  22 ++-
+ kernel/audit_watch.c                 |   7 +-
+ kernel/auditsc.c                     |   4 +-
+ 24 files changed, 527 insertions(+), 464 deletions(-)
+ create mode 100644 include/linux/fsnotify_obj.h
+Merging ext4/dev (4f2f76f75143 ext4: fix fencepost error in check for inode count overflow during resize)
+$ git merge ext4/dev
+Auto-merging include/linux/fs.h
+Auto-merging fs/ext4/super.c
+Auto-merging fs/ext4/inode.c
+Merge made by the 'recursive' strategy.
+ fs/ext4/balloc.c         | 33 ++++++-----------
+ fs/ext4/ext4.h           |  7 ++++
+ fs/ext4/extents_status.c |  3 +-
+ fs/ext4/file.c           | 93 ++++++++++++++++++++++++++++--------------------
+ fs/ext4/fsmap.c          |  4 +--
+ fs/ext4/ialloc.c         | 35 ++++++++----------
+ fs/ext4/indirect.c       | 14 +++++---
+ fs/ext4/inline.c         |  6 ++++
+ fs/ext4/inode.c          | 77 +++++++++++++++++++++++++++------------
+ fs/ext4/mballoc.c        | 23 ++++++------
+ fs/ext4/resize.c         |  2 +-
+ fs/ext4/super.c          | 68 +++++++++++++++++++++++++++--------
+ fs/ext4/xattr.c          |  2 +-
+ fs/ext4/xattr_security.c |  2 +-
+ fs/jbd2/journal.c        | 20 +++++------
+ fs/jbd2/revoke.c         | 12 +++----
+ fs/jbd2/transaction.c    |  6 ++--
+ include/linux/fs.h       |  5 +++
+ 18 files changed, 245 insertions(+), 167 deletions(-)
+Merging f2fs/dev (cba608493d3d f2fs: turn down IO priority of discard from background)
+$ git merge f2fs/dev
+Auto-merging include/linux/fscrypt_supp.h
+Auto-merging include/linux/fscrypt_notsupp.h
+Auto-merging fs/f2fs/super.c
+Auto-merging fs/f2fs/namei.c
+CONFLICT (content): Merge conflict in fs/f2fs/namei.c
+Auto-merging fs/crypto/fscrypt_private.h
+Auto-merging fs/crypto/crypto.c
+Resolved 'fs/f2fs/namei.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master 83ed385ecab7] Merge remote-tracking branch 'f2fs/dev'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/ABI/testing/sysfs-fs-f2fs |   1 +
+ Documentation/filesystems/f2fs.txt      |  16 ++-
+ fs/crypto/bio.c                         |  35 +++--
+ fs/crypto/crypto.c                      |   8 +-
+ fs/crypto/fscrypt_private.h             |   1 -
+ fs/ext4/readpage.c                      |   2 +-
+ fs/f2fs/checkpoint.c                    |   2 +-
+ fs/f2fs/data.c                          | 209 +++++++++++++++++----------
+ fs/f2fs/f2fs.h                          |  34 ++---
+ fs/f2fs/file.c                          |   6 +-
+ fs/f2fs/gc.c                            |   7 +-
+ fs/f2fs/inline.c                        |   3 +-
+ fs/f2fs/node.c                          |   3 +-
+ fs/f2fs/segment.c                       | 247 ++++++++++++++++----------------
+ fs/f2fs/super.c                         |  10 ++
+ fs/f2fs/sysfs.c                         |   3 +
+ include/linux/fscrypt_notsupp.h         |  13 +-
+ include/linux/fscrypt_supp.h            |   5 +-
+ 18 files changed, 359 insertions(+), 246 deletions(-)
+Merging fuse/for-next (5ba24197b94d fuse: add writeback documentation)
+$ git merge fuse/for-next
+Auto-merging fs/fuse/inode.c
+Merge made by the 'recursive' strategy.
+ Documentation/filesystems/fuse-io.txt | 38 +++++++++++++++++++++++++++++
+ fs/fuse/acl.c                         |  4 ++--
+ fs/fuse/control.c                     |  2 +-
+ fs/fuse/cuse.c                        | 11 ++++++---
+ fs/fuse/dev.c                         | 40 +++++++++++++++----------------
+ fs/fuse/dir.c                         | 45 +++++++++++++++++++++++++----------
+ fs/fuse/fuse_i.h                      | 14 +++++++++--
+ fs/fuse/inode.c                       | 40 +++++++++++++++++++------------
+ include/uapi/linux/fuse.h             |  7 +++++-
+ kernel/user_namespace.c               |  1 +
+ 10 files changed, 145 insertions(+), 57 deletions(-)
+ create mode 100644 Documentation/filesystems/fuse-io.txt
+Merging jfs/jfs-next (86313903430d MAINTAINERS: fix jfs tree location)
+$ git merge jfs/jfs-next
+Already up to date.
+Merging nfs/linux-next (036db8bd9637 Merge branch 'for-4.17-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/libata)
+$ git merge nfs/linux-next
+Already up to date.
+Merging nfs-anna/linux-next (ffbfa2ce88f1 sunrpc: Fix latency trace point crashes)
+$ git merge nfs-anna/linux-next
+Merge made by the 'recursive' strategy.
+Merging nfsd/nfsd-next (28771950c592 svcrdma: Fix incorrect return value/type in svc_rdma_post_recvs)
+$ git merge nfsd/nfsd-next
+Auto-merging net/sunrpc/xprtrdma/xprt_rdma.h
+Auto-merging net/sunrpc/xprtrdma/verbs.c
+Auto-merging net/sunrpc/xprtrdma/frwr_ops.c
+Auto-merging net/sunrpc/xprtrdma/fmr_ops.c
+Merge made by the 'recursive' strategy.
+ fs/nfsd/blocklayout.c                      |  18 +-
+ fs/nfsd/cache.h                            |   5 -
+ fs/nfsd/nfs4state.c                        |  20 +-
+ fs/nfsd/nfs4xdr.c                          |  45 ++-
+ fs/nfsd/nfscache.c                         |   6 +-
+ fs/nfsd/nfsd.h                             |   1 +
+ include/linux/nfs4.h                       |   8 +
+ include/linux/sunrpc/svc_rdma.h            |  95 ++---
+ include/trace/events/rpcrdma.h             | 584 ++++++++++++++++++++++++++++-
+ net/sunrpc/xprtrdma/backchannel.c          |   2 +
+ net/sunrpc/xprtrdma/fmr_ops.c              |   3 +
+ net/sunrpc/xprtrdma/frwr_ops.c             |   2 +
+ net/sunrpc/xprtrdma/module.c               |   4 +-
+ net/sunrpc/xprtrdma/rpc_rdma.c             |   7 +-
+ net/sunrpc/xprtrdma/svc_rdma.c             |   3 +-
+ net/sunrpc/xprtrdma/svc_rdma_backchannel.c |  54 ++-
+ net/sunrpc/xprtrdma/svc_rdma_recvfrom.c    | 439 ++++++++++++++++------
+ net/sunrpc/xprtrdma/svc_rdma_rw.c          | 133 +++----
+ net/sunrpc/xprtrdma/svc_rdma_sendto.c      | 510 ++++++++++++++++---------
+ net/sunrpc/xprtrdma/svc_rdma_transport.c   | 481 ++++--------------------
+ net/sunrpc/xprtrdma/transport.c            |   4 +
+ net/sunrpc/xprtrdma/verbs.c                |   1 +
+ net/sunrpc/xprtrdma/xprt_rdma.h            |   2 -
+ 23 files changed, 1536 insertions(+), 891 deletions(-)
+Merging orangefs/for-next (8d720dc81d4d orangefs: formatting cleanups)
+$ git merge orangefs/for-next
+Auto-merging fs/orangefs/namei.c
+Merge made by the 'recursive' strategy.
+ fs/orangefs/devorangefs-req.c | 17 +++++++------
+ fs/orangefs/file.c            |  4 +--
+ fs/orangefs/inode.c           | 57 ++++++++++++++++++++++++++-----------------
+ fs/orangefs/namei.c           |  7 ++++++
+ fs/orangefs/orangefs-bufmap.c |  4 +--
+ fs/orangefs/orangefs-kernel.h |  1 -
+ fs/orangefs/orangefs-utils.c  | 30 +++++++++++------------
+ fs/orangefs/super.c           | 13 +++++-----
+ 8 files changed, 78 insertions(+), 55 deletions(-)
+Merging overlayfs/overlayfs-next (b0ee6e9a4c53 ovl: Enable metadata only feature)
+$ git merge overlayfs/overlayfs-next
+Auto-merging include/uapi/linux/fs.h
+Auto-merging include/linux/fs.h
+Auto-merging fs/inode.c
+Auto-merging fs/btrfs/ioctl.c
+CONFLICT (content): Merge conflict in fs/btrfs/ioctl.c
+Auto-merging fs/btrfs/ctree.h
+Resolved 'fs/btrfs/ioctl.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master 7f67c675c615] Merge remote-tracking branch 'overlayfs/overlayfs-next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/filesystems/Locking       |   4 +-
+ Documentation/filesystems/overlayfs.txt |  97 ++++--
+ Documentation/filesystems/vfs.txt       |  19 +-
+ fs/btrfs/ctree.h                        |   5 +-
+ fs/btrfs/ioctl.c                        |   7 +-
+ fs/file_table.c                         |  13 +-
+ fs/inode.c                              | 210 +++++--------
+ fs/internal.h                           |  17 +-
+ fs/ioctl.c                              |   1 +
+ fs/locks.c                              |  20 +-
+ fs/namei.c                              |   2 +-
+ fs/namespace.c                          |  69 +----
+ fs/ocfs2/file.c                         |  10 +-
+ fs/open.c                               |  74 ++---
+ fs/overlayfs/Kconfig                    |  40 +++
+ fs/overlayfs/Makefile                   |   4 +-
+ fs/overlayfs/copy_up.c                  | 273 +++++++++-------
+ fs/overlayfs/dir.c                      | 312 +++++++++++++------
+ fs/overlayfs/export.c                   |  11 +-
+ fs/overlayfs/file.c                     | 530 ++++++++++++++++++++++++++++++++
+ fs/overlayfs/inode.c                    | 203 ++++++++----
+ fs/overlayfs/namei.c                    | 205 +++++++-----
+ fs/overlayfs/overlayfs.h                | 119 ++++---
+ fs/overlayfs/ovl_entry.h                |   7 +-
+ fs/overlayfs/super.c                    | 134 +++++---
+ fs/overlayfs/util.c                     | 252 ++++++++++++++-
+ fs/read_write.c                         |  91 +++---
+ fs/xattr.c                              |   9 +-
+ fs/xfs/xfs_file.c                       |   8 +-
+ include/linux/dcache.h                  |  15 +-
+ include/linux/fs.h                      |  36 ++-
+ include/linux/fsnotify.h                |  14 +-
+ include/uapi/linux/fs.h                 |   1 -
+ mm/util.c                               |   5 +
+ 34 files changed, 1981 insertions(+), 836 deletions(-)
+ create mode 100644 fs/overlayfs/file.c
+Merging ubifs/linux-next (b5094b7f135b ubi: Reject MLC NAND)
+$ git merge ubifs/linux-next
+Already up to date.
+Merging xfs/for-next (d25522f10cfa xfs: repair superblocks)
+$ git merge xfs/for-next
+Auto-merging include/uapi/linux/fs.h
+Auto-merging fs/xfs/xfs_file.c
+Auto-merging fs/inode.c
+Merge made by the 'recursive' strategy.
+ Documentation/ioctl/ioctl-number.txt |    3 +-
+ fs/inode.c                           |    1 +
+ fs/iomap.c                           |  235 +++++++-
+ fs/xfs/Kconfig                       |   18 +
+ fs/xfs/Makefile                      |    9 +
+ fs/xfs/libxfs/xfs_ag.c               |  464 +++++++++++++++
+ fs/xfs/libxfs/xfs_ag.h               |   30 +
+ fs/xfs/libxfs/xfs_alloc.c            |  129 +++-
+ fs/xfs/libxfs/xfs_alloc.h            |   23 +-
+ fs/xfs/libxfs/xfs_alloc_btree.c      |    9 +
+ fs/xfs/libxfs/xfs_alloc_btree.h      |    2 +
+ fs/xfs/libxfs/xfs_attr.c             |    4 +-
+ fs/xfs/libxfs/xfs_attr_remote.c      |    2 +-
+ fs/xfs/libxfs/xfs_bmap.c             |   92 +--
+ fs/xfs/libxfs/xfs_bmap.h             |   30 +-
+ fs/xfs/libxfs/xfs_bmap_btree.c       |    9 +
+ fs/xfs/libxfs/xfs_bmap_btree.h       |    3 +
+ fs/xfs/libxfs/xfs_btree.c            |   25 +-
+ fs/xfs/libxfs/xfs_btree.h            |    3 +-
+ fs/xfs/libxfs/xfs_defer.c            |   24 +-
+ fs/xfs/libxfs/xfs_defer.h            |    1 +
+ fs/xfs/libxfs/xfs_dquot_buf.c        |   94 +--
+ fs/xfs/libxfs/xfs_errortag.h         |    4 +-
+ fs/xfs/libxfs/xfs_format.h           |    7 +-
+ fs/xfs/libxfs/xfs_fs.h               |    9 +-
+ fs/xfs/libxfs/xfs_ialloc.c           |    2 +-
+ fs/xfs/libxfs/xfs_ialloc.h           |    3 +
+ fs/xfs/libxfs/xfs_ialloc_btree.c     |   11 +-
+ fs/xfs/libxfs/xfs_ialloc_btree.h     |    2 +
+ fs/xfs/libxfs/xfs_quota_defs.h       |    9 +-
+ fs/xfs/libxfs/xfs_refcount.c         |   21 +-
+ fs/xfs/libxfs/xfs_refcount.h         |    7 +
+ fs/xfs/libxfs/xfs_rmap.c             |   83 +++
+ fs/xfs/libxfs/xfs_rmap.h             |    4 +
+ fs/xfs/libxfs/xfs_sb.c               |  147 +++++
+ fs/xfs/libxfs/xfs_sb.h               |   16 +
+ fs/xfs/libxfs/xfs_shared.h           |   16 +-
+ fs/xfs/scrub/agheader.c              |   85 +--
+ fs/xfs/scrub/agheader_repair.c       |   70 +++
+ fs/xfs/scrub/alloc.c                 |    4 +-
+ fs/xfs/scrub/attr.c                  |    3 +-
+ fs/xfs/scrub/bmap.c                  |    8 +-
+ fs/xfs/scrub/btree.c                 |   42 +-
+ fs/xfs/scrub/common.c                |  131 +++-
+ fs/xfs/scrub/common.h                |   32 +-
+ fs/xfs/scrub/dir.c                   |   35 +-
+ fs/xfs/scrub/ialloc.c                |    7 +-
+ fs/xfs/scrub/inode.c                 |   10 +-
+ fs/xfs/scrub/parent.c                |   19 +-
+ fs/xfs/scrub/quota.c                 |  181 +++---
+ fs/xfs/scrub/refcount.c              |   10 +-
+ fs/xfs/scrub/repair.c                | 1089 ++++++++++++++++++++++++++++++++++
+ fs/xfs/scrub/repair.h                |  132 +++++
+ fs/xfs/scrub/rmap.c                  |    6 +-
+ fs/xfs/scrub/rtbitmap.c              |   37 +-
+ fs/xfs/scrub/scrub.c                 |  142 ++++-
+ fs/xfs/scrub/scrub.h                 |    5 +
+ fs/xfs/scrub/trace.h                 |  258 ++++++++
+ fs/xfs/xfs_aops.c                    |   12 +
+ fs/xfs/xfs_bmap_item.c               |    4 +-
+ fs/xfs/xfs_bmap_util.c               |   10 +-
+ fs/xfs/xfs_buf.c                     |  100 +++-
+ fs/xfs/xfs_buf.h                     |   29 +-
+ fs/xfs/xfs_buf_item.c                |   10 +-
+ fs/xfs/xfs_dquot.c                   |  780 ++++++++++++++----------
+ fs/xfs/xfs_dquot.h                   |   22 +-
+ fs/xfs/xfs_dquot_item.c              |    7 +-
+ fs/xfs/xfs_error.c                   |    3 +
+ fs/xfs/xfs_extfree_item.c            |    6 +-
+ fs/xfs/xfs_file.c                    |   55 +-
+ fs/xfs/xfs_fsops.c                   |  587 ++++--------------
+ fs/xfs/xfs_globals.c                 |    1 +
+ fs/xfs/xfs_icache.c                  |   94 ++-
+ fs/xfs/xfs_icache.h                  |    3 +
+ fs/xfs/xfs_icreate_item.c            |    4 +-
+ fs/xfs/xfs_inode.c                   |   47 +-
+ fs/xfs/xfs_inode.h                   |   14 +-
+ fs/xfs/xfs_inode_item.c              |    8 +-
+ fs/xfs/xfs_ioctl.c                   |   86 +++
+ fs/xfs/xfs_iomap.c                   |  186 +++---
+ fs/xfs/xfs_iops.c                    |    2 +-
+ fs/xfs/xfs_log.c                     |   12 +-
+ fs/xfs/xfs_log_cil.c                 |   22 +-
+ fs/xfs/xfs_log_recover.c             |    4 +-
+ fs/xfs/xfs_mount.c                   |    4 +-
+ fs/xfs/xfs_qm.c                      |  254 ++++----
+ fs/xfs/xfs_qm.h                      |    6 +-
+ fs/xfs/xfs_qm_bhv.c                  |    2 +-
+ fs/xfs/xfs_qm_syscalls.c             |   97 ++-
+ fs/xfs/xfs_quota.h                   |   22 +-
+ fs/xfs/xfs_quotaops.c                |    8 +-
+ fs/xfs/xfs_refcount_item.c           |    4 +-
+ fs/xfs/xfs_reflink.c                 |   18 +-
+ fs/xfs/xfs_rmap_item.c               |    4 +-
+ fs/xfs/xfs_super.c                   |   49 +-
+ fs/xfs/xfs_symlink.c                 |   10 +-
+ fs/xfs/xfs_sysctl.h                  |    1 +
+ fs/xfs/xfs_sysfs.c                   |   31 +
+ fs/xfs/xfs_trace.h                   |   72 ++-
+ fs/xfs/xfs_trans.c                   |   89 ++-
+ fs/xfs/xfs_trans.h                   |   29 +-
+ fs/xfs/xfs_trans_ail.c               |   50 +-
+ fs/xfs/xfs_trans_bmap.c              |    4 +-
+ fs/xfs/xfs_trans_buf.c               |   24 +-
+ fs/xfs/xfs_trans_dquot.c             |    4 +-
+ fs/xfs/xfs_trans_extfree.c           |   84 ++-
+ fs/xfs/xfs_trans_inode.c             |    3 +-
+ fs/xfs/xfs_trans_priv.h              |   11 +-
+ fs/xfs/xfs_trans_refcount.c          |    4 +-
+ fs/xfs/xfs_trans_rmap.c              |    4 +-
+ include/linux/iomap.h                |   11 +
+ include/uapi/linux/fs.h              |    8 +-
+ 112 files changed, 5151 insertions(+), 1721 deletions(-)
+ create mode 100644 fs/xfs/libxfs/xfs_ag.c
+ create mode 100644 fs/xfs/libxfs/xfs_ag.h
+ create mode 100644 fs/xfs/scrub/agheader_repair.c
+ create mode 100644 fs/xfs/scrub/repair.c
+ create mode 100644 fs/xfs/scrub/repair.h
+Merging file-locks/locks-next (7a107c0f55a3 fasync: Fix deadlock between task-context and interrupt-context kill_fasync())
+$ git merge file-locks/locks-next
+Auto-merging include/linux/fs.h
+Merge made by the 'recursive' strategy.
+ fs/fcntl.c         | 15 +++++++--------
+ include/linux/fs.h |  2 +-
+ 2 files changed, 8 insertions(+), 9 deletions(-)
+Merging vfs/for-next (1e9d358032bc Merge branches 'work.rmdir' and 'work.lookup' into for-next)
+$ git merge vfs/for-next
+Auto-merging security/selinux/hooks.c
+Auto-merging net/sctp/socket.c
+Auto-merging net/sctp/protocol.c
+Auto-merging net/sctp/ipv6.c
+Auto-merging net/packet/af_packet.c
+Auto-merging net/netfilter/x_tables.c
+Auto-merging net/netfilter/ipvs/ip_vs_ctl.c
+Auto-merging net/netfilter/ipvs/ip_vs_conn.c
+Auto-merging net/ipv4/route.c
+Auto-merging net/dccp/proto.c
+Auto-merging net/core/sock.c
+Auto-merging kernel/sys_ni.c
+Auto-merging kernel/cgroup/cgroup.c
+Auto-merging include/net/sctp/sctp.h
+Auto-merging include/linux/ide.h
+Auto-merging include/linux/fs.h
+Auto-merging fs/xfs/xfs_iops.c
+Auto-merging fs/xattr.c
+Auto-merging fs/read_write.c
+CONFLICT (content): Merge conflict in fs/read_write.c
+Auto-merging fs/proc/base.c
+Auto-merging fs/proc/array.c
+Auto-merging fs/orangefs/namei.c
+Auto-merging fs/namei.c
+Auto-merging fs/locks.c
+Auto-merging fs/f2fs/sysfs.c
+Auto-merging fs/ext4/mballoc.c
+Auto-merging fs/ext4/ext4.h
+Auto-merging fs/cifs/cifs_debug.c
+Auto-merging drivers/vhost/vhost.c
+Auto-merging drivers/scsi/sg.c
+Auto-merging drivers/parisc/sba_iommu.c
+Auto-merging drivers/parisc/ccio-dma.c
+Auto-merging drivers/macintosh/via-pmu.c
+Auto-merging arch/powerpc/kernel/rtas-proc.c
+Auto-merging arch/ia64/hp/common/sba_iommu.c
+Auto-merging Documentation/filesystems/vfs.txt
+Auto-merging Documentation/filesystems/Locking
+Resolved 'fs/read_write.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master 6bdd00132e2e] Merge remote-tracking branch 'vfs/for-next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/filesystems/Locking                  |  52 +-
+ Documentation/filesystems/vfs.txt                  |  15 +-
+ arch/arm/kernel/dma.c                              |  14 +-
+ arch/arm/kernel/swp_emulate.c                      |  15 +-
+ arch/arm/mach-rpc/ecard.c                          |  16 +-
+ arch/ia64/hp/common/sba_iommu.c                    |  15 +-
+ arch/ia64/hp/sim/simserial.c                       |  15 +-
+ arch/ia64/kernel/palinfo.c                         |  16 +-
+ arch/ia64/kernel/perfmon.c                         |  16 +-
+ arch/ia64/kernel/salinfo.c                         |  42 +-
+ arch/ia64/sn/kernel/sn2/prominfo_proc.c            |  32 +-
+ arch/ia64/sn/kernel/sn2/sn_proc_fs.c               |  62 +--
+ arch/m68k/kernel/setup_mm.c                        |  14 +-
+ arch/mips/pci/ops-pmcmsp.c                         |  28 +-
+ arch/mips/sibyte/common/bus_watcher.c              |  16 +-
+ arch/parisc/kernel/pci-dma.c                       |  17 +-
+ arch/parisc/kernel/pdc_chassis.c                   |  14 +-
+ arch/powerpc/kernel/eeh.c                          |  14 +-
+ arch/powerpc/kernel/rtas-proc.c                    |  32 +-
+ arch/powerpc/platforms/cell/spufs/sched.c          |  14 +-
+ arch/s390/kernel/sysinfo.c                         |  28 +-
+ arch/sh/drivers/dma/dma-api.c                      |  14 +-
+ arch/sparc/kernel/ioport.c                         |  19 +-
+ arch/um/drivers/ubd_kern.c                         |  16 +-
+ arch/x86/entry/syscalls/syscall_32.tbl             |   1 +
+ arch/x86/entry/syscalls/syscall_64.tbl             |   1 +
+ arch/x86/kernel/apm_32.c                           |  15 +-
+ arch/xtensa/platforms/iss/console.c                |  15 +-
+ block/genhd.c                                      |  28 +-
+ crypto/af_alg.c                                    |  14 +-
+ crypto/algif_aead.c                                |   4 +-
+ crypto/algif_hash.c                                |   2 -
+ crypto/algif_rng.c                                 |   1 -
+ crypto/algif_skcipher.c                            |   4 +-
+ crypto/proc.c                                      |  14 +-
+ drivers/acpi/ac.c                                  |  21 +-
+ drivers/acpi/battery.c                             | 121 +----
+ drivers/acpi/button.c                              |  19 +-
+ drivers/block/DAC960.c                             |  49 +-
+ drivers/block/drbd/drbd_int.h                      |   2 +-
+ drivers/block/drbd/drbd_main.c                     |   3 +-
+ drivers/block/drbd/drbd_proc.c                     |  34 +-
+ drivers/block/pktcdvd.c                            |  14 +-
+ drivers/block/ps3vram.c                            |  17 +-
+ drivers/char/apm-emulation.c                       |  15 +-
+ drivers/char/ds1620.c                              |  14 +-
+ drivers/char/efirtc.c                              |  15 +-
+ drivers/char/misc.c                                |  15 +-
+ drivers/char/nvram.c                               |  15 +-
+ drivers/char/pcmcia/synclink_cs.c                  |  15 +-
+ drivers/char/random.c                              |  29 +-
+ drivers/char/rtc.c                                 |  19 +-
+ drivers/char/toshiba.c                             |  15 +-
+ drivers/connector/connector.c                      |  15 +-
+ drivers/ide/ide-cd.c                               |  15 +-
+ drivers/ide/ide-disk_proc.c                        |  62 +--
+ drivers/ide/ide-floppy_proc.c                      |  17 +-
+ drivers/ide/ide-proc.c                             | 182 +------
+ drivers/ide/ide-tape.c                             |  17 +-
+ drivers/input/misc/hp_sdc_rtc.c                    |  14 +-
+ drivers/isdn/capi/capi.c                           |  30 +-
+ drivers/isdn/capi/capidrv.c                        |  15 +-
+ drivers/isdn/capi/kcapi.c                          |   3 +-
+ drivers/isdn/capi/kcapi_proc.c                     |  80 +--
+ drivers/isdn/gigaset/capi.c                        |  15 +-
+ drivers/isdn/hardware/avm/avmcard.h                |   4 +-
+ drivers/isdn/hardware/avm/b1.c                     |  17 +-
+ drivers/isdn/hardware/avm/b1dma.c                  |  17 +-
+ drivers/isdn/hardware/avm/b1isa.c                  |   2 +-
+ drivers/isdn/hardware/avm/b1pci.c                  |   4 +-
+ drivers/isdn/hardware/avm/b1pcmcia.c               |   2 +-
+ drivers/isdn/hardware/avm/c4.c                     |  15 +-
+ drivers/isdn/hardware/avm/t1isa.c                  |   2 +-
+ drivers/isdn/hardware/avm/t1pci.c                  |   2 +-
+ drivers/isdn/hardware/eicon/capimain.c             |  15 +-
+ drivers/isdn/hardware/eicon/diva_didd.c            |  17 +-
+ drivers/isdn/hardware/eicon/divasi.c               |  17 +-
+ drivers/isdn/hysdn/hycapi.c                        |  15 +-
+ drivers/isdn/mISDN/socket.c                        |   3 +-
+ drivers/macintosh/via-pmu.c                        |  57 +--
+ drivers/media/pci/saa7164/saa7164-core.c           |  14 +-
+ drivers/media/pci/zoran/videocodec.c               |  16 +-
+ drivers/message/fusion/mptbase.c                   |  57 +--
+ drivers/misc/sgi-gru/gruprocfs.c                   |  81 +--
+ drivers/mmc/core/sdio_uart.c                       |  15 +-
+ drivers/mtd/mtdcore.c                              |  14 +-
+ drivers/net/bonding/bond_procfs.c                  |  36 +-
+ drivers/net/hamradio/bpqether.c                    |  16 +-
+ drivers/net/hamradio/scc.c                         |  17 +-
+ drivers/net/hamradio/yam.c                         |  16 +-
+ drivers/net/ppp/pppoe.c                            |  20 +-
+ drivers/net/ppp/pptp.c                             |   1 -
+ drivers/net/wireless/atmel/atmel.c                 |  15 +-
+ drivers/net/wireless/intersil/hostap/hostap_ap.c   |  86 +---
+ drivers/net/wireless/intersil/hostap/hostap_hw.c   |  17 +-
+ drivers/net/wireless/intersil/hostap/hostap_proc.c | 143 +-----
+ drivers/net/wireless/ray_cs.c                      |  15 +-
+ drivers/nubus/proc.c                               |  51 +-
+ drivers/parisc/ccio-dma.c                          |  34 +-
+ drivers/parisc/sba_iommu.c                         |  32 +-
+ drivers/pci/proc.c                                 |  17 +-
+ drivers/platform/x86/toshiba_acpi.c                |  17 +-
+ drivers/pnp/pnpbios/proc.c                         |  78 +--
+ drivers/rtc/rtc-proc.c                             |  33 +-
+ drivers/s390/block/dasd_proc.c                     |  17 +-
+ drivers/s390/char/tape_proc.c                      |  19 +-
+ drivers/scsi/megaraid.c                            | 140 ++---
+ drivers/scsi/megaraid.h                            |  12 -
+ drivers/scsi/sg.c                                  | 124 +----
+ drivers/staging/comedi/drivers/serial2002.c        |   4 +-
+ drivers/staging/comedi/proc.c                      |  18 +-
+ drivers/staging/fwserial/fwserial.c                |  15 +-
+ drivers/staging/ipx/af_ipx.c                       |   2 +-
+ drivers/staging/ipx/ipx_proc.c                     |  45 +-
+ drivers/staging/ncpfs/dir.c                        |  42 +-
+ drivers/staging/rtl8192u/r8192U_core.c             |  67 +--
+ drivers/tty/amiserial.c                            |  15 +-
+ drivers/tty/cyclades.c                             |  15 +-
+ drivers/tty/serial/serial_core.c                   |  15 +-
+ drivers/tty/synclink.c                             |  15 +-
+ drivers/tty/synclink_gt.c                          |  15 +-
+ drivers/tty/synclinkmp.c                           |  15 +-
+ drivers/tty/tty_ldisc.c                            |  15 +-
+ drivers/usb/gadget/udc/at91_udc.c                  |  16 +-
+ drivers/usb/gadget/udc/fsl_udc_core.c              |  18 +-
+ drivers/usb/gadget/udc/goku_udc.c                  |  18 +-
+ drivers/usb/gadget/udc/omap_udc.c                  |  15 +-
+ drivers/usb/serial/usb-serial.c                    |  15 +-
+ drivers/vfio/virqfd.c                              |   2 +-
+ drivers/vhost/vhost.c                              |   2 +-
+ drivers/video/fbdev/core/fbmem.c                   |  15 +-
+ drivers/video/fbdev/via/viafbdev.c                 |  17 +-
+ drivers/zorro/proc.c                               |  17 +-
+ fs/9p/vfs_inode.c                                  |  35 +-
+ fs/adfs/dir.c                                      |  24 +-
+ fs/afs/proc.c                                      | 134 +----
+ fs/aio.c                                           | 562 +++++++++++++++------
+ fs/bfs/dir.c                                       |  43 +-
+ fs/cachefiles/proc.c                               |  19 +-
+ fs/cifs/cifs_debug.c                               |  15 +-
+ fs/cifs/dir.c                                      |  38 +-
+ fs/cramfs/inode.c                                  |   5 +-
+ fs/dcache.c                                        | 205 +++-----
+ fs/eventfd.c                                       |  15 +-
+ fs/eventpoll.c                                     |   5 +-
+ fs/ext4/ext4.h                                     |   2 +-
+ fs/ext4/mballoc.c                                  |  29 +-
+ fs/ext4/sysfs.c                                    |  49 +-
+ fs/f2fs/sysfs.c                                    |  29 +-
+ fs/fat/namei_msdos.c                               |   4 -
+ fs/fat/namei_vfat.c                                |  13 +-
+ fs/filesystems.c                                   |  14 +-
+ fs/freevxfs/vxfs_lookup.c                          |   8 +-
+ fs/fscache/histogram.c                             |  17 +-
+ fs/fscache/internal.h                              |   5 +-
+ fs/fscache/proc.c                                  |   8 +-
+ fs/fscache/stats.c                                 |  17 +-
+ fs/hfs/dir.c                                       |  20 +-
+ fs/hfs/inode.c                                     |   4 +-
+ fs/hfsplus/dir.c                                   |   3 +-
+ fs/jfs/jfs_debug.c                                 |  43 +-
+ fs/jfs/jfs_debug.h                                 |  10 +-
+ fs/jfs/jfs_logmgr.c                                |  14 +-
+ fs/jfs/jfs_metapage.c                              |  14 +-
+ fs/jfs/jfs_txnmgr.c                                |  28 +-
+ fs/jfs/jfs_xtree.c                                 |  14 +-
+ fs/locks.c                                         |  16 +-
+ fs/minix/namei.c                                   |   8 +-
+ fs/namei.c                                         |  12 +-
+ fs/nfs/client.c                                    |  43 +-
+ fs/omfs/dir.c                                      |   7 +-
+ fs/openpromfs/inode.c                              |   3 +-
+ fs/orangefs/namei.c                                |  64 +--
+ fs/pipe.c                                          |  22 +-
+ fs/proc/array.c                                    |  23 +-
+ fs/proc/base.c                                     | 154 +++---
+ fs/proc/cmdline.c                                  |  14 +-
+ fs/proc/consoles.c                                 |  14 +-
+ fs/proc/devices.c                                  |  14 +-
+ fs/proc/fd.c                                       | 138 ++---
+ fs/proc/generic.c                                  | 151 ++++--
+ fs/proc/internal.h                                 |  17 +-
+ fs/proc/interrupts.c                               |  14 +-
+ fs/proc/loadavg.c                                  |  14 +-
+ fs/proc/meminfo.c                                  |  14 +-
+ fs/proc/namespaces.c                               |  24 +-
+ fs/proc/nommu.c                                    |  14 +-
+ fs/proc/proc_net.c                                 | 104 ++--
+ fs/proc/proc_sysctl.c                              |  15 +-
+ fs/proc/proc_tty.c                                 |  22 +-
+ fs/proc/self.c                                     |   4 +-
+ fs/proc/softirqs.c                                 |  14 +-
+ fs/proc/thread_self.c                              |   4 +-
+ fs/proc/uptime.c                                   |  14 +-
+ fs/proc/version.c                                  |  14 +-
+ fs/qnx4/namei.c                                    |   8 +-
+ fs/qnx6/namei.c                                    |   8 +-
+ fs/read_write.c                                    |   1 -
+ fs/reiserfs/procfs.c                               |  16 +-
+ fs/romfs/super.c                                   |  36 +-
+ fs/select.c                                        |  85 ++--
+ fs/super.c                                         |   2 +-
+ fs/sysv/namei.c                                    |   9 +-
+ fs/timerfd.c                                       |  22 +-
+ fs/ubifs/dir.c                                     |  43 +-
+ fs/xattr.c                                         |   3 +-
+ fs/xfs/xfs_iops.c                                  |  16 +-
+ fs/xfs/xfs_stats.c                                 |  33 +-
+ include/crypto/if_alg.h                            |   3 +-
+ include/linux/aio.h                                |   2 -
+ include/linux/atalk.h                              |   7 +-
+ include/linux/compat.h                             |   7 +
+ include/linux/fs.h                                 |   4 +-
+ include/linux/ide.h                                |   6 +-
+ include/linux/isdn/capilli.h                       |   2 +-
+ include/linux/net.h                                |   1 +
+ include/linux/poll.h                               |  14 +-
+ include/linux/proc_fs.h                            |  39 ++
+ include/linux/seq_file_net.h                       |  19 +-
+ include/linux/skbuff.h                             |   3 +-
+ include/linux/sunrpc/rpc_pipe_fs.h                 |   2 -
+ include/linux/syscalls.h                           |   6 +
+ include/linux/tty.h                                |   3 +-
+ include/linux/tty_driver.h                         |   2 +-
+ include/linux/xattr.h                              |   1 -
+ include/net/ax25.h                                 |   5 +-
+ include/net/bluetooth/bluetooth.h                  |   2 +-
+ include/net/busy_poll.h                            |  15 +
+ include/net/ip6_fib.h                              |  10 +-
+ include/net/ip_vs.h                                |  12 -
+ include/net/iucv/af_iucv.h                         |   2 -
+ include/net/netrom.h                               |   5 +-
+ include/net/phonet/pn_dev.h                        |   4 +-
+ include/net/ping.h                                 |  11 -
+ include/net/raw.h                                  |   4 -
+ include/net/rose.h                                 |   6 +-
+ include/net/sctp/sctp.h                            |   3 +-
+ include/net/sock.h                                 |   2 -
+ include/net/tcp.h                                  |  14 +-
+ include/net/udp.h                                  |  22 +-
+ include/uapi/asm-generic/unistd.h                  |   4 +-
+ include/uapi/linux/aio_abi.h                       |  12 +-
+ include/uapi/linux/types.h                         |   4 -
+ kernel/cgroup/cgroup-internal.h                    |   2 +-
+ kernel/cgroup/cgroup-v1.c                          |  14 +-
+ kernel/cgroup/cgroup.c                             |   2 +-
+ kernel/dma.c                                       |  14 +-
+ kernel/exec_domain.c                               |  14 +-
+ kernel/irq/proc.c                                  |  82 +--
+ kernel/locking/lockdep_proc.c                      |  45 +-
+ kernel/resource.c                                  |  43 +-
+ kernel/sched/debug.c                               |  28 +-
+ kernel/sched/stats.c                               |  15 +-
+ kernel/sys_ni.c                                    |   2 +
+ kernel/time/timer_list.c                           |  16 +-
+ mm/memcontrol.c                                    |   2 +-
+ mm/vmalloc.c                                       |  21 +-
+ mm/vmstat.c                                        |  56 +-
+ net/8021q/vlanproc.c                               |  39 +-
+ net/9p/trans_fd.c                                  |  18 +-
+ net/appletalk/aarp.c                               |  20 +-
+ net/appletalk/atalk_proc.c                         |  51 +-
+ net/appletalk/ddp.c                                |   2 +-
+ net/atm/br2684.c                                   |  14 +-
+ net/atm/clip.c                                     |  17 +-
+ net/atm/common.c                                   |  11 +-
+ net/atm/common.h                                   |   2 +-
+ net/atm/lec.c                                      |  15 +-
+ net/atm/proc.c                                     | 131 +----
+ net/atm/pvc.c                                      |   2 +-
+ net/atm/svc.c                                      |   2 +-
+ net/ax25/af_ax25.c                                 |  23 +-
+ net/ax25/ax25_route.c                              |  15 +-
+ net/ax25/ax25_uid.c                                |  15 +-
+ net/bluetooth/af_bluetooth.c                       |  47 +-
+ net/bluetooth/bnep/sock.c                          |   1 -
+ net/bluetooth/cmtp/capi.c                          |  14 +-
+ net/bluetooth/cmtp/sock.c                          |   1 -
+ net/bluetooth/hci_sock.c                           |   2 +-
+ net/bluetooth/hidp/sock.c                          |   1 -
+ net/bluetooth/l2cap_sock.c                         |   2 +-
+ net/bluetooth/rfcomm/sock.c                        |   2 +-
+ net/bluetooth/sco.c                                |   2 +-
+ net/caif/caif_socket.c                             |  12 +-
+ net/can/bcm.c                                      |  18 +-
+ net/can/proc.c                                     | 127 +----
+ net/can/raw.c                                      |   2 +-
+ net/core/datagram.c                                |  13 +-
+ net/core/neighbour.c                               |  31 +-
+ net/core/net-procfs.c                              |  65 +--
+ net/core/sock.c                                    |  22 +-
+ net/dccp/dccp.h                                    |   3 +-
+ net/dccp/ipv4.c                                    |   2 +-
+ net/dccp/ipv6.c                                    |   2 +-
+ net/dccp/proto.c                                   |  13 +-
+ net/decnet/af_decnet.c                             |  23 +-
+ net/decnet/dn_dev.c                                |  15 +-
+ net/decnet/dn_neigh.c                              |  18 +-
+ net/decnet/dn_route.c                              |  19 +-
+ net/ieee802154/socket.c                            |   4 +-
+ net/ipv4/af_inet.c                                 |   8 +-
+ net/ipv4/arp.c                                     |  17 +-
+ net/ipv4/fib_trie.c                                |  48 +-
+ net/ipv4/igmp.c                                    |  33 +-
+ net/ipv4/ipconfig.c                                |  14 +-
+ net/ipv4/ipmr.c                                    |  32 +-
+ net/ipv4/ping.c                                    |  52 +-
+ net/ipv4/proc.c                                    |  48 +-
+ net/ipv4/raw.c                                     |  45 +-
+ net/ipv4/route.c                                   |  15 +-
+ net/ipv4/tcp.c                                     |  23 +-
+ net/ipv4/tcp_ipv4.c                                |  82 +--
+ net/ipv4/udp.c                                     | 104 ++--
+ net/ipv4/udplite.c                                 |  21 +-
+ net/ipv6/addrconf.c                                |  16 +-
+ net/ipv6/af_inet6.c                                |   4 +-
+ net/ipv6/anycast.c                                 |  16 +-
+ net/ipv6/ip6_fib.c                                 |  18 +-
+ net/ipv6/ip6_flowlabel.c                           |  42 +-
+ net/ipv6/ip6mr.c                                   |  32 +-
+ net/ipv6/mcast.c                                   |  34 +-
+ net/ipv6/ping.c                                    |  23 +-
+ net/ipv6/proc.c                                    |  48 +-
+ net/ipv6/raw.c                                     |  19 +-
+ net/ipv6/route.c                                   |  26 +-
+ net/ipv6/tcp_ipv6.c                                |  22 +-
+ net/ipv6/udp.c                                     |  23 +-
+ net/ipv6/udplite.c                                 |  22 +-
+ net/iucv/af_iucv.c                                 |   7 +-
+ net/kcm/kcmproc.c                                  |  83 +--
+ net/kcm/kcmsock.c                                  |  10 +-
+ net/key/af_key.c                                   |  18 +-
+ net/l2tp/l2tp_ip.c                                 |   2 +-
+ net/l2tp/l2tp_ip6.c                                |   2 +-
+ net/l2tp/l2tp_ppp.c                                |  24 +-
+ net/llc/af_llc.c                                   |   2 +-
+ net/llc/llc_proc.c                                 |  28 +-
+ net/netfilter/ipvs/ip_vs_app.c                     |  16 +-
+ net/netfilter/ipvs/ip_vs_conn.c                    |  35 +-
+ net/netfilter/ipvs/ip_vs_ctl.c                     |  47 +-
+ net/netfilter/nf_conntrack_expect.c                |  17 +-
+ net/netfilter/nf_conntrack_standalone.c            |  33 +-
+ net/netfilter/nf_log.c                             |  19 +-
+ net/netfilter/nf_synproxy_core.c                   |  17 +-
+ net/netfilter/nfnetlink_log.c                      |  18 +-
+ net/netfilter/nfnetlink_queue.c                    |  18 +-
+ net/netfilter/x_tables.c                           |  95 +---
+ net/netfilter/xt_hashlimit.c                       |  92 +---
+ net/netlink/af_netlink.c                           |  20 +-
+ net/netrom/af_netrom.c                             |  20 +-
+ net/netrom/nr_route.c                              |  29 +-
+ net/nfc/llcp_sock.c                                |   9 +-
+ net/nfc/rawsock.c                                  |   4 +-
+ net/packet/af_packet.c                             |  26 +-
+ net/phonet/pn_dev.c                                |   6 +-
+ net/phonet/socket.c                                |  39 +-
+ net/qrtr/qrtr.c                                    |   2 +-
+ net/rose/af_rose.c                                 |  28 +-
+ net/rose/rose_route.c                              |  44 +-
+ net/rxrpc/af_rxrpc.c                               |  10 +-
+ net/rxrpc/ar-internal.h                            |   4 +-
+ net/rxrpc/net_ns.c                                 |   7 +-
+ net/rxrpc/proc.c                                   |  31 +-
+ net/sched/sch_api.c                                |  14 +-
+ net/sctp/ipv6.c                                    |   2 +-
+ net/sctp/objcnt.c                                  |  16 +-
+ net/sctp/proc.c                                    |  71 +--
+ net/sctp/protocol.c                                |   2 +-
+ net/sctp/socket.c                                  |   4 +-
+ net/socket.c                                       |  55 +-
+ net/sunrpc/rpc_pipe.c                              |  16 -
+ net/tipc/socket.c                                  |  14 +-
+ net/unix/af_unix.c                                 |  47 +-
+ net/vmw_vsock/af_vsock.c                           |  19 +-
+ net/wireless/wext-proc.c                           |  17 +-
+ net/x25/af_x25.c                                   |   2 +-
+ net/x25/x25_proc.c                                 |  48 +-
+ net/xfrm/xfrm_proc.c                               |  16 +-
+ security/keys/proc.c                               |  34 +-
+ security/selinux/hooks.c                           |   7 +-
+ virt/kvm/eventfd.c                                 |   2 +-
+ 381 files changed, 2406 insertions(+), 7186 deletions(-)
+Merging vfs-miklos/next (0eb8af4916a5 vfs: use helper for calling f_op->fsync())
+$ git merge vfs-miklos/next
+Already up to date.
+Merging printk/for-next (913a61197c1f Merge branch 'for-4.18-vsprintf-cleanup' into for-4.18)
+$ git merge printk/for-next
+Auto-merging lib/vsprintf.c
+CONFLICT (content): Merge conflict in lib/vsprintf.c
+Auto-merging kernel/printk/printk.c
+Resolved 'lib/vsprintf.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master 252c4fade1a8] Merge remote-tracking branch 'printk/for-next'
+$ git diff -M --stat --summary HEAD^..
+ kernel/printk/printk.c      |  14 +----
+ kernel/printk/printk_safe.c |  14 ++---
+ lib/test_printf.c           |   2 +-
+ lib/vsprintf.c              | 130 ++++++++++++++++++--------------------------
+ 4 files changed, 63 insertions(+), 97 deletions(-)
+Merging pci/next (6bf16b354ddc Merge branch 'pci/kconfig')
+$ git merge pci/next
+Auto-merging include/linux/pci.h
+Auto-merging include/linux/acpi.h
+Removing drivers/pci/pcie/portdrv_acpi.c
+Auto-merging drivers/pci/pci.c
+Auto-merging drivers/pci/pci-driver.c
+Auto-merging drivers/pci/host/pci-aardvark.c
+Auto-merging drivers/pci/Kconfig
+Auto-merging drivers/nvme/host/pci.c
+Auto-merging drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+Auto-merging drivers/iommu/amd_iommu.c
+Auto-merging drivers/infiniband/hw/hfi1/pcie.c
+Auto-merging arch/xtensa/include/asm/pci.h
+Auto-merging arch/microblaze/include/asm/pci.h
+CONFLICT (content): Merge conflict in arch/microblaze/include/asm/pci.h
+Auto-merging arch/arm64/configs/defconfig
+Auto-merging MAINTAINERS
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.txt
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Resolved 'arch/microblaze/include/asm/pci.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master bb8234b5c487] Merge remote-tracking branch 'pci/next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/PCI/pci-error-recovery.txt           |   35 +-
+ Documentation/admin-guide/kernel-parameters.txt    |    2 +
+ .../devicetree/bindings/pci/designware-pcie.txt    |   24 +-
+ .../devicetree/bindings/pci/mobiveil-pcie.txt      |   73 +
+ .../devicetree/bindings/pci/pci-armada8k.txt       |    5 +-
+ Documentation/devicetree/bindings/pci/rcar-pci.txt |    6 +
+ .../devicetree/bindings/pci/rockchip-pcie-ep.txt   |   62 +
+ .../{rockchip-pcie.txt => rockchip-pcie-host.txt}  |    0
+ .../devicetree/bindings/vendor-prefixes.txt        |    1 +
+ MAINTAINERS                                        |   17 +-
+ arch/arm64/configs/defconfig                       |    3 +-
+ arch/microblaze/include/asm/pci.h                  |    4 -
+ arch/microblaze/pci/pci-common.c                   |   61 -
+ arch/mips/pci/pci-legacy.c                         |    8 +-
+ arch/sparc/kernel/leon_pci.c                       |   62 +-
+ arch/sparc/kernel/pci.c                            |  136 +-
+ arch/sparc/kernel/pci_common.c                     |   31 +-
+ arch/sparc/kernel/pci_msi.c                        |   10 +-
+ arch/sparc/kernel/pcic.c                           |   94 +-
+ arch/x86/pci/early.c                               |   19 +-
+ arch/x86/pci/fixup.c                               |    4 +
+ arch/xtensa/include/asm/pci.h                      |    2 -
+ arch/xtensa/kernel/pci.c                           |   69 +-
+ drivers/acpi/pci_root.c                            |   17 +-
+ drivers/infiniband/hw/hfi1/pcie.c                  |   24 +-
+ drivers/iommu/amd_iommu.c                          |   11 +-
+ drivers/iommu/intel-iommu.c                        |    3 +-
+ drivers/misc/pci_endpoint_test.c                   |   29 +-
+ drivers/net/ethernet/amazon/ena/ena_netdev.c       |   28 +-
+ drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c   |   23 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt.c          |   19 +-
+ drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c    |   75 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_main.c      |   47 +-
+ drivers/nvme/host/pci.c                            |   20 +-
+ drivers/pci/Kconfig                                |   12 +
+ drivers/pci/Makefile                               |    1 +
+ drivers/pci/ats.c                                  |    3 +
+ drivers/pci/dwc/Kconfig                            |   88 +-
+ drivers/pci/dwc/pci-dra7xx.c                       |   19 +-
+ drivers/pci/dwc/pci-imx6.c                         |    2 +-
+ drivers/pci/dwc/pci-keystone.c                     |    2 +-
+ drivers/pci/dwc/pcie-armada8k.c                    |   21 +-
+ drivers/pci/dwc/pcie-artpec6.c                     |    6 +-
+ drivers/pci/dwc/pcie-designware-ep.c               |   19 +-
+ drivers/pci/dwc/pcie-designware-host.c             |   80 +-
+ drivers/pci/dwc/pcie-designware-plat.c             |  155 +-
+ drivers/pci/dwc/pcie-designware.c                  |   22 +-
+ drivers/pci/dwc/pcie-designware.h                  |    1 +
+ drivers/pci/dwc/pcie-qcom.c                        |   11 +-
+ drivers/pci/endpoint/functions/pci-epf-test.c      |   35 +-
+ drivers/pci/endpoint/pci-epf-core.c                |   23 +-
+ drivers/pci/host/Kconfig                           |   55 +-
+ drivers/pci/host/Makefile                          |    2 +
+ drivers/pci/host/pci-aardvark.c                    |    7 +-
+ drivers/pci/host/pci-ftpci100.c                    |    6 +-
+ drivers/pci/host/pci-host-common.c                 |   13 +
+ drivers/pci/host/pci-host-generic.c                |    1 +
+ drivers/pci/host/pci-hyperv.c                      |  162 +-
+ drivers/pci/host/pci-mvebu.c                       |    2 +
+ drivers/pci/host/pci-rcar-gen2.c                   |    2 +
+ drivers/pci/host/pci-tegra.c                       |    2 +
+ drivers/pci/host/pci-v3-semi.c                     |    5 +-
+ drivers/pci/host/pci-versatile.c                   |    5 +-
+ drivers/pci/host/pci-xgene.c                       |    5 +-
+ drivers/pci/host/pcie-altera.c                     |    7 +-
+ drivers/pci/host/pcie-iproc-platform.c             |    5 +-
+ drivers/pci/host/pcie-mediatek.c                   |  236 +--
+ drivers/pci/host/pcie-mobiveil.c                   |  865 ++++++++++
+ drivers/pci/host/pcie-rcar.c                       |  284 ++--
+ drivers/pci/host/pcie-rockchip-ep.c                |  642 ++++++++
+ drivers/pci/host/pcie-rockchip-host.c              | 1142 +++++++++++++
+ drivers/pci/host/pcie-rockchip.c                   | 1684 +++-----------------
+ drivers/pci/host/pcie-rockchip.h                   |  338 ++++
+ drivers/pci/host/pcie-xilinx-nwl.c                 |    6 +-
+ drivers/pci/host/pcie-xilinx.c                     |    6 +-
+ drivers/pci/host/vmd.c                             |   91 +-
+ drivers/pci/hotplug/Kconfig                        |    5 +-
+ drivers/pci/hotplug/acpi_pcihp.c                   |   45 +-
+ drivers/pci/hotplug/acpiphp_glue.c                 |    4 +-
+ drivers/pci/hotplug/ibmphp_core.c                  |    2 +-
+ drivers/pci/hotplug/pciehp.h                       |    2 +-
+ drivers/pci/hotplug/pciehp_core.c                  |    2 +-
+ drivers/pci/hotplug/pciehp_hpc.c                   |   84 +-
+ drivers/pci/hotplug/pnv_php.c                      |    8 +-
+ drivers/pci/hotplug/shpchp.h                       |   11 -
+ drivers/pci/hotplug/shpchp_core.c                  |    2 +-
+ drivers/pci/iov.c                                  |   42 +-
+ drivers/pci/of.c                                   |   63 +-
+ drivers/pci/pci-acpi.c                             |   26 +-
+ drivers/pci/pci-driver.c                           |    2 +-
+ drivers/pci/pci-pf-stub.c                          |   54 +
+ drivers/pci/pci-sysfs.c                            |   15 +-
+ drivers/pci/pci.c                                  |   89 +-
+ drivers/pci/pci.h                                  |   45 +
+ drivers/pci/pcie/Makefile                          |    2 +-
+ drivers/pci/pcie/aer/aerdrv.c                      |    5 +-
+ drivers/pci/pcie/aer/aerdrv.h                      |   30 -
+ drivers/pci/pcie/aer/aerdrv_core.c                 |  325 +---
+ drivers/pci/pcie/aer/aerdrv_errprint.c             |   43 +-
+ drivers/pci/pcie/aspm.c                            |    9 +
+ drivers/pci/pcie/dpc.c                             |   74 +-
+ drivers/pci/pcie/err.c                             |  388 +++++
+ drivers/pci/pcie/portdrv.h                         |    5 +-
+ drivers/pci/pcie/portdrv_acpi.c                    |   57 -
+ drivers/pci/pcie/portdrv_core.c                    |   69 +-
+ drivers/pci/probe.c                                |   77 +-
+ drivers/pci/quirks.c                               | 1002 ++++++------
+ include/linux/acpi.h                               |    3 +-
+ include/linux/aer.h                                |    1 +
+ include/linux/of_pci.h                             |   34 -
+ include/linux/pci-ecam.h                           |    1 +
+ include/linux/pci-epc.h                            |    8 +
+ include/linux/pci-epf.h                            |    4 +-
+ include/linux/pci.h                                |   21 +-
+ include/linux/pci_hotplug.h                        |   11 +-
+ include/linux/pci_ids.h                            |    8 +
+ include/ras/ras_event.h                            |   22 +-
+ include/uapi/linux/pci_regs.h                      |    6 +
+ 118 files changed, 6024 insertions(+), 3779 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/pci/mobiveil-pcie.txt
+ create mode 100644 Documentation/devicetree/bindings/pci/rockchip-pcie-ep.txt
+ rename Documentation/devicetree/bindings/pci/{rockchip-pcie.txt => rockchip-pcie-host.txt} (100%)
+ create mode 100644 drivers/pci/host/pcie-mobiveil.c
+ create mode 100644 drivers/pci/host/pcie-rockchip-ep.c
+ create mode 100644 drivers/pci/host/pcie-rockchip-host.c
+ create mode 100644 drivers/pci/host/pcie-rockchip.h
+ create mode 100644 drivers/pci/pci-pf-stub.c
+ create mode 100644 drivers/pci/pcie/err.c
+ delete mode 100644 drivers/pci/pcie/portdrv_acpi.c
+Merging pstore/for-next/pstore (e698aaf37f9f pstore: fix crypto dependencies without compression)
+$ git merge pstore/for-next/pstore
+Already up to date.
+Merging hid/for-next (f1b6ae6701a7 Merge branch 'for-4.18/rmi' into for-next)
+$ git merge hid/for-next
+Merge made by the 'recursive' strategy.
+ drivers/hid/Kconfig           |   16 +
+ drivers/hid/Makefile          |    2 +
+ drivers/hid/hid-alps.c        |   30 +-
+ drivers/hid/hid-core.c        |   21 +-
+ drivers/hid/hid-generic.c     |   15 +
+ drivers/hid/hid-gfrm.c        |    2 +-
+ drivers/hid/hid-ids.h         |    5 +
+ drivers/hid/hid-input.c       |  123 ++++-
+ drivers/hid/hid-magicmouse.c  |    6 +-
+ drivers/hid/hid-multitouch.c  |  232 +++++----
+ drivers/hid/hid-plantronics.c |    6 +-
+ drivers/hid/hid-quirks.c      |    1 -
+ drivers/hid/hid-redragon.c    |   86 ++++
+ drivers/hid/hid-rmi.c         |   20 +
+ drivers/hid/hid-steam.c       | 1115 +++++++++++++++++++++++++++++++++++++++++
+ drivers/hid/i2c-hid/i2c-hid.c |   33 +-
+ drivers/hid/wacom_wac.c       |   74 +--
+ include/linux/hid.h           |   19 +-
+ 18 files changed, 1621 insertions(+), 185 deletions(-)
+ create mode 100644 drivers/hid/hid-redragon.c
+ create mode 100644 drivers/hid/hid-steam.c
+Merging i2c/i2c/for-next (bbfc642934a7 Merge branch 'i2c/for-4.18' into i2c/for-next)
+$ git merge i2c/i2c/for-next
+Removing include/linux/i2c-pnx.h
+Auto-merging arch/arm/mach-sa1100/simpad.c
+Auto-merging arch/arm/mach-pxa/viper.c
+Auto-merging arch/arm/mach-pxa/palmz72.c
+Auto-merging arch/arm/mach-omap2/omap_hwmod_7xx_data.c
+Auto-merging arch/arm/mach-omap2/omap_hwmod_54xx_data.c
+Auto-merging arch/arm/mach-omap2/omap_hwmod_44xx_data.c
+Auto-merging arch/arm/mach-omap2/omap_hwmod_3xxx_data.c
+Auto-merging arch/arm/mach-omap2/common.h
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/i2c/i2c-davinci.txt        |    2 +-
+ Documentation/devicetree/bindings/i2c/i2c-rcar.txt |    1 +
+ .../devicetree/bindings/i2c/i2c-s3c2410.txt        |    4 +-
+ Documentation/i2c/busses/i2c-mlxcpld               |    4 +
+ Documentation/i2c/busses/i2c-ocores                |    4 +-
+ Documentation/i2c/muxes/i2c-mux-gpio               |    4 +-
+ MAINTAINERS                                        |   16 +-
+ arch/arm/mach-ks8695/board-acs5k.c                 |    2 +-
+ arch/arm/mach-omap1/board-htcherald.c              |    2 +-
+ arch/arm/mach-omap1/common.h                       |    2 +-
+ arch/arm/mach-omap1/i2c.c                          |    2 +-
+ arch/arm/mach-omap2/common.h                       |    2 +-
+ arch/arm/mach-omap2/omap_hwmod_2420_data.c         |    2 +-
+ arch/arm/mach-omap2/omap_hwmod_2430_data.c         |    2 +-
+ arch/arm/mach-omap2/omap_hwmod_33xx_data.c         |    2 +-
+ arch/arm/mach-omap2/omap_hwmod_3xxx_data.c         |    2 +-
+ arch/arm/mach-omap2/omap_hwmod_44xx_data.c         |    2 +-
+ arch/arm/mach-omap2/omap_hwmod_54xx_data.c         |    2 +-
+ arch/arm/mach-omap2/omap_hwmod_7xx_data.c          |    2 +-
+ arch/arm/mach-pxa/palmz72.c                        |    2 +-
+ arch/arm/mach-pxa/viper.c                          |    2 +-
+ arch/arm/mach-sa1100/simpad.c                      |    2 +-
+ arch/mips/alchemy/board-gpr.c                      |    2 +-
+ arch/sh/boards/board-sh7785lcr.c                   |    2 +-
+ drivers/i2c/algos/i2c-algo-bit.c                   |    4 +-
+ drivers/i2c/algos/i2c-algo-pca.c                   |    5 +-
+ drivers/i2c/algos/i2c-algo-pcf.c                   |    8 +-
+ drivers/i2c/busses/Kconfig                         |    1 +
+ drivers/i2c/busses/Makefile                        |    3 +-
+ drivers/i2c/busses/i2c-aspeed.c                    |    3 +-
+ drivers/i2c/busses/i2c-at91.c                      |   12 +-
+ drivers/i2c/busses/i2c-axxia.c                     |   31 +-
+ drivers/i2c/busses/i2c-designware-common.c         |   20 +-
+ drivers/i2c/busses/i2c-designware-core.h           |   14 +-
+ drivers/i2c/busses/i2c-designware-master.c         |    8 +-
+ drivers/i2c/busses/i2c-designware-slave.c          |    6 +-
+ drivers/i2c/busses/i2c-diolan-u2c.c                |   11 +-
+ drivers/i2c/busses/i2c-efm32.c                     |    3 +-
+ drivers/i2c/busses/i2c-eg20t.c                     |    5 +-
+ drivers/i2c/busses/i2c-emev2.c                     |    2 +-
+ drivers/i2c/busses/i2c-exynos5.c                   |   29 +-
+ drivers/i2c/busses/i2c-gpio.c                      |    2 +-
+ drivers/i2c/busses/i2c-hix5hd2.c                   |   22 +-
+ drivers/i2c/busses/i2c-i801.c                      |    7 +-
+ drivers/i2c/busses/i2c-ibm_iic.c                   |    3 -
+ drivers/i2c/busses/i2c-imx-lpi2c.c                 |   16 +-
+ drivers/i2c/busses/i2c-imx.c                       |   21 +-
+ drivers/i2c/busses/i2c-kempld.c                    |    7 +-
+ drivers/i2c/busses/i2c-mlxcpld.c                   |   68 +-
+ drivers/i2c/busses/i2c-mt65xx.c                    |    8 +-
+ drivers/i2c/busses/i2c-mxs.c                       |   16 +-
+ drivers/i2c/busses/i2c-nomadik.c                   |    2 -
+ drivers/i2c/busses/i2c-ocores.c                    |    9 +-
+ drivers/i2c/busses/i2c-omap.c                      |    2 +-
+ drivers/i2c/busses/i2c-opal.c                      |    4 -
+ drivers/i2c/busses/i2c-pasemi.c                    |    2 +-
+ drivers/i2c/busses/i2c-pca-platform.c              |    2 +-
+ drivers/i2c/busses/i2c-pnx.c                       |   21 +-
+ drivers/i2c/busses/i2c-qup.c                       |   61 +-
+ drivers/i2c/busses/i2c-rcar.c                      |    4 +-
+ drivers/i2c/busses/i2c-riic.c                      |    5 +-
+ drivers/i2c/busses/i2c-rk3x.c                      |    2 -
+ drivers/i2c/busses/i2c-robotfuzz-osif.c            |   21 +-
+ drivers/i2c/busses/i2c-s3c2410.c                   |    2 -
+ drivers/i2c/busses/i2c-sh_mobile.c                 |   11 -
+ drivers/i2c/busses/i2c-stm32.c                     |  153 +++
+ drivers/i2c/busses/i2c-stm32.h                     |   37 +
+ drivers/i2c/busses/i2c-stm32f7.c                   | 1045 +++++++++++++++++++-
+ drivers/i2c/busses/i2c-stu300.c                    |   22 +-
+ drivers/i2c/busses/i2c-synquacer.c                 |    2 +-
+ drivers/i2c/busses/i2c-tegra.c                     |   33 -
+ drivers/i2c/busses/i2c-xiic.c                      |   17 +-
+ drivers/i2c/busses/i2c-xlp9xx.c                    |   89 +-
+ drivers/i2c/i2c-core-base.c                        |   10 +-
+ drivers/i2c/i2c-core-of.c                          |   51 +-
+ drivers/i2c/i2c-core-smbus.c                       |    2 +
+ drivers/i2c/i2c-mux.c                              |    2 +-
+ drivers/i2c/muxes/i2c-demux-pinctrl.c              |    5 +-
+ drivers/i2c/muxes/i2c-mux-gpio.c                   |    2 +-
+ drivers/i2c/muxes/i2c-mux-ltc4306.c                |    7 +-
+ drivers/i2c/muxes/i2c-mux-pca954x.c                |   18 +-
+ drivers/i2c/muxes/i2c-mux-reg.c                    |    4 +-
+ drivers/media/platform/marvell-ccic/mmp-driver.c   |    2 +-
+ drivers/mfd/sm501.c                                |    2 +-
+ drivers/mfd/timberdale.c                           |    4 +-
+ drivers/misc/eeprom/at24.c                         |   71 +-
+ include/linux/i2c-pnx.h                            |   38 -
+ include/linux/i2c.h                                |   12 +-
+ include/linux/{ => platform_data}/i2c-gpio.h       |    0
+ include/linux/{ => platform_data}/i2c-mux-gpio.h   |    0
+ include/linux/{ => platform_data}/i2c-ocores.h     |    0
+ include/linux/{ => platform_data}/i2c-omap.h       |    0
+ .../linux/{ => platform_data}/i2c-pca-platform.h   |    0
+ include/linux/{ => platform_data}/i2c-xiic.h       |    0
+ 94 files changed, 1695 insertions(+), 485 deletions(-)
+ create mode 100644 drivers/i2c/busses/i2c-stm32.c
+ delete mode 100644 include/linux/i2c-pnx.h
+ rename include/linux/{ => platform_data}/i2c-gpio.h (100%)
+ rename include/linux/{ => platform_data}/i2c-mux-gpio.h (100%)
+ rename include/linux/{ => platform_data}/i2c-ocores.h (100%)
+ rename include/linux/{ => platform_data}/i2c-omap.h (100%)
+ rename include/linux/{ => platform_data}/i2c-pca-platform.h (100%)
+ rename include/linux/{ => platform_data}/i2c-xiic.h (100%)
+Merging dmi/master (5cb06cfce4a8 firmware: dmi: Add access to the SKU ID string)
+$ git merge dmi/master
+Merge made by the 'recursive' strategy.
+ drivers/firmware/dmi-id.c       | 2 ++
+ drivers/firmware/dmi_scan.c     | 1 +
+ include/linux/mod_devicetable.h | 1 +
+ 3 files changed, 4 insertions(+)
+Merging hwmon-staging/hwmon-next (853d5e4b9a48 hwmon: (gpio-fan) Fix "#cooling-cells" property name in bindings)
+$ git merge hwmon-staging/hwmon-next
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/hwmon/gpio-fan.txt         |   2 +-
+ .../devicetree/bindings/hwmon/ltc2990.txt          |  36 ++++
+ Documentation/hwmon/hwmon-kernel-api.txt           |   3 +-
+ Documentation/hwmon/ltc2990                        |  24 ++-
+ MAINTAINERS                                        |   1 +
+ drivers/hwmon/Kconfig                              |   7 +-
+ drivers/hwmon/fschmd.c                             |   2 +-
+ drivers/hwmon/hwmon.c                              |   3 +
+ drivers/hwmon/k10temp.c                            |  60 +++++-
+ drivers/hwmon/ltc2990.c                            | 212 +++++++++++++++++----
+ drivers/hwmon/mc13783-adc.c                        |  60 ++++++
+ drivers/mfd/mc13xxx-core.c                         |  15 +-
+ include/linux/mfd/mc13xxx.h                        |   2 +
+ 13 files changed, 361 insertions(+), 66 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/hwmon/ltc2990.txt
+Merging jc_docs/docs-next (a49d9c0ae46e Documentation: document hung_task_panic kernel parameter)
+$ git merge jc_docs/docs-next
+Auto-merging mm/util.c
+Auto-merging mm/mmap.c
+Auto-merging mm/Kconfig
+Auto-merging fs/proc/task_mmu.c
+Auto-merging arch/powerpc/Kconfig
+Auto-merging arch/mips/Kconfig
+Auto-merging arch/ia64/Kconfig
+Auto-merging arch/alpha/Kconfig
+Auto-merging MAINTAINERS
+Auto-merging Documentation/vm/zswap.rst
+Auto-merging Documentation/vm/zsmalloc.rst
+Auto-merging Documentation/vm/z3fold.rst
+Auto-merging Documentation/vm/unevictable-lru.rst
+Removing Documentation/vm/transhuge.txt
+Auto-merging Documentation/vm/swap_numa.rst
+Auto-merging Documentation/vm/split_page_table_lock.rst
+Removing Documentation/vm/slub.txt
+Auto-merging Documentation/vm/remap_file_pages.rst
+Auto-merging Documentation/vm/page_owner.rst
+Auto-merging Documentation/vm/page_migration.rst
+Auto-merging Documentation/vm/page_frags.rst
+Removing Documentation/vm/overcommit-accounting
+Removing Documentation/vm/numa_memory_policy.txt
+Auto-merging Documentation/vm/numa.rst
+Removing Documentation/vm/mmu_notifier.txt
+Removing Documentation/vm/ksm.txt
+Auto-merging Documentation/vm/hwpoison.rst
+Auto-merging Documentation/vm/hugetlbfs_reserv.rst
+Auto-merging Documentation/vm/hmm.rst
+Auto-merging Documentation/vm/highmem.rst
+Auto-merging Documentation/vm/frontswap.rst
+Auto-merging Documentation/vm/cleancache.rst
+Auto-merging Documentation/vm/balance.rst
+Removing Documentation/vm/active_mm.txt
+Auto-merging Documentation/trace/ftrace.rst
+CONFLICT (modify/delete): Documentation/features/io/dma-api-debug/arch-support.txt deleted in HEAD and modified in jc_docs/docs-next. Version jc_docs/docs-next of Documentation/features/io/dma-api-debug/arch-support.txt left in tree.
+Auto-merging Documentation/features/core/eBPF-JIT/arch-support.txt
+Auto-merging Documentation/features/core/cBPF-JIT/arch-support.txt
+Auto-merging Documentation/admin-guide/mm/userfaultfd.rst
+Auto-merging Documentation/admin-guide/mm/soft-dirty.rst
+Auto-merging Documentation/admin-guide/mm/pagemap.rst
+Auto-merging Documentation/admin-guide/mm/idle_page_tracking.rst
+Auto-merging Documentation/admin-guide/mm/hugetlbpage.rst
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Automatic merge failed; fix conflicts and then commit the result.
+$ git rm -f Documentation/features/io/dma-api-debug/arch-support.txt
+Documentation/features/io/dma-api-debug/arch-support.txt: needs merge
+rm 'Documentation/features/io/dma-api-debug/arch-support.txt'
+$ git commit -v -a
+[master 59ad62997f77] Merge remote-tracking branch 'jc_docs/docs-next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/00-INDEX                             |  10 -
+ Documentation/ABI/stable/sysfs-devices-node        |   2 +-
+ .../ABI/testing/sysfs-kernel-mm-hugepages          |   2 +-
+ Documentation/ABI/testing/sysfs-kernel-mm-ksm      |   2 +-
+ Documentation/ABI/testing/sysfs-kernel-slab        |   4 +-
+ .../{bcache.txt => admin-guide/bcache.rst}         |   0
+ .../{cgroup-v2.txt => admin-guide/cgroup-v2.rst}   |   0
+ Documentation/admin-guide/index.rst                |   3 +
+ Documentation/admin-guide/kernel-parameters.txt    | 160 ++++---
+ Documentation/admin-guide/mm/concepts.rst          | 222 +++++++++
+ .../mm/hugetlbpage.rst}                            | 263 +++++-----
+ .../mm/idle_page_tracking.rst}                     |  56 ++-
+ Documentation/admin-guide/mm/index.rst             |  36 ++
+ Documentation/admin-guide/mm/ksm.rst               | 189 ++++++++
+ .../admin-guide/mm/numa_memory_policy.rst          | 495 +++++++++++++++++++
+ .../{vm/pagemap.txt => admin-guide/mm/pagemap.rst} | 184 +++----
+ .../mm/soft-dirty.rst}                             |  20 +-
+ Documentation/admin-guide/mm/transhuge.rst         | 418 ++++++++++++++++
+ .../mm/userfaultfd.rst}                            |  66 +--
+ Documentation/admin-guide/ramoops.rst              |   2 +-
+ Documentation/arm/Marvell/README                   |  15 +-
+ Documentation/block/cmdline-partition.txt          |  21 +-
+ .../{cachetlb.txt => core-api/cachetlb.rst}        |   0
+ .../circular-buffers.rst}                          |   0
+ Documentation/core-api/gfp_mask-from-fs-io.rst     |  66 +++
+ Documentation/core-api/index.rst                   |   3 +
+ Documentation/core-api/kernel-api.rst              |  60 +--
+ Documentation/core-api/refcount-vs-atomic.rst      |   2 +-
+ Documentation/crypto/index.rst                     |   1 +
+ Documentation/dev-tools/kasan.rst                  |   2 +-
+ Documentation/dev-tools/kselftest.rst              |   5 +
+ Documentation/{clk.txt => driver-api/clk.rst}      |   0
+ Documentation/driver-api/device_connection.rst     |   2 +-
+ Documentation/driver-api/gpio/driver.rst           |   6 +-
+ Documentation/driver-api/index.rst                 |   2 +
+ Documentation/driver-api/uio-howto.rst             |   3 +-
+ .../strncasecmp => core/cBPF-JIT}/arch-support.txt |  14 +-
+ .../core/{BPF-JIT => eBPF-JIT}/arch-support.txt    |   8 +-
+ .../core/generic-idle-thread/arch-support.txt      |   4 +-
+ .../features/core/jump-labels/arch-support.txt     |   2 +
+ .../features/core/tracehook/arch-support.txt       |   2 +
+ .../features/debug/KASAN/arch-support.txt          |   4 +-
+ .../debug/gcov-profile-all/arch-support.txt        |   2 +
+ Documentation/features/debug/kgdb/arch-support.txt |   4 +-
+ .../debug/kprobes-on-ftrace/arch-support.txt       |   2 +
+ .../features/debug/kprobes/arch-support.txt        |   4 +-
+ .../features/debug/kretprobes/arch-support.txt     |   4 +-
+ .../features/debug/optprobes/arch-support.txt      |   4 +-
+ .../features/debug/stackprotector/arch-support.txt |   2 +
+ .../features/debug/uprobes/arch-support.txt        |   6 +-
+ .../debug/user-ret-profiler/arch-support.txt       |   2 +
+ .../features/io/dma-contiguous/arch-support.txt    |   4 +-
+ .../features/io/sg-chain/arch-support.txt          |   2 +
+ .../locking/cmpxchg-local/arch-support.txt         |   4 +-
+ .../features/locking/lockdep/arch-support.txt      |   4 +-
+ .../locking/queued-rwlocks/arch-support.txt        |  10 +-
+ .../locking/queued-spinlocks/arch-support.txt      |   8 +-
+ .../locking/rwsem-optimized/arch-support.txt       |  10 +-
+ .../features/perf/kprobes-event/arch-support.txt   |   6 +-
+ .../features/perf/perf-regs/arch-support.txt       |   4 +-
+ .../features/perf/perf-stackdump/arch-support.txt  |   4 +-
+ .../sched/membarrier-sync-core/arch-support.txt    |   2 +
+ .../features/sched/numa-balancing/arch-support.txt |   6 +-
+ Documentation/features/scripts/features-refresh.sh |  98 ++++
+ .../seccomp/seccomp-filter/arch-support.txt        |   6 +-
+ .../time/arch-tick-broadcast/arch-support.txt      |   4 +-
+ .../features/time/clockevents/arch-support.txt     |   4 +-
+ .../time/context-tracking/arch-support.txt         |   2 +
+ .../features/time/irq-time-acct/arch-support.txt   |   4 +-
+ .../time/modern-timekeeping/arch-support.txt       |   2 +
+ .../features/time/virt-cpuacct/arch-support.txt    |   2 +
+ .../features/vm/ELF-ASLR/arch-support.txt          |   4 +-
+ .../features/vm/PG_uncached/arch-support.txt       |   2 +
+ Documentation/features/vm/THP/arch-support.txt     |   2 +
+ Documentation/features/vm/TLB/arch-support.txt     |   2 +
+ .../features/vm/huge-vmap/arch-support.txt         |   2 +
+ .../features/vm/ioremap_prot/arch-support.txt      |   2 +
+ .../features/vm/numa-memblock/arch-support.txt     |   4 +-
+ .../features/vm/pte_special/arch-support.txt       |   2 +
+ Documentation/filesystems/proc.txt                 |   8 +-
+ Documentation/filesystems/tmpfs.txt                |   5 +-
+ Documentation/index.rst                            |   3 +-
+ Documentation/ioctl/botching-up-ioctls.txt         |   4 +-
+ Documentation/memory-barriers.txt                  |   4 +-
+ Documentation/process/2.Process.rst                |  72 +--
+ Documentation/process/5.Posting.rst                |  16 +-
+ Documentation/process/index.rst                    |   1 +
+ Documentation/process/maintainer-pgp-guide.rst     |  39 +-
+ Documentation/process/submitting-patches.rst       |   2 +-
+ Documentation/security/index.rst                   |   2 +
+ Documentation/sound/alsa-configuration.rst         |   4 +-
+ Documentation/sound/soc/codec.rst                  |   2 +-
+ Documentation/sound/soc/platform.rst               |   2 +-
+ Documentation/sysctl/vm.txt                        |   6 +-
+ Documentation/trace/coresight.txt                  | 103 ++--
+ Documentation/trace/ftrace-uses.rst                |   4 +-
+ Documentation/trace/ftrace.rst                     |   7 +-
+ .../translations/ko_KR/memory-barriers.txt         |   4 +-
+ Documentation/vfio.txt                             |   5 +-
+ Documentation/vm/00-INDEX                          |  58 +--
+ Documentation/vm/active_mm.rst                     |  91 ++++
+ Documentation/vm/active_mm.txt                     |  83 ----
+ Documentation/vm/{balance => balance.rst}          |  15 +-
+ .../vm/{cleancache.txt => cleancache.rst}          | 105 ++--
+ Documentation/vm/conf.py                           |  10 +
+ Documentation/vm/{frontswap.txt => frontswap.rst}  |  59 ++-
+ Documentation/vm/{highmem.txt => highmem.rst}      |  87 ++--
+ Documentation/vm/{hmm.txt => hmm.rst}              |  78 ++-
+ .../{hugetlbfs_reserv.txt => hugetlbfs_reserv.rst} | 220 ++++++---
+ Documentation/vm/{hwpoison.txt => hwpoison.rst}    | 141 +++---
+ Documentation/vm/index.rst                         |  50 ++
+ Documentation/vm/ksm.rst                           |  87 ++++
+ Documentation/vm/ksm.txt                           | 178 -------
+ Documentation/vm/mmu_notifier.rst                  |  99 ++++
+ Documentation/vm/mmu_notifier.txt                  |  93 ----
+ Documentation/vm/{numa => numa.rst}                |   6 +-
+ Documentation/vm/numa_memory_policy.txt            | 452 ------------------
+ Documentation/vm/overcommit-accounting             |  80 ----
+ Documentation/vm/overcommit-accounting.rst         |  87 ++++
+ Documentation/vm/{page_frags => page_frags.rst}    |   5 +-
+ .../vm/{page_migration => page_migration.rst}      | 149 +++---
+ .../vm/{page_owner.txt => page_owner.rst}          |  34 +-
+ .../{remap_file_pages.txt => remap_file_pages.rst} |   6 +
+ Documentation/vm/slub.rst                          | 361 ++++++++++++++
+ Documentation/vm/slub.txt                          | 342 -------------
+ ...t_page_table_lock => split_page_table_lock.rst} |  12 +-
+ Documentation/vm/{swap_numa.txt => swap_numa.rst}  |  55 ++-
+ Documentation/vm/transhuge.rst                     | 197 ++++++++
+ Documentation/vm/transhuge.txt                     | 527 ---------------------
+ .../{unevictable-lru.txt => unevictable-lru.rst}   | 117 ++---
+ Documentation/vm/{z3fold.txt => z3fold.rst}        |   6 +-
+ Documentation/vm/{zsmalloc.txt => zsmalloc.rst}    |  60 ++-
+ Documentation/vm/{zswap.txt => zswap.rst}          |  71 +--
+ LICENSES/exceptions/Linux-syscall-note             |   2 +-
+ LICENSES/other/Apache-2.0                          | 183 +++++++
+ LICENSES/other/CC-BY-SA-4.0                        | 397 ++++++++++++++++
+ LICENSES/other/CDDL-1.0                            | 364 ++++++++++++++
+ LICENSES/other/Linux-OpenIB                        |  26 +
+ LICENSES/other/X11                                 |  37 ++
+ LICENSES/preferred/GPL-2.0                         |   6 +
+ MAINTAINERS                                        |   2 +-
+ arch/alpha/Kconfig                                 |   2 +-
+ arch/ia64/Kconfig                                  |   2 +-
+ arch/mips/Kconfig                                  |   2 +-
+ arch/powerpc/Kconfig                               |   2 +-
+ drivers/w1/w1_io.c                                 |   1 +
+ fs/Kconfig                                         |   2 +-
+ fs/dax.c                                           |   2 +-
+ fs/proc/task_mmu.c                                 |   4 +-
+ include/linux/hmm.h                                |   2 +-
+ include/linux/memremap.h                           |   4 +-
+ include/linux/mmu_notifier.h                       |   2 +-
+ include/linux/sched/mm.h                           |  42 +-
+ include/linux/swap.h                               |   2 +-
+ mm/Kconfig                                         |   7 +-
+ mm/cleancache.c                                    |   2 +-
+ mm/frontswap.c                                     |   2 +-
+ mm/hmm.c                                           |   2 +-
+ mm/huge_memory.c                                   |   4 +-
+ mm/hugetlb.c                                       |   4 +-
+ mm/ksm.c                                           |  23 +-
+ mm/mmap.c                                          |   2 +-
+ mm/rmap.c                                          |   6 +-
+ mm/util.c                                          |   2 +-
+ scripts/documentation-file-ref-check               | 125 ++++-
+ scripts/spdxcheck.py                               | 284 +++++++++++
+ 166 files changed, 5535 insertions(+), 2920 deletions(-)
+ rename Documentation/{bcache.txt => admin-guide/bcache.rst} (100%)
+ rename Documentation/{cgroup-v2.txt => admin-guide/cgroup-v2.rst} (100%)
+ create mode 100644 Documentation/admin-guide/mm/concepts.rst
+ rename Documentation/{vm/hugetlbpage.txt => admin-guide/mm/hugetlbpage.rst} (61%)
+ rename Documentation/{vm/idle_page_tracking.txt => admin-guide/mm/idle_page_tracking.rst} (72%)
+ create mode 100644 Documentation/admin-guide/mm/index.rst
+ create mode 100644 Documentation/admin-guide/mm/ksm.rst
+ create mode 100644 Documentation/admin-guide/mm/numa_memory_policy.rst
+ rename Documentation/{vm/pagemap.txt => admin-guide/mm/pagemap.rst} (53%)
+ rename Documentation/{vm/soft-dirty.txt => admin-guide/mm/soft-dirty.rst} (67%)
+ create mode 100644 Documentation/admin-guide/mm/transhuge.rst
+ rename Documentation/{vm/userfaultfd.txt => admin-guide/mm/userfaultfd.rst} (89%)
+ rename Documentation/{cachetlb.txt => core-api/cachetlb.rst} (100%)
+ rename Documentation/{circular-buffers.txt => core-api/circular-buffers.rst} (100%)
+ create mode 100644 Documentation/core-api/gfp_mask-from-fs-io.rst
+ rename Documentation/{clk.txt => driver-api/clk.rst} (100%)
+ rename Documentation/features/{lib/strncasecmp => core/cBPF-JIT}/arch-support.txt (70%)
+ rename Documentation/features/core/{BPF-JIT => eBPF-JIT}/arch-support.txt (79%)
+ create mode 100755 Documentation/features/scripts/features-refresh.sh
+ create mode 100644 Documentation/vm/active_mm.rst
+ delete mode 100644 Documentation/vm/active_mm.txt
+ rename Documentation/vm/{balance => balance.rst} (96%)
+ rename Documentation/vm/{cleancache.txt => cleancache.rst} (83%)
+ create mode 100644 Documentation/vm/conf.py
+ rename Documentation/vm/{frontswap.txt => frontswap.rst} (91%)
+ rename Documentation/vm/{highmem.txt => highmem.rst} (64%)
+ rename Documentation/vm/{hmm.txt => hmm.rst} (91%)
+ rename Documentation/vm/{hugetlbfs_reserv.txt => hugetlbfs_reserv.rst} (86%)
+ rename Documentation/vm/{hwpoison.txt => hwpoison.rst} (60%)
+ create mode 100644 Documentation/vm/index.rst
+ create mode 100644 Documentation/vm/ksm.rst
+ delete mode 100644 Documentation/vm/ksm.txt
+ create mode 100644 Documentation/vm/mmu_notifier.rst
+ delete mode 100644 Documentation/vm/mmu_notifier.txt
+ rename Documentation/vm/{numa => numa.rst} (98%)
+ delete mode 100644 Documentation/vm/numa_memory_policy.txt
+ delete mode 100644 Documentation/vm/overcommit-accounting
+ create mode 100644 Documentation/vm/overcommit-accounting.rst
+ rename Documentation/vm/{page_frags => page_frags.rst} (97%)
+ rename Documentation/vm/{page_migration => page_migration.rst} (63%)
+ rename Documentation/vm/{page_owner.txt => page_owner.rst} (86%)
+ rename Documentation/vm/{remap_file_pages.txt => remap_file_pages.rst} (92%)
+ create mode 100644 Documentation/vm/slub.rst
+ delete mode 100644 Documentation/vm/slub.txt
+ rename Documentation/vm/{split_page_table_lock => split_page_table_lock.rst} (95%)
+ rename Documentation/vm/{swap_numa.txt => swap_numa.rst} (74%)
+ create mode 100644 Documentation/vm/transhuge.rst
+ delete mode 100644 Documentation/vm/transhuge.txt
+ rename Documentation/vm/{unevictable-lru.txt => unevictable-lru.rst} (92%)
+ rename Documentation/vm/{z3fold.txt => z3fold.rst} (97%)
+ rename Documentation/vm/{zsmalloc.txt => zsmalloc.rst} (71%)
+ rename Documentation/vm/{zswap.txt => zswap.rst} (74%)
+ create mode 100644 LICENSES/other/Apache-2.0
+ create mode 100644 LICENSES/other/CC-BY-SA-4.0
+ create mode 100644 LICENSES/other/CDDL-1.0
+ create mode 100644 LICENSES/other/Linux-OpenIB
+ create mode 100644 LICENSES/other/X11
+ create mode 100755 scripts/spdxcheck.py
+Merging v4l-dvb/master (a00031c15974 media: ddbridge: conditionally enable fast TS for stv0910-equipped bridges)
+$ git merge v4l-dvb/master
+Removing include/media/videobuf-dvb.h
+Auto-merging drivers/staging/media/zoran/zoran_driver.c
+Auto-merging drivers/staging/media/zoran/Kconfig
+Removing drivers/staging/media/atomisp/platform/intel-mid/atomisp_gmin_platform.c
+Removing drivers/staging/media/atomisp/platform/intel-mid/Makefile
+Removing drivers/staging/media/atomisp/platform/Makefile
+Removing drivers/staging/media/atomisp/pci/atomisp2/mmu/sh_mmu_mrfld.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/mmu/isp_mmu.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/include/mmu/sh_mmu_mrfld.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/include/mmu/isp_mmu.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_vm.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_pool.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_common.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_bo.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_custom_host_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_vm.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_reserved_pool.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_dynamic_pool.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_version.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_uds.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_struct.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream_format.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream_format.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_sp.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_sp.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_shading.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_properties.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_pipe.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params_internal.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_shading.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_shading.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_dvs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_dvs.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_morph.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mmu.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mipi.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mipi.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metrics.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metrics.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metadata.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_legacy.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_internal.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_hrt.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_host_data.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_frac.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_dvs_info.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/timer/src/timer.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/tagger/interface/ia_css_tagger_common.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/src/spctrl.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/interface/ia_css_spctrl_comm.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/interface/ia_css_spctrl.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/src/rmgr_vbuf.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/src/rmgr.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/interface/ia_css_rmgr_vbuf.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/interface/ia_css_rmgr.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue_access.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue_access.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/interface/ia_css_queue_comm.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/interface/ia_css_queue.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/src/pipeline.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/interface/ia_css_pipeline_common.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/interface/ia_css_pipeline.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/virtual_isys.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/virtual_isys.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/rx.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_stream2mmio_rmgr.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_stream2mmio_rmgr.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_init.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_dma_rmgr.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_dma_rmgr.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/csi_rx_rmgr.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/csi_rx_rmgr.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/interface/ia_css_isys_comm.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/interface/ia_css_isys.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/src/isp_param.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/interface/ia_css_isp_param_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/interface/ia_css_isp_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/inputfifo/src/inputfifo.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/inputfifo/interface/ia_css_inputfifo.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/ifmtr/src/ifmtr.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/ifmtr/interface/ia_css_ifmtr.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/src/frame.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/interface/ia_css_frame_comm.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/interface/ia_css_frame.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/eventq/src/eventq.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/eventq/interface/ia_css_eventq.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/event/src/event.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/event/interface/ia_css_event.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/src/ia_css_debug.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug_pipe.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug_internal.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/src/bufq.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/interface/ia_css_bufq_comm.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/interface/ia_css_bufq.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/binary/src/binary.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/binary/interface/ia_css_binary.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/memory_realloc.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/isp_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/isp_const.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/input_buf.isp.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr_state.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_state.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/uds/uds_1.0/ia_css_uds_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/common/ia_css_sdis_common_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/common/ia_css_sdis_common.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_state.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/common/ia_css_common_io_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/common/ia_css_common_io_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de_state.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_state.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2_param.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_version_data.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_version.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_tpg.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_timer.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream_format.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_shading.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_properties.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_prbs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_pipe_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_pipe.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_morph.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mmu_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mmu.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mipi.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_metadata.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_memory_access.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_irq.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_input_port.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_host_data.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frame_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frame_format.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frac.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_firmware.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_event_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_err.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_env.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_dvs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_device_access.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_device_access.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_control.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_buffer.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_acc_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_3a.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/tag_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/sw_event_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/queue_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/queue_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/queue_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/vmem.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/vamem.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/type_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/timed_ctrl.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/tag.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/system_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/string_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/sp.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/socket.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/resource.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/queue.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/print_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/platform_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/pixelgen.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/mmu_device.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/misc_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/memory_realloc.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/memory_access/memory_access.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/math_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_stream2mmio.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_irq.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_dma.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isp.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/irq.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/input_system.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/input_formatter.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/ibuf_ctrl.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/vmem_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/vamem_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/timed_ctrl_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/tag_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/sp_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/pixelgen_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/mmu_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_stream2mmio_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_irq_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_dma_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isp_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/irq_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/input_system_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/input_formatter_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/ibuf_ctrl_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/hmem_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gpio_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gp_timer_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gp_device_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gdc_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/fifo_monitor_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/event_fifo_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/dma_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/debug_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/csi_rx_public.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/hmem.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gpio.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gp_timer.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gp_device.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gdc_device.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/fifo_monitor.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/event_fifo.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/error_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/dma.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/device_access/device_access.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/debug.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/csi_rx.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/bitop_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/assert_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/vmem_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/vamem_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/timed_ctrl_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/system_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/sp_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/mmu_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/isp_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/irq_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/input_system_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/input_formatter_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vamem_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vamem_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/system_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gpio_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gpio_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/hmem_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gpio_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gp_timer_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gp_device_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gdc_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/fifo_monitor_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/event_fifo_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/dma_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/debug_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_trace.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/spmem_dump.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/version.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/var.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/timed_controller_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/streaming_to_mipi_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/str2mem_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/scalar_processor_2400_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/mmu_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp_capture_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp_acquisition_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp2401_mamoiada_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp2400_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/irq_controller_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_system_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_system_ctrl_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_switch_2400_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_selector_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_formatter_subsystem_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/if_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_host_ids_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_2401_irq_types_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gpio_block_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gp_timer_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gdc_v2_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/dma_v2_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/css_receiver_2400_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/css_receiver_2400_common_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/cell_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/bits.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_states.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_states.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_params.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_configs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_configs.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/system_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/spmem_dump.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/pixelgen_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_stream2mmio_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_irq_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_dma_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/input_system_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/ibuf_ctrl_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/version.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/var.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/timed_controller_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/streaming_to_mipi_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/stream2mmio_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/str2mem_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/scalar_processor_2400_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/rx_csi_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mmu_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mipi_backend_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mipi_backend_common_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp_capture_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp_acquisition_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp2401_mamoiada_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp2400_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/irq_controller_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_system_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_system_ctrl_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_switch_2400_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_selector_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_formatter_subsystem_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/if_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/ibuf_cntrl_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_host_ids_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_2401_irq_types_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gpio_block_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gp_timer_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gdc_v2_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/dma_v2_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/css_receiver_2400_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/css_receiver_2400_common_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/cell_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/bits.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/PixelGen_SysBlock_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/system_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/pixelgen_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/pixelgen_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/input_system_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/input_system_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx_private.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx_local.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_states.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_states.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_params.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_configs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_configs.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/csi_rx_global.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/spmem_dump.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/version.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/var.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/timed_controller_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/streaming_to_mipi_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/str2mem_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/scalar_processor_2400_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/mmu_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp_capture_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp_acquisition_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp2400_support.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp2400_mamoiada_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/irq_controller_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_system_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_system_ctrl_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_switch_2400_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_selector_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_formatter_subsystem_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/if_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_types.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_irq_types_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_host_ids_hrt.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gpio_block_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gp_timer_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gdc_v2_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/dma_v2_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/css_receiver_2400_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/css_receiver_2400_common_defs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/cell_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/bits.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_states.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_states.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_params.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_params.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_configs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_configs.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/util/src/util.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/util/interface/ia_css_util.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_util.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_stagedesc.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_binarydesc.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_util.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_stagedesc.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_binarydesc.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/base/refcount/src/refcount.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/base/refcount/interface/ia_css_refcount.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/src/circbuf.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf_desc.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf_comm.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/css2400/Makefile
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_trace_event.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_tpg.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_tpg.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_tables.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_internal.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_helper.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_file.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_file.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_drvfs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_drvfs.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_dfs_tables.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_csi2.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_csi2.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_ioctl32.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_ioctl32.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_common.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_acc.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp_acc.c
+Removing drivers/staging/media/atomisp/pci/atomisp2/atomisp-regs.h
+Removing drivers/staging/media/atomisp/pci/atomisp2/Makefile
+Removing drivers/staging/media/atomisp/pci/Makefile
+Removing drivers/staging/media/atomisp/pci/Kconfig
+Removing drivers/staging/media/atomisp/include/media/lm3554.h
+Removing drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h
+Removing drivers/staging/media/atomisp/include/linux/atomisp_platform.h
+Removing drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h
+Removing drivers/staging/media/atomisp/include/linux/atomisp.h
+Removing drivers/staging/media/atomisp/i2c/ov5693/ov5693.h
+Removing drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c
+Removing drivers/staging/media/atomisp/i2c/ov5693/ad5823.h
+Removing drivers/staging/media/atomisp/i2c/ov5693/Makefile
+Removing drivers/staging/media/atomisp/i2c/ov5693/Kconfig
+Removing drivers/staging/media/atomisp/i2c/ov2722.h
+Removing drivers/staging/media/atomisp/i2c/ov2680.h
+Removing drivers/staging/media/atomisp/i2c/mt9m114.h
+Removing drivers/staging/media/atomisp/i2c/gc2235.h
+Removing drivers/staging/media/atomisp/i2c/gc0310.h
+Removing drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
+Removing drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
+Removing drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c
+Removing drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
+Removing drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c
+Removing drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
+Removing drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
+Removing drivers/staging/media/atomisp/i2c/Makefile
+Removing drivers/staging/media/atomisp/i2c/Kconfig
+Removing drivers/staging/media/atomisp/TODO
+Removing drivers/staging/media/atomisp/Makefile
+Removing drivers/staging/media/atomisp/Kconfig
+Removing drivers/media/v4l2-core/videobuf-dvb.c
+Auto-merging drivers/media/platform/vsp1/vsp1_brx.c
+Removing drivers/media/platform/vsp1/vsp1_bru.h
+Auto-merging drivers/media/platform/marvell-ccic/mmp-driver.c
+Removing drivers/media/pci/pt1/va1j5jf8007t.h
+Removing drivers/media/pci/pt1/va1j5jf8007t.c
+Removing drivers/media/pci/pt1/va1j5jf8007s.h
+Removing drivers/media/pci/pt1/va1j5jf8007s.c
+Auto-merging MAINTAINERS
+Auto-merging Documentation/media/uapi/v4l/selection-api-configuration.rst
+Removing Documentation/media/uapi/v4l/selection-api-005.rst
+Merge made by the 'recursive' strategy.
+ Documentation/ABI/testing/sysfs-class-rc           |    16 +-
+ Documentation/ABI/testing/sysfs-class-rc-nuvoton   |     2 +-
+ Documentation/ABI/testing/sysfs-devices-edac       |    14 +-
+ .../devicetree/bindings/media/cdns,csi2rx.txt      |   100 +
+ .../devicetree/bindings/media/cdns,csi2tx.txt      |    98 +
+ .../devicetree/bindings/media/i2c/ov7251.txt       |    52 +
+ .../devicetree/bindings/media/i2c/ov772x.txt       |    40 +
+ .../bindings/media/i2c/panasonic,amg88xx.txt       |    19 +
+ .../devicetree/bindings/media/rcar_vin.txt         |   138 +-
+ .../devicetree/bindings/media/renesas,ceu.txt      |     7 +-
+ .../bindings/media/renesas,rcar-csi2.txt           |   101 +
+ Documentation/media/kapi/cec-core.rst              |     5 +-
+ Documentation/media/uapi/cec/cec-ioc-receive.rst   |    24 +-
+ Documentation/media/uapi/dvb/dvbapi.rst            |     2 +-
+ Documentation/media/uapi/rc/lirc-dev-intro.rst     |     2 +-
+ .../media/uapi/rc/lirc-set-rec-timeout.rst         |    14 +-
+ Documentation/media/uapi/v4l/common.rst            |     2 +-
+ Documentation/media/uapi/v4l/crop.rst              |    22 +-
+ Documentation/media/uapi/v4l/selection-api-005.rst |    34 -
+ ...api-004.rst => selection-api-configuration.rst} |     2 +-
+ ...tion-api-006.rst => selection-api-examples.rst} |     0
+ ...lection-api-002.rst => selection-api-intro.rst} |     0
+ ...ction-api-003.rst => selection-api-targets.rst} |     0
+ .../media/uapi/v4l/selection-api-vs-crop-api.rst   |    39 +
+ Documentation/media/uapi/v4l/selection-api.rst     |    14 +-
+ Documentation/media/uapi/v4l/selection.svg         |     4 +-
+ Documentation/media/uapi/v4l/v4l2.rst              |     2 +-
+ .../media/v4l-drivers/cx23885-cardlist.rst         |    18 +-
+ .../media/v4l-drivers/em28xx-cardlist.rst          |    10 +-
+ MAINTAINERS                                        |    31 +-
+ drivers/gpu/drm/rcar-du/rcar_du_crtc.c             |   156 +-
+ drivers/gpu/drm/rcar-du/rcar_du_crtc.h             |    15 +
+ drivers/gpu/drm/rcar-du/rcar_du_vsp.c              |    12 +-
+ drivers/media/Kconfig                              |    12 +-
+ drivers/media/cec/cec-adap.c                       |    19 +-
+ drivers/media/cec/cec-core.c                       |     2 +-
+ drivers/media/cec/cec-pin-error-inj.c              |    33 +-
+ drivers/media/cec/cec-pin.c                        |     2 +-
+ drivers/media/common/b2c2/flexcop-fe-tuner.c       |     4 +-
+ drivers/media/common/b2c2/flexcop-i2c.c            |    47 +-
+ drivers/media/common/b2c2/flexcop.c                |     2 +-
+ drivers/media/common/b2c2/flexcop.h                |     1 +
+ drivers/media/common/saa7146/saa7146_i2c.c         |     4 +-
+ drivers/media/common/siano/smscoreapi.c            |    32 +-
+ drivers/media/common/siano/smscoreapi.h            |     3 +
+ drivers/media/common/siano/smsendian.c             |    14 +-
+ drivers/media/common/videobuf2/Kconfig             |     2 -
+ drivers/media/common/videobuf2/videobuf2-core.c    |     9 +-
+ drivers/media/dvb-core/dmxdev.c                    |     2 +-
+ drivers/media/dvb-core/dvb_ca_en50221.c            |     2 +
+ drivers/media/dvb-core/dvb_frontend.c              |   230 +-
+ drivers/media/dvb-core/dvb_net.c                   |     2 +-
+ drivers/media/dvb-core/dvbdev.c                    |     4 +
+ drivers/media/dvb-frontends/Kconfig                |     2 +-
+ drivers/media/dvb-frontends/as102_fe.h             |     2 +-
+ drivers/media/dvb-frontends/au8522_decoder.c       |    14 +-
+ drivers/media/dvb-frontends/cx24116.c              |     2 +-
+ drivers/media/dvb-frontends/cx24117.c              |     2 +-
+ drivers/media/dvb-frontends/cx24120.c              |     2 +-
+ drivers/media/dvb-frontends/cx24123.c              |     2 +-
+ drivers/media/dvb-frontends/cxd2099.c              |     4 +-
+ drivers/media/dvb-frontends/cxd2099.h              |     2 +-
+ drivers/media/dvb-frontends/cxd2820r_core.c        |     2 +-
+ .../cxd2880/cxd2880_tnrdmd_driver_version.h        |     4 +-
+ drivers/media/dvb-frontends/cxd2880/cxd2880_top.c  |    14 +-
+ drivers/media/dvb-frontends/dvb-pll.c              |   110 +
+ drivers/media/dvb-frontends/dvb-pll.h              |     6 +
+ drivers/media/dvb-frontends/l64781.c               |     4 +-
+ drivers/media/dvb-frontends/lgdt3306a.c            |     2 +-
+ drivers/media/dvb-frontends/lgdt330x.c             |   874 +-
+ drivers/media/dvb-frontends/lgdt330x.h             |    41 +-
+ drivers/media/dvb-frontends/mb86a20s.c             |     2 +-
+ drivers/media/dvb-frontends/mxl5xx.c               |     2 +-
+ drivers/media/dvb-frontends/s921.c                 |     2 +-
+ drivers/media/dvb-frontends/stv0910.c              |    15 +-
+ drivers/media/dvb-frontends/stv0910.h              |     1 +
+ drivers/media/dvb-frontends/tc90522.c              |    15 +-
+ drivers/media/dvb-frontends/tc90522.h              |    11 +-
+ drivers/media/i2c/Kconfig                          |    36 +
+ drivers/media/i2c/Makefile                         |     3 +
+ drivers/media/i2c/adv748x/adv748x-afe.c            |    12 +-
+ drivers/media/i2c/adv748x/adv748x-hdmi.c           |     8 +-
+ drivers/media/i2c/adv7511.c                        |    22 +-
+ drivers/media/i2c/imx258.c                         |  1318 +++
+ drivers/media/i2c/imx274.c                         |    74 +-
+ drivers/media/i2c/ir-kbd-i2c.c                     |     4 +-
+ drivers/media/i2c/ov13858.c                        |     1 -
+ drivers/media/i2c/ov2640.c                         |   112 +-
+ drivers/media/i2c/ov5640.c                         |   257 +-
+ drivers/media/i2c/ov5645.c                         |     6 +-
+ drivers/media/i2c/ov5695.c                         |     1 -
+ drivers/media/i2c/ov7251.c                         |  1503 +++
+ drivers/media/i2c/ov772x.c                         |     2 +-
+ drivers/media/i2c/ov7740.c                         |    22 +-
+ drivers/media/i2c/smiapp/smiapp-core.c             |    11 +-
+ drivers/media/i2c/tda1997x.c                       |     4 +-
+ drivers/media/i2c/tvp5150.c                        |   159 +-
+ drivers/media/i2c/video-i2c.c                      |   564 +
+ drivers/media/media-device.c                       |    21 +-
+ drivers/media/mmc/siano/smssdio.c                  |     2 +-
+ drivers/media/pci/Kconfig                          |     1 -
+ drivers/media/pci/Makefile                         |     1 -
+ drivers/media/pci/bt8xx/bttv-risc.c                |    17 +-
+ drivers/media/pci/bt8xx/dst.c                      |     2 +-
+ drivers/media/pci/bt8xx/dvb-bt8xx.c                |     4 +-
+ drivers/media/pci/cx23885/cx23885-core.c           |   132 +-
+ drivers/media/pci/cx23885/cx23885-dvb.c            |     6 +-
+ drivers/media/pci/cx23885/cx23885-reg.h            |    14 +
+ drivers/media/pci/cx88/cx88-dvb.c                  |     7 +-
+ drivers/media/pci/cx88/cx88-input.c                |    11 +-
+ drivers/media/pci/cx88/cx88-vbi.c                  |     1 -
+ drivers/media/pci/ddbridge/Kconfig                 |     1 +
+ drivers/media/pci/ddbridge/Makefile                |     2 +-
+ drivers/media/pci/ddbridge/ddbridge-ci.c           |     2 +-
+ drivers/media/pci/ddbridge/ddbridge-core.c         |   419 +-
+ drivers/media/pci/ddbridge/ddbridge-hw.c           |    11 +
+ drivers/media/pci/ddbridge/ddbridge-i2c.c          |     5 +-
+ drivers/media/pci/ddbridge/ddbridge-main.c         |    91 +-
+ drivers/media/pci/ddbridge/ddbridge-max.c          |    42 +
+ drivers/media/pci/ddbridge/ddbridge-max.h          |     1 +
+ drivers/media/pci/ddbridge/ddbridge-mci.c          |   551 +
+ drivers/media/pci/ddbridge/ddbridge-mci.h          |   156 +
+ drivers/media/pci/ddbridge/ddbridge-regs.h         |     4 +
+ drivers/media/pci/ddbridge/ddbridge.h              |    50 +-
+ drivers/media/pci/dt3155/Kconfig                   |     1 -
+ drivers/media/pci/intel/ipu3/Kconfig               |    16 +-
+ drivers/media/pci/intel/ipu3/ipu3-cio2.c           |    32 +-
+ drivers/media/pci/mantis/mantis_uart.c             |     7 +
+ drivers/media/pci/meye/Kconfig                     |     3 +-
+ drivers/media/pci/ngene/ngene-cards.c              |    18 +-
+ drivers/media/pci/ngene/ngene-dvb.c                |     2 +-
+ drivers/media/pci/pt1/Kconfig                      |     3 +
+ drivers/media/pci/pt1/Makefile                     |     3 +-
+ drivers/media/pci/pt1/pt1.c                        |   471 +-
+ drivers/media/pci/pt1/va1j5jf8007s.c               |   732 --
+ drivers/media/pci/pt1/va1j5jf8007s.h               |    42 -
+ drivers/media/pci/pt1/va1j5jf8007t.c               |   532 -
+ drivers/media/pci/pt1/va1j5jf8007t.h               |    42 -
+ drivers/media/pci/pt3/pt3.c                        |    70 +-
+ drivers/media/pci/pt3/pt3.h                        |    11 +-
+ drivers/media/pci/pt3/pt3_dma.c                    |    11 +-
+ drivers/media/pci/pt3/pt3_i2c.c                    |    11 +-
+ drivers/media/pci/saa7164/saa7164-fw.c             |     3 +-
+ drivers/media/pci/solo6x10/Kconfig                 |     1 -
+ drivers/media/pci/sta2x11/Kconfig                  |     3 +-
+ drivers/media/pci/sta2x11/sta2x11_vip.c            |    31 +-
+ drivers/media/pci/tw5864/Kconfig                   |     1 -
+ drivers/media/pci/tw686x/Kconfig                   |     1 -
+ drivers/media/pci/tw686x/tw686x-video.c            |     3 +-
+ drivers/media/platform/Kconfig                     |    57 +-
+ drivers/media/platform/Makefile                    |     1 +
+ drivers/media/platform/am437x/Kconfig              |     2 +-
+ drivers/media/platform/am437x/am437x-vpfe.c        |     6 +-
+ drivers/media/platform/atmel/Kconfig               |     4 +-
+ drivers/media/platform/cadence/Kconfig             |    34 +
+ drivers/media/platform/cadence/Makefile            |     4 +
+ drivers/media/platform/cadence/cdns-csi2rx.c       |   498 +
+ drivers/media/platform/cadence/cdns-csi2tx.c       |   563 +
+ drivers/media/platform/cec-gpio/cec-gpio.c         |     2 +-
+ drivers/media/platform/coda/coda-common.c          |    45 +-
+ drivers/media/platform/davinci/Kconfig             |    12 +-
+ drivers/media/platform/davinci/isif.c              |     4 +-
+ drivers/media/platform/davinci/vpbe.c              |    38 +-
+ drivers/media/platform/davinci/vpbe_display.c      |    33 +-
+ drivers/media/platform/davinci/vpbe_osd.c          |    21 +-
+ drivers/media/platform/davinci/vpbe_venc.c         |    11 +-
+ drivers/media/platform/davinci/vpfe_capture.c      |     2 +-
+ drivers/media/platform/exynos4-is/Kconfig          |     4 +-
+ drivers/media/platform/exynos4-is/fimc-lite-reg.c  |     2 +-
+ drivers/media/platform/fsl-viu.c                   |    63 +-
+ drivers/media/platform/marvell-ccic/Kconfig        |     5 +-
+ drivers/media/platform/marvell-ccic/mmp-driver.c   |     6 +-
+ drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c    |     5 +-
+ drivers/media/platform/omap/Kconfig                |     7 +-
+ drivers/media/platform/omap/omap_vout.c            |    17 +-
+ drivers/media/platform/omap/omap_vout_vrfb.c       |     4 +-
+ drivers/media/platform/omap3isp/isp.c              |    22 +-
+ drivers/media/platform/omap3isp/ispccdc.c          |     2 +-
+ drivers/media/platform/omap3isp/isph3a_aewb.c      |     2 +
+ drivers/media/platform/omap3isp/isph3a_af.c        |     2 +
+ drivers/media/platform/omap3isp/isphist.c          |     2 +
+ drivers/media/platform/omap3isp/isppreview.c       |     6 +-
+ drivers/media/platform/omap3isp/ispstat.c          |    37 +-
+ drivers/media/platform/omap3isp/ispstat.h          |     4 +-
+ drivers/media/platform/omap3isp/ispvideo.c         |     2 +-
+ drivers/media/platform/pxa_camera.c                |    17 +-
+ drivers/media/platform/rcar-vin/Kconfig            |    16 +-
+ drivers/media/platform/rcar-vin/Makefile           |     1 +
+ drivers/media/platform/rcar-vin/rcar-core.c        |   959 +-
+ drivers/media/platform/rcar-vin/rcar-csi2.c        |  1084 ++
+ drivers/media/platform/rcar-vin/rcar-dma.c         |   912 +-
+ drivers/media/platform/rcar-vin/rcar-v4l2.c        |   490 +-
+ drivers/media/platform/rcar-vin/rcar-vin.h         |   146 +-
+ drivers/media/platform/rcar_jpu.c                  |     4 +-
+ drivers/media/platform/renesas-ceu.c               |    23 +-
+ .../media/platform/s5p-jpeg/jpeg-hw-exynos3250.c   |     4 +-
+ drivers/media/platform/s5p-mfc/s5p_mfc_dec.c       |     4 +-
+ drivers/media/platform/s5p-mfc/s5p_mfc_enc.c       |     4 +-
+ drivers/media/platform/soc_camera/Kconfig          |     3 +-
+ .../platform/soc_camera/soc_camera_platform.c      |     3 +-
+ drivers/media/platform/sti/bdisp/bdisp-hw.c        |     2 +-
+ drivers/media/platform/sti/bdisp/bdisp-v4l2.c      |     4 +
+ drivers/media/platform/sti/c8sectpfe/Kconfig       |     2 +-
+ drivers/media/platform/sti/hva/hva-mem.c           |     2 +-
+ drivers/media/platform/sti/hva/hva-v4l2.c          |     4 +
+ drivers/media/platform/via-camera.c                |     2 +-
+ drivers/media/platform/video-mux.c                 |    16 +-
+ drivers/media/platform/vivid/vivid-vid-common.c    |     2 +-
+ drivers/media/platform/vsp1/Makefile               |     4 +-
+ drivers/media/platform/vsp1/vsp1.h                 |    16 +-
+ drivers/media/platform/vsp1/vsp1_bru.h             |    48 -
+ .../media/platform/vsp1/{vsp1_bru.c => vsp1_brx.c} |   218 +-
+ drivers/media/platform/vsp1/vsp1_brx.h             |    44 +
+ drivers/media/platform/vsp1/vsp1_clu.c             |   184 +-
+ drivers/media/platform/vsp1/vsp1_clu.h             |     7 +-
+ drivers/media/platform/vsp1/vsp1_dl.c              |   441 +-
+ drivers/media/platform/vsp1/vsp1_dl.h              |    36 +-
+ drivers/media/platform/vsp1/vsp1_drm.c             |   951 +-
+ drivers/media/platform/vsp1/vsp1_drm.h             |    31 +-
+ drivers/media/platform/vsp1/vsp1_drv.c             |    38 +-
+ drivers/media/platform/vsp1/vsp1_entity.c          |   137 +-
+ drivers/media/platform/vsp1/vsp1_entity.h          |    60 +-
+ drivers/media/platform/vsp1/vsp1_hgo.c             |    32 +-
+ drivers/media/platform/vsp1/vsp1_hgo.h             |     6 +-
+ drivers/media/platform/vsp1/vsp1_hgt.c             |    34 +-
+ drivers/media/platform/vsp1/vsp1_hgt.h             |     6 +-
+ drivers/media/platform/vsp1/vsp1_histo.c           |    67 +-
+ drivers/media/platform/vsp1/vsp1_histo.h           |     9 +-
+ drivers/media/platform/vsp1/vsp1_hsit.c            |    26 +-
+ drivers/media/platform/vsp1/vsp1_hsit.h            |     6 +-
+ drivers/media/platform/vsp1/vsp1_lif.c             |    96 +-
+ drivers/media/platform/vsp1/vsp1_lif.h             |     6 +-
+ drivers/media/platform/vsp1/vsp1_lut.c             |   151 +-
+ drivers/media/platform/vsp1/vsp1_lut.h             |     7 +-
+ drivers/media/platform/vsp1/vsp1_pipe.c            |   127 +-
+ drivers/media/platform/vsp1/vsp1_pipe.h            |    24 +-
+ drivers/media/platform/vsp1/vsp1_regs.h            |    46 +-
+ drivers/media/platform/vsp1/vsp1_rpf.c             |   207 +-
+ drivers/media/platform/vsp1/vsp1_rwpf.c            |     6 +-
+ drivers/media/platform/vsp1/vsp1_rwpf.h            |    10 +-
+ drivers/media/platform/vsp1/vsp1_sru.c             |    30 +-
+ drivers/media/platform/vsp1/vsp1_sru.h             |     6 +-
+ drivers/media/platform/vsp1/vsp1_uds.c             |    79 +-
+ drivers/media/platform/vsp1/vsp1_uds.h             |     8 +-
+ drivers/media/platform/vsp1/vsp1_uif.c             |   264 +
+ drivers/media/platform/vsp1/vsp1_uif.h             |    32 +
+ drivers/media/platform/vsp1/vsp1_video.c           |   220 +-
+ drivers/media/platform/vsp1/vsp1_video.h           |     9 +-
+ drivers/media/platform/vsp1/vsp1_wpf.c             |   340 +-
+ drivers/media/platform/xilinx/xilinx-dma.c         |    10 +-
+ drivers/media/radio/Kconfig                        |    45 +-
+ drivers/media/radio/si470x/Kconfig                 |    16 +-
+ drivers/media/radio/si470x/Makefile                |     8 +-
+ drivers/media/radio/si470x/radio-si470x-common.c   |    70 +-
+ drivers/media/radio/si470x/radio-si470x-i2c.c      |    24 +-
+ drivers/media/radio/si470x/radio-si470x-usb.c      |    18 +-
+ drivers/media/radio/si470x/radio-si470x.h          |    15 +-
+ drivers/media/rc/Kconfig                           |    10 +-
+ drivers/media/rc/ir-imon-decoder.c                 |   136 +-
+ drivers/media/rc/ir-jvc-decoder.c                  |     1 +
+ drivers/media/rc/ir-mce_kbd-decoder.c              |    64 +-
+ drivers/media/rc/ir-nec-decoder.c                  |     1 +
+ drivers/media/rc/ir-rc5-decoder.c                  |     4 +-
+ drivers/media/rc/ir-rc6-decoder.c                  |    11 +-
+ drivers/media/rc/ir-sanyo-decoder.c                |     1 +
+ drivers/media/rc/ir-sharp-decoder.c                |     1 +
+ drivers/media/rc/ir-sony-decoder.c                 |     1 +
+ drivers/media/rc/ir-spi.c                          |     4 +-
+ drivers/media/rc/ir-xmp-decoder.c                  |     1 +
+ drivers/media/rc/ite-cir.c                         |     8 +-
+ drivers/media/rc/ite-cir.h                         |     7 -
+ drivers/media/rc/lirc_dev.c                        |    31 +-
+ drivers/media/rc/mceusb.c                          |    53 +-
+ drivers/media/rc/mtk-cir.c                         |     4 +-
+ drivers/media/rc/nuvoton-cir.c                     |    89 +-
+ drivers/media/rc/rc-core-priv.h                    |     6 +
+ drivers/media/rc/rc-ir-raw.c                       |    81 +-
+ drivers/media/rc/rc-main.c                         |    72 +-
+ drivers/media/rc/st_rc.c                           |    16 +-
+ drivers/media/rc/winbond-cir.c                     |     4 +-
+ drivers/media/spi/cxd2880-spi.c                    |    32 +-
+ drivers/media/tuners/Kconfig                       |     7 +
+ drivers/media/tuners/Makefile                      |     1 +
+ drivers/media/tuners/mxl301rf.c                    |    11 +-
+ drivers/media/tuners/mxl301rf.h                    |    11 +-
+ drivers/media/tuners/qm1d1b0004.c                  |   266 +
+ drivers/media/tuners/qm1d1b0004.h                  |    24 +
+ drivers/media/tuners/qm1d1c0042.c                  |    11 +-
+ drivers/media/tuners/qm1d1c0042.h                  |    11 +-
+ drivers/media/usb/cx231xx/Kconfig                  |     5 +-
+ drivers/media/usb/cx231xx/cx231xx-417.c            |     1 +
+ drivers/media/usb/cx231xx/cx231xx-cards.c          |     9 +-
+ drivers/media/usb/cx231xx/cx231xx-dvb.c            |   382 +-
+ drivers/media/usb/cx231xx/cx231xx-pcb-cfg.c        |     2 +-
+ drivers/media/usb/cx231xx/cx231xx.h                |     3 -
+ drivers/media/usb/dvb-usb-v2/dvbsky.c              |   425 +-
+ drivers/media/usb/dvb-usb-v2/gl861.c               |    22 +-
+ drivers/media/usb/dvb-usb-v2/usb_urb.c             |    17 +-
+ drivers/media/usb/dvb-usb/cxusb.c                  |     9 +-
+ drivers/media/usb/dvb-usb/dvb-usb-dvb.c            |     4 +
+ drivers/media/usb/dvb-usb/gp8psk.c                 |     2 +-
+ drivers/media/usb/dvb-usb/usb-urb.c                |     6 +-
+ drivers/media/usb/em28xx/em28xx-cards.c            |    50 +-
+ drivers/media/usb/em28xx/em28xx-core.c             |     5 +-
+ drivers/media/usb/em28xx/em28xx-dvb.c              |     8 +-
+ drivers/media/usb/em28xx/em28xx-v4l.h              |     2 +-
+ drivers/media/usb/em28xx/em28xx.h                  |     1 +
+ drivers/media/usb/go7007/go7007-fw.c               |     3 +
+ drivers/media/usb/go7007/go7007-v4l2.c             |     2 +-
+ drivers/media/usb/gspca/Kconfig                    |     1 +
+ drivers/media/usb/gspca/gspca.c                    |   946 +-
+ drivers/media/usb/gspca/gspca.h                    |    38 +-
+ drivers/media/usb/gspca/jl2005bcd.c                |     2 +-
+ drivers/media/usb/gspca/m5602/m5602_core.c         |     4 +-
+ drivers/media/usb/gspca/ov534.c                    |     1 -
+ drivers/media/usb/gspca/sq905.c                    |     2 +-
+ drivers/media/usb/gspca/sq905c.c                   |     2 +-
+ drivers/media/usb/gspca/topro.c                    |     1 -
+ drivers/media/usb/gspca/vc032x.c                   |     2 +-
+ drivers/media/usb/gspca/vicam.c                    |     2 +-
+ drivers/media/usb/gspca/zc3xx.c                    |    58 +-
+ drivers/media/usb/hackrf/hackrf.c                  |    11 +-
+ drivers/media/usb/hdpvr/hdpvr-i2c.c                |     2 +-
+ drivers/media/usb/hdpvr/hdpvr-video.c              |     2 +-
+ drivers/media/usb/pvrusb2/pvrusb2-cx2584x-v4l.c    |     1 -
+ drivers/media/usb/pvrusb2/pvrusb2-devattr.c        |     4 +-
+ drivers/media/usb/pvrusb2/pvrusb2-v4l2.c           |    83 +-
+ drivers/media/usb/siano/smsusb.c                   |     2 +-
+ drivers/media/usb/stk1160/stk1160-core.c           |     4 +-
+ drivers/media/usb/usbtv/usbtv-video.c              |   117 +-
+ drivers/media/usb/usbtv/usbtv.h                    |     2 +-
+ drivers/media/usb/usbvision/usbvision-core.c       |     2 +-
+ drivers/media/usb/uvc/uvc_driver.c                 |    11 +-
+ drivers/media/usb/uvc/uvc_video.c                  |    24 +-
+ drivers/media/v4l2-core/Kconfig                    |     6 -
+ drivers/media/v4l2-core/Makefile                   |     1 -
+ drivers/media/v4l2-core/v4l2-compat-ioctl32.c      |   807 +-
+ drivers/media/v4l2-core/v4l2-dev.c                 |    73 +-
+ drivers/media/v4l2-core/v4l2-fwnode.c              |    28 +-
+ drivers/media/v4l2-core/v4l2-ioctl.c               |   284 +-
+ drivers/media/v4l2-core/v4l2-subdev.c              |    17 +-
+ drivers/media/v4l2-core/videobuf-dma-sg.c          |     6 +-
+ drivers/media/v4l2-core/videobuf-dvb.c             |   398 -
+ drivers/pnp/isapnp/Kconfig                         |     2 +-
+ drivers/staging/media/Kconfig                      |     4 +-
+ drivers/staging/media/Makefile                     |     2 +-
+ drivers/staging/media/atomisp/Kconfig              |    12 -
+ drivers/staging/media/atomisp/Makefile             |     6 -
+ drivers/staging/media/atomisp/TODO                 |    74 -
+ drivers/staging/media/atomisp/i2c/Kconfig          |    86 -
+ drivers/staging/media/atomisp/i2c/Makefile         |    25 -
+ drivers/staging/media/atomisp/i2c/atomisp-gc0310.c |  1392 ---
+ drivers/staging/media/atomisp/i2c/atomisp-gc2235.c |  1122 --
+ .../media/atomisp/i2c/atomisp-libmsrlisthelper.c   |   205 -
+ drivers/staging/media/atomisp/i2c/atomisp-lm3554.c |   968 --
+ .../staging/media/atomisp/i2c/atomisp-mt9m114.c    |  1917 ----
+ drivers/staging/media/atomisp/i2c/atomisp-ov2680.c |  1470 ---
+ drivers/staging/media/atomisp/i2c/atomisp-ov2722.c |  1271 ---
+ drivers/staging/media/atomisp/i2c/gc0310.h         |   404 -
+ drivers/staging/media/atomisp/i2c/gc2235.h         |   670 --
+ drivers/staging/media/atomisp/i2c/mt9m114.h        |  1777 ---
+ drivers/staging/media/atomisp/i2c/ov2680.h         |   856 --
+ drivers/staging/media/atomisp/i2c/ov2722.h         |  1262 ---
+ drivers/staging/media/atomisp/i2c/ov5693/Kconfig   |    11 -
+ drivers/staging/media/atomisp/i2c/ov5693/Makefile  |     9 -
+ drivers/staging/media/atomisp/i2c/ov5693/ad5823.h  |    63 -
+ .../media/atomisp/i2c/ov5693/atomisp-ov5693.c      |  1993 ----
+ drivers/staging/media/atomisp/i2c/ov5693/ov5693.h  |  1376 ---
+ .../staging/media/atomisp/include/linux/atomisp.h  |  1359 ---
+ .../atomisp/include/linux/atomisp_gmin_platform.h  |    36 -
+ .../media/atomisp/include/linux/atomisp_platform.h |   249 -
+ .../media/atomisp/include/linux/libmsrlisthelper.h |    28 -
+ .../staging/media/atomisp/include/media/lm3554.h   |   131 -
+ drivers/staging/media/atomisp/pci/Kconfig          |    14 -
+ drivers/staging/media/atomisp/pci/Makefile         |     5 -
+ .../staging/media/atomisp/pci/atomisp2/Makefile    |   349 -
+ .../media/atomisp/pci/atomisp2/atomisp-regs.h      |   205 -
+ .../media/atomisp/pci/atomisp2/atomisp_acc.c       |   604 -
+ .../media/atomisp/pci/atomisp2/atomisp_acc.h       |   120 -
+ .../media/atomisp/pci/atomisp2/atomisp_cmd.c       |  6696 -----------
+ .../media/atomisp/pci/atomisp2/atomisp_cmd.h       |   446 -
+ .../media/atomisp/pci/atomisp2/atomisp_common.h    |    75 -
+ .../media/atomisp/pci/atomisp2/atomisp_compat.h    |   662 --
+ .../atomisp/pci/atomisp2/atomisp_compat_css20.c    |  4704 --------
+ .../atomisp/pci/atomisp2/atomisp_compat_css20.h    |   277 -
+ .../atomisp/pci/atomisp2/atomisp_compat_ioctl32.c  |  1259 ---
+ .../atomisp/pci/atomisp2/atomisp_compat_ioctl32.h  |   365 -
+ .../media/atomisp/pci/atomisp2/atomisp_csi2.c      |   442 -
+ .../media/atomisp/pci/atomisp2/atomisp_csi2.h      |    57 -
+ .../atomisp/pci/atomisp2/atomisp_dfs_tables.h      |   408 -
+ .../media/atomisp/pci/atomisp2/atomisp_drvfs.c     |   205 -
+ .../media/atomisp/pci/atomisp2/atomisp_drvfs.h     |    24 -
+ .../media/atomisp/pci/atomisp2/atomisp_file.c      |   225 -
+ .../media/atomisp/pci/atomisp2/atomisp_file.h      |    43 -
+ .../media/atomisp/pci/atomisp2/atomisp_fops.c      |  1304 ---
+ .../media/atomisp/pci/atomisp2/atomisp_fops.h      |    50 -
+ .../media/atomisp/pci/atomisp2/atomisp_helper.h    |    29 -
+ .../media/atomisp/pci/atomisp2/atomisp_internal.h  |   310 -
+ .../media/atomisp/pci/atomisp2/atomisp_ioctl.c     |  3124 ------
+ .../media/atomisp/pci/atomisp2/atomisp_ioctl.h     |    69 -
+ .../media/atomisp/pci/atomisp2/atomisp_subdev.c    |  1422 ---
+ .../media/atomisp/pci/atomisp2/atomisp_subdev.h    |   467 -
+ .../media/atomisp/pci/atomisp2/atomisp_tables.h    |   187 -
+ .../media/atomisp/pci/atomisp2/atomisp_tpg.c       |   164 -
+ .../media/atomisp/pci/atomisp2/atomisp_tpg.h       |    38 -
+ .../atomisp/pci/atomisp2/atomisp_trace_event.h     |   129 -
+ .../media/atomisp/pci/atomisp2/atomisp_v4l2.c      |  1573 ---
+ .../media/atomisp/pci/atomisp2/atomisp_v4l2.h      |    40 -
+ .../media/atomisp/pci/atomisp2/css2400/Makefile    |     2 -
+ .../base/circbuf/interface/ia_css_circbuf.h        |   376 -
+ .../base/circbuf/interface/ia_css_circbuf_comm.h   |    56 -
+ .../base/circbuf/interface/ia_css_circbuf_desc.h   |   169 -
+ .../atomisp2/css2400/base/circbuf/src/circbuf.c    |   321 -
+ .../base/refcount/interface/ia_css_refcount.h      |    83 -
+ .../atomisp2/css2400/base/refcount/src/refcount.c  |   281 -
+ .../camera/pipe/interface/ia_css_pipe_binarydesc.h |   297 -
+ .../camera/pipe/interface/ia_css_pipe_stagedesc.h  |    52 -
+ .../camera/pipe/interface/ia_css_pipe_util.h       |    39 -
+ .../css2400/camera/pipe/src/pipe_binarydesc.c      |   880 --
+ .../css2400/camera/pipe/src/pipe_stagedesc.c       |   115 -
+ .../atomisp2/css2400/camera/pipe/src/pipe_util.c   |    51 -
+ .../css2400/camera/util/interface/ia_css_util.h    |   141 -
+ .../pci/atomisp2/css2400/camera/util/src/util.c    |   227 -
+ .../ia_css_isp_configs.c                           |   360 -
+ .../ia_css_isp_configs.h                           |   189 -
+ .../ia_css_isp_params.c                            |  3221 ------
+ .../ia_css_isp_params.h                            |   399 -
+ .../ia_css_isp_states.c                            |   214 -
+ .../ia_css_isp_states.h                            |    72 -
+ .../atomisp2/css2400/css_2400_system/hrt/bits.h    |   104 -
+ .../css2400/css_2400_system/hrt/cell_params.h      |    42 -
+ .../hrt/css_receiver_2400_common_defs.h            |   200 -
+ .../css_2400_system/hrt/css_receiver_2400_defs.h   |   258 -
+ .../atomisp2/css2400/css_2400_system/hrt/defs.h    |    36 -
+ .../css2400/css_2400_system/hrt/dma_v2_defs.h      |   199 -
+ .../css2400/css_2400_system/hrt/gdc_v2_defs.h      |   170 -
+ .../css2400/css_2400_system/hrt/gp_timer_defs.h    |    36 -
+ .../css2400/css_2400_system/hrt/gpio_block_defs.h  |    42 -
+ .../css_2400_system/hrt/hive_isp_css_defs.h        |   416 -
+ .../hrt/hive_isp_css_host_ids_hrt.h                |    84 -
+ .../hrt/hive_isp_css_irq_types_hrt.h               |    72 -
+ .../hrt/hive_isp_css_streaming_to_mipi_types_hrt.h |    26 -
+ .../css2400/css_2400_system/hrt/hive_types.h       |   128 -
+ .../atomisp2/css2400/css_2400_system/hrt/if_defs.h |    22 -
+ .../hrt/input_formatter_subsystem_defs.h           |    53 -
+ .../css_2400_system/hrt/input_selector_defs.h      |    89 -
+ .../css_2400_system/hrt/input_switch_2400_defs.h   |    30 -
+ .../css_2400_system/hrt/input_system_ctrl_defs.h   |   254 -
+ .../css_2400_system/hrt/input_system_defs.h        |   126 -
+ .../css_2400_system/hrt/irq_controller_defs.h      |    28 -
+ .../css_2400_system/hrt/isp2400_mamoiada_params.h  |   254 -
+ .../css2400/css_2400_system/hrt/isp2400_support.h  |    38 -
+ .../css_2400_system/hrt/isp_acquisition_defs.h     |   234 -
+ .../css2400/css_2400_system/hrt/isp_capture_defs.h |   310 -
+ .../css2400/css_2400_system/hrt/mmu_defs.h         |    23 -
+ .../hrt/scalar_processor_2400_params.h             |    20 -
+ .../css2400/css_2400_system/hrt/str2mem_defs.h     |    39 -
+ .../css_2400_system/hrt/streaming_to_mipi_defs.h   |    28 -
+ .../css_2400_system/hrt/timed_controller_defs.h    |    22 -
+ .../pci/atomisp2/css2400/css_2400_system/hrt/var.h |    74 -
+ .../atomisp2/css2400/css_2400_system/hrt/version.h |    20 -
+ .../atomisp2/css2400/css_2400_system/spmem_dump.c  |  3634 ------
+ .../css2400/css_2401_csi2p_system/csi_rx_global.h  |    63 -
+ .../ia_css_isp_configs.c                           |   360 -
+ .../ia_css_isp_configs.h                           |   189 -
+ .../ia_css_isp_params.c                            |  3220 ------
+ .../ia_css_isp_params.h                            |   399 -
+ .../ia_css_isp_states.c                            |   214 -
+ .../ia_css_isp_states.h                            |    72 -
+ .../css2400/css_2401_csi2p_system/host/csi_rx.c    |    41 -
+ .../css_2401_csi2p_system/host/csi_rx_local.h      |    61 -
+ .../css_2401_csi2p_system/host/csi_rx_private.h    |   282 -
+ .../css2400/css_2401_csi2p_system/host/ibuf_ctrl.c |    22 -
+ .../css_2401_csi2p_system/host/ibuf_ctrl_local.h   |    58 -
+ .../css_2401_csi2p_system/host/ibuf_ctrl_private.h |   233 -
+ .../host/input_system_local.h                      |   106 -
+ .../host/input_system_private.h                    |   128 -
+ .../css2400/css_2401_csi2p_system/host/isys_dma.c  |    40 -
+ .../css_2401_csi2p_system/host/isys_dma_local.h    |    20 -
+ .../css_2401_csi2p_system/host/isys_dma_private.h  |    60 -
+ .../css2400/css_2401_csi2p_system/host/isys_irq.c  |    39 -
+ .../css_2401_csi2p_system/host/isys_irq_local.h    |    35 -
+ .../css_2401_csi2p_system/host/isys_irq_private.h  |   108 -
+ .../css_2401_csi2p_system/host/isys_stream2mmio.c  |    21 -
+ .../host/isys_stream2mmio_local.h                  |    36 -
+ .../host/isys_stream2mmio_private.h                |   168 -
+ .../css_2401_csi2p_system/host/pixelgen_local.h    |    50 -
+ .../css_2401_csi2p_system/host/pixelgen_private.h  |   164 -
+ .../css_2401_csi2p_system/host/system_local.h      |   381 -
+ .../hrt/PixelGen_SysBlock_defs.h                   |   126 -
+ .../css2400/css_2401_csi2p_system/hrt/bits.h       |   104 -
+ .../css_2401_csi2p_system/hrt/cell_params.h        |    42 -
+ .../hrt/css_receiver_2400_common_defs.h            |   200 -
+ .../hrt/css_receiver_2400_defs.h                   |   258 -
+ .../css2400/css_2401_csi2p_system/hrt/defs.h       |    36 -
+ .../css_2401_csi2p_system/hrt/dma_v2_defs.h        |   199 -
+ .../css_2401_csi2p_system/hrt/gdc_v2_defs.h        |   170 -
+ .../css_2401_csi2p_system/hrt/gp_timer_defs.h      |    36 -
+ .../css_2401_csi2p_system/hrt/gpio_block_defs.h    |    42 -
+ .../hrt/hive_isp_css_2401_irq_types_hrt.h          |    68 -
+ .../css_2401_csi2p_system/hrt/hive_isp_css_defs.h  |   435 -
+ .../hrt/hive_isp_css_host_ids_hrt.h                |   119 -
+ .../hrt/hive_isp_css_streaming_to_mipi_types_hrt.h |    26 -
+ .../css2400/css_2401_csi2p_system/hrt/hive_types.h |   128 -
+ .../css_2401_csi2p_system/hrt/ibuf_cntrl_defs.h    |   138 -
+ .../css2400/css_2401_csi2p_system/hrt/if_defs.h    |    22 -
+ .../hrt/input_formatter_subsystem_defs.h           |    53 -
+ .../hrt/input_selector_defs.h                      |    89 -
+ .../hrt/input_switch_2400_defs.h                   |    30 -
+ .../hrt/input_system_ctrl_defs.h                   |   254 -
+ .../css_2401_csi2p_system/hrt/input_system_defs.h  |   126 -
+ .../hrt/irq_controller_defs.h                      |    28 -
+ .../css_2401_csi2p_system/hrt/isp2400_support.h    |    38 -
+ .../hrt/isp2401_mamoiada_params.h                  |   258 -
+ .../hrt/isp_acquisition_defs.h                     |   234 -
+ .../css_2401_csi2p_system/hrt/isp_capture_defs.h   |   310 -
+ .../hrt/mipi_backend_common_defs.h                 |   210 -
+ .../css_2401_csi2p_system/hrt/mipi_backend_defs.h  |   215 -
+ .../css2400/css_2401_csi2p_system/hrt/mmu_defs.h   |    23 -
+ .../css_2401_csi2p_system/hrt/rx_csi_defs.h        |   175 -
+ .../hrt/scalar_processor_2400_params.h             |    20 -
+ .../css_2401_csi2p_system/hrt/str2mem_defs.h       |    39 -
+ .../css_2401_csi2p_system/hrt/stream2mmio_defs.h   |    71 -
+ .../hrt/streaming_to_mipi_defs.h                   |    28 -
+ .../hrt/timed_controller_defs.h                    |    22 -
+ .../css2400/css_2401_csi2p_system/hrt/var.h        |    99 -
+ .../css2400/css_2401_csi2p_system/hrt/version.h    |    20 -
+ .../css_2401_csi2p_system/ibuf_ctrl_global.h       |    80 -
+ .../css_2401_csi2p_system/input_system_global.h    |   206 -
+ .../css_2401_csi2p_system/isys_dma_global.h        |    87 -
+ .../css_2401_csi2p_system/isys_irq_global.h        |    35 -
+ .../isys_stream2mmio_global.h                      |    39 -
+ .../css_2401_csi2p_system/pixelgen_global.h        |    91 -
+ .../css2400/css_2401_csi2p_system/spmem_dump.c     |  3686 ------
+ .../css2400/css_2401_csi2p_system/system_global.h  |   458 -
+ .../ia_css_isp_configs.c                           |   360 -
+ .../ia_css_isp_configs.h                           |   189 -
+ .../ia_css_isp_params.c                            |  3220 ------
+ .../ia_css_isp_params.h                            |   399 -
+ .../ia_css_isp_states.c                            |   214 -
+ .../ia_css_isp_states.h                            |    72 -
+ .../atomisp2/css2400/css_2401_system/hrt/bits.h    |   104 -
+ .../css2400/css_2401_system/hrt/cell_params.h      |    42 -
+ .../hrt/css_receiver_2400_common_defs.h            |   200 -
+ .../css_2401_system/hrt/css_receiver_2400_defs.h   |   258 -
+ .../atomisp2/css2400/css_2401_system/hrt/defs.h    |    36 -
+ .../css2400/css_2401_system/hrt/dma_v2_defs.h      |   199 -
+ .../css2400/css_2401_system/hrt/gdc_v2_defs.h      |   170 -
+ .../css2400/css_2401_system/hrt/gp_timer_defs.h    |    36 -
+ .../css2400/css_2401_system/hrt/gpio_block_defs.h  |    42 -
+ .../hrt/hive_isp_css_2401_irq_types_hrt.h          |    69 -
+ .../css_2401_system/hrt/hive_isp_css_defs.h        |   435 -
+ .../hrt/hive_isp_css_host_ids_hrt.h                |   119 -
+ .../hrt/hive_isp_css_streaming_to_mipi_types_hrt.h |    26 -
+ .../css2400/css_2401_system/hrt/hive_types.h       |   128 -
+ .../atomisp2/css2400/css_2401_system/hrt/if_defs.h |    22 -
+ .../hrt/input_formatter_subsystem_defs.h           |    53 -
+ .../css_2401_system/hrt/input_selector_defs.h      |    89 -
+ .../css_2401_system/hrt/input_switch_2400_defs.h   |    30 -
+ .../css_2401_system/hrt/input_system_ctrl_defs.h   |   254 -
+ .../css_2401_system/hrt/input_system_defs.h        |   126 -
+ .../css_2401_system/hrt/irq_controller_defs.h      |    28 -
+ .../css2400/css_2401_system/hrt/isp2400_support.h  |    38 -
+ .../css_2401_system/hrt/isp2401_mamoiada_params.h  |   258 -
+ .../css_2401_system/hrt/isp_acquisition_defs.h     |   234 -
+ .../css2400/css_2401_system/hrt/isp_capture_defs.h |   310 -
+ .../css2400/css_2401_system/hrt/mmu_defs.h         |    23 -
+ .../hrt/scalar_processor_2400_params.h             |    20 -
+ .../css2400/css_2401_system/hrt/str2mem_defs.h     |    39 -
+ .../css_2401_system/hrt/streaming_to_mipi_defs.h   |    28 -
+ .../css_2401_system/hrt/timed_controller_defs.h    |    22 -
+ .../pci/atomisp2/css2400/css_2401_system/hrt/var.h |    99 -
+ .../atomisp2/css2400/css_2401_system/hrt/version.h |    20 -
+ .../atomisp2/css2400/css_2401_system/spmem_dump.c  |  3634 ------
+ .../media/atomisp/pci/atomisp2/css2400/css_trace.h |   388 -
+ .../css2400/hive_isp_css_common/debug_global.h     |    83 -
+ .../css2400/hive_isp_css_common/dma_global.h       |   255 -
+ .../hive_isp_css_common/event_fifo_global.h        |    20 -
+ .../hive_isp_css_common/fifo_monitor_global.h      |    32 -
+ .../css2400/hive_isp_css_common/gdc_global.h       |    90 -
+ .../css2400/hive_isp_css_common/gp_device_global.h |    85 -
+ .../css2400/hive_isp_css_common/gp_timer_global.h  |    33 -
+ .../css2400/hive_isp_css_common/gpio_global.h      |    45 -
+ .../css2400/hive_isp_css_common/hmem_global.h      |    45 -
+ .../css2400/hive_isp_css_common/host/debug.c       |    72 -
+ .../css2400/hive_isp_css_common/host/debug_local.h |    21 -
+ .../hive_isp_css_common/host/debug_private.h       |    99 -
+ .../css2400/hive_isp_css_common/host/dma.c         |   299 -
+ .../css2400/hive_isp_css_common/host/dma_local.h   |   207 -
+ .../css2400/hive_isp_css_common/host/dma_private.h |    41 -
+ .../css2400/hive_isp_css_common/host/event_fifo.c  |    19 -
+ .../hive_isp_css_common/host/event_fifo_local.h    |    57 -
+ .../hive_isp_css_common/host/event_fifo_private.h  |    75 -
+ .../hive_isp_css_common/host/fifo_monitor.c        |   567 -
+ .../hive_isp_css_common/host/fifo_monitor_local.h  |    99 -
+ .../host/fifo_monitor_private.h                    |    79 -
+ .../css2400/hive_isp_css_common/host/gdc.c         |   127 -
+ .../css2400/hive_isp_css_common/host/gdc_local.h   |    20 -
+ .../css2400/hive_isp_css_common/host/gdc_private.h |    20 -
+ .../css2400/hive_isp_css_common/host/gp_device.c   |   108 -
+ .../hive_isp_css_common/host/gp_device_local.h     |   143 -
+ .../hive_isp_css_common/host/gp_device_private.h   |    46 -
+ .../css2400/hive_isp_css_common/host/gp_timer.c    |    70 -
+ .../hive_isp_css_common/host/gp_timer_local.h      |    45 -
+ .../hive_isp_css_common/host/gp_timer_private.h    |    22 -
+ .../css2400/hive_isp_css_common/host/gpio_local.h  |    20 -
+ .../hive_isp_css_common/host/gpio_private.h        |    44 -
+ .../css2400/hive_isp_css_common/host/hmem.c        |    19 -
+ .../css2400/hive_isp_css_common/host/hmem_local.h  |    20 -
+ .../hive_isp_css_common/host/hmem_private.h        |    30 -
+ .../hive_isp_css_common/host/input_formatter.c     |   228 -
+ .../host/input_formatter_local.h                   |   120 -
+ .../host/input_formatter_private.h                 |    46 -
+ .../hive_isp_css_common/host/input_system.c        |  1823 ---
+ .../hive_isp_css_common/host/input_system_local.h  |   533 -
+ .../host/input_system_private.h                    |   116 -
+ .../css2400/hive_isp_css_common/host/irq.c         |   448 -
+ .../css2400/hive_isp_css_common/host/irq_local.h   |   136 -
+ .../css2400/hive_isp_css_common/host/irq_private.h |    44 -
+ .../css2400/hive_isp_css_common/host/isp.c         |   129 -
+ .../css2400/hive_isp_css_common/host/isp_local.h   |    57 -
+ .../css2400/hive_isp_css_common/host/isp_private.h |   157 -
+ .../css2400/hive_isp_css_common/host/mmu.c         |    50 -
+ .../css2400/hive_isp_css_common/host/mmu_local.h   |    20 -
+ .../css2400/hive_isp_css_common/host/mmu_private.h |    44 -
+ .../atomisp2/css2400/hive_isp_css_common/host/sp.c |    81 -
+ .../css2400/hive_isp_css_common/host/sp_local.h    |   101 -
+ .../css2400/hive_isp_css_common/host/sp_private.h  |   163 -
+ .../hive_isp_css_common/host/system_local.h        |   306 -
+ .../css2400/hive_isp_css_common/host/timed_ctrl.c  |    74 -
+ .../hive_isp_css_common/host/timed_ctrl_local.h    |    20 -
+ .../hive_isp_css_common/host/timed_ctrl_private.h  |    34 -
+ .../css2400/hive_isp_css_common/host/vamem_local.h |    20 -
+ .../hive_isp_css_common/host/vamem_private.h       |    37 -
+ .../css2400/hive_isp_css_common/host/vmem.c        |   258 -
+ .../css2400/hive_isp_css_common/host/vmem_local.h  |    55 -
+ .../hive_isp_css_common/host/vmem_private.h        |    20 -
+ .../hive_isp_css_common/input_formatter_global.h   |   114 -
+ .../hive_isp_css_common/input_system_global.h      |   155 -
+ .../css2400/hive_isp_css_common/irq_global.h       |    45 -
+ .../css2400/hive_isp_css_common/isp_global.h       |   115 -
+ .../css2400/hive_isp_css_common/mmu_global.h       |    22 -
+ .../css2400/hive_isp_css_common/sp_global.h        |    93 -
+ .../css2400/hive_isp_css_common/system_global.h    |   348 -
+ .../hive_isp_css_common/timed_ctrl_global.h        |    56 -
+ .../css2400/hive_isp_css_common/vamem_global.h     |    34 -
+ .../css2400/hive_isp_css_common/vmem_global.h      |    28 -
+ .../css2400/hive_isp_css_include/assert_support.h  |   102 -
+ .../css2400/hive_isp_css_include/bitop_support.h   |    25 -
+ .../atomisp2/css2400/hive_isp_css_include/csi_rx.h |    43 -
+ .../atomisp2/css2400/hive_isp_css_include/debug.h  |    47 -
+ .../device_access/device_access.h                  |   194 -
+ .../atomisp2/css2400/hive_isp_css_include/dma.h    |    47 -
+ .../css2400/hive_isp_css_include/error_support.h   |    70 -
+ .../css2400/hive_isp_css_include/event_fifo.h      |    46 -
+ .../css2400/hive_isp_css_include/fifo_monitor.h    |    46 -
+ .../css2400/hive_isp_css_include/gdc_device.h      |    48 -
+ .../css2400/hive_isp_css_include/gp_device.h       |    46 -
+ .../css2400/hive_isp_css_include/gp_timer.h        |    46 -
+ .../atomisp2/css2400/hive_isp_css_include/gpio.h   |    46 -
+ .../atomisp2/css2400/hive_isp_css_include/hmem.h   |    46 -
+ .../hive_isp_css_include/host/csi_rx_public.h      |   135 -
+ .../hive_isp_css_include/host/debug_public.h       |    99 -
+ .../css2400/hive_isp_css_include/host/dma_public.h |    73 -
+ .../hive_isp_css_include/host/event_fifo_public.h  |    79 -
+ .../host/fifo_monitor_public.h                     |   110 -
+ .../css2400/hive_isp_css_include/host/gdc_public.h |    59 -
+ .../hive_isp_css_include/host/gp_device_public.h   |    58 -
+ .../hive_isp_css_include/host/gp_timer_public.h    |    34 -
+ .../hive_isp_css_include/host/gpio_public.h        |    45 -
+ .../hive_isp_css_include/host/hmem_public.h        |    32 -
+ .../hive_isp_css_include/host/ibuf_ctrl_public.h   |    93 -
+ .../host/input_formatter_public.h                  |   115 -
+ .../host/input_system_public.h                     |   376 -
+ .../css2400/hive_isp_css_include/host/irq_public.h |   184 -
+ .../css2400/hive_isp_css_include/host/isp_public.h |   186 -
+ .../hive_isp_css_include/host/isys_dma_public.h    |    38 -
+ .../hive_isp_css_include/host/isys_irq_public.h    |    45 -
+ .../hive_isp_css_include/host/isys_public.h        |    37 -
+ .../host/isys_stream2mmio_public.h                 |   101 -
+ .../css2400/hive_isp_css_include/host/mmu_public.h |    82 -
+ .../hive_isp_css_include/host/pixelgen_public.h    |    79 -
+ .../css2400/hive_isp_css_include/host/sp_public.h  |   223 -
+ .../css2400/hive_isp_css_include/host/tag_public.h |    41 -
+ .../hive_isp_css_include/host/timed_ctrl_public.h  |    59 -
+ .../hive_isp_css_include/host/vamem_public.h       |    20 -
+ .../hive_isp_css_include/host/vmem_public.h        |    20 -
+ .../css2400/hive_isp_css_include/ibuf_ctrl.h       |    48 -
+ .../css2400/hive_isp_css_include/input_formatter.h |    46 -
+ .../css2400/hive_isp_css_include/input_system.h    |    46 -
+ .../atomisp2/css2400/hive_isp_css_include/irq.h    |    46 -
+ .../atomisp2/css2400/hive_isp_css_include/isp.h    |    46 -
+ .../css2400/hive_isp_css_include/isys_dma.h        |    48 -
+ .../css2400/hive_isp_css_include/isys_irq.h        |    39 -
+ .../hive_isp_css_include/isys_stream2mmio.h        |    48 -
+ .../css2400/hive_isp_css_include/math_support.h    |   223 -
+ .../memory_access/memory_access.h                  |   174 -
+ .../css2400/hive_isp_css_include/memory_realloc.h  |    38 -
+ .../css2400/hive_isp_css_include/misc_support.h    |    26 -
+ .../css2400/hive_isp_css_include/mmu_device.h      |    48 -
+ .../css2400/hive_isp_css_include/pixelgen.h        |    48 -
+ .../hive_isp_css_include/platform_support.h        |    41 -
+ .../css2400/hive_isp_css_include/print_support.h   |    44 -
+ .../atomisp2/css2400/hive_isp_css_include/queue.h  |    46 -
+ .../css2400/hive_isp_css_include/resource.h        |    47 -
+ .../atomisp2/css2400/hive_isp_css_include/socket.h |    47 -
+ .../pci/atomisp2/css2400/hive_isp_css_include/sp.h |    46 -
+ .../css2400/hive_isp_css_include/string_support.h  |   165 -
+ .../css2400/hive_isp_css_include/system_types.h    |    25 -
+ .../atomisp2/css2400/hive_isp_css_include/tag.h    |    45 -
+ .../css2400/hive_isp_css_include/timed_ctrl.h      |    46 -
+ .../css2400/hive_isp_css_include/type_support.h    |    40 -
+ .../atomisp2/css2400/hive_isp_css_include/vamem.h  |    46 -
+ .../atomisp2/css2400/hive_isp_css_include/vmem.h   |    46 -
+ .../css2400/hive_isp_css_shared/host/queue_local.h |    20 -
+ .../hive_isp_css_shared/host/queue_private.h       |    18 -
+ .../css2400/hive_isp_css_shared/host/tag.c         |    95 -
+ .../css2400/hive_isp_css_shared/host/tag_local.h   |    22 -
+ .../css2400/hive_isp_css_shared/host/tag_private.h |    18 -
+ .../css2400/hive_isp_css_shared/queue_global.h     |    19 -
+ .../css2400/hive_isp_css_shared/sw_event_global.h  |    36 -
+ .../css2400/hive_isp_css_shared/tag_global.h       |    56 -
+ .../media/atomisp/pci/atomisp2/css2400/ia_css.h    |    57 -
+ .../media/atomisp/pci/atomisp2/css2400/ia_css_3a.h |   188 -
+ .../pci/atomisp2/css2400/ia_css_acc_types.h        |   468 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_buffer.h   |    84 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_control.h  |   157 -
+ .../pci/atomisp2/css2400/ia_css_device_access.c    |    95 -
+ .../pci/atomisp2/css2400/ia_css_device_access.h    |    59 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_dvs.h      |   299 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_env.h      |    94 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_err.h      |    63 -
+ .../pci/atomisp2/css2400/ia_css_event_public.h     |   196 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_firmware.h |    74 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_frac.h     |    37 -
+ .../pci/atomisp2/css2400/ia_css_frame_format.h     |   101 -
+ .../pci/atomisp2/css2400/ia_css_frame_public.h     |   352 -
+ .../pci/atomisp2/css2400/ia_css_host_data.h        |    46 -
+ .../pci/atomisp2/css2400/ia_css_input_port.h       |    60 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_irq.h      |   235 -
+ .../pci/atomisp2/css2400/ia_css_memory_access.c    |    83 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_metadata.h |    71 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_mipi.h     |    82 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_mmu.h      |    32 -
+ .../pci/atomisp2/css2400/ia_css_mmu_private.h      |    29 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_morph.h    |    39 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_pipe.h     |   195 -
+ .../pci/atomisp2/css2400/ia_css_pipe_public.h      |   579 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_prbs.h     |    53 -
+ .../pci/atomisp2/css2400/ia_css_properties.h       |    41 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_shading.h  |    40 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_stream.h   |   110 -
+ .../pci/atomisp2/css2400/ia_css_stream_format.h    |    29 -
+ .../pci/atomisp2/css2400/ia_css_stream_public.h    |   582 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_timer.h    |    84 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_tpg.h      |    78 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_types.h    |   616 -
+ .../atomisp/pci/atomisp2/css2400/ia_css_version.h  |    40 -
+ .../pci/atomisp2/css2400/ia_css_version_data.h     |    33 -
+ .../css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.c  |    32 -
+ .../css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.h  |    27 -
+ .../css2400/isp/kernels/aa/aa_2/ia_css_aa2_param.h |    24 -
+ .../css2400/isp/kernels/aa/aa_2/ia_css_aa2_types.h |    48 -
+ .../isp/kernels/anr/anr_1.0/ia_css_anr.host.c      |    60 -
+ .../isp/kernels/anr/anr_1.0/ia_css_anr.host.h      |    39 -
+ .../isp/kernels/anr/anr_1.0/ia_css_anr_param.h     |    25 -
+ .../isp/kernels/anr/anr_1.0/ia_css_anr_types.h     |    36 -
+ .../isp/kernels/anr/anr_2/ia_css_anr2.host.c       |    46 -
+ .../isp/kernels/anr/anr_2/ia_css_anr2.host.h       |    35 -
+ .../isp/kernels/anr/anr_2/ia_css_anr2_table.host.c |    52 -
+ .../isp/kernels/anr/anr_2/ia_css_anr2_table.host.h |    22 -
+ .../isp/kernels/anr/anr_2/ia_css_anr2_types.h      |    32 -
+ .../isp/kernels/anr/anr_2/ia_css_anr_param.h       |    27 -
+ .../css2400/isp/kernels/bh/bh_2/ia_css_bh.host.c   |    66 -
+ .../css2400/isp/kernels/bh/bh_2/ia_css_bh.host.h   |    32 -
+ .../css2400/isp/kernels/bh/bh_2/ia_css_bh_param.h  |    40 -
+ .../css2400/isp/kernels/bh/bh_2/ia_css_bh_types.h  |    37 -
+ .../css2400/isp/kernels/bnlm/ia_css_bnlm.host.c    |   183 -
+ .../css2400/isp/kernels/bnlm/ia_css_bnlm.host.h    |    40 -
+ .../css2400/isp/kernels/bnlm/ia_css_bnlm_param.h   |    63 -
+ .../css2400/isp/kernels/bnlm/ia_css_bnlm_types.h   |   106 -
+ .../isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c    |   122 -
+ .../isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h    |    35 -
+ .../isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h   |    47 -
+ .../isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h   |    71 -
+ .../isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c      |    64 -
+ .../isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h      |    34 -
+ .../isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h     |    30 -
+ .../isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c      |    28 -
+ .../isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h      |    25 -
+ .../isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h     |    24 -
+ .../isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c       |    76 -
+ .../isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h       |    43 -
+ .../isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h      |    32 -
+ .../isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h      |    55 -
+ .../isp/kernels/cnr/cnr_2/ia_css_cnr_param.h       |    20 -
+ .../conversion_1.0/ia_css_conversion.host.c        |    36 -
+ .../conversion_1.0/ia_css_conversion.host.h        |    33 -
+ .../conversion_1.0/ia_css_conversion_param.h       |    28 -
+ .../conversion_1.0/ia_css_conversion_types.h       |    32 -
+ .../copy_output_1.0/ia_css_copy_output.host.c      |    47 -
+ .../copy_output_1.0/ia_css_copy_output.host.h      |    34 -
+ .../copy_output_1.0/ia_css_copy_output_param.h     |    26 -
+ .../isp/kernels/crop/crop_1.0/ia_css_crop.host.c   |    64 -
+ .../isp/kernels/crop/crop_1.0/ia_css_crop.host.h   |    41 -
+ .../isp/kernels/crop/crop_1.0/ia_css_crop_param.h  |    32 -
+ .../isp/kernels/crop/crop_1.0/ia_css_crop_types.h  |    35 -
+ .../isp/kernels/csc/csc_1.0/ia_css_csc.host.c      |   132 -
+ .../isp/kernels/csc/csc_1.0/ia_css_csc.host.h      |    54 -
+ .../isp/kernels/csc/csc_1.0/ia_css_csc_param.h     |    34 -
+ .../isp/kernels/csc/csc_1.0/ia_css_csc_types.h     |    78 -
+ .../isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c    |   120 -
+ .../isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h    |    33 -
+ .../isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h   |    46 -
+ .../isp/kernels/ctc/ctc1_5/ia_css_ctc_param.h      |    20 -
+ .../isp/kernels/ctc/ctc2/ia_css_ctc2.host.c        |   156 -
+ .../isp/kernels/ctc/ctc2/ia_css_ctc2.host.h        |    33 -
+ .../isp/kernels/ctc/ctc2/ia_css_ctc2_param.h       |    49 -
+ .../isp/kernels/ctc/ctc2/ia_css_ctc2_types.h       |    55 -
+ .../isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c      |    63 -
+ .../isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h      |    36 -
+ .../isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h     |    44 -
+ .../kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c    |   215 -
+ .../kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h    |    24 -
+ .../isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h     |   110 -
+ .../css2400/isp/kernels/de/de_1.0/ia_css_de.host.c |    79 -
+ .../css2400/isp/kernels/de/de_1.0/ia_css_de.host.h |    44 -
+ .../isp/kernels/de/de_1.0/ia_css_de_param.h        |    27 -
+ .../isp/kernels/de/de_1.0/ia_css_de_state.h        |    26 -
+ .../isp/kernels/de/de_1.0/ia_css_de_types.h        |    43 -
+ .../css2400/isp/kernels/de/de_2/ia_css_de2.host.c  |    54 -
+ .../css2400/isp/kernels/de/de_2/ia_css_de2.host.h  |    38 -
+ .../css2400/isp/kernels/de/de_2/ia_css_de2_param.h |    30 -
+ .../css2400/isp/kernels/de/de_2/ia_css_de2_types.h |    42 -
+ .../css2400/isp/kernels/de/de_2/ia_css_de_param.h  |    20 -
+ .../css2400/isp/kernels/de/de_2/ia_css_de_state.h  |    21 -
+ .../css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.c |   132 -
+ .../css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.h |    47 -
+ .../isp/kernels/dp/dp_1.0/ia_css_dp_param.h        |    36 -
+ .../isp/kernels/dp/dp_1.0/ia_css_dp_types.h        |    50 -
+ .../css2400/isp/kernels/dpc2/ia_css_dpc2.host.c    |    65 -
+ .../css2400/isp/kernels/dpc2/ia_css_dpc2.host.h    |    39 -
+ .../css2400/isp/kernels/dpc2/ia_css_dpc2_param.h   |    53 -
+ .../css2400/isp/kernels/dpc2/ia_css_dpc2_types.h   |    59 -
+ .../isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c      |   306 -
+ .../isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h      |    60 -
+ .../isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h     |    39 -
+ .../isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h     |    30 -
+ .../isp/kernels/eed1_8/ia_css_eed1_8.host.c        |   321 -
+ .../isp/kernels/eed1_8/ia_css_eed1_8.host.h        |    45 -
+ .../isp/kernels/eed1_8/ia_css_eed1_8_param.h       |   154 -
+ .../isp/kernels/eed1_8/ia_css_eed1_8_types.h       |    86 -
+ .../isp/kernels/fc/fc_1.0/ia_css_formats.host.c    |    62 -
+ .../isp/kernels/fc/fc_1.0/ia_css_formats.host.h    |    45 -
+ .../isp/kernels/fc/fc_1.0/ia_css_formats_param.h   |    25 -
+ .../isp/kernels/fc/fc_1.0/ia_css_formats_types.h   |    38 -
+ .../fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h  |    33 -
+ .../fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h  |    26 -
+ .../isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c      |    89 -
+ .../isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h      |    44 -
+ .../isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h     |    35 -
+ .../isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h     |    52 -
+ .../css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.c |   118 -
+ .../css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.h |    65 -
+ .../isp/kernels/gc/gc_1.0/ia_css_gc_param.h        |    61 -
+ .../isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c   |   214 -
+ .../isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h   |    24 -
+ .../isp/kernels/gc/gc_1.0/ia_css_gc_types.h        |    97 -
+ .../css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.c  |   110 -
+ .../css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.h  |    79 -
+ .../css2400/isp/kernels/gc/gc_2/ia_css_gc2_param.h |    43 -
+ .../isp/kernels/gc/gc_2/ia_css_gc2_table.host.c    |   132 -
+ .../isp/kernels/gc/gc_2/ia_css_gc2_table.host.h    |    26 -
+ .../css2400/isp/kernels/gc/gc_2/ia_css_gc2_types.h |    54 -
+ .../css2400/isp/kernels/hdr/ia_css_hdr.host.c      |    41 -
+ .../css2400/isp/kernels/hdr/ia_css_hdr.host.h      |    31 -
+ .../css2400/isp/kernels/hdr/ia_css_hdr_param.h     |    53 -
+ .../css2400/isp/kernels/hdr/ia_css_hdr_types.h     |    64 -
+ .../io_ls/bayer_io_ls/ia_css_bayer_io.host.c       |    86 -
+ .../io_ls/bayer_io_ls/ia_css_bayer_io.host.h       |    31 -
+ .../io_ls/bayer_io_ls/ia_css_bayer_io_param.h      |    22 -
+ .../io_ls/bayer_io_ls/ia_css_bayer_io_types.h      |    22 -
+ .../kernels/io_ls/common/ia_css_common_io_param.h  |    22 -
+ .../kernels/io_ls/common/ia_css_common_io_types.h  |    31 -
+ .../io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h    |    22 -
+ .../io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h    |    22 -
+ .../ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c  |    86 -
+ .../ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h  |    31 -
+ .../ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h |    22 -
+ .../ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h |    22 -
+ .../ipu2_io_ls/common/ia_css_common_io_param.h     |    22 -
+ .../ipu2_io_ls/common/ia_css_common_io_types.h     |    31 -
+ .../yuv444_io_ls/ia_css_yuv444_io.host.c           |    86 -
+ .../yuv444_io_ls/ia_css_yuv444_io.host.h           |    31 -
+ .../yuv444_io_ls/ia_css_yuv444_io_param.h          |    22 -
+ .../yuv444_io_ls/ia_css_yuv444_io_types.h          |    22 -
+ .../iterator/iterator_1.0/ia_css_iterator.host.c   |    80 -
+ .../iterator/iterator_1.0/ia_css_iterator.host.h   |    34 -
+ .../iterator/iterator_1.0/ia_css_iterator_param.h  |    38 -
+ .../isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c |    74 -
+ .../isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h |    41 -
+ .../kernels/macc/macc1_5/ia_css_macc1_5_param.h    |    31 -
+ .../macc/macc1_5/ia_css_macc1_5_table.host.c       |    32 -
+ .../macc/macc1_5/ia_css_macc1_5_table.host.h       |    22 -
+ .../kernels/macc/macc1_5/ia_css_macc1_5_types.h    |    74 -
+ .../isp/kernels/macc/macc_1.0/ia_css_macc.host.c   |    49 -
+ .../isp/kernels/macc/macc_1.0/ia_css_macc.host.h   |    42 -
+ .../isp/kernels/macc/macc_1.0/ia_css_macc_param.h  |    25 -
+ .../kernels/macc/macc_1.0/ia_css_macc_table.host.c |    47 -
+ .../kernels/macc/macc_1.0/ia_css_macc_table.host.h |    23 -
+ .../isp/kernels/macc/macc_1.0/ia_css_macc_types.h  |    63 -
+ .../isp/kernels/norm/norm_1.0/ia_css_norm.host.c   |    16 -
+ .../isp/kernels/norm/norm_1.0/ia_css_norm.host.h   |    20 -
+ .../isp/kernels/norm/norm_1.0/ia_css_norm_param.h  |    19 -
+ .../css2400/isp/kernels/ob/ob2/ia_css_ob2.host.c   |    79 -
+ .../css2400/isp/kernels/ob/ob2/ia_css_ob2.host.h   |    40 -
+ .../css2400/isp/kernels/ob/ob2/ia_css_ob2_param.h  |    29 -
+ .../css2400/isp/kernels/ob/ob2/ia_css_ob2_types.h  |    45 -
+ .../css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.c |   159 -
+ .../css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.h |    53 -
+ .../isp/kernels/ob/ob_1.0/ia_css_ob_param.h        |    48 -
+ .../isp/kernels/ob/ob_1.0/ia_css_ob_types.h        |    69 -
+ .../kernels/output/output_1.0/ia_css_output.host.c |   162 -
+ .../kernels/output/output_1.0/ia_css_output.host.h |    75 -
+ .../output/output_1.0/ia_css_output_param.h        |    36 -
+ .../output/output_1.0/ia_css_output_types.h        |    48 -
+ .../kernels/qplane/qplane_2/ia_css_qplane.host.c   |    61 -
+ .../kernels/qplane/qplane_2/ia_css_qplane.host.h   |    43 -
+ .../kernels/qplane/qplane_2/ia_css_qplane_param.h  |    30 -
+ .../kernels/qplane/qplane_2/ia_css_qplane_types.h  |    33 -
+ .../isp/kernels/raw/raw_1.0/ia_css_raw.host.c      |   136 -
+ .../isp/kernels/raw/raw_1.0/ia_css_raw.host.h      |    38 -
+ .../isp/kernels/raw/raw_1.0/ia_css_raw_param.h     |    38 -
+ .../isp/kernels/raw/raw_1.0/ia_css_raw_types.h     |    37 -
+ .../raw_aa_binning_1.0/ia_css_raa.host.c           |    35 -
+ .../raw_aa_binning_1.0/ia_css_raa.host.h           |    27 -
+ .../isp/kernels/ref/ref_1.0/ia_css_ref.host.c      |    74 -
+ .../isp/kernels/ref/ref_1.0/ia_css_ref.host.h      |    41 -
+ .../isp/kernels/ref/ref_1.0/ia_css_ref_param.h     |    36 -
+ .../isp/kernels/ref/ref_1.0/ia_css_ref_state.h     |    26 -
+ .../isp/kernels/ref/ref_1.0/ia_css_ref_types.h     |    28 -
+ .../isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c      |   386 -
+ .../isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h      |    77 -
+ .../isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h     |    54 -
+ .../isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h     |   220 -
+ .../css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.c |   130 -
+ .../css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.h |    77 -
+ .../isp/kernels/sc/sc_1.0/ia_css_sc_param.h        |    71 -
+ .../isp/kernels/sc/sc_1.0/ia_css_sc_types.h        |   136 -
+ .../kernels/sdis/common/ia_css_sdis_common.host.h  |    99 -
+ .../kernels/sdis/common/ia_css_sdis_common_types.h |   219 -
+ .../isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c   |   423 -
+ .../isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h   |   101 -
+ .../isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h  |    53 -
+ .../isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c    |   338 -
+ .../isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h    |    95 -
+ .../isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h   |    69 -
+ .../isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c      |    76 -
+ .../isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h      |    38 -
+ .../isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h     |    43 -
+ .../isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h     |    53 -
+ .../isp/kernels/tnr/tnr3/ia_css_tnr3_types.h       |    61 -
+ .../isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c      |   130 -
+ .../isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h      |    56 -
+ .../isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h     |    48 -
+ .../isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h     |    26 -
+ .../isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h     |    60 -
+ .../isp/kernels/uds/uds_1.0/ia_css_uds_param.h     |    31 -
+ .../css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.c |   140 -
+ .../css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.h |    47 -
+ .../isp/kernels/vf/vf_1.0/ia_css_vf_param.h        |    37 -
+ .../isp/kernels/vf/vf_1.0/ia_css_vf_types.h        |    32 -
+ .../css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.c |    89 -
+ .../css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.h |    39 -
+ .../isp/kernels/wb/wb_1.0/ia_css_wb_param.h        |    29 -
+ .../isp/kernels/wb/wb_1.0/ia_css_wb_types.h        |    47 -
+ .../isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c      |    66 -
+ .../isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h      |    47 -
+ .../isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h     |    51 -
+ .../kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c    |    81 -
+ .../kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h    |    22 -
+ .../isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h     |    71 -
+ .../isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c     |   265 -
+ .../isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h     |    42 -
+ .../isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h    |    96 -
+ .../isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h    |    98 -
+ .../isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c      |   219 -
+ .../isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h      |    60 -
+ .../isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h     |    49 -
+ .../isp/kernels/ynr/ynr_1.0/ia_css_ynr_state.h     |    26 -
+ .../isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h     |    81 -
+ .../isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c       |   125 -
+ .../isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h       |    56 -
+ .../isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h      |    45 -
+ .../isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h      |    94 -
+ .../isp/kernels/ynr/ynr_2/ia_css_ynr_param.h       |    20 -
+ .../isp/kernels/ynr/ynr_2/ia_css_ynr_state.h       |    21 -
+ .../css2400/isp/modes/interface/input_buf.isp.h    |    73 -
+ .../css2400/isp/modes/interface/isp_const.h        |   482 -
+ .../css2400/isp/modes/interface/isp_types.h        |   128 -
+ .../atomisp/pci/atomisp2/css2400/memory_realloc.c  |    81 -
+ .../runtime/binary/interface/ia_css_binary.h       |   257 -
+ .../atomisp2/css2400/runtime/binary/src/binary.c   |  1838 ---
+ .../css2400/runtime/bufq/interface/ia_css_bufq.h   |   197 -
+ .../runtime/bufq/interface/ia_css_bufq_comm.h      |    66 -
+ .../pci/atomisp2/css2400/runtime/bufq/src/bufq.c   |   589 -
+ .../css2400/runtime/debug/interface/ia_css_debug.h |   509 -
+ .../debug/interface/ia_css_debug_internal.h        |    31 -
+ .../runtime/debug/interface/ia_css_debug_pipe.h    |    84 -
+ .../css2400/runtime/debug/src/ia_css_debug.c       |  3596 ------
+ .../css2400/runtime/event/interface/ia_css_event.h |    46 -
+ .../pci/atomisp2/css2400/runtime/event/src/event.c |   126 -
+ .../runtime/eventq/interface/ia_css_eventq.h       |    69 -
+ .../atomisp2/css2400/runtime/eventq/src/eventq.c   |    77 -
+ .../css2400/runtime/frame/interface/ia_css_frame.h |   180 -
+ .../runtime/frame/interface/ia_css_frame_comm.h    |   132 -
+ .../pci/atomisp2/css2400/runtime/frame/src/frame.c |  1026 --
+ .../css2400/runtime/ifmtr/interface/ia_css_ifmtr.h |    49 -
+ .../pci/atomisp2/css2400/runtime/ifmtr/src/ifmtr.c |   569 -
+ .../runtime/inputfifo/interface/ia_css_inputfifo.h |    69 -
+ .../css2400/runtime/inputfifo/src/inputfifo.c      |   613 -
+ .../runtime/isp_param/interface/ia_css_isp_param.h |   118 -
+ .../isp_param/interface/ia_css_isp_param_types.h   |    98 -
+ .../css2400/runtime/isp_param/src/isp_param.c      |   227 -
+ .../css2400/runtime/isys/interface/ia_css_isys.h   |   201 -
+ .../runtime/isys/interface/ia_css_isys_comm.h      |    69 -
+ .../css2400/runtime/isys/src/csi_rx_rmgr.c         |   179 -
+ .../css2400/runtime/isys/src/csi_rx_rmgr.h         |    43 -
+ .../css2400/runtime/isys/src/ibuf_ctrl_rmgr.c      |   140 -
+ .../css2400/runtime/isys/src/ibuf_ctrl_rmgr.h      |    55 -
+ .../css2400/runtime/isys/src/isys_dma_rmgr.c       |   103 -
+ .../css2400/runtime/isys/src/isys_dma_rmgr.h       |    41 -
+ .../atomisp2/css2400/runtime/isys/src/isys_init.c  |   139 -
+ .../runtime/isys/src/isys_stream2mmio_rmgr.c       |   105 -
+ .../runtime/isys/src/isys_stream2mmio_rmgr.h       |    41 -
+ .../pci/atomisp2/css2400/runtime/isys/src/rx.c     |   607 -
+ .../css2400/runtime/isys/src/virtual_isys.c        |   898 --
+ .../css2400/runtime/isys/src/virtual_isys.h        |    41 -
+ .../runtime/pipeline/interface/ia_css_pipeline.h   |   302 -
+ .../pipeline/interface/ia_css_pipeline_common.h    |    42 -
+ .../css2400/runtime/pipeline/src/pipeline.c        |   805 --
+ .../css2400/runtime/queue/interface/ia_css_queue.h |   192 -
+ .../runtime/queue/interface/ia_css_queue_comm.h    |    69 -
+ .../pci/atomisp2/css2400/runtime/queue/src/queue.c |   412 -
+ .../css2400/runtime/queue/src/queue_access.c       |   192 -
+ .../css2400/runtime/queue/src/queue_access.h       |   101 -
+ .../css2400/runtime/rmgr/interface/ia_css_rmgr.h   |    88 -
+ .../runtime/rmgr/interface/ia_css_rmgr_vbuf.h      |   115 -
+ .../pci/atomisp2/css2400/runtime/rmgr/src/rmgr.c   |    55 -
+ .../atomisp2/css2400/runtime/rmgr/src/rmgr_vbuf.c  |   330 -
+ .../runtime/spctrl/interface/ia_css_spctrl.h       |    87 -
+ .../runtime/spctrl/interface/ia_css_spctrl_comm.h  |    61 -
+ .../atomisp2/css2400/runtime/spctrl/src/spctrl.c   |   193 -
+ .../tagger/interface/ia_css_tagger_common.h        |    59 -
+ .../pci/atomisp2/css2400/runtime/timer/src/timer.c |    48 -
+ .../media/atomisp/pci/atomisp2/css2400/sh_css.c    | 11094 -------------------
+ .../atomisp/pci/atomisp2/css2400/sh_css_defs.h     |   410 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_dvs_info.h |    36 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_firmware.c |   315 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_firmware.h |    54 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_frac.h     |    40 -
+ .../pci/atomisp2/css2400/sh_css_host_data.c        |    42 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_hrt.c      |    84 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_hrt.h      |    34 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_internal.h |  1089 --
+ .../atomisp/pci/atomisp2/css2400/sh_css_legacy.h   |    77 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_metadata.c |    16 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_metrics.c  |   176 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_metrics.h  |    55 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_mipi.c     |   749 --
+ .../atomisp/pci/atomisp2/css2400/sh_css_mipi.h     |    49 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_mmu.c      |    56 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_morph.c    |    16 -
+ .../pci/atomisp2/css2400/sh_css_param_dvs.c        |   267 -
+ .../pci/atomisp2/css2400/sh_css_param_dvs.h        |    86 -
+ .../pci/atomisp2/css2400/sh_css_param_shading.c    |   417 -
+ .../pci/atomisp2/css2400/sh_css_param_shading.h    |    39 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_params.c   |  5253 ---------
+ .../atomisp/pci/atomisp2/css2400/sh_css_params.h   |   188 -
+ .../pci/atomisp2/css2400/sh_css_params_internal.h  |    21 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_pipe.c     |    16 -
+ .../pci/atomisp2/css2400/sh_css_properties.c       |    43 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_shading.c  |    16 -
+ .../media/atomisp/pci/atomisp2/css2400/sh_css_sp.c |  1803 ---
+ .../media/atomisp/pci/atomisp2/css2400/sh_css_sp.h |   248 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_stream.c   |    16 -
+ .../pci/atomisp2/css2400/sh_css_stream_format.c    |    76 -
+ .../pci/atomisp2/css2400/sh_css_stream_format.h    |    23 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_struct.h   |    80 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_uds.h      |    37 -
+ .../atomisp/pci/atomisp2/css2400/sh_css_version.c  |    30 -
+ .../staging/media/atomisp/pci/atomisp2/hmm/hmm.c   |   727 --
+ .../media/atomisp/pci/atomisp2/hmm/hmm_bo.c        |  1528 ---
+ .../atomisp/pci/atomisp2/hmm/hmm_dynamic_pool.c    |   233 -
+ .../atomisp/pci/atomisp2/hmm/hmm_reserved_pool.c   |   252 -
+ .../media/atomisp/pci/atomisp2/hmm/hmm_vm.c        |   212 -
+ .../atomisp2/hrt/hive_isp_css_custom_host_hrt.h    |   103 -
+ .../atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.c |   125 -
+ .../atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.h |    56 -
+ .../media/atomisp/pci/atomisp2/include/hmm/hmm.h   |   102 -
+ .../atomisp/pci/atomisp2/include/hmm/hmm_bo.h      |   319 -
+ .../atomisp/pci/atomisp2/include/hmm/hmm_common.h  |    96 -
+ .../atomisp/pci/atomisp2/include/hmm/hmm_pool.h    |   115 -
+ .../atomisp/pci/atomisp2/include/hmm/hmm_vm.h      |    64 -
+ .../atomisp/pci/atomisp2/include/mmu/isp_mmu.h     |   169 -
+ .../pci/atomisp2/include/mmu/sh_mmu_mrfld.h        |    24 -
+ .../media/atomisp/pci/atomisp2/mmu/isp_mmu.c       |   584 -
+ .../media/atomisp/pci/atomisp2/mmu/sh_mmu_mrfld.c  |    75 -
+ drivers/staging/media/atomisp/platform/Makefile    |     5 -
+ .../media/atomisp/platform/intel-mid/Makefile      |     4 -
+ .../platform/intel-mid/atomisp_gmin_platform.c     |   785 --
+ drivers/staging/media/davinci_vpfe/Kconfig         |     4 +-
+ drivers/staging/media/davinci_vpfe/Makefile        |     5 +
+ drivers/staging/media/davinci_vpfe/TODO            |     1 +
+ drivers/staging/media/davinci_vpfe/dm365_ipipe.c   |   143 +-
+ .../staging/media/davinci_vpfe/dm365_ipipe_hw.c    |    19 +-
+ drivers/staging/media/davinci_vpfe/dm365_ipipeif.c |    14 +-
+ drivers/staging/media/davinci_vpfe/dm365_isif.c    |     9 +-
+ drivers/staging/media/davinci_vpfe/dm365_resizer.c |    15 +-
+ .../staging/media/davinci_vpfe/vpfe_mc_capture.c   |     2 +-
+ drivers/staging/media/davinci_vpfe/vpfe_video.c    |     4 +-
+ drivers/staging/media/imx/imx-media-capture.c      |     4 +-
+ drivers/staging/media/imx/imx-media-csi.c          |     3 +-
+ drivers/staging/media/imx/imx-media-utils.c        |     9 +
+ drivers/staging/media/omap4iss/Kconfig             |     4 +-
+ drivers/staging/media/tegra-vde/tegra-vde.c        |   105 +-
+ drivers/{media/pci => staging/media}/zoran/Kconfig |     2 +-
+ .../{media/pci => staging/media}/zoran/Makefile    |     0
+ drivers/staging/media/zoran/TODO                   |     4 +
+ .../pci => staging/media}/zoran/videocodec.c       |     0
+ .../pci => staging/media}/zoran/videocodec.h       |     0
+ drivers/{media/pci => staging/media}/zoran/zoran.h |     0
+ .../pci => staging/media}/zoran/zoran_card.c       |     0
+ .../pci => staging/media}/zoran/zoran_card.h       |     0
+ .../pci => staging/media}/zoran/zoran_device.c     |     0
+ .../pci => staging/media}/zoran/zoran_device.h     |     0
+ .../pci => staging/media}/zoran/zoran_driver.c     |     4 +-
+ .../pci => staging/media}/zoran/zoran_procfs.c     |     0
+ .../pci => staging/media}/zoran/zoran_procfs.h     |     0
+ .../{media/pci => staging/media}/zoran/zr36016.c   |     0
+ .../{media/pci => staging/media}/zoran/zr36016.h   |     0
+ .../{media/pci => staging/media}/zoran/zr36050.c   |     0
+ .../{media/pci => staging/media}/zoran/zr36050.h   |     0
+ .../{media/pci => staging/media}/zoran/zr36057.h   |     0
+ .../{media/pci => staging/media}/zoran/zr36060.c   |     0
+ .../{media/pci => staging/media}/zoran/zr36060.h   |     0
+ drivers/video/fbdev/omap2/Kconfig                  |     2 +-
+ include/linux/omap-iommu.h                         |     5 +
+ include/linux/platform_data/media/mmp-camera.h     |    19 +
+ include/linux/sony-laptop.h                        |     4 +
+ include/media/dvb-usb-ids.h                        |     1 +
+ include/media/dvbdev.h                             |     2 +
+ include/media/media-entity.h                       |     2 +-
+ include/media/rc-core.h                            |     1 +
+ include/media/v4l2-dev.h                           |    25 +-
+ include/media/v4l2-device.h                        |     4 +-
+ include/media/v4l2-fwnode.h                        |     2 +-
+ include/media/v4l2-ioctl.h                         |    12 -
+ include/media/videobuf-dvb.h                       |    59 -
+ include/media/vsp1.h                               |    45 +-
+ include/uapi/linux/lirc.h                          |     6 +
+ include/uapi/linux/omap3isp.h                      |    22 +
+ include/video/omapfb_dss.h                         |    54 +-
+ sound/isa/Kconfig                                  |     3 +-
+ 1164 files changed, 16773 insertions(+), 176676 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/media/cdns,csi2rx.txt
+ create mode 100644 Documentation/devicetree/bindings/media/cdns,csi2tx.txt
+ create mode 100644 Documentation/devicetree/bindings/media/i2c/ov7251.txt
+ create mode 100644 Documentation/devicetree/bindings/media/i2c/ov772x.txt
+ create mode 100644 Documentation/devicetree/bindings/media/i2c/panasonic,amg88xx.txt
+ create mode 100644 Documentation/devicetree/bindings/media/renesas,rcar-csi2.txt
+ delete mode 100644 Documentation/media/uapi/v4l/selection-api-005.rst
+ rename Documentation/media/uapi/v4l/{selection-api-004.rst => selection-api-configuration.rst} (98%)
+ rename Documentation/media/uapi/v4l/{selection-api-006.rst => selection-api-examples.rst} (100%)
+ rename Documentation/media/uapi/v4l/{selection-api-002.rst => selection-api-intro.rst} (100%)
+ rename Documentation/media/uapi/v4l/{selection-api-003.rst => selection-api-targets.rst} (100%)
+ create mode 100644 Documentation/media/uapi/v4l/selection-api-vs-crop-api.rst
+ create mode 100644 drivers/media/i2c/imx258.c
+ create mode 100644 drivers/media/i2c/ov7251.c
+ create mode 100644 drivers/media/i2c/video-i2c.c
+ create mode 100644 drivers/media/pci/ddbridge/ddbridge-mci.c
+ create mode 100644 drivers/media/pci/ddbridge/ddbridge-mci.h
+ delete mode 100644 drivers/media/pci/pt1/va1j5jf8007s.c
+ delete mode 100644 drivers/media/pci/pt1/va1j5jf8007s.h
+ delete mode 100644 drivers/media/pci/pt1/va1j5jf8007t.c
+ delete mode 100644 drivers/media/pci/pt1/va1j5jf8007t.h
+ create mode 100644 drivers/media/platform/cadence/Kconfig
+ create mode 100644 drivers/media/platform/cadence/Makefile
+ create mode 100644 drivers/media/platform/cadence/cdns-csi2rx.c
+ create mode 100644 drivers/media/platform/cadence/cdns-csi2tx.c
+ create mode 100644 drivers/media/platform/rcar-vin/rcar-csi2.c
+ delete mode 100644 drivers/media/platform/vsp1/vsp1_bru.h
+ rename drivers/media/platform/vsp1/{vsp1_bru.c => vsp1_brx.c} (60%)
+ create mode 100644 drivers/media/platform/vsp1/vsp1_brx.h
+ create mode 100644 drivers/media/platform/vsp1/vsp1_uif.c
+ create mode 100644 drivers/media/platform/vsp1/vsp1_uif.h
+ create mode 100644 drivers/media/tuners/qm1d1b0004.c
+ create mode 100644 drivers/media/tuners/qm1d1b0004.h
+ delete mode 100644 drivers/media/v4l2-core/videobuf-dvb.c
+ delete mode 100644 drivers/staging/media/atomisp/Kconfig
+ delete mode 100644 drivers/staging/media/atomisp/Makefile
+ delete mode 100644 drivers/staging/media/atomisp/TODO
+ delete mode 100644 drivers/staging/media/atomisp/i2c/Kconfig
+ delete mode 100644 drivers/staging/media/atomisp/i2c/Makefile
+ delete mode 100644 drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
+ delete mode 100644 drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
+ delete mode 100644 drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c
+ delete mode 100644 drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
+ delete mode 100644 drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c
+ delete mode 100644 drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
+ delete mode 100644 drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
+ delete mode 100644 drivers/staging/media/atomisp/i2c/gc0310.h
+ delete mode 100644 drivers/staging/media/atomisp/i2c/gc2235.h
+ delete mode 100644 drivers/staging/media/atomisp/i2c/mt9m114.h
+ delete mode 100644 drivers/staging/media/atomisp/i2c/ov2680.h
+ delete mode 100644 drivers/staging/media/atomisp/i2c/ov2722.h
+ delete mode 100644 drivers/staging/media/atomisp/i2c/ov5693/Kconfig
+ delete mode 100644 drivers/staging/media/atomisp/i2c/ov5693/Makefile
+ delete mode 100644 drivers/staging/media/atomisp/i2c/ov5693/ad5823.h
+ delete mode 100644 drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c
+ delete mode 100644 drivers/staging/media/atomisp/i2c/ov5693/ov5693.h
+ delete mode 100644 drivers/staging/media/atomisp/include/linux/atomisp.h
+ delete mode 100644 drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h
+ delete mode 100644 drivers/staging/media/atomisp/include/linux/atomisp_platform.h
+ delete mode 100644 drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h
+ delete mode 100644 drivers/staging/media/atomisp/include/media/lm3554.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/Kconfig
+ delete mode 100644 drivers/staging/media/atomisp/pci/Makefile
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/Makefile
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp-regs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_acc.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_acc.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_common.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_ioctl32.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_ioctl32.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_csi2.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_csi2.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_dfs_tables.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_drvfs.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_drvfs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_file.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_file.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_helper.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_internal.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_tables.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_tpg.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_tpg.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_trace_event.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/Makefile
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf_comm.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf_desc.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/src/circbuf.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/base/refcount/interface/ia_css_refcount.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/base/refcount/src/refcount.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_binarydesc.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_stagedesc.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_util.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_binarydesc.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_stagedesc.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_util.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/util/interface/ia_css_util.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/util/src/util.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_configs.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_configs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_params.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_states.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_states.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/bits.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/cell_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/css_receiver_2400_common_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/css_receiver_2400_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/dma_v2_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gdc_v2_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gp_timer_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gpio_block_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_host_ids_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_irq_types_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/if_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_formatter_subsystem_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_selector_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_switch_2400_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_system_ctrl_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_system_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/irq_controller_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp2400_mamoiada_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp2400_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp_acquisition_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp_capture_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/mmu_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/scalar_processor_2400_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/str2mem_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/streaming_to_mipi_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/timed_controller_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/var.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/version.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/spmem_dump.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/csi_rx_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_configs.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_configs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_params.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_states.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_states.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/input_system_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/input_system_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/pixelgen_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/pixelgen_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/system_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/PixelGen_SysBlock_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/bits.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/cell_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/css_receiver_2400_common_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/css_receiver_2400_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/dma_v2_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gdc_v2_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gp_timer_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gpio_block_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_2401_irq_types_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_host_ids_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/ibuf_cntrl_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/if_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_formatter_subsystem_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_selector_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_switch_2400_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_system_ctrl_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_system_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/irq_controller_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp2400_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp2401_mamoiada_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp_acquisition_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp_capture_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mipi_backend_common_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mipi_backend_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mmu_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/rx_csi_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/scalar_processor_2400_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/str2mem_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/stream2mmio_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/streaming_to_mipi_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/timed_controller_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/var.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/version.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/ibuf_ctrl_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/input_system_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_dma_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_irq_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_stream2mmio_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/pixelgen_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/spmem_dump.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/system_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_configs.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_configs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_params.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_states.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_states.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/bits.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/cell_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/css_receiver_2400_common_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/css_receiver_2400_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/dma_v2_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gdc_v2_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gp_timer_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gpio_block_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_2401_irq_types_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_host_ids_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/if_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_formatter_subsystem_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_selector_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_switch_2400_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_system_ctrl_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_system_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/irq_controller_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp2400_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp2401_mamoiada_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp_acquisition_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp_capture_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/mmu_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/scalar_processor_2400_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/str2mem_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/streaming_to_mipi_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/timed_controller_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/var.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/version.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/spmem_dump.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/css_trace.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/debug_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/dma_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/event_fifo_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/fifo_monitor_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gdc_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gp_device_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gp_timer_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gpio_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/hmem_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gpio_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gpio_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/system_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vamem_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vamem_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/input_formatter_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/input_system_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/irq_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/isp_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/mmu_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/sp_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/system_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/timed_ctrl_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/vamem_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/vmem_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/assert_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/bitop_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/csi_rx.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/debug.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/device_access/device_access.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/dma.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/error_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/event_fifo.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/fifo_monitor.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gdc_device.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gp_device.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gp_timer.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gpio.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/hmem.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/csi_rx_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/debug_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/dma_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/event_fifo_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/fifo_monitor_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gdc_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gp_device_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gp_timer_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gpio_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/hmem_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/ibuf_ctrl_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/input_formatter_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/input_system_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/irq_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isp_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_dma_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_irq_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_stream2mmio_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/mmu_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/pixelgen_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/sp_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/tag_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/timed_ctrl_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/vamem_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/vmem_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/ibuf_ctrl.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/input_formatter.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/input_system.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/irq.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isp.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_dma.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_irq.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_stream2mmio.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/math_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/memory_access/memory_access.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/memory_realloc.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/misc_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/mmu_device.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/pixelgen.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/platform_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/print_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/queue.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/resource.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/socket.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/sp.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/string_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/system_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/tag.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/timed_ctrl.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/type_support.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/vamem.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/vmem.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/queue_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/queue_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag_local.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/queue_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/sw_event_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/tag_global.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_3a.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_acc_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_buffer.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_control.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_device_access.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_device_access.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_dvs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_env.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_err.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_event_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_firmware.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frac.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frame_format.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frame_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_host_data.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_input_port.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_irq.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_memory_access.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_metadata.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mipi.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mmu.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mmu_private.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_morph.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_pipe.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_pipe_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_prbs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_properties.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_shading.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream_format.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream_public.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_timer.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_tpg.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_version.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_version_data.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_state.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de_state.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/common/ia_css_common_io_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/common/ia_css_common_io_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_state.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/common/ia_css_sdis_common.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/common/ia_css_sdis_common_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/uds/uds_1.0/ia_css_uds_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_state.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr_state.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/input_buf.isp.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/isp_const.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/isp_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/memory_realloc.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/binary/interface/ia_css_binary.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/binary/src/binary.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/interface/ia_css_bufq.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/interface/ia_css_bufq_comm.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/src/bufq.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug_internal.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug_pipe.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/src/ia_css_debug.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/event/interface/ia_css_event.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/event/src/event.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/eventq/interface/ia_css_eventq.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/eventq/src/eventq.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/interface/ia_css_frame.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/interface/ia_css_frame_comm.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/src/frame.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/ifmtr/interface/ia_css_ifmtr.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/ifmtr/src/ifmtr.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/inputfifo/interface/ia_css_inputfifo.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/inputfifo/src/inputfifo.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/interface/ia_css_isp_param.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/interface/ia_css_isp_param_types.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/src/isp_param.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/interface/ia_css_isys.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/interface/ia_css_isys_comm.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/csi_rx_rmgr.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/csi_rx_rmgr.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_dma_rmgr.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_dma_rmgr.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_init.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_stream2mmio_rmgr.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_stream2mmio_rmgr.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/rx.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/virtual_isys.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/virtual_isys.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/interface/ia_css_pipeline.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/interface/ia_css_pipeline_common.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/src/pipeline.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/interface/ia_css_queue.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/interface/ia_css_queue_comm.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue_access.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue_access.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/interface/ia_css_rmgr.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/interface/ia_css_rmgr_vbuf.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/src/rmgr.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/src/rmgr_vbuf.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/interface/ia_css_spctrl.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/interface/ia_css_spctrl_comm.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/src/spctrl.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/tagger/interface/ia_css_tagger_common.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/timer/src/timer.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_defs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_dvs_info.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_frac.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_host_data.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_hrt.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_internal.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_legacy.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metadata.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metrics.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metrics.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mipi.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mipi.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mmu.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_morph.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_dvs.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_dvs.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_shading.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_shading.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params_internal.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_pipe.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_properties.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_shading.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_sp.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_sp.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream_format.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream_format.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_struct.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_uds.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_version.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_dynamic_pool.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_reserved_pool.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_vm.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_custom_host_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_bo.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_common.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_pool.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_vm.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/include/mmu/isp_mmu.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/include/mmu/sh_mmu_mrfld.h
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/mmu/isp_mmu.c
+ delete mode 100644 drivers/staging/media/atomisp/pci/atomisp2/mmu/sh_mmu_mrfld.c
+ delete mode 100644 drivers/staging/media/atomisp/platform/Makefile
+ delete mode 100644 drivers/staging/media/atomisp/platform/intel-mid/Makefile
+ delete mode 100644 drivers/staging/media/atomisp/platform/intel-mid/atomisp_gmin_platform.c
+ rename drivers/{media/pci => staging/media}/zoran/Kconfig (97%)
+ rename drivers/{media/pci => staging/media}/zoran/Makefile (100%)
+ create mode 100644 drivers/staging/media/zoran/TODO
+ rename drivers/{media/pci => staging/media}/zoran/videocodec.c (100%)
+ rename drivers/{media/pci => staging/media}/zoran/videocodec.h (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zoran.h (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zoran_card.c (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zoran_card.h (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zoran_device.c (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zoran_device.h (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zoran_driver.c (99%)
+ rename drivers/{media/pci => staging/media}/zoran/zoran_procfs.c (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zoran_procfs.h (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zr36016.c (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zr36016.h (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zr36050.c (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zr36050.h (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zr36057.h (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zr36060.c (100%)
+ rename drivers/{media/pci => staging/media}/zoran/zr36060.h (100%)
+ delete mode 100644 include/media/videobuf-dvb.h
+Merging v4l-dvb-next/master (f10379aad39e media: include/video/omapfb_dss.h: use IS_ENABLED())
+$ git merge v4l-dvb-next/master
+Already up to date.
+Merging fbdev/fbdev-for-next (dbeef4fe697b video: fbdev: pxafb: match_string() conversion fixup)
+$ git merge fbdev/fbdev-for-next
+Removing include/video/sh_mobile_meram.h
+Removing include/video/auo_k190xfb.h
+Auto-merging drivers/video/fbdev/via/viafbdev.c
+Removing drivers/video/fbdev/sh_mobile_meram.c
+Removing drivers/video/fbdev/auo_k190x.h
+Removing drivers/video/fbdev/auo_k190x.c
+Removing drivers/video/fbdev/auo_k1901fb.c
+Removing drivers/video/fbdev/auo_k1900fb.c
+Auto-merging drivers/media/platform/via-camera.c
+Merge made by the 'recursive' strategy.
+ drivers/gpu/drm/shmobile/Kconfig                   |    1 -
+ drivers/gpu/drm/shmobile/shmob_drm_crtc.c          |   42 -
+ drivers/gpu/drm/shmobile/shmob_drm_crtc.h          |    1 -
+ drivers/gpu/drm/shmobile/shmob_drm_drv.h           |    2 -
+ drivers/gpu/drm/shmobile/shmob_drm_kms.c           |   11 -
+ drivers/gpu/drm/shmobile/shmob_drm_kms.h           |    1 -
+ drivers/gpu/drm/shmobile/shmob_drm_plane.c         |    2 -
+ drivers/media/platform/via-camera.c                |    5 +
+ drivers/video/fbdev/Kconfig                        |   48 +-
+ drivers/video/fbdev/Makefile                       |    4 -
+ drivers/video/fbdev/aty/aty128fb.c                 |    2 +-
+ drivers/video/fbdev/aty/radeon_pm.c                |    8 +-
+ drivers/video/fbdev/au1100fb.c                     |    2 +-
+ drivers/video/fbdev/au1200fb.c                     |    2 +-
+ drivers/video/fbdev/auo_k1900fb.c                  |  204 ----
+ drivers/video/fbdev/auo_k1901fb.c                  |  257 -----
+ drivers/video/fbdev/auo_k190x.c                    | 1195 --------------------
+ drivers/video/fbdev/auo_k190x.h                    |  129 ---
+ drivers/video/fbdev/core/fb_defio.c                |    4 +-
+ drivers/video/fbdev/mmp/fb/mmpfb.c                 |    5 +-
+ drivers/video/fbdev/mmp/hw/mmp_ctrl.c              |    8 +-
+ drivers/video/fbdev/nvidia/nvidia.c                |    2 +-
+ drivers/video/fbdev/omap/omapfb_main.c             |    2 +-
+ .../fbdev/omap2/omapfb/displays/panel-dsi-cm.c     |   18 +-
+ drivers/video/fbdev/pxafb.c                        |    8 +-
+ drivers/video/fbdev/savage/savagefb_driver.c       |    8 +-
+ drivers/video/fbdev/sh_mobile_lcdcfb.c             |   77 +-
+ drivers/video/fbdev/sh_mobile_lcdcfb.h             |    1 -
+ drivers/video/fbdev/sh_mobile_meram.c              |  758 -------------
+ drivers/video/fbdev/sm501fb.c                      |    3 +-
+ drivers/video/fbdev/via/global.h                   |    6 +
+ drivers/video/fbdev/via/hw.c                       |    1 -
+ drivers/video/fbdev/via/via-core.c                 |    1 -
+ drivers/video/fbdev/via/via_clock.c                |    2 +-
+ drivers/video/fbdev/via/viafbdev.c                 |    1 -
+ include/linux/platform_data/shmob_drm.h            |    4 -
+ include/video/auo_k190xfb.h                        |  107 --
+ include/video/sh_mobile_lcdc.h                     |    3 -
+ include/video/sh_mobile_meram.h                    |   95 --
+ 39 files changed, 48 insertions(+), 2982 deletions(-)
+ delete mode 100644 drivers/video/fbdev/auo_k1900fb.c
+ delete mode 100644 drivers/video/fbdev/auo_k1901fb.c
+ delete mode 100644 drivers/video/fbdev/auo_k190x.c
+ delete mode 100644 drivers/video/fbdev/auo_k190x.h
+ delete mode 100644 drivers/video/fbdev/sh_mobile_meram.c
+ delete mode 100644 include/video/auo_k190xfb.h
+ delete mode 100644 include/video/sh_mobile_meram.h
+Merging pm/linux-next (c7533d0fd82f Merge branch 'pm-cpuidle' into linux-next)
+$ git merge pm/linux-next
+Auto-merging include/linux/acpi.h
+Auto-merging drivers/soc/tegra/pmc.c
+Auto-merging drivers/pci/pci.c
+Auto-merging drivers/pci/pci-driver.c
+Auto-merging drivers/i2c/i2c-core-base.c
+Auto-merging drivers/base/platform.c
+Auto-merging drivers/amba/bus.c
+Auto-merging drivers/acpi/tables.c
+Auto-merging drivers/acpi/battery.c
+Auto-merging drivers/acpi/ac.c
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ Documentation/acpi/cppc_sysfs.txt                  |  69 +++
+ .../devicetree/bindings/opp/kryo-cpufreq.txt       | 680 +++++++++++++++++++++
+ Documentation/devicetree/bindings/opp/opp.txt      |   7 +-
+ .../devicetree/bindings/power/power_domain.txt     |   6 +-
+ .../bindings/power/rockchip-io-domain.txt          |  15 +
+ MAINTAINERS                                        |   7 +
+ drivers/acpi/ac.c                                  |  48 +-
+ drivers/acpi/acpi_apd.c                            |  47 ++
+ drivers/acpi/acpi_lpss.c                           |   6 +-
+ drivers/acpi/acpi_watchdog.c                       |  72 ++-
+ drivers/acpi/acpica/acapps.h                       |   2 +
+ drivers/acpi/acpica/acglobal.h                     |   2 +-
+ drivers/acpi/acpica/dbnames.c                      |  12 +-
+ drivers/acpi/acpica/dbtest.c                       |  59 +-
+ drivers/acpi/acpica/dswscope.c                     |   8 +-
+ drivers/acpi/acpica/evgpe.c                        |   6 +
+ drivers/acpi/acpica/evxfgpe.c                      |  22 +
+ drivers/acpi/acpica/hwregs.c                       |   4 +-
+ drivers/acpi/acpica/hwxface.c                      |   4 +-
+ drivers/acpi/acpica/rsdump.c                       |   2 +-
+ drivers/acpi/acpica/tbinstal.c                     |   2 +-
+ drivers/acpi/acpica/utbuffer.c                     |   4 +-
+ drivers/acpi/acpica/utmutex.c                      |   4 +-
+ drivers/acpi/acpica/utobject.c                     |   2 +
+ drivers/acpi/acpica/utprint.c                      |   1 +
+ drivers/acpi/acpica/utstring.c                     |   2 +-
+ drivers/acpi/battery.c                             |  81 ++-
+ drivers/acpi/cppc_acpi.c                           | 159 +++--
+ drivers/acpi/device_pm.c                           |   9 +-
+ drivers/acpi/ec.c                                  |   6 +
+ drivers/acpi/internal.h                            |   1 +
+ drivers/acpi/reboot.c                              |   4 +-
+ drivers/acpi/sleep.c                               |   7 +
+ drivers/acpi/tables.c                              |   9 +-
+ drivers/amba/bus.c                                 |   4 +-
+ drivers/base/base.h                                |   3 +
+ drivers/base/core.c                                |  20 +
+ drivers/base/dd.c                                  |   7 +-
+ drivers/base/platform.c                            |  17 +-
+ drivers/base/power/common.c                        |  10 +-
+ drivers/base/power/domain.c                        | 212 +++++--
+ drivers/base/power/main.c                          |  37 +-
+ drivers/base/power/power.h                         |  30 -
+ drivers/base/power/runtime.c                       |  29 +-
+ drivers/base/power/wakeup.c                        |  96 ++-
+ drivers/base/property.c                            | 104 +++-
+ drivers/clk/x86/Makefile                           |   3 +-
+ drivers/clk/x86/clk-st.c                           |  77 +++
+ drivers/cpufreq/Kconfig.arm                        |  13 +-
+ drivers/cpufreq/Makefile                           |   1 +
+ drivers/cpufreq/armada-37xx-cpufreq.c              | 100 ++-
+ drivers/cpufreq/cppc_cpufreq.c                     |  80 ++-
+ drivers/cpufreq/cpufreq-dt-platdev.c               |   5 +-
+ drivers/cpufreq/cpufreq-dt.c                       |  10 +-
+ drivers/cpufreq/cpufreq-dt.h                       |   5 +
+ drivers/cpufreq/cpufreq.c                          |  78 ++-
+ drivers/cpufreq/cpufreq_governor.c                 |   2 +-
+ drivers/cpufreq/intel_pstate.c                     |  46 +-
+ drivers/cpufreq/qcom-cpufreq-kryo.c                | 212 +++++++
+ drivers/cpufreq/s3c2440-cpufreq.c                  |   2 +-
+ drivers/cpufreq/speedstep-lib.c                    |   2 +-
+ drivers/cpufreq/tegra20-cpufreq.c                  | 200 +++---
+ drivers/cpuidle/governor.c                         |  17 +-
+ drivers/cpuidle/governors/ladder.c                 |  10 +-
+ drivers/cpuidle/governors/menu.c                   |  10 +-
+ drivers/firmware/efi/apple-properties.c            |   8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c            |   2 +-
+ drivers/i2c/i2c-core-base.c                        |   2 +-
+ drivers/mailbox/pcc.c                              |  81 ++-
+ drivers/mmc/core/sdio_bus.c                        |   2 +-
+ drivers/opp/core.c                                 | 267 +++-----
+ drivers/opp/debugfs.c                              |  15 +-
+ drivers/opp/of.c                                   | 184 +++++-
+ drivers/opp/opp.h                                  |   6 +-
+ drivers/pci/pci-driver.c                           |   5 +-
+ drivers/pci/pci.c                                  |   6 +-
+ drivers/power/avs/rockchip-io-domain.c             |  68 +++
+ drivers/soc/tegra/pmc.c                            |  20 +-
+ drivers/soundwire/bus_type.c                       |  15 +-
+ drivers/spi/spi.c                                  |  11 +-
+ include/acpi/acnames.h                             |   7 +-
+ include/acpi/acpiosxf.h                            |  21 +
+ include/acpi/acpixf.h                              |   3 +-
+ include/acpi/actypes.h                             |   4 +
+ include/acpi/cppc_acpi.h                           |  14 +-
+ include/acpi/platform/aclinux.h                    |   5 +
+ include/acpi/platform/aclinuxex.h                  |  30 +
+ include/linux/acpi.h                               |   3 +-
+ include/linux/cpufreq.h                            |   2 +-
+ include/linux/cpuidle.h                            |   1 +
+ include/linux/kernel.h                             |   1 +
+ include/linux/notifier.h                           |  34 +-
+ include/linux/platform_data/clk-st.h               |  17 +
+ include/linux/pm_domain.h                          |  79 ++-
+ include/linux/pm_opp.h                             |  27 +-
+ include/linux/pm_runtime.h                         |   6 +-
+ include/linux/property.h                           |  52 +-
+ include/linux/srcutiny.h                           |   6 +-
+ include/linux/srcutree.h                           |   6 +-
+ kernel/power/hibernate.c                           |   7 +
+ kernel/power/qos.c                                 |   1 -
+ kernel/power/suspend.c                             |  27 +-
+ kernel/power/user.c                                |   5 +
+ kernel/power/wakelock.c                            |   1 +
+ kernel/sched/cpufreq_schedutil.c                   | 262 +++++---
+ kernel/time/tick-common.c                          |   2 +
+ tools/power/pm-graph/bootgraph.py                  |   2 +-
+ tools/power/pm-graph/sleepgraph.8                  |   4 +
+ tools/power/pm-graph/sleepgraph.py                 | 399 +++++++-----
+ .../x86/intel_pstate_tracer/intel_pstate_tracer.py |  54 +-
+ 110 files changed, 3421 insertions(+), 1164 deletions(-)
+ create mode 100644 Documentation/acpi/cppc_sysfs.txt
+ create mode 100644 Documentation/devicetree/bindings/opp/kryo-cpufreq.txt
+ create mode 100644 drivers/clk/x86/clk-st.c
+ create mode 100644 drivers/cpufreq/qcom-cpufreq-kryo.c
+ create mode 100644 include/linux/platform_data/clk-st.h
+Merging cpupower/cpupower (f9652d5cae04 cpupower : Fix header name to read idle state name)
+$ git merge cpupower/cpupower
+Merge made by the 'recursive' strategy.
+ tools/power/cpupower/bench/parse.c                 |  2 +-
+ .../cpupower/utils/idle_monitor/cpuidle_sysfs.c    | 15 ++++++++++
+ .../cpupower/utils/idle_monitor/cpupower-monitor.c | 35 ++++++++++++----------
+ .../cpupower/utils/idle_monitor/cpupower-monitor.h |  9 ++++++
+ 4 files changed, 45 insertions(+), 16 deletions(-)
+Merging idle/next (8a5776a5f498 Linux 4.14-rc4)
+$ git merge idle/next
+Already up to date.
+Merging opp/opp/linux-next (8a352fd8787c OPP: Allow same OPP table to be used for multiple genpd)
+$ git merge opp/opp/linux-next
+Already up to date.
+Merging thermal/next (e76a4386b2f5 thermal: Use DEVICE_ATTR_{RO|RW|WO}() variants)
+$ git merge thermal/next
+Merge made by the 'recursive' strategy.
+ drivers/thermal/thermal_core.c  |  6 ++--
+ drivers/thermal/thermal_core.h  | 13 +++----
+ drivers/thermal/thermal_sysfs.c | 76 +++++++++++++++--------------------------
+ 3 files changed, 35 insertions(+), 60 deletions(-)
+Merging thermal-soc/next (9efc58dfa14a thermal: mediatek: use of_device_get_match_data())
+$ git merge thermal-soc/next
+Removing drivers/thermal/samsung/exynos_tmu.h
+Auto-merging drivers/thermal/samsung/exynos_tmu.c
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/thermal/exynos-thermal.txt |  14 +-
+ .../devicetree/bindings/thermal/imx-thermal.txt    |   9 +-
+ .../bindings/thermal/mediatek-thermal.txt          |   1 +
+ .../bindings/thermal/rcar-gen3-thermal.txt         |   3 +-
+ .../bindings/thermal/uniphier-thermal.txt          |   1 +
+ drivers/thermal/imx_thermal.c                      | 295 +++++--
+ drivers/thermal/mtk_thermal.c                      |  40 +-
+ drivers/thermal/rcar_gen3_thermal.c                |  17 +-
+ drivers/thermal/samsung/exynos_tmu.c               | 953 +++++++--------------
+ drivers/thermal/samsung/exynos_tmu.h               |  75 --
+ drivers/thermal/tegra/soctherm.c                   |  29 +-
+ drivers/thermal/uniphier_thermal.c                 |   4 +
+ 12 files changed, 628 insertions(+), 813 deletions(-)
+ delete mode 100644 drivers/thermal/samsung/exynos_tmu.h
+Merging ieee1394/for-next (188775181bc0 firewire-ohci: work around oversized DMA reads on JMicron controllers)
+$ git merge ieee1394/for-next
+Already up to date.
+Merging dlm/next (da3627c30d22 dlm: remove O_NONBLOCK flag in sctp_connect_to_sock)
+$ git merge dlm/next
+Merge made by the 'recursive' strategy.
+ fs/dlm/lowcomms.c | 16 ++++++++++++++--
+ 1 file changed, 14 insertions(+), 2 deletions(-)
+Merging swiotlb/linux-next (4855c92dbb7b xen-swiotlb: fix the check condition for xen_swiotlb_free_coherent)
+$ git merge swiotlb/linux-next
+Already up to date.
+Merging rdma/for-next (f3ca0ab114e0 Merge branch 'mini_cqe' into git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma for-next)
+$ git merge rdma/for-next
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+Auto-merging drivers/infiniband/hw/mlx5/main.c
+Auto-merging drivers/infiniband/hw/hfi1/pcie.c
+Merge made by the 'recursive' strategy.
+ drivers/infiniband/Kconfig                         |  11 +
+ drivers/infiniband/core/Makefile                   |   7 +-
+ drivers/infiniband/core/addr.c                     | 172 +++----
+ drivers/infiniband/core/cache.c                    | 112 ++---
+ drivers/infiniband/core/cm.c                       |  58 ++-
+ drivers/infiniband/core/cma.c                      |  34 +-
+ drivers/infiniband/core/core_priv.h                |   3 -
+ drivers/infiniband/core/device.c                   |   4 +-
+ drivers/infiniband/core/nldev.c                    | 122 ++++-
+ drivers/infiniband/core/restrack.c                 |   7 +
+ drivers/infiniband/core/roce_gid_mgmt.c            |   2 -
+ drivers/infiniband/core/security.c                 |   4 -
+ drivers/infiniband/core/umem.c                     |  13 +-
+ drivers/infiniband/core/uverbs_cmd.c               |  23 +
+ drivers/infiniband/hw/cxgb4/Makefile               |   3 +-
+ drivers/infiniband/hw/cxgb4/cm.c                   |   1 +
+ drivers/infiniband/hw/cxgb4/iw_cxgb4.h             |   5 +
+ drivers/infiniband/hw/cxgb4/provider.c             |   8 +
+ drivers/infiniband/hw/cxgb4/qp.c                   |   9 +-
+ drivers/infiniband/hw/cxgb4/restrack.c             | 501 +++++++++++++++++++++
+ drivers/infiniband/hw/hfi1/Makefile                |  10 +-
+ drivers/infiniband/hw/hfi1/affinity.c              | 497 ++++++++++++++++++--
+ drivers/infiniband/hw/hfi1/affinity.h              |  10 +-
+ drivers/infiniband/hw/hfi1/chip.c                  |  70 +--
+ drivers/infiniband/hw/hfi1/chip.h                  |  15 +-
+ drivers/infiniband/hw/hfi1/chip_registers.h        |   7 +-
+ drivers/infiniband/hw/hfi1/debugfs.c               | 292 +-----------
+ drivers/infiniband/hw/hfi1/debugfs.h               | 102 ++---
+ drivers/infiniband/hw/hfi1/driver.c                |  53 ++-
+ drivers/infiniband/hw/hfi1/exp_rcv.c               |  39 +-
+ drivers/infiniband/hw/hfi1/exp_rcv.h               |  24 +-
+ drivers/infiniband/hw/hfi1/fault.c                 | 375 +++++++++++++++
+ drivers/infiniband/hw/hfi1/fault.h                 | 109 +++++
+ drivers/infiniband/hw/hfi1/file_ops.c              |   6 +-
+ drivers/infiniband/hw/hfi1/hfi.h                   |  57 ++-
+ drivers/infiniband/hw/hfi1/init.c                  |  32 +-
+ drivers/infiniband/hw/hfi1/mad.c                   |  37 +-
+ drivers/infiniband/hw/hfi1/pcie.c                  |   8 +
+ drivers/infiniband/hw/hfi1/pio.c                   |  44 +-
+ drivers/infiniband/hw/hfi1/rc.c                    |  10 +-
+ drivers/infiniband/hw/hfi1/ruc.c                   | 154 +------
+ drivers/infiniband/hw/hfi1/trace.c                 |  43 +-
+ drivers/infiniband/hw/hfi1/trace_dbg.h             |   3 +-
+ drivers/infiniband/hw/hfi1/trace_ibhdrs.h          | 160 ++++---
+ drivers/infiniband/hw/hfi1/uc.c                    |   4 +-
+ drivers/infiniband/hw/hfi1/ud.c                    |  61 ++-
+ drivers/infiniband/hw/hfi1/user_exp_rcv.c          |   1 -
+ drivers/infiniband/hw/hfi1/verbs.c                 |  45 +-
+ drivers/infiniband/hw/hfi1/verbs.h                 |  15 +-
+ drivers/infiniband/hw/hns/hns_roce_device.h        |   4 +
+ drivers/infiniband/hw/hns/hns_roce_hw_v2.c         |  40 +-
+ drivers/infiniband/hw/hns/hns_roce_hw_v2.h         |   6 +-
+ drivers/infiniband/hw/hns/hns_roce_main.c          |   1 -
+ drivers/infiniband/hw/hns/hns_roce_mr.c            |   6 -
+ drivers/infiniband/hw/hns/hns_roce_pd.c            |  10 +-
+ drivers/infiniband/hw/i40iw/i40iw_cm.c             |  67 ++-
+ drivers/infiniband/hw/i40iw/i40iw_cm.h             |   4 +-
+ drivers/infiniband/hw/i40iw/i40iw_hw.c             |  34 +-
+ drivers/infiniband/hw/i40iw/i40iw_main.c           |   2 +-
+ drivers/infiniband/hw/mlx4/main.c                  |   5 +-
+ drivers/infiniband/hw/mlx4/mr.c                    |  50 +-
+ drivers/infiniband/hw/mlx4/qp.c                    |   2 +-
+ drivers/infiniband/hw/mlx5/cq.c                    |  59 ++-
+ drivers/infiniband/hw/mlx5/main.c                  | 169 ++++++-
+ drivers/infiniband/hw/mlx5/qp.c                    |  43 +-
+ drivers/infiniband/hw/qedr/verbs.c                 |   2 +-
+ drivers/infiniband/hw/qib/qib.h                    |   4 +-
+ drivers/infiniband/hw/qib/qib_file_ops.c           |  12 +-
+ drivers/infiniband/hw/qib/qib_init.c               |  13 +
+ drivers/infiniband/hw/qib/qib_rc.c                 |   8 +-
+ drivers/infiniband/hw/qib/qib_ruc.c                | 154 +------
+ drivers/infiniband/hw/qib/qib_uc.c                 |   4 +-
+ drivers/infiniband/hw/qib/qib_ud.c                 |   4 +-
+ drivers/infiniband/hw/qib/qib_user_pages.c         |  20 +-
+ drivers/infiniband/hw/qib/qib_verbs.c              |   6 +-
+ drivers/infiniband/hw/qib/qib_verbs.h              |   2 -
+ drivers/infiniband/sw/rdmavt/Kconfig               |   2 +-
+ drivers/infiniband/sw/rdmavt/cq.c                  |  74 ++-
+ drivers/infiniband/sw/rdmavt/cq.h                  |   6 +-
+ drivers/infiniband/sw/rdmavt/qp.c                  | 151 ++++++-
+ drivers/infiniband/sw/rdmavt/trace_cq.h            |  35 +-
+ drivers/infiniband/sw/rdmavt/vt.c                  |  35 +-
+ drivers/infiniband/sw/rxe/Kconfig                  |   1 +
+ drivers/infiniband/sw/rxe/rxe.c                    |  11 +-
+ drivers/infiniband/sw/rxe/rxe.h                    |   4 +-
+ drivers/infiniband/sw/rxe/rxe_comp.c               |  10 +-
+ drivers/infiniband/sw/rxe/rxe_loc.h                |  13 +-
+ drivers/infiniband/sw/rxe/rxe_mr.c                 |  13 +-
+ drivers/infiniband/sw/rxe/rxe_net.c                |  18 +-
+ drivers/infiniband/sw/rxe/rxe_net.h                |   3 -
+ drivers/infiniband/sw/rxe/rxe_recv.c               |   7 +-
+ drivers/infiniband/sw/rxe/rxe_req.c                |   2 +-
+ drivers/infiniband/sw/rxe/rxe_resp.c               |  20 +-
+ drivers/infiniband/sw/rxe/rxe_verbs.c              |   6 +-
+ drivers/infiniband/ulp/ipoib/ipoib.h               |   2 +-
+ drivers/infiniband/ulp/ipoib/ipoib_main.c          |  33 +-
+ drivers/infiniband/ulp/ipoib/ipoib_multicast.c     |  15 +-
+ drivers/infiniband/ulp/isert/ib_isert.c            |  26 +-
+ drivers/net/ethernet/mellanox/mlx5/core/cmd.c      |  28 +-
+ .../mellanox/mlx5/core/diag/fs_tracepoint.c        |   2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c  |   2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_main.c  |   1 -
+ drivers/net/ethernet/mellanox/mlx5/core/en_tc.c    |   3 +
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.c  |   2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.h  |   2 +
+ .../ethernet/mellanox/mlx5/core/eswitch_offloads.c |  21 +-
+ drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c   |   9 +
+ drivers/net/ethernet/mellanox/mlx5/core/fs_core.c  |  26 +-
+ drivers/net/ethernet/mellanox/mlx5/core/fs_core.h  |   2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/mr.c       |   4 +-
+ drivers/net/ethernet/mellanox/mlx5/core/qp.c       |  18 +-
+ drivers/net/ethernet/mellanox/mlx5/core/vport.c    |   6 +-
+ drivers/net/ethernet/mellanox/mlx5/core/wq.h       |   1 -
+ include/linux/mlx5/device.h                        |  12 +
+ include/linux/mlx5/fs.h                            |   6 +-
+ include/linux/mlx5/mlx5_ifc.h                      |  66 ++-
+ include/rdma/ib_addr.h                             |  20 +-
+ include/rdma/ib_cache.h                            |   1 +
+ include/rdma/ib_verbs.h                            |  46 ++
+ include/rdma/rdma_cm.h                             |   3 +
+ include/rdma/rdma_vt.h                             |   7 +-
+ include/rdma/rdmavt_cq.h                           |   5 +-
+ include/rdma/rdmavt_qp.h                           |   1 +
+ include/rdma/restrack.h                            |  20 +
+ include/uapi/rdma/ib_user_verbs.h                  |  50 ++
+ include/uapi/rdma/mlx5-abi.h                       |   6 +-
+ include/uapi/rdma/rdma_netlink.h                   |  30 +-
+ 127 files changed, 3453 insertions(+), 1547 deletions(-)
+ create mode 100644 drivers/infiniband/hw/cxgb4/restrack.c
+ create mode 100644 drivers/infiniband/hw/hfi1/fault.c
+ create mode 100644 drivers/infiniband/hw/hfi1/fault.h
+Merging net-next/master (37c9102f216c net: davinci: fix building davinci mdio code without CONFIG_OF)
+$ git merge net-next/master
+Auto-merging tools/testing/selftests/bpf/test_tunnel_kern.c
+Auto-merging tools/testing/selftests/bpf/test_sockmap_kern.h
+Auto-merging tools/testing/selftests/bpf/test_sockmap.c
+Auto-merging security/selinux/hooks.c
+Removing samples/sockmap/sockmap_test.sh
+Removing samples/sockmap/Makefile
+Removing samples/bpf/test_tunnel_bpf.sh
+Auto-merging samples/bpf/bpf_insn.h
+Auto-merging net/tipc/socket.c
+Auto-merging net/sctp/socket.c
+Auto-merging net/packet/af_packet.c
+Auto-merging net/netfilter/nft_meta.c
+Auto-merging net/netfilter/nfnetlink_queue.c
+Auto-merging net/netfilter/nfnetlink_log.c
+Auto-merging net/netfilter/nf_tables_core.c
+Auto-merging net/netfilter/nf_tables_api.c
+Auto-merging net/netfilter/nf_flow_table_core.c
+Auto-merging net/netfilter/ipvs/ip_vs_ctl.c
+Auto-merging net/l2tp/l2tp_ppp.c
+Auto-merging net/ipv6/xfrm6_policy.c
+Auto-merging net/ipv6/udp.c
+Auto-merging net/ipv6/tcp_ipv6.c
+Auto-merging net/ipv6/seg6_iptunnel.c
+Auto-merging net/ipv6/route.c
+Auto-merging net/ipv6/ip6mr.c
+Auto-merging net/ipv6/ip6_fib.c
+Auto-merging net/ipv6/anycast.c
+Auto-merging net/ipv6/af_inet6.c
+Auto-merging net/ipv6/addrconf.c
+Auto-merging net/ipv4/udp.c
+Auto-merging net/ipv4/tcp_ipv4.c
+Auto-merging net/ipv4/tcp.c
+Auto-merging net/ipv4/route.c
+Auto-merging net/ipv4/proc.c
+Auto-merging net/ipv4/ipmr.c
+Auto-merging net/ipv4/ipconfig.c
+CONFLICT (content): Merge conflict in net/ipv4/ipconfig.c
+Auto-merging net/ipv4/fib_trie.c
+Auto-merging net/ipv4/af_inet.c
+Auto-merging net/core/sock.c
+Auto-merging net/core/neighbour.c
+Auto-merging net/core/dev.c
+Removing net/bridge/netfilter/nft_meta_bridge.c
+Auto-merging net/bridge/netfilter/ebtables.c
+Auto-merging init/Kconfig
+Auto-merging include/uapi/linux/pci_regs.h
+Removing include/trace/events/bpf.h
+Auto-merging include/net/udp.h
+Auto-merging include/net/tcp.h
+Auto-merging include/net/sock.h
+Auto-merging include/net/sctp/sctp.h
+Removing include/net/netfilter/nft_meta.h
+Auto-merging include/net/ip_vs.h
+Auto-merging include/net/ip6_fib.h
+Auto-merging include/linux/skbuff.h
+Removing include/linux/platform_data/mdio-gpio.h
+Auto-merging include/linux/net.h
+Auto-merging include/linux/mlx5/mlx5_ifc.h
+Auto-merging include/linux/mlx5/device.h
+Auto-merging drivers/vhost/net.c
+Auto-merging drivers/soc/ti/knav_qmss_queue.c
+Auto-merging drivers/soc/ti/knav_qmss.h
+Auto-merging drivers/net/wireless/ralink/rt2x00/rt2x00queue.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/pcie/trans.c
+Removing drivers/net/wireless/intel/iwlwifi/fw/nvm.c
+Auto-merging drivers/net/usb/qmi_wwan.c
+Auto-merging drivers/net/tun.c
+Auto-merging drivers/net/ethernet/socionext/netsec.c
+Auto-merging drivers/net/ethernet/sfc/efx.c
+Auto-merging drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+Auto-merging drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+Removing drivers/net/ethernet/freescale/gianfar_ptp.c
+Auto-merging drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+Auto-merging drivers/net/ethernet/broadcom/bnxt/bnxt.c
+Removing arch/x86/net/bpf_jit.S
+Auto-merging arch/x86/Kconfig
+Removing arch/sparc/net/bpf_jit_asm_64.S
+Removing arch/s390/net/bpf_jit.S
+Auto-merging arch/s390/net/Makefile
+CONFLICT (content): Merge conflict in arch/s390/net/Makefile
+Removing arch/powerpc/net/bpf_jit_asm64.S
+Auto-merging arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
+CONFLICT (content): Merge conflict in arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
+Auto-merging MAINTAINERS
+Removing Documentation/bpf/bpf_devel_QA.txt
+Removing Documentation/bpf/bpf_design_QA.txt
+Resolved 'arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi' using previous resolution.
+Resolved 'arch/s390/net/Makefile' using previous resolution.
+Resolved 'net/ipv4/ipconfig.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master f75ed129de31] Merge remote-tracking branch 'net-next/master'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/bpf/README.rst                       |   36 +
+ Documentation/bpf/bpf_design_QA.rst                |  221 ++
+ Documentation/bpf/bpf_design_QA.txt                |  156 --
+ Documentation/bpf/bpf_devel_QA.rst                 |  640 ++++++
+ Documentation/bpf/bpf_devel_QA.txt                 |  570 -----
+ .../devicetree/bindings/arm/stm32/stm32-syscon.txt |   14 +
+ .../devicetree/bindings/arm/{ => stm32}/stm32.txt  |    0
+ Documentation/devicetree/bindings/net/dsa/dsa.txt  |    6 -
+ .../devicetree/bindings/net/dsa/qca8k.txt          |   23 +-
+ .../devicetree/bindings/net/dwmac-sun8i.txt        |   21 +-
+ .../devicetree/bindings/net/fsl-tsec-phy.txt       |   68 +-
+ .../devicetree/bindings/net/meson-dwmac.txt        |    1 +
+ .../devicetree/bindings/net/microchip,lan78xx.txt  |   54 +
+ .../devicetree/bindings/net/mscc-miim.txt          |   26 +
+ .../devicetree/bindings/net/mscc-ocelot.txt        |   82 +
+ Documentation/devicetree/bindings/net/sff,sfp.txt  |    4 +-
+ Documentation/devicetree/bindings/net/sh_eth.txt   |    1 +
+ .../bindings/net/socionext,uniphier-ave4.txt       |   19 +-
+ .../devicetree/bindings/net/stm32-dwmac.txt        |   18 +-
+ .../bindings/net/wireless/qcom,ath10k.txt          |   31 +
+ .../devicetree/bindings/ptp/ptp-qoriq.txt          |   69 +
+ .../bindings/soc/ti/keystone-navigator-qmss.txt    |    9 +-
+ Documentation/filesystems/nfs/nfsroot.txt          |   70 +-
+ Documentation/networking/af_xdp.rst                |  297 +++
+ Documentation/networking/bonding.txt               |    2 +-
+ Documentation/networking/failover.rst              |   18 +
+ Documentation/networking/filter.txt                |   21 +-
+ Documentation/networking/index.rst                 |    1 +
+ Documentation/networking/ip-sysctl.txt             |   30 +-
+ Documentation/networking/net_failover.rst          |  116 +
+ Documentation/networking/netdev-features.txt       |    7 +
+ Documentation/sysctl/net.txt                       |    1 +
+ MAINTAINERS                                        |   41 +-
+ arch/arm/net/bpf_jit_32.c                          |   90 +-
+ arch/arm64/net/bpf_jit_comp.c                      |  180 +-
+ arch/mips/net/ebpf_jit.c                           |  130 --
+ arch/powerpc/net/Makefile                          |    2 +-
+ arch/powerpc/net/bpf_jit64.h                       |   37 +-
+ arch/powerpc/net/bpf_jit_asm64.S                   |  180 --
+ arch/powerpc/net/bpf_jit_comp64.c                  |  217 +-
+ arch/s390/net/Makefile                             |    2 +-
+ arch/s390/net/bpf_jit.S                            |  120 -
+ arch/s390/net/bpf_jit.h                            |   20 +-
+ arch/s390/net/bpf_jit_comp.c                       |  127 +-
+ arch/sparc/net/Makefile                            |    5 +-
+ arch/sparc/net/bpf_jit_64.h                        |   29 -
+ arch/sparc/net/bpf_jit_asm_64.S                    |  162 --
+ arch/sparc/net/bpf_jit_comp_64.c                   |   80 +-
+ arch/x86/Kconfig                                   |    2 +-
+ arch/x86/include/asm/nospec-branch.h               |   41 +-
+ arch/x86/net/Makefile                              |    7 +-
+ arch/x86/net/bpf_jit.S                             |  154 --
+ arch/x86/net/bpf_jit_comp.c                        |  343 +--
+ arch/x86/net/bpf_jit_comp32.c                      | 2419 ++++++++++++++++++++
+ drivers/bluetooth/Kconfig                          |    1 +
+ drivers/bluetooth/btbcm.c                          |  201 +-
+ drivers/bluetooth/btbcm.h                          |    5 +-
+ drivers/bluetooth/btqca.c                          |  104 +-
+ drivers/bluetooth/btqca.h                          |   11 +-
+ drivers/bluetooth/btqcomsmd.c                      |   10 +
+ drivers/bluetooth/btusb.c                          |    2 +
+ drivers/bluetooth/hci_bcm.c                        |   35 +-
+ drivers/bluetooth/hci_ldisc.c                      |    2 +
+ drivers/bluetooth/hci_qca.c                        |  116 +-
+ drivers/connector/cn_proc.c                        |    4 +
+ drivers/crypto/chelsio/chtls/chtls.h               |    1 +
+ drivers/crypto/chelsio/chtls/chtls_hw.c            |    6 +-
+ drivers/crypto/chelsio/chtls/chtls_io.c            |  104 +-
+ drivers/crypto/chelsio/chtls/chtls_main.c          |    3 +-
+ drivers/dca/dca-core.c                             |    2 +-
+ drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.c   |   21 +-
+ .../infiniband/ulp/opa_vnic/opa_vnic_internal.h    |    2 +-
+ drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c  |    2 +-
+ drivers/net/Kconfig                                |   14 +
+ drivers/net/Makefile                               |    1 +
+ drivers/net/bonding/bond_alb.c                     |   58 +-
+ drivers/net/bonding/bond_main.c                    |  117 +-
+ drivers/net/bonding/bond_options.c                 |    2 +-
+ drivers/net/dsa/b53/b53_common.c                   |   81 +-
+ drivers/net/dsa/b53/b53_priv.h                     |    6 +-
+ drivers/net/dsa/bcm_sf2.c                          |  200 +-
+ drivers/net/dsa/dsa_loop.c                         |   43 +-
+ drivers/net/dsa/lan9303-core.c                     |   11 +-
+ drivers/net/dsa/microchip/ksz_common.c             |   11 +-
+ drivers/net/dsa/mt7530.c                           |   11 +-
+ drivers/net/dsa/mv88e6xxx/chip.c                   |  388 +++-
+ drivers/net/dsa/mv88e6xxx/chip.h                   |   18 +-
+ drivers/net/dsa/mv88e6xxx/global1.c                |  109 +-
+ drivers/net/dsa/mv88e6xxx/global1.h                |   45 +-
+ drivers/net/dsa/mv88e6xxx/global2.c                |   65 +-
+ drivers/net/dsa/mv88e6xxx/global2.h                |   25 +-
+ drivers/net/dsa/mv88e6xxx/port.c                   |   39 +
+ drivers/net/dsa/mv88e6xxx/port.h                   |    3 +
+ drivers/net/dsa/mv88e6xxx/serdes.c                 |   20 +
+ drivers/net/dsa/mv88e6xxx/serdes.h                 |    3 +
+ drivers/net/dsa/qca8k.c                            |   74 +-
+ drivers/net/dsa/qca8k.h                            |    7 +-
+ drivers/net/ethernet/3com/3c59x.c                  |   62 +-
+ drivers/net/ethernet/8390/Kconfig                  |   17 +-
+ drivers/net/ethernet/8390/Makefile                 |    1 +
+ drivers/net/ethernet/8390/ax88796.c                |  228 +-
+ drivers/net/ethernet/8390/xsurf100.c               |  382 ++++
+ drivers/net/ethernet/Kconfig                       |   15 +-
+ drivers/net/ethernet/Makefile                      |    1 +
+ drivers/net/ethernet/amd/amd8111e.c                |   16 +-
+ drivers/net/ethernet/amd/xgbe/xgbe-drv.c           |  137 +-
+ drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c       |  217 ++
+ drivers/net/ethernet/amd/xgbe/xgbe-main.c          |   20 +-
+ drivers/net/ethernet/amd/xgbe/xgbe-mdio.c          |  167 +-
+ drivers/net/ethernet/amd/xgbe/xgbe-pci.c           |   36 +-
+ drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c        |  349 ++-
+ drivers/net/ethernet/amd/xgbe/xgbe.h               |   31 +
+ drivers/net/ethernet/broadcom/bcmsysport.c         |    6 +-
+ drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c    |    7 +-
+ drivers/net/ethernet/broadcom/bnxt/Makefile        |    1 +
+ drivers/net/ethernet/broadcom/bnxt/bnxt.c          |  164 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt.h          |   19 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c      |  166 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c  |  124 +
+ drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.h  |   23 +
+ drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c      |    8 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c  |   60 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c    |   22 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.h    |   17 +
+ drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c      |    2 +-
+ drivers/net/ethernet/broadcom/genet/bcmgenet.c     |    6 +-
+ drivers/net/ethernet/cadence/macb_main.c           |   12 +-
+ .../ethernet/cavium/liquidio/cn23xx_pf_device.c    |   60 +-
+ .../ethernet/cavium/liquidio/cn23xx_pf_device.h    |   14 +
+ drivers/net/ethernet/cavium/liquidio/lio_core.c    |  510 ++++-
+ drivers/net/ethernet/cavium/liquidio/lio_ethtool.c |  727 ++++--
+ drivers/net/ethernet/cavium/liquidio/lio_main.c    |  370 ++-
+ drivers/net/ethernet/cavium/liquidio/lio_vf_main.c |  259 +--
+ drivers/net/ethernet/cavium/liquidio/lio_vf_rep.c  |   15 +-
+ .../net/ethernet/cavium/liquidio/liquidio_common.h |   87 +-
+ .../net/ethernet/cavium/liquidio/octeon_device.c   |   12 +-
+ .../net/ethernet/cavium/liquidio/octeon_device.h   |   16 +-
+ drivers/net/ethernet/cavium/liquidio/octeon_iq.h   |    4 +-
+ .../net/ethernet/cavium/liquidio/octeon_mailbox.c  |   52 +
+ .../net/ethernet/cavium/liquidio/octeon_mailbox.h  |    7 +
+ .../net/ethernet/cavium/liquidio/octeon_network.h  |   79 +-
+ drivers/net/ethernet/cavium/thunder/nicvf_main.c   |    2 +-
+ drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h  |   20 +
+ drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h      |    1 +
+ drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c     |   87 +-
+ drivers/net/ethernet/chelsio/cxgb4/cxgb4.h         |   42 +-
+ drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c   |   28 +-
+ drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h   |    3 +
+ drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c |   22 +-
+ drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c  |   94 +-
+ drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c    |  125 +-
+ .../net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c   |   18 +
+ drivers/net/ethernet/chelsio/cxgb4/l2t.c           |    2 +-
+ drivers/net/ethernet/chelsio/cxgb4/sge.c           |  120 +-
+ drivers/net/ethernet/chelsio/cxgb4/srq.c           |    3 +-
+ drivers/net/ethernet/chelsio/cxgb4/t4_chip_type.h  |    2 +
+ drivers/net/ethernet/chelsio/cxgb4/t4_hw.c         |  188 +-
+ drivers/net/ethernet/chelsio/cxgb4/t4_msg.h        |    5 +
+ drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h |    6 +
+ drivers/net/ethernet/chelsio/cxgb4/t4_regs.h       |    5 +
+ drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h      |   44 +-
+ drivers/net/ethernet/chelsio/cxgb4/t4fw_version.h  |   12 +-
+ .../net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c    |   21 +-
+ drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h |    5 +-
+ drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c     |   73 +-
+ drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.h  |   15 +-
+ drivers/net/ethernet/ethoc.c                       |    6 +-
+ drivers/net/ethernet/freescale/Kconfig             |    2 +-
+ drivers/net/ethernet/freescale/Makefile            |    1 -
+ drivers/net/ethernet/freescale/fec.h               |    2 +-
+ drivers/net/ethernet/freescale/fec_main.c          |   12 +-
+ drivers/net/ethernet/freescale/fec_ptp.c           |    6 -
+ drivers/net/ethernet/freescale/fman/fman_port.c    |    8 +-
+ drivers/net/ethernet/freescale/gianfar.h           |    3 -
+ drivers/net/ethernet/freescale/gianfar_ethtool.c   |   23 +-
+ drivers/net/ethernet/freescale/gianfar_ptp.c       |  572 -----
+ drivers/net/ethernet/hisilicon/hns3/hnae3.c        |   88 +-
+ drivers/net/ethernet/hisilicon/hns3/hnae3.h        |   16 +-
+ drivers/net/ethernet/hisilicon/hns3/hns3_enet.c    |  505 ++--
+ drivers/net/ethernet/hisilicon/hns3/hns3_enet.h    |    6 +
+ drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c |   25 +-
+ .../net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c |   29 +-
+ .../net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h |   22 +-
+ .../ethernet/hisilicon/hns3/hns3pf/hclge_main.c    |  476 ++--
+ .../ethernet/hisilicon/hns3/hns3pf/hclge_main.h    |   31 +-
+ .../net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c |   35 +-
+ .../ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c    |    7 +-
+ .../net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c  |   50 +-
+ .../net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h  |    5 +
+ .../ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c  |   46 +-
+ .../ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h  |    2 +-
+ drivers/net/ethernet/huawei/hinic/Kconfig          |    2 +-
+ drivers/net/ethernet/huawei/hinic/hinic_main.c     |    8 +-
+ drivers/net/ethernet/ibm/ibmvnic.c                 |  223 +-
+ drivers/net/ethernet/ibm/ibmvnic.h                 |    2 +
+ drivers/net/ethernet/intel/e100.c                  |   28 +-
+ drivers/net/ethernet/intel/e1000/Makefile          |   26 -
+ drivers/net/ethernet/intel/e1000/e1000.h           |   29 +-
+ drivers/net/ethernet/intel/e1000/e1000_ethtool.c   |   23 +-
+ drivers/net/ethernet/intel/e1000/e1000_hw.c        |   28 +-
+ drivers/net/ethernet/intel/e1000/e1000_hw.h        |   28 +-
+ drivers/net/ethernet/intel/e1000/e1000_main.c      |   28 +-
+ drivers/net/ethernet/intel/e1000/e1000_osdep.h     |   29 +-
+ drivers/net/ethernet/intel/e1000/e1000_param.c     |   28 +-
+ drivers/net/ethernet/intel/e1000e/80003es2lan.c    |   21 +-
+ drivers/net/ethernet/intel/e1000e/80003es2lan.h    |   21 +-
+ drivers/net/ethernet/intel/e1000e/82571.c          |   21 +-
+ drivers/net/ethernet/intel/e1000e/82571.h          |   21 +-
+ drivers/net/ethernet/intel/e1000e/Makefile         |   27 +-
+ drivers/net/ethernet/intel/e1000e/defines.h        |   21 +-
+ drivers/net/ethernet/intel/e1000e/e1000.h          |   21 +-
+ drivers/net/ethernet/intel/e1000e/ethtool.c        |   21 +-
+ drivers/net/ethernet/intel/e1000e/hw.h             |   21 +-
+ drivers/net/ethernet/intel/e1000e/ich8lan.c        |   21 +-
+ drivers/net/ethernet/intel/e1000e/ich8lan.h        |   21 +-
+ drivers/net/ethernet/intel/e1000e/mac.c            |   21 +-
+ drivers/net/ethernet/intel/e1000e/mac.h            |   21 +-
+ drivers/net/ethernet/intel/e1000e/manage.c         |   21 +-
+ drivers/net/ethernet/intel/e1000e/manage.h         |   21 +-
+ drivers/net/ethernet/intel/e1000e/netdev.c         |   21 +-
+ drivers/net/ethernet/intel/e1000e/nvm.c            |   21 +-
+ drivers/net/ethernet/intel/e1000e/nvm.h            |   21 +-
+ drivers/net/ethernet/intel/e1000e/param.c          |   21 +-
+ drivers/net/ethernet/intel/e1000e/phy.c            |   21 +-
+ drivers/net/ethernet/intel/e1000e/phy.h            |   21 +-
+ drivers/net/ethernet/intel/e1000e/ptp.c            |   21 +-
+ drivers/net/ethernet/intel/e1000e/regs.h           |   21 +-
+ drivers/net/ethernet/intel/fm10k/Makefile          |   23 +-
+ drivers/net/ethernet/intel/fm10k/fm10k.h           |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_common.c    |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_common.h    |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_dcbnl.c     |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_debugfs.c   |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c   |  136 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_iov.c       |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_main.c      |   27 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_mbx.c       |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_mbx.h       |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_netdev.c    |   94 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_pci.c       |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_pf.c        |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_pf.h        |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_tlv.c       |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_tlv.h       |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_type.h      |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_vf.c        |   20 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_vf.h        |   20 +-
+ drivers/net/ethernet/intel/i40e/Makefile           |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e.h             |   33 +-
+ drivers/net/ethernet/intel/i40e/i40e_adminq.c      |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_adminq.h      |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h  |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_alloc.h       |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_client.c      |   32 +-
+ drivers/net/ethernet/intel/i40e/i40e_client.h      |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_common.c      |   63 +-
+ drivers/net/ethernet/intel/i40e/i40e_dcb.c         |  117 +-
+ drivers/net/ethernet/intel/i40e/i40e_dcb.h         |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_dcb_nl.c      |   37 +-
+ drivers/net/ethernet/intel/i40e/i40e_debugfs.c     |   34 +-
+ drivers/net/ethernet/intel/i40e/i40e_devids.h      |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_diag.c        |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_diag.h        |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_ethtool.c     |  542 +++--
+ drivers/net/ethernet/intel/i40e/i40e_hmc.c         |   27 +-
+ drivers/net/ethernet/intel/i40e/i40e_hmc.h         |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_lan_hmc.c     |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_lan_hmc.h     |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_main.c        |  196 +-
+ drivers/net/ethernet/intel/i40e/i40e_nvm.c         |   27 +-
+ drivers/net/ethernet/intel/i40e/i40e_osdep.h       |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_prototype.h   |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_ptp.c         |   84 +-
+ drivers/net/ethernet/intel/i40e/i40e_register.h    |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_status.h      |   26 +-
+ drivers/net/ethernet/intel/i40e/i40e_trace.h       |   23 +-
+ drivers/net/ethernet/intel/i40e/i40e_txrx.c        |   87 +-
+ drivers/net/ethernet/intel/i40e/i40e_txrx.h        |   29 +-
+ drivers/net/ethernet/intel/i40e/i40e_type.h        |   34 +-
+ drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c |  111 +-
+ drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h |   26 +-
+ drivers/net/ethernet/intel/i40evf/Makefile         |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_adminq.c    |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_adminq.h    |   26 +-
+ .../net/ethernet/intel/i40evf/i40e_adminq_cmd.h    |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_alloc.h     |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_common.c    |   27 +-
+ drivers/net/ethernet/intel/i40evf/i40e_devids.h    |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_hmc.h       |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_lan_hmc.h   |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_osdep.h     |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_prototype.h |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_register.h  |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_status.h    |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_trace.h     |   23 +-
+ drivers/net/ethernet/intel/i40evf/i40e_txrx.c      |   30 +-
+ drivers/net/ethernet/intel/i40evf/i40e_txrx.h      |   26 +-
+ drivers/net/ethernet/intel/i40evf/i40e_type.h      |   36 +-
+ drivers/net/ethernet/intel/i40evf/i40evf.h         |   27 +-
+ drivers/net/ethernet/intel/i40evf/i40evf_client.c  |    6 +-
+ drivers/net/ethernet/intel/i40evf/i40evf_client.h  |    8 +-
+ drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c |   33 +-
+ drivers/net/ethernet/intel/i40evf/i40evf_main.c    |   59 +-
+ .../net/ethernet/intel/i40evf/i40evf_virtchnl.c    |   37 +-
+ drivers/net/ethernet/intel/ice/ice_adminq_cmd.h    |   13 +-
+ drivers/net/ethernet/intel/ice/ice_nvm.c           |    7 +-
+ drivers/net/ethernet/intel/igb/Makefile            |   28 +-
+ drivers/net/ethernet/intel/igb/e1000_82575.c       |   23 +-
+ drivers/net/ethernet/intel/igb/e1000_82575.h       |   23 +-
+ drivers/net/ethernet/intel/igb/e1000_defines.h     |   25 +-
+ drivers/net/ethernet/intel/igb/e1000_hw.h          |   22 +-
+ drivers/net/ethernet/intel/igb/e1000_i210.c        |   23 +-
+ drivers/net/ethernet/intel/igb/e1000_i210.h        |   23 +-
+ drivers/net/ethernet/intel/igb/e1000_mac.c         |   23 +-
+ drivers/net/ethernet/intel/igb/e1000_mac.h         |   23 +-
+ drivers/net/ethernet/intel/igb/e1000_mbx.c         |   23 +-
+ drivers/net/ethernet/intel/igb/e1000_mbx.h         |   23 +-
+ drivers/net/ethernet/intel/igb/e1000_nvm.c         |   22 +-
+ drivers/net/ethernet/intel/igb/e1000_nvm.h         |   23 +-
+ drivers/net/ethernet/intel/igb/e1000_phy.c         |   23 +-
+ drivers/net/ethernet/intel/igb/e1000_phy.h         |   23 +-
+ drivers/net/ethernet/intel/igb/e1000_regs.h        |   23 +-
+ drivers/net/ethernet/intel/igb/igb.h               |   36 +-
+ drivers/net/ethernet/intel/igb/igb_ethtool.c       |   96 +-
+ drivers/net/ethernet/intel/igb/igb_hwmon.c         |   23 +-
+ drivers/net/ethernet/intel/igb/igb_main.c          |  389 +++-
+ drivers/net/ethernet/intel/igb/igb_ptp.c           |   19 +-
+ drivers/net/ethernet/intel/igbvf/Makefile          |   28 +-
+ drivers/net/ethernet/intel/igbvf/defines.h         |   26 +-
+ drivers/net/ethernet/intel/igbvf/ethtool.c         |   26 +-
+ drivers/net/ethernet/intel/igbvf/igbvf.h           |   26 +-
+ drivers/net/ethernet/intel/igbvf/mbx.c             |   26 +-
+ drivers/net/ethernet/intel/igbvf/mbx.h             |   26 +-
+ drivers/net/ethernet/intel/igbvf/netdev.c          |   26 +-
+ drivers/net/ethernet/intel/igbvf/regs.h            |   26 +-
+ drivers/net/ethernet/intel/igbvf/vf.c              |   26 +-
+ drivers/net/ethernet/intel/igbvf/vf.h              |   26 +-
+ drivers/net/ethernet/intel/ixgb/Makefile           |   27 -
+ drivers/net/ethernet/intel/ixgb/ixgb.h             |   28 +-
+ drivers/net/ethernet/intel/ixgb/ixgb_ee.c          |   29 +-
+ drivers/net/ethernet/intel/ixgb/ixgb_ee.h          |   28 +-
+ drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c     |   29 +-
+ drivers/net/ethernet/intel/ixgb/ixgb_hw.c          |   29 +-
+ drivers/net/ethernet/intel/ixgb/ixgb_hw.h          |   28 +-
+ drivers/net/ethernet/intel/ixgb/ixgb_ids.h         |   28 +-
+ drivers/net/ethernet/intel/ixgb/ixgb_main.c        |   29 +-
+ drivers/net/ethernet/intel/ixgb/ixgb_osdep.h       |   28 +-
+ drivers/net/ethernet/intel/ixgb/ixgb_param.c       |   29 +-
+ drivers/net/ethernet/intel/ixgbe/Makefile          |   29 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe.h           |   32 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_82598.c     |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c     |   41 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_common.c    |   30 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_common.h    |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.c       |   30 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.h       |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.c |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.h |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.c |   29 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.h |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c    |   29 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c   |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c   |   29 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c      |   31 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.h      |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c     |   53 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.h     |   27 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c       |   29 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_main.c      |  425 ++--
+ drivers/net/ethernet/intel/ixgbe/ixgbe_mbx.c       |   29 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_mbx.h       |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_model.h     |   42 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c       |   29 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_phy.h       |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c       |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c     |   39 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h     |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_sysfs.c     |   29 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_type.h      |   28 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c      |   29 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_x540.h      |   24 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c      |   35 +-
+ drivers/net/ethernet/intel/ixgbevf/Makefile        |   28 +-
+ drivers/net/ethernet/intel/ixgbevf/defines.h       |   26 +-
+ drivers/net/ethernet/intel/ixgbevf/ethtool.c       |   27 +-
+ drivers/net/ethernet/intel/ixgbevf/ixgbevf.h       |   26 +-
+ drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c  |   34 +-
+ drivers/net/ethernet/intel/ixgbevf/mbx.c           |   27 +-
+ drivers/net/ethernet/intel/ixgbevf/mbx.h           |   26 +-
+ drivers/net/ethernet/intel/ixgbevf/regs.h          |   26 +-
+ drivers/net/ethernet/intel/ixgbevf/vf.c            |   27 +-
+ drivers/net/ethernet/intel/ixgbevf/vf.h            |   26 +-
+ drivers/net/ethernet/marvell/Kconfig               |    1 +
+ drivers/net/ethernet/marvell/mvmdio.c              |    5 +-
+ drivers/net/ethernet/marvell/mvpp2.c               | 1005 ++++----
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c        |    4 +-
+ drivers/net/ethernet/mellanox/mlx4/en_rx.c         |   23 +-
+ drivers/net/ethernet/mellanox/mlx4/en_tx.c         |    2 +-
+ drivers/net/ethernet/mellanox/mlx4/fw.c            |   12 +
+ drivers/net/ethernet/mellanox/mlx4/main.c          |   56 +-
+ drivers/net/ethernet/mellanox/mlx4/mlx4.h          |    2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/Kconfig    |   13 +
+ drivers/net/ethernet/mellanox/mlx5/core/Makefile   |   10 +-
+ .../net/ethernet/mellanox/mlx5/core/accel/tls.c    |   71 +
+ .../net/ethernet/mellanox/mlx5/core/accel/tls.h    |   86 +
+ drivers/net/ethernet/mellanox/mlx5/core/en.h       |   67 +-
+ .../net/ethernet/mellanox/mlx5/core/en/Makefile    |    1 +
+ drivers/net/ethernet/mellanox/mlx5/core/en/port.c  |  237 ++
+ drivers/net/ethernet/mellanox/mlx5/core/en/port.h  |   48 +
+ .../ethernet/mellanox/mlx5/core/en/port_buffer.c   |  327 +++
+ .../ethernet/mellanox/mlx5/core/en/port_buffer.h   |   75 +
+ .../mellanox/mlx5/core/en_accel/en_accel.h         |   72 +
+ .../ethernet/mellanox/mlx5/core/en_accel/ipsec.h   |    3 -
+ .../net/ethernet/mellanox/mlx5/core/en_accel/tls.c |  197 ++
+ .../net/ethernet/mellanox/mlx5/core/en_accel/tls.h |   87 +
+ .../mellanox/mlx5/core/en_accel/tls_rxtx.c         |  278 +++
+ .../mellanox/mlx5/core/en_accel/tls_rxtx.h         |   50 +
+ .../mellanox/mlx5/core/en_accel/tls_stats.c        |   89 +
+ drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c |  132 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_dim.c   |   28 +-
+ .../net/ethernet/mellanox/mlx5/core/en_ethtool.c   |  137 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_fs.c    |    1 -
+ drivers/net/ethernet/mellanox/mlx5/core/en_main.c  |  258 ++-
+ drivers/net/ethernet/mellanox/mlx5/core/en_rep.c   |  114 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_rep.h   |    1 +
+ drivers/net/ethernet/mellanox/mlx5/core/en_rx.c    |  187 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_stats.c |  133 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_stats.h |   12 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_tc.c    |  452 ++--
+ drivers/net/ethernet/mellanox/mlx5/core/en_tc.h    |   24 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_tx.c    |  301 ++-
+ drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c  |   39 +-
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.c  |   22 +-
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.h  |   31 +-
+ .../ethernet/mellanox/mlx5/core/eswitch_offloads.c |  141 +-
+ .../net/ethernet/mellanox/mlx5/core/fpga/conn.c    |   14 +-
+ .../net/ethernet/mellanox/mlx5/core/fpga/conn.h    |    2 +-
+ .../net/ethernet/mellanox/mlx5/core/fpga/core.h    |    1 +
+ .../net/ethernet/mellanox/mlx5/core/fpga/ipsec.c   |    5 +-
+ drivers/net/ethernet/mellanox/mlx5/core/fpga/sdk.h |    2 +
+ drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.c |  562 +++++
+ drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.h |   68 +
+ drivers/net/ethernet/mellanox/mlx5/core/fs_core.c  |    2 +-
+ .../net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h  |   26 +
+ drivers/net/ethernet/mellanox/mlx5/core/main.c     |   11 +
+ drivers/net/ethernet/mellanox/mlx5/core/wq.c       |   94 +-
+ drivers/net/ethernet/mellanox/mlx5/core/wq.h       |   50 +-
+ drivers/net/ethernet/mellanox/mlxsw/cmd.h          |   47 +-
+ drivers/net/ethernet/mellanox/mlxsw/core.c         |   48 +-
+ drivers/net/ethernet/mellanox/mlxsw/core.h         |    7 +-
+ drivers/net/ethernet/mellanox/mlxsw/pci.c          |  278 ++-
+ drivers/net/ethernet/mellanox/mlxsw/pci_hw.h       |   74 +-
+ drivers/net/ethernet/mellanox/mlxsw/reg.h          |   31 +
+ drivers/net/ethernet/mellanox/mlxsw/resources.h    |    6 +
+ drivers/net/ethernet/mellanox/mlxsw/spectrum.c     |   71 +-
+ drivers/net/ethernet/mellanox/mlxsw/spectrum.h     |    1 +
+ .../net/ethernet/mellanox/mlxsw/spectrum_router.c  |  175 +-
+ .../net/ethernet/mellanox/mlxsw/spectrum_span.c    |  205 +-
+ .../net/ethernet/mellanox/mlxsw/spectrum_span.h    |    1 +
+ .../ethernet/mellanox/mlxsw/spectrum_switchdev.c   |   76 +-
+ .../ethernet/mellanox/mlxsw/spectrum_switchdev.h   |   43 +
+ drivers/net/ethernet/mellanox/mlxsw/switchx2.c     |   11 +-
+ drivers/net/ethernet/mscc/Kconfig                  |   30 +
+ drivers/net/ethernet/mscc/Makefile                 |    5 +
+ drivers/net/ethernet/mscc/ocelot.c                 | 1333 +++++++++++
+ drivers/net/ethernet/mscc/ocelot.h                 |  572 +++++
+ drivers/net/ethernet/mscc/ocelot_ana.h             |  625 +++++
+ drivers/net/ethernet/mscc/ocelot_board.c           |  316 +++
+ drivers/net/ethernet/mscc/ocelot_dev.h             |  275 +++
+ drivers/net/ethernet/mscc/ocelot_dev_gmii.h        |  154 ++
+ drivers/net/ethernet/mscc/ocelot_hsio.h            |  785 +++++++
+ drivers/net/ethernet/mscc/ocelot_io.c              |  116 +
+ drivers/net/ethernet/mscc/ocelot_qs.h              |   78 +
+ drivers/net/ethernet/mscc/ocelot_qsys.h            |  270 +++
+ drivers/net/ethernet/mscc/ocelot_regs.c            |  497 ++++
+ drivers/net/ethernet/mscc/ocelot_rew.h             |   81 +
+ drivers/net/ethernet/mscc/ocelot_sys.h             |  144 ++
+ drivers/net/ethernet/neterion/vxge/vxge-config.c   |   12 +-
+ drivers/net/ethernet/neterion/vxge/vxge-config.h   |    2 +-
+ drivers/net/ethernet/neterion/vxge/vxge-ethtool.c  |    2 +-
+ drivers/net/ethernet/neterion/vxge/vxge-main.c     |    4 +-
+ drivers/net/ethernet/netronome/Kconfig             |   13 +
+ drivers/net/ethernet/netronome/nfp/Makefile        |    8 +
+ drivers/net/ethernet/netronome/nfp/abm/ctrl.c      |  333 +++
+ drivers/net/ethernet/netronome/nfp/abm/main.c      |  765 +++++++
+ drivers/net/ethernet/netronome/nfp/abm/main.h      |  142 ++
+ drivers/net/ethernet/netronome/nfp/bpf/cmsg.c      |   16 +-
+ drivers/net/ethernet/netronome/nfp/bpf/fw.h        |   21 +-
+ drivers/net/ethernet/netronome/nfp/bpf/jit.c       |  764 +++++--
+ drivers/net/ethernet/netronome/nfp/bpf/main.c      |   39 +-
+ drivers/net/ethernet/netronome/nfp/bpf/main.h      |   66 +-
+ drivers/net/ethernet/netronome/nfp/bpf/offload.c   |  174 +-
+ drivers/net/ethernet/netronome/nfp/bpf/verifier.c  |  114 +-
+ drivers/net/ethernet/netronome/nfp/flower/action.c |  131 +-
+ drivers/net/ethernet/netronome/nfp/flower/cmsg.c   |   11 +-
+ drivers/net/ethernet/netronome/nfp/flower/cmsg.h   |   14 +
+ .../net/ethernet/netronome/nfp/flower/lag_conf.c   |  726 ++++++
+ drivers/net/ethernet/netronome/nfp/flower/main.c   |   74 +-
+ drivers/net/ethernet/netronome/nfp/flower/main.h   |   61 +-
+ .../net/ethernet/netronome/nfp/flower/metadata.c   |   20 +-
+ .../net/ethernet/netronome/nfp/flower/offload.c    |   52 +-
+ drivers/net/ethernet/netronome/nfp/nfp_abi.h       |  143 ++
+ drivers/net/ethernet/netronome/nfp/nfp_app.c       |   27 +-
+ drivers/net/ethernet/netronome/nfp/nfp_app.h       |   26 +
+ drivers/net/ethernet/netronome/nfp/nfp_app_nic.c   |    5 +-
+ drivers/net/ethernet/netronome/nfp/nfp_asm.h       |   40 +-
+ drivers/net/ethernet/netronome/nfp/nfp_devlink.c   |   40 +-
+ drivers/net/ethernet/netronome/nfp/nfp_main.c      |  145 ++
+ drivers/net/ethernet/netronome/nfp/nfp_main.h      |   27 +-
+ drivers/net/ethernet/netronome/nfp/nfp_net.h       |    7 +-
+ .../net/ethernet/netronome/nfp/nfp_net_common.c    |   23 +-
+ .../net/ethernet/netronome/nfp/nfp_net_debugfs.c   |    4 +-
+ .../net/ethernet/netronome/nfp/nfp_net_ethtool.c   |   10 +-
+ drivers/net/ethernet/netronome/nfp/nfp_net_main.c  |   72 +-
+ drivers/net/ethernet/netronome/nfp/nfp_net_repr.c  |   17 +-
+ drivers/net/ethernet/netronome/nfp/nfp_net_repr.h  |   11 +-
+ .../net/ethernet/netronome/nfp/nfp_netvf_main.c    |    2 +-
+ drivers/net/ethernet/netronome/nfp/nfp_port.c      |    8 +-
+ drivers/net/ethernet/netronome/nfp/nfp_port.h      |    8 +
+ .../net/ethernet/netronome/nfp/nfp_shared_buf.c    |  180 ++
+ drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h   |    2 +
+ .../ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c  |   95 +-
+ .../net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h   |    7 +
+ .../net/ethernet/netronome/nfp/nfpcore/nfp_mutex.c |   45 +
+ .../net/ethernet/netronome/nfp/nfpcore/nfp_nffw.h  |    2 +
+ .../ethernet/netronome/nfp/nfpcore/nfp_resource.c  |   59 +
+ .../net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c |   43 +
+ drivers/net/ethernet/qlogic/qed/Makefile           |    2 +-
+ drivers/net/ethernet/qlogic/qed/qed.h              |   74 +-
+ drivers/net/ethernet/qlogic/qed/qed_dcbx.c         |   14 +-
+ drivers/net/ethernet/qlogic/qed/qed_debug.c        |  118 +-
+ drivers/net/ethernet/qlogic/qed/qed_dev.c          |  113 +-
+ drivers/net/ethernet/qlogic/qed/qed_fcoe.c         |    3 +
+ drivers/net/ethernet/qlogic/qed/qed_hsi.h          |  259 +++
+ drivers/net/ethernet/qlogic/qed/qed_l2.c           |   56 +-
+ drivers/net/ethernet/qlogic/qed/qed_ll2.c          |   46 +-
+ drivers/net/ethernet/qlogic/qed/qed_main.c         |  168 +-
+ drivers/net/ethernet/qlogic/qed/qed_mcp.c          |   98 +-
+ drivers/net/ethernet/qlogic/qed/qed_mcp.h          |   81 +-
+ drivers/net/ethernet/qlogic/qed/qed_mng_tlv.c      | 1337 +++++++++++
+ drivers/net/ethernet/qlogic/qed/qed_sp.h           |   12 +-
+ drivers/net/ethernet/qlogic/qed/qed_sp_commands.c  |   76 +-
+ drivers/net/ethernet/qlogic/qed/qed_sriov.c        |  247 +-
+ drivers/net/ethernet/qlogic/qed/qed_vf.c           |   29 +
+ drivers/net/ethernet/qlogic/qed/qed_vf.h           |   21 +
+ drivers/net/ethernet/qlogic/qede/qede.h            |    6 +-
+ drivers/net/ethernet/qlogic/qede/qede_ethtool.c    |    4 +-
+ drivers/net/ethernet/qlogic/qede/qede_filter.c     |  663 ++++--
+ drivers/net/ethernet/qlogic/qede/qede_fp.c         |  227 +-
+ drivers/net/ethernet/qlogic/qede/qede_main.c       |  182 +-
+ drivers/net/ethernet/qualcomm/emac/emac-mac.c      |   11 +-
+ drivers/net/ethernet/qualcomm/emac/emac-sgmii.c    |  140 +-
+ drivers/net/ethernet/qualcomm/emac/emac-sgmii.h    |   32 +-
+ drivers/net/ethernet/qualcomm/emac/emac.c          |    9 +-
+ drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h |   13 +
+ .../net/ethernet/qualcomm/rmnet/rmnet_handlers.c   |   21 +-
+ .../ethernet/qualcomm/rmnet/rmnet_map_command.c    |   14 +-
+ .../net/ethernet/qualcomm/rmnet/rmnet_map_data.c   |   64 +-
+ drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c    |   51 +
+ drivers/net/ethernet/realtek/8139too.c             |    2 -
+ drivers/net/ethernet/realtek/r8169.c               | 1144 +++------
+ drivers/net/ethernet/renesas/sh_eth.c              |  107 +-
+ drivers/net/ethernet/renesas/sh_eth.h              |   28 +-
+ drivers/net/ethernet/rocker/rocker_main.c          |    4 +
+ drivers/net/ethernet/sfc/efx.c                     |   36 +
+ drivers/net/ethernet/sfc/tx.c                      |   33 +-
+ drivers/net/ethernet/socionext/Kconfig             |    2 +
+ drivers/net/ethernet/socionext/netsec.c            |   27 +-
+ drivers/net/ethernet/socionext/sni_ave.c           |  252 +-
+ drivers/net/ethernet/stmicro/stmmac/Makefile       |    3 +-
+ drivers/net/ethernet/stmicro/stmmac/chain_mode.c   |   34 +-
+ drivers/net/ethernet/stmicro/stmmac/common.h       |  234 +-
+ .../net/ethernet/stmicro/stmmac/dwmac-meson8b.c    |  120 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c     |   24 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c  |  267 ++-
+ drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c  |  221 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac1000.h    |    1 -
+ .../net/ethernet/stmicro/stmmac/dwmac1000_core.c   |   29 +-
+ .../net/ethernet/stmicro/stmmac/dwmac1000_dma.c    |   92 +-
+ .../net/ethernet/stmicro/stmmac/dwmac100_core.c    |   23 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c |   35 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac4.h       |    5 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c  |   42 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c |   38 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c   |   10 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h   |    1 -
+ drivers/net/ethernet/stmicro/stmmac/dwmac5.c       |  214 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac5.h       |   19 +-
+ drivers/net/ethernet/stmicro/stmmac/enh_desc.c     |   24 +-
+ drivers/net/ethernet/stmicro/stmmac/hwif.c         |  268 +++
+ drivers/net/ethernet/stmicro/stmmac/hwif.h         |  470 ++++
+ drivers/net/ethernet/stmicro/stmmac/norm_desc.c    |   24 +-
+ drivers/net/ethernet/stmicro/stmmac/ring_mode.c    |   39 +-
+ drivers/net/ethernet/stmicro/stmmac/stmmac.h       |   33 +-
+ .../net/ethernet/stmicro/stmmac/stmmac_ethtool.c   |   94 +-
+ .../net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c  |   34 +-
+ drivers/net/ethernet/stmicro/stmmac/stmmac_main.c  |  730 +++---
+ drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c  |    5 +-
+ .../net/ethernet/stmicro/stmmac/stmmac_platform.c  |    3 +-
+ drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c   |   18 +-
+ drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c    |  295 +++
+ drivers/net/ethernet/ti/Kconfig                    |   10 +-
+ drivers/net/ethernet/ti/cpsw-phy-sel.c             |    8 +-
+ drivers/net/ethernet/ti/cpsw.c                     |  113 +-
+ drivers/net/ethernet/ti/cpts.c                     |    5 +-
+ drivers/net/ethernet/ti/davinci_cpdma.c            |   10 +-
+ drivers/net/ethernet/ti/davinci_emac.c             |    4 +-
+ drivers/net/ethernet/ti/davinci_mdio.c             |   27 +-
+ drivers/net/ethernet/ti/netcp.h                    |    3 +
+ drivers/net/ethernet/ti/netcp_core.c               |   32 +
+ drivers/net/ethernet/ti/netcp_ethss.c              |  181 +-
+ drivers/net/geneve.c                               |   72 +-
+ drivers/net/hamradio/mkiss.c                       |    2 +-
+ drivers/net/hippi/rrunner.c                        |    2 +-
+ drivers/net/hyperv/Kconfig                         |    2 +
+ drivers/net/hyperv/hyperv_net.h                    |  168 +-
+ drivers/net/hyperv/netvsc.c                        |   61 +-
+ drivers/net/hyperv/netvsc_drv.c                    |  238 +-
+ drivers/net/hyperv/rndis_filter.c                  |   35 +-
+ drivers/net/ifb.c                                  |    2 +-
+ drivers/net/macvlan.c                              |   68 +-
+ drivers/net/net_failover.c                         |  836 +++++++
+ drivers/net/phy/Kconfig                            |   27 +-
+ drivers/net/phy/Makefile                           |    4 +
+ drivers/net/phy/asix.c                             |   63 +
+ drivers/net/phy/bcm-phy-lib.c                      |    6 +-
+ drivers/net/phy/dp83tc811.c                        |  347 +++
+ drivers/net/phy/marvell.c                          |    5 +-
+ drivers/net/phy/mdio-bitbang.c                     |    9 -
+ drivers/net/phy/mdio-boardinfo.c                   |    5 +-
+ drivers/net/phy/mdio-gpio.c                        |  128 +-
+ drivers/net/phy/mdio-mscc-miim.c                   |  193 ++
+ drivers/net/phy/micrel.c                           |    5 +-
+ drivers/net/phy/microchip.c                        |   25 +
+ drivers/net/phy/microchip_t1.c                     |   74 +
+ drivers/net/phy/phylink.c                          |   20 +-
+ drivers/net/phy/realtek.c                          |   16 +
+ drivers/net/phy/sfp-bus.c                          |    7 +
+ drivers/net/phy/sfp.c                              |   37 +-
+ drivers/net/phy/smsc.c                             |    5 +-
+ drivers/net/team/team.c                            |   10 +-
+ drivers/net/tun.c                                  |  127 +-
+ drivers/net/usb/Kconfig                            |    1 +
+ drivers/net/usb/lan78xx.c                          |  235 +-
+ drivers/net/usb/qmi_wwan.c                         |    2 +-
+ drivers/net/virtio_net.c                           |  158 +-
+ drivers/net/vmxnet3/vmxnet3_drv.c                  |    6 +-
+ drivers/net/vmxnet3/vmxnet3_ethtool.c              |    2 +-
+ drivers/net/vrf.c                                  |   27 +-
+ drivers/net/vxlan.c                                |   17 +-
+ drivers/net/wireless/ath/ath10k/Kconfig            |   12 +
+ drivers/net/wireless/ath/ath10k/Makefile           |    7 +-
+ drivers/net/wireless/ath/ath10k/ce.c               |  269 ++-
+ drivers/net/wireless/ath/ath10k/ce.h               |   24 +-
+ drivers/net/wireless/ath/ath10k/core.c             |  183 +-
+ drivers/net/wireless/ath/ath10k/core.h             |    2 +
+ drivers/net/wireless/ath/ath10k/hif.h              |   15 +-
+ drivers/net/wireless/ath/ath10k/htc.c              |    6 +
+ drivers/net/wireless/ath/ath10k/htc.h              |    4 +
+ drivers/net/wireless/ath/ath10k/htt.c              |    4 +-
+ drivers/net/wireless/ath/ath10k/htt.h              |  111 +-
+ drivers/net/wireless/ath/ath10k/htt_rx.c           |   24 +-
+ drivers/net/wireless/ath/ath10k/htt_tx.c           |   19 +-
+ drivers/net/wireless/ath/ath10k/hw.c               |    9 +-
+ drivers/net/wireless/ath/ath10k/hw.h               |   22 +-
+ drivers/net/wireless/ath/ath10k/mac.c              |   26 +-
+ drivers/net/wireless/ath/ath10k/pci.c              |    8 +-
+ drivers/net/wireless/ath/ath10k/sdio.c             |   24 +-
+ drivers/net/wireless/ath/ath10k/snoc.c             | 1414 ++++++++++++
+ drivers/net/wireless/ath/ath10k/snoc.h             |   95 +
+ drivers/net/wireless/ath/ath10k/txrx.c             |    8 +
+ drivers/net/wireless/ath/ath10k/wmi-ops.h          |   46 +
+ drivers/net/wireless/ath/ath10k/wmi-tlv.c          |   63 +
+ drivers/net/wireless/ath/ath10k/wmi-tlv.h          |  357 ++-
+ drivers/net/wireless/ath/ath10k/wmi.c              |   34 +-
+ drivers/net/wireless/ath/ath10k/wmi.h              |   28 +
+ drivers/net/wireless/ath/ath10k/wow.c              |  138 +-
+ drivers/net/wireless/ath/ath6kl/debug.c            |    2 +-
+ drivers/net/wireless/ath/ath6kl/main.c             |   14 +-
+ drivers/net/wireless/ath/ath9k/dfs.c               |    6 +-
+ drivers/net/wireless/ath/ath9k/main.c              |    3 +-
+ drivers/net/wireless/ath/wcn36xx/dxe.c             |   36 +-
+ drivers/net/wireless/ath/wcn36xx/dxe.h             |    1 -
+ drivers/net/wireless/ath/wcn36xx/hal.h             |    8 +-
+ drivers/net/wireless/ath/wcn36xx/main.c            |   40 +-
+ drivers/net/wireless/ath/wcn36xx/smd.c             |   33 +-
+ drivers/net/wireless/ath/wcn36xx/smd.h             |    2 +
+ drivers/net/wireless/ath/wcn36xx/txrx.c            |   15 +-
+ drivers/net/wireless/ath/wcn36xx/wcn36xx.h         |    7 +-
+ drivers/net/wireless/ath/wil6210/debugfs.c         |   22 +-
+ drivers/net/wireless/ath/wil6210/main.c            |    3 +-
+ drivers/net/wireless/ath/wil6210/wmi.c             |   19 +-
+ drivers/net/wireless/broadcom/b43/dma.c            |   14 +-
+ drivers/net/wireless/broadcom/b43legacy/dma.c      |    2 +-
+ .../wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c  |    1 +
+ .../broadcom/brcm80211/brcmfmac/cfg80211.c         |   51 +-
+ .../wireless/broadcom/brcm80211/brcmfmac/chip.c    |    1 +
+ .../broadcom/brcm80211/brcmfmac/firmware.c         |    2 +-
+ .../broadcom/brcm80211/brcmfmac/firmware.h         |    2 +-
+ .../wireless/broadcom/brcm80211/brcmfmac/msgbuf.h  |    6 +-
+ .../net/wireless/broadcom/brcm80211/brcmfmac/p2p.c |    7 +
+ .../wireless/broadcom/brcm80211/brcmfmac/pcie.c    |   40 +-
+ .../wireless/broadcom/brcm80211/brcmfmac/sdio.c    |    2 +-
+ .../net/wireless/broadcom/brcm80211/brcmfmac/usb.c |    2 +-
+ .../broadcom/brcm80211/brcmsmac/phy/phy_lcn.c      |    9 +-
+ .../broadcom/brcm80211/include/brcm_hw_ids.h       |    1 +
+ drivers/net/wireless/intel/ipw2x00/ipw2100.c       |    2 +-
+ drivers/net/wireless/intel/ipw2x00/ipw2100.h       |    2 +-
+ drivers/net/wireless/intel/ipw2x00/ipw2200.c       |    2 +-
+ drivers/net/wireless/intel/iwlwifi/Makefile        |    2 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/1000.c      |    8 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/2000.c      |   13 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/22000.c     |   11 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/5000.c      |    8 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/6000.c      |   19 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/7000.c      |   12 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/8000.c      |   10 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/9000.c      |    6 +-
+ drivers/net/wireless/intel/iwlwifi/dvm/main.c      |    8 +-
+ .../net/wireless/intel/iwlwifi/fw/api/datapath.h   |    5 -
+ .../net/wireless/intel/iwlwifi/fw/api/nvm-reg.h    |   42 +-
+ drivers/net/wireless/intel/iwlwifi/fw/api/rs.h     |  156 +-
+ drivers/net/wireless/intel/iwlwifi/fw/api/txq.h    |    2 +
+ drivers/net/wireless/intel/iwlwifi/fw/dbg.h        |   36 +
+ drivers/net/wireless/intel/iwlwifi/fw/debugfs.c    |    1 +
+ drivers/net/wireless/intel/iwlwifi/fw/debugfs.h    |   31 -
+ drivers/net/wireless/intel/iwlwifi/fw/file.h       |    3 +
+ drivers/net/wireless/intel/iwlwifi/fw/img.h        |    6 +
+ drivers/net/wireless/intel/iwlwifi/fw/nvm.c        |  162 --
+ drivers/net/wireless/intel/iwlwifi/fw/paging.c     |   78 +-
+ drivers/net/wireless/intel/iwlwifi/fw/runtime.h    |    1 -
+ drivers/net/wireless/intel/iwlwifi/iwl-config.h    |   92 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-csr.h       |   28 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-drv.c       |   14 +
+ .../net/wireless/intel/iwlwifi/iwl-eeprom-parse.c  |    4 +-
+ .../net/wireless/intel/iwlwifi/iwl-eeprom-parse.h  |    5 +-
+ .../net/wireless/intel/iwlwifi/iwl-eeprom-read.c   |    8 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-modparams.h |    2 +
+ drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c |  353 ++-
+ drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h |   56 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-trans.h     |   13 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/coex.c      |   37 +
+ drivers/net/wireless/intel/iwlwifi/mvm/constants.h |    7 +
+ drivers/net/wireless/intel/iwlwifi/mvm/d3.c        |   22 +-
+ .../net/wireless/intel/iwlwifi/mvm/debugfs-vif.c   |    2 +
+ drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c   |   25 +
+ drivers/net/wireless/intel/iwlwifi/mvm/fw.c        |   10 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c  |   39 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/mvm.h       |  101 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/nvm.c       |  208 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/ops.c       |   18 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c     |  117 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rs.c        |   15 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rs.h        |    3 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rx.c        |  103 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c      |   33 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/scan.c      |  194 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/sta.c       |   28 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/sta.h       |    7 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/tx.c        |   56 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/utils.c     |  438 +++-
+ .../net/wireless/intel/iwlwifi/pcie/ctxt-info.c    |    2 +-
+ drivers/net/wireless/intel/iwlwifi/pcie/internal.h |    8 +-
+ drivers/net/wireless/intel/iwlwifi/pcie/rx.c       |    3 +-
+ .../net/wireless/intel/iwlwifi/pcie/trans-gen2.c   |   15 +-
+ drivers/net/wireless/intel/iwlwifi/pcie/trans.c    |  139 +-
+ drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c  |   55 +-
+ drivers/net/wireless/intel/iwlwifi/pcie/tx.c       |   52 +-
+ drivers/net/wireless/mac80211_hwsim.c              |    2 +-
+ drivers/net/wireless/marvell/mwifiex/cfg80211.c    |    7 +-
+ drivers/net/wireless/marvell/mwifiex/cmdevt.c      |    1 -
+ drivers/net/wireless/marvell/mwifiex/ie.c          |    1 +
+ drivers/net/wireless/marvell/mwifiex/main.c        |   43 +-
+ drivers/net/wireless/marvell/mwifiex/main.h        |    7 +-
+ drivers/net/wireless/marvell/mwifiex/pcie.c        |    3 +-
+ drivers/net/wireless/marvell/mwifiex/uap_event.c   |   25 +-
+ drivers/net/wireless/mediatek/mt76/agg-rx.c        |   16 +-
+ drivers/net/wireless/mediatek/mt76/mac80211.c      |   13 +-
+ drivers/net/wireless/mediatek/mt76/mt76.h          |    1 +
+ drivers/net/wireless/mediatek/mt76/mt76x2.h        |    3 +-
+ drivers/net/wireless/mediatek/mt76/mt76x2_dma.h    |    7 +-
+ drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.c |    6 +-
+ drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.h |    6 +
+ drivers/net/wireless/mediatek/mt76/mt76x2_init.c   |   13 +-
+ drivers/net/wireless/mediatek/mt76/mt76x2_mac.c    |    1 -
+ drivers/net/wireless/mediatek/mt76/mt76x2_mac.h    |    1 -
+ drivers/net/wireless/mediatek/mt76/mt76x2_main.c   |    1 +
+ drivers/net/wireless/mediatek/mt76/mt76x2_phy.c    |   84 +-
+ drivers/net/wireless/mediatek/mt76/tx.c            |    6 +
+ drivers/net/wireless/mediatek/mt7601u/mac.c        |    4 +-
+ drivers/net/wireless/mediatek/mt7601u/main.c       |    6 +
+ drivers/net/wireless/mediatek/mt7601u/mt7601u.h    |    5 +-
+ drivers/net/wireless/mediatek/mt7601u/phy.c        |   11 +-
+ drivers/net/wireless/quantenna/qtnfmac/cfg80211.c  |    9 +
+ drivers/net/wireless/quantenna/qtnfmac/core.c      |    2 +-
+ drivers/net/wireless/quantenna/qtnfmac/event.c     |   52 +-
+ .../net/wireless/quantenna/qtnfmac/pearl/pcie.c    |    4 +
+ drivers/net/wireless/quantenna/qtnfmac/qlink.h     |    7 +-
+ drivers/net/wireless/ralink/rt2x00/rt2800.h        |   17 +-
+ drivers/net/wireless/ralink/rt2x00/rt2800lib.c     |   20 +-
+ drivers/net/wireless/ralink/rt2x00/rt2800lib.h     |    5 +-
+ drivers/net/wireless/ralink/rt2x00/rt2800mmio.c    |    1 +
+ drivers/net/wireless/ralink/rt2x00/rt2800pci.c     |    6 +-
+ drivers/net/wireless/ralink/rt2x00/rt2800soc.c     |    6 +-
+ drivers/net/wireless/ralink/rt2x00/rt2800usb.c     |    6 +-
+ drivers/net/wireless/ralink/rt2x00/rt2x00.h        |    4 -
+ drivers/net/wireless/ralink/rt2x00/rt2x00mac.c     |    3 +-
+ drivers/net/wireless/ralink/rt2x00/rt2x00queue.c   |    2 +
+ .../realtek/rtlwifi/btcoexist/halbtc8192e2ant.c    |    2 +-
+ .../realtek/rtlwifi/btcoexist/halbtc8723b2ant.c    |    2 +-
+ .../realtek/rtlwifi/btcoexist/halbtc8821a1ant.c    |   19 +-
+ .../realtek/rtlwifi/btcoexist/halbtc8821a2ant.c    |    2 +-
+ .../net/wireless/realtek/rtlwifi/rtl8192ee/hw.c    |    4 +-
+ drivers/net/wireless/realtek/rtlwifi/wifi.h        |    5 -
+ drivers/net/wireless/rsi/rsi_91x_coex.c            |    1 +
+ drivers/net/wireless/rsi/rsi_91x_core.c            |   33 +-
+ drivers/net/wireless/rsi/rsi_91x_hal.c             |  108 +-
+ drivers/net/wireless/rsi/rsi_91x_mac80211.c        |   72 +-
+ drivers/net/wireless/rsi/rsi_91x_mgmt.c            |   34 +-
+ drivers/net/wireless/rsi/rsi_91x_sdio.c            |   27 +-
+ drivers/net/wireless/rsi/rsi_91x_usb.c             |    8 +
+ drivers/net/wireless/rsi/rsi_boot_params.h         |    3 +-
+ drivers/net/wireless/rsi/rsi_hal.h                 |    3 +
+ drivers/net/wireless/rsi/rsi_main.h                |    7 +-
+ drivers/net/wireless/rsi/rsi_mgmt.h                |   19 +-
+ drivers/net/wireless/rsi/rsi_sdio.h                |    2 +-
+ drivers/net/wireless/rsi/rsi_usb.h                 |    1 +
+ drivers/net/wireless/st/cw1200/txrx.c              |    2 +-
+ drivers/net/wireless/ti/wlcore/sdio.c              |   27 +-
+ drivers/of/of_mdio.c                               |    3 +
+ drivers/phy/marvell/phy-mvebu-cp110-comphy.c       |   17 +-
+ drivers/ptp/Kconfig                                |   14 +-
+ drivers/ptp/Makefile                               |    1 +
+ drivers/ptp/ptp_pch.c                              |    7 +-
+ drivers/ptp/ptp_qoriq.c                            |  454 ++++
+ drivers/s390/net/lcs.c                             |    3 +
+ drivers/s390/net/qeth_core.h                       |   61 +-
+ drivers/s390/net/qeth_core_main.c                  |  148 +-
+ drivers/s390/net/qeth_core_mpc.h                   |    2 +
+ drivers/s390/net/qeth_core_sys.c                   |    2 +
+ drivers/s390/net/qeth_l2_main.c                    |   98 +-
+ drivers/s390/net/qeth_l3_main.c                    |  207 +-
+ drivers/scsi/csiostor/csio_hw.c                    |    2 +-
+ drivers/scsi/qedf/qedf.h                           |    7 +
+ drivers/scsi/qedf/qedf_debugfs.c                   |    1 -
+ drivers/scsi/qedf/qedf_fip.c                       |    7 +-
+ drivers/scsi/qedf/qedf_io.c                        |   11 +
+ drivers/scsi/qedf/qedf_main.c                      |  106 +-
+ drivers/scsi/qedi/qedi.h                           |    3 +
+ drivers/scsi/qedi/qedi_iscsi.c                     |    2 +-
+ drivers/scsi/qedi/qedi_iscsi.h                     |    6 +
+ drivers/scsi/qedi/qedi_main.c                      |  198 ++
+ drivers/soc/ti/knav_dma.c                          |    8 +
+ drivers/soc/ti/knav_qmss.h                         |    6 +
+ drivers/soc/ti/knav_qmss_queue.c                   |   98 +-
+ drivers/vhost/net.c                                |   19 +-
+ fs/exec.c                                          |   38 +-
+ fs/proc/Kconfig                                    |   15 +
+ fs/proc/vmcore.c                                   |  386 +++-
+ include/dt-bindings/net/microchip-lan78xx.h        |   21 +
+ include/linux/avf/virtchnl.h                       |    4 +-
+ include/linux/binfmts.h                            |    1 +
+ include/linux/bpf.h                                |   85 +-
+ include/linux/bpf_trace.h                          |    1 -
+ include/linux/bpf_types.h                          |    9 +-
+ include/linux/bpf_verifier.h                       |   13 +-
+ include/linux/bpfilter.h                           |   15 +
+ include/linux/btf.h                                |   50 +
+ include/linux/crash_dump.h                         |   18 +
+ include/linux/ethtool.h                            |    5 +
+ include/linux/filter.h                             |   37 +-
+ include/linux/fsl/ptp_qoriq.h                      |  141 ++
+ include/linux/if_bridge.h                          |   29 +
+ include/linux/if_macvlan.h                         |   29 +-
+ include/linux/if_tun.h                             |    4 +-
+ include/linux/inetdevice.h                         |    1 +
+ include/linux/kcore.h                              |    6 +
+ include/linux/mdio-bitbang.h                       |    2 -
+ include/linux/mdio-gpio.h                          |    9 +
+ include/linux/microchipphy.h                       |    3 +
+ include/linux/mlx4/device.h                        |    1 +
+ include/linux/mlx5/device.h                        |    3 +
+ include/linux/mlx5/driver.h                        |   18 +-
+ include/linux/mlx5/mlx5_ifc.h                      |   67 +-
+ include/linux/mlx5/mlx5_ifc_fpga.h                 |   77 +
+ include/linux/mmc/sdio_ids.h                       |    1 +
+ include/linux/net.h                                |    1 +
+ include/linux/net_dim.h                            |   69 +-
+ include/linux/netdev_features.h                    |    6 +-
+ include/linux/netdevice.h                          |   81 +-
+ include/linux/netfilter.h                          |   34 +-
+ include/linux/netfilter/nf_osf.h                   |   33 +
+ include/linux/netfilter_bridge/ebtables.h          |    4 -
+ include/linux/perf_event.h                         |    5 +
+ include/linux/phy.h                                |   69 +-
+ include/linux/phy/phy.h                            |    1 +
+ include/linux/platform_data/b53.h                  |    4 +
+ include/linux/platform_data/mdio-gpio.h            |   33 -
+ include/linux/platform_data/mv88e6xxx.h            |   18 +
+ include/linux/qed/qed_eth_if.h                     |    5 +
+ include/linux/qed/qed_if.h                         |  283 ++-
+ include/linux/qed/qed_ll2_if.h                     |   10 +-
+ include/linux/rhashtable.h                         |   38 +-
+ include/linux/skb_array.h                          |    5 +
+ include/linux/skbuff.h                             |   26 +-
+ include/linux/soc/ti/knav_dma.h                    |   12 +
+ include/linux/soc/ti/knav_qmss.h                   |    1 +
+ include/linux/socket.h                             |    5 +-
+ include/linux/tcp.h                                |    5 +-
+ include/linux/tnum.h                               |    4 +-
+ include/linux/trace_events.h                       |   17 +
+ include/linux/u64_stats_sync.h                     |   14 -
+ include/linux/udp.h                                |    3 +
+ include/linux/umh.h                                |   12 +
+ include/net/addrconf.h                             |   43 +
+ include/net/ax88796.h                              |   14 +
+ include/net/bluetooth/hci_core.h                   |    2 +
+ include/net/bonding.h                              |   11 +-
+ include/net/cfg80211.h                             |  131 +-
+ include/net/dcbnl.h                                |    4 +
+ include/net/devlink.h                              |   32 +-
+ include/net/dsa.h                                  |   37 +-
+ include/net/erspan.h                               |   28 +
+ include/net/failover.h                             |   36 +
+ include/net/fib_rules.h                            |    3 +-
+ include/net/flow_dissector.h                       |    7 +-
+ include/net/if_inet6.h                             |    6 +-
+ include/net/inet_connection_sock.h                 |   24 +-
+ include/net/inet_sock.h                            |    1 +
+ include/net/inet_timewait_sock.h                   |    1 +
+ include/net/ip.h                                   |    9 +-
+ include/net/ip6_fib.h                              |  211 +-
+ include/net/ip6_route.h                            |   85 +-
+ include/net/ip_fib.h                               |    2 +
+ include/net/ip_tunnels.h                           |   15 +-
+ include/net/ip_vs.h                                |    1 +
+ include/net/ipv6.h                                 |    6 +-
+ include/net/mac80211.h                             |   18 +-
+ include/net/neighbour.h                            |   19 +-
+ include/net/net_failover.h                         |   40 +
+ include/net/netfilter/ipv4/nf_nat_masquerade.h     |    2 +-
+ include/net/netfilter/ipv6/nf_nat_masquerade.h     |    2 +-
+ include/net/netfilter/nf_flow_table.h              |   24 +-
+ include/net/netfilter/nf_nat.h                     |    6 +-
+ include/net/netfilter/nf_nat_core.h                |   11 +-
+ include/net/netfilter/nf_nat_l3proto.h             |   64 +-
+ include/net/netfilter/nf_nat_l4proto.h             |    8 +-
+ include/net/netfilter/nf_nat_redirect.h            |    2 +-
+ include/net/netfilter/nf_tables.h                  |   61 +-
+ include/net/netfilter/nf_tables_core.h             |    3 +
+ include/net/netfilter/nfnetlink_log.h              |   17 -
+ include/net/netfilter/nft_meta.h                   |   44 -
+ include/net/netns/ipv4.h                           |    2 +
+ include/net/netns/ipv6.h                           |    4 +-
+ include/net/netns/nftables.h                       |    2 -
+ include/net/page_pool.h                            |  144 ++
+ include/net/pkt_cls.h                              |   20 +-
+ include/net/route.h                                |    1 +
+ include/net/sch_generic.h                          |   22 +-
+ include/net/sctp/constants.h                       |    5 +-
+ include/net/sctp/sctp.h                            |   52 +-
+ include/net/sctp/sm.h                              |    4 +-
+ include/net/sctp/structs.h                         |    6 +-
+ include/net/seg6.h                                 |    7 +-
+ include/net/seg6_local.h                           |   32 +
+ include/net/sock.h                                 |   25 +-
+ include/net/switchdev.h                            |    1 +
+ include/net/tcp.h                                  |   27 +-
+ include/net/tipc.h                                 |    4 +-
+ include/net/tls.h                                  |  127 +-
+ include/net/udp.h                                  |    4 +
+ include/net/vxlan.h                                |    1 +
+ include/net/xdp.h                                  |   85 +
+ include/net/xdp_sock.h                             |   57 +
+ include/trace/events/bpf.h                         |  355 ---
+ include/trace/events/fib.h                         |  107 +-
+ include/trace/events/fib6.h                        |   43 +-
+ include/trace/events/tcp.h                         |   75 +-
+ include/trace/events/xdp.h                         |   50 +-
+ include/uapi/linux/bpf.h                           | 2135 +++++++++++++----
+ include/uapi/linux/bpfilter.h                      |   21 +
+ include/uapi/linux/btf.h                           |  113 +
+ include/uapi/linux/cn_proc.h                       |    4 +
+ include/uapi/linux/dcbnl.h                         |   11 +
+ include/uapi/linux/devlink.h                       |   14 +
+ include/uapi/linux/elf.h                           |    1 +
+ include/uapi/linux/if_addr.h                       |    1 +
+ include/uapi/linux/if_link.h                       |    2 +
+ include/uapi/linux/if_xdp.h                        |   78 +
+ include/uapi/linux/netfilter/nf_nat.h              |   12 +-
+ include/uapi/linux/netfilter/nf_osf.h              |   86 +
+ include/uapi/linux/netfilter/nf_tables.h           |   12 +
+ include/uapi/linux/netfilter/nfnetlink_conntrack.h |    1 +
+ include/uapi/linux/netfilter/xt_osf.h              |  106 +-
+ include/uapi/linux/netfilter_bridge/ebtables.h     |    6 +
+ include/uapi/linux/netfilter_ipv6/ip6t_srh.h       |   43 +-
+ include/uapi/linux/nl80211.h                       |   99 +-
+ include/uapi/linux/openvswitch.h                   |   28 +
+ include/uapi/linux/pci_regs.h                      |    2 +
+ include/uapi/linux/pkt_cls.h                       |    1 +
+ include/uapi/linux/rtnetlink.h                     |    3 +
+ include/uapi/linux/seg6_local.h                    |   12 +
+ include/uapi/linux/snmp.h                          |    3 +
+ include/uapi/linux/tcp.h                           |   16 +
+ include/uapi/linux/tipc.h                          |   12 +-
+ include/uapi/linux/tipc_config.h                   |    5 +
+ include/uapi/linux/tipc_netlink.h                  |    1 +
+ include/uapi/linux/udp.h                           |    1 +
+ include/uapi/linux/virtio_net.h                    |    3 +
+ include/uapi/linux/vmcore.h                        |   18 +
+ init/Kconfig                                       |    1 +
+ kernel/bpf/Makefile                                |    4 +
+ kernel/bpf/arraymap.c                              |   50 +
+ kernel/bpf/btf.c                                   | 2324 +++++++++++++++++++
+ kernel/bpf/core.c                                  |  109 +-
+ kernel/bpf/cpumap.c                                |  132 +-
+ kernel/bpf/devmap.c                                |  131 +-
+ kernel/bpf/inode.c                                 |  172 +-
+ kernel/bpf/offload.c                               |    6 +-
+ kernel/bpf/sockmap.c                               |  640 +++++-
+ kernel/bpf/stackmap.c                              |  139 +-
+ kernel/bpf/syscall.c                               |  331 ++-
+ kernel/bpf/tnum.c                                  |   10 +
+ kernel/bpf/verifier.c                              |  308 ++-
+ kernel/bpf/xskmap.c                                |  232 ++
+ kernel/events/core.c                               |    8 +
+ kernel/trace/bpf_trace.c                           |  100 +-
+ kernel/trace/trace_kprobe.c                        |   29 +
+ kernel/trace/trace_uprobe.c                        |   22 +
+ kernel/umh.c                                       |  125 +-
+ lib/kobject_uevent.c                               |  178 +-
+ lib/rhashtable.c                                   |   51 +-
+ lib/test_bpf.c                                     |  570 +++--
+ net/8021q/vlan.c                                   |   11 +-
+ net/8021q/vlan.h                                   |    3 +-
+ net/8021q/vlan_dev.c                               |    4 +-
+ net/8021q/vlan_netlink.c                           |   45 +-
+ net/9p/mod.c                                       |    2 +-
+ net/Kconfig                                        |   22 +
+ net/Makefile                                       |    2 +
+ net/batman-adv/Kconfig                             |    6 +-
+ net/batman-adv/bat_v_elp.c                         |   15 +-
+ net/batman-adv/main.h                              |    2 +-
+ net/batman-adv/multicast.c                         |   29 +-
+ net/batman-adv/soft-interface.c                    |    5 +-
+ net/batman-adv/types.h                             |   23 +-
+ net/bluetooth/hci_core.c                           |   31 +
+ net/bluetooth/hci_event.c                          |   12 +-
+ net/bluetooth/hci_request.c                        |   30 +-
+ net/bpf/test_run.c                                 |    3 +-
+ net/bpfilter/Kconfig                               |   16 +
+ net/bpfilter/Makefile                              |   30 +
+ net/bpfilter/bpfilter_kern.c                       |  114 +
+ net/bpfilter/main.c                                |   63 +
+ net/bpfilter/msgfmt.h                              |   17 +
+ net/bridge/br.c                                    |   16 +-
+ net/bridge/br_fdb.c                                |   69 +-
+ net/bridge/br_forward.c                            |    6 +-
+ net/bridge/br_if.c                                 |   11 +-
+ net/bridge/br_input.c                              |    1 +
+ net/bridge/br_netlink.c                            |    9 +-
+ net/bridge/br_private.h                            |   28 +-
+ net/bridge/br_switchdev.c                          |   12 +-
+ net/bridge/br_sysfs_if.c                           |    2 +
+ net/bridge/br_vlan.c                               |   41 +
+ net/bridge/netfilter/Kconfig                       |    7 -
+ net/bridge/netfilter/Makefile                      |    1 -
+ net/bridge/netfilter/ebtables.c                    |   63 +-
+ net/bridge/netfilter/nft_meta_bridge.c             |  135 --
+ net/core/Makefile                                  |    2 +
+ net/core/dev.c                                     |  148 +-
+ net/core/devlink.c                                 |   93 +-
+ net/core/dst.c                                     |    1 +
+ net/core/ethtool.c                                 |   63 +-
+ net/core/failover.c                                |  315 +++
+ net/core/fib_rules.c                               |  495 ++--
+ net/core/filter.c                                  | 1416 ++++++++++--
+ net/core/flow_dissector.c                          |   17 +-
+ net/core/neighbour.c                               |    8 +-
+ net/core/net-traces.c                              |    4 -
+ net/core/page_pool.c                               |  317 +++
+ net/core/rtnetlink.c                               |    8 +-
+ net/core/skbuff.c                                  |   25 +-
+ net/core/sock.c                                    |   25 +-
+ net/core/xdp.c                                     |  290 +++
+ net/dcb/dcbnl.c                                    |   20 +
+ net/decnet/dn_rules.c                              |    7 +-
+ net/dsa/Kconfig                                    |    2 +-
+ net/dsa/dsa2.c                                     |   24 +
+ net/dsa/dsa_priv.h                                 |    9 -
+ net/dsa/master.c                                   |   62 +-
+ net/dsa/port.c                                     |   90 +-
+ net/dsa/slave.c                                    |  307 +--
+ net/ethernet/eth.c                                 |    6 +-
+ net/ipv4/Makefile                                  |    5 +-
+ net/ipv4/af_inet.c                                 |    5 +-
+ net/ipv4/bpfilter/Makefile                         |    2 +
+ net/ipv4/bpfilter/sockopt.c                        |   43 +
+ net/ipv4/devinet.c                                 |   15 +
+ net/ipv4/fib_frontend.c                            |   58 +-
+ net/ipv4/fib_rules.c                               |    7 +-
+ net/ipv4/fib_semantics.c                           |   43 +-
+ net/ipv4/fib_trie.c                                |   14 +-
+ net/ipv4/inet_connection_sock.c                    |    5 -
+ net/ipv4/ip_gre.c                                  |   12 +
+ net/ipv4/ip_output.c                               |   45 +-
+ net/ipv4/ip_sockglue.c                             |   17 +
+ net/ipv4/ip_tunnel_core.c                          |    6 +-
+ net/ipv4/ipconfig.c                                |  149 +-
+ net/ipv4/ipmr.c                                    |    3 +-
+ net/ipv4/metrics.c                                 |   53 +
+ net/ipv4/netfilter/ip_tables.c                     |    7 +-
+ net/ipv4/netfilter/ipt_MASQUERADE.c                |    2 +-
+ net/ipv4/netfilter/iptable_nat.c                   |   88 +-
+ net/ipv4/netfilter/nf_flow_table_ipv4.c            |  255 +--
+ net/ipv4/netfilter/nf_nat_h323.c                   |    4 +-
+ net/ipv4/netfilter/nf_nat_l3proto_ipv4.c           |  143 +-
+ net/ipv4/netfilter/nf_nat_masquerade_ipv4.c        |    4 +-
+ net/ipv4/netfilter/nf_nat_pptp.c                   |    2 +-
+ net/ipv4/netfilter/nf_nat_proto_gre.c              |    2 +-
+ net/ipv4/netfilter/nf_nat_proto_icmp.c             |    2 +-
+ net/ipv4/netfilter/nft_chain_nat_ipv4.c            |   53 +-
+ net/ipv4/netfilter/nft_masq_ipv4.c                 |    2 +-
+ net/ipv4/netlink.c                                 |   23 +
+ net/ipv4/proc.c                                    |    3 +
+ net/ipv4/route.c                                   |  177 +-
+ net/ipv4/sysctl_net_ipv4.c                         |   17 +
+ net/ipv4/tcp.c                                     |  204 +-
+ net/ipv4/tcp_input.c                               |  243 +-
+ net/ipv4/tcp_ipv4.c                                |   18 +-
+ net/ipv4/tcp_minisocks.c                           |    1 +
+ net/ipv4/tcp_output.c                              |  112 +-
+ net/ipv4/tcp_recovery.c                            |   80 +-
+ net/ipv4/tcp_timer.c                               |   27 +-
+ net/ipv4/udp.c                                     |   96 +-
+ net/ipv4/udp_offload.c                             |  101 +-
+ net/ipv6/Kconfig                                   |    5 +
+ net/ipv6/addrconf.c                                |  523 +++--
+ net/ipv6/addrconf_core.c                           |   41 +-
+ net/ipv6/af_inet6.c                                |   65 +-
+ net/ipv6/anycast.c                                 |   33 +-
+ net/ipv6/exthdrs.c                                 |   55 +-
+ net/ipv6/exthdrs_core.c                            |    2 +-
+ net/ipv6/fib6_rules.c                              |  145 +-
+ net/ipv6/ip6_fib.c                                 |  639 +++---
+ net/ipv6/ip6_gre.c                                 |   51 +-
+ net/ipv6/ip6_input.c                               |    2 +-
+ net/ipv6/ip6_offload.c                             |    6 +-
+ net/ipv6/ip6_output.c                              |   93 +-
+ net/ipv6/ip6_vti.c                                 |    2 +-
+ net/ipv6/ip6mr.c                                   |    3 +-
+ net/ipv6/ndisc.c                                   |   42 +-
+ net/ipv6/netfilter/ip6_tables.c                    |    6 +-
+ net/ipv6/netfilter/ip6t_MASQUERADE.c               |    2 +-
+ net/ipv6/netfilter/ip6t_rpfilter.c                 |    2 +
+ net/ipv6/netfilter/ip6t_srh.c                      |  173 +-
+ net/ipv6/netfilter/ip6table_nat.c                  |   87 +-
+ net/ipv6/netfilter/nf_flow_table_ipv6.c            |  246 +-
+ net/ipv6/netfilter/nf_nat_l3proto_ipv6.c           |  137 +-
+ net/ipv6/netfilter/nf_nat_masquerade_ipv6.c        |    4 +-
+ net/ipv6/netfilter/nf_nat_proto_icmpv6.c           |    2 +-
+ net/ipv6/netfilter/nft_chain_nat_ipv6.c            |   51 +-
+ net/ipv6/netfilter/nft_masq_ipv6.c                 |    2 +-
+ net/ipv6/netfilter/nft_redir_ipv6.c                |    2 +-
+ net/ipv6/reassembly.c                              |   25 +-
+ net/ipv6/route.c                                   | 1881 ++++++++-------
+ net/ipv6/seg6.c                                    |    1 -
+ net/ipv6/seg6_iptunnel.c                           |   24 +-
+ net/ipv6/seg6_local.c                              |  190 +-
+ net/ipv6/sysctl_net_ipv6.c                         |    8 +
+ net/ipv6/tcp_ipv6.c                                |    6 +-
+ net/ipv6/udp.c                                     |   44 +-
+ net/ipv6/udp_offload.c                             |    5 +-
+ net/ipv6/xfrm6_policy.c                            |    2 -
+ net/ipv6/xfrm6_state.c                             |    6 +-
+ net/l2tp/l2tp_debugfs.c                            |   20 +-
+ net/l2tp/l2tp_ppp.c                                |   21 +-
+ net/mac80211/cfg.c                                 |  103 +-
+ net/mac80211/driver-ops.h                          |    8 +-
+ net/mac80211/ethtool.c                             |   13 +-
+ net/mac80211/ht.c                                  |   44 +-
+ net/mac80211/ieee80211_i.h                         |    3 +
+ net/mac80211/main.c                                |    3 +
+ net/mac80211/mlme.c                                |   17 +-
+ net/mac80211/rx.c                                  |   40 +-
+ net/mac80211/sta_info.c                            |   38 +-
+ net/mac80211/sta_info.h                            |    5 +-
+ net/mac80211/status.c                              |    2 +
+ net/mac80211/trace.h                               |   25 +-
+ net/mac80211/tx.c                                  |   45 +
+ net/mac80211/util.c                                |    6 +-
+ net/ncsi/internal.h                                |   34 +-
+ net/ncsi/ncsi-manage.c                             |  226 +-
+ net/ncsi/ncsi-netlink.c                            |   21 +-
+ net/ncsi/ncsi-rsp.c                                |  179 +-
+ net/netfilter/Kconfig                              |   25 +-
+ net/netfilter/Makefile                             |    8 +-
+ net/netfilter/core.c                               |  102 +-
+ net/netfilter/ipvs/Kconfig                         |   37 +
+ net/netfilter/ipvs/Makefile                        |    1 +
+ net/netfilter/ipvs/ip_vs_ctl.c                     |    4 +
+ net/netfilter/ipvs/ip_vs_dh.c                      |    3 +-
+ net/netfilter/ipvs/ip_vs_lblc.c                    |    4 +-
+ net/netfilter/ipvs/ip_vs_lblcr.c                   |    4 +-
+ net/netfilter/ipvs/ip_vs_mh.c                      |  540 +++++
+ net/netfilter/ipvs/ip_vs_proto_tcp.c               |    4 +-
+ net/netfilter/ipvs/ip_vs_sh.c                      |    3 +-
+ net/netfilter/ipvs/ip_vs_xmit.c                    |    5 +-
+ net/netfilter/nf_conntrack_core.c                  |   92 +-
+ net/netfilter/nf_conntrack_ftp.c                   |    3 +-
+ net/netfilter/nf_conntrack_irc.c                   |    6 +-
+ net/netfilter/nf_conntrack_netlink.c               |   13 +-
+ net/netfilter/nf_conntrack_sane.c                  |    3 +-
+ net/netfilter/nf_conntrack_sip.c                   |    2 +-
+ net/netfilter/nf_conntrack_tftp.c                  |    2 +-
+ .../{nf_flow_table.c => nf_flow_table_core.c}      |  309 ++-
+ net/netfilter/nf_flow_table_inet.c                 |    3 +-
+ net/netfilter/nf_flow_table_ip.c                   |  487 ++++
+ net/netfilter/nf_internals.h                       |    5 +
+ net/netfilter/nf_nat_core.c                        |  321 ++-
+ net/netfilter/nf_nat_helper.c                      |    2 +-
+ net/netfilter/nf_nat_proto_common.c                |    9 +-
+ net/netfilter/nf_nat_proto_dccp.c                  |    2 +-
+ net/netfilter/nf_nat_proto_sctp.c                  |    2 +-
+ net/netfilter/nf_nat_proto_tcp.c                   |    2 +-
+ net/netfilter/nf_nat_proto_udp.c                   |    4 +-
+ net/netfilter/nf_nat_proto_unknown.c               |    2 +-
+ net/netfilter/nf_nat_redirect.c                    |    6 +-
+ net/netfilter/nf_nat_sip.c                         |    2 +-
+ net/netfilter/nf_osf.c                             |  218 ++
+ net/netfilter/nf_tables_api.c                      |  711 +++---
+ net/netfilter/nf_tables_core.c                     |   32 +-
+ net/netfilter/nfnetlink_log.c                      |    8 +-
+ net/netfilter/nfnetlink_queue.c                    |   28 +-
+ net/netfilter/nft_dynset.c                         |    7 +-
+ net/netfilter/nft_exthdr.c                         |   23 +-
+ net/netfilter/nft_flow_offload.c                   |    5 +-
+ net/netfilter/nft_hash.c                           |  133 +-
+ net/netfilter/nft_meta.c                           |  112 +-
+ net/netfilter/nft_nat.c                            |    2 +-
+ net/netfilter/nft_numgen.c                         |  161 +-
+ net/netfilter/nft_objref.c                         |    4 +-
+ net/netfilter/nft_rt.c                             |   22 +-
+ net/netfilter/nft_set_bitmap.c                     |   34 +-
+ net/netfilter/nft_set_hash.c                       |  153 +-
+ net/netfilter/nft_set_rbtree.c                     |  109 +-
+ net/netfilter/xt_NETMAP.c                          |    8 +-
+ net/netfilter/xt_NFLOG.c                           |   15 +-
+ net/netfilter/xt_REDIRECT.c                        |    2 +-
+ net/netfilter/xt_nat.c                             |   72 +-
+ net/netfilter/xt_osf.c                             |  202 +-
+ net/nfc/netlink.c                                  |   17 -
+ net/openvswitch/Kconfig                            |    3 +-
+ net/openvswitch/conntrack.c                        |  555 ++++-
+ net/openvswitch/conntrack.h                        |    9 +-
+ net/openvswitch/datapath.c                         |    7 +-
+ net/openvswitch/datapath.h                         |    3 +
+ net/packet/af_packet.c                             |   42 +-
+ net/qrtr/Kconfig                                   |    7 +
+ net/qrtr/Makefile                                  |    2 +
+ net/qrtr/tun.c                                     |  161 ++
+ net/rfkill/core.c                                  |   66 +-
+ net/sched/act_api.c                                |   20 +-
+ net/sched/act_csum.c                               |    6 +
+ net/sched/cls_api.c                                |    5 +-
+ net/sched/cls_basic.c                              |   24 +-
+ net/sched/cls_bpf.c                                |   22 +-
+ net/sched/cls_cgroup.c                             |   23 +-
+ net/sched/cls_flow.c                               |   24 +-
+ net/sched/cls_flower.c                             |  315 +--
+ net/sched/cls_fw.c                                 |   24 +-
+ net/sched/cls_matchall.c                           |   21 +-
+ net/sched/cls_route.c                              |   23 +-
+ net/sched/cls_rsvp.h                               |   20 +-
+ net/sched/cls_tcindex.c                            |   41 +-
+ net/sched/cls_u32.c                                |   37 +-
+ net/sched/sch_generic.c                            |   46 +-
+ net/sched/sch_mq.c                                 |   37 +
+ net/sctp/associola.c                               |   85 +-
+ net/sctp/chunk.c                                   |   12 +-
+ net/sctp/output.c                                  |   28 +-
+ net/sctp/outqueue.c                                |  660 +++---
+ net/sctp/sm_make_chunk.c                           |  143 +-
+ net/sctp/socket.c                                  |   43 +-
+ net/sctp/transport.c                               |   37 +-
+ net/smc/af_smc.c                                   |  803 ++++---
+ net/smc/smc.h                                      |   68 +-
+ net/smc/smc_cdc.c                                  |  101 +-
+ net/smc/smc_cdc.h                                  |   15 +-
+ net/smc/smc_clc.c                                  |    6 +-
+ net/smc/smc_clc.h                                  |    2 +-
+ net/smc/smc_core.c                                 |  199 +-
+ net/smc/smc_core.h                                 |   29 +-
+ net/smc/smc_diag.c                                 |   44 +-
+ net/smc/smc_ib.c                                   |   13 +-
+ net/smc/smc_llc.c                                  |  242 +-
+ net/smc/smc_llc.h                                  |    8 +-
+ net/smc/smc_rx.c                                   |  308 ++-
+ net/smc/smc_rx.h                                   |   11 +-
+ net/smc/smc_tx.c                                   |  111 +-
+ net/smc/smc_tx.h                                   |    5 +-
+ net/smc/smc_wr.c                                   |    1 +
+ net/tipc/bearer.c                                  |   29 +-
+ net/tipc/bearer.h                                  |    3 +
+ net/tipc/name_table.c                              |  103 +-
+ net/tipc/node.c                                    |   33 +-
+ net/tipc/node.h                                    |    3 +-
+ net/tipc/socket.c                                  |   13 +-
+ net/tipc/udp_media.c                               |    4 +-
+ net/tipc/udp_media.h                               |   14 +
+ net/tls/Kconfig                                    |   10 +
+ net/tls/Makefile                                   |    2 +
+ net/tls/tls_device.c                               |  766 +++++++
+ net/tls/tls_device_fallback.c                      |  450 ++++
+ net/tls/tls_main.c                                 |  139 +-
+ net/tls/tls_sw.c                                   |  141 +-
+ net/wireless/core.c                                |    4 +
+ net/wireless/nl80211.c                             |  304 ++-
+ net/wireless/rdev-ops.h                            |   12 +
+ net/wireless/reg.c                                 |   39 +-
+ net/wireless/sme.c                                 |   88 +-
+ net/wireless/trace.h                               |   14 +
+ net/wireless/util.c                                |   11 +
+ net/xdp/Kconfig                                    |    7 +
+ net/xdp/Makefile                                   |    1 +
+ net/xdp/xdp_umem.c                                 |  248 ++
+ net/xdp/xdp_umem.h                                 |   57 +
+ net/xdp/xdp_umem_props.h                           |   14 +
+ net/xdp/xsk.c                                      |  674 ++++++
+ net/xdp/xsk_queue.c                                |   63 +
+ net/xdp/xsk_queue.h                                |  255 +++
+ net/xfrm/xfrm_state.c                              |    9 +-
+ samples/bpf/Makefile                               |  172 +-
+ samples/bpf/{libbpf.h => bpf_insn.h}               |    8 +-
+ samples/bpf/bpf_load.c                             |  111 +-
+ samples/bpf/bpf_load.h                             |   13 +-
+ samples/bpf/cookie_uid_helper_example.c            |    2 +-
+ samples/bpf/cpustat_user.c                         |    2 +-
+ samples/bpf/fds_example.c                          |    4 +-
+ samples/bpf/lathist_user.c                         |    2 +-
+ samples/bpf/load_sock_ops.c                        |    2 +-
+ samples/bpf/lwt_len_hist_user.c                    |    2 +-
+ samples/bpf/map_perf_test_user.c                   |    2 +-
+ samples/bpf/offwaketime_user.c                     |    1 +
+ samples/bpf/sampleip_user.c                        |    1 +
+ samples/bpf/sock_example.c                         |    7 +-
+ samples/bpf/sock_example.h                         |    1 -
+ samples/bpf/sockex1_user.c                         |    2 +-
+ samples/bpf/sockex2_user.c                         |    2 +-
+ samples/bpf/sockex3_user.c                         |    2 +-
+ samples/bpf/spintest_user.c                        |    1 +
+ samples/bpf/syscall_tp_user.c                      |    2 +-
+ samples/bpf/task_fd_query_kern.c                   |   19 +
+ samples/bpf/task_fd_query_user.c                   |  382 ++++
+ samples/bpf/tc_l2_redirect_user.c                  |    2 +-
+ samples/bpf/test_cgrp2_array_pin.c                 |    2 +-
+ samples/bpf/test_cgrp2_attach.c                    |    3 +-
+ samples/bpf/test_cgrp2_attach2.c                   |    3 +-
+ samples/bpf/test_cgrp2_sock.c                      |    3 +-
+ samples/bpf/test_cgrp2_sock2.c                     |    3 +-
+ samples/bpf/test_current_task_under_cgroup_user.c  |    2 +-
+ samples/bpf/test_lru_dist.c                        |    2 +-
+ samples/bpf/test_map_in_map_user.c                 |    2 +-
+ samples/bpf/test_overhead_user.c                   |    2 +-
+ samples/bpf/test_probe_write_user_user.c           |    2 +-
+ samples/bpf/test_tunnel_bpf.sh                     |  319 ---
+ samples/bpf/trace_event_user.c                     |    1 +
+ samples/bpf/trace_output_user.c                    |  112 +-
+ samples/bpf/tracex1_user.c                         |    2 +-
+ samples/bpf/tracex2_user.c                         |    2 +-
+ samples/bpf/tracex3_user.c                         |    2 +-
+ samples/bpf/tracex4_user.c                         |    2 +-
+ samples/bpf/tracex5_user.c                         |    2 +-
+ samples/bpf/tracex6_user.c                         |    2 +-
+ samples/bpf/tracex7_user.c                         |    2 +-
+ samples/bpf/xdp1_user.c                            |   31 +-
+ samples/bpf/xdp_adjust_tail_kern.c                 |  152 ++
+ samples/bpf/xdp_adjust_tail_user.c                 |  150 ++
+ samples/bpf/xdp_fwd_kern.c                         |  138 ++
+ samples/bpf/xdp_fwd_user.c                         |  136 ++
+ samples/bpf/xdp_monitor_kern.c                     |   49 +
+ samples/bpf/xdp_monitor_user.c                     |   77 +-
+ samples/bpf/xdp_redirect_cpu_user.c                |    2 +-
+ samples/bpf/xdp_redirect_map_user.c                |    2 +-
+ samples/bpf/xdp_redirect_user.c                    |    2 +-
+ samples/bpf/xdp_router_ipv4_user.c                 |    2 +-
+ samples/bpf/xdp_rxq_info_user.c                    |   46 +-
+ samples/bpf/xdp_tx_iptunnel_user.c                 |    2 +-
+ samples/bpf/xdpsock.h                              |   11 +
+ samples/bpf/xdpsock_kern.c                         |   56 +
+ samples/bpf/xdpsock_user.c                         |  967 ++++++++
+ samples/sockmap/Makefile                           |   78 -
+ samples/sockmap/sockmap_test.sh                    |  488 ----
+ scripts/bpf_helpers_doc.py                         |  421 ++++
+ security/selinux/hooks.c                           |    4 +-
+ security/selinux/include/classmap.h                |    4 +-
+ tools/bpf/bpftool/.gitignore                       |    3 +
+ tools/bpf/bpftool/Documentation/bpftool-cgroup.rst |   11 +-
+ tools/bpf/bpftool/Documentation/bpftool-map.rst    |   53 +-
+ tools/bpf/bpftool/Documentation/bpftool-perf.rst   |   81 +
+ tools/bpf/bpftool/Documentation/bpftool-prog.rst   |    3 +-
+ tools/bpf/bpftool/Documentation/bpftool.rst        |    7 +-
+ tools/bpf/bpftool/Makefile                         |    7 +-
+ tools/bpf/bpftool/bash-completion/bpftool          |   59 +-
+ tools/bpf/bpftool/cgroup.c                         |   15 +-
+ tools/bpf/bpftool/common.c                         |   77 +-
+ tools/bpf/bpftool/main.c                           |    3 +-
+ tools/bpf/bpftool/main.h                           |    8 +-
+ tools/bpf/bpftool/map.c                            |   90 +-
+ tools/bpf/bpftool/map_perf_ring.c                  |  306 +++
+ tools/bpf/bpftool/perf.c                           |  246 ++
+ tools/bpf/bpftool/prog.c                           |  111 +-
+ tools/bpf/bpftool/xlated_dumper.c                  |   14 +-
+ tools/bpf/bpftool/xlated_dumper.h                  |    3 +
+ tools/include/uapi/asm/bitsperlong.h               |   18 +
+ tools/include/uapi/asm/errno.h                     |   18 +
+ tools/include/uapi/linux/bpf.h                     | 2135 +++++++++++++----
+ tools/include/uapi/linux/btf.h                     |  113 +
+ tools/include/uapi/linux/erspan.h                  |   52 +
+ tools/lib/bpf/Build                                |    2 +-
+ tools/lib/bpf/Makefile                             |    2 +-
+ tools/lib/bpf/bpf.c                                |  127 +-
+ tools/lib/bpf/bpf.h                                |   23 +
+ tools/lib/bpf/btf.c                                |  373 +++
+ tools/lib/bpf/btf.h                                |   22 +
+ tools/lib/bpf/libbpf.c                             |  286 ++-
+ tools/lib/bpf/libbpf.h                             |   69 +-
+ tools/testing/selftests/bpf/.gitignore             |    2 +
+ tools/testing/selftests/bpf/Makefile               |   50 +-
+ tools/testing/selftests/bpf/bpf_helpers.h          |   32 +
+ tools/testing/selftests/bpf/bpf_rand.h             |   80 +
+ tools/testing/selftests/bpf/test_adjust_tail.c     |   30 +
+ tools/testing/selftests/bpf/test_btf.c             | 2270 ++++++++++++++++++
+ tools/testing/selftests/bpf/test_btf_haskv.c       |   48 +
+ tools/testing/selftests/bpf/test_btf_nokv.c        |   43 +
+ tools/testing/selftests/bpf/test_get_stack_rawtp.c |  102 +
+ tools/testing/selftests/bpf/test_lwt_seg6local.c   |  437 ++++
+ tools/testing/selftests/bpf/test_lwt_seg6local.sh  |  140 ++
+ tools/testing/selftests/bpf/test_progs.c           |  566 ++++-
+ tools/testing/selftests/bpf/test_sockhash_kern.c   |    5 +
+ .../testing/selftests/bpf/test_sockmap.c           |  893 ++++++--
+ tools/testing/selftests/bpf/test_sockmap_kern.c    |    5 +
+ .../testing/selftests/bpf/test_sockmap_kern.h      |   64 +-
+ .../selftests/bpf/test_stacktrace_build_id.c       |   20 +-
+ tools/testing/selftests/bpf/test_stacktrace_map.c  |   19 +-
+ tools/testing/selftests/bpf/test_tunnel.sh         |  729 ++++++
+ .../testing/selftests/bpf/test_tunnel_kern.c       |  263 ++-
+ tools/testing/selftests/bpf/test_verifier.c        |  754 +++++-
+ tools/testing/selftests/bpf/trace_helpers.c        |  165 ++
+ tools/testing/selftests/bpf/trace_helpers.h        |   21 +
+ tools/testing/selftests/bpf/urandom_read.c         |   10 +-
+ tools/testing/selftests/net/.gitignore             |    4 +
+ tools/testing/selftests/net/Makefile               |    7 +-
+ tools/testing/selftests/net/fib_rule_tests.sh      |  248 ++
+ tools/testing/selftests/net/fib_tests.sh           |  858 ++++++-
+ .../selftests/net/forwarding/bridge_vlan_aware.sh  |   26 +-
+ .../net/forwarding/bridge_vlan_unaware.sh          |   26 +-
+ tools/testing/selftests/net/forwarding/lib.sh      |  167 +-
+ .../testing/selftests/net/forwarding/mirror_gre.sh |  159 ++
+ .../selftests/net/forwarding/mirror_gre_bound.sh   |  226 ++
+ .../net/forwarding/mirror_gre_bridge_1d_vlan.sh    |  109 +
+ .../selftests/net/forwarding/mirror_gre_changes.sh |  278 +++
+ .../selftests/net/forwarding/mirror_gre_flower.sh  |  137 ++
+ .../selftests/net/forwarding/mirror_gre_lib.sh     |   98 +
+ .../selftests/net/forwarding/mirror_gre_neigh.sh   |  115 +
+ .../selftests/net/forwarding/mirror_gre_nh.sh      |  127 +
+ .../net/forwarding/mirror_gre_topo_lib.sh          |   94 +
+ .../selftests/net/forwarding/mirror_gre_vlan.sh    |   92 +
+ .../net/forwarding/mirror_gre_vlan_bridge_1q.sh    |  140 ++
+ .../testing/selftests/net/forwarding/mirror_lib.sh |   94 +
+ .../selftests/net/forwarding/mirror_topo_lib.sh    |  101 +
+ .../selftests/net/forwarding/mirror_vlan.sh        |  169 ++
+ tools/testing/selftests/net/forwarding/router.sh   |   14 +-
+ .../selftests/net/forwarding/router_multipath.sh   |   29 +-
+ .../testing/selftests/net/forwarding/tc_actions.sh |   25 +-
+ .../testing/selftests/net/forwarding/tc_chains.sh  |    7 +-
+ .../testing/selftests/net/forwarding/tc_flower.sh  |   80 +-
+ .../selftests/net/forwarding/tc_shblocks.sh        |    5 +-
+ tools/testing/selftests/net/pmtu.sh                |    4 +-
+ tools/testing/selftests/net/rtnetlink.sh           |  103 +
+ tools/testing/selftests/net/tcp_inq.c              |  189 ++
+ tools/testing/selftests/net/tcp_mmap.c             |  447 ++++
+ tools/testing/selftests/net/udpgso.c               |  620 +++++
+ tools/testing/selftests/net/udpgso.sh              |   29 +
+ tools/testing/selftests/net/udpgso_bench.sh        |   74 +
+ tools/testing/selftests/net/udpgso_bench_rx.c      |  265 +++
+ tools/testing/selftests/net/udpgso_bench_tx.c      |  420 ++++
+ .../tc-testing/tc-tests/actions/csum.json          |   74 +-
+ .../selftests/tc-testing/tc-tests/actions/ife.json | 1036 ++++++++-
+ .../tc-testing/tc-tests/actions/mirred.json        |   24 +-
+ .../tc-testing/tc-tests/actions/police.json        |    4 +-
+ .../tc-testing/tc-tests/actions/sample.json        |  588 +++++
+ .../tc-testing/tc-tests/actions/vlan.json          |  320 ++-
+ tools/testing/selftests/uevent/Makefile            |   17 +
+ tools/testing/selftests/uevent/config              |    2 +
+ tools/testing/selftests/uevent/uevent_filtering.c  |  486 ++++
+ 1496 files changed, 92526 insertions(+), 29514 deletions(-)
+ create mode 100644 Documentation/bpf/README.rst
+ create mode 100644 Documentation/bpf/bpf_design_QA.rst
+ delete mode 100644 Documentation/bpf/bpf_design_QA.txt
+ create mode 100644 Documentation/bpf/bpf_devel_QA.rst
+ delete mode 100644 Documentation/bpf/bpf_devel_QA.txt
+ create mode 100644 Documentation/devicetree/bindings/arm/stm32/stm32-syscon.txt
+ rename Documentation/devicetree/bindings/arm/{ => stm32}/stm32.txt (100%)
+ create mode 100644 Documentation/devicetree/bindings/net/microchip,lan78xx.txt
+ create mode 100644 Documentation/devicetree/bindings/net/mscc-miim.txt
+ create mode 100644 Documentation/devicetree/bindings/net/mscc-ocelot.txt
+ create mode 100644 Documentation/devicetree/bindings/ptp/ptp-qoriq.txt
+ create mode 100644 Documentation/networking/af_xdp.rst
+ create mode 100644 Documentation/networking/failover.rst
+ create mode 100644 Documentation/networking/net_failover.rst
+ delete mode 100644 arch/powerpc/net/bpf_jit_asm64.S
+ delete mode 100644 arch/s390/net/bpf_jit.S
+ delete mode 100644 arch/sparc/net/bpf_jit_asm_64.S
+ delete mode 100644 arch/x86/net/bpf_jit.S
+ create mode 100644 arch/x86/net/bpf_jit_comp32.c
+ create mode 100644 drivers/net/ethernet/8390/xsurf100.c
+ create mode 100644 drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c
+ create mode 100644 drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.h
+ delete mode 100644 drivers/net/ethernet/freescale/gianfar_ptp.c
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/accel/tls.c
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/Makefile
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/port.c
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/port.h
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.h
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_stats.c
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.c
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.h
+ create mode 100644 drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.h
+ create mode 100644 drivers/net/ethernet/mscc/Kconfig
+ create mode 100644 drivers/net/ethernet/mscc/Makefile
+ create mode 100644 drivers/net/ethernet/mscc/ocelot.c
+ create mode 100644 drivers/net/ethernet/mscc/ocelot.h
+ create mode 100644 drivers/net/ethernet/mscc/ocelot_ana.h
+ create mode 100644 drivers/net/ethernet/mscc/ocelot_board.c
+ create mode 100644 drivers/net/ethernet/mscc/ocelot_dev.h
+ create mode 100644 drivers/net/ethernet/mscc/ocelot_dev_gmii.h
+ create mode 100644 drivers/net/ethernet/mscc/ocelot_hsio.h
+ create mode 100644 drivers/net/ethernet/mscc/ocelot_io.c
+ create mode 100644 drivers/net/ethernet/mscc/ocelot_qs.h
+ create mode 100644 drivers/net/ethernet/mscc/ocelot_qsys.h
+ create mode 100644 drivers/net/ethernet/mscc/ocelot_regs.c
+ create mode 100644 drivers/net/ethernet/mscc/ocelot_rew.h
+ create mode 100644 drivers/net/ethernet/mscc/ocelot_sys.h
+ create mode 100644 drivers/net/ethernet/netronome/nfp/abm/ctrl.c
+ create mode 100644 drivers/net/ethernet/netronome/nfp/abm/main.c
+ create mode 100644 drivers/net/ethernet/netronome/nfp/abm/main.h
+ create mode 100644 drivers/net/ethernet/netronome/nfp/flower/lag_conf.c
+ create mode 100644 drivers/net/ethernet/netronome/nfp/nfp_abi.h
+ create mode 100644 drivers/net/ethernet/netronome/nfp/nfp_shared_buf.c
+ create mode 100644 drivers/net/ethernet/qlogic/qed/qed_mng_tlv.c
+ create mode 100644 drivers/net/ethernet/stmicro/stmmac/hwif.c
+ create mode 100644 drivers/net/ethernet/stmicro/stmmac/hwif.h
+ create mode 100644 drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c
+ create mode 100644 drivers/net/net_failover.c
+ create mode 100644 drivers/net/phy/asix.c
+ create mode 100644 drivers/net/phy/dp83tc811.c
+ create mode 100644 drivers/net/phy/mdio-mscc-miim.c
+ create mode 100644 drivers/net/phy/microchip_t1.c
+ create mode 100644 drivers/net/wireless/ath/ath10k/snoc.c
+ create mode 100644 drivers/net/wireless/ath/ath10k/snoc.h
+ delete mode 100644 drivers/net/wireless/intel/iwlwifi/fw/nvm.c
+ create mode 100644 drivers/ptp/ptp_qoriq.c
+ create mode 100644 include/dt-bindings/net/microchip-lan78xx.h
+ create mode 100644 include/linux/bpfilter.h
+ create mode 100644 include/linux/btf.h
+ create mode 100644 include/linux/fsl/ptp_qoriq.h
+ create mode 100644 include/linux/mdio-gpio.h
+ create mode 100644 include/linux/netfilter/nf_osf.h
+ delete mode 100644 include/linux/platform_data/mdio-gpio.h
+ create mode 100644 include/linux/platform_data/mv88e6xxx.h
+ create mode 100644 include/net/failover.h
+ create mode 100644 include/net/net_failover.h
+ delete mode 100644 include/net/netfilter/nft_meta.h
+ create mode 100644 include/net/page_pool.h
+ create mode 100644 include/net/seg6_local.h
+ create mode 100644 include/net/xdp_sock.h
+ delete mode 100644 include/trace/events/bpf.h
+ create mode 100644 include/uapi/linux/bpfilter.h
+ create mode 100644 include/uapi/linux/btf.h
+ create mode 100644 include/uapi/linux/if_xdp.h
+ create mode 100644 include/uapi/linux/netfilter/nf_osf.h
+ create mode 100644 include/uapi/linux/vmcore.h
+ create mode 100644 kernel/bpf/btf.c
+ create mode 100644 kernel/bpf/xskmap.c
+ create mode 100644 net/bpfilter/Kconfig
+ create mode 100644 net/bpfilter/Makefile
+ create mode 100644 net/bpfilter/bpfilter_kern.c
+ create mode 100644 net/bpfilter/main.c
+ create mode 100644 net/bpfilter/msgfmt.h
+ delete mode 100644 net/bridge/netfilter/nft_meta_bridge.c
+ create mode 100644 net/core/failover.c
+ create mode 100644 net/core/page_pool.c
+ create mode 100644 net/ipv4/bpfilter/Makefile
+ create mode 100644 net/ipv4/bpfilter/sockopt.c
+ create mode 100644 net/ipv4/metrics.c
+ create mode 100644 net/ipv4/netlink.c
+ create mode 100644 net/netfilter/ipvs/ip_vs_mh.c
+ rename net/netfilter/{nf_flow_table.c => nf_flow_table_core.c} (67%)
+ create mode 100644 net/netfilter/nf_flow_table_ip.c
+ create mode 100644 net/netfilter/nf_osf.c
+ create mode 100644 net/qrtr/tun.c
+ create mode 100644 net/tls/tls_device.c
+ create mode 100644 net/tls/tls_device_fallback.c
+ create mode 100644 net/xdp/Kconfig
+ create mode 100644 net/xdp/Makefile
+ create mode 100644 net/xdp/xdp_umem.c
+ create mode 100644 net/xdp/xdp_umem.h
+ create mode 100644 net/xdp/xdp_umem_props.h
+ create mode 100644 net/xdp/xsk.c
+ create mode 100644 net/xdp/xsk_queue.c
+ create mode 100644 net/xdp/xsk_queue.h
+ rename samples/bpf/{libbpf.h => bpf_insn.h} (98%)
+ create mode 100644 samples/bpf/task_fd_query_kern.c
+ create mode 100644 samples/bpf/task_fd_query_user.c
+ delete mode 100755 samples/bpf/test_tunnel_bpf.sh
+ create mode 100644 samples/bpf/xdp_adjust_tail_kern.c
+ create mode 100644 samples/bpf/xdp_adjust_tail_user.c
+ create mode 100644 samples/bpf/xdp_fwd_kern.c
+ create mode 100644 samples/bpf/xdp_fwd_user.c
+ create mode 100644 samples/bpf/xdpsock.h
+ create mode 100644 samples/bpf/xdpsock_kern.c
+ create mode 100644 samples/bpf/xdpsock_user.c
+ delete mode 100644 samples/sockmap/Makefile
+ delete mode 100755 samples/sockmap/sockmap_test.sh
+ create mode 100755 scripts/bpf_helpers_doc.py
+ create mode 100644 tools/bpf/bpftool/.gitignore
+ create mode 100644 tools/bpf/bpftool/Documentation/bpftool-perf.rst
+ create mode 100644 tools/bpf/bpftool/map_perf_ring.c
+ create mode 100644 tools/bpf/bpftool/perf.c
+ create mode 100644 tools/include/uapi/asm/bitsperlong.h
+ create mode 100644 tools/include/uapi/asm/errno.h
+ create mode 100644 tools/include/uapi/linux/btf.h
+ create mode 100644 tools/include/uapi/linux/erspan.h
+ create mode 100644 tools/lib/bpf/btf.c
+ create mode 100644 tools/lib/bpf/btf.h
+ create mode 100644 tools/testing/selftests/bpf/bpf_rand.h
+ create mode 100644 tools/testing/selftests/bpf/test_adjust_tail.c
+ create mode 100644 tools/testing/selftests/bpf/test_btf.c
+ create mode 100644 tools/testing/selftests/bpf/test_btf_haskv.c
+ create mode 100644 tools/testing/selftests/bpf/test_btf_nokv.c
+ create mode 100644 tools/testing/selftests/bpf/test_get_stack_rawtp.c
+ create mode 100644 tools/testing/selftests/bpf/test_lwt_seg6local.c
+ create mode 100755 tools/testing/selftests/bpf/test_lwt_seg6local.sh
+ create mode 100644 tools/testing/selftests/bpf/test_sockhash_kern.c
+ rename samples/sockmap/sockmap_user.c => tools/testing/selftests/bpf/test_sockmap.c (56%)
+ create mode 100644 tools/testing/selftests/bpf/test_sockmap_kern.c
+ rename samples/sockmap/sockmap_kern.c => tools/testing/selftests/bpf/test_sockmap_kern.h (88%)
+ create mode 100755 tools/testing/selftests/bpf/test_tunnel.sh
+ rename samples/bpf/tcbpf2_kern.c => tools/testing/selftests/bpf/test_tunnel_kern.c (68%)
+ create mode 100644 tools/testing/selftests/bpf/trace_helpers.c
+ create mode 100644 tools/testing/selftests/bpf/trace_helpers.h
+ create mode 100755 tools/testing/selftests/net/fib_rule_tests.sh
+ mode change 100755 => 100644 tools/testing/selftests/net/fib_tests.sh
+ create mode 100755 tools/testing/selftests/net/forwarding/mirror_gre.sh
+ create mode 100755 tools/testing/selftests/net/forwarding/mirror_gre_bound.sh
+ create mode 100755 tools/testing/selftests/net/forwarding/mirror_gre_bridge_1d_vlan.sh
+ create mode 100755 tools/testing/selftests/net/forwarding/mirror_gre_changes.sh
+ create mode 100755 tools/testing/selftests/net/forwarding/mirror_gre_flower.sh
+ create mode 100644 tools/testing/selftests/net/forwarding/mirror_gre_lib.sh
+ create mode 100755 tools/testing/selftests/net/forwarding/mirror_gre_neigh.sh
+ create mode 100755 tools/testing/selftests/net/forwarding/mirror_gre_nh.sh
+ create mode 100644 tools/testing/selftests/net/forwarding/mirror_gre_topo_lib.sh
+ create mode 100755 tools/testing/selftests/net/forwarding/mirror_gre_vlan.sh
+ create mode 100755 tools/testing/selftests/net/forwarding/mirror_gre_vlan_bridge_1q.sh
+ create mode 100644 tools/testing/selftests/net/forwarding/mirror_lib.sh
+ create mode 100644 tools/testing/selftests/net/forwarding/mirror_topo_lib.sh
+ create mode 100755 tools/testing/selftests/net/forwarding/mirror_vlan.sh
+ create mode 100644 tools/testing/selftests/net/tcp_inq.c
+ create mode 100644 tools/testing/selftests/net/tcp_mmap.c
+ create mode 100644 tools/testing/selftests/net/udpgso.c
+ create mode 100755 tools/testing/selftests/net/udpgso.sh
+ create mode 100755 tools/testing/selftests/net/udpgso_bench.sh
+ create mode 100644 tools/testing/selftests/net/udpgso_bench_rx.c
+ create mode 100644 tools/testing/selftests/net/udpgso_bench_tx.c
+ create mode 100644 tools/testing/selftests/tc-testing/tc-tests/actions/sample.json
+ create mode 100644 tools/testing/selftests/uevent/Makefile
+ create mode 100644 tools/testing/selftests/uevent/config
+ create mode 100644 tools/testing/selftests/uevent/uevent_filtering.c
+$ git am -3 ../patches/0001-xsk-update-for-net-convert-datagram_poll-users-tp-po.patch
+Applying: xsk: update for "net: convert datagram_poll users tp ->poll_mask"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	net/xdp/xsk.c
+$ git add -A .
+$ git commit -v -a --amend
+[master 5c4849d1e267] Merge remote-tracking branch 'net-next/master'
+ Date: Thu May 31 12:30:18 2018 +1000
+$ git am -3 ../patches/0001-Revert-bpfilter-don-t-pass-O_CREAT-when-opening-cons.patch
+Applying: Revert "bpfilter: don't pass O_CREAT when opening console for debug"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	net/bpfilter/main.c
+$ git add -A .
+$ git commit -v -a --amend
+[master c13f7b864224] Merge remote-tracking branch 'net-next/master'
+ Date: Thu May 31 12:30:18 2018 +1000
+$ git am -3 ../patches/0002-Revert-bpfilter-fix-build-dependency.patch
+Applying: Revert "bpfilter: fix build dependency"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	net/bpfilter/Kconfig
+$ git add -A .
+$ git commit -v -a --amend
+[master 1541b74e2a98] Merge remote-tracking branch 'net-next/master'
+ Date: Thu May 31 12:30:18 2018 +1000
+$ git am -3 ../patches/0003-Revert-net-add-skeleton-of-bpfilter-kernel-module.patch
+Applying: Revert "net: add skeleton of bpfilter kernel module"
+$ git reset HEAD^
+Unstaged changes after reset:
+D	include/linux/bpfilter.h
+D	include/uapi/linux/bpfilter.h
+M	net/Kconfig
+M	net/Makefile
+D	net/bpfilter/Kconfig
+D	net/bpfilter/Makefile
+D	net/bpfilter/bpfilter_kern.c
+D	net/bpfilter/main.c
+D	net/bpfilter/msgfmt.h
+M	net/ipv4/Makefile
+D	net/ipv4/bpfilter/Makefile
+D	net/ipv4/bpfilter/sockopt.c
+M	net/ipv4/ip_sockglue.c
+$ git add -A .
+$ git commit -v -a --amend
+[master 6e99b530004e] Merge remote-tracking branch 'net-next/master'
+ Date: Thu May 31 12:30:18 2018 +1000
+Merging bpf-next/master (bcece5dc40b9 bpf: Change bpf_fib_lookup to return -EAFNOSUPPORT for unsupported address families)
+$ git merge bpf-next/master
+Auto-merging net/ipv6/udp.c
+Auto-merging net/ipv4/udp.c
+Auto-merging drivers/media/rc/rc-ir-raw.c
+Auto-merging drivers/media/rc/rc-core-priv.h
+Auto-merging drivers/media/rc/lirc_dev.c
+Auto-merging drivers/media/rc/Kconfig
+Merge made by the 'recursive' strategy.
+ drivers/media/rc/Kconfig                           |   13 +
+ drivers/media/rc/Makefile                          |    1 +
+ drivers/media/rc/bpf-lirc.c                        |  313 ++++++
+ drivers/media/rc/lirc_dev.c                        |   30 +
+ drivers/media/rc/rc-core-priv.h                    |   21 +
+ drivers/media/rc/rc-ir-raw.c                       |   12 +-
+ include/linux/bpf-cgroup.h                         |   24 +-
+ include/linux/bpf_lirc.h                           |   29 +
+ include/linux/bpf_types.h                          |    3 +
+ include/linux/filter.h                             |    1 +
+ include/uapi/linux/bpf.h                           |  108 +-
+ kernel/bpf/btf.c                                   |    2 +-
+ kernel/bpf/cgroup.c                                |   11 +-
+ kernel/bpf/core.c                                  |   11 +-
+ kernel/bpf/devmap.c                                |    2 +-
+ kernel/bpf/stackmap.c                              |    7 +-
+ kernel/bpf/syscall.c                               |   15 +
+ kernel/trace/bpf_trace.c                           |    2 +
+ net/core/filter.c                                  |   51 +-
+ net/ipv4/udp.c                                     |   20 +-
+ net/ipv6/udp.c                                     |   24 +
+ tools/bpf/bpftool/Documentation/bpftool-cgroup.rst |    9 +-
+ tools/bpf/bpftool/bash-completion/bpftool          |    5 +-
+ tools/bpf/bpftool/cgroup.c                         |    4 +-
+ tools/bpf/bpftool/prog.c                           |    1 +
+ tools/include/uapi/linux/bpf.h                     |  108 +-
+ tools/include/uapi/linux/lirc.h                    |  217 ++++
+ tools/include/uapi/linux/seg6.h                    |   55 +
+ tools/include/uapi/linux/seg6_local.h              |   80 ++
+ tools/lib/bpf/Makefile                             |    1 +
+ tools/lib/bpf/libbpf.c                             |    3 +
+ tools/testing/selftests/bpf/.gitignore             |    1 +
+ tools/testing/selftests/bpf/Makefile               |    7 +-
+ tools/testing/selftests/bpf/bpf_helpers.h          |    5 +
+ tools/testing/selftests/bpf/sendmsg4_prog.c        |   49 +
+ tools/testing/selftests/bpf/sendmsg6_prog.c        |   60 +
+ tools/testing/selftests/bpf/test_lirc_mode2.sh     |   28 +
+ tools/testing/selftests/bpf/test_lirc_mode2_kern.c |   23 +
+ tools/testing/selftests/bpf/test_lirc_mode2_user.c |  149 +++
+ tools/testing/selftests/bpf/test_sock_addr.c       | 1155 ++++++++++++++++----
+ 40 files changed, 2395 insertions(+), 265 deletions(-)
+ create mode 100644 drivers/media/rc/bpf-lirc.c
+ create mode 100644 include/linux/bpf_lirc.h
+ create mode 100644 tools/include/uapi/linux/lirc.h
+ create mode 100644 tools/include/uapi/linux/seg6.h
+ create mode 100644 tools/include/uapi/linux/seg6_local.h
+ create mode 100644 tools/testing/selftests/bpf/sendmsg4_prog.c
+ create mode 100644 tools/testing/selftests/bpf/sendmsg6_prog.c
+ create mode 100755 tools/testing/selftests/bpf/test_lirc_mode2.sh
+ create mode 100644 tools/testing/selftests/bpf/test_lirc_mode2_kern.c
+ create mode 100644 tools/testing/selftests/bpf/test_lirc_mode2_user.c
+Merging ipsec-next/master (f03329627b46 Merge branch 'hns3-next')
+$ git merge ipsec-next/master
+Already up to date.
+Merging netfilter-next/master (e523452ac0e0 netfilter: nf_tables: remove unused variables)
+$ git merge netfilter-next/master
+Auto-merging net/netfilter/nf_tables_core.c
+Auto-merging net/netfilter/nf_tables_api.c
+Merge made by the 'recursive' strategy.
+ include/net/netfilter/nf_socket.h           |   6 +-
+ include/net/netfilter/nf_tables.h           |   5 +
+ net/ipv4/netfilter/Kconfig                  |   5 +-
+ net/ipv4/netfilter/Makefile                 |   4 +-
+ net/ipv4/netfilter/nf_nat_masquerade_ipv4.c |   4 -
+ net/ipv6/netfilter/Kconfig                  |   5 +-
+ net/ipv6/netfilter/Makefile                 |   2 +-
+ net/ipv6/netfilter/nf_nat_masquerade_ipv6.c |   4 -
+ net/netfilter/Kconfig                       |   6 +-
+ net/netfilter/Makefile                      |   2 +-
+ net/netfilter/nf_nat_core.c                 |   2 +-
+ net/netfilter/nf_nat_redirect.c             |   4 -
+ net/netfilter/nf_tables_api.c               | 317 +++++++++++++++++++++++-----
+ net/netfilter/nf_tables_core.c              |  40 ++--
+ net/netfilter/nfnetlink.c                   |  17 +-
+ net/netfilter/nft_compat.c                  |  29 ++-
+ 16 files changed, 331 insertions(+), 121 deletions(-)
+Merging nfc-next/master (4d63adfe12dd NFC: Add NFC_CMD_DEACTIVATE_TARGET support)
+$ git merge nfc-next/master
+Already up to date.
+Merging ipvs-next/master (90278871d4b0 Merge git://git.kernel.org/pub/scm/linux/kernel/git/pablo/nf-next)
+$ git merge ipvs-next/master
+Already up to date.
+Merging wireless-drivers-next/master (76606886c9f4 Merge ath-next from git://git.kernel.org/pub/scm/linux/kernel/git/kvalo/ath.git)
+$ git merge wireless-drivers-next/master
+Auto-merging drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+Merge made by the 'recursive' strategy.
+ drivers/net/wireless/ath/ath10k/ahb.c              |   9 +-
+ drivers/net/wireless/ath/ath10k/core.h             |  22 ++
+ drivers/net/wireless/ath/ath10k/coredump.c         |  98 ++++++
+ drivers/net/wireless/ath/ath10k/debug.c            |  28 +-
+ drivers/net/wireless/ath/ath10k/debugfs_sta.c      |  30 +-
+ drivers/net/wireless/ath/ath10k/hw.h               |   4 +-
+ drivers/net/wireless/ath/ath10k/mac.c              |  37 +-
+ drivers/net/wireless/ath/ath10k/rx_desc.h          | 136 ++++----
+ drivers/net/wireless/ath/ath10k/sdio.c             |   2 +-
+ drivers/net/wireless/ath/ath10k/snoc.c             |   1 -
+ drivers/net/wireless/ath/ath10k/wmi-ops.h          |  32 ++
+ drivers/net/wireless/ath/ath10k/wmi-tlv.c          |  33 +-
+ drivers/net/wireless/ath/ath10k/wmi-tlv.h          |  16 +
+ drivers/net/wireless/ath/ath10k/wmi.c              | 206 +++++++++--
+ drivers/net/wireless/ath/ath10k/wmi.h              |  34 +-
+ drivers/net/wireless/ath/ath6kl/core.h             |   2 +-
+ drivers/net/wireless/ath/ath6kl/txrx.c             |   2 +-
+ drivers/net/wireless/ath/ath9k/dfs.c               |   2 +-
+ drivers/net/wireless/ath/dfs_pattern_detector.c    |   5 +-
+ drivers/net/wireless/ath/dfs_pattern_detector.h    |   3 +-
+ drivers/net/wireless/ath/dfs_pri_detector.h        |   3 +-
+ drivers/net/wireless/ath/regd.h                    |   7 +
+ drivers/net/wireless/ath/regd_common.h             |  60 ++--
+ drivers/net/wireless/ath/wcn36xx/Makefile          |   2 +
+ drivers/net/wireless/ath/wcn36xx/dxe.c             | 176 ++++++----
+ drivers/net/wireless/ath/wcn36xx/hal.h             |  16 +
+ drivers/net/wireless/ath/wcn36xx/main.c            |  13 +
+ drivers/net/wireless/ath/wcn36xx/smd.c             | 113 +++++-
+ drivers/net/wireless/ath/wcn36xx/smd.h             |   4 +
+ drivers/net/wireless/ath/wcn36xx/testmode.c        | 149 ++++++++
+ drivers/net/wireless/ath/wcn36xx/testmode.h        |  46 +++
+ drivers/net/wireless/ath/wcn36xx/testmode_i.h      |  29 ++
+ drivers/net/wireless/ath/wcn36xx/wcn36xx.h         |   2 +
+ drivers/net/wireless/ath/wil6210/Kconfig           |   2 +-
+ drivers/net/wireless/ath/wil6210/cfg80211.c        |  61 +---
+ drivers/net/wireless/ath/wil6210/debugfs.c         |   5 +-
+ drivers/net/wireless/ath/wil6210/main.c            |   2 +
+ drivers/net/wireless/ath/wil6210/netdev.c          |   8 +-
+ drivers/net/wireless/ath/wil6210/rx_reorder.c      |   7 +-
+ drivers/net/wireless/ath/wil6210/txrx.c            |  12 +-
+ drivers/net/wireless/ath/wil6210/wil6210.h         |  13 +-
+ drivers/net/wireless/ath/wil6210/wmi.c             | 152 +++++---
+ drivers/net/wireless/ath/wil6210/wmi.h             | 387 +++++++++++++++++++--
+ drivers/net/wireless/atmel/atmel_pci.c             |   4 +-
+ .../wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c  |   1 +
+ .../net/wireless/broadcom/brcm80211/brcmfmac/bus.h |   2 +
+ .../wireless/broadcom/brcm80211/brcmfmac/common.c  |   2 -
+ .../wireless/broadcom/brcm80211/brcmfmac/common.h  |   2 -
+ .../wireless/broadcom/brcm80211/brcmfmac/core.c    |   8 +
+ .../wireless/broadcom/brcm80211/brcmfmac/debug.c   |   3 +-
+ .../wireless/broadcom/brcm80211/brcmfmac/feature.c |  36 ++
+ .../broadcom/brcm80211/brcmfmac/flowring.c         |   2 +
+ .../wireless/broadcom/brcm80211/brcmfmac/pcie.c    |   6 +
+ .../wireless/broadcom/brcm80211/brcmfmac/sdio.c    |   4 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/6000.c      |   1 +
+ drivers/net/wireless/intel/iwlwifi/cfg/9000.c      |   2 +-
+ drivers/net/wireless/intel/iwlwifi/fw/api/rx.h     |   1 +
+ drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c |   3 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c  |   8 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rs.c        |  38 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rs.h        |   7 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c      |  36 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/sta.c       |   2 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/sta.h       |  10 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/tdls.c      |  16 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/tx.c        |  16 +-
+ drivers/net/wireless/intel/iwlwifi/pcie/drv.c      |   3 +
+ drivers/net/wireless/intel/iwlwifi/pcie/rx.c       |   2 +
+ drivers/net/wireless/marvell/mwifiex/cfg80211.c    |  24 +-
+ drivers/net/wireless/marvell/mwifiex/debugfs.c     |  31 +-
+ drivers/net/wireless/marvell/mwifiex/fw.h          |   1 +
+ drivers/net/wireless/marvell/mwifiex/main.h        |   1 +
+ drivers/net/wireless/marvell/mwifiex/pcie.c        |  18 +-
+ drivers/net/wireless/marvell/mwifiex/scan.c        |  10 +-
+ drivers/net/wireless/marvell/mwifiex/sdio.c        |  12 +
+ drivers/net/wireless/marvell/mwifiex/sta_event.c   |   3 +
+ drivers/net/wireless/marvell/mwifiex/usb.c         |  15 +
+ drivers/net/wireless/marvell/mwifiex/util.c        |  15 +-
+ drivers/net/wireless/mediatek/mt76/dma.c           |   4 +
+ drivers/net/wireless/mediatek/mt76/mac80211.c      |  43 ++-
+ drivers/net/wireless/mediatek/mt76/mt76.h          |   5 +
+ drivers/net/wireless/mediatek/mt76/mt76x2.h        |   3 +
+ drivers/net/wireless/mediatek/mt76/mt76x2_init.c   |  22 +-
+ drivers/net/wireless/mediatek/mt76/mt76x2_mac.c    |   3 +
+ drivers/net/wireless/mediatek/mt76/mt76x2_main.c   |  15 +-
+ drivers/net/wireless/mediatek/mt76/tx.c            |   4 +-
+ drivers/net/wireless/quantenna/qtnfmac/bus.h       |   3 +-
+ drivers/net/wireless/quantenna/qtnfmac/cfg80211.c  |  56 +--
+ drivers/net/wireless/quantenna/qtnfmac/commands.c  |  35 +-
+ drivers/net/wireless/quantenna/qtnfmac/commands.h  |   5 -
+ drivers/net/wireless/quantenna/qtnfmac/core.c      |   3 +-
+ drivers/net/wireless/quantenna/qtnfmac/core.h      |   2 -
+ drivers/net/wireless/quantenna/qtnfmac/event.c     |   8 +-
+ .../net/wireless/quantenna/qtnfmac/pearl/pcie.c    |  15 +-
+ drivers/net/wireless/quantenna/qtnfmac/qlink.h     |   2 +
+ drivers/net/wireless/quantenna/qtnfmac/trans.c     |   4 +-
+ drivers/net/wireless/realtek/rtlwifi/base.c        | 208 ++++++++---
+ drivers/net/wireless/realtek/rtlwifi/base.h        |   7 +-
+ .../realtek/rtlwifi/btcoexist/halbtc8723b2ant.c    |  23 +-
+ drivers/net/wireless/realtek/rtlwifi/pci.c         |  18 +-
+ .../net/wireless/realtek/rtlwifi/rtl8188ee/def.h   |  32 --
+ .../net/wireless/realtek/rtlwifi/rtl8188ee/sw.c    |   2 -
+ .../net/wireless/realtek/rtlwifi/rtl8188ee/trx.c   |   7 -
+ .../net/wireless/realtek/rtlwifi/rtl8188ee/trx.h   |   4 -
+ .../net/wireless/realtek/rtlwifi/rtl8192ce/def.h   |  25 --
+ .../net/wireless/realtek/rtlwifi/rtl8192ee/def.h   |   8 -
+ .../net/wireless/realtek/rtlwifi/rtl8192ee/fw.c    |  78 +----
+ .../net/wireless/realtek/rtlwifi/rtl8192ee/fw.h    |  16 +-
+ .../net/wireless/realtek/rtlwifi/rtl8192ee/sw.c    |   3 +-
+ .../net/wireless/realtek/rtlwifi/rtl8192ee/trx.c   |  29 +-
+ .../net/wireless/realtek/rtlwifi/rtl8192ee/trx.h   |   3 -
+ .../net/wireless/realtek/rtlwifi/rtl8723ae/def.h   |  25 --
+ .../wireless/realtek/rtlwifi/rtl8723ae/hal_btc.c   |   6 +-
+ .../wireless/realtek/rtlwifi/rtl8723ae/hal_btc.h   |  20 +-
+ .../net/wireless/realtek/rtlwifi/rtl8723ae/sw.c    |   1 -
+ .../net/wireless/realtek/rtlwifi/rtl8723ae/trx.c   |   7 -
+ .../net/wireless/realtek/rtlwifi/rtl8723ae/trx.h   |   3 -
+ .../net/wireless/realtek/rtlwifi/rtl8723be/def.h   |   8 -
+ .../net/wireless/realtek/rtlwifi/rtl8723be/fw.c    |  69 ----
+ .../net/wireless/realtek/rtlwifi/rtl8723be/fw.h    |  13 -
+ .../net/wireless/realtek/rtlwifi/rtl8723be/sw.c    |   2 -
+ .../net/wireless/realtek/rtlwifi/rtl8723be/trx.c   |  30 +-
+ .../net/wireless/realtek/rtlwifi/rtl8723be/trx.h   |   3 -
+ .../net/wireless/realtek/rtlwifi/rtl8821ae/def.h   |  33 --
+ .../net/wireless/realtek/rtlwifi/rtl8821ae/fw.c    |  69 +---
+ .../net/wireless/realtek/rtlwifi/rtl8821ae/fw.h    |  21 +-
+ .../net/wireless/realtek/rtlwifi/rtl8821ae/sw.c    |   3 +-
+ .../net/wireless/realtek/rtlwifi/rtl8821ae/trx.c   |  31 +-
+ .../net/wireless/realtek/rtlwifi/rtl8821ae/trx.h   |   3 -
+ drivers/net/wireless/realtek/rtlwifi/wifi.h        |  56 ++-
+ drivers/net/wireless/rsi/rsi_91x_mac80211.c        |   3 +-
+ drivers/net/wireless/rsi/rsi_91x_mgmt.c            |   1 +
+ drivers/net/wireless/ti/wlcore/sdio.c              |  21 +-
+ 133 files changed, 2282 insertions(+), 1157 deletions(-)
+ create mode 100644 drivers/net/wireless/ath/wcn36xx/testmode.c
+ create mode 100644 drivers/net/wireless/ath/wcn36xx/testmode.h
+ create mode 100644 drivers/net/wireless/ath/wcn36xx/testmode_i.h
+Merging bluetooth/master (1cd2fabf4bdc Bluetooth: btusb: Add additional device ID for RTL8822BE)
+$ git merge bluetooth/master
+Merge made by the 'recursive' strategy.
+ drivers/bluetooth/btmrvl_debugfs.c | 55 +++-----------------------------------
+ drivers/bluetooth/btmrvl_drv.h     |  2 --
+ drivers/bluetooth/btmrvl_main.c    |  6 -----
+ drivers/bluetooth/btmrvl_sdio.c    | 11 +++++---
+ drivers/bluetooth/btusb.c          | 43 ++++++++++++++++++++++++-----
+ drivers/bluetooth/hci_bcm.c        | 10 +------
+ drivers/bluetooth/hci_ldisc.c      | 22 ++++++++-------
+ drivers/bluetooth/hci_ll.c         |  3 ---
+ drivers/bluetooth/hci_nokia.c      |  3 ---
+ drivers/bluetooth/hci_qca.c        |  2 +-
+ drivers/bluetooth/hci_serdev.c     | 32 ++++++++++++++--------
+ drivers/bluetooth/hci_uart.h       |  1 +
+ net/bluetooth/hci_core.c           | 23 +++++-----------
+ net/bluetooth/hci_debugfs.c        | 24 ++++++-----------
+ net/bluetooth/smp.c                | 12 +++------
+ 15 files changed, 102 insertions(+), 147 deletions(-)
+Merging mac80211-next/master (f03329627b46 Merge branch 'hns3-next')
+$ git merge mac80211-next/master
+Already up to date.
+Merging gfs2/for-next (9a38662ba4e2 gfs2: Remove sdp->sd_jheightsize)
+$ git merge gfs2/for-next
+Merge made by the 'recursive' strategy.
+ fs/gfs2/bmap.c       | 14 +-------------
+ fs/gfs2/incore.h     |  2 --
+ fs/gfs2/ops_fstype.c | 19 -------------------
+ 3 files changed, 1 insertion(+), 34 deletions(-)
+Merging mtd/mtd/next (34c819073c8b mtd: cmdlinepart: Update comment for introduction of OFFSET_CONTINUOUS)
+$ git merge mtd/mtd/next
+Auto-merging drivers/mtd/mtdcore.c
+Auto-merging drivers/mtd/chips/cfi_cmdset_0002.c
+Merge made by the 'recursive' strategy.
+ Documentation/ABI/testing/sysfs-class-mtd          |  8 ++++
+ .../devicetree/bindings/mtd/partition.txt          |  2 +-
+ .../partitions/brcm,bcm947xx-cfe-partitions.txt    | 42 +++++++++++++++++++++
+ drivers/mtd/bcm47xxpart.c                          | 29 ++++++++++++--
+ drivers/mtd/chips/cfi_cmdset_0002.c                | 26 ++++++-------
+ drivers/mtd/chips/cfi_probe.c                      | 42 +++++++++++++++++++++
+ drivers/mtd/cmdlinepart.c                          |  5 ++-
+ drivers/mtd/devices/docg3.c                        |  3 +-
+ drivers/mtd/devices/mtd_dataflash.c                |  2 +-
+ drivers/mtd/inftlmount.c                           | 23 +++++++----
+ drivers/mtd/maps/pismo.c                           |  1 -
+ drivers/mtd/mtdcore.c                              | 24 +++++++-----
+ drivers/mtd/mtdcore.h                              |  1 -
+ drivers/mtd/mtdpart.c                              | 44 ++++++++--------------
+ drivers/mtd/nand/raw/Kconfig                       |  8 +---
+ drivers/mtd/nftlmount.c                            | 23 +++++++----
+ drivers/mtd/spi-nor/Kconfig                        |  2 +-
+ 17 files changed, 202 insertions(+), 83 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/mtd/partitions/brcm,bcm947xx-cfe-partitions.txt
+Merging nand/nand/next (782d1967d047 mtd: rawnand: Do not check FAIL bit when executing a SET_FEATURES op)
+$ git merge nand/nand/next
+Auto-merging include/linux/mtd/rawnand.h
+Auto-merging drivers/mtd/nand/raw/nand_base.c
+Auto-merging drivers/mtd/nand/raw/davinci_nand.c
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/mtd/gpmi-nand.txt          |   5 +
+ Documentation/devicetree/bindings/mtd/mtk-nand.txt |  24 ++-
+ .../devicetree/bindings/mtd/sunxi-nand.txt         |   2 -
+ MAINTAINERS                                        |   8 +
+ drivers/mtd/nand/onenand/samsung.c                 |   6 +-
+ drivers/mtd/nand/raw/davinci_nand.c                |  25 +--
+ drivers/mtd/nand/raw/diskonchip.c                  |   4 +-
+ drivers/mtd/nand/raw/fsl_elbc_nand.c               |  13 +-
+ drivers/mtd/nand/raw/fsl_ifc_nand.c                |  29 ++--
+ drivers/mtd/nand/raw/fsmc_nand.c                   |  27 +--
+ drivers/mtd/nand/raw/gpmi-nand/gpmi-lib.c          |  54 +++---
+ drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c         | 188 +++++++--------------
+ drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.h         |  25 +--
+ drivers/mtd/nand/raw/hisi504_nand.c                |  35 ++--
+ drivers/mtd/nand/raw/lpc32xx_mlc.c                 |  38 +++--
+ drivers/mtd/nand/raw/lpc32xx_slc.c                 |  26 +--
+ drivers/mtd/nand/raw/mtk_ecc.c                     |   7 +-
+ drivers/mtd/nand/raw/mtk_nand.c                    |  10 +-
+ drivers/mtd/nand/raw/nand_base.c                   |  89 +++++++---
+ drivers/mtd/nand/raw/sunxi_nand.c                  |  91 ++--------
+ include/linux/mtd/rawnand.h                        |  35 +++-
+ 21 files changed, 325 insertions(+), 416 deletions(-)
+Merging spi-nor/spi-nor/next (771ff17e82fa mtd: spi-nor: Add support for EN25QH32)
+$ git merge spi-nor/spi-nor/next
+Auto-merging drivers/mtd/spi-nor/cadence-quadspi.c
+Auto-merging drivers/mtd/spi-nor/Kconfig
+Merge made by the 'recursive' strategy.
+ drivers/mtd/spi-nor/Kconfig           |  4 +-
+ drivers/mtd/spi-nor/cadence-quadspi.c | 96 ++++++++++++++++++++++++++++++++++-
+ drivers/mtd/spi-nor/fsl-quadspi.c     | 15 +++++-
+ drivers/mtd/spi-nor/hisi-sfc.c        | 12 ++---
+ drivers/mtd/spi-nor/intel-spi.c       | 80 +++++++++++++++++++++++++----
+ drivers/mtd/spi-nor/spi-nor.c         | 33 ++++++++++++
+ drivers/mtd/spi-nor/stm32-quadspi.c   |  2 +-
+ include/linux/mtd/spi-nor.h           |  2 +
+ 8 files changed, 220 insertions(+), 24 deletions(-)
+Merging crypto/master (b268b3506d99 crypto: chtls - free beyond end rspq_skb_cache)
+$ git merge crypto/master
+Removing include/crypto/salsa20.h
+Auto-merging drivers/crypto/inside-secure/safexcel.c
+Auto-merging drivers/crypto/chelsio/chtls/chtls_main.c
+Auto-merging drivers/crypto/chelsio/chtls/chtls_io.c
+Auto-merging drivers/crypto/chelsio/chtls/chtls.h
+Removing arch/x86/crypto/salsa20_glue.c
+Removing arch/x86/crypto/salsa20-x86_64-asm_64.S
+Removing arch/x86/crypto/salsa20-i586-asm_32.S
+Merge made by the 'recursive' strategy.
+ arch/arm/crypto/sha1-armv4-large.S             |    10 +
+ arch/arm/crypto/sha256-armv4.pl                |    11 +-
+ arch/arm/crypto/sha256-core.S_shipped          |    11 +-
+ arch/arm/crypto/sha512-armv4.pl                |    11 +-
+ arch/arm/crypto/sha512-core.S_shipped          |    11 +-
+ arch/arm64/crypto/Kconfig                      |     6 +
+ arch/arm64/crypto/Makefile                     |     3 +
+ arch/arm64/crypto/aes-ce-ccm-core.S            |   150 +-
+ arch/arm64/crypto/aes-ce.S                     |    15 +-
+ arch/arm64/crypto/aes-modes.S                  |   331 +-
+ arch/arm64/crypto/aes-neonbs-core.S            |   305 +-
+ arch/arm64/crypto/crc32-ce-core.S              |    40 +-
+ arch/arm64/crypto/crct10dif-ce-core.S          |    32 +-
+ arch/arm64/crypto/ghash-ce-core.S              |   113 +-
+ arch/arm64/crypto/ghash-ce-glue.c              |    28 +-
+ arch/arm64/crypto/sha1-ce-core.S               |    42 +-
+ arch/arm64/crypto/sha2-ce-core.S               |    37 +-
+ arch/arm64/crypto/sha256-core.S_shipped        |    12 +-
+ arch/arm64/crypto/sha3-ce-core.S               |    77 +-
+ arch/arm64/crypto/sha512-armv8.pl              |    12 +-
+ arch/arm64/crypto/sha512-ce-core.S             |    27 +-
+ arch/arm64/crypto/sha512-core.S_shipped        |    12 +-
+ arch/arm64/crypto/sm4-ce-core.S                |    36 +
+ arch/arm64/crypto/sm4-ce-glue.c                |    73 +
+ arch/x86/crypto/Makefile                       |    25 +-
+ arch/x86/crypto/aegis128-aesni-asm.S           |   749 +
+ arch/x86/crypto/aegis128-aesni-glue.c          |   407 +
+ arch/x86/crypto/aegis128l-aesni-asm.S          |   825 +
+ arch/x86/crypto/aegis128l-aesni-glue.c         |   407 +
+ arch/x86/crypto/aegis256-aesni-asm.S           |   702 +
+ arch/x86/crypto/aegis256-aesni-glue.c          |   407 +
+ arch/x86/crypto/ghash-clmulni-intel_glue.c     |     2 +-
+ arch/x86/crypto/morus1280-avx2-asm.S           |   621 +
+ arch/x86/crypto/morus1280-avx2-glue.c          |    68 +
+ arch/x86/crypto/morus1280-sse2-asm.S           |   895 +
+ arch/x86/crypto/morus1280-sse2-glue.c          |    68 +
+ arch/x86/crypto/morus1280_glue.c               |   302 +
+ arch/x86/crypto/morus640-sse2-asm.S            |   614 +
+ arch/x86/crypto/morus640-sse2-glue.c           |    68 +
+ arch/x86/crypto/morus640_glue.c                |   298 +
+ arch/x86/crypto/salsa20-i586-asm_32.S          |   938 -
+ arch/x86/crypto/salsa20-x86_64-asm_64.S        |   805 -
+ arch/x86/crypto/salsa20_glue.c                 |    91 -
+ crypto/Kconfig                                 |   138 +-
+ crypto/Makefile                                |     6 +
+ crypto/aegis.h                                 |    80 +
+ crypto/aegis128.c                              |   463 +
+ crypto/aegis128l.c                             |   527 +
+ crypto/aegis256.c                              |   478 +
+ crypto/algapi.c                                |    10 +
+ crypto/authenc.c                               |     1 +
+ crypto/authencesn.c                            |     1 +
+ crypto/cfb.c                                   |     7 +-
+ crypto/cipher.c                                |     3 +-
+ crypto/crc32_generic.c                         |    15 +-
+ crypto/crc32c_generic.c                        |     8 +-
+ crypto/ctr.c                                   |     4 +-
+ crypto/cts.c                                   |     5 +-
+ crypto/ecc.c                                   |    66 +-
+ crypto/ecc.h                                   |     4 +-
+ crypto/ecdh.c                                  |     4 +-
+ crypto/morus1280.c                             |   549 +
+ crypto/morus640.c                              |   544 +
+ crypto/pcbc.c                                  |     5 +-
+ crypto/rsa.c                                   |     1 -
+ crypto/salsa20_generic.c                       |    20 +-
+ crypto/sm4_generic.c                           |    10 +-
+ crypto/tcrypt.c                                |   122 +-
+ crypto/testmgr.c                               |   613 +-
+ crypto/testmgr.h                               | 47127 ++++++++++-------------
+ crypto/zstd.c                                  |   265 +
+ drivers/char/hw_random/Kconfig                 |     1 +
+ drivers/char/hw_random/n2-drv.c                |     2 +-
+ drivers/char/hw_random/stm32-rng.c             |     9 +-
+ drivers/char/hw_random/via-rng.c               |     2 +-
+ drivers/crypto/Kconfig                         |    16 +-
+ drivers/crypto/amcc/crypto4xx_alg.c            |   231 +-
+ drivers/crypto/amcc/crypto4xx_core.c           |   317 +-
+ drivers/crypto/amcc/crypto4xx_core.h           |    35 +-
+ drivers/crypto/caam/caamalg.c                  |   231 +-
+ drivers/crypto/caam/caamalg_desc.c             |     4 +-
+ drivers/crypto/caam/caamalg_qi.c               |   227 +-
+ drivers/crypto/caam/caampkc.c                  |    71 +-
+ drivers/crypto/caam/caampkc.h                  |     8 +
+ drivers/crypto/caam/ctrl.c                     |    78 +-
+ drivers/crypto/caam/ctrl.h                     |     2 -
+ drivers/crypto/caam/intern.h                   |     1 +
+ drivers/crypto/caam/qi.c                       |     5 +-
+ drivers/crypto/caam/regs.h                     |     6 +
+ drivers/crypto/cavium/zip/common.h             |    21 +
+ drivers/crypto/cavium/zip/zip_crypto.c         |    22 +-
+ drivers/crypto/cavium/zip/zip_deflate.c        |     4 +-
+ drivers/crypto/cavium/zip/zip_device.c         |     4 +-
+ drivers/crypto/cavium/zip/zip_inflate.c        |     4 +-
+ drivers/crypto/cavium/zip/zip_main.c           |    24 +-
+ drivers/crypto/cavium/zip/zip_main.h           |     1 -
+ drivers/crypto/cavium/zip/zip_regs.h           |    42 +-
+ drivers/crypto/ccp/psp-dev.c                   |   143 +-
+ drivers/crypto/ccp/psp-dev.h                   |     4 +
+ drivers/crypto/ccree/cc_cipher.c               |   350 +-
+ drivers/crypto/ccree/cc_cipher.h               |    30 +-
+ drivers/crypto/ccree/cc_debugfs.c              |     7 +-
+ drivers/crypto/ccree/cc_driver.c               |    36 +-
+ drivers/crypto/ccree/cc_driver.h               |     2 +
+ drivers/crypto/ccree/cc_host_regs.h            |     6 +-
+ drivers/crypto/chelsio/chcr_algo.c             |   303 +-
+ drivers/crypto/chelsio/chcr_algo.h             |     3 +-
+ drivers/crypto/chelsio/chcr_core.h             |     2 +-
+ drivers/crypto/chelsio/chcr_crypto.h           |    15 +-
+ drivers/crypto/chelsio/chcr_ipsec.c            |    35 +-
+ drivers/crypto/chelsio/chtls/chtls.h           |    10 +-
+ drivers/crypto/chelsio/chtls/chtls_cm.c        |    28 +-
+ drivers/crypto/chelsio/chtls/chtls_io.c        |    54 +-
+ drivers/crypto/chelsio/chtls/chtls_main.c      |    13 +-
+ drivers/crypto/exynos-rng.c                    |     6 +-
+ drivers/crypto/inside-secure/safexcel.c        |    32 +
+ drivers/crypto/inside-secure/safexcel.h        |    44 +-
+ drivers/crypto/inside-secure/safexcel_cipher.c |   642 +-
+ drivers/crypto/inside-secure/safexcel_hash.c   |    23 +-
+ drivers/crypto/nx/nx-842-powernv.c             |     2 +-
+ drivers/crypto/omap-sham.c                     |     2 +-
+ drivers/crypto/picoxcell_crypto.c              |     6 +-
+ drivers/crypto/qat/qat_c3xxx/adf_drv.c         |     2 +
+ drivers/crypto/qat/qat_c62x/adf_drv.c          |     2 +
+ drivers/crypto/qat/qat_dh895xcc/adf_drv.c      |     1 +
+ drivers/crypto/vmx/aes.c                       |     2 -
+ drivers/crypto/vmx/aes_cbc.c                   |     3 -
+ drivers/crypto/vmx/aes_ctr.c                   |     2 -
+ drivers/crypto/vmx/aes_xts.c                   |     2 -
+ drivers/crypto/vmx/aesp8-ppc.pl                |    49 +-
+ drivers/crypto/vmx/ghash.c                     |     2 -
+ drivers/crypto/vmx/ghashp8-ppc.pl              |     9 +
+ include/crypto/algapi.h                        |     8 +
+ include/crypto/morus1280_glue.h                |   137 +
+ include/crypto/morus640_glue.h                 |   137 +
+ include/crypto/morus_common.h                  |    23 +
+ include/crypto/salsa20.h                       |    27 -
+ include/crypto/sm4.h                           |     3 +
+ include/linux/psp-sev.h                        |    23 +
+ include/uapi/linux/psp-sev.h                   |    12 +
+ 140 files changed, 34804 insertions(+), 29579 deletions(-)
+ create mode 100644 arch/arm64/crypto/sm4-ce-core.S
+ create mode 100644 arch/arm64/crypto/sm4-ce-glue.c
+ create mode 100644 arch/x86/crypto/aegis128-aesni-asm.S
+ create mode 100644 arch/x86/crypto/aegis128-aesni-glue.c
+ create mode 100644 arch/x86/crypto/aegis128l-aesni-asm.S
+ create mode 100644 arch/x86/crypto/aegis128l-aesni-glue.c
+ create mode 100644 arch/x86/crypto/aegis256-aesni-asm.S
+ create mode 100644 arch/x86/crypto/aegis256-aesni-glue.c
+ create mode 100644 arch/x86/crypto/morus1280-avx2-asm.S
+ create mode 100644 arch/x86/crypto/morus1280-avx2-glue.c
+ create mode 100644 arch/x86/crypto/morus1280-sse2-asm.S
+ create mode 100644 arch/x86/crypto/morus1280-sse2-glue.c
+ create mode 100644 arch/x86/crypto/morus1280_glue.c
+ create mode 100644 arch/x86/crypto/morus640-sse2-asm.S
+ create mode 100644 arch/x86/crypto/morus640-sse2-glue.c
+ create mode 100644 arch/x86/crypto/morus640_glue.c
+ delete mode 100644 arch/x86/crypto/salsa20-i586-asm_32.S
+ delete mode 100644 arch/x86/crypto/salsa20-x86_64-asm_64.S
+ delete mode 100644 arch/x86/crypto/salsa20_glue.c
+ create mode 100644 crypto/aegis.h
+ create mode 100644 crypto/aegis128.c
+ create mode 100644 crypto/aegis128l.c
+ create mode 100644 crypto/aegis256.c
+ create mode 100644 crypto/morus1280.c
+ create mode 100644 crypto/morus640.c
+ create mode 100644 crypto/zstd.c
+ create mode 100644 include/crypto/morus1280_glue.h
+ create mode 100644 include/crypto/morus640_glue.h
+ create mode 100644 include/crypto/morus_common.h
+ delete mode 100644 include/crypto/salsa20.h
+Merging drm/drm-next (568cf2e6aa0c Merge tag 'drm-amdkfd-next-2018-05-28' of git://people.freedesktop.org/~gabbayo/linux into drm-next)
+$ git merge drm/drm-next
+Auto-merging drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+Auto-merging drivers/gpu/drm/vc4/vc4_drv.c
+Auto-merging drivers/gpu/drm/scheduler/gpu_scheduler_trace.h
+Auto-merging drivers/gpu/drm/rcar-du/rcar_du_vsp.c
+Auto-merging drivers/gpu/drm/rcar-du/rcar_du_crtc.h
+Auto-merging drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+Auto-merging drivers/gpu/drm/nouveau/nvkm/engine/dma/gv100.c
+Removing drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygt215.c
+Removing drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgt215.c
+Removing drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgk104.c
+Removing drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h
+Removing drivers/gpu/drm/nouveau/nvkm/engine/disp/coregt215.c
+Removing drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp100.c
+Removing drivers/gpu/drm/nouveau/nvkm/engine/disp/coregm200.c
+Removing drivers/gpu/drm/nouveau/nvkm/engine/disp/coregm107.c
+Removing drivers/gpu/drm/nouveau/nvkm/engine/disp/coregk110.c
+Auto-merging drivers/gpu/drm/nouveau/nvkm/engine/disp/changv100.c
+Auto-merging drivers/gpu/drm/nouveau/nvkm/engine/ce/gv100.c
+Auto-merging drivers/gpu/drm/nouveau/nvif/userc361.c
+Auto-merging drivers/gpu/drm/nouveau/nvif/disp.c
+Removing drivers/gpu/drm/nouveau/nv50_display.c
+Auto-merging drivers/gpu/drm/nouveau/dispnv50/sorc37d.c
+Auto-merging drivers/gpu/drm/nouveau/dispnv50/sor907d.c
+Auto-merging drivers/gpu/drm/nouveau/dispnv50/dac907d.c
+Auto-merging drivers/gpu/drm/nouveau/dispnv50/curs907a.c
+Auto-merging drivers/gpu/drm/nouveau/dispnv50/core917d.c
+Auto-merging drivers/gpu/drm/nouveau/dispnv50/core907d.c
+Auto-merging drivers/gpu/drm/i915/selftests/i915_timeline.c
+Auto-merging drivers/gpu/drm/i915/intel_lrc.c
+Auto-merging drivers/gpu/drm/i915/intel_engine_cs.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/i915/intel_engine_cs.c
+Auto-merging drivers/gpu/drm/i915/i915_timeline.h
+Auto-merging drivers/gpu/drm/i915/i915_reg.h
+Removing drivers/gpu/drm/i915/i915_gem_timeline.c
+Removing drivers/gpu/drm/exynos/exynos_drm_gsc.h
+Removing drivers/gpu/drm/exynos/exynos_drm_fimc.h
+Auto-merging drivers/gpu/drm/drm_drv.c
+Auto-merging drivers/gpu/drm/drm_dp_helper.c
+Removing drivers/gpu/drm/amd/display/include/fixed32_32.h
+Removing drivers/gpu/drm/amd/display/dc/basics/fixpt32_32.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/df_v3_6.h
+Auto-merging drivers/gpu/drm/amd/amdgpu/df_v1_7.h
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+Auto-merging MAINTAINERS
+Resolved 'drivers/gpu/drm/i915/intel_engine_cs.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master 03a8ae4422e4] Merge remote-tracking branch 'drm/drm-next'
+$ git diff -M --stat --summary HEAD^..
+ .../bindings/display/bridge/adi,adv7511.txt        |   18 +-
+ .../bindings/display/bridge/cdns,dsi.txt           |  133 +
+ .../bindings/display/bridge/renesas,dw-hdmi.txt    |    1 +
+ .../devicetree/bindings/display/bridge/tda998x.txt |    3 +
+ .../bindings/display/bridge/thine,thc63lvd1024.txt |   60 +
+ .../bindings/display/exynos/exynos5433-decon.txt   |    9 +-
+ .../devicetree/bindings/display/renesas,du.txt     |   28 +-
+ .../bindings/display/sunxi/sun6i-dsi.txt           |   93 +
+ .../devicetree/bindings/gpu/brcm,bcm-v3d.txt       |   28 +
+ .../devicetree/bindings/gpu/samsung-scaler.txt     |   27 +
+ Documentation/gpu/drivers.rst                      |    2 +
+ Documentation/gpu/i915.rst                         |  141 +-
+ Documentation/gpu/kms-properties.csv               |    1 -
+ Documentation/gpu/todo.rst                         |   18 +
+ Documentation/gpu/xen-front.rst                    |   31 +
+ MAINTAINERS                                        |   21 +-
+ drivers/dma-buf/sync_debug.h                       |   10 -
+ drivers/gpu/drm/Kconfig                            |   13 +-
+ drivers/gpu/drm/Makefile                           |    4 +-
+ drivers/gpu/drm/amd/amdgpu/Makefile                |   24 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu.h                |   45 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c            |   11 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c         |   87 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h         |   25 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c  |   10 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c  |   10 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c  | 1043 +++++
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c   |  584 ++-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c   |   44 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h   |    1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c   |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c      |   15 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c            |  247 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c     |    8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c             |   18 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c            |   81 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c        |  195 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |  121 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_display.c        |   40 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_display.h        |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c            |   20 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h            |   11 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            |   21 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c             |   12 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c          |   11 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c           |   17 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c            |   50 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c             |    8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c            |   70 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c             |  111 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h             |   11 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h           |    1 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c         |  120 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.h         |   38 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c             |  365 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c          |   14 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c            |    1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c      |   14 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c           |   21 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h           |   13 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_test.c           |   18 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h          |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c            |  496 ++-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h            |    1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c          |   54 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h          |   22 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c            |  292 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h            |   19 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c            |   40 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c            |   25 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h            |   11 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c             |  273 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h             |   15 +-
+ drivers/gpu/drm/amd/amdgpu/ci_dpm.c                |    4 +-
+ drivers/gpu/drm/amd/amdgpu/cik.c                   |    7 +
+ drivers/gpu/drm/amd/amdgpu/dce_v10_0.c             |   17 +-
+ drivers/gpu/drm/amd/amdgpu/dce_v11_0.c             |   30 +-
+ drivers/gpu/drm/amd/amdgpu/dce_v6_0.c              |   17 +-
+ drivers/gpu/drm/amd/amdgpu/dce_v8_0.c              |   17 +-
+ drivers/gpu/drm/amd/amdgpu/dce_virtual.c           |   10 +-
+ drivers/gpu/drm/amd/amdgpu/df_v1_7.c               |  120 +
+ .../disp/basegk104.c => amd/amdgpu/df_v1_7.h}      |   30 +-
+ drivers/gpu/drm/amd/amdgpu/df_v3_6.c               |  116 +
+ .../disp/basegk110.c => amd/amdgpu/df_v3_6.h}      |   30 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c              |  102 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c              |  344 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c              |   25 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c              |   25 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c              |   33 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c              |  146 +-
+ drivers/gpu/drm/amd/amdgpu/kv_dpm.c                |    4 +-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c            |    1 +
+ drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c              |    4 +-
+ drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c             |   18 +-
+ drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h            |   67 +-
+ drivers/gpu/drm/amd/amdgpu/psp_v10_0.c             |    9 +
+ drivers/gpu/drm/amd/amdgpu/psp_v3_1.c              |    3 +
+ drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c             |   12 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c             |   96 +-
+ drivers/gpu/drm/amd/amdgpu/si.c                    |    7 +
+ drivers/gpu/drm/amd/amdgpu/si_dpm.c                |    2 +-
+ drivers/gpu/drm/amd/amdgpu/soc15.c                 |  125 +-
+ drivers/gpu/drm/amd/amdgpu/soc15.h                 |    1 +
+ drivers/gpu/drm/amd/amdgpu/soc15_common.h          |   15 +
+ drivers/gpu/drm/amd/amdgpu/soc15d.h                |    6 +
+ drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c              |   46 +-
+ drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c              |   40 +-
+ drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c              |  112 +-
+ drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c              | 1073 ++---
+ drivers/gpu/drm/amd/amdgpu/vce_v3_0.c              |    7 +-
+ drivers/gpu/drm/amd/amdgpu/vce_v4_0.c              |    1 +
+ drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c              |  188 +-
+ drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c       |   53 +
+ drivers/gpu/drm/amd/amdgpu/vi.c                    |  130 +-
+ drivers/gpu/drm/amd/amdkfd/Makefile                |   10 +-
+ drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c   |   20 +-
+ drivers/gpu/drm/amd/amdkfd/cik_regs.h              |    3 +-
+ drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h     |  560 +++
+ .../gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm  |  274 +-
+ .../gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm  | 1214 ++++++
+ drivers/gpu/drm/amd/amdkfd/kfd_chardev.c           |   52 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_crat.c              |   11 +
+ drivers/gpu/drm/amd/amdkfd/kfd_device.c            |  131 +-
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  |  114 +-
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h  |    2 +
+ .../drm/amd/amdkfd/kfd_device_queue_manager_v9.c   |   84 +
+ drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c          |   65 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_events.c            |    4 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c       |  119 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c    |   92 +
+ drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c         |    8 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c      |   39 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h      |    7 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_cik.c  |    9 +
+ drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c   |  340 ++
+ drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c   |  319 ++
+ drivers/gpu/drm/amd/amdkfd/kfd_module.c            |    7 +
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c       |    3 +
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c   |    6 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c    |  443 ++
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c    |    2 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c    |  392 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h    |  583 +++
+ drivers/gpu/drm/amd/amdkfd/kfd_priv.h              |  112 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_process.c           |   50 +-
+ .../gpu/drm/amd/amdkfd/kfd_process_queue_manager.c |   22 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_queue.c             |    8 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_topology.c          |    6 +
+ drivers/gpu/drm/amd/amdkfd/kfd_topology.h          |    1 +
+ drivers/gpu/drm/amd/amdkfd/soc15_int.h             |   47 +
+ drivers/gpu/drm/amd/display/Kconfig                |    8 -
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  |  226 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h  |    5 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_color.c    |   15 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c  |    5 -
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_services.c |   13 +-
+ drivers/gpu/drm/amd/display/dc/basics/Makefile     |    2 +-
+ drivers/gpu/drm/amd/display/dc/basics/conversion.c |   28 +-
+ drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c |  376 +-
+ drivers/gpu/drm/amd/display/dc/basics/fixpt32_32.c |  161 -
+ .../gpu/drm/amd/display/dc/basics/log_helpers.c    |    1 -
+ drivers/gpu/drm/amd/display/dc/basics/logger.c     |    3 +-
+ drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c |    3 +
+ .../gpu/drm/amd/display/dc/bios/command_table.c    |   22 +-
+ .../gpu/drm/amd/display/dc/bios/command_table2.c   |   26 +-
+ .../drm/amd/display/dc/bios/command_table_helper.c |    1 +
+ .../amd/display/dc/bios/command_table_helper2.c    |    1 +
+ .../gpu/drm/amd/display/dc/calcs/calcs_logger.h    |  579 +++
+ .../gpu/drm/amd/display/dc/calcs/custom_float.c    |   46 +-
+ drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c   |  256 +-
+ drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c   |   82 +-
+ drivers/gpu/drm/amd/display/dc/core/dc.c           |  112 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_debug.c     |   38 +-
+ .../gpu/drm/amd/display/dc/core/dc_hw_sequencer.c  |   24 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_link.c      |  170 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c   |   30 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c |    2 -
+ drivers/gpu/drm/amd/display/dc/core/dc_resource.c  |  345 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_stream.c    |   24 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_surface.c   |    8 +-
+ drivers/gpu/drm/amd/display/dc/dc.h                |   46 +-
+ drivers/gpu/drm/amd/display/dc/dc_dp_types.h       |    2 +
+ drivers/gpu/drm/amd/display/dc/dc_helper.c         |   59 +
+ drivers/gpu/drm/amd/display/dc/dc_hw_types.h       |    5 +
+ drivers/gpu/drm/amd/display/dc/dc_link.h           |    9 +
+ drivers/gpu/drm/amd/display/dc/dc_stream.h         |   28 +-
+ drivers/gpu/drm/amd/display/dc/dc_types.h          |   17 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_abm.c       |    2 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_audio.c     |    9 +-
+ .../gpu/drm/amd/display/dc/dce/dce_clock_source.c  |   73 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c    |   33 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c      |    2 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.c     |    3 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c       |    6 +-
+ .../gpu/drm/amd/display/dc/dce/dce_link_encoder.c  |    6 +-
+ .../gpu/drm/amd/display/dc/dce/dce_link_encoder.h  |    2 +
+ drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c |  103 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h |   17 +
+ .../gpu/drm/amd/display/dc/dce/dce_scl_filters.c   |   48 +-
+ .../drm/amd/display/dc/dce/dce_stream_encoder.c    |   76 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_transform.c |   26 +-
+ .../drm/amd/display/dc/dce100/dce100_resource.c    |   33 -
+ .../amd/display/dc/dce110/dce110_hw_sequencer.c    |  200 +-
+ .../drm/amd/display/dc/dce110/dce110_mem_input_v.c |    1 +
+ .../drm/amd/display/dc/dce110/dce110_resource.c    |   33 -
+ .../display/dc/dce110/dce110_timing_generator.c    |   16 +-
+ .../display/dc/dce110/dce110_timing_generator_v.c  |    7 -
+ .../drm/amd/display/dc/dce110/dce110_transform_v.c |    8 +-
+ .../drm/amd/display/dc/dce112/dce112_resource.c    |   35 +-
+ .../drm/amd/display/dc/dce112/dce112_resource.h    |    5 -
+ .../drm/amd/display/dc/dce120/dce120_resource.c    |   50 +-
+ .../display/dc/dce120/dce120_timing_generator.c    |   12 +-
+ .../gpu/drm/amd/display/dc/dce80/dce80_resource.c  |   49 -
+ drivers/gpu/drm/amd/display/dc/dcn10/Makefile      |    2 +-
+ .../gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c |   98 +-
+ drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c   |   43 +-
+ drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h   |   76 +-
+ .../gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c    |   96 +-
+ .../gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c  |   38 +-
+ .../gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c    |  229 +-
+ .../gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h    |   14 +-
+ drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c  |  215 +-
+ drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h  |   82 +-
+ .../drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c  |  435 +-
+ .../drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h  |    2 +
+ .../drm/amd/display/dc/dcn10/dcn10_link_encoder.c  | 1362 ++++++
+ .../drm/amd/display/dc/dcn10/dcn10_link_encoder.h  |  330 ++
+ drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c   |   24 +-
+ drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.h   |    5 +
+ drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c  |  104 +-
+ drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h  |   26 +
+ .../gpu/drm/amd/display/dc/dcn10/dcn10_resource.c  |  349 +-
+ .../amd/display/dc/dcn10/dcn10_stream_encoder.c    | 1490 +++++++
+ .../amd/display/dc/dcn10/dcn10_stream_encoder.h    |  524 +++
+ drivers/gpu/drm/amd/display/dc/dm_services.h       |    8 +-
+ .../drm/amd/display/dc/dml/display_mode_enums.h    |   13 +
+ .../gpu/drm/amd/display/dc/dml/display_mode_lib.c  |  138 +-
+ .../drm/amd/display/dc/dml/display_mode_structs.h  |  969 +++--
+ .../gpu/drm/amd/display/dc/dml/dml_inline_defs.h   |   10 +
+ drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c   |    1 +
+ drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c |    1 +
+ .../dc/i2caux/dce110/i2c_hw_engine_dce110.c        |    2 +-
+ drivers/gpu/drm/amd/display/dc/i2caux/i2caux.c     |    1 +
+ drivers/gpu/drm/amd/display/dc/inc/core_types.h    |    6 +-
+ drivers/gpu/drm/amd/display/dc/inc/dce_calcs.h     |    2 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h   |   64 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h        |   18 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h       |    4 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h  |   10 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/ipp.h        |    2 +-
+ .../gpu/drm/amd/display/dc/inc/hw/link_encoder.h   |    1 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h  |    1 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h        |   17 +
+ .../gpu/drm/amd/display/dc/inc/hw/stream_encoder.h |   36 +-
+ .../drm/amd/display/dc/inc/hw/timing_generator.h   |    3 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/transform.h  |    2 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h  |   22 +-
+ drivers/gpu/drm/amd/display/dc/inc/reg_helper.h    |   56 +
+ drivers/gpu/drm/amd/display/dc/inc/resource.h      |    4 -
+ drivers/gpu/drm/amd/display/dc/irq_types.h         |    9 +
+ drivers/gpu/drm/amd/display/include/dal_asic_id.h  |   11 +-
+ drivers/gpu/drm/amd/display/include/dal_types.h    |    1 +
+ drivers/gpu/drm/amd/display/include/fixed31_32.h   |  272 +-
+ drivers/gpu/drm/amd/display/include/fixed32_32.h   |  129 -
+ .../gpu/drm/amd/display/include/logger_interface.h |    9 +
+ drivers/gpu/drm/amd/display/include/logger_types.h |   66 +-
+ .../drm/amd/display/modules/color/color_gamma.c    |  692 ++-
+ .../drm/amd/display/modules/color/color_gamma.h    |   48 +-
+ .../gpu/drm/amd/display/modules/inc/mod_stats.h    |    4 +
+ drivers/gpu/drm/amd/display/modules/stats/stats.c  |  254 +-
+ drivers/gpu/drm/amd/include/amd_shared.h           |   23 +-
+ .../drm/amd/include/asic_reg/dce/dce_12_0_offset.h |   12 +
+ .../amd/include/asic_reg/dce/dce_12_0_sh_mask.h    |  152 +
+ .../drm/amd/include/asic_reg/dcn/dcn_1_0_offset.h  |   19 +-
+ .../drm/amd/include/asic_reg/dcn/dcn_1_0_sh_mask.h |    8 +
+ .../drm/amd/include/asic_reg/df/df_1_7_default.h   |   26 +
+ .../drm/amd/include/asic_reg/df/df_1_7_offset.h    |   37 +
+ .../drm/amd/include/asic_reg/df/df_1_7_sh_mask.h   |   52 +
+ .../drm/amd/include/asic_reg/df/df_3_6_default.h   |   26 +
+ .../drm/amd/include/asic_reg/df/df_3_6_offset.h    |   33 +
+ .../drm/amd/include/asic_reg/df/df_3_6_sh_mask.h   |   48 +
+ drivers/gpu/drm/amd/include/atombios.h             |    7 +
+ drivers/gpu/drm/amd/include/atomfirmware.h         |   35 +
+ drivers/gpu/drm/amd/include/cgs_common.h           |  170 -
+ drivers/gpu/drm/amd/include/kgd_kfd_interface.h    |   26 +-
+ drivers/gpu/drm/amd/include/kgd_pp_interface.h     |   13 +-
+ drivers/gpu/drm/amd/include/soc15_ih_clientid.h    |    1 +
+ drivers/gpu/drm/amd/include/v9_structs.h           |   48 +-
+ drivers/gpu/drm/amd/include/vega20_ip_offset.h     | 1051 +++++
+ drivers/gpu/drm/amd/powerplay/amd_powerplay.c      |  489 +--
+ .../gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c  |   41 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c        |   91 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c       |   11 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c   |  222 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h   |   15 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c |   99 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h |    2 +
+ .../amd/powerplay/hwmgr/process_pptables_v1_0.c    |   39 +-
+ .../gpu/drm/amd/powerplay/hwmgr/processpptables.c  |    4 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c  |  208 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.h  |    3 +-
+ .../amd/powerplay/hwmgr/smu7_clockpowergating.c    |   16 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c   |  330 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.h   |    2 +
+ .../gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c   |  229 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c   |   37 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c   |   98 +
+ drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.h   |   27 +
+ drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c |  951 ++--
+ drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h |   26 +-
+ .../gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c |  121 +-
+ .../amd/powerplay/hwmgr/vega10_processpptables.c   |    2 +-
+ .../gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c   |  107 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c |   95 +-
+ drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h |    2 +-
+ .../amd/powerplay/hwmgr/vega12_processpptables.c   |    7 +-
+ .../gpu/drm/amd/powerplay/hwmgr/vega12_thermal.c   |   37 +-
+ .../gpu/drm/amd/powerplay/inc/hardwaremanager.h    |    9 +-
+ drivers/gpu/drm/amd/powerplay/inc/hwmgr.h          |   35 +-
+ drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h       |    4 +-
+ drivers/gpu/drm/amd/powerplay/inc/smu75.h          |  760 ++++
+ drivers/gpu/drm/amd/powerplay/inc/smu75_discrete.h |  886 ++++
+ drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h   |    1 +
+ drivers/gpu/drm/amd/powerplay/smumgr/Makefile      |    2 +-
+ drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c   |   12 +-
+ drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c |   24 +-
+ .../gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c  |    9 +-
+ .../drm/amd/powerplay/smumgr/polaris10_smumgr.c    |   25 +-
+ .../gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c    |   39 +-
+ drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c |   46 +-
+ drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h |    1 -
+ drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c      |    2 +
+ .../gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c    |   19 +-
+ .../gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c   |   52 +-
+ .../gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c   |   56 +-
+ .../gpu/drm/amd/powerplay/smumgr/vegam_smumgr.c    | 2383 ++++++++++
+ .../gpu/drm/amd/powerplay/smumgr/vegam_smumgr.h    |   75 +
+ drivers/gpu/drm/ast/ast_mode.c                     |    2 +-
+ drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h       |   13 -
+ drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c    |   89 +-
+ drivers/gpu/drm/bochs/bochs_kms.c                  |    2 +-
+ drivers/gpu/drm/bridge/Kconfig                     |   16 +
+ drivers/gpu/drm/bridge/Makefile                    |    2 +
+ drivers/gpu/drm/bridge/adv7511/Kconfig             |    2 +-
+ drivers/gpu/drm/bridge/adv7511/adv7511.h           |    6 +
+ drivers/gpu/drm/bridge/adv7511/adv7511_drv.c       |   46 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_core.c |  331 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_core.h |    5 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c  |  236 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h  |    7 +
+ drivers/gpu/drm/bridge/cdns-dsi.c                  | 1623 +++++++
+ .../gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c    |    1 -
+ drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c      |   18 +-
+ drivers/gpu/drm/bridge/tc358767.c                  |    2 +-
+ drivers/gpu/drm/bridge/thc63lvd1024.c              |  206 +
+ drivers/gpu/drm/drm_atomic.c                       |   26 +-
+ drivers/gpu/drm/drm_atomic_helper.c                |  109 +-
+ drivers/gpu/drm/drm_blend.c                        |   39 +
+ drivers/gpu/drm/drm_connector.c                    |   50 +-
+ drivers/gpu/drm/drm_crtc.c                         |   54 +-
+ drivers/gpu/drm/drm_crtc_internal.h                |    2 +
+ drivers/gpu/drm/drm_dp_helper.c                    |   22 +-
+ drivers/gpu/drm/drm_dp_mst_topology.c              |    9 +-
+ drivers/gpu/drm/drm_drv.c                          |   64 +-
+ drivers/gpu/drm/drm_edid.c                         |   43 +-
+ drivers/gpu/drm/drm_fb_helper.c                    |   12 +-
+ drivers/gpu/drm/drm_framebuffer.c                  |   33 +-
+ drivers/gpu/drm/drm_gem.c                          |    9 +-
+ drivers/gpu/drm/drm_gem_framebuffer_helper.c       |   19 +
+ drivers/gpu/drm/drm_ioc32.c                        |    4 +-
+ drivers/gpu/drm/drm_ioctl.c                        |   85 +-
+ drivers/gpu/drm/drm_lease.c                        |    2 +-
+ drivers/gpu/drm/drm_modes.c                        |  179 +-
+ drivers/gpu/drm/drm_panel_orientation_quirks.c     |   29 +-
+ drivers/gpu/drm/drm_plane.c                        |   54 +-
+ drivers/gpu/drm/drm_prime.c                        |   21 +-
+ drivers/gpu/drm/drm_property.c                     |   27 +-
+ drivers/gpu/drm/drm_rect.c                         |   74 +-
+ drivers/gpu/drm/drm_scdc_helper.c                  |   10 +-
+ drivers/gpu/drm/drm_simple_kms_helper.c            |    4 +-
+ drivers/gpu/drm/drm_sysfs.c                        |    4 +-
+ drivers/gpu/drm/etnaviv/Kconfig                    |    8 -
+ drivers/gpu/drm/etnaviv/etnaviv_buffer.c           |   16 +-
+ drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c       |   15 +-
+ drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c           |   15 +-
+ drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h           |   13 +-
+ drivers/gpu/drm/etnaviv/etnaviv_drv.c              |   68 +-
+ drivers/gpu/drm/etnaviv/etnaviv_drv.h              |   45 +-
+ drivers/gpu/drm/etnaviv/etnaviv_dump.c             |   15 +-
+ drivers/gpu/drm/etnaviv/etnaviv_dump.h             |   16 +-
+ drivers/gpu/drm/etnaviv/etnaviv_gem.c              |   15 +-
+ drivers/gpu/drm/etnaviv/etnaviv_gem.h              |   15 +-
+ drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c        |   16 +-
+ drivers/gpu/drm/etnaviv/etnaviv_gpu.c              |   19 +-
+ drivers/gpu/drm/etnaviv/etnaviv_gpu.h              |   19 +-
+ drivers/gpu/drm/etnaviv/etnaviv_hwdb.c             |   13 +-
+ drivers/gpu/drm/etnaviv/etnaviv_iommu.c            |   49 +-
+ drivers/gpu/drm/etnaviv/etnaviv_iommu.h            |   15 +-
+ drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c         |  143 +-
+ drivers/gpu/drm/etnaviv/etnaviv_mmu.c              |   31 +-
+ drivers/gpu/drm/etnaviv/etnaviv_mmu.h              |   16 +-
+ drivers/gpu/drm/etnaviv/etnaviv_perfmon.c          |   13 +-
+ drivers/gpu/drm/etnaviv/etnaviv_perfmon.h          |   13 +-
+ drivers/gpu/drm/etnaviv/etnaviv_sched.c            |   13 +-
+ drivers/gpu/drm/etnaviv/etnaviv_sched.h            |   13 +-
+ drivers/gpu/drm/exynos/Kconfig                     |   18 +-
+ drivers/gpu/drm/exynos/Makefile                    |    2 +
+ drivers/gpu/drm/exynos/exynos5433_drm_decon.c      |   21 +-
+ drivers/gpu/drm/exynos/exynos_dp.c                 |    2 +-
+ drivers/gpu/drm/exynos/exynos_drm_crtc.c           |    2 +-
+ drivers/gpu/drm/exynos/exynos_drm_drv.c            |   55 +-
+ drivers/gpu/drm/exynos/exynos_drm_drv.h            |   11 +-
+ drivers/gpu/drm/exynos/exynos_drm_dsi.c            |   46 +-
+ drivers/gpu/drm/exynos/exynos_drm_fb.c             |    4 +-
+ drivers/gpu/drm/exynos/exynos_drm_fimc.c           | 1080 ++---
+ drivers/gpu/drm/exynos/exynos_drm_fimc.h           |   23 -
+ drivers/gpu/drm/exynos/exynos_drm_fimd.c           |    8 +
+ drivers/gpu/drm/exynos/exynos_drm_gem.c            |   21 +-
+ drivers/gpu/drm/exynos/exynos_drm_gem.h            |    3 +-
+ drivers/gpu/drm/exynos/exynos_drm_gsc.c            | 1075 ++---
+ drivers/gpu/drm/exynos/exynos_drm_gsc.h            |   24 -
+ drivers/gpu/drm/exynos/exynos_drm_ipp.c            |  916 ++++
+ drivers/gpu/drm/exynos/exynos_drm_ipp.h            |  175 +
+ drivers/gpu/drm/exynos/exynos_drm_plane.c          |    9 +-
+ drivers/gpu/drm/exynos/exynos_drm_rotator.c        |  758 +---
+ drivers/gpu/drm/exynos/exynos_drm_scaler.c         |  694 +++
+ drivers/gpu/drm/exynos/regs-scaler.h               |  426 ++
+ drivers/gpu/drm/gma500/cdv_device.c                |    4 +-
+ drivers/gpu/drm/gma500/cdv_intel_crt.c             |    2 +-
+ drivers/gpu/drm/gma500/cdv_intel_dp.c              |    2 +-
+ drivers/gpu/drm/gma500/cdv_intel_hdmi.c            |    2 +-
+ drivers/gpu/drm/gma500/cdv_intel_lvds.c            |    2 +-
+ drivers/gpu/drm/gma500/mdfld_dsi_output.c          |    2 +-
+ drivers/gpu/drm/gma500/oaktrail_hdmi.c             |    2 +-
+ drivers/gpu/drm/gma500/psb_intel_drv.h             |    2 +-
+ drivers/gpu/drm/gma500/psb_intel_lvds.c            |    2 +-
+ drivers/gpu/drm/gma500/psb_intel_sdvo.c            |    4 +-
+ drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c   |    2 +-
+ drivers/gpu/drm/i2c/Kconfig                        |    6 +
+ drivers/gpu/drm/i2c/Makefile                       |    1 +
+ drivers/gpu/drm/i2c/tda9950.c                      |  509 +++
+ drivers/gpu/drm/i2c/tda998x_drv.c                  |  244 +-
+ drivers/gpu/drm/i915/Kconfig.debug                 |   15 +-
+ drivers/gpu/drm/i915/Makefile                      |   20 +-
+ drivers/gpu/drm/i915/gvt/cmd_parser.c              |   81 +-
+ drivers/gpu/drm/i915/gvt/debugfs.c                 |   72 +-
+ drivers/gpu/drm/i915/gvt/gvt.h                     |    2 +-
+ drivers/gpu/drm/i915/gvt/handlers.c                |   35 +-
+ drivers/gpu/drm/i915/gvt/mmio.c                    |    2 -
+ drivers/gpu/drm/i915/gvt/mmio_context.c            |    2 +-
+ drivers/gpu/drm/i915/gvt/sched_policy.c            |   31 +-
+ drivers/gpu/drm/i915/gvt/scheduler.c               |   89 +-
+ drivers/gpu/drm/i915/gvt/scheduler.h               |    1 +
+ drivers/gpu/drm/i915/gvt/trace.h                   |   24 +-
+ drivers/gpu/drm/i915/i915_debugfs.c                |  566 +--
+ drivers/gpu/drm/i915/i915_drv.c                    |   89 +-
+ drivers/gpu/drm/i915/i915_drv.h                    |  424 +-
+ drivers/gpu/drm/i915/i915_gem.c                    |  394 +-
+ drivers/gpu/drm/i915/i915_gem.h                    |   13 +
+ drivers/gpu/drm/i915/i915_gem_batch_pool.c         |   30 +-
+ drivers/gpu/drm/i915/i915_gem_batch_pool.h         |   29 +-
+ drivers/gpu/drm/i915/i915_gem_context.c            |   62 +-
+ drivers/gpu/drm/i915/i915_gem_context.h            |   43 +-
+ drivers/gpu/drm/i915/i915_gem_execbuffer.c         |   32 +-
+ drivers/gpu/drm/i915/i915_gem_gtt.c                |   70 +-
+ drivers/gpu/drm/i915/i915_gem_gtt.h                |    5 +-
+ drivers/gpu/drm/i915/i915_gem_stolen.c             |  178 +-
+ drivers/gpu/drm/i915/i915_gem_timeline.c           |  154 -
+ drivers/gpu/drm/i915/i915_gpu_error.c              |   66 +-
+ drivers/gpu/drm/i915/i915_gpu_error.h              |  366 ++
+ drivers/gpu/drm/i915/i915_irq.c                    |  410 +-
+ drivers/gpu/drm/i915/i915_oa_icl.c                 |  118 +
+ drivers/gpu/drm/i915/i915_oa_icl.h                 |   34 +
+ drivers/gpu/drm/i915/i915_params.c                 |    3 +
+ drivers/gpu/drm/i915/i915_params.h                 |    3 +-
+ drivers/gpu/drm/i915/i915_pci.c                    |    1 +
+ drivers/gpu/drm/i915/i915_perf.c                   |   96 +-
+ drivers/gpu/drm/i915/i915_pmu.c                    |   27 +-
+ drivers/gpu/drm/i915/i915_pmu.h                    |   30 +-
+ drivers/gpu/drm/i915/i915_reg.h                    |  879 ++--
+ drivers/gpu/drm/i915/i915_request.c                |  438 +-
+ drivers/gpu/drm/i915/i915_request.h                |   49 +-
+ drivers/gpu/drm/i915/i915_scheduler.h              |   72 +
+ drivers/gpu/drm/i915/i915_timeline.c               |  105 +
+ .../i915/{i915_gem_timeline.h => i915_timeline.h}  |   71 +-
+ drivers/gpu/drm/i915/i915_trace.h                  |  129 +-
+ drivers/gpu/drm/i915/i915_utils.h                  |   10 +-
+ drivers/gpu/drm/i915/i915_vma.c                    |   73 +-
+ drivers/gpu/drm/i915/i915_vma.h                    |    6 +
+ drivers/gpu/drm/i915/intel_atomic.c                |   19 +-
+ drivers/gpu/drm/i915/intel_atomic_plane.c          |    7 +-
+ drivers/gpu/drm/i915/intel_bios.c                  |   18 +-
+ drivers/gpu/drm/i915/intel_breadcrumbs.c           |   52 +-
+ drivers/gpu/drm/i915/intel_crt.c                   |    8 +-
+ drivers/gpu/drm/i915/intel_csr.c                   |    9 +-
+ drivers/gpu/drm/i915/intel_ddi.c                   |  451 +-
+ drivers/gpu/drm/i915/intel_device_info.c           |  169 +-
+ drivers/gpu/drm/i915/intel_device_info.h           |    4 +-
+ drivers/gpu/drm/i915/intel_display.c               |  472 +-
+ drivers/gpu/drm/i915/intel_display.h               |    4 +
+ drivers/gpu/drm/i915/intel_dp.c                    |  298 +-
+ drivers/gpu/drm/i915/intel_dp_link_training.c      |    5 +
+ drivers/gpu/drm/i915/intel_dp_mst.c                |    8 +-
+ drivers/gpu/drm/i915/intel_dpio_phy.c              |   11 +-
+ drivers/gpu/drm/i915/intel_dpll_mgr.c              |  909 +++-
+ drivers/gpu/drm/i915/intel_dpll_mgr.h              |   97 +-
+ drivers/gpu/drm/i915/intel_drv.h                   |   84 +-
+ drivers/gpu/drm/i915/intel_dsi_vbt.c               |   34 +-
+ drivers/gpu/drm/i915/intel_engine_cs.c             |  937 +---
+ drivers/gpu/drm/i915/intel_fbc.c                   |   28 +
+ drivers/gpu/drm/i915/intel_fbdev.c                 |    7 +-
+ drivers/gpu/drm/i915/intel_frontbuffer.c           |    2 +-
+ drivers/gpu/drm/i915/intel_gpu_commands.h          |  274 ++
+ drivers/gpu/drm/i915/intel_guc.c                   |  231 +-
+ drivers/gpu/drm/i915/intel_guc.h                   |   82 +-
+ drivers/gpu/drm/i915/intel_guc_ads.c               |    9 +-
+ drivers/gpu/drm/i915/intel_guc_ct.c                |  545 ++-
+ drivers/gpu/drm/i915/intel_guc_ct.h                |   18 +-
+ drivers/gpu/drm/i915/intel_guc_fw.c                |    7 +-
+ drivers/gpu/drm/i915/intel_guc_fwif.h              |  162 +-
+ drivers/gpu/drm/i915/intel_guc_log.c               |  544 +--
+ drivers/gpu/drm/i915/intel_guc_log.h               |   59 +-
+ drivers/gpu/drm/i915/intel_guc_reg.h               |   14 +-
+ drivers/gpu/drm/i915/intel_guc_submission.c        |  114 +-
+ drivers/gpu/drm/i915/intel_hangcheck.c             |   16 +-
+ drivers/gpu/drm/i915/intel_hdcp.c                  |  185 +-
+ drivers/gpu/drm/i915/intel_hdmi.c                  |   40 +-
+ drivers/gpu/drm/i915/intel_hotplug.c               |    3 +
+ drivers/gpu/drm/i915/intel_huc.c                   |   30 +-
+ drivers/gpu/drm/i915/intel_huc.h                   |    7 +
+ drivers/gpu/drm/i915/intel_huc_fw.c                |    8 +-
+ drivers/gpu/drm/i915/intel_lrc.c                   |  538 ++-
+ drivers/gpu/drm/i915/intel_lrc.h                   |    2 +-
+ drivers/gpu/drm/i915/intel_mocs.c                  |    5 +-
+ drivers/gpu/drm/i915/intel_overlay.c               |    1 +
+ drivers/gpu/drm/i915/intel_pipe_crc.c              |   75 +-
+ drivers/gpu/drm/i915/intel_pm.c                    |  578 ++-
+ drivers/gpu/drm/i915/intel_psr.c                   |  444 +-
+ drivers/gpu/drm/i915/intel_ringbuffer.c            |   78 +-
+ drivers/gpu/drm/i915/intel_ringbuffer.h            |   71 +-
+ drivers/gpu/drm/i915/intel_runtime_pm.c            |  101 +-
+ drivers/gpu/drm/i915/intel_sdvo.c                  |    5 +-
+ drivers/gpu/drm/i915/intel_sprite.c                |  178 +-
+ drivers/gpu/drm/i915/intel_uc.c                    |  132 +-
+ drivers/gpu/drm/i915/intel_uc.h                    |    5 +-
+ drivers/gpu/drm/i915/intel_uc_fw.c                 |   13 +-
+ drivers/gpu/drm/i915/intel_uc_fw.h                 |   24 +-
+ drivers/gpu/drm/i915/intel_uncore.c                |  175 +-
+ drivers/gpu/drm/i915/intel_uncore.h                |    1 +
+ drivers/gpu/drm/i915/intel_wopcm.c                 |  275 ++
+ drivers/gpu/drm/i915/intel_wopcm.h                 |   31 +
+ drivers/gpu/drm/i915/intel_workarounds.c           |  949 ++++
+ drivers/gpu/drm/i915/intel_workarounds.h           |   17 +
+ drivers/gpu/drm/i915/selftests/huge_pages.c        |    5 +-
+ drivers/gpu/drm/i915/selftests/i915_gem_context.c  |    3 +
+ .../gpu/drm/i915/selftests/i915_live_selftests.h   |    2 +
+ .../gpu/drm/i915/selftests/i915_mock_selftests.h   |    1 +
+ .../{i915_gem_timeline.c => i915_timeline.c}       |   94 +-
+ drivers/gpu/drm/i915/selftests/i915_vma.c          |    2 +-
+ drivers/gpu/drm/i915/selftests/igt_flush_test.c    |   70 +
+ drivers/gpu/drm/i915/selftests/igt_flush_test.h    |   14 +
+ drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c |    5 +-
+ drivers/gpu/drm/i915/selftests/intel_engine_cs.c   |   58 +
+ drivers/gpu/drm/i915/selftests/intel_hangcheck.c   |  414 +-
+ drivers/gpu/drm/i915/selftests/intel_lrc.c         |  459 ++
+ drivers/gpu/drm/i915/selftests/intel_workarounds.c |  291 ++
+ drivers/gpu/drm/i915/selftests/mock_engine.c       |   67 +-
+ drivers/gpu/drm/i915/selftests/mock_gem_device.c   |   21 +-
+ drivers/gpu/drm/i915/selftests/mock_gtt.c          |    1 -
+ drivers/gpu/drm/i915/selftests/mock_timeline.c     |   45 +-
+ drivers/gpu/drm/i915/selftests/mock_timeline.h     |   28 +-
+ drivers/gpu/drm/mediatek/Kconfig                   |    1 +
+ drivers/gpu/drm/mediatek/mtk_dpi.c                 |   60 +-
+ drivers/gpu/drm/mediatek/mtk_drm_gem.c             |    2 +-
+ drivers/gpu/drm/mediatek/mtk_dsi.c                 |   14 +-
+ drivers/gpu/drm/mgag200/mgag200_mode.c             |    2 +-
+ drivers/gpu/drm/msm/adreno/adreno_device.c         |    1 -
+ drivers/gpu/drm/msm/msm_debugfs.c                  |    3 -
+ drivers/gpu/drm/mxsfb/mxsfb_drv.c                  |   11 +-
+ drivers/gpu/drm/nouveau/Kbuild                     |    8 +-
+ drivers/gpu/drm/nouveau/dispnv50/Kbuild            |   51 +
+ drivers/gpu/drm/nouveau/dispnv50/atom.h            |  222 +
+ drivers/gpu/drm/nouveau/dispnv50/base.c            |   53 +
+ drivers/gpu/drm/nouveau/dispnv50/base.h            |   31 +
+ drivers/gpu/drm/nouveau/dispnv50/base507c.c        |  286 ++
+ drivers/gpu/drm/nouveau/dispnv50/base827c.c        |   71 +
+ drivers/gpu/drm/nouveau/dispnv50/base907c.c        |  110 +
+ drivers/gpu/drm/nouveau/dispnv50/base917c.c        |   48 +
+ drivers/gpu/drm/nouveau/dispnv50/core.c            |   70 +
+ drivers/gpu/drm/nouveau/dispnv50/core.h            |   50 +
+ drivers/gpu/drm/nouveau/dispnv50/core507d.c        |  115 +
+ drivers/gpu/drm/nouveau/dispnv50/core827d.c        |   41 +
+ .../disp/basegt215.c => dispnv50/core907d.c}       |   34 +-
+ .../disp/basegt200.c => dispnv50/core917d.c}       |   34 +-
+ drivers/gpu/drm/nouveau/dispnv50/corec37d.c        |  110 +
+ drivers/gpu/drm/nouveau/dispnv50/curs.c            |   52 +
+ drivers/gpu/drm/nouveau/dispnv50/curs.h            |   14 +
+ drivers/gpu/drm/nouveau/dispnv50/curs507a.c        |  145 +
+ .../fifo/gpfifogk110.c => dispnv50/curs907a.c}     |   22 +-
+ drivers/gpu/drm/nouveau/dispnv50/cursc37a.c        |   50 +
+ drivers/gpu/drm/nouveau/dispnv50/dac507d.c         |   44 +
+ .../engine/disp/cursgt215.c => dispnv50/dac907d.c} |   30 +-
+ drivers/gpu/drm/nouveau/dispnv50/disp.c            | 2238 ++++++++++
+ drivers/gpu/drm/nouveau/dispnv50/disp.h            |   89 +
+ drivers/gpu/drm/nouveau/dispnv50/head.c            |  511 +++
+ drivers/gpu/drm/nouveau/dispnv50/head.h            |   78 +
+ drivers/gpu/drm/nouveau/dispnv50/head507d.c        |  325 ++
+ drivers/gpu/drm/nouveau/dispnv50/head827d.c        |  124 +
+ drivers/gpu/drm/nouveau/dispnv50/head907d.c        |  284 ++
+ drivers/gpu/drm/nouveau/dispnv50/head917d.c        |  100 +
+ drivers/gpu/drm/nouveau/dispnv50/headc37d.c        |  212 +
+ drivers/gpu/drm/nouveau/dispnv50/lut.c             |   95 +
+ drivers/gpu/drm/nouveau/dispnv50/lut.h             |   15 +
+ drivers/gpu/drm/nouveau/dispnv50/oimm.c            |   51 +
+ drivers/gpu/drm/nouveau/dispnv50/oimm.h            |    8 +
+ drivers/gpu/drm/nouveau/dispnv50/oimm507b.c        |   52 +
+ drivers/gpu/drm/nouveau/dispnv50/ovly.c            |   57 +
+ drivers/gpu/drm/nouveau/dispnv50/ovly.h            |   30 +
+ drivers/gpu/drm/nouveau/dispnv50/ovly507e.c        |  217 +
+ drivers/gpu/drm/nouveau/dispnv50/ovly827e.c        |  107 +
+ drivers/gpu/drm/nouveau/dispnv50/ovly907e.c        |   70 +
+ drivers/gpu/drm/nouveau/dispnv50/ovly917e.c        |   45 +
+ drivers/gpu/drm/nouveau/dispnv50/pior507d.c        |   44 +
+ drivers/gpu/drm/nouveau/dispnv50/sor507d.c         |   44 +
+ .../engine/disp/coregt200.c => dispnv50/sor907d.c} |   31 +-
+ .../engine/disp/cursgk104.c => dispnv50/sorc37d.c} |   30 +-
+ drivers/gpu/drm/nouveau/dispnv50/wimm.c            |   47 +
+ drivers/gpu/drm/nouveau/dispnv50/wimm.h            |    8 +
+ drivers/gpu/drm/nouveau/dispnv50/wimmc37b.c        |   86 +
+ drivers/gpu/drm/nouveau/dispnv50/wndw.c            |  641 +++
+ drivers/gpu/drm/nouveau/dispnv50/wndw.h            |   96 +
+ drivers/gpu/drm/nouveau/dispnv50/wndwc37e.c        |  278 ++
+ drivers/gpu/drm/nouveau/include/nvif/cl0080.h      |   47 +
+ drivers/gpu/drm/nouveau/include/nvif/cla06f.h      |   18 +-
+ drivers/gpu/drm/nouveau/include/nvif/class.h       |   14 +
+ drivers/gpu/drm/nouveau/include/nvif/clc37b.h      |   11 +
+ drivers/gpu/drm/nouveau/include/nvif/clc37e.h      |   13 +
+ drivers/gpu/drm/nouveau/include/nvif/device.h      |    9 +-
+ drivers/gpu/drm/nouveau/include/nvif/disp.h        |   12 +
+ drivers/gpu/drm/nouveau/include/nvif/fifo.h        |   18 +
+ drivers/gpu/drm/nouveau/include/nvif/mem.h         |    2 +
+ drivers/gpu/drm/nouveau/include/nvif/mmu.h         |    1 +
+ drivers/gpu/drm/nouveau/include/nvif/object.h      |   16 +
+ drivers/gpu/drm/nouveau/include/nvif/user.h        |   19 +
+ drivers/gpu/drm/nouveau/include/nvkm/core/device.h |   13 +-
+ drivers/gpu/drm/nouveau/include/nvkm/core/engine.h |    1 +
+ drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h |    2 +
+ drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h   |    1 +
+ drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h |    1 +
+ drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h  |    1 +
+ drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h |    3 +
+ drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h   |    2 +
+ .../gpu/drm/nouveau/include/nvkm/subdev/bios/dp.h  |    4 +
+ .../gpu/drm/nouveau/include/nvkm/subdev/devinit.h  |    1 +
+ .../gpu/drm/nouveau/include/nvkm/subdev/fault.h    |   33 +
+ drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h   |    1 +
+ drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h  |    3 +
+ drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h  |    1 +
+ drivers/gpu/drm/nouveau/nouveau_abi16.c            |   36 +-
+ drivers/gpu/drm/nouveau/nouveau_acpi.c             |    2 +-
+ drivers/gpu/drm/nouveau/nouveau_bo.c               |    2 +
+ drivers/gpu/drm/nouveau/nouveau_chan.c             |   39 +-
+ drivers/gpu/drm/nouveau/nouveau_chan.h             |    1 +
+ drivers/gpu/drm/nouveau/nouveau_connector.c        |   16 +-
+ drivers/gpu/drm/nouveau/nouveau_display.c          |   39 +-
+ drivers/gpu/drm/nouveau/nouveau_display.h          |    3 +-
+ drivers/gpu/drm/nouveau/nouveau_dma.c              |   15 +-
+ drivers/gpu/drm/nouveau/nouveau_dma.h              |    6 +-
+ drivers/gpu/drm/nouveau/nouveau_drm.c              |   90 +-
+ drivers/gpu/drm/nouveau/nouveau_drv.h              |    6 +
+ drivers/gpu/drm/nouveau/nouveau_encoder.h          |    3 +-
+ drivers/gpu/drm/nouveau/nouveau_fence.c            |   11 +-
+ drivers/gpu/drm/nouveau/nouveau_fence.h            |    2 -
+ drivers/gpu/drm/nouveau/nouveau_gem.c              |   50 +-
+ drivers/gpu/drm/nouveau/nouveau_hwmon.c            |   16 +-
+ drivers/gpu/drm/nouveau/nouveau_vmm.c              |    1 +
+ drivers/gpu/drm/nouveau/nouveau_vmm.h              |    2 +
+ drivers/gpu/drm/nouveau/nv04_fence.c               |    2 -
+ drivers/gpu/drm/nouveau/nv10_fence.c               |    2 -
+ drivers/gpu/drm/nouveau/nv17_fence.c               |    2 -
+ drivers/gpu/drm/nouveau/nv50_display.c             | 4558 --------------------
+ drivers/gpu/drm/nouveau/nv50_display.h             |    1 -
+ drivers/gpu/drm/nouveau/nv50_fence.c               |    2 -
+ drivers/gpu/drm/nouveau/nv84_fence.c               |   11 +-
+ drivers/gpu/drm/nouveau/nvif/Kbuild                |    6 +
+ drivers/gpu/drm/nouveau/nvif/device.c              |    5 +
+ .../inc/pp_soc15.h => nouveau/nvif/disp.c}         |   62 +-
+ drivers/gpu/drm/nouveau/nvif/fifo.c                |   99 +
+ drivers/gpu/drm/nouveau/nvif/mem.c                 |   13 +
+ drivers/gpu/drm/nouveau/nvif/mmu.c                 |   11 +
+ drivers/gpu/drm/nouveau/nvif/user.c                |   64 +
+ .../engine/fifo/gpfifogp100.c => nvif/userc361.c}  |   21 +-
+ drivers/gpu/drm/nouveau/nvif/vmm.c                 |    2 +-
+ drivers/gpu/drm/nouveau/nvkm/core/engine.c         |   15 +
+ drivers/gpu/drm/nouveau/nvkm/core/subdev.c         |   12 +
+ drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild      |    1 +
+ .../nvkm/engine/{disp/cursg84.c => ce/gv100.c}     |   29 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/device/base.c  |   73 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h  |    1 +
+ drivers/gpu/drm/nouveau/nvkm/engine/device/user.c  |   74 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild    |   30 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c    |   23 +
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/baseg84.c |   26 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/basegf119.c   |   24 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/basegp102.c   |   22 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/basenv50.c    |   42 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/changf119.c   |   13 +
+ .../{fifo/gpfifogm200.c => disp/changv100.c}       |   24 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/channv50.c    |  146 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/channv50.h    |  190 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg84.c |   24 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c |   24 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/coregf119.c   |   47 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/coregk104.c   |   24 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/coregk110.c   |   38 -
+ .../gpu/drm/nouveau/nvkm/engine/disp/coregm107.c   |   38 -
+ .../gpu/drm/nouveau/nvkm/engine/disp/coregm200.c   |   38 -
+ .../gpu/drm/nouveau/nvkm/engine/disp/coregp100.c   |   38 -
+ .../gpu/drm/nouveau/nvkm/engine/disp/coregp102.c   |   34 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/coregt215.c   |   38 -
+ .../gpu/drm/nouveau/nvkm/engine/disp/coregv100.c   |  204 +
+ .../gpu/drm/nouveau/nvkm/engine/disp/corenv50.c    |   61 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/cursgf119.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/cursgp102.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/cursgv100.c   |   81 +
+ .../gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c    |   36 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/dacgf119.c    |   11 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/dacnv50.c |   11 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c   |   40 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c   |   77 +
+ .../gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c    |  179 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h    |  102 -
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/g84.c     |   12 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/g94.c     |   12 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c   |   91 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c   |   10 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/gk110.c   |   10 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c   |   10 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c   |   10 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c   |    8 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c   |    9 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/gt200.c   |   12 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/gt215.c   |   12 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c   |  427 ++
+ .../gpu/drm/nouveau/nvkm/engine/disp/hdmigv100.c   |   85 +
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h    |    8 +
+ .../gpu/drm/nouveau/nvkm/engine/disp/headgf119.c   |   13 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/headgv100.c   |  105 +
+ .../gpu/drm/nouveau/nvkm/engine/disp/headnv50.c    |    7 +
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h     |   24 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp77.c   |   12 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp89.c   |   12 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c    |  207 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h    |   50 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/oimmgf119.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/oimmgk104.c   |   37 -
+ .../gpu/drm/nouveau/nvkm/engine/disp/oimmgp102.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/oimmgt215.c   |   37 -
+ .../gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c    |   36 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlyg84.c |   26 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/ovlygf119.c   |   24 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/ovlygk104.c   |   24 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/ovlygp102.c   |   22 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/ovlygt200.c   |   24 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/ovlygt215.c   |   38 -
+ .../gpu/drm/nouveau/nvkm/engine/disp/ovlynv50.c    |   42 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/piocgf119.c   |   13 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/piocnv50.c    |    6 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/piornv50.c    |   11 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h    |    3 +
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/rootg84.c |   19 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/rootg94.c |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootgf119.c   |  101 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootgk104.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootgk110.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootgm107.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootgm200.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootgp100.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootgp102.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootgt200.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootgt215.c   |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootgv100.c   |   52 +
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c    |  175 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h    |   22 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg84.c  |    2 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg94.c  |   10 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c    |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/sorgk104.c    |    2 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c    |    2 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c    |    8 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/sorgt215.c    |    2 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c    |  120 +
+ .../gpu/drm/nouveau/nvkm/engine/disp/sormcp77.c    |    2 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/sormcp89.c    |    2 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/sornv50.c |   19 +-
+ .../gpu/drm/nouveau/nvkm/engine/disp/wimmgv100.c   |   82 +
+ .../gpu/drm/nouveau/nvkm/engine/disp/wndwgv100.c   |  184 +
+ drivers/gpu/drm/nouveau/nvkm/engine/dma/Kbuild     |    2 +
+ .../nvkm/engine/{disp/oimmg84.c => dma/gv100.c}    |   27 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/dma/user.h     |    2 +
+ .../gpu/drm/nouveau/nvkm/engine/dma/usergv100.c    |  119 +
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild    |    7 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c    |   46 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h    |   11 +
+ .../gpu/drm/nouveau/nvkm/engine/fifo/changk104.h   |   20 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c   |  373 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h   |   57 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c   |   28 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c   |   20 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c   |   10 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c   |   27 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c   |   10 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c   |   10 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c   |   11 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c   |   11 +-
+ .../gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c |  155 +-
+ .../gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c |  225 +
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c   |  306 ++
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h    |    7 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h    |    6 +
+ .../gpu/drm/nouveau/nvkm/engine/fifo/usergv100.c   |   45 +
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild      |    4 +
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c  |  423 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h  |   79 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf104.c  |   10 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c  |   17 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf110.c  |   15 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c  |  114 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf119.c  |   15 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c  |  190 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110.c  |   28 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110b.c |   14 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk208.c  |   20 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c  |    9 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c  |   93 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c  |  120 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c  |   22 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c  |  100 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c  |   42 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp104.c  |   48 +
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp107.c  |   11 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c  |  215 +
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c     |  417 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h     |  109 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c     |   14 +
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c     |   21 +
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c     |   14 +
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c     |   42 +
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c     |   14 +
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c     |  143 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c     |   31 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c    |   17 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c     |   16 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c     |   37 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c     |  178 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c     |  175 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gm20b.c     |    5 +
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c     |  180 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c     |   77 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c     |   66 +
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c     |   21 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c     |   27 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c     |  120 +
+ drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c     |    1 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild         |    1 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c      |    2 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/bios/pll.c     |   19 +-
+ .../gpu/drm/nouveau/nvkm/subdev/bios/shadowramin.c |    5 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c     |   10 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild |    1 +
+ .../gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c    |    2 +-
+ .../gpu/drm/nouveau/nvkm/subdev/devinit/gv100.c    |   79 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h |    2 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild   |    3 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c   |  179 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c  |   69 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c  |  206 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h   |   34 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild      |    1 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c     |    4 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c     |    2 -
+ drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c     |    4 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/fb/gv100.c     |   46 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h      |    2 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild     |    1 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c     |   10 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c    |    6 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp102.c    |   51 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h     |    5 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c     |   20 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c     |    2 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h      |    2 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild     |    2 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c    |   43 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h      |    3 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c |   87 +
+ .../gpu/drm/nouveau/nvkm/subdev/secboot/gp108.c    |   21 +
+ .../nvkm/subdev/secboot/ls_ucode_msgqueue.c        |    7 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c    |    3 +-
+ drivers/gpu/drm/omapdrm/omap_drv.c                 |    3 +
+ drivers/gpu/drm/omapdrm/omap_plane.c               |    2 +-
+ .../gpu/drm/panel/panel-raspberrypi-touchscreen.c  |   14 +-
+ drivers/gpu/drm/pl111/Makefile                     |    1 +
+ drivers/gpu/drm/pl111/pl111_display.c              |   11 +-
+ drivers/gpu/drm/pl111/pl111_drm.h                  |    1 +
+ drivers/gpu/drm/pl111/pl111_drv.c                  |   34 +-
+ drivers/gpu/drm/pl111/pl111_versatile.c            |   56 +-
+ drivers/gpu/drm/pl111/pl111_vexpress.c             |  134 +
+ drivers/gpu/drm/pl111/pl111_vexpress.h             |   29 +
+ drivers/gpu/drm/qxl/qxl_cmd.c                      |   36 +-
+ drivers/gpu/drm/qxl/qxl_display.c                  |  278 +-
+ drivers/gpu/drm/qxl/qxl_drv.h                      |    3 -
+ drivers/gpu/drm/qxl/qxl_fb.c                       |    2 -
+ drivers/gpu/drm/qxl/qxl_irq.c                      |    3 +-
+ drivers/gpu/drm/qxl/qxl_ttm.c                      |    8 +-
+ drivers/gpu/drm/radeon/atombios_crtc.c             |   32 +-
+ drivers/gpu/drm/radeon/radeon_atpx_handler.c       |    2 +-
+ drivers/gpu/drm/radeon/radeon_connectors.c         |   10 +-
+ drivers/gpu/drm/radeon/radeon_device.c             |    6 +-
+ drivers/gpu/drm/radeon/radeon_display.c            |   53 +-
+ drivers/gpu/drm/radeon/radeon_drv.c                |    5 +
+ drivers/gpu/drm/radeon/radeon_fb.c                 |   20 +-
+ drivers/gpu/drm/radeon/radeon_legacy_crtc.c        |   20 +-
+ drivers/gpu/drm/radeon/radeon_mode.h               |    8 +-
+ drivers/gpu/drm/rcar-du/rcar_du_crtc.c             |   26 +-
+ drivers/gpu/drm/rcar-du/rcar_du_crtc.h             |    3 +-
+ drivers/gpu/drm/rcar-du/rcar_du_drv.c              |   51 +-
+ drivers/gpu/drm/rcar-du/rcar_du_drv.h              |    5 +-
+ drivers/gpu/drm/rcar-du/rcar_du_group.c            |   16 +-
+ drivers/gpu/drm/rcar-du/rcar_du_group.h            |    2 +
+ drivers/gpu/drm/rcar-du/rcar_du_kms.c              |   41 +-
+ drivers/gpu/drm/rcar-du/rcar_du_of.c               |    1 +
+ drivers/gpu/drm/rcar-du/rcar_du_of.h               |    2 +-
+ drivers/gpu/drm/rcar-du/rcar_du_plane.c            |   15 +-
+ drivers/gpu/drm/rcar-du/rcar_du_plane.h            |    2 -
+ drivers/gpu/drm/rcar-du/rcar_du_regs.h             |   16 +-
+ drivers/gpu/drm/rcar-du/rcar_du_vsp.c              |   51 +-
+ drivers/gpu/drm/rcar-du/rcar_du_vsp.h              |    3 -
+ drivers/gpu/drm/rockchip/analogix_dp-rockchip.c    |   37 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_drv.h        |    1 +
+ drivers/gpu/drm/rockchip/rockchip_drm_fb.c         |   61 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_gem.c        |    4 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_psr.c        |  158 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_psr.h        |    7 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_vop.c        |   28 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_vop.h        |    1 +
+ drivers/gpu/drm/rockchip/rockchip_vop_reg.c        |    1 +
+ drivers/gpu/drm/scheduler/gpu_scheduler.c          |   91 +-
+ .../gpu/drm/scheduler}/gpu_scheduler_trace.h       |    2 +-
+ drivers/gpu/drm/scheduler/sched_fence.c            |    8 +-
+ drivers/gpu/drm/selftests/Makefile                 |    2 +-
+ drivers/gpu/drm/selftests/drm_helper_selftests.h   |    9 +
+ drivers/gpu/drm/selftests/test-drm-helper.c        |  247 ++
+ drivers/gpu/drm/sti/Kconfig                        |    3 +-
+ drivers/gpu/drm/sti/sti_crtc.c                     |    2 +-
+ drivers/gpu/drm/sti/sti_drv.c                      |   24 +-
+ drivers/gpu/drm/sti/sti_plane.c                    |    9 +-
+ drivers/gpu/drm/stm/drv.c                          |    2 -
+ drivers/gpu/drm/stm/ltdc.c                         |   88 +-
+ drivers/gpu/drm/stm/ltdc.h                         |   10 +-
+ drivers/gpu/drm/sun4i/Kconfig                      |   10 +
+ drivers/gpu/drm/sun4i/Makefile                     |    4 +
+ drivers/gpu/drm/sun4i/sun4i_backend.c              |   16 +-
+ drivers/gpu/drm/sun4i/sun4i_backend.h              |    3 +
+ drivers/gpu/drm/sun4i/sun4i_layer.c                |    2 +
+ drivers/gpu/drm/sun4i/sun4i_tcon.c                 |   86 +-
+ drivers/gpu/drm/sun4i/sun4i_tcon.h                 |   46 +
+ drivers/gpu/drm/sun4i/sun6i_mipi_dphy.c            |  292 ++
+ drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c             | 1107 +++++
+ drivers/gpu/drm/sun4i/sun6i_mipi_dsi.h             |   63 +
+ drivers/gpu/drm/tegra/dc.c                         |  300 +-
+ drivers/gpu/drm/tegra/dc.h                         |   11 +-
+ drivers/gpu/drm/tegra/drm.c                        |  154 +-
+ drivers/gpu/drm/tegra/drm.h                        |   13 +-
+ drivers/gpu/drm/tegra/fb.c                         |   99 +-
+ drivers/gpu/drm/tegra/gem.c                        |   20 +-
+ drivers/gpu/drm/tegra/gr2d.c                       |   57 +-
+ drivers/gpu/drm/tegra/gr3d.c                       |   60 +-
+ drivers/gpu/drm/tegra/hub.c                        |    2 +-
+ drivers/gpu/drm/tegra/plane.c                      |  194 +-
+ drivers/gpu/drm/tegra/plane.h                      |   15 +-
+ drivers/gpu/drm/tegra/vic.c                        |    5 +
+ drivers/gpu/drm/tilcdc/tilcdc_crtc.c               |    2 +-
+ drivers/gpu/drm/tinydrm/core/tinydrm-core.c        |    2 +-
+ drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c     |   30 +
+ drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c        |   22 +-
+ drivers/gpu/drm/tinydrm/ili9225.c                  |   28 +-
+ drivers/gpu/drm/tinydrm/mi0283qt.c                 |   49 +-
+ drivers/gpu/drm/tinydrm/mipi-dbi.c                 |   30 +-
+ drivers/gpu/drm/tinydrm/repaper.c                  |   33 +-
+ drivers/gpu/drm/tinydrm/st7586.c                   |   28 +-
+ drivers/gpu/drm/tinydrm/st7735r.c                  |    7 +-
+ drivers/gpu/drm/ttm/ttm_agp_backend.c              |    1 +
+ drivers/gpu/drm/ttm/ttm_bo.c                       |    2 +-
+ drivers/gpu/drm/ttm/ttm_bo_manager.c               |    1 +
+ drivers/gpu/drm/ttm/ttm_bo_util.c                  |   51 +-
+ drivers/gpu/drm/ttm/ttm_bo_vm.c                    |    1 +
+ drivers/gpu/drm/ttm/ttm_execbuf_util.c             |    1 +
+ drivers/gpu/drm/ttm/ttm_lock.c                     |    1 +
+ drivers/gpu/drm/ttm/ttm_memory.c                   |    1 +
+ drivers/gpu/drm/ttm/ttm_module.c                   |    1 +
+ drivers/gpu/drm/ttm/ttm_object.c                   |    1 +
+ drivers/gpu/drm/ttm/ttm_tt.c                       |    1 +
+ drivers/gpu/drm/tve200/tve200_display.c            |   11 +-
+ drivers/gpu/drm/udl/udl_connector.c                |    2 +-
+ drivers/gpu/drm/udl/udl_dmabuf.c                   |    5 +-
+ drivers/gpu/drm/udl/udl_drv.c                      |    2 +-
+ drivers/gpu/drm/udl/udl_drv.h                      |    2 +
+ drivers/gpu/drm/udl/udl_gem.c                      |    5 +-
+ drivers/gpu/drm/udl/udl_main.c                     |    2 +
+ drivers/gpu/drm/v3d/Kconfig                        |    9 +
+ drivers/gpu/drm/v3d/Makefile                       |   18 +
+ drivers/gpu/drm/v3d/v3d_bo.c                       |  389 ++
+ drivers/gpu/drm/v3d/v3d_debugfs.c                  |  191 +
+ drivers/gpu/drm/v3d/v3d_drv.c                      |  371 ++
+ drivers/gpu/drm/v3d/v3d_drv.h                      |  294 ++
+ drivers/gpu/drm/v3d/v3d_fence.c                    |   58 +
+ drivers/gpu/drm/v3d/v3d_gem.c                      |  668 +++
+ drivers/gpu/drm/v3d/v3d_irq.c                      |  206 +
+ drivers/gpu/drm/v3d/v3d_mmu.c                      |  122 +
+ drivers/gpu/drm/v3d/v3d_regs.h                     |  295 ++
+ drivers/gpu/drm/v3d/v3d_sched.c                    |  228 +
+ drivers/gpu/drm/v3d/v3d_trace.h                    |   82 +
+ drivers/gpu/drm/v3d/v3d_trace_points.c             |    9 +
+ drivers/gpu/drm/vc4/vc4_crtc.c                     |   75 +-
+ drivers/gpu/drm/vc4/vc4_drv.c                      |    9 +-
+ drivers/gpu/drm/vc4/vc4_drv.h                      |   38 +
+ drivers/gpu/drm/vc4/vc4_dsi.c                      |    5 +
+ drivers/gpu/drm/vc4/vc4_gem.c                      |   57 +-
+ drivers/gpu/drm/vc4/vc4_hvs.c                      |    4 +
+ drivers/gpu/drm/vc4/vc4_kms.c                      |  224 +-
+ drivers/gpu/drm/vc4/vc4_plane.c                    |  152 +-
+ drivers/gpu/drm/vc4/vc4_regs.h                     |   97 +
+ drivers/gpu/drm/vc4/vc4_v3d.c                      |    3 +-
+ drivers/gpu/drm/virtio/virtgpu_display.c           |    2 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_drv.c                |    8 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_kms.c                |    6 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c                |   35 -
+ drivers/gpu/drm/xen/Kconfig                        |   17 +
+ drivers/gpu/drm/xen/Makefile                       |   11 +
+ drivers/gpu/drm/xen/xen_drm_front.c                |  840 ++++
+ drivers/gpu/drm/xen/xen_drm_front.h                |  158 +
+ drivers/gpu/drm/xen/xen_drm_front_cfg.c            |   77 +
+ drivers/gpu/drm/xen/xen_drm_front_cfg.h            |   37 +
+ drivers/gpu/drm/xen/xen_drm_front_conn.c           |  115 +
+ drivers/gpu/drm/xen/xen_drm_front_conn.h           |   27 +
+ drivers/gpu/drm/xen/xen_drm_front_evtchnl.c        |  387 ++
+ drivers/gpu/drm/xen/xen_drm_front_evtchnl.h        |   81 +
+ drivers/gpu/drm/xen/xen_drm_front_gem.c            |  308 ++
+ drivers/gpu/drm/xen/xen_drm_front_gem.h            |   40 +
+ drivers/gpu/drm/xen/xen_drm_front_kms.c            |  366 ++
+ drivers/gpu/drm/xen/xen_drm_front_kms.h            |   26 +
+ drivers/gpu/drm/xen/xen_drm_front_shbuf.c          |  414 ++
+ drivers/gpu/drm/xen/xen_drm_front_shbuf.h          |   64 +
+ drivers/gpu/drm/zte/zx_plane.c                     |    2 +-
+ drivers/gpu/drm/zte/zx_vou.c                       |    5 +-
+ drivers/gpu/drm/zte/zx_vou.h                       |    3 +-
+ drivers/gpu/host1x/cdma.c                          |    6 +-
+ drivers/gpu/host1x/cdma.h                          |    4 +-
+ drivers/gpu/host1x/debug.c                         |    2 +-
+ drivers/gpu/host1x/dev.c                           |   11 +-
+ drivers/gpu/host1x/dev.h                           |    8 -
+ drivers/gpu/host1x/hw/channel_hw.c                 |    5 +-
+ drivers/gpu/host1x/hw/syncpt_hw.c                  |   11 -
+ drivers/gpu/host1x/intr.c                          |   16 +-
+ drivers/gpu/host1x/intr.h                          |    8 +-
+ drivers/gpu/host1x/job.c                           |  147 +-
+ drivers/gpu/host1x/job.h                           |    4 +-
+ drivers/gpu/host1x/syncpt.c                        |   10 +-
+ drivers/gpu/host1x/syncpt.h                        |    3 -
+ drivers/staging/vboxvideo/vbox_drv.c               |    2 +-
+ drivers/video/hdmi.c                               |    3 +
+ include/drm/amd_asic_type.h                        |    2 +
+ include/drm/bridge/analogix_dp.h                   |    3 +-
+ include/drm/drmP.h                                 |   28 +-
+ include/drm/drm_blend.h                            |    3 +
+ include/drm/drm_device.h                           |   10 +-
+ include/drm/drm_dp_helper.h                        |   24 +-
+ include/drm/drm_drv.h                              |   15 +-
+ include/drm/drm_edid.h                             |    2 -
+ include/drm/drm_file.h                             |   23 +-
+ include/drm/drm_gem_framebuffer_helper.h           |    3 +
+ include/drm/drm_ioctl.h                            |    7 -
+ include/drm/drm_legacy.h                           |    4 +-
+ include/drm/drm_mode_config.h                      |    8 +
+ include/drm/drm_modes.h                            |   22 +
+ include/drm/drm_modeset_helper_vtables.h           |    5 +-
+ include/drm/drm_plane.h                            |   21 +-
+ include/drm/drm_property.h                         |   28 +-
+ include/drm/drm_rect.h                             |    3 +-
+ include/drm/drm_simple_kms_helper.h                |    6 +-
+ include/drm/gpu_scheduler.h                        |   56 +-
+ include/drm/i915_pciids.h                          |    1 +
+ include/drm/tinydrm/mipi-dbi.h                     |    4 +-
+ include/drm/tinydrm/tinydrm-helpers.h              |    5 +
+ include/drm/tinydrm/tinydrm.h                      |    8 +-
+ include/linux/dma-fence.h                          |  236 +-
+ include/linux/host1x.h                             |   24 +-
+ include/linux/platform_data/tda9950.h              |   16 +
+ include/trace/events/host1x.h                      |   16 +-
+ include/uapi/drm/amdgpu_drm.h                      |   20 +
+ include/uapi/drm/drm.h                             |    7 +
+ include/uapi/drm/drm_mode.h                        |    6 +
+ include/uapi/drm/exynos_drm.h                      |  240 ++
+ include/uapi/drm/tegra_drm.h                       |  492 ++-
+ include/uapi/drm/v3d_drm.h                         |  194 +
+ include/uapi/drm/vc4_drm.h                         |   13 +-
+ include/uapi/linux/virtio_gpu.h                    |    1 +
+ scripts/coccinelle/api/drm-get-put.cocci           |   10 -
+ 1110 files changed, 72649 insertions(+), 26344 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/display/bridge/cdns,dsi.txt
+ create mode 100644 Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.txt
+ create mode 100644 Documentation/devicetree/bindings/display/sunxi/sun6i-dsi.txt
+ create mode 100644 Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt
+ create mode 100644 Documentation/devicetree/bindings/gpu/samsung-scaler.txt
+ create mode 100644 Documentation/gpu/xen-front.rst
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/df_v1_7.c
+ rename drivers/gpu/drm/{nouveau/nvkm/engine/disp/basegk104.c => amd/amdgpu/df_v1_7.h} (71%)
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/df_v3_6.c
+ rename drivers/gpu/drm/{nouveau/nvkm/engine/disp/basegk110.c => amd/amdgpu/df_v3_6.h} (71%)
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c
+ create mode 100644 drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
+ create mode 100644 drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
+ create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
+ create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
+ create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c
+ create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+ create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h
+ create mode 100644 drivers/gpu/drm/amd/amdkfd/soc15_int.h
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/basics/fixpt32_32.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/calcs/calcs_logger.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
+ delete mode 100644 drivers/gpu/drm/amd/display/include/fixed32_32.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_default.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_default.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/vega20_ip_offset.h
+ create mode 100644 drivers/gpu/drm/amd/powerplay/inc/smu75.h
+ create mode 100644 drivers/gpu/drm/amd/powerplay/inc/smu75_discrete.h
+ create mode 100644 drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.c
+ create mode 100644 drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.h
+ create mode 100644 drivers/gpu/drm/bridge/cdns-dsi.c
+ create mode 100644 drivers/gpu/drm/bridge/thc63lvd1024.c
+ delete mode 100644 drivers/gpu/drm/exynos/exynos_drm_fimc.h
+ delete mode 100644 drivers/gpu/drm/exynos/exynos_drm_gsc.h
+ create mode 100644 drivers/gpu/drm/exynos/exynos_drm_ipp.c
+ create mode 100644 drivers/gpu/drm/exynos/exynos_drm_ipp.h
+ create mode 100644 drivers/gpu/drm/exynos/exynos_drm_scaler.c
+ create mode 100644 drivers/gpu/drm/exynos/regs-scaler.h
+ create mode 100644 drivers/gpu/drm/i2c/tda9950.c
+ delete mode 100644 drivers/gpu/drm/i915/i915_gem_timeline.c
+ create mode 100644 drivers/gpu/drm/i915/i915_gpu_error.h
+ create mode 100644 drivers/gpu/drm/i915/i915_oa_icl.c
+ create mode 100644 drivers/gpu/drm/i915/i915_oa_icl.h
+ create mode 100644 drivers/gpu/drm/i915/i915_scheduler.h
+ create mode 100644 drivers/gpu/drm/i915/i915_timeline.c
+ rename drivers/gpu/drm/i915/{i915_gem_timeline.h => i915_timeline.h} (68%)
+ create mode 100644 drivers/gpu/drm/i915/intel_gpu_commands.h
+ create mode 100644 drivers/gpu/drm/i915/intel_wopcm.c
+ create mode 100644 drivers/gpu/drm/i915/intel_wopcm.h
+ create mode 100644 drivers/gpu/drm/i915/intel_workarounds.c
+ create mode 100644 drivers/gpu/drm/i915/intel_workarounds.h
+ rename drivers/gpu/drm/i915/selftests/{i915_gem_timeline.c => i915_timeline.c} (70%)
+ create mode 100644 drivers/gpu/drm/i915/selftests/igt_flush_test.c
+ create mode 100644 drivers/gpu/drm/i915/selftests/igt_flush_test.h
+ create mode 100644 drivers/gpu/drm/i915/selftests/intel_engine_cs.c
+ create mode 100644 drivers/gpu/drm/i915/selftests/intel_lrc.c
+ create mode 100644 drivers/gpu/drm/i915/selftests/intel_workarounds.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/Kbuild
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/atom.h
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/base.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/base.h
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/base507c.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/base827c.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/base907c.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/base917c.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/core.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/core.h
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/core507d.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/core827d.c
+ rename drivers/gpu/drm/nouveau/{nvkm/engine/disp/basegt215.c => dispnv50/core907d.c} (71%)
+ rename drivers/gpu/drm/nouveau/{nvkm/engine/disp/basegt200.c => dispnv50/core917d.c} (71%)
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/corec37d.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/curs.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/curs.h
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/curs507a.c
+ rename drivers/gpu/drm/nouveau/{nvkm/engine/fifo/gpfifogk110.c => dispnv50/curs907a.c} (79%)
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/cursc37a.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/dac507d.c
+ rename drivers/gpu/drm/nouveau/{nvkm/engine/disp/cursgt215.c => dispnv50/dac907d.c} (74%)
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/disp.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/disp.h
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/head.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/head.h
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/head507d.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/head827d.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/head907d.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/head917d.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/headc37d.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/lut.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/lut.h
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/oimm.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/oimm.h
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/oimm507b.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/ovly.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/ovly.h
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/ovly507e.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/ovly827e.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/ovly907e.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/ovly917e.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/pior507d.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/sor507d.c
+ rename drivers/gpu/drm/nouveau/{nvkm/engine/disp/coregt200.c => dispnv50/sor907d.c} (74%)
+ rename drivers/gpu/drm/nouveau/{nvkm/engine/disp/cursgk104.c => dispnv50/sorc37d.c} (74%)
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/wimm.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/wimm.h
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/wimmc37b.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/wndw.c
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/wndw.h
+ create mode 100644 drivers/gpu/drm/nouveau/dispnv50/wndwc37e.c
+ create mode 100644 drivers/gpu/drm/nouveau/include/nvif/clc37b.h
+ create mode 100644 drivers/gpu/drm/nouveau/include/nvif/clc37e.h
+ create mode 100644 drivers/gpu/drm/nouveau/include/nvif/disp.h
+ create mode 100644 drivers/gpu/drm/nouveau/include/nvif/fifo.h
+ create mode 100644 drivers/gpu/drm/nouveau/include/nvif/user.h
+ create mode 100644 drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h
+ delete mode 100644 drivers/gpu/drm/nouveau/nv50_display.c
+ rename drivers/gpu/drm/{amd/powerplay/inc/pp_soc15.h => nouveau/nvif/disp.c} (57%)
+ create mode 100644 drivers/gpu/drm/nouveau/nvif/fifo.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvif/user.c
+ rename drivers/gpu/drm/nouveau/{nvkm/engine/fifo/gpfifogp100.c => nvif/userc361.c} (79%)
+ rename drivers/gpu/drm/nouveau/nvkm/engine/{disp/cursg84.c => ce/gv100.c} (76%)
+ rename drivers/gpu/drm/nouveau/nvkm/engine/{fifo/gpfifogm200.c => disp/changv100.c} (79%)
+ delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/coregk110.c
+ delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/coregm107.c
+ delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/coregm200.c
+ delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp100.c
+ delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/coregt215.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c
+ delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/headgv100.c
+ delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgk104.c
+ delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgt215.c
+ delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygt215.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/wimmgv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/wndwgv100.c
+ rename drivers/gpu/drm/nouveau/nvkm/engine/{disp/oimmg84.c => dma/gv100.c} (76%)
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/dma/usergv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/usergv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp104.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp102.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c
+ create mode 100644 drivers/gpu/drm/pl111/pl111_vexpress.c
+ create mode 100644 drivers/gpu/drm/pl111/pl111_vexpress.h
+ rename {include/drm => drivers/gpu/drm/scheduler}/gpu_scheduler_trace.h (97%)
+ create mode 100644 drivers/gpu/drm/selftests/drm_helper_selftests.h
+ create mode 100644 drivers/gpu/drm/selftests/test-drm-helper.c
+ create mode 100644 drivers/gpu/drm/sun4i/sun6i_mipi_dphy.c
+ create mode 100644 drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c
+ create mode 100644 drivers/gpu/drm/sun4i/sun6i_mipi_dsi.h
+ create mode 100644 drivers/gpu/drm/v3d/Kconfig
+ create mode 100644 drivers/gpu/drm/v3d/Makefile
+ create mode 100644 drivers/gpu/drm/v3d/v3d_bo.c
+ create mode 100644 drivers/gpu/drm/v3d/v3d_debugfs.c
+ create mode 100644 drivers/gpu/drm/v3d/v3d_drv.c
+ create mode 100644 drivers/gpu/drm/v3d/v3d_drv.h
+ create mode 100644 drivers/gpu/drm/v3d/v3d_fence.c
+ create mode 100644 drivers/gpu/drm/v3d/v3d_gem.c
+ create mode 100644 drivers/gpu/drm/v3d/v3d_irq.c
+ create mode 100644 drivers/gpu/drm/v3d/v3d_mmu.c
+ create mode 100644 drivers/gpu/drm/v3d/v3d_regs.h
+ create mode 100644 drivers/gpu/drm/v3d/v3d_sched.c
+ create mode 100644 drivers/gpu/drm/v3d/v3d_trace.h
+ create mode 100644 drivers/gpu/drm/v3d/v3d_trace_points.c
+ create mode 100644 drivers/gpu/drm/xen/Kconfig
+ create mode 100644 drivers/gpu/drm/xen/Makefile
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front.c
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front.h
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_cfg.c
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_cfg.h
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_conn.c
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_conn.h
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_evtchnl.c
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_evtchnl.h
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_gem.c
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_gem.h
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_kms.c
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_kms.h
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_shbuf.c
+ create mode 100644 drivers/gpu/drm/xen/xen_drm_front_shbuf.h
+ create mode 100644 include/linux/platform_data/tda9950.h
+ create mode 100644 include/uapi/drm/v3d_drm.h
+$ git am -3 ../patches/0001-drm-xen-front-merge-fix-for-drivers-remove-force-dma.patch
+Applying: drm/xen-front: merge fix for "drivers: remove force dma flag from buses"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/gpu/drm/xen/xen_drm_front.c
+$ git add -A .
+$ git commit -v -a --amend
+[master 126e90fcf75f] Merge remote-tracking branch 'drm/drm-next'
+ Date: Thu May 31 13:07:40 2018 +1000
+Merging drm-panel/drm/panel/for-next (e4bac408b084 drm/panel: simple: Add support for Winstar WF35LTIACD)
+$ git merge drm-panel/drm/panel/for-next
+Already up to date.
+Merging drm-intel/for-linux-next (315852b42297 drm: rcar-du: Fix build failure)
+$ git merge drm-intel/for-linux-next
+Already up to date.
+Merging drm-tegra/drm/tegra/for-next (6134534ca24f drm/tegra: Add kerneldoc for UAPI)
+$ git merge drm-tegra/drm/tegra/for-next
+Already up to date.
+Merging drm-misc/for-linux-next (fbecef131676 drm/v3d: add CONFIG_MMU dependency)
+$ git merge drm-misc/for-linux-next
+Merge made by the 'recursive' strategy.
+ drivers/gpu/drm/v3d/Kconfig | 1 +
+ 1 file changed, 1 insertion(+)
+Merging drm-msm/msm-next (64cb96587ba6 drm/msm: Switch to atomic_helper_commit())
+$ git merge drm-msm/msm-next
+Merge made by the 'recursive' strategy.
+ drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c |  17 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c  |   9 +
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c   | 105 ++++++++----
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h   |  27 +--
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c |  12 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c  |  20 ++-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c |  16 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c   |  17 +-
+ drivers/gpu/drm/msm/dsi/dsi_host.c         |   4 +-
+ drivers/gpu/drm/msm/msm_atomic.c           | 258 +++--------------------------
+ drivers/gpu/drm/msm/msm_drv.c              |  11 +-
+ drivers/gpu/drm/msm/msm_drv.h              |   9 +-
+ drivers/gpu/drm/msm/msm_kms.h              |  14 --
+ 13 files changed, 167 insertions(+), 352 deletions(-)
+Merging hdlcd/for-upstream/hdlcd (f73e8b825315 drm/arm: Replace instances of drm_dev_unref with drm_dev_put.)
+$ git merge hdlcd/for-upstream/hdlcd
+Already up to date.
+Merging mali-dp/for-upstream/mali-dp (6e810eb508f4 drm: mali-dp: Add YUV->RGB conversion support for video layers)
+$ git merge mali-dp/for-upstream/mali-dp
+Already up to date.
+Merging sunxi-drm/sunxi-drm/for-next (7dafb83edd32 Merge branches 'sunxi/drm-fixes-for-4.13' and 'sunxi/drm-for-4.14' into sunxi-drm/for-next)
+$ git merge sunxi-drm/sunxi-drm/for-next
+Already up to date!
+Merge made by the 'recursive' strategy.
+Merging imx-drm/imx-drm/next (d34515f6779c gpu: ipu-v3: csi: support RGB565 on parallel bus)
+$ git merge imx-drm/imx-drm/next
+Merge made by the 'recursive' strategy.
+ drivers/gpu/drm/imx/imx-drm.h    |  1 -
+ drivers/gpu/drm/imx/imx-ldb.c    |  9 ++++++---
+ drivers/gpu/drm/imx/ipuv3-crtc.c |  1 -
+ drivers/gpu/ipu-v3/ipu-csi.c     | 34 ++++++++++++++++++++++++++++------
+ 4 files changed, 34 insertions(+), 11 deletions(-)
+Merging etnaviv/etnaviv/next (f6ffbd4fc1a1 drm/etnaviv: replace license text with SPDX tags)
+$ git merge etnaviv/etnaviv/next
+Already up to date.
+Merging kconfig/for-next (bebc6082da0a Linux 4.14)
+$ git merge kconfig/for-next
+Already up to date.
+Merging regmap/for-next (16c66b8804fa Merge branch 'regmap-4.18' into regmap-next)
+$ git merge regmap/for-next
+Merge made by the 'recursive' strategy.
+ drivers/base/regmap/regmap-slimbus.c |  2 +-
+ include/linux/regmap.h               | 18 +++++++++++++++++-
+ 2 files changed, 18 insertions(+), 2 deletions(-)
+Merging sound/for-next (e46dcbb17d79 ALSA: xen-front: freeing an error pointer)
+$ git merge sound/for-next
+Auto-merging MAINTAINERS
+Auto-merging Documentation/sound/alsa-configuration.rst
+Merge made by the 'recursive' strategy.
+ Documentation/sound/alsa-configuration.rst |    7 +
+ Documentation/sound/hd-audio/models.rst    |    2 +
+ MAINTAINERS                                |    7 +
+ include/linux/usb/audio-v2.h               |    7 +
+ include/linux/usb/audio-v3.h               |   40 +
+ include/sound/core.h                       |    2 +-
+ include/sound/emu10k1.h                    |    4 +-
+ include/sound/hdaudio.h                    |    5 +
+ include/sound/memalloc.h                   |    2 -
+ include/uapi/linux/usb/audio.h             |   19 +-
+ include/uapi/sound/tlv.h                   |   16 +
+ sound/Kconfig                              |    2 +
+ sound/Makefile                             |    2 +-
+ sound/core/compress_offload.c              |    2 +-
+ sound/core/device.c                        |    9 +
+ sound/core/info.c                          |    6 +-
+ sound/core/init.c                          |    4 +-
+ sound/core/oss/mixer_oss.c                 |    2 +-
+ sound/core/oss/pcm_oss.c                   |    2 +-
+ sound/core/pcm.c                           |   10 +-
+ sound/core/pcm_compat.c                    |   10 +-
+ sound/core/pcm_lib.c                       |   15 +-
+ sound/core/pcm_local.h                     |   18 +
+ sound/core/pcm_memory.c                    |    2 +-
+ sound/core/pcm_native.c                    |  261 +--
+ sound/core/seq/seq_ports.c                 |    2 +-
+ sound/core/seq/seq_timer.c                 |    4 +-
+ sound/core/timer.c                         |   48 +-
+ sound/core/vmaster.c                       |   16 +-
+ sound/drivers/aloop.c                      |   19 +-
+ sound/drivers/dummy.c                      |    2 +-
+ sound/drivers/mts64.c                      |    6 +-
+ sound/drivers/opl4/opl4_proc.c             |    2 +-
+ sound/drivers/portman2x4.c                 |    6 +-
+ sound/firewire/bebob/bebob_proc.c          |    2 +-
+ sound/firewire/dice/Makefile               |    3 +-
+ sound/firewire/dice/dice-alesis.c          |   52 +
+ sound/firewire/dice/dice-extension.c       |  172 ++
+ sound/firewire/dice/dice-interface.h       |    9 +-
+ sound/firewire/dice/dice-midi.c            |   23 +-
+ sound/firewire/dice/dice-mytek.c           |   46 +
+ sound/firewire/dice/dice-pcm.c             |  233 ++-
+ sound/firewire/dice/dice-proc.c            |   80 +-
+ sound/firewire/dice/dice-stream.c          |  283 ++-
+ sound/firewire/dice/dice-tcelectronic.c    |  104 +
+ sound/firewire/dice/dice-transaction.c     |   49 +-
+ sound/firewire/dice/dice.c                 |  156 +-
+ sound/firewire/dice/dice.h                 |   25 +-
+ sound/firewire/digi00x/digi00x-proc.c      |    2 +-
+ sound/firewire/fireface/ff-proc.c          |    2 +-
+ sound/firewire/fireworks/fireworks_proc.c  |    2 +-
+ sound/firewire/isight.c                    |   10 +-
+ sound/firewire/motu/motu-proc.c            |    2 +-
+ sound/firewire/oxfw/oxfw-proc.c            |    2 +-
+ sound/firewire/oxfw/oxfw.c                 |    8 +-
+ sound/firewire/tascam/tascam-proc.c        |    2 +-
+ sound/hda/hdac_regmap.c                    |    4 +-
+ sound/isa/cmi8328.c                        |    4 +-
+ sound/isa/msnd/msnd_pinnacle.c             |   32 +-
+ sound/isa/sc6000.c                         |    4 +-
+ sound/pci/ac97/ac97_proc.c                 |    4 +-
+ sound/pci/ad1889.c                         |    4 +-
+ sound/pci/asihpi/asihpi.c                  |   12 +-
+ sound/pci/asihpi/hpioctl.c                 |    4 +-
+ sound/pci/ca0106/ca0106_proc.c             |    6 +-
+ sound/pci/cmipci.c                         |    2 +-
+ sound/pci/cs46xx/cs46xx.c                  |    2 +-
+ sound/pci/cs46xx/cs46xx_lib.c              |    2 +-
+ sound/pci/cs46xx/dsp_spos.c                |   14 +-
+ sound/pci/cs46xx/dsp_spos_scb_lib.c        |    2 +-
+ sound/pci/ctxfi/cttimer.c                  |    2 +-
+ sound/pci/ctxfi/xfi.c                      |    4 +-
+ sound/pci/echoaudio/echoaudio.c            |    2 +-
+ sound/pci/echoaudio/echoaudio.h            |    6 -
+ sound/pci/emu10k1/emu10k1x.c               |    2 +-
+ sound/pci/emu10k1/emufx.c                  |   11 +-
+ sound/pci/emu10k1/emupcm.c                 |    2 +-
+ sound/pci/emu10k1/emuproc.c                |   24 +-
+ sound/pci/emu10k1/memory.c                 |    6 +-
+ sound/pci/hda/Kconfig                      |    4 +
+ sound/pci/hda/hda_auto_parser.c            |   10 +-
+ sound/pci/hda/hda_codec.c                  |   70 +-
+ sound/pci/hda/hda_generic.c                |   29 +-
+ sound/pci/hda/hda_intel.c                  |   11 +
+ sound/pci/hda/hda_sysfs.c                  |   20 +-
+ sound/pci/hda/hp_x360_helper.c             |   95 +
+ sound/pci/hda/patch_ca0132.c               | 3067 ++++++++++++++++++++++++++--
+ sound/pci/hda/patch_conexant.c             |    7 +-
+ sound/pci/hda/patch_hdmi.c                 |    2 +-
+ sound/pci/hda/patch_realtek.c              |   55 +-
+ sound/pci/ice1712/pontis.c                 |    2 +-
+ sound/pci/ice1712/prodigy_hifi.c           |    2 +-
+ sound/pci/lola/lola_proc.c                 |    2 +-
+ sound/pci/pcxhr/pcxhr.c                    |    2 +-
+ sound/soc/codecs/cs43130.c                 |    8 +-
+ sound/soc/codecs/wm_adsp.c                 |   11 +-
+ sound/soc/fsl/fsl_ssi_dbg.c                |    2 +-
+ sound/sound_core.c                         |    6 +-
+ sound/sparc/dbri.c                         |    4 +-
+ sound/usb/card.c                           |  226 +-
+ sound/usb/clock.c                          |   27 +-
+ sound/usb/helper.h                         |    4 -
+ sound/usb/mixer.c                          |  639 +++++-
+ sound/usb/mixer.h                          |    6 +
+ sound/usb/mixer_maps.c                     |   65 +
+ sound/usb/mixer_quirks.c                   |   39 +-
+ sound/usb/mixer_quirks.h                   |    4 +
+ sound/usb/mixer_scarlett.c                 |    6 +-
+ sound/usb/pcm.c                            |  270 +--
+ sound/usb/pcm.h                            |    1 +
+ sound/usb/quirks-table.h                   |   10 +
+ sound/usb/quirks.c                         |   59 +
+ sound/usb/stream.c                         |  693 ++++---
+ sound/usb/usbaudio.h                       |    8 +
+ sound/xen/Kconfig                          |   10 +
+ sound/xen/Makefile                         |    9 +
+ sound/xen/xen_snd_front.c                  |  397 ++++
+ sound/xen/xen_snd_front.h                  |   54 +
+ sound/xen/xen_snd_front_alsa.c             |  822 ++++++++
+ sound/xen/xen_snd_front_alsa.h             |   23 +
+ sound/xen/xen_snd_front_cfg.c              |  519 +++++
+ sound/xen/xen_snd_front_cfg.h              |   46 +
+ sound/xen/xen_snd_front_evtchnl.c          |  494 +++++
+ sound/xen/xen_snd_front_evtchnl.h          |   95 +
+ sound/xen/xen_snd_front_shbuf.c            |  194 ++
+ sound/xen/xen_snd_front_shbuf.h            |   36 +
+ 126 files changed, 8730 insertions(+), 1418 deletions(-)
+ create mode 100644 sound/firewire/dice/dice-alesis.c
+ create mode 100644 sound/firewire/dice/dice-extension.c
+ create mode 100644 sound/firewire/dice/dice-mytek.c
+ create mode 100644 sound/firewire/dice/dice-tcelectronic.c
+ create mode 100644 sound/pci/hda/hp_x360_helper.c
+ create mode 100644 sound/xen/Kconfig
+ create mode 100644 sound/xen/Makefile
+ create mode 100644 sound/xen/xen_snd_front.c
+ create mode 100644 sound/xen/xen_snd_front.h
+ create mode 100644 sound/xen/xen_snd_front_alsa.c
+ create mode 100644 sound/xen/xen_snd_front_alsa.h
+ create mode 100644 sound/xen/xen_snd_front_cfg.c
+ create mode 100644 sound/xen/xen_snd_front_cfg.h
+ create mode 100644 sound/xen/xen_snd_front_evtchnl.c
+ create mode 100644 sound/xen/xen_snd_front_evtchnl.h
+ create mode 100644 sound/xen/xen_snd_front_shbuf.c
+ create mode 100644 sound/xen/xen_snd_front_shbuf.h
+Merging sound-asoc/for-next (64ed10ff091f Merge branch 'asoc-4.18' into asoc-next)
+$ git merge sound-asoc/for-next
+Removing sound/soc/soc-cache.c
+Removing sound/soc/omap/omap-pcm.c
+Auto-merging sound/soc/fsl/fsl_ssi_dbg.c
+Auto-merging sound/soc/codecs/wm_adsp.c
+Removing include/sound/rt5640.h
+Removing include/sound/omap-pcm.h
+Auto-merging include/linux/mod_devicetable.h
+Auto-merging drivers/soc/qcom/Makefile
+Auto-merging drivers/soc/qcom/Kconfig
+Auto-merging drivers/media/i2c/tda1997x.c
+Auto-merging Documentation/sound/soc/platform.rst
+Auto-merging Documentation/sound/soc/codec.rst
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/soc/qcom/qcom,apr.txt      |   84 +
+ .../devicetree/bindings/sound/adi,ssm2305.txt      |   14 +
+ .../devicetree/bindings/sound/atmel-i2s.txt        |   47 +
+ .../devicetree/bindings/sound/cs42xx8.txt          |    2 +-
+ .../devicetree/bindings/sound/fsl,asrc.txt         |   10 +-
+ .../devicetree/bindings/sound/fsl,esai.txt         |    2 +
+ .../devicetree/bindings/sound/fsl,spdif.txt        |    2 +
+ .../devicetree/bindings/sound/fsl-sai.txt          |    8 +-
+ .../devicetree/bindings/sound/mt2701-afe-pcm.txt   |    4 +-
+ Documentation/devicetree/bindings/sound/mt6351.txt |   16 +
+ .../devicetree/bindings/sound/mt6797-afe-pcm.txt   |   42 +
+ .../devicetree/bindings/sound/mt6797-mt6351.txt    |   14 +
+ .../devicetree/bindings/sound/qcom,apq8096.txt     |  109 +
+ .../devicetree/bindings/sound/qcom,q6adm.txt       |   33 +
+ .../devicetree/bindings/sound/qcom,q6afe.txt       |  172 ++
+ .../devicetree/bindings/sound/qcom,q6asm.txt       |   33 +
+ .../devicetree/bindings/sound/qcom,q6core.txt      |   21 +
+ Documentation/devicetree/bindings/sound/rt274.txt  |    2 +-
+ Documentation/devicetree/bindings/sound/rt5514.txt |    2 +-
+ Documentation/devicetree/bindings/sound/rt5616.txt |    2 +-
+ Documentation/devicetree/bindings/sound/rt5640.txt |   35 +
+ Documentation/devicetree/bindings/sound/rt5645.txt |    2 +-
+ Documentation/devicetree/bindings/sound/rt5651.txt |    2 +-
+ Documentation/devicetree/bindings/sound/rt5663.txt |    2 +-
+ Documentation/devicetree/bindings/sound/rt5668.txt |   50 +
+ .../devicetree/bindings/sound/sgtl5000.txt         |    2 +-
+ .../devicetree/bindings/sound/ti,tas6424.txt       |    2 +
+ .../devicetree/bindings/sound/tscs42xx.txt         |    6 +
+ Documentation/devicetree/bindings/sound/wm8510.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8523.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8524.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8580.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8711.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8728.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8731.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8737.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8741.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8750.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8753.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8770.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8776.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8804.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8903.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8960.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8962.txt |    2 +-
+ Documentation/devicetree/bindings/sound/wm8994.txt |    2 +-
+ Documentation/sound/soc/codec.rst                  |    8 +-
+ Documentation/sound/soc/platform.rst               |   30 +-
+ arch/arm/mach-ep93xx/core.c                        |    1 +
+ drivers/media/i2c/tda1997x.c                       |   25 +-
+ drivers/soc/qcom/Kconfig                           |    9 +
+ drivers/soc/qcom/Makefile                          |    1 +
+ drivers/soc/qcom/apr.c                             |  378 +++
+ include/dt-bindings/soc/qcom,apr.h                 |   28 +
+ include/dt-bindings/sound/fsl-imx-audmux.h         |    7 +
+ include/dt-bindings/sound/qcom,q6afe.h             |  111 +
+ include/dt-bindings/sound/qcom,q6asm.h             |   22 +
+ include/dt-bindings/sound/rt5640.h                 |   25 +
+ include/linux/mfd/wm8350/audio.h                   |    3 -
+ include/linux/mod_devicetable.h                    |   11 +
+ include/linux/soc/qcom/apr.h                       |  128 +
+ include/sound/omap-pcm.h                           |   30 -
+ include/sound/rt5640.h                             |   27 -
+ include/sound/rt5668.h                             |   40 +
+ include/sound/soc-dai.h                            |    1 -
+ include/sound/soc.h                                |  397 +--
+ include/trace/events/asoc.h                        |    1 -
+ include/uapi/sound/asoc.h                          |   29 +-
+ sound/soc/Makefile                                 |    2 +-
+ sound/soc/amd/acp-da7219-max98357a.c               |   90 +-
+ sound/soc/amd/acp-pcm-dma.c                        |  623 +++--
+ sound/soc/amd/acp.h                                |   97 +-
+ sound/soc/atmel/Kconfig                            |    9 +
+ sound/soc/atmel/Makefile                           |    2 +
+ sound/soc/atmel/atmel-i2s.c                        |  765 ++++++
+ sound/soc/atmel/atmel_ssc_dai.c                    |    8 +-
+ sound/soc/bcm/Kconfig                              |    3 +-
+ sound/soc/cirrus/Kconfig                           |   17 +
+ sound/soc/cirrus/ep93xx-i2s.c                      |   80 +-
+ sound/soc/codecs/Kconfig                           |   25 +-
+ sound/soc/codecs/Makefile                          |    8 +
+ sound/soc/codecs/adau17x1.c                        |    9 +
+ sound/soc/codecs/max9860.c                         |   44 +-
+ sound/soc/codecs/max9860.h                         |   10 +-
+ sound/soc/codecs/mt6351.c                          | 1505 +++++++++++
+ sound/soc/codecs/mt6351.h                          |  105 +
+ sound/soc/codecs/nau8810.c                         |   19 +-
+ sound/soc/codecs/nau8824.c                         |   13 +-
+ sound/soc/codecs/pcm512x-i2c.c                     |   17 +-
+ sound/soc/codecs/rt1305.c                          | 1191 +++++++++
+ sound/soc/codecs/rt1305.h                          |  276 ++
+ sound/soc/codecs/rt5640.c                          |  553 +++-
+ sound/soc/codecs/rt5640.h                          |   46 +-
+ sound/soc/codecs/rt5645.c                          |   23 +
+ sound/soc/codecs/rt5663.c                          |   55 +-
+ sound/soc/codecs/rt5663.h                          |    2 -
+ sound/soc/codecs/rt5668.c                          | 2639 ++++++++++++++++++++
+ sound/soc/codecs/rt5668.h                          | 1318 ++++++++++
+ sound/soc/codecs/rt5670.c                          |    2 +-
+ sound/soc/codecs/rt5677.c                          |   13 +-
+ sound/soc/codecs/sgtl5000.c                        |   18 +-
+ sound/soc/codecs/sgtl5000.h                        |    5 +-
+ sound/soc/codecs/ssm2305.c                         |  104 +
+ sound/soc/codecs/tas6424.c                         |   72 +-
+ sound/soc/codecs/tas6424.h                         |    4 +
+ sound/soc/codecs/tfa9879.c                         |   48 +-
+ sound/soc/codecs/tfa9879.h                         |    7 +-
+ sound/soc/codecs/tscs42xx.c                        |  203 +-
+ sound/soc/codecs/tscs42xx.h                        |    2 +-
+ sound/soc/codecs/wm2200.c                          |    4 +-
+ sound/soc/codecs/wm5100.c                          |    8 +-
+ sound/soc/codecs/wm8782.c                          |    9 +
+ sound/soc/codecs/wm8904.c                          |    2 +-
+ sound/soc/codecs/wm_adsp.c                         |    8 +-
+ sound/soc/davinci/Kconfig                          |    2 +-
+ sound/soc/davinci/davinci-mcasp.c                  |   10 +-
+ sound/soc/fsl/fsl_esai.c                           |   20 +-
+ sound/soc/fsl/fsl_esai.h                           |    5 +-
+ sound/soc/fsl/fsl_sai.c                            |   16 +-
+ sound/soc/fsl/fsl_sai.h                            |    5 +-
+ sound/soc/fsl/fsl_spdif.c                          |   24 +-
+ sound/soc/fsl/fsl_spdif.h                          |    5 +-
+ sound/soc/fsl/fsl_ssi.c                            |   60 +-
+ sound/soc/fsl/fsl_ssi.h                            |    6 +-
+ sound/soc/fsl/fsl_ssi_dbg.c                        |   18 +-
+ sound/soc/hisilicon/hi6210-i2s.c                   |    2 +-
+ sound/soc/intel/Kconfig                            |    2 +-
+ sound/soc/intel/boards/bxt_da7219_max98357a.c      |    2 +-
+ sound/soc/intel/boards/bxt_rt298.c                 |    2 +-
+ sound/soc/intel/boards/byt-max98090.c              |    2 +-
+ sound/soc/intel/boards/bytcht_es8316.c             |    2 +-
+ sound/soc/intel/boards/bytcr_rt5640.c              |  568 ++++-
+ sound/soc/intel/boards/bytcr_rt5651.c              |    2 +-
+ sound/soc/intel/boards/cht_bsw_max98090_ti.c       |    2 +-
+ sound/soc/intel/boards/cht_bsw_nau8824.c           |    4 +-
+ sound/soc/intel/boards/cht_bsw_rt5645.c            |    2 +-
+ sound/soc/intel/boards/cht_bsw_rt5672.c            |   30 +-
+ sound/soc/intel/boards/kbl_da7219_max98357a.c      |    2 +-
+ sound/soc/intel/boards/kbl_rt5663_max98927.c       |    5 +-
+ .../soc/intel/boards/kbl_rt5663_rt5514_max98927.c  |    4 +-
+ sound/soc/intel/boards/skl_nau88l25_max98357a.c    |    2 +-
+ sound/soc/intel/boards/skl_nau88l25_ssm4567.c      |    2 +-
+ sound/soc/intel/boards/skl_rt286.c                 |    2 +-
+ sound/soc/intel/skylake/skl-debug.c                |    4 +-
+ sound/soc/intel/skylake/skl-messages.c             |    2 +-
+ sound/soc/intel/skylake/skl-pcm.c                  |   36 +-
+ sound/soc/intel/skylake/skl-sst-dsp.h              |    3 -
+ sound/soc/intel/skylake/skl-sst.c                  |   34 +-
+ sound/soc/intel/skylake/skl.c                      |    7 +
+ sound/soc/kirkwood/Kconfig                         |    1 -
+ sound/soc/mediatek/Kconfig                         |   20 +
+ sound/soc/mediatek/Makefile                        |    2 +
+ sound/soc/mediatek/common/Makefile                 |   14 +-
+ sound/soc/mediatek/common/mtk-afe-fe-dai.c         |   30 +-
+ sound/soc/mediatek/common/mtk-afe-fe-dai.h         |   10 +-
+ .../soc/mediatek/common/mtk-afe-platform-driver.c  |   96 +-
+ .../soc/mediatek/common/mtk-afe-platform-driver.h  |   22 +-
+ sound/soc/mediatek/common/mtk-base-afe.h           |   30 +-
+ sound/soc/mediatek/mt2701/Makefile                 |   14 +-
+ sound/soc/mediatek/mt2701/mt2701-afe-clock-ctrl.c  |   66 +-
+ sound/soc/mediatek/mt2701/mt2701-afe-clock-ctrl.h  |   23 +-
+ sound/soc/mediatek/mt2701/mt2701-afe-common.h      |   38 +-
+ sound/soc/mediatek/mt2701/mt2701-afe-pcm.c         |  349 +--
+ sound/soc/mediatek/mt2701/mt2701-cs42448.c         |   13 +-
+ sound/soc/mediatek/mt2701/mt2701-reg.h             |   11 +-
+ sound/soc/mediatek/mt2701/mt2701-wm8960.c          |   10 +-
+ sound/soc/mediatek/mt6797/Makefile                 |   12 +
+ sound/soc/mediatek/mt6797/mt6797-afe-clk.c         |  123 +
+ sound/soc/mediatek/mt6797/mt6797-afe-clk.h         |   17 +
+ sound/soc/mediatek/mt6797/mt6797-afe-common.h      |   52 +
+ sound/soc/mediatek/mt6797/mt6797-afe-pcm.c         |  912 +++++++
+ sound/soc/mediatek/mt6797/mt6797-dai-adda.c        |  384 +++
+ sound/soc/mediatek/mt6797/mt6797-interconnection.h |   33 +
+ sound/soc/mediatek/mt6797/mt6797-mt6351.c          |  177 ++
+ sound/soc/mediatek/mt6797/mt6797-reg.h             |  838 +++++++
+ sound/soc/mediatek/mt8173/mt8173-afe-common.h      |   10 +-
+ sound/soc/mediatek/mt8173/mt8173-afe-pcm.c         |   38 +-
+ sound/soc/mediatek/mt8173/mt8173-max98090.c        |   10 +-
+ sound/soc/mediatek/mt8173/mt8173-rt5650-rt5514.c   |   10 +-
+ sound/soc/mediatek/mt8173/mt8173-rt5650-rt5676.c   |   10 +-
+ sound/soc/mediatek/mt8173/mt8173-rt5650.c          |   10 +-
+ sound/soc/omap/Kconfig                             |   28 +-
+ sound/soc/omap/Makefile                            |    4 +-
+ sound/soc/omap/n810.c                              |   11 +-
+ sound/soc/omap/omap-dmic.c                         |    4 +-
+ sound/soc/omap/omap-hdmi-audio.c                   |    5 +-
+ sound/soc/omap/omap-mcbsp.c                        |    4 +-
+ sound/soc/omap/omap-mcpdm.c                        |    4 +-
+ sound/soc/omap/omap-pcm.c                          |  262 --
+ sound/soc/omap/sdma-pcm.c                          |   74 +
+ sound/soc/omap/sdma-pcm.h                          |   21 +
+ sound/soc/pxa/Kconfig                              |    1 -
+ sound/soc/pxa/pxa-ssp.c                            |   88 +-
+ sound/soc/qcom/Kconfig                             |   57 +-
+ sound/soc/qcom/Makefile                            |    5 +
+ sound/soc/qcom/apq8096.c                           |  255 ++
+ sound/soc/qcom/qdsp6/Makefile                      |    8 +
+ sound/soc/qcom/qdsp6/q6adm.c                       |  646 +++++
+ sound/soc/qcom/qdsp6/q6adm.h                       |   27 +
+ sound/soc/qcom/qdsp6/q6afe-dai.c                   | 1303 ++++++++++
+ sound/soc/qcom/qdsp6/q6afe.c                       | 1495 +++++++++++
+ sound/soc/qcom/qdsp6/q6afe.h                       |  211 ++
+ sound/soc/qcom/qdsp6/q6asm-dai.c                   |  624 +++++
+ sound/soc/qcom/qdsp6/q6asm.c                       | 1399 +++++++++++
+ sound/soc/qcom/qdsp6/q6asm.h                       |   69 +
+ sound/soc/qcom/qdsp6/q6core.c                      |  380 +++
+ sound/soc/qcom/qdsp6/q6core.h                      |   15 +
+ sound/soc/qcom/qdsp6/q6dsp-common.c                |   66 +
+ sound/soc/qcom/qdsp6/q6dsp-common.h                |   24 +
+ sound/soc/qcom/qdsp6/q6dsp-errno.h                 |   51 +
+ sound/soc/qcom/qdsp6/q6routing.c                   | 1006 ++++++++
+ sound/soc/qcom/qdsp6/q6routing.h                   |    9 +
+ sound/soc/rockchip/rk3399_gru_sound.c              |   46 +-
+ sound/soc/sh/Kconfig                               |    6 +-
+ sound/soc/sh/rcar/cmd.c                            |   15 +-
+ sound/soc/sh/rcar/core.c                           |   49 +-
+ sound/soc/sh/rcar/dma.c                            |   11 +-
+ sound/soc/sh/rcar/gen.c                            |    3 +-
+ sound/soc/sh/rcar/rsnd.h                           |    4 +-
+ sound/soc/sh/rcar/ssi.c                            |   13 +-
+ sound/soc/soc-cache.c                              |   53 -
+ sound/soc/soc-compress.c                           |  391 +--
+ sound/soc/soc-core.c                               |  794 +-----
+ sound/soc/soc-devres.c                             |   35 -
+ sound/soc/soc-io.c                                 |   83 -
+ sound/soc/soc-jack.c                               |   22 -
+ sound/soc/soc-pcm.c                                |  147 +-
+ sound/soc/soc-topology.c                           |   30 +-
+ sound/soc/uniphier/aio-compress.c                  |   13 -
+ sound/soc/uniphier/aio-core.c                      |   71 +-
+ sound/soc/uniphier/aio-cpu.c                       |  153 +-
+ sound/soc/uniphier/aio-dma.c                       |   13 -
+ sound/soc/uniphier/aio-ld11.c                      |   13 -
+ sound/soc/uniphier/aio-reg.h                       |   46 +-
+ sound/soc/uniphier/aio.h                           |   20 +-
+ sound/soc/uniphier/evea.c                          |   55 +-
+ sound/soc/zte/zx-i2s.c                             |    5 +-
+ 237 files changed, 22826 insertions(+), 3831 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/soc/qcom/qcom,apr.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/adi,ssm2305.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/atmel-i2s.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/mt6351.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/mt6797-afe-pcm.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/mt6797-mt6351.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/qcom,apq8096.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/qcom,q6adm.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/qcom,q6afe.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/qcom,q6asm.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/qcom,q6core.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/rt5668.txt
+ create mode 100644 drivers/soc/qcom/apr.c
+ create mode 100644 include/dt-bindings/soc/qcom,apr.h
+ create mode 100644 include/dt-bindings/sound/qcom,q6afe.h
+ create mode 100644 include/dt-bindings/sound/qcom,q6asm.h
+ create mode 100644 include/dt-bindings/sound/rt5640.h
+ create mode 100644 include/linux/soc/qcom/apr.h
+ delete mode 100644 include/sound/omap-pcm.h
+ delete mode 100644 include/sound/rt5640.h
+ create mode 100644 include/sound/rt5668.h
+ create mode 100644 sound/soc/atmel/atmel-i2s.c
+ create mode 100644 sound/soc/codecs/mt6351.c
+ create mode 100644 sound/soc/codecs/mt6351.h
+ create mode 100644 sound/soc/codecs/rt1305.c
+ create mode 100644 sound/soc/codecs/rt1305.h
+ create mode 100644 sound/soc/codecs/rt5668.c
+ create mode 100644 sound/soc/codecs/rt5668.h
+ create mode 100644 sound/soc/codecs/ssm2305.c
+ create mode 100644 sound/soc/mediatek/mt6797/Makefile
+ create mode 100644 sound/soc/mediatek/mt6797/mt6797-afe-clk.c
+ create mode 100644 sound/soc/mediatek/mt6797/mt6797-afe-clk.h
+ create mode 100644 sound/soc/mediatek/mt6797/mt6797-afe-common.h
+ create mode 100644 sound/soc/mediatek/mt6797/mt6797-afe-pcm.c
+ create mode 100644 sound/soc/mediatek/mt6797/mt6797-dai-adda.c
+ create mode 100644 sound/soc/mediatek/mt6797/mt6797-interconnection.h
+ create mode 100644 sound/soc/mediatek/mt6797/mt6797-mt6351.c
+ create mode 100644 sound/soc/mediatek/mt6797/mt6797-reg.h
+ delete mode 100644 sound/soc/omap/omap-pcm.c
+ create mode 100644 sound/soc/omap/sdma-pcm.c
+ create mode 100644 sound/soc/omap/sdma-pcm.h
+ create mode 100644 sound/soc/qcom/apq8096.c
+ create mode 100644 sound/soc/qcom/qdsp6/Makefile
+ create mode 100644 sound/soc/qcom/qdsp6/q6adm.c
+ create mode 100644 sound/soc/qcom/qdsp6/q6adm.h
+ create mode 100644 sound/soc/qcom/qdsp6/q6afe-dai.c
+ create mode 100644 sound/soc/qcom/qdsp6/q6afe.c
+ create mode 100644 sound/soc/qcom/qdsp6/q6afe.h
+ create mode 100644 sound/soc/qcom/qdsp6/q6asm-dai.c
+ create mode 100644 sound/soc/qcom/qdsp6/q6asm.c
+ create mode 100644 sound/soc/qcom/qdsp6/q6asm.h
+ create mode 100644 sound/soc/qcom/qdsp6/q6core.c
+ create mode 100644 sound/soc/qcom/qdsp6/q6core.h
+ create mode 100644 sound/soc/qcom/qdsp6/q6dsp-common.c
+ create mode 100644 sound/soc/qcom/qdsp6/q6dsp-common.h
+ create mode 100644 sound/soc/qcom/qdsp6/q6dsp-errno.h
+ create mode 100644 sound/soc/qcom/qdsp6/q6routing.c
+ create mode 100644 sound/soc/qcom/qdsp6/q6routing.h
+ delete mode 100644 sound/soc/soc-cache.c
+Merging modules/modules-next (c554b8986801 module: Allow to always show the status of modsign)
+$ git merge modules/modules-next
+Auto-merging kernel/module.c
+Merge made by the 'recursive' strategy.
+ kernel/module.c | 4 +---
+ 1 file changed, 1 insertion(+), 3 deletions(-)
+Merging input/next (22a844b85368 Input: ti_am335x_tsc - prevent system suspend when TSC is in use)
+$ git merge input/next
+Auto-merging drivers/input/mouse/synaptics.c
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/input/elan_i2c.txt         |   1 +
+ MAINTAINERS                                        |   6 +
+ drivers/input/input.c                              |   3 +-
+ drivers/input/joystick/as5011.c                    |   4 +-
+ drivers/input/joystick/gamecon.c                   |  27 +-
+ drivers/input/joystick/xpad.c                      |  33 +-
+ drivers/input/keyboard/cros_ec_keyb.c              |  33 +-
+ drivers/input/misc/ati_remote2.c                   |   2 +-
+ drivers/input/mouse/Kconfig                        |  12 +
+ drivers/input/mouse/alps.c                         |  80 ++--
+ drivers/input/mouse/elan_i2c_core.c                |  88 +++-
+ drivers/input/mouse/elantech.c                     | 479 ++++++++++++++-----
+ drivers/input/mouse/elantech.h                     |  69 ++-
+ drivers/input/mouse/psmouse-base.c                 |  21 +-
+ drivers/input/mouse/psmouse-smbus.c                |  24 +-
+ drivers/input/mouse/psmouse.h                      |   2 +
+ drivers/input/mouse/synaptics.c                    |   2 +-
+ drivers/input/touchscreen/Kconfig                  |  11 +
+ drivers/input/touchscreen/Makefile                 |   1 +
+ drivers/input/touchscreen/atmel_mxt_ts.c           | 170 +------
+ drivers/input/touchscreen/chipone_icn8505.c        | 520 +++++++++++++++++++++
+ drivers/input/touchscreen/mk712.c                  |   2 +-
+ drivers/input/touchscreen/ti_am335x_tsc.c          |   7 +
+ drivers/input/touchscreen/usbtouchscreen.c         |  94 ++++
+ drivers/mfd/cros_ec.c                              |  19 +-
+ drivers/platform/chrome/chromeos_laptop.c          | 307 ++++++++++--
+ 26 files changed, 1599 insertions(+), 418 deletions(-)
+ create mode 100644 drivers/input/touchscreen/chipone_icn8505.c
+Merging block/for-next (04c4950d5b37 block: fixup bioset_integrity_create() call)
+$ git merge block/for-next
+Auto-merging include/linux/fs.h
+Auto-merging fs/xfs/xfs_super.c
+Auto-merging fs/xfs/xfs_aops.c
+Auto-merging fs/nfsd/blocklayout.c
+Auto-merging fs/btrfs/extent_io.c
+Auto-merging fs/block_dev.c
+Auto-merging drivers/scsi/sg.c
+Auto-merging drivers/scsi/scsi_transport_srp.c
+Auto-merging drivers/scsi/scsi_lib.c
+Removing drivers/sbus/char/jsflash.c
+Auto-merging drivers/s390/block/dasd.c
+Auto-merging drivers/nvme/target/io-cmd-bdev.c
+Auto-merging drivers/nvme/host/pci.c
+Auto-merging drivers/nvme/host/nvme.h
+Auto-merging drivers/nvme/host/core.c
+Auto-merging drivers/mmc/core/block.c
+Auto-merging drivers/md/dm.c
+Auto-merging drivers/md/dm-integrity.c
+Auto-merging drivers/ide/ide-tape.c
+Auto-merging drivers/ide/ide-cd.c
+Auto-merging drivers/block/rbd.c
+Auto-merging drivers/block/pktcdvd.c
+Auto-merging drivers/block/loop.c
+Auto-merging drivers/block/drbd/drbd_main.c
+CONFLICT (content): Merge conflict in drivers/block/drbd/drbd_main.c
+Auto-merging drivers/block/drbd/drbd_int.h
+Auto-merging drivers/block/DAC960.c
+CONFLICT (content): Merge conflict in drivers/block/DAC960.c
+Auto-merging drivers/ata/libata-eh.c
+Auto-merging block/genhd.c
+Removing arch/sparc/include/uapi/asm/jsflash.h
+Auto-merging MAINTAINERS
+Resolved 'drivers/block/DAC960.c' using previous resolution.
+Resolved 'drivers/block/drbd/drbd_main.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master e401b51c1556] Merge remote-tracking branch 'block/for-next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/block/null_blk.txt                |   9 +-
+ Documentation/scsi/scsi_eh.txt                  |  15 +-
+ MAINTAINERS                                     |   2 +-
+ arch/sparc/include/uapi/asm/jsflash.h           |  40 --
+ block/bfq-cgroup.c                              |  40 +-
+ block/bfq-iosched.c                             | 203 +++++---
+ block/bfq-iosched.h                             |  16 +-
+ block/bio-integrity.c                           |  29 +-
+ block/bio.c                                     | 189 ++++---
+ block/blk-core.c                                | 101 ++--
+ block/blk-integrity.c                           |  12 +-
+ block/blk-lib.c                                 |  12 +-
+ block/blk-merge.c                               |  19 +-
+ block/blk-mq-debugfs.c                          |   1 -
+ block/blk-mq-sched.c                            |  34 +-
+ block/blk-mq-sysfs.c                            |   6 +-
+ block/blk-mq-tag.c                              |  14 +-
+ block/blk-mq.c                                  | 338 ++++--------
+ block/blk-mq.h                                  |  42 +-
+ block/blk-stat.c                                |  10 +-
+ block/blk-stat.h                                |  45 +-
+ block/blk-sysfs.c                               |  80 +--
+ block/blk-throttle.c                            |  35 +-
+ block/blk-timeout.c                             |   6 +-
+ block/blk-wbt.c                                 | 129 +++--
+ block/blk-wbt.h                                 |  55 +-
+ block/blk-zoned.c                               |   8 +-
+ block/blk.h                                     |   2 +-
+ block/bounce.c                                  |  52 +-
+ block/bsg-lib.c                                 |   6 +-
+ block/bsg.c                                     |  44 +-
+ block/cfq-iosched.c                             |  66 ++-
+ block/deadline-iosched.c                        |   3 +-
+ block/genhd.c                                   |  37 +-
+ block/kyber-iosched.c                           | 199 +++++--
+ block/mq-deadline.c                             |   3 +-
+ block/partition-generic.c                       |  26 +-
+ block/scsi_ioctl.c                              |  10 +-
+ drivers/ata/libata-eh.c                         |  51 --
+ drivers/block/DAC960.c                          |  11 +-
+ drivers/block/DAC960.h                          |   1 -
+ drivers/block/aoe/aoeblk.c                      |  11 +-
+ drivers/block/aoe/aoecmd.c                      |   3 +-
+ drivers/block/brd.c                             |  10 +-
+ drivers/block/drbd/drbd_bitmap.c                |   5 +-
+ drivers/block/drbd/drbd_debugfs.c               |  20 +-
+ drivers/block/drbd/drbd_int.h                   |  10 +-
+ drivers/block/drbd/drbd_main.c                  |  73 +--
+ drivers/block/drbd/drbd_receiver.c              |   6 +-
+ drivers/block/drbd/drbd_req.c                   |   4 +-
+ drivers/block/drbd/drbd_req.h                   |   2 +-
+ drivers/block/floppy.c                          |   2 +-
+ drivers/block/loop.c                            |  17 +-
+ drivers/block/loop.h                            |   1 +
+ drivers/block/mtip32xx/mtip32xx.c               |  29 +-
+ drivers/block/nbd.c                             |  77 ++-
+ drivers/block/null_blk.c                        |  36 +-
+ drivers/block/paride/pd.c                       |   2 +-
+ drivers/block/pktcdvd.c                         |  60 +--
+ drivers/block/ps3disk.c                         |   2 -
+ drivers/block/rbd.c                             |  44 +-
+ drivers/block/rsxx/core.c                       |   6 +-
+ drivers/block/sx8.c                             |   2 +-
+ drivers/block/virtio_blk.c                      |   8 +-
+ drivers/block/xen-blkback/blkback.c             |   2 +-
+ drivers/block/xen-blkback/xenbus.c              |   4 +-
+ drivers/block/xen-blkfront.c                    |   7 +-
+ drivers/cdrom/cdrom.c                           |   2 +-
+ drivers/ide/ide-atapi.c                         |   2 +-
+ drivers/ide/ide-cd.c                            |   2 +-
+ drivers/ide/ide-cd_ioctl.c                      |   2 +-
+ drivers/ide/ide-devsets.c                       |   2 +-
+ drivers/ide/ide-disk.c                          |   2 +-
+ drivers/ide/ide-ioctls.c                        |   4 +-
+ drivers/ide/ide-park.c                          |   4 +-
+ drivers/ide/ide-pm.c                            |   5 +-
+ drivers/ide/ide-tape.c                          |   4 +-
+ drivers/ide/ide-taskfile.c                      |   4 +-
+ drivers/lightnvm/pblk-core.c                    |  30 +-
+ drivers/lightnvm/pblk-init.c                    |  72 +--
+ drivers/lightnvm/pblk-read.c                    |   4 +-
+ drivers/lightnvm/pblk-recovery.c                |   2 +-
+ drivers/lightnvm/pblk-write.c                   |   8 +-
+ drivers/lightnvm/pblk.h                         |  14 +-
+ drivers/md/bcache/bcache.h                      |  14 +-
+ drivers/md/bcache/bset.c                        |  13 +-
+ drivers/md/bcache/bset.h                        |   2 +-
+ drivers/md/bcache/btree.c                       |   4 +-
+ drivers/md/bcache/io.c                          |   4 +-
+ drivers/md/bcache/request.c                     |  18 +-
+ drivers/md/bcache/super.c                       | 109 ++--
+ drivers/md/bcache/sysfs.c                       |  51 +-
+ drivers/md/bcache/util.c                        |  35 --
+ drivers/md/bcache/util.h                        |   5 -
+ drivers/md/dm-bio-prison-v1.c                   |  13 +-
+ drivers/md/dm-bio-prison-v2.c                   |  13 +-
+ drivers/md/dm-cache-target.c                    |  25 +-
+ drivers/md/dm-core.h                            |   4 +-
+ drivers/md/dm-crypt.c                           |  60 +--
+ drivers/md/dm-integrity.c                       |  15 +-
+ drivers/md/dm-io.c                              |  29 +-
+ drivers/md/dm-kcopyd.c                          |  22 +-
+ drivers/md/dm-log-userspace-base.c              |  19 +-
+ drivers/md/dm-mpath.c                           |   3 +-
+ drivers/md/dm-region-hash.c                     |  23 +-
+ drivers/md/dm-rq.c                              |   4 +-
+ drivers/md/dm-snap.c                            |  17 +-
+ drivers/md/dm-thin.c                            |  32 +-
+ drivers/md/dm-verity-fec.c                      |  55 +-
+ drivers/md/dm-verity-fec.h                      |   8 +-
+ drivers/md/dm-zoned-target.c                    |  13 +-
+ drivers/md/dm.c                                 |  55 +-
+ drivers/md/md-faulty.c                          |   2 +-
+ drivers/md/md-linear.c                          |   2 +-
+ drivers/md/md-multipath.c                       |  17 +-
+ drivers/md/md-multipath.h                       |   2 +-
+ drivers/md/md.c                                 |  61 +--
+ drivers/md/md.h                                 |   4 +-
+ drivers/md/raid0.c                              |   5 +-
+ drivers/md/raid1.c                              |  76 ++-
+ drivers/md/raid1.h                              |   6 +-
+ drivers/md/raid10.c                             |  60 ++-
+ drivers/md/raid10.h                             |   6 +-
+ drivers/md/raid5-cache.c                        |  43 +-
+ drivers/md/raid5-ppl.c                          |  42 +-
+ drivers/md/raid5.c                              |  12 +-
+ drivers/md/raid5.h                              |   2 +-
+ drivers/memstick/core/ms_block.c                |   6 -
+ drivers/memstick/core/mspro_block.c             |   6 -
+ drivers/message/fusion/mptsas.c                 |   2 +-
+ drivers/mmc/core/block.c                        |  12 +-
+ drivers/mmc/core/queue.c                        |   5 +-
+ drivers/mtd/mtd_blkdevs.c                       |  20 +-
+ drivers/nvme/host/core.c                        |  42 +-
+ drivers/nvme/host/fabrics.c                     |  21 +-
+ drivers/nvme/host/fc.c                          |  15 +-
+ drivers/nvme/host/nvme.h                        |   5 +-
+ drivers/nvme/host/pci.c                         | 227 ++++----
+ drivers/nvme/host/rdma.c                        |  12 +-
+ drivers/nvme/host/trace.h                       |   4 +-
+ drivers/nvme/target/Makefile                    |   4 +-
+ drivers/nvme/target/admin-cmd.c                 |  10 +-
+ drivers/nvme/target/core.c                      |  55 +-
+ drivers/nvme/target/discovery.c                 |   2 -
+ drivers/nvme/target/fabrics-cmd.c               |   4 -
+ drivers/nvme/target/fc.c                        |   2 +-
+ drivers/nvme/target/{io-cmd.c => io-cmd-bdev.c} |  77 +--
+ drivers/nvme/target/io-cmd-file.c               | 300 +++++++++++
+ drivers/nvme/target/loop.c                      |   4 +-
+ drivers/nvme/target/nvmet.h                     |  28 +-
+ drivers/s390/block/dasd.c                       |   6 +-
+ drivers/sbus/char/Kconfig                       |   7 -
+ drivers/sbus/char/Makefile                      |   1 -
+ drivers/sbus/char/jsflash.c                     | 658 ------------------------
+ drivers/scsi/gdth.c                             |   2 +-
+ drivers/scsi/libiscsi.c                         |   6 +-
+ drivers/scsi/megaraid/megaraid_sas_base.c       |   2 +-
+ drivers/scsi/mvumi.c                            |   2 +-
+ drivers/scsi/osd/osd_initiator.c                |  24 +-
+ drivers/scsi/osst.c                             |   2 +-
+ drivers/scsi/qla4xxx/ql4_os.c                   |   2 +-
+ drivers/scsi/scsi_error.c                       |  10 +-
+ drivers/scsi/scsi_lib.c                         |   4 +-
+ drivers/scsi/scsi_transport_fc.c                |  16 +-
+ drivers/scsi/scsi_transport_iscsi.c             |   2 +-
+ drivers/scsi/scsi_transport_sas.c               |  19 +-
+ drivers/scsi/scsi_transport_srp.c               |   4 +-
+ drivers/scsi/sg.c                               |   2 +-
+ drivers/scsi/st.c                               |   2 +-
+ drivers/scsi/ufs/ufshcd.c                       |   6 +-
+ drivers/target/target_core_iblock.c             |  16 +-
+ drivers/target/target_core_iblock.h             |   2 +-
+ drivers/target/target_core_pscsi.c              |   3 +-
+ fs/block_dev.c                                  |  24 +-
+ fs/btrfs/extent_io.c                            |  25 +-
+ fs/direct-io.c                                  |   4 +-
+ fs/exofs/ore.c                                  |  10 +-
+ fs/exofs/super.c                                |   2 +-
+ fs/nfsd/blocklayout.c                           |   2 +-
+ fs/xfs/xfs_aops.c                               |   2 +-
+ fs/xfs/xfs_aops.h                               |   2 +-
+ fs/xfs/xfs_super.c                              |  11 +-
+ include/linux/bio.h                             |  42 +-
+ include/linux/blk-mq.h                          |   3 +-
+ include/linux/blk_types.h                       |  49 +-
+ include/linux/blkdev.h                          | 110 ++--
+ include/linux/bsg-lib.h                         |   3 +-
+ include/linux/bsg.h                             |   6 +-
+ include/linux/fs.h                              |   2 +-
+ include/linux/libata.h                          |   2 -
+ include/linux/mempool.h                         |  34 ++
+ include/linux/pktcdvd.h                         |   2 +-
+ include/linux/sbitmap.h                         |  36 ++
+ include/scsi/osd_initiator.h                    |   6 +-
+ include/scsi/scsi_host.h                        |   2 +-
+ kernel/power/swap.c                             |  14 +-
+ lib/sbitmap.c                                   | 113 ++--
+ mm/backing-dev.c                                |  18 +-
+ mm/mempool.c                                    | 108 +++-
+ 199 files changed, 2810 insertions(+), 3095 deletions(-)
+ delete mode 100644 arch/sparc/include/uapi/asm/jsflash.h
+ rename drivers/nvme/target/{io-cmd.c => io-cmd-bdev.c} (76%)
+ create mode 100644 drivers/nvme/target/io-cmd-file.c
+ delete mode 100644 drivers/sbus/char/jsflash.c
+Merging lightnvm/for-next (1c6286f26301 lightnvm: fix some error code in pblk-init.c)
+$ git merge lightnvm/for-next
+Already up to date.
+Merging device-mapper/for-next (1ea5c3756693 x86: optimize memcpy_flushcache)
+$ git merge device-mapper/for-next
+Merge made by the 'recursive' strategy.
+ Documentation/device-mapper/writecache.txt |   68 +
+ arch/x86/include/asm/string_64.h           |   28 +-
+ arch/x86/lib/usercopy_64.c                 |    4 +-
+ drivers/md/Kconfig                         |   11 +
+ drivers/md/Makefile                        |    1 +
+ drivers/md/dm-ioctl.c                      |    3 +-
+ drivers/md/dm-writecache.c                 | 2325 ++++++++++++++++++++++++++++
+ kernel/sched/swait.c                       |    2 +
+ 8 files changed, 2438 insertions(+), 4 deletions(-)
+ create mode 100644 Documentation/device-mapper/writecache.txt
+ create mode 100644 drivers/md/dm-writecache.c
+$ git am -3 ../patches/0001-dm-fixup-for-block-Drop-bioset_create.patch
+Applying: dm: fixup for "block: Drop bioset_create()"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/md/dm-writecache.c
+$ git add -A .
+$ git commit -v -a --amend
+[master 254350c3b7b0] Merge remote-tracking branch 'device-mapper/for-next'
+ Date: Thu May 31 13:45:47 2018 +1000
+Merging pcmcia/pcmcia-next (c3a5307873d4 pcmcia: Use module_pcmcia_driver for scsi drivers)
+$ git merge pcmcia/pcmcia-next
+Merge made by the 'recursive' strategy.
+ drivers/char/pcmcia/cm4040_cs.c    |  2 +-
+ drivers/scsi/pcmcia/aha152x_stub.c | 14 +-------------
+ drivers/scsi/pcmcia/nsp_cs.c       | 15 +--------------
+ drivers/scsi/pcmcia/nsp_cs.h       |  4 ----
+ drivers/scsi/pcmcia/qlogic_stub.c  | 13 +------------
+ drivers/scsi/pcmcia/sym53c500_cs.c | 16 +---------------
+ 6 files changed, 5 insertions(+), 59 deletions(-)
+Merging mmc/next (af6b8ff4bce4 mmc: sunxi: Use ifdef rather than __maybe_unused)
+$ git merge mmc/next
+Auto-merging include/linux/mmc/sdio_ids.h
+Auto-merging drivers/mmc/core/block.c
+Auto-merging arch/mips/boot/dts/ingenic/jz4780.dtsi
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/mmc/amlogic,meson-gx.txt   |   3 +
+ .../devicetree/bindings/mmc/bluefield-dw-mshc.txt  |  29 +++
+ Documentation/devicetree/bindings/mmc/jz4740.txt   |  38 ++++
+ Documentation/devicetree/bindings/mmc/mmc.txt      |   6 +
+ Documentation/devicetree/bindings/mmc/tmio_mmc.txt |   2 +
+ arch/mips/boot/dts/ingenic/ci20.dts                |  34 ++++
+ arch/mips/boot/dts/ingenic/jz4780.dtsi             |  52 ++++++
+ arch/mips/configs/ci20_defconfig                   |   4 +
+ drivers/mmc/core/block.c                           |  11 +-
+ drivers/mmc/core/card.h                            |   6 +
+ drivers/mmc/core/core.c                            |  67 +++++--
+ drivers/mmc/core/core.h                            |   4 +-
+ drivers/mmc/core/host.c                            |  16 +-
+ drivers/mmc/core/mmc.c                             |  16 +-
+ drivers/mmc/core/pwrseq_simple.c                   |  14 +-
+ drivers/mmc/core/quirks.h                          |   3 +
+ drivers/mmc/core/sd.c                              |  12 +-
+ drivers/mmc/core/sdio.c                            |  70 ++++---
+ drivers/mmc/core/slot-gpio.c                       |  22 ++-
+ drivers/mmc/host/Kconfig                           |  32 ++--
+ drivers/mmc/host/Makefile                          |   1 +
+ drivers/mmc/host/android-goldfish.c                |   8 +-
+ drivers/mmc/host/atmel-mci.c                       |  21 +--
+ drivers/mmc/host/au1xmmc.c                         |  12 +-
+ drivers/mmc/host/davinci_mmc.c                     |   6 +-
+ drivers/mmc/host/dw_mmc-bluefield.c                |  81 ++++++++
+ drivers/mmc/host/dw_mmc-rockchip.c                 |   5 +-
+ drivers/mmc/host/dw_mmc.c                          |   4 +
+ drivers/mmc/host/jz4740_mmc.c                      | 203 ++++++++++++++-------
+ drivers/mmc/host/meson-gx-mmc.c                    |  72 ++++++--
+ drivers/mmc/host/mmci.c                            |   5 -
+ drivers/mmc/host/mtk-sd.c                          |  63 +++++--
+ drivers/mmc/host/mxcmmc.c                          |  22 ++-
+ drivers/mmc/host/renesas_sdhi_internal_dmac.c      |  31 ++--
+ drivers/mmc/host/rtsx_usb_sdmmc.c                  |  19 +-
+ drivers/mmc/host/sdhci-bcm-kona.c                  |   4 +-
+ drivers/mmc/host/sdhci-cadence.c                   |  22 ++-
+ drivers/mmc/host/sdhci-esdhc-imx.c                 |  22 +++
+ drivers/mmc/host/sdhci-msm.c                       |  84 ++++++++-
+ drivers/mmc/host/sdhci-of-arasan.c                 |   9 +-
+ drivers/mmc/host/sdhci-pci-core.c                  |  38 +++-
+ drivers/mmc/host/sdhci-pci.h                       |   1 +
+ drivers/mmc/host/sdhci-pic32.c                     |   4 +-
+ drivers/mmc/host/sdhci-pxav2.c                     |   4 +-
+ drivers/mmc/host/sdhci-pxav3.c                     |   4 +-
+ drivers/mmc/host/sdhci-s3c.c                       |   4 +-
+ drivers/mmc/host/sdhci-spear.c                     |   4 +-
+ drivers/mmc/host/sdhci-st.c                        |   4 +-
+ drivers/mmc/host/sdhci-tegra.c                     |   2 +-
+ drivers/mmc/host/sdhci-xenon-phy.c                 |  11 +-
+ drivers/mmc/host/sunxi-mmc.c                       | 187 ++++++++++++-------
+ drivers/mmc/host/ushc.c                            |   4 +-
+ drivers/mmc/host/wbsd.c                            |  68 +++----
+ drivers/mmc/host/wmt-sdmmc.c                       |   6 +-
+ include/dt-bindings/dma/jz4780-dma.h               |  49 +++++
+ include/linux/mmc/card.h                           |   2 +
+ include/linux/mmc/core.h                           |   1 +
+ include/linux/mmc/host.h                           |   2 +
+ include/linux/mmc/sdio_ids.h                       |   1 +
+ 59 files changed, 1142 insertions(+), 389 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/mmc/bluefield-dw-mshc.txt
+ create mode 100644 Documentation/devicetree/bindings/mmc/jz4740.txt
+ create mode 100644 drivers/mmc/host/dw_mmc-bluefield.c
+ create mode 100644 include/dt-bindings/dma/jz4780-dma.h
+Merging kgdb/kgdb-next (2cf2f0d5b91f kdb: use memmove instead of overlapping memcpy)
+$ git merge kgdb/kgdb-next
+Already up to date.
+Merging md/for-next (5a409b4f56d5 MD: fix lock contention for flush bios)
+$ git merge md/for-next
+Auto-merging drivers/md/raid5.h
+Auto-merging drivers/md/raid5.c
+Auto-merging drivers/md/raid10.c
+Auto-merging drivers/md/raid1.c
+Auto-merging drivers/md/md.h
+Auto-merging drivers/md/md.c
+CONFLICT (content): Merge conflict in drivers/md/md.c
+Recorded preimage for 'drivers/md/md.c'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+Recorded resolution for 'drivers/md/md.c'.
+[master d0fb73f0150d] Merge remote-tracking branch 'md/for-next'
+$ git diff -M --stat --summary HEAD^..
+ drivers/md/md.c     | 165 +++++++++++++++++++++++++++++++++++-----------------
+ drivers/md/md.h     |  22 ++++---
+ drivers/md/raid1.c  |   4 +-
+ drivers/md/raid10.c |  10 ++--
+ drivers/md/raid5.c  |  12 +++-
+ drivers/md/raid5.h  |   1 +
+ 6 files changed, 144 insertions(+), 70 deletions(-)
+Merging mfd/for-mfd-next (d28b62520830 mfd: intel-lpss: Program REMAP register in PIO mode)
+$ git merge mfd/for-mfd-next
+Removing include/linux/mfd/syscon/exynos5-pmu.h
+Removing include/linux/mfd/syscon/exynos4-pmu.h
+Auto-merging drivers/mfd/sm501.c
+Removing drivers/mfd/cros_ec_acpi_gpe.c
+Auto-merging drivers/mfd/cros_ec.c
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/input/mtk-pmic-keys.txt    |  43 ++
+ .../devicetree/bindings/mfd/motorola-cpcap.txt     |  42 ++
+ Documentation/devicetree/bindings/mfd/mt6397.txt   |   6 +
+ MAINTAINERS                                        |  10 +
+ drivers/input/keyboard/Kconfig                     |   9 +
+ drivers/input/keyboard/Makefile                    |   1 +
+ drivers/input/keyboard/mtk-pmic-keys.c             | 339 +++++++++++++++
+ drivers/mfd/Makefile                               |   1 -
+ drivers/mfd/ab8500-debugfs.c                       |   3 +-
+ drivers/mfd/abx500-core.c                          |  25 +-
+ drivers/mfd/arizona-core.c                         |  53 ++-
+ drivers/mfd/atmel-smc.c                            |   1 +
+ drivers/mfd/axp20x.c                               | 252 +++--------
+ drivers/mfd/cros_ec.c                              |  41 +-
+ drivers/mfd/cros_ec_acpi_gpe.c                     | 103 -----
+ drivers/mfd/cros_ec_dev.c                          |  31 +-
+ drivers/mfd/cros_ec_i2c.c                          |  17 +-
+ drivers/mfd/da9062-core.c                          | 462 +++++----------------
+ drivers/mfd/htc-i2cpld.c                           |   4 +-
+ drivers/mfd/intel-lpss.c                           |   4 +-
+ drivers/mfd/janz-cmodio.c                          |   4 +-
+ drivers/mfd/jz4740-adc.c                           |   4 +-
+ drivers/mfd/max8997.c                              |   4 +-
+ drivers/mfd/mt6397-core.c                          |  26 +-
+ drivers/mfd/omap-usb-tll.c                         |  60 ++-
+ drivers/mfd/pcf50633-core.c                        |   7 +-
+ drivers/mfd/rave-sp.c                              | 118 +++++-
+ drivers/mfd/rc5t583.c                              |   6 +-
+ drivers/mfd/si476x-i2c.c                           |   6 +-
+ drivers/mfd/sm501.c                                |  32 +-
+ drivers/mfd/smsc-ece1099.c                         |   7 +-
+ drivers/mfd/sprd-sc27xx-spi.c                      |   3 +
+ drivers/mfd/stm32-timers.c                         | 201 ++++++++-
+ drivers/mfd/syscon.c                               |   2 +
+ drivers/mfd/ti_am335x_tscadc.c                     |   5 +-
+ drivers/mfd/tps65090.c                             |   4 +-
+ drivers/mfd/tps6586x.c                             |   4 +-
+ drivers/mfd/tps65910.c                             |  18 +-
+ drivers/mfd/tps65911-comparator.c                  |  17 +-
+ drivers/mfd/tps68470.c                             |  10 +-
+ drivers/mfd/tps80031.c                             |   4 +-
+ drivers/mfd/twl-core.c                             |   2 +-
+ drivers/mfd/twl6030-irq.c                          |   4 +-
+ drivers/mfd/viperboard.c                           |   4 +-
+ drivers/platform/chrome/cros_ec_debugfs.c          |  20 +
+ drivers/pwm/pwm-stm32.c                            | 257 ++++++++++++
+ drivers/rtc/rtc-mt6397.c                           |   7 +-
+ include/linux/mfd/arizona/pdata.h                  |   3 +-
+ include/linux/mfd/axp20x.h                         |  10 +-
+ include/linux/mfd/cros_ec.h                        |  20 +-
+ include/linux/mfd/stm32-timers.h                   |  58 +++
+ include/linux/mfd/syscon/exynos4-pmu.h             |  21 -
+ include/linux/mfd/syscon/exynos5-pmu.h             |  19 -
+ include/linux/mfd/tps65218.h                       |   4 +-
+ include/linux/mfd/tps68470.h                       |  17 +-
+ 55 files changed, 1504 insertions(+), 931 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/input/mtk-pmic-keys.txt
+ create mode 100644 drivers/input/keyboard/mtk-pmic-keys.c
+ delete mode 100644 drivers/mfd/cros_ec_acpi_gpe.c
+ delete mode 100644 include/linux/mfd/syscon/exynos4-pmu.h
+ delete mode 100644 include/linux/mfd/syscon/exynos5-pmu.h
+Merging backlight/for-backlight-next (cd4270258cbe backlight: as3711_bl: fix device-tree node leaks)
+$ git merge backlight/for-backlight-next
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ .../bindings/leds/backlight/pwm-backlight.txt      |   6 +
+ .../leds/backlight/zii,rave-sp-backlight.txt       |  23 ++
+ MAINTAINERS                                        |   1 +
+ drivers/staging/fbtft/fbtft-core.c                 |   4 +-
+ drivers/staging/fbtft/fbtft.h                      |   1 +
+ drivers/video/backlight/Kconfig                    |  13 ++
+ drivers/video/backlight/Makefile                   |   2 +
+ drivers/video/backlight/generic_bl.c               |   5 -
+ drivers/video/backlight/otm3225a.c                 | 252 +++++++++++++++++++++
+ drivers/video/backlight/pandora_bl.c               |  25 +-
+ drivers/video/backlight/pwm_bl.c                   |  34 ++-
+ drivers/video/backlight/rave-sp-backlight.c        |  82 +++++++
+ include/linux/backlight.h                          |   4 -
+ include/linux/mfd/rave-sp.h                        |   1 +
+ include/linux/pwm_backlight.h                      |   2 +
+ 15 files changed, 430 insertions(+), 25 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/leds/backlight/zii,rave-sp-backlight.txt
+ create mode 100644 drivers/video/backlight/otm3225a.c
+ create mode 100644 drivers/video/backlight/rave-sp-backlight.c
+Merging battery/for-next (1f140ff46708 Merge tag 'tags/tcpm-pps-4.18' into psy-next)
+$ git merge battery/for-next
+Auto-merging include/linux/usb/audio-v2.h
+Auto-merging drivers/usb/typec/tcpm.c
+Auto-merging drivers/usb/core/hub.c
+Auto-merging drivers/usb/core/hcd.c
+CONFLICT (content): Merge conflict in drivers/usb/core/hcd.c
+Auto-merging MAINTAINERS
+Resolved 'drivers/usb/core/hcd.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master 7b8a761ee7da] Merge remote-tracking branch 'battery/for-next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/ABI/testing/sysfs-bus-usb            |   18 +
+ Documentation/ABI/testing/sysfs-class-power        |  455 +++++++++
+ .../devicetree/bindings/power/supply/bq27xxx.txt   |    1 +
+ .../devicetree/bindings/usb/ci-hdrc-usb2.txt       |    6 +
+ .../devicetree/bindings/usb/fcs,fusb302.txt        |    6 -
+ .../devicetree/bindings/usb/richtek,rt1711h.txt    |   17 +
+ MAINTAINERS                                        |    1 +
+ drivers/power/reset/gpio-poweroff.c                |    4 +-
+ drivers/power/supply/ab8500_bmdata.c               |   63 --
+ drivers/power/supply/ab8500_btemp.c                |   93 +-
+ drivers/power/supply/ab8500_charger.c              |  131 +--
+ drivers/power/supply/ab8500_fg.c                   |   14 +-
+ drivers/power/supply/abx500_chargalg.c             |   62 --
+ drivers/power/supply/axp288_charger.c              |   26 +-
+ drivers/power/supply/axp288_fuel_gauge.c           |   27 +-
+ drivers/power/supply/bq27xxx_battery.c             |    9 +
+ drivers/power/supply/bq27xxx_battery_i2c.c         |    2 +
+ drivers/power/supply/charger-manager.c             |    5 +-
+ drivers/power/supply/gpio-charger.c                |    3 +-
+ drivers/power/supply/power_supply_core.c           |   11 +-
+ drivers/power/supply/power_supply_sysfs.c          |  123 ++-
+ drivers/power/supply/s3c_adc_battery.c             |    8 +-
+ drivers/staging/typec/Kconfig                      |    8 +
+ drivers/staging/typec/Makefile                     |    1 +
+ drivers/staging/typec/tcpci.h                      |    1 +
+ drivers/staging/typec/tcpci_rt1711h.c              |  312 ++++++
+ drivers/usb/core/hcd.c                             |   49 +-
+ drivers/usb/core/hub.c                             |   23 +-
+ drivers/usb/core/sysfs.c                           |   22 +
+ drivers/usb/gadget/udc/Kconfig                     |    4 +-
+ drivers/usb/host/Kconfig                           |    4 +-
+ drivers/usb/host/ehci-omap.c                       |    5 +-
+ drivers/usb/host/ehci-tegra.c                      |   87 +-
+ drivers/usb/isp1760/isp1760-core.c                 |    2 +-
+ drivers/usb/isp1760/isp1760-hcd.c                  |    2 +-
+ drivers/usb/mon/mon_bin.c                          |    2 +-
+ drivers/usb/mtu3/Kconfig                           |    2 +-
+ drivers/usb/mtu3/mtu3_plat.c                       |    6 +-
+ drivers/usb/musb/omap2430.c                        |   95 +-
+ drivers/usb/phy/Kconfig                            |    9 +
+ drivers/usb/phy/Makefile                           |    2 +-
+ drivers/usb/phy/phy-am335x.c                       |    6 +-
+ drivers/usb/phy/phy-tegra-usb.c                    |  140 ++-
+ drivers/usb/phy/phy.c                              |  133 +--
+ drivers/usb/renesas_usbhs/common.h                 |    1 -
+ drivers/usb/renesas_usbhs/rcar2.c                  |   29 -
+ drivers/usb/roles/intel-xhci-usb-role-switch.c     |    2 +
+ drivers/usb/storage/freecom.c                      |    4 +-
+ drivers/usb/typec/Kconfig                          |    1 +
+ drivers/usb/typec/fusb302/Kconfig                  |    2 +-
+ drivers/usb/typec/fusb302/fusb302.c                |  105 +-
+ drivers/usb/typec/tcpm.c                           | 1059 ++++++++++++++++++--
+ drivers/usb/typec/typec_wcove.c                    |    4 +-
+ include/linux/mfd/abx500.h                         |    1 -
+ include/linux/mfd/abx500/ab8500-bm.h               |    2 -
+ include/linux/mfd/abx500/ux500_chargalg.h          |    4 -
+ include/linux/power/bq27xxx_battery.h              |    3 +-
+ include/linux/power_supply.h                       |   16 +
+ include/linux/usb.h                                |    4 +
+ include/linux/usb/audio-v2.h                       |    2 +-
+ include/linux/usb/hcd.h                            |    2 +-
+ include/linux/usb/pd.h                             |    4 +-
+ include/linux/usb/phy.h                            |   36 -
+ include/linux/usb/tcpm.h                           |   10 +-
+ include/linux/usb/tegra_usb_phy.h                  |    2 +
+ include/uapi/linux/usb/ch11.h                      |    5 +
+ 66 files changed, 2345 insertions(+), 953 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/usb/richtek,rt1711h.txt
+ create mode 100644 drivers/staging/typec/tcpci_rt1711h.c
+Merging regulator/for-next (16bf416e6d33 Merge branch 'regulator-4.18' into regulator-next)
+$ git merge regulator/for-next
+Auto-merging arch/arm/mach-omap2/pdata-quirks.c
+Auto-merging arch/arm/mach-omap1/board-ams-delta.c
+CONFLICT (content): Merge conflict in arch/arm/mach-omap1/board-ams-delta.c
+Auto-merging MAINTAINERS
+Resolved 'arch/arm/mach-omap1/board-ams-delta.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master ce656036ee92] Merge remote-tracking branch 'regulator/for-next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/devicetree/bindings/mfd/axp20x.txt   |    3 +-
+ .../devicetree/bindings/mfd/bd9571mwv.txt          |   21 +
+ .../devicetree/bindings/regulator/pfuze100.txt     |    6 +-
+ .../bindings/regulator/qcom,spmi-regulator.txt     |   45 +
+ .../devicetree/bindings/regulator/regulator.txt    |   10 +
+ .../bindings/regulator/rohm,bd71837-regulator.txt  |  126 ++
+ .../bindings/regulator/sy8106a-regulator.txt       |   23 +
+ MAINTAINERS                                        |    6 +
+ arch/arm/mach-imx/mach-mx21ads.c                   |   13 +-
+ arch/arm/mach-imx/mach-mx27ads.c                   |   12 +-
+ arch/arm/mach-mmp/brownstone.c                     |   12 +-
+ arch/arm/mach-omap1/board-ams-delta.c              |   14 +-
+ arch/arm/mach-omap2/pdata-quirks.c                 |   16 +-
+ arch/arm/mach-pxa/em-x270.c                        |    1 -
+ arch/arm/mach-pxa/ezx.c                            |   33 +-
+ arch/arm/mach-pxa/hx4700.c                         |   12 +-
+ arch/arm/mach-pxa/magician.c                       |   13 +-
+ arch/arm/mach-pxa/raumfeld.c                       |   12 +-
+ arch/arm/mach-pxa/zeus.c                           |   23 +-
+ arch/arm/mach-s3c64xx/mach-crag6410-module.c       |   28 +-
+ arch/arm/mach-s3c64xx/mach-crag6410.c              |    1 -
+ arch/arm/mach-s3c64xx/mach-smdk6410.c              |    1 -
+ arch/arm/mach-sa1100/assabet.c                     |   21 +-
+ arch/arm/mach-sa1100/generic.c                     |    5 +-
+ arch/arm/mach-sa1100/generic.h                     |    3 +-
+ arch/arm/mach-sa1100/shannon.c                     |    4 +-
+ arch/sh/boards/mach-ecovec24/setup.c               |   22 +-
+ .../intel-mid/device_libs/platform_bcm43xx.c       |   17 +-
+ drivers/mfd/bd9571mwv.c                            |    2 +
+ drivers/regulator/Kconfig                          |   18 +
+ drivers/regulator/Makefile                         |    3 +-
+ drivers/regulator/ab8500.c                         | 1779 ++------------------
+ drivers/regulator/anatop-regulator.c               |   22 +-
+ drivers/regulator/arizona-ldo1.c                   |   19 +-
+ drivers/regulator/axp20x-regulator.c               |    2 +
+ drivers/regulator/bd71837-regulator.c              |  640 +++++++
+ drivers/regulator/bd9571mwv-regulator.c            |  127 +-
+ drivers/regulator/core.c                           |  244 ++-
+ drivers/regulator/cpcap-regulator.c                |    2 +-
+ drivers/regulator/fixed-helper.c                   |    1 -
+ drivers/regulator/fixed.c                          |   33 +-
+ drivers/regulator/gpio-regulator.c                 |   23 +-
+ drivers/regulator/internal.h                       |   37 +-
+ drivers/regulator/lm363x-regulator.c               |   20 +-
+ drivers/regulator/lp87565-regulator.c              |   17 +-
+ drivers/regulator/lp8788-ldo.c                     |   32 +-
+ drivers/regulator/max8952.c                        |   18 +-
+ drivers/regulator/max8973-regulator.c              |   54 +-
+ drivers/regulator/mc13783-regulator.c              |   18 +-
+ drivers/regulator/mc13892-regulator.c              |   18 +-
+ drivers/regulator/mc13xxx-regulator-core.c         |   28 +-
+ drivers/regulator/of_regulator.c                   |  193 ++-
+ drivers/regulator/pfuze100-regulator.c             |   40 +-
+ drivers/regulator/qcom_spmi-regulator.c            |  133 +-
+ drivers/regulator/s5m8767.c                        |   26 +-
+ drivers/regulator/sy8106a-regulator.c              |  167 ++
+ drivers/regulator/tps65090-regulator.c             |   50 +-
+ drivers/regulator/tps6586x-regulator.c             |    1 +
+ drivers/regulator/twl-regulator.c                  |    2 +-
+ drivers/regulator/wm8350-regulator.c               |    1 +
+ include/linux/mfd/bd9571mwv.h                      |    5 +
+ include/linux/mfd/lp8788.h                         |   16 -
+ include/linux/mfd/samsung/core.h                   |    4 +-
+ include/linux/mfd/tps65090.h                       |    8 +-
+ include/linux/mfd/tps6586x.h                       |    1 +
+ include/linux/regulator/ab8500.h                   |  157 +-
+ include/linux/regulator/arizona-ldo1.h             |    3 -
+ include/linux/regulator/consumer.h                 |    1 +
+ include/linux/regulator/driver.h                   |   27 +-
+ include/linux/regulator/fixed.h                    |    3 -
+ include/linux/regulator/gpio-regulator.h           |    3 -
+ include/linux/regulator/machine.h                  |    4 +
+ include/linux/regulator/max8952.h                  |    1 -
+ 73 files changed, 2393 insertions(+), 2113 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.txt
+ create mode 100644 Documentation/devicetree/bindings/regulator/sy8106a-regulator.txt
+ create mode 100644 drivers/regulator/bd71837-regulator.c
+ create mode 100644 drivers/regulator/sy8106a-regulator.c
+Merging security/next-testing (d19289133b1d Merge branch 'next-integrity' of git://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity into next-testing)
+$ git merge security/next-testing
+Auto-merging security/selinux/hooks.c
+Auto-merging net/socket.c
+Auto-merging drivers/char/tpm/eventlog/tpm2.c
+Auto-merging drivers/char/tpm/eventlog/tpm1.c
+Auto-merging drivers/char/tpm/eventlog/of.c
+Auto-merging drivers/char/tpm/eventlog/efi.c
+Auto-merging drivers/char/tpm/eventlog/acpi.c
+Merge made by the 'recursive' strategy.
+ Documentation/ABI/testing/evm                      |  13 ++
+ Documentation/ABI/testing/ima_policy               |   2 +-
+ drivers/char/tpm/Makefile                          |  10 +-
+ .../tpm/{tpm_eventlog_acpi.c => eventlog/acpi.c}   |   3 +-
+ drivers/char/tpm/eventlog/common.c                 | 195 ++++++++++++++++++++
+ drivers/char/tpm/eventlog/common.h                 |  35 ++++
+ .../tpm/{tpm_eventlog_efi.c => eventlog/efi.c}     |   6 +-
+ .../char/tpm/{tpm_eventlog_of.c => eventlog/of.c}  |  11 +-
+ .../char/tpm/{tpm1_eventlog.c => eventlog/tpm1.c}  | 200 ++-------------------
+ .../char/tpm/{tpm2_eventlog.c => eventlog/tpm2.c}  |   3 +-
+ drivers/char/tpm/st33zp24/spi.c                    |   4 +-
+ drivers/char/tpm/st33zp24/st33zp24.c               |   2 -
+ drivers/char/tpm/tpm-interface.c                   |   5 +-
+ drivers/char/tpm/tpm.h                             |  32 +---
+ drivers/char/tpm/tpm2-space.c                      |   3 +-
+ drivers/char/tpm/tpm_crb.c                         |  10 +-
+ drivers/char/tpm/tpm_tis_core.c                    |  58 +++++-
+ include/linux/lsm_hooks.h                          |   7 +
+ include/linux/security.h                           |  21 +--
+ include/uapi/linux/audit.h                         |   1 +
+ net/socket.c                                       |   7 +
+ security/integrity/evm/Kconfig                     |  11 ++
+ security/integrity/evm/evm.h                       |   7 +-
+ security/integrity/evm/evm_crypto.c                |  10 +-
+ security/integrity/evm/evm_main.c                  |  79 ++++----
+ security/integrity/evm/evm_secfs.c                 | 200 ++++++++++++++++++++-
+ security/integrity/iint.c                          |  18 ++
+ security/integrity/ima/ima.h                       |   1 -
+ security/integrity/ima/ima_fs.c                    |  18 +-
+ security/integrity/ima/ima_kexec.c                 |   2 +
+ security/integrity/ima/ima_main.c                  |   8 +-
+ security/integrity/ima/ima_policy.c                |  53 ++++--
+ security/integrity/ima/ima_template_lib.c          |   2 +
+ security/integrity/integrity.h                     |   2 +
+ security/keys/big_key.c                            |  11 +-
+ security/keys/dh.c                                 |  35 ++--
+ security/security.c                                |   6 +
+ security/selinux/hooks.c                           |  13 ++
+ security/smack/smack_lsm.c                         |  22 +++
+ 39 files changed, 781 insertions(+), 345 deletions(-)
+ rename drivers/char/tpm/{tpm_eventlog_acpi.c => eventlog/acpi.c} (98%)
+ create mode 100644 drivers/char/tpm/eventlog/common.c
+ create mode 100644 drivers/char/tpm/eventlog/common.h
+ rename drivers/char/tpm/{tpm_eventlog_efi.c => eventlog/efi.c} (91%)
+ rename drivers/char/tpm/{tpm_eventlog_of.c => eventlog/of.c} (91%)
+ rename drivers/char/tpm/{tpm1_eventlog.c => eventlog/tpm1.c} (58%)
+ rename drivers/char/tpm/{tpm2_eventlog.c => eventlog/tpm2.c} (99%)
+Merging apparmor/apparmor-next (430837708c21 apparmor: Fix memory leak of rule on error exit path)
+$ git merge apparmor/apparmor-next
+Merge made by the 'recursive' strategy.
+ Documentation/admin-guide/LSM/apparmor.rst |   6 +-
+ security/apparmor/audit.c                  |  90 +++++++++++-
+ security/apparmor/domain.c                 |   2 +-
+ security/apparmor/include/audit.h          |   6 +
+ security/apparmor/include/label.h          |   2 +-
+ security/apparmor/include/path.h           |  32 ++---
+ security/apparmor/include/secid.h          |  15 +-
+ security/apparmor/label.c                  |  14 +-
+ security/apparmor/lib.c                    |   2 +-
+ security/apparmor/lsm.c                    |  44 ++++--
+ security/apparmor/match.c                  |   2 +-
+ security/apparmor/mount.c                  |   2 +-
+ security/apparmor/policy.c                 |   4 +-
+ security/apparmor/secid.c                  | 221 ++++++++++++++++++++++++++---
+ 14 files changed, 373 insertions(+), 69 deletions(-)
+Merging integrity/next-integrity (6f0911a666d1 ima: fix updating the ima_appraise flag)
+$ git merge integrity/next-integrity
+Already up to date.
+Merging keys/keys-next (1e684d3820d8 pkcs7: Set the module licence to prevent tainting)
+$ git merge keys/keys-next
+Already up to date.
+Merging selinux/next (d141136f523a audit: normalize MAC_POLICY_LOAD record)
+$ git merge selinux/next
+Merge made by the 'recursive' strategy.
+ security/selinux/selinuxfs.c | 18 ++++++++++++------
+ 1 file changed, 12 insertions(+), 6 deletions(-)
+Merging tpmdd/next (3ab2011ea368 tpm: fix race condition in tpm_common_write())
+$ git merge tpmdd/next
+Merge made by the 'recursive' strategy.
+ drivers/char/tpm/tpm-dev-common.c | 40 ++++++++++++++++++---------------------
+ drivers/char/tpm/tpm-dev.h        |  2 +-
+ 2 files changed, 19 insertions(+), 23 deletions(-)
+Merging watchdog/master (1c85ffc86a2b dt-bindings: watchdog: renesas-wdt: Add R-Car Gen2 support)
+$ git merge watchdog/master
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/watchdog/renesas-wdt.txt   | 23 +++++++++++++++-------
+ drivers/watchdog/cadence_wdt.c                     |  6 ++----
+ drivers/watchdog/hpwdt.c                           |  2 +-
+ drivers/watchdog/mena21_wdt.c                      | 18 -----------------
+ drivers/watchdog/of_xilinx_wdt.c                   |  6 ++----
+ drivers/watchdog/renesas_wdt.c                     |  2 +-
+ drivers/watchdog/sp805_wdt.c                       | 14 +++++++++++++
+ drivers/watchdog/wdat_wdt.c                        |  6 ++----
+ 8 files changed, 38 insertions(+), 39 deletions(-)
+Merging iommu/next (1f568357117c Merge branches 'arm/io-pgtable', 'arm/qcom', 'arm/tegra', 'x86/vt-d', 'x86/amd' and 'core' into next)
+$ git merge iommu/next
+Auto-merging drivers/iommu/intel-iommu.c
+Auto-merging drivers/iommu/amd_iommu.c
+Auto-merging drivers/iommu/Kconfig
+Merge made by the 'recursive' strategy.
+ drivers/iommu/Kconfig              |   5 +-
+ drivers/iommu/amd_iommu.c          |  69 ++++++++++++-------------
+ drivers/iommu/amd_iommu_types.h    |   1 +
+ drivers/iommu/dmar.c               |   8 ++-
+ drivers/iommu/intel-iommu.c        | 101 ++++++++++++++++++-------------------
+ drivers/iommu/intel-svm.c          |   2 +-
+ drivers/iommu/io-pgtable-arm-v7s.c |   5 +-
+ drivers/iommu/io-pgtable-arm.c     |  18 ++++---
+ drivers/iommu/iommu.c              |   7 +--
+ drivers/iommu/qcom_iommu.c         |   6 +--
+ drivers/iommu/tegra-gart.c         |  15 +++++-
+ include/linux/intel-iommu.h        |   1 -
+ 12 files changed, 122 insertions(+), 116 deletions(-)
+Merging dwmw2-iommu/master (d8a5b80568a9 Linux 4.15)
+$ git merge dwmw2-iommu/master
+Already up to date.
+Merging vfio/next (a823b3a1b88b vfio: platform: Fix using devices in PM Domains)
+$ git merge vfio/next
+Merge made by the 'recursive' strategy.
+ Documentation/vfio-mediated-device.txt       |    5 +
+ drivers/vfio/mdev/mdev_core.c                |  102 +-
+ drivers/vfio/mdev/mdev_private.h             |    2 +-
+ drivers/vfio/mdev/mdev_sysfs.c               |   14 +-
+ drivers/vfio/platform/vfio_platform_common.c |   30 +-
+ drivers/vfio/vfio.c                          |   11 +-
+ drivers/vfio/vfio_iommu_type1.c              |  570 ++++++++++-
+ include/uapi/linux/vfio.h                    |   23 +
+ samples/Kconfig                              |   30 +
+ samples/vfio-mdev/Makefile                   |    3 +
+ samples/vfio-mdev/mbochs.c                   | 1406 ++++++++++++++++++++++++++
+ samples/vfio-mdev/mdpy-defs.h                |   22 +
+ samples/vfio-mdev/mdpy-fb.c                  |  232 +++++
+ samples/vfio-mdev/mdpy.c                     |  807 +++++++++++++++
+ 14 files changed, 3129 insertions(+), 128 deletions(-)
+ create mode 100644 samples/vfio-mdev/mbochs.c
+ create mode 100644 samples/vfio-mdev/mdpy-defs.h
+ create mode 100644 samples/vfio-mdev/mdpy-fb.c
+ create mode 100644 samples/vfio-mdev/mdpy.c
+Merging trivial/for-next (75a24b822d38 kfifo: fix inaccurate comment)
+$ git merge trivial/for-next
+Already up to date.
+Merging audit/next (5b71388663c0 audit: Fix wrong task in comparison of session ID)
+$ git merge audit/next
+Auto-merging security/selinux/selinuxfs.c
+CONFLICT (content): Merge conflict in security/selinux/selinuxfs.c
+Auto-merging security/selinux/hooks.c
+Auto-merging net/netfilter/x_tables.c
+Auto-merging net/core/dev.c
+Auto-merging net/bridge/netfilter/ebtables.c
+Auto-merging kernel/seccomp.c
+Auto-merging kernel/fork.c
+Auto-merging kernel/auditsc.c
+Auto-merging kernel/audit_watch.c
+Auto-merging include/uapi/linux/audit.h
+Auto-merging include/net/xfrm.h
+Resolved 'security/selinux/selinuxfs.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master 88547ce5a841] Merge remote-tracking branch 'audit/next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/userspace-api/seccomp_filter.rst |   7 --
+ include/linux/audit.h                          |  39 ++++---
+ include/net/xfrm.h                             |   4 +-
+ include/uapi/linux/audit.h                     |   1 +
+ init/init_task.c                               |   3 +-
+ kernel/audit.c                                 |   6 +-
+ kernel/audit_watch.c                           |   2 +-
+ kernel/auditfilter.c                           |   6 +-
+ kernel/auditsc.c                               | 135 ++++++++++++++++---------
+ kernel/fork.c                                  |   2 +-
+ kernel/seccomp.c                               | 126 ++++++++++++++++-------
+ net/bridge/netfilter/ebtables.c                |   2 +-
+ net/core/dev.c                                 |  18 ++--
+ net/netfilter/x_tables.c                       |   2 +-
+ net/netlabel/netlabel_user.c                   |   2 +-
+ security/integrity/ima/ima_api.c               |   2 +-
+ security/integrity/integrity_audit.c           |   2 +-
+ security/lsm_audit.c                           |   2 +-
+ security/selinux/hooks.c                       |   7 +-
+ security/selinux/selinuxfs.c                   |   6 +-
+ security/selinux/ss/services.c                 |  12 +--
+ 21 files changed, 242 insertions(+), 144 deletions(-)
+Merging devicetree/for-next (399d9e069577 dt-bindings: Add vendor prefix for Logic PD)
+$ git merge devicetree/for-next
+Auto-merging scripts/Makefile.lib
+Auto-merging drivers/of/platform.c
+Auto-merging MAINTAINERS
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.txt
+Auto-merging Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.txt
+Auto-merging Documentation/devicetree/bindings/mtd/mtk-nand.txt
+Auto-merging Documentation/devicetree/bindings/mmc/tmio_mmc.txt
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/arm/ux500/boards.txt       |   2 +-
+ Documentation/devicetree/bindings/dma/k3dma.txt    |   1 -
+ Documentation/devicetree/bindings/dma/ti-edma.txt  |   1 -
+ .../adc/samsung,exynos-adc.txt}                    |   0
+ .../devicetree/bindings/mips/lantiq/rcu.txt        |   2 -
+ Documentation/devicetree/bindings/mmc/tmio_mmc.txt |   4 -
+ .../{powerpc/4xx/ndfc.txt => mtd/ibm,ndfc.txt}     |   0
+ Documentation/devicetree/bindings/mtd/mtk-nand.txt |   4 -
+ .../{powerpc/4xx/emac.txt => net/ibm,emac.txt}     |   0
+ .../devicetree/bindings/pci/xgene-pci.txt          |   7 -
+ Documentation/devicetree/bindings/pps/pps-gpio.txt |   1 -
+ .../devicetree/bindings/pwm/pwm-omap-dmtimer.txt   |   2 +-
+ .../samsung,exynos4-rng.txt}                       |   0
+ .../bindings/{ => rng}/sparc_sun_oracle_rng.txt    |   0
+ .../bindings/thermal/rcar-gen3-thermal.txt         |   6 +-
+ .../{nios2/timer.txt => timer/altr,timer-1.0.txt}  |   0
+ .../arch_timer.txt => timer/arm,arch_timer.txt}    |   0
+ .../arm,armv7m-systick.txt}                        |   0
+ .../arm,global_timer.txt}                          |   0
+ .../bindings/{arm/twd.txt => timer/arm,twd.txt}    |   0
+ .../{powerpc/fsl/gtm.txt => timer/fsl,gtm.txt}     |   0
+ .../mrvl/timer.txt => timer/mrvl,mmp-timer.txt}    |   0
+ .../msm/timer.txt => timer/qcom,msm-timer.txt}     |   0
+ .../spear-timer.txt => timer/st,spear-timer.txt}   |   0
+ .../{c6x/timer64.txt => timer/ti,c64x+timer64.txt} |   0
+ .../{arm/omap/timer.txt => timer/ti,timer.txt}     |   0
+ .../{arm/vt8500 => timer}/via,vt8500-timer.txt     |   0
+ .../devicetree/bindings/vendor-prefixes.txt        |   5 +
+ MAINTAINERS                                        |   2 +
+ drivers/gpu/drm/rcar-du/Makefile                   |   7 +
+ drivers/of/of_numa.c                               |   1 -
+ drivers/of/platform.c                              |  11 ++
+ drivers/of/resolver.c                              |   5 +
+ drivers/of/unittest.c                              |   8 +-
+ scripts/Makefile.lib                               |   3 +
+ scripts/dtc/checks.c                               | 199 ++++++++++++++++++++-
+ scripts/dtc/dtc-lexer.l                            |   7 +
+ scripts/dtc/dtc-parser.y                           |  39 ++--
+ scripts/dtc/dtc.h                                  |   4 +
+ scripts/dtc/livetree.c                             |  26 ++-
+ scripts/dtc/version_gen.h                          |   2 +-
+ 41 files changed, 303 insertions(+), 46 deletions(-)
+ rename Documentation/devicetree/bindings/{arm/samsung/exynos-adc.txt => iio/adc/samsung,exynos-adc.txt} (100%)
+ rename Documentation/devicetree/bindings/{powerpc/4xx/ndfc.txt => mtd/ibm,ndfc.txt} (100%)
+ rename Documentation/devicetree/bindings/{powerpc/4xx/emac.txt => net/ibm,emac.txt} (100%)
+ rename Documentation/devicetree/bindings/{crypto/samsung,exynos-rng4.txt => rng/samsung,exynos4-rng.txt} (100%)
+ rename Documentation/devicetree/bindings/{ => rng}/sparc_sun_oracle_rng.txt (100%)
+ rename Documentation/devicetree/bindings/{nios2/timer.txt => timer/altr,timer-1.0.txt} (100%)
+ rename Documentation/devicetree/bindings/{arm/arch_timer.txt => timer/arm,arch_timer.txt} (100%)
+ rename Documentation/devicetree/bindings/{arm/armv7m_systick.txt => timer/arm,armv7m-systick.txt} (100%)
+ rename Documentation/devicetree/bindings/{arm/global_timer.txt => timer/arm,global_timer.txt} (100%)
+ rename Documentation/devicetree/bindings/{arm/twd.txt => timer/arm,twd.txt} (100%)
+ rename Documentation/devicetree/bindings/{powerpc/fsl/gtm.txt => timer/fsl,gtm.txt} (100%)
+ rename Documentation/devicetree/bindings/{arm/mrvl/timer.txt => timer/mrvl,mmp-timer.txt} (100%)
+ rename Documentation/devicetree/bindings/{arm/msm/timer.txt => timer/qcom,msm-timer.txt} (100%)
+ rename Documentation/devicetree/bindings/{arm/spear-timer.txt => timer/st,spear-timer.txt} (100%)
+ rename Documentation/devicetree/bindings/{c6x/timer64.txt => timer/ti,c64x+timer64.txt} (100%)
+ rename Documentation/devicetree/bindings/{arm/omap/timer.txt => timer/ti,timer.txt} (100%)
+ rename Documentation/devicetree/bindings/{arm/vt8500 => timer}/via,vt8500-timer.txt (100%)
+Merging mailbox/mailbox-for-next (dfbc9c5841fc mailbox: Remove depends on HAS_DMA in case of platform dependency)
+$ git merge mailbox/mailbox-for-next
+Merge made by the 'recursive' strategy.
+ drivers/mailbox/Kconfig | 2 --
+ 1 file changed, 2 deletions(-)
+Merging spi/for-next (953083ffb6f2 Merge branch 'spi-4.18' into spi-next)
+$ git merge spi/for-next
+Auto-merging drivers/spi/spi.c
+Removing drivers/spi/spi-bcm53xx.h
+Removing drivers/spi/spi-bcm53xx.c
+Merge made by the 'recursive' strategy.
+ drivers/mtd/devices/Kconfig     |   1 +
+ drivers/mtd/devices/m25p80.c    | 238 ++++++++---------------
+ drivers/spi/Kconfig             |  27 +--
+ drivers/spi/Makefile            |   2 +-
+ drivers/spi/internals.h         |  43 +++++
+ drivers/spi/spi-bcm-qspi.c      | 162 ++++++++--------
+ drivers/spi/spi-bcm53xx.c       | 360 -----------------------------------
+ drivers/spi/spi-bcm53xx.h       |  73 -------
+ drivers/spi/spi-bcm63xx-hsspi.c |  25 ++-
+ drivers/spi/spi-cadence.c       |   6 +-
+ drivers/spi/spi-fsl-lpspi.c     |  21 +-
+ drivers/spi/spi-imx.c           |  22 +--
+ drivers/spi/spi-mem.c           | 410 ++++++++++++++++++++++++++++++++++++++++
+ drivers/spi/spi-meson-spicc.c   |  11 +-
+ drivers/spi/spi-mpc52xx.c       |   2 +-
+ drivers/spi/spi-mxs.c           |  48 ++---
+ drivers/spi/spi-omap2-mcspi.c   | 111 +++++------
+ drivers/spi/spi-pxa2xx-dma.c    |  28 +--
+ drivers/spi/spi-pxa2xx.c        | 257 +++++++++----------------
+ drivers/spi/spi-pxa2xx.h        |  17 +-
+ drivers/spi/spi-s3c64xx.c       | 160 +++++++++-------
+ drivers/spi/spi-sh-msiof.c      |  67 +++----
+ drivers/spi/spi-stm32.c         |   2 +-
+ drivers/spi/spi-ti-qspi.c       |  87 +++++----
+ drivers/spi/spi-zynqmp-gqspi.c  |  92 +++++++--
+ drivers/spi/spi.c               | 145 +++++++-------
+ include/linux/spi/spi-mem.h     | 249 ++++++++++++++++++++++++
+ include/linux/spi/spi.h         |  60 +-----
+ 28 files changed, 1419 insertions(+), 1307 deletions(-)
+ create mode 100644 drivers/spi/internals.h
+ delete mode 100644 drivers/spi/spi-bcm53xx.c
+ delete mode 100644 drivers/spi/spi-bcm53xx.h
+ create mode 100644 drivers/spi/spi-mem.c
+ create mode 100644 include/linux/spi/spi-mem.h
+Merging tip/auto-latest (760c3aa70029 Merge branch 'x86/vdso')
+$ git merge tip/auto-latest
+Removing tools/testing/selftests/rcutorture/bin/parse-torture.sh
+Auto-merging lib/iov_iter.c
+Auto-merging kernel/time/timer_list.c
+Auto-merging kernel/time/tick-common.c
+Auto-merging kernel/sched/sched.h
+Auto-merging ipc/shm.c
+Auto-merging include/linux/syscalls.h
+Auto-merging include/linux/sched.h
+Auto-merging include/linux/compat.h
+Auto-merging drivers/s390/net/qeth_core_main.c
+Auto-merging drivers/nvme/host/core.c
+Removing drivers/irqchip/irq-mvebu-gicp.h
+Auto-merging arch/x86/net/bpf_jit_comp.c
+CONFLICT (content): Merge conflict in arch/x86/net/bpf_jit_comp.c
+Auto-merging arch/x86/lib/usercopy_64.c
+Auto-merging arch/x86/kernel/smpboot.c
+Auto-merging arch/x86/kernel/cpu/mtrr/mtrr.c
+Auto-merging arch/x86/kernel/cpu/intel.c
+Auto-merging arch/x86/kernel/cpu/cpu.h
+Auto-merging arch/x86/kernel/cpu/common.c
+Auto-merging arch/x86/kernel/cpu/cacheinfo.c
+Auto-merging arch/x86/kernel/cpu/amd.c
+Auto-merging arch/x86/include/asm/string_64.h
+Auto-merging arch/x86/Kconfig
+Removing arch/s390/include/uapi/asm/shmbuf.h
+Removing arch/s390/include/uapi/asm/sembuf.h
+Removing arch/s390/include/uapi/asm/msgbuf.h
+Auto-merging arch/powerpc/kernel/asm-offsets.c
+Auto-merging arch/nds32/include/asm/Kbuild
+Removing arch/ia64/include/uapi/asm/shmbuf.h
+Removing arch/ia64/include/uapi/asm/sembuf.h
+Removing arch/ia64/include/uapi/asm/msgbuf.h
+Removing arch/ia64/include/uapi/asm/ipcbuf.h
+Auto-merging arch/c6x/include/asm/Kbuild
+Auto-merging arch/arc/include/asm/Kbuild
+Removing arch/alpha/include/uapi/asm/shmbuf.h
+Removing arch/alpha/include/uapi/asm/sembuf.h
+Removing arch/alpha/include/uapi/asm/msgbuf.h
+Removing arch/alpha/include/uapi/asm/ipcbuf.h
+Auto-merging arch/Kconfig
+Auto-merging MAINTAINERS
+Auto-merging Documentation/translations/ko_KR/memory-barriers.txt
+Auto-merging Documentation/memory-barriers.txt
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Resolved 'arch/x86/net/bpf_jit_comp.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master d9df11c86e90] Merge remote-tracking branch 'tip/auto-latest'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/RCU/whatisRCU.txt                    |   2 +
+ Documentation/admin-guide/kernel-parameters.txt    |   5 -
+ Documentation/core-api/atomic_ops.rst              |  13 +-
+ .../bindings/interrupt-controller/arm,gic-v3.txt   |  17 +
+ Documentation/memory-barriers.txt                  |  17 +-
+ Documentation/scheduler/sched-deadline.txt         |  25 +-
+ .../translations/ko_KR/memory-barriers.txt         |  50 ++-
+ MAINTAINERS                                        |   3 +-
+ arch/Kconfig                                       |  15 +
+ arch/alpha/include/asm/Kbuild                      |   1 +
+ arch/alpha/include/uapi/asm/Kbuild                 |   4 +
+ arch/alpha/include/uapi/asm/ipcbuf.h               |   2 -
+ arch/alpha/include/uapi/asm/msgbuf.h               |  28 --
+ arch/alpha/include/uapi/asm/sembuf.h               |  23 --
+ arch/alpha/include/uapi/asm/shmbuf.h               |  39 ---
+ arch/arc/include/asm/Kbuild                        |   1 +
+ arch/arm/include/asm/Kbuild                        |   1 +
+ arch/arm64/include/asm/compat.h                    |  43 +--
+ arch/arm64/include/asm/spinlock.h                  |   5 -
+ arch/arm64/include/asm/stat.h                      |   1 +
+ arch/arm64/kernel/hw_breakpoint.c                  |   1 -
+ arch/arm64/kernel/perf_regs.c                      |   2 +-
+ arch/c6x/include/asm/Kbuild                        |   1 +
+ arch/h8300/include/asm/Kbuild                      |   1 +
+ arch/hexagon/include/asm/Kbuild                    |   1 +
+ arch/ia64/include/asm/Kbuild                       |   1 +
+ arch/ia64/include/asm/hardirq.h                    |   2 +-
+ arch/ia64/include/uapi/asm/Kbuild                  |   4 +
+ arch/ia64/include/uapi/asm/ipcbuf.h                |   2 -
+ arch/ia64/include/uapi/asm/msgbuf.h                |  28 --
+ arch/ia64/include/uapi/asm/sembuf.h                |  23 --
+ arch/ia64/include/uapi/asm/shmbuf.h                |  39 ---
+ arch/m68k/include/asm/Kbuild                       |   1 +
+ arch/microblaze/include/asm/Kbuild                 |   1 +
+ arch/mips/include/asm/compat.h                     |  51 ++-
+ arch/mips/include/uapi/asm/msgbuf.h                |  57 ++--
+ arch/mips/include/uapi/asm/sembuf.h                |  15 +-
+ arch/mips/include/uapi/asm/shmbuf.h                |  23 +-
+ arch/mips/kernel/signal32.c                        |   2 +-
+ arch/nds32/include/asm/Kbuild                      |   1 +
+ arch/nios2/include/asm/Kbuild                      |   1 +
+ arch/openrisc/include/asm/Kbuild                   |   1 +
+ arch/parisc/include/asm/compat.h                   |  43 +--
+ arch/parisc/include/asm/hardirq.h                  |   8 -
+ arch/parisc/include/uapi/asm/msgbuf.h              |  33 +-
+ arch/parisc/include/uapi/asm/sembuf.h              |  16 +-
+ arch/parisc/include/uapi/asm/shmbuf.h              |  19 +-
+ arch/powerpc/include/asm/compat.h                  |  43 +--
+ arch/powerpc/include/asm/hardirq.h                 |   7 -
+ arch/powerpc/include/uapi/asm/msgbuf.h             |  18 +-
+ arch/powerpc/include/uapi/asm/sembuf.h             |  14 +-
+ arch/powerpc/include/uapi/asm/shmbuf.h             |  19 +-
+ arch/powerpc/kernel/asm-offsets.c                  |   2 +-
+ arch/powerpc/oprofile/backtrace.c                  |   1 +
+ arch/s390/hypfs/hypfs_sprp.c                       |   1 -
+ arch/s390/include/asm/compat.h                     |  43 +--
+ arch/s390/include/asm/elf.h                        |   4 +-
+ arch/s390/include/asm/hardirq.h                    |   2 +
+ arch/s390/include/uapi/asm/Kbuild                  |   3 +
+ arch/s390/include/uapi/asm/msgbuf.h                |  38 ---
+ arch/s390/include/uapi/asm/sembuf.h                |  30 --
+ arch/s390/include/uapi/asm/shmbuf.h                |  49 ---
+ arch/s390/kvm/priv.c                               |   1 -
+ arch/s390/pci/pci_clp.c                            |   1 -
+ arch/sh/include/asm/Kbuild                         |   1 +
+ arch/sh/kernel/irq.c                               |   2 +-
+ arch/sparc/include/asm/compat.h                    |  47 ++-
+ arch/sparc/include/asm/hardirq_64.h                |   5 +-
+ arch/sparc/include/uapi/asm/msgbuf.h               |  22 +-
+ arch/sparc/include/uapi/asm/sembuf.h               |  16 +-
+ arch/sparc/include/uapi/asm/shmbuf.h               |  21 +-
+ arch/um/include/asm/Kbuild                         |   1 +
+ arch/unicore32/include/asm/Kbuild                  |   1 +
+ arch/x86/Kconfig                                   |   1 +
+ arch/x86/boot/compressed/eboot.c                   | 114 ++-----
+ arch/x86/entry/vdso/Makefile                       |  11 +-
+ arch/x86/events/amd/ibs.c                          |   2 +-
+ arch/x86/events/amd/uncore.c                       |  21 +-
+ arch/x86/events/core.c                             |   2 +-
+ arch/x86/include/asm/cacheinfo.h                   |   7 +
+ arch/x86/include/asm/compat.h                      |  43 +--
+ arch/x86/include/asm/ftrace.h                      |   2 +-
+ arch/x86/include/asm/hardirq.h                     |   8 -
+ arch/x86/include/asm/linkage.h                     |   2 +-
+ arch/x86/include/asm/processor.h                   |   9 -
+ arch/x86/include/asm/qspinlock.h                   |  21 +-
+ arch/x86/include/asm/qspinlock_paravirt.h          |   3 +-
+ arch/x86/include/asm/smp.h                         |   1 -
+ arch/x86/include/asm/stacktrace.h                  |   2 +
+ arch/x86/include/asm/string_64.h                   |  10 +-
+ arch/x86/include/asm/uaccess_64.h                  |  14 +
+ arch/x86/include/asm/x86_init.h                    |   1 +
+ arch/x86/include/uapi/asm/sembuf.h                 |  11 +-
+ arch/x86/kernel/cpu/Makefile                       |   2 +-
+ arch/x86/kernel/cpu/amd.c                          |  36 +--
+ .../kernel/cpu/{intel_cacheinfo.c => cacheinfo.c}  |  46 ++-
+ arch/x86/kernel/cpu/centaur.c                      |  53 +++
+ arch/x86/kernel/cpu/common.c                       |  20 ++
+ arch/x86/kernel/cpu/cpu.h                          |  10 +
+ arch/x86/kernel/cpu/intel.c                        |  34 +-
+ arch/x86/kernel/cpu/mcheck/mce-inject.c            |   2 +-
+ arch/x86/kernel/cpu/mcheck/mce.c                   |  18 ++
+ arch/x86/kernel/cpu/mtrr/Makefile                  |   2 +-
+ arch/x86/kernel/cpu/mtrr/if.c                      |  33 +-
+ arch/x86/kernel/cpu/mtrr/{main.c => mtrr.c}        |  33 +-
+ arch/x86/kernel/cpu/topology.c                     |   8 +-
+ arch/x86/kernel/dumpstack.c                        | 144 ++++-----
+ arch/x86/kernel/e820.c                             |  32 +-
+ arch/x86/kernel/early-quirks.c                     |   8 +-
+ arch/x86/kernel/hpet.c                             |   5 +-
+ arch/x86/kernel/i8237.c                            |  25 ++
+ arch/x86/kernel/platform-quirks.c                  |   7 +-
+ arch/x86/kernel/process_32.c                       |   8 +-
+ arch/x86/kernel/smpboot.c                          |   7 -
+ arch/x86/kernel/sys_x86_64.c                       |   2 +-
+ arch/x86/kernel/uprobes.c                          |   4 +-
+ arch/x86/kernel/vmlinux.lds.S                      |   4 +-
+ arch/x86/lib/memcpy_64.S                           | 102 +++---
+ arch/x86/lib/usercopy_64.c                         |  30 +-
+ arch/x86/mm/fault.c                                |   7 +-
+ arch/x86/mm/numa.c                                 |  22 +-
+ arch/x86/um/vdso/Makefile                          |   4 +-
+ arch/x86/xen/efi.c                                 |  57 ++++
+ arch/xtensa/include/asm/Kbuild                     |   1 +
+ arch/xtensa/include/uapi/asm/msgbuf.h              |  25 +-
+ arch/xtensa/include/uapi/asm/sembuf.h              |  17 +-
+ arch/xtensa/include/uapi/asm/shmbuf.h              |  37 +--
+ drivers/base/platform-msi.c                        |   3 +
+ drivers/bus/fsl-mc/fsl-mc-msi.c                    |   2 +
+ drivers/firmware/efi/Kconfig                       |   5 +
+ drivers/firmware/efi/Makefile                      |   1 +
+ drivers/firmware/efi/capsule-loader.c              |  14 +-
+ drivers/firmware/efi/cper-arm.c                    |   6 +-
+ drivers/firmware/efi/cper-x86.c                    | 356 +++++++++++++++++++++
+ drivers/firmware/efi/cper.c                        |  16 +-
+ drivers/firmware/efi/libstub/secureboot.c          |   3 +
+ drivers/firmware/efi/libstub/tpm.c                 |   2 +-
+ drivers/irqchip/Makefile                           |   2 +-
+ drivers/irqchip/irq-gic-v3-mbi.c                   | 331 +++++++++++++++++++
+ drivers/irqchip/irq-gic-v3.c                       |   7 +
+ drivers/irqchip/irq-mvebu-gicp.c                   |  38 +--
+ drivers/irqchip/irq-mvebu-gicp.h                   |  12 -
+ drivers/irqchip/irq-mvebu-icu.c                    |  33 +-
+ drivers/nvdimm/claim.c                             |   3 +-
+ drivers/nvdimm/pmem.c                              |   6 +-
+ drivers/nvme/host/core.c                           |   2 +-
+ drivers/pci/msi.c                                  |   3 +
+ drivers/s390/block/dasd_ioctl.c                    |   1 -
+ drivers/s390/char/fs3270.c                         |   1 -
+ drivers/s390/char/sclp_ctl.c                       |   1 -
+ drivers/s390/char/vmcp.c                           |   1 -
+ drivers/s390/cio/chsc_sch.c                        |   1 -
+ drivers/s390/net/qeth_core_main.c                  |   2 +-
+ include/asm-generic/atomic-long.h                  |  19 ++
+ include/asm-generic/barrier.h                      |  27 +-
+ include/asm-generic/compat.h                       |   3 +
+ include/asm-generic/qspinlock.h                    |   4 +-
+ include/asm-generic/qspinlock_types.h              |  32 +-
+ include/linux/atomic.h                             |   2 +
+ include/linux/compat.h                             |  12 +-
+ include/linux/compat_time.h                        |  23 ++
+ include/linux/cper.h                               |   4 +-
+ include/linux/delayacct.h                          |   2 +-
+ include/linux/dma-iommu.h                          |   1 +
+ include/linux/efi.h                                |   6 +-
+ include/linux/interrupt.h                          |  13 +-
+ include/linux/irq.h                                |   1 +
+ include/linux/irq_cpustat.h                        |  10 +-
+ include/linux/irq_sim.h                            |  13 +-
+ include/linux/irqchip/arm-gic-v3.h                 |   1 +
+ include/linux/irqdomain.h                          |   8 +-
+ include/linux/msi.h                                |   2 +
+ include/linux/mutex.h                              |   3 -
+ include/linux/rcupdate.h                           |   4 +-
+ include/linux/rcutree.h                            |   1 +
+ include/linux/restart_block.h                      |   7 +-
+ include/linux/sched.h                              |   9 +-
+ include/linux/spinlock.h                           |  18 ++
+ include/linux/srcu.h                               |  36 ++-
+ include/linux/string.h                             |   4 +-
+ include/linux/swait.h                              |   1 +
+ include/linux/syscalls.h                           |  19 +-
+ include/linux/time.h                               |   4 +-
+ include/linux/time64.h                             |  10 +-
+ include/linux/uio.h                                |  15 +
+ include/trace/events/rcu.h                         |  13 +-
+ include/uapi/asm-generic/msgbuf.h                  |  27 +-
+ include/uapi/asm-generic/posix_types.h             |   1 +
+ include/uapi/asm-generic/sembuf.h                  |  26 +-
+ include/uapi/asm-generic/shmbuf.h                  |  41 ++-
+ include/uapi/linux/time.h                          |   7 +
+ ipc/mqueue.c                                       |  86 ++---
+ ipc/msg.c                                          |  20 +-
+ ipc/sem.c                                          |  27 +-
+ ipc/shm.c                                          |  14 +-
+ ipc/syscall.c                                      |  13 +-
+ ipc/util.h                                         |   4 +-
+ kernel/compat.c                                    |  52 +--
+ kernel/delayacct.c                                 |  17 +-
+ kernel/irq/irq_sim.c                               |   7 +-
+ kernel/irq/msi.c                                   |  33 +-
+ kernel/locking/lockdep.c                           |  70 ++--
+ kernel/locking/mcs_spinlock.h                      |  10 +-
+ kernel/locking/mutex.c                             |   3 +-
+ kernel/locking/qspinlock.c                         | 247 +++++++-------
+ kernel/locking/qspinlock_paravirt.h                |  49 +--
+ kernel/locking/qspinlock_stat.h                    |   9 +-
+ kernel/rcu/rcu.h                                   |  12 +-
+ kernel/rcu/rcu_segcblist.c                         |  18 --
+ kernel/rcu/rcu_segcblist.h                         |   2 -
+ kernel/rcu/rcuperf.c                               |   2 +-
+ kernel/rcu/rcutorture.c                            |  15 +-
+ kernel/rcu/srcutiny.c                              |   9 +-
+ kernel/rcu/srcutree.c                              |  30 +-
+ kernel/rcu/tree.c                                  | 355 ++++++++------------
+ kernel/rcu/tree.h                                  |  36 ++-
+ kernel/rcu/tree_exp.h                              | 235 ++++++++------
+ kernel/rcu/tree_plugin.h                           |  98 +++---
+ kernel/rcu/update.c                                |  50 +--
+ kernel/sched/core.c                                |  53 ++-
+ kernel/sched/fair.c                                | 108 ++++---
+ kernel/sched/sched.h                               |   6 +
+ kernel/softirq.c                                   |   7 +-
+ kernel/stop_machine.c                              |  24 +-
+ kernel/time/clocksource.c                          |  35 +-
+ kernel/time/hrtimer.c                              |  10 +-
+ kernel/time/posix-stubs.c                          |  12 +-
+ kernel/time/posix-timers.c                         |  24 +-
+ kernel/time/tick-common.c                          |   3 +-
+ kernel/time/time.c                                 |  58 +++-
+ kernel/time/timer.c                                |  14 +-
+ kernel/time/timer_list.c                           |   2 -
+ kernel/torture.c                                   |   2 +-
+ kernel/trace/trace_benchmark.c                     |   4 +-
+ lib/iov_iter.c                                     |  61 ++++
+ tools/lib/symbol/kallsyms.c                        |   6 +
+ tools/lib/symbol/kallsyms.h                        |   2 +
+ tools/memory-model/Documentation/cheatsheet.txt    |   7 +-
+ tools/memory-model/Documentation/explanation.txt   | 221 ++++++++-----
+ tools/memory-model/Documentation/references.txt    |  17 +-
+ tools/memory-model/README                          |   2 +-
+ tools/memory-model/linux-kernel.bell               |   4 +-
+ tools/memory-model/linux-kernel.cat                |  41 ++-
+ tools/memory-model/linux-kernel.def                |  34 +-
+ tools/memory-model/litmus-tests/.gitignore         |   1 +
+ .../litmus-tests/IRIW+mbonceonces+OnceOnce.litmus  |   2 +-
+ .../MP+polockmbonce+poacquiresilsil.litmus         |  35 ++
+ .../MP+polockonce+poacquiresilsil.litmus           |  34 ++
+ tools/memory-model/litmus-tests/README             |  19 +-
+ .../WRC+pooncerelease+rmbonceonce+Once.litmus      |   4 +-
+ tools/memory-model/lock.cat                        | 107 +++++--
+ tools/memory-model/scripts/checkalllitmus.sh       |  73 +++++
+ tools/memory-model/scripts/checklitmus.sh          |  86 +++++
+ tools/objtool/arch/x86/decode.c                    |   5 +
+ tools/perf/Documentation/Makefile                  |  29 +-
+ tools/perf/Documentation/asciidoctor-extensions.rb |  29 ++
+ tools/perf/Documentation/perf-buildid-cache.txt    |   7 +-
+ tools/perf/Documentation/perf-stat.txt             |  16 +
+ tools/perf/Makefile.config                         |  14 +
+ tools/perf/Makefile.perf                           |   8 +
+ tools/perf/arch/arm/tests/dwarf-unwind.c           |   2 +-
+ tools/perf/arch/arm64/tests/dwarf-unwind.c         |   2 +-
+ tools/perf/arch/powerpc/tests/dwarf-unwind.c       |   2 +-
+ tools/perf/arch/powerpc/util/skip-callchain-idx.c  |   3 +-
+ tools/perf/arch/x86/tests/dwarf-unwind.c           |   2 +-
+ tools/perf/builtin-annotate.c                      |   2 +-
+ tools/perf/builtin-buildid-cache.c                 |  81 ++++-
+ tools/perf/builtin-inject.c                        |   4 +-
+ tools/perf/builtin-kallsyms.c                      |   2 +-
+ tools/perf/builtin-kmem.c                          |   6 +-
+ tools/perf/builtin-report.c                        |  14 +-
+ tools/perf/builtin-script.c                        |  30 +-
+ tools/perf/builtin-stat.c                          |  92 +++++-
+ tools/perf/builtin-timechart.c                     |   8 +-
+ tools/perf/builtin-top.c                           |   7 +-
+ tools/perf/builtin-trace.c                         |   9 +-
+ tools/perf/check-headers.sh                        |  30 +-
+ tools/perf/examples/bpf/5sec.c                     |  49 +++
+ tools/perf/examples/bpf/empty.c                    |   3 +
+ tools/perf/include/bpf/bpf.h                       |  13 +
+ tools/perf/tests/builtin-test.c                    |   9 +
+ tools/perf/tests/code-reading.c                    |   5 +-
+ tools/perf/tests/hists_common.c                    |   6 +-
+ tools/perf/tests/mmap-thread-lookup.c              |   7 +-
+ tools/perf/tests/parse-events.c                    |  13 +
+ tools/perf/tests/vmlinux-kallsyms.c                |  20 +-
+ tools/perf/ui/browsers/map.c                       |   2 +-
+ tools/perf/ui/stdio/hist.c                         |   3 +-
+ tools/perf/util/Build                              |   2 +
+ tools/perf/util/auxtrace.c                         |  12 +-
+ tools/perf/util/bpf-loader.c                       |   6 +-
+ tools/perf/util/build-id.c                         |   4 +-
+ tools/perf/util/cs-etm.c                           |   4 +-
+ tools/perf/util/db-export.c                        |   7 +-
+ tools/perf/util/dso.c                              |  34 +-
+ tools/perf/util/dso.h                              |  37 ++-
+ tools/perf/util/event.c                            |  37 +--
+ tools/perf/util/evsel.h                            |   1 +
+ tools/perf/util/genelf.c                           |   2 +-
+ tools/perf/util/intel-bts.c                        |   3 +-
+ tools/perf/util/intel-pt.c                         |   8 +-
+ tools/perf/util/llvm-utils.c                       |  19 +-
+ tools/perf/util/machine.c                          | 162 ++++------
+ tools/perf/util/machine.h                          |  47 ++-
+ tools/perf/util/map.c                              |  99 ++----
+ tools/perf/util/map.h                              |  59 +---
+ tools/perf/util/parse-events.c                     | 130 +++++++-
+ tools/perf/util/parse-events.h                     |   7 +-
+ tools/perf/util/parse-events.y                     |   8 +-
+ tools/perf/util/probe-event.c                      |  29 +-
+ tools/perf/util/session.c                          |  13 +-
+ tools/perf/util/sort.c                             |   6 +-
+ tools/perf/util/srcline.c                          |   1 +
+ tools/perf/util/stat.h                             |   3 +-
+ tools/perf/util/symbol-elf.c                       | 275 ++++++++--------
+ tools/perf/util/symbol-minimal.c                   |   3 +-
+ tools/perf/util/symbol.c                           | 219 ++++++-------
+ tools/perf/util/symbol.h                           |  24 +-
+ tools/perf/util/symbol_fprintf.c                   |   4 +-
+ tools/perf/util/thread.c                           |  35 +-
+ tools/perf/util/thread.h                           |  13 +-
+ tools/perf/util/unwind-libdw.c                     |  23 +-
+ tools/perf/util/unwind-libunwind-local.c           |  19 +-
+ tools/perf/util/vdso.c                             |   6 +-
+ .../selftests/rcutorture/bin/kvm-find-errors.sh    |  56 ++++
+ .../selftests/rcutorture/bin/kvm-recheck-rcu.sh    |  12 +-
+ .../selftests/rcutorture/bin/kvm-recheck.sh        |   4 -
+ .../selftests/rcutorture/bin/kvm-test-1-run.sh     |   1 -
+ .../selftests/rcutorture/bin/parse-console.sh      | 115 ++++++-
+ .../selftests/rcutorture/bin/parse-torture.sh      | 105 ------
+ 330 files changed, 4836 insertions(+), 3468 deletions(-)
+ delete mode 100644 arch/alpha/include/uapi/asm/ipcbuf.h
+ delete mode 100644 arch/alpha/include/uapi/asm/msgbuf.h
+ delete mode 100644 arch/alpha/include/uapi/asm/sembuf.h
+ delete mode 100644 arch/alpha/include/uapi/asm/shmbuf.h
+ delete mode 100644 arch/ia64/include/uapi/asm/ipcbuf.h
+ delete mode 100644 arch/ia64/include/uapi/asm/msgbuf.h
+ delete mode 100644 arch/ia64/include/uapi/asm/sembuf.h
+ delete mode 100644 arch/ia64/include/uapi/asm/shmbuf.h
+ delete mode 100644 arch/s390/include/uapi/asm/msgbuf.h
+ delete mode 100644 arch/s390/include/uapi/asm/sembuf.h
+ delete mode 100644 arch/s390/include/uapi/asm/shmbuf.h
+ create mode 100644 arch/x86/include/asm/cacheinfo.h
+ rename arch/x86/kernel/cpu/{intel_cacheinfo.c => cacheinfo.c} (95%)
+ rename arch/x86/kernel/cpu/mtrr/{main.c => mtrr.c} (96%)
+ create mode 100644 drivers/firmware/efi/cper-x86.c
+ create mode 100644 drivers/irqchip/irq-gic-v3-mbi.c
+ delete mode 100644 drivers/irqchip/irq-mvebu-gicp.h
+ create mode 100644 include/asm-generic/compat.h
+ create mode 100644 include/linux/compat_time.h
+ create mode 100644 tools/memory-model/litmus-tests/.gitignore
+ create mode 100644 tools/memory-model/litmus-tests/MP+polockmbonce+poacquiresilsil.litmus
+ create mode 100644 tools/memory-model/litmus-tests/MP+polockonce+poacquiresilsil.litmus
+ create mode 100644 tools/memory-model/scripts/checkalllitmus.sh
+ create mode 100644 tools/memory-model/scripts/checklitmus.sh
+ create mode 100644 tools/perf/Documentation/asciidoctor-extensions.rb
+ create mode 100644 tools/perf/examples/bpf/5sec.c
+ create mode 100644 tools/perf/examples/bpf/empty.c
+ create mode 100644 tools/perf/include/bpf/bpf.h
+ create mode 100755 tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh
+ delete mode 100755 tools/testing/selftests/rcutorture/bin/parse-torture.sh
+Merging clockevents/clockevents/next (6bd9549d8a7d clocksource/drivers/arc_timer: Add comments about locking while read GFRC)
+$ git merge clockevents/clockevents/next
+Merge made by the 'recursive' strategy.
+ drivers/clocksource/Kconfig          |  4 +++-
+ drivers/clocksource/arc_timer.c      | 14 ++++++++++++++
+ drivers/clocksource/mips-gic-timer.c | 18 ++++++++++--------
+ 3 files changed, 27 insertions(+), 9 deletions(-)
+Merging edac/linux_next (345fb0a9a634 Merge tag 'edac_for_4.11' of git://git.kernel.org/pub/scm/linux/kernel/git/bp/bp)
+$ git merge edac/linux_next
+Already up to date.
+Merging edac-amd/for-next (eaa3a1d46cfd EDAC, ghes: Make platform-based whitelisting x86-only)
+$ git merge edac-amd/for-next
+Auto-merging Documentation/devicetree/bindings/edac/socfpga-eccmgr.txt
+Merge made by the 'recursive' strategy.
+ .../{arm/altera => edac}/socfpga-eccmgr.txt        |  35 ++
+ arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts |   6 +-
+ arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi  |  83 +++-
+ .../boot/dts/altera/socfpga_stratix10_socdk.dts    |  44 +-
+ drivers/acpi/apei/ghes.c                           |  16 +-
+ drivers/edac/Kconfig                               |   3 +-
+ drivers/edac/altera_edac.c                         | 504 ++++++++++++++++++++-
+ drivers/edac/altera_edac.h                         | 132 +++++-
+ drivers/edac/ghes_edac.c                           |  55 ++-
+ drivers/edac/i7core_edac.c                         |   2 +-
+ include/acpi/ghes.h                                |   7 +-
+ 11 files changed, 817 insertions(+), 70 deletions(-)
+ rename Documentation/devicetree/bindings/{arm/altera => edac}/socfpga-eccmgr.txt (87%)
+Merging irqchip/irq/irqchip-next (868c4e07533c irqchip/meson-gpio: Add support for Meson-AXG SoCs)
+$ git merge irqchip/irq/irqchip-next
+Merge made by the 'recursive' strategy.
+ .../bindings/interrupt-controller/amlogic,meson-gpio-intc.txt | 11 ++++++-----
+ drivers/irqchip/irq-meson-gpio.c                              |  5 +++++
+ 2 files changed, 11 insertions(+), 5 deletions(-)
+Merging ftrace/for-next (0c5a9acc8b4e tracing: Fix the file mode of stack tracer)
+$ git merge ftrace/for-next
+Already up to date.
+Merging rcu/rcu/next (09ad34c32895 cpufreq: Use static SRCU initializer)
+$ git merge rcu/rcu/next
+Removing tools/testing/selftests/rcutorture/configs/rcu/TREE08-T.boot
+Auto-merging arch/x86/kernel/cpu/mtrr/mtrr.c
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Merge made by the 'recursive' strategy.
+ .../Design/Data-Structures/Data-Structures.html    | 118 +--
+ .../Memory-Ordering/Tree-RCU-Memory-Ordering.html  |  22 +-
+ .../Design/Memory-Ordering/TreeRCU-gp-cleanup.svg  | 123 +--
+ .../Design/Memory-Ordering/TreeRCU-gp-init-1.svg   |  16 +-
+ .../Design/Memory-Ordering/TreeRCU-gp-init-3.svg   |  56 +-
+ .../RCU/Design/Memory-Ordering/TreeRCU-gp.svg      | 237 +++---
+ .../RCU/Design/Memory-Ordering/TreeRCU-qs.svg      |  12 +-
+ Documentation/RCU/stallwarn.txt                    |  24 +-
+ Documentation/admin-guide/kernel-parameters.txt    |   4 +-
+ arch/x86/kernel/cpu/mtrr/mtrr.c                    |   4 +
+ include/linux/rcupdate.h                           |  21 +-
+ include/linux/rcutiny.h                            |   3 +-
+ include/linux/rcutree.h                            |   1 +
+ include/linux/srcu.h                               |  17 +
+ include/linux/torture.h                            |   4 +-
+ include/trace/events/rcu.h                         | 112 ++-
+ kernel/locking/locktorture.c                       |   5 +-
+ kernel/rcu/rcu.h                                   | 111 ++-
+ kernel/rcu/rcuperf.c                               |  57 +-
+ kernel/rcu/rcutorture.c                            | 185 ++--
+ kernel/rcu/srcutree.c                              |  36 +-
+ kernel/rcu/tiny.c                                  |   4 +-
+ kernel/rcu/tree.c                                  | 947 +++++++++++----------
+ kernel/rcu/tree.h                                  |  71 +-
+ kernel/rcu/tree_exp.h                              |   5 +-
+ kernel/rcu/tree_plugin.h                           | 174 ++--
+ kernel/rcu/update.c                                |  45 +-
+ kernel/torture.c                                   |  15 +-
+ .../testing/selftests/rcutorture/bin/configinit.sh |  26 +-
+ .../testing/selftests/rcutorture/bin/kvm-build.sh  |  11 +-
+ .../selftests/rcutorture/bin/kvm-recheck-rcu.sh    |   1 +
+ .../selftests/rcutorture/bin/kvm-recheck.sh        |   1 +
+ .../selftests/rcutorture/bin/kvm-test-1-run.sh     |   5 +-
+ tools/testing/selftests/rcutorture/bin/kvm.sh      |   2 +-
+ .../selftests/rcutorture/bin/parse-console.sh      |   7 +
+ .../selftests/rcutorture/configs/rcu/TREE03.boot   |   4 +-
+ .../selftests/rcutorture/configs/rcu/TREE08-T.boot |   1 -
+ .../rcutorture/configs/rcu/ver_functions.sh        |   2 +-
+ 38 files changed, 1326 insertions(+), 1163 deletions(-)
+ delete mode 100644 tools/testing/selftests/rcutorture/configs/rcu/TREE08-T.boot
+Merging kvm/linux-next (0ea3286e2df7 KVM: x86: Expose CLDEMOTE CPU feature to guest VM)
+$ git merge kvm/linux-next
+Auto-merging arch/x86/kvm/x86.c
+Auto-merging arch/x86/kvm/vmx.c
+Auto-merging arch/x86/kvm/svm.c
+Auto-merging arch/x86/kvm/lapic.c
+Auto-merging arch/x86/kvm/cpuid.c
+Auto-merging arch/x86/include/asm/kvm_host.h
+Merge made by the 'recursive' strategy.
+ arch/x86/include/asm/hyperv-tlfs.h |   9 +-
+ arch/x86/include/asm/kvm_host.h    |   6 +-
+ arch/x86/include/asm/vmx.h         |   2 +
+ arch/x86/kvm/cpuid.c               |   3 +-
+ arch/x86/kvm/emulate.c             |   4 +-
+ arch/x86/kvm/lapic.c               |  12 +-
+ arch/x86/kvm/lapic.h               |  14 ++
+ arch/x86/kvm/mmu.c                 |  78 ++++----
+ arch/x86/kvm/svm.c                 |  12 +-
+ arch/x86/kvm/vmx.c                 | 363 +++++++++++++++++++++++++++++--------
+ arch/x86/kvm/x86.c                 |  34 ++--
+ 11 files changed, 393 insertions(+), 144 deletions(-)
+Merging kvm-arm/next (e25028c8ded0 KVM: arm/arm64: Bump VGIC_V3_MAX_CPUS to 512)
+$ git merge kvm-arm/next
+Auto-merging virt/kvm/arm/vgic/vgic-v3.c
+Auto-merging include/linux/sched.h
+Auto-merging include/linux/kvm_host.h
+Auto-merging arch/arm64/kernel/ptrace.c
+Auto-merging arch/arm64/kernel/fpsimd.c
+Auto-merging arch/arm64/Kconfig
+Auto-merging arch/arm/include/asm/kvm_host.h
+Merge made by the 'recursive' strategy.
+ Documentation/virtual/kvm/devices/arm-vgic-v3.txt |  30 +++-
+ arch/arm/include/asm/kvm_host.h                   |  10 +-
+ arch/arm/include/uapi/asm/kvm.h                   |   1 +
+ arch/arm64/Kconfig                                |   7 +
+ arch/arm64/include/asm/cpufeature.h               |  29 ----
+ arch/arm64/include/asm/fpsimd.h                   |  21 +++
+ arch/arm64/include/asm/kvm_asm.h                  |   8 +-
+ arch/arm64/include/asm/kvm_host.h                 |  45 ++++--
+ arch/arm64/include/asm/processor.h                |  15 +-
+ arch/arm64/include/asm/thread_info.h              |  13 +-
+ arch/arm64/include/uapi/asm/kvm.h                 |   1 +
+ arch/arm64/kernel/fpsimd.c                        | 177 +++++++++++-----------
+ arch/arm64/kernel/ptrace.c                        |   1 +
+ arch/arm64/kvm/Kconfig                            |   1 +
+ arch/arm64/kvm/Makefile                           |   2 +-
+ arch/arm64/kvm/debug.c                            |   8 +-
+ arch/arm64/kvm/fpsimd.c                           | 110 ++++++++++++++
+ arch/arm64/kvm/hyp/debug-sr.c                     |   6 +-
+ arch/arm64/kvm/hyp/entry.S                        |  43 ------
+ arch/arm64/kvm/hyp/hyp-entry.S                    |  19 ---
+ arch/arm64/kvm/hyp/switch.c                       | 124 ++++++++++-----
+ arch/arm64/kvm/hyp/sysreg-sr.c                    |   4 +-
+ arch/arm64/kvm/sys_regs.c                         |   9 +-
+ include/kvm/arm_vgic.h                            |  17 ++-
+ include/linux/kvm_host.h                          |   9 ++
+ include/linux/sched.h                             |   6 +
+ include/linux/thread_info.h                       |  11 ++
+ virt/kvm/Kconfig                                  |   3 +
+ virt/kvm/arm/arm.c                                |  15 +-
+ virt/kvm/arm/vgic/vgic-init.c                     | 100 ++++++------
+ virt/kvm/arm/vgic/vgic-kvm-device.c               |  53 ++++++-
+ virt/kvm/arm/vgic/vgic-mmio-v3.c                  | 112 ++++++++++++--
+ virt/kvm/arm/vgic/vgic-v3.c                       |  99 +++++++++---
+ virt/kvm/arm/vgic/vgic.h                          |  42 ++++-
+ virt/kvm/kvm_main.c                               |   7 +-
+ 35 files changed, 809 insertions(+), 349 deletions(-)
+ create mode 100644 arch/arm64/kvm/fpsimd.c
+Merging kvm-mips/next (dc44abd6aad2 KVM: MIPS/Emulate: Properly implement TLBR for T&E)
+$ git merge kvm-mips/next
+Already up to date.
+Merging kvm-ppc/kvm-ppc-next (acc9eb9305fe KVM: PPC: Reimplement LOAD_VMX/STORE_VMX instruction mmio emulation with analyse_instr() input)
+$ git merge kvm-ppc/kvm-ppc-next
+Auto-merging arch/powerpc/kvm/book3s_hv_rmhandlers.S
+Auto-merging arch/powerpc/kvm/book3s_hv.c
+Auto-merging arch/powerpc/kernel/asm-offsets.c
+Merge made by the 'recursive' strategy.
+ .../powerpc/include/asm/book3s/64/tlbflush-radix.h |   7 +
+ arch/powerpc/include/asm/kvm_book3s.h              |  21 +-
+ arch/powerpc/include/asm/kvm_book3s_64.h           |  20 +-
+ arch/powerpc/include/asm/kvm_booke.h               |  20 +-
+ arch/powerpc/include/asm/kvm_host.h                |  20 +-
+ arch/powerpc/include/asm/kvm_ppc.h                 |  17 +-
+ arch/powerpc/include/asm/reg.h                     |   1 +
+ arch/powerpc/include/uapi/asm/kvm.h                |   1 +
+ arch/powerpc/kernel/asm-offsets.c                  |  18 +-
+ arch/powerpc/kernel/kvm.c                          |   4 +-
+ arch/powerpc/kvm/book3s.c                          |   4 +-
+ arch/powerpc/kvm/book3s_32_mmu.c                   |   2 +-
+ arch/powerpc/kvm/book3s_64_mmu_hv.c                |   3 +
+ arch/powerpc/kvm/book3s_64_mmu_radix.c             | 348 ++++++----
+ arch/powerpc/kvm/book3s_64_vio.c                   |  71 +-
+ arch/powerpc/kvm/book3s_64_vio_hv.c                |  54 +-
+ arch/powerpc/kvm/book3s_hv.c                       |  97 ++-
+ arch/powerpc/kvm/book3s_hv_builtin.c               |  21 +-
+ arch/powerpc/kvm/book3s_hv_interrupts.S            |   2 +-
+ arch/powerpc/kvm/book3s_hv_rm_mmu.c                |  36 +-
+ arch/powerpc/kvm/book3s_hv_rm_xics.c               |   2 +-
+ arch/powerpc/kvm/book3s_hv_rmhandlers.S            |  15 +-
+ arch/powerpc/kvm/book3s_hv_tm.c                    |  10 +-
+ arch/powerpc/kvm/book3s_hv_tm_builtin.c            |  10 +-
+ arch/powerpc/kvm/book3s_pr.c                       |  84 +--
+ arch/powerpc/kvm/book3s_xive_template.c            |   4 +-
+ arch/powerpc/kvm/booke.c                           |  41 +-
+ arch/powerpc/kvm/booke_emulate.c                   |   6 +-
+ arch/powerpc/kvm/e500_emulate.c                    |   6 +-
+ arch/powerpc/kvm/e500_mmu.c                        |   2 +-
+ arch/powerpc/kvm/e500_mmu_host.c                   |   8 +-
+ arch/powerpc/kvm/emulate_loadstore.c               | 752 ++++++---------------
+ arch/powerpc/kvm/powerpc.c                         | 299 ++++++--
+ arch/powerpc/mm/tlb-radix.c                        | 207 ++++++
+ 34 files changed, 1300 insertions(+), 913 deletions(-)
+Merging kvms390/next (2c8180e885c1 KVM: s390: vsie: simplify < 8k address checks)
+$ git merge kvms390/next
+Auto-merging arch/s390/kvm/priv.c
+Merge made by the 'recursive' strategy.
+ arch/s390/include/asm/ctl_reg.h     | 12 ++++++++
+ arch/s390/include/asm/kvm_host.h    |  1 +
+ arch/s390/include/asm/mmu.h         |  2 +-
+ arch/s390/include/asm/mmu_context.h |  2 +-
+ arch/s390/include/asm/pgtable.h     |  4 +--
+ arch/s390/kvm/guestdbg.c            |  2 +-
+ arch/s390/kvm/interrupt.c           | 20 ++++++-------
+ arch/s390/kvm/kvm-s390.c            | 59 +++++++++++++++++++------------------
+ arch/s390/kvm/kvm-s390.h            | 13 ++++++++
+ arch/s390/kvm/priv.c                | 28 ++++++++++--------
+ arch/s390/kvm/vsie.c                | 10 +++----
+ arch/s390/mm/gmap.c                 |  6 ++--
+ arch/s390/mm/pgtable.c              |  4 +--
+ 13 files changed, 98 insertions(+), 65 deletions(-)
+Merging xen-tip/linux-next (c78ad1667036 xen/PVH: Make GDT selectors PVH-specific)
+$ git merge xen-tip/linux-next
+Merge made by the 'recursive' strategy.
+ arch/x86/xen/xen-pvh.S | 47 +++++++++++++++++++++++++++++++++++++----------
+ 1 file changed, 37 insertions(+), 10 deletions(-)
+Merging percpu/for-next (b3a5d1119944 percpu_ref: Update doc to dissuade users from depending on internal RCU grace periods)
+$ git merge percpu/for-next
+Already up to date.
+Merging workqueues/for-next (197f6accacda workqueue: Make sure struct worker is accessible for wq_worker_comm())
+$ git merge workqueues/for-next
+Auto-merging fs/proc/internal.h
+Auto-merging fs/proc/base.c
+Auto-merging fs/proc/array.c
+Merge made by the 'recursive' strategy.
+ fs/proc/array.c             |  33 ++++++-----
+ fs/proc/base.c              |   5 +-
+ fs/proc/internal.h          |   2 +
+ include/linux/workqueue.h   |   1 +
+ kernel/workqueue.c          | 135 +++++++++++++++++++++++++++++---------------
+ kernel/workqueue_internal.h |   3 +-
+ 6 files changed, 117 insertions(+), 62 deletions(-)
+Merging drivers-x86/for-next (8b0695841da8 platform/x86: dell-wmi: Set correct keycode for Fn + left arrow)
+$ git merge drivers-x86/for-next
+Merge made by the 'recursive' strategy.
+ .../ABI/testing/sysfs-platform-ideapad-laptop      | 13 +++++
+ Documentation/laptops/thinkpad-acpi.txt            |  2 +
+ drivers/platform/mellanox/mlxreg-hotplug.c         |  6 ++-
+ drivers/platform/x86/acer-wmi.c                    |  1 +
+ drivers/platform/x86/apple-gmux.c                  |  2 +-
+ drivers/platform/x86/asus-laptop.c                 |  3 +-
+ drivers/platform/x86/asus-wmi.c                    |  3 +-
+ drivers/platform/x86/dell-smbios-base.c            |  5 +-
+ drivers/platform/x86/dell-wmi.c                    |  2 +-
+ drivers/platform/x86/fujitsu-laptop.c              | 47 ++++++++---------
+ drivers/platform/x86/ideapad-laptop.c              | 59 ++++++++++++++++++++--
+ drivers/platform/x86/intel_scu_ipc.c               |  4 +-
+ drivers/platform/x86/samsung-laptop.c              |  3 +-
+ drivers/platform/x86/thinkpad_acpi.c               | 30 +++++++----
+ include/linux/platform_data/mlxreg.h               |  3 +-
+ include/linux/string_helpers.h                     |  1 +
+ 16 files changed, 132 insertions(+), 52 deletions(-)
+Merging chrome-platform/for-next (79a3d60300fc platform/chrome: Use to_cros_ec_dev more broadly)
+$ git merge chrome-platform/for-next
+Auto-merging include/linux/mfd/cros_ec.h
+Merge made by the 'recursive' strategy.
+ drivers/platform/chrome/Kconfig            |  11 +++
+ drivers/platform/chrome/Makefile           |   1 +
+ drivers/platform/chrome/chromeos_laptop.c  |  10 ++-
+ drivers/platform/chrome/chromeos_tbmc.c    | 111 +++++++++++++++++++++++++++++
+ drivers/platform/chrome/cros_ec_lightbar.c |  21 ++----
+ drivers/platform/chrome/cros_ec_lpc.c      |  13 ++--
+ drivers/platform/chrome/cros_ec_sysfs.c    |   2 -
+ drivers/platform/chrome/cros_ec_vbc.c      |   9 +--
+ include/linux/mfd/cros_ec.h                |   2 +
+ 9 files changed, 151 insertions(+), 29 deletions(-)
+ create mode 100644 drivers/platform/chrome/chromeos_tbmc.c
+Merging hsi/for-next (70d52ba9e57e hsi: clients: Change return type to vm_fault_t)
+$ git merge hsi/for-next
+Merge made by the 'recursive' strategy.
+ drivers/hsi/clients/cmt_speech.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging leds/for-next (6d71021ab3b0 leds: class: ensure workqueue is initialized before setting brightness)
+$ git merge leds/for-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.txt
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/leds/leds-cr0014114.txt    |  54 +++
+ .../devicetree/bindings/leds/leds-lm3601x.txt      |  45 ++
+ .../devicetree/bindings/leds/leds-sc27xx-bltc.txt  |  41 ++
+ .../devicetree/bindings/vendor-prefixes.txt        |   1 +
+ drivers/leds/Kconfig                               |  33 ++
+ drivers/leds/Makefile                              |   3 +
+ drivers/leds/led-class.c                           |  10 +-
+ drivers/leds/leds-cr0014114.c                      | 315 +++++++++++++
+ drivers/leds/leds-lm3601x.c                        | 487 +++++++++++++++++++++
+ drivers/leds/leds-sc27xx-bltc.c                    | 244 +++++++++++
+ drivers/leds/leds-wm831x-status.c                  |  22 +-
+ 11 files changed, 1237 insertions(+), 18 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/leds/leds-cr0014114.txt
+ create mode 100644 Documentation/devicetree/bindings/leds/leds-lm3601x.txt
+ create mode 100644 Documentation/devicetree/bindings/leds/leds-sc27xx-bltc.txt
+ create mode 100644 drivers/leds/leds-cr0014114.c
+ create mode 100644 drivers/leds/leds-lm3601x.c
+ create mode 100644 drivers/leds/leds-sc27xx-bltc.c
+Merging ipmi/for-next (048f7c3e352e ipmi: Properly release srcu locks on error conditions)
+$ git merge ipmi/for-next
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/ipmi/npcm7xx-kcs-bmc.txt   |   39 +
+ drivers/char/ipmi/Kconfig                          |   23 +-
+ drivers/char/ipmi/Makefile                         |    1 +
+ drivers/char/ipmi/ipmi_bt_sm.c                     |    3 +-
+ drivers/char/ipmi/ipmi_devintf.c                   |  129 +-
+ drivers/char/ipmi/ipmi_msghandler.c                | 2123 +++++++++-----------
+ drivers/char/ipmi/ipmi_poweroff.c                  |   32 +-
+ drivers/char/ipmi/ipmi_si_intf.c                   |  198 +-
+ drivers/char/ipmi/ipmi_ssif.c                      |  183 +-
+ drivers/char/ipmi/ipmi_watchdog.c                  |  407 ++--
+ drivers/char/ipmi/kcs_bmc_npcm7xx.c                |  215 ++
+ include/linux/ipmi.h                               |  153 +-
+ include/linux/ipmi_smi.h                           |  129 +-
+ 13 files changed, 1752 insertions(+), 1883 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/ipmi/npcm7xx-kcs-bmc.txt
+ create mode 100644 drivers/char/ipmi/kcs_bmc_npcm7xx.c
+Merging driver-core/driver-core-next (0dda2bb62423 driver-core: return EINVAL error instead of BUG_ON())
+$ git merge driver-core/driver-core-next
+Auto-merging drivers/net/wireless/ath/ath10k/core.c
+Auto-merging drivers/base/core.c
+Merge made by the 'recursive' strategy.
+ Documentation/dell_rbu.txt                         |   5 +-
+ .../driver-api/firmware/fallback-mechanisms.rst    |  14 +-
+ .../driver-api/firmware/firmware_cache.rst         |   4 +-
+ .../driver-api/firmware/request_firmware.rst       |   5 +
+ drivers/base/Kconfig                               |  90 +++---------
+ drivers/base/core.c                                |  18 ++-
+ drivers/base/driver.c                              |   6 +-
+ drivers/base/firmware_loader/Kconfig               | 154 +++++++++++++++++++++
+ drivers/base/firmware_loader/fallback.c            |  53 +++++--
+ drivers/base/firmware_loader/fallback.h            |  18 +--
+ drivers/base/firmware_loader/firmware.h            |  37 +++--
+ drivers/base/firmware_loader/main.c                |  57 ++++++--
+ drivers/base/memory.c                              |   8 +-
+ drivers/net/wireless/ath/ath10k/core.c             |   2 +-
+ drivers/net/wireless/ath/ath10k/testmode.c         |   2 +-
+ fs/debugfs/file.c                                  |  10 +-
+ fs/debugfs/inode.c                                 |   4 +-
+ fs/kernfs/file.c                                   |   8 +-
+ include/linux/firmware.h                           |  10 ++
+ 19 files changed, 355 insertions(+), 150 deletions(-)
+ create mode 100644 drivers/base/firmware_loader/Kconfig
+Merging usb/usb-next (2c093cdb0a5e Merge tag 'phy-for-4.18' of git://git.kernel.org/pub/scm/linux/kernel/git/kishon/linux-phy into usb-next)
+$ git merge usb/usb-next
+Removing include/linux/usb/atmel_usba_udc.h
+Removing drivers/usb/musb/musbhsdma.h
+Auto-merging drivers/usb/gadget/udc/fsl_udc_core.c
+Auto-merging drivers/i2c/busses/Kconfig
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/phy/phy-mtk-xsphy.txt      | 109 +++
+ .../devicetree/bindings/phy/qcom-qmp-phy.txt       |   3 +-
+ .../devicetree/bindings/phy/qcom-qusb2-phy.txt     |  23 +-
+ Documentation/devicetree/bindings/usb/dwc3.txt     |  21 +
+ .../bindings/usb/hisilicon,histb-xhci.txt          |  45 ++
+ .../devicetree/bindings/usb/qcom,dwc3.txt          |  85 ++-
+ Documentation/driver-api/usb/dwc3.rst              |   3 +-
+ MAINTAINERS                                        |   8 +
+ drivers/i2c/busses/Kconfig                         |   3 +-
+ drivers/phy/mediatek/Kconfig                       |   9 +
+ drivers/phy/mediatek/Makefile                      |   1 +
+ drivers/phy/mediatek/phy-mtk-xsphy.c               | 600 +++++++++++++++
+ drivers/phy/motorola/phy-mapphone-mdm6600.c        |  85 +++
+ drivers/phy/phy-core.c                             |  18 +
+ drivers/phy/qualcomm/Kconfig                       |  11 +-
+ drivers/phy/qualcomm/Makefile                      |   1 +
+ drivers/phy/qualcomm/phy-ath79-usb.c               | 108 +++
+ drivers/phy/qualcomm/phy-qcom-qmp.c                | 169 ++++-
+ drivers/phy/qualcomm/phy-qcom-qmp.h                |   5 +
+ drivers/phy/qualcomm/phy-qcom-qusb2.c              | 130 +++-
+ drivers/phy/samsung/phy-exynos-mipi-video.c        |  34 +-
+ drivers/phy/st/phy-stm32-usbphyc.c                 |   9 +-
+ drivers/phy/tegra/xusb.c                           |  15 +-
+ drivers/platform/x86/Kconfig                       |   4 +-
+ drivers/power/supply/power_supply_core.c           |   4 +-
+ drivers/usb/class/usbtmc.c                         |  81 +-
+ drivers/usb/dwc2/core.c                            |   7 +
+ drivers/usb/dwc2/core.h                            | 188 ++++-
+ drivers/usb/dwc2/core_intr.c                       |   8 +
+ drivers/usb/dwc2/debug.h                           |   2 +-
+ drivers/usb/dwc2/debugfs.c                         |  22 +-
+ drivers/usb/dwc2/gadget.c                          | 334 ++++----
+ drivers/usb/dwc2/hcd.c                             |   3 +-
+ drivers/usb/dwc2/hcd.h                             |  14 +-
+ drivers/usb/dwc2/hcd_ddma.c                        |   1 +
+ drivers/usb/dwc2/hcd_intr.c                        |  12 +
+ drivers/usb/dwc2/hcd_queue.c                       |   5 +-
+ drivers/usb/dwc2/hw.h                              |   2 +
+ drivers/usb/dwc2/params.c                          |  14 +-
+ drivers/usb/dwc2/pci.c                             |   6 +
+ drivers/usb/dwc3/Kconfig                           |  12 +
+ drivers/usb/dwc3/Makefile                          |   1 +
+ drivers/usb/dwc3/core.c                            | 154 +++-
+ drivers/usb/dwc3/core.h                            |  25 +-
+ drivers/usb/dwc3/debug.h                           |  26 +-
+ drivers/usb/dwc3/drd.c                             |  34 +-
+ drivers/usb/dwc3/dwc3-of-simple.c                  |   2 +-
+ drivers/usb/dwc3/dwc3-qcom.c                       | 620 +++++++++++++++
+ drivers/usb/dwc3/ep0.c                             |   6 +-
+ drivers/usb/dwc3/gadget.c                          | 773 +++++++++----------
+ drivers/usb/dwc3/gadget.h                          |   5 +-
+ drivers/usb/dwc3/trace.h                           |  12 +-
+ drivers/usb/gadget/composite.c                     |   2 +-
+ drivers/usb/gadget/function/f_ecm.c                |   3 +-
+ drivers/usb/gadget/function/f_fs.c                 |  11 +
+ drivers/usb/gadget/function/f_midi.c               |  26 +-
+ drivers/usb/gadget/function/f_printer.c            |   6 +-
+ drivers/usb/gadget/function/rndis.c                |   3 +
+ drivers/usb/gadget/function/u_ether.c              |   4 +
+ drivers/usb/gadget/udc/Kconfig                     |   2 +
+ drivers/usb/gadget/udc/Makefile                    |   1 +
+ drivers/usb/gadget/udc/aspeed-vhub/Kconfig         |   7 +
+ drivers/usb/gadget/udc/aspeed-vhub/Makefile        |   4 +
+ drivers/usb/gadget/udc/aspeed-vhub/core.c          | 425 +++++++++++
+ drivers/usb/gadget/udc/aspeed-vhub/dev.c           | 589 ++++++++++++++
+ drivers/usb/gadget/udc/aspeed-vhub/ep0.c           | 486 ++++++++++++
+ drivers/usb/gadget/udc/aspeed-vhub/epn.c           | 843 +++++++++++++++++++++
+ drivers/usb/gadget/udc/aspeed-vhub/hub.c           | 829 ++++++++++++++++++++
+ drivers/usb/gadget/udc/aspeed-vhub/vhub.h          | 514 +++++++++++++
+ drivers/usb/gadget/udc/atmel_usba_udc.c            |  22 +-
+ drivers/usb/gadget/udc/atmel_usba_udc.h            |   1 -
+ drivers/usb/gadget/udc/core.c                      |   6 +
+ drivers/usb/gadget/udc/fsl_udc_core.c              |   1 +
+ drivers/usb/gadget/udc/renesas_usb3.c              |  37 +-
+ drivers/usb/gadget/usbstring.c                     |   2 +-
+ drivers/usb/host/Kconfig                           |   7 +
+ drivers/usb/host/Makefile                          |   1 +
+ drivers/usb/host/xhci-dbgcap.c                     |   1 +
+ drivers/usb/host/xhci-dbgcap.h                     |   2 +-
+ drivers/usb/host/xhci-dbgtty.c                     |   1 +
+ drivers/usb/host/xhci-debugfs.c                    |  85 +++
+ drivers/usb/host/xhci-histb.c                      | 410 ++++++++++
+ drivers/usb/host/xhci-hub.c                        | 244 +++---
+ drivers/usb/host/xhci-mem.c                        | 140 ++--
+ drivers/usb/host/xhci-mtk-sch.c                    |   4 +-
+ drivers/usb/host/xhci-ring.c                       | 126 +--
+ drivers/usb/host/xhci-tegra.c                      | 176 +++--
+ drivers/usb/host/xhci.c                            |  52 +-
+ drivers/usb/host/xhci.h                            |  39 +-
+ drivers/usb/misc/sisusbvga/sisusb.c                |   2 +-
+ drivers/usb/mtu3/Kconfig                           |   3 +-
+ drivers/usb/mtu3/mtu3.h                            |  15 +-
+ drivers/usb/mtu3/mtu3_core.c                       |  26 +-
+ drivers/usb/mtu3/mtu3_dr.c                         |  69 +-
+ drivers/usb/mtu3/mtu3_gadget.c                     |   9 +-
+ drivers/usb/mtu3/mtu3_gadget_ep0.c                 |  12 +-
+ drivers/usb/musb/am35x.c                           |   3 -
+ drivers/usb/musb/da8xx.c                           |   2 -
+ drivers/usb/musb/davinci.c                         |  49 +-
+ drivers/usb/musb/musb_core.c                       | 833 ++++++++++----------
+ drivers/usb/musb/musb_core.h                       |  16 +-
+ drivers/usb/musb/musb_cppi41.c                     |   4 +-
+ drivers/usb/musb/musb_dma.h                        |  10 +-
+ drivers/usb/musb/musb_dsps.c                       |  30 +-
+ drivers/usb/musb/musb_gadget.c                     |   8 +-
+ drivers/usb/musb/musb_gadget.h                     |  10 +-
+ drivers/usb/musb/musb_host.c                       |   8 +-
+ drivers/usb/musb/musb_io.h                         |   6 +-
+ drivers/usb/musb/musb_regs.h                       |  55 --
+ drivers/usb/musb/musb_virthub.c                    |   2 +-
+ drivers/usb/musb/musbhsdma.c                       |  74 +-
+ drivers/usb/musb/musbhsdma.h                       |  72 --
+ drivers/usb/musb/omap2430.c                        |   5 -
+ drivers/usb/musb/sunxi.c                           |  14 +-
+ drivers/usb/musb/ux500.c                           |   2 -
+ drivers/usb/roles/intel-xhci-usb-role-switch.c     |  37 +-
+ drivers/usb/storage/uas.c                          |   6 +
+ drivers/usb/storage/unusual_devs.h                 |   9 +
+ drivers/usb/storage/unusual_uas.h                  |   9 +
+ drivers/usb/typec/fusb302/fusb302.c                |   1 +
+ drivers/usb/typec/mux.c                            |   2 +-
+ drivers/usb/typec/tcpm.c                           |   4 +-
+ drivers/usb/typec/typec_wcove.c                    |  30 +-
+ drivers/usb/usbip/vhci_sysfs.c                     |  24 +-
+ include/dt-bindings/phy/phy-qcom-qusb2.h           |  37 +
+ include/linux/power_supply.h                       |   2 +
+ include/linux/usb/atmel_usba_udc.h                 |  24 -
+ include/linux/usb/gadget.h                         |   2 +-
+ include/linux/usb/musb.h                           |  15 -
+ 129 files changed, 8446 insertions(+), 2079 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt
+ create mode 100644 Documentation/devicetree/bindings/usb/hisilicon,histb-xhci.txt
+ create mode 100644 drivers/phy/mediatek/phy-mtk-xsphy.c
+ create mode 100644 drivers/phy/qualcomm/phy-ath79-usb.c
+ create mode 100644 drivers/usb/dwc3/dwc3-qcom.c
+ create mode 100644 drivers/usb/gadget/udc/aspeed-vhub/Kconfig
+ create mode 100644 drivers/usb/gadget/udc/aspeed-vhub/Makefile
+ create mode 100644 drivers/usb/gadget/udc/aspeed-vhub/core.c
+ create mode 100644 drivers/usb/gadget/udc/aspeed-vhub/dev.c
+ create mode 100644 drivers/usb/gadget/udc/aspeed-vhub/ep0.c
+ create mode 100644 drivers/usb/gadget/udc/aspeed-vhub/epn.c
+ create mode 100644 drivers/usb/gadget/udc/aspeed-vhub/hub.c
+ create mode 100644 drivers/usb/gadget/udc/aspeed-vhub/vhub.h
+ create mode 100644 drivers/usb/host/xhci-histb.c
+ delete mode 100644 drivers/usb/musb/musbhsdma.h
+ create mode 100644 include/dt-bindings/phy/phy-qcom-qusb2.h
+ delete mode 100644 include/linux/usb/atmel_usba_udc.h
+Merging usb-gadget/next (47265c067c0d usb: dwc2: gadget: Fix coverity issue)
+$ git merge usb-gadget/next
+Already up to date.
+Merging usb-serial/usb-next (7041d9c3f01b USB: serial: pl2303: add support for tx xon/xoff flow control)
+$ git merge usb-serial/usb-next
+Auto-merging include/linux/tty.h
+Auto-merging drivers/usb/serial/usb-serial.c
+Auto-merging drivers/usb/serial/option.c
+Merge made by the 'recursive' strategy.
+ drivers/tty/tty_ioctl.c         |   2 +-
+ drivers/usb/serial/bus.c        |   3 +-
+ drivers/usb/serial/ftdi_sio.c   | 194 ++++++++++++++++------------------------
+ drivers/usb/serial/option.c     |   3 +-
+ drivers/usb/serial/pl2303.c     |  16 +++-
+ drivers/usb/serial/usb-serial.c |   2 +-
+ include/linux/tty.h             |   2 +-
+ 7 files changed, 99 insertions(+), 123 deletions(-)
+Merging usb-chipidea-next/ci-for-usb-next (8bcf4c0bd7fb usb: chipidea: properly handle host or gadget initialization failure)
+$ git merge usb-chipidea-next/ci-for-usb-next
+Merge made by the 'recursive' strategy.
+Merging phy-next/next (c1eb8f83845b phy: mediatek: add XS-PHY driver)
+$ git merge phy-next/next
+Already up to date.
+Merging tty/tty-next (4b4ecd9cb853 vt: Perform safe console erase only once)
+$ git merge tty/tty-next
+Auto-merging drivers/tty/serial/serial_core.c
+Auto-merging arch/arm/boot/dts/armada-38x.dtsi
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Merge made by the 'recursive' strategy.
+ Documentation/admin-guide/kernel-parameters.txt |   6 +
+ arch/arm/boot/dts/armada-38x.dtsi               |   4 +-
+ drivers/tty/ipwireless/network.c                |   2 +-
+ drivers/tty/n_gsm.c                             |   2 +-
+ drivers/tty/nozomi.c                            | 104 +++++++-------
+ drivers/tty/pty.c                               |   3 +
+ drivers/tty/serial/8250/8250_aspeed_vuart.c     | 125 +++++++++++++++++
+ drivers/tty/serial/8250/8250_dw.c               |  35 ++++-
+ drivers/tty/serial/8250/8250_early.c            |   2 +-
+ drivers/tty/serial/8250/8250_of.c               |  63 +++++----
+ drivers/tty/serial/8250/8250_omap.c             |  27 +++-
+ drivers/tty/serial/8250/8250_port.c             |   6 +-
+ drivers/tty/serial/Kconfig                      |   8 +-
+ drivers/tty/serial/amba-pl011.c                 |  16 +++
+ drivers/tty/serial/atmel_serial.c               |   5 +-
+ drivers/tty/serial/earlycon.c                   |   5 +-
+ drivers/tty/serial/imx.c                        |  43 +++---
+ drivers/tty/serial/msm_serial.c                 |  23 +++
+ drivers/tty/serial/mvebu-uart.c                 |  64 +++++++++
+ drivers/tty/serial/qcom_geni_serial.c           | 177 ++++++++++++++++--------
+ drivers/tty/serial/samsung.c                    |  40 ++++--
+ drivers/tty/serial/sc16is7xx.c                  |   5 +-
+ drivers/tty/serial/serial_core.c                |   4 +-
+ drivers/tty/serial/sh-sci.c                     |  81 +++++++----
+ drivers/tty/serial/sh-sci.h                     |   4 +
+ drivers/tty/serial/st-asc.c                     |   6 +-
+ drivers/tty/serial/xilinx_uartps.c              | 108 ++++++---------
+ drivers/tty/vt/vt.c                             |  16 +--
+ include/linux/serial_8250.h                     |   1 +
+ include/linux/serial_core.h                     |   4 +-
+ include/uapi/linux/tty_flags.h                  |   2 +-
+ 31 files changed, 694 insertions(+), 297 deletions(-)
+Merging char-misc/char-misc-next (d6edc07cca93 MAINTAINERS: Add driver-api/fpga path)
+$ git merge char-misc/char-misc-next
+Auto-merging include/sound/soc-dai.h
+Auto-merging drivers/amba/bus.c
+Auto-merging MAINTAINERS
+Auto-merging Documentation/ioctl/ioctl-number.txt
+Removing Documentation/fpga/overview.txt
+Removing Documentation/fpga/fpga-region.txt
+Removing Documentation/fpga/fpga-mgr.txt
+Auto-merging Documentation/driver-api/index.rst
+Merge made by the 'recursive' strategy.
+ Documentation/ABI/stable/sysfs-bus-vmbus           |   40 +-
+ .../bindings/fpga/lattice-machxo2-spi.txt          |   29 +
+ .../bindings/nvmem/zii,rave-sp-eeprom.txt          |   40 +
+ Documentation/driver-api/fpga/fpga-bridge.rst      |   49 +
+ Documentation/driver-api/fpga/fpga-mgr.rst         |  220 ++
+ Documentation/driver-api/fpga/fpga-region.rst      |  102 +
+ Documentation/driver-api/fpga/index.rst            |   13 +
+ Documentation/driver-api/fpga/intro.rst            |   54 +
+ Documentation/driver-api/index.rst                 |    1 +
+ .../driver-api/soundwire/error_handling.rst        |   65 +
+ Documentation/driver-api/soundwire/index.rst       |    3 +
+ Documentation/driver-api/soundwire/locking.rst     |  106 +
+ Documentation/driver-api/soundwire/stream.rst      |  372 +++
+ Documentation/fpga/fpga-mgr.txt                    |  199 --
+ Documentation/fpga/fpga-region.txt                 |   95 -
+ Documentation/fpga/overview.txt                    |   23 -
+ Documentation/ioctl/ioctl-number.txt               |    1 +
+ Documentation/misc-devices/ibmvmc.rst              |  226 ++
+ Documentation/trace/coresight-cpu-debug.txt        |    4 +-
+ MAINTAINERS                                        |    7 +
+ arch/powerpc/include/asm/hvcall.h                  |    1 +
+ drivers/amba/bus.c                                 |    4 +-
+ drivers/android/Kconfig                            |   15 +-
+ drivers/android/binder.c                           |   17 +-
+ drivers/android/binder_alloc.c                     |    6 +-
+ drivers/char/mspec.c                               |   11 +-
+ drivers/firmware/google/Kconfig                    |    8 +
+ drivers/firmware/google/Makefile                   |    1 +
+ drivers/firmware/google/coreboot_table-acpi.c      |    2 +-
+ drivers/firmware/google/coreboot_table-of.c        |    2 +-
+ drivers/firmware/google/coreboot_table.c           |  130 +-
+ drivers/firmware/google/coreboot_table.h           |   72 +-
+ drivers/firmware/google/framebuffer-coreboot.c     |  115 +
+ drivers/firmware/google/memconsole-coreboot.c      |   49 +-
+ drivers/firmware/google/vpd.c                      |   43 +-
+ drivers/fpga/Kconfig                               |    8 +-
+ drivers/fpga/Makefile                              |    1 +
+ drivers/fpga/altera-cvp.c                          |   19 +-
+ drivers/fpga/altera-fpga2sdram.c                   |   34 +-
+ drivers/fpga/altera-freeze-bridge.c                |   35 +-
+ drivers/fpga/altera-hps2fpga.c                     |   37 +-
+ drivers/fpga/altera-pr-ip-core-plat.c              |   13 +-
+ drivers/fpga/altera-pr-ip-core.c                   |   31 +-
+ drivers/fpga/altera-ps-spi.c                       |   20 +-
+ drivers/fpga/fpga-bridge.c                         |   86 +-
+ drivers/fpga/fpga-mgr.c                            |  129 +-
+ drivers/fpga/fpga-region.c                         |   91 +-
+ drivers/fpga/ice40-spi.c                           |   21 +-
+ drivers/fpga/machxo2-spi.c                         |  415 ++++
+ drivers/fpga/of-fpga-region.c                      |   28 +-
+ drivers/fpga/socfpga-a10.c                         |   28 +-
+ drivers/fpga/socfpga.c                             |   32 +-
+ drivers/fpga/ts73xx-fpga.c                         |   20 +-
+ drivers/fpga/xilinx-pr-decoupler.c                 |   22 +-
+ drivers/fpga/xilinx-spi.c                          |   20 +-
+ drivers/fpga/zynq-fpga.c                           |   14 +-
+ drivers/hv/connection.c                            |   44 +-
+ drivers/hv/hyperv_vmbus.h                          |    3 +
+ drivers/hwtracing/coresight/coresight-cpu-debug.c  |   16 +-
+ .../coresight/coresight-dynamic-replicator.c       |   10 +-
+ drivers/hwtracing/coresight/coresight-etb10.c      |   13 +-
+ drivers/hwtracing/coresight/coresight-etm-cp14.c   |   13 +-
+ drivers/hwtracing/coresight/coresight-etm-perf.c   |   13 +-
+ drivers/hwtracing/coresight/coresight-etm-perf.h   |   13 +-
+ drivers/hwtracing/coresight/coresight-etm.h        |   13 +-
+ .../hwtracing/coresight/coresight-etm3x-sysfs.c    |   13 +-
+ drivers/hwtracing/coresight/coresight-etm3x.c      |   13 +-
+ .../hwtracing/coresight/coresight-etm4x-sysfs.c    |   13 +-
+ drivers/hwtracing/coresight/coresight-etm4x.c      |   13 +-
+ drivers/hwtracing/coresight/coresight-etm4x.h      |   13 +-
+ drivers/hwtracing/coresight/coresight-funnel.c     |   13 +-
+ drivers/hwtracing/coresight/coresight-priv.h       |   13 +-
+ drivers/hwtracing/coresight/coresight-replicator.c |   13 +-
+ drivers/hwtracing/coresight/coresight-stm.c        |   13 +-
+ drivers/hwtracing/coresight/coresight-tmc-etf.c    |   13 +-
+ drivers/hwtracing/coresight/coresight-tmc-etr.c    |   20 +-
+ drivers/hwtracing/coresight/coresight-tmc.c        |   10 +-
+ drivers/hwtracing/coresight/coresight-tmc.h        |   13 +-
+ drivers/hwtracing/coresight/coresight-tpiu.c       |   13 +-
+ drivers/hwtracing/coresight/coresight.c            |   21 +-
+ drivers/hwtracing/coresight/of_coresight.c         |   13 +-
+ drivers/hwtracing/stm/ftrace.c                     |   10 +-
+ drivers/misc/Kconfig                               |   14 +
+ drivers/misc/Makefile                              |    1 +
+ drivers/misc/cxl/context.c                         |    7 +-
+ drivers/misc/ibmvmc.c                              | 2418 ++++++++++++++++++++
+ drivers/misc/ibmvmc.h                              |  209 ++
+ drivers/misc/sgi-gru/grumain.c                     |    2 +-
+ drivers/misc/sgi-gru/grutables.h                   |    3 +-
+ drivers/misc/sgi-xp/xpnet.c                        |    2 +-
+ drivers/misc/ti-st/st_kim.c                        |    2 +-
+ drivers/misc/tifm_7xx1.c                           |    6 +-
+ drivers/mux/adg792a.c                              |    5 +-
+ drivers/nubus/bus.c                                |   19 +-
+ drivers/nubus/nubus.c                              |    2 +-
+ drivers/nvmem/Kconfig                              |    6 +
+ drivers/nvmem/Makefile                             |    3 +
+ drivers/nvmem/core.c                               |   33 +-
+ drivers/nvmem/meson-efuse.c                        |   41 +-
+ drivers/nvmem/rave-sp-eeprom.c                     |  357 +++
+ drivers/slimbus/qcom-ctrl.c                        |    3 +-
+ drivers/soundwire/Kconfig                          |    2 +-
+ drivers/soundwire/Makefile                         |    2 +-
+ drivers/soundwire/bus.c                            |   43 +
+ drivers/soundwire/bus.h                            |   72 +
+ drivers/soundwire/cadence_master.c                 |  451 +++-
+ drivers/soundwire/cadence_master.h                 |  151 ++
+ drivers/soundwire/intel.c                          |  525 ++++-
+ drivers/soundwire/intel.h                          |    4 +
+ drivers/soundwire/intel_init.c                     |    3 +
+ drivers/soundwire/stream.c                         | 1479 ++++++++++++
+ drivers/uio/uio.c                                  |  123 +-
+ drivers/uio/uio_fsl_elbc_gpcm.c                    |    1 -
+ drivers/virt/vboxguest/vboxguest_linux.c           |    4 +-
+ drivers/w1/masters/mxc_w1.c                        |   20 +-
+ drivers/w1/w1.c                                    |    2 +-
+ include/linux/coresight.h                          |   13 +-
+ include/linux/fpga/altera-pr-ip-core.h             |   13 +-
+ include/linux/fpga/fpga-bridge.h                   |    9 +-
+ include/linux/fpga/fpga-mgr.h                      |   23 +-
+ include/linux/fpga/fpga-region.h                   |   13 +-
+ include/linux/hyperv.h                             |   26 +-
+ include/linux/nubus.h                              |    2 +-
+ include/linux/nvmem-provider.h                     |   11 +
+ include/linux/soundwire/sdw.h                      |  332 ++-
+ include/linux/soundwire/sdw_intel.h                |   14 +
+ include/linux/uio_driver.h                         |    4 +-
+ include/sound/soc-dai.h                            |   23 +
+ scripts/mod/devicetable-offsets.c                  |    3 +
+ scripts/mod/file2alias.c                           |   11 +
+ scripts/ver_linux                                  |   81 +-
+ 131 files changed, 8973 insertions(+), 1217 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/fpga/lattice-machxo2-spi.txt
+ create mode 100644 Documentation/devicetree/bindings/nvmem/zii,rave-sp-eeprom.txt
+ create mode 100644 Documentation/driver-api/fpga/fpga-bridge.rst
+ create mode 100644 Documentation/driver-api/fpga/fpga-mgr.rst
+ create mode 100644 Documentation/driver-api/fpga/fpga-region.rst
+ create mode 100644 Documentation/driver-api/fpga/index.rst
+ create mode 100644 Documentation/driver-api/fpga/intro.rst
+ create mode 100644 Documentation/driver-api/soundwire/error_handling.rst
+ create mode 100644 Documentation/driver-api/soundwire/locking.rst
+ create mode 100644 Documentation/driver-api/soundwire/stream.rst
+ delete mode 100644 Documentation/fpga/fpga-mgr.txt
+ delete mode 100644 Documentation/fpga/fpga-region.txt
+ delete mode 100644 Documentation/fpga/overview.txt
+ create mode 100644 Documentation/misc-devices/ibmvmc.rst
+ create mode 100644 drivers/firmware/google/framebuffer-coreboot.c
+ create mode 100644 drivers/fpga/machxo2-spi.c
+ create mode 100644 drivers/misc/ibmvmc.c
+ create mode 100644 drivers/misc/ibmvmc.h
+ create mode 100644 drivers/nvmem/rave-sp-eeprom.c
+ create mode 100644 drivers/soundwire/stream.c
+Merging extcon/extcon-next (60cc43fc8884 Linux 4.17-rc1)
+$ git merge extcon/extcon-next
+Already up to date.
+Merging staging/staging-next (7a5abc3d9699 staging: lustre: fix more build errors in errno.c)
+$ git merge staging/staging-next
+Auto-merging include/linux/kernel.h
+Auto-merging include/linux/iio/common/cros_ec_sensors_core.h
+Auto-merging drivers/staging/rtl8723bs/include/hal_phy_reg_8723b.h
+Auto-merging drivers/staging/rtl8723bs/include/hal_phy_cfg.h
+Auto-merging drivers/staging/rtl8723bs/hal/hal_pwr_seq.c
+Auto-merging drivers/staging/rtl8192u/r8192U_core.c
+Removing drivers/staging/most/dim2/dim2.h
+CONFLICT (modify/delete): drivers/staging/media/atomisp/TODO deleted in HEAD and modified in staging/staging-next. Version staging/staging-next of drivers/staging/media/atomisp/TODO left in tree.
+Removing drivers/staging/lustre/lnet/libcfs/linux/linux-module.c
+Removing drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
+Auto-merging drivers/staging/lustre/lnet/libcfs/linux-tracefile.c
+Auto-merging drivers/staging/lustre/lnet/libcfs/linux-debug.c
+Auto-merging drivers/staging/lustre/lnet/libcfs/linux-crypto.c
+Removing drivers/staging/lustre/include/linux/libcfs/linux/linux-time.h
+Removing drivers/staging/lustre/include/linux/libcfs/linux/linux-cpu.h
+Removing drivers/staging/lustre/include/linux/libcfs/linux/libcfs.h
+Removing drivers/staging/lustre/include/linux/libcfs/libcfs_time.h
+Removing drivers/staging/lustre/include/linux/libcfs/libcfs_prim.h
+Removing drivers/staging/lustre/include/linux/libcfs/curproc.h
+Removing drivers/staging/ks7010/ks7010_sdio.h
+Removing drivers/staging/iio/meter/ade7759.c
+Removing drivers/staging/iio/meter/ade7758_trigger.c
+Removing drivers/staging/iio/meter/ade7758_ring.c
+Removing drivers/staging/iio/meter/ade7758_core.c
+Removing drivers/staging/iio/meter/ade7758.h
+Removing drivers/staging/iio/meter/ade7754.c
+Removing drivers/staging/iio/meter/ade7753.c
+Removing drivers/staging/iio/light/tsl2x7x.h
+Removing drivers/staging/iio/light/tsl2x7x.c
+Removing drivers/staging/iio/light/Makefile
+Removing drivers/staging/iio/light/Kconfig
+Removing drivers/staging/iio/accel/adis16201.c
+Removing drivers/staging/goldfish/goldfish_nand_reg.h
+Removing drivers/staging/goldfish/goldfish_nand.c
+Auto-merging drivers/iio/resolver/ad2s1200.c
+Auto-merging drivers/iio/common/hid-sensors/hid-sensor-trigger.c
+Auto-merging drivers/iio/adc/stm32-dfsdm-adc.c
+Auto-merging drivers/iio/accel/adis16209.c
+Auto-merging MAINTAINERS
+Automatic merge failed; fix conflicts and then commit the result.
+$ git rm -f drivers/staging/media/atomisp/TODO
+drivers/staging/media/atomisp/TODO: needs merge
+rm 'drivers/staging/media/atomisp/TODO'
+$ git commit -v -a
+[master 376a457fb0af] Merge remote-tracking branch 'staging/staging-next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/ABI/testing/sysfs-bus-iio            |    9 +
+ .../bindings/iio/adc/amlogic,meson-saradc.txt      |    1 +
+ .../devicetree/bindings/iio/adc/mcp320x.txt        |    2 +-
+ .../devicetree/bindings/iio/adc/st,stm32-adc.txt   |    6 +-
+ .../bindings/iio/adc/st,stm32-dfsdm-adc.txt        |    7 +-
+ .../bindings/iio/afe/current-sense-amplifier.txt   |   26 +
+ .../bindings/iio/afe/current-sense-shunt.txt       |   41 +
+ .../bindings/iio/afe/voltage-divider.txt           |   53 +
+ .../devicetree/bindings/iio/dac/ltc2632.txt        |   14 +
+ .../devicetree/bindings/iio/dac/ti,dac5571.txt     |   24 +
+ .../devicetree/bindings/iio/imu/inv_mpu6050.txt    |   12 +-
+ .../devicetree/bindings/iio/imu/st_lsm6dsx.txt     |    1 +
+ .../bindings/iio/potentiostat/lmp91000.txt         |    9 +-
+ MAINTAINERS                                        |   23 +
+ drivers/iio/Kconfig                                |    2 +
+ drivers/iio/Makefile                               |    2 +
+ drivers/iio/accel/Kconfig                          |   24 +
+ drivers/iio/accel/Makefile                         |    2 +
+ drivers/iio/accel/adis16201.c                      |  321 ++
+ drivers/{staging => }/iio/accel/adis16209.c        |   52 +-
+ drivers/iio/accel/bmc150-accel-core.c              |   23 +-
+ drivers/iio/accel/mma8452.c                        |   24 +-
+ drivers/iio/accel/sca3000.c                        |   13 +-
+ drivers/iio/accel/st_accel_i2c.c                   |    1 +
+ drivers/iio/adc/ad7791.c                           |   87 +-
+ drivers/iio/adc/imx7d_adc.c                        |    2 +-
+ drivers/iio/adc/meson_saradc.c                     |   83 +-
+ drivers/iio/adc/stm32-adc-core.c                   |   66 +-
+ drivers/iio/adc/stm32-adc.c                        |   47 +-
+ drivers/iio/adc/stm32-dfsdm-adc.c                  |   17 +-
+ drivers/iio/adc/stm32-dfsdm-core.c                 |   26 +
+ drivers/iio/adc/stx104.c                           |   11 +
+ drivers/iio/adc/ti-ads8688.c                       |   48 +-
+ drivers/iio/afe/Kconfig                            |   19 +
+ drivers/iio/afe/Makefile                           |    6 +
+ drivers/iio/afe/iio-rescale.c                      |  359 ++
+ drivers/iio/amplifiers/ad8366.c                    |    6 +-
+ drivers/iio/chemical/atlas-ph-sensor.c             |   14 +-
+ .../iio/common/cros_ec_sensors/cros_ec_sensors.c   |    3 +-
+ .../common/cros_ec_sensors/cros_ec_sensors_core.c  |    9 +-
+ .../iio/common/hid-sensors/hid-sensor-trigger.c    |    9 +-
+ drivers/iio/dac/Kconfig                            |   33 +-
+ drivers/iio/dac/Makefile                           |    3 +
+ drivers/iio/dac/ad5686-spi.c                       |  119 +
+ drivers/iio/dac/ad5686.c                           |  428 +-
+ drivers/iio/dac/ad5686.h                           |  145 +
+ drivers/iio/dac/ad5696-i2c.c                       |  102 +
+ drivers/iio/dac/ltc2632.c                          |   89 +-
+ drivers/iio/dac/ti-dac5571.c                       |  439 ++
+ drivers/iio/imu/inv_mpu6050/inv_mpu_core.c         |  284 +-
+ drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c          |   86 +-
+ drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h          |   22 +-
+ drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c         |   47 +-
+ drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c          |   12 +-
+ drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c      |   52 +-
+ drivers/iio/imu/st_lsm6dsx/Kconfig                 |    3 +-
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h            |    2 +
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c     |   10 +-
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c       |    7 +-
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c        |    5 +
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c        |    5 +
+ drivers/iio/light/Kconfig                          |    8 +
+ drivers/iio/light/Makefile                         |    1 +
+ drivers/iio/light/cros_ec_light_prox.c             |    3 +-
+ drivers/iio/light/tsl2583.c                        |    2 +-
+ drivers/iio/light/tsl2772.c                        | 1800 ++++++++
+ drivers/iio/magnetometer/mag3110.c                 |  158 +-
+ drivers/iio/potentiometer/mcp4018.c                |   41 +-
+ drivers/iio/potentiometer/mcp4531.c                |  145 +-
+ drivers/iio/potentiostat/lmp91000.c                |    2 +
+ drivers/iio/pressure/cros_ec_baro.c                |    3 +-
+ drivers/iio/resolver/Kconfig                       |   17 +
+ drivers/iio/resolver/Makefile                      |    5 +
+ drivers/{staging => }/iio/resolver/ad2s1200.c      |  167 +-
+ drivers/staging/android/Kconfig                    |   11 +-
+ drivers/staging/android/Makefile                   |    1 +
+ drivers/staging/android/TODO                       |    9 +
+ drivers/staging/android/ashmem.h                   |    6 +-
+ drivers/staging/android/ion/Kconfig                |    2 +-
+ drivers/staging/android/ion/ion.c                  |   22 +-
+ drivers/staging/android/ion/ion.h                  |    2 +-
+ drivers/staging/android/ion/ion_page_pool.c        |    5 +
+ drivers/staging/android/uapi/ashmem.h              |    6 +-
+ drivers/staging/android/uapi/ion.h                 |    2 +-
+ drivers/staging/android/uapi/vsoc_shm.h            |  295 ++
+ drivers/staging/android/vsoc.c                     | 1152 +++++
+ drivers/staging/board/armadillo800eva.c            |   10 +-
+ drivers/staging/board/board.c                      |    5 +-
+ drivers/staging/comedi/comedi_compat32.c           |   12 +-
+ drivers/staging/comedi/comedidev.h                 |    2 +-
+ drivers/staging/comedi/drivers/cb_pcidas64.c       |    6 +-
+ drivers/staging/emxx_udc/Kconfig                   |    2 +-
+ drivers/staging/emxx_udc/TODO                      |    2 +
+ drivers/staging/fbtft/TODO                         |    4 +
+ drivers/staging/fbtft/fbtft-bus.c                  |   13 +-
+ drivers/staging/fsl-dpaa2/Kconfig                  |    8 +
+ drivers/staging/fsl-dpaa2/Makefile                 |    5 +-
+ drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c     |  111 +-
+ drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.h     |   49 +-
+ drivers/staging/fsl-dpaa2/ethernet/dpaa2-ethtool.c |   23 +
+ drivers/staging/fsl-dpaa2/ethsw/dpsw-cmd.h         |   13 +
+ drivers/staging/fsl-dpaa2/ethsw/dpsw.c             |   42 +
+ drivers/staging/fsl-dpaa2/ethsw/dpsw.h             |    6 +
+ drivers/staging/fsl-dpaa2/ethsw/ethsw.c            |   39 +-
+ drivers/staging/fsl-dpaa2/rtc/Makefile             |    7 +
+ drivers/staging/fsl-dpaa2/rtc/dprtc-cmd.h          |  137 +
+ drivers/staging/fsl-dpaa2/rtc/dprtc.c              |  701 +++
+ drivers/staging/fsl-dpaa2/rtc/dprtc.h              |  164 +
+ drivers/staging/fsl-dpaa2/rtc/rtc.c                |  229 +
+ drivers/staging/fsl-dpaa2/rtc/rtc.h                |   14 +
+ drivers/staging/fsl-mc/bus/dpio/dpio-driver.c      |   28 +-
+ drivers/staging/fsl-mc/bus/dpio/qbman-portal.h     |    4 +-
+ drivers/staging/fsl-mc/include/dpaa2-io.h          |    2 +-
+ drivers/staging/gdm724x/gdm_endian.c               |   14 +-
+ drivers/staging/gdm724x/gdm_endian.h               |   14 +-
+ drivers/staging/gdm724x/gdm_lte.c                  |   16 +-
+ drivers/staging/gdm724x/gdm_lte.h                  |   14 +-
+ drivers/staging/gdm724x/gdm_mux.c                  |   14 +-
+ drivers/staging/gdm724x/gdm_mux.h                  |   14 +-
+ drivers/staging/gdm724x/gdm_tty.c                  |   14 +-
+ drivers/staging/gdm724x/gdm_tty.h                  |   14 +-
+ drivers/staging/gdm724x/gdm_usb.c                  |   14 +-
+ drivers/staging/gdm724x/gdm_usb.h                  |   14 +-
+ drivers/staging/gdm724x/hci.h                      |   14 +-
+ drivers/staging/gdm724x/hci_packet.h               |   14 +-
+ drivers/staging/gdm724x/netlink_k.c                |   14 +-
+ drivers/staging/gdm724x/netlink_k.h                |   14 +-
+ drivers/staging/goldfish/Kconfig                   |    9 +-
+ drivers/staging/goldfish/Makefile                  |    1 -
+ drivers/staging/goldfish/goldfish_nand.c           |  441 --
+ drivers/staging/goldfish/goldfish_nand_reg.h       |   76 -
+ drivers/staging/greybus/TODO                       |    5 +
+ drivers/staging/greybus/arche-platform.c           |    9 +-
+ drivers/staging/greybus/audio_codec.h              |    5 +-
+ drivers/staging/greybus/audio_topology.c           |    2 +-
+ drivers/staging/greybus/svc.c                      |    3 -
+ drivers/staging/iio/Kconfig                        |    1 -
+ drivers/staging/iio/Makefile                       |    1 -
+ drivers/staging/iio/TODO                           |    9 +-
+ drivers/staging/iio/accel/Kconfig                  |   24 -
+ drivers/staging/iio/accel/Makefile                 |    2 -
+ drivers/staging/iio/accel/adis16201.c              |  385 --
+ drivers/staging/iio/adc/ad7606_par.c               |    6 +-
+ drivers/staging/iio/adc/ad7780.c                   |    2 +-
+ drivers/staging/iio/cdc/ad7746.c                   |   44 +-
+ drivers/staging/iio/light/Kconfig                  |   14 -
+ drivers/staging/iio/light/Makefile                 |    5 -
+ drivers/staging/iio/light/tsl2x7x.c                | 1889 ---------
+ drivers/staging/iio/light/tsl2x7x.h                |   99 -
+ drivers/staging/iio/meter/Kconfig                  |   42 -
+ drivers/staging/iio/meter/Makefile                 |    8 -
+ drivers/staging/iio/meter/ade7753.c                |  630 ---
+ drivers/staging/iio/meter/ade7754.c                |  664 ---
+ drivers/staging/iio/meter/ade7758.h                |  183 -
+ drivers/staging/iio/meter/ade7758_core.c           |  955 -----
+ drivers/staging/iio/meter/ade7758_ring.c           |  177 -
+ drivers/staging/iio/meter/ade7758_trigger.c        |  108 -
+ drivers/staging/iio/meter/ade7759.c                |  558 ---
+ drivers/staging/iio/meter/ade7854-i2c.c            |  238 +-
+ drivers/staging/iio/meter/ade7854-spi.c            |  268 +-
+ drivers/staging/iio/meter/ade7854.c                |   40 +-
+ drivers/staging/iio/meter/ade7854.h                |   16 +-
+ drivers/staging/iio/resolver/Kconfig               |   12 -
+ drivers/staging/iio/resolver/Makefile              |    1 -
+ drivers/staging/ks7010/eap_packet.h                |   76 +-
+ drivers/staging/ks7010/ks7010_sdio.c               |  632 +--
+ drivers/staging/ks7010/ks7010_sdio.h               |  164 -
+ drivers/staging/ks7010/ks_hostif.c                 | 1043 ++---
+ drivers/staging/ks7010/ks_hostif.h                 |  465 +-
+ drivers/staging/ks7010/ks_wlan.h                   |  278 +-
+ drivers/staging/ks7010/ks_wlan_ioctl.h             |    7 +-
+ drivers/staging/ks7010/ks_wlan_net.c               | 1118 ++---
+ drivers/staging/ks7010/michael_mic.c               |   28 +-
+ drivers/staging/ks7010/michael_mic.h               |   11 +-
+ drivers/staging/lustre/README.txt                  |    2 +-
+ drivers/staging/lustre/TODO                        |    2 +-
+ .../staging/lustre/include/linux/libcfs/curproc.h  |   85 -
+ .../staging/lustre/include/linux/libcfs/libcfs.h   |  103 +-
+ .../lustre/include/linux/libcfs/libcfs_cpu.h       |  217 +-
+ .../lustre/include/linux/libcfs/libcfs_crypto.h    |    3 +
+ .../lustre/include/linux/libcfs/libcfs_debug.h     |   40 +
+ .../lustre/include/linux/libcfs/libcfs_fail.h      |    9 +-
+ .../lustre/include/linux/libcfs/libcfs_hash.h      |    3 +
+ .../lustre/include/linux/libcfs/libcfs_prim.h      |   82 -
+ .../lustre/include/linux/libcfs/libcfs_string.h    |    2 +
+ .../lustre/include/linux/libcfs/libcfs_time.h      |   79 -
+ .../lustre/include/linux/libcfs/linux/libcfs.h     |  133 -
+ .../lustre/include/linux/libcfs/linux/linux-cpu.h  |   78 -
+ .../lustre/include/linux/libcfs/linux/linux-time.h |  103 -
+ .../staging/lustre/include/linux/lnet/lib-lnet.h   |    8 +
+ .../lustre/include/uapi/linux/lnet/lnet-dlc.h      |    3 +-
+ .../lustre/include/uapi/linux/lnet/lnetctl.h       |   11 -
+ .../include/uapi/linux/lustre/lustre_fiemap.h      |    2 +-
+ .../lustre/include/uapi/linux/lustre/lustre_idl.h  |    6 +-
+ .../include/uapi/linux/lustre/lustre_ioctl.h       |    2 -
+ .../lustre/include/uapi/linux/lustre/lustre_user.h |    6 +-
+ .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c    |   40 +-
+ .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h    |   24 +-
+ .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c |   77 +-
+ .../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c  |    9 +
+ .../staging/lustre/lnet/klnds/socklnd/socklnd.c    |   19 +-
+ .../staging/lustre/lnet/klnds/socklnd/socklnd.h    |    3 +-
+ .../staging/lustre/lnet/klnds/socklnd/socklnd_cb.c |   98 +-
+ .../lustre/lnet/klnds/socklnd/socklnd_lib.c        |    1 +
+ drivers/staging/lustre/lnet/libcfs/Makefile        |   19 +-
+ drivers/staging/lustre/lnet/libcfs/debug.c         |    9 +-
+ drivers/staging/lustre/lnet/libcfs/fail.c          |    4 +
+ drivers/staging/lustre/lnet/libcfs/hash.c          |    5 +-
+ drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c    |  942 ++++-
+ drivers/staging/lustre/lnet/libcfs/libcfs_lock.c   |    3 +
+ drivers/staging/lustre/lnet/libcfs/libcfs_mem.c    |    6 +-
+ drivers/staging/lustre/lnet/libcfs/libcfs_string.c |    6 +
+ .../lnet/libcfs/{linux => }/linux-crypto-adler.c   |    0
+ .../lustre/lnet/libcfs/{linux => }/linux-crypto.c  |   10 +-
+ .../lustre/lnet/libcfs/{linux => }/linux-crypto.h  |    0
+ .../lustre/lnet/libcfs/{linux => }/linux-debug.c   |    5 +-
+ .../lnet/libcfs/{linux => }/linux-tracefile.c      |    5 +-
+ .../staging/lustre/lnet/libcfs/linux/linux-cpu.c   | 1079 -----
+ .../lustre/lnet/libcfs/linux/linux-module.c        |  197 -
+ drivers/staging/lustre/lnet/libcfs/module.c        |  296 +-
+ drivers/staging/lustre/lnet/libcfs/tracefile.c     |   27 +-
+ drivers/staging/lustre/lnet/libcfs/tracefile.h     |   11 +
+ drivers/staging/lustre/lnet/lnet/api-ni.c          |    6 +-
+ drivers/staging/lustre/lnet/lnet/config.c          |    1 +
+ drivers/staging/lustre/lnet/lnet/lib-eq.c          |    2 +-
+ drivers/staging/lustre/lnet/lnet/lib-move.c        |   16 +-
+ drivers/staging/lustre/lnet/lnet/lib-socket.c      |    1 -
+ drivers/staging/lustre/lnet/lnet/module.c          |   42 +-
+ drivers/staging/lustre/lnet/lnet/net_fault.c       |   40 +-
+ drivers/staging/lustre/lnet/lnet/nidstrings.c      |    3 +
+ drivers/staging/lustre/lnet/lnet/peer.c            |    2 +-
+ drivers/staging/lustre/lnet/lnet/router.c          |   17 +-
+ drivers/staging/lustre/lnet/lnet/router_proc.c     |   16 +-
+ drivers/staging/lustre/lnet/selftest/conctl.c      |   28 +-
+ drivers/staging/lustre/lnet/selftest/conrpc.c      |   15 +-
+ drivers/staging/lustre/lnet/selftest/conrpc.h      |    1 -
+ drivers/staging/lustre/lnet/selftest/console.c     |   15 +-
+ drivers/staging/lustre/lnet/selftest/console.h     |    4 +-
+ drivers/staging/lustre/lnet/selftest/framework.c   |    4 +-
+ drivers/staging/lustre/lnet/selftest/module.c      |    6 +-
+ drivers/staging/lustre/lnet/selftest/selftest.h    |    1 -
+ drivers/staging/lustre/lnet/selftest/timer.c       |    4 +-
+ drivers/staging/lustre/lustre/fid/fid_internal.h   |    1 -
+ drivers/staging/lustre/lustre/fid/fid_lib.c        |    1 -
+ drivers/staging/lustre/lustre/fid/fid_request.c    |    7 +-
+ drivers/staging/lustre/lustre/fid/lproc_fid.c      |    1 -
+ drivers/staging/lustre/lustre/fld/fld_cache.c      |    1 -
+ drivers/staging/lustre/lustre/fld/fld_internal.h   |    1 -
+ drivers/staging/lustre/lustre/fld/fld_request.c    |    7 +-
+ drivers/staging/lustre/lustre/fld/lproc_fld.c      |    1 -
+ drivers/staging/lustre/lustre/include/cl_object.h  |   10 +-
+ .../staging/lustre/lustre/include/lprocfs_status.h |    2 +-
+ drivers/staging/lustre/lustre/include/lu_object.h  |   42 +-
+ .../staging/lustre/lustre/include/lustre_compat.h  |    1 +
+ .../staging/lustre/lustre/include/lustre_disk.h    |    1 -
+ drivers/staging/lustre/lustre/include/lustre_dlm.h |   18 +-
+ .../staging/lustre/lustre/include/lustre_export.h  |    9 +-
+ drivers/staging/lustre/lustre/include/lustre_fld.h |    1 -
+ .../staging/lustre/lustre/include/lustre_handles.h |    2 -
+ .../staging/lustre/lustre/include/lustre_import.h  |    1 +
+ .../staging/lustre/lustre/include/lustre_intent.h  |    2 +
+ drivers/staging/lustre/lustre/include/lustre_lib.h |   22 +-
+ drivers/staging/lustre/lustre/include/lustre_mds.h |    1 -
+ drivers/staging/lustre/lustre/include/lustre_net.h |   11 +-
+ .../lustre/include/lustre_patchless_compat.h       |    1 +
+ drivers/staging/lustre/lustre/include/lustre_sec.h |    2 +
+ drivers/staging/lustre/lustre/include/obd.h        |   28 +-
+ drivers/staging/lustre/lustre/include/obd_class.h  |   15 +-
+ .../staging/lustre/lustre/include/obd_support.h    |   28 -
+ drivers/staging/lustre/lustre/ldlm/l_lock.c        |    1 -
+ drivers/staging/lustre/lustre/ldlm/ldlm_extent.c   |    1 -
+ drivers/staging/lustre/lustre/ldlm/ldlm_flock.c    |   11 +-
+ drivers/staging/lustre/lustre/ldlm/ldlm_lib.c      |    1 -
+ drivers/staging/lustre/lustre/ldlm/ldlm_lock.c     |   25 +-
+ drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c    |   16 +-
+ drivers/staging/lustre/lustre/ldlm/ldlm_request.c  |   67 +-
+ drivers/staging/lustre/lustre/ldlm/ldlm_resource.c |   31 +-
+ drivers/staging/lustre/lustre/llite/dir.c          |    4 +-
+ drivers/staging/lustre/lustre/llite/file.c         |   24 +-
+ drivers/staging/lustre/lustre/llite/glimpse.c      |    1 -
+ drivers/staging/lustre/lustre/llite/lcommon_cl.c   |    9 +-
+ .../staging/lustre/lustre/llite/llite_internal.h   |    4 +-
+ drivers/staging/lustre/lustre/llite/llite_lib.c    |   11 +-
+ drivers/staging/lustre/lustre/llite/llite_mmap.c   |   14 +-
+ drivers/staging/lustre/lustre/llite/llite_nfs.c    |    2 +-
+ drivers/staging/lustre/lustre/llite/lproc_llite.c  |   12 +-
+ drivers/staging/lustre/lustre/llite/namei.c        |   11 +-
+ drivers/staging/lustre/lustre/llite/range_lock.c   |    1 +
+ drivers/staging/lustre/lustre/llite/range_lock.h   |    2 +-
+ drivers/staging/lustre/lustre/llite/statahead.c    |   10 +-
+ drivers/staging/lustre/lustre/llite/super25.c      |    4 +
+ drivers/staging/lustre/lustre/llite/vvp_dev.c      |  167 +-
+ drivers/staging/lustre/lustre/llite/vvp_internal.h |    8 +-
+ drivers/staging/lustre/lustre/llite/vvp_object.c   |    2 -
+ drivers/staging/lustre/lustre/llite/xattr.c        |  227 +-
+ drivers/staging/lustre/lustre/llite/xattr_cache.c  |   83 +-
+ drivers/staging/lustre/lustre/lmv/lmv_intent.c     |   12 +-
+ drivers/staging/lustre/lustre/lmv/lmv_internal.h   |    2 +-
+ drivers/staging/lustre/lustre/lmv/lmv_obd.c        |   31 +-
+ .../staging/lustre/lustre/lov/lov_cl_internal.h    |   20 +-
+ drivers/staging/lustre/lustre/lov/lov_ea.c         |    1 -
+ drivers/staging/lustre/lustre/lov/lov_internal.h   |   13 +-
+ drivers/staging/lustre/lustre/lov/lov_merge.c      |    4 +-
+ drivers/staging/lustre/lustre/lov/lov_obd.c        |   25 +-
+ drivers/staging/lustre/lustre/lov/lov_object.c     |    8 +-
+ drivers/staging/lustre/lustre/lov/lov_offset.c     |    2 -
+ drivers/staging/lustre/lustre/lov/lov_pool.c       |  161 +-
+ drivers/staging/lustre/lustre/lov/lov_request.c    |    6 +-
+ drivers/staging/lustre/lustre/mdc/mdc_internal.h   |    8 +-
+ drivers/staging/lustre/lustre/mdc/mdc_lib.c        |   17 +-
+ drivers/staging/lustre/lustre/mdc/mdc_locks.c      |   68 +-
+ drivers/staging/lustre/lustre/mdc/mdc_reint.c      |    2 +-
+ drivers/staging/lustre/lustre/mdc/mdc_request.c    |   12 +-
+ drivers/staging/lustre/lustre/mgc/mgc_internal.h   |    1 -
+ drivers/staging/lustre/lustre/mgc/mgc_request.c    |    7 +
+ drivers/staging/lustre/lustre/obdclass/cl_io.c     |    1 -
+ drivers/staging/lustre/lustre/obdclass/cl_object.c |    2 -
+ drivers/staging/lustre/lustre/obdclass/cl_page.c   |    1 -
+ drivers/staging/lustre/lustre/obdclass/class_obd.c |   15 +-
+ drivers/staging/lustre/lustre/obdclass/genops.c    |   36 +-
+ .../staging/lustre/lustre/obdclass/kernelcomm.c    |    2 +
+ .../lustre/lustre/obdclass/linux/linux-module.c    |    4 +-
+ drivers/staging/lustre/lustre/obdclass/llog.c      |    3 +-
+ .../lustre/lustre/obdclass/lprocfs_counters.c      |    1 +
+ .../lustre/lustre/obdclass/lprocfs_status.c        |   12 +-
+ drivers/staging/lustre/lustre/obdclass/lu_object.c |  266 +-
+ drivers/staging/lustre/lustre/obdclass/lu_ref.c    |    2 -
+ .../lustre/lustre/obdclass/lustre_handles.c        |    1 +
+ .../staging/lustre/lustre/obdclass/obd_config.c    |  163 +-
+ drivers/staging/lustre/lustre/obdclass/obd_mount.c |    1 +
+ drivers/staging/lustre/lustre/obdclass/uuid.c      |    2 -
+ .../staging/lustre/lustre/obdecho/echo_client.c    |    9 +-
+ drivers/staging/lustre/lustre/osc/osc_cache.c      |    4 +-
+ .../staging/lustre/lustre/osc/osc_cl_internal.h    |   12 +-
+ drivers/staging/lustre/lustre/osc/osc_internal.h   |    7 +-
+ drivers/staging/lustre/lustre/osc/osc_io.c         |    2 +-
+ drivers/staging/lustre/lustre/osc/osc_lock.c       |    1 -
+ drivers/staging/lustre/lustre/osc/osc_object.c     |   11 +-
+ drivers/staging/lustre/lustre/osc/osc_page.c       |    4 +-
+ drivers/staging/lustre/lustre/osc/osc_quota.c      |  136 +-
+ drivers/staging/lustre/lustre/osc/osc_request.c    |   50 +-
+ drivers/staging/lustre/lustre/ptlrpc/client.c      |   20 +-
+ drivers/staging/lustre/lustre/ptlrpc/connection.c  |  164 +-
+ drivers/staging/lustre/lustre/ptlrpc/errno.c       |    4 +-
+ drivers/staging/lustre/lustre/ptlrpc/import.c      |   11 +-
+ drivers/staging/lustre/lustre/ptlrpc/llog_client.c |    2 -
+ drivers/staging/lustre/lustre/ptlrpc/llog_net.c    |    2 -
+ drivers/staging/lustre/lustre/ptlrpc/niobuf.c      |    7 +-
+ drivers/staging/lustre/lustre/ptlrpc/nrs.c         |    3 +-
+ drivers/staging/lustre/lustre/ptlrpc/nrs_fifo.c    |    3 +-
+ .../staging/lustre/lustre/ptlrpc/pack_generic.c    |    2 -
+ drivers/staging/lustre/lustre/ptlrpc/pinger.c      |   23 +-
+ .../staging/lustre/lustre/ptlrpc/ptlrpc_module.c   |    4 +
+ drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c     |   14 +-
+ drivers/staging/lustre/lustre/ptlrpc/sec.c         |    2 +-
+ drivers/staging/lustre/lustre/ptlrpc/service.c     |   21 +-
+ drivers/staging/most/cdev/cdev.c                   |   16 +-
+ drivers/staging/most/core.c                        |   68 +-
+ drivers/staging/most/core.h                        |    4 +
+ drivers/staging/most/dim2/Kconfig                  |    2 +-
+ drivers/staging/most/dim2/dim2.c                   |  397 +-
+ drivers/staging/most/dim2/dim2.h                   |   21 -
+ drivers/staging/most/dim2/hal.c                    |    9 +-
+ drivers/staging/most/dim2/reg.h                    |   84 +-
+ drivers/staging/most/i2c/i2c.c                     |  140 +-
+ drivers/staging/most/sound/sound.c                 |  123 +-
+ drivers/staging/most/usb/usb.c                     |   55 +-
+ drivers/staging/most/video/video.c                 |   32 -
+ drivers/staging/mt7621-dts/gbpc1.dts               |    3 +-
+ drivers/staging/mt7621-dts/mt7621.dtsi             |   20 +-
+ drivers/staging/mt7621-eth/ethtool.c               |   54 +-
+ drivers/staging/mt7621-eth/gsw_mt7620.h            |    4 +-
+ drivers/staging/mt7621-eth/gsw_mt7621.c            |   10 +-
+ drivers/staging/mt7621-eth/mdio.c                  |    6 +-
+ drivers/staging/mt7621-eth/mtk_eth_soc.c           |   44 +-
+ drivers/staging/mt7621-eth/mtk_eth_soc.h           |   33 +-
+ drivers/staging/mt7621-eth/soc_mt7621.c            |    7 +-
+ drivers/staging/mt7621-gpio/TODO                   |    2 -
+ drivers/staging/mt7621-gpio/gpio-mt7621.c          |  144 +-
+ .../staging/mt7621-gpio/mediatek,mt7621-gpio.txt   |   58 +
+ drivers/staging/mt7621-mmc/board.h                 |   82 +-
+ drivers/staging/mt7621-mmc/dbg.c                   |  397 +-
+ drivers/staging/mt7621-mmc/dbg.h                   |   97 +-
+ drivers/staging/mt7621-mmc/mt6575_sd.h             | 1087 +++--
+ drivers/staging/mt7621-mmc/sd.c                    | 4458 +++++++++-----------
+ drivers/staging/mt7621-pci/pci-mt7621.c            |  478 +--
+ drivers/staging/mt7621-pinctrl/pinctrl-rt2880.c    |    1 -
+ drivers/staging/mt7621-spi/spi-mt7621.c            |    4 -
+ drivers/staging/nvec/nvec.c                        |   81 +-
+ drivers/staging/nvec/nvec.h                        |    2 +-
+ drivers/staging/olpc_dcon/TODO                     |    4 +
+ .../pi433/Documentation/devicetree/pi433.txt       |    6 +-
+ drivers/staging/pi433/Documentation/pi433.txt      |   10 +-
+ drivers/staging/pi433/Kconfig                      |    2 +-
+ drivers/staging/pi433/pi433_if.c                   |   39 +-
+ drivers/staging/pi433/rf69.c                       |  182 +-
+ drivers/staging/pi433/rf69.h                       |   28 +-
+ drivers/staging/rtl8188eu/TODO                     |    5 -
+ drivers/staging/rtl8188eu/os_dep/recv_linux.c      |    4 -
+ drivers/staging/rtl8192e/dot11d.c                  |    2 +-
+ drivers/staging/rtl8192e/rtllib_tx.c               |   14 +-
+ .../rtl8192u/ieee80211/ieee80211_crypt_ccmp.c      |    2 +-
+ .../rtl8192u/ieee80211/ieee80211_crypt_tkip.c      |    2 +-
+ .../rtl8192u/ieee80211/ieee80211_crypt_wep.c       |    2 +-
+ .../staging/rtl8192u/ieee80211/ieee80211_softmac.c |    2 +-
+ .../staging/rtl8192u/ieee80211/rtl819x_HTProc.c    |  170 +-
+ drivers/staging/rtl8192u/r8192U_core.c             |    2 +-
+ drivers/staging/rtl8192u/r8192U_dm.c               |    4 +-
+ drivers/staging/rtl8723bs/Makefile                 |    2 +-
+ drivers/staging/rtl8723bs/core/rtw_ap.c            |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_btcoex.c        |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_cmd.c           |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_debug.c         |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_eeprom.c        |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_efuse.c         |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_ieee80211.c     |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_io.c            |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_ioctl_set.c     |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_mlme.c          |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_mlme_ext.c      |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_odm.c           |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_pwrctrl.c       |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_recv.c          |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_rf.c            |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_security.c      |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_sta_mgt.c       |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_wlan_util.c     |   10 +-
+ drivers/staging/rtl8723bs/core/rtw_xmit.c          |   10 +-
+ drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c    |   10 +-
+ drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.h    |   10 +-
+ drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c    |   12 +-
+ drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.h    |   10 +-
+ drivers/staging/rtl8723bs/hal/HalBtcOutSrc.h       |   10 +-
+ drivers/staging/rtl8723bs/hal/HalHWImg8723B_BB.c   |   10 +-
+ drivers/staging/rtl8723bs/hal/HalHWImg8723B_BB.h   |   10 +-
+ drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.c  |   10 +-
+ drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.h  |   10 +-
+ drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.c   |   10 +-
+ drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.h   |   10 +-
+ drivers/staging/rtl8723bs/hal/HalPhyRf.c           |   10 +-
+ drivers/staging/rtl8723bs/hal/HalPhyRf.h           |   10 +-
+ drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c     |   10 +-
+ drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.h     |   10 +-
+ drivers/staging/rtl8723bs/hal/HalPwrSeqCmd.c       |   10 +-
+ drivers/staging/rtl8723bs/hal/Mp_Precomp.h         |   10 +-
+ drivers/staging/rtl8723bs/hal/hal_btcoex.c         |   10 +-
+ drivers/staging/rtl8723bs/hal/hal_com.c            |   10 +-
+ drivers/staging/rtl8723bs/hal/hal_com_phycfg.c     |   10 +-
+ drivers/staging/rtl8723bs/hal/hal_intf.c           |   10 +-
+ drivers/staging/rtl8723bs/hal/hal_phy.c            |   10 +-
+ .../hal/{Hal8723BPwrSeq.c => hal_pwr_seq.c}        |   12 +-
+ drivers/staging/rtl8723bs/hal/hal_sdio.c           |   10 +-
+ drivers/staging/rtl8723bs/hal/odm.c                |   10 +-
+ drivers/staging/rtl8723bs/hal/odm.h                |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_AntDiv.c         |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_AntDiv.h         |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_CfoTracking.c    |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_CfoTracking.h    |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_DIG.c            |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_DIG.h            |   10 +-
+ .../rtl8723bs/hal/odm_DynamicBBPowerSaving.c       |   10 +-
+ .../rtl8723bs/hal/odm_DynamicBBPowerSaving.h       |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_DynamicTxPower.c |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_DynamicTxPower.h |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_EdcaTurboCheck.c |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_EdcaTurboCheck.h |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_HWConfig.c       |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_HWConfig.h       |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_NoiseMonitor.c   |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_NoiseMonitor.h   |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_PathDiv.c        |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_PathDiv.h        |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_RTL8723B.c       |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_RTL8723B.h       |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_RegConfig8723B.c |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_RegConfig8723B.h |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_RegDefine11N.h   |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_debug.c          |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_debug.h          |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_interface.h      |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_precomp.h        |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_reg.h            |   10 +-
+ drivers/staging/rtl8723bs/hal/odm_types.h          |   10 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c       |   10 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_dm.c        |   10 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c  |   40 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c    |   10 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c    |   10 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_rxdesc.c    |   10 +-
+ drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c     |   10 +-
+ drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c     |   10 +-
+ drivers/staging/rtl8723bs/hal/sdio_halinit.c       |   10 +-
+ drivers/staging/rtl8723bs/hal/sdio_ops.c           |   10 +-
+ drivers/staging/rtl8723bs/include/Hal8192CPhyReg.h |   10 +-
+ drivers/staging/rtl8723bs/include/HalPwrSeqCmd.h   |   10 +-
+ drivers/staging/rtl8723bs/include/HalVerDef.h      |   10 +-
+ drivers/staging/rtl8723bs/include/autoconf.h       |   10 +-
+ drivers/staging/rtl8723bs/include/basic_types.h    |   10 +-
+ drivers/staging/rtl8723bs/include/cmd_osdep.h      |   10 +-
+ drivers/staging/rtl8723bs/include/drv_conf.h       |   10 +-
+ drivers/staging/rtl8723bs/include/drv_types.h      |   10 +-
+ drivers/staging/rtl8723bs/include/drv_types_sdio.h |   10 +-
+ drivers/staging/rtl8723bs/include/ethernet.h       |   10 +-
+ drivers/staging/rtl8723bs/include/hal_btcoex.h     |   10 +-
+ drivers/staging/rtl8723bs/include/hal_com.h        |   10 +-
+ drivers/staging/rtl8723bs/include/hal_com_h2c.h    |   10 +-
+ drivers/staging/rtl8723bs/include/hal_com_phycfg.h |   10 +-
+ drivers/staging/rtl8723bs/include/hal_com_reg.h    |   10 +-
+ drivers/staging/rtl8723bs/include/hal_data.h       |   10 +-
+ drivers/staging/rtl8723bs/include/hal_intf.h       |   10 +-
+ drivers/staging/rtl8723bs/include/hal_pg.h         |   10 +-
+ drivers/staging/rtl8723bs/include/hal_phy.h        |   10 +-
+ .../include/{Hal8723BPhyCfg.h => hal_phy_cfg.h}    |   10 +-
+ drivers/staging/rtl8723bs/include/hal_phy_reg.h    |   10 +-
+ .../{Hal8723BPhyReg.h => hal_phy_reg_8723b.h}      |   10 +-
+ .../include/{Hal8723BPwrSeq.h => hal_pwr_seq.h}    |    0
+ drivers/staging/rtl8723bs/include/hal_sdio.h       |   10 +-
+ drivers/staging/rtl8723bs/include/ieee80211.h      |   10 +-
+ drivers/staging/rtl8723bs/include/ioctl_cfg80211.h |   10 +-
+ drivers/staging/rtl8723bs/include/mlme_osdep.h     |   10 +-
+ drivers/staging/rtl8723bs/include/osdep_intf.h     |   10 +-
+ drivers/staging/rtl8723bs/include/osdep_service.h  |   10 +-
+ .../rtl8723bs/include/osdep_service_linux.h        |   10 +-
+ drivers/staging/rtl8723bs/include/recv_osdep.h     |   10 +-
+ drivers/staging/rtl8723bs/include/rtl8192c_recv.h  |   10 +-
+ drivers/staging/rtl8723bs/include/rtl8192c_rf.h    |   10 +-
+ drivers/staging/rtl8723bs/include/rtl8723b_cmd.h   |   10 +-
+ drivers/staging/rtl8723bs/include/rtl8723b_dm.h    |   10 +-
+ drivers/staging/rtl8723bs/include/rtl8723b_hal.h   |  283 +-
+ drivers/staging/rtl8723bs/include/rtl8723b_recv.h  |   10 +-
+ drivers/staging/rtl8723bs/include/rtl8723b_rf.h    |   10 +-
+ drivers/staging/rtl8723bs/include/rtl8723b_spec.h  |   10 +-
+ drivers/staging/rtl8723bs/include/rtl8723b_xmit.h  |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_ap.h         |   10 +-
+ .../staging/rtl8723bs/include/rtw_beamforming.h    |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_br_ext.h     |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_btcoex.h     |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_byteorder.h  |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_cmd.h        |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_debug.h      |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_eeprom.h     |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_efuse.h      |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_event.h      |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_ht.h         |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_io.h         |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_ioctl.h      |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_ioctl_set.h  |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_mlme.h       |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_mlme_ext.h   |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_mp.h         |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_odm.h        |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_pwrctrl.h    |   35 +-
+ drivers/staging/rtl8723bs/include/rtw_qos.h        |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_recv.h       |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_rf.h         |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_security.h   |   10 +-
+ drivers/staging/rtl8723bs/include/rtw_version.h    |    1 +
+ drivers/staging/rtl8723bs/include/rtw_xmit.h       |   10 +-
+ drivers/staging/rtl8723bs/include/sdio_hal.h       |   10 +-
+ drivers/staging/rtl8723bs/include/sdio_ops.h       |   10 +-
+ drivers/staging/rtl8723bs/include/sdio_ops_linux.h |   10 +-
+ drivers/staging/rtl8723bs/include/sdio_osintf.h    |   10 +-
+ drivers/staging/rtl8723bs/include/sta_info.h       |   10 +-
+ drivers/staging/rtl8723bs/include/wifi.h           |   22 +-
+ drivers/staging/rtl8723bs/include/wlan_bssdef.h    |   10 +-
+ drivers/staging/rtl8723bs/include/xmit_osdep.h     |   10 +-
+ drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c  |   12 +-
+ drivers/staging/rtl8723bs/os_dep/ioctl_linux.c     |   10 +-
+ drivers/staging/rtl8723bs/os_dep/mlme_linux.c      |   10 +-
+ drivers/staging/rtl8723bs/os_dep/os_intfs.c        |   10 +-
+ drivers/staging/rtl8723bs/os_dep/osdep_service.c   |   10 +-
+ drivers/staging/rtl8723bs/os_dep/recv_linux.c      |   10 +-
+ drivers/staging/rtl8723bs/os_dep/rtw_proc.c        |   10 +-
+ drivers/staging/rtl8723bs/os_dep/rtw_proc.h        |   10 +-
+ drivers/staging/rtl8723bs/os_dep/sdio_intf.c       |   10 +-
+ drivers/staging/rtl8723bs/os_dep/sdio_ops_linux.c  |   10 +-
+ drivers/staging/rtl8723bs/os_dep/xmit_linux.c      |   10 +-
+ drivers/staging/rtlwifi/base.c                     |   13 +-
+ drivers/staging/rtlwifi/base.h                     |   13 +-
+ drivers/staging/rtlwifi/btcoexist/halbt_precomp.h  |   13 +-
+ .../staging/rtlwifi/btcoexist/halbtc8822b1ant.c    |   13 +-
+ .../staging/rtlwifi/btcoexist/halbtc8822b1ant.h    |   13 +-
+ .../staging/rtlwifi/btcoexist/halbtc8822b2ant.c    |   13 +-
+ .../staging/rtlwifi/btcoexist/halbtc8822b2ant.h    |   13 +-
+ .../rtlwifi/btcoexist/halbtc8822bwifionly.c        |   13 +-
+ .../rtlwifi/btcoexist/halbtc8822bwifionly.h        |   13 +-
+ drivers/staging/rtlwifi/btcoexist/halbtcoutsrc.c   |   13 +-
+ drivers/staging/rtlwifi/btcoexist/halbtcoutsrc.h   |   13 +-
+ drivers/staging/rtlwifi/btcoexist/rtl_btc.c        |   13 +-
+ drivers/staging/rtlwifi/btcoexist/rtl_btc.h        |   13 +-
+ drivers/staging/rtlwifi/cam.c                      |   13 +-
+ drivers/staging/rtlwifi/cam.h                      |   13 +-
+ drivers/staging/rtlwifi/core.c                     |   13 +-
+ drivers/staging/rtlwifi/core.h                     |   13 +-
+ drivers/staging/rtlwifi/debug.c                    |   13 +-
+ drivers/staging/rtlwifi/debug.h                    |   13 +-
+ drivers/staging/rtlwifi/efuse.c                    |   13 +-
+ drivers/staging/rtlwifi/efuse.h                    |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_2_platform.h |   13 +-
+ .../halmac_88xx/halmac_8822b/halmac_8822b_cfg.h    |   13 +-
+ .../halmac_88xx/halmac_8822b/halmac_8822b_phy.c    |   13 +-
+ .../halmac_8822b/halmac_8822b_pwr_seq.c            |   13 +-
+ .../halmac_8822b/halmac_8822b_pwr_seq.h            |   13 +-
+ .../halmac_88xx/halmac_8822b/halmac_api_8822b.c    |   13 +-
+ .../halmac_88xx/halmac_8822b/halmac_api_8822b.h    |   13 +-
+ .../halmac_8822b/halmac_api_8822b_pcie.c           |   13 +-
+ .../halmac_8822b/halmac_api_8822b_pcie.h           |   13 +-
+ .../halmac_8822b/halmac_api_8822b_sdio.c           |   13 +-
+ .../halmac_8822b/halmac_api_8822b_sdio.h           |   13 +-
+ .../halmac_8822b/halmac_api_8822b_usb.c            |   13 +-
+ .../halmac_8822b/halmac_api_8822b_usb.h            |   13 +-
+ .../halmac_88xx/halmac_8822b/halmac_func_8822b.c   |   13 +-
+ .../halmac_88xx/halmac_8822b/halmac_func_8822b.h   |   13 +-
+ .../rtlwifi/halmac/halmac_88xx/halmac_88xx_cfg.h   |   13 +-
+ .../rtlwifi/halmac/halmac_88xx/halmac_api_88xx.c   |   13 +-
+ .../rtlwifi/halmac/halmac_88xx/halmac_api_88xx.h   |   13 +-
+ .../halmac/halmac_88xx/halmac_api_88xx_pcie.c      |   13 +-
+ .../halmac/halmac_88xx/halmac_api_88xx_pcie.h      |   13 +-
+ .../halmac/halmac_88xx/halmac_api_88xx_sdio.c      |   13 +-
+ .../halmac/halmac_88xx/halmac_api_88xx_sdio.h      |   13 +-
+ .../halmac/halmac_88xx/halmac_api_88xx_usb.c       |   13 +-
+ .../halmac/halmac_88xx/halmac_api_88xx_usb.h       |   13 +-
+ .../rtlwifi/halmac/halmac_88xx/halmac_func_88xx.c  |   13 +-
+ .../rtlwifi/halmac/halmac_88xx/halmac_func_88xx.h  |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_api.c        |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_api.h        |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_bit2.h       |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_bit_8822b.h  |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_fw_info.h    |   13 +-
+ .../rtlwifi/halmac/halmac_fw_offload_c2h_nic.h     |   13 +-
+ .../rtlwifi/halmac/halmac_fw_offload_h2c_nic.h     |   13 +-
+ .../rtlwifi/halmac/halmac_h2c_extra_info_nic.h     |   13 +-
+ .../staging/rtlwifi/halmac/halmac_intf_phy_cmd.h   |   13 +-
+ .../rtlwifi/halmac/halmac_original_c2h_nic.h       |   13 +-
+ .../rtlwifi/halmac/halmac_original_h2c_nic.h       |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_pcie_reg.h   |   13 +-
+ .../staging/rtlwifi/halmac/halmac_pwr_seq_cmd.h    |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_reg2.h       |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_reg_8822b.h  |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_rx_bd_chip.h |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_rx_bd_nic.h  |   13 +-
+ .../staging/rtlwifi/halmac/halmac_rx_desc_chip.h   |   13 +-
+ .../staging/rtlwifi/halmac/halmac_rx_desc_nic.h    |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_sdio_reg.h   |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_tx_bd_chip.h |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_tx_bd_nic.h  |   13 +-
+ .../staging/rtlwifi/halmac/halmac_tx_desc_chip.h   |   13 +-
+ .../staging/rtlwifi/halmac/halmac_tx_desc_nic.h    |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_type.h       |   13 +-
+ drivers/staging/rtlwifi/halmac/halmac_usb_reg.h    |   13 +-
+ drivers/staging/rtlwifi/halmac/rtl_halmac.c        |   13 +-
+ drivers/staging/rtlwifi/halmac/rtl_halmac.h        |   13 +-
+ drivers/staging/rtlwifi/pci.c                      |   13 +-
+ drivers/staging/rtlwifi/pci.h                      |   13 +-
+ drivers/staging/rtlwifi/phydm/halphyrf_ce.c        |   13 +-
+ drivers/staging/rtlwifi/phydm/halphyrf_ce.h        |   13 +-
+ drivers/staging/rtlwifi/phydm/mp_precomp.h         |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm.c              |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm.h              |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_acs.c          |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_acs.h          |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_adaptivity.c   |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_adaptivity.h   |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_adc_sampling.c |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_adc_sampling.h |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_antdiv.c       |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_antdiv.h       |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_beamforming.h  |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_ccx.c          |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_ccx.h          |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_cfotracking.c  |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_cfotracking.h  |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_debug.c        |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_debug.h        |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_dfs.h          |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_dig.c          |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_dig.h          |   13 +-
+ .../staging/rtlwifi/phydm/phydm_dynamic_rx_path.h  |   13 +-
+ .../rtlwifi/phydm/phydm_dynamicbbpowersaving.c     |   13 +-
+ .../rtlwifi/phydm/phydm_dynamicbbpowersaving.h     |   13 +-
+ .../staging/rtlwifi/phydm/phydm_dynamictxpower.c   |   13 +-
+ .../staging/rtlwifi/phydm/phydm_dynamictxpower.h   |   13 +-
+ .../staging/rtlwifi/phydm/phydm_edcaturbocheck.c   |   13 +-
+ .../staging/rtlwifi/phydm/phydm_edcaturbocheck.h   |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_features.h     |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_hwconfig.c     |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_hwconfig.h     |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_interface.c    |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_interface.h    |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_iqk.h          |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_kfree.c        |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_kfree.h        |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_noisemonitor.c |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_noisemonitor.h |   13 +-
+ .../staging/rtlwifi/phydm/phydm_powertracking_ce.c |   13 +-
+ .../staging/rtlwifi/phydm/phydm_powertracking_ce.h |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_pre_define.h   |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_precomp.h      |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_psd.c          |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_psd.h          |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_rainfo.c       |   17 +-
+ drivers/staging/rtlwifi/phydm/phydm_rainfo.h       |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_reg.h          |   13 +-
+ .../staging/rtlwifi/phydm/phydm_regdefine11ac.h    |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_regdefine11n.h |   13 +-
+ drivers/staging/rtlwifi/phydm/phydm_types.h        |   13 +-
+ .../rtlwifi/phydm/rtl8822b/halhwimg8822b_bb.c      |   13 +-
+ .../rtlwifi/phydm/rtl8822b/halhwimg8822b_bb.h      |   13 +-
+ .../rtlwifi/phydm/rtl8822b/halhwimg8822b_mac.c     |   13 +-
+ .../rtlwifi/phydm/rtl8822b/halhwimg8822b_mac.h     |   13 +-
+ .../rtlwifi/phydm/rtl8822b/halhwimg8822b_rf.c      |   13 +-
+ .../rtlwifi/phydm/rtl8822b/halhwimg8822b_rf.h      |   13 +-
+ .../rtlwifi/phydm/rtl8822b/halphyrf_8822b.c        |   47 +-
+ .../rtlwifi/phydm/rtl8822b/halphyrf_8822b.h        |   13 +-
+ .../rtlwifi/phydm/rtl8822b/phydm_hal_api8822b.c    |   13 +-
+ .../rtlwifi/phydm/rtl8822b/phydm_hal_api8822b.h    |   13 +-
+ .../rtlwifi/phydm/rtl8822b/phydm_iqk_8822b.c       |   13 +-
+ .../rtlwifi/phydm/rtl8822b/phydm_iqk_8822b.h       |   13 +-
+ .../rtlwifi/phydm/rtl8822b/phydm_regconfig8822b.c  |   13 +-
+ .../rtlwifi/phydm/rtl8822b/phydm_regconfig8822b.h  |   13 +-
+ .../rtlwifi/phydm/rtl8822b/phydm_rtl8822b.c        |   13 +-
+ .../rtlwifi/phydm/rtl8822b/phydm_rtl8822b.h        |   13 +-
+ .../rtlwifi/phydm/rtl8822b/version_rtl8822b.h      |   13 +-
+ drivers/staging/rtlwifi/phydm/rtl_phydm.c          |   13 +-
+ drivers/staging/rtlwifi/phydm/rtl_phydm.h          |   13 +-
+ drivers/staging/rtlwifi/phydm/txbf/halcomtxbf.h    |   13 +-
+ drivers/staging/rtlwifi/phydm/txbf/haltxbf8822b.h  |   13 +-
+ .../staging/rtlwifi/phydm/txbf/haltxbfinterface.h  |   13 +-
+ drivers/staging/rtlwifi/phydm/txbf/haltxbfjaguar.h |   13 +-
+ .../rtlwifi/phydm/txbf/phydm_hal_txbf_api.h        |   13 +-
+ drivers/staging/rtlwifi/ps.c                       |   13 +-
+ drivers/staging/rtlwifi/ps.h                       |   13 +-
+ drivers/staging/rtlwifi/pwrseqcmd.h                |   13 +-
+ drivers/staging/rtlwifi/rc.c                       |   13 +-
+ drivers/staging/rtlwifi/rc.h                       |   13 +-
+ drivers/staging/rtlwifi/regd.c                     |   13 +-
+ drivers/staging/rtlwifi/regd.h                     |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/def.h            |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/fw.c             |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/fw.h             |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/hw.c             |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/hw.h             |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/led.c            |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/led.h            |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/phy.c            |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/phy.h            |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/reg.h            |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/sw.c             |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/sw.h             |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/trx.c            |   13 +-
+ drivers/staging/rtlwifi/rtl8822be/trx.h            |   13 +-
+ drivers/staging/rtlwifi/stats.c                    |   13 +-
+ drivers/staging/rtlwifi/stats.h                    |   13 +-
+ drivers/staging/rtlwifi/wifi.h                     |   13 +-
+ drivers/staging/sm750fb/TODO                       |    5 +-
+ drivers/staging/speakup/buffers.c                  |    4 +
+ drivers/staging/speakup/main.c                     |    6 +
+ drivers/staging/speakup/speakup.h                  |    3 +-
+ drivers/staging/speakup/speakup_dummy.c            |    1 +
+ drivers/staging/speakup/speakup_soft.c             |    3 +-
+ drivers/staging/speakup/spk_types.h                |    2 +-
+ drivers/staging/speakup/varhandlers.c              |    1 +
+ drivers/staging/unisys/visornic/visornic_main.c    |    2 +-
+ drivers/staging/vboxvideo/vbox_main.c              |    3 +-
+ drivers/staging/vboxvideo/vbox_mode.c              |    2 +-
+ drivers/staging/vc04_services/Kconfig              |    1 -
+ drivers/staging/vc04_services/Makefile             |    2 +-
+ .../vc04_services/bcm2835-audio/bcm2835-vchiq.c    |   64 +-
+ .../staging/vc04_services/bcm2835-audio/bcm2835.c  |   43 +-
+ .../staging/vc04_services/bcm2835-audio/bcm2835.h  |   12 +
+ drivers/staging/vc04_services/bcm2835-camera/TODO  |   11 -
+ .../vc04_services/bcm2835-camera/bcm2835-camera.c  |  402 +-
+ .../vc04_services/bcm2835-camera/mmal-common.h     |    9 +-
+ .../vc04_services/bcm2835-camera/mmal-msg-port.h   |    2 +-
+ .../vc04_services/bcm2835-camera/mmal-parameters.h |   13 +
+ .../vc04_services/bcm2835-camera/mmal-vchiq.c      |  345 +-
+ .../vc04_services/bcm2835-camera/mmal-vchiq.h      |   15 +-
+ drivers/staging/vc04_services/interface/vchi/TODO  |    9 +-
+ .../vc04_services/interface/vchi/vchi_common.h     |    2 -
+ .../interface/vchiq_arm/vchiq_2835_arm.c           |    6 +-
+ .../vc04_services/interface/vchiq_arm/vchiq_arm.c  |   83 +-
+ .../vc04_services/interface/vchiq_arm/vchiq_core.c |   55 +-
+ .../vc04_services/interface/vchiq_arm/vchiq_core.h |    1 -
+ .../vc04_services/interface/vchiq_arm/vchiq_shim.c |    1 -
+ drivers/staging/vt6655/card.c                      |    5 +-
+ drivers/staging/vt6655/device_main.c               |  150 +-
+ drivers/staging/wilc1000/TODO                      |   12 +-
+ drivers/staging/wilc1000/coreconfigurator.c        |  140 +-
+ drivers/staging/wilc1000/coreconfigurator.h        |   37 +-
+ drivers/staging/wilc1000/host_interface.c          |  549 ++-
+ drivers/staging/wilc1000/host_interface.h          |    6 +-
+ drivers/staging/wilc1000/linux_mon.c               |   90 +-
+ drivers/staging/wilc1000/linux_wlan.c              |  378 +-
+ drivers/staging/wilc1000/wilc_debugfs.c            |    2 -
+ drivers/staging/wilc1000/wilc_sdio.c               |  109 +-
+ drivers/staging/wilc1000/wilc_spi.c                |   99 +-
+ drivers/staging/wilc1000/wilc_wfi_cfgoperations.c  | 1049 +++--
+ drivers/staging/wilc1000/wilc_wfi_cfgoperations.h  |   18 +-
+ drivers/staging/wilc1000/wilc_wfi_netdevice.h      |   83 +-
+ drivers/staging/wilc1000/wilc_wlan.c               |   26 +-
+ drivers/staging/wilc1000/wilc_wlan.h               |   37 +-
+ drivers/staging/wilc1000/wilc_wlan_cfg.c           |    1 -
+ drivers/staging/wilc1000/wilc_wlan_if.h            |   16 +-
+ drivers/staging/wlan-ng/hfa384x.h                  |    2 +-
+ drivers/staging/wlan-ng/hfa384x_usb.c              |    7 +-
+ drivers/staging/wlan-ng/p80211conv.c               |    8 +-
+ drivers/staging/wlan-ng/p80211conv.h               |   26 +-
+ drivers/staging/wlan-ng/p80211hdr.h                |    2 +-
+ drivers/staging/wlan-ng/p80211ioctl.h              |    2 +-
+ drivers/staging/wlan-ng/p80211metadef.h            |    2 +-
+ drivers/staging/wlan-ng/p80211metastruct.h         |   90 +-
+ drivers/staging/wlan-ng/p80211mgmt.h               |    2 +-
+ drivers/staging/wlan-ng/p80211msg.h                |    2 +-
+ drivers/staging/wlan-ng/p80211netdev.c             |    8 +-
+ drivers/staging/wlan-ng/p80211netdev.h             |    8 +-
+ drivers/staging/wlan-ng/p80211req.h                |    2 +-
+ drivers/staging/wlan-ng/p80211types.h              |    2 +-
+ drivers/staging/wlan-ng/prism2mgmt.h               |    2 +-
+ drivers/staging/wlan-ng/prism2sta.c                |   52 +-
+ include/linux/iio/adc/ad_sigma_delta.h             |   24 +-
+ include/linux/iio/adc/stm32-dfsdm-adc.h            |    2 +
+ .../linux/iio/common}/cros_ec_sensors_core.h       |    3 +
+ include/linux/iio/iio.h                            |   24 +-
+ include/linux/kernel.h                             |   16 +
+ include/linux/platform_data/tsl2772.h              |  101 +
+ tools/iio/iio_generic_buffer.c                     |   13 +-
+ 826 files changed, 19290 insertions(+), 24222 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/iio/afe/current-sense-amplifier.txt
+ create mode 100644 Documentation/devicetree/bindings/iio/afe/current-sense-shunt.txt
+ create mode 100644 Documentation/devicetree/bindings/iio/afe/voltage-divider.txt
+ create mode 100644 Documentation/devicetree/bindings/iio/dac/ti,dac5571.txt
+ create mode 100644 drivers/iio/accel/adis16201.c
+ rename drivers/{staging => }/iio/accel/adis16209.c (92%)
+ create mode 100644 drivers/iio/afe/Kconfig
+ create mode 100644 drivers/iio/afe/Makefile
+ create mode 100644 drivers/iio/afe/iio-rescale.c
+ create mode 100644 drivers/iio/dac/ad5686-spi.c
+ create mode 100644 drivers/iio/dac/ad5686.h
+ create mode 100644 drivers/iio/dac/ad5696-i2c.c
+ create mode 100644 drivers/iio/dac/ti-dac5571.c
+ create mode 100644 drivers/iio/light/tsl2772.c
+ create mode 100644 drivers/iio/resolver/Kconfig
+ create mode 100644 drivers/iio/resolver/Makefile
+ rename drivers/{staging => }/iio/resolver/ad2s1200.c (52%)
+ create mode 100644 drivers/staging/android/uapi/vsoc_shm.h
+ create mode 100644 drivers/staging/android/vsoc.c
+ create mode 100644 drivers/staging/fbtft/TODO
+ create mode 100644 drivers/staging/fsl-dpaa2/rtc/Makefile
+ create mode 100644 drivers/staging/fsl-dpaa2/rtc/dprtc-cmd.h
+ create mode 100644 drivers/staging/fsl-dpaa2/rtc/dprtc.c
+ create mode 100644 drivers/staging/fsl-dpaa2/rtc/dprtc.h
+ create mode 100644 drivers/staging/fsl-dpaa2/rtc/rtc.c
+ create mode 100644 drivers/staging/fsl-dpaa2/rtc/rtc.h
+ delete mode 100644 drivers/staging/goldfish/goldfish_nand.c
+ delete mode 100644 drivers/staging/goldfish/goldfish_nand_reg.h
+ create mode 100644 drivers/staging/greybus/TODO
+ delete mode 100644 drivers/staging/iio/accel/adis16201.c
+ delete mode 100644 drivers/staging/iio/light/Kconfig
+ delete mode 100644 drivers/staging/iio/light/Makefile
+ delete mode 100644 drivers/staging/iio/light/tsl2x7x.c
+ delete mode 100644 drivers/staging/iio/light/tsl2x7x.h
+ delete mode 100644 drivers/staging/iio/meter/ade7753.c
+ delete mode 100644 drivers/staging/iio/meter/ade7754.c
+ delete mode 100644 drivers/staging/iio/meter/ade7758.h
+ delete mode 100644 drivers/staging/iio/meter/ade7758_core.c
+ delete mode 100644 drivers/staging/iio/meter/ade7758_ring.c
+ delete mode 100644 drivers/staging/iio/meter/ade7758_trigger.c
+ delete mode 100644 drivers/staging/iio/meter/ade7759.c
+ delete mode 100644 drivers/staging/ks7010/ks7010_sdio.h
+ delete mode 100644 drivers/staging/lustre/include/linux/libcfs/curproc.h
+ delete mode 100644 drivers/staging/lustre/include/linux/libcfs/libcfs_prim.h
+ delete mode 100644 drivers/staging/lustre/include/linux/libcfs/libcfs_time.h
+ delete mode 100644 drivers/staging/lustre/include/linux/libcfs/linux/libcfs.h
+ delete mode 100644 drivers/staging/lustre/include/linux/libcfs/linux/linux-cpu.h
+ delete mode 100644 drivers/staging/lustre/include/linux/libcfs/linux/linux-time.h
+ rename drivers/staging/lustre/lnet/libcfs/{linux => }/linux-crypto-adler.c (100%)
+ rename drivers/staging/lustre/lnet/libcfs/{linux => }/linux-crypto.c (98%)
+ rename drivers/staging/lustre/lnet/libcfs/{linux => }/linux-crypto.h (100%)
+ rename drivers/staging/lustre/lnet/libcfs/{linux => }/linux-debug.c (97%)
+ rename drivers/staging/lustre/lnet/libcfs/{linux => }/linux-tracefile.c (98%)
+ delete mode 100644 drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
+ delete mode 100644 drivers/staging/lustre/lnet/libcfs/linux/linux-module.c
+ delete mode 100644 drivers/staging/most/dim2/dim2.h
+ create mode 100644 drivers/staging/mt7621-gpio/mediatek,mt7621-gpio.txt
+ rename drivers/staging/rtl8723bs/hal/{Hal8723BPwrSeq.c => hal_pwr_seq.c} (86%)
+ rename drivers/staging/rtl8723bs/include/{Hal8723BPhyCfg.h => hal_phy_cfg.h} (84%)
+ rename drivers/staging/rtl8723bs/include/{Hal8723BPhyReg.h => hal_phy_reg_8723b.h} (79%)
+ rename drivers/staging/rtl8723bs/include/{Hal8723BPwrSeq.h => hal_pwr_seq.h} (100%)
+ rename {drivers/iio/common/cros_ec_sensors => include/linux/iio/common}/cros_ec_sensors_core.h (98%)
+ create mode 100644 include/linux/platform_data/tsl2772.h
+Merging mux/for-next (6a3d4baac84a mux: add mux_control_get_optional() API)
+$ git merge mux/for-next
+Merge made by the 'recursive' strategy.
+ Documentation/driver-model/devres.txt |   1 +
+ drivers/mux/core.c                    | 104 +++++++++++++++++++++++++++-------
+ include/linux/mux/consumer.h          |   4 ++
+ 3 files changed, 89 insertions(+), 20 deletions(-)
+Merging slave-dma/next (cece9f043feb Merge branch 'topic/sprd' into next)
+$ git merge slave-dma/next
+Auto-merging drivers/dma/ti/omap-dma.c
+Auto-merging drivers/dma/ti/edma.c
+Auto-merging drivers/dma/ti/cppi41.c
+Auto-merging MAINTAINERS
+Auto-merging Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/dma/renesas,rcar-dmac.txt  |   1 +
+ .../devicetree/bindings/dma/renesas,usb-dmac.txt   |   2 +
+ MAINTAINERS                                        |   8 +
+ drivers/dma/Kconfig                                |  31 +-
+ drivers/dma/Makefile                               |   5 +-
+ drivers/dma/at_hdmac.c                             |   9 +-
+ drivers/dma/at_xdmac.c                             |   9 +-
+ drivers/dma/dma-axi-dmac.c                         |   2 +-
+ drivers/dma/dmatest.c                              |  16 +-
+ drivers/dma/dw/platform.c                          |   6 +-
+ drivers/dma/fsldma.c                               |   6 +-
+ drivers/dma/idma64.c                               |   6 +-
+ drivers/dma/imx-dma.c                              |  26 +-
+ drivers/dma/imx-sdma.c                             |  29 +-
+ drivers/dma/mxs-dma.c                              |  14 +-
+ drivers/dma/pl330.c                                | 237 +++++++++-----
+ drivers/dma/qcom/bam_dma.c                         |  10 +-
+ drivers/dma/qcom/hidma.c                           |   3 +-
+ drivers/dma/qcom/hidma_mgmt_sys.c                  |   6 +-
+ drivers/dma/sh/shdmac.c                            |  50 ++-
+ drivers/dma/sprd-dma.c                             | 349 ++++++++++++---------
+ drivers/dma/ste_dma40.c                            |  12 +-
+ drivers/dma/stm32-mdma.c                           | 100 +++---
+ drivers/dma/ti/Kconfig                             |  37 +++
+ drivers/dma/ti/Makefile                            |   5 +
+ drivers/dma/{ => ti}/cppi41.c                      |   2 +-
+ .../dma/{ti-dma-crossbar.c => ti/dma-crossbar.c}   |   0
+ drivers/dma/{ => ti}/edma.c                        |   4 +-
+ drivers/dma/{ => ti}/omap-dma.c                    |   2 +-
+ drivers/dma/txx9dmac.c                             |   8 +-
+ include/linux/dma/sprd-dma.h                       |  61 ++++
+ 31 files changed, 631 insertions(+), 425 deletions(-)
+ create mode 100644 drivers/dma/ti/Kconfig
+ create mode 100644 drivers/dma/ti/Makefile
+ rename drivers/dma/{ => ti}/cppi41.c (99%)
+ rename drivers/dma/{ti-dma-crossbar.c => ti/dma-crossbar.c} (100%)
+ rename drivers/dma/{ => ti}/edma.c (99%)
+ rename drivers/dma/{ => ti}/omap-dma.c (99%)
+ create mode 100644 include/linux/dma/sprd-dma.h
+Merging cgroup/for-next (d8742e229021 cgroup: css_set_lock should nest inside tasklist_lock)
+$ git merge cgroup/for-next
+Removing kernel/cgroup/stat.c
+Auto-merging kernel/cgroup/cgroup.c
+Auto-merging kernel/cgroup/cgroup-internal.h
+Merge made by the 'recursive' strategy.
+ include/linux/cgroup-defs.h     |  52 +++--
+ include/linux/cgroup.h          |  12 +-
+ kernel/cgroup/Makefile          |   2 +-
+ kernel/cgroup/cgroup-internal.h |  11 +-
+ kernel/cgroup/cgroup.c          | 105 +++++++---
+ kernel/cgroup/rdma.c            |  35 ++--
+ kernel/cgroup/rstat.c           | 416 ++++++++++++++++++++++++++++++++++++++++
+ kernel/cgroup/stat.c            | 338 --------------------------------
+ 8 files changed, 554 insertions(+), 417 deletions(-)
+ create mode 100644 kernel/cgroup/rstat.c
+ delete mode 100644 kernel/cgroup/stat.c
+Merging scsi/for-next (7ce1b53b1b5d Merge branch 'misc' into for-next)
+$ git merge scsi/for-next
+Auto-merging kernel/workqueue.c
+Auto-merging include/linux/libata.h
+Auto-merging include/linux/hyperv.h
+Auto-merging drivers/target/target_core_pscsi.c
+Auto-merging drivers/scsi/ufs/ufshcd.c
+Auto-merging drivers/scsi/st.c
+Auto-merging drivers/scsi/sg.c
+Auto-merging drivers/scsi/scsi_transport_sas.c
+Auto-merging drivers/scsi/scsi_lib.c
+Auto-merging drivers/scsi/scsi_error.c
+Auto-merging drivers/scsi/qedf/qedf_main.c
+Auto-merging drivers/scsi/qedf/qedf_io.c
+Auto-merging drivers/scsi/qedf/qedf_fip.c
+Auto-merging drivers/scsi/qedf/qedf_debugfs.c
+Auto-merging drivers/scsi/qedf/qedf.h
+CONFLICT (content): Merge conflict in drivers/scsi/qedf/qedf.h
+Auto-merging drivers/scsi/osd/osd_initiator.c
+Auto-merging drivers/scsi/mvumi.c
+Auto-merging drivers/scsi/megaraid/megaraid_sas_base.c
+Auto-merging drivers/scsi/megaraid.c
+Auto-merging drivers/net/hyperv/netvsc_drv.c
+Auto-merging drivers/net/hyperv/netvsc.c
+Auto-merging drivers/net/hyperv/hyperv_net.h
+Auto-merging drivers/message/fusion/mptsas.c
+Auto-merging drivers/message/fusion/mptbase.c
+Auto-merging MAINTAINERS
+Auto-merging Documentation/ioctl/ioctl-number.txt
+Auto-merging Documentation/driver-api/index.rst
+Resolved 'drivers/scsi/qedf/qedf.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master ce8d04846017] Merge remote-tracking branch 'scsi/for-next'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/driver-api/index.rst          |    1 +
+ Documentation/driver-api/scsi.rst           |    2 +-
+ Documentation/driver-api/target.rst         |   64 ++
+ Documentation/ioctl/ioctl-number.txt        |    2 +-
+ MAINTAINERS                                 |    1 -
+ drivers/ata/libata-scsi.c                   |   12 +
+ drivers/hv/ring_buffer.c                    |    2 +
+ drivers/message/fusion/lsi/mpi_cnfg.h       |    4 +-
+ drivers/message/fusion/mptbase.c            |    2 +-
+ drivers/message/fusion/mptfc.c              |    2 +-
+ drivers/message/fusion/mptlan.c             |    2 +-
+ drivers/message/fusion/mptsas.c             |    2 +-
+ drivers/net/hyperv/hyperv_net.h             |    1 -
+ drivers/net/hyperv/netvsc.c                 |   18 +-
+ drivers/net/hyperv/netvsc_drv.c             |    3 -
+ drivers/s390/scsi/zfcp_dbf.c                |   90 +-
+ drivers/s390/scsi/zfcp_dbf.h                |   21 +-
+ drivers/s390/scsi/zfcp_erp.c                |  194 ++--
+ drivers/s390/scsi/zfcp_ext.h                |   16 +-
+ drivers/s390/scsi/zfcp_fc.c                 |   11 +-
+ drivers/s390/scsi/zfcp_fc.h                 |   22 +-
+ drivers/s390/scsi/zfcp_fsf.c                |   61 +-
+ drivers/s390/scsi/zfcp_fsf.h                |    6 +-
+ drivers/s390/scsi/zfcp_scsi.c               |  141 ++-
+ drivers/s390/scsi/zfcp_sysfs.c              |    5 +-
+ drivers/scsi/3w-9xxx.c                      |    5 +
+ drivers/scsi/3w-xxxx.c                      |    3 +
+ drivers/scsi/Kconfig                        |   14 +
+ drivers/scsi/Makefile                       |    3 +-
+ drivers/scsi/a100u2w.c                      |   13 +-
+ drivers/scsi/am53c974.c                     |   13 +-
+ drivers/scsi/cxlflash/Kconfig               |    2 +-
+ drivers/scsi/cxlflash/Makefile              |    4 +-
+ drivers/scsi/cxlflash/backend.h             |   55 +-
+ drivers/scsi/cxlflash/common.h              |   12 +-
+ drivers/scsi/cxlflash/cxl_hw.c              |   13 +
+ drivers/scsi/cxlflash/lunmgt.c              |    4 +-
+ drivers/scsi/cxlflash/main.c                |   97 +-
+ drivers/scsi/cxlflash/main.h                |   21 +
+ drivers/scsi/cxlflash/ocxl_hw.c             | 1436 +++++++++++++++++++++++++++
+ drivers/scsi/cxlflash/ocxl_hw.h             |   77 ++
+ drivers/scsi/cxlflash/sislite.h             |   41 +-
+ drivers/scsi/cxlflash/superpipe.c           |   23 +-
+ drivers/scsi/cxlflash/vlun.c                |    3 +-
+ drivers/scsi/dpt_i2o.c                      |   21 +-
+ drivers/scsi/esas2r/esas2r_init.c           |    5 +-
+ drivers/scsi/esas2r/esas2r_ioctl.c          |    2 +-
+ drivers/scsi/esas2r/esas2r_main.c           |    2 +-
+ drivers/scsi/hisi_sas/hisi_sas.h            |   52 +-
+ drivers/scsi/hisi_sas/hisi_sas_main.c       |  638 ++++++++----
+ drivers/scsi/hisi_sas/hisi_sas_v1_hw.c      |  164 ++-
+ drivers/scsi/hisi_sas/hisi_sas_v2_hw.c      |  284 +++---
+ drivers/scsi/hisi_sas/hisi_sas_v3_hw.c      |  452 ++++++---
+ drivers/scsi/ipr.c                          |    2 +
+ drivers/scsi/ips.c                          |   78 +-
+ drivers/scsi/ips.h                          |   11 +-
+ drivers/scsi/isci/init.c                    |    3 -
+ drivers/scsi/iscsi_tcp.c                    |    1 -
+ drivers/scsi/libsas/sas_ata.c               |    5 +
+ drivers/scsi/libsas/sas_discover.c          |    1 +
+ drivers/scsi/lpfc/lpfc.h                    |    9 +-
+ drivers/scsi/lpfc/lpfc_attr.c               |  124 ++-
+ drivers/scsi/lpfc/lpfc_bsg.c                |   20 +-
+ drivers/scsi/lpfc/lpfc_ct.c                 |   10 +
+ drivers/scsi/lpfc/lpfc_debugfs.c            |   98 +-
+ drivers/scsi/lpfc/lpfc_els.c                |    1 -
+ drivers/scsi/lpfc/lpfc_hbadisc.c            |    7 +-
+ drivers/scsi/lpfc/lpfc_hw4.h                |    7 +-
+ drivers/scsi/lpfc/lpfc_init.c               |  153 ++-
+ drivers/scsi/lpfc/lpfc_nportdisc.c          |   35 +-
+ drivers/scsi/lpfc/lpfc_nvme.c               |  238 +++--
+ drivers/scsi/lpfc/lpfc_nvme.h               |   17 +-
+ drivers/scsi/lpfc/lpfc_nvmet.h              |    6 +-
+ drivers/scsi/lpfc/lpfc_scsi.c               |   33 +-
+ drivers/scsi/lpfc/lpfc_scsi.h               |    2 +-
+ drivers/scsi/lpfc/lpfc_sli.c                |  116 ++-
+ drivers/scsi/lpfc/lpfc_version.h            |    4 +-
+ drivers/scsi/megaraid.c                     |    3 +
+ drivers/scsi/megaraid/megaraid_sas.h        |    4 +-
+ drivers/scsi/megaraid/megaraid_sas_base.c   |   27 +-
+ drivers/scsi/megaraid/megaraid_sas_fusion.c |    8 +-
+ drivers/scsi/mpt3sas/mpi/mpi2.h             |    9 +-
+ drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h        |   30 +-
+ drivers/scsi/mpt3sas/mpi/mpi2_init.h        |    2 +-
+ drivers/scsi/mpt3sas/mpi/mpi2_ioc.h         |    7 +-
+ drivers/scsi/mpt3sas/mpt3sas_base.c         |  477 ++++++---
+ drivers/scsi/mpt3sas/mpt3sas_base.h         |   60 +-
+ drivers/scsi/mpt3sas/mpt3sas_ctl.c          |   33 +-
+ drivers/scsi/mpt3sas/mpt3sas_ctl.h          |    2 +-
+ drivers/scsi/mpt3sas/mpt3sas_scsih.c        |  491 ++++++---
+ drivers/scsi/mpt3sas/mpt3sas_warpdrive.c    |    3 +-
+ drivers/scsi/mvumi.c                        |   20 +-
+ drivers/scsi/osd/osd_initiator.c            |   16 +-
+ drivers/scsi/pm8001/pm8001_hwi.c            |    2 +-
+ drivers/scsi/qedf/drv_fcoe_fw_funcs.c       |    2 +-
+ drivers/scsi/qedf/drv_fcoe_fw_funcs.h       |    2 +-
+ drivers/scsi/qedf/drv_scsi_fw_funcs.c       |    2 +-
+ drivers/scsi/qedf/drv_scsi_fw_funcs.h       |    2 +-
+ drivers/scsi/qedf/qedf.h                    |    6 +-
+ drivers/scsi/qedf/qedf_attr.c               |    2 +-
+ drivers/scsi/qedf/qedf_dbg.c                |    4 +-
+ drivers/scsi/qedf/qedf_dbg.h                |    2 +-
+ drivers/scsi/qedf/qedf_debugfs.c            |    2 +-
+ drivers/scsi/qedf/qedf_els.c                |   35 +-
+ drivers/scsi/qedf/qedf_fip.c                |    5 +-
+ drivers/scsi/qedf/qedf_hsi.h                |    2 +-
+ drivers/scsi/qedf/qedf_io.c                 |   87 +-
+ drivers/scsi/qedf/qedf_main.c               |  130 ++-
+ drivers/scsi/qedf/qedf_version.h            |    6 +-
+ drivers/scsi/qla2xxx/qla_def.h              |    4 +-
+ drivers/scsi/qla2xxx/qla_gbl.h              |    3 +-
+ drivers/scsi/qla2xxx/qla_gs.c               |   41 +-
+ drivers/scsi/qla2xxx/qla_init.c             |  105 +-
+ drivers/scsi/qla2xxx/qla_os.c               |    4 +
+ drivers/scsi/qla2xxx/qla_target.c           |  192 +++-
+ drivers/scsi/qla2xxx/qla_target.h           |    4 +-
+ drivers/scsi/qla2xxx/qla_version.h          |    2 +-
+ drivers/scsi/qla2xxx/tcm_qla2xxx.c          |   37 +-
+ drivers/scsi/qlogicpti.c                    |    6 +-
+ drivers/scsi/scsi_debugfs.c                 |    2 +-
+ drivers/scsi/scsi_devinfo.c                 |   31 +-
+ drivers/scsi/scsi_dh.c                      |    5 +-
+ drivers/scsi/scsi_error.c                   |    7 +
+ drivers/scsi/scsi_lib.c                     |    4 +
+ drivers/scsi/scsi_sysfs.c                   |    2 +-
+ drivers/scsi/scsi_transport_sas.c           |    4 -
+ drivers/scsi/sd.h                           |   12 +-
+ drivers/scsi/sd_zbc.c                       |   10 -
+ drivers/scsi/sg.c                           |    2 +-
+ drivers/scsi/snic/snic_scsi.c               |    6 +-
+ drivers/scsi/st.c                           |    6 +-
+ drivers/scsi/storvsc_drv.c                  |   85 +-
+ drivers/scsi/ufs/ufs-qcom.c                 |    2 +-
+ drivers/scsi/ufs/ufshcd.c                   |  308 +++++-
+ drivers/scsi/ufs/ufshcd.h                   |   21 +
+ drivers/scsi/wd719x.c                       |   13 +-
+ drivers/scsi/zorro_esp.c                    | 1172 ++++++++++++++++++++++
+ drivers/target/target_core_configfs.c       |   25 +
+ drivers/target/target_core_file.c           |  137 ++-
+ drivers/target/target_core_file.h           |    1 +
+ drivers/target/target_core_internal.h       |    1 +
+ drivers/target/target_core_pscsi.c          |   26 +-
+ drivers/target/target_core_transport.c      |   64 +-
+ drivers/target/target_core_user.c           |  160 ++-
+ include/linux/hyperv.h                      |   12 +
+ include/linux/libata.h                      |    2 +
+ include/linux/log2.h                        |   35 +-
+ include/scsi/scsi_device.h                  |    2 +-
+ include/scsi/scsi_devinfo.h                 |   75 +-
+ include/target/target_core_base.h           |    1 +
+ include/uapi/linux/target_core_user.h       |   11 +-
+ kernel/workqueue.c                          |    1 +
+ 152 files changed, 7234 insertions(+), 2017 deletions(-)
+ create mode 100644 Documentation/driver-api/target.rst
+ create mode 100644 drivers/scsi/cxlflash/ocxl_hw.c
+ create mode 100644 drivers/scsi/cxlflash/ocxl_hw.h
+ create mode 100644 drivers/scsi/zorro_esp.c
+Merging scsi-mkp/for-next (1b5c2cb19668 scsi: lpfc: update driver version to 12.0.0.4)
+$ git merge scsi-mkp/for-next
+Already up to date.
+Merging target-updates/for-next (1c130ae00b76 iscsi-target: make sure to wake up sleeping login worker)
+$ git merge target-updates/for-next
+Already up to date.
+Merging target-bva/for-next (60cc43fc8884 Linux 4.17-rc1)
+$ git merge target-bva/for-next
+Already up to date.
+Merging libata/for-next (1b1a8f3d0161 Merge branch 'for-4.17-fixes' into for-next)
+$ git merge libata/for-next
+Auto-merging include/linux/libata.h
+Auto-merging drivers/ata/libata-scsi.c
+Auto-merging drivers/ata/libata-eh.c
+Merge made by the 'recursive' strategy.
+ drivers/ata/acard-ahci.c     |  4 +-
+ drivers/ata/ahci.h           |  2 +-
+ drivers/ata/libahci.c        |  8 ++--
+ drivers/ata/libata-core.c    | 61 ++++++++++++---------------
+ drivers/ata/libata-eh.c      | 56 ++++++++++++++++++++-----
+ drivers/ata/libata-scsi.c    | 19 +++++----
+ drivers/ata/libata-zpodd.c   |  2 +-
+ drivers/ata/pata_hpt37x.c    | 13 +++---
+ drivers/ata/sata_dwc_460ex.c | 14 +++----
+ drivers/ata/sata_fsl.c       | 10 ++---
+ drivers/ata/sata_mv.c        | 26 ++++++------
+ drivers/ata/sata_nv.c        | 98 +++++++++++++++++++-------------------------
+ drivers/ata/sata_sil24.c     |  6 +--
+ include/linux/libata.h       | 22 +++++-----
+ 14 files changed, 178 insertions(+), 163 deletions(-)
+Merging vhost/linux-next (cfcde40e8ce3 ACPI: disable extra P_LVLx access on KVM)
+$ git merge vhost/linux-next
+Auto-merging mm/page_alloc.c
+Auto-merging include/linux/mm.h
+Merge made by the 'recursive' strategy.
+ drivers/acpi/processor_idle.c       |   7 +-
+ drivers/virtio/virtio_balloon.c     | 298 +++++++++++++++++++++++++++++++-----
+ include/linux/mm.h                  |   6 +
+ include/linux/ptr_ring.h            |  79 +++++++++-
+ include/linux/virtio_ring.h         |   4 +-
+ include/uapi/linux/virtio_balloon.h |   7 +
+ mm/page_alloc.c                     |  97 ++++++++++++
+ mm/page_poison.c                    |   6 +
+ 8 files changed, 460 insertions(+), 44 deletions(-)
+Merging rpmsg/for-next (454575aba6e4 Merge branches 'hwspinlock-next', 'rpmsg-next' and 'rproc-next' into for-next)
+$ git merge rpmsg/for-next
+Removing include/linux/ste_modem_shm.h
+Merge made by the 'recursive' strategy.
+ Documentation/ABI/testing/sysfs-bus-rpmsg          |  20 ++
+ .../devicetree/bindings/remoteproc/qcom,q6v5.txt   |   1 +
+ .../devicetree/bindings/soc/qcom/qcom,smd.txt      |   8 +-
+ drivers/hwspinlock/Kconfig                         |   1 +
+ drivers/hwspinlock/hwspinlock_core.c               |  77 +++++---
+ drivers/hwspinlock/hwspinlock_internal.h           |  10 +-
+ drivers/hwspinlock/omap_hwspinlock.c               |  10 +-
+ drivers/hwspinlock/qcom_hwspinlock.c               |  10 +-
+ drivers/hwspinlock/sirf_hwspinlock.c               |   3 +-
+ drivers/hwspinlock/sprd_hwspinlock.c               |  10 +-
+ drivers/hwspinlock/u8500_hsem.c                    |  10 +-
+ drivers/remoteproc/Kconfig                         |   1 -
+ drivers/remoteproc/da8xx_remoteproc.c              |  12 +-
+ drivers/remoteproc/qcom_q6v5_pil.c                 | 201 ++++++++++++++++++---
+ drivers/rpmsg/Kconfig                              |   1 +
+ drivers/rpmsg/qcom_smd.c                           |  68 +++++--
+ drivers/rpmsg/rpmsg_core.c                         |  39 ++++
+ include/linux/hwspinlock.h                         |  68 ++++++-
+ include/linux/ste_modem_shm.h                      |  56 ------
+ 19 files changed, 422 insertions(+), 184 deletions(-)
+ delete mode 100644 include/linux/ste_modem_shm.h
+Merging gpio/for-next (84bf021ecee4 gpio: pxa: Include the right header)
+$ git merge gpio/for-next
+Auto-merging drivers/gpio/gpiolib.c
+Auto-merging drivers/gpio/gpio-aspeed.c
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/gpio/gpio-pca953x.txt      |  34 +++++
+ .../devicetree/bindings/gpio/renesas,gpio-rcar.txt |   3 +
+ .../devicetree/bindings/gpio/snps-dwapb-gpio.txt   |   9 +-
+ Documentation/driver-api/gpio/board.rst            |  16 +++
+ Documentation/driver-api/gpio/drivers-on-gpio.rst  |   4 +
+ drivers/gpio/Kconfig                               |  12 ++
+ drivers/gpio/gpio-104-dio-48e.c                    |   2 +-
+ drivers/gpio/gpio-104-idi-48.c                     |   2 +-
+ drivers/gpio/gpio-74xx-mmio.c                      |   9 +-
+ drivers/gpio/gpio-aspeed.c                         |  24 +++-
+ drivers/gpio/gpio-dwapb.c                          |  57 ++++----
+ drivers/gpio/gpio-eic-sprd.c                       |  73 +++++++++++
+ drivers/gpio/gpio-ge.c                             |   4 +-
+ drivers/gpio/gpio-gpio-mm.c                        |   2 +-
+ drivers/gpio/gpio-ingenic.c                        |   4 +-
+ drivers/gpio/gpio-loongson.c                       | 116 ++++++++++-------
+ drivers/gpio/gpio-lp3943.c                         |   2 +-
+ drivers/gpio/gpio-lp873x.c                         |   2 +-
+ drivers/gpio/gpio-lpc32xx.c                        |   3 +-
+ drivers/gpio/gpio-lynxpoint.c                      |   2 +-
+ drivers/gpio/gpio-max730x.c                        |   2 +-
+ drivers/gpio/gpio-mc33880.c                        |   2 +-
+ drivers/gpio/gpio-mc9s08dz60.c                     |   2 +-
+ drivers/gpio/gpio-ml-ioh.c                         |   2 +-
+ drivers/gpio/gpio-mm-lantiq.c                      |   2 +-
+ drivers/gpio/gpio-mockup.c                         |   7 +-
+ drivers/gpio/gpio-msic.c                           |   2 +-
+ drivers/gpio/gpio-mvebu.c                          |  20 ++-
+ drivers/gpio/gpio-mxc.c                            |  54 ++++----
+ drivers/gpio/gpio-mxs.c                            |  32 ++---
+ drivers/gpio/gpio-octeon.c                         |   2 +-
+ drivers/gpio/gpio-omap.c                           |   2 +-
+ drivers/gpio/gpio-palmas.c                         |   8 +-
+ drivers/gpio/gpio-pca953x.c                        |  81 ++++++++----
+ drivers/gpio/gpio-pcf857x.c                        |   2 +-
+ drivers/gpio/gpio-pch.c                            |   2 +-
+ drivers/gpio/gpio-pl061.c                          |   2 +-
+ drivers/gpio/gpio-pmic-eic-sprd.c                  |  58 ++++++++-
+ drivers/gpio/gpio-pxa.c                            |  10 +-
+ drivers/gpio/gpio-stmpe.c                          |   9 +-
+ drivers/gpio/gpio-syscon.c                         |   9 +-
+ drivers/gpio/gpio-ts4900.c                         |   7 +-
+ drivers/gpio/gpio-vf610.c                          |   4 +-
+ drivers/gpio/gpio-xlp.c                            |   9 +-
+ drivers/gpio/gpio-xra1403.c                        |  15 +--
+ drivers/gpio/gpio-zynq.c                           |  14 +-
+ drivers/gpio/gpiolib-of.c                          |   7 +-
+ drivers/gpio/gpiolib.c                             | 143 ++++++++++++++++++---
+ drivers/gpio/gpiolib.h                             |   2 +-
+ drivers/mfd/intel_quark_i2c_gpio.c                 |   3 +-
+ include/linux/gpio/consumer.h                      |  10 +-
+ include/linux/gpio/machine.h                       |  31 +++++
+ include/linux/platform_data/gpio-dwapb.h           |   3 +-
+ 53 files changed, 647 insertions(+), 291 deletions(-)
+Merging pinctrl/for-next (6c1b317ff0d8 Merge branch 'devel' into for-next)
+$ git merge pinctrl/for-next
+Removing drivers/pinctrl/samsung/pinctrl-exynos5440.c
+Auto-merging drivers/gpio/gpiolib.c
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ .../bindings/pinctrl/actions,s900-pinctrl.txt      |   16 +
+ .../bindings/pinctrl/allwinner,sunxi-pinctrl.txt   |    1 +
+ .../bindings/pinctrl/brcm,bcm2835-gpio.txt         |   18 +
+ .../devicetree/bindings/pinctrl/meson,pinctrl.txt  |    2 +
+ .../bindings/pinctrl/pinctrl-mcp23s08.txt          |    4 +-
+ .../devicetree/bindings/pinctrl/pinctrl-mt7622.txt |   10 +
+ .../bindings/pinctrl/renesas,pfc-pinctrl.txt       |    2 +
+ .../bindings/pinctrl/rockchip,pinctrl.txt          |    1 +
+ MAINTAINERS                                        |    3 +
+ drivers/gpio/gpiolib.c                             |   10 +
+ drivers/pinctrl/Kconfig                            |    1 +
+ drivers/pinctrl/Makefile                           |    1 +
+ drivers/pinctrl/actions/Kconfig                    |   15 +
+ drivers/pinctrl/actions/Makefile                   |    2 +
+ drivers/pinctrl/actions/pinctrl-owl.c              |  785 ++++++
+ drivers/pinctrl/actions/pinctrl-owl.h              |  162 ++
+ drivers/pinctrl/actions/pinctrl-s900.c             | 1888 ++++++++++++++
+ drivers/pinctrl/bcm/Kconfig                        |    1 +
+ drivers/pinctrl/bcm/pinctrl-bcm2835.c              |  100 +-
+ drivers/pinctrl/berlin/berlin-bg2.c                |    5 +-
+ drivers/pinctrl/berlin/berlin-bg2cd.c              |    5 +-
+ drivers/pinctrl/berlin/berlin-bg2q.c               |    5 +-
+ drivers/pinctrl/berlin/berlin-bg4ct.c              |   13 +-
+ drivers/pinctrl/berlin/berlin.c                    |    5 +-
+ drivers/pinctrl/berlin/berlin.h                    |    5 +-
+ drivers/pinctrl/freescale/pinctrl-imx.c            |   42 +-
+ drivers/pinctrl/freescale/pinctrl-imx.h            |    6 +-
+ drivers/pinctrl/freescale/pinctrl-imx1-core.c      |   27 +-
+ drivers/pinctrl/freescale/pinctrl-imx1.c           |   15 +-
+ drivers/pinctrl/freescale/pinctrl-imx1.h           |    6 +-
+ drivers/pinctrl/freescale/pinctrl-imx21.c          |   15 +-
+ drivers/pinctrl/freescale/pinctrl-imx23.c          |   19 +-
+ drivers/pinctrl/freescale/pinctrl-imx25.c          |   28 +-
+ drivers/pinctrl/freescale/pinctrl-imx27.c          |   19 +-
+ drivers/pinctrl/freescale/pinctrl-imx28.c          |   19 +-
+ drivers/pinctrl/freescale/pinctrl-imx35.c          |   24 +-
+ drivers/pinctrl/freescale/pinctrl-imx50.c          |   19 +-
+ drivers/pinctrl/freescale/pinctrl-imx51.c          |   21 +-
+ drivers/pinctrl/freescale/pinctrl-imx53.c          |   21 +-
+ drivers/pinctrl/freescale/pinctrl-imx6dl.c         |   16 +-
+ drivers/pinctrl/freescale/pinctrl-imx6q.c          |   21 +-
+ drivers/pinctrl/freescale/pinctrl-imx6sl.c         |   16 +-
+ drivers/pinctrl/freescale/pinctrl-imx6sll.c        |    8 +-
+ drivers/pinctrl/freescale/pinctrl-imx6sx.c         |   16 +-
+ drivers/pinctrl/freescale/pinctrl-imx6ul.c         |   16 +-
+ drivers/pinctrl/freescale/pinctrl-imx7d.c          |   16 +-
+ drivers/pinctrl/freescale/pinctrl-imx7ulp.c        |   17 +-
+ drivers/pinctrl/freescale/pinctrl-mxs.c            |   13 +-
+ drivers/pinctrl/freescale/pinctrl-mxs.h            |    8 +-
+ drivers/pinctrl/freescale/pinctrl-vf610.c          |   15 +-
+ drivers/pinctrl/intel/pinctrl-cherryview.c         |    4 +
+ drivers/pinctrl/mediatek/Kconfig                   |    6 +
+ drivers/pinctrl/mediatek/Makefile                  |    1 +
+ drivers/pinctrl/mediatek/mtk-eint.c                |  492 ++++
+ drivers/pinctrl/mediatek/mtk-eint.h                |  106 +
+ drivers/pinctrl/mediatek/pinctrl-mt2701.c          |   25 +-
+ drivers/pinctrl/mediatek/pinctrl-mt2712.c          |   25 +-
+ drivers/pinctrl/mediatek/pinctrl-mt7622.c          |  143 ++
+ drivers/pinctrl/mediatek/pinctrl-mt8127.c          |   25 +-
+ drivers/pinctrl/mediatek/pinctrl-mt8135.c          |   25 +-
+ drivers/pinctrl/mediatek/pinctrl-mt8173.c          |   25 +-
+ drivers/pinctrl/mediatek/pinctrl-mtk-common.c      |  608 +----
+ drivers/pinctrl/mediatek/pinctrl-mtk-common.h      |   13 +-
+ drivers/pinctrl/meson/pinctrl-meson-axg.c          |  107 +
+ drivers/pinctrl/meson/pinctrl-meson-gxbb.c         |    4 +-
+ drivers/pinctrl/meson/pinctrl-meson-gxl.c          |    4 +-
+ drivers/pinctrl/meson/pinctrl-meson8.c             |   23 +-
+ drivers/pinctrl/mvebu/pinctrl-armada-37xx.c        |   32 +-
+ drivers/pinctrl/mvebu/pinctrl-armada-xp.c          |   22 +-
+ drivers/pinctrl/pinctrl-at91-pio4.c                |    4 +-
+ drivers/pinctrl/pinctrl-rockchip.c                 |  246 ++
+ drivers/pinctrl/pinctrl-single.c                   |   72 +-
+ drivers/pinctrl/qcom/pinctrl-msm.c                 |   92 +-
+ drivers/pinctrl/qcom/pinctrl-qdf2xxx.c             |  114 +-
+ drivers/pinctrl/samsung/Kconfig                    |   10 +-
+ drivers/pinctrl/samsung/Makefile                   |    1 -
+ drivers/pinctrl/samsung/pinctrl-exynos-arm.c       |   30 +-
+ drivers/pinctrl/samsung/pinctrl-exynos-arm64.c     |   20 +
+ drivers/pinctrl/samsung/pinctrl-exynos.h           |    2 +-
+ drivers/pinctrl/samsung/pinctrl-exynos5440.c       | 1005 --------
+ drivers/pinctrl/samsung/pinctrl-samsung.c          |   29 +
+ drivers/pinctrl/sh-pfc/Kconfig                     |   10 +
+ drivers/pinctrl/sh-pfc/Makefile                    |    2 +
+ drivers/pinctrl/sh-pfc/core.c                      |   12 +
+ drivers/pinctrl/sh-pfc/pfc-r8a77470.c              | 2343 +++++++++++++++++
+ drivers/pinctrl/sh-pfc/pfc-r8a7795-es1.c           |    6 +-
+ drivers/pinctrl/sh-pfc/pfc-r8a7795.c               |    8 +-
+ drivers/pinctrl/sh-pfc/pfc-r8a7796.c               |    8 +-
+ drivers/pinctrl/sh-pfc/pfc-r8a77965.c              | 1592 +++++++++++-
+ drivers/pinctrl/sh-pfc/pfc-r8a77970.c              |   32 +-
+ drivers/pinctrl/sh-pfc/pfc-r8a77980.c              |   52 +-
+ drivers/pinctrl/sh-pfc/pfc-r8a77990.c              | 2695 ++++++++++++++++++++
+ drivers/pinctrl/sh-pfc/sh_pfc.h                    |   10 +-
+ drivers/pinctrl/sunxi/Kconfig                      |    4 +
+ drivers/pinctrl/sunxi/Makefile                     |    1 +
+ drivers/pinctrl/sunxi/pinctrl-sun50i-h6-r.c        |  128 +
+ drivers/pinctrl/tegra/pinctrl-tegra.c              |   11 -
+ drivers/pinctrl/tegra/pinctrl-tegra.h              |   11 +
+ drivers/pinctrl/tegra/pinctrl-tegra20.c            |   30 +-
+ drivers/pinctrl/uniphier/pinctrl-uniphier-ld11.c   |   49 +
+ drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c   |   54 +
+ include/dt-bindings/pinctrl/mt7623-pinfunc.h       |   90 +-
+ 102 files changed, 11832 insertions(+), 2134 deletions(-)
+ create mode 100644 drivers/pinctrl/actions/Kconfig
+ create mode 100644 drivers/pinctrl/actions/Makefile
+ create mode 100644 drivers/pinctrl/actions/pinctrl-owl.c
+ create mode 100644 drivers/pinctrl/actions/pinctrl-owl.h
+ create mode 100644 drivers/pinctrl/actions/pinctrl-s900.c
+ create mode 100644 drivers/pinctrl/mediatek/mtk-eint.c
+ create mode 100644 drivers/pinctrl/mediatek/mtk-eint.h
+ delete mode 100644 drivers/pinctrl/samsung/pinctrl-exynos5440.c
+ create mode 100644 drivers/pinctrl/sh-pfc/pfc-r8a77470.c
+ create mode 100644 drivers/pinctrl/sh-pfc/pfc-r8a77990.c
+ create mode 100644 drivers/pinctrl/sunxi/pinctrl-sun50i-h6-r.c
+Merging pinctrl-samsung/for-next (7c24e71c8a64 pinctrl: samsung: Fix suspend/resume for Exynos5433 GPF1..5 banks)
+$ git merge pinctrl-samsung/for-next
+Already up to date.
+Merging pwm/for-next (692099cdcf27 pwm: simplify getting .drvdata)
+$ git merge pwm/for-next
+Merge made by the 'recursive' strategy.
+ drivers/pwm/pwm-atmel-tcb.c | 6 ++----
+ drivers/pwm/pwm-meson.c     | 4 ++--
+ drivers/pwm/pwm-rcar.c      | 3 +--
+ 3 files changed, 5 insertions(+), 8 deletions(-)
+Merging userns/for-next (4b245b90065b Merge of my siginfo-next and userns-next trees for testing in linux-next)
+$ git merge userns/for-next
+Auto-merging security/integrity/evm/evm_crypto.c
+Auto-merging kernel/signal.c
+Auto-merging fs/namespace.c
+Auto-merging fs/namei.c
+Auto-merging fs/ioctl.c
+Auto-merging arch/x86/mm/fault.c
+Auto-merging arch/x86/kvm/mmu.c
+Auto-merging arch/powerpc/mm/fault.c
+Auto-merging arch/parisc/kernel/traps.c
+Auto-merging arch/arm64/mm/fault.c
+Auto-merging arch/arm64/kernel/fpsimd.c
+Auto-merging arch/arm/vfp/vfpmodule.c
+Auto-merging arch/arm/mm/fault.c
+CONFLICT (content): Merge conflict in arch/arm/mm/fault.c
+Auto-merging arch/arm/kernel/traps.c
+Auto-merging arch/arm/kernel/swp_emulate.c
+Resolved 'arch/arm/mm/fault.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master d8ecc069f9d7] Merge remote-tracking branch 'userns/for-next'
+$ git diff -M --stat --summary HEAD^..
+ arch/alpha/include/uapi/asm/siginfo.h     |  14 ---
+ arch/alpha/kernel/osf_sys.c               |  11 +-
+ arch/alpha/kernel/signal.c                |  20 +---
+ arch/alpha/kernel/traps.c                 |  79 ++++---------
+ arch/alpha/mm/fault.c                     |  13 +--
+ arch/arc/mm/fault.c                       |   2 +
+ arch/arm/kernel/ptrace.c                  |   1 +
+ arch/arm/kernel/swp_emulate.c             |   1 +
+ arch/arm/kernel/traps.c                   |   5 +
+ arch/arm/mm/alignment.c                   |   1 +
+ arch/arm/mm/fault.c                       |   4 +
+ arch/arm/vfp/vfpmodule.c                  |   3 +-
+ arch/arm64/kernel/fpsimd.c                |   2 +-
+ arch/arm64/kernel/sys_compat.c            |   1 +
+ arch/arm64/kernel/traps.c                 |   1 +
+ arch/arm64/mm/fault.c                     |  18 ++-
+ arch/c6x/kernel/traps.c                   |   9 +-
+ arch/hexagon/kernel/traps.c               |   9 +-
+ arch/hexagon/mm/vm_fault.c                |  20 ++--
+ arch/ia64/include/uapi/asm/siginfo.h      |   7 --
+ arch/ia64/kernel/brl_emu.c                |   1 +
+ arch/ia64/kernel/signal.c                 |   2 +
+ arch/ia64/kernel/traps.c                  |  31 ++++-
+ arch/ia64/kernel/unaligned.c              |   1 +
+ arch/ia64/mm/fault.c                      |   4 +-
+ arch/m68k/kernel/traps.c                  |  58 +++++-----
+ arch/m68k/mm/fault.c                      |  25 ++---
+ arch/microblaze/kernel/exceptions.c       |   8 +-
+ arch/microblaze/mm/fault.c                |  12 +-
+ arch/mips/kernel/traps.c                  |  65 +++--------
+ arch/mips/mm/fault.c                      |  18 +--
+ arch/nds32/kernel/traps.c                 |  35 ++----
+ arch/nds32/mm/fault.c                     |  18 +--
+ arch/nios2/kernel/traps.c                 |   8 +-
+ arch/openrisc/kernel/traps.c              |  30 +----
+ arch/openrisc/mm/fault.c                  |  18 +--
+ arch/parisc/kernel/ptrace.c               |  10 +-
+ arch/parisc/kernel/traps.c                |  61 ++++------
+ arch/parisc/kernel/unaligned.c            |  15 +--
+ arch/parisc/math-emu/driver.c             |   8 +-
+ arch/parisc/mm/fault.c                    |  54 +++++----
+ arch/powerpc/include/uapi/asm/siginfo.h   |  15 ---
+ arch/powerpc/kernel/process.c             |   1 +
+ arch/powerpc/kernel/traps.c               |  13 +--
+ arch/powerpc/mm/fault.c                   |   1 +
+ arch/powerpc/platforms/cell/spufs/fault.c |   2 +-
+ arch/riscv/kernel/traps.c                 |  16 +--
+ arch/s390/kernel/traps.c                  |  29 +----
+ arch/s390/mm/fault.c                      |  21 +---
+ arch/sh/kernel/hw_breakpoint.c            |   9 +-
+ arch/sh/kernel/traps_32.c                 |  20 ++--
+ arch/sh/math-emu/math.c                   |   8 +-
+ arch/sh/mm/fault.c                        |   9 +-
+ arch/sparc/include/uapi/asm/siginfo.h     |   7 --
+ arch/sparc/kernel/process_64.c            |   9 +-
+ arch/sparc/kernel/sys_sparc_32.c          |   8 +-
+ arch/sparc/kernel/sys_sparc_64.c          |   8 +-
+ arch/sparc/kernel/traps_32.c              | 104 ++++-------------
+ arch/sparc/kernel/traps_64.c              | 131 +++++----------------
+ arch/sparc/kernel/unaligned_32.c          |  11 +-
+ arch/sparc/mm/fault_32.c                  |  12 +-
+ arch/sparc/mm/fault_64.c                  |   8 +-
+ arch/um/kernel/ptrace.c                   |  13 +--
+ arch/um/kernel/trap.c                     |  62 ++++------
+ arch/unicore32/kernel/fpu-ucf64.c         |   8 +-
+ arch/unicore32/mm/fault.c                 |   3 +
+ arch/x86/entry/vsyscall/vsyscall_64.c     |   2 +-
+ arch/x86/kernel/ptrace.c                  |   2 +-
+ arch/x86/kernel/signal_compat.c           |   2 +-
+ arch/x86/kernel/traps.c                   |   3 +
+ arch/x86/kernel/umip.c                    |   1 +
+ arch/x86/kvm/mmu.c                        |   1 +
+ arch/x86/mm/fault.c                       |   1 +
+ arch/xtensa/kernel/traps.c                |   9 +-
+ arch/xtensa/mm/fault.c                    |  18 +--
+ fs/attr.c                                 |  36 ++++--
+ fs/ioctl.c                                |   4 +-
+ fs/namei.c                                |  16 ++-
+ fs/namespace.c                            |   4 +-
+ fs/signalfd.c                             |  93 +++++++--------
+ include/linux/ptrace.h                    |   1 -
+ include/linux/signal.h                    |   3 +
+ include/linux/tracehook.h                 |   1 +
+ include/uapi/asm-generic/siginfo.h        |   3 +-
+ include/uapi/linux/signalfd.h             |   6 +-
+ kernel/signal.c                           | 181 +++++++++---------------------
+ security/commoncap.c                      |   8 +-
+ security/integrity/evm/evm_crypto.c       |   3 +-
+ virt/kvm/arm/mmu.c                        |   1 +
+ 89 files changed, 545 insertions(+), 1085 deletions(-)
+Merging ktest/for-next (6cd110a91f52 ktest: Take submenu into account for grub2 menus)
+$ git merge ktest/for-next
+Already up to date.
+Merging random/dev (9e66317d3c92 Linux 4.14-rc3)
+$ git merge random/dev
+Already up to date.
+Merging aio/master (2a8a98673c13 fs: aio: fix the increment of aio-nr and counting against aio-max-nr)
+$ git merge aio/master
+Already up to date.
+Merging kselftest/next (fa32156921da selftests: lib: fix prime_numbers module search and skip logic)
+$ git merge kselftest/next
+Removing tools/testing/selftests/timers/rtctest.c
+Auto-merging tools/testing/selftests/net/rtnetlink.sh
+Auto-merging tools/testing/selftests/net/pmtu.sh
+Auto-merging tools/testing/selftests/net/fib_tests.sh
+Auto-merging tools/testing/selftests/memfd/run_hugetlbfs_test.sh
+Auto-merging MAINTAINERS
+Merge made by the 'recursive' strategy.
+ MAINTAINERS                                        |    2 +-
+ tools/testing/selftests/Makefile                   |    5 +-
+ tools/testing/selftests/android/Makefile           |    8 -
+ tools/testing/selftests/android/ion/ion_test.sh    |    7 +-
+ .../breakpoints/step_after_suspend_test.c          |    6 +-
+ tools/testing/selftests/cgroup/Makefile            |   10 +
+ tools/testing/selftests/cgroup/cgroup_util.c       |  331 +++++++
+ tools/testing/selftests/cgroup/cgroup_util.h       |   41 +
+ tools/testing/selftests/cgroup/test_memcontrol.c   | 1015 ++++++++++++++++++++
+ .../selftests/cpu-hotplug/cpu-on-off-test.sh       |   14 +-
+ tools/testing/selftests/cpufreq/main.sh            |    5 +-
+ tools/testing/selftests/efivarfs/efivarfs.sh       |    7 +-
+ tools/testing/selftests/exec/execveat.c            |    6 +-
+ tools/testing/selftests/filesystems/Makefile       |    1 +
+ tools/testing/selftests/filesystems/devpts_pts.c   |   15 +-
+ tools/testing/selftests/firmware/fw_fallback.sh    |    4 +-
+ tools/testing/selftests/firmware/fw_filesystem.sh  |    4 +-
+ tools/testing/selftests/firmware/fw_lib.sh         |    7 +-
+ tools/testing/selftests/futex/Makefile             |   12 -
+ tools/testing/selftests/gpio/gpio-mockup.sh        |   12 +-
+ tools/testing/selftests/intel_pstate/aperf.c       |    6 +-
+ tools/testing/selftests/intel_pstate/run.sh        |   47 +-
+ tools/testing/selftests/ipc/msgque.c               |    7 +-
+ tools/testing/selftests/kmod/kmod.sh               |   13 +-
+ tools/testing/selftests/kselftest.h                |    2 +-
+ tools/testing/selftests/kvm/.gitignore             |    3 +
+ tools/testing/selftests/kvm/lib/assert.c           |    9 +-
+ tools/testing/selftests/kvm/vmx_tsc_adjust_test.c  |    2 +
+ tools/testing/selftests/lib.mk                     |   55 +-
+ tools/testing/selftests/lib/Makefile               |    2 +-
+ tools/testing/selftests/lib/bitmap.sh              |    8 +-
+ tools/testing/selftests/lib/prime_numbers.sh       |    9 +-
+ tools/testing/selftests/lib/printf.sh              |    8 +-
+ tools/testing/selftests/locking/Makefile           |   10 +
+ tools/testing/selftests/locking/ww_mutex.sh        |    8 +
+ tools/testing/selftests/media_tests/Makefile       |    3 +-
+ .../selftests/media_tests/media_device_open.c      |    8 +-
+ .../selftests/media_tests/media_device_test.c      |   10 +-
+ .../testing/selftests/membarrier/membarrier_test.c |    7 +-
+ tools/testing/selftests/memfd/Makefile             |    6 +-
+ .../memfd/{run_tests.sh => run_hugetlbfs_test.sh}  |   18 +-
+ tools/testing/selftests/memory-hotplug/Makefile    |    5 +-
+ .../selftests/memory-hotplug/mem-on-off-test.sh    |   14 +-
+ tools/testing/selftests/mount/Makefile             |   12 +-
+ tools/testing/selftests/mount/run_tests.sh         |   12 +
+ tools/testing/selftests/mqueue/Makefile            |   12 +-
+ tools/testing/selftests/mqueue/mq_open_tests.c     |   37 +-
+ tools/testing/selftests/mqueue/mq_perf_tests.c     |    8 +-
+ tools/testing/selftests/net/fib_tests.sh           |    8 +-
+ tools/testing/selftests/net/netdevice.sh           |   16 +-
+ tools/testing/selftests/net/pmtu.sh                |    5 +-
+ tools/testing/selftests/net/psock_tpacket.c        |    4 +-
+ tools/testing/selftests/net/rtnetlink.sh           |   31 +-
+ tools/testing/selftests/rtc/.gitignore             |    2 +
+ tools/testing/selftests/rtc/Makefile               |    9 +
+ tools/testing/selftests/rtc/rtctest.c              |  238 +++++
+ .../{timers/rtctest_setdate.c => rtc/setdate.c}    |    0
+ tools/testing/selftests/timers/.gitignore          |    3 +-
+ tools/testing/selftests/timers/Makefile            |    4 +-
+ tools/testing/selftests/timers/rtcpie.c            |  134 +++
+ tools/testing/selftests/timers/rtctest.c           |  403 --------
+ tools/testing/selftests/x86/Makefile               |    8 +-
+ tools/testing/selftests/x86/trivial_program.c      |   10 +
+ 63 files changed, 2113 insertions(+), 615 deletions(-)
+ create mode 100644 tools/testing/selftests/cgroup/Makefile
+ create mode 100644 tools/testing/selftests/cgroup/cgroup_util.c
+ create mode 100644 tools/testing/selftests/cgroup/cgroup_util.h
+ create mode 100644 tools/testing/selftests/cgroup/test_memcontrol.c
+ create mode 100644 tools/testing/selftests/kvm/.gitignore
+ create mode 100644 tools/testing/selftests/locking/Makefile
+ mode change 100644 => 100755 tools/testing/selftests/locking/ww_mutex.sh
+ rename tools/testing/selftests/memfd/{run_tests.sh => run_hugetlbfs_test.sh} (88%)
+ create mode 100755 tools/testing/selftests/mount/run_tests.sh
+ create mode 100644 tools/testing/selftests/rtc/.gitignore
+ create mode 100644 tools/testing/selftests/rtc/Makefile
+ create mode 100644 tools/testing/selftests/rtc/rtctest.c
+ rename tools/testing/selftests/{timers/rtctest_setdate.c => rtc/setdate.c} (100%)
+ create mode 100644 tools/testing/selftests/timers/rtcpie.c
+ delete mode 100644 tools/testing/selftests/timers/rtctest.c
+ create mode 100644 tools/testing/selftests/x86/trivial_program.c
+Merging y2038/y2038 (5dc0b1529d21 y2038: ipc: Redirect ipc(SEMTIMEDOP, ...) to compat_ksys_semtimedop)
+$ git merge y2038/y2038
+Already up to date.
+Merging livepatching/for-next (d7ab8e67aa52 Merge branch 'for-4.18/upstream' into for-next)
+$ git merge livepatching/for-next
+Merge made by the 'recursive' strategy.
+ Documentation/livepatch/livepatch.txt | 24 ------------------------
+ 1 file changed, 24 deletions(-)
+Merging coresight/next (ec1df1673899 coresight: Remove function coresight_vpid_to_pid())
+$ git merge coresight/next
+Auto-merging include/linux/coresight.h
+Auto-merging drivers/hwtracing/coresight/coresight-etm4x.h
+Auto-merging drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
+Auto-merging drivers/hwtracing/coresight/coresight-etm3x.c
+Auto-merging drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
+Auto-merging drivers/hwtracing/coresight/coresight-etm.h
+Merge made by the 'recursive' strategy.
+ drivers/hwtracing/coresight/coresight-etm.h        |  3 --
+ .../hwtracing/coresight/coresight-etm3x-sysfs.c    | 43 +++++++++++++++++---
+ drivers/hwtracing/coresight/coresight-etm3x.c      |  4 +-
+ .../hwtracing/coresight/coresight-etm4x-sysfs.c    | 47 +++++++++++++++++-----
+ drivers/hwtracing/coresight/coresight-etm4x.h      |  3 --
+ include/linux/coresight.h                          | 20 ---------
+ 6 files changed, 76 insertions(+), 44 deletions(-)
+Merging rtc/rtc-next (b22e568f658f Fix a typo in RTC documentation for RTC_DRV_JZ4740)
+$ git merge rtc/rtc-next
+Merge made by the 'recursive' strategy.
+ .../devicetree/bindings/rtc/nxp,rtc-2123.txt       |   2 +-
+ .../devicetree/bindings/rtc/st,stm32-rtc.txt       |  37 ++-
+ drivers/rtc/Kconfig                                |   2 +-
+ drivers/rtc/nvmem.c                                |   2 +-
+ drivers/rtc/rtc-88pm80x.c                          |  58 ++--
+ drivers/rtc/rtc-ab-b5ze-s3.c                       |  30 +-
+ drivers/rtc/rtc-at91rm9200.c                       |   2 +
+ drivers/rtc/rtc-bq4802.c                           |   6 +-
+ drivers/rtc/rtc-brcmstb-waketimer.c                |  20 +-
+ drivers/rtc/rtc-cmos.c                             | 155 ++++++++--
+ drivers/rtc/rtc-ds1216.c                           |   6 +-
+ drivers/rtc/rtc-ds1307.c                           |   2 +
+ drivers/rtc/rtc-ds1511.c                           |   9 +-
+ drivers/rtc/rtc-ds1553.c                           |  15 +-
+ drivers/rtc/rtc-ds1685.c                           |  21 +-
+ drivers/rtc/rtc-ds1742.c                           |   8 +-
+ drivers/rtc/rtc-lpc32xx.c                          |  16 +-
+ drivers/rtc/rtc-ls1x.c                             |  32 +-
+ drivers/rtc/rtc-m48t59.c                           |  41 +--
+ drivers/rtc/rtc-mv.c                               |   3 +-
+ drivers/rtc/rtc-mxc.c                              |  34 +--
+ drivers/rtc/rtc-mxc_v2.c                           |  29 +-
+ drivers/rtc/rtc-pcap.c                             |  15 +-
+ drivers/rtc/rtc-rx8581.c                           | 210 ++++---------
+ drivers/rtc/rtc-sc27xx.c                           |  16 +-
+ drivers/rtc/rtc-sh.c                               |  15 +-
+ drivers/rtc/rtc-snvs.c                             |  13 +-
+ drivers/rtc/rtc-st-lpc.c                           |  26 +-
+ drivers/rtc/rtc-stk17ta8.c                         |  15 +-
+ drivers/rtc/rtc-stm32.c                            | 339 +++++++++++++++------
+ drivers/rtc/rtc-test.c                             |  22 +-
+ drivers/rtc/rtc-tps6586x.c                         |  55 ++--
+ drivers/rtc/rtc-tps65910.c                         |  26 +-
+ drivers/rtc/rtc-vr41xx.c                           |  35 ++-
+ drivers/rtc/rtc-zynqmp.c                           |  10 +-
+ include/linux/rtc.h                                |   2 +-
+ 36 files changed, 692 insertions(+), 637 deletions(-)
+Merging nvdimm/libnvdimm-for-next (1dcdf20fcf9c Merge branch 'for-4.18/mcsafe' into libnvdimm-for-next)
+$ git merge nvdimm/libnvdimm-for-next
+Auto-merging mm/hmm.c
+CONFLICT (content): Merge conflict in mm/hmm.c
+Auto-merging mm/gup.c
+Auto-merging mm/Kconfig
+Auto-merging lib/Kconfig
+Auto-merging include/linux/mm.h
+Auto-merging include/linux/memremap.h
+Auto-merging fs/xfs/xfs_iops.c
+Auto-merging fs/xfs/xfs_ioctl.c
+Auto-merging fs/xfs/xfs_inode.h
+Auto-merging fs/xfs/xfs_file.c
+Auto-merging fs/dax.c
+Auto-merging fs/Kconfig
+Auto-merging drivers/md/dm.c
+Resolved 'mm/hmm.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master b7e5d50281e7] Merge remote-tracking branch 'nvdimm/libnvdimm-for-next'
+$ git diff -M --stat --summary HEAD^..
+ arch/x86/Kconfig.debug             |   3 +
+ arch/x86/include/asm/mcsafe_test.h |  75 +++++++++++++
+ arch/x86/lib/memcpy_64.S           |  10 ++
+ drivers/dax/super.c                |  24 ++++-
+ drivers/md/dm-linear.c             |  16 +++
+ drivers/md/dm-log-writes.c         |  15 +++
+ drivers/md/dm-stripe.c             |  21 ++++
+ drivers/md/dm.c                    |  25 +++++
+ drivers/nvdimm/pfn_devs.c          |   2 -
+ drivers/nvdimm/pmem.c              |  32 ++++++
+ drivers/s390/block/dcssblk.c       |   7 ++
+ fs/Kconfig                         |   1 +
+ fs/dax.c                           | 118 +++++++++++++++++++--
+ fs/xfs/xfs_file.c                  |  72 +++++++++++--
+ fs/xfs/xfs_inode.h                 |  16 +++
+ fs/xfs/xfs_ioctl.c                 |   8 +-
+ fs/xfs/xfs_iops.c                  |  16 +--
+ fs/xfs/xfs_pnfs.c                  |  15 +--
+ fs/xfs/xfs_pnfs.h                  |   5 +-
+ include/linux/dax.h                |  12 +++
+ include/linux/device-mapper.h      |   5 +-
+ include/linux/memremap.h           |  36 ++-----
+ include/linux/mm.h                 |  71 +++++++++----
+ include/linux/uio.h                |   2 +-
+ kernel/Makefile                    |   3 +-
+ kernel/iomem.c                     | 167 +++++++++++++++++++++++++++++
+ kernel/memremap.c                  | 210 ++++++-------------------------------
+ lib/Kconfig                        |   3 +
+ mm/Kconfig                         |   5 +
+ mm/gup.c                           |  36 +++++--
+ mm/hmm.c                           |  13 +--
+ mm/swap.c                          |   3 +-
+ tools/testing/nvdimm/test/nfit.c   | 104 ++++++++++++++++++
+ 33 files changed, 860 insertions(+), 291 deletions(-)
+ create mode 100644 arch/x86/include/asm/mcsafe_test.h
+ create mode 100644 kernel/iomem.c
+Merging at24/at24/for-next (73b0d922cd5b eeprom: at24: provide a separate routine for creating dummy i2c clients)
+$ git merge at24/at24/for-next
+Already up to date.
+Merging ntb/ntb-next (5e083738f65d ntb: ntb_transport: Replace GFP_ATOMIC with GFP_KERNEL in ntb_transport_create_queue)
+$ git merge ntb/ntb-next
+Auto-merging drivers/ntb/hw/intel/ntb_hw_gen1.c
+Merge made by the 'recursive' strategy.
+ drivers/ntb/hw/idt/ntb_hw_idt.c                    |   2 +-
+ drivers/ntb/hw/intel/Makefile                      |   1 +
+ .../ntb/hw/intel/{ntb_hw_intel.c => ntb_hw_gen1.c} | 713 ++-------------------
+ drivers/ntb/hw/intel/ntb_hw_gen1.h                 | 182 ++++++
+ drivers/ntb/hw/intel/ntb_hw_gen3.c                 | 597 +++++++++++++++++
+ drivers/ntb/hw/intel/ntb_hw_gen3.h                 | 110 ++++
+ drivers/ntb/hw/intel/ntb_hw_intel.h                | 203 ++----
+ drivers/ntb/ntb_transport.c                        |   6 +-
+ 8 files changed, 1001 insertions(+), 813 deletions(-)
+ rename drivers/ntb/hw/intel/{ntb_hw_intel.c => ntb_hw_gen1.c} (74%)
+ create mode 100644 drivers/ntb/hw/intel/ntb_hw_gen1.h
+ create mode 100644 drivers/ntb/hw/intel/ntb_hw_gen3.c
+ create mode 100644 drivers/ntb/hw/intel/ntb_hw_gen3.h
+Merging kspp/for-next/kspp (45888b40d2a6 rslib: Allocate decoder buffers to avoid VLAs)
+$ git merge kspp/for-next/kspp
+Auto-merging drivers/mtd/nand/raw/diskonchip.c
+Auto-merging drivers/md/dm-verity-fec.c
+Merge made by the 'recursive' strategy.
+ drivers/md/dm-verity-fec.c        |   2 +-
+ drivers/mtd/nand/raw/cafe_nand.c  |   7 +-
+ drivers/mtd/nand/raw/diskonchip.c |  67 +++++------
+ include/linux/rslib.h             |  74 +++++++-----
+ lib/reed_solomon/decode_rs.c      |  34 +++---
+ lib/reed_solomon/encode_rs.c      |  15 +--
+ lib/reed_solomon/reed_solomon.c   | 240 ++++++++++++++++++++++----------------
+ 7 files changed, 243 insertions(+), 196 deletions(-)
+Merging init_task/init_task (e1e871aff3de Expand INIT_STRUCT_PID and remove)
+$ git merge init_task/init_task
+Already up to date.
+Merging cisco/for-next (5997aab0a11e Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs)
+$ git merge cisco/for-next
+Already up to date.
+Merging akpm-current/current (06a5af82ba13 ipc: use new return type vm_fault_t)
+$ git merge --no-ff akpm-current/current
+Auto-merging security/selinux/hooks.c
+Auto-merging security/apparmor/lsm.c
+Auto-merging scripts/tags.sh
+Auto-merging scripts/checkpatch.pl
+Auto-merging mm/vmalloc.c
+Auto-merging mm/page_alloc.c
+Auto-merging mm/mmap.c
+Auto-merging mm/memcontrol.c
+Auto-merging mm/ksm.c
+Auto-merging mm/hugetlb.c
+Auto-merging mm/huge_memory.c
+Auto-merging mm/hmm.c
+Auto-merging mm/gup.c
+Auto-merging mm/backing-dev.c
+Auto-merging mm/Kconfig
+Auto-merging lib/Kconfig.debug
+Auto-merging kernel/sched/core.c
+Auto-merging kernel/memremap.c
+Auto-merging kernel/fork.c
+Auto-merging kernel/cgroup/cgroup.c
+Auto-merging ipc/shm.c
+Auto-merging ipc/sem.c
+Auto-merging include/linux/swap.h
+Auto-merging include/linux/sched/mm.h
+Auto-merging include/linux/sched.h
+Auto-merging include/linux/mm.h
+Auto-merging include/linux/memremap.h
+Auto-merging include/linux/kernel.h
+Auto-merging include/linux/hmm.h
+Auto-merging include/linux/fsnotify_backend.h
+Auto-merging include/linux/fs.h
+Auto-merging include/linux/dax.h
+Auto-merging include/linux/cgroup-defs.h
+Auto-merging fs/proc/task_mmu.c
+Auto-merging fs/proc/internal.h
+Auto-merging fs/proc/fd.c
+Auto-merging fs/proc/base.c
+CONFLICT (content): Merge conflict in fs/proc/base.c
+Auto-merging fs/proc/array.c
+Auto-merging fs/ocfs2/file.c
+Auto-merging fs/notify/inotify/inotify_user.c
+Auto-merging fs/notify/inotify/inotify_fsnotify.c
+Auto-merging fs/notify/group.c
+Auto-merging fs/notify/fanotify/fanotify_user.c
+Auto-merging fs/notify/fanotify/fanotify.c
+Auto-merging fs/notify/dnotify/dnotify.c
+Auto-merging fs/namei.c
+Auto-merging fs/fcntl.c
+Auto-merging fs/exofs/super.c
+Auto-merging fs/exofs/ore.c
+Auto-merging fs/exec.c
+Auto-merging fs/dcache.c
+Auto-merging fs/dax.c
+Auto-merging fs/autofs/waitq.c
+Auto-merging fs/autofs/symlink.c
+Auto-merging fs/autofs/root.c
+Auto-merging fs/autofs/inode.c
+Auto-merging fs/autofs/init.c
+Auto-merging fs/autofs/expire.c
+Auto-merging fs/autofs/dev-ioctl.c
+Auto-merging fs/autofs/autofs_i.h
+Auto-merging fs/Kconfig
+Auto-merging block/genhd.c
+Auto-merging arch/x86/Kconfig
+Auto-merging arch/sparc/Kconfig
+Auto-merging arch/sh/Kconfig
+CONFLICT (content): Merge conflict in arch/sh/Kconfig
+Auto-merging arch/s390/include/asm/pgtable.h
+Auto-merging arch/s390/Kconfig
+Auto-merging arch/riscv/Kconfig
+Auto-merging arch/powerpc/include/asm/pte-common.h
+Auto-merging arch/powerpc/include/asm/book3s/64/pgtable.h
+Auto-merging arch/powerpc/Kconfig
+Auto-merging arch/arm64/Kconfig
+Auto-merging arch/arm/boot/compressed/Makefile
+Auto-merging arch/arm/Kconfig
+Auto-merging arch/arc/Kconfig
+Auto-merging MAINTAINERS
+Auto-merging Documentation/vm/hmm.rst
+Auto-merging Documentation/filesystems/autofs.txt
+Auto-merging Documentation/filesystems/autofs-mount-control.txt
+Auto-merging Documentation/features/vm/pte_special/arch-support.txt
+Auto-merging Documentation/admin-guide/cgroup-v2.rst
+Resolved 'arch/sh/Kconfig' using previous resolution.
+Resolved 'fs/proc/base.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit -v -a
+[master 5cccae69f3e7] Merge branch 'akpm-current/current'
+$ git diff -M --stat --summary HEAD^..
+ Documentation/ABI/testing/sysfs-class-bdi          |   8 +
+ Documentation/admin-guide/cgroup-v2.rst            | 146 +++-
+ Documentation/blockdev/zram.txt                    |  25 +
+ .../features/vm/pte_special/arch-support.txt       |   2 +-
+ Documentation/filesystems/00-INDEX                 |   4 +-
+ ...-mount-control.txt => autofs-mount-control.txt} |   8 +-
+ .../filesystems/{autofs4.txt => autofs.txt}        |  10 +-
+ Documentation/filesystems/automount-support.txt    |   2 +-
+ Documentation/filesystems/path-lookup.md           |   2 +-
+ Documentation/sysctl/fs.txt                        |  36 +
+ Documentation/vm/hmm.rst                           |   1 -
+ MAINTAINERS                                        |   4 +-
+ arch/arc/Kconfig                                   |   1 +
+ arch/arc/include/asm/pgtable.h                     |   2 -
+ arch/arm/Kconfig                                   |   4 +-
+ arch/arm/boot/compressed/Makefile                  |   3 +
+ arch/arm/include/asm/page.h                        |   2 +
+ arch/arm/include/asm/pgtable-3level.h              |   1 -
+ arch/arm/kvm/hyp/Makefile                          |   8 +
+ arch/arm/vdso/Makefile                             |   3 +
+ arch/arm64/Kconfig                                 |   1 +
+ arch/arm64/include/asm/pgtable.h                   |   2 -
+ arch/powerpc/Kconfig                               |   1 +
+ arch/powerpc/include/asm/book3s/64/pgtable.h       |   3 -
+ arch/powerpc/include/asm/pte-common.h              |   3 -
+ arch/riscv/Kconfig                                 |   1 +
+ arch/riscv/include/asm/pgtable-bits.h              |   3 -
+ arch/s390/Kconfig                                  |   1 +
+ arch/s390/include/asm/pgtable.h                    |   1 -
+ arch/s390/mm/pgalloc.c                             |  21 +-
+ arch/sh/Kconfig                                    |   5 +-
+ arch/sh/include/asm/pgtable.h                      |   2 -
+ arch/sparc/Kconfig                                 |   1 +
+ arch/sparc/include/asm/pgtable_64.h                |   3 -
+ arch/x86/Kconfig                                   |   1 +
+ arch/x86/include/asm/pgtable_types.h               |   1 -
+ arch/x86/mm/pgtable.c                              |   5 +-
+ block/genhd.c                                      |   2 +-
+ drivers/block/zram/Kconfig                         |  14 +-
+ drivers/block/zram/zram_drv.c                      | 165 ++++-
+ drivers/block/zram/zram_drv.h                      |  14 +-
+ fs/9p/v9fs.c                                       |  29 +-
+ fs/Kconfig                                         |   4 +
+ fs/Makefile                                        |   1 +
+ fs/autofs/Kconfig                                  |  20 +
+ fs/autofs/Makefile                                 |   7 +
+ fs/{autofs4 => autofs}/autofs_i.h                  |  92 +--
+ fs/{autofs4 => autofs}/dev-ioctl.c                 |  31 +-
+ fs/{autofs4 => autofs}/expire.c                    | 132 ++--
+ fs/{autofs4 => autofs}/init.c                      |  12 +-
+ fs/{autofs4 => autofs}/inode.c                     |  54 +-
+ fs/{autofs4 => autofs}/root.c                      | 277 ++++----
+ fs/{autofs4 => autofs}/symlink.c                   |  16 +-
+ fs/{autofs4 => autofs}/waitq.c                     |  57 +-
+ fs/autofs4/Kconfig                                 |  46 +-
+ fs/autofs4/Makefile                                |   4 +-
+ fs/bfs/inode.c                                     |   9 +-
+ fs/binfmt_elf.c                                    |  16 +-
+ fs/compat_ioctl.c                                  |   2 -
+ fs/dax.c                                           |  78 +-
+ fs/dcache.c                                        |   3 +-
+ fs/exec.c                                          |   4 +-
+ fs/exofs/ore.c                                     |  84 ++-
+ fs/exofs/ore_raid.c                                |  75 +-
+ fs/exofs/super.c                                   |  23 +-
+ fs/fcntl.c                                         |   2 +-
+ fs/namei.c                                         |  53 +-
+ fs/notify/dnotify/dnotify.c                        |   5 +-
+ fs/notify/fanotify/fanotify.c                      |   6 +-
+ fs/notify/fanotify/fanotify_user.c                 |   5 +-
+ fs/notify/group.c                                  |   4 +
+ fs/notify/inotify/inotify_fsnotify.c               |   2 +-
+ fs/notify/inotify/inotify_user.c                   |   5 +-
+ fs/ocfs2/buffer_head_io.c                          |  77 +-
+ fs/ocfs2/dlmglue.c                                 | 121 +++-
+ fs/ocfs2/dlmglue.h                                 |   1 +
+ fs/ocfs2/file.c                                    |  14 +-
+ fs/ocfs2/file.h                                    |   2 -
+ fs/ocfs2/ioctl.c                                   |   2 +-
+ fs/ocfs2/journal.c                                 |  51 +-
+ fs/ocfs2/namei.c                                   |   3 +-
+ fs/ocfs2/ocfs2_fs.h                                |   8 +-
+ fs/ocfs2/stackglue.c                               |   6 -
+ fs/ocfs2/stackglue.h                               |   3 -
+ fs/proc/array.c                                    |   2 +-
+ fs/proc/base.c                                     | 143 ++--
+ fs/proc/fd.c                                       |   2 +-
+ fs/proc/internal.h                                 |   2 +-
+ fs/proc/page.c                                     |   2 +
+ fs/proc/task_mmu.c                                 |  26 +-
+ fs/userfaultfd.c                                   |  22 +-
+ include/asm-generic/int-ll64.h                     |  19 +-
+ include/linux/cgroup-defs.h                        |   5 +
+ include/linux/dax.h                                |   4 +-
+ include/linux/fs.h                                 |   2 +
+ include/linux/fsnotify_backend.h                   |  12 +-
+ include/linux/gfp.h                                |  10 +-
+ include/linux/hmm.h                                |  12 +-
+ include/linux/kcov.h                               |  14 +
+ include/linux/kernel.h                             |   1 +
+ include/linux/ksm.h                                |  17 -
+ include/linux/memcontrol.h                         |  97 ++-
+ include/linux/memfd.h                              |  16 +
+ include/linux/memory_hotplug.h                     |   1 -
+ include/linux/memremap.h                           |   1 +
+ include/linux/mm.h                                 |  10 +-
+ include/linux/mm_types.h                           | 251 +++----
+ include/linux/oom.h                                |  12 +-
+ include/linux/page-flags.h                         |  51 +-
+ include/linux/page_counter.h                       |  26 +-
+ include/linux/pfn_t.h                              |   4 +-
+ include/linux/sched.h                              |   5 +-
+ include/linux/sched/mm.h                           |  36 +-
+ include/linux/sched/signal.h                       |   3 +
+ include/linux/shmem_fs.h                           |  13 -
+ include/linux/slab.h                               |  61 ++
+ include/linux/slab_def.h                           |   5 +-
+ include/linux/slub_def.h                           |   4 +-
+ include/linux/swap.h                               |  13 +-
+ include/linux/types.h                              |  34 +-
+ include/linux/userfaultfd_k.h                      |   6 +-
+ include/uapi/linux/auto_fs.h                       | 169 ++++-
+ include/uapi/linux/auto_fs4.h                      | 153 +---
+ include/uapi/linux/kernel-page-flags.h             |   2 +-
+ include/uapi/linux/prctl.h                         |   4 +
+ ipc/sem.c                                          |  18 +-
+ ipc/shm.c                                          |   2 +-
+ kernel/cgroup/cgroup.c                             |  13 +-
+ kernel/crash_core.c                                |   1 +
+ kernel/cred.c                                      |   1 +
+ kernel/exit.c                                      |  93 +--
+ kernel/fork.c                                      |  30 +-
+ kernel/hung_task.c                                 |  11 +-
+ kernel/kcov.c                                      |  21 +-
+ kernel/memremap.c                                  |  39 +-
+ kernel/relay.c                                     |   2 +-
+ kernel/sched/core.c                                |   4 +
+ kernel/sys.c                                       |  21 +-
+ kernel/sysctl.c                                    |  18 +
+ lib/Kconfig.debug                                  |  36 +-
+ lib/bitmap.c                                       |   5 +-
+ lib/idr.c                                          |  10 +-
+ lib/nmi_backtrace.c                                |   5 +-
+ lib/percpu_ida.c                                   |  63 +-
+ mm/Kconfig                                         |   3 +
+ mm/Makefile                                        |   1 +
+ mm/backing-dev.c                                   |  39 +-
+ mm/debug.c                                         |   4 +-
+ mm/filemap.c                                       |   8 +-
+ mm/gup.c                                           |  42 +-
+ mm/hmm.c                                           | 297 ++------
+ mm/huge_memory.c                                   |   7 +-
+ mm/hugetlb.c                                       |  62 +-
+ mm/hugetlb_cgroup.c                                |   6 +-
+ mm/init-mm.c                                       |   1 +
+ mm/ksm.c                                           |  25 +-
+ mm/list_lru.c                                      |   7 +
+ mm/memblock.c                                      |  27 +-
+ mm/memcontrol.c                                    | 783 +++++++++++++++++----
+ mm/memfd.c                                         | 345 +++++++++
+ mm/memory.c                                        |  40 +-
+ mm/memory_hotplug.c                                |  23 +
+ mm/mincore.c                                       |  12 +-
+ mm/mmap.c                                          |   4 +-
+ mm/nommu.c                                         |   2 +-
+ mm/oom_kill.c                                      | 234 ++++--
+ mm/page_alloc.c                                    |  74 +-
+ mm/page_counter.c                                  | 100 ++-
+ mm/page_owner.c                                    |   4 +-
+ mm/shmem.c                                         | 379 ++--------
+ mm/slab.c                                          |   8 +-
+ mm/slab.h                                          |   6 +-
+ mm/slab_common.c                                   | 122 +++-
+ mm/slob.c                                          |   4 +-
+ mm/slub.c                                          | 126 +---
+ mm/sparse-vmemmap.c                                |   1 -
+ mm/sparse.c                                        |  35 +-
+ mm/swap_slots.c                                    |  10 +-
+ mm/swap_state.c                                    |  19 +-
+ mm/swapfile.c                                      | 156 +++-
+ mm/userfaultfd.c                                   |  22 +-
+ mm/vmalloc.c                                       |  41 +-
+ mm/vmpressure.c                                    |  35 +-
+ mm/vmscan.c                                        |  38 +-
+ net/9p/client.c                                    |  13 +-
+ net/9p/trans_xen.c                                 |   1 -
+ scripts/checkpatch.pl                              |  15 +
+ scripts/get_maintainer.pl                          |  13 +-
+ scripts/tags.sh                                    |   6 +-
+ security/apparmor/lsm.c                            |   1 +
+ security/selinux/hooks.c                           |   1 +
+ tools/testing/selftests/proc/.gitignore            |   3 +
+ tools/testing/selftests/proc/Makefile              |   5 +-
+ tools/testing/selftests/proc/fd-001-lookup.c       | 168 +++++
+ tools/testing/selftests/proc/fd-002-posix-eq.c     |  57 ++
+ tools/testing/selftests/proc/fd-003-kthread.c      | 178 +++++
+ tools/testing/selftests/proc/proc-uptime.h         |  16 +-
+ tools/testing/selftests/proc/proc.h                |  39 +
+ tools/testing/selftests/proc/read.c                |  17 +-
+ tools/vm/page-types.c                              |   1 +
+ 200 files changed, 4662 insertions(+), 2657 deletions(-)
+ rename Documentation/filesystems/{autofs4-mount-control.txt => autofs-mount-control.txt} (98%)
+ rename Documentation/filesystems/{autofs4.txt => autofs.txt} (98%)
+ create mode 100644 fs/autofs/Kconfig
+ create mode 100644 fs/autofs/Makefile
+ rename fs/{autofs4 => autofs}/autofs_i.h (67%)
+ rename fs/{autofs4 => autofs}/dev-ioctl.c (96%)
+ rename fs/{autofs4 => autofs}/expire.c (79%)
+ rename fs/{autofs4 => autofs}/init.c (78%)
+ rename fs/{autofs4 => autofs}/inode.c (85%)
+ rename fs/{autofs4 => autofs}/root.c (72%)
+ rename fs/{autofs4 => autofs}/symlink.c (59%)
+ rename fs/{autofs4 => autofs}/waitq.c (90%)
+ create mode 100644 include/linux/memfd.h
+ create mode 100644 mm/memfd.c
+ create mode 100644 tools/testing/selftests/proc/fd-001-lookup.c
+ create mode 100644 tools/testing/selftests/proc/fd-002-posix-eq.c
+ create mode 100644 tools/testing/selftests/proc/fd-003-kthread.c
+ create mode 100644 tools/testing/selftests/proc/proc.h
+$ git clone -s -l -n -q . ../rebase-tmp
+$ cd ../rebase-tmp
+$ git checkout -b akpm remotes/origin/akpm/master
+Switched to a new branch 'akpm'
+$ git rebase --onto master remotes/origin/akpm/master-base
+First, rewinding head to replay your work on top of it...
+Applying: mm: use octal not symbolic permissions
+Applying: treewide: use PHYS_ADDR_MAX to avoid type casting ULLONG_MAX
+Applying: mm: fix oom_kill event handling
+Applying: hexagon: fix printk format warning in setup.c
+Applying: hexagon: drop the unused variable zero_page_mask
+Applying: mm, memcg: propagate memory effective protection on setting memory.min/low
+Applying: mm, memcg: don't skip memory guarantee calculations
+Applying: selftests: cgroup: add test for memory.low corner cases
+Applying: kernel/kexec_file.c: add walk_system_ram_res_rev()
+Applying: kernel/kexec_file.c: load kernel at top of system RAM if required
+Applying: drivers/media/platform/sti/delta/delta-ipc.c: fix read buffer overflow
+Applying: sparc64: NG4 memset 32 bits overflow
+$ cd ../next
+$ git fetch -f ../rebase-tmp akpm:akpm/master
+From ../rebase-tmp
+ + a47b3194e592...7c2656ad4cbd akpm       -> akpm/master  (forced update)
+$ rm -rf ../rebase-tmp
+Merging akpm/master (7c2656ad4cbd sparc64: NG4 memset 32 bits overflow)
+$ git merge --no-ff akpm/master
+Merge made by the 'recursive' strategy.
+ arch/arm64/mm/init.c                             |   6 +-
+ arch/hexagon/include/asm/pgtable.h               |   1 -
+ arch/hexagon/kernel/setup.c                      |   2 +-
+ arch/hexagon/mm/init.c                           |   3 -
+ arch/mips/kernel/setup.c                         |   4 +-
+ arch/powerpc/mm/mem.c                            |   2 +-
+ arch/sparc/lib/NG4memset.S                       |  26 ++---
+ arch/sparc/mm/init_64.c                          |   2 +-
+ arch/x86/mm/init_32.c                            |   2 +-
+ arch/x86/mm/init_64.c                            |   2 +-
+ drivers/firmware/efi/arm-init.c                  |   2 +-
+ drivers/media/platform/sti/delta/delta-ipc.c     |   4 +-
+ drivers/remoteproc/qcom_q6v5_pil.c               |   2 +-
+ drivers/soc/qcom/mdt_loader.c                    |   4 +-
+ include/linux/ioport.h                           |   3 +
+ include/linux/memcontrol.h                       |  26 ++++-
+ kernel/kexec_file.c                              |   2 +
+ kernel/resource.c                                |  63 +++++++++++++
+ mm/cleancache.c                                  |  10 +-
+ mm/cma_debug.c                                   |  25 ++---
+ mm/compaction.c                                  |   2 +-
+ mm/dmapool.c                                     |   2 +-
+ mm/failslab.c                                    |   2 +-
+ mm/frontswap.c                                   |  11 +--
+ mm/memblock.c                                    |   9 +-
+ mm/memcontrol.c                                  |  35 ++++---
+ mm/oom_kill.c                                    |   2 +-
+ mm/page_alloc.c                                  |   2 +-
+ mm/page_idle.c                                   |   2 +-
+ mm/page_owner.c                                  |   4 +-
+ mm/shmem.c                                       |   9 +-
+ mm/slab_common.c                                 |   4 +-
+ mm/vmalloc.c                                     |   4 +-
+ mm/zsmalloc.c                                    |   5 +-
+ mm/zswap.c                                       |  38 ++++----
+ tools/testing/selftests/cgroup/test_memcontrol.c | 115 +++++++++++++++++++++++
+ 36 files changed, 322 insertions(+), 115 deletions(-)
diff --git a/Next/quilt-import.log b/Next/quilt-import.log
new file mode 100644
index 000000000000..5a4e5a9ccdce
--- /dev/null
+++ b/Next/quilt-import.log
@@ -0,0 +1,2 @@
+Importing dmi based on 4.17-rc2
+Unchanged quilt series dmi
diff --git a/arch/Kconfig b/arch/Kconfig
index 75dd23acf133..d3869ca34c18 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -278,9 +278,6 @@ config HAVE_CLK
 	  The <linux/clk.h> calls support software clock gating and
 	  thus are a key power management tool on many systems.
 
-config HAVE_DMA_API_DEBUG
-	bool
-
 config HAVE_HW_BREAKPOINT
 	bool
 	depends on PERF_EVENTS
@@ -401,6 +398,15 @@ config SECCOMP_FILTER
 
 	  See Documentation/prctl/seccomp_filter.txt for details.
 
+preferred-plugin-hostcc := $(if-success,[ $(gcc-version) -ge 40800 ],$(HOSTCXX),$(HOSTCC))
+
+config PLUGIN_HOSTCC
+	string
+	default "$(shell,$(srctree)/scripts/gcc-plugin.sh $(preferred-plugin-hostcc) $(HOSTCXX) $(CC))"
+	help
+	  Host compiler used to build GCC plugins.  This can be $(HOSTCXX),
+	  $(HOSTCC), or a null string if GCC plugin is unsupported.
+
 config HAVE_GCC_PLUGINS
 	bool
 	help
@@ -410,7 +416,7 @@ config HAVE_GCC_PLUGINS
 menuconfig GCC_PLUGINS
 	bool "GCC plugins"
 	depends on HAVE_GCC_PLUGINS
-	depends on !COMPILE_TEST
+	depends on PLUGIN_HOSTCC != ""
 	help
 	  GCC plugins are loadable modules that provide extra features to the
 	  compiler. They are useful for runtime instrumentation and static analysis.
@@ -420,7 +426,7 @@ menuconfig GCC_PLUGINS
 config GCC_PLUGIN_CYC_COMPLEXITY
 	bool "Compute the cyclomatic complexity of a function" if EXPERT
 	depends on GCC_PLUGINS
-	depends on !COMPILE_TEST
+	depends on !COMPILE_TEST	# too noisy
 	help
 	  The complexity M of a function's control flow graph is defined as:
 	   M = E - N + 2P
@@ -487,7 +493,7 @@ config GCC_PLUGIN_STRUCTLEAK_BYREF_ALL
 config GCC_PLUGIN_STRUCTLEAK_VERBOSE
 	bool "Report forcefully initialized variables"
 	depends on GCC_PLUGIN_STRUCTLEAK
-	depends on !COMPILE_TEST
+	depends on !COMPILE_TEST	# too noisy
 	help
 	  This option will cause a warning to be printed each time the
 	  structleak plugin finds a variable it thinks needs to be
@@ -527,7 +533,7 @@ config GCC_PLUGIN_RANDSTRUCT
 config GCC_PLUGIN_RANDSTRUCT_PERFORMANCE
 	bool "Use cacheline-aware structure randomization"
 	depends on GCC_PLUGIN_RANDSTRUCT
-	depends on !COMPILE_TEST
+	depends on !COMPILE_TEST	# do not reduce test coverage
 	help
 	  If you say Y here, the RANDSTRUCT randomization will make a
 	  best effort at restricting randomization to cacheline-sized
@@ -539,13 +545,16 @@ config HAVE_CC_STACKPROTECTOR
 	bool
 	help
 	  An arch should select this symbol if:
-	  - its compiler supports the -fstack-protector option
 	  - it has implemented a stack canary (e.g. __stack_chk_guard)
 
-choice
-	prompt "Stack Protector buffer overflow detection"
+config CC_HAS_STACKPROTECTOR_NONE
+	def_bool $(cc-option,-fno-stack-protector)
+
+config CC_STACKPROTECTOR
+	bool "Stack Protector buffer overflow detection"
 	depends on HAVE_CC_STACKPROTECTOR
-	default CC_STACKPROTECTOR_AUTO
+	depends on $(cc-option,-fstack-protector)
+	default y
 	help
 	  This option turns on the "stack-protector" GCC feature. This
 	  feature puts, at the beginning of functions, a canary value on
@@ -555,14 +564,6 @@ choice
 	  overwrite the canary, which gets detected and the attack is then
 	  neutralized via a kernel panic.
 
-config CC_STACKPROTECTOR_NONE
-	bool "None"
-	help
-	  Disable "stack-protector" GCC feature.
-
-config CC_STACKPROTECTOR_REGULAR
-	bool "Regular"
-	help
 	  Functions will have the stack-protector canary logic added if they
 	  have an 8-byte or larger character array on the stack.
 
@@ -574,7 +575,10 @@ config CC_STACKPROTECTOR_REGULAR
 	  by about 0.3%.
 
 config CC_STACKPROTECTOR_STRONG
-	bool "Strong"
+	bool "Strong Stack Protector"
+	depends on CC_STACKPROTECTOR
+	depends on $(cc-option,-fstack-protector-strong)
+	default y
 	help
 	  Functions will have the stack-protector canary logic added in any
 	  of the following conditions:
@@ -592,29 +596,6 @@ config CC_STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
-config CC_STACKPROTECTOR_AUTO
-	bool "Automatic"
-	help
-	  If the compiler supports it, the best available stack-protector
-	  option will be chosen.
-
-endchoice
-
-config LD_DEAD_CODE_DATA_ELIMINATION
-	bool
-	help
-	  Select this if the architecture wants to do dead code and
-	  data elimination with the linker by compiling with
-	  -ffunction-sections -fdata-sections and linking with
-	  --gc-sections.
-
-	  This requires that the arch annotates or otherwise protects
-	  its external entry points from being discarded. Linker scripts
-	  must also merge .text.*, .data.*, and .bss.* correctly into
-	  output sections. Care must be taken not to pull in unrelated
-	  sections (e.g., '.text.init'). Typically '.' in section names
-	  is used to distinguish them from label names / C identifiers.
-
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
@@ -690,12 +671,6 @@ config MODULES_USE_ELF_REL
 	  Modules only use ELF REL relocations.  Modules with ELF RELA
 	  relocations will give an error.
 
-config HAVE_UNDERSCORE_SYMBOL_PREFIX
-	bool
-	help
-	  Some architectures generate an _ in front of C symbols; things like
-	  module loading and assembly files need to know about this.
-
 config HAVE_IRQ_EXIT_ON_IRQ_STACK
 	bool
 	help
@@ -874,6 +849,21 @@ config OLD_SIGACTION
 config COMPAT_OLD_SIGACTION
 	bool
 
+config 64BIT_TIME
+	def_bool ARCH_HAS_64BIT_TIME
+	help
+	  This should be selected by all architectures that need to support
+	  new system calls with a 64-bit time_t. This is relevant on all 32-bit
+	  architectures, and 64-bit architectures as part of compat syscall
+	  handling.
+
+config COMPAT_32BIT_TIME
+	def_bool (!64BIT && 64BIT_TIME) || COMPAT
+	help
+	  This enables 32 bit time_t support in addition to 64 bit time_t support.
+	  This is relevant on all 32-bit architectures, and 64-bit architectures
+	  as part of compat syscall handling.
+
 config ARCH_NO_COHERENT_DMA_MMAP
 	bool
 
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig
index f19dc31288c8..0c4805a572c8 100644
--- a/arch/alpha/Kconfig
+++ b/arch/alpha/Kconfig
@@ -10,6 +10,8 @@ config ALPHA
 	select HAVE_OPROFILE
 	select HAVE_PCSPKR_PLATFORM
 	select HAVE_PERF_EVENTS
+	select NEED_DMA_MAP_STATE
+	select NEED_SG_DMA_LENGTH
 	select VIRT_TO_BUS
 	select GENERIC_IRQ_PROBE
 	select AUTO_IRQ_AFFINITY if SMP
@@ -64,15 +66,6 @@ config ZONE_DMA
 	bool
 	default y
 
-config ARCH_DMA_ADDR_T_64BIT
-	def_bool y
-
-config NEED_DMA_MAP_STATE
-       def_bool y
-
-config NEED_SG_DMA_LENGTH
-	def_bool y
-
 config GENERIC_ISA_DMA
 	bool
 	default y
@@ -346,9 +339,6 @@ config PCI_DOMAINS
 config PCI_SYSCALL
 	def_bool PCI
 
-config IOMMU_HELPER
-	def_bool PCI
-
 config ALPHA_NONAME
 	bool
 	depends on ALPHA_BOOK1 || ALPHA_NONAME_CH
@@ -586,7 +576,7 @@ config ARCH_DISCONTIGMEM_ENABLE
 	  Say Y to support efficient handling of discontiguous physical memory,
 	  for architectures which are either NUMA (Non-Uniform Memory Access)
 	  or have huge holes in the physical address space for other reasons.
-	  See <file:Documentation/vm/numa> for more.
+	  See <file:Documentation/vm/numa.rst> for more.
 
 source "mm/Kconfig"
 
diff --git a/arch/alpha/include/asm/Kbuild b/arch/alpha/include/asm/Kbuild
index 9b68790013e2..0580cb8c84b2 100644
--- a/arch/alpha/include/asm/Kbuild
+++ b/arch/alpha/include/asm/Kbuild
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 
 
+generic-y += compat.h
 generic-y += exec.h
 generic-y += export.h
 generic-y += fb.h
diff --git a/arch/alpha/include/asm/pci.h b/arch/alpha/include/asm/pci.h
index b9ec55351924..cf6bc1e64d66 100644
--- a/arch/alpha/include/asm/pci.h
+++ b/arch/alpha/include/asm/pci.h
@@ -56,11 +56,6 @@ struct pci_controller {
 
 /* IOMMU controls.  */
 
-/* The PCI address space does not equal the physical memory address space.
-   The networking and block device layers use this boolean for bounce buffer
-   decisions.  */
-#define PCI_DMA_BUS_IS_PHYS  0
-
 /* TODO: integrate with include/asm-generic/pci.h ? */
 static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel)
 {
diff --git a/arch/alpha/include/uapi/asm/Kbuild b/arch/alpha/include/uapi/asm/Kbuild
index 9afaba5e5503..1a5b75310cf4 100644
--- a/arch/alpha/include/uapi/asm/Kbuild
+++ b/arch/alpha/include/uapi/asm/Kbuild
@@ -2,4 +2,8 @@
 include include/uapi/asm-generic/Kbuild.asm
 
 generic-y += bpf_perf_event.h
+generic-y += ipcbuf.h
+generic-y += msgbuf.h
 generic-y += poll.h
+generic-y += sembuf.h
+generic-y += shmbuf.h
diff --git a/arch/alpha/include/uapi/asm/ipcbuf.h b/arch/alpha/include/uapi/asm/ipcbuf.h
deleted file mode 100644
index 90d6445a14df..000000000000
--- a/arch/alpha/include/uapi/asm/ipcbuf.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#include <asm-generic/ipcbuf.h>
diff --git a/arch/alpha/include/uapi/asm/msgbuf.h b/arch/alpha/include/uapi/asm/msgbuf.h
deleted file mode 100644
index 8c5d4d8c1b16..000000000000
--- a/arch/alpha/include/uapi/asm/msgbuf.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _ALPHA_MSGBUF_H
-#define _ALPHA_MSGBUF_H
-
-/* 
- * The msqid64_ds structure for alpha architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space is left for:
- * - 2 miscellaneous 64-bit values
- */
-
-struct msqid64_ds {
-	struct ipc64_perm msg_perm;
-	__kernel_time_t msg_stime;	/* last msgsnd time */
-	__kernel_time_t msg_rtime;	/* last msgrcv time */
-	__kernel_time_t msg_ctime;	/* last change time */
-	unsigned long  msg_cbytes;	/* current number of bytes on queue */
-	unsigned long  msg_qnum;	/* number of messages in queue */
-	unsigned long  msg_qbytes;	/* max number of bytes on queue */
-	__kernel_pid_t msg_lspid;	/* pid of last msgsnd */
-	__kernel_pid_t msg_lrpid;	/* last receive pid */
-	unsigned long  __unused1;
-	unsigned long  __unused2;
-};
-
-#endif /* _ALPHA_MSGBUF_H */
diff --git a/arch/alpha/include/uapi/asm/sembuf.h b/arch/alpha/include/uapi/asm/sembuf.h
deleted file mode 100644
index f28ffa668b2f..000000000000
--- a/arch/alpha/include/uapi/asm/sembuf.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _ALPHA_SEMBUF_H
-#define _ALPHA_SEMBUF_H
-
-/* 
- * The semid64_ds structure for alpha architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space is left for:
- * - 2 miscellaneous 64-bit values
- */
-
-struct semid64_ds {
-	struct ipc64_perm sem_perm;		/* permissions .. see ipc.h */
-	__kernel_time_t	sem_otime;		/* last semop time */
-	__kernel_time_t	sem_ctime;		/* last change time */
-	unsigned long	sem_nsems;		/* no. of semaphores in array */
-	unsigned long	__unused1;
-	unsigned long	__unused2;
-};
-
-#endif /* _ALPHA_SEMBUF_H */
diff --git a/arch/alpha/include/uapi/asm/shmbuf.h b/arch/alpha/include/uapi/asm/shmbuf.h
deleted file mode 100644
index 7e041ca2eb40..000000000000
--- a/arch/alpha/include/uapi/asm/shmbuf.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _ALPHA_SHMBUF_H
-#define _ALPHA_SHMBUF_H
-
-/* 
- * The shmid64_ds structure for alpha architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space is left for:
- * - 2 miscellaneous 64-bit values
- */
-
-struct shmid64_ds {
-	struct ipc64_perm	shm_perm;	/* operation perms */
-	size_t			shm_segsz;	/* size of segment (bytes) */
-	__kernel_time_t		shm_atime;	/* last attach time */
-	__kernel_time_t		shm_dtime;	/* last detach time */
-	__kernel_time_t		shm_ctime;	/* last change time */
-	__kernel_pid_t		shm_cpid;	/* pid of creator */
-	__kernel_pid_t		shm_lpid;	/* pid of last operator */
-	unsigned long		shm_nattch;	/* no. of current attaches */
-	unsigned long		__unused1;
-	unsigned long		__unused2;
-};
-
-struct shminfo64 {
-	unsigned long	shmmax;
-	unsigned long	shmmin;
-	unsigned long	shmmni;
-	unsigned long	shmseg;
-	unsigned long	shmall;
-	unsigned long	__unused1;
-	unsigned long	__unused2;
-	unsigned long	__unused3;
-	unsigned long	__unused4;
-};
-
-#endif /* _ALPHA_SHMBUF_H */
diff --git a/arch/alpha/include/uapi/asm/siginfo.h b/arch/alpha/include/uapi/asm/siginfo.h
index 0cf3b527b274..db3f0138536f 100644
--- a/arch/alpha/include/uapi/asm/siginfo.h
+++ b/arch/alpha/include/uapi/asm/siginfo.h
@@ -7,18 +7,4 @@
 
 #include <asm-generic/siginfo.h>
 
-/*
- * SIGFPE si_codes
- */
-#ifdef __KERNEL__
-#define FPE_FIXME	0	/* Broken dup of SI_USER */
-#endif /* __KERNEL__ */
-
-/*
- * SIGTRAP si_codes
- */
-#ifdef __KERNEL__
-#define TRAP_FIXME	0	/* Broken dup of SI_USER */
-#endif /* __KERNEL__ */
-
 #endif
diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c
index 89faa6f4de47..6e921754c8fc 100644
--- a/arch/alpha/kernel/osf_sys.c
+++ b/arch/alpha/kernel/osf_sys.c
@@ -871,8 +871,7 @@ SYSCALL_DEFINE5(osf_setsysinfo, unsigned long, op, void __user *, buffer,
 		   send a signal.  Old exceptions are not signaled.  */
 		fex = (exc >> IEEE_STATUS_TO_EXCSUM_SHIFT) & swcr;
  		if (fex) {
-			siginfo_t info;
-			int si_code = 0;
+			int si_code = FPE_FLTUNK;
 
 			if (fex & IEEE_TRAP_ENABLE_DNO) si_code = FPE_FLTUND;
 			if (fex & IEEE_TRAP_ENABLE_INE) si_code = FPE_FLTRES;
@@ -881,11 +880,9 @@ SYSCALL_DEFINE5(osf_setsysinfo, unsigned long, op, void __user *, buffer,
 			if (fex & IEEE_TRAP_ENABLE_DZE) si_code = FPE_FLTDIV;
 			if (fex & IEEE_TRAP_ENABLE_INV) si_code = FPE_FLTINV;
 
-			info.si_signo = SIGFPE;
-			info.si_errno = 0;
-			info.si_code = si_code;
-			info.si_addr = NULL;  /* FIXME */
- 			send_sig_info(SIGFPE, &info, current);
+			send_sig_fault(SIGFPE, si_code,
+				       (void __user *)NULL,  /* FIXME */
+				       0, current);
  		}
 		return 0;
 	}
diff --git a/arch/alpha/kernel/signal.c b/arch/alpha/kernel/signal.c
index 9ebb3bcbc626..8c0c4ee0be6e 100644
--- a/arch/alpha/kernel/signal.c
+++ b/arch/alpha/kernel/signal.c
@@ -219,14 +219,8 @@ do_sigreturn(struct sigcontext __user *sc)
 
 	/* Send SIGTRAP if we're single-stepping: */
 	if (ptrace_cancel_bpt (current)) {
-		siginfo_t info;
-
-		info.si_signo = SIGTRAP;
-		info.si_errno = 0;
-		info.si_code = TRAP_BRKPT;
-		info.si_addr = (void __user *) regs->pc;
-		info.si_trapno = 0;
-		send_sig_info(SIGTRAP, &info, current);
+		send_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *) regs->pc, 0,
+			       current);
 	}
 	return;
 
@@ -253,14 +247,8 @@ do_rt_sigreturn(struct rt_sigframe __user *frame)
 
 	/* Send SIGTRAP if we're single-stepping: */
 	if (ptrace_cancel_bpt (current)) {
-		siginfo_t info;
-
-		info.si_signo = SIGTRAP;
-		info.si_errno = 0;
-		info.si_code = TRAP_BRKPT;
-		info.si_addr = (void __user *) regs->pc;
-		info.si_trapno = 0;
-		send_sig_info(SIGTRAP, &info, current);
+		send_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *) regs->pc, 0,
+			       current);
 	}
 	return;
 
diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c
index f43bd05dede2..bc9627698796 100644
--- a/arch/alpha/kernel/traps.c
+++ b/arch/alpha/kernel/traps.c
@@ -213,7 +213,6 @@ do_entArith(unsigned long summary, unsigned long write_mask,
 	    struct pt_regs *regs)
 {
 	long si_code = FPE_FLTINV;
-	siginfo_t info;
 
 	if (summary & 1) {
 		/* Software-completion summary bit is set, so try to
@@ -228,17 +227,12 @@ do_entArith(unsigned long summary, unsigned long write_mask,
 	}
 	die_if_kernel("Arithmetic fault", regs, 0, NULL);
 
-	info.si_signo = SIGFPE;
-	info.si_errno = 0;
-	info.si_code = si_code;
-	info.si_addr = (void __user *) regs->pc;
-	send_sig_info(SIGFPE, &info, current);
+	send_sig_fault(SIGFPE, si_code, (void __user *) regs->pc, 0, current);
 }
 
 asmlinkage void
 do_entIF(unsigned long type, struct pt_regs *regs)
 {
-	siginfo_t info;
 	int signo, code;
 
 	if ((regs->ps & ~IPL_MAX) == 0) {
@@ -270,31 +264,20 @@ do_entIF(unsigned long type, struct pt_regs *regs)
 
 	switch (type) {
 	      case 0: /* breakpoint */
-		info.si_signo = SIGTRAP;
-		info.si_errno = 0;
-		info.si_code = TRAP_BRKPT;
-		info.si_trapno = 0;
-		info.si_addr = (void __user *) regs->pc;
-
 		if (ptrace_cancel_bpt(current)) {
 			regs->pc -= 4;	/* make pc point to former bpt */
 		}
 
-		send_sig_info(SIGTRAP, &info, current);
+		send_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc, 0,
+			       current);
 		return;
 
 	      case 1: /* bugcheck */
-		info.si_signo = SIGTRAP;
-		info.si_errno = 0;
-		info.si_code = TRAP_FIXME;
-		info.si_addr = (void __user *) regs->pc;
-		info.si_trapno = 0;
-		send_sig_info(SIGTRAP, &info, current);
+		send_sig_fault(SIGTRAP, TRAP_UNK, (void __user *) regs->pc, 0,
+			       current);
 		return;
 		
 	      case 2: /* gentrap */
-		info.si_addr = (void __user *) regs->pc;
-		info.si_trapno = regs->r16;
 		switch ((long) regs->r16) {
 		case GEN_INTOVF:
 			signo = SIGFPE;
@@ -326,7 +309,7 @@ do_entIF(unsigned long type, struct pt_regs *regs)
 			break;
 		case GEN_ROPRAND:
 			signo = SIGFPE;
-			code = FPE_FIXME;
+			code = FPE_FLTUNK;
 			break;
 
 		case GEN_DECOVF:
@@ -348,15 +331,12 @@ do_entIF(unsigned long type, struct pt_regs *regs)
 		case GEN_SUBRNG7:
 		default:
 			signo = SIGTRAP;
-			code = TRAP_FIXME;
+			code = TRAP_UNK;
 			break;
 		}
 
-		info.si_signo = signo;
-		info.si_errno = 0;
-		info.si_code = code;
-		info.si_addr = (void __user *) regs->pc;
-		send_sig_info(signo, &info, current);
+		send_sig_fault(signo, code, (void __user *) regs->pc, regs->r16,
+			       current);
 		return;
 
 	      case 4: /* opDEC */
@@ -380,11 +360,9 @@ do_entIF(unsigned long type, struct pt_regs *regs)
 			if (si_code == 0)
 				return;
 			if (si_code > 0) {
-				info.si_signo = SIGFPE;
-				info.si_errno = 0;
-				info.si_code = si_code;
-				info.si_addr = (void __user *) regs->pc;
-				send_sig_info(SIGFPE, &info, current);
+				send_sig_fault(SIGFPE, si_code,
+					       (void __user *) regs->pc, 0,
+					       current);
 				return;
 			}
 		}
@@ -409,11 +387,7 @@ do_entIF(unsigned long type, struct pt_regs *regs)
 		      ;
 	}
 
-	info.si_signo = SIGILL;
-	info.si_errno = 0;
-	info.si_code = ILL_ILLOPC;
-	info.si_addr = (void __user *) regs->pc;
-	send_sig_info(SIGILL, &info, current);
+	send_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->pc, 0, current);
 }
 
 /* There is an ifdef in the PALcode in MILO that enables a 
@@ -426,15 +400,9 @@ do_entIF(unsigned long type, struct pt_regs *regs)
 asmlinkage void
 do_entDbg(struct pt_regs *regs)
 {
-	siginfo_t info;
-
 	die_if_kernel("Instruction fault", regs, 0, NULL);
 
-	info.si_signo = SIGILL;
-	info.si_errno = 0;
-	info.si_code = ILL_ILLOPC;
-	info.si_addr = (void __user *) regs->pc;
-	force_sig_info(SIGILL, &info, current);
+	force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->pc, 0, current);
 }
 
 
@@ -758,7 +726,7 @@ do_entUnaUser(void __user * va, unsigned long opcode,
 
 	unsigned long tmp1, tmp2, tmp3, tmp4;
 	unsigned long fake_reg, *reg_addr = &fake_reg;
-	siginfo_t info;
+	int si_code;
 	long error;
 
 	/* Check the UAC bits to decide what the user wants us to do
@@ -981,34 +949,27 @@ do_entUnaUser(void __user * va, unsigned long opcode,
 
 give_sigsegv:
 	regs->pc -= 4;  /* make pc point to faulting insn */
-	info.si_signo = SIGSEGV;
-	info.si_errno = 0;
 
 	/* We need to replicate some of the logic in mm/fault.c,
 	   since we don't have access to the fault code in the
 	   exception handling return path.  */
 	if ((unsigned long)va >= TASK_SIZE)
-		info.si_code = SEGV_ACCERR;
+		si_code = SEGV_ACCERR;
 	else {
 		struct mm_struct *mm = current->mm;
 		down_read(&mm->mmap_sem);
 		if (find_vma(mm, (unsigned long)va))
-			info.si_code = SEGV_ACCERR;
+			si_code = SEGV_ACCERR;
 		else
-			info.si_code = SEGV_MAPERR;
+			si_code = SEGV_MAPERR;
 		up_read(&mm->mmap_sem);
 	}
-	info.si_addr = va;
-	send_sig_info(SIGSEGV, &info, current);
+	send_sig_fault(SIGSEGV, si_code, va, 0, current);
 	return;
 
 give_sigbus:
 	regs->pc -= 4;
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRALN;
-	info.si_addr = va;
-	send_sig_info(SIGBUS, &info, current);
+	send_sig_fault(SIGBUS, BUS_ADRALN, va, 0, current);
 	return;
 }
 
diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c
index cd3c572ee912..de2bd217adad 100644
--- a/arch/alpha/mm/fault.c
+++ b/arch/alpha/mm/fault.c
@@ -88,7 +88,6 @@ do_page_fault(unsigned long address, unsigned long mmcsr,
 	struct mm_struct *mm = current->mm;
 	const struct exception_table_entry *fixup;
 	int fault, si_code = SEGV_MAPERR;
-	siginfo_t info;
 	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 
 	/* As of EV6, a load into $31/$f31 is a prefetch, and never faults
@@ -221,21 +220,13 @@ do_page_fault(unsigned long address, unsigned long mmcsr,
 	up_read(&mm->mmap_sem);
 	/* Send a sigbus, regardless of whether we were in kernel
 	   or user mode.  */
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRERR;
-	info.si_addr = (void __user *) address;
-	force_sig_info(SIGBUS, &info, current);
+	force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *) address, 0, current);
 	if (!user_mode(regs))
 		goto no_context;
 	return;
 
  do_sigsegv:
-	info.si_signo = SIGSEGV;
-	info.si_errno = 0;
-	info.si_code = si_code;
-	info.si_addr = (void __user *) address;
-	force_sig_info(SIGSEGV, &info, current);
+	force_sig_fault(SIGSEGV, si_code, (void __user *) address, 0, current);
 	return;
 
 #ifdef CONFIG_ALPHA_LARGE_VMALLOC
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
index d76bf4a83740..e81bcd271be7 100644
--- a/arch/arc/Kconfig
+++ b/arch/arc/Kconfig
@@ -9,11 +9,15 @@
 config ARC
 	def_bool y
 	select ARC_TIMERS
+	select ARCH_HAS_SYNC_DMA_FOR_CPU
+	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
 	select ARCH_HAS_SG_CHAIN
 	select ARCH_SUPPORTS_ATOMIC_RMW if ARC_HAS_LLSC
 	select BUILDTIME_EXTABLE_SORT
 	select CLONE_BACKWARDS
 	select COMMON_CLK
+	select DMA_NONCOHERENT_OPS
+	select DMA_NONCOHERENT_MMAP
 	select GENERIC_ATOMIC64 if !ISA_ARCV2 || !(ARC_HAS_LL64 && ARC_HAS_LLSC)
 	select GENERIC_CLOCKEVENTS
 	select GENERIC_FIND_FIRST_BIT
@@ -44,6 +48,7 @@ config ARC
 	select HAVE_GENERIC_DMA_COHERENT
 	select HAVE_KERNEL_GZIP
 	select HAVE_KERNEL_LZMA
+	select ARCH_HAS_PTE_SPECIAL
 
 config MIGHT_HAVE_PCI
 	bool
@@ -453,16 +458,11 @@ config ARC_HAS_PAE40
 	default n
 	depends on ISA_ARCV2
 	select HIGHMEM
+	select PHYS_ADDR_T_64BIT
 	help
 	  Enable access to physical memory beyond 4G, only supported on
 	  ARC cores with 40 bit Physical Addressing support
 
-config ARCH_PHYS_ADDR_T_64BIT
-	def_bool ARC_HAS_PAE40
-
-config ARCH_DMA_ADDR_T_64BIT
-	bool
-
 config ARC_KVADDR_SIZE
 	int "Kernel Virtual Address Space size (MB)"
 	range 0 512
diff --git a/arch/arc/include/asm/Kbuild b/arch/arc/include/asm/Kbuild
index 4bd5d4369e05..feed50ce89fa 100644
--- a/arch/arc/include/asm/Kbuild
+++ b/arch/arc/include/asm/Kbuild
@@ -1,7 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0
 generic-y += bugs.h
+generic-y += compat.h
 generic-y += device.h
 generic-y += div64.h
+generic-y += dma-mapping.h
 generic-y += emergency-restart.h
 generic-y += extable.h
 generic-y += fb.h
diff --git a/arch/arc/include/asm/dma-mapping.h b/arch/arc/include/asm/dma-mapping.h
deleted file mode 100644
index 7a16824bfe98..000000000000
--- a/arch/arc/include/asm/dma-mapping.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * DMA Mapping glue for ARC
- *
- * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef ASM_ARC_DMA_MAPPING_H
-#define ASM_ARC_DMA_MAPPING_H
-
-extern const struct dma_map_ops arc_dma_ops;
-
-static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
-{
-	return &arc_dma_ops;
-}
-
-#endif
diff --git a/arch/arc/include/asm/pci.h b/arch/arc/include/asm/pci.h
index ba56c23c1b20..4ff53c041c64 100644
--- a/arch/arc/include/asm/pci.h
+++ b/arch/arc/include/asm/pci.h
@@ -16,12 +16,6 @@
 #define PCIBIOS_MIN_MEM 0x100000
 
 #define pcibios_assign_all_busses()	1
-/*
- * The PCI address space does equal the physical memory address space.
- * The networking and block device layers use this boolean for bounce
- * buffer decisions.
- */
-#define PCI_DMA_BUS_IS_PHYS	1
 
 #endif /* __KERNEL__ */
 
diff --git a/arch/arc/include/asm/pgtable.h b/arch/arc/include/asm/pgtable.h
index 08fe33830d4b..8ec5599a0957 100644
--- a/arch/arc/include/asm/pgtable.h
+++ b/arch/arc/include/asm/pgtable.h
@@ -320,8 +320,6 @@ PTE_BIT_FUNC(mkexec,	|= (_PAGE_EXECUTE));
 PTE_BIT_FUNC(mkspecial,	|= (_PAGE_SPECIAL));
 PTE_BIT_FUNC(mkhuge,	|= (_PAGE_HW_SZ));
 
-#define __HAVE_ARCH_PTE_SPECIAL
-
 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 {
 	return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot));
diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c
index 1dcc404b5aec..8c1071840979 100644
--- a/arch/arc/mm/dma.c
+++ b/arch/arc/mm/dma.c
@@ -16,13 +16,12 @@
  * The default DMA address == Phy address which is 0x8000_0000 based.
  */
 
-#include <linux/dma-mapping.h>
+#include <linux/dma-noncoherent.h>
 #include <asm/cache.h>
 #include <asm/cacheflush.h>
 
-
-static void *arc_dma_alloc(struct device *dev, size_t size,
-		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
+void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
+		gfp_t gfp, unsigned long attrs)
 {
 	unsigned long order = get_order(size);
 	struct page *page;
@@ -89,7 +88,7 @@ static void *arc_dma_alloc(struct device *dev, size_t size,
 	return kvaddr;
 }
 
-static void arc_dma_free(struct device *dev, size_t size, void *vaddr,
+void arch_dma_free(struct device *dev, size_t size, void *vaddr,
 		dma_addr_t dma_handle, unsigned long attrs)
 {
 	phys_addr_t paddr = dma_handle;
@@ -105,9 +104,9 @@ static void arc_dma_free(struct device *dev, size_t size, void *vaddr,
 	__free_pages(page, get_order(size));
 }
 
-static int arc_dma_mmap(struct device *dev, struct vm_area_struct *vma,
-			void *cpu_addr, dma_addr_t dma_addr, size_t size,
-			unsigned long attrs)
+int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma,
+		void *cpu_addr, dma_addr_t dma_addr, size_t size,
+		unsigned long attrs)
 {
 	unsigned long user_count = vma_pages(vma);
 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
@@ -130,149 +129,14 @@ static int arc_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 	return ret;
 }
 
-/*
- * streaming DMA Mapping API...
- * CPU accesses page via normal paddr, thus needs to explicitly made
- * consistent before each use
- */
-static void _dma_cache_sync(phys_addr_t paddr, size_t size,
-		enum dma_data_direction dir)
+void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
+		size_t size, enum dma_data_direction dir)
 {
-	switch (dir) {
-	case DMA_FROM_DEVICE:
-		dma_cache_inv(paddr, size);
-		break;
-	case DMA_TO_DEVICE:
-		dma_cache_wback(paddr, size);
-		break;
-	case DMA_BIDIRECTIONAL:
-		dma_cache_wback_inv(paddr, size);
-		break;
-	default:
-		pr_err("Invalid DMA dir [%d] for OP @ %pa[p]\n", dir, &paddr);
-	}
+	dma_cache_wback(paddr, size);
 }
 
-/*
- * arc_dma_map_page - map a portion of a page for streaming DMA
- *
- * Ensure that any data held in the cache is appropriately discarded
- * or written back.
- *
- * The device owns this memory once this call has completed.  The CPU
- * can regain ownership by calling dma_unmap_page().
- *
- * Note: while it takes struct page as arg, caller can "abuse" it to pass
- * a region larger than PAGE_SIZE, provided it is physically contiguous
- * and this still works correctly
- */
-static dma_addr_t arc_dma_map_page(struct device *dev, struct page *page,
-		unsigned long offset, size_t size, enum dma_data_direction dir,
-		unsigned long attrs)
+void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
+		size_t size, enum dma_data_direction dir)
 {
-	phys_addr_t paddr = page_to_phys(page) + offset;
-
-	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
-		_dma_cache_sync(paddr, size, dir);
-
-	return paddr;
-}
-
-/*
- * arc_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
- *
- * After this call, reads by the CPU to the buffer are guaranteed to see
- * whatever the device wrote there.
- *
- * Note: historically this routine was not implemented for ARC
- */
-static void arc_dma_unmap_page(struct device *dev, dma_addr_t handle,
-			       size_t size, enum dma_data_direction dir,
-			       unsigned long attrs)
-{
-	phys_addr_t paddr = handle;
-
-	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
-		_dma_cache_sync(paddr, size, dir);
+	dma_cache_inv(paddr, size);
 }
-
-static int arc_dma_map_sg(struct device *dev, struct scatterlist *sg,
-	   int nents, enum dma_data_direction dir, unsigned long attrs)
-{
-	struct scatterlist *s;
-	int i;
-
-	for_each_sg(sg, s, nents, i)
-		s->dma_address = dma_map_page(dev, sg_page(s), s->offset,
-					       s->length, dir);
-
-	return nents;
-}
-
-static void arc_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
-			     int nents, enum dma_data_direction dir,
-			     unsigned long attrs)
-{
-	struct scatterlist *s;
-	int i;
-
-	for_each_sg(sg, s, nents, i)
-		arc_dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir,
-				   attrs);
-}
-
-static void arc_dma_sync_single_for_cpu(struct device *dev,
-		dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
-{
-	_dma_cache_sync(dma_handle, size, DMA_FROM_DEVICE);
-}
-
-static void arc_dma_sync_single_for_device(struct device *dev,
-		dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
-{
-	_dma_cache_sync(dma_handle, size, DMA_TO_DEVICE);
-}
-
-static void arc_dma_sync_sg_for_cpu(struct device *dev,
-		struct scatterlist *sglist, int nelems,
-		enum dma_data_direction dir)
-{
-	int i;
-	struct scatterlist *sg;
-
-	for_each_sg(sglist, sg, nelems, i)
-		_dma_cache_sync(sg_phys(sg), sg->length, dir);
-}
-
-static void arc_dma_sync_sg_for_device(struct device *dev,
-		struct scatterlist *sglist, int nelems,
-		enum dma_data_direction dir)
-{
-	int i;
-	struct scatterlist *sg;
-
-	for_each_sg(sglist, sg, nelems, i)
-		_dma_cache_sync(sg_phys(sg), sg->length, dir);
-}
-
-static int arc_dma_supported(struct device *dev, u64 dma_mask)
-{
-	/* Support 32 bit DMA mask exclusively */
-	return dma_mask == DMA_BIT_MASK(32);
-}
-
-const struct dma_map_ops arc_dma_ops = {
-	.alloc			= arc_dma_alloc,
-	.free			= arc_dma_free,
-	.mmap			= arc_dma_mmap,
-	.map_page		= arc_dma_map_page,
-	.unmap_page		= arc_dma_unmap_page,
-	.map_sg			= arc_dma_map_sg,
-	.unmap_sg		= arc_dma_unmap_sg,
-	.sync_single_for_device	= arc_dma_sync_single_for_device,
-	.sync_single_for_cpu	= arc_dma_sync_single_for_cpu,
-	.sync_sg_for_cpu	= arc_dma_sync_sg_for_cpu,
-	.sync_sg_for_device	= arc_dma_sync_sg_for_device,
-	.dma_supported		= arc_dma_supported,
-};
-EXPORT_SYMBOL(arc_dma_ops);
diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c
index a0b7bd6d030d..b884bbd6f354 100644
--- a/arch/arc/mm/fault.c
+++ b/arch/arc/mm/fault.c
@@ -70,6 +70,8 @@ void do_page_fault(unsigned long address, struct pt_regs *regs)
 	int write = regs->ecr_cause & ECR_C_PROTV_STORE;  /* ST/EX */
 	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 
+	clear_siginfo(&info);
+
 	/*
 	 * We fault-in kernel-space virtual memory on-demand. The
 	 * 'reference' page table is init_mm.pgd.
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index a7f8e7f4b88f..d07e9af52e8d 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -8,8 +8,10 @@ config ARM
 	select ARCH_HAS_DEVMEM_IS_ALLOWED
 	select ARCH_HAS_ELF_RANDOMIZE
 	select ARCH_HAS_FORTIFY_SOURCE
-	select ARCH_HAS_SET_MEMORY
+	select ARCH_HAS_KCOV
+	select ARCH_HAS_PTE_SPECIAL if ARM_LPAE
 	select ARCH_HAS_PHYS_TO_DMA
+	select ARCH_HAS_SET_MEMORY
 	select ARCH_HAS_STRICT_KERNEL_RWX if MMU && !XIP_KERNEL
 	select ARCH_HAS_STRICT_MODULE_RWX if MMU
 	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
@@ -60,7 +62,6 @@ config ARM
 	select HAVE_CONTEXT_TRACKING
 	select HAVE_C_RECORDMCOUNT
 	select HAVE_DEBUG_KMEMLEAK
-	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_CONTIGUOUS if MMU
 	select HAVE_DYNAMIC_FTRACE if (!XIP_KERNEL) && !CPU_ENDIAN_BE32 && MMU
 	select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE
@@ -96,6 +97,7 @@ config ARM
 	select HAVE_VIRT_CPU_ACCOUNTING_GEN
 	select IRQ_FORCED_THREADING
 	select MODULES_USE_ELF_REL
+	select NEED_DMA_MAP_STATE
 	select NO_BOOTMEM
 	select OF_EARLY_FLATTREE if OF
 	select OF_RESERVED_MEM if OF
@@ -119,9 +121,6 @@ config ARM_HAS_SG_CHAIN
 	select ARCH_HAS_SG_CHAIN
 	bool
 
-config NEED_SG_DMA_LENGTH
-	bool
-
 config ARM_DMA_USE_IOMMU
 	bool
 	select ARM_HAS_SG_CHAIN
@@ -224,9 +223,6 @@ config ARCH_MAY_HAVE_PC_FDC
 config ZONE_DMA
 	bool
 
-config NEED_DMA_MAP_STATE
-       def_bool y
-
 config ARCH_SUPPORTS_UPROBES
 	def_bool y
 
@@ -1467,7 +1463,7 @@ config ARM_PSCI
 config ARCH_NR_GPIO
 	int
 	default 2048 if ARCH_SOCFPGA
-	default 1024 if ARCH_BRCMSTB || ARCH_SHMOBILE || ARCH_TEGRA || \
+	default 1024 if ARCH_BRCMSTB || ARCH_RENESAS || ARCH_TEGRA || \
 		ARCH_ZYNQ
 	default 512 if ARCH_EXYNOS || ARCH_KEYSTONE || SOC_OMAP5 || \
 		SOC_DRA7XX || ARCH_S3C24XX || ARCH_S3C64XX || ARCH_S5PV210
@@ -1704,6 +1700,7 @@ config ARCH_WANT_GENERAL_HUGETLB
 config ARM_MODULE_PLTS
 	bool "Use PLTs to allow module memory to spill over into vmalloc area"
 	depends on MODULES
+	default y
 	help
 	  Allocate PLTs when loading modules so that jumps and calls whose
 	  targets are too far away for their relative offsets to be encoded
@@ -1714,7 +1711,8 @@ config ARM_MODULE_PLTS
 	  rounding up to page size, the actual memory footprint is usually
 	  the same.
 
-	  Say y if you are getting out of memory errors while loading modules
+	  Disabling this is usually safe for small single-platform
+	  configurations. If unsure, say y.
 
 source "mm/Kconfig"
 
@@ -1778,12 +1776,6 @@ config SECCOMP
 	  and the task is only allowed to execute a few safe syscalls
 	  defined by each seccomp mode.
 
-config SWIOTLB
-	def_bool y
-
-config IOMMU_HELPER
-	def_bool SWIOTLB
-
 config PARAVIRT
 	bool "Enable paravirtualization code"
 	help
@@ -1815,6 +1807,7 @@ config XEN
 	depends on MMU
 	select ARCH_DMA_ADDR_T_64BIT
 	select ARM_PSCI
+	select SWIOTLB
 	select SWIOTLB_XEN
 	select PARAVIRT
 	help
diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug
index 199ebc1c4538..693f84392f1b 100644
--- a/arch/arm/Kconfig.debug
+++ b/arch/arm/Kconfig.debug
@@ -942,6 +942,13 @@ choice
 		  via SCIF0 on Renesas RZ/G1M (R8A7743), R-Car H2 (R8A7790),
 		  M2-W (R8A7791), V2H (R8A7792), or M2-N (R8A7793).
 
+	config DEBUG_RCAR_GEN2_SCIF1
+		bool "Kernel low-level debugging messages via SCIF1 on R8A77470"
+		depends on ARCH_R8A77470
+		help
+		  Say Y here if you want kernel low-level debugging support
+		  via SCIF1 on Renesas RZ/G1C (R8A77470).
+
 	config DEBUG_RCAR_GEN2_SCIF2
 		bool "Kernel low-level debugging messages via SCIF2 on R8A7794"
 		depends on ARCH_R8A7794
@@ -1495,6 +1502,7 @@ config DEBUG_LL_INCLUDE
 	default "debug/renesas-scif.S" if DEBUG_RCAR_GEN1_SCIF0
 	default "debug/renesas-scif.S" if DEBUG_RCAR_GEN1_SCIF2
 	default "debug/renesas-scif.S" if DEBUG_RCAR_GEN2_SCIF0
+	default "debug/renesas-scif.S" if DEBUG_RCAR_GEN2_SCIF1
 	default "debug/renesas-scif.S" if DEBUG_RCAR_GEN2_SCIF2
 	default "debug/renesas-scif.S" if DEBUG_RCAR_GEN2_SCIF4
 	default "debug/renesas-scif.S" if DEBUG_RMOBILE_SCIFA0
@@ -1617,6 +1625,7 @@ config DEBUG_UART_PHYS
 	default 0xe6c80000 if DEBUG_RMOBILE_SCIFA4
 	default 0xe6e58000 if DEBUG_RCAR_GEN2_SCIF2
 	default 0xe6e60000 if DEBUG_RCAR_GEN2_SCIF0
+	default 0xe6e68000 if DEBUG_RCAR_GEN2_SCIF1
 	default 0xe6ee0000 if DEBUG_RCAR_GEN2_SCIF4
 	default 0xe8008000 if DEBUG_R7S72100_SCIF2
 	default 0xf0000be0 if ARCH_EBSA110
@@ -1651,8 +1660,8 @@ config DEBUG_UART_PHYS
 		DEBUG_NETX_UART || \
 		DEBUG_QCOM_UARTDM || DEBUG_R7S72100_SCIF2 || \
 		DEBUG_RCAR_GEN1_SCIF0 || DEBUG_RCAR_GEN1_SCIF2 || \
-		DEBUG_RCAR_GEN2_SCIF0 || DEBUG_RCAR_GEN2_SCIF2 || \
-		DEBUG_RCAR_GEN2_SCIF4 || \
+		DEBUG_RCAR_GEN2_SCIF0 || DEBUG_RCAR_GEN2_SCIF1 || \
+		DEBUG_RCAR_GEN2_SCIF2 || DEBUG_RCAR_GEN2_SCIF4 || \
 		DEBUG_RMOBILE_SCIFA0 || DEBUG_RMOBILE_SCIFA1 || \
 		DEBUG_RMOBILE_SCIFA4 || DEBUG_S3C24XX_UART || \
 		DEBUG_S3C64XX_UART || \
diff --git a/arch/arm/Makefile b/arch/arm/Makefile
index e4e537f27339..336c0cfcb579 100644
--- a/arch/arm/Makefile
+++ b/arch/arm/Makefile
@@ -106,7 +106,7 @@ tune-$(CONFIG_CPU_V6K)		=$(call cc-option,-mtune=arm1136j-s,-mtune=strongarm)
 tune-y := $(tune-y)
 
 ifeq ($(CONFIG_AEABI),y)
-CFLAGS_ABI	:=-mabi=aapcs-linux -mno-thumb-interwork -mfpu=vfp
+CFLAGS_ABI	:=-mabi=aapcs-linux -mfpu=vfp
 else
 CFLAGS_ABI	:=$(call cc-option,-mapcs-32,-mabi=apcs-gnu) $(call cc-option,-mno-thumb-interwork,)
 endif
@@ -212,7 +212,7 @@ machine-$(CONFIG_ARCH_S3C24XX)		+= s3c24xx
 machine-$(CONFIG_ARCH_S3C64XX)		+= s3c64xx
 machine-$(CONFIG_ARCH_S5PV210)		+= s5pv210
 machine-$(CONFIG_ARCH_SA1100)		+= sa1100
-machine-$(CONFIG_ARCH_SHMOBILE) 	+= shmobile
+machine-$(CONFIG_ARCH_RENESAS)	 	+= shmobile
 machine-$(CONFIG_ARCH_SIRF)		+= prima2
 machine-$(CONFIG_ARCH_SOCFPGA)		+= socfpga
 machine-$(CONFIG_ARCH_STI)		+= sti
diff --git a/arch/arm/arm-soc-for-next-contents.txt b/arch/arm/arm-soc-for-next-contents.txt
new file mode 100644
index 000000000000..cef2b2cc3e30
--- /dev/null
+++ b/arch/arm/arm-soc-for-next-contents.txt
@@ -0,0 +1,203 @@
+next/soc
+	ux500/machine
+		git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-stericsson tags/ux500-machine-arm-soc
+	patch
+		ARM: stm32: Select DMA, DMAMUX and MDMA support on STM32MP157C
+	omap/ti-sysc
+		git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap tags/omap-for-v4.18/ti-sysc-signed
+	brcm/soc
+		https://github.com/Broadcom/stblinux tags/arm-soc/for-4.18/soc
+	samsung/soc
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-soc-4.18
+	davinci/soc
+		git://git.kernel.org/pub/scm/linux/kernel/git/nsekhar/linux-davinci tags/davinci-for-v4.18/soc
+	tegra/soc
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-4.18-arm-soc
+	imx/soc
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-soc-4.18
+	sunxi/core
+		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux tags/sunxi-core-for-4.18
+	omap/ti-sysc-fix
+		git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap tags/omap-for-v4.18/ti-sysc-fix-signed
+	omap/soc
+		git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap tags/omap-for-v4.18/soc-signed
+	berlin/core
+		git://git.kernel.org/pub/scm/linux/kernel/git/jszhang/linux-berlin tags/berlin-core-for-v4.18
+	omap/soc-late
+		git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap tags/omap-for-v4.18/soc-late-signed
+	patch
+		ARM: stm32: Don't select DMA unconditionally on STM32MP157C
+
+next/dt
+	ux500/dts
+		git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-stericsson tags/ux500-dts-arm-soc
+	stm32/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32 tags/stm32-dt-for-v4.18-1
+	omap/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap tags/omap-for-v4.18/dt-signed
+	keystone/dts
+		git://git.kernel.org/pub/scm/linux/kernel/git/ssantosh/linux-keystone tags/keystone_dts_for_4.18
+	omap/dt-sdhci
+		git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap tags/omap-for-v4.18/dt-sdhci-signed
+		contains depends/sdhci-omap
+	brcm/dt
+		https://github.com/Broadcom/stblinux tags/arm-soc/for-4.18/devicetree
+	gemini/dts
+		git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-nomadik tags/gemini-dts-arm-soc
+	samsung/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt-4.18
+	samsung/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt64-4.18
+	mediatek/dts32
+		https://git.kernel.org/pub/scm/linux/kernel/git/matthias.bgg/linux tags/v4.17-next-dts32
+	mediatek/dts64
+		https://git.kernel.org/pub/scm/linux/kernel/git/matthias.bgg/linux tags/v4.17-next-dts64
+	sti/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/pchotard/sti tags/sti-dt-for-v4.18-round1
+	vexpress/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux tags/vexpress-updates-4.18
+	juno/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux tags/juno-updates-4.18
+	rockchip/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v4.18-rockchip-dts32-1
+	rockchip/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v4.18-rockchip-dts64-1
+	hisi/dt64-v2
+		git://github.com/hisilicon/linux-hisi tags/hisi-arm64-dt-for-4.18v2
+	amlogic/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-amlogic tags/amlogic-dt
+	amlogic/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-amlogic tags/amlogic-dt64
+	davinci/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/nsekhar/linux-davinci tags/davinci-for-v4.18/dt
+	tegra/dt-bindings
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-4.18-dt-bindings
+	tegra/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-4.18-arm-dt
+	mvebu/dt
+		git://git.infradead.org/linux-mvebu tags/mvebu-dt-4.18-1
+	mvebu/dt64
+		git://git.infradead.org/linux-mvebu tags/mvebu-dt64-4.18-1
+	uniphier/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-uniphier tags/uniphier-dt-v4.18
+	uniphier/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-uniphier tags/uniphier-dt64-v4.18
+	imx/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-dt-4.18
+	imx/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-dt64-4.18
+	sunxi/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux tags/sunxi-dt-for-4.18
+	sunxi/h3-h5
+		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux tags/sunxi-h3-h5-for-4.18
+	sunxi/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux tags/sunxi-dt64-for-4.18
+	omap/dt-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap tags/omap-for-v4.18/dt-part2-signed
+	qcom/arm64
+		git://git.kernel.org/pub/scm/linux/kernel/git/agross/linux tags/qcom-arm64-for-4.18
+	qcom/dts
+		git://git.kernel.org/pub/scm/linux/kernel/git/agross/linux tags/qcom-dts-for-4.18
+	amlogic/dt-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-amlogic tags/amlogic-dt-2
+	amlogic/dt64-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-amlogic tags/amlogic-dt64-2
+	amlogic/dt64-3
+		https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-amlogic tags/amlogic-dt64-3
+	berlin/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/jszhang/linux-berlin tags/berlin64-dt-for-v4.18
+	berlin/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/jszhang/linux-berlin tags/berlin-dt-for-v4.18
+	samsung/dt-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt-4.18-2
+	samsung/dt64-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt64-4.18-2
+	patch
+		arm64: dts: sprd: fix typo in 'remote-endpoint'
+	aspeed/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/joel/aspeed tags/aspeed-4.18-devicetree
+	patch
+		arm64: dts: sprd: Add GPIO and GPIO keys device nodes
+		arm64: dts: sprd: whale2: Add the rtc enable clock for watchdog
+	at91/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux tags/at91-ab-4.18-dt
+	qcom/arm64-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/agross/linux tags/qcom-arm64-for-4.18-2
+	qcom/dts-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/agross/linux tags/qcom-dts-for-4.18-2
+	renesas/dt-bindings
+		https://git.kernel.org/pub/scm/linux/kernel/git/horms/renesas tags/renesas-dt-bindings-for-v4.18
+
+next/drivers
+	patch
+		bus: arm-cci: remove unnecessary unreachable()
+	keystone/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/ssantosh/linux-keystone tags/soc_drivers_for_4.18
+	omap/gpmc
+		https://github.com/rogerq/linux tags/gpmc-omap-for-v4.18
+	hisi/drivers
+		git://github.com/hisilicon/linux-hisi tags/hisi-drivers-for-4.18
+	brcm/drivers
+		https://github.com/Broadcom/stblinux tags/arm-soc/for-4.18/drivers
+	samsung/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-drivers-4.18
+	mediatek/soc
+		https://git.kernel.org/pub/scm/linux/kernel/git/matthias.bgg/linux tags/v4.17-next-soc
+		contains depends/regmap-ktime-fix
+	drivers/scmi
+		git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux tags/scmi-updates-4.18
+	drivers/reset
+		git://git.pengutronix.de/pza/linux tags/reset-for-4.18
+	rockchip/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v4.18-rockchip-drivers-1
+	tegra/memory
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-4.18-memory-v2
+	imx/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-drivers-4.18
+	rockchip/drivers-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v4.18-rockchip-drivers-2
+
+next/defconfig
+	patch
+		ARM: multi_v7_defconfig: Add STM32F7 I2C & STM32 USBPHYC support
+	brcm/defconfig
+		https://github.com/Broadcom/stblinux tags/arm-soc/for-4.18/defconfig
+	brcm/defconfig-arm64
+		https://github.com/Broadcom/stblinux tags/arm-soc/for-4.18/defconfig-arm64
+	samsung/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-defconfig-4.18
+	mediatek/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/matthias.bgg/linux tags/v4.17-next-defconfig
+	rockchip/defconfig
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v4.18-rockchip-defconfig64-1
+	amlogic/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-amlogic tags/amlogic-defconfig
+	davinci/defconfig
+		git://git.kernel.org/pub/scm/linux/kernel/git/nsekhar/linux-davinci tags/davinci-for-v4.18/defconfig
+	renesas/arm64-defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/horms/renesas tags/renesas-arm64-defconfig-for-v4.18
+	mvebu/arm64
+		git://git.infradead.org/linux-mvebu tags/mvebu-arm64-4.18-1
+	imx/defconfig
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-defconfig-4.18
+	hisi/defconfig
+		git://github.com/hisilicon/linux-hisi tags/hisi-defconfig-for-4.18v3
+	qcom/arm64-defconfig
+		git://git.kernel.org/pub/scm/linux/kernel/git/agross/linux tags/qcom-arm64-defconfig-for-4.18
+	renesas/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/horms/renesas tags/renesas-defconfig-for-v4.18
+
+next/late
+	qcom/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/agross/linux tags/qcom-drivers-for-4.18
+
+fixes
+	<no branch> (d89ad4f0b44dde8c5333ef7a5d5824829cc72de3)
+		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux tags/sunxi-fixes-for-4.17
+	<no branch> (5ec3444c8346c922316f4effafa8afc17defda0e)
+		git://git.kernel.org/pub/scm/linux/kernel/git/agross/linux tags/qcom-fixes-for-4.17-rc7
+	patch
+		ARM: Fix i2c-gpio GPIO descriptor tables
+	<no branch> (9c6d26df1fae6ad4718d51c48e6517913304ed27)
+		git://github.com/hisilicon/linux-hisi tags/hisi-fixes-for-4.17v2
+
diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile
index 6a4e7341ecd3..1f5a5ffe7fcf 100644
--- a/arch/arm/boot/compressed/Makefile
+++ b/arch/arm/boot/compressed/Makefile
@@ -25,6 +25,9 @@ endif
 
 GCOV_PROFILE		:= n
 
+# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in.
+KCOV_INSTRUMENT		:= n
+
 #
 # Architecture dependencies
 #
@@ -113,7 +116,7 @@ CFLAGS_fdt_ro.o := $(nossp_flags)
 CFLAGS_fdt_rw.o := $(nossp_flags)
 CFLAGS_fdt_wip.o := $(nossp_flags)
 
-ccflags-y := -fpic -mno-single-pic-base -fno-builtin -I$(obj)
+ccflags-y := -fpic $(call cc-option,-mno-single-pic-base,) -fno-builtin -I$(obj)
 asflags-y := -DZIMAGE
 
 # Supply kernel BSS size to the decompressor via a linker symbol.
diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile
index 7e2424957809..37a3de760d40 100644
--- a/arch/arm/boot/dts/Makefile
+++ b/arch/arm/boot/dts/Makefile
@@ -75,6 +75,7 @@ dtb-$(CONFIG_ARCH_BCM2835) += \
 	bcm2835-rpi-a-plus.dtb \
 	bcm2836-rpi-2-b.dtb \
 	bcm2837-rpi-3-b.dtb \
+	bcm2837-rpi-3-b-plus.dtb \
 	bcm2835-rpi-zero.dtb \
 	bcm2835-rpi-zero-w.dtb
 dtb-$(CONFIG_ARCH_BCM_5301X) += \
@@ -102,8 +103,10 @@ dtb-$(CONFIG_ARCH_BCM_5301X) += \
 	bcm47094-dlink-dir-885l.dtb \
 	bcm47094-linksys-panamera.dtb \
 	bcm47094-luxul-abr-4500.dtb \
+	bcm47094-luxul-xap-1610.dtb \
 	bcm47094-luxul-xbr-4500.dtb \
 	bcm47094-luxul-xwr-3100.dtb \
+	bcm47094-luxul-xwr-3150-v1.dtb \
 	bcm47094-netgear-r8500.dtb \
 	bcm94708.dtb \
 	bcm94709.dtb \
@@ -140,6 +143,7 @@ dtb-$(CONFIG_ARCH_BCM_NSP) += \
 dtb-$(CONFIG_ARCH_BERLIN) += \
 	berlin2-sony-nsz-gs7.dtb \
 	berlin2cd-google-chromecast.dtb \
+	berlin2cd-valve-steamlink.dtb \
 	berlin2q-marvell-dmp.dtb
 dtb-$(CONFIG_ARCH_BRCMSTB) += \
 	bcm7445-bcm97445svmb.dtb
@@ -190,8 +194,6 @@ dtb-$(CONFIG_ARCH_EXYNOS5) += \
 	exynos5422-odroidxu3.dtb \
 	exynos5422-odroidxu3-lite.dtb \
 	exynos5422-odroidxu4.dtb \
-	exynos5440-sd5v1.dtb \
-	exynos5440-ssdk5440.dtb \
 	exynos5800-peach-pi.dtb
 dtb-$(CONFIG_ARCH_GEMINI) += \
 	gemini-dlink-dir-685.dtb \
@@ -312,14 +314,14 @@ dtb-$(CONFIG_ARCH_NPCM7XX) += \
 dtb-$(CONFIG_MACH_MESON6) += \
 	meson6-atv1200.dtb
 dtb-$(CONFIG_MACH_MESON8) += \
-	meson8-minix-neo-x8.dtb
+	meson8-minix-neo-x8.dtb \
+	meson8b-mxq.dtb \
+	meson8b-odroidc1.dtb \
+	meson8m2-mxiii-plus.dtb
 dtb-$(CONFIG_ARCH_MMP) += \
 	pxa168-aspenite.dtb \
 	pxa910-dkb.dtb \
 	mmp2-brownstone.dtb
-dtb-$(CONFIG_MACH_MESON8B) += \
-	meson8b-mxq.dtb \
-	meson8b-odroidc1.dtb
 dtb-$(CONFIG_ARCH_MPS2) += \
 	mps2-an385.dtb \
 	mps2-an399.dtb
@@ -399,6 +401,7 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6dl-hummingboard2-som-v15.dtb \
 	imx6dl-icore.dtb \
 	imx6dl-icore-rqs.dtb \
+	imx6dl-mamoj.dtb \
 	imx6dl-nit6xlite.dtb \
 	imx6dl-nitrogen6x.dtb \
 	imx6dl-phytec-mira-rdk-nand.dtb \
@@ -439,6 +442,7 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6q-cubox-i-emmc-som-v15.dtb \
 	imx6q-cubox-i-som-v15.dtb \
 	imx6q-dfi-fs700-m60.dtb \
+	imx6q-dhcom-pdk2.dtb \
 	imx6q-display5-tianma-tm070-1280x768.dtb \
 	imx6q-dmo-edmqmx6.dtb \
 	imx6q-dms-ba16.dtb \
@@ -463,9 +467,11 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6q-hummingboard2-emmc-som-v15.dtb \
 	imx6q-hummingboard2-som-v15.dtb \
 	imx6q-icore.dtb \
+	imx6q-icore-mipi.dtb \
 	imx6q-icore-ofcap10.dtb \
 	imx6q-icore-ofcap12.dtb \
 	imx6q-icore-rqs.dtb \
+	imx6q-kp-tpc.dtb \
 	imx6q-marsboard.dtb \
 	imx6q-mccmon6.dtb \
 	imx6q-nitrogen6x.dtb \
@@ -688,6 +694,7 @@ dtb-$(CONFIG_SOC_AM33XX) += \
 	am335x-pdu001.dtb \
 	am335x-pepper.dtb \
 	am335x-phycore-rdk.dtb \
+	am335x-pocketbeagle.dtb \
 	am335x-shc.dtb \
 	am335x-sbc-t335.dtb \
 	am335x-sl50.dtb \
@@ -760,12 +767,17 @@ dtb-$(CONFIG_ARCH_QCOM) += \
 	qcom-apq8084-ifc6540.dtb \
 	qcom-apq8084-mtp.dtb \
 	qcom-ipq4019-ap.dk01.1-c1.dtb \
+	qcom-ipq4019-ap.dk04.1-c1.dtb \
+	qcom-ipq4019-ap.dk04.1-c3.dtb \
+	qcom-ipq4019-ap.dk07.1-c1.dtb \
+	qcom-ipq4019-ap.dk07.1-c2.dtb \
 	qcom-ipq8064-ap148.dtb \
 	qcom-msm8660-surf.dtb \
 	qcom-msm8960-cdp.dtb \
 	qcom-msm8974-fairphone-fp2.dtb \
 	qcom-msm8974-lge-nexus5-hammerhead.dtb \
 	qcom-msm8974-samsung-klte.dtb \
+	qcom-msm8974-sony-xperia-amami.dtb \
 	qcom-msm8974-sony-xperia-castor.dtb \
 	qcom-msm8974-sony-xperia-honami.dtb \
 	qcom-mdm9615-wp8548-mangoh-green.dtb
@@ -795,6 +807,7 @@ dtb-$(CONFIG_ARCH_RENESAS) += \
 	r8a7745-iwg22d-sodimm.dtb \
 	r8a7745-iwg22d-sodimm-dbhd-ca.dtb \
 	r8a7745-sk-rzg1e.dtb \
+	r8a77470-iwg23s-sbc.dtb \
 	r8a7778-bockw.dtb \
 	r8a7779-marzen.dtb \
 	r8a7790-lager.dtb \
@@ -959,6 +972,7 @@ dtb-$(CONFIG_MACH_SUN7I) += \
 	sun7i-a20-m3.dtb \
 	sun7i-a20-mk808c.dtb \
 	sun7i-a20-olimex-som-evb.dtb \
+	sun7i-a20-olimex-som-evb-emmc.dtb \
 	sun7i-a20-olimex-som204-evb.dtb \
 	sun7i-a20-olimex-som204-evb-emmc.dtb \
 	sun7i-a20-olinuxino-lime.dtb \
@@ -992,8 +1006,9 @@ dtb-$(CONFIG_MACH_SUN8I) += \
 	sun8i-a83t-bananapi-m3.dtb \
 	sun8i-a83t-cubietruck-plus.dtb \
 	sun8i-a83t-tbs-a711.dtb \
-	sun8i-h2-plus-orangepi-r1.dtb \
 	sun8i-h2-plus-bananapi-m2-zero.dtb \
+	sun8i-h2-plus-libretech-all-h3-cc.dtb \
+	sun8i-h2-plus-orangepi-r1.dtb \
 	sun8i-h2-plus-orangepi-zero.dtb \
 	sun8i-h3-bananapi-m2-plus.dtb \
 	sun8i-h3-beelink-x2.dtb \
@@ -1010,6 +1025,8 @@ dtb-$(CONFIG_MACH_SUN8I) += \
 	sun8i-h3-orangepi-plus.dtb \
 	sun8i-h3-orangepi-plus2e.dtb \
 	sun8i-r16-bananapi-m2m.dtb \
+	sun8i-r16-nintendo-nes-classic.dtb \
+	sun8i-r16-nintendo-super-nes-classic.dtb \
 	sun8i-r16-parrot.dtb \
 	sun8i-r40-bananapi-m2-ultra.dtb \
 	sun8i-v3s-licheepi-zero.dtb \
@@ -1054,9 +1071,7 @@ dtb-$(CONFIG_ARCH_U8500) += \
 	ste-hrefprev60-stuib.dtb \
 	ste-hrefprev60-tvk.dtb \
 	ste-hrefv60plus-stuib.dtb \
-	ste-hrefv60plus-tvk.dtb \
-	ste-ccu8540.dtb \
-	ste-ccu9540.dtb
+	ste-hrefv60plus-tvk.dtb
 dtb-$(CONFIG_ARCH_UNIPHIER) += \
 	uniphier-ld4-ref.dtb \
 	uniphier-ld6b-ref.dtb \
@@ -1150,6 +1165,9 @@ dtb-$(CONFIG_ARCH_MEDIATEK) += \
 	mt6580-evbp1.dtb \
 	mt6589-aquaris5.dtb \
 	mt6592-evb.dtb \
+	mt7623a-rfb-emmc.dtb \
+	mt7623a-rfb-nand.dtb \
+	mt7623n-rfb-emmc.dtb \
 	mt7623n-rfb-nand.dtb \
 	mt7623n-bananapi-bpi-r2.dtb \
 	mt8127-moose.dtb \
@@ -1158,8 +1176,11 @@ dtb-$(CONFIG_ARCH_ZX) += zx296702-ad1.dtb
 dtb-$(CONFIG_ARCH_ASPEED) += \
 	aspeed-ast2500-evb.dtb \
 	aspeed-bmc-arm-centriq2400-rep.dtb \
+	aspeed-bmc-intel-s2600wf.dtb \
+	aspeed-bmc-opp-lanyang.dtb \
 	aspeed-bmc-opp-palmetto.dtb \
 	aspeed-bmc-opp-romulus.dtb \
 	aspeed-bmc-opp-witherspoon.dtb \
 	aspeed-bmc-opp-zaius.dtb \
+	aspeed-bmc-portwell-neptune.dtb \
 	aspeed-bmc-quanta-q71l.dtb
diff --git a/arch/arm/boot/dts/am335x-baltos-ir3220.dts b/arch/arm/boot/dts/am335x-baltos-ir3220.dts
index 46df1b22022c..1b215c425c57 100644
--- a/arch/arm/boot/dts/am335x-baltos-ir3220.dts
+++ b/arch/arm/boot/dts/am335x-baltos-ir3220.dts
@@ -85,7 +85,7 @@
 		gpio-controller;
 		#gpio-cells = <2>;
 		interrupt-parent = <&gpio0>;
-		interrupts = <20 GPIO_ACTIVE_LOW>;
+		interrupts = <20 IRQ_TYPE_EDGE_RISING>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&tca6416_pins>;
 	};
diff --git a/arch/arm/boot/dts/am335x-baltos-ir5221.dts b/arch/arm/boot/dts/am335x-baltos-ir5221.dts
index 5d56355ba040..832ead864dc5 100644
--- a/arch/arm/boot/dts/am335x-baltos-ir5221.dts
+++ b/arch/arm/boot/dts/am335x-baltos-ir5221.dts
@@ -94,7 +94,7 @@
 		gpio-controller;
 		#gpio-cells = <2>;
 		interrupt-parent = <&gpio0>;
-		interrupts = <20 GPIO_ACTIVE_LOW>;
+		interrupts = <20 IRQ_TYPE_EDGE_RISING>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&tca6416_pins>;
 	};
diff --git a/arch/arm/boot/dts/am335x-baltos.dtsi b/arch/arm/boot/dts/am335x-baltos.dtsi
index ec6052c521ef..ed7a5a3daa42 100644
--- a/arch/arm/boot/dts/am335x-baltos.dtsi
+++ b/arch/arm/boot/dts/am335x-baltos.dtsi
@@ -249,7 +249,7 @@
 		gpio-controller;
 		#gpio-cells = <2>;
 		interrupt-parent = <&gpio1>;
-		interrupts = <28 GPIO_ACTIVE_LOW>;
+		interrupts = <28 IRQ_TYPE_EDGE_RISING>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&tps65910_pins>;
 	};
diff --git a/arch/arm/boot/dts/am335x-bone-common.dtsi b/arch/arm/boot/dts/am335x-bone-common.dtsi
index e67b4d65c8d0..f9e8667f5886 100644
--- a/arch/arm/boot/dts/am335x-bone-common.dtsi
+++ b/arch/arm/boot/dts/am335x-bone-common.dtsi
@@ -161,7 +161,14 @@
 
 	mmc1_pins: pinmux_mmc1_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) /* GPIO0_6 */
+			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7)		/* spio0_cs1.gpio0_6 */
+			AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat0.mmc0_dat0 */
+			AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat1.mmc0_dat1 */
+			AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat2.mmc0_dat2 */
+			AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat3.mmc0_dat3 */
+			AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_cmd.mmc0_cmd */
+			AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_clk.mmc0_clk */
+			AM33XX_IOPAD(0x9a0, PIN_INPUT | MUX_MODE4)		/* mcasp0_aclkr.mmc0_sdwp */
 		>;
 	};
 
diff --git a/arch/arm/boot/dts/am335x-boneblue.dts b/arch/arm/boot/dts/am335x-boneblue.dts
index 58baee158e64..7bcd72691f06 100644
--- a/arch/arm/boot/dts/am335x-boneblue.dts
+++ b/arch/arm/boot/dts/am335x-boneblue.dts
@@ -364,7 +364,7 @@
 		compatible = "invensense,mpu9250";
 		reg = <0x68>;
 		interrupt-parent = <&gpio3>;
-		interrupts = <21 GPIO_ACTIVE_LOW>;
+		interrupts = <21 IRQ_TYPE_EDGE_RISING>;
 		i2c-gate {
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/arch/arm/boot/dts/am335x-evm.dts b/arch/arm/boot/dts/am335x-evm.dts
index fee6b3ee1741..1356fd6f8da3 100644
--- a/arch/arm/boot/dts/am335x-evm.dts
+++ b/arch/arm/boot/dts/am335x-evm.dts
@@ -303,7 +303,14 @@
 
 	mmc1_pins: pinmux_mmc1_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) /* spi0_cs1.gpio0_6 */
+			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7)		/* spi0_cs1.gpio0_6 */
+			AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat0.mmc0_dat0 */
+			AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat1.mmc0_dat1 */
+			AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat2.mmc0_dat2 */
+			AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat3.mmc0_dat3 */
+			AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_cmd.mmc0_cmd */
+			AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_clk.mmc0_clk */
+			AM33XX_IOPAD(0x9a0, PIN_INPUT | MUX_MODE4)		/* mcasp0_aclkr.mmc0_sdwp */
 		>;
 	};
 
diff --git a/arch/arm/boot/dts/am335x-evmsk.dts b/arch/arm/boot/dts/am335x-evmsk.dts
index fa608cd5dc14..0c096a795e37 100644
--- a/arch/arm/boot/dts/am335x-evmsk.dts
+++ b/arch/arm/boot/dts/am335x-evmsk.dts
@@ -137,7 +137,7 @@
 		};
 	};
 
-	backlight {
+	lcd_bl: backlight {
 		compatible = "pwm-backlight";
 		pwms = <&ecap2 0 50000 PWM_POLARITY_INVERTED>;
 		brightness-levels = <0 58 61 66 75 90 125 170 255>;
@@ -172,6 +172,7 @@
 		pinctrl-names = "default", "sleep";
 		pinctrl-0 = <&lcd_pins_default>;
 		pinctrl-1 = <&lcd_pins_sleep>;
+		backlight = <&lcd_bl>;
 		status = "okay";
 		panel-info {
 			ac-bias		= <255>;
@@ -399,7 +400,14 @@
 
 	mmc1_pins: pinmux_mmc1_pins {
 		pinctrl-single,pins = <
-			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) /* spi0_cs1.gpio0_6 */
+			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) 		/* spi0_cs1.gpio0_6 */
+			AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat0.mmc0_dat0 */
+			AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat1.mmc0_dat1 */
+			AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat2.mmc0_dat2 */
+			AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_dat3.mmc0_dat3 */
+			AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_cmd.mmc0_cmd */
+			AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0)	/* mmc0_clk.mmc0_clk */
+			AM33XX_IOPAD(0x9a0, PIN_INPUT | MUX_MODE4)		/* mcasp0_aclkr.mmc0_sdwp */
 		>;
 	};
 
diff --git a/arch/arm/boot/dts/am335x-osd335x-common.dtsi b/arch/arm/boot/dts/am335x-osd335x-common.dtsi
new file mode 100644
index 000000000000..f8ff473f94f0
--- /dev/null
+++ b/arch/arm/boot/dts/am335x-osd335x-common.dtsi
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Author: Robert Nelson <robertcnelson@gmail.com>
+ */
+
+/ {
+	cpus {
+		cpu@0 {
+			cpu0-supply = <&dcdc2_reg>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0x20000000>; /* 512 MB */
+	};
+};
+
+&cpu0_opp_table {
+	/*
+	* Octavo Systems:
+	* The EFUSE_SMA register is not programmed for any of the AM335x wafers
+	* we get and we are not programming them during our production test.
+	* Therefore, from a DEVICE_ID revision point of view, the silicon looks
+	* like it is Revision 2.1.  However, from an EFUSE_SMA point of view for
+	* the HW OPP table, the silicon looks like it is Revision 1.0 (ie the
+	* EFUSE_SMA register reads as all zeros).
+	*/
+	oppnitro-1000000000 {
+		opp-supported-hw = <0x06 0x0100>;
+	};
+};
+
+&am33xx_pinmux {
+	i2c0_pins: pinmux-i2c0-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0)	/* (C17) I2C0_SDA.I2C0_SDA */
+			AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0)	/* (C16) I2C0_SCL.I2C0_SCL */
+		>;
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins>;
+
+	status = "okay";
+	clock-frequency = <400000>;
+
+	tps: tps@24 {
+		reg = <0x24>;
+	};
+};
+
+/include/ "tps65217.dtsi"
+
+&tps {
+	interrupts = <7>; /* NMI */
+	interrupt-parent = <&intc>;
+
+	ti,pmic-shutdown-controller;
+
+	pwrbutton {
+		interrupts = <2>;
+		status = "okay";
+	};
+
+	regulators {
+		dcdc1_reg: regulator@0 {
+			regulator-name = "vdds_dpr";
+			regulator-always-on;
+		};
+
+		dcdc2_reg: regulator@1 {
+			/* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */
+			regulator-name = "vdd_mpu";
+			regulator-min-microvolt = <925000>;
+			regulator-max-microvolt = <1351500>;
+			regulator-boot-on;
+			regulator-always-on;
+		};
+
+		dcdc3_reg: regulator@2 {
+			/* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */
+			regulator-name = "vdd_core";
+			regulator-min-microvolt = <925000>;
+			regulator-max-microvolt = <1150000>;
+			regulator-boot-on;
+			regulator-always-on;
+		};
+
+		ldo1_reg: regulator@3 {
+			regulator-name = "vio,vrtc,vdds";
+			regulator-always-on;
+		};
+
+		ldo2_reg: regulator@4 {
+			regulator-name = "vdd_3v3aux";
+			regulator-always-on;
+		};
+
+		ldo3_reg: regulator@5 {
+			regulator-name = "vdd_1v8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-always-on;
+		};
+
+		ldo4_reg: regulator@6 {
+			regulator-name = "vdd_3v3a";
+			regulator-always-on;
+		};
+	};
+};
+
+&aes {
+	status = "okay";
+};
+
+&sham {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/am335x-pocketbeagle.dts b/arch/arm/boot/dts/am335x-pocketbeagle.dts
new file mode 100644
index 000000000000..62fe5cab9fae
--- /dev/null
+++ b/arch/arm/boot/dts/am335x-pocketbeagle.dts
@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Author: Robert Nelson <robertcnelson@gmail.com>
+ */
+/dts-v1/;
+
+#include "am33xx.dtsi"
+#include "am335x-osd335x-common.dtsi"
+
+/ {
+	model = "TI AM335x PocketBeagle";
+	compatible = "ti,am335x-pocketbeagle", "ti,am335x-bone", "ti,am33xx";
+
+	chosen {
+		stdout-path = &uart0;
+	};
+
+	leds {
+		pinctrl-names = "default";
+		pinctrl-0 = <&usr_leds_pins>;
+
+		compatible = "gpio-leds";
+
+		usr0 {
+			label = "beaglebone:green:usr0";
+			gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+			default-state = "off";
+		};
+
+		usr1 {
+			label = "beaglebone:green:usr1";
+			gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "mmc0";
+			default-state = "off";
+		};
+
+		usr2 {
+			label = "beaglebone:green:usr2";
+			gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "cpu0";
+			default-state = "off";
+		};
+
+		usr3 {
+			label = "beaglebone:green:usr3";
+			gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
+
+	vmmcsd_fixed: fixedregulator0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmcsd_fixed";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+};
+
+&am33xx_pinmux {
+	i2c2_pins: pinmux-i2c2-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3)	/* (D17) uart1_rtsn.I2C2_SCL */
+			AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3)	/* (D18) uart1_ctsn.I2C2_SDA */
+		>;
+	};
+
+	ehrpwm0_pins: pinmux-ehrpwm0-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x990, PIN_OUTPUT_PULLDOWN | MUX_MODE1)	/* (A13) mcasp0_aclkx.ehrpwm0A */
+		>;
+	};
+
+	ehrpwm1_pins: pinmux-ehrpwm1-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x848, PIN_OUTPUT_PULLDOWN | MUX_MODE6)	/* (U14) gpmc_a2.ehrpwm1A */
+		>;
+	};
+
+	mmc0_pins: pinmux-mmc0-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7)		/* (C15) spi0_cs1.gpio0[6] */
+			AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0)	/* (G16) mmc0_dat0.mmc0_dat0 */
+			AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0)	/* (G15) mmc0_dat1.mmc0_dat1 */
+			AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0)	/* (F18) mmc0_dat2.mmc0_dat2 */
+			AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0)	/* (F17) mmc0_dat3.mmc0_dat3 */
+			AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0)	/* (G18) mmc0_cmd.mmc0_cmd */
+			AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0)	/* (G17) mmc0_clk.mmc0_clk */
+			AM33XX_IOPAD(0x9a0, PIN_INPUT | MUX_MODE4)		/* (B12) mcasp0_aclkr.mmc0_sdwp */
+		>;
+	};
+
+	spi0_pins: pinmux-spi0-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x950, PIN_INPUT_PULLUP | MUX_MODE0)	/* (A17) spi0_sclk.spi0_sclk */
+			AM33XX_IOPAD(0x954, PIN_INPUT_PULLUP | MUX_MODE0)	/* (B17) spi0_d0.spi0_d0 */
+			AM33XX_IOPAD(0x958, PIN_INPUT_PULLUP | MUX_MODE0)	/* (B16) spi0_d1.spi0_d1 */
+			AM33XX_IOPAD(0x95c, PIN_INPUT_PULLUP | MUX_MODE0)	/* (A16) spi0_cs0.spi0_cs0 */
+		>;
+	};
+
+	spi1_pins: pinmux-spi1-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x964, PIN_INPUT_PULLUP | MUX_MODE4)	/* (C18) eCAP0_in_PWM0_out.spi1_sclk */
+			AM33XX_IOPAD(0x968, PIN_INPUT_PULLUP | MUX_MODE4)	/* (E18) uart0_ctsn.spi1_d0 */
+			AM33XX_IOPAD(0x96c, PIN_INPUT_PULLUP | MUX_MODE4)	/* (E17) uart0_rtsn.spi1_d1 */
+			AM33XX_IOPAD(0x9b0, PIN_INPUT_PULLUP | MUX_MODE4)	/* (A15) xdma_event_intr0.spi1_cs1 */
+		>;
+	};
+
+	usr_leds_pins: pinmux-usr-leds-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x854, PIN_OUTPUT | MUX_MODE7)		/* (V15) gpmc_a5.gpio1[21] - USR_LED_0 */
+			AM33XX_IOPAD(0x858, PIN_OUTPUT | MUX_MODE7)		/* (U15) gpmc_a6.gpio1[22] - USR_LED_1 */
+			AM33XX_IOPAD(0x85c, PIN_OUTPUT | MUX_MODE7)		/* (T15) gpmc_a7.gpio1[23] - USR_LED_2 */
+			AM33XX_IOPAD(0x860, PIN_OUTPUT | MUX_MODE7)		/* (V16) gpmc_a8.gpio1[24] - USR_LED_3 */
+		>;
+	};
+
+	uart0_pins: pinmux-uart0-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0)	/* (E15) uart0_rxd.uart0_rxd */
+			AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* (E16) uart0_txd.uart0_txd */
+		>;
+	};
+
+	uart4_pins: pinmux-uart4-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x870, PIN_INPUT_PULLUP | MUX_MODE6)	/* (T17) gpmc_wait0.uart4_rxd */
+			AM33XX_IOPAD(0x874, PIN_OUTPUT_PULLDOWN | MUX_MODE6)	/* (U17) gpmc_wpn.uart4_txd */
+		>;
+	};
+};
+
+&epwmss0 {
+	status = "okay";
+};
+
+&ehrpwm0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&ehrpwm0_pins>;
+};
+
+&epwmss1 {
+	status = "okay";
+};
+
+&ehrpwm1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&ehrpwm1_pins>;
+};
+
+&i2c0 {
+	eeprom: eeprom@50 {
+		compatible = "atmel,24c256";
+		reg = <0x50>;
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_pins>;
+
+	status = "okay";
+	clock-frequency = <400000>;
+};
+
+&mmc1 {
+	status = "okay";
+	vmmc-supply = <&vmmcsd_fixed>;
+	bus-width = <4>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc0_pins>;
+	cd-gpios = <&gpio0 6 GPIO_ACTIVE_LOW>;
+};
+
+&rtc {
+	system-power-controller;
+};
+
+&tscadc {
+	status = "okay";
+	adc {
+		ti,adc-channels = <0 1 2 3 4 5 6 7>;
+		ti,chan-step-avg = <16 16 16 16 16 16 16 16>;
+		ti,chan-step-opendelay = <0x98 0x98 0x98 0x98 0x98 0x98 0x98 0x98>;
+		ti,chan-step-sampledelay = <0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0>;
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pins>;
+
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart4_pins>;
+
+	status = "okay";
+};
+
+&usb {
+	status = "okay";
+};
+
+&usb_ctrl_mod {
+	status = "okay";
+};
+
+&usb0_phy {
+	status = "okay";
+};
+
+&usb0 {
+	status = "okay";
+	dr_mode = "otg";
+};
+
+&usb1_phy {
+	status = "okay";
+};
+
+&usb1 {
+	status = "okay";
+	dr_mode = "host";
+};
+
+&cppi41dma  {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/am3517-evm.dts b/arch/arm/boot/dts/am3517-evm.dts
index 0e4a125f78e3..98aadb0f81c5 100644
--- a/arch/arm/boot/dts/am3517-evm.dts
+++ b/arch/arm/boot/dts/am3517-evm.dts
@@ -8,11 +8,17 @@
 /dts-v1/;
 
 #include "am3517.dtsi"
+#include "am3517-som.dtsi"
+#include <dt-bindings/input/input.h>
 
 / {
 	model = "TI AM3517 EVM (AM3517/05 TMDSEVM3517)";
 	compatible = "ti,am3517-evm", "ti,am3517", "ti,omap3";
 
+	aliases {
+		display0 = &lcd0;
+	};
+
 	memory@80000000 {
 		device_type = "memory";
 		reg = <0x80000000 0x10000000>; /* 256 MB */
@@ -24,6 +30,144 @@
                 regulator-min-microvolt = <3300000>;
                 regulator-max-microvolt = <3300000>;
         };
+
+	gpio-keys {
+		compatible = "gpio-keys-polled";
+		poll-interval = <100>;
+
+		user_pb {
+			label = "User Push Button";
+			linux,code = <BTN_0>;
+			gpios = <&tca6416 5 GPIO_ACTIVE_LOW>;
+		};
+
+		user_sw_1 {
+			label = "User Switch 1";
+			linux,code = <BTN_1>;
+			gpios = <&tca6416 8 GPIO_ACTIVE_LOW>;
+		};
+
+		user_sw_2 {
+			label = "User Switch 2";
+			linux,code = <BTN_2>;
+			gpios = <&tca6416 9 GPIO_ACTIVE_LOW>;
+		};
+
+		user_sw_3 {
+			label = "User Switch 3";
+			linux,code = <BTN_3>;
+			gpios = <&tca6416 10 GPIO_ACTIVE_LOW>;
+		};
+
+		user_sw_4 {
+			label = "User Switch 4";
+			linux,code = <BTN_4>;
+			gpios = <&tca6416 11 GPIO_ACTIVE_LOW>;
+		};
+
+		user_sw_5 {
+			label = "User Switch 5";
+			linux,code = <BTN_5>;
+			gpios = <&tca6416 12 GPIO_ACTIVE_LOW>;
+		};
+
+		user_sw_6 {
+			label = "User Switch 6";
+			linux,code = <BTN_6>;
+			gpios = <&tca6416 13 GPIO_ACTIVE_LOW>;
+		};
+
+		user_sw_7 {
+			label = "User Switch 7";
+			linux,code = <BTN_7>;
+			gpios = <&tca6416 14 GPIO_ACTIVE_LOW>;
+		};
+
+		user_sw_8 {
+			label = "User Switch 8";
+			linux,code = <BTN_8>;
+			gpios = <&tca6416 15 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&leds_pins>;
+
+		user_led_1 {
+			label = "am3517evm:green:user_led_1";
+			gpios = <&tca6416 7 GPIO_ACTIVE_LOW>;
+			default-state = "on";
+		};
+
+		user_led_2 {
+			label = "am3517evm:green:user_led_2";
+			gpios = <&tca6416 6 GPIO_ACTIVE_LOW>;
+			default-state = "on";
+		};
+
+		user_led_3 {
+			label = "am3517evm:green:user_led_3";
+			gpios = <&gpio1 11 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "mmc0"; /* SD/MMC card activity */
+		};
+
+		user_led_4 {
+			label = "am3517evm:green:user_led_4";
+			gpios = <&gpio1 31 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	lcd0: display@0 {
+		compatible = "panel-dpi";
+		label = "15";
+		status = "okay";
+		pinctrl-names = "default";
+		enable-gpios = <&gpio6 16 GPIO_ACTIVE_HIGH>;	/* gpio176, lcd INI */
+
+		port {
+			lcd_in: endpoint {
+				remote-endpoint = <&dpi_out>;
+			};
+		};
+
+		panel-timing {
+			clock-frequency = <9000000>;
+			hactive = <480>;
+			vactive = <272>;
+			hfront-porch = <3>;
+			hback-porch = <2>;
+			hsync-len = <42>;
+			vback-porch = <3>;
+			vfront-porch = <4>;
+			vsync-len = <11>;
+			hsync-active = <0>;
+			vsync-active = <0>;
+			de-active = <1>;
+			pixelclk-active = <1>;
+		};
+	};
+
+	bl: backlight {
+		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&backlight_pins>;
+		pwms = <&pwm11 0 5000000 0>;
+		brightness-levels = <0 10 20 30 40 50 60 70 80 90 100>;
+		default-brightness-level = <7>;
+		enable-gpios = <&gpio6 22 GPIO_ACTIVE_HIGH>; /* gpio_182 */
+	};
+
+	pwm11: dmtimer-pwm@11 {
+		compatible = "ti,omap-dmtimer-pwm";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pwm_pins>;
+		ti,timers = <&timer11>;
+		#pwm-cells = <3>;
+	};
 };
 
 &davinci_emac {
@@ -34,12 +178,32 @@
 	     status = "okay";
 };
 
-&i2c1 {
-	clock-frequency = <400000>;
+&dss {
+	status = "ok";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&dss_dpi_pins>;
+
+	vdds_dsi-supply = <&vdd_io_reg>;
+	vdda_video-supply = <&vdd_io_reg>;
+
+	port {
+		dpi_out: endpoint {
+			remote-endpoint = <&lcd_in>;
+			data-lines = <16>;
+		};
+	};
 };
 
 &i2c2 {
 	clock-frequency = <400000>;
+	/* User DIP swithes [1:8] / User LEDS [1:2] */
+	tca6416: gpio@21 {
+		compatible = "ti,tca6416";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
 };
 
 &i2c3 {
@@ -47,8 +211,13 @@
 };
 
 &mmc1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc1_pins>;
 	vmmc-supply = <&vmmc_fixed>;
 	bus-width = <4>;
+	wp-gpios = <&gpio4 30 GPIO_ACTIVE_HIGH>; /* gpio_126 */
+	cd-gpios = <&gpio4 31 GPIO_ACTIVE_HIGH>; /* gpio_127 */
 };
 
 &mmc2 {
@@ -59,3 +228,63 @@
       status = "disabled";
 };
 
+&omap3_pmx_core {
+
+	leds_pins: pinmux_leds_pins {
+		pinctrl-single,pins = <
+			OMAP3_WKUP_IOPAD(0x2a24, PIN_OUTPUT_PULLUP | MUX_MODE4)	/* jtag_emu0.gpio_11 */
+			OMAP3_WKUP_IOPAD(0x2a26, PIN_OUTPUT_PULLUP | MUX_MODE4)	/* jtag_emu1.gpio_31 */
+		>;
+	};
+
+	mmc1_pins: pinmux_mmc1_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x2144, PIN_INPUT_PULLUP | MUX_MODE0)	/* sdmmc1_clk.sdmmc1_clk */
+			OMAP3_CORE1_IOPAD(0x2146, PIN_INPUT_PULLUP | MUX_MODE0)	/* sdmmc1_cmd.sdmmc1_cmd */
+			OMAP3_CORE1_IOPAD(0x2148, PIN_INPUT_PULLUP | MUX_MODE0)	/* sdmmc1_dat0.sdmmc1_dat0 */
+			OMAP3_CORE1_IOPAD(0x214a, PIN_INPUT_PULLUP | MUX_MODE0)	/* sdmmc1_dat1.sdmmc1_dat1 */
+			OMAP3_CORE1_IOPAD(0x214c, PIN_INPUT_PULLUP | MUX_MODE0)	/* sdmmc1_dat2.sdmmc1_dat2 */
+			OMAP3_CORE1_IOPAD(0x214e, PIN_INPUT_PULLUP | MUX_MODE0)	/* sdmmc1_dat3.sdmmc1_dat3 */
+			OMAP3_CORE1_IOPAD(0x2150, PIN_INPUT_PULLUP | MUX_MODE4) /* sdmmc1_dat4.gpio_126 */
+			OMAP3_CORE1_IOPAD(0x2152, PIN_INPUT_PULLUP | MUX_MODE4) /* sdmmc1_dat5.gpio_127 */
+		>;
+	};
+
+	pwm_pins: pinmux_pwm_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x21dc, PIN_OUTPUT | MUX_MODE1)       /* mcspi2_cs0.gpt11_pwm */
+		>;
+	};
+
+	backlight_pins: pinmux_backlight_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x21de, PIN_OUTPUT | MUX_MODE4)       /* mcspi2_cs1.gpio_182 */
+		>;
+	};
+
+	dss_dpi_pins: pinmux_dss_dpi_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x21d2, PIN_OUTPUT | MUX_MODE4)       /* mcspi1_cs2.gpio_176 */
+			OMAP3_CORE1_IOPAD(0x20d4, PIN_OUTPUT | MUX_MODE0)       /* dss_pclk.dss_pclk */
+			OMAP3_CORE1_IOPAD(0x20d6, PIN_OUTPUT | MUX_MODE0)       /* dss_hsync.dss_hsync */
+			OMAP3_CORE1_IOPAD(0x20d8, PIN_OUTPUT | MUX_MODE0)       /* dss_vsync.dss_vsync */
+			OMAP3_CORE1_IOPAD(0x20da, PIN_OUTPUT | MUX_MODE0)       /* dss_acbias.dss_acbias */
+			OMAP3_CORE1_IOPAD(0x20dc, PIN_OUTPUT | MUX_MODE0)       /* dss_data0.dss_data0 */
+			OMAP3_CORE1_IOPAD(0x20de, PIN_OUTPUT | MUX_MODE0)       /* dss_data1.dss_data1 */
+			OMAP3_CORE1_IOPAD(0x20e0, PIN_OUTPUT | MUX_MODE0)       /* dss_data2.dss_data2 */
+			OMAP3_CORE1_IOPAD(0x20e2, PIN_OUTPUT | MUX_MODE0)       /* dss_data3.dss_data3 */
+			OMAP3_CORE1_IOPAD(0x20e4, PIN_OUTPUT | MUX_MODE0)       /* dss_data4.dss_data4 */
+			OMAP3_CORE1_IOPAD(0x20e6, PIN_OUTPUT | MUX_MODE0)       /* dss_data5.dss_data5 */
+			OMAP3_CORE1_IOPAD(0x20e8, PIN_OUTPUT | MUX_MODE0)       /* dss_data6.dss_data6 */
+			OMAP3_CORE1_IOPAD(0x20ea, PIN_OUTPUT | MUX_MODE0)       /* dss_data7.dss_data7 */
+			OMAP3_CORE1_IOPAD(0x20ec, PIN_OUTPUT | MUX_MODE0)       /* dss_data8.dss_data8 */
+			OMAP3_CORE1_IOPAD(0x20ee, PIN_OUTPUT | MUX_MODE0)       /* dss_data9.dss_data9 */
+			OMAP3_CORE1_IOPAD(0x20f0, PIN_OUTPUT | MUX_MODE0)       /* dss_data10.dss_data10 */
+			OMAP3_CORE1_IOPAD(0x20f2, PIN_OUTPUT | MUX_MODE0)       /* dss_data11.dss_data11 */
+			OMAP3_CORE1_IOPAD(0x20f4, PIN_OUTPUT | MUX_MODE0)       /* dss_data12.dss_data12 */
+			OMAP3_CORE1_IOPAD(0x20f6, PIN_OUTPUT | MUX_MODE0)       /* dss_data13.dss_data13 */
+			OMAP3_CORE1_IOPAD(0x20f8, PIN_OUTPUT | MUX_MODE0)       /* dss_data14.dss_data14 */
+			OMAP3_CORE1_IOPAD(0x20fa, PIN_OUTPUT | MUX_MODE0)       /* dss_data15.dss_data15 */
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/am3517-som.dtsi b/arch/arm/boot/dts/am3517-som.dtsi
new file mode 100644
index 000000000000..a6d5ff73c163
--- /dev/null
+++ b/arch/arm/boot/dts/am3517-som.dtsi
@@ -0,0 +1,142 @@
+/*
+ * Copyright (C) 2016 Derald D. Woods <woods.technical@gmail.com>
+ *
+ * Based on am3517-evm.dts
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/ {
+	cpus {
+		cpu@0 {
+			cpu0-supply = <&vdd_core_reg>;
+		};
+	};
+};
+
+&gpmc {
+	ranges = <0 0 0x30000000 0x1000000>;	/* CS0: 16MB for NAND */
+
+	nand@0,0 {
+		compatible = "ti,omap2-nand";
+		linux,mtd-name = "micron,mt29f4g16abchch";
+		reg = <0 0 4>; /* CS0, offset 0, IO size 4 */
+		nand-bus-width = <16>;
+		ti,nand-ecc-opt = "bch8";
+		gpmc,sync-clk-ps = <0>;
+		gpmc,cs-on-ns = <0>;
+		gpmc,cs-rd-off-ns = <44>;
+		gpmc,cs-wr-off-ns = <44>;
+		gpmc,adv-on-ns = <6>;
+		gpmc,adv-rd-off-ns = <34>;
+		gpmc,adv-wr-off-ns = <44>;
+		gpmc,we-off-ns = <40>;
+		gpmc,oe-off-ns = <54>;
+		gpmc,access-ns = <64>;
+		gpmc,rd-cycle-ns = <82>;
+		gpmc,wr-cycle-ns = <82>;
+		gpmc,wr-access-ns = <40>;
+		gpmc,wr-data-mux-bus-ns = <0>;
+		gpmc,device-width = <2>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+
+	s35390a: s35390a@30 {
+		compatible = "sii,s35390a";
+		reg = <0x30>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&rtc_pins>;
+		interrupts-extended = <&gpio2 23 IRQ_TYPE_EDGE_FALLING>; /* gpio_55 */
+	};
+
+	tps: tps65023@48 {
+		compatible = "ti,tps65023";
+		reg = <0x48>;
+
+		regulators {
+			vdd_core_reg: VDCDC1 {
+				regulator-name = "vdd_core";
+				compatible = "regulator-fixed";
+				regulator-always-on;
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+			};
+
+			vdd_io_reg: VDCDC2 {
+				regulator-name = "vdd_io";
+				compatible = "regulator-fixed";
+				regulator-always-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			vdd_1v8_reg: VDCDC3 {
+				regulator-name = "vdd_1v8";
+				compatible = "regulator-fixed";
+				regulator-always-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			vdd_usb18_reg: LDO1 {
+				regulator-name = "vdd_usb18";
+				compatible = "regulator-fixed";
+				regulator-always-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			vdd_usb33_reg: LDO2 {
+				regulator-name = "vdd_usb33";
+				compatible = "regulator-fixed";
+				regulator-always-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+			};
+		};
+	};
+
+	touchscreen: tsc2004@4b {
+		compatible = "ti,tsc2004";
+		reg = <0x4b>;
+
+		vio-supply = <&vdd_io_reg>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&tsc2004_pins>;
+		interrupts-extended = <&gpio3 1 IRQ_TYPE_EDGE_RISING>; /* gpio_65 */
+
+		touchscreen-fuzz-x = <4>;
+		touchscreen-fuzz-y = <7>;
+		touchscreen-fuzz-pressure = <2>;
+		touchscreen-size-x = <480>;
+		touchscreen-size-y = <272>;
+		touchscreen-max-pressure = <2048>;
+
+		ti,x-plate-ohms = <280>;
+		ti,esd-recovery-timeout-ms = <8000>;
+	};
+};
+
+&omap3_pmx_core {
+
+	rtc_pins: pinmux_rtc_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x20b6, PIN_INPUT_PULLUP | MUX_MODE4) /* gpmc_ncs4.gpio_55 */
+		>;
+	};
+
+	tsc2004_pins: pinmux_tsc2004_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x20d2, PIN_INPUT | MUX_MODE4) /* gpmc_wait3.gpio_65 */
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/am437x-cm-t43.dts b/arch/arm/boot/dts/am437x-cm-t43.dts
index 3b9a94c274a7..bff5abe69bdb 100644
--- a/arch/arm/boot/dts/am437x-cm-t43.dts
+++ b/arch/arm/boot/dts/am437x-cm-t43.dts
@@ -203,7 +203,7 @@
 	tps65218: tps65218@24 {
 		compatible = "ti,tps65218";
 		reg = <0x24>;
-		interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>; /* NMIn */
+		interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* NMIn */
 		interrupt-parent = <&gic>;
 		interrupt-controller;
 		#interrupt-cells = <2>;
diff --git a/arch/arm/boot/dts/am437x-gp-evm.dts b/arch/arm/boot/dts/am437x-gp-evm.dts
index 8fe95cd7232a..60414b1ca404 100644
--- a/arch/arm/boot/dts/am437x-gp-evm.dts
+++ b/arch/arm/boot/dts/am437x-gp-evm.dts
@@ -543,7 +543,7 @@
 	tps65218: tps65218@24 {
 		reg = <0x24>;
 		compatible = "ti,tps65218";
-		interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>; /* NMIn */
+		interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* NMIn */
 		interrupt-controller;
 		#interrupt-cells = <2>;
 
diff --git a/arch/arm/boot/dts/am437x-sk-evm.dts b/arch/arm/boot/dts/am437x-sk-evm.dts
index 4118802b7fea..440351ad0b80 100644
--- a/arch/arm/boot/dts/am437x-sk-evm.dts
+++ b/arch/arm/boot/dts/am437x-sk-evm.dts
@@ -15,6 +15,7 @@
 #include <dt-bindings/pwm/pwm.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/irq.h>
 
 / {
 	model = "TI AM437x SK EVM";
@@ -158,6 +159,22 @@
 			};
 		};
 	};
+
+	vmmcwl_fixed: fixedregulator-mmcwl {
+		/*
+		 * WL_EN is not SDIO standard compliant. It is an out of band
+		 * signal and hard to be dealt with in a standard way by the
+		 * SDIO core driver.
+		 * So modelling the WL_EN line as a regulator was a natural
+		 * choice as the MMC core already deals with MMC supplies.
+		 */
+		compatible = "regulator-fixed";
+		regulator-name = "vmmcwl_fixed";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		gpio = <&gpio4 8 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
 };
 
 &am43xx_pinmux {
@@ -418,6 +435,62 @@
 			AM4372_IOPAD(0xac4, PIN_OUTPUT | MUX_MODE0) /* usb0_drvvbus.usb0_drvvbus */
 		>;
 	};
+
+	mmc3_pins_default: pinmux_mmc3_pins_default {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x9f0, PIN_INPUT_PULLUP | MUX_MODE3) /* (AD21) cam1_data2.mmc2_clk */
+			AM4372_IOPAD(0x9f4, PIN_INPUT_PULLUP | MUX_MODE3) /* (AE22) cam1_data3.mmc2_cmd */
+			AM4372_IOPAD(0x9f8, PIN_INPUT_PULLUP | MUX_MODE3) /* (AD22) cam1_data4.mmc2_dat0 */
+			AM4372_IOPAD(0x9fc, PIN_INPUT_PULLUP | MUX_MODE3) /* (AE23) cam1_data5.mmc2_dat1 */
+			AM4372_IOPAD(0xa00, PIN_INPUT_PULLUP | MUX_MODE3) /* (AD23) cam1_data6.mmc2_dat2 */
+			AM4372_IOPAD(0xa04, PIN_INPUT_PULLUP | MUX_MODE3) /* (AE24) cam1_data7.mmc2_dat3 */
+		>;
+	};
+
+	mmc3_pins_sleep: pinmux_mmc3_pins_sleep {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x9f0, PIN_INPUT_PULLDOWN | MUX_MODE7) /* (AD21) cam1_data2.mmc2_clk */
+			AM4372_IOPAD(0x9f4, PIN_INPUT_PULLDOWN | MUX_MODE7) /* (AE22) cam1_data3.mmc2_cmd */
+			AM4372_IOPAD(0x9f8, PIN_INPUT_PULLDOWN | MUX_MODE7) /* (AD22) cam1_data4.mmc2_dat0 */
+			AM4372_IOPAD(0x9fc, PIN_INPUT_PULLDOWN | MUX_MODE7) /* (AE23) cam1_data5.mmc2_dat1 */
+			AM4372_IOPAD(0xa00, PIN_INPUT_PULLDOWN | MUX_MODE7) /* (AD23) cam1_data6.mmc2_dat2 */
+			AM4372_IOPAD(0xa04, PIN_INPUT_PULLDOWN | MUX_MODE7) /* (AE24) cam1_data7.mmc2_dat3 */
+		>;
+	};
+
+	wlan_pins_default: pinmux_wlan_pins_default {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x9d0, PIN_OUTPUT_PULLDOWN | MUX_MODE7)		/* cam1_data8.gpio4_8 WL_EN */
+			AM4372_IOPAD(0x9e4, PIN_INPUT | WAKEUP_ENABLE | MUX_MODE7)	/* cam1_wen.gpio4_13 WL_IRQ */
+		>;
+	};
+
+	wlan_pins_sleep: pinmux_wlan_pins_sleep {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x9d0, PIN_OUTPUT_PULLDOWN | MUX_MODE7)		/* cam1_data8.gpio4_8 WL_EN */
+			AM4372_IOPAD(0x9e4, PIN_INPUT | WAKEUP_ENABLE | MUX_MODE7)	/* cam1_wen.gpio4_13 WL_IRQ */
+		>;
+	};
+
+	uart1_bt_pins_default: pinmux_uart1_bt_pins_default {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x980, PIN_INPUT | MUX_MODE0)		/* uart1_rxd.uart1_rxd */
+			AM4372_IOPAD(0x984, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* uart1_txd.uart1_txd */
+			AM4372_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE0)	/* uart1_ctsn.uart1_ctsn */
+			AM4372_IOPAD(0x97c, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* uart1_rtsn.uart1_rtsn */
+			AM4372_IOPAD(0x9cc, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* cam1_data9.gpio4_7 BT_EN */
+		>;
+	};
+
+	uart1_bt_pins_sleep: pinmux_uart1_bt_pins_sleep {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x980, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* uart1_rxd.uart1_rxd */
+			AM4372_IOPAD(0x984, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* uart1_txd.uart1_txd */
+			AM4372_IOPAD(0x978, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* uart1_ctsn.uart1_ctsn */
+			AM4372_IOPAD(0x97c, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* uart1_rtsn.uart1_rtsn */
+			AM4372_IOPAD(0x9cc, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* cam1_data9.gpio4_7 BT_EN */
+		>;
+	};
 };
 
 &i2c0 {
@@ -581,6 +654,10 @@
 	status = "okay";
 };
 
+&gpio4 {
+	status = "okay";
+};
+
 &gpio5 {
 	status = "okay";
 };
@@ -595,6 +672,44 @@
 	cd-gpios = <&gpio0 6 GPIO_ACTIVE_LOW>;
 };
 
+&uart1 {
+	status = "okay";
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&uart1_bt_pins_default>;
+	pinctrl-1 = <&uart1_bt_pins_sleep>;
+};
+
+&mmc3 {
+	status = "okay";
+	/*
+	 * these are on the crossbar and are outlined in the
+	 * xbar-event-map element
+	 */
+	dmas = <&edma_xbar 30 0 1>,
+		<&edma_xbar 31 0 2>;
+	dma-names = "tx", "rx";
+	vmmc-supply = <&vmmcwl_fixed>;
+	bus-width = <4>;
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&mmc3_pins_default>;
+	pinctrl-1 = <&mmc3_pins_sleep>;
+	cap-power-off-card;
+	keep-power-in-suspend;
+	ti,non-removable;
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+	wlcore: wlcore@2 {
+		compatible = "ti,wl1835";
+		pinctrl-names = "default", "sleep";
+		pinctrl-0 = <&wlan_pins_default>;
+		pinctrl-1 = <&wlan_pins_sleep>;
+		reg = <2>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <13 IRQ_TYPE_LEVEL_HIGH>;
+	};
+};
+
 &usb2_phy1 {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/am43x-epos-evm.dts b/arch/arm/boot/dts/am43x-epos-evm.dts
index a66941885c11..6502d3397653 100644
--- a/arch/arm/boot/dts/am43x-epos-evm.dts
+++ b/arch/arm/boot/dts/am43x-epos-evm.dts
@@ -595,7 +595,7 @@
 	tps65218: tps65218@24 {
 		reg = <0x24>;
 		compatible = "ti,tps65218";
-		interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>; /* NMIn */
+		interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* NMIn */
 		interrupt-controller;
 		#interrupt-cells = <2>;
 
diff --git a/arch/arm/boot/dts/am571x-idk.dts b/arch/arm/boot/dts/am571x-idk.dts
index a2555140babc..5bb9d68d6e90 100644
--- a/arch/arm/boot/dts/am571x-idk.dts
+++ b/arch/arm/boot/dts/am571x-idk.dts
@@ -10,6 +10,7 @@
 #include "dra72x.dtsi"
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/irq.h>
+#include "dra7-mmc-iodelay.dtsi"
 #include "dra72x-mmc-iodelay.dtsi"
 #include "am57xx-idk-common.dtsi"
 
@@ -102,7 +103,7 @@
 
 &mmc1 {
 	pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50", "ddr50", "sdr104";
-	pinctrl-0 = <&mmc1_pins_default>;
+	pinctrl-0 = <&mmc1_pins_default_no_clk_pu>;
 	pinctrl-1 = <&mmc1_pins_hs>;
 	pinctrl-2 = <&mmc1_pins_sdr12>;
 	pinctrl-3 = <&mmc1_pins_sdr25>;
@@ -112,7 +113,7 @@
 };
 
 &mmc2 {
-	pinctrl-names = "default", "hs", "ddr_1_8v";
+	pinctrl-names = "default", "hs", "ddr_3_3v";
 	pinctrl-0 = <&mmc2_pins_default>;
 	pinctrl-1 = <&mmc2_pins_hs>;
 	pinctrl-2 = <&mmc2_pins_ddr_rev20 &mmc2_iodelay_ddr_conf>;
diff --git a/arch/arm/boot/dts/am572x-idk.dts b/arch/arm/boot/dts/am572x-idk.dts
index 3a02ed720957..3ef9111d0e8b 100644
--- a/arch/arm/boot/dts/am572x-idk.dts
+++ b/arch/arm/boot/dts/am572x-idk.dts
@@ -9,6 +9,7 @@
 /dts-v1/;
 
 #include "dra74x.dtsi"
+#include "dra7-mmc-iodelay.dtsi"
 #include "dra74x-mmc-iodelay.dtsi"
 #include "am572x-idk-common.dtsi"
 
@@ -20,7 +21,7 @@
 
 &mmc1 {
 	pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50", "ddr50", "sdr104";
-	pinctrl-0 = <&mmc1_pins_default>;
+	pinctrl-0 = <&mmc1_pins_default_no_clk_pu>;
 	pinctrl-1 = <&mmc1_pins_hs>;
 	pinctrl-2 = <&mmc1_pins_sdr12>;
 	pinctrl-3 = <&mmc1_pins_sdr25>;
@@ -30,7 +31,7 @@
 };
 
 &mmc2 {
-	pinctrl-names = "default", "hs", "ddr_1_8v";
+	pinctrl-names = "default", "hs", "ddr_3_3v";
 	pinctrl-0 = <&mmc2_pins_default>;
 	pinctrl-1 = <&mmc2_pins_hs>;
 	pinctrl-2 = <&mmc2_pins_ddr_rev20>;
diff --git a/arch/arm/boot/dts/am574x-idk.dts b/arch/arm/boot/dts/am574x-idk.dts
index 41e12a382d2f..378dfa780ac1 100644
--- a/arch/arm/boot/dts/am574x-idk.dts
+++ b/arch/arm/boot/dts/am574x-idk.dts
@@ -7,6 +7,8 @@
 /dts-v1/;
 
 #include "dra76x.dtsi"
+#include "dra7-mmc-iodelay.dtsi"
+#include "dra76x-mmc-iodelay.dtsi"
 #include "am572x-idk-common.dtsi"
 
 / {
@@ -20,3 +22,21 @@
 		spi-max-frequency = <96000000>;
 	};
 };
+
+&mmc1 {
+	pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50", "ddr50", "sdr104";
+	pinctrl-0 = <&mmc1_pins_default_no_clk_pu>;
+	pinctrl-1 = <&mmc1_pins_hs>;
+	pinctrl-2 = <&mmc1_pins_default>;
+	pinctrl-3 = <&mmc1_pins_hs>;
+	pinctrl-4 = <&mmc1_pins_sdr50>;
+	pinctrl-5 = <&mmc1_pins_ddr50 &mmc1_iodelay_ddr_conf>;
+	pinctrl-6 = <&mmc1_pins_ddr50 &mmc1_iodelay_sdr104_conf>;
+};
+
+&mmc2 {
+	pinctrl-names = "default", "hs", "ddr_3_3v";
+	pinctrl-0 = <&mmc2_pins_default>;
+	pinctrl-1 = <&mmc2_pins_default>;
+	pinctrl-2 = <&mmc2_pins_default>;
+};
diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi
index 6204a266212a..ad953113cefb 100644
--- a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi
+++ b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi
@@ -444,8 +444,8 @@
 	vmmc-supply = <&vdd_3v3>;
 	vqmmc-supply = <&vdd_3v3>;
 	bus-width = <8>;
-	ti,non-removable;
-	cap-mmc-dual-data-rate;
+	non-removable;
+	no-1-8-v;
 };
 
 &sata {
diff --git a/arch/arm/boot/dts/am57xx-beagle-x15.dts b/arch/arm/boot/dts/am57xx-beagle-x15.dts
index d6689106d2a8..70a71c641066 100644
--- a/arch/arm/boot/dts/am57xx-beagle-x15.dts
+++ b/arch/arm/boot/dts/am57xx-beagle-x15.dts
@@ -25,10 +25,11 @@
 	pinctrl-1 = <&mmc1_pins_hs>;
 
 	vmmc-supply = <&ldo1_reg>;
+	no-1-8-v;
 };
 
 &mmc2 {
-	pinctrl-names = "default", "hs", "ddr_1_8v";
+	pinctrl-names = "default", "hs", "ddr_3_3v";
 	pinctrl-0 = <&mmc2_pins_default>;
 	pinctrl-1 = <&mmc2_pins_hs>;
 	pinctrl-2 = <&mmc2_pins_ddr_3_3v_rev11 &mmc2_iodelay_ddr_3_3v_rev11_conf>;
diff --git a/arch/arm/boot/dts/am57xx-idk-common.dtsi b/arch/arm/boot/dts/am57xx-idk-common.dtsi
index 43cdf523a8a0..ad87f1ae904d 100644
--- a/arch/arm/boot/dts/am57xx-idk-common.dtsi
+++ b/arch/arm/boot/dts/am57xx-idk-common.dtsi
@@ -115,17 +115,6 @@
 			DRA7XX_CORE_IOPAD(0x37d4, MUX_MODE15 | PULL_UP)	/* dcan1_rx.off */
 		>;
 	};
-
-	mmc1_pins_default: mmc1_pins_default {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x3754, PIN_INPUT_PULLDOWN | MUX_MODE0) /* mmc1_clk.clk */
-			DRA7XX_CORE_IOPAD(0x3758, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_cmd.cmd */
-			DRA7XX_CORE_IOPAD(0x375c, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat0.dat0 */
-			DRA7XX_CORE_IOPAD(0x3760, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat1.dat1 */
-			DRA7XX_CORE_IOPAD(0x3764, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat2.dat2 */
-			DRA7XX_CORE_IOPAD(0x3768, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat3.dat3 */
-		>;
-	};
 };
 
 &i2c1 {
@@ -423,8 +412,9 @@
 	vmmc-supply = <&v3_3d>;
 	vqmmc-supply = <&v3_3d>;
 	bus-width = <8>;
-	ti,non-removable;
+	non-removable;
 	max-frequency = <96000000>;
+	no-1-8-v;
 };
 
 &dcan1 {
diff --git a/arch/arm/boot/dts/armada-370-db.dts b/arch/arm/boot/dts/armada-370-db.dts
index afe46097a403..77261a2fb949 100644
--- a/arch/arm/boot/dts/armada-370-db.dts
+++ b/arch/arm/boot/dts/armada-370-db.dts
@@ -105,33 +105,6 @@
 			usb@51000 {
 				status = "okay";
 			};
-
-			nand@d0000 {
-				status = "okay";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-
-				partitions {
-					compatible = "fixed-partitions";
-					#address-cells = <1>;
-					#size-cells = <1>;
-
-					partition@0 {
-						label = "U-Boot";
-						reg = <0 0x800000>;
-					};
-					partition@800000 {
-						label = "Linux";
-						reg = <0x800000 0x800000>;
-					};
-					partition@1000000 {
-						label = "Filesystem";
-						reg = <0x1000000 0x3f000000>;
-					};
-				};
-			};
 		};
 	};
 
@@ -239,3 +212,33 @@
 	};
 };
 
+&nand_controller {
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "U-Boot";
+				reg = <0 0x800000>;
+			};
+			partition@800000 {
+				label = "Linux";
+				reg = <0x800000 0x800000>;
+			};
+			partition@1000000 {
+				label = "Filesystem";
+				reg = <0x1000000 0x3f000000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-370-dlink-dns327l.dts b/arch/arm/boot/dts/armada-370-dlink-dns327l.dts
index 8e46f63cbaa1..baa459dd51e4 100644
--- a/arch/arm/boot/dts/armada-370-dlink-dns327l.dts
+++ b/arch/arm/boot/dts/armada-370-dlink-dns327l.dts
@@ -44,61 +44,6 @@
 			usb@50000 {
 				status = "okay";
 			};
-
-			nand@d0000 {
-				status = "okay";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-				nand-ecc-strength = <4>;
-				nand-ecc-step-size = <512>;
-
-				partition@0 {
-					label = "u-boot";
-					/* 1.0 MiB */
-					reg = <0x0000000 0x100000>;
-					read-only;
-				};
-
-				partition@100000 {
-					label = "u-boot-env";
-					/* 128 KiB */
-					reg = <0x100000 0x20000>;
-					read-only;
-				};
-
-				partition@120000 {
-					label = "uImage";
-					/* 7 MiB */
-					reg = <0x120000 0x700000>;
-				};
-
-				partition@820000 {
-					label = "ubifs";
-					/* ~ 84 MiB */
-					reg = <0x820000 0x54e0000>;
-				};
-
-				/* Hardcoded into stock bootloader */
-				partition@5d00000 {
-					label = "failsafe-uImage";
-					/* 5 MiB */
-					reg = <0x5d00000 0x500000>;
-				};
-
-				partition@6200000 {
-					label = "failsafe-fs";
-					/* 29 MiB */
-					reg = <0x6200000 0x1d00000>;
-				};
-
-				partition@7f00000 {
-					label = "bbt";
-					/* 1 MiB for BBT */
-					reg = <0x7f00000 0x100000>;
-				};
-			};
 		};
 	};
 
@@ -319,3 +264,68 @@
 	clock-frequency = <100000>;
 	status = "okay";
 };
+
+&nand_controller {
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+		nand-ecc-strength = <4>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "u-boot";
+				/* 1.0 MiB */
+				reg = <0x0000000 0x100000>;
+				read-only;
+			};
+
+			partition@100000 {
+				label = "u-boot-env";
+				/* 128 KiB */
+				reg = <0x100000 0x20000>;
+				read-only;
+			};
+
+			partition@120000 {
+				label = "uImage";
+				/* 7 MiB */
+				reg = <0x120000 0x700000>;
+			};
+
+			partition@820000 {
+				label = "ubifs";
+				/* ~ 84 MiB */
+				reg = <0x820000 0x54e0000>;
+			};
+
+			/* Hardcoded into stock bootloader */
+			partition@5d00000 {
+				label = "failsafe-uImage";
+				/* 5 MiB */
+				reg = <0x5d00000 0x500000>;
+			};
+
+			partition@6200000 {
+				label = "failsafe-fs";
+				/* 29 MiB */
+				reg = <0x6200000 0x1d00000>;
+			};
+
+			partition@7f00000 {
+				label = "bbt";
+				/* 1 MiB for BBT */
+				reg = <0x7f00000 0x100000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-370-mirabox.dts b/arch/arm/boot/dts/armada-370-mirabox.dts
index 996f31b00729..7c2f5a79b50d 100644
--- a/arch/arm/boot/dts/armada-370-mirabox.dts
+++ b/arch/arm/boot/dts/armada-370-mirabox.dts
@@ -108,27 +108,6 @@
 					reg = <0x25>;
 				};
 			};
-
-			nand@d0000 {
-				status = "okay";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-
-				partition@0 {
-					label = "U-Boot";
-					reg = <0 0x400000>;
-				};
-				partition@400000 {
-					label = "Linux";
-					reg = <0x400000 0x400000>;
-				};
-				partition@800000 {
-					label = "Filesystem";
-					reg = <0x800000 0x3f800000>;
-				};
-			};
 		};
 	};
 };
@@ -173,3 +152,33 @@
 	};
 };
 
+&nand_controller {
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "U-Boot";
+				reg = <0 0x400000>;
+			};
+			partition@400000 {
+				label = "Linux";
+				reg = <0x400000 0x400000>;
+			};
+			partition@800000 {
+				label = "Filesystem";
+				reg = <0x800000 0x3f800000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-370-netgear-rn102.dts b/arch/arm/boot/dts/armada-370-netgear-rn102.dts
index 56634803e16b..b0b640b7de40 100644
--- a/arch/arm/boot/dts/armada-370-netgear-rn102.dts
+++ b/arch/arm/boot/dts/armada-370-netgear-rn102.dts
@@ -81,46 +81,6 @@
 					pwm_polarity = <0>;
 				};
 			};
-
-			nand@d0000 {
-				status = "okay";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-
-				/* Use Hardware BCH ECC */
-				nand-ecc-strength = <4>;
-				nand-ecc-step-size = <512>;
-
-				partition@0 {
-					label = "u-boot";
-					reg = <0x0000000 0x180000>;  /* 1.5MB */
-					read-only;
-				};
-
-				partition@180000 {
-					label = "u-boot-env";
-					reg = <0x180000 0x20000>;    /* 128KB */
-					read-only;
-				};
-
-				partition@200000 {
-					label = "uImage";
-					reg = <0x0200000 0x600000>;    /* 6MB */
-				};
-
-				partition@800000 {
-					label = "minirootfs";
-					reg = <0x0800000 0x400000>;    /* 4MB */
-				};
-
-				/* Last MB is for the BBT, i.e. not writable */
-				partition@c00000 {
-					label = "ubifs";
-					reg = <0x0c00000 0x7400000>; /* 116MB */
-				};
-			};
 		};
 	};
 
@@ -264,3 +224,53 @@
 		marvell,function = "gpio";
 	};
 };
+
+&nand_controller {
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+
+		/* Use Hardware BCH ECC */
+		nand-ecc-strength = <4>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "u-boot";
+				reg = <0x0000000 0x180000>;  /* 1.5MB */
+				read-only;
+			};
+
+			partition@180000 {
+				label = "u-boot-env";
+				reg = <0x180000 0x20000>;    /* 128KB */
+				read-only;
+			};
+
+			partition@200000 {
+				label = "uImage";
+				reg = <0x0200000 0x600000>;    /* 6MB */
+			};
+
+			partition@800000 {
+				label = "minirootfs";
+				reg = <0x0800000 0x400000>;    /* 4MB */
+			};
+
+			/* Last MB is for the BBT, i.e. not writable */
+			partition@c00000 {
+				label = "ubifs";
+				reg = <0x0c00000 0x7400000>; /* 116MB */
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-370-netgear-rn104.dts b/arch/arm/boot/dts/armada-370-netgear-rn104.dts
index 16d0307f786a..9fd1cb9f4992 100644
--- a/arch/arm/boot/dts/armada-370-netgear-rn104.dts
+++ b/arch/arm/boot/dts/armada-370-netgear-rn104.dts
@@ -90,46 +90,6 @@
 					reg = <0x23>;
 				};
 			};
-
-			nand@d0000 {
-				status = "okay";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-
-				/* Use Hardware BCH ECC */
-				nand-ecc-strength = <4>;
-				nand-ecc-step-size = <512>;
-
-				partition@0 {
-					label = "u-boot";
-					reg = <0x0000000 0x180000>;  /* 1.5MB */
-					read-only;
-				};
-
-				partition@180000 {
-					label = "u-boot-env";
-					reg = <0x180000 0x20000>;    /* 128KB */
-					read-only;
-				};
-
-				partition@200000 {
-					label = "uImage";
-					reg = <0x0200000 0x600000>;    /* 6MB */
-				};
-
-				partition@800000 {
-					label = "minirootfs";
-					reg = <0x0800000 0x400000>;    /* 4MB */
-				};
-
-				/* Last MB is for the BBT, i.e. not writable */
-				partition@c00000 {
-					label = "ubifs";
-					reg = <0x0c00000 0x7400000>; /* 116MB */
-				};
-			};
 		};
 	};
 
@@ -276,3 +236,53 @@
 		marvell,function = "gpio";
 	};
 };
+
+&nand_controller {
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+
+		/* Use Hardware BCH ECC */
+		nand-ecc-strength = <4>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "u-boot";
+				reg = <0x0000000 0x180000>;  /* 1.5MB */
+				read-only;
+			};
+
+			partition@180000 {
+				label = "u-boot-env";
+				reg = <0x180000 0x20000>;    /* 128KB */
+				read-only;
+			};
+
+			partition@200000 {
+				label = "uImage";
+				reg = <0x0200000 0x600000>;    /* 6MB */
+			};
+
+			partition@800000 {
+				label = "minirootfs";
+				reg = <0x0800000 0x400000>;    /* 4MB */
+			};
+
+			/* Last MB is for the BBT, i.e. not writable */
+			partition@c00000 {
+				label = "ubifs";
+				reg = <0x0c00000 0x7400000>; /* 116MB */
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-370-rd.dts b/arch/arm/boot/dts/armada-370-rd.dts
index cc2f774eb267..2bfb3108b5b2 100644
--- a/arch/arm/boot/dts/armada-370-rd.dts
+++ b/arch/arm/boot/dts/armada-370-rd.dts
@@ -112,27 +112,6 @@
 					default-state = "keep";
 				};
 			};
-
-			nand@d0000 {
-				status = "okay";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-
-				partition@0 {
-					label = "U-Boot";
-					reg = <0 0x800000>;
-				};
-				partition@800000 {
-					label = "Linux";
-					reg = <0x800000 0x800000>;
-				};
-				partition@1000000 {
-					label = "Filesystem";
-					reg = <0x1000000 0x3f000000>;
-				};
-			};
 		};
 	};
 
@@ -288,3 +267,34 @@
 		marvell,function = "gpio";
 	};
 };
+
+&nand_controller {
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "U-Boot";
+				reg = <0 0x800000>;
+			};
+			partition@800000 {
+				label = "Linux";
+				reg = <0x800000 0x800000>;
+			};
+			partition@1000000 {
+				label = "Filesystem";
+				reg = <0x1000000 0x3f000000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-370-seagate-nas-xbay.dtsi b/arch/arm/boot/dts/armada-370-seagate-nas-xbay.dtsi
index a5206db0ebbd..b52634ecf1d9 100644
--- a/arch/arm/boot/dts/armada-370-seagate-nas-xbay.dtsi
+++ b/arch/arm/boot/dts/armada-370-seagate-nas-xbay.dtsi
@@ -66,33 +66,6 @@
 					interrupts = <110>;
 				};
 			};
-
-			nand@d0000 {
-				status = "okay";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-				nand-ecc-strength = <4>;
-				nand-ecc-step-size = <512>;
-
-				partition@0 {
-					label = "u-boot";
-					reg = <0x0 0x300000>;
-				};
-				partition@300000 {
-					label = "device-tree";
-					reg = <0x300000 0x20000>;
-				};
-				partition@320000 {
-					label = "linux";
-					reg = <0x320000 0x2000000>;
-				};
-				partition@2320000 {
-					label = "rootfs";
-					reg = <0x2320000 0xdce0000>;
-				};
-			};
 		};
 
 	};
@@ -227,3 +200,40 @@
 		marvell,function = "gpio";
 	};
 };
+
+&nand_controller {
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+		nand-ecc-strength = <4>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "u-boot";
+				reg = <0x0 0x300000>;
+			};
+			partition@300000 {
+				label = "device-tree";
+				reg = <0x300000 0x20000>;
+			};
+			partition@320000 {
+				label = "linux";
+				reg = <0x320000 0x2000000>;
+			};
+			partition@2320000 {
+				label = "rootfs";
+				reg = <0x2320000 0xdce0000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-370-xp.dtsi b/arch/arm/boot/dts/armada-370-xp.dtsi
index 11fc3271dad4..c15f5e92f97f 100644
--- a/arch/arm/boot/dts/armada-370-xp.dtsi
+++ b/arch/arm/boot/dts/armada-370-xp.dtsi
@@ -244,11 +244,11 @@
 				status = "disabled";
 			};
 
-			nand: nand@d0000 {
-				compatible = "marvell,armada370-nand";
+			nand_controller: nand-controller@d0000 {
+				compatible = "marvell,armada370-nand-controller";
 				reg = <0xd0000 0x54>;
 				#address-cells = <1>;
-				#size-cells = <1>;
+				#size-cells = <0>;
 				interrupts = <113>;
 				clocks = <&coredivclk 0>;
 				status = "disabled";
diff --git a/arch/arm/boot/dts/armada-375-db.dts b/arch/arm/boot/dts/armada-375-db.dts
index e4ecd7e75644..0e679465cbb5 100644
--- a/arch/arm/boot/dts/armada-375-db.dts
+++ b/arch/arm/boot/dts/armada-375-db.dts
@@ -103,28 +103,38 @@
 	nr-ports = <2>;
 };
 
-&nand {
+&nand_controller {
+	status = "okay";
 	pinctrl-0 = <&nand_pins>;
 	pinctrl-names = "default";
-	status = "okay";
-	num-cs = <1>;
-	marvell,nand-keep-config;
-	marvell,nand-enable-arbiter;
-	nand-on-flash-bbt;
-	nand-ecc-strength = <4>;
-	nand-ecc-step-size = <512>;
-
-	partition@0 {
-		label = "U-Boot";
-		reg = <0 0x800000>;
-	};
-	partition@800000 {
-		label = "Linux";
-		reg = <0x800000 0x800000>;
-	};
-	partition@1000000 {
-		label = "Filesystem";
-		reg = <0x1000000 0x3f000000>;
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+		nand-ecc-strength = <4>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "U-Boot";
+				reg = <0 0x800000>;
+			};
+			partition@800000 {
+				label = "Linux";
+				reg = <0x800000 0x800000>;
+			};
+			partition@1000000 {
+				label = "Filesystem";
+				reg = <0x1000000 0x3f000000>;
+			};
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/armada-375.dtsi b/arch/arm/boot/dts/armada-375.dtsi
index 53ead6f26a0e..2932a29ae272 100644
--- a/arch/arm/boot/dts/armada-375.dtsi
+++ b/arch/arm/boot/dts/armada-375.dtsi
@@ -502,11 +502,11 @@
 				status = "disabled";
 			};
 
-			nand: nand@d0000 {
-				compatible = "marvell,armada370-nand";
+			nand_controller: nand-controller@d0000 {
+				compatible = "marvell,armada370-nand-controller";
 				reg = <0xd0000 0x54>;
 				#address-cells = <1>;
-				#size-cells = <1>;
+				#size-cells = <0>;
 				interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gateclk 11>;
 				status = "disabled";
diff --git a/arch/arm/boot/dts/armada-385-db-ap.dts b/arch/arm/boot/dts/armada-385-db-ap.dts
index d294f24281a5..0e4613bb56ee 100644
--- a/arch/arm/boot/dts/armada-385-db-ap.dts
+++ b/arch/arm/boot/dts/armada-385-db-ap.dts
@@ -135,39 +135,6 @@
 				status = "okay";
 			};
 
-			nfc: flash@d0000 {
-				status = "okay";
-				num-cs = <1>;
-				nand-ecc-strength = <4>;
-				nand-ecc-step-size = <512>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-
-				partitions {
-					compatible = "fixed-partitions";
-					#address-cells = <1>;
-					#size-cells = <1>;
-
-					partition@0 {
-						label = "U-Boot";
-						reg = <0x00000000 0x00800000>;
-						read-only;
-					};
-
-					partition@800000 {
-						label = "uImage";
-						reg = <0x00800000 0x00400000>;
-						read-only;
-					};
-
-					partition@c00000 {
-						label = "Root";
-						reg = <0x00c00000 0x3f400000>;
-					};
-				};
-			};
-
 			usb3@f0000 {
 				status = "okay";
 				usb-phy = <&usb3_phy>;
@@ -233,3 +200,39 @@
 		spi-max-frequency = <54000000>;
 	};
 };
+
+&nand_controller {
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		nand-on-flash-bbt;
+		nand-ecc-strength = <4>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "U-Boot";
+				reg = <0x00000000 0x00800000>;
+				read-only;
+			};
+
+			partition@800000 {
+				label = "uImage";
+				reg = <0x00800000 0x00400000>;
+				read-only;
+			};
+
+			partition@c00000 {
+				label = "Root";
+				reg = <0x00c00000 0x3f400000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-385-linksys-caiman.dts b/arch/arm/boot/dts/armada-385-linksys-caiman.dts
index 1f30993af405..a03050c97084 100644
--- a/arch/arm/boot/dts/armada-385-linksys-caiman.dts
+++ b/arch/arm/boot/dts/armada-385-linksys-caiman.dts
@@ -73,67 +73,72 @@
 
 &nand {
 	/* 128MiB */
-
-	partition@0 {
-		label = "u-boot";
-		reg = <0x0000000 0x200000>;  /* 2MiB */
-		read-only;
-	};
-
-	partition@100000 {
-		label = "u_env";
-		reg = <0x200000 0x40000>;    /* 256KiB */
-	};
-
-	partition@140000 {
-		label = "s_env";
-		reg = <0x240000 0x40000>;    /* 256KiB */
-	};
-
-	partition@900000 {
-		label = "devinfo";
-		reg = <0x900000 0x100000>;   /* 1MiB */
-		read-only;
-	};
-
-	/* kernel1 overlaps with rootfs1 by design */
-	partition@a00000 {
-		label = "kernel1";
-		reg = <0xa00000 0x2800000>;  /* 40MiB */
-	};
-
-	partition@1000000 {
-		label = "rootfs1";
-		reg = <0x1000000 0x2200000>;  /* 34MiB */
-	};
-
-	/* kernel2 overlaps with rootfs2 by design */
-	partition@3200000 {
-		label = "kernel2";
-		reg = <0x3200000 0x2800000>; /* 40MiB */
-	};
-
-	partition@3800000 {
-		label = "rootfs2";
-		reg = <0x3800000 0x2200000>; /* 34MiB */
-	};
-
-	/*
-	 * 38MiB, last MiB is for the BBT, not writable
-	 */
-	partition@5a00000 {
-		label = "syscfg";
-		reg = <0x5a00000 0x2600000>;
-	};
-
-	/*
-	 * Unused area between "s_env" and "devinfo".
-	 * Moved here because otherwise the renumbered
-	 * partitions would break the bootloader
-	 * supplied bootargs
-	 */
-	partition@180000 {
-		label = "unused_area";
-		reg = <0x280000 0x680000>;   /* 6.5MiB */
+	partitions {
+		compatible = "fixed-partitions";
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		partition@0 {
+			label = "u-boot";
+			reg = <0x0000000 0x200000>;  /* 2MiB */
+			read-only;
+		};
+
+		partition@100000 {
+			label = "u_env";
+			reg = <0x200000 0x40000>;    /* 256KiB */
+		};
+
+		partition@140000 {
+			label = "s_env";
+			reg = <0x240000 0x40000>;    /* 256KiB */
+		};
+
+		partition@900000 {
+			label = "devinfo";
+			reg = <0x900000 0x100000>;   /* 1MiB */
+			read-only;
+		};
+
+		/* kernel1 overlaps with rootfs1 by design */
+		partition@a00000 {
+			label = "kernel1";
+			reg = <0xa00000 0x2800000>;  /* 40MiB */
+		};
+
+		partition@1000000 {
+			label = "rootfs1";
+			reg = <0x1000000 0x2200000>;  /* 34MiB */
+		};
+
+		/* kernel2 overlaps with rootfs2 by design */
+		partition@3200000 {
+			label = "kernel2";
+			reg = <0x3200000 0x2800000>; /* 40MiB */
+		};
+
+		partition@3800000 {
+			label = "rootfs2";
+			reg = <0x3800000 0x2200000>; /* 34MiB */
+		};
+
+		/*
+		 * 38MiB, last MiB is for the BBT, not writable
+		 */
+		partition@5a00000 {
+			label = "syscfg";
+			reg = <0x5a00000 0x2600000>;
+		};
+
+		/*
+		 * Unused area between "s_env" and "devinfo".
+		 * Moved here because otherwise the renumbered
+		 * partitions would break the bootloader
+		 * supplied bootargs
+		 */
+		partition@180000 {
+			label = "unused_area";
+			reg = <0x280000 0x680000>;   /* 6.5MiB */
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/armada-385-linksys-cobra.dts b/arch/arm/boot/dts/armada-385-linksys-cobra.dts
index bc34802ce6bc..e3e4877a6f49 100644
--- a/arch/arm/boot/dts/armada-385-linksys-cobra.dts
+++ b/arch/arm/boot/dts/armada-385-linksys-cobra.dts
@@ -73,67 +73,72 @@
 
 &nand {
 	/* 128MiB */
-
-	partition@0 {
-		label = "u-boot";
-		reg = <0x0000000 0x200000>;  /* 2MiB */
-		read-only;
-	};
-
-	partition@100000 {
-		label = "u_env";
-		reg = <0x200000 0x40000>;    /* 256KiB */
-	};
-
-	partition@140000 {
-		label = "s_env";
-		reg = <0x240000 0x40000>;    /* 256KiB */
-	};
-
-	partition@900000 {
-		label = "devinfo";
-		reg = <0x900000 0x100000>;   /* 1MiB */
-		read-only;
-	};
-
-	/* kernel1 overlaps with rootfs1 by design */
-	partition@a00000 {
-		label = "kernel1";
-		reg = <0xa00000 0x2800000>;  /* 40MiB */
-	};
-
-	partition@1000000 {
-		label = "rootfs1";
-		reg = <0x1000000 0x2200000>;  /* 34MiB */
-	};
-
-	/* kernel2 overlaps with rootfs2 by design */
-	partition@3200000 {
-		label = "kernel2";
-		reg = <0x3200000 0x2800000>; /* 40MiB */
-	};
-
-	partition@3800000 {
-		label = "rootfs2";
-		reg = <0x3800000 0x2200000>; /* 34MiB */
-	};
-
-	/*
-	 * 38MiB, last MiB is for the BBT, not writable
-	 */
-	partition@5a00000 {
-		label = "syscfg";
-		reg = <0x5a00000 0x2600000>;
-	};
-
-	/*
-	 * Unused area between "s_env" and "devinfo".
-	 * Moved here because otherwise the renumbered
-	 * partitions would break the bootloader
-	 * supplied bootargs
-	 */
-	partition@180000 {
-		label = "unused_area";
-		reg = <0x280000 0x680000>;   /* 6.5MiB */
+	partitions {
+		compatible = "fixed-partitions";
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		partition@0 {
+			label = "u-boot";
+			reg = <0x0000000 0x200000>;  /* 2MiB */
+			read-only;
+		};
+
+		partition@100000 {
+			label = "u_env";
+			reg = <0x200000 0x40000>;    /* 256KiB */
+		};
+
+		partition@140000 {
+			label = "s_env";
+			reg = <0x240000 0x40000>;    /* 256KiB */
+		};
+
+		partition@900000 {
+			label = "devinfo";
+			reg = <0x900000 0x100000>;   /* 1MiB */
+			read-only;
+		};
+
+		/* kernel1 overlaps with rootfs1 by design */
+		partition@a00000 {
+			label = "kernel1";
+			reg = <0xa00000 0x2800000>;  /* 40MiB */
+		};
+
+		partition@1000000 {
+			label = "rootfs1";
+			reg = <0x1000000 0x2200000>;  /* 34MiB */
+		};
+
+		/* kernel2 overlaps with rootfs2 by design */
+		partition@3200000 {
+			label = "kernel2";
+			reg = <0x3200000 0x2800000>; /* 40MiB */
+		};
+
+		partition@3800000 {
+			label = "rootfs2";
+			reg = <0x3800000 0x2200000>; /* 34MiB */
+		};
+
+		/*
+		 * 38MiB, last MiB is for the BBT, not writable
+		 */
+		partition@5a00000 {
+			label = "syscfg";
+			reg = <0x5a00000 0x2600000>;
+		};
+
+		/*
+		 * Unused area between "s_env" and "devinfo".
+		 * Moved here because otherwise the renumbered
+		 * partitions would break the bootloader
+		 * supplied bootargs
+		 */
+		partition@180000 {
+			label = "unused_area";
+			reg = <0x280000 0x680000>;   /* 6.5MiB */
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/armada-385-linksys-rango.dts b/arch/arm/boot/dts/armada-385-linksys-rango.dts
index 5b745a0ccce5..3c4af57ec2b9 100644
--- a/arch/arm/boot/dts/armada-385-linksys-rango.dts
+++ b/arch/arm/boot/dts/armada-385-linksys-rango.dts
@@ -81,74 +81,79 @@
 
 &nand {
 	/* AMD/Spansion S34ML02G2 256MiB, OEM Layout */
-
-	partition@0 {
-		label = "u-boot";
-		reg = <0x0000000 0x200000>;  /* 2MiB */
-		read-only;
-	};
-
-	partition@200000 {
-		label = "u_env";
-		reg = <0x200000 0x20000>;    /* 128KiB */
-	};
-
-	partition@220000 {
-		label = "s_env";
-		reg = <0x220000 0x40000>;    /* 256KiB */
-	};
-
-	partition@7e0000 {
-		label = "devinfo";
-		reg = <0x7e0000 0x40000>;   /* 256KiB */
-		read-only;
-	};
-
-	partition@820000 {
-		label = "sysdiag";
-		reg = <0x820000 0x1e0000>;   /* 1920KiB */
-		read-only;
-	};
-
-	/* kernel1 overlaps with rootfs1 by design */
-	partition@a00000 {
-		label = "kernel1";
-		reg = <0xa00000 0x5000000>;  /* 80MiB */
-	};
-
-	partition@1000000 {
-		label = "rootfs1";
-		reg = <0x1000000 0x4a00000>;  /* 74MiB */
-	};
-
-	/* kernel2 overlaps with rootfs2 by design */
-	partition@5a00000 {
-		label = "kernel2";
-		reg = <0x5a00000 0x5000000>; /* 80MiB */
-	};
-
-	partition@6000000 {
-		label = "rootfs2";
-		reg = <0x6000000 0x4a00000>; /* 74MiB */
-	};
-
-	/*
-	 * 86MiB, last MiB is for the BBT, not writable
-	 */
-	partition@aa00000 {
-		label = "syscfg";
-		reg = <0xaa00000 0x5600000>;
-	};
-
-	/*
-	 * Unused area between "s_env" and "devinfo".
-	 * Moved here because otherwise the renumbered
-	 * partitions would break the bootloader
-	 * supplied bootargs
-	 */
-	partition@180000 {
-		label = "unused_area";
-		reg = <0x260000 0x5c0000>;   /* 5.75MiB */
+	partitions {
+		compatible = "fixed-partitions";
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		partition@0 {
+			label = "u-boot";
+			reg = <0x0000000 0x200000>;  /* 2MiB */
+			read-only;
+		};
+
+		partition@200000 {
+			label = "u_env";
+			reg = <0x200000 0x20000>;    /* 128KiB */
+		};
+
+		partition@220000 {
+			label = "s_env";
+			reg = <0x220000 0x40000>;    /* 256KiB */
+		};
+
+		partition@7e0000 {
+			label = "devinfo";
+			reg = <0x7e0000 0x40000>;   /* 256KiB */
+			read-only;
+		};
+
+		partition@820000 {
+			label = "sysdiag";
+			reg = <0x820000 0x1e0000>;   /* 1920KiB */
+			read-only;
+		};
+
+		/* kernel1 overlaps with rootfs1 by design */
+		partition@a00000 {
+			label = "kernel1";
+			reg = <0xa00000 0x5000000>;  /* 80MiB */
+		};
+
+		partition@1000000 {
+			label = "rootfs1";
+			reg = <0x1000000 0x4a00000>;  /* 74MiB */
+		};
+
+		/* kernel2 overlaps with rootfs2 by design */
+		partition@5a00000 {
+			label = "kernel2";
+			reg = <0x5a00000 0x5000000>; /* 80MiB */
+		};
+
+		partition@6000000 {
+			label = "rootfs2";
+			reg = <0x6000000 0x4a00000>; /* 74MiB */
+		};
+
+		/*
+		 * 86MiB, last MiB is for the BBT, not writable
+		 */
+		partition@aa00000 {
+			label = "syscfg";
+			reg = <0xaa00000 0x5600000>;
+		};
+
+		/*
+		 * Unused area between "s_env" and "devinfo".
+		 * Moved here because otherwise the renumbered
+		 * partitions would break the bootloader
+		 * supplied bootargs
+		 */
+		partition@180000 {
+			label = "unused_area";
+			reg = <0x260000 0x5c0000>;   /* 5.75MiB */
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/armada-385-linksys-shelby.dts b/arch/arm/boot/dts/armada-385-linksys-shelby.dts
index 44f5aeb5fc33..3451cd3e5dff 100644
--- a/arch/arm/boot/dts/armada-385-linksys-shelby.dts
+++ b/arch/arm/boot/dts/armada-385-linksys-shelby.dts
@@ -73,67 +73,72 @@
 
 &nand {
 	/* 128MiB */
-
-	partition@0 {
-		label = "u-boot";
-		reg = <0x0000000 0x200000>;  /* 2MiB */
-		read-only;
-	};
-
-	partition@100000 {
-		label = "u_env";
-		reg = <0x200000 0x40000>;    /* 256KiB */
-	};
-
-	partition@140000 {
-		label = "s_env";
-		reg = <0x240000 0x40000>;    /* 256KiB */
-	};
-
-	partition@900000 {
-		label = "devinfo";
-		reg = <0x900000 0x100000>;   /* 1MiB */
-		read-only;
-	};
-
-	/* kernel1 overlaps with rootfs1 by design */
-	partition@a00000 {
-		label = "kernel1";
-		reg = <0xa00000 0x2800000>;  /* 40MiB */
-	};
-
-	partition@1000000 {
-		label = "rootfs1";
-		reg = <0x1000000 0x2200000>;  /* 34MiB */
-	};
-
-	/* kernel2 overlaps with rootfs2 by design */
-	partition@3200000 {
-		label = "kernel2";
-		reg = <0x3200000 0x2800000>; /* 40MiB */
-	};
-
-	partition@3800000 {
-		label = "rootfs2";
-		reg = <0x3800000 0x2200000>; /* 34MiB */
-	};
-
-	/*
-	 * 38MiB, last MiB is for the BBT, not writable
-	 */
-	partition@5a00000 {
-		label = "syscfg";
-		reg = <0x5a00000 0x2600000>;
-	};
-
-	/*
-	 * Unused area between "s_env" and "devinfo".
-	 * Moved here because otherwise the renumbered
-	 * partitions would break the bootloader
-	 * supplied bootargs
-	 */
-	partition@180000 {
-		label = "unused_area";
-		reg = <0x280000 0x680000>;   /* 6.5MiB */
+	partitions {
+		compatible = "fixed-partitions";
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		partition@0 {
+			label = "u-boot";
+			reg = <0x0000000 0x200000>;  /* 2MiB */
+			read-only;
+		};
+
+		partition@100000 {
+			label = "u_env";
+			reg = <0x200000 0x40000>;    /* 256KiB */
+		};
+
+		partition@140000 {
+			label = "s_env";
+			reg = <0x240000 0x40000>;    /* 256KiB */
+		};
+
+		partition@900000 {
+			label = "devinfo";
+			reg = <0x900000 0x100000>;   /* 1MiB */
+			read-only;
+		};
+
+		/* kernel1 overlaps with rootfs1 by design */
+		partition@a00000 {
+			label = "kernel1";
+			reg = <0xa00000 0x2800000>;  /* 40MiB */
+		};
+
+		partition@1000000 {
+			label = "rootfs1";
+			reg = <0x1000000 0x2200000>;  /* 34MiB */
+		};
+
+		/* kernel2 overlaps with rootfs2 by design */
+		partition@3200000 {
+			label = "kernel2";
+			reg = <0x3200000 0x2800000>; /* 40MiB */
+		};
+
+		partition@3800000 {
+			label = "rootfs2";
+			reg = <0x3800000 0x2200000>; /* 34MiB */
+		};
+
+		/*
+		 * 38MiB, last MiB is for the BBT, not writable
+		 */
+		partition@5a00000 {
+			label = "syscfg";
+			reg = <0x5a00000 0x2600000>;
+		};
+
+		/*
+		 * Unused area between "s_env" and "devinfo".
+		 * Moved here because otherwise the renumbered
+		 * partitions would break the bootloader
+		 * supplied bootargs
+		 */
+		partition@180000 {
+			label = "unused_area";
+			reg = <0x280000 0x680000>;   /* 6.5MiB */
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/armada-385-linksys.dtsi b/arch/arm/boot/dts/armada-385-linksys.dtsi
index 4a0d7360110b..827e82be2201 100644
--- a/arch/arm/boot/dts/armada-385-linksys.dtsi
+++ b/arch/arm/boot/dts/armada-385-linksys.dtsi
@@ -138,13 +138,19 @@
 	};
 };
 
-&nand {
+&nand_controller {
 	/* 128MiB or 256MiB */
 	status = "okay";
-	num-cs = <1>;
-	marvell,nand-keep-config;
-	marvell,nand-enable-arbiter;
-	nand-on-flash-bbt;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	nand: nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+	};
 };
 
 &mdio {
diff --git a/arch/arm/boot/dts/armada-385-synology-ds116.dts b/arch/arm/boot/dts/armada-385-synology-ds116.dts
index 6782ce481ac9..d8769956cbfc 100644
--- a/arch/arm/boot/dts/armada-385-synology-ds116.dts
+++ b/arch/arm/boot/dts/armada-385-synology-ds116.dts
@@ -139,7 +139,7 @@
 					      3700 5
 					      3900 6
 					      4000 7>;
-			cooling-cells = <2>;
+			#cooling-cells = <2>;
 		};
 
 		gpio-leds {
diff --git a/arch/arm/boot/dts/armada-388-db.dts b/arch/arm/boot/dts/armada-388-db.dts
index 05250d426dc4..a2bec07bf4c5 100644
--- a/arch/arm/boot/dts/armada-388-db.dts
+++ b/arch/arm/boot/dts/armada-388-db.dts
@@ -91,29 +91,6 @@
 				status = "okay";
 			};
 
-			flash@d0000 {
-				status = "okay";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-				nand-ecc-strength = <4>;
-				nand-ecc-step-size = <512>;
-
-				partition@0 {
-					label = "U-Boot";
-					reg = <0 0x800000>;
-				};
-				partition@800000 {
-					label = "Linux";
-					reg = <0x800000 0x800000>;
-				};
-				partition@1000000 {
-					label = "Filesystem";
-					reg = <0x1000000 0x3f000000>;
-				};
-			};
-
 			sdhci@d8000 {
 				broken-cd;
 				wp-inverted;
@@ -165,3 +142,35 @@
 	};
 };
 
+&nand_controller {
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+		nand-ecc-strength = <4>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "U-Boot";
+				reg = <0 0x800000>;
+			};
+			partition@800000 {
+				label = "Linux";
+				reg = <0x800000 0x800000>;
+			};
+			partition@1000000 {
+				label = "Filesystem";
+				reg = <0x1000000 0x3f000000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-38x.dtsi b/arch/arm/boot/dts/armada-38x.dtsi
index 4cc09e43eea2..18edc9bc7927 100644
--- a/arch/arm/boot/dts/armada-38x.dtsi
+++ b/arch/arm/boot/dts/armada-38x.dtsi
@@ -163,7 +163,7 @@
 			};
 
 			uart0: serial@12000 {
-				compatible = "snps,dw-apb-uart";
+				compatible = "marvell,armada-38x-uart";
 				reg = <0x12000 0x100>;
 				reg-shift = <2>;
 				interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
@@ -173,7 +173,7 @@
 			};
 
 			uart1: serial@12100 {
-				compatible = "snps,dw-apb-uart";
+				compatible = "marvell,armada-38x-uart";
 				reg = <0x12100 0x100>;
 				reg-shift = <2>;
 				interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
@@ -551,11 +551,11 @@
 				status = "okay";
 			};
 
-			nand: flash@d0000 {
-				compatible = "marvell,armada370-nand";
+			nand_controller: nand-controller@d0000 {
+				compatible = "marvell,armada370-nand-controller";
 				reg = <0xd0000 0x54>;
 				#address-cells = <1>;
-				#size-cells = <1>;
+				#size-cells = <0>;
 				interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&coredivclk 0>;
 				status = "disabled";
diff --git a/arch/arm/boot/dts/armada-390-db.dts b/arch/arm/boot/dts/armada-390-db.dts
index 1b2362e4c831..0e29474ae9a2 100644
--- a/arch/arm/boot/dts/armada-390-db.dts
+++ b/arch/arm/boot/dts/armada-390-db.dts
@@ -49,37 +49,6 @@
 				status = "okay";
 			};
 
-			flash@d0000 {
-				status = "okay";
-				pinctrl-0 = <&nand_pins>;
-				pinctrl-names = "default";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-				nand-ecc-strength = <8>;
-				nand-ecc-step-size = <512>;
-
-				partitions {
-					compatible = "fixed-partitions";
-					#address-cells = <1>;
-					#size-cells = <1>;
-
-					partition@0 {
-						label = "U-Boot";
-						reg = <0 0x800000>;
-					};
-					partition@800000 {
-						label = "Linux";
-						reg = <0x800000 0x800000>;
-					};
-					partition@1000000 {
-						label = "Filesystem";
-						reg = <0x1000000 0x3f000000>;
-					};
-				};
-			};
-
 			/* CON98 */
 			usb3@f8000 {
 				status = "okay";
@@ -136,3 +105,38 @@
 		};
 	};
 };
+
+&nand_controller {
+	status = "okay";
+	pinctrl-0 = <&nand_pins>;
+	pinctrl-names = "default";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+		nand-ecc-strength = <8>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "U-Boot";
+				reg = <0 0x800000>;
+			};
+			partition@800000 {
+				label = "Linux";
+				reg = <0x800000 0x800000>;
+			};
+			partition@1000000 {
+				label = "Filesystem";
+				reg = <0x1000000 0x3f000000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-395-gp.dts b/arch/arm/boot/dts/armada-395-gp.dts
index 2a9de192b423..6dd9e9077f84 100644
--- a/arch/arm/boot/dts/armada-395-gp.dts
+++ b/arch/arm/boot/dts/armada-395-gp.dts
@@ -57,41 +57,6 @@
 				status = "okay";
 			};
 
-			flash@d0000 {
-				status = "okay";
-				pinctrl-0 = <&nand_pins>;
-				pinctrl-names = "default";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-				nand-ecc-strength = <4>;
-				nand-ecc-step-size = <512>;
-
-				partitions {
-					compatible = "fixed-partitions";
-					#address-cells = <1>;
-					#size-cells = <1>;
-
-					partition@0 {
-						label = "U-Boot";
-						reg = <0x00000000 0x00600000>;
-						read-only;
-					};
-
-					partition@800000 {
-						label = "uImage";
-						reg = <0x00600000 0x00400000>;
-						read-only;
-					};
-
-					partition@1000000 {
-						label = "Root";
-						reg = <0x00a00000 0x3f600000>;
-					};
-				};
-			};
-
 			/* CON18 */
 			sdhci@d8000 {
 				clock-frequency = <200000000>;
@@ -130,3 +95,42 @@
 		};
 	};
 };
+
+&nand_controller {
+	status = "okay";
+	pinctrl-0 = <&nand_pins>;
+	pinctrl-names = "default";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+		nand-ecc-strength = <4>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "U-Boot";
+				reg = <0x00000000 0x00600000>;
+				read-only;
+			};
+
+			partition@800000 {
+				label = "uImage";
+				reg = <0x00600000 0x00400000>;
+				read-only;
+			};
+
+			partition@1000000 {
+				label = "Root";
+				reg = <0x00a00000 0x3f600000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-398-db.dts b/arch/arm/boot/dts/armada-398-db.dts
index 2337f24784f7..fc28308e5bc5 100644
--- a/arch/arm/boot/dts/armada-398-db.dts
+++ b/arch/arm/boot/dts/armada-398-db.dts
@@ -51,31 +51,6 @@
 				status = "okay";
 			};
 
-			flash@d0000 {
-				status = "okay";
-				pinctrl-0 = <&nand_pins>;
-				pinctrl-names = "default";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-				nand-ecc-strength = <8>;
-				nand-ecc-step-size = <512>;
-
-				partition@0 {
-					label = "U-Boot";
-					reg = <0 0x800000>;
-				};
-				partition@800000 {
-					label = "Linux";
-					reg = <0x800000 0x800000>;
-				};
-				partition@1000000 {
-					label = "Filesystem";
-					reg = <0x1000000 0x3f000000>;
-				};
-			};
-
 			usb3@f8000 {
 				status = "okay";
 			};
@@ -122,3 +97,38 @@
 		};
 	};
 };
+
+&nand_controller {
+	status = "okay";
+	pinctrl-0 = <&nand_pins>;
+	pinctrl-names = "default";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+		nand-ecc-strength = <8>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "U-Boot";
+				reg = <0 0x800000>;
+			};
+			partition@800000 {
+				label = "Linux";
+				reg = <0x800000 0x800000>;
+			};
+			partition@1000000 {
+				label = "Filesystem";
+				reg = <0x1000000 0x3f000000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-39x.dtsi b/arch/arm/boot/dts/armada-39x.dtsi
index c1737c0a8325..f0c949831efb 100644
--- a/arch/arm/boot/dts/armada-39x.dtsi
+++ b/arch/arm/boot/dts/armada-39x.dtsi
@@ -367,11 +367,11 @@
 				interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
 			};
 
-			flash@d0000 {
-				compatible = "marvell,armada370-nand";
+			nand_controller: nand-controller@d0000 {
+				compatible = "marvell,armada370-nand-controller";
 				reg = <0xd0000 0x54>;
 				#address-cells = <1>;
-				#size-cells = <1>;
+				#size-cells = <0>;
 				interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&coredivclk 0>;
 				status = "disabled";
diff --git a/arch/arm/boot/dts/armada-xp-98dx3236.dtsi b/arch/arm/boot/dts/armada-xp-98dx3236.dtsi
index a5da44fb35ed..8d708cc22495 100644
--- a/arch/arm/boot/dts/armada-xp-98dx3236.dtsi
+++ b/arch/arm/boot/dts/armada-xp-98dx3236.dtsi
@@ -306,6 +306,19 @@
 &pinctrl {
 	compatible = "marvell,98dx3236-pinctrl";
 
+	nand_pins: nand-pins {
+		marvell,pins = "mpp20", "mpp21", "mpp22",
+			       "mpp23", "mpp24", "mpp25",
+			       "mpp26", "mpp27", "mpp28",
+			       "mpp29", "mpp30";
+		marvell,function = "dev";
+	};
+
+	nand_rb: nand-rb {
+		marvell,pins = "mpp19";
+		marvell,function = "nand";
+	};
+
 	spi0_pins: spi0-pins {
 		marvell,pins = "mpp0", "mpp1",
 			       "mpp2", "mpp3";
diff --git a/arch/arm/boot/dts/armada-xp-db-dxbc2.dts b/arch/arm/boot/dts/armada-xp-db-dxbc2.dts
index 4c64923f1c52..f42fc6118b7c 100644
--- a/arch/arm/boot/dts/armada-xp-db-dxbc2.dts
+++ b/arch/arm/boot/dts/armada-xp-db-dxbc2.dts
@@ -70,9 +70,9 @@
 
 &nand {
 	status = "okay";
+	label = "pxa3xx_nand-0";
 	num-cs = <1>;
 	marvell,nand-keep-config;
-	marvell,nand-enable-arbiter;
 	nand-on-flash-bbt;
 	nand-ecc-strength = <4>;
 	nand-ecc-step-size = <512>;
diff --git a/arch/arm/boot/dts/armada-xp-db-xc3-24g4xg.dts b/arch/arm/boot/dts/armada-xp-db-xc3-24g4xg.dts
index a0ebb52683f1..8432f517e346 100644
--- a/arch/arm/boot/dts/armada-xp-db-xc3-24g4xg.dts
+++ b/arch/arm/boot/dts/armada-xp-db-xc3-24g4xg.dts
@@ -69,9 +69,9 @@
 
 &nand {
 	status = "okay";
+	label = "pxa3xx_nand-0";
 	num-cs = <1>;
 	marvell,nand-keep-config;
-	marvell,nand-enable-arbiter;
 	nand-on-flash-bbt;
 	nand-ecc-strength = <4>;
 	nand-ecc-step-size = <512>;
diff --git a/arch/arm/boot/dts/armada-xp-db.dts b/arch/arm/boot/dts/armada-xp-db.dts
index 73d3f5cb9828..f3ac7483afed 100644
--- a/arch/arm/boot/dts/armada-xp-db.dts
+++ b/arch/arm/boot/dts/armada-xp-db.dts
@@ -146,9 +146,9 @@
 
 			nand@d0000 {
 				status = "okay";
+				label = "pxa3xx_nand-0";
 				num-cs = <1>;
 				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
 				nand-on-flash-bbt;
 
 				partitions {
diff --git a/arch/arm/boot/dts/armada-xp-gp.dts b/arch/arm/boot/dts/armada-xp-gp.dts
index c143556bbb7b..1139e9469a83 100644
--- a/arch/arm/boot/dts/armada-xp-gp.dts
+++ b/arch/arm/boot/dts/armada-xp-gp.dts
@@ -162,9 +162,9 @@
 
 			nand@d0000 {
 				status = "okay";
+				label = "pxa3xx_nand-0";
 				num-cs = <1>;
 				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
 				nand-on-flash-bbt;
 			};
 		};
diff --git a/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts b/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts
index def62e9e835b..bbbb38888bb8 100644
--- a/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts
+++ b/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts
@@ -83,9 +83,9 @@
 
 			nand@d0000 {
 				status = "okay";
+				label = "pxa3xx_nand-0";
 				num-cs = <1>;
 				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
 				nand-on-flash-bbt;
 
 				partitions {
diff --git a/arch/arm/boot/dts/armada-xp-linksys-mamba.dts b/arch/arm/boot/dts/armada-xp-linksys-mamba.dts
index f8b60d937818..7a2606c3b62e 100644
--- a/arch/arm/boot/dts/armada-xp-linksys-mamba.dts
+++ b/arch/arm/boot/dts/armada-xp-linksys-mamba.dts
@@ -165,79 +165,6 @@
 			bm@c8000 {
 				status = "okay";
 			};
-
-			nand@d0000 {
-				status = "okay";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-				nand-ecc-strength = <4>;
-				nand-ecc-step-size = <512>;
-
-				partition@0 {
-					label = "u-boot";
-					reg = <0x0000000 0x100000>;  /* 1MB */
-					read-only;
-				};
-
-				partition@100000 {
-					label = "u_env";
-					reg = <0x100000 0x40000>;    /* 256KB */
-				};
-
-				partition@140000 {
-					label = "s_env";
-					reg = <0x140000 0x40000>;    /* 256KB */
-				};
-
-				partition@900000 {
-					label = "devinfo";
-					reg = <0x900000 0x100000>;   /* 1MB */
-					read-only;
-				};
-
-				/* kernel1 overlaps with rootfs1 by design */
-				partition@a00000 {
-					label = "kernel1";
-					reg = <0xa00000 0x2800000>;  /* 40MB */
-				};
-
-				partition@d00000 {
-					label = "rootfs1";
-					reg = <0xd00000 0x2500000>;  /* 37MB */
-				};
-
-				/* kernel2 overlaps with rootfs2 by design */
-				partition@3200000 {
-					label = "kernel2";
-					reg = <0x3200000 0x2800000>; /* 40MB */
-				};
-
-				partition@3500000 {
-					label = "rootfs2";
-					reg = <0x3500000 0x2500000>; /* 37MB */
-				};
-
-				/*
-				 * 38MB, last MB is for the BBT, not writable
-				 */
-				partition@5a00000 {
-					label = "syscfg";
-					reg = <0x5a00000 0x2600000>;
-				};
-
-				/*
-				 * Unused area between "s_env" and "devinfo".
-				 * Moved here because otherwise the renumbered
-				 * partitions would break the bootloader
-				 * supplied bootargs
-				 */
-				partition@180000 {
-					label = "unused_area";
-					reg = <0x180000 0x780000>;   /* 7.5MB */
-				};
-			};
 		};
 
 		bm-bppi {
@@ -434,3 +361,86 @@
 		};
 	};
 };
+
+&nand_controller {
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+		nand-ecc-strength = <4>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "u-boot";
+				reg = <0x0000000 0x100000>;  /* 1MB */
+				read-only;
+			};
+
+			partition@100000 {
+				label = "u_env";
+				reg = <0x100000 0x40000>;    /* 256KB */
+			};
+
+			partition@140000 {
+				label = "s_env";
+				reg = <0x140000 0x40000>;    /* 256KB */
+			};
+
+			partition@900000 {
+				label = "devinfo";
+				reg = <0x900000 0x100000>;   /* 1MB */
+				read-only;
+			};
+
+			/* kernel1 overlaps with rootfs1 by design */
+			partition@a00000 {
+				label = "kernel1";
+				reg = <0xa00000 0x2800000>;  /* 40MB */
+			};
+
+			partition@d00000 {
+				label = "rootfs1";
+				reg = <0xd00000 0x2500000>;  /* 37MB */
+			};
+
+			/* kernel2 overlaps with rootfs2 by design */
+			partition@3200000 {
+				label = "kernel2";
+				reg = <0x3200000 0x2800000>; /* 40MB */
+			};
+
+			partition@3500000 {
+				label = "rootfs2";
+				reg = <0x3500000 0x2500000>; /* 37MB */
+			};
+
+			/*
+			 * 38MB, last MB is for the BBT, not writable
+			 */
+			partition@5a00000 {
+				label = "syscfg";
+				reg = <0x5a00000 0x2600000>;
+			};
+
+			/*
+			 * Unused area between "s_env" and "devinfo".
+			 * Moved here because otherwise the renumbered
+			 * partitions would break the bootloader
+			 * supplied bootargs
+			 */
+			partition@180000 {
+				label = "unused_area";
+				reg = <0x180000 0x780000>;   /* 7.5MB */
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/armada-xp-netgear-rn2120.dts b/arch/arm/boot/dts/armada-xp-netgear-rn2120.dts
index c350b1cf5201..8ea73587db81 100644
--- a/arch/arm/boot/dts/armada-xp-netgear-rn2120.dts
+++ b/arch/arm/boot/dts/armada-xp-netgear-rn2120.dts
@@ -117,46 +117,6 @@
 				nr-ports = <2>;
 				status = "okay";
 			};
-
-			nand@d0000 {
-				status = "okay";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-
-				/* Use Hardware BCH ECC */
-				nand-ecc-strength = <4>;
-				nand-ecc-step-size = <512>;
-
-				partition@0 {
-					label = "u-boot";
-					reg = <0x0000000 0x180000>;  /* 1.5MB */
-					read-only;
-				};
-
-				partition@180000 {
-					label = "u-boot-env";
-					reg = <0x180000 0x20000>;    /* 128KB */
-					read-only;
-				};
-
-				partition@200000 {
-					label = "uImage";
-					reg = <0x0200000 0x600000>;    /* 6MB */
-				};
-
-				partition@800000 {
-					label = "minirootfs";
-					reg = <0x0800000 0x400000>;    /* 4MB */
-				};
-
-				/* Last MB is for the BBT, i.e. not writable */
-				partition@c00000 {
-					label = "ubifs";
-					reg = <0x0c00000 0x7400000>; /* 116MB */
-				};
-			};
 		};
 	};
 
@@ -345,3 +305,53 @@
 		marvell,function = "gpio";
 	};
 };
+
+&nand_controller {
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		label = "pxa3xx_nand-0";
+		nand-rb = <0>;
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+
+		/* Use Hardware BCH ECC */
+		nand-ecc-strength = <4>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "u-boot";
+				reg = <0x0000000 0x180000>;  /* 1.5MB */
+				read-only;
+			};
+
+			partition@180000 {
+				label = "u-boot-env";
+				reg = <0x180000 0x20000>;    /* 128KB */
+				read-only;
+			};
+
+			partition@200000 {
+				label = "uImage";
+				reg = <0x0200000 0x600000>;    /* 6MB */
+			};
+
+			partition@800000 {
+				label = "minirootfs";
+				reg = <0x0800000 0x400000>;    /* 4MB */
+			};
+
+			/* Last MB is for the BBT, i.e. not writable */
+			partition@c00000 {
+				label = "ubifs";
+				reg = <0x0c00000 0x7400000>; /* 116MB */
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/aspeed-ast2500-evb.dts b/arch/arm/boot/dts/aspeed-ast2500-evb.dts
index 91a36c1f029b..ede11c597673 100644
--- a/arch/arm/boot/dts/aspeed-ast2500-evb.dts
+++ b/arch/arm/boot/dts/aspeed-ast2500-evb.dts
@@ -79,3 +79,21 @@
 		reg = <0x4d>;
 	};
 };
+
+&ehci0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usb2ah_default>;
+};
+
+&ehci1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usb2bh_default>;
+};
+
+&uhci {
+	status = "okay";
+
+	/* No pinctrl, this follows the above EHCI settings */
+};
diff --git a/arch/arm/boot/dts/aspeed-bmc-intel-s2600wf.dts b/arch/arm/boot/dts/aspeed-bmc-intel-s2600wf.dts
new file mode 100644
index 000000000000..7a291de02543
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed-bmc-intel-s2600wf.dts
@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2017 Intel Corporation
+/dts-v1/;
+
+#include "aspeed-g5.dtsi"
+
+/ {
+	model = "S2600WF BMC";
+	compatible = "intel,s2600wf-bmc", "aspeed,ast2500";
+
+	chosen {
+		stdout-path = &uart5;
+		bootargs = "earlyprintk";
+	};
+
+	memory {
+		reg = <0x80000000 0x20000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		vga_memory: framebuffer@7f000000 {
+			no-map;
+			reg = <0x7f000000 0x01000000>;
+		};
+	};
+
+	iio-hwmon {
+		compatible = "iio-hwmon";
+		io-channels = <&adc 0>, <&adc 1>, <&adc 2>, <&adc 3>,
+			<&adc 4>, <&adc 5>, <&adc 6>, <&adc 7>,
+			<&adc 8>, <&adc 9>, <&adc 10>, <&adc 11>,
+			<&adc 12>, <&adc 13>, <&adc 14>, <&adc 15>;
+	};
+
+};
+
+&fmc {
+	status = "okay";
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "bmc";
+#include "openbmc-flash-layout.dtsi"
+	};
+};
+
+&spi1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi1_default>;
+
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "pnor";
+	};
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&mac0 {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rmii1_default>;
+	use-ncsi;
+};
+
+&mac1 {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rgmii2_default &pinctrl_mdio2_default>;
+};
+
+&i2c1 {
+	status = "okay";
+};
+
+&i2c2 {
+	status = "okay";
+};
+
+&i2c3 {
+	status = "okay";
+};
+
+&i2c4 {
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+};
+
+&i2c6 {
+	status = "okay";
+};
+
+&i2c7 {
+	status = "okay";
+};
+
+&i2c13 {
+	status = "okay";
+};
+
+&gfx {
+	status = "okay";
+};
+
+&pinctrl {
+	aspeed,external-nodes = <&gfx &lhc>;
+};
+
+&pwm_tacho {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm0_default &pinctrl_pwm1_default
+			 &pinctrl_pwm2_default &pinctrl_pwm3_default
+			 &pinctrl_pwm4_default &pinctrl_pwm5_default
+			 &pinctrl_pwm6_default &pinctrl_pwm7_default>;
+};
diff --git a/arch/arm/boot/dts/aspeed-bmc-opp-lanyang.dts b/arch/arm/boot/dts/aspeed-bmc-opp-lanyang.dts
new file mode 100644
index 000000000000..d598b6391362
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed-bmc-opp-lanyang.dts
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2018 Inventec Corporation
+/dts-v1/;
+
+#include "aspeed-g5.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+
+/ {
+	model = "Lanyang BMC";
+	compatible = "inventec,lanyang-bmc", "aspeed,ast2500";
+
+	chosen {
+		stdout-path = &uart5;
+		bootargs = "console=ttyS4,115200 earlyprintk";
+	};
+
+	memory {
+		reg = <0x80000000 0x40000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		flash_memory: region@98000000 {
+			no-map;
+			reg = <0x98000000 0x04000000>; /* 64M */
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		sys_boot_status {
+			label = "System_boot_status";
+			gpios = <&gpio ASPEED_GPIO(B, 6) GPIO_ACTIVE_LOW>;
+		};
+
+		attention {
+			label = "Attention_locator";
+			gpios = <&gpio ASPEED_GPIO(B, 7) GPIO_ACTIVE_HIGH>;
+		};
+
+		plt_fault {
+			label = "Platform_fault";
+			gpios = <&gpio ASPEED_GPIO(B, 1) GPIO_ACTIVE_HIGH>;
+		};
+
+		hdd_fault {
+			label = "Onboard_drive_fault";
+			gpios = <&gpio ASPEED_GPIO(B, 3) GPIO_ACTIVE_HIGH>;
+		};
+		bmc_err {
+			lable = "BMC_fault";
+			gpios = <&gpio ASPEED_GPIO(H, 6) GPIO_ACTIVE_HIGH>;
+		};
+
+		sys_err {
+			lable = "Sys_fault";
+			gpios = <&gpio ASPEED_GPIO(H, 7) GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	fsi: gpio-fsi {
+		compatible = "fsi-master-gpio", "fsi-master";
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		clock-gpios = <&gpio ASPEED_GPIO(J, 0) GPIO_ACTIVE_HIGH>;
+		data-gpios = <&gpio ASPEED_GPIO(J, 1) GPIO_ACTIVE_HIGH>;
+		trans-gpios = <&gpio ASPEED_GPIO(D, 5) GPIO_ACTIVE_HIGH>;
+		enable-gpios = <&gpio ASPEED_GPIO(D, 0) GPIO_ACTIVE_HIGH>;
+		mux-gpios = <&gpio ASPEED_GPIO(H, 2) GPIO_ACTIVE_HIGH>;
+	};
+
+	iio-hwmon {
+		compatible = "iio-hwmon";
+		io-channels = <&adc 0>, <&adc 1>, <&adc 2>, <&adc 3>,
+			<&adc 4>, <&adc 5>, <&adc 6>, <&adc 7>,
+			<&adc 8>, <&adc 9>, <&adc 10>, <&adc 11>,
+			<&adc 13>, <&adc 14>, <&adc 15>;
+	};
+
+	iio-hwmon-battery {
+		compatible = "iio-hwmon";
+		io-channels = <&adc 12>;
+	};
+};
+
+&pwm_tacho {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm0_default &pinctrl_pwm1_default
+		&pinctrl_pwm2_default &pinctrl_pwm3_default>;
+
+	fan@0 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x00>;
+	};
+
+	fan@1 {
+		reg = <0x01>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x01>;
+	};
+
+	fan@2 {
+		reg = <0x02>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x02>;
+	};
+
+	fan@3 {
+		reg = <0x03>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x03>;
+	};
+};
+
+&fmc {
+	status = "okay";
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "bmc";
+#include "openbmc-flash-layout.dtsi"
+	};
+};
+
+&spi1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi1_default>;
+
+	flash@0 {
+		status = "okay";
+		label = "pnor";
+		m25p,fast-read;
+	};
+};
+
+&spi2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2ck_default
+		     &pinctrl_spi2cs0_default
+		     &pinctrl_spi2cs1_default
+		     &pinctrl_spi2miso_default
+		     &pinctrl_spi2mosi_default>;
+
+	flash@0 {
+		status = "okay";
+	};
+};
+
+&uart1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_txd1_default
+		     &pinctrl_rxd1_default>;
+};
+
+&lpc_ctrl {
+	status = "okay";
+	memory-region = <&flash_memory>;
+	flash = <&spi1>;
+};
+
+&lpc_snoop {
+	status = "okay";
+	snoop-ports = <0x80>;
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&mac0 {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rmii1_default>;
+	use-ncsi;
+};
+
+&mac1 {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rgmii2_default &pinctrl_mdio2_default>;
+};
+
+&i2c0 {
+	status = "okay";
+
+	eeprom@55 {
+		compatible = "atmel,24c64";
+		reg = <0x55>;
+		pagesize = <32>;
+	};
+
+	rtc@68 {
+		compatible = "nxp,pcf8523";
+		reg = <0x68>;
+	};
+
+	tmp75@48 {
+		compatible = "ti,tmp75";
+		reg = <0x48>;
+	};
+};
+
+&i2c1 {
+	status = "okay";
+};
+
+&i2c2 {
+	status = "okay";
+};
+
+&i2c3 {
+	status = "okay";
+};
+
+&i2c4 {
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+};
+
+&i2c6 {
+	status = "okay";
+};
+
+&i2c7 {
+	status = "okay";
+};
+
+&i2c8 {
+	status = "okay";
+};
+
+&i2c9 {
+	status = "okay";
+};
+
+&i2c10 {
+	status = "okay";
+};
+
+&i2c11 {
+	status = "okay";
+};
+
+&vuart {
+	status = "okay";
+};
+
+&gfx {
+	status = "okay";
+};
+
+&pinctrl {
+	aspeed,external-nodes = <&gfx &lhc>;
+};
+
+&gpio {
+	pin_gpio_b0 {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(B, 0) GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "BMC_HDD1_PWR_EN";
+	};
+
+	pin_gpio_b5 {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(B, 5) GPIO_ACTIVE_HIGH>;
+		input;
+		line-name = "BMC_USB1_OCI2";
+	};
+
+	pin_gpio_h5 {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(H, 5) GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "BMC_CP0_PERST_ENABLE_R";
+	};
+
+	pin_gpio_z2 {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(Z, 2) GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "RST_PCA9546_U177_N";
+	};
+
+	pin_gpio_aa6 {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(AA, 6) GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "BMC_CP0_RESET_N";
+	};
+
+	pin_gpio_aa7 {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(AA, 7) GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "BMC_TPM_RESET_N";
+	};
+
+	pin_gpio_ab0 {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(AB, 0) GPIO_ACTIVE_LOW>;
+		output-high;
+		line-name = "BMC_USB_PWRON_N";
+	};
+};
+
+&ibt {
+	status = "okay";
+};
+
+&adc {
+	status = "okay";
+};
+
diff --git a/arch/arm/boot/dts/aspeed-bmc-opp-romulus.dts b/arch/arm/boot/dts/aspeed-bmc-opp-romulus.dts
index 51bc6a2e9dd5..389f5f83bef9 100644
--- a/arch/arm/boot/dts/aspeed-bmc-opp-romulus.dts
+++ b/arch/arm/boot/dts/aspeed-bmc-opp-romulus.dts
@@ -68,6 +68,12 @@
 			gpios = <&gpio ASPEED_GPIO(J, 2) GPIO_ACTIVE_LOW>;
 			linux,code = <ASPEED_GPIO(J, 2)>;
 		};
+
+		id-button {
+			label = "id-button";
+			gpios = <&gpio ASPEED_GPIO(Q, 7) GPIO_ACTIVE_LOW>;
+			linux,code = <ASPEED_GPIO(Q, 7)>;
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/aspeed-bmc-opp-witherspoon.dts b/arch/arm/boot/dts/aspeed-bmc-opp-witherspoon.dts
index 7056231cbee6..78a511e6e482 100644
--- a/arch/arm/boot/dts/aspeed-bmc-opp-witherspoon.dts
+++ b/arch/arm/boot/dts/aspeed-bmc-opp-witherspoon.dts
@@ -28,6 +28,34 @@
 		};
 	};
 
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		air-water {
+			label = "air-water";
+			gpios = <&gpio ASPEED_GPIO(B, 5) GPIO_ACTIVE_LOW>;
+			linux,code = <ASPEED_GPIO(B, 5)>;
+		};
+
+		checkstop {
+			label = "checkstop";
+			gpios = <&gpio ASPEED_GPIO(J, 2) GPIO_ACTIVE_LOW>;
+			linux,code = <ASPEED_GPIO(J, 2)>;
+		};
+
+		ps0-presence {
+			label = "ps0-presence";
+			gpios = <&gpio ASPEED_GPIO(P, 7) GPIO_ACTIVE_LOW>;
+			linux,code = <ASPEED_GPIO(P, 7)>;
+		};
+
+		ps1-presence {
+			label = "ps1-presence";
+			gpios = <&gpio ASPEED_GPIO(N, 0) GPIO_ACTIVE_LOW>;
+			linux,code = <ASPEED_GPIO(N, 0)>;
+		};
+	};
+
 	gpio-keys-polled {
 		compatible = "gpio-keys-polled";
 		#address-cells = <1>;
@@ -547,6 +575,10 @@
 	pinctrl-0 = <&pinctrl_wdtrst1_default>;
 };
 
+&wdt2 {
+	aspeed,alt-boot;
+};
+
 &ibt {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/aspeed-bmc-opp-zaius.dts b/arch/arm/boot/dts/aspeed-bmc-opp-zaius.dts
index ebe726a0d311..ccbf645ab84d 100644
--- a/arch/arm/boot/dts/aspeed-bmc-opp-zaius.dts
+++ b/arch/arm/boot/dts/aspeed-bmc-opp-zaius.dts
@@ -55,6 +55,12 @@
 			gpios = <&gpio ASPEED_GPIO(F, 7) GPIO_ACTIVE_LOW>;
 			linux,code = <ASPEED_GPIO(F, 7)>;
 		};
+
+		pcie-e2b-present{
+			label = "pcie-e2b-present";
+			gpios = <&gpio ASPEED_GPIO(E, 7) GPIO_ACTIVE_LOW>;
+			linux,code = <ASPEED_GPIO(E, 7)>;
+		};
 	};
 
 	leds {
diff --git a/arch/arm/boot/dts/aspeed-bmc-portwell-neptune.dts b/arch/arm/boot/dts/aspeed-bmc-portwell-neptune.dts
new file mode 100644
index 000000000000..43ed13963d35
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed-bmc-portwell-neptune.dts
@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2017 Facebook Inc.
+/dts-v1/;
+
+#include "aspeed-g5.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+
+/ {
+	model = "Portwell Neptune BMC";
+	compatible = "portwell,neptune-bmc", "aspeed,ast2500";
+	aliases {
+		serial0 = &uart1;
+		serial4 = &uart5;
+	};
+	chosen {
+		stdout-path = &uart5;
+		bootargs = "console=ttyS4,115200 earlyprintk";
+	};
+
+	memory {
+		reg = <0x80000000 0x20000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		postcode0 {
+			label="BMC_UP";
+			gpios = <&gpio ASPEED_GPIO(H, 0) GPIO_ACTIVE_HIGH>;
+			default-state = "on";
+		};
+		postcode1 {
+			label="BMC_HB";
+			gpios = <&gpio ASPEED_GPIO(H, 1) GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+		postcode2 {
+			label="FAULT";
+			gpios = <&gpio ASPEED_GPIO(H, 2) GPIO_ACTIVE_HIGH>;
+		};
+		// postcode3-7 are GPIOH3-H7
+	};
+};
+
+&fmc {
+	status = "okay";
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+#include "openbmc-flash-layout.dtsi"
+	};
+};
+
+&spi1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi1_default>;
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "pnor";
+	};
+};
+
+&uart1 {
+	// Host Console
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_txd1_default
+		     &pinctrl_rxd1_default>;
+};
+
+&uart5 {
+	// BMC Console
+	status = "okay";
+};
+
+&mac0 {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rmii1_default
+		     &pinctrl_mdio1_default>;
+};
+
+&mac1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rmii2_default>;
+	use-ncsi;
+};
+
+&i2c1 {
+	status = "okay";
+	// To PCIe slot SMBUS
+};
+
+&i2c2 {
+	status = "okay";
+	// To LAN I210
+};
+
+&i2c3 {
+	status = "okay";
+	// SMBus to COMe AB
+};
+
+&i2c4 {
+	status = "okay";
+	// I2C to COMe AB
+};
+
+&i2c5 {
+	status = "okay";
+// 	USB Debug card
+	pca9555@27 {
+		compatible = "nxp,pca9555";
+		reg = <0x27>;
+	};
+};
+
+&i2c6 {
+	status = "okay";
+	tpm@20 {
+		compatible = "infineon,slb9645tt";
+		reg = <0x20>;
+	};
+	tmp421@4e {
+		compatible = "ti,tmp421";
+		reg = <0x4e>;
+	};
+	tmp421@4f {
+		compatible = "ti,tmp421";
+		reg = <0x4f>;
+	};
+};
+
+&i2c8 {
+	status = "okay";
+	eeprom@51 {
+		compatible = "atmel,24c128";
+		reg = <0x51>;
+		pagesize = <32>;
+	};
+};
+
+&pwm_tacho {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm0_default &pinctrl_pwm1_default>;
+	fan@0 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x00>;
+	};
+
+	fan@1 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x01>;
+	};
+};
diff --git a/arch/arm/boot/dts/aspeed-g4.dtsi b/arch/arm/boot/dts/aspeed-g4.dtsi
index 518d2bc7c7fc..5e947ed496c2 100644
--- a/arch/arm/boot/dts/aspeed-g4.dtsi
+++ b/arch/arm/boot/dts/aspeed-g4.dtsi
@@ -108,6 +108,23 @@
 			status = "disabled";
 		};
 
+		ehci0: usb@1e6a1000 {
+			compatible = "aspeed,ast2400-ehci", "generic-ehci";
+			reg = <0x1e6a1000 0x100>;
+			interrupts = <5>;
+			clocks = <&syscon ASPEED_CLK_GATE_USBPORT1CLK>;
+			status = "disabled";
+		};
+
+		uhci: usb@1e6b0000 {
+			compatible = "aspeed,ast2400-uhci", "generic-uhci";
+			reg = <0x1e6b0000 0x100>;
+			interrupts = <14>;
+			#ports = <3>;
+			clocks = <&syscon ASPEED_CLK_GATE_USBUHCICLK>;
+			status = "disabled";
+		};
+
 		apb {
 			compatible = "simple-bus";
 			#address-cells = <1>;
@@ -125,6 +142,14 @@
 				pinctrl: pinctrl {
 					compatible = "aspeed,g4-pinctrl";
 				};
+
+			};
+
+			rng: hwrng@78 {
+				compatible = "timeriomem_rng";
+				reg = <0x78 0x4>;
+				period = <1>;
+				quality = <100>;
 			};
 
 			adc: adc@1e6e9000 {
@@ -1250,6 +1275,16 @@
 		groups = "USBCKI";
 	};
 
+	pinctrl_usb2h_default: usb2h_default {
+		function = "USB2H1";
+		groups = "USB2H1";
+	};
+
+	pinctrl_usb2d_default: usb2d_default {
+		function = "USB2D1";
+		groups = "USB2D1";
+	};
+
 	pinctrl_vgabios_rom_default: vgabios_rom_default {
 		function = "VGABIOS_ROM";
 		groups = "VGABIOS_ROM";
diff --git a/arch/arm/boot/dts/aspeed-g5.dtsi b/arch/arm/boot/dts/aspeed-g5.dtsi
index f9917717dd08..24eec00c4a95 100644
--- a/arch/arm/boot/dts/aspeed-g5.dtsi
+++ b/arch/arm/boot/dts/aspeed-g5.dtsi
@@ -143,6 +143,31 @@
 			status = "disabled";
 		};
 
+		ehci0: usb@1e6a1000 {
+			compatible = "aspeed,ast2500-ehci", "generic-ehci";
+			reg = <0x1e6a1000 0x100>;
+			interrupts = <5>;
+			clocks = <&syscon ASPEED_CLK_GATE_USBPORT1CLK>;
+			status = "disabled";
+		};
+
+		ehci1: usb@1e6a3000 {
+			compatible = "aspeed,ast2500-ehci", "generic-ehci";
+			reg = <0x1e6a3000 0x100>;
+			interrupts = <13>;
+			clocks = <&syscon ASPEED_CLK_GATE_USBPORT2CLK>;
+			status = "disabled";
+		};
+
+		uhci: usb@1e6b0000 {
+			compatible = "aspeed,ast2500-uhci", "generic-uhci";
+			reg = <0x1e6b0000 0x100>;
+			interrupts = <14>;
+			#ports = <2>;
+			clocks = <&syscon ASPEED_CLK_GATE_USBUHCICLK>;
+			status = "disabled";
+		};
+
 		apb {
 			compatible = "simple-bus";
 			#address-cells = <1>;
@@ -164,6 +189,13 @@
 				};
 			};
 
+			rng: hwrng@78 {
+				compatible = "timeriomem_rng";
+				reg = <0x78 0x4>;
+				period = <1>;
+				quality = <100>;
+			};
+
 			gfx: display@1e6e6000 {
 				compatible = "aspeed,ast2500-gfx", "syscon";
 				reg = <0x1e6e6000 0x1000>;
@@ -1380,6 +1412,21 @@
 		groups = "USBCKI";
 	};
 
+	pinctrl_usb2ah_default: usb2ah_default {
+		function = "USB2AH";
+		groups = "USB2AH";
+	};
+
+	pinctrl_usb11bhid_default: usb11bhid_default {
+		function = "USB11BHID";
+		groups = "USB11BHID";
+	};
+
+	pinctrl_usb2bh_default: usb2bh_default {
+		function = "USB2BH";
+		groups = "USB2BH";
+	};
+
 	pinctrl_vgabiosrom_default: vgabiosrom_default {
 		function = "VGABIOSROM";
 		groups = "VGABIOSROM";
diff --git a/arch/arm/boot/dts/at91-sama5d2_xplained.dts b/arch/arm/boot/dts/at91-sama5d2_xplained.dts
index e4bbb7e0f793..fcc85d70f36e 100644
--- a/arch/arm/boot/dts/at91-sama5d2_xplained.dts
+++ b/arch/arm/boot/dts/at91-sama5d2_xplained.dts
@@ -232,7 +232,7 @@
 						pinctrl-names = "default";
 						pinctrl-0 = <&pinctrl_charger_chglev &pinctrl_charger_lbo &pinctrl_charger_irq>;
 						interrupt-parent = <&pioA>;
-						interrupts = <PIN_PB13 GPIO_ACTIVE_LOW>;
+						interrupts = <PIN_PB13 IRQ_TYPE_EDGE_RISING>;
 
 						active-semi,chglev-gpios = <&pioA PIN_PA12 GPIO_ACTIVE_HIGH>;
 						active-semi,lbo-gpios = <&pioA PIN_PC8 GPIO_ACTIVE_LOW>;
diff --git a/arch/arm/boot/dts/at91-sama5d4ek.dts b/arch/arm/boot/dts/at91-sama5d4ek.dts
index 7887a7160a54..0702a2f2b336 100644
--- a/arch/arm/boot/dts/at91-sama5d4ek.dts
+++ b/arch/arm/boot/dts/at91-sama5d4ek.dts
@@ -129,8 +129,8 @@
 					wakeup-source;
 				};
 
-				atmel_mxt_ts@4c {
-					compatible = "atmel,atmel_mxt_ts";
+				touchscreen@4c {
+					compatible = "atmel,maxtouch";
 					reg = <0x4c>;
 					interrupt-parent = <&pioE>;
 					interrupts = <24 0x0>;
diff --git a/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts b/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts
index aa1fc7babfea..2cd9c5e4f892 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts
@@ -101,6 +101,12 @@
 	hpd-gpios = <&gpio 46 GPIO_ACTIVE_LOW>;
 };
 
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm0_gpio40 &pwm1_gpio45>;
+	status = "okay";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_gpio14>;
diff --git a/arch/arm/boot/dts/bcm2835-rpi-a.dts b/arch/arm/boot/dts/bcm2835-rpi-a.dts
index 425f6b0a5ef8..067d1f07a2d3 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-a.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-a.dts
@@ -96,6 +96,12 @@
 	hpd-gpios = <&gpio 46 GPIO_ACTIVE_HIGH>;
 };
 
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm0_gpio40 &pwm1_gpio45>;
+	status = "okay";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_gpio14>;
diff --git a/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts b/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts
index effa195e7895..cfbdaacbaeba 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts
@@ -103,6 +103,12 @@
 	hpd-gpios = <&gpio 46 GPIO_ACTIVE_LOW>;
 };
 
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm0_gpio40 &pwm1_gpio45>;
+	status = "okay";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_gpio14>;
diff --git a/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts b/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts
index 772ec3b48231..5641d162dfdb 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts
@@ -96,6 +96,12 @@
 	hpd-gpios = <&gpio 46 GPIO_ACTIVE_LOW>;
 };
 
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm0_gpio40 &pwm1_gpio45>;
+	status = "okay";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_gpio14>;
diff --git a/arch/arm/boot/dts/bcm2835-rpi-b.dts b/arch/arm/boot/dts/bcm2835-rpi-b.dts
index 434483d6fc14..31ff602e2cd3 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-b.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-b.dts
@@ -91,6 +91,12 @@
 	hpd-gpios = <&gpio 46 GPIO_ACTIVE_HIGH>;
 };
 
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm0_gpio40 &pwm1_gpio45>;
+	status = "okay";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_gpio14>;
diff --git a/arch/arm/boot/dts/bcm2835-rpi.dtsi b/arch/arm/boot/dts/bcm2835-rpi.dtsi
index 6c3cfaa77f3d..cb2d6d78a7fb 100644
--- a/arch/arm/boot/dts/bcm2835-rpi.dtsi
+++ b/arch/arm/boot/dts/bcm2835-rpi.dtsi
@@ -83,12 +83,6 @@
 	bus-width = <4>;
 };
 
-&pwm {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pwm0_gpio40 &pwm1_gpio45>;
-	status = "okay";
-};
-
 &usb {
 	power-domains = <&power RPI_POWER_DOMAIN_USB>;
 };
diff --git a/arch/arm/boot/dts/bcm2836-rpi-2-b.dts b/arch/arm/boot/dts/bcm2836-rpi-2-b.dts
index 5c339adabdf0..2fef70a09953 100644
--- a/arch/arm/boot/dts/bcm2836-rpi-2-b.dts
+++ b/arch/arm/boot/dts/bcm2836-rpi-2-b.dts
@@ -41,6 +41,12 @@
 	hpd-gpios = <&gpio 46 GPIO_ACTIVE_LOW>;
 };
 
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm0_gpio40 &pwm1_gpio45>;
+	status = "okay";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_gpio14>;
diff --git a/arch/arm/boot/dts/bcm2837-rpi-3-b-plus.dts b/arch/arm/boot/dts/bcm2837-rpi-3-b-plus.dts
new file mode 100644
index 000000000000..4adb85e66be3
--- /dev/null
+++ b/arch/arm/boot/dts/bcm2837-rpi-3-b-plus.dts
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0
+/dts-v1/;
+#include "bcm2837.dtsi"
+#include "bcm2835-rpi.dtsi"
+#include "bcm283x-rpi-lan7515.dtsi"
+#include "bcm283x-rpi-usb-host.dtsi"
+
+/ {
+	compatible = "raspberrypi,3-model-b-plus", "brcm,bcm2837";
+	model = "Raspberry Pi 3 Model B+";
+
+	chosen {
+		/* 8250 auxiliary UART instead of pl011 */
+		stdout-path = "serial1:115200n8";
+	};
+
+	memory {
+		reg = <0 0x40000000>;
+	};
+
+	leds {
+		act {
+			gpios = <&gpio 29 GPIO_ACTIVE_HIGH>;
+		};
+
+		pwr {
+			label = "PWR";
+			gpios = <&expgpio 2 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	wifi_pwrseq: wifi-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&expgpio 1 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&firmware {
+	expgpio: gpio {
+		compatible = "raspberrypi,firmware-gpio";
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-line-names = "BT_ON",
+				  "WL_ON",
+				  "STATUS_LED",
+				  "LAN_RUN",
+				  "",
+				  "CAM_GPIO0",
+				  "CAM_GPIO1",
+				  "";
+		status = "okay";
+	};
+};
+
+&hdmi {
+	hpd-gpios = <&gpio 28 GPIO_ACTIVE_LOW>;
+};
+
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm0_gpio40 &pwm1_gpio41>;
+	status = "okay";
+};
+
+/* SDHCI is used to control the SDIO for wireless */
+&sdhci {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_gpio34>;
+	status = "okay";
+	bus-width = <4>;
+	non-removable;
+	mmc-pwrseq = <&wifi_pwrseq>;
+
+	brcmf: wifi@1 {
+		reg = <1>;
+		compatible = "brcm,bcm4329-fmac";
+	};
+};
+
+/* SDHOST is used to drive the SD card */
+&sdhost {
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdhost_gpio48>;
+	status = "okay";
+	bus-width = <4>;
+};
+
+/* uart0 communicates with the BT module */
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_ctsrts_gpio30 &uart0_gpio32 &gpclk2_gpio43>;
+	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm43438-bt";
+		max-speed = <2000000>;
+		shutdown-gpios = <&expgpio 0 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+/* uart1 is mapped to the pin header */
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart1_gpio14>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/bcm2837-rpi-3-b.dts b/arch/arm/boot/dts/bcm2837-rpi-3-b.dts
index 0b31d995a066..c318bcbc6ba7 100644
--- a/arch/arm/boot/dts/bcm2837-rpi-3-b.dts
+++ b/arch/arm/boot/dts/bcm2837-rpi-3-b.dts
@@ -20,9 +20,14 @@
 
 	leds {
 		act {
-			gpios = <&gpio 47 GPIO_ACTIVE_HIGH>;
+			gpios = <&expgpio 2 GPIO_ACTIVE_HIGH>;
 		};
 	};
+
+	wifi_pwrseq: wifi-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&expgpio 1 GPIO_ACTIVE_HIGH>;
+	};
 };
 
 &firmware {
@@ -42,6 +47,16 @@
 	};
 };
 
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm0_gpio40 &pwm1_gpio41>;
+	status = "okay";
+};
+
+&hdmi {
+	hpd-gpios = <&expgpio 4 GPIO_ACTIVE_LOW>;
+};
+
 /* uart0 communicates with the BT module */
 &uart0 {
 	pinctrl-names = "default";
@@ -51,6 +66,7 @@
 	bluetooth {
 		compatible = "brcm,bcm43438-bt";
 		max-speed = <2000000>;
+		shutdown-gpios = <&expgpio 0 GPIO_ACTIVE_HIGH>;
 	};
 };
 
@@ -63,11 +79,19 @@
 
 /* SDHCI is used to control the SDIO for wireless */
 &sdhci {
+	#address-cells = <1>;
+	#size-cells = <0>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&emmc_gpio34>;
 	status = "okay";
 	bus-width = <4>;
 	non-removable;
+	mmc-pwrseq = <&wifi_pwrseq>;
+
+	brcmf: wifi@1 {
+		reg = <1>;
+		compatible = "brcm,bcm4329-fmac";
+	};
 };
 
 /* SDHOST is used to drive the SD card */
diff --git a/arch/arm/boot/dts/bcm283x-rpi-lan7515.dtsi b/arch/arm/boot/dts/bcm283x-rpi-lan7515.dtsi
new file mode 100644
index 000000000000..9403da0990d0
--- /dev/null
+++ b/arch/arm/boot/dts/bcm283x-rpi-lan7515.dtsi
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0
+/ {
+	aliases {
+		ethernet0 = &ethernet;
+	};
+};
+
+&usb {
+	usb-port@1 {
+		compatible = "usb424,2514";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		usb-port@1 {
+			compatible = "usb424,2514";
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			ethernet: ethernet@1 {
+				compatible = "usb424,7800";
+				reg = <1>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/bcm283x.dtsi b/arch/arm/boot/dts/bcm283x.dtsi
index ac00e730f898..61315cf734ef 100644
--- a/arch/arm/boot/dts/bcm283x.dtsi
+++ b/arch/arm/boot/dts/bcm283x.dtsi
@@ -136,6 +136,7 @@
 		rng@7e104000 {
 			compatible = "brcm,bcm2835-rng";
 			reg = <0x7e104000 0x10>;
+			interrupts = <2 29>;
 		};
 
 		mailbox: mailbox@7e00b880 {
diff --git a/arch/arm/boot/dts/bcm4708-asus-rt-ac56u.dts b/arch/arm/boot/dts/bcm4708-asus-rt-ac56u.dts
index 4175174e589a..ff2e551b9058 100644
--- a/arch/arm/boot/dts/bcm4708-asus-rt-ac56u.dts
+++ b/arch/arm/boot/dts/bcm4708-asus-rt-ac56u.dts
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for Asus RT-AC56U
  *
  * Copyright (C) 2015 Rafał Miłecki <zajec5@gmail.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
- * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
- * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm4708-asus-rt-ac68u.dts b/arch/arm/boot/dts/bcm4708-asus-rt-ac68u.dts
index 8fa033fea959..3bcc03788f38 100644
--- a/arch/arm/boot/dts/bcm4708-asus-rt-ac68u.dts
+++ b/arch/arm/boot/dts/bcm4708-asus-rt-ac68u.dts
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for Asus RT-AC68U
  *
  * Copyright (C) 2015 Rafał Miłecki <zajec5@gmail.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
- * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
- * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm4708-buffalo-wzr-1750dhp.dts b/arch/arm/boot/dts/bcm4708-buffalo-wzr-1750dhp.dts
index 8b64caabaad8..a587384f8e40 100644
--- a/arch/arm/boot/dts/bcm4708-buffalo-wzr-1750dhp.dts
+++ b/arch/arm/boot/dts/bcm4708-buffalo-wzr-1750dhp.dts
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for Buffalo WZR-1750DHP
  *
  * Copyright (C) 2014 Rafał Miłecki <zajec5@gmail.com>
- *
- * Licensed under the GNU/GPL. See COPYING for details.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm4708-linksys-ea6300-v1.dts b/arch/arm/boot/dts/bcm4708-linksys-ea6300-v1.dts
index 126ab5867772..6c8f0ad82332 100644
--- a/arch/arm/boot/dts/bcm4708-linksys-ea6300-v1.dts
+++ b/arch/arm/boot/dts/bcm4708-linksys-ea6300-v1.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright (C) 2017 Rafał Miłecki <rafal@milecki.pl>
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm4708-luxul-xap-1510.dts b/arch/arm/boot/dts/bcm4708-luxul-xap-1510.dts
index f591b0f256d8..ebda45fe9ea4 100644
--- a/arch/arm/boot/dts/bcm4708-luxul-xap-1510.dts
+++ b/arch/arm/boot/dts/bcm4708-luxul-xap-1510.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright 2016 Luxul Inc.
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm4708-luxul-xwc-1000.dts b/arch/arm/boot/dts/bcm4708-luxul-xwc-1000.dts
index 50d65d8fbd9a..9dd0e22c906a 100644
--- a/arch/arm/boot/dts/bcm4708-luxul-xwc-1000.dts
+++ b/arch/arm/boot/dts/bcm4708-luxul-xwc-1000.dts
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for Luxul XWC-1000
  *
  * Copyright 2014 Luxul Inc.
- *
- * Licensed under the GNU/GPL. See COPYING for details.
  */
 
 /dts-v1/;
@@ -26,9 +25,15 @@
 
 	nand: nand@18028000 {
 		nandcs@0 {
-			partition@0 {
-				label = "ubi";
-				reg = <0x00000000 0x08000000>;
+			partitions {
+				compatible = "fixed-partitions";
+				#address-cells = <1>;
+				#size-cells = <1>;
+
+				partition@0 {
+					label = "ubi";
+					reg = <0x00000000 0x08000000>;
+				};
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/bcm4708-netgear-r6300-v2.dts b/arch/arm/boot/dts/bcm4708-netgear-r6300-v2.dts
index bb66cebe0bd8..2642494c97a1 100644
--- a/arch/arm/boot/dts/bcm4708-netgear-r6300-v2.dts
+++ b/arch/arm/boot/dts/bcm4708-netgear-r6300-v2.dts
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for Netgear R6300 V2
  *
  * Copyright (C) 2014 Rafał Miłecki <zajec5@gmail.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
- * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
- * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm4708-smartrg-sr400ac.dts b/arch/arm/boot/dts/bcm4708-smartrg-sr400ac.dts
index 83a4c60bb431..e7b09b7b7d25 100644
--- a/arch/arm/boot/dts/bcm4708-smartrg-sr400ac.dts
+++ b/arch/arm/boot/dts/bcm4708-smartrg-sr400ac.dts
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X arm platform code.
  * DTS for SmartRG SR400ac
  *
  * Copyright (C) 2015 Rafał Miłecki <zajec5@gmail.com>
- *
- * Licensed under the GNU/GPL. See COPYING for details.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm47081-asus-rt-n18u.dts b/arch/arm/boot/dts/bcm47081-asus-rt-n18u.dts
index 0800a964f2fe..16314fcc6e56 100644
--- a/arch/arm/boot/dts/bcm47081-asus-rt-n18u.dts
+++ b/arch/arm/boot/dts/bcm47081-asus-rt-n18u.dts
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for Asus RT-N18U
  *
  * Copyright (C) 2014 Rafał Miłecki <zajec5@gmail.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
- * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
- * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm47081-buffalo-wzr-600dhp2.dts b/arch/arm/boot/dts/bcm47081-buffalo-wzr-600dhp2.dts
index c2af33eb47de..328aa90240ce 100644
--- a/arch/arm/boot/dts/bcm47081-buffalo-wzr-600dhp2.dts
+++ b/arch/arm/boot/dts/bcm47081-buffalo-wzr-600dhp2.dts
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for Buffalo WZR-600DHP2
  *
  * Copyright (C) 2014 Rafał Miłecki <zajec5@gmail.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
- * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
- * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm47081-buffalo-wzr-900dhp.dts b/arch/arm/boot/dts/bcm47081-buffalo-wzr-900dhp.dts
index 87ea6ba664f5..8ea46eed26e2 100644
--- a/arch/arm/boot/dts/bcm47081-buffalo-wzr-900dhp.dts
+++ b/arch/arm/boot/dts/bcm47081-buffalo-wzr-900dhp.dts
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for Buffalo WZR-900DHP
  *
  * Copyright (C) 2015 Rafał Miłecki <zajec5@gmail.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
- * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
- * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm47081-luxul-xap-1410.dts b/arch/arm/boot/dts/bcm47081-luxul-xap-1410.dts
index 9b5759849983..5eeac7302329 100644
--- a/arch/arm/boot/dts/bcm47081-luxul-xap-1410.dts
+++ b/arch/arm/boot/dts/bcm47081-luxul-xap-1410.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright 2017 Luxul Inc.
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm47081-luxul-xwr-1200.dts b/arch/arm/boot/dts/bcm47081-luxul-xwr-1200.dts
index ba1c19b1b3eb..da4d9ec62fc6 100644
--- a/arch/arm/boot/dts/bcm47081-luxul-xwr-1200.dts
+++ b/arch/arm/boot/dts/bcm47081-luxul-xwr-1200.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright 2017 Luxul Inc.
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm47081-tplink-archer-c5-v2.dts b/arch/arm/boot/dts/bcm47081-tplink-archer-c5-v2.dts
index 3ed8de42cb48..c94c732188fb 100644
--- a/arch/arm/boot/dts/bcm47081-tplink-archer-c5-v2.dts
+++ b/arch/arm/boot/dts/bcm47081-tplink-archer-c5-v2.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright (C) 2017 Rafał Miłecki <rafal@milecki.pl>
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts b/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts
index df473cc41572..22271818f901 100644
--- a/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts
+++ b/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for Asus RT-AC87U
  *
  * Copyright (C) 2015 Rafał Miłecki <zajec5@gmail.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
- * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
- * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts b/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts
index 92058c73ee59..79a9633ec417 100644
--- a/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts
+++ b/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for Buffalo WXR-1900DHP
  *
  * Copyright (C) 2015 Felix Fietkau <nbd@openwrt.org>
- *
- * Licensed under the GNU/GPL. See COPYING for details.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts b/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts
index 3d1d9c2c4efc..db744a5e122d 100644
--- a/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts
+++ b/arch/arm/boot/dts/bcm4709-linksys-ea9200.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright (C) 2017 Rafał Miłecki <rafal@milecki.pl>
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm4709-netgear-r7000.dts b/arch/arm/boot/dts/bcm4709-netgear-r7000.dts
index f43ab4721456..9e267d38df4c 100644
--- a/arch/arm/boot/dts/bcm4709-netgear-r7000.dts
+++ b/arch/arm/boot/dts/bcm4709-netgear-r7000.dts
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for Netgear R7000
  *
  * Copyright (C) 2015 Rafał Miłecki <zajec5@gmail.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
- * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
- * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts b/arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts
index ec4a50e440f6..f5bf6586ae07 100644
--- a/arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts
+++ b/arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright (C) 2016 Rafał Miłecki <rafal@milecki.pl>
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts b/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts
index 7cc7d344fe5b..d173bcd93b91 100644
--- a/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts
+++ b/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X ARM platform code.
  * DTS for D-Link DIR-885L
  *
  * Copyright (C) 2016 Rafał Miłecki <zajec5@gmail.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
- * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
- * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
  */
 
 /dts-v1/;
@@ -37,9 +26,15 @@
 
 	nand: nand@18028000 {
 		nandcs@0 {
-			partition@0 {
-				label = "firmware";
-				reg = <0x00000000 0x08000000>;
+			partitions {
+				compatible = "fixed-partitions";
+				#address-cells = <1>;
+				#size-cells = <1>;
+
+				partition@0 {
+					label = "firmware";
+					reg = <0x00000000 0x08000000>;
+				};
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/bcm47094-linksys-panamera.dts b/arch/arm/boot/dts/bcm47094-linksys-panamera.dts
index b6750f70dffb..f47afe36d857 100644
--- a/arch/arm/boot/dts/bcm47094-linksys-panamera.dts
+++ b/arch/arm/boot/dts/bcm47094-linksys-panamera.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright (C) 2017 Rafał Miłecki <rafal@milecki.pl>
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm47094-luxul-abr-4500.dts b/arch/arm/boot/dts/bcm47094-luxul-abr-4500.dts
index ecd22a246746..a5cef51cfe4f 100644
--- a/arch/arm/boot/dts/bcm47094-luxul-abr-4500.dts
+++ b/arch/arm/boot/dts/bcm47094-luxul-abr-4500.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright (C) 2017 Luxul Inc.
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm47094-luxul-xap-1610.dts b/arch/arm/boot/dts/bcm47094-luxul-xap-1610.dts
new file mode 100644
index 000000000000..7fd85475893d
--- /dev/null
+++ b/arch/arm/boot/dts/bcm47094-luxul-xap-1610.dts
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright 2018 Luxul Inc.
+ */
+
+/dts-v1/;
+
+#include "bcm47094.dtsi"
+
+/ {
+	compatible = "luxul,xap-1610-v1", "brcm,bcm47094", "brcm,bcm4708";
+	model = "Luxul XAP-1610 V1";
+
+	chosen {
+		bootargs = "earlycon";
+	};
+
+	memory {
+		reg = <0x00000000 0x08000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		status	{
+			label = "bcm53xx:green:status";
+			gpios = <&chipcommon 0 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "timer";
+		};
+
+		2ghz {
+			label = "bcm53xx:blue:2ghz";
+			gpios = <&chipcommon 13 GPIO_ACTIVE_LOW>;
+		};
+
+		5ghz {
+			label = "bcm53xx:blue:5ghz";
+			gpios = <&chipcommon 14 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		restart {
+			label = "Reset";
+			linux,code = <KEY_RESTART>;
+			gpios = <&chipcommon 17 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&spi_nor {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/bcm47094-luxul-xbr-4500.dts b/arch/arm/boot/dts/bcm47094-luxul-xbr-4500.dts
index 15ffb1abc440..7496aabf8f77 100644
--- a/arch/arm/boot/dts/bcm47094-luxul-xbr-4500.dts
+++ b/arch/arm/boot/dts/bcm47094-luxul-xbr-4500.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright (C) 2017 Luxul Inc.
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts b/arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts
index bc1d1e10d4ac..53aaa5212610 100644
--- a/arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts
+++ b/arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright 2016 Luxul Inc.
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm47094-luxul-xwr-3150-v1.dts b/arch/arm/boot/dts/bcm47094-luxul-xwr-3150-v1.dts
new file mode 100644
index 000000000000..bdad7267255a
--- /dev/null
+++ b/arch/arm/boot/dts/bcm47094-luxul-xwr-3150-v1.dts
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright 2018 Luxul Inc.
+ */
+
+/dts-v1/;
+
+#include "bcm47094.dtsi"
+#include "bcm5301x-nand-cs0-bch8.dtsi"
+
+/ {
+	compatible = "luxul,xwr-3150-v1", "brcm,bcm47094", "brcm,bcm4708";
+	model = "Luxul XWR-3150 V1";
+
+	chosen {
+		bootargs = "earlycon";
+	};
+
+	memory {
+		reg = <0x00000000 0x08000000
+		       0x88000000 0x18000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		power	{
+			label = "bcm53xx:green:power";
+			gpios = <&chipcommon 0 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-on";
+		};
+
+		usb3	{
+			label = "bcm53xx:green:usb3";
+			gpios = <&chipcommon 8 GPIO_ACTIVE_LOW>;
+			trigger-sources = <&ohci_port1>, <&ehci_port1>,
+					  <&xhci_port1>;
+			linux,default-trigger = "usbport";
+		};
+
+		status	{
+			label = "bcm53xx:green:status";
+			gpios = <&chipcommon 10 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "timer";
+		};
+
+		2ghz {
+			label = "bcm53xx:green:2ghz";
+			gpios = <&chipcommon 13 GPIO_ACTIVE_LOW>;
+		};
+
+		5ghz {
+			label = "bcm53xx:green:5ghz";
+			gpios = <&chipcommon 14 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		restart {
+			label = "Reset";
+			linux,code = <KEY_RESTART>;
+			gpios = <&chipcommon 17 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&usb3 {
+	vcc-gpio = <&chipcommon 18 GPIO_ACTIVE_HIGH>;
+};
+
+&spi_nor {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/bcm47094-netgear-r8500.dts b/arch/arm/boot/dts/bcm47094-netgear-r8500.dts
index 859929973158..0e718edc065a 100644
--- a/arch/arm/boot/dts/bcm47094-netgear-r8500.dts
+++ b/arch/arm/boot/dts/bcm47094-netgear-r8500.dts
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright (C) 2016 Rafał Miłecki <rafal@milecki.pl>
- *
- * Licensed under the ISC license.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/bcm5301x-nand-cs0-bch1.dtsi b/arch/arm/boot/dts/bcm5301x-nand-cs0-bch1.dtsi
index 24b099c00f13..c349e8f0afc5 100644
--- a/arch/arm/boot/dts/bcm5301x-nand-cs0-bch1.dtsi
+++ b/arch/arm/boot/dts/bcm5301x-nand-cs0-bch1.dtsi
@@ -1,9 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom Northstar NAND.
  *
  * Copyright (C) 2016 Rafał Miłecki <rafal.milecki@gmail.com>
- *
- * Licensed under the ISC license.
  */
 
 #include "bcm5301x-nand-cs0.dtsi"
diff --git a/arch/arm/boot/dts/bcm5301x-nand-cs0-bch4.dtsi b/arch/arm/boot/dts/bcm5301x-nand-cs0-bch4.dtsi
index b4e875df9528..18e25e302b13 100644
--- a/arch/arm/boot/dts/bcm5301x-nand-cs0-bch4.dtsi
+++ b/arch/arm/boot/dts/bcm5301x-nand-cs0-bch4.dtsi
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Copyright 2016 Luxul Inc.
- *
- * Licensed under the ISC license.
  */
 
 #include "bcm5301x-nand-cs0.dtsi"
diff --git a/arch/arm/boot/dts/bcm5301x-nand-cs0-bch8.dtsi b/arch/arm/boot/dts/bcm5301x-nand-cs0-bch8.dtsi
index 9a9630ded306..c8e56d30bd6f 100644
--- a/arch/arm/boot/dts/bcm5301x-nand-cs0-bch8.dtsi
+++ b/arch/arm/boot/dts/bcm5301x-nand-cs0-bch8.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom BCM470X / BCM5301X Nand chip defaults.
  *
@@ -5,8 +6,6 @@
  * and uses 8 bit ECC.
  *
  * Copyright (C) 2015 Hauke Mehrtens <hauke@hauke-m.de>
- *
- * Licensed under the GNU/GPL. See COPYING for details.
  */
 
 #include "bcm5301x-nand-cs0.dtsi"
diff --git a/arch/arm/boot/dts/bcm5301x-nand-cs0.dtsi b/arch/arm/boot/dts/bcm5301x-nand-cs0.dtsi
index 168495106b82..e5a2d62daf92 100644
--- a/arch/arm/boot/dts/bcm5301x-nand-cs0.dtsi
+++ b/arch/arm/boot/dts/bcm5301x-nand-cs0.dtsi
@@ -1,9 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
 /*
  * Broadcom Northstar NAND.
  *
  * Copyright (C) 2015 Hauke Mehrtens <hauke@hauke-m.de>
- *
- * Licensed under the GNU/GPL. See COPYING for details.
  */
 
 / {
diff --git a/arch/arm/boot/dts/berlin2-sony-nsz-gs7.dts b/arch/arm/boot/dts/berlin2-sony-nsz-gs7.dts
index 1c475796d17f..64a297759eb9 100644
--- a/arch/arm/boot/dts/berlin2-sony-nsz-gs7.dts
+++ b/arch/arm/boot/dts/berlin2-sony-nsz-gs7.dts
@@ -1,39 +1,8 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
 /*
  * Device Tree file for Sony NSZ-GS7
  *
  * Sebastian Hesselbarth <sebastian.hesselbarth@gmail.com>
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is licensed under the terms of the GNU General Public
- *     License version 2. This program is licensed "as is" without any
- *     warranty of any kind, whether express or implied.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/berlin2.dtsi b/arch/arm/boot/dts/berlin2.dtsi
index d575823c5750..db67377af266 100644
--- a/arch/arm/boot/dts/berlin2.dtsi
+++ b/arch/arm/boot/dts/berlin2.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
 /*
  * Device Tree Include file for Marvell Armada 1500 (Berlin BG2) SoC
  *
@@ -5,38 +6,6 @@
  *
  * based on GPL'ed 2.6 kernel sources
  *  (c) Marvell International Ltd.
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is licensed under the terms of the GNU General Public
- *     License version 2. This program is licensed "as is" without any
- *     warranty of any kind, whether express or implied.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #include <dt-bindings/clock/berlin2.h>
@@ -149,7 +118,7 @@
 		local-timer@ad0600 {
 			compatible = "arm,cortex-a9-twd-timer";
 			reg = <0xad0600 0x20>;
-			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>;
+			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_EDGE_RISING)>;
 			clocks = <&chip_clk CLKID_TWD>;
 		};
 
diff --git a/arch/arm/boot/dts/berlin2cd-google-chromecast.dts b/arch/arm/boot/dts/berlin2cd-google-chromecast.dts
index ca24def0ce13..56fa951bc86f 100644
--- a/arch/arm/boot/dts/berlin2cd-google-chromecast.dts
+++ b/arch/arm/boot/dts/berlin2cd-google-chromecast.dts
@@ -1,39 +1,8 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
 /*
  * Device Tree file for Google Chromecast
  *
  * Sebastian Hesselbarth <sebastian.hesselbarth@gmail.com>
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is licensed under the terms of the GNU General Public
- *     License version 2. This program is licensed "as is" without any
- *     warranty of any kind, whether express or implied.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
 /dts-v1/;
@@ -52,22 +21,35 @@
 
 	memory@0 {
 		device_type = "memory";
-		reg = <0x00000000 0x20000000>; /* 512 MB */
+
+		/*
+		 * We're using "linux,usable-memory" instead of "reg" here
+		 * because the (signed and encrypted) bootloader that shipped
+		 * with this device provides an incorrect memory range in
+		 * ATAG_MEM. Linux helpfully overrides the "reg" property with
+		 * data from the ATAG, so we can't specify the proper range
+		 * normally. Fortunately, this alternate property is checked
+		 * first by the OF driver, so we can (ab)use it instead.
+		 */
+		linux,usable-memory = <0x00000000 0x20000000>; /* 512 MB */
 	};
 
 	leds {
-		compatible = "gpio-leds";
+		compatible = "pwm-leds";
+		pinctrl-0 = <&ledpwm_pmux>;
+		pinctrl-names = "default";
 
 		white {
 			label = "white";
-			gpios = <&portc 1 GPIO_ACTIVE_HIGH>;
-			default-state = "keep";
+			pwms = <&pwm 0 600000 0>;
+			max-brightness = <255>;
+			linux,default-trigger = "default-on";
 		};
 
 		red {
 			label = "red";
-			gpios = <&portc 2 GPIO_ACTIVE_HIGH>;
-			default-state = "keep";
+			pwms = <&pwm 1 600000 0>;
+			max-brightness = <255>;
 		};
 	};
 };
@@ -86,3 +68,10 @@
 &usb_phy1 { status = "okay"; };
 
 &usb1 { status = "okay"; };
+
+&soc_pinctrl {
+	ledpwm_pmux: ledpwm-pmux {
+		groups = "G0";
+		function = "pwm";
+	};
+};
diff --git a/arch/arm/boot/dts/berlin2cd-valve-steamlink.dts b/arch/arm/boot/dts/berlin2cd-valve-steamlink.dts
new file mode 100644
index 000000000000..79ac842ae461
--- /dev/null
+++ b/arch/arm/boot/dts/berlin2cd-valve-steamlink.dts
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright 2018 Alexander Monakov <amonakov@gmail.com>
+ */
+/dts-v1/;
+
+#include "berlin2cd.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "Valve Steam Link";
+	compatible = "valve,steamlink", "marvell,berlin2cd", "marvell,berlin";
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x00000000 0x20000000>; /* 512 MB */
+	};
+
+	gpio-restart {
+		compatible = "gpio-restart";
+		gpios = <&porta 6 GPIO_ACTIVE_HIGH>;
+		active-delay = <100>;
+		inactive-delay = <10>;
+		wait-delay = <100>;
+		priority = <200>;
+	};
+};
+
+&cpu {
+	cpu-supply = <&vcpu>;
+	operating-points = <
+		/* kHz    uV */
+		1000000 1325000
+	>;
+};
+
+&i2c0 {
+	status = "okay";
+
+	/* There are two regulators on the board. One is accessible via I2C,
+	 * with buck1 providing SoC power (set up by bootloader to 1.325V or
+	 * less depending on leakage value in OTP), and buck2 likely used for
+	 * DRAM (providing 1.35V). The other regulator on the opposite side
+	 * of the board is probably supplying SDIO and NAND fixed voltages. */
+	regulator@19 {
+		compatible = "marvell,88pg868";
+		reg = <0x19>;
+
+		vcpu: buck1 {
+			regulator-boot-on;
+			regulator-always-on;
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1325000>;
+		};
+	};
+};
+
+/* Fixed interface to on-board Marvell 8897 Wi-Fi/Bluetooth/NFC chip. */
+&sdhci0 {
+	keep-power-in-suspend;
+	non-removable;
+	status = "okay";
+};
+
+&uart0 {
+	/* RX/TX are routed to TP50/TP51 on the board. */
+	status = "okay";
+};
+
+/* The SoC is connected to on-board USB hub that in turn has one downstream
+ * port wired to the on-board Steam Controller wireless receiver chip. */
+&usb_phy1 { status = "okay"; };
+
+&usb1 {
+	dr_mode = "host";
+	status = "okay";
+};
+
+&eth1 { status = "okay"; };
diff --git a/arch/arm/boot/dts/berlin2cd.dtsi b/arch/arm/boot/dts/berlin2cd.dtsi
index 501c59d97eae..e5c1f4213ff9 100644
--- a/arch/arm/boot/dts/berlin2cd.dtsi
+++ b/arch/arm/boot/dts/berlin2cd.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
 /*
  * Device Tree Include file for Marvell Armada 1500-mini (Berlin BG2CD) SoC
  *
@@ -5,38 +6,6 @@
  *
  * based on GPL'ed 2.6 kernel sources
  *  (c) Marvell International Ltd.
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is licensed under the terms of the GNU General Public
- *     License version 2. This program is licensed "as is" without any
- *     warranty of any kind, whether express or implied.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #include <dt-bindings/clock/berlin2.h>
@@ -57,7 +26,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		cpu@0 {
+		cpu: cpu@0 {
 			compatible = "arm,cortex-a9";
 			device_type = "cpu";
 			next-level-cache = <&l2>;
@@ -73,6 +42,12 @@
 		};
 	};
 
+	pmu {
+		compatible = "arm,cortex-a9-pmu";
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
 	refclk: oscillator {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
@@ -87,11 +62,6 @@
 
 		ranges = <0 0xf7000000 0x1000000>;
 
-		pmu {
-			compatible = "arm,cortex-a9-pmu";
-			interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
 		sdhci0: sdhci@ab0000 {
 			compatible = "mrvl,pxav3-mmc";
 			reg = <0xab0000 0x200>;
@@ -108,6 +78,11 @@
 			cache-level = <2>;
 		};
 
+		snoop-control-unit@ad0000 {
+			compatible = "arm,cortex-a9-scu";
+			reg = <0xad0000 0x100>;
+		};
+
 		gic: interrupt-controller@ad1000 {
 			compatible = "arm,cortex-a9-gic";
 			reg = <0xad1000 0x1000>, <0xad0100 0x0100>;
@@ -115,10 +90,24 @@
 			#interrupt-cells = <3>;
 		};
 
+		global-timer@ad0200 {
+			compatible = "arm,cortex-a9-global-timer";
+			reg = <0xad0200 0x20>;
+			interrupts = <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_EDGE_RISING)>;
+			clocks = <&chip_clk CLKID_TWD>;
+		};
+
 		local-timer@ad0600 {
 			compatible = "arm,cortex-a9-twd-timer";
 			reg = <0xad0600 0x20>;
-			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_HIGH)>;
+			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_EDGE_RISING)>;
+			clocks = <&chip_clk CLKID_TWD>;
+		};
+
+		local-wdt@ad0620 {
+			compatible = "arm,cortex-a9-twd-wdt";
+			reg = <0xad0620 0x20>;
+			interrupts = <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_EDGE_RISING)>;
 			clocks = <&chip_clk CLKID_TWD>;
 		};
 
@@ -254,6 +243,60 @@
 				};
 			};
 
+			i2c0: i2c@1400 {
+				compatible = "snps,designware-i2c";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x1400 0x100>;
+				interrupts = <16>;
+				clocks = <&chip_clk CLKID_CFG>;
+				status = "disabled";
+			};
+
+			i2c1: i2c@1800 {
+				compatible = "snps,designware-i2c";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x1800 0x100>;
+				interrupts = <17>;
+				clocks = <&chip_clk CLKID_CFG>;
+				status = "disabled";
+			};
+
+			spi0: spi@1c00 {
+				compatible = "snps,dw-apb-ssi";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x1c00 0x100>;
+				interrupts = <4>;
+				clocks = <&chip_clk CLKID_CFG>;
+				status = "disabled";
+			};
+
+			wdt4: watchdog@2000 {
+				compatible = "snps,dw-wdt";
+				reg = <0x2000 0x100>;
+				clocks = <&chip_clk CLKID_CFG>;
+				interrupts = <5>;
+				status = "disabled";
+			};
+
+			wdt5: watchdog@2400 {
+				compatible = "snps,dw-wdt";
+				reg = <0x2400 0x100>;
+				clocks = <&chip_clk CLKID_CFG>;
+				interrupts = <6>;
+				status = "disabled";
+			};
+
+			wdt6: watchdog@2800 {
+				compatible = "snps,dw-wdt";
+				reg = <0x2800 0x100>;
+				clocks = <&chip_clk CLKID_CFG>;
+				interrupts = <7>;
+				status = "disabled";
+			};
+
 			timer0: timer@2c00 {
 				compatible = "snps,dw-apb-timer";
 				reg = <0x2c00 0x14>;
@@ -435,6 +478,36 @@
 				};
 			};
 
+			spi1: spi@6000 {
+				compatible = "snps,dw-apb-ssi";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x6000 0x100>;
+				clocks = <&refclk>;
+				interrupts = <5>;
+				status = "disabled";
+			};
+
+			i2c2: i2c@7000 {
+				compatible = "snps,designware-i2c";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x7000 0x100>;
+				interrupts = <6>;
+				clocks = <&refclk>;
+				status = "disabled";
+			};
+
+			i2c3: i2c@8000 {
+				compatible = "snps,designware-i2c";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x8000 0x100>;
+				interrupts = <7>;
+				clocks = <&refclk>;
+				status = "disabled";
+			};
+
 			sm_gpio0: gpio@c000 {
 				compatible = "snps,dw-apb-gpio";
 				reg = <0xc000 0x400>;
@@ -472,6 +545,16 @@
 				status = "disabled";
 			};
 
+			uart2: serial@b000 {
+				compatible = "snps,dw-apb-uart";
+				reg = <0xb000 0x100>;
+				reg-shift = <2>;
+				reg-io-width = <1>;
+				interrupts = <10>;
+				clocks = <&refclk>;
+				status = "disabled";
+			};
+
 			sysctrl: system-controller@d000 {
 				compatible = "simple-mfd", "syscon";
 				reg = <0xd000 0x100>;
@@ -479,6 +562,12 @@
 				sys_pinctrl: pin-controller {
 					compatible = "marvell,berlin2cd-system-pinctrl";
 				};
+
+				adc: adc {
+					compatible = "marvell,berlin2-adc";
+					interrupts = <12>, <14>;
+					interrupt-names = "adc", "tsen";
+				};
 			};
 
 			sic: interrupt-controller@e000 {
diff --git a/arch/arm/boot/dts/berlin2q-marvell-dmp.dts b/arch/arm/boot/dts/berlin2q-marvell-dmp.dts
index 57aa5f8a7c77..c162f98cb8e8 100644
--- a/arch/arm/boot/dts/berlin2q-marvell-dmp.dts
+++ b/arch/arm/boot/dts/berlin2q-marvell-dmp.dts
@@ -1,37 +1,6 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
 /*
  * Copyright (C) 2014 Antoine Ténart <antoine.tenart@free-electrons.com>
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is licensed under the terms of the GNU General Public
- *     License version 2. This program is licensed "as is" without any
- *     warranty of any kind, whether express or implied.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/berlin2q.dtsi b/arch/arm/boot/dts/berlin2q.dtsi
index bf3a6c9a1d34..516a7ce25791 100644
--- a/arch/arm/boot/dts/berlin2q.dtsi
+++ b/arch/arm/boot/dts/berlin2q.dtsi
@@ -1,37 +1,6 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
 /*
  * Copyright (C) 2014 Antoine Ténart <antoine.tenart@free-electrons.com>
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is licensed under the terms of the GNU General Public
- *     License version 2. This program is licensed "as is" without any
- *     warranty of any kind, whether express or implied.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #include <dt-bindings/clock/berlin2q.h>
@@ -53,7 +22,7 @@
 		#size-cells = <0>;
 		enable-method = "marvell,berlin-smp";
 
-		cpu@0 {
+		cpu0: cpu@0 {
 			compatible = "arm,cortex-a9";
 			device_type = "cpu";
 			next-level-cache = <&l2>;
@@ -71,21 +40,21 @@
 			>;
 		};
 
-		cpu@1 {
+		cpu1: cpu@1 {
 			compatible = "arm,cortex-a9";
 			device_type = "cpu";
 			next-level-cache = <&l2>;
 			reg = <1>;
 		};
 
-		cpu@2 {
+		cpu2: cpu@2 {
 			compatible = "arm,cortex-a9";
 			device_type = "cpu";
 			next-level-cache = <&l2>;
 			reg = <2>;
 		};
 
-		cpu@3 {
+		cpu3: cpu@3 {
 			compatible = "arm,cortex-a9";
 			device_type = "cpu";
 			next-level-cache = <&l2>;
@@ -93,6 +62,19 @@
 		};
 	};
 
+	pmu {
+		compatible = "arm,cortex-a9-pmu";
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>,
+				     <&cpu1>,
+				     <&cpu2>,
+				     <&cpu3>;
+	};
+
 	refclk: oscillator {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
@@ -107,14 +89,6 @@
 		ranges = <0 0xf7000000 0x1000000>;
 		interrupt-parent = <&gic>;
 
-		pmu {
-			compatible = "arm,cortex-a9-pmu";
-			interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
 		sdhci0: sdhci@ab0000 {
 			compatible = "mrvl,pxav3-mmc";
 			reg = <0xab0000 0x200>;
@@ -145,6 +119,7 @@
 		l2: l2-cache-controller@ac0000 {
 			compatible = "arm,pl310-cache";
 			reg = <0xac0000 0x1000>;
+			cache-unified;
 			cache-level = <2>;
 			arm,data-latency = <2 2 2>;
 			arm,tag-latency = <2 2 2>;
@@ -159,7 +134,7 @@
 			compatible = "arm,cortex-a9-twd-timer";
 			reg = <0xad0600 0x20>;
 			clocks = <&chip_clk CLKID_TWD>;
-			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_EDGE_RISING)>;
 		};
 
 		gic: interrupt-controller@ad1000 {
diff --git a/arch/arm/boot/dts/da850-evm.dts b/arch/arm/boot/dts/da850-evm.dts
index 3962fa4b07f5..0e82bb988fde 100644
--- a/arch/arm/boot/dts/da850-evm.dts
+++ b/arch/arm/boot/dts/da850-evm.dts
@@ -27,143 +27,6 @@
 		spi0 = &spi1;
 	};
 
-	soc@1c00000 {
-		pmx_core: pinmux@14120 {
-			status = "okay";
-
-			mcasp0_pins: pinmux_mcasp0_pins {
-				pinctrl-single,bits = <
-					/*
-					 * AHCLKX, ACLKX, AFSX, AHCLKR, ACLKR,
-					 * AFSR, AMUTE
-					 */
-					0x00 0x11111111 0xffffffff
-					/* AXR11, AXR12 */
-					0x04 0x00011000 0x000ff000
-				>;
-			};
-			nand_pins: nand_pins {
-				pinctrl-single,bits = <
-					/* EMA_WAIT[0], EMA_OE, EMA_WE, EMA_CS[4], EMA_CS[3] */
-					0x1c 0x10110110  0xf0ff0ff0
-					/*
-					 * EMA_D[0], EMA_D[1], EMA_D[2],
-					 * EMA_D[3], EMA_D[4], EMA_D[5],
-					 * EMA_D[6], EMA_D[7]
-					 */
-					0x24 0x11111111  0xffffffff
-					/* EMA_A[1], EMA_A[2] */
-					0x30 0x01100000  0x0ff00000
-				>;
-			};
-		};
-		serial0: serial@42000 {
-			status = "okay";
-		};
-		serial1: serial@10c000 {
-			status = "okay";
-		};
-		serial2: serial@10d000 {
-			status = "okay";
-		};
-		rtc0: rtc@23000 {
-			status = "okay";
-		};
-		i2c0: i2c@22000 {
-			status = "okay";
-			clock-frequency = <100000>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2c0_pins>;
-
-			tps: tps@48 {
-				reg = <0x48>;
-			};
-			tlv320aic3106: tlv320aic3106@18 {
-				#sound-dai-cells = <0>;
-				compatible = "ti,tlv320aic3106";
-				reg = <0x18>;
-				status = "okay";
-
-				/* Regulators */
-				IOVDD-supply = <&vdcdc2_reg>;
-				/* Derived from VBAT: Baseboard 3.3V / 1.8V */
-				AVDD-supply = <&vbat>;
-				DRVDD-supply = <&vbat>;
-				DVDD-supply = <&vbat>;
-			};
-			tca6416: gpio@20 {
-				compatible = "ti,tca6416";
-				reg = <0x20>;
-				gpio-controller;
-				#gpio-cells = <2>;
-			};
-		};
-		wdt: wdt@21000 {
-			status = "okay";
-		};
-		mmc0: mmc@40000 {
-			max-frequency = <50000000>;
-			bus-width = <4>;
-			status = "okay";
-			pinctrl-names = "default";
-			pinctrl-0 = <&mmc0_pins>;
-		};
-		spi1: spi@30e000 {
-			status = "okay";
-			pinctrl-names = "default";
-			pinctrl-0 = <&spi1_pins &spi1_cs0_pin>;
-			flash: m25p80@0 {
-				#address-cells = <1>;
-				#size-cells = <1>;
-				compatible = "m25p64";
-				spi-max-frequency = <30000000>;
-				m25p,fast-read;
-				reg = <0>;
-				partition@0 {
-					label = "U-Boot-SPL";
-					reg = <0x00000000 0x00010000>;
-					read-only;
-				};
-				partition@1 {
-					label = "U-Boot";
-					reg = <0x00010000 0x00080000>;
-					read-only;
-				};
-				partition@2 {
-					label = "U-Boot-Env";
-					reg = <0x00090000 0x00010000>;
-					read-only;
-				};
-				partition@3 {
-					label = "Kernel";
-					reg = <0x000a0000 0x00280000>;
-				};
-				partition@4 {
-					label = "Filesystem";
-					reg = <0x00320000 0x00400000>;
-				};
-				partition@5 {
-					label = "MAC-Address";
-					reg = <0x007f0000 0x00010000>;
-					read-only;
-				};
-			};
-		};
-		mdio: mdio@224000 {
-			status = "okay";
-			pinctrl-names = "default";
-			pinctrl-0 = <&mdio_pins>;
-			bus_freq = <2200000>;
-		};
-		eth0: ethernet@220000 {
-			status = "okay";
-			pinctrl-names = "default";
-			pinctrl-0 = <&mii_pins>;
-		};
-		gpio: gpio@226000 {
-			status = "okay";
-		};
-	};
 	vbat: fixedregulator0 {
 		compatible = "regulator-fixed";
 		regulator-name = "vbat";
@@ -200,6 +63,155 @@
 	};
 };
 
+&pmx_core {
+	status = "okay";
+
+	mcasp0_pins: pinmux_mcasp0_pins {
+		pinctrl-single,bits = <
+			/*
+			 * AHCLKX, ACLKX, AFSX, AHCLKR, ACLKR,
+			 * AFSR, AMUTE
+			 */
+			0x00 0x11111111 0xffffffff
+			/* AXR11, AXR12 */
+			0x04 0x00011000 0x000ff000
+		>;
+	};
+	nand_pins: nand_pins {
+		pinctrl-single,bits = <
+			/* EMA_WAIT[0], EMA_OE, EMA_WE, EMA_CS[4], EMA_CS[3] */
+			0x1c 0x10110110  0xf0ff0ff0
+			/*
+			 * EMA_D[0], EMA_D[1], EMA_D[2],
+			 * EMA_D[3], EMA_D[4], EMA_D[5],
+			 * EMA_D[6], EMA_D[7]
+			 */
+			0x24 0x11111111  0xffffffff
+			/* EMA_A[1], EMA_A[2] */
+			0x30 0x01100000  0x0ff00000
+		>;
+	};
+};
+
+&serial0 {
+	status = "okay";
+};
+
+&serial1 {
+	status = "okay";
+};
+
+&serial2 {
+	status = "okay";
+};
+
+&rtc0 {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins>;
+
+	tps: tps@48 {
+		reg = <0x48>;
+	};
+	tlv320aic3106: tlv320aic3106@18 {
+		#sound-dai-cells = <0>;
+		compatible = "ti,tlv320aic3106";
+		reg = <0x18>;
+		status = "okay";
+
+		/* Regulators */
+		IOVDD-supply = <&vdcdc2_reg>;
+		/* Derived from VBAT: Baseboard 3.3V / 1.8V */
+		AVDD-supply = <&vbat>;
+		DRVDD-supply = <&vbat>;
+		DVDD-supply = <&vbat>;
+	};
+	tca6416: gpio@20 {
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
+
+&wdt {
+	status = "okay";
+};
+
+&mmc0 {
+	max-frequency = <50000000>;
+	bus-width = <4>;
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc0_pins>;
+	cd-gpios = <&gpio 64 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&gpio 65 GPIO_ACTIVE_HIGH>;
+};
+
+&spi1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi1_pins &spi1_cs0_pin>;
+	flash: m25p80@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "m25p64";
+		spi-max-frequency = <30000000>;
+		m25p,fast-read;
+		reg = <0>;
+		partition@0 {
+			label = "U-Boot-SPL";
+			reg = <0x00000000 0x00010000>;
+			read-only;
+		};
+		partition@1 {
+			label = "U-Boot";
+			reg = <0x00010000 0x00080000>;
+			read-only;
+		};
+		partition@2 {
+			label = "U-Boot-Env";
+			reg = <0x00090000 0x00010000>;
+			read-only;
+		};
+		partition@3 {
+			label = "Kernel";
+			reg = <0x000a0000 0x00280000>;
+		};
+		partition@4 {
+			label = "Filesystem";
+			reg = <0x00320000 0x00400000>;
+		};
+		partition@5 {
+			label = "MAC-Address";
+			reg = <0x007f0000 0x00010000>;
+			read-only;
+		};
+	};
+};
+
+&mdio {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&mdio_pins>;
+	bus_freq = <2200000>;
+};
+
+&eth0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&mii_pins>;
+};
+
+&gpio {
+	status = "okay";
+};
+
 /include/ "tps6507x.dtsi"
 
 &tps {
@@ -309,6 +321,18 @@
 	};
 };
 
+&usb_phy {
+	status = "okay";
+};
+
+&usb0 {
+	status = "okay";
+};
+
+&usb1 {
+	status = "okay";
+};
+
 &vpif {
 	pinctrl-names = "default";
 	pinctrl-0 = <&vpif_capture_pins>, <&vpif_display_pins>;
diff --git a/arch/arm/boot/dts/da850-lego-ev3.dts b/arch/arm/boot/dts/da850-lego-ev3.dts
index 1ffd87796cac..ee3932475ce7 100644
--- a/arch/arm/boot/dts/da850-lego-ev3.dts
+++ b/arch/arm/boot/dts/da850-lego-ev3.dts
@@ -33,11 +33,9 @@
 	 */
 	gpio_keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
 		label = "EV3 Brick Buttons";
 		pinctrl-names = "default";
-		pinctrl-0 = <&button_pins>, <&button_bias>;
+		pinctrl-0 = <&button_bias>;
 
 		center {
 			label = "Center";
@@ -81,8 +79,6 @@
 	 */
 	leds {
 		compatible = "gpio-leds";
-		pinctrl-names = "default";
-		pinctrl-0 = <&led_pins>;
 
 		left_green {
 			label = "led0:green:brick-status";
@@ -119,8 +115,6 @@
 	gpio-poweroff {
 		compatible = "gpio-poweroff";
 		gpios = <&gpio 107 GPIO_ACTIVE_LOW>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&system_power_pin>;
 	};
 
 	sound {
@@ -136,8 +130,6 @@
 	 * the sensor (input) ports, the motor (output) ports and the A/DC.
 	 */
 	vcc5v: regulator1 {
-		pinctrl-names = "default";
-		pinctrl-0 = <&vcc5v_pins>;
 		compatible = "regulator-fixed";
 		regulator-name = "vcc5v";
 		regulator-min-microvolt = <5000000>;
@@ -165,8 +157,6 @@
 	 * This is the amplifier for the speaker.
 	 */
 	amp: regulator3 {
-		pinctrl-names = "default";
-		pinctrl-0 = <&amp_pins>;
 		compatible = "regulator-fixed";
 		regulator-name = "amp";
 		gpio = <&gpio 111 GPIO_ACTIVE_HIGH>;
@@ -177,8 +167,6 @@
 	 * The EV3 can use 6-AA batteries or a rechargeable Li-ion battery pack.
 	 */
 	battery {
-		pinctrl-names = "default";
-		pinctrl-0 = <&battery_pins>;
 		compatible = "lego,ev3-battery";
 		io-channels = <&adc 4>, <&adc 3>;
 		io-channel-names = "voltage", "current";
@@ -206,73 +194,10 @@
 &pmx_core {
 	status = "okay";
 
-	mmc0_cd_pin: pinmux_mmc0_cd {
-		pinctrl-single,bits = <
-			/* GP5[14] */
-			0x2C 0x00000080 0x000000f0
-		>;
-	};
-
-	button_pins: pinmux_button_pins {
-		pinctrl-single,bits = <
-			/* GP1[13] */
-			0x8 0x00000800 0x00000f00
-			/* GP6[10] */
-			0x34 0x00800000 0x00f00000
-			/* GP6[6] */
-			0x38 0x00000080 0x000000f0
-			/* GP7[12], GP7[14], GP7[15] */
-			0x40 0x00808800 0x00f0ff00
-		>;
-	};
-
-	led_pins: pinmux_led_pins {
-		pinctrl-single,bits = <
-			/* GP6[12], GP6[13], GP6[14] */
-			0x34 0x00008880 0x0000fff0
-			/* GP6[7] */
-			0x38 0x00000008 0x0000000f
-		>;
-	};
-
-	system_power_pin: pinmux_system_power {
-		pinctrl-single,bits = <
-			/* GP6[11] */
-			0x34 0x00080000 0x000f0000
-		>;
-	};
-
-	vcc5v_pins: pinmux_vcc5v {
-		pinctrl-single,bits = <
-			/* GP6[5] */
-			0x40 0x00000080 0x000000f0
-			/* GP6[3] */
-			0x4c 0x00008000 0x0000f000
-		>;
-	};
-
-	amp_pins: pinmux_amp_pins {
-		pinctrl-single,bits = <
-			/* GP6[15] */
-			0x34 0x00000008 0x0000000f
-		>;
-	};
-
-	battery_pins: pinmux_battery_pins {
-		pinctrl-single,bits = <
-			/* GP0[6] */
-			0x04 0x00000080 0x000000f0
-			/* GP8[8] */
-			0x4c 0x00000080 0x000000f0
-		>;
-	};
-
 	ev3_lcd_pins: pinmux_lcd {
 		pinctrl-single,bits = <
-			/* SIMO, GP2[11], GP2[12], CLK */
-			0x14 0x00188100 0x00ffff00
-			/* GP5[0] */
-			0x30 0x80000000 0xf0000000
+			/* SIMO, CLK */
+			0x14 0x00100100 0x00f00f00
 		>;
 	};
 };
@@ -327,7 +252,7 @@
 	bus-width = <4>;
 	cd-gpios = <&gpio 94 GPIO_ACTIVE_LOW>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&mmc0_pins>, <&mmc0_cd_pin>;
+	pinctrl-0 = <&mmc0_pins>;
 };
 
 &spi0 {
diff --git a/arch/arm/boot/dts/da850.dtsi b/arch/arm/boot/dts/da850.dtsi
index 12010002dbdb..f6f1597b03df 100644
--- a/arch/arm/boot/dts/da850.dtsi
+++ b/arch/arm/boot/dts/da850.dtsi
@@ -59,8 +59,18 @@
 			pinctrl-single,bit-per-mux;
 			pinctrl-single,register-width = <32>;
 			pinctrl-single,function-mask = <0xf>;
+			/* pin base, nr pins & gpio function */
+			pinctrl-single,gpio-range = <&range   0 17 0x8>,
+						    <&range  17  8 0x4>,
+						    <&range  26  8 0x4>,
+						    <&range  34 80 0x8>,
+						    <&range 129 31 0x8>;
 			status = "disabled";
 
+			range: gpio-range {
+				#pinctrl-single,gpio-range-cells = <3>;
+			};
+
 			serial0_rtscts_pins: pinmux_serial0_rtscts_pins {
 				pinctrl-single,bits = <
 					/* UART0_RTS UART0_CTS */
@@ -549,6 +559,150 @@
 			status = "disabled";
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <&pmx_core   0  15 1>,
+				      <&pmx_core   1  14 1>,
+				      <&pmx_core   2  13 1>,
+				      <&pmx_core   3  12 1>,
+				      <&pmx_core   4  11 1>,
+				      <&pmx_core   5  10 1>,
+				      <&pmx_core   6   9 1>,
+				      <&pmx_core   7   8 1>,
+				      <&pmx_core   8   7 1>,
+				      <&pmx_core   9   6 1>,
+				      <&pmx_core  10   5 1>,
+				      <&pmx_core  11   4 1>,
+				      <&pmx_core  12   3 1>,
+				      <&pmx_core  13   2 1>,
+				      <&pmx_core  14   1 1>,
+				      <&pmx_core  15   0 1>,
+				      <&pmx_core  16  39 1>,
+				      <&pmx_core  17  38 1>,
+				      <&pmx_core  18  37 1>,
+				      <&pmx_core  19  36 1>,
+				      <&pmx_core  20  35 1>,
+				      <&pmx_core  21  34 1>,
+				      <&pmx_core  22  33 1>,
+				      <&pmx_core  23  32 1>,
+				      <&pmx_core  24  24 1>,
+				      <&pmx_core  25  22 1>,
+				      <&pmx_core  26  21 1>,
+				      <&pmx_core  27  20 1>,
+				      <&pmx_core  28  19 1>,
+				      <&pmx_core  29  18 1>,
+				      <&pmx_core  30  17 1>,
+				      <&pmx_core  31  16 1>,
+				      <&pmx_core  32  55 1>,
+				      <&pmx_core  33  54 1>,
+				      <&pmx_core  34  53 1>,
+				      <&pmx_core  35  52 1>,
+				      <&pmx_core  36  51 1>,
+				      <&pmx_core  37  50 1>,
+				      <&pmx_core  38  49 1>,
+				      <&pmx_core  39  48 1>,
+				      <&pmx_core  40  47 1>,
+				      <&pmx_core  41  46 1>,
+				      <&pmx_core  42  45 1>,
+				      <&pmx_core  43  44 1>,
+				      <&pmx_core  44  43 1>,
+				      <&pmx_core  45  42 1>,
+				      <&pmx_core  46  41 1>,
+				      <&pmx_core  47  40 1>,
+				      <&pmx_core  48  71 1>,
+				      <&pmx_core  49  70 1>,
+				      <&pmx_core  50  69 1>,
+				      <&pmx_core  51  68 1>,
+				      <&pmx_core  52  67 1>,
+				      <&pmx_core  53  66 1>,
+				      <&pmx_core  54  65 1>,
+				      <&pmx_core  55  64 1>,
+				      <&pmx_core  56  63 1>,
+				      <&pmx_core  57  62 1>,
+				      <&pmx_core  58  61 1>,
+				      <&pmx_core  59  60 1>,
+				      <&pmx_core  60  59 1>,
+				      <&pmx_core  61  58 1>,
+				      <&pmx_core  62  57 1>,
+				      <&pmx_core  63  56 1>,
+				      <&pmx_core  64  87 1>,
+				      <&pmx_core  65  86 1>,
+				      <&pmx_core  66  85 1>,
+				      <&pmx_core  67  84 1>,
+				      <&pmx_core  68  83 1>,
+				      <&pmx_core  69  82 1>,
+				      <&pmx_core  70  81 1>,
+				      <&pmx_core  71  80 1>,
+				      <&pmx_core  72  70 1>,
+				      <&pmx_core  73  78 1>,
+				      <&pmx_core  74  77 1>,
+				      <&pmx_core  75  76 1>,
+				      <&pmx_core  76  75 1>,
+				      <&pmx_core  77  74 1>,
+				      <&pmx_core  78  73 1>,
+				      <&pmx_core  79  72 1>,
+				      <&pmx_core  80 103 1>,
+				      <&pmx_core  81 102 1>,
+				      <&pmx_core  82 101 1>,
+				      <&pmx_core  83 100 1>,
+				      <&pmx_core  84  99 1>,
+				      <&pmx_core  85  98 1>,
+				      <&pmx_core  86  97 1>,
+				      <&pmx_core  87  96 1>,
+				      <&pmx_core  88  95 1>,
+				      <&pmx_core  89  94 1>,
+				      <&pmx_core  90  93 1>,
+				      <&pmx_core  91  92 1>,
+				      <&pmx_core  92  91 1>,
+				      <&pmx_core  93  90 1>,
+				      <&pmx_core  94  89 1>,
+				      <&pmx_core  95  88 1>,
+				      <&pmx_core  96 158 1>,
+				      <&pmx_core  97 157 1>,
+				      <&pmx_core  98 156 1>,
+				      <&pmx_core  99 155 1>,
+				      <&pmx_core 100 154 1>,
+				      <&pmx_core 101 129 1>,
+				      <&pmx_core 102 113 1>,
+				      <&pmx_core 103 112 1>,
+				      <&pmx_core 104 111 1>,
+				      <&pmx_core 105 110 1>,
+				      <&pmx_core 106 109 1>,
+				      <&pmx_core 107 108 1>,
+				      <&pmx_core 108 107 1>,
+				      <&pmx_core 109 106 1>,
+				      <&pmx_core 110 105 1>,
+				      <&pmx_core 111 104 1>,
+				      <&pmx_core 112 145 1>,
+				      <&pmx_core 113 144 1>,
+				      <&pmx_core 114 143 1>,
+				      <&pmx_core 115 142 1>,
+				      <&pmx_core 116 141 1>,
+				      <&pmx_core 117 140 1>,
+				      <&pmx_core 118 139 1>,
+				      <&pmx_core 119 138 1>,
+				      <&pmx_core 120 137 1>,
+				      <&pmx_core 121 136 1>,
+				      <&pmx_core 122 135 1>,
+				      <&pmx_core 123 134 1>,
+				      <&pmx_core 124 133 1>,
+				      <&pmx_core 125 132 1>,
+				      <&pmx_core 126 131 1>,
+				      <&pmx_core 127 130 1>,
+				      <&pmx_core 128 159 1>,
+				      <&pmx_core 129  31 1>,
+				      <&pmx_core 130  30 1>,
+				      <&pmx_core 131  20 1>,
+				      <&pmx_core 132  28 1>,
+				      <&pmx_core 133  27 1>,
+				      <&pmx_core 134  26 1>,
+				      <&pmx_core 135  23 1>,
+				      <&pmx_core 136 153 1>,
+				      <&pmx_core 137 152 1>,
+				      <&pmx_core 138 151 1>,
+				      <&pmx_core 139 150 1>,
+				      <&pmx_core 140 149 1>,
+				      <&pmx_core 141 148 1>,
+				      <&pmx_core 142 147 1>,
+				      <&pmx_core 143 146 1>;
 		};
 		pinconf: pin-controller@22c00c {
 			compatible = "ti,da850-pupd";
diff --git a/arch/arm/boot/dts/dm8148-t410.dts b/arch/arm/boot/dts/dm8148-t410.dts
index 6418f9cdbe83..c46a227b543d 100644
--- a/arch/arm/boot/dts/dm8148-t410.dts
+++ b/arch/arm/boot/dts/dm8148-t410.dts
@@ -77,7 +77,7 @@
 			DM814X_IOPAD(0x09dc, PIN_INPUT_PULLUP | 0x1)	/* SD2_DAT[0] */
 			DM814X_IOPAD(0x09e0, PIN_INPUT | 0x1)		/* SD2_CLK */
 			DM814X_IOPAD(0x09f4, PIN_INPUT_PULLUP | 0x2)	/* SD2_CMD */
-			DM814X_IOPAD(0x0920, PIN_INPUT | 40)	/* SD2_SDCD */
+			DM814X_IOPAD(0x0920, PIN_INPUT | 0x40)	/* SD2_SDCD */
 			>;
 	};
 
diff --git a/arch/arm/boot/dts/dra7-evm-common.dtsi b/arch/arm/boot/dts/dra7-evm-common.dtsi
index 05a7b1a01bc3..33230c8b2951 100644
--- a/arch/arm/boot/dts/dra7-evm-common.dtsi
+++ b/arch/arm/boot/dts/dra7-evm-common.dtsi
@@ -260,3 +260,18 @@
 &pcie1_rc {
 	status = "okay";
 };
+
+&mmc4 {
+	bus-width = <4>;
+	cap-power-off-card;
+	keep-power-in-suspend;
+	non-removable;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	wifi@2 {
+		compatible = "ti,wl1835";
+		reg = <2>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <7 IRQ_TYPE_EDGE_RISING>;
+	};
+};
diff --git a/arch/arm/boot/dts/dra7-evm.dts b/arch/arm/boot/dts/dra7-evm.dts
index f1425b0f3a54..0894593860d6 100644
--- a/arch/arm/boot/dts/dra7-evm.dts
+++ b/arch/arm/boot/dts/dra7-evm.dts
@@ -20,6 +20,16 @@
 		reg = <0x0 0x80000000 0x0 0x60000000>; /* 1536 MB */
 	};
 
+	evm_12v0: fixedregulator-evm_12v0 {
+		/* main supply */
+		compatible = "regulator-fixed";
+		regulator-name = "evm_12v0";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
 	evm_1v8_sw: fixedregulator-evm_1v8 {
 		compatible = "regulator-fixed";
 		regulator-name = "evm_1v8";
@@ -54,6 +64,48 @@
 		regulator-max-microvolt = <1800000>;
 	};
 
+	evm_3v3: fixedregulator-evm3v3 {
+		/* Output of Cntlr A of TPS43351-Q1 on dra7-evm */
+		compatible = "regulator-fixed";
+		regulator-name = "evm_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&evm_12v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	evm_5v0: fixedregulator-evm_5v0 {
+		/* Output of Cntlr B of TPS43351-Q1 on dra7-evm */
+		compatible = "regulator-fixed";
+		regulator-name = "evm_5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&evm_12v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	evm_3v6: fixedregulator-evm_3v6 {
+		compatible = "regulator-fixed";
+		regulator-name = "evm_3v6";
+		regulator-min-microvolt = <3600000>;
+		regulator-max-microvolt = <3600000>;
+		vin-supply = <&evm_5v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vmmcwl_fixed: fixedregulator-mmcwl {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmcwl_fixed";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		gpio = <&gpio5 8 0>;
+		startup-delay-us = <70000>;
+		enable-active-high;
+	};
+
 	extcon_usb2: extcon_usb2 {
 		compatible = "linux,extcon-usb-gpio";
 		id-gpio = <&pcf_gpio_21 2 GPIO_ACTIVE_HIGH>;
@@ -325,6 +377,7 @@
 	vmmc-supply = <&evm_1v8_sw>;
 	vqmmc-supply = <&evm_1v8_sw>;
 	bus-width = <8>;
+	non-removable;
 	pinctrl-names = "default", "hs", "ddr_1_8v-rev11", "ddr_1_8v", "hs200_1_8v-rev11", "hs200_1_8v";
 	pinctrl-0 = <&mmc2_pins_default>;
 	pinctrl-1 = <&mmc2_pins_hs>;
@@ -334,6 +387,21 @@
 	pinctrl-5 = <&mmc2_pins_hs200 &mmc2_iodelay_hs200_rev20_conf>;
 };
 
+&mmc4 {
+	status = "okay";
+	vmmc-supply = <&evm_3v6>;
+	vqmmc-supply = <&vmmcwl_fixed>;
+	pinctrl-names = "default-rev11", "default", "hs-rev11", "hs", "sdr12-rev11", "sdr12", "sdr25-rev11", "sdr25";
+	pinctrl-0 = <&mmc4_pins_default &mmc4_iodelay_ds_rev11_conf>;
+	pinctrl-1 = <&mmc4_pins_default &mmc4_iodelay_ds_rev20_conf>;
+	pinctrl-2 = <&mmc4_pins_hs &mmc4_iodelay_sdr12_hs_sdr25_rev11_conf>;
+	pinctrl-3 = <&mmc4_pins_hs &mmc4_iodelay_sdr12_hs_sdr25_rev20_conf>;
+	pinctrl-4 = <&mmc4_pins_sdr12 &mmc4_iodelay_sdr12_hs_sdr25_rev11_conf>;
+	pinctrl-5 = <&mmc4_pins_sdr12 &mmc4_iodelay_sdr12_hs_sdr25_rev20_conf>;
+	pinctrl-6 = <&mmc4_pins_sdr25 &mmc4_iodelay_sdr12_hs_sdr25_rev11_conf>;
+	pinctrl-7 = <&mmc4_pins_sdr25 &mmc4_iodelay_sdr12_hs_sdr25_rev20_conf>;
+};
+
 &cpu0 {
 	vdd-supply = <&smps123_reg>;
 };
diff --git a/arch/arm/boot/dts/dra7-mmc-iodelay.dtsi b/arch/arm/boot/dts/dra7-mmc-iodelay.dtsi
new file mode 100644
index 000000000000..aa0947266526
--- /dev/null
+++ b/arch/arm/boot/dts/dra7-mmc-iodelay.dtsi
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * MMC IOdelay values for TI's DRA7xx SoCs.
+ * Copyright (C) 2018 Texas Instruments
+ * Author: Kishon Vijay Abraham I <kishon@ti.com>
+ */
+
+&dra7_pmx_core {
+	mmc1_pins_default_no_clk_pu: mmc1_pins_default_no_clk_pu {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x3754, PIN_INPUT_PULLDOWN | MUX_MODE0) /* mmc1_clk.clk */
+			DRA7XX_CORE_IOPAD(0x3758, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_cmd.cmd */
+			DRA7XX_CORE_IOPAD(0x375c, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat0.dat0 */
+			DRA7XX_CORE_IOPAD(0x3760, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat1.dat1 */
+			DRA7XX_CORE_IOPAD(0x3764, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat2.dat2 */
+			DRA7XX_CORE_IOPAD(0x3768, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat3.dat3 */
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi
index f4ddd86f2c77..93fc2d79d337 100644
--- a/arch/arm/boot/dts/dra7.dtsi
+++ b/arch/arm/boot/dts/dra7.dtsi
@@ -1079,17 +1079,15 @@
 		};
 
 		mmc1: mmc@4809c000 {
-			compatible = "ti,omap4-hsmmc";
+			compatible = "ti,dra7-sdhci";
 			reg = <0x4809c000 0x400>;
 			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "mmc1";
-			ti,dual-volt;
-			ti,needs-special-reset;
-			dmas = <&sdma_xbar 61>, <&sdma_xbar 62>;
-			dma-names = "tx", "rx";
 			status = "disabled";
 			pbias-supply = <&pbias_mmc_reg>;
 			max-frequency = <192000000>;
+			mmc-ddr-1_8v;
+			mmc-ddr-3_3v;
 		};
 
 		hdqw1w: 1w@480b2000 {
@@ -1100,40 +1098,40 @@
 		};
 
 		mmc2: mmc@480b4000 {
-			compatible = "ti,omap4-hsmmc";
+			compatible = "ti,dra7-sdhci";
 			reg = <0x480b4000 0x400>;
 			interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "mmc2";
-			ti,needs-special-reset;
-			dmas = <&sdma_xbar 47>, <&sdma_xbar 48>;
-			dma-names = "tx", "rx";
 			status = "disabled";
 			max-frequency = <192000000>;
+			/* SDR104/DDR50/SDR50 bits in CAPA2 is not supported */
+			sdhci-caps-mask = <0x7 0x0>;
+			mmc-hs200-1_8v;
+			mmc-ddr-1_8v;
+			mmc-ddr-3_3v;
 		};
 
 		mmc3: mmc@480ad000 {
-			compatible = "ti,omap4-hsmmc";
+			compatible = "ti,dra7-sdhci";
 			reg = <0x480ad000 0x400>;
 			interrupts = <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "mmc3";
-			ti,needs-special-reset;
-			dmas = <&sdma_xbar 77>, <&sdma_xbar 78>;
-			dma-names = "tx", "rx";
 			status = "disabled";
 			/* Errata i887 limits max-frequency of MMC3 to 64 MHz */
 			max-frequency = <64000000>;
+			/* SDMA is not supported */
+			sdhci-caps-mask = <0x0 0x400000>;
 		};
 
 		mmc4: mmc@480d1000 {
-			compatible = "ti,omap4-hsmmc";
+			compatible = "ti,dra7-sdhci";
 			reg = <0x480d1000 0x400>;
 			interrupts = <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "mmc4";
-			ti,needs-special-reset;
-			dmas = <&sdma_xbar 57>, <&sdma_xbar 58>;
-			dma-names = "tx", "rx";
 			status = "disabled";
 			max-frequency = <192000000>;
+			/* SDMA is not supported */
+			sdhci-caps-mask = <0x0 0x400000>;
 		};
 
 		mmu0_dsp1: mmu@40d01000 {
@@ -1895,7 +1893,7 @@
 			};
 		};
 
-		dcan1: can@481cc000 {
+		dcan1: can@4ae3c000 {
 			compatible = "ti,dra7-d_can";
 			ti,hwmods = "dcan1";
 			reg = <0x4ae3c000 0x2000>;
@@ -1905,7 +1903,7 @@
 			status = "disabled";
 		};
 
-		dcan2: can@481d0000 {
+		dcan2: can@48480000 {
 			compatible = "ti,dra7-d_can";
 			ti,hwmods = "dcan2";
 			reg = <0x48480000 0x2000>;
diff --git a/arch/arm/boot/dts/dra71-evm.dts b/arch/arm/boot/dts/dra71-evm.dts
index ebc4bbae981e..b7aeaeeead3b 100644
--- a/arch/arm/boot/dts/dra71-evm.dts
+++ b/arch/arm/boot/dts/dra71-evm.dts
@@ -7,6 +7,7 @@
  */
 
 #include "dra72-evm-common.dtsi"
+#include "dra7-mmc-iodelay.dtsi"
 #include "dra72x-mmc-iodelay.dtsi"
 #include <dt-bindings/net/ti-dp83867.h>
 
@@ -50,19 +51,6 @@
 	};
 };
 
-&dra7_pmx_core {
-	mmc1_pins_default: mmc1_pins_default {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x3754, PIN_INPUT_PULLDOWN | MUX_MODE0) /* mmc1_clk.clk */
-			DRA7XX_CORE_IOPAD(0x3758, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_cmd.cmd */
-			DRA7XX_CORE_IOPAD(0x375c, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat0.dat0 */
-			DRA7XX_CORE_IOPAD(0x3760, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat1.dat1 */
-			DRA7XX_CORE_IOPAD(0x3764, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat2.dat2 */
-			DRA7XX_CORE_IOPAD(0x3768, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat3.dat3 */
-		>;
-	};
-};
-
 &i2c1 {
 	status = "okay";
 	clock-frequency = <400000>;
@@ -187,7 +175,7 @@
 
 &mmc1 {
 	pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50", "ddr50", "sdr104";
-	pinctrl-0 = <&mmc1_pins_default>;
+	pinctrl-0 = <&mmc1_pins_default_no_clk_pu>;
 	pinctrl-1 = <&mmc1_pins_hs>;
 	pinctrl-2 = <&mmc1_pins_sdr12>;
 	pinctrl-3 = <&mmc1_pins_sdr25>;
@@ -204,6 +192,7 @@
 	pinctrl-2 = <&mmc2_pins_ddr_rev20 &mmc2_iodelay_ddr_conf>;
 	pinctrl-3 = <&mmc2_pins_hs200 &mmc2_iodelay_hs200_rev20_conf>;
 	vmmc-supply = <&evm_1v8_sw>;
+	vqmmc-supply = <&evm_1v8_sw>;
 };
 
 &mac {
diff --git a/arch/arm/boot/dts/dra72-evm-common.dtsi b/arch/arm/boot/dts/dra72-evm-common.dtsi
index e85f560a2f78..df174f5c15d1 100644
--- a/arch/arm/boot/dts/dra72-evm-common.dtsi
+++ b/arch/arm/boot/dts/dra72-evm-common.dtsi
@@ -44,6 +44,16 @@
 		regulator-boot-on;
 	};
 
+	evm_3v6: fixedregulator-evm_3v6 {
+		compatible = "regulator-fixed";
+		regulator-name = "evm_3v6";
+		regulator-min-microvolt = <3600000>;
+		regulator-max-microvolt = <3600000>;
+		vin-supply = <&evm_5v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
 	vsys_3v3: fixedregulator-vsys3v3 {
 		/* Output 2 of TPS43351QDAPRQ1 on dra72-evm */
 		/* Output 2 of LM5140QRWGTQ1 on dra71-evm */
@@ -171,36 +181,18 @@
 			clocks = <&atl_clkin2_ck>;
 		};
 	};
-};
-
-&dra7_pmx_core {
-	mmc1_pins_default: mmc1_pins_default {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x376c, PIN_INPUT | MUX_MODE14)	/* mmc1sdcd.gpio219 */
-			DRA7XX_CORE_IOPAD(0x3754, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_clk.clk */
-			DRA7XX_CORE_IOPAD(0x3758, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_cmd.cmd */
-			DRA7XX_CORE_IOPAD(0x375c, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat0.dat0 */
-			DRA7XX_CORE_IOPAD(0x3760, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat1.dat1 */
-			DRA7XX_CORE_IOPAD(0x3764, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat2.dat2 */
-			DRA7XX_CORE_IOPAD(0x3768, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc1_dat3.dat3 */
-		>;
-	};
 
-	mmc2_pins_default: mmc2_pins_default {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x349c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_a23.mmc2_clk */
-			DRA7XX_CORE_IOPAD(0x34b0, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_cs1.mmc2_cmd */
-			DRA7XX_CORE_IOPAD(0x34a0, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_a24.mmc2_dat0 */
-			DRA7XX_CORE_IOPAD(0x34a4, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_a25.mmc2_dat1 */
-			DRA7XX_CORE_IOPAD(0x34a8, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_a26.mmc2_dat2 */
-			DRA7XX_CORE_IOPAD(0x34ac, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_a27.mmc2_dat3 */
-			DRA7XX_CORE_IOPAD(0x348c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_a19.mmc2_dat4 */
-			DRA7XX_CORE_IOPAD(0x3490, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_a20.mmc2_dat5 */
-			DRA7XX_CORE_IOPAD(0x3494, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_a21.mmc2_dat6 */
-			DRA7XX_CORE_IOPAD(0x3498, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_a22.mmc2_dat7 */
-		>;
+	vmmcwl_fixed: fixedregulator-mmcwl {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmcwl_fixed";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		gpio = <&gpio5 8 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
 	};
+};
 
+&dra7_pmx_core {
 	dcan1_pins_default: dcan1_pins_default {
 		pinctrl-single,pins = <
 			DRA7XX_CORE_IOPAD(0x37d0, PIN_OUTPUT_PULLUP | MUX_MODE0) /* dcan1_tx */
@@ -421,10 +413,33 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc2_pins_default>;
 	bus-width = <8>;
-	ti,non-removable;
+	non-removable;
 	max-frequency = <192000000>;
 };
 
+&mmc4 {
+	status = "okay";
+	vmmc-supply = <&evm_3v6>;
+	vqmmc-supply = <&vmmcwl_fixed>;
+	bus-width = <4>;
+	cap-power-off-card;
+	keep-power-in-suspend;
+	non-removable;
+	pinctrl-names = "default", "hs", "sdr12", "sdr25";
+	pinctrl-0 = <&mmc4_pins_default>;
+	pinctrl-1 = <&mmc4_pins_default>;
+	pinctrl-2 = <&mmc4_pins_default>;
+	pinctrl-3 = <&mmc4_pins_default>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	wifi@2 {
+		compatible = "ti,wl1835";
+		reg = <2>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <7 IRQ_TYPE_EDGE_RISING>;
+	};
+};
+
 &mac {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/dra72x-mmc-iodelay.dtsi b/arch/arm/boot/dts/dra72x-mmc-iodelay.dtsi
index 088013c6dc6e..edad87c4292c 100644
--- a/arch/arm/boot/dts/dra72x-mmc-iodelay.dtsi
+++ b/arch/arm/boot/dts/dra72x-mmc-iodelay.dtsi
@@ -202,6 +202,17 @@
 			DRA7XX_CORE_IOPAD(0x3498, PIN_INPUT_PULLUP | MODE_SELECT | MUX_MODE1) /* gpmc_a22.mmc2_dat7 */
 		>;
 	};
+
+	mmc4_pins_default: mmc4_pins_default {
+		pinctrl-single,pins = <
+			DRA7XX_CORE_IOPAD(0x37e8, PIN_INPUT_PULLUP | MUX_MODE3) /* uart1_ctsn.mmc4_clk */
+			DRA7XX_CORE_IOPAD(0x37ec, PIN_INPUT_PULLUP | MUX_MODE3) /* uart1_rtsn.mmc4_cmd */
+			DRA7XX_CORE_IOPAD(0x37f0, PIN_INPUT_PULLUP | MUX_MODE3) /* uart2_rxd.mmc4_dat0 */
+			DRA7XX_CORE_IOPAD(0x37f4, PIN_INPUT_PULLUP | MUX_MODE3) /* uart2_txd.mmc4_dat1 */
+			DRA7XX_CORE_IOPAD(0x37f8, PIN_INPUT_PULLUP | MUX_MODE3) /* uart2_ctsn.mmc4_dat2 */
+			DRA7XX_CORE_IOPAD(0x37fc, PIN_INPUT_PULLUP | MUX_MODE3) /* uart2_rtsn.mmc4_dat3 */
+		>;
+	};
 };
 
 &dra7_iodelay_core {
diff --git a/arch/arm/boot/dts/dra76-evm.dts b/arch/arm/boot/dts/dra76-evm.dts
index 2deb96405d06..c07f0051844d 100644
--- a/arch/arm/boot/dts/dra76-evm.dts
+++ b/arch/arm/boot/dts/dra76-evm.dts
@@ -42,6 +42,16 @@
 		regulator-boot-on;
 	};
 
+	vio_3v6: fixedregulator-vio_3v6 {
+		compatible = "regulator-fixed";
+		regulator-name = "vio_3v6";
+		regulator-min-microvolt = <3600000>;
+		regulator-max-microvolt = <3600000>;
+		vin-supply = <&vsys_5v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
 	vsys_3v3: fixedregulator-vsys3v3 {
 		/* Output of Cntlr A of TPS43351-Q1 on dra76-evm */
 		compatible = "regulator-fixed";
@@ -81,6 +91,16 @@
 		vin-supply = <&smps5_reg>;
 	};
 
+	vmmcwl_fixed: fixedregulator-mmcwl {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmcwl_fixed";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		gpio = <&gpio5 8 0>;	/* gpio5_8 */
+		startup-delay-us = <70000>;
+		enable-active-high;
+	};
+
 	vtt_fixed: fixedregulator-vtt {
 		compatible = "regulator-fixed";
 		regulator-name = "vtt_fixed";
@@ -307,7 +327,7 @@
 &mmc1 {
 	status = "okay";
 	vmmc-supply = <&vio_3v3_sd>;
-	vmmc_aux-supply = <&ldo4_reg>;
+	vqmmc-supply = <&ldo4_reg>;
 	bus-width = <4>;
 	/*
 	 * SDCD signal is not being used here - using the fact that GPIO mode
@@ -324,6 +344,7 @@
 	vmmc-supply = <&vio_1v8>;
 	vqmmc-supply = <&vio_1v8>;
 	bus-width = <8>;
+	non-removable;
 	pinctrl-names = "default", "hs", "ddr_1_8v", "hs200_1_8v";
 	pinctrl-0 = <&mmc2_pins_default>;
 	pinctrl-1 = <&mmc2_pins_default>;
@@ -331,6 +352,17 @@
 	pinctrl-3 = <&mmc2_pins_hs200 &mmc2_iodelay_hs200_conf>;
 };
 
+&mmc4 {
+	status = "okay";
+	vmmc-supply = <&vio_3v6>;
+	vqmmc-supply = <&vmmcwl_fixed>;
+	pinctrl-names = "default", "hs", "sdr12", "sdr25";
+	pinctrl-0 = <&mmc4_pins_hs &mmc4_iodelay_default_conf>;
+	pinctrl-1 = <&mmc4_pins_hs &mmc4_iodelay_manual1_conf>;
+	pinctrl-2 = <&mmc4_pins_hs &mmc4_iodelay_manual1_conf>;
+	pinctrl-3 = <&mmc4_pins_hs &mmc4_iodelay_manual1_conf>;
+};
+
 /* No RTC on this device */
 &rtc {
 	status = "disabled";
diff --git a/arch/arm/boot/dts/emev2-kzm9d.dts b/arch/arm/boot/dts/emev2-kzm9d.dts
index c238407133bf..0af44b7eadb9 100644
--- a/arch/arm/boot/dts/emev2-kzm9d.dts
+++ b/arch/arm/boot/dts/emev2-kzm9d.dts
@@ -34,9 +34,6 @@
 
 	gpio_keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		one {
 			debounce-interval = <50>;
 			wakeup-source;
diff --git a/arch/arm/boot/dts/emev2.dtsi b/arch/arm/boot/dts/emev2.dtsi
index 42ea246e71cb..fec1241b858f 100644
--- a/arch/arm/boot/dts/emev2.dtsi
+++ b/arch/arm/boot/dts/emev2.dtsi
@@ -31,13 +31,13 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		cpu@0 {
+		cpu0: cpu@0 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <0>;
 			clock-frequency = <533000000>;
 		};
-		cpu@1 {
+		cpu1: cpu@1 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <1>;
@@ -57,6 +57,7 @@
 		compatible = "arm,cortex-a9-pmu";
 		interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>;
 	};
 
 	clocks@e0110000 {
diff --git a/arch/arm/boot/dts/exynos-syscon-restart.dtsi b/arch/arm/boot/dts/exynos-syscon-restart.dtsi
index 4b3dd0549a54..ecf416690a15 100644
--- a/arch/arm/boot/dts/exynos-syscon-restart.dtsi
+++ b/arch/arm/boot/dts/exynos-syscon-restart.dtsi
@@ -3,22 +3,18 @@
  * Samsung's Exynos SoC syscon reboot/poweroff nodes common definition.
  */
 
-/ {
-	soc {
-		compatible = "simple-bus";
-
-		poweroff: syscon-poweroff {
-			compatible = "syscon-poweroff";
-			regmap = <&pmu_system_controller>;
-			offset = <0x330C>; /* PS_HOLD_CONTROL */
-			mask = <0x5200>; /* reset value */
-		};
+&pmu_system_controller {
+	poweroff: syscon-poweroff {
+		compatible = "syscon-poweroff";
+		regmap = <&pmu_system_controller>;
+		offset = <0x330C>; /* PS_HOLD_CONTROL */
+		mask = <0x5200>; /* reset value */
+	};
 
-		reboot: syscon-reboot {
-			compatible = "syscon-reboot";
-			regmap = <&pmu_system_controller>;
-			offset = <0x0400>; /* SWRESET */
-			mask = <0x1>;
-		};
+	reboot: syscon-reboot {
+		compatible = "syscon-reboot";
+		regmap = <&pmu_system_controller>;
+		offset = <0x0400>; /* SWRESET */
+		mask = <0x1>;
 	};
 };
diff --git a/arch/arm/boot/dts/exynos3250-rinato.dts b/arch/arm/boot/dts/exynos3250-rinato.dts
index 2c3460781cc6..2a6b828c01b7 100644
--- a/arch/arm/boot/dts/exynos3250-rinato.dts
+++ b/arch/arm/boot/dts/exynos3250-rinato.dts
@@ -24,6 +24,10 @@
 		i2c7 = &i2c_max77836;
 	};
 
+	chosen {
+		stdout-path = &serial_1;
+	};
+
 	memory@40000000 {
 		device_type = "memory";
 		reg =  <0x40000000 0x1ff00000>;
diff --git a/arch/arm/boot/dts/exynos3250.dtsi b/arch/arm/boot/dts/exynos3250.dtsi
index 0a5f989d963b..962af97c1883 100644
--- a/arch/arm/boot/dts/exynos3250.dtsi
+++ b/arch/arm/boot/dts/exynos3250.dtsi
@@ -15,7 +15,6 @@
  */
 
 #include "exynos4-cpu-thermal.dtsi"
-#include "exynos-syscon-restart.dtsi"
 #include <dt-bindings/clock/exynos3250.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -919,3 +918,4 @@
 };
 
 #include "exynos3250-pinctrl.dtsi"
+#include "exynos-syscon-restart.dtsi"
diff --git a/arch/arm/boot/dts/exynos4.dtsi b/arch/arm/boot/dts/exynos4.dtsi
index 909a9f2bf5be..dfe41b698745 100644
--- a/arch/arm/boot/dts/exynos4.dtsi
+++ b/arch/arm/boot/dts/exynos4.dtsi
@@ -20,7 +20,6 @@
 #include <dt-bindings/clock/exynos-audss-clk.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
-#include "exynos-syscon-restart.dtsi"
 
 / {
 	interrupt-parent = <&gic>;
@@ -1025,3 +1024,5 @@
 		};
 	};
 };
+
+#include "exynos-syscon-restart.dtsi"
diff --git a/arch/arm/boot/dts/exynos4210-origen.dts b/arch/arm/boot/dts/exynos4210-origen.dts
index 9a310e841d5d..2ab99f9f3d0a 100644
--- a/arch/arm/boot/dts/exynos4210-origen.dts
+++ b/arch/arm/boot/dts/exynos4210-origen.dts
@@ -34,26 +34,17 @@
 		stdout-path = &serial_2;
 	};
 
-	regulators {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		mmc_reg: regulator@0 {
-			compatible = "regulator-fixed";
-			reg = <0>;
-			regulator-name = "VMEM_VDD_2.8V";
-			regulator-min-microvolt = <2800000>;
-			regulator-max-microvolt = <2800000>;
-			gpio = <&gpx1 1 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-		};
+	mmc_reg: voltage-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "VMEM_VDD_2.8V";
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+		gpio = <&gpx1 1 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
 	};
 
 	gpio_keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
 
 		up {
 			label = "Up";
@@ -131,12 +122,23 @@
 	cpu0-supply = <&buck1_reg>;
 };
 
+&exynos_usbphy {
+	status = "okay";
+};
+
 &fimd {
 	pinctrl-0 = <&lcd_en &lcd_clk &lcd_data24 &pwm0_out>;
 	pinctrl-names = "default";
 	status = "okay";
 };
 
+&hsotg {
+	vusb_d-supply = <&ldo3_reg>;
+	vusb_a-supply = <&ldo8_reg>;
+	dr_mode = "peripheral";
+	status = "okay";
+};
+
 &i2c_0 {
 	status = "okay";
 	samsung,i2c-sda-delay = <100>;
diff --git a/arch/arm/boot/dts/exynos4210-trats.dts b/arch/arm/boot/dts/exynos4210-trats.dts
index eaeeb4f6b84a..6f1d76cb7951 100644
--- a/arch/arm/boot/dts/exynos4210-trats.dts
+++ b/arch/arm/boot/dts/exynos4210-trats.dts
@@ -259,8 +259,8 @@
 		reg = <0x48>;
 		interrupt-parent = <&gpx0>;
 		interrupts = <4 IRQ_TYPE_EDGE_FALLING>;
-		x-size = <720>;
-		y-size = <1280>;
+		touchscreen-size-x = <720>;
+		touchscreen-size-y = <1280>;
 		avdd-supply = <&tsp_reg>;
 		vdd-supply = <&tsp_reg>;
 	};
diff --git a/arch/arm/boot/dts/exynos4412-galaxy-s3.dtsi b/arch/arm/boot/dts/exynos4412-galaxy-s3.dtsi
index ee8e1f445370..30eee5942eff 100644
--- a/arch/arm/boot/dts/exynos4412-galaxy-s3.dtsi
+++ b/arch/arm/boot/dts/exynos4412-galaxy-s3.dtsi
@@ -15,24 +15,22 @@
 		i2c10 = &i2c_cm36651;
 	};
 
-	regulators {
-		lcd_vdd3_reg: voltage-regulator-2 {
-			compatible = "regulator-fixed";
-			regulator-name = "LCD_VDD_2.2V";
-			regulator-min-microvolt = <2200000>;
-			regulator-max-microvolt = <2200000>;
-			gpio = <&gpc0 1 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-		};
+	lcd_vdd3_reg: voltage-regulator-6 {
+		compatible = "regulator-fixed";
+		regulator-name = "LCD_VDD_2.2V";
+		regulator-min-microvolt = <2200000>;
+		regulator-max-microvolt = <2200000>;
+		gpio = <&gpc0 1 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
 
-		ps_als_reg: voltage-regulator-5 {
-			compatible = "regulator-fixed";
-			regulator-name = "LED_A_3.0V";
-			regulator-min-microvolt = <3000000>;
-			regulator-max-microvolt = <3000000>;
-			gpio = <&gpj0 5 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-		};
+	ps_als_reg: voltage-regulator-7 {
+		compatible = "regulator-fixed";
+		regulator-name = "LED_A_3.0V";
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3000000>;
+		gpio = <&gpj0 5 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
 	};
 
 	i2c_ak8975: i2c-gpio-0 {
@@ -120,8 +118,8 @@
 		reg = <0x48>;
 		interrupt-parent = <&gpm2>;
 		interrupts = <3 IRQ_TYPE_EDGE_FALLING>;
-		x-size = <720>;
-		y-size = <1280>;
+		touchscreen-size-x = <720>;
+		touchscreen-size-y = <1280>;
 		avdd-supply = <&ldo23_reg>;
 		vdd-supply = <&ldo24_reg>;
 	};
diff --git a/arch/arm/boot/dts/exynos4412-midas.dtsi b/arch/arm/boot/dts/exynos4412-midas.dtsi
index 76f2b30f1731..dc11ca1673e8 100644
--- a/arch/arm/boot/dts/exynos4412-midas.dtsi
+++ b/arch/arm/boot/dts/exynos4412-midas.dtsi
@@ -46,56 +46,50 @@
 		};
 	};
 
-	regulators {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		cam_io_reg: voltage-regulator-1 {
-			compatible = "regulator-fixed";
-			regulator-name = "CAM_SENSOR_A";
-			regulator-min-microvolt = <2800000>;
-			regulator-max-microvolt = <2800000>;
-			enable-active-high;
-			status = "disabled";
-		};
+	cam_io_reg: voltage-regulator-1 {
+		compatible = "regulator-fixed";
+		regulator-name = "CAM_SENSOR_A";
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+		enable-active-high;
+		status = "disabled";
+	};
 
-		cam_af_reg: voltage-regulator-3 {
-			compatible = "regulator-fixed";
-			regulator-name = "CAM_AF";
-			regulator-min-microvolt = <2800000>;
-			regulator-max-microvolt = <2800000>;
-			enable-active-high;
-			status = "disabled";
-		};
+	cam_af_reg: voltage-regulator-2 {
+		compatible = "regulator-fixed";
+		regulator-name = "CAM_AF";
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+		enable-active-high;
+		status = "disabled";
+	};
 
-		vsil12: voltage-regulator-6 {
-			compatible = "regulator-fixed";
-			regulator-name = "VSIL_1.2V";
-			regulator-min-microvolt = <1200000>;
-			regulator-max-microvolt = <1200000>;
-			gpio = <&gpl0 4 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-			vin-supply = <&buck7_reg>;
-		};
+	vsil12: voltage-regulator-3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VSIL_1.2V";
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		gpio = <&gpl0 4 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		vin-supply = <&buck7_reg>;
+	};
 
-		vcc33mhl: voltage-regulator-7 {
-			compatible = "regulator-fixed";
-			regulator-name = "VCC_3.3_MHL";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			gpio = <&gpl0 4 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-		};
+	vcc33mhl: voltage-regulator-4 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3.3_MHL";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpl0 4 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
 
-		vcc18mhl: voltage-regulator-8 {
-			compatible = "regulator-fixed";
-			regulator-name = "VCC_1.8_MHL";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-			gpio = <&gpl0 4 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-		};
+	vcc18mhl: voltage-regulator-5 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_1.8_MHL";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		gpio = <&gpl0 4 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
 	};
 
 	gpio-keys {
diff --git a/arch/arm/boot/dts/exynos4412-n710x.dts b/arch/arm/boot/dts/exynos4412-n710x.dts
index eb402a0d6651..fe2bfd76cc4e 100644
--- a/arch/arm/boot/dts/exynos4412-n710x.dts
+++ b/arch/arm/boot/dts/exynos4412-n710x.dts
@@ -13,15 +13,13 @@
 
 	/* bootargs are passed in by bootloader */
 
-	regulators {
-		cam_vdda_reg: voltage-regulator-9 {
-			compatible = "regulator-fixed";
-			regulator-name = "CAM_SENSOR_CORE_1.2V";
-			regulator-min-microvolt = <1200000>;
-			regulator-max-microvolt = <1200000>;
-			gpio = <&gpm4 1 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-		};
+	cam_vdda_reg: voltage-regulator-6 {
+		compatible = "regulator-fixed";
+		regulator-name = "CAM_SENSOR_CORE_1.2V";
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		gpio = <&gpm4 1 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
 	};
 };
 
diff --git a/arch/arm/boot/dts/exynos4412-odroid-common.dtsi b/arch/arm/boot/dts/exynos4412-odroid-common.dtsi
index d7ad07fd48f9..a09e46c9dbc0 100644
--- a/arch/arm/boot/dts/exynos4412-odroid-common.dtsi
+++ b/arch/arm/boot/dts/exynos4412-odroid-common.dtsi
@@ -37,20 +37,14 @@
 	};
 
 	sound: sound {
-		compatible = "simple-audio-card";
+		compatible = "hardkernel,odroid-xu4-audio";
 
-		simple-audio-card,format = "i2s";
-		simple-audio-card,bitclock-master = <&link0_codec>;
-		simple-audio-card,frame-master = <&link0_codec>;
-
-		simple-audio-card,cpu {
+		cpu {
 			sound-dai = <&i2s0 0>;
-			system-clock-frequency = <19200000>;
 		};
 
-		link0_codec: simple-audio-card,codec {
-			sound-dai = <&max98090>;
-			clocks = <&i2s0 CLK_I2S_CDCLK>;
+		codec {
+			sound-dai = <&hdmi>, <&max98090>;
 		};
 	};
 
@@ -142,14 +136,25 @@
 	pinctrl-0 = <>;
 };
 
+&clock {
+	assigned-clocks = <&clock CLK_FOUT_EPLL>;
+	assigned-clock-rates = <45158401>;
+};
+
 &clock_audss {
 	assigned-clocks = <&clock_audss EXYNOS_MOUT_AUDSS>,
 			<&clock_audss EXYNOS_MOUT_I2S>,
 			<&clock_audss EXYNOS_DOUT_SRP>,
-			<&clock_audss EXYNOS_DOUT_AUD_BUS>;
+			<&clock_audss EXYNOS_DOUT_AUD_BUS>,
+			<&clock_audss EXYNOS_DOUT_I2S>;
+
 	assigned-clock-parents = <&clock CLK_FOUT_EPLL>,
-			<&clock_audss EXYNOS_MOUT_AUDSS>;
-	assigned-clock-rates = <0>, <0>, <192000000>, <19200000>;
+			  <&clock_audss EXYNOS_MOUT_AUDSS>;
+
+	assigned-clock-rates = <0>, <0>,
+			<196608001>,
+			<(196608001 / 2)>,
+			<(196608001 / 8)>;
 };
 
 &cpu0 {
@@ -498,6 +503,8 @@
 	pinctrl-0 = <&i2s0_bus>;
 	pinctrl-names = "default";
 	status = "okay";
+	assigned-clocks = <&i2s0 CLK_I2S_RCLK_SRC>;
+	assigned-clock-parents = <&clock_audss EXYNOS_SCLK_I2S>;
 };
 
 &mixer {
diff --git a/arch/arm/boot/dts/exynos4412-odroidu3.dts b/arch/arm/boot/dts/exynos4412-odroidu3.dts
index bdcd4523cc1c..459919b65df8 100644
--- a/arch/arm/boot/dts/exynos4412-odroidu3.dts
+++ b/arch/arm/boot/dts/exynos4412-odroidu3.dts
@@ -113,11 +113,11 @@
 };
 
 &sound {
-	simple-audio-card,name = "Odroid-U3";
-	simple-audio-card,widgets =
+	model = "Odroid-U3";
+	samsung,audio-widgets =
 		"Headphone", "Headphone Jack",
 		"Speakers", "Speakers";
-	simple-audio-card,routing =
+	samsung,audio-routing =
 		"Headphone Jack", "HPL",
 		"Headphone Jack", "HPR",
 		"Headphone Jack", "MICBIAS",
diff --git a/arch/arm/boot/dts/exynos4412-odroidx.dts b/arch/arm/boot/dts/exynos4412-odroidx.dts
index 2dff129bc2ad..348556fcdd9d 100644
--- a/arch/arm/boot/dts/exynos4412-odroidx.dts
+++ b/arch/arm/boot/dts/exynos4412-odroidx.dts
@@ -97,12 +97,12 @@
 };
 
 &sound {
-	simple-audio-card,name = "Odroid-X";
-	simple-audio-card,widgets =
+	model = "Odroid-X";
+	samsung,audio-widgets =
 		"Headphone", "Headphone Jack",
 		"Microphone", "Mic Jack",
 		"Microphone", "DMIC";
-	simple-audio-card,routing =
+	samsung,audio-routing =
 		"Headphone Jack", "HPL",
 		"Headphone Jack", "HPR",
 		"IN1", "Mic Jack",
diff --git a/arch/arm/boot/dts/exynos4412-origen.dts b/arch/arm/boot/dts/exynos4412-origen.dts
index 1514f0dbaff8..346f71932457 100644
--- a/arch/arm/boot/dts/exynos4412-origen.dts
+++ b/arch/arm/boot/dts/exynos4412-origen.dts
@@ -90,7 +90,7 @@
 	samsung,vbus-gpio = <&gpx3 5 1>;
 	status = "okay";
 
-	port@1{
+	port@1 {
 		status = "okay";
 	};
 	port@2 {
diff --git a/arch/arm/boot/dts/exynos4412.dtsi b/arch/arm/boot/dts/exynos4412.dtsi
index 2ae1ab602f4b..7b43c10c510b 100644
--- a/arch/arm/boot/dts/exynos4412.dtsi
+++ b/arch/arm/boot/dts/exynos4412.dtsi
@@ -298,7 +298,7 @@
 			status = "disabled";
 		};
 
-		sysmmu_g2d: sysmmu@10A40000{
+		sysmmu_g2d: sysmmu@10a40000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x10A40000 0x1000>;
 			interrupt-parent = <&combiner>;
diff --git a/arch/arm/boot/dts/exynos5.dtsi b/arch/arm/boot/dts/exynos5.dtsi
index f8d7de1144f1..5d2f0a06fbef 100644
--- a/arch/arm/boot/dts/exynos5.dtsi
+++ b/arch/arm/boot/dts/exynos5.dtsi
@@ -12,7 +12,6 @@
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
-#include "exynos-syscon-restart.dtsi"
 
 / {
 	interrupt-parent = <&gic>;
@@ -197,8 +196,6 @@
 			reg = <0x145B0000 0x1000>;
 			interrupts = <10 3>;
 			interrupt-parent = <&combiner>;
-			#address-cells = <1>;
-			#size-cells = <0>;
 			status = "disabled";
 		};
 
diff --git a/arch/arm/boot/dts/exynos5250.dtsi b/arch/arm/boot/dts/exynos5250.dtsi
index 45283a6c5eee..2daf505b3d08 100644
--- a/arch/arm/boot/dts/exynos5250.dtsi
+++ b/arch/arm/boot/dts/exynos5250.dtsi
@@ -132,10 +132,6 @@
 			reg = <0x100440A0 0x20>;
 			#power-domain-cells = <0>;
 			label = "DISP1";
-			clocks = <&clock CLK_FIN_PLL>,
-				 <&clock CLK_MOUT_ACLK200_DISP1_SUB>,
-				 <&clock CLK_MOUT_ACLK300_DISP1_SUB>;
-			clock-names = "oscclk", "clk0", "clk1";
 		};
 
 		pd_mau: power-domain@100440c0 {
@@ -882,7 +878,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_fimc_dis1: sysmmu@132E0000{
+		sysmmu_fimc_dis1: sysmmu@132e0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x132E0000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -1117,3 +1113,4 @@
 };
 
 #include "exynos5250-pinctrl.dtsi"
+#include "exynos-syscon-restart.dtsi"
diff --git a/arch/arm/boot/dts/exynos5410.dtsi b/arch/arm/boot/dts/exynos5410.dtsi
index 55509c690328..20e789ea136f 100644
--- a/arch/arm/boot/dts/exynos5410.dtsi
+++ b/arch/arm/boot/dts/exynos5410.dtsi
@@ -439,3 +439,4 @@
 };
 
 #include "exynos5410-pinctrl.dtsi"
+#include "exynos-syscon-restart.dtsi"
diff --git a/arch/arm/boot/dts/exynos5420-peach-pit.dts b/arch/arm/boot/dts/exynos5420-peach-pit.dts
index 244f0091c21f..57c2332bf282 100644
--- a/arch/arm/boot/dts/exynos5420-peach-pit.dts
+++ b/arch/arm/boot/dts/exynos5420-peach-pit.dts
@@ -29,7 +29,7 @@
 
 	aliases {
 		/* Assign 20 so we don't get confused w/ builtin ones */
-		i2c20 = "/spi@12d40000/cros-ec@0/i2c-tunnel";
+		i2c20 = &i2c_tunnel;
 	};
 
 	backlight: backlight {
@@ -970,7 +970,7 @@
 			samsung,spi-feedback-delay = <1>;
 		};
 
-		i2c-tunnel {
+		i2c_tunnel: i2c-tunnel {
 			compatible = "google,cros-ec-i2c-tunnel";
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/arch/arm/boot/dts/exynos5420.dtsi b/arch/arm/boot/dts/exynos5420.dtsi
index 2f3cb2a97f71..f4e8c5823bc2 100644
--- a/arch/arm/boot/dts/exynos5420.dtsi
+++ b/arch/arm/boot/dts/exynos5420.dtsi
@@ -276,10 +276,6 @@
 			reg = <0x10044000 0x20>;
 			#power-domain-cells = <0>;
 			label = "GSC";
-			clocks = <&clock CLK_FIN_PLL>,
-				 <&clock CLK_MOUT_USER_ACLK300_GSCL>,
-				 <&clock CLK_GSCL0>, <&clock CLK_GSCL1>;
-			clock-names = "oscclk", "clk0", "asb0", "asb1";
 		};
 
 		isp_pd: power-domain@10044020 {
@@ -292,10 +288,6 @@
 		mfc_pd: power-domain@10044060 {
 			compatible = "samsung,exynos4210-pd";
 			reg = <0x10044060 0x20>;
-			clocks = <&clock CLK_FIN_PLL>,
-				 <&clock CLK_MOUT_USER_ACLK333>,
-				 <&clock CLK_ACLK333>;
-			clock-names = "oscclk", "clk0","asb0";
 			#power-domain-cells = <0>;
 			label = "MFC";
 		};
@@ -312,12 +304,6 @@
 			reg = <0x100440C0 0x20>;
 			#power-domain-cells = <0>;
 			label = "DISP";
-			clocks = <&clock CLK_FIN_PLL>,
-				 <&clock CLK_MOUT_USER_ACLK200_DISP1>,
-				 <&clock CLK_MOUT_USER_ACLK300_DISP1>,
-				 <&clock CLK_MOUT_USER_ACLK400_DISP1>,
-				 <&clock CLK_FIMD1>, <&clock CLK_MIXER>;
-			clock-names = "oscclk", "clk0", "clk1", "clk2", "asb0", "asb1";
 		};
 
 		mau_pd: power-domain@100440e0 {
@@ -687,6 +673,36 @@
 			iommus = <&sysmmu_gscl1>;
 		};
 
+		scaler_0: scaler@12800000 {
+			compatible = "samsung,exynos5420-scaler";
+			reg = <0x12800000 0x1294>;
+			interrupts = <0 220 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clock CLK_MSCL0>;
+			clock-names = "mscl";
+			power-domains = <&msc_pd>;
+			iommus = <&sysmmu_scaler0r>, <&sysmmu_scaler0w>;
+		};
+
+		scaler_1: scaler@12810000 {
+			compatible = "samsung,exynos5420-scaler";
+			reg = <0x12810000 0x1294>;
+			interrupts = <0 221 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clock CLK_MSCL1>;
+			clock-names = "mscl";
+			power-domains = <&msc_pd>;
+			iommus = <&sysmmu_scaler1r>, <&sysmmu_scaler1w>;
+		};
+
+		scaler_2: scaler@12820000 {
+			compatible = "samsung,exynos5420-scaler";
+			reg = <0x12820000 0x1294>;
+			interrupts = <0 222 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clock CLK_MSCL2>;
+			clock-names = "mscl";
+			power-domains = <&msc_pd>;
+			iommus = <&sysmmu_scaler2r>, <&sysmmu_scaler2w>;
+		};
+
 		jpeg_0: jpeg@11f50000 {
 			compatible = "samsung,exynos5420-jpeg";
 			reg = <0x11F50000 0x1000>;
@@ -761,7 +777,7 @@
 			#include "exynos5420-tmu-sensor-conf.dtsi"
 		};
 
-		sysmmu_g2dr: sysmmu@0x10A60000 {
+		sysmmu_g2dr: sysmmu@10a60000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x10A60000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -771,7 +787,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_g2dw: sysmmu@0x10A70000 {
+		sysmmu_g2dw: sysmmu@10a70000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x10A70000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -781,7 +797,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_tv: sysmmu@0x14650000 {
+		sysmmu_tv: sysmmu@14650000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x14650000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -792,7 +808,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_gscl0: sysmmu@0x13E80000 {
+		sysmmu_gscl0: sysmmu@13e80000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x13E80000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -803,7 +819,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_gscl1: sysmmu@0x13E90000 {
+		sysmmu_gscl1: sysmmu@13e90000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x13E90000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -814,65 +830,71 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_scaler0r: sysmmu@0x12880000 {
+		sysmmu_scaler0r: sysmmu@12880000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x12880000 0x1000>;
 			interrupt-parent = <&combiner>;
 			interrupts = <22 4>;
 			clock-names = "sysmmu", "master";
 			clocks = <&clock CLK_SMMU_MSCL0>, <&clock CLK_MSCL0>;
+			power-domains = <&msc_pd>;
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_scaler1r: sysmmu@0x12890000 {
+		sysmmu_scaler1r: sysmmu@12890000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x12890000 0x1000>;
 			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "sysmmu", "master";
 			clocks = <&clock CLK_SMMU_MSCL1>, <&clock CLK_MSCL1>;
+			power-domains = <&msc_pd>;
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_scaler2r: sysmmu@0x128A0000 {
+		sysmmu_scaler2r: sysmmu@128a0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x128A0000 0x1000>;
 			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "sysmmu", "master";
 			clocks = <&clock CLK_SMMU_MSCL2>, <&clock CLK_MSCL2>;
+			power-domains = <&msc_pd>;
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_scaler0w: sysmmu@0x128C0000 {
+		sysmmu_scaler0w: sysmmu@128c0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x128C0000 0x1000>;
 			interrupt-parent = <&combiner>;
 			interrupts = <27 2>;
 			clock-names = "sysmmu", "master";
 			clocks = <&clock CLK_SMMU_MSCL0>, <&clock CLK_MSCL0>;
+			power-domains = <&msc_pd>;
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_scaler1w: sysmmu@0x128D0000 {
+		sysmmu_scaler1w: sysmmu@128d0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x128D0000 0x1000>;
 			interrupt-parent = <&combiner>;
 			interrupts = <22 6>;
 			clock-names = "sysmmu", "master";
 			clocks = <&clock CLK_SMMU_MSCL1>, <&clock CLK_MSCL1>;
+			power-domains = <&msc_pd>;
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_scaler2w: sysmmu@0x128E0000 {
+		sysmmu_scaler2w: sysmmu@128e0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x128E0000 0x1000>;
 			interrupt-parent = <&combiner>;
 			interrupts = <19 6>;
 			clock-names = "sysmmu", "master";
 			clocks = <&clock CLK_SMMU_MSCL2>, <&clock CLK_MSCL2>;
+			power-domains = <&msc_pd>;
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_rotator: sysmmu@0x11D40000 {
+		sysmmu_rotator: sysmmu@11d40000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11D40000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -882,7 +904,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_jpeg0: sysmmu@0x11F10000 {
+		sysmmu_jpeg0: sysmmu@11f10000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11F10000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -892,7 +914,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_jpeg1: sysmmu@0x11F20000 {
+		sysmmu_jpeg1: sysmmu@11f20000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11F20000 0x1000>;
 			interrupts = <GIC_SPI 169 IRQ_TYPE_LEVEL_HIGH>;
@@ -901,7 +923,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_mfc_l: sysmmu@0x11200000 {
+		sysmmu_mfc_l: sysmmu@11200000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11200000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -912,7 +934,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_mfc_r: sysmmu@0x11210000 {
+		sysmmu_mfc_r: sysmmu@11210000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11210000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -923,7 +945,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_fimd1_0: sysmmu@0x14640000 {
+		sysmmu_fimd1_0: sysmmu@14640000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x14640000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -934,7 +956,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_fimd1_1: sysmmu@0x14680000 {
+		sysmmu_fimd1_1: sysmmu@14680000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x14680000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -1531,3 +1553,4 @@
 };
 
 #include "exynos5420-pinctrl.dtsi"
+#include "exynos-syscon-restart.dtsi"
diff --git a/arch/arm/boot/dts/exynos5422-odroid-core.dtsi b/arch/arm/boot/dts/exynos5422-odroid-core.dtsi
index d31249f25ccf..2f4f40882dab 100644
--- a/arch/arm/boot/dts/exynos5422-odroid-core.dtsi
+++ b/arch/arm/boot/dts/exynos5422-odroid-core.dtsi
@@ -23,7 +23,7 @@
 		stdout-path = "serial2:115200n8";
 	};
 
-	firmware@02073000 {
+	firmware@2073000 {
 		compatible = "samsung,secure-firmware";
 		reg = <0x02073000 0x1000>;
 	};
diff --git a/arch/arm/boot/dts/exynos5440-sd5v1.dts b/arch/arm/boot/dts/exynos5440-sd5v1.dts
deleted file mode 100644
index c4b8392d1ae1..000000000000
--- a/arch/arm/boot/dts/exynos5440-sd5v1.dts
+++ /dev/null
@@ -1,42 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * SAMSUNG SD5v1 board device tree source
- *
- * Copyright (c) 2013 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- */
-
-/dts-v1/;
-#include "exynos5440.dtsi"
-
-/ {
-	model = "SAMSUNG SD5v1 board based on EXYNOS5440";
-	compatible = "samsung,sd5v1", "samsung,exynos5440", "samsung,exynos5";
-
-	chosen {
-		bootargs = "root=/dev/sda2 rw rootwait ignore_loglevel earlyprintk no_console_suspend mem=2048M@0x80000000 mem=6144M@0x100000000 console=ttySAC0,115200";
-	};
-
-	/* FIXME: set reg property with correct start address and size */
-	memory@0 {
-		device_type = "memory";
-		reg = <0 0>;
-	};
-
-	fixed-rate-clocks {
-		xtal {
-			compatible = "samsung,clock-xtal";
-			clock-frequency = <50000000>;
-		};
-	};
-
-	spi {
-		status = "disabled";
-	};
-
-};
-
-&gmac {
-	fixed_phy;
-	phy_addr = <1>;
-};
diff --git a/arch/arm/boot/dts/exynos5440-ssdk5440.dts b/arch/arm/boot/dts/exynos5440-ssdk5440.dts
deleted file mode 100644
index a33c4fc29ae5..000000000000
--- a/arch/arm/boot/dts/exynos5440-ssdk5440.dts
+++ /dev/null
@@ -1,81 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * SAMSUNG SSDK5440 board device tree source
- *
- * Copyright (c) 2012 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- */
-
-/dts-v1/;
-#include "exynos5440.dtsi"
-#include <dt-bindings/gpio/gpio.h>
-
-/ {
-	model = "SAMSUNG SSDK5440 board based on EXYNOS5440";
-	compatible = "samsung,ssdk5440", "samsung,exynos5440", "samsung,exynos5";
-
-	chosen {
-		bootargs = "root=/dev/sda2 rw rootwait ignore_loglevel earlyprintk no_console_suspend mem=2048M@0x80000000 mem=6144M@0x100000000 console=ttySAC0,115200";
-	};
-
-	/* FIXME: set reg property with correct start address and size */
-	memory@0 {
-		device_type = "memory";
-		reg = <0 0>;
-	};
-
-	fixed-rate-clocks {
-		xtal {
-			compatible = "samsung,clock-xtal";
-			clock-frequency = <50000000>;
-		};
-	};
-};
-
-&pcie_0 {
-	reset-gpio = <&pin_ctrl 5 GPIO_ACTIVE_HIGH>;
-	status = "okay";
-};
-
-&pcie_1 {
-	reset-gpio = <&pin_ctrl 22 GPIO_ACTIVE_HIGH>;
-	status = "okay";
-};
-
-&spi_0 {
-	flash: w25q128@0 {
-		#address-cells = <1>;
-		#size-cells = <1>;
-		compatible = "winbond,w25q128";
-		spi-max-frequency = <15625000>;
-		reg = <0>;
-		controller-data {
-			samsung,spi-feedback-delay = <0>;
-		};
-
-		partition@0 {
-			label = "BootLoader";
-			reg = <0x60000 0x80000>;
-			read-only;
-		};
-
-		partition@e0000 {
-			label = "Recovery-Kernel";
-			reg = <0xe0000 0x300000>;
-			read-only;
-		};
-
-		partition@3e0000 {
-			label = "CRAM-FS";
-			reg = <0x3e0000 0x700000>;
-			read-only;
-		};
-
-		partition@ae0000 {
-			label = "User-Data";
-			reg = <0xae0000 0x520000>;
-		};
-
-	};
-
-};
diff --git a/arch/arm/boot/dts/exynos5440-tmu-sensor-conf.dtsi b/arch/arm/boot/dts/exynos5440-tmu-sensor-conf.dtsi
deleted file mode 100644
index 0421c3d42905..000000000000
--- a/arch/arm/boot/dts/exynos5440-tmu-sensor-conf.dtsi
+++ /dev/null
@@ -1,20 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Device tree sources for Exynos5440 TMU sensor configuration
- *
- * Copyright (c) 2014 Lukasz Majewski <l.majewski@samsung.com>
- */
-
-#include <dt-bindings/thermal/thermal_exynos.h>
-
-#thermal-sensor-cells = <0>;
-samsung,tmu_gain = <5>;
-samsung,tmu_reference_voltage = <16>;
-samsung,tmu_noise_cancel_mode = <4>;
-samsung,tmu_efuse_value = <0x5d2d>;
-samsung,tmu_min_efuse_value = <16>;
-samsung,tmu_max_efuse_value = <76>;
-samsung,tmu_first_point_trim = <25>;
-samsung,tmu_second_point_trim = <70>;
-samsung,tmu_default_temp_offset = <25>;
-samsung,tmu_cal_type = <TYPE_ONE_POINT_TRIMMING>;
diff --git a/arch/arm/boot/dts/exynos5440-trip-points.dtsi b/arch/arm/boot/dts/exynos5440-trip-points.dtsi
deleted file mode 100644
index a2b04fed7d0b..000000000000
--- a/arch/arm/boot/dts/exynos5440-trip-points.dtsi
+++ /dev/null
@@ -1,21 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Device tree sources for default Exynos5440 thermal zone definition
- *
- * Copyright (c) 2014 Lukasz Majewski <l.majewski@samsung.com>
- */
-
-polling-delay-passive = <0>;
-polling-delay = <0>;
-trips {
-	cpu-alert-0 {
-		temperature = <100000>; /* millicelsius */
-		hysteresis = <0>; /* millicelsius */
-		type = "active";
-	};
-	cpu-crit-0 {
-		temperature = <105000>; /* millicelsius */
-		hysteresis = <0>; /* millicelsius */
-		type = "critical";
-	};
-};
diff --git a/arch/arm/boot/dts/exynos5440.dtsi b/arch/arm/boot/dts/exynos5440.dtsi
deleted file mode 100644
index f3abecc44657..000000000000
--- a/arch/arm/boot/dts/exynos5440.dtsi
+++ /dev/null
@@ -1,355 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * SAMSUNG EXYNOS5440 SoC device tree source
- *
- * Copyright (c) 2012 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- */
-
-#include <dt-bindings/clock/exynos5440.h>
-#include <dt-bindings/interrupt-controller/arm-gic.h>
-#include <dt-bindings/interrupt-controller/irq.h>
-
-/ {
-	compatible = "samsung,exynos5440", "samsung,exynos5";
-
-	interrupt-parent = <&gic>;
-	#address-cells = <1>;
-	#size-cells = <1>;
-
-	aliases {
-		serial0 = &serial_0;
-		serial1 = &serial_1;
-		spi0 = &spi_0;
-		tmuctrl0 = &tmuctrl_0;
-		tmuctrl1 = &tmuctrl_1;
-		tmuctrl2 = &tmuctrl_2;
-	};
-
-	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		cpu@0 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a15";
-			reg = <0>;
-		};
-		cpu@1 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a15";
-			reg = <1>;
-		};
-		cpu@2 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a15";
-			reg = <2>;
-		};
-		cpu@3 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a15";
-			reg = <3>;
-		};
-	};
-
-	soc: soc {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges;
-
-		clock: clock-controller@160000 {
-			compatible = "samsung,exynos5440-clock";
-			reg = <0x160000 0x1000>;
-			#clock-cells = <1>;
-		};
-
-		gic: interrupt-controller@2e0000 {
-			compatible = "arm,cortex-a15-gic";
-			#interrupt-cells = <3>;
-			interrupt-controller;
-			reg =	<0x2E1000 0x1000>,
-				<0x2E2000 0x2000>,
-				<0x2E4000 0x2000>,
-				<0x2E6000 0x2000>;
-			interrupts = <GIC_PPI 9
-					(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
-		};
-
-
-		arm-pmu {
-			compatible = "arm,cortex-a15-pmu", "arm,cortex-a9-pmu";
-			interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
-		timer {
-			compatible = "arm,cortex-a15-timer",
-				     "arm,armv7-timer";
-			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
-			clock-frequency = <50000000>;
-		};
-
-		cpufreq@160000 {
-			compatible = "samsung,exynos5440-cpufreq";
-			reg = <0x160000 0x1000>;
-			interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>;
-			operating-points = <
-					/* KHz	  uV */
-					1500000 1100000
-					1400000 1075000
-					1300000 1050000
-					1200000 1025000
-					1100000 1000000
-					1000000 975000
-					900000  950000
-					800000  925000
-			>;
-		};
-
-		serial_0: serial@b0000 {
-			compatible = "samsung,exynos4210-uart";
-			reg = <0xB0000 0x1000>;
-			interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_B_125>, <&clock CLK_B_125>;
-			clock-names = "uart", "clk_uart_baud0";
-		};
-
-		serial_1: serial@c0000 {
-			compatible = "samsung,exynos4210-uart";
-			reg = <0xC0000 0x1000>;
-			interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_B_125>, <&clock CLK_B_125>;
-			clock-names = "uart", "clk_uart_baud0";
-		};
-
-		spi_0: spi@d0000 {
-			compatible = "samsung,exynos5440-spi";
-			reg = <0xD0000 0x100>;
-			interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			samsung,spi-src-clk = <0>;
-			num-cs = <1>;
-			clocks = <&clock CLK_B_125>, <&clock CLK_SPI_BAUD>;
-			clock-names = "spi", "spi_busclk0";
-		};
-
-		pin_ctrl: pinctrl@e0000 {
-			compatible = "samsung,exynos5440-pinctrl";
-			reg = <0xE0000 0x1000>;
-			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-controller;
-			#interrupt-cells = <2>;
-			#gpio-cells = <2>;
-
-			fan: fan {
-				samsung,exynos5440-pin-function = <1>;
-			};
-
-			hdd_led0: hdd_led0 {
-				samsung,exynos5440-pin-function = <2>;
-			};
-
-			hdd_led1: hdd_led1 {
-				samsung,exynos5440-pin-function = <3>;
-			};
-
-			uart1: uart1 {
-				samsung,exynos5440-pin-function = <4>;
-			};
-		};
-
-		i2c@f0000 {
-			compatible = "samsung,exynos5440-i2c";
-			reg = <0xF0000 0x1000>;
-			interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			clocks = <&clock CLK_B_125>;
-			clock-names = "i2c";
-		};
-
-		i2c@100000 {
-			compatible = "samsung,exynos5440-i2c";
-			reg = <0x100000 0x1000>;
-			interrupts = <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			clocks = <&clock CLK_B_125>;
-			clock-names = "i2c";
-		};
-
-		watchdog@110000 {
-			compatible = "samsung,s3c6410-wdt";
-			reg = <0x110000 0x1000>;
-			interrupts = <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_B_125>;
-			clock-names = "watchdog";
-		};
-
-		gmac: ethernet@230000 {
-			compatible = "snps,dwmac-3.70a", "snps,dwmac";
-			reg = <0x00230000 0x8000>;
-			interrupt-parent = <&gic>;
-			interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "macirq";
-			phy-mode = "sgmii";
-			clocks = <&clock CLK_GMAC0>;
-			clock-names = "stmmaceth";
-		};
-
-		amba {
-			#address-cells = <1>;
-			#size-cells = <1>;
-			compatible = "simple-bus";
-			interrupt-parent = <&gic>;
-			ranges;
-		};
-
-		rtc@130000 {
-			compatible = "samsung,s3c6410-rtc";
-			reg = <0x130000 0x1000>;
-			interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_B_125>;
-			clock-names = "rtc";
-		};
-
-		tmuctrl_0: tmuctrl@160118 {
-			compatible = "samsung,exynos5440-tmu";
-			reg = <0x160118 0x230>, <0x160368 0x10>;
-			interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_B_125>;
-			clock-names = "tmu_apbif";
-			#include "exynos5440-tmu-sensor-conf.dtsi"
-		};
-
-		tmuctrl_1: tmuctrl@16011c {
-			compatible = "samsung,exynos5440-tmu";
-			reg = <0x16011C 0x230>, <0x160368 0x10>;
-			interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_B_125>;
-			clock-names = "tmu_apbif";
-			#include "exynos5440-tmu-sensor-conf.dtsi"
-		};
-
-		tmuctrl_2: tmuctrl@160120 {
-			compatible = "samsung,exynos5440-tmu";
-			reg = <0x160120 0x230>, <0x160368 0x10>;
-			interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_B_125>;
-			clock-names = "tmu_apbif";
-			#include "exynos5440-tmu-sensor-conf.dtsi"
-		};
-
-		sata@210000 {
-			compatible = "snps,exynos5440-ahci";
-			reg = <0x210000 0x10000>;
-			interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_SATA>;
-			clock-names = "sata";
-		};
-
-		ohci@220000 {
-			compatible = "samsung,exynos5440-ohci";
-			reg = <0x220000 0x1000>;
-			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_USB>;
-			clock-names = "usbhost";
-		};
-
-		ehci@221000 {
-			compatible = "samsung,exynos5440-ehci";
-			reg = <0x221000 0x1000>;
-			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_USB>;
-			clock-names = "usbhost";
-		};
-
-		pcie_phy0: pcie-phy@270000 {
-			#phy-cells = <0>;
-			compatible = "samsung,exynos5440-pcie-phy";
-			reg = <0x270000 0x1000>, <0x271000 0x40>;
-		};
-
-		pcie_phy1: pcie-phy@272000 {
-			#phy-cells = <0>;
-			compatible = "samsung,exynos5440-pcie-phy";
-			reg = <0x272000 0x1000>, <0x271040 0x40>;
-		};
-
-		pcie_0: pcie@290000 {
-			compatible = "samsung,exynos5440-pcie", "snps,dw-pcie";
-			reg = <0x290000 0x1000>, <0x40000000 0x1000>;
-			reg-names = "elbi", "config";
-			interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_PR0_250_O>, <&clock CLK_PB0_250_O>;
-			clock-names = "pcie", "pcie_bus";
-			#address-cells = <3>;
-			#size-cells = <2>;
-			device_type = "pci";
-			phys = <&pcie_phy0>;
-			ranges = <0x81000000 0 0	  0x40001000 0 0x00010000   /* downstream I/O */
-				  0x82000000 0 0x40011000 0x40011000 0 0x1ffef000>; /* non-prefetchable memory */
-			bus-range = <0x00 0xff>;
-			#interrupt-cells = <1>;
-			interrupt-map-mask = <0 0 0 0>;
-			interrupt-map = <0x0 0 &gic 53>;
-			num-lanes = <4>;
-			status = "disabled";
-		};
-
-		pcie_1: pcie@2a0000 {
-			compatible = "samsung,exynos5440-pcie", "snps,dw-pcie";
-			reg = <0x2a0000 0x1000>, <0x60000000 0x1000>;
-			reg-names = "elbi", "config";
-			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_PR1_250_O>, <&clock CLK_PB0_250_O>;
-			clock-names = "pcie", "pcie_bus";
-			#address-cells = <3>;
-			#size-cells = <2>;
-			device_type = "pci";
-			phys = <&pcie_phy1>;
-			ranges = <0x81000000 0 0	  0x60001000 0 0x00010000   /* downstream I/O */
-				  0x82000000 0 0x60011000 0x60011000 0 0x1ffef000>; /* non-prefetchable memory */
-			bus-range = <0x00 0xff>;
-			#interrupt-cells = <1>;
-			interrupt-map-mask = <0 0 0 0>;
-			interrupt-map = <0x0 0 &gic 56>;
-			num-lanes = <4>;
-			status = "disabled";
-		};
-	};
-
-	thermal-zones {
-		cpu0_thermal: cpu0-thermal {
-			thermal-sensors = <&tmuctrl_0>;
-			#include "exynos5440-trip-points.dtsi"
-		};
-		cpu1_thermal: cpu1-thermal {
-		       thermal-sensors = <&tmuctrl_1>;
-		       #include "exynos5440-trip-points.dtsi"
-		};
-		cpu2_thermal: cpu2-thermal {
-		       thermal-sensors = <&tmuctrl_2>;
-		       #include "exynos5440-trip-points.dtsi"
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/exynos5800-peach-pi.dts b/arch/arm/boot/dts/exynos5800-peach-pi.dts
index 2f8df9244f72..d80ab9085da1 100644
--- a/arch/arm/boot/dts/exynos5800-peach-pi.dts
+++ b/arch/arm/boot/dts/exynos5800-peach-pi.dts
@@ -27,7 +27,7 @@
 
 	aliases {
 		/* Assign 20 so we don't get confused w/ builtin ones */
-		i2c20 = "/spi@12d40000/cros-ec@0/i2c-tunnel";
+		i2c20 = &i2c_tunnel;
 	};
 
 	backlight: backlight {
@@ -939,7 +939,7 @@
 			samsung,spi-feedback-delay = <1>;
 		};
 
-		i2c-tunnel {
+		i2c_tunnel: i2c-tunnel {
 			compatible = "google,cros-ec-i2c-tunnel";
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/arch/arm/boot/dts/gemini-dlink-dir-685.dts b/arch/arm/boot/dts/gemini-dlink-dir-685.dts
index cadde92bc6b5..fb5c954ab95a 100644
--- a/arch/arm/boot/dts/gemini-dlink-dir-685.dts
+++ b/arch/arm/boot/dts/gemini-dlink-dir-685.dts
@@ -13,22 +13,22 @@
 	#address-cells = <1>;
 	#size-cells = <1>;
 
-	memory {
+	memory@0 {
 		/* 128 MB SDRAM in 2 x Hynix HY5DU121622DTP-D43 */
 		device_type = "memory";
 		reg = <0x00000000 0x8000000>;
 	};
 
 	chosen {
-		stdout-path = "uart0:115200n8";
+		bootargs = "console=ttyS0,19200n8 root=/dev/sda1 rw rootwait";
+		stdout-path = "uart0:19200n8";
 	};
 
 	gpio_keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
+
 		button-esc {
-			debounce_interval = <50>;
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <KEY_ESC>;
 			label = "reset";
@@ -36,7 +36,7 @@
 			gpios = <&gpio0 8 GPIO_ACTIVE_LOW>;
 		};
 		button-eject {
-			debounce_interval = <50>;
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <KEY_EJECTCD>;
 			label = "unmount";
@@ -98,7 +98,7 @@
 		/*
 		 * These two LEDs are on the side of the device.
 		 * For electrical reasons, both LEDs cannot be active
-		 * at the same time so only blue or orange can on at
+		 * at the same time so only blue or orange can be on at
 		 * one time. Enabling both makes the LED go dark.
 		 * The LEDs both sit inside the unmount button and the
 		 * label on the case says "unmount".
@@ -108,12 +108,14 @@
 			/* Collides with LPC_SERIRQ, UART DTR, SSP FSC pins */
 			gpios = <&gpio0 11 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
+			linux,default-trigger = "disk-read";
 		};
 		led-orange-hd {
 			label = "dir685:orange:HD";
 			/* Collides with LPC_LAD[2], UART DSR, SSP ECLK pins */
 			gpios = <&gpio0 12 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
+			linux,default-trigger = "disk-write";
 		};
 	};
 
diff --git a/arch/arm/boot/dts/gemini-dlink-dns-313.dts b/arch/arm/boot/dts/gemini-dlink-dns-313.dts
index 403364a7aab9..d1329322b968 100644
--- a/arch/arm/boot/dts/gemini-dlink-dns-313.dts
+++ b/arch/arm/boot/dts/gemini-dlink-dns-313.dts
@@ -15,7 +15,7 @@
 	#address-cells = <1>;
 	#size-cells = <1>;
 
-	memory {
+	memory@0 {
 		/* 64 MB SDRAM in a Nanya NT5DS32M16BS-6K package */
 		device_type = "memory";
 		reg = <0x00000000 0x4000000>;
@@ -26,15 +26,15 @@
 	};
 
 	chosen {
+		bootargs = "console=ttyS0,19200n8 root=/dev/sda4 rw rootwait";
 		stdout-path = "uart0:19200n8";
 	};
 
 	gpio_keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
+
 		button-esc {
-			debounce_interval = <50>;
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <KEY_ESC>;
 			label = "reset";
@@ -59,14 +59,13 @@
 			label = "dns313:green:disk";
 			gpios = <&gpio0 3 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
-			linux,default-trigger = "ide-disk";
-			/* Ideally should activate while reading */
+			linux,default-trigger = "disk-read";
 		};
 		led-disk-red {
 			label = "dns313:red:disk";
 			gpios = <&gpio0 4 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
-			/* Ideally should activate while writing */
+			linux,default-trigger = "disk-write";
 		};
 	};
 
@@ -158,8 +157,12 @@
 
 	soc {
 		flash@30000000 {
+			/*
+			 * This is a Eon EN29LV400AB 512 KiB flash with
+			 * three partitions.
+			 */
+			compatible = "cortina,gemini-flash", "jedec-flash";
 			status = "okay";
-			/* 512KB of flash */
 			reg = <0x30000000 0x00080000>;
 
 			/*
diff --git a/arch/arm/boot/dts/gemini-nas4220b.dts b/arch/arm/boot/dts/gemini-nas4220b.dts
index 4785fbcc41ed..963ea890c87f 100644
--- a/arch/arm/boot/dts/gemini-nas4220b.dts
+++ b/arch/arm/boot/dts/gemini-nas4220b.dts
@@ -14,7 +14,7 @@
 	#address-cells = <1>;
 	#size-cells = <1>;
 
-	memory { /* 128 MB */
+	memory@0 { /* 128 MB */
 		device_type = "memory";
 		reg = <0x00000000 0x8000000>;
 	};
@@ -26,19 +26,17 @@
 
 	gpio_keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
 
-		button@29 {
-			debounce_interval = <50>;
+		button-setup {
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <KEY_SETUP>;
 			label = "Backup button";
 			/* Conflict with TVC */
 			gpios = <&gpio1 29 GPIO_ACTIVE_LOW>;
 		};
-		button@31 {
-			debounce_interval = <50>;
+		button-restart {
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <KEY_RESTART>;
 			label = "Softreset button";
@@ -49,13 +47,13 @@
 
 	leds {
 		compatible = "gpio-leds";
-		led@28 {
+		led-orange-hdd {
 			label = "nas4220b:orange:hdd";
 			/* Conflict with TVC */
 			gpios = <&gpio1 28 GPIO_ACTIVE_HIGH>;
 			default-state = "on";
 		};
-		led@30 {
+		led-green-os {
 			label = "nas4220b:green:os";
 			/* Conflict with TVC */
 			gpios = <&gpio1 30 GPIO_ACTIVE_HIGH>;
@@ -202,5 +200,9 @@
 		ata@63000000 {
 			status = "okay";
 		};
+
+		ata@63400000 {
+			status = "okay";
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/gemini-rut1xx.dts b/arch/arm/boot/dts/gemini-rut1xx.dts
index 15f20178642c..eb4f0bf074da 100644
--- a/arch/arm/boot/dts/gemini-rut1xx.dts
+++ b/arch/arm/boot/dts/gemini-rut1xx.dts
@@ -14,7 +14,7 @@
 	#address-cells = <1>;
 	#size-cells = <1>;
 
-	memory { /* 128 MB */
+	memory@0 { /* 128 MB */
 		device_type = "memory";
 		reg = <0x00000000 0x8000000>;
 	};
@@ -26,11 +26,9 @@
 
 	gpio_keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
 
-		button@28 {
-			debounce_interval = <50>;
+		button-setup {
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <KEY_SETUP>;
 			label = "Reset to defaults";
@@ -41,14 +39,14 @@
 
 	leds {
 		compatible = "gpio-leds";
-		led@7 {
+		led-gsm {
 			/* FIXME: add the LED color */
 			label = "rut1xx::gsm";
 			/* Conflict with ICE */
 			gpios = <&gpio0 7 GPIO_ACTIVE_HIGH>;
 			default-state = "on";
 		};
-		led@31 {
+		led-power {
 			/* FIXME: add the LED color */
 			label = "rut1xx::power";
 			/* Conflict with NAND CE0 */
diff --git a/arch/arm/boot/dts/gemini-sq201.dts b/arch/arm/boot/dts/gemini-sq201.dts
index 63c02ca9513c..e5cf9d1a98cd 100644
--- a/arch/arm/boot/dts/gemini-sq201.dts
+++ b/arch/arm/boot/dts/gemini-sq201.dts
@@ -14,7 +14,7 @@
 	#address-cells = <1>;
 	#size-cells = <1>;
 
-	memory { /* 128 MB */
+	memory@0 { /* 128 MB */
 		device_type = "memory";
 		reg = <0x00000000 0x8000000>;
 	};
@@ -26,11 +26,9 @@
 
 	gpio_keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
 
-		button@18 {
-			debounce_interval = <50>;
+		button-setup {
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <KEY_SETUP>;
 			label = "factory reset";
@@ -41,14 +39,14 @@
 
 	leds {
 		compatible = "gpio-leds";
-		led@20 {
+		led-green-info {
 			label = "sq201:green:info";
 			/* Conflict with parallel flash */
 			gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>;
 			default-state = "on";
 			linux,default-trigger = "heartbeat";
 		};
-		led@31 {
+		led-green-usb {
 			label = "sq201:green:usb";
 			/* Conflict with parallel and NAND flash */
 			gpios = <&gpio0 31 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm/boot/dts/gemini-wbd111.dts b/arch/arm/boot/dts/gemini-wbd111.dts
index b4ec9ad85d72..29af86cd10f7 100644
--- a/arch/arm/boot/dts/gemini-wbd111.dts
+++ b/arch/arm/boot/dts/gemini-wbd111.dts
@@ -14,7 +14,8 @@
 	#address-cells = <1>;
 	#size-cells = <1>;
 
-	memory { /* 128 MB */
+	memory@0 {
+		/* 128 MB */
 		device_type = "memory";
 		reg = <0x00000000 0x8000000>;
 	};
@@ -26,11 +27,9 @@
 
 	gpio_keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
 
-		button@5 {
-			debounce_interval = <50>;
+		button-setup {
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <KEY_SETUP>;
 			label = "reset";
@@ -42,25 +41,25 @@
 	leds {
 		compatible = "gpio-leds";
 
-		led@1 {
+		led-red-l3 {
 			label = "wbd111:red:L3";
 			/* Conflict with TVC and extended parallel flash */
 			gpios = <&gpio0 1 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
-		led@2 {
+		led-green-l4 {
 			label = "wbd111:green:L4";
 			/* Conflict with TVC and extended parallel flash */
 			gpios = <&gpio0 2 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
-		led@3 {
+		led-red-l4 {
 			label = "wbd111:red:L4";
 			/* Conflict with TVC and extended parallel flash */
 			gpios = <&gpio0 3 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
-		led@5 {
+		led-greeb-l3 {
 			label = "wbd111:green:L3";
 			/* Conflict with TVC and extended parallel flash */
 			gpios = <&gpio0 5 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm/boot/dts/gemini-wbd222.dts b/arch/arm/boot/dts/gemini-wbd222.dts
index 6d25bcc046e7..24e6ae3616f7 100644
--- a/arch/arm/boot/dts/gemini-wbd222.dts
+++ b/arch/arm/boot/dts/gemini-wbd222.dts
@@ -14,7 +14,7 @@
 	#address-cells = <1>;
 	#size-cells = <1>;
 
-	memory { /* 128 MB */
+	memory@0 { /* 128 MB */
 		device_type = "memory";
 		reg = <0x00000000 0x8000000>;
 	};
@@ -26,11 +26,9 @@
 
 	gpio_keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
 
-		button@5 {
-			debounce_interval = <50>;
+		button-setup {
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <KEY_SETUP>;
 			label = "reset";
@@ -42,25 +40,25 @@
 	leds {
 		compatible = "gpio-leds";
 
-		led@1 {
+		led-red-l3 {
 			label = "wbd111:red:L3";
 			/* Conflict with TVC and extended parallel flash */
 			gpios = <&gpio0 1 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
-		led@2 {
+		led-green-l4 {
 			label = "wbd111:green:L4";
 			/* Conflict with TVC and extended parallel flash */
 			gpios = <&gpio0 2 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
-		led@3 {
+		led-red-l4 {
 			label = "wbd111:red:L4";
 			/* Conflict with TVC and extended parallel flash */
 			gpios = <&gpio0 3 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
-		led@5 {
+		led-green-l3 {
 			label = "wbd111:green:L3";
 			/* Conflict with TVC and extended parallel flash */
 			gpios = <&gpio0 5 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm/boot/dts/gemini.dtsi b/arch/arm/boot/dts/gemini.dtsi
index 0568baca500a..eb752e9495de 100644
--- a/arch/arm/boot/dts/gemini.dtsi
+++ b/arch/arm/boot/dts/gemini.dtsi
@@ -3,8 +3,6 @@
  * Device Tree file for Cortina systems Gemini SoC
  */
 
-/include/ "skeleton.dtsi"
-
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/clock/cortina,gemini-clock.h>
 #include <dt-bindings/reset/cortina,gemini-reset.h>
diff --git a/arch/arm/boot/dts/imx1-ads.dts b/arch/arm/boot/dts/imx1-ads.dts
index 6354e4c87313..a1d81badb5c8 100644
--- a/arch/arm/boot/dts/imx1-ads.dts
+++ b/arch/arm/boot/dts/imx1-ads.dts
@@ -23,17 +23,6 @@
 	memory@8000000 {
 		reg = <0x08000000 0x04000000>;
 	};
-
-	clocks {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		clk32 {
-			compatible = "fsl,imx-clk32", "fixed-clock";
-			#clock-cells = <0>;
-			clock-frequency = <32000>;
-		};
-	};
 };
 
 &cspi1 {
diff --git a/arch/arm/boot/dts/imx1.dtsi b/arch/arm/boot/dts/imx1.dtsi
index f7b9edf93f5e..3edc7b5550d8 100644
--- a/arch/arm/boot/dts/imx1.dtsi
+++ b/arch/arm/boot/dts/imx1.dtsi
@@ -1,13 +1,6 @@
-/*
- * Copyright (C) 2014 Alexander Shiyan <shc_work@mail.ru>
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (C) 2014 Alexander Shiyan <shc_work@mail.ru>
 
 #include "imx1-pinfunc.h"
 
@@ -62,6 +55,14 @@
 		};
 	};
 
+	clocks {
+		clk32 {
+			compatible = "fsl,imx-clk32", "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <32000>;
+		};
+	};
+
 	soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/arch/arm/boot/dts/imx23-evk.dts b/arch/arm/boot/dts/imx23-evk.dts
index 9d92ece82560..9fb47724b9c1 100644
--- a/arch/arm/boot/dts/imx23-evk.dts
+++ b/arch/arm/boot/dts/imx23-evk.dts
@@ -1,13 +1,6 @@
-/*
- * Copyright 2012 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 #include "imx23.dtsi"
diff --git a/arch/arm/boot/dts/imx23.dtsi b/arch/arm/boot/dts/imx23.dtsi
index cb0a3fe32718..71bfd2b15609 100644
--- a/arch/arm/boot/dts/imx23.dtsi
+++ b/arch/arm/boot/dts/imx23.dtsi
@@ -1,13 +1,6 @@
-/*
- * Copyright 2012 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
 
 #include "imx23-pinfunc.h"
 
diff --git a/arch/arm/boot/dts/imx25-pdk.dts b/arch/arm/boot/dts/imx25-pdk.dts
index 7f9bd052b84e..a5626b46ac4e 100644
--- a/arch/arm/boot/dts/imx25-pdk.dts
+++ b/arch/arm/boot/dts/imx25-pdk.dts
@@ -1,13 +1,6 @@
-/*
- * Copyright 2013 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2013 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 #include <dt-bindings/gpio/gpio.h>
@@ -291,7 +284,6 @@
 };
 
 &ssi1 {
-	codec-handle = <&codec>;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/imx25.dtsi b/arch/arm/boot/dts/imx25.dtsi
index cf70df20b19c..85c15ee63272 100644
--- a/arch/arm/boot/dts/imx25.dtsi
+++ b/arch/arm/boot/dts/imx25.dtsi
@@ -1,13 +1,6 @@
-/*
- * Copyright 2012 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
 
 #include <dt-bindings/gpio/gpio.h>
 #include "imx25-pinfunc.h"
@@ -70,9 +63,6 @@
 	};
 
 	clocks {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		osc {
 			compatible = "fsl,imx-osc", "fixed-clock";
 			#clock-cells = <0>;
diff --git a/arch/arm/boot/dts/imx27-apf27.dts b/arch/arm/boot/dts/imx27-apf27.dts
index 66941cdbf244..3eddd805a793 100644
--- a/arch/arm/boot/dts/imx27-apf27.dts
+++ b/arch/arm/boot/dts/imx27-apf27.dts
@@ -22,17 +22,10 @@
 	memory@a0000000 {
 		reg = <0xa0000000 0x04000000>;
 	};
+};
 
-	clocks {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		osc26m {
-			compatible = "fsl,imx-osc26m", "fixed-clock";
-			#clock-cells = <0>;
-			clock-frequency = <0>;
-		};
-	};
+&clk_osc26m {
+	clock-frequency = <0>;
 };
 
 &iomuxc {
diff --git a/arch/arm/boot/dts/imx27-pdk.dts b/arch/arm/boot/dts/imx27-pdk.dts
index 924b90c9985d..f9a882d99132 100644
--- a/arch/arm/boot/dts/imx27-pdk.dts
+++ b/arch/arm/boot/dts/imx27-pdk.dts
@@ -1,13 +1,6 @@
-/*
- * Copyright 2012 Sascha Hauer, Pengutronix
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Sascha Hauer, Pengutronix
 
 /dts-v1/;
 #include "imx27.dtsi"
diff --git a/arch/arm/boot/dts/imx27.dtsi b/arch/arm/boot/dts/imx27.dtsi
index 6585b00c3917..753d88df1627 100644
--- a/arch/arm/boot/dts/imx27.dtsi
+++ b/arch/arm/boot/dts/imx27.dtsi
@@ -1,13 +1,6 @@
-/*
- * Copyright 2012 Sascha Hauer, Pengutronix
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Sascha Hauer, Pengutronix
 
 #include "imx27-pinfunc.h"
 
@@ -57,10 +50,7 @@
 	};
 
 	clocks {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		osc26m {
+		clk_osc26m: osc26m {
 			compatible = "fsl,imx-osc26m", "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <26000000>;
diff --git a/arch/arm/boot/dts/imx28-cfa10049.dts b/arch/arm/boot/dts/imx28-cfa10049.dts
index 60e5c7fd5035..f1c8315b3e01 100644
--- a/arch/arm/boot/dts/imx28-cfa10049.dts
+++ b/arch/arm/boot/dts/imx28-cfa10049.dts
@@ -398,8 +398,6 @@
 		compatible = "gpio-keys";
 		pinctrl-names = "default";
 		pinctrl-0 = <&rotary_btn_pins_cfa10049>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 
 		rotary_button {
 			label = "rotary_button";
diff --git a/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts b/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts
index 7f8d40a9c67e..22215337f72a 100644
--- a/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts
+++ b/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts
@@ -206,8 +206,6 @@
 
 	gpio-keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&enocean_button>;
 
diff --git a/arch/arm/boot/dts/imx28-evk.dts b/arch/arm/boot/dts/imx28-evk.dts
index b0d39654aeb3..6b0ae667640f 100644
--- a/arch/arm/boot/dts/imx28-evk.dts
+++ b/arch/arm/boot/dts/imx28-evk.dts
@@ -1,13 +1,6 @@
-/*
- * Copyright 2012 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 #include "imx28.dtsi"
diff --git a/arch/arm/boot/dts/imx28-tx28.dts b/arch/arm/boot/dts/imx28-tx28.dts
index 687186358c18..b8f46432e2a2 100644
--- a/arch/arm/boot/dts/imx28-tx28.dts
+++ b/arch/arm/boot/dts/imx28-tx28.dts
@@ -140,15 +140,10 @@
 		regulator-boot-on;
 	};
 
-	clocks {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		mclk: clock@0 {
-			compatible = "fixed-clock";
-			reg = <0>;
-			#clock-cells = <0>;
-			clock-frequency = <26000000>;
-		};
+	mclk: clock-mclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <26000000>;
 	};
 
 	sound {
@@ -345,6 +340,7 @@
 		interrupts = <5 IRQ_TYPE_EDGE_FALLING>;
 		reset-gpios = <&gpio2 6 GPIO_ACTIVE_LOW>;
 		wake-gpios = <&gpio4 9 GPIO_ACTIVE_HIGH>;
+		wakeup-source;
 	};
 
 	touchscreen: tsc2007@48 {
diff --git a/arch/arm/boot/dts/imx28.dtsi b/arch/arm/boot/dts/imx28.dtsi
index 9ad8d3556859..5107fdc482ea 100644
--- a/arch/arm/boot/dts/imx28.dtsi
+++ b/arch/arm/boot/dts/imx28.dtsi
@@ -1,13 +1,6 @@
-/*
- * Copyright 2012 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
 
 #include <dt-bindings/gpio/gpio.h>
 #include "imx28-pinfunc.h"
diff --git a/arch/arm/boot/dts/imx31.dtsi b/arch/arm/boot/dts/imx31.dtsi
index ebc3f2dbb6fd..4642c8169a65 100644
--- a/arch/arm/boot/dts/imx31.dtsi
+++ b/arch/arm/boot/dts/imx31.dtsi
@@ -1,13 +1,6 @@
-/*
- * Copyright 2012 Denis 'GNUtoo' Carikli <GNUtoo@no-log.org>
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Denis 'GNUtoo' Carikli <GNUtoo@no-log.org>
 
 / {
 	#address-cells = <1>;
diff --git a/arch/arm/boot/dts/imx35-pdk.dts b/arch/arm/boot/dts/imx35-pdk.dts
index 646b1257bba2..df613e88fd2c 100644
--- a/arch/arm/boot/dts/imx35-pdk.dts
+++ b/arch/arm/boot/dts/imx35-pdk.dts
@@ -1,14 +1,7 @@
-/*
- * Copyright 2013 Eukréa Electromatique <denis@eukrea.com>
- * Copyright 2014 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2013 Eukréa Electromatique <denis@eukrea.com>
+// Copyright 2014 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 #include "imx35.dtsi"
diff --git a/arch/arm/boot/dts/imx35.dtsi b/arch/arm/boot/dts/imx35.dtsi
index 54111ed218b1..1c50b785cad4 100644
--- a/arch/arm/boot/dts/imx35.dtsi
+++ b/arch/arm/boot/dts/imx35.dtsi
@@ -1,12 +1,8 @@
-/*
- * Copyright 2012 Steffen Trumtrar, Pengutronix
- *
- * based on imx27.dtsi
- *
- * This program is free software; you can redistribute it and/or modify it under
- * the terms of the GNU General Public License version 2 as published by the
- * Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2012 Steffen Trumtrar, Pengutronix
+//
+// based on imx27.dtsi
 
 #include "imx35-pinfunc.h"
 
diff --git a/arch/arm/boot/dts/imx50-evk.dts b/arch/arm/boot/dts/imx50-evk.dts
index 23f1833e23fa..f0622ec4ba9c 100644
--- a/arch/arm/boot/dts/imx50-evk.dts
+++ b/arch/arm/boot/dts/imx50-evk.dts
@@ -1,15 +1,8 @@
-/*
- * Copyright 2013 Greg Ungerer <gerg@uclinux.org>
- * Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2013 Greg Ungerer <gerg@uclinux.org>
+// Copyright 2011 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 /dts-v1/;
 #include "imx50.dtsi"
diff --git a/arch/arm/boot/dts/imx50.dtsi b/arch/arm/boot/dts/imx50.dtsi
index 7954e79d0a16..a9b712db9f6c 100644
--- a/arch/arm/boot/dts/imx50.dtsi
+++ b/arch/arm/boot/dts/imx50.dtsi
@@ -60,9 +60,6 @@
 	};
 
 	clocks {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		ckil {
 			compatible = "fsl,imx-ckil", "fixed-clock";
 			#clock-cells = <0>;
diff --git a/arch/arm/boot/dts/imx51-babbage.dts b/arch/arm/boot/dts/imx51-babbage.dts
index cf7a1963df25..b8ca73d3d379 100644
--- a/arch/arm/boot/dts/imx51-babbage.dts
+++ b/arch/arm/boot/dts/imx51-babbage.dts
@@ -1,14 +1,7 @@
-/*
- * Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2011 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 /dts-v1/;
 #include "imx51.dtsi"
diff --git a/arch/arm/boot/dts/imx51-zii-rdu1.dts b/arch/arm/boot/dts/imx51-zii-rdu1.dts
index 6464f2560e06..df9eca94d812 100644
--- a/arch/arm/boot/dts/imx51-zii-rdu1.dts
+++ b/arch/arm/boot/dts/imx51-zii-rdu1.dts
@@ -207,8 +207,6 @@
 
 		switch@0 {
 			compatible = "marvell,mv88e6085";
-			#address-cells = <1>;
-			#size-cells = <0>;
 			reg = <0>;
 			dsa,member = <0 0>;
 
@@ -462,7 +460,10 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_esdhc1>;
 	bus-width = <4>;
+	no-1-8-v;
 	non-removable;
+	no-sdio;
+	no-sd;
 	status = "okay";
 };
 
@@ -591,6 +592,7 @@
 	phy_type = "ulpi";
 	fsl,usbphy = <&usbh1phy>;
 	disable-over-current;
+	maximum-speed = "full-speed";
 	vbus-supply = <&reg_5p0v_main>;
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/imx51.dtsi b/arch/arm/boot/dts/imx51.dtsi
index 5d390a64e976..fe01b890c715 100644
--- a/arch/arm/boot/dts/imx51.dtsi
+++ b/arch/arm/boot/dts/imx51.dtsi
@@ -1,14 +1,7 @@
-/*
- * Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2011 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 #include "imx51-pinfunc.h"
 #include <dt-bindings/clock/imx5-clock.h>
@@ -56,9 +49,6 @@
 	};
 
 	clocks {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		ckil {
 			compatible = "fsl,imx-ckil", "fixed-clock";
 			#clock-cells = <0>;
diff --git a/arch/arm/boot/dts/imx53-ard.dts b/arch/arm/boot/dts/imx53-ard.dts
index 80fc00705d92..117bd002dd1d 100644
--- a/arch/arm/boot/dts/imx53-ard.dts
+++ b/arch/arm/boot/dts/imx53-ard.dts
@@ -11,6 +11,7 @@
  */
 
 /dts-v1/;
+#include <dt-bindings/input/input.h>
 #include "imx53.dtsi"
 
 / {
@@ -68,34 +69,34 @@
 		home {
 			label = "Home";
 			gpios = <&gpio5 10 0>;
-			linux,code = <102>; /* KEY_HOME */
+			linux,code = <KEY_HOME>;
 			wakeup-source;
 		};
 
 		back {
 			label = "Back";
 			gpios = <&gpio5 11 0>;
-			linux,code = <158>; /* KEY_BACK */
+			linux,code = <KEY_BACK>;
 			wakeup-source;
 		};
 
 		program {
 			label = "Program";
 			gpios = <&gpio5 12 0>;
-			linux,code = <362>; /* KEY_PROGRAM */
+			linux,code = <KEY_PROGRAM >;
 			wakeup-source;
 		};
 
 		volume-up {
 			label = "Volume Up";
 			gpios = <&gpio5 13 0>;
-			linux,code = <115>; /* KEY_VOLUMEUP */
+			linux,code = <KEY_VOLUMEUP>;
 		};
 
 		volume-down {
 			label = "Volume Down";
 			gpios = <&gpio4 0 0>;
-			linux,code = <114>; /* KEY_VOLUMEDOWN */
+			linux,code = <KEY_VOLUMEDOWN>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx53-m53.dtsi b/arch/arm/boot/dts/imx53-m53.dtsi
index 3da6dd5edb79..ce45f08e3051 100644
--- a/arch/arm/boot/dts/imx53-m53.dtsi
+++ b/arch/arm/boot/dts/imx53-m53.dtsi
@@ -53,8 +53,6 @@
 
 	stmpe610@41 {
 		compatible = "st,stmpe610";
-		#address-cells = <1>;
-		#size-cells = <0>;
 		reg = <0x41>;
 		id = <0>;
 		blocks = <0x5>;
diff --git a/arch/arm/boot/dts/imx53-ppd.dts b/arch/arm/boot/dts/imx53-ppd.dts
index d5628af2e301..3aa6f693fa9f 100644
--- a/arch/arm/boot/dts/imx53-ppd.dts
+++ b/arch/arm/boot/dts/imx53-ppd.dts
@@ -180,8 +180,6 @@
 
 	power-gpio-keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
 
 		power-button {
 			label = "Power button";
@@ -192,8 +190,6 @@
 
 	touch-lock-key {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
 
 		touch-lock-button {
 			label = "Touch lock button";
@@ -300,7 +296,7 @@
 		compatible = "dlg,da9053-aa";
 		reg = <0>;
 		interrupt-parent = <&gpio3>;
-		interrupts = <12 0x8>;
+		interrupts = <12 IRQ_TYPE_LEVEL_LOW>;
 		spi-max-frequency = <1000000>;
 		dlg,tsi-as-adc;
 		tsiref-supply = <&reg_tsiref>;
@@ -473,7 +469,7 @@
 				compatible = "fsl,mma8453";
 				reg = <0x1c>;
 				interrupt-parent = <&gpio1>;
-				interrupts = <6 0>;
+				interrupts = <6 IRQ_TYPE_NONE>;
 				interrupt-names = "INT1";
 			};
 
@@ -539,7 +535,7 @@
 		reset-gpio = <&gpio5 19 GPIO_ACTIVE_HIGH>;
 		reg = <0x4b>;
 		interrupt-parent = <&gpio5>;
-		interrupts = <4 0x8>;
+		interrupts = <4 IRQ_TYPE_LEVEL_LOW>;
 	};
 };
 
@@ -559,8 +555,6 @@
 		status = "okay";
 
 		port@2 {
-			reg = <2>;
-
 			lvds0_out: endpoint {
 				remote-endpoint = <&panel_in_lvds0>;
 			};
diff --git a/arch/arm/boot/dts/imx53-qsb-common.dtsi b/arch/arm/boot/dts/imx53-qsb-common.dtsi
index 485a69d45e1c..ef7658a78836 100644
--- a/arch/arm/boot/dts/imx53-qsb-common.dtsi
+++ b/arch/arm/boot/dts/imx53-qsb-common.dtsi
@@ -1,14 +1,7 @@
-/*
- * Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2011 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 #include "imx53.dtsi"
 
diff --git a/arch/arm/boot/dts/imx53-qsb.dts b/arch/arm/boot/dts/imx53-qsb.dts
index d3d662e37677..6831836bd726 100644
--- a/arch/arm/boot/dts/imx53-qsb.dts
+++ b/arch/arm/boot/dts/imx53-qsb.dts
@@ -1,14 +1,7 @@
-/*
- * Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2011 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 /dts-v1/;
 #include "imx53-qsb-common.dtsi"
@@ -23,7 +16,7 @@
 		compatible = "dlg,da9053-aa", "dlg,da9052";
 		reg = <0x48>;
 		interrupt-parent = <&gpio7>;
-		interrupts = <11 0x8>; /* low-level active IRQ at GPIO7_11 */
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>; /* low-level active IRQ at GPIO7_11 */
 
 		regulators {
 			buck1_reg: buck1 {
diff --git a/arch/arm/boot/dts/imx53-qsrb.dts b/arch/arm/boot/dts/imx53-qsrb.dts
index 4e103a905dc9..1bbf24ad308a 100644
--- a/arch/arm/boot/dts/imx53-qsrb.dts
+++ b/arch/arm/boot/dts/imx53-qsrb.dts
@@ -1,14 +1,7 @@
-/*
- * Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2011 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 /dts-v1/;
 
diff --git a/arch/arm/boot/dts/imx53-smd.dts b/arch/arm/boot/dts/imx53-smd.dts
index fd030128666c..462071c9ddd7 100644
--- a/arch/arm/boot/dts/imx53-smd.dts
+++ b/arch/arm/boot/dts/imx53-smd.dts
@@ -1,16 +1,10 @@
-/*
- * Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2011 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 /dts-v1/;
+#include <dt-bindings/input/input.h>
 #include "imx53.dtsi"
 
 / {
@@ -27,13 +21,13 @@
 		volume-up {
 			label = "Volume Up";
 			gpios = <&gpio2 14 0>;
-			linux,code = <115>; /* KEY_VOLUMEUP */
+			linux,code = <KEY_VOLUMEUP>;
 		};
 
 		volume-down {
 			label = "Volume Down";
 			gpios = <&gpio2 15 0>;
-			linux,code = <114>; /* KEY_VOLUMEDOWN */
+			linux,code = <KEY_VOLUMEDOWN>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx53-tx53-x03x.dts b/arch/arm/boot/dts/imx53-tx53-x03x.dts
index af8ec5e4417b..a7f77527269d 100644
--- a/arch/arm/boot/dts/imx53-tx53-x03x.dts
+++ b/arch/arm/boot/dts/imx53-tx53-x03x.dts
@@ -245,6 +245,7 @@
 		interrupts = <15 IRQ_TYPE_EDGE_FALLING>;
 		reset-gpios = <&gpio2 22 GPIO_ACTIVE_LOW>;
 		wake-gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>;
+		wakeup-source;
 	};
 
 	touchscreen: tsc2007@48 {
diff --git a/arch/arm/boot/dts/imx53-tx53.dtsi b/arch/arm/boot/dts/imx53-tx53.dtsi
index 69a2af7d6c11..54cf3e67069a 100644
--- a/arch/arm/boot/dts/imx53-tx53.dtsi
+++ b/arch/arm/boot/dts/imx53-tx53.dtsi
@@ -58,7 +58,7 @@
 		can0 = &can2; /* Make the can interface indices consistent with TX28/TX48 modules */
 		can1 = &can1;
 		ipu = &ipu;
-		reg_can_xcvr = &reg_can_xcvr;
+		reg-can-xcvr = &reg_can_xcvr;
 		usbh1 = &usbh1;
 		usbotg = &usbotg;
 	};
@@ -67,13 +67,12 @@
 		ckih1 {
 			clock-frequency = <0>;
 		};
+	};
 
-		mclk: clock@0 {
-			compatible = "fixed-clock";
-			reg = <0>;
-			#clock-cells = <0>;
-			clock-frequency = <26000000>;
-		};
+	mclk: clock-mclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <26000000>;
 	};
 
 	gpio-keys {
@@ -550,7 +549,6 @@
 };
 
 &ssi1 {
-	codec-handle = <&sgtl5000>;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/imx53-voipac-dmm-668.dtsi b/arch/arm/boot/dts/imx53-voipac-dmm-668.dtsi
index df8dafe2564d..f83a8c62ea53 100644
--- a/arch/arm/boot/dts/imx53-voipac-dmm-668.dtsi
+++ b/arch/arm/boot/dts/imx53-voipac-dmm-668.dtsi
@@ -150,7 +150,7 @@
 		compatible = "dlg,da9053-aa", "dlg,da9052";
 		reg = <0x48>;
 		interrupt-parent = <&gpio7>;
-		interrupts = <11 0x8>; /* low-level active IRQ at GPIO7_11 */
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>; /* low-level active IRQ at GPIO7_11 */
 
 		regulators {
 			buck1_reg: buck1 {
diff --git a/arch/arm/boot/dts/imx53.dtsi b/arch/arm/boot/dts/imx53.dtsi
index 3d65c0192f69..1a7a7bb3df45 100644
--- a/arch/arm/boot/dts/imx53.dtsi
+++ b/arch/arm/boot/dts/imx53.dtsi
@@ -88,9 +88,6 @@
 	};
 
 	clocks {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		ckil {
 			compatible = "fsl,imx-ckil", "fixed-clock";
 			#clock-cells = <0>;
@@ -488,6 +485,10 @@
 							remote-endpoint = <&ipu_di0_lvds0>;
 						};
 					};
+
+					port@2 {
+						reg = <2>;
+					};
 				};
 
 				lvds-channel@1 {
@@ -503,6 +504,10 @@
 							remote-endpoint = <&ipu_di1_lvds1>;
 						};
 					};
+
+					port@2 {
+						reg = <2>;
+					};
 				};
 			};
 
diff --git a/arch/arm/boot/dts/imx6dl-aristainetos2_4.dts b/arch/arm/boot/dts/imx6dl-aristainetos2_4.dts
index 7128c76d5721..29940ba215a8 100644
--- a/arch/arm/boot/dts/imx6dl-aristainetos2_4.dts
+++ b/arch/arm/boot/dts/imx6dl-aristainetos2_4.dts
@@ -78,8 +78,6 @@
 
 &ecspi1 {
 	lcd_panel: display@0 {
-		#address-cells = <1>;
-		#size-cells = <1>;
 		compatible = "lg,lg4573";
 		spi-max-frequency = <10000000>;
 		reg = <0>;
diff --git a/arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts b/arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts
index ea184d108491..3dee3af1a4c1 100644
--- a/arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts
+++ b/arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts
@@ -72,15 +72,12 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	clocks {
-		/* Fixed crystal dedicated to mcp251x */
-		clk16m: clk@1 {
-			compatible = "fixed-clock";
-			reg = <1>;
-			#clock-cells = <0>;
-			clock-frequency = <16000000>;
-			clock-output-names = "clk16m";
-		};
+	/* Fixed crystal dedicated to mcp251x */
+	clk16m: clock-16m {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <16000000>;
+		clock-output-names = "clk16m";
 	};
 
 	gpio-keys {
diff --git a/arch/arm/boot/dts/imx6dl-mamoj.dts b/arch/arm/boot/dts/imx6dl-mamoj.dts
new file mode 100644
index 000000000000..6b2d29138bed
--- /dev/null
+++ b/arch/arm/boot/dts/imx6dl-mamoj.dts
@@ -0,0 +1,224 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2018 BTicino
+ * Copyright (C) 2018 Amarula Solutions B.V.
+ */
+
+/dts-v1/;
+
+#include "imx6dl.dtsi"
+
+/ {
+	model = "BTicino i.MX6DL Mamoj board";
+	compatible = "bticino,imx6dl-mamoj", "fsl,imx6dl";
+};
+
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet>;
+	phy-mode = "mii";
+	status = "okay";
+};
+
+&i2c3 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	status = "okay";
+};
+
+&i2c4 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c4>;
+	status = "okay";
+
+	pfuze100: pmic@8 {
+		compatible = "fsl,pfuze100";
+		reg = <0x08>;
+
+		regulators {
+			/* CPU vdd_arm core */
+			sw1a_reg: sw1ab {
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1875000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			/* SOC vdd_soc */
+			sw1c_reg: sw1c {
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1875000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			/* I/O power GEN_3V3 */
+			sw2_reg: sw2 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* DDR memory */
+			sw3a_reg: sw3a {
+				regulator-min-microvolt = <400000>;
+				regulator-max-microvolt = <1975000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* DDR memory */
+			sw3b_reg: sw3b {
+				regulator-min-microvolt = <400000>;
+				regulator-max-microvolt = <1975000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* not used */
+			sw4_reg: sw4 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			/* not used */
+			swbst_reg: swbst {
+				regulator-min-microvolt = <5000000>;
+				regulator-max-microvolt = <5150000>;
+			};
+
+			/* PMIC vsnvs. EX boot mode */
+			snvs_reg: vsnvs {
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vref_reg: vrefddr {
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* not used */
+			vgen1_reg: vgen1 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1550000>;
+			};
+
+			/* not used */
+			vgen2_reg: vgen2 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1550000>;
+			};
+
+			/* not used */
+			vgen3_reg: vgen3 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			/* 1v8 general power */
+			vgen4_reg: vgen4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			/* 2v8 general power IMX6 */
+			vgen5_reg: vgen5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			/* 3v3 Ethernet */
+			vgen6_reg: vgen6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	status = "okay";
+};
+
+&usdhc3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	bus-width = <8>;
+	non-removable;
+	keep-power-in-suspend;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_enet: enetgrp {
+		fsl,pins = <
+			MX6QDL_PAD_ENET_MDIO__ENET_MDIO		0x1b0b0
+			MX6QDL_PAD_ENET_MDC__ENET_MDC		0x1b0b0
+			MX6QDL_PAD_ENET_REF_CLK__ENET_TX_CLK	0x1b0b1
+			MX6QDL_PAD_ENET_TXD0__ENET_TX_DATA0	0x1b0b0
+			MX6QDL_PAD_ENET_TXD1__ENET_TX_DATA1	0x1b0b0
+			MX6QDL_PAD_KEY_ROW2__ENET_TX_DATA2	0x1b0b0
+			MX6QDL_PAD_KEY_ROW0__ENET_TX_DATA3	0x1b0b0
+			MX6QDL_PAD_ENET_TX_EN__ENET_TX_EN	0x1b0b0
+			MX6QDL_PAD_GPIO_19__ENET_TX_ER		0x1b0b0
+			MX6QDL_PAD_GPIO_18__ENET_RX_CLK		0x1b0b1
+			MX6QDL_PAD_ENET_RXD0__ENET_RX_DATA0	0x1b0b0
+			MX6QDL_PAD_ENET_RXD1__ENET_RX_DATA1	0x1b0b0
+			MX6QDL_PAD_KEY_COL2__ENET_RX_DATA2	0x1b0b0
+			MX6QDL_PAD_KEY_COL0__ENET_RX_DATA3      0x1b0b0
+			MX6QDL_PAD_ENET_CRS_DV__ENET_RX_EN	0x1b0b0
+			MX6QDL_PAD_ENET_RX_ER__ENET_RX_ER	0x1b0b0
+			MX6QDL_PAD_KEY_COL3__ENET_CRS		0x1b0b0
+			MX6QDL_PAD_KEY_ROW1__ENET_COL		0x1b0b0
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_3__I2C3_SCL     0x4001b8b1
+			MX6QDL_PAD_GPIO_6__I2C3_SDA     0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c4: i2c4grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_7__I2C4_SCL	0x4001b8b1
+			MX6QDL_PAD_GPIO_8__I2C4_SDA	0x4001b8b1
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D24__UART3_TX_DATA	0x1b0b1
+			MX6QDL_PAD_EIM_D25__UART3_RX_DATA	0x1b0b1
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_CMD__SD3_CMD	0x17059
+			MX6QDL_PAD_SD3_CLK__SD3_CLK	0x10059
+			MX6QDL_PAD_SD3_DAT0__SD3_DATA0	0x17059
+			MX6QDL_PAD_SD3_DAT1__SD3_DATA1	0x17059
+			MX6QDL_PAD_SD3_DAT2__SD3_DATA2	0x17059
+			MX6QDL_PAD_SD3_DAT3__SD3_DATA3	0x17059
+			MX6QDL_PAD_SD3_DAT4__SD3_DATA4	0x17059
+			MX6QDL_PAD_SD3_DAT5__SD3_DATA5	0x17059
+			MX6QDL_PAD_SD3_DAT6__SD3_DATA6	0x17059
+			MX6QDL_PAD_SD3_DAT7__SD3_DATA7	0x17059
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/imx6dl-sabreauto.dts b/arch/arm/boot/dts/imx6dl-sabreauto.dts
index a6ce7b487ad7..660d52a245ba 100644
--- a/arch/arm/boot/dts/imx6dl-sabreauto.dts
+++ b/arch/arm/boot/dts/imx6dl-sabreauto.dts
@@ -1,10 +1,6 @@
-/*
- * Copyright (C) 2013 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2013 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 
diff --git a/arch/arm/boot/dts/imx6dl-sabresd.dts b/arch/arm/boot/dts/imx6dl-sabresd.dts
index 9607afe088fc..cd6bbf22a16f 100644
--- a/arch/arm/boot/dts/imx6dl-sabresd.dts
+++ b/arch/arm/boot/dts/imx6dl-sabresd.dts
@@ -1,10 +1,6 @@
-/*
- * Copyright (C) 2013 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2013 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 
diff --git a/arch/arm/boot/dts/imx6dl-udoo.dts b/arch/arm/boot/dts/imx6dl-udoo.dts
index e3713f00e819..d871cac1711f 100644
--- a/arch/arm/boot/dts/imx6dl-udoo.dts
+++ b/arch/arm/boot/dts/imx6dl-udoo.dts
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2013 Freescale Semiconductor, Inc.
  *
  * Author: Fabio Estevam <fabio.estevam@freescale.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 /dts-v1/;
 #include "imx6dl.dtsi"
diff --git a/arch/arm/boot/dts/imx6dl-wandboard-revb1.dts b/arch/arm/boot/dts/imx6dl-wandboard-revb1.dts
index 5727fa48cfd5..738db4fc7702 100644
--- a/arch/arm/boot/dts/imx6dl-wandboard-revb1.dts
+++ b/arch/arm/boot/dts/imx6dl-wandboard-revb1.dts
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2013 Freescale Semiconductor, Inc.
  *
  * Author: Fabio Estevam <fabio.estevam@freescale.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 /dts-v1/;
 #include "imx6dl.dtsi"
diff --git a/arch/arm/boot/dts/imx6dl-wandboard-revd1.dts b/arch/arm/boot/dts/imx6dl-wandboard-revd1.dts
index a72c07db7dda..51de6b4bd7d8 100644
--- a/arch/arm/boot/dts/imx6dl-wandboard-revd1.dts
+++ b/arch/arm/boot/dts/imx6dl-wandboard-revd1.dts
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2013 Freescale Semiconductor, Inc.
  *
  * Author: Fabio Estevam <fabio.estevam@freescale.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 /dts-v1/;
 #include "imx6dl.dtsi"
diff --git a/arch/arm/boot/dts/imx6dl-wandboard.dts b/arch/arm/boot/dts/imx6dl-wandboard.dts
index a09f274cd1f4..b43454deaa1a 100644
--- a/arch/arm/boot/dts/imx6dl-wandboard.dts
+++ b/arch/arm/boot/dts/imx6dl-wandboard.dts
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2013 Freescale Semiconductor, Inc.
  *
  * Author: Fabio Estevam <fabio.estevam@freescale.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 /dts-v1/;
 #include "imx6dl.dtsi"
diff --git a/arch/arm/boot/dts/imx6dl.dtsi b/arch/arm/boot/dts/imx6dl.dtsi
index 558bce81209d..b384913c34dd 100644
--- a/arch/arm/boot/dts/imx6dl.dtsi
+++ b/arch/arm/boot/dts/imx6dl.dtsi
@@ -1,12 +1,6 @@
-
-/*
- * Copyright 2013 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2013 Freescale Semiconductor, Inc.
 
 #include <dt-bindings/interrupt-controller/irq.h>
 #include "imx6dl-pinfunc.h"
diff --git a/arch/arm/boot/dts/imx6q-b850v3.dts b/arch/arm/boot/dts/imx6q-b850v3.dts
index 35edbdc7bcd1..044a5bebe1c5 100644
--- a/arch/arm/boot/dts/imx6q-b850v3.dts
+++ b/arch/arm/boot/dts/imx6q-b850v3.dts
@@ -156,8 +156,6 @@
 
 	stdp2690@72 {
 		compatible = "megachips,stdp2690-ge-b850v3-fw";
-		#address-cells = <1>;
-		#size-cells = <0>;
 		reg = <0x72>;
 
 		ports {
@@ -184,8 +182,6 @@
 
 	stdp4028@73 {
 		compatible = "megachips,stdp4028-ge-b850v3-fw";
-		#address-cells = <1>;
-		#size-cells = <0>;
 		reg = <0x73>;
 		interrupt-parent = <&gpio2>;
 		interrupts = <0 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/imx6q-ba16.dtsi b/arch/arm/boot/dts/imx6q-ba16.dtsi
index bf4bdb385de9..e903c488287b 100644
--- a/arch/arm/boot/dts/imx6q-ba16.dtsi
+++ b/arch/arm/boot/dts/imx6q-ba16.dtsi
@@ -157,7 +157,12 @@
 
 		partition@d0000 {
 			label = "spare";
-			reg = <0xd0000 0x130000>;
+			reg = <0xd0000 0x320000>;
+		};
+
+		partition@3f0000 {
+			label = "mfg";
+			reg = <0x3f0000 0x10000>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx6q-bx50v3.dtsi b/arch/arm/boot/dts/imx6q-bx50v3.dtsi
index 990e411cbca0..d3cba09be0cb 100644
--- a/arch/arm/boot/dts/imx6q-bx50v3.dtsi
+++ b/arch/arm/boot/dts/imx6q-bx50v3.dtsi
@@ -43,13 +43,10 @@
 #include "imx6q-ba16.dtsi"
 
 / {
-	clocks {
-		mclk: clock@0 {
-			compatible = "fixed-clock";
-			reg = <0>;
-			#clock-cells = <0>;
-			clock-frequency = <22000000>;
-		};
+	mclk: clock-mclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <22000000>;
 	};
 
 	gpio-poweroff {
@@ -107,8 +104,6 @@
 
 		switch@0 {
 			compatible = "marvell,mv88e6085"; /* 88e6240*/
-			#address-cells = <1>;
-			#size-cells = <0>;
 			reg = <0>;
 
 			switch_ports: ports {
diff --git a/arch/arm/boot/dts/imx6q-dhcom-pdk2.dts b/arch/arm/boot/dts/imx6q-dhcom-pdk2.dts
new file mode 100644
index 000000000000..9c61e3be2d9a
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-dhcom-pdk2.dts
@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: (GPL-2.0+)
+/*
+ * Copyright (C) 2015 DH electronics GmbH
+ * Copyright (C) 2018 Marek Vasut <marex@denx.de>
+ */
+
+/dts-v1/;
+
+#include "imx6q-dhcom-som.dtsi"
+
+/ {
+	model = "Freescale i.MX6 Quad DHCOM Premium Developer Kit (2)";
+	compatible = "dh,imx6q-dhcom-pdk2", "dh,imx6q-dhcom-som", "fsl,imx6q";
+
+	chosen {
+		stdout-path = &uart1;
+	};
+
+	clk_ext_audio_codec: clock-codec {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+	};
+
+	sound {
+		compatible = "fsl,imx-audio-sgtl5000";
+		model = "imx-sgtl5000";
+		ssi-controller = <&ssi1>;
+		audio-codec = <&sgtl5000>;
+		audio-routing =
+			"MIC_IN", "Mic Jack",
+			"Mic Jack", "Mic Bias",
+			"LINE_IN", "Line In Jack",
+			"Headphone Jack", "HP_OUT";
+		mux-int-port = <1>;
+		mux-ext-port = <3>;
+	};
+};
+
+&audmux {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_audmux_ext>;
+	status = "okay";
+};
+
+&hdmi {
+	ddc-i2c-bus = <&i2c2>;
+	status = "okay";
+};
+
+&i2c2 {
+	sgtl5000: codec@a {
+		compatible = "fsl,sgtl5000";
+		reg = <0x0a>;
+		#sound-dai-cells = <0>;
+		clocks = <&clk_ext_audio_codec>;
+		VDDA-supply = <&reg_3p3v>;
+		VDDIO-supply = <&reg_3p3v>;
+	};
+};
+
+&iomuxc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hog_base &pinctrl_hog>;
+
+	pinctrl_hog: hog-grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_2__GPIO1_IO02		0x400120b0
+			MX6QDL_PAD_GPIO_4__GPIO1_IO04		0x400120b0
+			MX6QDL_PAD_GPIO_5__GPIO1_IO05		0x400120b0
+			MX6QDL_PAD_CSI0_DAT17__GPIO6_IO03	0x400120b0
+			MX6QDL_PAD_GPIO_19__GPIO4_IO05		0x120b0
+			MX6QDL_PAD_DI0_PIN4__GPIO4_IO20		0x400120b0
+			MX6QDL_PAD_EIM_D27__GPIO3_IO27		0x120b0
+			MX6QDL_PAD_KEY_ROW0__GPIO4_IO07		0x120b0
+			MX6QDL_PAD_KEY_COL1__GPIO4_IO08		0x400120b0
+			MX6QDL_PAD_NANDF_CS1__GPIO6_IO14	0x400120b0
+			MX6QDL_PAD_NANDF_CS2__GPIO6_IO15	0x400120b0
+			MX6QDL_PAD_KEY_ROW1__GPIO4_IO09		0x400120b0
+			MX6QDL_PAD_SD3_DAT5__GPIO7_IO00		0x400120b0
+			MX6QDL_PAD_SD3_DAT4__GPIO7_IO01		0x400120b0
+			MX6QDL_PAD_CSI0_VSYNC__GPIO5_IO21	0x400120b0
+			MX6QDL_PAD_GPIO_18__GPIO7_IO13		0x400120b0
+			MX6QDL_PAD_SD1_CMD__GPIO1_IO18		0x400120b0
+			MX6QDL_PAD_SD1_DAT0__GPIO1_IO16		0x400120b0
+			MX6QDL_PAD_SD1_DAT1__GPIO1_IO17		0x400120b0
+			MX6QDL_PAD_SD1_DAT2__GPIO1_IO19		0x400120b0
+			MX6QDL_PAD_SD1_CLK__GPIO1_IO20		0x400120b0
+			MX6QDL_PAD_CSI0_PIXCLK__GPIO5_IO18	0x400120b0
+			MX6QDL_PAD_CSI0_MCLK__GPIO5_IO19	0x400120b0
+			MX6QDL_PAD_KEY_COL0__GPIO4_IO06		0x400120b0
+		>;
+	};
+
+	pinctrl_audmux_ext: audmux-ext-grp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT7__AUD3_RXD		0x130b0
+			MX6QDL_PAD_CSI0_DAT4__AUD3_TXC		0x130b0
+			MX6QDL_PAD_CSI0_DAT5__AUD3_TXD		0x110b0
+			MX6QDL_PAD_CSI0_DAT6__AUD3_TXFS		0x130b0
+		>;
+	};
+
+	pinctrl_enet_1G: enet-1G-grp {
+		fsl,pins = <
+			MX6QDL_PAD_ENET_MDIO__ENET_MDIO		0x100b0
+			MX6QDL_PAD_ENET_MDC__ENET_MDC		0x100b0
+			MX6QDL_PAD_RGMII_TXC__RGMII_TXC		0x100b0
+			MX6QDL_PAD_RGMII_TD0__RGMII_TD0		0x100b0
+			MX6QDL_PAD_RGMII_TD1__RGMII_TD1		0x100b0
+			MX6QDL_PAD_RGMII_TD2__RGMII_TD2		0x100b0
+			MX6QDL_PAD_RGMII_TD3__RGMII_TD3		0x100b0
+			MX6QDL_PAD_RGMII_TX_CTL__RGMII_TX_CTL	0x100b0
+			MX6QDL_PAD_ENET_REF_CLK__ENET_TX_CLK	0x100b0
+			MX6QDL_PAD_RGMII_RXC__RGMII_RXC		0x1b0b0
+			MX6QDL_PAD_RGMII_RD0__RGMII_RD0		0x1b0b0
+			MX6QDL_PAD_RGMII_RD1__RGMII_RD1		0x1b0b0
+			MX6QDL_PAD_RGMII_RD2__RGMII_RD2		0x1b0b0
+			MX6QDL_PAD_RGMII_RD3__RGMII_RD3		0x1b0b0
+			MX6QDL_PAD_RGMII_RX_CTL__RGMII_RX_CTL	0x1b0b0
+			MX6QDL_PAD_EIM_D29__GPIO3_IO29		0x000b0
+			MX6QDL_PAD_GPIO_0__GPIO1_IO00		0x000b1
+			MX6QDL_PAD_EIM_D26__GPIO3_IO26		0x000b1
+		>;
+	};
+
+	pinctrl_pcie: pcie-grp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DATA_EN__GPIO5_IO20	0x1b0b1
+		>;
+	};
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pcie>;
+	reset-gpio = <&gpio6 14 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&ssi1 {
+	status = "okay";
+};
+
+&sata {
+	status = "okay";
+};
+
+&usdhc3 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6q-dhcom-som.dtsi b/arch/arm/boot/dts/imx6q-dhcom-som.dtsi
new file mode 100644
index 000000000000..bbba0671f0f4
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-dhcom-som.dtsi
@@ -0,0 +1,476 @@
+// SPDX-License-Identifier: (GPL-2.0+)
+/*
+ * Copyright (C) 2015 DH electronics GmbH
+ * Copyright (C) 2018 Marek Vasut <marex@denx.de>
+ */
+
+#include "imx6q.dtsi"
+#include <dt-bindings/pwm/pwm.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/clock/imx6qdl-clock.h>
+#include <dt-bindings/input/input.h>
+
+/ {
+	aliases {
+		mmc0 = &usdhc2;
+		mmc1 = &usdhc3;
+		mmc2 = &usdhc4;
+		mmc3 = &usdhc1;
+	};
+
+	memory@10000000 {
+		reg = <0x10000000 0x40000000>;
+	};
+
+	reg_usb_otg_vbus: regulator-usb-otg-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_otg_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	reg_usb_h1_vbus: regulator-usb-h1-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_h1_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio3 31 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_3p3v: regulator-3P3V {
+		compatible = "regulator-fixed";
+		regulator-name = "3P3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+};
+
+&can1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	status = "okay";
+};
+
+&can2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	status = "okay";
+};
+
+&ecspi1 {
+	cs-gpios = <&gpio2 30 GPIO_ACTIVE_HIGH>, <&gpio4 11 GPIO_ACTIVE_HIGH>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	status = "okay";
+
+	flash@0 {	/* S25FL116K */
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "jedec,spi-nor";
+		spi-max-frequency = <50000000>;
+		reg = <0>;
+		m25p,fast-read;
+	};
+};
+
+&ecspi2 {
+	cs-gpios = <&gpio5 29 GPIO_ACTIVE_LOW>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi2>;
+	status = "okay";
+};
+
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet_100M>;
+	phy-mode = "rmii";
+	phy-handle = <&ethphy0>;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {	/* SMSC LAN8710Ai */
+			reg = <0>;
+			max-speed = <100>;
+			reset-gpios = <&gpio5 0 GPIO_ACTIVE_LOW>;
+			reset-delay-us = <1000>;
+			reset-post-delay-us = <1000>;
+		};
+	};
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	status = "okay";
+};
+
+&i2c2 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	status = "okay";
+};
+
+&i2c3 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	status = "okay";
+
+	ltc3676: pmic@3c {
+		compatible = "lltc,ltc3676";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pmic_hw300>;
+		reg = <0x3c>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <2 IRQ_TYPE_EDGE_FALLING>;
+
+		regulators {
+			sw1_reg: sw1 {
+				regulator-min-microvolt = <787500>;
+				regulator-max-microvolt = <1527272>;
+				lltc,fb-voltage-divider = <100000 110000>;
+				regulator-suspend-mem-microvolt = <1040000>;
+				regulator-ramp-delay = <7000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			sw2_reg: sw2 {
+				regulator-min-microvolt = <1885714>;
+				regulator-max-microvolt = <3657142>;
+				lltc,fb-voltage-divider = <100000 28000>;
+				regulator-ramp-delay = <7000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			sw3_reg: sw3 {
+				regulator-min-microvolt = <787500>;
+				regulator-max-microvolt = <1527272>;
+				lltc,fb-voltage-divider = <100000 110000>;
+				regulator-suspend-mem-microvolt = <980000>;
+				regulator-ramp-delay = <7000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			sw4_reg: sw4 {
+				regulator-min-microvolt = <855571>;
+				regulator-max-microvolt = <1659291>;
+				lltc,fb-voltage-divider = <100000 93100>;
+				regulator-ramp-delay = <7000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo1_reg: ldo1 {
+				regulator-min-microvolt = <3240306>;
+				regulator-max-microvolt = <3240306>;
+				lltc,fb-voltage-divider = <102000 29400>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo2_reg: ldo2 {
+				regulator-min-microvolt = <2484708>;
+				regulator-max-microvolt = <2484708>;
+				lltc,fb-voltage-divider = <100000 41200>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
+
+	touchscreen@49 {	/* TSC2004 */
+		compatible = "ti,tsc2004";
+		reg = <0x49>;
+		vio-supply = <&reg_3p3v>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_tsc2004_hw300>;
+		interrupts-extended = <&gpio4 14 IRQ_TYPE_EDGE_FALLING>;
+		status = "disabled";
+	};
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+
+	rtc@56 {
+		compatible = "rv3029c2";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_rtc_hw300>;
+		reg = <0x56>;
+		interrupt-parent = <&gpio7>;
+		interrupts = <12 2>;
+	};
+};
+
+&iomuxc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hog_base>;
+
+	pinctrl_hog_base: hog-base-grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_A19__GPIO2_IO19		0x120b0
+			MX6QDL_PAD_EIM_A23__GPIO6_IO06		0x120b0
+			MX6QDL_PAD_EIM_A22__GPIO2_IO16		0x120b0
+			MX6QDL_PAD_EIM_A16__GPIO2_IO22		0x120b0
+			MX6QDL_PAD_EIM_A17__GPIO2_IO21		0x120b0
+		>;
+	};
+
+	pinctrl_ecspi1: ecspi1-grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D17__ECSPI1_MISO		0x100b1
+			MX6QDL_PAD_EIM_D18__ECSPI1_MOSI		0x100b1
+			MX6QDL_PAD_EIM_D16__ECSPI1_SCLK		0x100b1
+			MX6QDL_PAD_EIM_EB2__GPIO2_IO30		0x1b0b0
+			MX6QDL_PAD_KEY_ROW2__GPIO4_IO11		0x1b0b0
+		>;
+	};
+
+	pinctrl_ecspi2: ecspi2-grp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT10__ECSPI2_MISO	0x100b1
+			MX6QDL_PAD_CSI0_DAT9__ECSPI2_MOSI	0x100b1
+			MX6QDL_PAD_CSI0_DAT8__ECSPI2_SCLK	0x100b1
+			MX6QDL_PAD_CSI0_DAT11__GPIO5_IO29	0x1b0b0
+		>;
+	};
+
+	pinctrl_enet_100M: enet-100M-grp {
+		fsl,pins = <
+			MX6QDL_PAD_ENET_MDIO__ENET_MDIO		0x1b0b0
+			MX6QDL_PAD_ENET_MDC__ENET_MDC		0x1b0b0
+			MX6QDL_PAD_ENET_CRS_DV__ENET_RX_EN	0x1b0b0
+			MX6QDL_PAD_ENET_RX_ER__ENET_RX_ER	0x1b0b0
+			MX6QDL_PAD_ENET_RXD0__ENET_RX_DATA0	0x1b0b0
+			MX6QDL_PAD_ENET_RXD1__ENET_RX_DATA1	0x1b0b0
+			MX6QDL_PAD_ENET_TX_EN__ENET_TX_EN	0x1b0b0
+			MX6QDL_PAD_ENET_TXD0__ENET_TX_DATA0	0x1b0b0
+			MX6QDL_PAD_ENET_TXD1__ENET_TX_DATA1	0x1b0b0
+			MX6QDL_PAD_GPIO_16__ENET_REF_CLK	0x4001b0a8
+			MX6QDL_PAD_EIM_WAIT__GPIO5_IO00		0x000b0
+			MX6QDL_PAD_KEY_ROW4__GPIO4_IO15		0x000b1
+			MX6QDL_PAD_GPIO_7__GPIO1_IO07		0x120b0
+		>;
+	};
+
+	pinctrl_flexcan1: flexcan1-grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL2__FLEXCAN1_TX	0x1b0b0
+			MX6QDL_PAD_GPIO_8__FLEXCAN1_RX		0x1b0b0
+		>;
+	};
+
+	pinctrl_flexcan2: flexcan2-grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_DAT0__FLEXCAN2_TX	0x1b0b0
+			MX6QDL_PAD_SD3_DAT1__FLEXCAN2_RX	0x1b0b0
+		>;
+	};
+
+	pinctrl_i2c1: i2c1-grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D21__I2C1_SCL		0x4001b8b1
+			MX6QDL_PAD_EIM_D28__I2C1_SDA		0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c2: i2c2-grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL3__I2C2_SCL		0x4001b8b1
+			MX6QDL_PAD_KEY_ROW3__I2C2_SDA		0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c3: i2c3-grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_3__I2C3_SCL		0x4001b8b1
+			MX6QDL_PAD_GPIO_6__I2C3_SDA		0x4001b8b1
+		>;
+	};
+
+	pinctrl_pmic_hw300: pmic-hw300-grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_A25__GPIO5_IO02		0x1B0B0
+		>;
+	};
+
+	pinctrl_rtc_hw300: rtc-hw300-grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_17__GPIO7_IO12		0x120B0
+		>;
+	};
+
+	pinctrl_tsc2004_hw300: tsc2004-hw300-grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL4__GPIO4_IO14		0x120B0
+		>;
+	};
+
+	pinctrl_uart1: uart1-grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_DAT7__UART1_TX_DATA	0x1b0b1
+			MX6QDL_PAD_SD3_DAT6__UART1_RX_DATA	0x1b0b1
+			MX6QDL_PAD_EIM_D20__UART1_RTS_B		0x1b0b1
+			MX6QDL_PAD_EIM_D19__UART1_CTS_B		0x4001b0b1
+			MX6QDL_PAD_EIM_D23__GPIO3_IO23		0x4001b0b1
+			MX6QDL_PAD_EIM_D24__GPIO3_IO24		0x4001b0b1
+			MX6QDL_PAD_EIM_D25__GPIO3_IO25		0x4001b0b1
+			MX6QDL_PAD_EIM_EB3__GPIO2_IO31		0x4001b0b1
+		>;
+	};
+
+	pinctrl_uart4: uart4-grp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT12__UART4_TX_DATA	0x1b0b1
+			MX6QDL_PAD_CSI0_DAT13__UART4_RX_DATA	0x1b0b1
+		>;
+	};
+
+	pinctrl_uart5: uart5-grp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT14__UART5_TX_DATA	0x1b0b1
+			MX6QDL_PAD_CSI0_DAT15__UART5_RX_DATA	0x1b0b1
+			MX6QDL_PAD_CSI0_DAT18__UART5_RTS_B	0x1b0b1
+			MX6QDL_PAD_CSI0_DAT19__UART5_CTS_B	0x4001b0b1
+		>;
+	};
+
+	pinctrl_usbh1: usbh1-grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D31__GPIO3_IO31		0x120B0
+		>;
+	};
+
+	pinctrl_usbotg: usbotg-grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_1__USB_OTG_ID		0x17059
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2-grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD2_CMD__SD2_CMD		0x17059
+			MX6QDL_PAD_SD2_CLK__SD2_CLK		0x10059
+			MX6QDL_PAD_SD2_DAT0__SD2_DATA0		0x17059
+			MX6QDL_PAD_SD2_DAT1__SD2_DATA1		0x17059
+			MX6QDL_PAD_SD2_DAT2__SD2_DATA2		0x17059
+			MX6QDL_PAD_SD2_DAT3__SD2_DATA3		0x17059
+			MX6QDL_PAD_NANDF_CS3__GPIO6_IO16	0x120B0
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3-grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_CMD__SD3_CMD		0x17059
+			MX6QDL_PAD_SD3_CLK__SD3_CLK		0x10059
+			MX6QDL_PAD_SD3_DAT0__SD3_DATA0		0x17059
+			MX6QDL_PAD_SD3_DAT1__SD3_DATA1		0x17059
+			MX6QDL_PAD_SD3_DAT2__SD3_DATA2		0x17059
+			MX6QDL_PAD_SD3_DAT3__SD3_DATA3		0x17059
+			MX6QDL_PAD_SD3_RST__GPIO7_IO08		0x120B0
+		>;
+	};
+
+	pinctrl_usdhc4: usdhc4-grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD4_CMD__SD4_CMD		0x17059
+			MX6QDL_PAD_SD4_CLK__SD4_CLK		0x10059
+			MX6QDL_PAD_SD4_DAT0__SD4_DATA0		0x17059
+			MX6QDL_PAD_SD4_DAT1__SD4_DATA1		0x17059
+			MX6QDL_PAD_SD4_DAT2__SD4_DATA2		0x17059
+			MX6QDL_PAD_SD4_DAT3__SD4_DATA3		0x17059
+			MX6QDL_PAD_SD4_DAT4__SD4_DATA4		0x17059
+			MX6QDL_PAD_SD4_DAT5__SD4_DATA5		0x17059
+			MX6QDL_PAD_SD4_DAT6__SD4_DATA6		0x17059
+			MX6QDL_PAD_SD4_DAT7__SD4_DATA7		0x17059
+		>;
+	};
+};
+
+&reg_arm {
+	vin-supply = <&sw3_reg>;
+};
+
+&reg_soc {
+	vin-supply = <&sw1_reg>;
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	uart-has-rtscts;
+	dtr-gpios = <&gpio3 24 GPIO_ACTIVE_LOW>;
+	dsr-gpios = <&gpio3 25 GPIO_ACTIVE_LOW>;
+	dcd-gpios = <&gpio3 23 GPIO_ACTIVE_LOW>;
+	rng-gpios = <&gpio2 31 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart4>;
+	status = "okay";
+};
+
+&uart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart5>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&usbh1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usbh1>;
+	vbus-supply = <&reg_usb_h1_vbus>;
+	dr_mode = "host";
+	status = "okay";
+};
+
+&usbotg {
+	vbus-supply = <&reg_usb_otg_vbus>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usbotg>;
+	disable-over-current;
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&usdhc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	cd-gpios = <&gpio6 16 GPIO_ACTIVE_HIGH>;
+	keep-power-in-suspend;
+	status = "okay";
+};
+
+&usdhc3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	cd-gpios = <&gpio7 8 GPIO_ACTIVE_LOW>;
+	fsl,wp-controller;
+	keep-power-in-suspend;
+	status = "disabled";
+};
+
+&usdhc4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc4>;
+	non-removable;
+	bus-width = <8>;
+	no-1-8-v;
+	keep-power-in-suspend;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6q-gk802.dts b/arch/arm/boot/dts/imx6q-gk802.dts
index 0be375611382..84d3540b3a97 100644
--- a/arch/arm/boot/dts/imx6q-gk802.dts
+++ b/arch/arm/boot/dts/imx6q-gk802.dts
@@ -8,6 +8,7 @@
 
 /dts-v1/;
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
 #include "imx6q.dtsi"
 
 / {
@@ -43,7 +44,7 @@
 		recovery-button {
 			label = "recovery";
 			gpios = <&gpio3 16 1>;
-			linux,code = <0x198>; /* KEY_RESTART */
+			linux,code = <KEY_RESTART>;
 			wakeup-source;
 		};
 	};
diff --git a/arch/arm/boot/dts/imx6q-icore-mipi.dts b/arch/arm/boot/dts/imx6q-icore-mipi.dts
new file mode 100644
index 000000000000..acd3d33476d4
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-icore-mipi.dts
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2017 Engicam S.r.l.
+ * Copyright (C) 2017 Amarula Solutions B.V.
+ * Author: Jagan Teki <jagan@amarulasolutions.com>
+ */
+
+/dts-v1/;
+
+#include "imx6q.dtsi"
+#include "imx6qdl-icore.dtsi"
+
+/ {
+	model = "Engicam i.CoreM6 Quad/Dual MIPI Starter Kit";
+	compatible = "engicam,imx6-icore", "fsl,imx6q";
+};
+
+&hdmi {
+	ddc-i2c-bus = <&i2c2>;
+	status = "okay";
+};
+
+&usdhc3 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6q-icore-ofcap12.dts b/arch/arm/boot/dts/imx6q-icore-ofcap12.dts
index 9e230f56c5fb..6e27c8143f82 100644
--- a/arch/arm/boot/dts/imx6q-icore-ofcap12.dts
+++ b/arch/arm/boot/dts/imx6q-icore-ofcap12.dts
@@ -48,28 +48,31 @@
 / {
 	model = "Engicam i.CoreM6 Quad/Dual OpenFrame Capacitive touch 12 Kit";
 	compatible = "engicam,imx6-icore", "fsl,imx6q";
+
+	panel {
+		compatible = "koe,tx31d200vm0baa";
+		backlight = <&backlight_lvds>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&lvds0_out>;
+			};
+		};
+	};
 };
 
 &ldb {
 	status = "okay";
 
 	lvds-channel@0 {
-		fsl,data-mapping = "spwg";
-		fsl,data-width = <18>;
+		reg = <0>;
 		status = "okay";
 
-		display-timings {
-			native-mode = <&timing0>;
-			timing0: timing0 {
-				clock-frequency = <46800000>;
-				hactive = <1280>;
-				vactive = <480>;
-				hback-porch = <353>;
-				hfront-porch = <47>;
-				vback-porch = <39>;
-				vfront-porch = <4>;
-				hsync-len = <8>;
-				vsync-len = <2>;
+		port@4 {
+			reg = <4>;
+
+			lvds0_out: endpoint {
+				remote-endpoint = <&panel_in>;
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/imx6q-kp-tpc.dts b/arch/arm/boot/dts/imx6q-kp-tpc.dts
new file mode 100644
index 000000000000..302d8d06e4cc
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-kp-tpc.dts
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2018
+ * Lukasz Majewski, DENX Software Engineering, lukma@denx.de
+ */
+
+/dts-v1/;
+
+#include "imx6q-kp.dtsi"
+
+/ {
+	model = "Freescale i.MX6 Qwuad K+P TPC Board";
+	compatible = "kiebackpeter,imx6q-tpc", "fsl,imx6q";
+
+	memory@10000000 {
+		reg = <0x10000000 0x40000000>;
+	};
+};
+
+&ipu1_di0_disp0 {
+	remote-endpoint = <&lcd_display_in>;
+};
diff --git a/arch/arm/boot/dts/imx6q-kp.dtsi b/arch/arm/boot/dts/imx6q-kp.dtsi
new file mode 100644
index 000000000000..24c8169baf44
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-kp.dtsi
@@ -0,0 +1,432 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2018
+ * Lukasz Majewski, DENX Software Engineering, lukma@denx.de
+ */
+
+/dts-v1/;
+
+#include "imx6q.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pwm/pwm.h>
+#include <dt-bindings/sound/fsl-imx-audmux.h>
+
+/ {
+	backlight_lcd: backlight-lcd {
+		compatible = "pwm-backlight";
+		pwms = <&pwm1 0 5000000>;
+		brightness-levels = <0 255>;
+		num-interpolated-steps = <255>;
+		default-brightness-level = <250>;
+	};
+
+	beeper {
+		compatible = "pwm-beeper";
+		pwms = <&pwm2 0 500000>;
+	};
+
+	lcd_display: display {
+		compatible = "fsl,imx-parallel-display";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		interface-pix-fmt = "rgb24";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_ipu1>;
+
+		port@0 {
+			reg = <0>;
+
+			lcd_display_in: endpoint {
+				remote-endpoint = <&ipu1_di0_disp0>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+
+			lcd_display_out: endpoint {
+				remote-endpoint = <&lcd_panel_in>;
+			};
+		};
+	};
+
+	lcd_panel: lcd-panel {
+		compatible = "auo,g070vvn01";
+		backlight = <&backlight_lcd>;
+		power-supply = <&reg_display>;
+
+		port {
+			lcd_panel_in: endpoint {
+				remote-endpoint = <&lcd_display_out>;
+			};
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		green {
+			label = "led1";
+			gpios = <&gpio3 16 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "gpio";
+			default-state = "off";
+		};
+
+		red {
+			label = "led0";
+			gpios = <&gpio3 23 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "gpio";
+			default-state = "off";
+		};
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "3P3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_audio: regulator-audio {
+		compatible = "regulator-fixed";
+		regulator-name = "sgtl5000-supply";
+		gpio = <&gpio6 31 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
+	reg_display: regulator-display {
+		compatible = "regulator-fixed";
+		regulator-name = "display-supply";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_usb_h1_vbus: regulator-usb_h1_vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_h1_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		enable-active-high;
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "imx6q-sgtl5000-audio";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&codec_dai>;
+		simple-audio-card,frame-master = <&codec_dai>;
+
+		cpu_dai: simple-audio-card,cpu {
+			sound-dai = <&ssi1>;
+		};
+
+		codec_dai: simple-audio-card,codec {
+			sound-dai = <&sgtl5000>;
+		};
+	};
+};
+
+&audmux {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_audmux>;
+	status = "okay";
+
+	ssi1 {
+		fsl,audmux-port = <0>;
+		fsl,port-config = <
+			(IMX_AUDMUX_V2_PTCR_SYN |
+			IMX_AUDMUX_V2_PTCR_TFSEL(2) |
+			IMX_AUDMUX_V2_PTCR_TCSEL(2) |
+			IMX_AUDMUX_V2_PTCR_TFSDIR |
+			IMX_AUDMUX_V2_PTCR_TCLKDIR)
+			IMX_AUDMUX_V2_PDCR_RXDSEL(2)
+		>;
+	};
+
+	aud3 {
+		fsl,audmux-port = <2>;
+		fsl,port-config = <
+			IMX_AUDMUX_V2_PTCR_SYN
+			IMX_AUDMUX_V2_PDCR_RXDSEL(0)
+		>;
+	};
+};
+
+&can1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+};
+
+&can2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+};
+
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet>;
+	phy-mode = "rgmii";
+	fsl,magic-packet;
+	status = "okay";
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	status = "okay";
+
+	touchscreen@5d {
+		compatible = "goodix,gt911";
+		reg = <0x5d>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_ts>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <9 IRQ_TYPE_EDGE_FALLING>;
+		irq-gpios = <&gpio1 9 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpio5 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	ds1307: rtc@32 {
+		compatible = "dallas,ds1307";
+		reg = <0x32>;
+	};
+};
+
+&i2c2 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	status = "okay";
+
+	sgtl5000: audio-codec@a {
+		compatible = "fsl,sgtl5000";
+		#sound-dai-cells = <0>;
+		reg = <0x0a>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_codec>;
+		clocks = <&clks IMX6QDL_CLK_CKO>;
+		VDDA-supply = <&reg_3p3v>;
+		VDDIO-supply = <&reg_3p3v>;
+	};
+};
+
+&iomuxc {
+	pinctrl_audmux: audmuxgrp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT7__AUD3_RXD		0x130b0
+			MX6QDL_PAD_CSI0_DAT4__AUD3_TXC		0x130b0
+			MX6QDL_PAD_CSI0_DAT5__AUD3_TXD		0x110b0
+			MX6QDL_PAD_CSI0_DAT6__AUD3_TXFS	0x130b0
+		>;
+	};
+
+	pinctrl_codec: codecgrp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_BCLK__GPIO6_IO31   0x1b0b0
+			/* sgtl5000 sys_mclk clock routed to CLKO1 */
+			MX6QDL_PAD_GPIO_0__CCM_CLKO1      0x000b0
+		>;
+	};
+
+	pinctrl_enet: enetgrp {
+		fsl,pins = <
+			MX6QDL_PAD_ENET_MDIO__ENET_MDIO	0x1b0b0
+			MX6QDL_PAD_ENET_MDC__ENET_MDC		0x1b0b0
+			MX6QDL_PAD_RGMII_TXC__RGMII_TXC	0x1b0b0
+			MX6QDL_PAD_RGMII_TD0__RGMII_TD0	0x1b0b0
+			MX6QDL_PAD_RGMII_TD1__RGMII_TD1	0x1b0b0
+			MX6QDL_PAD_RGMII_TD2__RGMII_TD2	0x1b0b0
+			MX6QDL_PAD_RGMII_TD3__RGMII_TD3	0x1b0b0
+			MX6QDL_PAD_RGMII_TX_CTL__RGMII_TX_CTL	0x1b0b0
+			MX6QDL_PAD_ENET_REF_CLK__ENET_TX_CLK	0x1b0b0
+			MX6QDL_PAD_RGMII_RXC__RGMII_RXC	0x1b0b0
+			MX6QDL_PAD_RGMII_RD0__RGMII_RD0	0x1b0b0
+			MX6QDL_PAD_RGMII_RD1__RGMII_RD1	0x1b0b0
+			MX6QDL_PAD_RGMII_RD2__RGMII_RD2	0x1b0b0
+			MX6QDL_PAD_RGMII_RD3__RGMII_RD3	0x1b0b0
+			MX6QDL_PAD_RGMII_RX_CTL__RGMII_RX_CTL	0x1b0b0
+			MX6QDL_PAD_GPIO_16__ENET_REF_CLK 0x4001b0a8
+		>;
+	};
+
+	pinctrl_flexcan1: can1grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_7__FLEXCAN1_TX        0x1b0b0
+			MX6QDL_PAD_GPIO_8__FLEXCAN1_RX        0x1b0b0
+		>;
+	};
+
+	pinctrl_flexcan2: can2grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL4__FLEXCAN2_TX        0x1b0b0
+			MX6QDL_PAD_KEY_ROW4__FLEXCAN2_RX        0x1b0b0
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT8__I2C1_SDA	0x4001b8b1
+			MX6QDL_PAD_CSI0_DAT9__I2C1_SCL	0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL3__I2C2_SCL	0x4001b8b1
+			MX6QDL_PAD_KEY_ROW3__I2C2_SDA	0x4001b8b1
+		 >;
+	};
+
+	pinctrl_ipu1: ipu1grp {
+		fsl,pins = <
+			MX6QDL_PAD_DI0_DISP_CLK__IPU1_DI0_DISP_CLK 0x10
+			MX6QDL_PAD_DI0_PIN15__IPU1_DI0_PIN15       0x10
+			MX6QDL_PAD_DI0_PIN2__IPU1_DI0_PIN02        0x10
+			MX6QDL_PAD_DI0_PIN3__IPU1_DI0_PIN03        0x10
+			MX6QDL_PAD_DISP0_DAT0__IPU1_DISP0_DATA00   0x10
+			MX6QDL_PAD_DISP0_DAT1__IPU1_DISP0_DATA01   0x10
+			MX6QDL_PAD_DISP0_DAT2__IPU1_DISP0_DATA02   0x10
+			MX6QDL_PAD_DISP0_DAT3__IPU1_DISP0_DATA03   0x10
+			MX6QDL_PAD_DISP0_DAT4__IPU1_DISP0_DATA04   0x10
+			MX6QDL_PAD_DISP0_DAT5__IPU1_DISP0_DATA05   0x10
+			MX6QDL_PAD_DISP0_DAT6__IPU1_DISP0_DATA06   0x10
+			MX6QDL_PAD_DISP0_DAT7__IPU1_DISP0_DATA07   0x10
+			MX6QDL_PAD_DISP0_DAT8__IPU1_DISP0_DATA08   0x10
+			MX6QDL_PAD_DISP0_DAT9__IPU1_DISP0_DATA09   0x10
+			MX6QDL_PAD_DISP0_DAT10__IPU1_DISP0_DATA10  0x10
+			MX6QDL_PAD_DISP0_DAT11__IPU1_DISP0_DATA11  0x10
+			MX6QDL_PAD_DISP0_DAT12__IPU1_DISP0_DATA12  0x10
+			MX6QDL_PAD_DISP0_DAT13__IPU1_DISP0_DATA13  0x10
+			MX6QDL_PAD_DISP0_DAT14__IPU1_DISP0_DATA14  0x10
+			MX6QDL_PAD_DISP0_DAT15__IPU1_DISP0_DATA15  0x10
+			MX6QDL_PAD_DISP0_DAT16__IPU1_DISP0_DATA16  0x10
+			MX6QDL_PAD_DISP0_DAT17__IPU1_DISP0_DATA17  0x10
+			MX6QDL_PAD_DISP0_DAT18__IPU1_DISP0_DATA18  0x10
+			MX6QDL_PAD_DISP0_DAT19__IPU1_DISP0_DATA19  0x10
+			MX6QDL_PAD_DISP0_DAT20__IPU1_DISP0_DATA20  0x10
+			MX6QDL_PAD_DISP0_DAT21__IPU1_DISP0_DATA21  0x10
+			MX6QDL_PAD_DISP0_DAT22__IPU1_DISP0_DATA22  0x10
+			MX6QDL_PAD_DISP0_DAT23__IPU1_DISP0_DATA23  0x10
+		>;
+	};
+
+	pinctrl_pwm1: pwm1grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD1_DAT3__PWM1_OUT		0x1b0b1
+		>;
+	};
+
+	pinctrl_pwm2: pwm2grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD1_DAT2__PWM2_OUT		0x1b0b1
+		>;
+	};
+
+	pinctrl_ts: tsgrp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_9__GPIO1_IO09 0x1b0b0
+			MX6QDL_PAD_EIM_A25__GPIO5_IO02 0x1b0b0
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_DAT7__UART1_TX_DATA	0x1b0b1
+			MX6QDL_PAD_SD3_DAT6__UART1_RX_DATA	0x1b0b1
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D26__UART2_TX_DATA       0x1b0b1
+			MX6QDL_PAD_EIM_D27__UART2_RX_DATA       0x1b0b1
+			MX6QDL_PAD_EIM_D28__UART2_CTS_B         0x1b0b1
+			MX6QDL_PAD_EIM_D29__UART2_RTS_B         0x1b0b1
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD2_CMD__SD2_CMD		0x17059
+			MX6QDL_PAD_SD2_CLK__SD2_CLK		0x10059
+			MX6QDL_PAD_SD2_DAT0__SD2_DATA0		0x17059
+			MX6QDL_PAD_SD2_DAT1__SD2_DATA1		0x17059
+			MX6QDL_PAD_SD2_DAT2__SD2_DATA2		0x17059
+			MX6QDL_PAD_SD2_DAT3__SD2_DATA3		0x17059
+		>;
+	};
+
+	pinctrl_usdhc4: usdhc4grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD4_CMD__SD4_CMD		0x17059
+			MX6QDL_PAD_SD4_CLK__SD4_CLK		0x10059
+			MX6QDL_PAD_SD4_DAT0__SD4_DATA0		0x17059
+			MX6QDL_PAD_SD4_DAT1__SD4_DATA1		0x17059
+			MX6QDL_PAD_SD4_DAT2__SD4_DATA2		0x17059
+			MX6QDL_PAD_SD4_DAT3__SD4_DATA3		0x17059
+			MX6QDL_PAD_SD4_DAT4__SD4_DATA4		0x17059
+			MX6QDL_PAD_SD4_DAT5__SD4_DATA5		0x17059
+			MX6QDL_PAD_SD4_DAT6__SD4_DATA6		0x17059
+			MX6QDL_PAD_SD4_DAT7__SD4_DATA7		0x17059
+		>;
+	};
+};
+
+&pwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm1>;
+	status = "okay";
+};
+
+&pwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm2>;
+	status = "okay";
+};
+
+&ssi1 {
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	uart-has-rtscts;
+};
+
+&usbh1 {
+	status = "okay";
+};
+
+&usdhc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	bus-width = <4>;
+	cd-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&usdhc4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc4>;
+	bus-width = <8>;
+	non-removable;
+	no-1-8-v;
+	keep-power-in-suspend;
+	status = "okay";
+};
+
+&wdog1 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6q-novena.dts b/arch/arm/boot/dts/imx6q-novena.dts
index 52f39371188d..fcd824dc485b 100644
--- a/arch/arm/boot/dts/imx6q-novena.dts
+++ b/arch/arm/boot/dts/imx6q-novena.dts
@@ -268,8 +268,6 @@
 	touch: stmpe811@44 {
 		compatible = "st,stmpe811";
 		reg = <0x44>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 		irq-gpio = <&gpio5 13 GPIO_ACTIVE_HIGH>;
 		id = <0>;
 		blocks = <0x5>;
diff --git a/arch/arm/boot/dts/imx6q-pistachio.dts b/arch/arm/boot/dts/imx6q-pistachio.dts
index bd57b3b74db7..a31e83cd07a3 100644
--- a/arch/arm/boot/dts/imx6q-pistachio.dts
+++ b/arch/arm/boot/dts/imx6q-pistachio.dts
@@ -614,7 +614,7 @@
 &uart5 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart5>;
-	fsl,uart-has-rtscts;
+	uart-has-rtscts;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/imx6q-sabreauto.dts b/arch/arm/boot/dts/imx6q-sabreauto.dts
index 334b9247e78c..6e981a3e0a83 100644
--- a/arch/arm/boot/dts/imx6q-sabreauto.dts
+++ b/arch/arm/boot/dts/imx6q-sabreauto.dts
@@ -1,14 +1,7 @@
-/*
- * Copyright 2012 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 /dts-v1/;
 
diff --git a/arch/arm/boot/dts/imx6q-sabresd.dts b/arch/arm/boot/dts/imx6q-sabresd.dts
index 527772b62fee..eec944673c0b 100644
--- a/arch/arm/boot/dts/imx6q-sabresd.dts
+++ b/arch/arm/boot/dts/imx6q-sabresd.dts
@@ -1,14 +1,7 @@
-/*
- * Copyright 2012 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 /dts-v1/;
 
diff --git a/arch/arm/boot/dts/imx6q-udoo.dts b/arch/arm/boot/dts/imx6q-udoo.dts
index c3e64ff3d544..52e9f4a211d0 100644
--- a/arch/arm/boot/dts/imx6q-udoo.dts
+++ b/arch/arm/boot/dts/imx6q-udoo.dts
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2013 Freescale Semiconductor, Inc.
  *
  * Author: Fabio Estevam <fabio.estevam@freescale.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 /dts-v1/;
 #include "imx6q.dtsi"
diff --git a/arch/arm/boot/dts/imx6q-utilite-pro.dts b/arch/arm/boot/dts/imx6q-utilite-pro.dts
index f5d9c34b0d39..d16ff2083d62 100644
--- a/arch/arm/boot/dts/imx6q-utilite-pro.dts
+++ b/arch/arm/boot/dts/imx6q-utilite-pro.dts
@@ -61,8 +61,6 @@
 
 	encoder {
 		compatible = "ti,tfp410";
-		#address-cells = <1>;
-		#size-cells = <0>;
 
 		ports {
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/imx6q-var-dt6customboard.dts b/arch/arm/boot/dts/imx6q-var-dt6customboard.dts
index e0728d475f6f..f2368a073d07 100644
--- a/arch/arm/boot/dts/imx6q-var-dt6customboard.dts
+++ b/arch/arm/boot/dts/imx6q-var-dt6customboard.dts
@@ -26,8 +26,6 @@
 
 	gpio-keys {
 		compatible = "gpio-keys";
-		#address-cells = <1>;
-		#size-cells = <0>;
 		autorepeat;
 
 		back {
diff --git a/arch/arm/boot/dts/imx6q-wandboard-revb1.dts b/arch/arm/boot/dts/imx6q-wandboard-revb1.dts
index b763352cddae..be85b980bdfe 100644
--- a/arch/arm/boot/dts/imx6q-wandboard-revb1.dts
+++ b/arch/arm/boot/dts/imx6q-wandboard-revb1.dts
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2013 Freescale Semiconductor, Inc.
  *
  * Author: Fabio Estevam <fabio.estevam@freescale.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 /dts-v1/;
 #include "imx6q.dtsi"
diff --git a/arch/arm/boot/dts/imx6q-wandboard-revd1.dts b/arch/arm/boot/dts/imx6q-wandboard-revd1.dts
index 8691fab21058..fcfba28764d4 100644
--- a/arch/arm/boot/dts/imx6q-wandboard-revd1.dts
+++ b/arch/arm/boot/dts/imx6q-wandboard-revd1.dts
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2013 Freescale Semiconductor, Inc.
  *
  * Author: Fabio Estevam <fabio.estevam@freescale.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 /dts-v1/;
 #include "imx6q.dtsi"
diff --git a/arch/arm/boot/dts/imx6q-wandboard.dts b/arch/arm/boot/dts/imx6q-wandboard.dts
index 2a3d98c1489a..fa36fe183fc0 100644
--- a/arch/arm/boot/dts/imx6q-wandboard.dts
+++ b/arch/arm/boot/dts/imx6q-wandboard.dts
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2013 Freescale Semiconductor, Inc.
  *
  * Author: Fabio Estevam <fabio.estevam@freescale.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 /dts-v1/;
 #include "imx6q.dtsi"
diff --git a/arch/arm/boot/dts/imx6q.dtsi b/arch/arm/boot/dts/imx6q.dtsi
index ae7b3f107893..70483ce72ba6 100644
--- a/arch/arm/boot/dts/imx6q.dtsi
+++ b/arch/arm/boot/dts/imx6q.dtsi
@@ -1,12 +1,6 @@
-
-/*
- * Copyright 2013 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2013 Freescale Semiconductor, Inc.
 
 #include <dt-bindings/interrupt-controller/irq.h>
 #include "imx6q-pinfunc.h"
@@ -162,22 +156,27 @@
 				#size-cells = <0>;
 				reg = <2>;
 
-				ipu2_di0_disp0: disp0-endpoint {
+				ipu2_di0_disp0: endpoint@0 {
+					reg = <0>;
 				};
 
-				ipu2_di0_hdmi: hdmi-endpoint {
+				ipu2_di0_hdmi: endpoint@1 {
+					reg = <1>;
 					remote-endpoint = <&hdmi_mux_2>;
 				};
 
-				ipu2_di0_mipi: mipi-endpoint {
+				ipu2_di0_mipi: endpoint@2 {
+					reg = <2>;
 					remote-endpoint = <&mipi_mux_2>;
 				};
 
-				ipu2_di0_lvds0: lvds0-endpoint {
+				ipu2_di0_lvds0: endpoint@3 {
+					reg = <3>;
 					remote-endpoint = <&lvds0_mux_2>;
 				};
 
-				ipu2_di0_lvds1: lvds1-endpoint {
+				ipu2_di0_lvds1: endpoint@4 {
+					reg = <4>;
 					remote-endpoint = <&lvds1_mux_2>;
 				};
 			};
@@ -187,19 +186,23 @@
 				#size-cells = <0>;
 				reg = <3>;
 
-				ipu2_di1_hdmi: hdmi-endpoint {
+				ipu2_di1_hdmi: endpoint@1 {
+					reg = <1>;
 					remote-endpoint = <&hdmi_mux_3>;
 				};
 
-				ipu2_di1_mipi: mipi-endpoint {
+				ipu2_di1_mipi: endpoint@2 {
+					reg = <2>;
 					remote-endpoint = <&mipi_mux_3>;
 				};
 
-				ipu2_di1_lvds0: lvds0-endpoint {
+				ipu2_di1_lvds0: endpoint@3 {
+					reg = <3>;
 					remote-endpoint = <&lvds0_mux_3>;
 				};
 
-				ipu2_di1_lvds1: lvds1-endpoint {
+				ipu2_di1_lvds1: endpoint@4 {
+					reg = <4>;
 					remote-endpoint = <&lvds1_mux_3>;
 				};
 			};
diff --git a/arch/arm/boot/dts/imx6qdl-apalis.dtsi b/arch/arm/boot/dts/imx6qdl-apalis.dtsi
index 8206683172d2..64fbee61de44 100644
--- a/arch/arm/boot/dts/imx6qdl-apalis.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-apalis.dtsi
@@ -331,8 +331,6 @@
 		compatible = "st,stmpe811";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_touch_int>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 		reg = <0x41>;
 		interrupts = <10 IRQ_TYPE_LEVEL_LOW>;
 		interrupt-parent = <&gpio4>;
diff --git a/arch/arm/boot/dts/imx6qdl-colibri.dtsi b/arch/arm/boot/dts/imx6qdl-colibri.dtsi
index e4eb300549d4..76035db96f67 100644
--- a/arch/arm/boot/dts/imx6qdl-colibri.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-colibri.dtsi
@@ -262,8 +262,6 @@
 		compatible = "st,stmpe811";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_touch_int>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 		reg = <0x41>;
 		interrupts = <20 IRQ_TYPE_LEVEL_LOW>;
 		interrupt-parent = <&gpio6>;
diff --git a/arch/arm/boot/dts/imx6qdl-gw5904.dtsi b/arch/arm/boot/dts/imx6qdl-gw5904.dtsi
index 58124adfd65b..3c52bdb453f3 100644
--- a/arch/arm/boot/dts/imx6qdl-gw5904.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-gw5904.dtsi
@@ -162,8 +162,6 @@
 
 		switch@0 {
 			compatible = "marvell,mv88e6085";
-			#address-cells = <1>;
-			#size-cells = <0>;
 			reg = <0>;
 
 			ports {
diff --git a/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi b/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi
index 7e20b47de839..0e64016e765f 100644
--- a/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi
@@ -38,6 +38,7 @@
  *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
+#include <dt-bindings/sound/fsl-imx-audmux.h>
 
 / {
 	/* Will be filled by the bootloader */
@@ -110,17 +111,27 @@
 		vin-supply = <&v_5v0>;
 	};
 
-	sound-sgtl5000 {
-		audio-codec = <&sgtl5000>;
-		audio-routing =
-			"MIC_IN", "Mic Jack",
-			"Mic Jack", "Mic Bias",
+	audio: sound-sgtl5000 {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "On-board Codec";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&sound_codec>;
+		simple-audio-card,frame-master = <&sound_codec>;
+		simple-audio-card,widgets =
+			"Microphone", "Headphone Jack",
+			"Headphone", "Headphone Jack";
+		simple-audio-card,routing =
+			"MIC_IN", "Headphone Jack",
+			"Headphone Jack", "Mic Bias",
 			"Headphone Jack", "HP_OUT";
-		compatible = "fsl,imx-audio-sgtl5000";
-		model = "On-board Codec";
-		mux-ext-port = <5>;
-		mux-int-port = <1>;
-		ssi-controller = <&ssi1>;
+
+		sound_cpu: simple-audio-card,cpu {
+			sound-dai = <&ssi1>;
+		};
+
+		sound_codec: simple-audio-card,codec {
+			sound-dai = <&sgtl5000>;
+		};
 	};
 
 	sound-spdif {
@@ -134,6 +145,26 @@
 
 &audmux {
 	status = "okay";
+
+	ssi1 {
+		fsl,audmux-port = <0>;
+		fsl,port-config = <
+			(IMX_AUDMUX_V2_PTCR_SYN |
+			 IMX_AUDMUX_V2_PTCR_TFSEL(4) |
+			 IMX_AUDMUX_V2_PTCR_TCSEL(4) |
+			 IMX_AUDMUX_V2_PTCR_TFSDIR |
+			 IMX_AUDMUX_V2_PTCR_TCLKDIR)
+			 IMX_AUDMUX_V2_PDCR_RXDSEL(4)
+		>;
+	};
+
+	pins5 {
+		fsl,audmux-port = <4>;
+		fsl,port-config = <
+			IMX_AUDMUX_V2_PTCR_SYN
+			IMX_AUDMUX_V2_PDCR_RXDSEL(0)
+		>;
+	};
 };
 
 &can1 {
@@ -166,6 +197,7 @@
 		compatible = "fsl,sgtl5000";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_hummingboard_sgtl5000>;
+		#sound-dai-cells = <0>;
 		reg = <0x0a>;
 		VDDA-supply = <&v_3v2>;
 		VDDIO-supply = <&v_3v2>;
diff --git a/arch/arm/boot/dts/imx6qdl-hummingboard2.dtsi b/arch/arm/boot/dts/imx6qdl-hummingboard2.dtsi
index 98241acb08a6..c413f9c3540f 100644
--- a/arch/arm/boot/dts/imx6qdl-hummingboard2.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-hummingboard2.dtsi
@@ -38,6 +38,7 @@
  *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
+#include <dt-bindings/sound/fsl-imx-audmux.h>
 
 / {
 	/* Will be filled by the bootloader */
@@ -150,22 +151,52 @@
 		vin-supply = <&v_5v0>;
 	};
 
-	sound-sgtl5000 {
-		audio-codec = <&sgtl5000>;
-		audio-routing =
+	audio: sound-sgtl5000 {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "On-board Codec";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&sound_codec>;
+		simple-audio-card,frame-master = <&sound_codec>;
+		simple-audio-card,widgets =
+			"Microphone", "Mic Jack",
+			"Headphone", "Headphone Jack";
+		simple-audio-card,routing =
 			"MIC_IN", "Mic Jack",
 			"Mic Jack", "Mic Bias",
 			"Headphone Jack", "HP_OUT";
-		compatible = "fsl,imx-audio-sgtl5000";
-		model = "On-board Codec";
-		mux-ext-port = <5>;
-		mux-int-port = <1>;
-		ssi-controller = <&ssi1>;
+
+		sound_cpu: simple-audio-card,cpu {
+			sound-dai = <&ssi1>;
+		};
+
+		sound_codec: simple-audio-card,codec {
+			sound-dai = <&sgtl5000>;
+		};
 	};
 };
 
 &audmux {
 	status = "okay";
+
+	ssi1 {
+		fsl,audmux-port = <0>;
+		fsl,port-config = <
+			(IMX_AUDMUX_V2_PTCR_SYN |
+			 IMX_AUDMUX_V2_PTCR_TFSEL(4) |
+			 IMX_AUDMUX_V2_PTCR_TCSEL(4) |
+			 IMX_AUDMUX_V2_PTCR_TFSDIR |
+			 IMX_AUDMUX_V2_PTCR_TCLKDIR)
+			 IMX_AUDMUX_V2_PDCR_RXDSEL(4)
+		>;
+	};
+
+	pins5 {
+		fsl,audmux-port = <4>;
+		fsl,port-config = <
+			IMX_AUDMUX_V2_PTCR_SYN
+			IMX_AUDMUX_V2_PDCR_RXDSEL(0)
+		>;
+	};
 };
 
 &ecspi2 {
diff --git a/arch/arm/boot/dts/imx6qdl-icore.dtsi b/arch/arm/boot/dts/imx6qdl-icore.dtsi
index b3a463a5908b..0a1574998fc6 100644
--- a/arch/arm/boot/dts/imx6qdl-icore.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-icore.dtsi
@@ -49,7 +49,7 @@
 		reg = <0x10000000 0x80000000>;
 	};
 
-	backlight {
+	backlight_lvds: backlight-lvds {
 		compatible = "pwm-backlight";
 		pwms = <&pwm3 0 100000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -265,6 +265,14 @@
 	status = "okay";
 };
 
+&usdhc3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	no-1-8-v;
+	non-removable;
+	status = "disabled";
+};
+
 &iomuxc {
 	pinctrl_audmux: audmux {
 		fsl,pins = <
@@ -378,4 +386,19 @@
 			MX6QDL_PAD_SD1_DAT3__SD1_DATA3 0x17070
 		>;
 	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_CMD__SD3_CMD    0x17059
+			MX6QDL_PAD_SD3_CLK__SD3_CLK    0x10059
+			MX6QDL_PAD_SD3_DAT0__SD3_DATA0 0x17059
+			MX6QDL_PAD_SD3_DAT1__SD3_DATA1 0x17059
+			MX6QDL_PAD_SD3_DAT2__SD3_DATA2 0x17059
+			MX6QDL_PAD_SD3_DAT3__SD3_DATA3 0x17059
+			MX6QDL_PAD_SD3_DAT4__SD3_DATA4 0x17059
+			MX6QDL_PAD_SD3_DAT5__SD3_DATA5 0x17059
+			MX6QDL_PAD_SD3_DAT6__SD3_DATA6 0x17059
+			MX6QDL_PAD_SD3_DAT7__SD3_DATA7 0x17059
+		>;
+	};
 };
diff --git a/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi b/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi
index c58f3443d55d..ed1aafd56973 100644
--- a/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi
@@ -115,7 +115,7 @@
 		compatible = "dlg,da9063";
 		reg = <0x58>;
 		interrupt-parent = <&gpio2>;
-		interrupts = <9 0x8>; /* active-low GPIO2_9 */
+		interrupts = <9 IRQ_TYPE_LEVEL_LOW>; /* active-low GPIO2_9 */
 
 		regulators {
 			vddcore_reg: bcore1 {
diff --git a/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi b/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
index 54b0139e978d..0e28e36ddbb2 100644
--- a/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
@@ -1,14 +1,7 @@
-/*
- * Copyright 2012 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 #include <dt-bindings/gpio/gpio.h>
 
diff --git a/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi b/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi
index 18b65052553d..654cf2c9b073 100644
--- a/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi
@@ -379,9 +379,6 @@
 		powerdown-gpios = <&gpio6 9 GPIO_ACTIVE_HIGH>; /* NANDF_WP_B */
 
 		port {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
 			ov5640_to_mipi_csi2: endpoint {
 				remote-endpoint = <&mipi_csi2_in>;
 				clock-lanes = <0>;
diff --git a/arch/arm/boot/dts/imx6qdl-sabresd.dtsi b/arch/arm/boot/dts/imx6qdl-sabresd.dtsi
index f019f9900369..15744ad52535 100644
--- a/arch/arm/boot/dts/imx6qdl-sabresd.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-sabresd.dtsi
@@ -1,14 +1,7 @@
-/*
- * Copyright 2012 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 #include <dt-bindings/clock/imx6qdl-clock.h>
 #include <dt-bindings/gpio/gpio.h>
@@ -294,9 +287,6 @@
 		reset-gpios = <&gpio1 20 GPIO_ACTIVE_LOW>;
 
 		port {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
 			ov5640_to_mipi_csi2: endpoint {
 				remote-endpoint = <&mipi_csi2_in>;
 				clock-lanes = <0>;
diff --git a/arch/arm/boot/dts/imx6qdl-tx6-lcd.dtsi b/arch/arm/boot/dts/imx6qdl-tx6-lcd.dtsi
index 5102fc47380b..79f2354886b7 100644
--- a/arch/arm/boot/dts/imx6qdl-tx6-lcd.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-tx6-lcd.dtsi
@@ -77,7 +77,6 @@
 		enable-gpios = <&gpio3 29 GPIO_ACTIVE_HIGH>;
 		power-supply = <&reg_3v3>;
 		backlight = <&backlight>;
-		bus-format-override = "rgb24";
 
 		port {
 			lcd_panel_in: endpoint {
diff --git a/arch/arm/boot/dts/imx6qdl-tx6-mb7.dtsi b/arch/arm/boot/dts/imx6qdl-tx6-mb7.dtsi
index 4c4e2e1a931f..410972e1dca9 100644
--- a/arch/arm/boot/dts/imx6qdl-tx6-mb7.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-tx6-mb7.dtsi
@@ -54,19 +54,16 @@
 
 	lcd-panel {
 		compatible = "edt,et057090dhu";
-		bus-format-override = "rgb24";
 		pixelclk-active = <0>;
 	};
 
 	lvds0-panel {
 		compatible = "edt,etml1010g0dka";
-		bus-format-override = "spwg-18";
 		pixelclk-active = <0>;
 	};
 
 	lvds1-panel {
 		compatible = "edt,etml1010g0dka";
-		bus-format-override = "spwg-18";
 		pixelclk-active = <0>;
 	};
 };
diff --git a/arch/arm/boot/dts/imx6qdl-tx6.dtsi b/arch/arm/boot/dts/imx6qdl-tx6.dtsi
index f015e2d1cf35..a98fb2564c63 100644
--- a/arch/arm/boot/dts/imx6qdl-tx6.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-tx6.dtsi
@@ -50,11 +50,11 @@
 		can0 = &can2;
 		can1 = &can1;
 		ethernet0 = &fec;
-		lcdif_23bit_pins_a = &pinctrl_disp0_1;
-		lcdif_24bit_pins_a = &pinctrl_disp0_2;
+		lcdif-23bit-pins-a = &pinctrl_disp0_1;
+		lcdif-24bit-pins-a = &pinctrl_disp0_2;
 		pwm0 = &pwm1;
 		pwm1 = &pwm2;
-		reg_can_xcvr = &reg_can_xcvr;
+		reg-can-xcvr = &reg_can_xcvr;
 		stk5led = &user_led;
 		usbotg = &usbotg;
 		sdhc0 = &usdhc1;
diff --git a/arch/arm/boot/dts/imx6qdl-udoo.dtsi b/arch/arm/boot/dts/imx6qdl-udoo.dtsi
index 906387915dc5..4f27861bbb32 100644
--- a/arch/arm/boot/dts/imx6qdl-udoo.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-udoo.dtsi
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2013 Freescale Semiconductor, Inc.
  *
  * Author: Fabio Estevam <fabio.estevam@freescale.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 
 / {
diff --git a/arch/arm/boot/dts/imx6qdl-wandboard-revb1.dtsi b/arch/arm/boot/dts/imx6qdl-wandboard-revb1.dtsi
index a32089132263..855dc6f9df75 100644
--- a/arch/arm/boot/dts/imx6qdl-wandboard-revb1.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-wandboard-revb1.dtsi
@@ -17,7 +17,6 @@
 	imx6qdl-wandboard {
 		pinctrl_hog: hoggrp {
 			fsl,pins = <
-				MX6QDL_PAD_GPIO_0__CCM_CLKO1		0x130b0		/* GPIO_0_CLKO */
 				MX6QDL_PAD_GPIO_2__GPIO1_IO02		0x80000000	/* uSDHC1 CD */
 				MX6QDL_PAD_EIM_DA9__GPIO3_IO09		0x80000000	/* uSDHC3 CD */
 				MX6QDL_PAD_EIM_EB1__GPIO2_IO29		0x0f0b0		/* WL_REF_ON */
diff --git a/arch/arm/boot/dts/imx6qdl-wandboard-revc1.dtsi b/arch/arm/boot/dts/imx6qdl-wandboard-revc1.dtsi
index 8d893a78cdf0..49a0a557e62e 100644
--- a/arch/arm/boot/dts/imx6qdl-wandboard-revc1.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-wandboard-revc1.dtsi
@@ -17,7 +17,6 @@
 	imx6qdl-wandboard {
 		pinctrl_hog: hoggrp {
 			fsl,pins = <
-				MX6QDL_PAD_GPIO_0__CCM_CLKO1		0x130b0		/* GPIO_0_CLKO */
 				MX6QDL_PAD_GPIO_2__GPIO1_IO02		0x80000000	/* uSDHC1 CD */
 				MX6QDL_PAD_EIM_DA9__GPIO3_IO09		0x80000000	/* uSDHC3 CD */
 				MX6QDL_PAD_CSI0_DAT14__GPIO6_IO00	0x0f0b0		/* WIFI_ON (reset, active low) */
diff --git a/arch/arm/boot/dts/imx6qdl-wandboard-revd1.dtsi b/arch/arm/boot/dts/imx6qdl-wandboard-revd1.dtsi
index 3a8a4952d45e..69d9c8661439 100644
--- a/arch/arm/boot/dts/imx6qdl-wandboard-revd1.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-wandboard-revd1.dtsi
@@ -147,7 +147,6 @@
 	imx6qdl-wandboard {
 		pinctrl_hog: hoggrp {
 			fsl,pins = <
-				MX6QDL_PAD_GPIO_0__CCM_CLKO1     	0x130b0
 				MX6QDL_PAD_EIM_D22__USB_OTG_PWR		0x80000000	/* USB Power Enable */
 				MX6QDL_PAD_GPIO_2__GPIO1_IO02		0x80000000	/* USDHC1 CD */
 				MX6QDL_PAD_EIM_DA9__GPIO3_IO09		0x80000000	/* uSDHC3 CD */
diff --git a/arch/arm/boot/dts/imx6qdl-wandboard.dtsi b/arch/arm/boot/dts/imx6qdl-wandboard.dtsi
index ed96d7b5feab..e1afa54404d0 100644
--- a/arch/arm/boot/dts/imx6qdl-wandboard.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-wandboard.dtsi
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2013 Freescale Semiconductor, Inc.
  *
  * Author: Fabio Estevam <fabio.estevam@freescale.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 
 #include <dt-bindings/gpio/gpio.h>
@@ -83,6 +79,8 @@
 	status = "okay";
 
 	codec: sgtl5000@a {
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_mclk>;
 		compatible = "fsl,sgtl5000";
 		reg = <0x0a>;
 		clocks = <&clks IMX6QDL_CLK_CKO>;
@@ -142,6 +140,12 @@
 			>;
 		};
 
+		pinctrl_mclk: mclkgrp {
+			fsl,pins = <
+				MX6QDL_PAD_GPIO_0__CCM_CLKO1		0x130b0
+			>;
+		};
+
 		pinctrl_spdif: spdifgrp {
 			fsl,pins = <
 				MX6QDL_PAD_ENET_RXD0__SPDIF_OUT		0x1b0b0
diff --git a/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi b/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi
index 911f7f0e3cea..19a075aee19e 100644
--- a/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi
@@ -263,6 +263,17 @@
 	};
 };
 
+&cpu0 {
+	fsl,soc-operating-points = <
+		/* ARM kHz  SOC-PU uV */
+		1200000 1300000
+		996000	1275000
+		852000	1275000
+		792000	1200000
+		396000	1200000
+	>;
+};
+
 &reg_arm {
 	vin-supply = <&sw1a_reg>;
 };
@@ -571,6 +582,17 @@
 		};
 	};
 
+	touchscreen@2a {
+		compatible = "eeti,egalax_ts";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_ts>;
+		reg = <0x2a>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <8 IRQ_TYPE_LEVEL_LOW>;
+		wakeup-gpios = <&gpio1 8 GPIO_ACTIVE_LOW>;
+		status = "disabled";
+	};
+
 	hpa1: amp@60 {
 		compatible = "ti,tpa6130a2";
 		pinctrl-names = "default";
@@ -666,8 +688,6 @@
 			compatible = "marvell,mv88e6085";
 			pinctrl-0 = <&pinctrl_switch_irq>;
 			pinctrl-names = "default";
-			#address-cells = <1>;
-			#size-cells = <0>;
 			reg = <0>;
 			dsa,member = <0 0>;
 			eeprom-length = <512>;
diff --git a/arch/arm/boot/dts/imx6qdl.dtsi b/arch/arm/boot/dts/imx6qdl.dtsi
index c003e62bf290..911141e24681 100644
--- a/arch/arm/boot/dts/imx6qdl.dtsi
+++ b/arch/arm/boot/dts/imx6qdl.dtsi
@@ -1,14 +1,7 @@
-/*
- * Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2011 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2011 Freescale Semiconductor, Inc.
+// Copyright 2011 Linaro Ltd.
 
 #include <dt-bindings/clock/imx6qdl-clock.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
@@ -58,9 +51,6 @@
 	};
 
 	clocks {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		ckil {
 			compatible = "fsl,imx-ckil", "fixed-clock";
 			#clock-cells = <0>;
@@ -695,11 +685,8 @@
 				interrupts = <0 49 IRQ_TYPE_LEVEL_HIGH>,
 					     <0 54 IRQ_TYPE_LEVEL_HIGH>,
 					     <0 127 IRQ_TYPE_LEVEL_HIGH>;
-				#address-cells = <1>;
-				#size-cells = <0>;
 
-				regulator-1p1@20c8110 {
-					reg = <0x20c8110>;
+				regulator-1p1 {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd1p1";
 					regulator-min-microvolt = <1000000>;
@@ -714,8 +701,7 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-3p0@20c8120 {
-					reg = <0x20c8120>;
+				regulator-3p0 {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd3p0";
 					regulator-min-microvolt = <2800000>;
@@ -730,8 +716,7 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-2p5@20c8130 {
-					reg = <0x20c8130>;
+				regulator-2p5 {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd2p5";
 					regulator-min-microvolt = <2250000>;
@@ -746,8 +731,7 @@
 					anatop-enable-bit = <0>;
 				};
 
-				reg_arm: regulator-vddcore@20c8140 {
-					reg = <0x20c8140>;
+				reg_arm: regulator-vddcore {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddarm";
 					regulator-min-microvolt = <725000>;
@@ -764,8 +748,7 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_pu: regulator-vddpu@20c8140 {
-					reg = <0x20c8140>;
+				reg_pu: regulator-vddpu {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddpu";
 					regulator-min-microvolt = <725000>;
@@ -782,8 +765,7 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_soc: regulator-vddsoc@20c8140 {
-					reg = <0x20c8140>;
+				reg_soc: regulator-vddsoc {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddsoc";
 					regulator-min-microvolt = <725000>;
@@ -1187,8 +1169,6 @@
 			};
 
 			mipi_dsi: mipi@21e0000 {
-				#address-cells = <1>;
-				#size-cells = <0>;
 				reg = <0x021e0000 0x4000>;
 				status = "disabled";
 
@@ -1300,22 +1280,27 @@
 				#size-cells = <0>;
 				reg = <2>;
 
-				ipu1_di0_disp0: disp0-endpoint {
+				ipu1_di0_disp0: endpoint@0 {
+					reg = <0>;
 				};
 
-				ipu1_di0_hdmi: hdmi-endpoint {
+				ipu1_di0_hdmi: endpoint@1 {
+					reg = <1>;
 					remote-endpoint = <&hdmi_mux_0>;
 				};
 
-				ipu1_di0_mipi: mipi-endpoint {
+				ipu1_di0_mipi: endpoint@2 {
+					reg = <2>;
 					remote-endpoint = <&mipi_mux_0>;
 				};
 
-				ipu1_di0_lvds0: lvds0-endpoint {
+				ipu1_di0_lvds0: endpoint@3 {
+					reg = <3>;
 					remote-endpoint = <&lvds0_mux_0>;
 				};
 
-				ipu1_di0_lvds1: lvds1-endpoint {
+				ipu1_di0_lvds1: endpoint@4 {
+					reg = <4>;
 					remote-endpoint = <&lvds1_mux_0>;
 				};
 			};
@@ -1325,22 +1310,27 @@
 				#size-cells = <0>;
 				reg = <3>;
 
-				ipu1_di1_disp1: disp1-endpoint {
+				ipu1_di1_disp1: endpoint@0 {
+					reg = <0>;
 				};
 
-				ipu1_di1_hdmi: hdmi-endpoint {
+				ipu1_di1_hdmi: endpoint@1 {
+					reg = <1>;
 					remote-endpoint = <&hdmi_mux_1>;
 				};
 
-				ipu1_di1_mipi: mipi-endpoint {
+				ipu1_di1_mipi: endpoint@2 {
+					reg = <2>;
 					remote-endpoint = <&mipi_mux_1>;
 				};
 
-				ipu1_di1_lvds0: lvds0-endpoint {
+				ipu1_di1_lvds0: endpoint@3 {
+					reg = <3>;
 					remote-endpoint = <&lvds0_mux_1>;
 				};
 
-				ipu1_di1_lvds1: lvds1-endpoint {
+				ipu1_di1_lvds1: endpoint@4 {
+					reg = <4>;
 					remote-endpoint = <&lvds1_mux_1>;
 				};
 			};
diff --git a/arch/arm/boot/dts/imx6qp-sabreauto.dts b/arch/arm/boot/dts/imx6qp-sabreauto.dts
index 5ce3840d83d3..d4caeeb0af70 100644
--- a/arch/arm/boot/dts/imx6qp-sabreauto.dts
+++ b/arch/arm/boot/dts/imx6qp-sabreauto.dts
@@ -1,44 +1,6 @@
-/*
- * Copyright 2016 Freescale Semiconductor, Inc.
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This file is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+//
+// Copyright 2016 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 
diff --git a/arch/arm/boot/dts/imx6qp-sabresd.dts b/arch/arm/boot/dts/imx6qp-sabresd.dts
index a8a5004dd9c8..f1b9cb104fdd 100644
--- a/arch/arm/boot/dts/imx6qp-sabresd.dts
+++ b/arch/arm/boot/dts/imx6qp-sabresd.dts
@@ -1,44 +1,6 @@
-/*
- * Copyright 2016 Freescale Semiconductor, Inc.
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This file is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+//
+// Copyright 2016 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 
diff --git a/arch/arm/boot/dts/imx6qp-wandboard-revd1.dts b/arch/arm/boot/dts/imx6qp-wandboard-revd1.dts
index 907ba0c74ba6..bcca5ac5fa51 100644
--- a/arch/arm/boot/dts/imx6qp-wandboard-revd1.dts
+++ b/arch/arm/boot/dts/imx6qp-wandboard-revd1.dts
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2013 Freescale Semiconductor, Inc.
  *
  * Author: Fabio Estevam <fabio.estevam@freescale.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 /dts-v1/;
 #include "imx6qp.dtsi"
diff --git a/arch/arm/boot/dts/imx6qp-zii-rdu2.dts b/arch/arm/boot/dts/imx6qp-zii-rdu2.dts
index de5b50df833c..8c293e9f36a7 100644
--- a/arch/arm/boot/dts/imx6qp-zii-rdu2.dts
+++ b/arch/arm/boot/dts/imx6qp-zii-rdu2.dts
@@ -53,3 +53,8 @@
 		reg = <0x10000000 0>;
 	};
 };
+
+&gpu_3d {
+	assigned-clocks = <&clks IMX6QDL_CLK_GPU3D_SHADER_SEL>;
+	assigned-clock-parents = <&clks IMX6QDL_CLK_PLL2_PFD1_594M>;
+};
diff --git a/arch/arm/boot/dts/imx6qp.dtsi b/arch/arm/boot/dts/imx6qp.dtsi
index 5f4fdce715c1..5f51f8e5c1fa 100644
--- a/arch/arm/boot/dts/imx6qp.dtsi
+++ b/arch/arm/boot/dts/imx6qp.dtsi
@@ -1,44 +1,6 @@
-/*
- * Copyright 2016 Freescale Semiconductor, Inc.
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This file is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+//
+// Copyright 2016 Freescale Semiconductor, Inc.
 
 #include "imx6q.dtsi"
 
diff --git a/arch/arm/boot/dts/imx6sl-evk.dts b/arch/arm/boot/dts/imx6sl-evk.dts
index 37e792fdc160..92ad01f676e3 100644
--- a/arch/arm/boot/dts/imx6sl-evk.dts
+++ b/arch/arm/boot/dts/imx6sl-evk.dts
@@ -1,10 +1,6 @@
-/*
- * Copyright (C) 2013 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+//Copyright (C) 2013 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 
diff --git a/arch/arm/boot/dts/imx6sl.dtsi b/arch/arm/boot/dts/imx6sl.dtsi
index ab6a7e2e7e8f..994e48dc1df0 100644
--- a/arch/arm/boot/dts/imx6sl.dtsi
+++ b/arch/arm/boot/dts/imx6sl.dtsi
@@ -1,11 +1,6 @@
-/*
- * Copyright 2013 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2013 Freescale Semiconductor, Inc.
 
 #include <dt-bindings/interrupt-controller/irq.h>
 #include "imx6sl-pinfunc.h"
@@ -86,9 +81,6 @@
 	};
 
 	clocks {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		ckil {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
@@ -527,11 +519,8 @@
 				interrupts = <0 49 IRQ_TYPE_LEVEL_HIGH>,
 					     <0 54 IRQ_TYPE_LEVEL_HIGH>,
 					     <0 127 IRQ_TYPE_LEVEL_HIGH>;
-				#address-cells = <1>;
-				#size-cells = <0>;
 
-				regulator-1p1@20c8110 {
-					reg = <0x20c8110>;
+				regulator-1p1 {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd1p1";
 					regulator-min-microvolt = <800000>;
@@ -546,8 +535,7 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-3p0@20c8120 {
-					reg = <0x20c8120>;
+				regulator-3p0 {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd3p0";
 					regulator-min-microvolt = <2800000>;
@@ -562,8 +550,7 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-2p5@20c8130 {
-					reg = <0x20c8130>;
+				regulator-2p5 {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd2p5";
 					regulator-min-microvolt = <2100000>;
@@ -578,8 +565,7 @@
 					anatop-enable-bit = <0>;
 				};
 
-				reg_arm: regulator-vddcore@20c8140 {
-					reg = <0x20c8140>;
+				reg_arm: regulator-vddcore {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddarm";
 					regulator-min-microvolt = <725000>;
@@ -596,8 +582,7 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_pu: regulator-vddpu@20c8140 {
-					reg = <0x20c8140>;
+				reg_pu: regulator-vddpu {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddpu";
 					regulator-min-microvolt = <725000>;
@@ -614,8 +599,7 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_soc: regulator-vddsoc@20c8140 {
-					reg = <0x20c8140>;
+				reg_soc: regulator-vddsoc {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddsoc";
 					regulator-min-microvolt = <725000>;
diff --git a/arch/arm/boot/dts/imx6sx-nitrogen6sx.dts b/arch/arm/boot/dts/imx6sx-nitrogen6sx.dts
index b58f770c40d9..59e52f504922 100644
--- a/arch/arm/boot/dts/imx6sx-nitrogen6sx.dts
+++ b/arch/arm/boot/dts/imx6sx-nitrogen6sx.dts
@@ -48,8 +48,8 @@
 	compatible = "boundary,imx6sx-nitrogen6sx", "fsl,imx6sx";
 
 	aliases {
-		fb_lcd = &lcdif1;
-		t_lcd = &t_lcd;
+		fb-lcd = &lcdif1;
+		t-lcd = &t_lcd;
 	};
 
 	memory@80000000 {
diff --git a/arch/arm/boot/dts/imx6sx-sabreauto.dts b/arch/arm/boot/dts/imx6sx-sabreauto.dts
index 72da5acf35a2..841a27f3198f 100644
--- a/arch/arm/boot/dts/imx6sx-sabreauto.dts
+++ b/arch/arm/boot/dts/imx6sx-sabreauto.dts
@@ -1,10 +1,6 @@
-/*
- * Copyright (C) 2014 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2014 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 
@@ -18,25 +14,67 @@
 		reg = <0x80000000 0x80000000>;
 	};
 
-	regulators {
-		compatible = "simple-bus";
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_led>;
+
+		user {
+			label = "debug";
+			gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	vcc_sd3: regulator-vcc-sd3 {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_vcc_sd3>;
+		regulator-name = "VCC_SD3";
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3000000>;
+		gpio = <&gpio2 11 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+};
+
+&anaclk2 {
+	clock-frequency = <24576000>;
+};
+
+&fec1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet1>;
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy1>;
+	fsl,magic-packet;
+	status = "okay";
+
+	mdio {
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		vcc_sd3: regulator@0 {
-			compatible = "regulator-fixed";
+		ethphy0: ethernet-phy@0 {
+			compatible = "ethernet-phy-ieee802.3-c22";
 			reg = <0>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_vcc_sd3>;
-			regulator-name = "VCC_SD3";
-			regulator-min-microvolt = <3000000>;
-			regulator-max-microvolt = <3000000>;
-			gpio = <&gpio2 11 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
+		};
+
+		ethphy1: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
 		};
 	};
 };
 
+&fec2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet2>;
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy0>;
+	fsl,magic-packet;
+	status = "okay";
+};
+
 &uart1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart1>;
@@ -69,78 +107,297 @@
 };
 
 &iomuxc {
-	imx6x-sabreauto {
-		pinctrl_uart1: uart1grp {
-			fsl,pins = <
-				MX6SX_PAD_GPIO1_IO04__UART1_TX		0x1b0b1
-				MX6SX_PAD_GPIO1_IO05__UART1_RX		0x1b0b1
-			>;
-		};
+	pinctrl_egalax_int: egalax-intgrp {
+		fsl,pins = <
+			MX6SX_PAD_SD4_RESET_B__GPIO6_IO_22      0x10b0
+		>;
+	};
 
-		pinctrl_usdhc3: usdhc3grp {
-			fsl,pins = <
-				MX6SX_PAD_SD3_CMD__USDHC3_CMD		0x17059
-				MX6SX_PAD_SD3_CLK__USDHC3_CLK		0x10059
-				MX6SX_PAD_SD3_DATA0__USDHC3_DATA0	0x17059
-				MX6SX_PAD_SD3_DATA1__USDHC3_DATA1	0x17059
-				MX6SX_PAD_SD3_DATA2__USDHC3_DATA2	0x17059
-				MX6SX_PAD_SD3_DATA3__USDHC3_DATA3	0x17059
-				MX6SX_PAD_SD3_DATA4__USDHC3_DATA4	0x17059
-				MX6SX_PAD_SD3_DATA5__USDHC3_DATA5	0x17059
-				MX6SX_PAD_SD3_DATA6__USDHC3_DATA6	0x17059
-				MX6SX_PAD_SD3_DATA7__USDHC3_DATA7	0x17059
-				MX6SX_PAD_KEY_COL0__GPIO2_IO_10		0x17059 /* CD */
-				MX6SX_PAD_KEY_ROW0__GPIO2_IO_15		0x17059 /* WP */
-			>;
-		};
+	pinctrl_enet1: enet1grp {
+		fsl,pins = <
+			MX6SX_PAD_ENET1_MDIO__ENET1_MDIO        0xa0b1
+			MX6SX_PAD_ENET1_MDC__ENET1_MDC          0xa0b1
+			MX6SX_PAD_RGMII1_TXC__ENET1_RGMII_TXC   0xa0b9
+			MX6SX_PAD_RGMII1_TD0__ENET1_TX_DATA_0   0xa0b1
+			MX6SX_PAD_RGMII1_TD1__ENET1_TX_DATA_1   0xa0b1
+			MX6SX_PAD_RGMII1_TD2__ENET1_TX_DATA_2   0xa0b1
+			MX6SX_PAD_RGMII1_TD3__ENET1_TX_DATA_3   0xa0b1
+			MX6SX_PAD_RGMII1_TX_CTL__ENET1_TX_EN    0xa0b1
+			MX6SX_PAD_RGMII1_RXC__ENET1_RX_CLK      0x3081
+			MX6SX_PAD_RGMII1_RD0__ENET1_RX_DATA_0   0x3081
+			MX6SX_PAD_RGMII1_RD1__ENET1_RX_DATA_1   0x3081
+			MX6SX_PAD_RGMII1_RD2__ENET1_RX_DATA_2   0x3081
+			MX6SX_PAD_RGMII1_RD3__ENET1_RX_DATA_3   0x3081
+			MX6SX_PAD_RGMII1_RX_CTL__ENET1_RX_EN    0x3081
+		>;
+	};
 
-		pinctrl_usdhc3_100mhz: usdhc3grp-100mhz {
-			fsl,pins = <
-				MX6SX_PAD_SD3_CMD__USDHC3_CMD		0x170b9
-				MX6SX_PAD_SD3_CLK__USDHC3_CLK		0x100b9
-				MX6SX_PAD_SD3_DATA0__USDHC3_DATA0	0x170b9
-				MX6SX_PAD_SD3_DATA1__USDHC3_DATA1	0x170b9
-				MX6SX_PAD_SD3_DATA2__USDHC3_DATA2	0x170b9
-				MX6SX_PAD_SD3_DATA3__USDHC3_DATA3	0x170b9
-				MX6SX_PAD_SD3_DATA4__USDHC3_DATA4	0x170b9
-				MX6SX_PAD_SD3_DATA5__USDHC3_DATA5	0x170b9
-				MX6SX_PAD_SD3_DATA6__USDHC3_DATA6	0x170b9
-				MX6SX_PAD_SD3_DATA7__USDHC3_DATA7	0x170b9
-			>;
-		};
+	pinctrl_enet2: enet2grp {
+		fsl,pins = <
+			MX6SX_PAD_RGMII2_TXC__ENET2_RGMII_TXC   0xa0b9
+			MX6SX_PAD_RGMII2_TD0__ENET2_TX_DATA_0   0xa0b1
+			MX6SX_PAD_RGMII2_TD1__ENET2_TX_DATA_1   0xa0b1
+			MX6SX_PAD_RGMII2_TD2__ENET2_TX_DATA_2   0xa0b1
+			MX6SX_PAD_RGMII2_TD3__ENET2_TX_DATA_3   0xa0b1
+			MX6SX_PAD_RGMII2_TX_CTL__ENET2_TX_EN    0xa0b1
+			MX6SX_PAD_RGMII2_RXC__ENET2_RX_CLK      0x3081
+			MX6SX_PAD_RGMII2_RD0__ENET2_RX_DATA_0   0x3081
+			MX6SX_PAD_RGMII2_RD1__ENET2_RX_DATA_1   0x3081
+			MX6SX_PAD_RGMII2_RD2__ENET2_RX_DATA_2   0x3081
+			MX6SX_PAD_RGMII2_RD3__ENET2_RX_DATA_3   0x3081
+			MX6SX_PAD_RGMII2_RX_CTL__ENET2_RX_EN    0x3081
+		>;
+	};
 
-		pinctrl_usdhc3_200mhz: usdhc3grp-200mhz {
-			fsl,pins = <
-				MX6SX_PAD_SD3_CMD__USDHC3_CMD		0x170f9
-				MX6SX_PAD_SD3_CLK__USDHC3_CLK		0x100f9
-				MX6SX_PAD_SD3_DATA0__USDHC3_DATA0	0x170f9
-				MX6SX_PAD_SD3_DATA1__USDHC3_DATA1	0x170f9
-				MX6SX_PAD_SD3_DATA2__USDHC3_DATA2	0x170f9
-				MX6SX_PAD_SD3_DATA3__USDHC3_DATA3	0x170f9
-				MX6SX_PAD_SD3_DATA4__USDHC3_DATA4	0x170f9
-				MX6SX_PAD_SD3_DATA5__USDHC3_DATA5	0x170f9
-				MX6SX_PAD_SD3_DATA6__USDHC3_DATA6	0x170f9
-				MX6SX_PAD_SD3_DATA7__USDHC3_DATA7	0x170f9
-			>;
-		};
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX6SX_PAD_GPIO1_IO03__I2C2_SDA          0x4001b8b1
+			MX6SX_PAD_GPIO1_IO02__I2C2_SCL          0x4001b8b1
+		>;
+	};
 
-		pinctrl_usdhc4: usdhc4grp {
-			fsl,pins = <
-				MX6SX_PAD_SD4_CMD__USDHC4_CMD		0x17059
-				MX6SX_PAD_SD4_CLK__USDHC4_CLK		0x10059
-				MX6SX_PAD_SD4_DATA0__USDHC4_DATA0	0x17059
-				MX6SX_PAD_SD4_DATA1__USDHC4_DATA1	0x17059
-				MX6SX_PAD_SD4_DATA2__USDHC4_DATA2	0x17059
-				MX6SX_PAD_SD4_DATA3__USDHC4_DATA3	0x17059
-				MX6SX_PAD_SD4_DATA7__GPIO6_IO_21	0x17059 /* CD */
-				MX6SX_PAD_SD4_DATA6__GPIO6_IO_20	0x17059 /* WP */
-			>;
-		};
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX6SX_PAD_KEY_ROW4__I2C3_SDA            0x4001b8b1
+			MX6SX_PAD_KEY_COL4__I2C3_SCL            0x4001b8b1
+		>;
+	};
+
+	pinctrl_led: ledgrp {
+		fsl,pins = <
+			MX6SX_PAD_CSI_PIXCLK__GPIO1_IO_24 0x17059
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX6SX_PAD_GPIO1_IO04__UART1_TX		0x1b0b1
+			MX6SX_PAD_GPIO1_IO05__UART1_RX		0x1b0b1
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX6SX_PAD_SD3_CMD__USDHC3_CMD		0x17059
+			MX6SX_PAD_SD3_CLK__USDHC3_CLK		0x10059
+			MX6SX_PAD_SD3_DATA0__USDHC3_DATA0	0x17059
+			MX6SX_PAD_SD3_DATA1__USDHC3_DATA1	0x17059
+			MX6SX_PAD_SD3_DATA2__USDHC3_DATA2	0x17059
+			MX6SX_PAD_SD3_DATA3__USDHC3_DATA3	0x17059
+			MX6SX_PAD_SD3_DATA4__USDHC3_DATA4	0x17059
+			MX6SX_PAD_SD3_DATA5__USDHC3_DATA5	0x17059
+			MX6SX_PAD_SD3_DATA6__USDHC3_DATA6	0x17059
+			MX6SX_PAD_SD3_DATA7__USDHC3_DATA7	0x17059
+			MX6SX_PAD_KEY_COL0__GPIO2_IO_10		0x17059 /* CD */
+			MX6SX_PAD_KEY_ROW0__GPIO2_IO_15		0x17059 /* WP */
+		>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3grp-100mhz {
+		fsl,pins = <
+			MX6SX_PAD_SD3_CMD__USDHC3_CMD		0x170b9
+			MX6SX_PAD_SD3_CLK__USDHC3_CLK		0x100b9
+			MX6SX_PAD_SD3_DATA0__USDHC3_DATA0	0x170b9
+			MX6SX_PAD_SD3_DATA1__USDHC3_DATA1	0x170b9
+			MX6SX_PAD_SD3_DATA2__USDHC3_DATA2	0x170b9
+			MX6SX_PAD_SD3_DATA3__USDHC3_DATA3	0x170b9
+			MX6SX_PAD_SD3_DATA4__USDHC3_DATA4	0x170b9
+			MX6SX_PAD_SD3_DATA5__USDHC3_DATA5	0x170b9
+			MX6SX_PAD_SD3_DATA6__USDHC3_DATA6	0x170b9
+			MX6SX_PAD_SD3_DATA7__USDHC3_DATA7	0x170b9
+		>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3grp-200mhz {
+		fsl,pins = <
+			MX6SX_PAD_SD3_CMD__USDHC3_CMD		0x170f9
+			MX6SX_PAD_SD3_CLK__USDHC3_CLK		0x100f9
+			MX6SX_PAD_SD3_DATA0__USDHC3_DATA0	0x170f9
+			MX6SX_PAD_SD3_DATA1__USDHC3_DATA1	0x170f9
+			MX6SX_PAD_SD3_DATA2__USDHC3_DATA2	0x170f9
+			MX6SX_PAD_SD3_DATA3__USDHC3_DATA3	0x170f9
+			MX6SX_PAD_SD3_DATA4__USDHC3_DATA4	0x170f9
+			MX6SX_PAD_SD3_DATA5__USDHC3_DATA5	0x170f9
+			MX6SX_PAD_SD3_DATA6__USDHC3_DATA6	0x170f9
+			MX6SX_PAD_SD3_DATA7__USDHC3_DATA7	0x170f9
+		>;
+	};
+
+	pinctrl_usdhc4: usdhc4grp {
+		fsl,pins = <
+			MX6SX_PAD_SD4_CMD__USDHC4_CMD		0x17059
+			MX6SX_PAD_SD4_CLK__USDHC4_CLK		0x10059
+			MX6SX_PAD_SD4_DATA0__USDHC4_DATA0	0x17059
+			MX6SX_PAD_SD4_DATA1__USDHC4_DATA1	0x17059
+			MX6SX_PAD_SD4_DATA2__USDHC4_DATA2	0x17059
+			MX6SX_PAD_SD4_DATA3__USDHC4_DATA3	0x17059
+			MX6SX_PAD_SD4_DATA7__GPIO6_IO_21	0x17059 /* CD */
+			MX6SX_PAD_SD4_DATA6__GPIO6_IO_20	0x17059 /* WP */
+		>;
+	};
+
+	pinctrl_vcc_sd3: vccsd3grp {
+		fsl,pins = <
+			MX6SX_PAD_KEY_COL1__GPIO2_IO_11		0x17059
+		>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <
+			MX6SX_PAD_GPIO1_IO13__WDOG1_WDOG_ANY	0x30b0
+		>;
+	};
+};
+
+&i2c2 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	status = "okay";
+
+	touchscreen@4 {
+		compatible = "eeti,egalax_ts";
+		reg = <0x04>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_egalax_int>;
+		interrupt-parent = <&gpio6>;
+		interrupts = <22 IRQ_TYPE_EDGE_FALLING>;
+		wakeup-gpios = <&gpio6 22 GPIO_ACTIVE_HIGH>;
+	};
+
+	pfuze100: pmic@8 {
+		compatible = "fsl,pfuze100";
+		reg = <0x08>;
+
+		regulators {
+			sw1a_reg: sw1ab {
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1875000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			sw1c_reg: sw1c {
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1875000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			sw2_reg: sw2 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			sw3a_reg: sw3a {
+				regulator-min-microvolt = <400000>;
+				regulator-max-microvolt = <1975000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			sw3b_reg: sw3b {
+				regulator-min-microvolt = <400000>;
+				regulator-max-microvolt = <1975000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			sw4_reg: sw4 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			swbst_reg: swbst {
+				regulator-min-microvolt = <5000000>;
+				regulator-max-microvolt = <5150000>;
+			};
+
+			snvs_reg: vsnvs {
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
 
-		pinctrl_vcc_sd3: vccsd3grp {
-			fsl,pins = <
-				MX6SX_PAD_KEY_COL1__GPIO2_IO_11		0x17059
-			>;
+			vref_reg: vrefddr {
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vgen1_reg: vgen1 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1550000>;
+				regulator-always-on;
+			};
+
+			vgen2_reg: vgen2 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1550000>;
+			};
+
+			vgen3_reg: vgen3 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			vgen4_reg: vgen4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			vgen5_reg: vgen5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			vgen6_reg: vgen6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
 		};
 	};
+
+	max7322: gpio@68 {
+		compatible = "maxim,max7322";
+		reg = <0x68>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
+
+&i2c3 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	status = "okay";
+
+	max7310_a: gpio@30 {
+		compatible = "maxim,max7310";
+		reg = <0x30>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	max7310_b: gpio@32 {
+		compatible = "maxim,max7310";
+		reg = <0x32>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
+
+&wdog1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
 };
diff --git a/arch/arm/boot/dts/imx6sx.dtsi b/arch/arm/boot/dts/imx6sx.dtsi
index 49c7205b8db8..d8b94f47498b 100644
--- a/arch/arm/boot/dts/imx6sx.dtsi
+++ b/arch/arm/boot/dts/imx6sx.dtsi
@@ -1,10 +1,6 @@
-/*
- * Copyright 2014 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2014 Freescale Semiconductor, Inc.
 
 #include <dt-bindings/clock/imx6sx-clock.h>
 #include <dt-bindings/gpio/gpio.h>
@@ -104,41 +100,46 @@
 		interrupt-parent = <&intc>;
 	};
 
-	clocks {
-		#address-cells = <1>;
-		#size-cells = <0>;
+	ckil: clock-ckil {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		clock-output-names = "ckil";
+	};
 
-		ckil: clock@0 {
-			compatible = "fixed-clock";
-			reg = <0>;
-			#clock-cells = <0>;
-			clock-frequency = <32768>;
-			clock-output-names = "ckil";
-		};
+	osc: clock-osc {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+		clock-output-names = "osc";
+	};
 
-		osc: clock@1 {
-			compatible = "fixed-clock";
-			reg = <1>;
-			#clock-cells = <0>;
-			clock-frequency = <24000000>;
-			clock-output-names = "osc";
-		};
+	ipp_di0: clock-ipp-di0 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+		clock-output-names = "ipp_di0";
+	};
 
-		ipp_di0: clock@2 {
-			compatible = "fixed-clock";
-			reg = <2>;
-			#clock-cells = <0>;
-			clock-frequency = <0>;
-			clock-output-names = "ipp_di0";
-		};
+	ipp_di1: clock-ipp-di1 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+		clock-output-names = "ipp_di1";
+	};
 
-		ipp_di1: clock@3 {
-			compatible = "fixed-clock";
-			reg = <3>;
-			#clock-cells = <0>;
-			clock-frequency = <0>;
-			clock-output-names = "ipp_di1";
-		};
+	anaclk1: clock-anaclk1 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+		clock-output-names = "anaclk1";
+	};
+
+	anaclk2: clock-anaclk2 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+		clock-output-names = "anaclk2";
 	};
 
 	tempmon: tempmon {
@@ -575,8 +576,8 @@
 				interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>;
 				#clock-cells = <1>;
-				clocks = <&ckil>, <&osc>, <&ipp_di0>, <&ipp_di1>;
-				clock-names = "ckil", "osc", "ipp_di0", "ipp_di1";
+				clocks = <&ckil>, <&osc>, <&ipp_di0>, <&ipp_di1>, <&anaclk1>, <&anaclk2>;
+				clock-names = "ckil", "osc", "ipp_di0", "ipp_di1", "anaclk1", "anaclk2";
 			};
 
 			anatop: anatop@20c8000 {
@@ -586,11 +587,8 @@
 				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>;
-				#address-cells = <1>;
-				#size-cells = <0>;
 
-				regulator-1p1@20c8110 {
-					reg = <0x20c8110>;
+				regulator-1p1 {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd1p1";
 					regulator-min-microvolt = <800000>;
@@ -605,8 +603,7 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-3p0@20c8120 {
-					reg = <0x20c8120>;
+				regulator-3p0 {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd3p0";
 					regulator-min-microvolt = <2800000>;
@@ -621,8 +618,7 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-2p5@20c8130 {
-					reg = <0x20c8130>;
+				regulator-2p5 {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd2p5";
 					regulator-min-microvolt = <2100000>;
@@ -637,8 +633,7 @@
 					anatop-enable-bit = <0>;
 				};
 
-				reg_arm: regulator-vddcore@20c8140 {
-					reg = <0x20c8140>;
+				reg_arm: regulator-vddcore {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddarm";
 					regulator-min-microvolt = <725000>;
@@ -655,8 +650,7 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_pcie: regulator-vddpcie@20c8140 {
-					reg = <0x20c8140>;
+				reg_pcie: regulator-vddpcie {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddpcie";
 					regulator-min-microvolt = <725000>;
@@ -672,8 +666,7 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_soc: regulator-vddsoc@20c8140 {
-					reg = <0x20c8140>;
+				reg_soc: regulator-vddsoc {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddsoc";
 					regulator-min-microvolt = <725000>;
diff --git a/arch/arm/boot/dts/imx6ul-14x14-evk.dts b/arch/arm/boot/dts/imx6ul-14x14-evk.dts
index 6d720b20e7ed..2438669f149a 100644
--- a/arch/arm/boot/dts/imx6ul-14x14-evk.dts
+++ b/arch/arm/boot/dts/imx6ul-14x14-evk.dts
@@ -1,10 +1,6 @@
-/*
- * Copyright (C) 2015 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2015 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 
diff --git a/arch/arm/boot/dts/imx6ul-isiot.dtsi b/arch/arm/boot/dts/imx6ul-isiot.dtsi
index 921e12c69a00..cd9928551154 100644
--- a/arch/arm/boot/dts/imx6ul-isiot.dtsi
+++ b/arch/arm/boot/dts/imx6ul-isiot.dtsi
@@ -153,8 +153,6 @@
 	stmpe811: gpio-expander@44 {
 		compatible = "st,stmpe811";
 		reg = <0x44>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_stmpe>;
 		interrupt-parent = <&gpio1>;
diff --git a/arch/arm/boot/dts/imx6ul-tx6ul-mainboard.dts b/arch/arm/boot/dts/imx6ul-tx6ul-mainboard.dts
index 2d80f7b50bc0..97686097a86e 100644
--- a/arch/arm/boot/dts/imx6ul-tx6ul-mainboard.dts
+++ b/arch/arm/boot/dts/imx6ul-tx6ul-mainboard.dts
@@ -48,7 +48,7 @@
 	compatible = "karo,imx6ul-tx6ul", "fsl,imx6ul";
 
 	aliases {
-		lcdif_24bit_pins_a = &pinctrl_disp0_3;
+		lcdif-24bit-pins-a = &pinctrl_disp0_3;
 		mmc0 = &usdhc1;
 		/delete-property/ mmc1;
 		serial2 = &uart3;
diff --git a/arch/arm/boot/dts/imx6ul-tx6ul.dtsi b/arch/arm/boot/dts/imx6ul-tx6ul.dtsi
index f678d18ad44a..02b5ba42cd59 100644
--- a/arch/arm/boot/dts/imx6ul-tx6ul.dtsi
+++ b/arch/arm/boot/dts/imx6ul-tx6ul.dtsi
@@ -53,10 +53,10 @@
 		i2c2 = &i2c1;
 		i2c3 = &i2c3;
 		i2c4 = &i2c4;
-		lcdif_23bit_pins_a = &pinctrl_disp0_1;
-		lcdif_24bit_pins_a = &pinctrl_disp0_2;
+		lcdif-23bit-pins-a = &pinctrl_disp0_1;
+		lcdif-24bit-pins-a = &pinctrl_disp0_2;
 		pwm0 = &pwm5;
-		reg_can_xcvr = &reg_can_xcvr;
+		reg-can-xcvr = &reg_can_xcvr;
 		serial2 = &uart5;
 		serial4 = &uart3;
 		spi0 = &ecspi2;
diff --git a/arch/arm/boot/dts/imx6ul.dtsi b/arch/arm/boot/dts/imx6ul.dtsi
index 1241972b16ba..47a3453a4211 100644
--- a/arch/arm/boot/dts/imx6ul.dtsi
+++ b/arch/arm/boot/dts/imx6ul.dtsi
@@ -1,10 +1,6 @@
-/*
- * Copyright 2015 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2015 Freescale Semiconductor, Inc.
 
 #include <dt-bindings/clock/imx6ul-clock.h>
 #include <dt-bindings/gpio/gpio.h>
@@ -551,11 +547,8 @@
 				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>;
-				#address-cells = <1>;
-				#size-cells = <0>;
 
-				reg_3p0: regulator-3p0@20c8110 {
-					reg = <0x20c8110>;
+				reg_3p0: regulator-3p0 {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd3p0";
 					regulator-min-microvolt = <2625000>;
@@ -569,8 +562,7 @@
 					anatop-enable-bit = <0>;
 				};
 
-				reg_arm: regulator-vddcore@20c8140 {
-					reg = <0x20c8140>;
+				reg_arm: regulator-vddcore {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "cpu";
 					regulator-min-microvolt = <725000>;
@@ -587,8 +579,7 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_soc: regulator-vddsoc@20c8140 {
-					reg = <0x20c8140>;
+				reg_soc: regulator-vddsoc {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddsoc";
 					regulator-min-microvolt = <725000>;
@@ -769,6 +760,36 @@
 			reg = <0x02100000 0x100000>;
 			ranges;
 
+			crypto: caam@2140000 {
+				compatible = "fsl,imx6ul-caam", "fsl,sec-v4.0";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				reg = <0x2140000 0x3c000>;
+				ranges = <0 0x2140000 0x3c000>;
+				interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clks IMX6UL_CLK_CAAM_IPG>, <&clks IMX6UL_CLK_CAAM_ACLK>,
+					 <&clks IMX6UL_CLK_CAAM_MEM>;
+				clock-names = "ipg", "aclk", "mem";
+
+				sec_jr0: jr0@1000 {
+					compatible = "fsl,sec-v4.0-job-ring";
+					reg = <0x1000 0x1000>;
+					interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
+				};
+
+				sec_jr1: jr1@2000 {
+					compatible = "fsl,sec-v4.0-job-ring";
+					reg = <0x2000 0x1000>;
+					interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
+				};
+
+				sec_jr2: jr2@3000 {
+					compatible = "fsl,sec-v4.0-job-ring";
+					reg = <0x3000 0x1000>;
+					interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
+				};
+			};
+
 			usbotg1: usb@2184000 {
 				compatible = "fsl,imx6ul-usb", "fsl,imx27-usb";
 				reg = <0x02184000 0x200>;
diff --git a/arch/arm/boot/dts/imx6ull-pinfunc.h b/arch/arm/boot/dts/imx6ull-pinfunc.h
index 118202336691..fdc46bb09cc1 100644
--- a/arch/arm/boot/dts/imx6ull-pinfunc.h
+++ b/arch/arm/boot/dts/imx6ull-pinfunc.h
@@ -14,6 +14,14 @@
  * The pin function ID is a tuple of
  * <mux_reg conf_reg input_reg mux_mode input_val>
  */
+#define MX6ULL_PAD_UART1_TX_DATA__UART5_DTE_RX                    0x0084 0x0310 0x0644 0x9 0x4
+#define MX6ULL_PAD_UART1_RX_DATA__UART5_DCE_RX                    0x0088 0x0314 0x0644 0x9 0x5
+#define MX6ULL_PAD_UART1_CTS_B__UART5_DCE_RTS                     0x008C 0x0318 0x0640 0x9 0x3
+#define MX6ULL_PAD_UART1_RTS_B__UART5_DTE_RTS                     0x0090 0x031C 0x0640 0x9 0x4
+#define MX6ULL_PAD_UART5_TX_DATA__UART5_DTE_RX                    0x00BC 0x0348 0x0644 0x0 0x6
+#define MX6ULL_PAD_UART5_RX_DATA__UART5_DCE_RX                    0x00C0 0x034C 0x0644 0x0 0x7
+#define MX6ULL_PAD_ENET1_RX_EN__UART5_DCE_RTS                     0x00CC 0x0358 0x0640 0x1 0x5
+#define MX6ULL_PAD_ENET1_TX_DATA0__UART5_DTE_RTS                  0x00D0 0x035C 0x0640 0x1 0x6
 #define MX6ULL_PAD_ENET2_RX_DATA0__EPDC_SDDO08                    0x00E4 0x0370 0x0000 0x9 0x0
 #define MX6ULL_PAD_ENET2_RX_DATA1__EPDC_SDDO09                    0x00E8 0x0374 0x0000 0x9 0x0
 #define MX6ULL_PAD_ENET2_RX_EN__EPDC_SDDO10                       0x00EC 0x0378 0x0000 0x9 0x0
@@ -47,6 +55,7 @@
 #define MX6ULL_PAD_CSI_DATA00__ESAI_TX_HF_CLK                     0x01E4 0x0470 0x0000 0x9 0x0
 #define MX6ULL_PAD_CSI_DATA01__ESAI_RX_HF_CLK                     0x01E8 0x0474 0x0000 0x9 0x0
 #define MX6ULL_PAD_CSI_DATA02__ESAI_RX_FS                         0x01EC 0x0478 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_DATA02__UART5_DCE_RTS                      0x01EC 0x0478 0x0640 0x8 0x7
 #define MX6ULL_PAD_CSI_DATA03__ESAI_RX_CLK                        0x01F0 0x047C 0x0000 0x9 0x0
 #define MX6ULL_PAD_CSI_DATA04__ESAI_TX_FS                         0x01F4 0x0480 0x0000 0x9 0x0
 #define MX6ULL_PAD_CSI_DATA05__ESAI_TX_CLK                        0x01F8 0x0484 0x0000 0x9 0x0
diff --git a/arch/arm/boot/dts/imx6ull.dtsi b/arch/arm/boot/dts/imx6ull.dtsi
index 571ddd71cdba..ebc25c98e5e1 100644
--- a/arch/arm/boot/dts/imx6ull.dtsi
+++ b/arch/arm/boot/dts/imx6ull.dtsi
@@ -45,6 +45,8 @@
 
 /* Delete UART8 in AIPS-1 (i.MX6UL specific) */
 /delete-node/ &uart8;
+/* Delete CAAM node in AIPS-2 (i.MX6UL specific) */
+/delete-node/ &crypto;
 
 / {
 	soc {
diff --git a/arch/arm/boot/dts/imx7d-cl-som-imx7.dts b/arch/arm/boot/dts/imx7d-cl-som-imx7.dts
index 7f645683f53b..8bf365d28cac 100644
--- a/arch/arm/boot/dts/imx7d-cl-som-imx7.dts
+++ b/arch/arm/boot/dts/imx7d-cl-som-imx7.dts
@@ -33,7 +33,7 @@
 };
 
 &cpu0 {
-	arm-supply = <&sw1a_reg>;
+	cpu-supply = <&sw1a_reg>;
 };
 
 &fec1 {
diff --git a/arch/arm/boot/dts/imx7d-nitrogen7.dts b/arch/arm/boot/dts/imx7d-nitrogen7.dts
index 52167298984d..70c53e50b2fc 100644
--- a/arch/arm/boot/dts/imx7d-nitrogen7.dts
+++ b/arch/arm/boot/dts/imx7d-nitrogen7.dts
@@ -49,8 +49,8 @@
 	compatible = "boundary,imx7d-nitrogen7", "fsl,imx7d";
 
 	aliases {
-		fb_lcd = &lcdif;
-		t_lcd = &t_lcd;
+		fb-lcd = &lcdif;
+		t-lcd = &t_lcd;
 	};
 
 	memory@80000000 {
@@ -144,7 +144,7 @@
 };
 
 &cpu0 {
-	arm-supply = <&sw1a_reg>;
+	cpu-supply = <&sw1a_reg>;
 };
 
 &fec1 {
diff --git a/arch/arm/boot/dts/imx7d-pinfunc.h b/arch/arm/boot/dts/imx7d-pinfunc.h
index f2493bc63da4..aa9dbead4b8b 100644
--- a/arch/arm/boot/dts/imx7d-pinfunc.h
+++ b/arch/arm/boot/dts/imx7d-pinfunc.h
@@ -592,7 +592,7 @@
 #define MX7D_PAD_UART2_RX_DATA__UART2_DCE_RX                      0x0130 0x03A0 0x06FC 0x0 0x2
 #define MX7D_PAD_UART2_RX_DATA__UART2_DTE_TX                      0x0130 0x03A0 0x0000 0x0 0x0
 #define MX7D_PAD_UART2_RX_DATA__I2C2_SCL                          0x0130 0x03A0 0x05DC 0x1 0x0
-#define MX7D_PAD_UART2_RX_DATA__SAI3_RX_BCLK                      0x0130 0x03A0 0x0000 0x2 0x0
+#define MX7D_PAD_UART2_RX_DATA__SAI3_RX_BCLK                      0x0130 0x03A0 0x06C4 0x2 0x0
 #define MX7D_PAD_UART2_RX_DATA__ECSPI1_SS3                        0x0130 0x03A0 0x0000 0x3 0x0
 #define MX7D_PAD_UART2_RX_DATA__ENET2_1588_EVENT1_IN              0x0130 0x03A0 0x0000 0x4 0x0
 #define MX7D_PAD_UART2_RX_DATA__GPIO4_IO2                         0x0130 0x03A0 0x0000 0x5 0x0
@@ -1112,13 +1112,13 @@
 #define MX7D_PAD_ENET1_RGMII_TD3__GPIO7_IO9                       0x0250 0x04C0 0x0000 0x5 0x0
 #define MX7D_PAD_ENET1_RGMII_TD3__CAAM_RNG_OSC_OBS                0x0250 0x04C0 0x0000 0x7 0x0
 #define MX7D_PAD_ENET1_RGMII_TX_CTL__ENET1_RGMII_TX_CTL           0x0254 0x04C4 0x0000 0x0 0x0
-#define MX7D_PAD_ENET1_RGMII_TX_CTL__SAI1_RX_SYNC                 0x0254 0x04C4 0x0000 0x2 0x0
+#define MX7D_PAD_ENET1_RGMII_TX_CTL__SAI1_RX_SYNC                 0x0254 0x04C4 0x06A4 0x2 0x1
 #define MX7D_PAD_ENET1_RGMII_TX_CTL__GPT2_COMPARE1                0x0254 0x04C4 0x0000 0x3 0x0
 #define MX7D_PAD_ENET1_RGMII_TX_CTL__EPDC_PWR_CTRL2               0x0254 0x04C4 0x0000 0x4 0x0
 #define MX7D_PAD_ENET1_RGMII_TX_CTL__GPIO7_IO10                   0x0254 0x04C4 0x0000 0x5 0x0
 #define MX7D_PAD_ENET1_RGMII_TXC__ENET1_RGMII_TXC                 0x0258 0x04C8 0x0000 0x0 0x0
 #define MX7D_PAD_ENET1_RGMII_TXC__ENET1_TX_ER                     0x0258 0x04C8 0x0000 0x1 0x0
-#define MX7D_PAD_ENET1_RGMII_TXC__SAI1_RX_BCLK                    0x0258 0x04C8 0x0000 0x2 0x0
+#define MX7D_PAD_ENET1_RGMII_TXC__SAI1_RX_BCLK                    0x0258 0x04C8 0x069C 0x2 0x1
 #define MX7D_PAD_ENET1_RGMII_TXC__GPT2_COMPARE2                   0x0258 0x04C8 0x0000 0x3 0x0
 #define MX7D_PAD_ENET1_RGMII_TXC__EPDC_PWR_CTRL3                  0x0258 0x04C8 0x0000 0x4 0x0
 #define MX7D_PAD_ENET1_RGMII_TXC__GPIO7_IO11                      0x0258 0x04C8 0x0000 0x5 0x0
diff --git a/arch/arm/boot/dts/imx7d-sdb-sht11.dts b/arch/arm/boot/dts/imx7d-sdb-sht11.dts
index 64a20ed1713a..996555596d40 100644
--- a/arch/arm/boot/dts/imx7d-sdb-sht11.dts
+++ b/arch/arm/boot/dts/imx7d-sdb-sht11.dts
@@ -1,44 +1,6 @@
-/*
- * Copyright (C) 2015 Freescale Semiconductor, Inc.
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This file is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+//
+// Copyright (C) 2015 Freescale Semiconductor, Inc.
 
 #include "imx7d-sdb.dts"
 
diff --git a/arch/arm/boot/dts/imx7d-sdb.dts b/arch/arm/boot/dts/imx7d-sdb.dts
index 5d6a08be397f..940849163104 100644
--- a/arch/arm/boot/dts/imx7d-sdb.dts
+++ b/arch/arm/boot/dts/imx7d-sdb.dts
@@ -1,44 +1,6 @@
-/*
- * Copyright (C) 2015 Freescale Semiconductor, Inc.
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This file is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+//
+// Copyright (C) 2015 Freescale Semiconductor, Inc.
 
 /dts-v1/;
 
@@ -52,6 +14,24 @@
 		reg = <0x80000000 0x80000000>;
 	};
 
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_keys>;
+
+		volume-up {
+			label = "Volume Up";
+			gpios = <&gpio5 11 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+		};
+
+		volume-down {
+			label = "Volume Down";
+			gpios = <&gpio5 10 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEDOWN>;
+		};
+	};
+
 	spi4 {
 		compatible = "spi-gpio";
 		pinctrl-names = "default";
@@ -161,7 +141,7 @@
 };
 
 &cpu0 {
-	arm-supply = <&sw1a_reg>;
+	cpu-supply = <&sw1a_reg>;
 };
 
 &ecspi3 {
@@ -519,6 +499,12 @@
 			>;
 		};
 
+		pinctrl_gpio_keys: gpio_keysgrp {
+			fsl,pins = <
+				MX7D_PAD_SD2_RESET_B__GPIO5_IO11	0x59
+				MX7D_PAD_SD2_WP__GPIO5_IO10		0x59
+			>;
+		};
 
 		pinctrl_hog: hoggrp {
 			fsl,pins = <
diff --git a/arch/arm/boot/dts/imx7d.dtsi b/arch/arm/boot/dts/imx7d.dtsi
index 200714e3feea..f579fe5c9941 100644
--- a/arch/arm/boot/dts/imx7d.dtsi
+++ b/arch/arm/boot/dts/imx7d.dtsi
@@ -1,45 +1,7 @@
-/*
- * Copyright 2015 Freescale Semiconductor, Inc.
- * Copyright 2016 Toradex AG
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This file is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+//
+// Copyright 2015 Freescale Semiconductor, Inc.
+// Copyright 2016 Toradex AG
 
 #include "imx7s.dtsi"
 #include <dt-bindings/reset/imx7-reset.h>
@@ -47,12 +9,8 @@
 / {
 	cpus {
 		cpu0: cpu@0 {
-			operating-points = <
-				/* KHz	uV */
-				996000	1075000
-				792000	975000
-			>;
 			clock-frequency = <996000000>;
+			operating-points-v2 = <&cpu0_opp_table>;
 		};
 
 		cpu1: cpu@1 {
@@ -60,6 +18,25 @@
 			device_type = "cpu";
 			reg = <1>;
 			clock-frequency = <996000000>;
+			operating-points-v2 = <&cpu0_opp_table>;
+		};
+	};
+
+	cpu0_opp_table: opp-table {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-792000000 {
+			opp-hz = /bits/ 64 <792000000>;
+			opp-microvolt = <975000>;
+			clock-latency-ns = <150000>;
+		};
+
+		opp-996000000 {
+			opp-hz = /bits/ 64 <996000000>;
+			opp-microvolt = <1075000>;
+			clock-latency-ns = <150000>;
+			opp-suspend;
 		};
 	};
 
diff --git a/arch/arm/boot/dts/imx7s-warp.dts b/arch/arm/boot/dts/imx7s-warp.dts
index 8a30b148534d..fa390da636de 100644
--- a/arch/arm/boot/dts/imx7s-warp.dts
+++ b/arch/arm/boot/dts/imx7s-warp.dts
@@ -113,10 +113,6 @@
 	assigned-clock-rates = <884736000>;
 };
 
-&cpu0 {
-	arm-supply = <&sw1a_reg>;
-};
-
 &i2c1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_i2c1>;
diff --git a/arch/arm/boot/dts/imx7s.dtsi b/arch/arm/boot/dts/imx7s.dtsi
index ce85b3ca1a55..7d3399305be2 100644
--- a/arch/arm/boot/dts/imx7s.dtsi
+++ b/arch/arm/boot/dts/imx7s.dtsi
@@ -1,45 +1,7 @@
-/*
- * Copyright 2015 Freescale Semiconductor, Inc.
- * Copyright 2016 Toradex AG
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This file is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+//
+// Copyright 2015 Freescale Semiconductor, Inc.
+// Copyright 2016 Toradex AG
 
 #include <dt-bindings/clock/imx7d-clock.h>
 #include <dt-bindings/power/imx7-power.h>
@@ -173,6 +135,17 @@
 		};
 	};
 
+	tempmon: tempmon {
+		compatible = "fsl,imx7d-tempmon";
+		interrupt-parent = <&gpc>;
+		interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
+		fsl,tempmon =<&anatop>;
+		nvmem-cells = <&tempmon_calib>,
+			<&tempmon_temp_grade>;
+		nvmem-cell-names = "calib", "temp_grade";
+		clocks = <&clks IMX7D_PLL_SYS_MAIN_CLK>;
+	};
+
 	timer {
 		compatible = "arm,armv7-timer";
 		interrupt-parent = <&intc>;
@@ -321,7 +294,7 @@
 			port {
 				tpiu_in_port: endpoint {
 					slave-mode;
-					remote-endpoint = <&replicator_out_port1>;
+					remote-endpoint = <&replicator_out_port0>;
 				};
 			};
 		};
@@ -540,27 +513,14 @@
 				};
 			};
 
-			tempmon: tempmon {
-				compatible = "fsl,imx7d-tempmon";
-				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
-				fsl,tempmon =<&anatop>;
-				nvmem-cells = <&tempmon_calib>,
-					<&tempmon_temp_grade>;
-				nvmem-cell-names = "calib", "temp_grade";
-				clocks = <&clks IMX7D_PLL_SYS_MAIN_CLK>;
-			};
-
 			anatop: anatop@30360000 {
 				compatible = "fsl,imx7d-anatop", "fsl,imx6q-anatop",
 					"syscon", "simple-bus";
 				reg = <0x30360000 0x10000>;
 				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>;
-				#address-cells = <1>;
-				#size-cells = <0>;
 
-				reg_1p0d: regulator-vdd1p0d@30360210 {
-					reg = <0x30360210>;
+				reg_1p0d: regulator-vdd1p0d {
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd1p0d";
 					regulator-min-microvolt = <800000>;
@@ -573,6 +533,20 @@
 					anatop-max-voltage = <1200000>;
 					anatop-enable-bit = <0>;
 				};
+
+				reg_1p2: regulator-vdd1p2 {
+					compatible = "fsl,anatop-regulator";
+					regulator-name = "vdd1p2";
+					regulator-min-microvolt = <1100000>;
+					regulator-max-microvolt = <1300000>;
+					anatop-reg-offset = <0x220>;
+					anatop-vol-bit-shift = <8>;
+					anatop-vol-bit-width = <5>;
+					anatop-min-bit-val = <0x14>;
+					anatop-min-voltage = <1100000>;
+					anatop-max-voltage = <1300000>;
+					anatop-enable-bit = <0>;
+				};
 			};
 
 			snvs: snvs@30370000 {
diff --git a/arch/arm/boot/dts/keystone-k2g-evm.dts b/arch/arm/boot/dts/keystone-k2g-evm.dts
index 6a4657799b99..154fdd7a7022 100644
--- a/arch/arm/boot/dts/keystone-k2g-evm.dts
+++ b/arch/arm/boot/dts/keystone-k2g-evm.dts
@@ -114,6 +114,20 @@
 			K2G_CORE_IOPAD(0x11f0) (BUFFER_CLASS_B | PIN_PULLDOWN | MUX_MODE0)      /* uart2_txd.uart2_txd */
 		>;
 	};
+
+	dcan0_pins: pinmux_dcan0_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x11fc) (BUFFER_CLASS_B | PULL_DISABLE  | MUX_MODE0)	/* dcan0tx.dcan0tx */
+			K2G_CORE_IOPAD(0x1200) (BUFFER_CLASS_B | PIN_PULLDOWN  | MUX_MODE0)	/* dcan0rx.dcan0rx */
+		>;
+	};
+
+	dcan1_pins: pinmux_dcan1_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x1224) (BUFFER_CLASS_B | PULL_DISABLE  | MUX_MODE1)	/* qspicsn2.dcan1tx */
+			K2G_CORE_IOPAD(0x1228) (BUFFER_CLASS_B | PIN_PULLDOWN  | MUX_MODE1)	/* qspicsn3.dcan1rx */
+		>;
+	};
 };
 
 &uart0 {
@@ -268,3 +282,15 @@
 	pinctrl-0 = <&uart2_pins>;
 	status = "okay";
 };
+
+&dcan0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dcan0_pins>;
+	status = "okay";
+};
+
+&dcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dcan1_pins>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/logicpd-som-lv.dtsi b/arch/arm/boot/dts/logicpd-som-lv.dtsi
index 6fa7bba3e801..3bb28c03ca74 100644
--- a/arch/arm/boot/dts/logicpd-som-lv.dtsi
+++ b/arch/arm/boot/dts/logicpd-som-lv.dtsi
@@ -35,6 +35,13 @@
 		reset-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>; /* gpio_4 */
 		#phy-cells = <0>;
 	};
+
+	/* fixed 26MHz oscillator */
+	hfclk_26m: oscillator {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <26000000>;
+	};
 };
 
 &gpmc {
@@ -79,6 +86,8 @@
 		reg = <0x48>;
 		interrupts = <7>; /* SYS_NIRQ cascaded to intc */
 		interrupt-parent = <&intc>;
+		clocks = <&hfclk_26m>;
+		clock-names = "fck";
 		twl_audio: audio {
 			compatible = "ti,twl4030-audio";
 			codec {
@@ -98,6 +107,25 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c3_pins>;
 	clock-frequency = <400000>;
+
+	touchscreen: tsc2004@48 {
+		compatible = "ti,tsc2004";
+		reg = <0x48>;
+		vio-supply = <&vaux1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&tsc2004_pins>;
+		interrupts-extended = <&gpio5 25 IRQ_TYPE_EDGE_RISING>; /* gpio 153 */
+
+		touchscreen-fuzz-x = <4>;
+		touchscreen-fuzz-y = <7>;
+		touchscreen-fuzz-pressure = <2>;
+		touchscreen-size-x = <4096>;
+		touchscreen-size-y = <4096>;
+		touchscreen-max-pressure = <2048>;
+
+		ti,x-plate-ohms = <280>;
+		ti,esd-recovery-timeout-ms = <8000>;
+	};
 };
 
 &mmc3 {
@@ -203,6 +231,12 @@
 			OMAP3_CORE1_IOPAD(0x20ba, PIN_OUTPUT | MUX_MODE4)        /* gpmc_ncs6.gpio_57 */
 		>;
 	};
+
+	tsc2004_pins: pinmux_tsc2004_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x2186, PIN_INPUT | MUX_MODE4)	/* mcbsp4_dr.gpio_153 */
+		>;
+	};
 };
 
 &omap3_pmx_wkup {
diff --git a/arch/arm/boot/dts/logicpd-torpedo-som.dtsi b/arch/arm/boot/dts/logicpd-torpedo-som.dtsi
index 3e174e474d3d..7d2302e8706c 100644
--- a/arch/arm/boot/dts/logicpd-torpedo-som.dtsi
+++ b/arch/arm/boot/dts/logicpd-torpedo-som.dtsi
@@ -30,6 +30,13 @@
 			linux,default-trigger = "none";
 		};
 	};
+
+	/* fixed 26MHz oscillator */
+	hfclk_26m: oscillator {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <26000000>;
+	};
 };
 
 &gpmc {
@@ -74,6 +81,9 @@
 		reg = <0x48>;
 		interrupts = <7>; /* SYS_NIRQ cascaded to intc */
 		interrupt-parent = <&intc>;
+		clocks = <&hfclk_26m>;
+		clock-names = "fck";
+
 		twl_audio: audio {
 			compatible = "ti,twl4030-audio";
 			codec {
diff --git a/arch/arm/boot/dts/meson8.dtsi b/arch/arm/boot/dts/meson8.dtsi
index dcc9292d2ffa..d77dcf890cfc 100644
--- a/arch/arm/boot/dts/meson8.dtsi
+++ b/arch/arm/boot/dts/meson8.dtsi
@@ -57,7 +57,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		cpu@200 {
+		cpu0: cpu@200 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			next-level-cache = <&L2>;
@@ -66,7 +66,7 @@
 			resets = <&clkc CLKC_RESET_CPU0_SOFT_RESET>;
 		};
 
-		cpu@201 {
+		cpu1: cpu@201 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			next-level-cache = <&L2>;
@@ -75,7 +75,7 @@
 			resets = <&clkc CLKC_RESET_CPU1_SOFT_RESET>;
 		};
 
-		cpu@202 {
+		cpu2: cpu@202 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			next-level-cache = <&L2>;
@@ -84,7 +84,7 @@
 			resets = <&clkc CLKC_RESET_CPU2_SOFT_RESET>;
 		};
 
-		cpu@203 {
+		cpu3: cpu@203 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			next-level-cache = <&L2>;
@@ -94,6 +94,15 @@
 		};
 	};
 
+	pmu {
+		compatible = "arm,cortex-a9-pmu";
+		interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
+	};
+
 	reserved-memory {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -272,6 +281,22 @@
 				function = "pwm_e";
 			};
 		};
+
+		uart_a1_pins: uart-a1 {
+			mux {
+				groups = "uart_tx_a1",
+				       "uart_rx_a1";
+				function = "uart_a";
+			};
+		};
+
+		uart_a1_cts_rts_pins: uart-a1-cts-rts {
+			mux {
+				groups = "uart_cts_a1",
+				       "uart_rts_a1";
+				function = "uart_a";
+			};
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/meson8b-odroidc1.dts b/arch/arm/boot/dts/meson8b-odroidc1.dts
index 3a5603d95b70..ef3177d3da3d 100644
--- a/arch/arm/boot/dts/meson8b-odroidc1.dts
+++ b/arch/arm/boot/dts/meson8b-odroidc1.dts
@@ -103,10 +103,34 @@
 	};
 };
 
-&uart_AO {
+&ethmac {
 	status = "okay";
-	pinctrl-0 = <&uart_ao_a_pins>;
+
+	snps,reset-gpio = <&gpio GPIOH_4 GPIO_ACTIVE_HIGH>;
+	snps,reset-active-low;
+	snps,reset-delays-us = <0 10000 30000>;
+
+	pinctrl-0 = <&eth_rgmii_pins>;
 	pinctrl-names = "default";
+
+	phy-mode = "rgmii";
+	phy-handle = <&eth_phy>;
+	amlogic,tx-delay-ns = <4>;
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* Realtek RTL8211F (0x001cc916) */
+		eth_phy: ethernet-phy@0 {
+			reg = <0>;
+			eee-broken-1000t;
+			interrupt-parent = <&gpio_intc>;
+			/* GPIOH_3 */
+			interrupts = <17 IRQ_TYPE_LEVEL_LOW>;
+		};
+	};
 };
 
 &gpio_ao {
@@ -124,12 +148,10 @@
 	};
 };
 
-&usb1_phy {
-	status = "okay";
-};
-
-&usb1 {
+&ir_receiver {
 	status = "okay";
+	pinctrl-0 = <&ir_recv_pins>;
+	pinctrl-names = "default";
 };
 
 &sdio {
@@ -158,32 +180,16 @@
 	};
 };
 
-&ethmac {
+&uart_AO {
 	status = "okay";
-
-	snps,reset-gpio = <&gpio GPIOH_4 GPIO_ACTIVE_HIGH>;
-	snps,reset-active-low;
-	snps,reset-delays-us = <0 10000 30000>;
-
-	pinctrl-0 = <&eth_rgmii_pins>;
+	pinctrl-0 = <&uart_ao_a_pins>;
 	pinctrl-names = "default";
+};
 
-	phy-mode = "rgmii";
-	phy-handle = <&eth_phy>;
-	amlogic,tx-delay-ns = <4>;
-
-	mdio {
-		compatible = "snps,dwmac-mdio";
-		#address-cells = <1>;
-		#size-cells = <0>;
+&usb1_phy {
+	status = "okay";
+};
 
-		/* Realtek RTL8211F (0x001cc916) */
-		eth_phy: ethernet-phy@0 {
-			reg = <0>;
-			eee-broken-1000t;
-			interrupt-parent = <&gpio_intc>;
-			/* GPIOH_3 */
-			interrupts = <17 IRQ_TYPE_LEVEL_LOW>;
-		};
-	};
+&usb1 {
+	status = "okay";
 };
diff --git a/arch/arm/boot/dts/meson8b.dtsi b/arch/arm/boot/dts/meson8b.dtsi
index 553b82174604..08f7f6be7254 100644
--- a/arch/arm/boot/dts/meson8b.dtsi
+++ b/arch/arm/boot/dts/meson8b.dtsi
@@ -55,7 +55,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		cpu@200 {
+		cpu0: cpu@200 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a5";
 			next-level-cache = <&L2>;
@@ -64,7 +64,7 @@
 			resets = <&clkc CLKC_RESET_CPU0_SOFT_RESET>;
 		};
 
-		cpu@201 {
+		cpu1: cpu@201 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a5";
 			next-level-cache = <&L2>;
@@ -73,7 +73,7 @@
 			resets = <&clkc CLKC_RESET_CPU1_SOFT_RESET>;
 		};
 
-		cpu@202 {
+		cpu2: cpu@202 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a5";
 			next-level-cache = <&L2>;
@@ -82,7 +82,7 @@
 			resets = <&clkc CLKC_RESET_CPU2_SOFT_RESET>;
 		};
 
-		cpu@203 {
+		cpu3: cpu@203 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a5";
 			next-level-cache = <&L2>;
@@ -92,6 +92,15 @@
 		};
 	};
 
+	pmu {
+		compatible = "arm,cortex-a5-pmu";
+		interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
+	};
+
 	reserved-memory {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -139,6 +148,13 @@
 				function = "uart_ao";
 			};
 		};
+
+		ir_recv_pins: remote {
+			mux {
+				groups = "remote_input";
+				function = "remote";
+			};
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/meson8m2-mxiii-plus.dts b/arch/arm/boot/dts/meson8m2-mxiii-plus.dts
new file mode 100644
index 000000000000..f5853610b20b
--- /dev/null
+++ b/arch/arm/boot/dts/meson8m2-mxiii-plus.dts
@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2018 Oleg Ivanov <balbes-150@yandex.ru>
+ * Copyright (c) 2018 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ */
+
+/dts-v1/;
+
+#include "meson8m2.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/ {
+	model = "Tronsmart MXIII Plus";
+	compatible = "tronsmart,mxiii-plus", "amlogic,meson8m2";
+
+	aliases {
+		ethernet0 = &ethmac;
+		i2c0 = &i2c_AO;
+		serial0 = &uart_AO;
+		serial1 = &uart_A;
+		mmc0 = &sd_card_slot;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory {
+		reg = <0x40000000 0x80000000>;
+	};
+
+	adc-keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 0>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <1710000>;
+
+		button-function {
+			label = "Function";
+			linux,code = <KEY_FN>;
+			press-threshold-microvolt = <10000>;
+		};
+	};
+
+	vcc_3v3: regulator-vcc3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+};
+
+&cpu0 {
+	cpu-supply = <&vcck>;
+};
+
+&ethmac {
+	status = "okay";
+
+	pinctrl-0 = <&eth_rgmii_pins>;
+	pinctrl-names = "default";
+
+	phy-handle = <&eth_phy0>;
+	phy-mode = "rgmii";
+
+	amlogic,tx-delay-ns = <4>;
+
+	snps,reset-gpio = <&gpio GPIOH_4 0>;
+	snps,reset-delays-us = <0 10000 1000000>;
+	snps,reset-active-low;
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		eth_phy0: ethernet-phy@0 {
+			/* Realtek RTL8211F (0x001cc916) */
+			reg = <0>;
+		};
+	};
+};
+
+&ir_receiver {
+	status = "okay";
+	pinctrl-0 = <&ir_recv_pins>;
+	pinctrl-names = "default";
+};
+
+&i2c_AO {
+	status = "okay";
+	pinctrl-0 = <&i2c_ao_pins>;
+	pinctrl-names = "default";
+
+	pmic@32 {
+		compatible = "ricoh,rn5t618";
+		reg = <0x32>;
+		system-power-controller;
+
+		regulators {
+			vcck: DCDC1 {
+				regulator-name = "VCCK";
+				regulator-min-microvolt = <825000>;
+				regulator-max-microvolt = <1150000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			DCDC2 {
+				regulator-name = "VDDAO";
+				regulator-min-microvolt = <950000>;
+				regulator-max-microvolt = <1150000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			DCDC3 {
+				regulator-name = "VDD_DDR";
+				regulator-min-microvolt = <1500000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			LDO1 {
+				regulator-name = "VDDIO_AO28";
+				regulator-min-microvolt = <2900000>;
+				regulator-max-microvolt = <2900000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vddio_ao1v8: LDO2 {
+				regulator-name = "VDDIO_AO18";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			LDO3 {
+				regulator-name = "VCC1V8";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			LDO4 {
+				regulator-name = "VCC2V8";
+				regulator-min-microvolt = <2850000>;
+				regulator-max-microvolt = <2850000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			LDO5 {
+				regulator-name = "AVDD1V8";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			LDORTC1 {
+				regulator-name = "VDD_LDO";
+				regulator-min-microvolt = <2700000>;
+				regulator-max-microvolt = <2700000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			LDORTC2 {
+				regulator-name = "RTC_0V9";
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&saradc {
+	status = "okay";
+	vref-supply = <&vddio_ao1v8>;
+};
+
+&sdio {
+	status = "okay";
+
+	pinctrl-0 = <&sd_b_pins>;
+	pinctrl-names = "default";
+
+	/* SD card */
+	sd_card_slot: slot@1 {
+		compatible = "mmc-slot";
+		reg = <1>;
+		status = "okay";
+
+		bus-width = <4>;
+		no-sdio;
+		cap-mmc-highspeed;
+		cap-sd-highspeed;
+		disable-wp;
+
+		cd-gpios = <&gpio CARD_6 GPIO_ACTIVE_HIGH>;
+		cd-inverted;
+
+		vmmc-supply = <&vcc_3v3>;
+	};
+};
+
+/* connected to the Bluetooth module */
+&uart_A {
+	status = "okay";
+	pinctrl-0 = <&uart_a1_pins>, <&uart_a1_cts_rts_pins>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+};
+
+&uart_AO {
+	status = "okay";
+	pinctrl-0 = <&uart_ao_a_pins>;
+	pinctrl-names = "default";
+};
+
+&usb0 {
+	status = "okay";
+};
+
+&usb1 {
+	status = "okay";
+};
+
+&usb0_phy {
+	status = "okay";
+};
+
+&usb1_phy {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/meson8m2.dtsi b/arch/arm/boot/dts/meson8m2.dtsi
new file mode 100644
index 000000000000..3e1f92273d7b
--- /dev/null
+++ b/arch/arm/boot/dts/meson8m2.dtsi
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2017 Martin Blumenstingl <martin.blumenstingl@googlemail.com>.
+ */
+
+#include "meson8.dtsi"
+
+/ {
+	model = "Amlogic Meson8m2 SoC";
+	compatible = "amlogic,meson8m2";
+}; /* end of / */
+
+&clkc {
+	compatible = "amlogic,meson8m2-clkc", "amlogic,meson8-clkc";
+};
+
+&ethmac {
+	compatible = "amlogic,meson8m2-dwmac", "snps,dwmac";
+	reg = <0xc9410000 0x10000
+		0xc1108140 0x8>;
+	clocks = <&clkc CLKID_ETH>,
+		 <&clkc CLKID_MPLL2>,
+		 <&clkc CLKID_MPLL2>;
+	clock-names = "stmmaceth", "clkin0", "clkin1";
+	resets = <&reset RESET_ETHERNET>;
+	reset-names = "stmmaceth";
+};
+
+&pinctrl_aobus {
+	compatible = "amlogic,meson8m2-aobus-pinctrl",
+		     "amlogic,meson8-aobus-pinctrl";
+};
+
+&pinctrl_cbus {
+	compatible = "amlogic,meson8m2-cbus-pinctrl",
+		     "amlogic,meson8-cbus-pinctrl";
+
+	eth_rgmii_pins: ethernet {
+		mux {
+			groups = "eth_tx_clk_50m", "eth_tx_en",
+				 "eth_txd3", "eth_txd2",
+				 "eth_txd1", "eth_txd0",
+				 "eth_rx_clk_in", "eth_rx_dv",
+				 "eth_rxd3", "eth_rxd2",
+				 "eth_rxd1", "eth_rxd0",
+				 "eth_mdio", "eth_mdc";
+			function = "ethernet";
+		};
+	};
+};
+
+&wdt {
+	compatible = "amlogic,meson8m2-wdt", "amlogic,meson8b-wdt";
+};
diff --git a/arch/arm/boot/dts/mt2701-evb.dts b/arch/arm/boot/dts/mt2701-evb.dts
index 63af4b13a36f..be0edb3dae6c 100644
--- a/arch/arm/boot/dts/mt2701-evb.dts
+++ b/arch/arm/boot/dts/mt2701-evb.dts
@@ -1,15 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2015 MediaTek Inc.
  * Author: Erin Lo <erin.lo@mediatek.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/mt2701.dtsi b/arch/arm/boot/dts/mt2701.dtsi
index 05557fce0f1d..180377e56ef4 100644
--- a/arch/arm/boot/dts/mt2701.dtsi
+++ b/arch/arm/boot/dts/mt2701.dtsi
@@ -1,15 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2015 MediaTek Inc.
  * Author: Erin.Lo <erin.lo@mediatek.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <dt-bindings/clock/mt2701-clk.h>
@@ -426,104 +419,96 @@
 		status = "disabled";
 	};
 
-	afe: audio-controller@11220000 {
-		compatible = "mediatek,mt2701-audio";
-		reg = <0 0x11220000 0 0x2000>,
-		      <0 0x112a0000 0 0x20000>;
-		interrupts =  <GIC_SPI 104 IRQ_TYPE_LEVEL_LOW>,
-			      <GIC_SPI 132 IRQ_TYPE_LEVEL_LOW>;
-		interrupt-names	= "afe", "asys";
-		power-domains = <&scpsys MT2701_POWER_DOMAIN_IFR_MSC>;
-
-		clocks = <&infracfg CLK_INFRA_AUDIO>,
-			 <&topckgen CLK_TOP_AUD_MUX1_SEL>,
-			 <&topckgen CLK_TOP_AUD_MUX2_SEL>,
-			 <&topckgen CLK_TOP_AUD_MUX1_DIV>,
-			 <&topckgen CLK_TOP_AUD_MUX2_DIV>,
-			 <&topckgen CLK_TOP_AUD_48K_TIMING>,
-			 <&topckgen CLK_TOP_AUD_44K_TIMING>,
-			 <&topckgen CLK_TOP_AUDPLL_MUX_SEL>,
-			 <&topckgen CLK_TOP_APLL_SEL>,
-			 <&topckgen CLK_TOP_AUD1PLL_98M>,
-			 <&topckgen CLK_TOP_AUD2PLL_90M>,
-			 <&topckgen CLK_TOP_HADDS2PLL_98M>,
-			 <&topckgen CLK_TOP_HADDS2PLL_294M>,
-			 <&topckgen CLK_TOP_AUDPLL>,
-			 <&topckgen CLK_TOP_AUDPLL_D4>,
-			 <&topckgen CLK_TOP_AUDPLL_D8>,
-			 <&topckgen CLK_TOP_AUDPLL_D16>,
-			 <&topckgen CLK_TOP_AUDPLL_D24>,
-			 <&topckgen CLK_TOP_AUDINTBUS_SEL>,
-			 <&clk26m>,
-			 <&topckgen CLK_TOP_SYSPLL1_D4>,
-			 <&topckgen CLK_TOP_AUD_K1_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K2_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K3_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K4_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K5_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K6_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K1_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K2_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K3_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K4_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K5_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K6_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_I2S1_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S2_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S3_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S4_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S5_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S6_MCLK>,
-			 <&topckgen CLK_TOP_ASM_M_SEL>,
-			 <&topckgen CLK_TOP_ASM_H_SEL>,
-			 <&topckgen CLK_TOP_UNIVPLL2_D4>,
-			 <&topckgen CLK_TOP_UNIVPLL2_D2>,
-			 <&topckgen CLK_TOP_SYSPLL_D5>;
-
-		clock-names = "infra_sys_audio_clk",
-			 "top_audio_mux1_sel",
-			 "top_audio_mux2_sel",
-			 "top_audio_mux1_div",
-			 "top_audio_mux2_div",
-			 "top_audio_48k_timing",
-			 "top_audio_44k_timing",
-			 "top_audpll_mux_sel",
-			 "top_apll_sel",
-			 "top_aud1_pll_98M",
-			 "top_aud2_pll_90M",
-			 "top_hadds2_pll_98M",
-			 "top_hadds2_pll_294M",
-			 "top_audpll",
-			 "top_audpll_d4",
-			 "top_audpll_d8",
-			 "top_audpll_d16",
-			 "top_audpll_d24",
-			 "top_audintbus_sel",
-			 "clk_26m",
-			 "top_syspll1_d4",
-			 "top_aud_k1_src_sel",
-			 "top_aud_k2_src_sel",
-			 "top_aud_k3_src_sel",
-			 "top_aud_k4_src_sel",
-			 "top_aud_k5_src_sel",
-			 "top_aud_k6_src_sel",
-			 "top_aud_k1_src_div",
-			 "top_aud_k2_src_div",
-			 "top_aud_k3_src_div",
-			 "top_aud_k4_src_div",
-			 "top_aud_k5_src_div",
-			 "top_aud_k6_src_div",
-			 "top_aud_i2s1_mclk",
-			 "top_aud_i2s2_mclk",
-			 "top_aud_i2s3_mclk",
-			 "top_aud_i2s4_mclk",
-			 "top_aud_i2s5_mclk",
-			 "top_aud_i2s6_mclk",
-			 "top_asm_m_sel",
-			 "top_asm_h_sel",
-			 "top_univpll2_d4",
-			 "top_univpll2_d2",
-			 "top_syspll_d5";
+	audsys: clock-controller@11220000 {
+		compatible = "mediatek,mt2701-audsys", "syscon";
+		reg = <0 0x11220000 0 0x2000>;
+		#clock-cells = <1>;
+
+		afe: audio-controller {
+			compatible = "mediatek,mt2701-audio";
+			interrupts =  <GIC_SPI 104 IRQ_TYPE_LEVEL_LOW>,
+				      <GIC_SPI 132 IRQ_TYPE_LEVEL_LOW>;
+			interrupt-names	= "afe", "asys";
+			power-domains = <&scpsys MT2701_POWER_DOMAIN_IFR_MSC>;
+
+			clocks = <&infracfg CLK_INFRA_AUDIO>,
+				 <&topckgen CLK_TOP_AUD_MUX1_SEL>,
+				 <&topckgen CLK_TOP_AUD_MUX2_SEL>,
+				 <&topckgen CLK_TOP_AUD_48K_TIMING>,
+				 <&topckgen CLK_TOP_AUD_44K_TIMING>,
+				 <&topckgen CLK_TOP_AUD_K1_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K2_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K3_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K4_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K1_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_K2_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_K3_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_K4_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_I2S1_MCLK>,
+				 <&topckgen CLK_TOP_AUD_I2S2_MCLK>,
+				 <&topckgen CLK_TOP_AUD_I2S3_MCLK>,
+				 <&topckgen CLK_TOP_AUD_I2S4_MCLK>,
+				 <&audsys CLK_AUD_I2SO1>,
+				 <&audsys CLK_AUD_I2SO2>,
+				 <&audsys CLK_AUD_I2SO3>,
+				 <&audsys CLK_AUD_I2SO4>,
+				 <&audsys CLK_AUD_I2SIN1>,
+				 <&audsys CLK_AUD_I2SIN2>,
+				 <&audsys CLK_AUD_I2SIN3>,
+				 <&audsys CLK_AUD_I2SIN4>,
+				 <&audsys CLK_AUD_ASRCO1>,
+				 <&audsys CLK_AUD_ASRCO2>,
+				 <&audsys CLK_AUD_ASRCO3>,
+				 <&audsys CLK_AUD_ASRCO4>,
+				 <&audsys CLK_AUD_AFE>,
+				 <&audsys CLK_AUD_AFE_CONN>,
+				 <&audsys CLK_AUD_A1SYS>,
+				 <&audsys CLK_AUD_A2SYS>,
+				 <&audsys CLK_AUD_AFE_MRGIF>;
+
+			clock-names = "infra_sys_audio_clk",
+				      "top_audio_mux1_sel",
+				      "top_audio_mux2_sel",
+				      "top_audio_a1sys_hp",
+				      "top_audio_a2sys_hp",
+				      "i2s0_src_sel",
+				      "i2s1_src_sel",
+				      "i2s2_src_sel",
+				      "i2s3_src_sel",
+				      "i2s0_src_div",
+				      "i2s1_src_div",
+				      "i2s2_src_div",
+				      "i2s3_src_div",
+				      "i2s0_mclk_en",
+				      "i2s1_mclk_en",
+				      "i2s2_mclk_en",
+				      "i2s3_mclk_en",
+				      "i2so0_hop_ck",
+				      "i2so1_hop_ck",
+				      "i2so2_hop_ck",
+				      "i2so3_hop_ck",
+				      "i2si0_hop_ck",
+				      "i2si1_hop_ck",
+				      "i2si2_hop_ck",
+				      "i2si3_hop_ck",
+				      "asrc0_out_ck",
+				      "asrc1_out_ck",
+				      "asrc2_out_ck",
+				      "asrc3_out_ck",
+				      "audio_afe_pd",
+				      "audio_afe_conn_pd",
+				      "audio_a1sys_pd",
+				      "audio_a2sys_pd",
+				      "audio_mrgif_pd";
+
+			assigned-clocks = <&topckgen CLK_TOP_AUD_MUX1_SEL>,
+					  <&topckgen CLK_TOP_AUD_MUX2_SEL>,
+					  <&topckgen CLK_TOP_AUD_MUX1_DIV>,
+					  <&topckgen CLK_TOP_AUD_MUX2_DIV>;
+			assigned-clock-parents = <&topckgen CLK_TOP_AUD1PLL_98M>,
+						 <&topckgen CLK_TOP_AUD2PLL_90M>;
+			assigned-clock-rates = <0>, <0>, <49152000>, <45158400>;
+		};
 	};
 
 	mmsys: syscon@14000000 {
diff --git a/arch/arm/boot/dts/mt6323.dtsi b/arch/arm/boot/dts/mt6323.dtsi
index 7c783d6c750e..ba397407c1dd 100644
--- a/arch/arm/boot/dts/mt6323.dtsi
+++ b/arch/arm/boot/dts/mt6323.dtsi
@@ -1,15 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2017 MediaTek Inc.
+ * Copyright (c) 2017-2018 MediaTek Inc.
  * Author: John Crispin <john@phrozen.org>
  *	   Sean Wang <sean.wang@mediatek.com>
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 &pwrap {
@@ -20,6 +14,13 @@
 		interrupt-controller;
 		#interrupt-cells = <2>;
 
+		mt6323_leds: leds {
+			compatible = "mediatek,mt6323-led";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		mt6323regulator: mt6323regulator{
 			compatible = "mediatek,mt6323-regulator";
 
diff --git a/arch/arm/boot/dts/mt6580-evbp1.dts b/arch/arm/boot/dts/mt6580-evbp1.dts
index 17daeae6bbe8..ca137897ed60 100644
--- a/arch/arm/boot/dts/mt6580-evbp1.dts
+++ b/arch/arm/boot/dts/mt6580-evbp1.dts
@@ -1,15 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2015 MediaTek Inc.
  * Author: Mars.C <mars.cheng@mediatek.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/mt6580.dtsi b/arch/arm/boot/dts/mt6580.dtsi
index a349dba5ff79..2bdc5ed12fca 100644
--- a/arch/arm/boot/dts/mt6580.dtsi
+++ b/arch/arm/boot/dts/mt6580.dtsi
@@ -1,15 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2015 MediaTek Inc.
  * Author: Mars.C <mars.cheng@mediatek.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <dt-bindings/interrupt-controller/irq.h>
diff --git a/arch/arm/boot/dts/mt6589-aquaris5.dts b/arch/arm/boot/dts/mt6589-aquaris5.dts
index 594a6f3bebda..7bbaa1279a26 100644
--- a/arch/arm/boot/dts/mt6589-aquaris5.dts
+++ b/arch/arm/boot/dts/mt6589-aquaris5.dts
@@ -1,16 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * Copyright (c) 2014 MundoReader S.L.
  * Author: Matthias Brugger <matthias.bgg@gmail.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/mt6589.dtsi b/arch/arm/boot/dts/mt6589.dtsi
index 41df742d7891..28df8495686a 100644
--- a/arch/arm/boot/dts/mt6589.dtsi
+++ b/arch/arm/boot/dts/mt6589.dtsi
@@ -1,17 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * Copyright (c) 2014 MundoReader S.L.
  * Author: Matthias Brugger <matthias.bgg@gmail.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
+*/
 
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
diff --git a/arch/arm/boot/dts/mt6592-evb.dts b/arch/arm/boot/dts/mt6592-evb.dts
index b57237e6394a..02849f6548e3 100644
--- a/arch/arm/boot/dts/mt6592-evb.dts
+++ b/arch/arm/boot/dts/mt6592-evb.dts
@@ -1,15 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2014 MediaTek Inc.
  * Author: Howard Chen <ibanezchen@gmail.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/mt6592.dtsi b/arch/arm/boot/dts/mt6592.dtsi
index c69201ffff72..8696ac891d60 100644
--- a/arch/arm/boot/dts/mt6592.dtsi
+++ b/arch/arm/boot/dts/mt6592.dtsi
@@ -1,15 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2014 MediaTek Inc.
  * Author: Howard Chen <ibanezchen@gmail.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <dt-bindings/interrupt-controller/irq.h>
diff --git a/arch/arm/boot/dts/mt7623.dtsi b/arch/arm/boot/dts/mt7623.dtsi
index e10c03496524..d1eb123bc73b 100644
--- a/arch/arm/boot/dts/mt7623.dtsi
+++ b/arch/arm/boot/dts/mt7623.dtsi
@@ -1,16 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2017 MediaTek Inc.
+ * Copyright (c) 2017-2018 MediaTek Inc.
  * Author: John Crispin <john@phrozen.org>
  *	   Sean Wang <sean.wang@mediatek.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -22,11 +15,12 @@
 #include <dt-bindings/phy/phy.h>
 #include <dt-bindings/reset/mt2701-resets.h>
 #include <dt-bindings/thermal/thermal.h>
-#include "skeleton64.dtsi"
 
 / {
 	compatible = "mediatek,mt7623";
 	interrupt-parent = <&sysirq>;
+	#address-cells = <2>;
+	#size-cells = <2>;
 
 	cpu_opp_table: opp-table {
 		compatible = "operating-points-v2";
@@ -130,14 +124,14 @@
 		#clock-cells = <0>;
 	};
 
-	rtc32k: oscillator@1 {
+	rtc32k: oscillator-1 {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <32000>;
 		clock-output-names = "rtc32k";
 	};
 
-	clk26m: oscillator@0 {
+	clk26m: oscillator-0 {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <26000000>;
@@ -492,6 +486,18 @@
 		nvmem-cell-names = "calibration-data";
 	};
 
+	btif: serial@1100c000 {
+		compatible = "mediatek,mt7623-btif",
+			     "mediatek,mtk-btif";
+		reg = <0 0x1100c000 0 0x1000>;
+		interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_LOW>;
+		clocks = <&pericfg CLK_PERI_BTIF>;
+		clock-names = "main";
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		status = "disabled";
+	};
+
 	nandc: nfi@1100d000 {
 		compatible = "mediatek,mt7623-nfc",
 			     "mediatek,mt2701-nfc";
@@ -517,6 +523,18 @@
 		status = "disabled";
 	};
 
+	nor_flash: spi@11014000 {
+		compatible = "mediatek,mt7623-nor",
+			     "mediatek,mt8173-nor";
+		reg = <0 0x11014000 0 0x1000>;
+		clocks = <&pericfg CLK_PERI_FLASH>,
+			 <&topckgen CLK_TOP_FLASH_SEL>;
+		clock-names = "spi", "sf";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+
 	spi1: spi@11016000 {
 		compatible = "mediatek,mt7623-spi",
 			     "mediatek,mt2701-spi";
@@ -545,105 +563,99 @@
 		status = "disabled";
 	};
 
-	afe: audio-controller@11220000 {
-		compatible = "mediatek,mt7623-audio",
-			     "mediatek,mt2701-audio";
-		reg = <0 0x11220000 0 0x2000>,
-		      <0 0x112a0000 0 0x20000>;
-		interrupts =  <GIC_SPI 104 IRQ_TYPE_LEVEL_LOW>,
-			      <GIC_SPI 132 IRQ_TYPE_LEVEL_LOW>;
-		interrupt-names	= "afe", "asys";
-		power-domains = <&scpsys MT2701_POWER_DOMAIN_IFR_MSC>;
+	audsys: clock-controller@11220000 {
+		compatible = "mediatek,mt7623-audsys",
+			     "mediatek,mt2701-audsys",
+			     "syscon";
+		reg = <0 0x11220000 0 0x2000>;
+		#clock-cells = <1>;
 
-		clocks = <&infracfg CLK_INFRA_AUDIO>,
-			 <&topckgen CLK_TOP_AUD_MUX1_SEL>,
-			 <&topckgen CLK_TOP_AUD_MUX2_SEL>,
-			 <&topckgen CLK_TOP_AUD_MUX1_DIV>,
-			 <&topckgen CLK_TOP_AUD_MUX2_DIV>,
-			 <&topckgen CLK_TOP_AUD_48K_TIMING>,
-			 <&topckgen CLK_TOP_AUD_44K_TIMING>,
-			 <&topckgen CLK_TOP_AUDPLL_MUX_SEL>,
-			 <&topckgen CLK_TOP_APLL_SEL>,
-			 <&topckgen CLK_TOP_AUD1PLL_98M>,
-			 <&topckgen CLK_TOP_AUD2PLL_90M>,
-			 <&topckgen CLK_TOP_HADDS2PLL_98M>,
-			 <&topckgen CLK_TOP_HADDS2PLL_294M>,
-			 <&topckgen CLK_TOP_AUDPLL>,
-			 <&topckgen CLK_TOP_AUDPLL_D4>,
-			 <&topckgen CLK_TOP_AUDPLL_D8>,
-			 <&topckgen CLK_TOP_AUDPLL_D16>,
-			 <&topckgen CLK_TOP_AUDPLL_D24>,
-			 <&topckgen CLK_TOP_AUDINTBUS_SEL>,
-			 <&clk26m>,
-			 <&topckgen CLK_TOP_SYSPLL1_D4>,
-			 <&topckgen CLK_TOP_AUD_K1_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K2_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K3_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K4_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K5_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K6_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K1_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K2_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K3_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K4_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K5_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K6_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_I2S1_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S2_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S3_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S4_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S5_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S6_MCLK>,
-			 <&topckgen CLK_TOP_ASM_M_SEL>,
-			 <&topckgen CLK_TOP_ASM_H_SEL>,
-			 <&topckgen CLK_TOP_UNIVPLL2_D4>,
-			 <&topckgen CLK_TOP_UNIVPLL2_D2>,
-			 <&topckgen CLK_TOP_SYSPLL_D5>;
-
-		clock-names = "infra_sys_audio_clk",
-			 "top_audio_mux1_sel",
-			 "top_audio_mux2_sel",
-			 "top_audio_mux1_div",
-			 "top_audio_mux2_div",
-			 "top_audio_48k_timing",
-			 "top_audio_44k_timing",
-			 "top_audpll_mux_sel",
-			 "top_apll_sel",
-			 "top_aud1_pll_98M",
-			 "top_aud2_pll_90M",
-			 "top_hadds2_pll_98M",
-			 "top_hadds2_pll_294M",
-			 "top_audpll",
-			 "top_audpll_d4",
-			 "top_audpll_d8",
-			 "top_audpll_d16",
-			 "top_audpll_d24",
-			 "top_audintbus_sel",
-			 "clk_26m",
-			 "top_syspll1_d4",
-			 "top_aud_k1_src_sel",
-			 "top_aud_k2_src_sel",
-			 "top_aud_k3_src_sel",
-			 "top_aud_k4_src_sel",
-			 "top_aud_k5_src_sel",
-			 "top_aud_k6_src_sel",
-			 "top_aud_k1_src_div",
-			 "top_aud_k2_src_div",
-			 "top_aud_k3_src_div",
-			 "top_aud_k4_src_div",
-			 "top_aud_k5_src_div",
-			 "top_aud_k6_src_div",
-			 "top_aud_i2s1_mclk",
-			 "top_aud_i2s2_mclk",
-			 "top_aud_i2s3_mclk",
-			 "top_aud_i2s4_mclk",
-			 "top_aud_i2s5_mclk",
-			 "top_aud_i2s6_mclk",
-			 "top_asm_m_sel",
-			 "top_asm_h_sel",
-			 "top_univpll2_d4",
-			 "top_univpll2_d2",
-			 "top_syspll_d5";
+		afe: audio-controller {
+			compatible = "mediatek,mt7623-audio",
+				     "mediatek,mt2701-audio";
+			interrupts =  <GIC_SPI 104 IRQ_TYPE_LEVEL_LOW>,
+				      <GIC_SPI 132 IRQ_TYPE_LEVEL_LOW>;
+			interrupt-names	= "afe", "asys";
+			power-domains = <&scpsys MT2701_POWER_DOMAIN_IFR_MSC>;
+
+			clocks = <&infracfg CLK_INFRA_AUDIO>,
+				 <&topckgen CLK_TOP_AUD_MUX1_SEL>,
+				 <&topckgen CLK_TOP_AUD_MUX2_SEL>,
+				 <&topckgen CLK_TOP_AUD_48K_TIMING>,
+				 <&topckgen CLK_TOP_AUD_44K_TIMING>,
+				 <&topckgen CLK_TOP_AUD_K1_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K2_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K3_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K4_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K1_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_K2_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_K3_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_K4_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_I2S1_MCLK>,
+				 <&topckgen CLK_TOP_AUD_I2S2_MCLK>,
+				 <&topckgen CLK_TOP_AUD_I2S3_MCLK>,
+				 <&topckgen CLK_TOP_AUD_I2S4_MCLK>,
+				 <&audsys CLK_AUD_I2SO1>,
+				 <&audsys CLK_AUD_I2SO2>,
+				 <&audsys CLK_AUD_I2SO3>,
+				 <&audsys CLK_AUD_I2SO4>,
+				 <&audsys CLK_AUD_I2SIN1>,
+				 <&audsys CLK_AUD_I2SIN2>,
+				 <&audsys CLK_AUD_I2SIN3>,
+				 <&audsys CLK_AUD_I2SIN4>,
+				 <&audsys CLK_AUD_ASRCO1>,
+				 <&audsys CLK_AUD_ASRCO2>,
+				 <&audsys CLK_AUD_ASRCO3>,
+				 <&audsys CLK_AUD_ASRCO4>,
+				 <&audsys CLK_AUD_AFE>,
+				 <&audsys CLK_AUD_AFE_CONN>,
+				 <&audsys CLK_AUD_A1SYS>,
+				 <&audsys CLK_AUD_A2SYS>,
+				 <&audsys CLK_AUD_AFE_MRGIF>;
+
+			clock-names = "infra_sys_audio_clk",
+				      "top_audio_mux1_sel",
+				      "top_audio_mux2_sel",
+				      "top_audio_a1sys_hp",
+				      "top_audio_a2sys_hp",
+				      "i2s0_src_sel",
+				      "i2s1_src_sel",
+				      "i2s2_src_sel",
+				      "i2s3_src_sel",
+				      "i2s0_src_div",
+				      "i2s1_src_div",
+				      "i2s2_src_div",
+				      "i2s3_src_div",
+				      "i2s0_mclk_en",
+				      "i2s1_mclk_en",
+				      "i2s2_mclk_en",
+				      "i2s3_mclk_en",
+				      "i2so0_hop_ck",
+				      "i2so1_hop_ck",
+				      "i2so2_hop_ck",
+				      "i2so3_hop_ck",
+				      "i2si0_hop_ck",
+				      "i2si1_hop_ck",
+				      "i2si2_hop_ck",
+				      "i2si3_hop_ck",
+				      "asrc0_out_ck",
+				      "asrc1_out_ck",
+				      "asrc2_out_ck",
+				      "asrc3_out_ck",
+				      "audio_afe_pd",
+				      "audio_afe_conn_pd",
+				      "audio_a1sys_pd",
+				      "audio_a2sys_pd",
+				      "audio_mrgif_pd";
+
+			assigned-clocks = <&topckgen CLK_TOP_AUD_MUX1_SEL>,
+					  <&topckgen CLK_TOP_AUD_MUX2_SEL>,
+					  <&topckgen CLK_TOP_AUD_MUX1_DIV>,
+					  <&topckgen CLK_TOP_AUD_MUX2_DIV>;
+			assigned-clock-parents = <&topckgen CLK_TOP_AUD1PLL_98M>,
+						 <&topckgen CLK_TOP_AUD2PLL_90M>;
+			assigned-clock-rates = <0>, <0>, <49152000>, <45158400>;
+		};
 	};
 
 	mmc0: mmc@11230000 {
@@ -873,6 +885,16 @@
 		#reset-cells = <1>;
 	};
 
+	hsdma: dma-controller@1b007000 {
+		compatible = "mediatek,mt7623-hsdma";
+		reg = <0 0x1b007000 0 0x1000>;
+		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_LOW>;
+		clocks = <&ethsys CLK_ETHSYS_HSDMA>;
+		clock-names = "hsdma";
+		power-domains = <&scpsys MT2701_POWER_DOMAIN_ETH>;
+		#dma-cells = <1>;
+	};
+
 	eth: ethernet@1b100000 {
 		compatible = "mediatek,mt7623-eth",
 			     "mediatek,mt2701-eth",
@@ -913,3 +935,298 @@
 		status = "disabled";
 	};
 };
+
+&pio {
+	cir_pins_a:cir-default {
+		pins-cir {
+			pinmux = <MT7623_PIN_46_IR_FUNC_IR>;
+			bias-disable;
+		};
+	};
+
+	i2c0_pins_a: i2c0-default {
+		pins-i2c0 {
+			pinmux = <MT7623_PIN_75_SDA0_FUNC_SDA0>,
+				 <MT7623_PIN_76_SCL0_FUNC_SCL0>;
+			bias-disable;
+		};
+	};
+
+	i2c1_pins_a: i2c1-default {
+		pin-i2c1 {
+			pinmux = <MT7623_PIN_57_SDA1_FUNC_SDA1>,
+				 <MT7623_PIN_58_SCL1_FUNC_SCL1>;
+			bias-disable;
+		};
+	};
+
+	i2c1_pins_b: i2c1-alt {
+		pin-i2c1 {
+			pinmux = <MT7623_PIN_242_URTS2_FUNC_SCL1>,
+				 <MT7623_PIN_243_UCTS2_FUNC_SDA1>;
+			bias-disable;
+		};
+	};
+
+	i2c2_pins_a: i2c2-default {
+		pin-i2c2 {
+			pinmux = <MT7623_PIN_77_SDA2_FUNC_SDA2>,
+				 <MT7623_PIN_78_SCL2_FUNC_SCL2>;
+			bias-disable;
+		};
+	};
+
+	i2c2_pins_b: i2c2-alt {
+		pin-i2c2 {
+			pinmux = <MT7623_PIN_122_GPIO122_FUNC_SDA2>,
+				 <MT7623_PIN_123_HTPLG_FUNC_SCL2>;
+			bias-disable;
+		};
+	};
+
+	i2s0_pins_a: i2s0-default {
+		pin-i2s0 {
+			pinmux = <MT7623_PIN_49_I2S0_DATA_FUNC_I2S0_DATA>,
+				 <MT7623_PIN_72_I2S0_DATA_IN_FUNC_I2S0_DATA_IN>,
+				 <MT7623_PIN_73_I2S0_LRCK_FUNC_I2S0_LRCK>,
+				 <MT7623_PIN_74_I2S0_BCK_FUNC_I2S0_BCK>,
+				 <MT7623_PIN_126_I2S0_MCLK_FUNC_I2S0_MCLK>;
+			drive-strength = <MTK_DRIVE_12mA>;
+			bias-pull-down;
+		};
+	};
+
+	i2s1_pins_a: i2s1-default {
+		pin-i2s1 {
+			pinmux = <MT7623_PIN_33_I2S1_DATA_FUNC_I2S1_DATA>,
+				 <MT7623_PIN_34_I2S1_DATA_IN_FUNC_I2S1_DATA_IN>,
+				 <MT7623_PIN_35_I2S1_BCK_FUNC_I2S1_BCK>,
+				 <MT7623_PIN_36_I2S1_LRCK_FUNC_I2S1_LRCK>,
+				 <MT7623_PIN_37_I2S1_MCLK_FUNC_I2S1_MCLK>;
+			drive-strength = <MTK_DRIVE_12mA>;
+			bias-pull-down;
+		};
+	};
+
+	key_pins_a: keys-alt {
+		pins-keys {
+			pinmux = <MT7623_PIN_256_GPIO256_FUNC_GPIO256>,
+				 <MT7623_PIN_257_GPIO257_FUNC_GPIO257> ;
+			input-enable;
+		};
+	};
+
+	led_pins_a: leds-alt {
+		pins-leds {
+			pinmux = <MT7623_PIN_239_EXT_SDIO0_FUNC_GPIO239>,
+				 <MT7623_PIN_240_EXT_XCS_FUNC_GPIO240>,
+				 <MT7623_PIN_241_EXT_SCK_FUNC_GPIO241>;
+		};
+	};
+
+	mmc0_pins_default: mmc0default {
+		pins-cmd-dat {
+			pinmux = <MT7623_PIN_111_MSDC0_DAT7_FUNC_MSDC0_DAT7>,
+				 <MT7623_PIN_112_MSDC0_DAT6_FUNC_MSDC0_DAT6>,
+				 <MT7623_PIN_113_MSDC0_DAT5_FUNC_MSDC0_DAT5>,
+				 <MT7623_PIN_114_MSDC0_DAT4_FUNC_MSDC0_DAT4>,
+				 <MT7623_PIN_118_MSDC0_DAT3_FUNC_MSDC0_DAT3>,
+				 <MT7623_PIN_119_MSDC0_DAT2_FUNC_MSDC0_DAT2>,
+				 <MT7623_PIN_120_MSDC0_DAT1_FUNC_MSDC0_DAT1>,
+				 <MT7623_PIN_121_MSDC0_DAT0_FUNC_MSDC0_DAT0>,
+				 <MT7623_PIN_116_MSDC0_CMD_FUNC_MSDC0_CMD>;
+			input-enable;
+			bias-pull-up;
+		};
+
+		pins-clk {
+			pinmux = <MT7623_PIN_117_MSDC0_CLK_FUNC_MSDC0_CLK>;
+			bias-pull-down;
+		};
+
+		pins-rst {
+			pinmux = <MT7623_PIN_115_MSDC0_RSTB_FUNC_MSDC0_RSTB>;
+			bias-pull-up;
+		};
+	};
+
+	mmc0_pins_uhs: mmc0 {
+		pins-cmd-dat {
+			pinmux = <MT7623_PIN_111_MSDC0_DAT7_FUNC_MSDC0_DAT7>,
+				 <MT7623_PIN_112_MSDC0_DAT6_FUNC_MSDC0_DAT6>,
+				 <MT7623_PIN_113_MSDC0_DAT5_FUNC_MSDC0_DAT5>,
+				 <MT7623_PIN_114_MSDC0_DAT4_FUNC_MSDC0_DAT4>,
+				 <MT7623_PIN_118_MSDC0_DAT3_FUNC_MSDC0_DAT3>,
+				 <MT7623_PIN_119_MSDC0_DAT2_FUNC_MSDC0_DAT2>,
+				 <MT7623_PIN_120_MSDC0_DAT1_FUNC_MSDC0_DAT1>,
+				 <MT7623_PIN_121_MSDC0_DAT0_FUNC_MSDC0_DAT0>,
+				 <MT7623_PIN_116_MSDC0_CMD_FUNC_MSDC0_CMD>;
+			input-enable;
+			drive-strength = <MTK_DRIVE_2mA>;
+			bias-pull-up = <MTK_PUPD_SET_R1R0_01>;
+		};
+
+		pins-clk {
+			pinmux = <MT7623_PIN_117_MSDC0_CLK_FUNC_MSDC0_CLK>;
+			drive-strength = <MTK_DRIVE_2mA>;
+			bias-pull-down = <MTK_PUPD_SET_R1R0_01>;
+		};
+
+		pins-rst {
+			pinmux = <MT7623_PIN_115_MSDC0_RSTB_FUNC_MSDC0_RSTB>;
+			bias-pull-up;
+		};
+	};
+
+	mmc1_pins_default: mmc1default {
+		pins-cmd-dat {
+			pinmux = <MT7623_PIN_107_MSDC1_DAT0_FUNC_MSDC1_DAT0>,
+				 <MT7623_PIN_108_MSDC1_DAT1_FUNC_MSDC1_DAT1>,
+				 <MT7623_PIN_109_MSDC1_DAT2_FUNC_MSDC1_DAT2>,
+				 <MT7623_PIN_110_MSDC1_DAT3_FUNC_MSDC1_DAT3>,
+				 <MT7623_PIN_105_MSDC1_CMD_FUNC_MSDC1_CMD>;
+			input-enable;
+			drive-strength = <MTK_DRIVE_4mA>;
+			bias-pull-up = <MTK_PUPD_SET_R1R0_10>;
+		};
+
+		pins-clk {
+			pinmux = <MT7623_PIN_106_MSDC1_CLK_FUNC_MSDC1_CLK>;
+			bias-pull-down;
+			drive-strength = <MTK_DRIVE_4mA>;
+		};
+
+		pins-wp {
+			pinmux = <MT7623_PIN_29_EINT7_FUNC_MSDC1_WP>;
+			input-enable;
+			bias-pull-up;
+		};
+
+		pins-insert {
+			pinmux = <MT7623_PIN_261_MSDC1_INS_FUNC_GPIO261>;
+			bias-pull-up;
+		};
+	};
+
+	mmc1_pins_uhs: mmc1 {
+		pins-cmd-dat {
+			pinmux = <MT7623_PIN_107_MSDC1_DAT0_FUNC_MSDC1_DAT0>,
+				 <MT7623_PIN_108_MSDC1_DAT1_FUNC_MSDC1_DAT1>,
+				 <MT7623_PIN_109_MSDC1_DAT2_FUNC_MSDC1_DAT2>,
+				 <MT7623_PIN_110_MSDC1_DAT3_FUNC_MSDC1_DAT3>,
+				 <MT7623_PIN_105_MSDC1_CMD_FUNC_MSDC1_CMD>;
+			input-enable;
+			drive-strength = <MTK_DRIVE_4mA>;
+			bias-pull-up = <MTK_PUPD_SET_R1R0_10>;
+		};
+
+		pins-clk {
+			pinmux = <MT7623_PIN_106_MSDC1_CLK_FUNC_MSDC1_CLK>;
+			drive-strength = <MTK_DRIVE_4mA>;
+			bias-pull-down = <MTK_PUPD_SET_R1R0_10>;
+		};
+	};
+
+	nand_pins_default: nanddefault {
+		pins-ale {
+			pinmux = <MT7623_PIN_116_MSDC0_CMD_FUNC_NALE>;
+			drive-strength = <MTK_DRIVE_8mA>;
+			bias-pull-down = <MTK_PUPD_SET_R1R0_10>;
+		};
+
+		pins-dat {
+			pinmux = <MT7623_PIN_111_MSDC0_DAT7_FUNC_NLD7>,
+				 <MT7623_PIN_112_MSDC0_DAT6_FUNC_NLD6>,
+				 <MT7623_PIN_114_MSDC0_DAT4_FUNC_NLD4>,
+				 <MT7623_PIN_118_MSDC0_DAT3_FUNC_NLD3>,
+				 <MT7623_PIN_121_MSDC0_DAT0_FUNC_NLD0>,
+				 <MT7623_PIN_120_MSDC0_DAT1_FUNC_NLD1>,
+				 <MT7623_PIN_113_MSDC0_DAT5_FUNC_NLD5>,
+				 <MT7623_PIN_115_MSDC0_RSTB_FUNC_NLD8>,
+				 <MT7623_PIN_119_MSDC0_DAT2_FUNC_NLD2>;
+			input-enable;
+			drive-strength = <MTK_DRIVE_8mA>;
+			bias-pull-up;
+		};
+
+		pins-we {
+			pinmux = <MT7623_PIN_117_MSDC0_CLK_FUNC_NWEB>;
+			drive-strength = <MTK_DRIVE_8mA>;
+			bias-pull-up = <MTK_PUPD_SET_R1R0_10>;
+		};
+	};
+
+	pcie_default: pcie_pin_default {
+		pins_cmd_dat {
+			pinmux = <MT7623_PIN_208_AUD_EXT_CK1_FUNC_PCIE0_PERST_N>,
+				 <MT7623_PIN_209_AUD_EXT_CK2_FUNC_PCIE1_PERST_N>;
+			bias-disable;
+		};
+	};
+
+	pwm_pins_a: pwm-default {
+		pins-pwm {
+			pinmux = <MT7623_PIN_203_PWM0_FUNC_PWM0>,
+				 <MT7623_PIN_204_PWM1_FUNC_PWM1>,
+				 <MT7623_PIN_205_PWM2_FUNC_PWM2>,
+				 <MT7623_PIN_206_PWM3_FUNC_PWM3>,
+				 <MT7623_PIN_207_PWM4_FUNC_PWM4>;
+		};
+	};
+
+	spi0_pins_a: spi0-default {
+		pins-spi {
+			pinmux = <MT7623_PIN_53_SPI0_CSN_FUNC_SPI0_CS>,
+				<MT7623_PIN_54_SPI0_CK_FUNC_SPI0_CK>,
+				<MT7623_PIN_55_SPI0_MI_FUNC_SPI0_MI>,
+				<MT7623_PIN_56_SPI0_MO_FUNC_SPI0_MO>;
+			bias-disable;
+		};
+	};
+
+	spi1_pins_a: spi1-default {
+		pins-spi {
+			pinmux = <MT7623_PIN_7_SPI1_CSN_FUNC_SPI1_CS>,
+				<MT7623_PIN_199_SPI1_CK_FUNC_SPI1_CK>,
+				<MT7623_PIN_8_SPI1_MI_FUNC_SPI1_MI>,
+				<MT7623_PIN_9_SPI1_MO_FUNC_SPI1_MO>;
+		};
+	};
+
+	spi2_pins_a: spi2-default {
+		pins-spi {
+			pinmux = <MT7623_PIN_101_SPI2_CSN_FUNC_SPI2_CS>,
+				 <MT7623_PIN_104_SPI2_CK_FUNC_SPI2_CK>,
+				 <MT7623_PIN_102_SPI2_MI_FUNC_SPI2_MI>,
+				 <MT7623_PIN_103_SPI2_MO_FUNC_SPI2_MO>;
+		};
+	};
+
+	uart0_pins_a: uart0-default {
+		pins-dat {
+			pinmux = <MT7623_PIN_79_URXD0_FUNC_URXD0>,
+				 <MT7623_PIN_80_UTXD0_FUNC_UTXD0>;
+		};
+	};
+
+	uart1_pins_a: uart1-default {
+		pins-dat {
+			pinmux = <MT7623_PIN_81_URXD1_FUNC_URXD1>,
+				 <MT7623_PIN_82_UTXD1_FUNC_UTXD1>;
+		};
+	};
+
+	uart2_pins_a: uart2-default {
+		pins-dat {
+			pinmux = <MT7623_PIN_14_GPIO14_FUNC_URXD2>,
+				 <MT7623_PIN_15_GPIO15_FUNC_UTXD2>;
+		};
+	};
+
+	uart2_pins_b: uart2-alt {
+		pins-dat {
+			pinmux = <MT7623_PIN_200_URXD2_FUNC_URXD2>,
+				 <MT7623_PIN_201_UTXD2_FUNC_UTXD2>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/mt7623a-rfb-emmc.dts b/arch/arm/boot/dts/mt7623a-rfb-emmc.dts
new file mode 100644
index 000000000000..13c86936d1c8
--- /dev/null
+++ b/arch/arm/boot/dts/mt7623a-rfb-emmc.dts
@@ -0,0 +1,291 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2017-2018 MediaTek Inc.
+ * Author: Sean Wang <sean.wang@mediatek.com>
+ *
+ */
+
+/dts-v1/;
+#include <dt-bindings/input/input.h>
+#include "mt7623a.dtsi"
+#include "mt6323.dtsi"
+
+/ {
+	model = "MediaTek MT7623A with eMMC reference board";
+	compatible = "mediatek,mt7623a-rfb-emmc", "mediatek,mt7623";
+
+	aliases {
+		serial2 = &uart2;
+	};
+
+	chosen {
+		stdout-path = "serial2:115200n8";
+	};
+
+	cpus {
+		cpu@0 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+
+		cpu@1 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+
+		cpu@2 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+
+		cpu@3 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&key_pins_a>;
+
+		factory {
+			label = "factory";
+			linux,code = <BTN_0>;
+			gpios = <&pio 256 GPIO_ACTIVE_LOW>;
+		};
+
+		wps {
+			label = "wps";
+			linux,code = <KEY_WPS_BUTTON>;
+			gpios = <&pio 257 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0 0x80000000 0 0x20000000>;
+	};
+
+	reg_1p8v: regulator-1p8v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_5v: regulator-5v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	sound {
+		compatible = "mediatek,mt2701-wm8960-machine";
+		mediatek,platform = <&afe>;
+		audio-routing =
+			"Headphone", "HP_L",
+			"Headphone", "HP_R",
+			"LINPUT1", "AMIC",
+			"RINPUT1", "AMIC";
+		mediatek,audio-codec = <&wm8960>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&i2s0_pins_a>;
+	};
+};
+
+&btif {
+	status = "okay";
+};
+
+&crypto {
+	status = "okay";
+};
+
+&eth {
+	status = "okay";
+
+	gmac0: mac@0 {
+		compatible = "mediatek,eth-mac";
+		reg = <0>;
+		phy-mode = "trgmii";
+
+		fixed-link {
+			speed = <1000>;
+			full-duplex;
+			pause;
+		};
+	};
+
+	mdio-bus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		switch@0 {
+			compatible = "mediatek,mt7530";
+			reg = <0>;
+			mediatek,mcm;
+			resets = <&ethsys MT2701_ETHSYS_MCM_RST>;
+			reset-names = "mcm";
+			core-supply = <&mt6323_vpa_reg>;
+			io-supply = <&mt6323_vemc3v3_reg>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					label = "lan0";
+				};
+
+				port@1 {
+					reg = <1>;
+					label = "lan1";
+				};
+
+				port@2 {
+					reg = <2>;
+					label = "lan2";
+				};
+
+				port@3 {
+					reg = <3>;
+					label = "lan3";
+				};
+
+				port@4 {
+					reg = <4>;
+					label = "wan";
+				};
+
+				port@6 {
+					reg = <6>;
+					label = "cpu";
+					ethernet = <&gmac0>;
+					phy-mode = "trgmii";
+
+					fixed-link {
+						speed = <1000>;
+						full-duplex;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins_a>;
+	status = "okay";
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins_b>;
+	status = "okay";
+
+	wm8960: wm8960@1a {
+		compatible = "wlf,wm8960";
+		reg = <0x1a>;
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_pins_b>;
+	status = "okay";
+};
+
+&mmc0 {
+	pinctrl-names = "default", "state_uhs";
+	pinctrl-0 = <&mmc0_pins_default>;
+	pinctrl-1 = <&mmc0_pins_uhs>;
+	status = "okay";
+	bus-width = <8>;
+	max-frequency = <50000000>;
+	cap-mmc-highspeed;
+	vmmc-supply = <&reg_3p3v>;
+	vqmmc-supply = <&reg_1p8v>;
+	non-removable;
+};
+
+&mmc1 {
+	pinctrl-names = "default", "state_uhs";
+	pinctrl-0 = <&mmc1_pins_default>;
+	pinctrl-1 = <&mmc1_pins_uhs>;
+	status = "okay";
+	bus-width = <4>;
+	max-frequency = <50000000>;
+	cap-sd-highspeed;
+	cd-gpios = <&pio 261 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&reg_3p3v>;
+	vqmmc-supply = <&reg_3p3v>;
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie_default>;
+	status = "okay";
+
+	pcie@0,0 {
+		status = "okay";
+	};
+
+	pcie@1,0 {
+		status = "okay";
+	};
+};
+
+&pcie0_phy {
+	status = "okay";
+};
+
+&pcie1_phy {
+	status = "okay";
+};
+
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm_pins_a>;
+	status = "okay";
+};
+
+&spi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi0_pins_a>;
+	status = "okay";
+};
+
+&spi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi1_pins_a>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_pins_b>;
+	status = "okay";
+};
+
+&usb1 {
+	vusb33-supply = <&reg_3p3v>;
+	vbus-supply = <&reg_5v>;
+	status = "okay";
+};
+
+&u3phy1 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/mt7623a-rfb-nand.dts b/arch/arm/boot/dts/mt7623a-rfb-nand.dts
new file mode 100644
index 000000000000..88d8f0b2f4c2
--- /dev/null
+++ b/arch/arm/boot/dts/mt7623a-rfb-nand.dts
@@ -0,0 +1,337 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2017-2018 MediaTek Inc.
+ * Author: Sean Wang <sean.wang@mediatek.com>
+ *
+ */
+
+/dts-v1/;
+#include <dt-bindings/input/input.h>
+#include "mt7623a.dtsi"
+#include "mt6323.dtsi"
+
+/ {
+	model = "MediaTek MT7623A with NAND reference board";
+	compatible = "mediatek,mt7623a-rfb-nand", "mediatek,mt7623";
+
+	aliases {
+		serial2 = &uart2;
+	};
+
+	chosen {
+		stdout-path = "serial2:115200n8";
+	};
+
+	cpus {
+		cpu@0 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+
+		cpu@1 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+
+		cpu@2 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+
+		cpu@3 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&key_pins_a>;
+
+		factory {
+			label = "factory";
+			linux,code = <BTN_0>;
+			gpios = <&pio 256 GPIO_ACTIVE_LOW>;
+		};
+
+		wps {
+			label = "wps";
+			linux,code = <KEY_WPS_BUTTON>;
+			gpios = <&pio 257 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0 0x80000000 0 0x20000000>;
+	};
+
+	reg_1p8v: regulator-1p8v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_5v: regulator-5v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	sound {
+		compatible = "mediatek,mt2701-wm8960-machine";
+		mediatek,platform = <&afe>;
+		audio-routing =
+			"Headphone", "HP_L",
+			"Headphone", "HP_R",
+			"LINPUT1", "AMIC",
+			"RINPUT1", "AMIC";
+		mediatek,audio-codec = <&wm8960>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&i2s0_pins_a>;
+	};
+};
+
+&bch {
+	status = "okay";
+};
+
+&btif {
+	status = "okay";
+};
+
+&crypto {
+	status = "okay";
+};
+
+&eth {
+	status = "okay";
+
+	gmac0: mac@0 {
+		compatible = "mediatek,eth-mac";
+		reg = <0>;
+		phy-mode = "trgmii";
+
+		fixed-link {
+			speed = <1000>;
+			full-duplex;
+			pause;
+		};
+	};
+
+	mdio-bus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		switch@0 {
+			compatible = "mediatek,mt7530";
+			reg = <0>;
+			mediatek,mcm;
+			resets = <&ethsys MT2701_ETHSYS_MCM_RST>;
+			reset-names = "mcm";
+			core-supply = <&mt6323_vpa_reg>;
+			io-supply = <&mt6323_vemc3v3_reg>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					label = "lan0";
+				};
+
+				port@1 {
+					reg = <1>;
+					label = "lan1";
+				};
+
+				port@2 {
+					reg = <2>;
+					label = "lan2";
+				};
+
+				port@3 {
+					reg = <3>;
+					label = "lan3";
+				};
+
+				port@4 {
+					reg = <4>;
+					label = "wan";
+				};
+
+				port@6 {
+					reg = <6>;
+					label = "cpu";
+					ethernet = <&gmac0>;
+					phy-mode = "trgmii";
+
+					fixed-link {
+						speed = <1000>;
+						full-duplex;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins_a>;
+	status = "okay";
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins_b>;
+	status = "okay";
+
+	wm8960: wm8960@1a {
+		compatible = "wlf,wm8960";
+		reg = <0x1a>;
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_pins_b>;
+	status = "okay";
+};
+
+&mmc1 {
+	pinctrl-names = "default", "state_uhs";
+	pinctrl-0 = <&mmc1_pins_default>;
+	pinctrl-1 = <&mmc1_pins_uhs>;
+	status = "okay";
+	bus-width = <4>;
+	max-frequency = <50000000>;
+	cap-sd-highspeed;
+	cd-gpios = <&pio 261 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&reg_3p3v>;
+	vqmmc-supply = <&reg_3p3v>;
+};
+
+&nandc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&nand_pins_default>;
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+		spare_per_sector = <64>;
+		nand-ecc-mode = "hw";
+		nand-ecc-strength = <12>;
+		nand-ecc-step-size = <1024>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "preloader";
+				reg = <0x0 0x40000>;
+			};
+
+			partition@40000 {
+				label = "uboot";
+				reg = <0x40000 0x80000>;
+			};
+
+			partition@c0000 {
+				label = "uboot-env";
+				reg = <0xC0000 0x40000>;
+			};
+
+			partition@140000 {
+				label = "bootimg";
+				reg = <0x140000 0x2000000>;
+			};
+
+			partition@2140000 {
+				label = "recovery";
+				reg = <0x2140000 0x2000000>;
+			};
+
+			partition@4140000 {
+				label = "rootfs";
+				reg = <0x4140000 0x1000000>;
+			};
+
+			partition@5140000 {
+				label = "usrdata";
+				reg = <0x5140000 0x1000000>;
+			};
+		};
+	};
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie_default>;
+	status = "okay";
+
+	pcie@0,0 {
+		status = "okay";
+	};
+
+	pcie@1,0 {
+		status = "okay";
+	};
+};
+
+&pcie0_phy {
+	status = "okay";
+};
+
+&pcie1_phy {
+	status = "okay";
+};
+
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm_pins_a>;
+	status = "okay";
+};
+
+&spi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi0_pins_a>;
+	status = "okay";
+};
+
+&spi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi1_pins_a>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_pins_b>;
+	status = "okay";
+};
+
+&usb1 {
+	vusb33-supply = <&reg_3p3v>;
+	vbus-supply = <&reg_5v>;
+	status = "okay";
+};
+
+&u3phy1 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/mt7623a.dtsi b/arch/arm/boot/dts/mt7623a.dtsi
new file mode 100644
index 000000000000..0735a1fb8ad9
--- /dev/null
+++ b/arch/arm/boot/dts/mt7623a.dtsi
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2017-2018 MediaTek Inc.
+ * Author: Sean Wang <sean.wang@mediatek.com>
+ *
+ */
+
+/dts-v1/;
+#include <dt-bindings/power/mt7623a-power.h>
+#include "mt7623.dtsi"
+
+&afe {
+	power-domains = <&scpsys MT7623A_POWER_DOMAIN_IFR_MSC>;
+};
+
+&crypto {
+	power-domains = <&scpsys MT7623A_POWER_DOMAIN_ETH>;
+};
+
+&eth {
+	power-domains = <&scpsys MT7623A_POWER_DOMAIN_ETH>;
+};
+
+&nandc {
+	power-domains = <&scpsys MT7623A_POWER_DOMAIN_IFR_MSC>;
+};
+
+&pcie {
+	power-domains = <&scpsys MT7623A_POWER_DOMAIN_HIF>;
+};
+
+&scpsys {
+	compatible = "mediatek,mt7623a-scpsys";
+	clocks = <&topckgen CLK_TOP_ETHIF_SEL>;
+	clock-names = "ethif";
+};
+
+&usb1 {
+	power-domains = <&scpsys MT7623A_POWER_DOMAIN_HIF>;
+};
+
+&usb2 {
+	power-domains = <&scpsys MT7623A_POWER_DOMAIN_HIF>;
+};
diff --git a/arch/arm/boot/dts/mt7623n-bananapi-bpi-r2.dts b/arch/arm/boot/dts/mt7623n-bananapi-bpi-r2.dts
index bbf56f855e46..531d905d924f 100644
--- a/arch/arm/boot/dts/mt7623n-bananapi-bpi-r2.dts
+++ b/arch/arm/boot/dts/mt7623n-bananapi-bpi-r2.dts
@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 Sean Wang <sean.wang@mediatek.com>
+ * Copyright 2017-2018 Sean Wang <sean.wang@mediatek.com>
  *
  * SPDX-License-Identifier: (GPL-2.0+ OR MIT)
  */
@@ -109,10 +109,15 @@
 	};
 
 	memory@80000000 {
-		reg = <0 0x80000000 0 0x40000000>;
+		device_type = "memory";
+		reg = <0 0x80000000 0 0x80000000>;
 	};
 };
 
+&btif {
+	status = "okay";
+};
+
 &cir {
 	pinctrl-names = "default";
 	pinctrl-0 = <&cir_pins_a>;
@@ -144,8 +149,6 @@
 
 		switch@0 {
 			compatible = "mediatek,mt7530";
-			#address-cells = <1>;
-			#size-cells = <0>;
 			reg = <0>;
 			reset-gpios = <&pio 33 0>;
 			core-supply = <&mt6323_vpa_reg>;
@@ -154,7 +157,6 @@
 			ports {
 				#address-cells = <1>;
 				#size-cells = <0>;
-				reg = <0>;
 
 				port@0 {
 					reg = <0>;
@@ -235,6 +237,28 @@
 	vqmmc-supply = <&reg_3p3v>;
 };
 
+&mt6323_leds {
+	status = "okay";
+
+	led@0 {
+		reg = <0>;
+		label = "bpi-r2:isink:green";
+		default-state = "off";
+	};
+
+	led@1 {
+		reg = <1>;
+		label = "bpi-r2:isink:red";
+		default-state = "off";
+	};
+
+	led@2 {
+		reg = <2>;
+		label = "bpi-r2:isink:blue";
+		default-state = "off";
+	};
+};
+
 &pcie {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie_default>;
@@ -257,257 +281,12 @@
 	status = "okay";
 };
 
-&pio {
-	cir_pins_a:cir@0 {
-		pins-cir {
-			pinmux = <MT7623_PIN_46_IR_FUNC_IR>;
-			bias-disable;
-		};
-	};
-
-	i2c0_pins_a: i2c@0 {
-		pins-i2c0 {
-			pinmux = <MT7623_PIN_75_SDA0_FUNC_SDA0>,
-				 <MT7623_PIN_76_SCL0_FUNC_SCL0>;
-			bias-disable;
-		};
-	};
-
-	i2c1_pins_a: i2c@1 {
-		pin-i2c1 {
-			pinmux = <MT7623_PIN_57_SDA1_FUNC_SDA1>,
-				 <MT7623_PIN_58_SCL1_FUNC_SCL1>;
-			bias-disable;
-		};
-	};
-
-	i2s0_pins_a: i2s@0 {
-		pin-i2s0 {
-			pinmux = <MT7623_PIN_49_I2S0_DATA_FUNC_I2S0_DATA>,
-				 <MT7623_PIN_72_I2S0_DATA_IN_FUNC_I2S0_DATA_IN>,
-				 <MT7623_PIN_73_I2S0_LRCK_FUNC_I2S0_LRCK>,
-				 <MT7623_PIN_74_I2S0_BCK_FUNC_I2S0_BCK>,
-				 <MT7623_PIN_126_I2S0_MCLK_FUNC_I2S0_MCLK>;
-			drive-strength = <MTK_DRIVE_12mA>;
-			bias-pull-down;
-		};
-	};
-
-	i2s1_pins_a: i2s@1 {
-		pin-i2s1 {
-			pinmux = <MT7623_PIN_33_I2S1_DATA_FUNC_I2S1_DATA>,
-				 <MT7623_PIN_34_I2S1_DATA_IN_FUNC_I2S1_DATA_IN>,
-				 <MT7623_PIN_35_I2S1_BCK_FUNC_I2S1_BCK>,
-				 <MT7623_PIN_36_I2S1_LRCK_FUNC_I2S1_LRCK>,
-				 <MT7623_PIN_37_I2S1_MCLK_FUNC_I2S1_MCLK>;
-			drive-strength = <MTK_DRIVE_12mA>;
-			bias-pull-down;
-		};
-	};
-
-	key_pins_a: keys@0 {
-		pins-keys {
-			pinmux = <MT7623_PIN_256_GPIO256_FUNC_GPIO256>,
-				 <MT7623_PIN_257_GPIO257_FUNC_GPIO257> ;
-			input-enable;
-		};
-	};
-
-	led_pins_a: leds@0 {
-		pins-leds {
-			pinmux = <MT7623_PIN_239_EXT_SDIO0_FUNC_GPIO239>,
-				 <MT7623_PIN_240_EXT_XCS_FUNC_GPIO240>,
-				 <MT7623_PIN_241_EXT_SCK_FUNC_GPIO241>;
-		};
-	};
-
-	mmc0_pins_default: mmc0default {
-		pins-cmd-dat {
-			pinmux = <MT7623_PIN_111_MSDC0_DAT7_FUNC_MSDC0_DAT7>,
-				 <MT7623_PIN_112_MSDC0_DAT6_FUNC_MSDC0_DAT6>,
-				 <MT7623_PIN_113_MSDC0_DAT5_FUNC_MSDC0_DAT5>,
-				 <MT7623_PIN_114_MSDC0_DAT4_FUNC_MSDC0_DAT4>,
-				 <MT7623_PIN_118_MSDC0_DAT3_FUNC_MSDC0_DAT3>,
-				 <MT7623_PIN_119_MSDC0_DAT2_FUNC_MSDC0_DAT2>,
-				 <MT7623_PIN_120_MSDC0_DAT1_FUNC_MSDC0_DAT1>,
-				 <MT7623_PIN_121_MSDC0_DAT0_FUNC_MSDC0_DAT0>,
-				 <MT7623_PIN_116_MSDC0_CMD_FUNC_MSDC0_CMD>;
-			input-enable;
-			bias-pull-up;
-		};
-
-		pins-clk {
-			pinmux = <MT7623_PIN_117_MSDC0_CLK_FUNC_MSDC0_CLK>;
-			bias-pull-down;
-		};
-
-		pins-rst {
-			pinmux = <MT7623_PIN_115_MSDC0_RSTB_FUNC_MSDC0_RSTB>;
-			bias-pull-up;
-		};
-	};
-
-	mmc0_pins_uhs: mmc0 {
-		pins-cmd-dat {
-			pinmux = <MT7623_PIN_111_MSDC0_DAT7_FUNC_MSDC0_DAT7>,
-				 <MT7623_PIN_112_MSDC0_DAT6_FUNC_MSDC0_DAT6>,
-				 <MT7623_PIN_113_MSDC0_DAT5_FUNC_MSDC0_DAT5>,
-				 <MT7623_PIN_114_MSDC0_DAT4_FUNC_MSDC0_DAT4>,
-				 <MT7623_PIN_118_MSDC0_DAT3_FUNC_MSDC0_DAT3>,
-				 <MT7623_PIN_119_MSDC0_DAT2_FUNC_MSDC0_DAT2>,
-				 <MT7623_PIN_120_MSDC0_DAT1_FUNC_MSDC0_DAT1>,
-				 <MT7623_PIN_121_MSDC0_DAT0_FUNC_MSDC0_DAT0>,
-				 <MT7623_PIN_116_MSDC0_CMD_FUNC_MSDC0_CMD>;
-			input-enable;
-			drive-strength = <MTK_DRIVE_2mA>;
-			bias-pull-up = <MTK_PUPD_SET_R1R0_01>;
-		};
-
-		pins-clk {
-			pinmux = <MT7623_PIN_117_MSDC0_CLK_FUNC_MSDC0_CLK>;
-			drive-strength = <MTK_DRIVE_2mA>;
-			bias-pull-down = <MTK_PUPD_SET_R1R0_01>;
-		};
-
-		pins-rst {
-			pinmux = <MT7623_PIN_115_MSDC0_RSTB_FUNC_MSDC0_RSTB>;
-			bias-pull-up;
-		};
-	};
-
-	mmc1_pins_default: mmc1default {
-		pins-cmd-dat {
-			pinmux = <MT7623_PIN_107_MSDC1_DAT0_FUNC_MSDC1_DAT0>,
-				 <MT7623_PIN_108_MSDC1_DAT1_FUNC_MSDC1_DAT1>,
-				 <MT7623_PIN_109_MSDC1_DAT2_FUNC_MSDC1_DAT2>,
-				 <MT7623_PIN_110_MSDC1_DAT3_FUNC_MSDC1_DAT3>,
-				 <MT7623_PIN_105_MSDC1_CMD_FUNC_MSDC1_CMD>;
-			input-enable;
-			drive-strength = <MTK_DRIVE_4mA>;
-			bias-pull-up = <MTK_PUPD_SET_R1R0_10>;
-		};
-
-		pins-clk {
-			pinmux = <MT7623_PIN_106_MSDC1_CLK_FUNC_MSDC1_CLK>;
-			bias-pull-down;
-			drive-strength = <MTK_DRIVE_4mA>;
-		};
-
-		pins-wp {
-			pinmux = <MT7623_PIN_29_EINT7_FUNC_MSDC1_WP>;
-			input-enable;
-			bias-pull-up;
-		};
-
-		pins-insert {
-			pinmux = <MT7623_PIN_261_MSDC1_INS_FUNC_GPIO261>;
-			bias-pull-up;
-		};
-	};
-
-	mmc1_pins_uhs: mmc1 {
-		pins-cmd-dat {
-			pinmux = <MT7623_PIN_107_MSDC1_DAT0_FUNC_MSDC1_DAT0>,
-				 <MT7623_PIN_108_MSDC1_DAT1_FUNC_MSDC1_DAT1>,
-				 <MT7623_PIN_109_MSDC1_DAT2_FUNC_MSDC1_DAT2>,
-				 <MT7623_PIN_110_MSDC1_DAT3_FUNC_MSDC1_DAT3>,
-				 <MT7623_PIN_105_MSDC1_CMD_FUNC_MSDC1_CMD>;
-			input-enable;
-			drive-strength = <MTK_DRIVE_4mA>;
-			bias-pull-up = <MTK_PUPD_SET_R1R0_10>;
-		};
-
-		pins-clk {
-			pinmux = <MT7623_PIN_106_MSDC1_CLK_FUNC_MSDC1_CLK>;
-			drive-strength = <MTK_DRIVE_4mA>;
-			bias-pull-down = <MTK_PUPD_SET_R1R0_10>;
-		};
-	};
-
-	pcie_default: pcie_pin_default {
-		pins_cmd_dat {
-			pinmux = <MT7623_PIN_208_AUD_EXT_CK1_FUNC_PCIE0_PERST_N>,
-				 <MT7623_PIN_209_AUD_EXT_CK2_FUNC_PCIE1_PERST_N>;
-			bias-disable;
-		};
-	};
-
-	pwm_pins_a: pwm@0 {
-		pins-pwm {
-			pinmux = <MT7623_PIN_203_PWM0_FUNC_PWM0>,
-				 <MT7623_PIN_204_PWM1_FUNC_PWM1>,
-				 <MT7623_PIN_205_PWM2_FUNC_PWM2>,
-				 <MT7623_PIN_206_PWM3_FUNC_PWM3>,
-				 <MT7623_PIN_207_PWM4_FUNC_PWM4>;
-		};
-	};
-
-	spi0_pins_a: spi@0 {
-		pins-spi {
-			pinmux = <MT7623_PIN_53_SPI0_CSN_FUNC_SPI0_CS>,
-				<MT7623_PIN_54_SPI0_CK_FUNC_SPI0_CK>,
-				<MT7623_PIN_55_SPI0_MI_FUNC_SPI0_MI>,
-				<MT7623_PIN_56_SPI0_MO_FUNC_SPI0_MO>;
-			bias-disable;
-		};
-	};
-
-	uart0_pins_a: uart@0 {
-		pins-dat {
-			pinmux = <MT7623_PIN_79_URXD0_FUNC_URXD0>,
-				 <MT7623_PIN_80_UTXD0_FUNC_UTXD0>;
-		};
-	};
-
-	uart1_pins_a: uart@1 {
-		pins-dat {
-			pinmux = <MT7623_PIN_81_URXD1_FUNC_URXD1>,
-				 <MT7623_PIN_82_UTXD1_FUNC_UTXD1>;
-		};
-	};
-
-	uart2_pins_a: uart@2 {
-		pins-dat {
-			pinmux = <MT7623_PIN_14_GPIO14_FUNC_URXD2>,
-				 <MT7623_PIN_15_GPIO15_FUNC_UTXD2>;
-		};
-	};
-};
-
 &pwm {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pwm_pins_a>;
 	status = "okay";
 };
 
-&pwrap {
-	mt6323 {
-		mt6323led: led {
-			compatible = "mediatek,mt6323-led";
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			led@0 {
-				reg = <0>;
-				label = "bpi-r2:isink:green";
-				default-state = "off";
-			};
-
-			led@1 {
-				reg = <1>;
-				label = "bpi-r2:isink:red";
-				default-state = "off";
-			};
-
-			led@2 {
-				reg = <2>;
-				label = "bpi-r2:isink:blue";
-				default-state = "off";
-			};
-		};
-	};
-};
-
 &spi0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&spi0_pins_a>;
diff --git a/arch/arm/boot/dts/mt7623n-rfb-emmc.dts b/arch/arm/boot/dts/mt7623n-rfb-emmc.dts
new file mode 100644
index 000000000000..b7606130ade9
--- /dev/null
+++ b/arch/arm/boot/dts/mt7623n-rfb-emmc.dts
@@ -0,0 +1,326 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2017-2018 MediaTek Inc.
+ * Author: Sean Wang <sean.wang@mediatek.com>
+ *
+ */
+
+/dts-v1/;
+#include <dt-bindings/input/input.h>
+#include "mt7623.dtsi"
+#include "mt6323.dtsi"
+
+/ {
+	model = "MediaTek MT7623N with eMMC reference board";
+	compatible = "mediatek,mt7623n-rfb-emmc", "mediatek,mt7623";
+
+	aliases {
+		serial0 = &uart0;
+		serial1 = &uart1;
+		serial2 = &uart2;
+	};
+
+	chosen {
+		stdout-path = "serial2:115200n8";
+	};
+
+	cpus {
+		cpu@0 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+
+		cpu@1 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+
+		cpu@2 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+
+		cpu@3 {
+			proc-supply = <&mt6323_vproc_reg>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&key_pins_a>;
+
+		factory {
+			label = "factory";
+			linux,code = <BTN_0>;
+			gpios = <&pio 256 GPIO_ACTIVE_LOW>;
+		};
+
+		wps {
+			label = "wps";
+			linux,code = <KEY_WPS_BUTTON>;
+			gpios = <&pio 257 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0 0x80000000 0 0x40000000>;
+	};
+
+	reg_1p8v: regulator-1p8v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_5v: regulator-5v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	sound {
+		compatible = "mediatek,mt2701-wm8960-machine";
+		mediatek,platform = <&afe>;
+		audio-routing =
+			"Headphone", "HP_L",
+			"Headphone", "HP_R",
+			"LINPUT1", "AMIC",
+			"RINPUT1", "AMIC";
+		mediatek,audio-codec = <&wm8960>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&i2s0_pins_a>;
+	};
+};
+
+&btif {
+	status = "okay";
+};
+
+&cir {
+	pinctrl-names = "default";
+	pinctrl-0 = <&cir_pins_a>;
+	status = "okay";
+};
+
+&crypto {
+	status = "okay";
+};
+
+&eth {
+	status = "okay";
+
+	gmac0: mac@0 {
+		compatible = "mediatek,eth-mac";
+		reg = <0>;
+		phy-mode = "trgmii";
+
+		fixed-link {
+			speed = <1000>;
+			full-duplex;
+			pause;
+		};
+	};
+
+	mac@1 {
+		compatible = "mediatek,eth-mac";
+		reg = <1>;
+		phy-handle = <&phy5>;
+	};
+
+	mdio-bus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		phy5: ethernet-phy@5 {
+			reg = <5>;
+			phy-mode = "rgmii-rxid";
+		};
+
+		switch@0 {
+			compatible = "mediatek,mt7530";
+			reg = <0>;
+			reset-gpios = <&pio 33 0>;
+			core-supply = <&mt6323_vpa_reg>;
+			io-supply = <&mt6323_vemc3v3_reg>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					label = "lan0";
+				};
+
+				port@1 {
+					reg = <1>;
+					label = "lan1";
+				};
+
+				port@2 {
+					reg = <2>;
+					label = "lan2";
+				};
+
+				port@3 {
+					reg = <3>;
+					label = "lan3";
+				};
+
+				port@4 {
+					reg = <4>;
+					label = "wan";
+				};
+
+				port@6 {
+					reg = <6>;
+					label = "cpu";
+					ethernet = <&gmac0>;
+					phy-mode = "trgmii";
+
+					fixed-link {
+						speed = <1000>;
+						full-duplex;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins_a>;
+	status = "okay";
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins_b>;
+	status = "okay";
+
+	wm8960: wm8960@1a {
+		compatible = "wlf,wm8960";
+		reg = <0x1a>;
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_pins_a>;
+	status = "okay";
+};
+
+&mmc0 {
+	pinctrl-names = "default", "state_uhs";
+	pinctrl-0 = <&mmc0_pins_default>;
+	pinctrl-1 = <&mmc0_pins_uhs>;
+	status = "okay";
+	bus-width = <8>;
+	max-frequency = <50000000>;
+	cap-mmc-highspeed;
+	vmmc-supply = <&reg_3p3v>;
+	vqmmc-supply = <&reg_1p8v>;
+	non-removable;
+};
+
+&mmc1 {
+	pinctrl-names = "default", "state_uhs";
+	pinctrl-0 = <&mmc1_pins_default>;
+	pinctrl-1 = <&mmc1_pins_uhs>;
+	status = "okay";
+	bus-width = <4>;
+	max-frequency = <50000000>;
+	cap-sd-highspeed;
+	cd-gpios = <&pio 261 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&reg_3p3v>;
+	vqmmc-supply = <&reg_3p3v>;
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie_default>;
+	status = "okay";
+
+	pcie@0,0 {
+		status = "okay";
+	};
+
+	pcie@1,0 {
+		status = "okay";
+	};
+};
+
+&pcie0_phy {
+	status = "okay";
+};
+
+&pcie1_phy {
+	status = "okay";
+};
+
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm_pins_a>;
+	status = "okay";
+};
+
+&spi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi0_pins_a>;
+	status = "okay";
+};
+
+&spi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi1_pins_a>;
+	status = "okay";
+};
+
+&spi2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi2_pins_a>;
+	status = "okay";
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pins_a>;
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart1_pins_a>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_pins_a>;
+	status = "okay";
+};
+
+&usb1 {
+	vusb33-supply = <&reg_3p3v>;
+	vbus-supply = <&reg_5v>;
+	status = "okay";
+};
+
+&u3phy1 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/mt7623n-rfb-nand.dts b/arch/arm/boot/dts/mt7623n-rfb-nand.dts
index f729c718aba1..96ff3c9068ae 100644
--- a/arch/arm/boot/dts/mt7623n-rfb-nand.dts
+++ b/arch/arm/boot/dts/mt7623n-rfb-nand.dts
@@ -1,15 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2017 MediaTek Inc.
  * Author: John Crispin <john@phrozen.org>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /dts-v1/;
@@ -78,34 +71,3 @@
 		};
 	};
 };
-
-&pio {
-	nand_pins_default: nanddefault {
-		pins-ale {
-			pinmux = <MT7623_PIN_116_MSDC0_CMD_FUNC_NALE>;
-			drive-strength = <MTK_DRIVE_8mA>;
-			bias-pull-down = <MTK_PUPD_SET_R1R0_10>;
-		};
-
-		pins-dat {
-			pinmux = <MT7623_PIN_111_MSDC0_DAT7_FUNC_NLD7>,
-				 <MT7623_PIN_112_MSDC0_DAT6_FUNC_NLD6>,
-				 <MT7623_PIN_114_MSDC0_DAT4_FUNC_NLD4>,
-				 <MT7623_PIN_118_MSDC0_DAT3_FUNC_NLD3>,
-				 <MT7623_PIN_121_MSDC0_DAT0_FUNC_NLD0>,
-				 <MT7623_PIN_120_MSDC0_DAT1_FUNC_NLD1>,
-				 <MT7623_PIN_113_MSDC0_DAT5_FUNC_NLD5>,
-				 <MT7623_PIN_115_MSDC0_RSTB_FUNC_NLD8>,
-				 <MT7623_PIN_119_MSDC0_DAT2_FUNC_NLD2>;
-			input-enable;
-			drive-strength = <MTK_DRIVE_8mA>;
-			bias-pull-up;
-		};
-
-		pins-we {
-			pinmux = <MT7623_PIN_117_MSDC0_CLK_FUNC_NWEB>;
-			drive-strength = <MTK_DRIVE_8mA>;
-			bias-pull-up = <MTK_PUPD_SET_R1R0_10>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/mt7623n-rfb.dtsi b/arch/arm/boot/dts/mt7623n-rfb.dtsi
index 256c5fd947bf..5c5cc7da5dd2 100644
--- a/arch/arm/boot/dts/mt7623n-rfb.dtsi
+++ b/arch/arm/boot/dts/mt7623n-rfb.dtsi
@@ -1,16 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2017 MediaTek Inc.
  * Author: John Crispin <john@phrozen.org>
  *	   Sean Wang <sean.wang@mediatek.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /dts-v1/;
@@ -47,10 +40,11 @@
 	};
 
 	memory@80000000 {
+		device_type = "memory";
 		reg = <0 0x80000000 0 0x40000000>;
 	};
 
-	usb_p1_vbus: regulator@0 {
+	usb_p1_vbus: regulator-5v {
 		compatible = "regulator-fixed";
 		regulator-name = "usb_vbus";
 		regulator-min-microvolt = <5000000>;
diff --git a/arch/arm/boot/dts/mt8127-moose.dts b/arch/arm/boot/dts/mt8127-moose.dts
index 073e295a1cb4..308829b2da86 100644
--- a/arch/arm/boot/dts/mt8127-moose.dts
+++ b/arch/arm/boot/dts/mt8127-moose.dts
@@ -1,15 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2014 MediaTek Inc.
  * Author: Joe.C <yingjoe.chen@mediatek.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/mt8127.dtsi b/arch/arm/boot/dts/mt8127.dtsi
index 916c095d11b9..3adfc6f7859c 100644
--- a/arch/arm/boot/dts/mt8127.dtsi
+++ b/arch/arm/boot/dts/mt8127.dtsi
@@ -1,15 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2014 MediaTek Inc.
  * Author: Joe.C <yingjoe.chen@mediatek.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <dt-bindings/interrupt-controller/irq.h>
diff --git a/arch/arm/boot/dts/mt8135-evbp1.dts b/arch/arm/boot/dts/mt8135-evbp1.dts
index 460db6d05952..0ace7a40a60d 100644
--- a/arch/arm/boot/dts/mt8135-evbp1.dts
+++ b/arch/arm/boot/dts/mt8135-evbp1.dts
@@ -1,15 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2014 MediaTek Inc.
  * Author: Joe.C <yingjoe.chen@mediatek.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/mt8135.dtsi b/arch/arm/boot/dts/mt8135.dtsi
index a97b4ee4ae79..688069dc1533 100644
--- a/arch/arm/boot/dts/mt8135.dtsi
+++ b/arch/arm/boot/dts/mt8135.dtsi
@@ -1,15 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2014 MediaTek Inc.
  * Author: Joe.C <yingjoe.chen@mediatek.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <dt-bindings/clock/mt8135-clk.h>
diff --git a/arch/arm/boot/dts/omap2420-n810.dts b/arch/arm/boot/dts/omap2420-n810.dts
index 7c485fbfa535..96b9913ecc1f 100644
--- a/arch/arm/boot/dts/omap2420-n810.dts
+++ b/arch/arm/boot/dts/omap2420-n810.dts
@@ -6,11 +6,70 @@
 / {
 	model = "Nokia N810";
 	compatible = "nokia,n810", "nokia,n8x0", "ti,omap2420", "ti,omap2";
+
+	vio_ape: vio_ape {
+		compatible = "regulator-fixed";
+		regulator-name = "vio_ape";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	v28_aic: v28_aic {
+		compatible = "regulator-fixed";
+		regulator-name = "v28_aic";
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+	};
+};
+
+&omap2420_pmx {
+	mcbsp2_pins: mcbsp2_pins {
+		pinctrl-single,pins = <
+			OMAP2420_CORE_IOPAD(0x0124, PIN_INPUT | MUX_MODE1)	/* eac_ac_sclk.mcbsp2_clkx */
+			OMAP2420_CORE_IOPAD(0x0125, PIN_INPUT | MUX_MODE1)	/* eac_ac_fs.mcbsp2_fsx */
+			OMAP2420_CORE_IOPAD(0x0126, PIN_INPUT | MUX_MODE1)	/* eac_ac_din.mcbsp2_dr */
+			OMAP2420_CORE_IOPAD(0x0127, PIN_OUTPUT | MUX_MODE1)	/* eac_ac_dout.mcbsp2_dx */
+		>;
+	};
+
+	aic33_pins: aic33_pins {
+		pinctrl-single,pins = <
+			OMAP2420_CORE_IOPAD(0x0129, PIN_OUTPUT | MUX_MODE3)	/* eac_ac_rst.gpio118 */
+			OMAP2420_CORE_IOPAD(0x00e8, PIN_OUTPUT | MUX_MODE2)	/* vlynq_tx1.sys_clkout2 */
+		>;
+	};
 };
 
 &i2c2 {
-	aic3x@18 {
-		compatible = "tlv320aic3x";
+	aic33@18 {
+		compatible = "ti,tlv320aic33";
 		reg = <0x18>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&aic33_pins>;
+
+		gpio-reset = <&gpio4 22 GPIO_ACTIVE_LOW>; /* gpio118 */
+
+		ai3x-gpio-func = <
+			10 /* AIC3X_GPIO1_FUNC_DIGITAL_MIC_MODCLK */
+			5 /* AIC3X_GPIO2_FUNC_DIGITAL_MIC_INPUT */
+		>;
+		ai3x-micbias-vg = <1>; /* 2V */
+
+		AVDD-supply = <&v28_aic>;
+		DRVDD-supply = <&v28_aic>;
+		IOVDD-supply = <&vio_ape>;
+		DVDD-supply = <&vio_ape>;
+
+		assigned-clocks = <&sys_clkout2_src>, <&sys_clkout2>;
+		assigned-clock-parents = <&func_96m_ck>;
+		assigned-clock-rates = <0>, <12000000>;
 	};
 };
+
+&mcbsp2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcbsp2_pins>;
+
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/omap3-beagle-xm.dts b/arch/arm/boot/dts/omap3-beagle-xm.dts
index 0349fcc9dc26..d80587de0bbf 100644
--- a/arch/arm/boot/dts/omap3-beagle-xm.dts
+++ b/arch/arm/boot/dts/omap3-beagle-xm.dts
@@ -30,6 +30,13 @@
 		ethernet = &ethernet;
 	};
 
+	/* fixed 26MHz oscillator */
+	hfclk_26m: oscillator {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <26000000>;
+	};
+
 	leds {
 		compatible = "gpio-leds";
 
@@ -274,6 +281,9 @@
 		interrupts = <7>; /* SYS_NIRQ cascaded to intc */
 		interrupt-parent = <&intc>;
 
+		clocks = <&hfclk_26m>;
+		clock-names = "fck";
+
 		twl_audio: audio {
 			compatible = "ti,twl4030-audio";
 			codec {
diff --git a/arch/arm/boot/dts/omap3-cm-t3x.dtsi b/arch/arm/boot/dts/omap3-cm-t3x.dtsi
index 9dcb18d22cde..cdb632df152a 100644
--- a/arch/arm/boot/dts/omap3-cm-t3x.dtsi
+++ b/arch/arm/boot/dts/omap3-cm-t3x.dtsi
@@ -60,7 +60,7 @@
 		regulator-max-microvolt = <3300000>;
 	};
 
-	tv0: connector {
+	tv0: svideo-connector {
 		compatible = "svideo-connector";
 		label = "tv";
 
diff --git a/arch/arm/boot/dts/omap3-devkit8000-common.dtsi b/arch/arm/boot/dts/omap3-devkit8000-common.dtsi
index 0c0bb1b01b0b..746a658e84b6 100644
--- a/arch/arm/boot/dts/omap3-devkit8000-common.dtsi
+++ b/arch/arm/boot/dts/omap3-devkit8000-common.dtsi
@@ -349,10 +349,17 @@
 	vdda_dac-supply = <&vdac>;
 
 	port {
-		dpi_dvi_out: endpoint {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dpi_dvi_out: endpoint@0 {
+			reg = <0>;
 			remote-endpoint = <&tfp410_in>;
 			data-lines = <24>;
 		};
+
+		endpoint@1 {
+			reg = <1>;
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/omap3-devkit8000-lcd-common.dtsi b/arch/arm/boot/dts/omap3-devkit8000-lcd-common.dtsi
index 2d64bcffaaa8..1093387259e2 100644
--- a/arch/arm/boot/dts/omap3-devkit8000-lcd-common.dtsi
+++ b/arch/arm/boot/dts/omap3-devkit8000-lcd-common.dtsi
@@ -30,7 +30,10 @@
 
 &dss {
 	port {
-		dpi_lcd_out: endpoint {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dpi_lcd_out: endpoint@1 {
+			reg = <1>;
 			remote-endpoint = <&lcd_in>;
 			data-lines = <24>;
 		};
diff --git a/arch/arm/boot/dts/omap3-gta04.dtsi b/arch/arm/boot/dts/omap3-gta04.dtsi
index 4170be70460e..ac830b917776 100644
--- a/arch/arm/boot/dts/omap3-gta04.dtsi
+++ b/arch/arm/boot/dts/omap3-gta04.dtsi
@@ -30,6 +30,13 @@
 		display0 = &lcd;
 	};
 
+	/* fixed 26MHz oscillator */
+	hfclk_26m: oscillator {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <26000000>;
+	};
+
 	gpio-keys {
 		compatible = "gpio-keys";
 
@@ -312,6 +319,9 @@
 		interrupts = <7>; /* SYS_NIRQ cascaded to intc */
 		interrupt-parent = <&intc>;
 
+		clocks = <&hfclk_26m>;
+		clock-names = "fck";
+
 		twl_audio: audio {
 			compatible = "ti,twl4030-audio";
 			ti,enable-vibra = <1>;
diff --git a/arch/arm/boot/dts/omap3-pandora-common.dtsi b/arch/arm/boot/dts/omap3-pandora-common.dtsi
index f83b1029b3b7..90c98f95b2b3 100644
--- a/arch/arm/boot/dts/omap3-pandora-common.dtsi
+++ b/arch/arm/boot/dts/omap3-pandora-common.dtsi
@@ -27,6 +27,13 @@
 		display0 = &lcd;
 	};
 
+	/* fixed 26MHz oscillator */
+	hfclk_26m: oscillator {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <26000000>;
+	};
+
 	tv: connector {
 		compatible = "connector-analog-tv";
 		label = "tv";
@@ -357,6 +364,9 @@
 		interrupts = <7>; /* SYS_NIRQ cascaded to intc */
 		interrupt-parent = <&intc>;
 
+		clocks = <&hfclk_26m>;
+		clock-names = "fck";
+
 		twl_power: power {
 			compatible = "ti,twl4030-power-reset";
 			ti,use_poweroff;
@@ -611,7 +621,7 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&penirq_pins>;
 		interrupt-parent = <&gpio3>;
-		interrupts = <30 0>;	/* GPIO_94 */
+		interrupts = <30 IRQ_TYPE_NONE>;	/* GPIO_94 */
 		pendown-gpio = <&gpio3 30 GPIO_ACTIVE_HIGH>;
 		vcc-supply = <&vaux4>;
 
diff --git a/arch/arm/boot/dts/omap3-sb-t35.dtsi b/arch/arm/boot/dts/omap3-sb-t35.dtsi
index 22b4c8bdcc65..fb9842fa922c 100644
--- a/arch/arm/boot/dts/omap3-sb-t35.dtsi
+++ b/arch/arm/boot/dts/omap3-sb-t35.dtsi
@@ -34,7 +34,7 @@
 		};
 	};
 
-	dvi0: connector {
+	dvi0: dvi-connector {
 		compatible = "dvi-connector";
 		label = "dvi";
 
diff --git a/arch/arm/boot/dts/qcom-apq8064.dtsi b/arch/arm/boot/dts/qcom-apq8064.dtsi
index 5341a39c0392..4a99c9255104 100644
--- a/arch/arm/boot/dts/qcom-apq8064.dtsi
+++ b/arch/arm/boot/dts/qcom-apq8064.dtsi
@@ -444,7 +444,7 @@
 				compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm";
 				reg = <0x12450000 0x100>,
 				      <0x12400000 0x03>;
-				interrupts = <0 193 0x0>;
+				interrupts = <0 193 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GSBI1_UART_CLK>, <&gcc GSBI1_H_CLK>;
 				clock-names = "core", "iface";
 				status = "disabled";
@@ -456,11 +456,12 @@
 				pinctrl-1 = <&i2c1_pins_sleep>;
 				pinctrl-names = "default", "sleep";
 				reg = <0x12460000 0x1000>;
-				interrupts = <0 194 IRQ_TYPE_NONE>;
+				interrupts = <0 194 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GSBI1_QUP_CLK>, <&gcc GSBI1_H_CLK>;
 				clock-names = "core", "iface";
 				#address-cells = <1>;
 				#size-cells = <0>;
+				status = "disabled";
 			};
 
 		};
@@ -484,11 +485,12 @@
 				pinctrl-0 = <&i2c2_pins>;
 				pinctrl-1 = <&i2c2_pins_sleep>;
 				pinctrl-names = "default", "sleep";
-				interrupts = <0 196 IRQ_TYPE_NONE>;
+				interrupts = <0 196 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GSBI2_QUP_CLK>, <&gcc GSBI2_H_CLK>;
 				clock-names = "core", "iface";
 				#address-cells = <1>;
 				#size-cells = <0>;
+				status = "disabled";
 			};
 		};
 
@@ -508,12 +510,13 @@
 				pinctrl-1 = <&i2c3_pins_sleep>;
 				pinctrl-names = "default", "sleep";
 				reg = <0x16280000 0x1000>;
-				interrupts = <GIC_SPI 151 IRQ_TYPE_NONE>;
+				interrupts = <GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GSBI3_QUP_CLK>,
 					 <&gcc GSBI3_H_CLK>;
 				clock-names = "core", "iface";
 				#address-cells = <1>;
 				#size-cells = <0>;
+				status = "disabled";
 			};
 		};
 
@@ -534,10 +537,11 @@
 				pinctrl-1 = <&i2c4_pins_sleep>;
 				pinctrl-names = "default", "sleep";
 				reg = <0x16380000 0x1000>;
-				interrupts = <GIC_SPI 153 IRQ_TYPE_NONE>;
+				interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GSBI4_QUP_CLK>,
 					 <&gcc GSBI4_H_CLK>;
 				clock-names = "core", "iface";
+				status = "disabled";
 			};
 		};
 
@@ -556,7 +560,7 @@
 				compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm";
 				reg = <0x1a240000 0x100>,
 				      <0x1a200000 0x03>;
-				interrupts = <0 154 0x0>;
+				interrupts = <0 154 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GSBI5_UART_CLK>, <&gcc GSBI5_H_CLK>;
 				clock-names = "core", "iface";
 				status = "disabled";
@@ -565,7 +569,7 @@
 			gsbi5_spi: spi@1a280000 {
 				compatible = "qcom,spi-qup-v1.1.1";
 				reg = <0x1a280000 0x1000>;
-				interrupts = <0 155 0>;
+				interrupts = <0 155 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&spi5_default>;
 				pinctrl-1 = <&spi5_sleep>;
 				pinctrl-names = "default", "sleep";
@@ -592,7 +596,7 @@
 				compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm";
 				reg = <0x16540000 0x100>,
 				      <0x16500000 0x03>;
-				interrupts = <0 156 0x0>;
+				interrupts = <0 156 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GSBI6_UART_CLK>, <&gcc GSBI6_H_CLK>;
 				clock-names = "core", "iface";
 				status = "disabled";
@@ -604,7 +608,7 @@
 				pinctrl-1 = <&i2c6_pins_sleep>;
 				pinctrl-names = "default", "sleep";
 				reg = <0x16580000 0x1000>;
-				interrupts = <GIC_SPI 157 IRQ_TYPE_NONE>;
+				interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GSBI6_QUP_CLK>,
 					 <&gcc GSBI6_H_CLK>;
 				clock-names = "core", "iface";
@@ -628,7 +632,7 @@
 				compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm";
 				reg = <0x16640000 0x1000>,
 				      <0x16600000 0x1000>;
-				interrupts = <0 158 0x0>;
+				interrupts = <0 158 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GSBI7_UART_CLK>, <&gcc GSBI7_H_CLK>;
 				clock-names = "core", "iface";
 				status = "disabled";
@@ -640,7 +644,7 @@
 				pinctrl-1 = <&i2c7_pins_sleep>;
 				pinctrl-names = "default", "sleep";
 				reg = <0x16680000 0x1000>;
-				interrupts = <GIC_SPI 159 IRQ_TYPE_NONE>;
+				interrupts = <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GSBI7_QUP_CLK>,
 					 <&gcc GSBI7_H_CLK>;
 				clock-names = "core", "iface";
@@ -1056,7 +1060,7 @@
 			compatible		= "qcom,apq8064-ahci", "generic-ahci";
 			status			= "disabled";
 			reg			= <0x29000000 0x180>;
-			interrupts		= <GIC_SPI 209 IRQ_TYPE_NONE>;
+			interrupts		= <GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH>;
 
 			clocks			= <&gcc SFAB_SATA_S_H_CLK>,
 						<&gcc SATA_H_CLK>,
@@ -1082,7 +1086,7 @@
 		sdcc1bam:dma@12402000{
 			compatible = "qcom,bam-v1.3.0";
 			reg = <0x12402000 0x8000>;
-			interrupts = <0 98 0>;
+			interrupts = <0 98 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc SDC1_H_CLK>;
 			clock-names = "bam_clk";
 			#dma-cells = <1>;
@@ -1092,7 +1096,7 @@
 		sdcc3bam:dma@12182000{
 			compatible = "qcom,bam-v1.3.0";
 			reg = <0x12182000 0x8000>;
-			interrupts = <0 96 0>;
+			interrupts = <0 96 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc SDC3_H_CLK>;
 			clock-names = "bam_clk";
 			#dma-cells = <1>;
@@ -1102,7 +1106,7 @@
 		sdcc4bam:dma@121c2000{
 			compatible = "qcom,bam-v1.3.0";
 			reg = <0x121c2000 0x8000>;
-			interrupts = <0 95 0>;
+			interrupts = <0 95 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc SDC4_H_CLK>;
 			clock-names = "bam_clk";
 			#dma-cells = <1>;
@@ -1181,7 +1185,7 @@
 			compatible = "qcom,adreno-3xx";
 			reg = <0x04300000 0x20000>;
 			reg-names = "kgsl_3d0_reg_memory";
-			interrupts = <GIC_SPI 80 0>;
+			interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "kgsl_3d0_irq";
 			clock-names =
 			    "core_clk",
@@ -1281,7 +1285,7 @@
 			label = "MDSS DSI CTRL->0";
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <GIC_SPI 82 0>;
+			interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>;
 			reg = <0x04700000 0x200>;
 			reg-names = "dsi_ctrl";
 
@@ -1350,8 +1354,8 @@
 			    <&mmcc MDP_AXI_CLK>;
 			reg = <0x07500000 0x100000>;
 			interrupts =
-			    <GIC_SPI 63 0>,
-			    <GIC_SPI 64 0>;
+			    <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>,
+			    <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>;
 			qcom,ncb = <2>;
 		};
 
@@ -1366,8 +1370,8 @@
 			    <&mmcc MDP_AXI_CLK>;
 			reg = <0x07600000 0x100000>;
 			interrupts =
-			    <GIC_SPI 61 0>,
-			    <GIC_SPI 62 0>;
+			    <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>,
+			    <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
 			qcom,ncb = <2>;
 		};
 
@@ -1382,8 +1386,8 @@
 			    <&mmcc GFX3D_AXI_CLK>;
 			reg = <0x07c00000 0x100000>;
 			interrupts =
-			    <GIC_SPI 69 0>,
-			    <GIC_SPI 70 0>;
+			    <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>,
+			    <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
 			qcom,ncb = <3>;
 		};
 
@@ -1398,8 +1402,8 @@
 			    <&mmcc GFX3D_AXI_CLK>;
 			reg = <0x07d00000 0x100000>;
 			interrupts =
-			    <GIC_SPI 210 0>,
-			    <GIC_SPI 211 0>;
+			    <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>,
+			    <GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH>;
 			qcom,ncb = <3>;
 		};
 
@@ -1417,8 +1421,8 @@
 			#address-cells = <3>;
 			#size-cells = <2>;
 			ranges = <0x81000000 0 0 0x0fe00000 0 0x00100000   /* I/O */
-				  0x82000000 0 0 0x08000000 0 0x07e00000>; /* memory */
-			interrupts = <GIC_SPI 238 IRQ_TYPE_NONE>;
+				  0x82000000 0 0x08000000 0x08000000 0 0x07e00000>; /* memory */
+			interrupts = <GIC_SPI 238 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "msi";
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi b/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi
index e413b21ee331..418f9a022336 100644
--- a/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi
+++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi
@@ -20,6 +20,14 @@
 	model = "Qualcomm Technologies, Inc. IPQ4019/AP-DK01.1";
 	compatible = "qcom,ipq4019";
 
+	aliases {
+		serial0 = &blsp1_uart1;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
 	soc {
 		rng@22000 {
 			status = "ok";
@@ -61,7 +69,7 @@
 			status = "ok";
 		};
 
-		spi_0: spi@78b5000 {
+		spi@78b5000 {
 			pinctrl-0 = <&spi_0_pins>;
 			pinctrl-names = "default";
 			status = "ok";
diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1-c1.dts b/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1-c1.dts
new file mode 100644
index 000000000000..7a96f300bc8d
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1-c1.dts
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2018, The Linux Foundation. All rights reserved.
+
+#include "qcom-ipq4019-ap.dk04.1.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ4019/AP-DK04.1-C1";
+	compatible = "qcom,ipq4019-dk04.1-c1";
+
+	soc {
+		dma@7984000 {
+			status = "ok";
+		};
+
+		qpic-nand@79b0000 {
+			status = "ok";
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1-c3.dts b/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1-c3.dts
new file mode 100644
index 000000000000..2d1c4c6e42f1
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1-c3.dts
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2018, The Linux Foundation. All rights reserved.
+
+#include "qcom-ipq4019-ap.dk04.1.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ4019/AP-DK04.1-C3";
+	compatible = "qcom,ipq4019-ap-dk04.1-c3";
+};
diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi b/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi
new file mode 100644
index 000000000000..7c1eb1963c67
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2018, The Linux Foundation. All rights reserved.
+
+#include "qcom-ipq4019.dtsi"
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ4019/AP-DK04.1";
+
+	aliases {
+		serial0 = &blsp1_uart1;
+		serial1 = &blsp1_uart2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory {
+		device_type = "memory";
+		reg = <0x80000000 0x10000000>; /* 256MB */
+	};
+
+	soc {
+		pinctrl@1000000 {
+			serial_0_pins: serial0-pinmux {
+				pins = "gpio16", "gpio17";
+				function = "blsp_uart0";
+				bias-disable;
+			};
+
+			serial_1_pins: serial1-pinmux {
+				pins = "gpio8", "gpio9",
+					"gpio10", "gpio11";
+				function = "blsp_uart1";
+				bias-disable;
+			};
+
+			spi_0_pins: spi-0-pinmux {
+				pinmux {
+					function = "blsp_spi0";
+					pins = "gpio13", "gpio14", "gpio15";
+					bias-disable;
+				};
+				pinmux_cs {
+					function = "gpio";
+					pins = "gpio12";
+					bias-disable;
+					output-high;
+				};
+			};
+
+			i2c_0_pins: i2c-0-pinmux {
+				pins = "gpio20", "gpio21";
+				function = "blsp_i2c0";
+				bias-disable;
+			};
+
+			nand_pins: nand-pins {
+				pins = "gpio53", "gpio55", "gpio56",
+					"gpio57", "gpio58", "gpio59",
+					"gpio60", "gpio62", "gpio63",
+					"gpio64", "gpio65", "gpio66",
+					"gpio67", "gpio68", "gpio69";
+				function = "qpic";
+			};
+		};
+
+		serial@78af000 {
+			pinctrl-0 = <&serial_0_pins>;
+			pinctrl-names = "default";
+			status = "ok";
+		};
+
+		serial@78b0000 {
+			pinctrl-0 = <&serial_1_pins>;
+			pinctrl-names = "default";
+			status = "ok";
+		};
+
+		dma@7884000 {
+			status = "ok";
+		};
+
+		spi@78b5000 { /* BLSP1 QUP1 */
+			pinctrl-0 = <&spi_0_pins>;
+			pinctrl-names = "default";
+			status = "ok";
+			cs-gpios = <&tlmm 12 0>;
+
+			m25p80@0 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				reg = <0>;
+				compatible = "n25q128a11";
+				spi-max-frequency = <24000000>;
+			};
+		};
+
+		pci@40000000 {
+			status = "ok";
+			perst-gpio = <&tlmm 38 0x1>;
+		};
+
+		qpic-nand@79b0000 {
+			pinctrl-0 = <&nand_pins>;
+			pinctrl-names = "default";
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts b/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts
new file mode 100644
index 000000000000..8c7ef6537ae6
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2018, The Linux Foundation. All rights reserved.
+
+#include "qcom-ipq4019-ap.dk07.1.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ4019/AP-DK07.1-C1";
+	compatible = "qcom,ipq4019-ap-dk07.1-c1";
+
+	soc {
+		pci@40000000 {
+			status = "ok";
+			perst-gpio = <&tlmm 38 0x1>;
+		};
+
+		spi@78b6000 {
+			status = "ok";
+		};
+
+		pinctrl@1000000 {
+			serial_1_pins: serial1-pinmux {
+				pins = "gpio8", "gpio9",
+					"gpio10", "gpio11";
+				function = "blsp_uart1";
+				bias-disable;
+			};
+
+			spi_0_pins: spi-0-pinmux {
+				pinmux {
+					function = "blsp_spi0";
+					pins = "gpio13", "gpio14", "gpio15";
+					bias-disable;
+				};
+				pinmux_cs {
+					function = "gpio";
+					pins = "gpio12";
+					bias-disable;
+					output-high;
+				};
+			};
+		};
+
+		serial@78b0000 {
+			pinctrl-0 = <&serial_1_pins>;
+			pinctrl-names = "default";
+			status = "ok";
+		};
+
+		spi@78b5000 {
+			pinctrl-0 = <&spi_0_pins>;
+			pinctrl-names = "default";
+			status = "ok";
+			cs-gpios = <&tlmm 12 0>;
+
+			m25p80@0 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				reg = <0>;
+				compatible = "n25q128a11";
+				spi-max-frequency = <24000000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c2.dts b/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c2.dts
new file mode 100644
index 000000000000..af7a9028d492
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c2.dts
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2018, The Linux Foundation. All rights reserved.
+
+#include "qcom-ipq4019-ap.dk07.1.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ4019/AP-DK07.1-C2";
+	compatible = "qcom,ipq4019-ap-dk07.1-c2";
+
+	soc {
+		pinctrl@1000000 {
+			serial_1_pins: serial1-pinmux {
+				pins = "gpio8", "gpio9";
+				function = "blsp_uart1";
+				bias-disable;
+			};
+		};
+
+		serial@78b0000 {
+			pinctrl-0 = <&serial_1_pins>;
+			pinctrl-names = "default";
+			status = "ok";
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1.dtsi b/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1.dtsi
new file mode 100644
index 000000000000..9f1a5a668772
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1.dtsi
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2018, The Linux Foundation. All rights reserved.
+
+#include "qcom-ipq4019.dtsi"
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ4019/AP-DK07.1";
+
+	memory {
+		device_type = "memory";
+		reg = <0x80000000 0x20000000>; /* 512MB */
+	};
+
+	aliases {
+		serial0 = &blsp1_uart1;
+		serial1 = &blsp1_uart2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	soc {
+		pinctrl@1000000 {
+			serial_0_pins: serial0-pinmux {
+				pins = "gpio16", "gpio17";
+				function = "blsp_uart0";
+				bias-disable;
+			};
+
+			i2c_0_pins: i2c-0-pinmux {
+				pins = "gpio20", "gpio21";
+				function = "blsp_i2c0";
+				bias-disable;
+			};
+
+			nand_pins: nand-pins {
+				pins = "gpio53", "gpio55", "gpio56",
+				       "gpio57", "gpio58", "gpio59",
+				       "gpio60", "gpio62", "gpio63",
+				       "gpio64", "gpio65", "gpio66",
+				       "gpio67", "gpio68", "gpio69";
+				function = "qpic";
+                        };
+		};
+
+		serial@78af000 {
+			pinctrl-0 = <&serial_0_pins>;
+			pinctrl-names = "default";
+			status = "ok";
+		};
+
+		dma@7884000 {
+			status = "ok";
+		};
+
+		i2c@78b7000 { /* BLSP1 QUP2 */
+			pinctrl-0 = <&i2c_0_pins>;
+			pinctrl-names = "default";
+			status = "ok";
+		};
+
+		dma@7984000 {
+			status = "ok";
+		};
+
+		qpic-nand@79b0000 {
+			pinctrl-0 = <&nand_pins>;
+			pinctrl-names = "default";
+			status = "ok";
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/qcom-ipq4019.dtsi b/arch/arm/boot/dts/qcom-ipq4019.dtsi
index 10d112a4078e..7bcd7635e723 100644
--- a/arch/arm/boot/dts/qcom-ipq4019.dtsi
+++ b/arch/arm/boot/dts/qcom-ipq4019.dtsi
@@ -23,9 +23,27 @@
 	compatible = "qcom,ipq4019";
 	interrupt-parent = <&intc>;
 
+	reserved-memory {
+		#address-cells = <0x1>;
+		#size-cells = <0x1>;
+		ranges;
+
+		smem_region: smem@87e00000 {
+			reg = <0x87e00000 0x080000>;
+			no-map;
+		};
+
+		tz@87e80000 {
+			reg = <0x87e80000 0x180000>;
+			no-map;
+		};
+	};
+
 	aliases {
-		spi0 = &spi_0;
-		i2c0 = &i2c_0;
+		spi0 = &blsp1_spi1;
+		spi1 = &blsp1_spi2;
+		i2c0 = &blsp1_i2c3;
+		i2c1 = &blsp1_i2c4;
 	};
 
 	cpus {
@@ -45,7 +63,7 @@
 				48000	1100000
 				200000	1100000
 				500000	1100000
-				666000	1100000
+				716000  1100000
 			>;
 			clock-latency = <256000>;
 		};
@@ -104,6 +122,12 @@
 		};
 	};
 
+	firmware {
+		scm {
+			compatible = "qcom,scm-ipq4019";
+		};
+	};
+
 	timer {
 		compatible = "arm,armv7-timer";
 		interrupts = <1 2 0xf08>,
@@ -149,13 +173,13 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
-			interrupts = <0 208 0>;
+			interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		blsp_dma: dma@7884000 {
 			compatible = "qcom,bam-v1.7.0";
 			reg = <0x07884000 0x23000>;
-			interrupts = <GIC_SPI 238 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 238 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP1_AHB_CLK>;
 			clock-names = "bam_clk";
 			#dma-cells = <1>;
@@ -163,7 +187,7 @@
 			status = "disabled";
 		};
 
-		spi_0: spi@78b5000 {
+		blsp1_spi1: spi@78b5000 { /* BLSP1 QUP1 */
 			compatible = "qcom,spi-qup-v2.2.1";
 			reg = <0x78b5000 0x600>;
 			interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
@@ -172,10 +196,26 @@
 			clock-names = "core", "iface";
 			#address-cells = <1>;
 			#size-cells = <0>;
+			dmas = <&blsp_dma 5>, <&blsp_dma 4>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		blsp1_spi2: spi@78b6000 { /* BLSP1 QUP2 */
+			compatible = "qcom,spi-qup-v2.2.1";
+			reg = <0x78b6000 0x600>;
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_BLSP1_QUP2_SPI_APPS_CLK>,
+				<&gcc GCC_BLSP1_AHB_CLK>;
+			clock-names = "core", "iface";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&blsp_dma 7>, <&blsp_dma 6>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
-		i2c_0: i2c@78b7000 {
+		blsp1_i2c3: i2c@78b7000 { /* BLSP1 QUP3 */
 			compatible = "qcom,i2c-qup-v2.2.1";
 			reg = <0x78b7000 0x600>;
 			interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>;
@@ -184,14 +224,29 @@
 			clock-names = "iface", "core";
 			#address-cells = <1>;
 			#size-cells = <0>;
+			dmas = <&blsp_dma 9>, <&blsp_dma 8>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
+		blsp1_i2c4: i2c@78b8000 { /* BLSP1 QUP4 */
+			compatible = "qcom,i2c-qup-v2.2.1";
+			reg = <0x78b8000 0x600>;
+			interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_BLSP1_AHB_CLK>,
+				 <&gcc GCC_BLSP1_QUP2_I2C_APPS_CLK>;
+			clock-names = "iface", "core";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&blsp_dma 11>, <&blsp_dma 10>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
 
 		cryptobam: dma@8e04000 {
 			compatible = "qcom,bam-v1.7.0";
 			reg = <0x08e04000 0x20000>;
-			interrupts = <GIC_SPI 207 0>;
+			interrupts = <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_CRYPTO_AHB_CLK>;
 			clock-names = "bam_clk";
 			#dma-cells = <1>;
@@ -256,10 +311,10 @@
                         regulator;
                 };
 
-		serial@78af000 {
+		blsp1_uart1: serial@78af000 {
 			compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm";
 			reg = <0x78af000 0x200>;
-			interrupts = <0 107 0>;
+			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 			clocks = <&gcc GCC_BLSP1_UART1_APPS_CLK>,
 				<&gcc GCC_BLSP1_AHB_CLK>;
@@ -268,10 +323,10 @@
 			dma-names = "rx", "tx";
 		};
 
-		serial@78b0000 {
+		blsp1_uart2: serial@78b0000 {
 			compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm";
 			reg = <0x78b0000 0x200>;
-			interrupts = <0 108 0>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 			clocks = <&gcc GCC_BLSP1_UART2_APPS_CLK>,
 				<&gcc GCC_BLSP1_AHB_CLK>;
@@ -293,6 +348,101 @@
 			reg = <0x4ab000 0x4>;
 		};
 
+		pcie0: pci@40000000 {
+			compatible = "qcom,pcie-ipq4019", "snps,dw-pcie";
+			reg =  <0x40000000 0xf1d
+				0x40000f20 0xa8
+				0x80000 0x2000
+				0x40100000 0x1000>;
+			reg-names = "dbi", "elbi", "parf", "config";
+			device_type = "pci";
+			linux,pci-domain = <0>;
+			bus-range = <0x00 0xff>;
+			num-lanes = <1>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			ranges = <0x81000000 0 0x40200000 0x40200000 0 0x00100000
+				  0x82000000 0 0x48000000 0x48000000 0 0x10000000>;
+
+			interrupts = <GIC_SPI 141 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "msi";
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc 0 142 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
+					<0 0 0 2 &intc 0 143 IRQ_TYPE_LEVEL_HIGH>, /* int_b */
+					<0 0 0 3 &intc 0 144 IRQ_TYPE_LEVEL_HIGH>, /* int_c */
+					<0 0 0 4 &intc 0 145 IRQ_TYPE_LEVEL_HIGH>; /* int_d */
+			clocks = <&gcc GCC_PCIE_AHB_CLK>,
+				 <&gcc GCC_PCIE_AXI_M_CLK>,
+				 <&gcc GCC_PCIE_AXI_S_CLK>;
+			clock-names = "aux",
+				      "master_bus",
+				      "slave_bus";
+
+			resets = <&gcc PCIE_AXI_M_ARES>,
+				 <&gcc PCIE_AXI_S_ARES>,
+				 <&gcc PCIE_PIPE_ARES>,
+				 <&gcc PCIE_AXI_M_VMIDMT_ARES>,
+				 <&gcc PCIE_AXI_S_XPU_ARES>,
+				 <&gcc PCIE_PARF_XPU_ARES>,
+				 <&gcc PCIE_PHY_ARES>,
+				 <&gcc PCIE_AXI_M_STICKY_ARES>,
+				 <&gcc PCIE_PIPE_STICKY_ARES>,
+				 <&gcc PCIE_PWR_ARES>,
+				 <&gcc PCIE_AHB_ARES>,
+				 <&gcc PCIE_PHY_AHB_ARES>;
+			reset-names = "axi_m",
+				      "axi_s",
+				      "pipe",
+				      "axi_m_vmid",
+				      "axi_s_xpu",
+				      "parf",
+				      "phy",
+				      "axi_m_sticky",
+				      "pipe_sticky",
+				      "pwr",
+				      "ahb",
+				      "phy_ahb";
+
+			status = "disabled";
+		};
+
+		qpic_bam: dma@7984000 {
+			compatible = "qcom,bam-v1.7.0";
+			reg = <0x7984000 0x1a000>;
+			interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_QPIC_CLK>;
+			clock-names = "bam_clk";
+			#dma-cells = <1>;
+			qcom,ee = <0>;
+			status = "disabled";
+		};
+
+		nand: qpic-nand@79b0000 {
+			compatible = "qcom,ipq4019-nand";
+			reg = <0x79b0000 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&gcc GCC_QPIC_CLK>,
+				 <&gcc GCC_QPIC_AHB_CLK>;
+			clock-names = "core", "aon";
+
+			dmas = <&qpic_bam 0>,
+			       <&qpic_bam 1>,
+			       <&qpic_bam 2>;
+			dma-names = "tx", "rx", "cmd";
+			status = "disabled";
+
+			nand@0 {
+				reg = <0>;
+
+				nand-ecc-strength = <4>;
+				nand-ecc-step-size = <512>;
+				nand-bus-width = <8>;
+			};
+		};
+
 		wifi0: wifi@a000000 {
 			compatible = "qcom,ipq4019-wifi";
 			reg = <0xa000000 0x200000>;
@@ -326,7 +476,7 @@
 				     <GIC_SPI 45 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 46 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 47 IRQ_TYPE_EDGE_RISING>,
-				     <GIC_SPI 168 IRQ_TYPE_NONE>;
+				     <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names =  "msi0",  "msi1",  "msi2",  "msi3",
 					   "msi4",  "msi5",  "msi6",  "msi7",
 					   "msi8",  "msi9", "msi10", "msi11",
@@ -368,7 +518,7 @@
 				     <GIC_SPI 61 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 62 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 63 IRQ_TYPE_EDGE_RISING>,
-				     <GIC_SPI 169 IRQ_TYPE_NONE>;
+				     <GIC_SPI 169 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names =  "msi0",  "msi1",  "msi2",  "msi3",
 					   "msi4",  "msi5",  "msi6",  "msi7",
 					   "msi8",  "msi9", "msi10", "msi11",
diff --git a/arch/arm/boot/dts/qcom-msm8660.dtsi b/arch/arm/boot/dts/qcom-msm8660.dtsi
index 33030f9419fe..70698941f64c 100644
--- a/arch/arm/boot/dts/qcom-msm8660.dtsi
+++ b/arch/arm/boot/dts/qcom-msm8660.dtsi
@@ -452,7 +452,7 @@
 			clock-names = "ram";
 
 			rpmcc: clock-controller {
-				compatible	= "qcom,rpmcc-apq8660", "qcom,rpmcc";
+				compatible	= "qcom,rpmcc-msm8660", "qcom,rpmcc";
 				#clock-cells = <1>;
 			};
 
diff --git a/arch/arm/boot/dts/qcom-msm8974-sony-xperia-amami.dts b/arch/arm/boot/dts/qcom-msm8974-sony-xperia-amami.dts
new file mode 100644
index 000000000000..5669f5f58a86
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-msm8974-sony-xperia-amami.dts
@@ -0,0 +1,436 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "qcom-msm8974.dtsi"
+#include "qcom-pm8841.dtsi"
+#include "qcom-pm8941.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+
+/ {
+	model = "Sony Xperia Z1 Compact";
+	compatible = "sony,xperia-amami", "qcom,msm8974";
+
+	aliases {
+		serial0 = &blsp1_uart2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		input-name = "gpio-keys";
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio_keys_pin_a>;
+
+		volume-down {
+			label = "volume_down";
+			gpios = <&pm8941_gpios 2 GPIO_ACTIVE_LOW>;
+			linux,input-type = <1>;
+			linux,code = <KEY_VOLUMEDOWN>;
+		};
+
+		camera-snapshot {
+			label = "camera_snapshot";
+			gpios = <&pm8941_gpios 3 GPIO_ACTIVE_LOW>;
+			linux,input-type = <1>;
+			linux,code = <KEY_CAMERA>;
+		};
+
+		camera-focus {
+			label = "camera_focus";
+			gpios = <&pm8941_gpios 4 GPIO_ACTIVE_LOW>;
+			linux,input-type = <1>;
+			linux,code = <KEY_CAMERA_FOCUS>;
+		};
+
+		volume-up {
+			label = "volume_up";
+			gpios = <&pm8941_gpios 5 GPIO_ACTIVE_LOW>;
+			linux,input-type = <1>;
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	memory@0 {
+		reg = <0 0x40000000>, <0x40000000 0x40000000>;
+		device_type = "memory";
+	};
+
+	smd {
+		rpm {
+			rpm_requests {
+				pm8841-regulators {
+					s1 {
+						regulator-min-microvolt = <675000>;
+						regulator-max-microvolt = <1050000>;
+					};
+
+					s2 {
+						regulator-min-microvolt = <500000>;
+						regulator-max-microvolt = <1050000>;
+					};
+
+					s3 {
+						regulator-min-microvolt = <500000>;
+						regulator-max-microvolt = <1050000>;
+					};
+
+					s4 {
+						regulator-min-microvolt = <500000>;
+						regulator-max-microvolt = <1050000>;
+					};
+				};
+
+				pm8941-regulators {
+					vdd_l1_l3-supply = <&pm8941_s1>;
+					vdd_l2_lvs1_2_3-supply = <&pm8941_s3>;
+					vdd_l4_l11-supply = <&pm8941_s1>;
+					vdd_l5_l7-supply = <&pm8941_s2>;
+					vdd_l6_l12_l14_l15-supply = <&pm8941_s2>;
+					vdd_l9_l10_l17_l22-supply = <&vreg_boost>;
+					vdd_l13_l20_l23_l24-supply = <&vreg_boost>;
+					vdd_l21-supply = <&vreg_boost>;
+
+					s1 {
+						regulator-min-microvolt = <1300000>;
+						regulator-max-microvolt = <1300000>;
+						regulator-always-on;
+						regulator-boot-on;
+					};
+
+					s2 {
+						regulator-min-microvolt = <2150000>;
+						regulator-max-microvolt = <2150000>;
+						regulator-boot-on;
+					};
+
+					s3 {
+						regulator-min-microvolt = <1800000>;
+						regulator-max-microvolt = <1800000>;
+						regulator-always-on;
+						regulator-boot-on;
+					};
+
+					s4 {
+						regulator-min-microvolt = <5000000>;
+						regulator-max-microvolt = <5000000>;
+					};
+
+					l1 {
+						regulator-min-microvolt = <1225000>;
+						regulator-max-microvolt = <1225000>;
+
+						regulator-always-on;
+						regulator-boot-on;
+					};
+
+					l2 {
+						regulator-min-microvolt = <1200000>;
+						regulator-max-microvolt = <1200000>;
+					};
+
+					l3 {
+						regulator-min-microvolt = <1200000>;
+						regulator-max-microvolt = <1200000>;
+					};
+
+					l4 {
+						regulator-min-microvolt = <1225000>;
+						regulator-max-microvolt = <1225000>;
+					};
+
+					l5 {
+						regulator-min-microvolt = <1800000>;
+						regulator-max-microvolt = <1800000>;
+					};
+
+					l6 {
+						regulator-min-microvolt = <1800000>;
+						regulator-max-microvolt = <1800000>;
+
+						regulator-boot-on;
+					};
+
+					l7 {
+						regulator-min-microvolt = <1800000>;
+						regulator-max-microvolt = <1800000>;
+
+						regulator-boot-on;
+					};
+
+					l8 {
+						regulator-min-microvolt = <1800000>;
+						regulator-max-microvolt = <1800000>;
+					};
+
+					l9 {
+						regulator-min-microvolt = <1800000>;
+						regulator-max-microvolt = <2950000>;
+					};
+
+					l11 {
+						regulator-min-microvolt = <1300000>;
+						regulator-max-microvolt = <1350000>;
+					};
+
+					l12 {
+						regulator-min-microvolt = <1800000>;
+						regulator-max-microvolt = <1800000>;
+
+						regulator-always-on;
+						regulator-boot-on;
+					};
+
+					l13 {
+						regulator-min-microvolt = <1800000>;
+						regulator-max-microvolt = <2950000>;
+
+						regulator-boot-on;
+					};
+
+					l14 {
+						regulator-min-microvolt = <1800000>;
+						regulator-max-microvolt = <1800000>;
+					};
+
+					l15 {
+						regulator-min-microvolt = <2050000>;
+						regulator-max-microvolt = <2050000>;
+					};
+
+					l16 {
+						regulator-min-microvolt = <2700000>;
+						regulator-max-microvolt = <2700000>;
+					};
+
+					l17 {
+						regulator-min-microvolt = <2700000>;
+						regulator-max-microvolt = <2700000>;
+					};
+
+					l18 {
+						regulator-min-microvolt = <2850000>;
+						regulator-max-microvolt = <2850000>;
+					};
+
+					l19 {
+						regulator-min-microvolt = <3300000>;
+						regulator-max-microvolt = <3300000>;
+					};
+
+					l20 {
+						regulator-min-microvolt = <2950000>;
+						regulator-max-microvolt = <2950000>;
+
+						regulator-allow-set-load;
+						regulator-boot-on;
+						regulator-system-load = <200000>;
+					};
+
+					l21 {
+						regulator-min-microvolt = <2950000>;
+						regulator-max-microvolt = <2950000>;
+
+						regulator-boot-on;
+					};
+
+					l22 {
+						regulator-min-microvolt = <3000000>;
+						regulator-max-microvolt = <3000000>;
+					};
+
+					l23 {
+						regulator-min-microvolt = <2800000>;
+						regulator-max-microvolt = <2800000>;
+					};
+
+					l24 {
+						regulator-min-microvolt = <3075000>;
+						regulator-max-microvolt = <3075000>;
+
+						regulator-boot-on;
+					};
+				};
+			};
+		};
+	};
+};
+
+&soc {
+	sdhci@f9824900 {
+		status = "ok";
+
+		vmmc-supply = <&pm8941_l20>;
+		vqmmc-supply = <&pm8941_s3>;
+
+		bus-width = <8>;
+		non-removable;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&sdhc1_pin_a>;
+	};
+
+	sdhci@f98a4900 {
+		status = "ok";
+
+		bus-width = <4>;
+
+		vmmc-supply = <&pm8941_l21>;
+		vqmmc-supply = <&pm8941_l13>;
+
+		cd-gpios = <&msmgpio 62 GPIO_ACTIVE_LOW>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&sdhc2_pin_a>, <&sdhc2_cd_pin_a>;
+	};
+
+	serial@f991e000 {
+		status = "ok";
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&blsp1_uart2_pin_a>;
+	};
+
+
+	pinctrl@fd510000 {
+		blsp1_uart2_pin_a: blsp1-uart2-pin-active {
+			rx {
+				pins = "gpio5";
+				function = "blsp_uart2";
+
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			tx {
+				pins = "gpio4";
+				function = "blsp_uart2";
+
+				drive-strength = <4>;
+				bias-disable;
+			};
+		};
+
+		i2c2_pins: i2c2 {
+			mux {
+				pins = "gpio6", "gpio7";
+				function = "blsp_i2c2";
+
+				drive-strength = <2>;
+				bias-disable;
+			};
+		};
+
+		sdhc1_pin_a: sdhc1-pin-active {
+			clk {
+				pins = "sdc1_clk";
+				drive-strength = <16>;
+				bias-disable;
+			};
+
+			cmd-data {
+				pins = "sdc1_cmd", "sdc1_data";
+				drive-strength = <10>;
+				bias-pull-up;
+			};
+		};
+
+		sdhc2_cd_pin_a: sdhc2-cd-pin-active {
+			pins = "gpio62";
+			function = "gpio";
+
+			drive-strength = <2>;
+			bias-disable;
+		 };
+
+		sdhc2_pin_a: sdhc2-pin-active {
+			clk {
+				pins = "sdc2_clk";
+				drive-strength = <10>;
+				bias-disable;
+			};
+
+			cmd-data {
+				pins = "sdc2_cmd", "sdc2_data";
+				drive-strength = <6>;
+				bias-pull-up;
+			};
+		};
+	};
+
+	dma-controller@f9944000 {
+		qcom,controlled-remotely;
+	};
+
+	usb@f9a55000 {
+		status = "ok";
+
+		phys = <&usb_hs1_phy>;
+		phy-select = <&tcsr 0xb000 0>;
+		extcon = <&smbb>, <&usb_id>;
+		vbus-supply = <&chg_otg>;
+
+		hnp-disable;
+		srp-disable;
+		adp-disable;
+
+		ulpi {
+			phy@a {
+				status = "ok";
+
+				v1p8-supply = <&pm8941_l6>;
+				v3p3-supply = <&pm8941_l24>;
+
+				extcon = <&smbb>;
+				qcom,init-seq = /bits/ 8 <0x1 0x64>;
+			};
+		};
+	};
+};
+
+&spmi_bus {
+	pm8941@0 {
+		charger@1000 {
+			qcom,fast-charge-safe-current = <1300000>;
+			qcom,fast-charge-current-limit = <1300000>;
+			qcom,dc-current-limit = <1300000>;
+			qcom,fast-charge-safe-voltage = <4400000>;
+			qcom,fast-charge-high-threshold-voltage = <4350000>;
+			qcom,fast-charge-low-threshold-voltage = <3400000>;
+			qcom,auto-recharge-threshold-voltage = <4200000>;
+			qcom,minimum-input-voltage = <4300000>;
+		};
+
+		gpios@c000 {
+			gpio_keys_pin_a: gpio-keys-active {
+				pins = "gpio2", "gpio3", "gpio4", "gpio5";
+				function = "normal";
+
+				bias-pull-up;
+				power-source = <PM8941_GPIO_S3>;
+			};
+		};
+
+		coincell@2800 {
+			status = "ok";
+			qcom,rset-ohms = <2100>;
+			qcom,vset-millivolts = <3000>;
+		};
+	};
+
+	pm8941@1 {
+		wled@d800 {
+			status = "ok";
+
+			qcom,cs-out;
+			qcom,current-limit = <20>;
+			qcom,current-boost-limit = <805>;
+			qcom,switching-freq = <1600>;
+			qcom,ovp = <29>;
+			qcom,num-strings = <2>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/qcom-pm8941.dtsi b/arch/arm/boot/dts/qcom-pm8941.dtsi
index 1d5ef55c7ee5..2515c5c217ac 100644
--- a/arch/arm/boot/dts/qcom-pm8941.dtsi
+++ b/arch/arm/boot/dts/qcom-pm8941.dtsi
@@ -139,6 +139,9 @@
 			#size-cells = <0>;
 			#io-channel-cells = <1>;
 
+			bat_temp {
+				reg = <VADC_LR_MUX1_BAT_THERM>;
+			};
 			die_temp {
 				reg = <VADC_DIE_TEMP>;
 			};
@@ -154,6 +157,9 @@
 			ref_vdd {
 				reg = <VADC_VDD_VADC>;
 			};
+			vbat_sns {
+				reg = <VADC_VBAT_SNS>;
+			};
 		};
 
 		pm8941_iadc: iadc@3600 {
diff --git a/arch/arm/boot/dts/r7s72100.dtsi b/arch/arm/boot/dts/r7s72100.dtsi
index ab9645a42eca..a54822e97bac 100644
--- a/arch/arm/boot/dts/r7s72100.dtsi
+++ b/arch/arm/boot/dts/r7s72100.dtsi
@@ -15,7 +15,6 @@
 
 / {
 	compatible = "renesas,r7s72100";
-	interrupt-parent = <&gic>;
 	#address-cells = <1>;
 	#size-cells = <1>;
 
@@ -31,61 +30,370 @@
 		spi4 = &spi4;
 	};
 
-	clocks {
-		ranges;
+	/* Fixed factor clocks */
+	b_clk: b {
+		#clock-cells = <0>;
+		compatible = "fixed-factor-clock";
+		clocks = <&cpg_clocks R7S72100_CLK_PLL>;
+		clock-mult = <1>;
+		clock-div = <3>;
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a9";
+			reg = <0>;
+			clock-frequency = <400000000>;
+			clocks = <&cpg_clocks R7S72100_CLK_I>;
+			next-level-cache = <&L2>;
+		};
+	};
+
+	/* External clocks */
+	extal_clk: extal {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		/* If clk present, value must be set by board */
+		clock-frequency = <0>;
+	};
+
+	p0_clk: p0 {
+		#clock-cells = <0>;
+		compatible = "fixed-factor-clock";
+		clocks = <&cpg_clocks R7S72100_CLK_PLL>;
+		clock-mult = <1>;
+		clock-div = <12>;
+	};
+
+	p1_clk: p1 {
+		#clock-cells = <0>;
+		compatible = "fixed-factor-clock";
+		clocks = <&cpg_clocks R7S72100_CLK_PLL>;
+		clock-mult = <1>;
+		clock-div = <6>;
+	};
+
+	pmu {
+		compatible = "arm,cortex-a9-pmu";
+		interrupts-extended = <&gic GIC_PPI 0 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	rtc_x1_clk: rtc_x1 {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		/* If clk present, value must be set by board to 32678 */
+		clock-frequency = <0>;
+	};
+
+	rtc_x3_clk: rtc_x3 {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		/* If clk present, value must be set by board to 4000000 */
+		clock-frequency = <0>;
+	};
+
+	soc {
+		compatible = "simple-bus";
+		interrupt-parent = <&gic>;
+
 		#address-cells = <1>;
 		#size-cells = <1>;
+		ranges;
+
+		L2: cache-controller@3ffff000 {
+			compatible = "arm,pl310-cache";
+			reg = <0x3ffff000 0x1000>;
+			interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+			arm,early-bresp-disable;
+			arm,full-line-zero-disable;
+			cache-unified;
+			cache-level = <2>;
+		};
+
+		scif0: serial@e8007000 {
+			compatible = "renesas,scif-r7s72100", "renesas,scif";
+			reg = <0xe8007000 64>;
+			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp4_clks R7S72100_CLK_SCIF0>;
+			clock-names = "fck";
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
+		};
+
+		scif1: serial@e8007800 {
+			compatible = "renesas,scif-r7s72100", "renesas,scif";
+			reg = <0xe8007800 64>;
+			interrupts = <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp4_clks R7S72100_CLK_SCIF1>;
+			clock-names = "fck";
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
+		};
+
+		scif2: serial@e8008000 {
+			compatible = "renesas,scif-r7s72100", "renesas,scif";
+			reg = <0xe8008000 64>;
+			interrupts = <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp4_clks R7S72100_CLK_SCIF2>;
+			clock-names = "fck";
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
+		};
+
+		scif3: serial@e8008800 {
+			compatible = "renesas,scif-r7s72100", "renesas,scif";
+			reg = <0xe8008800 64>;
+			interrupts = <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp4_clks R7S72100_CLK_SCIF3>;
+			clock-names = "fck";
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
+		};
 
-		/* External clocks */
-		extal_clk: extal {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			/* If clk present, value must be set by board */
-			clock-frequency = <0>;
+		scif4: serial@e8009000 {
+			compatible = "renesas,scif-r7s72100", "renesas,scif";
+			reg = <0xe8009000 64>;
+			interrupts = <GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp4_clks R7S72100_CLK_SCIF4>;
+			clock-names = "fck";
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
 
-		usb_x1_clk: usb_x1 {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			/* If clk present, value must be set by board */
-			clock-frequency = <0>;
+		scif5: serial@e8009800 {
+			compatible = "renesas,scif-r7s72100", "renesas,scif";
+			reg = <0xe8009800 64>;
+			interrupts = <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp4_clks R7S72100_CLK_SCIF5>;
+			clock-names = "fck";
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
 
-		rtc_x1_clk: rtc_x1 {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			/* If clk present, value must be set by board to 32678 */
-			clock-frequency = <0>;
+		scif6: serial@e800a000 {
+			compatible = "renesas,scif-r7s72100", "renesas,scif";
+			reg = <0xe800a000 64>;
+			interrupts = <GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp4_clks R7S72100_CLK_SCIF6>;
+			clock-names = "fck";
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
 
-		rtc_x3_clk: rtc_x3 {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			/* If clk present, value must be set by board to 4000000 */
-			clock-frequency = <0>;
+		scif7: serial@e800a800 {
+			compatible = "renesas,scif-r7s72100", "renesas,scif";
+			reg = <0xe800a800 64>;
+			interrupts = <GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp4_clks R7S72100_CLK_SCIF7>;
+			clock-names = "fck";
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
 
-		/* Fixed factor clocks */
-		b_clk: b {
-			#clock-cells = <0>;
-			compatible = "fixed-factor-clock";
-			clocks = <&cpg_clocks R7S72100_CLK_PLL>;
-			clock-mult = <1>;
-			clock-div = <3>;
+		spi0: spi@e800c800 {
+			compatible = "renesas,rspi-r7s72100", "renesas,rspi-rz";
+			reg = <0xe800c800 0x24>;
+			interrupts = <GIC_SPI 238 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 239 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 240 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error", "rx", "tx";
+			clocks = <&mstp10_clks R7S72100_CLK_SPI0>;
+			power-domains = <&cpg_clocks>;
+			num-cs = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
 		};
-		p1_clk: p1 {
-			#clock-cells = <0>;
-			compatible = "fixed-factor-clock";
-			clocks = <&cpg_clocks R7S72100_CLK_PLL>;
-			clock-mult = <1>;
-			clock-div = <6>;
+
+		spi1: spi@e800d000 {
+			compatible = "renesas,rspi-r7s72100", "renesas,rspi-rz";
+			reg = <0xe800d000 0x24>;
+			interrupts = <GIC_SPI 241 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 242 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 243 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error", "rx", "tx";
+			clocks = <&mstp10_clks R7S72100_CLK_SPI1>;
+			power-domains = <&cpg_clocks>;
+			num-cs = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		spi2: spi@e800d800 {
+			compatible = "renesas,rspi-r7s72100", "renesas,rspi-rz";
+			reg = <0xe800d800 0x24>;
+			interrupts = <GIC_SPI 244 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 245 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error", "rx", "tx";
+			clocks = <&mstp10_clks R7S72100_CLK_SPI2>;
+			power-domains = <&cpg_clocks>;
+			num-cs = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
 		};
-		p0_clk: p0 {
-			#clock-cells = <0>;
-			compatible = "fixed-factor-clock";
-			clocks = <&cpg_clocks R7S72100_CLK_PLL>;
-			clock-mult = <1>;
-			clock-div = <12>;
+
+		spi3: spi@e800e000 {
+			compatible = "renesas,rspi-r7s72100", "renesas,rspi-rz";
+			reg = <0xe800e000 0x24>;
+			interrupts = <GIC_SPI 247 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 248 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 249 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error", "rx", "tx";
+			clocks = <&mstp10_clks R7S72100_CLK_SPI3>;
+			power-domains = <&cpg_clocks>;
+			num-cs = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		spi4: spi@e800e800 {
+			compatible = "renesas,rspi-r7s72100", "renesas,rspi-rz";
+			reg = <0xe800e800 0x24>;
+			interrupts = <GIC_SPI 250 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 251 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 252 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error", "rx", "tx";
+			clocks = <&mstp10_clks R7S72100_CLK_SPI4>;
+			power-domains = <&cpg_clocks>;
+			num-cs = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		usbhs0: usb@e8010000 {
+			compatible = "renesas,usbhs-r7s72100", "renesas,rza1-usbhs";
+			reg = <0xe8010000 0x1a0>;
+			interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp7_clks R7S72100_CLK_USB0>;
+			renesas,buswait = <4>;
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
+		};
+
+		usbhs1: usb@e8207000 {
+			compatible = "renesas,usbhs-r7s72100", "renesas,rza1-usbhs";
+			reg = <0xe8207000 0x1a0>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp7_clks R7S72100_CLK_USB1>;
+			renesas,buswait = <4>;
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
+		};
+
+		mmcif: mmc@e804c800 {
+			compatible = "renesas,mmcif-r7s72100", "renesas,sh-mmcif";
+			reg = <0xe804c800 0x80>;
+			interrupts = <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 269 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 267 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp8_clks R7S72100_CLK_MMCIF>;
+			power-domains = <&cpg_clocks>;
+			reg-io-width = <4>;
+			bus-width = <8>;
+			status = "disabled";
+		};
+
+		sdhi0: sd@e804e000 {
+			compatible = "renesas,sdhi-r7s72100";
+			reg = <0xe804e000 0x100>;
+			interrupts = <GIC_SPI 270 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 271 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&mstp12_clks R7S72100_CLK_SDHI00>,
+				 <&mstp12_clks R7S72100_CLK_SDHI01>;
+			clock-names = "core", "cd";
+			power-domains = <&cpg_clocks>;
+			cap-sd-highspeed;
+			cap-sdio-irq;
+			status = "disabled";
+		};
+
+		sdhi1: sd@e804e800 {
+			compatible = "renesas,sdhi-r7s72100";
+			reg = <0xe804e800 0x100>;
+			interrupts = <GIC_SPI 273 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 274 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 275 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&mstp12_clks R7S72100_CLK_SDHI10>,
+				 <&mstp12_clks R7S72100_CLK_SDHI11>;
+			clock-names = "core", "cd";
+			power-domains = <&cpg_clocks>;
+			cap-sd-highspeed;
+			cap-sdio-irq;
+			status = "disabled";
+		};
+
+		gic: interrupt-controller@e8201000 {
+			compatible = "arm,pl390";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0xe8201000 0x1000>,
+				<0xe8202000 0x1000>;
+		};
+
+		ether: ethernet@e8203000 {
+			compatible = "renesas,ether-r7s72100";
+			reg = <0xe8203000 0x800>,
+			      <0xe8204800 0x200>;
+			interrupts = <GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp7_clks R7S72100_CLK_ETHER>;
+			power-domains = <&cpg_clocks>;
+			phy-mode = "mii";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		ceu: camera@e8210000 {
+			reg = <0xe8210000 0x3000>;
+			compatible = "renesas,r7s72100-ceu";
+			interrupts = <GIC_SPI 332 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp6_clks R7S72100_CLK_CEU>;
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
+		};
+
+		wdt: watchdog@fcfe0000 {
+			compatible = "renesas,r7s72100-wdt", "renesas,rza-wdt";
+			reg = <0xfcfe0000 0x6>;
+			interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&p0_clk>;
 		};
 
 		/* Special CPG clocks */
@@ -135,9 +443,9 @@
 			#clock-cells = <1>;
 			compatible = "renesas,r7s72100-mstp-clocks", "renesas,cpg-mstp-clocks";
 			reg = <0xfcfe042c 4>;
-			clocks = <&p0_clk>;
-			clock-indices = <R7S72100_CLK_RTC>;
-			clock-output-names = "rtc";
+			clocks = <&b_clk>, <&p0_clk>;
+			clock-indices = <R7S72100_CLK_CEU R7S72100_CLK_RTC>;
+			clock-output-names = "ceu", "rtc";
 		};
 
 		mstp7_clks: mstp7_clks@fcfe0430 {
@@ -192,479 +500,209 @@
 			>;
 			clock-output-names = "sdhi00", "sdhi01", "sdhi10", "sdhi11";
 		};
-	};
-
-	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		cpu@0 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a9";
-			reg = <0>;
-			clock-frequency = <400000000>;
-			clocks = <&cpg_clocks R7S72100_CLK_I>;
-			next-level-cache = <&L2>;
-		};
-	};
-
-	pinctrl: pin-controller@fcfe3000 {
-		compatible = "renesas,r7s72100-ports";
-
-		reg = <0xfcfe3000 0x4230>;
-
-		port0: gpio-0 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 0 6>;
-		};
-
-		port1: gpio-1 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 16 16>;
-		};
 
-		port2: gpio-2 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 32 16>;
+		pinctrl: pin-controller@fcfe3000 {
+			compatible = "renesas,r7s72100-ports";
+
+			reg = <0xfcfe3000 0x4230>;
+
+			port0: gpio-0 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 0 6>;
+			};
+
+			port1: gpio-1 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 16 16>;
+			};
+
+			port2: gpio-2 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 32 16>;
+			};
+
+			port3: gpio-3 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 48 16>;
+			};
+
+			port4: gpio-4 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 64 16>;
+			};
+
+			port5: gpio-5 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 80 11>;
+			};
+
+			port6: gpio-6 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 96 16>;
+			};
+
+			port7: gpio-7 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 112 16>;
+			};
+
+			port8: gpio-8 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 128 16>;
+			};
+
+			port9: gpio-9 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 144 8>;
+			};
+
+			port10: gpio-10 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 160 16>;
+			};
+
+			port11: gpio-11 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&pinctrl 0 176 16>;
+			};
 		};
 
-		port3: gpio-3 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 48 16>;
+		ostm0: timer@fcfec000 {
+			compatible = "renesas,r7s72100-ostm", "renesas,ostm";
+			reg = <0xfcfec000 0x30>;
+			interrupts = <GIC_SPI 102 IRQ_TYPE_EDGE_RISING>;
+			clocks = <&mstp5_clks R7S72100_CLK_OSTM0>;
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
 
-		port4: gpio-4 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 64 16>;
+		ostm1: timer@fcfec400 {
+			compatible = "renesas,r7s72100-ostm", "renesas,ostm";
+			reg = <0xfcfec400 0x30>;
+			interrupts = <GIC_SPI 103 IRQ_TYPE_EDGE_RISING>;
+			clocks = <&mstp5_clks R7S72100_CLK_OSTM1>;
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
 
-		port5: gpio-5 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 80 11>;
+		i2c0: i2c@fcfee000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,riic-r7s72100", "renesas,riic-rz";
+			reg = <0xfcfee000 0x44>;
+			interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 158 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 159 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp9_clks R7S72100_CLK_I2C0>;
+			clock-frequency = <100000>;
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
 
-		port6: gpio-6 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 96 16>;
+		i2c1: i2c@fcfee400 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,riic-r7s72100", "renesas,riic-rz";
+			reg = <0xfcfee400 0x44>;
+			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 166 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 167 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 169 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 172 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp9_clks R7S72100_CLK_I2C1>;
+			clock-frequency = <100000>;
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
 
-		port7: gpio-7 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 112 16>;
+		i2c2: i2c@fcfee800 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,riic-r7s72100", "renesas,riic-rz";
+			reg = <0xfcfee800 0x44>;
+			interrupts = <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 174 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 175 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 178 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 179 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp9_clks R7S72100_CLK_I2C2>;
+			clock-frequency = <100000>;
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
 
-		port8: gpio-8 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 128 16>;
+		i2c3: i2c@fcfeec00 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,riic-r7s72100", "renesas,riic-rz";
+			reg = <0xfcfeec00 0x44>;
+			interrupts = <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 182 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 183 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp9_clks R7S72100_CLK_I2C3>;
+			clock-frequency = <100000>;
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
 
-		port9: gpio-9 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 144 8>;
+		mtu2: timer@fcff0000 {
+			compatible = "renesas,mtu2-r7s72100", "renesas,mtu2";
+			reg = <0xfcff0000 0x400>;
+			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "tgi0a";
+			clocks = <&mstp3_clks R7S72100_CLK_MTU2>;
+			clock-names = "fck";
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
 
-		port10: gpio-10 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 160 16>;
+		rtc: rtc@fcff1000 {
+			compatible = "renesas,r7s72100-rtc", "renesas,sh-rtc";
+			reg = <0xfcff1000 0x2e>;
+			interrupts = <GIC_SPI 276 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 277 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 278 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "alarm", "period", "carry";
+			clocks = <&mstp6_clks R7S72100_CLK_RTC>, <&rtc_x1_clk>,
+				 <&rtc_x3_clk>, <&extal_clk>;
+			clock-names = "fck", "rtc_x1", "rtc_x3", "extal";
+			power-domains = <&cpg_clocks>;
+			status = "disabled";
 		};
-
-		port11: gpio-11 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl 0 176 16>;
-		};
-	};
-
-	scif0: serial@e8007000 {
-		compatible = "renesas,scif-r7s72100", "renesas,scif";
-		reg = <0xe8007000 64>;
-		interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp4_clks R7S72100_CLK_SCIF0>;
-		clock-names = "fck";
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	scif1: serial@e8007800 {
-		compatible = "renesas,scif-r7s72100", "renesas,scif";
-		reg = <0xe8007800 64>;
-		interrupts = <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp4_clks R7S72100_CLK_SCIF1>;
-		clock-names = "fck";
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	scif2: serial@e8008000 {
-		compatible = "renesas,scif-r7s72100", "renesas,scif";
-		reg = <0xe8008000 64>;
-		interrupts = <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp4_clks R7S72100_CLK_SCIF2>;
-		clock-names = "fck";
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	scif3: serial@e8008800 {
-		compatible = "renesas,scif-r7s72100", "renesas,scif";
-		reg = <0xe8008800 64>;
-		interrupts = <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp4_clks R7S72100_CLK_SCIF3>;
-		clock-names = "fck";
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	scif4: serial@e8009000 {
-		compatible = "renesas,scif-r7s72100", "renesas,scif";
-		reg = <0xe8009000 64>;
-		interrupts = <GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp4_clks R7S72100_CLK_SCIF4>;
-		clock-names = "fck";
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	scif5: serial@e8009800 {
-		compatible = "renesas,scif-r7s72100", "renesas,scif";
-		reg = <0xe8009800 64>;
-		interrupts = <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp4_clks R7S72100_CLK_SCIF5>;
-		clock-names = "fck";
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	scif6: serial@e800a000 {
-		compatible = "renesas,scif-r7s72100", "renesas,scif";
-		reg = <0xe800a000 64>;
-		interrupts = <GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp4_clks R7S72100_CLK_SCIF6>;
-		clock-names = "fck";
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	scif7: serial@e800a800 {
-		compatible = "renesas,scif-r7s72100", "renesas,scif";
-		reg = <0xe800a800 64>;
-		interrupts = <GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp4_clks R7S72100_CLK_SCIF7>;
-		clock-names = "fck";
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	spi0: spi@e800c800 {
-		compatible = "renesas,rspi-r7s72100", "renesas,rspi-rz";
-		reg = <0xe800c800 0x24>;
-		interrupts = <GIC_SPI 238 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 239 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 240 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "error", "rx", "tx";
-		clocks = <&mstp10_clks R7S72100_CLK_SPI0>;
-		power-domains = <&cpg_clocks>;
-		num-cs = <1>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		status = "disabled";
-	};
-
-	spi1: spi@e800d000 {
-		compatible = "renesas,rspi-r7s72100", "renesas,rspi-rz";
-		reg = <0xe800d000 0x24>;
-		interrupts = <GIC_SPI 241 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 242 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 243 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "error", "rx", "tx";
-		clocks = <&mstp10_clks R7S72100_CLK_SPI1>;
-		power-domains = <&cpg_clocks>;
-		num-cs = <1>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		status = "disabled";
-	};
-
-	spi2: spi@e800d800 {
-		compatible = "renesas,rspi-r7s72100", "renesas,rspi-rz";
-		reg = <0xe800d800 0x24>;
-		interrupts = <GIC_SPI 244 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 245 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "error", "rx", "tx";
-		clocks = <&mstp10_clks R7S72100_CLK_SPI2>;
-		power-domains = <&cpg_clocks>;
-		num-cs = <1>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		status = "disabled";
-	};
-
-	spi3: spi@e800e000 {
-		compatible = "renesas,rspi-r7s72100", "renesas,rspi-rz";
-		reg = <0xe800e000 0x24>;
-		interrupts = <GIC_SPI 247 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 248 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 249 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "error", "rx", "tx";
-		clocks = <&mstp10_clks R7S72100_CLK_SPI3>;
-		power-domains = <&cpg_clocks>;
-		num-cs = <1>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		status = "disabled";
-	};
-
-	spi4: spi@e800e800 {
-		compatible = "renesas,rspi-r7s72100", "renesas,rspi-rz";
-		reg = <0xe800e800 0x24>;
-		interrupts = <GIC_SPI 250 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 251 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 252 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "error", "rx", "tx";
-		clocks = <&mstp10_clks R7S72100_CLK_SPI4>;
-		power-domains = <&cpg_clocks>;
-		num-cs = <1>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		status = "disabled";
-	};
-
-	gic: interrupt-controller@e8201000 {
-		compatible = "arm,pl390";
-		#interrupt-cells = <3>;
-		#address-cells = <0>;
-		interrupt-controller;
-		reg = <0xe8201000 0x1000>,
-			<0xe8202000 0x1000>;
-	};
-
-	L2: cache-controller@3ffff000 {
-		compatible = "arm,pl310-cache";
-		reg = <0x3ffff000 0x1000>;
-		interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
-		arm,early-bresp-disable;
-		arm,full-line-zero-disable;
-		cache-unified;
-		cache-level = <2>;
-	};
-
-	wdt: watchdog@fcfe0000 {
-		compatible = "renesas,r7s72100-wdt", "renesas,rza-wdt";
-		reg = <0xfcfe0000 0x6>;
-		interrupts = <GIC_SPI 106 IRQ_TYPE_EDGE_RISING>;
-		clocks = <&p0_clk>;
-	};
-
-	i2c0: i2c@fcfee000 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		compatible = "renesas,riic-r7s72100", "renesas,riic-rz";
-		reg = <0xfcfee000 0x44>;
-		interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 158 IRQ_TYPE_EDGE_RISING>,
-			     <GIC_SPI 159 IRQ_TYPE_EDGE_RISING>,
-			     <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp9_clks R7S72100_CLK_I2C0>;
-		clock-frequency = <100000>;
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	i2c1: i2c@fcfee400 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		compatible = "renesas,riic-r7s72100", "renesas,riic-rz";
-		reg = <0xfcfee400 0x44>;
-		interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 166 IRQ_TYPE_EDGE_RISING>,
-			     <GIC_SPI 167 IRQ_TYPE_EDGE_RISING>,
-			     <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 169 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 172 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp9_clks R7S72100_CLK_I2C1>;
-		clock-frequency = <100000>;
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	i2c2: i2c@fcfee800 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		compatible = "renesas,riic-r7s72100", "renesas,riic-rz";
-		reg = <0xfcfee800 0x44>;
-		interrupts = <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 174 IRQ_TYPE_EDGE_RISING>,
-			     <GIC_SPI 175 IRQ_TYPE_EDGE_RISING>,
-			     <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 178 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 179 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp9_clks R7S72100_CLK_I2C2>;
-		clock-frequency = <100000>;
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	i2c3: i2c@fcfeec00 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		compatible = "renesas,riic-r7s72100", "renesas,riic-rz";
-		reg = <0xfcfeec00 0x44>;
-		interrupts = <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 182 IRQ_TYPE_EDGE_RISING>,
-			     <GIC_SPI 183 IRQ_TYPE_EDGE_RISING>,
-			     <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp9_clks R7S72100_CLK_I2C3>;
-		clock-frequency = <100000>;
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	mtu2: timer@fcff0000 {
-		compatible = "renesas,mtu2-r7s72100", "renesas,mtu2";
-		reg = <0xfcff0000 0x400>;
-		interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "tgi0a";
-		clocks = <&mstp3_clks R7S72100_CLK_MTU2>;
-		clock-names = "fck";
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	ether: ethernet@e8203000 {
-		compatible = "renesas,ether-r7s72100";
-		reg = <0xe8203000 0x800>,
-		      <0xe8204800 0x200>;
-		interrupts = <GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp7_clks R7S72100_CLK_ETHER>;
-		power-domains = <&cpg_clocks>;
-		phy-mode = "mii";
-		#address-cells = <1>;
-		#size-cells = <0>;
-		status = "disabled";
-	};
-
-	mmcif: mmc@e804c800 {
-		compatible = "renesas,mmcif-r7s72100", "renesas,sh-mmcif";
-		reg = <0xe804c800 0x80>;
-		interrupts = <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH
-			      GIC_SPI 269 IRQ_TYPE_LEVEL_HIGH
-			      GIC_SPI 267 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&mstp8_clks R7S72100_CLK_MMCIF>;
-		power-domains = <&cpg_clocks>;
-		reg-io-width = <4>;
-		bus-width = <8>;
-		status = "disabled";
-	};
-
-	sdhi0: sd@e804e000 {
-		compatible = "renesas,sdhi-r7s72100";
-		reg = <0xe804e000 0x100>;
-		interrupts = <GIC_SPI 270 IRQ_TYPE_LEVEL_HIGH
-			      GIC_SPI 271 IRQ_TYPE_LEVEL_HIGH
-			      GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>;
-
-		clocks = <&mstp12_clks R7S72100_CLK_SDHI00>,
-			 <&mstp12_clks R7S72100_CLK_SDHI01>;
-		clock-names = "core", "cd";
-		power-domains = <&cpg_clocks>;
-		cap-sd-highspeed;
-		cap-sdio-irq;
-		status = "disabled";
-	};
-
-	sdhi1: sd@e804e800 {
-		compatible = "renesas,sdhi-r7s72100";
-		reg = <0xe804e800 0x100>;
-		interrupts = <GIC_SPI 273 IRQ_TYPE_LEVEL_HIGH
-			      GIC_SPI 274 IRQ_TYPE_LEVEL_HIGH
-			      GIC_SPI 275 IRQ_TYPE_LEVEL_HIGH>;
-
-		clocks = <&mstp12_clks R7S72100_CLK_SDHI10>,
-			 <&mstp12_clks R7S72100_CLK_SDHI11>;
-		clock-names = "core", "cd";
-		power-domains = <&cpg_clocks>;
-		cap-sd-highspeed;
-		cap-sdio-irq;
-		status = "disabled";
-	};
-
-	ostm0: timer@fcfec000 {
-		compatible = "renesas,r7s72100-ostm", "renesas,ostm";
-		reg = <0xfcfec000 0x30>;
-		interrupts = <GIC_SPI 102 IRQ_TYPE_EDGE_RISING>;
-		clocks = <&mstp5_clks R7S72100_CLK_OSTM0>;
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
-	};
-
-	ostm1: timer@fcfec400 {
-		compatible = "renesas,r7s72100-ostm", "renesas,ostm";
-		reg = <0xfcfec400 0x30>;
-		interrupts = <GIC_SPI 103 IRQ_TYPE_EDGE_RISING>;
-		clocks = <&mstp5_clks R7S72100_CLK_OSTM1>;
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
 	};
 
-	rtc: rtc@fcff1000 {
-		compatible = "renesas,r7s72100-rtc", "renesas,sh-rtc";
-		reg = <0xfcff1000 0x2e>;
-		interrupts = <GIC_SPI 276 IRQ_TYPE_EDGE_RISING
-			      GIC_SPI 277 IRQ_TYPE_EDGE_RISING
-			      GIC_SPI 278 IRQ_TYPE_EDGE_RISING>;
-		interrupt-names = "alarm", "period", "carry";
-		clocks = <&mstp6_clks R7S72100_CLK_RTC>, <&rtc_x1_clk>,
-			 <&rtc_x3_clk>, <&extal_clk>;
-		clock-names = "fck", "rtc_x1", "rtc_x3", "extal";
-		power-domains = <&cpg_clocks>;
-		status = "disabled";
+	usb_x1_clk: usb_x1 {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		/* If clk present, value must be set by board */
+		clock-frequency = <0>;
 	};
 };
diff --git a/arch/arm/boot/dts/r8a73a4-ape6evm.dts b/arch/arm/boot/dts/r8a73a4-ape6evm.dts
index ec7c86e06538..125c39c0222f 100644
--- a/arch/arm/boot/dts/r8a73a4-ape6evm.dts
+++ b/arch/arm/boot/dts/r8a73a4-ape6evm.dts
@@ -234,7 +234,7 @@
 &sdhi0 {
 	vmmc-supply = <&vcc_sdhi0>;
 	bus-width = <4>;
-	toshiba,mmc-wrprotect-disable;
+	disable-wp;
 	pinctrl-names = "default";
 	pinctrl-0 = <&sdhi0_pins>;
 	status = "okay";
@@ -244,7 +244,7 @@
 	vmmc-supply = <&ape6evm_fixed_3v3>;
 	bus-width = <4>;
 	broken-cd;
-	toshiba,mmc-wrprotect-disable;
+	disable-wp;
 	pinctrl-names = "default";
 	pinctrl-0 = <&sdhi1_pins>;
 	status = "okay";
diff --git a/arch/arm/boot/dts/r8a73a4.dtsi b/arch/arm/boot/dts/r8a73a4.dtsi
index 8e48090e4fdc..080d037f5733 100644
--- a/arch/arm/boot/dts/r8a73a4.dtsi
+++ b/arch/arm/boot/dts/r8a73a4.dtsi
@@ -57,10 +57,10 @@
 
 	timer {
 		compatible = "arm,armv7-timer";
-		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
+		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>;
 	};
 
 	dbsc1: memory-controller@e6790000 {
@@ -464,7 +464,7 @@
 			<0 0xf1002000 0 0x2000>,
 			<0 0xf1004000 0 0x2000>,
 			<0 0xf1006000 0 0x2000>;
-		interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+		interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_HIGH)>;
 		clocks = <&mstp4_clks R8A73A4_CLK_INTC_SYS>;
 		clock-names = "clk";
 		power-domains = <&pd_c4>;
diff --git a/arch/arm/boot/dts/r8a7740.dtsi b/arch/arm/boot/dts/r8a7740.dtsi
index afd3bc5e6cf2..eb9a911deefb 100644
--- a/arch/arm/boot/dts/r8a7740.dtsi
+++ b/arch/arm/boot/dts/r8a7740.dtsi
@@ -67,6 +67,24 @@
 		power-domains = <&pd_d4>;
 	};
 
+	ceu0: ceu@fe910000 {
+		reg = <0xfe910000 0x3000>;
+		compatible = "renesas,r8a7740-ceu";
+		interrupts = <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&mstp1_clks R8A7740_CLK_CEU20>;
+		power-domains = <&pd_a4r>;
+		status = "disabled";
+	};
+
+	ceu1: ceu@fe914000 {
+		reg = <0xfe914000 0x3000>;
+		compatible = "renesas,r8a7740-ceu";
+		interrupts = <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&mstp1_clks R8A7740_CLK_CEU21>;
+		power-domains = <&pd_a4r>;
+		status = "disabled";
+	};
+
 	cmt1: timer@e6138000 {
 		compatible = "renesas,cmt-48-r8a7740", "renesas,cmt-48";
 		reg = <0xe6138000 0x170>;
diff --git a/arch/arm/boot/dts/r8a7743-iwg20m.dtsi b/arch/arm/boot/dts/r8a7743-iwg20m.dtsi
index 1d3e9503c5bd..d364685d9184 100644
--- a/arch/arm/boot/dts/r8a7743-iwg20m.dtsi
+++ b/arch/arm/boot/dts/r8a7743-iwg20m.dtsi
@@ -91,6 +91,11 @@
 	};
 };
 
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
+
 &sdhi0 {
 	pinctrl-0 = <&sdhi0_pins>;
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/r8a7743.dtsi b/arch/arm/boot/dts/r8a7743.dtsi
index 1d9073ba0ce0..142949d7066f 100644
--- a/arch/arm/boot/dts/r8a7743.dtsi
+++ b/arch/arm/boot/dts/r8a7743.dtsi
@@ -125,6 +125,13 @@
 		clock-frequency = <0>;
 	};
 
+	pmu {
+		compatible = "arm,cortex-a15-pmu";
+		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>;
+	};
+
 	/* External SCIF clock */
 	scif_clk: scif {
 		compatible = "fixed-clock";
@@ -297,6 +304,16 @@
 			reg = <0 0xe6160000 0 0x100>;
 		};
 
+		rwdt: watchdog@e6020000 {
+			compatible = "renesas,r8a7743-wdt",
+				     "renesas,rcar-gen2-wdt";
+			reg = <0 0xe6020000 0 0x0c>;
+			clocks = <&cpg CPG_MOD 402>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 402>;
+			status = "disabled";
+		};
+
 		sysc: system-controller@e6180000 {
 			compatible = "renesas,r8a7743-sysc";
 			reg = <0 0xe6180000 0 0x200>;
@@ -407,7 +424,7 @@
 
 			smp-sram@0 {
 				compatible = "renesas,smp-sram";
-				reg = <0 0x10>;
+				reg = <0 0x100>;
 			};
 		};
 
diff --git a/arch/arm/boot/dts/r8a7745-iwg22m.dtsi b/arch/arm/boot/dts/r8a7745-iwg22m.dtsi
index 8d0a392b6811..29b6e10fdf96 100644
--- a/arch/arm/boot/dts/r8a7745-iwg22m.dtsi
+++ b/arch/arm/boot/dts/r8a7745-iwg22m.dtsi
@@ -91,6 +91,11 @@
 	};
 };
 
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
+
 &sdhi1 {
 	pinctrl-0 = <&sdhi1_pins>;
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/r8a7745.dtsi b/arch/arm/boot/dts/r8a7745.dtsi
index dd49a8b48f3e..1cb7a7ab0418 100644
--- a/arch/arm/boot/dts/r8a7745.dtsi
+++ b/arch/arm/boot/dts/r8a7745.dtsi
@@ -105,6 +105,13 @@
 		clock-frequency = <0>;
 	};
 
+	pmu {
+		compatible = "arm,cortex-a7-pmu";
+		interrupts-extended = <&gic GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>;
+	};
+
 	/* External SCIF clock */
 	scif_clk: scif {
 		compatible = "fixed-clock";
@@ -262,6 +269,16 @@
 			reg = <0 0xe6160000 0 0x100>;
 		};
 
+		rwdt: watchdog@e6020000 {
+			compatible = "renesas,r8a7745-wdt",
+				     "renesas,rcar-gen2-wdt";
+			reg = <0 0xe6020000 0 0x0c>;
+			clocks = <&cpg CPG_MOD 402>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 402>;
+			status = "disabled";
+		};
+
 		sysc: system-controller@e6180000 {
 			compatible = "renesas,r8a7745-sysc";
 			reg = <0 0xe6180000 0 0x200>;
@@ -360,7 +377,7 @@
 
 			smp-sram@0 {
 				compatible = "renesas,smp-sram";
-				reg = <0 0x10>;
+				reg = <0 0x100>;
 			};
 		};
 
diff --git a/arch/arm/boot/dts/r8a77470-iwg23s-sbc.dts b/arch/arm/boot/dts/r8a77470-iwg23s-sbc.dts
new file mode 100644
index 000000000000..e3585daafdd6
--- /dev/null
+++ b/arch/arm/boot/dts/r8a77470-iwg23s-sbc.dts
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for the iWave-RZ/G1C single board computer
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ */
+
+/dts-v1/;
+#include "r8a77470.dtsi"
+/ {
+	model = "iWave iW-RainboW-G23S single board computer based on RZ/G1C";
+	compatible = "iwave,g23s", "renesas,r8a77470";
+
+	aliases {
+		ethernet0 = &avb;
+		serial1 = &scif1;
+	};
+
+	chosen {
+		bootargs = "ignore_loglevel rw root=/dev/nfs ip=dhcp";
+		stdout-path = "serial1:115200n8";
+	};
+
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0 0x40000000 0 0x20000000>;
+	};
+};
+
+&avb {
+	phy-handle = <&phy3>;
+	phy-mode = "gmii";
+	renesas,no-ether-link;
+	status = "okay";
+
+	phy3: ethernet-phy@3 {
+		reg = <3>;
+		micrel,led-mode = <1>;
+	};
+};
+
+&extal_clk {
+	clock-frequency = <20000000>;
+};
+
+&scif1 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/r8a77470.dtsi b/arch/arm/boot/dts/r8a77470.dtsi
new file mode 100644
index 000000000000..c85032f9605b
--- /dev/null
+++ b/arch/arm/boot/dts/r8a77470.dtsi
@@ -0,0 +1,336 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for the r8a77470 SoC
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/renesas-cpg-mssr.h>
+/ {
+	compatible = "renesas,r8a77470";
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a7";
+			reg = <0>;
+			clock-frequency = <1000000000>;
+			clocks = <&cpg CPG_CORE 0>;
+			power-domains = <&sysc 5>;
+			next-level-cache = <&L2_CA7>;
+		};
+
+
+		L2_CA7: cache-controller-0 {
+			compatible = "cache";
+			cache-unified;
+			cache-level = <2>;
+			power-domains = <&sysc 21>;
+		};
+	};
+
+	/* External root clock */
+	extal_clk: extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+
+	/* External SCIF clock */
+	scif_clk: scif {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+
+	soc {
+		compatible = "simple-bus";
+		interrupt-parent = <&gic>;
+
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		cpg: clock-controller@e6150000 {
+			compatible = "renesas,r8a77470-cpg-mssr";
+			reg = <0 0xe6150000 0 0x1000>;
+			clocks = <&extal_clk>, <&usb_extal_clk>;
+			clock-names = "extal", "usb_extal";
+			#clock-cells = <2>;
+			#power-domain-cells = <0>;
+			#reset-cells = <1>;
+		};
+
+		rst: reset-controller@e6160000 {
+			compatible = "renesas,r8a77470-rst";
+			reg = <0 0xe6160000 0 0x100>;
+		};
+
+		sysc: system-controller@e6180000 {
+			compatible = "renesas,r8a77470-sysc";
+			reg = <0 0xe6180000 0 0x200>;
+			#power-domain-cells = <1>;
+		};
+
+		irqc: interrupt-controller@e61c0000 {
+			compatible = "renesas,irqc-r8a77470", "renesas,irqc";
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			reg = <0 0xe61c0000 0 0x200>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 407>;
+			power-domains = <&sysc 32>;
+			resets = <&cpg 407>;
+		};
+
+		icram0:	sram@e63a0000 {
+			compatible = "mmio-sram";
+			reg = <0 0xe63a0000 0 0x12000>;
+		};
+
+		icram1:	sram@e63c0000 {
+			compatible = "mmio-sram";
+			reg = <0 0xe63c0000 0 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0 0xe63c0000 0x1000>;
+
+			smp-sram@0 {
+				compatible = "renesas,smp-sram";
+				reg = <0 0x100>;
+			};
+		};
+
+		icram2:	sram@e6300000 {
+			compatible = "mmio-sram";
+			reg = <0 0xe6300000 0 0x20000>;
+		};
+
+		dmac0: dma-controller@e6700000 {
+			compatible = "renesas,dmac-r8a77470",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe6700000 0 0x20000>;
+			interrupts = <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14";
+			clocks = <&cpg CPG_MOD 219>;
+			clock-names = "fck";
+			power-domains = <&sysc 32>;
+			resets = <&cpg 219>;
+			#dma-cells = <1>;
+			dma-channels = <15>;
+		};
+
+		dmac1: dma-controller@e6720000 {
+			compatible = "renesas,dmac-r8a77470",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe6720000 0 0x20000>;
+			interrupts = <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 310 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 313 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14";
+			clocks = <&cpg CPG_MOD 218>;
+			clock-names = "fck";
+			power-domains = <&sysc 32>;
+			resets = <&cpg 218>;
+			#dma-cells = <1>;
+			dma-channels = <15>;
+		};
+
+		avb: ethernet@e6800000 {
+			compatible = "renesas,etheravb-r8a77470",
+				     "renesas,etheravb-rcar-gen2";
+			reg = <0 0xe6800000 0 0x800>, <0 0xee0e8000 0 0x4000>;
+			interrupts = <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 812>;
+			power-domains = <&sysc 32>;
+			resets = <&cpg 812>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		scif0: serial@e6e60000 {
+			compatible = "renesas,scif-r8a77470",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6e60000 0 0x40>;
+			interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 721>,
+				 <&cpg CPG_CORE 5>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x29>, <&dmac0 0x2a>,
+			       <&dmac1 0x29>, <&dmac1 0x2a>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc 32>;
+			resets = <&cpg 721>;
+			status = "disabled";
+		};
+
+		scif1: serial@e6e68000 {
+			compatible = "renesas,scif-r8a77470",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6e68000 0 0x40>;
+			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 720>,
+				 <&cpg CPG_CORE 5>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x2d>, <&dmac0 0x2e>,
+			       <&dmac1 0x2d>, <&dmac1 0x2e>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc 32>;
+			resets = <&cpg 720>;
+			status = "disabled";
+		};
+
+		scif2: serial@e6e58000 {
+			compatible = "renesas,scif-r8a77470",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6e58000 0 0x40>;
+			interrupts = <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 719>,
+				 <&cpg CPG_CORE 5>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x2b>, <&dmac0 0x2c>,
+			       <&dmac1 0x2b>, <&dmac1 0x2c>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc 32>;
+			resets = <&cpg 719>;
+			status = "disabled";
+		};
+
+		scif3: serial@e6ea8000 {
+			compatible = "renesas,scif-r8a77470",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6ea8000 0 0x40>;
+			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 718>,
+				 <&cpg CPG_CORE 5>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x2f>, <&dmac0 0x30>,
+			       <&dmac1 0x2f>, <&dmac1 0x30>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc 32>;
+			resets = <&cpg 718>;
+			status = "disabled";
+		};
+
+		scif4: serial@e6ee0000 {
+			compatible = "renesas,scif-r8a77470",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6ee0000 0 0x40>;
+			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 715>,
+				 <&cpg CPG_CORE 5>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0xfb>, <&dmac0 0xfc>,
+			       <&dmac1 0xfb>, <&dmac1 0xfc>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc 32>;
+			resets = <&cpg 715>;
+			status = "disabled";
+		};
+
+		scif5: serial@e6ee8000 {
+			compatible = "renesas,scif-r8a77470",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6ee8000 0 0x40>;
+			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 714>,
+				 <&cpg CPG_CORE 5>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0xfd>, <&dmac0 0xfe>,
+			       <&dmac1 0xfd>, <&dmac1 0xfe>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc 32>;
+			resets = <&cpg 714>;
+			status = "disabled";
+		};
+
+		gic: interrupt-controller@f1001000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0 0xf1001000 0 0x1000>, <0 0xf1002000 0 0x2000>,
+			      <0 0xf1004000 0 0x2000>, <0 0xf1006000 0 0x2000>;
+			interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>;
+			clocks = <&cpg CPG_MOD 408>;
+			clock-names = "clk";
+			power-domains = <&sysc 32>;
+			resets = <&cpg 408>;
+		};
+
+		prr: chipid@ff000044 {
+			compatible = "renesas,prr";
+			reg = <0 0xff000044 0 4>;
+		};
+	};
+
+	timer {
+		compatible = "arm,armv7-timer";
+		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
+	};
+
+	/* External USB clock - can be overridden by the board */
+	usb_extal_clk: usb_extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <48000000>;
+	};
+};
diff --git a/arch/arm/boot/dts/r8a7790-lager.dts b/arch/arm/boot/dts/r8a7790-lager.dts
index f07f9018c3e7..092610e3f953 100644
--- a/arch/arm/boot/dts/r8a7790-lager.dts
+++ b/arch/arm/boot/dts/r8a7790-lager.dts
@@ -902,9 +902,6 @@
 	status = "okay";
 
 	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		vin1ep0: endpoint {
 			remote-endpoint = <&adv7180>;
 			bus-width = <8>;
@@ -929,6 +926,11 @@
 	};
 };
 
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
+
 &ssi1 {
 	shared-pin;
 };
diff --git a/arch/arm/boot/dts/r8a7790.dtsi b/arch/arm/boot/dts/r8a7790.dtsi
index 05a0fc23ac88..4d06b154bd7e 100644
--- a/arch/arm/boot/dts/r8a7790.dtsi
+++ b/arch/arm/boot/dts/r8a7790.dtsi
@@ -202,6 +202,24 @@
 		clock-frequency = <0>;
 	};
 
+	pmu-0 {
+		compatible = "arm,cortex-a15-pmu";
+		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
+	};
+
+	pmu-1 {
+		compatible = "arm,cortex-a7-pmu";
+		interrupts-extended = <&gic GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu4>, <&cpu5>, <&cpu6>, <&cpu7>;
+	};
+
 	/* External SCIF clock */
 	scif_clk: scif {
 		compatible = "fixed-clock";
@@ -218,6 +236,16 @@
 		#size-cells = <2>;
 		ranges;
 
+		rwdt: watchdog@e6020000 {
+			compatible = "renesas,r8a7790-wdt",
+				     "renesas,rcar-gen2-wdt";
+			reg = <0 0xe6020000 0 0x0c>;
+			clocks = <&cpg CPG_MOD 402>;
+			power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
+			resets = <&cpg 402>;
+			status = "disabled";
+		};
+
 		gpio0: gpio@e6050000 {
 			compatible = "renesas,gpio-r8a7790",
 				     "renesas,rcar-gen2-gpio";
@@ -443,7 +471,7 @@
 
 			smp-sram@0 {
 				compatible = "renesas,smp-sram";
-				reg = <0 0x10>;
+				reg = <0 0x100>;
 			};
 		};
 
@@ -1544,7 +1572,7 @@
 			interrupt-controller;
 			reg = <0 0xf1001000 0 0x1000>, <0 0xf1002000 0 0x2000>,
 			      <0 0xf1004000 0 0x2000>, <0 0xf1006000 0 0x2000>;
-			interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+			interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&cpg CPG_MOD 408>;
 			clock-names = "clk";
 			power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
@@ -1615,6 +1643,33 @@
 			resets = <&cpg 127>;
 		};
 
+		fdp1@fe940000 {
+			compatible = "renesas,fdp1";
+			reg = <0 0xfe940000 0 0x2400>;
+			interrupts = <GIC_SPI 262 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 119>;
+			power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
+			resets = <&cpg 119>;
+		};
+
+		fdp1@fe944000 {
+			compatible = "renesas,fdp1";
+			reg = <0 0xfe944000 0 0x2400>;
+			interrupts = <GIC_SPI 263 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 118>;
+			power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
+			resets = <&cpg 118>;
+		};
+
+		fdp1@fe948000 {
+			compatible = "renesas,fdp1";
+			reg = <0 0xfe948000 0 0x2400>;
+			interrupts = <GIC_SPI 264 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 117>;
+			power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
+			resets = <&cpg 117>;
+		};
+
 		jpu: jpeg-codec@fe980000 {
 			compatible = "renesas,jpu-r8a7790",
 				     "renesas,rcar-gen2-jpu";
@@ -1773,10 +1828,10 @@
 
 	timer {
 		compatible = "arm,armv7-timer";
-		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
+		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>;
 	};
 
 	/* External USB clock - can be overridden by the board */
diff --git a/arch/arm/boot/dts/r8a7791-koelsch.dts b/arch/arm/boot/dts/r8a7791-koelsch.dts
index 9d7213a0b8b8..8ab793d8b2fd 100644
--- a/arch/arm/boot/dts/r8a7791-koelsch.dts
+++ b/arch/arm/boot/dts/r8a7791-koelsch.dts
@@ -643,6 +643,11 @@
 	status = "okay";
 };
 
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
+
 &sata0 {
 	status = "okay";
 };
@@ -850,9 +855,6 @@
 	pinctrl-names = "default";
 
 	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		vin0ep2: endpoint {
 			remote-endpoint = <&adv7612_out>;
 			bus-width = <24>;
@@ -871,9 +873,6 @@
 	pinctrl-names = "default";
 
 	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		vin1ep: endpoint {
 			remote-endpoint = <&adv7180>;
 			bus-width = <8>;
diff --git a/arch/arm/boot/dts/r8a7791-porter.dts b/arch/arm/boot/dts/r8a7791-porter.dts
index ae9ed9ff53ef..a01101b49d99 100644
--- a/arch/arm/boot/dts/r8a7791-porter.dts
+++ b/arch/arm/boot/dts/r8a7791-porter.dts
@@ -386,9 +386,6 @@
 	pinctrl-names = "default";
 
 	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		vin0ep: endpoint {
 			remote-endpoint = <&adv7180>;
 			bus-width = <8>;
@@ -481,6 +478,11 @@
 	};
 };
 
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
+
 &ssi1 {
 	shared-pin;
 };
diff --git a/arch/arm/boot/dts/r8a7791.dtsi b/arch/arm/boot/dts/r8a7791.dtsi
index 506b20885413..6e1dd7ad7bd6 100644
--- a/arch/arm/boot/dts/r8a7791.dtsi
+++ b/arch/arm/boot/dts/r8a7791.dtsi
@@ -126,6 +126,13 @@
 		clock-frequency = <0>;
 	};
 
+	pmu {
+		compatible = "arm,cortex-a15-pmu";
+		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>;
+	};
+
 	/* External SCIF clock */
 	scif_clk: scif {
 		compatible = "fixed-clock";
@@ -142,6 +149,16 @@
 		#size-cells = <2>;
 		ranges;
 
+		rwdt: watchdog@e6020000 {
+			compatible = "renesas,r8a7791-wdt",
+				     "renesas,rcar-gen2-wdt";
+			reg = <0 0xe6020000 0 0x0c>;
+			clocks = <&cpg CPG_MOD 402>;
+			power-domains = <&sysc R8A7791_PD_ALWAYS_ON>;
+			resets = <&cpg 402>;
+			status = "disabled";
+		};
+
 		gpio0: gpio@e6050000 {
 			compatible = "renesas,gpio-r8a7791",
 				     "renesas,rcar-gen2-gpio";
@@ -407,7 +424,7 @@
 
 			smp-sram@0 {
 				compatible = "renesas,smp-sram";
-				reg = <0 0x10>;
+				reg = <0 0x100>;
 			};
 		};
 
@@ -1621,6 +1638,24 @@
 			resets = <&cpg 127>;
 		};
 
+		fdp1@fe940000 {
+			compatible = "renesas,fdp1";
+			reg = <0 0xfe940000 0 0x2400>;
+			interrupts = <GIC_SPI 262 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 119>;
+			power-domains = <&sysc R8A7791_PD_ALWAYS_ON>;
+			resets = <&cpg 119>;
+		};
+
+		fdp1@fe944000 {
+			compatible = "renesas,fdp1";
+			reg = <0 0xfe944000 0 0x2400>;
+			interrupts = <GIC_SPI 263 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 118>;
+			power-domains = <&sysc R8A7791_PD_ALWAYS_ON>;
+			resets = <&cpg 118>;
+		};
+
 		jpu: jpeg-codec@fe980000 {
 			compatible = "renesas,jpu-r8a7791",
 				     "renesas,rcar-gen2-jpu";
diff --git a/arch/arm/boot/dts/r8a7792-blanche.dts b/arch/arm/boot/dts/r8a7792-blanche.dts
index 9b67dca6c9ef..04fb70931b3b 100644
--- a/arch/arm/boot/dts/r8a7792-blanche.dts
+++ b/arch/arm/boot/dts/r8a7792-blanche.dts
@@ -239,6 +239,11 @@
 	};
 };
 
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
+
 &scif0 {
 	pinctrl-0 = <&scif0_pins>;
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/r8a7792-wheat.dts b/arch/arm/boot/dts/r8a7792-wheat.dts
index b9471b67b728..db01de7a3811 100644
--- a/arch/arm/boot/dts/r8a7792-wheat.dts
+++ b/arch/arm/boot/dts/r8a7792-wheat.dts
@@ -168,6 +168,11 @@
 	};
 };
 
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
+
 &scif0 {
 	pinctrl-0 = <&scif0_pins>;
 	pinctrl-names = "default";
@@ -240,9 +245,15 @@
 	status = "okay";
 	clock-frequency = <400000>;
 
+	/*
+	 * The adv75xx resets its addresses to defaults during low power mode.
+	 * Because we have two ADV7513 devices on the same bus, we must change
+	 * both of them away from the defaults so that they do not conflict.
+	 */
 	hdmi@3d {
 		compatible = "adi,adv7513";
-		reg = <0x3d>;
+		reg = <0x3d>, <0x2d>, <0x4d>, <0x5d>;
+		reg-names = "main", "cec", "edid", "packet";
 
 		adi,input-depth = <8>;
 		adi,input-colorspace = "rgb";
@@ -272,7 +283,8 @@
 
 	hdmi@39 {
 		compatible = "adi,adv7513";
-		reg = <0x39>;
+		reg = <0x39>, <0x29>, <0x49>, <0x59>;
+		reg-names = "main", "cec", "edid", "packet";
 
 		adi,input-depth = <8>;
 		adi,input-colorspace = "rgb";
diff --git a/arch/arm/boot/dts/r8a7792.dtsi b/arch/arm/boot/dts/r8a7792.dtsi
index 268987ff0201..f44257dd86f6 100644
--- a/arch/arm/boot/dts/r8a7792.dtsi
+++ b/arch/arm/boot/dts/r8a7792.dtsi
@@ -85,6 +85,13 @@
 		clock-frequency = <0>;
 	};
 
+	pmu {
+		compatible = "arm,cortex-a15-pmu";
+		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>;
+	};
+
 	/* External SCIF clock */
 	scif_clk: scif {
 		compatible = "fixed-clock";
@@ -101,6 +108,16 @@
 		#size-cells = <2>;
 		ranges;
 
+		rwdt: watchdog@e6020000 {
+			compatible = "renesas,r8a7792-wdt",
+				     "renesas,rcar-gen2-wdt";
+			reg = <0 0xe6020000 0 0x0c>;
+			clocks = <&cpg CPG_MOD 402>;
+			power-domains = <&sysc R8A7792_PD_ALWAYS_ON>;
+			resets = <&cpg 402>;
+			status = "disabled";
+		};
+
 		gpio0: gpio@e6050000 {
 			compatible = "renesas,gpio-r8a7792",
 				     "renesas,rcar-gen2-gpio";
@@ -341,7 +358,7 @@
 
 			smp-sram@0 {
 				compatible = "renesas,smp-sram";
-				reg = <0 0x10>;
+				reg = <0 0x100>;
 			};
 		};
 
diff --git a/arch/arm/boot/dts/r8a7793-gose.dts b/arch/arm/boot/dts/r8a7793-gose.dts
index 96e117d8b2cc..aa209f6e5d71 100644
--- a/arch/arm/boot/dts/r8a7793-gose.dts
+++ b/arch/arm/boot/dts/r8a7793-gose.dts
@@ -599,6 +599,11 @@
 	status = "okay";
 };
 
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
+
 &scif0 {
 	pinctrl-0 = <&scif0_pins>;
 	pinctrl-names = "default";
@@ -758,9 +763,6 @@
 	pinctrl-names = "default";
 
 	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		vin0ep2: endpoint {
 			remote-endpoint = <&adv7612_out>;
 			bus-width = <24>;
@@ -780,9 +782,6 @@
 	status = "okay";
 
 	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		vin1ep: endpoint {
 			remote-endpoint = <&adv7180_out>;
 			bus-width = <8>;
diff --git a/arch/arm/boot/dts/r8a7793.dtsi b/arch/arm/boot/dts/r8a7793.dtsi
index 4f526030dc7c..4abecfc0ca98 100644
--- a/arch/arm/boot/dts/r8a7793.dtsi
+++ b/arch/arm/boot/dts/r8a7793.dtsi
@@ -110,6 +110,13 @@
 		clock-frequency = <0>;
 	};
 
+	pmu {
+		compatible = "arm,cortex-a15-pmu";
+		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>;
+	};
+
 	/* External SCIF clock */
 	scif_clk: scif {
 		compatible = "fixed-clock";
@@ -126,6 +133,16 @@
 		#size-cells = <2>;
 		ranges;
 
+		rwdt: watchdog@e6020000 {
+			compatible = "renesas,r8a7793-wdt",
+				     "renesas,rcar-gen2-wdt";
+			reg = <0 0xe6020000 0 0x0c>;
+			clocks = <&cpg CPG_MOD 402>;
+			power-domains = <&sysc R8A7793_PD_ALWAYS_ON>;
+			resets = <&cpg 402>;
+			status = "disabled";
+		};
+
 		gpio0: gpio@e6050000 {
 			compatible = "renesas,gpio-r8a7793",
 				     "renesas,rcar-gen2-gpio";
@@ -392,7 +409,7 @@
 
 			smp-sram@0 {
 				compatible = "renesas,smp-sram";
-				reg = <0 0x10>;
+				reg = <0 0x100>;
 			};
 		};
 
@@ -1290,6 +1307,24 @@
 			resets = <&cpg 408>;
 		};
 
+		fdp1@fe940000 {
+			compatible = "renesas,fdp1";
+			reg = <0 0xfe940000 0 0x2400>;
+			interrupts = <GIC_SPI 262 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 119>;
+			power-domains = <&sysc R8A7793_PD_ALWAYS_ON>;
+			resets = <&cpg 119>;
+		};
+
+		fdp1@fe944000 {
+			compatible = "renesas,fdp1";
+			reg = <0 0xfe944000 0 0x2400>;
+			interrupts = <GIC_SPI 263 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 118>;
+			power-domains = <&sysc R8A7793_PD_ALWAYS_ON>;
+			resets = <&cpg 118>;
+		};
+
 		du: display@feb00000 {
 			compatible = "renesas,du-r8a7793";
 			reg = <0 0xfeb00000 0 0x40000>;
diff --git a/arch/arm/boot/dts/r8a7794-alt.dts b/arch/arm/boot/dts/r8a7794-alt.dts
index 26a883484ea8..e17027532941 100644
--- a/arch/arm/boot/dts/r8a7794-alt.dts
+++ b/arch/arm/boot/dts/r8a7794-alt.dts
@@ -181,6 +181,12 @@
 				};
 			};
 		};
+
+		eeprom@50 {
+			compatible = "renesas,r1ex24002", "atmel,24c02";
+			reg = <0x50>;
+			pagesize = <16>;
+		};
 	};
 
 	/*
@@ -330,6 +336,11 @@
 	status = "okay";
 };
 
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
+
 &sdhi0 {
 	pinctrl-0 = <&sdhi0_pins>;
 	pinctrl-1 = <&sdhi0_pins_uhs>;
@@ -375,9 +386,6 @@
 	pinctrl-names = "default";
 
 	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		vin0ep: endpoint {
 			remote-endpoint = <&adv7180>;
 			bus-width = <8>;
diff --git a/arch/arm/boot/dts/r8a7794-silk.dts b/arch/arm/boot/dts/r8a7794-silk.dts
index 351cb3b3d966..7808aaee6644 100644
--- a/arch/arm/boot/dts/r8a7794-silk.dts
+++ b/arch/arm/boot/dts/r8a7794-silk.dts
@@ -475,9 +475,6 @@
 	pinctrl-names = "default";
 
 	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		vin0ep: endpoint {
 			remote-endpoint = <&adv7180>;
 			bus-width = <8>;
@@ -540,6 +537,11 @@
 	};
 };
 
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
+
 &ssi1 {
 	shared-pin;
 };
diff --git a/arch/arm/boot/dts/r8a7794.dtsi b/arch/arm/boot/dts/r8a7794.dtsi
index d588efa6aeaa..736196903d22 100644
--- a/arch/arm/boot/dts/r8a7794.dtsi
+++ b/arch/arm/boot/dts/r8a7794.dtsi
@@ -103,6 +103,13 @@
 		clock-frequency = <0>;
 	};
 
+	pmu {
+		compatible = "arm,cortex-a7-pmu";
+		interrupts-extended = <&gic GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>;
+	};
+
 	/* External SCIF clock */
 	scif_clk: scif {
 		compatible = "fixed-clock";
@@ -119,6 +126,16 @@
 		#size-cells = <2>;
 		ranges;
 
+		rwdt: watchdog@e6020000 {
+			compatible = "renesas,r8a7794-wdt",
+				     "renesas,rcar-gen2-wdt";
+			reg = <0 0xe6020000 0 0x0c>;
+			clocks = <&cpg CPG_MOD 402>;
+			power-domains = <&sysc R8A7794_PD_ALWAYS_ON>;
+			resets = <&cpg 402>;
+			status = "disabled";
+		};
+
 		gpio0: gpio@e6050000 {
 			compatible = "renesas,gpio-r8a7794",
 				     "renesas,rcar-gen2-gpio";
@@ -348,7 +365,7 @@
 
 			smp-sram@0 {
 				compatible = "renesas,smp-sram";
-				reg = <0 0x10>;
+				reg = <0 0x100>;
 			};
 		};
 
@@ -1323,6 +1340,15 @@
 			resets = <&cpg 128>;
 		};
 
+		fdp1@fe940000 {
+			compatible = "renesas,fdp1";
+			reg = <0 0xfe940000 0 0x2400>;
+			interrupts = <GIC_SPI 262 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 119>;
+			power-domains = <&sysc R8A7794_PD_ALWAYS_ON>;
+			resets = <&cpg 119>;
+		};
+
 		du: display@feb00000 {
 			compatible = "renesas,du-r8a7794";
 			reg = <0 0xfeb00000 0 0x40000>;
diff --git a/arch/arm/boot/dts/rk3036.dtsi b/arch/arm/boot/dts/rk3036.dtsi
index a97458112ff6..567a6a725f9c 100644
--- a/arch/arm/boot/dts/rk3036.dtsi
+++ b/arch/arm/boot/dts/rk3036.dtsi
@@ -197,6 +197,8 @@
 		reg = <0x10118300 0x100>;
 		interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vop_mmu";
+		clocks = <&cru ACLK_LCDC>, <&cru HCLK_LCDC>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
diff --git a/arch/arm/boot/dts/rk322x.dtsi b/arch/arm/boot/dts/rk322x.dtsi
index df1e47858675..be80e9a2c9af 100644
--- a/arch/arm/boot/dts/rk322x.dtsi
+++ b/arch/arm/boot/dts/rk322x.dtsi
@@ -584,6 +584,8 @@
 		reg = <0x20020800 0x100>;
 		interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vpu_mmu";
+		clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>;
+		clock-names = "aclk", "iface";
 		iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -593,6 +595,8 @@
 		reg = <0x20030480 0x40>, <0x200304c0 0x40>;
 		interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vdec_mmu";
+		clocks = <&cru ACLK_RKVDEC>, <&cru HCLK_RKVDEC>;
+		clock-names = "aclk", "iface";
 		iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -602,6 +606,8 @@
 		reg = <0x20053f00 0x100>;
 		interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vop_mmu";
+		clocks = <&cru ACLK_VOP>, <&cru HCLK_VOP>;
+		clock-names = "aclk", "iface";
 		iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -611,6 +617,8 @@
 		reg = <0x20070800 0x100>;
 		interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "iep_mmu";
+		clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>;
+		clock-names = "aclk", "iface";
 		iommu-cells = <0>;
 		status = "disabled";
 	};
diff --git a/arch/arm/boot/dts/rk3288-phycore-som.dtsi b/arch/arm/boot/dts/rk3288-phycore-som.dtsi
index f13bcb1cd3d9..aaab2d171ffe 100644
--- a/arch/arm/boot/dts/rk3288-phycore-som.dtsi
+++ b/arch/arm/boot/dts/rk3288-phycore-som.dtsi
@@ -151,6 +151,7 @@
 			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
 			ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
 			enet-phy-lane-no-swap;
+			ti,clk-output-sel = <DP83867_CLK_O_SEL_CHN_A_TCLK>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/rk3288-tinker.dts b/arch/arm/boot/dts/rk3288-tinker.dts
index 346b0d8b474d..127488f9f174 100644
--- a/arch/arm/boot/dts/rk3288-tinker.dts
+++ b/arch/arm/boot/dts/rk3288-tinker.dts
@@ -49,6 +49,10 @@
 	model = "Rockchip RK3288 Tinker Board";
 	compatible = "asus,rk3288-tinker", "rockchip,rk3288";
 
+	chosen {
+		stdout-path = "serial2:115200n8";
+	};
+
 	memory {
 		reg = <0x0 0x0 0x0 0x80000000>;
 		device_type = "memory";
diff --git a/arch/arm/boot/dts/rk3288-veyron-chromebook.dtsi b/arch/arm/boot/dts/rk3288-veyron-chromebook.dtsi
index be487111d025..b16d570ff029 100644
--- a/arch/arm/boot/dts/rk3288-veyron-chromebook.dtsi
+++ b/arch/arm/boot/dts/rk3288-veyron-chromebook.dtsi
@@ -95,7 +95,8 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&bl_en>;
 		pwms = <&pwm0 0 1000000 0>;
-		pwm-delay-us = <10000>;
+		post-pwm-on-delay-ms = <10>;
+		pwm-off-delay-ms = <10>;
 	};
 
 	gpio-charger {
diff --git a/arch/arm/boot/dts/rk3288-veyron-minnie.dts b/arch/arm/boot/dts/rk3288-veyron-minnie.dts
index 544de6027aaa..4c5307e62001 100644
--- a/arch/arm/boot/dts/rk3288-veyron-minnie.dts
+++ b/arch/arm/boot/dts/rk3288-veyron-minnie.dts
@@ -123,6 +123,8 @@
 			240 241 242 243 244 245 246 247
 			248 249 250 251 252 253 254 255>;
 	power-supply = <&backlight_regulator>;
+	post-pwm-on-delay-ms = <200>;
+	pwm-off-delay-ms = <200>;
 };
 
 &emmc {
diff --git a/arch/arm/boot/dts/rk3288.dtsi b/arch/arm/boot/dts/rk3288.dtsi
index 354aff45c1af..d7e49d29ace5 100644
--- a/arch/arm/boot/dts/rk3288.dtsi
+++ b/arch/arm/boot/dts/rk3288.dtsi
@@ -959,6 +959,8 @@
 		reg = <0x0 0xff900800 0x0 0x40>;
 		interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "iep_mmu";
+		clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -968,6 +970,8 @@
 		reg = <0x0 0xff914000 0x0 0x100>, <0x0 0xff915000 0x0 0x100>;
 		interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "isp_mmu";
+		clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		rockchip,disable-mmu-reset;
 		status = "disabled";
@@ -1027,6 +1031,8 @@
 		reg = <0x0 0xff930300 0x0 0x100>;
 		interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vopb_mmu";
+		clocks = <&cru ACLK_VOP0>, <&cru HCLK_VOP0>;
+		clock-names = "aclk", "iface";
 		power-domains = <&power RK3288_PD_VIO>;
 		#iommu-cells = <0>;
 		status = "disabled";
@@ -1075,6 +1081,8 @@
 		reg = <0x0 0xff940300 0x0 0x100>;
 		interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vopl_mmu";
+		clocks = <&cru ACLK_VOP1>, <&cru HCLK_VOP1>;
+		clock-names = "aclk", "iface";
 		power-domains = <&power RK3288_PD_VIO>;
 		#iommu-cells = <0>;
 		status = "disabled";
@@ -1206,6 +1214,8 @@
 		reg = <0x0 0xff9a0800 0x0 0x100>;
 		interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vpu_mmu";
+		clocks = <&cru ACLK_VCODEC>, <&cru HCLK_VCODEC>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -1215,6 +1225,8 @@
 		reg = <0x0 0xff9c0440 0x0 0x40>, <0x0 0xff9c0480 0x0 0x40>;
 		interrupts = <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "hevc_mmu";
+		clocks = <&cru ACLK_HEVC>, <&cru HCLK_HEVC>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -1848,16 +1860,16 @@
 
 		uart4 {
 			uart4_xfer: uart4-xfer {
-				rockchip,pins = <5 12 3 &pcfg_pull_up>,
-						<5 13 3 &pcfg_pull_none>;
+				rockchip,pins = <5 15 3 &pcfg_pull_up>,
+						<5 14 3 &pcfg_pull_none>;
 			};
 
 			uart4_cts: uart4-cts {
-				rockchip,pins = <5 14 3 &pcfg_pull_up>;
+				rockchip,pins = <5 12 3 &pcfg_pull_up>;
 			};
 
 			uart4_rts: uart4-rts {
-				rockchip,pins = <5 15 3 &pcfg_pull_none>;
+				rockchip,pins = <5 13 3 &pcfg_pull_none>;
 			};
 		};
 
diff --git a/arch/arm/boot/dts/s3c2416-smdk2416.dts b/arch/arm/boot/dts/s3c2416-smdk2416.dts
index a1c9d8c695cc..5164386aff3a 100644
--- a/arch/arm/boot/dts/s3c2416-smdk2416.dts
+++ b/arch/arm/boot/dts/s3c2416-smdk2416.dts
@@ -12,14 +12,13 @@
 	model = "SMDK2416";
 	compatible = "samsung,s3c2416";
 
-	memory {
+	memory@30000000 {
+		device_type = "memory";
 		reg =  <0x30000000 0x4000000>;
 	};
 
 	clocks {
 		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <1>;
 
 		xti: xti {
 			compatible = "fixed-clock";
diff --git a/arch/arm/boot/dts/s3c2416.dtsi b/arch/arm/boot/dts/s3c2416.dtsi
index 3c7385cab248..6adf64ea3ff2 100644
--- a/arch/arm/boot/dts/s3c2416.dtsi
+++ b/arch/arm/boot/dts/s3c2416.dtsi
@@ -18,9 +18,6 @@
 	};
 
 	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		cpu {
 			compatible = "arm,arm926ej-s";
 		};
@@ -30,7 +27,7 @@
 		compatible = "samsung,s3c2416-irq";
 	};
 
-	clocks: clock-controller@0x4c000000 {
+	clocks: clock-controller@4c000000 {
 		compatible = "samsung,s3c2416-clock";
 		reg = <0x4c000000 0x40>;
 		#clock-cells = <1>;
@@ -69,7 +66,7 @@
 				<&clocks SCLK_UART>;
 	};
 
-	uart_3: serial@5000C000 {
+	uart_3: serial@5000c000 {
 		compatible = "samsung,s3c2440-uart";
 		reg = <0x5000C000 0x4000>;
 		interrupts = <1 18 24 4>, <1 18 25 4>;
@@ -80,7 +77,7 @@
 		status = "disabled";
 	};
 
-	sdhci_1: sdhci@4AC00000 {
+	sdhci_1: sdhci@4ac00000 {
 		compatible = "samsung,s3c6410-sdhci";
 		reg = <0x4AC00000 0x100>;
 		interrupts = <0 0 21 3>;
@@ -91,7 +88,7 @@
 		status = "disabled";
 	};
 
-	sdhci_0: sdhci@4A800000 {
+	sdhci_0: sdhci@4a800000 {
 		compatible = "samsung,s3c6410-sdhci";
 		reg = <0x4A800000 0x100>;
 		interrupts = <0 0 20 3>;
diff --git a/arch/arm/boot/dts/s3c24xx.dtsi b/arch/arm/boot/dts/s3c24xx.dtsi
index 34c7fe6751cf..6d8dd3cdd3c0 100644
--- a/arch/arm/boot/dts/s3c24xx.dtsi
+++ b/arch/arm/boot/dts/s3c24xx.dtsi
@@ -5,11 +5,11 @@
  * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
  */
 
-#include "skeleton.dtsi"
-
 / {
 	compatible = "samsung,s3c24xx";
 	interrupt-parent = <&intc>;
+	#address-cells = <1>;
+	#size-cells = <1>;
 
 	aliases {
 		pinctrl0 = &pinctrl_0;
diff --git a/arch/arm/boot/dts/s3c6410-mini6410.dts b/arch/arm/boot/dts/s3c6410-mini6410.dts
index f68601bd9c91..0e159c884f97 100644
--- a/arch/arm/boot/dts/s3c6410-mini6410.dts
+++ b/arch/arm/boot/dts/s3c6410-mini6410.dts
@@ -19,7 +19,8 @@
 	model = "FriendlyARM Mini6410 board based on S3C6410";
 	compatible = "friendlyarm,mini6410", "samsung,s3c6410";
 
-	memory {
+	memory@50000000 {
+		device_type = "memory";
 		reg = <0x50000000 0x10000000>;
 	};
 
diff --git a/arch/arm/boot/dts/s3c6410-smdk6410.dts b/arch/arm/boot/dts/s3c6410-smdk6410.dts
index b6b5afcd7602..a9a5689dc462 100644
--- a/arch/arm/boot/dts/s3c6410-smdk6410.dts
+++ b/arch/arm/boot/dts/s3c6410-smdk6410.dts
@@ -19,7 +19,8 @@
 	model = "SAMSUNG SMDK6410 board based on S3C6410";
 	compatible = "samsung,mini6410", "samsung,s3c6410";
 
-	memory {
+	memory@50000000 {
+		device_type = "memory";
 		reg = <0x50000000 0x8000000>;
 	};
 
diff --git a/arch/arm/boot/dts/s3c64xx.dtsi b/arch/arm/boot/dts/s3c64xx.dtsi
index e2be3fbdd3f3..2e611df37911 100644
--- a/arch/arm/boot/dts/s3c64xx.dtsi
+++ b/arch/arm/boot/dts/s3c64xx.dtsi
@@ -13,10 +13,12 @@
  * nodes can be added to this file.
  */
 
-#include "skeleton.dtsi"
 #include <dt-bindings/clock/samsung,s3c64xx-clock.h>
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		i2c0 = &i2c0;
 		pinctrl0 = &pinctrl0;
diff --git a/arch/arm/boot/dts/sh73a0.dtsi b/arch/arm/boot/dts/sh73a0.dtsi
index 914a7c2a584f..c953648a5f41 100644
--- a/arch/arm/boot/dts/sh73a0.dtsi
+++ b/arch/arm/boot/dts/sh73a0.dtsi
@@ -22,7 +22,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		cpu@0 {
+		cpu0: cpu@0 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <0>;
@@ -31,7 +31,7 @@
 			power-domains = <&pd_a2sl>;
 			next-level-cache = <&L2>;
 		};
-		cpu@1 {
+		cpu1: cpu@1 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <1>;
@@ -91,6 +91,7 @@
 		compatible = "arm,cortex-a9-pmu";
 		interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>;
 	};
 
 	cmt1: timer@e6138000 {
@@ -336,7 +337,7 @@
 			      GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp3_clks SH73A0_CLK_SDHI1>;
 		power-domains = <&pd_a3sp>;
-		toshiba,mmc-wrprotect-disable;
+		disable-wp;
 		cap-sd-highspeed;
 		status = "disabled";
 	};
@@ -348,7 +349,7 @@
 			      GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp3_clks SH73A0_CLK_SDHI2>;
 		power-domains = <&pd_a3sp>;
-		toshiba,mmc-wrprotect-disable;
+		disable-wp;
 		cap-sd-highspeed;
 		status = "disabled";
 	};
diff --git a/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts b/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
index 2459d133f1be..f50b19447de6 100644
--- a/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
+++ b/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
@@ -161,7 +161,7 @@
 	};
 
 	at24@50 {
-		compatible = "at24,24c01";
+		compatible = "atmel,24c01";
 		pagesize = <8>;
 		reg = <0x50>;
 	};
@@ -213,7 +213,7 @@
 			#size-cells = <0>;
 			reg = <6>;
 			eeprom@51 {
-				compatible = "at,24c01";
+				compatible = "atmel,24c01";
 				pagesize = <8>;
 				reg = <0x51>;
 			};
@@ -224,7 +224,7 @@
 			#size-cells = <0>;
 			reg = <7>;
 			eeprom@51 {
-				compatible = "at,24c01";
+				compatible = "atmel,24c01";
 				pagesize = <8>;
 				reg = <0x51>;
 			};
diff --git a/arch/arm/boot/dts/ste-ccu8540-pinctrl.dtsi b/arch/arm/boot/dts/ste-ccu8540-pinctrl.dtsi
deleted file mode 100644
index 52dba2e39c71..000000000000
--- a/arch/arm/boot/dts/ste-ccu8540-pinctrl.dtsi
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Copyright 2012 ST-Ericsson
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
-#include "ste-nomadik-pinctrl.dtsi"
-
-/ {
-	soc {
-		pinctrl {
-			uart0 {
-				uart0_default_mux: uart0_mux {
-					default_mux {
-						function = "u0";
-						groups = "u0_a_1";
-					};
-				};
-
-				uart0_default_mode: uart0_default {
-					default_cfg1 {
-						pins = "GPIO0", "GPIO2";
-						ste,config = <&in_pu>;
-					};
-
-					default_cfg2 {
-						pins = "GPIO1", "GPIO3";
-						ste,config = <&out_hi>;
-					};
-				};
-
-				uart0_sleep_mode: uart0_sleep {
-					sleep_cfg1 {
-						pins = "GPIO0", "GPIO2";
-						ste,config = <&slpm_in_pu>;
-					};
-
-					sleep_cfg2 {
-						pins = "GPIO1", "GPIO3";
-						ste,config = <&slpm_out_hi>;
-					};
-				};
-			};
-
-			uart2 {
-				uart2_default_mode: uart2_default {
-					default_mux {
-						function = "u2";
-						groups = "u2txrx_a_1";
-					};
-
-					default_cfg1 {
-						pins = "GPIO120";
-						ste,config = <&in_pu>;
-					};
-
-					default_cfg2 {
-						pins = "GPIO121";
-						ste,config = <&out_hi>;
-					};
-				};
-
-				uart2_sleep_mode: uart2_sleep {
-					sleep_cfg1 {
-						pins = "GPIO120";
-						ste,config = <&slpm_in_pu>;
-					};
-
-					sleep_cfg2 {
-						pins = "GPIO121";
-						ste,config = <&slpm_out_hi>;
-					};
-				};
-			};
-
-			i2c0 {
-				i2c0_default_mux: i2c_mux {
-					default_mux {
-						function = "i2c0";
-						groups = "i2c0_a_1";
-					};
-				};
-
-				i2c0_default_mode: i2c_default {
-					default_cfg1 {
-						pins = "GPIO147", "GPIO148";
-						ste,config = <&in_pu>;
-					};
-				};
-
-				i2c0_sleep_mode: i2c_sleep {
-					sleep_cfg1 {
-						pins = "GPIO147", "GPIO148";
-						ste,config = <&slpm_in_pu>;
-					};
-				};
-			};
-
-			i2c1 {
-				i2c1_default_mux: i2c_mux {
-					default_mux {
-						function = "i2c1";
-						groups = "i2c1_b_2";
-					};
-				};
-
-				i2c1_default_mode: i2c_default {
-					default_cfg1 {
-						pins = "GPIO16", "GPIO17";
-						ste,config = <&in_pu>;
-					};
-				};
-
-				i2c1_sleep_mode: i2c_sleep {
-					sleep_cfg1 {
-						pins = "GPIO16", "GPIO17";
-						ste,config = <&slpm_in_pu>;
-					};
-				};
-			};
-
-			i2c2 {
-				i2c2_default_mux: i2c_mux {
-					default_mux {
-						function = "i2c2";
-						groups = "i2c2_b_2";
-					};
-				};
-
-				i2c2_default_mode: i2c_default {
-					default_cfg1 {
-						pins = "GPIO10", "GPIO11";
-						ste,config = <&in_pu>;
-					};
-				};
-
-				i2c2_sleep_mode: i2c_sleep {
-					sleep_cfg1 {
-						pins = "GPIO11", "GPIO11";
-						ste,config = <&slpm_in_pu>;
-					};
-				};
-			};
-
-			i2c4 {
-				i2c4_default_mux: i2c_mux {
-					default_mux {
-						function = "i2c4";
-						groups = "i2c4_b_2";
-					};
-				};
-
-				i2c4_default_mode: i2c_default {
-					default_cfg1 {
-						pins = "GPIO122", "GPIO123";
-						ste,config = <&in_pu>;
-					};
-				};
-
-				i2c4_sleep_mode: i2c_sleep {
-					sleep_cfg1 {
-						pins = "GPIO122", "GPIO123";
-						ste,config = <&slpm_in_pu>;
-					};
-				};
-			};
-
-			i2c5 {
-				i2c5_default_mux: i2c_mux {
-					default_mux {
-						function = "i2c5";
-						groups = "i2c5_c_2";
-					};
-				};
-
-				i2c5_default_mode: i2c_default {
-					default_cfg1 {
-						pins = "GPIO118", "GPIO119";
-						ste,config = <&in_pu>;
-					};
-				};
-
-				i2c5_sleep_mode: i2c_sleep {
-					sleep_cfg1 {
-						pins = "GPIO118", "GPIO119";
-						ste,config = <&slpm_in_pu>;
-					};
-				};
-			};
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/ste-ccu8540.dts b/arch/arm/boot/dts/ste-ccu8540.dts
deleted file mode 100644
index 6eaaf638e52e..000000000000
--- a/arch/arm/boot/dts/ste-ccu8540.dts
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 2013 ST-Ericsson AB
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
-
-/dts-v1/;
-#include "ste-dbx5x0.dtsi"
-#include "ste-ccu8540-pinctrl.dtsi"
-
-/ {
-	model = "ST-Ericsson U8540 platform with Device Tree";
-	compatible = "st-ericsson,ccu8540", "st-ericsson,u8540";
-
-	/* This stablilizes the serial port enumeration */
-	aliases {
-		serial0 = &ux500_serial0;
-		serial1 = &ux500_serial1;
-		serial2 = &ux500_serial2;
-	};
-
-	memory@0 {
-		device_type = "memory";
-		reg = <0x20000000 0x1f000000>, <0xc0000000 0x3f000000>;
-	};
-
-	soc {
-		pinctrl {
-			compatible = "stericsson,db8540-pinctrl";
-		};
-
-		prcmu@80157000 {
-			reg = <0x80157000 0x2000>, <0x801b0000 0x8000>, <0x801b8000 0x3000>;
-			reg-names = "prcmu", "prcmu-tcpm", "prcmu-tcdm";
-		};
-
-		uart@80120000 {
-			pinctrl-names = "default", "sleep";
-			pinctrl-0 = <&uart0_default_mux>, <&uart0_default_mode>;
-			pinctrl-1 = <&uart0_sleep_mode>;
-			status = "okay";
-		};
-
-		uart@80121000 {
-			status = "okay";
-		};
-
-		uart@80007000 {
-			pinctrl-names = "default", "sleep";
-			pinctrl-0 = <&uart2_default_mode>;
-			pinctrl-1 = <&uart2_sleep_mode>;
-			status = "okay";
-		};
-
-		i2c0: i2c@80004000 {
-			pinctrl-names = "default","sleep";
-			pinctrl-0 = <&i2c0_default_mux>, <&i2c0_default_mode>;
-			pinctrl-1 = <&i2c0_sleep_mode>;
-		};
-
-		i2c1: i2c@80122000 {
-			pinctrl-names = "default","sleep";
-			pinctrl-0 = <&i2c1_default_mux>, <&i2c1_default_mode>;
-			pinctrl-1 = <&i2c1_sleep_mode>;
-		};
-
-		i2c2: i2c@80128000 {
-			pinctrl-names = "default","sleep";
-			pinctrl-0 = <&i2c2_default_mux>, <&i2c2_default_mode>;
-			pinctrl-1 = <&i2c2_sleep_mode>;
-		};
-
-		i2c3: i2c@80110000 {
-			status = "disabled";
-		};
-
-		i2c4: i2c@8012a000 {
-			pinctrl-names = "default","sleep";
-			pinctrl-0 = <&i2c4_default_mux>, <&i2c4_default_mode>;
-			pinctrl-1 = <&i2c4_sleep_mode>;
-		};
-
-		i2c5: i2c@80001000 {
-			pinctrl-names = "default","sleep";
-			pinctrl-0 = <&i2c5_default_mux>, <&i2c5_default_mode>;
-			pinctrl-1 = <&i2c5_sleep_mode>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/ste-ccu9540.dts b/arch/arm/boot/dts/ste-ccu9540.dts
deleted file mode 100644
index b3b9bb8e1aa8..000000000000
--- a/arch/arm/boot/dts/ste-ccu9540.dts
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright 2012 ST-Ericsson AB
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
-
-/dts-v1/;
-#include "ste-dbx5x0.dtsi"
-
-/ {
-	model = "ST-Ericsson CCU9540 platform with Device Tree";
-	compatible = "st-ericsson,ccu9540", "st-ericsson,u9540";
-
-	/* This stablilizes the serial port enumeration */
-	aliases {
-		serial0 = &ux500_serial0;
-		serial1 = &ux500_serial1;
-		serial2 = &ux500_serial2;
-	};
-
-	memory {
-		reg = <0x00000000 0x20000000>;
-	};
-
-	soc {
-		uart@80120000 {
-			status = "okay";
-		};
-
-		uart@80121000 {
-			status = "okay";
-		};
-
-		uart@80007000 {
-			status = "okay";
-		};
-
-		// External Micro SD slot
-		sdi0_per1@80126000 {
-			arm,primecell-periphid = <0x10480180>;
-			max-frequency = <100000000>;
-			bus-width = <4>;
-			cap-sd-highspeed;
-			cap-mmc-highspeed;
-			vmmc-supply = <&ab8500_ldo_aux3_reg>;
-
-			cd-gpios  = <&gpio7 6 GPIO_ACTIVE_HIGH>; // 230
-			cd-inverted;
-
-			status = "okay";
-		};
-
-
-		// WLAN SDIO channel
-		sdi1_per2@80118000 {
-			arm,primecell-periphid = <0x10480180>;
-			max-frequency = <100000000>;
-			bus-width = <4>;
-
-			status = "okay";
-		};
-
-		// On-board eMMC
-		sdi4_per2@80114000 {
-			arm,primecell-periphid = <0x10480180>;
-		        max-frequency = <100000000>;
-			bus-width = <8>;
-			cap-mmc-highspeed;
-			vmmc-supply = <&ab8500_ldo_aux2_reg>;
-
-			status = "okay";
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/ste-snowball.dts b/arch/arm/boot/dts/ste-snowball.dts
index ade1d0d4e5f4..b0b94d053098 100644
--- a/arch/arm/boot/dts/ste-snowball.dts
+++ b/arch/arm/boot/dts/ste-snowball.dts
@@ -46,35 +46,35 @@
 		#size-cells = <0>;
 
 		button@1 {
-			debounce_interval = <50>;
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <2>;
 			label = "userpb";
 			gpios = <&gpio1 0 GPIO_ACTIVE_HIGH>;
 		};
 		button@2 {
-			debounce_interval = <50>;
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <3>;
 			label = "extkb1";
 			gpios = <&gpio4 23 GPIO_ACTIVE_HIGH>;
 		};
 		button@3 {
-			debounce_interval = <50>;
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <4>;
 			label = "extkb2";
 			gpios = <&gpio4 24 GPIO_ACTIVE_HIGH>;
 		};
 		button@4 {
-			debounce_interval = <50>;
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <5>;
 			label = "extkb3";
 			gpios = <&gpio5 1 GPIO_ACTIVE_HIGH>;
 		};
 		button@5 {
-			debounce_interval = <50>;
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <6>;
 			label = "extkb4";
diff --git a/arch/arm/boot/dts/stih407-family.dtsi b/arch/arm/boot/dts/stih407-family.dtsi
index f7362c31de29..9e29a4499938 100644
--- a/arch/arm/boot/dts/stih407-family.dtsi
+++ b/arch/arm/boot/dts/stih407-family.dtsi
@@ -206,19 +206,19 @@
 		vtg_main: sti-vtg-main@8d02800 {
 			compatible = "st,vtg";
 			reg = <0x8d02800 0x200>;
-			interrupts = <GIC_SPI 108 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		vtg_aux: sti-vtg-aux@8d00200 {
 			compatible = "st,vtg";
 			reg = <0x8d00200 0x100>;
-			interrupts = <GIC_SPI 109 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		serial@9830000 {
 			compatible = "st,asc";
 			reg = <0x9830000 0x2c>;
-			interrupts = <GIC_SPI 122 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clk_s_c0_flexgen CLK_EXT2F_A9>;
 			/* Pinctrl moved out to a per-board configuration */
 
@@ -228,7 +228,7 @@
 		serial@9831000 {
 			compatible = "st,asc";
 			reg = <0x9831000 0x2c>;
-			interrupts = <GIC_SPI 123 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_serial1>;
 			clocks = <&clk_s_c0_flexgen CLK_EXT2F_A9>;
@@ -239,7 +239,7 @@
 		serial@9832000 {
 			compatible = "st,asc";
 			reg = <0x9832000 0x2c>;
-			interrupts = <GIC_SPI 124 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_serial2>;
 			clocks = <&clk_s_c0_flexgen CLK_EXT2F_A9>;
@@ -251,7 +251,7 @@
 		sbc_serial0: serial@9530000 {
 			compatible = "st,asc";
 			reg = <0x9530000 0x2c>;
-			interrupts = <GIC_SPI 138 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_sbc_serial0>;
 			clocks = <&clk_sysin>;
@@ -262,7 +262,7 @@
 		serial@9531000 {
 			compatible = "st,asc";
 			reg = <0x9531000 0x2c>;
-			interrupts = <GIC_SPI 139 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_sbc_serial1>;
 			clocks = <&clk_sysin>;
@@ -574,7 +574,7 @@
 			status = "disabled";
 			reg = <0x09060000 0x7ff>, <0x9061008 0x20>;
 			reg-names = "mmc", "top-mmc-delay";
-			interrupts = <GIC_SPI 92 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "mmcirq";
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_mmc0>;
@@ -589,7 +589,7 @@
 			status = "disabled";
 			reg = <0x09080000 0x7ff>;
 			reg-names = "mmc";
-			interrupts = <GIC_SPI 90 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "mmcirq";
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_sd1>;
@@ -623,7 +623,7 @@
 			compatible = "st,ahci";
 			reg = <0x9b20000 0x1000>;
 
-			interrupts = <GIC_SPI 159 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hostc";
 
 			phys = <&phy_port0 PHY_TYPE_SATA>;
@@ -646,7 +646,7 @@
 			compatible = "st,ahci";
 			reg = <0x9b28000 0x1000>;
 
-			interrupts = <GIC_SPI 170 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hostc";
 
 			phys = <&phy_port1 PHY_TYPE_SATA>;
@@ -687,7 +687,7 @@
 			dwc3: dwc3@9900000 {
 				compatible	= "snps,dwc3";
 				reg		= <0x09900000 0x100000>;
-				interrupts	= <GIC_SPI 155 IRQ_TYPE_NONE>;
+				interrupts	= <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
 				dr_mode		= "host";
 				phy-names	= "usb2-phy", "usb3-phy";
 				phys		= <&usb2_picophy0>,
@@ -701,7 +701,7 @@
 			compatible	= "st,sti-pwm";
 			#pwm-cells	= <2>;
 			reg		= <0x9810000 0x68>;
-			interrupts      = <GIC_SPI 128 IRQ_TYPE_NONE>;
+			interrupts      = <GIC_SPI 128 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names	= "default";
 			pinctrl-0	= <&pinctrl_pwm0_chan0_default>;
 			clock-names	= "pwm";
@@ -716,7 +716,7 @@
 			compatible	= "st,sti-pwm";
 			#pwm-cells	= <2>;
 			reg		= <0x9510000 0x68>;
-			interrupts      = <GIC_SPI 131 IRQ_TYPE_NONE>;
+			interrupts      = <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names	= "default";
 			pinctrl-0	= <&pinctrl_pwm1_chan0_default
 					&pinctrl_pwm1_chan1_default
@@ -755,8 +755,8 @@
 			resets = <&softreset STIH407_ETH1_SOFTRESET>;
 			reset-names = "stmmaceth";
 
-			interrupts = <GIC_SPI 98 IRQ_TYPE_NONE>,
-				     <GIC_SPI 99 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "macirq", "eth_wake_irq";
 
 			/* DMA Bus Mode */
@@ -787,7 +787,7 @@
 		mailbox0: mailbox@8f00000  {
 			compatible	= "st,stih407-mailbox";
 			reg		= <0x8f00000 0x1000>;
-			interrupts	= <GIC_SPI 1 IRQ_TYPE_NONE>;
+			interrupts	= <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>;
 			#mbox-cells	= <2>;
 			mbox-name	= "a9";
 			status		= "okay";
@@ -857,7 +857,7 @@
 				 <&clk_s_c0_flexgen CLK_EXT2F_A9>,
 				 <&clk_s_c0_flexgen CLK_EXT2F_A9>,
 				 <&clk_s_c0_flexgen CLK_EXT2F_A9>;
-			interrupts = <GIC_SPI 5 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
 			dma-channels = <16>;
 			#dma-cells = <3>;
 		};
@@ -875,7 +875,7 @@
 				<&clk_s_c0_flexgen CLK_TX_ICN_DMU>,
 				<&clk_s_c0_flexgen CLK_EXT2F_A9>;
 
-			interrupts = <GIC_SPI 7 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
 			dma-channels = <16>;
 			#dma-cells = <3>;
 
@@ -890,7 +890,7 @@
 			      <0x8e77000 0x1000>,
 			      <0x8e78000 0x8000>;
 			reg-names = "slimcore", "dmem", "peripherals", "imem";
-			interrupts = <GIC_SPI 9 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
 			dma-channels = <16>;
 			#dma-cells = <3>;
 			clocks = <&clk_s_c0_flexgen CLK_FDMA>,
@@ -910,7 +910,7 @@
 			assigned-clock-parents = <0>, <&clk_s_d0_quadfs 0>;
 			assigned-clock-rates = <50000000>;
 			reg = <0x8d80000 0x158>;
-			interrupts = <GIC_SPI 84 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&fdma0 2 0 1>;
 			dma-names = "tx";
 
@@ -926,7 +926,7 @@
 			assigned-clock-parents = <0>, <&clk_s_d0_quadfs 1>;
 			assigned-clock-rates = <50000000>;
 			reg = <0x8d81000 0x158>;
-			interrupts = <GIC_SPI 85 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&fdma0 3 0 1>;
 			dma-names = "tx";
 
@@ -942,7 +942,7 @@
 			assigned-clock-parents = <0>, <&clk_s_d0_quadfs 2>;
 			assigned-clock-rates = <50000000>;
 			reg = <0x8d82000 0x158>;
-			interrupts = <GIC_SPI 86 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&fdma0 4 0 1>;
 			dma-names = "tx";
 
@@ -958,7 +958,7 @@
 			assigned-clock-parents = <0>, <&clk_s_d0_quadfs 3>;
 			assigned-clock-rates = <50000000>;
 			reg = <0x8d85000 0x158>;
-			interrupts = <GIC_SPI 89 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&fdma0 7 0 1>;
 			dma-names = "tx";
 
@@ -970,7 +970,7 @@
 			#sound-dai-cells = <0>;
 			st,syscfg = <&syscfg_core>;
 			reg = <0x8d83000 0x158>;
-			interrupts = <GIC_SPI 87 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&fdma0 5 0 1>;
 			dma-names = "rx";
 
@@ -982,7 +982,7 @@
 			#sound-dai-cells = <0>;
 			st,syscfg = <&syscfg_core>;
 			reg = <0x8d84000 0x158>;
-			interrupts = <GIC_SPI 88 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&fdma0 6 0 1>;
 			dma-names = "rx";
 
diff --git a/arch/arm/boot/dts/stih407-pinctrl.dtsi b/arch/arm/boot/dts/stih407-pinctrl.dtsi
index 53c6888d1fc0..e393519fb84c 100644
--- a/arch/arm/boot/dts/stih407-pinctrl.dtsi
+++ b/arch/arm/boot/dts/stih407-pinctrl.dtsi
@@ -52,7 +52,7 @@
 			st,syscfg = <&syscfg_sbc>;
 			reg = <0x0961f080 0x4>;
 			reg-names = "irqmux";
-			interrupts = <GIC_SPI 188 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "irqmux";
 			ranges = <0 0x09610000 0x6000>;
 
@@ -376,7 +376,7 @@
 			st,syscfg = <&syscfg_front>;
 			reg = <0x0920f080 0x4>;
 			reg-names = "irqmux";
-			interrupts = <GIC_SPI 189 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "irqmux";
 			ranges = <0 0x09200000 0x10000>;
 
@@ -936,7 +936,7 @@
 			st,syscfg = <&syscfg_front>;
 			reg = <0x0921f080 0x4>;
 			reg-names = "irqmux";
-			interrupts = <GIC_SPI 190 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "irqmux";
 			ranges = <0 0x09210000 0x10000>;
 
@@ -969,7 +969,7 @@
 			st,syscfg = <&syscfg_rear>;
 			reg = <0x0922f080 0x4>;
 			reg-names = "irqmux";
-			interrupts = <GIC_SPI 191 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "irqmux";
 			ranges = <0 0x09220000 0x6000>;
 
@@ -1164,7 +1164,7 @@
 			st,syscfg = <&syscfg_flash>;
 			reg = <0x0923f080 0x4>;
 			reg-names = "irqmux";
-			interrupts = <GIC_SPI 192 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "irqmux";
 			ranges = <0 0x09230000 0x3000>;
 
diff --git a/arch/arm/boot/dts/stih407.dtsi b/arch/arm/boot/dts/stih407.dtsi
index 57efc87dec2b..5b7951ffc350 100644
--- a/arch/arm/boot/dts/stih407.dtsi
+++ b/arch/arm/boot/dts/stih407.dtsi
@@ -108,7 +108,7 @@
 				reg = <0x8d04000 0x1000>;
 				reg-names = "hdmi-reg";
 				#sound-dai-cells = <0>;
-				interrupts = <GIC_SPI 106 IRQ_TYPE_NONE>;
+				interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
 				interrupt-names	= "irq";
 				clock-names = "pix",
 					      "tmds",
diff --git a/arch/arm/boot/dts/stih410.dtsi b/arch/arm/boot/dts/stih410.dtsi
index 3313005ee15c..888548ea9b5c 100644
--- a/arch/arm/boot/dts/stih410.dtsi
+++ b/arch/arm/boot/dts/stih410.dtsi
@@ -43,7 +43,7 @@
 		ohci0: usb@9a03c00 {
 			compatible = "st,st-ohci-300x";
 			reg = <0x9a03c00 0x100>;
-			interrupts = <GIC_SPI 180 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>,
 				 <&clk_s_c0_flexgen CLK_RX_ICN_DISP_0>;
 			resets = <&powerdown STIH407_USB2_PORT0_POWERDOWN>,
@@ -58,7 +58,7 @@
 		ehci0: usb@9a03e00 {
 			compatible = "st,st-ehci-300x";
 			reg = <0x9a03e00 0x100>;
-			interrupts = <GIC_SPI 151 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb0>;
 			clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>,
@@ -75,7 +75,7 @@
 		ohci1: usb@9a83c00 {
 			compatible = "st,st-ohci-300x";
 			reg = <0x9a83c00 0x100>;
-			interrupts = <GIC_SPI 181 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>,
 				 <&clk_s_c0_flexgen CLK_RX_ICN_DISP_0>;
 			resets = <&powerdown STIH407_USB2_PORT1_POWERDOWN>,
@@ -90,7 +90,7 @@
 		ehci1: usb@9a83e00 {
 			compatible = "st,st-ehci-300x";
 			reg = <0x9a83e00 0x100>;
-			interrupts = <GIC_SPI 153 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb1>;
 			clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>,
@@ -202,7 +202,7 @@
 				reg = <0x8d04000 0x1000>;
 				reg-names = "hdmi-reg";
 				#sound-dai-cells = <0>;
-				interrupts = <GIC_SPI 106 IRQ_TYPE_NONE>;
+				interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
 				interrupt-names	= "irq";
 				clock-names = "pix",
 					      "tmds",
@@ -254,7 +254,7 @@
 		bdisp0:bdisp@9f10000 {
 			compatible = "st,stih407-bdisp";
 			reg = <0x9f10000 0x1000>;
-			interrupts = <GIC_SPI 38 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "bdisp";
 			clocks = <&clk_s_c0_flexgen CLK_IC_BDISP_0>;
 		};
@@ -263,8 +263,8 @@
 			compatible = "st,st-hva";
 			reg = <0x8c85000 0x400>, <0x6000000 0x40000>;
 			reg-names = "hva_registers", "hva_esram";
-			interrupts = <GIC_SPI 58 IRQ_TYPE_NONE>,
-				     <GIC_SPI 59 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "clk_hva";
 			clocks = <&clk_s_c0_flexgen CLK_HVA>;
 		};
@@ -292,7 +292,7 @@
 			reg = <0x94a087c 0x64>;
 			clocks = <&clk_sysin>;
 			clock-names = "cec-clk";
-			interrupts = <GIC_SPI 140 IRQ_TYPE_NONE>;
+			interrupts = <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "cec-irq";
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_cec0_default>;
diff --git a/arch/arm/boot/dts/stihxxx-b2120.dtsi b/arch/arm/boot/dts/stihxxx-b2120.dtsi
index c67edb1a8121..4dedfcb0fcb3 100644
--- a/arch/arm/boot/dts/stihxxx-b2120.dtsi
+++ b/arch/arm/boot/dts/stihxxx-b2120.dtsi
@@ -154,8 +154,8 @@
 			reg		= <0x08a20000 0x10000>,
 					  <0x08a00000 0x4000>;
 			reg-names	= "c8sectpfe", "c8sectpfe-ram";
-			interrupts	= <GIC_SPI 34 IRQ_TYPE_NONE>,
-					  <GIC_SPI 35 IRQ_TYPE_NONE>;
+			interrupts	= <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>,
+					  <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names	= "c8sectpfe-error-irq",
 					  "c8sectpfe-idle-irq";
 			pinctrl-0	= <&pinctrl_tsin0_serial>;
diff --git a/arch/arm/boot/dts/stm32f469-disco.dts b/arch/arm/boot/dts/stm32f469-disco.dts
index 2f76726bf335..3ee768cb86fc 100644
--- a/arch/arm/boot/dts/stm32f469-disco.dts
+++ b/arch/arm/boot/dts/stm32f469-disco.dts
@@ -46,7 +46,7 @@
  */
 
 /dts-v1/;
-#include "stm32f429.dtsi"
+#include "stm32f469.dtsi"
 #include "stm32f469-pinctrl.dtsi"
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
@@ -112,7 +112,7 @@
 	vcc5v_otg: vcc5v-otg-regulator {
 		compatible = "regulator-fixed";
 		enable-active-high;
-		gpio = <&gpiob 2 0>;
+		gpio = <&gpiob 2 GPIO_ACTIVE_HIGH>;
 		regulator-name = "vcc5_host1";
 		regulator-always-on;
 	};
@@ -126,6 +126,55 @@
 	clock-frequency = <8000000>;
 };
 
+&dsi {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			dsi_in: endpoint {
+				remote-endpoint = <&ltdc_out_dsi>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			dsi_out: endpoint {
+				remote-endpoint = <&dsi_panel_in>;
+			};
+		};
+	};
+
+	panel-dsi@0 {
+		compatible = "orisetech,otm8009a";
+		reg = <0>; /* dsi virtual channel (0..3) */
+		reset-gpios = <&gpioh 7 GPIO_ACTIVE_LOW>;
+		status = "okay";
+
+		port {
+			dsi_panel_in: endpoint {
+				remote-endpoint = <&dsi_out>;
+			};
+		};
+	};
+};
+
+&ltdc {
+	dma-ranges;
+	status = "okay";
+
+	port {
+		ltdc_out_dsi: endpoint@0 {
+			remote-endpoint = <&dsi_in>;
+		};
+	};
+};
+
 &rtc {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/stm32f469.dtsi b/arch/arm/boot/dts/stm32f469.dtsi
new file mode 100644
index 000000000000..5ae5213f68cb
--- /dev/null
+++ b/arch/arm/boot/dts/stm32f469.dtsi
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
+/* Copyright (C) STMicroelectronics 2017 - All Rights Reserved */
+
+#include "stm32f429.dtsi"
+
+/ {
+	soc {
+		dsi: dsi@40016c00 {
+			compatible = "st,stm32-dsi";
+			reg = <0x40016c00 0x800>;
+			interrupts = <92>;
+			resets = <&rcc STM32F4_APB2_RESET(DSI)>;
+			reset-names = "apb";
+			clocks = <&rcc 1 CLK_F469_DSI>, <&clk_hse>;
+			clock-names = "pclk", "ref";
+			status = "disabled";
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/stm32f746-disco.dts b/arch/arm/boot/dts/stm32f746-disco.dts
index be94c6ad7e94..f9ad71f7c807 100644
--- a/arch/arm/boot/dts/stm32f746-disco.dts
+++ b/arch/arm/boot/dts/stm32f746-disco.dts
@@ -90,6 +90,14 @@
 	clock-frequency = <25000000>;
 };
 
+&i2c1 {
+	pinctrl-0 = <&i2c1_pins_b>;
+	pinctrl-names = "default";
+	i2c-scl-rising-time-ns = <185>;
+	i2c-scl-falling-time-ns = <20>;
+	status = "okay";
+};
+
 &sdio1 {
 	status = "okay";
 	vmmc-supply = <&mmc_vcard>;
diff --git a/arch/arm/boot/dts/stm32f746.dtsi b/arch/arm/boot/dts/stm32f746.dtsi
index 4be2ee575b19..1479e3eb05fa 100644
--- a/arch/arm/boot/dts/stm32f746.dtsi
+++ b/arch/arm/boot/dts/stm32f746.dtsi
@@ -345,6 +345,42 @@
 			status = "disabled";
 		};
 
+		i2c2: i2c@40005800 {
+			compatible = "st,stm32f7-i2c";
+			reg = <0x40005800 0x400>;
+			interrupts = <33>,
+				     <34>;
+			resets = <&rcc STM32F7_APB1_RESET(I2C2)>;
+			clocks = <&rcc 1 CLK_I2C2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c3: i2c@40005C00 {
+			compatible = "st,stm32f7-i2c";
+			reg = <0x40005C00 0x400>;
+			interrupts = <72>,
+				     <73>;
+			resets = <&rcc STM32F7_APB1_RESET(I2C3)>;
+			clocks = <&rcc 1 CLK_I2C3>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c4: i2c@40006000 {
+			compatible = "st,stm32f7-i2c";
+			reg = <0x40006000 0x400>;
+			interrupts = <95>,
+				     <96>;
+			resets = <&rcc STM32F7_APB1_RESET(I2C4)>;
+			clocks = <&rcc 1 CLK_I2C4>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		cec: cec@40006c00 {
 			compatible = "st,stm32-cec";
 			reg = <0x40006C00 0x400>;
diff --git a/arch/arm/boot/dts/stm32f769-disco.dts b/arch/arm/boot/dts/stm32f769-disco.dts
index 2241eecdabfe..677276ba4dbe 100644
--- a/arch/arm/boot/dts/stm32f769-disco.dts
+++ b/arch/arm/boot/dts/stm32f769-disco.dts
@@ -111,6 +111,14 @@
 	clock-frequency = <25000000>;
 };
 
+&i2c1 {
+	pinctrl-0 = <&i2c1_pins_b>;
+	pinctrl-names = "default";
+	i2c-scl-rising-time-ns = <185>;
+	i2c-scl-falling-time-ns = <20>;
+	status = "okay";
+};
+
 &rtc {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/stm32h743-pinctrl.dtsi b/arch/arm/boot/dts/stm32h743-pinctrl.dtsi
index 0f15dfb98381..24be8e63dec8 100644
--- a/arch/arm/boot/dts/stm32h743-pinctrl.dtsi
+++ b/arch/arm/boot/dts/stm32h743-pinctrl.dtsi
@@ -163,6 +163,16 @@
 				#interrupt-cells = <2>;
 			};
 
+			i2c1_pins_a: i2c1@0 {
+				pins {
+					pinmux = <STM32_PINMUX('B', 6, AF4)>, /* I2C1_SCL */
+						 <STM32_PINMUX('B', 7, AF4)>; /* I2C1_SDA */
+					bias-disable;
+					drive-open-drain;
+					slew-rate = <0>;
+				};
+			};
+
 			usart1_pins: usart1@0 {
 				pins1 {
 					pinmux = <STM32_PINMUX('B', 14, AF4)>; /* USART1_TX */
diff --git a/arch/arm/boot/dts/stm32h743.dtsi b/arch/arm/boot/dts/stm32h743.dtsi
index 2bb103e1194d..637beffe5067 100644
--- a/arch/arm/boot/dts/stm32h743.dtsi
+++ b/arch/arm/boot/dts/stm32h743.dtsi
@@ -86,6 +86,7 @@
 
 			pwm {
 				compatible = "st,stm32-pwm-lp";
+				#pwm-cells = <3>;
 				status = "disabled";
 			};
 
@@ -130,6 +131,42 @@
 			clocks = <&rcc USART2_CK>;
 		};
 
+		i2c1: i2c@40005400 {
+			compatible = "st,stm32f7-i2c";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x40005400 0x400>;
+			interrupts = <31>,
+				     <32>;
+			resets = <&rcc STM32H7_APB1L_RESET(I2C1)>;
+			clocks = <&rcc I2C1_CK>;
+			status = "disabled";
+		};
+
+		i2c2: i2c@40005800 {
+			compatible = "st,stm32f7-i2c";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x40005800 0x400>;
+			interrupts = <33>,
+				     <34>;
+			resets = <&rcc STM32H7_APB1L_RESET(I2C2)>;
+			clocks = <&rcc I2C2_CK>;
+			status = "disabled";
+		};
+
+		i2c3: i2c@40005C00 {
+			compatible = "st,stm32f7-i2c";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x40005C00 0x400>;
+			interrupts = <72>,
+				     <73>;
+			resets = <&rcc STM32H7_APB1L_RESET(I2C3)>;
+			clocks = <&rcc I2C3_CK>;
+			status = "disabled";
+		};
+
 		dac: dac@40007400 {
 			compatible = "st,stm32h7-dac-core";
 			reg = <0x40007400 0x400>;
@@ -323,6 +360,18 @@
 			status = "disabled";
 		};
 
+		i2c4: i2c@58001C00 {
+			compatible = "st,stm32f7-i2c";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x58001C00 0x400>;
+			interrupts = <95>,
+				     <96>;
+			resets = <&rcc STM32H7_APB4_RESET(I2C4)>;
+			clocks = <&rcc I2C4_CK>;
+			status = "disabled";
+		};
+
 		lptimer2: timer@58002400 {
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -334,6 +383,7 @@
 
 			pwm {
 				compatible = "st,stm32-pwm-lp";
+				#pwm-cells = <3>;
 				status = "disabled";
 			};
 
@@ -360,6 +410,7 @@
 
 			pwm {
 				compatible = "st,stm32-pwm-lp";
+				#pwm-cells = <3>;
 				status = "disabled";
 			};
 
@@ -381,6 +432,7 @@
 
 			pwm {
 				compatible = "st,stm32-pwm-lp";
+				#pwm-cells = <3>;
 				status = "disabled";
 			};
 		};
@@ -396,6 +448,7 @@
 
 			pwm {
 				compatible = "st,stm32-pwm-lp";
+				#pwm-cells = <3>;
 				status = "disabled";
 			};
 		};
diff --git a/arch/arm/boot/dts/stm32h743i-eval.dts b/arch/arm/boot/dts/stm32h743i-eval.dts
index c7187e18ea16..3f8e0c4a998d 100644
--- a/arch/arm/boot/dts/stm32h743i-eval.dts
+++ b/arch/arm/boot/dts/stm32h743i-eval.dts
@@ -92,6 +92,14 @@
 	clock-frequency = <25000000>;
 };
 
+&i2c1 {
+	pinctrl-0 = <&i2c1_pins_a>;
+	pinctrl-names = "default";
+	i2c-scl-rising-time-ns = <185>;
+	i2c-scl-falling-time-ns = <20>;
+	status = "okay";
+};
+
 &rtc {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/stm32mp157-pinctrl.dtsi b/arch/arm/boot/dts/stm32mp157-pinctrl.dtsi
index c0743305f31b..88e91335b9d7 100644
--- a/arch/arm/boot/dts/stm32mp157-pinctrl.dtsi
+++ b/arch/arm/boot/dts/stm32mp157-pinctrl.dtsi
@@ -7,7 +7,7 @@
 
 / {
 	soc {
-		pinctrl: pin-controller {
+		pinctrl: pin-controller@50002000 {
 			#address-cells = <1>;
 			#size-cells = <1>;
 			compatible = "st,stm32mp157-pinctrl";
@@ -20,7 +20,7 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0x0 0x400>;
-				clocks = <&clk_pll3_p>;
+				clocks = <&rcc GPIOA>;
 				st,bank-name = "GPIOA";
 				ngpios = <16>;
 				gpio-ranges = <&pinctrl 0 0 16>;
@@ -32,7 +32,7 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0x1000 0x400>;
-				clocks = <&clk_pll3_p>;
+				clocks = <&rcc GPIOB>;
 				st,bank-name = "GPIOB";
 				ngpios = <16>;
 				gpio-ranges = <&pinctrl 0 16 16>;
@@ -44,7 +44,7 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0x2000 0x400>;
-				clocks = <&clk_pll3_p>;
+				clocks = <&rcc GPIOC>;
 				st,bank-name = "GPIOC";
 				ngpios = <16>;
 				gpio-ranges = <&pinctrl 0 32 16>;
@@ -56,7 +56,7 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0x3000 0x400>;
-				clocks = <&clk_pll3_p>;
+				clocks = <&rcc GPIOD>;
 				st,bank-name = "GPIOD";
 				ngpios = <16>;
 				gpio-ranges = <&pinctrl 0 48 16>;
@@ -68,7 +68,7 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0x4000 0x400>;
-				clocks = <&clk_pll3_p>;
+				clocks = <&rcc GPIOE>;
 				st,bank-name = "GPIOE";
 				ngpios = <16>;
 				gpio-ranges = <&pinctrl 0 64 16>;
@@ -80,7 +80,7 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0x5000 0x400>;
-				clocks = <&clk_pll3_p>;
+				clocks = <&rcc GPIOF>;
 				st,bank-name = "GPIOF";
 				ngpios = <16>;
 				gpio-ranges = <&pinctrl 0 80 16>;
@@ -92,7 +92,7 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0x6000 0x400>;
-				clocks = <&clk_pll3_p>;
+				clocks = <&rcc GPIOG>;
 				st,bank-name = "GPIOG";
 				ngpios = <16>;
 				gpio-ranges = <&pinctrl 0 96 16>;
@@ -104,7 +104,7 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0x7000 0x400>;
-				clocks = <&clk_pll3_p>;
+				clocks = <&rcc GPIOH>;
 				st,bank-name = "GPIOH";
 				ngpios = <16>;
 				gpio-ranges = <&pinctrl 0 112 16>;
@@ -116,7 +116,7 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0x8000 0x400>;
-				clocks = <&clk_pll3_p>;
+				clocks = <&rcc GPIOI>;
 				st,bank-name = "GPIOI";
 				ngpios = <16>;
 				gpio-ranges = <&pinctrl 0 128 16>;
@@ -128,7 +128,7 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0x9000 0x400>;
-				clocks = <&clk_pll3_p>;
+				clocks = <&rcc GPIOJ>;
 				st,bank-name = "GPIOJ";
 				ngpios = <16>;
 				gpio-ranges = <&pinctrl 0 144 16>;
@@ -140,13 +140,124 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0xa000 0x400>;
-				clocks = <&clk_pll3_p>;
+				clocks = <&rcc GPIOK>;
 				st,bank-name = "GPIOK";
 				ngpios = <8>;
 				gpio-ranges = <&pinctrl 0 160 8>;
 			};
 
-			uart4_pins_a: uart4@0 {
+			cec_pins_a: cec-0 {
+				pins {
+					pinmux = <STM32_PINMUX('A', 15, AF4)>;
+					bias-disable;
+					drive-open-drain;
+					slew-rate = <0>;
+				};
+			};
+
+			i2c1_pins_a: i2c1-0 {
+				pins {
+					pinmux = <STM32_PINMUX('D', 12, AF5)>, /* I2C1_SCL */
+						 <STM32_PINMUX('F', 15, AF5)>; /* I2C1_SDA */
+					bias-disable;
+					drive-open-drain;
+					slew-rate = <0>;
+				};
+			};
+
+			i2c2_pins_a: i2c2-0 {
+				pins {
+					pinmux = <STM32_PINMUX('H', 4, AF4)>, /* I2C2_SCL */
+						 <STM32_PINMUX('H', 5, AF4)>; /* I2C2_SDA */
+					bias-disable;
+					drive-open-drain;
+					slew-rate = <0>;
+				};
+			};
+
+			i2c5_pins_a: i2c5-0 {
+				pins {
+					pinmux = <STM32_PINMUX('A', 11, AF4)>, /* I2C5_SCL */
+						 <STM32_PINMUX('A', 12, AF4)>; /* I2C5_SDA */
+					bias-disable;
+					drive-open-drain;
+					slew-rate = <0>;
+				};
+			};
+
+			pwm2_pins_a: pwm2-0 {
+				pins {
+					pinmux = <STM32_PINMUX('A', 3, AF1)>; /* TIM2_CH4 */
+					bias-pull-down;
+					drive-push-pull;
+					slew-rate = <0>;
+				};
+			};
+
+			pwm8_pins_a: pwm8-0 {
+				pins {
+					pinmux = <STM32_PINMUX('I', 2, AF3)>; /* TIM8_CH4 */
+					bias-pull-down;
+					drive-push-pull;
+					slew-rate = <0>;
+				};
+			};
+
+			pwm12_pins_a: pwm12-0 {
+				pins {
+					pinmux = <STM32_PINMUX('H', 6, AF2)>; /* TIM12_CH1 */
+					bias-pull-down;
+					drive-push-pull;
+					slew-rate = <0>;
+				};
+			};
+
+			qspi_clk_pins_a: qspi-clk-0 {
+				pins {
+					pinmux = <STM32_PINMUX('F', 10, AF9)>; /* QSPI_CLK */
+					bias-disable;
+					drive-push-pull;
+					slew-rate = <3>;
+				};
+			};
+
+			qspi_bk1_pins_a: qspi-bk1-0 {
+				pins1 {
+					pinmux = <STM32_PINMUX('F', 8, AF10)>, /* QSPI_BK1_IO0 */
+						 <STM32_PINMUX('F', 9, AF10)>, /* QSPI_BK1_IO1 */
+						 <STM32_PINMUX('F', 7, AF9)>, /* QSPI_BK1_IO2 */
+						 <STM32_PINMUX('F', 6, AF9)>; /* QSPI_BK1_IO3 */
+					bias-disable;
+					drive-push-pull;
+					slew-rate = <3>;
+				};
+				pins2 {
+					pinmux = <STM32_PINMUX('B', 6, AF10)>; /* QSPI_BK1_NCS */
+					bias-pull-up;
+					drive-push-pull;
+					slew-rate = <3>;
+				};
+			};
+
+			qspi_bk2_pins_a: qspi-bk2-0 {
+				pins1 {
+					pinmux = <STM32_PINMUX('H', 2, AF9)>, /* QSPI_BK2_IO0 */
+						 <STM32_PINMUX('H', 3, AF9)>, /* QSPI_BK2_IO1 */
+						 <STM32_PINMUX('G', 10, AF11)>, /* QSPI_BK2_IO2 */
+						 <STM32_PINMUX('G', 7, AF11)>; /* QSPI_BK2_IO3 */
+					bias-disable;
+					drive-push-pull;
+					slew-rate = <3>;
+				};
+				pins2 {
+					pinmux = <STM32_PINMUX('C', 0, AF10)>; /* QSPI_BK2_NCS */
+					bias-pull-up;
+					drive-push-pull;
+					slew-rate = <3>;
+				};
+			};
+
+			uart4_pins_a: uart4-0 {
 				pins1 {
 					pinmux = <STM32_PINMUX('G', 11, AF6)>; /* UART4_TX */
 					bias-disable;
@@ -160,7 +271,7 @@
 			};
 		};
 
-		pinctrl_z: pin-controller-z {
+		pinctrl_z: pin-controller-z@54004000 {
 			#address-cells = <1>;
 			#size-cells = <1>;
 			compatible = "st,stm32mp157-z-pinctrl";
@@ -174,12 +285,22 @@
 				interrupt-controller;
 				#interrupt-cells = <2>;
 				reg = <0 0x400>;
-				clocks = <&clk_pll2_p>;
+				clocks = <&rcc GPIOZ>;
 				st,bank-name = "GPIOZ";
 				st,bank-ioport = <11>;
 				ngpios = <8>;
 				gpio-ranges = <&pinctrl_z 0 400 8>;
 			};
+
+			i2c4_pins_a: i2c4-0 {
+				pins {
+					pinmux = <STM32_PINMUX('Z', 4, AF6)>, /* I2C4_SCL */
+						 <STM32_PINMUX('Z', 5, AF6)>; /* I2C4_SDA */
+					bias-disable;
+					drive-open-drain;
+					slew-rate = <0>;
+				};
+			};
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/stm32mp157c-ed1.dts b/arch/arm/boot/dts/stm32mp157c-ed1.dts
index 9f90337a22e3..ae336530b59b 100644
--- a/arch/arm/boot/dts/stm32mp157c-ed1.dts
+++ b/arch/arm/boot/dts/stm32mp157c-ed1.dts
@@ -16,13 +16,56 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@c0000000 {
 		reg = <0xC0000000 0x40000000>;
 	};
 
 	aliases {
 		serial0 = &uart4;
 	};
+
+	reg11: reg11 {
+		compatible = "regulator-fixed";
+		regulator-name = "reg11";
+		regulator-min-microvolt = <1100000>;
+		regulator-max-microvolt = <1100000>;
+		regulator-always-on;
+	};
+
+	reg18: reg18 {
+		compatible = "regulator-fixed";
+		regulator-name = "reg18";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+	};
+
+	vdd_usb: vdd-usb {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_usb";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+};
+
+&rng1 {
+	status = "okay";
+};
+
+&timers6 {
+	status = "okay";
+	timer@5 {
+		status = "okay";
+	};
+};
+
+&i2c4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c4_pins_a>;
+	i2c-scl-rising-time-ns = <185>;
+	i2c-scl-falling-time-ns = <20>;
+	status = "okay";
 };
 
 &uart4 {
@@ -30,3 +73,15 @@
 	pinctrl-0 = <&uart4_pins_a>;
 	status = "okay";
 };
+
+&usbphyc_port0 {
+	phy-supply = <&vdd_usb>;
+	vdda1v1-supply = <&reg11>;
+	vdda1v8-supply = <&reg18>;
+};
+
+&usbphyc_port1 {
+	phy-supply = <&vdd_usb>;
+	vdda1v1-supply = <&reg11>;
+	vdda1v8-supply = <&reg18>;
+};
diff --git a/arch/arm/boot/dts/stm32mp157c-ev1.dts b/arch/arm/boot/dts/stm32mp157c-ev1.dts
index 57e6dbc52e09..9382d8063031 100644
--- a/arch/arm/boot/dts/stm32mp157c-ev1.dts
+++ b/arch/arm/boot/dts/stm32mp157c-ev1.dts
@@ -19,3 +19,90 @@
 		serial0 = &uart4;
 	};
 };
+
+&cec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&cec_pins_a>;
+	status = "okay";
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_pins_a>;
+	i2c-scl-rising-time-ns = <185>;
+	i2c-scl-falling-time-ns = <20>;
+	status = "okay";
+};
+
+&i2c5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c5_pins_a>;
+	i2c-scl-rising-time-ns = <185>;
+	i2c-scl-falling-time-ns = <20>;
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_clk_pins_a &qspi_bk1_pins_a &qspi_bk2_pins_a>;
+	reg = <0x58003000 0x1000>, <0x70000000 0x4000000>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	flash0: mx66l51235l@0 {
+		reg = <0>;
+		spi-rx-bus-width = <4>;
+		spi-max-frequency = <108000000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+
+	flash1: mx66l51235l@1 {
+		reg = <1>;
+		spi-rx-bus-width = <4>;
+		spi-max-frequency = <108000000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+};
+
+&timers2 {
+	status = "disabled";
+	pwm {
+		pinctrl-0 = <&pwm2_pins_a>;
+		pinctrl-names = "default";
+		status = "okay";
+	};
+	timer@1 {
+		status = "okay";
+	};
+};
+
+&timers8 {
+	status = "disabled";
+	pwm {
+		pinctrl-0 = <&pwm8_pins_a>;
+		pinctrl-names = "default";
+		status = "okay";
+	};
+	timer@7 {
+		status = "okay";
+	};
+};
+
+&timers12 {
+	status = "disabled";
+	pwm {
+		pinctrl-0 = <&pwm12_pins_a>;
+		pinctrl-names = "default";
+		status = "okay";
+	};
+	timer@11 {
+		status = "okay";
+	};
+};
+
+&usbphyc {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/stm32mp157c.dtsi b/arch/arm/boot/dts/stm32mp157c.dtsi
index 9e17e42b02b2..b66f673b5038 100644
--- a/arch/arm/boot/dts/stm32mp157c.dtsi
+++ b/arch/arm/boot/dts/stm32mp157c.dtsi
@@ -4,6 +4,8 @@
  * Author: Ludovic Barre <ludovic.barre@st.com> for STMicroelectronics.
  */
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/stm32mp1-clks.h>
+#include <dt-bindings/reset/stm32mp1-resets.h>
 
 / {
 	#address-cells = <1>;
@@ -71,12 +73,6 @@
 			clock-frequency = <24000000>;
 		};
 
-		clk_pll_per: clk-pll-per {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			clock-frequency = <64000000>;
-		};
-
 		clk_hsi: clk-hsi {
 			#clock-cells = <0>;
 			compatible = "fixed-clock";
@@ -100,24 +96,6 @@
 			compatible = "fixed-clock";
 			clock-frequency = <4000000>;
 		};
-
-		clk_pclk1: clk-pclk1 {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			clock-frequency = <86000000>;
-		};
-
-		clk_pll3_p: clk-pll3_p {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			clock-frequency = <172000000>;
-		};
-
-		clk_pll2_p: clk-pll2_p {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			clock-frequency = <264000000>;
-		};
 	};
 
 	soc {
@@ -127,67 +105,729 @@
 		interrupt-parent = <&intc>;
 		ranges;
 
+		timers2: timer@40000000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x40000000 0x400>;
+			clocks = <&rcc TIM2_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+
+			timer@1 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <1>;
+				status = "disabled";
+			};
+		};
+
+		timers3: timer@40001000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x40001000 0x400>;
+			clocks = <&rcc TIM3_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+
+			timer@2 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <2>;
+				status = "disabled";
+			};
+		};
+
+		timers4: timer@40002000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x40002000 0x400>;
+			clocks = <&rcc TIM4_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+
+			timer@3 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <3>;
+				status = "disabled";
+			};
+		};
+
+		timers5: timer@40003000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x40003000 0x400>;
+			clocks = <&rcc TIM5_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+
+			timer@4 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <4>;
+				status = "disabled";
+			};
+		};
+
+		timers6: timer@40004000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x40004000 0x400>;
+			clocks = <&rcc TIM6_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			timer@5 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <5>;
+				status = "disabled";
+			};
+		};
+
+		timers7: timer@40005000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x40005000 0x400>;
+			clocks = <&rcc TIM7_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			timer@6 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <6>;
+				status = "disabled";
+			};
+		};
+
+		timers12: timer@40006000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x40006000 0x400>;
+			clocks = <&rcc TIM12_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+
+			timer@11 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <11>;
+				status = "disabled";
+			};
+		};
+
+		timers13: timer@40007000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x40007000 0x400>;
+			clocks = <&rcc TIM13_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+
+			timer@12 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <12>;
+				status = "disabled";
+			};
+		};
+
+		timers14: timer@40008000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x40008000 0x400>;
+			clocks = <&rcc TIM14_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+
+			timer@13 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <13>;
+				status = "disabled";
+			};
+		};
+
+		lptimer1: timer@40009000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-lptimer";
+			reg = <0x40009000 0x400>;
+			clocks = <&rcc LPTIM1_K>;
+			clock-names = "mux";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm-lp";
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			trigger@0 {
+				compatible = "st,stm32-lptimer-trigger";
+				reg = <0>;
+				status = "disabled";
+			};
+
+			counter {
+				compatible = "st,stm32-lptimer-counter";
+				status = "disabled";
+			};
+		};
+
 		usart2: serial@4000e000 {
 			compatible = "st,stm32h7-uart";
 			reg = <0x4000e000 0x400>;
-			interrupts = <GIC_SPI 38 IRQ_TYPE_NONE>;
-			clocks = <&clk_pclk1>;
+			interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc USART2_K>;
 			status = "disabled";
 		};
 
 		usart3: serial@4000f000 {
 			compatible = "st,stm32h7-uart";
 			reg = <0x4000f000 0x400>;
-			interrupts = <GIC_SPI 39 IRQ_TYPE_NONE>;
-			clocks = <&clk_pclk1>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc USART3_K>;
 			status = "disabled";
 		};
 
 		uart4: serial@40010000 {
 			compatible = "st,stm32h7-uart";
 			reg = <0x40010000 0x400>;
-			interrupts = <GIC_SPI 52 IRQ_TYPE_NONE>;
-			clocks = <&clk_pclk1>;
+			interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc UART4_K>;
 			status = "disabled";
 		};
 
 		uart5: serial@40011000 {
 			compatible = "st,stm32h7-uart";
 			reg = <0x40011000 0x400>;
-			interrupts = <GIC_SPI 53 IRQ_TYPE_NONE>;
-			clocks = <&clk_pclk1>;
+			interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc UART5_K>;
+			status = "disabled";
+		};
+
+		i2c1: i2c@40012000 {
+			compatible = "st,stm32f7-i2c";
+			reg = <0x40012000 0x400>;
+			interrupt-names = "event", "error";
+			interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc I2C1_K>;
+			resets = <&rcc I2C1_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c2: i2c@40013000 {
+			compatible = "st,stm32f7-i2c";
+			reg = <0x40013000 0x400>;
+			interrupt-names = "event", "error";
+			interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc I2C2_K>;
+			resets = <&rcc I2C2_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c3: i2c@40014000 {
+			compatible = "st,stm32f7-i2c";
+			reg = <0x40014000 0x400>;
+			interrupt-names = "event", "error";
+			interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc I2C3_K>;
+			resets = <&rcc I2C3_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c5: i2c@40015000 {
+			compatible = "st,stm32f7-i2c";
+			reg = <0x40015000 0x400>;
+			interrupt-names = "event", "error";
+			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc I2C5_K>;
+			resets = <&rcc I2C5_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			status = "disabled";
 		};
 
+		cec: cec@40016000 {
+			compatible = "st,stm32-cec";
+			reg = <0x40016000 0x400>;
+			interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc CEC_K>, <&clk_lse>;
+			clock-names = "cec", "hdmi-cec";
+			status = "disabled";
+		};
+
+		dac: dac@40017000 {
+			compatible = "st,stm32h7-dac-core";
+			reg = <0x40017000 0x400>;
+			clocks = <&rcc DAC12>;
+			clock-names = "pclk";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			dac1: dac@1 {
+				compatible = "st,stm32-dac";
+				#io-channels-cells = <1>;
+				reg = <1>;
+				status = "disabled";
+			};
+
+			dac2: dac@2 {
+				compatible = "st,stm32-dac";
+				#io-channels-cells = <1>;
+				reg = <2>;
+				status = "disabled";
+			};
+		};
+
 		uart7: serial@40018000 {
 			compatible = "st,stm32h7-uart";
 			reg = <0x40018000 0x400>;
-			interrupts = <GIC_SPI 82 IRQ_TYPE_NONE>;
-			clocks = <&clk_pclk1>;
+			interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc UART7_K>;
 			status = "disabled";
 		};
 
 		uart8: serial@40019000 {
 			compatible = "st,stm32h7-uart";
 			reg = <0x40019000 0x400>;
-			interrupts = <GIC_SPI 83 IRQ_TYPE_NONE>;
-			clocks = <&clk_pclk1>;
+			interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc UART8_K>;
 			status = "disabled";
 		};
 
+		timers1: timer@44000000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x44000000 0x400>;
+			clocks = <&rcc TIM1_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+
+			timer@0 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <0>;
+				status = "disabled";
+			};
+		};
+
+		timers8: timer@44001000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x44001000 0x400>;
+			clocks = <&rcc TIM8_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+
+			timer@7 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <7>;
+				status = "disabled";
+			};
+		};
+
 		usart6: serial@44003000 {
 			compatible = "st,stm32h7-uart";
 			reg = <0x44003000 0x400>;
-			interrupts = <GIC_SPI 71 IRQ_TYPE_NONE>;
-			clocks = <&clk_pclk1>;
+			interrupts = <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc USART6_K>;
+			status = "disabled";
+		};
+
+		timers15: timer@44006000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x44006000 0x400>;
+			clocks = <&rcc TIM15_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+
+			timer@14 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <14>;
+				status = "disabled";
+			};
+		};
+
+		timers16: timer@44007000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x44007000 0x400>;
+			clocks = <&rcc TIM16_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+			timer@15 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <15>;
+				status = "disabled";
+			};
+		};
+
+		timers17: timer@44008000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-timers";
+			reg = <0x44008000 0x400>;
+			clocks = <&rcc TIM17_K>;
+			clock-names = "int";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm";
+				status = "disabled";
+			};
+
+			timer@16 {
+				compatible = "st,stm32h7-timer-trigger";
+				reg = <16>;
+				status = "disabled";
+			};
+		};
+
+		dma1: dma@48000000 {
+			compatible = "st,stm32-dma";
+			reg = <0x48000000 0x400>;
+			interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc DMA1>;
+			#dma-cells = <4>;
+			st,mem2mem;
+			dma-requests = <8>;
+		};
+
+		dma2: dma@48001000 {
+			compatible = "st,stm32-dma";
+			reg = <0x48001000 0x400>;
+			interrupts = <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc DMA2>;
+			#dma-cells = <4>;
+			st,mem2mem;
+			dma-requests = <8>;
+		};
+
+		dmamux1: dma-router@48002000 {
+			compatible = "st,stm32h7-dmamux";
+			reg = <0x48002000 0x1c>;
+			#dma-cells = <3>;
+			dma-requests = <128>;
+			dma-masters = <&dma1 &dma2>;
+			dma-channels = <16>;
+			clocks = <&rcc DMAMUX>;
+		};
+
+		rcc: rcc@50000000 {
+			compatible = "st,stm32mp1-rcc", "syscon";
+			reg = <0x50000000 0x1000>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		lptimer2: timer@50021000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-lptimer";
+			reg = <0x50021000 0x400>;
+			clocks = <&rcc LPTIM2_K>;
+			clock-names = "mux";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm-lp";
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			trigger@1 {
+				compatible = "st,stm32-lptimer-trigger";
+				reg = <1>;
+				status = "disabled";
+			};
+
+			counter {
+				compatible = "st,stm32-lptimer-counter";
+				status = "disabled";
+			};
+		};
+
+		lptimer3: timer@50022000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32-lptimer";
+			reg = <0x50022000 0x400>;
+			clocks = <&rcc LPTIM3_K>;
+			clock-names = "mux";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm-lp";
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			trigger@2 {
+				compatible = "st,stm32-lptimer-trigger";
+				reg = <2>;
+				status = "disabled";
+			};
+		};
+
+		lptimer4: timer@50023000 {
+			compatible = "st,stm32-lptimer";
+			reg = <0x50023000 0x400>;
+			clocks = <&rcc LPTIM4_K>;
+			clock-names = "mux";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm-lp";
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+		};
+
+		lptimer5: timer@50024000 {
+			compatible = "st,stm32-lptimer";
+			reg = <0x50024000 0x400>;
+			clocks = <&rcc LPTIM5_K>;
+			clock-names = "mux";
+			status = "disabled";
+
+			pwm {
+				compatible = "st,stm32-pwm-lp";
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+		};
+
+		vrefbuf: vrefbuf@50025000 {
+			compatible = "st,stm32-vrefbuf";
+			reg = <0x50025000 0x8>;
+			regulator-min-microvolt = <1500000>;
+			regulator-max-microvolt = <2500000>;
+			clocks = <&rcc VREF>;
+			status = "disabled";
+		};
+
+		cryp1: cryp@54001000 {
+			compatible = "st,stm32mp1-cryp";
+			reg = <0x54001000 0x400>;
+			interrupts = <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc CRYP1>;
+			resets = <&rcc CRYP1_R>;
+			status = "disabled";
+		};
+
+		rng1: rng@54003000 {
+			compatible = "st,stm32-rng";
+			reg = <0x54003000 0x400>;
+			clocks = <&rcc RNG1_K>;
+			resets = <&rcc RNG1_R>;
+			status = "disabled";
+		};
+
+		mdma1: dma@58000000 {
+			compatible = "st,stm32h7-mdma";
+			reg = <0x58000000 0x1000>;
+			interrupts = <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc MDMA>;
+			#dma-cells = <5>;
+			dma-channels = <32>;
+			dma-requests = <48>;
+		};
+
+		qspi: qspi@58003000 {
+			compatible = "st,stm32f469-qspi";
+			reg = <0x58003000 0x1000>, <0x70000000 0x10000000>;
+			reg-names = "qspi", "qspi_mm";
+			interrupts = <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc QSPI_K>;
+			resets = <&rcc QSPI_R>;
+			status = "disabled";
+		};
+
+		crc1: crc@58009000 {
+			compatible = "st,stm32f7-crc";
+			reg = <0x58009000 0x400>;
+			clocks = <&rcc CRC1>;
+			status = "disabled";
+		};
+
+		usbh_ohci: usbh-ohci@5800c000 {
+			compatible = "generic-ohci";
+			reg = <0x5800c000 0x1000>;
+			clocks = <&rcc USBH>;
+			resets = <&rcc USBH_R>;
+			interrupts = <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		usbh_ehci: usbh-ehci@5800d000 {
+			compatible = "generic-ehci";
+			reg = <0x5800d000 0x1000>;
+			clocks = <&rcc USBH>;
+			resets = <&rcc USBH_R>;
+			interrupts = <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
+			companion = <&usbh_ohci>;
+			status = "disabled";
+		};
+
+		dsi: dsi@5a000000 {
+			compatible = "st,stm32-dsi";
+			reg = <0x5a000000 0x800>;
+			clocks = <&rcc DSI_K>, <&clk_hse>, <&rcc DSI_PX>;
+			clock-names = "pclk", "ref", "px_clk";
+			resets = <&rcc DSI_R>;
+			reset-names = "apb";
+			status = "disabled";
+		};
+
+		ltdc: display-controller@5a001000 {
+			compatible = "st,stm32-ltdc";
+			reg = <0x5a001000 0x400>;
+			interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc LTDC_PX>;
+			clock-names = "lcd";
+			resets = <&rcc LTDC_R>;
 			status = "disabled";
 		};
 
+		usbphyc: usbphyc@5a006000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32mp1-usbphyc";
+			reg = <0x5a006000 0x1000>;
+			clocks = <&rcc USBPHY_K>;
+			resets = <&rcc USBPHY_R>;
+			status = "disabled";
+
+			usbphyc_port0: usb-phy@0 {
+				#phy-cells = <0>;
+				reg = <0>;
+			};
+
+			usbphyc_port1: usb-phy@1 {
+				#phy-cells = <1>;
+				reg = <1>;
+			};
+		};
+
 		usart1: serial@5c000000 {
 			compatible = "st,stm32h7-uart";
 			reg = <0x5c000000 0x400>;
-			interrupts = <GIC_SPI 37 IRQ_TYPE_NONE>;
-			clocks = <&clk_pclk1>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc USART1_K>;
+			status = "disabled";
+		};
+
+		i2c4: i2c@5c002000 {
+			compatible = "st,stm32f7-i2c";
+			reg = <0x5c002000 0x400>;
+			interrupt-names = "event", "error";
+			interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc I2C4_K>;
+			resets = <&rcc I2C4_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c6: i2c@5c009000 {
+			compatible = "st,stm32f7-i2c";
+			reg = <0x5c009000 0x400>;
+			interrupt-names = "event", "error";
+			interrupts = <GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc I2C6_K>;
+			resets = <&rcc I2C6_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			status = "disabled";
 		};
 	};
diff --git a/arch/arm/boot/dts/sun7i-a20-olimex-som-evb-emmc.dts b/arch/arm/boot/dts/sun7i-a20-olimex-som-evb-emmc.dts
new file mode 100644
index 000000000000..81ebc97b76ee
--- /dev/null
+++ b/arch/arm/boot/dts/sun7i-a20-olimex-som-evb-emmc.dts
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Device Tree Source for A20-Olimex-SOM-EVB-eMMC Board
+ *
+ * Copyright (C) 2018 Olimex Ltd.
+ *   Author: Stefan Mavrodiev <stefan@olimex.com>
+ */
+
+/dts-v1/;
+#include "sun7i-a20-olimex-som-evb.dts"
+
+/ {
+
+	model = "Olimex A20-Olimex-SOM-EVB-eMMC";
+	compatible = "olimex,a20-olimex-som-evb-emmc", "allwinner,sun7i-a20";
+
+	mmc2_pwrseq: mmc2_pwrseq {
+		compatible = "mmc-pwrseq-emmc";
+		reset-gpios = <&pio 2 18 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&mmc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc2_pins_a>;
+	vmmc-supply = <&reg_vcc3v3>;
+	mmc-pwrseq = <&mmc2_pwrseq>;
+	bus-width = <4>;
+	non-removable;
+	status = "okay";
+
+	emmc: emmc@0 {
+		reg = <0>;
+		compatible = "mmc-card";
+		broken-hpi;
+	};
+};
diff --git a/arch/arm/boot/dts/sun7i-a20-olimex-som204-evb.dts b/arch/arm/boot/dts/sun7i-a20-olimex-som204-evb.dts
index eae8e267b9ef..3d7b5c848fef 100644
--- a/arch/arm/boot/dts/sun7i-a20-olimex-som204-evb.dts
+++ b/arch/arm/boot/dts/sun7i-a20-olimex-som204-evb.dts
@@ -172,8 +172,7 @@
 	pinctrl-0 = <&mmc0_pins_a>;
 	vmmc-supply = <&reg_vcc3v3>;
 	bus-width = <4>;
-	cd-gpios = <&pio 7 1 GPIO_ACTIVE_HIGH>;
-	cd-inverted;
+	cd-gpios = <&pio 7 1 GPIO_ACTIVE_LOW>;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/sun8i-a23-a33.dtsi b/arch/arm/boot/dts/sun8i-a23-a33.dtsi
index 971f9be699a7..44f3cad3de75 100644
--- a/arch/arm/boot/dts/sun8i-a23-a33.dtsi
+++ b/arch/arm/boot/dts/sun8i-a23-a33.dtsi
@@ -198,6 +198,8 @@
 			clock-names = "ahb", "mod";
 			resets = <&ccu RST_BUS_NAND>;
 			reset-names = "ahb";
+			pinctrl-names = "default";
+			pinctrl-0 = <&nand_pins &nand_pins_cs0 &nand_pins_rb0>;
 			status = "disabled";
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -315,6 +317,37 @@
 				bias-pull-up;
 			};
 
+			nand_pins: nand-pins {
+				pins = "PC0", "PC1", "PC2", "PC5",
+				       "PC8", "PC9", "PC10", "PC11",
+				       "PC12", "PC13", "PC14", "PC15";
+				function = "nand0";
+			};
+
+			nand_pins_cs0: nand-pins-cs0 {
+				pins = "PC4";
+				function = "nand0";
+				bias-pull-up;
+			};
+
+			nand_pins_cs1: nand-pins-cs1 {
+				pins = "PC3";
+				function = "nand0";
+				bias-pull-up;
+			};
+
+			nand_pins_rb0: nand-pins-rb0 {
+				pins = "PC6";
+				function = "nand0";
+				bias-pull-up;
+			};
+
+			nand_pins_rb1: nand-pins-rb1 {
+				pins = "PC7";
+				function = "nand0";
+				bias-pull-up;
+			};
+
 			pwm0_pins: pwm0 {
 				pins = "PH0";
 				function = "pwm0";
diff --git a/arch/arm/boot/dts/sun8i-a33.dtsi b/arch/arm/boot/dts/sun8i-a33.dtsi
index a21f2ed07a52..8d278ee001e9 100644
--- a/arch/arm/boot/dts/sun8i-a33.dtsi
+++ b/arch/arm/boot/dts/sun8i-a33.dtsi
@@ -236,6 +236,11 @@
 					#address-cells = <1>;
 					#size-cells = <0>;
 					reg = <1>;
+
+					tcon0_out_dsi: endpoint@1 {
+						reg = <1>;
+						remote-endpoint = <&dsi_in_tcon0>;
+					};
 				};
 			};
 		};
@@ -280,6 +285,45 @@
 			#io-channel-cells = <0>;
 		};
 
+		dsi: dsi@1ca0000 {
+			compatible = "allwinner,sun6i-a31-mipi-dsi";
+			reg = <0x01ca0000 0x1000>;
+			interrupts = <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_MIPI_DSI>,
+				 <&ccu CLK_DSI_SCLK>;
+			clock-names = "bus", "mod";
+			resets = <&ccu RST_BUS_MIPI_DSI>;
+			phys = <&dphy>;
+			phy-names = "dphy";
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0>;
+
+					dsi_in_tcon0: endpoint {
+						remote-endpoint = <&tcon0_out_dsi>;
+					};
+				};
+			};
+		};
+
+		dphy: d-phy@1ca1000 {
+			compatible = "allwinner,sun6i-a31-mipi-dphy";
+			reg = <0x01ca1000 0x1000>;
+			clocks = <&ccu CLK_BUS_MIPI_DSI>,
+				 <&ccu CLK_DSI_DPHY>;
+			clock-names = "bus", "mod";
+			resets = <&ccu RST_BUS_MIPI_DSI>;
+			status = "disabled";
+			#phy-cells = <0>;
+		};
+
 		fe0: display-frontend@1e00000 {
 			compatible = "allwinner,sun8i-a33-display-frontend";
 			reg = <0x01e00000 0x20000>;
diff --git a/arch/arm/boot/dts/sun8i-a83t.dtsi b/arch/arm/boot/dts/sun8i-a83t.dtsi
index 568307639be8..2be23d600957 100644
--- a/arch/arm/boot/dts/sun8i-a83t.dtsi
+++ b/arch/arm/boot/dts/sun8i-a83t.dtsi
@@ -66,6 +66,8 @@
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			operating-points-v2 = <&cpu0_opp_table>;
+			cci-control-port = <&cci_control0>;
+			enable-method = "allwinner,sun8i-a83t-smp";
 			reg = <0>;
 		};
 
@@ -73,6 +75,8 @@
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			operating-points-v2 = <&cpu0_opp_table>;
+			cci-control-port = <&cci_control0>;
+			enable-method = "allwinner,sun8i-a83t-smp";
 			reg = <1>;
 		};
 
@@ -80,6 +84,8 @@
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			operating-points-v2 = <&cpu0_opp_table>;
+			cci-control-port = <&cci_control0>;
+			enable-method = "allwinner,sun8i-a83t-smp";
 			reg = <2>;
 		};
 
@@ -87,6 +93,8 @@
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			operating-points-v2 = <&cpu0_opp_table>;
+			cci-control-port = <&cci_control0>;
+			enable-method = "allwinner,sun8i-a83t-smp";
 			reg = <3>;
 		};
 
@@ -96,6 +104,8 @@
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			operating-points-v2 = <&cpu1_opp_table>;
+			cci-control-port = <&cci_control1>;
+			enable-method = "allwinner,sun8i-a83t-smp";
 			reg = <0x100>;
 		};
 
@@ -103,6 +113,8 @@
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			operating-points-v2 = <&cpu1_opp_table>;
+			cci-control-port = <&cci_control1>;
+			enable-method = "allwinner,sun8i-a83t-smp";
 			reg = <0x101>;
 		};
 
@@ -110,6 +122,8 @@
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			operating-points-v2 = <&cpu1_opp_table>;
+			cci-control-port = <&cci_control1>;
+			enable-method = "allwinner,sun8i-a83t-smp";
 			reg = <0x102>;
 		};
 
@@ -117,6 +131,8 @@
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			operating-points-v2 = <&cpu1_opp_table>;
+			cci-control-port = <&cci_control1>;
+			enable-method = "allwinner,sun8i-a83t-smp";
 			reg = <0x103>;
 		};
 	};
@@ -349,6 +365,44 @@
 			};
 		};
 
+		cpucfg@1700000 {
+			compatible = "allwinner,sun8i-a83t-cpucfg";
+			reg = <0x01700000 0x400>;
+		};
+
+		cci@1790000 {
+			compatible = "arm,cci-400";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			reg = <0x01790000 0x10000>;
+			ranges = <0x0 0x01790000 0x10000>;
+
+			cci_control0: slave-if@4000 {
+				compatible = "arm,cci-400-ctrl-if";
+				interface-type = "ace";
+				reg = <0x4000 0x1000>;
+			};
+
+			cci_control1: slave-if@5000 {
+				compatible = "arm,cci-400-ctrl-if";
+				interface-type = "ace";
+				reg = <0x5000 0x1000>;
+			};
+
+			pmu@9000 {
+				compatible = "arm,cci-400-pmu,r1";
+				reg = <0x9000 0x5000>;
+				interrupts = <GIC_SPI 132 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 133 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+			};
+		};
+
 		syscon: syscon@1c00000 {
 			compatible = "allwinner,sun8i-a83t-system-controller",
 				"syscon";
@@ -492,6 +546,11 @@
 			#size-cells = <0>;
 		};
 
+		sid: eeprom@1c14000 {
+			compatible = "allwinner,sun8i-a83t-sid";
+			reg = <0x1c14000 0x400>;
+		};
+
 		usb_otg: usb@1c19000 {
 			compatible = "allwinner,sun8i-a83t-musb",
 				     "allwinner,sun8i-a33-musb";
@@ -928,6 +987,11 @@
 			#reset-cells = <1>;
 		};
 
+		r_cpucfg@1f01c00 {
+			compatible = "allwinner,sun8i-a83t-r-cpucfg";
+			reg = <0x1f01c00 0x400>;
+		};
+
 		r_pio: pinctrl@1f02c00 {
 			compatible = "allwinner,sun8i-a83t-r-pinctrl";
 			reg = <0x01f02c00 0x400>;
diff --git a/arch/arm/boot/dts/sun8i-h2-plus-libretech-all-h3-cc.dts b/arch/arm/boot/dts/sun8i-h2-plus-libretech-all-h3-cc.dts
new file mode 100644
index 000000000000..4db0d4bb65eb
--- /dev/null
+++ b/arch/arm/boot/dts/sun8i-h2-plus-libretech-all-h3-cc.dts
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2018 Chen-Yu Tsai <wens@csie.org>
+ */
+
+/dts-v1/;
+#include "sun8i-h3.dtsi"
+#include "sunxi-libretech-all-h3-cc.dtsi"
+
+/ {
+	model = "Libre Computer Board ALL-H3-CC H2+";
+	compatible = "libretech,all-h3-cc-h2-plus", "allwinner,sun8i-h2-plus";
+};
diff --git a/arch/arm/boot/dts/sun8i-h2-plus-orangepi-zero.dts b/arch/arm/boot/dts/sun8i-h2-plus-orangepi-zero.dts
index 0bc031fe4c56..84cd9c061227 100644
--- a/arch/arm/boot/dts/sun8i-h2-plus-orangepi-zero.dts
+++ b/arch/arm/boot/dts/sun8i-h2-plus-orangepi-zero.dts
@@ -89,6 +89,23 @@
 		gpio = <&pio 0 20 GPIO_ACTIVE_HIGH>;
 	};
 
+	reg_vdd_cpux: vdd-cpux-regulator {
+		compatible = "regulator-gpio";
+		regulator-name = "vdd-cpux";
+		regulator-type = "voltage";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <1100000>;
+		regulator-max-microvolt = <1300000>;
+		regulator-ramp-delay = <50>; /* 4ms */
+
+		gpios = <&r_pio 0 6 GPIO_ACTIVE_HIGH>; /* PL6 */
+		enable-active-high;
+		gpios-states = <1>;
+		states = <1100000 0
+			  1300000 1>;
+	};
+
 	wifi_pwrseq: wifi_pwrseq {
 		compatible = "mmc-pwrseq-simple";
 		reset-gpios = <&r_pio 0 7 GPIO_ACTIVE_LOW>;
@@ -96,6 +113,10 @@
 	};
 };
 
+&cpu0 {
+	cpu-supply = <&reg_vdd_cpux>;
+};
+
 &ehci0 {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dts b/arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dts
index b20a710da7bc..a8b2f0f1c11d 100644
--- a/arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dts
+++ b/arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dts
@@ -6,213 +6,9 @@
 
 /dts-v1/;
 #include "sun8i-h3.dtsi"
-
-#include <dt-bindings/gpio/gpio.h>
-#include <dt-bindings/input/input.h>
+#include "sunxi-libretech-all-h3-cc.dtsi"
 
 / {
 	model = "Libre Computer Board ALL-H3-CC H3";
 	compatible = "libretech,all-h3-cc-h3", "allwinner,sun8i-h3";
-
-	aliases {
-		ethernet0 = &emac;
-		serial0 = &uart0;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-
-	connector {
-		compatible = "hdmi-connector";
-		type = "a";
-
-		port {
-			hdmi_con_in: endpoint {
-				remote-endpoint = <&hdmi_out_con>;
-			};
-		};
-	};
-
-	leds {
-		compatible = "gpio-leds";
-
-		pwr_led {
-			label = "librecomputer:green:pwr";
-			gpios = <&r_pio 0 10 GPIO_ACTIVE_HIGH>; /* PL10 */
-			default-state = "on";
-		};
-
-		status_led {
-			label = "librecomputer:blue:status";
-			gpios = <&pio 0 7 GPIO_ACTIVE_HIGH>; /* PA7 */
-		};
-	};
-
-	gpio_keys {
-		compatible = "gpio-keys";
-
-		power {
-			label = "power";
-			linux,code = <KEY_POWER>;
-			gpios = <&r_pio 0 2 GPIO_ACTIVE_LOW>; /* PL2 */
-		};
-	};
-
-	reg_vcc1v2: vcc1v2 {
-		compatible = "regulator-fixed";
-		regulator-name = "vcc1v2";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		regulator-always-on;
-		regulator-boot-on;
-		vin-supply = <&reg_vcc5v0>;
-		gpio = <&r_pio 0 8 GPIO_ACTIVE_HIGH>; /* PL8 */
-		enable-active-high;
-	};
-
-	reg_vcc3v3: vcc3v3 {
-		compatible = "regulator-fixed";
-		regulator-name = "vcc3v3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&reg_vcc5v0>;
-	};
-
-	/* This represents the board's 5V input */
-	reg_vcc5v0: vcc5v0 {
-		compatible = "regulator-fixed";
-		regulator-name = "vcc5v0";
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-	};
-
-	reg_vcc_dram: vcc-dram {
-		compatible = "regulator-fixed";
-		regulator-name = "vcc-dram";
-		regulator-min-microvolt = <1500000>;
-		regulator-max-microvolt = <1500000>;
-		regulator-always-on;
-		regulator-boot-on;
-		vin-supply = <&reg_vcc5v0>;
-		gpio = <&r_pio 0 9 GPIO_ACTIVE_HIGH>; /* PL9 */
-		enable-active-high;
-	};
-
-	reg_vcc_io: vcc-io {
-		compatible = "regulator-fixed";
-		regulator-name = "vcc-io";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		regulator-always-on;
-		regulator-boot-on;
-		vin-supply = <&reg_vcc3v3>;
-		gpio = <&r_pio 0 5 GPIO_ACTIVE_LOW>; /* PL5 */
-	};
-
-	reg_vdd_cpux: vdd-cpux {
-		compatible = "regulator-fixed";
-		regulator-name = "vdd-cpux";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		regulator-always-on;
-		regulator-boot-on;
-		vin-supply = <&reg_vcc5v0>;
-		gpio = <&r_pio 0 8 GPIO_ACTIVE_HIGH>; /* PL8 */
-		enable-active-high;
-	};
-};
-
-&codec {
-	allwinner,audio-routing =
-		"Line Out", "LINEOUT",
-		"MIC1", "Mic",
-		"Mic",  "MBIAS";
-	status = "okay";
-};
-
-&de {
-	status = "okay";
-};
-
-&ehci0 {
-	status = "okay";
-};
-
-&ehci1 {
-	status = "okay";
-};
-
-&ehci2 {
-	status = "okay";
-};
-
-&ehci3 {
-	status = "okay";
-};
-
-&emac {
-	phy-handle = <&int_mii_phy>;
-	phy-mode = "mii";
-	allwinner,leds-active-low;
-	status = "okay";
-};
-
-&hdmi {
-	status = "okay";
-};
-
-&hdmi_out {
-	hdmi_out_con: endpoint {
-		remote-endpoint = <&hdmi_con_in>;
-	};
-};
-
-&ir {
-	pinctrl-names = "default";
-	pinctrl-0 = <&ir_pins_a>;
-	status = "okay";
-};
-
-&mmc0 {
-	vmmc-supply = <&reg_vcc_io>;
-	bus-width = <4>;
-	cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>; /* PF6 */
-	status = "okay";
-};
-
-&ohci0 {
-	status = "okay";
-};
-
-&ohci1 {
-	status = "okay";
-};
-
-&ohci2 {
-	status = "okay";
-};
-
-&ohci3 {
-	status = "okay";
-};
-
-&uart0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&uart0_pins_a>;
-	status = "okay";
-};
-
-&usb_otg {
-	dr_mode = "host";
-	status = "okay";
-};
-
-&usbphy {
-	/* VBUS on USB ports are always on */
-	usb0_vbus-supply = <&reg_vcc5v0>;
-	usb1_vbus-supply = <&reg_vcc5v0>;
-	usb2_vbus-supply = <&reg_vcc5v0>;
-	usb3_vbus-supply = <&reg_vcc5v0>;
-	status = "okay";
 };
diff --git a/arch/arm/boot/dts/sun8i-h3-orangepi-one.dts b/arch/arm/boot/dts/sun8i-h3-orangepi-one.dts
index 232f124ce62c..245fd658defb 100644
--- a/arch/arm/boot/dts/sun8i-h3-orangepi-one.dts
+++ b/arch/arm/boot/dts/sun8i-h3-orangepi-one.dts
@@ -99,6 +99,27 @@
 			gpios = <&r_pio 0 3 GPIO_ACTIVE_LOW>;
 		};
 	};
+
+	reg_vdd_cpux: vdd-cpux-regulator {
+		compatible = "regulator-gpio";
+		regulator-name = "vdd-cpux";
+		regulator-type = "voltage";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <1100000>;
+		regulator-max-microvolt = <1300000>;
+		regulator-ramp-delay = <50>; /* 4ms */
+
+		gpios = <&r_pio 0 6 GPIO_ACTIVE_HIGH>; /* PL6 */
+		enable-active-high;
+		gpios-states = <0x1>;
+		states = <1100000 0x0
+			  1300000 0x1>;
+	};
+};
+
+&cpu0 {
+	cpu-supply = <&reg_vdd_cpux>;
 };
 
 &de {
diff --git a/arch/arm/boot/dts/sun8i-h3-orangepi-pc.dts b/arch/arm/boot/dts/sun8i-h3-orangepi-pc.dts
index cea4d647ecbf..46240334128f 100644
--- a/arch/arm/boot/dts/sun8i-h3-orangepi-pc.dts
+++ b/arch/arm/boot/dts/sun8i-h3-orangepi-pc.dts
@@ -113,6 +113,10 @@
 	status = "okay";
 };
 
+&cpu0 {
+	cpu-supply = <&reg_vdd_cpux>;
+};
+
 &ehci0 {
 	status = "okay";
 };
@@ -182,6 +186,30 @@
 	};
 };
 
+&r_i2c {
+	status = "okay";
+
+	reg_vdd_cpux: regulator@65 {
+		compatible = "silergy,sy8106a";
+		reg = <0x65>;
+		regulator-name = "vdd-cpux";
+		silergy,fixed-microvolt = <1200000>;
+		/*
+		 * The datasheet uses 1.1V as the minimum value of VDD-CPUX,
+		 * however both the Armbian DVFS table and the official one
+		 * have operating points with voltage under 1.1V, and both
+		 * DVFS table are known to work properly at the lowest
+		 * operating point.
+		 *
+		 * Use 1.0V as the minimum voltage instead.
+		 */
+		regulator-min-microvolt = <1000000>;
+		regulator-max-microvolt = <1300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+};
+
 &r_pio {
 	leds_r_opc: led_pins {
 		pins = "PL10";
diff --git a/arch/arm/boot/dts/sun8i-h3.dtsi b/arch/arm/boot/dts/sun8i-h3.dtsi
index 10da8ed7db81..41d57c76f290 100644
--- a/arch/arm/boot/dts/sun8i-h3.dtsi
+++ b/arch/arm/boot/dts/sun8i-h3.dtsi
@@ -43,32 +43,62 @@
 #include "sunxi-h3-h5.dtsi"
 
 / {
+	cpu0_opp_table: opp_table0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp@648000000 {
+			opp-hz = /bits/ 64 <648000000>;
+			opp-microvolt = <1040000 1040000 1300000>;
+			clock-latency-ns = <244144>; /* 8 32k periods */
+		};
+
+		opp@816000000 {
+			opp-hz = /bits/ 64 <816000000>;
+			opp-microvolt = <1100000 1100000 1300000>;
+			clock-latency-ns = <244144>; /* 8 32k periods */
+		};
+
+		opp@1008000000 {
+			opp-hz = /bits/ 64 <1008000000>;
+			opp-microvolt = <1200000 1200000 1300000>;
+			clock-latency-ns = <244144>; /* 8 32k periods */
+		};
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		cpu@0 {
+		cpu0: cpu@0 {
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			reg = <0>;
+			clocks = <&ccu CLK_CPUX>;
+			clock-names = "cpu";
+			operating-points-v2 = <&cpu0_opp_table>;
+			#cooling-cells = <2>;
 		};
 
 		cpu@1 {
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			reg = <1>;
+			operating-points-v2 = <&cpu0_opp_table>;
 		};
 
 		cpu@2 {
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			reg = <2>;
+			operating-points-v2 = <&cpu0_opp_table>;
 		};
 
 		cpu@3 {
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			reg = <3>;
+			operating-points-v2 = <&cpu0_opp_table>;
 		};
 	};
 
diff --git a/arch/arm/boot/dts/sun8i-r16-nintendo-nes-classic.dts b/arch/arm/boot/dts/sun8i-r16-nintendo-nes-classic.dts
new file mode 100644
index 000000000000..fc0658cfa319
--- /dev/null
+++ b/arch/arm/boot/dts/sun8i-r16-nintendo-nes-classic.dts
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0 OR X11
+/* Copyright (c) 2016 FUKAUMI Naoki <naobsd@gmail.com> */
+
+/dts-v1/;
+#include "sun8i-a33.dtsi"
+#include "sunxi-common-regulators.dtsi"
+
+/ {
+	model = "Nintendo NES Classic Edition";
+	compatible = "nintendo,nes-classic", "allwinner,sun8i-r16",
+		     "allwinner,sun8i-a33";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+};
+
+&uart0 {
+	/*
+	 * UART0 is available on two ports: PB and PF, both are accessible.
+	 * PF can also be used for the SD card so PB is preferred.
+	 */
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pins_a>;
+	status = "okay";
+};
+
+&nfc {
+	status = "okay";
+
+	/* 2Gb Macronix MX30LF2G18AC (3V) */
+	nand@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		reg = <0>;
+		allwinner,rb = <0>;
+		nand-ecc-mode = "hw";
+		nand-ecc-strength = <16>;
+		nand-ecc-step-size = <1024>;
+	};
+};
+
+&usb_otg {
+	status = "okay";
+	dr_mode = "otg";
+};
+
+&usbphy {
+	/* VBUS is always on because it is wired to the power supply */
+	usb1_vbus-supply = <&reg_vcc5v0>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/sun8i-r16-nintendo-super-nes-classic.dts b/arch/arm/boot/dts/sun8i-r16-nintendo-super-nes-classic.dts
new file mode 100644
index 000000000000..80761d7904ec
--- /dev/null
+++ b/arch/arm/boot/dts/sun8i-r16-nintendo-super-nes-classic.dts
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0 OR X11
+/* Copyright (c) 2018 Miquèl RAYNAL <miquel.raynal@bootlin.com> */
+
+/dts-v1/;
+#include "sun8i-r16-nintendo-nes-classic.dts"
+
+/ {
+	model = "Nintendo SuperNES Classic Edition";
+	compatible = "nintendo,super-nes-classic", "nintendo,nes-classic",
+		     "allwinner,sun8i-r16", "allwinner,sun8i-a33";
+};
diff --git a/arch/arm/boot/dts/sun8i-r40-bananapi-m2-ultra.dts b/arch/arm/boot/dts/sun8i-r40-bananapi-m2-ultra.dts
index 27d9ccd0ef2f..25fb048c7df2 100644
--- a/arch/arm/boot/dts/sun8i-r40-bananapi-m2-ultra.dts
+++ b/arch/arm/boot/dts/sun8i-r40-bananapi-m2-ultra.dts
@@ -51,6 +51,7 @@
 	compatible = "sinovoip,bpi-m2-ultra", "allwinner,sun8i-r40";
 
 	aliases {
+		ethernet0 = &gmac;
 		serial0 = &uart0;
 	};
 
@@ -101,6 +102,22 @@
 	status = "okay";
 };
 
+&gmac {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac_rgmii_pins>;
+	phy-handle = <&phy1>;
+	phy-mode = "rgmii";
+	phy-supply = <&reg_dc1sw>;
+	status = "okay";
+};
+
+&gmac_mdio {
+	phy1: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <1>;
+	};
+};
+
 &i2c0 {
 	status = "okay";
 
@@ -114,6 +131,48 @@
 
 #include "axp22x.dtsi"
 
+&mmc0 {
+	vmmc-supply = <&reg_dcdc1>;
+	bus-width = <4>;
+	cd-gpios = <&pio 7 13 GPIO_ACTIVE_HIGH>; /* PH13 */
+	cd-inverted;
+	status = "okay";
+};
+
+&mmc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc1_pg_pins>;
+	vmmc-supply = <&reg_dldo2>;
+	vqmmc-supply = <&reg_dldo1>;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	bus-width = <4>;
+	non-removable;
+	status = "okay";
+};
+
+&mmc2 {
+	vmmc-supply = <&reg_dcdc1>;
+	vqmmc-supply = <&reg_dcdc1>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+&ohci1 {
+	status = "okay";
+};
+
+&ohci2 {
+	status = "okay";
+};
+
+&reg_aldo2 {
+	regulator-always-on;
+	regulator-min-microvolt = <2500000>;
+	regulator-max-microvolt = <2500000>;
+	regulator-name = "vcc-pa";
+};
+
 &reg_aldo3 {
 	regulator-always-on;
 	regulator-min-microvolt = <2700000>;
@@ -121,6 +180,12 @@
 	regulator-name = "avcc";
 };
 
+&reg_dc1sw {
+	regulator-min-microvolt = <3000000>;
+	regulator-max-microvolt = <3000000>;
+	regulator-name = "vcc-gmac-phy";
+};
+
 &reg_dcdc1 {
 	regulator-always-on;
 	regulator-min-microvolt = <3000000>;
@@ -161,40 +226,6 @@
 	regulator-name = "vcc-wifi";
 };
 
-&mmc0 {
-	vmmc-supply = <&reg_dcdc1>;
-	bus-width = <4>;
-	cd-gpios = <&pio 7 13 GPIO_ACTIVE_LOW>; /* PH13 */
-	status = "okay";
-};
-
-&mmc1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&mmc1_pg_pins>;
-	vmmc-supply = <&reg_dldo2>;
-	vqmmc-supply = <&reg_dldo1>;
-	mmc-pwrseq = <&wifi_pwrseq>;
-	bus-width = <4>;
-	non-removable;
-	status = "okay";
-};
-
-&mmc2 {
-	vmmc-supply = <&reg_dcdc1>;
-	vqmmc-supply = <&reg_dcdc1>;
-	bus-width = <8>;
-	non-removable;
-	status = "okay";
-};
-
-&ohci1 {
-	status = "okay";
-};
-
-&ohci2 {
-	status = "okay";
-};
-
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_pb_pins>;
diff --git a/arch/arm/boot/dts/sun8i-r40.dtsi b/arch/arm/boot/dts/sun8i-r40.dtsi
index 173dcc1652d2..bd97ca3dc2fa 100644
--- a/arch/arm/boot/dts/sun8i-r40.dtsi
+++ b/arch/arm/boot/dts/sun8i-r40.dtsi
@@ -265,6 +265,19 @@
 			#interrupt-cells = <3>;
 			#gpio-cells = <3>;
 
+			gmac_rgmii_pins: gmac-rgmii-pins {
+				pins = "PA0", "PA1", "PA2", "PA3",
+				       "PA4", "PA5", "PA6", "PA7",
+				       "PA8", "PA10", "PA11", "PA12",
+				       "PA13", "PA15", "PA16";
+				function = "gmac";
+				/*
+				 * data lines in RGMII mode use DDR mode
+				 * and need a higher signal drive strength
+				 */
+				drive-strength = <40>;
+			};
+
 			i2c0_pins: i2c0-pins {
 				pins = "PB0", "PB1";
 				function = "i2c0";
@@ -451,6 +464,27 @@
 			#size-cells = <0>;
 		};
 
+		gmac: ethernet@1c50000 {
+			compatible = "allwinner,sun8i-r40-gmac";
+			syscon = <&ccu>;
+			reg = <0x01c50000 0x10000>;
+			interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq";
+			resets = <&ccu RST_BUS_GMAC>;
+			reset-names = "stmmaceth";
+			clocks = <&ccu CLK_BUS_GMAC>;
+			clock-names = "stmmaceth";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			gmac_mdio: mdio {
+				compatible = "snps,dwmac-mdio";
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
 		gic: interrupt-controller@1c81000 {
 			compatible = "arm,gic-400";
 			reg = <0x01c81000 0x1000>,
diff --git a/arch/arm/boot/dts/sun8i-v40-bananapi-m2-berry.dts b/arch/arm/boot/dts/sun8i-v40-bananapi-m2-berry.dts
index a26d72c3f9b5..35859d8f3267 100644
--- a/arch/arm/boot/dts/sun8i-v40-bananapi-m2-berry.dts
+++ b/arch/arm/boot/dts/sun8i-v40-bananapi-m2-berry.dts
@@ -87,6 +87,11 @@
 	};
 };
 
+&ehci1 {
+	/* Terminus Tech FE 1.1s 4-port USB 2.0 hub here */
+	status = "okay";
+};
+
 &i2c0 {
 	status = "okay";
 
@@ -170,3 +175,8 @@
 	pinctrl-0 = <&uart0_pb_pins>;
 	status = "okay";
 };
+
+&usbphy {
+	usb1_vbus-supply = <&reg_vcc5v0>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/sunxi-h3-h5.dtsi b/arch/arm/boot/dts/sunxi-h3-h5.dtsi
index 1be1a02d6df2..c3bff1105e5d 100644
--- a/arch/arm/boot/dts/sunxi-h3-h5.dtsi
+++ b/arch/arm/boot/dts/sunxi-h3-h5.dtsi
@@ -822,6 +822,19 @@
 			status = "disabled";
 		};
 
+		r_i2c: i2c@1f02400 {
+			compatible = "allwinner,sun6i-a31-i2c";
+			reg = <0x01f02400 0x400>;
+			interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&r_i2c_pins>;
+			clocks = <&r_ccu CLK_APB0_I2C>;
+			resets = <&r_ccu RST_APB0_I2C>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
 		r_pio: pinctrl@1f02c00 {
 			compatible = "allwinner,sun8i-h3-r-pinctrl";
 			reg = <0x01f02c00 0x400>;
@@ -837,6 +850,11 @@
 				pins = "PL11";
 				function = "s_cir_rx";
 			};
+
+			r_i2c_pins: r-i2c {
+				pins = "PL0", "PL1";
+				function = "s_i2c";
+			};
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/sunxi-libretech-all-h3-cc.dtsi b/arch/arm/boot/dts/sunxi-libretech-all-h3-cc.dtsi
new file mode 100644
index 000000000000..f7ffdd6658a2
--- /dev/null
+++ b/arch/arm/boot/dts/sunxi-libretech-all-h3-cc.dtsi
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2017 Chen-Yu Tsai <wens@csie.org>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/ {
+	aliases {
+		ethernet0 = &emac;
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		pwr_led {
+			label = "librecomputer:green:pwr";
+			gpios = <&r_pio 0 10 GPIO_ACTIVE_HIGH>; /* PL10 */
+			default-state = "on";
+		};
+
+		status_led {
+			label = "librecomputer:blue:status";
+			gpios = <&pio 0 7 GPIO_ACTIVE_HIGH>; /* PA7 */
+		};
+	};
+
+	gpio_keys {
+		compatible = "gpio-keys";
+
+		power {
+			label = "power";
+			linux,code = <KEY_POWER>;
+			gpios = <&r_pio 0 2 GPIO_ACTIVE_LOW>; /* PL2 */
+		};
+	};
+
+	reg_vcc1v2: vcc1v2 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v2";
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&reg_vcc5v0>;
+		gpio = <&r_pio 0 8 GPIO_ACTIVE_HIGH>; /* PL8 */
+		enable-active-high;
+	};
+
+	reg_vcc3v3: vcc3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&reg_vcc5v0>;
+	};
+
+	/* This represents the board's 5V input */
+	reg_vcc5v0: vcc5v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	reg_vcc_dram: vcc-dram {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc-dram";
+		regulator-min-microvolt = <1500000>;
+		regulator-max-microvolt = <1500000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&reg_vcc5v0>;
+		gpio = <&r_pio 0 9 GPIO_ACTIVE_HIGH>; /* PL9 */
+		enable-active-high;
+	};
+
+	reg_vcc_io: vcc-io {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc-io";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&reg_vcc3v3>;
+		gpio = <&r_pio 0 5 GPIO_ACTIVE_LOW>; /* PL5 */
+	};
+
+	reg_vdd_cpux: vdd-cpux {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd-cpux";
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&reg_vcc5v0>;
+		gpio = <&r_pio 0 8 GPIO_ACTIVE_HIGH>; /* PL8 */
+		enable-active-high;
+	};
+};
+
+&codec {
+	allwinner,audio-routing =
+		"Line Out", "LINEOUT",
+		"MIC1", "Mic",
+		"Mic",  "MBIAS";
+	status = "okay";
+};
+
+&cpu0 {
+	cpu-supply = <&reg_vdd_cpux>;
+};
+
+&de {
+	status = "okay";
+};
+
+&ehci0 {
+	status = "okay";
+};
+
+&ehci1 {
+	status = "okay";
+};
+
+&ehci2 {
+	status = "okay";
+};
+
+&ehci3 {
+	status = "okay";
+};
+
+&emac {
+	phy-handle = <&int_mii_phy>;
+	phy-mode = "mii";
+	allwinner,leds-active-low;
+	status = "okay";
+};
+
+&hdmi {
+	status = "okay";
+};
+
+&hdmi_out {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
+&ir {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ir_pins_a>;
+	status = "okay";
+};
+
+&mmc0 {
+	vmmc-supply = <&reg_vcc_io>;
+	bus-width = <4>;
+	cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>; /* PF6 */
+	status = "okay";
+};
+
+&ohci0 {
+	status = "okay";
+};
+
+&ohci1 {
+	status = "okay";
+};
+
+&ohci2 {
+	status = "okay";
+};
+
+&ohci3 {
+	status = "okay";
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pins_a>;
+	status = "okay";
+};
+
+&usb_otg {
+	dr_mode = "host";
+	status = "okay";
+};
+
+&usbphy {
+	/* VBUS on USB ports are always on */
+	usb0_vbus-supply = <&reg_vcc5v0>;
+	usb1_vbus-supply = <&reg_vcc5v0>;
+	usb2_vbus-supply = <&reg_vcc5v0>;
+	usb3_vbus-supply = <&reg_vcc5v0>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/tegra114.dtsi b/arch/arm/boot/dts/tegra114.dtsi
index 0e4a13295d8a..84c4358dacac 100644
--- a/arch/arm/boot/dts/tegra114.dtsi
+++ b/arch/arm/boot/dts/tegra114.dtsi
@@ -19,6 +19,7 @@
 		clocks = <&tegra_car TEGRA114_CLK_HOST1X>;
 		resets = <&tegra_car 28>;
 		reset-names = "host1x";
+		iommus = <&mc TEGRA_SWGROUP_HC>;
 
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -32,6 +33,8 @@
 			clocks = <&tegra_car TEGRA114_CLK_GR2D>;
 			resets = <&tegra_car 21>;
 			reset-names = "2d";
+
+			iommus = <&mc TEGRA_SWGROUP_G2>;
 		};
 
 		gr3d@54180000 {
@@ -40,6 +43,8 @@
 			clocks = <&tegra_car TEGRA114_CLK_GR3D>;
 			resets = <&tegra_car 24>;
 			reset-names = "3d";
+
+			iommus = <&mc TEGRA_SWGROUP_NV>;
 		};
 
 		dc@54200000 {
diff --git a/arch/arm/boot/dts/tegra124-apalis-v1.2.dtsi b/arch/arm/boot/dts/tegra124-apalis-v1.2.dtsi
index bb67edb016c5..3455822350c5 100644
--- a/arch/arm/boot/dts/tegra124-apalis-v1.2.dtsi
+++ b/arch/arm/boot/dts/tegra124-apalis-v1.2.dtsi
@@ -1536,15 +1536,15 @@
 	};
 
 	serial@70006040 {
-		compatible = "nvidia,tegra124-hsuart";
+		compatible = "nvidia,tegra124-hsuart", "nvidia,tegra30-hsuart";
 	};
 
 	serial@70006200 {
-		compatible = "nvidia,tegra124-hsuart";
+		compatible = "nvidia,tegra124-hsuart", "nvidia,tegra30-hsuart";
 	};
 
 	serial@70006300 {
-		compatible = "nvidia,tegra124-hsuart";
+		compatible = "nvidia,tegra124-hsuart", "nvidia,tegra30-hsuart";
 	};
 
 	hdmi_ddc: i2c@7000c700 {
diff --git a/arch/arm/boot/dts/tegra124-apalis.dtsi b/arch/arm/boot/dts/tegra124-apalis.dtsi
index 65a2161b9b8e..9f960c84ba10 100644
--- a/arch/arm/boot/dts/tegra124-apalis.dtsi
+++ b/arch/arm/boot/dts/tegra124-apalis.dtsi
@@ -1565,15 +1565,15 @@
 	};
 
 	serial@70006040 {
-		compatible = "nvidia,tegra124-hsuart";
+		compatible = "nvidia,tegra124-hsuart", "nvidia,tegra30-hsuart";
 	};
 
 	serial@70006200 {
-		compatible = "nvidia,tegra124-hsuart";
+		compatible = "nvidia,tegra124-hsuart", "nvidia,tegra30-hsuart";
 	};
 
 	serial@70006300 {
-		compatible = "nvidia,tegra124-hsuart";
+		compatible = "nvidia,tegra124-hsuart", "nvidia,tegra30-hsuart";
 	};
 
 	hdmi_ddc: i2c@7000c400 {
diff --git a/arch/arm/boot/dts/tegra30.dtsi b/arch/arm/boot/dts/tegra30.dtsi
index a110cf84d85f..09087b9c5e26 100644
--- a/arch/arm/boot/dts/tegra30.dtsi
+++ b/arch/arm/boot/dts/tegra30.dtsi
@@ -112,6 +112,7 @@
 		clocks = <&tegra_car TEGRA30_CLK_HOST1X>;
 		resets = <&tegra_car 28>;
 		reset-names = "host1x";
+		iommus = <&mc TEGRA_SWGROUP_HC>;
 
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -125,6 +126,8 @@
 			clocks = <&tegra_car TEGRA30_CLK_MPE>;
 			resets = <&tegra_car 60>;
 			reset-names = "mpe";
+
+			iommus = <&mc TEGRA_SWGROUP_MPE>;
 		};
 
 		vi@54080000 {
@@ -134,6 +137,8 @@
 			clocks = <&tegra_car TEGRA30_CLK_VI>;
 			resets = <&tegra_car 20>;
 			reset-names = "vi";
+
+			iommus = <&mc TEGRA_SWGROUP_VI>;
 		};
 
 		epp@540c0000 {
@@ -143,6 +148,8 @@
 			clocks = <&tegra_car TEGRA30_CLK_EPP>;
 			resets = <&tegra_car 19>;
 			reset-names = "epp";
+
+			iommus = <&mc TEGRA_SWGROUP_EPP>;
 		};
 
 		isp@54100000 {
@@ -152,6 +159,8 @@
 			clocks = <&tegra_car TEGRA30_CLK_ISP>;
 			resets = <&tegra_car 23>;
 			reset-names = "isp";
+
+			iommus = <&mc TEGRA_SWGROUP_ISP>;
 		};
 
 		gr2d@54140000 {
@@ -161,6 +170,8 @@
 			clocks = <&tegra_car TEGRA30_CLK_GR2D>;
 			resets = <&tegra_car 21>;
 			reset-names = "2d";
+
+			iommus = <&mc TEGRA_SWGROUP_G2>;
 		};
 
 		gr3d@54180000 {
@@ -172,6 +183,9 @@
 			resets = <&tegra_car 24>,
 				 <&tegra_car 98>;
 			reset-names = "3d", "3d2";
+
+			iommus = <&mc TEGRA_SWGROUP_NV>,
+				 <&mc TEGRA_SWGROUP_NV2>;
 		};
 
 		dc@54200000 {
diff --git a/arch/arm/boot/dts/uniphier-pro4.dtsi b/arch/arm/boot/dts/uniphier-pro4.dtsi
index 844124bc9c9c..49539f035219 100644
--- a/arch/arm/boot/dts/uniphier-pro4.dtsi
+++ b/arch/arm/boot/dts/uniphier-pro4.dtsi
@@ -286,7 +286,7 @@
 			has-transaction-translator;
 		};
 
-		soc-glue@5f800000 {
+		soc_glue: soc-glue@5f800000 {
 			compatible = "socionext,uniphier-pro4-soc-glue",
 				     "simple-mfd", "syscon";
 			reg = <0x5f800000 0x2000>;
@@ -371,10 +371,14 @@
 			interrupts = <0 66 4>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_ether_rgmii>;
-			clocks = <&sys_clk 6>;
-			resets = <&sys_rst 6>;
+			clock-names = "gio", "ether", "ether-gb", "ether-phy";
+			clocks = <&sys_clk 12>, <&sys_clk 6>, <&sys_clk 7>,
+				 <&sys_clk 10>;
+			reset-names = "gio", "ether";
+			resets = <&sys_rst 12>, <&sys_rst 6>;
 			phy-mode = "rgmii";
 			local-mac-address = [00 00 00 00 00 00];
+			socionext,syscon-phy-mode = <&soc_glue 0>;
 
 			mdio: mdio {
 				#address-cells = <1>;
diff --git a/arch/arm/boot/dts/uniphier-pxs2.dtsi b/arch/arm/boot/dts/uniphier-pxs2.dtsi
index debcbd15c24b..641d96119d4f 100644
--- a/arch/arm/boot/dts/uniphier-pxs2.dtsi
+++ b/arch/arm/boot/dts/uniphier-pxs2.dtsi
@@ -506,10 +506,13 @@
 			interrupts = <0 66 4>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_ether_rgmii>;
+			clock-names = "ether";
 			clocks = <&sys_clk 6>;
+			reset-names = "ether";
 			resets = <&sys_rst 6>;
 			phy-mode = "rgmii";
 			local-mac-address = [00 00 00 00 00 00];
+			socionext,syscon-phy-mode = <&soc_glue 0>;
 
 			mdio: mdio {
 				#address-cells = <1>;
diff --git a/arch/arm/boot/dts/vexpress-v2m-rs1.dtsi b/arch/arm/boot/dts/vexpress-v2m-rs1.dtsi
index 35714ff6f467..4488c8fe213a 100644
--- a/arch/arm/boot/dts/vexpress-v2m-rs1.dtsi
+++ b/arch/arm/boot/dts/vexpress-v2m-rs1.dtsi
@@ -17,426 +17,436 @@
  * CHANGES TO vexpress-v2m.dtsi!
  */
 
-	motherboard {
-		model = "V2M-P1";
-		arm,hbi = <0x190>;
-		arm,vexpress,site = <0>;
-		arm,v2m-memory-map = "rs1";
-		compatible = "arm,vexpress,v2m-p1", "simple-bus";
-		#address-cells = <2>; /* SMB chipselect number and offset */
-		#size-cells = <1>;
-		#interrupt-cells = <1>;
-		ranges;
-
-		flash@0,00000000 {
-			compatible = "arm,vexpress-flash", "cfi-flash";
-			reg = <0 0x00000000 0x04000000>,
-			      <4 0x00000000 0x04000000>;
-			bank-width = <4>;
-		};
+/ {
+	smb@8000000 {
+		motherboard {
+			model = "V2M-P1";
+			arm,hbi = <0x190>;
+			arm,vexpress,site = <0>;
+			arm,v2m-memory-map = "rs1";
+			compatible = "arm,vexpress,v2m-p1", "simple-bus";
+			#address-cells = <2>; /* SMB chipselect number and offset */
+			#size-cells = <1>;
+			#interrupt-cells = <1>;
+			ranges;
 
-		psram@1,00000000 {
-			compatible = "arm,vexpress-psram", "mtd-ram";
-			reg = <1 0x00000000 0x02000000>;
-			bank-width = <4>;
-		};
+			flash@0,00000000 {
+				compatible = "arm,vexpress-flash", "cfi-flash";
+				reg = <0 0x00000000 0x04000000>,
+				      <4 0x00000000 0x04000000>;
+				bank-width = <4>;
+			};
 
-		v2m_video_ram: vram@2,00000000 {
-			compatible = "arm,vexpress-vram";
-			reg = <2 0x00000000 0x00800000>;
-		};
+			psram@1,00000000 {
+				compatible = "arm,vexpress-psram", "mtd-ram";
+				reg = <1 0x00000000 0x02000000>;
+				bank-width = <4>;
+			};
 
-		ethernet@2,02000000 {
-			compatible = "smsc,lan9118", "smsc,lan9115";
-			reg = <2 0x02000000 0x10000>;
-			interrupts = <15>;
-			phy-mode = "mii";
-			reg-io-width = <4>;
-			smsc,irq-active-high;
-			smsc,irq-push-pull;
-			vdd33a-supply = <&v2m_fixed_3v3>;
-			vddvario-supply = <&v2m_fixed_3v3>;
-		};
+			v2m_video_ram: vram@2,00000000 {
+				compatible = "arm,vexpress-vram";
+				reg = <2 0x00000000 0x00800000>;
+			};
 
-		usb@2,03000000 {
-			compatible = "nxp,usb-isp1761";
-			reg = <2 0x03000000 0x20000>;
-			interrupts = <16>;
-			port1-otg;
-		};
+			ethernet@2,02000000 {
+				compatible = "smsc,lan9118", "smsc,lan9115";
+				reg = <2 0x02000000 0x10000>;
+				interrupts = <15>;
+				phy-mode = "mii";
+				reg-io-width = <4>;
+				smsc,irq-active-high;
+				smsc,irq-push-pull;
+				vdd33a-supply = <&v2m_fixed_3v3>;
+				vddvario-supply = <&v2m_fixed_3v3>;
+			};
 
-		iofpga@3,00000000 {
-			compatible = "simple-bus";
-			#address-cells = <1>;
-			#size-cells = <1>;
-			ranges = <0 3 0 0x200000>;
+			usb@2,03000000 {
+				compatible = "nxp,usb-isp1761";
+				reg = <2 0x03000000 0x20000>;
+				interrupts = <16>;
+				port1-otg;
+			};
 
-			v2m_sysreg: sysreg@10000 {
-				compatible = "arm,vexpress-sysreg";
-				reg = <0x010000 0x1000>;
+			iofpga@3,00000000 {
+				compatible = "simple-bus";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 3 0 0x200000>;
+
+				v2m_sysreg: sysreg@10000 {
+					compatible = "arm,vexpress-sysreg";
+					reg = <0x010000 0x1000>;
+					#address-cells = <1>;
+					#size-cells = <1>;
+					ranges = <0 0x10000 0x1000>;
+
+					v2m_led_gpios: gpio@8 {
+						compatible = "arm,vexpress-sysreg,sys_led";
+						reg = <0x008 4>;
+						gpio-controller;
+						#gpio-cells = <2>;
+					};
 
-				v2m_led_gpios: sys_led {
-					compatible = "arm,vexpress-sysreg,sys_led";
-					gpio-controller;
-					#gpio-cells = <2>;
-				};
+					v2m_mmc_gpios: gpio@48 {
+						compatible = "arm,vexpress-sysreg,sys_mci";
+						reg = <0x048 4>;
+						gpio-controller;
+						#gpio-cells = <2>;
+					};
 
-				v2m_mmc_gpios: sys_mci {
-					compatible = "arm,vexpress-sysreg,sys_mci";
-					gpio-controller;
-					#gpio-cells = <2>;
+					v2m_flash_gpios: gpio@4c {
+						compatible = "arm,vexpress-sysreg,sys_flash";
+						reg = <0x04c 4>;
+						gpio-controller;
+						#gpio-cells = <2>;
+					};
 				};
 
-				v2m_flash_gpios: sys_flash {
-					compatible = "arm,vexpress-sysreg,sys_flash";
-					gpio-controller;
-					#gpio-cells = <2>;
+				v2m_sysctl: sysctl@20000 {
+					compatible = "arm,sp810", "arm,primecell";
+					reg = <0x020000 0x1000>;
+					clocks = <&v2m_refclk32khz>, <&v2m_refclk1mhz>, <&smbclk>;
+					clock-names = "refclk", "timclk", "apb_pclk";
+					#clock-cells = <1>;
+					clock-output-names = "timerclken0", "timerclken1", "timerclken2", "timerclken3";
+					assigned-clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&v2m_sysctl 3>, <&v2m_sysctl 3>;
+					assigned-clock-parents = <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>;
 				};
-			};
 
-			v2m_sysctl: sysctl@20000 {
-				compatible = "arm,sp810", "arm,primecell";
-				reg = <0x020000 0x1000>;
-				clocks = <&v2m_refclk32khz>, <&v2m_refclk1mhz>, <&smbclk>;
-				clock-names = "refclk", "timclk", "apb_pclk";
-				#clock-cells = <1>;
-				clock-output-names = "timerclken0", "timerclken1", "timerclken2", "timerclken3";
-				assigned-clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&v2m_sysctl 3>, <&v2m_sysctl 3>;
-				assigned-clock-parents = <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>;
-			};
+				/* PCI-E I2C bus */
+				v2m_i2c_pcie: i2c@30000 {
+					compatible = "arm,versatile-i2c";
+					reg = <0x030000 0x1000>;
 
-			/* PCI-E I2C bus */
-			v2m_i2c_pcie: i2c@30000 {
-				compatible = "arm,versatile-i2c";
-				reg = <0x030000 0x1000>;
+					#address-cells = <1>;
+					#size-cells = <0>;
 
-				#address-cells = <1>;
-				#size-cells = <0>;
+					pcie-switch@60 {
+						compatible = "idt,89hpes32h8";
+						reg = <0x60>;
+					};
+				};
 
-				pcie-switch@60 {
-					compatible = "idt,89hpes32h8";
-					reg = <0x60>;
+				aaci@40000 {
+					compatible = "arm,pl041", "arm,primecell";
+					reg = <0x040000 0x1000>;
+					interrupts = <11>;
+					clocks = <&smbclk>;
+					clock-names = "apb_pclk";
 				};
-			};
 
-			aaci@40000 {
-				compatible = "arm,pl041", "arm,primecell";
-				reg = <0x040000 0x1000>;
-				interrupts = <11>;
-				clocks = <&smbclk>;
-				clock-names = "apb_pclk";
-			};
+				mmci@50000 {
+					compatible = "arm,pl180", "arm,primecell";
+					reg = <0x050000 0x1000>;
+					interrupts = <9 10>;
+					cd-gpios = <&v2m_mmc_gpios 0 0>;
+					wp-gpios = <&v2m_mmc_gpios 1 0>;
+					max-frequency = <12000000>;
+					vmmc-supply = <&v2m_fixed_3v3>;
+					clocks = <&v2m_clk24mhz>, <&smbclk>;
+					clock-names = "mclk", "apb_pclk";
+				};
 
-			mmci@50000 {
-				compatible = "arm,pl180", "arm,primecell";
-				reg = <0x050000 0x1000>;
-				interrupts = <9 10>;
-				cd-gpios = <&v2m_mmc_gpios 0 0>;
-				wp-gpios = <&v2m_mmc_gpios 1 0>;
-				max-frequency = <12000000>;
-				vmmc-supply = <&v2m_fixed_3v3>;
-				clocks = <&v2m_clk24mhz>, <&smbclk>;
-				clock-names = "mclk", "apb_pclk";
-			};
+				kmi@60000 {
+					compatible = "arm,pl050", "arm,primecell";
+					reg = <0x060000 0x1000>;
+					interrupts = <12>;
+					clocks = <&v2m_clk24mhz>, <&smbclk>;
+					clock-names = "KMIREFCLK", "apb_pclk";
+				};
 
-			kmi@60000 {
-				compatible = "arm,pl050", "arm,primecell";
-				reg = <0x060000 0x1000>;
-				interrupts = <12>;
-				clocks = <&v2m_clk24mhz>, <&smbclk>;
-				clock-names = "KMIREFCLK", "apb_pclk";
-			};
+				kmi@70000 {
+					compatible = "arm,pl050", "arm,primecell";
+					reg = <0x070000 0x1000>;
+					interrupts = <13>;
+					clocks = <&v2m_clk24mhz>, <&smbclk>;
+					clock-names = "KMIREFCLK", "apb_pclk";
+				};
 
-			kmi@70000 {
-				compatible = "arm,pl050", "arm,primecell";
-				reg = <0x070000 0x1000>;
-				interrupts = <13>;
-				clocks = <&v2m_clk24mhz>, <&smbclk>;
-				clock-names = "KMIREFCLK", "apb_pclk";
-			};
+				v2m_serial0: uart@90000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x090000 0x1000>;
+					interrupts = <5>;
+					clocks = <&v2m_oscclk2>, <&smbclk>;
+					clock-names = "uartclk", "apb_pclk";
+				};
 
-			v2m_serial0: uart@90000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x090000 0x1000>;
-				interrupts = <5>;
-				clocks = <&v2m_oscclk2>, <&smbclk>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+				v2m_serial1: uart@a0000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x0a0000 0x1000>;
+					interrupts = <6>;
+					clocks = <&v2m_oscclk2>, <&smbclk>;
+					clock-names = "uartclk", "apb_pclk";
+				};
 
-			v2m_serial1: uart@a0000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x0a0000 0x1000>;
-				interrupts = <6>;
-				clocks = <&v2m_oscclk2>, <&smbclk>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+				v2m_serial2: uart@b0000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x0b0000 0x1000>;
+					interrupts = <7>;
+					clocks = <&v2m_oscclk2>, <&smbclk>;
+					clock-names = "uartclk", "apb_pclk";
+				};
 
-			v2m_serial2: uart@b0000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x0b0000 0x1000>;
-				interrupts = <7>;
-				clocks = <&v2m_oscclk2>, <&smbclk>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+				v2m_serial3: uart@c0000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x0c0000 0x1000>;
+					interrupts = <8>;
+					clocks = <&v2m_oscclk2>, <&smbclk>;
+					clock-names = "uartclk", "apb_pclk";
+				};
 
-			v2m_serial3: uart@c0000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x0c0000 0x1000>;
-				interrupts = <8>;
-				clocks = <&v2m_oscclk2>, <&smbclk>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+				wdt@f0000 {
+					compatible = "arm,sp805", "arm,primecell";
+					reg = <0x0f0000 0x1000>;
+					interrupts = <0>;
+					clocks = <&v2m_refclk32khz>, <&smbclk>;
+					clock-names = "wdogclk", "apb_pclk";
+				};
 
-			wdt@f0000 {
-				compatible = "arm,sp805", "arm,primecell";
-				reg = <0x0f0000 0x1000>;
-				interrupts = <0>;
-				clocks = <&v2m_refclk32khz>, <&smbclk>;
-				clock-names = "wdogclk", "apb_pclk";
-			};
+				v2m_timer01: timer@110000 {
+					compatible = "arm,sp804", "arm,primecell";
+					reg = <0x110000 0x1000>;
+					interrupts = <2>;
+					clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&smbclk>;
+					clock-names = "timclken1", "timclken2", "apb_pclk";
+				};
 
-			v2m_timer01: timer@110000 {
-				compatible = "arm,sp804", "arm,primecell";
-				reg = <0x110000 0x1000>;
-				interrupts = <2>;
-				clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&smbclk>;
-				clock-names = "timclken1", "timclken2", "apb_pclk";
-			};
+				v2m_timer23: timer@120000 {
+					compatible = "arm,sp804", "arm,primecell";
+					reg = <0x120000 0x1000>;
+					interrupts = <3>;
+					clocks = <&v2m_sysctl 2>, <&v2m_sysctl 3>, <&smbclk>;
+					clock-names = "timclken1", "timclken2", "apb_pclk";
+				};
 
-			v2m_timer23: timer@120000 {
-				compatible = "arm,sp804", "arm,primecell";
-				reg = <0x120000 0x1000>;
-				interrupts = <3>;
-				clocks = <&v2m_sysctl 2>, <&v2m_sysctl 3>, <&smbclk>;
-				clock-names = "timclken1", "timclken2", "apb_pclk";
-			};
+				/* DVI I2C bus */
+				v2m_i2c_dvi: i2c@160000 {
+					compatible = "arm,versatile-i2c";
+					reg = <0x160000 0x1000>;
 
-			/* DVI I2C bus */
-			v2m_i2c_dvi: i2c@160000 {
-				compatible = "arm,versatile-i2c";
-				reg = <0x160000 0x1000>;
+					#address-cells = <1>;
+					#size-cells = <0>;
 
-				#address-cells = <1>;
-				#size-cells = <0>;
+					dvi-transmitter@39 {
+						compatible = "sil,sii9022-tpi", "sil,sii9022";
+						reg = <0x39>;
+					};
 
-				dvi-transmitter@39 {
-					compatible = "sil,sii9022-tpi", "sil,sii9022";
-					reg = <0x39>;
+					dvi-transmitter@60 {
+						compatible = "sil,sii9022-cpi", "sil,sii9022";
+						reg = <0x60>;
+					};
 				};
 
-				dvi-transmitter@60 {
-					compatible = "sil,sii9022-cpi", "sil,sii9022";
-					reg = <0x60>;
+				rtc@170000 {
+					compatible = "arm,pl031", "arm,primecell";
+					reg = <0x170000 0x1000>;
+					interrupts = <4>;
+					clocks = <&smbclk>;
+					clock-names = "apb_pclk";
 				};
-			};
 
-			rtc@170000 {
-				compatible = "arm,pl031", "arm,primecell";
-				reg = <0x170000 0x1000>;
-				interrupts = <4>;
-				clocks = <&smbclk>;
-				clock-names = "apb_pclk";
-			};
-
-			compact-flash@1a0000 {
-				compatible = "arm,vexpress-cf", "ata-generic";
-				reg = <0x1a0000 0x100
-				       0x1a0100 0xf00>;
-				reg-shift = <2>;
-			};
-
-			clcd@1f0000 {
-				compatible = "arm,pl111", "arm,primecell";
-				reg = <0x1f0000 0x1000>;
-				interrupt-names = "combined";
-				interrupts = <14>;
-				clocks = <&v2m_oscclk1>, <&smbclk>;
-				clock-names = "clcdclk", "apb_pclk";
-				memory-region = <&v2m_video_ram>;
-				max-memory-bandwidth = <50350000>; /* 16bpp @ 25.175MHz */
-
-				port {
-					v2m_clcd_pads: endpoint {
-						remote-endpoint = <&v2m_clcd_panel>;
-						arm,pl11x,tft-r0g0b0-pads = <0 8 16>;
-					};
+				compact-flash@1a0000 {
+					compatible = "arm,vexpress-cf", "ata-generic";
+					reg = <0x1a0000 0x100
+					       0x1a0100 0xf00>;
+					reg-shift = <2>;
 				};
 
-				panel {
-					compatible = "panel-dpi";
+				clcd@1f0000 {
+					compatible = "arm,pl111", "arm,primecell";
+					reg = <0x1f0000 0x1000>;
+					interrupt-names = "combined";
+					interrupts = <14>;
+					clocks = <&v2m_oscclk1>, <&smbclk>;
+					clock-names = "clcdclk", "apb_pclk";
+					memory-region = <&v2m_video_ram>;
+					max-memory-bandwidth = <50350000>; /* 16bpp @ 25.175MHz */
 
 					port {
-						v2m_clcd_panel: endpoint {
-							remote-endpoint = <&v2m_clcd_pads>;
+						v2m_clcd_pads: endpoint {
+							remote-endpoint = <&v2m_clcd_panel>;
+							arm,pl11x,tft-r0g0b0-pads = <0 8 16>;
 						};
 					};
 
-					panel-timing {
-						clock-frequency = <25175000>;
-						hactive = <640>;
-						hback-porch = <40>;
-						hfront-porch = <24>;
-						hsync-len = <96>;
-						vactive = <480>;
-						vback-porch = <32>;
-						vfront-porch = <11>;
-						vsync-len = <2>;
+					panel {
+						compatible = "panel-dpi";
+
+						port {
+							v2m_clcd_panel: endpoint {
+								remote-endpoint = <&v2m_clcd_pads>;
+							};
+						};
+
+						panel-timing {
+							clock-frequency = <25175000>;
+							hactive = <640>;
+							hback-porch = <40>;
+							hfront-porch = <24>;
+							hsync-len = <96>;
+							vactive = <480>;
+							vback-porch = <32>;
+							vfront-porch = <11>;
+							vsync-len = <2>;
+						};
 					};
 				};
 			};
-		};
 
-		v2m_fixed_3v3: fixed-regulator-0 {
-			compatible = "regulator-fixed";
-			regulator-name = "3V3";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-always-on;
-		};
+			v2m_fixed_3v3: fixed-regulator-0 {
+				compatible = "regulator-fixed";
+				regulator-name = "3V3";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
 
-		v2m_clk24mhz: clk24mhz {
-			compatible = "fixed-clock";
-			#clock-cells = <0>;
-			clock-frequency = <24000000>;
-			clock-output-names = "v2m:clk24mhz";
-		};
+			v2m_clk24mhz: clk24mhz {
+				compatible = "fixed-clock";
+				#clock-cells = <0>;
+				clock-frequency = <24000000>;
+				clock-output-names = "v2m:clk24mhz";
+			};
 
-		v2m_refclk1mhz: refclk1mhz {
-			compatible = "fixed-clock";
-			#clock-cells = <0>;
-			clock-frequency = <1000000>;
-			clock-output-names = "v2m:refclk1mhz";
-		};
+			v2m_refclk1mhz: refclk1mhz {
+				compatible = "fixed-clock";
+				#clock-cells = <0>;
+				clock-frequency = <1000000>;
+				clock-output-names = "v2m:refclk1mhz";
+			};
 
-		v2m_refclk32khz: refclk32khz {
-			compatible = "fixed-clock";
-			#clock-cells = <0>;
-			clock-frequency = <32768>;
-			clock-output-names = "v2m:refclk32khz";
-		};
+			v2m_refclk32khz: refclk32khz {
+				compatible = "fixed-clock";
+				#clock-cells = <0>;
+				clock-frequency = <32768>;
+				clock-output-names = "v2m:refclk32khz";
+			};
 
-		leds {
-			compatible = "gpio-leds";
+			leds {
+				compatible = "gpio-leds";
 
-			user1 {
-				label = "v2m:green:user1";
-				gpios = <&v2m_led_gpios 0 0>;
-				linux,default-trigger = "heartbeat";
-			};
+				user1 {
+					label = "v2m:green:user1";
+					gpios = <&v2m_led_gpios 0 0>;
+					linux,default-trigger = "heartbeat";
+				};
 
-			user2 {
-				label = "v2m:green:user2";
-				gpios = <&v2m_led_gpios 1 0>;
-				linux,default-trigger = "mmc0";
-			};
+				user2 {
+					label = "v2m:green:user2";
+					gpios = <&v2m_led_gpios 1 0>;
+					linux,default-trigger = "mmc0";
+				};
 
-			user3 {
-				label = "v2m:green:user3";
-				gpios = <&v2m_led_gpios 2 0>;
-				linux,default-trigger = "cpu0";
-			};
+				user3 {
+					label = "v2m:green:user3";
+					gpios = <&v2m_led_gpios 2 0>;
+					linux,default-trigger = "cpu0";
+				};
 
-			user4 {
-				label = "v2m:green:user4";
-				gpios = <&v2m_led_gpios 3 0>;
-				linux,default-trigger = "cpu1";
-			};
+				user4 {
+					label = "v2m:green:user4";
+					gpios = <&v2m_led_gpios 3 0>;
+					linux,default-trigger = "cpu1";
+				};
 
-			user5 {
-				label = "v2m:green:user5";
-				gpios = <&v2m_led_gpios 4 0>;
-				linux,default-trigger = "cpu2";
-			};
+				user5 {
+					label = "v2m:green:user5";
+					gpios = <&v2m_led_gpios 4 0>;
+					linux,default-trigger = "cpu2";
+				};
 
-			user6 {
-				label = "v2m:green:user6";
-				gpios = <&v2m_led_gpios 5 0>;
-				linux,default-trigger = "cpu3";
-			};
+				user6 {
+					label = "v2m:green:user6";
+					gpios = <&v2m_led_gpios 5 0>;
+					linux,default-trigger = "cpu3";
+				};
 
-			user7 {
-				label = "v2m:green:user7";
-				gpios = <&v2m_led_gpios 6 0>;
-				linux,default-trigger = "cpu4";
-			};
+				user7 {
+					label = "v2m:green:user7";
+					gpios = <&v2m_led_gpios 6 0>;
+					linux,default-trigger = "cpu4";
+				};
 
-			user8 {
-				label = "v2m:green:user8";
-				gpios = <&v2m_led_gpios 7 0>;
-				linux,default-trigger = "cpu5";
+				user8 {
+					label = "v2m:green:user8";
+					gpios = <&v2m_led_gpios 7 0>;
+					linux,default-trigger = "cpu5";
+				};
 			};
-		};
 
-		mcc {
-			compatible = "arm,vexpress,config-bus";
-			arm,vexpress,config-bridge = <&v2m_sysreg>;
+			mcc {
+				compatible = "arm,vexpress,config-bus";
+				arm,vexpress,config-bridge = <&v2m_sysreg>;
 
-			oscclk0 {
-				/* MCC static memory clock */
-				compatible = "arm,vexpress-osc";
-				arm,vexpress-sysreg,func = <1 0>;
-				freq-range = <25000000 60000000>;
-				#clock-cells = <0>;
-				clock-output-names = "v2m:oscclk0";
-			};
+				oscclk0 {
+					/* MCC static memory clock */
+					compatible = "arm,vexpress-osc";
+					arm,vexpress-sysreg,func = <1 0>;
+					freq-range = <25000000 60000000>;
+					#clock-cells = <0>;
+					clock-output-names = "v2m:oscclk0";
+				};
 
-			v2m_oscclk1: oscclk1 {
-				/* CLCD clock */
-				compatible = "arm,vexpress-osc";
-				arm,vexpress-sysreg,func = <1 1>;
-				freq-range = <23750000 65000000>;
-				#clock-cells = <0>;
-				clock-output-names = "v2m:oscclk1";
-			};
+				v2m_oscclk1: oscclk1 {
+					/* CLCD clock */
+					compatible = "arm,vexpress-osc";
+					arm,vexpress-sysreg,func = <1 1>;
+					freq-range = <23750000 65000000>;
+					#clock-cells = <0>;
+					clock-output-names = "v2m:oscclk1";
+				};
 
-			v2m_oscclk2: oscclk2 {
-				/* IO FPGA peripheral clock */
-				compatible = "arm,vexpress-osc";
-				arm,vexpress-sysreg,func = <1 2>;
-				freq-range = <24000000 24000000>;
-				#clock-cells = <0>;
-				clock-output-names = "v2m:oscclk2";
-			};
+				v2m_oscclk2: oscclk2 {
+					/* IO FPGA peripheral clock */
+					compatible = "arm,vexpress-osc";
+					arm,vexpress-sysreg,func = <1 2>;
+					freq-range = <24000000 24000000>;
+					#clock-cells = <0>;
+					clock-output-names = "v2m:oscclk2";
+				};
 
-			volt-vio {
-				/* Logic level voltage */
-				compatible = "arm,vexpress-volt";
-				arm,vexpress-sysreg,func = <2 0>;
-				regulator-name = "VIO";
-				regulator-always-on;
-				label = "VIO";
-			};
+				volt-vio {
+					/* Logic level voltage */
+					compatible = "arm,vexpress-volt";
+					arm,vexpress-sysreg,func = <2 0>;
+					regulator-name = "VIO";
+					regulator-always-on;
+					label = "VIO";
+				};
 
-			temp-mcc {
-				/* MCC internal operating temperature */
-				compatible = "arm,vexpress-temp";
-				arm,vexpress-sysreg,func = <4 0>;
-				label = "MCC";
-			};
+				temp-mcc {
+					/* MCC internal operating temperature */
+					compatible = "arm,vexpress-temp";
+					arm,vexpress-sysreg,func = <4 0>;
+					label = "MCC";
+				};
 
-			reset {
-				compatible = "arm,vexpress-reset";
-				arm,vexpress-sysreg,func = <5 0>;
-			};
+				reset {
+					compatible = "arm,vexpress-reset";
+					arm,vexpress-sysreg,func = <5 0>;
+				};
 
-			muxfpga {
-				compatible = "arm,vexpress-muxfpga";
-				arm,vexpress-sysreg,func = <7 0>;
-			};
+				muxfpga {
+					compatible = "arm,vexpress-muxfpga";
+					arm,vexpress-sysreg,func = <7 0>;
+				};
 
-			shutdown {
-				compatible = "arm,vexpress-shutdown";
-				arm,vexpress-sysreg,func = <8 0>;
-			};
+				shutdown {
+					compatible = "arm,vexpress-shutdown";
+					arm,vexpress-sysreg,func = <8 0>;
+				};
 
-			reboot {
-				compatible = "arm,vexpress-reboot";
-				arm,vexpress-sysreg,func = <9 0>;
-			};
+				reboot {
+					compatible = "arm,vexpress-reboot";
+					arm,vexpress-sysreg,func = <9 0>;
+				};
 
-			dvimode {
-				compatible = "arm,vexpress-dvimode";
-				arm,vexpress-sysreg,func = <11 0>;
+				dvimode {
+					compatible = "arm,vexpress-dvimode";
+					arm,vexpress-sysreg,func = <11 0>;
+				};
 			};
 		};
 	};
+};
diff --git a/arch/arm/boot/dts/vexpress-v2m.dtsi b/arch/arm/boot/dts/vexpress-v2m.dtsi
index b0021a816028..4db42f6326a3 100644
--- a/arch/arm/boot/dts/vexpress-v2m.dtsi
+++ b/arch/arm/boot/dts/vexpress-v2m.dtsi
@@ -18,425 +18,435 @@
  * CHANGES TO vexpress-v2m-rs1.dtsi!
  */
 
-	motherboard {
-		model = "V2M-P1";
-		arm,hbi = <0x190>;
-		arm,vexpress,site = <0>;
-		compatible = "arm,vexpress,v2m-p1", "simple-bus";
-		#address-cells = <2>; /* SMB chipselect number and offset */
-		#size-cells = <1>;
-		#interrupt-cells = <1>;
-		ranges;
-
-		flash@0,00000000 {
-			compatible = "arm,vexpress-flash", "cfi-flash";
-			reg = <0 0x00000000 0x04000000>,
-			      <1 0x00000000 0x04000000>;
-			bank-width = <4>;
-		};
+/ {
+	smb@4000000 {
+		motherboard {
+			model = "V2M-P1";
+			arm,hbi = <0x190>;
+			arm,vexpress,site = <0>;
+			compatible = "arm,vexpress,v2m-p1", "simple-bus";
+			#address-cells = <2>; /* SMB chipselect number and offset */
+			#size-cells = <1>;
+			#interrupt-cells = <1>;
+			ranges;
 
-		psram@2,00000000 {
-			compatible = "arm,vexpress-psram", "mtd-ram";
-			reg = <2 0x00000000 0x02000000>;
-			bank-width = <4>;
-		};
+			flash@0,00000000 {
+				compatible = "arm,vexpress-flash", "cfi-flash";
+				reg = <0 0x00000000 0x04000000>,
+				      <1 0x00000000 0x04000000>;
+				bank-width = <4>;
+			};
 
-		v2m_video_ram: vram@3,00000000 {
-			compatible = "arm,vexpress-vram";
-			reg = <3 0x00000000 0x00800000>;
-		};
+			psram@2,00000000 {
+				compatible = "arm,vexpress-psram", "mtd-ram";
+				reg = <2 0x00000000 0x02000000>;
+				bank-width = <4>;
+			};
 
-		ethernet@3,02000000 {
-			compatible = "smsc,lan9118", "smsc,lan9115";
-			reg = <3 0x02000000 0x10000>;
-			interrupts = <15>;
-			phy-mode = "mii";
-			reg-io-width = <4>;
-			smsc,irq-active-high;
-			smsc,irq-push-pull;
-			vdd33a-supply = <&v2m_fixed_3v3>;
-			vddvario-supply = <&v2m_fixed_3v3>;
-		};
+			v2m_video_ram: vram@3,00000000 {
+				compatible = "arm,vexpress-vram";
+				reg = <3 0x00000000 0x00800000>;
+			};
 
-		usb@3,03000000 {
-			compatible = "nxp,usb-isp1761";
-			reg = <3 0x03000000 0x20000>;
-			interrupts = <16>;
-			port1-otg;
-		};
+			ethernet@3,02000000 {
+				compatible = "smsc,lan9118", "smsc,lan9115";
+				reg = <3 0x02000000 0x10000>;
+				interrupts = <15>;
+				phy-mode = "mii";
+				reg-io-width = <4>;
+				smsc,irq-active-high;
+				smsc,irq-push-pull;
+				vdd33a-supply = <&v2m_fixed_3v3>;
+				vddvario-supply = <&v2m_fixed_3v3>;
+			};
 
-		iofpga@7,00000000 {
-			compatible = "simple-bus";
-			#address-cells = <1>;
-			#size-cells = <1>;
-			ranges = <0 7 0 0x20000>;
+			usb@3,03000000 {
+				compatible = "nxp,usb-isp1761";
+				reg = <3 0x03000000 0x20000>;
+				interrupts = <16>;
+				port1-otg;
+			};
 
-			v2m_sysreg: sysreg@0 {
-				compatible = "arm,vexpress-sysreg";
-				reg = <0x00000 0x1000>;
+			iofpga@7,00000000 {
+				compatible = "simple-bus";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 7 0 0x20000>;
+
+				v2m_sysreg: sysreg@0 {
+					compatible = "arm,vexpress-sysreg";
+					reg = <0x00000 0x1000>;
+					#address-cells = <1>;
+					#size-cells = <1>;
+					ranges = <0 0 0x1000>;
+
+					v2m_led_gpios: gpio@8 {
+						compatible = "arm,vexpress-sysreg,sys_led";
+						reg = <0x008 4>;
+						gpio-controller;
+						#gpio-cells = <2>;
+					};
 
-				v2m_led_gpios: sys_led {
-					compatible = "arm,vexpress-sysreg,sys_led";
-					gpio-controller;
-					#gpio-cells = <2>;
-				};
+					v2m_mmc_gpios: gpio@48 {
+						compatible = "arm,vexpress-sysreg,sys_mci";
+						reg = <0x048 4>;
+						gpio-controller;
+						#gpio-cells = <2>;
+					};
 
-				v2m_mmc_gpios: sys_mci {
-					compatible = "arm,vexpress-sysreg,sys_mci";
-					gpio-controller;
-					#gpio-cells = <2>;
+					v2m_flash_gpios: gpio@4c {
+						compatible = "arm,vexpress-sysreg,sys_flash";
+						reg = <0x04c 4>;
+						gpio-controller;
+						#gpio-cells = <2>;
+					};
 				};
 
-				v2m_flash_gpios: sys_flash {
-					compatible = "arm,vexpress-sysreg,sys_flash";
-					gpio-controller;
-					#gpio-cells = <2>;
+				v2m_sysctl: sysctl@1000 {
+					compatible = "arm,sp810", "arm,primecell";
+					reg = <0x01000 0x1000>;
+					clocks = <&v2m_refclk32khz>, <&v2m_refclk1mhz>, <&smbclk>;
+					clock-names = "refclk", "timclk", "apb_pclk";
+					#clock-cells = <1>;
+					clock-output-names = "timerclken0", "timerclken1", "timerclken2", "timerclken3";
+					assigned-clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&v2m_sysctl 3>, <&v2m_sysctl 3>;
+					assigned-clock-parents = <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>;
 				};
-			};
 
-			v2m_sysctl: sysctl@1000 {
-				compatible = "arm,sp810", "arm,primecell";
-				reg = <0x01000 0x1000>;
-				clocks = <&v2m_refclk32khz>, <&v2m_refclk1mhz>, <&smbclk>;
-				clock-names = "refclk", "timclk", "apb_pclk";
-				#clock-cells = <1>;
-				clock-output-names = "timerclken0", "timerclken1", "timerclken2", "timerclken3";
-				assigned-clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&v2m_sysctl 3>, <&v2m_sysctl 3>;
-				assigned-clock-parents = <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>;
-			};
+				/* PCI-E I2C bus */
+				v2m_i2c_pcie: i2c@2000 {
+					compatible = "arm,versatile-i2c";
+					reg = <0x02000 0x1000>;
 
-			/* PCI-E I2C bus */
-			v2m_i2c_pcie: i2c@2000 {
-				compatible = "arm,versatile-i2c";
-				reg = <0x02000 0x1000>;
+					#address-cells = <1>;
+					#size-cells = <0>;
 
-				#address-cells = <1>;
-				#size-cells = <0>;
+					pcie-switch@60 {
+						compatible = "idt,89hpes32h8";
+						reg = <0x60>;
+					};
+				};
 
-				pcie-switch@60 {
-					compatible = "idt,89hpes32h8";
-					reg = <0x60>;
+				aaci@4000 {
+					compatible = "arm,pl041", "arm,primecell";
+					reg = <0x04000 0x1000>;
+					interrupts = <11>;
+					clocks = <&smbclk>;
+					clock-names = "apb_pclk";
 				};
-			};
 
-			aaci@4000 {
-				compatible = "arm,pl041", "arm,primecell";
-				reg = <0x04000 0x1000>;
-				interrupts = <11>;
-				clocks = <&smbclk>;
-				clock-names = "apb_pclk";
-			};
+				mmci@5000 {
+					compatible = "arm,pl180", "arm,primecell";
+					reg = <0x05000 0x1000>;
+					interrupts = <9 10>;
+					cd-gpios = <&v2m_mmc_gpios 0 0>;
+					wp-gpios = <&v2m_mmc_gpios 1 0>;
+					max-frequency = <12000000>;
+					vmmc-supply = <&v2m_fixed_3v3>;
+					clocks = <&v2m_clk24mhz>, <&smbclk>;
+					clock-names = "mclk", "apb_pclk";
+				};
 
-			mmci@5000 {
-				compatible = "arm,pl180", "arm,primecell";
-				reg = <0x05000 0x1000>;
-				interrupts = <9 10>;
-				cd-gpios = <&v2m_mmc_gpios 0 0>;
-				wp-gpios = <&v2m_mmc_gpios 1 0>;
-				max-frequency = <12000000>;
-				vmmc-supply = <&v2m_fixed_3v3>;
-				clocks = <&v2m_clk24mhz>, <&smbclk>;
-				clock-names = "mclk", "apb_pclk";
-			};
+				kmi@6000 {
+					compatible = "arm,pl050", "arm,primecell";
+					reg = <0x06000 0x1000>;
+					interrupts = <12>;
+					clocks = <&v2m_clk24mhz>, <&smbclk>;
+					clock-names = "KMIREFCLK", "apb_pclk";
+				};
 
-			kmi@6000 {
-				compatible = "arm,pl050", "arm,primecell";
-				reg = <0x06000 0x1000>;
-				interrupts = <12>;
-				clocks = <&v2m_clk24mhz>, <&smbclk>;
-				clock-names = "KMIREFCLK", "apb_pclk";
-			};
+				kmi@7000 {
+					compatible = "arm,pl050", "arm,primecell";
+					reg = <0x07000 0x1000>;
+					interrupts = <13>;
+					clocks = <&v2m_clk24mhz>, <&smbclk>;
+					clock-names = "KMIREFCLK", "apb_pclk";
+				};
 
-			kmi@7000 {
-				compatible = "arm,pl050", "arm,primecell";
-				reg = <0x07000 0x1000>;
-				interrupts = <13>;
-				clocks = <&v2m_clk24mhz>, <&smbclk>;
-				clock-names = "KMIREFCLK", "apb_pclk";
-			};
+				v2m_serial0: uart@9000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x09000 0x1000>;
+					interrupts = <5>;
+					clocks = <&v2m_oscclk2>, <&smbclk>;
+					clock-names = "uartclk", "apb_pclk";
+				};
 
-			v2m_serial0: uart@9000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x09000 0x1000>;
-				interrupts = <5>;
-				clocks = <&v2m_oscclk2>, <&smbclk>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+				v2m_serial1: uart@a000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x0a000 0x1000>;
+					interrupts = <6>;
+					clocks = <&v2m_oscclk2>, <&smbclk>;
+					clock-names = "uartclk", "apb_pclk";
+				};
 
-			v2m_serial1: uart@a000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x0a000 0x1000>;
-				interrupts = <6>;
-				clocks = <&v2m_oscclk2>, <&smbclk>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+				v2m_serial2: uart@b000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x0b000 0x1000>;
+					interrupts = <7>;
+					clocks = <&v2m_oscclk2>, <&smbclk>;
+					clock-names = "uartclk", "apb_pclk";
+				};
 
-			v2m_serial2: uart@b000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x0b000 0x1000>;
-				interrupts = <7>;
-				clocks = <&v2m_oscclk2>, <&smbclk>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+				v2m_serial3: uart@c000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x0c000 0x1000>;
+					interrupts = <8>;
+					clocks = <&v2m_oscclk2>, <&smbclk>;
+					clock-names = "uartclk", "apb_pclk";
+				};
 
-			v2m_serial3: uart@c000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x0c000 0x1000>;
-				interrupts = <8>;
-				clocks = <&v2m_oscclk2>, <&smbclk>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+				wdt@f000 {
+					compatible = "arm,sp805", "arm,primecell";
+					reg = <0x0f000 0x1000>;
+					interrupts = <0>;
+					clocks = <&v2m_refclk32khz>, <&smbclk>;
+					clock-names = "wdogclk", "apb_pclk";
+				};
 
-			wdt@f000 {
-				compatible = "arm,sp805", "arm,primecell";
-				reg = <0x0f000 0x1000>;
-				interrupts = <0>;
-				clocks = <&v2m_refclk32khz>, <&smbclk>;
-				clock-names = "wdogclk", "apb_pclk";
-			};
+				v2m_timer01: timer@11000 {
+					compatible = "arm,sp804", "arm,primecell";
+					reg = <0x11000 0x1000>;
+					interrupts = <2>;
+					clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&smbclk>;
+					clock-names = "timclken1", "timclken2", "apb_pclk";
+				};
 
-			v2m_timer01: timer@11000 {
-				compatible = "arm,sp804", "arm,primecell";
-				reg = <0x11000 0x1000>;
-				interrupts = <2>;
-				clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&smbclk>;
-				clock-names = "timclken1", "timclken2", "apb_pclk";
-			};
+				v2m_timer23: timer@12000 {
+					compatible = "arm,sp804", "arm,primecell";
+					reg = <0x12000 0x1000>;
+					interrupts = <3>;
+					clocks = <&v2m_sysctl 2>, <&v2m_sysctl 3>, <&smbclk>;
+					clock-names = "timclken1", "timclken2", "apb_pclk";
+				};
 
-			v2m_timer23: timer@12000 {
-				compatible = "arm,sp804", "arm,primecell";
-				reg = <0x12000 0x1000>;
-				interrupts = <3>;
-				clocks = <&v2m_sysctl 2>, <&v2m_sysctl 3>, <&smbclk>;
-				clock-names = "timclken1", "timclken2", "apb_pclk";
-			};
+				/* DVI I2C bus */
+				v2m_i2c_dvi: i2c@16000 {
+					compatible = "arm,versatile-i2c";
+					reg = <0x16000 0x1000>;
 
-			/* DVI I2C bus */
-			v2m_i2c_dvi: i2c@16000 {
-				compatible = "arm,versatile-i2c";
-				reg = <0x16000 0x1000>;
+					#address-cells = <1>;
+					#size-cells = <0>;
 
-				#address-cells = <1>;
-				#size-cells = <0>;
+					dvi-transmitter@39 {
+						compatible = "sil,sii9022-tpi", "sil,sii9022";
+						reg = <0x39>;
+					};
 
-				dvi-transmitter@39 {
-					compatible = "sil,sii9022-tpi", "sil,sii9022";
-					reg = <0x39>;
+					dvi-transmitter@60 {
+						compatible = "sil,sii9022-cpi", "sil,sii9022";
+						reg = <0x60>;
+					};
 				};
 
-				dvi-transmitter@60 {
-					compatible = "sil,sii9022-cpi", "sil,sii9022";
-					reg = <0x60>;
+				rtc@17000 {
+					compatible = "arm,pl031", "arm,primecell";
+					reg = <0x17000 0x1000>;
+					interrupts = <4>;
+					clocks = <&smbclk>;
+					clock-names = "apb_pclk";
 				};
-			};
 
-			rtc@17000 {
-				compatible = "arm,pl031", "arm,primecell";
-				reg = <0x17000 0x1000>;
-				interrupts = <4>;
-				clocks = <&smbclk>;
-				clock-names = "apb_pclk";
-			};
-
-			compact-flash@1a000 {
-				compatible = "arm,vexpress-cf", "ata-generic";
-				reg = <0x1a000 0x100
-				       0x1a100 0xf00>;
-				reg-shift = <2>;
-			};
-
-			clcd@1f000 {
-				compatible = "arm,pl111", "arm,primecell";
-				reg = <0x1f000 0x1000>;
-				interrupt-names = "combined";
-				interrupts = <14>;
-				clocks = <&v2m_oscclk1>, <&smbclk>;
-				clock-names = "clcdclk", "apb_pclk";
-				memory-region = <&v2m_video_ram>;
-				max-memory-bandwidth = <50350000>; /* 16bpp @ 25.175MHz */
-
-				port {
-					v2m_clcd_pads: endpoint {
-						remote-endpoint = <&v2m_clcd_panel>;
-						arm,pl11x,tft-r0g0b0-pads = <0 8 16>;
-					};
+				compact-flash@1a000 {
+					compatible = "arm,vexpress-cf", "ata-generic";
+					reg = <0x1a000 0x100
+					       0x1a100 0xf00>;
+					reg-shift = <2>;
 				};
 
-				panel {
-					compatible = "panel-dpi";
+				clcd@1f000 {
+					compatible = "arm,pl111", "arm,primecell";
+					reg = <0x1f000 0x1000>;
+					interrupt-names = "combined";
+					interrupts = <14>;
+					clocks = <&v2m_oscclk1>, <&smbclk>;
+					clock-names = "clcdclk", "apb_pclk";
+					memory-region = <&v2m_video_ram>;
+					max-memory-bandwidth = <50350000>; /* 16bpp @ 25.175MHz */
 
 					port {
-						v2m_clcd_panel: endpoint {
-							remote-endpoint = <&v2m_clcd_pads>;
+						v2m_clcd_pads: endpoint {
+							remote-endpoint = <&v2m_clcd_panel>;
+							arm,pl11x,tft-r0g0b0-pads = <0 8 16>;
 						};
 					};
 
-					panel-timing {
-						clock-frequency = <25175000>;
-						hactive = <640>;
-						hback-porch = <40>;
-						hfront-porch = <24>;
-						hsync-len = <96>;
-						vactive = <480>;
-						vback-porch = <32>;
-						vfront-porch = <11>;
-						vsync-len = <2>;
+					panel {
+						compatible = "panel-dpi";
+
+						port {
+							v2m_clcd_panel: endpoint {
+								remote-endpoint = <&v2m_clcd_pads>;
+							};
+						};
+
+						panel-timing {
+							clock-frequency = <25175000>;
+							hactive = <640>;
+							hback-porch = <40>;
+							hfront-porch = <24>;
+							hsync-len = <96>;
+							vactive = <480>;
+							vback-porch = <32>;
+							vfront-porch = <11>;
+							vsync-len = <2>;
+						};
 					};
 				};
 			};
-		};
 
-		v2m_fixed_3v3: fixed-regulator-0 {
-			compatible = "regulator-fixed";
-			regulator-name = "3V3";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-always-on;
-		};
+			v2m_fixed_3v3: fixed-regulator-0 {
+				compatible = "regulator-fixed";
+				regulator-name = "3V3";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
 
-		v2m_clk24mhz: clk24mhz {
-			compatible = "fixed-clock";
-			#clock-cells = <0>;
-			clock-frequency = <24000000>;
-			clock-output-names = "v2m:clk24mhz";
-		};
+			v2m_clk24mhz: clk24mhz {
+				compatible = "fixed-clock";
+				#clock-cells = <0>;
+				clock-frequency = <24000000>;
+				clock-output-names = "v2m:clk24mhz";
+			};
 
-		v2m_refclk1mhz: refclk1mhz {
-			compatible = "fixed-clock";
-			#clock-cells = <0>;
-			clock-frequency = <1000000>;
-			clock-output-names = "v2m:refclk1mhz";
-		};
+			v2m_refclk1mhz: refclk1mhz {
+				compatible = "fixed-clock";
+				#clock-cells = <0>;
+				clock-frequency = <1000000>;
+				clock-output-names = "v2m:refclk1mhz";
+			};
 
-		v2m_refclk32khz: refclk32khz {
-			compatible = "fixed-clock";
-			#clock-cells = <0>;
-			clock-frequency = <32768>;
-			clock-output-names = "v2m:refclk32khz";
-		};
+			v2m_refclk32khz: refclk32khz {
+				compatible = "fixed-clock";
+				#clock-cells = <0>;
+				clock-frequency = <32768>;
+				clock-output-names = "v2m:refclk32khz";
+			};
 
-		leds {
-			compatible = "gpio-leds";
+			leds {
+				compatible = "gpio-leds";
 
-			user1 {
-				label = "v2m:green:user1";
-				gpios = <&v2m_led_gpios 0 0>;
-				linux,default-trigger = "heartbeat";
-			};
+				user1 {
+					label = "v2m:green:user1";
+					gpios = <&v2m_led_gpios 0 0>;
+					linux,default-trigger = "heartbeat";
+				};
 
-			user2 {
-				label = "v2m:green:user2";
-				gpios = <&v2m_led_gpios 1 0>;
-				linux,default-trigger = "mmc0";
-			};
+				user2 {
+					label = "v2m:green:user2";
+					gpios = <&v2m_led_gpios 1 0>;
+					linux,default-trigger = "mmc0";
+				};
 
-			user3 {
-				label = "v2m:green:user3";
-				gpios = <&v2m_led_gpios 2 0>;
-				linux,default-trigger = "cpu0";
-			};
+				user3 {
+					label = "v2m:green:user3";
+					gpios = <&v2m_led_gpios 2 0>;
+					linux,default-trigger = "cpu0";
+				};
 
-			user4 {
-				label = "v2m:green:user4";
-				gpios = <&v2m_led_gpios 3 0>;
-				linux,default-trigger = "cpu1";
-			};
+				user4 {
+					label = "v2m:green:user4";
+					gpios = <&v2m_led_gpios 3 0>;
+					linux,default-trigger = "cpu1";
+				};
 
-			user5 {
-				label = "v2m:green:user5";
-				gpios = <&v2m_led_gpios 4 0>;
-				linux,default-trigger = "cpu2";
-			};
+				user5 {
+					label = "v2m:green:user5";
+					gpios = <&v2m_led_gpios 4 0>;
+					linux,default-trigger = "cpu2";
+				};
 
-			user6 {
-				label = "v2m:green:user6";
-				gpios = <&v2m_led_gpios 5 0>;
-				linux,default-trigger = "cpu3";
-			};
+				user6 {
+					label = "v2m:green:user6";
+					gpios = <&v2m_led_gpios 5 0>;
+					linux,default-trigger = "cpu3";
+				};
 
-			user7 {
-				label = "v2m:green:user7";
-				gpios = <&v2m_led_gpios 6 0>;
-				linux,default-trigger = "cpu4";
-			};
+				user7 {
+					label = "v2m:green:user7";
+					gpios = <&v2m_led_gpios 6 0>;
+					linux,default-trigger = "cpu4";
+				};
 
-			user8 {
-				label = "v2m:green:user8";
-				gpios = <&v2m_led_gpios 7 0>;
-				linux,default-trigger = "cpu5";
+				user8 {
+					label = "v2m:green:user8";
+					gpios = <&v2m_led_gpios 7 0>;
+					linux,default-trigger = "cpu5";
+				};
 			};
-		};
 
-		mcc {
-			compatible = "arm,vexpress,config-bus";
-			arm,vexpress,config-bridge = <&v2m_sysreg>;
+			mcc {
+				compatible = "arm,vexpress,config-bus";
+				arm,vexpress,config-bridge = <&v2m_sysreg>;
 
-			oscclk0 {
-				/* MCC static memory clock */
-				compatible = "arm,vexpress-osc";
-				arm,vexpress-sysreg,func = <1 0>;
-				freq-range = <25000000 60000000>;
-				#clock-cells = <0>;
-				clock-output-names = "v2m:oscclk0";
-			};
+				oscclk0 {
+					/* MCC static memory clock */
+					compatible = "arm,vexpress-osc";
+					arm,vexpress-sysreg,func = <1 0>;
+					freq-range = <25000000 60000000>;
+					#clock-cells = <0>;
+					clock-output-names = "v2m:oscclk0";
+				};
 
-			v2m_oscclk1: oscclk1 {
-				/* CLCD clock */
-				compatible = "arm,vexpress-osc";
-				arm,vexpress-sysreg,func = <1 1>;
-				freq-range = <23750000 65000000>;
-				#clock-cells = <0>;
-				clock-output-names = "v2m:oscclk1";
-			};
+				v2m_oscclk1: oscclk1 {
+					/* CLCD clock */
+					compatible = "arm,vexpress-osc";
+					arm,vexpress-sysreg,func = <1 1>;
+					freq-range = <23750000 65000000>;
+					#clock-cells = <0>;
+					clock-output-names = "v2m:oscclk1";
+				};
 
-			v2m_oscclk2: oscclk2 {
-				/* IO FPGA peripheral clock */
-				compatible = "arm,vexpress-osc";
-				arm,vexpress-sysreg,func = <1 2>;
-				freq-range = <24000000 24000000>;
-				#clock-cells = <0>;
-				clock-output-names = "v2m:oscclk2";
-			};
+				v2m_oscclk2: oscclk2 {
+					/* IO FPGA peripheral clock */
+					compatible = "arm,vexpress-osc";
+					arm,vexpress-sysreg,func = <1 2>;
+					freq-range = <24000000 24000000>;
+					#clock-cells = <0>;
+					clock-output-names = "v2m:oscclk2";
+				};
 
-			volt-vio {
-				/* Logic level voltage */
-				compatible = "arm,vexpress-volt";
-				arm,vexpress-sysreg,func = <2 0>;
-				regulator-name = "VIO";
-				regulator-always-on;
-				label = "VIO";
-			};
+				volt-vio {
+					/* Logic level voltage */
+					compatible = "arm,vexpress-volt";
+					arm,vexpress-sysreg,func = <2 0>;
+					regulator-name = "VIO";
+					regulator-always-on;
+					label = "VIO";
+				};
 
-			temp-mcc {
-				/* MCC internal operating temperature */
-				compatible = "arm,vexpress-temp";
-				arm,vexpress-sysreg,func = <4 0>;
-				label = "MCC";
-			};
+				temp-mcc {
+					/* MCC internal operating temperature */
+					compatible = "arm,vexpress-temp";
+					arm,vexpress-sysreg,func = <4 0>;
+					label = "MCC";
+				};
 
-			reset {
-				compatible = "arm,vexpress-reset";
-				arm,vexpress-sysreg,func = <5 0>;
-			};
+				reset {
+					compatible = "arm,vexpress-reset";
+					arm,vexpress-sysreg,func = <5 0>;
+				};
 
-			muxfpga {
-				compatible = "arm,vexpress-muxfpga";
-				arm,vexpress-sysreg,func = <7 0>;
-			};
+				muxfpga {
+					compatible = "arm,vexpress-muxfpga";
+					arm,vexpress-sysreg,func = <7 0>;
+				};
 
-			shutdown {
-				compatible = "arm,vexpress-shutdown";
-				arm,vexpress-sysreg,func = <8 0>;
-			};
+				shutdown {
+					compatible = "arm,vexpress-shutdown";
+					arm,vexpress-sysreg,func = <8 0>;
+				};
 
-			reboot {
-				compatible = "arm,vexpress-reboot";
-				arm,vexpress-sysreg,func = <9 0>;
-			};
+				reboot {
+					compatible = "arm,vexpress-reboot";
+					arm,vexpress-sysreg,func = <9 0>;
+				};
 
-			dvimode {
-				compatible = "arm,vexpress-dvimode";
-				arm,vexpress-sysreg,func = <11 0>;
+				dvimode {
+					compatible = "arm,vexpress-dvimode";
+					arm,vexpress-sysreg,func = <11 0>;
+				};
 			};
 		};
 	};
+};
\ No newline at end of file
diff --git a/arch/arm/boot/dts/vexpress-v2p-ca15-tc1.dts b/arch/arm/boot/dts/vexpress-v2p-ca15-tc1.dts
index a8ac4e2ed290..3971427a105b 100644
--- a/arch/arm/boot/dts/vexpress-v2p-ca15-tc1.dts
+++ b/arch/arm/boot/dts/vexpress-v2p-ca15-tc1.dts
@@ -9,6 +9,7 @@
  */
 
 /dts-v1/;
+#include "vexpress-v2m-rs1.dtsi"
 
 / {
 	model = "V2P-CA15";
@@ -278,8 +279,6 @@
 				<0 0 40 &gic 0 40 4>,
 				<0 0 41 &gic 0 41 4>,
 				<0 0 42 &gic 0 42 4>;
-
-		/include/ "vexpress-v2m-rs1.dtsi"
 	};
 
 	site2: hsb@40000000 {
diff --git a/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts b/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts
index a4c7713edfcd..ac6b90e9d806 100644
--- a/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts
+++ b/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts
@@ -9,6 +9,7 @@
  */
 
 /dts-v1/;
+#include "vexpress-v2m-rs1.dtsi"
 
 / {
 	model = "V2P-CA15_CA7";
@@ -203,7 +204,7 @@
 			     <1 10 0xf08>;
 	};
 
-	pmu_a15 {
+	pmu-a15 {
 		compatible = "arm,cortex-a15-pmu";
 		interrupts = <0 68 4>,
 			     <0 69 4>;
@@ -211,7 +212,7 @@
 				     <&cpu1>;
 	};
 
-	pmu_a7 {
+	pmu-a7 {
 		compatible = "arm,cortex-a7-pmu";
 		interrupts = <0 128 4>,
 			     <0 129 4>,
@@ -584,7 +585,7 @@
 		};
 	};
 
-	smb@8000000 {
+	smb: smb@8000000 {
 		compatible = "simple-bus";
 
 		#address-cells = <2>;
@@ -641,8 +642,6 @@
 				<0 0 40 &gic 0 40 4>,
 				<0 0 41 &gic 0 41 4>,
 				<0 0 42 &gic 0 42 4>;
-
-		/include/ "vexpress-v2m-rs1.dtsi"
 	};
 
 	site2: hsb@40000000 {
diff --git a/arch/arm/boot/dts/vexpress-v2p-ca5s.dts b/arch/arm/boot/dts/vexpress-v2p-ca5s.dts
index 32f1906ffecf..e5b4a7570a01 100644
--- a/arch/arm/boot/dts/vexpress-v2p-ca5s.dts
+++ b/arch/arm/boot/dts/vexpress-v2p-ca5s.dts
@@ -9,6 +9,7 @@
  */
 
 /dts-v1/;
+#include "vexpress-v2m-rs1.dtsi"
 
 / {
 	model = "V2P-CA5s";
@@ -191,7 +192,7 @@
 		};
 	};
 
-	smb@8000000 {
+	smb: smb@8000000 {
 		compatible = "simple-bus";
 
 		#address-cells = <2>;
@@ -248,8 +249,6 @@
 				<0 0 40 &gic 0 40 4>,
 				<0 0 41 &gic 0 41 4>,
 				<0 0 42 &gic 0 42 4>;
-
-		/include/ "vexpress-v2m-rs1.dtsi"
 	};
 
 	site2: hsb@40000000 {
diff --git a/arch/arm/boot/dts/vexpress-v2p-ca9.dts b/arch/arm/boot/dts/vexpress-v2p-ca9.dts
index 5814460e0549..fc43873cbdff 100644
--- a/arch/arm/boot/dts/vexpress-v2p-ca9.dts
+++ b/arch/arm/boot/dts/vexpress-v2p-ca9.dts
@@ -9,6 +9,7 @@
  */
 
 /dts-v1/;
+#include "vexpress-v2m.dtsi"
 
 / {
 	model = "V2P-CA9";
@@ -301,7 +302,7 @@
 		};
 	};
 
-	smb@4000000 {
+	smb: smb@4000000 {
 		compatible = "simple-bus";
 
 		#address-cells = <2>;
@@ -357,8 +358,6 @@
 				<0 0 40 &gic 0 40 4>,
 				<0 0 41 &gic 0 41 4>,
 				<0 0 42 &gic 0 42 4>;
-
-		/include/ "vexpress-v2m.dtsi"
 	};
 
 	site2: hsb@e0000000 {
diff --git a/arch/arm/boot/dts/vf-colibri-eval-v3.dtsi b/arch/arm/boot/dts/vf-colibri-eval-v3.dtsi
index d8b2972527eb..e2da122a63f4 100644
--- a/arch/arm/boot/dts/vf-colibri-eval-v3.dtsi
+++ b/arch/arm/boot/dts/vf-colibri-eval-v3.dtsi
@@ -117,7 +117,7 @@
 		clocks = <&clk16m>;
 		spi-max-frequency = <10000000>;
 		interrupt-parent = <&gpio1>;
-		interrupts = <11 GPIO_ACTIVE_LOW>;
+		interrupts = <11 IRQ_TYPE_EDGE_RISING>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
index 782b69a3acdf..bd79e00bf615 100644
--- a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
+++ b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
@@ -70,8 +70,6 @@
 				compatible = "marvell,mv88e6085";
 				pinctrl-0 = <&pinctrl_gpio_switch0>;
 				pinctrl-names = "default";
-				#address-cells = <1>;
-				#size-cells = <0>;
 				reg = <0>;
 				dsa,member = <0 0>;
 				interrupt-parent = <&gpio0>;
@@ -156,8 +154,6 @@
 				compatible = "marvell,mv88e6085";
 				pinctrl-0 = <&pinctrl_gpio_switch1>;
 				pinctrl-names = "default";
-				#address-cells = <1>;
-				#size-cells = <0>;
 				reg = <0>;
 				dsa,member = <0 1>;
 				interrupt-parent = <&gpio0>;
@@ -243,8 +239,6 @@
 
 			switch2: switch@0 {
 				compatible = "marvell,mv88e6085";
-				#address-cells = <1>;
-				#size-cells = <0>;
 				reg = <0>;
 				dsa,member = <0 2>;
 
diff --git a/arch/arm/boot/dts/vf610-zii-dev-rev-c.dts b/arch/arm/boot/dts/vf610-zii-dev-rev-c.dts
index c6f134c78303..0b1e94c6f25b 100644
--- a/arch/arm/boot/dts/vf610-zii-dev-rev-c.dts
+++ b/arch/arm/boot/dts/vf610-zii-dev-rev-c.dts
@@ -69,8 +69,6 @@
 				compatible = "marvell,mv88e6190";
 				pinctrl-0 = <&pinctrl_gpio_switch0>;
 				pinctrl-names = "default";
-				#address-cells = <1>;
-				#size-cells = <0>;
 				reg = <0>;
 				dsa,member = <0 0>;
 				eeprom-length = <65536>;
@@ -166,8 +164,6 @@
 				compatible = "marvell,mv88e6190";
 				pinctrl-0 = <&pinctrl_gpio_switch1>;
 				pinctrl-names = "default";
-				#address-cells = <1>;
-				#size-cells = <0>;
 				reg = <0>;
 				dsa,member = <0 1>;
 				eeprom-length = <65536>;
diff --git a/arch/arm/boot/dts/vf610-zii-dev.dtsi b/arch/arm/boot/dts/vf610-zii-dev.dtsi
index 4890b8a5aa44..5ae5abfe1d55 100644
--- a/arch/arm/boot/dts/vf610-zii-dev.dtsi
+++ b/arch/arm/boot/dts/vf610-zii-dev.dtsi
@@ -222,6 +222,10 @@
 	status = "okay";
 };
 
+&tempsensor {
+	io-channels = <&adc0 16>;
+};
+
 &iomuxc {
 	pinctrl_adc0_ad5: adc0ad5grp {
 		fsl,pins = <
diff --git a/arch/arm/boot/dts/vfxxx.dtsi b/arch/arm/boot/dts/vfxxx.dtsi
index c3f09b737924..d392794d9c13 100644
--- a/arch/arm/boot/dts/vfxxx.dtsi
+++ b/arch/arm/boot/dts/vfxxx.dtsi
@@ -84,7 +84,7 @@
 		mask = <0x1000>;
 	};
 
-	iio-hwmon {
+	tempsensor: iio-hwmon {
 		compatible = "iio-hwmon";
 		io-channels = <&adc0 16>, <&adc1 16>;
 	};
diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile
index 70b4a14ed993..1e9f7af8f70f 100644
--- a/arch/arm/common/Makefile
+++ b/arch/arm/common/Makefile
@@ -10,6 +10,7 @@ obj-$(CONFIG_DMABOUNCE)		+= dmabounce.o
 obj-$(CONFIG_SHARP_LOCOMO)	+= locomo.o
 obj-$(CONFIG_SHARP_PARAM)	+= sharpsl_param.o
 obj-$(CONFIG_SHARP_SCOOP)	+= scoop.o
+obj-$(CONFIG_SMP)		+= secure_cntvoff.o
 obj-$(CONFIG_PCI_HOST_ITE8152)  += it8152.o
 obj-$(CONFIG_MCPM)		+= mcpm_head.o mcpm_entry.o mcpm_platsmp.o vlock.o
 CFLAGS_REMOVE_mcpm_entry.o	= -pg
diff --git a/arch/arm/common/mcpm_entry.c b/arch/arm/common/mcpm_entry.c
index 2b913f17d50f..ad574d20415c 100644
--- a/arch/arm/common/mcpm_entry.c
+++ b/arch/arm/common/mcpm_entry.c
@@ -9,6 +9,7 @@
  * published by the Free Software Foundation.
  */
 
+#include <linux/export.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/irqflags.h>
@@ -174,6 +175,7 @@ bool mcpm_is_available(void)
 {
 	return (platform_ops) ? true : false;
 }
+EXPORT_SYMBOL_GPL(mcpm_is_available);
 
 /*
  * We can't use regular spinlocks. In the switcher case, it is possible
diff --git a/arch/arm/common/secure_cntvoff.S b/arch/arm/common/secure_cntvoff.S
new file mode 100644
index 000000000000..53fc7bdb6c2e
--- /dev/null
+++ b/arch/arm/common/secure_cntvoff.S
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2014 Renesas Electronics Corporation
+ *
+ * Initialization of CNTVOFF register from secure mode
+ *
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+ENTRY(secure_cntvoff_init)
+	.arch	armv7-a
+	/*
+	 * CNTVOFF has to be initialized either from non-secure Hypervisor
+	 * mode or secure Monitor mode with SCR.NS==1. If TrustZone is enabled
+	 * then it should be handled by the secure code. The CPU must implement
+	 * the virtualization extensions.
+	 */
+	cps	#MON_MODE
+	mrc	p15, 0, r1, c1, c1, 0		/* Get Secure Config */
+	orr	r0, r1, #1
+	mcr	p15, 0, r0, c1, c1, 0		/* Set Non Secure bit */
+	isb
+	mov	r0, #0
+	mcrr	p15, 4, r0, r0, c14		/* CNTVOFF = 0 */
+	isb
+	mcr	p15, 0, r1, c1, c1, 0		/* Set Secure bit */
+	isb
+	cps	#SVC_MODE
+	ret	lr
+ENDPROC(secure_cntvoff_init)
diff --git a/arch/arm/configs/bcm2835_defconfig b/arch/arm/configs/bcm2835_defconfig
index 8682b15336b9..e4d188f0a4b4 100644
--- a/arch/arm/configs/bcm2835_defconfig
+++ b/arch/arm/configs/bcm2835_defconfig
@@ -64,6 +64,7 @@ CONFIG_BLK_DEV_SD=y
 CONFIG_SCSI_CONSTANTS=y
 CONFIG_SCSI_SCAN_ASYNC=y
 CONFIG_NETDEVICES=y
+CONFIG_USB_LAN78XX=y
 CONFIG_USB_USBNET=y
 CONFIG_USB_NET_SMSC95XX=y
 CONFIG_BRCMFMAC=m
@@ -127,6 +128,7 @@ CONFIG_LEDS_TRIGGER_CAMERA=y
 CONFIG_DMADEVICES=y
 CONFIG_DMA_BCM2835=y
 CONFIG_STAGING=y
+CONFIG_BCM2835_VCHIQ=m
 CONFIG_MAILBOX=y
 CONFIG_BCM2835_MBOX=y
 # CONFIG_IOMMU_SUPPORT is not set
diff --git a/arch/arm/configs/davinci_all_defconfig b/arch/arm/configs/davinci_all_defconfig
index c302a04e8cbc..21b2d7791df4 100644
--- a/arch/arm/configs/davinci_all_defconfig
+++ b/arch/arm/configs/davinci_all_defconfig
@@ -56,7 +56,7 @@ CONFIG_IP_PNP_DHCP=y
 CONFIG_NETFILTER=y
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FW_LOADER is not set
+CONFIG_FW_LOADER=m
 CONFIG_DMA_CMA=y
 CONFIG_DA8XX_MSTPRI=y
 CONFIG_MTD=m
@@ -212,6 +212,8 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_OMAP=m
 CONFIG_DMADEVICES=y
 CONFIG_TI_EDMA=y
+CONFIG_REMOTEPROC=m
+CONFIG_DA8XX_REMOTEPROC=m
 CONFIG_MEMORY=y
 CONFIG_TI_AEMIF=m
 CONFIG_DA8XX_DDRCTL=y
diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig
index 629189c62fd1..85b2369d6b20 100644
--- a/arch/arm/configs/exynos_defconfig
+++ b/arch/arm/configs/exynos_defconfig
@@ -208,6 +208,7 @@ CONFIG_DRM_EXYNOS_DSI=y
 CONFIG_DRM_EXYNOS_HDMI=y
 CONFIG_DRM_PANEL_SIMPLE=y
 CONFIG_DRM_PANEL_SAMSUNG_LD9040=y
+CONFIG_DRM_PANEL_SAMSUNG_S6E63J0X03=y
 CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0=y
 CONFIG_DRM_NXP_PTN3460=y
 CONFIG_DRM_PARADE_PS8622=y
diff --git a/arch/arm/configs/imx_v6_v7_defconfig b/arch/arm/configs/imx_v6_v7_defconfig
index 3a308437b088..f70507ab91ee 100644
--- a/arch/arm/configs/imx_v6_v7_defconfig
+++ b/arch/arm/configs/imx_v6_v7_defconfig
@@ -38,6 +38,7 @@ CONFIG_SOC_IMX51=y
 CONFIG_SOC_IMX53=y
 CONFIG_SOC_IMX6Q=y
 CONFIG_SOC_IMX6SL=y
+CONFIG_SOC_IMX6SLL=y
 CONFIG_SOC_IMX6SX=y
 CONFIG_SOC_IMX6UL=y
 CONFIG_SOC_IMX7D=y
@@ -153,6 +154,9 @@ CONFIG_USB_RTL8152=m
 CONFIG_USB_USBNET=y
 CONFIG_USB_NET_CDC_EEM=m
 CONFIG_BRCMFMAC=m
+CONFIG_MWIFIEX=m
+CONFIG_MWIFIEX_SDIO=m
+CONFIG_MWIFIEX_PCIE=m
 CONFIG_WL12XX=m
 CONFIG_WL18XX=m
 CONFIG_WLCORE_SDIO=m
@@ -199,6 +203,7 @@ CONFIG_SPI_GPIO=y
 CONFIG_SPI_IMX=y
 CONFIG_SPI_FSL_DSPI=y
 CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_MAX732X=y
 CONFIG_GPIO_MC9S08DZ60=y
 CONFIG_GPIO_PCA953X=y
 CONFIG_GPIO_STMPE=y
@@ -214,11 +219,13 @@ CONFIG_CPU_THERMAL=y
 CONFIG_IMX_THERMAL=y
 CONFIG_WATCHDOG=y
 CONFIG_DA9062_WATCHDOG=y
+CONFIG_RN5T618_WATCHDOG=y
 CONFIG_IMX2_WDT=y
 CONFIG_MFD_DA9052_I2C=y
 CONFIG_MFD_DA9062=y
 CONFIG_MFD_MC13XXX_SPI=y
 CONFIG_MFD_MC13XXX_I2C=y
+CONFIG_MFD_RN5T618=y
 CONFIG_MFD_STMPE=y
 CONFIG_REGULATOR=y
 CONFIG_REGULATOR_FIXED_VOLTAGE=y
@@ -229,6 +236,7 @@ CONFIG_REGULATOR_GPIO=y
 CONFIG_REGULATOR_MC13783=y
 CONFIG_REGULATOR_MC13892=y
 CONFIG_REGULATOR_PFUZE100=y
+CONFIG_REGULATOR_RN5T618=y
 CONFIG_RC_CORE=y
 CONFIG_RC_DEVICES=y
 CONFIG_IR_GPIO_CIR=y
@@ -374,6 +382,7 @@ CONFIG_PWM=y
 CONFIG_PWM_FSL_FTM=y
 CONFIG_PWM_IMX=y
 CONFIG_NVMEM_IMX_OCOTP=y
+CONFIG_NVMEM_VF610_OCOTP=y
 CONFIG_TEE=y
 CONFIG_OPTEE=y
 CONFIG_MUX_MMIO=y
diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig
index e6b3c96d4c09..7e1c543162c3 100644
--- a/arch/arm/configs/multi_v7_defconfig
+++ b/arch/arm/configs/multi_v7_defconfig
@@ -90,6 +90,7 @@ CONFIG_ARCH_R8A73A4=y
 CONFIG_ARCH_R8A7740=y
 CONFIG_ARCH_R8A7743=y
 CONFIG_ARCH_R8A7745=y
+CONFIG_ARCH_R8A77470=y
 CONFIG_ARCH_R8A7778=y
 CONFIG_ARCH_R8A7779=y
 CONFIG_ARCH_R8A7790=y
@@ -187,6 +188,8 @@ CONFIG_B53_MMAP_DRIVER=m
 CONFIG_B53_SRAB_DRIVER=m
 CONFIG_CAN_SUN4I=y
 CONFIG_BT=m
+CONFIG_BT_HCIUART=m
+CONFIG_BT_HCIUART_BCM=y
 CONFIG_BT_MRVL=m
 CONFIG_BT_MRVL_SDIO=m
 CONFIG_CFG80211=m
@@ -280,6 +283,7 @@ CONFIG_FIXED_PHY=y
 CONFIG_ROCKCHIP_PHY=y
 CONFIG_USB_PEGASUS=y
 CONFIG_USB_RTL8152=m
+CONFIG_USB_LAN78XX=m
 CONFIG_USB_USBNET=y
 CONFIG_USB_NET_SMSC75XX=y
 CONFIG_USB_NET_SMSC95XX=y
@@ -360,10 +364,12 @@ CONFIG_SERIAL_ST_ASC=y
 CONFIG_SERIAL_ST_ASC_CONSOLE=y
 CONFIG_SERIAL_STM32=y
 CONFIG_SERIAL_STM32_CONSOLE=y
+CONFIG_SERIAL_DEV_BUS=y
 CONFIG_HVC_DRIVER=y
 CONFIG_VIRTIO_CONSOLE=y
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_DAVINCI=y
+CONFIG_I2C_MESON=y
 CONFIG_I2C_MUX=y
 CONFIG_I2C_ARB_GPIO_CHALLENGE=m
 CONFIG_I2C_MUX_PCA954x=y
@@ -385,6 +391,7 @@ CONFIG_I2C_S3C2410=y
 CONFIG_I2C_SH_MOBILE=y
 CONFIG_I2C_SIRF=y
 CONFIG_I2C_ST=y
+CONFIG_I2C_STM32F7=y
 CONFIG_I2C_SUN6I_P2WI=y
 CONFIG_I2C_TEGRA=y
 CONFIG_I2C_UNIPHIER=y
@@ -497,6 +504,7 @@ CONFIG_TEGRA_WATCHDOG=m
 CONFIG_MESON_WATCHDOG=y
 CONFIG_DW_WATCHDOG=y
 CONFIG_DIGICOLOR_WATCHDOG=y
+CONFIG_RENESAS_WDT=m
 CONFIG_BCM2835_WDT=y
 CONFIG_BCM47XX_WDT=y
 CONFIG_BCM7038_WDT=m
@@ -638,6 +646,7 @@ CONFIG_DRM_SUN4I=m
 CONFIG_DRM_FSL_DCU=m
 CONFIG_DRM_TEGRA=y
 CONFIG_DRM_PANEL_SAMSUNG_LD9040=m
+CONFIG_DRM_PANEL_SAMSUNG_S6E63J0X03=m
 CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0=m
 CONFIG_DRM_PANEL_SIMPLE=y
 CONFIG_DRM_SII9234=m
@@ -650,7 +659,6 @@ CONFIG_FB_EFI=y
 CONFIG_FB_WM8505=y
 CONFIG_FB_SH_MOBILE_LCDC=y
 CONFIG_FB_SIMPLE=y
-CONFIG_FB_SH_MOBILE_MERAM=y
 CONFIG_BACKLIGHT_LCD_SUPPORT=y
 CONFIG_BACKLIGHT_CLASS_DEVICE=y
 CONFIG_LCD_PLATFORM=m
@@ -947,6 +955,7 @@ CONFIG_PWM_ATMEL=m
 CONFIG_PWM_ATMEL_HLCDC_PWM=m
 CONFIG_PWM_ATMEL_TCB=m
 CONFIG_PWM_FSL_FTM=m
+CONFIG_PWM_MESON=m
 CONFIG_PWM_RCAR=m
 CONFIG_PWM_RENESAS_TPU=y
 CONFIG_PWM_ROCKCHIP=m
@@ -972,6 +981,7 @@ CONFIG_PHY_QCOM_APQ8064_SATA=m
 CONFIG_PHY_MIPHY28LP=y
 CONFIG_PHY_RCAR_GEN2=m
 CONFIG_PHY_STIH407_USB=y
+CONFIG_PHY_STM32_USBPHYC=y
 CONFIG_PHY_SUN4I_USB=y
 CONFIG_PHY_SUN9I_USB=y
 CONFIG_PHY_SAMSUNG_USB2=m
diff --git a/arch/arm/configs/shmobile_defconfig b/arch/arm/configs/shmobile_defconfig
index a701601fbd76..b49887e86a3d 100644
--- a/arch/arm/configs/shmobile_defconfig
+++ b/arch/arm/configs/shmobile_defconfig
@@ -14,6 +14,7 @@ CONFIG_ARCH_R8A73A4=y
 CONFIG_ARCH_R8A7740=y
 CONFIG_ARCH_R8A7743=y
 CONFIG_ARCH_R8A7745=y
+CONFIG_ARCH_R8A77470=y
 CONFIG_ARCH_R8A7778=y
 CONFIG_ARCH_R8A7779=y
 CONFIG_ARCH_R8A7790=y
@@ -127,6 +128,7 @@ CONFIG_CPU_THERMAL=y
 CONFIG_RCAR_THERMAL=y
 CONFIG_WATCHDOG=y
 CONFIG_DA9063_WATCHDOG=y
+CONFIG_RENESAS_WDT=y
 CONFIG_MFD_AS3711=y
 CONFIG_MFD_DA9063=y
 CONFIG_REGULATOR_FIXED_VOLTAGE=y
@@ -156,7 +158,6 @@ CONFIG_DRM_DUMB_VGA_DAC=y
 CONFIG_DRM_I2C_ADV7511=y
 CONFIG_DRM_I2C_ADV7511_AUDIO=y
 CONFIG_FB_SH_MOBILE_LCDC=y
-CONFIG_FB_SH_MOBILE_MERAM=y
 # CONFIG_LCD_CLASS_DEVICE is not set
 # CONFIG_BACKLIGHT_GENERIC is not set
 CONFIG_BACKLIGHT_PWM=y
diff --git a/arch/arm/crypto/sha1-armv4-large.S b/arch/arm/crypto/sha1-armv4-large.S
index 99207c45ec10..f82cd8cf5a09 100644
--- a/arch/arm/crypto/sha1-armv4-large.S
+++ b/arch/arm/crypto/sha1-armv4-large.S
@@ -1,4 +1,14 @@
 #define __ARM_ARCH__ __LINUX_ARM_ARCH__
+@ SPDX-License-Identifier: GPL-2.0
+
+@ This code is taken from the OpenSSL project but the author (Andy Polyakov)
+@ has relicensed it under the GPLv2. Therefore this program is free software;
+@ you can redistribute it and/or modify it under the terms of the GNU General
+@ Public License version 2 as published by the Free Software Foundation.
+@
+@ The original headers, including the original license headers, are
+@ included below for completeness.
+
 @ ====================================================================
 @ Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
 @ project. The module is, however, dual licensed under OpenSSL and
diff --git a/arch/arm/crypto/sha256-armv4.pl b/arch/arm/crypto/sha256-armv4.pl
index fac0533ea633..b9ec44060ed3 100644
--- a/arch/arm/crypto/sha256-armv4.pl
+++ b/arch/arm/crypto/sha256-armv4.pl
@@ -1,12 +1,19 @@
 #!/usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0
+
+# This code is taken from the OpenSSL project but the author (Andy Polyakov)
+# has relicensed it under the GPLv2. Therefore this program is free software;
+# you can redistribute it and/or modify it under the terms of the GNU General
+# Public License version 2 as published by the Free Software Foundation.
+#
+# The original headers, including the original license headers, are
+# included below for completeness.
 
 # ====================================================================
 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
 # project. The module is, however, dual licensed under OpenSSL and
 # CRYPTOGAMS licenses depending on where you obtain it. For further
 # details see http://www.openssl.org/~appro/cryptogams/.
-#
-# Permission to use under GPL terms is granted.
 # ====================================================================
 
 # SHA256 block procedure for ARMv4. May 2007.
diff --git a/arch/arm/crypto/sha256-core.S_shipped b/arch/arm/crypto/sha256-core.S_shipped
index 555a1a8eec90..3b58300d611c 100644
--- a/arch/arm/crypto/sha256-core.S_shipped
+++ b/arch/arm/crypto/sha256-core.S_shipped
@@ -1,11 +1,18 @@
+@ SPDX-License-Identifier: GPL-2.0
+
+@ This code is taken from the OpenSSL project but the author (Andy Polyakov)
+@ has relicensed it under the GPLv2. Therefore this program is free software;
+@ you can redistribute it and/or modify it under the terms of the GNU General
+@ Public License version 2 as published by the Free Software Foundation.
+@
+@ The original headers, including the original license headers, are
+@ included below for completeness.
 
 @ ====================================================================
 @ Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
 @ project. The module is, however, dual licensed under OpenSSL and
 @ CRYPTOGAMS licenses depending on where you obtain it. For further
 @ details see http://www.openssl.org/~appro/cryptogams/.
-@
-@ Permission to use under GPL terms is granted.
 @ ====================================================================
 
 @ SHA256 block procedure for ARMv4. May 2007.
diff --git a/arch/arm/crypto/sha512-armv4.pl b/arch/arm/crypto/sha512-armv4.pl
index a2b11a844357..fb5d15048c0b 100644
--- a/arch/arm/crypto/sha512-armv4.pl
+++ b/arch/arm/crypto/sha512-armv4.pl
@@ -1,12 +1,19 @@
 #!/usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0
+
+# This code is taken from the OpenSSL project but the author (Andy Polyakov)
+# has relicensed it under the GPLv2. Therefore this program is free software;
+# you can redistribute it and/or modify it under the terms of the GNU General
+# Public License version 2 as published by the Free Software Foundation.
+#
+# The original headers, including the original license headers, are
+# included below for completeness.
 
 # ====================================================================
 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
 # project. The module is, however, dual licensed under OpenSSL and
 # CRYPTOGAMS licenses depending on where you obtain it. For further
 # details see http://www.openssl.org/~appro/cryptogams/.
-#
-# Permission to use under GPL terms is granted.
 # ====================================================================
 
 # SHA512 block procedure for ARMv4. September 2007.
diff --git a/arch/arm/crypto/sha512-core.S_shipped b/arch/arm/crypto/sha512-core.S_shipped
index 3694c4d4ca2b..b1c334a49cda 100644
--- a/arch/arm/crypto/sha512-core.S_shipped
+++ b/arch/arm/crypto/sha512-core.S_shipped
@@ -1,11 +1,18 @@
+@ SPDX-License-Identifier: GPL-2.0
+
+@ This code is taken from the OpenSSL project but the author (Andy Polyakov)
+@ has relicensed it under the GPLv2. Therefore this program is free software;
+@ you can redistribute it and/or modify it under the terms of the GNU General
+@ Public License version 2 as published by the Free Software Foundation.
+@
+@ The original headers, including the original license headers, are
+@ included below for completeness.
 
 @ ====================================================================
 @ Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
 @ project. The module is, however, dual licensed under OpenSSL and
 @ CRYPTOGAMS licenses depending on where you obtain it. For further
 @ details see http://www.openssl.org/~appro/cryptogams/.
-@
-@ Permission to use under GPL terms is granted.
 @ ====================================================================
 
 @ SHA512 block procedure for ARMv4. September 2007.
diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild
index 873e3c189279..1d66db9c9db5 100644
--- a/arch/arm/include/asm/Kbuild
+++ b/arch/arm/include/asm/Kbuild
@@ -1,3 +1,4 @@
+generic-y += compat.h
 generic-y += current.h
 generic-y += early_ioremap.h
 generic-y += emergency-restart.h
diff --git a/arch/arm/include/asm/bugs.h b/arch/arm/include/asm/bugs.h
index a97f1ea708d1..73a99c72a930 100644
--- a/arch/arm/include/asm/bugs.h
+++ b/arch/arm/include/asm/bugs.h
@@ -10,12 +10,14 @@
 #ifndef __ASM_BUGS_H
 #define __ASM_BUGS_H
 
-#ifdef CONFIG_MMU
 extern void check_writebuffer_bugs(void);
 
-#define check_bugs() check_writebuffer_bugs()
+#ifdef CONFIG_MMU
+extern void check_bugs(void);
+extern void check_other_bugs(void);
 #else
 #define check_bugs() do { } while (0)
+#define check_other_bugs() do { } while (0)
 #endif
 
 #endif
diff --git a/arch/arm/include/asm/cp15.h b/arch/arm/include/asm/cp15.h
index 4c9fa72b59f5..07e27f212dc7 100644
--- a/arch/arm/include/asm/cp15.h
+++ b/arch/arm/include/asm/cp15.h
@@ -65,6 +65,9 @@
 #define __write_sysreg(v, r, w, c, t)	asm volatile(w " " c : : "r" ((t)(v)))
 #define write_sysreg(v, ...)		__write_sysreg(v, __VA_ARGS__)
 
+#define BPIALL				__ACCESS_CP15(c7, 0, c5, 6)
+#define ICIALLU				__ACCESS_CP15(c7, 0, c5, 0)
+
 extern unsigned long cr_alignment;	/* defined in entry-armv.S */
 
 static inline unsigned long get_cr(void)
diff --git a/arch/arm/include/asm/cputype.h b/arch/arm/include/asm/cputype.h
index cb546425da8a..0d289240b6ca 100644
--- a/arch/arm/include/asm/cputype.h
+++ b/arch/arm/include/asm/cputype.h
@@ -2,9 +2,6 @@
 #ifndef __ASM_ARM_CPUTYPE_H
 #define __ASM_ARM_CPUTYPE_H
 
-#include <linux/stringify.h>
-#include <linux/kernel.h>
-
 #define CPUID_ID	0
 #define CPUID_CACHETYPE	1
 #define CPUID_TCM	2
@@ -62,6 +59,7 @@
 	((mpidr >> (MPIDR_LEVEL_BITS * level)) & MPIDR_LEVEL_MASK)
 
 #define ARM_CPU_IMP_ARM			0x41
+#define ARM_CPU_IMP_BRCM		0x42
 #define ARM_CPU_IMP_DEC			0x44
 #define ARM_CPU_IMP_INTEL		0x69
 
@@ -77,8 +75,17 @@
 #define ARM_CPU_PART_CORTEX_A12		0x4100c0d0
 #define ARM_CPU_PART_CORTEX_A17		0x4100c0e0
 #define ARM_CPU_PART_CORTEX_A15		0x4100c0f0
+#define ARM_CPU_PART_CORTEX_A53		0x4100d030
+#define ARM_CPU_PART_CORTEX_A57		0x4100d070
+#define ARM_CPU_PART_CORTEX_A72		0x4100d080
+#define ARM_CPU_PART_CORTEX_A73		0x4100d090
+#define ARM_CPU_PART_CORTEX_A75		0x4100d0a0
 #define ARM_CPU_PART_MASK		0xff00fff0
 
+/* Broadcom implemented processors */
+#define ARM_CPU_PART_BRAHMA_B15		0x420000f0
+#define ARM_CPU_PART_BRAHMA_B53		0x42001000
+
 /* DEC implemented cores */
 #define ARM_CPU_PART_SA1100		0x4400a110
 
@@ -98,6 +105,11 @@
 /* Qualcomm implemented cores */
 #define ARM_CPU_PART_SCORPION		0x510002d0
 
+#ifndef __ASSEMBLY__
+
+#include <linux/stringify.h>
+#include <linux/kernel.h>
+
 extern unsigned int processor_id;
 
 #ifdef CONFIG_CPU_CP15
@@ -326,4 +338,6 @@ static inline int __attribute_const__ cpuid_feature_extract_field(u32 features,
 #define cpuid_feature_extract(reg, field) \
 	cpuid_feature_extract_field(read_cpuid_ext(reg), field)
 
+#endif /* __ASSEMBLY__ */
+
 #endif
diff --git a/arch/arm/include/asm/kgdb.h b/arch/arm/include/asm/kgdb.h
index 3b73fdcf3627..8de1100d1067 100644
--- a/arch/arm/include/asm/kgdb.h
+++ b/arch/arm/include/asm/kgdb.h
@@ -77,7 +77,7 @@ extern int kgdb_fault_expected;
 
 #define KGDB_MAX_NO_CPUS	1
 #define BUFMAX			400
-#define NUMREGBYTES		(DBG_MAX_REG_NUM << 2)
+#define NUMREGBYTES		(GDB_MAX_REGS << 2)
 #define NUMCRITREGBYTES		(32 << 2)
 
 #define _R0			0
diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index 5a953ecb0d78..231e87ad45d5 100644
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -61,8 +61,6 @@ struct kvm_vcpu;
 extern char __kvm_hyp_init[];
 extern char __kvm_hyp_init_end[];
 
-extern char __kvm_hyp_vector[];
-
 extern void __kvm_flush_vm_context(void);
 extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
 extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index c7c28c885a19..f9ed85226ef7 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -21,6 +21,7 @@
 
 #include <linux/types.h>
 #include <linux/kvm_types.h>
+#include <asm/cputype.h>
 #include <asm/kvm.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_mmio.h>
@@ -280,6 +281,7 @@ void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
 
 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
 
+static inline bool kvm_arch_check_sve_has_vhe(void) { return true; }
 static inline void kvm_arch_hardware_unsetup(void) {}
 static inline void kvm_arch_sync_events(struct kvm *kvm) {}
 static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
@@ -303,16 +305,30 @@ int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
 int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
 			       struct kvm_device_attr *attr);
 
-/* All host FP/SIMD state is restored on guest exit, so nothing to save: */
-static inline void kvm_fpsimd_flush_cpu_state(void) {}
+/*
+ * VFP/NEON switching is all done by the hyp switch code, so no need to
+ * coordinate with host context handling for this state:
+ */
+static inline void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu) {}
+static inline void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) {}
+static inline void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) {}
 
 static inline void kvm_arm_vhe_guest_enter(void) {}
 static inline void kvm_arm_vhe_guest_exit(void) {}
 
 static inline bool kvm_arm_harden_branch_predictor(void)
 {
-	/* No way to detect it yet, pretend it is not there. */
-	return false;
+	switch(read_cpuid_part()) {
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+	case ARM_CPU_PART_BRAHMA_B15:
+	case ARM_CPU_PART_CORTEX_A12:
+	case ARM_CPU_PART_CORTEX_A15:
+	case ARM_CPU_PART_CORTEX_A17:
+		return true;
+#endif
+	default:
+		return false;
+	}
 }
 
 static inline void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu) {}
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index f675162663f0..c94d291fd1a8 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -327,7 +327,28 @@ static inline int kvm_read_guest_lock(struct kvm *kvm,
 
 static inline void *kvm_get_hyp_vector(void)
 {
-	return kvm_ksym_ref(__kvm_hyp_vector);
+	switch(read_cpuid_part()) {
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+	case ARM_CPU_PART_CORTEX_A12:
+	case ARM_CPU_PART_CORTEX_A17:
+	{
+		extern char __kvm_hyp_vector_bp_inv[];
+		return kvm_ksym_ref(__kvm_hyp_vector_bp_inv);
+	}
+
+	case ARM_CPU_PART_BRAHMA_B15:
+	case ARM_CPU_PART_CORTEX_A15:
+	{
+		extern char __kvm_hyp_vector_ic_inv[];
+		return kvm_ksym_ref(__kvm_hyp_vector_ic_inv);
+	}
+#endif
+	default:
+	{
+		extern char __kvm_hyp_vector[];
+		return kvm_ksym_ref(__kvm_hyp_vector);
+	}
+	}
 }
 
 static inline int kvm_map_vectors(void)
diff --git a/arch/arm/include/asm/mpu.h b/arch/arm/include/asm/mpu.h
index 6d1491c8ee22..5e088c83d3d8 100644
--- a/arch/arm/include/asm/mpu.h
+++ b/arch/arm/include/asm/mpu.h
@@ -12,60 +12,101 @@
 /* ID_MMFR0 data relevant to MPU */
 #define MMFR0_PMSA		(0xF << 4)
 #define MMFR0_PMSAv7		(3 << 4)
+#define MMFR0_PMSAv8		(4 << 4)
 
 /* MPU D/I Size Register fields */
-#define MPU_RSR_SZ		1
-#define MPU_RSR_EN		0
-#define MPU_RSR_SD		8
+#define PMSAv7_RSR_SZ		1
+#define PMSAv7_RSR_EN		0
+#define PMSAv7_RSR_SD		8
 
 /* Number of subregions (SD) */
-#define MPU_NR_SUBREGS		8
-#define MPU_MIN_SUBREG_SIZE	256
+#define PMSAv7_NR_SUBREGS	8
+#define PMSAv7_MIN_SUBREG_SIZE	256
 
 /* The D/I RSR value for an enabled region spanning the whole of memory */
-#define MPU_RSR_ALL_MEM		63
+#define PMSAv7_RSR_ALL_MEM	63
 
 /* Individual bits in the DR/IR ACR */
-#define MPU_ACR_XN		(1 << 12)
-#define MPU_ACR_SHARED		(1 << 2)
+#define PMSAv7_ACR_XN		(1 << 12)
+#define PMSAv7_ACR_SHARED	(1 << 2)
 
 /* C, B and TEX[2:0] bits only have semantic meanings when grouped */
-#define MPU_RGN_CACHEABLE	0xB
-#define MPU_RGN_SHARED_CACHEABLE (MPU_RGN_CACHEABLE | MPU_ACR_SHARED)
-#define MPU_RGN_STRONGLY_ORDERED 0
+#define PMSAv7_RGN_CACHEABLE		0xB
+#define PMSAv7_RGN_SHARED_CACHEABLE	(PMSAv7_RGN_CACHEABLE | PMSAv7_ACR_SHARED)
+#define PMSAv7_RGN_STRONGLY_ORDERED	0
 
 /* Main region should only be shared for SMP */
 #ifdef CONFIG_SMP
-#define MPU_RGN_NORMAL		(MPU_RGN_CACHEABLE | MPU_ACR_SHARED)
+#define PMSAv7_RGN_NORMAL	(PMSAv7_RGN_CACHEABLE | PMSAv7_ACR_SHARED)
 #else
-#define MPU_RGN_NORMAL		MPU_RGN_CACHEABLE
+#define PMSAv7_RGN_NORMAL	PMSAv7_RGN_CACHEABLE
 #endif
 
 /* Access permission bits of ACR (only define those that we use)*/
-#define MPU_AP_PL1RO_PL0NA	(0x5 << 8)
-#define MPU_AP_PL1RW_PL0RW	(0x3 << 8)
-#define MPU_AP_PL1RW_PL0R0	(0x2 << 8)
-#define MPU_AP_PL1RW_PL0NA	(0x1 << 8)
+#define PMSAv7_AP_PL1RO_PL0NA	(0x5 << 8)
+#define PMSAv7_AP_PL1RW_PL0RW	(0x3 << 8)
+#define PMSAv7_AP_PL1RW_PL0R0	(0x2 << 8)
+#define PMSAv7_AP_PL1RW_PL0NA	(0x1 << 8)
+
+#define PMSAv8_BAR_XN		1
+
+#define PMSAv8_LAR_EN		1
+#define PMSAv8_LAR_IDX(n)	(((n) & 0x7) << 1)
+
+
+#define PMSAv8_AP_PL1RW_PL0NA	(0 << 1)
+#define PMSAv8_AP_PL1RW_PL0RW	(1 << 1)
+#define PMSAv8_AP_PL1RO_PL0RO	(3 << 1)
+
+#ifdef CONFIG_SMP
+#define PMSAv8_RGN_SHARED	(3 << 3) // inner sharable
+#else
+#define PMSAv8_RGN_SHARED	(0 << 3)
+#endif
+
+#define PMSAv8_RGN_DEVICE_nGnRnE	0
+#define PMSAv8_RGN_NORMAL		1
+
+#define PMSAv8_MAIR(attr, mt)	((attr) << ((mt) * 8))
+
+#ifdef CONFIG_CPU_V7M
+#define PMSAv8_MINALIGN		32
+#else
+#define PMSAv8_MINALIGN		64
+#endif
 
 /* For minimal static MPU region configurations */
-#define MPU_PROBE_REGION	0
-#define MPU_BG_REGION		1
-#define MPU_RAM_REGION		2
-#define MPU_ROM_REGION		3
+#define PMSAv7_PROBE_REGION	0
+#define PMSAv7_BG_REGION	1
+#define PMSAv7_RAM_REGION	2
+#define PMSAv7_ROM_REGION	3
+
+/* Fixed for PMSAv8 only */
+#define PMSAv8_XIP_REGION	0
+#define PMSAv8_KERNEL_REGION	1
 
 /* Maximum number of regions Linux is interested in */
-#define MPU_MAX_REGIONS		16
+#define MPU_MAX_REGIONS	16
 
-#define MPU_DATA_SIDE		0
-#define MPU_INSTR_SIDE		1
+#define PMSAv7_DATA_SIDE	0
+#define PMSAv7_INSTR_SIDE	1
 
 #ifndef __ASSEMBLY__
 
 struct mpu_rgn {
 	/* Assume same attributes for d/i-side  */
-	u32 drbar;
-	u32 drsr;
-	u32 dracr;
+	union {
+		u32 drbar;   /* PMSAv7 */
+		u32 prbar;   /* PMSAv8 */
+	};
+	union {
+		u32 drsr;   /* PMSAv7 */
+		u32 prlar;  /* PMSAv8 */
+	};
+	union {
+		u32 dracr;  /* PMSAv7 */
+		u32 unused; /* not used in PMSAv8 */
+	};
 };
 
 struct mpu_rgn_info {
@@ -75,16 +116,17 @@ struct mpu_rgn_info {
 extern struct mpu_rgn_info mpu_rgn_info;
 
 #ifdef CONFIG_ARM_MPU
+extern void __init pmsav7_adjust_lowmem_bounds(void);
+extern void __init pmsav8_adjust_lowmem_bounds(void);
 
-extern void __init adjust_lowmem_bounds_mpu(void);
-extern void __init mpu_setup(void);
-
+extern void __init pmsav7_setup(void);
+extern void __init pmsav8_setup(void);
 #else
-
-static inline void adjust_lowmem_bounds_mpu(void) {}
-static inline void mpu_setup(void) {}
-
-#endif /* !CONFIG_ARM_MPU */
+static inline void pmsav7_adjust_lowmem_bounds(void) {};
+static inline void pmsav8_adjust_lowmem_bounds(void) {};
+static inline void pmsav7_setup(void) {};
+static inline void pmsav8_setup(void) {};
+#endif
 
 #endif /* __ASSEMBLY__ */
 
diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h
index 4355f0ec44d6..f98baaec0a15 100644
--- a/arch/arm/include/asm/page.h
+++ b/arch/arm/include/asm/page.h
@@ -17,6 +17,8 @@
 
 #ifndef __ASSEMBLY__
 
+#include <linux/personality.h>	/* For READ_IMPLIES_EXEC */
+
 #ifndef CONFIG_MMU
 
 #include <asm/page-nommu.h>
diff --git a/arch/arm/include/asm/pci.h b/arch/arm/include/asm/pci.h
index 1f0de808d111..0abd389cf0ec 100644
--- a/arch/arm/include/asm/pci.h
+++ b/arch/arm/include/asm/pci.h
@@ -19,13 +19,6 @@ static inline int pci_proc_domain(struct pci_bus *bus)
 }
 #endif /* CONFIG_PCI_DOMAINS */
 
-/*
- * The PCI address space does equal the physical memory address space.
- * The networking and block device layers use this boolean for bounce
- * buffer decisions.
- */
-#define PCI_DMA_BUS_IS_PHYS     (1)
-
 #define HAVE_PCI_MMAP
 #define ARCH_GENERIC_PCI_MMAP_RESOURCE
 
diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h
index 2a4836087358..6d50a11d7793 100644
--- a/arch/arm/include/asm/pgtable-3level.h
+++ b/arch/arm/include/asm/pgtable-3level.h
@@ -219,7 +219,6 @@ static inline pte_t pte_mkspecial(pte_t pte)
 	pte_val(pte) |= L_PTE_SPECIAL;
 	return pte;
 }
-#define	__HAVE_ARCH_PTE_SPECIAL
 
 #define pmd_write(pmd)		(pmd_isclear((pmd), L_PMD_SECT_RDONLY))
 #define pmd_dirty(pmd)		(pmd_isset((pmd), L_PMD_SECT_DIRTY))
diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h
index f2e1af45bd6f..e25f4392e1b2 100644
--- a/arch/arm/include/asm/proc-fns.h
+++ b/arch/arm/include/asm/proc-fns.h
@@ -36,6 +36,10 @@ extern struct processor {
 	 * Set up any processor specifics
 	 */
 	void (*_proc_init)(void);
+	/*
+	 * Check for processor bugs
+	 */
+	void (*check_bugs)(void);
 	/*
 	 * Disable any processor specifics
 	 */
diff --git a/arch/arm/include/asm/secure_cntvoff.h b/arch/arm/include/asm/secure_cntvoff.h
new file mode 100644
index 000000000000..1f93aee1f630
--- /dev/null
+++ b/arch/arm/include/asm/secure_cntvoff.h
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __ASMARM_ARCH_CNTVOFF_H
+#define __ASMARM_ARCH_CNTVOFF_H
+
+extern void secure_cntvoff_init(void);
+
+#endif
diff --git a/arch/arm/include/asm/system_misc.h b/arch/arm/include/asm/system_misc.h
index 78f6db114faf..8e76db83c498 100644
--- a/arch/arm/include/asm/system_misc.h
+++ b/arch/arm/include/asm/system_misc.h
@@ -8,6 +8,7 @@
 #include <linux/linkage.h>
 #include <linux/irqflags.h>
 #include <linux/reboot.h>
+#include <linux/percpu.h>
 
 extern void cpu_init(void);
 
@@ -15,6 +16,20 @@ void soft_restart(unsigned long);
 extern void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd);
 extern void (*arm_pm_idle)(void);
 
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+typedef void (*harden_branch_predictor_fn_t)(void);
+DECLARE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn);
+static inline void harden_branch_predictor(void)
+{
+	harden_branch_predictor_fn_t fn = per_cpu(harden_branch_predictor_fn,
+						  smp_processor_id());
+	if (fn)
+		fn();
+}
+#else
+#define harden_branch_predictor() do { } while (0)
+#endif
+
 #define UDBG_UNDEFINED	(1 << 0)
 #define UDBG_SYSCALL	(1 << 1)
 #define UDBG_BADABORT	(1 << 2)
diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h
index 0bf2347495f1..3d614e90c19f 100644
--- a/arch/arm/include/asm/uaccess.h
+++ b/arch/arm/include/asm/uaccess.h
@@ -152,7 +152,7 @@ extern int __get_user_64t_4(void *);
 #define __get_user_check(x, p)						\
 	({								\
 		unsigned long __limit = current_thread_info()->addr_limit - 1; \
-		register const typeof(*(p)) __user *__p asm("r0") = (p);\
+		register typeof(*(p)) __user *__p asm("r0") = (p);	\
 		register typeof(x) __r2 asm("r2");			\
 		register unsigned long __l asm("r1") = __limit;		\
 		register int __e asm("r0");				\
diff --git a/arch/arm/include/asm/v7m.h b/arch/arm/include/asm/v7m.h
index 634e77107425..187ccf6496ad 100644
--- a/arch/arm/include/asm/v7m.h
+++ b/arch/arm/include/asm/v7m.h
@@ -64,9 +64,17 @@
 #define MPU_CTRL_ENABLE		1
 #define MPU_CTRL_PRIVDEFENA	(1 << 2)
 
-#define MPU_RNR			0x98
-#define MPU_RBAR		0x9c
-#define MPU_RASR		0xa0
+#define PMSAv7_RNR		0x98
+#define PMSAv7_RBAR		0x9c
+#define PMSAv7_RASR		0xa0
+
+#define PMSAv8_RNR		0x98
+#define PMSAv8_RBAR		0x9c
+#define PMSAv8_RLAR		0xa0
+#define PMSAv8_RBAR_A(n)	(PMSAv8_RBAR + 8*(n))
+#define PMSAv8_RLAR_A(n)	(PMSAv8_RLAR + 8*(n))
+#define PMSAv8_MAIR0		0xc0
+#define PMSAv8_MAIR1		0xc4
 
 /* Cache opeartions */
 #define	V7M_SCB_ICIALLU		0x250	/* I-cache invalidate all to PoU */
diff --git a/arch/arm/include/debug/brcmstb.S b/arch/arm/include/debug/brcmstb.S
index c826f15d2f80..0f580caa81e5 100644
--- a/arch/arm/include/debug/brcmstb.S
+++ b/arch/arm/include/debug/brcmstb.S
@@ -11,20 +11,25 @@
  * GNU General Public License for more details.
  */
 #include <linux/serial_reg.h>
+#include <asm/cputype.h>
 
 /* Physical register offset and virtual register offset */
 #define REG_PHYS_BASE		0xf0000000
+#define REG_PHYS_BASE_V7	0x08000000
 #define REG_VIRT_BASE		0xfc000000
 #define REG_PHYS_ADDR(x)	((x) + REG_PHYS_BASE)
+#define REG_PHYS_ADDR_V7(x)	((x) + REG_PHYS_BASE_V7)
 
 /* Product id can be read from here */
 #define SUN_TOP_CTRL_BASE	REG_PHYS_ADDR(0x404000)
+#define SUN_TOP_CTRL_BASE_V7	REG_PHYS_ADDR_V7(0x404000)
 
 #define UARTA_3390		REG_PHYS_ADDR(0x40a900)
 #define UARTA_7250		REG_PHYS_ADDR(0x40b400)
 #define UARTA_7260		REG_PHYS_ADDR(0x40c000)
 #define UARTA_7268		UARTA_7260
 #define UARTA_7271		UARTA_7268
+#define UARTA_7278		REG_PHYS_ADDR_V7(0x40c000)
 #define UARTA_7364		REG_PHYS_ADDR(0x40b000)
 #define UARTA_7366		UARTA_7364
 #define UARTA_74371		REG_PHYS_ADDR(0x406b00)
@@ -55,8 +60,21 @@
 		mov	\rv, #0			@ yes; record init is done
 		str	\rv, [\tmp]
 
+		/* Check for V7 memory map if B53 */
+		mrc	p15, 0, \rv, c0, c0, 0	@ get Main ID register
+		ldr	\rp, =ARM_CPU_PART_MASK
+		and	\rv, \rv, \rp
+		ldr	\rp, =ARM_CPU_PART_BRAHMA_B53	@ check for B53 CPU
+		cmp	\rv, \rp
+		bne	10f
+
+		/* if PERIPHBASE doesn't overlap REG_PHYS_BASE use V7 map */
+		mrc	p15, 1, \rv, c15, c3, 0	@ get PERIPHBASE from CBAR
+		ands	\rv, \rv, #REG_PHYS_BASE
+		ldreq	\rp, =SUN_TOP_CTRL_BASE_V7
+
 		/* Check SUN_TOP_CTRL base */
-		ldr	\rp, =SUN_TOP_CTRL_BASE	@ load SUN_TOP_CTRL PA
+10:		ldrne	\rp, =SUN_TOP_CTRL_BASE	@ load SUN_TOP_CTRL PA
 		ldr	\rv, [\rp, #0]		@ get register contents
 ARM_BE8(	rev	\rv, \rv )
 		and	\rv, \rv, #0xffffff00	@ strip revision bits [7:0]
@@ -72,6 +90,7 @@ ARM_BE8(	rev	\rv, \rv )
 27:		checkuart(\rp, \rv, 0x07437100, 74371)
 28:		checkuart(\rp, \rv, 0x74390000, 7439)
 29:		checkuart(\rp, \rv, 0x74450000, 7445)
+30:		checkuart(\rp, \rv, 0x72780000, 7278)
 
 		/* No valid UART found */
 90:		mov	\rp, #0
diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h
index caae4843cb70..16e006f708ca 100644
--- a/arch/arm/include/uapi/asm/kvm.h
+++ b/arch/arm/include/uapi/asm/kvm.h
@@ -91,6 +91,7 @@ struct kvm_regs {
 #define KVM_VGIC_V3_ADDR_TYPE_DIST	2
 #define KVM_VGIC_V3_ADDR_TYPE_REDIST	3
 #define KVM_VGIC_ITS_ADDR_TYPE		4
+#define KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION	5
 
 #define KVM_VGIC_V3_DIST_SIZE		SZ_64K
 #define KVM_VGIC_V3_REDIST_SIZE		(2 * SZ_64K)
diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile
index b59ac4bf82b8..8cad59465af3 100644
--- a/arch/arm/kernel/Makefile
+++ b/arch/arm/kernel/Makefile
@@ -31,6 +31,7 @@ else
 obj-y		+= entry-armv.o
 endif
 
+obj-$(CONFIG_MMU)		+= bugs.o
 obj-$(CONFIG_CPU_IDLE)		+= cpuidle.o
 obj-$(CONFIG_ISA_DMA_API)	+= dma.o
 obj-$(CONFIG_FIQ)		+= fiq.o fiqasm.o
diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c
index f369ece99958..27c5381518d8 100644
--- a/arch/arm/kernel/asm-offsets.c
+++ b/arch/arm/kernel/asm-offsets.c
@@ -194,9 +194,11 @@ int main(void)
   DEFINE(MPU_RNG_INFO_USED,	offsetof(struct mpu_rgn_info, used));
 
   DEFINE(MPU_RNG_SIZE,		sizeof(struct mpu_rgn));
-  DEFINE(MPU_RGN_DRBAR,		offsetof(struct mpu_rgn, drbar));
-  DEFINE(MPU_RGN_DRSR,		offsetof(struct mpu_rgn, drsr));
-  DEFINE(MPU_RGN_DRACR,		offsetof(struct mpu_rgn, dracr));
+  DEFINE(MPU_RGN_DRBAR,	offsetof(struct mpu_rgn, drbar));
+  DEFINE(MPU_RGN_DRSR,	offsetof(struct mpu_rgn, drsr));
+  DEFINE(MPU_RGN_DRACR,	offsetof(struct mpu_rgn, dracr));
+  DEFINE(MPU_RGN_PRBAR,	offsetof(struct mpu_rgn, prbar));
+  DEFINE(MPU_RGN_PRLAR,	offsetof(struct mpu_rgn, prlar));
 #endif
   return 0; 
 }
diff --git a/arch/arm/kernel/bugs.c b/arch/arm/kernel/bugs.c
new file mode 100644
index 000000000000..7be511310191
--- /dev/null
+++ b/arch/arm/kernel/bugs.c
@@ -0,0 +1,18 @@
+// SPDX-Identifier: GPL-2.0
+#include <linux/init.h>
+#include <asm/bugs.h>
+#include <asm/proc-fns.h>
+
+void check_other_bugs(void)
+{
+#ifdef MULTI_CPU
+	if (processor.check_bugs)
+		processor.check_bugs();
+#endif
+}
+
+void __init check_bugs(void)
+{
+	check_writebuffer_bugs();
+	check_other_bugs();
+}
diff --git a/arch/arm/kernel/dma.c b/arch/arm/kernel/dma.c
index e651c4d0a0d9..6739d37c2bc5 100644
--- a/arch/arm/kernel/dma.c
+++ b/arch/arm/kernel/dma.c
@@ -276,21 +276,9 @@ static int proc_dma_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int proc_dma_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_dma_show, NULL);
-}
-
-static const struct file_operations proc_dma_operations = {
-	.open		= proc_dma_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int __init proc_dma_init(void)
 {
-	proc_create("dma", 0, NULL, &proc_dma_operations);
+	proc_create_single("dma", 0, NULL, proc_dma_show);
 	return 0;
 }
 
diff --git a/arch/arm/kernel/head-nommu.S b/arch/arm/kernel/head-nommu.S
index 2e38f85b757a..dd546d65a383 100644
--- a/arch/arm/kernel/head-nommu.S
+++ b/arch/arm/kernel/head-nommu.S
@@ -68,14 +68,6 @@ ENTRY(stext)
 	beq	__error_p				@ yes, error 'p'
 
 #ifdef CONFIG_ARM_MPU
-	/* Calculate the size of a region covering just the kernel */
-	ldr	r5, =PLAT_PHYS_OFFSET		@ Region start: PHYS_OFFSET
-	ldr     r6, =(_end)			@ Cover whole kernel
-	sub	r6, r6, r5			@ Minimum size of region to map
-	clz	r6, r6				@ Region size must be 2^N...
-	rsb	r6, r6, #31			@ ...so round up region size
-	lsl	r6, r6, #MPU_RSR_SZ		@ Put size in right field
-	orr	r6, r6, #(1 << MPU_RSR_EN)	@ Set region enabled bit
 	bl	__setup_mpu
 #endif
 
@@ -83,8 +75,8 @@ ENTRY(stext)
 	ldr	r12, [r10, #PROCINFO_INITFUNC]
 	add	r12, r12, r10
 	ret	r12
-1:	bl	__after_proc_init
-	b	__mmap_switched
+1:	ldr	lr, =__mmap_switched
+	b	__after_proc_init
 ENDPROC(stext)
 
 #ifdef CONFIG_SMP
@@ -110,8 +102,6 @@ ENTRY(secondary_startup)
 	ldr	r7, __secondary_data
 
 #ifdef CONFIG_ARM_MPU
-	/* Use MPU region info supplied by __cpu_up */
-	ldr	r6, [r7]			@ get secondary_data.mpu_rgn_info
 	bl      __secondary_setup_mpu		@ Initialize the MPU
 #endif
 
@@ -133,12 +123,45 @@ __secondary_data:
 /*
  * Set the Control Register and Read the process ID.
  */
+	.text
 __after_proc_init:
+#ifdef CONFIG_ARM_MPU
+M_CLASS(movw	r12, #:lower16:BASEADDR_V7M_SCB)
+M_CLASS(movt	r12, #:upper16:BASEADDR_V7M_SCB)
+M_CLASS(ldr	r3, [r12, 0x50])
+AR_CLASS(mrc	p15, 0, r3, c0, c1, 4)          @ Read ID_MMFR0
+	and	r3, r3, #(MMFR0_PMSA)           @ PMSA field
+	teq	r3, #(MMFR0_PMSAv7)             @ PMSA v7
+	beq	1f
+	teq	r3, #(MMFR0_PMSAv8)		@ PMSA v8
+	/*
+	 * Memory region attributes for PMSAv8:
+	 *
+	 *   n = AttrIndx[2:0]
+	 *                      n       MAIR
+	 *   DEVICE_nGnRnE      000     00000000
+	 *   NORMAL             001     11111111
+	 */
+	ldreq	r3, =PMSAv8_MAIR(0x00, PMSAv8_RGN_DEVICE_nGnRnE) | \
+		     PMSAv8_MAIR(0xff, PMSAv8_RGN_NORMAL)
+AR_CLASS(mcreq	p15, 0, r3, c10, c2, 0)		@ MAIR 0
+M_CLASS(streq	r3, [r12, #PMSAv8_MAIR0])
+	moveq	r3, #0
+AR_CLASS(mcreq	p15, 0, r3, c10, c2, 1)		@ MAIR 1
+M_CLASS(streq	r3, [r12, #PMSAv8_MAIR1])
+
+1:
+#endif
 #ifdef CONFIG_CPU_CP15
 	/*
 	 * CP15 system control register value returned in r0 from
 	 * the CPU init function.
 	 */
+
+#ifdef CONFIG_ARM_MPU
+	biceq	r0, r0, #CR_BR			@ Disable the 'default mem-map'
+	orreq	r0, r0, #CR_M			@ Set SCTRL.M (MPU on)
+#endif
 #if defined(CONFIG_ALIGNMENT_TRAP) && __LINUX_ARM_ARCH__ < 6
 	orr	r0, r0, #CR_A
 #else
@@ -154,7 +177,15 @@ __after_proc_init:
 	bic	r0, r0, #CR_I
 #endif
 	mcr	p15, 0, r0, c1, c0, 0		@ write control reg
+	isb
 #elif defined (CONFIG_CPU_V7M)
+#ifdef CONFIG_ARM_MPU
+	ldreq	r3, [r12, MPU_CTRL]
+	biceq	r3, #MPU_CTRL_PRIVDEFENA
+	orreq	r3, #MPU_CTRL_ENABLE
+	streq	r3, [r12, MPU_CTRL]
+	isb
+#endif
 	/* For V7M systems we want to modify the CCR similarly to the SCTLR */
 #ifdef CONFIG_CPU_DCACHE_DISABLE
 	bic	r0, r0, #V7M_SCB_CCR_DC
@@ -165,9 +196,7 @@ __after_proc_init:
 #ifdef CONFIG_CPU_ICACHE_DISABLE
 	bic	r0, r0, #V7M_SCB_CCR_IC
 #endif
-	movw	r3, #:lower16:(BASEADDR_V7M_SCB + V7M_SCB_CCR)
-	movt	r3, #:upper16:(BASEADDR_V7M_SCB + V7M_SCB_CCR)
-	str	r0, [r3]
+	str	r0, [r12, V7M_SCB_CCR]
 #endif /* CONFIG_CPU_CP15 elif CONFIG_CPU_V7M */
 	ret	lr
 ENDPROC(__after_proc_init)
@@ -184,7 +213,7 @@ ENDPROC(__after_proc_init)
 .endm
 
 /* Setup a single MPU region, either D or I side (D-side for unified) */
-.macro setup_region bar, acr, sr, side = MPU_DATA_SIDE, unused
+.macro setup_region bar, acr, sr, side = PMSAv7_DATA_SIDE, unused
 	mcr	p15, 0, \bar, c6, c1, (0 + \side)	@ I/DRBAR
 	mcr	p15, 0, \acr, c6, c1, (4 + \side)	@ I/DRACR
 	mcr	p15, 0, \sr, c6, c1, (2 + \side)		@ I/DRSR
@@ -192,14 +221,14 @@ ENDPROC(__after_proc_init)
 #else
 .macro set_region_nr tmp, rgnr, base
 	mov	\tmp, \rgnr
-	str     \tmp, [\base, #MPU_RNR]
+	str     \tmp, [\base, #PMSAv7_RNR]
 .endm
 
 .macro setup_region bar, acr, sr, unused, base
 	lsl     \acr, \acr, #16
 	orr     \acr, \acr, \sr
-	str     \bar, [\base, #MPU_RBAR]
-	str     \acr, [\base, #MPU_RASR]
+	str     \bar, [\base, #PMSAv7_RBAR]
+	str     \acr, [\base, #PMSAv7_RASR]
 .endm
 
 #endif
@@ -210,8 +239,9 @@ ENDPROC(__after_proc_init)
  * Region 2: Normal, Shared, cacheable for RAM. From PHYS_OFFSET, size from r6
  * Region 3: Normal, shared, inaccessible from PL0 to protect the vectors page
  *
- * r6: Value to be written to DRSR (and IRSR if required) for MPU_RAM_REGION
+ * r6: Value to be written to DRSR (and IRSR if required) for PMSAv7_RAM_REGION
 */
+	__HEAD
 
 ENTRY(__setup_mpu)
 
@@ -223,7 +253,22 @@ AR_CLASS(mrc	p15, 0, r0, c0, c1, 4)		@ Read ID_MMFR0
 M_CLASS(ldr	r0, [r12, 0x50])
 	and	r0, r0, #(MMFR0_PMSA)		@ PMSA field
 	teq	r0, #(MMFR0_PMSAv7)		@ PMSA v7
-	bxne	lr
+	beq	__setup_pmsa_v7
+	teq	r0, #(MMFR0_PMSAv8)		@ PMSA v8
+	beq	__setup_pmsa_v8
+
+	ret	lr
+ENDPROC(__setup_mpu)
+
+ENTRY(__setup_pmsa_v7)
+	/* Calculate the size of a region covering just the kernel */
+	ldr	r5, =PLAT_PHYS_OFFSET		@ Region start: PHYS_OFFSET
+	ldr     r6, =(_end)			@ Cover whole kernel
+	sub	r6, r6, r5			@ Minimum size of region to map
+	clz	r6, r6				@ Region size must be 2^N...
+	rsb	r6, r6, #31			@ ...so round up region size
+	lsl	r6, r6, #PMSAv7_RSR_SZ		@ Put size in right field
+	orr	r6, r6, #(1 << PMSAv7_RSR_EN)	@ Set region enabled bit
 
 	/* Determine whether the D/I-side memory map is unified. We set the
 	 * flags here and continue to use them for the rest of this function */
@@ -234,77 +279,189 @@ M_CLASS(ldr    r0, [r12, #MPU_TYPE])
 	tst	r0, #MPUIR_nU			@ MPUIR_nU = 0 for unified
 
 	/* Setup second region first to free up r6 */
-	set_region_nr r0, #MPU_RAM_REGION, r12
+	set_region_nr r0, #PMSAv7_RAM_REGION, r12
 	isb
 	/* Full access from PL0, PL1, shared for CONFIG_SMP, cacheable */
 	ldr	r0, =PLAT_PHYS_OFFSET		@ RAM starts at PHYS_OFFSET
-	ldr	r5,=(MPU_AP_PL1RW_PL0RW | MPU_RGN_NORMAL)
+	ldr	r5,=(PMSAv7_AP_PL1RW_PL0RW | PMSAv7_RGN_NORMAL)
 
-	setup_region r0, r5, r6, MPU_DATA_SIDE, r12	@ PHYS_OFFSET, shared, enabled
+	setup_region r0, r5, r6, PMSAv7_DATA_SIDE, r12	@ PHYS_OFFSET, shared, enabled
 	beq	1f					@ Memory-map not unified
-	setup_region r0, r5, r6, MPU_INSTR_SIDE, r12	@ PHYS_OFFSET, shared, enabled
+	setup_region r0, r5, r6, PMSAv7_INSTR_SIDE, r12	@ PHYS_OFFSET, shared, enabled
 1:	isb
 
 	/* First/background region */
-	set_region_nr r0, #MPU_BG_REGION, r12
+	set_region_nr r0, #PMSAv7_BG_REGION, r12
 	isb
 	/* Execute Never,  strongly ordered, inaccessible to PL0, rw PL1  */
 	mov	r0, #0				@ BG region starts at 0x0
-	ldr	r5,=(MPU_ACR_XN | MPU_RGN_STRONGLY_ORDERED | MPU_AP_PL1RW_PL0NA)
-	mov	r6, #MPU_RSR_ALL_MEM		@ 4GB region, enabled
+	ldr	r5,=(PMSAv7_ACR_XN | PMSAv7_RGN_STRONGLY_ORDERED | PMSAv7_AP_PL1RW_PL0NA)
+	mov	r6, #PMSAv7_RSR_ALL_MEM		@ 4GB region, enabled
 
-	setup_region r0, r5, r6, MPU_DATA_SIDE, r12	@ 0x0, BG region, enabled
+	setup_region r0, r5, r6, PMSAv7_DATA_SIDE, r12	@ 0x0, BG region, enabled
 	beq	2f					@ Memory-map not unified
-	setup_region r0, r5, r6, MPU_INSTR_SIDE r12	@ 0x0, BG region, enabled
+	setup_region r0, r5, r6, PMSAv7_INSTR_SIDE r12	@ 0x0, BG region, enabled
 2:	isb
 
 #ifdef CONFIG_XIP_KERNEL
-	set_region_nr r0, #MPU_ROM_REGION, r12
+	set_region_nr r0, #PMSAv7_ROM_REGION, r12
 	isb
 
-	ldr	r5,=(MPU_AP_PL1RO_PL0NA | MPU_RGN_NORMAL)
+	ldr	r5,=(PMSAv7_AP_PL1RO_PL0NA | PMSAv7_RGN_NORMAL)
 
 	ldr	r0, =CONFIG_XIP_PHYS_ADDR		@ ROM start
 	ldr     r6, =(_exiprom)				@ ROM end
 	sub	r6, r6, r0				@ Minimum size of region to map
 	clz	r6, r6					@ Region size must be 2^N...
 	rsb	r6, r6, #31				@ ...so round up region size
-	lsl	r6, r6, #MPU_RSR_SZ			@ Put size in right field
-	orr	r6, r6, #(1 << MPU_RSR_EN)		@ Set region enabled bit
+	lsl	r6, r6, #PMSAv7_RSR_SZ			@ Put size in right field
+	orr	r6, r6, #(1 << PMSAv7_RSR_EN)		@ Set region enabled bit
 
-	setup_region r0, r5, r6, MPU_DATA_SIDE, r12	@ XIP_PHYS_ADDR, shared, enabled
+	setup_region r0, r5, r6, PMSAv7_DATA_SIDE, r12	@ XIP_PHYS_ADDR, shared, enabled
 	beq	3f					@ Memory-map not unified
-	setup_region r0, r5, r6, MPU_INSTR_SIDE, r12	@ XIP_PHYS_ADDR, shared, enabled
+	setup_region r0, r5, r6, PMSAv7_INSTR_SIDE, r12	@ XIP_PHYS_ADDR, shared, enabled
 3:	isb
 #endif
+	ret	lr
+ENDPROC(__setup_pmsa_v7)
+
+ENTRY(__setup_pmsa_v8)
+	mov	r0, #0
+AR_CLASS(mcr	p15, 0, r0, c6, c2, 1)		@ PRSEL
+M_CLASS(str	r0, [r12, #PMSAv8_RNR])
+	isb
+
+#ifdef CONFIG_XIP_KERNEL
+	ldr	r5, =CONFIG_XIP_PHYS_ADDR		@ ROM start
+	ldr     r6, =(_exiprom)				@ ROM end
+	sub	r6, r6, #1
+	bic	r6, r6, #(PMSAv8_MINALIGN - 1)
+
+	orr	r5, r5, #(PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED)
+	orr	r6, r6, #(PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN)
+
+AR_CLASS(mcr	p15, 0, r5, c6, c8, 0)			@ PRBAR0
+AR_CLASS(mcr	p15, 0, r6, c6, c8, 1)			@ PRLAR0
+M_CLASS(str	r5, [r12, #PMSAv8_RBAR_A(0)])
+M_CLASS(str	r6, [r12, #PMSAv8_RLAR_A(0)])
+#endif
+
+	ldr	r5, =KERNEL_START
+	ldr	r6, =KERNEL_END
+	sub	r6, r6, #1
+	bic	r6, r6, #(PMSAv8_MINALIGN - 1)
+
+	orr	r5, r5, #(PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED)
+	orr	r6, r6, #(PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN)
+
+AR_CLASS(mcr	p15, 0, r5, c6, c8, 4)			@ PRBAR1
+AR_CLASS(mcr	p15, 0, r6, c6, c8, 5)			@ PRLAR1
+M_CLASS(str	r5, [r12, #PMSAv8_RBAR_A(1)])
+M_CLASS(str	r6, [r12, #PMSAv8_RLAR_A(1)])
+
+	/* Setup Background: 0x0 - min(KERNEL_START, XIP_PHYS_ADDR) */
+#ifdef CONFIG_XIP_KERNEL
+	ldr	r6, =KERNEL_START
+	ldr	r5, =CONFIG_XIP_PHYS_ADDR
+	cmp	r6, r5
+	movcs	r6, r5
+#else
+	ldr	r6, =KERNEL_START
+#endif
+	cmp	r6, #0
+	beq	1f
+
+	mov	r5, #0
+	sub	r6, r6, #1
+	bic	r6, r6, #(PMSAv8_MINALIGN - 1)
+
+	orr	r5, r5, #(PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED | PMSAv8_BAR_XN)
+	orr	r6, r6, #(PMSAv8_LAR_IDX(PMSAv8_RGN_DEVICE_nGnRnE) | PMSAv8_LAR_EN)
+
+AR_CLASS(mcr	p15, 0, r5, c6, c9, 0)			@ PRBAR2
+AR_CLASS(mcr	p15, 0, r6, c6, c9, 1)			@ PRLAR2
+M_CLASS(str	r5, [r12, #PMSAv8_RBAR_A(2)])
+M_CLASS(str	r6, [r12, #PMSAv8_RLAR_A(2)])
+
+1:
+	/* Setup Background: max(KERNEL_END, _exiprom) - 0xffffffff */
+#ifdef CONFIG_XIP_KERNEL
+	ldr	r5, =KERNEL_END
+	ldr	r6, =(_exiprom)
+	cmp	r5, r6
+	movcc	r5, r6
+#else
+	ldr	r5, =KERNEL_END
+#endif
+	mov	r6, #0xffffffff
+	bic	r6, r6, #(PMSAv8_MINALIGN - 1)
+
+	orr	r5, r5, #(PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED | PMSAv8_BAR_XN)
+	orr	r6, r6, #(PMSAv8_LAR_IDX(PMSAv8_RGN_DEVICE_nGnRnE) | PMSAv8_LAR_EN)
 
-	/* Enable the MPU */
-AR_CLASS(mrc	p15, 0, r0, c1, c0, 0)		@ Read SCTLR
-AR_CLASS(bic	r0, r0, #CR_BR)			@ Disable the 'default mem-map'
-AR_CLASS(orr	r0, r0, #CR_M)			@ Set SCTRL.M (MPU on)
-AR_CLASS(mcr	p15, 0, r0, c1, c0, 0)		@ Enable MPU
+AR_CLASS(mcr	p15, 0, r5, c6, c9, 4)			@ PRBAR3
+AR_CLASS(mcr	p15, 0, r6, c6, c9, 5)			@ PRLAR3
+M_CLASS(str	r5, [r12, #PMSAv8_RBAR_A(3)])
+M_CLASS(str	r6, [r12, #PMSAv8_RLAR_A(3)])
 
-M_CLASS(ldr	r0, [r12, #MPU_CTRL])
-M_CLASS(bic	r0, #MPU_CTRL_PRIVDEFENA)
-M_CLASS(orr	r0, #MPU_CTRL_ENABLE)
-M_CLASS(str	r0, [r12, #MPU_CTRL])
+#ifdef CONFIG_XIP_KERNEL
+	/* Setup Background: min(_exiprom, KERNEL_END) - max(KERNEL_START, XIP_PHYS_ADDR) */
+	ldr	r5, =(_exiprom)
+	ldr	r6, =KERNEL_END
+	cmp	r5, r6
+	movcs	r5, r6
+
+	ldr	r6, =KERNEL_START
+	ldr	r0, =CONFIG_XIP_PHYS_ADDR
+	cmp	r6, r0
+	movcc	r6, r0
+
+	sub	r6, r6, #1
+	bic	r6, r6, #(PMSAv8_MINALIGN - 1)
+
+	orr	r5, r5, #(PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED | PMSAv8_BAR_XN)
+	orr	r6, r6, #(PMSAv8_LAR_IDX(PMSAv8_RGN_DEVICE_nGnRnE) | PMSAv8_LAR_EN)
+
+#ifdef CONFIG_CPU_V7M
+	/* There is no alias for n == 4 */
+	mov	r0, #4
+	str	r0, [r12, #PMSAv8_RNR]			@ PRSEL
 	isb
 
+	str	r5, [r12, #PMSAv8_RBAR_A(0)]
+	str	r6, [r12, #PMSAv8_RLAR_A(0)]
+#else
+	mcr	p15, 0, r5, c6, c10, 1			@ PRBAR4
+	mcr	p15, 0, r6, c6, c10, 2			@ PRLAR4
+#endif
+#endif
 	ret	lr
-ENDPROC(__setup_mpu)
+ENDPROC(__setup_pmsa_v8)
 
 #ifdef CONFIG_SMP
 /*
  * r6: pointer at mpu_rgn_info
  */
 
+	.text
 ENTRY(__secondary_setup_mpu)
+	/* Use MPU region info supplied by __cpu_up */
+	ldr	r6, [r7]			@ get secondary_data.mpu_rgn_info
+
 	/* Probe for v7 PMSA compliance */
 	mrc	p15, 0, r0, c0, c1, 4		@ Read ID_MMFR0
 	and	r0, r0, #(MMFR0_PMSA)		@ PMSA field
 	teq	r0, #(MMFR0_PMSAv7)		@ PMSA v7
-	bne	__error_p
+	beq	__secondary_setup_pmsa_v7
+	teq	r0, #(MMFR0_PMSAv8)		@ PMSA v8
+	beq	__secondary_setup_pmsa_v8
+	b	__error_p
+ENDPROC(__secondary_setup_mpu)
 
+/*
+ * r6: pointer at mpu_rgn_info
+ */
+ENTRY(__secondary_setup_pmsa_v7)
 	/* Determine whether the D/I-side memory map is unified. We set the
 	 * flags here and continue to use them for the rest of this function */
 	mrc	p15, 0, r0, c0, c0, 4		@ MPUIR
@@ -328,25 +485,45 @@ ENTRY(__secondary_setup_mpu)
 	ldr	r6, [r3, #MPU_RGN_DRSR]
 	ldr	r5, [r3, #MPU_RGN_DRACR]
 
-	setup_region r0, r5, r6, MPU_DATA_SIDE
+	setup_region r0, r5, r6, PMSAv7_DATA_SIDE
 	beq	2f
-	setup_region r0, r5, r6, MPU_INSTR_SIDE
+	setup_region r0, r5, r6, PMSAv7_INSTR_SIDE
 2:	isb
 
 	mrc	p15, 0, r0, c0, c0, 4		@ Reevaluate the MPUIR
 	cmp	r4, #0
 	bgt	1b
 
-	/* Enable the MPU */
-	mrc	p15, 0, r0, c1, c0, 0		@ Read SCTLR
-	bic	r0, r0, #CR_BR			@ Disable the 'default mem-map'
-	orr	r0, r0, #CR_M			@ Set SCTRL.M (MPU on)
-	mcr	p15, 0, r0, c1, c0, 0		@ Enable MPU
+	ret	lr
+ENDPROC(__secondary_setup_pmsa_v7)
+
+ENTRY(__secondary_setup_pmsa_v8)
+	ldr	r4, [r6, #MPU_RNG_INFO_USED]
+#ifndef CONFIG_XIP_KERNEL
+	add	r4, r4, #1
+#endif
+	mov	r5, #MPU_RNG_SIZE
+	add	r3, r6, #MPU_RNG_INFO_RNGS
+	mla	r3, r4, r5, r3
+
+1:
+	sub	r3, r3, #MPU_RNG_SIZE
+	sub	r4, r4, #1
+
+	mcr	p15, 0, r4, c6, c2, 1		@ PRSEL
 	isb
 
-	ret	lr
-ENDPROC(__secondary_setup_mpu)
+	ldr	r5, [r3, #MPU_RGN_PRBAR]
+	ldr	r6, [r3, #MPU_RGN_PRLAR]
 
+	mcr	p15, 0, r5, c6, c3, 0		@ PRBAR
+	mcr	p15, 0, r6, c6, c3, 1           @ PRLAR
+
+	cmp	r4, #0
+	bgt	1b
+
+	ret	lr
+ENDPROC(__secondary_setup_pmsa_v8)
 #endif /* CONFIG_SMP */
 #endif /* CONFIG_ARM_MPU */
 #include "head-common.S"
diff --git a/arch/arm/kernel/perf_event_v6.c b/arch/arm/kernel/perf_event_v6.c
index 1d7061a38922..be42c4f66a40 100644
--- a/arch/arm/kernel/perf_event_v6.c
+++ b/arch/arm/kernel/perf_event_v6.c
@@ -303,12 +303,10 @@ static void armv6pmu_enable_event(struct perf_event *event)
 }
 
 static irqreturn_t
-armv6pmu_handle_irq(int irq_num,
-		    void *dev)
+armv6pmu_handle_irq(struct arm_pmu *cpu_pmu)
 {
 	unsigned long pmcr = armv6_pmcr_read();
 	struct perf_sample_data data;
-	struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
 	struct pt_regs *regs;
 	int idx;
diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c
index 870b66c1e4ef..57f01e059f39 100644
--- a/arch/arm/kernel/perf_event_v7.c
+++ b/arch/arm/kernel/perf_event_v7.c
@@ -946,11 +946,10 @@ static void armv7pmu_disable_event(struct perf_event *event)
 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 }
 
-static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
+static irqreturn_t armv7pmu_handle_irq(struct arm_pmu *cpu_pmu)
 {
 	u32 pmnc;
 	struct perf_sample_data data;
-	struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
 	struct pt_regs *regs;
 	int idx;
diff --git a/arch/arm/kernel/perf_event_xscale.c b/arch/arm/kernel/perf_event_xscale.c
index fcf218da660e..88d1a76f5367 100644
--- a/arch/arm/kernel/perf_event_xscale.c
+++ b/arch/arm/kernel/perf_event_xscale.c
@@ -142,11 +142,10 @@ xscale1_pmnc_counter_has_overflowed(unsigned long pmnc,
 }
 
 static irqreturn_t
-xscale1pmu_handle_irq(int irq_num, void *dev)
+xscale1pmu_handle_irq(struct arm_pmu *cpu_pmu)
 {
 	unsigned long pmnc;
 	struct perf_sample_data data;
-	struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
 	struct pt_regs *regs;
 	int idx;
@@ -489,11 +488,10 @@ xscale2_pmnc_counter_has_overflowed(unsigned long of_flags,
 }
 
 static irqreturn_t
-xscale2pmu_handle_irq(int irq_num, void *dev)
+xscale2pmu_handle_irq(struct arm_pmu *cpu_pmu)
 {
 	unsigned long pmnc, of_flags;
 	struct perf_sample_data data;
-	struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
 	struct pt_regs *regs;
 	int idx;
diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c
index 7724b0f661b3..36718a424358 100644
--- a/arch/arm/kernel/ptrace.c
+++ b/arch/arm/kernel/ptrace.c
@@ -205,6 +205,7 @@ void ptrace_break(struct task_struct *tsk, struct pt_regs *regs)
 {
 	siginfo_t info;
 
+	clear_siginfo(&info);
 	info.si_signo = SIGTRAP;
 	info.si_errno = 0;
 	info.si_code  = TRAP_BRKPT;
diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c
index fc40a2b40595..35ca494c028c 100644
--- a/arch/arm/kernel/setup.c
+++ b/arch/arm/kernel/setup.c
@@ -754,7 +754,7 @@ int __init arm_add_memory(u64 start, u64 size)
 	else
 		size -= aligned_start - start;
 
-#ifndef CONFIG_ARCH_PHYS_ADDR_T_64BIT
+#ifndef CONFIG_PHYS_ADDR_T_64BIT
 	if (aligned_start > ULONG_MAX) {
 		pr_crit("Ignoring memory at 0x%08llx outside 32-bit physical address space\n",
 			(long long)start);
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 2da087926ebe..0978282d5fc2 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -31,6 +31,7 @@
 #include <linux/irq_work.h>
 
 #include <linux/atomic.h>
+#include <asm/bugs.h>
 #include <asm/smp.h>
 #include <asm/cacheflush.h>
 #include <asm/cpu.h>
@@ -236,8 +237,6 @@ int __cpu_disable(void)
 	flush_cache_louis();
 	local_flush_tlb_all();
 
-	clear_tasks_mm_cpumask(cpu);
-
 	return 0;
 }
 
@@ -255,6 +254,7 @@ void __cpu_die(unsigned int cpu)
 	}
 	pr_debug("CPU%u: shutdown\n", cpu);
 
+	clear_tasks_mm_cpumask(cpu);
 	/*
 	 * platform_cpu_kill() is generally expected to do the powering off
 	 * and/or cutting of clocks to the dying CPU.  Optionally, this may
@@ -405,6 +405,9 @@ asmlinkage void secondary_start_kernel(void)
 	 * before we continue - which happens after __cpu_up returns.
 	 */
 	set_cpu_online(cpu, true);
+
+	check_other_bugs();
+
 	complete(&cpu_running);
 
 	local_irq_enable();
diff --git a/arch/arm/kernel/suspend.c b/arch/arm/kernel/suspend.c
index a40ebb7c0896..d08099269e35 100644
--- a/arch/arm/kernel/suspend.c
+++ b/arch/arm/kernel/suspend.c
@@ -3,6 +3,7 @@
 #include <linux/slab.h>
 #include <linux/mm_types.h>
 
+#include <asm/bugs.h>
 #include <asm/cacheflush.h>
 #include <asm/idmap.h>
 #include <asm/pgalloc.h>
@@ -36,6 +37,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
 		cpu_switch_mm(mm->pgd, mm);
 		local_flush_bp_all();
 		local_flush_tlb_all();
+		check_other_bugs();
 	}
 
 	return ret;
diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c
index 3bda08bee674..80517f293eb9 100644
--- a/arch/arm/kernel/swp_emulate.c
+++ b/arch/arm/kernel/swp_emulate.c
@@ -91,18 +91,6 @@ static int proc_status_show(struct seq_file *m, void *v)
 		seq_printf(m, "Last process:\t\t%d\n", previous_pid);
 	return 0;
 }
-
-static int proc_status_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_status_show, PDE_DATA(inode));
-}
-
-static const struct file_operations proc_status_fops = {
-	.open		= proc_status_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
 #endif
 
 /*
@@ -112,6 +100,7 @@ static void set_segfault(struct pt_regs *regs, unsigned long addr)
 {
 	siginfo_t info;
 
+	clear_siginfo(&info);
 	down_read(&current->mm->mmap_sem);
 	if (find_vma(current->mm, addr) == NULL)
 		info.si_code = SEGV_MAPERR;
@@ -260,7 +249,8 @@ static int __init swp_emulation_init(void)
 		return 0;
 
 #ifdef CONFIG_PROC_FS
-	if (!proc_create("cpu/swp_emulation", S_IRUGO, NULL, &proc_status_fops))
+	if (!proc_create_single("cpu/swp_emulation", S_IRUGO, NULL,
+			proc_status_show))
 		return -ENOMEM;
 #endif /* CONFIG_PROC_FS */
 
diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c
index 2fe87109ae46..badf02ca3693 100644
--- a/arch/arm/kernel/traps.c
+++ b/arch/arm/kernel/traps.c
@@ -441,6 +441,7 @@ asmlinkage void do_undefinstr(struct pt_regs *regs)
 	siginfo_t info;
 	void __user *pc;
 
+	clear_siginfo(&info);
 	pc = (void __user *)instruction_pointer(regs);
 
 	if (processor_mode(regs) == SVC_MODE) {
@@ -540,6 +541,7 @@ static int bad_syscall(int n, struct pt_regs *regs)
 {
 	siginfo_t info;
 
+	clear_siginfo(&info);
 	if ((current->personality & PER_MASK) != PER_LINUX) {
 		send_sig(SIGSEGV, current, 1);
 		return regs->ARM_r0;
@@ -607,6 +609,7 @@ asmlinkage int arm_syscall(int no, struct pt_regs *regs)
 {
 	siginfo_t info;
 
+	clear_siginfo(&info);
 	if ((no >> 16) != (__ARM_NR_BASE>> 16))
 		return bad_syscall(no, regs);
 
@@ -743,6 +746,8 @@ baddataabort(int code, unsigned long instr, struct pt_regs *regs)
 	unsigned long addr = instruction_pointer(regs);
 	siginfo_t info;
 
+	clear_siginfo(&info);
+
 #ifdef CONFIG_DEBUG_USER
 	if (user_debug & UDBG_BADABORT) {
 		pr_err("[%d] %s: bad data abort: code %d instr 0x%08lx\n",
diff --git a/arch/arm/kernel/vmlinux-xip.lds.S b/arch/arm/kernel/vmlinux-xip.lds.S
index d32f5d35f602..3593d5c1acd2 100644
--- a/arch/arm/kernel/vmlinux-xip.lds.S
+++ b/arch/arm/kernel/vmlinux-xip.lds.S
@@ -13,6 +13,7 @@
 #include <asm/cache.h>
 #include <asm/thread_info.h>
 #include <asm/memory.h>
+#include <asm/mpu.h>
 #include <asm/page.h>
 
 #include "vmlinux.lds.h"
@@ -148,6 +149,9 @@ SECTIONS
 	__init_end = .;
 
 	BSS_SECTION(0, 0, 8)
+#ifdef CONFIG_ARM_MPU
+	. = ALIGN(PMSAv8_MINALIGN);
+#endif
 	_end = .;
 
 	STABS_DEBUG
diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index b77dc675ae55..23150c0f0f4d 100644
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -12,6 +12,7 @@
 #include <asm/cache.h>
 #include <asm/thread_info.h>
 #include <asm/memory.h>
+#include <asm/mpu.h>
 #include <asm/page.h>
 #include <asm/pgtable.h>
 
@@ -54,6 +55,9 @@ SECTIONS
 	. = ALIGN(1<<SECTION_SHIFT);
 #endif
 
+#ifdef CONFIG_ARM_MPU
+	. = ALIGN(PMSAv8_MINALIGN);
+#endif
 	.text : {			/* Real text segment		*/
 		_stext = .;		/* Text and read-only data	*/
 		ARM_TEXT
@@ -143,6 +147,9 @@ SECTIONS
 	_edata = .;
 
 	BSS_SECTION(0, 0, 0)
+#ifdef CONFIG_ARM_MPU
+	. = ALIGN(PMSAv8_MINALIGN);
+#endif
 	_end = .;
 
 	STABS_DEBUG
diff --git a/arch/arm/kernel/vmlinux.lds.h b/arch/arm/kernel/vmlinux.lds.h
index 71281e08e1d4..ae5fdff18406 100644
--- a/arch/arm/kernel/vmlinux.lds.h
+++ b/arch/arm/kernel/vmlinux.lds.h
@@ -27,24 +27,24 @@
 
 #define PROC_INFO							\
 		. = ALIGN(4);						\
-		VMLINUX_SYMBOL(__proc_info_begin) = .;			\
+		__proc_info_begin = .;					\
 		*(.proc.info.init)					\
-		VMLINUX_SYMBOL(__proc_info_end) = .;
+		__proc_info_end = .;
 
 #define HYPERVISOR_TEXT							\
-		VMLINUX_SYMBOL(__hyp_text_start) = .;			\
+		__hyp_text_start = .;					\
 		*(.hyp.text)						\
-		VMLINUX_SYMBOL(__hyp_text_end) = .;
+		__hyp_text_end = .;
 
 #define IDMAP_TEXT							\
 		ALIGN_FUNCTION();					\
-		VMLINUX_SYMBOL(__idmap_text_start) = .;			\
+		__idmap_text_start = .;					\
 		*(.idmap.text)						\
-		VMLINUX_SYMBOL(__idmap_text_end) = .;			\
+		__idmap_text_end = .;					\
 		. = ALIGN(PAGE_SIZE);					\
-		VMLINUX_SYMBOL(__hyp_idmap_text_start) = .;		\
+		__hyp_idmap_text_start = .;				\
 		*(.hyp.idmap.text)					\
-		VMLINUX_SYMBOL(__hyp_idmap_text_end) = .;
+		__hyp_idmap_text_end = .;
 
 #define ARM_DISCARD							\
 		*(.ARM.exidx.exit.text)					\
diff --git a/arch/arm/kvm/hyp/Makefile b/arch/arm/kvm/hyp/Makefile
index 7fc0638f263a..d2b5ec9c4b92 100644
--- a/arch/arm/kvm/hyp/Makefile
+++ b/arch/arm/kvm/hyp/Makefile
@@ -23,3 +23,11 @@ obj-$(CONFIG_KVM_ARM_HOST) += hyp-entry.o
 obj-$(CONFIG_KVM_ARM_HOST) += switch.o
 CFLAGS_switch.o		   += $(CFLAGS_ARMV7VE)
 obj-$(CONFIG_KVM_ARM_HOST) += s2-setup.o
+
+# KVM code is run at a different exception code with a different map, so
+# compiler instrumentation that inserts callbacks or checks into the code may
+# cause crashes. Just disable it.
+GCOV_PROFILE	:= n
+KASAN_SANITIZE	:= n
+UBSAN_SANITIZE	:= n
+KCOV_INSTRUMENT	:= n
diff --git a/arch/arm/kvm/hyp/hyp-entry.S b/arch/arm/kvm/hyp/hyp-entry.S
index 95a2faefc070..aa3f9a9837ac 100644
--- a/arch/arm/kvm/hyp/hyp-entry.S
+++ b/arch/arm/kvm/hyp/hyp-entry.S
@@ -16,6 +16,7 @@
  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
+#include <linux/arm-smccc.h>
 #include <linux/linkage.h>
 #include <asm/kvm_arm.h>
 #include <asm/kvm_asm.h>
@@ -71,6 +72,90 @@ __kvm_hyp_vector:
 	W(b)	hyp_irq
 	W(b)	hyp_fiq
 
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+	.align 5
+__kvm_hyp_vector_ic_inv:
+	.global __kvm_hyp_vector_ic_inv
+
+	/*
+	 * We encode the exception entry in the bottom 3 bits of
+	 * SP, and we have to guarantee to be 8 bytes aligned.
+	 */
+	W(add)	sp, sp, #1	/* Reset 	  7 */
+	W(add)	sp, sp, #1	/* Undef	  6 */
+	W(add)	sp, sp, #1	/* Syscall	  5 */
+	W(add)	sp, sp, #1	/* Prefetch abort 4 */
+	W(add)	sp, sp, #1	/* Data abort	  3 */
+	W(add)	sp, sp, #1	/* HVC		  2 */
+	W(add)	sp, sp, #1	/* IRQ		  1 */
+	W(nop)			/* FIQ		  0 */
+
+	mcr	p15, 0, r0, c7, c5, 0	/* ICIALLU */
+	isb
+
+	b	decode_vectors
+
+	.align 5
+__kvm_hyp_vector_bp_inv:
+	.global __kvm_hyp_vector_bp_inv
+
+	/*
+	 * We encode the exception entry in the bottom 3 bits of
+	 * SP, and we have to guarantee to be 8 bytes aligned.
+	 */
+	W(add)	sp, sp, #1	/* Reset 	  7 */
+	W(add)	sp, sp, #1	/* Undef	  6 */
+	W(add)	sp, sp, #1	/* Syscall	  5 */
+	W(add)	sp, sp, #1	/* Prefetch abort 4 */
+	W(add)	sp, sp, #1	/* Data abort	  3 */
+	W(add)	sp, sp, #1	/* HVC		  2 */
+	W(add)	sp, sp, #1	/* IRQ		  1 */
+	W(nop)			/* FIQ		  0 */
+
+	mcr	p15, 0, r0, c7, c5, 6	/* BPIALL */
+	isb
+
+decode_vectors:
+
+#ifdef CONFIG_THUMB2_KERNEL
+	/*
+	 * Yet another silly hack: Use VPIDR as a temp register.
+	 * Thumb2 is really a pain, as SP cannot be used with most
+	 * of the bitwise instructions. The vect_br macro ensures
+	 * things gets cleaned-up.
+	 */
+	mcr	p15, 4, r0, c0, c0, 0	/* VPIDR */
+	mov	r0, sp
+	and	r0, r0, #7
+	sub	sp, sp, r0
+	push	{r1, r2}
+	mov	r1, r0
+	mrc	p15, 4, r0, c0, c0, 0	/* VPIDR */
+	mrc	p15, 0, r2, c0, c0, 0	/* MIDR  */
+	mcr	p15, 4, r2, c0, c0, 0	/* VPIDR */
+#endif
+
+.macro vect_br val, targ
+ARM(	eor	sp, sp, #\val	)
+ARM(	tst	sp, #7		)
+ARM(	eorne	sp, sp, #\val	)
+
+THUMB(	cmp	r1, #\val	)
+THUMB(	popeq	{r1, r2}	)
+
+	beq	\targ
+.endm
+
+	vect_br	0, hyp_fiq
+	vect_br	1, hyp_irq
+	vect_br	2, hyp_hvc
+	vect_br	3, hyp_dabt
+	vect_br	4, hyp_pabt
+	vect_br	5, hyp_svc
+	vect_br	6, hyp_undef
+	vect_br	7, hyp_reset
+#endif
+
 .macro invalid_vector label, cause
 	.align
 \label:	mov	r0, #\cause
@@ -118,7 +203,7 @@ hyp_hvc:
 	lsr     r2, r2, #16
 	and     r2, r2, #0xff
 	cmp     r2, #0
-	bne	guest_trap		@ Guest called HVC
+	bne	guest_hvc_trap		@ Guest called HVC
 
 	/*
 	 * Getting here means host called HVC, we shift parameters and branch
@@ -149,7 +234,14 @@ hyp_hvc:
 	bx	ip
 
 1:
-	push	{lr}
+	/*
+	 * Pushing r2 here is just a way of keeping the stack aligned to
+	 * 8 bytes on any path that can trigger a HYP exception. Here,
+	 * we may well be about to jump into the guest, and the guest
+	 * exit would otherwise be badly decoded by our fancy
+	 * "decode-exception-without-a-branch" code...
+	 */
+	push	{r2, lr}
 
 	mov	lr, r0
 	mov	r0, r1
@@ -159,7 +251,21 @@ hyp_hvc:
 THUMB(	orr	lr, #1)
 	blx	lr			@ Call the HYP function
 
-	pop	{lr}
+	pop	{r2, lr}
+	eret
+
+guest_hvc_trap:
+	movw	r2, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1
+	movt	r2, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1
+	ldr	r0, [sp]		@ Guest's r0
+	teq	r0, r2
+	bne	guest_trap
+	add	sp, sp, #12
+	@ Returns:
+	@ r0 = 0
+	@ r1 = HSR value (perfectly predictable)
+	@ r2 = ARM_SMCCC_ARCH_WORKAROUND_1
+	mov	r0, #0
 	eret
 
 guest_trap:
diff --git a/arch/arm/mach-axxia/Kconfig b/arch/arm/mach-axxia/Kconfig
index bb2ce1c63fd9..d3eae6037913 100644
--- a/arch/arm/mach-axxia/Kconfig
+++ b/arch/arm/mach-axxia/Kconfig
@@ -2,7 +2,6 @@
 config ARCH_AXXIA
 	bool "LSI Axxia platforms"
 	depends on ARCH_MULTI_V7 && ARM_LPAE
-	select ARCH_DMA_ADDR_T_64BIT
 	select ARM_AMBA
 	select ARM_GIC
 	select ARM_TIMER_SP804
diff --git a/arch/arm/mach-bcm/Kconfig b/arch/arm/mach-bcm/Kconfig
index c2f3b0d216a4..c46a728df44e 100644
--- a/arch/arm/mach-bcm/Kconfig
+++ b/arch/arm/mach-bcm/Kconfig
@@ -211,7 +211,6 @@ config ARCH_BRCMSTB
 	select BRCMSTB_L2_IRQ
 	select BCM7120_L2_IRQ
 	select ARCH_HAS_HOLES_MEMORYMODEL
-	select ARCH_DMA_ADDR_T_64BIT if ARM_LPAE
 	select ZONE_DMA if ARM_LPAE
 	select SOC_BRCMSTB
 	select SOC_BUS
diff --git a/arch/arm/mach-berlin/Kconfig b/arch/arm/mach-berlin/Kconfig
index 63ab1d368625..3d719cf645e3 100644
--- a/arch/arm/mach-berlin/Kconfig
+++ b/arch/arm/mach-berlin/Kconfig
@@ -23,8 +23,12 @@ config MACH_BERLIN_BG2
 
 config MACH_BERLIN_BG2CD
 	bool "Marvell Armada 1500-mini (BG2CD)"
+	select ARM_ERRATA_754322
+	select ARM_ERRATA_775420
+	select ARM_GLOBAL_TIMER
 	select CACHE_L2X0
-	select HAVE_ARM_TWD if SMP
+	select HAVE_ARM_SCU
+	select HAVE_ARM_TWD
 	select PINCTRL_BERLIN_BG2CD
 
 config MACH_BERLIN_BG2Q
diff --git a/arch/arm/mach-berlin/berlin.c b/arch/arm/mach-berlin/berlin.c
index ac181c6797ee..2424ad40190c 100644
--- a/arch/arm/mach-berlin/berlin.c
+++ b/arch/arm/mach-berlin/berlin.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device Tree support for Marvell Berlin SoCs.
  *
@@ -5,10 +6,6 @@
  *
  * based on GPL'ed 2.6 kernel sources
  *  (c) Marvell International Ltd.
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2.  This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
  */
 
 #include <linux/init.h>
diff --git a/arch/arm/mach-berlin/headsmp.S b/arch/arm/mach-berlin/headsmp.S
index dc82a3486b05..3057885d9772 100644
--- a/arch/arm/mach-berlin/headsmp.S
+++ b/arch/arm/mach-berlin/headsmp.S
@@ -1,11 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2014 Marvell Technology Group Ltd.
  *
  * Antoine Ténart <antoine.tenart@free-electrons.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/linkage.h>
diff --git a/arch/arm/mach-berlin/platsmp.c b/arch/arm/mach-berlin/platsmp.c
index 7586b7aec272..593fc4a69d84 100644
--- a/arch/arm/mach-berlin/platsmp.c
+++ b/arch/arm/mach-berlin/platsmp.c
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2014 Marvell Technology Group Ltd.
  *
  * Antoine Ténart <antoine.tenart@free-electrons.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/io.h>
@@ -81,7 +78,6 @@ static void __init berlin_smp_prepare_cpus(unsigned int max_cpus)
 		goto unmap_scu;
 
 	scu_enable(scu_base);
-	flush_cache_all();
 
 	/*
 	 * Write the first instruction the CPU will execute after being reset
diff --git a/arch/arm/mach-davinci/aemif.c b/arch/arm/mach-davinci/aemif.c
index ff8b7e76b6e9..e4ab3f3a2a1f 100644
--- a/arch/arm/mach-davinci/aemif.c
+++ b/arch/arm/mach-davinci/aemif.c
@@ -189,7 +189,7 @@ int davinci_aemif_setup(struct platform_device *pdev)
 	 * Setup Async configuration register in case we did not boot
 	 * from NAND and so bootloader did not bother to set it up.
 	 */
-	val = davinci_aemif_readl(base, A1CR_OFFSET + pdev->id * 4);
+	val = davinci_aemif_readl(base, A1CR_OFFSET + pdata->core_chipsel * 4);
 	/*
 	 * Extended Wait is not valid and Select Strobe mode is not
 	 * used
@@ -198,13 +198,13 @@ int davinci_aemif_setup(struct platform_device *pdev)
 	if (pdata->options & NAND_BUSWIDTH_16)
 		val |= 0x1;
 
-	davinci_aemif_writel(base, A1CR_OFFSET + pdev->id * 4, val);
+	davinci_aemif_writel(base, A1CR_OFFSET + pdata->core_chipsel * 4, val);
 
 	clkrate = clk_get_rate(clk);
 
 	if (pdata->timing)
-		ret = davinci_aemif_setup_timing(pdata->timing, base, pdev->id,
-						 clkrate);
+		ret = davinci_aemif_setup_timing(pdata->timing, base,
+						 pdata->core_chipsel, clkrate);
 
 	if (ret < 0)
 		dev_dbg(&pdev->dev, "NAND timing values setup fail\n");
diff --git a/arch/arm/mach-davinci/board-da830-evm.c b/arch/arm/mach-davinci/board-da830-evm.c
index d1e8ce7b4bd2..14a6fc061744 100644
--- a/arch/arm/mach-davinci/board-da830-evm.c
+++ b/arch/arm/mach-davinci/board-da830-evm.c
@@ -315,6 +315,7 @@ static struct davinci_aemif_timing da830_evm_nandflash_timing = {
 };
 
 static struct davinci_nand_pdata da830_evm_nand_pdata = {
+	.core_chipsel	= 1,
 	.parts		= da830_evm_nand_partitions,
 	.nr_parts	= ARRAY_SIZE(da830_evm_nand_partitions),
 	.ecc_mode	= NAND_ECC_HW,
diff --git a/arch/arm/mach-davinci/board-da850-evm.c b/arch/arm/mach-davinci/board-da850-evm.c
index 158ed9a1483f..e22fb40e34bc 100644
--- a/arch/arm/mach-davinci/board-da850-evm.c
+++ b/arch/arm/mach-davinci/board-da850-evm.c
@@ -244,6 +244,7 @@ static struct davinci_aemif_timing da850_evm_nandflash_timing = {
 };
 
 static struct davinci_nand_pdata da850_evm_nandflash_data = {
+	.core_chipsel	= 1,
 	.parts		= da850_evm_nandflash_partition,
 	.nr_parts	= ARRAY_SIZE(da850_evm_nandflash_partition),
 	.ecc_mode	= NAND_ECC_HW,
diff --git a/arch/arm/mach-davinci/board-dm355-evm.c b/arch/arm/mach-davinci/board-dm355-evm.c
index 23ab9e8bc04c..a3377f959444 100644
--- a/arch/arm/mach-davinci/board-dm355-evm.c
+++ b/arch/arm/mach-davinci/board-dm355-evm.c
@@ -78,6 +78,7 @@ static struct mtd_partition davinci_nand_partitions[] = {
 };
 
 static struct davinci_nand_pdata davinci_nand_data = {
+	.core_chipsel		= 0,
 	.mask_chipsel		= BIT(14),
 	.parts			= davinci_nand_partitions,
 	.nr_parts		= ARRAY_SIZE(davinci_nand_partitions),
diff --git a/arch/arm/mach-davinci/board-dm355-leopard.c b/arch/arm/mach-davinci/board-dm355-leopard.c
index 59743bd76793..8249a0bf69f0 100644
--- a/arch/arm/mach-davinci/board-dm355-leopard.c
+++ b/arch/arm/mach-davinci/board-dm355-leopard.c
@@ -72,6 +72,7 @@ static struct mtd_partition davinci_nand_partitions[] = {
 };
 
 static struct davinci_nand_pdata davinci_nand_data = {
+	.core_chipsel		= 0,
 	.mask_chipsel		= BIT(14),
 	.parts			= davinci_nand_partitions,
 	.nr_parts		= ARRAY_SIZE(davinci_nand_partitions),
diff --git a/arch/arm/mach-davinci/board-dm365-evm.c b/arch/arm/mach-davinci/board-dm365-evm.c
index 0ac085b58a2b..435f7ec7d9af 100644
--- a/arch/arm/mach-davinci/board-dm365-evm.c
+++ b/arch/arm/mach-davinci/board-dm365-evm.c
@@ -138,6 +138,7 @@ static struct mtd_partition davinci_nand_partitions[] = {
 };
 
 static struct davinci_nand_pdata davinci_nand_data = {
+	.core_chipsel		= 0,
 	.mask_chipsel		= BIT(14),
 	.parts			= davinci_nand_partitions,
 	.nr_parts		= ARRAY_SIZE(davinci_nand_partitions),
diff --git a/arch/arm/mach-davinci/board-dm644x-evm.c b/arch/arm/mach-davinci/board-dm644x-evm.c
index 509e64ab1994..48436f74fd71 100644
--- a/arch/arm/mach-davinci/board-dm644x-evm.c
+++ b/arch/arm/mach-davinci/board-dm644x-evm.c
@@ -153,6 +153,7 @@ static struct davinci_aemif_timing davinci_evm_nandflash_timing = {
 };
 
 static struct davinci_nand_pdata davinci_evm_nandflash_data = {
+	.core_chipsel	= 0,
 	.parts		= davinci_evm_nandflash_partition,
 	.nr_parts	= ARRAY_SIZE(davinci_evm_nandflash_partition),
 	.ecc_mode	= NAND_ECC_HW,
@@ -772,6 +773,8 @@ static __init void davinci_evm_init(void)
 	struct clk *aemif_clk;
 	struct davinci_soc_info *soc_info = &davinci_soc_info;
 
+	dm644x_init_devices();
+
 	ret = dm644x_gpio_register();
 	if (ret)
 		pr_warn("%s: GPIO init failed: %d\n", __func__, ret);
diff --git a/arch/arm/mach-davinci/board-dm646x-evm.c b/arch/arm/mach-davinci/board-dm646x-evm.c
index a3c0d1e87647..584064fdabf5 100644
--- a/arch/arm/mach-davinci/board-dm646x-evm.c
+++ b/arch/arm/mach-davinci/board-dm646x-evm.c
@@ -84,6 +84,7 @@ static struct davinci_aemif_timing dm6467tevm_nandflash_timing = {
 };
 
 static struct davinci_nand_pdata davinci_nand_data = {
+	.core_chipsel		= 0,
 	.mask_cle 		= 0x80000,
 	.mask_ale 		= 0x40000,
 	.parts			= davinci_nand_partitions,
diff --git a/arch/arm/mach-davinci/board-mityomapl138.c b/arch/arm/mach-davinci/board-mityomapl138.c
index d1c85484c2e2..37b3e48a21d1 100644
--- a/arch/arm/mach-davinci/board-mityomapl138.c
+++ b/arch/arm/mach-davinci/board-mityomapl138.c
@@ -400,6 +400,7 @@ static struct mtd_partition mityomapl138_nandflash_partition[] = {
 };
 
 static struct davinci_nand_pdata mityomapl138_nandflash_data = {
+	.core_chipsel	= 1,
 	.parts		= mityomapl138_nandflash_partition,
 	.nr_parts	= ARRAY_SIZE(mityomapl138_nandflash_partition),
 	.ecc_mode	= NAND_ECC_HW,
diff --git a/arch/arm/mach-davinci/board-neuros-osd2.c b/arch/arm/mach-davinci/board-neuros-osd2.c
index f2875770fbff..25ad9b0612be 100644
--- a/arch/arm/mach-davinci/board-neuros-osd2.c
+++ b/arch/arm/mach-davinci/board-neuros-osd2.c
@@ -87,6 +87,7 @@ static struct mtd_partition davinci_ntosd2_nandflash_partition[] = {
 };
 
 static struct davinci_nand_pdata davinci_ntosd2_nandflash_data = {
+	.core_chipsel	= 0,
 	.parts		= davinci_ntosd2_nandflash_partition,
 	.nr_parts	= ARRAY_SIZE(davinci_ntosd2_nandflash_partition),
 	.ecc_mode	= NAND_ECC_HW,
@@ -174,6 +175,8 @@ static __init void davinci_ntosd2_init(void)
 	struct clk *aemif_clk;
 	struct davinci_soc_info *soc_info = &davinci_soc_info;
 
+	dm644x_init_devices();
+
 	ret = dm644x_gpio_register();
 	if (ret)
 		pr_warn("%s: GPIO init failed: %d\n", __func__, ret);
diff --git a/arch/arm/mach-davinci/board-sffsdr.c b/arch/arm/mach-davinci/board-sffsdr.c
index 2922da9d1684..e7c1728b0833 100644
--- a/arch/arm/mach-davinci/board-sffsdr.c
+++ b/arch/arm/mach-davinci/board-sffsdr.c
@@ -134,6 +134,8 @@ static __init void davinci_sffsdr_init(void)
 {
 	struct davinci_soc_info *soc_info = &davinci_soc_info;
 
+	dm644x_init_devices();
+
 	platform_add_devices(davinci_sffsdr_devices,
 			     ARRAY_SIZE(davinci_sffsdr_devices));
 	sffsdr_init_i2c();
diff --git a/arch/arm/mach-davinci/davinci.h b/arch/arm/mach-davinci/davinci.h
index 270cef85750a..376cdd51ce9d 100644
--- a/arch/arm/mach-davinci/davinci.h
+++ b/arch/arm/mach-davinci/davinci.h
@@ -104,6 +104,7 @@ int dm365_gpio_register(void);
 
 /* DM644x function declarations */
 void dm644x_init(void);
+void dm644x_init_devices(void);
 void dm644x_init_time(void);
 void dm644x_init_asp(void);
 int dm644x_init_video(struct vpfe_config *, struct vpbe_config *);
diff --git a/arch/arm/mach-davinci/dm644x.c b/arch/arm/mach-davinci/dm644x.c
index b409801649e1..a2e8586c8a6d 100644
--- a/arch/arm/mach-davinci/dm644x.c
+++ b/arch/arm/mach-davinci/dm644x.c
@@ -961,19 +961,14 @@ int __init dm644x_init_video(struct vpfe_config *vpfe_cfg,
 	return 0;
 }
 
-static int __init dm644x_init_devices(void)
+void __init dm644x_init_devices(void)
 {
 	struct platform_device *edma_pdev;
-	int ret = 0;
-
-	if (!cpu_is_davinci_dm644x())
-		return 0;
+	int ret;
 
 	edma_pdev = platform_device_register_full(&dm644x_edma_device);
-	if (IS_ERR(edma_pdev)) {
+	if (IS_ERR(edma_pdev))
 		pr_warn("%s: Failed to register eDMA\n", __func__);
-		return PTR_ERR(edma_pdev);
-	}
 
 	platform_device_register(&dm644x_mdio_device);
 	platform_device_register(&dm644x_emac_device);
@@ -982,6 +977,4 @@ static int __init dm644x_init_devices(void)
 	if (ret)
 		pr_warn("%s: watchdog init failed: %d\n", __func__, ret);
 
-	return ret;
 }
-postcore_initcall(dm644x_init_devices);
diff --git a/arch/arm/mach-ep93xx/core.c b/arch/arm/mach-ep93xx/core.c
index 0581ffbedddd..faf48a3b1fea 100644
--- a/arch/arm/mach-ep93xx/core.c
+++ b/arch/arm/mach-ep93xx/core.c
@@ -635,6 +635,7 @@ EXPORT_SYMBOL(ep93xx_keypad_release_gpio);
  *************************************************************************/
 static struct resource ep93xx_i2s_resource[] = {
 	DEFINE_RES_MEM(EP93XX_I2S_PHYS_BASE, 0x100),
+	DEFINE_RES_IRQ(IRQ_EP93XX_SAI),
 };
 
 static struct platform_device ep93xx_i2s_device = {
diff --git a/arch/arm/mach-exynos/Kconfig b/arch/arm/mach-exynos/Kconfig
index 647c319f9f5f..b40963cf91c7 100644
--- a/arch/arm/mach-exynos/Kconfig
+++ b/arch/arm/mach-exynos/Kconfig
@@ -8,7 +8,6 @@
 menuconfig ARCH_EXYNOS
 	bool "Samsung EXYNOS"
 	depends on ARCH_MULTI_V7
-	select ARCH_HAS_BANDGAP
 	select ARCH_HAS_HOLES_MEMORYMODEL
 	select ARCH_SUPPORTS_BIG_ENDIAN
 	select ARM_AMBA
@@ -108,18 +107,6 @@ config SOC_EXYNOS5420
 	default y
 	depends on ARCH_EXYNOS5
 
-config SOC_EXYNOS5440
-	bool "SAMSUNG EXYNOS5440"
-	default y
-	depends on ARCH_EXYNOS5
-	select ARCH_DMA_ADDR_T_64BIT if ARM_LPAE
-	select HAVE_ARM_ARCH_TIMER
-	select AUTO_ZRELADDR
-	select PINCTRL_EXYNOS5440
-	select PM_OPP
-	help
-	  Enable EXYNOS5440 SoC support
-
 config SOC_EXYNOS5800
 	bool "SAMSUNG EXYNOS5800"
 	default y
diff --git a/arch/arm/mach-exynos/common.h b/arch/arm/mach-exynos/common.h
index 098f84a149a3..dcd21bb95e3b 100644
--- a/arch/arm/mach-exynos/common.h
+++ b/arch/arm/mach-exynos/common.h
@@ -21,7 +21,6 @@
 #define EXYNOS5250_SOC_ID	0x43520000
 #define EXYNOS5410_SOC_ID	0xE5410000
 #define EXYNOS5420_SOC_ID	0xE5420000
-#define EXYNOS5440_SOC_ID	0xE5440000
 #define EXYNOS5800_SOC_ID	0xE5422000
 #define EXYNOS5_SOC_MASK	0xFFFFF000
 
@@ -39,7 +38,6 @@ IS_SAMSUNG_CPU(exynos4412, EXYNOS4412_CPU_ID, EXYNOS4_CPU_MASK)
 IS_SAMSUNG_CPU(exynos5250, EXYNOS5250_SOC_ID, EXYNOS5_SOC_MASK)
 IS_SAMSUNG_CPU(exynos5410, EXYNOS5410_SOC_ID, EXYNOS5_SOC_MASK)
 IS_SAMSUNG_CPU(exynos5420, EXYNOS5420_SOC_ID, EXYNOS5_SOC_MASK)
-IS_SAMSUNG_CPU(exynos5440, EXYNOS5440_SOC_ID, EXYNOS5_SOC_MASK)
 IS_SAMSUNG_CPU(exynos5800, EXYNOS5800_SOC_ID, EXYNOS5_SOC_MASK)
 
 #if defined(CONFIG_SOC_EXYNOS3250)
@@ -82,22 +80,12 @@ IS_SAMSUNG_CPU(exynos5800, EXYNOS5800_SOC_ID, EXYNOS5_SOC_MASK)
 # define soc_is_exynos5420()	0
 #endif
 
-#if defined(CONFIG_SOC_EXYNOS5440)
-# define soc_is_exynos5440()	is_samsung_exynos5440()
-#else
-# define soc_is_exynos5440()	0
-#endif
-
 #if defined(CONFIG_SOC_EXYNOS5800)
 # define soc_is_exynos5800()	is_samsung_exynos5800()
 #else
 # define soc_is_exynos5800()	0
 #endif
 
-#define soc_is_exynos4() (soc_is_exynos4210() || soc_is_exynos4412())
-#define soc_is_exynos5() (soc_is_exynos5250() || soc_is_exynos5410() || \
-			  soc_is_exynos5420() || soc_is_exynos5800())
-
 extern u32 cp15_save_diag;
 extern u32 cp15_save_power;
 
@@ -149,6 +137,11 @@ extern void exynos_cpu_restore_register(void);
 extern void exynos_pm_central_suspend(void);
 extern int exynos_pm_central_resume(void);
 extern void exynos_enter_aftr(void);
+#ifdef CONFIG_SMP
+extern void exynos_scu_enable(void);
+#else
+static inline void exynos_scu_enable(void) { }
+#endif
 
 extern struct cpuidle_exynos_data cpuidle_coupled_exynos_data;
 
diff --git a/arch/arm/mach-exynos/exynos.c b/arch/arm/mach-exynos/exynos.c
index 8c4f5e342dc1..f4b6c93a7fd0 100644
--- a/arch/arm/mach-exynos/exynos.c
+++ b/arch/arm/mach-exynos/exynos.c
@@ -24,15 +24,6 @@
 
 #include "common.h"
 
-static struct map_desc exynos4_iodesc[] __initdata = {
-	{
-		.virtual	= (unsigned long)S5P_VA_COREPERI_BASE,
-		.pfn		= __phys_to_pfn(EXYNOS4_PA_COREPERI),
-		.length		= SZ_8K,
-		.type		= MT_DEVICE,
-	},
-};
-
 static struct platform_device exynos_cpuidle = {
 	.name              = "exynos_cpuidle",
 #ifdef CONFIG_ARM_EXYNOS_CPUIDLE
@@ -63,15 +54,6 @@ void __init exynos_sysram_init(void)
 	}
 }
 
-static void __init exynos_init_late(void)
-{
-	if (of_machine_is_compatible("samsung,exynos5440"))
-		/* to be supported later */
-		return;
-
-	exynos_pm_init();
-}
-
 static int __init exynos_fdt_map_chipid(unsigned long node, const char *uname,
 					int depth, void *data)
 {
@@ -79,8 +61,7 @@ static int __init exynos_fdt_map_chipid(unsigned long node, const char *uname,
 	const __be32 *reg;
 	int len;
 
-	if (!of_flat_dt_is_compatible(node, "samsung,exynos4210-chipid") &&
-		!of_flat_dt_is_compatible(node, "samsung,exynos5440-clock"))
+	if (!of_flat_dt_is_compatible(node, "samsung,exynos4210-chipid"))
 		return 0;
 
 	reg = of_get_flat_dt_prop(node, "reg", &len);
@@ -95,17 +76,6 @@ static int __init exynos_fdt_map_chipid(unsigned long node, const char *uname,
 	return 1;
 }
 
-/*
- * exynos_map_io
- *
- * register the standard cpu IO areas
- */
-static void __init exynos_map_io(void)
-{
-	if (soc_is_exynos4())
-		iotable_init(exynos4_iodesc, ARRAY_SIZE(exynos4_iodesc));
-}
-
 static void __init exynos_init_io(void)
 {
 	debug_ll_io_init();
@@ -114,8 +84,6 @@ static void __init exynos_init_io(void)
 
 	/* detect cpu id and rev. */
 	s5p_init_cpu(S5P_VA_CHIPID);
-
-	exynos_map_io();
 }
 
 /*
@@ -209,7 +177,6 @@ static char const *const exynos_dt_compat[] __initconst = {
 	"samsung,exynos5250",
 	"samsung,exynos5260",
 	"samsung,exynos5420",
-	"samsung,exynos5440",
 	NULL
 };
 
@@ -232,7 +199,7 @@ DT_MACHINE_START(EXYNOS_DT, "SAMSUNG EXYNOS (Flattened Device Tree)")
 	.init_early	= exynos_firmware_init,
 	.init_irq	= exynos_init_irq,
 	.init_machine	= exynos_dt_machine_init,
-	.init_late	= exynos_init_late,
+	.init_late	= exynos_pm_init,
 	.dt_compat	= exynos_dt_compat,
 	.dt_fixup	= exynos_dt_fixup,
 MACHINE_END
diff --git a/arch/arm/mach-exynos/include/mach/map.h b/arch/arm/mach-exynos/include/mach/map.h
index 37a5ea5e2602..22ebe3654633 100644
--- a/arch/arm/mach-exynos/include/mach/map.h
+++ b/arch/arm/mach-exynos/include/mach/map.h
@@ -15,6 +15,4 @@
 
 #define EXYNOS_PA_CHIPID		0x10000000
 
-#define EXYNOS4_PA_COREPERI		0x10500000
-
 #endif /* __ASM_ARCH_MAP_H */
diff --git a/arch/arm/mach-exynos/platsmp.c b/arch/arm/mach-exynos/platsmp.c
index 5156fe70e030..6a1e682371b3 100644
--- a/arch/arm/mach-exynos/platsmp.c
+++ b/arch/arm/mach-exynos/platsmp.c
@@ -163,6 +163,26 @@ int exynos_cluster_power_state(int cluster)
 		S5P_CORE_LOCAL_PWR_EN);
 }
 
+/**
+ * exynos_scu_enable : enables SCU for Cortex-A9 based system
+ */
+void exynos_scu_enable(void)
+{
+	struct device_node *np;
+	static void __iomem *scu_base;
+
+	if (!scu_base) {
+		np = of_find_compatible_node(NULL, NULL, "arm,cortex-a9-scu");
+		if (np) {
+			scu_base = of_iomap(np, 0);
+			of_node_put(np);
+		} else {
+			scu_base = ioremap(scu_a9_get_base(), SZ_4K);
+		}
+	}
+	scu_enable(scu_base);
+}
+
 static void __iomem *cpu_boot_reg_base(void)
 {
 	if (soc_is_exynos4210() && samsung_rev() == EXYNOS4210_REV_1_1)
@@ -219,11 +239,6 @@ static void write_pen_release(int val)
 	sync_cache_w(&pen_release);
 }
 
-static void __iomem *scu_base_addr(void)
-{
-	return (void __iomem *)(S5P_VA_SCU);
-}
-
 static DEFINE_SPINLOCK(boot_lock);
 
 static void exynos_secondary_init(unsigned int cpu)
@@ -389,7 +404,7 @@ static void __init exynos_smp_prepare_cpus(unsigned int max_cpus)
 	exynos_set_delayed_reset_assertion(true);
 
 	if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A9)
-		scu_enable(scu_base_addr());
+		exynos_scu_enable();
 
 	/*
 	 * Write the address of secondary startup into the
diff --git a/arch/arm/mach-exynos/pm.c b/arch/arm/mach-exynos/pm.c
index a822c5073715..48e7fb38613e 100644
--- a/arch/arm/mach-exynos/pm.c
+++ b/arch/arm/mach-exynos/pm.c
@@ -22,8 +22,6 @@
 #include <asm/suspend.h>
 #include <asm/cacheflush.h>
 
-#include <mach/map.h>
-
 #include "common.h"
 
 static inline void __iomem *exynos_boot_vector_addr(void)
@@ -172,7 +170,7 @@ void exynos_enter_aftr(void)
 	cpu_suspend(0, exynos_aftr_finisher);
 
 	if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A9) {
-		scu_enable(S5P_VA_SCU);
+		exynos_scu_enable();
 		if (call_firmware_op(resume) == -ENOSYS)
 			exynos_cpu_restore_register();
 	}
diff --git a/arch/arm/mach-exynos/suspend.c b/arch/arm/mach-exynos/suspend.c
index c2ed997fedef..d3db306a5a70 100644
--- a/arch/arm/mach-exynos/suspend.c
+++ b/arch/arm/mach-exynos/suspend.c
@@ -30,8 +30,6 @@
 #include <asm/smp_scu.h>
 #include <asm/suspend.h>
 
-#include <mach/map.h>
-
 #include <plat/pm-common.h>
 
 #include "common.h"
@@ -401,7 +399,7 @@ static void exynos_pm_resume(void)
 		goto early_wakeup;
 
 	if (cpuid == ARM_CPU_PART_CORTEX_A9)
-		scu_enable(S5P_VA_SCU);
+		exynos_scu_enable();
 
 	if (call_firmware_op(resume) == -ENOSYS
 	    && cpuid == ARM_CPU_PART_CORTEX_A9)
diff --git a/arch/arm/mach-highbank/Kconfig b/arch/arm/mach-highbank/Kconfig
index 81110ec34226..5552968f07f8 100644
--- a/arch/arm/mach-highbank/Kconfig
+++ b/arch/arm/mach-highbank/Kconfig
@@ -1,7 +1,6 @@
 config ARCH_HIGHBANK
 	bool "Calxeda ECX-1000/2000 (Highbank/Midway)"
 	depends on ARCH_MULTI_V7
-	select ARCH_DMA_ADDR_T_64BIT if ARM_LPAE
 	select ARCH_HAS_HOLES_MEMORYMODEL
 	select ARCH_SUPPORTS_BIG_ENDIAN
 	select ARM_AMBA
diff --git a/arch/arm/mach-imx/Kconfig b/arch/arm/mach-imx/Kconfig
index e47fa13f4b0c..6f4232384774 100644
--- a/arch/arm/mach-imx/Kconfig
+++ b/arch/arm/mach-imx/Kconfig
@@ -501,6 +501,7 @@ config SOC_IMX6SL
 
 config SOC_IMX6SLL
 	bool "i.MX6 SoloLiteLite support"
+	select PINCTRL_IMX6SLL
 	select SOC_IMX6
 
 	help
diff --git a/arch/arm/mach-imx/mach-mx21ads.c b/arch/arm/mach-imx/mach-mx21ads.c
index 5e366824814f..5d3b6b4fe6db 100644
--- a/arch/arm/mach-imx/mach-mx21ads.c
+++ b/arch/arm/mach-imx/mach-mx21ads.c
@@ -18,6 +18,7 @@
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/physmap.h>
 #include <linux/gpio/driver.h>
+#include <linux/gpio/machine.h>
 #include <linux/gpio.h>
 #include <linux/regulator/fixed.h>
 #include <linux/regulator/machine.h>
@@ -175,6 +176,7 @@ static struct resource mx21ads_mmgpio_resource =
 	DEFINE_RES_MEM_NAMED(MX21ADS_IO_REG, SZ_2, "dat");
 
 static struct bgpio_pdata mx21ads_mmgpio_pdata = {
+	.label	= "mx21ads-mmgpio",
 	.base	= MX21ADS_MMGPIO_BASE,
 	.ngpio	= 16,
 };
@@ -203,7 +205,6 @@ static struct regulator_init_data mx21ads_lcd_regulator_init_data = {
 static struct fixed_voltage_config mx21ads_lcd_regulator_pdata = {
 	.supply_name	= "LCD",
 	.microvolts	= 3300000,
-	.gpio		= MX21ADS_IO_LCDON,
 	.enable_high	= 1,
 	.init_data	= &mx21ads_lcd_regulator_init_data,
 };
@@ -216,6 +217,15 @@ static struct platform_device mx21ads_lcd_regulator = {
 	},
 };
 
+static struct gpiod_lookup_table mx21ads_lcd_regulator_gpiod_table = {
+	.dev_id = "reg-fixed-voltage.0", /* Let's hope ID 0 is what we get */
+	.table = {
+		GPIO_LOOKUP("mx21ads-mmgpio", 9,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 /*
  * Connected is a portrait Sharp-QVGA display
  * of type: LQ035Q7DB02
@@ -311,6 +321,7 @@ static void __init mx21ads_late_init(void)
 {
 	imx21_add_mxc_mmc(0, &mx21ads_sdhc_pdata);
 
+	gpiod_add_lookup_table(&mx21ads_lcd_regulator_gpiod_table);
 	platform_add_devices(platform_devices, ARRAY_SIZE(platform_devices));
 
 	mx21ads_cs8900_resources[1].start =
diff --git a/arch/arm/mach-imx/mach-mx27ads.c b/arch/arm/mach-imx/mach-mx27ads.c
index a04bb094ded1..0fdb88db0cbd 100644
--- a/arch/arm/mach-imx/mach-mx27ads.c
+++ b/arch/arm/mach-imx/mach-mx27ads.c
@@ -16,6 +16,7 @@
 #include <linux/gpio/driver.h>
 /* Needed for gpio_to_irq() */
 #include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/platform_device.h>
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/map.h>
@@ -230,10 +231,17 @@ static struct regulator_init_data mx27ads_lcd_regulator_init_data = {
 static struct fixed_voltage_config mx27ads_lcd_regulator_pdata = {
 	.supply_name	= "LCD",
 	.microvolts	= 3300000,
-	.gpio		= MX27ADS_LCD_GPIO,
 	.init_data	= &mx27ads_lcd_regulator_init_data,
 };
 
+static struct gpiod_lookup_table mx27ads_lcd_regulator_gpiod_table = {
+	.dev_id = "reg-fixed-voltage.0", /* Let's hope ID 0 is what we get */
+	.table = {
+		GPIO_LOOKUP("LCD", 0, "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 static void __init mx27ads_regulator_init(void)
 {
 	struct gpio_chip *vchip;
@@ -247,6 +255,8 @@ static void __init mx27ads_regulator_init(void)
 	vchip->set		= vgpio_set;
 	gpiochip_add_data(vchip, NULL);
 
+	gpiod_add_lookup_table(&mx27ads_lcd_regulator_gpiod_table);
+
 	platform_device_register_data(NULL, "reg-fixed-voltage",
 				      PLATFORM_DEVID_AUTO,
 				      &mx27ads_lcd_regulator_pdata,
diff --git a/arch/arm/mach-imx/mach-mx31_3ds.c b/arch/arm/mach-imx/mach-mx31_3ds.c
index 68c3f0799d5b..9d87f1dcf7bb 100644
--- a/arch/arm/mach-imx/mach-mx31_3ds.c
+++ b/arch/arm/mach-imx/mach-mx31_3ds.c
@@ -374,26 +374,12 @@ static struct imx_ssi_platform_data mx31_3ds_ssi_pdata = {
 };
 
 /* SPI */
-static int spi0_internal_chipselect[] = {
-	MXC_SPI_CS(0),
-	MXC_SPI_CS(1),
-	MXC_SPI_CS(2),
-};
-
 static const struct spi_imx_master spi0_pdata __initconst = {
-	.chipselect	= spi0_internal_chipselect,
-	.num_chipselect	= ARRAY_SIZE(spi0_internal_chipselect),
-};
-
-static int spi1_internal_chipselect[] = {
-	MXC_SPI_CS(0),
-	MXC_SPI_CS(1),
-	MXC_SPI_CS(2),
+	.num_chipselect	= 3,
 };
 
 static const struct spi_imx_master spi1_pdata __initconst = {
-	.chipselect	= spi1_internal_chipselect,
-	.num_chipselect	= ARRAY_SIZE(spi1_internal_chipselect),
+	.num_chipselect	= 3,
 };
 
 static struct spi_board_info mx31_3ds_spi_devs[] __initdata = {
diff --git a/arch/arm/mach-imx/mach-mx31lilly.c b/arch/arm/mach-imx/mach-mx31lilly.c
index 6fd463642954..8bf52819d4d9 100644
--- a/arch/arm/mach-imx/mach-mx31lilly.c
+++ b/arch/arm/mach-imx/mach-mx31lilly.c
@@ -226,20 +226,12 @@ static void __init lilly1131_usb_init(void)
 
 /* SPI */
 
-static int spi_internal_chipselect[] = {
-	MXC_SPI_CS(0),
-	MXC_SPI_CS(1),
-	MXC_SPI_CS(2),
-};
-
 static const struct spi_imx_master spi0_pdata __initconst = {
-	.chipselect = spi_internal_chipselect,
-	.num_chipselect = ARRAY_SIZE(spi_internal_chipselect),
+	.num_chipselect = 3,
 };
 
 static const struct spi_imx_master spi1_pdata __initconst = {
-	.chipselect = spi_internal_chipselect,
-	.num_chipselect = ARRAY_SIZE(spi_internal_chipselect),
+	.num_chipselect = 3,
 };
 
 static struct mc13xxx_platform_data mc13783_pdata __initdata = {
diff --git a/arch/arm/mach-imx/mach-mx31lite.c b/arch/arm/mach-imx/mach-mx31lite.c
index a3250bc7f114..a3cbba6c955b 100644
--- a/arch/arm/mach-imx/mach-mx31lite.c
+++ b/arch/arm/mach-imx/mach-mx31lite.c
@@ -83,15 +83,8 @@ static const struct imxuart_platform_data uart_pdata __initconst = {
 };
 
 /* SPI */
-static int spi0_internal_chipselect[] = {
-	MXC_SPI_CS(0),
-	MXC_SPI_CS(1),
-	MXC_SPI_CS(2),
-};
-
 static const struct spi_imx_master spi0_pdata __initconst = {
-	.chipselect	= spi0_internal_chipselect,
-	.num_chipselect	= ARRAY_SIZE(spi0_internal_chipselect),
+	.num_chipselect	= 3,
 };
 
 static const struct mxc_nand_platform_data
@@ -133,13 +126,8 @@ static struct platform_device smsc911x_device = {
  * The MC13783 is the only hard-wired SPI device on the module.
  */
 
-static int spi1_internal_chipselect[] = {
-	MXC_SPI_CS(0),
-};
-
 static const struct spi_imx_master spi1_pdata __initconst = {
-	.chipselect	= spi1_internal_chipselect,
-	.num_chipselect	= ARRAY_SIZE(spi1_internal_chipselect),
+	.num_chipselect	= 1,
 };
 
 static struct mc13xxx_platform_data mc13783_pdata __initdata = {
diff --git a/arch/arm/mach-imx/mach-mx31moboard.c b/arch/arm/mach-imx/mach-mx31moboard.c
index 7716f83aecdd..643a3d749703 100644
--- a/arch/arm/mach-imx/mach-mx31moboard.c
+++ b/arch/arm/mach-imx/mach-mx31moboard.c
@@ -152,14 +152,8 @@ static const struct imxi2c_platform_data moboard_i2c1_data __initconst = {
 	.bitrate = 100000,
 };
 
-static int moboard_spi1_cs[] = {
-	MXC_SPI_CS(0),
-	MXC_SPI_CS(2),
-};
-
 static const struct spi_imx_master moboard_spi1_pdata __initconst = {
-	.chipselect	= moboard_spi1_cs,
-	.num_chipselect	= ARRAY_SIZE(moboard_spi1_cs),
+	.num_chipselect	= 3,
 };
 
 static struct regulator_consumer_supply sdhc_consumers[] = {
@@ -296,19 +290,14 @@ static struct spi_board_info moboard_spi_board_info[] __initdata = {
 		/* irq number is run-time assigned */
 		.max_speed_hz = 300000,
 		.bus_num = 1,
-		.chip_select = 1,
+		.chip_select = 0,
 		.platform_data = &moboard_pmic,
 		.mode = SPI_CS_HIGH,
 	},
 };
 
-static int moboard_spi2_cs[] = {
-	MXC_SPI_CS(0), MXC_SPI_CS(1),
-};
-
 static const struct spi_imx_master moboard_spi2_pdata __initconst = {
-	.chipselect	= moboard_spi2_cs,
-	.num_chipselect	= ARRAY_SIZE(moboard_spi2_cs),
+	.num_chipselect	= 2,
 };
 
 #define SDHC1_CD IOMUX_TO_GPIO(MX31_PIN_ATA_CS0)
diff --git a/arch/arm/mach-imx/mach-pca100.c b/arch/arm/mach-imx/mach-pca100.c
index ed675863655b..5714e2f1b106 100644
--- a/arch/arm/mach-imx/mach-pca100.c
+++ b/arch/arm/mach-imx/mach-pca100.c
@@ -20,7 +20,7 @@
 #include <linux/platform_device.h>
 #include <linux/io.h>
 #include <linux/i2c.h>
-#include <linux/platform_data/at24.h>
+#include <linux/property.h>
 #include <linux/dma-mapping.h>
 #include <linux/spi/spi.h>
 #include <linux/spi/eeprom.h>
@@ -168,16 +168,15 @@ static const struct imxi2c_platform_data pca100_i2c1_data __initconst = {
 	.bitrate = 100000,
 };
 
-static struct at24_platform_data board_eeprom = {
-	.byte_len = 4096,
-	.page_size = 32,
-	.flags = AT24_FLAG_ADDR16,
+static const struct property_entry board_eeprom_properties[] = {
+	PROPERTY_ENTRY_U32("pagesize", 32),
+	{ }
 };
 
 static struct i2c_board_info pca100_i2c_devices[] = {
 	{
-		I2C_BOARD_INFO("at24", 0x52), /* E0=0, E1=1, E2=0 */
-		.platform_data = &board_eeprom,
+		I2C_BOARD_INFO("24c32", 0x52), /* E0=0, E1=1, E2=0 */
+		.properties = board_eeprom_properties,
 	}, {
 		I2C_BOARD_INFO("pcf8563", 0x51),
 	}, {
diff --git a/arch/arm/mach-imx/mach-pcm037.c b/arch/arm/mach-imx/mach-pcm037.c
index b787ba6897e4..004737c40fda 100644
--- a/arch/arm/mach-imx/mach-pcm037.c
+++ b/arch/arm/mach-imx/mach-pcm037.c
@@ -23,7 +23,7 @@
 #include <linux/smsc911x.h>
 #include <linux/interrupt.h>
 #include <linux/i2c.h>
-#include <linux/platform_data/at24.h>
+#include <linux/property.h>
 #include <linux/delay.h>
 #include <linux/spi/spi.h>
 #include <linux/irq.h>
@@ -263,16 +263,15 @@ static const struct imxi2c_platform_data pcm037_i2c2_data __initconst = {
 	.bitrate = 20000,
 };
 
-static struct at24_platform_data board_eeprom = {
-	.byte_len = 4096,
-	.page_size = 32,
-	.flags = AT24_FLAG_ADDR16,
+static const struct property_entry board_eeprom_properties[] = {
+	PROPERTY_ENTRY_U32("pagesize", 32),
+	{ }
 };
 
 static struct i2c_board_info pcm037_i2c_devices[] = {
 	{
-		I2C_BOARD_INFO("at24", 0x52), /* E0=0, E1=1, E2=0 */
-		.platform_data = &board_eeprom,
+		I2C_BOARD_INFO("24c32", 0x52), /* E0=0, E1=1, E2=0 */
+		.properties = board_eeprom_properties,
 	}, {
 		I2C_BOARD_INFO("pcf8563", 0x51),
 	}
diff --git a/arch/arm/mach-imx/mach-pcm037_eet.c b/arch/arm/mach-imx/mach-pcm037_eet.c
index 95bd97710494..15bc956d466b 100644
--- a/arch/arm/mach-imx/mach-pcm037_eet.c
+++ b/arch/arm/mach-imx/mach-pcm037_eet.c
@@ -56,11 +56,8 @@ static struct spi_board_info pcm037_spi_dev[] = {
 };
 
 /* Platform Data for MXC CSPI */
-static int pcm037_spi1_cs[] = { MXC_SPI_CS(0), MXC_SPI_CS(1), };
-
 static const struct spi_imx_master pcm037_spi1_pdata __initconst = {
-	.chipselect = pcm037_spi1_cs,
-	.num_chipselect = ARRAY_SIZE(pcm037_spi1_cs),
+	.num_chipselect = 2,
 };
 
 /* GPIO-keys input device */
diff --git a/arch/arm/mach-imx/mach-pcm043.c b/arch/arm/mach-imx/mach-pcm043.c
index 78e2bf8dcd96..e595e5368676 100644
--- a/arch/arm/mach-imx/mach-pcm043.c
+++ b/arch/arm/mach-imx/mach-pcm043.c
@@ -24,7 +24,7 @@
 #include <linux/interrupt.h>
 #include <linux/delay.h>
 #include <linux/i2c.h>
-#include <linux/platform_data/at24.h>
+#include <linux/property.h>
 #include <linux/usb/otg.h>
 #include <linux/usb/ulpi.h>
 
@@ -110,16 +110,15 @@ static const struct imxi2c_platform_data pcm043_i2c0_data __initconst = {
 	.bitrate = 50000,
 };
 
-static struct at24_platform_data board_eeprom = {
-	.byte_len = 4096,
-	.page_size = 32,
-	.flags = AT24_FLAG_ADDR16,
+static const struct property_entry board_eeprom_properties[] = {
+	PROPERTY_ENTRY_U32("pagesize", 32),
+	{ }
 };
 
 static struct i2c_board_info pcm043_i2c_devices[] = {
 	{
-		I2C_BOARD_INFO("at24", 0x52), /* E0=0, E1=1, E2=0 */
-		.platform_data = &board_eeprom,
+		I2C_BOARD_INFO("24c32", 0x52), /* E0=0, E1=1, E2=0 */
+		.properties = board_eeprom_properties,
 	}, {
 		I2C_BOARD_INFO("pcf8563", 0x51),
 	},
diff --git a/arch/arm/mach-imx/mach-vpr200.c b/arch/arm/mach-imx/mach-vpr200.c
index 5ff154c9a086..da3336aaa4c5 100644
--- a/arch/arm/mach-imx/mach-vpr200.c
+++ b/arch/arm/mach-imx/mach-vpr200.c
@@ -29,7 +29,6 @@
 #include <asm/mach/time.h>
 
 #include <linux/i2c.h>
-#include <linux/platform_data/at24.h>
 #include <linux/mfd/mc13xxx.h>
 
 #include "common.h"
@@ -145,15 +144,9 @@ static const struct imxi2c_platform_data vpr200_i2c0_data __initconst = {
 	.bitrate = 50000,
 };
 
-static struct at24_platform_data vpr200_eeprom = {
-	.byte_len = 2048 / 8,
-	.page_size = 1,
-};
-
 static struct i2c_board_info vpr200_i2c_devices[] = {
 	{
-		I2C_BOARD_INFO("at24", 0x50), /* E0=0, E1=0, E2=0 */
-		.platform_data = &vpr200_eeprom,
+		I2C_BOARD_INFO("24c02", 0x50), /* E0=0, E1=0, E2=0 */
 	}, {
 		I2C_BOARD_INFO("mc13892", 0x08),
 		.platform_data = &vpr200_pmic,
diff --git a/arch/arm/mach-ks8695/board-acs5k.c b/arch/arm/mach-ks8695/board-acs5k.c
index 937eb1d47e7b..ef835d82cdb9 100644
--- a/arch/arm/mach-ks8695/board-acs5k.c
+++ b/arch/arm/mach-ks8695/board-acs5k.c
@@ -19,7 +19,7 @@
 #include <linux/gpio/machine.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
-#include <linux/i2c-gpio.h>
+#include <linux/platform_data/i2c-gpio.h>
 #include <linux/platform_data/pca953x.h>
 
 #include <linux/mtd/mtd.h>
diff --git a/arch/arm/mach-meson/Kconfig b/arch/arm/mach-meson/Kconfig
index d90f61e6254f..d51cfda953d4 100644
--- a/arch/arm/mach-meson/Kconfig
+++ b/arch/arm/mach-meson/Kconfig
@@ -19,14 +19,7 @@ config MACH_MESON6
 	select MESON6_TIMER
 
 config MACH_MESON8
-	bool "Amlogic Meson8 SoCs support"
-	default ARCH_MESON
-	select MESON6_TIMER
-	select COMMON_CLK_MESON8B
-	select MESON_IRQ_GPIO
-
-config MACH_MESON8B
-	bool "Amlogic Meson8b SoCs support"
+	bool "Amlogic Meson8, Meson8b and Meson8m2 SoCs support"
 	default ARCH_MESON
 	select MESON6_TIMER
 	select COMMON_CLK_MESON8B
diff --git a/arch/arm/mach-meson/meson.c b/arch/arm/mach-meson/meson.c
index 4e2357178625..c8d99df32f9b 100644
--- a/arch/arm/mach-meson/meson.c
+++ b/arch/arm/mach-meson/meson.c
@@ -20,6 +20,7 @@ static const char * const meson_common_board_compat[] = {
 	"amlogic,meson6",
 	"amlogic,meson8",
 	"amlogic,meson8b",
+	"amlogic,meson8m2",
 	NULL,
 };
 
diff --git a/arch/arm/mach-mmp/brownstone.c b/arch/arm/mach-mmp/brownstone.c
index d1613b954926..563b5a278d65 100644
--- a/arch/arm/mach-mmp/brownstone.c
+++ b/arch/arm/mach-mmp/brownstone.c
@@ -15,6 +15,7 @@
 #include <linux/platform_device.h>
 #include <linux/io.h>
 #include <linux/gpio-pxa.h>
+#include <linux/gpio/machine.h>
 #include <linux/regulator/machine.h>
 #include <linux/regulator/max8649.h>
 #include <linux/regulator/fixed.h>
@@ -148,7 +149,6 @@ static struct regulator_init_data brownstone_v_5vp_data = {
 static struct fixed_voltage_config brownstone_v_5vp = {
 	.supply_name		= "v_5vp",
 	.microvolts		= 5000000,
-	.gpio			= GPIO_5V_ENABLE,
 	.enable_high		= 1,
 	.enabled_at_boot	= 1,
 	.init_data		= &brownstone_v_5vp_data,
@@ -162,6 +162,15 @@ static struct platform_device brownstone_v_5vp_device = {
 	},
 };
 
+static struct gpiod_lookup_table brownstone_v_5vp_gpiod_table = {
+	.dev_id = "reg-fixed-voltage.1", /* .id set to 1 above */
+	.table = {
+		GPIO_LOOKUP("gpio-pxa", GPIO_5V_ENABLE,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 static struct max8925_platform_data brownstone_max8925_info = {
 	.irq_base		= MMP_NR_IRQS,
 };
@@ -217,6 +226,7 @@ static void __init brownstone_init(void)
 	mmp2_add_isram(&mmp2_isram_platdata);
 
 	/* enable 5v regulator */
+	gpiod_add_lookup_table(&brownstone_v_5vp_gpiod_table);
 	platform_device_register(&brownstone_v_5vp_device);
 }
 
diff --git a/arch/arm/mach-omap1/board-ams-delta.c b/arch/arm/mach-omap1/board-ams-delta.c
index 52e8e53ca154..914218150cec 100644
--- a/arch/arm/mach-omap1/board-ams-delta.c
+++ b/arch/arm/mach-omap1/board-ams-delta.c
@@ -12,6 +12,7 @@
  * published by the Free Software Foundation.
  */
 #include <linux/gpio/driver.h>
+#include <linux/gpio/machine.h>
 #include <linux/gpio.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
@@ -202,7 +203,10 @@ static struct resource latch2_resources[] = {
 	},
 };
 
+#define LATCH2_LABEL	"ams-delta-latch2"
+
 static struct bgpio_pdata latch2_pdata = {
+	.label	= LATCH2_LABEL,
 	.base	= AMS_DELTA_LATCH2_GPIO_BASE,
 	.ngpio	= AMS_DELTA_LATCH2_NGPIO,
 };
@@ -217,6 +221,23 @@ static struct platform_device latch2_gpio_device = {
 	},
 };
 
+#define LATCH2_PIN_LCD_VBLEN		0
+#define LATCH2_PIN_LCD_NDISP		1
+#define LATCH2_PIN_NAND_NCE		2
+#define LATCH2_PIN_NAND_NRE		3
+#define LATCH2_PIN_NAND_NWP		4
+#define LATCH2_PIN_NAND_NWE		5
+#define LATCH2_PIN_NAND_ALE		6
+#define LATCH2_PIN_NAND_CLE		7
+#define LATCH2_PIN_KEYBRD_PWR		8
+#define LATCH2_PIN_KEYBRD_DATAOUT	9
+#define LATCH2_PIN_SCARD_RSTIN		10
+#define LATCH2_PIN_SCARD_CMDVCC		11
+#define LATCH2_PIN_MODEM_NRESET		12
+#define LATCH2_PIN_MODEM_CODEC		13
+#define LATCH2_PIN_HOOKFLASH1		14
+#define LATCH2_PIN_HOOKFLASH2		15
+
 static const struct gpio latch_gpios[] __initconst = {
 	{
 		.gpio	= LATCH1_GPIO_BASE + 6,
@@ -238,11 +259,6 @@ static const struct gpio latch_gpios[] __initconst = {
 		.flags	= GPIOF_OUT_INIT_LOW,
 		.label	= "scard_cmdvcc",
 	},
-	{
-		.gpio	= AMS_DELTA_GPIO_PIN_MODEM_CODEC,
-		.flags	= GPIOF_OUT_INIT_LOW,
-		.label	= "modem_codec",
-	},
 	{
 		.gpio	= AMS_DELTA_LATCH2_GPIO_BASE + 14,
 		.flags	= GPIOF_OUT_INIT_LOW,
@@ -272,7 +288,6 @@ static struct regulator_init_data modem_nreset_data = {
 static struct fixed_voltage_config modem_nreset_config = {
 	.supply_name		= "modem_nreset",
 	.microvolts		= 3300000,
-	.gpio			= AMS_DELTA_GPIO_PIN_MODEM_NRESET,
 	.startup_delay		= 25000,
 	.enable_high		= 1,
 	.enabled_at_boot	= 1,
@@ -287,6 +302,15 @@ static struct platform_device modem_nreset_device = {
 	},
 };
 
+static struct gpiod_lookup_table modem_nreset_gpiod_table = {
+	.dev_id = "reg-fixed-voltage",
+	.table = {
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_MODEM_NRESET,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 struct modem_private_data {
 	struct regulator *regulator;
 };
@@ -323,6 +347,22 @@ static struct platform_device ams_delta_nand_device = {
 	.resource	= ams_delta_nand_resources,
 };
 
+#define OMAP_GPIO_LABEL	"gpio-0-15"
+
+static struct gpiod_lookup_table ams_delta_nand_gpio_table = {
+	.table = {
+		GPIO_LOOKUP(OMAP_GPIO_LABEL, AMS_DELTA_GPIO_PIN_NAND_RB, "rdy",
+			    0),
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NCE, "nce", 0),
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NRE, "nre", 0),
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NWP, "nwp", 0),
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NWE, "nwe", 0),
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_ALE, "ale", 0),
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_CLE, "cle", 0),
+		{ },
+	},
+};
+
 static struct resource ams_delta_kp_resources[] = {
 	[0] = {
 		.start	= INT_KEYBOARD,
@@ -358,6 +398,14 @@ static struct platform_device ams_delta_lcd_device = {
 	.id	= -1,
 };
 
+static struct gpiod_lookup_table ams_delta_lcd_gpio_table = {
+	.table = {
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_LCD_VBLEN, "vblen", 0),
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_LCD_NDISP, "ndisp", 0),
+		{ },
+	},
+};
+
 static const struct gpio_led gpio_leds[] __initconst = {
 	{
 		.name		 = "camera",
@@ -449,11 +497,35 @@ static struct platform_device ams_delta_audio_device = {
 	.id     = -1,
 };
 
+static struct gpiod_lookup_table ams_delta_audio_gpio_table = {
+	.table = {
+		GPIO_LOOKUP(OMAP_GPIO_LABEL, AMS_DELTA_GPIO_PIN_HOOK_SWITCH,
+			    "hook_switch", 0),
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_MODEM_CODEC,
+			    "modem_codec", 0),
+		{ },
+	},
+};
+
 static struct platform_device cx20442_codec_device = {
 	.name   = "cx20442-codec",
 	.id     = -1,
 };
 
+static struct gpiod_lookup_table ams_delta_serio_gpio_table = {
+	.table = {
+		GPIO_LOOKUP(OMAP_GPIO_LABEL, AMS_DELTA_GPIO_PIN_KEYBRD_DATA,
+			    "data", 0),
+		GPIO_LOOKUP(OMAP_GPIO_LABEL, AMS_DELTA_GPIO_PIN_KEYBRD_CLK,
+			    "clock", 0),
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_KEYBRD_PWR,
+			    "power", 0),
+		GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_KEYBRD_DATAOUT,
+			    "dataout", 0),
+		{ },
+	},
+};
+
 static struct platform_device *ams_delta_devices[] __initdata = {
 	&latch1_gpio_device,
 	&latch2_gpio_device,
@@ -468,6 +540,16 @@ static struct platform_device *late_devices[] __initdata = {
 	&cx20442_codec_device,
 };
 
+static struct gpiod_lookup_table *ams_delta_gpio_tables[] __initdata = {
+	&ams_delta_audio_gpio_table,
+	&ams_delta_serio_gpio_table,
+};
+
+static struct gpiod_lookup_table *late_gpio_tables[] __initdata = {
+	&ams_delta_lcd_gpio_table,
+	&ams_delta_nand_gpio_table,
+};
+
 static void __init ams_delta_init(void)
 {
 	/* mux pins for uarts */
@@ -500,6 +582,20 @@ static void __init ams_delta_init(void)
 	gpio_led_register_device(-1, &leds_pdata);
 	platform_add_devices(ams_delta_devices, ARRAY_SIZE(ams_delta_devices));
 
+	/*
+	 * As soon as devices have been registered, assign their dev_names
+	 * to respective GPIO lookup tables before they are added.
+	 */
+	ams_delta_audio_gpio_table.dev_id =
+			dev_name(&ams_delta_audio_device.dev);
+	/*
+	 * No device name is assigned to GPIO lookup table for serio device
+	 * as long as serio driver is not converted to platform device driver.
+	 */
+
+	gpiod_add_lookup_tables(ams_delta_gpio_tables,
+				ARRAY_SIZE(ams_delta_gpio_tables));
+
 	ams_delta_init_fiq();
 
 	omap_writew(omap_readw(ARM_RSTCT1) | 0x0004, ARM_RSTCT1);
@@ -570,6 +666,15 @@ static int __init late_init(void)
 
 	platform_add_devices(late_devices, ARRAY_SIZE(late_devices));
 
+	/*
+	 * As soon as devices have been registered, assign their dev_names
+	 * to respective GPIO lookup tables before they are added.
+	 */
+	ams_delta_lcd_gpio_table.dev_id = dev_name(&ams_delta_lcd_device.dev);
+	ams_delta_nand_gpio_table.dev_id = dev_name(&ams_delta_nand_device.dev);
+
+	gpiod_add_lookup_tables(late_gpio_tables, ARRAY_SIZE(late_gpio_tables));
+	gpiod_add_lookup_table(&modem_nreset_gpiod_table);
 	err = platform_device_register(&modem_nreset_device);
 	if (err) {
 		pr_err("Couldn't register the modem regulator device\n");
diff --git a/arch/arm/mach-omap1/board-htcherald.c b/arch/arm/mach-omap1/board-htcherald.c
index 67d46690a56e..da8f3fc3180f 100644
--- a/arch/arm/mach-omap1/board-htcherald.c
+++ b/arch/arm/mach-omap1/board-htcherald.c
@@ -31,7 +31,7 @@
 #include <linux/gpio.h>
 #include <linux/gpio_keys.h>
 #include <linux/i2c.h>
-#include <linux/i2c-gpio.h>
+#include <linux/platform_data/i2c-gpio.h>
 #include <linux/htcpld.h>
 #include <linux/leds.h>
 #include <linux/spi/spi.h>
diff --git a/arch/arm/mach-omap1/board-osk.c b/arch/arm/mach-omap1/board-osk.c
index c66372ed29e2..9ffa8d755a59 100644
--- a/arch/arm/mach-omap1/board-osk.c
+++ b/arch/arm/mach-omap1/board-osk.c
@@ -303,22 +303,22 @@ static const struct omap_lcd_config osk_lcd_config __initconst = {
 #ifdef	CONFIG_OMAP_OSK_MISTRAL
 
 #include <linux/input.h>
-#include <linux/platform_data/at24.h>
+#include <linux/property.h>
 #include <linux/spi/spi.h>
 #include <linux/spi/ads7846.h>
 
 #include <linux/platform_data/keypad-omap.h>
 
-static struct at24_platform_data at24c04 = {
-	.byte_len	= SZ_4K / 8,
-	.page_size	= 16,
+static const struct property_entry mistral_at24_properties[] = {
+	PROPERTY_ENTRY_U32("pagesize", 16),
+	{ }
 };
 
 static struct i2c_board_info __initdata mistral_i2c_board_info[] = {
 	{
 		/* NOTE:  powered from LCD supply */
 		I2C_BOARD_INFO("24c04", 0x50),
-		.platform_data	= &at24c04,
+		.properties = mistral_at24_properties,
 	},
 	/* TODO when driver support is ready:
 	 *  - optionally ov9640 camera sensor at 0x30
diff --git a/arch/arm/mach-omap1/common.h b/arch/arm/mach-omap1/common.h
index d83ff257eaa8..c6537d2c2859 100644
--- a/arch/arm/mach-omap1/common.h
+++ b/arch/arm/mach-omap1/common.h
@@ -27,7 +27,7 @@
 #define __ARCH_ARM_MACH_OMAP1_COMMON_H
 
 #include <linux/mtd/mtd.h>
-#include <linux/i2c-omap.h>
+#include <linux/platform_data/i2c-omap.h>
 #include <linux/reboot.h>
 
 #include <asm/exception.h>
diff --git a/arch/arm/mach-omap1/i2c.c b/arch/arm/mach-omap1/i2c.c
index 5bdf3c4190f9..9250f263ac51 100644
--- a/arch/arm/mach-omap1/i2c.c
+++ b/arch/arm/mach-omap1/i2c.c
@@ -20,7 +20,7 @@
  */
 
 #include <linux/i2c.h>
-#include <linux/i2c-omap.h>
+#include <linux/platform_data/i2c-omap.h>
 #include <mach/mux.h>
 #include "soc.h"
 
diff --git a/arch/arm/mach-omap2/Makefile b/arch/arm/mach-omap2/Makefile
index 0d9ce58bc464..01377c292db4 100644
--- a/arch/arm/mach-omap2/Makefile
+++ b/arch/arm/mach-omap2/Makefile
@@ -78,7 +78,6 @@ endif
 omap-4-5-pm-common			= omap-mpuss-lowpower.o
 obj-$(CONFIG_ARCH_OMAP4)		+= $(omap-4-5-pm-common)
 obj-$(CONFIG_SOC_OMAP5)			+= $(omap-4-5-pm-common)
-obj-$(CONFIG_OMAP_PM_NOOP)		+= omap-pm-noop.o
 
 ifeq ($(CONFIG_PM),y)
 obj-$(CONFIG_ARCH_OMAP2)		+= pm24xx.o
diff --git a/arch/arm/mach-omap2/board-generic.c b/arch/arm/mach-omap2/board-generic.c
index 6c61ecc62905..6b4f4975cf7a 100644
--- a/arch/arm/mach-omap2/board-generic.c
+++ b/arch/arm/mach-omap2/board-generic.c
@@ -31,8 +31,6 @@ static const struct of_device_id omap_dt_match_table[] __initconst = {
 static void __init __maybe_unused omap_generic_init(void)
 {
 	pdata_quirks_init(omap_dt_match_table);
-
-	omapdss_init_of();
 	omap_soc_device_init();
 }
 
diff --git a/arch/arm/mach-omap2/clockdomain.c b/arch/arm/mach-omap2/clockdomain.c
index b79b1ca9aee9..6d44fe05a3fe 100644
--- a/arch/arm/mach-omap2/clockdomain.c
+++ b/arch/arm/mach-omap2/clockdomain.c
@@ -23,6 +23,7 @@
 #include <linux/limits.h>
 #include <linux/err.h>
 #include <linux/clk-provider.h>
+#include <linux/cpu_pm.h>
 
 #include <linux/io.h>
 
@@ -31,6 +32,7 @@
 #include "soc.h"
 #include "clock.h"
 #include "clockdomain.h"
+#include "pm.h"
 
 /* clkdm_list contains all registered struct clockdomains */
 static LIST_HEAD(clkdm_list);
@@ -39,6 +41,8 @@ static LIST_HEAD(clkdm_list);
 static struct clkdm_autodep *autodeps;
 
 static struct clkdm_ops *arch_clkdm;
+void clkdm_save_context(void);
+void clkdm_restore_context(void);
 
 /* Private functions */
 
@@ -449,6 +453,22 @@ int clkdm_register_autodeps(struct clkdm_autodep *ia)
 	return 0;
 }
 
+static int cpu_notifier(struct notifier_block *nb, unsigned long cmd, void *v)
+{
+	switch (cmd) {
+	case CPU_CLUSTER_PM_ENTER:
+		if (enable_off_mode)
+			clkdm_save_context();
+		break;
+	case CPU_CLUSTER_PM_EXIT:
+		if (enable_off_mode)
+			clkdm_restore_context();
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
 /**
  * clkdm_complete_init - set up the clockdomain layer
  *
@@ -460,6 +480,7 @@ int clkdm_register_autodeps(struct clkdm_autodep *ia)
 int clkdm_complete_init(void)
 {
 	struct clockdomain *clkdm;
+	static struct notifier_block nb;
 
 	if (list_empty(&clkdm_list))
 		return -EACCES;
@@ -474,6 +495,12 @@ int clkdm_complete_init(void)
 		clkdm_clear_all_sleepdeps(clkdm);
 	}
 
+	/* Only AM43XX can lose clkdm context during rtc-ddr suspend */
+	if (soc_is_am43xx()) {
+		nb.notifier_call = cpu_notifier;
+		cpu_pm_register_notifier(&nb);
+	}
+
 	return 0;
 }
 
@@ -1307,3 +1334,49 @@ int clkdm_hwmod_disable(struct clockdomain *clkdm, struct omap_hwmod *oh)
 	return 0;
 }
 
+/**
+ * _clkdm_save_context - save the context for the control of this clkdm
+ *
+ * Due to a suspend or hibernation operation, the state of the registers
+ * controlling this clkdm will be lost, save their context.
+ */
+static int _clkdm_save_context(struct clockdomain *clkdm, void *ununsed)
+{
+	if (!arch_clkdm || !arch_clkdm->clkdm_save_context)
+		return -EINVAL;
+
+	return arch_clkdm->clkdm_save_context(clkdm);
+}
+
+/**
+ * _clkdm_restore_context - restore context for control of this clkdm
+ *
+ * Restore the register values for this clockdomain.
+ */
+static int _clkdm_restore_context(struct clockdomain *clkdm, void *ununsed)
+{
+	if (!arch_clkdm || !arch_clkdm->clkdm_restore_context)
+		return -EINVAL;
+
+	return arch_clkdm->clkdm_restore_context(clkdm);
+}
+
+/**
+ * clkdm_save_context - Saves the context for each registered clkdm
+ *
+ * Save the context for each registered clockdomain.
+ */
+void clkdm_save_context(void)
+{
+	clkdm_for_each(_clkdm_save_context, NULL);
+}
+
+/**
+ * clkdm_restore_context - Restores the context for each registered clkdm
+ *
+ * Restore the context for each registered clockdomain.
+ */
+void clkdm_restore_context(void)
+{
+	clkdm_for_each(_clkdm_restore_context, NULL);
+}
diff --git a/arch/arm/mach-omap2/clockdomain.h b/arch/arm/mach-omap2/clockdomain.h
index 24667a5a9dc0..c7d0953e4aa2 100644
--- a/arch/arm/mach-omap2/clockdomain.h
+++ b/arch/arm/mach-omap2/clockdomain.h
@@ -141,6 +141,7 @@ struct clockdomain {
 	int usecount;
 	int forcewake_count;
 	struct list_head node;
+	u32 context;
 };
 
 /**
@@ -159,6 +160,8 @@ struct clockdomain {
  * @clkdm_deny_idle: Disable hw supervised idle transitions for clock domain
  * @clkdm_clk_enable: Put the clkdm in right state for a clock enable
  * @clkdm_clk_disable: Put the clkdm in right state for a clock disable
+ * @clkdm_save_context: Save the current clkdm context
+ * @clkdm_restore_context: Restore the clkdm context
  */
 struct clkdm_ops {
 	int	(*clkdm_add_wkdep)(struct clockdomain *clkdm1, struct clockdomain *clkdm2);
@@ -175,6 +178,8 @@ struct clkdm_ops {
 	void	(*clkdm_deny_idle)(struct clockdomain *clkdm);
 	int	(*clkdm_clk_enable)(struct clockdomain *clkdm);
 	int	(*clkdm_clk_disable)(struct clockdomain *clkdm);
+	int	(*clkdm_save_context)(struct clockdomain *clkdm);
+	int	(*clkdm_restore_context)(struct clockdomain *clkdm);
 };
 
 int clkdm_register_platform_funcs(struct clkdm_ops *co);
@@ -214,6 +219,9 @@ int clkdm_clk_disable(struct clockdomain *clkdm, struct clk *clk);
 int clkdm_hwmod_enable(struct clockdomain *clkdm, struct omap_hwmod *oh);
 int clkdm_hwmod_disable(struct clockdomain *clkdm, struct omap_hwmod *oh);
 
+void clkdm_save_context(void);
+void clkdm_restore_context(void);
+
 extern void __init omap242x_clockdomains_init(void);
 extern void __init omap243x_clockdomains_init(void);
 extern void __init omap3xxx_clockdomains_init(void);
diff --git a/arch/arm/mach-omap2/cm33xx.c b/arch/arm/mach-omap2/cm33xx.c
index 1cc0247a2cb5..084d454f6074 100644
--- a/arch/arm/mach-omap2/cm33xx.c
+++ b/arch/arm/mach-omap2/cm33xx.c
@@ -72,6 +72,17 @@ static inline u32 am33xx_cm_rmw_reg_bits(u32 mask, u32 bits, s16 inst, s16 idx)
 	return v;
 }
 
+static inline u32 am33xx_cm_read_reg_bits(u16 inst, s16 idx, u32 mask)
+{
+	u32 v;
+
+	v = am33xx_cm_read_reg(inst, idx);
+	v &= mask;
+	v >>= __ffs(mask);
+
+	return v;
+}
+
 /**
  * _clkctrl_idlest - read a CM_*_CLKCTRL register; mask & shift IDLEST bitfield
  * @inst: CM instance register offset (*_INST macro)
@@ -338,6 +349,46 @@ static u32 am33xx_cm_xlate_clkctrl(u8 part, u16 inst, u16 offset)
 	return cm_base.pa + inst + offset;
 }
 
+/**
+ * am33xx_clkdm_save_context - Save the clockdomain transition context
+ * @clkdm: The clockdomain pointer whose context needs to be saved
+ *
+ * Save the clockdomain transition context.
+ */
+static int am33xx_clkdm_save_context(struct clockdomain *clkdm)
+{
+	clkdm->context = am33xx_cm_read_reg_bits(clkdm->cm_inst,
+						 clkdm->clkdm_offs,
+						 AM33XX_CLKTRCTRL_MASK);
+
+	return 0;
+}
+
+/**
+ * am33xx_restore_save_context - Restore the clockdomain transition context
+ * @clkdm: The clockdomain pointer whose context needs to be restored
+ *
+ * Restore the clockdomain transition context.
+ */
+static int am33xx_clkdm_restore_context(struct clockdomain *clkdm)
+{
+	switch (clkdm->context) {
+	case OMAP34XX_CLKSTCTRL_DISABLE_AUTO:
+		am33xx_clkdm_deny_idle(clkdm);
+		break;
+	case OMAP34XX_CLKSTCTRL_FORCE_SLEEP:
+		am33xx_clkdm_sleep(clkdm);
+		break;
+	case OMAP34XX_CLKSTCTRL_FORCE_WAKEUP:
+		am33xx_clkdm_wakeup(clkdm);
+		break;
+	case OMAP34XX_CLKSTCTRL_ENABLE_AUTO:
+		am33xx_clkdm_allow_idle(clkdm);
+		break;
+	}
+	return 0;
+}
+
 struct clkdm_ops am33xx_clkdm_operations = {
 	.clkdm_sleep		= am33xx_clkdm_sleep,
 	.clkdm_wakeup		= am33xx_clkdm_wakeup,
@@ -345,6 +396,8 @@ struct clkdm_ops am33xx_clkdm_operations = {
 	.clkdm_deny_idle	= am33xx_clkdm_deny_idle,
 	.clkdm_clk_enable	= am33xx_clkdm_clk_enable,
 	.clkdm_clk_disable	= am33xx_clkdm_clk_disable,
+	.clkdm_save_context	= am33xx_clkdm_save_context,
+	.clkdm_restore_context	= am33xx_clkdm_restore_context,
 };
 
 static const struct cm_ll_data am33xx_cm_ll_data = {
diff --git a/arch/arm/mach-omap2/cminst44xx.c b/arch/arm/mach-omap2/cminst44xx.c
index 7deefee49fc3..c11ac492b626 100644
--- a/arch/arm/mach-omap2/cminst44xx.c
+++ b/arch/arm/mach-omap2/cminst44xx.c
@@ -481,6 +481,47 @@ static u32 omap4_cminst_xlate_clkctrl(u8 part, u16 inst, u16 offset)
 	return _cm_bases[part].pa + inst + offset;
 }
 
+/**
+ * omap4_clkdm_save_context - Save the clockdomain modulemode context
+ * @clkdm: The clockdomain pointer whose context needs to be saved
+ *
+ * Save the clockdomain modulemode context.
+ */
+static int omap4_clkdm_save_context(struct clockdomain *clkdm)
+{
+	clkdm->context = omap4_cminst_read_inst_reg(clkdm->prcm_partition,
+						    clkdm->cm_inst,
+						    clkdm->clkdm_offs +
+						    OMAP4_CM_CLKSTCTRL);
+	clkdm->context &= OMAP4430_MODULEMODE_MASK;
+	return 0;
+}
+
+/**
+ * omap4_clkdm_restore_context - Restore the clockdomain modulemode context
+ * @clkdm: The clockdomain pointer whose context needs to be restored
+ *
+ * Restore the clockdomain modulemode context.
+ */
+static int omap4_clkdm_restore_context(struct clockdomain *clkdm)
+{
+	switch (clkdm->context) {
+	case OMAP34XX_CLKSTCTRL_DISABLE_AUTO:
+		omap4_clkdm_deny_idle(clkdm);
+		break;
+	case OMAP34XX_CLKSTCTRL_FORCE_SLEEP:
+		omap4_clkdm_sleep(clkdm);
+		break;
+	case OMAP34XX_CLKSTCTRL_FORCE_WAKEUP:
+		omap4_clkdm_wakeup(clkdm);
+		break;
+	case OMAP34XX_CLKSTCTRL_ENABLE_AUTO:
+		omap4_clkdm_allow_idle(clkdm);
+		break;
+	}
+	return 0;
+}
+
 struct clkdm_ops omap4_clkdm_operations = {
 	.clkdm_add_wkdep	= omap4_clkdm_add_wkup_sleep_dep,
 	.clkdm_del_wkdep	= omap4_clkdm_del_wkup_sleep_dep,
@@ -496,6 +537,8 @@ struct clkdm_ops omap4_clkdm_operations = {
 	.clkdm_deny_idle	= omap4_clkdm_deny_idle,
 	.clkdm_clk_enable	= omap4_clkdm_clk_enable,
 	.clkdm_clk_disable	= omap4_clkdm_clk_disable,
+	.clkdm_save_context	= omap4_clkdm_save_context,
+	.clkdm_restore_context	= omap4_clkdm_restore_context,
 };
 
 struct clkdm_ops am43xx_clkdm_operations = {
diff --git a/arch/arm/mach-omap2/common.h b/arch/arm/mach-omap2/common.h
index fbe0b78bf489..129455e822e4 100644
--- a/arch/arm/mach-omap2/common.h
+++ b/arch/arm/mach-omap2/common.h
@@ -30,7 +30,7 @@
 #include <linux/delay.h>
 #include <linux/i2c.h>
 #include <linux/mfd/twl.h>
-#include <linux/i2c-omap.h>
+#include <linux/platform_data/i2c-omap.h>
 #include <linux/reboot.h>
 #include <linux/irqchip/irq-omap-intc.h>
 
@@ -44,6 +44,9 @@
 
 #define OMAP_INTC_START		NR_IRQS
 
+extern int (*omap_pm_soc_init)(void);
+int omap_pm_nop_init(void);
+
 #if defined(CONFIG_PM) && defined(CONFIG_ARCH_OMAP2)
 int omap2_pm_init(void);
 #else
@@ -79,9 +82,12 @@ static inline int omap4_pm_init_early(void)
 
 #if defined(CONFIG_PM) && (defined(CONFIG_SOC_AM33XX) || \
 	defined(CONFIG_SOC_AM43XX))
-void amx3_common_pm_init(void);
+int amx3_common_pm_init(void);
 #else
-static inline void amx3_common_pm_init(void) { }
+static inline int amx3_common_pm_init(void)
+{
+	return 0;
+}
 #endif
 
 extern void omap2_init_common_infrastructure(void);
@@ -122,14 +128,10 @@ void am43xx_init_early(void);
 void am43xx_init_late(void);
 void omap4430_init_early(void);
 void omap5_init_early(void);
-void omap3_init_late(void);	/* Do not use this one */
+void omap3_init_late(void);
 void omap4430_init_late(void);
 void omap2420_init_late(void);
 void omap2430_init_late(void);
-void omap3430_init_late(void);
-void omap35xx_init_late(void);
-void omap3630_init_late(void);
-void am35xx_init_late(void);
 void ti81xx_init_late(void);
 void am33xx_init_late(void);
 void omap5_init_late(void);
@@ -350,7 +352,5 @@ extern int omap_dss_reset(struct omap_hwmod *);
 /* SoC specific clock initializer */
 int omap_clk_init(void);
 
-int __init omapdss_init_of(void);
-
 #endif /* __ASSEMBLER__ */
 #endif /* __ARCH_ARM_MACH_OMAP2PLUS_COMMON_H */
diff --git a/arch/arm/mach-omap2/control.c b/arch/arm/mach-omap2/control.c
index 180da403639e..0bbfb20e193f 100644
--- a/arch/arm/mach-omap2/control.c
+++ b/arch/arm/mach-omap2/control.c
@@ -17,6 +17,7 @@
 #include <linux/of_address.h>
 #include <linux/regmap.h>
 #include <linux/mfd/syscon.h>
+#include <linux/cpu_pm.h>
 
 #include "soc.h"
 #include "iomap.h"
@@ -621,6 +622,110 @@ void __init omap3_ctrl_init(void)
 }
 #endif /* CONFIG_ARCH_OMAP3 && CONFIG_PM */
 
+static unsigned long am43xx_control_reg_offsets[] = {
+	AM33XX_CONTROL_SYSCONFIG_OFFSET,
+	AM33XX_CONTROL_STATUS_OFFSET,
+	AM43XX_CONTROL_MPU_L2_CTRL_OFFSET,
+	AM33XX_CONTROL_CORE_SLDO_CTRL_OFFSET,
+	AM33XX_CONTROL_MPU_SLDO_CTRL_OFFSET,
+	AM33XX_CONTROL_CLK32KDIVRATIO_CTRL_OFFSET,
+	AM33XX_CONTROL_BANDGAP_CTRL_OFFSET,
+	AM33XX_CONTROL_BANDGAP_TRIM_OFFSET,
+	AM33XX_CONTROL_PLL_CLKINPULOW_CTRL_OFFSET,
+	AM33XX_CONTROL_MOSC_CTRL_OFFSET,
+	AM33XX_CONTROL_DEEPSLEEP_CTRL_OFFSET,
+	AM43XX_CONTROL_DISPLAY_PLL_SEL_OFFSET,
+	AM33XX_CONTROL_INIT_PRIORITY_0_OFFSET,
+	AM33XX_CONTROL_INIT_PRIORITY_1_OFFSET,
+	AM33XX_CONTROL_TPTC_CFG_OFFSET,
+	AM33XX_CONTROL_USB_CTRL0_OFFSET,
+	AM33XX_CONTROL_USB_CTRL1_OFFSET,
+	AM43XX_CONTROL_USB_CTRL2_OFFSET,
+	AM43XX_CONTROL_GMII_SEL_OFFSET,
+	AM43XX_CONTROL_MPUSS_CTRL_OFFSET,
+	AM43XX_CONTROL_TIMER_CASCADE_CTRL_OFFSET,
+	AM43XX_CONTROL_PWMSS_CTRL_OFFSET,
+	AM33XX_CONTROL_MREQPRIO_0_OFFSET,
+	AM33XX_CONTROL_MREQPRIO_1_OFFSET,
+	AM33XX_CONTROL_HW_EVENT_SEL_GRP1_OFFSET,
+	AM33XX_CONTROL_HW_EVENT_SEL_GRP2_OFFSET,
+	AM33XX_CONTROL_HW_EVENT_SEL_GRP3_OFFSET,
+	AM33XX_CONTROL_HW_EVENT_SEL_GRP4_OFFSET,
+	AM33XX_CONTROL_SMRT_CTRL_OFFSET,
+	AM33XX_CONTROL_MPUSS_HW_DEBUG_SEL_OFFSET,
+	AM43XX_CONTROL_CQDETECT_STS_OFFSET,
+	AM43XX_CONTROL_CQDETECT_STS2_OFFSET,
+	AM43XX_CONTROL_VTP_CTRL_OFFSET,
+	AM33XX_CONTROL_VREF_CTRL_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_0_3_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_4_7_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_8_11_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_12_15_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_16_19_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_20_23_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_24_27_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_28_31_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_32_35_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_36_39_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_40_43_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_44_47_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_48_51_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_52_55_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_56_59_OFFSET,
+	AM33XX_CONTROL_TPCC_EVT_MUX_60_63_OFFSET,
+	AM33XX_CONTROL_TIMER_EVT_CAPT_OFFSET,
+	AM33XX_CONTROL_ECAP_EVT_CAPT_OFFSET,
+	AM33XX_CONTROL_ADC_EVT_CAPT_OFFSET,
+	AM43XX_CONTROL_ADC1_EVT_CAPT_OFFSET,
+	AM33XX_CONTROL_RESET_ISO_OFFSET,
+};
+
+static u32 am33xx_control_vals[ARRAY_SIZE(am43xx_control_reg_offsets)];
+
+/**
+ * am43xx_control_save_context - Save the wakeup domain registers
+ *
+ * Save the wkup domain registers
+ */
+void am43xx_control_save_context(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(am43xx_control_reg_offsets); i++)
+		am33xx_control_vals[i] =
+				omap_ctrl_readl(am43xx_control_reg_offsets[i]);
+}
+
+/**
+ * am43xx_control_restore_context - Restore the wakeup domain registers
+ *
+ * Restore the wkup domain registers
+ */
+void am43xx_control_restore_context(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(am43xx_control_reg_offsets); i++)
+		omap_ctrl_writel(am33xx_control_vals[i],
+				 am43xx_control_reg_offsets[i]);
+}
+
+static int cpu_notifier(struct notifier_block *nb, unsigned long cmd, void *v)
+{
+	switch (cmd) {
+	case CPU_CLUSTER_PM_ENTER:
+		if (enable_off_mode)
+			am43xx_control_save_context();
+		break;
+	case CPU_CLUSTER_PM_EXIT:
+		if (enable_off_mode)
+			am43xx_control_restore_context();
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
 struct control_init_data {
 	int index;
 	void __iomem *mem;
@@ -699,6 +804,7 @@ int __init omap_control_init(void)
 	const struct omap_prcm_init_data *data;
 	int ret;
 	struct regmap *syscon;
+	static struct notifier_block nb;
 
 	for_each_matching_node_and_match(np, omap_scrm_dt_match_table, &match) {
 		data = match->data;
@@ -731,6 +837,12 @@ int __init omap_control_init(void)
 		}
 	}
 
+	/* Only AM43XX can lose ctrl registers context during rtc-ddr suspend */
+	if (soc_is_am43xx()) {
+		nb.notifier_call = cpu_notifier;
+		cpu_pm_register_notifier(&nb);
+	}
+
 	return 0;
 }
 
diff --git a/arch/arm/mach-omap2/control.h b/arch/arm/mach-omap2/control.h
index ec406bc2c6d4..393b42110511 100644
--- a/arch/arm/mach-omap2/control.h
+++ b/arch/arm/mach-omap2/control.h
@@ -409,6 +409,67 @@
 #define AM33XX_DEV_FEATURE		0x604
 #define AM33XX_SGX_MASK			BIT(29)
 
+/* Additional AM33XX/AM43XX CONTROL registers */
+#define AM33XX_CONTROL_SYSCONFIG_OFFSET			0x0010
+#define AM33XX_CONTROL_STATUS_OFFSET			0x0040
+#define AM43XX_CONTROL_MPU_L2_CTRL_OFFSET		0x01e0
+#define AM33XX_CONTROL_CORTEX_VBBLDO_CTRL_OFFSET	0x041c
+#define AM33XX_CONTROL_CORE_SLDO_CTRL_OFFSET		0x0428
+#define AM33XX_CONTROL_MPU_SLDO_CTRL_OFFSET		0x042c
+#define AM33XX_CONTROL_CLK32KDIVRATIO_CTRL_OFFSET	0x0444
+#define AM33XX_CONTROL_BANDGAP_CTRL_OFFSET		0x0448
+#define AM33XX_CONTROL_BANDGAP_TRIM_OFFSET		0x044c
+#define AM33XX_CONTROL_PLL_CLKINPULOW_CTRL_OFFSET	0x0458
+#define AM33XX_CONTROL_MOSC_CTRL_OFFSET			0x0468
+#define AM33XX_CONTROL_RCOSC_CTRL_OFFSET		0x046c
+#define AM33XX_CONTROL_DEEPSLEEP_CTRL_OFFSET		0x0470
+#define AM43XX_CONTROL_DISPLAY_PLL_SEL_OFFSET		0x0534
+#define AM33XX_CONTROL_INIT_PRIORITY_0_OFFSET		0x0608
+#define AM33XX_CONTROL_INIT_PRIORITY_1_OFFSET		0x060c
+#define AM33XX_CONTROL_MMU_CFG_OFFSET			0x0610
+#define AM33XX_CONTROL_TPTC_CFG_OFFSET			0x0614
+#define AM33XX_CONTROL_USB_CTRL0_OFFSET			0x0620
+#define AM33XX_CONTROL_USB_CTRL1_OFFSET			0x0628
+#define AM33XX_CONTROL_USB_WKUP_CTRL_OFFSET		0x0648
+#define AM43XX_CONTROL_USB_CTRL2_OFFSET			0x064c
+#define AM43XX_CONTROL_GMII_SEL_OFFSET			0x0650
+#define AM43XX_CONTROL_MPUSS_CTRL_OFFSET		0x0654
+#define AM43XX_CONTROL_TIMER_CASCADE_CTRL_OFFSET	0x0658
+#define AM43XX_CONTROL_PWMSS_CTRL_OFFSET		0x0664
+#define AM33XX_CONTROL_MREQPRIO_0_OFFSET		0x0670
+#define AM33XX_CONTROL_MREQPRIO_1_OFFSET		0x0674
+#define AM33XX_CONTROL_HW_EVENT_SEL_GRP1_OFFSET		0x0690
+#define AM33XX_CONTROL_HW_EVENT_SEL_GRP2_OFFSET		0x0694
+#define AM33XX_CONTROL_HW_EVENT_SEL_GRP3_OFFSET		0x0698
+#define AM33XX_CONTROL_HW_EVENT_SEL_GRP4_OFFSET		0x069c
+#define AM33XX_CONTROL_SMRT_CTRL_OFFSET			0x06a0
+#define AM33XX_CONTROL_MPUSS_HW_DEBUG_SEL_OFFSET	0x06a4
+#define AM43XX_CONTROL_CQDETECT_STS_OFFSET		0x0e00
+#define AM43XX_CONTROL_CQDETECT_STS2_OFFSET		0x0e08
+#define AM43XX_CONTROL_VTP_CTRL_OFFSET			0x0e0c
+#define AM33XX_CONTROL_VREF_CTRL_OFFSET			0x0e14
+#define AM33XX_CONTROL_TPCC_EVT_MUX_0_3_OFFSET		0x0f90
+#define AM33XX_CONTROL_TPCC_EVT_MUX_4_7_OFFSET		0x0f94
+#define AM33XX_CONTROL_TPCC_EVT_MUX_8_11_OFFSET		0x0f98
+#define AM33XX_CONTROL_TPCC_EVT_MUX_12_15_OFFSET	0x0f9c
+#define AM33XX_CONTROL_TPCC_EVT_MUX_16_19_OFFSET	0x0fa0
+#define AM33XX_CONTROL_TPCC_EVT_MUX_20_23_OFFSET	0x0fa4
+#define AM33XX_CONTROL_TPCC_EVT_MUX_24_27_OFFSET	0x0fa8
+#define AM33XX_CONTROL_TPCC_EVT_MUX_28_31_OFFSET	0x0fac
+#define AM33XX_CONTROL_TPCC_EVT_MUX_32_35_OFFSET	0x0fb0
+#define AM33XX_CONTROL_TPCC_EVT_MUX_36_39_OFFSET	0x0fb4
+#define AM33XX_CONTROL_TPCC_EVT_MUX_40_43_OFFSET	0x0fb8
+#define AM33XX_CONTROL_TPCC_EVT_MUX_44_47_OFFSET	0x0fbc
+#define AM33XX_CONTROL_TPCC_EVT_MUX_48_51_OFFSET	0x0fc0
+#define AM33XX_CONTROL_TPCC_EVT_MUX_52_55_OFFSET	0x0fc4
+#define AM33XX_CONTROL_TPCC_EVT_MUX_56_59_OFFSET	0x0fc8
+#define AM33XX_CONTROL_TPCC_EVT_MUX_60_63_OFFSET	0x0fcc
+#define AM33XX_CONTROL_TIMER_EVT_CAPT_OFFSET		0x0fd0
+#define AM33XX_CONTROL_ECAP_EVT_CAPT_OFFSET		0x0fd4
+#define AM33XX_CONTROL_ADC_EVT_CAPT_OFFSET		0x0fd8
+#define AM43XX_CONTROL_ADC1_EVT_CAPT_OFFSET		0x0fdc
+#define AM33XX_CONTROL_RESET_ISO_OFFSET			0x1000
+
 /* CONTROL OMAP STATUS register to identify OMAP3 features */
 #define OMAP3_CONTROL_OMAP_STATUS	0x044c
 
diff --git a/arch/arm/mach-omap2/display.c b/arch/arm/mach-omap2/display.c
index b3f6eb5d04a2..9500b6e27380 100644
--- a/arch/arm/mach-omap2/display.c
+++ b/arch/arm/mach-omap2/display.c
@@ -32,7 +32,6 @@
 #include <linux/platform_data/omapdss.h>
 #include "omap_hwmod.h"
 #include "omap_device.h"
-#include "omap-pm.h"
 #include "common.h"
 
 #include "soc.h"
@@ -126,11 +125,6 @@ static void omap_dsi_disable_pads(int dsi_id, unsigned lane_mask)
 		omap4_dsi_mux_pads(dsi_id, 0);
 }
 
-static int omap_dss_set_min_bus_tput(struct device *dev, unsigned long tput)
-{
-	return omap_pm_set_min_bus_tput(dev, OCP_INITIATOR_AGENT, tput);
-}
-
 static enum omapdss_version __init omap_display_get_version(void)
 {
 	if (cpu_is_omap24xx())
@@ -169,7 +163,6 @@ static int __init omapdss_init_fbdev(void)
 	static struct omap_dss_board_info board_data = {
 		.dsi_enable_pads = omap_dsi_enable_pads,
 		.dsi_disable_pads = omap_dsi_disable_pads,
-		.set_min_bus_tput = omap_dss_set_min_bus_tput,
 	};
 	struct device_node *node;
 	int r;
@@ -392,7 +385,7 @@ static struct device_node * __init omapdss_find_dss_of_node(void)
 	return NULL;
 }
 
-int __init omapdss_init_of(void)
+static int __init omapdss_init_of(void)
 {
 	int r;
 	struct device_node *node;
@@ -422,3 +415,4 @@ int __init omapdss_init_of(void)
 
 	return omapdss_init_fbdev();
 }
+omap_device_initcall(omapdss_init_of);
diff --git a/arch/arm/mach-omap2/hsmmc.c b/arch/arm/mach-omap2/hsmmc.c
index b064066d431c..0103548b0b15 100644
--- a/arch/arm/mach-omap2/hsmmc.c
+++ b/arch/arm/mach-omap2/hsmmc.c
@@ -18,7 +18,6 @@
 
 #include "soc.h"
 #include "omap_device.h"
-#include "omap-pm.h"
 
 #include "hsmmc.h"
 #include "control.h"
diff --git a/arch/arm/mach-omap2/i2c.c b/arch/arm/mach-omap2/i2c.c
index 91a21c3923b2..37ff25ee3d89 100644
--- a/arch/arm/mach-omap2/i2c.c
+++ b/arch/arm/mach-omap2/i2c.c
@@ -22,7 +22,6 @@
 #include "soc.h"
 #include "omap_hwmod.h"
 #include "omap_device.h"
-#include "omap-pm.h"
 
 #include "prm.h"
 #include "common.h"
diff --git a/arch/arm/mach-omap2/io.c b/arch/arm/mach-omap2/io.c
index cf546dfe3b32..bb8e0bb7ef5d 100644
--- a/arch/arm/mach-omap2/io.c
+++ b/arch/arm/mach-omap2/io.c
@@ -37,7 +37,6 @@
 #include "clock.h"
 #include "clock2xxx.h"
 #include "clock3xxx.h"
-#include "omap-pm.h"
 #include "sdrc.h"
 #include "control.h"
 #include "serial.h"
@@ -421,13 +420,6 @@ static void __init __maybe_unused omap_hwmod_init_postsetup(void)
 	postsetup_state = _HWMOD_STATE_ENABLED;
 #endif
 	omap_hwmod_for_each(_set_hwmod_postsetup_state, &postsetup_state);
-
-	omap_pm_if_early_init();
-}
-
-static void __init __maybe_unused omap_common_late_init(void)
-{
-	omap2_common_pm_late_init();
 }
 
 #ifdef CONFIG_SOC_OMAP2420
@@ -450,9 +442,7 @@ void __init omap2420_init_early(void)
 
 void __init omap2420_init_late(void)
 {
-	omap_common_late_init();
-	omap2_pm_init();
-	omap2_clk_enable_autoidle_all();
+	omap_pm_soc_init = omap2_pm_init;
 }
 #endif
 
@@ -476,9 +466,7 @@ void __init omap2430_init_early(void)
 
 void __init omap2430_init_late(void)
 {
-	omap_common_late_init();
-	omap2_pm_init();
-	omap2_clk_enable_autoidle_all();
+	omap_pm_soc_init = omap2_pm_init;
 }
 #endif
 
@@ -529,43 +517,12 @@ void __init am35xx_init_early(void)
 
 void __init omap3_init_late(void)
 {
-	omap_common_late_init();
-	omap3_pm_init();
-	omap2_clk_enable_autoidle_all();
-}
-
-void __init omap3430_init_late(void)
-{
-	omap_common_late_init();
-	omap3_pm_init();
-	omap2_clk_enable_autoidle_all();
-}
-
-void __init omap35xx_init_late(void)
-{
-	omap_common_late_init();
-	omap3_pm_init();
-	omap2_clk_enable_autoidle_all();
-}
-
-void __init omap3630_init_late(void)
-{
-	omap_common_late_init();
-	omap3_pm_init();
-	omap2_clk_enable_autoidle_all();
-}
-
-void __init am35xx_init_late(void)
-{
-	omap_common_late_init();
-	omap3_pm_init();
-	omap2_clk_enable_autoidle_all();
+	omap_pm_soc_init = omap3_pm_init;
 }
 
 void __init ti81xx_init_late(void)
 {
-	omap_common_late_init();
-	omap2_clk_enable_autoidle_all();
+	omap_pm_soc_init = omap_pm_nop_init;
 }
 #endif
 
@@ -621,8 +578,7 @@ void __init am33xx_init_early(void)
 
 void __init am33xx_init_late(void)
 {
-	omap_common_late_init();
-	amx3_common_pm_init();
+	omap_pm_soc_init = amx3_common_pm_init;
 }
 #endif
 
@@ -645,9 +601,7 @@ void __init am43xx_init_early(void)
 
 void __init am43xx_init_late(void)
 {
-	omap_common_late_init();
-	omap2_clk_enable_autoidle_all();
-	amx3_common_pm_init();
+	omap_pm_soc_init = amx3_common_pm_init;
 }
 #endif
 
@@ -675,9 +629,7 @@ void __init omap4430_init_early(void)
 
 void __init omap4430_init_late(void)
 {
-	omap_common_late_init();
-	omap4_pm_init();
-	omap2_clk_enable_autoidle_all();
+	omap_pm_soc_init = omap4_pm_init;
 }
 #endif
 
@@ -703,9 +655,7 @@ void __init omap5_init_early(void)
 
 void __init omap5_init_late(void)
 {
-	omap_common_late_init();
-	omap4_pm_init();
-	omap2_clk_enable_autoidle_all();
+	omap_pm_soc_init = omap4_pm_init;
 }
 #endif
 
@@ -728,9 +678,7 @@ void __init dra7xx_init_early(void)
 
 void __init dra7xx_init_late(void)
 {
-	omap_common_late_init();
-	omap4_pm_init();
-	omap2_clk_enable_autoidle_all();
+	omap_pm_soc_init = omap4_pm_init;
 }
 #endif
 
diff --git a/arch/arm/mach-omap2/omap-pm-noop.c b/arch/arm/mach-omap2/omap-pm-noop.c
deleted file mode 100644
index 4ead077ea4e7..000000000000
--- a/arch/arm/mach-omap2/omap-pm-noop.c
+++ /dev/null
@@ -1,176 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * omap-pm-noop.c - OMAP power management interface - dummy version
- *
- * This code implements the OMAP power management interface to
- * drivers, CPUIdle, CPUFreq, and DSP Bridge.  It is strictly for
- * debug/demonstration use, as it does nothing but printk() whenever a
- * function is called (when DEBUG is defined, below)
- *
- * Copyright (C) 2008-2009 Texas Instruments, Inc.
- * Copyright (C) 2008-2009 Nokia Corporation
- * Paul Walmsley
- *
- * Interface developed by (in alphabetical order):
- * Karthik Dasu, Tony Lindgren, Rajendra Nayak, Sakari Poussa, Veeramanikandan
- * Raju, Anand Sawant, Igor Stoppa, Paul Walmsley, Richard Woodruff
- */
-
-#undef DEBUG
-
-#include <linux/init.h>
-#include <linux/cpufreq.h>
-#include <linux/device.h>
-#include <linux/platform_device.h>
-
-#include "omap_device.h"
-#include "omap-pm.h"
-
-static bool off_mode_enabled;
-static int dummy_context_loss_counter;
-
-/*
- * Device-driver-originated constraints (via board-*.c files)
- */
-
-int omap_pm_set_max_mpu_wakeup_lat(struct device *dev, long t)
-{
-	if (!dev || t < -1) {
-		WARN(1, "OMAP PM: %s: invalid parameter(s)", __func__);
-		return -EINVAL;
-	}
-
-	if (t == -1)
-		pr_debug("OMAP PM: remove max MPU wakeup latency constraint: dev %s\n",
-			 dev_name(dev));
-	else
-		pr_debug("OMAP PM: add max MPU wakeup latency constraint: dev %s, t = %ld usec\n",
-			 dev_name(dev), t);
-
-	/*
-	 * For current Linux, this needs to map the MPU to a
-	 * powerdomain, then go through the list of current max lat
-	 * constraints on the MPU and find the smallest.  If
-	 * the latency constraint has changed, the code should
-	 * recompute the state to enter for the next powerdomain
-	 * state.
-	 *
-	 * TI CDP code can call constraint_set here.
-	 */
-
-	return 0;
-}
-
-int omap_pm_set_min_bus_tput(struct device *dev, u8 agent_id, unsigned long r)
-{
-	if (!dev || (agent_id != OCP_INITIATOR_AGENT &&
-	    agent_id != OCP_TARGET_AGENT)) {
-		WARN(1, "OMAP PM: %s: invalid parameter(s)", __func__);
-		return -EINVAL;
-	}
-
-	if (r == 0)
-		pr_debug("OMAP PM: remove min bus tput constraint: dev %s for agent_id %d\n",
-			 dev_name(dev), agent_id);
-	else
-		pr_debug("OMAP PM: add min bus tput constraint: dev %s for agent_id %d: rate %ld KiB\n",
-			 dev_name(dev), agent_id, r);
-
-	/*
-	 * This code should model the interconnect and compute the
-	 * required clock frequency, convert that to a VDD2 OPP ID, then
-	 * set the VDD2 OPP appropriately.
-	 *
-	 * TI CDP code can call constraint_set here on the VDD2 OPP.
-	 */
-
-	return 0;
-}
-
-/*
- * DSP Bridge-specific constraints
- */
-
-
-/**
- * omap_pm_enable_off_mode - notify OMAP PM that off-mode is enabled
- *
- * Intended for use only by OMAP PM core code to notify this layer
- * that off mode has been enabled.
- */
-void omap_pm_enable_off_mode(void)
-{
-	off_mode_enabled = true;
-}
-
-/**
- * omap_pm_disable_off_mode - notify OMAP PM that off-mode is disabled
- *
- * Intended for use only by OMAP PM core code to notify this layer
- * that off mode has been disabled.
- */
-void omap_pm_disable_off_mode(void)
-{
-	off_mode_enabled = false;
-}
-
-/*
- * Device context loss tracking
- */
-
-#ifdef CONFIG_ARCH_OMAP2PLUS
-
-int omap_pm_get_dev_context_loss_count(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	int count;
-
-	if (WARN_ON(!dev))
-		return -ENODEV;
-
-	if (dev->pm_domain == &omap_device_pm_domain) {
-		count = omap_device_get_context_loss_count(pdev);
-	} else {
-		WARN_ONCE(off_mode_enabled, "omap_pm: using dummy context loss counter; device %s should be converted to omap_device",
-			  dev_name(dev));
-
-		count = dummy_context_loss_counter;
-
-		if (off_mode_enabled) {
-			count++;
-			/*
-			 * Context loss count has to be a non-negative value.
-			 * Clear the sign bit to get a value range from 0 to
-			 * INT_MAX.
-			 */
-			count &= INT_MAX;
-			dummy_context_loss_counter = count;
-		}
-	}
-
-	pr_debug("OMAP PM: context loss count for dev %s = %d\n",
-		 dev_name(dev), count);
-
-	return count;
-}
-
-#else
-
-int omap_pm_get_dev_context_loss_count(struct device *dev)
-{
-	return dummy_context_loss_counter;
-}
-
-#endif
-
-/* Should be called before clk framework init */
-int __init omap_pm_if_early_init(void)
-{
-	return 0;
-}
-
-/* Must be called after clock framework is initialized */
-int __init omap_pm_if_init(void)
-{
-	return 0;
-}
diff --git a/arch/arm/mach-omap2/omap-pm.h b/arch/arm/mach-omap2/omap-pm.h
deleted file mode 100644
index 5ba5df47f91b..000000000000
--- a/arch/arm/mach-omap2/omap-pm.h
+++ /dev/null
@@ -1,161 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * omap-pm.h - OMAP power management interface
- *
- * Copyright (C) 2008-2010 Texas Instruments, Inc.
- * Copyright (C) 2008-2010 Nokia Corporation
- * Paul Walmsley
- *
- * Interface developed by (in alphabetical order): Karthik Dasu, Jouni
- * Högander, Tony Lindgren, Rajendra Nayak, Sakari Poussa,
- * Veeramanikandan Raju, Anand Sawant, Igor Stoppa, Paul Walmsley,
- * Richard Woodruff
- */
-
-#ifndef ASM_ARM_ARCH_OMAP_OMAP_PM_H
-#define ASM_ARM_ARCH_OMAP_OMAP_PM_H
-
-#include <linux/device.h>
-#include <linux/cpufreq.h>
-#include <linux/clk.h>
-#include <linux/pm_opp.h>
-
-/*
- * agent_id values for use with omap_pm_set_min_bus_tput():
- *
- * OCP_INITIATOR_AGENT is only valid for devices that can act as
- * initiators -- it represents the device's L3 interconnect
- * connection.  OCP_TARGET_AGENT represents the device's L4
- * interconnect connection.
- */
-#define OCP_TARGET_AGENT		1
-#define OCP_INITIATOR_AGENT		2
-
-/**
- * omap_pm_if_early_init - OMAP PM init code called before clock fw init
- * @mpu_opp_table: array ptr to struct omap_opp for MPU
- * @dsp_opp_table: array ptr to struct omap_opp for DSP
- * @l3_opp_table : array ptr to struct omap_opp for CORE
- *
- * Initialize anything that must be configured before the clock
- * framework starts.  The "_if_" is to avoid name collisions with the
- * PM idle-loop code.
- */
-int __init omap_pm_if_early_init(void);
-
-/**
- * omap_pm_if_init - OMAP PM init code called after clock fw init
- *
- * The main initialization code.  OPP tables are passed in here.  The
- * "_if_" is to avoid name collisions with the PM idle-loop code.
- */
-int __init omap_pm_if_init(void);
-
-/*
- * Device-driver-originated constraints (via board-*.c files, platform_data)
- */
-
-
-/**
- * omap_pm_set_max_mpu_wakeup_lat - set the maximum MPU wakeup latency
- * @dev: struct device * requesting the constraint
- * @t: maximum MPU wakeup latency in microseconds
- *
- * Request that the maximum interrupt latency for the MPU to be no
- * greater than @t microseconds. "Interrupt latency" in this case is
- * defined as the elapsed time from the occurrence of a hardware or
- * timer interrupt to the time when the device driver's interrupt
- * service routine has been entered by the MPU.
- *
- * It is intended that underlying PM code will use this information to
- * determine what power state to put the MPU powerdomain into, and
- * possibly the CORE powerdomain as well, since interrupt handling
- * code currently runs from SDRAM.  Advanced PM or board*.c code may
- * also configure interrupt controller priorities, OCP bus priorities,
- * CPU speed(s), etc.
- *
- * This function will not affect device wakeup latency, e.g., time
- * elapsed from when a device driver enables a hardware device with
- * clk_enable(), to when the device is ready for register access or
- * other use.  To control this device wakeup latency, use
- * omap_pm_set_max_dev_wakeup_lat()
- *
- * Multiple calls to omap_pm_set_max_mpu_wakeup_lat() will replace the
- * previous t value.  To remove the latency target for the MPU, call
- * with t = -1.
- *
- * XXX This constraint will be deprecated soon in favor of the more
- * general omap_pm_set_max_dev_wakeup_lat()
- *
- * Returns -EINVAL for an invalid argument, -ERANGE if the constraint
- * is not satisfiable, or 0 upon success.
- */
-int omap_pm_set_max_mpu_wakeup_lat(struct device *dev, long t);
-
-
-/**
- * omap_pm_set_min_bus_tput - set minimum bus throughput needed by device
- * @dev: struct device * requesting the constraint
- * @tbus_id: interconnect to operate on (OCP_{INITIATOR,TARGET}_AGENT)
- * @r: minimum throughput (in KiB/s)
- *
- * Request that the minimum data throughput on the OCP interconnect
- * attached to device @dev interconnect agent @tbus_id be no less
- * than @r KiB/s.
- *
- * It is expected that the OMAP PM or bus code will use this
- * information to set the interconnect clock to run at the lowest
- * possible speed that satisfies all current system users.  The PM or
- * bus code will adjust the estimate based on its model of the bus, so
- * device driver authors should attempt to specify an accurate
- * quantity for their device use case, and let the PM or bus code
- * overestimate the numbers as necessary to handle request/response
- * latency, other competing users on the system, etc.  On OMAP2/3, if
- * a driver requests a minimum L4 interconnect speed constraint, the
- * code will also need to add an minimum L3 interconnect speed
- * constraint,
- *
- * Multiple calls to omap_pm_set_min_bus_tput() will replace the
- * previous rate value for this device.  To remove the interconnect
- * throughput restriction for this device, call with r = 0.
- *
- * Returns -EINVAL for an invalid argument, -ERANGE if the constraint
- * is not satisfiable, or 0 upon success.
- */
-int omap_pm_set_min_bus_tput(struct device *dev, u8 agent_id, unsigned long r);
-
-
-/*
- * CPUFreq-originated constraint
- *
- * In the future, this should be handled by custom OPP clocktype
- * functions.
- */
-
-
-/*
- * Device context loss tracking
- */
-
-/**
- * omap_pm_get_dev_context_loss_count - return count of times dev has lost ctx
- * @dev: struct device *
- *
- * This function returns the number of times that the device @dev has
- * lost its internal context.  This generally occurs on a powerdomain
- * transition to OFF.  Drivers use this as an optimization to avoid restoring
- * context if the device hasn't lost it.  To use, drivers should initially
- * call this in their context save functions and store the result.  Early in
- * the driver's context restore function, the driver should call this function
- * again, and compare the result to the stored counter.  If they differ, the
- * driver must restore device context.   If the number of context losses
- * exceeds the maximum positive integer, the function will wrap to 0 and
- * continue counting.  Returns the number of context losses for this device,
- * or negative value upon error.
- */
-int omap_pm_get_dev_context_loss_count(struct device *dev);
-
-void omap_pm_enable_off_mode(void);
-void omap_pm_disable_off_mode(void);
-
-#endif
diff --git a/arch/arm/mach-omap2/omap_device.c b/arch/arm/mach-omap2/omap_device.c
index 3b829a50d1db..ac219b9e6a4c 100644
--- a/arch/arm/mach-omap2/omap_device.c
+++ b/arch/arm/mach-omap2/omap_device.c
@@ -143,7 +143,7 @@ static int omap_device_build_from_dt(struct platform_device *pdev)
 	struct resource res;
 	const char *oh_name;
 	int oh_cnt, i, ret = 0;
-	bool device_active = false;
+	bool device_active = false, skip_pm_domain = false;
 
 	oh_cnt = of_property_count_strings(node, "ti,hwmods");
 	if (oh_cnt <= 0) {
@@ -151,8 +151,15 @@ static int omap_device_build_from_dt(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
+	/* SDMA still needs special handling for omap_device_build() */
+	ret = of_property_read_string_index(node, "ti,hwmods", 0, &oh_name);
+	if (!ret && (!strncmp("dma_system", oh_name, 10) ||
+		     !strncmp("dma", oh_name, 3)))
+		skip_pm_domain = true;
+
 	/* Use ti-sysc driver instead of omap_device? */
-	if (!omap_hwmod_parse_module_range(NULL, node, &res))
+	if (!skip_pm_domain &&
+	    !omap_hwmod_parse_module_range(NULL, node, &res))
 		return -ENODEV;
 
 	hwmods = kzalloc(sizeof(struct omap_hwmod *) * oh_cnt, GFP_KERNEL);
@@ -191,11 +198,12 @@ static int omap_device_build_from_dt(struct platform_device *pdev)
 			r->name = dev_name(&pdev->dev);
 	}
 
-	dev_pm_domain_set(&pdev->dev, &omap_device_pm_domain);
-
-	if (device_active) {
-		omap_device_enable(pdev);
-		pm_runtime_set_active(&pdev->dev);
+	if (!skip_pm_domain) {
+		dev_pm_domain_set(&pdev->dev, &omap_device_pm_domain);
+		if (device_active) {
+			omap_device_enable(pdev);
+			pm_runtime_set_active(&pdev->dev);
+		}
 	}
 
 odbfd_exit1:
diff --git a/arch/arm/mach-omap2/omap_hwmod.c b/arch/arm/mach-omap2/omap_hwmod.c
index e7d23e200ecc..2ceffd85dd3d 100644
--- a/arch/arm/mach-omap2/omap_hwmod.c
+++ b/arch/arm/mach-omap2/omap_hwmod.c
@@ -481,7 +481,7 @@ static int _wait_softreset_complete(struct omap_hwmod *oh)
 
 	sysc = oh->class->sysc;
 
-	if (sysc->sysc_flags & SYSS_HAS_RESET_STATUS)
+	if (sysc->sysc_flags & SYSS_HAS_RESET_STATUS && sysc->syss_offs > 0)
 		omap_test_timeout((omap_hwmod_read(oh, sysc->syss_offs)
 				   & SYSS_RESETDONE_MASK),
 				  MAX_MODULE_SOFTRESET_WAIT, c);
@@ -3171,19 +3171,19 @@ static int omap_hwmod_init_regbits(struct device *dev,
  */
 int omap_hwmod_init_reg_offs(struct device *dev,
 			     const struct ti_sysc_module_data *data,
-			     u32 *rev_offs, u32 *sysc_offs, u32 *syss_offs)
+			     s32 *rev_offs, s32 *sysc_offs, s32 *syss_offs)
 {
-	*rev_offs = 0;
+	*rev_offs = -ENODEV;
 	*sysc_offs = 0;
 	*syss_offs = 0;
 
-	if (data->offsets[SYSC_REVISION] > 0)
+	if (data->offsets[SYSC_REVISION] >= 0)
 		*rev_offs = data->offsets[SYSC_REVISION];
 
-	if (data->offsets[SYSC_SYSCONFIG] > 0)
+	if (data->offsets[SYSC_SYSCONFIG] >= 0)
 		*sysc_offs = data->offsets[SYSC_SYSCONFIG];
 
-	if (data->offsets[SYSC_SYSSTATUS] > 0)
+	if (data->offsets[SYSC_SYSSTATUS] >= 0)
 		*syss_offs = data->offsets[SYSC_SYSSTATUS];
 
 	return 0;
@@ -3312,8 +3312,8 @@ static int omap_hwmod_check_module(struct device *dev,
 				   struct omap_hwmod *oh,
 				   const struct ti_sysc_module_data *data,
 				   struct sysc_regbits *sysc_fields,
-				   u32 rev_offs, u32 sysc_offs,
-				   u32 syss_offs, u32 sysc_flags,
+				   s32 rev_offs, s32 sysc_offs,
+				   s32 syss_offs, u32 sysc_flags,
 				   u32 idlemodes)
 {
 	if (!oh->class->sysc)
@@ -3365,7 +3365,7 @@ static int omap_hwmod_check_module(struct device *dev,
 int omap_hwmod_allocate_module(struct device *dev, struct omap_hwmod *oh,
 			       const struct ti_sysc_module_data *data,
 			       struct sysc_regbits *sysc_fields,
-			       u32 rev_offs, u32 sysc_offs, u32 syss_offs,
+			       s32 rev_offs, s32 sysc_offs, s32 syss_offs,
 			       u32 sysc_flags, u32 idlemodes)
 {
 	struct omap_hwmod_class_sysconfig *sysc;
@@ -3425,7 +3425,8 @@ int omap_hwmod_init_module(struct device *dev,
 {
 	struct omap_hwmod *oh;
 	struct sysc_regbits *sysc_fields;
-	u32 rev_offs, sysc_offs, syss_offs, sysc_flags, idlemodes;
+	s32 rev_offs, sysc_offs, syss_offs;
+	u32 sysc_flags, idlemodes;
 	int error;
 
 	if (!dev || !data)
diff --git a/arch/arm/mach-omap2/omap_hwmod.h b/arch/arm/mach-omap2/omap_hwmod.h
index c7122abbf977..b70cdc21f8a2 100644
--- a/arch/arm/mach-omap2/omap_hwmod.h
+++ b/arch/arm/mach-omap2/omap_hwmod.h
@@ -317,9 +317,9 @@ struct omap_hwmod_ocp_if {
  * then this field has to be populated with the correct offset structure.
  */
 struct omap_hwmod_class_sysconfig {
-	u32 rev_offs;
-	u32 sysc_offs;
-	u32 syss_offs;
+	s32 rev_offs;
+	s32 sysc_offs;
+	s32 syss_offs;
 	u16 sysc_flags;
 	struct sysc_regbits *sysc_fields;
 	u8 srst_udelay;
diff --git a/arch/arm/mach-omap2/omap_hwmod_2420_data.c b/arch/arm/mach-omap2/omap_hwmod_2420_data.c
index fe66cf247874..d684fac8f592 100644
--- a/arch/arm/mach-omap2/omap_hwmod_2420_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_2420_data.c
@@ -13,7 +13,7 @@
  * XXX these should be marked initdata for multi-OMAP kernels
  */
 
-#include <linux/i2c-omap.h>
+#include <linux/platform_data/i2c-omap.h>
 #include <linux/omap-dma.h>
 
 #include "omap_hwmod.h"
diff --git a/arch/arm/mach-omap2/omap_hwmod_2430_data.c b/arch/arm/mach-omap2/omap_hwmod_2430_data.c
index 74eefd30518c..abef9f6f9bf5 100644
--- a/arch/arm/mach-omap2/omap_hwmod_2430_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_2430_data.c
@@ -13,7 +13,7 @@
  * XXX these should be marked initdata for multi-OMAP kernels
  */
 
-#include <linux/i2c-omap.h>
+#include <linux/platform_data/i2c-omap.h>
 #include <linux/platform_data/hsmmc-omap.h>
 #include <linux/omap-dma.h>
 
diff --git a/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c b/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c
index 5efe91c6e95b..9ded7bf972e7 100644
--- a/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c
@@ -629,6 +629,7 @@ struct omap_hwmod am33xx_gpmc_hwmod = {
 
 /* 'i2c' class */
 static struct omap_hwmod_class_sysconfig am33xx_i2c_sysc = {
+	.rev_offs	= 0,
 	.sysc_offs	= 0x0010,
 	.syss_offs	= 0x0090,
 	.sysc_flags	= (SYSC_HAS_AUTOIDLE | SYSC_HAS_CLOCKACTIVITY |
diff --git a/arch/arm/mach-omap2/omap_hwmod_33xx_data.c b/arch/arm/mach-omap2/omap_hwmod_33xx_data.c
index 53e1ac3724f2..c9483bc06228 100644
--- a/arch/arm/mach-omap2/omap_hwmod_33xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_33xx_data.c
@@ -14,7 +14,7 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/i2c-omap.h>
+#include <linux/platform_data/i2c-omap.h>
 
 #include "omap_hwmod.h"
 #include "omap_hwmod_common_data.h"
diff --git a/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c b/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c
index 23336b6c7125..23e6a41a18eb 100644
--- a/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c
@@ -15,7 +15,7 @@
  * XXX these should be marked initdata for multi-OMAP kernels
  */
 
-#include <linux/i2c-omap.h>
+#include <linux/platform_data/i2c-omap.h>
 #include <linux/power/smartreflex.h>
 #include <linux/platform_data/hsmmc-omap.h>
 
@@ -885,6 +885,7 @@ static struct omap_hwmod omap3xxx_dma_system_hwmod = {
  */
 
 static struct omap_hwmod_class_sysconfig omap3xxx_mcbsp_sysc = {
+	.rev_offs	= -ENODEV,
 	.sysc_offs	= 0x008c,
 	.sysc_flags	= (SYSC_HAS_CLOCKACTIVITY | SYSC_HAS_ENAWAKEUP |
 			   SYSC_HAS_SIDLEMODE | SYSC_HAS_SOFTRESET),
@@ -990,6 +991,7 @@ static struct omap_hwmod omap3xxx_mcbsp5_hwmod = {
 
 /* 'mcbsp sidetone' class */
 static struct omap_hwmod_class_sysconfig omap3xxx_mcbsp_sidetone_sysc = {
+	.rev_offs	= -ENODEV,
 	.sysc_offs	= 0x0010,
 	.sysc_flags	= SYSC_HAS_AUTOIDLE,
 	.sysc_fields	= &omap_hwmod_sysc_type1,
@@ -1018,6 +1020,7 @@ static struct omap_hwmod omap3xxx_mcbsp3_sidetone_hwmod = {
 
 /* SR common */
 static struct omap_hwmod_class_sysconfig omap34xx_sr_sysc = {
+	.rev_offs	= -ENODEV,
 	.sysc_offs	= 0x24,
 	.sysc_flags	= (SYSC_HAS_CLOCKACTIVITY | SYSC_NO_CACHE),
 	.sysc_fields	= &omap34xx_sr_sysc_fields,
@@ -1030,6 +1033,7 @@ static struct omap_hwmod_class omap34xx_smartreflex_hwmod_class = {
 };
 
 static struct omap_hwmod_class_sysconfig omap36xx_sr_sysc = {
+	.rev_offs	= -ENODEV,
 	.sysc_offs	= 0x38,
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART),
 	.sysc_flags	= (SYSC_HAS_SIDLEMODE | SYSC_HAS_ENAWAKEUP |
diff --git a/arch/arm/mach-omap2/omap_hwmod_43xx_data.c b/arch/arm/mach-omap2/omap_hwmod_43xx_data.c
index 5f73b730d4fc..aa271ac5ebac 100644
--- a/arch/arm/mach-omap2/omap_hwmod_43xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_43xx_data.c
@@ -378,6 +378,7 @@ static struct omap_hwmod am43xx_usb_otg_ss1_hwmod = {
 };
 
 static struct omap_hwmod_class_sysconfig am43xx_qspi_sysc = {
+	.rev_offs	= 0,
 	.sysc_offs      = 0x0010,
 	.sysc_flags     = SYSC_HAS_SIDLEMODE,
 	.idlemodes      = (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
diff --git a/arch/arm/mach-omap2/omap_hwmod_44xx_data.c b/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
index e4f8ae9cd637..a95dbac57a81 100644
--- a/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
@@ -23,7 +23,7 @@
 #include <linux/io.h>
 #include <linux/platform_data/hsmmc-omap.h>
 #include <linux/power/smartreflex.h>
-#include <linux/i2c-omap.h>
+#include <linux/platform_data/i2c-omap.h>
 
 #include <linux/omap-dma.h>
 
@@ -1360,6 +1360,7 @@ static struct omap_hwmod omap44xx_hsi_hwmod = {
  */
 
 static struct omap_hwmod_class_sysconfig omap44xx_i2c_sysc = {
+	.rev_offs	= 0,
 	.sysc_offs	= 0x0010,
 	.syss_offs	= 0x0090,
 	.sysc_flags	= (SYSC_HAS_AUTOIDLE | SYSC_HAS_CLOCKACTIVITY |
@@ -1634,6 +1635,7 @@ static struct omap_hwmod omap44xx_mailbox_hwmod = {
 
 /* The IP is not compliant to type1 / type2 scheme */
 static struct omap_hwmod_class_sysconfig omap44xx_mcasp_sysc = {
+	.rev_offs	= 0,
 	.sysc_offs	= 0x0004,
 	.sysc_flags	= SYSC_HAS_SIDLEMODE,
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
@@ -1667,6 +1669,7 @@ static struct omap_hwmod omap44xx_mcasp_hwmod = {
  */
 
 static struct omap_hwmod_class_sysconfig omap44xx_mcbsp_sysc = {
+	.rev_offs	= -ENODEV,
 	.sysc_offs	= 0x008c,
 	.sysc_flags	= (SYSC_HAS_CLOCKACTIVITY | SYSC_HAS_ENAWAKEUP |
 			   SYSC_HAS_SIDLEMODE | SYSC_HAS_SOFTRESET),
@@ -2353,6 +2356,7 @@ static struct omap_hwmod omap44xx_slimbus2_hwmod = {
 
 /* The IP is not compliant to type1 / type2 scheme */
 static struct omap_hwmod_class_sysconfig omap44xx_smartreflex_sysc = {
+	.rev_offs	= -ENODEV,
 	.sysc_offs	= 0x0038,
 	.sysc_flags	= (SYSC_HAS_ENAWAKEUP | SYSC_HAS_SIDLEMODE),
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
diff --git a/arch/arm/mach-omap2/omap_hwmod_54xx_data.c b/arch/arm/mach-omap2/omap_hwmod_54xx_data.c
index c72cd84b07ec..115473d441cd 100644
--- a/arch/arm/mach-omap2/omap_hwmod_54xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_54xx_data.c
@@ -20,7 +20,7 @@
 #include <linux/io.h>
 #include <linux/platform_data/hsmmc-omap.h>
 #include <linux/power/smartreflex.h>
-#include <linux/i2c-omap.h>
+#include <linux/platform_data/i2c-omap.h>
 
 #include <linux/omap-dma.h>
 
@@ -804,6 +804,7 @@ static struct omap_hwmod omap54xx_gpio8_hwmod = {
  */
 
 static struct omap_hwmod_class_sysconfig omap54xx_i2c_sysc = {
+	.rev_offs	= 0,
 	.sysc_offs	= 0x0010,
 	.syss_offs	= 0x0090,
 	.sysc_flags	= (SYSC_HAS_AUTOIDLE | SYSC_HAS_CLOCKACTIVITY |
@@ -974,6 +975,7 @@ static struct omap_hwmod omap54xx_mailbox_hwmod = {
  */
 
 static struct omap_hwmod_class_sysconfig omap54xx_mcbsp_sysc = {
+	.rev_offs	= -ENODEV,
 	.sysc_offs	= 0x008c,
 	.sysc_flags	= (SYSC_HAS_CLOCKACTIVITY | SYSC_HAS_ENAWAKEUP |
 			   SYSC_HAS_SIDLEMODE | SYSC_HAS_SOFTRESET),
@@ -1997,6 +1999,7 @@ static struct omap_hwmod omap54xx_ocp2scp3_hwmod = {
  */
 
 static struct omap_hwmod_class_sysconfig omap54xx_sata_sysc = {
+	.rev_offs	= 0x00fc,
 	.sysc_offs	= 0x0000,
 	.sysc_flags	= (SYSC_HAS_MIDLEMODE | SYSC_HAS_SIDLEMODE),
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
diff --git a/arch/arm/mach-omap2/omap_hwmod_7xx_data.c b/arch/arm/mach-omap2/omap_hwmod_7xx_data.c
index 62352d1e6361..e6c7061a8e73 100644
--- a/arch/arm/mach-omap2/omap_hwmod_7xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_7xx_data.c
@@ -20,7 +20,7 @@
 #include <linux/io.h>
 #include <linux/platform_data/hsmmc-omap.h>
 #include <linux/power/smartreflex.h>
-#include <linux/i2c-omap.h>
+#include <linux/platform_data/i2c-omap.h>
 
 #include <linux/omap-dma.h>
 
@@ -1070,6 +1070,7 @@ static struct omap_hwmod dra7xx_hdq1w_hwmod = {
  */
 
 static struct omap_hwmod_class_sysconfig dra7xx_i2c_sysc = {
+	.rev_offs	= 0,
 	.sysc_offs	= 0x0010,
 	.syss_offs	= 0x0090,
 	.sysc_flags	= (SYSC_HAS_AUTOIDLE | SYSC_HAS_CLOCKACTIVITY |
@@ -1440,6 +1441,7 @@ static struct omap_hwmod dra7xx_mcspi4_hwmod = {
  *
  */
 static struct omap_hwmod_class_sysconfig dra7xx_mcasp_sysc = {
+	.rev_offs	= 0,
 	.sysc_offs	= 0x0004,
 	.sysc_flags	= SYSC_HAS_SIDLEMODE,
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART),
@@ -1898,6 +1900,7 @@ static struct omap_hwmod dra7xx_pciess2_hwmod = {
  */
 
 static struct omap_hwmod_class_sysconfig dra7xx_qspi_sysc = {
+	.rev_offs	= 0,
 	.sysc_offs	= 0x0010,
 	.sysc_flags	= SYSC_HAS_SIDLEMODE,
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
@@ -1930,6 +1933,7 @@ static struct omap_hwmod dra7xx_qspi_hwmod = {
  *
  */
 static struct omap_hwmod_class_sysconfig dra7xx_rtcss_sysc = {
+	.rev_offs	= 0x0074,
 	.sysc_offs	= 0x0078,
 	.sysc_flags	= SYSC_HAS_SIDLEMODE,
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
@@ -1965,6 +1969,7 @@ static struct omap_hwmod dra7xx_rtcss_hwmod = {
  */
 
 static struct omap_hwmod_class_sysconfig dra7xx_sata_sysc = {
+	.rev_offs	= 0x00fc,
 	.sysc_offs	= 0x0000,
 	.sysc_flags	= (SYSC_HAS_MIDLEMODE | SYSC_HAS_SIDLEMODE),
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
@@ -2003,6 +2008,7 @@ static struct omap_hwmod dra7xx_sata_hwmod = {
 
 /* The IP is not compliant to type1 / type2 scheme */
 static struct omap_hwmod_class_sysconfig dra7xx_smartreflex_sysc = {
+	.rev_offs	= -ENODEV,
 	.sysc_offs	= 0x0038,
 	.sysc_flags	= (SYSC_HAS_ENAWAKEUP | SYSC_HAS_SIDLEMODE),
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
diff --git a/arch/arm/mach-omap2/omap_hwmod_81xx_data.c b/arch/arm/mach-omap2/omap_hwmod_81xx_data.c
index 686655f884c1..8e44e2728620 100644
--- a/arch/arm/mach-omap2/omap_hwmod_81xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_81xx_data.c
@@ -954,6 +954,7 @@ static struct omap_hwmod_ocp_if dm816x_l4_hs__emac1 = {
 };
 
 static struct omap_hwmod_class_sysconfig dm81xx_sata_sysc = {
+	.rev_offs	= 0x00fc,
 	.sysc_offs	= 0x1100,
 	.sysc_flags	= SYSC_HAS_SIDLEMODE,
 	.idlemodes	= SIDLE_FORCE,
diff --git a/arch/arm/mach-omap2/pdata-quirks.c b/arch/arm/mach-omap2/pdata-quirks.c
index 6459816c2879..74aa74180c0c 100644
--- a/arch/arm/mach-omap2/pdata-quirks.c
+++ b/arch/arm/mach-omap2/pdata-quirks.c
@@ -10,6 +10,7 @@
 #include <linux/clk.h>
 #include <linux/davinci_emac.h>
 #include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/of_platform.h>
@@ -26,14 +27,12 @@
 #include <linux/platform_data/iommu-omap.h>
 #include <linux/platform_data/ti-sysc.h>
 #include <linux/platform_data/wkup_m3.h>
-#include <linux/platform_data/media/ir-rx51.h>
 #include <linux/platform_data/asoc-ti-mcbsp.h>
 
 #include "common.h"
 #include "common-board-devices.h"
 #include "control.h"
 #include "omap_device.h"
-#include "omap-pm.h"
 #include "omap-secure.h"
 #include "soc.h"
 #include "hsmmc.h"
@@ -330,7 +329,6 @@ static struct regulator_init_data pandora_vmmc3 = {
 static struct fixed_voltage_config pandora_vwlan = {
 	.supply_name		= "vwlan",
 	.microvolts		= 1800000, /* 1.8V */
-	.gpio			= PANDORA_WIFI_NRESET_GPIO,
 	.startup_delay		= 50000, /* 50ms */
 	.enable_high		= 1,
 	.init_data		= &pandora_vmmc3,
@@ -344,6 +342,19 @@ static struct platform_device pandora_vwlan_device = {
 	},
 };
 
+static struct gpiod_lookup_table pandora_vwlan_gpiod_table = {
+	.dev_id = "reg-fixed-voltage.1",
+	.table = {
+		/*
+		 * As this is a low GPIO number it should be at the first
+		 * GPIO bank.
+		 */
+		GPIO_LOOKUP("gpio-0-31", PANDORA_WIFI_NRESET_GPIO,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 static void pandora_wl1251_init_card(struct mmc_card *card)
 {
 	/*
@@ -405,6 +416,7 @@ static void __init pandora_wl1251_init(void)
 static void __init omap3_pandora_legacy_init(void)
 {
 	platform_device_register(&pandora_backlight);
+	gpiod_add_lookup_table(&pandora_vwlan_gpiod_table);
 	platform_device_register(&pandora_vwlan_device);
 	omap_hsmmc_init(pandora_mmc3);
 	omap_hsmmc_late_init(pandora_mmc3);
@@ -514,18 +526,6 @@ void omap_auxdata_legacy_init(struct device *dev)
 	dev->platform_data = &twl_gpio_auxdata;
 }
 
-static struct ir_rx51_platform_data __maybe_unused rx51_ir_data = {
-	.set_max_mpu_wakeup_lat = omap_pm_set_max_mpu_wakeup_lat,
-};
-
-static struct platform_device __maybe_unused rx51_ir_device = {
-	.name           = "ir_rx51",
-	.id             = -1,
-	.dev            = {
-		.platform_data = &rx51_ir_data,
-	},
-};
-
 #if IS_ENABLED(CONFIG_SND_OMAP_SOC_MCBSP)
 static struct omap_mcbsp_platform_data mcbsp_pdata;
 static void __init omap3_mcbsp_init(void)
@@ -569,7 +569,6 @@ static struct of_dev_auxdata omap_auxdata_lookup[] = {
 		       "480c9000.smartreflex", &omap_sr_pdata[OMAP_SR_MPU]),
 	OF_DEV_AUXDATA("ti,omap3-hsmmc", 0x4809c000, "4809c000.mmc", &mmc_pdata[0]),
 	OF_DEV_AUXDATA("ti,omap3-hsmmc", 0x480b4000, "480b4000.mmc", &mmc_pdata[1]),
-	OF_DEV_AUXDATA("nokia,n900-ir", 0, "n900-ir", &rx51_ir_data),
 	/* Only on am3517 */
 	OF_DEV_AUXDATA("ti,davinci_mdio", 0x5c030000, "davinci_mdio.0", NULL),
 	OF_DEV_AUXDATA("ti,am3517-emac", 0x5c000000, "davinci_emac.0",
diff --git a/arch/arm/mach-omap2/pm-debug.c b/arch/arm/mach-omap2/pm-debug.c
index 5c46ea6756d7..acb698d5780f 100644
--- a/arch/arm/mach-omap2/pm-debug.c
+++ b/arch/arm/mach-omap2/pm-debug.c
@@ -31,7 +31,6 @@
 #include "clock.h"
 #include "powerdomain.h"
 #include "clockdomain.h"
-#include "omap-pm.h"
 
 #include "soc.h"
 #include "cm2xxx_3xxx.h"
@@ -240,10 +239,6 @@ static int option_set(void *data, u64 val)
 	*option = val;
 
 	if (option == &enable_off_mode) {
-		if (val)
-			omap_pm_enable_off_mode();
-		else
-			omap_pm_disable_off_mode();
 		if (cpu_is_omap34xx())
 			omap3_pm_off_mode_enable(val);
 	}
diff --git a/arch/arm/mach-omap2/pm.c b/arch/arm/mach-omap2/pm.c
index 6f68576e5695..ca03af8fe43f 100644
--- a/arch/arm/mach-omap2/pm.c
+++ b/arch/arm/mach-omap2/pm.c
@@ -16,11 +16,11 @@
 #include <linux/pm_opp.h>
 #include <linux/export.h>
 #include <linux/suspend.h>
+#include <linux/clk.h>
 #include <linux/cpu.h>
 
 #include <asm/system_misc.h>
 
-#include "omap-pm.h"
 #include "omap_device.h"
 #include "common.h"
 
@@ -230,16 +230,20 @@ static void __init omap4_init_voltages(void)
 	omap2_set_init_voltage("iva", "dpll_iva_m5x2_ck", "iva");
 }
 
-static int __init omap2_common_pm_init(void)
+int __maybe_unused omap_pm_nop_init(void)
 {
-	omap_pm_if_init();
-
 	return 0;
 }
-omap_postcore_initcall(omap2_common_pm_init);
+
+int (*omap_pm_soc_init)(void);
 
 int __init omap2_common_pm_late_init(void)
 {
+	int error;
+
+	if (!omap_pm_soc_init)
+		return 0;
+
 	/* Init the voltage layer */
 	omap3_twl_init();
 	omap4_twl_init();
@@ -252,5 +256,12 @@ int __init omap2_common_pm_late_init(void)
 	/* Smartreflex device init */
 	omap_devinit_smartreflex();
 
+	error = omap_pm_soc_init();
+	if (error)
+		pr_warn("%s: pm soc init failed: %i\n", __func__, error);
+
+	omap2_clk_enable_autoidle_all();
+
 	return 0;
 }
+omap_late_initcall(omap2_common_pm_late_init);
diff --git a/arch/arm/mach-omap2/pm33xx-core.c b/arch/arm/mach-omap2/pm33xx-core.c
index 93c0b5ba9f09..9b3755a2e2ec 100644
--- a/arch/arm/mach-omap2/pm33xx-core.c
+++ b/arch/arm/mach-omap2/pm33xx-core.c
@@ -173,7 +173,7 @@ static struct am33xx_pm_platform_data *am33xx_pm_get_pdata(void)
 		return NULL;
 }
 
-void __init amx3_common_pm_init(void)
+int __init amx3_common_pm_init(void)
 {
 	struct am33xx_pm_platform_data *pdata;
 	struct platform_device_info devinfo;
@@ -186,4 +186,6 @@ void __init amx3_common_pm_init(void)
 	devinfo.size_data = sizeof(*pdata);
 	devinfo.id = -1;
 	platform_device_register_full(&devinfo);
+
+	return 0;
 }
diff --git a/arch/arm/mach-omap2/pm44xx.c b/arch/arm/mach-omap2/pm44xx.c
index b3870220612e..78e1ace7d17d 100644
--- a/arch/arm/mach-omap2/pm44xx.c
+++ b/arch/arm/mach-omap2/pm44xx.c
@@ -131,6 +131,19 @@ static int __init pwrdms_setup(struct powerdomain *pwrdm, void *unused)
 		return 0;
 	}
 
+	/*
+	 * Bootloader or kexec boot may have LOGICRETSTATE cleared
+	 * for some domains. This is the case when kexec booting from
+	 * Android kernels that support off mode for example.
+	 * Make sure it's set at least for core and per, otherwise
+	 * we currently will see lost GPIO interrupts for wlcore and
+	 * smsc911x at least if per hits retention during idle.
+	 */
+	if (!strncmp(pwrdm->name, "core", 4) ||
+	    !strncmp(pwrdm->name, "l4per", 5) ||
+	    !strncmp(pwrdm->name, "wkup", 4))
+		pwrdm_set_logic_retst(pwrdm, PWRDM_POWER_RET);
+
 	pwrst = kmalloc(sizeof(struct power_state), GFP_ATOMIC);
 	if (!pwrst)
 		return -ENOMEM;
diff --git a/arch/arm/mach-omap2/powerdomain.c b/arch/arm/mach-omap2/powerdomain.c
index 1e6a967cd2d5..1a0f69c0a376 100644
--- a/arch/arm/mach-omap2/powerdomain.c
+++ b/arch/arm/mach-omap2/powerdomain.c
@@ -14,6 +14,7 @@
  */
 #undef DEBUG
 
+#include <linux/cpu_pm.h>
 #include <linux/kernel.h>
 #include <linux/types.h>
 #include <linux/list.h>
@@ -39,6 +40,9 @@
 
 #define PWRDM_TRACE_STATES_FLAG	(1<<31)
 
+void pwrdms_save_context(void);
+void pwrdms_restore_context(void);
+
 enum {
 	PWRDM_STATE_NOW = 0,
 	PWRDM_STATE_PREV,
@@ -333,6 +337,22 @@ int pwrdm_register_pwrdms(struct powerdomain **ps)
 	return 0;
 }
 
+static int cpu_notifier(struct notifier_block *nb, unsigned long cmd, void *v)
+{
+	switch (cmd) {
+	case CPU_CLUSTER_PM_ENTER:
+		if (enable_off_mode)
+			pwrdms_save_context();
+		break;
+	case CPU_CLUSTER_PM_EXIT:
+		if (enable_off_mode)
+			pwrdms_restore_context();
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
 /**
  * pwrdm_complete_init - set up the powerdomain layer
  *
@@ -347,6 +367,7 @@ int pwrdm_register_pwrdms(struct powerdomain **ps)
 int pwrdm_complete_init(void)
 {
 	struct powerdomain *temp_p;
+	static struct notifier_block nb;
 
 	if (list_empty(&pwrdm_list))
 		return -EACCES;
@@ -354,6 +375,12 @@ int pwrdm_complete_init(void)
 	list_for_each_entry(temp_p, &pwrdm_list, node)
 		pwrdm_set_next_pwrst(temp_p, PWRDM_POWER_ON);
 
+	/* Only AM43XX can lose pwrdm context during rtc-ddr suspend */
+	if (soc_is_am43xx()) {
+		nb.notifier_call = cpu_notifier;
+		cpu_pm_register_notifier(&nb);
+	}
+
 	return 0;
 }
 
@@ -1199,3 +1226,63 @@ bool pwrdm_can_ever_lose_context(struct powerdomain *pwrdm)
 
 	return 0;
 }
+
+/**
+ * pwrdm_save_context - save powerdomain registers
+ *
+ * Register state is going to be lost due to a suspend or hibernate
+ * event. Save the powerdomain registers.
+ */
+static int pwrdm_save_context(struct powerdomain *pwrdm, void *unused)
+{
+	if (arch_pwrdm && arch_pwrdm->pwrdm_save_context)
+		arch_pwrdm->pwrdm_save_context(pwrdm);
+	return 0;
+}
+
+/**
+ * pwrdm_save_context - restore powerdomain registers
+ *
+ * Restore powerdomain control registers after a suspend or resume
+ * event.
+ */
+static int pwrdm_restore_context(struct powerdomain *pwrdm, void *unused)
+{
+	if (arch_pwrdm && arch_pwrdm->pwrdm_restore_context)
+		arch_pwrdm->pwrdm_restore_context(pwrdm);
+	return 0;
+}
+
+static int pwrdm_lost_power(struct powerdomain *pwrdm, void *unused)
+{
+	int state;
+
+	/*
+	 * Power has been lost across all powerdomains, increment the
+	 * counter.
+	 */
+
+	state = pwrdm_read_pwrst(pwrdm);
+	if (state != PWRDM_POWER_OFF) {
+		pwrdm->state_counter[state]++;
+		pwrdm->state_counter[PWRDM_POWER_OFF]++;
+	}
+	pwrdm->state = state;
+
+	return 0;
+}
+
+void pwrdms_save_context(void)
+{
+	pwrdm_for_each(pwrdm_save_context, NULL);
+}
+
+void pwrdms_restore_context(void)
+{
+	pwrdm_for_each(pwrdm_restore_context, NULL);
+}
+
+void pwrdms_lost_power(void)
+{
+	pwrdm_for_each(pwrdm_lost_power, NULL);
+}
diff --git a/arch/arm/mach-omap2/powerdomain.h b/arch/arm/mach-omap2/powerdomain.h
index 28a796ce07d7..9a907fb14044 100644
--- a/arch/arm/mach-omap2/powerdomain.h
+++ b/arch/arm/mach-omap2/powerdomain.h
@@ -144,6 +144,7 @@ struct powerdomain {
 	s64 timer;
 	s64 state_timer[PWRDM_MAX_PWRSTS];
 #endif
+	u32 context;
 };
 
 /**
@@ -198,6 +199,8 @@ struct pwrdm_ops {
 	int	(*pwrdm_set_lowpwrstchange)(struct powerdomain *pwrdm);
 	int	(*pwrdm_wait_transition)(struct powerdomain *pwrdm);
 	int	(*pwrdm_has_voltdm)(void);
+	void	(*pwrdm_save_context)(struct powerdomain *pwrdm);
+	void	(*pwrdm_restore_context)(struct powerdomain *pwrdm);
 };
 
 int pwrdm_register_platform_funcs(struct pwrdm_ops *custom_funcs);
@@ -273,4 +276,8 @@ extern struct powerdomain gfx_omap2_pwrdm;
 extern void pwrdm_lock(struct powerdomain *pwrdm);
 extern void pwrdm_unlock(struct powerdomain *pwrdm);
 
+extern void pwrdms_save_context(void);
+extern void pwrdms_restore_context(void);
+
+extern void pwrdms_lost_power(void);
 #endif
diff --git a/arch/arm/mach-omap2/prm33xx.c b/arch/arm/mach-omap2/prm33xx.c
index ebaf80d72a10..d5141669c28d 100644
--- a/arch/arm/mach-omap2/prm33xx.c
+++ b/arch/arm/mach-omap2/prm33xx.c
@@ -342,6 +342,35 @@ static void am33xx_prm_global_warm_sw_reset(void)
 				  AM33XX_PRM_RSTCTRL_OFFSET);
 }
 
+static void am33xx_pwrdm_save_context(struct powerdomain *pwrdm)
+{
+	pwrdm->context = am33xx_prm_read_reg(pwrdm->prcm_offs,
+						pwrdm->pwrstctrl_offs);
+	/*
+	 * Do not save LOWPOWERSTATECHANGE, writing a 1 indicates a request,
+	 * reading back a 1 indicates a request in progress.
+	 */
+	pwrdm->context &= ~AM33XX_LOWPOWERSTATECHANGE_MASK;
+}
+
+static void am33xx_pwrdm_restore_context(struct powerdomain *pwrdm)
+{
+	int st, ctrl;
+
+	st = am33xx_prm_read_reg(pwrdm->prcm_offs,
+				 pwrdm->pwrstst_offs);
+
+	am33xx_prm_write_reg(pwrdm->context, pwrdm->prcm_offs,
+			     pwrdm->pwrstctrl_offs);
+
+	/* Make sure we only wait for a transition if there is one */
+	st &= OMAP_POWERSTATEST_MASK;
+	ctrl = OMAP_POWERSTATEST_MASK & pwrdm->context;
+
+	if (st != ctrl)
+		am33xx_pwrdm_wait_transition(pwrdm);
+}
+
 struct pwrdm_ops am33xx_pwrdm_operations = {
 	.pwrdm_set_next_pwrst		= am33xx_pwrdm_set_next_pwrst,
 	.pwrdm_read_next_pwrst		= am33xx_pwrdm_read_next_pwrst,
@@ -357,6 +386,8 @@ struct pwrdm_ops am33xx_pwrdm_operations = {
 	.pwrdm_set_mem_retst		= am33xx_pwrdm_set_mem_retst,
 	.pwrdm_wait_transition		= am33xx_pwrdm_wait_transition,
 	.pwrdm_has_voltdm		= am33xx_check_vcvp,
+	.pwrdm_save_context		= am33xx_pwrdm_save_context,
+	.pwrdm_restore_context		= am33xx_pwrdm_restore_context,
 };
 
 static struct prm_ll_data am33xx_prm_ll_data = {
diff --git a/arch/arm/mach-omap2/prm44xx.c b/arch/arm/mach-omap2/prm44xx.c
index acb95936dfe7..7b95729e8359 100644
--- a/arch/arm/mach-omap2/prm44xx.c
+++ b/arch/arm/mach-omap2/prm44xx.c
@@ -12,6 +12,7 @@
  * published by the Free Software Foundation.
  */
 
+#include <linux/cpu_pm.h>
 #include <linux/kernel.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
@@ -30,6 +31,7 @@
 #include "prcm44xx.h"
 #include "prminst44xx.h"
 #include "powerdomain.h"
+#include "pm.h"
 
 /* Static data */
 
@@ -57,6 +59,13 @@ static struct omap_prcm_irq_setup omap4_prcm_irq_setup = {
 	.reconfigure_io_chain	= &omap44xx_prm_reconfigure_io_chain,
 };
 
+struct omap_prm_irq_context {
+	unsigned long irq_enable;
+	unsigned long pm_ctrl;
+};
+
+static struct omap_prm_irq_context omap_prm_context;
+
 /*
  * omap44xx_prm_reset_src_map - map from bits in the PRM_RSTST
  *   hardware register (which are specific to OMAP44xx SoCs) to reset
@@ -667,6 +676,54 @@ static int omap4_check_vcvp(void)
 	return 0;
 }
 
+/**
+ * omap4_pwrdm_save_context - Saves the powerdomain state
+ * @pwrdm: pointer to individual powerdomain
+ *
+ * The function saves the powerdomain state control information.
+ * This is needed in rtc+ddr modes where we lose powerdomain context.
+ */
+static void omap4_pwrdm_save_context(struct powerdomain *pwrdm)
+{
+	pwrdm->context = omap4_prminst_read_inst_reg(pwrdm->prcm_partition,
+						     pwrdm->prcm_offs,
+						     pwrdm->pwrstctrl_offs);
+
+	/*
+	 * Do not save LOWPOWERSTATECHANGE, writing a 1 indicates a request,
+	 * reading back a 1 indicates a request in progress.
+	 */
+	pwrdm->context &= ~OMAP4430_LOWPOWERSTATECHANGE_MASK;
+}
+
+/**
+ * omap4_pwrdm_restore_context - Restores the powerdomain state
+ * @pwrdm: pointer to individual powerdomain
+ *
+ * The function restores the powerdomain state control information.
+ * This is needed in rtc+ddr modes where we lose powerdomain context.
+ */
+static void omap4_pwrdm_restore_context(struct powerdomain *pwrdm)
+{
+	int st, ctrl;
+
+	st = omap4_prminst_read_inst_reg(pwrdm->prcm_partition,
+					 pwrdm->prcm_offs,
+					 pwrdm->pwrstctrl_offs);
+
+	omap4_prminst_write_inst_reg(pwrdm->context,
+				     pwrdm->prcm_partition,
+				     pwrdm->prcm_offs,
+				     pwrdm->pwrstctrl_offs);
+
+	/* Make sure we only wait for a transition if there is one */
+	st &= OMAP_POWERSTATEST_MASK;
+	ctrl = OMAP_POWERSTATEST_MASK & pwrdm->context;
+
+	if (st != ctrl)
+		omap4_pwrdm_wait_transition(pwrdm);
+}
+
 struct pwrdm_ops omap4_pwrdm_operations = {
 	.pwrdm_set_next_pwrst	= omap4_pwrdm_set_next_pwrst,
 	.pwrdm_read_next_pwrst	= omap4_pwrdm_read_next_pwrst,
@@ -685,10 +742,50 @@ struct pwrdm_ops omap4_pwrdm_operations = {
 	.pwrdm_set_mem_retst	= omap4_pwrdm_set_mem_retst,
 	.pwrdm_wait_transition	= omap4_pwrdm_wait_transition,
 	.pwrdm_has_voltdm	= omap4_check_vcvp,
+	.pwrdm_save_context	= omap4_pwrdm_save_context,
+	.pwrdm_restore_context	= omap4_pwrdm_restore_context,
 };
 
 static int omap44xx_prm_late_init(void);
 
+void prm_save_context(void)
+{
+	omap_prm_context.irq_enable =
+			omap4_prm_read_inst_reg(AM43XX_PRM_OCP_SOCKET_INST,
+						omap4_prcm_irq_setup.mask);
+
+	omap_prm_context.pm_ctrl =
+			omap4_prm_read_inst_reg(AM43XX_PRM_DEVICE_INST,
+						omap4_prcm_irq_setup.pm_ctrl);
+}
+
+void prm_restore_context(void)
+{
+	omap4_prm_write_inst_reg(omap_prm_context.irq_enable,
+				 OMAP4430_PRM_OCP_SOCKET_INST,
+				 omap4_prcm_irq_setup.mask);
+
+	omap4_prm_write_inst_reg(omap_prm_context.pm_ctrl,
+				 AM43XX_PRM_DEVICE_INST,
+				 omap4_prcm_irq_setup.pm_ctrl);
+}
+
+static int cpu_notifier(struct notifier_block *nb, unsigned long cmd, void *v)
+{
+	switch (cmd) {
+	case CPU_CLUSTER_PM_ENTER:
+		if (enable_off_mode)
+			prm_save_context();
+		break;
+	case CPU_CLUSTER_PM_EXIT:
+		if (enable_off_mode)
+			prm_restore_context();
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
 /*
  * XXX document
  */
@@ -709,6 +806,7 @@ static const struct omap_prcm_init_data *prm_init_data;
 
 int __init omap44xx_prm_init(const struct omap_prcm_init_data *data)
 {
+	static struct notifier_block nb;
 	omap_prm_base_init();
 
 	prm_init_data = data;
@@ -730,6 +828,12 @@ int __init omap44xx_prm_init(const struct omap_prcm_init_data *data)
 		omap4_prcm_irq_setup.mask = AM43XX_PRM_IRQENABLE_MPU_OFFSET;
 	}
 
+	/* Only AM43XX can lose prm context during rtc-ddr suspend */
+	if (soc_is_am43xx()) {
+		nb.notifier_call = cpu_notifier;
+		cpu_pm_register_notifier(&nb);
+	}
+
 	return prm_register(&omap44xx_prm_ll_data);
 }
 
diff --git a/arch/arm/mach-omap2/timer.c b/arch/arm/mach-omap2/timer.c
index 4fb4dc24e5e9..98ed5ac073bc 100644
--- a/arch/arm/mach-omap2/timer.c
+++ b/arch/arm/mach-omap2/timer.c
@@ -50,7 +50,6 @@
 #include "omap_device.h"
 #include <plat/counter-32k.h>
 #include <clocksource/timer-ti-dm.h>
-#include "omap-pm.h"
 
 #include "soc.h"
 #include "common.h"
@@ -71,6 +70,9 @@ static struct clock_event_device clockevent_gpt;
 /* Clockevent hwmod for am335x and am437x suspend */
 static struct omap_hwmod *clockevent_gpt_hwmod;
 
+/* Clockesource hwmod for am437x suspend */
+static struct omap_hwmod *clocksource_gpt_hwmod;
+
 #ifdef CONFIG_SOC_HAS_REALTIME_COUNTER
 static unsigned long arch_timer_freq;
 
@@ -168,6 +170,43 @@ static const struct of_device_id omap_timer_match[] __initconst = {
 	{ }
 };
 
+static int omap_timer_add_disabled_property(struct device_node *np)
+{
+	struct property *prop;
+
+	prop = kzalloc(sizeof(*prop), GFP_KERNEL);
+	if (!prop)
+		return -ENOMEM;
+
+	prop->name = "status";
+	prop->value = "disabled";
+	prop->length = strlen(prop->value);
+
+	return of_add_property(np, prop);
+}
+
+static int omap_timer_update_dt(struct device_node *np)
+{
+	int error = 0;
+
+	if (!of_device_is_compatible(np, "ti,omap-counter32k")) {
+		error = omap_timer_add_disabled_property(np);
+		if (error)
+			return error;
+	}
+
+	/* No parent interconnect target module configured? */
+	if (of_get_property(np, "ti,hwmods", NULL))
+		return error;
+
+	/* Tag parent interconnect target module disabled */
+	error = omap_timer_add_disabled_property(np->parent);
+	if (error)
+		return error;
+
+	return 0;
+}
+
 /**
  * omap_get_timer_dt - get a timer using device-tree
  * @match	- device-tree match structure for matching a device type
@@ -183,6 +222,7 @@ static struct device_node * __init omap_get_timer_dt(const struct of_device_id *
 						     const char *property)
 {
 	struct device_node *np;
+	int error;
 
 	for_each_matching_node(np, match) {
 		if (!of_device_is_available(np))
@@ -197,17 +237,9 @@ static struct device_node * __init omap_get_timer_dt(const struct of_device_id *
 				  of_get_property(np, "ti,timer-secure", NULL)))
 			continue;
 
-		if (!of_device_is_compatible(np, "ti,omap-counter32k")) {
-			struct property *prop;
+		error = omap_timer_update_dt(np);
+		WARN(error, "%s: Could not update dt: %i\n", __func__, error);
 
-			prop = kzalloc(sizeof(*prop), GFP_KERNEL);
-			if (!prop)
-				return NULL;
-			prop->name = "status";
-			prop->value = "disabled";
-			prop->length = strlen(prop->value);
-			of_add_property(np, prop);
-		}
 		return np;
 	}
 
@@ -266,8 +298,12 @@ static int __init omap_dm_timer_init_one(struct omap_dm_timer *timer,
 		return -ENODEV;
 
 	of_property_read_string_index(np, "ti,hwmods", 0, &oh_name);
-	if (!oh_name)
-		return -ENODEV;
+	if (!oh_name) {
+		of_property_read_string_index(np->parent, "ti,hwmods", 0,
+					      &oh_name);
+		if (!oh_name)
+			return -ENODEV;
+	}
 
 	timer->irq = irq_of_parse_and_map(np, 0);
 	if (!timer->irq)
@@ -419,9 +455,12 @@ static int __init __maybe_unused omap2_sync32k_clocksource_init(void)
 	if (!np)
 		return -ENODEV;
 
-	of_property_read_string_index(np, "ti,hwmods", 0, &oh_name);
-	if (!oh_name)
-		return -ENODEV;
+	of_property_read_string_index(np->parent, "ti,hwmods", 0, &oh_name);
+	if (!oh_name) {
+		of_property_read_string_index(np, "ti,hwmods", 0, &oh_name);
+		if (!oh_name)
+			return -ENODEV;
+	}
 
 	/*
 	 * First check hwmod data is available for sync32k counter
@@ -442,6 +481,26 @@ static int __init __maybe_unused omap2_sync32k_clocksource_init(void)
 	return ret;
 }
 
+static unsigned int omap2_gptimer_clksrc_load;
+
+static void omap2_gptimer_clksrc_suspend(struct clocksource *unused)
+{
+	omap2_gptimer_clksrc_load =
+		__omap_dm_timer_read_counter(&clksrc, OMAP_TIMER_NONPOSTED);
+
+	omap_hwmod_idle(clocksource_gpt_hwmod);
+}
+
+static void omap2_gptimer_clksrc_resume(struct clocksource *unused)
+{
+	omap_hwmod_enable(clocksource_gpt_hwmod);
+
+	__omap_dm_timer_load_start(&clksrc,
+				   OMAP_TIMER_CTRL_ST | OMAP_TIMER_CTRL_AR,
+				   omap2_gptimer_clksrc_load,
+				   OMAP_TIMER_NONPOSTED);
+}
+
 static void __init omap2_gptimer_clocksource_init(int gptimer_id,
 						  const char *fck_source,
 						  const char *property)
@@ -454,6 +513,15 @@ static void __init omap2_gptimer_clocksource_init(int gptimer_id,
 	res = omap_dm_timer_init_one(&clksrc, fck_source, property,
 				     &clocksource_gpt.name,
 				     OMAP_TIMER_NONPOSTED);
+
+	if (soc_is_am43xx()) {
+		clocksource_gpt.suspend = omap2_gptimer_clksrc_suspend;
+		clocksource_gpt.resume = omap2_gptimer_clksrc_resume;
+
+		clocksource_gpt_hwmod =
+			omap_hwmod_lookup(clocksource_gpt.name);
+	}
+
 	BUG_ON(res);
 
 	__omap_dm_timer_load_start(&clksrc,
diff --git a/arch/arm/mach-pxa/em-x270.c b/arch/arm/mach-pxa/em-x270.c
index 49022ad338e9..6d7d93981098 100644
--- a/arch/arm/mach-pxa/em-x270.c
+++ b/arch/arm/mach-pxa/em-x270.c
@@ -987,7 +987,6 @@ static struct fixed_voltage_config camera_dummy_config = {
 	.supply_name		= "camera_vdd",
 	.input_supply		= "vcc cam",
 	.microvolts		= 2800000,
-	.gpio			= -1,
 	.enable_high		= 0,
 	.init_data		= &camera_dummy_initdata,
 };
diff --git a/arch/arm/mach-pxa/ezx.c b/arch/arm/mach-pxa/ezx.c
index 2c90b58f347d..2b4bd6d94855 100644
--- a/arch/arm/mach-pxa/ezx.c
+++ b/arch/arm/mach-pxa/ezx.c
@@ -21,6 +21,7 @@
 #include <linux/regulator/fixed.h>
 #include <linux/input.h>
 #include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/gpio_keys.h>
 #include <linux/leds-lp3944.h>
 #include <linux/platform_data/i2c-pxa.h>
@@ -698,31 +699,39 @@ static struct pxa27x_keypad_platform_data e2_keypad_platform_data = {
 
 #if defined(CONFIG_MACH_EZX_A780) || defined(CONFIG_MACH_EZX_A910)
 /* camera */
-static struct regulator_consumer_supply camera_dummy_supplies[] = {
+static struct regulator_consumer_supply camera_regulator_supplies[] = {
 	REGULATOR_SUPPLY("vdd", "0-005d"),
 };
 
-static struct regulator_init_data camera_dummy_initdata = {
-	.consumer_supplies = camera_dummy_supplies,
-	.num_consumer_supplies = ARRAY_SIZE(camera_dummy_supplies),
+static struct regulator_init_data camera_regulator_initdata = {
+	.consumer_supplies = camera_regulator_supplies,
+	.num_consumer_supplies = ARRAY_SIZE(camera_regulator_supplies),
 	.constraints = {
 		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
 	},
 };
 
-static struct fixed_voltage_config camera_dummy_config = {
+static struct fixed_voltage_config camera_regulator_config = {
 	.supply_name		= "camera_vdd",
 	.microvolts		= 2800000,
-	.gpio			= GPIO50_nCAM_EN,
 	.enable_high		= 0,
-	.init_data		= &camera_dummy_initdata,
+	.init_data		= &camera_regulator_initdata,
 };
 
-static struct platform_device camera_supply_dummy_device = {
+static struct platform_device camera_supply_regulator_device = {
 	.name	= "reg-fixed-voltage",
 	.id	= 1,
 	.dev	= {
-		.platform_data = &camera_dummy_config,
+		.platform_data = &camera_regulator_config,
+	},
+};
+
+static struct gpiod_lookup_table camera_supply_gpiod_table = {
+	.dev_id = "reg-fixed-voltage.1",
+	.table = {
+		GPIO_LOOKUP("gpio-pxa", GPIO50_nCAM_EN,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
 	},
 };
 #endif
@@ -800,7 +809,7 @@ static struct i2c_board_info a780_i2c_board_info[] = {
 
 static struct platform_device *a780_devices[] __initdata = {
 	&a780_gpio_keys,
-	&camera_supply_dummy_device,
+	&camera_supply_regulator_device,
 };
 
 static void __init a780_init(void)
@@ -823,6 +832,7 @@ static void __init a780_init(void)
 	if (a780_camera_init() == 0)
 		pxa_set_camera_info(&a780_pxacamera_platform_data);
 
+	gpiod_add_lookup_table(&camera_supply_gpiod_table);
 	pwm_add_table(ezx_pwm_lookup, ARRAY_SIZE(ezx_pwm_lookup));
 	platform_add_devices(ARRAY_AND_SIZE(ezx_devices));
 	platform_add_devices(ARRAY_AND_SIZE(a780_devices));
@@ -892,6 +902,7 @@ static void __init e680_init(void)
 
 	pxa_set_keypad_info(&e680_keypad_platform_data);
 
+	gpiod_add_lookup_table(&camera_supply_gpiod_table);
 	pwm_add_table(ezx_pwm_lookup, ARRAY_SIZE(ezx_pwm_lookup));
 	platform_add_devices(ARRAY_AND_SIZE(ezx_devices));
 	platform_add_devices(ARRAY_AND_SIZE(e680_devices));
@@ -1098,7 +1109,7 @@ static struct i2c_board_info __initdata a910_i2c_board_info[] = {
 
 static struct platform_device *a910_devices[] __initdata = {
 	&a910_gpio_keys,
-	&camera_supply_dummy_device,
+	&camera_supply_regulator_device,
 };
 
 static void __init a910_init(void)
diff --git a/arch/arm/mach-pxa/hx4700.c b/arch/arm/mach-pxa/hx4700.c
index e2e7f247a645..6717a10180eb 100644
--- a/arch/arm/mach-pxa/hx4700.c
+++ b/arch/arm/mach-pxa/hx4700.c
@@ -20,6 +20,7 @@
 #include <linux/delay.h>
 #include <linux/fb.h>
 #include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/gpio_keys.h>
 #include <linux/input.h>
 #include <linux/input/navpoint.h>
@@ -711,7 +712,6 @@ static struct gpio_regulator_state bq24022_states[] = {
 static struct gpio_regulator_config bq24022_info = {
 	.supply_name = "bq24022",
 
-	.enable_gpio = GPIO72_HX4700_BQ24022_nCHARGE_EN,
 	.enable_high = 0,
 	.enabled_at_boot = 0,
 
@@ -733,6 +733,15 @@ static struct platform_device bq24022 = {
 	},
 };
 
+static struct gpiod_lookup_table bq24022_gpiod_table = {
+	.dev_id = "gpio-regulator",
+	.table = {
+		GPIO_LOOKUP("gpio-pxa", GPIO72_HX4700_BQ24022_nCHARGE_EN,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 /*
  * StrataFlash
  */
@@ -875,6 +884,7 @@ static void __init hx4700_init(void)
 	pxa_set_btuart_info(NULL);
 	pxa_set_stuart_info(NULL);
 
+	gpiod_add_lookup_table(&bq24022_gpiod_table);
 	platform_add_devices(devices, ARRAY_SIZE(devices));
 	pwm_add_table(hx4700_pwm_lookup, ARRAY_SIZE(hx4700_pwm_lookup));
 
diff --git a/arch/arm/mach-pxa/magician.c b/arch/arm/mach-pxa/magician.c
index c5325d1ae77b..9a5bda3ea194 100644
--- a/arch/arm/mach-pxa/magician.c
+++ b/arch/arm/mach-pxa/magician.c
@@ -18,6 +18,7 @@
 #include <linux/platform_device.h>
 #include <linux/delay.h>
 #include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/gpio_keys.h>
 #include <linux/input.h>
 #include <linux/mfd/htc-pasic3.h>
@@ -656,7 +657,6 @@ static struct gpio_regulator_state bq24022_states[] = {
 static struct gpio_regulator_config bq24022_info = {
 	.supply_name		= "bq24022",
 
-	.enable_gpio		= GPIO30_MAGICIAN_BQ24022_nCHARGE_EN,
 	.enable_high		= 0,
 	.enabled_at_boot	= 1,
 
@@ -678,6 +678,15 @@ static struct platform_device bq24022 = {
 	},
 };
 
+static struct gpiod_lookup_table bq24022_gpiod_table = {
+	.dev_id = "gpio-regulator",
+	.table = {
+		GPIO_LOOKUP("gpio-pxa", GPIO30_MAGICIAN_BQ24022_nCHARGE_EN,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 /*
  * fixed regulator for ads7846
  */
@@ -696,7 +705,6 @@ static struct regulator_init_data vads7846_regulator = {
 static struct fixed_voltage_config vads7846 = {
 	.supply_name	= "vads7846",
 	.microvolts	= 3300000, /* probably */
-	.gpio		= -EINVAL,
 	.startup_delay	= 0,
 	.init_data	= &vads7846_regulator,
 };
@@ -1007,6 +1015,7 @@ static void __init magician_init(void)
 	regulator_register_always_on(0, "power", pwm_backlight_supply,
 		ARRAY_SIZE(pwm_backlight_supply), 5000000);
 
+	gpiod_add_lookup_table(&bq24022_gpiod_table);
 	platform_add_devices(ARRAY_AND_SIZE(devices));
 }
 
diff --git a/arch/arm/mach-pxa/palmz72.c b/arch/arm/mach-pxa/palmz72.c
index 0adb1bd6208e..4d475f6f4a77 100644
--- a/arch/arm/mach-pxa/palmz72.c
+++ b/arch/arm/mach-pxa/palmz72.c
@@ -30,7 +30,7 @@
 #include <linux/wm97xx.h>
 #include <linux/power_supply.h>
 #include <linux/usb/gpio_vbus.h>
-#include <linux/i2c-gpio.h>
+#include <linux/platform_data/i2c-gpio.h>
 #include <linux/gpio/machine.h>
 
 #include <asm/mach-types.h>
diff --git a/arch/arm/mach-pxa/raumfeld.c b/arch/arm/mach-pxa/raumfeld.c
index 034345546f84..ee766e4ebddc 100644
--- a/arch/arm/mach-pxa/raumfeld.c
+++ b/arch/arm/mach-pxa/raumfeld.c
@@ -886,7 +886,6 @@ static struct regulator_init_data audio_va_initdata = {
 static struct fixed_voltage_config audio_va_config = {
 	.supply_name		= "audio_va",
 	.microvolts		= 5000000,
-	.gpio			= GPIO_AUDIO_VA_ENABLE,
 	.enable_high		= 1,
 	.enabled_at_boot	= 0,
 	.init_data		= &audio_va_initdata,
@@ -900,6 +899,15 @@ static struct platform_device audio_va_device = {
 	},
 };
 
+static struct gpiod_lookup_table audio_va_gpiod_table = {
+	.dev_id = "reg-fixed-voltage.0",
+	.table = {
+		GPIO_LOOKUP("gpio-pxa", GPIO_AUDIO_VA_ENABLE,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 /* Dummy supplies for Codec's VD/VLC */
 
 static struct regulator_consumer_supply audio_dummy_supplies[] = {
@@ -918,7 +926,6 @@ static struct regulator_init_data audio_dummy_initdata = {
 static struct fixed_voltage_config audio_dummy_config = {
 	.supply_name		= "audio_vd",
 	.microvolts		= 3300000,
-	.gpio			= -1,
 	.init_data		= &audio_dummy_initdata,
 };
 
@@ -1033,6 +1040,7 @@ static void __init raumfeld_audio_init(void)
 	else
 		gpio_direction_output(GPIO_MCLK_RESET, 1);
 
+	gpiod_add_lookup_table(&audio_va_gpiod_table);
 	platform_add_devices(ARRAY_AND_SIZE(audio_regulator_devices));
 }
 
diff --git a/arch/arm/mach-pxa/viper.c b/arch/arm/mach-pxa/viper.c
index 207dcc2e94e7..ab2f89266bbd 100644
--- a/arch/arm/mach-pxa/viper.c
+++ b/arch/arm/mach-pxa/viper.c
@@ -35,7 +35,7 @@
 #include <linux/sched.h>
 #include <linux/gpio.h>
 #include <linux/jiffies.h>
-#include <linux/i2c-gpio.h>
+#include <linux/platform_data/i2c-gpio.h>
 #include <linux/gpio/machine.h>
 #include <linux/platform_data/i2c-pxa.h>
 #include <linux/serial_8250.h>
diff --git a/arch/arm/mach-pxa/zeus.c b/arch/arm/mach-pxa/zeus.c
index e3851795d6d7..58e05afcece0 100644
--- a/arch/arm/mach-pxa/zeus.c
+++ b/arch/arm/mach-pxa/zeus.c
@@ -17,6 +17,7 @@
 #include <linux/irq.h>
 #include <linux/pm.h>
 #include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/serial_8250.h>
 #include <linux/dm9000.h>
 #include <linux/mmc/host.h>
@@ -410,7 +411,6 @@ static struct regulator_init_data can_regulator_init_data = {
 static struct fixed_voltage_config can_regulator_pdata = {
 	.supply_name	= "CAN_SHDN",
 	.microvolts	= 3300000,
-	.gpio		= ZEUS_CAN_SHDN_GPIO,
 	.init_data	= &can_regulator_init_data,
 };
 
@@ -422,6 +422,15 @@ static struct platform_device can_regulator_device = {
 	},
 };
 
+static struct gpiod_lookup_table can_regulator_gpiod_table = {
+	.dev_id = "reg-fixed-voltage.0",
+	.table = {
+		GPIO_LOOKUP("gpio-pxa", ZEUS_CAN_SHDN_GPIO,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 static struct mcp251x_platform_data zeus_mcp2515_pdata = {
 	.oscillator_frequency	= 16*1000*1000,
 };
@@ -538,7 +547,6 @@ static struct regulator_init_data zeus_ohci_regulator_data = {
 static struct fixed_voltage_config zeus_ohci_regulator_config = {
 	.supply_name		= "vbus2",
 	.microvolts		= 5000000, /* 5.0V */
-	.gpio			= ZEUS_USB2_PWREN_GPIO,
 	.enable_high		= 1,
 	.startup_delay		= 0,
 	.init_data		= &zeus_ohci_regulator_data,
@@ -552,6 +560,15 @@ static struct platform_device zeus_ohci_regulator_device = {
 	},
 };
 
+static struct gpiod_lookup_table zeus_ohci_regulator_gpiod_table = {
+	.dev_id = "reg-fixed-voltage.0",
+	.table = {
+		GPIO_LOOKUP("gpio-pxa", ZEUS_USB2_PWREN_GPIO,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 static struct pxaohci_platform_data zeus_ohci_platform_data = {
 	.port_mode	= PMM_NPS_MODE,
 	/* Clear Power Control Polarity Low and set Power Sense
@@ -855,6 +872,8 @@ static void __init zeus_init(void)
 
 	pxa2xx_mfp_config(ARRAY_AND_SIZE(zeus_pin_config));
 
+	gpiod_add_lookup_table(&can_regulator_gpiod_table);
+	gpiod_add_lookup_table(&zeus_ohci_regulator_gpiod_table);
 	platform_add_devices(zeus_devices, ARRAY_SIZE(zeus_devices));
 
 	zeus_register_ohci();
diff --git a/arch/arm/mach-rockchip/Kconfig b/arch/arm/mach-rockchip/Kconfig
index a4065966881a..fafd3d7f9f8c 100644
--- a/arch/arm/mach-rockchip/Kconfig
+++ b/arch/arm/mach-rockchip/Kconfig
@@ -3,7 +3,6 @@ config ARCH_ROCKCHIP
 	depends on ARCH_MULTI_V7
 	select PINCTRL
 	select PINCTRL_ROCKCHIP
-	select ARCH_DMA_ADDR_T_64BIT if ARM_LPAE
 	select ARCH_HAS_RESET_CONTROLLER
 	select ARM_AMBA
 	select ARM_GIC
diff --git a/arch/arm/mach-rpc/ecard.c b/arch/arm/mach-rpc/ecard.c
index bdb5ec1cf560..39aef4876ed4 100644
--- a/arch/arm/mach-rpc/ecard.c
+++ b/arch/arm/mach-rpc/ecard.c
@@ -657,25 +657,13 @@ static int ecard_devices_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int ecard_devices_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ecard_devices_proc_show, NULL);
-}
-
-static const struct file_operations bus_ecard_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= ecard_devices_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static struct proc_dir_entry *proc_bus_ecard_dir = NULL;
 
 static void ecard_proc_init(void)
 {
 	proc_bus_ecard_dir = proc_mkdir("bus/ecard", NULL);
-	proc_create("devices", 0, proc_bus_ecard_dir, &bus_ecard_proc_fops);
+	proc_create_single("devices", 0, proc_bus_ecard_dir,
+			ecard_devices_proc_show);
 }
 
 #define ec_set_resource(ec,nr,st,sz)				\
diff --git a/arch/arm/mach-s3c24xx/h1940-bluetooth.c b/arch/arm/mach-s3c24xx/h1940-bluetooth.c
index 46ad20ea87d1..186b5321658e 100644
--- a/arch/arm/mach-s3c24xx/h1940-bluetooth.c
+++ b/arch/arm/mach-s3c24xx/h1940-bluetooth.c
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-1.0
+// SPDX-License-Identifier: GPL-1.0+
 //
 // Copyright (c) Arnaud Patard <arnaud.patard@rtp-net.org>
 //
diff --git a/arch/arm/mach-s3c24xx/mach-mini2440.c b/arch/arm/mach-s3c24xx/mach-mini2440.c
index 95753e0bc073..f9fc1f8d2b28 100644
--- a/arch/arm/mach-s3c24xx/mach-mini2440.c
+++ b/arch/arm/mach-s3c24xx/mach-mini2440.c
@@ -20,7 +20,7 @@
 #include <linux/serial_core.h>
 #include <linux/serial_s3c.h>
 #include <linux/dm9000.h>
-#include <linux/platform_data/at24.h>
+#include <linux/property.h>
 #include <linux/platform_device.h>
 #include <linux/gpio_keys.h>
 #include <linux/i2c.h>
@@ -481,15 +481,15 @@ static struct platform_device mini2440_audio = {
 /*
  * I2C devices
  */
-static struct at24_platform_data at24c08 = {
-	.byte_len	= SZ_8K / 8,
-	.page_size	= 16,
+static const struct property_entry mini2440_at24_properties[] = {
+	PROPERTY_ENTRY_U32("pagesize", 16),
+	{ }
 };
 
 static struct i2c_board_info mini2440_i2c_devs[] __initdata = {
 	{
 		I2C_BOARD_INFO("24c08", 0x50),
-		.platform_data = &at24c08,
+		.properties = mini2440_at24_properties,
 	},
 };
 
diff --git a/arch/arm/mach-s3c64xx/mach-crag6410-module.c b/arch/arm/mach-s3c64xx/mach-crag6410-module.c
index f00988705408..5aa472892465 100644
--- a/arch/arm/mach-s3c64xx/mach-crag6410-module.c
+++ b/arch/arm/mach-s3c64xx/mach-crag6410-module.c
@@ -9,6 +9,7 @@
 #include <linux/interrupt.h>
 #include <linux/i2c.h>
 #include <linux/spi/spi.h>
+#include <linux/gpio/machine.h>
 
 #include <linux/mfd/wm831x/irq.h>
 #include <linux/mfd/wm831x/gpio.h>
@@ -206,9 +207,6 @@ static const struct i2c_board_info wm1277_devs[] = {
 };
 
 static struct arizona_pdata wm5102_reva_pdata = {
-	.ldo1 = {
-		.ldoena = S3C64XX_GPN(7),
-	},
 	.gpio_base = CODEC_GPIO_BASE,
 	.irq_flags = IRQF_TRIGGER_HIGH,
 	.micd_pol_gpio = CODEC_GPIO_BASE + 4,
@@ -237,10 +235,16 @@ static struct spi_board_info wm5102_reva_spi_devs[] = {
 	},
 };
 
-static struct arizona_pdata wm5102_pdata = {
-	.ldo1 = {
-		.ldoena = S3C64XX_GPN(7),
+static struct gpiod_lookup_table wm5102_reva_gpiod_table = {
+	.dev_id = "spi0.1", /* SPI device name */
+	.table = {
+		GPIO_LOOKUP("GPION", 7,
+			    "wlf,ldoena", GPIO_ACTIVE_HIGH),
+		{ },
 	},
+};
+
+static struct arizona_pdata wm5102_pdata = {
 	.gpio_base = CODEC_GPIO_BASE,
 	.irq_flags = IRQF_TRIGGER_HIGH,
 	.micd_pol_gpio = CODEC_GPIO_BASE + 2,
@@ -264,6 +268,15 @@ static struct spi_board_info wm5102_spi_devs[] = {
 	},
 };
 
+static struct gpiod_lookup_table wm5102_gpiod_table = {
+	.dev_id = "spi0.1", /* SPI device name */
+	.table = {
+		GPIO_LOOKUP("GPION", 7,
+			    "wlf,ldo1ena", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 static struct spi_board_info wm5110_spi_devs[] = {
 	[0] = {
 		.modalias	= "wm5110",
@@ -366,6 +379,9 @@ static int wlf_gf_module_probe(struct i2c_client *i2c,
 					    rev == gf_mods[i].rev))
 			break;
 
+	gpiod_add_lookup_table(&wm5102_reva_gpiod_table);
+	gpiod_add_lookup_table(&wm5102_gpiod_table);
+
 	if (i < ARRAY_SIZE(gf_mods)) {
 		dev_info(&i2c->dev, "%s revision %d\n",
 			 gf_mods[i].name, rev + 1);
diff --git a/arch/arm/mach-s3c64xx/mach-crag6410.c b/arch/arm/mach-s3c64xx/mach-crag6410.c
index f04650297487..379424d72ae7 100644
--- a/arch/arm/mach-s3c64xx/mach-crag6410.c
+++ b/arch/arm/mach-s3c64xx/mach-crag6410.c
@@ -352,7 +352,6 @@ static struct fixed_voltage_config wallvdd_pdata = {
 	.supply_name = "WALLVDD",
 	.microvolts = 5000000,
 	.init_data = &wallvdd_data,
-	.gpio = -EINVAL,
 };
 
 static struct platform_device wallvdd_device = {
diff --git a/arch/arm/mach-s3c64xx/mach-smdk6410.c b/arch/arm/mach-s3c64xx/mach-smdk6410.c
index c46fa5dfd2e0..908e5aa831c8 100644
--- a/arch/arm/mach-s3c64xx/mach-smdk6410.c
+++ b/arch/arm/mach-s3c64xx/mach-smdk6410.c
@@ -222,7 +222,6 @@ static struct fixed_voltage_config smdk6410_b_pwr_5v_pdata = {
 	.supply_name = "B_PWR_5V",
 	.microvolts = 5000000,
 	.init_data = &smdk6410_b_pwr_5v_data,
-	.gpio = -EINVAL,
 };
 
 static struct platform_device smdk6410_b_pwr_5v = {
diff --git a/arch/arm/mach-sa1100/assabet.c b/arch/arm/mach-sa1100/assabet.c
index 575ec085cffa..dbb53c520165 100644
--- a/arch/arm/mach-sa1100/assabet.c
+++ b/arch/arm/mach-sa1100/assabet.c
@@ -101,7 +101,7 @@ static int __init assabet_init_gpio(void __iomem *reg, u32 def_val)
 
 	assabet_bcr_gc = gc;
 
-	return gc->base;
+	return 0;
 }
 
 /*
@@ -471,6 +471,14 @@ static struct fixed_voltage_config assabet_cf_vcc_pdata __initdata = {
 	.enable_high = 1,
 };
 
+static struct gpiod_lookup_table assabet_cf_vcc_gpio_table = {
+	.dev_id = "reg-fixed-voltage.0",
+	.table = {
+		GPIO_LOOKUP("assabet", 0, "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 static void __init assabet_init(void)
 {
 	/*
@@ -517,9 +525,11 @@ static void __init assabet_init(void)
 			neponset_resources, ARRAY_SIZE(neponset_resources));
 #endif
 	} else {
+		gpiod_add_lookup_table(&assabet_cf_vcc_gpio_table);
 		sa11x0_register_fixed_regulator(0, &assabet_cf_vcc_pdata,
-					 assabet_cf_vcc_consumers,
-					 ARRAY_SIZE(assabet_cf_vcc_consumers));
+					assabet_cf_vcc_consumers,
+					ARRAY_SIZE(assabet_cf_vcc_consumers),
+					true);
 
 	}
 
@@ -802,7 +812,6 @@ fs_initcall(assabet_leds_init);
 
 void __init assabet_init_irq(void)
 {
-	unsigned int assabet_gpio_base;
 	u32 def_val;
 
 	sa1100_init_irq();
@@ -817,9 +826,7 @@ void __init assabet_init_irq(void)
 	 *
 	 * This must precede any driver calls to BCR_set() or BCR_clear().
 	 */
-	assabet_gpio_base = assabet_init_gpio((void *)&ASSABET_BCR, def_val);
-
-	assabet_cf_vcc_pdata.gpio = assabet_gpio_base + 0;
+	assabet_init_gpio((void *)&ASSABET_BCR, def_val);
 }
 
 MACHINE_START(ASSABET, "Intel-Assabet")
diff --git a/arch/arm/mach-sa1100/generic.c b/arch/arm/mach-sa1100/generic.c
index 7167ddf84a0e..800321c6cbd8 100644
--- a/arch/arm/mach-sa1100/generic.c
+++ b/arch/arm/mach-sa1100/generic.c
@@ -348,7 +348,8 @@ void __init sa11x0_init_late(void)
 
 int __init sa11x0_register_fixed_regulator(int n,
 	struct fixed_voltage_config *cfg,
-	struct regulator_consumer_supply *supplies, unsigned num_supplies)
+	struct regulator_consumer_supply *supplies, unsigned num_supplies,
+	bool uses_gpio)
 {
 	struct regulator_init_data *id;
 
@@ -356,7 +357,7 @@ int __init sa11x0_register_fixed_regulator(int n,
 	if (!cfg->init_data)
 		return -ENOMEM;
 
-	if (cfg->gpio < 0)
+	if (!uses_gpio)
 		id->constraints.always_on = 1;
 	id->constraints.name = cfg->supply_name;
 	id->constraints.min_uV = cfg->microvolts;
diff --git a/arch/arm/mach-sa1100/generic.h b/arch/arm/mach-sa1100/generic.h
index 5f3cb52fa6ab..158a4fd5ca24 100644
--- a/arch/arm/mach-sa1100/generic.h
+++ b/arch/arm/mach-sa1100/generic.h
@@ -54,4 +54,5 @@ void sa11x0_register_pcmcia(int socket, struct gpiod_lookup_table *);
 struct fixed_voltage_config;
 struct regulator_consumer_supply;
 int sa11x0_register_fixed_regulator(int n, struct fixed_voltage_config *cfg,
-	struct regulator_consumer_supply *supplies, unsigned num_supplies);
+	struct regulator_consumer_supply *supplies, unsigned num_supplies,
+	bool uses_gpio);
diff --git a/arch/arm/mach-sa1100/shannon.c b/arch/arm/mach-sa1100/shannon.c
index 22f7fe0b809f..5bc82e2671c6 100644
--- a/arch/arm/mach-sa1100/shannon.c
+++ b/arch/arm/mach-sa1100/shannon.c
@@ -102,14 +102,14 @@ static struct fixed_voltage_config shannon_cf_vcc_pdata __initdata = {
 	.supply_name = "cf-power",
 	.microvolts = 3300000,
 	.enabled_at_boot = 1,
-	.gpio = -EINVAL,
 };
 
 static void __init shannon_init(void)
 {
 	sa11x0_register_fixed_regulator(0, &shannon_cf_vcc_pdata,
 					shannon_cf_vcc_consumers,
-					ARRAY_SIZE(shannon_cf_vcc_consumers));
+					ARRAY_SIZE(shannon_cf_vcc_consumers),
+					false);
 	sa11x0_register_pcmcia(0, &shannon_pcmcia0_gpio_table);
 	sa11x0_register_pcmcia(1, &shannon_pcmcia1_gpio_table);
 	sa11x0_ppc_configure_mcp();
diff --git a/arch/arm/mach-sa1100/simpad.c b/arch/arm/mach-sa1100/simpad.c
index f45aed2519ba..406487e76a5c 100644
--- a/arch/arm/mach-sa1100/simpad.c
+++ b/arch/arm/mach-sa1100/simpad.c
@@ -37,7 +37,7 @@
 #include <linux/input.h>
 #include <linux/gpio_keys.h>
 #include <linux/leds.h>
-#include <linux/i2c-gpio.h>
+#include <linux/platform_data/i2c-gpio.h>
 
 #include "generic.h"
 
diff --git a/arch/arm/mach-shmobile/Kconfig b/arch/arm/mach-shmobile/Kconfig
index 280e7312a9e1..0b67254eabb2 100644
--- a/arch/arm/mach-shmobile/Kconfig
+++ b/arch/arm/mach-shmobile/Kconfig
@@ -29,7 +29,6 @@ config ARCH_RMOBILE
 menuconfig ARCH_RENESAS
 	bool "Renesas ARM SoCs"
 	depends on ARCH_MULTI_V7 && MMU
-	select ARCH_DMA_ADDR_T_64BIT if ARM_LPAE
 	select ARCH_SHMOBILE
 	select ARM_GIC
 	select GPIOLIB
@@ -75,6 +74,10 @@ config ARCH_R8A7745
 	bool "RZ/G1E (R8A77450)"
 	select ARCH_RCAR_GEN2
 
+config ARCH_R8A77470
+	bool "RZ/G1C (R8A77470)"
+	select ARCH_RCAR_GEN2
+
 config ARCH_R8A7778
 	bool "R-Car M1A (R8A77781)"
 	select ARCH_RCAR_GEN1
@@ -110,6 +113,15 @@ config ARCH_R8A7794
 	bool "R-Car E2 (R8A77940)"
 	select ARCH_RCAR_GEN2
 
+config ARCH_R9A06G032
+	bool "RZ/N1D (R9A06G032)"
+	select ARCH_RZN1
+
+config ARCH_RZN1
+	bool "RZ/N1 (R9A06G0xx) Family"
+	select ARM_AMBA
+	select CPU_V7
+
 config ARCH_SH73A0
 	bool "SH-Mobile AG5 (R8A73A00)"
 	select ARCH_RMOBILE
diff --git a/arch/arm/mach-shmobile/common.h b/arch/arm/mach-shmobile/common.h
index 43c1ac696274..2109f123bdfb 100644
--- a/arch/arm/mach-shmobile/common.h
+++ b/arch/arm/mach-shmobile/common.h
@@ -2,7 +2,6 @@
 #ifndef __ARCH_MACH_COMMON_H
 #define __ARCH_MACH_COMMON_H
 
-extern void shmobile_init_cntvoff(void);
 extern void shmobile_init_delay(void);
 extern void shmobile_boot_vector(void);
 extern unsigned long shmobile_boot_fn;
diff --git a/arch/arm/mach-shmobile/headsmp-apmu.S b/arch/arm/mach-shmobile/headsmp-apmu.S
index 5672b5849401..d49ab194766a 100644
--- a/arch/arm/mach-shmobile/headsmp-apmu.S
+++ b/arch/arm/mach-shmobile/headsmp-apmu.S
@@ -11,29 +11,9 @@
 #include <linux/linkage.h>
 #include <asm/assembler.h>
 
-ENTRY(shmobile_init_cntvoff)
-	/*
-	 * CNTVOFF has to be initialized either from non-secure Hypervisor
-	 * mode or secure Monitor mode with SCR.NS==1. If TrustZone is enabled
-	 * then it should be handled by the secure code
-	 */
-	cps	#MON_MODE
-	mrc	p15, 0, r1, c1, c1, 0		/* Get Secure Config */
-	orr	r0, r1, #1
-	mcr	p15, 0, r0, c1, c1, 0		/* Set Non Secure bit */
-	instr_sync
-	mov	r0, #0
-	mcrr	p15, 4, r0, r0, c14		/* CNTVOFF = 0 */
-	instr_sync
-	mcr	p15, 0, r1, c1, c1, 0		/* Set Secure bit */
-	instr_sync
-	cps	#SVC_MODE
-	ret	lr
-ENDPROC(shmobile_init_cntvoff)
-
 #ifdef CONFIG_SMP
 ENTRY(shmobile_boot_apmu)
-	bl	shmobile_init_cntvoff
+	bl	secure_cntvoff_init
 	b	secondary_startup
 ENDPROC(shmobile_boot_apmu)
 #endif
diff --git a/arch/arm/mach-shmobile/setup-rcar-gen2.c b/arch/arm/mach-shmobile/setup-rcar-gen2.c
index 5561dbed7a33..88fdc1801d90 100644
--- a/arch/arm/mach-shmobile/setup-rcar-gen2.c
+++ b/arch/arm/mach-shmobile/setup-rcar-gen2.c
@@ -26,6 +26,7 @@
 #include <linux/of_fdt.h>
 #include <linux/of_platform.h>
 #include <asm/mach/arch.h>
+#include <asm/secure_cntvoff.h>
 #include "common.h"
 #include "rcar-gen2.h"
 
@@ -70,9 +71,10 @@ void __init rcar_gen2_timer_init(void)
 	void __iomem *base;
 	u32 freq;
 
-	shmobile_init_cntvoff();
+	secure_cntvoff_init();
 
 	if (of_machine_is_compatible("renesas,r8a7745") ||
+	    of_machine_is_compatible("renesas,r8a77470") ||
 	    of_machine_is_compatible("renesas,r8a7792") ||
 	    of_machine_is_compatible("renesas,r8a7794")) {
 		freq = 260000000 / 8;	/* ZS / 8 */
@@ -205,6 +207,7 @@ MACHINE_END
 static const char * const rz_g1_boards_compat_dt[] __initconst = {
 	"renesas,r8a7743",
 	"renesas,r8a7745",
+	"renesas,r8a77470",
 	NULL,
 };
 
diff --git a/arch/arm/mach-sunxi/Kconfig b/arch/arm/mach-sunxi/Kconfig
index ce53ceaf4cc5..d9c8ecf88ec6 100644
--- a/arch/arm/mach-sunxi/Kconfig
+++ b/arch/arm/mach-sunxi/Kconfig
@@ -51,7 +51,7 @@ config MACH_SUN9I
 config ARCH_SUNXI_MC_SMP
 	bool
 	depends on SMP
-	default MACH_SUN9I
+	default MACH_SUN9I || MACH_SUN8I
 	select ARM_CCI400_PORT_CTRL
 	select ARM_CPU_SUSPEND
 
diff --git a/arch/arm/mach-sunxi/Makefile b/arch/arm/mach-sunxi/Makefile
index 7de9cc286d53..71429aa85143 100644
--- a/arch/arm/mach-sunxi/Makefile
+++ b/arch/arm/mach-sunxi/Makefile
@@ -1,5 +1,5 @@
 CFLAGS_mc_smp.o	+= -march=armv7-a
 
 obj-$(CONFIG_ARCH_SUNXI) += sunxi.o
-obj-$(CONFIG_ARCH_SUNXI_MC_SMP) += mc_smp.o
+obj-$(CONFIG_ARCH_SUNXI_MC_SMP) += mc_smp.o headsmp.o
 obj-$(CONFIG_SMP) += platsmp.o
diff --git a/arch/arm/mach-sunxi/headsmp.S b/arch/arm/mach-sunxi/headsmp.S
new file mode 100644
index 000000000000..32d76be98541
--- /dev/null
+++ b/arch/arm/mach-sunxi/headsmp.S
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (c) 2018 Chen-Yu Tsai
+ * Copyright (c) 2018 Bootlin
+ *
+ * Chen-Yu Tsai <wens@csie.org>
+ * Mylène Josserand <mylene.josserand@bootlin.com>
+ *
+ * SMP support for sunxi based systems with Cortex A7/A15
+ *
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+#include <asm/cputype.h>
+
+ENTRY(sunxi_mc_smp_cluster_cache_enable)
+	.arch	armv7-a
+	/*
+	 * Enable cluster-level coherency, in preparation for turning on the MMU.
+	 *
+	 * Also enable regional clock gating and L2 data latency settings for
+	 * Cortex-A15. These settings are from the vendor kernel.
+	 */
+	mrc	p15, 0, r1, c0, c0, 0
+	movw	r2, #(ARM_CPU_PART_MASK & 0xffff)
+	movt	r2, #(ARM_CPU_PART_MASK >> 16)
+	and	r1, r1, r2
+	movw	r2, #(ARM_CPU_PART_CORTEX_A15 & 0xffff)
+	movt	r2, #(ARM_CPU_PART_CORTEX_A15 >> 16)
+	cmp	r1, r2
+	bne	not_a15
+
+	/* The following is Cortex-A15 specific */
+
+	/* ACTLR2: Enable CPU regional clock gates */
+	mrc p15, 1, r1, c15, c0, 4
+	orr r1, r1, #(0x1 << 31)
+	mcr p15, 1, r1, c15, c0, 4
+
+	/* L2ACTLR */
+	mrc p15, 1, r1, c15, c0, 0
+	/* Enable L2, GIC, and Timer regional clock gates */
+	orr r1, r1, #(0x1 << 26)
+	/* Disable clean/evict from being pushed to external */
+	orr r1, r1, #(0x1<<3)
+	mcr p15, 1, r1, c15, c0, 0
+
+	/* L2CTRL: L2 data RAM latency */
+	mrc p15, 1, r1, c9, c0, 2
+	bic r1, r1, #(0x7 << 0)
+	orr r1, r1, #(0x3 << 0)
+	mcr p15, 1, r1, c9, c0, 2
+
+	/* End of Cortex-A15 specific setup */
+	not_a15:
+
+	/* Get value of sunxi_mc_smp_first_comer */
+	adr	r1, first
+	ldr	r0, [r1]
+	ldr	r0, [r1, r0]
+
+	/* Skip cci_enable_port_for_self if not first comer */
+	cmp	r0, #0
+	bxeq	lr
+	b	cci_enable_port_for_self
+
+	.align 2
+	first: .word sunxi_mc_smp_first_comer - .
+ENDPROC(sunxi_mc_smp_cluster_cache_enable)
+
+ENTRY(sunxi_mc_smp_secondary_startup)
+	bl	sunxi_mc_smp_cluster_cache_enable
+	bl	secure_cntvoff_init
+	b	secondary_startup
+ENDPROC(sunxi_mc_smp_secondary_startup)
+
+ENTRY(sunxi_mc_smp_resume)
+	bl	sunxi_mc_smp_cluster_cache_enable
+	b	cpu_resume
+ENDPROC(sunxi_mc_smp_resume)
diff --git a/arch/arm/mach-sunxi/mc_smp.c b/arch/arm/mach-sunxi/mc_smp.c
index c0246ec54a0a..b4037b603897 100644
--- a/arch/arm/mach-sunxi/mc_smp.c
+++ b/arch/arm/mach-sunxi/mc_smp.c
@@ -55,22 +55,35 @@
 #define CPUCFG_CX_RST_CTRL_L2_RST	BIT(8)
 #define CPUCFG_CX_RST_CTRL_CX_RST(n)	BIT(4 + (n))
 #define CPUCFG_CX_RST_CTRL_CORE_RST(n)	BIT(n)
+#define CPUCFG_CX_RST_CTRL_CORE_RST_ALL	(0xf << 0)
 
 #define PRCM_CPU_PO_RST_CTRL(c)		(0x4 + 0x4 * (c))
 #define PRCM_CPU_PO_RST_CTRL_CORE(n)	BIT(n)
 #define PRCM_CPU_PO_RST_CTRL_CORE_ALL	0xf
 #define PRCM_PWROFF_GATING_REG(c)	(0x100 + 0x4 * (c))
-#define PRCM_PWROFF_GATING_REG_CLUSTER	BIT(4)
+/* The power off register for clusters are different from a80 and a83t */
+#define PRCM_PWROFF_GATING_REG_CLUSTER_SUN8I	BIT(0)
+#define PRCM_PWROFF_GATING_REG_CLUSTER_SUN9I	BIT(4)
 #define PRCM_PWROFF_GATING_REG_CORE(n)	BIT(n)
 #define PRCM_PWR_SWITCH_REG(c, cpu)	(0x140 + 0x10 * (c) + 0x4 * (cpu))
 #define PRCM_CPU_SOFT_ENTRY_REG		0x164
 
+/* R_CPUCFG registers, specific to sun8i-a83t */
+#define R_CPUCFG_CLUSTER_PO_RST_CTRL(c)	(0x30 + (c) * 0x4)
+#define R_CPUCFG_CLUSTER_PO_RST_CTRL_CORE(n)	BIT(n)
+#define R_CPUCFG_CPU_SOFT_ENTRY_REG		0x01a4
+
 #define CPU0_SUPPORT_HOTPLUG_MAGIC0	0xFA50392F
 #define CPU0_SUPPORT_HOTPLUG_MAGIC1	0x790DCA3A
 
 static void __iomem *cpucfg_base;
 static void __iomem *prcm_base;
 static void __iomem *sram_b_smp_base;
+static void __iomem *r_cpucfg_base;
+
+extern void sunxi_mc_smp_secondary_startup(void);
+extern void sunxi_mc_smp_resume(void);
+static bool is_a83t;
 
 static bool sunxi_core_is_cortex_a15(unsigned int core, unsigned int cluster)
 {
@@ -157,6 +170,16 @@ static int sunxi_cpu_powerup(unsigned int cpu, unsigned int cluster)
 	reg &= ~PRCM_CPU_PO_RST_CTRL_CORE(cpu);
 	writel(reg, prcm_base + PRCM_CPU_PO_RST_CTRL(cluster));
 
+	if (is_a83t) {
+		/* assert cpu power-on reset */
+		reg  = readl(r_cpucfg_base +
+			     R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster));
+		reg &= ~(R_CPUCFG_CLUSTER_PO_RST_CTRL_CORE(cpu));
+		writel(reg, r_cpucfg_base +
+		       R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster));
+		udelay(10);
+	}
+
 	/* Cortex-A7: hold L1 reset disable signal low */
 	if (!sunxi_core_is_cortex_a15(cpu, cluster)) {
 		reg = readl(cpucfg_base + CPUCFG_CX_CTRL_REG0(cluster));
@@ -180,17 +203,38 @@ static int sunxi_cpu_powerup(unsigned int cpu, unsigned int cluster)
 	/* open power switch */
 	sunxi_cpu_power_switch_set(cpu, cluster, true);
 
+	/* Handle A83T bit swap */
+	if (is_a83t) {
+		if (cpu == 0)
+			cpu = 4;
+	}
+
 	/* clear processor power gate */
 	reg = readl(prcm_base + PRCM_PWROFF_GATING_REG(cluster));
 	reg &= ~PRCM_PWROFF_GATING_REG_CORE(cpu);
 	writel(reg, prcm_base + PRCM_PWROFF_GATING_REG(cluster));
 	udelay(20);
 
+	/* Handle A83T bit swap */
+	if (is_a83t) {
+		if (cpu == 4)
+			cpu = 0;
+	}
+
 	/* de-assert processor power-on reset */
 	reg = readl(prcm_base + PRCM_CPU_PO_RST_CTRL(cluster));
 	reg |= PRCM_CPU_PO_RST_CTRL_CORE(cpu);
 	writel(reg, prcm_base + PRCM_CPU_PO_RST_CTRL(cluster));
 
+	if (is_a83t) {
+		reg  = readl(r_cpucfg_base +
+			     R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster));
+		reg |= R_CPUCFG_CLUSTER_PO_RST_CTRL_CORE(cpu);
+		writel(reg, r_cpucfg_base +
+		       R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster));
+		udelay(10);
+	}
+
 	/* de-assert all processor resets */
 	reg = readl(cpucfg_base + CPUCFG_CX_RST_CTRL(cluster));
 	reg |= CPUCFG_CX_RST_CTRL_DBG_RST(cpu);
@@ -212,6 +256,14 @@ static int sunxi_cluster_powerup(unsigned int cluster)
 	if (cluster >= SUNXI_NR_CLUSTERS)
 		return -EINVAL;
 
+	/* For A83T, assert cluster cores resets */
+	if (is_a83t) {
+		reg = readl(cpucfg_base + CPUCFG_CX_RST_CTRL(cluster));
+		reg &= ~CPUCFG_CX_RST_CTRL_CORE_RST_ALL;   /* Core Reset    */
+		writel(reg, cpucfg_base + CPUCFG_CX_RST_CTRL(cluster));
+		udelay(10);
+	}
+
 	/* assert ACINACTM */
 	reg = readl(cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster));
 	reg |= CPUCFG_CX_CTRL_REG1_ACINACTM;
@@ -222,6 +274,16 @@ static int sunxi_cluster_powerup(unsigned int cluster)
 	reg &= ~PRCM_CPU_PO_RST_CTRL_CORE_ALL;
 	writel(reg, prcm_base + PRCM_CPU_PO_RST_CTRL(cluster));
 
+	/* assert cluster cores resets */
+	if (is_a83t) {
+		reg  = readl(r_cpucfg_base +
+			     R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster));
+		reg &= ~CPUCFG_CX_RST_CTRL_CORE_RST_ALL;
+		writel(reg, r_cpucfg_base +
+		       R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster));
+		udelay(10);
+	}
+
 	/* assert cluster resets */
 	reg = readl(cpucfg_base + CPUCFG_CX_RST_CTRL(cluster));
 	reg &= ~CPUCFG_CX_RST_CTRL_DBG_SOC_RST;
@@ -252,7 +314,10 @@ static int sunxi_cluster_powerup(unsigned int cluster)
 
 	/* clear cluster power gate */
 	reg = readl(prcm_base + PRCM_PWROFF_GATING_REG(cluster));
-	reg &= ~PRCM_PWROFF_GATING_REG_CLUSTER;
+	if (is_a83t)
+		reg &= ~PRCM_PWROFF_GATING_REG_CLUSTER_SUN8I;
+	else
+		reg &= ~PRCM_PWROFF_GATING_REG_CLUSTER_SUN9I;
 	writel(reg, prcm_base + PRCM_PWROFF_GATING_REG(cluster));
 	udelay(20);
 
@@ -300,74 +365,7 @@ static void sunxi_cluster_cache_disable_without_axi(void)
 }
 
 static int sunxi_mc_smp_cpu_table[SUNXI_NR_CLUSTERS][SUNXI_CPUS_PER_CLUSTER];
-static int sunxi_mc_smp_first_comer;
-
-/*
- * Enable cluster-level coherency, in preparation for turning on the MMU.
- *
- * Also enable regional clock gating and L2 data latency settings for
- * Cortex-A15. These settings are from the vendor kernel.
- */
-static void __naked sunxi_mc_smp_cluster_cache_enable(void)
-{
-	asm volatile (
-		"mrc	p15, 0, r1, c0, c0, 0\n"
-		"movw	r2, #" __stringify(ARM_CPU_PART_MASK & 0xffff) "\n"
-		"movt	r2, #" __stringify(ARM_CPU_PART_MASK >> 16) "\n"
-		"and	r1, r1, r2\n"
-		"movw	r2, #" __stringify(ARM_CPU_PART_CORTEX_A15 & 0xffff) "\n"
-		"movt	r2, #" __stringify(ARM_CPU_PART_CORTEX_A15 >> 16) "\n"
-		"cmp	r1, r2\n"
-		"bne	not_a15\n"
-
-		/* The following is Cortex-A15 specific */
-
-		/* ACTLR2: Enable CPU regional clock gates */
-		"mrc p15, 1, r1, c15, c0, 4\n"
-		"orr r1, r1, #(0x1<<31)\n"
-		"mcr p15, 1, r1, c15, c0, 4\n"
-
-		/* L2ACTLR */
-		"mrc p15, 1, r1, c15, c0, 0\n"
-		/* Enable L2, GIC, and Timer regional clock gates */
-		"orr r1, r1, #(0x1<<26)\n"
-		/* Disable clean/evict from being pushed to external */
-		"orr r1, r1, #(0x1<<3)\n"
-		"mcr p15, 1, r1, c15, c0, 0\n"
-
-		/* L2CTRL: L2 data RAM latency */
-		"mrc p15, 1, r1, c9, c0, 2\n"
-		"bic r1, r1, #(0x7<<0)\n"
-		"orr r1, r1, #(0x3<<0)\n"
-		"mcr p15, 1, r1, c9, c0, 2\n"
-
-		/* End of Cortex-A15 specific setup */
-		"not_a15:\n"
-
-		/* Get value of sunxi_mc_smp_first_comer */
-		"adr	r1, first\n"
-		"ldr	r0, [r1]\n"
-		"ldr	r0, [r1, r0]\n"
-
-		/* Skip cci_enable_port_for_self if not first comer */
-		"cmp	r0, #0\n"
-		"bxeq	lr\n"
-		"b	cci_enable_port_for_self\n"
-
-		".align 2\n"
-		"first: .word sunxi_mc_smp_first_comer - .\n"
-	);
-}
-
-static void __naked sunxi_mc_smp_secondary_startup(void)
-{
-	asm volatile(
-		"bl	sunxi_mc_smp_cluster_cache_enable\n"
-		"b	secondary_startup"
-		/* Let compiler know about sunxi_mc_smp_cluster_cache_enable */
-		:: "i" (sunxi_mc_smp_cluster_cache_enable)
-	);
-}
+int sunxi_mc_smp_first_comer;
 
 static DEFINE_SPINLOCK(boot_lock);
 
@@ -516,7 +514,10 @@ static int sunxi_cluster_powerdown(unsigned int cluster)
 	/* gate cluster power */
 	pr_debug("%s: gate cluster power\n", __func__);
 	reg = readl(prcm_base + PRCM_PWROFF_GATING_REG(cluster));
-	reg |= PRCM_PWROFF_GATING_REG_CLUSTER;
+	if (is_a83t)
+		reg |= PRCM_PWROFF_GATING_REG_CLUSTER_SUN8I;
+	else
+		reg |= PRCM_PWROFF_GATING_REG_CLUSTER_SUN9I;
 	writel(reg, prcm_base + PRCM_PWROFF_GATING_REG(cluster));
 	udelay(20);
 
@@ -598,8 +599,12 @@ static int sunxi_mc_smp_cpu_kill(unsigned int l_cpu)
 	return !ret;
 }
 
-static bool sunxi_mc_smp_cpu_can_disable(unsigned int __unused)
+static bool sunxi_mc_smp_cpu_can_disable(unsigned int cpu)
 {
+	/* CPU0 hotplug not handled for sun8i-a83t */
+	if (is_a83t)
+		if (cpu == 0)
+			return false;
 	return true;
 }
 #endif
@@ -637,16 +642,6 @@ static bool __init sunxi_mc_smp_cpu_table_init(void)
  */
 typedef typeof(cpu_reset) phys_reset_t;
 
-static void __init __naked sunxi_mc_smp_resume(void)
-{
-	asm volatile(
-		"bl	sunxi_mc_smp_cluster_cache_enable\n"
-		"b	cpu_resume"
-		/* Let compiler know about sunxi_mc_smp_cluster_cache_enable */
-		:: "i" (sunxi_mc_smp_cluster_cache_enable)
-	);
-}
-
 static int __init nocache_trampoline(unsigned long __unused)
 {
 	phys_reset_t phys_reset;
@@ -692,12 +687,14 @@ struct sunxi_mc_smp_nodes {
 	struct device_node *prcm_node;
 	struct device_node *cpucfg_node;
 	struct device_node *sram_node;
+	struct device_node *r_cpucfg_node;
 };
 
 /* This structure holds SoC-specific bits tied to an enable-method string. */
 struct sunxi_mc_smp_data {
 	const char *enable_method;
 	int (*get_smp_nodes)(struct sunxi_mc_smp_nodes *nodes);
+	bool is_a83t;
 };
 
 static void __init sunxi_mc_smp_put_nodes(struct sunxi_mc_smp_nodes *nodes)
@@ -705,6 +702,7 @@ static void __init sunxi_mc_smp_put_nodes(struct sunxi_mc_smp_nodes *nodes)
 	of_node_put(nodes->prcm_node);
 	of_node_put(nodes->cpucfg_node);
 	of_node_put(nodes->sram_node);
+	of_node_put(nodes->r_cpucfg_node);
 	memset(nodes, 0, sizeof(*nodes));
 }
 
@@ -734,11 +732,42 @@ static int __init sun9i_a80_get_smp_nodes(struct sunxi_mc_smp_nodes *nodes)
 	return 0;
 }
 
+static int __init sun8i_a83t_get_smp_nodes(struct sunxi_mc_smp_nodes *nodes)
+{
+	nodes->prcm_node = of_find_compatible_node(NULL, NULL,
+						   "allwinner,sun8i-a83t-r-ccu");
+	if (!nodes->prcm_node) {
+		pr_err("%s: PRCM not available\n", __func__);
+		return -ENODEV;
+	}
+
+	nodes->cpucfg_node = of_find_compatible_node(NULL, NULL,
+						     "allwinner,sun8i-a83t-cpucfg");
+	if (!nodes->cpucfg_node) {
+		pr_err("%s: CPUCFG not available\n", __func__);
+		return -ENODEV;
+	}
+
+	nodes->r_cpucfg_node = of_find_compatible_node(NULL, NULL,
+						       "allwinner,sun8i-a83t-r-cpucfg");
+	if (!nodes->r_cpucfg_node) {
+		pr_err("%s: RCPUCFG not available\n", __func__);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
 static const struct sunxi_mc_smp_data sunxi_mc_smp_data[] __initconst = {
 	{
 		.enable_method	= "allwinner,sun9i-a80-smp",
 		.get_smp_nodes	= sun9i_a80_get_smp_nodes,
 	},
+	{
+		.enable_method	= "allwinner,sun8i-a83t-smp",
+		.get_smp_nodes	= sun8i_a83t_get_smp_nodes,
+		.is_a83t	= true,
+	},
 };
 
 static int __init sunxi_mc_smp_init(void)
@@ -746,6 +775,7 @@ static int __init sunxi_mc_smp_init(void)
 	struct sunxi_mc_smp_nodes nodes = { 0 };
 	struct device_node *node;
 	struct resource res;
+	void __iomem *addr;
 	int i, ret;
 
 	/*
@@ -771,6 +801,8 @@ static int __init sunxi_mc_smp_init(void)
 			break;
 	}
 
+	is_a83t = sunxi_mc_smp_data[i].is_a83t;
+
 	of_node_put(node);
 	if (ret)
 		return -ENODEV;
@@ -808,12 +840,23 @@ static int __init sunxi_mc_smp_init(void)
 		goto err_unmap_prcm;
 	}
 
-	sram_b_smp_base = of_io_request_and_map(nodes.sram_node, 0,
-						"sunxi-mc-smp");
-	if (IS_ERR(sram_b_smp_base)) {
-		ret = PTR_ERR(sram_b_smp_base);
-		pr_err("%s: failed to map secure SRAM\n", __func__);
-		goto err_unmap_release_cpucfg;
+	if (is_a83t) {
+		r_cpucfg_base = of_io_request_and_map(nodes.r_cpucfg_node,
+						      0, "sunxi-mc-smp");
+		if (IS_ERR(r_cpucfg_base)) {
+			ret = PTR_ERR(r_cpucfg_base);
+			pr_err("%s: failed to map R-CPUCFG registers\n",
+			       __func__);
+			goto err_unmap_release_cpucfg;
+		}
+	} else {
+		sram_b_smp_base = of_io_request_and_map(nodes.sram_node, 0,
+							"sunxi-mc-smp");
+		if (IS_ERR(sram_b_smp_base)) {
+			ret = PTR_ERR(sram_b_smp_base);
+			pr_err("%s: failed to map secure SRAM\n", __func__);
+			goto err_unmap_release_cpucfg;
+		}
 	}
 
 	/* Configure CCI-400 for boot cluster */
@@ -821,15 +864,18 @@ static int __init sunxi_mc_smp_init(void)
 	if (ret) {
 		pr_err("%s: failed to configure boot cluster: %d\n",
 		       __func__, ret);
-		goto err_unmap_release_secure_sram;
+		goto err_unmap_release_sram_rcpucfg;
 	}
 
 	/* We don't need the device nodes anymore */
 	sunxi_mc_smp_put_nodes(&nodes);
 
 	/* Set the hardware entry point address */
-	writel(__pa_symbol(sunxi_mc_smp_secondary_startup),
-	       prcm_base + PRCM_CPU_SOFT_ENTRY_REG);
+	if (is_a83t)
+		addr = r_cpucfg_base + R_CPUCFG_CPU_SOFT_ENTRY_REG;
+	else
+		addr = prcm_base + PRCM_CPU_SOFT_ENTRY_REG;
+	writel(__pa_symbol(sunxi_mc_smp_secondary_startup), addr);
 
 	/* Actually enable multi cluster SMP */
 	smp_set_ops(&sunxi_mc_smp_smp_ops);
@@ -838,9 +884,14 @@ static int __init sunxi_mc_smp_init(void)
 
 	return 0;
 
-err_unmap_release_secure_sram:
-	iounmap(sram_b_smp_base);
-	of_address_to_resource(nodes.sram_node, 0, &res);
+err_unmap_release_sram_rcpucfg:
+	if (is_a83t) {
+		iounmap(r_cpucfg_base);
+		of_address_to_resource(nodes.r_cpucfg_node, 0, &res);
+	} else {
+		iounmap(sram_b_smp_base);
+		of_address_to_resource(nodes.sram_node, 0, &res);
+	}
 	release_mem_region(res.start, resource_size(&res));
 err_unmap_release_cpucfg:
 	iounmap(cpucfg_base);
diff --git a/arch/arm/mach-sunxi/sunxi.c b/arch/arm/mach-sunxi/sunxi.c
index 5e9602ce1573..de4b0e932f22 100644
--- a/arch/arm/mach-sunxi/sunxi.c
+++ b/arch/arm/mach-sunxi/sunxi.c
@@ -16,6 +16,7 @@
 #include <linux/platform_device.h>
 
 #include <asm/mach/arch.h>
+#include <asm/secure_cntvoff.h>
 
 static const char * const sunxi_board_dt_compat[] = {
 	"allwinner,sun4i-a10",
@@ -62,7 +63,6 @@ MACHINE_END
 static const char * const sun8i_board_dt_compat[] = {
 	"allwinner,sun8i-a23",
 	"allwinner,sun8i-a33",
-	"allwinner,sun8i-a83t",
 	"allwinner,sun8i-h2-plus",
 	"allwinner,sun8i-h3",
 	"allwinner,sun8i-r40",
@@ -75,6 +75,24 @@ DT_MACHINE_START(SUN8I_DT, "Allwinner sun8i Family")
 	.dt_compat	= sun8i_board_dt_compat,
 MACHINE_END
 
+static void __init sun8i_a83t_cntvoff_init(void)
+{
+#ifdef CONFIG_SMP
+	secure_cntvoff_init();
+#endif
+}
+
+static const char * const sun8i_a83t_cntvoff_board_dt_compat[] = {
+	"allwinner,sun8i-a83t",
+	NULL,
+};
+
+DT_MACHINE_START(SUN8I_A83T_CNTVOFF_DT, "Allwinner A83t board")
+	.init_early	= sun8i_a83t_cntvoff_init,
+	.init_time	= sun6i_timer_init,
+	.dt_compat	= sun8i_a83t_cntvoff_board_dt_compat,
+MACHINE_END
+
 static const char * const sun9i_board_dt_compat[] = {
 	"allwinner,sun9i-a80",
 	NULL,
diff --git a/arch/arm/mach-tegra/Kconfig b/arch/arm/mach-tegra/Kconfig
index 1e0aeb47bac6..7f3b83e0d324 100644
--- a/arch/arm/mach-tegra/Kconfig
+++ b/arch/arm/mach-tegra/Kconfig
@@ -15,6 +15,5 @@ menuconfig ARCH_TEGRA
 	select RESET_CONTROLLER
 	select SOC_BUS
 	select ZONE_DMA if ARM_LPAE
-	select ARCH_DMA_ADDR_T_64BIT if ARM_LPAE
 	help
 	  This enables support for NVIDIA Tegra based systems.
diff --git a/arch/arm/mach-tegra/tegra.c b/arch/arm/mach-tegra/tegra.c
index 02e712d2ea30..f9587be48235 100644
--- a/arch/arm/mach-tegra/tegra.c
+++ b/arch/arm/mach-tegra/tegra.c
@@ -97,6 +97,10 @@ static void __init tegra_dt_init_late(void)
 	if (IS_ENABLED(CONFIG_ARCH_TEGRA_2x_SOC) &&
 	    of_machine_is_compatible("compal,paz00"))
 		tegra_paz00_wifikill_init();
+
+	if (IS_ENABLED(CONFIG_ARCH_TEGRA_2x_SOC) &&
+	    of_machine_is_compatible("nvidia,tegra20"))
+		platform_device_register_simple("tegra20-cpufreq", -1, NULL, 0);
 }
 
 static const char * const tegra_dt_board_compat[] = {
diff --git a/arch/arm/mach-ux500/Kconfig b/arch/arm/mach-ux500/Kconfig
index f98332ea2ef2..c1086ebe0050 100644
--- a/arch/arm/mach-ux500/Kconfig
+++ b/arch/arm/mach-ux500/Kconfig
@@ -9,64 +9,33 @@ menuconfig ARCH_U8500
 	select ARM_ERRATA_764369 if SMP
 	select ARM_GIC
 	select CACHE_L2X0
+	select CLKSRC_DBX500_PRCMU
 	select CLKSRC_NOMADIK_MTU
 	select GPIOLIB
 	select HAVE_ARM_SCU if SMP
 	select HAVE_ARM_TWD if SMP
+	select I2C
+	select I2C_NOMADIK
+	select MFD_DB8500_PRCMU
 	select PINCTRL
+	select PINCTRL_AB8500
+	select PINCTRL_AB8505
 	select PINCTRL_ABX500
+	select PINCTRL_DB8500
 	select PINCTRL_NOMADIK
 	select PL310_ERRATA_753970 if CACHE_L2X0
-	help
-	  Support for ST-Ericsson's Ux500 architecture
-
-if ARCH_U8500
-
-config UX500_SOC_DB8500
-	bool
-	select MFD_DB8500_PRCMU
-	select PINCTRL_DB8500
-	select PINCTRL_DB8540
-	select PINCTRL_AB8500
-	select PINCTRL_AB8505
-	select PINCTRL_AB9540
-	select PINCTRL_AB8540
-	select REGULATOR
-	select REGULATOR_DB8500_PRCMU
-	select CLKSRC_DBX500_PRCMU
 	select PM_GENERIC_DOMAINS if PM
-
-config MACH_MOP500
-	bool "U8500 Development platform, MOP500 versions"
-	select I2C
-	select I2C_NOMADIK
 	select REGULATOR
+	select REGULATOR_DB8500_PRCMU
 	select REGULATOR_FIXED_VOLTAGE
 	select SOC_BUS
-	select UX500_SOC_DB8500
 	help
-	  Include support for the MOP500 development platform.
-
-config MACH_HREFV60
-	bool "U8500 Development platform, HREFv60 version"
-	select MACH_MOP500
-	help
-	  Include support for the HREFv60 new development platform.
-	  Includes HREFv70, v71 etc.
+	  Support for ST-Ericsson's Ux500 architecture
 
-config MACH_SNOWBALL
-	bool "U8500 Snowball platform"
-	select MACH_MOP500
-	help
-	  Include support for the snowball development platform.
+if ARCH_U8500
 
-config UX500_AUTO_PLATFORM
+config UX500_SOC_DB8500
 	def_bool y
-	select MACH_MOP500
-	help
-	  At least one platform needs to be selected in order to build
-	  a working kernel. If everything else is disabled, this
-	  automatically enables MACH_MOP500.
 
 config UX500_DEBUG_UART
 	int "Ux500 UART to use for low-level debug"
diff --git a/arch/arm/mach-ux500/cpu-db8500.c b/arch/arm/mach-ux500/cpu-db8500.c
index 36cd23c8be9b..389ecf6faa00 100644
--- a/arch/arm/mach-ux500/cpu-db8500.c
+++ b/arch/arm/mach-ux500/cpu-db8500.c
@@ -111,11 +111,6 @@ static void ux500_restart(enum reboot_mode mode, const char *cmd)
 	prcmu_system_reset(0);
 }
 
-static struct of_dev_auxdata u8540_auxdata_lookup[] __initdata = {
-	OF_DEV_AUXDATA("stericsson,db8500-prcmu", 0x80157000, "db8500-prcmu", NULL),
-	{},
-};
-
 static const struct of_device_id u8500_local_bus_nodes[] = {
 	/* only create devices below soc node */
 	{ .compatible = "stericsson,db8500", },
@@ -129,20 +124,13 @@ static void __init u8500_init_machine(void)
 	/* Initialize ux500 power domains */
 	ux500_pm_domains_init();
 
-	/* automatically probe child nodes of dbx5x0 devices */
-	if (of_machine_is_compatible("st-ericsson,u8540"))
-		of_platform_populate(NULL, u8500_local_bus_nodes,
-				     u8540_auxdata_lookup, NULL);
-	else
-		of_platform_populate(NULL, u8500_local_bus_nodes,
-				     NULL, NULL);
+	of_platform_populate(NULL, u8500_local_bus_nodes,
+			     NULL, NULL);
 }
 
 static const char * stericsson_dt_platform_compat[] = {
 	"st-ericsson,u8500",
-	"st-ericsson,u8540",
 	"st-ericsson,u9500",
-	"st-ericsson,u9540",
 	NULL,
 };
 
diff --git a/arch/arm/mach-ux500/db8500-regs.h b/arch/arm/mach-ux500/db8500-regs.h
index 27399553c841..3d6e1955119a 100644
--- a/arch/arm/mach-ux500/db8500-regs.h
+++ b/arch/arm/mach-ux500/db8500-regs.h
@@ -41,10 +41,6 @@
 /* ASIC ID is at 0xbf4 offset within this region */
 #define U8500_ASIC_ID_BASE	0x9001D000
 
-#define U9540_BOOT_ROM_BASE	0xFFFE0000
-/* ASIC ID is at 0xbf4 offset within this region */
-#define U9540_ASIC_ID_BASE	0xFFFFD000
-
 #define U8500_PER6_BASE		0xa03c0000
 #define U8500_PER7_BASE		0xa03d0000
 #define U8500_PER5_BASE		0xa03e0000
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 7f14acf67caf..96a7b6cf459b 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -415,6 +415,7 @@ config CPU_V7
 	select CPU_CP15_MPU if !MMU
 	select CPU_HAS_ASID if MMU
 	select CPU_PABRT_V7
+	select CPU_SPECTRE if MMU
 	select CPU_THUMB_CAPABLE
 	select CPU_TLB_V7 if MMU
 
@@ -661,6 +662,7 @@ config ARM_LPAE
 	bool "Support for the Large Physical Address Extension"
 	depends on MMU && CPU_32v7 && !CPU_32v6 && !CPU_32v5 && \
 		!CPU_32v4 && !CPU_32v3
+	select PHYS_ADDR_T_64BIT
 	help
 	  Say Y if you have an ARMv7 processor supporting the LPAE page
 	  table format and you would like to access memory beyond the
@@ -673,12 +675,6 @@ config ARM_PV_FIXUP
 	def_bool y
 	depends on ARM_LPAE && ARM_PATCH_PHYS_VIRT && ARCH_KEYSTONE
 
-config ARCH_PHYS_ADDR_T_64BIT
-	def_bool ARM_LPAE
-
-config ARCH_DMA_ADDR_T_64BIT
-	bool
-
 config ARM_THUMB
 	bool "Support Thumb user binaries" if !CPU_THUMBONLY && EXPERT
 	depends on CPU_THUMB_CAPABLE
@@ -826,6 +822,28 @@ config CPU_BPREDICT_DISABLE
 	help
 	  Say Y here to disable branch prediction.  If unsure, say N.
 
+config CPU_SPECTRE
+	bool
+
+config HARDEN_BRANCH_PREDICTOR
+	bool "Harden the branch predictor against aliasing attacks" if EXPERT
+	depends on CPU_SPECTRE
+	default y
+	help
+	   Speculation attacks against some high-performance processors rely
+	   on being able to manipulate the branch predictor for a victim
+	   context by executing aliasing branches in the attacker context.
+	   Such attacks can be partially mitigated against by clearing
+	   internal branch predictor state and limiting the prediction
+	   logic in some situations.
+
+	   This config option will take CPU-specific actions to harden
+	   the branch predictor against aliasing attacks and may rely on
+	   specific instruction sequences or control bits being set by
+	   the system firmware.
+
+	   If unsure, say Y.
+
 config TLS_REG_EMUL
 	bool
 	select NEED_KUSER_HELPERS
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index 9dbb84923e12..7cb1699fbfc4 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -10,7 +10,7 @@ obj-$(CONFIG_MMU)		+= fault-armv.o flush.o idmap.o ioremap.o \
 
 ifneq ($(CONFIG_MMU),y)
 obj-y				+= nommu.o
-obj-$(CONFIG_ARM_MPU)		+= pmsa-v7.o
+obj-$(CONFIG_ARM_MPU)		+= pmsa-v7.o pmsa-v8.o
 endif
 
 obj-$(CONFIG_ARM_PTDUMP_CORE)	+= dump.o
@@ -97,7 +97,7 @@ obj-$(CONFIG_CPU_MOHAWK)	+= proc-mohawk.o
 obj-$(CONFIG_CPU_FEROCEON)	+= proc-feroceon.o
 obj-$(CONFIG_CPU_V6)		+= proc-v6.o
 obj-$(CONFIG_CPU_V6K)		+= proc-v6.o
-obj-$(CONFIG_CPU_V7)		+= proc-v7.o
+obj-$(CONFIG_CPU_V7)		+= proc-v7.o proc-v7-bugs.o
 obj-$(CONFIG_CPU_V7M)		+= proc-v7m.o
 
 AFLAGS_proc-v6.o	:=-Wa,-march=armv6
diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c
index 2c96190e018b..bd2c739d8083 100644
--- a/arch/arm/mm/alignment.c
+++ b/arch/arm/mm/alignment.c
@@ -950,6 +950,7 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	if (ai_usermode & UM_SIGNAL) {
 		siginfo_t si;
 
+		clear_siginfo(&si);
 		si.si_signo = SIGBUS;
 		si.si_errno = 0;
 		si.si_code = BUS_ADRALN;
diff --git a/arch/arm/mm/cache-b15-rac.c b/arch/arm/mm/cache-b15-rac.c
index d9586ba2e63c..c6ed14840c3c 100644
--- a/arch/arm/mm/cache-b15-rac.c
+++ b/arch/arm/mm/cache-b15-rac.c
@@ -33,7 +33,10 @@ extern void v7_flush_kern_cache_all(void);
 #define  RAC_CPU_SHIFT			(8)
 #define  RACCFG_MASK			(0xff)
 #define RAC_CONFIG1_REG			(0x7c)
-#define RAC_FLUSH_REG			(0x80)
+/* Brahma-B15 is a quad-core only design */
+#define B15_RAC_FLUSH_REG		(0x80)
+/* Brahma-B53 is an octo-core design */
+#define B53_RAC_FLUSH_REG		(0x84)
 #define  FLUSH_RAC			(1 << 0)
 
 /* Bitmask to enable instruction and data prefetching with a 256-bytes stride */
@@ -52,6 +55,7 @@ static void __iomem *b15_rac_base;
 static DEFINE_SPINLOCK(rac_lock);
 
 static u32 rac_config0_reg;
+static u32 rac_flush_offset;
 
 /* Initialization flag to avoid checking for b15_rac_base, and to prevent
  * multi-platform kernels from crashing here as well.
@@ -70,14 +74,14 @@ static inline void __b15_rac_flush(void)
 {
 	u32 reg;
 
-	__raw_writel(FLUSH_RAC, b15_rac_base + RAC_FLUSH_REG);
+	__raw_writel(FLUSH_RAC, b15_rac_base + rac_flush_offset);
 	do {
 		/* This dmb() is required to force the Bus Interface Unit
 		 * to clean oustanding writes, and forces an idle cycle
 		 * to be inserted.
 		 */
 		dmb();
-		reg = __raw_readl(b15_rac_base + RAC_FLUSH_REG);
+		reg = __raw_readl(b15_rac_base + rac_flush_offset);
 	} while (reg & FLUSH_RAC);
 }
 
@@ -287,7 +291,7 @@ static struct syscore_ops b15_rac_syscore_ops = {
 
 static int __init b15_rac_init(void)
 {
-	struct device_node *dn;
+	struct device_node *dn, *cpu_dn;
 	int ret = 0, cpu;
 	u32 reg, en_mask = 0;
 
@@ -305,6 +309,24 @@ static int __init b15_rac_init(void)
 		goto out;
 	}
 
+	cpu_dn = of_get_cpu_node(0, NULL);
+	if (!cpu_dn) {
+		ret = -ENODEV;
+		goto out;
+	}
+
+	if (of_device_is_compatible(cpu_dn, "brcm,brahma-b15"))
+		rac_flush_offset = B15_RAC_FLUSH_REG;
+	else if (of_device_is_compatible(cpu_dn, "brcm,brahma-b53"))
+		rac_flush_offset = B53_RAC_FLUSH_REG;
+	else {
+		pr_err("Unsupported CPU\n");
+		of_node_put(cpu_dn);
+		ret = -EINVAL;
+		goto out;
+	}
+	of_node_put(cpu_dn);
+
 	ret = register_reboot_notifier(&b15_rac_reboot_nb);
 	if (ret) {
 		pr_err("failed to register reboot notifier\n");
diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c
index 619f24a42d09..f448a0663b10 100644
--- a/arch/arm/mm/dma-mapping-nommu.c
+++ b/arch/arm/mm/dma-mapping-nommu.c
@@ -241,12 +241,3 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
 void arch_teardown_dma_ops(struct device *dev)
 {
 }
-
-#define PREALLOC_DMA_DEBUG_ENTRIES	4096
-
-static int __init dma_debug_do_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-	return 0;
-}
-core_initcall(dma_debug_do_init);
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index ada8eb206a90..af27f1c22d93 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -831,7 +831,7 @@ static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 		 unsigned long attrs)
 {
 	int ret;
-	unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	unsigned long nr_vma_pages = vma_pages(vma);
 	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
 	unsigned long pfn = dma_to_pfn(dev, dma_addr);
 	unsigned long off = vma->vm_pgoff;
@@ -1151,15 +1151,6 @@ int arm_dma_supported(struct device *dev, u64 mask)
 	return __dma_supported(dev, mask, false);
 }
 
-#define PREALLOC_DMA_DEBUG_ENTRIES	4096
-
-static int __init dma_debug_do_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-	return 0;
-}
-core_initcall(dma_debug_do_init);
-
 #ifdef CONFIG_ARM_DMA_USE_IOMMU
 
 static int __dma_info_to_prot(enum dma_data_direction dir, unsigned long attrs)
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index b75eada23d0a..84becc911ee3 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -163,6 +163,11 @@ __do_user_fault(struct task_struct *tsk, unsigned long addr,
 {
 	struct siginfo si;
 
+	if (addr > TASK_SIZE)
+		harden_branch_predictor();
+
+	clear_siginfo(&si);
+
 #ifdef CONFIG_DEBUG_USER
 	if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) ||
 	    ((user_debug & UDBG_BUS)  && (sig == SIGBUS))) {
@@ -557,6 +562,7 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 		inf->name, fsr, addr);
 	show_pte(current->mm, addr);
 
+	clear_siginfo(&info);
 	info.si_signo = inf->sig;
 	info.si_errno = 0;
 	info.si_code  = inf->code;
@@ -589,6 +595,7 @@ do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
 	pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
 		inf->name, ifsr, addr);
 
+	clear_siginfo(&info);
 	info.si_signo = inf->sig;
 	info.si_errno = 0;
 	info.si_code  = inf->code;
diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c
index 7c087961b7ce..5dd6c58d653b 100644
--- a/arch/arm/mm/nommu.c
+++ b/arch/arm/mm/nommu.c
@@ -99,6 +99,38 @@ void __init arm_mm_memblock_reserve(void)
 	memblock_reserve(0, 1);
 }
 
+static void __init adjust_lowmem_bounds_mpu(void)
+{
+	unsigned long pmsa = read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA;
+
+	switch (pmsa) {
+	case MMFR0_PMSAv7:
+		pmsav7_adjust_lowmem_bounds();
+		break;
+	case MMFR0_PMSAv8:
+		pmsav8_adjust_lowmem_bounds();
+		break;
+	default:
+		break;
+	}
+}
+
+static void __init mpu_setup(void)
+{
+	unsigned long pmsa = read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA;
+
+	switch (pmsa) {
+	case MMFR0_PMSAv7:
+		pmsav7_setup();
+		break;
+	case MMFR0_PMSAv8:
+		pmsav8_setup();
+		break;
+	default:
+		break;
+	}
+}
+
 void __init adjust_lowmem_bounds(void)
 {
 	phys_addr_t end;
diff --git a/arch/arm/mm/pmsa-v7.c b/arch/arm/mm/pmsa-v7.c
index e2853bfff74e..699fa2e88725 100644
--- a/arch/arm/mm/pmsa-v7.c
+++ b/arch/arm/mm/pmsa-v7.c
@@ -102,7 +102,7 @@ static inline u32 irbar_read(void)
 
 static inline void rgnr_write(u32 v)
 {
-	writel_relaxed(v, BASEADDR_V7M_SCB + MPU_RNR);
+	writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv7_RNR);
 }
 
 /* Data-side / unified region attributes */
@@ -110,28 +110,28 @@ static inline void rgnr_write(u32 v)
 /* Region access control register */
 static inline void dracr_write(u32 v)
 {
-	u32 rsr = readl_relaxed(BASEADDR_V7M_SCB + MPU_RASR) & GENMASK(15, 0);
+	u32 rsr = readl_relaxed(BASEADDR_V7M_SCB + PMSAv7_RASR) & GENMASK(15, 0);
 
-	writel_relaxed((v << 16) | rsr, BASEADDR_V7M_SCB + MPU_RASR);
+	writel_relaxed((v << 16) | rsr, BASEADDR_V7M_SCB + PMSAv7_RASR);
 }
 
 /* Region size register */
 static inline void drsr_write(u32 v)
 {
-	u32 racr = readl_relaxed(BASEADDR_V7M_SCB + MPU_RASR) & GENMASK(31, 16);
+	u32 racr = readl_relaxed(BASEADDR_V7M_SCB + PMSAv7_RASR) & GENMASK(31, 16);
 
-	writel_relaxed(v | racr, BASEADDR_V7M_SCB + MPU_RASR);
+	writel_relaxed(v | racr, BASEADDR_V7M_SCB + PMSAv7_RASR);
 }
 
 /* Region base address register */
 static inline void drbar_write(u32 v)
 {
-	writel_relaxed(v, BASEADDR_V7M_SCB + MPU_RBAR);
+	writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv7_RBAR);
 }
 
 static inline u32 drbar_read(void)
 {
-	return readl_relaxed(BASEADDR_V7M_SCB + MPU_RBAR);
+	return readl_relaxed(BASEADDR_V7M_SCB + PMSAv7_RBAR);
 }
 
 /* ARMv7-M only supports a unified MPU, so I-side operations are nop */
@@ -143,11 +143,6 @@ static inline unsigned long irbar_read(void) {return 0;}
 
 #endif
 
-static int __init mpu_present(void)
-{
-	return ((read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA) == MMFR0_PMSAv7);
-}
-
 static bool __init try_split_region(phys_addr_t base, phys_addr_t size, struct region *region)
 {
 	unsigned long  subreg, bslots, sslots;
@@ -161,7 +156,7 @@ static bool __init try_split_region(phys_addr_t base, phys_addr_t size, struct r
 
 	bdiff = base - abase;
 	sdiff = p2size - asize;
-	subreg = p2size / MPU_NR_SUBREGS;
+	subreg = p2size / PMSAv7_NR_SUBREGS;
 
 	if ((bdiff % subreg) || (sdiff % subreg))
 		return false;
@@ -172,17 +167,17 @@ static bool __init try_split_region(phys_addr_t base, phys_addr_t size, struct r
 	if (bslots || sslots) {
 		int i;
 
-		if (subreg < MPU_MIN_SUBREG_SIZE)
+		if (subreg < PMSAv7_MIN_SUBREG_SIZE)
 			return false;
 
-		if (bslots + sslots > MPU_NR_SUBREGS)
+		if (bslots + sslots > PMSAv7_NR_SUBREGS)
 			return false;
 
 		for (i = 0; i < bslots; i++)
 			_set_bit(i, &region->subreg);
 
 		for (i = 1; i <= sslots; i++)
-			_set_bit(MPU_NR_SUBREGS - i, &region->subreg);
+			_set_bit(PMSAv7_NR_SUBREGS - i, &region->subreg);
 	}
 
 	region->base = abase;
@@ -233,7 +228,7 @@ static int __init allocate_region(phys_addr_t base, phys_addr_t size,
 }
 
 /* MPU initialisation functions */
-void __init adjust_lowmem_bounds_mpu(void)
+void __init pmsav7_adjust_lowmem_bounds(void)
 {
 	phys_addr_t  specified_mem_size = 0, total_mem_size = 0;
 	struct memblock_region *reg;
@@ -243,10 +238,7 @@ void __init adjust_lowmem_bounds_mpu(void)
 	unsigned int mem_max_regions;
 	int num, i;
 
-	if (!mpu_present())
-		return;
-
-	/* Free-up MPU_PROBE_REGION */
+	/* Free-up PMSAv7_PROBE_REGION */
 	mpu_min_region_order = __mpu_min_region_order();
 
 	/* How many regions are supported */
@@ -301,12 +293,12 @@ void __init adjust_lowmem_bounds_mpu(void)
 	num = allocate_region(mem_start, specified_mem_size, mem_max_regions, mem);
 
 	for (i = 0; i < num; i++) {
-		unsigned long  subreg = mem[i].size / MPU_NR_SUBREGS;
+		unsigned long  subreg = mem[i].size / PMSAv7_NR_SUBREGS;
 
 		total_mem_size += mem[i].size - subreg * hweight_long(mem[i].subreg);
 
 		pr_debug("MPU: base %pa size %pa disable subregions: %*pbl\n",
-			 &mem[i].base, &mem[i].size, MPU_NR_SUBREGS, &mem[i].subreg);
+			 &mem[i].base, &mem[i].size, PMSAv7_NR_SUBREGS, &mem[i].subreg);
 	}
 
 	if (total_mem_size != specified_mem_size) {
@@ -349,7 +341,7 @@ static int __init __mpu_min_region_order(void)
 	u32 drbar_result, irbar_result;
 
 	/* We've kept a region free for this probing */
-	rgnr_write(MPU_PROBE_REGION);
+	rgnr_write(PMSAv7_PROBE_REGION);
 	isb();
 	/*
 	 * As per ARM ARM, write 0xFFFFFFFC to DRBAR to find the minimum
@@ -388,8 +380,8 @@ static int __init mpu_setup_region(unsigned int number, phys_addr_t start,
 		return -ENOMEM;
 
 	/* Writing N to bits 5:1 (RSR_SZ)  specifies region size 2^N+1 */
-	size_data = ((size_order - 1) << MPU_RSR_SZ) | 1 << MPU_RSR_EN;
-	size_data |= subregions << MPU_RSR_SD;
+	size_data = ((size_order - 1) << PMSAv7_RSR_SZ) | 1 << PMSAv7_RSR_EN;
+	size_data |= subregions << PMSAv7_RSR_SD;
 
 	if (need_flush)
 		flush_cache_all();
@@ -424,18 +416,15 @@ static int __init mpu_setup_region(unsigned int number, phys_addr_t start,
 /*
 * Set up default MPU regions, doing nothing if there is no MPU
 */
-void __init mpu_setup(void)
+void __init pmsav7_setup(void)
 {
 	int i, region = 0, err = 0;
 
-	if (!mpu_present())
-		return;
-
 	/* Setup MPU (order is important) */
 
 	/* Background */
 	err |= mpu_setup_region(region++, 0, 32,
-				MPU_ACR_XN | MPU_RGN_STRONGLY_ORDERED | MPU_AP_PL1RW_PL0RW,
+				PMSAv7_ACR_XN | PMSAv7_RGN_STRONGLY_ORDERED | PMSAv7_AP_PL1RW_PL0RW,
 				0, false);
 
 #ifdef CONFIG_XIP_KERNEL
@@ -448,13 +437,13 @@ void __init mpu_setup(void)
                  * with BG region (which is uncachable), thus we need
                  * to clean and invalidate cache.
 		 */
-		bool need_flush = region == MPU_RAM_REGION;
+		bool need_flush = region == PMSAv7_RAM_REGION;
 
 		if (!xip[i].size)
 			continue;
 
 		err |= mpu_setup_region(region++, xip[i].base, ilog2(xip[i].size),
-					MPU_AP_PL1RO_PL0NA | MPU_RGN_NORMAL,
+					PMSAv7_AP_PL1RO_PL0NA | PMSAv7_RGN_NORMAL,
 					xip[i].subreg, need_flush);
 	}
 #endif
@@ -465,14 +454,14 @@ void __init mpu_setup(void)
 			continue;
 
 		err |= mpu_setup_region(region++, mem[i].base, ilog2(mem[i].size),
-					MPU_AP_PL1RW_PL0RW | MPU_RGN_NORMAL,
+					PMSAv7_AP_PL1RW_PL0RW | PMSAv7_RGN_NORMAL,
 					mem[i].subreg, false);
 	}
 
 	/* Vectors */
 #ifndef CONFIG_CPU_V7M
 	err |= mpu_setup_region(region++, vectors_base, ilog2(2 * PAGE_SIZE),
-				MPU_AP_PL1RW_PL0NA | MPU_RGN_NORMAL,
+				PMSAv7_AP_PL1RW_PL0NA | PMSAv7_RGN_NORMAL,
 				0, false);
 #endif
 	if (err) {
diff --git a/arch/arm/mm/pmsa-v8.c b/arch/arm/mm/pmsa-v8.c
new file mode 100644
index 000000000000..617a83def88a
--- /dev/null
+++ b/arch/arm/mm/pmsa-v8.c
@@ -0,0 +1,307 @@
+/*
+ * Based on linux/arch/arm/pmsa-v7.c
+ *
+ * ARM PMSAv8 supporting functions.
+ */
+
+#include <linux/memblock.h>
+#include <linux/range.h>
+
+#include <asm/cp15.h>
+#include <asm/cputype.h>
+#include <asm/mpu.h>
+
+#include <asm/memory.h>
+#include <asm/sections.h>
+
+#include "mm.h"
+
+#ifndef CONFIG_CPU_V7M
+
+#define PRSEL	__ACCESS_CP15(c6, 0, c2, 1)
+#define PRBAR	__ACCESS_CP15(c6, 0, c3, 0)
+#define PRLAR	__ACCESS_CP15(c6, 0, c3, 1)
+
+static inline u32 prlar_read(void)
+{
+	return read_sysreg(PRLAR);
+}
+
+static inline u32 prbar_read(void)
+{
+	return read_sysreg(PRBAR);
+}
+
+static inline void prsel_write(u32 v)
+{
+	write_sysreg(v, PRSEL);
+}
+
+static inline void prbar_write(u32 v)
+{
+	write_sysreg(v, PRBAR);
+}
+
+static inline void prlar_write(u32 v)
+{
+	write_sysreg(v, PRLAR);
+}
+#else
+
+static inline u32 prlar_read(void)
+{
+	return readl_relaxed(BASEADDR_V7M_SCB + PMSAv8_RLAR);
+}
+
+static inline u32 prbar_read(void)
+{
+	return readl_relaxed(BASEADDR_V7M_SCB + PMSAv8_RBAR);
+}
+
+static inline void prsel_write(u32 v)
+{
+	writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv8_RNR);
+}
+
+static inline void prbar_write(u32 v)
+{
+	writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv8_RBAR);
+}
+
+static inline void prlar_write(u32 v)
+{
+	writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv8_RLAR);
+}
+
+#endif
+
+static struct range __initdata io[MPU_MAX_REGIONS];
+static struct range __initdata mem[MPU_MAX_REGIONS];
+
+static unsigned int __initdata mpu_max_regions;
+
+static __init bool is_region_fixed(int number)
+{
+	switch (number) {
+	case PMSAv8_XIP_REGION:
+	case PMSAv8_KERNEL_REGION:
+		return true;
+	default:
+		return false;
+	}
+}
+
+void __init pmsav8_adjust_lowmem_bounds(void)
+{
+	phys_addr_t mem_end;
+	struct memblock_region *reg;
+	bool first = true;
+
+	for_each_memblock(memory, reg) {
+		if (first) {
+			phys_addr_t phys_offset = PHYS_OFFSET;
+
+			/*
+			 * Initially only use memory continuous from
+			 * PHYS_OFFSET */
+			if (reg->base != phys_offset)
+				panic("First memory bank must be contiguous from PHYS_OFFSET");
+			mem_end = reg->base + reg->size;
+			first = false;
+		} else {
+			/*
+			 * memblock auto merges contiguous blocks, remove
+			 * all blocks afterwards in one go (we can't remove
+			 * blocks separately while iterating)
+			 */
+			pr_notice("Ignoring RAM after %pa, memory at %pa ignored\n",
+				  &mem_end, &reg->base);
+			memblock_remove(reg->base, 0 - reg->base);
+			break;
+		}
+	}
+}
+
+static int __init __mpu_max_regions(void)
+{
+	static int max_regions;
+	u32 mpuir;
+
+	if (max_regions)
+		return max_regions;
+
+	mpuir = read_cpuid_mputype();
+
+	max_regions  = (mpuir & MPUIR_DREGION_SZMASK) >> MPUIR_DREGION;
+
+	return max_regions;
+}
+
+static int __init __pmsav8_setup_region(unsigned int number, u32 bar, u32 lar)
+{
+	if (number > mpu_max_regions
+	    || number >= MPU_MAX_REGIONS)
+		return -ENOENT;
+
+	dsb();
+	prsel_write(number);
+	isb();
+	prbar_write(bar);
+	prlar_write(lar);
+
+	mpu_rgn_info.rgns[number].prbar = bar;
+	mpu_rgn_info.rgns[number].prlar = lar;
+
+	mpu_rgn_info.used++;
+
+	return 0;
+}
+
+static int __init pmsav8_setup_ram(unsigned int number, phys_addr_t start,phys_addr_t end)
+{
+	u32 bar, lar;
+
+	if (is_region_fixed(number))
+		return -EINVAL;
+
+	bar = start;
+	lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);;
+
+	bar |= PMSAv8_AP_PL1RW_PL0RW | PMSAv8_RGN_SHARED;
+	lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN;
+
+	return __pmsav8_setup_region(number, bar, lar);
+}
+
+static int __init pmsav8_setup_io(unsigned int number, phys_addr_t start,phys_addr_t end)
+{
+	u32 bar, lar;
+
+	if (is_region_fixed(number))
+		return -EINVAL;
+
+	bar = start;
+	lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);;
+
+	bar |= PMSAv8_AP_PL1RW_PL0RW | PMSAv8_RGN_SHARED | PMSAv8_BAR_XN;
+	lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_DEVICE_nGnRnE) | PMSAv8_LAR_EN;
+
+	return __pmsav8_setup_region(number, bar, lar);
+}
+
+static int __init pmsav8_setup_fixed(unsigned int number, phys_addr_t start,phys_addr_t end)
+{
+	u32 bar, lar;
+
+	if (!is_region_fixed(number))
+		return -EINVAL;
+
+	bar = start;
+	lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);
+
+	bar |= PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED;
+	lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN;
+
+	prsel_write(number);
+	isb();
+
+	if (prbar_read() != bar || prlar_read() != lar)
+		return -EINVAL;
+
+	/* Reserved region was set up early, we just need a record for secondaries */
+	mpu_rgn_info.rgns[number].prbar = bar;
+	mpu_rgn_info.rgns[number].prlar = lar;
+
+	mpu_rgn_info.used++;
+
+	return 0;
+}
+
+#ifndef CONFIG_CPU_V7M
+static int __init pmsav8_setup_vector(unsigned int number, phys_addr_t start,phys_addr_t end)
+{
+	u32 bar, lar;
+
+	if (number == PMSAv8_KERNEL_REGION)
+		return -EINVAL;
+
+	bar = start;
+	lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);
+
+	bar |= PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED;
+	lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN;
+
+	return __pmsav8_setup_region(number, bar, lar);
+}
+#endif
+
+void __init pmsav8_setup(void)
+{
+	int i, err = 0;
+	int region = PMSAv8_KERNEL_REGION;
+
+	/* How many regions are supported ? */
+	mpu_max_regions = __mpu_max_regions();
+
+	/* RAM: single chunk of memory */
+	add_range(mem,  ARRAY_SIZE(mem), 0,  memblock.memory.regions[0].base,
+		  memblock.memory.regions[0].base + memblock.memory.regions[0].size);
+
+	/* IO: cover full 4G range */
+	add_range(io, ARRAY_SIZE(io), 0, 0, 0xffffffff);
+
+	/* RAM and IO: exclude kernel */
+	subtract_range(mem, ARRAY_SIZE(mem), __pa(KERNEL_START), __pa(KERNEL_END));
+	subtract_range(io, ARRAY_SIZE(io),  __pa(KERNEL_START), __pa(KERNEL_END));
+
+#ifdef CONFIG_XIP_KERNEL
+	/* RAM and IO: exclude xip */
+	subtract_range(mem, ARRAY_SIZE(mem), CONFIG_XIP_PHYS_ADDR, __pa(_exiprom));
+	subtract_range(io, ARRAY_SIZE(io), CONFIG_XIP_PHYS_ADDR, __pa(_exiprom));
+#endif
+
+#ifndef CONFIG_CPU_V7M
+	/* RAM and IO: exclude vectors */
+	subtract_range(mem, ARRAY_SIZE(mem),  vectors_base, vectors_base + 2 * PAGE_SIZE);
+	subtract_range(io, ARRAY_SIZE(io),  vectors_base, vectors_base + 2 * PAGE_SIZE);
+#endif
+	/* IO: exclude RAM */
+	for (i = 0; i < ARRAY_SIZE(mem); i++)
+		subtract_range(io, ARRAY_SIZE(io), mem[i].start, mem[i].end);
+
+	/* Now program MPU */
+
+#ifdef CONFIG_XIP_KERNEL
+	/* ROM */
+	err |= pmsav8_setup_fixed(PMSAv8_XIP_REGION, CONFIG_XIP_PHYS_ADDR, __pa(_exiprom));
+#endif
+	/* Kernel */
+	err |= pmsav8_setup_fixed(region++, __pa(KERNEL_START), __pa(KERNEL_END));
+
+
+	/* IO */
+	for (i = 0; i < ARRAY_SIZE(io); i++) {
+		if (!io[i].end)
+			continue;
+
+		err |= pmsav8_setup_io(region++, io[i].start, io[i].end);
+	}
+
+	/* RAM */
+	for (i = 0; i < ARRAY_SIZE(mem); i++) {
+		if (!mem[i].end)
+			continue;
+
+		err |= pmsav8_setup_ram(region++, mem[i].start, mem[i].end);
+	}
+
+	/* Vectors */
+#ifndef CONFIG_CPU_V7M
+	err |= pmsav8_setup_vector(region++, vectors_base, vectors_base + 2 * PAGE_SIZE);
+#endif
+	if (err)
+		pr_warn("MPU region initialization failure! %d", err);
+	else
+		pr_info("Using ARM PMSAv8 Compliant MPU. Used %d of %d regions\n",
+			mpu_rgn_info.used, mpu_max_regions);
+}
diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S
index f10e31d0730a..81d0efb055c6 100644
--- a/arch/arm/mm/proc-macros.S
+++ b/arch/arm/mm/proc-macros.S
@@ -273,13 +273,14 @@
 	mcr	p15, 0, ip, c7, c10, 4		@ data write barrier
 	.endm
 
-.macro define_processor_functions name:req, dabort:req, pabort:req, nommu=0, suspend=0
+.macro define_processor_functions name:req, dabort:req, pabort:req, nommu=0, suspend=0, bugs=0
 	.type	\name\()_processor_functions, #object
 	.align 2
 ENTRY(\name\()_processor_functions)
 	.word	\dabort
 	.word	\pabort
 	.word	cpu_\name\()_proc_init
+	.word	\bugs
 	.word	cpu_\name\()_proc_fin
 	.word	cpu_\name\()_reset
 	.word	cpu_\name\()_do_idle
diff --git a/arch/arm/mm/proc-v7-2level.S b/arch/arm/mm/proc-v7-2level.S
index c6141a5435c3..f8d45ad2a515 100644
--- a/arch/arm/mm/proc-v7-2level.S
+++ b/arch/arm/mm/proc-v7-2level.S
@@ -41,11 +41,6 @@
  *	even on Cortex-A8 revisions not affected by 430973.
  *	If IBE is not set, the flush BTAC/BTB won't do anything.
  */
-ENTRY(cpu_ca8_switch_mm)
-#ifdef CONFIG_MMU
-	mov	r2, #0
-	mcr	p15, 0, r2, c7, c5, 6		@ flush BTAC/BTB
-#endif
 ENTRY(cpu_v7_switch_mm)
 #ifdef CONFIG_MMU
 	mmid	r1, r1				@ get mm->context.id
@@ -66,7 +61,6 @@ ENTRY(cpu_v7_switch_mm)
 #endif
 	bx	lr
 ENDPROC(cpu_v7_switch_mm)
-ENDPROC(cpu_ca8_switch_mm)
 
 /*
  *	cpu_v7_set_pte_ext(ptep, pte)
diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c
new file mode 100644
index 000000000000..5ba1f94e58a8
--- /dev/null
+++ b/arch/arm/mm/proc-v7-bugs.c
@@ -0,0 +1,176 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/arm-smccc.h>
+#include <linux/kernel.h>
+#include <linux/psci.h>
+#include <linux/smp.h>
+
+#include <asm/cp15.h>
+#include <asm/cputype.h>
+#include <asm/proc-fns.h>
+#include <asm/system_misc.h>
+
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn);
+
+extern void cpu_v7_iciallu_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
+extern void cpu_v7_bpiall_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
+extern void cpu_v7_smc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
+extern void cpu_v7_hvc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
+
+static void harden_branch_predictor_bpiall(void)
+{
+	write_sysreg(0, BPIALL);
+}
+
+static void harden_branch_predictor_iciallu(void)
+{
+	write_sysreg(0, ICIALLU);
+}
+
+static void __maybe_unused call_smc_arch_workaround_1(void)
+{
+	arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
+}
+
+static void __maybe_unused call_hvc_arch_workaround_1(void)
+{
+	arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
+}
+
+static void cpu_v7_spectre_init(void)
+{
+	const char *spectre_v2_method = NULL;
+	int cpu = smp_processor_id();
+	u32 cpuid;
+
+	if (per_cpu(harden_branch_predictor_fn, cpu))
+		return;
+
+	cpuid = read_cpuid_part();
+	switch (cpuid) {
+	case ARM_CPU_PART_CORTEX_A8:
+	case ARM_CPU_PART_CORTEX_A9:
+	case ARM_CPU_PART_CORTEX_A12:
+	case ARM_CPU_PART_CORTEX_A17:
+	case ARM_CPU_PART_CORTEX_A73:
+	case ARM_CPU_PART_CORTEX_A75:
+		if (processor.switch_mm != cpu_v7_bpiall_switch_mm)
+			goto bl_error;
+		per_cpu(harden_branch_predictor_fn, cpu) =
+			harden_branch_predictor_bpiall;
+		spectre_v2_method = "BPIALL";
+		break;
+
+	case ARM_CPU_PART_CORTEX_A15:
+	case ARM_CPU_PART_BRAHMA_B15:
+		if (processor.switch_mm != cpu_v7_iciallu_switch_mm)
+			goto bl_error;
+		per_cpu(harden_branch_predictor_fn, cpu) =
+			harden_branch_predictor_iciallu;
+		spectre_v2_method = "ICIALLU";
+		break;
+
+#ifdef CONFIG_ARM_PSCI
+	default:
+		/* Other ARM CPUs require no workaround */
+		if (cpuid >> 24 == ARM_CPU_IMP_ARM)
+			break;
+		/* fallthrough */
+		/* Cortex A57/A72 require firmware workaround */
+	case ARM_CPU_PART_CORTEX_A57:
+	case ARM_CPU_PART_CORTEX_A72: {
+		struct arm_smccc_res res;
+
+		if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
+			break;
+
+		switch (psci_ops.conduit) {
+		case PSCI_CONDUIT_HVC:
+			arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
+					  ARM_SMCCC_ARCH_WORKAROUND_1, &res);
+			if ((int)res.a0 != 0)
+				break;
+			if (processor.switch_mm != cpu_v7_hvc_switch_mm && cpu)
+				goto bl_error;
+			per_cpu(harden_branch_predictor_fn, cpu) =
+				call_hvc_arch_workaround_1;
+			processor.switch_mm = cpu_v7_hvc_switch_mm;
+			spectre_v2_method = "hypervisor";
+			break;
+
+		case PSCI_CONDUIT_SMC:
+			arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
+					  ARM_SMCCC_ARCH_WORKAROUND_1, &res);
+			if ((int)res.a0 != 0)
+				break;
+			if (processor.switch_mm != cpu_v7_smc_switch_mm && cpu)
+				goto bl_error;
+			per_cpu(harden_branch_predictor_fn, cpu) =
+				call_smc_arch_workaround_1;
+			processor.switch_mm = cpu_v7_smc_switch_mm;
+			spectre_v2_method = "firmware";
+			break;
+
+		default:
+			break;
+		}
+	}
+#endif
+	}
+
+	if (spectre_v2_method)
+		pr_info("CPU%u: Spectre v2: using %s workaround\n",
+			smp_processor_id(), spectre_v2_method);
+	return;
+
+bl_error:
+	pr_err("CPU%u: Spectre v2: incorrect context switching function, system vulnerable\n",
+		cpu);
+}
+#else
+static void cpu_v7_spectre_init(void)
+{
+}
+#endif
+
+static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned,
+						  u32 mask, const char *msg)
+{
+	u32 aux_cr;
+
+	asm("mrc p15, 0, %0, c1, c0, 1" : "=r" (aux_cr));
+
+	if ((aux_cr & mask) != mask) {
+		if (!*warned)
+			pr_err("CPU%u: %s", smp_processor_id(), msg);
+		*warned = true;
+		return false;
+	}
+	return true;
+}
+
+static DEFINE_PER_CPU(bool, spectre_warned);
+
+static bool check_spectre_auxcr(bool *warned, u32 bit)
+{
+	return IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR) &&
+		cpu_v7_check_auxcr_set(warned, bit,
+				       "Spectre v2: firmware did not set auxiliary control register IBE bit, system vulnerable\n");
+}
+
+void cpu_v7_ca8_ibe(void)
+{
+	if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6)))
+		cpu_v7_spectre_init();
+}
+
+void cpu_v7_ca15_ibe(void)
+{
+	if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0)))
+		cpu_v7_spectre_init();
+}
+
+void cpu_v7_bugs_init(void)
+{
+	cpu_v7_spectre_init();
+}
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index b528a15f460d..6fe52819e014 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -9,6 +9,7 @@
  *
  *  This is the "shell" of the ARMv7 processor support.
  */
+#include <linux/arm-smccc.h>
 #include <linux/init.h>
 #include <linux/linkage.h>
 #include <asm/assembler.h>
@@ -93,6 +94,37 @@ ENTRY(cpu_v7_dcache_clean_area)
 	ret	lr
 ENDPROC(cpu_v7_dcache_clean_area)
 
+#ifdef CONFIG_ARM_PSCI
+	.arch_extension sec
+ENTRY(cpu_v7_smc_switch_mm)
+	stmfd	sp!, {r0 - r3}
+	movw	r0, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1
+	movt	r0, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1
+	smc	#0
+	ldmfd	sp!, {r0 - r3}
+	b	cpu_v7_switch_mm
+ENDPROC(cpu_v7_smc_switch_mm)
+	.arch_extension virt
+ENTRY(cpu_v7_hvc_switch_mm)
+	stmfd	sp!, {r0 - r3}
+	movw	r0, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1
+	movt	r0, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1
+	hvc	#0
+	ldmfd	sp!, {r0 - r3}
+	b	cpu_v7_switch_mm
+ENDPROC(cpu_v7_smc_switch_mm)
+#endif
+ENTRY(cpu_v7_iciallu_switch_mm)
+	mov	r3, #0
+	mcr	p15, 0, r3, c7, c5, 0		@ ICIALLU
+	b	cpu_v7_switch_mm
+ENDPROC(cpu_v7_iciallu_switch_mm)
+ENTRY(cpu_v7_bpiall_switch_mm)
+	mov	r3, #0
+	mcr	p15, 0, r3, c7, c5, 6		@ flush BTAC/BTB
+	b	cpu_v7_switch_mm
+ENDPROC(cpu_v7_bpiall_switch_mm)
+
 	string	cpu_v7_name, "ARMv7 Processor"
 	.align
 
@@ -158,31 +190,6 @@ ENTRY(cpu_v7_do_resume)
 ENDPROC(cpu_v7_do_resume)
 #endif
 
-/*
- * Cortex-A8
- */
-	globl_equ	cpu_ca8_proc_init,	cpu_v7_proc_init
-	globl_equ	cpu_ca8_proc_fin,	cpu_v7_proc_fin
-	globl_equ	cpu_ca8_reset,		cpu_v7_reset
-	globl_equ	cpu_ca8_do_idle,	cpu_v7_do_idle
-	globl_equ	cpu_ca8_dcache_clean_area, cpu_v7_dcache_clean_area
-	globl_equ	cpu_ca8_set_pte_ext,	cpu_v7_set_pte_ext
-	globl_equ	cpu_ca8_suspend_size,	cpu_v7_suspend_size
-#ifdef CONFIG_ARM_CPU_SUSPEND
-	globl_equ	cpu_ca8_do_suspend,	cpu_v7_do_suspend
-	globl_equ	cpu_ca8_do_resume,	cpu_v7_do_resume
-#endif
-
-/*
- * Cortex-A9 processor functions
- */
-	globl_equ	cpu_ca9mp_proc_init,	cpu_v7_proc_init
-	globl_equ	cpu_ca9mp_proc_fin,	cpu_v7_proc_fin
-	globl_equ	cpu_ca9mp_reset,	cpu_v7_reset
-	globl_equ	cpu_ca9mp_do_idle,	cpu_v7_do_idle
-	globl_equ	cpu_ca9mp_dcache_clean_area, cpu_v7_dcache_clean_area
-	globl_equ	cpu_ca9mp_switch_mm,	cpu_v7_switch_mm
-	globl_equ	cpu_ca9mp_set_pte_ext,	cpu_v7_set_pte_ext
 .globl	cpu_ca9mp_suspend_size
 .equ	cpu_ca9mp_suspend_size, cpu_v7_suspend_size + 4 * 2
 #ifdef CONFIG_ARM_CPU_SUSPEND
@@ -547,12 +554,79 @@ __v7_setup_stack:
 
 	__INITDATA
 
+	.weak cpu_v7_bugs_init
+
 	@ define struct processor (see <asm/proc-fns.h> and proc-macros.S)
-	define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
+	define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init
+
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+	@ generic v7 bpiall on context switch
+	globl_equ	cpu_v7_bpiall_proc_init,	cpu_v7_proc_init
+	globl_equ	cpu_v7_bpiall_proc_fin,		cpu_v7_proc_fin
+	globl_equ	cpu_v7_bpiall_reset,		cpu_v7_reset
+	globl_equ	cpu_v7_bpiall_do_idle,		cpu_v7_do_idle
+	globl_equ	cpu_v7_bpiall_dcache_clean_area, cpu_v7_dcache_clean_area
+	globl_equ	cpu_v7_bpiall_set_pte_ext,	cpu_v7_set_pte_ext
+	globl_equ	cpu_v7_bpiall_suspend_size,	cpu_v7_suspend_size
+#ifdef CONFIG_ARM_CPU_SUSPEND
+	globl_equ	cpu_v7_bpiall_do_suspend,	cpu_v7_do_suspend
+	globl_equ	cpu_v7_bpiall_do_resume,	cpu_v7_do_resume
+#endif
+	define_processor_functions v7_bpiall, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init
+
+#define HARDENED_BPIALL_PROCESSOR_FUNCTIONS v7_bpiall_processor_functions
+#else
+#define HARDENED_BPIALL_PROCESSOR_FUNCTIONS v7_processor_functions
+#endif
+
 #ifndef CONFIG_ARM_LPAE
-	define_processor_functions ca8, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
-	define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
+	@ Cortex-A8 - always needs bpiall switch_mm implementation
+	globl_equ	cpu_ca8_proc_init,	cpu_v7_proc_init
+	globl_equ	cpu_ca8_proc_fin,	cpu_v7_proc_fin
+	globl_equ	cpu_ca8_reset,		cpu_v7_reset
+	globl_equ	cpu_ca8_do_idle,	cpu_v7_do_idle
+	globl_equ	cpu_ca8_dcache_clean_area, cpu_v7_dcache_clean_area
+	globl_equ	cpu_ca8_set_pte_ext,	cpu_v7_set_pte_ext
+	globl_equ	cpu_ca8_switch_mm,	cpu_v7_bpiall_switch_mm
+	globl_equ	cpu_ca8_suspend_size,	cpu_v7_suspend_size
+#ifdef CONFIG_ARM_CPU_SUSPEND
+	globl_equ	cpu_ca8_do_suspend,	cpu_v7_do_suspend
+	globl_equ	cpu_ca8_do_resume,	cpu_v7_do_resume
+#endif
+	define_processor_functions ca8, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_ca8_ibe
+
+	@ Cortex-A9 - needs more registers preserved across suspend/resume
+	@ and bpiall switch_mm for hardening
+	globl_equ	cpu_ca9mp_proc_init,	cpu_v7_proc_init
+	globl_equ	cpu_ca9mp_proc_fin,	cpu_v7_proc_fin
+	globl_equ	cpu_ca9mp_reset,	cpu_v7_reset
+	globl_equ	cpu_ca9mp_do_idle,	cpu_v7_do_idle
+	globl_equ	cpu_ca9mp_dcache_clean_area, cpu_v7_dcache_clean_area
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+	globl_equ	cpu_ca9mp_switch_mm,	cpu_v7_bpiall_switch_mm
+#else
+	globl_equ	cpu_ca9mp_switch_mm,	cpu_v7_switch_mm
+#endif
+	globl_equ	cpu_ca9mp_set_pte_ext,	cpu_v7_set_pte_ext
+	define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init
 #endif
+
+	@ Cortex-A15 - needs iciallu switch_mm for hardening
+	globl_equ	cpu_ca15_proc_init,	cpu_v7_proc_init
+	globl_equ	cpu_ca15_proc_fin,	cpu_v7_proc_fin
+	globl_equ	cpu_ca15_reset,		cpu_v7_reset
+	globl_equ	cpu_ca15_do_idle,	cpu_v7_do_idle
+	globl_equ	cpu_ca15_dcache_clean_area, cpu_v7_dcache_clean_area
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+	globl_equ	cpu_ca15_switch_mm,	cpu_v7_iciallu_switch_mm
+#else
+	globl_equ	cpu_ca15_switch_mm,	cpu_v7_switch_mm
+#endif
+	globl_equ	cpu_ca15_set_pte_ext,	cpu_v7_set_pte_ext
+	globl_equ	cpu_ca15_suspend_size,	cpu_v7_suspend_size
+	globl_equ	cpu_ca15_do_suspend,	cpu_v7_do_suspend
+	globl_equ	cpu_ca15_do_resume,	cpu_v7_do_resume
+	define_processor_functions ca15, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_ca15_ibe
 #ifdef CONFIG_CPU_PJ4B
 	define_processor_functions pj4b, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
 #endif
@@ -669,7 +743,7 @@ __v7_ca7mp_proc_info:
 __v7_ca12mp_proc_info:
 	.long	0x410fc0d0
 	.long	0xff0ffff0
-	__v7_proc __v7_ca12mp_proc_info, __v7_ca12mp_setup
+	__v7_proc __v7_ca12mp_proc_info, __v7_ca12mp_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
 	.size	__v7_ca12mp_proc_info, . - __v7_ca12mp_proc_info
 
 	/*
@@ -679,7 +753,7 @@ __v7_ca12mp_proc_info:
 __v7_ca15mp_proc_info:
 	.long	0x410fc0f0
 	.long	0xff0ffff0
-	__v7_proc __v7_ca15mp_proc_info, __v7_ca15mp_setup
+	__v7_proc __v7_ca15mp_proc_info, __v7_ca15mp_setup, proc_fns = ca15_processor_functions
 	.size	__v7_ca15mp_proc_info, . - __v7_ca15mp_proc_info
 
 	/*
@@ -689,7 +763,7 @@ __v7_ca15mp_proc_info:
 __v7_b15mp_proc_info:
 	.long	0x420f00f0
 	.long	0xff0ffff0
-	__v7_proc __v7_b15mp_proc_info, __v7_b15mp_setup, cache_fns = b15_cache_fns
+	__v7_proc __v7_b15mp_proc_info, __v7_b15mp_setup, proc_fns = ca15_processor_functions, cache_fns = b15_cache_fns
 	.size	__v7_b15mp_proc_info, . - __v7_b15mp_proc_info
 
 	/*
@@ -699,9 +773,25 @@ __v7_b15mp_proc_info:
 __v7_ca17mp_proc_info:
 	.long	0x410fc0e0
 	.long	0xff0ffff0
-	__v7_proc __v7_ca17mp_proc_info, __v7_ca17mp_setup
+	__v7_proc __v7_ca17mp_proc_info, __v7_ca17mp_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
 	.size	__v7_ca17mp_proc_info, . - __v7_ca17mp_proc_info
 
+	/* ARM Ltd. Cortex A73 processor */
+	.type	__v7_ca73_proc_info, #object
+__v7_ca73_proc_info:
+	.long	0x410fd090
+	.long	0xff0ffff0
+	__v7_proc __v7_ca73_proc_info, __v7_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
+	.size	__v7_ca73_proc_info, . - __v7_ca73_proc_info
+
+	/* ARM Ltd. Cortex A75 processor */
+	.type	__v7_ca75_proc_info, #object
+__v7_ca75_proc_info:
+	.long	0x410fd0a0
+	.long	0xff0ffff0
+	__v7_proc __v7_ca75_proc_info, __v7_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
+	.size	__v7_ca75_proc_info, . - __v7_ca75_proc_info
+
 	/*
 	 * Qualcomm Inc. Krait processors.
 	 */
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index b5030e1a41d8..d3ea6454e775 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -234,18 +234,11 @@ static void jit_fill_hole(void *area, unsigned int size)
 #define SCRATCH_SIZE 80
 
 /* total stack size used in JITed code */
-#define _STACK_SIZE \
-	(ctx->prog->aux->stack_depth + \
-	 + SCRATCH_SIZE + \
-	 + 4 /* extra for skb_copy_bits buffer */)
-
-#define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT)
+#define _STACK_SIZE	(ctx->prog->aux->stack_depth + SCRATCH_SIZE)
+#define STACK_SIZE	ALIGN(_STACK_SIZE, STACK_ALIGNMENT)
 
 /* Get the offset of eBPF REGISTERs stored on scratch space. */
-#define STACK_VAR(off) (STACK_SIZE-off-4)
-
-/* Offset of skb_copy_bits buffer */
-#define SKB_BUFFER STACK_VAR(SCRATCH_SIZE)
+#define STACK_VAR(off) (STACK_SIZE - off)
 
 #if __LINUX_ARM_ARCH__ < 7
 
@@ -1452,83 +1445,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 			emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
 		emit_ldx_r(dst, rn, dstk, off, ctx, BPF_SIZE(code));
 		break;
-	/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
-	case BPF_LD | BPF_ABS | BPF_W:
-	case BPF_LD | BPF_ABS | BPF_H:
-	case BPF_LD | BPF_ABS | BPF_B:
-	/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
-	case BPF_LD | BPF_IND | BPF_W:
-	case BPF_LD | BPF_IND | BPF_H:
-	case BPF_LD | BPF_IND | BPF_B:
-	{
-		const u8 r4 = bpf2a32[BPF_REG_6][1]; /* r4 = ptr to sk_buff */
-		const u8 r0 = bpf2a32[BPF_REG_0][1]; /*r0: struct sk_buff *skb*/
-						     /* rtn value */
-		const u8 r1 = bpf2a32[BPF_REG_0][0]; /* r1: int k */
-		const u8 r2 = bpf2a32[BPF_REG_1][1]; /* r2: unsigned int size */
-		const u8 r3 = bpf2a32[BPF_REG_1][0]; /* r3: void *buffer */
-		const u8 r6 = bpf2a32[TMP_REG_1][1]; /* r6: void *(*func)(..) */
-		int size;
-
-		/* Setting up first argument */
-		emit(ARM_MOV_R(r0, r4), ctx);
-
-		/* Setting up second argument */
-		emit_a32_mov_i(r1, imm, false, ctx);
-		if (BPF_MODE(code) == BPF_IND)
-			emit_a32_alu_r(r1, src_lo, false, sstk, ctx,
-				       false, false, BPF_ADD);
-
-		/* Setting up third argument */
-		switch (BPF_SIZE(code)) {
-		case BPF_W:
-			size = 4;
-			break;
-		case BPF_H:
-			size = 2;
-			break;
-		case BPF_B:
-			size = 1;
-			break;
-		default:
-			return -EINVAL;
-		}
-		emit_a32_mov_i(r2, size, false, ctx);
-
-		/* Setting up fourth argument */
-		emit(ARM_ADD_I(r3, ARM_SP, imm8m(SKB_BUFFER)), ctx);
-
-		/* Setting up function pointer to call */
-		emit_a32_mov_i(r6, (unsigned int)bpf_load_pointer, false, ctx);
-		emit_blx_r(r6, ctx);
-
-		emit(ARM_EOR_R(r1, r1, r1), ctx);
-		/* Check if return address is NULL or not.
-		 * if NULL then jump to epilogue
-		 * else continue to load the value from retn address
-		 */
-		emit(ARM_CMP_I(r0, 0), ctx);
-		jmp_offset = epilogue_offset(ctx);
-		check_imm24(jmp_offset);
-		_emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
-
-		/* Load value from the address */
-		switch (BPF_SIZE(code)) {
-		case BPF_W:
-			emit(ARM_LDR_I(r0, r0, 0), ctx);
-			emit_rev32(r0, r0, ctx);
-			break;
-		case BPF_H:
-			emit(ARM_LDRH_I(r0, r0, 0), ctx);
-			emit_rev16(r0, r0, ctx);
-			break;
-		case BPF_B:
-			emit(ARM_LDRB_I(r0, r0, 0), ctx);
-			/* No need to reverse */
-			break;
-		}
-		break;
-	}
 	/* ST: *(size *)(dst + off) = imm */
 	case BPF_ST | BPF_MEM | BPF_W:
 	case BPF_ST | BPF_MEM | BPF_H:
diff --git a/arch/arm/plat-omap/Kconfig b/arch/arm/plat-omap/Kconfig
index afc1a1d4f7a5..c0a242cae79a 100644
--- a/arch/arm/plat-omap/Kconfig
+++ b/arch/arm/plat-omap/Kconfig
@@ -115,16 +115,6 @@ config OMAP_SERIAL_WAKE
 	  to data on the serial RX line. This allows you to wake the
 	  system from serial console.
 
-choice
-	prompt "OMAP PM layer selection"
-	depends on ARCH_OMAP
-	default OMAP_PM_NOOP
-
-config OMAP_PM_NOOP
-	bool "No-op/debug PM layer"
-
-endchoice
-
 endmenu
 
 endif
diff --git a/arch/arm/plat-samsung/adc.c b/arch/arm/plat-samsung/adc.c
index 42bac8d5ab5d..2da35735fa38 100644
--- a/arch/arm/plat-samsung/adc.c
+++ b/arch/arm/plat-samsung/adc.c
@@ -413,8 +413,7 @@ static int s3c_adc_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM
 static int s3c_adc_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct adc_device *adc = platform_get_drvdata(pdev);
+	struct adc_device *adc = dev_get_drvdata(dev);
 	unsigned long flags;
 	u32 con;
 
diff --git a/arch/arm/plat-samsung/include/plat/map-s5p.h b/arch/arm/plat-samsung/include/plat/map-s5p.h
index f5769e93544a..d69a0ca09fb5 100644
--- a/arch/arm/plat-samsung/include/plat/map-s5p.h
+++ b/arch/arm/plat-samsung/include/plat/map-s5p.h
@@ -11,10 +11,6 @@
 
 #define S5P_VA_CHIPID		S3C_ADDR(0x02000000)
 
-#define S5P_VA_COREPERI_BASE	S3C_ADDR(0x02800000)
-#define S5P_VA_COREPERI(x)	(S5P_VA_COREPERI_BASE + (x))
-#define S5P_VA_SCU		S5P_VA_COREPERI(0x0)
-
 #define VA_VIC(x)		(S3C_VA_IRQ + ((x) * 0x10000))
 #define VA_VIC0			VA_VIC(0)
 #define VA_VIC1			VA_VIC(1)
diff --git a/arch/arm/vdso/Makefile b/arch/arm/vdso/Makefile
index bb4118213fee..f4efff9d3afb 100644
--- a/arch/arm/vdso/Makefile
+++ b/arch/arm/vdso/Makefile
@@ -30,6 +30,9 @@ CFLAGS_vgettimeofday.o = -O2
 # Disable gcov profiling for VDSO code
 GCOV_PROFILE := n
 
+# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in.
+KCOV_INSTRUMENT := n
+
 # Force dependency
 $(obj)/vdso.o : $(obj)/vdso.so
 
diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c
index af4ee2cef2f9..35d0f823e823 100644
--- a/arch/arm/vfp/vfpmodule.c
+++ b/arch/arm/vfp/vfpmodule.c
@@ -218,8 +218,7 @@ static void vfp_raise_sigfpe(unsigned int sicode, struct pt_regs *regs)
 {
 	siginfo_t info;
 
-	memset(&info, 0, sizeof(info));
-
+	clear_siginfo(&info);
 	info.si_signo = SIGFPE;
 	info.si_code = sicode;
 	info.si_addr = (void __user *)(instruction_pointer(regs) - 4);
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index eb2cf4938f6d..6eea2c05b020 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -7,16 +7,19 @@ config ARM64
 	select ACPI_REDUCED_HARDWARE_ONLY if ACPI
 	select ACPI_MCFG if ACPI
 	select ACPI_SPCR_TABLE if ACPI
+	select ACPI_PPTT if ACPI
 	select ARCH_CLOCKSOURCE_DATA
 	select ARCH_HAS_DEBUG_VIRTUAL
 	select ARCH_HAS_DEVMEM_IS_ALLOWED
 	select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI
 	select ARCH_HAS_ELF_RANDOMIZE
+	select ARCH_HAS_FAST_MULTIPLIER
 	select ARCH_HAS_FORTIFY_SOURCE
 	select ARCH_HAS_GCOV_PROFILE_ALL
 	select ARCH_HAS_GIGANTIC_PAGE if (MEMORY_ISOLATION && COMPACTION) || CMA
 	select ARCH_HAS_KCOV
 	select ARCH_HAS_MEMBARRIER_SYNC_CORE
+	select ARCH_HAS_PTE_SPECIAL
 	select ARCH_HAS_SET_MEMORY
 	select ARCH_HAS_SG_CHAIN
 	select ARCH_HAS_STRICT_KERNEL_RWX
@@ -105,7 +108,6 @@ config ARM64
 	select HAVE_CONTEXT_TRACKING
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_DEBUG_KMEMLEAK
-	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_CONTIGUOUS
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_EFFICIENT_UNALIGNED_ACCESS
@@ -133,6 +135,8 @@ config ARM64
 	select IRQ_FORCED_THREADING
 	select MODULES_USE_ELF_RELA
 	select MULTI_IRQ_HANDLER
+	select NEED_DMA_MAP_STATE
+	select NEED_SG_DMA_LENGTH
 	select NO_BOOTMEM
 	select OF
 	select OF_EARLY_FLATTREE
@@ -142,6 +146,7 @@ config ARM64
 	select POWER_SUPPLY
 	select REFCOUNT_FULL
 	select SPARSE_IRQ
+	select SWIOTLB
 	select SYSCTL_EXCEPTION_TRACE
 	select THREAD_INFO_IN_TASK
 	help
@@ -150,9 +155,6 @@ config ARM64
 config 64BIT
 	def_bool y
 
-config ARCH_PHYS_ADDR_T_64BIT
-	def_bool y
-
 config MMU
 	def_bool y
 
@@ -237,24 +239,9 @@ config ZONE_DMA32
 config HAVE_GENERIC_GUP
 	def_bool y
 
-config ARCH_DMA_ADDR_T_64BIT
-	def_bool y
-
-config NEED_DMA_MAP_STATE
-	def_bool y
-
-config NEED_SG_DMA_LENGTH
-	def_bool y
-
 config SMP
 	def_bool y
 
-config SWIOTLB
-	def_bool y
-
-config IOMMU_HELPER
-	def_bool SWIOTLB
-
 config KERNEL_MODE_NEON
 	def_bool y
 
@@ -1130,6 +1117,7 @@ endmenu
 config ARM64_SVE
 	bool "ARM Scalable Vector Extension support"
 	default y
+	depends on !KVM || ARM64_VHE
 	help
 	  The Scalable Vector Extension (SVE) is an extension to the AArch64
 	  execution state which complements and extends the SIMD functionality
@@ -1155,6 +1143,12 @@ config ARM64_SVE
 	  booting the kernel.  If unsure and you are not observing these
 	  symptoms, you should assume that it is safe to say Y.
 
+	  CPUs that support SVE are architecturally required to support the
+	  Virtualization Host Extensions (VHE), so the kernel makes no
+	  provision for supporting SVE alongside KVM without VHE enabled.
+	  Thus, you will need to enable CONFIG_ARM64_VHE if you want to support
+	  KVM in the same kernel image.
+
 config ARM64_MODULE_PLTS
 	bool
 	select HAVE_MOD_ARCH_SPECIFIC
diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms
index 2b1535cdeb7c..d5aeac351fc3 100644
--- a/arch/arm64/Kconfig.platforms
+++ b/arch/arm64/Kconfig.platforms
@@ -208,6 +208,12 @@ config ARCH_R8A77980
 	help
 	  This enables support for the Renesas R-Car V3H SoC.
 
+config ARCH_R8A77990
+	bool "Renesas R-Car E3 SoC Platform"
+	depends on ARCH_RENESAS
+	help
+	  This enables support for the Renesas R-Car E3 SoC.
+
 config ARCH_R8A77995
 	bool "Renesas R-Car D3 SoC Platform"
 	depends on ARCH_RENESAS
diff --git a/arch/arm64/boot/dts/Makefile b/arch/arm64/boot/dts/Makefile
index 4aa50b9b26bc..3543bc324553 100644
--- a/arch/arm64/boot/dts/Makefile
+++ b/arch/arm64/boot/dts/Makefile
@@ -22,5 +22,6 @@ subdir-y += renesas
 subdir-y += rockchip
 subdir-y += socionext
 subdir-y += sprd
+subdir-y += synaptics
 subdir-y += xilinx
 subdir-y += zte
diff --git a/arch/arm64/boot/dts/allwinner/Makefile b/arch/arm64/boot/dts/allwinner/Makefile
index 8bebe7da5ed9..67ce8c500b2e 100644
--- a/arch/arm64/boot/dts/allwinner/Makefile
+++ b/arch/arm64/boot/dts/allwinner/Makefile
@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
+dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a64-amarula-relic.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a64-bananapi-m64.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a64-nanopi-a64.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a64-olinuxino.dtb
@@ -6,10 +7,11 @@ dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a64-orangepi-win.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a64-pine64-plus.dtb sun50i-a64-pine64.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a64-sopine-baseboard.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a64-teres-i.dtb
+dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-libretech-all-h3-cc.dtb
+dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-nanopi-neo2.dtb
+dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-nanopi-neo-plus2.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-orangepi-pc2.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-orangepi-prime.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-orangepi-zero-plus.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-orangepi-zero-plus2.dtb
-dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-nanopi-neo2.dtb
-dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-nanopi-neo-plus2.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-pine-h64.dtb
diff --git a/arch/arm64/boot/dts/allwinner/axp803.dtsi b/arch/arm64/boot/dts/allwinner/axp803.dtsi
index ff8af52743ff..e5eae8bafc42 100644
--- a/arch/arm64/boot/dts/allwinner/axp803.dtsi
+++ b/arch/arm64/boot/dts/allwinner/axp803.dtsi
@@ -146,5 +146,10 @@
 			regulator-max-microvolt = <3000000>;
 			regulator-name = "rtc-ldo";
 		};
+
+		reg_drivevbus: drivevbus {
+			regulator-name = "drivevbus";
+			status = "disabled";
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-amarula-relic.dts b/arch/arm64/boot/dts/allwinner/sun50i-a64-amarula-relic.dts
new file mode 100644
index 000000000000..ce4a256ff086
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-amarula-relic.dts
@@ -0,0 +1,188 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2018 Amarula Solutions B.V.
+ * Author: Jagan Teki <jagan@amarulasolutions.com>
+ */
+
+/dts-v1/;
+
+#include "sun50i-a64.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "Amarula A64-Relic";
+	compatible = "amarula,a64-relic", "allwinner,sun50i-a64";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+};
+
+&ehci0 {
+	status = "okay";
+};
+
+&mmc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc2_pins>;
+	vmmc-supply = <&reg_dcdc1>;
+	bus-width = <8>;
+	non-removable;
+	cap-mmc-hw-reset;
+	status = "okay";
+};
+
+&ohci0 {
+	status = "okay";
+};
+
+&r_rsb {
+	status = "okay";
+
+	axp803: pmic@3a3 {
+		compatible = "x-powers,axp803";
+		reg = <0x3a3>;
+		interrupt-parent = <&r_intc>;
+		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+		x-powers,drive-vbus-en; /* set N_VBUSEN as output pin */
+	};
+};
+
+#include "axp803.dtsi"
+
+&reg_aldo1 {
+	regulator-always-on;
+	regulator-min-microvolt = <2800000>;
+	regulator-max-microvolt = <2800000>;
+	regulator-name = "avdd-csi";
+};
+
+&reg_aldo2 {
+	regulator-always-on;
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-pl";
+};
+
+&reg_aldo3 {
+	regulator-always-on;
+	regulator-min-microvolt = <3000000>;
+	regulator-max-microvolt = <3000000>;
+	regulator-name = "vcc-pll-avcc";
+};
+
+&reg_dcdc1 {
+	regulator-always-on;
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-3v3";
+};
+
+&reg_dcdc2 {
+	regulator-always-on;
+	regulator-min-microvolt = <1040000>;
+	regulator-max-microvolt = <1300000>;
+	regulator-name = "vdd-cpux";
+};
+
+/* DCDC3 is polyphased with DCDC2 */
+
+&reg_dcdc5 {
+	regulator-always-on;
+	regulator-min-microvolt = <1500000>;
+	regulator-max-microvolt = <1500000>;
+	regulator-name = "vcc-dram";
+};
+
+&reg_dcdc6 {
+	regulator-always-on;
+	regulator-min-microvolt = <1100000>;
+	regulator-max-microvolt = <1100000>;
+	regulator-name = "vdd-sys";
+};
+
+&reg_dldo1 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-hdmi-dsi-sensor";
+};
+
+&reg_dldo2 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-mipi";
+};
+
+&reg_dldo3 {
+	regulator-min-microvolt = <2800000>;
+	regulator-max-microvolt = <2800000>;
+	regulator-name = "dovdd-csi";
+};
+
+&reg_dldo4 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-wifi-io";
+};
+
+&reg_drivevbus {
+	regulator-name = "usb0-vbus";
+	status = "okay";
+};
+
+&reg_eldo1 {
+	regulator-always-on;
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "cpvdd";
+};
+
+&reg_eldo3 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "dvdd-csi";
+};
+
+&reg_fldo1 {
+	regulator-min-microvolt = <1200000>;
+	regulator-max-microvolt = <1200000>;
+	regulator-name = "vcc-1v2-hsic";
+};
+
+/*
+ * The A64 chip cannot work without this regulator off, although
+ * it seems to be only driving the AR100 core.
+ * Maybe we don't still know well about CPUs domain.
+ */
+&reg_fldo2 {
+	regulator-always-on;
+	regulator-min-microvolt = <1100000>;
+	regulator-max-microvolt = <1100000>;
+	regulator-name = "vdd-cpus";
+};
+
+&reg_rtc_ldo {
+	regulator-name = "vcc-rtc";
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pins_a>;
+	status = "okay";
+};
+
+&usb_otg {
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&usbphy {
+	usb0_id_det-gpios = <&pio 7 9 GPIO_ACTIVE_HIGH>; /* PH9 */
+	usb0_vbus-supply = <&reg_drivevbus>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-bananapi-m64.dts b/arch/arm64/boot/dts/allwinner/sun50i-a64-bananapi-m64.dts
index 2250dec9974c..0716b1441187 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-a64-bananapi-m64.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-bananapi-m64.dts
@@ -86,6 +86,10 @@
 	};
 };
 
+&ehci0 {
+	status = "okay";
+};
+
 &ehci1 {
 	status = "okay";
 };
@@ -155,6 +159,10 @@
 	status = "okay";
 };
 
+&ohci0 {
+	status = "okay";
+};
+
 &ohci1 {
 	status = "okay";
 };
@@ -167,6 +175,7 @@
 		reg = <0x3a3>;
 		interrupt-parent = <&r_intc>;
 		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+		x-powers,drive-vbus-en; /* set N_VBUSEN as output pin */
 	};
 };
 
@@ -254,6 +263,11 @@
 	regulator-name = "vcc-wifi-io";
 };
 
+&reg_drivevbus {
+	regulator-name = "usb0-vbus";
+	status = "okay";
+};
+
 &reg_eldo1 {
 	regulator-min-microvolt = <1800000>;
 	regulator-max-microvolt = <1800000>;
@@ -294,6 +308,13 @@
 	status = "okay";
 };
 
+&usb_otg {
+	dr_mode = "otg";
+	status = "okay";
+};
+
 &usbphy {
+	usb0_id_det-gpios = <&pio 7 9 GPIO_ACTIVE_HIGH>; /* PH9 */
+	usb0_vbus-supply = <&reg_drivevbus>;
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-sopine.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a64-sopine.dtsi
index 43418bd881d8..b94f93c04acb 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-a64-sopine.dtsi
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-sopine.dtsi
@@ -66,6 +66,18 @@
 	};
 };
 
+&spi0  {
+	status = "okay";
+
+	flash@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <40000000>;
+	};
+};
+
 #include "axp803.dtsi"
 
 &reg_aldo2 {
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5-libretech-all-h3-cc.dts b/arch/arm64/boot/dts/allwinner/sun50i-h5-libretech-all-h3-cc.dts
new file mode 100644
index 000000000000..95e113ce8699
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h5-libretech-all-h3-cc.dts
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2018 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ */
+
+/dts-v1/;
+#include "sun50i-h5.dtsi"
+#include <arm/sunxi-libretech-all-h3-cc.dtsi>
+
+/ {
+	model = "Libre Computer Board ALL-H3-CC H5";
+	compatible = "libretech,all-h3-cc-h5", "allwinner,sun50i-h5";
+};
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-pc2.dts b/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-pc2.dts
index 98862c7c7258..3e0d5a9c096d 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-pc2.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-pc2.dts
@@ -207,6 +207,18 @@
 	status = "okay";
 };
 
+&spi0  {
+	status = "okay";
+
+	flash@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <40000000>;
+	};
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_pins_a>;
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi
index e237c05cfdb4..62d646baac3c 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi
@@ -47,7 +47,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		cpu@0 {
+		cpu0: cpu@0 {
 			compatible = "arm,cortex-a53", "arm,armv8";
 			device_type = "cpu";
 			reg = <0>;
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6-pine-h64.dts b/arch/arm64/boot/dts/allwinner/sun50i-h6-pine-h64.dts
index d36de5eb81f3..b6f2d6b2ecae 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h6-pine-h64.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h6-pine-h64.dts
@@ -22,6 +22,16 @@
 	};
 };
 
+&r_i2c {
+	status = "okay";
+
+	pcf8563: rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+		#clock-cells = <0>;
+	};
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_ph_pins>;
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi
index 56563150d61a..c72da8cd9ef5 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi
@@ -4,6 +4,8 @@
  */
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/sun50i-h6-ccu.h>
+#include <dt-bindings/reset/sun50i-h6-ccu.h>
 
 / {
 	interrupt-parent = <&gic>;
@@ -115,7 +117,7 @@
 				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&ccu 26>, <&osc24M>, <&osc32k>;
+			clocks = <&ccu CLK_APB1>, <&osc24M>, <&osc32k>;
 			clock-names = "apb", "hosc", "losc";
 			gpio-controller;
 			#gpio-cells = <3>;
@@ -134,8 +136,8 @@
 			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clocks = <&ccu 70>;
-			resets = <&ccu 21>;
+			clocks = <&ccu CLK_BUS_UART0>;
+			resets = <&ccu RST_BUS_UART0>;
 			status = "disabled";
 		};
 
@@ -145,8 +147,8 @@
 			interrupts = <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clocks = <&ccu 71>;
-			resets = <&ccu 22>;
+			clocks = <&ccu CLK_BUS_UART1>;
+			resets = <&ccu RST_BUS_UART1>;
 			status = "disabled";
 		};
 
@@ -156,8 +158,8 @@
 			interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clocks = <&ccu 72>;
-			resets = <&ccu 23>;
+			clocks = <&ccu CLK_BUS_UART2>;
+			resets = <&ccu RST_BUS_UART2>;
 			status = "disabled";
 		};
 
@@ -167,9 +169,59 @@
 			interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clocks = <&ccu 73>;
-			resets = <&ccu 24>;
+			clocks = <&ccu CLK_BUS_UART3>;
+			resets = <&ccu RST_BUS_UART3>;
 			status = "disabled";
 		};
+
+		r_ccu: clock@7010000 {
+			compatible = "allwinner,sun50i-h6-r-ccu";
+			reg = <0x07010000 0x400>;
+			clocks = <&osc24M>, <&osc32k>, <&iosc>,
+				 <&ccu CLK_PLL_PERIPH0>;
+			clock-names = "hosc", "losc", "iosc", "pll-periph";
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		r_intc: interrupt-controller@7021000 {
+			compatible = "allwinner,sun50i-h6-r-intc",
+				     "allwinner,sun6i-a31-r-intc";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			reg = <0x07021000 0x400>;
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		r_pio: pinctrl@7022000 {
+			compatible = "allwinner,sun50i-h6-r-pinctrl";
+			reg = <0x07022000 0x400>;
+			interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&r_ccu 2>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
+			gpio-controller;
+			#gpio-cells = <3>;
+			interrupt-controller;
+			#interrupt-cells = <3>;
+
+			r_i2c_pins: r-i2c {
+				pins = "PL0", "PL1";
+				function = "s_i2c";
+			};
+		};
+
+		r_i2c: i2c@7081400 {
+			compatible = "allwinner,sun6i-a31-i2c";
+			reg = <0x07081400 0x400>;
+			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&r_ccu 8>;
+			resets = <&r_ccu 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&r_i2c_pins>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
index c89d0c307f8d..e6b059378dc0 100644
--- a/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
+++ b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
@@ -17,6 +17,7 @@
 /dts-v1/;
 #include <dt-bindings/reset/altr,rst-mgr-s10.h>
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/clock/stratix10-clock.h>
 
 / {
 	compatible = "altr,socfpga-stratix10";
@@ -92,9 +93,32 @@
 		interrupt-parent = <&intc>;
 		ranges = <0 0 0 0xffffffff>;
 
-		clkmgr@ffd1000 {
-			compatible = "altr,clk-mgr";
+		clkmgr: clock-controller@ffd10000 {
+			compatible = "intel,stratix10-clkmgr";
 			reg = <0xffd10000 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		clocks {
+			cb_intosc_hs_div2_clk: cb-intosc-hs-div2-clk {
+				#clock-cells = <0>;
+				compatible = "fixed-clock";
+			};
+
+			cb_intosc_ls_clk: cb-intosc-ls-clk {
+				#clock-cells = <0>;
+				compatible = "fixed-clock";
+			};
+
+			f2s_free_clk: f2s-free-clk {
+				#clock-cells = <0>;
+				compatible = "fixed-clock";
+			};
+
+			osc1: osc1 {
+				#clock-cells = <0>;
+				compatible = "fixed-clock";
+			};
 		};
 
 		gmac0: ethernet@ff800000 {
@@ -105,6 +129,8 @@
 			mac-address = [00 00 00 00 00 00];
 			resets = <&rst EMAC0_RESET>;
 			reset-names = "stmmaceth";
+			clocks = <&clkmgr STRATIX10_EMAC0_CLK>;
+			clock-names = "stmmaceth";
 			status = "disabled";
 		};
 
@@ -116,6 +142,8 @@
 			mac-address = [00 00 00 00 00 00];
 			resets = <&rst EMAC1_RESET>;
 			reset-names = "stmmaceth";
+			clocks = <&clkmgr STRATIX10_EMAC1_CLK>;
+			clock-names = "stmmaceth";
 			status = "disabled";
 		};
 
@@ -127,6 +155,8 @@
 			mac-address = [00 00 00 00 00 00];
 			resets = <&rst EMAC2_RESET>;
 			reset-names = "stmmaceth";
+			clocks = <&clkmgr STRATIX10_EMAC2_CLK>;
+			clock-names = "stmmaceth";
 			status = "disabled";
 		};
 
@@ -177,6 +207,7 @@
 			reg = <0xffc02800 0x100>;
 			interrupts = <0 103 4>;
 			resets = <&rst I2C0_RESET>;
+			clocks = <&clkmgr STRATIX10_L4_SP_CLK>;
 			status = "disabled";
 		};
 
@@ -187,6 +218,7 @@
 			reg = <0xffc02900 0x100>;
 			interrupts = <0 104 4>;
 			resets = <&rst I2C1_RESET>;
+			clocks = <&clkmgr STRATIX10_L4_SP_CLK>;
 			status = "disabled";
 		};
 
@@ -197,6 +229,7 @@
 			reg = <0xffc02a00 0x100>;
 			interrupts = <0 105 4>;
 			resets = <&rst I2C2_RESET>;
+			clocks = <&clkmgr STRATIX10_L4_SP_CLK>;
 			status = "disabled";
 		};
 
@@ -207,6 +240,7 @@
 			reg = <0xffc02b00 0x100>;
 			interrupts = <0 106 4>;
 			resets = <&rst I2C3_RESET>;
+			clocks = <&clkmgr STRATIX10_L4_SP_CLK>;
 			status = "disabled";
 		};
 
@@ -217,6 +251,7 @@
 			reg = <0xffc02c00 0x100>;
 			interrupts = <0 107 4>;
 			resets = <&rst I2C4_RESET>;
+			clocks = <&clkmgr STRATIX10_L4_SP_CLK>;
 			status = "disabled";
 		};
 
@@ -229,6 +264,9 @@
 			fifo-depth = <0x400>;
 			resets = <&rst SDMMC_RESET>;
 			reset-names = "reset";
+			clocks = <&clkmgr STRATIX10_L4_MP_CLK>,
+				 <&clkmgr STRATIX10_SDMMC_CLK>;
+			clock-names = "biu", "ciu";
 			status = "disabled";
 		};
 
@@ -237,6 +275,25 @@
 			reg = <0xffe00000 0x100000>;
 		};
 
+		pdma: pdma@ffda0000 {
+			compatible = "arm,pl330", "arm,primecell";
+			reg = <0xffda0000 0x1000>;
+			interrupts = <0 81 4>,
+				     <0 82 4>,
+				     <0 83 4>,
+				     <0 84 4>,
+				     <0 85 4>,
+				     <0 86 4>,
+				     <0 87 4>,
+				     <0 88 4>,
+				     <0 89 4>;
+			#dma-cells = <1>;
+			#dma-channels = <8>;
+			#dma-requests = <32>;
+			clocks = <&clkmgr STRATIX10_L4_MAIN_CLK>;
+			clock-names = "apb_pclk";
+		};
+
 		rst: rstmgr@ffd11000 {
 			#reset-cells = <1>;
 			compatible = "altr,rst-mgr";
@@ -288,24 +345,32 @@
 			compatible = "snps,dw-apb-timer";
 			interrupts = <0 113 4>;
 			reg = <0xffc03000 0x100>;
+			clocks = <&clkmgr STRATIX10_L4_SP_CLK>;
+			clock-names = "timer";
 		};
 
 		timer1: timer1@ffc03100 {
 			compatible = "snps,dw-apb-timer";
 			interrupts = <0 114 4>;
 			reg = <0xffc03100 0x100>;
+			clocks = <&clkmgr STRATIX10_L4_SP_CLK>;
+			clock-names = "timer";
 		};
 
 		timer2: timer2@ffd00000 {
 			compatible = "snps,dw-apb-timer";
 			interrupts = <0 115 4>;
 			reg = <0xffd00000 0x100>;
+			clocks = <&clkmgr STRATIX10_L4_SP_CLK>;
+			clock-names = "timer";
 		};
 
 		timer3: timer3@ffd00100 {
 			compatible = "snps,dw-apb-timer";
 			interrupts = <0 116 4>;
 			reg = <0xffd00100 0x100>;
+			clocks = <&clkmgr STRATIX10_L4_SP_CLK>;
+			clock-names = "timer";
 		};
 
 		uart0: serial0@ffc02000 {
@@ -315,6 +380,7 @@
 			reg-shift = <2>;
 			reg-io-width = <4>;
 			resets = <&rst UART0_RESET>;
+			clocks = <&clkmgr STRATIX10_L4_SP_CLK>;
 			status = "disabled";
 		};
 
@@ -325,6 +391,7 @@
 			reg-shift = <2>;
 			reg-io-width = <4>;
 			resets = <&rst UART1_RESET>;
+			clocks = <&clkmgr STRATIX10_L4_SP_CLK>;
 			status = "disabled";
 		};
 
@@ -387,5 +454,17 @@
 			resets = <&rst WATCHDOG3_RESET>;
 			status = "disabled";
 		};
+
+		eccmgr {
+			compatible = "altr,socfpga-s10-ecc-manager";
+			interrupts = <0 15 4>, <0 95 4>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+
+			sdramedac {
+				compatible = "altr,sdram-edac-s10";
+				interrupts = <16 4>, <48 4>;
+			};
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts
index eaf13fe29287..f9b1ef12db48 100644
--- a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts
+++ b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts
@@ -50,6 +50,21 @@
 		/* We expect the bootloader to fill in the reg */
 		reg = <0 0 0 0>;
 	};
+
+	ref_033v: 033-v-ref {
+		compatible = "regulator-fixed";
+		regulator-name = "0.33V";
+		regulator-min-microvolt = <330000>;
+		regulator-max-microvolt = <330000>;
+	};
+
+	soc {
+		clocks {
+			osc1 {
+				clock-frequency = <25000000>;
+			};
+		};
+	};
 };
 
 &gpio1 {
@@ -79,7 +94,7 @@
 			rxd2-skew-ps = <420>; /* 0ps */
 			rxd3-skew-ps = <420>; /* 0ps */
 			txen-skew-ps = <0>; /* -420ps */
-			txc-skew-ps = <1860>; /* 960ps */
+			txc-skew-ps = <900>; /* 0ps */
 			rxdv-skew-ps = <420>; /* 0ps */
 			rxc-skew-ps = <1680>; /* 780ps */
 		};
@@ -105,3 +120,30 @@
 &watchdog0 {
 	status = "okay";
 };
+
+&i2c1 {
+	status = "okay";
+	clock-frequency = <100000>;
+
+	adc@14 {
+		compatible = "lltc,ltc2497";
+		reg = <0x14>;
+		vref-supply = <&ref_033v>;
+	};
+
+	temp@4c {
+		compatible = "maxim,max1619";
+		reg = <0x4c>;
+	};
+
+	eeprom@51 {
+		compatible = "atmel,24c32";
+		reg = <0x51>;
+		pagesize = <32>;
+	};
+
+	rtc@68 {
+		compatible = "dallas,ds1339";
+		reg = <0x68>;
+	};
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts b/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts
index 57eedced5a51..4b3331fbfe39 100644
--- a/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts
@@ -15,6 +15,53 @@
 		serial0 = &uart_AO;
 		serial1 = &uart_A;
 	};
+
+	vddio_boot: regulator-vddio_boot {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDIO_BOOT";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	vddao_3v3: regulator-vddao_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDAO_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	vddio_ao18: regulator-vddio_ao18 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDIO_AO18";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	vcc_3v3: regulator-vcc_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	emmc_pwrseq: emmc-pwrseq {
+		compatible = "mmc-pwrseq-emmc";
+		reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpio GPIOX_7 GPIO_ACTIVE_LOW>;
+		clocks = <&wifi32k>;
+		clock-names = "ext_clock";
+	};
+
+	wifi32k: wifi32k {
+		compatible = "pwm-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		pwms = <&pwm_ab 0 30518 0>; /* PWM_A at 32.768KHz */
+	};
 };
 
 &ethmac {
@@ -47,3 +94,62 @@
 	pinctrl-0 = <&i2c1_z_pins>;
 	pinctrl-names = "default";
 };
+
+&i2c_AO {
+	status = "okay";
+	pinctrl-0 = <&i2c_ao_sck_10_pins>, <&i2c_ao_sda_11_pins>;
+	pinctrl-names = "default";
+};
+
+&pwm_ab {
+	status = "okay";
+	pinctrl-0 = <&pwm_a_x20_pins>;
+	pinctrl-names = "default";
+};
+
+/* emmc storage */
+&sd_emmc_c {
+	status = "okay";
+	pinctrl-0 = <&emmc_pins>;
+	pinctrl-1 = <&emmc_clk_gate_pins>;
+	pinctrl-names = "default", "clk-gate";
+
+	bus-width = <8>;
+	cap-sd-highspeed;
+	cap-mmc-highspeed;
+	max-frequency = <180000000>;
+	non-removable;
+	disable-wp;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+};
+
+/* wifi module */
+&sd_emmc_b {
+	status = "okay";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	pinctrl-0 = <&sdio_pins>;
+	pinctrl-1 = <&sdio_clk_gate_pins>;
+	pinctrl-names = "default", "clk-gate";
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+	non-removable;
+	disable-wp;
+
+	mmc-pwrseq = <&sdio_pwrseq>;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+
+	brcmf: wifi@1 {
+		reg = <1>;
+		compatible = "brcm,bcm4329-fmac";
+	};
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi
index b58808eb3cc8..fee87737a201 100644
--- a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi
@@ -7,6 +7,9 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/clock/axg-clkc.h>
+#include <dt-bindings/clock/axg-aoclkc.h>
+#include <dt-bindings/gpio/meson-axg-gpio.h>
+#include <dt-bindings/reset/amlogic,meson-axg-reset.h>
 
 / {
 	compatible = "amlogic,meson-axg";
@@ -107,12 +110,51 @@
 		#clock-cells = <0>;
 	};
 
+	ao_alt_xtal: ao_alt_xtal-clk {
+		compatible = "fixed-clock";
+		clock-frequency = <32000000>;
+		clock-output-names = "ao_alt_xtal";
+		#clock-cells = <0>;
+	};
+
 	soc {
 		compatible = "simple-bus";
 		#address-cells = <2>;
 		#size-cells = <2>;
 		ranges;
 
+		apb: apb@ffe00000 {
+			compatible = "simple-bus";
+			reg = <0x0 0xffe00000 0x0 0x200000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x0 0x0 0x0 0xffe00000 0x0 0x200000>;
+
+			sd_emmc_b: sd@5000 {
+				compatible = "amlogic,meson-axg-mmc";
+				reg = <0x0 0x5000 0x0 0x2000>;
+				interrupts = <GIC_SPI 217 IRQ_TYPE_EDGE_RISING>;
+				status = "disabled";
+				clocks = <&clkc CLKID_SD_EMMC_B>,
+					<&clkc CLKID_SD_EMMC_B_CLK0>,
+					<&clkc CLKID_FCLK_DIV2>;
+				clock-names = "core", "clkin0", "clkin1";
+				resets = <&reset RESET_SD_EMMC_B>;
+			};
+
+			sd_emmc_c: mmc@7000 {
+				compatible = "amlogic,meson-axg-mmc";
+				reg = <0x0 0x7000 0x0 0x2000>;
+				interrupts = <GIC_SPI 218 IRQ_TYPE_EDGE_RISING>;
+				status = "disabled";
+				clocks = <&clkc CLKID_SD_EMMC_C>,
+					<&clkc CLKID_SD_EMMC_C_CLK0>,
+					<&clkc CLKID_FCLK_DIV2>;
+				clock-names = "core", "clkin0", "clkin1";
+				resets = <&reset RESET_SD_EMMC_C>;
+			};
+		};
+
 		cbus: bus@ffd00000 {
 			compatible = "simple-bus";
 			reg = <0x0 0xffd00000 0x0 0x25000>;
@@ -120,6 +162,15 @@
 			#size-cells = <2>;
 			ranges = <0x0 0x0 0x0 0xffd00000 0x0 0x25000>;
 
+			gpio_intc: interrupt-controller@f080 {
+				compatible = "amlogic,meson-gpio-intc";
+				reg = <0x0 0xf080 0x0 0x10>;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				amlogic,channel-interrupts = <64 65 66 67 68 69 70 71>;
+				status = "disabled";
+			};
+
 			pwm_ab: pwm@1b000 {
 				compatible = "amlogic,meson-axg-ee-pwm";
 				reg = <0x0 0x1b000 0x0 0x20>;
@@ -164,50 +215,42 @@
 
 			i2c0: i2c@1f000 {
 				compatible = "amlogic,meson-axg-i2c";
-				status = "disabled";
 				reg = <0x0 0x1f000 0x0 0x20>;
-				interrupts = <GIC_SPI 21 IRQ_TYPE_EDGE_RISING>,
-					<GIC_SPI 47 IRQ_TYPE_EDGE_RISING>;
+				interrupts = <GIC_SPI 21 IRQ_TYPE_EDGE_RISING>;
+				clocks = <&clkc CLKID_I2C>;
 				#address-cells = <1>;
 				#size-cells = <0>;
-				clocks = <&clkc CLKID_I2C>;
-				clock-names = "clk_i2c";
+				status = "disabled";
 			};
 
 			i2c1: i2c@1e000 {
 				compatible = "amlogic,meson-axg-i2c";
+				reg = <0x0 0x1e000 0x0 0x20>;
+				interrupts = <GIC_SPI 214 IRQ_TYPE_EDGE_RISING>;
+				clocks = <&clkc CLKID_I2C>;
 				#address-cells = <1>;
 				#size-cells = <0>;
-				reg = <0x0 0x1e000 0x0 0x20>;
 				status = "disabled";
-				interrupts = <GIC_SPI 214 IRQ_TYPE_EDGE_RISING>,
-					<GIC_SPI 48 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&clkc CLKID_I2C>;
-				clock-names = "clk_i2c";
 			};
 
 			i2c2: i2c@1d000 {
 				compatible = "amlogic,meson-axg-i2c";
-				status = "disabled";
 				reg = <0x0 0x1d000 0x0 0x20>;
-				interrupts = <GIC_SPI 215 IRQ_TYPE_EDGE_RISING>,
-					<GIC_SPI 49 IRQ_TYPE_EDGE_RISING>;
+				interrupts = <GIC_SPI 215 IRQ_TYPE_EDGE_RISING>;
+				clocks = <&clkc CLKID_I2C>;
 				#address-cells = <1>;
 				#size-cells = <0>;
-				clocks = <&clkc CLKID_I2C>;
-				clock-names = "clk_i2c";
+				status = "disabled";
 			};
 
 			i2c3: i2c@1c000 {
 				compatible = "amlogic,meson-axg-i2c";
-				status = "disabled";
 				reg = <0x0 0x1c000 0x0 0x20>;
-				interrupts = <GIC_SPI 39 IRQ_TYPE_EDGE_RISING>,
-					<GIC_SPI 50 IRQ_TYPE_EDGE_RISING>;
+				interrupts = <GIC_SPI 39 IRQ_TYPE_EDGE_RISING>;
+				clocks = <&clkc CLKID_I2C>;
 				#address-cells = <1>;
 				#size-cells = <0>;
-				clocks = <&clkc CLKID_I2C>;
-				clock-names = "clk_i2c";
+				status = "disabled";
 			};
 
 			uart_A: serial@24000 {
@@ -262,10 +305,14 @@
 			#size-cells = <2>;
 			ranges = <0x0 0x0 0x0 0xff63c000 0x0 0x1c00>;
 
-			clkc: clock-controller@0 {
-				compatible = "amlogic,axg-clkc";
-				#clock-cells = <1>;
-				reg = <0x0 0x0 0x0 0x320>;
+			sysctrl: system-controller@0 {
+				compatible = "amlogic,meson-axg-hhi-sysctrl", "syscon", "simple-mfd";
+				reg = <0 0 0 0x400>;
+
+				clkc: clock-controller {
+					compatible = "amlogic,axg-clkc";
+					#clock-cells = <1>;
+				};
 			};
 		};
 
@@ -309,6 +356,57 @@
 					gpio-ranges = <&pinctrl_periphs 0 0 86>;
 				};
 
+				emmc_pins: emmc {
+					mux {
+						groups = "emmc_nand_d0",
+							"emmc_nand_d1",
+							"emmc_nand_d2",
+							"emmc_nand_d3",
+							"emmc_nand_d4",
+							"emmc_nand_d5",
+							"emmc_nand_d6",
+							"emmc_nand_d7",
+							"emmc_clk",
+							"emmc_cmd",
+							"emmc_ds";
+						function = "emmc";
+					};
+				};
+
+				emmc_clk_gate_pins: emmc_clk_gate {
+					mux {
+						groups = "BOOT_8";
+						function = "gpio_periphs";
+					};
+					cfg-pull-down {
+						pins = "BOOT_8";
+						bias-pull-down;
+					};
+				};
+
+				sdio_pins: sdio {
+					mux {
+						groups = "sdio_d0",
+							"sdio_d1",
+							"sdio_d2",
+							"sdio_d3",
+							"sdio_cmd",
+							"sdio_clk";
+						function = "sdio";
+					};
+				};
+
+				sdio_clk_gate_pins: sdio_clk_gate {
+					mux {
+						groups = "GPIOX_4";
+						function = "gpio_periphs";
+					};
+					cfg-pull-down {
+						pins = "GPIOX_4";
+						bias-pull-down;
+					};
+				};
+
 				eth_rmii_x_pins: eth-x-rmii {
 					mux {
 						groups = "eth_mdio_x",
@@ -660,6 +758,251 @@
 						function = "uart_ao_b_z";
 					};
 				};
+
+				mclk_b_pins: mclk_b {
+					mux {
+						groups = "mclk_b";
+						function = "mclk_b";
+					};
+				};
+
+				mclk_c_pins: mclk_c {
+					mux {
+						groups = "mclk_c";
+						function = "mclk_c";
+					};
+				};
+
+				tdma_sclk_pins: tdma_sclk {
+					mux {
+						groups = "tdma_sclk";
+						function = "tdma";
+					};
+				};
+
+				tdma_sclk_slv_pins: tdma_sclk_slv {
+					mux {
+						groups = "tdma_sclk_slv";
+						function = "tdma";
+					};
+				};
+
+				tdma_fs_pins: tdma_fs {
+					mux {
+						groups = "tdma_fs";
+						function = "tdma";
+					};
+				};
+
+				tdma_fs_slv_pins: tdma_fs_slv {
+					mux {
+						groups = "tdma_fs_slv";
+						function = "tdma";
+					};
+				};
+
+				tdma_din0_pins: tdma_din0 {
+					mux {
+						groups = "tdma_din0";
+						function = "tdma";
+					};
+				};
+
+				tdma_dout0_x14_pins: tdma_dout0_x14 {
+					mux {
+						groups = "tdma_dout0_x14";
+						function = "tdma";
+					};
+				};
+
+				tdma_dout0_x15_pins: tdma_dout0_x15 {
+					mux {
+						groups = "tdma_dout0_x15";
+						function = "tdma";
+					};
+				};
+
+				tdma_dout1_pins: tdma_dout1 {
+					mux {
+						groups = "tdma_dout1";
+						function = "tdma";
+					};
+				};
+
+				tdma_din1_pins: tdma_din1 {
+					mux {
+						groups = "tdma_din1";
+						function = "tdma";
+					};
+				};
+
+				tdmb_sclk_pins: tdmb_sclk {
+					mux {
+						groups = "tdmb_sclk";
+						function = "tdmb";
+					};
+				};
+
+				tdmb_sclk_slv_pins: tdmb_sclk_slv {
+					mux {
+						groups = "tdmb_sclk_slv";
+						function = "tdmb";
+					};
+				};
+
+				tdmb_fs_pins: tdmb_fs {
+					mux {
+						groups = "tdmb_fs";
+						function = "tdmb";
+					};
+				};
+
+				tdmb_fs_slv_pins: tdmb_fs_slv {
+					mux {
+						groups = "tdmb_fs_slv";
+						function = "tdmb";
+					};
+				};
+
+				tdmb_din0_pins: tdmb_din0 {
+					mux {
+						groups = "tdmb_din0";
+						function = "tdmb";
+					};
+				};
+
+				tdmb_dout0_pins: tdmb_dout0 {
+					mux {
+						groups = "tdmb_dout0";
+						function = "tdmb";
+					};
+				};
+
+				tdmb_din1_pins: tdmb_din1 {
+					mux {
+						groups = "tdmb_din1";
+						function = "tdmb";
+					};
+				};
+
+				tdmb_dout1_pins: tdmb_dout1 {
+					mux {
+						groups = "tdmb_dout1";
+						function = "tdmb";
+					};
+				};
+
+				tdmb_din2_pins: tdmb_din2 {
+					mux {
+						groups = "tdmb_din2";
+						function = "tdmb";
+					};
+				};
+
+				tdmb_dout2_pins: tdmb_dout2 {
+					mux {
+						groups = "tdmb_dout2";
+						function = "tdmb";
+					};
+				};
+
+				tdmb_din3_pins: tdmb_din3 {
+					mux {
+						groups = "tdmb_din3";
+						function = "tdmb";
+					};
+				};
+
+				tdmb_dout3_pins: tdmb_dout3 {
+					mux {
+						groups = "tdmb_dout3";
+						function = "tdmb";
+					};
+				};
+
+				tdmc_sclk_pins: tdmc_sclk {
+					mux {
+						groups = "tdmc_sclk";
+						function = "tdmc";
+					};
+				};
+
+				tdmc_sclk_slv_pins: tdmc_sclk_slv {
+					mux {
+						groups = "tdmc_sclk_slv";
+						function = "tdmc";
+					};
+				};
+
+				tdmc_fs_pins: tdmc_fs {
+					mux {
+						groups = "tdmc_fs";
+						function = "tdmc";
+					};
+				};
+
+				tdmc_fs_slv_pins: tdmc_fs_slv {
+					mux {
+						groups = "tdmc_fs_slv";
+						function = "tdmc";
+					};
+				};
+
+				tdmc_din0_pins: tdmc_din0 {
+					mux {
+						groups = "tdmc_din0";
+						function = "tdmc";
+					};
+				};
+
+				tdmc_dout0_pins: tdmc_dout0 {
+					mux {
+						groups = "tdmc_dout0";
+						function = "tdmc";
+					};
+				};
+
+				tdmc_din1_pins: tdmc_din1 {
+					mux {
+						groups = "tdmc_din1";
+						function = "tdmc";
+					};
+				};
+
+				tdmc_dout1_pins: tdmc_dout1 {
+					mux {
+						groups = "tdmc_dout1";
+						function = "tdmc";
+					};
+				};
+
+				tdmc_din2_pins: tdmc_din2 {
+					mux {
+						groups = "tdmc_din2";
+						function = "tdmc";
+					};
+				};
+
+				tdmc_dout2_pins: tdmc_dout2 {
+					mux {
+						groups = "tdmc_dout2";
+						function = "tdmc";
+					};
+				};
+
+				tdmc_din3_pins: tdmc_din3 {
+					mux {
+						groups = "tdmc_din3";
+						function = "tdmc";
+					};
+				};
+
+				tdmc_dout3_pins: tdmc_dout3 {
+					mux {
+						groups = "tdmc_dout3";
+						function = "tdmc";
+					};
+				};
 			};
 		};
 
@@ -688,6 +1031,17 @@
 			#size-cells = <2>;
 			ranges = <0x0 0x0 0x0 0xff800000 0x0 0x100000>;
 
+			sysctrl_AO: sys-ctrl@0 {
+				compatible = "amlogic,meson-axg-ao-sysctrl", "syscon", "simple-mfd";
+				reg =  <0x0 0x0 0x0 0x100>;
+
+				clkc_AO: clock-controller {
+					compatible = "amlogic,meson-axg-aoclkc";
+					#clock-cells = <1>;
+					#reset-cells = <1>;
+				};
+			};
+
 			pinctrl_aobus: pinctrl@14 {
 				compatible = "amlogic,meson-axg-aobus-pinctrl";
 				#address-cells = <2>;
@@ -704,6 +1058,48 @@
 					gpio-ranges = <&pinctrl_aobus 0 0 15>;
 				};
 
+				i2c_ao_sck_4_pins: i2c_ao_sck_4 {
+					mux {
+						groups = "i2c_ao_sck_4";
+						function = "i2c_ao";
+					};
+				};
+
+				i2c_ao_sck_8_pins: i2c_ao_sck_8 {
+					mux {
+						groups = "i2c_ao_sck_8";
+						function = "i2c_ao";
+					};
+				};
+
+				i2c_ao_sck_10_pins: i2c_ao_sck_10 {
+					mux {
+						groups = "i2c_ao_sck_10";
+						function = "i2c_ao";
+					};
+				};
+
+				i2c_ao_sda_5_pins: i2c_ao_sda_5 {
+					mux {
+						groups = "i2c_ao_sda_5";
+						function = "i2c_ao";
+					};
+				};
+
+				i2c_ao_sda_9_pins: i2c_ao_sda_9 {
+					mux {
+						groups = "i2c_ao_sda_9";
+						function = "i2c_ao";
+					};
+				};
+
+				i2c_ao_sda_11_pins: i2c_ao_sda_11 {
+					mux {
+						groups = "i2c_ao_sda_11";
+						function = "i2c_ao";
+					};
+				};
+
 				remote_input_ao_pins: remote_input_ao {
 					mux {
 						groups = "remote_input_ao";
@@ -766,20 +1162,19 @@
 
 			i2c_AO: i2c@5000 {
 				compatible = "amlogic,meson-axg-i2c";
-				status = "disabled";
 				reg = <0x0 0x05000 0x0 0x20>;
 				interrupts = <GIC_SPI 195 IRQ_TYPE_EDGE_RISING>;
+				clocks = <&clkc CLKID_AO_I2C>;
 				#address-cells = <1>;
 				#size-cells = <0>;
-				clocks = <&clkc CLKID_I2C>;
-				clock-names = "clk_i2c";
+				status = "disabled";
 			};
 
 			uart_AO: serial@3000 {
 				compatible = "amlogic,meson-gx-uart", "amlogic,meson-ao-uart";
 				reg = <0x0 0x3000 0x0 0x18>;
 				interrupts = <GIC_SPI 193 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&xtal>, <&clkc CLKID_CLK81>, <&xtal>;
+				clocks = <&xtal>, <&clkc_AO CLKID_AO_UART1>, <&xtal>;
 				clock-names = "xtal", "pclk", "baud";
 				status = "disabled";
 			};
@@ -788,7 +1183,7 @@
 				compatible = "amlogic,meson-gx-uart", "amlogic,meson-ao-uart";
 				reg = <0x0 0x4000 0x0 0x18>;
 				interrupts = <GIC_SPI 197 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&xtal>, <&clkc CLKID_CLK81>, <&xtal>;
+				clocks = <&xtal>, <&clkc_AO CLKID_AO_UART2>, <&xtal>;
 				clock-names = "xtal", "pclk", "baud";
 				status = "disabled";
 			};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
index 562c26a0ba33..98cbba6809ca 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
@@ -307,11 +307,10 @@
 	clock-names = "isfr", "iahb", "venci";
 };
 
-&hiubus {
-	clkc: clock-controller@0 {
+&sysctrl {
+	clkc: clock-controller {
 		compatible = "amlogic,gxbb-clkc";
 		#clock-cells = <1>;
-		reg = <0x0 0x0 0x0 0x3db>;
 	};
 };
 
@@ -716,6 +715,7 @@
 		 <&clkc CLKID_SD_EMMC_A_CLK0>,
 		 <&clkc CLKID_FCLK_DIV2>;
 	clock-names = "core", "clkin0", "clkin1";
+	resets = <&reset RESET_SD_EMMC_A>;
 };
 
 &sd_emmc_b {
@@ -723,6 +723,7 @@
 		 <&clkc CLKID_SD_EMMC_B_CLK0>,
 		 <&clkc CLKID_FCLK_DIV2>;
 	clock-names = "core", "clkin0", "clkin1";
+	resets = <&reset RESET_SD_EMMC_B>;
 };
 
 &sd_emmc_c {
@@ -730,6 +731,7 @@
 		 <&clkc CLKID_SD_EMMC_C_CLK0>,
 		 <&clkc CLKID_FCLK_DIV2>;
 	clock-names = "core", "clkin0", "clkin1";
+	resets = <&reset RESET_SD_EMMC_C>;
 };
 
 &spicc {
@@ -749,12 +751,12 @@
 };
 
 &uart_AO {
-	clocks = <&xtal>, <&clkc CLKID_CLK81>, <&xtal>;
+	clocks = <&xtal>, <&clkc_AO CLKID_AO_UART1>, <&xtal>;
 	clock-names = "xtal", "pclk", "baud";
 };
 
 &uart_AO_B {
-	clocks = <&xtal>, <&clkc CLKID_CLK81>, <&xtal>;
+	clocks = <&xtal>, <&clkc_AO CLKID_AO_UART2>, <&xtal>;
 	clock-names = "xtal", "pclk", "baud";
 };
 
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-libretech-cc.dts b/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-libretech-cc.dts
index 3e3eb31748a3..f63bceb88caa 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-libretech-cc.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-libretech-cc.dts
@@ -234,9 +234,6 @@
 
 	bus-width = <4>;
 	cap-sd-highspeed;
-	sd-uhs-sdr12;
-	sd-uhs-sdr25;
-	sd-uhs-sdr50;
 	max-frequency = <100000000>;
 	disable-wp;
 
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi
index dba365ed4bd5..27538eea547b 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi
@@ -267,11 +267,10 @@
 	clock-names = "isfr", "iahb", "venci";
 };
 
-&hiubus {
-	clkc: clock-controller@0 {
-		compatible = "amlogic,gxl-clkc", "amlogic,gxbb-clkc";
+&sysctrl {
+	clkc: clock-controller {
+		compatible = "amlogic,gxl-clkc";
 		#clock-cells = <1>;
-		reg = <0x0 0x0 0x0 0x3db>;
 	};
 };
 
@@ -725,13 +724,15 @@
 		 <&clkc CLKID_SD_EMMC_A_CLK0>,
 		 <&clkc CLKID_FCLK_DIV2>;
 	clock-names = "core", "clkin0", "clkin1";
+	resets = <&reset RESET_SD_EMMC_A>;
 };
 
 &sd_emmc_b {
 	clocks = <&clkc CLKID_SD_EMMC_B>,
 		 <&clkc CLKID_SD_EMMC_B_CLK0>,
 		 <&clkc CLKID_FCLK_DIV2>;
-       clock-names = "core", "clkin0", "clkin1";
+	clock-names = "core", "clkin0", "clkin1";
+	resets = <&reset RESET_SD_EMMC_B>;
 };
 
 &sd_emmc_c {
@@ -739,6 +740,7 @@
 		 <&clkc CLKID_SD_EMMC_C_CLK0>,
 		 <&clkc CLKID_FCLK_DIV2>;
 	clock-names = "core", "clkin0", "clkin1";
+	resets = <&reset RESET_SD_EMMC_C>;
 };
 
 &spicc {
@@ -758,12 +760,12 @@
 };
 
 &uart_AO {
-	clocks = <&xtal>, <&clkc CLKID_CLK81>, <&xtal>;
+	clocks = <&xtal>, <&clkc_AO CLKID_AO_UART1>, <&xtal>;
 	clock-names = "xtal", "pclk", "baud";
 };
 
 &uart_AO_B {
-	clocks = <&xtal>, <&clkc CLKID_CLK81>, <&xtal>;
+	clocks = <&xtal>, <&clkc_AO CLKID_AO_UART2>, <&xtal>;
 	clock-names = "xtal", "pclk", "baud";
 };
 
diff --git a/arch/arm64/boot/dts/arm/juno-base.dtsi b/arch/arm64/boot/dts/arm/juno-base.dtsi
index eb749c50a736..ce56a4acda4f 100644
--- a/arch/arm64/boot/dts/arm/juno-base.dtsi
+++ b/arch/arm64/boot/dts/arm/juno-base.dtsi
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 #include "juno-clocks.dtsi"
+#include "juno-motherboard.dtsi"
 
 / {
 	/*
@@ -572,14 +573,14 @@
 			thermal-sensors = <&scpi_sensors0 3>;
 		};
 
-		big_cluster_thermal_zone: big_cluster {
+		big_cluster_thermal_zone: big-cluster {
 			polling-delay = <1000>;
 			polling-delay-passive = <100>;
 			thermal-sensors = <&scpi_sensors0 21>;
 			status = "disabled";
 		};
 
-		little_cluster_thermal_zone: little_cluster {
+		little_cluster_thermal_zone: little-cluster {
 			polling-delay = <1000>;
 			polling-delay-passive = <100>;
 			thermal-sensors = <&scpi_sensors0 22>;
@@ -677,7 +678,7 @@
 		clock-names = "pxlclk";
 
 		port {
-			hdlcd1_output: hdlcd1-endpoint {
+			hdlcd1_output: endpoint {
 				remote-endpoint = <&tda998x_1_input>;
 			};
 		};
@@ -692,7 +693,7 @@
 		clock-names = "pxlclk";
 
 		port {
-			hdlcd0_output: hdlcd0-endpoint {
+			hdlcd0_output: endpoint {
 				remote-endpoint = <&tda998x_0_input>;
 			};
 		};
@@ -720,7 +721,7 @@
 			compatible = "nxp,tda998x";
 			reg = <0x70>;
 			port {
-				tda998x_0_input: tda998x-0-endpoint {
+				tda998x_0_input: endpoint {
 					remote-endpoint = <&hdlcd0_output>;
 				};
 			};
@@ -730,7 +731,7 @@
 			compatible = "nxp,tda998x";
 			reg = <0x71>;
 			port {
-				tda998x_1_input: tda998x-1-endpoint {
+				tda998x_1_input: endpoint {
 					remote-endpoint = <&hdlcd1_output>;
 				};
 			};
@@ -795,8 +796,6 @@
 				<0 0 10 &gic 0 0 0 167 IRQ_TYPE_LEVEL_HIGH>,
 				<0 0 11 &gic 0 0 0 168 IRQ_TYPE_LEVEL_HIGH>,
 				<0 0 12 &gic 0 0 0 169 IRQ_TYPE_LEVEL_HIGH>;
-
-		/include/ "juno-motherboard.dtsi"
 	};
 
 	site2: tlx@60000000 {
diff --git a/arch/arm64/boot/dts/arm/juno-motherboard.dtsi b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi
index 69804c5f1197..1792b074e9a3 100644
--- a/arch/arm64/boot/dts/arm/juno-motherboard.dtsi
+++ b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi
@@ -7,6 +7,8 @@
  *
  */
 
+/ {
+	smb@8000000 {
 		mb_clk24mhz: clk24mhz {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
@@ -54,46 +56,46 @@
 				regulator-always-on;
 			};
 
-			gpio_keys {
+			gpio-keys {
 				compatible = "gpio-keys";
 
 				power-button {
-					debounce_interval = <50>;
+					debounce-interval = <50>;
 					wakeup-source;
 					linux,code = <116>;
 					label = "POWER";
 					gpios = <&iofpga_gpio0 0 0x4>;
 				};
 				home-button {
-					debounce_interval = <50>;
+					debounce-interval = <50>;
 					wakeup-source;
 					linux,code = <102>;
 					label = "HOME";
 					gpios = <&iofpga_gpio0 1 0x4>;
 				};
 				rlock-button {
-					debounce_interval = <50>;
+					debounce-interval = <50>;
 					wakeup-source;
 					linux,code = <152>;
 					label = "RLOCK";
 					gpios = <&iofpga_gpio0 2 0x4>;
 				};
 				vol-up-button {
-					debounce_interval = <50>;
+					debounce-interval = <50>;
 					wakeup-source;
 					linux,code = <115>;
 					label = "VOL+";
 					gpios = <&iofpga_gpio0 3 0x4>;
 				};
 				vol-down-button {
-					debounce_interval = <50>;
+					debounce-interval = <50>;
 					wakeup-source;
 					linux,code = <114>;
 					label = "VOL-";
 					gpios = <&iofpga_gpio0 4 0x4>;
 				};
 				nmi-button {
-					debounce_interval = <50>;
+					debounce-interval = <50>;
 					wakeup-source;
 					linux,code = <99>;
 					label = "NMI";
@@ -287,3 +289,5 @@
 				};
 			};
 		};
+	};
+};
diff --git a/arch/arm64/boot/dts/arm/juno-r1.dts b/arch/arm64/boot/dts/arm/juno-r1.dts
index aed6389468c4..2c5db03f226c 100644
--- a/arch/arm64/boot/dts/arm/juno-r1.dts
+++ b/arch/arm64/boot/dts/arm/juno-r1.dts
@@ -201,7 +201,7 @@
 		};
 	};
 
-	pmu_a57 {
+	pmu-a57 {
 		compatible = "arm,cortex-a57-pmu";
 		interrupts = <GIC_SPI 02 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 06 IRQ_TYPE_LEVEL_HIGH>;
@@ -209,7 +209,7 @@
 				     <&A57_1>;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>,
@@ -278,6 +278,10 @@
 	remote-endpoint = <&csys2_funnel_out_port>;
 };
 
+&csys1_funnel_in_port0 {
+	remote-endpoint = <&stm_out_port>;
+};
+
 &stm_out_port {
 	remote-endpoint = <&csys1_funnel_in_port0>;
 };
diff --git a/arch/arm64/boot/dts/arm/juno-r2.dts b/arch/arm64/boot/dts/arm/juno-r2.dts
index b39b6d6ec5aa..c51950f4a1b6 100644
--- a/arch/arm64/boot/dts/arm/juno-r2.dts
+++ b/arch/arm64/boot/dts/arm/juno-r2.dts
@@ -201,7 +201,7 @@
 		};
 	};
 
-	pmu_a72 {
+	pmu-a72 {
 		compatible = "arm,cortex-a72-pmu";
 		interrupts = <GIC_SPI 02 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 06 IRQ_TYPE_LEVEL_HIGH>;
@@ -209,7 +209,7 @@
 				     <&A72_1>;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>,
@@ -278,6 +278,10 @@
 	remote-endpoint = <&csys2_funnel_out_port>;
 };
 
+&csys1_funnel_in_port0 {
+	remote-endpoint = <&stm_out_port>;
+};
+
 &stm_out_port {
 	remote-endpoint = <&csys1_funnel_in_port0>;
 };
diff --git a/arch/arm64/boot/dts/arm/juno.dts b/arch/arm64/boot/dts/arm/juno.dts
index c9236c4b967d..2b2bf39c30ef 100644
--- a/arch/arm64/boot/dts/arm/juno.dts
+++ b/arch/arm64/boot/dts/arm/juno.dts
@@ -200,7 +200,7 @@
 		};
 	};
 
-	pmu_a57 {
+	pmu-a57 {
 		compatible = "arm,cortex-a57-pmu";
 		interrupts = <GIC_SPI 02 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 06 IRQ_TYPE_LEVEL_HIGH>;
@@ -208,7 +208,7 @@
 				     <&A57_1>;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm64/boot/dts/arm/rtsm_ve-aemv8a.dts b/arch/arm64/boot/dts/arm/rtsm_ve-aemv8a.dts
index 06c8117e812a..602f63f72c37 100644
--- a/arch/arm64/boot/dts/arm/rtsm_ve-aemv8a.dts
+++ b/arch/arm64/boot/dts/arm/rtsm_ve-aemv8a.dts
@@ -12,6 +12,8 @@
 
 /memreserve/ 0x80000000 0x00010000;
 
+#include "rtsm_ve-motherboard.dtsi"
+
 / {
 	model = "RTSM_VE_AEMv8A";
 	compatible = "arm,rtsm_ve,aemv8a", "arm,vexpress";
@@ -162,7 +164,5 @@
 				<0 0 40 &gic 0 40 4>,
 				<0 0 41 &gic 0 41 4>,
 				<0 0 42 &gic 0 42 4>;
-
-		/include/ "rtsm_ve-motherboard.dtsi"
 	};
 };
diff --git a/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi b/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi
index 1134e5d8df18..d2dbc3f39263 100644
--- a/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi
+++ b/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi
@@ -7,270 +7,273 @@
  *
  * VEMotherBoard.lisa
  */
-
-	motherboard {
-		arm,v2m-memory-map = "rs1";
-		compatible = "arm,vexpress,v2m-p1", "simple-bus";
-		#address-cells = <2>; /* SMB chipselect number and offset */
-		#size-cells = <1>;
-		#interrupt-cells = <1>;
-		ranges;
-
-		flash@0,00000000 {
-			compatible = "arm,vexpress-flash", "cfi-flash";
-			reg = <0 0x00000000 0x04000000>,
-			      <4 0x00000000 0x04000000>;
-			bank-width = <4>;
-		};
-
-		v2m_video_ram: vram@2,00000000 {
-			compatible = "arm,vexpress-vram";
-			reg = <2 0x00000000 0x00800000>;
-		};
-
-		ethernet@2,02000000 {
-			compatible = "smsc,lan91c111";
-			reg = <2 0x02000000 0x10000>;
-			interrupts = <15>;
-		};
-
-		v2m_clk24mhz: clk24mhz {
-			compatible = "fixed-clock";
-			#clock-cells = <0>;
-			clock-frequency = <24000000>;
-			clock-output-names = "v2m:clk24mhz";
-		};
-
-		v2m_refclk1mhz: refclk1mhz {
-			compatible = "fixed-clock";
-			#clock-cells = <0>;
-			clock-frequency = <1000000>;
-			clock-output-names = "v2m:refclk1mhz";
-		};
-
-		v2m_refclk32khz: refclk32khz {
-			compatible = "fixed-clock";
-			#clock-cells = <0>;
-			clock-frequency = <32768>;
-			clock-output-names = "v2m:refclk32khz";
-		};
-
-		iofpga@3,00000000 {
-			compatible = "simple-bus";
-			#address-cells = <1>;
+/ {
+	smb@8000000 {
+		motherboard {
+			arm,v2m-memory-map = "rs1";
+			compatible = "arm,vexpress,v2m-p1", "simple-bus";
+			#address-cells = <2>; /* SMB chipselect number and offset */
 			#size-cells = <1>;
-			ranges = <0 3 0 0x200000>;
-
-			v2m_sysreg: sysreg@10000 {
-				compatible = "arm,vexpress-sysreg";
-				reg = <0x010000 0x1000>;
-				gpio-controller;
-				#gpio-cells = <2>;
+			#interrupt-cells = <1>;
+			ranges;
+
+			flash@0,00000000 {
+				compatible = "arm,vexpress-flash", "cfi-flash";
+				reg = <0 0x00000000 0x04000000>,
+				      <4 0x00000000 0x04000000>;
+				bank-width = <4>;
 			};
 
-			v2m_sysctl: sysctl@20000 {
-				compatible = "arm,sp810", "arm,primecell";
-				reg = <0x020000 0x1000>;
-				clocks = <&v2m_refclk32khz>, <&v2m_refclk1mhz>, <&v2m_clk24mhz>;
-				clock-names = "refclk", "timclk", "apb_pclk";
-				#clock-cells = <1>;
-				clock-output-names = "timerclken0", "timerclken1", "timerclken2", "timerclken3";
-				assigned-clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&v2m_sysctl 3>, <&v2m_sysctl 3>;
-				assigned-clock-parents = <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>;
+			v2m_video_ram: vram@2,00000000 {
+				compatible = "arm,vexpress-vram";
+				reg = <2 0x00000000 0x00800000>;
 			};
 
-			aaci@40000 {
-				compatible = "arm,pl041", "arm,primecell";
-				reg = <0x040000 0x1000>;
-				interrupts = <11>;
-				clocks = <&v2m_clk24mhz>;
-				clock-names = "apb_pclk";
+			ethernet@2,02000000 {
+				compatible = "smsc,lan91c111";
+				reg = <2 0x02000000 0x10000>;
+				interrupts = <15>;
 			};
 
-			mmci@50000 {
-				compatible = "arm,pl180", "arm,primecell";
-				reg = <0x050000 0x1000>;
-				interrupts = <9 10>;
-				cd-gpios = <&v2m_sysreg 0 0>;
-				wp-gpios = <&v2m_sysreg 1 0>;
-				max-frequency = <12000000>;
-				vmmc-supply = <&v2m_fixed_3v3>;
-				clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
-				clock-names = "mclk", "apb_pclk";
+			v2m_clk24mhz: clk24mhz {
+				compatible = "fixed-clock";
+				#clock-cells = <0>;
+				clock-frequency = <24000000>;
+				clock-output-names = "v2m:clk24mhz";
 			};
 
-			kmi@60000 {
-				compatible = "arm,pl050", "arm,primecell";
-				reg = <0x060000 0x1000>;
-				interrupts = <12>;
-				clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
-				clock-names = "KMIREFCLK", "apb_pclk";
+			v2m_refclk1mhz: refclk1mhz {
+				compatible = "fixed-clock";
+				#clock-cells = <0>;
+				clock-frequency = <1000000>;
+				clock-output-names = "v2m:refclk1mhz";
 			};
 
-			kmi@70000 {
-				compatible = "arm,pl050", "arm,primecell";
-				reg = <0x070000 0x1000>;
-				interrupts = <13>;
-				clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
-				clock-names = "KMIREFCLK", "apb_pclk";
+			v2m_refclk32khz: refclk32khz {
+				compatible = "fixed-clock";
+				#clock-cells = <0>;
+				clock-frequency = <32768>;
+				clock-output-names = "v2m:refclk32khz";
 			};
 
-			v2m_serial0: uart@90000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x090000 0x1000>;
-				interrupts = <5>;
-				clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+			iofpga@3,00000000 {
+				compatible = "simple-bus";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 3 0 0x200000>;
+
+				v2m_sysreg: sysreg@10000 {
+					compatible = "arm,vexpress-sysreg";
+					reg = <0x010000 0x1000>;
+					gpio-controller;
+					#gpio-cells = <2>;
+				};
 
-			v2m_serial1: uart@a0000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x0a0000 0x1000>;
-				interrupts = <6>;
-				clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+				v2m_sysctl: sysctl@20000 {
+					compatible = "arm,sp810", "arm,primecell";
+					reg = <0x020000 0x1000>;
+					clocks = <&v2m_refclk32khz>, <&v2m_refclk1mhz>, <&v2m_clk24mhz>;
+					clock-names = "refclk", "timclk", "apb_pclk";
+					#clock-cells = <1>;
+					clock-output-names = "timerclken0", "timerclken1", "timerclken2", "timerclken3";
+					assigned-clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&v2m_sysctl 3>, <&v2m_sysctl 3>;
+					assigned-clock-parents = <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>;
+				};
 
-			v2m_serial2: uart@b0000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x0b0000 0x1000>;
-				interrupts = <7>;
-				clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+				aaci@40000 {
+					compatible = "arm,pl041", "arm,primecell";
+					reg = <0x040000 0x1000>;
+					interrupts = <11>;
+					clocks = <&v2m_clk24mhz>;
+					clock-names = "apb_pclk";
+				};
 
-			v2m_serial3: uart@c0000 {
-				compatible = "arm,pl011", "arm,primecell";
-				reg = <0x0c0000 0x1000>;
-				interrupts = <8>;
-				clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
-				clock-names = "uartclk", "apb_pclk";
-			};
+				mmci@50000 {
+					compatible = "arm,pl180", "arm,primecell";
+					reg = <0x050000 0x1000>;
+					interrupts = <9 10>;
+					cd-gpios = <&v2m_sysreg 0 0>;
+					wp-gpios = <&v2m_sysreg 1 0>;
+					max-frequency = <12000000>;
+					vmmc-supply = <&v2m_fixed_3v3>;
+					clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
+					clock-names = "mclk", "apb_pclk";
+				};
 
-			wdt@f0000 {
-				compatible = "arm,sp805", "arm,primecell";
-				reg = <0x0f0000 0x1000>;
-				interrupts = <0>;
-				clocks = <&v2m_refclk32khz>, <&v2m_clk24mhz>;
-				clock-names = "wdogclk", "apb_pclk";
-			};
+				kmi@60000 {
+					compatible = "arm,pl050", "arm,primecell";
+					reg = <0x060000 0x1000>;
+					interrupts = <12>;
+					clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
+					clock-names = "KMIREFCLK", "apb_pclk";
+				};
 
-			v2m_timer01: timer@110000 {
-				compatible = "arm,sp804", "arm,primecell";
-				reg = <0x110000 0x1000>;
-				interrupts = <2>;
-				clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&v2m_clk24mhz>;
-				clock-names = "timclken1", "timclken2", "apb_pclk";
-			};
+				kmi@70000 {
+					compatible = "arm,pl050", "arm,primecell";
+					reg = <0x070000 0x1000>;
+					interrupts = <13>;
+					clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
+					clock-names = "KMIREFCLK", "apb_pclk";
+				};
 
-			v2m_timer23: timer@120000 {
-				compatible = "arm,sp804", "arm,primecell";
-				reg = <0x120000 0x1000>;
-				interrupts = <3>;
-				clocks = <&v2m_sysctl 2>, <&v2m_sysctl 3>, <&v2m_clk24mhz>;
-				clock-names = "timclken1", "timclken2", "apb_pclk";
-			};
+				v2m_serial0: uart@90000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x090000 0x1000>;
+					interrupts = <5>;
+					clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
+					clock-names = "uartclk", "apb_pclk";
+				};
 
-			rtc@170000 {
-				compatible = "arm,pl031", "arm,primecell";
-				reg = <0x170000 0x1000>;
-				interrupts = <4>;
-				clocks = <&v2m_clk24mhz>;
-				clock-names = "apb_pclk";
-			};
+				v2m_serial1: uart@a0000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x0a0000 0x1000>;
+					interrupts = <6>;
+					clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
+					clock-names = "uartclk", "apb_pclk";
+				};
 
-			clcd@1f0000 {
-				compatible = "arm,pl111", "arm,primecell";
-				reg = <0x1f0000 0x1000>;
-				interrupt-names = "combined";
-				interrupts = <14>;
-				clocks = <&v2m_oscclk1>, <&v2m_clk24mhz>;
-				clock-names = "clcdclk", "apb_pclk";
-				arm,pl11x,framebuffer = <0x18000000 0x00180000>;
-				memory-region = <&v2m_video_ram>;
-				max-memory-bandwidth = <130000000>; /* 16bpp @ 63.5MHz */
-
-				port {
-					v2m_clcd_pads: endpoint {
-						remote-endpoint = <&v2m_clcd_panel>;
-						arm,pl11x,tft-r0g0b0-pads = <0 8 16>;
-					};
+				v2m_serial2: uart@b0000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x0b0000 0x1000>;
+					interrupts = <7>;
+					clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
+					clock-names = "uartclk", "apb_pclk";
 				};
 
-				panel {
-					compatible = "panel-dpi";
+				v2m_serial3: uart@c0000 {
+					compatible = "arm,pl011", "arm,primecell";
+					reg = <0x0c0000 0x1000>;
+					interrupts = <8>;
+					clocks = <&v2m_clk24mhz>, <&v2m_clk24mhz>;
+					clock-names = "uartclk", "apb_pclk";
+				};
+
+				wdt@f0000 {
+					compatible = "arm,sp805", "arm,primecell";
+					reg = <0x0f0000 0x1000>;
+					interrupts = <0>;
+					clocks = <&v2m_refclk32khz>, <&v2m_clk24mhz>;
+					clock-names = "wdogclk", "apb_pclk";
+				};
+
+				v2m_timer01: timer@110000 {
+					compatible = "arm,sp804", "arm,primecell";
+					reg = <0x110000 0x1000>;
+					interrupts = <2>;
+					clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&v2m_clk24mhz>;
+					clock-names = "timclken1", "timclken2", "apb_pclk";
+				};
+
+				v2m_timer23: timer@120000 {
+					compatible = "arm,sp804", "arm,primecell";
+					reg = <0x120000 0x1000>;
+					interrupts = <3>;
+					clocks = <&v2m_sysctl 2>, <&v2m_sysctl 3>, <&v2m_clk24mhz>;
+					clock-names = "timclken1", "timclken2", "apb_pclk";
+				};
+
+				rtc@170000 {
+					compatible = "arm,pl031", "arm,primecell";
+					reg = <0x170000 0x1000>;
+					interrupts = <4>;
+					clocks = <&v2m_clk24mhz>;
+					clock-names = "apb_pclk";
+				};
+
+				clcd@1f0000 {
+					compatible = "arm,pl111", "arm,primecell";
+					reg = <0x1f0000 0x1000>;
+					interrupt-names = "combined";
+					interrupts = <14>;
+					clocks = <&v2m_oscclk1>, <&v2m_clk24mhz>;
+					clock-names = "clcdclk", "apb_pclk";
+					arm,pl11x,framebuffer = <0x18000000 0x00180000>;
+					memory-region = <&v2m_video_ram>;
+					max-memory-bandwidth = <130000000>; /* 16bpp @ 63.5MHz */
 
 					port {
-						v2m_clcd_panel: endpoint {
-							remote-endpoint = <&v2m_clcd_pads>;
+						v2m_clcd_pads: endpoint {
+							remote-endpoint = <&v2m_clcd_panel>;
+							arm,pl11x,tft-r0g0b0-pads = <0 8 16>;
 						};
 					};
 
-					panel-timing {
-						clock-frequency = <63500127>;
-						hactive = <1024>;
-						hback-porch = <152>;
-						hfront-porch = <48>;
-						hsync-len = <104>;
-						vactive = <768>;
-						vback-porch = <23>;
-						vfront-porch = <3>;
-						vsync-len = <4>;
+					panel {
+						compatible = "panel-dpi";
+
+						port {
+							v2m_clcd_panel: endpoint {
+								remote-endpoint = <&v2m_clcd_pads>;
+							};
+						};
+
+						panel-timing {
+							clock-frequency = <63500127>;
+							hactive = <1024>;
+							hback-porch = <152>;
+							hfront-porch = <48>;
+							hsync-len = <104>;
+							vactive = <768>;
+							vback-porch = <23>;
+							vfront-porch = <3>;
+							vsync-len = <4>;
+						};
 					};
 				};
-			};
 
-			virtio-block@130000 {
-				compatible = "virtio,mmio";
-				reg = <0x130000 0x200>;
-				interrupts = <42>;
+				virtio-block@130000 {
+					compatible = "virtio,mmio";
+					reg = <0x130000 0x200>;
+					interrupts = <42>;
+				};
 			};
-		};
-
-		v2m_fixed_3v3: v2m-3v3 {
-			compatible = "regulator-fixed";
-			regulator-name = "3V3";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-always-on;
-		};
-
-		mcc {
-			compatible = "arm,vexpress,config-bus";
-			arm,vexpress,config-bridge = <&v2m_sysreg>;
 
-			v2m_oscclk1: oscclk1 {
-				/* CLCD clock */
-				compatible = "arm,vexpress-osc";
-				arm,vexpress-sysreg,func = <1 1>;
-				freq-range = <23750000 63500000>;
-				#clock-cells = <0>;
-				clock-output-names = "v2m:oscclk1";
+			v2m_fixed_3v3: v2m-3v3 {
+				compatible = "regulator-fixed";
+				regulator-name = "3V3";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
 			};
 
-			reset {
-				compatible = "arm,vexpress-reset";
-				arm,vexpress-sysreg,func = <5 0>;
-			};
+			mcc {
+				compatible = "arm,vexpress,config-bus";
+				arm,vexpress,config-bridge = <&v2m_sysreg>;
+
+				v2m_oscclk1: oscclk1 {
+					/* CLCD clock */
+					compatible = "arm,vexpress-osc";
+					arm,vexpress-sysreg,func = <1 1>;
+					freq-range = <23750000 63500000>;
+					#clock-cells = <0>;
+					clock-output-names = "v2m:oscclk1";
+				};
 
-			muxfpga {
-				compatible = "arm,vexpress-muxfpga";
-				arm,vexpress-sysreg,func = <7 0>;
-			};
+				reset {
+					compatible = "arm,vexpress-reset";
+					arm,vexpress-sysreg,func = <5 0>;
+				};
 
-			shutdown {
-				compatible = "arm,vexpress-shutdown";
-				arm,vexpress-sysreg,func = <8 0>;
-			};
+				muxfpga {
+					compatible = "arm,vexpress-muxfpga";
+					arm,vexpress-sysreg,func = <7 0>;
+				};
 
-			reboot {
-				compatible = "arm,vexpress-reboot";
-				arm,vexpress-sysreg,func = <9 0>;
-			};
+				shutdown {
+					compatible = "arm,vexpress-shutdown";
+					arm,vexpress-sysreg,func = <8 0>;
+				};
+
+				reboot {
+					compatible = "arm,vexpress-reboot";
+					arm,vexpress-sysreg,func = <9 0>;
+				};
 
-			dvimode {
-				compatible = "arm,vexpress-dvimode";
-				arm,vexpress-sysreg,func = <11 0>;
+				dvimode {
+					compatible = "arm,vexpress-dvimode";
+					arm,vexpress-sysreg,func = <11 0>;
+				};
 			};
 		};
 	};
+};
diff --git a/arch/arm64/boot/dts/arm/vexpress-v2f-1xv7-ca53x2.dts b/arch/arm64/boot/dts/arm/vexpress-v2f-1xv7-ca53x2.dts
index 1c9eadc2d71e..38880380e0fa 100644
--- a/arch/arm64/boot/dts/arm/vexpress-v2f-1xv7-ca53x2.dts
+++ b/arch/arm64/boot/dts/arm/vexpress-v2f-1xv7-ca53x2.dts
@@ -13,6 +13,7 @@
 /dts-v1/;
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include "vexpress-v2m-rs1.dtsi"
 
 / {
 	model = "V2F-1XV7 Cortex-A53x2 SMM";
@@ -129,7 +130,7 @@
 		};
 	};
 
-	smb@8000000 {
+	smb: smb@8000000 {
 		compatible = "simple-bus";
 
 		#address-cells = <2>;
@@ -186,7 +187,5 @@
 				<0 0 40 &gic GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
 				<0 0 41 &gic GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
 				<0 0 42 &gic GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
-
-		/include/ "vexpress-v2m-rs1.dtsi"
 	};
 };
diff --git a/arch/arm64/boot/dts/broadcom/Makefile b/arch/arm64/boot/dts/broadcom/Makefile
index 2a2591ef1fee..1193a9e34bbb 100644
--- a/arch/arm64/boot/dts/broadcom/Makefile
+++ b/arch/arm64/boot/dts/broadcom/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
-dtb-$(CONFIG_ARCH_BCM2835) += bcm2837-rpi-3-b.dtb
+dtb-$(CONFIG_ARCH_BCM2835) += bcm2837-rpi-3-b.dtb \
+			      bcm2837-rpi-3-b-plus.dtb
 
 subdir-y	+= northstar2
 subdir-y	+= stingray
diff --git a/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b-plus.dts b/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b-plus.dts
new file mode 100644
index 000000000000..46ad2023cccf
--- /dev/null
+++ b/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b-plus.dts
@@ -0,0 +1,2 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "arm/bcm2837-rpi-3-b-plus.dts"
diff --git a/arch/arm64/boot/dts/exynos/exynos5433.dtsi b/arch/arm64/boot/dts/exynos/exynos5433.dtsi
index 1ad8677f6a0a..038c99792ccb 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433.dtsi
@@ -18,8 +18,8 @@
 
 / {
 	compatible = "samsung,exynos5433";
-	#address-cells = <2>;
-	#size-cells = <2>;
+	#address-cells = <1>;
+	#size-cells = <1>;
 
 	interrupt-parent = <&gic>;
 
@@ -231,18 +231,11 @@
 		cpu_on = <0xC4000003>;
 	};
 
-	reboot: syscon-reboot {
-		compatible = "syscon-reboot";
-		regmap = <&pmu_system_controller>;
-		offset = <0x400>; /* SWRESET */
-		mask = <0x1>;
-	};
-
 	soc: soc {
 		compatible = "simple-bus";
 		#address-cells = <1>;
 		#size-cells = <1>;
-		ranges = <0x0 0x0 0x0 0x18000000>;
+		ranges;
 
 		arm_a53_pmu {
 			compatible = "arm,cortex-a53-pmu", "arm,armv8-pmuv3";
@@ -799,6 +792,13 @@
 			#clock-cells = <1>;
 			clock-names = "clkout16";
 			clocks = <&xxti>;
+
+			reboot: syscon-reboot {
+				compatible = "syscon-reboot";
+				regmap = <&pmu_system_controller>;
+				offset = <0x400>; /* SWRESET */
+				mask = <0x1>;
+			};
 		};
 
 		gic: interrupt-controller@11001000 {
@@ -829,11 +829,16 @@
 				<&cmu_disp CLK_ACLK_SMMU_DECON0X>,
 				<&cmu_disp CLK_ACLK_XIU_DECON0X>,
 				<&cmu_disp CLK_PCLK_SMMU_DECON0X>,
+				<&cmu_disp CLK_ACLK_SMMU_DECON1X>,
+				<&cmu_disp CLK_ACLK_XIU_DECON1X>,
+				<&cmu_disp CLK_PCLK_SMMU_DECON1X>,
 				<&cmu_disp CLK_SCLK_DECON_VCLK>,
 				<&cmu_disp CLK_SCLK_DECON_ECLK>;
 			clock-names = "pclk", "aclk_decon", "aclk_smmu_decon0x",
 				"aclk_xiu_decon0x", "pclk_smmu_decon0x",
-				"sclk_decon_vclk", "sclk_decon_eclk";
+				"aclk_smmu_decon1x", "aclk_xiu_decon1x",
+				"pclk_smmu_decon1x", "sclk_decon_vclk",
+				"sclk_decon_eclk";
 			power-domains = <&pd_disp>;
 			interrupt-names = "fifo", "vsync", "lcd_sys";
 			interrupts = <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>,
@@ -866,11 +871,16 @@
 				 <&cmu_disp CLK_ACLK_SMMU_TV0X>,
 				 <&cmu_disp CLK_ACLK_XIU_TV0X>,
 				 <&cmu_disp CLK_PCLK_SMMU_TV0X>,
+				 <&cmu_disp CLK_ACLK_SMMU_TV1X>,
+				 <&cmu_disp CLK_ACLK_XIU_TV1X>,
+				 <&cmu_disp CLK_PCLK_SMMU_TV1X>,
 				 <&cmu_disp CLK_SCLK_DECON_TV_VCLK>,
 				 <&cmu_disp CLK_SCLK_DECON_TV_ECLK>;
 			clock-names = "pclk", "aclk_decon", "aclk_smmu_decon0x",
 				      "aclk_xiu_decon0x", "pclk_smmu_decon0x",
-				      "sclk_decon_vclk", "sclk_decon_eclk";
+				      "aclk_smmu_decon1x", "aclk_xiu_decon1x",
+				      "pclk_smmu_decon1x", "sclk_decon_vclk",
+				      "sclk_decon_eclk";
 			samsung,disp-sysreg = <&syscon_disp>;
 			power-domains = <&pd_disp>;
 			interrupt-names = "fifo", "vsync", "lcd_sys";
@@ -1034,6 +1044,30 @@
 			power-domains = <&pd_gscl>;
 		};
 
+		scaler_0: scaler@15000000 {
+			compatible = "samsung,exynos5433-scaler";
+			reg = <0x15000000 0x1294>;
+			interrupts = <0 402 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk", "aclk_xiu";
+			clocks = <&cmu_mscl CLK_PCLK_M2MSCALER0>,
+				 <&cmu_mscl CLK_ACLK_M2MSCALER0>,
+				 <&cmu_mscl CLK_ACLK_XIU_MSCLX>;
+			iommus = <&sysmmu_scaler_0>;
+			power-domains = <&pd_mscl>;
+		};
+
+		scaler_1: scaler@15010000 {
+			compatible = "samsung,exynos5433-scaler";
+			reg = <0x15010000 0x1294>;
+			interrupts = <0 403 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk", "aclk_xiu";
+			clocks = <&cmu_mscl CLK_PCLK_M2MSCALER1>,
+				 <&cmu_mscl CLK_ACLK_M2MSCALER1>,
+				 <&cmu_mscl CLK_ACLK_XIU_MSCLX>;
+			iommus = <&sysmmu_scaler_1>;
+			power-domains = <&pd_mscl>;
+		};
+
 		jpeg: codec@15020000 {
 			compatible = "samsung,exynos5433-jpeg";
 			reg = <0x15020000 0x10000>;
@@ -1137,6 +1171,28 @@
 			power-domains = <&pd_gscl>;
 		};
 
+		sysmmu_scaler_0: sysmmu@0x15040000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x15040000 0x1000>;
+			interrupts = <GIC_SPI 404 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk";
+			clocks = <&cmu_mscl CLK_PCLK_SMMU_M2MSCALER0>,
+				 <&cmu_mscl CLK_ACLK_SMMU_M2MSCALER0>;
+			#iommu-cells = <0>;
+			power-domains = <&pd_mscl>;
+		};
+
+		sysmmu_scaler_1: sysmmu@0x15050000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x15050000 0x1000>;
+			interrupts = <GIC_SPI 406 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk";
+			clocks = <&cmu_mscl CLK_PCLK_SMMU_M2MSCALER1>,
+				 <&cmu_mscl CLK_ACLK_SMMU_M2MSCALER1>;
+			#iommu-cells = <0>;
+			power-domains = <&pd_mscl>;
+		};
+
 		sysmmu_jpeg: sysmmu@15060000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x15060000 0x1000>;
diff --git a/arch/arm64/boot/dts/exynos/exynos7.dtsi b/arch/arm64/boot/dts/exynos/exynos7.dtsi
index ad9dce6894ce..93a84338938a 100644
--- a/arch/arm64/boot/dts/exynos/exynos7.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7.dtsi
@@ -12,8 +12,8 @@
 / {
 	compatible = "samsung,exynos7";
 	interrupt-parent = <&gic>;
-	#address-cells = <2>;
-	#size-cells = <2>;
+	#address-cells = <1>;
+	#size-cells = <1>;
 
 	aliases {
 		pinctrl0 = &pinctrl_alive;
@@ -70,7 +70,7 @@
 		compatible = "simple-bus";
 		#address-cells = <1>;
 		#size-cells = <1>;
-		ranges = <0 0 0 0x18000000>;
+		ranges;
 
 		chipid@10000000 {
 			compatible = "samsung,exynos4210-chipid";
@@ -494,13 +494,13 @@
 		pmu_system_controller: system-controller@105c0000 {
 			compatible = "samsung,exynos7-pmu", "syscon";
 			reg = <0x105c0000 0x5000>;
-		};
 
-		reboot: syscon-reboot {
-			compatible = "syscon-reboot";
-			regmap = <&pmu_system_controller>;
-			offset = <0x0400>;
-			mask = <0x1>;
+			reboot: syscon-reboot {
+				compatible = "syscon-reboot";
+				regmap = <&pmu_system_controller>;
+				offset = <0x0400>;
+				mask = <0x1>;
+			};
 		};
 
 		rtc: rtc@10590000 {
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1012a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1012a.dtsi
index bb788eddf9f4..205f0f4c5df0 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1012a.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1012a.dtsi
@@ -53,11 +53,11 @@
 
 	aliases {
 		crypto = &crypto;
-		rtic_a = &rtic_a;
-		rtic_b = &rtic_b;
-		rtic_c = &rtic_c;
-		rtic_d = &rtic_d;
-		sec_mon = &sec_mon;
+		rtic-a = &rtic_a;
+		rtic-b = &rtic_b;
+		rtic-c = &rtic_c;
+		rtic-d = &rtic_d;
+		sec-mon = &sec_mon;
 	};
 
 	cpus {
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi
index 5498c705ae6a..061647bd97b8 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi
@@ -134,7 +134,7 @@
 
 &dspi {
 	status = "okay";
-	dflash0: n25q512a {
+	dflash0: n25q512a@0 {
 		#address-cells = <1>;
 		#size-cells = <1>;
 		compatible = "st,m25p80";
diff --git a/arch/arm64/boot/dts/hisilicon/hi3660.dtsi b/arch/arm64/boot/dts/hisilicon/hi3660.dtsi
index ec3eb8e33a3a..8d477dcbfa58 100644
--- a/arch/arm64/boot/dts/hisilicon/hi3660.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hi3660.dtsi
@@ -7,6 +7,7 @@
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/clock/hi3660-clock.h>
+#include <dt-bindings/thermal/thermal.h>
 
 / {
 	compatible = "hisilicon,hi3660";
@@ -62,6 +63,10 @@
 			next-level-cache = <&A53_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_0>;
 			capacity-dmips-mhz = <592>;
+			clocks = <&stub_clock HI3660_CLK_STUB_CLUSTER0>;
+			operating-points-v2 = <&cluster0_opp>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <110>;
 		};
 
 		cpu1: cpu@1 {
@@ -72,6 +77,8 @@
 			next-level-cache = <&A53_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_0>;
 			capacity-dmips-mhz = <592>;
+			clocks = <&stub_clock HI3660_CLK_STUB_CLUSTER0>;
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		cpu2: cpu@2 {
@@ -82,6 +89,8 @@
 			next-level-cache = <&A53_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_0>;
 			capacity-dmips-mhz = <592>;
+			clocks = <&stub_clock HI3660_CLK_STUB_CLUSTER0>;
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		cpu3: cpu@3 {
@@ -92,6 +101,8 @@
 			next-level-cache = <&A53_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_0>;
 			capacity-dmips-mhz = <592>;
+			clocks = <&stub_clock HI3660_CLK_STUB_CLUSTER0>;
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		cpu4: cpu@100 {
@@ -102,6 +113,10 @@
 			next-level-cache = <&A73_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_1>;
 			capacity-dmips-mhz = <1024>;
+			clocks = <&stub_clock HI3660_CLK_STUB_CLUSTER1>;
+			operating-points-v2 = <&cluster1_opp>;
+			#cooling-cells = <2>;
+			dynamic-power-coefficient = <550>;
 		};
 
 		cpu5: cpu@101 {
@@ -112,6 +127,8 @@
 			next-level-cache = <&A73_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_1>;
 			capacity-dmips-mhz = <1024>;
+			clocks = <&stub_clock HI3660_CLK_STUB_CLUSTER1>;
+			operating-points-v2 = <&cluster1_opp>;
 		};
 
 		cpu6: cpu@102 {
@@ -122,6 +139,8 @@
 			next-level-cache = <&A73_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_1>;
 			capacity-dmips-mhz = <1024>;
+			clocks = <&stub_clock HI3660_CLK_STUB_CLUSTER1>;
+			operating-points-v2 = <&cluster1_opp>;
 		};
 
 		cpu7: cpu@103 {
@@ -132,6 +151,8 @@
 			next-level-cache = <&A73_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_1>;
 			capacity-dmips-mhz = <1024>;
+			clocks = <&stub_clock HI3660_CLK_STUB_CLUSTER1>;
+			operating-points-v2 = <&cluster1_opp>;
 		};
 
 		idle-states {
@@ -174,6 +195,76 @@
 		};
 	};
 
+	cluster0_opp: opp_table0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp00 {
+			opp-hz = /bits/ 64 <533000000>;
+			opp-microvolt = <700000>;
+			clock-latency-ns = <300000>;
+		};
+
+		opp01 {
+			opp-hz = /bits/ 64 <999000000>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <300000>;
+		};
+
+		opp02 {
+			opp-hz = /bits/ 64 <1402000000>;
+			opp-microvolt = <900000>;
+			clock-latency-ns = <300000>;
+		};
+
+		opp03 {
+			opp-hz = /bits/ 64 <1709000000>;
+			opp-microvolt = <1000000>;
+			clock-latency-ns = <300000>;
+		};
+
+		opp04 {
+			opp-hz = /bits/ 64 <1844000000>;
+			opp-microvolt = <1100000>;
+			clock-latency-ns = <300000>;
+		};
+	};
+
+	cluster1_opp: opp_table1 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp10 {
+			opp-hz = /bits/ 64 <903000000>;
+			opp-microvolt = <700000>;
+			clock-latency-ns = <300000>;
+		};
+
+		opp11 {
+			opp-hz = /bits/ 64 <1421000000>;
+			opp-microvolt = <800000>;
+			clock-latency-ns = <300000>;
+		};
+
+		opp12 {
+			opp-hz = /bits/ 64 <1805000000>;
+			opp-microvolt = <900000>;
+			clock-latency-ns = <300000>;
+		};
+
+		opp13 {
+			opp-hz = /bits/ 64 <2112000000>;
+			opp-microvolt = <1000000>;
+			clock-latency-ns = <300000>;
+		};
+
+		opp14 {
+			opp-hz = /bits/ 64 <2362000000>;
+			opp-microvolt = <1100000>;
+			clock-latency-ns = <300000>;
+		};
+	};
+
 	gic: interrupt-controller@e82b0000 {
 		compatible = "arm,gic-400";
 		reg = <0x0 0xe82b1000 0 0x1000>, /* GICD */
@@ -274,6 +365,21 @@
 			#reset-cells = <2>;
 		};
 
+		mailbox: mailbox@e896b000 {
+			compatible = "hisilicon,hi3660-mbox";
+			reg = <0x0 0xe896b000 0x0 0x1000>;
+			interrupts = <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <3>;
+		};
+
+		stub_clock: stub_clock@e896b500 {
+			compatible = "hisilicon,hi3660-stub-clk";
+			reg = <0x0 0xe896b500 0x0 0x0100>;
+			#clock-cells = <1>;
+			mboxes = <&mailbox 13 3 0>;
+		};
+
 		dual_timer0: timer@fff14000 {
 			compatible = "arm,sp804", "arm,primecell";
 			reg = <0x0 0xfff14000 0x0 0x1000>;
@@ -872,6 +978,8 @@
 				  0x0 0x02000000>;
 			num-lanes = <1>;
 			#interrupt-cells = <1>;
+			interrupts = <0 283 4>;
+			interrupt-names = "msi";
 			interrupt-map-mask = <0xf800 0 0 7>;
 			interrupt-map = <0x0 0 0 1
 					 &gic GIC_SPI 282 IRQ_TYPE_LEVEL_HIGH>,
@@ -972,5 +1080,44 @@
 			interrupts = <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>;
 			#thermal-sensor-cells = <1>;
 		};
+
+		thermal-zones {
+
+			cls0: cls0 {
+				polling-delay = <1000>;
+				polling-delay-passive = <100>;
+				sustainable-power = <4500>;
+
+				/* sensor ID */
+				thermal-sensors = <&tsensor 1>;
+
+				trips {
+					threshold: trip-point@0 {
+						temperature = <65000>;
+						hysteresis = <1000>;
+						type = "passive";
+					};
+
+					target: trip-point@1 {
+						temperature = <75000>;
+						hysteresis = <1000>;
+						type = "passive";
+					};
+				};
+
+				cooling-maps {
+					map0 {
+						trip = <&target>;
+						contribution = <1024>;
+						cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+					};
+					map1 {
+						trip = <&target>;
+						contribution = <512>;
+						cooling-device = <&cpu4 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+					};
+				};
+			};
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts b/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts
index 4d5d644abb12..d30f6eb8a5ee 100644
--- a/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts
+++ b/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts
@@ -11,6 +11,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include "hi3798cv200.dtsi"
+#include "poplar-pinctrl.dtsi"
 
 / {
 	model = "HiSilicon Poplar Development Board";
@@ -61,6 +62,33 @@
 			default-state = "off";
 		};
 	};
+
+	reg_pcie: regulator-pcie {
+		compatible = "regulator-fixed";
+		regulator-name = "3V3_PCIE0";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio6 7 0>;
+		enable-active-high;
+	};
+};
+
+&ehci {
+	status = "okay";
+};
+
+&emmc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_pins_1 &emmc_pins_2
+		     &emmc_pins_3 &emmc_pins_4>;
+	fifo-depth = <256>;
+	clock-frequency = <200000000>;
+	cap-mmc-highspeed;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+	non-removable;
+	bus-width = <8>;
+	status = "okay";
 };
 
 &gmac1 {
@@ -146,6 +174,16 @@
 	status = "okay";
 };
 
+&ohci {
+	status = "okay";
+};
+
+&pcie {
+	reset-gpios = <&gpio4 4 GPIO_ACTIVE_HIGH>;
+	vpcie-supply = <&reg_pcie>;
+	status = "okay";
+};
+
 &sd0 {
 	bus-width = <4>;
 	cap-sd-highspeed;
diff --git a/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi b/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi
index 962bd79139e4..7c0fddd7c8cf 100644
--- a/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi
@@ -8,7 +8,9 @@
  */
 
 #include <dt-bindings/clock/histb-clock.h>
+#include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/phy/phy.h>
 #include <dt-bindings/reset/ti-syscon.h>
 
 / {
@@ -106,6 +108,113 @@
 			#reset-cells = <2>;
 		};
 
+		perictrl: peripheral-controller@8a20000 {
+			compatible = "hisilicon,hi3798cv200-perictrl", "syscon",
+				     "simple-mfd";
+			reg = <0x8a20000 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x8a20000 0x1000>;
+
+			usb2_phy1: usb2-phy@120 {
+				compatible = "hisilicon,hi3798cv200-usb2-phy";
+				reg = <0x120 0x4>;
+				clocks = <&crg HISTB_USB2_PHY1_REF_CLK>;
+				resets = <&crg 0xbc 4>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				usb2_phy1_port0: phy@0 {
+					reg = <0>;
+					#phy-cells = <0>;
+					resets = <&crg 0xbc 8>;
+				};
+
+				usb2_phy1_port1: phy@1 {
+					reg = <1>;
+					#phy-cells = <0>;
+					resets = <&crg 0xbc 9>;
+				};
+			};
+
+			usb2_phy2: usb2-phy@124 {
+				compatible = "hisilicon,hi3798cv200-usb2-phy";
+				reg = <0x124 0x4>;
+				clocks = <&crg HISTB_USB2_PHY2_REF_CLK>;
+				resets = <&crg 0xbc 6>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				usb2_phy2_port0: phy@0 {
+					reg = <0>;
+					#phy-cells = <0>;
+					resets = <&crg 0xbc 10>;
+				};
+			};
+
+			combphy0: phy@850 {
+				compatible = "hisilicon,hi3798cv200-combphy";
+				reg = <0x850 0x8>;
+				#phy-cells = <1>;
+				clocks = <&crg HISTB_COMBPHY0_CLK>;
+				resets = <&crg 0x188 4>;
+				assigned-clocks = <&crg HISTB_COMBPHY0_CLK>;
+				assigned-clock-rates = <100000000>;
+				hisilicon,fixed-mode = <PHY_TYPE_USB3>;
+			};
+
+			combphy1: phy@858 {
+				compatible = "hisilicon,hi3798cv200-combphy";
+				reg = <0x858 0x8>;
+				#phy-cells = <1>;
+				clocks = <&crg HISTB_COMBPHY1_CLK>;
+				resets = <&crg 0x188 12>;
+				assigned-clocks = <&crg HISTB_COMBPHY1_CLK>;
+				assigned-clock-rates = <100000000>;
+				hisilicon,mode-select-bits = <0x0008 11 (0x3 << 11)>;
+			};
+		};
+
+		pmx0: pinconf@8a21000 {
+			compatible = "pinconf-single";
+			reg = <0x8a21000 0x180>;
+			pinctrl-single,register-width = <32>;
+			pinctrl-single,function-mask = <7>;
+			pinctrl-single,gpio-range = <
+				&range 0  8 2  /* GPIO 0 */
+				&range 8  1 0  /* GPIO 1 */
+				&range 9  4 2
+				&range 13 1 0
+				&range 14 1 1
+				&range 15 1 0
+				&range 16 5 0  /* GPIO 2 */
+				&range 21 3 1
+				&range 24 4 1  /* GPIO 3 */
+				&range 28 2 2
+				&range 86 1 1
+				&range 87 1 0
+				&range 30 4 2  /* GPIO 4 */
+				&range 34 3 0
+				&range 37 1 2
+				&range 38 3 2  /* GPIO 6 */
+				&range 41 5 0
+				&range 46 8 1  /* GPIO 7 */
+				&range 54 8 1  /* GPIO 8 */
+				&range 64 7 1  /* GPIO 9 */
+				&range 71 1 0
+				&range 72 6 1  /* GPIO 10 */
+				&range 78 1 0
+				&range 79 1 1
+				&range 80 6 1  /* GPIO 11 */
+				&range 70 2 1
+				&range 88 8 0  /* GPIO 12 */
+			>;
+
+			range: gpio-range {
+				#pinctrl-single,gpio-range-cells = <3>;
+			};
+		};
+
 		uart0: serial@8b00000 {
 			compatible = "arm,pl011", "arm,primecell";
 			reg = <0x8b00000 0x1000>;
@@ -205,12 +314,17 @@
 		};
 
 		emmc: mmc@9830000 {
-			compatible = "snps,dw-mshc";
+			compatible = "hisilicon,hi3798cv200-dw-mshc";
 			reg = <0x9830000 0x10000>;
 			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&crg HISTB_MMC_CIU_CLK>,
-				 <&crg HISTB_MMC_BIU_CLK>;
-			clock-names = "ciu", "biu";
+				 <&crg HISTB_MMC_BIU_CLK>,
+				 <&crg HISTB_MMC_SAMPLE_CLK>,
+				 <&crg HISTB_MMC_DRV_CLK>;
+			clock-names = "ciu", "biu", "ciu-sample", "ciu-drive";
+			resets = <&crg 0xa0 4>;
+			reset-names = "reset";
+			status = "disabled";
 		};
 
 		gpio0: gpio@8b20000 {
@@ -221,6 +335,7 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <&pmx0 0 0 8>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -234,6 +349,13 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <
+				&pmx0 0 8 1
+				&pmx0 1 9 4
+				&pmx0 5 13 1
+				&pmx0 6 14 1
+				&pmx0 7 15 1
+			>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -247,6 +369,7 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <&pmx0 0 16 5 &pmx0 5 21 3>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -260,6 +383,12 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <
+				&pmx0 0 24 4
+				&pmx0 4 28 2
+				&pmx0 6 86 1
+				&pmx0 7 87 1
+			>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -273,6 +402,7 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <&pmx0 0 30 4 &pmx0 4 34 3 &pmx0 7 37 1>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -299,6 +429,7 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <&pmx0 0 38 3 &pmx0 0 41 5>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -312,6 +443,7 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <&pmx0 0 46 8>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -325,6 +457,7 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <&pmx0 0 54 8>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -338,6 +471,7 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <&pmx0 0 64 7 &pmx0 71 1>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -351,6 +485,7 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <&pmx0 0 72 6 &pmx0 6 78 1 &pmx0 7 79 1>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -364,6 +499,7 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <&pmx0 0 80 6 &pmx0 6 70 2>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -377,6 +513,7 @@
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			gpio-ranges = <&pmx0 0 88 8>;
 			clocks = <&crg HISTB_APB_CLK>;
 			clock-names = "apb_pclk";
 			status = "disabled";
@@ -419,5 +556,67 @@
 			clocks = <&sysctrl HISTB_IR_CLK>;
 			status = "disabled";
 		};
+
+		pcie: pcie@9860000 {
+			compatible = "hisilicon,hi3798cv200-pcie";
+			reg = <0x9860000 0x1000>,
+			      <0x0 0x2000>,
+			      <0x2000000 0x01000000>;
+			reg-names = "control", "rc-dbi", "config";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			device_type = "pci";
+			bus-range = <0 15>;
+			num-lanes = <1>;
+			ranges = <0x81000000 0x0 0x00000000 0x4f00000 0x0 0x100000
+				  0x82000000 0x0 0x3000000 0x3000000 0x0 0x01f00000>;
+			interrupts = <GIC_SPI 128 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi";
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0>;
+			interrupt-map = <0 0 0 0 &gic 0 131 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&crg HISTB_PCIE_AUX_CLK>,
+				 <&crg HISTB_PCIE_PIPE_CLK>,
+				 <&crg HISTB_PCIE_SYS_CLK>,
+				 <&crg HISTB_PCIE_BUS_CLK>;
+			clock-names = "aux", "pipe", "sys", "bus";
+			resets = <&crg 0x18c 6>, <&crg 0x18c 5>, <&crg 0x18c 4>;
+			reset-names = "soft", "sys", "bus";
+			phys = <&combphy1 PHY_TYPE_PCIE>;
+			phy-names = "phy";
+			status = "disabled";
+		};
+
+		ohci: ohci@9880000 {
+			compatible = "generic-ohci";
+			reg = <0x9880000 0x10000>;
+			interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&crg HISTB_USB2_BUS_CLK>,
+				 <&crg HISTB_USB2_12M_CLK>,
+				 <&crg HISTB_USB2_48M_CLK>;
+			clock-names = "bus", "clk12", "clk48";
+			resets = <&crg 0xb8 12>;
+			reset-names = "bus";
+			phys = <&usb2_phy1_port0>;
+			phy-names = "usb";
+			status = "disabled";
+		};
+
+		ehci: ehci@9890000 {
+			compatible = "generic-ehci";
+			reg = <0x9890000 0x10000>;
+			interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&crg HISTB_USB2_BUS_CLK>,
+				 <&crg HISTB_USB2_PHY_CLK>,
+				 <&crg HISTB_USB2_UTMI_CLK>;
+			clock-names = "bus", "phy", "utmi";
+			resets = <&crg 0xb8 12>,
+				 <&crg 0xb8 16>,
+				 <&crg 0xb8 13>;
+			reset-names = "bus", "phy", "utmi";
+			phys = <&usb2_phy1_port0>;
+			phy-names = "usb";
+			status = "disabled";
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/hisilicon/hip06-d03.dts b/arch/arm64/boot/dts/hisilicon/hip06-d03.dts
index 9af633021a42..a95c6f5619bf 100644
--- a/arch/arm64/boot/dts/hisilicon/hip06-d03.dts
+++ b/arch/arm64/boot/dts/hisilicon/hip06-d03.dts
@@ -25,6 +25,14 @@
 	chosen { };
 };
 
+&ipmi0 {
+	status = "ok";
+};
+
+&uart0 {
+	status = "ok";
+};
+
 &eth0 {
 	status = "ok";
 };
diff --git a/arch/arm64/boot/dts/hisilicon/hip06.dtsi b/arch/arm64/boot/dts/hisilicon/hip06.dtsi
index 35202ebe62a7..d78a6a755d03 100644
--- a/arch/arm64/boot/dts/hisilicon/hip06.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hip06.dtsi
@@ -350,6 +350,27 @@
 		#size-cells = <2>;
 		ranges;
 
+		isa@a01b0000 {
+			compatible = "hisilicon,hip06-lpc";
+			#size-cells = <1>;
+			#address-cells = <2>;
+			reg = <0x0 0xa01b0000 0x0 0x1000>;
+
+			ipmi0: bt@e4 {
+				compatible = "ipmi-bt";
+				device_type = "ipmi";
+				reg = <0x01 0xe4 0x04>;
+				status = "disabled";
+			};
+
+			uart0: lpc-uart@2f8 {
+				compatible = "ns16550a";
+				clock-frequency = <1843200>;
+				reg = <0x01 0x2f8 0x08>;
+				status = "disabled";
+			};
+		};
+
 		refclk: refclk {
 			compatible = "fixed-clock";
 			clock-frequency = <50000000>;
diff --git a/arch/arm64/boot/dts/hisilicon/hip07-d05.dts b/arch/arm64/boot/dts/hisilicon/hip07-d05.dts
index fe7c16c36025..21147e8e3f94 100644
--- a/arch/arm64/boot/dts/hisilicon/hip07-d05.dts
+++ b/arch/arm64/boot/dts/hisilicon/hip07-d05.dts
@@ -57,6 +57,10 @@
 	status = "ok";
 };
 
+&ipmi0 {
+	status = "ok";
+};
+
 &usb_ohci {
 	status = "ok";
 };
diff --git a/arch/arm64/boot/dts/hisilicon/hip07.dtsi b/arch/arm64/boot/dts/hisilicon/hip07.dtsi
index 0600a6a84ab7..9c10030a07f8 100644
--- a/arch/arm64/boot/dts/hisilicon/hip07.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hip07.dtsi
@@ -1114,6 +1114,20 @@
 		#size-cells = <2>;
 		ranges;
 
+		isa@a01b0000 {
+			compatible = "hisilicon,hip07-lpc";
+			#size-cells = <1>;
+			#address-cells = <2>;
+			reg = <0x0 0xa01b0000 0x0 0x1000>;
+
+			ipmi0: bt@e4 {
+				compatible = "ipmi-bt";
+				device_type = "ipmi";
+				reg = <0x01 0xe4 0x04>;
+				status = "disabled";
+			};
+		};
+
 		uart0: uart@602b0000 {
 			compatible = "arm,sbsa-uart";
 			reg = <0x0 0x602b0000 0x0 0x1000>;
diff --git a/arch/arm64/boot/dts/hisilicon/poplar-pinctrl.dtsi b/arch/arm64/boot/dts/hisilicon/poplar-pinctrl.dtsi
new file mode 100644
index 000000000000..7bb19e4b084a
--- /dev/null
+++ b/arch/arm64/boot/dts/hisilicon/poplar-pinctrl.dtsi
@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Pinctrl dts file for HiSilicon Poplar board
+ *
+ * Copyright (c) 2016-2018 HiSilicon Technologies Co., Ltd.
+ */
+
+#include <dt-bindings/pinctrl/hisi.h>
+
+/* value, enable bits, disable bits, mask */
+#define PINCTRL_PULLDOWN(value, enable, disable, mask) \
+	(value << 13) (enable << 13) (disable << 13) (mask << 13)
+#define PINCTRL_PULLUP(value, enable, disable, mask) \
+	(value << 12) (enable << 12) (disable << 12) (mask << 12)
+#define PINCTRL_SLEW_RATE(value, mask)	  (value << 8) (mask << 8)
+#define PINCTRL_DRV_STRENGTH(value, mask) (value << 4) (mask << 4)
+
+&pmx0 {
+	emmc_pins_1: emmc-pins-1 {
+		pinctrl-single,pins = <
+			0x000 MUX_M2
+			0x004 MUX_M2
+			0x008 MUX_M2
+			0x00c MUX_M2
+			0x010 MUX_M2
+			0x014 MUX_M2
+			0x018 MUX_M2
+			0x01c MUX_M2
+			0x024 MUX_M2
+		>;
+		pinctrl-single,bias-pulldown = <
+			PINCTRL_PULLDOWN(0, 1, 0, 1)
+		>;
+		pinctrl-single,bias-pullup = <
+			PINCTRL_PULLUP(0, 1, 0, 1)
+		>;
+		pinctrl-single,slew-rate = <
+			PINCTRL_SLEW_RATE(1, 1)
+		>;
+		pinctrl-single,drive-strength = <
+			PINCTRL_DRV_STRENGTH(0xb, 0xf)
+		>;
+	};
+
+	emmc_pins_2: emmc-pins-2 {
+		pinctrl-single,pins = <
+			0x028 MUX_M2
+		>;
+		pinctrl-single,bias-pulldown = <
+			PINCTRL_PULLDOWN(0, 1, 0, 1)
+		>;
+		pinctrl-single,bias-pullup = <
+			PINCTRL_PULLUP(0, 1, 0, 1)
+		>;
+		pinctrl-single,slew-rate = <
+			PINCTRL_SLEW_RATE(1, 1)
+		>;
+		pinctrl-single,drive-strength = <
+			PINCTRL_DRV_STRENGTH(0x9, 0xf)
+		>;
+	};
+
+	emmc_pins_3: emmc-pins-3 {
+		pinctrl-single,pins = <
+			0x02c MUX_M2
+		>;
+		pinctrl-single,bias-pulldown = <
+			PINCTRL_PULLDOWN(0, 1, 0, 1)
+		>;
+		pinctrl-single,bias-pullup = <
+			PINCTRL_PULLUP(0, 1, 0, 1)
+		>;
+		pinctrl-single,slew-rate = <
+			PINCTRL_SLEW_RATE(1, 1)
+		>;
+		pinctrl-single,drive-strength = <
+			PINCTRL_DRV_STRENGTH(3, 3)
+		>;
+	};
+
+	emmc_pins_4: emmc-pins-4 {
+		pinctrl-single,pins = <
+			0x030 MUX_M2
+		>;
+		pinctrl-single,bias-pulldown = <
+			PINCTRL_PULLDOWN(1, 1, 0, 1)
+		>;
+		pinctrl-single,bias-pullup = <
+			PINCTRL_PULLUP(0, 1, 0, 1)
+		>;
+		pinctrl-single,slew-rate = <
+			PINCTRL_SLEW_RATE(1, 1)
+		>;
+		pinctrl-single,drive-strength = <
+			PINCTRL_DRV_STRENGTH(3, 3)
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/marvell/Makefile b/arch/arm64/boot/dts/marvell/Makefile
index cb454beede55..ea9d49f2a911 100644
--- a/arch/arm64/boot/dts/marvell/Makefile
+++ b/arch/arm64/boot/dts/marvell/Makefile
@@ -1,8 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-# Berlin SoC Family
-dtb-$(CONFIG_ARCH_BERLIN) += berlin4ct-dmp.dtb
-dtb-$(CONFIG_ARCH_BERLIN) += berlin4ct-stb.dtb
-
 # Mvebu SoC Family
 dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-db.dtb
 dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-espressobin.dtb
diff --git a/arch/arm64/boot/dts/marvell/armada-3720-espressobin.dts b/arch/arm64/boot/dts/marvell/armada-3720-espressobin.dts
index ef7fd2ca2515..3ab25ad402b9 100644
--- a/arch/arm64/boot/dts/marvell/armada-3720-espressobin.dts
+++ b/arch/arm64/boot/dts/marvell/armada-3720-espressobin.dts
@@ -63,6 +63,33 @@
 	status = "okay";
 };
 
+&spi0 {
+	status = "okay";
+
+	flash@0 {
+		reg = <0>;
+		compatible = "winbond,w25q32dw", "jedec,spi-flash";
+		spi-max-frequency = <104000000>;
+		m25p,fast-read;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "uboot";
+				reg = <0 0x180000>;
+			};
+
+			partition@180000 {
+				label = "ubootenv";
+				reg = <0x180000 0x10000>;
+			};
+		};
+	};
+};
+
 /* Exported on the micro USB connector J5 through an FTDI */
 &uart0 {
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
index 97207a61bc79..3353252d78a0 100644
--- a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
@@ -148,10 +148,13 @@
 				compatible = "marvell,armada3710-nb-pinctrl",
 					     "syscon", "simple-mfd";
 				reg = <0x13800 0x100>, <0x13C00 0x20>;
+				/* MPP1[19:0] */
 				gpionb: gpio {
 					#gpio-cells = <2>;
 					gpio-ranges = <&pinctrl_nb 0 0 36>;
 					gpio-controller;
+					interrupt-controller;
+					#interrupt-cells = <2>;
 					interrupts =
 					<GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
@@ -209,10 +212,13 @@
 				compatible = "marvell,armada3710-sb-pinctrl",
 					     "syscon", "simple-mfd";
 				reg = <0x18800 0x100>, <0x18C00 0x20>;
+				/* MPP2[23:0] */
 				gpiosb: gpio {
 					#gpio-cells = <2>;
 					gpio-ranges = <&pinctrl_sb 0 0 30>;
 					gpio-controller;
+					interrupt-controller;
+					#interrupt-cells = <2>;
 					interrupts =
 					<GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm64/boot/dts/marvell/armada-7040-db.dts b/arch/arm64/boot/dts/marvell/armada-7040-db.dts
index d6bec058a30a..412efdb46e7c 100644
--- a/arch/arm64/boot/dts/marvell/armada-7040-db.dts
+++ b/arch/arm64/boot/dts/marvell/armada-7040-db.dts
@@ -242,6 +242,11 @@
 	phy-mode = "10gbase-kr";
 	/* Generic PHY, providing serdes lanes */
 	phys = <&cp0_comphy2 0>;
+
+	fixed-link {
+		speed = <10000>;
+		full-duplex;
+	};
 };
 
 &cp0_eth1 {
diff --git a/arch/arm64/boot/dts/marvell/armada-8040-db.dts b/arch/arm64/boot/dts/marvell/armada-8040-db.dts
index 5689fb23bbab..1bac437369a1 100644
--- a/arch/arm64/boot/dts/marvell/armada-8040-db.dts
+++ b/arch/arm64/boot/dts/marvell/armada-8040-db.dts
@@ -177,6 +177,11 @@
 &cp0_eth0 {
 	status = "okay";
 	phy-mode = "10gbase-kr";
+
+	fixed-link {
+		speed = <10000>;
+		full-duplex;
+	};
 };
 
 &cp0_eth2 {
@@ -303,6 +308,11 @@
 &cp1_eth0 {
 	status = "okay";
 	phy-mode = "10gbase-kr";
+
+	fixed-link {
+		speed = <10000>;
+		full-duplex;
+	};
 };
 
 &cp1_eth1 {
diff --git a/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dts b/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dts
index 81de03ef860d..a66958ff4de6 100644
--- a/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dts
+++ b/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dts
@@ -27,6 +27,7 @@
 		ethernet0 = &cp0_eth0;
 		ethernet1 = &cp1_eth0;
 		ethernet2 = &cp1_eth1;
+		ethernet3 = &cp1_eth2;
 	};
 
 	/* Regulator labels correspond with schematics */
@@ -64,6 +65,42 @@
 		compatible = "usb-nop-xceiv";
 		vcc-supply = <&v_5v0_usb3_hst_vbus>;
 	};
+
+	sfp_eth0: sfp-eth0 {
+		/* CON15,16 - CPM lane 4 */
+		compatible = "sff,sfp";
+		i2c-bus = <&sfpp0_i2c>;
+		los-gpio = <&cp1_gpio1 28 GPIO_ACTIVE_HIGH>;
+		mod-def0-gpio = <&cp1_gpio1 27 GPIO_ACTIVE_LOW>;
+		tx-disable-gpio = <&cp1_gpio1 29 GPIO_ACTIVE_HIGH>;
+		tx-fault-gpio  = <&cp1_gpio1 26 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cp1_sfpp0_pins>;
+	};
+
+	sfp_eth1: sfp-eth1 {
+		/* CON17,18 - CPS lane 4 */
+		compatible = "sff,sfp";
+		i2c-bus = <&sfpp1_i2c>;
+		los-gpio = <&cp1_gpio1 8 GPIO_ACTIVE_HIGH>;
+		mod-def0-gpio = <&cp1_gpio1 11 GPIO_ACTIVE_LOW>;
+		tx-disable-gpio = <&cp1_gpio1 10 GPIO_ACTIVE_HIGH>;
+		tx-fault-gpio = <&cp0_gpio2 30 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cp1_sfpp1_pins &cp0_sfpp1_pins>;
+	};
+
+	sfp_eth3: sfp-eth3 {
+		/* CON3,4 - CPS lane 5 */
+		compatible = "sff,sfp";
+		i2c-bus = <&sfp_1g_i2c>;
+		los-gpio = <&cp0_gpio2 22 GPIO_ACTIVE_HIGH>;
+		mod-def0-gpio = <&cp0_gpio2 21 GPIO_ACTIVE_LOW>;
+		tx-disable-gpio = <&cp1_gpio1 24 GPIO_ACTIVE_HIGH>;
+		tx-fault-gpio = <&cp0_gpio2 19 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cp0_sfp_1g_pins &cp1_sfp_1g_pins>;
+	};
 };
 
 &uart0 {
@@ -171,6 +208,10 @@
 		marvell,pins = "mpp47";
 		marvell,function = "gpio";
 	};
+	cp0_sfp_1g_pins: sfp-1g-pins {
+		marvell,pins = "mpp51", "mpp53", "mpp54";
+		marvell,function = "gpio";
+	};
 	cp0_pcie_pins: pcie-pins {
 		marvell,pins = "mpp52";
 		marvell,function = "gpio";
@@ -180,6 +221,10 @@
 			       "mpp60", "mpp61";
 		marvell,function = "sdio";
 	};
+	cp0_sfpp1_pins: sfpp1-pins {
+		marvell,pins = "mpp62";
+		marvell,function = "gpio";
+	};
 };
 
 &cp0_xmdio {
@@ -188,11 +233,13 @@
 	phy0: ethernet-phy@0 {
 		compatible = "ethernet-phy-ieee802.3-c45";
 		reg = <0>;
+		sfp = <&sfp_eth0>;
 	};
 
 	phy8: ethernet-phy@8 {
 		compatible = "ethernet-phy-ieee802.3-c45";
 		reg = <8>;
+		sfp = <&sfp_eth1>;
 	};
 };
 
@@ -257,7 +304,22 @@
 	phys = <&cp1_comphy0 1>;
 };
 
+&cp1_eth2 {
+	/* CPS Lane 5 */
+	status = "okay";
+	/* Network PHY */
+	phy-mode = "2500base-x";
+	managed = "in-band-status";
+	/* Generic PHY, providing serdes lanes */
+	phys = <&cp1_comphy5 2>;
+	sfp = <&sfp_eth3>;
+};
+
 &cp1_pinctrl {
+	cp1_sfpp1_pins: sfpp1-pins {
+		marvell,pins = "mpp8", "mpp10", "mpp11";
+		marvell,function = "gpio";
+	};
 	cp1_spi1_pins: spi1-pins {
 		marvell,pins = "mpp12", "mpp13", "mpp14", "mpp15", "mpp16";
 		marvell,function = "spi1";
@@ -266,6 +328,14 @@
 		marvell,pins = "mpp6", "mpp7";
 		marvell,function = "uart0";
 	};
+	cp1_sfp_1g_pins: sfp-1g-pins {
+		marvell,pins = "mpp24";
+		marvell,function = "gpio";
+	};
+	cp1_sfpp0_pins: sfpp0-pins {
+		marvell,pins = "mpp26", "mpp27", "mpp28", "mpp29";
+		marvell,function = "gpio";
+	};
 };
 
 /* J27 UART header */
diff --git a/arch/arm64/boot/dts/marvell/armada-cp110.dtsi b/arch/arm64/boot/dts/marvell/armada-cp110.dtsi
index ed2f1237ea1e..1c6ff8197a88 100644
--- a/arch/arm64/boot/dts/marvell/armada-cp110.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-cp110.dtsi
@@ -149,7 +149,7 @@
 
 		CP110_LABEL(icu): interrupt-controller@1e0000 {
 			compatible = "marvell,cp110-icu";
-			reg = <0x1e0000 0x10>;
+			reg = <0x1e0000 0x440>;
 			#interrupt-cells = <3>;
 			interrupt-controller;
 			msi-parent = <&gicp>;
@@ -236,6 +236,7 @@
 			compatible = "marvell,armada-8k-ahci",
 			"generic-ahci";
 			reg = <0x540000 0x30000>;
+			dma-coherent;
 			interrupts = <ICU_GRP_NSR 107 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&CP110_LABEL(clk) 1 15>,
 				 <&CP110_LABEL(clk) 1 16>;
diff --git a/arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts b/arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts
deleted file mode 100644
index fae6c6924705..000000000000
--- a/arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (C) 2015 Marvell Technology Group Ltd.
- *
- * Author: Jisheng Zhang <jszhang@marvell.com>
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPLv2 or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/dts-v1/;
-
-#include "berlin4ct.dtsi"
-
-/ {
-	model = "Marvell BG4CT DMP board";
-	compatible = "marvell,berlin4ct-dmp", "marvell,berlin4ct", "marvell,berlin";
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-
-	memory@1000000 {
-		device_type = "memory";
-		/* the first 16MB is for firmwares' usage */
-		reg = <0 0x01000000 0 0x7f000000>;
-	};
-};
-
-&uart0 {
-	status = "okay";
-};
diff --git a/arch/arm64/boot/dts/marvell/berlin4ct-stb.dts b/arch/arm64/boot/dts/marvell/berlin4ct-stb.dts
deleted file mode 100644
index d47edad13e68..000000000000
--- a/arch/arm64/boot/dts/marvell/berlin4ct-stb.dts
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (C) 2015 Marvell Technology Group Ltd.
- *
- * Author: Jisheng Zhang <jszhang@marvell.com>
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPLv2 or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/dts-v1/;
-
-#include "berlin4ct.dtsi"
-
-/ {
-	model = "Marvell BG4CT STB board";
-	compatible = "marvell,berlin4ct-stb", "marvell,berlin4ct", "marvell,berlin";
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-
-	memory@1000000 {
-		device_type = "memory";
-		/* the first 16MB is for firmwares' usage */
-		reg = <0 0x01000000 0 0x7f000000>;
-	};
-};
-
-&uart0 {
-	status = "okay";
-};
diff --git a/arch/arm64/boot/dts/marvell/berlin4ct.dtsi b/arch/arm64/boot/dts/marvell/berlin4ct.dtsi
deleted file mode 100644
index d2f88b92d8e2..000000000000
--- a/arch/arm64/boot/dts/marvell/berlin4ct.dtsi
+++ /dev/null
@@ -1,351 +0,0 @@
-/*
- * Copyright (C) 2015 Marvell Technology Group Ltd.
- *
- * Author: Jisheng Zhang <jszhang@marvell.com>
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPLv2 or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include <dt-bindings/interrupt-controller/arm-gic.h>
-
-/ {
-	compatible = "marvell,berlin4ct", "marvell,berlin";
-	interrupt-parent = <&gic>;
-	#address-cells = <2>;
-	#size-cells = <2>;
-
-	aliases {
-		serial0 = &uart0;
-	};
-
-	psci {
-		compatible = "arm,psci-1.0", "arm,psci-0.2";
-		method = "smc";
-	};
-
-	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		cpu0: cpu@0 {
-			compatible = "arm,cortex-a53", "arm,armv8";
-			device_type = "cpu";
-			reg = <0x0>;
-			enable-method = "psci";
-			next-level-cache = <&l2>;
-			cpu-idle-states = <&CPU_SLEEP_0>;
-		};
-
-		cpu1: cpu@1 {
-			compatible = "arm,cortex-a53", "arm,armv8";
-			device_type = "cpu";
-			reg = <0x1>;
-			enable-method = "psci";
-			next-level-cache = <&l2>;
-			cpu-idle-states = <&CPU_SLEEP_0>;
-		};
-
-		cpu2: cpu@2 {
-			compatible = "arm,cortex-a53", "arm,armv8";
-			device_type = "cpu";
-			reg = <0x2>;
-			enable-method = "psci";
-			next-level-cache = <&l2>;
-			cpu-idle-states = <&CPU_SLEEP_0>;
-		};
-
-		cpu3: cpu@3 {
-			compatible = "arm,cortex-a53", "arm,armv8";
-			device_type = "cpu";
-			reg = <0x3>;
-			enable-method = "psci";
-			next-level-cache = <&l2>;
-			cpu-idle-states = <&CPU_SLEEP_0>;
-		};
-
-		l2: cache {
-			compatible = "cache";
-		};
-
-		idle-states {
-			entry-method = "psci";
-			CPU_SLEEP_0: cpu-sleep-0 {
-				compatible = "arm,idle-state";
-				local-timer-stop;
-				arm,psci-suspend-param = <0x0010000>;
-				entry-latency-us = <75>;
-				exit-latency-us = <155>;
-				min-residency-us = <1000>;
-			};
-		};
-	};
-
-	osc: osc {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <25000000>;
-	};
-
-	pmu {
-		compatible = "arm,cortex-a53-pmu", "arm,armv8-pmuv3";
-		interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-affinity = <&cpu0>,
-				     <&cpu1>,
-				     <&cpu2>,
-				     <&cpu3>;
-	};
-
-	timer {
-		compatible = "arm,armv8-timer";
-		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
-	};
-
-	soc@f7000000 {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges = <0 0 0xf7000000 0x1000000>;
-
-		gic: interrupt-controller@901000 {
-			compatible = "arm,gic-400";
-			#interrupt-cells = <3>;
-			interrupt-controller;
-			reg = <0x901000 0x1000>,
-			      <0x902000 0x2000>,
-			      <0x904000 0x2000>,
-			      <0x906000 0x2000>;
-			interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
-		};
-
-		apb@e80000 {
-			compatible = "simple-bus";
-			#address-cells = <1>;
-			#size-cells = <1>;
-
-			ranges = <0 0xe80000 0x10000>;
-			interrupt-parent = <&aic>;
-
-			gpio0: gpio@400 {
-				compatible = "snps,dw-apb-gpio";
-				reg = <0x0400 0x400>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				porta: gpio-port@0 {
-					compatible = "snps,dw-apb-gpio-port";
-					gpio-controller;
-					#gpio-cells = <2>;
-					snps,nr-gpios = <32>;
-					reg = <0>;
-					interrupt-controller;
-					#interrupt-cells = <2>;
-					interrupts = <0>;
-				};
-			};
-
-			gpio1: gpio@800 {
-				compatible = "snps,dw-apb-gpio";
-				reg = <0x0800 0x400>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				portb: gpio-port@1 {
-					compatible = "snps,dw-apb-gpio-port";
-					gpio-controller;
-					#gpio-cells = <2>;
-					snps,nr-gpios = <32>;
-					reg = <0>;
-					interrupt-controller;
-					#interrupt-cells = <2>;
-					interrupts = <1>;
-				};
-			};
-
-			gpio2: gpio@c00 {
-				compatible = "snps,dw-apb-gpio";
-				reg = <0x0c00 0x400>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				portc: gpio-port@2 {
-					compatible = "snps,dw-apb-gpio-port";
-					gpio-controller;
-					#gpio-cells = <2>;
-					snps,nr-gpios = <32>;
-					reg = <0>;
-					interrupt-controller;
-					#interrupt-cells = <2>;
-					interrupts = <2>;
-				};
-			};
-
-			gpio3: gpio@1000 {
-				compatible = "snps,dw-apb-gpio";
-				reg = <0x1000 0x400>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				portd: gpio-port@3 {
-					compatible = "snps,dw-apb-gpio-port";
-					gpio-controller;
-					#gpio-cells = <2>;
-					snps,nr-gpios = <32>;
-					reg = <0>;
-					interrupt-controller;
-					#interrupt-cells = <2>;
-					interrupts = <3>;
-				};
-			};
-
-			aic: interrupt-controller@3800 {
-				compatible = "snps,dw-apb-ictl";
-				reg = <0x3800 0x30>;
-				interrupt-controller;
-				#interrupt-cells = <1>;
-				interrupt-parent = <&gic>;
-				interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
-			};
-		};
-
-		soc_pinctrl: pin-controller@ea8000 {
-			compatible = "marvell,berlin4ct-soc-pinctrl";
-			reg = <0xea8000 0x14>;
-		};
-
-		avio_pinctrl: pin-controller@ea8400 {
-			compatible = "marvell,berlin4ct-avio-pinctrl";
-			reg = <0xea8400 0x8>;
-		};
-
-		apb@fc0000 {
-			compatible = "simple-bus";
-			#address-cells = <1>;
-			#size-cells = <1>;
-			ranges = <0 0xfc0000 0x10000>;
-			interrupt-parent = <&sic>;
-
-			sic: interrupt-controller@1000 {
-				compatible = "snps,dw-apb-ictl";
-				reg = <0x1000 0x30>;
-				interrupt-controller;
-				#interrupt-cells = <1>;
-				interrupt-parent = <&gic>;
-				interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
-			};
-
-			wdt0: watchdog@3000 {
-				compatible = "snps,dw-wdt";
-				reg = <0x3000 0x100>;
-				clocks = <&osc>;
-				interrupts = <0>;
-			};
-
-			wdt1: watchdog@4000 {
-				compatible = "snps,dw-wdt";
-				reg = <0x4000 0x100>;
-				clocks = <&osc>;
-				interrupts = <1>;
-			};
-
-			wdt2: watchdog@5000 {
-				compatible = "snps,dw-wdt";
-				reg = <0x5000 0x100>;
-				clocks = <&osc>;
-				interrupts = <2>;
-			};
-
-			sm_gpio0: gpio@8000 {
-				compatible = "snps,dw-apb-gpio";
-				reg = <0x8000 0x400>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				porte: gpio-port@4 {
-					compatible = "snps,dw-apb-gpio-port";
-					gpio-controller;
-					#gpio-cells = <2>;
-					snps,nr-gpios = <32>;
-					reg = <0>;
-				};
-			};
-
-			sm_gpio1: gpio@9000 {
-				compatible = "snps,dw-apb-gpio";
-				reg = <0x9000 0x400>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				portf: gpio-port@5 {
-					compatible = "snps,dw-apb-gpio-port";
-					gpio-controller;
-					#gpio-cells = <2>;
-					snps,nr-gpios = <32>;
-					reg = <0>;
-				};
-			};
-
-			uart0: uart@d000 {
-				compatible = "snps,dw-apb-uart";
-				reg = <0xd000 0x100>;
-				interrupts = <8>;
-				clocks = <&osc>;
-				reg-shift = <2>;
-				status = "disabled";
-				pinctrl-0 = <&uart0_pmux>;
-				pinctrl-names = "default";
-			};
-		};
-
-		system_pinctrl: pin-controller@fe2200 {
-			compatible = "marvell,berlin4ct-system-pinctrl";
-			reg = <0xfe2200 0xc>;
-
-			uart0_pmux: uart0-pmux {
-				groups = "SM_URT0_TXD", "SM_URT0_RXD";
-				function = "uart0";
-			};
-		};
-	};
-};
diff --git a/arch/arm64/boot/dts/mediatek/mt2712-pinfunc.h b/arch/arm64/boot/dts/mediatek/mt2712-pinfunc.h
new file mode 100644
index 000000000000..1b4cb0c55744
--- /dev/null
+++ b/arch/arm64/boot/dts/mediatek/mt2712-pinfunc.h
@@ -0,0 +1,1123 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 MediaTek Inc.
+ * Author: Zhiyong Tao <zhiyong.tao@mediatek.com>
+ *
+ */
+#ifndef __DTS_MT2712_PINFUNC_H
+#define __DTS_MT2712_PINFUNC_H
+
+#include <dt-bindings/pinctrl/mt65xx.h>
+
+#define MT2712_PIN_0_EINT0__FUNC_GPIO0 (MTK_PIN_NO(0) | 0)
+#define MT2712_PIN_0_EINT0__FUNC_EINT0 (MTK_PIN_NO(0) | 1)
+#define MT2712_PIN_0_EINT0__FUNC_MBIST_DIAG_SCANOUT (MTK_PIN_NO(0) | 2)
+#define MT2712_PIN_0_EINT0__FUNC_DSIA_TE (MTK_PIN_NO(0) | 3)
+#define MT2712_PIN_0_EINT0__FUNC_DSIC_TE (MTK_PIN_NO(0) | 4)
+#define MT2712_PIN_0_EINT0__FUNC_DIN_D3 (MTK_PIN_NO(0) | 5)
+#define MT2712_PIN_0_EINT0__FUNC_PURE_HW_PROTECT (MTK_PIN_NO(0) | 6)
+
+#define MT2712_PIN_1_EINT1__FUNC_GPIO1 (MTK_PIN_NO(1) | 0)
+#define MT2712_PIN_1_EINT1__FUNC_EINT1 (MTK_PIN_NO(1) | 1)
+#define MT2712_PIN_1_EINT1__FUNC_IR_IN (MTK_PIN_NO(1) | 2)
+#define MT2712_PIN_1_EINT1__FUNC_DSIB_TE (MTK_PIN_NO(1) | 3)
+#define MT2712_PIN_1_EINT1__FUNC_DSID_TE (MTK_PIN_NO(1) | 4)
+#define MT2712_PIN_1_EINT1__FUNC_DIN_D4 (MTK_PIN_NO(1) | 5)
+
+#define MT2712_PIN_2_EINT2__FUNC_GPIO2 (MTK_PIN_NO(2) | 0)
+#define MT2712_PIN_2_EINT2__FUNC_EINT2 (MTK_PIN_NO(2) | 1)
+#define MT2712_PIN_2_EINT2__FUNC_IR_IN (MTK_PIN_NO(2) | 2)
+#define MT2712_PIN_2_EINT2__FUNC_LCM_RST1 (MTK_PIN_NO(2) | 3)
+#define MT2712_PIN_2_EINT2__FUNC_DIN_D5 (MTK_PIN_NO(2) | 5)
+
+#define MT2712_PIN_3_EINT3__FUNC_GPIO3 (MTK_PIN_NO(3) | 0)
+#define MT2712_PIN_3_EINT3__FUNC_EINT3 (MTK_PIN_NO(3) | 1)
+#define MT2712_PIN_3_EINT3__FUNC_IR_IN (MTK_PIN_NO(3) | 2)
+#define MT2712_PIN_3_EINT3__FUNC_LCM_RST0 (MTK_PIN_NO(3) | 3)
+#define MT2712_PIN_3_EINT3__FUNC_DIN_D6 (MTK_PIN_NO(3) | 5)
+
+#define MT2712_PIN_4_PWM0__FUNC_GPIO4 (MTK_PIN_NO(4) | 0)
+#define MT2712_PIN_4_PWM0__FUNC_PWM0 (MTK_PIN_NO(4) | 1)
+#define MT2712_PIN_4_PWM0__FUNC_DISP0_PWM (MTK_PIN_NO(4) | 2)
+#define MT2712_PIN_4_PWM0__FUNC_DISP1_PWM (MTK_PIN_NO(4) | 3)
+#define MT2712_PIN_4_PWM0__FUNC_DIN_CLK (MTK_PIN_NO(4) | 5)
+
+#define MT2712_PIN_5_PWM1__FUNC_GPIO5 (MTK_PIN_NO(5) | 0)
+#define MT2712_PIN_5_PWM1__FUNC_PWM1 (MTK_PIN_NO(5) | 1)
+#define MT2712_PIN_5_PWM1__FUNC_DISP1_PWM (MTK_PIN_NO(5) | 2)
+#define MT2712_PIN_5_PWM1__FUNC_DISP0_PWM (MTK_PIN_NO(5) | 3)
+#define MT2712_PIN_5_PWM1__FUNC_DIN_VSYNC (MTK_PIN_NO(5) | 5)
+
+#define MT2712_PIN_6_PWM2__FUNC_GPIO6 (MTK_PIN_NO(6) | 0)
+#define MT2712_PIN_6_PWM2__FUNC_PWM2 (MTK_PIN_NO(6) | 1)
+#define MT2712_PIN_6_PWM2__FUNC_DISP0_PWM (MTK_PIN_NO(6) | 2)
+#define MT2712_PIN_6_PWM2__FUNC_DISP1_PWM (MTK_PIN_NO(6) | 3)
+#define MT2712_PIN_6_PWM2__FUNC_DISP2_PWM (MTK_PIN_NO(6) | 4)
+#define MT2712_PIN_6_PWM2__FUNC_DIN_HSYNC (MTK_PIN_NO(6) | 5)
+
+#define MT2712_PIN_7_PWM3__FUNC_GPIO7 (MTK_PIN_NO(7) | 0)
+#define MT2712_PIN_7_PWM3__FUNC_PWM3 (MTK_PIN_NO(7) | 1)
+#define MT2712_PIN_7_PWM3__FUNC_DISP1_PWM (MTK_PIN_NO(7) | 2)
+#define MT2712_PIN_7_PWM3__FUNC_DISP0_PWM (MTK_PIN_NO(7) | 3)
+#define MT2712_PIN_7_PWM3__FUNC_LCM_RST2 (MTK_PIN_NO(7) | 4)
+#define MT2712_PIN_7_PWM3__FUNC_DIN_D0 (MTK_PIN_NO(7) | 5)
+
+#define MT2712_PIN_8_PWM4__FUNC_GPIO8 (MTK_PIN_NO(8) | 0)
+#define MT2712_PIN_8_PWM4__FUNC_PWM4 (MTK_PIN_NO(8) | 1)
+#define MT2712_PIN_8_PWM4__FUNC_DISP0_PWM (MTK_PIN_NO(8) | 2)
+#define MT2712_PIN_8_PWM4__FUNC_DISP1_PWM (MTK_PIN_NO(8) | 3)
+#define MT2712_PIN_8_PWM4__FUNC_DSIA_TE (MTK_PIN_NO(8) | 4)
+#define MT2712_PIN_8_PWM4__FUNC_DIN_D1 (MTK_PIN_NO(8) | 5)
+
+#define MT2712_PIN_9_PWM5__FUNC_GPIO9 (MTK_PIN_NO(9) | 0)
+#define MT2712_PIN_9_PWM5__FUNC_PWM5 (MTK_PIN_NO(9) | 1)
+#define MT2712_PIN_9_PWM5__FUNC_DISP1_PWM (MTK_PIN_NO(9) | 2)
+#define MT2712_PIN_9_PWM5__FUNC_DISP0_PWM (MTK_PIN_NO(9) | 3)
+#define MT2712_PIN_9_PWM5__FUNC_DSIB_TE (MTK_PIN_NO(9) | 4)
+#define MT2712_PIN_9_PWM5__FUNC_DIN_D2 (MTK_PIN_NO(9) | 5)
+
+#define MT2712_PIN_10_PWM6__FUNC_GPIO10 (MTK_PIN_NO(10) | 0)
+#define MT2712_PIN_10_PWM6__FUNC_PWM6 (MTK_PIN_NO(10) | 1)
+#define MT2712_PIN_10_PWM6__FUNC_DISP0_PWM (MTK_PIN_NO(10) | 2)
+#define MT2712_PIN_10_PWM6__FUNC_DISP1_PWM (MTK_PIN_NO(10) | 3)
+#define MT2712_PIN_10_PWM6__FUNC_LCM_RST0 (MTK_PIN_NO(10) | 4)
+
+#define MT2712_PIN_11_PWM7__FUNC_GPIO11 (MTK_PIN_NO(11) | 0)
+#define MT2712_PIN_11_PWM7__FUNC_PWM7 (MTK_PIN_NO(11) | 1)
+#define MT2712_PIN_11_PWM7__FUNC_DISP1_PWM (MTK_PIN_NO(11) | 2)
+#define MT2712_PIN_11_PWM7__FUNC_DISP0_PWM (MTK_PIN_NO(11) | 3)
+#define MT2712_PIN_11_PWM7__FUNC_LCM_RST1 (MTK_PIN_NO(11) | 4)
+
+#define MT2712_PIN_12_IDDIG_P0__FUNC_GPIO12 (MTK_PIN_NO(12) | 0)
+#define MT2712_PIN_12_IDDIG_P0__FUNC_IDDIG_A (MTK_PIN_NO(12) | 1)
+#define MT2712_PIN_12_IDDIG_P0__FUNC_DIN_D7 (MTK_PIN_NO(12) | 5)
+
+#define MT2712_PIN_13_DRV_VBUS_P0__FUNC_GPIO13 (MTK_PIN_NO(13) | 0)
+#define MT2712_PIN_13_DRV_VBUS_P0__FUNC_DRV_VBUS_A (MTK_PIN_NO(13) | 1)
+
+#define MT2712_PIN_14_IDDIG_P1__FUNC_GPIO14 (MTK_PIN_NO(14) | 0)
+#define MT2712_PIN_14_IDDIG_P1__FUNC_IDDIG_B (MTK_PIN_NO(14) | 1)
+
+#define MT2712_PIN_15_DRV_VBUS_P1__FUNC_GPIO15 (MTK_PIN_NO(15) | 0)
+#define MT2712_PIN_15_DRV_VBUS_P1__FUNC_DRV_VBUS_B (MTK_PIN_NO(15) | 1)
+
+#define MT2712_PIN_16_DRV_VBUS_P2__FUNC_GPIO16 (MTK_PIN_NO(16) | 0)
+#define MT2712_PIN_16_DRV_VBUS_P2__FUNC_DRV_VBUS_C (MTK_PIN_NO(16) | 1)
+
+#define MT2712_PIN_17_DRV_VBUS_P3__FUNC_GPIO17 (MTK_PIN_NO(17) | 0)
+#define MT2712_PIN_17_DRV_VBUS_P3__FUNC_DRV_VBUS_D (MTK_PIN_NO(17) | 1)
+
+#define MT2712_PIN_18_KPROW0__FUNC_GPIO18 (MTK_PIN_NO(18) | 0)
+#define MT2712_PIN_18_KPROW0__FUNC_KROW0 (MTK_PIN_NO(18) | 1)
+
+#define MT2712_PIN_19_KPCOL0__FUNC_GPIO19 (MTK_PIN_NO(19) | 0)
+#define MT2712_PIN_19_KPCOL0__FUNC_KCOL0 (MTK_PIN_NO(19) | 1)
+
+#define MT2712_PIN_20_KPROW1__FUNC_GPIO20 (MTK_PIN_NO(20) | 0)
+#define MT2712_PIN_20_KPROW1__FUNC_KROW1 (MTK_PIN_NO(20) | 1)
+
+#define MT2712_PIN_21_KPCOL1__FUNC_GPIO21 (MTK_PIN_NO(21) | 0)
+#define MT2712_PIN_21_KPCOL1__FUNC_KCOL1 (MTK_PIN_NO(21) | 1)
+
+#define MT2712_PIN_22_KPROW2__FUNC_GPIO22 (MTK_PIN_NO(22) | 0)
+#define MT2712_PIN_22_KPROW2__FUNC_KROW2 (MTK_PIN_NO(22) | 1)
+#define MT2712_PIN_22_KPROW2__FUNC_DISP1_PWM (MTK_PIN_NO(22) | 2)
+
+#define MT2712_PIN_23_KPCOL2__FUNC_GPIO23 (MTK_PIN_NO(23) | 0)
+#define MT2712_PIN_23_KPCOL2__FUNC_KCOL2 (MTK_PIN_NO(23) | 1)
+#define MT2712_PIN_23_KPCOL2__FUNC_DISP0_PWM (MTK_PIN_NO(23) | 2)
+
+#define MT2712_PIN_24_CMMCLK__FUNC_GPIO24 (MTK_PIN_NO(24) | 0)
+#define MT2712_PIN_24_CMMCLK__FUNC_CMMCLK (MTK_PIN_NO(24) | 1)
+#define MT2712_PIN_24_CMMCLK__FUNC_DBG_MON_A_1_ (MTK_PIN_NO(24) | 7)
+
+#define MT2712_PIN_25_CM2MCLK__FUNC_GPIO25 (MTK_PIN_NO(25) | 0)
+#define MT2712_PIN_25_CM2MCLK__FUNC_CM2MCLK (MTK_PIN_NO(25) | 1)
+#define MT2712_PIN_25_CM2MCLK__FUNC_DBG_MON_A_2_ (MTK_PIN_NO(25) | 7)
+
+#define MT2712_PIN_26_PCM_TX__FUNC_GPIO26 (MTK_PIN_NO(26) | 0)
+#define MT2712_PIN_26_PCM_TX__FUNC_PCM1_DO (MTK_PIN_NO(26) | 1)
+#define MT2712_PIN_26_PCM_TX__FUNC_MRG_TX (MTK_PIN_NO(26) | 2)
+#define MT2712_PIN_26_PCM_TX__FUNC_DAI_TX (MTK_PIN_NO(26) | 3)
+#define MT2712_PIN_26_PCM_TX__FUNC_MRG_RX (MTK_PIN_NO(26) | 4)
+#define MT2712_PIN_26_PCM_TX__FUNC_DAI_RX (MTK_PIN_NO(26) | 5)
+#define MT2712_PIN_26_PCM_TX__FUNC_PCM1_DI (MTK_PIN_NO(26) | 6)
+#define MT2712_PIN_26_PCM_TX__FUNC_DBG_MON_A_3_ (MTK_PIN_NO(26) | 7)
+
+#define MT2712_PIN_27_PCM_CLK__FUNC_GPIO27 (MTK_PIN_NO(27) | 0)
+#define MT2712_PIN_27_PCM_CLK__FUNC_PCM1_CLK (MTK_PIN_NO(27) | 1)
+#define MT2712_PIN_27_PCM_CLK__FUNC_MRG_CLK (MTK_PIN_NO(27) | 2)
+#define MT2712_PIN_27_PCM_CLK__FUNC_DAI_CLK (MTK_PIN_NO(27) | 3)
+#define MT2712_PIN_27_PCM_CLK__FUNC_DBG_MON_A_4_ (MTK_PIN_NO(27) | 7)
+
+#define MT2712_PIN_28_PCM_RX__FUNC_GPIO28 (MTK_PIN_NO(28) | 0)
+#define MT2712_PIN_28_PCM_RX__FUNC_PCM1_DI (MTK_PIN_NO(28) | 1)
+#define MT2712_PIN_28_PCM_RX__FUNC_MRG_RX (MTK_PIN_NO(28) | 2)
+#define MT2712_PIN_28_PCM_RX__FUNC_DAI_RX (MTK_PIN_NO(28) | 3)
+#define MT2712_PIN_28_PCM_RX__FUNC_MRG_TX (MTK_PIN_NO(28) | 4)
+#define MT2712_PIN_28_PCM_RX__FUNC_DAI_TX (MTK_PIN_NO(28) | 5)
+#define MT2712_PIN_28_PCM_RX__FUNC_PCM1_DO (MTK_PIN_NO(28) | 6)
+#define MT2712_PIN_28_PCM_RX__FUNC_DBG_MON_A_5_ (MTK_PIN_NO(28) | 7)
+
+#define MT2712_PIN_29_PCM_SYNC__FUNC_GPIO29 (MTK_PIN_NO(29) | 0)
+#define MT2712_PIN_29_PCM_SYNC__FUNC_PCM1_SYNC (MTK_PIN_NO(29) | 1)
+#define MT2712_PIN_29_PCM_SYNC__FUNC_MRG_SYNC (MTK_PIN_NO(29) | 2)
+#define MT2712_PIN_29_PCM_SYNC__FUNC_DAI_SYNC (MTK_PIN_NO(29) | 3)
+#define MT2712_PIN_29_PCM_SYNC__FUNC_DBG_MON_A_6_ (MTK_PIN_NO(29) | 7)
+
+#define MT2712_PIN_30_NCEB0__FUNC_GPIO30 (MTK_PIN_NO(30) | 0)
+#define MT2712_PIN_30_NCEB0__FUNC_NCEB0 (MTK_PIN_NO(30) | 1)
+#define MT2712_PIN_30_NCEB0__FUNC_USB0_FT_SDA (MTK_PIN_NO(30) | 2)
+#define MT2712_PIN_30_NCEB0__FUNC_DBG_MON_A_7_ (MTK_PIN_NO(30) | 7)
+
+#define MT2712_PIN_31_NCEB1__FUNC_GPIO31 (MTK_PIN_NO(31) | 0)
+#define MT2712_PIN_31_NCEB1__FUNC_NCEB1 (MTK_PIN_NO(31) | 1)
+#define MT2712_PIN_31_NCEB1__FUNC_USB1_FT_SCL (MTK_PIN_NO(31) | 2)
+#define MT2712_PIN_31_NCEB1__FUNC_DBG_MON_A_8_ (MTK_PIN_NO(31) | 7)
+
+#define MT2712_PIN_32_NF_DQS__FUNC_GPIO32 (MTK_PIN_NO(32) | 0)
+#define MT2712_PIN_32_NF_DQS__FUNC_NF_DQS (MTK_PIN_NO(32) | 1)
+#define MT2712_PIN_32_NF_DQS__FUNC_USB1_FT_SDA (MTK_PIN_NO(32) | 2)
+#define MT2712_PIN_32_NF_DQS__FUNC_DBG_MON_A_9_ (MTK_PIN_NO(32) | 7)
+
+#define MT2712_PIN_33_NWEB__FUNC_GPIO33 (MTK_PIN_NO(33) | 0)
+#define MT2712_PIN_33_NWEB__FUNC_NWEB (MTK_PIN_NO(33) | 1)
+#define MT2712_PIN_33_NWEB__FUNC_USB2_FT_SCL (MTK_PIN_NO(33) | 2)
+#define MT2712_PIN_33_NWEB__FUNC_DBG_MON_A_10_ (MTK_PIN_NO(33) | 7)
+
+#define MT2712_PIN_34_NREB__FUNC_GPIO34 (MTK_PIN_NO(34) | 0)
+#define MT2712_PIN_34_NREB__FUNC_NREB (MTK_PIN_NO(34) | 1)
+#define MT2712_PIN_34_NREB__FUNC_USB2_FT_SDA (MTK_PIN_NO(34) | 2)
+#define MT2712_PIN_34_NREB__FUNC_DBG_MON_A_11_ (MTK_PIN_NO(34) | 7)
+
+#define MT2712_PIN_35_NCLE__FUNC_GPIO35 (MTK_PIN_NO(35) | 0)
+#define MT2712_PIN_35_NCLE__FUNC_NCLE (MTK_PIN_NO(35) | 1)
+#define MT2712_PIN_35_NCLE__FUNC_USB3_FT_SCL (MTK_PIN_NO(35) | 2)
+#define MT2712_PIN_35_NCLE__FUNC_DBG_MON_A_12_ (MTK_PIN_NO(35) | 7)
+
+#define MT2712_PIN_36_NALE__FUNC_GPIO36 (MTK_PIN_NO(36) | 0)
+#define MT2712_PIN_36_NALE__FUNC_NALE (MTK_PIN_NO(36) | 1)
+#define MT2712_PIN_36_NALE__FUNC_USB3_FT_SDA (MTK_PIN_NO(36) | 2)
+#define MT2712_PIN_36_NALE__FUNC_DBG_MON_A_13_ (MTK_PIN_NO(36) | 7)
+
+#define MT2712_PIN_37_MSDC0E_CLK__FUNC_GPIO37 (MTK_PIN_NO(37) | 0)
+#define MT2712_PIN_37_MSDC0E_CLK__FUNC_MSDC0_CLK (MTK_PIN_NO(37) | 1)
+#define MT2712_PIN_37_MSDC0E_CLK__FUNC_USB0_FT_SCL (MTK_PIN_NO(37) | 2)
+#define MT2712_PIN_37_MSDC0E_CLK__FUNC_DBG_MON_A_0_ (MTK_PIN_NO(37) | 7)
+
+#define MT2712_PIN_38_MSDC0E_DAT7__FUNC_GPIO38 (MTK_PIN_NO(38) | 0)
+#define MT2712_PIN_38_MSDC0E_DAT7__FUNC_MSDC0_DAT7 (MTK_PIN_NO(38) | 1)
+#define MT2712_PIN_38_MSDC0E_DAT7__FUNC_NAND_ND7 (MTK_PIN_NO(38) | 2)
+#define MT2712_PIN_38_MSDC0E_DAT7__FUNC_DBG_MON_A_14_ (MTK_PIN_NO(38) | 7)
+
+#define MT2712_PIN_39_MSDC0E_DAT6__FUNC_GPIO39 (MTK_PIN_NO(39) | 0)
+#define MT2712_PIN_39_MSDC0E_DAT6__FUNC_MSDC0_DAT6 (MTK_PIN_NO(39) | 1)
+#define MT2712_PIN_39_MSDC0E_DAT6__FUNC_NAND_ND6 (MTK_PIN_NO(39) | 2)
+#define MT2712_PIN_39_MSDC0E_DAT6__FUNC_DBG_MON_A_15_ (MTK_PIN_NO(39) | 7)
+
+#define MT2712_PIN_40_MSDC0E_DAT5__FUNC_GPIO40 (MTK_PIN_NO(40) | 0)
+#define MT2712_PIN_40_MSDC0E_DAT5__FUNC_MSDC0_DAT5 (MTK_PIN_NO(40) | 1)
+#define MT2712_PIN_40_MSDC0E_DAT5__FUNC_NAND_ND5 (MTK_PIN_NO(40) | 2)
+#define MT2712_PIN_40_MSDC0E_DAT5__FUNC_DBG_MON_A_16_ (MTK_PIN_NO(40) | 7)
+
+#define MT2712_PIN_41_MSDC0E_DAT4__FUNC_GPIO41 (MTK_PIN_NO(41) | 0)
+#define MT2712_PIN_41_MSDC0E_DAT4__FUNC_MSDC0_DAT4 (MTK_PIN_NO(41) | 1)
+#define MT2712_PIN_41_MSDC0E_DAT4__FUNC_NAND_ND4 (MTK_PIN_NO(41) | 2)
+#define MT2712_PIN_41_MSDC0E_DAT4__FUNC_DBG_MON_A_17_ (MTK_PIN_NO(41) | 7)
+
+#define MT2712_PIN_42_MSDC0E_DAT3__FUNC_GPIO42 (MTK_PIN_NO(42) | 0)
+#define MT2712_PIN_42_MSDC0E_DAT3__FUNC_MSDC0_DAT3 (MTK_PIN_NO(42) | 1)
+#define MT2712_PIN_42_MSDC0E_DAT3__FUNC_NAND_ND3 (MTK_PIN_NO(42) | 2)
+#define MT2712_PIN_42_MSDC0E_DAT3__FUNC_DBG_MON_A_18_ (MTK_PIN_NO(42) | 7)
+
+#define MT2712_PIN_43_MSDC0E_DAT2__FUNC_GPIO43 (MTK_PIN_NO(43) | 0)
+#define MT2712_PIN_43_MSDC0E_DAT2__FUNC_MSDC0_DAT2 (MTK_PIN_NO(43) | 1)
+#define MT2712_PIN_43_MSDC0E_DAT2__FUNC_NAND_ND2 (MTK_PIN_NO(43) | 2)
+#define MT2712_PIN_43_MSDC0E_DAT2__FUNC_DBG_MON_A_19_ (MTK_PIN_NO(43) | 7)
+
+#define MT2712_PIN_44_MSDC0E_DAT1__FUNC_GPIO44 (MTK_PIN_NO(44) | 0)
+#define MT2712_PIN_44_MSDC0E_DAT1__FUNC_MSDC0_DAT1 (MTK_PIN_NO(44) | 1)
+#define MT2712_PIN_44_MSDC0E_DAT1__FUNC_NAND_ND1 (MTK_PIN_NO(44) | 2)
+#define MT2712_PIN_44_MSDC0E_DAT1__FUNC_DBG_MON_A_20_ (MTK_PIN_NO(44) | 7)
+
+#define MT2712_PIN_45_MSDC0E_DAT0__FUNC_GPIO45 (MTK_PIN_NO(45) | 0)
+#define MT2712_PIN_45_MSDC0E_DAT0__FUNC_MSDC0_DAT0 (MTK_PIN_NO(45) | 1)
+#define MT2712_PIN_45_MSDC0E_DAT0__FUNC_NAND_ND0 (MTK_PIN_NO(45) | 2)
+#define MT2712_PIN_45_MSDC0E_DAT0__FUNC_DBG_MON_A_21_ (MTK_PIN_NO(45) | 7)
+
+#define MT2712_PIN_46_MSDC0E_CMD__FUNC_GPIO46 (MTK_PIN_NO(46) | 0)
+#define MT2712_PIN_46_MSDC0E_CMD__FUNC_MSDC0_CMD (MTK_PIN_NO(46) | 1)
+#define MT2712_PIN_46_MSDC0E_CMD__FUNC_NAND_NRNB (MTK_PIN_NO(46) | 2)
+#define MT2712_PIN_46_MSDC0E_CMD__FUNC_DBG_MON_A_22_ (MTK_PIN_NO(46) | 7)
+
+#define MT2712_PIN_47_MSDC0E_DSL__FUNC_GPIO47 (MTK_PIN_NO(47) | 0)
+#define MT2712_PIN_47_MSDC0E_DSL__FUNC_MSDC0_DSL (MTK_PIN_NO(47) | 1)
+#define MT2712_PIN_47_MSDC0E_DSL__FUNC_DBG_MON_A_23_ (MTK_PIN_NO(47) | 7)
+
+#define MT2712_PIN_48_MSDC0E_RSTB__FUNC_GPIO48 (MTK_PIN_NO(48) | 0)
+#define MT2712_PIN_48_MSDC0E_RSTB__FUNC_MSDC0_RSTB (MTK_PIN_NO(48) | 1)
+#define MT2712_PIN_48_MSDC0E_RSTB__FUNC_DBG_MON_A_24_ (MTK_PIN_NO(48) | 7)
+
+#define MT2712_PIN_49_MSDC3_DAT3__FUNC_GPIO49 (MTK_PIN_NO(49) | 0)
+#define MT2712_PIN_49_MSDC3_DAT3__FUNC_MSDC3_DAT3 (MTK_PIN_NO(49) | 1)
+#define MT2712_PIN_49_MSDC3_DAT3__FUNC_DBG_MON_A_25_ (MTK_PIN_NO(49) | 7)
+
+#define MT2712_PIN_50_MSDC3_DAT2__FUNC_GPIO50 (MTK_PIN_NO(50) | 0)
+#define MT2712_PIN_50_MSDC3_DAT2__FUNC_MSDC3_DAT2 (MTK_PIN_NO(50) | 1)
+#define MT2712_PIN_50_MSDC3_DAT2__FUNC_DBG_MON_A_26_ (MTK_PIN_NO(50) | 7)
+
+#define MT2712_PIN_51_MSDC3_DAT1__FUNC_GPIO51 (MTK_PIN_NO(51) | 0)
+#define MT2712_PIN_51_MSDC3_DAT1__FUNC_MSDC3_DAT1 (MTK_PIN_NO(51) | 1)
+#define MT2712_PIN_51_MSDC3_DAT1__FUNC_DBG_MON_A_27_ (MTK_PIN_NO(51) | 7)
+
+#define MT2712_PIN_52_MSDC3_DAT0__FUNC_GPIO52 (MTK_PIN_NO(52) | 0)
+#define MT2712_PIN_52_MSDC3_DAT0__FUNC_MSDC3_DAT0 (MTK_PIN_NO(52) | 1)
+#define MT2712_PIN_52_MSDC3_DAT0__FUNC_DBG_MON_A_28_ (MTK_PIN_NO(52) | 7)
+
+#define MT2712_PIN_53_MSDC3_CMD__FUNC_GPIO53 (MTK_PIN_NO(53) | 0)
+#define MT2712_PIN_53_MSDC3_CMD__FUNC_MSDC3_CMD (MTK_PIN_NO(53) | 1)
+#define MT2712_PIN_53_MSDC3_CMD__FUNC_DBG_MON_A_29_ (MTK_PIN_NO(53) | 7)
+
+#define MT2712_PIN_54_MSDC3_INS__FUNC_GPIO54 (MTK_PIN_NO(54) | 0)
+#define MT2712_PIN_54_MSDC3_INS__FUNC_MSDC3_INS (MTK_PIN_NO(54) | 1)
+#define MT2712_PIN_54_MSDC3_INS__FUNC_DBG_MON_A_30_ (MTK_PIN_NO(54) | 7)
+
+#define MT2712_PIN_55_MSDC3_DSL__FUNC_GPIO55 (MTK_PIN_NO(55) | 0)
+#define MT2712_PIN_55_MSDC3_DSL__FUNC_MSDC3_DSL (MTK_PIN_NO(55) | 1)
+#define MT2712_PIN_55_MSDC3_DSL__FUNC_DBG_MON_A_31_ (MTK_PIN_NO(55) | 7)
+
+#define MT2712_PIN_56_MSDC3_CLK__FUNC_GPIO56 (MTK_PIN_NO(56) | 0)
+#define MT2712_PIN_56_MSDC3_CLK__FUNC_MSDC3_CLK (MTK_PIN_NO(56) | 1)
+#define MT2712_PIN_56_MSDC3_CLK__FUNC_DBG_MON_A_32_ (MTK_PIN_NO(56) | 7)
+
+#define MT2712_PIN_57_NOR_CS__FUNC_GPIO57 (MTK_PIN_NO(57) | 0)
+#define MT2712_PIN_57_NOR_CS__FUNC_NOR_CS (MTK_PIN_NO(57) | 1)
+
+#define MT2712_PIN_58_NOR_CK__FUNC_GPIO58 (MTK_PIN_NO(58) | 0)
+#define MT2712_PIN_58_NOR_CK__FUNC_NOR_CK (MTK_PIN_NO(58) | 1)
+
+#define MT2712_PIN_59_NOR_IO0__FUNC_GPIO59 (MTK_PIN_NO(59) | 0)
+#define MT2712_PIN_59_NOR_IO0__FUNC_NOR_IO0 (MTK_PIN_NO(59) | 1)
+
+#define MT2712_PIN_60_NOR_IO1__FUNC_GPIO60 (MTK_PIN_NO(60) | 0)
+#define MT2712_PIN_60_NOR_IO1__FUNC_NOR_IO1 (MTK_PIN_NO(60) | 1)
+
+#define MT2712_PIN_61_NOR_IO2__FUNC_GPIO61 (MTK_PIN_NO(61) | 0)
+#define MT2712_PIN_61_NOR_IO2__FUNC_NOR_IO2 (MTK_PIN_NO(61) | 1)
+
+#define MT2712_PIN_62_NOR_IO3__FUNC_GPIO62 (MTK_PIN_NO(62) | 0)
+#define MT2712_PIN_62_NOR_IO3__FUNC_NOR_IO3 (MTK_PIN_NO(62) | 1)
+
+#define MT2712_PIN_63_MSDC1_CLK__FUNC_GPIO63 (MTK_PIN_NO(63) | 0)
+#define MT2712_PIN_63_MSDC1_CLK__FUNC_MSDC1_CLK (MTK_PIN_NO(63) | 1)
+#define MT2712_PIN_63_MSDC1_CLK__FUNC_UDI_TCK (MTK_PIN_NO(63) | 2)
+
+#define MT2712_PIN_64_MSDC1_DAT3__FUNC_GPIO64 (MTK_PIN_NO(64) | 0)
+#define MT2712_PIN_64_MSDC1_DAT3__FUNC_MSDC1_DAT3 (MTK_PIN_NO(64) | 1)
+#define MT2712_PIN_64_MSDC1_DAT3__FUNC_UDI_TDI (MTK_PIN_NO(64) | 2)
+
+#define MT2712_PIN_65_MSDC1_DAT1__FUNC_GPIO65 (MTK_PIN_NO(65) | 0)
+#define MT2712_PIN_65_MSDC1_DAT1__FUNC_MSDC1_DAT1 (MTK_PIN_NO(65) | 1)
+#define MT2712_PIN_65_MSDC1_DAT1__FUNC_UDI_TMS (MTK_PIN_NO(65) | 2)
+
+#define MT2712_PIN_66_MSDC1_DAT2__FUNC_GPIO66 (MTK_PIN_NO(66) | 0)
+#define MT2712_PIN_66_MSDC1_DAT2__FUNC_MSDC1_DAT2 (MTK_PIN_NO(66) | 1)
+#define MT2712_PIN_66_MSDC1_DAT2__FUNC_UDI_TDO (MTK_PIN_NO(66) | 2)
+
+#define MT2712_PIN_67_MSDC1_PSW__FUNC_GPIO67 (MTK_PIN_NO(67) | 0)
+#define MT2712_PIN_67_MSDC1_PSW__FUNC_UDI_NTRST (MTK_PIN_NO(67) | 2)
+
+#define MT2712_PIN_68_MSDC1_DAT0__FUNC_GPIO68 (MTK_PIN_NO(68) | 0)
+#define MT2712_PIN_68_MSDC1_DAT0__FUNC_MSDC1_DAT0 (MTK_PIN_NO(68) | 1)
+
+#define MT2712_PIN_69_MSDC1_CMD__FUNC_GPIO69 (MTK_PIN_NO(69) | 0)
+#define MT2712_PIN_69_MSDC1_CMD__FUNC_MSDC1_CMD (MTK_PIN_NO(69) | 1)
+
+#define MT2712_PIN_70_MSDC1_INS__FUNC_GPIO70 (MTK_PIN_NO(70) | 0)
+
+#define MT2712_PIN_71_GBE_TXD3__FUNC_GPIO71 (MTK_PIN_NO(71) | 0)
+#define MT2712_PIN_71_GBE_TXD3__FUNC_GBE_TXD3 (MTK_PIN_NO(71) | 1)
+#define MT2712_PIN_71_GBE_TXD3__FUNC_DBG_MON_B_0_ (MTK_PIN_NO(71) | 7)
+
+#define MT2712_PIN_72_GBE_TXD2__FUNC_GPIO72 (MTK_PIN_NO(72) | 0)
+#define MT2712_PIN_72_GBE_TXD2__FUNC_GBE_TXD2 (MTK_PIN_NO(72) | 1)
+#define MT2712_PIN_72_GBE_TXD2__FUNC_DBG_MON_B_1_ (MTK_PIN_NO(72) | 7)
+
+#define MT2712_PIN_73_GBE_TXD1__FUNC_GPIO73 (MTK_PIN_NO(73) | 0)
+#define MT2712_PIN_73_GBE_TXD1__FUNC_GBE_TXD1 (MTK_PIN_NO(73) | 1)
+#define MT2712_PIN_73_GBE_TXD1__FUNC_DBG_MON_B_2_ (MTK_PIN_NO(73) | 7)
+
+#define MT2712_PIN_74_GBE_TXD0__FUNC_GPIO74 (MTK_PIN_NO(74) | 0)
+#define MT2712_PIN_74_GBE_TXD0__FUNC_GBE_TXD0 (MTK_PIN_NO(74) | 1)
+#define MT2712_PIN_74_GBE_TXD0__FUNC_DBG_MON_B_3_ (MTK_PIN_NO(74) | 7)
+
+#define MT2712_PIN_75_GBE_TXC__FUNC_GPIO75 (MTK_PIN_NO(75) | 0)
+#define MT2712_PIN_75_GBE_TXC__FUNC_GBE_TXC (MTK_PIN_NO(75) | 1)
+#define MT2712_PIN_75_GBE_TXC__FUNC_DBG_MON_B_4_ (MTK_PIN_NO(75) | 7)
+
+#define MT2712_PIN_76_GBE_TXEN__FUNC_GPIO76 (MTK_PIN_NO(76) | 0)
+#define MT2712_PIN_76_GBE_TXEN__FUNC_GBE_TXEN (MTK_PIN_NO(76) | 1)
+#define MT2712_PIN_76_GBE_TXEN__FUNC_DBG_MON_B_5_ (MTK_PIN_NO(76) | 7)
+
+#define MT2712_PIN_77_GBE_TXER__FUNC_GPIO77 (MTK_PIN_NO(77) | 0)
+#define MT2712_PIN_77_GBE_TXER__FUNC_GBE_TXER (MTK_PIN_NO(77) | 1)
+#define MT2712_PIN_77_GBE_TXER__FUNC_DBG_MON_B_6_ (MTK_PIN_NO(77) | 7)
+
+#define MT2712_PIN_78_GBE_RXD3__FUNC_GPIO78 (MTK_PIN_NO(78) | 0)
+#define MT2712_PIN_78_GBE_RXD3__FUNC_GBE_RXD3 (MTK_PIN_NO(78) | 1)
+#define MT2712_PIN_78_GBE_RXD3__FUNC_DBG_MON_B_7_ (MTK_PIN_NO(78) | 7)
+
+#define MT2712_PIN_79_GBE_RXD2__FUNC_GPIO79 (MTK_PIN_NO(79) | 0)
+#define MT2712_PIN_79_GBE_RXD2__FUNC_GBE_RXD2 (MTK_PIN_NO(79) | 1)
+#define MT2712_PIN_79_GBE_RXD2__FUNC_DBG_MON_B_8_ (MTK_PIN_NO(79) | 7)
+
+#define MT2712_PIN_80_GBE_RXD1__FUNC_GPIO80 (MTK_PIN_NO(80) | 0)
+#define MT2712_PIN_80_GBE_RXD1__FUNC_GBE_RXD1 (MTK_PIN_NO(80) | 1)
+#define MT2712_PIN_80_GBE_RXD1__FUNC_DBG_MON_B_9_ (MTK_PIN_NO(80) | 7)
+
+#define MT2712_PIN_81_GBE_RXD0__FUNC_GPIO81 (MTK_PIN_NO(81) | 0)
+#define MT2712_PIN_81_GBE_RXD0__FUNC_GBE_RXD0 (MTK_PIN_NO(81) | 1)
+#define MT2712_PIN_81_GBE_RXD0__FUNC_DBG_MON_B_10_ (MTK_PIN_NO(81) | 7)
+
+#define MT2712_PIN_82_GBE_RXDV__FUNC_GPIO82 (MTK_PIN_NO(82) | 0)
+#define MT2712_PIN_82_GBE_RXDV__FUNC_GBE_RXDV (MTK_PIN_NO(82) | 1)
+#define MT2712_PIN_82_GBE_RXDV__FUNC_DBG_MON_B_11_ (MTK_PIN_NO(82) | 7)
+
+#define MT2712_PIN_83_GBE_RXER__FUNC_GPIO83 (MTK_PIN_NO(83) | 0)
+#define MT2712_PIN_83_GBE_RXER__FUNC_GBE_RXER (MTK_PIN_NO(83) | 1)
+#define MT2712_PIN_83_GBE_RXER__FUNC_DBG_MON_B_12_ (MTK_PIN_NO(83) | 7)
+
+#define MT2712_PIN_84_GBE_RXC__FUNC_GPIO84 (MTK_PIN_NO(84) | 0)
+#define MT2712_PIN_84_GBE_RXC__FUNC_GBE_RXC (MTK_PIN_NO(84) | 1)
+#define MT2712_PIN_84_GBE_RXC__FUNC_DBG_MON_B_13_ (MTK_PIN_NO(84) | 7)
+
+#define MT2712_PIN_85_GBE_MDC__FUNC_GPIO85 (MTK_PIN_NO(85) | 0)
+#define MT2712_PIN_85_GBE_MDC__FUNC_GBE_MDC (MTK_PIN_NO(85) | 1)
+#define MT2712_PIN_85_GBE_MDC__FUNC_DBG_MON_B_14_ (MTK_PIN_NO(85) | 7)
+
+#define MT2712_PIN_86_GBE_MDIO__FUNC_GPIO86 (MTK_PIN_NO(86) | 0)
+#define MT2712_PIN_86_GBE_MDIO__FUNC_GBE_MDIO (MTK_PIN_NO(86) | 1)
+#define MT2712_PIN_86_GBE_MDIO__FUNC_DBG_MON_B_15_ (MTK_PIN_NO(86) | 7)
+
+#define MT2712_PIN_87_GBE_COL__FUNC_GPIO87 (MTK_PIN_NO(87) | 0)
+#define MT2712_PIN_87_GBE_COL__FUNC_GBE_COL (MTK_PIN_NO(87) | 1)
+#define MT2712_PIN_87_GBE_COL__FUNC_DBG_MON_B_16_ (MTK_PIN_NO(87) | 7)
+
+#define MT2712_PIN_88_GBE_INTR__FUNC_GPIO88 (MTK_PIN_NO(88) | 0)
+#define MT2712_PIN_88_GBE_INTR__FUNC_GBE_INTR (MTK_PIN_NO(88) | 1)
+#define MT2712_PIN_88_GBE_INTR__FUNC_GBE_CRS (MTK_PIN_NO(88) | 2)
+#define MT2712_PIN_88_GBE_INTR__FUNC_DBG_MON_B_17_ (MTK_PIN_NO(88) | 7)
+
+#define MT2712_PIN_89_MSDC2_CLK__FUNC_GPIO89 (MTK_PIN_NO(89) | 0)
+#define MT2712_PIN_89_MSDC2_CLK__FUNC_MSDC2_CLK (MTK_PIN_NO(89) | 1)
+#define MT2712_PIN_89_MSDC2_CLK__FUNC_DBG_MON_B_18_ (MTK_PIN_NO(89) | 7)
+
+#define MT2712_PIN_90_MSDC2_DAT3__FUNC_GPIO90 (MTK_PIN_NO(90) | 0)
+#define MT2712_PIN_90_MSDC2_DAT3__FUNC_MSDC2_DAT3 (MTK_PIN_NO(90) | 1)
+#define MT2712_PIN_90_MSDC2_DAT3__FUNC_DBG_MON_B_19_ (MTK_PIN_NO(90) | 7)
+
+#define MT2712_PIN_91_MSDC2_DAT2__FUNC_GPIO91 (MTK_PIN_NO(91) | 0)
+#define MT2712_PIN_91_MSDC2_DAT2__FUNC_MSDC2_DAT2 (MTK_PIN_NO(91) | 1)
+#define MT2712_PIN_91_MSDC2_DAT2__FUNC_DBG_MON_B_20_ (MTK_PIN_NO(91) | 7)
+
+#define MT2712_PIN_92_MSDC2_DAT1__FUNC_GPIO92 (MTK_PIN_NO(92) | 0)
+#define MT2712_PIN_92_MSDC2_DAT1__FUNC_MSDC2_DAT1 (MTK_PIN_NO(92) | 1)
+#define MT2712_PIN_92_MSDC2_DAT1__FUNC_DBG_MON_B_21_ (MTK_PIN_NO(92) | 7)
+
+#define MT2712_PIN_93_MSDC2_DAT0__FUNC_GPIO93 (MTK_PIN_NO(93) | 0)
+#define MT2712_PIN_93_MSDC2_DAT0__FUNC_MSDC2_DAT0 (MTK_PIN_NO(93) | 1)
+#define MT2712_PIN_93_MSDC2_DAT0__FUNC_DBG_MON_B_22_ (MTK_PIN_NO(93) | 7)
+
+#define MT2712_PIN_94_MSDC2_INS__FUNC_GPIO94 (MTK_PIN_NO(94) | 0)
+#define MT2712_PIN_94_MSDC2_INS__FUNC_DBG_MON_B_23_ (MTK_PIN_NO(94) | 7)
+
+#define MT2712_PIN_95_MSDC2_CMD__FUNC_GPIO95 (MTK_PIN_NO(95) | 0)
+#define MT2712_PIN_95_MSDC2_CMD__FUNC_MSDC2_CMD (MTK_PIN_NO(95) | 1)
+#define MT2712_PIN_95_MSDC2_CMD__FUNC_DBG_MON_B_24_ (MTK_PIN_NO(95) | 7)
+
+#define MT2712_PIN_96_MSDC2_PSW__FUNC_GPIO96 (MTK_PIN_NO(96) | 0)
+#define MT2712_PIN_96_MSDC2_PSW__FUNC_DBG_MON_B_25_ (MTK_PIN_NO(96) | 7)
+
+#define MT2712_PIN_97_URXD4__FUNC_GPIO97 (MTK_PIN_NO(97) | 0)
+#define MT2712_PIN_97_URXD4__FUNC_URXD4 (MTK_PIN_NO(97) | 1)
+#define MT2712_PIN_97_URXD4__FUNC_UTXD4 (MTK_PIN_NO(97) | 2)
+#define MT2712_PIN_97_URXD4__FUNC_MRG_CLK (MTK_PIN_NO(97) | 3)
+#define MT2712_PIN_97_URXD4__FUNC_PCM1_CLK (MTK_PIN_NO(97) | 4)
+#define MT2712_PIN_97_URXD4__FUNC_I2S_IQ2_SDQB (MTK_PIN_NO(97) | 5)
+#define MT2712_PIN_97_URXD4__FUNC_I2SO1_WS (MTK_PIN_NO(97) | 6)
+#define MT2712_PIN_97_URXD4__FUNC_DBG_MON_B_26_ (MTK_PIN_NO(97) | 7)
+
+#define MT2712_PIN_98_URTS4__FUNC_GPIO98 (MTK_PIN_NO(98) | 0)
+#define MT2712_PIN_98_URTS4__FUNC_URTS4 (MTK_PIN_NO(98) | 1)
+#define MT2712_PIN_98_URTS4__FUNC_UCTS4 (MTK_PIN_NO(98) | 2)
+#define MT2712_PIN_98_URTS4__FUNC_MRG_RX (MTK_PIN_NO(98) | 3)
+#define MT2712_PIN_98_URTS4__FUNC_PCM1_DI (MTK_PIN_NO(98) | 4)
+#define MT2712_PIN_98_URTS4__FUNC_I2S_IQ1_SDIB (MTK_PIN_NO(98) | 5)
+#define MT2712_PIN_98_URTS4__FUNC_I2SO1_MCK (MTK_PIN_NO(98) | 6)
+#define MT2712_PIN_98_URTS4__FUNC_DBG_MON_B_27_ (MTK_PIN_NO(98) | 7)
+
+#define MT2712_PIN_99_UTXD4__FUNC_GPIO99 (MTK_PIN_NO(99) | 0)
+#define MT2712_PIN_99_UTXD4__FUNC_UTXD4 (MTK_PIN_NO(99) | 1)
+#define MT2712_PIN_99_UTXD4__FUNC_URXD4 (MTK_PIN_NO(99) | 2)
+#define MT2712_PIN_99_UTXD4__FUNC_MRG_SYNC (MTK_PIN_NO(99) | 3)
+#define MT2712_PIN_99_UTXD4__FUNC_PCM1_SYNC (MTK_PIN_NO(99) | 4)
+#define MT2712_PIN_99_UTXD4__FUNC_I2S_IQ0_SDQB (MTK_PIN_NO(99) | 5)
+#define MT2712_PIN_99_UTXD4__FUNC_I2SO1_BCK (MTK_PIN_NO(99) | 6)
+#define MT2712_PIN_99_UTXD4__FUNC_DBG_MON_B_28_ (MTK_PIN_NO(99) | 7)
+
+#define MT2712_PIN_100_UCTS4__FUNC_GPIO100 (MTK_PIN_NO(100) | 0)
+#define MT2712_PIN_100_UCTS4__FUNC_UCTS4 (MTK_PIN_NO(100) | 1)
+#define MT2712_PIN_100_UCTS4__FUNC_URTS4 (MTK_PIN_NO(100) | 2)
+#define MT2712_PIN_100_UCTS4__FUNC_MRG_TX (MTK_PIN_NO(100) | 3)
+#define MT2712_PIN_100_UCTS4__FUNC_PCM1_DO (MTK_PIN_NO(100) | 4)
+#define MT2712_PIN_100_UCTS4__FUNC_I2S_IQ0_SDIB (MTK_PIN_NO(100) | 5)
+#define MT2712_PIN_100_UCTS4__FUNC_I2SO1_DO (MTK_PIN_NO(100) | 6)
+#define MT2712_PIN_100_UCTS4__FUNC_DBG_MON_B_29_ (MTK_PIN_NO(100) | 7)
+
+#define MT2712_PIN_101_URXD5__FUNC_GPIO101 (MTK_PIN_NO(101) | 0)
+#define MT2712_PIN_101_URXD5__FUNC_URXD5 (MTK_PIN_NO(101) | 1)
+#define MT2712_PIN_101_URXD5__FUNC_UTXD5 (MTK_PIN_NO(101) | 2)
+#define MT2712_PIN_101_URXD5__FUNC_I2SO3_WS (MTK_PIN_NO(101) | 3)
+#define MT2712_PIN_101_URXD5__FUNC_TDMIN_LRCK (MTK_PIN_NO(101) | 4)
+#define MT2712_PIN_101_URXD5__FUNC_I2SO0_WS (MTK_PIN_NO(101) | 6)
+#define MT2712_PIN_101_URXD5__FUNC_DBG_MON_B_30_ (MTK_PIN_NO(101) | 7)
+
+#define MT2712_PIN_102_URTS5__FUNC_GPIO102 (MTK_PIN_NO(102) | 0)
+#define MT2712_PIN_102_URTS5__FUNC_URTS5 (MTK_PIN_NO(102) | 1)
+#define MT2712_PIN_102_URTS5__FUNC_UCTS5 (MTK_PIN_NO(102) | 2)
+#define MT2712_PIN_102_URTS5__FUNC_I2SO3_MCK (MTK_PIN_NO(102) | 3)
+#define MT2712_PIN_102_URTS5__FUNC_TDMIN_MCLK (MTK_PIN_NO(102) | 4)
+#define MT2712_PIN_102_URTS5__FUNC_IR_IN (MTK_PIN_NO(102) | 5)
+#define MT2712_PIN_102_URTS5__FUNC_I2SO0_MCK (MTK_PIN_NO(102) | 6)
+#define MT2712_PIN_102_URTS5__FUNC_DBG_MON_B_31_ (MTK_PIN_NO(102) | 7)
+
+#define MT2712_PIN_103_UTXD5__FUNC_GPIO103 (MTK_PIN_NO(103) | 0)
+#define MT2712_PIN_103_UTXD5__FUNC_UTXD5 (MTK_PIN_NO(103) | 1)
+#define MT2712_PIN_103_UTXD5__FUNC_URXD5 (MTK_PIN_NO(103) | 2)
+#define MT2712_PIN_103_UTXD5__FUNC_I2SO3_BCK (MTK_PIN_NO(103) | 3)
+#define MT2712_PIN_103_UTXD5__FUNC_TDMIN_BCK (MTK_PIN_NO(103) | 4)
+#define MT2712_PIN_103_UTXD5__FUNC_I2SO0_BCK (MTK_PIN_NO(103) | 6)
+#define MT2712_PIN_103_UTXD5__FUNC_DBG_MON_B_32_ (MTK_PIN_NO(103) | 7)
+
+#define MT2712_PIN_104_UCTS5__FUNC_GPIO104 (MTK_PIN_NO(104) | 0)
+#define MT2712_PIN_104_UCTS5__FUNC_UCTS5 (MTK_PIN_NO(104) | 1)
+#define MT2712_PIN_104_UCTS5__FUNC_URTS5 (MTK_PIN_NO(104) | 2)
+#define MT2712_PIN_104_UCTS5__FUNC_I2SO0_DO1 (MTK_PIN_NO(104) | 3)
+#define MT2712_PIN_104_UCTS5__FUNC_TDMIN_DI (MTK_PIN_NO(104) | 4)
+#define MT2712_PIN_104_UCTS5__FUNC_IR_IN (MTK_PIN_NO(104) | 5)
+#define MT2712_PIN_104_UCTS5__FUNC_I2SO0_DO0 (MTK_PIN_NO(104) | 6)
+
+#define MT2712_PIN_105_I2C_SDA0__FUNC_GPIO105 (MTK_PIN_NO(105) | 0)
+#define MT2712_PIN_105_I2C_SDA0__FUNC_SDA0 (MTK_PIN_NO(105) | 1)
+
+#define MT2712_PIN_106_I2C_SDA1__FUNC_GPIO106 (MTK_PIN_NO(106) | 0)
+#define MT2712_PIN_106_I2C_SDA1__FUNC_SDA1 (MTK_PIN_NO(106) | 1)
+
+#define MT2712_PIN_107_I2C_SDA2__FUNC_GPIO107 (MTK_PIN_NO(107) | 0)
+#define MT2712_PIN_107_I2C_SDA2__FUNC_SDA2 (MTK_PIN_NO(107) | 1)
+
+#define MT2712_PIN_108_I2C_SDA3__FUNC_GPIO108 (MTK_PIN_NO(108) | 0)
+#define MT2712_PIN_108_I2C_SDA3__FUNC_SDA3 (MTK_PIN_NO(108) | 1)
+
+#define MT2712_PIN_109_I2C_SDA4__FUNC_GPIO109 (MTK_PIN_NO(109) | 0)
+#define MT2712_PIN_109_I2C_SDA4__FUNC_SDA4 (MTK_PIN_NO(109) | 1)
+
+#define MT2712_PIN_110_I2C_SDA5__FUNC_GPIO110 (MTK_PIN_NO(110) | 0)
+#define MT2712_PIN_110_I2C_SDA5__FUNC_SDA5 (MTK_PIN_NO(110) | 1)
+
+#define MT2712_PIN_111_I2C_SCL0__FUNC_GPIO111 (MTK_PIN_NO(111) | 0)
+#define MT2712_PIN_111_I2C_SCL0__FUNC_SCL0 (MTK_PIN_NO(111) | 1)
+
+#define MT2712_PIN_112_I2C_SCL1__FUNC_GPIO112 (MTK_PIN_NO(112) | 0)
+#define MT2712_PIN_112_I2C_SCL1__FUNC_SCL1 (MTK_PIN_NO(112) | 1)
+
+#define MT2712_PIN_113_I2C_SCL2__FUNC_GPIO113 (MTK_PIN_NO(113) | 0)
+#define MT2712_PIN_113_I2C_SCL2__FUNC_SCL2 (MTK_PIN_NO(113) | 1)
+
+#define MT2712_PIN_114_I2C_SCL3__FUNC_GPIO114 (MTK_PIN_NO(114) | 0)
+#define MT2712_PIN_114_I2C_SCL3__FUNC_SCL3 (MTK_PIN_NO(114) | 1)
+
+#define MT2712_PIN_115_I2C_SCL4__FUNC_GPIO115 (MTK_PIN_NO(115) | 0)
+#define MT2712_PIN_115_I2C_SCL4__FUNC_SCL4 (MTK_PIN_NO(115) | 1)
+
+#define MT2712_PIN_116_I2C_SCL5__FUNC_GPIO116 (MTK_PIN_NO(116) | 0)
+#define MT2712_PIN_116_I2C_SCL5__FUNC_SCL5 (MTK_PIN_NO(116) | 1)
+
+#define MT2712_PIN_117_URXD0__FUNC_GPIO117 (MTK_PIN_NO(117) | 0)
+#define MT2712_PIN_117_URXD0__FUNC_URXD0 (MTK_PIN_NO(117) | 1)
+#define MT2712_PIN_117_URXD0__FUNC_UTXD0 (MTK_PIN_NO(117) | 2)
+
+#define MT2712_PIN_118_URXD1__FUNC_GPIO118 (MTK_PIN_NO(118) | 0)
+#define MT2712_PIN_118_URXD1__FUNC_URXD1 (MTK_PIN_NO(118) | 1)
+#define MT2712_PIN_118_URXD1__FUNC_UTXD1 (MTK_PIN_NO(118) | 2)
+
+#define MT2712_PIN_119_URXD2__FUNC_GPIO119 (MTK_PIN_NO(119) | 0)
+#define MT2712_PIN_119_URXD2__FUNC_URXD2 (MTK_PIN_NO(119) | 1)
+#define MT2712_PIN_119_URXD2__FUNC_UTXD2 (MTK_PIN_NO(119) | 2)
+
+#define MT2712_PIN_120_UTXD0__FUNC_GPIO120 (MTK_PIN_NO(120) | 0)
+#define MT2712_PIN_120_UTXD0__FUNC_UTXD0 (MTK_PIN_NO(120) | 1)
+#define MT2712_PIN_120_UTXD0__FUNC_URXD0 (MTK_PIN_NO(120) | 2)
+
+#define MT2712_PIN_121_UTXD1__FUNC_GPIO121 (MTK_PIN_NO(121) | 0)
+#define MT2712_PIN_121_UTXD1__FUNC_UTXD1 (MTK_PIN_NO(121) | 1)
+#define MT2712_PIN_121_UTXD1__FUNC_URXD1 (MTK_PIN_NO(121) | 2)
+
+#define MT2712_PIN_122_UTXD2__FUNC_GPIO122 (MTK_PIN_NO(122) | 0)
+#define MT2712_PIN_122_UTXD2__FUNC_UTXD2 (MTK_PIN_NO(122) | 1)
+#define MT2712_PIN_122_UTXD2__FUNC_URXD2 (MTK_PIN_NO(122) | 2)
+
+#define MT2712_PIN_123_URXD3__FUNC_GPIO123 (MTK_PIN_NO(123) | 0)
+#define MT2712_PIN_123_URXD3__FUNC_URXD3 (MTK_PIN_NO(123) | 1)
+#define MT2712_PIN_123_URXD3__FUNC_UTXD3 (MTK_PIN_NO(123) | 2)
+#define MT2712_PIN_123_URXD3__FUNC_PURE_HW_PROTECT (MTK_PIN_NO(123) | 3)
+
+#define MT2712_PIN_124_UTXD3__FUNC_GPIO124 (MTK_PIN_NO(124) | 0)
+#define MT2712_PIN_124_UTXD3__FUNC_UTXD3 (MTK_PIN_NO(124) | 1)
+#define MT2712_PIN_124_UTXD3__FUNC_URXD3 (MTK_PIN_NO(124) | 2)
+#define MT2712_PIN_124_UTXD3__FUNC_PURE_HW_PROTECT (MTK_PIN_NO(124) | 3)
+
+#define MT2712_PIN_125_URTS3__FUNC_GPIO125 (MTK_PIN_NO(125) | 0)
+#define MT2712_PIN_125_URTS3__FUNC_URTS3 (MTK_PIN_NO(125) | 1)
+#define MT2712_PIN_125_URTS3__FUNC_UCTS3 (MTK_PIN_NO(125) | 2)
+#define MT2712_PIN_125_URTS3__FUNC_WATCH_DOG (MTK_PIN_NO(125) | 3)
+
+#define MT2712_PIN_126_UCTS3__FUNC_GPIO126 (MTK_PIN_NO(126) | 0)
+#define MT2712_PIN_126_UCTS3__FUNC_UCTS3 (MTK_PIN_NO(126) | 1)
+#define MT2712_PIN_126_UCTS3__FUNC_URTS3 (MTK_PIN_NO(126) | 2)
+#define MT2712_PIN_126_UCTS3__FUNC_SRCLKENA0 (MTK_PIN_NO(126) | 3)
+
+#define MT2712_PIN_127_SPI2_CSN__FUNC_GPIO127 (MTK_PIN_NO(127) | 0)
+#define MT2712_PIN_127_SPI2_CSN__FUNC_SPI_CS_2_ (MTK_PIN_NO(127) | 1)
+#define MT2712_PIN_127_SPI2_CSN__FUNC_SPI_CS_1_ (MTK_PIN_NO(127) | 2)
+
+#define MT2712_PIN_128_SPI2_MO__FUNC_GPIO128 (MTK_PIN_NO(128) | 0)
+#define MT2712_PIN_128_SPI2_MO__FUNC_SPI_MO_2_ (MTK_PIN_NO(128) | 1)
+#define MT2712_PIN_128_SPI2_MO__FUNC_SPI_SO_1_ (MTK_PIN_NO(128) | 2)
+
+#define MT2712_PIN_129_SPI2_MI__FUNC_GPIO129 (MTK_PIN_NO(129) | 0)
+#define MT2712_PIN_129_SPI2_MI__FUNC_SPI_MI_2_ (MTK_PIN_NO(129) | 1)
+#define MT2712_PIN_129_SPI2_MI__FUNC_SPI_SI_1_ (MTK_PIN_NO(129) | 2)
+
+#define MT2712_PIN_130_SPI2_CK__FUNC_GPIO130 (MTK_PIN_NO(130) | 0)
+#define MT2712_PIN_130_SPI2_CK__FUNC_SPI_CK_2_ (MTK_PIN_NO(130) | 1)
+#define MT2712_PIN_130_SPI2_CK__FUNC_SPI_CK_1_ (MTK_PIN_NO(130) | 2)
+
+#define MT2712_PIN_131_SPI3_CSN__FUNC_GPIO131 (MTK_PIN_NO(131) | 0)
+#define MT2712_PIN_131_SPI3_CSN__FUNC_SPI_CS_3_ (MTK_PIN_NO(131) | 1)
+
+#define MT2712_PIN_132_SPI3_MO__FUNC_GPIO132 (MTK_PIN_NO(132) | 0)
+#define MT2712_PIN_132_SPI3_MO__FUNC_SPI_MO_3_ (MTK_PIN_NO(132) | 1)
+
+#define MT2712_PIN_133_SPI3_MI__FUNC_GPIO133 (MTK_PIN_NO(133) | 0)
+#define MT2712_PIN_133_SPI3_MI__FUNC_SPI_MI_3_ (MTK_PIN_NO(133) | 1)
+
+#define MT2712_PIN_134_SPI3_CK__FUNC_GPIO134 (MTK_PIN_NO(134) | 0)
+#define MT2712_PIN_134_SPI3_CK__FUNC_SPI_CK_3_ (MTK_PIN_NO(134) | 1)
+
+#define MT2712_PIN_135_KPROW3__FUNC_GPIO135 (MTK_PIN_NO(135) | 0)
+#define MT2712_PIN_135_KPROW3__FUNC_KROW3 (MTK_PIN_NO(135) | 1)
+#define MT2712_PIN_135_KPROW3__FUNC_DSIC_TE (MTK_PIN_NO(135) | 2)
+
+#define MT2712_PIN_136_KPROW4__FUNC_GPIO136 (MTK_PIN_NO(136) | 0)
+#define MT2712_PIN_136_KPROW4__FUNC_KROW4 (MTK_PIN_NO(136) | 1)
+#define MT2712_PIN_136_KPROW4__FUNC_DSID_TE (MTK_PIN_NO(136) | 2)
+
+#define MT2712_PIN_137_KPCOL3__FUNC_GPIO137 (MTK_PIN_NO(137) | 0)
+#define MT2712_PIN_137_KPCOL3__FUNC_KCOL3 (MTK_PIN_NO(137) | 1)
+#define MT2712_PIN_137_KPCOL3__FUNC_DISP2_PWM (MTK_PIN_NO(137) | 2)
+
+#define MT2712_PIN_138_KPCOL4__FUNC_GPIO138 (MTK_PIN_NO(138) | 0)
+#define MT2712_PIN_138_KPCOL4__FUNC_KCOL4 (MTK_PIN_NO(138) | 1)
+#define MT2712_PIN_138_KPCOL4__FUNC_LCM_RST2 (MTK_PIN_NO(138) | 2)
+
+#define MT2712_PIN_139_KPCOL5__FUNC_GPIO139 (MTK_PIN_NO(139) | 0)
+#define MT2712_PIN_139_KPCOL5__FUNC_KCOL5 (MTK_PIN_NO(139) | 1)
+#define MT2712_PIN_139_KPCOL5__FUNC_DSIA_TE (MTK_PIN_NO(139) | 3)
+#define MT2712_PIN_139_KPCOL5__FUNC_PURE_HW_PROTECT (MTK_PIN_NO(139) | 4)
+
+#define MT2712_PIN_140_KPCOL6__FUNC_GPIO140 (MTK_PIN_NO(140) | 0)
+#define MT2712_PIN_140_KPCOL6__FUNC_KCOL6 (MTK_PIN_NO(140) | 1)
+#define MT2712_PIN_140_KPCOL6__FUNC_WATCH_DOG (MTK_PIN_NO(140) | 2)
+#define MT2712_PIN_140_KPCOL6__FUNC_LCM_RST1 (MTK_PIN_NO(140) | 3)
+
+#define MT2712_PIN_141_KPROW5__FUNC_GPIO141 (MTK_PIN_NO(141) | 0)
+#define MT2712_PIN_141_KPROW5__FUNC_KROW5 (MTK_PIN_NO(141) | 1)
+#define MT2712_PIN_141_KPROW5__FUNC_LCM_RST0 (MTK_PIN_NO(141) | 3)
+#define MT2712_PIN_141_KPROW5__FUNC_PURE_HW_PROTECT (MTK_PIN_NO(141) | 4)
+
+#define MT2712_PIN_142_KPROW6__FUNC_GPIO142 (MTK_PIN_NO(142) | 0)
+#define MT2712_PIN_142_KPROW6__FUNC_KROW6 (MTK_PIN_NO(142) | 1)
+#define MT2712_PIN_142_KPROW6__FUNC_SRCLKENA0 (MTK_PIN_NO(142) | 2)
+#define MT2712_PIN_142_KPROW6__FUNC_DSIB_TE (MTK_PIN_NO(142) | 3)
+
+#define MT2712_PIN_143_JTDO_ICE__FUNC_GPIO143 (MTK_PIN_NO(143) | 0)
+#define MT2712_PIN_143_JTDO_ICE__FUNC_JTDO_ICE (MTK_PIN_NO(143) | 1)
+#define MT2712_PIN_143_JTDO_ICE__FUNC_DFD_TDO (MTK_PIN_NO(143) | 3)
+
+#define MT2712_PIN_144_JTCK_ICE__FUNC_GPIO144 (MTK_PIN_NO(144) | 0)
+#define MT2712_PIN_144_JTCK_ICE__FUNC_JTCK_ICE (MTK_PIN_NO(144) | 1)
+#define MT2712_PIN_144_JTCK_ICE__FUNC_DFD_TCK (MTK_PIN_NO(144) | 3)
+
+#define MT2712_PIN_145_JTDI_ICE__FUNC_GPIO145 (MTK_PIN_NO(145) | 0)
+#define MT2712_PIN_145_JTDI_ICE__FUNC_JTDI_ICE (MTK_PIN_NO(145) | 1)
+#define MT2712_PIN_145_JTDI_ICE__FUNC_DFD_TDI (MTK_PIN_NO(145) | 3)
+
+#define MT2712_PIN_146_JTMS_ICE__FUNC_GPIO146 (MTK_PIN_NO(146) | 0)
+#define MT2712_PIN_146_JTMS_ICE__FUNC_JTMS_ICE (MTK_PIN_NO(146) | 1)
+#define MT2712_PIN_146_JTMS_ICE__FUNC_DFD_TMS (MTK_PIN_NO(146) | 3)
+
+#define MT2712_PIN_147_JTRSTB_ICE__FUNC_GPIO147 (MTK_PIN_NO(147) | 0)
+#define MT2712_PIN_147_JTRSTB_ICE__FUNC_JTRST_B_ICE (MTK_PIN_NO(147) | 1)
+#define MT2712_PIN_147_JTRSTB_ICE__FUNC_DFD_NTRST (MTK_PIN_NO(147) | 3)
+
+#define MT2712_PIN_148_GPIO148__FUNC_GPIO148 (MTK_PIN_NO(148) | 0)
+#define MT2712_PIN_148_GPIO148__FUNC_JTRSTB_CM4 (MTK_PIN_NO(148) | 1)
+#define MT2712_PIN_148_GPIO148__FUNC_DFD_NTRST (MTK_PIN_NO(148) | 3)
+
+#define MT2712_PIN_149_GPIO149__FUNC_GPIO149 (MTK_PIN_NO(149) | 0)
+#define MT2712_PIN_149_GPIO149__FUNC_JTCK_CM4 (MTK_PIN_NO(149) | 1)
+#define MT2712_PIN_149_GPIO149__FUNC_DFD_TCK (MTK_PIN_NO(149) | 3)
+
+#define MT2712_PIN_150_GPIO150__FUNC_GPIO150 (MTK_PIN_NO(150) | 0)
+#define MT2712_PIN_150_GPIO150__FUNC_JTMS_CM4 (MTK_PIN_NO(150) | 1)
+#define MT2712_PIN_150_GPIO150__FUNC_DFD_TMS (MTK_PIN_NO(150) | 3)
+
+#define MT2712_PIN_151_GPIO151__FUNC_GPIO151 (MTK_PIN_NO(151) | 0)
+#define MT2712_PIN_151_GPIO151__FUNC_JTDI_CM4 (MTK_PIN_NO(151) | 1)
+#define MT2712_PIN_151_GPIO151__FUNC_DFD_TDI (MTK_PIN_NO(151) | 3)
+
+#define MT2712_PIN_152_GPIO152__FUNC_GPIO152 (MTK_PIN_NO(152) | 0)
+#define MT2712_PIN_152_GPIO152__FUNC_JTDO_CM4 (MTK_PIN_NO(152) | 1)
+#define MT2712_PIN_152_GPIO152__FUNC_DFD_TDO (MTK_PIN_NO(152) | 3)
+
+#define MT2712_PIN_153_SPI0_CSN__FUNC_GPIO153 (MTK_PIN_NO(153) | 0)
+#define MT2712_PIN_153_SPI0_CSN__FUNC_SPI_CS_0_ (MTK_PIN_NO(153) | 1)
+#define MT2712_PIN_153_SPI0_CSN__FUNC_SRCLKENA0 (MTK_PIN_NO(153) | 2)
+#define MT2712_PIN_153_SPI0_CSN__FUNC_UTXD0 (MTK_PIN_NO(153) | 3)
+#define MT2712_PIN_153_SPI0_CSN__FUNC_I2SO0_DO1 (MTK_PIN_NO(153) | 4)
+#define MT2712_PIN_153_SPI0_CSN__FUNC_TDMO0_DATA1 (MTK_PIN_NO(153) | 6)
+#define MT2712_PIN_153_SPI0_CSN__FUNC_I2S_IQ2_SDQB (MTK_PIN_NO(153) | 7)
+
+#define MT2712_PIN_154_SPI0_MI__FUNC_GPIO154 (MTK_PIN_NO(154) | 0)
+#define MT2712_PIN_154_SPI0_MI__FUNC_SPI_MI_0_ (MTK_PIN_NO(154) | 1)
+#define MT2712_PIN_154_SPI0_MI__FUNC_SRCLKENA0 (MTK_PIN_NO(154) | 2)
+#define MT2712_PIN_154_SPI0_MI__FUNC_URXD0 (MTK_PIN_NO(154) | 3)
+#define MT2712_PIN_154_SPI0_MI__FUNC_I2SO0_DO0 (MTK_PIN_NO(154) | 4)
+#define MT2712_PIN_154_SPI0_MI__FUNC_I2SO1_DO (MTK_PIN_NO(154) | 5)
+#define MT2712_PIN_154_SPI0_MI__FUNC_TDMO0_DATA (MTK_PIN_NO(154) | 6)
+#define MT2712_PIN_154_SPI0_MI__FUNC_I2S_IQ1_SDIB (MTK_PIN_NO(154) | 7)
+
+#define MT2712_PIN_155_SPI0_CK__FUNC_GPIO155 (MTK_PIN_NO(155) | 0)
+#define MT2712_PIN_155_SPI0_CK__FUNC_SPI_CK_0_ (MTK_PIN_NO(155) | 1)
+#define MT2712_PIN_155_SPI0_CK__FUNC_SC_APBIAS_OFF (MTK_PIN_NO(155) | 2)
+#define MT2712_PIN_155_SPI0_CK__FUNC_UTXD1 (MTK_PIN_NO(155) | 3)
+#define MT2712_PIN_155_SPI0_CK__FUNC_I2SO0_BCK (MTK_PIN_NO(155) | 4)
+#define MT2712_PIN_155_SPI0_CK__FUNC_I2SO1_BCK (MTK_PIN_NO(155) | 5)
+#define MT2712_PIN_155_SPI0_CK__FUNC_TDMO0_BCK (MTK_PIN_NO(155) | 6)
+#define MT2712_PIN_155_SPI0_CK__FUNC_I2S_IQ0_SDQB (MTK_PIN_NO(155) | 7)
+
+#define MT2712_PIN_156_SPI0_MO__FUNC_GPIO156 (MTK_PIN_NO(156) | 0)
+#define MT2712_PIN_156_SPI0_MO__FUNC_SPI_MO_0_ (MTK_PIN_NO(156) | 1)
+#define MT2712_PIN_156_SPI0_MO__FUNC_SC_APBIAS_OFF (MTK_PIN_NO(156) | 2)
+#define MT2712_PIN_156_SPI0_MO__FUNC_URXD1 (MTK_PIN_NO(156) | 3)
+#define MT2712_PIN_156_SPI0_MO__FUNC_I2SO0_WS (MTK_PIN_NO(156) | 4)
+#define MT2712_PIN_156_SPI0_MO__FUNC_I2SO1_WS (MTK_PIN_NO(156) | 5)
+#define MT2712_PIN_156_SPI0_MO__FUNC_TDMO0_LRCK (MTK_PIN_NO(156) | 6)
+#define MT2712_PIN_156_SPI0_MO__FUNC_I2S_IQ0_SDIB (MTK_PIN_NO(156) | 7)
+
+#define MT2712_PIN_157_SPI5_CSN__FUNC_GPIO157 (MTK_PIN_NO(157) | 0)
+#define MT2712_PIN_157_SPI5_CSN__FUNC_SPI_CS_5_ (MTK_PIN_NO(157) | 1)
+#define MT2712_PIN_157_SPI5_CSN__FUNC_LCM_RST0 (MTK_PIN_NO(157) | 2)
+#define MT2712_PIN_157_SPI5_CSN__FUNC_UTXD2 (MTK_PIN_NO(157) | 3)
+#define MT2712_PIN_157_SPI5_CSN__FUNC_I2SO0_MCK (MTK_PIN_NO(157) | 4)
+#define MT2712_PIN_157_SPI5_CSN__FUNC_I2SO1_MCK (MTK_PIN_NO(157) | 5)
+#define MT2712_PIN_157_SPI5_CSN__FUNC_TDMO0_MCLK (MTK_PIN_NO(157) | 6)
+
+#define MT2712_PIN_158_SPI5_MI__FUNC_GPIO158 (MTK_PIN_NO(158) | 0)
+#define MT2712_PIN_158_SPI5_MI__FUNC_SPI_MI_5_ (MTK_PIN_NO(158) | 1)
+#define MT2712_PIN_158_SPI5_MI__FUNC_DSIA_TE (MTK_PIN_NO(158) | 2)
+#define MT2712_PIN_158_SPI5_MI__FUNC_URXD2 (MTK_PIN_NO(158) | 3)
+
+#define MT2712_PIN_159_SPI5_MO__FUNC_GPIO159 (MTK_PIN_NO(159) | 0)
+#define MT2712_PIN_159_SPI5_MO__FUNC_SPI_MO_5_ (MTK_PIN_NO(159) | 1)
+#define MT2712_PIN_159_SPI5_MO__FUNC_DSIB_TE (MTK_PIN_NO(159) | 2)
+#define MT2712_PIN_159_SPI5_MO__FUNC_UTXD3 (MTK_PIN_NO(159) | 3)
+
+#define MT2712_PIN_160_SPI5_CK__FUNC_GPIO160 (MTK_PIN_NO(160) | 0)
+#define MT2712_PIN_160_SPI5_CK__FUNC_SPI_CK_5_ (MTK_PIN_NO(160) | 1)
+#define MT2712_PIN_160_SPI5_CK__FUNC_LCM_RST1 (MTK_PIN_NO(160) | 2)
+#define MT2712_PIN_160_SPI5_CK__FUNC_URXD3 (MTK_PIN_NO(160) | 3)
+
+#define MT2712_PIN_161_SPI1_CSN__FUNC_GPIO161 (MTK_PIN_NO(161) | 0)
+#define MT2712_PIN_161_SPI1_CSN__FUNC_SPI_CS_1_ (MTK_PIN_NO(161) | 1)
+#define MT2712_PIN_161_SPI1_CSN__FUNC_SPI_CS_4_ (MTK_PIN_NO(161) | 2)
+#define MT2712_PIN_161_SPI1_CSN__FUNC_I2S_IQ2_SDQB (MTK_PIN_NO(161) | 4)
+#define MT2712_PIN_161_SPI1_CSN__FUNC_I2SO2_DO (MTK_PIN_NO(161) | 5)
+#define MT2712_PIN_161_SPI1_CSN__FUNC_TDMO0_DATA1 (MTK_PIN_NO(161) | 6)
+#define MT2712_PIN_161_SPI1_CSN__FUNC_I2SO0_DO1 (MTK_PIN_NO(161) | 7)
+
+#define MT2712_PIN_162_SPI1_SI__FUNC_GPIO162 (MTK_PIN_NO(162) | 0)
+#define MT2712_PIN_162_SPI1_SI__FUNC_SPI_SI_1_ (MTK_PIN_NO(162) | 1)
+#define MT2712_PIN_162_SPI1_SI__FUNC_SPI_MI_4_ (MTK_PIN_NO(162) | 2)
+#define MT2712_PIN_162_SPI1_SI__FUNC_I2S_IQ1_SDIB (MTK_PIN_NO(162) | 4)
+#define MT2712_PIN_162_SPI1_SI__FUNC_I2SO2_BCK (MTK_PIN_NO(162) | 5)
+#define MT2712_PIN_162_SPI1_SI__FUNC_TDMO0_DATA (MTK_PIN_NO(162) | 6)
+#define MT2712_PIN_162_SPI1_SI__FUNC_I2SO0_DO0 (MTK_PIN_NO(162) | 7)
+
+#define MT2712_PIN_163_SPI1_CK__FUNC_GPIO163 (MTK_PIN_NO(163) | 0)
+#define MT2712_PIN_163_SPI1_CK__FUNC_SPI_CK_1_ (MTK_PIN_NO(163) | 1)
+#define MT2712_PIN_163_SPI1_CK__FUNC_SPI_CK_4_ (MTK_PIN_NO(163) | 2)
+#define MT2712_PIN_163_SPI1_CK__FUNC_I2S_IQ0_SDQB (MTK_PIN_NO(163) | 4)
+#define MT2712_PIN_163_SPI1_CK__FUNC_I2SO2_WS (MTK_PIN_NO(163) | 5)
+#define MT2712_PIN_163_SPI1_CK__FUNC_TDMO0_BCK (MTK_PIN_NO(163) | 6)
+#define MT2712_PIN_163_SPI1_CK__FUNC_I2SO0_BCK (MTK_PIN_NO(163) | 7)
+
+#define MT2712_PIN_164_SPI1_SO__FUNC_GPIO164 (MTK_PIN_NO(164) | 0)
+#define MT2712_PIN_164_SPI1_SO__FUNC_SPI_SO_1_ (MTK_PIN_NO(164) | 1)
+#define MT2712_PIN_164_SPI1_SO__FUNC_SPI_MO_4_ (MTK_PIN_NO(164) | 2)
+#define MT2712_PIN_164_SPI1_SO__FUNC_I2S_IQ0_SDIB (MTK_PIN_NO(164) | 4)
+#define MT2712_PIN_164_SPI1_SO__FUNC_I2SO2_MCK (MTK_PIN_NO(164) | 5)
+#define MT2712_PIN_164_SPI1_SO__FUNC_TDMO0_LRCK (MTK_PIN_NO(164) | 6)
+#define MT2712_PIN_164_SPI1_SO__FUNC_I2SO0_WS (MTK_PIN_NO(164) | 7)
+
+#define MT2712_PIN_165_SPI4_CSN__FUNC_GPIO165 (MTK_PIN_NO(165) | 0)
+#define MT2712_PIN_165_SPI4_CSN__FUNC_SPI_CS_4_ (MTK_PIN_NO(165) | 1)
+#define MT2712_PIN_165_SPI4_CSN__FUNC_LCM_RST0 (MTK_PIN_NO(165) | 2)
+#define MT2712_PIN_165_SPI4_CSN__FUNC_SPI_CS_1_ (MTK_PIN_NO(165) | 3)
+#define MT2712_PIN_165_SPI4_CSN__FUNC_UTXD4 (MTK_PIN_NO(165) | 4)
+#define MT2712_PIN_165_SPI4_CSN__FUNC_I2SO1_DO (MTK_PIN_NO(165) | 5)
+#define MT2712_PIN_165_SPI4_CSN__FUNC_TDMO0_MCLK (MTK_PIN_NO(165) | 6)
+#define MT2712_PIN_165_SPI4_CSN__FUNC_I2SO0_MCK (MTK_PIN_NO(165) | 7)
+
+#define MT2712_PIN_166_SPI4_MI__FUNC_GPIO166 (MTK_PIN_NO(166) | 0)
+#define MT2712_PIN_166_SPI4_MI__FUNC_SPI_MI_4_ (MTK_PIN_NO(166) | 1)
+#define MT2712_PIN_166_SPI4_MI__FUNC_DSIA_TE (MTK_PIN_NO(166) | 2)
+#define MT2712_PIN_166_SPI4_MI__FUNC_SPI_SI_1_ (MTK_PIN_NO(166) | 3)
+#define MT2712_PIN_166_SPI4_MI__FUNC_URXD4 (MTK_PIN_NO(166) | 4)
+#define MT2712_PIN_166_SPI4_MI__FUNC_I2SO1_BCK (MTK_PIN_NO(166) | 5)
+
+#define MT2712_PIN_167_SPI4_MO__FUNC_GPIO167 (MTK_PIN_NO(167) | 0)
+#define MT2712_PIN_167_SPI4_MO__FUNC_SPI_MO_4_ (MTK_PIN_NO(167) | 1)
+#define MT2712_PIN_167_SPI4_MO__FUNC_DSIB_TE (MTK_PIN_NO(167) | 2)
+#define MT2712_PIN_167_SPI4_MO__FUNC_SPI_SO_1_ (MTK_PIN_NO(167) | 3)
+#define MT2712_PIN_167_SPI4_MO__FUNC_UTXD5 (MTK_PIN_NO(167) | 4)
+#define MT2712_PIN_167_SPI4_MO__FUNC_I2SO1_WS (MTK_PIN_NO(167) | 5)
+
+#define MT2712_PIN_168_SPI4_CK__FUNC_GPIO168 (MTK_PIN_NO(168) | 0)
+#define MT2712_PIN_168_SPI4_CK__FUNC_SPI_CK_4_ (MTK_PIN_NO(168) | 1)
+#define MT2712_PIN_168_SPI4_CK__FUNC_LCM_RST1 (MTK_PIN_NO(168) | 2)
+#define MT2712_PIN_168_SPI4_CK__FUNC_SPI_CK_1_ (MTK_PIN_NO(168) | 3)
+#define MT2712_PIN_168_SPI4_CK__FUNC_URXD5 (MTK_PIN_NO(168) | 4)
+#define MT2712_PIN_168_SPI4_CK__FUNC_I2SO1_MCK (MTK_PIN_NO(168) | 5)
+
+#define MT2712_PIN_169_I2SI0_DATA__FUNC_GPIO169 (MTK_PIN_NO(169) | 0)
+#define MT2712_PIN_169_I2SI0_DATA__FUNC_I2SI0_DI (MTK_PIN_NO(169) | 1)
+#define MT2712_PIN_169_I2SI0_DATA__FUNC_I2SI1_DI (MTK_PIN_NO(169) | 2)
+#define MT2712_PIN_169_I2SI0_DATA__FUNC_I2SI2_DI (MTK_PIN_NO(169) | 3)
+#define MT2712_PIN_169_I2SI0_DATA__FUNC_TDMIN_DI (MTK_PIN_NO(169) | 4)
+
+#define MT2712_PIN_170_I2SI0_LRCK__FUNC_GPIO170 (MTK_PIN_NO(170) | 0)
+#define MT2712_PIN_170_I2SI0_LRCK__FUNC_I2SI0_WS (MTK_PIN_NO(170) | 1)
+#define MT2712_PIN_170_I2SI0_LRCK__FUNC_I2SI1_WS (MTK_PIN_NO(170) | 2)
+#define MT2712_PIN_170_I2SI0_LRCK__FUNC_I2SI2_WS (MTK_PIN_NO(170) | 3)
+#define MT2712_PIN_170_I2SI0_LRCK__FUNC_TDMIN_LRCK (MTK_PIN_NO(170) | 4)
+#define MT2712_PIN_170_I2SI0_LRCK__FUNC_TDMO0_DATA3 (MTK_PIN_NO(170) | 5)
+#define MT2712_PIN_170_I2SI0_LRCK__FUNC_TDMO1_DATA3 (MTK_PIN_NO(170) | 6)
+
+#define MT2712_PIN_171_I2SI0_MCLK__FUNC_GPIO171 (MTK_PIN_NO(171) | 0)
+#define MT2712_PIN_171_I2SI0_MCLK__FUNC_I2SI0_MCK (MTK_PIN_NO(171) | 1)
+#define MT2712_PIN_171_I2SI0_MCLK__FUNC_I2SI1_MCK (MTK_PIN_NO(171) | 2)
+#define MT2712_PIN_171_I2SI0_MCLK__FUNC_I2SI2_MCK (MTK_PIN_NO(171) | 3)
+#define MT2712_PIN_171_I2SI0_MCLK__FUNC_TDMIN_MCLK (MTK_PIN_NO(171) | 4)
+#define MT2712_PIN_171_I2SI0_MCLK__FUNC_TDMO0_DATA2 (MTK_PIN_NO(171) | 5)
+#define MT2712_PIN_171_I2SI0_MCLK__FUNC_TDMO1_DATA2 (MTK_PIN_NO(171) | 6)
+
+#define MT2712_PIN_172_I2SI0_BCK__FUNC_GPIO172 (MTK_PIN_NO(172) | 0)
+#define MT2712_PIN_172_I2SI0_BCK__FUNC_I2SI0_BCK (MTK_PIN_NO(172) | 1)
+#define MT2712_PIN_172_I2SI0_BCK__FUNC_I2SI1_BCK (MTK_PIN_NO(172) | 2)
+#define MT2712_PIN_172_I2SI0_BCK__FUNC_I2SI2_BCK (MTK_PIN_NO(172) | 3)
+#define MT2712_PIN_172_I2SI0_BCK__FUNC_TDMIN_BCK (MTK_PIN_NO(172) | 4)
+#define MT2712_PIN_172_I2SI0_BCK__FUNC_TDMO0_DATA1 (MTK_PIN_NO(172) | 5)
+#define MT2712_PIN_172_I2SI0_BCK__FUNC_TDMO1_DATA1 (MTK_PIN_NO(172) | 6)
+
+#define MT2712_PIN_173_I2SI2_DATA__FUNC_GPIO173 (MTK_PIN_NO(173) | 0)
+#define MT2712_PIN_173_I2SI2_DATA__FUNC_I2SI2_DI (MTK_PIN_NO(173) | 1)
+#define MT2712_PIN_173_I2SI2_DATA__FUNC_I2SI0_DI (MTK_PIN_NO(173) | 2)
+#define MT2712_PIN_173_I2SI2_DATA__FUNC_I2SI1_DI (MTK_PIN_NO(173) | 3)
+#define MT2712_PIN_173_I2SI2_DATA__FUNC_PCM1_DI (MTK_PIN_NO(173) | 4)
+#define MT2712_PIN_173_I2SI2_DATA__FUNC_TDMIN_DI (MTK_PIN_NO(173) | 5)
+#define MT2712_PIN_173_I2SI2_DATA__FUNC_PCM1_DO (MTK_PIN_NO(173) | 6)
+
+#define MT2712_PIN_174_I2SI2_MCLK__FUNC_GPIO174 (MTK_PIN_NO(174) | 0)
+#define MT2712_PIN_174_I2SI2_MCLK__FUNC_I2SI2_MCK (MTK_PIN_NO(174) | 1)
+#define MT2712_PIN_174_I2SI2_MCLK__FUNC_I2SI0_MCK (MTK_PIN_NO(174) | 2)
+#define MT2712_PIN_174_I2SI2_MCLK__FUNC_I2SI1_MCK (MTK_PIN_NO(174) | 3)
+#define MT2712_PIN_174_I2SI2_MCLK__FUNC_PCM1_DO (MTK_PIN_NO(174) | 4)
+#define MT2712_PIN_174_I2SI2_MCLK__FUNC_TDMIN_MCLK (MTK_PIN_NO(174) | 5)
+#define MT2712_PIN_174_I2SI2_MCLK__FUNC_PCM1_DI (MTK_PIN_NO(174) | 6)
+#define MT2712_PIN_174_I2SI2_MCLK__FUNC_I2S_IQ2_SDQB (MTK_PIN_NO(174) | 7)
+
+#define MT2712_PIN_175_I2SI2_BCK__FUNC_GPIO175 (MTK_PIN_NO(175) | 0)
+#define MT2712_PIN_175_I2SI2_BCK__FUNC_I2SI2_BCK (MTK_PIN_NO(175) | 1)
+#define MT2712_PIN_175_I2SI2_BCK__FUNC_I2SI0_BCK (MTK_PIN_NO(175) | 2)
+#define MT2712_PIN_175_I2SI2_BCK__FUNC_I2SI1_BCK (MTK_PIN_NO(175) | 3)
+#define MT2712_PIN_175_I2SI2_BCK__FUNC_PCM1_CLK (MTK_PIN_NO(175) | 4)
+#define MT2712_PIN_175_I2SI2_BCK__FUNC_TDMIN_BCK (MTK_PIN_NO(175) | 5)
+
+#define MT2712_PIN_176_I2SI2_LRCK__FUNC_GPIO176 (MTK_PIN_NO(176) | 0)
+#define MT2712_PIN_176_I2SI2_LRCK__FUNC_I2SI2_WS (MTK_PIN_NO(176) | 1)
+#define MT2712_PIN_176_I2SI2_LRCK__FUNC_I2SI0_WS (MTK_PIN_NO(176) | 2)
+#define MT2712_PIN_176_I2SI2_LRCK__FUNC_I2SI1_WS (MTK_PIN_NO(176) | 3)
+#define MT2712_PIN_176_I2SI2_LRCK__FUNC_PCM1_SYNC (MTK_PIN_NO(176) | 4)
+#define MT2712_PIN_176_I2SI2_LRCK__FUNC_TDMIN_LRCK (MTK_PIN_NO(176) | 5)
+
+#define MT2712_PIN_177_I2SI1_DATA__FUNC_GPIO177 (MTK_PIN_NO(177) | 0)
+#define MT2712_PIN_177_I2SI1_DATA__FUNC_I2SI1_DI (MTK_PIN_NO(177) | 1)
+#define MT2712_PIN_177_I2SI1_DATA__FUNC_I2SI0_DI (MTK_PIN_NO(177) | 2)
+#define MT2712_PIN_177_I2SI1_DATA__FUNC_I2SI2_DI (MTK_PIN_NO(177) | 3)
+#define MT2712_PIN_177_I2SI1_DATA__FUNC_TDMIN_DI (MTK_PIN_NO(177) | 4)
+
+#define MT2712_PIN_178_I2SI1_BCK__FUNC_GPIO178 (MTK_PIN_NO(178) | 0)
+#define MT2712_PIN_178_I2SI1_BCK__FUNC_I2SI1_BCK (MTK_PIN_NO(178) | 1)
+#define MT2712_PIN_178_I2SI1_BCK__FUNC_I2SI0_BCK (MTK_PIN_NO(178) | 2)
+#define MT2712_PIN_178_I2SI1_BCK__FUNC_I2SI2_BCK (MTK_PIN_NO(178) | 3)
+#define MT2712_PIN_178_I2SI1_BCK__FUNC_TDMIN_BCK (MTK_PIN_NO(178) | 4)
+#define MT2712_PIN_178_I2SI1_BCK__FUNC_TDMO0_DATA3 (MTK_PIN_NO(178) | 5)
+#define MT2712_PIN_178_I2SI1_BCK__FUNC_TDMO1_DATA3 (MTK_PIN_NO(178) | 6)
+
+#define MT2712_PIN_179_I2SI1_LRCK__FUNC_GPIO179 (MTK_PIN_NO(179) | 0)
+#define MT2712_PIN_179_I2SI1_LRCK__FUNC_I2SI1_WS (MTK_PIN_NO(179) | 1)
+#define MT2712_PIN_179_I2SI1_LRCK__FUNC_I2SI0_WS (MTK_PIN_NO(179) | 2)
+#define MT2712_PIN_179_I2SI1_LRCK__FUNC_I2SI2_WS (MTK_PIN_NO(179) | 3)
+#define MT2712_PIN_179_I2SI1_LRCK__FUNC_TDMIN_LRCK (MTK_PIN_NO(179) | 4)
+#define MT2712_PIN_179_I2SI1_LRCK__FUNC_TDMO0_DATA2 (MTK_PIN_NO(179) | 5)
+#define MT2712_PIN_179_I2SI1_LRCK__FUNC_TDMO1_DATA2 (MTK_PIN_NO(179) | 6)
+
+#define MT2712_PIN_180_I2SI1_MCLK__FUNC_GPIO180 (MTK_PIN_NO(180) | 0)
+#define MT2712_PIN_180_I2SI1_MCLK__FUNC_I2SI1_MCK (MTK_PIN_NO(180) | 1)
+#define MT2712_PIN_180_I2SI1_MCLK__FUNC_I2SI0_MCK (MTK_PIN_NO(180) | 2)
+#define MT2712_PIN_180_I2SI1_MCLK__FUNC_I2SI2_MCK (MTK_PIN_NO(180) | 3)
+#define MT2712_PIN_180_I2SI1_MCLK__FUNC_TDMIN_MCLK (MTK_PIN_NO(180) | 4)
+#define MT2712_PIN_180_I2SI1_MCLK__FUNC_TDMO0_DATA1 (MTK_PIN_NO(180) | 5)
+#define MT2712_PIN_180_I2SI1_MCLK__FUNC_TDMO1_DATA1 (MTK_PIN_NO(180) | 6)
+#define MT2712_PIN_180_I2SI1_MCLK__FUNC_I2S_IQ2_SDIB (MTK_PIN_NO(180) | 7)
+
+#define MT2712_PIN_181_I2SO1_DATA0__FUNC_GPIO181 (MTK_PIN_NO(181) | 0)
+#define MT2712_PIN_181_I2SO1_DATA0__FUNC_I2SO1_DO (MTK_PIN_NO(181) | 1)
+#define MT2712_PIN_181_I2SO1_DATA0__FUNC_I2SO0_DO0 (MTK_PIN_NO(181) | 2)
+#define MT2712_PIN_181_I2SO1_DATA0__FUNC_I2SO2_DO (MTK_PIN_NO(181) | 3)
+#define MT2712_PIN_181_I2SO1_DATA0__FUNC_DAI_TX (MTK_PIN_NO(181) | 4)
+#define MT2712_PIN_181_I2SO1_DATA0__FUNC_TDMIN_MCLK (MTK_PIN_NO(181) | 5)
+#define MT2712_PIN_181_I2SO1_DATA0__FUNC_I2S_IQ2_SDIA (MTK_PIN_NO(181) | 7)
+
+#define MT2712_PIN_182_I2SO1_BCK__FUNC_GPIO182 (MTK_PIN_NO(182) | 0)
+#define MT2712_PIN_182_I2SO1_BCK__FUNC_I2SO1_BCK (MTK_PIN_NO(182) | 1)
+#define MT2712_PIN_182_I2SO1_BCK__FUNC_I2SO0_BCK (MTK_PIN_NO(182) | 2)
+#define MT2712_PIN_182_I2SO1_BCK__FUNC_I2SO2_BCK (MTK_PIN_NO(182) | 3)
+#define MT2712_PIN_182_I2SO1_BCK__FUNC_DAI_SYNC (MTK_PIN_NO(182) | 4)
+#define MT2712_PIN_182_I2SO1_BCK__FUNC_TDMIN_BCK (MTK_PIN_NO(182) | 5)
+#define MT2712_PIN_182_I2SO1_BCK__FUNC_TDMO0_DATA3 (MTK_PIN_NO(182) | 6)
+#define MT2712_PIN_182_I2SO1_BCK__FUNC_I2S_IQ2_BCK (MTK_PIN_NO(182) | 7)
+
+#define MT2712_PIN_183_I2SO1_LRCK__FUNC_GPIO183 (MTK_PIN_NO(183) | 0)
+#define MT2712_PIN_183_I2SO1_LRCK__FUNC_I2SO1_WS (MTK_PIN_NO(183) | 1)
+#define MT2712_PIN_183_I2SO1_LRCK__FUNC_I2SO0_WS (MTK_PIN_NO(183) | 2)
+#define MT2712_PIN_183_I2SO1_LRCK__FUNC_I2SO2_WS (MTK_PIN_NO(183) | 3)
+#define MT2712_PIN_183_I2SO1_LRCK__FUNC_DAI_CLK (MTK_PIN_NO(183) | 4)
+#define MT2712_PIN_183_I2SO1_LRCK__FUNC_TDMIN_DI (MTK_PIN_NO(183) | 5)
+#define MT2712_PIN_183_I2SO1_LRCK__FUNC_TDMO0_DATA2 (MTK_PIN_NO(183) | 6)
+#define MT2712_PIN_183_I2SO1_LRCK__FUNC_I2S_IQ2_WS (MTK_PIN_NO(183) | 7)
+
+#define MT2712_PIN_184_I2SO1_MCLK__FUNC_GPIO184 (MTK_PIN_NO(184) | 0)
+#define MT2712_PIN_184_I2SO1_MCLK__FUNC_I2SO1_MCK (MTK_PIN_NO(184) | 1)
+#define MT2712_PIN_184_I2SO1_MCLK__FUNC_I2SO0_MCK (MTK_PIN_NO(184) | 2)
+#define MT2712_PIN_184_I2SO1_MCLK__FUNC_I2SO2_MCK (MTK_PIN_NO(184) | 3)
+#define MT2712_PIN_184_I2SO1_MCLK__FUNC_DAI_RX (MTK_PIN_NO(184) | 4)
+#define MT2712_PIN_184_I2SO1_MCLK__FUNC_TDMIN_LRCK (MTK_PIN_NO(184) | 5)
+#define MT2712_PIN_184_I2SO1_MCLK__FUNC_TDMO0_DATA1 (MTK_PIN_NO(184) | 6)
+#define MT2712_PIN_184_I2SO1_MCLK__FUNC_I2S_IQ2_SDQA (MTK_PIN_NO(184) | 7)
+
+#define MT2712_PIN_185_AUD_EXT_CK2__FUNC_GPIO185 (MTK_PIN_NO(185) | 0)
+#define MT2712_PIN_185_AUD_EXT_CK2__FUNC_AUD_EXT_CK2 (MTK_PIN_NO(185) | 1)
+#define MT2712_PIN_185_AUD_EXT_CK2__FUNC_AUD_EXT_CK1 (MTK_PIN_NO(185) | 2)
+#define MT2712_PIN_185_AUD_EXT_CK2__FUNC_I2SO1_DO (MTK_PIN_NO(185) | 3)
+#define MT2712_PIN_185_AUD_EXT_CK2__FUNC_I2SI2_DI (MTK_PIN_NO(185) | 4)
+#define MT2712_PIN_185_AUD_EXT_CK2__FUNC_MRG_RX (MTK_PIN_NO(185) | 5)
+#define MT2712_PIN_185_AUD_EXT_CK2__FUNC_PCM1_DI (MTK_PIN_NO(185) | 6)
+#define MT2712_PIN_185_AUD_EXT_CK2__FUNC_I2S_IQ0_SDQB (MTK_PIN_NO(185) | 7)
+
+#define MT2712_PIN_186_AUD_EXT_CK1__FUNC_GPIO186 (MTK_PIN_NO(186) | 0)
+#define MT2712_PIN_186_AUD_EXT_CK1__FUNC_AUD_EXT_CK1 (MTK_PIN_NO(186) | 1)
+#define MT2712_PIN_186_AUD_EXT_CK1__FUNC_AUD_EXT_CK2 (MTK_PIN_NO(186) | 2)
+#define MT2712_PIN_186_AUD_EXT_CK1__FUNC_I2SO0_DO1 (MTK_PIN_NO(186) | 3)
+#define MT2712_PIN_186_AUD_EXT_CK1__FUNC_I2SI1_DI (MTK_PIN_NO(186) | 4)
+#define MT2712_PIN_186_AUD_EXT_CK1__FUNC_MRG_TX (MTK_PIN_NO(186) | 5)
+#define MT2712_PIN_186_AUD_EXT_CK1__FUNC_PCM1_DO (MTK_PIN_NO(186) | 6)
+#define MT2712_PIN_186_AUD_EXT_CK1__FUNC_I2S_IQ0_SDIB (MTK_PIN_NO(186) | 7)
+
+#define MT2712_PIN_187_I2SO2_BCK__FUNC_GPIO187 (MTK_PIN_NO(187) | 0)
+#define MT2712_PIN_187_I2SO2_BCK__FUNC_I2SO2_BCK (MTK_PIN_NO(187) | 1)
+#define MT2712_PIN_187_I2SO2_BCK__FUNC_I2SO0_BCK (MTK_PIN_NO(187) | 2)
+#define MT2712_PIN_187_I2SO2_BCK__FUNC_I2SO1_BCK (MTK_PIN_NO(187) | 3)
+#define MT2712_PIN_187_I2SO2_BCK__FUNC_PCM1_CLK (MTK_PIN_NO(187) | 4)
+#define MT2712_PIN_187_I2SO2_BCK__FUNC_MRG_SYNC (MTK_PIN_NO(187) | 5)
+#define MT2712_PIN_187_I2SO2_BCK__FUNC_TDMO1_DATA3 (MTK_PIN_NO(187) | 6)
+#define MT2712_PIN_187_I2SO2_BCK__FUNC_I2S_IQ0_BCK (MTK_PIN_NO(187) | 7)
+
+#define MT2712_PIN_188_I2SO2_LRCK__FUNC_GPIO188 (MTK_PIN_NO(188) | 0)
+#define MT2712_PIN_188_I2SO2_LRCK__FUNC_I2SO2_WS (MTK_PIN_NO(188) | 1)
+#define MT2712_PIN_188_I2SO2_LRCK__FUNC_I2SO0_WS (MTK_PIN_NO(188) | 2)
+#define MT2712_PIN_188_I2SO2_LRCK__FUNC_I2SO1_WS (MTK_PIN_NO(188) | 3)
+#define MT2712_PIN_188_I2SO2_LRCK__FUNC_PCM1_SYNC (MTK_PIN_NO(188) | 4)
+#define MT2712_PIN_188_I2SO2_LRCK__FUNC_MRG_CLK (MTK_PIN_NO(188) | 5)
+#define MT2712_PIN_188_I2SO2_LRCK__FUNC_TDMO1_DATA2 (MTK_PIN_NO(188) | 6)
+#define MT2712_PIN_188_I2SO2_LRCK__FUNC_I2S_IQ0_WS (MTK_PIN_NO(188) | 7)
+
+#define MT2712_PIN_189_I2SO2_MCLK__FUNC_GPIO189 (MTK_PIN_NO(189) | 0)
+#define MT2712_PIN_189_I2SO2_MCLK__FUNC_I2SO2_MCK (MTK_PIN_NO(189) | 1)
+#define MT2712_PIN_189_I2SO2_MCLK__FUNC_I2SO0_MCK (MTK_PIN_NO(189) | 2)
+#define MT2712_PIN_189_I2SO2_MCLK__FUNC_I2SO1_MCK (MTK_PIN_NO(189) | 3)
+#define MT2712_PIN_189_I2SO2_MCLK__FUNC_PCM1_DO (MTK_PIN_NO(189) | 4)
+#define MT2712_PIN_189_I2SO2_MCLK__FUNC_MRG_RX (MTK_PIN_NO(189) | 5)
+#define MT2712_PIN_189_I2SO2_MCLK__FUNC_TDMO1_DATA1 (MTK_PIN_NO(189) | 6)
+#define MT2712_PIN_189_I2SO2_MCLK__FUNC_I2S_IQ0_SDQA (MTK_PIN_NO(189) | 7)
+
+#define MT2712_PIN_190_I2SO2_DATA0__FUNC_GPIO190 (MTK_PIN_NO(190) | 0)
+#define MT2712_PIN_190_I2SO2_DATA0__FUNC_I2SO2_DO (MTK_PIN_NO(190) | 1)
+#define MT2712_PIN_190_I2SO2_DATA0__FUNC_I2SO0_DO0 (MTK_PIN_NO(190) | 2)
+#define MT2712_PIN_190_I2SO2_DATA0__FUNC_I2SO1_DO (MTK_PIN_NO(190) | 3)
+#define MT2712_PIN_190_I2SO2_DATA0__FUNC_PCM1_DI (MTK_PIN_NO(190) | 4)
+#define MT2712_PIN_190_I2SO2_DATA0__FUNC_MRG_TX (MTK_PIN_NO(190) | 5)
+#define MT2712_PIN_190_I2SO2_DATA0__FUNC_PCM1_DO (MTK_PIN_NO(190) | 6)
+#define MT2712_PIN_190_I2SO2_DATA0__FUNC_I2S_IQ0_SDIA (MTK_PIN_NO(190) | 7)
+
+#define MT2712_PIN_191_I2SO0_DATA1__FUNC_GPIO191 (MTK_PIN_NO(191) | 0)
+#define MT2712_PIN_191_I2SO0_DATA1__FUNC_I2SO0_DO1 (MTK_PIN_NO(191) | 1)
+#define MT2712_PIN_191_I2SO0_DATA1__FUNC_I2SI0_DI (MTK_PIN_NO(191) | 2)
+#define MT2712_PIN_191_I2SO0_DATA1__FUNC_I2SI1_DI (MTK_PIN_NO(191) | 3)
+#define MT2712_PIN_191_I2SO0_DATA1__FUNC_I2SI2_DI (MTK_PIN_NO(191) | 4)
+#define MT2712_PIN_191_I2SO0_DATA1__FUNC_DAI_TX (MTK_PIN_NO(191) | 5)
+#define MT2712_PIN_191_I2SO0_DATA1__FUNC_I2S_IQ0_SDQB (MTK_PIN_NO(191) | 6)
+#define MT2712_PIN_191_I2SO0_DATA1__FUNC_I2S_IQ1_SDQB (MTK_PIN_NO(191) | 7)
+
+#define MT2712_PIN_192_I2SO0_MCLK__FUNC_GPIO192 (MTK_PIN_NO(192) | 0)
+#define MT2712_PIN_192_I2SO0_MCLK__FUNC_I2SO0_MCK (MTK_PIN_NO(192) | 1)
+#define MT2712_PIN_192_I2SO0_MCLK__FUNC_I2SO1_MCK (MTK_PIN_NO(192) | 2)
+#define MT2712_PIN_192_I2SO0_MCLK__FUNC_I2SO2_MCK (MTK_PIN_NO(192) | 3)
+#define MT2712_PIN_192_I2SO0_MCLK__FUNC_USB4_FT_SCL (MTK_PIN_NO(192) | 4)
+#define MT2712_PIN_192_I2SO0_MCLK__FUNC_TDMO1_DATA3 (MTK_PIN_NO(192) | 5)
+#define MT2712_PIN_192_I2SO0_MCLK__FUNC_I2S_IQ0_SDIB (MTK_PIN_NO(192) | 6)
+#define MT2712_PIN_192_I2SO0_MCLK__FUNC_I2S_IQ1_SDQA (MTK_PIN_NO(192) | 7)
+
+#define MT2712_PIN_193_I2SO0_DATA0__FUNC_GPIO193 (MTK_PIN_NO(193) | 0)
+#define MT2712_PIN_193_I2SO0_DATA0__FUNC_I2SO0_DO0 (MTK_PIN_NO(193) | 1)
+#define MT2712_PIN_193_I2SO0_DATA0__FUNC_I2SO1_DO (MTK_PIN_NO(193) | 2)
+#define MT2712_PIN_193_I2SO0_DATA0__FUNC_I2SO2_DO (MTK_PIN_NO(193) | 3)
+#define MT2712_PIN_193_I2SO0_DATA0__FUNC_USB4_FT_SDA (MTK_PIN_NO(193) | 4)
+#define MT2712_PIN_193_I2SO0_DATA0__FUNC_I2S_IQ1_SDIA (MTK_PIN_NO(193) | 7)
+
+#define MT2712_PIN_194_I2SO0_LRCK__FUNC_GPIO194 (MTK_PIN_NO(194) | 0)
+#define MT2712_PIN_194_I2SO0_LRCK__FUNC_I2SO0_WS (MTK_PIN_NO(194) | 1)
+#define MT2712_PIN_194_I2SO0_LRCK__FUNC_I2SO1_WS (MTK_PIN_NO(194) | 2)
+#define MT2712_PIN_194_I2SO0_LRCK__FUNC_I2SO2_WS (MTK_PIN_NO(194) | 3)
+#define MT2712_PIN_194_I2SO0_LRCK__FUNC_USB5_FT_SCL (MTK_PIN_NO(194) | 4)
+#define MT2712_PIN_194_I2SO0_LRCK__FUNC_TDMO1_DATA2 (MTK_PIN_NO(194) | 5)
+#define MT2712_PIN_194_I2SO0_LRCK__FUNC_I2S_IQ1_WS (MTK_PIN_NO(194) | 7)
+
+#define MT2712_PIN_195_I2SO0_BCK__FUNC_GPIO195 (MTK_PIN_NO(195) | 0)
+#define MT2712_PIN_195_I2SO0_BCK__FUNC_I2SO0_BCK (MTK_PIN_NO(195) | 1)
+#define MT2712_PIN_195_I2SO0_BCK__FUNC_I2SO1_BCK (MTK_PIN_NO(195) | 2)
+#define MT2712_PIN_195_I2SO0_BCK__FUNC_I2SO2_BCK (MTK_PIN_NO(195) | 3)
+#define MT2712_PIN_195_I2SO0_BCK__FUNC_USB5_FT_SDA (MTK_PIN_NO(195) | 4)
+#define MT2712_PIN_195_I2SO0_BCK__FUNC_TDMO1_DATA1 (MTK_PIN_NO(195) | 5)
+#define MT2712_PIN_195_I2SO0_BCK__FUNC_I2S_IQ1_BCK (MTK_PIN_NO(195) | 7)
+
+#define MT2712_PIN_196_TDMO1_MCLK__FUNC_GPIO196 (MTK_PIN_NO(196) | 0)
+#define MT2712_PIN_196_TDMO1_MCLK__FUNC_TDMO1_MCLK (MTK_PIN_NO(196) | 1)
+#define MT2712_PIN_196_TDMO1_MCLK__FUNC_TDMO0_MCLK (MTK_PIN_NO(196) | 2)
+#define MT2712_PIN_196_TDMO1_MCLK__FUNC_TDMIN_MCLK (MTK_PIN_NO(196) | 3)
+#define MT2712_PIN_196_TDMO1_MCLK__FUNC_I2SO0_DO1 (MTK_PIN_NO(196) | 6)
+#define MT2712_PIN_196_TDMO1_MCLK__FUNC_I2S_IQ1_SDIB (MTK_PIN_NO(196) | 7)
+
+#define MT2712_PIN_197_TDMO1_LRCK__FUNC_GPIO197 (MTK_PIN_NO(197) | 0)
+#define MT2712_PIN_197_TDMO1_LRCK__FUNC_TDMO1_LRCK (MTK_PIN_NO(197) | 1)
+#define MT2712_PIN_197_TDMO1_LRCK__FUNC_TDMO0_LRCK (MTK_PIN_NO(197) | 2)
+#define MT2712_PIN_197_TDMO1_LRCK__FUNC_TDMIN_LRCK (MTK_PIN_NO(197) | 3)
+#define MT2712_PIN_197_TDMO1_LRCK__FUNC_TDMO0_DATA3 (MTK_PIN_NO(197) | 4)
+#define MT2712_PIN_197_TDMO1_LRCK__FUNC_TDMO1_DATA3 (MTK_PIN_NO(197) | 5)
+#define MT2712_PIN_197_TDMO1_LRCK__FUNC_I2SO3_MCK (MTK_PIN_NO(197) | 6)
+#define MT2712_PIN_197_TDMO1_LRCK__FUNC_TDMO1_DATA2 (MTK_PIN_NO(197) | 7)
+
+#define MT2712_PIN_198_TDMO1_BCK__FUNC_GPIO198 (MTK_PIN_NO(198) | 0)
+#define MT2712_PIN_198_TDMO1_BCK__FUNC_TDMO1_BCK (MTK_PIN_NO(198) | 1)
+#define MT2712_PIN_198_TDMO1_BCK__FUNC_TDMO0_BCK (MTK_PIN_NO(198) | 2)
+#define MT2712_PIN_198_TDMO1_BCK__FUNC_TDMIN_BCK (MTK_PIN_NO(198) | 3)
+#define MT2712_PIN_198_TDMO1_BCK__FUNC_TDMO0_DATA2 (MTK_PIN_NO(198) | 4)
+#define MT2712_PIN_198_TDMO1_BCK__FUNC_TDMO1_DATA2 (MTK_PIN_NO(198) | 5)
+#define MT2712_PIN_198_TDMO1_BCK__FUNC_I2SO3_BCK (MTK_PIN_NO(198) | 6)
+#define MT2712_PIN_198_TDMO1_BCK__FUNC_TDMO1_DATA1 (MTK_PIN_NO(198) | 7)
+
+#define MT2712_PIN_199_TDMO1_DATA__FUNC_GPIO199 (MTK_PIN_NO(199) | 0)
+#define MT2712_PIN_199_TDMO1_DATA__FUNC_TDMO1_DATA (MTK_PIN_NO(199) | 1)
+#define MT2712_PIN_199_TDMO1_DATA__FUNC_TDMO0_DATA (MTK_PIN_NO(199) | 2)
+#define MT2712_PIN_199_TDMO1_DATA__FUNC_TDMIN_DI (MTK_PIN_NO(199) | 3)
+#define MT2712_PIN_199_TDMO1_DATA__FUNC_TDMO0_DATA1 (MTK_PIN_NO(199) | 4)
+#define MT2712_PIN_199_TDMO1_DATA__FUNC_TDMO1_DATA1 (MTK_PIN_NO(199) | 5)
+#define MT2712_PIN_199_TDMO1_DATA__FUNC_I2SO3_WS (MTK_PIN_NO(199) | 6)
+
+#define MT2712_PIN_200_TDMO0_MCLK__FUNC_GPIO200 (MTK_PIN_NO(200) | 0)
+#define MT2712_PIN_200_TDMO0_MCLK__FUNC_TDMO0_MCLK0 (MTK_PIN_NO(200) | 1)
+#define MT2712_PIN_200_TDMO0_MCLK__FUNC_TDMO1_MCLK0 (MTK_PIN_NO(200) | 2)
+#define MT2712_PIN_200_TDMO0_MCLK__FUNC_PCM1_DI (MTK_PIN_NO(200) | 3)
+#define MT2712_PIN_200_TDMO0_MCLK__FUNC_TDMO0_MCLK1 (MTK_PIN_NO(200) | 4)
+#define MT2712_PIN_200_TDMO0_MCLK__FUNC_TDMO1_MCLK1 (MTK_PIN_NO(200) | 5)
+#define MT2712_PIN_200_TDMO0_MCLK__FUNC_MRG_TX (MTK_PIN_NO(200) | 6)
+#define MT2712_PIN_200_TDMO0_MCLK__FUNC_I2SO2_MCK (MTK_PIN_NO(200) | 7)
+
+#define MT2712_PIN_201_TDMO0_LRCK__FUNC_GPIO201 (MTK_PIN_NO(201) | 0)
+#define MT2712_PIN_201_TDMO0_LRCK__FUNC_TDMO0_LRCK0 (MTK_PIN_NO(201) | 1)
+#define MT2712_PIN_201_TDMO0_LRCK__FUNC_TDMO1_LRCK0 (MTK_PIN_NO(201) | 2)
+#define MT2712_PIN_201_TDMO0_LRCK__FUNC_PCM1_SYNC (MTK_PIN_NO(201) | 3)
+#define MT2712_PIN_201_TDMO0_LRCK__FUNC_TDMO0_LRCK1 (MTK_PIN_NO(201) | 4)
+#define MT2712_PIN_201_TDMO0_LRCK__FUNC_TDMO1_LRCK1 (MTK_PIN_NO(201) | 5)
+#define MT2712_PIN_201_TDMO0_LRCK__FUNC_MRG_RX (MTK_PIN_NO(201) | 6)
+#define MT2712_PIN_201_TDMO0_LRCK__FUNC_I2SO2_WS (MTK_PIN_NO(201) | 7)
+
+#define MT2712_PIN_202_TDMO0_BCK__FUNC_GPIO202 (MTK_PIN_NO(202) | 0)
+#define MT2712_PIN_202_TDMO0_BCK__FUNC_TDMO0_BCK0 (MTK_PIN_NO(202) | 1)
+#define MT2712_PIN_202_TDMO0_BCK__FUNC_TDMO1_BCK0 (MTK_PIN_NO(202) | 2)
+#define MT2712_PIN_202_TDMO0_BCK__FUNC_PCM1_CLK (MTK_PIN_NO(202) | 3)
+#define MT2712_PIN_202_TDMO0_BCK__FUNC_TDMO0_BCK1 (MTK_PIN_NO(202) | 4)
+#define MT2712_PIN_202_TDMO0_BCK__FUNC_TDMO1_BCK1 (MTK_PIN_NO(202) | 5)
+#define MT2712_PIN_202_TDMO0_BCK__FUNC_MRG_SYNC (MTK_PIN_NO(202) | 6)
+#define MT2712_PIN_202_TDMO0_BCK__FUNC_I2SO2_BCK (MTK_PIN_NO(202) | 7)
+
+#define MT2712_PIN_203_TDMO0_DATA__FUNC_GPIO203 (MTK_PIN_NO(203) | 0)
+#define MT2712_PIN_203_TDMO0_DATA__FUNC_TDMO0_DATA0 (MTK_PIN_NO(203) | 1)
+#define MT2712_PIN_203_TDMO0_DATA__FUNC_TDMO1_DATA0 (MTK_PIN_NO(203) | 2)
+#define MT2712_PIN_203_TDMO0_DATA__FUNC_PCM1_DO (MTK_PIN_NO(203) | 3)
+#define MT2712_PIN_203_TDMO0_DATA__FUNC_TDMO0_DATA1 (MTK_PIN_NO(203) | 4)
+#define MT2712_PIN_203_TDMO0_DATA__FUNC_TDMO1_DATA1 (MTK_PIN_NO(203) | 5)
+#define MT2712_PIN_203_TDMO0_DATA__FUNC_MRG_CLK (MTK_PIN_NO(203) | 6)
+#define MT2712_PIN_203_TDMO0_DATA__FUNC_I2SO2_DO (MTK_PIN_NO(203) | 7)
+
+#define MT2712_PIN_204_PERSTB_P0__FUNC_GPIO204 (MTK_PIN_NO(204) | 0)
+#define MT2712_PIN_204_PERSTB_P0__FUNC_PERST_B_P0 (MTK_PIN_NO(204) | 1)
+
+#define MT2712_PIN_205_CLKREQN_P0__FUNC_GPIO205 (MTK_PIN_NO(205) | 0)
+#define MT2712_PIN_205_CLKREQN_P0__FUNC_CLKREQ_N_P0 (MTK_PIN_NO(205) | 1)
+
+#define MT2712_PIN_206_WAKEEN_P0__FUNC_GPIO206 (MTK_PIN_NO(206) | 0)
+#define MT2712_PIN_206_WAKEEN_P0__FUNC_WAKE_EN_P0 (MTK_PIN_NO(206) | 1)
+
+#define MT2712_PIN_207_PERSTB_P1__FUNC_GPIO207 (MTK_PIN_NO(207) | 0)
+#define MT2712_PIN_207_PERSTB_P1__FUNC_PERST_B_P1 (MTK_PIN_NO(207) | 1)
+
+#define MT2712_PIN_208_CLKREQN_P1__FUNC_GPIO208 (MTK_PIN_NO(208) | 0)
+#define MT2712_PIN_208_CLKREQN_P1__FUNC_CLKREQ_N_P1 (MTK_PIN_NO(208) | 1)
+
+#define MT2712_PIN_209_WAKEEN_P1__FUNC_GPIO209 (MTK_PIN_NO(209) | 0)
+#define MT2712_PIN_209_WAKEEN_P1__FUNC_WAKE_EN_P1 (MTK_PIN_NO(209) | 1)
+
+#endif /* __DTS_MT2712_PINFUNC_H */
diff --git a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi
index 9d88f41aefa0..6d8532af8346 100644
--- a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi
@@ -9,6 +9,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/power/mt2712-power.h>
+#include "mt2712-pinfunc.h"
 
 / {
 	compatible = "mediatek,mt2712";
@@ -199,6 +200,34 @@
 		clock-output-names = "clkaud_ext_i_2";
 	};
 
+	clki2si0_mck_i: oscillator@6 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <30000000>;
+		clock-output-names = "clki2si0_mck_i";
+	};
+
+	clki2si1_mck_i: oscillator@7 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <30000000>;
+		clock-output-names = "clki2si1_mck_i";
+	};
+
+	clki2si2_mck_i: oscillator@8 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <30000000>;
+		clock-output-names = "clki2si2_mck_i";
+	};
+
+	clktdmin_mclk_i: oscillator@9 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <30000000>;
+		clock-output-names = "clktdmin_mclk_i";
+	};
+
 	timer {
 		compatible = "arm,armv8-timer";
 		interrupt-parent = <&gic>;
@@ -230,6 +259,23 @@
 		#clock-cells = <1>;
 	};
 
+	syscfg_pctl_a: syscfg_pctl_a@10005000 {
+		compatible = "mediatek,mt2712-pctl-a-syscfg", "syscon";
+		reg = <0 0x10005000 0 0x1000>;
+	};
+
+	pio: pinctrl@10005000 {
+		compatible = "mediatek,mt2712-pinctrl";
+		reg = <0 0x1000b000 0 0x1000>;
+		mediatek,pctl-regmap = <&syscfg_pctl_a>;
+		pins-are-numbered;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
 	scpsys: scpsys@10006000 {
 		compatible = "mediatek,mt2712-scpsys", "syscon";
 		#power-domain-cells = <1>;
diff --git a/arch/arm64/boot/dts/mediatek/mt7622-rfb1.dts b/arch/arm64/boot/dts/mediatek/mt7622-rfb1.dts
index 45d8655ee423..b7837642c33a 100644
--- a/arch/arm64/boot/dts/mediatek/mt7622-rfb1.dts
+++ b/arch/arm64/boot/dts/mediatek/mt7622-rfb1.dts
@@ -18,7 +18,7 @@
 	compatible = "mediatek,mt7622-rfb1", "mediatek,mt7622";
 
 	chosen {
-		bootargs = "console=ttyS0,115200n1";
+		bootargs = "console=ttyS0,115200n1 swiotlb=512";
 	};
 
 	cpus {
@@ -163,10 +163,17 @@
 	i2s1_pins: i2s1-pins {
 		mux {
 			function = "i2s";
-			groups =  "i2s_out_bclk_ws_mclk",
+			groups =  "i2s_out_mclk_bclk_ws",
 				  "i2s1_in_data",
 				  "i2s1_out_data";
 		};
+
+		conf {
+			pins = "I2S1_IN", "I2S1_OUT", "I2S_BCLK",
+			       "I2S_WS", "I2S_MCLK";
+			drive-strength = <12>;
+			bias-pull-down;
+		};
 	};
 
 	irrx_pins: irrx-pins {
diff --git a/arch/arm64/boot/dts/mediatek/mt7622.dtsi b/arch/arm64/boot/dts/mediatek/mt7622.dtsi
index e9d5130df8d1..9213c966c224 100644
--- a/arch/arm64/boot/dts/mediatek/mt7622.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt7622.dtsi
@@ -527,6 +527,95 @@
 		status = "disabled";
 	};
 
+	audsys: clock-controller@11220000 {
+		compatible = "mediatek,mt7622-audsys", "syscon";
+		reg = <0 0x11220000 0 0x2000>;
+		#clock-cells = <1>;
+
+		afe: audio-controller {
+			compatible = "mediatek,mt7622-audio";
+			interrupts =  <GIC_SPI 144 IRQ_TYPE_LEVEL_LOW>,
+				      <GIC_SPI 145 IRQ_TYPE_LEVEL_LOW>;
+			interrupt-names	= "afe", "asys";
+
+			clocks = <&infracfg CLK_INFRA_AUDIO_PD>,
+				 <&topckgen CLK_TOP_AUD1_SEL>,
+				 <&topckgen CLK_TOP_AUD2_SEL>,
+				 <&topckgen CLK_TOP_A1SYS_HP_DIV_PD>,
+				 <&topckgen CLK_TOP_A2SYS_HP_DIV_PD>,
+				 <&topckgen CLK_TOP_I2S0_MCK_SEL>,
+				 <&topckgen CLK_TOP_I2S1_MCK_SEL>,
+				 <&topckgen CLK_TOP_I2S2_MCK_SEL>,
+				 <&topckgen CLK_TOP_I2S3_MCK_SEL>,
+				 <&topckgen CLK_TOP_I2S0_MCK_DIV>,
+				 <&topckgen CLK_TOP_I2S1_MCK_DIV>,
+				 <&topckgen CLK_TOP_I2S2_MCK_DIV>,
+				 <&topckgen CLK_TOP_I2S3_MCK_DIV>,
+				 <&topckgen CLK_TOP_I2S0_MCK_DIV_PD>,
+				 <&topckgen CLK_TOP_I2S1_MCK_DIV_PD>,
+				 <&topckgen CLK_TOP_I2S2_MCK_DIV_PD>,
+				 <&topckgen CLK_TOP_I2S3_MCK_DIV_PD>,
+				 <&audsys CLK_AUDIO_I2SO1>,
+				 <&audsys CLK_AUDIO_I2SO2>,
+				 <&audsys CLK_AUDIO_I2SO3>,
+				 <&audsys CLK_AUDIO_I2SO4>,
+				 <&audsys CLK_AUDIO_I2SIN1>,
+				 <&audsys CLK_AUDIO_I2SIN2>,
+				 <&audsys CLK_AUDIO_I2SIN3>,
+				 <&audsys CLK_AUDIO_I2SIN4>,
+				 <&audsys CLK_AUDIO_ASRCO1>,
+				 <&audsys CLK_AUDIO_ASRCO2>,
+				 <&audsys CLK_AUDIO_ASRCO3>,
+				 <&audsys CLK_AUDIO_ASRCO4>,
+				 <&audsys CLK_AUDIO_AFE>,
+				 <&audsys CLK_AUDIO_AFE_CONN>,
+				 <&audsys CLK_AUDIO_A1SYS>,
+				 <&audsys CLK_AUDIO_A2SYS>;
+
+			clock-names = "infra_sys_audio_clk",
+				      "top_audio_mux1_sel",
+				      "top_audio_mux2_sel",
+				      "top_audio_a1sys_hp",
+				      "top_audio_a2sys_hp",
+				      "i2s0_src_sel",
+				      "i2s1_src_sel",
+				      "i2s2_src_sel",
+				      "i2s3_src_sel",
+				      "i2s0_src_div",
+				      "i2s1_src_div",
+				      "i2s2_src_div",
+				      "i2s3_src_div",
+				      "i2s0_mclk_en",
+				      "i2s1_mclk_en",
+				      "i2s2_mclk_en",
+				      "i2s3_mclk_en",
+				      "i2so0_hop_ck",
+				      "i2so1_hop_ck",
+				      "i2so2_hop_ck",
+				      "i2so3_hop_ck",
+				      "i2si0_hop_ck",
+				      "i2si1_hop_ck",
+				      "i2si2_hop_ck",
+				      "i2si3_hop_ck",
+				      "asrc0_out_ck",
+				      "asrc1_out_ck",
+				      "asrc2_out_ck",
+				      "asrc3_out_ck",
+				      "audio_afe_pd",
+				      "audio_afe_conn_pd",
+				      "audio_a1sys_pd",
+				      "audio_a2sys_pd";
+
+			assigned-clocks = <&topckgen CLK_TOP_A1SYS_HP_SEL>,
+					  <&topckgen CLK_TOP_A2SYS_HP_SEL>,
+					  <&topckgen CLK_TOP_A1SYS_HP_DIV>,
+					  <&topckgen CLK_TOP_A2SYS_HP_DIV>;
+			assigned-clock-parents = <&topckgen CLK_TOP_AUD1PLL>,
+						 <&topckgen CLK_TOP_AUD2PLL>;
+			assigned-clock-rates = <0>, <0>, <49152000>, <45158400>;
+		};
+	};
+
 	mmc0: mmc@11230000 {
 		compatible = "mediatek,mt7622-mmc";
 		reg = <0 0x11230000 0 0x1000>;
@@ -735,6 +824,16 @@
 		#reset-cells = <1>;
 	};
 
+	hsdma: dma-controller@1b007000 {
+		compatible = "mediatek,mt7622-hsdma";
+		reg = <0 0x1b007000 0 0x1000>;
+		interrupts = <GIC_SPI 219 IRQ_TYPE_LEVEL_LOW>;
+		clocks = <&ethsys CLK_ETH_HSDMA_EN>;
+		clock-names = "hsdma";
+		power-domains = <&scpsys MT7622_POWER_DOMAIN_ETHSYS>;
+		#dma-cells = <1>;
+	};
+
 	eth: ethernet@1b100000 {
 		compatible = "mediatek,mt7622-eth",
 			     "mediatek,mt2701-eth",
diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile
index 55ec5ee7f7e8..9319e74b8906 100644
--- a/arch/arm64/boot/dts/qcom/Makefile
+++ b/arch/arm64/boot/dts/qcom/Makefile
@@ -6,3 +6,4 @@ dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-mtp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8992-bullhead-rev-101.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8994-angler-rev-101.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8996-mtp.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sdm845-mtp.dtb
diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c-pins.dtsi b/arch/arm64/boot/dts/qcom/apq8096-db820c-pins.dtsi
index 24552f19b3fa..6a573875d45a 100644
--- a/arch/arm64/boot/dts/qcom/apq8096-db820c-pins.dtsi
+++ b/arch/arm64/boot/dts/qcom/apq8096-db820c-pins.dtsi
@@ -36,4 +36,30 @@
 			drive-strength = <2>;	/* 2 MA */
 		};
 	};
+
+	blsp1_uart1_default: blsp1_uart1_default {
+		mux {
+			pins = "gpio41", "gpio42", "gpio43", "gpio44";
+			function = "blsp_uart2";
+		};
+
+		config {
+			pins = "gpio41", "gpio42", "gpio43", "gpio44";
+			drive-strength = <16>;
+			bias-disable;
+		};
+	};
+
+	blsp1_uart1_sleep: blsp1_uart1_sleep {
+		mux {
+			pins = "gpio41", "gpio42", "gpio43", "gpio44";
+			function = "gpio";
+		};
+
+		config {
+			pins = "gpio41", "gpio42", "gpio43", "gpio44";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
 };
diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c-pmic-pins.dtsi b/arch/arm64/boot/dts/qcom/apq8096-db820c-pmic-pins.dtsi
index 59b29ddfb6e9..a6ad3d7fe655 100644
--- a/arch/arm64/boot/dts/qcom/apq8096-db820c-pmic-pins.dtsi
+++ b/arch/arm64/boot/dts/qcom/apq8096-db820c-pmic-pins.dtsi
@@ -4,7 +4,7 @@
 &pm8994_gpios {
 
 	pinctrl-names = "default";
-	pinctrl-0 = <&ls_exp_gpio_f>;
+	pinctrl-0 = <&ls_exp_gpio_f &bt_en_gpios>;
 
 	ls_exp_gpio_f: pm8994_gpio5 {
 		pinconf {
@@ -14,6 +14,28 @@
 		};
 	};
 
+	bt_en_gpios: bt_en_gpios {
+		pinconf {
+			pins = "gpio19";
+			function = PMIC_GPIO_FUNC_NORMAL;
+			output-low;
+			power-source = <PM8994_GPIO_S4>; // 1.8V
+			qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
+			bias-pull-down;
+		};
+	};
+
+	wlan_en_gpios: wlan_en_gpios {
+		pinconf {
+			pins = "gpio8";
+			function = PMIC_GPIO_FUNC_NORMAL;
+			output-low;
+			power-source = <PM8994_GPIO_S4>; // 1.8V
+			qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
+			bias-pull-down;
+		};
+	};
+
 	volume_up_gpio: pm8996_gpio2 {
 		pinconf {
 			pins = "gpio2";
@@ -26,6 +48,16 @@
 		};
 	};
 
+	divclk4_pin_a: divclk4 {
+		pinconf {
+			pins = "gpio18";
+			function = PMIC_GPIO_FUNC_FUNC2;
+
+			bias-disable;
+			power-source = <PM8994_GPIO_S4>;
+		};
+	};
+
 	usb3_vbus_det_gpio: pm8996_gpio22 {
 		pinconf {
 			pins = "gpio22";
diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi b/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
index 1c8f1b86472d..0f829db33efe 100644
--- a/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
+++ b/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
@@ -23,6 +23,7 @@
 	aliases {
 		serial0 = &blsp2_uart1;
 		serial1 = &blsp2_uart2;
+		serial2 = &blsp1_uart1;
 		i2c0	= &blsp1_i2c2;
 		i2c1	= &blsp2_i2c1;
 		i2c2	= &blsp2_i2c0;
@@ -34,7 +35,36 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	clocks {
+		divclk4: divclk4 {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <32768>;
+			clock-output-names = "divclk4";
+
+			pinctrl-names = "default";
+			pinctrl-0 = <&divclk4_pin_a>;
+		};
+	};
+
 	soc {
+		serial@7570000 {
+			label = "BT-UART";
+			status = "okay";
+			pinctrl-names = "default", "sleep";
+			pinctrl-0 = <&blsp1_uart1_default>;
+			pinctrl-1 = <&blsp1_uart1_sleep>;
+
+			bluetooth {
+				compatible = "qcom,qca6174-bt";
+
+				/* bt_disable_n gpio */
+				enable-gpios = <&pm8994_gpios 19 GPIO_ACTIVE_HIGH>;
+
+				clocks = <&divclk4>;
+			};
+		};
+
 		serial@75b0000 {
 			label = "LS-UART1";
 			status = "okay";
@@ -87,6 +117,16 @@
 			pinctrl-0 = <&sdc2_clk_on &sdc2_cmd_on &sdc2_data_on &sdc2_cd_on>;
 			pinctrl-1 = <&sdc2_clk_off &sdc2_cmd_off &sdc2_data_off &sdc2_cd_off>;
 			cd-gpios = <&msmgpio 38 0x1>;
+			vmmc-supply = <&pm8994_l21>;
+			vqmmc-supply = <&pm8994_l13>;
+			status = "okay";
+		};
+
+		phy@627000 {
+			status = "okay";
+		};
+
+		ufshc@624000 {
 			status = "okay";
 		};
 
@@ -139,17 +179,34 @@
 			pinctrl-0 = <&usb2_vbus_det_gpio>;
 		};
 
+		wlan_en: wlan-en-1-8v {
+			pinctrl-names = "default";
+			pinctrl-0 = <&wlan_en_gpios>;
+			compatible = "regulator-fixed";
+			regulator-name = "wlan-en-regulator";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+
+			gpio = <&pm8994_gpios 8 0>;
+
+			/* WLAN card specific delay */
+			startup-delay-us = <70000>;
+			enable-active-high;
+		};
+
 		agnoc@0 {
-			qcom,pcie@600000 {
+			pcie@600000 {
+				status = "okay";
 				perst-gpio = <&msmgpio 35 GPIO_ACTIVE_LOW>;
+				vddpe-3v3-supply = <&wlan_en>;
 			};
 
-			qcom,pcie@608000 {
+			pcie@608000 {
 				status = "okay";
 				perst-gpio = <&msmgpio 130 GPIO_ACTIVE_LOW>;
 			};
 
-			qcom,pcie@610000 {
+			pcie@610000 {
 				status = "okay";
 				perst-gpio = <&msmgpio 114 GPIO_ACTIVE_LOW>;
 			};
diff --git a/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts b/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts
index 6a838b5d321e..c13ddee8262b 100644
--- a/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts
+++ b/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts
@@ -21,6 +21,7 @@
 
 	aliases {
 		serial0 = &blsp1_uart5;
+		serial1 = &blsp1_uart3;
 	};
 
 	chosen {
@@ -33,20 +34,61 @@
 	};
 
 	soc {
-		pinctrl@1000000 {
-			serial_4_pins: serial4_pinmux {
-				mux {
-					pins = "gpio23", "gpio24";
-					function = "blsp4_uart1";
-					bias-disable;
-				};
+		serial@78b3000 {
+			status = "ok";
+		};
+
+		spi@78b5000 {
+			status = "ok";
+
+			m25p80@0 {
+				  #address-cells = <1>;
+				  #size-cells = <1>;
+				  compatible = "jedec,spi-nor";
+				  reg = <0>;
+				  spi-max-frequency = <50000000>;
 			};
 		};
 
-		serial@78b3000 {
-			pinctrl-0 = <&serial_4_pins>;
-			pinctrl-names = "default";
+		serial@78b1000 {
+			 status = "ok";
+		};
+
+		i2c@78b6000 {
+			 status = "ok";
+		};
+
+		dma@7984000 {
+			 status = "ok";
+		};
+
+		nand@79b0000 {
+			status = "ok";
+
+			nand@0 {
+				reg = <0>;
+				nand-ecc-strength = <4>;
+				nand-ecc-step-size = <512>;
+				nand-bus-width = <8>;
+			};
+		};
+
+		phy@86000 {
+			status = "ok";
+		};
+
+		phy@8e000 {
+			status = "ok";
+		};
+
+		pci@20000000 {
+			status = "ok";
+			perst-gpio = <&tlmm 58 0x1>;
+		};
+
+		pci@10000000 {
 			status = "ok";
+			perst-gpio = <&tlmm 61 0x1>;
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/qcom/ipq8074.dtsi b/arch/arm64/boot/dts/qcom/ipq8074.dtsi
index 2bc5dec5614d..18226980f7c3 100644
--- a/arch/arm64/boot/dts/qcom/ipq8074.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq8074.dtsi
@@ -24,7 +24,7 @@
 		ranges = <0 0 0 0xffffffff>;
 		compatible = "simple-bus";
 
-		pinctrl@1000000 {
+		tlmm: pinctrl@1000000 {
 			compatible = "qcom,ipq8074-pinctrl";
 			reg = <0x1000000 0x300000>;
 			interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
@@ -32,6 +32,45 @@
 			#gpio-cells = <0x2>;
 			interrupt-controller;
 			#interrupt-cells = <0x2>;
+
+			serial_4_pins: serial4-pinmux {
+				pins = "gpio23", "gpio24";
+				function = "blsp4_uart1";
+				drive-strength = <8>;
+				bias-disable;
+			};
+
+			i2c_0_pins: i2c-0-pinmux {
+				pins = "gpio42", "gpio43";
+				function = "blsp1_i2c";
+				drive-strength = <8>;
+				bias-disable;
+			};
+
+			spi_0_pins: spi-0-pins {
+				pins = "gpio38", "gpio39", "gpio40", "gpio41";
+				function = "blsp0_spi";
+				drive-strength = <8>;
+				bias-disable;
+			};
+
+			hsuart_pins: hsuart-pins {
+				pins = "gpio46", "gpio47", "gpio48", "gpio49";
+				function = "blsp2_uart";
+				drive-strength = <8>;
+				bias-disable;
+			};
+
+			qpic_pins: qpic-pins {
+				pins = "gpio1", "gpio3", "gpio4",
+				       "gpio5", "gpio6", "gpio7",
+				       "gpio8", "gpio10", "gpio11",
+				       "gpio12", "gpio13", "gpio14",
+				       "gpio15", "gpio16", "gpio17";
+				function = "qpic";
+				drive-strength = <8>;
+				bias-disable;
+			};
 		};
 
 		intc: interrupt-controller@b000000 {
@@ -122,6 +161,276 @@
 			clocks = <&gcc GCC_BLSP1_UART5_APPS_CLK>,
 				 <&gcc GCC_BLSP1_AHB_CLK>;
 			clock-names = "core", "iface";
+			pinctrl-0 = <&serial_4_pins>;
+			pinctrl-names = "default";
+			status = "disabled";
+		};
+
+		blsp_dma: dma@7884000 {
+			compatible = "qcom,bam-v1.7.0";
+			reg = <0x7884000 0x2b000>;
+			interrupts = <GIC_SPI 238 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_BLSP1_AHB_CLK>;
+			clock-names = "bam_clk";
+			#dma-cells = <1>;
+			qcom,ee = <0>;
+		};
+
+		blsp1_uart1: serial@78af000 {
+			compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm";
+			reg = <0x78af000 0x200>;
+			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_BLSP1_UART1_APPS_CLK>,
+				 <&gcc GCC_BLSP1_AHB_CLK>;
+			clock-names = "core", "iface";
+			status = "disabled";
+		};
+
+		blsp1_uart3: serial@78b1000 {
+			compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm";
+			reg = <0x78b1000 0x200>;
+			interrupts = <GIC_SPI 306 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_BLSP1_UART3_APPS_CLK>,
+				<&gcc GCC_BLSP1_AHB_CLK>;
+			clock-names = "core", "iface";
+			dmas = <&blsp_dma 4>,
+				<&blsp_dma 5>;
+			dma-names = "tx", "rx";
+			pinctrl-0 = <&hsuart_pins>;
+			pinctrl-names = "default";
+			status = "disabled";
+		};
+
+		blsp1_spi1: spi@78b5000 {
+			compatible = "qcom,spi-qup-v2.2.1";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x78b5000 0x600>;
+			interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
+			spi-max-frequency = <50000000>;
+			clocks = <&gcc GCC_BLSP1_QUP1_SPI_APPS_CLK>,
+				<&gcc GCC_BLSP1_AHB_CLK>;
+			clock-names = "core", "iface";
+			dmas = <&blsp_dma 12>, <&blsp_dma 13>;
+			dma-names = "tx", "rx";
+			pinctrl-0 = <&spi_0_pins>;
+			pinctrl-names = "default";
+			status = "disabled";
+		};
+
+		blsp1_i2c2: i2c@78b6000 {
+			compatible = "qcom,i2c-qup-v2.2.1";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x78b6000 0x600>;
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_BLSP1_AHB_CLK>,
+				<&gcc GCC_BLSP1_QUP2_I2C_APPS_CLK>;
+			clock-names = "iface", "core";
+			clock-frequency = <400000>;
+			dmas = <&blsp_dma 15>, <&blsp_dma 14>;
+			dma-names = "rx", "tx";
+			pinctrl-0 = <&i2c_0_pins>;
+			pinctrl-names = "default";
+			status = "disabled";
+		};
+
+		blsp1_i2c3: i2c@78b7000 {
+			compatible = "qcom,i2c-qup-v2.2.1";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x78b7000 0x600>;
+			interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_BLSP1_AHB_CLK>,
+				<&gcc GCC_BLSP1_QUP3_I2C_APPS_CLK>;
+			clock-names = "iface", "core";
+			clock-frequency = <100000>;
+			dmas = <&blsp_dma 17>, <&blsp_dma 16>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		qpic_bam: dma@7984000 {
+			compatible = "qcom,bam-v1.7.0";
+			reg = <0x7984000 0x1a000>;
+			interrupts = <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_QPIC_AHB_CLK>;
+			clock-names = "bam_clk";
+			#dma-cells = <1>;
+			qcom,ee = <0>;
+			status = "disabled";
+		};
+
+		qpic_nand: nand@79b0000 {
+			compatible = "qcom,ipq8074-nand";
+			reg = <0x79b0000 0x10000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&gcc GCC_QPIC_CLK>,
+				 <&gcc GCC_QPIC_AHB_CLK>;
+			clock-names = "core", "aon";
+
+			dmas = <&qpic_bam 0>,
+			       <&qpic_bam 1>,
+			       <&qpic_bam 2>;
+			dma-names = "tx", "rx", "cmd";
+			pinctrl-0 = <&qpic_pins>;
+			pinctrl-names = "default";
+			status = "disabled";
+		};
+
+		pcie_phy0: phy@86000 {
+			compatible = "qcom,ipq8074-qmp-pcie-phy";
+			reg = <0x86000 0x1000>;
+			#phy-cells = <0>;
+			clocks = <&gcc GCC_PCIE0_PIPE_CLK>;
+			clock-names = "pipe_clk";
+			clock-output-names = "pcie20_phy0_pipe_clk";
+
+			resets = <&gcc GCC_PCIE0_PHY_BCR>,
+				<&gcc GCC_PCIE0PHY_PHY_BCR>;
+			reset-names = "phy",
+				      "common";
+			status = "disabled";
+		};
+
+		pcie0: pci@20000000 {
+			compatible = "qcom,pcie-ipq8074";
+			reg =  <0x20000000 0xf1d
+				0x20000f20 0xa8
+				0x80000 0x2000
+				0x20100000 0x1000>;
+			reg-names = "dbi", "elbi", "parf", "config";
+			device_type = "pci";
+			linux,pci-domain = <0>;
+			bus-range = <0x00 0xff>;
+			num-lanes = <1>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			phys = <&pcie_phy0>;
+			phy-names = "pciephy";
+
+			ranges = <0x81000000 0 0x20200000 0x20200000
+				  0 0x100000   /* downstream I/O */
+				  0x82000000 0 0x20300000 0x20300000
+				  0 0xd00000>; /* non-prefetchable memory */
+
+			interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi";
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc 0 75
+					 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
+					<0 0 0 2 &intc 0 78
+					 IRQ_TYPE_LEVEL_HIGH>, /* int_b */
+					<0 0 0 3 &intc 0 79
+					 IRQ_TYPE_LEVEL_HIGH>, /* int_c */
+					<0 0 0 4 &intc 0 83
+					 IRQ_TYPE_LEVEL_HIGH>; /* int_d */
+
+			clocks = <&gcc GCC_SYS_NOC_PCIE0_AXI_CLK>,
+				 <&gcc GCC_PCIE0_AXI_M_CLK>,
+				 <&gcc GCC_PCIE0_AXI_S_CLK>,
+				 <&gcc GCC_PCIE0_AHB_CLK>,
+				 <&gcc GCC_PCIE0_AUX_CLK>;
+
+			clock-names = "iface",
+				      "axi_m",
+				      "axi_s",
+				      "ahb",
+				      "aux";
+			resets = <&gcc GCC_PCIE0_PIPE_ARES>,
+				 <&gcc GCC_PCIE0_SLEEP_ARES>,
+				 <&gcc GCC_PCIE0_CORE_STICKY_ARES>,
+				 <&gcc GCC_PCIE0_AXI_MASTER_ARES>,
+				 <&gcc GCC_PCIE0_AXI_SLAVE_ARES>,
+				 <&gcc GCC_PCIE0_AHB_ARES>,
+				 <&gcc GCC_PCIE0_AXI_MASTER_STICKY_ARES>;
+			reset-names = "pipe",
+				      "sleep",
+				      "sticky",
+				      "axi_m",
+				      "axi_s",
+				      "ahb",
+				      "axi_m_sticky";
+			status = "disabled";
+		};
+
+		pcie_phy1: phy@8e000 {
+			compatible = "qcom,ipq8074-qmp-pcie-phy";
+			reg = <0x8e000 0x1000>;
+			#phy-cells = <0>;
+			clocks = <&gcc GCC_PCIE1_PIPE_CLK>;
+			clock-names = "pipe_clk";
+			clock-output-names = "pcie20_phy1_pipe_clk";
+
+			resets = <&gcc GCC_PCIE1_PHY_BCR>,
+				<&gcc GCC_PCIE1PHY_PHY_BCR>;
+			reset-names = "phy",
+				      "common";
+			status = "disabled";
+		};
+
+		pcie1: pci@10000000 {
+			compatible = "qcom,pcie-ipq8074";
+			reg =  <0x10000000 0xf1d
+				0x10000f20 0xa8
+				0x88000 0x2000
+				0x10100000 0x1000>;
+			reg-names = "dbi", "elbi", "parf", "config";
+			device_type = "pci";
+			linux,pci-domain = <1>;
+			bus-range = <0x00 0xff>;
+			num-lanes = <1>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			phys = <&pcie_phy1>;
+			phy-names = "pciephy";
+
+			ranges = <0x81000000 0 0x10200000 0x10200000
+				  0 0x100000   /* downstream I/O */
+				  0x82000000 0 0x10300000 0x10300000
+				  0 0xd00000>; /* non-prefetchable memory */
+
+			interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi";
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc 0 142
+					 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
+					<0 0 0 2 &intc 0 143
+					 IRQ_TYPE_LEVEL_HIGH>, /* int_b */
+					<0 0 0 3 &intc 0 144
+					 IRQ_TYPE_LEVEL_HIGH>, /* int_c */
+					<0 0 0 4 &intc 0 145
+					 IRQ_TYPE_LEVEL_HIGH>; /* int_d */
+
+			clocks = <&gcc GCC_SYS_NOC_PCIE1_AXI_CLK>,
+				 <&gcc GCC_PCIE1_AXI_M_CLK>,
+				 <&gcc GCC_PCIE1_AXI_S_CLK>,
+				 <&gcc GCC_PCIE1_AHB_CLK>,
+				 <&gcc GCC_PCIE1_AUX_CLK>;
+			clock-names = "iface",
+				      "axi_m",
+				      "axi_s",
+				      "ahb",
+				      "aux";
+			resets = <&gcc GCC_PCIE1_PIPE_ARES>,
+				 <&gcc GCC_PCIE1_SLEEP_ARES>,
+				 <&gcc GCC_PCIE1_CORE_STICKY_ARES>,
+				 <&gcc GCC_PCIE1_AXI_MASTER_ARES>,
+				 <&gcc GCC_PCIE1_AXI_SLAVE_ARES>,
+				 <&gcc GCC_PCIE1_AHB_ARES>,
+				 <&gcc GCC_PCIE1_AXI_MASTER_STICKY_ARES>;
+			reset-names = "pipe",
+				      "sleep",
+				      "sticky",
+				      "axi_m",
+				      "axi_s",
+				      "ahb",
+				      "axi_m_sticky";
 			status = "disabled";
 		};
 	};
@@ -175,7 +484,7 @@
 
 	pmu {
 		compatible = "arm,armv8-pmuv3";
-		interrupts = <GIC_PPI 7 GIC_CPU_MASK_SIMPLE(4)>;
+		interrupts = <GIC_PPI 7 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 	};
 
 	clocks {
diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi
index 66b318e1de80..650f356f69ca 100644
--- a/arch/arm64/boot/dts/qcom/msm8916.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi
@@ -179,7 +179,7 @@
 
 	pmu {
 		compatible = "arm,cortex-a53-pmu";
-		interrupts = <GIC_PPI 7 GIC_CPU_MASK_SIMPLE(4)>;
+		interrupts = <GIC_PPI 7 (GIC_CPU_MASK_SIMPLE(4)| IRQ_TYPE_LEVEL_HIGH)>;
 	};
 
 	thermal-zones {
@@ -512,7 +512,7 @@
 		blsp_i2c2: i2c@78b6000 {
 			compatible = "qcom,i2c-qup-v2.2.1";
 			reg = <0x078b6000 0x500>;
-			interrupts = <GIC_SPI 96 0>;
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP1_AHB_CLK>,
 				 <&gcc GCC_BLSP1_QUP2_I2C_APPS_CLK>;
 			clock-names = "iface", "core";
@@ -527,7 +527,7 @@
 		blsp_i2c4: i2c@78b8000 {
 			compatible = "qcom,i2c-qup-v2.2.1";
 			reg = <0x078b8000 0x500>;
-			interrupts = <GIC_SPI 98 0>;
+			interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP1_AHB_CLK>,
 				 <&gcc GCC_BLSP1_QUP4_I2C_APPS_CLK>;
 			clock-names = "iface", "core";
@@ -542,7 +542,7 @@
 		blsp_i2c6: i2c@78ba000 {
 			compatible = "qcom,i2c-qup-v2.2.1";
 			reg = <0x078ba000 0x500>;
-			interrupts = <GIC_SPI 100 0>;
+			interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP1_AHB_CLK>,
 				 <&gcc GCC_BLSP1_QUP6_I2C_APPS_CLK>;
 			clock-names = "iface", "core";
@@ -574,7 +574,7 @@
 					"mi2s-bit-clk3";
 			#sound-dai-cells = <1>;
 
-			interrupts = <0 160 0>;
+			interrupts = <0 160 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "lpass-irq-lpaif";
 			reg = <0x07708000 0x10000>;
 			reg-names = "lpass-lpaif";
@@ -594,7 +594,7 @@
 			reg = <0x07824900 0x11c>, <0x07824000 0x800>;
 			reg-names = "hc_mem", "core_mem";
 
-			interrupts = <0 123 0>, <0 138 0>;
+			interrupts = <0 123 IRQ_TYPE_LEVEL_HIGH>, <0 138 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
 			clocks = <&gcc GCC_SDCC1_APPS_CLK>,
 				 <&gcc GCC_SDCC1_AHB_CLK>,
@@ -611,7 +611,7 @@
 			reg = <0x07864900 0x11c>, <0x07864000 0x800>;
 			reg-names = "hc_mem", "core_mem";
 
-			interrupts = <0 125 0>, <0 221 0>;
+			interrupts = <0 125 IRQ_TYPE_LEVEL_HIGH>, <0 221 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
 			clocks = <&gcc GCC_SDCC2_APPS_CLK>,
 				 <&gcc GCC_SDCC2_AHB_CLK>,
@@ -818,7 +818,7 @@
 			iommu-ctx@2000 {
 				compatible = "qcom,msm-iommu-v1-ns";
 				reg = <0x2000 0x1000>;
-				interrupts = <GIC_SPI 242 0>;
+				interrupts = <GIC_SPI 242 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
 
@@ -862,7 +862,7 @@
 				      "bus_clk",
 				      "vsync_clk";
 
-			interrupts = <0 72 0>;
+			interrupts = <0 72 IRQ_TYPE_LEVEL_HIGH>;
 
 			interrupt-controller;
 			#interrupt-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/msm8992-bullhead-rev-101.dts b/arch/arm64/boot/dts/qcom/msm8992-bullhead-rev-101.dts
index 454213391671..8c69516f97ed 100644
--- a/arch/arm64/boot/dts/qcom/msm8992-bullhead-rev-101.dts
+++ b/arch/arm64/boot/dts/qcom/msm8992-bullhead-rev-101.dts
@@ -38,4 +38,21 @@
 			pinctrl-1 = <&blsp1_uart2_sleep>;
 		};
 	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ramoops@1ff00000 {
+			compatible = "ramoops";
+			reg = <0x0 0x1ff00000 0x0 0x40000>;
+			console-size = <0x10000>;
+			record-size = <0x10000>;
+			ftrace-size = <0x10000>;
+			pmsg-size = <0x20000>;
+		};
+	};
 };
+
+#include "msm8994-smd-rpm.dtsi"
diff --git a/arch/arm64/boot/dts/qcom/msm8992-pins.dtsi b/arch/arm64/boot/dts/qcom/msm8992-pins.dtsi
index d2a26f0f8d73..31bc9d98e31f 100644
--- a/arch/arm64/boot/dts/qcom/msm8992-pins.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8992-pins.dtsi
@@ -35,4 +35,64 @@
 			bias-pull-down;
 		};
 	};
+
+	/* 0-3 for sdc1 4-6 for sdc2 */
+	/* Order of pins */
+	/* SDC1: CLK -> 0, CMD -> 1, DATA -> 2, RCLK -> 3 */
+	/* SDC2: CLK -> 4, CMD -> 5, DATA -> 6 */
+	sdc1_clk_on: clk-on {
+		pinconf {
+			pins = "sdc1_clk";
+			bias-disable = <0>; /* No pull */
+			drive-strength = <16>; /* 16mA */
+		};
+	};
+
+	sdc1_clk_off: clk-off {
+		pinconf {
+			pins = "sdc1_clk";
+			bias-disable = <0>; /* No pull */
+			drive-strength = <2>; /* 2mA */
+		};
+	};
+
+	sdc1_cmd_on: cmd-on {
+		pinconf {
+			pins = "sdc1_cmd";
+			bias-pull-up;
+			drive-strength = <8>;
+		};
+	};
+
+	sdc1_cmd_off: cmd-off {
+		pinconf {
+			pins = "sdc1_cmd";
+			bias-pull-up = <0x3>; /* same as 3.10 ?? */
+			drive-strength = <2>; /* 2mA */
+		};
+	};
+
+	sdc1_data_on: data-on {
+		pinconf {
+			pins = "sdc1_data";
+			bias-pull-up;
+			drive-strength = <8>; /* 8mA */
+		};
+	};
+
+	sdc1_data_off: data-off {
+		pinconf {
+			pins = "sdc1_data";
+			bias-pull-up;
+			drive-strength = <2>;
+		};
+	};
+
+	sdc1_rclk_on: rclk-on {
+		bias-pull-down; /* pull down */
+	};
+
+	sdc1_rclk_off: rclk-off {
+		bias-pull-down; /* pull down */
+	};
 };
diff --git a/arch/arm64/boot/dts/qcom/msm8992.dtsi b/arch/arm64/boot/dts/qcom/msm8992.dtsi
index 171578747ed0..cf5cacdd624d 100644
--- a/arch/arm64/boot/dts/qcom/msm8992.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8992.dtsi
@@ -202,6 +202,31 @@
 			reg = <0xfc400000 0x2000>;
 		};
 
+		sdhci1: mmc@f9824900 {
+			compatible = "qcom,sdhci-msm-v4";
+			reg = <0xf9824900 0x1a0>, <0xf9824000 0x800>;
+			reg-names = "hc_mem", "core_mem";
+
+			interrupts = <GIC_SPI 123 IRQ_TYPE_NONE>,
+					<GIC_SPI 138 IRQ_TYPE_NONE>;
+			interrupt-names = "hc_irq", "pwr_irq";
+
+			clocks = <&clock_gcc GCC_SDCC1_APPS_CLK>,
+				<&clock_gcc GCC_SDCC1_AHB_CLK>;
+			clock-names = "core", "iface";
+
+			pinctrl-names = "default", "sleep";
+			pinctrl-0 = <&sdc1_clk_on &sdc1_cmd_on &sdc1_data_on
+					&sdc1_rclk_on>;
+			pinctrl-1 = <&sdc1_clk_off &sdc1_cmd_off &sdc1_data_off
+					&sdc1_rclk_off>;
+
+			regulator-always-on;
+			bus-width = <8>;
+			mmc-hs400-1_8v;
+			status = "okay";
+		};
+
 		rpm_msg_ram: memory@fc428000 {
 			compatible = "qcom,rpm-msg-ram";
 			reg = <0xfc428000 0x4000>;
@@ -231,7 +256,67 @@
 		};
 	};
 
+	smd_rpm: smd {
+		compatible = "qcom,smd";
+		rpm {
+			interrupts = <GIC_SPI 168 IRQ_TYPE_EDGE_RISING>;
+			qcom,ipc = <&apcs 8 0>;
+			qcom,smd-edge = <15>;
+			qcom,local-pid = <0>;
+			qcom,remote-pid = <6>;
+
+			rpm-requests {
+				compatible = "qcom,rpm-msm8994";
+				qcom,smd-channels = "rpm_requests";
+
+				pm8994-regulators {
+					compatible = "qcom,rpm-pm8994-regulators";
+
+					pm8994_s1: s1 {};
+					pm8994_s2: s2 {};
+					pm8994_s3: s3 {};
+					pm8994_s4: s4 {};
+					pm8994_s5: s5 {};
+					pm8994_s6: s6 {};
+					pm8994_s7: s7 {};
+
+					pm8994_l1: l1 {};
+					pm8994_l2: l2 {};
+					pm8994_l3: l3 {};
+					pm8994_l4: l4 {};
+					pm8994_l6: l6 {};
+					pm8994_l8: l8 {};
+					pm8994_l9: l9 {};
+					pm8994_l10: l10 {};
+					pm8994_l11: l11 {};
+					pm8994_l12: l12 {};
+					pm8994_l13: l13 {};
+					pm8994_l14: l14 {};
+					pm8994_l15: l15 {};
+					pm8994_l16: l16 {};
+					pm8994_l17: l17 {};
+					pm8994_l18: l18 {};
+					pm8994_l19: l19 {};
+					pm8994_l20: l20 {};
+					pm8994_l21: l21 {};
+					pm8994_l22: l22 {};
+					pm8994_l23: l23 {};
+					pm8994_l24: l24 {};
+					pm8994_l25: l25 {};
+					pm8994_l26: l26 {};
+					pm8994_l27: l27 {};
+					pm8994_l28: l28 {};
+					pm8994_l29: l29 {};
+					pm8994_l30: l30 {};
+					pm8994_l31: l31 {};
+					pm8994_l32: l32 {};
+
+					pm8994_lvs1: lvs1 {};
+					pm8994_lvs2: lvs2 {};
+				};
+			};
+		};
+	};
 };
 
-
 #include "msm8992-pins.dtsi"
diff --git a/arch/arm64/boot/dts/qcom/msm8994-smd-rpm.dtsi b/arch/arm64/boot/dts/qcom/msm8994-smd-rpm.dtsi
new file mode 100644
index 000000000000..47ebd16cb680
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8994-smd-rpm.dtsi
@@ -0,0 +1,276 @@
+/* Copyright (c) 2015, LGE Inc. All rights reserved.
+ * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+&smd_rpm {
+	rpm {
+		rpm_requests {
+			pm8994-regulators {
+
+				vdd_l1-supply = <&pm8994_s1>;
+				vdd_l2_26_28-supply = <&pm8994_s3>;
+				vdd_l3_11-supply = <&pm8994_s3>;
+				vdd_l4_27_31-supply = <&pm8994_s3>;
+				vdd_l5_7-supply = <&pm8994_s3>;
+				vdd_l6_12_32-supply = <&pm8994_s5>;
+				vdd_l8_16_30-supply = <&vreg_vph_pwr>;
+				vdd_l9_10_18_22-supply = <&vreg_vph_pwr>;
+				vdd_l13_19_23_24-supply = <&vreg_vph_pwr>;
+				vdd_l14_15-supply = <&pm8994_s5>;
+				vdd_l17_29-supply = <&vreg_vph_pwr>;
+				vdd_l20_21-supply = <&vreg_vph_pwr>;
+				vdd_l25-supply = <&pm8994_s5>;
+				vdd_lvs1_2 = <&pm8994_s4>;
+
+				s1 {
+					regulator-min-microvolt = <800000>;
+					regulator-max-microvolt = <800000>;
+				};
+
+				s2 {
+					/* TODO */
+				};
+
+				s3 {
+					regulator-min-microvolt = <1300000>;
+					regulator-max-microvolt = <1300000>;
+				};
+
+				s4 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					regulator-allow-set-load;
+					regulator-system-load = <325000>;
+				};
+
+				s5 {
+					regulator-min-microvolt = <2150000>;
+					regulator-max-microvolt = <2150000>;
+				};
+
+				s7 {
+					regulator-min-microvolt = <1000000>;
+					regulator-max-microvolt = <1000000>;
+				};
+
+				l1 {
+					regulator-min-microvolt = <1000000>;
+					regulator-max-microvolt = <1000000>;
+				};
+
+				l2 {
+					regulator-min-microvolt = <1250000>;
+					regulator-max-microvolt = <1250000>;
+				};
+
+				l3 {
+					regulator-min-microvolt = <1200000>;
+					regulator-max-microvolt = <1200000>;
+				};
+
+				l4 {
+					regulator-min-microvolt = <1225000>;
+					regulator-max-microvolt = <1225000>;
+				};
+
+				l5 {
+					/* TODO */
+				};
+
+				l6 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+				};
+
+				l7 {
+					/* TODO */
+				};
+
+				l8 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+				};
+
+				l9 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+				};
+
+				l10 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					qcom,init-voltage = <1800000>;
+				};
+
+				l11 {
+					regulator-min-microvolt = <1200000>;
+					regulator-max-microvolt = <1200000>;
+					qcom,init-voltage = <1200000>;
+				};
+
+				l12 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					qcom,init-voltage = <1800000>;
+					proxy-supply = <&pm8994_l12>;
+					qcom,proxy-consumer-enable;
+					qcom,proxy-consumer-current = <10000>;
+					status = "okay";
+				};
+
+				l13 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <2950000>;
+					qcom,init-voltage = <2950000>;
+					status = "okay";
+				};
+
+				l14 {
+					regulator-min-microvolt = <1200000>;
+					regulator-max-microvolt = <1200000>;
+					qcom,init-voltage = <1200000>;
+					proxy-supply = <&pm8994_l14>;
+					qcom,proxy-consumer-enable;
+					qcom,proxy-consumer-current = <10000>;
+					status = "okay";
+				};
+
+				l15 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					qcom,init-voltage = <1800000>;
+					status = "okay";
+				};
+
+				l16 {
+					regulator-min-microvolt = <2700000>;
+					regulator-max-microvolt = <2700000>;
+					qcom,init-voltage = <2700000>;
+					status = "okay";
+				};
+
+				l17 {
+					regulator-min-microvolt = <2700000>;
+					regulator-max-microvolt = <2700000>;
+					qcom,init-voltage = <2700000>;
+					status = "okay";
+				};
+
+				l18 {
+					regulator-min-microvolt = <3000000>;
+					regulator-max-microvolt = <3000000>;
+					regulator-always-on;
+					qcom,init-voltage = <3000000>;
+					qcom,init-ldo-mode = <1>;
+				};
+
+				l19 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					qcom,init-voltage = <1800000>;
+					status = "okay";
+				};
+
+				l20 {
+					regulator-min-microvolt = <2950000>;
+					regulator-max-microvolt = <2950000>;
+					regulator-always-on;
+					regulator-boot-on;
+					regulator-allow-set-load;
+					regulator-system-load = <570000>;
+				};
+
+				l21 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					regulator-always-on;
+					qcom,init-voltage = <1800000>;
+				};
+
+				l22 {
+					regulator-min-microvolt = <3100000>;
+					regulator-max-microvolt = <3100000>;
+					qcom,init-voltage = <3100000>;
+				};
+
+				l23 {
+					regulator-min-microvolt = <2800000>;
+					regulator-max-microvolt = <2800000>;
+					qcom,init-voltage = <2800000>;
+				};
+
+				l24 {
+					regulator-min-microvolt = <3075000>;
+					regulator-max-microvolt = <3150000>;
+					qcom,init-voltage = <3075000>;
+				};
+
+				l25 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					qcom,init-voltage = <1800000>;
+				};
+
+				l26 {
+					/* TODO: value from downstream
+					regulator-min-microvolt = <987500>;
+					fails to apply */
+				};
+
+				l27 {
+					regulator-min-microvolt = <1050000>;
+					regulator-max-microvolt = <1050000>;
+					qcom,init-voltage = <1050000>;
+				};
+
+				l28 {
+					regulator-min-microvolt = <1000000>;
+					regulator-max-microvolt = <1000000>;
+					qcom,init-voltage = <1000000>;
+					proxy-supply = <&pm8994_l28>;
+					qcom,proxy-consumer-enable;
+					qcom,proxy-consumer-current = <10000>;
+				};
+
+				l29 {
+					/* TODO: Unsupported voltage range.
+					regulator-min-microvolt = <2800000>;
+					regulator-max-microvolt = <2800000>;
+					qcom,init-voltage = <2800000>;
+					*/
+				};
+
+				l30 {
+					/* TODO: get this verified
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					qcom,init-voltage = <1800000>;
+					*/
+				};
+
+				l31 {
+					regulator-min-microvolt = <1262500>;
+					regulator-max-microvolt = <1262500>;
+					qcom,init-voltage = <1262500>;
+				};
+
+				l32 {
+					/* TODO: get this verified
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					qcom,init-voltage = <1800000>;
+					*/
+				};
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8996.dtsi b/arch/arm64/boot/dts/qcom/msm8996.dtsi
index 410ae787ebb4..8c7f9ca25b53 100644
--- a/arch/arm64/boot/dts/qcom/msm8996.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8996.dtsi
@@ -419,6 +419,16 @@
 			#clock-cells = <1>;
 		};
 
+		blsp1_uart1: serial@7570000 {
+			compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm";
+			reg = <0x07570000 0x1000>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_BLSP1_UART2_APPS_CLK>,
+				 <&gcc GCC_BLSP1_AHB_CLK>;
+			clock-names = "core", "iface";
+			status = "disabled";
+		};
+
 		blsp1_spi0: spi@7575000 {
 			compatible = "qcom,spi-qup-v2.2.1";
 			reg = <0x07575000 0x600>;
@@ -437,7 +447,7 @@
 		blsp2_i2c0: i2c@75b5000 {
 			compatible = "qcom,i2c-qup-v2.2.1";
 			reg = <0x075b5000 0x1000>;
-			interrupts = <GIC_SPI 101 0>;
+			interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP2_AHB_CLK>,
 				<&gcc GCC_BLSP2_QUP1_I2C_APPS_CLK>;
 			clock-names = "iface", "core";
@@ -468,7 +478,7 @@
 		blsp2_i2c1: i2c@75b6000 {
 			compatible = "qcom,i2c-qup-v2.2.1";
 			reg = <0x075b6000 0x1000>;
-			interrupts = <GIC_SPI 102 0>;
+			interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP2_AHB_CLK>,
 				<&gcc GCC_BLSP2_QUP2_I2C_APPS_CLK>;
 			clock-names = "iface", "core";
@@ -493,7 +503,7 @@
 		blsp1_i2c2: i2c@7577000 {
 			compatible = "qcom,i2c-qup-v2.2.1";
 			reg = <0x07577000 0x1000>;
-			interrupts = <GIC_SPI 97 0>;
+			interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP1_AHB_CLK>,
 				<&gcc GCC_BLSP1_QUP3_I2C_APPS_CLK>;
 			clock-names = "iface", "core";
@@ -526,7 +536,8 @@
 			 reg = <0x74a4900 0x314>, <0x74a4000 0x800>;
 			 reg-names = "hc_mem", "core_mem";
 
-			 interrupts = <0 125 0>, <0 221 0>;
+			 interrupts = <0 125 IRQ_TYPE_LEVEL_HIGH>,
+				      <0 221 IRQ_TYPE_LEVEL_HIGH>;
 			 interrupt-names = "hc_irq", "pwr_irq";
 
 			 clock-names = "iface", "core", "xo";
@@ -623,6 +634,91 @@
 			#interrupt-cells = <4>;
 		};
 
+		ufsphy: phy@627000 {
+			compatible = "qcom,msm8996-ufs-phy-qmp-14nm";
+			reg = <0x627000 0xda8>;
+			reg-names = "phy_mem";
+			#phy-cells = <0>;
+
+			vdda-phy-supply = <&pm8994_l28>;
+			vdda-pll-supply = <&pm8994_l12>;
+
+			vdda-phy-max-microamp = <18380>;
+			vdda-pll-max-microamp = <9440>;
+
+			vddp-ref-clk-supply = <&pm8994_l25>;
+			vddp-ref-clk-max-microamp = <100>;
+			vddp-ref-clk-always-on;
+
+			clock-names = "ref_clk_src", "ref_clk";
+			clocks = <&rpmcc RPM_SMD_LN_BB_CLK>,
+				 <&gcc GCC_UFS_CLKREF_CLK>;
+			status = "disabled";
+		};
+
+		ufshc@624000 {
+			compatible = "qcom,ufshc";
+			reg = <0x624000 0x2500>;
+			interrupts = <GIC_SPI 265 IRQ_TYPE_LEVEL_HIGH>;
+
+			phys = <&ufsphy>;
+			phy-names = "ufsphy";
+
+			vcc-supply = <&pm8994_l20>;
+			vccq-supply = <&pm8994_l25>;
+			vccq2-supply = <&pm8994_s4>;
+
+			vcc-max-microamp = <600000>;
+			vccq-max-microamp = <450000>;
+			vccq2-max-microamp = <450000>;
+
+			power-domains = <&gcc UFS_GDSC>;
+
+			clock-names =
+				"core_clk_src",
+				"core_clk",
+				"bus_clk",
+				"bus_aggr_clk",
+				"iface_clk",
+				"core_clk_unipro_src",
+				"core_clk_unipro",
+				"core_clk_ice",
+				"ref_clk",
+				"tx_lane0_sync_clk",
+				"rx_lane0_sync_clk";
+			clocks =
+				<&gcc UFS_AXI_CLK_SRC>,
+				<&gcc GCC_UFS_AXI_CLK>,
+				<&gcc GCC_SYS_NOC_UFS_AXI_CLK>,
+				<&gcc GCC_AGGRE2_UFS_AXI_CLK>,
+				<&gcc GCC_UFS_AHB_CLK>,
+				<&gcc UFS_ICE_CORE_CLK_SRC>,
+				<&gcc GCC_UFS_UNIPRO_CORE_CLK>,
+				<&gcc GCC_UFS_ICE_CORE_CLK>,
+				<&rpmcc RPM_SMD_LN_BB_CLK>,
+				<&gcc GCC_UFS_TX_SYMBOL_0_CLK>,
+				<&gcc GCC_UFS_RX_SYMBOL_0_CLK>;
+			freq-table-hz =
+				<100000000 200000000>,
+				<0 0>,
+				<0 0>,
+				<0 0>,
+				<0 0>,
+				<150000000 300000000>,
+				<0 0>,
+				<0 0>,
+				<0 0>,
+				<0 0>,
+				<0 0>;
+
+			lanes-per-direction = <1>;
+			status = "disabled";
+
+			ufs_variant {
+				compatible = "qcom,ufs_variant";
+			};
+		};
+
 		mmcc: clock-controller@8c0000 {
 			compatible = "qcom,mmcc-msm8996";
 			#clock-cells = <1>;
@@ -809,7 +905,7 @@
 			dwc3@7600000 {
 				compatible = "snps,dwc3";
 				reg = <0x7600000 0xcc00>;
-				interrupts = <0 138 0>;
+				interrupts = <0 138 IRQ_TYPE_LEVEL_HIGH>;
 				phys = <&hsusb_phy2>;
 				phy-names = "usb2-phy";
 			};
@@ -838,7 +934,7 @@
 			dwc3@6a00000 {
 				compatible = "snps,dwc3";
 				reg = <0x6a00000 0xcc00>;
-				interrupts = <0 131 0>;
+				interrupts = <0 131 IRQ_TYPE_LEVEL_HIGH>;
 				phys = <&hsusb_phy1>, <&ssusb_phy_0>;
 				phy-names = "usb2-phy", "usb3-phy";
 			};
@@ -851,7 +947,7 @@
 			#size-cells = <1>;
 			ranges;
 
-			pcie0: qcom,pcie@600000 {
+			pcie0: pcie@600000 {
 				compatible = "qcom,pcie-msm8996", "snps,dw-pcie";
 				status = "disabled";
 				power-domains = <&gcc PCIE0_GDSC>;
@@ -872,7 +968,7 @@
 				ranges = <0x01000000 0x0 0x0c200000 0x0c200000 0x0 0x100000>,
 					<0x02000000 0x0 0x0c300000 0x0c300000 0x0 0xd00000>;
 
-				interrupts = <GIC_SPI 405 IRQ_TYPE_NONE>;
+				interrupts = <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>;
 				interrupt-names = "msi";
 				#interrupt-cells = <1>;
 				interrupt-map-mask = <0 0 0 0x7>;
@@ -904,7 +1000,7 @@
 
 			};
 
-			pcie1: qcom,pcie@608000 {
+			pcie1: pcie@608000 {
 				compatible = "qcom,pcie-msm8996", "snps,dw-pcie";
 				power-domains = <&gcc PCIE1_GDSC>;
 				bus-range = <0x00 0xff>;
@@ -927,7 +1023,7 @@
 				ranges = <0x01000000 0x0 0x0d200000 0x0d200000 0x0 0x100000>,
 					<0x02000000 0x0 0x0d300000 0x0d300000 0x0 0xd00000>;
 
-				interrupts = <GIC_SPI 413 IRQ_TYPE_NONE>;
+				interrupts = <GIC_SPI 413 IRQ_TYPE_LEVEL_HIGH>;
 				interrupt-names = "msi";
 				#interrupt-cells = <1>;
 				interrupt-map-mask = <0 0 0 0x7>;
@@ -957,7 +1053,7 @@
 						"bus_slave";
 			};
 
-			pcie2: qcom,pcie@610000 {
+			pcie2: pcie@610000 {
 				compatible = "qcom,pcie-msm8996", "snps,dw-pcie";
 				power-domains = <&gcc PCIE2_GDSC>;
 				bus-range = <0x00 0xff>;
@@ -980,7 +1076,7 @@
 
 				device_type = "pci";
 
-				interrupts = <GIC_SPI 421 IRQ_TYPE_NONE>;
+				interrupts = <GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH>;
 				interrupt-names = "msi";
 				#interrupt-cells = <1>;
 				interrupt-map-mask = <0 0 0 0x7>;
diff --git a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts
new file mode 100644
index 000000000000..979ab49913f1
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * SDM845 MTP board device tree source
+ *
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ */
+
+/dts-v1/;
+
+#include "sdm845.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. SDM845 MTP";
+	compatible = "qcom,sdm845-mtp";
+};
diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi
new file mode 100644
index 000000000000..cdaabeb3c995
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi
@@ -0,0 +1,327 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * SDM845 SoC device tree source
+ *
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	interrupt-parent = <&intc>;
+
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	chosen { };
+
+	memory@80000000 {
+		device_type = "memory";
+		/* We expect the bootloader to fill in the size */
+		reg = <0 0x80000000 0 0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		memory@85fc0000 {
+			reg = <0 0x85fc0000 0 0x20000>;
+			no-map;
+		};
+
+		memory@85fe0000 {
+			compatible = "qcom,cmd-db";
+			reg = <0x0 0x85fe0000 0x0 0x20000>;
+			no-map;
+		};
+
+		smem_mem: memory@86000000 {
+			reg = <0x0 0x86000000 0x0 0x200000>;
+			no-map;
+		};
+
+		memory@86200000 {
+			reg = <0 0x86200000 0 0x2d00000>;
+			no-map;
+		};
+	};
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		CPU0: cpu@0 {
+			device_type = "cpu";
+			compatible = "qcom,kryo385";
+			reg = <0x0 0x0>;
+			enable-method = "psci";
+			next-level-cache = <&L2_0>;
+			L2_0: l2-cache {
+				compatible = "cache";
+				next-level-cache = <&L3_0>;
+				L3_0: l3-cache {
+				      compatible = "cache";
+				};
+			};
+		};
+
+		CPU1: cpu@100 {
+			device_type = "cpu";
+			compatible = "qcom,kryo385";
+			reg = <0x0 0x100>;
+			enable-method = "psci";
+			next-level-cache = <&L2_100>;
+			L2_100: l2-cache {
+				compatible = "cache";
+				next-level-cache = <&L3_0>;
+			};
+		};
+
+		CPU2: cpu@200 {
+			device_type = "cpu";
+			compatible = "qcom,kryo385";
+			reg = <0x0 0x200>;
+			enable-method = "psci";
+			next-level-cache = <&L2_200>;
+			L2_200: l2-cache {
+				compatible = "cache";
+				next-level-cache = <&L3_0>;
+			};
+		};
+
+		CPU3: cpu@300 {
+			device_type = "cpu";
+			compatible = "qcom,kryo385";
+			reg = <0x0 0x300>;
+			enable-method = "psci";
+			next-level-cache = <&L2_300>;
+			L2_300: l2-cache {
+				compatible = "cache";
+				next-level-cache = <&L3_0>;
+			};
+		};
+
+		CPU4: cpu@400 {
+			device_type = "cpu";
+			compatible = "qcom,kryo385";
+			reg = <0x0 0x400>;
+			enable-method = "psci";
+			next-level-cache = <&L2_400>;
+			L2_400: l2-cache {
+				compatible = "cache";
+				next-level-cache = <&L3_0>;
+			};
+		};
+
+		CPU5: cpu@500 {
+			device_type = "cpu";
+			compatible = "qcom,kryo385";
+			reg = <0x0 0x500>;
+			enable-method = "psci";
+			next-level-cache = <&L2_500>;
+			L2_500: l2-cache {
+				compatible = "cache";
+				next-level-cache = <&L3_0>;
+			};
+		};
+
+		CPU6: cpu@600 {
+			device_type = "cpu";
+			compatible = "qcom,kryo385";
+			reg = <0x0 0x600>;
+			enable-method = "psci";
+			next-level-cache = <&L2_600>;
+			L2_600: l2-cache {
+				compatible = "cache";
+				next-level-cache = <&L3_0>;
+			};
+		};
+
+		CPU7: cpu@700 {
+			device_type = "cpu";
+			compatible = "qcom,kryo385";
+			reg = <0x0 0x700>;
+			enable-method = "psci";
+			next-level-cache = <&L2_700>;
+			L2_700: l2-cache {
+				compatible = "cache";
+				next-level-cache = <&L3_0>;
+			};
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 1 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 2 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 3 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 0 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	clocks {
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <38400000>;
+			clock-output-names = "xo_board";
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <32764>;
+		};
+	};
+
+	tcsr_mutex: hwlock {
+		compatible = "qcom,tcsr-mutex";
+		syscon = <&tcsr_mutex_regs 0 0x1000>;
+		#hwlock-cells = <1>;
+	};
+
+	smem {
+		compatible = "qcom,smem";
+		memory-region = <&smem_mem>;
+		hwlocks = <&tcsr_mutex 3>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+	};
+
+	soc: soc {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0 0 0xffffffff>;
+		compatible = "simple-bus";
+
+		gcc: clock-controller@100000 {
+			compatible = "qcom,gcc-sdm845";
+			reg = <0x100000 0x1f0000>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		tcsr_mutex_regs: syscon@1f40000 {
+			compatible = "syscon";
+			reg = <0x1f40000 0x40000>;
+		};
+
+		tlmm: pinctrl@3400000 {
+			compatible = "qcom,sdm845-pinctrl";
+			reg = <0x03400000 0xc00000>;
+			interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		spmi_bus: spmi@c440000 {
+			compatible = "qcom,spmi-pmic-arb";
+			reg = <0xc440000 0x1100>,
+			      <0xc600000 0x2000000>,
+			      <0xe600000 0x100000>,
+			      <0xe700000 0xa0000>,
+			      <0xc40a000 0x26000>;
+			reg-names = "core", "chnls", "obsrvr", "intr", "cnfg";
+			interrupt-names = "periph_irq";
+			interrupts = <GIC_SPI 481 IRQ_TYPE_LEVEL_HIGH>;
+			qcom,ee = <0>;
+			qcom,channel = <0>;
+			#address-cells = <2>;
+			#size-cells = <0>;
+			interrupt-controller;
+			#interrupt-cells = <4>;
+			cell-index = <0>;
+		};
+
+		apss_shared: mailbox@17990000 {
+			compatible = "qcom,sdm845-apss-shared";
+			reg = <0x17990000 0x1000>;
+			#mbox-cells = <1>;
+		};
+
+		intc: interrupt-controller@17a00000 {
+			compatible = "arm,gic-v3";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			reg = <0x17a00000 0x10000>,     /* GICD */
+			      <0x17a60000 0x100000>;    /* GICR * 8 */
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+
+			gic-its@17a40000 {
+				compatible = "arm,gic-v3-its";
+				msi-controller;
+				#msi-cells = <1>;
+				reg = <0x17a40000 0x20000>;
+				status = "disabled";
+			};
+		};
+
+		timer@17c90000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			compatible = "arm,armv7-timer-mem";
+			reg = <0x17c90000 0x1000>;
+
+			frame@17ca0000 {
+				frame-number = <0>;
+				interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0x17ca0000 0x1000>,
+				      <0x17cb0000 0x1000>;
+			};
+
+			frame@17cc0000 {
+				frame-number = <1>;
+				interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0x17cc0000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@17cd0000 {
+				frame-number = <2>;
+				interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0x17cd0000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@17ce0000 {
+				frame-number = <3>;
+				interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0x17ce0000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@17cf0000 {
+				frame-number = <4>;
+				interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0x17cf0000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@17d00000 {
+				frame-number = <5>;
+				interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0x17d00000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@17d10000 {
+				frame-number = <6>;
+				interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0x17d10000 0x1000>;
+				status = "disabled";
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/Makefile b/arch/arm64/boot/dts/renesas/Makefile
index 5ede06000ea4..9e2394bc3c62 100644
--- a/arch/arm64/boot/dts/renesas/Makefile
+++ b/arch/arm64/boot/dts/renesas/Makefile
@@ -9,5 +9,6 @@ dtb-$(CONFIG_ARCH_R8A7796) += r8a7796-m3ulcb-kf.dtb
 dtb-$(CONFIG_ARCH_R8A7796) += r8a7796-salvator-xs.dtb
 dtb-$(CONFIG_ARCH_R8A77965) += r8a77965-salvator-x.dtb r8a77965-salvator-xs.dtb
 dtb-$(CONFIG_ARCH_R8A77970) += r8a77970-eagle.dtb r8a77970-v3msk.dtb
-dtb-$(CONFIG_ARCH_R8A77980) += r8a77980-condor.dtb
+dtb-$(CONFIG_ARCH_R8A77980) += r8a77980-condor.dtb r8a77980-v3hsk.dtb
+dtb-$(CONFIG_ARCH_R8A77990) += r8a77990-ebisu.dtb
 dtb-$(CONFIG_ARCH_R8A77995) += r8a77995-draak.dtb
diff --git a/arch/arm64/boot/dts/renesas/r8a7795-es1-salvator-x.dts b/arch/arm64/boot/dts/renesas/r8a7795-es1-salvator-x.dts
index 7f2a3d923f21..3f46345a4644 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795-es1-salvator-x.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7795-es1-salvator-x.dts
@@ -56,6 +56,12 @@
 	status = "okay";
 };
 
+&sound_card {
+	dais = <&rsnd_port0	/* ak4613 */
+		&rsnd_port1	/* HDMI0  */
+		&rsnd_port2>;	/* HDMI1  */
+};
+
 &hdmi0 {
 	status = "okay";
 
@@ -66,6 +72,12 @@
 				remote-endpoint = <&hdmi0_con>;
 			};
 		};
+		port@2 {
+			reg = <2>;
+			dw_hdmi0_snd_in: endpoint {
+				remote-endpoint = <&rsnd_endpoint1>;
+			};
+		};
 	};
 };
 
@@ -83,6 +95,12 @@
 				remote-endpoint = <&hdmi1_con>;
 			};
 		};
+		port@2 {
+			reg = <2>;
+			dw_hdmi1_snd_in: endpoint {
+				remote-endpoint = <&rsnd_endpoint2>;
+			};
+		};
 	};
 };
 
@@ -94,6 +112,34 @@
 	status = "okay";
 };
 
+&rcar_sound {
+	ports {
+		/* rsnd_port0 is on salvator-common */
+		rsnd_port1: port@1 {
+			rsnd_endpoint1: endpoint {
+				remote-endpoint = <&dw_hdmi0_snd_in>;
+
+				dai-format = "i2s";
+				bitclock-master = <&rsnd_endpoint1>;
+				frame-master = <&rsnd_endpoint1>;
+
+				playback = <&ssi2>;
+			};
+		};
+		rsnd_port2: port@2 {
+			rsnd_endpoint2: endpoint {
+				remote-endpoint = <&dw_hdmi1_snd_in>;
+
+				dai-format = "i2s";
+				bitclock-master = <&rsnd_endpoint2>;
+				frame-master = <&rsnd_endpoint2>;
+
+				playback = <&ssi3>;
+			};
+		};
+	};
+};
+
 &pfc {
 	usb2_pins: usb2 {
 		groups = "usb2";
diff --git a/arch/arm64/boot/dts/renesas/r8a7795-es1.dtsi b/arch/arm64/boot/dts/renesas/r8a7795-es1.dtsi
index f9acd125d687..e19dcd6cb767 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795-es1.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a7795-es1.dtsi
@@ -39,7 +39,6 @@
 		reg = <0 0xe7730000 0 0x1000>;
 		renesas,ipmmu-main = <&ipmmu_mm 8>;
 		#iommu-cells = <1>;
-		status = "disabled";
 	};
 
 	/delete-node/ usb-phy@ee0e0200;
@@ -108,6 +107,61 @@
 		resets = <&cpg 117>;
 		renesas,fcp = <&fcpf2>;
 	};
+
+	csi21: csi2@fea90000 {
+		compatible = "renesas,r8a7795-csi2";
+		reg = <0 0xfea90000 0 0x10000>;
+		interrupts = <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cpg CPG_MOD 713>;
+		power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+		resets = <&cpg 713>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@1 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				reg = <1>;
+
+				csi21vin0: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&vin0csi21>;
+				};
+				csi21vin1: endpoint@1 {
+					reg = <1>;
+					remote-endpoint = <&vin1csi21>;
+				};
+				csi21vin2: endpoint@2 {
+					reg = <2>;
+					remote-endpoint = <&vin2csi21>;
+				};
+				csi21vin3: endpoint@3 {
+					reg = <3>;
+					remote-endpoint = <&vin3csi21>;
+				};
+				csi21vin4: endpoint@4 {
+					reg = <4>;
+					remote-endpoint = <&vin4csi21>;
+				};
+				csi21vin5: endpoint@5 {
+					reg = <5>;
+					remote-endpoint = <&vin5csi21>;
+				};
+				csi21vin6: endpoint@6 {
+					reg = <6>;
+					remote-endpoint = <&vin6csi21>;
+				};
+				csi21vin7: endpoint@7 {
+					reg = <7>;
+					remote-endpoint = <&vin7csi21>;
+				};
+			};
+		};
+	};
 };
 
 &gpio1 {
@@ -175,3 +229,91 @@
 &du {
 	vsps = <&vspd0 &vspd1 &vspd2 &vspd3>;
 };
+
+&vin0 {
+	ports {
+		port@1 {
+			vin0csi21: endpoint@1 {
+				reg = <1>;
+				remote-endpoint= <&csi21vin0>;
+			};
+		};
+	};
+};
+
+&vin1 {
+	ports {
+		port@1 {
+			vin1csi21: endpoint@1 {
+				reg = <1>;
+				remote-endpoint= <&csi21vin1>;
+			};
+		};
+	};
+};
+
+&vin2 {
+	ports {
+		port@1 {
+			vin2csi21: endpoint@1 {
+				reg = <1>;
+				remote-endpoint= <&csi21vin2>;
+			};
+		};
+	};
+};
+
+&vin3 {
+	ports {
+		port@1 {
+			vin3csi21: endpoint@1 {
+				reg = <1>;
+				remote-endpoint= <&csi21vin3>;
+			};
+		};
+	};
+};
+
+&vin4 {
+	ports {
+		port@1 {
+			vin4csi21: endpoint@1 {
+				reg = <1>;
+				remote-endpoint= <&csi21vin4>;
+			};
+		};
+	};
+};
+
+&vin5 {
+	ports {
+		port@1 {
+			vin5csi21: endpoint@1 {
+				reg = <1>;
+				remote-endpoint= <&csi21vin5>;
+			};
+		};
+	};
+};
+
+&vin6 {
+	ports {
+		port@1 {
+			vin6csi21: endpoint@1 {
+				reg = <1>;
+				remote-endpoint= <&csi21vin6>;
+			};
+		};
+	};
+};
+
+&vin7 {
+	ports {
+		port@1 {
+			vin7csi21: endpoint@1 {
+				reg = <1>;
+				remote-endpoint= <&csi21vin7>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts b/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts
index af467419266a..0efbef5ea9b7 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts
@@ -56,6 +56,12 @@
 	status = "okay";
 };
 
+&sound_card {
+	dais = <&rsnd_port0	/* ak4613 */
+		&rsnd_port1	/* HDMI0  */
+		&rsnd_port2>;	/* HDMI1  */
+};
+
 &hdmi0 {
 	status = "okay";
 
@@ -66,6 +72,12 @@
 				remote-endpoint = <&hdmi0_con>;
 			};
 		};
+		port@2 {
+			reg = <2>;
+			dw_hdmi0_snd_in: endpoint {
+				remote-endpoint = <&rsnd_endpoint1>;
+			};
+		};
 	};
 };
 
@@ -83,6 +95,12 @@
 				remote-endpoint = <&hdmi1_con>;
 			};
 		};
+		port@2 {
+			reg = <2>;
+			dw_hdmi1_snd_in: endpoint {
+				remote-endpoint = <&rsnd_endpoint2>;
+			};
+		};
 	};
 };
 
@@ -94,6 +112,34 @@
 	status = "okay";
 };
 
+&rcar_sound {
+	ports {
+		/* rsnd_port0 is on salvator-common */
+		rsnd_port1: port@1 {
+			rsnd_endpoint1: endpoint {
+				remote-endpoint = <&dw_hdmi0_snd_in>;
+
+				dai-format = "i2s";
+				bitclock-master = <&rsnd_endpoint1>;
+				frame-master = <&rsnd_endpoint1>;
+
+				playback = <&ssi2>;
+			};
+		};
+		rsnd_port2: port@2 {
+			rsnd_endpoint2: endpoint {
+				remote-endpoint = <&dw_hdmi1_snd_in>;
+
+				dai-format = "i2s";
+				bitclock-master = <&rsnd_endpoint2>;
+				frame-master = <&rsnd_endpoint2>;
+
+				playback = <&ssi3>;
+			};
+		};
+	};
+};
+
 &pfc {
 	usb2_pins: usb2 {
 		groups = "usb2";
diff --git a/arch/arm64/boot/dts/renesas/r8a7795-salvator-xs.dts b/arch/arm64/boot/dts/renesas/r8a7795-salvator-xs.dts
index 8b50ceb746e8..e231b5a7cbab 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795-salvator-xs.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7795-salvator-xs.dts
@@ -56,6 +56,22 @@
 	status = "okay";
 };
 
+&ehci3 {
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&hsusb3 {
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&sound_card {
+	dais = <&rsnd_port0	/* ak4613 */
+		&rsnd_port1	/* HDMI0  */
+		&rsnd_port2>;	/* HDMI1  */
+};
+
 &hdmi0 {
 	status = "okay";
 
@@ -66,6 +82,12 @@
 				remote-endpoint = <&hdmi0_con>;
 			};
 		};
+		port@2 {
+			reg = <2>;
+			dw_hdmi0_snd_in: endpoint {
+				remote-endpoint = <&rsnd_endpoint1>;
+			};
+		};
 	};
 };
 
@@ -83,6 +105,12 @@
 				remote-endpoint = <&hdmi1_con>;
 			};
 		};
+		port@2 {
+			reg = <2>;
+			dw_hdmi1_snd_in: endpoint {
+				remote-endpoint = <&rsnd_endpoint2>;
+			};
+		};
 	};
 };
 
@@ -94,11 +122,61 @@
 	status = "okay";
 };
 
+&ohci3 {
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&rcar_sound {
+	ports {
+		/* rsnd_port0 is on salvator-common */
+		rsnd_port1: port@1 {
+			rsnd_endpoint1: endpoint {
+				remote-endpoint = <&dw_hdmi0_snd_in>;
+
+				dai-format = "i2s";
+				bitclock-master = <&rsnd_endpoint1>;
+				frame-master = <&rsnd_endpoint1>;
+
+				playback = <&ssi2>;
+			};
+		};
+		rsnd_port2: port@2 {
+			rsnd_endpoint2: endpoint {
+				remote-endpoint = <&dw_hdmi1_snd_in>;
+
+				dai-format = "i2s";
+				bitclock-master = <&rsnd_endpoint2>;
+				frame-master = <&rsnd_endpoint2>;
+
+				playback = <&ssi3>;
+			};
+		};
+	};
+};
+
 &pfc {
 	usb2_pins: usb2 {
 		groups = "usb2";
 		function = "usb2";
 	};
+
+	/*
+	 * - On Salvator-X[S], GP6_3[01] are connected to ADV7482 as irq pins
+	 *   (when SW31 is the default setting on Salvator-XS).
+	 * - If SW31 is the default setting, you cannot use USB2.0 ch3 on
+	 *   r8a7795 with Salvator-XS.
+	 *   Hence the SW31 setting must be changed like 2) below.
+	 *   1) Default setting of SW31: ON-ON-OFF-OFF-OFF-OFF:
+	 *	- Connect GP6_3[01] to ADV7842.
+	 *   2) Changed setting of SW31: OFF-OFF-ON-ON-ON-ON:
+	 *	- Connect GP6_3[01] to BD082065 (USB2.0 ch3's host power).
+	 *	- Connect GP6_{04,21} to ADV7842.
+	 */
+	usb2_ch3_pins: usb2_ch3 {
+		groups = "usb2_ch3";
+		function = "usb2_ch3";
+	};
 };
 
 &usb2_phy2 {
@@ -107,3 +185,10 @@
 
 	status = "okay";
 };
+
+&usb2_phy3 {
+	pinctrl-0 = <&usb2_ch3_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a7795.dtsi b/arch/arm64/boot/dts/renesas/r8a7795.dtsi
index 1d5e3ac0231c..d842940b2f43 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a7795.dtsi
@@ -30,6 +30,91 @@
 		i2c7 = &i2c_dvfs;
 	};
 
+	/*
+	 * The external audio clocks are configured as 0 Hz fixed frequency
+	 * clocks by default.
+	 * Boards that provide audio clocks should override them.
+	 */
+	audio_clk_a: audio_clk_a {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	audio_clk_b: audio_clk_b {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	audio_clk_c: audio_clk_c {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	/* External CAN clock - to be overridden by boards that provide it */
+	can_clk: can {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	cluster0_opp: opp_table0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-500000000 {
+			opp-hz = /bits/ 64 <500000000>;
+			opp-microvolt = <830000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <830000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1500000000 {
+			opp-hz = /bits/ 64 <1500000000>;
+			opp-microvolt = <830000>;
+			clock-latency-ns = <300000>;
+			opp-suspend;
+		};
+		opp-1600000000 {
+			opp-hz = /bits/ 64 <1600000000>;
+			opp-microvolt = <900000>;
+			clock-latency-ns = <300000>;
+			turbo-mode;
+		};
+		opp-1700000000 {
+			opp-hz = /bits/ 64 <1700000000>;
+			opp-microvolt = <960000>;
+			clock-latency-ns = <300000>;
+			turbo-mode;
+		};
+	};
+
+	cluster1_opp: opp_table1 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -47,7 +132,7 @@
 		};
 
 		a57_1: cpu@1 {
-			compatible = "arm,cortex-a57","arm,armv8";
+			compatible = "arm,cortex-a57", "arm,armv8";
 			reg = <0x1>;
 			device_type = "cpu";
 			power-domains = <&sysc R8A7795_PD_CA57_CPU1>;
@@ -59,7 +144,7 @@
 		};
 
 		a57_2: cpu@2 {
-			compatible = "arm,cortex-a57","arm,armv8";
+			compatible = "arm,cortex-a57", "arm,armv8";
 			reg = <0x2>;
 			device_type = "cpu";
 			power-domains = <&sysc R8A7795_PD_CA57_CPU2>;
@@ -71,7 +156,7 @@
 		};
 
 		a57_3: cpu@3 {
-			compatible = "arm,cortex-a57","arm,armv8";
+			compatible = "arm,cortex-a57", "arm,armv8";
 			reg = <0x3>;
 			device_type = "cpu";
 			power-domains = <&sysc R8A7795_PD_CA57_CPU3>;
@@ -94,7 +179,7 @@
 		};
 
 		a53_1: cpu@101 {
-			compatible = "arm,cortex-a53","arm,armv8";
+			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0x101>;
 			device_type = "cpu";
 			power-domains = <&sysc R8A7795_PD_CA53_CPU1>;
@@ -105,7 +190,7 @@
 		};
 
 		a53_2: cpu@102 {
-			compatible = "arm,cortex-a53","arm,armv8";
+			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0x102>;
 			device_type = "cpu";
 			power-domains = <&sysc R8A7795_PD_CA53_CPU2>;
@@ -116,7 +201,7 @@
 		};
 
 		a53_3: cpu@103 {
-			compatible = "arm,cortex-a53","arm,armv8";
+			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0x103>;
 			device_type = "cpu";
 			power-domains = <&sysc R8A7795_PD_CA53_CPU3>;
@@ -155,91 +240,6 @@
 		clock-frequency = <0>;
 	};
 
-	/*
-	 * The external audio clocks are configured as 0 Hz fixed frequency
-	 * clocks by default.
-	 * Boards that provide audio clocks should override them.
-	 */
-	audio_clk_a: audio_clk_a {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
-	};
-
-	audio_clk_b: audio_clk_b {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
-	};
-
-	audio_clk_c: audio_clk_c {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
-	};
-
-	/* External CAN clock - to be overridden by boards that provide it */
-	can_clk: can {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
-	};
-
-	cluster0_opp: opp_table0 {
-		compatible = "operating-points-v2";
-		opp-shared;
-
-		opp-500000000 {
-			opp-hz = /bits/ 64 <500000000>;
-			opp-microvolt = <830000>;
-			clock-latency-ns = <300000>;
-		};
-		opp-1000000000 {
-			opp-hz = /bits/ 64 <1000000000>;
-			opp-microvolt = <830000>;
-			clock-latency-ns = <300000>;
-		};
-		opp-1500000000 {
-			opp-hz = /bits/ 64 <1500000000>;
-			opp-microvolt = <830000>;
-			clock-latency-ns = <300000>;
-			opp-suspend;
-		};
-		opp-1600000000 {
-			opp-hz = /bits/ 64 <1600000000>;
-			opp-microvolt = <900000>;
-			clock-latency-ns = <300000>;
-			turbo-mode;
-		};
-		opp-1700000000 {
-			opp-hz = /bits/ 64 <1700000000>;
-			opp-microvolt = <960000>;
-			clock-latency-ns = <300000>;
-			turbo-mode;
-		};
-	};
-
-	cluster1_opp: opp_table1 {
-		compatible = "operating-points-v2";
-		opp-shared;
-
-		opp-800000000 {
-			opp-hz = /bits/ 64 <800000000>;
-			opp-microvolt = <820000>;
-			clock-latency-ns = <300000>;
-		};
-		opp-1000000000 {
-			opp-hz = /bits/ 64 <1000000000>;
-			opp-microvolt = <820000>;
-			clock-latency-ns = <300000>;
-		};
-		opp-1200000000 {
-			opp-hz = /bits/ 64 <1200000000>;
-			opp-microvolt = <820000>;
-			clock-latency-ns = <300000>;
-		};
-	};
-
 	/* External PCIe clock - can be overridden by the board */
 	pcie_bus_clk: pcie_bus {
 		compatible = "fixed-clock";
@@ -247,18 +247,6 @@
 		clock-frequency = <0>;
 	};
 
-	pmu_a57 {
-		compatible = "arm,cortex-a57-pmu";
-		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
-		                      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
-				      <&gic GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>,
-				      <&gic GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-affinity = <&a57_0>,
-				     <&a57_1>,
-				     <&a57_2>,
-				     <&a57_3>;
-	};
-
 	pmu_a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts-extended = <&gic GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
@@ -271,6 +259,18 @@
 				     <&a53_3>;
 	};
 
+	pmu_a57 {
+		compatible = "arm,cortex-a57-pmu";
+		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&a57_0>,
+				     <&a57_1>,
+				     <&a57_2>,
+				     <&a57_3>;
+	};
+
 	psci {
 		compatible = "arm,psci-1.0", "arm,psci-0.2";
 		method = "smc";
@@ -291,23 +291,6 @@
 		#size-cells = <2>;
 		ranges;
 
-		gic: interrupt-controller@f1010000 {
-			compatible = "arm,gic-400";
-			#interrupt-cells = <3>;
-			#address-cells = <0>;
-			interrupt-controller;
-			reg = <0x0 0xf1010000 0 0x1000>,
-			      <0x0 0xf1020000 0 0x20000>,
-			      <0x0 0xf1040000 0 0x20000>,
-			      <0x0 0xf1060000 0 0x20000>;
-			interrupts = <GIC_PPI 9
-					(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_HIGH)>;
-			clocks = <&cpg CPG_MOD 408>;
-			clock-names = "clk";
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 408>;
-		};
-
 		wdt0: watchdog@e6020000 {
 			compatible = "renesas,r8a7795-wdt", "renesas,rcar-gen3-wdt";
 			reg = <0 0xe6020000 0 0x0c>;
@@ -437,6 +420,11 @@
 			resets = <&cpg 905>;
 		};
 
+		pfc: pin-controller@e6060000 {
+			compatible = "renesas,pfc-r8a7795";
+			reg = <0 0xe6060000 0 0x50c>;
+		};
+
 		cpg: clock-controller@e6150000 {
 			compatible = "renesas,r8a7795-cpg-mssr";
 			reg = <0 0xe6150000 0 0x1000>;
@@ -452,20 +440,25 @@
 			reg = <0 0xe6160000 0 0x0200>;
 		};
 
-		prr: chipid@fff00044 {
-			compatible = "renesas,prr";
-			reg = <0 0xfff00044 0 4>;
-		};
-
 		sysc: system-controller@e6180000 {
 			compatible = "renesas,r8a7795-sysc";
 			reg = <0 0xe6180000 0 0x0400>;
 			#power-domain-cells = <1>;
 		};
 
-		pfc: pin-controller@e6060000 {
-			compatible = "renesas,pfc-r8a7795";
-			reg = <0 0xe6060000 0 0x50c>;
+		tsc: thermal@e6198000 {
+			compatible = "renesas,r8a7795-thermal";
+			reg = <0 0xe6198000 0 0x100>,
+			      <0 0xe61a0000 0 0x100>,
+			      <0 0xe61a8000 0 0x100>;
+			interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 522>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 522>;
+			#thermal-sensor-cells = <1>;
+			status = "okay";
 		};
 
 		intc_ex: interrupt-controller@e61c0000 {
@@ -484,153 +477,326 @@
 			resets = <&cpg 407>;
 		};
 
-		ipmmu_vi0: mmu@febd0000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xfebd0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 14>;
+		i2c0: i2c@e6500000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7795",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe6500000 0 0x40>;
+			interrupts = <GIC_SPI 287 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 931>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-		};
-
-		ipmmu_vi1: mmu@febe0000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xfebe0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 15>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 931>;
+			dmas = <&dmac1 0x91>, <&dmac1 0x90>,
+			       <&dmac2 0x91>, <&dmac2 0x90>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
 			status = "disabled";
 		};
 
-		ipmmu_vp0: mmu@fe990000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xfe990000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 16>;
-			power-domains = <&sysc R8A7795_PD_A3VP>;
-			#iommu-cells = <1>;
+		i2c1: i2c@e6508000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7795",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe6508000 0 0x40>;
+			interrupts = <GIC_SPI 288 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 930>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 930>;
+			dmas = <&dmac1 0x93>, <&dmac1 0x92>,
+			       <&dmac2 0x93>, <&dmac2 0x92>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
 			status = "disabled";
 		};
 
-		ipmmu_vp1: mmu@fe980000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xfe980000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 17>;
-			power-domains = <&sysc R8A7795_PD_A3VP>;
-			#iommu-cells = <1>;
+		i2c2: i2c@e6510000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7795",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe6510000 0 0x40>;
+			interrupts = <GIC_SPI 286 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 929>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 929>;
+			dmas = <&dmac1 0x95>, <&dmac1 0x94>,
+			       <&dmac2 0x95>, <&dmac2 0x94>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
+			status = "disabled";
 		};
 
-		ipmmu_vc0: mmu@fe6b0000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xfe6b0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 12>;
-			power-domains = <&sysc R8A7795_PD_A3VC>;
-			#iommu-cells = <1>;
+		i2c3: i2c@e66d0000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7795",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe66d0000 0 0x40>;
+			interrupts = <GIC_SPI 290 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 928>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 928>;
+			dmas = <&dmac0 0x97>, <&dmac0 0x96>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
 			status = "disabled";
 		};
 
-		ipmmu_vc1: mmu@fe6f0000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xfe6f0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 13>;
-			power-domains = <&sysc R8A7795_PD_A3VC>;
-			#iommu-cells = <1>;
+		i2c4: i2c@e66d8000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7795",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe66d8000 0 0x40>;
+			interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 927>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 927>;
+			dmas = <&dmac0 0x99>, <&dmac0 0x98>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
 			status = "disabled";
 		};
 
-		ipmmu_pv0: mmu@fd800000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xfd800000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 6>;
+		i2c5: i2c@e66e0000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7795",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe66e0000 0 0x40>;
+			interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 919>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 919>;
+			dmas = <&dmac0 0x9b>, <&dmac0 0x9a>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
 			status = "disabled";
 		};
 
-		ipmmu_pv1: mmu@fd950000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xfd950000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 7>;
+		i2c6: i2c@e66e8000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7795",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe66e8000 0 0x40>;
+			interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 918>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 918>;
+			dmas = <&dmac0 0x9d>, <&dmac0 0x9c>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
 			status = "disabled";
 		};
 
-		ipmmu_pv2: mmu@fd960000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xfd960000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 8>;
+		i2c_dvfs: i2c@e60b0000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,iic-r8a7795",
+				     "renesas,rcar-gen3-iic",
+				     "renesas,rmobile-iic";
+			reg = <0 0xe60b0000 0 0x425>;
+			interrupts = <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 926>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 926>;
+			dmas = <&dmac0 0x11>, <&dmac0 0x10>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
-		ipmmu_pv3: mmu@fd970000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xfd970000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 9>;
+		hscif0: serial@e6540000 {
+			compatible = "renesas,hscif-r8a7795",
+				     "renesas,rcar-gen3-hscif",
+				     "renesas,hscif";
+			reg = <0 0xe6540000 0 96>;
+			interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 520>,
+				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac1 0x31>, <&dmac1 0x30>,
+			       <&dmac2 0x31>, <&dmac2 0x30>;
+			dma-names = "tx", "rx", "tx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 520>;
 			status = "disabled";
 		};
 
-		ipmmu_ir: mmu@ff8b0000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xff8b0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 3>;
-			power-domains = <&sysc R8A7795_PD_A3IR>;
-			#iommu-cells = <1>;
+		hscif1: serial@e6550000 {
+			compatible = "renesas,hscif-r8a7795",
+				     "renesas,rcar-gen3-hscif",
+				     "renesas,hscif";
+			reg = <0 0xe6550000 0 96>;
+			interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 519>,
+				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac1 0x33>, <&dmac1 0x32>,
+			       <&dmac2 0x33>, <&dmac2 0x32>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 519>;
 			status = "disabled";
 		};
 
-		ipmmu_hc: mmu@e6570000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xe6570000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 2>;
+		hscif2: serial@e6560000 {
+			compatible = "renesas,hscif-r8a7795",
+				     "renesas,rcar-gen3-hscif",
+				     "renesas,hscif";
+			reg = <0 0xe6560000 0 96>;
+			interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 518>,
+				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac1 0x35>, <&dmac1 0x34>,
+			       <&dmac2 0x35>, <&dmac2 0x34>;
+			dma-names = "tx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 518>;
 			status = "disabled";
 		};
 
-		ipmmu_rt: mmu@ffc80000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xffc80000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 10>;
+		hscif3: serial@e66a0000 {
+			compatible = "renesas,hscif-r8a7795",
+				     "renesas,rcar-gen3-hscif",
+				     "renesas,hscif";
+			reg = <0 0xe66a0000 0 96>;
+			interrupts = <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 517>,
+				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x37>, <&dmac0 0x36>;
+			dma-names = "tx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 517>;
 			status = "disabled";
 		};
 
-		ipmmu_mp0: mmu@ec670000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xec670000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 4>;
+		hscif4: serial@e66b0000 {
+			compatible = "renesas,hscif-r8a7795",
+				     "renesas,rcar-gen3-hscif",
+				     "renesas,hscif";
+			reg = <0 0xe66b0000 0 96>;
+			interrupts = <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 516>,
+				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x39>, <&dmac0 0x38>;
+			dma-names = "tx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 516>;
 			status = "disabled";
 		};
 
-		ipmmu_ds0: mmu@e6740000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xe6740000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 0>;
+		hsusb: usb@e6590000 {
+			compatible = "renesas,usbhs-r8a7795",
+				     "renesas,rcar-gen3-usbhs";
+			reg = <0 0xe6590000 0 0x100>;
+			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 704>;
+			dmas = <&usb_dmac0 0>, <&usb_dmac0 1>,
+			       <&usb_dmac1 0>, <&usb_dmac1 1>;
+			dma-names = "ch0", "ch1", "ch2", "ch3";
+			renesas,buswait = <11>;
+			phys = <&usb2_phy0>;
+			phy-names = "usb";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 704>;
+			status = "disabled";
 		};
 
-		ipmmu_ds1: mmu@e7740000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xe7740000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 1>;
+		hsusb3: usb@e659c000 {
+			compatible = "renesas,usbhs-r8a7795",
+				     "renesas,rcar-gen3-usbhs";
+			reg = <0 0xe659c000 0 0x100>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 705>;
+			dmas = <&usb_dmac2 0>, <&usb_dmac2 1>,
+			       <&usb_dmac3 0>, <&usb_dmac3 1>;
+			dma-names = "ch0", "ch1", "ch2", "ch3";
+			renesas,buswait = <11>;
+			phys = <&usb2_phy3>;
+			phy-names = "usb";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 705>;
+			status = "disabled";
 		};
 
-		ipmmu_mm: mmu@e67b0000 {
-			compatible = "renesas,ipmmu-r8a7795";
-			reg = <0 0xe67b0000 0 0x1000>;
-			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+		usb_dmac0: dma-controller@e65a0000 {
+			compatible = "renesas,r8a7795-usb-dmac",
+				     "renesas,usb-dmac";
+			reg = <0 0xe65a0000 0 0x100>;
+			interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1";
+			clocks = <&cpg CPG_MOD 330>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 330>;
+			#dma-cells = <1>;
+			dma-channels = <2>;
+		};
+
+		usb_dmac1: dma-controller@e65b0000 {
+			compatible = "renesas,r8a7795-usb-dmac",
+				     "renesas,usb-dmac";
+			reg = <0 0xe65b0000 0 0x100>;
+			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1";
+			clocks = <&cpg CPG_MOD 331>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 331>;
+			#dma-cells = <1>;
+			dma-channels = <2>;
+		};
+
+		usb_dmac2: dma-controller@e6460000 {
+			compatible = "renesas,r8a7795-usb-dmac",
+				     "renesas,usb-dmac";
+			reg = <0 0xe6460000 0 0x100>;
+			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1";
+			clocks = <&cpg CPG_MOD 326>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 326>;
+			#dma-cells = <1>;
+			dma-channels = <2>;
+		};
+
+		usb_dmac3: dma-controller@e6470000 {
+			compatible = "renesas,r8a7795-usb-dmac",
+				     "renesas,usb-dmac";
+			reg = <0 0xe6470000 0 0x100>;
+			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1";
+			clocks = <&cpg CPG_MOD 329>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 329>;
+			#dma-cells = <1>;
+			dma-channels = <2>;
+		};
+
+		usb3_phy0: usb-phy@e65ee000 {
+			compatible = "renesas,r8a7795-usb3-phy",
+				     "renesas,rcar-gen3-usb3-phy";
+			reg = <0 0xe65ee000 0 0x90>;
+			clocks = <&cpg CPG_MOD 328>, <&usb3s0_clk>,
+				 <&usb_extal_clk>;
+			clock-names = "usb3-if", "usb3s_clk", "usb_extal";
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 328>;
+			#phy-cells = <0>;
+			status = "disabled";
 		};
 
 		dmac0: dma-controller@e6700000 {
@@ -759,155 +925,208 @@
 			       <&ipmmu_ds1 30>, <&ipmmu_ds1 31>;
 		};
 
-		audma0: dma-controller@ec700000 {
-			compatible = "renesas,dmac-r8a7795",
-				     "renesas,rcar-dmac";
-			reg = <0 0xec700000 0 0x10000>;
-			interrupts = <GIC_SPI 350 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 320 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 321 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 322 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 323 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 324 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 332 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 333 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 334 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 335 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "error",
-					"ch0", "ch1", "ch2", "ch3",
-					"ch4", "ch5", "ch6", "ch7",
-					"ch8", "ch9", "ch10", "ch11",
-					"ch12", "ch13", "ch14", "ch15";
-			clocks = <&cpg CPG_MOD 502>;
-			clock-names = "fck";
+		ipmmu_ds0: mmu@e6740000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xe6740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 0>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 502>;
-			#dma-cells = <1>;
-			dma-channels = <16>;
-			iommus = <&ipmmu_mp0 0>, <&ipmmu_mp0 1>,
-			       <&ipmmu_mp0 2>, <&ipmmu_mp0 3>,
-			       <&ipmmu_mp0 4>, <&ipmmu_mp0 5>,
-			       <&ipmmu_mp0 6>, <&ipmmu_mp0 7>,
-			       <&ipmmu_mp0 8>, <&ipmmu_mp0 9>,
-			       <&ipmmu_mp0 10>, <&ipmmu_mp0 11>,
-			       <&ipmmu_mp0 12>, <&ipmmu_mp0 13>,
-			       <&ipmmu_mp0 14>, <&ipmmu_mp0 15>;
+			#iommu-cells = <1>;
 		};
 
-		audma1: dma-controller@ec720000 {
-			compatible = "renesas,dmac-r8a7795",
-				     "renesas,rcar-dmac";
-			reg = <0 0xec720000 0 0x10000>;
-			interrupts = <GIC_SPI 351 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 336 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 338 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 339 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 340 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 341 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 342 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 343 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 344 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 345 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 346 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 347 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 348 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 349 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 382 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 383 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "error",
-					"ch0", "ch1", "ch2", "ch3",
-					"ch4", "ch5", "ch6", "ch7",
-					"ch8", "ch9", "ch10", "ch11",
-					"ch12", "ch13", "ch14", "ch15";
-			clocks = <&cpg CPG_MOD 501>;
-			clock-names = "fck";
+		ipmmu_ds1: mmu@e7740000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xe7740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 1>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 501>;
-			#dma-cells = <1>;
-			dma-channels = <16>;
-			iommus = <&ipmmu_mp0 16>, <&ipmmu_mp0 17>,
-			       <&ipmmu_mp0 18>, <&ipmmu_mp0 19>,
-			       <&ipmmu_mp0 20>, <&ipmmu_mp0 21>,
-			       <&ipmmu_mp0 22>, <&ipmmu_mp0 23>,
-			       <&ipmmu_mp0 24>, <&ipmmu_mp0 25>,
-			       <&ipmmu_mp0 26>, <&ipmmu_mp0 27>,
-			       <&ipmmu_mp0 28>, <&ipmmu_mp0 29>,
-			       <&ipmmu_mp0 30>, <&ipmmu_mp0 31>;
+			#iommu-cells = <1>;
 		};
 
-		avb: ethernet@e6800000 {
-			compatible = "renesas,etheravb-r8a7795",
-				     "renesas,etheravb-rcar-gen3";
-			reg = <0 0xe6800000 0 0x800>, <0 0xe6a00000 0 0x10000>;
-			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "ch0", "ch1", "ch2", "ch3",
-					  "ch4", "ch5", "ch6", "ch7",
-					  "ch8", "ch9", "ch10", "ch11",
-					  "ch12", "ch13", "ch14", "ch15",
-					  "ch16", "ch17", "ch18", "ch19",
-					  "ch20", "ch21", "ch22", "ch23",
-					  "ch24";
-			clocks = <&cpg CPG_MOD 812>;
+		ipmmu_hc: mmu@e6570000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xe6570000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 2>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 812>;
-			phy-mode = "rgmii";
-			iommus = <&ipmmu_ds0 16>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			status = "disabled";
+			#iommu-cells = <1>;
 		};
 
-		can0: can@e6c30000 {
-			compatible = "renesas,can-r8a7795",
-				     "renesas,rcar-gen3-can";
-			reg = <0 0xe6c30000 0 0x1000>;
-			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 916>,
-			       <&cpg CPG_CORE R8A7795_CLK_CANFD>,
-			       <&can_clk>;
-			clock-names = "clkp1", "clkp2", "can_clk";
-			assigned-clocks = <&cpg CPG_CORE R8A7795_CLK_CANFD>;
-			assigned-clock-rates = <40000000>;
+		ipmmu_ir: mmu@ff8b0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xff8b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 3>;
+			power-domains = <&sysc R8A7795_PD_A3IR>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_mm: mmu@e67b0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xe67b0000 0 0x1000>;
+			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 916>;
-			status = "disabled";
+			#iommu-cells = <1>;
 		};
 
-		can1: can@e6c38000 {
-			compatible = "renesas,can-r8a7795",
-				     "renesas,rcar-gen3-can";
+		ipmmu_mp0: mmu@ec670000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xec670000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 4>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_pv0: mmu@fd800000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfd800000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 6>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_pv1: mmu@fd950000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfd950000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 7>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_pv2: mmu@fd960000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfd960000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 8>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_pv3: mmu@fd970000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfd970000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 9>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_rt: mmu@ffc80000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xffc80000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 10>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vc0: mmu@fe6b0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfe6b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 12>;
+			power-domains = <&sysc R8A7795_PD_A3VC>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vc1: mmu@fe6f0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfe6f0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 13>;
+			power-domains = <&sysc R8A7795_PD_A3VC>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vi0: mmu@febd0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfebd0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 14>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vi1: mmu@febe0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfebe0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 15>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vp0: mmu@fe990000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfe990000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 16>;
+			power-domains = <&sysc R8A7795_PD_A3VP>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vp1: mmu@fe980000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfe980000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 17>;
+			power-domains = <&sysc R8A7795_PD_A3VP>;
+			#iommu-cells = <1>;
+		};
+
+		avb: ethernet@e6800000 {
+			compatible = "renesas,etheravb-r8a7795",
+				     "renesas,etheravb-rcar-gen3";
+			reg = <0 0xe6800000 0 0x800>, <0 0xe6a00000 0 0x10000>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14", "ch15",
+					  "ch16", "ch17", "ch18", "ch19",
+					  "ch20", "ch21", "ch22", "ch23",
+					  "ch24";
+			clocks = <&cpg CPG_MOD 812>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 812>;
+			phy-mode = "rgmii";
+			iommus = <&ipmmu_ds0 16>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		can0: can@e6c30000 {
+			compatible = "renesas,can-r8a7795",
+				     "renesas,rcar-gen3-can";
+			reg = <0 0xe6c30000 0 0x1000>;
+			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 916>,
+			       <&cpg CPG_CORE R8A7795_CLK_CANFD>,
+			       <&can_clk>;
+			clock-names = "clkp1", "clkp2", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A7795_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 916>;
+			status = "disabled";
+		};
+
+		can1: can@e6c38000 {
+			compatible = "renesas,can-r8a7795",
+				     "renesas,rcar-gen3-can";
 			reg = <0 0xe6c38000 0 0x1000>;
 			interrupts = <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 915>,
@@ -946,211 +1165,173 @@
 			};
 		};
 
-		drif00: rif@e6f40000 {
-			compatible = "renesas,r8a7795-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f40000 0 0x64>;
-			interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 515>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x20>, <&dmac2 0x20>;
-			dma-names = "rx", "rx";
+		pwm0: pwm@e6e30000 {
+			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
+			reg = <0 0xe6e30000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 515>;
-			renesas,bonding = <&drif01>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
 			status = "disabled";
 		};
 
-		drif01: rif@e6f50000 {
-			compatible = "renesas,r8a7795-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f50000 0 0x64>;
-			interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 514>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x22>, <&dmac2 0x22>;
-			dma-names = "rx", "rx";
+		pwm1: pwm@e6e31000 {
+			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
+			reg = <0 0xe6e31000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 514>;
-			renesas,bonding = <&drif00>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
 			status = "disabled";
 		};
 
-		drif10: rif@e6f60000 {
-			compatible = "renesas,r8a7795-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f60000 0 0x64>;
-			interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 513>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x24>, <&dmac2 0x24>;
-			dma-names = "rx", "rx";
+		pwm2: pwm@e6e32000 {
+			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
+			reg = <0 0xe6e32000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 513>;
-			renesas,bonding = <&drif11>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
 			status = "disabled";
 		};
 
-		drif11: rif@e6f70000 {
-			compatible = "renesas,r8a7795-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f70000 0 0x64>;
-			interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 512>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x26>, <&dmac2 0x26>;
-			dma-names = "rx", "rx";
+		pwm3: pwm@e6e33000 {
+			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
+			reg = <0 0xe6e33000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 512>;
-			renesas,bonding = <&drif10>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
 			status = "disabled";
 		};
 
-		drif20: rif@e6f80000 {
-			compatible = "renesas,r8a7795-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f80000 0 0x64>;
-			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 511>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x28>, <&dmac2 0x28>;
-			dma-names = "rx", "rx";
+		pwm4: pwm@e6e34000 {
+			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
+			reg = <0 0xe6e34000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 511>;
-			renesas,bonding = <&drif21>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
 			status = "disabled";
 		};
 
-		drif21: rif@e6f90000 {
-			compatible = "renesas,r8a7795-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f90000 0 0x64>;
-			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 510>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x2a>, <&dmac2 0x2a>;
-			dma-names = "rx", "rx";
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 510>;
-			renesas,bonding = <&drif20>;
+		pwm5: pwm@e6e35000 {
+			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
+			reg = <0 0xe6e35000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
 			status = "disabled";
 		};
 
-		drif30: rif@e6fa0000 {
-			compatible = "renesas,r8a7795-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6fa0000 0 0x64>;
-			interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 509>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x2c>, <&dmac2 0x2c>;
-			dma-names = "rx", "rx";
+		pwm6: pwm@e6e36000 {
+			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
+			reg = <0 0xe6e36000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 509>;
-			renesas,bonding = <&drif31>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
 			status = "disabled";
 		};
 
-		drif31: rif@e6fb0000 {
-			compatible = "renesas,r8a7795-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6fb0000 0 0x64>;
-			interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 508>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x2e>, <&dmac2 0x2e>;
-			dma-names = "rx", "rx";
+		scif0: serial@e6e60000 {
+			compatible = "renesas,scif-r8a7795",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6e60000 0 64>;
+			interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 207>,
+				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac1 0x51>, <&dmac1 0x50>,
+			       <&dmac2 0x51>, <&dmac2 0x50>;
+			dma-names = "tx", "rx", "tx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 508>;
-			renesas,bonding = <&drif30>;
+			resets = <&cpg 207>;
 			status = "disabled";
 		};
 
-		hscif0: serial@e6540000 {
-			compatible = "renesas,hscif-r8a7795",
-				     "renesas,rcar-gen3-hscif",
-				     "renesas,hscif";
-			reg = <0 0xe6540000 0 96>;
-			interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 520>,
+		scif1: serial@e6e68000 {
+			compatible = "renesas,scif-r8a7795",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6e68000 0 64>;
+			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 206>,
 				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x31>, <&dmac1 0x30>,
-			       <&dmac2 0x31>, <&dmac2 0x30>;
+			dmas = <&dmac1 0x53>, <&dmac1 0x52>,
+			       <&dmac2 0x53>, <&dmac2 0x52>;
 			dma-names = "tx", "rx", "tx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 520>;
+			resets = <&cpg 206>;
 			status = "disabled";
 		};
 
-		hscif1: serial@e6550000 {
-			compatible = "renesas,hscif-r8a7795",
-				     "renesas,rcar-gen3-hscif",
-				     "renesas,hscif";
-			reg = <0 0xe6550000 0 96>;
-			interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 519>,
+		scif2: serial@e6e88000 {
+			compatible = "renesas,scif-r8a7795",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6e88000 0 64>;
+			interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 310>,
 				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x33>, <&dmac1 0x32>,
-			       <&dmac2 0x33>, <&dmac2 0x32>;
+			dmas = <&dmac1 0x13>, <&dmac1 0x12>,
+			       <&dmac2 0x13>, <&dmac2 0x12>;
 			dma-names = "tx", "rx", "tx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 519>;
+			resets = <&cpg 310>;
 			status = "disabled";
 		};
 
-		hscif2: serial@e6560000 {
-			compatible = "renesas,hscif-r8a7795",
-				     "renesas,rcar-gen3-hscif",
-				     "renesas,hscif";
-			reg = <0 0xe6560000 0 96>;
-			interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 518>,
+		scif3: serial@e6c50000 {
+			compatible = "renesas,scif-r8a7795",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6c50000 0 64>;
+			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 204>,
 				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x35>, <&dmac1 0x34>,
-			       <&dmac2 0x35>, <&dmac2 0x34>;
-			dma-names = "tx", "rx", "tx", "rx";
+			dmas = <&dmac0 0x57>, <&dmac0 0x56>;
+			dma-names = "tx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 518>;
+			resets = <&cpg 204>;
 			status = "disabled";
 		};
 
-		hscif3: serial@e66a0000 {
-			compatible = "renesas,hscif-r8a7795",
-				     "renesas,rcar-gen3-hscif",
-				     "renesas,hscif";
-			reg = <0 0xe66a0000 0 96>;
-			interrupts = <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 517>,
+		scif4: serial@e6c40000 {
+			compatible = "renesas,scif-r8a7795",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6c40000 0 64>;
+			interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 203>,
 				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac0 0x37>, <&dmac0 0x36>;
+			dmas = <&dmac0 0x59>, <&dmac0 0x58>;
 			dma-names = "tx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 517>;
+			resets = <&cpg 203>;
 			status = "disabled";
 		};
 
-		hscif4: serial@e66b0000 {
-			compatible = "renesas,hscif-r8a7795",
-				     "renesas,rcar-gen3-hscif",
-				     "renesas,hscif";
-			reg = <0 0xe66b0000 0 96>;
-			interrupts = <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 516>,
+		scif5: serial@e6f30000 {
+			compatible = "renesas,scif-r8a7795",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6f30000 0 64>;
+			interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 202>,
 				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac0 0x39>, <&dmac0 0x38>;
-			dma-names = "tx", "rx";
+			dmas = <&dmac1 0x5b>, <&dmac1 0x5a>,
+			       <&dmac2 0x5b>, <&dmac2 0x5a>;
+			dma-names = "tx", "rx", "tx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 516>;
+			resets = <&cpg 202>;
 			status = "disabled";
 		};
 
@@ -1216,304 +1397,379 @@
 			status = "disabled";
 		};
 
-		scif0: serial@e6e60000 {
-			compatible = "renesas,scif-r8a7795",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6e60000 0 64>;
-			interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 207>,
-				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x51>, <&dmac1 0x50>,
-			       <&dmac2 0x51>, <&dmac2 0x50>;
-			dma-names = "tx", "rx", "tx", "rx";
+		vin0: video@e6ef0000 {
+			compatible = "renesas,vin-r8a7795";
+			reg = <0 0xe6ef0000 0 0x1000>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 811>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 207>;
+			resets = <&cpg 811>;
+			renesas,id = <0>;
 			status = "disabled";
-		};
 
-		scif1: serial@e6e68000 {
-			compatible = "renesas,scif-r8a7795",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6e68000 0 64>;
-			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 206>,
-				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x53>, <&dmac1 0x52>,
-			       <&dmac2 0x53>, <&dmac2 0x52>;
-			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 206>;
-			status = "disabled";
-		};
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
 
-		scif2: serial@e6e88000 {
-			compatible = "renesas,scif-r8a7795",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6e88000 0 64>;
-			interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 310>,
-				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x13>, <&dmac1 0x12>,
-			       <&dmac2 0x13>, <&dmac2 0x12>;
-			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 310>;
-			status = "disabled";
-		};
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
 
-		scif3: serial@e6c50000 {
-			compatible = "renesas,scif-r8a7795",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6c50000 0 64>;
-			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 204>,
-				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac0 0x57>, <&dmac0 0x56>;
-			dma-names = "tx", "rx";
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 204>;
-			status = "disabled";
-		};
+					reg = <1>;
 
-		scif4: serial@e6c40000 {
-			compatible = "renesas,scif-r8a7795",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6c40000 0 64>;
-			interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 203>,
-				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac0 0x59>, <&dmac0 0x58>;
-			dma-names = "tx", "rx";
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 203>;
-			status = "disabled";
+					vin0csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin0>;
+					};
+					vin0csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin0>;
+					};
+				};
+			};
 		};
 
-		scif5: serial@e6f30000 {
-			compatible = "renesas,scif-r8a7795",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6f30000 0 64>;
-			interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 202>,
-				 <&cpg CPG_CORE R8A7795_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x5b>, <&dmac1 0x5a>,
-			       <&dmac2 0x5b>, <&dmac2 0x5a>;
-			dma-names = "tx", "rx", "tx", "rx";
+		vin1: video@e6ef1000 {
+			compatible = "renesas,vin-r8a7795";
+			reg = <0 0xe6ef1000 0 0x1000>;
+			interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 810>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 202>;
+			resets = <&cpg 810>;
+			renesas,id = <1>;
 			status = "disabled";
-		};
 
-		i2c_dvfs: i2c@e60b0000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,iic-r8a7795",
-				     "renesas,rcar-gen3-iic",
-				     "renesas,rmobile-iic";
-			reg = <0 0xe60b0000 0 0x425>;
-			interrupts = <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 926>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 926>;
-			dmas = <&dmac0 0x11>, <&dmac0 0x10>;
-			dma-names = "tx", "rx";
-			status = "disabled";
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin1csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin1>;
+					};
+					vin1csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin1>;
+					};
+				};
+			};
 		};
 
-		i2c0: i2c@e6500000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a7795",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe6500000 0 0x40>;
-			interrupts = <GIC_SPI 287 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 931>;
+		vin2: video@e6ef2000 {
+			compatible = "renesas,vin-r8a7795";
+			reg = <0 0xe6ef2000 0 0x1000>;
+			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 809>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 931>;
-			dmas = <&dmac1 0x91>, <&dmac1 0x90>,
-			       <&dmac2 0x91>, <&dmac2 0x90>;
-			dma-names = "tx", "rx", "tx", "rx";
-			i2c-scl-internal-delay-ns = <110>;
+			resets = <&cpg 809>;
+			renesas,id = <2>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin2csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin2>;
+					};
+					vin2csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin2>;
+					};
+				};
+			};
 		};
 
-		i2c1: i2c@e6508000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a7795",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe6508000 0 0x40>;
-			interrupts = <GIC_SPI 288 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 930>;
+		vin3: video@e6ef3000 {
+			compatible = "renesas,vin-r8a7795";
+			reg = <0 0xe6ef3000 0 0x1000>;
+			interrupts = <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 808>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 930>;
-			dmas = <&dmac1 0x93>, <&dmac1 0x92>,
-			       <&dmac2 0x93>, <&dmac2 0x92>;
-			dma-names = "tx", "rx", "tx", "rx";
-			i2c-scl-internal-delay-ns = <6>;
+			resets = <&cpg 808>;
+			renesas,id = <3>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin3csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin3>;
+					};
+					vin3csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin3>;
+					};
+				};
+			};
 		};
 
-		i2c2: i2c@e6510000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a7795",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe6510000 0 0x40>;
-			interrupts = <GIC_SPI 286 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 929>;
+		vin4: video@e6ef4000 {
+			compatible = "renesas,vin-r8a7795";
+			reg = <0 0xe6ef4000 0 0x1000>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 807>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 929>;
-			dmas = <&dmac1 0x95>, <&dmac1 0x94>,
-			       <&dmac2 0x95>, <&dmac2 0x94>;
-			dma-names = "tx", "rx", "tx", "rx";
-			i2c-scl-internal-delay-ns = <6>;
+			resets = <&cpg 807>;
+			renesas,id = <4>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin4csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin4>;
+					};
+					vin4csi41: endpoint@3 {
+						reg = <3>;
+						remote-endpoint= <&csi41vin4>;
+					};
+				};
+			};
 		};
 
-		i2c3: i2c@e66d0000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a7795",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe66d0000 0 0x40>;
-			interrupts = <GIC_SPI 290 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 928>;
+		vin5: video@e6ef5000 {
+			compatible = "renesas,vin-r8a7795";
+			reg = <0 0xe6ef5000 0 0x1000>;
+			interrupts = <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 806>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 928>;
-			dmas = <&dmac0 0x97>, <&dmac0 0x96>;
-			dma-names = "tx", "rx";
-			i2c-scl-internal-delay-ns = <110>;
+			resets = <&cpg 806>;
+			renesas,id = <5>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin5csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin5>;
+					};
+					vin5csi41: endpoint@3 {
+						reg = <3>;
+						remote-endpoint= <&csi41vin5>;
+					};
+				};
+			};
 		};
 
-		i2c4: i2c@e66d8000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a7795",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe66d8000 0 0x40>;
-			interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 927>;
+		vin6: video@e6ef6000 {
+			compatible = "renesas,vin-r8a7795";
+			reg = <0 0xe6ef6000 0 0x1000>;
+			interrupts = <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 805>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 927>;
-			dmas = <&dmac0 0x99>, <&dmac0 0x98>;
-			dma-names = "tx", "rx";
-			i2c-scl-internal-delay-ns = <110>;
+			resets = <&cpg 805>;
+			renesas,id = <6>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin6csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin6>;
+					};
+					vin6csi41: endpoint@3 {
+						reg = <3>;
+						remote-endpoint= <&csi41vin6>;
+					};
+				};
+			};
 		};
 
-		i2c5: i2c@e66e0000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a7795",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe66e0000 0 0x40>;
-			interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 919>;
+		vin7: video@e6ef7000 {
+			compatible = "renesas,vin-r8a7795";
+			reg = <0 0xe6ef7000 0 0x1000>;
+			interrupts = <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 804>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 919>;
-			dmas = <&dmac0 0x9b>, <&dmac0 0x9a>;
-			dma-names = "tx", "rx";
-			i2c-scl-internal-delay-ns = <110>;
+			resets = <&cpg 804>;
+			renesas,id = <7>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin7csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin7>;
+					};
+					vin7csi41: endpoint@3 {
+						reg = <3>;
+						remote-endpoint= <&csi41vin7>;
+					};
+				};
+			};
 		};
 
-		i2c6: i2c@e66e8000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a7795",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe66e8000 0 0x40>;
-			interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 918>;
+		drif00: rif@e6f40000 {
+			compatible = "renesas,r8a7795-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f40000 0 0x64>;
+			interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 515>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x20>, <&dmac2 0x20>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 918>;
-			dmas = <&dmac0 0x9d>, <&dmac0 0x9c>;
-			dma-names = "tx", "rx";
-			i2c-scl-internal-delay-ns = <6>;
+			resets = <&cpg 515>;
+			renesas,bonding = <&drif01>;
 			status = "disabled";
 		};
 
-		pwm0: pwm@e6e30000 {
-			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
-			reg = <0 0xe6e30000 0 0x8>;
-			clocks = <&cpg CPG_MOD 523>;
+		drif01: rif@e6f50000 {
+			compatible = "renesas,r8a7795-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f50000 0 0x64>;
+			interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 514>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x22>, <&dmac2 0x22>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 523>;
-			#pwm-cells = <2>;
+			resets = <&cpg 514>;
+			renesas,bonding = <&drif00>;
 			status = "disabled";
 		};
 
-		pwm1: pwm@e6e31000 {
-			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
-			reg = <0 0xe6e31000 0 0x8>;
-			clocks = <&cpg CPG_MOD 523>;
+		drif10: rif@e6f60000 {
+			compatible = "renesas,r8a7795-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f60000 0 0x64>;
+			interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 513>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x24>, <&dmac2 0x24>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 523>;
-			#pwm-cells = <2>;
+			resets = <&cpg 513>;
+			renesas,bonding = <&drif11>;
 			status = "disabled";
 		};
 
-		pwm2: pwm@e6e32000 {
-			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
-			reg = <0 0xe6e32000 0 0x8>;
-			clocks = <&cpg CPG_MOD 523>;
+		drif11: rif@e6f70000 {
+			compatible = "renesas,r8a7795-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f70000 0 0x64>;
+			interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 512>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x26>, <&dmac2 0x26>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 523>;
-			#pwm-cells = <2>;
+			resets = <&cpg 512>;
+			renesas,bonding = <&drif10>;
 			status = "disabled";
 		};
 
-		pwm3: pwm@e6e33000 {
-			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
-			reg = <0 0xe6e33000 0 0x8>;
-			clocks = <&cpg CPG_MOD 523>;
+		drif20: rif@e6f80000 {
+			compatible = "renesas,r8a7795-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f80000 0 0x64>;
+			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 511>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x28>, <&dmac2 0x28>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 523>;
-			#pwm-cells = <2>;
+			resets = <&cpg 511>;
+			renesas,bonding = <&drif21>;
 			status = "disabled";
 		};
 
-		pwm4: pwm@e6e34000 {
-			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
-			reg = <0 0xe6e34000 0 0x8>;
-			clocks = <&cpg CPG_MOD 523>;
+		drif21: rif@e6f90000 {
+			compatible = "renesas,r8a7795-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f90000 0 0x64>;
+			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 510>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x2a>, <&dmac2 0x2a>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 523>;
-			#pwm-cells = <2>;
+			resets = <&cpg 510>;
+			renesas,bonding = <&drif20>;
 			status = "disabled";
 		};
 
-		pwm5: pwm@e6e35000 {
-			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
-			reg = <0 0xe6e35000 0 0x8>;
-			clocks = <&cpg CPG_MOD 523>;
+		drif30: rif@e6fa0000 {
+			compatible = "renesas,r8a7795-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6fa0000 0 0x64>;
+			interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 509>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x2c>, <&dmac2 0x2c>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 523>;
-			#pwm-cells = <2>;
+			resets = <&cpg 509>;
+			renesas,bonding = <&drif31>;
 			status = "disabled";
 		};
 
-		pwm6: pwm@e6e36000 {
-			compatible = "renesas,pwm-r8a7795", "renesas,pwm-rcar";
-			reg = <0 0xe6e36000 0 0x8>;
-			clocks = <&cpg CPG_MOD 523>;
+		drif31: rif@e6fb0000 {
+			compatible = "renesas,r8a7795-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6fb0000 0 0x64>;
+			interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 508>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x2e>, <&dmac2 0x2e>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 523>;
-			#pwm-cells = <2>;
+			resets = <&cpg 508>;
+			renesas,bonding = <&drif30>;
 			status = "disabled";
 		};
 
@@ -1711,201 +1967,172 @@
 					dma-names = "rx", "tx", "rxu", "txu";
 				};
 			};
-		};
-
-		sata: sata@ee300000 {
-			compatible = "renesas,sata-r8a7795",
-				     "renesas,rcar-gen3-sata";
-			reg = <0 0xee300000 0 0x200000>;
-			interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 815>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 815>;
-			status = "disabled";
-			iommus = <&ipmmu_hc 2>;
-		};
-
-		usb3_phy0: usb-phy@e65ee000 {
-			compatible = "renesas,r8a7795-usb3-phy",
-				     "renesas,rcar-gen3-usb3-phy";
-			reg = <0 0xe65ee000 0 0x90>;
-			clocks = <&cpg CPG_MOD 328>, <&usb3s0_clk>,
-				 <&usb_extal_clk>;
-			clock-names = "usb3-if", "usb3s_clk", "usb_extal";
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 328>;
-			#phy-cells = <0>;
-			status = "disabled";
-		};
-
-		xhci0: usb@ee000000 {
-			compatible = "renesas,xhci-r8a7795", "renesas,rcar-gen3-xhci";
-			reg = <0 0xee000000 0 0xc00>;
-			interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 328>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 328>;
-			status = "disabled";
-		};
-
-		usb3_peri0: usb@ee020000 {
-			compatible = "renesas,r8a7795-usb3-peri",
-				     "renesas,rcar-gen3-usb3-peri";
-			reg = <0 0xee020000 0 0x400>;
-			interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 328>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 328>;
-			status = "disabled";
-		};
-
-		usb_dmac0: dma-controller@e65a0000 {
-			compatible = "renesas,r8a7795-usb-dmac",
-				     "renesas,usb-dmac";
-			reg = <0 0xe65a0000 0 0x100>;
-			interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "ch0", "ch1";
-			clocks = <&cpg CPG_MOD 330>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 330>;
-			#dma-cells = <1>;
-			dma-channels = <2>;
-		};
 
-		usb_dmac1: dma-controller@e65b0000 {
-			compatible = "renesas,r8a7795-usb-dmac",
-				     "renesas,usb-dmac";
-			reg = <0 0xe65b0000 0 0x100>;
-			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "ch0", "ch1";
-			clocks = <&cpg CPG_MOD 331>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 331>;
-			#dma-cells = <1>;
-			dma-channels = <2>;
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				port@0 {
+					reg = <0>;
+				};
+				port@1 {
+					reg = <1>;
+				};
+				port@2 {
+					reg = <2>;
+				};
+			};
 		};
 
-		usb_dmac2: dma-controller@e6460000 {
-			compatible = "renesas,r8a7795-usb-dmac",
-				     "renesas,usb-dmac";
-			reg = <0 0xe6460000 0 0x100>;
-			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "ch0", "ch1";
-			clocks = <&cpg CPG_MOD 326>;
+		audma0: dma-controller@ec700000 {
+			compatible = "renesas,dmac-r8a7795",
+				     "renesas,rcar-dmac";
+			reg = <0 0xec700000 0 0x10000>;
+			interrupts = <GIC_SPI 350 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 320 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 321 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 322 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 323 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 324 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 332 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 333 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 334 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 335 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 502>;
+			clock-names = "fck";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 326>;
+			resets = <&cpg 502>;
 			#dma-cells = <1>;
-			dma-channels = <2>;
+			dma-channels = <16>;
+			iommus = <&ipmmu_mp0 0>, <&ipmmu_mp0 1>,
+			       <&ipmmu_mp0 2>, <&ipmmu_mp0 3>,
+			       <&ipmmu_mp0 4>, <&ipmmu_mp0 5>,
+			       <&ipmmu_mp0 6>, <&ipmmu_mp0 7>,
+			       <&ipmmu_mp0 8>, <&ipmmu_mp0 9>,
+			       <&ipmmu_mp0 10>, <&ipmmu_mp0 11>,
+			       <&ipmmu_mp0 12>, <&ipmmu_mp0 13>,
+			       <&ipmmu_mp0 14>, <&ipmmu_mp0 15>;
 		};
 
-		usb_dmac3: dma-controller@e6470000 {
-			compatible = "renesas,r8a7795-usb-dmac",
-				     "renesas,usb-dmac";
-			reg = <0 0xe6470000 0 0x100>;
-			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "ch0", "ch1";
-			clocks = <&cpg CPG_MOD 329>;
+		audma1: dma-controller@ec720000 {
+			compatible = "renesas,dmac-r8a7795",
+				     "renesas,rcar-dmac";
+			reg = <0 0xec720000 0 0x10000>;
+			interrupts = <GIC_SPI 351 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 336 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 338 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 339 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 340 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 341 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 342 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 343 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 344 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 345 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 346 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 347 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 348 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 349 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 382 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 383 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 501>;
+			clock-names = "fck";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 329>;
+			resets = <&cpg 501>;
 			#dma-cells = <1>;
-			dma-channels = <2>;
-		};
-
-		sdhi0: sd@ee100000 {
-			compatible = "renesas,sdhi-r8a7795",
-				     "renesas,rcar-gen3-sdhi";
-			reg = <0 0xee100000 0 0x2000>;
-			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 314>;
-			max-frequency = <200000000>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 314>;
-			status = "disabled";
-		};
-
-		sdhi1: sd@ee120000 {
-			compatible = "renesas,sdhi-r8a7795",
-				     "renesas,rcar-gen3-sdhi";
-			reg = <0 0xee120000 0 0x2000>;
-			interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 313>;
-			max-frequency = <200000000>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 313>;
-			status = "disabled";
+			dma-channels = <16>;
+			iommus = <&ipmmu_mp0 16>, <&ipmmu_mp0 17>,
+			       <&ipmmu_mp0 18>, <&ipmmu_mp0 19>,
+			       <&ipmmu_mp0 20>, <&ipmmu_mp0 21>,
+			       <&ipmmu_mp0 22>, <&ipmmu_mp0 23>,
+			       <&ipmmu_mp0 24>, <&ipmmu_mp0 25>,
+			       <&ipmmu_mp0 26>, <&ipmmu_mp0 27>,
+			       <&ipmmu_mp0 28>, <&ipmmu_mp0 29>,
+			       <&ipmmu_mp0 30>, <&ipmmu_mp0 31>;
 		};
 
-		sdhi2: sd@ee140000 {
-			compatible = "renesas,sdhi-r8a7795",
-				     "renesas,rcar-gen3-sdhi";
-			reg = <0 0xee140000 0 0x2000>;
-			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 312>;
-			max-frequency = <200000000>;
+		xhci0: usb@ee000000 {
+			compatible = "renesas,xhci-r8a7795", "renesas,rcar-gen3-xhci";
+			reg = <0 0xee000000 0 0xc00>;
+			interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 328>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 312>;
+			resets = <&cpg 328>;
 			status = "disabled";
 		};
 
-		sdhi3: sd@ee160000 {
-			compatible = "renesas,sdhi-r8a7795",
-				     "renesas,rcar-gen3-sdhi";
-			reg = <0 0xee160000 0 0x2000>;
-			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 311>;
-			max-frequency = <200000000>;
+		usb3_peri0: usb@ee020000 {
+			compatible = "renesas,r8a7795-usb3-peri",
+				     "renesas,rcar-gen3-usb3-peri";
+			reg = <0 0xee020000 0 0x400>;
+			interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 328>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 311>;
+			resets = <&cpg 328>;
 			status = "disabled";
 		};
 
-		usb2_phy0: usb-phy@ee080200 {
-			compatible = "renesas,usb2-phy-r8a7795",
-				     "renesas,rcar-gen3-usb2-phy";
-			reg = <0 0xee080200 0 0x700>;
+		ohci0: usb@ee080000 {
+			compatible = "generic-ohci";
+			reg = <0 0xee080000 0 0x100>;
 			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 703>;
+			phys = <&usb2_phy0>;
+			phy-names = "usb";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 703>;
-			#phy-cells = <0>;
 			status = "disabled";
 		};
 
-		usb2_phy1: usb-phy@ee0a0200 {
-			compatible = "renesas,usb2-phy-r8a7795",
-				     "renesas,rcar-gen3-usb2-phy";
-			reg = <0 0xee0a0200 0 0x700>;
+		ohci1: usb@ee0a0000 {
+			compatible = "generic-ohci";
+			reg = <0 0xee0a0000 0 0x100>;
+			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 702>;
+			phys = <&usb2_phy1>;
+			phy-names = "usb";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 702>;
-			#phy-cells = <0>;
 			status = "disabled";
 		};
 
-		usb2_phy2: usb-phy@ee0c0200 {
-			compatible = "renesas,usb2-phy-r8a7795",
-				     "renesas,rcar-gen3-usb2-phy";
-			reg = <0 0xee0c0200 0 0x700>;
+		ohci2: usb@ee0c0000 {
+			compatible = "generic-ohci";
+			reg = <0 0xee0c0000 0 0x100>;
+			interrupts = <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 701>;
+			phys = <&usb2_phy2>;
+			phy-names = "usb";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 701>;
-			#phy-cells = <0>;
 			status = "disabled";
 		};
 
-		usb2_phy3: usb-phy@ee0e0200 {
-			compatible = "renesas,usb2-phy-r8a7795",
-				     "renesas,rcar-gen3-usb2-phy";
-			reg = <0 0xee0e0200 0 0x700>;
+		ohci3: usb@ee0e0000 {
+			compatible = "generic-ohci";
+			reg = <0 0xee0e0000 0 0x100>;
 			interrupts = <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 700>;
+			phys = <&usb2_phy3>;
+			phy-names = "usb";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 700>;
-			#phy-cells = <0>;
 			status = "disabled";
 		};
 
@@ -1961,88 +2188,129 @@
 			status = "disabled";
 		};
 
-		ohci0: usb@ee080000 {
-			compatible = "generic-ohci";
-			reg = <0 0xee080000 0 0x100>;
+		usb2_phy0: usb-phy@ee080200 {
+			compatible = "renesas,usb2-phy-r8a7795",
+				     "renesas,rcar-gen3-usb2-phy";
+			reg = <0 0xee080200 0 0x700>;
 			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 703>;
-			phys = <&usb2_phy0>;
-			phy-names = "usb";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 703>;
+			#phy-cells = <0>;
 			status = "disabled";
 		};
 
-		ohci1: usb@ee0a0000 {
-			compatible = "generic-ohci";
-			reg = <0 0xee0a0000 0 0x100>;
-			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
+		usb2_phy1: usb-phy@ee0a0200 {
+			compatible = "renesas,usb2-phy-r8a7795",
+				     "renesas,rcar-gen3-usb2-phy";
+			reg = <0 0xee0a0200 0 0x700>;
 			clocks = <&cpg CPG_MOD 702>;
-			phys = <&usb2_phy1>;
-			phy-names = "usb";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 702>;
+			#phy-cells = <0>;
 			status = "disabled";
 		};
 
-		ohci2: usb@ee0c0000 {
-			compatible = "generic-ohci";
-			reg = <0 0xee0c0000 0 0x100>;
-			interrupts = <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>;
+		usb2_phy2: usb-phy@ee0c0200 {
+			compatible = "renesas,usb2-phy-r8a7795",
+				     "renesas,rcar-gen3-usb2-phy";
+			reg = <0 0xee0c0200 0 0x700>;
 			clocks = <&cpg CPG_MOD 701>;
-			phys = <&usb2_phy2>;
-			phy-names = "usb";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 701>;
+			#phy-cells = <0>;
 			status = "disabled";
 		};
 
-		ohci3: usb@ee0e0000 {
-			compatible = "generic-ohci";
-			reg = <0 0xee0e0000 0 0x100>;
+		usb2_phy3: usb-phy@ee0e0200 {
+			compatible = "renesas,usb2-phy-r8a7795",
+				     "renesas,rcar-gen3-usb2-phy";
+			reg = <0 0xee0e0200 0 0x700>;
 			interrupts = <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 700>;
-			phys = <&usb2_phy3>;
-			phy-names = "usb";
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 700>;
+			#phy-cells = <0>;
 			status = "disabled";
 		};
 
-		hsusb: usb@e6590000 {
-			compatible = "renesas,usbhs-r8a7795",
-				     "renesas,rcar-gen3-usbhs";
-			reg = <0 0xe6590000 0 0x100>;
-			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 704>;
-			dmas = <&usb_dmac0 0>, <&usb_dmac0 1>,
-			       <&usb_dmac1 0>, <&usb_dmac1 1>;
-			dma-names = "ch0", "ch1", "ch2", "ch3";
-			renesas,buswait = <11>;
-			phys = <&usb2_phy0>;
-			phy-names = "usb";
+		sdhi0: sd@ee100000 {
+			compatible = "renesas,sdhi-r8a7795",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee100000 0 0x2000>;
+			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 314>;
+			max-frequency = <200000000>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 704>;
+			resets = <&cpg 314>;
 			status = "disabled";
 		};
 
-		hsusb3: usb@e659c000 {
-			compatible = "renesas,usbhs-r8a7795",
-				     "renesas,rcar-gen3-usbhs";
-			reg = <0 0xe659c000 0 0x100>;
-			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 705>;
-			dmas = <&usb_dmac2 0>, <&usb_dmac2 1>,
-			       <&usb_dmac3 0>, <&usb_dmac3 1>;
-			dma-names = "ch0", "ch1", "ch2", "ch3";
-			renesas,buswait = <11>;
-			phys = <&usb2_phy3>;
-			phy-names = "usb";
+		sdhi1: sd@ee120000 {
+			compatible = "renesas,sdhi-r8a7795",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee120000 0 0x2000>;
+			interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 313>;
+			max-frequency = <200000000>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 705>;
+			resets = <&cpg 313>;
 			status = "disabled";
 		};
 
+		sdhi2: sd@ee140000 {
+			compatible = "renesas,sdhi-r8a7795",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee140000 0 0x2000>;
+			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 312>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 312>;
+			status = "disabled";
+		};
+
+		sdhi3: sd@ee160000 {
+			compatible = "renesas,sdhi-r8a7795",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee160000 0 0x2000>;
+			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 311>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 311>;
+			status = "disabled";
+		};
+
+		sata: sata@ee300000 {
+			compatible = "renesas,sata-r8a7795",
+				     "renesas,rcar-gen3-sata";
+			reg = <0 0xee300000 0 0x200000>;
+			interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 815>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 815>;
+			status = "disabled";
+			iommus = <&ipmmu_hc 2>;
+		};
+
+		gic: interrupt-controller@f1010000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0x0 0xf1010000 0 0x1000>,
+			      <0x0 0xf1020000 0 0x20000>,
+			      <0x0 0xf1040000 0 0x20000>,
+			      <0x0 0xf1060000 0 0x20000>;
+			interrupts = <GIC_PPI 9
+					(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_HIGH)>;
+			clocks = <&cpg CPG_MOD 408>;
+			clock-names = "clk";
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 408>;
+		};
+
 		pciec0: pcie@fe000000 {
 			compatible = "renesas,pcie-r8a7795",
 				     "renesas,pcie-rcar-gen3";
@@ -2133,28 +2401,28 @@
 			reg = <0 0xfe890000 0 0x2000>;
 			interrupts = <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 820>;
-			power-domains = <&sysc R8A7795_PD_A3VC>;
-			resets = <&cpg 820>;
-		};
-
-		vspbc: vsp@fe920000 {
-			compatible = "renesas,vsp2";
-			reg = <0 0xfe920000 0 0x8000>;
-			interrupts = <GIC_SPI 465 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 624>;
-			power-domains = <&sysc R8A7795_PD_A3VP>;
-			resets = <&cpg 624>;
+			power-domains = <&sysc R8A7795_PD_A3VC>;
+			resets = <&cpg 820>;
+		};
 
-			renesas,fcp = <&fcpvb1>;
+		fdp1@fe940000 {
+			compatible = "renesas,fdp1";
+			reg = <0 0xfe940000 0 0x2400>;
+			interrupts = <GIC_SPI 262 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 119>;
+			power-domains = <&sysc R8A7795_PD_A3VP>;
+			resets = <&cpg 119>;
+			renesas,fcp = <&fcpf0>;
 		};
 
-		fcpvb1: fcp@fe92f000 {
-			compatible = "renesas,fcpv";
-			reg = <0 0xfe92f000 0 0x200>;
-			clocks = <&cpg CPG_MOD 606>;
+		fdp1@fe944000 {
+			compatible = "renesas,fdp1";
+			reg = <0 0xfe944000 0 0x2400>;
+			interrupts = <GIC_SPI 263 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 118>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
-			resets = <&cpg 606>;
-			iommus = <&ipmmu_vp1 7>;
+			resets = <&cpg 118>;
+			renesas,fcp = <&fcpf1>;
 		};
 
 		fcpf0: fcp@fe950000 {
@@ -2175,17 +2443,6 @@
 			iommus = <&ipmmu_vp1 1>;
 		};
 
-		vspbd: vsp@fe960000 {
-			compatible = "renesas,vsp2";
-			reg = <0 0xfe960000 0 0x8000>;
-			interrupts = <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 626>;
-			power-domains = <&sysc R8A7795_PD_A3VP>;
-			resets = <&cpg 626>;
-
-			renesas,fcp = <&fcpvb0>;
-		};
-
 		fcpvb0: fcp@fe96f000 {
 			compatible = "renesas,fcpv";
 			reg = <0 0xfe96f000 0 0x200>;
@@ -2195,15 +2452,13 @@
 			iommus = <&ipmmu_vp0 5>;
 		};
 
-		vspi0: vsp@fe9a0000 {
-			compatible = "renesas,vsp2";
-			reg = <0 0xfe9a0000 0 0x8000>;
-			interrupts = <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 631>;
+		fcpvb1: fcp@fe92f000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfe92f000 0 0x200>;
+			clocks = <&cpg CPG_MOD 606>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
-			resets = <&cpg 631>;
-
-			renesas,fcp = <&fcpvi0>;
+			resets = <&cpg 606>;
+			iommus = <&ipmmu_vp1 7>;
 		};
 
 		fcpvi0: fcp@fe9af000 {
@@ -2215,17 +2470,6 @@
 			iommus = <&ipmmu_vp0 8>;
 		};
 
-		vspi1: vsp@fe9b0000 {
-			compatible = "renesas,vsp2";
-			reg = <0 0xfe9b0000 0 0x8000>;
-			interrupts = <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 630>;
-			power-domains = <&sysc R8A7795_PD_A3VP>;
-			resets = <&cpg 630>;
-
-			renesas,fcp = <&fcpvi1>;
-		};
-
 		fcpvi1: fcp@fe9bf000 {
 			compatible = "renesas,fcpv";
 			reg = <0 0xfe9bf000 0 0x200>;
@@ -2235,6 +2479,55 @@
 			iommus = <&ipmmu_vp1 9>;
 		};
 
+		fcpvd0: fcp@fea27000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfea27000 0 0x200>;
+			clocks = <&cpg CPG_MOD 603>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 603>;
+			iommus = <&ipmmu_vi0 8>;
+		};
+
+		fcpvd1: fcp@fea2f000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfea2f000 0 0x200>;
+			clocks = <&cpg CPG_MOD 602>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 602>;
+			iommus = <&ipmmu_vi0 9>;
+		};
+
+		fcpvd2: fcp@fea37000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfea37000 0 0x200>;
+			clocks = <&cpg CPG_MOD 601>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 601>;
+			iommus = <&ipmmu_vi1 10>;
+		};
+
+		vspbd: vsp@fe960000 {
+			compatible = "renesas,vsp2";
+			reg = <0 0xfe960000 0 0x8000>;
+			interrupts = <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 626>;
+			power-domains = <&sysc R8A7795_PD_A3VP>;
+			resets = <&cpg 626>;
+
+			renesas,fcp = <&fcpvb0>;
+		};
+
+		vspbc: vsp@fe920000 {
+			compatible = "renesas,vsp2";
+			reg = <0 0xfe920000 0 0x8000>;
+			interrupts = <GIC_SPI 465 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 624>;
+			power-domains = <&sysc R8A7795_PD_A3VP>;
+			resets = <&cpg 624>;
+
+			renesas,fcp = <&fcpvb1>;
+		};
+
 		vspd0: vsp@fea20000 {
 			compatible = "renesas,vsp2";
 			reg = <0 0xfea20000 0 0x8000>;
@@ -2246,15 +2539,6 @@
 			renesas,fcp = <&fcpvd0>;
 		};
 
-		fcpvd0: fcp@fea27000 {
-			compatible = "renesas,fcpv";
-			reg = <0 0xfea27000 0 0x200>;
-			clocks = <&cpg CPG_MOD 603>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 603>;
-			iommus = <&ipmmu_vi0 8>;
-		};
-
 		vspd1: vsp@fea28000 {
 			compatible = "renesas,vsp2";
 			reg = <0 0xfea28000 0 0x8000>;
@@ -2266,15 +2550,6 @@
 			renesas,fcp = <&fcpvd1>;
 		};
 
-		fcpvd1: fcp@fea2f000 {
-			compatible = "renesas,fcpv";
-			reg = <0 0xfea2f000 0 0x200>;
-			clocks = <&cpg CPG_MOD 602>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 602>;
-			iommus = <&ipmmu_vi0 9>;
-		};
-
 		vspd2: vsp@fea30000 {
 			compatible = "renesas,vsp2";
 			reg = <0 0xfea30000 0 0x8000>;
@@ -2286,33 +2561,159 @@
 			renesas,fcp = <&fcpvd2>;
 		};
 
-		fcpvd2: fcp@fea37000 {
-			compatible = "renesas,fcpv";
-			reg = <0 0xfea37000 0 0x200>;
-			clocks = <&cpg CPG_MOD 601>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 601>;
-			iommus = <&ipmmu_vi1 10>;
+		vspi0: vsp@fe9a0000 {
+			compatible = "renesas,vsp2";
+			reg = <0 0xfe9a0000 0 0x8000>;
+			interrupts = <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 631>;
+			power-domains = <&sysc R8A7795_PD_A3VP>;
+			resets = <&cpg 631>;
+
+			renesas,fcp = <&fcpvi0>;
 		};
 
-		fdp1@fe940000 {
-			compatible = "renesas,fdp1";
-			reg = <0 0xfe940000 0 0x2400>;
-			interrupts = <GIC_SPI 262 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 119>;
+		vspi1: vsp@fe9b0000 {
+			compatible = "renesas,vsp2";
+			reg = <0 0xfe9b0000 0 0x8000>;
+			interrupts = <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 630>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
-			resets = <&cpg 119>;
-			renesas,fcp = <&fcpf0>;
+			resets = <&cpg 630>;
+
+			renesas,fcp = <&fcpvi1>;
 		};
 
-		fdp1@fe944000 {
-			compatible = "renesas,fdp1";
-			reg = <0 0xfe944000 0 0x2400>;
-			interrupts = <GIC_SPI 263 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 118>;
-			power-domains = <&sysc R8A7795_PD_A3VP>;
-			resets = <&cpg 118>;
-			renesas,fcp = <&fcpf1>;
+		csi20: csi2@fea80000 {
+			compatible = "renesas,r8a7795-csi2";
+			reg = <0 0xfea80000 0 0x10000>;
+			interrupts = <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 714>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 714>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					csi20vin0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&vin0csi20>;
+					};
+					csi20vin1: endpoint@1 {
+						reg = <1>;
+						remote-endpoint = <&vin1csi20>;
+					};
+					csi20vin2: endpoint@2 {
+						reg = <2>;
+						remote-endpoint = <&vin2csi20>;
+					};
+					csi20vin3: endpoint@3 {
+						reg = <3>;
+						remote-endpoint = <&vin3csi20>;
+					};
+					csi20vin4: endpoint@4 {
+						reg = <4>;
+						remote-endpoint = <&vin4csi20>;
+					};
+					csi20vin5: endpoint@5 {
+						reg = <5>;
+						remote-endpoint = <&vin5csi20>;
+					};
+					csi20vin6: endpoint@6 {
+						reg = <6>;
+						remote-endpoint = <&vin6csi20>;
+					};
+					csi20vin7: endpoint@7 {
+						reg = <7>;
+						remote-endpoint = <&vin7csi20>;
+					};
+				};
+			};
+		};
+
+		csi40: csi2@feaa0000 {
+			compatible = "renesas,r8a7795-csi2";
+			reg = <0 0xfeaa0000 0 0x10000>;
+			interrupts = <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 716>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 716>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					csi40vin0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&vin0csi40>;
+					};
+					csi40vin1: endpoint@1 {
+						reg = <1>;
+						remote-endpoint = <&vin1csi40>;
+					};
+					csi40vin2: endpoint@2 {
+						reg = <2>;
+						remote-endpoint = <&vin2csi40>;
+					};
+					csi40vin3: endpoint@3 {
+						reg = <3>;
+						remote-endpoint = <&vin3csi40>;
+					};
+				};
+			};
+		};
+
+		csi41: csi2@feab0000 {
+			compatible = "renesas,r8a7795-csi2";
+			reg = <0 0xfeab0000 0 0x10000>;
+			interrupts = <GIC_SPI 247 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 715>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 715>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					csi41vin4: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&vin4csi41>;
+					};
+					csi41vin5: endpoint@1 {
+						reg = <1>;
+						remote-endpoint = <&vin5csi41>;
+					};
+					csi41vin6: endpoint@2 {
+						reg = <2>;
+						remote-endpoint = <&vin6csi41>;
+					};
+					csi41vin7: endpoint@3 {
+						reg = <3>;
+						remote-endpoint = <&vin7csi41>;
+					};
+				};
+			};
 		};
 
 		hdmi0: hdmi@fead0000 {
@@ -2337,6 +2738,10 @@
 				port@1 {
 					reg = <1>;
 				};
+				port@2 {
+					/* HDMI sound */
+					reg = <2>;
+				};
 			};
 		};
 
@@ -2362,6 +2767,10 @@
 				port@1 {
 					reg = <1>;
 				};
+				port@2 {
+					/* HDMI sound */
+					reg = <2>;
+				};
 			};
 		};
 
@@ -2412,38 +2821,12 @@
 			};
 		};
 
-		tsc: thermal@e6198000 {
-			compatible = "renesas,r8a7795-thermal";
-			reg = <0 0xe6198000 0 0x100>,
-			      <0 0xe61a0000 0 0x100>,
-			      <0 0xe61a8000 0 0x100>;
-			interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 522>;
-			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
-			resets = <&cpg 522>;
-			#thermal-sensor-cells = <1>;
-			status = "okay";
+		prr: chipid@fff00044 {
+			compatible = "renesas,prr";
+			reg = <0 0xfff00044 0 4>;
 		};
 	};
 
-	timer {
-		compatible = "arm,armv8-timer";
-		interrupts-extended = <&gic GIC_PPI 13
-				       (GIC_CPU_MASK_SIMPLE(8) |
-				       IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 14
-				       (GIC_CPU_MASK_SIMPLE(8) |
-				       IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 11
-				       (GIC_CPU_MASK_SIMPLE(8) |
-				       IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 10
-				       (GIC_CPU_MASK_SIMPLE(8) |
-				       IRQ_TYPE_LEVEL_LOW)>;
-	};
-
 	thermal-zones {
 		sensor_thermal1: sensor-thermal1 {
 			polling-delay-passive = <250>;
@@ -2453,12 +2836,12 @@
 			trips {
 				sensor1_passive: sensor1-passive {
 					temperature = <95000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "passive";
 				};
 				sensor1_crit: sensor1-crit {
 					temperature = <120000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "critical";
 				};
 			};
@@ -2479,12 +2862,12 @@
 			trips {
 				sensor2_passive: sensor2-passive {
 					temperature = <95000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "passive";
 				};
 				sensor2_crit: sensor2-crit {
 					temperature = <120000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "critical";
 				};
 			};
@@ -2505,12 +2888,12 @@
 			trips {
 				sensor3_passive: sensor3-passive {
 					temperature = <95000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "passive";
 				};
 				sensor3_crit: sensor3-crit {
 					temperature = <120000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "critical";
 				};
 			};
@@ -2524,6 +2907,14 @@
 		};
 	};
 
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>;
+	};
+
 	/* External USB clocks - can be overridden by the board */
 	usb3s0_clk: usb3s0 {
 		compatible = "fixed-clock";
diff --git a/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts b/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts
index 498c9e807dc4..90cca09b9a5e 100644
--- a/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts
@@ -40,6 +40,11 @@
 		      "dclkin.0", "dclkin.1", "dclkin.2";
 };
 
+&sound_card {
+	dais = <&rsnd_port0	/* ak4613 */
+		&rsnd_port1>;	/* HDMI0  */
+};
+
 &hdmi0 {
 	status = "okay";
 
@@ -50,9 +55,32 @@
 				remote-endpoint = <&hdmi0_con>;
 			};
 		};
+		port@2 {
+			reg = <2>;
+			dw_hdmi0_snd_in: endpoint {
+				remote-endpoint = <&rsnd_endpoint1>;
+			};
+		};
 	};
 };
 
 &hdmi0_con {
 	remote-endpoint = <&rcar_dw_hdmi0_out>;
 };
+
+&rcar_sound {
+	ports {
+		/* rsnd_port0 is on salvator-common */
+		rsnd_port1: port@1 {
+			rsnd_endpoint1: endpoint {
+				remote-endpoint = <&dw_hdmi0_snd_in>;
+
+				dai-format = "i2s";
+				bitclock-master = <&rsnd_endpoint1>;
+				frame-master = <&rsnd_endpoint1>;
+
+				playback = <&ssi2>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a7796-salvator-xs.dts b/arch/arm64/boot/dts/renesas/r8a7796-salvator-xs.dts
index 2c37055efa94..ddf35d4cd5e5 100644
--- a/arch/arm64/boot/dts/renesas/r8a7796-salvator-xs.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7796-salvator-xs.dts
@@ -40,6 +40,11 @@
 		      "dclkin.0", "dclkin.1", "dclkin.2";
 };
 
+&sound_card {
+	dais = <&rsnd_port0	/* ak4613 */
+		&rsnd_port1>;	/* HDMI0  */
+};
+
 &hdmi0 {
 	status = "okay";
 
@@ -50,9 +55,32 @@
 				remote-endpoint = <&hdmi0_con>;
 			};
 		};
+		port@2 {
+			reg = <2>;
+			dw_hdmi0_snd_in: endpoint {
+				remote-endpoint = <&rsnd_endpoint1>;
+			};
+		};
 	};
 };
 
 &hdmi0_con {
 	remote-endpoint = <&rcar_dw_hdmi0_out>;
 };
+
+&rcar_sound {
+	ports {
+		/* rsnd_port0 is on salvator-common */
+		rsnd_port1: port@1 {
+			rsnd_endpoint1: endpoint {
+				remote-endpoint = <&dw_hdmi0_snd_in>;
+
+				dai-format = "i2s";
+				bitclock-master = <&rsnd_endpoint1>;
+				frame-master = <&rsnd_endpoint1>;
+
+				playback = <&ssi2>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a7796.dtsi b/arch/arm64/boot/dts/renesas/r8a7796.dtsi
index 556eb8e45499..7c25be6b5af3 100644
--- a/arch/arm64/boot/dts/renesas/r8a7796.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a7796.dtsi
@@ -60,6 +60,72 @@
 		clock-frequency = <0>;
 	};
 
+	cluster0_opp: opp_table0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-500000000 {
+			opp-hz = /bits/ 64 <500000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1500000000 {
+			opp-hz = /bits/ 64 <1500000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1600000000 {
+			opp-hz = /bits/ 64 <1600000000>;
+			opp-microvolt = <900000>;
+			clock-latency-ns = <300000>;
+			turbo-mode;
+		};
+		opp-1700000000 {
+			opp-hz = /bits/ 64 <1700000000>;
+			opp-microvolt = <900000>;
+			clock-latency-ns = <300000>;
+			turbo-mode;
+		};
+		opp-1800000000 {
+			opp-hz = /bits/ 64 <1800000000>;
+			opp-microvolt = <960000>;
+			clock-latency-ns = <300000>;
+			turbo-mode;
+		};
+	};
+
+	cluster1_opp: opp_table1 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1300000000 {
+			opp-hz = /bits/ 64 <1300000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+			turbo-mode;
+		};
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -77,7 +143,7 @@
 		};
 
 		a57_1: cpu@1 {
-			compatible = "arm,cortex-a57","arm,armv8";
+			compatible = "arm,cortex-a57", "arm,armv8";
 			reg = <0x1>;
 			device_type = "cpu";
 			power-domains = <&sysc R8A7796_PD_CA57_CPU1>;
@@ -100,7 +166,7 @@
 		};
 
 		a53_1: cpu@101 {
-			compatible = "arm,cortex-a53","arm,armv8";
+			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0x101>;
 			device_type = "cpu";
 			power-domains = <&sysc R8A7796_PD_CA53_CPU1>;
@@ -111,7 +177,7 @@
 		};
 
 		a53_2: cpu@102 {
-			compatible = "arm,cortex-a53","arm,armv8";
+			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0x102>;
 			device_type = "cpu";
 			power-domains = <&sysc R8A7796_PD_CA53_CPU2>;
@@ -122,7 +188,7 @@
 		};
 
 		a53_3: cpu@103 {
-			compatible = "arm,cortex-a53","arm,armv8";
+			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0x103>;
 			device_type = "cpu";
 			power-domains = <&sysc R8A7796_PD_CA53_CPU3>;
@@ -161,72 +227,6 @@
 		clock-frequency = <0>;
 	};
 
-	cluster0_opp: opp_table0 {
-		compatible = "operating-points-v2";
-		opp-shared;
-
-		opp-500000000 {
-			opp-hz = /bits/ 64 <500000000>;
-			opp-microvolt = <820000>;
-			clock-latency-ns = <300000>;
-		};
-		opp-1000000000 {
-			opp-hz = /bits/ 64 <1000000000>;
-			opp-microvolt = <820000>;
-			clock-latency-ns = <300000>;
-		};
-		opp-1500000000 {
-			opp-hz = /bits/ 64 <1500000000>;
-			opp-microvolt = <820000>;
-			clock-latency-ns = <300000>;
-		};
-		opp-1600000000 {
-			opp-hz = /bits/ 64 <1600000000>;
-			opp-microvolt = <900000>;
-			clock-latency-ns = <300000>;
-			turbo-mode;
-		};
-		opp-1700000000 {
-			opp-hz = /bits/ 64 <1700000000>;
-			opp-microvolt = <900000>;
-			clock-latency-ns = <300000>;
-			turbo-mode;
-		};
-		opp-1800000000 {
-			opp-hz = /bits/ 64 <1800000000>;
-			opp-microvolt = <960000>;
-			clock-latency-ns = <300000>;
-			turbo-mode;
-		};
-	};
-
-	cluster1_opp: opp_table1 {
-		compatible = "operating-points-v2";
-		opp-shared;
-
-		opp-800000000 {
-			opp-hz = /bits/ 64 <800000000>;
-			opp-microvolt = <820000>;
-			clock-latency-ns = <300000>;
-		};
-		opp-1000000000 {
-			opp-hz = /bits/ 64 <1000000000>;
-			opp-microvolt = <820000>;
-			clock-latency-ns = <300000>;
-		};
-		opp-1200000000 {
-			opp-hz = /bits/ 64 <1200000000>;
-			opp-microvolt = <820000>;
-			clock-latency-ns = <300000>;
-		};
-		opp-1300000000 {
-			opp-hz = /bits/ 64 <1300000000>;
-			opp-microvolt = <820000>;
-			clock-latency-ns = <300000>;
-			turbo-mode;
-		};
-	};
-
 	/* External PCIe clock - can be overridden by the board */
 	pcie_bus_clk: pcie_bus {
 		compatible = "fixed-clock";
@@ -234,13 +234,6 @@
 		clock-frequency = <0>;
 	};
 
-	pmu_a57 {
-		compatible = "arm,cortex-a57-pmu";
-		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
-				      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-affinity = <&a57_0>, <&a57_1>;
-	};
-
 	pmu_a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts-extended = <&gic GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
@@ -250,6 +243,13 @@
 		interrupt-affinity = <&a53_0>, <&a53_1>, <&a53_2>, <&a53_3>;
 	};
 
+	pmu_a57 {
+		compatible = "arm,cortex-a57-pmu";
+		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&a57_0>, <&a57_1>;
+	};
+
 	psci {
 		compatible = "arm,psci-1.0", "arm,psci-0.2";
 		method = "smc";
@@ -269,23 +269,6 @@
 		#size-cells = <2>;
 		ranges;
 
-		gic: interrupt-controller@f1010000 {
-			compatible = "arm,gic-400";
-			#interrupt-cells = <3>;
-			#address-cells = <0>;
-			interrupt-controller;
-			reg = <0x0 0xf1010000 0 0x1000>,
-			      <0x0 0xf1020000 0 0x20000>,
-			      <0x0 0xf1040000 0 0x20000>,
-			      <0x0 0xf1060000 0 0x20000>;
-			interrupts = <GIC_PPI 9
-					(GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_HIGH)>;
-			clocks = <&cpg CPG_MOD 408>;
-			clock-names = "clk";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 408>;
-		};
-
 		wdt0: watchdog@e6020000 {
 			compatible = "renesas,r8a7796-wdt",
 				     "renesas,rcar-gen3-wdt";
@@ -421,242 +404,72 @@
 			reg = <0 0xe6060000 0 0x50c>;
 		};
 
-		ipmmu_vi0: mmu@febd0000 {
-			compatible = "renesas,ipmmu-r8a7796";
-			reg = <0 0xfebd0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 9>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+		cpg: clock-controller@e6150000 {
+			compatible = "renesas,r8a7796-cpg-mssr";
+			reg = <0 0xe6150000 0 0x1000>;
+			clocks = <&extal_clk>, <&extalr_clk>;
+			clock-names = "extal", "extalr";
+			#clock-cells = <2>;
+			#power-domain-cells = <0>;
+			#reset-cells = <1>;
 		};
 
-		ipmmu_vc0: mmu@fe6b0000 {
-			compatible = "renesas,ipmmu-r8a7796";
-			reg = <0 0xfe6b0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 8>;
-			power-domains = <&sysc R8A7796_PD_A3VC>;
-			#iommu-cells = <1>;
-			status = "disabled";
+		rst: reset-controller@e6160000 {
+			compatible = "renesas,r8a7796-rst";
+			reg = <0 0xe6160000 0 0x0200>;
 		};
 
-		ipmmu_pv0: mmu@fd800000 {
-			compatible = "renesas,ipmmu-r8a7796";
-			reg = <0 0xfd800000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 5>;
+		sysc: system-controller@e6180000 {
+			compatible = "renesas,r8a7796-sysc";
+			reg = <0 0xe6180000 0 0x0400>;
+			#power-domain-cells = <1>;
+		};
+
+		tsc: thermal@e6198000 {
+			compatible = "renesas,r8a7796-thermal";
+			reg = <0 0xe6198000 0 0x100>,
+			      <0 0xe61a0000 0 0x100>,
+			      <0 0xe61a8000 0 0x100>;
+			interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 522>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
+			resets = <&cpg 522>;
+			#thermal-sensor-cells = <1>;
+			status = "okay";
 		};
 
-		ipmmu_pv1: mmu@fd950000 {
-			compatible = "renesas,ipmmu-r8a7796";
-			reg = <0 0xfd950000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 6>;
+		intc_ex: interrupt-controller@e61c0000 {
+			compatible = "renesas,intc-ex-r8a7796", "renesas,irqc";
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			reg = <0 0xe61c0000 0 0x200>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 407>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-			status = "disabled";
+			resets = <&cpg 407>;
 		};
 
-		ipmmu_ir: mmu@ff8b0000 {
-			compatible = "renesas,ipmmu-r8a7796";
-			reg = <0 0xff8b0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 3>;
-			power-domains = <&sysc R8A7796_PD_A3IR>;
-			#iommu-cells = <1>;
-			status = "disabled";
-		};
-
-		ipmmu_hc: mmu@e6570000 {
-			compatible = "renesas,ipmmu-r8a7796";
-			reg = <0 0xe6570000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 2>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-			status = "disabled";
-		};
-
-		ipmmu_rt: mmu@ffc80000 {
-			compatible = "renesas,ipmmu-r8a7796";
-			reg = <0 0xffc80000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 7>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-			status = "disabled";
-		};
-
-		ipmmu_mp: mmu@ec670000 {
-			compatible = "renesas,ipmmu-r8a7796";
-			reg = <0 0xec670000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 4>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-		};
-
-		ipmmu_ds0: mmu@e6740000 {
-			compatible = "renesas,ipmmu-r8a7796";
-			reg = <0 0xe6740000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 0>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-		};
-
-		ipmmu_ds1: mmu@e7740000 {
-			compatible = "renesas,ipmmu-r8a7796";
-			reg = <0 0xe7740000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 1>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-		};
-
-		ipmmu_mm: mmu@e67b0000 {
-			compatible = "renesas,ipmmu-r8a7796";
-			reg = <0 0xe67b0000 0 0x1000>;
-			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-		};
-
-		cpg: clock-controller@e6150000 {
-			compatible = "renesas,r8a7796-cpg-mssr";
-			reg = <0 0xe6150000 0 0x1000>;
-			clocks = <&extal_clk>, <&extalr_clk>;
-			clock-names = "extal", "extalr";
-			#clock-cells = <2>;
-			#power-domain-cells = <0>;
-			#reset-cells = <1>;
-		};
-
-		rst: reset-controller@e6160000 {
-			compatible = "renesas,r8a7796-rst";
-			reg = <0 0xe6160000 0 0x0200>;
-		};
-
-		prr: chipid@fff00044 {
-			compatible = "renesas,prr";
-			reg = <0 0xfff00044 0 4>;
-		};
-
-		sysc: system-controller@e6180000 {
-			compatible = "renesas,r8a7796-sysc";
-			reg = <0 0xe6180000 0 0x0400>;
-			#power-domain-cells = <1>;
-		};
-
-		intc_ex: interrupt-controller@e61c0000 {
-			compatible = "renesas,intc-ex-r8a7796", "renesas,irqc";
-			#interrupt-cells = <2>;
-			interrupt-controller;
-			reg = <0 0xe61c0000 0 0x200>;
-			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 407>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 407>;
-		};
-
-		i2c_dvfs: i2c@e60b0000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,iic-r8a7796",
-				     "renesas,rcar-gen3-iic",
-				     "renesas,rmobile-iic";
-			reg = <0 0xe60b0000 0 0x425>;
-			interrupts = <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 926>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 926>;
-			dmas = <&dmac0 0x11>, <&dmac0 0x10>;
-			dma-names = "tx", "rx";
-			status = "disabled";
-		};
-
-		pwm0: pwm@e6e30000 {
-			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
-			reg = <0 0xe6e30000 0 8>;
-			#pwm-cells = <2>;
-			clocks = <&cpg CPG_MOD 523>;
-			resets = <&cpg 523>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			status = "disabled";
-		};
-
-		pwm1: pwm@e6e31000 {
-			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
-			reg = <0 0xe6e31000 0 8>;
-			#pwm-cells = <2>;
-			clocks = <&cpg CPG_MOD 523>;
-			resets = <&cpg 523>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			status = "disabled";
-		};
-
-		pwm2: pwm@e6e32000 {
-			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
-			reg = <0 0xe6e32000 0 8>;
-			#pwm-cells = <2>;
-			clocks = <&cpg CPG_MOD 523>;
-			resets = <&cpg 523>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			status = "disabled";
-		};
-
-		pwm3: pwm@e6e33000 {
-			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
-			reg = <0 0xe6e33000 0 8>;
-			#pwm-cells = <2>;
-			clocks = <&cpg CPG_MOD 523>;
-			resets = <&cpg 523>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			status = "disabled";
-		};
-
-		pwm4: pwm@e6e34000 {
-			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
-			reg = <0 0xe6e34000 0 8>;
-			#pwm-cells = <2>;
-			clocks = <&cpg CPG_MOD 523>;
-			resets = <&cpg 523>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			status = "disabled";
-		};
-
-		pwm5: pwm@e6e35000 {
-			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
-			reg = <0 0xe6e35000 0 8>;
-			#pwm-cells = <2>;
-			clocks = <&cpg CPG_MOD 523>;
-			resets = <&cpg 523>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			status = "disabled";
-		};
-
-		pwm6: pwm@e6e36000 {
-			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
-			reg = <0 0xe6e36000 0 8>;
-			#pwm-cells = <2>;
-			clocks = <&cpg CPG_MOD 523>;
-			resets = <&cpg 523>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			status = "disabled";
-		};
-
-		i2c0: i2c@e6500000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a7796",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe6500000 0 0x40>;
-			interrupts = <GIC_SPI 287 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 931>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 931>;
-			dmas = <&dmac1 0x91>, <&dmac1 0x90>,
-			       <&dmac2 0x91>, <&dmac2 0x90>;
-			dma-names = "tx", "rx", "tx", "rx";
-			i2c-scl-internal-delay-ns = <110>;
+		i2c0: i2c@e6500000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7796",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe6500000 0 0x40>;
+			interrupts = <GIC_SPI 287 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 931>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 931>;
+			dmas = <&dmac1 0x91>, <&dmac1 0x90>,
+			       <&dmac2 0x91>, <&dmac2 0x90>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
 			status = "disabled";
 		};
 
@@ -758,244 +571,37 @@
 			status = "disabled";
 		};
 
-		can0: can@e6c30000 {
-			compatible = "renesas,can-r8a7796",
-				     "renesas,rcar-gen3-can";
-			reg = <0 0xe6c30000 0 0x1000>;
-			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 916>,
-			       <&cpg CPG_CORE R8A7796_CLK_CANFD>,
-			       <&can_clk>;
-			clock-names = "clkp1", "clkp2", "can_clk";
-			assigned-clocks = <&cpg CPG_CORE R8A7796_CLK_CANFD>;
-			assigned-clock-rates = <40000000>;
+		i2c_dvfs: i2c@e60b0000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,iic-r8a7796",
+				     "renesas,rcar-gen3-iic",
+				     "renesas,rmobile-iic";
+			reg = <0 0xe60b0000 0 0x425>;
+			interrupts = <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 926>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 916>;
+			resets = <&cpg 926>;
+			dmas = <&dmac0 0x11>, <&dmac0 0x10>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
-		can1: can@e6c38000 {
-			compatible = "renesas,can-r8a7796",
-				     "renesas,rcar-gen3-can";
-			reg = <0 0xe6c38000 0 0x1000>;
-			interrupts = <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 915>,
-			       <&cpg CPG_CORE R8A7796_CLK_CANFD>,
-			       <&can_clk>;
-			clock-names = "clkp1", "clkp2", "can_clk";
-			assigned-clocks = <&cpg CPG_CORE R8A7796_CLK_CANFD>;
-			assigned-clock-rates = <40000000>;
+		hscif0: serial@e6540000 {
+			compatible = "renesas,hscif-r8a7796",
+				     "renesas,rcar-gen3-hscif",
+				     "renesas,hscif";
+			reg = <0 0xe6540000 0 0x60>;
+			interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 520>,
+				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac1 0x31>, <&dmac1 0x30>,
+			       <&dmac2 0x31>, <&dmac2 0x30>;
+			dma-names = "tx", "rx", "tx", "rx";
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 915>;
-			status = "disabled";
-		};
-
-		canfd: can@e66c0000 {
-			compatible = "renesas,r8a7796-canfd",
-				     "renesas,rcar-gen3-canfd";
-			reg = <0 0xe66c0000 0 0x8000>;
-			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>,
-				   <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 914>,
-			       <&cpg CPG_CORE R8A7796_CLK_CANFD>,
-			       <&can_clk>;
-			clock-names = "fck", "canfd", "can_clk";
-			assigned-clocks = <&cpg CPG_CORE R8A7796_CLK_CANFD>;
-			assigned-clock-rates = <40000000>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 914>;
-			status = "disabled";
-
-			channel0 {
-				status = "disabled";
-			};
-
-			channel1 {
-				status = "disabled";
-			};
-		};
-
-		drif00: rif@e6f40000 {
-			compatible = "renesas,r8a7796-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f40000 0 0x64>;
-			interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 515>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x20>, <&dmac2 0x20>;
-			dma-names = "rx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 515>;
-			renesas,bonding = <&drif01>;
-			status = "disabled";
-		};
-
-		drif01: rif@e6f50000 {
-			compatible = "renesas,r8a7796-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f50000 0 0x64>;
-			interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 514>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x22>, <&dmac2 0x22>;
-			dma-names = "rx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 514>;
-			renesas,bonding = <&drif00>;
-			status = "disabled";
-		};
-
-		drif10: rif@e6f60000 {
-			compatible = "renesas,r8a7796-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f60000 0 0x64>;
-			interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 513>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x24>, <&dmac2 0x24>;
-			dma-names = "rx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 513>;
-			renesas,bonding = <&drif11>;
-			status = "disabled";
-		};
-
-		drif11: rif@e6f70000 {
-			compatible = "renesas,r8a7796-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f70000 0 0x64>;
-			interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 512>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x26>, <&dmac2 0x26>;
-			dma-names = "rx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 512>;
-			renesas,bonding = <&drif10>;
-			status = "disabled";
-		};
-
-		drif20: rif@e6f80000 {
-			compatible = "renesas,r8a7796-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f80000 0 0x64>;
-			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 511>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x28>, <&dmac2 0x28>;
-			dma-names = "rx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 511>;
-			renesas,bonding = <&drif21>;
-			status = "disabled";
-		};
-
-		drif21: rif@e6f90000 {
-			compatible = "renesas,r8a7796-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6f90000 0 0x64>;
-			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 510>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x2a>, <&dmac2 0x2a>;
-			dma-names = "rx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 510>;
-			renesas,bonding = <&drif20>;
-			status = "disabled";
-		};
-
-		drif30: rif@e6fa0000 {
-			compatible = "renesas,r8a7796-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6fa0000 0 0x64>;
-			interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 509>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x2c>, <&dmac2 0x2c>;
-			dma-names = "rx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 509>;
-			renesas,bonding = <&drif31>;
-			status = "disabled";
-		};
-
-		drif31: rif@e6fb0000 {
-			compatible = "renesas,r8a7796-drif",
-				     "renesas,rcar-gen3-drif";
-			reg = <0 0xe6fb0000 0 0x64>;
-			interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 508>;
-			clock-names = "fck";
-			dmas = <&dmac1 0x2e>, <&dmac2 0x2e>;
-			dma-names = "rx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 508>;
-			renesas,bonding = <&drif30>;
-			status = "disabled";
-		};
-
-		avb: ethernet@e6800000 {
-			compatible = "renesas,etheravb-r8a7796",
-				     "renesas,etheravb-rcar-gen3";
-			reg = <0 0xe6800000 0 0x800>, <0 0xe6a00000 0 0x10000>;
-			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "ch0", "ch1", "ch2", "ch3",
-					  "ch4", "ch5", "ch6", "ch7",
-					  "ch8", "ch9", "ch10", "ch11",
-					  "ch12", "ch13", "ch14", "ch15",
-					  "ch16", "ch17", "ch18", "ch19",
-					  "ch20", "ch21", "ch22", "ch23",
-					  "ch24";
-			clocks = <&cpg CPG_MOD 812>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 812>;
-			phy-mode = "rgmii";
-			iommus = <&ipmmu_ds0 16>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			status = "disabled";
-		};
-
-		hscif0: serial@e6540000 {
-			compatible = "renesas,hscif-r8a7796",
-				     "renesas,rcar-gen3-hscif",
-				     "renesas,hscif";
-			reg = <0 0xe6540000 0 0x60>;
-			interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 520>,
-				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x31>, <&dmac1 0x30>,
-			       <&dmac2 0x31>, <&dmac2 0x30>;
-			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 520>;
+			resets = <&cpg 520>;
 			status = "disabled";
 		};
 
@@ -1069,162 +675,61 @@
 			status = "disabled";
 		};
 
-		scif0: serial@e6e60000 {
-			compatible = "renesas,scif-r8a7796",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6e60000 0 64>;
-			interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 207>,
-				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x51>, <&dmac1 0x50>,
-			       <&dmac2 0x51>, <&dmac2 0x50>;
-			dma-names = "tx", "rx", "tx", "rx";
+		hsusb: usb@e6590000 {
+			compatible = "renesas,usbhs-r8a7796",
+				     "renesas,rcar-gen3-usbhs";
+			reg = <0 0xe6590000 0 0x100>;
+			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 704>;
+			dmas = <&usb_dmac0 0>, <&usb_dmac0 1>,
+			       <&usb_dmac1 0>, <&usb_dmac1 1>;
+			dma-names = "ch0", "ch1", "ch2", "ch3";
+			renesas,buswait = <11>;
+			phys = <&usb2_phy0>;
+			phy-names = "usb";
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 207>;
+			resets = <&cpg 704>;
 			status = "disabled";
 		};
 
-		scif1: serial@e6e68000 {
-			compatible = "renesas,scif-r8a7796",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6e68000 0 64>;
-			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 206>,
-				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x53>, <&dmac1 0x52>,
-			       <&dmac2 0x53>, <&dmac2 0x52>;
-			dma-names = "tx", "rx", "tx", "rx";
+		usb_dmac0: dma-controller@e65a0000 {
+			compatible = "renesas,r8a7796-usb-dmac",
+				     "renesas,usb-dmac";
+			reg = <0 0xe65a0000 0 0x100>;
+			interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1";
+			clocks = <&cpg CPG_MOD 330>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 206>;
-			status = "disabled";
+			resets = <&cpg 330>;
+			#dma-cells = <1>;
+			dma-channels = <2>;
 		};
 
-		scif2: serial@e6e88000 {
-			compatible = "renesas,scif-r8a7796",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6e88000 0 64>;
-			interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 310>,
-				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 310>;
-			status = "disabled";
-		};
-
-		scif3: serial@e6c50000 {
-			compatible = "renesas,scif-r8a7796",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6c50000 0 64>;
-			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 204>,
-				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac0 0x57>, <&dmac0 0x56>;
-			dma-names = "tx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 204>;
-			status = "disabled";
-		};
-
-		scif4: serial@e6c40000 {
-			compatible = "renesas,scif-r8a7796",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6c40000 0 64>;
-			interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 203>,
-				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac0 0x59>, <&dmac0 0x58>;
-			dma-names = "tx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 203>;
-			status = "disabled";
-		};
-
-		scif5: serial@e6f30000 {
-			compatible = "renesas,scif-r8a7796",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6f30000 0 64>;
-			interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 202>,
-				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x5b>, <&dmac1 0x5a>,
-			       <&dmac2 0x5b>, <&dmac2 0x5a>;
-			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 202>;
-			status = "disabled";
-		};
-
-		msiof0: spi@e6e90000 {
-			compatible = "renesas,msiof-r8a7796",
-				     "renesas,rcar-gen3-msiof";
-			reg = <0 0xe6e90000 0 0x0064>;
-			interrupts = <GIC_SPI 156 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 211>;
-			dmas = <&dmac1 0x41>, <&dmac1 0x40>,
-			       <&dmac2 0x41>, <&dmac2 0x40>;
-			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 211>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			status = "disabled";
-		};
-
-		msiof1: spi@e6ea0000 {
-			compatible = "renesas,msiof-r8a7796",
-				     "renesas,rcar-gen3-msiof";
-			reg = <0 0xe6ea0000 0 0x0064>;
-			interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 210>;
-			dmas = <&dmac1 0x43>, <&dmac1 0x42>,
-			       <&dmac2 0x43>, <&dmac2 0x42>;
-			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 210>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			status = "disabled";
-		};
-
-		msiof2: spi@e6c00000 {
-			compatible = "renesas,msiof-r8a7796",
-				     "renesas,rcar-gen3-msiof";
-			reg = <0 0xe6c00000 0 0x0064>;
-			interrupts = <GIC_SPI 158 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 209>;
-			dmas = <&dmac0 0x45>, <&dmac0 0x44>;
-			dma-names = "tx", "rx";
+		usb_dmac1: dma-controller@e65b0000 {
+			compatible = "renesas,r8a7796-usb-dmac",
+				     "renesas,usb-dmac";
+			reg = <0 0xe65b0000 0 0x100>;
+			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1";
+			clocks = <&cpg CPG_MOD 331>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 209>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			status = "disabled";
+			resets = <&cpg 331>;
+			#dma-cells = <1>;
+			dma-channels = <2>;
 		};
 
-		msiof3: spi@e6c10000 {
-			compatible = "renesas,msiof-r8a7796",
-				     "renesas,rcar-gen3-msiof";
-			reg = <0 0xe6c10000 0 0x0064>;
-			interrupts = <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 208>;
-			dmas = <&dmac0 0x47>, <&dmac0 0x46>;
-			dma-names = "tx", "rx";
+		usb3_phy0: usb-phy@e65ee000 {
+			compatible = "renesas,r8a7796-usb3-phy",
+				     "renesas,rcar-gen3-usb3-phy";
+			reg = <0 0xe65ee000 0 0x90>;
+			clocks = <&cpg CPG_MOD 328>, <&usb3s0_clk>,
+				 <&usb_extal_clk>;
+			clock-names = "usb3-if", "usb3s_clk", "usb_extal";
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 208>;
-			#address-cells = <1>;
-			#size-cells = <0>;
+			resets = <&cpg 328>;
+			#phy-cells = <0>;
 			status = "disabled";
 		};
 
@@ -1354,306 +859,803 @@
 			       <&ipmmu_ds1 30>, <&ipmmu_ds1 31>;
 		};
 
-		audma0: dma-controller@ec700000 {
-			compatible = "renesas,dmac-r8a7796",
-				     "renesas,rcar-dmac";
-			reg = <0 0xec700000 0 0x10000>;
-			interrupts = <GIC_SPI 350 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 320 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 321 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 322 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 323 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 324 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 332 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 333 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 334 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 335 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "error",
-					"ch0", "ch1", "ch2", "ch3",
-					"ch4", "ch5", "ch6", "ch7",
-					"ch8", "ch9", "ch10", "ch11",
-					"ch12", "ch13", "ch14", "ch15";
-			clocks = <&cpg CPG_MOD 502>;
-			clock-names = "fck";
+		ipmmu_ds0: mmu@e6740000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xe6740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 0>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 502>;
-			#dma-cells = <1>;
-			dma-channels = <16>;
-			iommus = <&ipmmu_mp 0>, <&ipmmu_mp 1>,
-			       <&ipmmu_mp 2>, <&ipmmu_mp 3>,
-			       <&ipmmu_mp 4>, <&ipmmu_mp 5>,
-			       <&ipmmu_mp 6>, <&ipmmu_mp 7>,
-			       <&ipmmu_mp 8>, <&ipmmu_mp 9>,
-			       <&ipmmu_mp 10>, <&ipmmu_mp 11>,
-			       <&ipmmu_mp 12>, <&ipmmu_mp 13>,
-			       <&ipmmu_mp 14>, <&ipmmu_mp 15>;
+			#iommu-cells = <1>;
 		};
 
-		audma1: dma-controller@ec720000 {
-			compatible = "renesas,dmac-r8a7796",
-				     "renesas,rcar-dmac";
-			reg = <0 0xec720000 0 0x10000>;
-			interrupts = <GIC_SPI 351 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 336 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 338 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 339 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 340 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 341 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 342 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 343 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 344 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 345 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 346 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 347 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 348 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 349 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 382 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 383 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "error",
-					"ch0", "ch1", "ch2", "ch3",
-					"ch4", "ch5", "ch6", "ch7",
-					"ch8", "ch9", "ch10", "ch11",
-					"ch12", "ch13", "ch14", "ch15";
-			clocks = <&cpg CPG_MOD 501>;
-			clock-names = "fck";
+		ipmmu_ds1: mmu@e7740000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xe7740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 1>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 501>;
-			#dma-cells = <1>;
-			dma-channels = <16>;
-			iommus = <&ipmmu_mp 16>, <&ipmmu_mp 17>,
-			       <&ipmmu_mp 18>, <&ipmmu_mp 19>,
-			       <&ipmmu_mp 20>, <&ipmmu_mp 21>,
-			       <&ipmmu_mp 22>, <&ipmmu_mp 23>,
-			       <&ipmmu_mp 24>, <&ipmmu_mp 25>,
-			       <&ipmmu_mp 26>, <&ipmmu_mp 27>,
-			       <&ipmmu_mp 28>, <&ipmmu_mp 29>,
-			       <&ipmmu_mp 30>, <&ipmmu_mp 31>;
+			#iommu-cells = <1>;
 		};
 
-		usb_dmac0: dma-controller@e65a0000 {
-			compatible = "renesas,r8a7796-usb-dmac",
-				     "renesas,usb-dmac";
-			reg = <0 0xe65a0000 0 0x100>;
-			interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "ch0", "ch1";
-			clocks = <&cpg CPG_MOD 330>;
+		ipmmu_hc: mmu@e6570000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xe6570000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 2>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 330>;
-			#dma-cells = <1>;
-			dma-channels = <2>;
+			#iommu-cells = <1>;
 		};
 
-		usb_dmac1: dma-controller@e65b0000 {
-			compatible = "renesas,r8a7796-usb-dmac",
-				     "renesas,usb-dmac";
-			reg = <0 0xe65b0000 0 0x100>;
-			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "ch0", "ch1";
-			clocks = <&cpg CPG_MOD 331>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 331>;
-			#dma-cells = <1>;
-			dma-channels = <2>;
+		ipmmu_ir: mmu@ff8b0000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xff8b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 3>;
+			power-domains = <&sysc R8A7796_PD_A3IR>;
+			#iommu-cells = <1>;
 		};
 
-		hsusb: usb@e6590000 {
-			compatible = "renesas,usbhs-r8a7796",
-				     "renesas,rcar-gen3-usbhs";
-			reg = <0 0xe6590000 0 0x100>;
-			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 704>;
-			dmas = <&usb_dmac0 0>, <&usb_dmac0 1>,
-			       <&usb_dmac1 0>, <&usb_dmac1 1>;
-			dma-names = "ch0", "ch1", "ch2", "ch3";
-			renesas,buswait = <11>;
-			phys = <&usb2_phy0>;
-			phy-names = "usb";
+		ipmmu_mm: mmu@e67b0000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xe67b0000 0 0x1000>;
+			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_mp: mmu@ec670000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xec670000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 4>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_pv0: mmu@fd800000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xfd800000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 5>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_pv1: mmu@fd950000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xfd950000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 6>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_rt: mmu@ffc80000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xffc80000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 7>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vc0: mmu@fe6b0000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xfe6b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 8>;
+			power-domains = <&sysc R8A7796_PD_A3VC>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vi0: mmu@febd0000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xfebd0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 9>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		avb: ethernet@e6800000 {
+			compatible = "renesas,etheravb-r8a7796",
+				     "renesas,etheravb-rcar-gen3";
+			reg = <0 0xe6800000 0 0x800>, <0 0xe6a00000 0 0x10000>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14", "ch15",
+					  "ch16", "ch17", "ch18", "ch19",
+					  "ch20", "ch21", "ch22", "ch23",
+					  "ch24";
+			clocks = <&cpg CPG_MOD 812>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 812>;
+			phy-mode = "rgmii";
+			iommus = <&ipmmu_ds0 16>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		can0: can@e6c30000 {
+			compatible = "renesas,can-r8a7796",
+				     "renesas,rcar-gen3-can";
+			reg = <0 0xe6c30000 0 0x1000>;
+			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 916>,
+			       <&cpg CPG_CORE R8A7796_CLK_CANFD>,
+			       <&can_clk>;
+			clock-names = "clkp1", "clkp2", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A7796_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 916>;
+			status = "disabled";
+		};
+
+		can1: can@e6c38000 {
+			compatible = "renesas,can-r8a7796",
+				     "renesas,rcar-gen3-can";
+			reg = <0 0xe6c38000 0 0x1000>;
+			interrupts = <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 915>,
+			       <&cpg CPG_CORE R8A7796_CLK_CANFD>,
+			       <&can_clk>;
+			clock-names = "clkp1", "clkp2", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A7796_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 915>;
+			status = "disabled";
+		};
+
+		canfd: can@e66c0000 {
+			compatible = "renesas,r8a7796-canfd",
+				     "renesas,rcar-gen3-canfd";
+			reg = <0 0xe66c0000 0 0x8000>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>,
+				   <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 914>,
+			       <&cpg CPG_CORE R8A7796_CLK_CANFD>,
+			       <&can_clk>;
+			clock-names = "fck", "canfd", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A7796_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 914>;
+			status = "disabled";
+
+			channel0 {
+				status = "disabled";
+			};
+
+			channel1 {
+				status = "disabled";
+			};
+		};
+
+		pwm0: pwm@e6e30000 {
+			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
+			reg = <0 0xe6e30000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm1: pwm@e6e31000 {
+			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
+			reg = <0 0xe6e31000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm2: pwm@e6e32000 {
+			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
+			reg = <0 0xe6e32000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm3: pwm@e6e33000 {
+			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
+			reg = <0 0xe6e33000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm4: pwm@e6e34000 {
+			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
+			reg = <0 0xe6e34000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm5: pwm@e6e35000 {
+			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
+			reg = <0 0xe6e35000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm6: pwm@e6e36000 {
+			compatible = "renesas,pwm-r8a7796", "renesas,pwm-rcar";
+			reg = <0 0xe6e36000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif0: serial@e6e60000 {
+			compatible = "renesas,scif-r8a7796",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6e60000 0 64>;
+			interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 207>,
+				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac1 0x51>, <&dmac1 0x50>,
+			       <&dmac2 0x51>, <&dmac2 0x50>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 207>;
+			status = "disabled";
+		};
+
+		scif1: serial@e6e68000 {
+			compatible = "renesas,scif-r8a7796",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6e68000 0 64>;
+			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 206>,
+				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac1 0x53>, <&dmac1 0x52>,
+			       <&dmac2 0x53>, <&dmac2 0x52>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 206>;
+			status = "disabled";
+		};
+
+		scif2: serial@e6e88000 {
+			compatible = "renesas,scif-r8a7796",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6e88000 0 64>;
+			interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 310>,
+				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 310>;
+			status = "disabled";
+		};
+
+		scif3: serial@e6c50000 {
+			compatible = "renesas,scif-r8a7796",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6c50000 0 64>;
+			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 204>,
+				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x57>, <&dmac0 0x56>;
+			dma-names = "tx", "rx";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 204>;
+			status = "disabled";
+		};
+
+		scif4: serial@e6c40000 {
+			compatible = "renesas,scif-r8a7796",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6c40000 0 64>;
+			interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 203>,
+				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x59>, <&dmac0 0x58>;
+			dma-names = "tx", "rx";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 203>;
+			status = "disabled";
+		};
+
+		scif5: serial@e6f30000 {
+			compatible = "renesas,scif-r8a7796",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6f30000 0 64>;
+			interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 202>,
+				 <&cpg CPG_CORE R8A7796_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac1 0x5b>, <&dmac1 0x5a>,
+			       <&dmac2 0x5b>, <&dmac2 0x5a>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 202>;
+			status = "disabled";
+		};
+
+		msiof0: spi@e6e90000 {
+			compatible = "renesas,msiof-r8a7796",
+				     "renesas,rcar-gen3-msiof";
+			reg = <0 0xe6e90000 0 0x0064>;
+			interrupts = <GIC_SPI 156 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 211>;
+			dmas = <&dmac1 0x41>, <&dmac1 0x40>,
+			       <&dmac2 0x41>, <&dmac2 0x40>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 211>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof1: spi@e6ea0000 {
+			compatible = "renesas,msiof-r8a7796",
+				     "renesas,rcar-gen3-msiof";
+			reg = <0 0xe6ea0000 0 0x0064>;
+			interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 210>;
+			dmas = <&dmac1 0x43>, <&dmac1 0x42>,
+			       <&dmac2 0x43>, <&dmac2 0x42>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 210>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof2: spi@e6c00000 {
+			compatible = "renesas,msiof-r8a7796",
+				     "renesas,rcar-gen3-msiof";
+			reg = <0 0xe6c00000 0 0x0064>;
+			interrupts = <GIC_SPI 158 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 209>;
+			dmas = <&dmac0 0x45>, <&dmac0 0x44>;
+			dma-names = "tx", "rx";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 209>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof3: spi@e6c10000 {
+			compatible = "renesas,msiof-r8a7796",
+				     "renesas,rcar-gen3-msiof";
+			reg = <0 0xe6c10000 0 0x0064>;
+			interrupts = <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 208>;
+			dmas = <&dmac0 0x47>, <&dmac0 0x46>;
+			dma-names = "tx", "rx";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 208>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		vin0: video@e6ef0000 {
+			compatible = "renesas,vin-r8a7796";
+			reg = <0 0xe6ef0000 0 0x1000>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 811>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 811>;
+			renesas,id = <0>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin0csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin0>;
+					};
+					vin0csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin0>;
+					};
+				};
+			};
+		};
+
+		vin1: video@e6ef1000 {
+			compatible = "renesas,vin-r8a7796";
+			reg = <0 0xe6ef1000 0 0x1000>;
+			interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 810>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 704>;
+			resets = <&cpg 810>;
+			renesas,id = <1>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin1csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin1>;
+					};
+					vin1csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin1>;
+					};
+				};
+			};
 		};
 
-		usb3_phy0: usb-phy@e65ee000 {
-			compatible = "renesas,r8a7796-usb3-phy",
-				     "renesas,rcar-gen3-usb3-phy";
-			reg = <0 0xe65ee000 0 0x90>;
-			clocks = <&cpg CPG_MOD 328>, <&usb3s0_clk>,
-				 <&usb_extal_clk>;
-			clock-names = "usb3-if", "usb3s_clk", "usb_extal";
+		vin2: video@e6ef2000 {
+			compatible = "renesas,vin-r8a7796";
+			reg = <0 0xe6ef2000 0 0x1000>;
+			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 809>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 328>;
-			#phy-cells = <0>;
+			resets = <&cpg 809>;
+			renesas,id = <2>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin2csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin2>;
+					};
+					vin2csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin2>;
+					};
+				};
+			};
 		};
 
-		xhci0: usb@ee000000 {
-			compatible = "renesas,xhci-r8a7796",
-				     "renesas,rcar-gen3-xhci";
-			reg = <0 0xee000000 0 0xc00>;
-			interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 328>;
+		vin3: video@e6ef3000 {
+			compatible = "renesas,vin-r8a7796";
+			reg = <0 0xe6ef3000 0 0x1000>;
+			interrupts = <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 808>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 328>;
+			resets = <&cpg 808>;
+			renesas,id = <3>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin3csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin3>;
+					};
+					vin3csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin3>;
+					};
+				};
+			};
 		};
 
-		usb3_peri0: usb@ee020000 {
-			compatible = "renesas,r8a7796-usb3-peri",
-				     "renesas,rcar-gen3-usb3-peri";
-			reg = <0 0xee020000 0 0x400>;
-			interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 328>;
+		vin4: video@e6ef4000 {
+			compatible = "renesas,vin-r8a7796";
+			reg = <0 0xe6ef4000 0 0x1000>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 807>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 328>;
+			resets = <&cpg 807>;
+			renesas,id = <4>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin4csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin4>;
+					};
+					vin4csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin4>;
+					};
+				};
+			};
 		};
 
-		ohci0: usb@ee080000 {
-			compatible = "generic-ohci";
-			reg = <0 0xee080000 0 0x100>;
-			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 703>;
-			phys = <&usb2_phy0>;
-			phy-names = "usb";
+		vin5: video@e6ef5000 {
+			compatible = "renesas,vin-r8a7796";
+			reg = <0 0xe6ef5000 0 0x1000>;
+			interrupts = <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 806>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 703>;
+			resets = <&cpg 806>;
+			renesas,id = <5>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin5csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin5>;
+					};
+					vin5csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin5>;
+					};
+				};
+			};
 		};
 
-		ehci0: usb@ee080100 {
-			compatible = "generic-ehci";
-			reg = <0 0xee080100 0 0x100>;
-			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 703>;
-			phys = <&usb2_phy0>;
-			phy-names = "usb";
-			companion= <&ohci0>;
+		vin6: video@e6ef6000 {
+			compatible = "renesas,vin-r8a7796";
+			reg = <0 0xe6ef6000 0 0x1000>;
+			interrupts = <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 805>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 703>;
+			resets = <&cpg 805>;
+			renesas,id = <6>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin6csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin6>;
+					};
+					vin6csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin6>;
+					};
+				};
+			};
+		};
+
+		vin7: video@e6ef7000 {
+			compatible = "renesas,vin-r8a7796";
+			reg = <0 0xe6ef7000 0 0x1000>;
+			interrupts = <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 804>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 804>;
+			renesas,id = <7>;
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin7csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin7>;
+					};
+					vin7csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin7>;
+					};
+				};
+			};
 		};
 
-		usb2_phy0: usb-phy@ee080200 {
-			compatible = "renesas,usb2-phy-r8a7796",
-				     "renesas,rcar-gen3-usb2-phy";
-			reg = <0 0xee080200 0 0x700>;
-			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 703>;
+		drif00: rif@e6f40000 {
+			compatible = "renesas,r8a7796-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f40000 0 0x64>;
+			interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 515>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x20>, <&dmac2 0x20>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 703>;
-			#phy-cells = <0>;
+			resets = <&cpg 515>;
+			renesas,bonding = <&drif01>;
 			status = "disabled";
 		};
 
-		ohci1: usb@ee0a0000 {
-			compatible = "generic-ohci";
-			reg = <0 0xee0a0000 0 0x100>;
-			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 702>;
-			phys = <&usb2_phy1>;
-			phy-names = "usb";
+		drif01: rif@e6f50000 {
+			compatible = "renesas,r8a7796-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f50000 0 0x64>;
+			interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 514>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x22>, <&dmac2 0x22>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 702>;
+			resets = <&cpg 514>;
+			renesas,bonding = <&drif00>;
 			status = "disabled";
 		};
 
-		ehci1: usb@ee0a0100 {
-			compatible = "generic-ehci";
-			reg = <0 0xee0a0100 0 0x100>;
-			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 702>;
-			phys = <&usb2_phy1>;
-			phy-names = "usb";
-			companion= <&ohci1>;
+		drif10: rif@e6f60000 {
+			compatible = "renesas,r8a7796-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f60000 0 0x64>;
+			interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 513>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x24>, <&dmac2 0x24>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 702>;
+			resets = <&cpg 513>;
+			renesas,bonding = <&drif11>;
 			status = "disabled";
 		};
 
-		usb2_phy1: usb-phy@ee0a0200 {
-			compatible = "renesas,usb2-phy-r8a7796",
-				     "renesas,rcar-gen3-usb2-phy";
-			reg = <0 0xee0a0200 0 0x700>;
-			clocks = <&cpg CPG_MOD 702>;
+		drif11: rif@e6f70000 {
+			compatible = "renesas,r8a7796-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f70000 0 0x64>;
+			interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 512>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x26>, <&dmac2 0x26>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 702>;
-			#phy-cells = <0>;
+			resets = <&cpg 512>;
+			renesas,bonding = <&drif10>;
 			status = "disabled";
 		};
 
-		sdhi0: sd@ee100000 {
-			compatible = "renesas,sdhi-r8a7796",
-				     "renesas,rcar-gen3-sdhi";
-			reg = <0 0xee100000 0 0x2000>;
-			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 314>;
-			max-frequency = <200000000>;
+		drif20: rif@e6f80000 {
+			compatible = "renesas,r8a7796-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f80000 0 0x64>;
+			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 511>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x28>, <&dmac2 0x28>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 314>;
+			resets = <&cpg 511>;
+			renesas,bonding = <&drif21>;
 			status = "disabled";
 		};
 
-		sdhi1: sd@ee120000 {
-			compatible = "renesas,sdhi-r8a7796",
-				     "renesas,rcar-gen3-sdhi";
-			reg = <0 0xee120000 0 0x2000>;
-			interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 313>;
-			max-frequency = <200000000>;
+		drif21: rif@e6f90000 {
+			compatible = "renesas,r8a7796-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6f90000 0 0x64>;
+			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 510>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x2a>, <&dmac2 0x2a>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 313>;
+			resets = <&cpg 510>;
+			renesas,bonding = <&drif20>;
 			status = "disabled";
 		};
 
-		sdhi2: sd@ee140000 {
-			compatible = "renesas,sdhi-r8a7796",
-				     "renesas,rcar-gen3-sdhi";
-			reg = <0 0xee140000 0 0x2000>;
-			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 312>;
-			max-frequency = <200000000>;
+		drif30: rif@e6fa0000 {
+			compatible = "renesas,r8a7796-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6fa0000 0 0x64>;
+			interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 509>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x2c>, <&dmac2 0x2c>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 312>;
+			resets = <&cpg 509>;
+			renesas,bonding = <&drif31>;
 			status = "disabled";
 		};
 
-		sdhi3: sd@ee160000 {
-			compatible = "renesas,sdhi-r8a7796",
-				     "renesas,rcar-gen3-sdhi";
-			reg = <0 0xee160000 0 0x2000>;
-			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 311>;
-			max-frequency = <200000000>;
+		drif31: rif@e6fb0000 {
+			compatible = "renesas,r8a7796-drif",
+				     "renesas,rcar-gen3-drif";
+			reg = <0 0xe6fb0000 0 0x64>;
+			interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 508>;
+			clock-names = "fck";
+			dmas = <&dmac1 0x2e>, <&dmac2 0x2e>;
+			dma-names = "rx", "rx";
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 311>;
+			resets = <&cpg 508>;
+			renesas,bonding = <&drif30>;
 			status = "disabled";
 		};
 
-		tsc: thermal@e6198000 {
-			compatible = "renesas,r8a7796-thermal";
-			reg = <0 0xe6198000 0 0x100>,
-			      <0 0xe61a0000 0 0x100>,
-			      <0 0xe61a8000 0 0x100>;
-			interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 522>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 522>;
-			#thermal-sensor-cells = <1>;
-			status = "okay";
-		};
-
 		rcar_sound: sound@ec500000 {
 			/*
 			 * #sound-dai-cells is required
@@ -1848,6 +1850,261 @@
 					dma-names = "rx", "tx", "rxu", "txu";
 				};
 			};
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				port@0 {
+					reg = <0>;
+				};
+				port@1 {
+					reg = <1>;
+				};
+			};
+		};
+
+		audma0: dma-controller@ec700000 {
+			compatible = "renesas,dmac-r8a7796",
+				     "renesas,rcar-dmac";
+			reg = <0 0xec700000 0 0x10000>;
+			interrupts = <GIC_SPI 350 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 320 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 321 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 322 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 323 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 324 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 332 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 333 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 334 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 335 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 502>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 502>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+			iommus = <&ipmmu_mp 0>, <&ipmmu_mp 1>,
+			       <&ipmmu_mp 2>, <&ipmmu_mp 3>,
+			       <&ipmmu_mp 4>, <&ipmmu_mp 5>,
+			       <&ipmmu_mp 6>, <&ipmmu_mp 7>,
+			       <&ipmmu_mp 8>, <&ipmmu_mp 9>,
+			       <&ipmmu_mp 10>, <&ipmmu_mp 11>,
+			       <&ipmmu_mp 12>, <&ipmmu_mp 13>,
+			       <&ipmmu_mp 14>, <&ipmmu_mp 15>;
+		};
+
+		audma1: dma-controller@ec720000 {
+			compatible = "renesas,dmac-r8a7796",
+				     "renesas,rcar-dmac";
+			reg = <0 0xec720000 0 0x10000>;
+			interrupts = <GIC_SPI 351 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 336 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 338 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 339 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 340 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 341 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 342 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 343 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 344 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 345 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 346 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 347 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 348 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 349 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 382 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 383 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 501>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 501>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+			iommus = <&ipmmu_mp 16>, <&ipmmu_mp 17>,
+			       <&ipmmu_mp 18>, <&ipmmu_mp 19>,
+			       <&ipmmu_mp 20>, <&ipmmu_mp 21>,
+			       <&ipmmu_mp 22>, <&ipmmu_mp 23>,
+			       <&ipmmu_mp 24>, <&ipmmu_mp 25>,
+			       <&ipmmu_mp 26>, <&ipmmu_mp 27>,
+			       <&ipmmu_mp 28>, <&ipmmu_mp 29>,
+			       <&ipmmu_mp 30>, <&ipmmu_mp 31>;
+		};
+
+		xhci0: usb@ee000000 {
+			compatible = "renesas,xhci-r8a7796",
+				     "renesas,rcar-gen3-xhci";
+			reg = <0 0xee000000 0 0xc00>;
+			interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 328>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 328>;
+			status = "disabled";
+		};
+
+		usb3_peri0: usb@ee020000 {
+			compatible = "renesas,r8a7796-usb3-peri",
+				     "renesas,rcar-gen3-usb3-peri";
+			reg = <0 0xee020000 0 0x400>;
+			interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 328>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 328>;
+			status = "disabled";
+		};
+
+		ohci0: usb@ee080000 {
+			compatible = "generic-ohci";
+			reg = <0 0xee080000 0 0x100>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 703>;
+			phys = <&usb2_phy0>;
+			phy-names = "usb";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 703>;
+			status = "disabled";
+		};
+
+		ohci1: usb@ee0a0000 {
+			compatible = "generic-ohci";
+			reg = <0 0xee0a0000 0 0x100>;
+			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 702>;
+			phys = <&usb2_phy1>;
+			phy-names = "usb";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 702>;
+			status = "disabled";
+		};
+
+		ehci0: usb@ee080100 {
+			compatible = "generic-ehci";
+			reg = <0 0xee080100 0 0x100>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 703>;
+			phys = <&usb2_phy0>;
+			phy-names = "usb";
+			companion= <&ohci0>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 703>;
+			status = "disabled";
+		};
+
+		ehci1: usb@ee0a0100 {
+			compatible = "generic-ehci";
+			reg = <0 0xee0a0100 0 0x100>;
+			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 702>;
+			phys = <&usb2_phy1>;
+			phy-names = "usb";
+			companion= <&ohci1>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 702>;
+			status = "disabled";
+		};
+
+		usb2_phy0: usb-phy@ee080200 {
+			compatible = "renesas,usb2-phy-r8a7796",
+				     "renesas,rcar-gen3-usb2-phy";
+			reg = <0 0xee080200 0 0x700>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 703>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 703>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		usb2_phy1: usb-phy@ee0a0200 {
+			compatible = "renesas,usb2-phy-r8a7796",
+				     "renesas,rcar-gen3-usb2-phy";
+			reg = <0 0xee0a0200 0 0x700>;
+			clocks = <&cpg CPG_MOD 702>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 702>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		sdhi0: sd@ee100000 {
+			compatible = "renesas,sdhi-r8a7796",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee100000 0 0x2000>;
+			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 314>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 314>;
+			status = "disabled";
+		};
+
+		sdhi1: sd@ee120000 {
+			compatible = "renesas,sdhi-r8a7796",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee120000 0 0x2000>;
+			interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 313>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 313>;
+			status = "disabled";
+		};
+
+		sdhi2: sd@ee140000 {
+			compatible = "renesas,sdhi-r8a7796",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee140000 0 0x2000>;
+			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 312>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 312>;
+			status = "disabled";
+		};
+
+		sdhi3: sd@ee160000 {
+			compatible = "renesas,sdhi-r8a7796",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee160000 0 0x2000>;
+			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 311>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 311>;
+			status = "disabled";
+		};
+
+		gic: interrupt-controller@f1010000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0x0 0xf1010000 0 0x1000>,
+			      <0x0 0xf1020000 0 0x20000>,
+			      <0x0 0xf1040000 0 0x20000>,
+			      <0x0 0xf1060000 0 0x20000>;
+			interrupts = <GIC_PPI 9
+					(GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_HIGH)>;
+			clocks = <&cpg CPG_MOD 408>;
+			clock-names = "clk";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 408>;
 		};
 
 		pciec0: pcie@fe000000 {
@@ -1860,6 +2117,26 @@
 			/* placeholder */
 		};
 
+		imr-lx4@fe860000 {
+			compatible = "renesas,r8a7796-imr-lx4",
+				     "renesas,imr-lx4";
+			reg = <0 0xfe860000 0 0x2000>;
+			interrupts = <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 823>;
+			power-domains = <&sysc R8A7796_PD_A3VC>;
+			resets = <&cpg 823>;
+		};
+
+		imr-lx4@fe870000 {
+			compatible = "renesas,r8a7796-imr-lx4",
+				     "renesas,imr-lx4";
+			reg = <0 0xfe870000 0 0x2000>;
+			interrupts = <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 822>;
+			power-domains = <&sysc R8A7796_PD_A3VC>;
+			resets = <&cpg 822>;
+		};
+
 		fdp1@fe940000 {
 			compatible = "renesas,fdp1";
 			reg = <0 0xfe940000 0 0x2400>;
@@ -1878,17 +2155,6 @@
 			resets = <&cpg 615>;
 		};
 
-		vspb: vsp@fe960000 {
-			compatible = "renesas,vsp2";
-			reg = <0 0xfe960000 0 0x8000>;
-			interrupts = <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 626>;
-			power-domains = <&sysc R8A7796_PD_A3VC>;
-			resets = <&cpg 626>;
-
-			renesas,fcp = <&fcpvb0>;
-		};
-
 		fcpvb0: fcp@fe96f000 {
 			compatible = "renesas,fcpv";
 			reg = <0 0xfe96f000 0 0x200>;
@@ -1897,17 +2163,6 @@
 			resets = <&cpg 607>;
 		};
 
-		vspi0: vsp@fe9a0000 {
-			compatible = "renesas,vsp2";
-			reg = <0 0xfe9a0000 0 0x8000>;
-			interrupts = <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 631>;
-			power-domains = <&sysc R8A7796_PD_A3VC>;
-			resets = <&cpg 631>;
-
-			renesas,fcp = <&fcpvi0>;
-		};
-
 		fcpvi0: fcp@fe9af000 {
 			compatible = "renesas,fcpv";
 			reg = <0 0xfe9af000 0 0x200>;
@@ -1917,6 +2172,44 @@
 			iommus = <&ipmmu_vc0 19>;
 		};
 
+		fcpvd0: fcp@fea27000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfea27000 0 0x200>;
+			clocks = <&cpg CPG_MOD 603>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 603>;
+			iommus = <&ipmmu_vi0 8>;
+		};
+
+		fcpvd1: fcp@fea2f000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfea2f000 0 0x200>;
+			clocks = <&cpg CPG_MOD 602>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 602>;
+			iommus = <&ipmmu_vi0 9>;
+		};
+
+		fcpvd2: fcp@fea37000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfea37000 0 0x200>;
+			clocks = <&cpg CPG_MOD 601>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 601>;
+			iommus = <&ipmmu_vi0 10>;
+		};
+
+		vspb: vsp@fe960000 {
+			compatible = "renesas,vsp2";
+			reg = <0 0xfe960000 0 0x8000>;
+			interrupts = <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 626>;
+			power-domains = <&sysc R8A7796_PD_A3VC>;
+			resets = <&cpg 626>;
+
+			renesas,fcp = <&fcpvb0>;
+		};
+
 		vspd0: vsp@fea20000 {
 			compatible = "renesas,vsp2";
 			reg = <0 0xfea20000 0 0x8000>;
@@ -1928,15 +2221,6 @@
 			renesas,fcp = <&fcpvd0>;
 		};
 
-		fcpvd0: fcp@fea27000 {
-			compatible = "renesas,fcpv";
-			reg = <0 0xfea27000 0 0x200>;
-			clocks = <&cpg CPG_MOD 603>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 603>;
-			iommus = <&ipmmu_vi0 8>;
-		};
-
 		vspd1: vsp@fea28000 {
 			compatible = "renesas,vsp2";
 			reg = <0 0xfea28000 0 0x8000>;
@@ -1948,15 +2232,6 @@
 			renesas,fcp = <&fcpvd1>;
 		};
 
-		fcpvd1: fcp@fea2f000 {
-			compatible = "renesas,fcpv";
-			reg = <0 0xfea2f000 0 0x200>;
-			clocks = <&cpg CPG_MOD 602>;
-			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 602>;
-			iommus = <&ipmmu_vi0 9>;
-		};
-
 		vspd2: vsp@fea30000 {
 			compatible = "renesas,vsp2";
 			reg = <0 0xfea30000 0 0x8000>;
@@ -1968,13 +2243,126 @@
 			renesas,fcp = <&fcpvd2>;
 		};
 
-		fcpvd2: fcp@fea37000 {
-			compatible = "renesas,fcpv";
-			reg = <0 0xfea37000 0 0x200>;
-			clocks = <&cpg CPG_MOD 601>;
+		vspi0: vsp@fe9a0000 {
+			compatible = "renesas,vsp2";
+			reg = <0 0xfe9a0000 0 0x8000>;
+			interrupts = <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 631>;
+			power-domains = <&sysc R8A7796_PD_A3VC>;
+			resets = <&cpg 631>;
+
+			renesas,fcp = <&fcpvi0>;
+		};
+
+		csi20: csi2@fea80000 {
+			compatible = "renesas,r8a7796-csi2";
+			reg = <0 0xfea80000 0 0x10000>;
+			interrupts = <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 714>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
-			resets = <&cpg 601>;
-			iommus = <&ipmmu_vi0 10>;
+			resets = <&cpg 714>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					csi20vin0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&vin0csi20>;
+					};
+					csi20vin1: endpoint@1 {
+						reg = <1>;
+						remote-endpoint = <&vin1csi20>;
+					};
+					csi20vin2: endpoint@2 {
+						reg = <2>;
+						remote-endpoint = <&vin2csi20>;
+					};
+					csi20vin3: endpoint@3 {
+						reg = <3>;
+						remote-endpoint = <&vin3csi20>;
+					};
+					csi20vin4: endpoint@4 {
+						reg = <4>;
+						remote-endpoint = <&vin4csi20>;
+					};
+					csi20vin5: endpoint@5 {
+						reg = <5>;
+						remote-endpoint = <&vin5csi20>;
+					};
+					csi20vin6: endpoint@6 {
+						reg = <6>;
+						remote-endpoint = <&vin6csi20>;
+					};
+					csi20vin7: endpoint@7 {
+						reg = <7>;
+						remote-endpoint = <&vin7csi20>;
+					};
+				};
+			};
+		};
+
+		csi40: csi2@feaa0000 {
+			compatible = "renesas,r8a7796-csi2";
+			reg = <0 0xfeaa0000 0 0x10000>;
+			interrupts = <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 716>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 716>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					csi40vin0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&vin0csi40>;
+					};
+					csi40vin1: endpoint@1 {
+						reg = <1>;
+						remote-endpoint = <&vin1csi40>;
+					};
+					csi40vin2: endpoint@2 {
+						reg = <2>;
+						remote-endpoint = <&vin2csi40>;
+					};
+					csi40vin3: endpoint@3 {
+						reg = <3>;
+						remote-endpoint = <&vin3csi40>;
+					};
+					csi40vin4: endpoint@4 {
+						reg = <4>;
+						remote-endpoint = <&vin4csi40>;
+					};
+					csi40vin5: endpoint@5 {
+						reg = <5>;
+						remote-endpoint = <&vin5csi40>;
+					};
+					csi40vin6: endpoint@6 {
+						reg = <6>;
+						remote-endpoint = <&vin6csi40>;
+					};
+					csi40vin7: endpoint@7 {
+						reg = <7>;
+						remote-endpoint = <&vin7csi40>;
+					};
+				};
+
+			};
 		};
 
 		hdmi0: hdmi@fead0000 {
@@ -1999,6 +2387,10 @@
 				port@1 {
 					reg = <1>;
 				};
+				port@2 {
+					/* HDMI sound */
+					reg = <2>;
+				};
 			};
 		};
 
@@ -2042,35 +2434,12 @@
 			};
 		};
 
-		imr-lx4@fe860000 {
-			compatible = "renesas,r8a7796-imr-lx4",
-				     "renesas,imr-lx4";
-			reg = <0 0xfe860000 0 0x2000>;
-			interrupts = <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 823>;
-			power-domains = <&sysc R8A7796_PD_A3VC>;
-			resets = <&cpg 823>;
-		};
-
-		imr-lx4@fe870000 {
-			compatible = "renesas,r8a7796-imr-lx4",
-				     "renesas,imr-lx4";
-			reg = <0 0xfe870000 0 0x2000>;
-			interrupts = <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 822>;
-			power-domains = <&sysc R8A7796_PD_A3VC>;
-			resets = <&cpg 822>;
+		prr: chipid@fff00044 {
+			compatible = "renesas,prr";
+			reg = <0 0xfff00044 0 4>;
 		};
 	};
 
-	timer {
-		compatible = "arm,armv8-timer";
-		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>;
-	};
-
 	thermal-zones {
 		sensor_thermal1: sensor-thermal1 {
 			polling-delay-passive = <250>;
@@ -2080,12 +2449,12 @@
 			trips {
 				sensor1_passive: sensor1-passive {
 					temperature = <95000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "passive";
 				};
 				sensor1_crit: sensor1-crit {
 					temperature = <120000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "critical";
 				};
 			};
@@ -2106,12 +2475,12 @@
 			trips {
 				sensor2_passive: sensor2-passive {
 					temperature = <95000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "passive";
 				};
 				sensor2_crit: sensor2-crit {
 					temperature = <120000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "critical";
 				};
 			};
@@ -2132,12 +2501,12 @@
 			trips {
 				sensor3_passive: sensor3-passive {
 					temperature = <95000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "passive";
 				};
 				sensor3_crit: sensor3-crit {
 					temperature = <120000>;
-					hysteresis = <2000>;
+					hysteresis = <1000>;
 					type = "critical";
 				};
 			};
@@ -2151,6 +2520,14 @@
 		};
 	};
 
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>;
+	};
+
 	/* External USB clocks - can be overridden by the board */
 	usb3s0_clk: usb3s0 {
 		compatible = "fixed-clock";
diff --git a/arch/arm64/boot/dts/renesas/r8a77965-salvator-x.dts b/arch/arm64/boot/dts/renesas/r8a77965-salvator-x.dts
index 75d890d91df9..340a3c72b65a 100644
--- a/arch/arm64/boot/dts/renesas/r8a77965-salvator-x.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77965-salvator-x.dts
@@ -19,3 +19,31 @@
 		reg = <0x0 0x48000000 0x0 0x78000000>;
 	};
 };
+
+&du {
+	clocks = <&cpg CPG_MOD 724>,
+		 <&cpg CPG_MOD 723>,
+		 <&cpg CPG_MOD 721>,
+		 <&versaclock5 1>,
+		 <&x21_clk>,
+		 <&versaclock5 2>;
+	clock-names = "du.0", "du.1", "du.3",
+		      "dclkin.0", "dclkin.1", "dclkin.3";
+};
+
+&hdmi0 {
+	status = "okay";
+
+	ports {
+		port@1 {
+			reg = <1>;
+			rcar_dw_hdmi0_out: endpoint {
+				remote-endpoint = <&hdmi0_con>;
+			};
+		};
+	};
+};
+
+&hdmi0_con {
+	remote-endpoint = <&rcar_dw_hdmi0_out>;
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77965-salvator-xs.dts b/arch/arm64/boot/dts/renesas/r8a77965-salvator-xs.dts
index a83a00deed9e..9de4e3db1621 100644
--- a/arch/arm64/boot/dts/renesas/r8a77965-salvator-xs.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77965-salvator-xs.dts
@@ -19,3 +19,31 @@
 		reg = <0x0 0x48000000 0x0 0x78000000>;
 	};
 };
+
+&du {
+	clocks = <&cpg CPG_MOD 724>,
+		 <&cpg CPG_MOD 723>,
+		 <&cpg CPG_MOD 721>,
+		 <&versaclock6 1>,
+		 <&x21_clk>,
+		 <&versaclock6 2>;
+	clock-names = "du.0", "du.1", "du.3",
+		      "dclkin.0", "dclkin.1", "dclkin.3";
+};
+
+&hdmi0 {
+	status = "okay";
+
+	ports {
+		port@1 {
+			reg = <1>;
+			rcar_dw_hdmi0_out: endpoint {
+				remote-endpoint = <&hdmi0_con>;
+			};
+		};
+	};
+};
+
+&hdmi0_con {
+	remote-endpoint = <&rcar_dw_hdmi0_out>;
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77965.dtsi b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
index f0871fcdd984..486aecacb22a 100644
--- a/arch/arm64/boot/dts/renesas/r8a77965.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
@@ -8,8 +8,9 @@
  * Copyright (C) 2016 Renesas Electronics Corp.
  */
 
-#include <dt-bindings/clock/renesas-cpg-mssr.h>
+#include <dt-bindings/clock/r8a77965-cpg-mssr.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/power/r8a77965-sysc.h>
 
 #define CPG_AUDIO_CLK_I		10
 
@@ -19,12 +20,44 @@
 	#size-cells = <2>;
 
 	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &i2c5;
+		i2c6 = &i2c6;
 		i2c7 = &i2c_dvfs;
 	};
 
-	psci {
-		compatible = "arm,psci-1.0", "arm,psci-0.2";
-		method = "smc";
+	/*
+	 * The external audio clocks are configured as 0 Hz fixed frequency
+	 * clocks by default.
+	 * Boards that provide audio clocks should override them.
+	 */
+	audio_clk_a: audio_clk_a {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	audio_clk_b: audio_clk_b {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	audio_clk_c: audio_clk_c {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	/* External CAN clock - to be overridden by boards that provide it */
+	can_clk: can {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
 	};
 
 	cpus {
@@ -35,23 +68,23 @@
 			compatible = "arm,cortex-a57", "arm,armv8";
 			reg = <0x0>;
 			device_type = "cpu";
-			power-domains = <&sysc 0>;
+			power-domains = <&sysc R8A77965_PD_CA57_CPU0>;
 			next-level-cache = <&L2_CA57>;
 			enable-method = "psci";
 		};
 
 		a57_1: cpu@1 {
-			compatible = "arm,cortex-a57","arm,armv8";
+			compatible = "arm,cortex-a57", "arm,armv8";
 			reg = <0x1>;
 			device_type = "cpu";
-			power-domains = <&sysc 1>;
+			power-domains = <&sysc R8A77965_PD_CA57_CPU1>;
 			next-level-cache = <&L2_CA57>;
 			enable-method = "psci";
 		};
 
 		L2_CA57: cache-controller-0 {
 			compatible = "cache";
-			power-domains = <&sysc 12>;
+			power-domains = <&sysc R8A77965_PD_CA57_SCU>;
 			cache-unified;
 			cache-level = <2>;
 		};
@@ -71,34 +104,24 @@
 		clock-frequency = <0>;
 	};
 
-	/*
-	 * The external audio clocks are configured as 0 Hz fixed frequency
-	 * clocks by default.
-	 * Boards that provide audio clocks should override them.
-	 */
-	audio_clk_a: audio_clk_a {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
-	};
-
-	audio_clk_b: audio_clk_b {
+	/* External PCIe clock - can be overridden by the board */
+	pcie_bus_clk: pcie_bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_c: audio_clk_c {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
+	pmu_a57 {
+		compatible = "arm,cortex-a57-pmu";
+		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&a57_0>,
+				     <&a57_1>;
 	};
 
-	/* External CAN clock - to be overridden by boards that provide it */
-	can_clk: can {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
+	psci {
+		compatible = "arm,psci-1.0", "arm,psci-0.2";
+		method = "smc";
 	};
 
 	/* External SCIF clock - to be overridden by boards that provide it */
@@ -108,42 +131,6 @@
 		clock-frequency = <0>;
 	};
 
-	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
-	};
-
-	/* External USB clocks - can be overridden by the board */
-	usb3s0_clk: usb3s0 {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
-	};
-
-	usb_extal_clk: usb_extal {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
-	};
-
-	timer {
-		compatible = "arm,armv8-timer";
-		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
-	};
-
-	pmu_a57 {
-		compatible = "arm,cortex-a57-pmu";
-		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
-				      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-affinity = <&a57_0>,
-				     <&a57_1>;
-	};
-
 	soc {
 		compatible = "simple-bus";
 		interrupt-parent = <&gic>;
@@ -151,52 +138,9 @@
 		#size-cells = <2>;
 		ranges;
 
-		gic: interrupt-controller@f1010000 {
-			compatible = "arm,gic-400";
-			#interrupt-cells = <3>;
-			#address-cells = <0>;
-			interrupt-controller;
-			reg = <0x0 0xf1010000 0 0x1000>,
-			      <0x0 0xf1020000 0 0x20000>,
-			      <0x0 0xf1040000 0 0x20000>,
-			      <0x0 0xf1060000 0 0x20000>;
-			interrupts = <GIC_PPI 9
-					(GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>;
-			clocks = <&cpg CPG_MOD 408>;
-			clock-names = "clk";
-			power-domains = <&sysc 32>;
-			resets = <&cpg 408>;
-		};
-
-		pfc: pin-controller@e6060000 {
-			compatible = "renesas,pfc-r8a77965";
-			reg = <0 0xe6060000 0 0x50c>;
-		};
-
-		cpg: clock-controller@e6150000 {
-			compatible = "renesas,r8a77965-cpg-mssr";
-			reg = <0 0xe6150000 0 0x1000>;
-			clocks = <&extal_clk>, <&extalr_clk>;
-			clock-names = "extal", "extalr";
-			#clock-cells = <2>;
-			#power-domain-cells = <0>;
-			#reset-cells = <1>;
-		};
-
-		rst: reset-controller@e6160000 {
-			compatible = "renesas,r8a77965-rst";
-			reg = <0 0xe6160000 0 0x0200>;
-		};
-
-		prr: chipid@fff00044 {
-			compatible = "renesas,prr";
-			reg = <0 0xfff00044 0 4>;
-		};
-
-		sysc: system-controller@e6180000 {
-			compatible = "renesas,r8a77965-sysc";
-			reg = <0 0xe6180000 0 0x0400>;
-			#power-domain-cells = <1>;
+		wdt0: watchdog@e6020000 {
+			reg = <0 0xe6020000 0 0x0c>;
+			/* placeholder */
 		};
 
 		gpio0: gpio@e6050000 {
@@ -210,7 +154,7 @@
 			#interrupt-cells = <2>;
 			interrupt-controller;
 			clocks = <&cpg CPG_MOD 912>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 912>;
 		};
 
@@ -225,7 +169,7 @@
 			#interrupt-cells = <2>;
 			interrupt-controller;
 			clocks = <&cpg CPG_MOD 911>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 911>;
 		};
 
@@ -240,7 +184,7 @@
 			#interrupt-cells = <2>;
 			interrupt-controller;
 			clocks = <&cpg CPG_MOD 910>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 910>;
 		};
 
@@ -255,7 +199,7 @@
 			#interrupt-cells = <2>;
 			interrupt-controller;
 			clocks = <&cpg CPG_MOD 909>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 909>;
 		};
 
@@ -270,7 +214,7 @@
 			#interrupt-cells = <2>;
 			interrupt-controller;
 			clocks = <&cpg CPG_MOD 908>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 908>;
 		};
 
@@ -285,7 +229,7 @@
 			#interrupt-cells = <2>;
 			interrupt-controller;
 			clocks = <&cpg CPG_MOD 907>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 907>;
 		};
 
@@ -300,7 +244,7 @@
 			#interrupt-cells = <2>;
 			interrupt-controller;
 			clocks = <&cpg CPG_MOD 906>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 906>;
 		};
 
@@ -315,10 +259,51 @@
 			#interrupt-cells = <2>;
 			interrupt-controller;
 			clocks = <&cpg CPG_MOD 905>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 905>;
 		};
 
+		pfc: pin-controller@e6060000 {
+			compatible = "renesas,pfc-r8a77965";
+			reg = <0 0xe6060000 0 0x50c>;
+		};
+
+		cpg: clock-controller@e6150000 {
+			compatible = "renesas,r8a77965-cpg-mssr";
+			reg = <0 0xe6150000 0 0x1000>;
+			clocks = <&extal_clk>, <&extalr_clk>;
+			clock-names = "extal", "extalr";
+			#clock-cells = <2>;
+			#power-domain-cells = <0>;
+			#reset-cells = <1>;
+		};
+
+		rst: reset-controller@e6160000 {
+			compatible = "renesas,r8a77965-rst";
+			reg = <0 0xe6160000 0 0x0200>;
+		};
+
+		sysc: system-controller@e6180000 {
+			compatible = "renesas,r8a77965-sysc";
+			reg = <0 0xe6180000 0 0x0400>;
+			#power-domain-cells = <1>;
+		};
+
+		tsc: thermal@e6198000 {
+			compatible = "renesas,r8a77965-thermal";
+			reg = <0 0xe6198000 0 0x100>,
+			      <0 0xe61a0000 0 0x100>,
+			      <0 0xe61a8000 0 0x100>;
+			interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 522>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 522>;
+			#thermal-sensor-cells = <1>;
+			status = "okay";
+		};
+
 		intc_ex: interrupt-controller@e61c0000 {
 			compatible = "renesas,intc-ex-r8a77965", "renesas,irqc";
 			#interrupt-cells = <2>;
@@ -331,84 +316,273 @@
 				      GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH
 				      GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 407>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 407>;
 		};
 
-		dmac0: dma-controller@e6700000 {
-			compatible = "renesas,dmac-r8a77965",
-				     "renesas,rcar-dmac";
-			reg = <0 0xe6700000 0 0x10000>;
-			interrupts = <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "error",
-					"ch0", "ch1", "ch2", "ch3",
-					"ch4", "ch5", "ch6", "ch7",
-					"ch8", "ch9", "ch10", "ch11",
-					"ch12", "ch13", "ch14", "ch15";
-			clocks = <&cpg CPG_MOD 219>;
-			clock-names = "fck";
-			power-domains = <&sysc 32>;
-			resets = <&cpg 219>;
-			#dma-cells = <1>;
-			dma-channels = <16>;
+		i2c0: i2c@e6500000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a77965",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe6500000 0 0x40>;
+			interrupts = <GIC_SPI 287 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 931>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 931>;
+			dmas = <&dmac1 0x91>, <&dmac1 0x90>,
+			       <&dmac2 0x91>, <&dmac2 0x90>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
+			status = "disabled";
 		};
 
-		dmac1: dma-controller@e7300000 {
-			compatible = "renesas,dmac-r8a77965",
-				     "renesas,rcar-dmac";
-			reg = <0 0xe7300000 0 0x10000>;
-			interrupts = <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 310 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 313 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 319 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "error",
-					"ch0", "ch1", "ch2", "ch3",
-					"ch4", "ch5", "ch6", "ch7",
-					"ch8", "ch9", "ch10", "ch11",
-					"ch12", "ch13", "ch14", "ch15";
-			clocks = <&cpg CPG_MOD 218>;
-			clock-names = "fck";
-			power-domains = <&sysc 32>;
-			resets = <&cpg 218>;
-			#dma-cells = <1>;
-			dma-channels = <16>;
+		i2c1: i2c@e6508000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a77965",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe6508000 0 0x40>;
+			interrupts = <GIC_SPI 288 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 930>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 930>;
+			dmas = <&dmac1 0x93>, <&dmac1 0x92>,
+			       <&dmac2 0x93>, <&dmac2 0x92>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
+			status = "disabled";
 		};
 
-		dmac2: dma-controller@e7310000 {
-			compatible = "renesas,dmac-r8a77965",
-				     "renesas,rcar-dmac";
-			reg = <0 0xe7310000 0 0x10000>;
-			interrupts = <GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 417 IRQ_TYPE_LEVEL_HIGH
+		i2c2: i2c@e6510000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a77965",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe6510000 0 0x40>;
+			interrupts = <GIC_SPI 286 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 929>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 929>;
+			dmas = <&dmac1 0x95>, <&dmac1 0x94>,
+			       <&dmac2 0x95>, <&dmac2 0x94>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
+			status = "disabled";
+		};
+
+		i2c3: i2c@e66d0000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a77965",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe66d0000 0 0x40>;
+			interrupts = <GIC_SPI 290 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 928>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 928>;
+			dmas = <&dmac0 0x97>, <&dmac0 0x96>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
+			status = "disabled";
+		};
+
+		i2c4: i2c@e66d8000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a77965",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe66d8000 0 0x40>;
+			interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 927>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 927>;
+			dmas = <&dmac0 0x99>, <&dmac0 0x98>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
+			status = "disabled";
+		};
+
+		i2c5: i2c@e66e0000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a77965",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe66e0000 0 0x40>;
+			interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 919>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 919>;
+			dmas = <&dmac0 0x9b>, <&dmac0 0x9a>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
+			status = "disabled";
+		};
+
+		i2c6: i2c@e66e8000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a77965",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe66e8000 0 0x40>;
+			interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 918>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 918>;
+			dmas = <&dmac0 0x9d>, <&dmac0 0x9c>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
+			status = "disabled";
+		};
+
+		i2c_dvfs: i2c@e60b0000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,iic-r8a77965",
+				     "renesas,rcar-gen3-iic",
+				     "renesas,rmobile-iic";
+			reg = <0 0xe60b0000 0 0x425>;
+			interrupts = <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 926>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 926>;
+			dmas = <&dmac0 0x11>, <&dmac0 0x10>;
+			dma-names = "tx", "rx";
+			status = "disabled";
+		};
+
+		hsusb: usb@e6590000 {
+			compatible = "renesas,usbhs-r8a7796",
+				     "renesas,rcar-gen3-usbhs";
+			reg = <0 0xe6590000 0 0x100>;
+			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 704>;
+			dmas = <&usb_dmac0 0>, <&usb_dmac0 1>,
+			       <&usb_dmac1 0>, <&usb_dmac1 1>;
+			dma-names = "ch0", "ch1", "ch2", "ch3";
+			renesas,buswait = <11>;
+			phys = <&usb2_phy0>;
+			phy-names = "usb";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 704>;
+			status = "disabled";
+		};
+
+		usb_dmac0: dma-controller@e65a0000 {
+			compatible = "renesas,r8a77965-usb-dmac",
+				     "renesas,usb-dmac";
+			reg = <0 0xe65a0000 0 0x100>;
+			interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1";
+			clocks = <&cpg CPG_MOD 330>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 330>;
+			#dma-cells = <1>;
+			dma-channels = <2>;
+		};
+
+		usb_dmac1: dma-controller@e65b0000 {
+			compatible = "renesas,r8a77965-usb-dmac",
+				     "renesas,usb-dmac";
+			reg = <0 0xe65b0000 0 0x100>;
+			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1";
+			clocks = <&cpg CPG_MOD 331>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 331>;
+			#dma-cells = <1>;
+			dma-channels = <2>;
+		};
+
+		usb3_phy0: usb-phy@e65ee000 {
+			compatible = "renesas,r8a77965-usb3-phy",
+				     "renesas,rcar-gen3-usb3-phy";
+			reg = <0 0xe65ee000 0 0x90>;
+			clocks = <&cpg CPG_MOD 328>, <&usb3s0_clk>,
+				 <&usb_extal_clk>;
+			clock-names = "usb3-if", "usb3s_clk", "usb_extal";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 328>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		dmac0: dma-controller@e6700000 {
+			compatible = "renesas,dmac-r8a77965",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe6700000 0 0x10000>;
+			interrupts = <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 219>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 219>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+		};
+
+		dmac1: dma-controller@e7300000 {
+			compatible = "renesas,dmac-r8a77965",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe7300000 0 0x10000>;
+			interrupts = <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 310 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 313 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 319 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 218>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 218>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+		};
+
+		dmac2: dma-controller@e7310000 {
+			compatible = "renesas,dmac-r8a77965",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe7310000 0 0x10000>;
+			interrupts = <GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 417 IRQ_TYPE_LEVEL_HIGH
 				      GIC_SPI 418 IRQ_TYPE_LEVEL_HIGH
 				      GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH
 				      GIC_SPI 420 IRQ_TYPE_LEVEL_HIGH
@@ -431,12 +605,127 @@
 					"ch12", "ch13", "ch14", "ch15";
 			clocks = <&cpg CPG_MOD 217>;
 			clock-names = "fck";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 217>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
 		};
 
+		avb: ethernet@e6800000 {
+			compatible = "renesas,etheravb-r8a77965",
+				     "renesas,etheravb-rcar-gen3";
+			reg = <0 0xe6800000 0 0x800>, <0 0xe6a00000 0 0x10000>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14", "ch15",
+					  "ch16", "ch17", "ch18", "ch19",
+					  "ch20", "ch21", "ch22", "ch23",
+					  "ch24";
+			clocks = <&cpg CPG_MOD 812>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 812>;
+			phy-mode = "rgmii";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		pwm0: pwm@e6e30000 {
+			compatible = "renesas,pwm-r8a77965", "renesas,pwm-rcar";
+			reg = <0 0xe6e30000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm1: pwm@e6e31000 {
+			compatible = "renesas,pwm-r8a77965", "renesas,pwm-rcar";
+			reg = <0 0xe6e31000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm2: pwm@e6e32000 {
+			compatible = "renesas,pwm-r8a77965", "renesas,pwm-rcar";
+			reg = <0 0xe6e32000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm3: pwm@e6e33000 {
+			compatible = "renesas,pwm-r8a77965", "renesas,pwm-rcar";
+			reg = <0 0xe6e33000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm4: pwm@e6e34000 {
+			compatible = "renesas,pwm-r8a77965", "renesas,pwm-rcar";
+			reg = <0 0xe6e34000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm5: pwm@e6e35000 {
+			compatible = "renesas,pwm-r8a77965", "renesas,pwm-rcar";
+			reg = <0 0xe6e35000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		pwm6: pwm@e6e36000 {
+			compatible = "renesas,pwm-r8a77965", "renesas,pwm-rcar";
+			reg = <0 0xe6e36000 0 8>;
+			#pwm-cells = <2>;
+			clocks = <&cpg CPG_MOD 523>;
+			resets = <&cpg 523>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
 		scif0: serial@e6e60000 {
 			compatible = "renesas,scif-r8a77965",
 				     "renesas,rcar-gen3-scif", "renesas,scif";
@@ -449,7 +738,7 @@
 			dmas = <&dmac1 0x51>, <&dmac1 0x50>,
 			       <&dmac2 0x51>, <&dmac2 0x50>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 207>;
 			status = "disabled";
 		};
@@ -466,7 +755,7 @@
 			dmas = <&dmac1 0x53>, <&dmac1 0x52>,
 			       <&dmac2 0x53>, <&dmac2 0x52>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 206>;
 			status = "disabled";
 		};
@@ -480,7 +769,7 @@
 				 <&cpg CPG_CORE 20>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 310>;
 			status = "disabled";
 		};
@@ -496,7 +785,7 @@
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac0 0x57>, <&dmac0 0x56>;
 			dma-names = "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 204>;
 			status = "disabled";
 		};
@@ -512,7 +801,7 @@
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac0 0x59>, <&dmac0 0x58>;
 			dma-names = "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 203>;
 			status = "disabled";
 		};
@@ -529,243 +818,772 @@
 			dmas = <&dmac1 0x5b>, <&dmac1 0x5a>,
 			       <&dmac2 0x5b>, <&dmac2 0x5a>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
 			resets = <&cpg 202>;
 			status = "disabled";
 		};
 
-		avb: ethernet@e6800000 {
-			compatible = "renesas,etheravb-r8a77965",
-				     "renesas,etheravb-rcar-gen3";
-			reg = <0 0xe6800000 0 0x800>, <0 0xe6a00000 0 0x10000>;
-			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "ch0", "ch1", "ch2", "ch3",
-					  "ch4", "ch5", "ch6", "ch7",
-					  "ch8", "ch9", "ch10", "ch11",
-					  "ch12", "ch13", "ch14", "ch15",
-					  "ch16", "ch17", "ch18", "ch19",
-					  "ch20", "ch21", "ch22", "ch23",
-					  "ch24";
-			clocks = <&cpg CPG_MOD 812>;
-			power-domains = <&sysc 32>;
-			resets = <&cpg 812>;
-			phy-mode = "rgmii";
+		msiof0: spi@e6e90000 {
+			compatible = "renesas,msiof-r8a77965",
+				     "renesas,rcar-gen3-msiof";
+			reg = <0 0xe6e90000 0 0x0064>;
+			interrupts = <GIC_SPI 156 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 211>;
+			dmas = <&dmac1 0x41>, <&dmac1 0x40>,
+			       <&dmac2 0x41>, <&dmac2 0x40>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 211>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			status = "disabled";
 		};
 
-		csi20: csi2@fea80000 {
-			reg = <0 0xfea80000 0 0x10000>;
-			/* placeholder */
+		msiof1: spi@e6ea0000 {
+			compatible = "renesas,msiof-r8a77965",
+				     "renesas,rcar-gen3-msiof";
+			reg = <0 0xe6ea0000 0 0x0064>;
+			interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 210>;
+			dmas = <&dmac1 0x43>, <&dmac1 0x42>,
+			       <&dmac2 0x43>, <&dmac2 0x42>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 210>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof2: spi@e6c00000 {
+			compatible = "renesas,msiof-r8a77965",
+				     "renesas,rcar-gen3-msiof";
+			reg = <0 0xe6c00000 0 0x0064>;
+			interrupts = <GIC_SPI 158 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 209>;
+			dmas = <&dmac0 0x45>, <&dmac0 0x44>;
+			dma-names = "tx", "rx";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 209>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof3: spi@e6c10000 {
+			compatible = "renesas,msiof-r8a77965",
+				     "renesas,rcar-gen3-msiof";
+			reg = <0 0xe6c10000 0 0x0064>;
+			interrupts = <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 208>;
+			dmas = <&dmac0 0x47>, <&dmac0 0x46>;
+			dma-names = "tx", "rx";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 208>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		vin0: video@e6ef0000 {
+			compatible = "renesas,vin-r8a77965";
+			reg = <0 0xe6ef0000 0 0x1000>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 811>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 811>;
+			renesas,id = <0>;
+			status = "disabled";
 
 			ports {
 				#address-cells = <1>;
 				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin0csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin0>;
+					};
+					vin0csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin0>;
+					};
+				};
 			};
 		};
 
-		csi40: csi2@feaa0000 {
-			reg = <0 0xfeaa0000 0 0x10000>;
-			/* placeholder */
+		vin1: video@e6ef1000 {
+			compatible = "renesas,vin-r8a77965";
+			reg = <0 0xe6ef1000 0 0x1000>;
+			interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 810>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 810>;
+			renesas,id = <1>;
+			status = "disabled";
 
 			ports {
 				#address-cells = <1>;
 				#size-cells = <0>;
-			};
-		};
 
-		vin0: video@e6ef0000 {
-			reg = <0 0xe6ef0000 0 0x1000>;
-			/* placeholder */
-		};
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
 
-		vin1: video@e6ef1000 {
-			reg = <0 0xe6ef1000 0 0x1000>;
-			/* placeholder */
+					reg = <1>;
+
+					vin1csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin1>;
+					};
+					vin1csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin1>;
+					};
+				};
+			};
 		};
 
 		vin2: video@e6ef2000 {
+			compatible = "renesas,vin-r8a77965";
 			reg = <0 0xe6ef2000 0 0x1000>;
-			/* placeholder */
+			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 809>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 809>;
+			renesas,id = <2>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin2csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin2>;
+					};
+					vin2csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin2>;
+					};
+				};
+			};
 		};
 
 		vin3: video@e6ef3000 {
+			compatible = "renesas,vin-r8a77965";
 			reg = <0 0xe6ef3000 0 0x1000>;
-			/* placeholder */
+			interrupts = <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 808>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 808>;
+			renesas,id = <3>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin3csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin3>;
+					};
+					vin3csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin3>;
+					};
+				};
+			};
 		};
 
 		vin4: video@e6ef4000 {
+			compatible = "renesas,vin-r8a77965";
 			reg = <0 0xe6ef4000 0 0x1000>;
-			/* placeholder */
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 807>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 807>;
+			renesas,id = <4>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin4csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin4>;
+					};
+					vin4csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin4>;
+					};
+				};
+			};
 		};
 
 		vin5: video@e6ef5000 {
+			compatible = "renesas,vin-r8a77965";
 			reg = <0 0xe6ef5000 0 0x1000>;
-			/* placeholder */
+			interrupts = <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 806>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 806>;
+			renesas,id = <5>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin5csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin5>;
+					};
+					vin5csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin5>;
+					};
+				};
+			};
 		};
 
 		vin6: video@e6ef6000 {
+			compatible = "renesas,vin-r8a77965";
 			reg = <0 0xe6ef6000 0 0x1000>;
-			/* placeholder */
+			interrupts = <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 805>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 805>;
+			renesas,id = <6>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin6csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin6>;
+					};
+					vin6csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin6>;
+					};
+				};
+			};
 		};
 
 		vin7: video@e6ef7000 {
+			compatible = "renesas,vin-r8a77965";
 			reg = <0 0xe6ef7000 0 0x1000>;
+			interrupts = <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 804>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 804>;
+			renesas,id = <7>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin7csi20: endpoint@0 {
+						reg = <0>;
+						remote-endpoint= <&csi20vin7>;
+					};
+					vin7csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin7>;
+					};
+				};
+			};
+		};
+
+		rcar_sound: sound@ec500000 {
+			reg =	<0 0xec500000 0 0x1000>, /* SCU */
+				<0 0xec5a0000 0 0x100>,  /* ADG */
+				<0 0xec540000 0 0x1000>, /* SSIU */
+				<0 0xec541000 0 0x280>,  /* SSI */
+				<0 0xec740000 0 0x200>;  /* Audio DMAC peri peri*/
 			/* placeholder */
+
+			rcar_sound,dvc {
+				dvc0: dvc-0 {
+				};
+				dvc1: dvc-1 {
+				};
+			};
+
+			rcar_sound,src {
+				src0: src-0 {
+				};
+				src1: src-1 {
+				};
+			};
+
+			rcar_sound,ssi {
+				ssi0: ssi-0 {
+				};
+				ssi1: ssi-1 {
+				};
+			};
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				port@0 {
+					reg = <0>;
+				};
+			};
+		};
+
+		xhci0: usb@ee000000 {
+			compatible = "renesas,xhci-r8a77965",
+				     "renesas,rcar-gen3-xhci";
+			reg = <0 0xee000000 0 0xc00>;
+			interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 328>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 328>;
+			status = "disabled";
+		};
+
+		usb3_peri0: usb@ee020000 {
+			compatible = "renesas,r8a77965-usb3-peri",
+				     "renesas,rcar-gen3-usb3-peri";
+			reg = <0 0xee020000 0 0x400>;
+			interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 328>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 328>;
+			status = "disabled";
 		};
 
 		ohci0: usb@ee080000 {
+			compatible = "generic-ohci";
 			reg = <0 0xee080000 0 0x100>;
-			/* placeholder */
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 703>;
+			phys = <&usb2_phy0>;
+			phy-names = "usb";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 703>;
+			status = "disabled";
+		};
+
+		ohci1: usb@ee0a0000 {
+			compatible = "generic-ohci";
+			reg = <0 0xee0a0000 0 0x100>;
+			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 702>;
+			phys = <&usb2_phy1>;
+			phy-names = "usb";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 702>;
+			status = "disabled";
 		};
 
 		ehci0: usb@ee080100 {
+			compatible = "generic-ehci";
 			reg = <0 0xee080100 0 0x100>;
-			/* placeholder */
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 703>;
+			phys = <&usb2_phy0>;
+			phy-names = "usb";
+			companion = <&ohci0>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 703>;
+			status = "disabled";
+		};
+
+		ehci1: usb@ee0a0100 {
+			compatible = "generic-ehci";
+			reg = <0 0xee0a0100 0 0x100>;
+			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 702>;
+			phys = <&usb2_phy1>;
+			phy-names = "usb";
+			companion = <&ohci1>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 702>;
+			status = "disabled";
 		};
 
 		usb2_phy0: usb-phy@ee080200 {
+			compatible = "renesas,usb2-phy-r8a77965",
+				     "renesas,rcar-gen3-usb2-phy";
 			reg = <0 0xee080200 0 0x700>;
-			/* placeholder */
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 703>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 703>;
+			#phy-cells = <0>;
+			status = "disabled";
 		};
 
 		usb2_phy1: usb-phy@ee0a0200 {
+			compatible = "renesas,usb2-phy-r8a77965",
+				     "renesas,rcar-gen3-usb2-phy";
 			reg = <0 0xee0a0200 0 0x700>;
-			/* placeholder */
+			clocks = <&cpg CPG_MOD 703>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 703>;
+			#phy-cells = <0>;
+			status = "disabled";
 		};
 
-		ohci1: usb@ee0a0000 {
-			reg = <0 0xee0a0000 0 0x100>;
-			/* placeholder */
+		sdhi0: sd@ee100000 {
+			compatible = "renesas,sdhi-r8a77965",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee100000 0 0x2000>;
+			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 314>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 314>;
+			status = "disabled";
 		};
 
-		ehci1: usb@ee0a0100 {
-			reg = <0 0xee0a0100 0 0x100>;
-			/* placeholder */
+		sdhi1: sd@ee120000 {
+			compatible = "renesas,sdhi-r8a77965",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee120000 0 0x2000>;
+			interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 313>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 313>;
+			status = "disabled";
 		};
 
-		i2c0: i2c@e6500000 {
-			reg = <0 0xe6500000 0 0x40>;
-			/* placeholder */
+		sdhi2: sd@ee140000 {
+			compatible = "renesas,sdhi-r8a77965",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee140000 0 0x2000>;
+			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 312>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 312>;
+			status = "disabled";
 		};
 
-		i2c1: i2c@e6508000 {
-			reg = <0 0xe6508000 0 0x40>;
-			/* placeholder */
+		sdhi3: sd@ee160000 {
+			compatible = "renesas,sdhi-r8a77965",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee160000 0 0x2000>;
+			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 311>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 311>;
+			status = "disabled";
 		};
 
-		i2c2: i2c@e6510000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
+		gic: interrupt-controller@f1010000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0x0 0xf1010000 0 0x1000>,
+			      <0x0 0xf1020000 0 0x20000>,
+			      <0x0 0xf1040000 0 0x20000>,
+			      <0x0 0xf1060000 0 0x20000>;
+			interrupts = <GIC_PPI 9
+					(GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>;
+			clocks = <&cpg CPG_MOD 408>;
+			clock-names = "clk";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 408>;
+		};
 
-			reg = <0 0xe6510000 0 0x40>;
+		pciec0: pcie@fe000000 {
+			reg = <0 0xfe000000 0 0x80000>;
 			/* placeholder */
 		};
 
-		i2c3: i2c@e66d0000 {
-			reg = <0 0xe66d0000 0 0x40>;
+		pciec1: pcie@ee800000 {
+			reg = <0 0xee800000 0 0x80000>;
 			/* placeholder */
 		};
 
-		i2c4: i2c@e66d8000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
+		fcpf0: fcp@fe950000 {
+			compatible = "renesas,fcpf";
+			reg = <0 0xfe950000 0 0x200>;
+			clocks = <&cpg CPG_MOD 615>;
+			power-domains = <&sysc R8A77965_PD_A3VP>;
+			resets = <&cpg 615>;
+		};
 
-			reg = <0 0xe66d8000 0 0x40>;
-			/* placeholder */
+		vspb: vsp@fe960000 {
+			compatible = "renesas,vsp2";
+			reg = <0 0xfe960000 0 0x8000>;
+			interrupts = <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 626>;
+			power-domains = <&sysc R8A77965_PD_A3VP>;
+			resets = <&cpg 626>;
+
+			renesas,fcp = <&fcpvb0>;
 		};
 
-		i2c5: i2c@e66e0000 {
-			reg = <0 0xe66e0000 0 0x40>;
-			/* placeholder */
+		fcpvb0: fcp@fe96f000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfe96f000 0 0x200>;
+			clocks = <&cpg CPG_MOD 607>;
+			power-domains = <&sysc R8A77965_PD_A3VP>;
+			resets = <&cpg 607>;
 		};
 
-		i2c6: i2c@e66e8000 {
-			reg = <0 0xe66e8000 0 0x40>;
-			/* placeholder */
+		vspi0: vsp@fe9a0000 {
+			compatible = "renesas,vsp2";
+			reg = <0 0xfe9a0000 0 0x8000>;
+			interrupts = <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 631>;
+			power-domains = <&sysc R8A77965_PD_A3VP>;
+			resets = <&cpg 631>;
+
+			renesas,fcp = <&fcpvi0>;
 		};
 
-		i2c_dvfs: i2c@e60b0000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,iic-r8a77965",
-				     "renesas,rcar-gen3-iic",
-				     "renesas,rmobile-iic";
-			reg = <0 0xe60b0000 0 0x425>;
-			interrupts = <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 926>;
-			power-domains = <&sysc 32>;
-			resets = <&cpg 926>;
-			dmas = <&dmac0 0x11>, <&dmac0 0x10>;
-			dma-names = "tx", "rx";
-			status = "disabled";
+		fcpvi0: fcp@fe9af000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfe9af000 0 0x200>;
+			clocks = <&cpg CPG_MOD 611>;
+			power-domains = <&sysc R8A77965_PD_A3VP>;
+			resets = <&cpg 611>;
 		};
 
-		pwm0: pwm@e6e30000 {
-			reg = <0 0xe6e30000 0 8>;
-			/* placeholder */
+		vspd0: vsp@fea20000 {
+			compatible = "renesas,vsp2";
+			reg = <0 0xfea20000 0 0x8000>;
+			interrupts = <GIC_SPI 466 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 623>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 623>;
+
+			renesas,fcp = <&fcpvd0>;
 		};
 
-		pwm1: pwm@e6e31000 {
-			reg = <0 0xe6e31000 0 8>;
-			#pwm-cells = <2>;
-			/* placeholder */
+		fcpvd0: fcp@fea27000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfea27000 0 0x200>;
+			clocks = <&cpg CPG_MOD 603>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 603>;
 		};
 
-		pwm2: pwm@e6e32000 {
-			reg = <0 0xe6e32000 0 8>;
-			/* placeholder */
+		vspd1: vsp@fea28000 {
+			compatible = "renesas,vsp2";
+			reg = <0 0xfea28000 0 0x8000>;
+			interrupts = <GIC_SPI 467 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 622>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 622>;
+
+			renesas,fcp = <&fcpvd1>;
 		};
 
-		pwm3: pwm@e6e33000 {
-			reg = <0 0xe6e33000 0 8>;
-			/* placeholder */
+		fcpvd1: fcp@fea2f000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfea2f000 0 0x200>;
+			clocks = <&cpg CPG_MOD 602>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 602>;
 		};
 
-		pwm4: pwm@e6e34000 {
-			reg = <0 0xe6e34000 0 8>;
-			/* placeholder */
+		csi20: csi2@fea80000 {
+			compatible = "renesas,r8a77965-csi2";
+			reg = <0 0xfea80000 0 0x10000>;
+			interrupts = <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 714>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 714>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					csi20vin0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&vin0csi20>;
+					};
+					csi20vin1: endpoint@1 {
+						reg = <1>;
+						remote-endpoint = <&vin1csi20>;
+					};
+					csi20vin2: endpoint@2 {
+						reg = <2>;
+						remote-endpoint = <&vin2csi20>;
+					};
+					csi20vin3: endpoint@3 {
+						reg = <3>;
+						remote-endpoint = <&vin3csi20>;
+					};
+					csi20vin4: endpoint@4 {
+						reg = <4>;
+						remote-endpoint = <&vin4csi20>;
+					};
+					csi20vin5: endpoint@5 {
+						reg = <5>;
+						remote-endpoint = <&vin5csi20>;
+					};
+					csi20vin6: endpoint@6 {
+						reg = <6>;
+						remote-endpoint = <&vin6csi20>;
+					};
+					csi20vin7: endpoint@7 {
+						reg = <7>;
+						remote-endpoint = <&vin7csi20>;
+					};
+				};
+			};
 		};
 
-		pwm5: pwm@e6e35000 {
-			reg = <0 0xe6e35000 0 8>;
-			/* placeholder */
+		csi40: csi2@feaa0000 {
+			compatible = "renesas,r8a77965-csi2";
+			reg = <0 0xfeaa0000 0 0x10000>;
+			interrupts = <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 716>;
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 716>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					csi40vin0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&vin0csi40>;
+					};
+					csi40vin1: endpoint@1 {
+						reg = <1>;
+						remote-endpoint = <&vin1csi40>;
+					};
+					csi40vin2: endpoint@2 {
+						reg = <2>;
+						remote-endpoint = <&vin2csi40>;
+					};
+					csi40vin3: endpoint@3 {
+						reg = <3>;
+						remote-endpoint = <&vin3csi40>;
+					};
+					csi40vin4: endpoint@4 {
+						reg = <4>;
+						remote-endpoint = <&vin4csi40>;
+					};
+					csi40vin5: endpoint@5 {
+						reg = <5>;
+						remote-endpoint = <&vin5csi40>;
+					};
+					csi40vin6: endpoint@6 {
+						reg = <6>;
+						remote-endpoint = <&vin6csi40>;
+					};
+					csi40vin7: endpoint@7 {
+						reg = <7>;
+						remote-endpoint = <&vin7csi40>;
+					};
+				};
+			};
 		};
 
-		pwm6: pwm@e6e36000 {
-			reg = <0 0xe6e36000 0 8>;
-			/* placeholder */
+		hdmi0: hdmi@fead0000 {
+			compatible = "renesas,r8a77965-hdmi",
+				     "renesas,rcar-gen3-hdmi";
+			reg = <0 0xfead0000 0 0x10000>;
+			interrupts = <GIC_SPI 389 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 729>,
+				 <&cpg CPG_CORE R8A77965_CLK_HDMI>;
+			clock-names = "iahb", "isfr";
+			power-domains = <&sysc R8A77965_PD_ALWAYS_ON>;
+			resets = <&cpg 729>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				port@0 {
+					reg = <0>;
+					dw_hdmi0_in: endpoint {
+						remote-endpoint = <&du_out_hdmi0>;
+					};
+				};
+				port@1 {
+					reg = <1>;
+				};
+			};
 		};
 
 		du: display@feb00000 {
-			reg = <0 0xfeb00000 0 0x80000>,
-			      <0 0xfeb90000 0 0x14>;
-			/* placeholder */
+			compatible = "renesas,du-r8a77965";
+			reg = <0 0xfeb00000 0 0x80000>;
+			reg-names = "du";
+			interrupts = <GIC_SPI 256 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 270 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 724>,
+				 <&cpg CPG_MOD 723>,
+				 <&cpg CPG_MOD 721>;
+			clock-names = "du.0", "du.1", "du.3";
+			status = "disabled";
+
+			vsps = <&vspd0 0 &vspd1 0 &vspd0 1>;
 
 			ports {
 				#address-cells = <1>;
@@ -779,6 +1597,7 @@
 				port@1 {
 					reg = <1>;
 					du_out_hdmi0: endpoint {
+						remote-endpoint = <&dw_hdmi0_in>;
 					};
 				};
 				port@2 {
@@ -789,90 +1608,74 @@
 			};
 		};
 
-		hsusb: usb@e6590000 {
-			reg = <0 0xe6590000 0 0x100>;
-			/* placeholder */
-		};
-
-		pciec0: pcie@fe000000 {
-			reg = <0 0xfe000000 0 0x80000>;
-			/* placeholder */
-		};
-
-		pciec1: pcie@ee800000 {
-			reg = <0 0xee800000 0 0x80000>;
-			/* placeholder */
+		prr: chipid@fff00044 {
+			compatible = "renesas,prr";
+			reg = <0 0xfff00044 0 4>;
 		};
+	};
 
-		rcar_sound: sound@ec500000 {
-			reg =	<0 0xec500000 0 0x1000>, /* SCU */
-				<0 0xec5a0000 0 0x100>,  /* ADG */
-				<0 0xec540000 0 0x1000>, /* SSIU */
-				<0 0xec541000 0 0x280>,  /* SSI */
-				<0 0xec740000 0 0x200>;  /* Audio DMAC peri peri*/
-			/* placeholder */
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
+	};
 
-			rcar_sound,dvc {
-				dvc0: dvc-0 {
-				};
-				dvc1: dvc-1 {
+	thermal-zones {
+		sensor_thermal1: sensor-thermal1 {
+			polling-delay-passive = <250>;
+			polling-delay = <1000>;
+			thermal-sensors = <&tsc 0>;
+
+			trips {
+				sensor1_crit: sensor1-crit {
+					temperature = <120000>;
+					hysteresis = <1000>;
+					type = "critical";
 				};
 			};
+		};
 
-			rcar_sound,src {
-				src0: src-0 {
-				};
-				src1: src-1 {
-				};
-			};
+		sensor_thermal2: sensor-thermal2 {
+			polling-delay-passive = <250>;
+			polling-delay = <1000>;
+			thermal-sensors = <&tsc 1>;
 
-			rcar_sound,ssi {
-				ssi0: ssi-0 {
-				};
-				ssi1: ssi-1 {
+			trips {
+				sensor2_crit: sensor2-crit {
+					temperature = <120000>;
+					hysteresis = <1000>;
+					type = "critical";
 				};
 			};
 		};
 
-		sdhi0: sd@ee100000 {
-			reg = <0 0xee100000 0 0x2000>;
-			/* placeholder */
-		};
-
-		sdhi1: sd@ee120000 {
-			reg = <0 0xee120000 0 0x2000>;
-			/* placeholder */
-		};
+		sensor_thermal3: sensor-thermal3 {
+			polling-delay-passive = <250>;
+			polling-delay = <1000>;
+			thermal-sensors = <&tsc 2>;
 
-		sdhi2: sd@ee140000 {
-			reg = <0 0xee140000 0 0x2000>;
-			/* placeholder */
-		};
-
-		sdhi3: sd@ee160000 {
-			reg = <0 0xee160000 0 0x2000>;
-			/* placeholder */
-		};
-
-		usb3_phy0: usb-phy@e65ee000 {
-			reg = <0 0xe65ee000 0 0x90>;
-			#phy-cells = <0>;
-			/* placeholder */
-		};
-
-		usb3_peri0: usb@ee020000 {
-			reg = <0 0xee020000 0 0x400>;
-			/* placeholder */
+			trips {
+				sensor3_crit: sensor3-crit {
+					temperature = <120000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
 		};
+	};
 
-		xhci0: usb@ee000000 {
-			reg = <0 0xee000000 0 0xc00>;
-			/* placeholder */
-		};
+	/* External USB clocks - can be overridden by the board */
+	usb3s0_clk: usb3s0 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
 
-		wdt0: watchdog@e6020000 {
-			reg = <0 0xe6020000 0 0x0c>;
-			/* placeholder */
-		};
+	usb_extal_clk: usb_extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
 	};
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
index 3c5f598c9766..21f9cf5c6e84 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
@@ -31,9 +31,57 @@
 		/* first 128MB is reserved for secure area. */
 		reg = <0x0 0x48000000 0x0 0x38000000>;
 	};
+
+	hdmi-out {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_out: endpoint {
+				remote-endpoint = <&adv7511_out>;
+			};
+		};
+	};
+
+	d3p3: regulator-fixed {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	lvds-decoder {
+		compatible = "thine,thc63lvd1024";
+
+		vcc-supply = <&d3p3>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				thc63lvd1024_in: endpoint {
+					remote-endpoint = <&lvds0_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+				thc63lvd1024_out: endpoint {
+					remote-endpoint = <&adv7511_in>;
+				};
+			};
+		};
+	};
 };
 
 &avb {
+	pinctrl-0 = <&avb_pins>;
+	pinctrl-names = "default";
+
 	renesas,no-ether-link;
 	phy-handle = <&phy0>;
 	phy-mode = "rgmii-id";
@@ -47,6 +95,16 @@
 	};
 };
 
+&canfd {
+	pinctrl-0 = <&canfd0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	channel0 {
+		status = "okay";
+	};
+};
+
 &extal_clk {
 	clock-frequency = <16666666>;
 };
@@ -68,9 +126,51 @@
 		gpio-controller;
 		#gpio-cells = <2>;
 	};
+
+	hdmi@39 {
+		compatible = "adi,adv7511w";
+		reg = <0x39>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <20 IRQ_TYPE_LEVEL_LOW>;
+
+		adi,input-depth = <8>;
+		adi,input-colorspace = "rgb";
+		adi,input-clock = "1x";
+		adi,input-style = <1>;
+		adi,input-justification = "evenly";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				adv7511_in: endpoint {
+					remote-endpoint = <&thc63lvd1024_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				adv7511_out: endpoint {
+					remote-endpoint = <&hdmi_con_out>;
+				};
+			};
+		};
+	};
 };
 
 &pfc {
+	avb_pins: avb0 {
+		groups = "avb0_mdio", "avb0_rgmii", "avb0_txcrefclk";
+		function = "avb0";
+	};
+
+	canfd0_pins: canfd0 {
+		groups = "canfd0_data_a";
+		function = "canfd0";
+	};
+
 	i2c0_pins: i2c0 {
 		groups = "i2c0";
 		function = "i2c0";
@@ -93,3 +193,19 @@
 
 	status = "okay";
 };
+
+&du {
+	status = "okay";
+};
+
+&lvds0 {
+	status = "okay";
+
+	ports {
+		port@1 {
+			lvds0_out: endpoint {
+				remote-endpoint = <&thc63lvd1024_in>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
index a8ceeac77992..9fce031a596f 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
@@ -29,9 +29,71 @@
 		/* first 128MB is reserved for secure area. */
 		reg = <0x0 0x48000000 0x0 0x38000000>;
 	};
+
+	osc5_clk: osc5-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <148500000>;
+	};
+
+	vcc_d1_8v: regulator-0 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_D1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	vcc_d3_3v: regulator-1 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_D3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	lvds-decoder {
+		compatible = "thine,thc63lvd1024";
+		vcc-supply = <&vcc_d3_3v>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				thc63lvd1024_in: endpoint {
+					remote-endpoint = <&lvds0_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+				thc63lvd1024_out: endpoint {
+					remote-endpoint = <&adv7511_in>;
+				};
+			};
+		};
+	};
+
+	hdmi-out {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&adv7511_out>;
+			};
+		};
+	};
 };
 
 &avb {
+	pinctrl-0 = <&avb_pins>;
+	pinctrl-names = "default";
+
 	renesas,no-ether-link;
 	phy-handle = <&phy0>;
 	phy-mode = "rgmii-id";
@@ -43,6 +105,13 @@
 	};
 };
 
+&du {
+	clocks = <&cpg CPG_MOD 724>,
+		 <&osc5_clk>;
+	clock-names = "du.0", "dclkin.0";
+	status = "okay";
+};
+
 &extal_clk {
 	clock-frequency = <16666666>;
 };
@@ -52,12 +121,80 @@
 };
 
 &pfc {
+	avb_pins: avb0 {
+		groups = "avb0_mdio", "avb0_rgmii", "avb0_txcrefclk";
+		function = "avb0";
+	};
+
+	i2c0_pins: i2c0 {
+		groups = "i2c0";
+		function = "i2c0";
+	};
+
 	scif0_pins: scif0 {
 		groups = "scif0_data";
 		function = "scif0";
 	};
 };
 
+&i2c0 {
+	pinctrl-0 = <&i2c0_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+	clock-frequency = <400000>;
+
+	hdmi@39{
+		compatible = "adi,adv7511w";
+		#sound-dai-cells = <0>;
+		reg = <0x39>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <20 IRQ_TYPE_LEVEL_LOW>;
+		avdd-supply = <&vcc_d1_8v>;
+		dvdd-supply = <&vcc_d1_8v>;
+		pvdd-supply = <&vcc_d1_8v>;
+		bgvdd-supply = <&vcc_d1_8v>;
+		dvdd-3v-supply = <&vcc_d3_3v>;
+
+		adi,input-depth = <8>;
+		adi,input-colorspace = "rgb";
+		adi,input-clock = "1x";
+		adi,input-style = <1>;
+		adi,input-justification = "evenly";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				adv7511_in: endpoint {
+					remote-endpoint = <&thc63lvd1024_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				adv7511_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+};
+
+&lvds0 {
+	status = "okay";
+
+	ports {
+		port@1 {
+			lvds0_out: endpoint {
+				remote-endpoint = <&thc63lvd1024_in>;
+			};
+		};
+	};
+};
+
 &scif0 {
 	pinctrl-0 = <&scif0_pins>;
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/renesas/r8a77970.dtsi b/arch/arm64/boot/dts/renesas/r8a77970.dtsi
index c6db8ea43906..98a2317a16c4 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77970.dtsi
@@ -41,6 +41,16 @@
 			enable-method = "psci";
 		};
 
+		a53_1: cpu@1 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <1>;
+			clocks = <&cpg CPG_CORE R8A77970_CLK_Z2>;
+			power-domains = <&sysc R8A77970_PD_CA53_CPU1>;
+			next-level-cache = <&L2_CA53>;
+			enable-method = "psci";
+		};
+
 		L2_CA53: cache-controller {
 			compatible = "cache";
 			power-domains = <&sysc R8A77970_PD_CA53_SCU>;
@@ -63,11 +73,25 @@
 		clock-frequency = <0>;
 	};
 
+	pmu_a53 {
+		compatible = "arm,cortex-a53-pmu";
+		interrupts-extended = <&gic GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&a53_0>, <&a53_1>;
+	};
+
 	psci {
 		compatible = "arm,psci-1.0", "arm,psci-0.2";
 		method = "smc";
 	};
 
+	/* External CAN clock - to be overridden by boards that provide it */
+	can_clk: can {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
 	/* External SCIF clock - to be overridden by boards that provide it */
 	scif_clk: scif {
 		compatible = "fixed-clock";
@@ -83,23 +107,6 @@
 		#size-cells = <2>;
 		ranges;
 
-		gic: interrupt-controller@f1010000 {
-			compatible = "arm,gic-400";
-			#interrupt-cells = <3>;
-			#address-cells = <0>;
-			interrupt-controller;
-			reg = <0 0xf1010000 0 0x1000>,
-			      <0 0xf1020000 0 0x20000>,
-			      <0 0xf1040000 0 0x20000>,
-			      <0 0xf1060000 0 0x20000>;
-			interrupts = <GIC_PPI 9	(GIC_CPU_MASK_SIMPLE(1) |
-				      IRQ_TYPE_LEVEL_HIGH)>;
-			clocks = <&cpg CPG_MOD 408>;
-			clock-names = "clk";
-			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
-			resets = <&cpg 408>;
-		};
-
 		rwdt: watchdog@e6020000 {
 			compatible = "renesas,r8a77970-wdt",
 				     "renesas,rcar-gen3-wdt";
@@ -110,75 +117,6 @@
 			status = "disabled";
 		};
 
-		cpg: clock-controller@e6150000 {
-			compatible = "renesas,r8a77970-cpg-mssr";
-			reg = <0 0xe6150000 0 0x1000>;
-			clocks = <&extal_clk>, <&extalr_clk>;
-			clock-names = "extal", "extalr";
-			#clock-cells = <2>;
-			#power-domain-cells = <0>;
-			#reset-cells = <1>;
-		};
-
-		rst: reset-controller@e6160000 {
-			compatible = "renesas,r8a77970-rst";
-			reg = <0 0xe6160000 0 0x200>;
-		};
-
-		sysc: system-controller@e6180000 {
-			compatible = "renesas,r8a77970-sysc";
-			reg = <0 0xe6180000 0 0x440>;
-			#power-domain-cells = <1>;
-		};
-
-		ipmmu_vi0: mmu@febd0000 {
-			compatible = "renesas,ipmmu-r8a77970";
-			reg = <0 0xfebd0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 9>;
-			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-			status = "disabled";
-		};
-
-		ipmmu_ir: mmu@ff8b0000 {
-			compatible = "renesas,ipmmu-r8a77970";
-			reg = <0 0xff8b0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 3>;
-			power-domains = <&sysc R8A77970_PD_A3IR>;
-			#iommu-cells = <1>;
-			status = "disabled";
-		};
-
-		ipmmu_rt: mmu@ffc80000 {
-			compatible = "renesas,ipmmu-r8a77970";
-			reg = <0 0xffc80000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 7>;
-			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-		};
-
-		ipmmu_ds1: mmu@e7740000 {
-			compatible = "renesas,ipmmu-r8a77970";
-			reg = <0 0xe7740000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 1>;
-			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-		};
-
-		ipmmu_mm: mmu@e67b0000 {
-			compatible = "renesas,ipmmu-r8a77970";
-			reg = <0 0xe67b0000 0 0x1000>;
-			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
-			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
-			#iommu-cells = <1>;
-		};
-
-		pfc: pin-controller@e6060000 {
-			compatible = "renesas,pfc-r8a77970";
-			reg = <0 0xe6060000 0 0x504>;
-		};
-
 		gpio0: gpio@e6050000 {
 			compatible = "renesas,gpio-r8a77970",
 				     "renesas,rcar-gen3-gpio";
@@ -269,6 +207,32 @@
 			resets = <&cpg 907>;
 		};
 
+		pfc: pin-controller@e6060000 {
+			compatible = "renesas,pfc-r8a77970";
+			reg = <0 0xe6060000 0 0x504>;
+		};
+
+		cpg: clock-controller@e6150000 {
+			compatible = "renesas,r8a77970-cpg-mssr";
+			reg = <0 0xe6150000 0 0x1000>;
+			clocks = <&extal_clk>, <&extalr_clk>;
+			clock-names = "extal", "extalr";
+			#clock-cells = <2>;
+			#power-domain-cells = <0>;
+			#reset-cells = <1>;
+		};
+
+		rst: reset-controller@e6160000 {
+			compatible = "renesas,r8a77970-rst";
+			reg = <0 0xe6160000 0 0x200>;
+		};
+
+		sysc: system-controller@e6180000 {
+			compatible = "renesas,r8a77970-sysc";
+			reg = <0 0xe6180000 0 0x440>;
+			#power-domain-cells = <1>;
+		};
+
 		intc_ex: interrupt-controller@e61c0000 {
 			compatible = "renesas,intc-ex-r8a77970", "renesas,irqc";
 			#interrupt-cells = <2>;
@@ -285,67 +249,6 @@
 			resets = <&cpg 407>;
 		};
 
-		prr: chipid@fff00044 {
-			compatible = "renesas,prr";
-			reg = <0 0xfff00044 0 4>;
-		};
-
-		dmac1: dma-controller@e7300000 {
-			compatible = "renesas,dmac-r8a77970",
-				     "renesas,rcar-dmac";
-			reg = <0 0xe7300000 0 0x10000>;
-			interrupts = <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 310 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "error",
-					  "ch0", "ch1", "ch2", "ch3",
-					  "ch4", "ch5", "ch6", "ch7";
-			clocks = <&cpg CPG_MOD 218>;
-			clock-names = "fck";
-			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
-			resets = <&cpg 218>;
-			#dma-cells = <1>;
-			dma-channels = <8>;
-			iommus = <&ipmmu_ds1 0>, <&ipmmu_ds1 1>,
-			       <&ipmmu_ds1 2>, <&ipmmu_ds1 3>,
-			       <&ipmmu_ds1 4>, <&ipmmu_ds1 5>,
-			       <&ipmmu_ds1 6>, <&ipmmu_ds1 7>;
-		};
-
-		dmac2: dma-controller@e7310000 {
-			compatible = "renesas,dmac-r8a77970",
-				     "renesas,rcar-dmac";
-			reg = <0 0xe7310000 0 0x10000>;
-			interrupts = <GIC_SPI 307 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 313 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH
-				      GIC_SPI 319 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "error",
-					  "ch0", "ch1", "ch2", "ch3",
-					  "ch4", "ch5", "ch6", "ch7";
-			clocks = <&cpg CPG_MOD 217>;
-			clock-names = "fck";
-			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
-			resets = <&cpg 217>;
-			#dma-cells = <1>;
-			dma-channels = <8>;
-			iommus = <&ipmmu_ds1 16>, <&ipmmu_ds1 17>,
-			       <&ipmmu_ds1 18>, <&ipmmu_ds1 19>,
-			       <&ipmmu_ds1 20>, <&ipmmu_ds1 21>,
-			       <&ipmmu_ds1 22>, <&ipmmu_ds1 23>;
-		};
-
 		i2c0: i2c@e6500000 {
 			compatible = "renesas,i2c-r8a77970",
 				     "renesas,rcar-gen3-i2c";
@@ -502,6 +405,77 @@
 			status = "disabled";
 		};
 
+		canfd: can@e66c0000 {
+			compatible = "renesas,r8a77970-canfd",
+				     "renesas,rcar-gen3-canfd";
+			reg = <0 0xe66c0000 0 0x8000>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 914>,
+				 <&cpg CPG_CORE R8A77970_CLK_CANFD>,
+				 <&can_clk>;
+			clock-names = "fck", "canfd", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A77970_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 914>;
+			status = "disabled";
+
+			channel0 {
+				status = "disabled";
+			};
+
+			channel1 {
+				status = "disabled";
+			};
+		};
+
+		avb: ethernet@e6800000 {
+			compatible = "renesas,etheravb-r8a77970",
+				     "renesas,etheravb-rcar-gen3";
+			reg = <0 0xe6800000 0 0x800>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14", "ch15",
+					  "ch16", "ch17", "ch18", "ch19",
+					  "ch20", "ch21", "ch22", "ch23",
+					  "ch24";
+			clocks = <&cpg CPG_MOD 812>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 812>;
+			phy-mode = "rgmii";
+			iommus = <&ipmmu_rt 3>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		scif0: serial@e6e60000 {
 			compatible = "renesas,scif-r8a77970",
 				     "renesas,rcar-gen3-scif",
@@ -573,57 +547,358 @@
 			status = "disabled";
 		};
 
-		avb: ethernet@e6800000 {
-			compatible = "renesas,etheravb-r8a77970",
-				     "renesas,etheravb-rcar-gen3";
-			reg = <0 0xe6800000 0 0x800>;
-			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "ch0", "ch1", "ch2", "ch3",
-					  "ch4", "ch5", "ch6", "ch7",
-					  "ch8", "ch9", "ch10", "ch11",
-					  "ch12", "ch13", "ch14", "ch15",
-					  "ch16", "ch17", "ch18", "ch19",
-					  "ch20", "ch21", "ch22", "ch23",
-					  "ch24";
-			clocks = <&cpg CPG_MOD 812>;
+
+		vin0: video@e6ef0000 {
+			compatible = "renesas,vin-r8a77970";
+			reg = <0 0xe6ef0000 0 0x1000>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 811>;
 			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
-			resets = <&cpg 812>;
-			phy-mode = "rgmii";
-			iommus = <&ipmmu_rt 3>;
-			#address-cells = <1>;
-			#size-cells = <0>;
+			resets = <&cpg 811>;
+			renesas,id = <0>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin0csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin0>;
+					};
+				};
+			};
+		};
+
+		vin1: video@e6ef1000 {
+			compatible = "renesas,vin-r8a77970";
+			reg = <0 0xe6ef1000 0 0x1000>;
+			interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 810>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 810>;
+			renesas,id = <1>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin1csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin1>;
+					};
+				};
+			};
+		};
+
+		vin2: video@e6ef2000 {
+			compatible = "renesas,vin-r8a77970";
+			reg = <0 0xe6ef2000 0 0x1000>;
+			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 809>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 809>;
+			renesas,id = <2>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin2csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin2>;
+					};
+				};
+			};
+		};
+
+		vin3: video@e6ef3000 {
+			compatible = "renesas,vin-r8a77970";
+			reg = <0 0xe6ef3000 0 0x1000>;
+			interrupts = <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 808>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 808>;
+			renesas,id = <3>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					vin3csi40: endpoint@2 {
+						reg = <2>;
+						remote-endpoint= <&csi40vin3>;
+					};
+				};
+			};
+		};
+
+		dmac1: dma-controller@e7300000 {
+			compatible = "renesas,dmac-r8a77970",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe7300000 0 0x10000>;
+			interrupts = <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 310 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7";
+			clocks = <&cpg CPG_MOD 218>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 218>;
+			#dma-cells = <1>;
+			dma-channels = <8>;
+			iommus = <&ipmmu_ds1 0>, <&ipmmu_ds1 1>,
+			       <&ipmmu_ds1 2>, <&ipmmu_ds1 3>,
+			       <&ipmmu_ds1 4>, <&ipmmu_ds1 5>,
+			       <&ipmmu_ds1 6>, <&ipmmu_ds1 7>;
+		};
+
+		dmac2: dma-controller@e7310000 {
+			compatible = "renesas,dmac-r8a77970",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe7310000 0 0x10000>;
+			interrupts = <GIC_SPI 307 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 313 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 319 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7";
+			clocks = <&cpg CPG_MOD 217>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 217>;
+			#dma-cells = <1>;
+			dma-channels = <8>;
+			iommus = <&ipmmu_ds1 16>, <&ipmmu_ds1 17>,
+			       <&ipmmu_ds1 18>, <&ipmmu_ds1 19>,
+			       <&ipmmu_ds1 20>, <&ipmmu_ds1 21>,
+			       <&ipmmu_ds1 22>, <&ipmmu_ds1 23>;
+		};
+
+		ipmmu_ds1: mmu@e7740000 {
+			compatible = "renesas,ipmmu-r8a77970";
+			reg = <0 0xe7740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 0>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_ir: mmu@ff8b0000 {
+			compatible = "renesas,ipmmu-r8a77970";
+			reg = <0 0xff8b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 3>;
+			power-domains = <&sysc R8A77970_PD_A3IR>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_mm: mmu@e67b0000 {
+			compatible = "renesas,ipmmu-r8a77970";
+			reg = <0 0xe67b0000 0 0x1000>;
+			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_rt: mmu@ffc80000 {
+			compatible = "renesas,ipmmu-r8a77970";
+			reg = <0 0xffc80000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 7>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vi0: mmu@febd0000 {
+			compatible = "renesas,ipmmu-r8a77970";
+			reg = <0 0xfebd0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 9>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		gic: interrupt-controller@f1010000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0 0xf1010000 0 0x1000>,
+			      <0 0xf1020000 0 0x20000>,
+			      <0 0xf1040000 0 0x20000>,
+			      <0 0xf1060000 0 0x20000>;
+			interrupts = <GIC_PPI 9	(GIC_CPU_MASK_SIMPLE(2) |
+				      IRQ_TYPE_LEVEL_HIGH)>;
+			clocks = <&cpg CPG_MOD 408>;
+			clock-names = "clk";
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 408>;
+		};
+
+		vspd0: vsp@fea20000 {
+			compatible = "renesas,vsp2";
+			reg = <0 0xfea20000 0 0x8000>;
+			interrupts = <GIC_SPI 169 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 623>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 623>;
+			renesas,fcp = <&fcpvd0>;
+		};
+
+		fcpvd0: fcp@fea27000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfea27000 0 0x200>;
+			clocks = <&cpg CPG_MOD 603>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 603>;
+		};
+
+		csi40: csi2@feaa0000 {
+			compatible = "renesas,r8a77970-csi2";
+			reg = <0 0xfeaa0000 0 0x10000>;
+			interrupts = <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 716>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 716>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					reg = <1>;
+
+					csi40vin0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&vin0csi40>;
+					};
+					csi40vin1: endpoint@1 {
+						reg = <1>;
+						remote-endpoint = <&vin1csi40>;
+					};
+					csi40vin2: endpoint@2 {
+						reg = <2>;
+						remote-endpoint = <&vin2csi40>;
+					};
+					csi40vin3: endpoint@3 {
+						reg = <3>;
+						remote-endpoint = <&vin3csi40>;
+					};
+				};
+			};
+		};
+
+		du: display@feb00000 {
+			compatible = "renesas,du-r8a77970";
+			reg = <0 0xfeb00000 0 0x80000>;
+			interrupts = <GIC_SPI 256 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 724>;
+			clock-names = "du.0";
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 724>;
+			vsps = <&vspd0>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					du_out_rgb: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					du_out_lvds0: endpoint {
+						remote-endpoint = <&lvds0_in>;
+					};
+				};
+			};
+		};
+
+		lvds0: lvds-encoder@feb90000 {
+			compatible = "renesas,r8a77970-lvds";
+			reg = <0 0xfeb90000 0 0x14>;
+			clocks = <&cpg CPG_MOD 727>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 727>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					lvds0_in: endpoint {
+						remote-endpoint =
+							<&du_out_lvds0>;
+					};
+				};
+				port@1 {
+					reg = <1>;
+					lvds0_out: endpoint {
+					};
+				};
+			};
+		};
+
+		prr: chipid@fff00044 {
+			compatible = "renesas,prr";
+			reg = <0 0xfff00044 0 4>;
 		};
 	};
 
 	timer {
 		compatible = "arm,armv8-timer";
-		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_LOW)>,
-				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_LOW)>;
+		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
 	};
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a77980-condor.dts b/arch/arm64/boot/dts/renesas/r8a77980-condor.dts
index 06cf6845765a..0b93a7d76585 100644
--- a/arch/arm64/boot/dts/renesas/r8a77980-condor.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77980-condor.dts
@@ -27,9 +27,30 @@
 		/* first 128MB is reserved for secure area. */
 		reg = <0 0x48000000 0 0x78000000>;
 	};
+
+	d3_3v: regulator-0 {
+		compatible = "regulator-fixed";
+		regulator-name = "D3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	vddq_vin01: regulator-1 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDQ_VIN01";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
 };
 
 &avb {
+	pinctrl-0 = <&avb_pins>;
+	pinctrl-names = "default";
+
 	phy-mode = "rgmii-id";
 	phy-handle = <&phy0>;
 	renesas,no-ether-link;
@@ -41,6 +62,16 @@
 	};
 };
 
+&canfd {
+	pinctrl-0 = <&canfd0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	channel0 {
+		status = "okay";
+	};
+};
+
 &extal_clk {
 	clock-frequency = <16666666>;
 };
@@ -49,7 +80,57 @@
 	clock-frequency = <32768>;
 };
 
+&mmc0 {
+	pinctrl-0 = <&mmc_pins>;
+	pinctrl-1 = <&mmc_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
+
+	vmmc-supply = <&d3_3v>;
+	vqmmc-supply = <&vddq_vin01>;
+	mmc-hs200-1_8v;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+&pfc {
+	avb_pins: avb {
+		groups = "avb_mdio", "avb_rgmii";
+		function = "avb";
+	};
+
+	canfd0_pins: canfd0 {
+		groups = "canfd0_data_a";
+		function = "canfd0";
+	};
+
+	mmc_pins: mmc {
+		groups = "mmc_data8", "mmc_ctrl", "mmc_ds";
+		function = "mmc";
+		power-source = <3300>;
+	};
+
+	mmc_pins_uhs: mmc_uhs {
+		groups = "mmc_data8", "mmc_ctrl", "mmc_ds";
+		function = "mmc";
+		power-source = <1800>;
+	};
+
+	scif0_pins: scif0 {
+		groups = "scif0_data";
+		function = "scif0";
+	};
+
+	scif_clk_pins: scif_clk {
+		groups = "scif_clk_b";
+		function = "scif_clk";
+	};
+};
+
 &scif0 {
+	pinctrl-0 = <&scif0_pins>, <&scif_clk_pins>;
+	pinctrl-names = "default";
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts b/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts
new file mode 100644
index 000000000000..c9680994555d
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for the V3H Starter Kit board
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ * Copyright (C) 2018 Cogent Embedded, Inc.
+ */
+
+/dts-v1/;
+#include "r8a77980.dtsi"
+
+/ {
+	model = "Renesas V3H Starter Kit board";
+	compatible = "renesas,v3hsk", "renesas,r8a77980";
+
+	aliases {
+		serial0 = &scif0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@48000000 {
+		device_type = "memory";
+		/* first 128MB is reserved for secure area. */
+		reg = <0 0x48000000 0 0x78000000>;
+	};
+};
+
+&extal_clk {
+	clock-frequency = <16666666>;
+};
+
+&extalr_clk {
+	clock-frequency = <32768>;
+};
+
+&pfc {
+	scif0_pins: scif0 {
+		groups = "scif0_data";
+		function = "scif0";
+	};
+
+	scif_clk_pins: scif_clk {
+		groups = "scif_clk_b";
+		function = "scif_clk";
+	};
+};
+
+&scif0 {
+	pinctrl-0 = <&scif0_pins>, <&scif_clk_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&scif_clk {
+	clock-frequency = <14745600>;
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77980.dtsi b/arch/arm64/boot/dts/renesas/r8a77980.dtsi
index 03845fd74996..4c40f9f0ebc9 100644
--- a/arch/arm64/boot/dts/renesas/r8a77980.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77980.dtsi
@@ -6,9 +6,10 @@
  * Copyright (C) 2018 Cogent Embedded, Inc.
  */
 
+#include <dt-bindings/clock/r8a77980-cpg-mssr.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
-#include <dt-bindings/clock/renesas-cpg-mssr.h>
+#include <dt-bindings/power/r8a77980-sysc.h>
 
 / {
 	compatible = "renesas,r8a77980";
@@ -23,20 +24,27 @@
 			device_type = "cpu";
 			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0>;
-			clocks = <&cpg CPG_CORE 0>;
-			power-domains = <&sysc 5>;
+			clocks = <&cpg CPG_CORE R8A77980_CLK_Z2>;
+			power-domains = <&sysc R8A77980_PD_CA53_CPU0>;
 			next-level-cache = <&L2_CA53>;
 			enable-method = "psci";
 		};
 
 		L2_CA53: cache-controller {
 			compatible = "cache";
-			power-domains = <&sysc 21>;
+			power-domains = <&sysc R8A77980_PD_CA53_SCU>;
 			cache-unified;
 			cache-level = <2>;
 		};
 	};
 
+	/* External CAN clock - to be overridden by boards that provide it */
+	can_clk: can {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
 	extal_clk: extal {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
@@ -71,6 +79,11 @@
 		#size-cells = <2>;
 		ranges;
 
+		pfc: pin-controller@e6060000 {
+			compatible = "renesas,pfc-r8a77980";
+			reg = <0 0xe6060000 0 0x50c>;
+		};
+
 		cpg: clock-controller@e6150000 {
 			compatible = "renesas,r8a77980-cpg-mssr";
 			reg = <0 0xe6150000 0 0x1000>;
@@ -99,13 +112,13 @@
 			reg = <0 0xe6540000 0 0x60>;
 			interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 520>,
-				 <&cpg CPG_CORE 19>,
+				 <&cpg CPG_CORE R8A77980_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x31>, <&dmac1 0x30>,
 			       <&dmac2 0x31>, <&dmac2 0x30>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 520>;
 			status = "disabled";
 		};
@@ -117,13 +130,13 @@
 			reg = <0 0xe6550000 0 0x60>;
 			interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 519>,
-				 <&cpg CPG_CORE 19>,
+				 <&cpg CPG_CORE R8A77980_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x33>, <&dmac1 0x32>,
 			       <&dmac2 0x33>, <&dmac2 0x32>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 519>;
 			status = "disabled";
 		};
@@ -135,13 +148,13 @@
 			reg = <0 0xe6560000 0 0x60>;
 			interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 518>,
-				 <&cpg CPG_CORE 19>,
+				 <&cpg CPG_CORE R8A77980_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x35>, <&dmac1 0x34>,
 			       <&dmac2 0x35>, <&dmac2 0x34>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 518>;
 			status = "disabled";
 		};
@@ -153,17 +166,42 @@
 			reg = <0 0xe66a0000 0 0x60>;
 			interrupts = <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 517>,
-				 <&cpg CPG_CORE 19>,
+				 <&cpg CPG_CORE R8A77980_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x37>, <&dmac1 0x36>,
 			       <&dmac2 0x37>, <&dmac2 0x36>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 517>;
 			status = "disabled";
 		};
 
+		canfd: can@e66c0000 {
+			compatible = "renesas,r8a77980-canfd",
+				     "renesas,rcar-gen3-canfd";
+			reg = <0 0xe66c0000 0 0x8000>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 914>,
+				 <&cpg CPG_CORE R8A77980_CLK_CANFD>,
+				 <&can_clk>;
+			clock-names = "fck", "canfd", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A77980_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
+			resets = <&cpg 914>;
+			status = "disabled";
+
+			channel0 {
+				status = "disabled";
+			};
+
+			channel1 {
+				status = "disabled";
+			};
+		};
+
 		avb: ethernet@e6800000 {
 			compatible = "renesas,etheravb-r8a77980",
 				     "renesas,etheravb-rcar-gen3";
@@ -201,11 +239,12 @@
 					  "ch20", "ch21", "ch22", "ch23",
 					  "ch24";
 			clocks = <&cpg CPG_MOD 812>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 812>;
 			phy-mode = "rgmii";
 			#address-cells = <1>;
 			#size-cells = <0>;
+			status = "disabled";
 		};
 
 		scif0: serial@e6e60000 {
@@ -215,13 +254,13 @@
 			reg = <0 0xe6e60000 0 0x40>;
 			interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 207>,
-				 <&cpg CPG_CORE 19>,
+				 <&cpg CPG_CORE R8A77980_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x51>, <&dmac1 0x50>,
 			       <&dmac2 0x51>, <&dmac2 0x50>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 207>;
 			status = "disabled";
 		};
@@ -233,13 +272,13 @@
 			reg = <0 0xe6e68000 0 0x40>;
 			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 206>,
-				 <&cpg CPG_CORE 19>,
+				 <&cpg CPG_CORE R8A77980_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x53>, <&dmac1 0x52>,
 			       <&dmac2 0x53>, <&dmac2 0x52>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 206>;
 			status = "disabled";
 		};
@@ -251,13 +290,13 @@
 			reg = <0 0xe6c50000 0 0x40>;
 			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 204>,
-				 <&cpg CPG_CORE 19>,
+				 <&cpg CPG_CORE R8A77980_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x57>, <&dmac1 0x56>,
 			       <&dmac2 0x57>, <&dmac2 0x56>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 204>;
 			status = "disabled";
 		};
@@ -269,13 +308,13 @@
 			reg = <0 0xe6c40000 0 0x40>;
 			interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 203>,
-				 <&cpg CPG_CORE 19>,
+				 <&cpg CPG_CORE R8A77980_CLK_S3D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x59>, <&dmac1 0x58>,
 			       <&dmac2 0x59>, <&dmac2 0x58>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 203>;
 			status = "disabled";
 		};
@@ -308,7 +347,7 @@
 					  "ch12", "ch13", "ch14", "ch15";
 			clocks = <&cpg CPG_MOD 218>;
 			clock-names = "fck";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 218>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
@@ -342,12 +381,24 @@
 					  "ch12", "ch13", "ch14", "ch15";
 			clocks = <&cpg CPG_MOD 217>;
 			clock-names = "fck";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 217>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
 		};
 
+		mmc0: mmc@ee140000 {
+			compatible = "renesas,sdhi-r8a77980",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee140000 0 0x2000>;
+			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 314>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
+			resets = <&cpg 314>;
+			max-frequency = <200000000>;
+			status = "disabled";
+		};
+
 		gic: interrupt-controller@f1010000 {
 			compatible = "arm,gic-400";
 			#interrupt-cells = <3>;
@@ -361,7 +412,7 @@
 				      IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&cpg CPG_MOD 408>;
 			clock-names = "clk";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 408>;
 		};
 
diff --git a/arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts b/arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts
new file mode 100644
index 000000000000..7a09d0524f9b
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Device Tree Source for the ebisu board
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ */
+
+/dts-v1/;
+#include "r8a77990.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "Renesas Ebisu board based on r8a77990";
+	compatible = "renesas,ebisu", "renesas,r8a77990";
+
+	aliases {
+		serial0 = &scif2;
+		ethernet0 = &avb;
+	};
+
+	chosen {
+		bootargs = "ignore_loglevel";
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@48000000 {
+		device_type = "memory";
+		/* first 128MB is reserved for secure area. */
+		reg = <0x0 0x48000000 0x0 0x38000000>;
+	};
+};
+
+&avb {
+	pinctrl-0 = <&avb_pins>;
+	pinctrl-names = "default";
+	renesas,no-ether-link;
+	phy-handle = <&phy0>;
+	phy-mode = "rgmii-txid";
+	status = "okay";
+
+	phy0: ethernet-phy@0 {
+		rxc-skew-ps = <1500>;
+		reg = <0>;
+		interrupt-parent = <&gpio2>;
+		interrupts = <21 IRQ_TYPE_LEVEL_LOW>;
+		reset-gpios = <&gpio1 20 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&extal_clk {
+	clock-frequency = <48000000>;
+};
+
+&pfc {
+	avb_pins: avb {
+		mux {
+			groups = "avb_link", "avb_mii";
+			function = "avb";
+		};
+	};
+};
+
+&scif2 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77990.dtsi b/arch/arm64/boot/dts/renesas/r8a77990.dtsi
new file mode 100644
index 000000000000..be4f519711a1
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a77990.dtsi
@@ -0,0 +1,281 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Device Tree Source for the r8a77990 SoC
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ */
+
+#include <dt-bindings/clock/renesas-cpg-mssr.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	compatible = "renesas,r8a77990";
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* 1 core only at this point */
+		a53_0: cpu@0 {
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0>;
+			device_type = "cpu";
+			power-domains = <&sysc 5>;
+			next-level-cache = <&L2_CA53>;
+			enable-method = "psci";
+		};
+
+		L2_CA53: cache-controller-0 {
+			compatible = "cache";
+			power-domains = <&sysc 21>;
+			cache-unified;
+			cache-level = <2>;
+		};
+	};
+
+	extal_clk: extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board */
+		clock-frequency = <0>;
+	};
+
+	pmu_a53 {
+		compatible = "arm,cortex-a53-pmu";
+		interrupts-extended = <&gic GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&a53_0>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0", "arm,psci-0.2";
+		method = "smc";
+	};
+
+	soc: soc {
+		compatible = "simple-bus";
+		interrupt-parent = <&gic>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		gpio0: gpio@e6050000 {
+			compatible = "renesas,gpio-r8a77990",
+				     "renesas,rcar-gen3-gpio";
+			reg = <0 0xe6050000 0 0x50>;
+			interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 0 18>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 912>;
+			power-domains = <&sysc 32>;
+			resets = <&cpg 912>;
+		};
+
+		gpio1: gpio@e6051000 {
+			compatible = "renesas,gpio-r8a77990",
+				     "renesas,rcar-gen3-gpio";
+			reg = <0 0xe6051000 0 0x50>;
+			interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 32 23>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 911>;
+			power-domains = <&sysc 32>;
+			resets = <&cpg 911>;
+		};
+
+		gpio2: gpio@e6052000 {
+			compatible = "renesas,gpio-r8a77990",
+				     "renesas,rcar-gen3-gpio";
+			reg = <0 0xe6052000 0 0x50>;
+			interrupts = <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 64 26>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 910>;
+			power-domains = <&sysc 32>;
+			resets = <&cpg 910>;
+		};
+
+		gpio3: gpio@e6053000 {
+			compatible = "renesas,gpio-r8a77990",
+				     "renesas,rcar-gen3-gpio";
+			reg = <0 0xe6053000 0 0x50>;
+			interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 96 16>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 909>;
+			power-domains = <&sysc 32>;
+			resets = <&cpg 909>;
+		};
+
+		gpio4: gpio@e6054000 {
+			compatible = "renesas,gpio-r8a77990",
+				     "renesas,rcar-gen3-gpio";
+			reg = <0 0xe6054000 0 0x50>;
+			interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 128 11>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 908>;
+			power-domains = <&sysc 32>;
+			resets = <&cpg 908>;
+		};
+
+		gpio5: gpio@e6055000 {
+			compatible = "renesas,gpio-r8a77990",
+				     "renesas,rcar-gen3-gpio";
+			reg = <0 0xe6055000 0 0x50>;
+			interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 160 20>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 907>;
+			power-domains = <&sysc 32>;
+			resets = <&cpg 907>;
+		};
+
+		gpio6: gpio@e6055400 {
+			compatible = "renesas,gpio-r8a77990",
+				     "renesas,rcar-gen3-gpio";
+			reg = <0 0xe6055400 0 0x50>;
+			interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 192 18>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 906>;
+			power-domains = <&sysc 32>;
+			resets = <&cpg 906>;
+		};
+
+		pfc: pin-controller@e6060000 {
+			compatible = "renesas,pfc-r8a77990";
+			reg = <0 0xe6060000 0 0x508>;
+		};
+
+		cpg: clock-controller@e6150000 {
+			compatible = "renesas,r8a77990-cpg-mssr";
+			reg = <0 0xe6150000 0 0x1000>;
+			clocks = <&extal_clk>;
+			clock-names = "extal";
+			#clock-cells = <2>;
+			#power-domain-cells = <0>;
+			#reset-cells = <1>;
+		};
+
+		rst: reset-controller@e6160000 {
+			compatible = "renesas,r8a77990-rst";
+			reg = <0 0xe6160000 0 0x0200>;
+		};
+
+		sysc: system-controller@e6180000 {
+			compatible = "renesas,r8a77990-sysc";
+			reg = <0 0xe6180000 0 0x0400>;
+			#power-domain-cells = <1>;
+		};
+
+		avb: ethernet@e6800000 {
+			compatible = "renesas,etheravb-r8a77990",
+				     "renesas,etheravb-rcar-gen3";
+			reg = <0 0xe6800000 0 0x800>, <0 0xe6a00000 0 0x10000>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14", "ch15",
+					  "ch16", "ch17", "ch18", "ch19",
+					  "ch20", "ch21", "ch22", "ch23",
+					  "ch24";
+			clocks = <&cpg CPG_MOD 812>;
+			power-domains = <&sysc 32>;
+			resets = <&cpg 812>;
+			phy-mode = "rgmii";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		scif2: serial@e6e88000 {
+			compatible = "renesas,scif-r8a77990",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6e88000 0 64>;
+			interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 310>;
+			clock-names = "fck";
+			power-domains = <&sysc 32>;
+			resets = <&cpg 310>;
+			status = "disabled";
+		};
+
+		gic: interrupt-controller@f1010000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0x0 0xf1010000 0 0x1000>,
+			      <0x0 0xf1020000 0 0x20000>,
+			      <0x0 0xf1040000 0 0x20000>,
+			      <0x0 0xf1060000 0 0x20000>;
+			interrupts = <GIC_PPI 9
+					(GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_HIGH)>;
+			clocks = <&cpg CPG_MOD 408>;
+			clock-names = "clk";
+			power-domains = <&sysc 32>;
+			resets = <&cpg 408>;
+		};
+
+		prr: chipid@fff00044 {
+			compatible = "renesas,prr";
+			reg = <0 0xfff00044 0 4>;
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_LOW)>;
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77995-draak.dts b/arch/arm64/boot/dts/renesas/r8a77995-draak.dts
index d03f19414028..9d73de8bc94d 100644
--- a/arch/arm64/boot/dts/renesas/r8a77995-draak.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77995-draak.dts
@@ -91,7 +91,7 @@
 &pfc {
 	avb0_pins: avb {
 		mux {
-			groups = "avb0_link", "avb0_mdc", "avb0_mii";
+			groups = "avb0_link", "avb0_mdio", "avb0_mii";
 			function = "avb0";
 		};
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a77995.dtsi b/arch/arm64/boot/dts/renesas/r8a77995.dtsi
index 82aed7ee984c..2506f46293e8 100644
--- a/arch/arm64/boot/dts/renesas/r8a77995.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77995.dtsi
@@ -18,9 +18,11 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
-	psci {
-		compatible = "arm,psci-1.0", "arm,psci-0.2";
-		method = "smc";
+	/* External CAN clock - to be overridden by boards that provide it */
+	can_clk: can {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
 	};
 
 	cpus {
@@ -51,18 +53,16 @@
 		clock-frequency = <0>;
 	};
 
-	/* External CAN clock - to be overridden by boards that provide it */
-	can_clk: can {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
-	};
-
 	pmu_a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts-extended = <&gic GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
+	psci {
+		compatible = "arm,psci-1.0", "arm,psci-0.2";
+		method = "smc";
+	};
+
 	scif_clk: scif {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
@@ -76,23 +76,6 @@
 		#size-cells = <2>;
 		ranges;
 
-		gic: interrupt-controller@f1010000 {
-			compatible = "arm,gic-400";
-			#interrupt-cells = <3>;
-			#address-cells = <0>;
-			interrupt-controller;
-			reg = <0x0 0xf1010000 0 0x1000>,
-			      <0x0 0xf1020000 0 0x20000>,
-			      <0x0 0xf1040000 0 0x20000>,
-			      <0x0 0xf1060000 0 0x20000>;
-			interrupts = <GIC_PPI 9
-					(GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_HIGH)>;
-			clocks = <&cpg CPG_MOD 408>;
-			clock-names = "clk";
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 408>;
-		};
-
 		rwdt: watchdog@e6020000 {
 			compatible = "renesas,r8a77995-wdt",
 				     "renesas,rcar-gen3-wdt";
@@ -103,88 +86,123 @@
 			status = "disabled";
 		};
 
-		ipmmu_vi0: mmu@febd0000 {
-			compatible = "renesas,ipmmu-r8a77995";
-			reg = <0 0xfebd0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 14>;
-			#iommu-cells = <1>;
-			status = "disabled";
-		};
-
-		ipmmu_vp0: mmu@fe990000 {
-			compatible = "renesas,ipmmu-r8a77995";
-			reg = <0 0xfe990000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 16>;
-			#iommu-cells = <1>;
-			status = "disabled";
-		};
-
-		ipmmu_vc0: mmu@fe6b0000 {
-			compatible = "renesas,ipmmu-r8a77995";
-			reg = <0 0xfe6b0000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 12>;
-			#iommu-cells = <1>;
-			status = "disabled";
+		gpio0: gpio@e6050000 {
+			compatible = "renesas,gpio-r8a77995",
+				     "renesas,rcar-gen3-gpio",
+				     "renesas,gpio-rcar";
+			reg = <0 0xe6050000 0 0x50>;
+			interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 0 9>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 912>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 912>;
 		};
 
-		ipmmu_pv0: mmu@fd800000 {
-			compatible = "renesas,ipmmu-r8a77995";
-			reg = <0 0xfd800000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 6>;
-			#iommu-cells = <1>;
-			status = "disabled";
+		gpio1: gpio@e6051000 {
+			compatible = "renesas,gpio-r8a77995",
+				     "renesas,rcar-gen3-gpio",
+				     "renesas,gpio-rcar";
+			reg = <0 0xe6051000 0 0x50>;
+			interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 32 32>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 911>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 911>;
 		};
 
-		ipmmu_hc: mmu@e6570000 {
-			compatible = "renesas,ipmmu-r8a77995";
-			reg = <0 0xe6570000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 2>;
-			#iommu-cells = <1>;
-			status = "disabled";
+		gpio2: gpio@e6052000 {
+			compatible = "renesas,gpio-r8a77995",
+				     "renesas,rcar-gen3-gpio",
+				     "renesas,gpio-rcar";
+			reg = <0 0xe6052000 0 0x50>;
+			interrupts = <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 64 32>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 910>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 910>;
 		};
 
-		ipmmu_rt: mmu@ffc80000 {
-			compatible = "renesas,ipmmu-r8a77995";
-			reg = <0 0xffc80000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 10>;
-			#iommu-cells = <1>;
-			status = "disabled";
+		gpio3: gpio@e6053000 {
+			compatible = "renesas,gpio-r8a77995",
+				     "renesas,rcar-gen3-gpio",
+				     "renesas,gpio-rcar";
+			reg = <0 0xe6053000 0 0x50>;
+			interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 96 10>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 909>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 909>;
 		};
 
-		ipmmu_mp: mmu@ec670000 {
-			compatible = "renesas,ipmmu-r8a77995";
-			reg = <0 0xec670000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 4>;
-			#iommu-cells = <1>;
-			status = "disabled";
+		gpio4: gpio@e6054000 {
+			compatible = "renesas,gpio-r8a77995",
+				     "renesas,rcar-gen3-gpio",
+				     "renesas,gpio-rcar";
+			reg = <0 0xe6054000 0 0x50>;
+			interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 128 32>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 908>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 908>;
 		};
 
-		ipmmu_ds0: mmu@e6740000 {
-			compatible = "renesas,ipmmu-r8a77995";
-			reg = <0 0xe6740000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 0>;
-			#iommu-cells = <1>;
-			status = "disabled";
+		gpio5: gpio@e6055000 {
+			compatible = "renesas,gpio-r8a77995",
+				     "renesas,rcar-gen3-gpio",
+				     "renesas,gpio-rcar";
+			reg = <0 0xe6055000 0 0x50>;
+			interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 160 21>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 907>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 907>;
 		};
 
-		ipmmu_ds1: mmu@e7740000 {
-			compatible = "renesas,ipmmu-r8a77995";
-			reg = <0 0xe7740000 0 0x1000>;
-			renesas,ipmmu-main = <&ipmmu_mm 1>;
-			#iommu-cells = <1>;
-			status = "disabled";
+		gpio6: gpio@e6055400 {
+			compatible = "renesas,gpio-r8a77995",
+				     "renesas,rcar-gen3-gpio",
+				     "renesas,gpio-rcar";
+			reg = <0 0xe6055400 0 0x50>;
+			interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pfc 0 192 14>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			clocks = <&cpg CPG_MOD 906>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 906>;
 		};
 
-		ipmmu_mm: mmu@e67b0000 {
-			compatible = "renesas,ipmmu-r8a77995";
-			reg = <0 0xe67b0000 0 0x1000>;
-			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
-			#iommu-cells = <1>;
-			status = "disabled";
+		pfc: pin-controller@e6060000 {
+			compatible = "renesas,pfc-r8a77995";
+			reg = <0 0xe6060000 0 0x508>;
 		};
 
-
 		cpg: clock-controller@e6150000 {
 			compatible = "renesas,r8a77995-cpg-mssr";
 			reg = <0 0xe6150000 0 0x1000>;
@@ -200,16 +218,6 @@
 			reg = <0 0xe6160000 0 0x0200>;
 		};
 
-		pfc: pin-controller@e6060000 {
-			compatible = "renesas,pfc-r8a77995";
-			reg = <0 0xe6060000 0 0x508>;
-		};
-
-		prr: chipid@fff00044 {
-			compatible = "renesas,prr";
-			reg = <0 0xfff00044 0 4>;
-		};
-
 		sysc: system-controller@e6180000 {
 			compatible = "renesas,r8a77995-sysc";
 			reg = <0 0xe6180000 0 0x0400>;
@@ -232,6 +240,98 @@
 			resets = <&cpg 407>;
 		};
 
+		i2c0: i2c@e6500000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a77995",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe6500000 0 0x40>;
+			interrupts = <GIC_SPI 287 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 931>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 931>;
+			dmas = <&dmac1 0x91>, <&dmac1 0x90>,
+			       <&dmac2 0x91>, <&dmac2 0x90>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
+			status = "disabled";
+		};
+
+		i2c1: i2c@e6508000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a77995",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe6508000 0 0x40>;
+			interrupts = <GIC_SPI 288 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 930>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 930>;
+			dmas = <&dmac1 0x93>, <&dmac1 0x92>,
+			       <&dmac2 0x93>, <&dmac2 0x92>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
+			status = "disabled";
+		};
+
+		i2c2: i2c@e6510000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a77995",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe6510000 0 0x40>;
+			interrupts = <GIC_SPI 286 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 929>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 929>;
+			dmas = <&dmac1 0x95>, <&dmac1 0x94>,
+			       <&dmac2 0x95>, <&dmac2 0x94>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
+			status = "disabled";
+		};
+
+		i2c3: i2c@e66d0000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a77995",
+				     "renesas,rcar-gen3-i2c";
+			reg = <0 0xe66d0000 0 0x40>;
+			interrupts = <GIC_SPI 290 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 928>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 928>;
+			dmas = <&dmac0 0x97>, <&dmac0 0x96>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
+			status = "disabled";
+		};
+
+		canfd: can@e66c0000 {
+			compatible = "renesas,r8a77995-canfd",
+				     "renesas,rcar-gen3-canfd";
+			reg = <0 0xe66c0000 0 0x8000>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>,
+				   <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 914>,
+			       <&cpg CPG_CORE R8A77995_CLK_CANFD>,
+			       <&can_clk>;
+			clock-names = "fck", "canfd", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A77995_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 914>;
+			status = "disabled";
+
+			channel0 {
+				status = "disabled";
+			};
+
+			channel1 {
+				status = "disabled";
+			};
+		};
+
 		dmac0: dma-controller@e6700000 {
 			compatible = "renesas,dmac-r8a77995",
 				     "renesas,rcar-dmac";
@@ -304,173 +404,75 @@
 			dma-channels = <8>;
 		};
 
-		gpio0: gpio@e6050000 {
-			compatible = "renesas,gpio-r8a77995",
-				     "renesas,rcar-gen3-gpio",
-				     "renesas,gpio-rcar";
-			reg = <0 0xe6050000 0 0x50>;
-			interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
-			#gpio-cells = <2>;
-			gpio-controller;
-			gpio-ranges = <&pfc 0 0 9>;
-			#interrupt-cells = <2>;
-			interrupt-controller;
-			clocks = <&cpg CPG_MOD 912>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 912>;
-		};
-
-		gpio1: gpio@e6051000 {
-			compatible = "renesas,gpio-r8a77995",
-				     "renesas,rcar-gen3-gpio",
-				     "renesas,gpio-rcar";
-			reg = <0 0xe6051000 0 0x50>;
-			interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
-			#gpio-cells = <2>;
-			gpio-controller;
-			gpio-ranges = <&pfc 0 32 32>;
-			#interrupt-cells = <2>;
-			interrupt-controller;
-			clocks = <&cpg CPG_MOD 911>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 911>;
-		};
-
-		gpio2: gpio@e6052000 {
-			compatible = "renesas,gpio-r8a77995",
-				     "renesas,rcar-gen3-gpio",
-				     "renesas,gpio-rcar";
-			reg = <0 0xe6052000 0 0x50>;
-			interrupts = <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
-			#gpio-cells = <2>;
-			gpio-controller;
-			gpio-ranges = <&pfc 0 64 32>;
-			#interrupt-cells = <2>;
-			interrupt-controller;
-			clocks = <&cpg CPG_MOD 910>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 910>;
-		};
-
-		gpio3: gpio@e6053000 {
-			compatible = "renesas,gpio-r8a77995",
-				     "renesas,rcar-gen3-gpio",
-				     "renesas,gpio-rcar";
-			reg = <0 0xe6053000 0 0x50>;
-			interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
-			#gpio-cells = <2>;
-			gpio-controller;
-			gpio-ranges = <&pfc 0 96 10>;
-			#interrupt-cells = <2>;
-			interrupt-controller;
-			clocks = <&cpg CPG_MOD 909>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 909>;
+		ipmmu_ds0: mmu@e6740000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xe6740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 0>;
+			#iommu-cells = <1>;
 		};
 
-		gpio4: gpio@e6054000 {
-			compatible = "renesas,gpio-r8a77995",
-				     "renesas,rcar-gen3-gpio",
-				     "renesas,gpio-rcar";
-			reg = <0 0xe6054000 0 0x50>;
-			interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
-			#gpio-cells = <2>;
-			gpio-controller;
-			gpio-ranges = <&pfc 0 128 32>;
-			#interrupt-cells = <2>;
-			interrupt-controller;
-			clocks = <&cpg CPG_MOD 908>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 908>;
+		ipmmu_ds1: mmu@e7740000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xe7740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 1>;
+			#iommu-cells = <1>;
 		};
 
-		gpio5: gpio@e6055000 {
-			compatible = "renesas,gpio-r8a77995",
-				     "renesas,rcar-gen3-gpio",
-				     "renesas,gpio-rcar";
-			reg = <0 0xe6055000 0 0x50>;
-			interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
-			#gpio-cells = <2>;
-			gpio-controller;
-			gpio-ranges = <&pfc 0 160 21>;
-			#interrupt-cells = <2>;
-			interrupt-controller;
-			clocks = <&cpg CPG_MOD 907>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 907>;
+		ipmmu_hc: mmu@e6570000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xe6570000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 2>;
+			#iommu-cells = <1>;
 		};
 
-		gpio6: gpio@e6055400 {
-			compatible = "renesas,gpio-r8a77995",
-				     "renesas,rcar-gen3-gpio",
-				     "renesas,gpio-rcar";
-			reg = <0 0xe6055400 0 0x50>;
-			interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
-			#gpio-cells = <2>;
-			gpio-controller;
-			gpio-ranges = <&pfc 0 192 14>;
-			#interrupt-cells = <2>;
-			interrupt-controller;
-			clocks = <&cpg CPG_MOD 906>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 906>;
+		ipmmu_mm: mmu@e67b0000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xe67b0000 0 0x1000>;
+			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+			#iommu-cells = <1>;
 		};
 
-		can0: can@e6c30000 {
-			compatible = "renesas,can-r8a77995",
-				     "renesas,rcar-gen3-can";
-			reg = <0 0xe6c30000 0 0x1000>;
-			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 916>,
-			       <&cpg CPG_CORE R8A77995_CLK_CANFD>,
-			       <&can_clk>;
-			clock-names = "clkp1", "clkp2", "can_clk";
-			assigned-clocks = <&cpg CPG_CORE R8A77995_CLK_CANFD>;
-			assigned-clock-rates = <40000000>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 916>;
-			status = "disabled";
+		ipmmu_mp: mmu@ec670000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xec670000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 4>;
+			#iommu-cells = <1>;
 		};
 
-		can1: can@e6c38000 {
-			compatible = "renesas,can-r8a77995",
-				     "renesas,rcar-gen3-can";
-			reg = <0 0xe6c38000 0 0x1000>;
-			interrupts = <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 915>,
-			       <&cpg CPG_CORE R8A77995_CLK_CANFD>,
-			       <&can_clk>;
-			clock-names = "clkp1", "clkp2", "can_clk";
-			assigned-clocks = <&cpg CPG_CORE R8A77995_CLK_CANFD>;
-			assigned-clock-rates = <40000000>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 915>;
-			status = "disabled";
+		ipmmu_pv0: mmu@fd800000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xfd800000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 6>;
+			#iommu-cells = <1>;
 		};
 
-		canfd: can@e66c0000 {
-			compatible = "renesas,r8a77995-canfd",
-				     "renesas,rcar-gen3-canfd";
-			reg = <0 0xe66c0000 0 0x8000>;
-			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>,
-				   <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 914>,
-			       <&cpg CPG_CORE R8A77995_CLK_CANFD>,
-			       <&can_clk>;
-			clock-names = "fck", "canfd", "can_clk";
-			assigned-clocks = <&cpg CPG_CORE R8A77995_CLK_CANFD>;
-			assigned-clock-rates = <40000000>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 914>;
-			status = "disabled";
+		ipmmu_rt: mmu@ffc80000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xffc80000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 10>;
+			#iommu-cells = <1>;
+		};
 
-			channel0 {
-				status = "disabled";
-			};
+		ipmmu_vc0: mmu@fe6b0000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xfe6b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 12>;
+			#iommu-cells = <1>;
+		};
 
-			channel1 {
-				status = "disabled";
-			};
+		ipmmu_vi0: mmu@febd0000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xfebd0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 14>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vp0: mmu@fe990000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xfe990000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 16>;
+			#iommu-cells = <1>;
 		};
 
 		avb: ethernet@e6800000 {
@@ -519,87 +521,35 @@
 			status = "disabled";
 		};
 
-		scif2: serial@e6e88000 {
-			compatible = "renesas,scif-r8a77995",
-				     "renesas,rcar-gen3-scif", "renesas,scif";
-			reg = <0 0xe6e88000 0 64>;
-			interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 310>,
-				 <&cpg CPG_CORE R8A77995_CLK_S3D1C>,
-				 <&scif_clk>;
-			clock-names = "fck", "brg_int", "scif_clk";
-			dmas = <&dmac1 0x13>, <&dmac1 0x12>,
-			       <&dmac2 0x13>, <&dmac2 0x12>;
-			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 310>;
-			status = "disabled";
-		};
-
-		i2c0: i2c@e6500000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a77995",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe6500000 0 0x40>;
-			interrupts = <GIC_SPI 287 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 931>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 931>;
-			dmas = <&dmac1 0x91>, <&dmac1 0x90>,
-			       <&dmac2 0x91>, <&dmac2 0x90>;
-			dma-names = "tx", "rx", "tx", "rx";
-			i2c-scl-internal-delay-ns = <6>;
-			status = "disabled";
-		};
-
-		i2c1: i2c@e6508000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a77995",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe6508000 0 0x40>;
-			interrupts = <GIC_SPI 288 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 930>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 930>;
-			dmas = <&dmac1 0x93>, <&dmac1 0x92>,
-			       <&dmac2 0x93>, <&dmac2 0x92>;
-			dma-names = "tx", "rx", "tx", "rx";
-			i2c-scl-internal-delay-ns = <6>;
-			status = "disabled";
-		};
-
-		i2c2: i2c@e6510000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a77995",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe6510000 0 0x40>;
-			interrupts = <GIC_SPI 286 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 929>;
+		can0: can@e6c30000 {
+			compatible = "renesas,can-r8a77995",
+				     "renesas,rcar-gen3-can";
+			reg = <0 0xe6c30000 0 0x1000>;
+			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 916>,
+			       <&cpg CPG_CORE R8A77995_CLK_CANFD>,
+			       <&can_clk>;
+			clock-names = "clkp1", "clkp2", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A77995_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
 			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 929>;
-			dmas = <&dmac1 0x95>, <&dmac1 0x94>,
-			       <&dmac2 0x95>, <&dmac2 0x94>;
-			dma-names = "tx", "rx", "tx", "rx";
-			i2c-scl-internal-delay-ns = <6>;
+			resets = <&cpg 916>;
 			status = "disabled";
 		};
 
-		i2c3: i2c@e66d0000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "renesas,i2c-r8a77995",
-				     "renesas,rcar-gen3-i2c";
-			reg = <0 0xe66d0000 0 0x40>;
-			interrupts = <GIC_SPI 290 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 928>;
+		can1: can@e6c38000 {
+			compatible = "renesas,can-r8a77995",
+				     "renesas,rcar-gen3-can";
+			reg = <0 0xe6c38000 0 0x1000>;
+			interrupts = <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 915>,
+			       <&cpg CPG_CORE R8A77995_CLK_CANFD>,
+			       <&can_clk>;
+			clock-names = "clkp1", "clkp2", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A77995_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
 			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 928>;
-			dmas = <&dmac0 0x97>, <&dmac0 0x96>;
-			dma-names = "tx", "rx";
-			i2c-scl-internal-delay-ns = <6>;
+			resets = <&cpg 915>;
 			status = "disabled";
 		};
 
@@ -643,38 +593,54 @@
 			status = "disabled";
 		};
 
-		sdhi2: sd@ee140000 {
-			compatible = "renesas,sdhi-r8a77995",
-				     "renesas,rcar-gen3-sdhi";
-			reg = <0 0xee140000 0 0x2000>;
-			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&cpg CPG_MOD 312>;
-			max-frequency = <200000000>;
+		scif2: serial@e6e88000 {
+			compatible = "renesas,scif-r8a77995",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6e88000 0 64>;
+			interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 310>,
+				 <&cpg CPG_CORE R8A77995_CLK_S3D1C>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac1 0x13>, <&dmac1 0x12>,
+			       <&dmac2 0x13>, <&dmac2 0x12>;
+			dma-names = "tx", "rx", "tx", "rx";
 			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 312>;
+			resets = <&cpg 310>;
 			status = "disabled";
 		};
 
-		ehci0: usb@ee080100 {
-			compatible = "generic-ehci";
-			reg = <0 0xee080100 0 0x100>;
+		vin4: video@e6ef4000 {
+			compatible = "renesas,vin-r8a77995";
+			reg = <0 0xe6ef4000 0 0x1000>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 807>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 807>;
+			renesas,id = <4>;
+			status = "disabled";
+		};
+
+		ohci0: usb@ee080000 {
+			compatible = "generic-ohci";
+			reg = <0 0xee080000 0 0x100>;
 			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 703>;
 			phys = <&usb2_phy0>;
 			phy-names = "usb";
-			companion = <&ohci0>;
 			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
 			resets = <&cpg 703>;
 			status = "disabled";
 		};
 
-		ohci0: usb@ee080000 {
-			compatible = "generic-ohci";
-			reg = <0 0xee080000 0 0x100>;
+		ehci0: usb@ee080100 {
+			compatible = "generic-ehci";
+			reg = <0 0xee080100 0 0x100>;
 			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 703>;
 			phys = <&usb2_phy0>;
 			phy-names = "usb";
+			companion = <&ohci0>;
 			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
 			resets = <&cpg 703>;
 			status = "disabled";
@@ -692,6 +658,35 @@
 			status = "disabled";
 		};
 
+		sdhi2: sd@ee140000 {
+			compatible = "renesas,sdhi-r8a77995",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee140000 0 0x2000>;
+			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 312>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 312>;
+			status = "disabled";
+		};
+
+		gic: interrupt-controller@f1010000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0x0 0xf1010000 0 0x1000>,
+			      <0x0 0xf1020000 0 0x20000>,
+			      <0x0 0xf1040000 0 0x20000>,
+			      <0x0 0xf1060000 0 0x20000>;
+			interrupts = <GIC_PPI 9
+					(GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_HIGH)>;
+			clocks = <&cpg CPG_MOD 408>;
+			clock-names = "clk";
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 408>;
+		};
+
 		vspbs: vsp@fe960000 {
 			compatible = "renesas,vsp2";
 			reg = <0 0xfe960000 0 0x8000>;
@@ -702,15 +697,6 @@
 			renesas,fcp = <&fcpvb0>;
 		};
 
-		fcpvb0: fcp@fe96f000 {
-			compatible = "renesas,fcpv";
-			reg = <0 0xfe96f000 0 0x200>;
-			clocks = <&cpg CPG_MOD 607>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 607>;
-			iommus = <&ipmmu_vp0 5>;
-		};
-
 		vspd0: vsp@fea20000 {
 			compatible = "renesas,vsp2";
 			reg = <0 0xfea20000 0 0x8000>;
@@ -721,15 +707,6 @@
 			renesas,fcp = <&fcpvd0>;
 		};
 
-		fcpvd0: fcp@fea27000 {
-			compatible = "renesas,fcpv";
-			reg = <0 0xfea27000 0 0x200>;
-			clocks = <&cpg CPG_MOD 603>;
-			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
-			resets = <&cpg 603>;
-			iommus = <&ipmmu_vi0 8>;
-		};
-
 		vspd1: vsp@fea28000 {
 			compatible = "renesas,vsp2";
 			reg = <0 0xfea28000 0 0x8000>;
@@ -740,6 +717,24 @@
 			renesas,fcp = <&fcpvd1>;
 		};
 
+		fcpvb0: fcp@fe96f000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfe96f000 0 0x200>;
+			clocks = <&cpg CPG_MOD 607>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 607>;
+			iommus = <&ipmmu_vp0 5>;
+		};
+
+		fcpvd0: fcp@fea27000 {
+			compatible = "renesas,fcpv";
+			reg = <0 0xfea27000 0 0x200>;
+			clocks = <&cpg CPG_MOD 603>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 603>;
+			iommus = <&ipmmu_vi0 8>;
+		};
+
 		fcpvd1: fcp@fea2f000 {
 			compatible = "renesas,fcpv";
 			reg = <0 0xfea2f000 0 0x200>;
@@ -783,6 +778,11 @@
 				};
 			};
 		};
+
+		prr: chipid@fff00044 {
+			compatible = "renesas,prr";
+			reg = <0 0xfff00044 0 4>;
+		};
 	};
 
 	timer {
diff --git a/arch/arm64/boot/dts/renesas/salvator-common.dtsi b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
index 2a7f36abd2dd..9256fbaaab7f 100644
--- a/arch/arm64/boot/dts/renesas/salvator-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
@@ -66,6 +66,29 @@
 		enable-gpios = <&gpio6 7 GPIO_ACTIVE_HIGH>;
 	};
 
+	cvbs-in {
+		compatible = "composite-video-connector";
+		label = "CVBS IN";
+
+		port {
+			cvbs_con: endpoint {
+				remote-endpoint = <&adv7482_ain7>;
+			};
+		};
+	};
+
+	hdmi-in {
+		compatible = "hdmi-connector";
+		label = "HDMI IN";
+		type = "a";
+
+		port {
+			hdmi_in_con: endpoint {
+				remote-endpoint = <&adv7482_hdmi>;
+			};
+		};
+	};
+
 	reg_1p8v: regulator0 {
 		compatible = "regulator-fixed";
 		regulator-name = "fixed-1.8V";
@@ -93,20 +116,12 @@
 		regulator-always-on;
 	};
 
-	rsnd_ak4613: sound {
-		compatible = "simple-audio-card";
-
-		simple-audio-card,format = "left_j";
-		simple-audio-card,bitclock-master = <&sndcpu>;
-		simple-audio-card,frame-master = <&sndcpu>;
+	sound_card: sound {
+		compatible = "audio-graph-card";
 
-		sndcpu: simple-audio-card,cpu {
-			sound-dai = <&rcar_sound>;
-		};
+		label = "rcar-sound";
 
-		sndcodec: simple-audio-card,codec {
-			sound-dai = <&ak4613>;
-		};
+		dais = <&rsnd_port0>;
 	};
 
 	vbus0_usb2: regulator-vbus0-usb2 {
@@ -268,6 +283,37 @@
 	};
 };
 
+&csi20 {
+	status = "okay";
+
+	ports {
+		port@0 {
+			reg = <0>;
+			csi20_in: endpoint {
+				clock-lanes = <0>;
+				data-lanes = <1>;
+				remote-endpoint = <&adv7482_txb>;
+			};
+		};
+	};
+};
+
+&csi40 {
+	status = "okay";
+
+	ports {
+		port@0 {
+			reg = <0>;
+
+			csi40_in: endpoint {
+				clock-lanes = <0>;
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&adv7482_txa>;
+			};
+		};
+	};
+};
+
 &du {
 	pinctrl-0 = <&du_pins>;
 	pinctrl-names = "default";
@@ -322,6 +368,12 @@
 		asahi-kasei,out4-single-end;
 		asahi-kasei,out5-single-end;
 		asahi-kasei,out6-single-end;
+
+		port {
+			ak4613_endpoint: endpoint {
+				remote-endpoint = <&rsnd_endpoint0>;
+			};
+		};
 	};
 
 	cs2000: clk_multiplier@4f {
@@ -359,6 +411,55 @@
 
 		shunt-resistor-micro-ohms = <5000>;
 	};
+
+	video-receiver@70 {
+		compatible = "adi,adv7482";
+		reg = <0x70>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		interrupt-parent = <&gpio6>;
+		interrupt-names = "intrq1", "intrq2";
+		interrupts = <30 IRQ_TYPE_LEVEL_LOW>,
+			     <31 IRQ_TYPE_LEVEL_LOW>;
+
+		port@7 {
+			reg = <7>;
+
+			adv7482_ain7: endpoint {
+				remote-endpoint = <&cvbs_con>;
+			};
+		};
+
+		port@8 {
+			reg = <8>;
+
+			adv7482_hdmi: endpoint {
+				remote-endpoint = <&hdmi_in_con>;
+			};
+		};
+
+		port@10 {
+			reg = <10>;
+
+			adv7482_txa: endpoint {
+				clock-lanes = <0>;
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&csi40_in>;
+			};
+		};
+
+		port@11 {
+			reg = <11>;
+
+			adv7482_txb: endpoint {
+				clock-lanes = <0>;
+				data-lanes = <1>;
+				remote-endpoint = <&csi20_in>;
+			};
+		};
+	};
 };
 
 &i2c_dvfs {
@@ -376,6 +477,8 @@
 		#interrupt-cells = <2>;
 		gpio-controller;
 		#gpio-cells = <2>;
+		rohm,ddr-backup-power = <0xf>;
+		rohm,rstbmode-level;
 
 		regulators {
 			dvfs: dvfs {
@@ -387,6 +490,12 @@
 			};
 		};
 	};
+
+	eeprom@50 {
+		compatible = "rohm,br24t01", "atmel,24c01";
+		reg = <0x50>;
+		pagesize = <8>;
+	};
 };
 
 &ohci0 {
@@ -416,12 +525,12 @@
 
 	avb_pins: avb {
 		mux {
-			groups = "avb_link", "avb_mdc", "avb_mii";
+			groups = "avb_link", "avb_mdio", "avb_mii";
 			function = "avb";
 		};
 
-		pins_mdc {
-			groups = "avb_mdc";
+		pins_mdio {
+			groups = "avb_mdio";
 			drive-strength = <24>;
 		};
 
@@ -581,10 +690,18 @@
 		 <&audio_clk_c>,
 		 <&cpg CPG_CORE CPG_AUDIO_CLK_I>;
 
-	rcar_sound,dai {
-		dai0 {
-			playback = <&ssi0 &src0 &dvc0>;
-			capture  = <&ssi1 &src1 &dvc1>;
+	ports {
+		rsnd_port0: port@0 {
+			rsnd_endpoint0: endpoint {
+				remote-endpoint = <&ak4613_endpoint>;
+
+				dai-format = "left_j";
+				bitclock-master = <&rsnd_endpoint0>;
+				frame-master = <&rsnd_endpoint0>;
+
+				playback = <&ssi0 &src0 &dvc0>;
+				capture  = <&ssi1 &src1 &dvc1>;
+			};
 		};
 	};
 };
@@ -689,6 +806,38 @@
 	clock-frequency = <100000000>;
 };
 
+&vin0 {
+	status = "okay";
+};
+
+&vin1 {
+	status = "okay";
+};
+
+&vin2 {
+	status = "okay";
+};
+
+&vin3 {
+	status = "okay";
+};
+
+&vin4 {
+	status = "okay";
+};
+
+&vin5 {
+	status = "okay";
+};
+
+&vin6 {
+	status = "okay";
+};
+
+&vin7 {
+	status = "okay";
+};
+
 &wdt0 {
 	timeout-sec = <60>;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/renesas/ulcb.dtsi b/arch/arm64/boot/dts/renesas/ulcb.dtsi
index 6f814845f8b6..0edb16e6b372 100644
--- a/arch/arm64/boot/dts/renesas/ulcb.dtsi
+++ b/arch/arm64/boot/dts/renesas/ulcb.dtsi
@@ -243,6 +243,32 @@
 
 &i2c_dvfs {
 	status = "okay";
+
+	pmic: pmic@30 {
+		pinctrl-0 = <&irq0_pins>;
+		pinctrl-names = "default";
+
+		compatible = "rohm,bd9571mwv";
+		reg = <0x30>;
+		interrupt-parent = <&intc_ex>;
+		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		rohm,ddr-backup-power = <0xf>;
+		rohm,rstbmode-pulse;
+
+		regulators {
+			dvfs: dvfs {
+				regulator-name = "dvfs";
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <1030000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
 };
 
 &ohci1 {
@@ -255,12 +281,12 @@
 
 	avb_pins: avb {
 		mux {
-			groups = "avb_link", "avb_mdc", "avb_mii";
+			groups = "avb_link", "avb_mdio", "avb_mii";
 			function = "avb";
 		};
 
-		pins_mdc {
-			groups = "avb_mdc";
+		pins_mdio {
+			groups = "avb_mdio";
 			drive-strength = <24>;
 		};
 
@@ -276,6 +302,11 @@
 		function = "i2c2";
 	};
 
+	irq0_pins: irq0 {
+		groups = "intc_ex_irq0";
+		function = "intc_ex";
+	};
+
 	scif2_pins: scif2 {
 		groups = "scif2_data_a";
 		function = "scif2";
diff --git a/arch/arm64/boot/dts/rockchip/rk3328.dtsi b/arch/arm64/boot/dts/rockchip/rk3328.dtsi
index be2bfbc6b483..b8e9da15e00c 100644
--- a/arch/arm64/boot/dts/rockchip/rk3328.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3328.dtsi
@@ -595,6 +595,8 @@
 		reg = <0x0 0xff330200 0 0x100>;
 		interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "h265e_mmu";
+		clocks = <&cru ACLK_H265>, <&cru PCLK_H265>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -604,6 +606,8 @@
 		reg = <0x0 0xff340800 0x0 0x40>;
 		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vepu_mmu";
+		clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -613,6 +617,8 @@
 		reg = <0x0 0xff350800 0x0 0x40>;
 		interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vpu_mmu";
+		clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -622,6 +628,8 @@
 		reg = <0x0 0xff360480 0x0 0x40>, <0x0 0xff3604c0 0x0 0x40>;
 		interrupts = <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "rkvdec_mmu";
+		clocks = <&cru ACLK_RKVDEC>, <&cru HCLK_RKVDEC>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -631,6 +639,8 @@
 		reg = <0x0 0xff373f00 0x0 0x100>;
 		interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vop_mmu";
+		clocks = <&cru ACLK_VOP>, <&cru HCLK_VOP>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
diff --git a/arch/arm64/boot/dts/rockchip/rk3368.dtsi b/arch/arm64/boot/dts/rockchip/rk3368.dtsi
index 03458ac44201..ad91ced78649 100644
--- a/arch/arm64/boot/dts/rockchip/rk3368.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3368.dtsi
@@ -742,6 +742,8 @@
 		reg = <0x0 0xff900800 0x0 0x100>;
 		interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "iep_mmu";
+		clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -752,6 +754,8 @@
 		      <0x0 0xff915000 0x0 0x100>;
 		interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "isp_mmu";
+		clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		rockchip,disable-mmu-reset;
 		status = "disabled";
@@ -762,6 +766,8 @@
 		reg = <0x0 0xff930300 0x0 0x100>;
 		interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vop_mmu";
+		clocks = <&cru ACLK_VOP>, <&cru HCLK_VOP>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -772,6 +778,8 @@
 		      <0x0 0xff9a0480 0x0 0x40>;
 		interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "hevc_mmu";
+		clocks = <&cru ACLK_VIDEO>, <&cru HCLK_VIDEO>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -782,6 +790,8 @@
 		interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "vepu_mmu", "vdpu_mmu";
+		clocks = <&cru ACLK_VIDEO>, <&cru HCLK_VIDEO>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-firefly.dts b/arch/arm64/boot/dts/rockchip/rk3399-firefly.dts
index 4f28628aa091..2a352763c848 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-firefly.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-firefly.dts
@@ -662,6 +662,14 @@
 	status = "okay";
 };
 
+&tcphy0 {
+	status = "okay";
+};
+
+&tcphy1 {
+	status = "okay";
+};
+
 &tsadc {
 	/* tshut mode 0:CRU 1:GPIO */
 	rockchip,hw-tshut-mode = <1>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts
index 191a6bcb1704..82179125bfb7 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts
@@ -255,7 +255,7 @@ ap_i2c_dig: &i2c2 {
 
 &ap_i2c_tp {
 	trackpad@4a {
-		compatible = "atmel,atmel_mxt_tp";
+		compatible = "atmel,maxtouch";
 		reg = <0x4a>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&trackpad_int_l>;
@@ -271,7 +271,7 @@ ap_i2c_dig: &i2c2 {
 
 &ap_i2c_ts {
 	touchscreen@4b {
-		compatible = "atmel,atmel_mxt_ts";
+		compatible = "atmel,maxtouch";
 		reg = <0x4b>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&touch_int_l>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
index 18f546f2dfd1..f49bfab75dd0 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
@@ -588,7 +588,9 @@
 		<&cru ACLK_PERILP0>, <&cru HCLK_PERILP0>,
 		<&cru PCLK_PERILP0>, <&cru ACLK_CCI>,
 		<&cru HCLK_PERILP1>, <&cru PCLK_PERILP1>,
-		<&cru ACLK_VIO>;
+		<&cru ACLK_VIO>, <&cru ACLK_HDCP>,
+		<&cru ACLK_GIC_PRE>,
+		<&cru PCLK_DDR>;
 	assigned-clock-rates =
 		<600000000>, <800000000>,
 		<1000000000>,
@@ -597,7 +599,9 @@
 		<100000000>, <100000000>,
 		<50000000>, <800000000>,
 		<100000000>, <50000000>,
-		<400000000>;
+		<400000000>, <400000000>,
+		<200000000>,
+		<200000000>;
 };
 
 &emmc_phy {
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts b/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts
index 7d3e8bfd51dd..e0afdd8b62bd 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-puma-haikou.dts
@@ -143,6 +143,11 @@
 	};
 };
 
+&hdmi {
+	ddc-i2c-bus = <&i2c3>;
+	status = "okay";
+};
+
 &i2c1 {
 	status = "okay";
 	clock-frequency = <400000>;
@@ -246,6 +251,10 @@
 	status = "okay";
 };
 
+&tcphy0 {
+	status = "okay";
+};
+
 &u2phy0 {
 	status = "okay";
 };
@@ -281,3 +290,19 @@
 &usb_host0_ohci {
 	status = "okay";
 };
+
+&vopb {
+	status = "okay";
+};
+
+&vopb_mmu {
+	status = "okay";
+};
+
+&vopl {
+	status = "okay";
+};
+
+&vopl_mmu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi
index 4a2d06abe9c1..14a0f1998639 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi
@@ -527,6 +527,10 @@
 	};
 };
 
+&tcphy1 {
+	status = "okay";
+};
+
 &tsadc {
 	rockchip,hw-tshut-mode = <1>;
 	rockchip,hw-tshut-polarity = <1>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator.dts
index 56952d1a3fb8..ad7548d3b93d 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator.dts
@@ -190,6 +190,18 @@
 	status = "okay";
 };
 
+&pcie_phy {
+	status = "okay";
+};
+
+&pcie0 {
+	ep-gpios = <&gpio2 RK_PA4 GPIO_ACTIVE_HIGH>;
+	num-lanes = <4>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie_clkreqn_cpm>;
+	status = "okay";
+};
+
 &pinctrl {
 	sdio-pwrseq {
 		wifi_enable_h: wifi-enable-h {
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi
index e5daed7d2026..941b627094d7 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi
@@ -471,21 +471,6 @@
 	gpio1830-supply = <&vcc_3v0>;
 };
 
-&pcie_phy {
-	status = "okay";
-};
-
-&pcie0 {
-	assigned-clocks = <&cru SCLK_PCIEPHY_REF>;
-	assigned-clock-parents = <&cru SCLK_PCIEPHY_REF100M>;
-	assigned-clock-rates = <100000000>;
-	ep-gpios = <&gpio2 RK_PA4 GPIO_ACTIVE_HIGH>;
-	num-lanes = <4>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&pcie_clkreqn_cpm>;
-	status = "okay";
-};
-
 &pmu_io_domains {
 	pmu1830-supply = <&vcc_3v0>;
 	status = "okay";
@@ -560,6 +545,14 @@
 	status = "okay";
 };
 
+&tcphy0 {
+	status = "okay";
+};
+
+&tcphy1 {
+	status = "okay";
+};
+
 &tsadc {
 	/* tshut mode 0:CRU 1:GPIO */
 	rockchip,hw-tshut-mode = <1>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
index 4550c0f82be9..e0040b648f43 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
@@ -312,6 +312,8 @@
 		reg = <0x0 0xfe320000 0x0 0x4000>;
 		interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH 0>;
 		max-frequency = <150000000>;
+		assigned-clocks = <&cru HCLK_SD>;
+		assigned-clock-rates = <200000000>;
 		clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>,
 			 <&cru SCLK_SDMMC_DRV>, <&cru SCLK_SDMMC_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
@@ -411,8 +413,8 @@
 			reg = <0x0 0xfe800000 0x0 0x100000>;
 			interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH 0>;
 			dr_mode = "otg";
-			phys = <&u2phy0_otg>;
-			phy-names = "usb2-phy";
+			phys = <&u2phy0_otg>, <&tcphy0_usb3>;
+			phy-names = "usb2-phy", "usb3-phy";
 			phy_type = "utmi_wide";
 			snps,dis_enblslpm_quirk;
 			snps,dis-u2-freeclk-exists-quirk;
@@ -444,8 +446,8 @@
 			reg = <0x0 0xfe900000 0x0 0x100000>;
 			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH 0>;
 			dr_mode = "otg";
-			phys = <&u2phy1_otg>;
-			phy-names = "usb2-phy";
+			phys = <&u2phy1_otg>, <&tcphy1_usb3>;
+			phy-names = "usb2-phy", "usb3-phy";
 			phy_type = "utmi_wide";
 			snps,dis_enblslpm_quirk;
 			snps,dis-u2-freeclk-exists-quirk;
@@ -461,8 +463,8 @@
 		compatible = "rockchip,rk3399-cdn-dp";
 		reg = <0x0 0xfec00000 0x0 0x100000>;
 		interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH 0>;
-		assigned-clocks = <&cru SCLK_DP_CORE>;
-		assigned-clock-rates = <100000000>;
+		assigned-clocks = <&cru SCLK_DP_CORE>, <&cru SCLK_SPDIF_REC_DPTX>;
+		assigned-clock-rates = <100000000>, <200000000>;
 		clocks = <&cru SCLK_DP_CORE>, <&cru PCLK_DP_CTRL>,
 			 <&cru SCLK_SPDIF_REC_DPTX>, <&cru PCLK_VIO_GRF>;
 		clock-names = "core-clk", "pclk", "spdif", "grf";
@@ -1234,6 +1236,8 @@
 		reg = <0x0 0xff650800 0x0 0x40>;
 		interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH 0>;
 		interrupt-names = "vpu_mmu";
+		clocks = <&cru ACLK_VCODEC>, <&cru HCLK_VCODEC>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -1243,6 +1247,8 @@
 		reg = <0x0 0xff660480 0x0 0x40>, <0x0 0xff6604c0 0x0 0x40>;
 		interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH 0>;
 		interrupt-names = "vdec_mmu";
+		clocks = <&cru ACLK_VDU>, <&cru HCLK_VDU>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -1252,6 +1258,8 @@
 		reg = <0x0 0xff670800 0x0 0x40>;
 		interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH 0>;
 		interrupt-names = "iep_mmu";
+		clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		status = "disabled";
 	};
@@ -1323,7 +1331,9 @@
 			<&cru ACLK_PERILP0>, <&cru HCLK_PERILP0>,
 			<&cru PCLK_PERILP0>, <&cru ACLK_CCI>,
 			<&cru HCLK_PERILP1>, <&cru PCLK_PERILP1>,
-			<&cru ACLK_VIO>;
+			<&cru ACLK_VIO>, <&cru ACLK_HDCP>,
+			<&cru ACLK_GIC_PRE>,
+			<&cru PCLK_DDR>;
 		assigned-clock-rates =
 			 <594000000>,  <800000000>,
 			<1000000000>,
@@ -1332,7 +1342,9 @@
 			 <100000000>,  <100000000>,
 			  <50000000>, <600000000>,
 			 <100000000>,   <50000000>,
-			 <400000000>;
+			 <400000000>, <400000000>,
+			 <200000000>,
+			 <200000000>;
 	};
 
 	grf: syscon@ff770000 {
@@ -1599,7 +1611,7 @@
 		interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH 0>;
 		interrupt-names = "vopl_mmu";
 		clocks = <&cru ACLK_VOP1>, <&cru HCLK_VOP1>;
-		clock-names = "aclk", "hclk";
+		clock-names = "aclk", "iface";
 		power-domains = <&power RK3399_PD_VOPL>;
 		#iommu-cells = <0>;
 		status = "disabled";
@@ -1656,7 +1668,7 @@
 		interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH 0>;
 		interrupt-names = "vopb_mmu";
 		clocks = <&cru ACLK_VOP0>, <&cru HCLK_VOP0>;
-		clock-names = "aclk", "hclk";
+		clock-names = "aclk", "iface";
 		power-domains = <&power RK3399_PD_VOPB>;
 		#iommu-cells = <0>;
 		status = "disabled";
@@ -1667,6 +1679,8 @@
 		reg = <0x0 0xff914000 0x0 0x100>, <0x0 0xff915000 0x0 0x100>;
 		interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH 0>;
 		interrupt-names = "isp0_mmu";
+		clocks = <&cru ACLK_ISP0_NOC>, <&cru HCLK_ISP0_NOC>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		rockchip,disable-mmu-reset;
 		status = "disabled";
@@ -1677,6 +1691,8 @@
 		reg = <0x0 0xff924000 0x0 0x100>, <0x0 0xff925000 0x0 0x100>;
 		interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH 0>;
 		interrupt-names = "isp1_mmu";
+		clocks = <&cru ACLK_ISP1_NOC>, <&cru HCLK_ISP1_NOC>;
+		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
 		rockchip,disable-mmu-reset;
 		status = "disabled";
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
index c32dd3419c87..d63b56e944de 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
@@ -549,10 +549,13 @@
 			status = "disabled";
 			reg = <0x65000000 0x8500>;
 			interrupts = <0 66 4>;
+			clock-names = "ether";
 			clocks = <&sys_clk 6>;
+			reset-names = "ether";
 			resets = <&sys_rst 6>;
-			phy-mode = "rmii";
+			phy-mode = "internal";
 			local-mac-address = [00 00 00 00 00 00];
+			socionext,syscon-phy-mode = <&soc_glue 0>;
 
 			mdio: mdio {
 				#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
index 3a5ed789c056..0298bd0d0e1a 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
@@ -604,10 +604,13 @@
 			interrupts = <0 66 4>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_ether_rgmii>;
+			clock-names = "ether";
 			clocks = <&sys_clk 6>;
+			reset-names = "ether";
 			resets = <&sys_rst 6>;
 			phy-mode = "rgmii";
 			local-mac-address = [00 00 00 00 00 00];
+			socionext,syscon-phy-mode = <&soc_glue 0>;
 
 			mdio: mdio {
 				#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi b/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi
index e85d6ddea3c2..2a4cf427f5d3 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi
+++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi
@@ -341,7 +341,7 @@
 			cdns,phy-dll-delay-sdclk-hsmmc = <21>;
 		};
 
-		soc-glue@5f800000 {
+		soc_glue: soc-glue@5f800000 {
 			compatible = "socionext,uniphier-pxs3-soc-glue",
 				     "simple-mfd", "syscon";
 			reg = <0x5f800000 0x2000>;
@@ -412,10 +412,13 @@
 			interrupts = <0 66 4>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_ether_rgmii>;
+			clock-names = "ether";
 			clocks = <&sys_clk 6>;
+			reset-names = "ether";
 			resets = <&sys_rst 6>;
 			phy-mode = "rgmii";
 			local-mac-address = [00 00 00 00 00 00];
+			socionext,syscon-phy-mode = <&soc_glue 0>;
 
 			mdio0: mdio {
 				#address-cells = <1>;
@@ -430,10 +433,13 @@
 			interrupts = <0 67 4>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_ether1_rgmii>;
+			clock-names = "ether";
 			clocks = <&sys_clk 7>;
+			reset-names = "ether";
 			resets = <&sys_rst 7>;
 			phy-mode = "rgmii";
 			local-mac-address = [00 00 00 00 00 00];
+			socionext,syscon-phy-mode = <&soc_glue 1>;
 
 			mdio1: mdio {
 				#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/sprd/sc2731.dtsi b/arch/arm64/boot/dts/sprd/sc2731.dtsi
index 4331006185bf..98d3b4fdb9ad 100644
--- a/arch/arm64/boot/dts/sprd/sc2731.dtsi
+++ b/arch/arm64/boot/dts/sprd/sc2731.dtsi
@@ -24,6 +24,17 @@
 			interrupts = <2 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
+		pmic_eic: gpio@300 {
+			compatible = "sprd,sc27xx-eic";
+			reg = <0x300>;
+			interrupt-parent = <&sc2731_pmic>;
+			interrupts = <5 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
 		regulators {
 			compatible = "sprd,sc27xx-regulator";
 
diff --git a/arch/arm64/boot/dts/sprd/sc9860.dtsi b/arch/arm64/boot/dts/sprd/sc9860.dtsi
index 5dbfb796d9f9..3f5160d2f130 100644
--- a/arch/arm64/boot/dts/sprd/sc9860.dtsi
+++ b/arch/arm64/boot/dts/sprd/sc9860.dtsi
@@ -7,6 +7,8 @@
  */
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/gpio/gpio.h>
 #include "whale2.dtsi"
 
 / {
@@ -326,7 +328,7 @@
 					reg = <4>;
 					soc_funnel_in_port1: endpoint {
 						slave-mode;
-						remote-endpioint =
+						remote-endpoint =
 							<&stm_out_port>;
 					};
 				};
@@ -679,5 +681,33 @@
 				};
 			};
 		};
+
+		gpio-keys {
+			compatible = "gpio-keys";
+
+			key-volumedown {
+				label = "Volume Down Key";
+				linux,code = <KEY_VOLUMEDOWN>;
+				gpios = <&eic_debounce 2 GPIO_ACTIVE_LOW>;
+				debounce-interval = <2>;
+				wakeup-source;
+			};
+
+			key-volumeup {
+				label = "Volume Up Key";
+				linux,code = <KEY_VOLUMEUP>;
+				gpios = <&pmic_eic 10 GPIO_ACTIVE_HIGH>;
+				debounce-interval = <2>;
+				wakeup-source;
+			};
+
+			key-power {
+				label = "Power Key";
+				linux,code = <KEY_POWER>;
+				gpios = <&pmic_eic 1 GPIO_ACTIVE_HIGH>;
+				debounce-interval = <2>;
+				wakeup-source;
+			};
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/sprd/whale2.dtsi b/arch/arm64/boot/dts/sprd/whale2.dtsi
index 66a881e6da92..e9db9108f3c0 100644
--- a/arch/arm64/boot/dts/sprd/whale2.dtsi
+++ b/arch/arm64/boot/dts/sprd/whale2.dtsi
@@ -154,6 +154,56 @@
 				clocks = <&aon_gate CLK_SPLK_EB>;
 			};
 
+			eic_debounce: gpio@40210000 {
+				compatible = "sprd,sc9860-eic-debounce";
+				reg = <0 0x40210000 0 0x80>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
+			eic_latch: gpio@40210080 {
+				compatible = "sprd,sc9860-eic-latch";
+				reg = <0 0x40210080 0 0x20>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
+			eic_async: gpio@402100a0 {
+				compatible = "sprd,sc9860-eic-async";
+				reg = <0 0x402100a0 0 0x20>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
+			eic_sync: gpio@402100c0 {
+				compatible = "sprd,sc9860-eic-sync";
+				reg = <0 0x402100c0 0 0x20>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
+			ap_gpio: gpio@40280000 {
+				compatible = "sprd,sc9860-gpio";
+				reg = <0 0x40280000 0 0x1000>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
 			pin_controller: pinctrl@402a0000 {
 				compatible = "sprd,sc9860-pinctrl";
 				reg = <0 0x402a0000 0 0x10000>;
@@ -164,8 +214,9 @@
 				reg = <0 0x40310000 0 0x1000>;
 				interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
 				timeout-sec = <12>;
-				clock-names = "enable";
-				clocks = <&aon_gate CLK_APCPU_WDG_EB>;
+				clock-names = "enable", "rtc_enable";
+				clocks = <&aon_gate CLK_APCPU_WDG_EB>,
+				       <&aon_gate CLK_AP_WDG_RTC_EB>;
 			};
 		};
 
diff --git a/arch/arm64/boot/dts/synaptics/Makefile b/arch/arm64/boot/dts/synaptics/Makefile
new file mode 100644
index 000000000000..de71ddda6835
--- /dev/null
+++ b/arch/arm64/boot/dts/synaptics/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+# Berlin SoC Family
+dtb-$(CONFIG_ARCH_BERLIN) += berlin4ct-dmp.dtb
+dtb-$(CONFIG_ARCH_BERLIN) += berlin4ct-stb.dtb
diff --git a/arch/arm64/boot/dts/synaptics/berlin4ct-dmp.dts b/arch/arm64/boot/dts/synaptics/berlin4ct-dmp.dts
new file mode 100644
index 000000000000..c64a179ebbb7
--- /dev/null
+++ b/arch/arm64/boot/dts/synaptics/berlin4ct-dmp.dts
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (C) 2015 Marvell Technology Group Ltd.
+ *
+ * Author: Jisheng Zhang <jszhang@marvell.com>
+ */
+
+/dts-v1/;
+
+#include "berlin4ct.dtsi"
+
+/ {
+	model = "Marvell BG4CT DMP board";
+	compatible = "marvell,berlin4ct-dmp", "marvell,berlin4ct", "marvell,berlin";
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@1000000 {
+		device_type = "memory";
+		/* the first 16MB is for firmwares' usage */
+		reg = <0 0x01000000 0 0x7f000000>;
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/synaptics/berlin4ct-stb.dts b/arch/arm64/boot/dts/synaptics/berlin4ct-stb.dts
new file mode 100644
index 000000000000..277dccfa05cb
--- /dev/null
+++ b/arch/arm64/boot/dts/synaptics/berlin4ct-stb.dts
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (C) 2015 Marvell Technology Group Ltd.
+ *
+ * Author: Jisheng Zhang <jszhang@marvell.com>
+ */
+
+/dts-v1/;
+
+#include "berlin4ct.dtsi"
+
+/ {
+	model = "Marvell BG4CT STB board";
+	compatible = "marvell,berlin4ct-stb", "marvell,berlin4ct", "marvell,berlin";
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@1000000 {
+		device_type = "memory";
+		/* the first 16MB is for firmwares' usage */
+		reg = <0 0x01000000 0 0x7f000000>;
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/synaptics/berlin4ct.dtsi b/arch/arm64/boot/dts/synaptics/berlin4ct.dtsi
new file mode 100644
index 000000000000..216767e2edf6
--- /dev/null
+++ b/arch/arm64/boot/dts/synaptics/berlin4ct.dtsi
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (C) 2015 Marvell Technology Group Ltd.
+ *
+ * Author: Jisheng Zhang <jszhang@marvell.com>
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	compatible = "marvell,berlin4ct", "marvell,berlin";
+	interrupt-parent = <&gic>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0", "arm,psci-0.2";
+		method = "smc";
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			compatible = "arm,cortex-a53", "arm,armv8";
+			device_type = "cpu";
+			reg = <0x0>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+			cpu-idle-states = <&CPU_SLEEP_0>;
+		};
+
+		cpu1: cpu@1 {
+			compatible = "arm,cortex-a53", "arm,armv8";
+			device_type = "cpu";
+			reg = <0x1>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+			cpu-idle-states = <&CPU_SLEEP_0>;
+		};
+
+		cpu2: cpu@2 {
+			compatible = "arm,cortex-a53", "arm,armv8";
+			device_type = "cpu";
+			reg = <0x2>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+			cpu-idle-states = <&CPU_SLEEP_0>;
+		};
+
+		cpu3: cpu@3 {
+			compatible = "arm,cortex-a53", "arm,armv8";
+			device_type = "cpu";
+			reg = <0x3>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+			cpu-idle-states = <&CPU_SLEEP_0>;
+		};
+
+		l2: cache {
+			compatible = "cache";
+		};
+
+		idle-states {
+			entry-method = "psci";
+			CPU_SLEEP_0: cpu-sleep-0 {
+				compatible = "arm,idle-state";
+				local-timer-stop;
+				arm,psci-suspend-param = <0x0010000>;
+				entry-latency-us = <75>;
+				exit-latency-us = <155>;
+				min-residency-us = <1000>;
+			};
+		};
+	};
+
+	osc: osc {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <25000000>;
+	};
+
+	pmu {
+		compatible = "arm,cortex-a53-pmu", "arm,armv8-pmuv3";
+		interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>,
+				     <&cpu1>,
+				     <&cpu2>,
+				     <&cpu3>;
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
+	};
+
+	soc@f7000000 {
+		compatible = "simple-bus";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0 0xf7000000 0x1000000>;
+
+		gic: interrupt-controller@901000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			reg = <0x901000 0x1000>,
+			      <0x902000 0x2000>,
+			      <0x904000 0x2000>,
+			      <0x906000 0x2000>;
+			interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+		};
+
+		apb@e80000 {
+			compatible = "simple-bus";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			ranges = <0 0xe80000 0x10000>;
+			interrupt-parent = <&aic>;
+
+			gpio0: gpio@400 {
+				compatible = "snps,dw-apb-gpio";
+				reg = <0x0400 0x400>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				porta: gpio-port@0 {
+					compatible = "snps,dw-apb-gpio-port";
+					gpio-controller;
+					#gpio-cells = <2>;
+					snps,nr-gpios = <32>;
+					reg = <0>;
+					interrupt-controller;
+					#interrupt-cells = <2>;
+					interrupts = <0>;
+				};
+			};
+
+			gpio1: gpio@800 {
+				compatible = "snps,dw-apb-gpio";
+				reg = <0x0800 0x400>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				portb: gpio-port@1 {
+					compatible = "snps,dw-apb-gpio-port";
+					gpio-controller;
+					#gpio-cells = <2>;
+					snps,nr-gpios = <32>;
+					reg = <0>;
+					interrupt-controller;
+					#interrupt-cells = <2>;
+					interrupts = <1>;
+				};
+			};
+
+			gpio2: gpio@c00 {
+				compatible = "snps,dw-apb-gpio";
+				reg = <0x0c00 0x400>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				portc: gpio-port@2 {
+					compatible = "snps,dw-apb-gpio-port";
+					gpio-controller;
+					#gpio-cells = <2>;
+					snps,nr-gpios = <32>;
+					reg = <0>;
+					interrupt-controller;
+					#interrupt-cells = <2>;
+					interrupts = <2>;
+				};
+			};
+
+			gpio3: gpio@1000 {
+				compatible = "snps,dw-apb-gpio";
+				reg = <0x1000 0x400>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				portd: gpio-port@3 {
+					compatible = "snps,dw-apb-gpio-port";
+					gpio-controller;
+					#gpio-cells = <2>;
+					snps,nr-gpios = <32>;
+					reg = <0>;
+					interrupt-controller;
+					#interrupt-cells = <2>;
+					interrupts = <3>;
+				};
+			};
+
+			aic: interrupt-controller@3800 {
+				compatible = "snps,dw-apb-ictl";
+				reg = <0x3800 0x30>;
+				interrupt-controller;
+				#interrupt-cells = <1>;
+				interrupt-parent = <&gic>;
+				interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+			};
+		};
+
+		soc_pinctrl: pin-controller@ea8000 {
+			compatible = "marvell,berlin4ct-soc-pinctrl";
+			reg = <0xea8000 0x14>;
+		};
+
+		avio_pinctrl: pin-controller@ea8400 {
+			compatible = "marvell,berlin4ct-avio-pinctrl";
+			reg = <0xea8400 0x8>;
+		};
+
+		apb@fc0000 {
+			compatible = "simple-bus";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0xfc0000 0x10000>;
+			interrupt-parent = <&sic>;
+
+			sic: interrupt-controller@1000 {
+				compatible = "snps,dw-apb-ictl";
+				reg = <0x1000 0x30>;
+				interrupt-controller;
+				#interrupt-cells = <1>;
+				interrupt-parent = <&gic>;
+				interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
+			wdt0: watchdog@3000 {
+				compatible = "snps,dw-wdt";
+				reg = <0x3000 0x100>;
+				clocks = <&osc>;
+				interrupts = <0>;
+			};
+
+			wdt1: watchdog@4000 {
+				compatible = "snps,dw-wdt";
+				reg = <0x4000 0x100>;
+				clocks = <&osc>;
+				interrupts = <1>;
+			};
+
+			wdt2: watchdog@5000 {
+				compatible = "snps,dw-wdt";
+				reg = <0x5000 0x100>;
+				clocks = <&osc>;
+				interrupts = <2>;
+			};
+
+			sm_gpio0: gpio@8000 {
+				compatible = "snps,dw-apb-gpio";
+				reg = <0x8000 0x400>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				porte: gpio-port@4 {
+					compatible = "snps,dw-apb-gpio-port";
+					gpio-controller;
+					#gpio-cells = <2>;
+					snps,nr-gpios = <32>;
+					reg = <0>;
+				};
+			};
+
+			sm_gpio1: gpio@9000 {
+				compatible = "snps,dw-apb-gpio";
+				reg = <0x9000 0x400>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				portf: gpio-port@5 {
+					compatible = "snps,dw-apb-gpio-port";
+					gpio-controller;
+					#gpio-cells = <2>;
+					snps,nr-gpios = <32>;
+					reg = <0>;
+				};
+			};
+
+			uart0: uart@d000 {
+				compatible = "snps,dw-apb-uart";
+				reg = <0xd000 0x100>;
+				interrupts = <8>;
+				clocks = <&osc>;
+				reg-shift = <2>;
+				status = "disabled";
+				pinctrl-0 = <&uart0_pmux>;
+				pinctrl-names = "default";
+			};
+		};
+
+		system_pinctrl: pin-controller@fe2200 {
+			compatible = "marvell,berlin4ct-system-pinctrl";
+			reg = <0xfe2200 0xc>;
+
+			uart0_pmux: uart0-pmux {
+				groups = "SM_URT0_TXD", "SM_URT0_RXD";
+				function = "uart0";
+			};
+		};
+	};
+};
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index ecf613761e78..3cfa8ca26738 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -53,6 +53,7 @@ CONFIG_ARCH_R8A7796=y
 CONFIG_ARCH_R8A77965=y
 CONFIG_ARCH_R8A77970=y
 CONFIG_ARCH_R8A77980=y
+CONFIG_ARCH_R8A77990=y
 CONFIG_ARCH_R8A77995=y
 CONFIG_ARCH_STRATIX10=y
 CONFIG_ARCH_TEGRA=y
@@ -75,10 +76,12 @@ CONFIG_PCI_HISI=y
 CONFIG_PCIE_QCOM=y
 CONFIG_PCIE_KIRIN=y
 CONFIG_PCIE_ARMADA_8K=y
+CONFIG_PCIE_HISI_STB=y
 CONFIG_PCI_AARDVARK=y
 CONFIG_PCI_TEGRA=y
 CONFIG_PCIE_RCAR=y
-CONFIG_PCIE_ROCKCHIP=m
+CONFIG_PCIE_ROCKCHIP=y
+CONFIG_PCIE_ROCKCHIP_HOST=m
 CONFIG_PCI_HOST_GENERIC=y
 CONFIG_PCI_XGENE=y
 CONFIG_PCI_HOST_THUNDER_PEM=y
@@ -157,8 +160,10 @@ CONFIG_BT_HIDP=m
 # CONFIG_BT_LE is not set
 CONFIG_BT_LEDS=y
 # CONFIG_BT_DEBUGFS is not set
+CONFIG_BT_HCIBTUSB=m
 CONFIG_BT_HCIUART=m
 CONFIG_BT_HCIUART_LL=y
+CONFIG_BT_HCIUART_BCM=y
 CONFIG_CFG80211=m
 CONFIG_MAC80211=m
 CONFIG_MAC80211_LEDS=y
@@ -169,6 +174,9 @@ CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 CONFIG_DMA_CMA=y
+CONFIG_CMA_SIZE_MBYTES=32
+CONFIG_HISILICON_LPC=y
+CONFIG_SIMPLE_PM_BUS=y
 CONFIG_MTD=y
 CONFIG_MTD_BLOCK=y
 CONFIG_MTD_M25P80=y
@@ -187,6 +195,9 @@ CONFIG_BLK_DEV_SD=y
 CONFIG_SCSI_SAS_ATA=y
 CONFIG_SCSI_HISI_SAS=y
 CONFIG_SCSI_HISI_SAS_PCI=y
+CONFIG_SCSI_UFSHCD=m
+CONFIG_SCSI_UFSHCD_PLATFORM=m
+CONFIG_SCSI_UFS_QCOM=m
 CONFIG_ATA=y
 CONFIG_SATA_AHCI=y
 CONFIG_SATA_AHCI_PLATFORM=y
@@ -206,8 +217,10 @@ CONFIG_VETH=m
 CONFIG_VIRTIO_NET=y
 CONFIG_AMD_XGBE=y
 CONFIG_NET_XGENE=y
+CONFIG_ATL1C=m
 CONFIG_MACB=y
 CONFIG_THUNDER_NIC_PF=y
+CONFIG_HIX5HD2_GMAC=y
 CONFIG_HNS_DSAF=y
 CONFIG_HNS_ENET=y
 CONFIG_E1000E=y
@@ -239,6 +252,7 @@ CONFIG_ROCKCHIP_PHY=y
 CONFIG_USB_PEGASUS=m
 CONFIG_USB_RTL8150=m
 CONFIG_USB_RTL8152=m
+CONFIG_USB_LAN78XX=m
 CONFIG_USB_USBNET=m
 CONFIG_USB_NET_DM9601=m
 CONFIG_USB_NET_SR9800=m
@@ -246,13 +260,19 @@ CONFIG_USB_NET_SMSC75XX=m
 CONFIG_USB_NET_SMSC95XX=m
 CONFIG_USB_NET_PLUSB=m
 CONFIG_USB_NET_MCS7830=m
+CONFIG_ATH10K=m
+CONFIG_ATH10K_PCI=m
 CONFIG_BRCMFMAC=m
+CONFIG_MWIFIEX=m
+CONFIG_MWIFIEX_PCIE=m
 CONFIG_WL18XX=m
 CONFIG_WLCORE_SDIO=m
 CONFIG_INPUT_EVDEV=y
 CONFIG_KEYBOARD_ADC=m
 CONFIG_KEYBOARD_CROS_EC=y
 CONFIG_KEYBOARD_GPIO=y
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_ATMEL_MXT=m
 CONFIG_INPUT_MISC=y
 CONFIG_INPUT_PM8941_PWRKEY=y
 CONFIG_INPUT_HISI_POWERKEY=y
@@ -286,6 +306,7 @@ CONFIG_SERIAL_MVEBU_UART=y
 CONFIG_SERIAL_DEV_BUS=y
 CONFIG_SERIAL_DEV_CTRL_TTYPORT=y
 CONFIG_VIRTIO_CONSOLE=y
+CONFIG_I2C_HID=m
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_MUX=y
 CONFIG_I2C_MUX_PCA954x=y
@@ -303,6 +324,7 @@ CONFIG_I2C_UNIPHIER_F=y
 CONFIG_I2C_RCAR=y
 CONFIG_I2C_CROS_EC_TUNNEL=y
 CONFIG_SPI=y
+CONFIG_SPI_ARMADA_3700=y
 CONFIG_SPI_MESON_SPICC=m
 CONFIG_SPI_MESON_SPIFC=m
 CONFIG_SPI_BCM2835=m
@@ -320,6 +342,7 @@ CONFIG_PINCTRL_MAX77620=y
 CONFIG_PINCTRL_MSM8916=y
 CONFIG_PINCTRL_MSM8994=y
 CONFIG_PINCTRL_MSM8996=y
+CONFIG_PINCTRL_MT7622=y
 CONFIG_PINCTRL_QDF2XXX=y
 CONFIG_PINCTRL_QCOM_SPMI_PMIC=y
 CONFIG_GPIO_DWAPB=y
@@ -332,6 +355,8 @@ CONFIG_GPIO_XGENE_SB=y
 CONFIG_GPIO_PCA953X=y
 CONFIG_GPIO_PCA953X_IRQ=y
 CONFIG_GPIO_MAX77620=y
+CONFIG_POWER_AVS=y
+CONFIG_ROCKCHIP_IODOMAIN=y
 CONFIG_POWER_RESET_MSM=y
 CONFIG_POWER_RESET_XGENE=y
 CONFIG_POWER_RESET_SYSCON=y
@@ -343,6 +368,7 @@ CONFIG_SENSORS_INA2XX=m
 CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
 CONFIG_CPU_THERMAL=y
 CONFIG_THERMAL_EMULATION=y
+CONFIG_ARMADA_THERMAL=y
 CONFIG_BRCMSTB_THERMAL=m
 CONFIG_EXYNOS_THERMAL=y
 CONFIG_RCAR_GEN3_THERMAL=y
@@ -361,6 +387,7 @@ CONFIG_MFD_AXP20X_RSB=y
 CONFIG_MFD_CROS_EC=y
 CONFIG_MFD_CROS_EC_I2C=y
 CONFIG_MFD_CROS_EC_SPI=y
+CONFIG_MFD_CROS_EC_CHARDEV=m
 CONFIG_MFD_EXYNOS_LPASS=m
 CONFIG_MFD_HI6421_PMIC=y
 CONFIG_MFD_HI655X_PMIC=y
@@ -439,7 +466,8 @@ CONFIG_SND_BCM2835_SOC_I2S=m
 CONFIG_SND_SOC_SAMSUNG=y
 CONFIG_SND_SOC_RCAR=m
 CONFIG_SND_SOC_AK4613=m
-CONFIG_SND_SIMPLE_CARD=y
+CONFIG_SND_SIMPLE_CARD=m
+CONFIG_SND_AUDIO_GRAPH_CARD=m
 CONFIG_USB=y
 CONFIG_USB_OTG=y
 CONFIG_USB_XHCI_HCD=y
@@ -485,6 +513,7 @@ CONFIG_MMC_SPI=y
 CONFIG_MMC_SDHI=y
 CONFIG_MMC_DW=y
 CONFIG_MMC_DW_EXYNOS=y
+CONFIG_MMC_DW_HI3798CV200=y
 CONFIG_MMC_DW_K3=y
 CONFIG_MMC_DW_ROCKCHIP=y
 CONFIG_MMC_SUNXI=y
@@ -514,6 +543,7 @@ CONFIG_RTC_DRV_SUN6I=y
 CONFIG_RTC_DRV_ARMADA38X=y
 CONFIG_RTC_DRV_TEGRA=y
 CONFIG_RTC_DRV_XGENE=y
+CONFIG_RTC_DRV_CROS_EC=y
 CONFIG_DMADEVICES=y
 CONFIG_DMA_BCM2835=m
 CONFIG_K3_DMA=y
@@ -556,6 +586,7 @@ CONFIG_TEGRA_IOMMU_SMMU=y
 CONFIG_ARM_SMMU=y
 CONFIG_ARM_SMMU_V3=y
 CONFIG_QCOM_IOMMU=y
+CONFIG_RPMSG_QCOM_GLINK_RPM=y
 CONFIG_RPMSG_QCOM_SMD=y
 CONFIG_RASPBERRYPI_POWER=y
 CONFIG_QCOM_SMEM=y
@@ -567,12 +598,18 @@ CONFIG_ARCH_TEGRA_132_SOC=y
 CONFIG_ARCH_TEGRA_210_SOC=y
 CONFIG_ARCH_TEGRA_186_SOC=y
 CONFIG_ARCH_TEGRA_194_SOC=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
 CONFIG_EXTCON_USB_GPIO=y
+CONFIG_EXTCON_USBC_CROS_EC=y
 CONFIG_MEMORY=y
 CONFIG_TEGRA_MC=y
 CONFIG_IIO=y
 CONFIG_EXYNOS_ADC=y
 CONFIG_ROCKCHIP_SARADC=m
+CONFIG_IIO_CROS_EC_SENSORS_CORE=m
+CONFIG_IIO_CROS_EC_SENSORS=m
+CONFIG_IIO_CROS_EC_LIGHT_PROX=m
+CONFIG_IIO_CROS_EC_BARO=m
 CONFIG_PWM=y
 CONFIG_PWM_BCM2835=m
 CONFIG_PWM_CROS_EC=m
@@ -581,21 +618,26 @@ CONFIG_PWM_RCAR=m
 CONFIG_PWM_ROCKCHIP=y
 CONFIG_PWM_SAMSUNG=y
 CONFIG_PWM_TEGRA=m
+CONFIG_PHY_HISTB_COMBPHY=y
+CONFIG_PHY_HISI_INNO_USB2=y
 CONFIG_PHY_RCAR_GEN3_USB2=y
 CONFIG_PHY_RCAR_GEN3_USB3=m
 CONFIG_PHY_HI6220_USB=y
 CONFIG_PHY_QCOM_USB_HS=y
 CONFIG_PHY_SUN4I_USB=y
 CONFIG_PHY_MVEBU_CP110_COMPHY=y
+CONFIG_PHY_QCOM_QMP=m
 CONFIG_PHY_ROCKCHIP_INNO_USB2=y
 CONFIG_PHY_ROCKCHIP_EMMC=y
 CONFIG_PHY_ROCKCHIP_PCIE=m
+CONFIG_PHY_ROCKCHIP_TYPEC=y
 CONFIG_PHY_XGENE=y
 CONFIG_PHY_TEGRA_XUSB=y
 CONFIG_QCOM_L2_PMU=y
 CONFIG_QCOM_L3_PMU=y
 CONFIG_MESON_EFUSE=m
 CONFIG_QCOM_QFPROM=y
+CONFIG_ROCKCHIP_EFUSE=y
 CONFIG_UNIPHIER_EFUSE=y
 CONFIG_TEE=y
 CONFIG_OPTEE=y
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index cb5a243110c4..e3fdb0fd6f70 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -47,6 +47,12 @@ config CRYPTO_SM3_ARM64_CE
 	select CRYPTO_HASH
 	select CRYPTO_SM3
 
+config CRYPTO_SM4_ARM64_CE
+	tristate "SM4 symmetric cipher (ARMv8.2 Crypto Extensions)"
+	depends on KERNEL_MODE_NEON
+	select CRYPTO_ALGAPI
+	select CRYPTO_SM4
+
 config CRYPTO_GHASH_ARM64_CE
 	tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions"
 	depends on KERNEL_MODE_NEON
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index f35ac684b1c0..bcafd016618e 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -23,6 +23,9 @@ sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o
 obj-$(CONFIG_CRYPTO_SM3_ARM64_CE) += sm3-ce.o
 sm3-ce-y := sm3-ce-glue.o sm3-ce-core.o
 
+obj-$(CONFIG_CRYPTO_SM4_ARM64_CE) += sm4-ce.o
+sm4-ce-y := sm4-ce-glue.o sm4-ce-core.o
+
 obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o
 ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
 
diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S
index e3a375c4cb83..88f5aef7934c 100644
--- a/arch/arm64/crypto/aes-ce-ccm-core.S
+++ b/arch/arm64/crypto/aes-ce-ccm-core.S
@@ -19,24 +19,33 @@
 	 *			     u32 *macp, u8 const rk[], u32 rounds);
 	 */
 ENTRY(ce_aes_ccm_auth_data)
-	ldr	w8, [x3]			/* leftover from prev round? */
+	frame_push	7
+
+	mov	x19, x0
+	mov	x20, x1
+	mov	x21, x2
+	mov	x22, x3
+	mov	x23, x4
+	mov	x24, x5
+
+	ldr	w25, [x22]			/* leftover from prev round? */
 	ld1	{v0.16b}, [x0]			/* load mac */
-	cbz	w8, 1f
-	sub	w8, w8, #16
+	cbz	w25, 1f
+	sub	w25, w25, #16
 	eor	v1.16b, v1.16b, v1.16b
-0:	ldrb	w7, [x1], #1			/* get 1 byte of input */
-	subs	w2, w2, #1
-	add	w8, w8, #1
+0:	ldrb	w7, [x20], #1			/* get 1 byte of input */
+	subs	w21, w21, #1
+	add	w25, w25, #1
 	ins	v1.b[0], w7
 	ext	v1.16b, v1.16b, v1.16b, #1	/* rotate in the input bytes */
 	beq	8f				/* out of input? */
-	cbnz	w8, 0b
+	cbnz	w25, 0b
 	eor	v0.16b, v0.16b, v1.16b
-1:	ld1	{v3.4s}, [x4]			/* load first round key */
-	prfm	pldl1strm, [x1]
-	cmp	w5, #12				/* which key size? */
-	add	x6, x4, #16
-	sub	w7, w5, #2			/* modified # of rounds */
+1:	ld1	{v3.4s}, [x23]			/* load first round key */
+	prfm	pldl1strm, [x20]
+	cmp	w24, #12			/* which key size? */
+	add	x6, x23, #16
+	sub	w7, w24, #2			/* modified # of rounds */
 	bmi	2f
 	bne	5f
 	mov	v5.16b, v3.16b
@@ -55,33 +64,43 @@ ENTRY(ce_aes_ccm_auth_data)
 	ld1	{v5.4s}, [x6], #16		/* load next round key */
 	bpl	3b
 	aese	v0.16b, v4.16b
-	subs	w2, w2, #16			/* last data? */
+	subs	w21, w21, #16			/* last data? */
 	eor	v0.16b, v0.16b, v5.16b		/* final round */
 	bmi	6f
-	ld1	{v1.16b}, [x1], #16		/* load next input block */
+	ld1	{v1.16b}, [x20], #16		/* load next input block */
 	eor	v0.16b, v0.16b, v1.16b		/* xor with mac */
-	bne	1b
-6:	st1	{v0.16b}, [x0]			/* store mac */
+	beq	6f
+
+	if_will_cond_yield_neon
+	st1	{v0.16b}, [x19]			/* store mac */
+	do_cond_yield_neon
+	ld1	{v0.16b}, [x19]			/* reload mac */
+	endif_yield_neon
+
+	b	1b
+6:	st1	{v0.16b}, [x19]			/* store mac */
 	beq	10f
-	adds	w2, w2, #16
+	adds	w21, w21, #16
 	beq	10f
-	mov	w8, w2
-7:	ldrb	w7, [x1], #1
+	mov	w25, w21
+7:	ldrb	w7, [x20], #1
 	umov	w6, v0.b[0]
 	eor	w6, w6, w7
-	strb	w6, [x0], #1
-	subs	w2, w2, #1
+	strb	w6, [x19], #1
+	subs	w21, w21, #1
 	beq	10f
 	ext	v0.16b, v0.16b, v0.16b, #1	/* rotate out the mac bytes */
 	b	7b
-8:	mov	w7, w8
-	add	w8, w8, #16
+8:	mov	w7, w25
+	add	w25, w25, #16
 9:	ext	v1.16b, v1.16b, v1.16b, #1
 	adds	w7, w7, #1
 	bne	9b
 	eor	v0.16b, v0.16b, v1.16b
-	st1	{v0.16b}, [x0]
-10:	str	w8, [x3]
+	st1	{v0.16b}, [x19]
+10:	str	w25, [x22]
+
+	frame_pop
 	ret
 ENDPROC(ce_aes_ccm_auth_data)
 
@@ -126,19 +145,29 @@ ENTRY(ce_aes_ccm_final)
 ENDPROC(ce_aes_ccm_final)
 
 	.macro	aes_ccm_do_crypt,enc
-	ldr	x8, [x6, #8]			/* load lower ctr */
-	ld1	{v0.16b}, [x5]			/* load mac */
-CPU_LE(	rev	x8, x8			)	/* keep swabbed ctr in reg */
+	frame_push	8
+
+	mov	x19, x0
+	mov	x20, x1
+	mov	x21, x2
+	mov	x22, x3
+	mov	x23, x4
+	mov	x24, x5
+	mov	x25, x6
+
+	ldr	x26, [x25, #8]			/* load lower ctr */
+	ld1	{v0.16b}, [x24]			/* load mac */
+CPU_LE(	rev	x26, x26		)	/* keep swabbed ctr in reg */
 0:	/* outer loop */
-	ld1	{v1.8b}, [x6]			/* load upper ctr */
-	prfm	pldl1strm, [x1]
-	add	x8, x8, #1
-	rev	x9, x8
-	cmp	w4, #12				/* which key size? */
-	sub	w7, w4, #2			/* get modified # of rounds */
+	ld1	{v1.8b}, [x25]			/* load upper ctr */
+	prfm	pldl1strm, [x20]
+	add	x26, x26, #1
+	rev	x9, x26
+	cmp	w23, #12			/* which key size? */
+	sub	w7, w23, #2			/* get modified # of rounds */
 	ins	v1.d[1], x9			/* no carry in lower ctr */
-	ld1	{v3.4s}, [x3]			/* load first round key */
-	add	x10, x3, #16
+	ld1	{v3.4s}, [x22]			/* load first round key */
+	add	x10, x22, #16
 	bmi	1f
 	bne	4f
 	mov	v5.16b, v3.16b
@@ -165,9 +194,9 @@ CPU_LE(	rev	x8, x8			)	/* keep swabbed ctr in reg */
 	bpl	2b
 	aese	v0.16b, v4.16b
 	aese	v1.16b, v4.16b
-	subs	w2, w2, #16
-	bmi	6f				/* partial block? */
-	ld1	{v2.16b}, [x1], #16		/* load next input block */
+	subs	w21, w21, #16
+	bmi	7f				/* partial block? */
+	ld1	{v2.16b}, [x20], #16		/* load next input block */
 	.if	\enc == 1
 	eor	v2.16b, v2.16b, v5.16b		/* final round enc+mac */
 	eor	v1.16b, v1.16b, v2.16b		/* xor with crypted ctr */
@@ -176,18 +205,29 @@ CPU_LE(	rev	x8, x8			)	/* keep swabbed ctr in reg */
 	eor	v1.16b, v2.16b, v5.16b		/* final round enc */
 	.endif
 	eor	v0.16b, v0.16b, v2.16b		/* xor mac with pt ^ rk[last] */
-	st1	{v1.16b}, [x0], #16		/* write output block */
-	bne	0b
-CPU_LE(	rev	x8, x8			)
-	st1	{v0.16b}, [x5]			/* store mac */
-	str	x8, [x6, #8]			/* store lsb end of ctr (BE) */
-5:	ret
-
-6:	eor	v0.16b, v0.16b, v5.16b		/* final round mac */
+	st1	{v1.16b}, [x19], #16		/* write output block */
+	beq	5f
+
+	if_will_cond_yield_neon
+	st1	{v0.16b}, [x24]			/* store mac */
+	do_cond_yield_neon
+	ld1	{v0.16b}, [x24]			/* reload mac */
+	endif_yield_neon
+
+	b	0b
+5:
+CPU_LE(	rev	x26, x26			)
+	st1	{v0.16b}, [x24]			/* store mac */
+	str	x26, [x25, #8]			/* store lsb end of ctr (BE) */
+
+6:	frame_pop
+	ret
+
+7:	eor	v0.16b, v0.16b, v5.16b		/* final round mac */
 	eor	v1.16b, v1.16b, v5.16b		/* final round enc */
-	st1	{v0.16b}, [x5]			/* store mac */
-	add	w2, w2, #16			/* process partial tail block */
-7:	ldrb	w9, [x1], #1			/* get 1 byte of input */
+	st1	{v0.16b}, [x24]			/* store mac */
+	add	w21, w21, #16			/* process partial tail block */
+8:	ldrb	w9, [x20], #1			/* get 1 byte of input */
 	umov	w6, v1.b[0]			/* get top crypted ctr byte */
 	umov	w7, v0.b[0]			/* get top mac byte */
 	.if	\enc == 1
@@ -197,13 +237,13 @@ CPU_LE(	rev	x8, x8			)
 	eor	w9, w9, w6
 	eor	w7, w7, w9
 	.endif
-	strb	w9, [x0], #1			/* store out byte */
-	strb	w7, [x5], #1			/* store mac byte */
-	subs	w2, w2, #1
-	beq	5b
+	strb	w9, [x19], #1			/* store out byte */
+	strb	w7, [x24], #1			/* store mac byte */
+	subs	w21, w21, #1
+	beq	6b
 	ext	v0.16b, v0.16b, v0.16b, #1	/* shift out mac byte */
 	ext	v1.16b, v1.16b, v1.16b, #1	/* shift out ctr byte */
-	b	7b
+	b	8b
 	.endm
 
 	/*
diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S
index 50330f5c3adc..623e74ed1c67 100644
--- a/arch/arm64/crypto/aes-ce.S
+++ b/arch/arm64/crypto/aes-ce.S
@@ -30,18 +30,21 @@
 	.endm
 
 	/* prepare for encryption with key in rk[] */
-	.macro		enc_prepare, rounds, rk, ignore
-	load_round_keys	\rounds, \rk
+	.macro		enc_prepare, rounds, rk, temp
+	mov		\temp, \rk
+	load_round_keys	\rounds, \temp
 	.endm
 
 	/* prepare for encryption (again) but with new key in rk[] */
-	.macro		enc_switch_key, rounds, rk, ignore
-	load_round_keys	\rounds, \rk
+	.macro		enc_switch_key, rounds, rk, temp
+	mov		\temp, \rk
+	load_round_keys	\rounds, \temp
 	.endm
 
 	/* prepare for decryption with key in rk[] */
-	.macro		dec_prepare, rounds, rk, ignore
-	load_round_keys	\rounds, \rk
+	.macro		dec_prepare, rounds, rk, temp
+	mov		\temp, \rk
+	load_round_keys	\rounds, \temp
 	.endm
 
 	.macro		do_enc_Nx, de, mc, k, i0, i1, i2, i3
diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S
index a68412e1e3a4..483a7130cf0e 100644
--- a/arch/arm64/crypto/aes-modes.S
+++ b/arch/arm64/crypto/aes-modes.S
@@ -14,12 +14,12 @@
 	.align		4
 
 aes_encrypt_block4x:
-	encrypt_block4x	v0, v1, v2, v3, w3, x2, x8, w7
+	encrypt_block4x	v0, v1, v2, v3, w22, x21, x8, w7
 	ret
 ENDPROC(aes_encrypt_block4x)
 
 aes_decrypt_block4x:
-	decrypt_block4x	v0, v1, v2, v3, w3, x2, x8, w7
+	decrypt_block4x	v0, v1, v2, v3, w22, x21, x8, w7
 	ret
 ENDPROC(aes_decrypt_block4x)
 
@@ -31,57 +31,71 @@ ENDPROC(aes_decrypt_block4x)
 	 */
 
 AES_ENTRY(aes_ecb_encrypt)
-	stp		x29, x30, [sp, #-16]!
-	mov		x29, sp
+	frame_push	5
 
-	enc_prepare	w3, x2, x5
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+
+.Lecbencrestart:
+	enc_prepare	w22, x21, x5
 
 .LecbencloopNx:
-	subs		w4, w4, #4
+	subs		w23, w23, #4
 	bmi		.Lecbenc1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
+	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 pt blocks */
 	bl		aes_encrypt_block4x
-	st1		{v0.16b-v3.16b}, [x0], #64
+	st1		{v0.16b-v3.16b}, [x19], #64
+	cond_yield_neon	.Lecbencrestart
 	b		.LecbencloopNx
 .Lecbenc1x:
-	adds		w4, w4, #4
+	adds		w23, w23, #4
 	beq		.Lecbencout
 .Lecbencloop:
-	ld1		{v0.16b}, [x1], #16		/* get next pt block */
-	encrypt_block	v0, w3, x2, x5, w6
-	st1		{v0.16b}, [x0], #16
-	subs		w4, w4, #1
+	ld1		{v0.16b}, [x20], #16		/* get next pt block */
+	encrypt_block	v0, w22, x21, x5, w6
+	st1		{v0.16b}, [x19], #16
+	subs		w23, w23, #1
 	bne		.Lecbencloop
 .Lecbencout:
-	ldp		x29, x30, [sp], #16
+	frame_pop
 	ret
 AES_ENDPROC(aes_ecb_encrypt)
 
 
 AES_ENTRY(aes_ecb_decrypt)
-	stp		x29, x30, [sp, #-16]!
-	mov		x29, sp
+	frame_push	5
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
 
-	dec_prepare	w3, x2, x5
+.Lecbdecrestart:
+	dec_prepare	w22, x21, x5
 
 .LecbdecloopNx:
-	subs		w4, w4, #4
+	subs		w23, w23, #4
 	bmi		.Lecbdec1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
+	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 ct blocks */
 	bl		aes_decrypt_block4x
-	st1		{v0.16b-v3.16b}, [x0], #64
+	st1		{v0.16b-v3.16b}, [x19], #64
+	cond_yield_neon	.Lecbdecrestart
 	b		.LecbdecloopNx
 .Lecbdec1x:
-	adds		w4, w4, #4
+	adds		w23, w23, #4
 	beq		.Lecbdecout
 .Lecbdecloop:
-	ld1		{v0.16b}, [x1], #16		/* get next ct block */
-	decrypt_block	v0, w3, x2, x5, w6
-	st1		{v0.16b}, [x0], #16
-	subs		w4, w4, #1
+	ld1		{v0.16b}, [x20], #16		/* get next ct block */
+	decrypt_block	v0, w22, x21, x5, w6
+	st1		{v0.16b}, [x19], #16
+	subs		w23, w23, #1
 	bne		.Lecbdecloop
 .Lecbdecout:
-	ldp		x29, x30, [sp], #16
+	frame_pop
 	ret
 AES_ENDPROC(aes_ecb_decrypt)
 
@@ -94,78 +108,100 @@ AES_ENDPROC(aes_ecb_decrypt)
 	 */
 
 AES_ENTRY(aes_cbc_encrypt)
-	ld1		{v4.16b}, [x5]			/* get iv */
-	enc_prepare	w3, x2, x6
+	frame_push	6
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+	mov		x24, x5
+
+.Lcbcencrestart:
+	ld1		{v4.16b}, [x24]			/* get iv */
+	enc_prepare	w22, x21, x6
 
 .Lcbcencloop4x:
-	subs		w4, w4, #4
+	subs		w23, w23, #4
 	bmi		.Lcbcenc1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
+	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 pt blocks */
 	eor		v0.16b, v0.16b, v4.16b		/* ..and xor with iv */
-	encrypt_block	v0, w3, x2, x6, w7
+	encrypt_block	v0, w22, x21, x6, w7
 	eor		v1.16b, v1.16b, v0.16b
-	encrypt_block	v1, w3, x2, x6, w7
+	encrypt_block	v1, w22, x21, x6, w7
 	eor		v2.16b, v2.16b, v1.16b
-	encrypt_block	v2, w3, x2, x6, w7
+	encrypt_block	v2, w22, x21, x6, w7
 	eor		v3.16b, v3.16b, v2.16b
-	encrypt_block	v3, w3, x2, x6, w7
-	st1		{v0.16b-v3.16b}, [x0], #64
+	encrypt_block	v3, w22, x21, x6, w7
+	st1		{v0.16b-v3.16b}, [x19], #64
 	mov		v4.16b, v3.16b
+	st1		{v4.16b}, [x24]			/* return iv */
+	cond_yield_neon	.Lcbcencrestart
 	b		.Lcbcencloop4x
 .Lcbcenc1x:
-	adds		w4, w4, #4
+	adds		w23, w23, #4
 	beq		.Lcbcencout
 .Lcbcencloop:
-	ld1		{v0.16b}, [x1], #16		/* get next pt block */
+	ld1		{v0.16b}, [x20], #16		/* get next pt block */
 	eor		v4.16b, v4.16b, v0.16b		/* ..and xor with iv */
-	encrypt_block	v4, w3, x2, x6, w7
-	st1		{v4.16b}, [x0], #16
-	subs		w4, w4, #1
+	encrypt_block	v4, w22, x21, x6, w7
+	st1		{v4.16b}, [x19], #16
+	subs		w23, w23, #1
 	bne		.Lcbcencloop
 .Lcbcencout:
-	st1		{v4.16b}, [x5]			/* return iv */
+	st1		{v4.16b}, [x24]			/* return iv */
+	frame_pop
 	ret
 AES_ENDPROC(aes_cbc_encrypt)
 
 
 AES_ENTRY(aes_cbc_decrypt)
-	stp		x29, x30, [sp, #-16]!
-	mov		x29, sp
+	frame_push	6
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+	mov		x24, x5
 
-	ld1		{v7.16b}, [x5]			/* get iv */
-	dec_prepare	w3, x2, x6
+.Lcbcdecrestart:
+	ld1		{v7.16b}, [x24]			/* get iv */
+	dec_prepare	w22, x21, x6
 
 .LcbcdecloopNx:
-	subs		w4, w4, #4
+	subs		w23, w23, #4
 	bmi		.Lcbcdec1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
+	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 ct blocks */
 	mov		v4.16b, v0.16b
 	mov		v5.16b, v1.16b
 	mov		v6.16b, v2.16b
 	bl		aes_decrypt_block4x
-	sub		x1, x1, #16
+	sub		x20, x20, #16
 	eor		v0.16b, v0.16b, v7.16b
 	eor		v1.16b, v1.16b, v4.16b
-	ld1		{v7.16b}, [x1], #16		/* reload 1 ct block */
+	ld1		{v7.16b}, [x20], #16		/* reload 1 ct block */
 	eor		v2.16b, v2.16b, v5.16b
 	eor		v3.16b, v3.16b, v6.16b
-	st1		{v0.16b-v3.16b}, [x0], #64
+	st1		{v0.16b-v3.16b}, [x19], #64
+	st1		{v7.16b}, [x24]			/* return iv */
+	cond_yield_neon	.Lcbcdecrestart
 	b		.LcbcdecloopNx
 .Lcbcdec1x:
-	adds		w4, w4, #4
+	adds		w23, w23, #4
 	beq		.Lcbcdecout
 .Lcbcdecloop:
-	ld1		{v1.16b}, [x1], #16		/* get next ct block */
+	ld1		{v1.16b}, [x20], #16		/* get next ct block */
 	mov		v0.16b, v1.16b			/* ...and copy to v0 */
-	decrypt_block	v0, w3, x2, x6, w7
+	decrypt_block	v0, w22, x21, x6, w7
 	eor		v0.16b, v0.16b, v7.16b		/* xor with iv => pt */
 	mov		v7.16b, v1.16b			/* ct is next iv */
-	st1		{v0.16b}, [x0], #16
-	subs		w4, w4, #1
+	st1		{v0.16b}, [x19], #16
+	subs		w23, w23, #1
 	bne		.Lcbcdecloop
 .Lcbcdecout:
-	st1		{v7.16b}, [x5]			/* return iv */
-	ldp		x29, x30, [sp], #16
+	st1		{v7.16b}, [x24]			/* return iv */
+	frame_pop
 	ret
 AES_ENDPROC(aes_cbc_decrypt)
 
@@ -176,19 +212,26 @@ AES_ENDPROC(aes_cbc_decrypt)
 	 */
 
 AES_ENTRY(aes_ctr_encrypt)
-	stp		x29, x30, [sp, #-16]!
-	mov		x29, sp
+	frame_push	6
 
-	enc_prepare	w3, x2, x6
-	ld1		{v4.16b}, [x5]
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+	mov		x24, x5
+
+.Lctrrestart:
+	enc_prepare	w22, x21, x6
+	ld1		{v4.16b}, [x24]
 
 	umov		x6, v4.d[1]		/* keep swabbed ctr in reg */
 	rev		x6, x6
-	cmn		w6, w4			/* 32 bit overflow? */
-	bcs		.Lctrloop
 .LctrloopNx:
-	subs		w4, w4, #4
+	subs		w23, w23, #4
 	bmi		.Lctr1x
+	cmn		w6, #4			/* 32 bit overflow? */
+	bcs		.Lctr1x
 	ldr		q8, =0x30000000200000001	/* addends 1,2,3[,0] */
 	dup		v7.4s, w6
 	mov		v0.16b, v4.16b
@@ -200,25 +243,27 @@ AES_ENTRY(aes_ctr_encrypt)
 	mov		v1.s[3], v8.s[0]
 	mov		v2.s[3], v8.s[1]
 	mov		v3.s[3], v8.s[2]
-	ld1		{v5.16b-v7.16b}, [x1], #48	/* get 3 input blocks */
+	ld1		{v5.16b-v7.16b}, [x20], #48	/* get 3 input blocks */
 	bl		aes_encrypt_block4x
 	eor		v0.16b, v5.16b, v0.16b
-	ld1		{v5.16b}, [x1], #16		/* get 1 input block  */
+	ld1		{v5.16b}, [x20], #16		/* get 1 input block  */
 	eor		v1.16b, v6.16b, v1.16b
 	eor		v2.16b, v7.16b, v2.16b
 	eor		v3.16b, v5.16b, v3.16b
-	st1		{v0.16b-v3.16b}, [x0], #64
+	st1		{v0.16b-v3.16b}, [x19], #64
 	add		x6, x6, #4
 	rev		x7, x6
 	ins		v4.d[1], x7
-	cbz		w4, .Lctrout
+	cbz		w23, .Lctrout
+	st1		{v4.16b}, [x24]		/* return next CTR value */
+	cond_yield_neon	.Lctrrestart
 	b		.LctrloopNx
 .Lctr1x:
-	adds		w4, w4, #4
+	adds		w23, w23, #4
 	beq		.Lctrout
 .Lctrloop:
 	mov		v0.16b, v4.16b
-	encrypt_block	v0, w3, x2, x8, w7
+	encrypt_block	v0, w22, x21, x8, w7
 
 	adds		x6, x6, #1		/* increment BE ctr */
 	rev		x7, x6
@@ -226,22 +271,22 @@ AES_ENTRY(aes_ctr_encrypt)
 	bcs		.Lctrcarry		/* overflow? */
 
 .Lctrcarrydone:
-	subs		w4, w4, #1
+	subs		w23, w23, #1
 	bmi		.Lctrtailblock		/* blocks <0 means tail block */
-	ld1		{v3.16b}, [x1], #16
+	ld1		{v3.16b}, [x20], #16
 	eor		v3.16b, v0.16b, v3.16b
-	st1		{v3.16b}, [x0], #16
+	st1		{v3.16b}, [x19], #16
 	bne		.Lctrloop
 
 .Lctrout:
-	st1		{v4.16b}, [x5]		/* return next CTR value */
-	ldp		x29, x30, [sp], #16
+	st1		{v4.16b}, [x24]		/* return next CTR value */
+.Lctrret:
+	frame_pop
 	ret
 
 .Lctrtailblock:
-	st1		{v0.16b}, [x0]
-	ldp		x29, x30, [sp], #16
-	ret
+	st1		{v0.16b}, [x19]
+	b		.Lctrret
 
 .Lctrcarry:
 	umov		x7, v4.d[0]		/* load upper word of ctr  */
@@ -274,10 +319,16 @@ CPU_LE(	.quad		1, 0x87		)
 CPU_BE(	.quad		0x87, 1		)
 
 AES_ENTRY(aes_xts_encrypt)
-	stp		x29, x30, [sp, #-16]!
-	mov		x29, sp
+	frame_push	6
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+	mov		x24, x6
 
-	ld1		{v4.16b}, [x6]
+	ld1		{v4.16b}, [x24]
 	cbz		w7, .Lxtsencnotfirst
 
 	enc_prepare	w3, x5, x8
@@ -286,15 +337,17 @@ AES_ENTRY(aes_xts_encrypt)
 	ldr		q7, .Lxts_mul_x
 	b		.LxtsencNx
 
+.Lxtsencrestart:
+	ld1		{v4.16b}, [x24]
 .Lxtsencnotfirst:
-	enc_prepare	w3, x2, x8
+	enc_prepare	w22, x21, x8
 .LxtsencloopNx:
 	ldr		q7, .Lxts_mul_x
 	next_tweak	v4, v4, v7, v8
 .LxtsencNx:
-	subs		w4, w4, #4
+	subs		w23, w23, #4
 	bmi		.Lxtsenc1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
+	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 pt blocks */
 	next_tweak	v5, v4, v7, v8
 	eor		v0.16b, v0.16b, v4.16b
 	next_tweak	v6, v5, v7, v8
@@ -307,35 +360,43 @@ AES_ENTRY(aes_xts_encrypt)
 	eor		v0.16b, v0.16b, v4.16b
 	eor		v1.16b, v1.16b, v5.16b
 	eor		v2.16b, v2.16b, v6.16b
-	st1		{v0.16b-v3.16b}, [x0], #64
+	st1		{v0.16b-v3.16b}, [x19], #64
 	mov		v4.16b, v7.16b
-	cbz		w4, .Lxtsencout
+	cbz		w23, .Lxtsencout
+	st1		{v4.16b}, [x24]
+	cond_yield_neon	.Lxtsencrestart
 	b		.LxtsencloopNx
 .Lxtsenc1x:
-	adds		w4, w4, #4
+	adds		w23, w23, #4
 	beq		.Lxtsencout
 .Lxtsencloop:
-	ld1		{v1.16b}, [x1], #16
+	ld1		{v1.16b}, [x20], #16
 	eor		v0.16b, v1.16b, v4.16b
-	encrypt_block	v0, w3, x2, x8, w7
+	encrypt_block	v0, w22, x21, x8, w7
 	eor		v0.16b, v0.16b, v4.16b
-	st1		{v0.16b}, [x0], #16
-	subs		w4, w4, #1
+	st1		{v0.16b}, [x19], #16
+	subs		w23, w23, #1
 	beq		.Lxtsencout
 	next_tweak	v4, v4, v7, v8
 	b		.Lxtsencloop
 .Lxtsencout:
-	st1		{v4.16b}, [x6]
-	ldp		x29, x30, [sp], #16
+	st1		{v4.16b}, [x24]
+	frame_pop
 	ret
 AES_ENDPROC(aes_xts_encrypt)
 
 
 AES_ENTRY(aes_xts_decrypt)
-	stp		x29, x30, [sp, #-16]!
-	mov		x29, sp
+	frame_push	6
 
-	ld1		{v4.16b}, [x6]
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+	mov		x24, x6
+
+	ld1		{v4.16b}, [x24]
 	cbz		w7, .Lxtsdecnotfirst
 
 	enc_prepare	w3, x5, x8
@@ -344,15 +405,17 @@ AES_ENTRY(aes_xts_decrypt)
 	ldr		q7, .Lxts_mul_x
 	b		.LxtsdecNx
 
+.Lxtsdecrestart:
+	ld1		{v4.16b}, [x24]
 .Lxtsdecnotfirst:
-	dec_prepare	w3, x2, x8
+	dec_prepare	w22, x21, x8
 .LxtsdecloopNx:
 	ldr		q7, .Lxts_mul_x
 	next_tweak	v4, v4, v7, v8
 .LxtsdecNx:
-	subs		w4, w4, #4
+	subs		w23, w23, #4
 	bmi		.Lxtsdec1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
+	ld1		{v0.16b-v3.16b}, [x20], #64	/* get 4 ct blocks */
 	next_tweak	v5, v4, v7, v8
 	eor		v0.16b, v0.16b, v4.16b
 	next_tweak	v6, v5, v7, v8
@@ -365,26 +428,28 @@ AES_ENTRY(aes_xts_decrypt)
 	eor		v0.16b, v0.16b, v4.16b
 	eor		v1.16b, v1.16b, v5.16b
 	eor		v2.16b, v2.16b, v6.16b
-	st1		{v0.16b-v3.16b}, [x0], #64
+	st1		{v0.16b-v3.16b}, [x19], #64
 	mov		v4.16b, v7.16b
-	cbz		w4, .Lxtsdecout
+	cbz		w23, .Lxtsdecout
+	st1		{v4.16b}, [x24]
+	cond_yield_neon	.Lxtsdecrestart
 	b		.LxtsdecloopNx
 .Lxtsdec1x:
-	adds		w4, w4, #4
+	adds		w23, w23, #4
 	beq		.Lxtsdecout
 .Lxtsdecloop:
-	ld1		{v1.16b}, [x1], #16
+	ld1		{v1.16b}, [x20], #16
 	eor		v0.16b, v1.16b, v4.16b
-	decrypt_block	v0, w3, x2, x8, w7
+	decrypt_block	v0, w22, x21, x8, w7
 	eor		v0.16b, v0.16b, v4.16b
-	st1		{v0.16b}, [x0], #16
-	subs		w4, w4, #1
+	st1		{v0.16b}, [x19], #16
+	subs		w23, w23, #1
 	beq		.Lxtsdecout
 	next_tweak	v4, v4, v7, v8
 	b		.Lxtsdecloop
 .Lxtsdecout:
-	st1		{v4.16b}, [x6]
-	ldp		x29, x30, [sp], #16
+	st1		{v4.16b}, [x24]
+	frame_pop
 	ret
 AES_ENDPROC(aes_xts_decrypt)
 
@@ -393,43 +458,61 @@ AES_ENDPROC(aes_xts_decrypt)
 	 *		  int blocks, u8 dg[], int enc_before, int enc_after)
 	 */
 AES_ENTRY(aes_mac_update)
-	ld1		{v0.16b}, [x4]			/* get dg */
+	frame_push	6
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+	mov		x24, x6
+
+	ld1		{v0.16b}, [x23]			/* get dg */
 	enc_prepare	w2, x1, x7
 	cbz		w5, .Lmacloop4x
 
 	encrypt_block	v0, w2, x1, x7, w8
 
 .Lmacloop4x:
-	subs		w3, w3, #4
+	subs		w22, w22, #4
 	bmi		.Lmac1x
-	ld1		{v1.16b-v4.16b}, [x0], #64	/* get next pt block */
+	ld1		{v1.16b-v4.16b}, [x19], #64	/* get next pt block */
 	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */
-	encrypt_block	v0, w2, x1, x7, w8
+	encrypt_block	v0, w21, x20, x7, w8
 	eor		v0.16b, v0.16b, v2.16b
-	encrypt_block	v0, w2, x1, x7, w8
+	encrypt_block	v0, w21, x20, x7, w8
 	eor		v0.16b, v0.16b, v3.16b
-	encrypt_block	v0, w2, x1, x7, w8
+	encrypt_block	v0, w21, x20, x7, w8
 	eor		v0.16b, v0.16b, v4.16b
-	cmp		w3, wzr
-	csinv		x5, x6, xzr, eq
+	cmp		w22, wzr
+	csinv		x5, x24, xzr, eq
 	cbz		w5, .Lmacout
-	encrypt_block	v0, w2, x1, x7, w8
+	encrypt_block	v0, w21, x20, x7, w8
+	st1		{v0.16b}, [x23]			/* return dg */
+	cond_yield_neon	.Lmacrestart
 	b		.Lmacloop4x
 .Lmac1x:
-	add		w3, w3, #4
+	add		w22, w22, #4
 .Lmacloop:
-	cbz		w3, .Lmacout
-	ld1		{v1.16b}, [x0], #16		/* get next pt block */
+	cbz		w22, .Lmacout
+	ld1		{v1.16b}, [x19], #16		/* get next pt block */
 	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */
 
-	subs		w3, w3, #1
-	csinv		x5, x6, xzr, eq
+	subs		w22, w22, #1
+	csinv		x5, x24, xzr, eq
 	cbz		w5, .Lmacout
 
-	encrypt_block	v0, w2, x1, x7, w8
+.Lmacenc:
+	encrypt_block	v0, w21, x20, x7, w8
 	b		.Lmacloop
 
 .Lmacout:
-	st1		{v0.16b}, [x4]			/* return dg */
+	st1		{v0.16b}, [x23]			/* return dg */
+	frame_pop
 	ret
+
+.Lmacrestart:
+	ld1		{v0.16b}, [x23]			/* get dg */
+	enc_prepare	w21, x20, x0
+	b		.Lmacloop4x
 AES_ENDPROC(aes_mac_update)
diff --git a/arch/arm64/crypto/aes-neonbs-core.S b/arch/arm64/crypto/aes-neonbs-core.S
index ca0472500433..e613a87f8b53 100644
--- a/arch/arm64/crypto/aes-neonbs-core.S
+++ b/arch/arm64/crypto/aes-neonbs-core.S
@@ -565,54 +565,61 @@ ENDPROC(aesbs_decrypt8)
 	 *		     int blocks)
 	 */
 	.macro		__ecb_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
-	stp		x29, x30, [sp, #-16]!
-	mov		x29, sp
+	frame_push	5
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
 
 99:	mov		x5, #1
-	lsl		x5, x5, x4
-	subs		w4, w4, #8
-	csel		x4, x4, xzr, pl
+	lsl		x5, x5, x23
+	subs		w23, w23, #8
+	csel		x23, x23, xzr, pl
 	csel		x5, x5, xzr, mi
 
-	ld1		{v0.16b}, [x1], #16
+	ld1		{v0.16b}, [x20], #16
 	tbnz		x5, #1, 0f
-	ld1		{v1.16b}, [x1], #16
+	ld1		{v1.16b}, [x20], #16
 	tbnz		x5, #2, 0f
-	ld1		{v2.16b}, [x1], #16
+	ld1		{v2.16b}, [x20], #16
 	tbnz		x5, #3, 0f
-	ld1		{v3.16b}, [x1], #16
+	ld1		{v3.16b}, [x20], #16
 	tbnz		x5, #4, 0f
-	ld1		{v4.16b}, [x1], #16
+	ld1		{v4.16b}, [x20], #16
 	tbnz		x5, #5, 0f
-	ld1		{v5.16b}, [x1], #16
+	ld1		{v5.16b}, [x20], #16
 	tbnz		x5, #6, 0f
-	ld1		{v6.16b}, [x1], #16
+	ld1		{v6.16b}, [x20], #16
 	tbnz		x5, #7, 0f
-	ld1		{v7.16b}, [x1], #16
+	ld1		{v7.16b}, [x20], #16
 
-0:	mov		bskey, x2
-	mov		rounds, x3
+0:	mov		bskey, x21
+	mov		rounds, x22
 	bl		\do8
 
-	st1		{\o0\().16b}, [x0], #16
+	st1		{\o0\().16b}, [x19], #16
 	tbnz		x5, #1, 1f
-	st1		{\o1\().16b}, [x0], #16
+	st1		{\o1\().16b}, [x19], #16
 	tbnz		x5, #2, 1f
-	st1		{\o2\().16b}, [x0], #16
+	st1		{\o2\().16b}, [x19], #16
 	tbnz		x5, #3, 1f
-	st1		{\o3\().16b}, [x0], #16
+	st1		{\o3\().16b}, [x19], #16
 	tbnz		x5, #4, 1f
-	st1		{\o4\().16b}, [x0], #16
+	st1		{\o4\().16b}, [x19], #16
 	tbnz		x5, #5, 1f
-	st1		{\o5\().16b}, [x0], #16
+	st1		{\o5\().16b}, [x19], #16
 	tbnz		x5, #6, 1f
-	st1		{\o6\().16b}, [x0], #16
+	st1		{\o6\().16b}, [x19], #16
 	tbnz		x5, #7, 1f
-	st1		{\o7\().16b}, [x0], #16
+	st1		{\o7\().16b}, [x19], #16
 
-	cbnz		x4, 99b
+	cbz		x23, 1f
+	cond_yield_neon
+	b		99b
 
-1:	ldp		x29, x30, [sp], #16
+1:	frame_pop
 	ret
 	.endm
 
@@ -632,43 +639,49 @@ ENDPROC(aesbs_ecb_decrypt)
 	 */
 	.align		4
 ENTRY(aesbs_cbc_decrypt)
-	stp		x29, x30, [sp, #-16]!
-	mov		x29, sp
+	frame_push	6
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+	mov		x24, x5
 
 99:	mov		x6, #1
-	lsl		x6, x6, x4
-	subs		w4, w4, #8
-	csel		x4, x4, xzr, pl
+	lsl		x6, x6, x23
+	subs		w23, w23, #8
+	csel		x23, x23, xzr, pl
 	csel		x6, x6, xzr, mi
 
-	ld1		{v0.16b}, [x1], #16
+	ld1		{v0.16b}, [x20], #16
 	mov		v25.16b, v0.16b
 	tbnz		x6, #1, 0f
-	ld1		{v1.16b}, [x1], #16
+	ld1		{v1.16b}, [x20], #16
 	mov		v26.16b, v1.16b
 	tbnz		x6, #2, 0f
-	ld1		{v2.16b}, [x1], #16
+	ld1		{v2.16b}, [x20], #16
 	mov		v27.16b, v2.16b
 	tbnz		x6, #3, 0f
-	ld1		{v3.16b}, [x1], #16
+	ld1		{v3.16b}, [x20], #16
 	mov		v28.16b, v3.16b
 	tbnz		x6, #4, 0f
-	ld1		{v4.16b}, [x1], #16
+	ld1		{v4.16b}, [x20], #16
 	mov		v29.16b, v4.16b
 	tbnz		x6, #5, 0f
-	ld1		{v5.16b}, [x1], #16
+	ld1		{v5.16b}, [x20], #16
 	mov		v30.16b, v5.16b
 	tbnz		x6, #6, 0f
-	ld1		{v6.16b}, [x1], #16
+	ld1		{v6.16b}, [x20], #16
 	mov		v31.16b, v6.16b
 	tbnz		x6, #7, 0f
-	ld1		{v7.16b}, [x1]
+	ld1		{v7.16b}, [x20]
 
-0:	mov		bskey, x2
-	mov		rounds, x3
+0:	mov		bskey, x21
+	mov		rounds, x22
 	bl		aesbs_decrypt8
 
-	ld1		{v24.16b}, [x5]			// load IV
+	ld1		{v24.16b}, [x24]		// load IV
 
 	eor		v1.16b, v1.16b, v25.16b
 	eor		v6.16b, v6.16b, v26.16b
@@ -679,34 +692,36 @@ ENTRY(aesbs_cbc_decrypt)
 	eor		v3.16b, v3.16b, v30.16b
 	eor		v5.16b, v5.16b, v31.16b
 
-	st1		{v0.16b}, [x0], #16
+	st1		{v0.16b}, [x19], #16
 	mov		v24.16b, v25.16b
 	tbnz		x6, #1, 1f
-	st1		{v1.16b}, [x0], #16
+	st1		{v1.16b}, [x19], #16
 	mov		v24.16b, v26.16b
 	tbnz		x6, #2, 1f
-	st1		{v6.16b}, [x0], #16
+	st1		{v6.16b}, [x19], #16
 	mov		v24.16b, v27.16b
 	tbnz		x6, #3, 1f
-	st1		{v4.16b}, [x0], #16
+	st1		{v4.16b}, [x19], #16
 	mov		v24.16b, v28.16b
 	tbnz		x6, #4, 1f
-	st1		{v2.16b}, [x0], #16
+	st1		{v2.16b}, [x19], #16
 	mov		v24.16b, v29.16b
 	tbnz		x6, #5, 1f
-	st1		{v7.16b}, [x0], #16
+	st1		{v7.16b}, [x19], #16
 	mov		v24.16b, v30.16b
 	tbnz		x6, #6, 1f
-	st1		{v3.16b}, [x0], #16
+	st1		{v3.16b}, [x19], #16
 	mov		v24.16b, v31.16b
 	tbnz		x6, #7, 1f
-	ld1		{v24.16b}, [x1], #16
-	st1		{v5.16b}, [x0], #16
-1:	st1		{v24.16b}, [x5]			// store IV
+	ld1		{v24.16b}, [x20], #16
+	st1		{v5.16b}, [x19], #16
+1:	st1		{v24.16b}, [x24]		// store IV
 
-	cbnz		x4, 99b
+	cbz		x23, 2f
+	cond_yield_neon
+	b		99b
 
-	ldp		x29, x30, [sp], #16
+2:	frame_pop
 	ret
 ENDPROC(aesbs_cbc_decrypt)
 
@@ -731,87 +746,93 @@ CPU_BE(	.quad		0x87, 1		)
 	 */
 __xts_crypt8:
 	mov		x6, #1
-	lsl		x6, x6, x4
-	subs		w4, w4, #8
-	csel		x4, x4, xzr, pl
+	lsl		x6, x6, x23
+	subs		w23, w23, #8
+	csel		x23, x23, xzr, pl
 	csel		x6, x6, xzr, mi
 
-	ld1		{v0.16b}, [x1], #16
+	ld1		{v0.16b}, [x20], #16
 	next_tweak	v26, v25, v30, v31
 	eor		v0.16b, v0.16b, v25.16b
 	tbnz		x6, #1, 0f
 
-	ld1		{v1.16b}, [x1], #16
+	ld1		{v1.16b}, [x20], #16
 	next_tweak	v27, v26, v30, v31
 	eor		v1.16b, v1.16b, v26.16b
 	tbnz		x6, #2, 0f
 
-	ld1		{v2.16b}, [x1], #16
+	ld1		{v2.16b}, [x20], #16
 	next_tweak	v28, v27, v30, v31
 	eor		v2.16b, v2.16b, v27.16b
 	tbnz		x6, #3, 0f
 
-	ld1		{v3.16b}, [x1], #16
+	ld1		{v3.16b}, [x20], #16
 	next_tweak	v29, v28, v30, v31
 	eor		v3.16b, v3.16b, v28.16b
 	tbnz		x6, #4, 0f
 
-	ld1		{v4.16b}, [x1], #16
-	str		q29, [sp, #16]
+	ld1		{v4.16b}, [x20], #16
+	str		q29, [sp, #.Lframe_local_offset]
 	eor		v4.16b, v4.16b, v29.16b
 	next_tweak	v29, v29, v30, v31
 	tbnz		x6, #5, 0f
 
-	ld1		{v5.16b}, [x1], #16
-	str		q29, [sp, #32]
+	ld1		{v5.16b}, [x20], #16
+	str		q29, [sp, #.Lframe_local_offset + 16]
 	eor		v5.16b, v5.16b, v29.16b
 	next_tweak	v29, v29, v30, v31
 	tbnz		x6, #6, 0f
 
-	ld1		{v6.16b}, [x1], #16
-	str		q29, [sp, #48]
+	ld1		{v6.16b}, [x20], #16
+	str		q29, [sp, #.Lframe_local_offset + 32]
 	eor		v6.16b, v6.16b, v29.16b
 	next_tweak	v29, v29, v30, v31
 	tbnz		x6, #7, 0f
 
-	ld1		{v7.16b}, [x1], #16
-	str		q29, [sp, #64]
+	ld1		{v7.16b}, [x20], #16
+	str		q29, [sp, #.Lframe_local_offset + 48]
 	eor		v7.16b, v7.16b, v29.16b
 	next_tweak	v29, v29, v30, v31
 
-0:	mov		bskey, x2
-	mov		rounds, x3
+0:	mov		bskey, x21
+	mov		rounds, x22
 	br		x7
 ENDPROC(__xts_crypt8)
 
 	.macro		__xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
-	stp		x29, x30, [sp, #-80]!
-	mov		x29, sp
+	frame_push	6, 64
 
-	ldr		q30, .Lxts_mul_x
-	ld1		{v25.16b}, [x5]
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+	mov		x24, x5
+
+0:	ldr		q30, .Lxts_mul_x
+	ld1		{v25.16b}, [x24]
 
 99:	adr		x7, \do8
 	bl		__xts_crypt8
 
-	ldp		q16, q17, [sp, #16]
-	ldp		q18, q19, [sp, #48]
+	ldp		q16, q17, [sp, #.Lframe_local_offset]
+	ldp		q18, q19, [sp, #.Lframe_local_offset + 32]
 
 	eor		\o0\().16b, \o0\().16b, v25.16b
 	eor		\o1\().16b, \o1\().16b, v26.16b
 	eor		\o2\().16b, \o2\().16b, v27.16b
 	eor		\o3\().16b, \o3\().16b, v28.16b
 
-	st1		{\o0\().16b}, [x0], #16
+	st1		{\o0\().16b}, [x19], #16
 	mov		v25.16b, v26.16b
 	tbnz		x6, #1, 1f
-	st1		{\o1\().16b}, [x0], #16
+	st1		{\o1\().16b}, [x19], #16
 	mov		v25.16b, v27.16b
 	tbnz		x6, #2, 1f
-	st1		{\o2\().16b}, [x0], #16
+	st1		{\o2\().16b}, [x19], #16
 	mov		v25.16b, v28.16b
 	tbnz		x6, #3, 1f
-	st1		{\o3\().16b}, [x0], #16
+	st1		{\o3\().16b}, [x19], #16
 	mov		v25.16b, v29.16b
 	tbnz		x6, #4, 1f
 
@@ -820,18 +841,22 @@ ENDPROC(__xts_crypt8)
 	eor		\o6\().16b, \o6\().16b, v18.16b
 	eor		\o7\().16b, \o7\().16b, v19.16b
 
-	st1		{\o4\().16b}, [x0], #16
+	st1		{\o4\().16b}, [x19], #16
 	tbnz		x6, #5, 1f
-	st1		{\o5\().16b}, [x0], #16
+	st1		{\o5\().16b}, [x19], #16
 	tbnz		x6, #6, 1f
-	st1		{\o6\().16b}, [x0], #16
+	st1		{\o6\().16b}, [x19], #16
 	tbnz		x6, #7, 1f
-	st1		{\o7\().16b}, [x0], #16
+	st1		{\o7\().16b}, [x19], #16
+
+	cbz		x23, 1f
+	st1		{v25.16b}, [x24]
 
-	cbnz		x4, 99b
+	cond_yield_neon	0b
+	b		99b
 
-1:	st1		{v25.16b}, [x5]
-	ldp		x29, x30, [sp], #80
+1:	st1		{v25.16b}, [x24]
+	frame_pop
 	ret
 	.endm
 
@@ -856,24 +881,31 @@ ENDPROC(aesbs_xts_decrypt)
 	 *		     int rounds, int blocks, u8 iv[], u8 final[])
 	 */
 ENTRY(aesbs_ctr_encrypt)
-	stp		x29, x30, [sp, #-16]!
-	mov		x29, sp
-
-	cmp		x6, #0
-	cset		x10, ne
-	add		x4, x4, x10		// do one extra block if final
-
-	ldp		x7, x8, [x5]
-	ld1		{v0.16b}, [x5]
+	frame_push	8
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+	mov		x24, x5
+	mov		x25, x6
+
+	cmp		x25, #0
+	cset		x26, ne
+	add		x23, x23, x26		// do one extra block if final
+
+98:	ldp		x7, x8, [x24]
+	ld1		{v0.16b}, [x24]
 CPU_LE(	rev		x7, x7		)
 CPU_LE(	rev		x8, x8		)
 	adds		x8, x8, #1
 	adc		x7, x7, xzr
 
 99:	mov		x9, #1
-	lsl		x9, x9, x4
-	subs		w4, w4, #8
-	csel		x4, x4, xzr, pl
+	lsl		x9, x9, x23
+	subs		w23, w23, #8
+	csel		x23, x23, xzr, pl
 	csel		x9, x9, xzr, le
 
 	tbnz		x9, #1, 0f
@@ -891,82 +923,85 @@ CPU_LE(	rev		x8, x8		)
 	tbnz		x9, #7, 0f
 	next_ctr	v7
 
-0:	mov		bskey, x2
-	mov		rounds, x3
+0:	mov		bskey, x21
+	mov		rounds, x22
 	bl		aesbs_encrypt8
 
-	lsr		x9, x9, x10		// disregard the extra block
+	lsr		x9, x9, x26		// disregard the extra block
 	tbnz		x9, #0, 0f
 
-	ld1		{v8.16b}, [x1], #16
+	ld1		{v8.16b}, [x20], #16
 	eor		v0.16b, v0.16b, v8.16b
-	st1		{v0.16b}, [x0], #16
+	st1		{v0.16b}, [x19], #16
 	tbnz		x9, #1, 1f
 
-	ld1		{v9.16b}, [x1], #16
+	ld1		{v9.16b}, [x20], #16
 	eor		v1.16b, v1.16b, v9.16b
-	st1		{v1.16b}, [x0], #16
+	st1		{v1.16b}, [x19], #16
 	tbnz		x9, #2, 2f
 
-	ld1		{v10.16b}, [x1], #16
+	ld1		{v10.16b}, [x20], #16
 	eor		v4.16b, v4.16b, v10.16b
-	st1		{v4.16b}, [x0], #16
+	st1		{v4.16b}, [x19], #16
 	tbnz		x9, #3, 3f
 
-	ld1		{v11.16b}, [x1], #16
+	ld1		{v11.16b}, [x20], #16
 	eor		v6.16b, v6.16b, v11.16b
-	st1		{v6.16b}, [x0], #16
+	st1		{v6.16b}, [x19], #16
 	tbnz		x9, #4, 4f
 
-	ld1		{v12.16b}, [x1], #16
+	ld1		{v12.16b}, [x20], #16
 	eor		v3.16b, v3.16b, v12.16b
-	st1		{v3.16b}, [x0], #16
+	st1		{v3.16b}, [x19], #16
 	tbnz		x9, #5, 5f
 
-	ld1		{v13.16b}, [x1], #16
+	ld1		{v13.16b}, [x20], #16
 	eor		v7.16b, v7.16b, v13.16b
-	st1		{v7.16b}, [x0], #16
+	st1		{v7.16b}, [x19], #16
 	tbnz		x9, #6, 6f
 
-	ld1		{v14.16b}, [x1], #16
+	ld1		{v14.16b}, [x20], #16
 	eor		v2.16b, v2.16b, v14.16b
-	st1		{v2.16b}, [x0], #16
+	st1		{v2.16b}, [x19], #16
 	tbnz		x9, #7, 7f
 
-	ld1		{v15.16b}, [x1], #16
+	ld1		{v15.16b}, [x20], #16
 	eor		v5.16b, v5.16b, v15.16b
-	st1		{v5.16b}, [x0], #16
+	st1		{v5.16b}, [x19], #16
 
 8:	next_ctr	v0
-	cbnz		x4, 99b
+	st1		{v0.16b}, [x24]
+	cbz		x23, 0f
+
+	cond_yield_neon	98b
+	b		99b
 
-0:	st1		{v0.16b}, [x5]
-	ldp		x29, x30, [sp], #16
+0:	frame_pop
 	ret
 
 	/*
 	 * If we are handling the tail of the input (x6 != NULL), return the
 	 * final keystream block back to the caller.
 	 */
-1:	cbz		x6, 8b
-	st1		{v1.16b}, [x6]
+1:	cbz		x25, 8b
+	st1		{v1.16b}, [x25]
 	b		8b
-2:	cbz		x6, 8b
-	st1		{v4.16b}, [x6]
+2:	cbz		x25, 8b
+	st1		{v4.16b}, [x25]
 	b		8b
-3:	cbz		x6, 8b
-	st1		{v6.16b}, [x6]
+3:	cbz		x25, 8b
+	st1		{v6.16b}, [x25]
 	b		8b
-4:	cbz		x6, 8b
-	st1		{v3.16b}, [x6]
+4:	cbz		x25, 8b
+	st1		{v3.16b}, [x25]
 	b		8b
-5:	cbz		x6, 8b
-	st1		{v7.16b}, [x6]
+5:	cbz		x25, 8b
+	st1		{v7.16b}, [x25]
 	b		8b
-6:	cbz		x6, 8b
-	st1		{v2.16b}, [x6]
+6:	cbz		x25, 8b
+	st1		{v2.16b}, [x25]
 	b		8b
-7:	cbz		x6, 8b
-	st1		{v5.16b}, [x6]
+7:	cbz		x25, 8b
+	st1		{v5.16b}, [x25]
 	b		8b
 ENDPROC(aesbs_ctr_encrypt)
diff --git a/arch/arm64/crypto/crc32-ce-core.S b/arch/arm64/crypto/crc32-ce-core.S
index 16ed3c7ebd37..8061bf0f9c66 100644
--- a/arch/arm64/crypto/crc32-ce-core.S
+++ b/arch/arm64/crypto/crc32-ce-core.S
@@ -100,9 +100,10 @@
 	dCONSTANT	.req	d0
 	qCONSTANT	.req	q0
 
-	BUF		.req	x0
-	LEN		.req	x1
-	CRC		.req	x2
+	BUF		.req	x19
+	LEN		.req	x20
+	CRC		.req	x21
+	CONST		.req	x22
 
 	vzr		.req	v9
 
@@ -123,7 +124,14 @@ ENTRY(crc32_pmull_le)
 ENTRY(crc32c_pmull_le)
 	adr_l		x3, .Lcrc32c_constants
 
-0:	bic		LEN, LEN, #15
+0:	frame_push	4, 64
+
+	mov		BUF, x0
+	mov		LEN, x1
+	mov		CRC, x2
+	mov		CONST, x3
+
+	bic		LEN, LEN, #15
 	ld1		{v1.16b-v4.16b}, [BUF], #0x40
 	movi		vzr.16b, #0
 	fmov		dCONSTANT, CRC
@@ -132,7 +140,7 @@ ENTRY(crc32c_pmull_le)
 	cmp		LEN, #0x40
 	b.lt		less_64
 
-	ldr		qCONSTANT, [x3]
+	ldr		qCONSTANT, [CONST]
 
 loop_64:		/* 64 bytes Full cache line folding */
 	sub		LEN, LEN, #0x40
@@ -162,10 +170,21 @@ loop_64:		/* 64 bytes Full cache line folding */
 	eor		v4.16b, v4.16b, v8.16b
 
 	cmp		LEN, #0x40
-	b.ge		loop_64
+	b.lt		less_64
+
+	if_will_cond_yield_neon
+	stp		q1, q2, [sp, #.Lframe_local_offset]
+	stp		q3, q4, [sp, #.Lframe_local_offset + 32]
+	do_cond_yield_neon
+	ldp		q1, q2, [sp, #.Lframe_local_offset]
+	ldp		q3, q4, [sp, #.Lframe_local_offset + 32]
+	ldr		qCONSTANT, [CONST]
+	movi		vzr.16b, #0
+	endif_yield_neon
+	b		loop_64
 
 less_64:		/* Folding cache line into 128bit */
-	ldr		qCONSTANT, [x3, #16]
+	ldr		qCONSTANT, [CONST, #16]
 
 	pmull2		v5.1q, v1.2d, vCONSTANT.2d
 	pmull		v1.1q, v1.1d, vCONSTANT.1d
@@ -204,8 +223,8 @@ fold_64:
 	eor		v1.16b, v1.16b, v2.16b
 
 	/* final 32-bit fold */
-	ldr		dCONSTANT, [x3, #32]
-	ldr		d3, [x3, #40]
+	ldr		dCONSTANT, [CONST, #32]
+	ldr		d3, [CONST, #40]
 
 	ext		v2.16b, v1.16b, vzr.16b, #4
 	and		v1.16b, v1.16b, v3.16b
@@ -213,7 +232,7 @@ fold_64:
 	eor		v1.16b, v1.16b, v2.16b
 
 	/* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
-	ldr		qCONSTANT, [x3, #48]
+	ldr		qCONSTANT, [CONST, #48]
 
 	and		v2.16b, v1.16b, v3.16b
 	ext		v2.16b, vzr.16b, v2.16b, #8
@@ -223,6 +242,7 @@ fold_64:
 	eor		v1.16b, v1.16b, v2.16b
 	mov		w0, v1.s[1]
 
+	frame_pop
 	ret
 ENDPROC(crc32_pmull_le)
 ENDPROC(crc32c_pmull_le)
diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm64/crypto/crct10dif-ce-core.S
index f179c01bd55c..663ea71cdb38 100644
--- a/arch/arm64/crypto/crct10dif-ce-core.S
+++ b/arch/arm64/crypto/crct10dif-ce-core.S
@@ -74,13 +74,19 @@
 	.text
 	.cpu		generic+crypto
 
-	arg1_low32	.req	w0
-	arg2		.req	x1
-	arg3		.req	x2
+	arg1_low32	.req	w19
+	arg2		.req	x20
+	arg3		.req	x21
 
 	vzr		.req	v13
 
 ENTRY(crc_t10dif_pmull)
+	frame_push	3, 128
+
+	mov		arg1_low32, w0
+	mov		arg2, x1
+	mov		arg3, x2
+
 	movi		vzr.16b, #0		// init zero register
 
 	// adjust the 16-bit initial_crc value, scale it to 32 bits
@@ -175,8 +181,25 @@ CPU_LE(	ext		v12.16b, v12.16b, v12.16b, #8	)
 	subs		arg3, arg3, #128
 
 	// check if there is another 64B in the buffer to be able to fold
-	b.ge		_fold_64_B_loop
+	b.lt		_fold_64_B_end
+
+	if_will_cond_yield_neon
+	stp		q0, q1, [sp, #.Lframe_local_offset]
+	stp		q2, q3, [sp, #.Lframe_local_offset + 32]
+	stp		q4, q5, [sp, #.Lframe_local_offset + 64]
+	stp		q6, q7, [sp, #.Lframe_local_offset + 96]
+	do_cond_yield_neon
+	ldp		q0, q1, [sp, #.Lframe_local_offset]
+	ldp		q2, q3, [sp, #.Lframe_local_offset + 32]
+	ldp		q4, q5, [sp, #.Lframe_local_offset + 64]
+	ldp		q6, q7, [sp, #.Lframe_local_offset + 96]
+	ldr_l		q10, rk3, x8
+	movi		vzr.16b, #0		// init zero register
+	endif_yield_neon
+
+	b		_fold_64_B_loop
 
+_fold_64_B_end:
 	// at this point, the buffer pointer is pointing at the last y Bytes
 	// of the buffer the 64B of folded data is in 4 of the vector
 	// registers: v0, v1, v2, v3
@@ -304,6 +327,7 @@ _barrett:
 _cleanup:
 	// scale the result back to 16 bits
 	lsr		x0, x0, #16
+	frame_pop
 	ret
 
 _less_than_128:
diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S
index 11ebf1ae248a..dcffb9e77589 100644
--- a/arch/arm64/crypto/ghash-ce-core.S
+++ b/arch/arm64/crypto/ghash-ce-core.S
@@ -213,22 +213,31 @@
 	.endm
 
 	.macro		__pmull_ghash, pn
-	ld1		{SHASH.2d}, [x3]
-	ld1		{XL.2d}, [x1]
+	frame_push	5
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+
+0:	ld1		{SHASH.2d}, [x22]
+	ld1		{XL.2d}, [x20]
 	ext		SHASH2.16b, SHASH.16b, SHASH.16b, #8
 	eor		SHASH2.16b, SHASH2.16b, SHASH.16b
 
 	__pmull_pre_\pn
 
 	/* do the head block first, if supplied */
-	cbz		x4, 0f
-	ld1		{T1.2d}, [x4]
-	b		1f
+	cbz		x23, 1f
+	ld1		{T1.2d}, [x23]
+	mov		x23, xzr
+	b		2f
 
-0:	ld1		{T1.2d}, [x2], #16
-	sub		w0, w0, #1
+1:	ld1		{T1.2d}, [x21], #16
+	sub		w19, w19, #1
 
-1:	/* multiply XL by SHASH in GF(2^128) */
+2:	/* multiply XL by SHASH in GF(2^128) */
 CPU_LE(	rev64		T1.16b, T1.16b	)
 
 	ext		T2.16b, XL.16b, XL.16b, #8
@@ -250,9 +259,18 @@ CPU_LE(	rev64		T1.16b, T1.16b	)
 	eor		T2.16b, T2.16b, XH.16b
 	eor		XL.16b, XL.16b, T2.16b
 
-	cbnz		w0, 0b
+	cbz		w19, 3f
+
+	if_will_cond_yield_neon
+	st1		{XL.2d}, [x20]
+	do_cond_yield_neon
+	b		0b
+	endif_yield_neon
+
+	b		1b
 
-	st1		{XL.2d}, [x1]
+3:	st1		{XL.2d}, [x20]
+	frame_pop
 	ret
 	.endm
 
@@ -304,38 +322,55 @@ ENDPROC(pmull_ghash_update_p8)
 	.endm
 
 	.macro		pmull_gcm_do_crypt, enc
-	ld1		{SHASH.2d}, [x4]
-	ld1		{XL.2d}, [x1]
-	ldr		x8, [x5, #8]			// load lower counter
+	frame_push	10
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+	mov		x22, x3
+	mov		x23, x4
+	mov		x24, x5
+	mov		x25, x6
+	mov		x26, x7
+	.if		\enc == 1
+	ldr		x27, [sp, #96]			// first stacked arg
+	.endif
+
+	ldr		x28, [x24, #8]			// load lower counter
+CPU_LE(	rev		x28, x28	)
+
+0:	mov		x0, x25
+	load_round_keys	w26, x0
+	ld1		{SHASH.2d}, [x23]
+	ld1		{XL.2d}, [x20]
 
 	movi		MASK.16b, #0xe1
 	ext		SHASH2.16b, SHASH.16b, SHASH.16b, #8
-CPU_LE(	rev		x8, x8		)
 	shl		MASK.2d, MASK.2d, #57
 	eor		SHASH2.16b, SHASH2.16b, SHASH.16b
 
 	.if		\enc == 1
-	ld1		{KS.16b}, [x7]
+	ld1		{KS.16b}, [x27]
 	.endif
 
-0:	ld1		{CTR.8b}, [x5]			// load upper counter
-	ld1		{INP.16b}, [x3], #16
-	rev		x9, x8
-	add		x8, x8, #1
-	sub		w0, w0, #1
+1:	ld1		{CTR.8b}, [x24]			// load upper counter
+	ld1		{INP.16b}, [x22], #16
+	rev		x9, x28
+	add		x28, x28, #1
+	sub		w19, w19, #1
 	ins		CTR.d[1], x9			// set lower counter
 
 	.if		\enc == 1
 	eor		INP.16b, INP.16b, KS.16b	// encrypt input
-	st1		{INP.16b}, [x2], #16
+	st1		{INP.16b}, [x21], #16
 	.endif
 
 	rev64		T1.16b, INP.16b
 
-	cmp		w6, #12
-	b.ge		2f				// AES-192/256?
+	cmp		w26, #12
+	b.ge		4f				// AES-192/256?
 
-1:	enc_round	CTR, v21
+2:	enc_round	CTR, v21
 
 	ext		T2.16b, XL.16b, XL.16b, #8
 	ext		IN1.16b, T1.16b, T1.16b, #8
@@ -390,27 +425,39 @@ CPU_LE(	rev		x8, x8		)
 
 	.if		\enc == 0
 	eor		INP.16b, INP.16b, KS.16b
-	st1		{INP.16b}, [x2], #16
+	st1		{INP.16b}, [x21], #16
 	.endif
 
-	cbnz		w0, 0b
+	cbz		w19, 3f
 
-CPU_LE(	rev		x8, x8		)
-	st1		{XL.2d}, [x1]
-	str		x8, [x5, #8]			// store lower counter
+	if_will_cond_yield_neon
+	st1		{XL.2d}, [x20]
+	.if		\enc == 1
+	st1		{KS.16b}, [x27]
+	.endif
+	do_cond_yield_neon
+	b		0b
+	endif_yield_neon
 
+	b		1b
+
+3:	st1		{XL.2d}, [x20]
 	.if		\enc == 1
-	st1		{KS.16b}, [x7]
+	st1		{KS.16b}, [x27]
 	.endif
 
+CPU_LE(	rev		x28, x28	)
+	str		x28, [x24, #8]			// store lower counter
+
+	frame_pop
 	ret
 
-2:	b.eq		3f				// AES-192?
+4:	b.eq		5f				// AES-192?
 	enc_round	CTR, v17
 	enc_round	CTR, v18
-3:	enc_round	CTR, v19
+5:	enc_round	CTR, v19
 	enc_round	CTR, v20
-	b		1b
+	b		2b
 	.endm
 
 	/*
diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c
index cfc9c92814fd..7cf0b1aa6ea8 100644
--- a/arch/arm64/crypto/ghash-ce-glue.c
+++ b/arch/arm64/crypto/ghash-ce-glue.c
@@ -63,11 +63,12 @@ static void (*pmull_ghash_update)(int blocks, u64 dg[], const char *src,
 
 asmlinkage void pmull_gcm_encrypt(int blocks, u64 dg[], u8 dst[],
 				  const u8 src[], struct ghash_key const *k,
-				  u8 ctr[], int rounds, u8 ks[]);
+				  u8 ctr[], u32 const rk[], int rounds,
+				  u8 ks[]);
 
 asmlinkage void pmull_gcm_decrypt(int blocks, u64 dg[], u8 dst[],
 				  const u8 src[], struct ghash_key const *k,
-				  u8 ctr[], int rounds);
+				  u8 ctr[], u32 const rk[], int rounds);
 
 asmlinkage void pmull_gcm_encrypt_block(u8 dst[], u8 const src[],
 					u32 const rk[], int rounds);
@@ -368,26 +369,29 @@ static int gcm_encrypt(struct aead_request *req)
 		pmull_gcm_encrypt_block(ks, iv, NULL,
 					num_rounds(&ctx->aes_key));
 		put_unaligned_be32(3, iv + GCM_IV_SIZE);
+		kernel_neon_end();
 
-		err = skcipher_walk_aead_encrypt(&walk, req, true);
+		err = skcipher_walk_aead_encrypt(&walk, req, false);
 
 		while (walk.nbytes >= AES_BLOCK_SIZE) {
 			int blocks = walk.nbytes / AES_BLOCK_SIZE;
 
+			kernel_neon_begin();
 			pmull_gcm_encrypt(blocks, dg, walk.dst.virt.addr,
 					  walk.src.virt.addr, &ctx->ghash_key,
-					  iv, num_rounds(&ctx->aes_key), ks);
+					  iv, ctx->aes_key.key_enc,
+					  num_rounds(&ctx->aes_key), ks);
+			kernel_neon_end();
 
 			err = skcipher_walk_done(&walk,
 						 walk.nbytes % AES_BLOCK_SIZE);
 		}
-		kernel_neon_end();
 	} else {
 		__aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv,
 				    num_rounds(&ctx->aes_key));
 		put_unaligned_be32(2, iv + GCM_IV_SIZE);
 
-		err = skcipher_walk_aead_encrypt(&walk, req, true);
+		err = skcipher_walk_aead_encrypt(&walk, req, false);
 
 		while (walk.nbytes >= AES_BLOCK_SIZE) {
 			int blocks = walk.nbytes / AES_BLOCK_SIZE;
@@ -467,15 +471,19 @@ static int gcm_decrypt(struct aead_request *req)
 		pmull_gcm_encrypt_block(tag, iv, ctx->aes_key.key_enc,
 					num_rounds(&ctx->aes_key));
 		put_unaligned_be32(2, iv + GCM_IV_SIZE);
+		kernel_neon_end();
 
-		err = skcipher_walk_aead_decrypt(&walk, req, true);
+		err = skcipher_walk_aead_decrypt(&walk, req, false);
 
 		while (walk.nbytes >= AES_BLOCK_SIZE) {
 			int blocks = walk.nbytes / AES_BLOCK_SIZE;
 
+			kernel_neon_begin();
 			pmull_gcm_decrypt(blocks, dg, walk.dst.virt.addr,
 					  walk.src.virt.addr, &ctx->ghash_key,
-					  iv, num_rounds(&ctx->aes_key));
+					  iv, ctx->aes_key.key_enc,
+					  num_rounds(&ctx->aes_key));
+			kernel_neon_end();
 
 			err = skcipher_walk_done(&walk,
 						 walk.nbytes % AES_BLOCK_SIZE);
@@ -483,14 +491,12 @@ static int gcm_decrypt(struct aead_request *req)
 		if (walk.nbytes)
 			pmull_gcm_encrypt_block(iv, iv, NULL,
 						num_rounds(&ctx->aes_key));
-
-		kernel_neon_end();
 	} else {
 		__aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv,
 				    num_rounds(&ctx->aes_key));
 		put_unaligned_be32(2, iv + GCM_IV_SIZE);
 
-		err = skcipher_walk_aead_decrypt(&walk, req, true);
+		err = skcipher_walk_aead_decrypt(&walk, req, false);
 
 		while (walk.nbytes >= AES_BLOCK_SIZE) {
 			int blocks = walk.nbytes / AES_BLOCK_SIZE;
diff --git a/arch/arm64/crypto/sha1-ce-core.S b/arch/arm64/crypto/sha1-ce-core.S
index 46049850727d..78eb35fb5056 100644
--- a/arch/arm64/crypto/sha1-ce-core.S
+++ b/arch/arm64/crypto/sha1-ce-core.S
@@ -69,30 +69,36 @@
 	 *			  int blocks)
 	 */
 ENTRY(sha1_ce_transform)
+	frame_push	3
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+
 	/* load round constants */
-	loadrc		k0.4s, 0x5a827999, w6
+0:	loadrc		k0.4s, 0x5a827999, w6
 	loadrc		k1.4s, 0x6ed9eba1, w6
 	loadrc		k2.4s, 0x8f1bbcdc, w6
 	loadrc		k3.4s, 0xca62c1d6, w6
 
 	/* load state */
-	ld1		{dgav.4s}, [x0]
-	ldr		dgb, [x0, #16]
+	ld1		{dgav.4s}, [x19]
+	ldr		dgb, [x19, #16]
 
 	/* load sha1_ce_state::finalize */
 	ldr_l		w4, sha1_ce_offsetof_finalize, x4
-	ldr		w4, [x0, x4]
+	ldr		w4, [x19, x4]
 
 	/* load input */
-0:	ld1		{v8.4s-v11.4s}, [x1], #64
-	sub		w2, w2, #1
+1:	ld1		{v8.4s-v11.4s}, [x20], #64
+	sub		w21, w21, #1
 
 CPU_LE(	rev32		v8.16b, v8.16b		)
 CPU_LE(	rev32		v9.16b, v9.16b		)
 CPU_LE(	rev32		v10.16b, v10.16b	)
 CPU_LE(	rev32		v11.16b, v11.16b	)
 
-1:	add		t0.4s, v8.4s, k0.4s
+2:	add		t0.4s, v8.4s, k0.4s
 	mov		dg0v.16b, dgav.16b
 
 	add_update	c, ev, k0,  8,  9, 10, 11, dgb
@@ -123,16 +129,25 @@ CPU_LE(	rev32		v11.16b, v11.16b	)
 	add		dgbv.2s, dgbv.2s, dg1v.2s
 	add		dgav.4s, dgav.4s, dg0v.4s
 
-	cbnz		w2, 0b
+	cbz		w21, 3f
+
+	if_will_cond_yield_neon
+	st1		{dgav.4s}, [x19]
+	str		dgb, [x19, #16]
+	do_cond_yield_neon
+	b		0b
+	endif_yield_neon
+
+	b		1b
 
 	/*
 	 * Final block: add padding and total bit count.
 	 * Skip if the input size was not a round multiple of the block size,
 	 * the padding is handled by the C code in that case.
 	 */
-	cbz		x4, 3f
+3:	cbz		x4, 4f
 	ldr_l		w4, sha1_ce_offsetof_count, x4
-	ldr		x4, [x0, x4]
+	ldr		x4, [x19, x4]
 	movi		v9.2d, #0
 	mov		x8, #0x80000000
 	movi		v10.2d, #0
@@ -141,10 +156,11 @@ CPU_LE(	rev32		v11.16b, v11.16b	)
 	mov		x4, #0
 	mov		v11.d[0], xzr
 	mov		v11.d[1], x7
-	b		1b
+	b		2b
 
 	/* store new state */
-3:	st1		{dgav.4s}, [x0]
-	str		dgb, [x0, #16]
+4:	st1		{dgav.4s}, [x19]
+	str		dgb, [x19, #16]
+	frame_pop
 	ret
 ENDPROC(sha1_ce_transform)
diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/crypto/sha2-ce-core.S
index 4c3c89b812ce..cd8b36412469 100644
--- a/arch/arm64/crypto/sha2-ce-core.S
+++ b/arch/arm64/crypto/sha2-ce-core.S
@@ -79,30 +79,36 @@
 	 */
 	.text
 ENTRY(sha2_ce_transform)
+	frame_push	3
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+
 	/* load round constants */
-	adr_l		x8, .Lsha2_rcon
+0:	adr_l		x8, .Lsha2_rcon
 	ld1		{ v0.4s- v3.4s}, [x8], #64
 	ld1		{ v4.4s- v7.4s}, [x8], #64
 	ld1		{ v8.4s-v11.4s}, [x8], #64
 	ld1		{v12.4s-v15.4s}, [x8]
 
 	/* load state */
-	ld1		{dgav.4s, dgbv.4s}, [x0]
+	ld1		{dgav.4s, dgbv.4s}, [x19]
 
 	/* load sha256_ce_state::finalize */
 	ldr_l		w4, sha256_ce_offsetof_finalize, x4
-	ldr		w4, [x0, x4]
+	ldr		w4, [x19, x4]
 
 	/* load input */
-0:	ld1		{v16.4s-v19.4s}, [x1], #64
-	sub		w2, w2, #1
+1:	ld1		{v16.4s-v19.4s}, [x20], #64
+	sub		w21, w21, #1
 
 CPU_LE(	rev32		v16.16b, v16.16b	)
 CPU_LE(	rev32		v17.16b, v17.16b	)
 CPU_LE(	rev32		v18.16b, v18.16b	)
 CPU_LE(	rev32		v19.16b, v19.16b	)
 
-1:	add		t0.4s, v16.4s, v0.4s
+2:	add		t0.4s, v16.4s, v0.4s
 	mov		dg0v.16b, dgav.16b
 	mov		dg1v.16b, dgbv.16b
 
@@ -131,16 +137,24 @@ CPU_LE(	rev32		v19.16b, v19.16b	)
 	add		dgbv.4s, dgbv.4s, dg1v.4s
 
 	/* handled all input blocks? */
-	cbnz		w2, 0b
+	cbz		w21, 3f
+
+	if_will_cond_yield_neon
+	st1		{dgav.4s, dgbv.4s}, [x19]
+	do_cond_yield_neon
+	b		0b
+	endif_yield_neon
+
+	b		1b
 
 	/*
 	 * Final block: add padding and total bit count.
 	 * Skip if the input size was not a round multiple of the block size,
 	 * the padding is handled by the C code in that case.
 	 */
-	cbz		x4, 3f
+3:	cbz		x4, 4f
 	ldr_l		w4, sha256_ce_offsetof_count, x4
-	ldr		x4, [x0, x4]
+	ldr		x4, [x19, x4]
 	movi		v17.2d, #0
 	mov		x8, #0x80000000
 	movi		v18.2d, #0
@@ -149,9 +163,10 @@ CPU_LE(	rev32		v19.16b, v19.16b	)
 	mov		x4, #0
 	mov		v19.d[0], xzr
 	mov		v19.d[1], x7
-	b		1b
+	b		2b
 
 	/* store new state */
-3:	st1		{dgav.4s, dgbv.4s}, [x0]
+4:	st1		{dgav.4s, dgbv.4s}, [x19]
+	frame_pop
 	ret
 ENDPROC(sha2_ce_transform)
diff --git a/arch/arm64/crypto/sha256-core.S_shipped b/arch/arm64/crypto/sha256-core.S_shipped
index 3ce82cc860bc..7c7ce2e3bad6 100644
--- a/arch/arm64/crypto/sha256-core.S_shipped
+++ b/arch/arm64/crypto/sha256-core.S_shipped
@@ -1,3 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// This code is taken from the OpenSSL project but the author (Andy Polyakov)
+// has relicensed it under the GPLv2. Therefore this program is free software;
+// you can redistribute it and/or modify it under the terms of the GNU General
+// Public License version 2 as published by the Free Software Foundation.
+//
+// The original headers, including the original license headers, are
+// included below for completeness.
+
 // Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
 //
 // Licensed under the OpenSSL license (the "License").  You may not use
@@ -10,8 +20,6 @@
 // project. The module is, however, dual licensed under OpenSSL and
 // CRYPTOGAMS licenses depending on where you obtain it. For further
 // details see http://www.openssl.org/~appro/cryptogams/.
-//
-// Permission to use under GPLv2 terms is granted.
 // ====================================================================
 //
 // SHA256/512 for ARMv8.
diff --git a/arch/arm64/crypto/sha3-ce-core.S b/arch/arm64/crypto/sha3-ce-core.S
index 332ad7530690..a7d587fa54f6 100644
--- a/arch/arm64/crypto/sha3-ce-core.S
+++ b/arch/arm64/crypto/sha3-ce-core.S
@@ -41,9 +41,16 @@
 	 */
 	.text
 ENTRY(sha3_ce_transform)
-	/* load state */
-	add	x8, x0, #32
-	ld1	{ v0.1d- v3.1d}, [x0]
+	frame_push	4
+
+	mov	x19, x0
+	mov	x20, x1
+	mov	x21, x2
+	mov	x22, x3
+
+0:	/* load state */
+	add	x8, x19, #32
+	ld1	{ v0.1d- v3.1d}, [x19]
 	ld1	{ v4.1d- v7.1d}, [x8], #32
 	ld1	{ v8.1d-v11.1d}, [x8], #32
 	ld1	{v12.1d-v15.1d}, [x8], #32
@@ -51,13 +58,13 @@ ENTRY(sha3_ce_transform)
 	ld1	{v20.1d-v23.1d}, [x8], #32
 	ld1	{v24.1d}, [x8]
 
-0:	sub	w2, w2, #1
+1:	sub	w21, w21, #1
 	mov	w8, #24
 	adr_l	x9, .Lsha3_rcon
 
 	/* load input */
-	ld1	{v25.8b-v28.8b}, [x1], #32
-	ld1	{v29.8b-v31.8b}, [x1], #24
+	ld1	{v25.8b-v28.8b}, [x20], #32
+	ld1	{v29.8b-v31.8b}, [x20], #24
 	eor	v0.8b, v0.8b, v25.8b
 	eor	v1.8b, v1.8b, v26.8b
 	eor	v2.8b, v2.8b, v27.8b
@@ -66,10 +73,10 @@ ENTRY(sha3_ce_transform)
 	eor	v5.8b, v5.8b, v30.8b
 	eor	v6.8b, v6.8b, v31.8b
 
-	tbnz	x3, #6, 2f		// SHA3-512
+	tbnz	x22, #6, 3f		// SHA3-512
 
-	ld1	{v25.8b-v28.8b}, [x1], #32
-	ld1	{v29.8b-v30.8b}, [x1], #16
+	ld1	{v25.8b-v28.8b}, [x20], #32
+	ld1	{v29.8b-v30.8b}, [x20], #16
 	eor	 v7.8b,  v7.8b, v25.8b
 	eor	 v8.8b,  v8.8b, v26.8b
 	eor	 v9.8b,  v9.8b, v27.8b
@@ -77,34 +84,34 @@ ENTRY(sha3_ce_transform)
 	eor	v11.8b, v11.8b, v29.8b
 	eor	v12.8b, v12.8b, v30.8b
 
-	tbnz	x3, #4, 1f		// SHA3-384 or SHA3-224
+	tbnz	x22, #4, 2f		// SHA3-384 or SHA3-224
 
 	// SHA3-256
-	ld1	{v25.8b-v28.8b}, [x1], #32
+	ld1	{v25.8b-v28.8b}, [x20], #32
 	eor	v13.8b, v13.8b, v25.8b
 	eor	v14.8b, v14.8b, v26.8b
 	eor	v15.8b, v15.8b, v27.8b
 	eor	v16.8b, v16.8b, v28.8b
-	b	3f
+	b	4f
 
-1:	tbz	x3, #2, 3f		// bit 2 cleared? SHA-384
+2:	tbz	x22, #2, 4f		// bit 2 cleared? SHA-384
 
 	// SHA3-224
-	ld1	{v25.8b-v28.8b}, [x1], #32
-	ld1	{v29.8b}, [x1], #8
+	ld1	{v25.8b-v28.8b}, [x20], #32
+	ld1	{v29.8b}, [x20], #8
 	eor	v13.8b, v13.8b, v25.8b
 	eor	v14.8b, v14.8b, v26.8b
 	eor	v15.8b, v15.8b, v27.8b
 	eor	v16.8b, v16.8b, v28.8b
 	eor	v17.8b, v17.8b, v29.8b
-	b	3f
+	b	4f
 
 	// SHA3-512
-2:	ld1	{v25.8b-v26.8b}, [x1], #16
+3:	ld1	{v25.8b-v26.8b}, [x20], #16
 	eor	 v7.8b,  v7.8b, v25.8b
 	eor	 v8.8b,  v8.8b, v26.8b
 
-3:	sub	w8, w8, #1
+4:	sub	w8, w8, #1
 
 	eor3	v29.16b,  v4.16b,  v9.16b, v14.16b
 	eor3	v26.16b,  v1.16b,  v6.16b, v11.16b
@@ -183,17 +190,33 @@ ENTRY(sha3_ce_transform)
 
 	eor	 v0.16b,  v0.16b, v31.16b
 
-	cbnz	w8, 3b
-	cbnz	w2, 0b
+	cbnz	w8, 4b
+	cbz	w21, 5f
+
+	if_will_cond_yield_neon
+	add	x8, x19, #32
+	st1	{ v0.1d- v3.1d}, [x19]
+	st1	{ v4.1d- v7.1d}, [x8], #32
+	st1	{ v8.1d-v11.1d}, [x8], #32
+	st1	{v12.1d-v15.1d}, [x8], #32
+	st1	{v16.1d-v19.1d}, [x8], #32
+	st1	{v20.1d-v23.1d}, [x8], #32
+	st1	{v24.1d}, [x8]
+	do_cond_yield_neon
+	b		0b
+	endif_yield_neon
+
+	b	1b
 
 	/* save state */
-	st1	{ v0.1d- v3.1d}, [x0], #32
-	st1	{ v4.1d- v7.1d}, [x0], #32
-	st1	{ v8.1d-v11.1d}, [x0], #32
-	st1	{v12.1d-v15.1d}, [x0], #32
-	st1	{v16.1d-v19.1d}, [x0], #32
-	st1	{v20.1d-v23.1d}, [x0], #32
-	st1	{v24.1d}, [x0]
+5:	st1	{ v0.1d- v3.1d}, [x19], #32
+	st1	{ v4.1d- v7.1d}, [x19], #32
+	st1	{ v8.1d-v11.1d}, [x19], #32
+	st1	{v12.1d-v15.1d}, [x19], #32
+	st1	{v16.1d-v19.1d}, [x19], #32
+	st1	{v20.1d-v23.1d}, [x19], #32
+	st1	{v24.1d}, [x19]
+	frame_pop
 	ret
 ENDPROC(sha3_ce_transform)
 
diff --git a/arch/arm64/crypto/sha512-armv8.pl b/arch/arm64/crypto/sha512-armv8.pl
index c55efb308544..2d8655d5b1af 100644
--- a/arch/arm64/crypto/sha512-armv8.pl
+++ b/arch/arm64/crypto/sha512-armv8.pl
@@ -1,4 +1,14 @@
 #! /usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0
+
+# This code is taken from the OpenSSL project but the author (Andy Polyakov)
+# has relicensed it under the GPLv2. Therefore this program is free software;
+# you can redistribute it and/or modify it under the terms of the GNU General
+# Public License version 2 as published by the Free Software Foundation.
+#
+# The original headers, including the original license headers, are
+# included below for completeness.
+
 # Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
 #
 # Licensed under the OpenSSL license (the "License").  You may not use
@@ -11,8 +21,6 @@
 # project. The module is, however, dual licensed under OpenSSL and
 # CRYPTOGAMS licenses depending on where you obtain it. For further
 # details see http://www.openssl.org/~appro/cryptogams/.
-#
-# Permission to use under GPLv2 terms is granted.
 # ====================================================================
 #
 # SHA256/512 for ARMv8.
diff --git a/arch/arm64/crypto/sha512-ce-core.S b/arch/arm64/crypto/sha512-ce-core.S
index 7f3bca5c59a2..ce65e3abe4f2 100644
--- a/arch/arm64/crypto/sha512-ce-core.S
+++ b/arch/arm64/crypto/sha512-ce-core.S
@@ -107,17 +107,23 @@
 	 */
 	.text
 ENTRY(sha512_ce_transform)
+	frame_push	3
+
+	mov		x19, x0
+	mov		x20, x1
+	mov		x21, x2
+
 	/* load state */
-	ld1		{v8.2d-v11.2d}, [x0]
+0:	ld1		{v8.2d-v11.2d}, [x19]
 
 	/* load first 4 round constants */
 	adr_l		x3, .Lsha512_rcon
 	ld1		{v20.2d-v23.2d}, [x3], #64
 
 	/* load input */
-0:	ld1		{v12.2d-v15.2d}, [x1], #64
-	ld1		{v16.2d-v19.2d}, [x1], #64
-	sub		w2, w2, #1
+1:	ld1		{v12.2d-v15.2d}, [x20], #64
+	ld1		{v16.2d-v19.2d}, [x20], #64
+	sub		w21, w21, #1
 
 CPU_LE(	rev64		v12.16b, v12.16b	)
 CPU_LE(	rev64		v13.16b, v13.16b	)
@@ -196,9 +202,18 @@ CPU_LE(	rev64		v19.16b, v19.16b	)
 	add		v11.2d, v11.2d, v3.2d
 
 	/* handled all input blocks? */
-	cbnz		w2, 0b
+	cbz		w21, 3f
+
+	if_will_cond_yield_neon
+	st1		{v8.2d-v11.2d}, [x19]
+	do_cond_yield_neon
+	b		0b
+	endif_yield_neon
+
+	b		1b
 
 	/* store new state */
-3:	st1		{v8.2d-v11.2d}, [x0]
+3:	st1		{v8.2d-v11.2d}, [x19]
+	frame_pop
 	ret
 ENDPROC(sha512_ce_transform)
diff --git a/arch/arm64/crypto/sha512-core.S_shipped b/arch/arm64/crypto/sha512-core.S_shipped
index bd0f59f06c9d..e063a6106720 100644
--- a/arch/arm64/crypto/sha512-core.S_shipped
+++ b/arch/arm64/crypto/sha512-core.S_shipped
@@ -1,3 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// This code is taken from the OpenSSL project but the author (Andy Polyakov)
+// has relicensed it under the GPLv2. Therefore this program is free software;
+// you can redistribute it and/or modify it under the terms of the GNU General
+// Public License version 2 as published by the Free Software Foundation.
+//
+// The original headers, including the original license headers, are
+// included below for completeness.
+
 // Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
 //
 // Licensed under the OpenSSL license (the "License").  You may not use
@@ -10,8 +20,6 @@
 // project. The module is, however, dual licensed under OpenSSL and
 // CRYPTOGAMS licenses depending on where you obtain it. For further
 // details see http://www.openssl.org/~appro/cryptogams/.
-//
-// Permission to use under GPLv2 terms is granted.
 // ====================================================================
 //
 // SHA256/512 for ARMv8.
diff --git a/arch/arm64/crypto/sm4-ce-core.S b/arch/arm64/crypto/sm4-ce-core.S
new file mode 100644
index 000000000000..af3bfbc3f4d4
--- /dev/null
+++ b/arch/arm64/crypto/sm4-ce-core.S
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.irp		b, 0, 1, 2, 3, 4, 5, 6, 7, 8
+	.set		.Lv\b\().4s, \b
+	.endr
+
+	.macro		sm4e, rd, rn
+	.inst		0xcec08400 | .L\rd | (.L\rn << 5)
+	.endm
+
+	/*
+	 * void sm4_ce_do_crypt(const u32 *rk, u32 *out, const u32 *in);
+	 */
+	.text
+ENTRY(sm4_ce_do_crypt)
+	ld1		{v8.4s}, [x2]
+	ld1		{v0.4s-v3.4s}, [x0], #64
+CPU_LE(	rev32		v8.16b, v8.16b		)
+	ld1		{v4.4s-v7.4s}, [x0]
+	sm4e		v8.4s, v0.4s
+	sm4e		v8.4s, v1.4s
+	sm4e		v8.4s, v2.4s
+	sm4e		v8.4s, v3.4s
+	sm4e		v8.4s, v4.4s
+	sm4e		v8.4s, v5.4s
+	sm4e		v8.4s, v6.4s
+	sm4e		v8.4s, v7.4s
+	rev64		v8.4s, v8.4s
+	ext		v8.16b, v8.16b, v8.16b, #8
+CPU_LE(	rev32		v8.16b, v8.16b		)
+	st1		{v8.4s}, [x1]
+	ret
+ENDPROC(sm4_ce_do_crypt)
diff --git a/arch/arm64/crypto/sm4-ce-glue.c b/arch/arm64/crypto/sm4-ce-glue.c
new file mode 100644
index 000000000000..b7fb5274b250
--- /dev/null
+++ b/arch/arm64/crypto/sm4-ce-glue.c
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <asm/neon.h>
+#include <asm/simd.h>
+#include <crypto/sm4.h>
+#include <linux/module.h>
+#include <linux/cpufeature.h>
+#include <linux/crypto.h>
+#include <linux/types.h>
+
+MODULE_ALIAS_CRYPTO("sm4");
+MODULE_ALIAS_CRYPTO("sm4-ce");
+MODULE_DESCRIPTION("SM4 symmetric cipher using ARMv8 Crypto Extensions");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+
+asmlinkage void sm4_ce_do_crypt(const u32 *rk, void *out, const void *in);
+
+static void sm4_ce_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	const struct crypto_sm4_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	if (!may_use_simd()) {
+		crypto_sm4_encrypt(tfm, out, in);
+	} else {
+		kernel_neon_begin();
+		sm4_ce_do_crypt(ctx->rkey_enc, out, in);
+		kernel_neon_end();
+	}
+}
+
+static void sm4_ce_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	const struct crypto_sm4_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	if (!may_use_simd()) {
+		crypto_sm4_decrypt(tfm, out, in);
+	} else {
+		kernel_neon_begin();
+		sm4_ce_do_crypt(ctx->rkey_dec, out, in);
+		kernel_neon_end();
+	}
+}
+
+static struct crypto_alg sm4_ce_alg = {
+	.cra_name			= "sm4",
+	.cra_driver_name		= "sm4-ce",
+	.cra_priority			= 200,
+	.cra_flags			= CRYPTO_ALG_TYPE_CIPHER,
+	.cra_blocksize			= SM4_BLOCK_SIZE,
+	.cra_ctxsize			= sizeof(struct crypto_sm4_ctx),
+	.cra_module			= THIS_MODULE,
+	.cra_u.cipher = {
+		.cia_min_keysize	= SM4_KEY_SIZE,
+		.cia_max_keysize	= SM4_KEY_SIZE,
+		.cia_setkey		= crypto_sm4_set_key,
+		.cia_encrypt		= sm4_ce_encrypt,
+		.cia_decrypt		= sm4_ce_decrypt
+	}
+};
+
+static int __init sm4_ce_mod_init(void)
+{
+	return crypto_register_alg(&sm4_ce_alg);
+}
+
+static void __exit sm4_ce_mod_fini(void)
+{
+	crypto_unregister_alg(&sm4_ce_alg);
+}
+
+module_cpu_feature_match(SM3, sm4_ce_mod_init);
+module_exit(sm4_ce_mod_fini);
diff --git a/arch/arm64/include/asm/acpi.h b/arch/arm64/include/asm/acpi.h
index 32f465a80e4e..0db62a4cbce2 100644
--- a/arch/arm64/include/asm/acpi.h
+++ b/arch/arm64/include/asm/acpi.h
@@ -86,6 +86,10 @@ static inline bool acpi_has_cpu_in_madt(void)
 }
 
 struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu);
+static inline u32 get_acpi_id_for_cpu(unsigned int cpu)
+{
+	return	acpi_cpu_get_madt_gicc(cpu)->uid;
+}
 
 static inline void arch_fix_phys_package_id(int num, u32 slot) { }
 void __init acpi_init_cpus(void);
diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h
index 9bbffc7a301f..5df5cfe1c143 100644
--- a/arch/arm64/include/asm/cache.h
+++ b/arch/arm64/include/asm/cache.h
@@ -33,7 +33,7 @@
 #define ICACHE_POLICY_VIPT	2
 #define ICACHE_POLICY_PIPT	3
 
-#define L1_CACHE_SHIFT		7
+#define L1_CACHE_SHIFT		(6)
 #define L1_CACHE_BYTES		(1 << L1_CACHE_SHIFT)
 
 /*
@@ -43,7 +43,7 @@
  * cache before the transfer is done, causing old data to be seen by
  * the CPU.
  */
-#define ARCH_DMA_MINALIGN	L1_CACHE_BYTES
+#define ARCH_DMA_MINALIGN	(128)
 
 #ifndef __ASSEMBLY__
 
@@ -77,7 +77,7 @@ static inline u32 cache_type_cwg(void)
 static inline int cache_line_size(void)
 {
 	u32 cwg = cache_type_cwg();
-	return cwg ? 4 << cwg : L1_CACHE_BYTES;
+	return cwg ? 4 << cwg : ARCH_DMA_MINALIGN;
 }
 
 #endif	/* __ASSEMBLY__ */
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
index 4f5fd2a36e6e..3b0938281541 100644
--- a/arch/arm64/include/asm/cmpxchg.h
+++ b/arch/arm64/include/asm/cmpxchg.h
@@ -204,7 +204,9 @@ static inline void __cmpwait_case_##name(volatile void *ptr,		\
 	unsigned long tmp;						\
 									\
 	asm volatile(							\
-	"	ldxr" #sz "\t%" #w "[tmp], %[v]\n"		\
+	"	sevl\n"							\
+	"	wfe\n"							\
+	"	ldxr" #sz "\t%" #w "[tmp], %[v]\n"			\
 	"	eor	%" #w "[tmp], %" #w "[tmp], %" #w "[val]\n"	\
 	"	cbnz	%" #w "[tmp], 1f\n"				\
 	"	wfe\n"							\
diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h
index c00c62e1a4a3..1a037b94eba1 100644
--- a/arch/arm64/include/asm/compat.h
+++ b/arch/arm64/include/asm/compat.h
@@ -34,7 +34,6 @@
 
 typedef u32		compat_size_t;
 typedef s32		compat_ssize_t;
-typedef s32		compat_time_t;
 typedef s32		compat_clock_t;
 typedef s32		compat_pid_t;
 typedef u16		__compat_uid_t;
@@ -66,16 +65,6 @@ typedef u32		compat_ulong_t;
 typedef u64		compat_u64;
 typedef u32		compat_uptr_t;
 
-struct compat_timespec {
-	compat_time_t	tv_sec;
-	s32		tv_nsec;
-};
-
-struct compat_timeval {
-	compat_time_t	tv_sec;
-	s32		tv_usec;
-};
-
 struct compat_stat {
 #ifdef __AARCH64EB__
 	short		st_dev;
@@ -192,10 +181,10 @@ struct compat_ipc64_perm {
 
 struct compat_semid64_ds {
 	struct compat_ipc64_perm sem_perm;
-	compat_time_t  sem_otime;
-	compat_ulong_t __unused1;
-	compat_time_t  sem_ctime;
-	compat_ulong_t __unused2;
+	compat_ulong_t sem_otime;
+	compat_ulong_t sem_otime_high;
+	compat_ulong_t sem_ctime;
+	compat_ulong_t sem_ctime_high;
 	compat_ulong_t sem_nsems;
 	compat_ulong_t __unused3;
 	compat_ulong_t __unused4;
@@ -203,12 +192,12 @@ struct compat_semid64_ds {
 
 struct compat_msqid64_ds {
 	struct compat_ipc64_perm msg_perm;
-	compat_time_t  msg_stime;
-	compat_ulong_t __unused1;
-	compat_time_t  msg_rtime;
-	compat_ulong_t __unused2;
-	compat_time_t  msg_ctime;
-	compat_ulong_t __unused3;
+	compat_ulong_t msg_stime;
+	compat_ulong_t msg_stime_high;
+	compat_ulong_t msg_rtime;
+	compat_ulong_t msg_rtime_high;
+	compat_ulong_t msg_ctime;
+	compat_ulong_t msg_ctime_high;
 	compat_ulong_t msg_cbytes;
 	compat_ulong_t msg_qnum;
 	compat_ulong_t msg_qbytes;
@@ -221,12 +210,12 @@ struct compat_msqid64_ds {
 struct compat_shmid64_ds {
 	struct compat_ipc64_perm shm_perm;
 	compat_size_t  shm_segsz;
-	compat_time_t  shm_atime;
-	compat_ulong_t __unused1;
-	compat_time_t  shm_dtime;
-	compat_ulong_t __unused2;
-	compat_time_t  shm_ctime;
-	compat_ulong_t __unused3;
+	compat_ulong_t shm_atime;
+	compat_ulong_t shm_atime_high;
+	compat_ulong_t shm_dtime;
+	compat_ulong_t shm_dtime_high;
+	compat_ulong_t shm_ctime;
+	compat_ulong_t shm_ctime_high;
 	compat_pid_t   shm_cpid;
 	compat_pid_t   shm_lpid;
 	compat_ulong_t shm_nattch;
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 09b0f2a80c8f..0a6b7133195e 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -11,9 +11,7 @@
 
 #include <asm/cpucaps.h>
 #include <asm/cputype.h>
-#include <asm/fpsimd.h>
 #include <asm/hwcap.h>
-#include <asm/sigcontext.h>
 #include <asm/sysreg.h>
 
 /*
@@ -510,33 +508,6 @@ static inline bool system_supports_sve(void)
 		cpus_have_const_cap(ARM64_SVE);
 }
 
-/*
- * Read the pseudo-ZCR used by cpufeatures to identify the supported SVE
- * vector length.
- *
- * Use only if SVE is present.
- * This function clobbers the SVE vector length.
- */
-static inline u64 read_zcr_features(void)
-{
-	u64 zcr;
-	unsigned int vq_max;
-
-	/*
-	 * Set the maximum possible VL, and write zeroes to all other
-	 * bits to see if they stick.
-	 */
-	sve_kernel_enable(NULL);
-	write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);
-
-	zcr = read_sysreg_s(SYS_ZCR_EL1);
-	zcr &= ~(u64)ZCR_ELx_LEN_MASK; /* find sticky 1s outside LEN field */
-	vq_max = sve_vq_from_vl(sve_get_vl());
-	zcr |= vq_max - 1; /* set LEN field to maximum effective value */
-
-	return zcr;
-}
-
 #endif /* __ASSEMBLY__ */
 
 #endif
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index aa7162ae93e3..fa92747a49c8 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -18,6 +18,8 @@
 
 #include <asm/ptrace.h>
 #include <asm/errno.h>
+#include <asm/processor.h>
+#include <asm/sigcontext.h>
 
 #ifndef __ASSEMBLY__
 
@@ -41,6 +43,8 @@ struct task_struct;
 extern void fpsimd_save_state(struct user_fpsimd_state *state);
 extern void fpsimd_load_state(struct user_fpsimd_state *state);
 
+extern void fpsimd_save(void);
+
 extern void fpsimd_thread_switch(struct task_struct *next);
 extern void fpsimd_flush_thread(void);
 
@@ -49,12 +53,27 @@ extern void fpsimd_preserve_current_state(void);
 extern void fpsimd_restore_current_state(void);
 extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
 
+extern void fpsimd_bind_task_to_cpu(void);
+extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state);
+
 extern void fpsimd_flush_task_state(struct task_struct *target);
+extern void fpsimd_flush_cpu_state(void);
 extern void sve_flush_cpu_state(void);
 
 /* Maximum VL that SVE VL-agnostic software can transparently support */
 #define SVE_VL_ARCH_MAX 0x100
 
+/* Offset of FFR in the SVE register dump */
+static inline size_t sve_ffr_offset(int vl)
+{
+	return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
+}
+
+static inline void *sve_pffr(struct thread_struct *thread)
+{
+	return (char *)thread->sve_state + sve_ffr_offset(thread->sve_vl);
+}
+
 extern void sve_save_state(void *state, u32 *pfpsr);
 extern void sve_load_state(void const *state, u32 const *pfpsr,
 			   unsigned long vq_minus_1);
@@ -63,6 +82,8 @@ extern unsigned int sve_get_vl(void);
 struct arm64_cpu_capabilities;
 extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused);
 
+extern u64 read_zcr_features(void);
+
 extern int __ro_after_init sve_max_vl;
 
 #ifdef CONFIG_ARM64_SVE
diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h
index e050d765ca9e..46843515d77b 100644
--- a/arch/arm64/include/asm/fpsimdmacros.h
+++ b/arch/arm64/include/asm/fpsimdmacros.h
@@ -207,12 +207,14 @@
 		str		w\nxtmp, [\xpfpsr, #4]
 .endm
 
-.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp
+.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2
 		mrs_s		x\nxtmp, SYS_ZCR_EL1
-		bic		x\nxtmp, x\nxtmp, ZCR_ELx_LEN_MASK
-		orr		x\nxtmp, x\nxtmp, \xvqminus1
-		msr_s		SYS_ZCR_EL1, x\nxtmp	// self-synchronising
-
+		bic		\xtmp2, x\nxtmp, ZCR_ELx_LEN_MASK
+		orr		\xtmp2, \xtmp2, \xvqminus1
+		cmp		\xtmp2, x\nxtmp
+		b.eq		921f
+		msr_s		SYS_ZCR_EL1, \xtmp2	// self-synchronising
+921:
  _for n, 0, 31,	_sve_ldr_v	\n, \nxbase, \n - 34
 		_sve_ldr_p	0, \nxbase
 		_sve_wrffr	0
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index f6648a3e4152..821a7032c0f7 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -30,19 +30,19 @@
 /* The hyp-stub will return this for any kvm_call_hyp() call */
 #define ARM_EXCEPTION_HYP_GONE	  HVC_STUB_ERR
 
-#define KVM_ARM64_DEBUG_DIRTY_SHIFT	0
-#define KVM_ARM64_DEBUG_DIRTY		(1 << KVM_ARM64_DEBUG_DIRTY_SHIFT)
+#ifndef __ASSEMBLY__
+
+#include <linux/mm.h>
 
 /* Translate a kernel address of @sym into its equivalent linear mapping */
 #define kvm_ksym_ref(sym)						\
 	({								\
 		void *val = &sym;					\
 		if (!is_kernel_in_hyp_mode())				\
-			val = phys_to_virt((u64)&sym - kimage_voffset);	\
+			val = lm_alias(&sym);				\
 		val;							\
 	 })
 
-#ifndef __ASSEMBLY__
 struct kvm;
 struct kvm_vcpu;
 
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 469de8acd06f..a4ca202ff3f2 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -30,6 +30,7 @@
 #include <asm/kvm.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_mmio.h>
+#include <asm/thread_info.h>
 
 #define __KVM_HAVE_ARCH_INTC_INITIALIZED
 
@@ -216,8 +217,8 @@ struct kvm_vcpu_arch {
 	/* Exception Information */
 	struct kvm_vcpu_fault_info fault;
 
-	/* Guest debug state */
-	u64 debug_flags;
+	/* Miscellaneous vcpu state flags */
+	u64 flags;
 
 	/*
 	 * We maintain more than a single set of debug registers to support
@@ -238,6 +239,10 @@ struct kvm_vcpu_arch {
 
 	/* Pointer to host CPU context */
 	kvm_cpu_context_t *host_cpu_context;
+
+	struct thread_info *host_thread_info;	/* hyp VA */
+	struct user_fpsimd_state *host_fpsimd_state;	/* hyp VA */
+
 	struct {
 		/* {Break,watch}point registers */
 		struct kvm_guest_debug_arch regs;
@@ -293,6 +298,12 @@ struct kvm_vcpu_arch {
 	bool sysregs_loaded_on_cpu;
 };
 
+/* vcpu_arch flags field values: */
+#define KVM_ARM64_DEBUG_DIRTY		(1 << 0)
+#define KVM_ARM64_FP_ENABLED		(1 << 1) /* guest FP regs loaded */
+#define KVM_ARM64_FP_HOST		(1 << 2) /* host FP regs loaded */
+#define KVM_ARM64_HOST_SVE_IN_USE	(1 << 3) /* backup for host TIF_SVE */
+
 #define vcpu_gp_regs(v)		(&(v)->arch.ctxt.gp_regs)
 
 /*
@@ -394,6 +405,19 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,
 	kvm_call_hyp(__kvm_set_tpidr_el2, tpidr_el2);
 }
 
+static inline bool kvm_arch_check_sve_has_vhe(void)
+{
+	/*
+	 * The Arm architecture specifies that implementation of SVE
+	 * requires VHE also to be implemented.  The KVM code for arm64
+	 * relies on this when SVE is present:
+	 */
+	if (system_supports_sve())
+		return has_vhe();
+	else
+		return true;
+}
+
 static inline void kvm_arch_hardware_unsetup(void) {}
 static inline void kvm_arch_sync_events(struct kvm *kvm) {}
 static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
@@ -420,15 +444,18 @@ static inline void __cpu_init_stage2(void)
 		  "PARange is %d bits, unsupported configuration!", parange);
 }
 
-/*
- * All host FP/SIMD state is restored on guest exit, so nothing needs
- * doing here except in the SVE case:
-*/
-static inline void kvm_fpsimd_flush_cpu_state(void)
+/* Guest/host FPSIMD coordination helpers */
+int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu);
+void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu);
+void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu);
+void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu);
+
+#ifdef CONFIG_KVM /* Avoid conflicts with core headers if CONFIG_KVM=n */
+static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
 {
-	if (system_supports_sve())
-		sve_flush_cpu_state();
+	return kvm_arch_vcpu_run_map_fp(vcpu);
 }
+#endif
 
 static inline void kvm_arm_vhe_guest_enter(void)
 {
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 6128992c2ded..513d8636e3ef 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -72,7 +72,6 @@
 #ifdef __ASSEMBLY__
 
 #include <asm/alternative.h>
-#include <asm/cpufeature.h>
 
 /*
  * Convert a kernel VA into a HYP VA.
diff --git a/arch/arm64/include/asm/pci.h b/arch/arm64/include/asm/pci.h
index 8747f7c5e0e7..9e690686e8aa 100644
--- a/arch/arm64/include/asm/pci.h
+++ b/arch/arm64/include/asm/pci.h
@@ -18,11 +18,6 @@
 #define pcibios_assign_all_busses() \
 	(pci_has_flag(PCI_REASSIGN_ALL_BUS))
 
-/*
- * PCI address space differs from physical memory address space
- */
-#define PCI_DMA_BUS_IS_PHYS	(0)
-
 #define ARCH_GENERIC_PCI_MMAP_RESOURCE	1
 
 extern int isa_dma_bridge_buggy;
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 7c4c8f318ba9..9f82d6b53851 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -306,8 +306,6 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b)
 #define HPAGE_MASK		(~(HPAGE_SIZE - 1))
 #define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
 
-#define __HAVE_ARCH_PTE_SPECIAL
-
 static inline pte_t pgd_pte(pgd_t pgd)
 {
 	return __pte(pgd_val(pgd));
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 767598932549..c99e657fdd57 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -156,7 +156,9 @@ static inline void arch_thread_struct_whitelist(unsigned long *offset,
 /* Sync TPIDR_EL0 back to thread_struct for current */
 void tls_preserve_current_state(void);
 
-#define INIT_THREAD  {	}
+#define INIT_THREAD {				\
+	.fpsimd_cpu = NR_CPUS,			\
+}
 
 static inline void start_thread_common(struct pt_regs *regs, unsigned long pc)
 {
@@ -244,6 +246,17 @@ void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused);
 void cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused);
 void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused);
 
+/*
+ * Not at the top of the file due to a direct #include cycle between
+ * <asm/fpsimd.h> and <asm/processor.h>.  Deferring this #include
+ * ensures that contents of processor.h are visible to fpsimd.h even if
+ * processor.h is included first.
+ *
+ * These prctl helpers are the only things in this file that require
+ * fpsimd.h.  The core code expects them to be in this header.
+ */
+#include <asm/fpsimd.h>
+
 /* Userspace interface for PR_SVE_{SET,GET}_VL prctl()s: */
 #define SVE_SET_VL(arg)	sve_set_current_vl(arg)
 #define SVE_GET_VL()	sve_get_current_vl()
diff --git a/arch/arm64/include/asm/spinlock.h b/arch/arm64/include/asm/spinlock.h
index ebdae15d665d..26c5bd7d88d8 100644
--- a/arch/arm64/include/asm/spinlock.h
+++ b/arch/arm64/include/asm/spinlock.h
@@ -122,11 +122,6 @@ static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
 
 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
 {
-	/*
-	 * Ensure prior spin_lock operations to other locks have completed
-	 * on this CPU before we test whether "lock" is locked.
-	 */
-	smp_mb(); /* ^^^ */
 	return !arch_spin_value_unlocked(READ_ONCE(*lock));
 }
 
diff --git a/arch/arm64/include/asm/stat.h b/arch/arm64/include/asm/stat.h
index 15e35598ac40..eab738019707 100644
--- a/arch/arm64/include/asm/stat.h
+++ b/arch/arm64/include/asm/stat.h
@@ -20,6 +20,7 @@
 
 #ifdef CONFIG_COMPAT
 
+#include <linux/compat_time.h>
 #include <asm/compat.h>
 
 /*
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index 740aa03c5f0d..af271f9a6c9f 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -45,12 +45,6 @@ struct thread_info {
 	int			preempt_count;	/* 0 => preemptable, <0 => bug */
 };
 
-#define INIT_THREAD_INFO(tsk)						\
-{									\
-	.preempt_count	= INIT_PREEMPT_COUNT,				\
-	.addr_limit	= KERNEL_DS,					\
-}
-
 #define thread_saved_pc(tsk)	\
 	((unsigned long)(tsk->thread.cpu_context.pc))
 #define thread_saved_sp(tsk)	\
@@ -117,5 +111,12 @@ void arch_release_task_struct(struct task_struct *tsk);
 				 _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
 				 _TIF_NOHZ)
 
+#define INIT_THREAD_INFO(tsk)						\
+{									\
+	.flags		= _TIF_FOREIGN_FPSTATE,				\
+	.preempt_count	= INIT_PREEMPT_COUNT,				\
+	.addr_limit	= KERNEL_DS,					\
+}
+
 #endif /* __KERNEL__ */
 #endif /* __ASM_THREAD_INFO_H */
diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h
index c4f2d50491eb..df48212f767b 100644
--- a/arch/arm64/include/asm/topology.h
+++ b/arch/arm64/include/asm/topology.h
@@ -7,14 +7,16 @@
 struct cpu_topology {
 	int thread_id;
 	int core_id;
-	int cluster_id;
+	int package_id;
+	int llc_id;
 	cpumask_t thread_sibling;
 	cpumask_t core_sibling;
+	cpumask_t llc_siblings;
 };
 
 extern struct cpu_topology cpu_topology[NR_CPUS];
 
-#define topology_physical_package_id(cpu)	(cpu_topology[cpu].cluster_id)
+#define topology_physical_package_id(cpu)	(cpu_topology[cpu].package_id)
 #define topology_core_id(cpu)		(cpu_topology[cpu].core_id)
 #define topology_core_cpumask(cpu)	(&cpu_topology[cpu].core_sibling)
 #define topology_sibling_cpumask(cpu)	(&cpu_topology[cpu].thread_sibling)
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 04b3256f8e6d..4e76630dd655 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -91,6 +91,7 @@ struct kvm_regs {
 #define KVM_VGIC_V3_ADDR_TYPE_DIST	2
 #define KVM_VGIC_V3_ADDR_TYPE_REDIST	3
 #define KVM_VGIC_ITS_ADDR_TYPE		4
+#define KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION	5
 
 #define KVM_VGIC_V3_DIST_SIZE		SZ_64K
 #define KVM_VGIC_V3_REDIST_SIZE		(2 * SZ_64K)
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index 6e47fc3ab549..97d45d5151d4 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -13,6 +13,7 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/sysctl.h>
+#include <linux/uaccess.h>
 
 #include <asm/cpufeature.h>
 #include <asm/insn.h>
@@ -20,8 +21,6 @@
 #include <asm/system_misc.h>
 #include <asm/traps.h>
 #include <asm/kprobes.h>
-#include <linux/uaccess.h>
-#include <asm/cpufeature.h>
 
 #define CREATE_TRACE_POINTS
 #include "trace-events-emulation.h"
diff --git a/arch/arm64/kernel/cacheinfo.c b/arch/arm64/kernel/cacheinfo.c
index 380f2e2fbed5..0bf0a835122f 100644
--- a/arch/arm64/kernel/cacheinfo.c
+++ b/arch/arm64/kernel/cacheinfo.c
@@ -17,6 +17,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/acpi.h>
 #include <linux/cacheinfo.h>
 #include <linux/of.h>
 
@@ -46,7 +47,7 @@ static void ci_leaf_init(struct cacheinfo *this_leaf,
 
 static int __init_cache_level(unsigned int cpu)
 {
-	unsigned int ctype, level, leaves, of_level;
+	unsigned int ctype, level, leaves, fw_level;
 	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
 
 	for (level = 1, leaves = 0; level <= MAX_CACHE_LEVEL; level++) {
@@ -59,15 +60,19 @@ static int __init_cache_level(unsigned int cpu)
 		leaves += (ctype == CACHE_TYPE_SEPARATE) ? 2 : 1;
 	}
 
-	of_level = of_find_last_cache_level(cpu);
-	if (level < of_level) {
+	if (acpi_disabled)
+		fw_level = of_find_last_cache_level(cpu);
+	else
+		fw_level = acpi_find_last_cache_level(cpu);
+
+	if (level < fw_level) {
 		/*
 		 * some external caches not specified in CLIDR_EL1
 		 * the information may be available in the device tree
 		 * only unified external caches are considered here
 		 */
-		leaves += (of_level - level);
-		level = of_level;
+		leaves += (fw_level - level);
+		level = fw_level;
 	}
 
 	this_cpu_ci->num_levels = level;
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 9d1b06d67c53..fbee8c17a4e6 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1606,7 +1606,6 @@ static void __init setup_system_capabilities(void)
 void __init setup_cpu_features(void)
 {
 	u32 cwg;
-	int cls;
 
 	setup_system_capabilities();
 	mark_const_caps_ready();
@@ -1627,13 +1626,9 @@ void __init setup_cpu_features(void)
 	 * Check for sane CTR_EL0.CWG value.
 	 */
 	cwg = cache_type_cwg();
-	cls = cache_line_size();
 	if (!cwg)
-		pr_warn("No Cache Writeback Granule information, assuming cache line size %d\n",
-			cls);
-	if (L1_CACHE_BYTES < cls)
-		pr_warn("L1_CACHE_BYTES smaller than the Cache Writeback Granule (%d < %d)\n",
-			L1_CACHE_BYTES, cls);
+		pr_warn("No Cache Writeback Granule information, assuming %d\n",
+			ARCH_DMA_MINALIGN);
 }
 
 static bool __maybe_unused
diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S
index 73f17bffcd23..12d4958e6429 100644
--- a/arch/arm64/kernel/entry-fpsimd.S
+++ b/arch/arm64/kernel/entry-fpsimd.S
@@ -49,7 +49,7 @@ ENTRY(sve_save_state)
 ENDPROC(sve_save_state)
 
 ENTRY(sve_load_state)
-	sve_load 0, x1, x2, 3
+	sve_load 0, x1, x2, 3, x4
 	ret
 ENDPROC(sve_load_state)
 
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 87a35364e750..99dbeef0a984 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -31,18 +31,19 @@
 #include <linux/percpu.h>
 #include <linux/prctl.h>
 #include <linux/preempt.h>
-#include <linux/prctl.h>
 #include <linux/ptrace.h>
 #include <linux/sched/signal.h>
 #include <linux/sched/task_stack.h>
 #include <linux/signal.h>
 #include <linux/slab.h>
+#include <linux/stddef.h>
 #include <linux/sysctl.h>
 
 #include <asm/esr.h>
 #include <asm/fpsimd.h>
 #include <asm/cpufeature.h>
 #include <asm/cputype.h>
+#include <asm/processor.h>
 #include <asm/simd.h>
 #include <asm/sigcontext.h>
 #include <asm/sysreg.h>
@@ -118,7 +119,6 @@
  */
 struct fpsimd_last_state_struct {
 	struct user_fpsimd_state *st;
-	bool sve_in_use;
 };
 
 static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
@@ -159,19 +159,6 @@ static void sve_free(struct task_struct *task)
 	__sve_free(task);
 }
 
-
-/* Offset of FFR in the SVE register dump */
-static size_t sve_ffr_offset(int vl)
-{
-	return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
-}
-
-static void *sve_pffr(struct task_struct *task)
-{
-	return (char *)task->thread.sve_state +
-		sve_ffr_offset(task->thread.sve_vl);
-}
-
 static void change_cpacr(u64 val, u64 mask)
 {
 	u64 cpacr = read_sysreg(CPACR_EL1);
@@ -252,31 +239,24 @@ static void task_fpsimd_load(void)
 	WARN_ON(!in_softirq() && !irqs_disabled());
 
 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
-		sve_load_state(sve_pffr(current),
+		sve_load_state(sve_pffr(&current->thread),
 			       &current->thread.uw.fpsimd_state.fpsr,
 			       sve_vq_from_vl(current->thread.sve_vl) - 1);
 	else
 		fpsimd_load_state(&current->thread.uw.fpsimd_state);
-
-	if (system_supports_sve()) {
-		/* Toggle SVE trapping for userspace if needed */
-		if (test_thread_flag(TIF_SVE))
-			sve_user_enable();
-		else
-			sve_user_disable();
-
-		/* Serialised by exception return to user */
-	}
 }
 
 /*
- * Ensure current's FPSIMD/SVE storage in thread_struct is up to date
- * with respect to the CPU registers.
+ * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
+ * date with respect to the CPU registers.
  *
  * Softirqs (and preemption) must be disabled.
  */
-static void task_fpsimd_save(void)
+void fpsimd_save(void)
 {
+	struct user_fpsimd_state *st = __this_cpu_read(fpsimd_last_state.st);
+	/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
+
 	WARN_ON(!in_softirq() && !irqs_disabled());
 
 	if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
@@ -291,10 +271,9 @@ static void task_fpsimd_save(void)
 				return;
 			}
 
-			sve_save_state(sve_pffr(current),
-				       &current->thread.uw.fpsimd_state.fpsr);
+			sve_save_state(sve_pffr(&current->thread), &st->fpsr);
 		} else
-			fpsimd_save_state(&current->thread.uw.fpsimd_state);
+			fpsimd_save_state(st);
 	}
 }
 
@@ -598,7 +577,7 @@ int sve_set_vector_length(struct task_struct *task,
 	if (task == current) {
 		local_bh_disable();
 
-		task_fpsimd_save();
+		fpsimd_save();
 		set_thread_flag(TIF_FOREIGN_FPSTATE);
 	}
 
@@ -618,10 +597,8 @@ int sve_set_vector_length(struct task_struct *task,
 	task->thread.sve_vl = vl;
 
 out:
-	if (flags & PR_SVE_VL_INHERIT)
-		set_tsk_thread_flag(task, TIF_SVE_VL_INHERIT);
-	else
-		clear_tsk_thread_flag(task, TIF_SVE_VL_INHERIT);
+	update_tsk_thread_flag(task, TIF_SVE_VL_INHERIT,
+			       flags & PR_SVE_VL_INHERIT);
 
 	return 0;
 }
@@ -765,6 +742,33 @@ void sve_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p)
 	isb();
 }
 
+/*
+ * Read the pseudo-ZCR used by cpufeatures to identify the supported SVE
+ * vector length.
+ *
+ * Use only if SVE is present.
+ * This function clobbers the SVE vector length.
+ */
+u64 read_zcr_features(void)
+{
+	u64 zcr;
+	unsigned int vq_max;
+
+	/*
+	 * Set the maximum possible VL, and write zeroes to all other
+	 * bits to see if they stick.
+	 */
+	sve_kernel_enable(NULL);
+	write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);
+
+	zcr = read_sysreg_s(SYS_ZCR_EL1);
+	zcr &= ~(u64)ZCR_ELx_LEN_MASK; /* find sticky 1s outside LEN field */
+	vq_max = sve_vq_from_vl(sve_get_vl());
+	zcr |= vq_max - 1; /* set LEN field to maximum effective value */
+
+	return zcr;
+}
+
 void __init sve_setup(void)
 {
 	u64 zcr;
@@ -839,7 +843,7 @@ asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
 
 	local_bh_disable();
 
-	task_fpsimd_save();
+	fpsimd_save();
 	fpsimd_to_sve(current);
 
 	/* Force ret_to_user to reload the registers: */
@@ -882,7 +886,7 @@ asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 			si_code = FPE_FLTRES;
 	}
 
-	memset(&info, 0, sizeof(info));
+	clear_siginfo(&info);
 	info.si_signo = SIGFPE;
 	info.si_code = si_code;
 	info.si_addr = (void __user *)instruction_pointer(regs);
@@ -892,31 +896,25 @@ asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 
 void fpsimd_thread_switch(struct task_struct *next)
 {
+	bool wrong_task, wrong_cpu;
+
 	if (!system_supports_fpsimd())
 		return;
+
+	/* Save unsaved fpsimd state, if any: */
+	fpsimd_save();
+
 	/*
-	 * Save the current FPSIMD state to memory, but only if whatever is in
-	 * the registers is in fact the most recent userland FPSIMD state of
-	 * 'current'.
+	 * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
+	 * state.  For kernel threads, FPSIMD registers are never loaded
+	 * and wrong_task and wrong_cpu will always be true.
 	 */
-	if (current->mm)
-		task_fpsimd_save();
+	wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
+					&next->thread.uw.fpsimd_state;
+	wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
 
-	if (next->mm) {
-		/*
-		 * If we are switching to a task whose most recent userland
-		 * FPSIMD state is already in the registers of *this* cpu,
-		 * we can skip loading the state from memory. Otherwise, set
-		 * the TIF_FOREIGN_FPSTATE flag so the state will be loaded
-		 * upon the next return to userland.
-		 */
-		if (__this_cpu_read(fpsimd_last_state.st) ==
-			&next->thread.uw.fpsimd_state
-		    && next->thread.fpsimd_cpu == smp_processor_id())
-			clear_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
-		else
-			set_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
-	}
+	update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
+			       wrong_task || wrong_cpu);
 }
 
 void fpsimd_flush_thread(void)
@@ -982,7 +980,7 @@ void fpsimd_preserve_current_state(void)
 		return;
 
 	local_bh_disable();
-	task_fpsimd_save();
+	fpsimd_save();
 	local_bh_enable();
 }
 
@@ -1002,14 +1000,33 @@ void fpsimd_signal_preserve_current_state(void)
  * Associate current's FPSIMD context with this cpu
  * Preemption must be disabled when calling this function.
  */
-static void fpsimd_bind_to_cpu(void)
+void fpsimd_bind_task_to_cpu(void)
 {
 	struct fpsimd_last_state_struct *last =
 		this_cpu_ptr(&fpsimd_last_state);
 
 	last->st = &current->thread.uw.fpsimd_state;
-	last->sve_in_use = test_thread_flag(TIF_SVE);
 	current->thread.fpsimd_cpu = smp_processor_id();
+
+	if (system_supports_sve()) {
+		/* Toggle SVE trapping for userspace if needed */
+		if (test_thread_flag(TIF_SVE))
+			sve_user_enable();
+		else
+			sve_user_disable();
+
+		/* Serialised by exception return to user */
+	}
+}
+
+void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st)
+{
+	struct fpsimd_last_state_struct *last =
+		this_cpu_ptr(&fpsimd_last_state);
+
+	WARN_ON(!in_softirq() && !irqs_disabled());
+
+	last->st = st;
 }
 
 /*
@@ -1026,7 +1043,7 @@ void fpsimd_restore_current_state(void)
 
 	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
 		task_fpsimd_load();
-		fpsimd_bind_to_cpu();
+		fpsimd_bind_task_to_cpu();
 	}
 
 	local_bh_enable();
@@ -1049,9 +1066,9 @@ void fpsimd_update_current_state(struct user_fpsimd_state const *state)
 		fpsimd_to_sve(current);
 
 	task_fpsimd_load();
+	fpsimd_bind_task_to_cpu();
 
-	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE))
-		fpsimd_bind_to_cpu();
+	clear_thread_flag(TIF_FOREIGN_FPSTATE);
 
 	local_bh_enable();
 }
@@ -1064,29 +1081,12 @@ void fpsimd_flush_task_state(struct task_struct *t)
 	t->thread.fpsimd_cpu = NR_CPUS;
 }
 
-static inline void fpsimd_flush_cpu_state(void)
+void fpsimd_flush_cpu_state(void)
 {
 	__this_cpu_write(fpsimd_last_state.st, NULL);
+	set_thread_flag(TIF_FOREIGN_FPSTATE);
 }
 
-/*
- * Invalidate any task SVE state currently held in this CPU's regs.
- *
- * This is used to prevent the kernel from trying to reuse SVE register data
- * that is detroyed by KVM guest enter/exit.  This function should go away when
- * KVM SVE support is implemented.  Don't use it for anything else.
- */
-#ifdef CONFIG_ARM64_SVE
-void sve_flush_cpu_state(void)
-{
-	struct fpsimd_last_state_struct const *last =
-		this_cpu_ptr(&fpsimd_last_state);
-
-	if (last->st && last->sve_in_use)
-		fpsimd_flush_cpu_state();
-}
-#endif /* CONFIG_ARM64_SVE */
-
 #ifdef CONFIG_KERNEL_MODE_NEON
 
 DEFINE_PER_CPU(bool, kernel_neon_busy);
@@ -1120,11 +1120,8 @@ void kernel_neon_begin(void)
 
 	__this_cpu_write(kernel_neon_busy, true);
 
-	/* Save unsaved task fpsimd state, if any: */
-	if (current->mm) {
-		task_fpsimd_save();
-		set_thread_flag(TIF_FOREIGN_FPSTATE);
-	}
+	/* Save unsaved fpsimd state, if any: */
+	fpsimd_save();
 
 	/* Invalidate any task state remaining in the fpsimd regs: */
 	fpsimd_flush_cpu_state();
@@ -1246,13 +1243,10 @@ static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
 {
 	switch (cmd) {
 	case CPU_PM_ENTER:
-		if (current->mm)
-			task_fpsimd_save();
+		fpsimd_save();
 		fpsimd_flush_cpu_state();
 		break;
 	case CPU_PM_EXIT:
-		if (current->mm)
-			set_thread_flag(TIF_FOREIGN_FPSTATE);
 		break;
 	case CPU_PM_ENTER_FAILED:
 	default:
diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c
index 74bb56f656ef..413dbe530da8 100644
--- a/arch/arm64/kernel/hw_breakpoint.c
+++ b/arch/arm64/kernel/hw_breakpoint.c
@@ -30,7 +30,6 @@
 #include <linux/smp.h>
 #include <linux/uaccess.h>
 
-#include <asm/compat.h>
 #include <asm/current.h>
 #include <asm/debug-monitors.h>
 #include <asm/hw_breakpoint.h>
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index 85a251b6dfa8..33147aacdafd 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -670,11 +670,10 @@ static void armv8pmu_disable_event(struct perf_event *event)
 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 }
 
-static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
+static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu)
 {
 	u32 pmovsr;
 	struct perf_sample_data data;
-	struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
 	struct pt_regs *regs;
 	int idx;
diff --git a/arch/arm64/kernel/perf_regs.c b/arch/arm64/kernel/perf_regs.c
index 1d091d048d04..0bbac612146e 100644
--- a/arch/arm64/kernel/perf_regs.c
+++ b/arch/arm64/kernel/perf_regs.c
@@ -1,11 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0
+#include <linux/compat.h>
 #include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/perf_event.h>
 #include <linux/bug.h>
 #include <linux/sched/task_stack.h>
 
-#include <asm/compat.h>
 #include <asm/perf_regs.h>
 #include <asm/ptrace.h>
 
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index 7ff81fed46e1..317ac57459f2 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -44,6 +44,7 @@
 #include <asm/compat.h>
 #include <asm/cpufeature.h>
 #include <asm/debug-monitors.h>
+#include <asm/fpsimd.h>
 #include <asm/pgtable.h>
 #include <asm/stacktrace.h>
 #include <asm/syscall.h>
@@ -1046,8 +1047,6 @@ static const struct user_regset_view user_aarch64_view = {
 };
 
 #ifdef CONFIG_COMPAT
-#include <linux/compat.h>
-
 enum compat_regset {
 	REGSET_COMPAT_GPR,
 	REGSET_COMPAT_VFP,
diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c
index 93ab57dcfc14..a6109825eeb9 100644
--- a/arch/arm64/kernel/sys_compat.c
+++ b/arch/arm64/kernel/sys_compat.c
@@ -112,6 +112,7 @@ long compat_arm_syscall(struct pt_regs *regs)
 		break;
 	}
 
+	clear_siginfo(&info);
 	info.si_signo = SIGILL;
 	info.si_errno = 0;
 	info.si_code  = ILL_ILLTRP;
diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c
index 21868530018e..7415c166281f 100644
--- a/arch/arm64/kernel/topology.c
+++ b/arch/arm64/kernel/topology.c
@@ -11,7 +11,9 @@
  * for more details.
  */
 
+#include <linux/acpi.h>
 #include <linux/arch_topology.h>
+#include <linux/cacheinfo.h>
 #include <linux/cpu.h>
 #include <linux/cpumask.h>
 #include <linux/init.h>
@@ -22,6 +24,7 @@
 #include <linux/sched.h>
 #include <linux/sched/topology.h>
 #include <linux/slab.h>
+#include <linux/smp.h>
 #include <linux/string.h>
 
 #include <asm/cpu.h>
@@ -47,7 +50,7 @@ static int __init get_cpu_for_node(struct device_node *node)
 	return cpu;
 }
 
-static int __init parse_core(struct device_node *core, int cluster_id,
+static int __init parse_core(struct device_node *core, int package_id,
 			     int core_id)
 {
 	char name[10];
@@ -63,7 +66,7 @@ static int __init parse_core(struct device_node *core, int cluster_id,
 			leaf = false;
 			cpu = get_cpu_for_node(t);
 			if (cpu >= 0) {
-				cpu_topology[cpu].cluster_id = cluster_id;
+				cpu_topology[cpu].package_id = package_id;
 				cpu_topology[cpu].core_id = core_id;
 				cpu_topology[cpu].thread_id = i;
 			} else {
@@ -85,7 +88,7 @@ static int __init parse_core(struct device_node *core, int cluster_id,
 			return -EINVAL;
 		}
 
-		cpu_topology[cpu].cluster_id = cluster_id;
+		cpu_topology[cpu].package_id = package_id;
 		cpu_topology[cpu].core_id = core_id;
 	} else if (leaf) {
 		pr_err("%pOF: Can't get CPU for leaf core\n", core);
@@ -101,7 +104,7 @@ static int __init parse_cluster(struct device_node *cluster, int depth)
 	bool leaf = true;
 	bool has_cores = false;
 	struct device_node *c;
-	static int cluster_id __initdata;
+	static int package_id __initdata;
 	int core_id = 0;
 	int i, ret;
 
@@ -140,7 +143,7 @@ static int __init parse_cluster(struct device_node *cluster, int depth)
 			}
 
 			if (leaf) {
-				ret = parse_core(c, cluster_id, core_id++);
+				ret = parse_core(c, package_id, core_id++);
 			} else {
 				pr_err("%pOF: Non-leaf cluster with core %s\n",
 				       cluster, name);
@@ -158,7 +161,7 @@ static int __init parse_cluster(struct device_node *cluster, int depth)
 		pr_warn("%pOF: empty cluster\n", cluster);
 
 	if (leaf)
-		cluster_id++;
+		package_id++;
 
 	return 0;
 }
@@ -194,7 +197,7 @@ static int __init parse_dt_topology(void)
 	 * only mark cores described in the DT as possible.
 	 */
 	for_each_possible_cpu(cpu)
-		if (cpu_topology[cpu].cluster_id == -1)
+		if (cpu_topology[cpu].package_id == -1)
 			ret = -EINVAL;
 
 out_map:
@@ -212,7 +215,19 @@ EXPORT_SYMBOL_GPL(cpu_topology);
 
 const struct cpumask *cpu_coregroup_mask(int cpu)
 {
-	return &cpu_topology[cpu].core_sibling;
+	const cpumask_t *core_mask = cpumask_of_node(cpu_to_node(cpu));
+
+	/* Find the smaller of NUMA, core or LLC siblings */
+	if (cpumask_subset(&cpu_topology[cpu].core_sibling, core_mask)) {
+		/* not numa in package, lets use the package siblings */
+		core_mask = &cpu_topology[cpu].core_sibling;
+	}
+	if (cpu_topology[cpu].llc_id != -1) {
+		if (cpumask_subset(&cpu_topology[cpu].llc_siblings, core_mask))
+			core_mask = &cpu_topology[cpu].llc_siblings;
+	}
+
+	return core_mask;
 }
 
 static void update_siblings_masks(unsigned int cpuid)
@@ -224,7 +239,10 @@ static void update_siblings_masks(unsigned int cpuid)
 	for_each_possible_cpu(cpu) {
 		cpu_topo = &cpu_topology[cpu];
 
-		if (cpuid_topo->cluster_id != cpu_topo->cluster_id)
+		if (cpuid_topo->llc_id == cpu_topo->llc_id)
+			cpumask_set_cpu(cpu, &cpuid_topo->llc_siblings);
+
+		if (cpuid_topo->package_id != cpu_topo->package_id)
 			continue;
 
 		cpumask_set_cpu(cpuid, &cpu_topo->core_sibling);
@@ -245,7 +263,7 @@ void store_cpu_topology(unsigned int cpuid)
 	struct cpu_topology *cpuid_topo = &cpu_topology[cpuid];
 	u64 mpidr;
 
-	if (cpuid_topo->cluster_id != -1)
+	if (cpuid_topo->package_id != -1)
 		goto topology_populated;
 
 	mpidr = read_cpuid_mpidr();
@@ -259,19 +277,19 @@ void store_cpu_topology(unsigned int cpuid)
 		/* Multiprocessor system : Multi-threads per core */
 		cpuid_topo->thread_id  = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 		cpuid_topo->core_id    = MPIDR_AFFINITY_LEVEL(mpidr, 1);
-		cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2) |
+		cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 2) |
 					 MPIDR_AFFINITY_LEVEL(mpidr, 3) << 8;
 	} else {
 		/* Multiprocessor system : Single-thread per core */
 		cpuid_topo->thread_id  = -1;
 		cpuid_topo->core_id    = MPIDR_AFFINITY_LEVEL(mpidr, 0);
-		cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1) |
+		cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 1) |
 					 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8 |
 					 MPIDR_AFFINITY_LEVEL(mpidr, 3) << 16;
 	}
 
 	pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n",
-		 cpuid, cpuid_topo->cluster_id, cpuid_topo->core_id,
+		 cpuid, cpuid_topo->package_id, cpuid_topo->core_id,
 		 cpuid_topo->thread_id, mpidr);
 
 topology_populated:
@@ -287,7 +305,11 @@ static void __init reset_cpu_topology(void)
 
 		cpu_topo->thread_id = -1;
 		cpu_topo->core_id = 0;
-		cpu_topo->cluster_id = -1;
+		cpu_topo->package_id = -1;
+
+		cpu_topo->llc_id = -1;
+		cpumask_clear(&cpu_topo->llc_siblings);
+		cpumask_set_cpu(cpu, &cpu_topo->llc_siblings);
 
 		cpumask_clear(&cpu_topo->core_sibling);
 		cpumask_set_cpu(cpu, &cpu_topo->core_sibling);
@@ -296,6 +318,59 @@ static void __init reset_cpu_topology(void)
 	}
 }
 
+#ifdef CONFIG_ACPI
+/*
+ * Propagate the topology information of the processor_topology_node tree to the
+ * cpu_topology array.
+ */
+static int __init parse_acpi_topology(void)
+{
+	bool is_threaded;
+	int cpu, topology_id;
+
+	is_threaded = read_cpuid_mpidr() & MPIDR_MT_BITMASK;
+
+	for_each_possible_cpu(cpu) {
+		int i, cache_id;
+
+		topology_id = find_acpi_cpu_topology(cpu, 0);
+		if (topology_id < 0)
+			return topology_id;
+
+		if (is_threaded) {
+			cpu_topology[cpu].thread_id = topology_id;
+			topology_id = find_acpi_cpu_topology(cpu, 1);
+			cpu_topology[cpu].core_id   = topology_id;
+		} else {
+			cpu_topology[cpu].thread_id  = -1;
+			cpu_topology[cpu].core_id    = topology_id;
+		}
+		topology_id = find_acpi_cpu_topology_package(cpu);
+		cpu_topology[cpu].package_id = topology_id;
+
+		i = acpi_find_last_cache_level(cpu);
+
+		if (i > 0) {
+			/*
+			 * this is the only part of cpu_topology that has
+			 * a direct relationship with the cache topology
+			 */
+			cache_id = find_acpi_cpu_cache_topology(cpu, i);
+			if (cache_id > 0)
+				cpu_topology[cpu].llc_id = cache_id;
+		}
+	}
+
+	return 0;
+}
+
+#else
+static inline int __init parse_acpi_topology(void)
+{
+	return -EINVAL;
+}
+#endif
+
 void __init init_cpu_topology(void)
 {
 	reset_cpu_topology();
@@ -304,6 +379,8 @@ void __init init_cpu_topology(void)
 	 * Discard anything that was parsed if we hit an error so we
 	 * don't use partial information.
 	 */
-	if (of_have_populated_dt() && parse_dt_topology())
+	if (!acpi_disabled && parse_acpi_topology())
+		reset_cpu_topology();
+	else if (of_have_populated_dt() && parse_dt_topology())
 		reset_cpu_topology();
 }
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 8bbdc17e49df..d399d459397b 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -635,6 +635,7 @@ asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
 	siginfo_t info;
 	void __user *pc = (void __user *)instruction_pointer(regs);
 
+	clear_siginfo(&info);
 	info.si_signo = SIGILL;
 	info.si_errno = 0;
 	info.si_code  = ILL_ILLOPC;
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 0221aca6493d..605d1b60469c 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -34,25 +34,25 @@ jiffies = jiffies_64;
 	 *    4 KB (see related ASSERT() below)		\
 	 */						\
 	. = ALIGN(SZ_4K);				\
-	VMLINUX_SYMBOL(__hyp_idmap_text_start) = .;	\
+	__hyp_idmap_text_start = .;			\
 	*(.hyp.idmap.text)				\
-	VMLINUX_SYMBOL(__hyp_idmap_text_end) = .;	\
-	VMLINUX_SYMBOL(__hyp_text_start) = .;		\
+	__hyp_idmap_text_end = .;			\
+	__hyp_text_start = .;				\
 	*(.hyp.text)					\
-	VMLINUX_SYMBOL(__hyp_text_end) = .;
+	__hyp_text_end = .;
 
 #define IDMAP_TEXT					\
 	. = ALIGN(SZ_4K);				\
-	VMLINUX_SYMBOL(__idmap_text_start) = .;		\
+	__idmap_text_start = .;				\
 	*(.idmap.text)					\
-	VMLINUX_SYMBOL(__idmap_text_end) = .;
+	__idmap_text_end = .;
 
 #ifdef CONFIG_HIBERNATION
 #define HIBERNATE_TEXT					\
 	. = ALIGN(SZ_4K);				\
-	VMLINUX_SYMBOL(__hibernate_exit_text_start) = .;\
+	__hibernate_exit_text_start = .;		\
 	*(.hibernate_exit.text)				\
-	VMLINUX_SYMBOL(__hibernate_exit_text_end) = .;
+	__hibernate_exit_text_end = .;
 #else
 #define HIBERNATE_TEXT
 #endif
@@ -60,10 +60,10 @@ jiffies = jiffies_64;
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
 #define TRAMP_TEXT					\
 	. = ALIGN(PAGE_SIZE);				\
-	VMLINUX_SYMBOL(__entry_tramp_text_start) = .;	\
+	__entry_tramp_text_start = .;			\
 	*(.entry.tramp.text)				\
 	. = ALIGN(PAGE_SIZE);				\
-	VMLINUX_SYMBOL(__entry_tramp_text_end) = .;
+	__entry_tramp_text_end = .;
 #else
 #define TRAMP_TEXT
 #endif
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index a2e3a5af1113..47b23bf617c7 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -39,6 +39,7 @@ config KVM
 	select HAVE_KVM_IRQ_ROUTING
 	select IRQ_BYPASS_MANAGER
 	select HAVE_KVM_IRQ_BYPASS
+	select HAVE_KVM_VCPU_RUN_PID_CHANGE
 	---help---
 	  Support hosting virtualized guest machines.
 	  We don't support KVM with 16K page tables yet, due to the multiple
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 93afff91cb7c..0f2a135ba15b 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -19,7 +19,7 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/psci.o $(KVM)/arm/perf.o
 kvm-$(CONFIG_KVM_ARM_HOST) += inject_fault.o regmap.o va_layout.o
 kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o
 kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o
-kvm-$(CONFIG_KVM_ARM_HOST) += vgic-sys-reg-v3.o
+kvm-$(CONFIG_KVM_ARM_HOST) += vgic-sys-reg-v3.o fpsimd.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/aarch32.o
 
 kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic.o
diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
index a1f4ebdfe6d3..00d422336a45 100644
--- a/arch/arm64/kvm/debug.c
+++ b/arch/arm64/kvm/debug.c
@@ -103,7 +103,7 @@ void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu)
  *
  * Additionally, KVM only traps guest accesses to the debug registers if
  * the guest is not actively using them (see the KVM_ARM64_DEBUG_DIRTY
- * flag on vcpu->arch.debug_flags).  Since the guest must not interfere
+ * flag on vcpu->arch.flags).  Since the guest must not interfere
  * with the hardware state when debugging the guest, we must ensure that
  * trapping is enabled whenever we are debugging the guest using the
  * debug registers.
@@ -111,7 +111,7 @@ void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu)
 
 void kvm_arm_setup_debug(struct kvm_vcpu *vcpu)
 {
-	bool trap_debug = !(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY);
+	bool trap_debug = !(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY);
 	unsigned long mdscr;
 
 	trace_kvm_arm_setup_debug(vcpu, vcpu->guest_debug);
@@ -184,7 +184,7 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu)
 			vcpu_write_sys_reg(vcpu, mdscr, MDSCR_EL1);
 
 			vcpu->arch.debug_ptr = &vcpu->arch.external_debug_state;
-			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
+			vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
 			trap_debug = true;
 
 			trace_kvm_arm_set_regset("BKPTS", get_num_brps(),
@@ -206,7 +206,7 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu)
 
 	/* If KDE or MDE are set, perform a full save/restore cycle. */
 	if (vcpu_read_sys_reg(vcpu, MDSCR_EL1) & (DBG_MDSCR_KDE | DBG_MDSCR_MDE))
-		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
+		vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
 
 	trace_kvm_arm_set_dreg32("MDCR_EL2", vcpu->arch.mdcr_el2);
 	trace_kvm_arm_set_dreg32("MDSCR_EL1", vcpu_read_sys_reg(vcpu, MDSCR_EL1));
diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c
new file mode 100644
index 000000000000..dc6ecfa5a2d2
--- /dev/null
+++ b/arch/arm64/kvm/fpsimd.c
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * arch/arm64/kvm/fpsimd.c: Guest/host FPSIMD context coordination helpers
+ *
+ * Copyright 2018 Arm Limited
+ * Author: Dave Martin <Dave.Martin@arm.com>
+ */
+#include <linux/bottom_half.h>
+#include <linux/sched.h>
+#include <linux/thread_info.h>
+#include <linux/kvm_host.h>
+#include <asm/kvm_asm.h>
+#include <asm/kvm_host.h>
+#include <asm/kvm_mmu.h>
+
+/*
+ * Called on entry to KVM_RUN unless this vcpu previously ran at least
+ * once and the most recent prior KVM_RUN for this vcpu was called from
+ * the same task as current (highly likely).
+ *
+ * This is guaranteed to execute before kvm_arch_vcpu_load_fp(vcpu),
+ * such that on entering hyp the relevant parts of current are already
+ * mapped.
+ */
+int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu)
+{
+	int ret;
+
+	struct thread_info *ti = &current->thread_info;
+	struct user_fpsimd_state *fpsimd = &current->thread.uw.fpsimd_state;
+
+	/*
+	 * Make sure the host task thread flags and fpsimd state are
+	 * visible to hyp:
+	 */
+	ret = create_hyp_mappings(ti, ti + 1, PAGE_HYP);
+	if (ret)
+		goto error;
+
+	ret = create_hyp_mappings(fpsimd, fpsimd + 1, PAGE_HYP);
+	if (ret)
+		goto error;
+
+	vcpu->arch.host_thread_info = kern_hyp_va(ti);
+	vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd);
+error:
+	return ret;
+}
+
+/*
+ * Prepare vcpu for saving the host's FPSIMD state and loading the guest's.
+ * The actual loading is done by the FPSIMD access trap taken to hyp.
+ *
+ * Here, we just set the correct metadata to indicate that the FPSIMD
+ * state in the cpu regs (if any) belongs to current on the host.
+ *
+ * TIF_SVE is backed up here, since it may get clobbered with guest state.
+ * This flag is restored by kvm_arch_vcpu_put_fp(vcpu).
+ */
+void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu)
+{
+	BUG_ON(!current->mm);
+
+	vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED | KVM_ARM64_HOST_SVE_IN_USE);
+	vcpu->arch.flags |= KVM_ARM64_FP_HOST;
+	if (test_thread_flag(TIF_SVE))
+		vcpu->arch.flags |= KVM_ARM64_HOST_SVE_IN_USE;
+}
+
+/*
+ * If the guest FPSIMD state was loaded, update the host's context
+ * tracking data mark the CPU FPSIMD regs as dirty and belonging to vcpu
+ * so that they will be written back if the kernel clobbers them due to
+ * kernel-mode NEON before re-entry into the guest.
+ */
+void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
+{
+	WARN_ON_ONCE(!irqs_disabled());
+
+	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
+		fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.gp_regs.fp_regs);
+		clear_thread_flag(TIF_FOREIGN_FPSTATE);
+		clear_thread_flag(TIF_SVE);
+	}
+}
+
+/*
+ * Write back the vcpu FPSIMD regs if they are dirty, and invalidate the
+ * cpu FPSIMD regs so that they can't be spuriously reused if this vcpu
+ * disappears and another task or vcpu appears that recycles the same
+ * struct fpsimd_state.
+ */
+void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
+{
+	local_bh_disable();
+
+	update_thread_flag(TIF_SVE,
+			   vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE);
+
+	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
+		/* Clean guest FP state to memory and invalidate cpu view */
+		fpsimd_save();
+		fpsimd_flush_cpu_state();
+	} else if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
+		/* Ensure user trap controls are correctly restored */
+		fpsimd_bind_task_to_cpu();
+	}
+
+	local_bh_enable();
+}
diff --git a/arch/arm64/kvm/hyp/debug-sr.c b/arch/arm64/kvm/hyp/debug-sr.c
index 3e717f66f011..50009766e5e5 100644
--- a/arch/arm64/kvm/hyp/debug-sr.c
+++ b/arch/arm64/kvm/hyp/debug-sr.c
@@ -163,7 +163,7 @@ void __hyp_text __debug_switch_to_guest(struct kvm_vcpu *vcpu)
 	if (!has_vhe())
 		__debug_save_spe_nvhe(&vcpu->arch.host_debug_state.pmscr_el1);
 
-	if (!(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY))
+	if (!(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY))
 		return;
 
 	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
@@ -185,7 +185,7 @@ void __hyp_text __debug_switch_to_host(struct kvm_vcpu *vcpu)
 	if (!has_vhe())
 		__debug_restore_spe_nvhe(vcpu->arch.host_debug_state.pmscr_el1);
 
-	if (!(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY))
+	if (!(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY))
 		return;
 
 	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
@@ -196,7 +196,7 @@ void __hyp_text __debug_switch_to_host(struct kvm_vcpu *vcpu)
 	__debug_save_state(vcpu, guest_dbg, guest_ctxt);
 	__debug_restore_state(vcpu, host_dbg, host_ctxt);
 
-	vcpu->arch.debug_flags &= ~KVM_ARM64_DEBUG_DIRTY;
+	vcpu->arch.flags &= ~KVM_ARM64_DEBUG_DIRTY;
 }
 
 u32 __hyp_text __kvm_get_mdcr_el2(void)
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index e41a161d313a..fad1e164fe48 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -166,46 +166,3 @@ abort_guest_exit_end:
 	orr	x0, x0, x5
 1:	ret
 ENDPROC(__guest_exit)
-
-ENTRY(__fpsimd_guest_restore)
-	// x0: esr
-	// x1: vcpu
-	// x2-x29,lr: vcpu regs
-	// vcpu x0-x1 on the stack
-	stp	x2, x3, [sp, #-16]!
-	stp	x4, lr, [sp, #-16]!
-
-alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
-	mrs	x2, cptr_el2
-	bic	x2, x2, #CPTR_EL2_TFP
-	msr	cptr_el2, x2
-alternative_else
-	mrs	x2, cpacr_el1
-	orr	x2, x2, #CPACR_EL1_FPEN
-	msr	cpacr_el1, x2
-alternative_endif
-	isb
-
-	mov	x3, x1
-
-	ldr	x0, [x3, #VCPU_HOST_CONTEXT]
-	kern_hyp_va x0
-	add	x0, x0, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
-	bl	__fpsimd_save_state
-
-	add	x2, x3, #VCPU_CONTEXT
-	add	x0, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
-	bl	__fpsimd_restore_state
-
-	// Skip restoring fpexc32 for AArch64 guests
-	mrs	x1, hcr_el2
-	tbnz	x1, #HCR_RW_SHIFT, 1f
-	ldr	x4, [x3, #VCPU_FPEXC32_EL2]
-	msr	fpexc32_el2, x4
-1:
-	ldp	x4, lr, [sp], #16
-	ldp	x2, x3, [sp], #16
-	ldp	x0, x1, [sp], #16
-
-	eret
-ENDPROC(__fpsimd_guest_restore)
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index bffece27b5c1..753b9d213651 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -113,25 +113,6 @@ el1_hvc_guest:
 
 el1_trap:
 	get_vcpu_ptr	x1, x0
-
-	mrs		x0, esr_el2
-	lsr		x0, x0, #ESR_ELx_EC_SHIFT
-	/*
-	 * x0: ESR_EC
-	 * x1: vcpu pointer
-	 */
-
-	/*
-	 * We trap the first access to the FP/SIMD to save the host context
-	 * and restore the guest context lazily.
-	 * If FP/SIMD is not implemented, handle the trap and inject an
-	 * undefined instruction exception to the guest.
-	 */
-alternative_if_not ARM64_HAS_NO_FPSIMD
-	cmp	x0, #ESR_ELx_EC_FP_ASIMD
-	b.eq	__fpsimd_guest_restore
-alternative_else_nop_endif
-
 	mov	x0, #ARM_EXCEPTION_TRAP
 	b	__guest_exit
 
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index d9645236e474..2d45bd719a5d 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -21,21 +21,25 @@
 
 #include <kvm/arm_psci.h>
 
+#include <asm/cpufeature.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_emulate.h>
+#include <asm/kvm_host.h>
 #include <asm/kvm_hyp.h>
 #include <asm/kvm_mmu.h>
 #include <asm/fpsimd.h>
 #include <asm/debug-monitors.h>
+#include <asm/processor.h>
+#include <asm/thread_info.h>
 
-static bool __hyp_text __fpsimd_enabled_nvhe(void)
+/* Check whether the FP regs were dirtied while in the host-side run loop: */
+static bool __hyp_text update_fp_enabled(struct kvm_vcpu *vcpu)
 {
-	return !(read_sysreg(cptr_el2) & CPTR_EL2_TFP);
-}
+	if (vcpu->arch.host_thread_info->flags & _TIF_FOREIGN_FPSTATE)
+		vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED |
+				      KVM_ARM64_FP_HOST);
 
-static bool fpsimd_enabled_vhe(void)
-{
-	return !!(read_sysreg(cpacr_el1) & CPACR_EL1_FPEN);
+	return !!(vcpu->arch.flags & KVM_ARM64_FP_ENABLED);
 }
 
 /* Save the 32-bit only FPSIMD system register state */
@@ -92,7 +96,10 @@ static void activate_traps_vhe(struct kvm_vcpu *vcpu)
 
 	val = read_sysreg(cpacr_el1);
 	val |= CPACR_EL1_TTA;
-	val &= ~(CPACR_EL1_FPEN | CPACR_EL1_ZEN);
+	val &= ~CPACR_EL1_ZEN;
+	if (!update_fp_enabled(vcpu))
+		val &= ~CPACR_EL1_FPEN;
+
 	write_sysreg(val, cpacr_el1);
 
 	write_sysreg(kvm_get_hyp_vector(), vbar_el1);
@@ -105,7 +112,10 @@ static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu)
 	__activate_traps_common(vcpu);
 
 	val = CPTR_EL2_DEFAULT;
-	val |= CPTR_EL2_TTA | CPTR_EL2_TFP | CPTR_EL2_TZ;
+	val |= CPTR_EL2_TTA | CPTR_EL2_TZ;
+	if (!update_fp_enabled(vcpu))
+		val |= CPTR_EL2_TFP;
+
 	write_sysreg(val, cptr_el2);
 }
 
@@ -318,6 +328,50 @@ static bool __hyp_text __skip_instr(struct kvm_vcpu *vcpu)
 	}
 }
 
+static bool __hyp_text __hyp_switch_fpsimd(struct kvm_vcpu *vcpu)
+{
+	struct user_fpsimd_state *host_fpsimd = vcpu->arch.host_fpsimd_state;
+
+	if (has_vhe())
+		write_sysreg(read_sysreg(cpacr_el1) | CPACR_EL1_FPEN,
+			     cpacr_el1);
+	else
+		write_sysreg(read_sysreg(cptr_el2) & ~(u64)CPTR_EL2_TFP,
+			     cptr_el2);
+
+	isb();
+
+	if (vcpu->arch.flags & KVM_ARM64_FP_HOST) {
+		/*
+		 * In the SVE case, VHE is assumed: it is enforced by
+		 * Kconfig and kvm_arch_init().
+		 */
+		if (system_supports_sve() &&
+		    (vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE)) {
+			struct thread_struct *thread = container_of(
+				host_fpsimd,
+				struct thread_struct, uw.fpsimd_state);
+
+			sve_save_state(sve_pffr(thread), &host_fpsimd->fpsr);
+		} else {
+			__fpsimd_save_state(host_fpsimd);
+		}
+
+		vcpu->arch.flags &= ~KVM_ARM64_FP_HOST;
+	}
+
+	__fpsimd_restore_state(&vcpu->arch.ctxt.gp_regs.fp_regs);
+
+	/* Skip restoring fpexc32 for AArch64 guests */
+	if (!(read_sysreg(hcr_el2) & HCR_RW))
+		write_sysreg(vcpu->arch.ctxt.sys_regs[FPEXC32_EL2],
+			     fpexc32_el2);
+
+	vcpu->arch.flags |= KVM_ARM64_FP_ENABLED;
+
+	return true;
+}
+
 /*
  * Return true when we were able to fixup the guest exit and should return to
  * the guest, false when we should restore the host state and return to the
@@ -334,11 +388,23 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
 	 * same PC once the SError has been injected, and replay the
 	 * trapping instruction.
 	 */
-	if (*exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu))
+	if (*exit_code != ARM_EXCEPTION_TRAP)
+		goto exit;
+
+	/*
+	 * We trap the first access to the FP/SIMD to save the host context
+	 * and restore the guest context lazily.
+	 * If FP/SIMD is not implemented, handle the trap and inject an
+	 * undefined instruction exception to the guest.
+	 */
+	if (system_supports_fpsimd() &&
+	    kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_FP_ASIMD)
+		return __hyp_switch_fpsimd(vcpu);
+
+	if (!__populate_fault_info(vcpu))
 		return true;
 
-	if (static_branch_unlikely(&vgic_v2_cpuif_trap) &&
-	    *exit_code == ARM_EXCEPTION_TRAP) {
+	if (static_branch_unlikely(&vgic_v2_cpuif_trap)) {
 		bool valid;
 
 		valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&
@@ -350,12 +416,8 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
 		if (valid) {
 			int ret = __vgic_v2_perform_cpuif_access(vcpu);
 
-			if (ret == 1) {
-				if (__skip_instr(vcpu))
-					return true;
-				else
-					*exit_code = ARM_EXCEPTION_TRAP;
-			}
+			if (ret ==  1 && __skip_instr(vcpu))
+				return true;
 
 			if (ret == -1) {
 				/* Promote an illegal access to an
@@ -368,23 +430,21 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
 					*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
 				*exit_code = ARM_EXCEPTION_EL1_SERROR;
 			}
+
+			goto exit;
 		}
 	}
 
 	if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&
-	    *exit_code == ARM_EXCEPTION_TRAP &&
 	    (kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 ||
 	     kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) {
 		int ret = __vgic_v3_perform_cpuif_access(vcpu);
 
-		if (ret == 1) {
-			if (__skip_instr(vcpu))
-				return true;
-			else
-				*exit_code = ARM_EXCEPTION_TRAP;
-		}
+		if (ret == 1 && __skip_instr(vcpu))
+			return true;
 	}
 
+exit:
 	/* Return to the host kernel and handle the exit */
 	return false;
 }
@@ -394,7 +454,6 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
 {
 	struct kvm_cpu_context *host_ctxt;
 	struct kvm_cpu_context *guest_ctxt;
-	bool fp_enabled;
 	u64 exit_code;
 
 	host_ctxt = vcpu->arch.host_cpu_context;
@@ -416,19 +475,14 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
 		/* And we're baaack! */
 	} while (fixup_guest_exit(vcpu, &exit_code));
 
-	fp_enabled = fpsimd_enabled_vhe();
-
 	sysreg_save_guest_state_vhe(guest_ctxt);
 
 	__deactivate_traps(vcpu);
 
 	sysreg_restore_host_state_vhe(host_ctxt);
 
-	if (fp_enabled) {
-		__fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
-		__fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
+	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)
 		__fpsimd_save_fpexc32(vcpu);
-	}
 
 	__debug_switch_to_host(vcpu);
 
@@ -440,7 +494,6 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
 {
 	struct kvm_cpu_context *host_ctxt;
 	struct kvm_cpu_context *guest_ctxt;
-	bool fp_enabled;
 	u64 exit_code;
 
 	vcpu = kern_hyp_va(vcpu);
@@ -472,8 +525,6 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
 		/* And we're baaack! */
 	} while (fixup_guest_exit(vcpu, &exit_code));
 
-	fp_enabled = __fpsimd_enabled_nvhe();
-
 	__sysreg_save_state_nvhe(guest_ctxt);
 	__sysreg32_save_state(vcpu);
 	__timer_disable_traps(vcpu);
@@ -484,11 +535,8 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
 
 	__sysreg_restore_state_nvhe(host_ctxt);
 
-	if (fp_enabled) {
-		__fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
-		__fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
+	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)
 		__fpsimd_save_fpexc32(vcpu);
-	}
 
 	/*
 	 * This must come after restoring the host sysregs, since a non-VHE
diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c
index b3894df6bf1a..35bc16832efe 100644
--- a/arch/arm64/kvm/hyp/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/sysreg-sr.c
@@ -196,7 +196,7 @@ void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
 	sysreg[DACR32_EL2] = read_sysreg(dacr32_el2);
 	sysreg[IFSR32_EL2] = read_sysreg(ifsr32_el2);
 
-	if (has_vhe() || vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
+	if (has_vhe() || vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY)
 		sysreg[DBGVCR32_EL2] = read_sysreg(dbgvcr32_el2);
 }
 
@@ -218,7 +218,7 @@ void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
 	write_sysreg(sysreg[DACR32_EL2], dacr32_el2);
 	write_sysreg(sysreg[IFSR32_EL2], ifsr32_el2);
 
-	if (has_vhe() || vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
+	if (has_vhe() || vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY)
 		write_sysreg(sysreg[DBGVCR32_EL2], dbgvcr32_el2);
 }
 
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 6e3b969391fd..a4363735d3f8 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -31,7 +31,6 @@
 #include <asm/debug-monitors.h>
 #include <asm/esr.h>
 #include <asm/kvm_arm.h>
-#include <asm/kvm_asm.h>
 #include <asm/kvm_coproc.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_host.h>
@@ -338,7 +337,7 @@ static bool trap_debug_regs(struct kvm_vcpu *vcpu,
 {
 	if (p->is_write) {
 		vcpu_write_sys_reg(vcpu, p->regval, r->reg);
-		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
+		vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
 	} else {
 		p->regval = vcpu_read_sys_reg(vcpu, r->reg);
 	}
@@ -369,7 +368,7 @@ static void reg_to_dbg(struct kvm_vcpu *vcpu,
 	}
 
 	*dbg_reg = val;
-	vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
+	vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
 }
 
 static void dbg_to_reg(struct kvm_vcpu *vcpu,
@@ -1441,7 +1440,7 @@ static bool trap_debug32(struct kvm_vcpu *vcpu,
 {
 	if (p->is_write) {
 		vcpu_cp14(vcpu, r->reg) = p->regval;
-		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
+		vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
 	} else {
 		p->regval = vcpu_cp14(vcpu, r->reg);
 	}
@@ -1473,7 +1472,7 @@ static bool trap_xvr(struct kvm_vcpu *vcpu,
 		val |= p->regval << 32;
 		*dbg_reg = val;
 
-		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
+		vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
 	} else {
 		p->regval = *dbg_reg >> 32;
 	}
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index a96ec0181818..49e217ac7e1e 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -504,20 +504,15 @@ static int __init arm64_dma_init(void)
 	    max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT))
 		swiotlb = 1;
 
+	WARN_TAINT(ARCH_DMA_MINALIGN < cache_line_size(),
+		   TAINT_CPU_OUT_OF_SPEC,
+		   "ARCH_DMA_MINALIGN smaller than CTR_EL0.CWG (%d < %d)",
+		   ARCH_DMA_MINALIGN, cache_line_size());
+
 	return atomic_pool_init();
 }
 arch_initcall(arm64_dma_init);
 
-#define PREALLOC_DMA_DEBUG_ENTRIES	4096
-
-static int __init dma_debug_do_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-	return 0;
-}
-fs_initcall(dma_debug_do_init);
-
-
 #ifdef CONFIG_IOMMU_DMA
 #include <linux/dma-iommu.h>
 #include <linux/platform_device.h>
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 2af3dd89bcdb..576f15153080 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -356,11 +356,12 @@ static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *re
 	 */
 	if (user_mode(regs)) {
 		const struct fault_info *inf = esr_to_fault_info(esr);
-		struct siginfo si = {
-			.si_signo	= inf->sig,
-			.si_code	= inf->code,
-			.si_addr	= (void __user *)addr,
-		};
+		struct siginfo si;
+
+		clear_siginfo(&si);
+		si.si_signo	= inf->sig;
+		si.si_code	= inf->code;
+		si.si_addr	= (void __user *)addr;
 
 		__do_user_fault(&si, esr);
 	} else {
@@ -634,6 +635,7 @@ static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 			nmi_exit();
 	}
 
+	clear_siginfo(&info);
 	info.si_signo = inf->sig;
 	info.si_errno = 0;
 	info.si_code  = inf->code;
@@ -738,6 +740,7 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
 		show_pte(addr);
 	}
 
+	clear_siginfo(&info);
 	info.si_signo = inf->sig;
 	info.si_errno = 0;
 	info.si_code  = inf->code;
@@ -780,6 +783,7 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
 		local_irq_enable();
 	}
 
+	clear_siginfo(&info);
 	info.si_signo = SIGBUS;
 	info.si_errno = 0;
 	info.si_code  = BUS_ADRALN;
@@ -823,7 +827,6 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
 					      struct pt_regs *regs)
 {
 	const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr);
-	struct siginfo info;
 	int rv;
 
 	/*
@@ -839,6 +842,9 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
 	if (!inf->fn(addr, esr, regs)) {
 		rv = 1;
 	} else {
+		struct siginfo info;
+
+		clear_siginfo(&info);
 		info.si_signo = inf->sig;
 		info.si_errno = 0;
 		info.si_code  = inf->code;
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 1b18b4722420..325cfb3b858a 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -310,7 +310,7 @@ static void __init arm64_memory_present(void)
 }
 #endif
 
-static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX;
+static phys_addr_t memory_limit = PHYS_ADDR_MAX;
 
 /*
  * Limit the memory size that was specified via FDT.
@@ -401,7 +401,7 @@ void __init arm64_memblock_init(void)
 	 * high up in memory, add back the kernel region that must be accessible
 	 * via the linear mapping.
 	 */
-	if (memory_limit != (phys_addr_t)ULLONG_MAX) {
+	if (memory_limit != PHYS_ADDR_MAX) {
 		memblock_mem_limit_remove_map(memory_limit);
 		memblock_add(__pa_symbol(_text), (u64)(_end - _text));
 	}
@@ -666,7 +666,7 @@ __setup("keepinitrd", keepinitrd_setup);
  */
 static int dump_mem_limit(struct notifier_block *self, unsigned long v, void *p)
 {
-	if (memory_limit != (phys_addr_t)ULLONG_MAX) {
+	if (memory_limit != PHYS_ADDR_MAX) {
 		pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20);
 	} else {
 		pr_emerg("Memory Limit: none\n");
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index a93350451e8e..a6fdaea07c63 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -21,7 +21,6 @@
 #include <linux/bpf.h>
 #include <linux/filter.h>
 #include <linux/printk.h>
-#include <linux/skbuff.h>
 #include <linux/slab.h>
 
 #include <asm/byteorder.h>
@@ -80,23 +79,66 @@ static inline void emit(const u32 insn, struct jit_ctx *ctx)
 	ctx->idx++;
 }
 
+static inline void emit_a64_mov_i(const int is64, const int reg,
+				  const s32 val, struct jit_ctx *ctx)
+{
+	u16 hi = val >> 16;
+	u16 lo = val & 0xffff;
+
+	if (hi & 0x8000) {
+		if (hi == 0xffff) {
+			emit(A64_MOVN(is64, reg, (u16)~lo, 0), ctx);
+		} else {
+			emit(A64_MOVN(is64, reg, (u16)~hi, 16), ctx);
+			if (lo != 0xffff)
+				emit(A64_MOVK(is64, reg, lo, 0), ctx);
+		}
+	} else {
+		emit(A64_MOVZ(is64, reg, lo, 0), ctx);
+		if (hi)
+			emit(A64_MOVK(is64, reg, hi, 16), ctx);
+	}
+}
+
+static int i64_i16_blocks(const u64 val, bool inverse)
+{
+	return (((val >>  0) & 0xffff) != (inverse ? 0xffff : 0x0000)) +
+	       (((val >> 16) & 0xffff) != (inverse ? 0xffff : 0x0000)) +
+	       (((val >> 32) & 0xffff) != (inverse ? 0xffff : 0x0000)) +
+	       (((val >> 48) & 0xffff) != (inverse ? 0xffff : 0x0000));
+}
+
 static inline void emit_a64_mov_i64(const int reg, const u64 val,
 				    struct jit_ctx *ctx)
 {
-	u64 tmp = val;
-	int shift = 0;
-
-	emit(A64_MOVZ(1, reg, tmp & 0xffff, shift), ctx);
-	tmp >>= 16;
-	shift += 16;
-	while (tmp) {
-		if (tmp & 0xffff)
-			emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx);
-		tmp >>= 16;
-		shift += 16;
+	u64 nrm_tmp = val, rev_tmp = ~val;
+	bool inverse;
+	int shift;
+
+	if (!(nrm_tmp >> 32))
+		return emit_a64_mov_i(0, reg, (u32)val, ctx);
+
+	inverse = i64_i16_blocks(nrm_tmp, true) < i64_i16_blocks(nrm_tmp, false);
+	shift = max(round_down((inverse ? (fls64(rev_tmp) - 1) :
+					  (fls64(nrm_tmp) - 1)), 16), 0);
+	if (inverse)
+		emit(A64_MOVN(1, reg, (rev_tmp >> shift) & 0xffff, shift), ctx);
+	else
+		emit(A64_MOVZ(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx);
+	shift -= 16;
+	while (shift >= 0) {
+		if (((nrm_tmp >> shift) & 0xffff) != (inverse ? 0xffff : 0x0000))
+			emit(A64_MOVK(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx);
+		shift -= 16;
 	}
 }
 
+/*
+ * This is an unoptimized 64 immediate emission used for BPF to BPF call
+ * addresses. It will always do a full 64 bit decomposition as otherwise
+ * more complexity in the last extra pass is required since we previously
+ * reserved 4 instructions for the address.
+ */
 static inline void emit_addr_mov_i64(const int reg, const u64 val,
 				     struct jit_ctx *ctx)
 {
@@ -111,26 +153,6 @@ static inline void emit_addr_mov_i64(const int reg, const u64 val,
 	}
 }
 
-static inline void emit_a64_mov_i(const int is64, const int reg,
-				  const s32 val, struct jit_ctx *ctx)
-{
-	u16 hi = val >> 16;
-	u16 lo = val & 0xffff;
-
-	if (hi & 0x8000) {
-		if (hi == 0xffff) {
-			emit(A64_MOVN(is64, reg, (u16)~lo, 0), ctx);
-		} else {
-			emit(A64_MOVN(is64, reg, (u16)~hi, 16), ctx);
-			emit(A64_MOVK(is64, reg, lo, 0), ctx);
-		}
-	} else {
-		emit(A64_MOVZ(is64, reg, lo, 0), ctx);
-		if (hi)
-			emit(A64_MOVK(is64, reg, hi, 16), ctx);
-	}
-}
-
 static inline int bpf2a64_offset(int bpf_to, int bpf_from,
 				 const struct jit_ctx *ctx)
 {
@@ -163,7 +185,7 @@ static inline int epilogue_offset(const struct jit_ctx *ctx)
 /* Tail call offset to jump into */
 #define PROLOGUE_OFFSET 7
 
-static int build_prologue(struct jit_ctx *ctx)
+static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
 {
 	const struct bpf_prog *prog = ctx->prog;
 	const u8 r6 = bpf2a64[BPF_REG_6];
@@ -188,7 +210,7 @@ static int build_prologue(struct jit_ctx *ctx)
 	 *                        | ... | BPF prog stack
 	 *                        |     |
 	 *                        +-----+ <= (BPF_FP - prog->aux->stack_depth)
-	 *                        |RSVD | JIT scratchpad
+	 *                        |RSVD | padding
 	 * current A64_SP =>      +-----+ <= (BPF_FP - ctx->stack_size)
 	 *                        |     |
 	 *                        | ... | Function call stack
@@ -210,19 +232,19 @@ static int build_prologue(struct jit_ctx *ctx)
 	/* Set up BPF prog stack base register */
 	emit(A64_MOV(1, fp, A64_SP), ctx);
 
-	/* Initialize tail_call_cnt */
-	emit(A64_MOVZ(1, tcc, 0, 0), ctx);
+	if (!ebpf_from_cbpf) {
+		/* Initialize tail_call_cnt */
+		emit(A64_MOVZ(1, tcc, 0, 0), ctx);
 
-	cur_offset = ctx->idx - idx0;
-	if (cur_offset != PROLOGUE_OFFSET) {
-		pr_err_once("PROLOGUE_OFFSET = %d, expected %d!\n",
-			    cur_offset, PROLOGUE_OFFSET);
-		return -1;
+		cur_offset = ctx->idx - idx0;
+		if (cur_offset != PROLOGUE_OFFSET) {
+			pr_err_once("PROLOGUE_OFFSET = %d, expected %d!\n",
+				    cur_offset, PROLOGUE_OFFSET);
+			return -1;
+		}
 	}
 
-	/* 4 byte extra for skb_copy_bits buffer */
-	ctx->stack_size = prog->aux->stack_depth + 4;
-	ctx->stack_size = STACK_ALIGN(ctx->stack_size);
+	ctx->stack_size = STACK_ALIGN(prog->aux->stack_depth);
 
 	/* Set up function call stack */
 	emit(A64_SUB_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
@@ -723,71 +745,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 		emit(A64_CBNZ(0, tmp3, jmp_offset), ctx);
 		break;
 
-	/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
-	case BPF_LD | BPF_ABS | BPF_W:
-	case BPF_LD | BPF_ABS | BPF_H:
-	case BPF_LD | BPF_ABS | BPF_B:
-	/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
-	case BPF_LD | BPF_IND | BPF_W:
-	case BPF_LD | BPF_IND | BPF_H:
-	case BPF_LD | BPF_IND | BPF_B:
-	{
-		const u8 r0 = bpf2a64[BPF_REG_0]; /* r0 = return value */
-		const u8 r6 = bpf2a64[BPF_REG_6]; /* r6 = pointer to sk_buff */
-		const u8 fp = bpf2a64[BPF_REG_FP];
-		const u8 r1 = bpf2a64[BPF_REG_1]; /* r1: struct sk_buff *skb */
-		const u8 r2 = bpf2a64[BPF_REG_2]; /* r2: int k */
-		const u8 r3 = bpf2a64[BPF_REG_3]; /* r3: unsigned int size */
-		const u8 r4 = bpf2a64[BPF_REG_4]; /* r4: void *buffer */
-		const u8 r5 = bpf2a64[BPF_REG_5]; /* r5: void *(*func)(...) */
-		int size;
-
-		emit(A64_MOV(1, r1, r6), ctx);
-		emit_a64_mov_i(0, r2, imm, ctx);
-		if (BPF_MODE(code) == BPF_IND)
-			emit(A64_ADD(0, r2, r2, src), ctx);
-		switch (BPF_SIZE(code)) {
-		case BPF_W:
-			size = 4;
-			break;
-		case BPF_H:
-			size = 2;
-			break;
-		case BPF_B:
-			size = 1;
-			break;
-		default:
-			return -EINVAL;
-		}
-		emit_a64_mov_i64(r3, size, ctx);
-		emit(A64_SUB_I(1, r4, fp, ctx->stack_size), ctx);
-		emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx);
-		emit(A64_BLR(r5), ctx);
-		emit(A64_MOV(1, r0, A64_R(0)), ctx);
-
-		jmp_offset = epilogue_offset(ctx);
-		check_imm19(jmp_offset);
-		emit(A64_CBZ(1, r0, jmp_offset), ctx);
-		emit(A64_MOV(1, r5, r0), ctx);
-		switch (BPF_SIZE(code)) {
-		case BPF_W:
-			emit(A64_LDR32(r0, r5, A64_ZR), ctx);
-#ifndef CONFIG_CPU_BIG_ENDIAN
-			emit(A64_REV32(0, r0, r0), ctx);
-#endif
-			break;
-		case BPF_H:
-			emit(A64_LDRH(r0, r5, A64_ZR), ctx);
-#ifndef CONFIG_CPU_BIG_ENDIAN
-			emit(A64_REV16(0, r0, r0), ctx);
-#endif
-			break;
-		case BPF_B:
-			emit(A64_LDRB(r0, r5, A64_ZR), ctx);
-			break;
-		}
-		break;
-	}
 	default:
 		pr_err_once("unknown opcode %02x\n", code);
 		return -EINVAL;
@@ -851,6 +808,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	struct bpf_prog *tmp, *orig_prog = prog;
 	struct bpf_binary_header *header;
 	struct arm64_jit_data *jit_data;
+	bool was_classic = bpf_prog_was_classic(prog);
 	bool tmp_blinded = false;
 	bool extra_pass = false;
 	struct jit_ctx ctx;
@@ -905,7 +863,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		goto out_off;
 	}
 
-	if (build_prologue(&ctx)) {
+	if (build_prologue(&ctx, was_classic)) {
 		prog = orig_prog;
 		goto out_off;
 	}
@@ -928,7 +886,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 skip_init_ctx:
 	ctx.idx = 0;
 
-	build_prologue(&ctx);
+	build_prologue(&ctx, was_classic);
 
 	if (build_body(&ctx)) {
 		bpf_jit_binary_free(header);
diff --git a/arch/c6x/Kconfig b/arch/c6x/Kconfig
index c6b4dd1418b4..bf59855628ac 100644
--- a/arch/c6x/Kconfig
+++ b/arch/c6x/Kconfig
@@ -6,11 +6,13 @@
 
 config C6X
 	def_bool y
+	select ARCH_HAS_SYNC_DMA_FOR_CPU
+	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
 	select CLKDEV_LOOKUP
+	select DMA_NONCOHERENT_OPS
 	select GENERIC_ATOMIC64
 	select GENERIC_IRQ_SHOW
 	select HAVE_ARCH_TRACEHOOK
-	select HAVE_DMA_API_DEBUG
 	select HAVE_MEMBLOCK
 	select SPARSE_IRQ
 	select IRQ_DOMAIN
diff --git a/arch/c6x/include/asm/Kbuild b/arch/c6x/include/asm/Kbuild
index fd4c840de837..33a2c94fed0d 100644
--- a/arch/c6x/include/asm/Kbuild
+++ b/arch/c6x/include/asm/Kbuild
@@ -1,10 +1,12 @@
 generic-y += atomic.h
 generic-y += barrier.h
 generic-y += bugs.h
+generic-y += compat.h
 generic-y += current.h
 generic-y += device.h
 generic-y += div64.h
 generic-y += dma.h
+generic-y += dma-mapping.h
 generic-y += emergency-restart.h
 generic-y += exec.h
 generic-y += extable.h
diff --git a/arch/c6x/include/asm/dma-mapping.h b/arch/c6x/include/asm/dma-mapping.h
deleted file mode 100644
index 05daf1038111..000000000000
--- a/arch/c6x/include/asm/dma-mapping.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- *  Port on Texas Instruments TMS320C6x architecture
- *
- *  Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated
- *  Author: Aurelien Jacquiot <aurelien.jacquiot@ti.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License version 2 as
- *  published by the Free Software Foundation.
- *
- */
-#ifndef _ASM_C6X_DMA_MAPPING_H
-#define _ASM_C6X_DMA_MAPPING_H
-
-extern const struct dma_map_ops c6x_dma_ops;
-
-static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
-{
-	return &c6x_dma_ops;
-}
-
-extern void coherent_mem_init(u32 start, u32 size);
-void *c6x_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
-		gfp_t gfp, unsigned long attrs);
-void c6x_dma_free(struct device *dev, size_t size, void *vaddr,
-		dma_addr_t dma_handle, unsigned long attrs);
-
-#endif	/* _ASM_C6X_DMA_MAPPING_H */
diff --git a/arch/c6x/include/asm/setup.h b/arch/c6x/include/asm/setup.h
index 852afb209afb..350f34debb19 100644
--- a/arch/c6x/include/asm/setup.h
+++ b/arch/c6x/include/asm/setup.h
@@ -28,5 +28,7 @@ extern unsigned char c6x_fuse_mac[6];
 extern void machine_init(unsigned long dt_ptr);
 extern void time_init(void);
 
+extern void coherent_mem_init(u32 start, u32 size);
+
 #endif /* !__ASSEMBLY__ */
 #endif /* _ASM_C6X_SETUP_H */
diff --git a/arch/c6x/kernel/Makefile b/arch/c6x/kernel/Makefile
index 02f340d7b8fe..fbe74174de87 100644
--- a/arch/c6x/kernel/Makefile
+++ b/arch/c6x/kernel/Makefile
@@ -8,6 +8,6 @@ extra-y := head.o vmlinux.lds
 obj-y := process.o traps.o irq.o signal.o ptrace.o
 obj-y += setup.o sys_c6x.o time.o devicetree.o
 obj-y += switch_to.o entry.o vectors.o c6x_ksyms.o
-obj-y += soc.o dma.o
+obj-y += soc.o
 
 obj-$(CONFIG_MODULES)           += module.o
diff --git a/arch/c6x/kernel/dma.c b/arch/c6x/kernel/dma.c
deleted file mode 100644
index 9fff8be75f58..000000000000
--- a/arch/c6x/kernel/dma.c
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- *  Copyright (C) 2011 Texas Instruments Incorporated
- *  Author: Mark Salter <msalter@redhat.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License version 2 as
- *  published by the Free Software Foundation.
- */
-#include <linux/module.h>
-#include <linux/dma-mapping.h>
-#include <linux/mm.h>
-#include <linux/mm_types.h>
-#include <linux/scatterlist.h>
-
-#include <asm/cacheflush.h>
-
-static void c6x_dma_sync(dma_addr_t handle, size_t size,
-			 enum dma_data_direction dir)
-{
-	unsigned long paddr = handle;
-
-	BUG_ON(!valid_dma_direction(dir));
-
-	switch (dir) {
-	case DMA_FROM_DEVICE:
-		L2_cache_block_invalidate(paddr, paddr + size);
-		break;
-	case DMA_TO_DEVICE:
-		L2_cache_block_writeback(paddr, paddr + size);
-		break;
-	case DMA_BIDIRECTIONAL:
-		L2_cache_block_writeback_invalidate(paddr, paddr + size);
-		break;
-	default:
-		break;
-	}
-}
-
-static dma_addr_t c6x_dma_map_page(struct device *dev, struct page *page,
-		unsigned long offset, size_t size, enum dma_data_direction dir,
-		unsigned long attrs)
-{
-	dma_addr_t handle = virt_to_phys(page_address(page) + offset);
-
-	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
-		c6x_dma_sync(handle, size, dir);
-
-	return handle;
-}
-
-static void c6x_dma_unmap_page(struct device *dev, dma_addr_t handle,
-		size_t size, enum dma_data_direction dir, unsigned long attrs)
-{
-	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
-		c6x_dma_sync(handle, size, dir);
-}
-
-static int c6x_dma_map_sg(struct device *dev, struct scatterlist *sglist,
-		int nents, enum dma_data_direction dir, unsigned long attrs)
-{
-	struct scatterlist *sg;
-	int i;
-
-	for_each_sg(sglist, sg, nents, i) {
-		sg->dma_address = sg_phys(sg);
-		if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
-			c6x_dma_sync(sg->dma_address, sg->length, dir);
-	}
-
-	return nents;
-}
-
-static void c6x_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
-		  int nents, enum dma_data_direction dir, unsigned long attrs)
-{
-	struct scatterlist *sg;
-	int i;
-
-	if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
-		return;
-
-	for_each_sg(sglist, sg, nents, i)
-		c6x_dma_sync(sg_dma_address(sg), sg->length, dir);
-}
-
-static void c6x_dma_sync_single_for_cpu(struct device *dev, dma_addr_t handle,
-		size_t size, enum dma_data_direction dir)
-{
-	c6x_dma_sync(handle, size, dir);
-
-}
-
-static void c6x_dma_sync_single_for_device(struct device *dev,
-		dma_addr_t handle, size_t size, enum dma_data_direction dir)
-{
-	c6x_dma_sync(handle, size, dir);
-
-}
-
-static void c6x_dma_sync_sg_for_cpu(struct device *dev,
-		struct scatterlist *sglist, int nents,
-		enum dma_data_direction dir)
-{
-	struct scatterlist *sg;
-	int i;
-
-	for_each_sg(sglist, sg, nents, i)
-		c6x_dma_sync_single_for_cpu(dev, sg_dma_address(sg),
-					sg->length, dir);
-
-}
-
-static void c6x_dma_sync_sg_for_device(struct device *dev,
-		struct scatterlist *sglist, int nents,
-		enum dma_data_direction dir)
-{
-	struct scatterlist *sg;
-	int i;
-
-	for_each_sg(sglist, sg, nents, i)
-		c6x_dma_sync_single_for_device(dev, sg_dma_address(sg),
-					   sg->length, dir);
-
-}
-
-const struct dma_map_ops c6x_dma_ops = {
-	.alloc			= c6x_dma_alloc,
-	.free			= c6x_dma_free,
-	.map_page		= c6x_dma_map_page,
-	.unmap_page		= c6x_dma_unmap_page,
-	.map_sg			= c6x_dma_map_sg,
-	.unmap_sg		= c6x_dma_unmap_sg,
-	.sync_single_for_device	= c6x_dma_sync_single_for_device,
-	.sync_single_for_cpu	= c6x_dma_sync_single_for_cpu,
-	.sync_sg_for_device	= c6x_dma_sync_sg_for_device,
-	.sync_sg_for_cpu	= c6x_dma_sync_sg_for_cpu,
-};
-EXPORT_SYMBOL(c6x_dma_ops);
-
-/* Number of entries preallocated for DMA-API debugging */
-#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
-
-static int __init dma_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-
-	return 0;
-}
-fs_initcall(dma_init);
diff --git a/arch/c6x/kernel/traps.c b/arch/c6x/kernel/traps.c
index 4c1d4b84dd2b..5c60aea3b75a 100644
--- a/arch/c6x/kernel/traps.c
+++ b/arch/c6x/kernel/traps.c
@@ -244,7 +244,6 @@ static struct exception_info eexcept_table[128] = {
 static void do_trap(struct exception_info *except_info, struct pt_regs *regs)
 {
 	unsigned long addr = instruction_pointer(regs);
-	siginfo_t info;
 
 	if (except_info->code != TRAP_BRKPT)
 		pr_err("TRAP: %s PC[0x%lx] signo[%d] code[%d]\n",
@@ -253,12 +252,8 @@ static void do_trap(struct exception_info *except_info, struct pt_regs *regs)
 
 	die_if_kernel(except_info->kernel_str, regs, addr);
 
-	info.si_signo = except_info->signo;
-	info.si_errno = 0;
-	info.si_code  = except_info->code;
-	info.si_addr  = (void __user *)addr;
-
-	force_sig_info(except_info->signo, &info, current);
+	force_sig_fault(except_info->signo, except_info->code,
+			(void __user *)addr, current);
 }
 
 /*
diff --git a/arch/c6x/mm/dma-coherent.c b/arch/c6x/mm/dma-coherent.c
index 95e38ad27c69..d0a8e0c4b27e 100644
--- a/arch/c6x/mm/dma-coherent.c
+++ b/arch/c6x/mm/dma-coherent.c
@@ -19,10 +19,12 @@
 #include <linux/bitops.h>
 #include <linux/module.h>
 #include <linux/interrupt.h>
-#include <linux/dma-mapping.h>
+#include <linux/dma-noncoherent.h>
 #include <linux/memblock.h>
 
+#include <asm/cacheflush.h>
 #include <asm/page.h>
+#include <asm/setup.h>
 
 /*
  * DMA coherent memory management, can be redefined using the memdma=
@@ -73,7 +75,7 @@ static void __free_dma_pages(u32 addr, int order)
  * Allocate DMA coherent memory space and return both the kernel
  * virtual and DMA address for that space.
  */
-void *c6x_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
+void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 		gfp_t gfp, unsigned long attrs)
 {
 	u32 paddr;
@@ -98,7 +100,7 @@ void *c6x_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 /*
  * Free DMA coherent memory as defined by the above mapping.
  */
-void c6x_dma_free(struct device *dev, size_t size, void *vaddr,
+void arch_dma_free(struct device *dev, size_t size, void *vaddr,
 		dma_addr_t dma_handle, unsigned long attrs)
 {
 	int order;
@@ -139,3 +141,35 @@ void __init coherent_mem_init(phys_addr_t start, u32 size)
 	dma_bitmap = phys_to_virt(bitmap_phys);
 	memset(dma_bitmap, 0, dma_pages * PAGE_SIZE);
 }
+
+static void c6x_dma_sync(struct device *dev, phys_addr_t paddr, size_t size,
+		enum dma_data_direction dir)
+{
+	BUG_ON(!valid_dma_direction(dir));
+
+	switch (dir) {
+	case DMA_FROM_DEVICE:
+		L2_cache_block_invalidate(paddr, paddr + size);
+		break;
+	case DMA_TO_DEVICE:
+		L2_cache_block_writeback(paddr, paddr + size);
+		break;
+	case DMA_BIDIRECTIONAL:
+		L2_cache_block_writeback_invalidate(paddr, paddr + size);
+		break;
+	default:
+		break;
+	}
+}
+
+void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
+		size_t size, enum dma_data_direction dir)
+{
+	return c6x_dma_sync(dev, paddr, size, dir);
+}
+
+void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
+		size_t size, enum dma_data_direction dir)
+{
+	return c6x_dma_sync(dev, paddr, size, dir);
+}
diff --git a/arch/h8300/include/asm/Kbuild b/arch/h8300/include/asm/Kbuild
index 14bac06b7116..a5d0b2991f47 100644
--- a/arch/h8300/include/asm/Kbuild
+++ b/arch/h8300/include/asm/Kbuild
@@ -3,6 +3,7 @@ generic-y += barrier.h
 generic-y += bugs.h
 generic-y += cacheflush.h
 generic-y += checksum.h
+generic-y += compat.h
 generic-y += current.h
 generic-y += delay.h
 generic-y += device.h
diff --git a/arch/h8300/include/asm/pci.h b/arch/h8300/include/asm/pci.h
index 7c9e55d62215..d4d345a52092 100644
--- a/arch/h8300/include/asm/pci.h
+++ b/arch/h8300/include/asm/pci.h
@@ -15,6 +15,4 @@ static inline void pcibios_penalize_isa_irq(int irq, int active)
 	/* We don't do dynamic PCI IRQ allocation */
 }
 
-#define PCI_DMA_BUS_IS_PHYS	(1)
-
 #endif /* _ASM_H8300_PCI_H */
diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig
index 76d2f20d525e..37adb2003033 100644
--- a/arch/hexagon/Kconfig
+++ b/arch/hexagon/Kconfig
@@ -19,6 +19,7 @@ config HEXAGON
 	select GENERIC_IRQ_SHOW
 	select HAVE_ARCH_KGDB
 	select HAVE_ARCH_TRACEHOOK
+	select NEED_SG_DMA_LENGTH
 	select NO_IOPORT_MAP
 	select GENERIC_IOMAP
 	select GENERIC_SMP_IDLE_THREAD
@@ -63,9 +64,6 @@ config GENERIC_CSUM
 config GENERIC_IRQ_PROBE
 	def_bool y
 
-config NEED_SG_DMA_LENGTH
-	def_bool y
-
 config RWSEM_GENERIC_SPINLOCK
 	def_bool n
 
diff --git a/arch/hexagon/include/asm/Kbuild b/arch/hexagon/include/asm/Kbuild
index e9743f689fb8..dd2fd9c0d292 100644
--- a/arch/hexagon/include/asm/Kbuild
+++ b/arch/hexagon/include/asm/Kbuild
@@ -2,6 +2,7 @@
 generic-y += barrier.h
 generic-y += bug.h
 generic-y += bugs.h
+generic-y += compat.h
 generic-y += current.h
 generic-y += device.h
 generic-y += div64.h
diff --git a/arch/hexagon/include/asm/pgtable.h b/arch/hexagon/include/asm/pgtable.h
index aef02f7ca8aa..65125d0b02dd 100644
--- a/arch/hexagon/include/asm/pgtable.h
+++ b/arch/hexagon/include/asm/pgtable.h
@@ -30,7 +30,6 @@
 
 /* A handy thing to have if one has the RAM. Declared in head.S */
 extern unsigned long empty_zero_page;
-extern unsigned long zero_page_mask;
 
 /*
  * The PTE model described here is that of the Hexagon Virtual Machine,
diff --git a/arch/hexagon/kernel/dma.c b/arch/hexagon/kernel/dma.c
index ad8347c29dcf..77459df34e2e 100644
--- a/arch/hexagon/kernel/dma.c
+++ b/arch/hexagon/kernel/dma.c
@@ -208,7 +208,6 @@ const struct dma_map_ops hexagon_dma_ops = {
 	.sync_single_for_cpu = hexagon_sync_single_for_cpu,
 	.sync_single_for_device = hexagon_sync_single_for_device,
 	.mapping_error	= hexagon_mapping_error,
-	.is_phys	= 1,
 };
 
 void __init hexagon_dma_init(void)
diff --git a/arch/hexagon/kernel/setup.c b/arch/hexagon/kernel/setup.c
index 6981949f5df3..dc8c7e75b5d1 100644
--- a/arch/hexagon/kernel/setup.c
+++ b/arch/hexagon/kernel/setup.c
@@ -66,7 +66,7 @@ void __init setup_arch(char **cmdline_p)
 	 */
 	__vmsetvec(_K_VM_event_vector);
 
-	printk(KERN_INFO "PHYS_OFFSET=0x%08x\n", PHYS_OFFSET);
+	printk(KERN_INFO "PHYS_OFFSET=0x%08lx\n", PHYS_OFFSET);
 
 	/*
 	 * Simulator has a few differences from the hardware.
diff --git a/arch/hexagon/kernel/traps.c b/arch/hexagon/kernel/traps.c
index 2942a9204a9a..91ee04842c22 100644
--- a/arch/hexagon/kernel/traps.c
+++ b/arch/hexagon/kernel/traps.c
@@ -412,10 +412,6 @@ void do_trap0(struct pt_regs *regs)
 	case TRAP_DEBUG:
 		/* Trap0 0xdb is debug breakpoint */
 		if (user_mode(regs)) {
-			struct siginfo info;
-
-			info.si_signo = SIGTRAP;
-			info.si_errno = 0;
 			/*
 			 * Some architecures add some per-thread state
 			 * to distinguish between breakpoint traps and
@@ -423,9 +419,8 @@ void do_trap0(struct pt_regs *regs)
 			 * set the si_code value appropriately, or we
 			 * may want to use a different trap0 flavor.
 			 */
-			info.si_code = TRAP_BRKPT;
-			info.si_addr = (void __user *) pt_elr(regs);
-			force_sig_info(SIGTRAP, &info, current);
+			force_sig_fault(SIGTRAP, TRAP_BRKPT,
+					(void __user *) pt_elr(regs), current);
 		} else {
 #ifdef CONFIG_KGDB
 			kgdb_handle_exception(pt_cause(regs), SIGTRAP,
diff --git a/arch/hexagon/mm/init.c b/arch/hexagon/mm/init.c
index 192584d5ac2f..1495d45e472d 100644
--- a/arch/hexagon/mm/init.c
+++ b/arch/hexagon/mm/init.c
@@ -39,9 +39,6 @@ unsigned long __phys_offset;	/*  physical kernel offset >> 12  */
 /*  Set as variable to limit PMD copies  */
 int max_kernel_seg = 0x303;
 
-/*  think this should be (page_size-1) the way it's used...*/
-unsigned long zero_page_mask;
-
 /*  indicate pfn's of high memory  */
 unsigned long highstart_pfn, highend_pfn;
 
diff --git a/arch/hexagon/mm/vm_fault.c b/arch/hexagon/mm/vm_fault.c
index 3eec33c5cfd7..933bbcef5363 100644
--- a/arch/hexagon/mm/vm_fault.c
+++ b/arch/hexagon/mm/vm_fault.c
@@ -50,7 +50,7 @@ void do_page_fault(unsigned long address, long cause, struct pt_regs *regs)
 {
 	struct vm_area_struct *vma;
 	struct mm_struct *mm = current->mm;
-	siginfo_t info;
+	int si_signo;
 	int si_code = SEGV_MAPERR;
 	int fault;
 	const struct exception_table_entry *fixup;
@@ -140,28 +140,22 @@ void do_page_fault(unsigned long address, long cause, struct pt_regs *regs)
 	 * unable to fix up the page fault.
 	 */
 	if (fault & VM_FAULT_SIGBUS) {
-		info.si_signo = SIGBUS;
-		info.si_code = BUS_ADRERR;
+		si_signo = SIGBUS;
+		si_code = BUS_ADRERR;
 	}
 	/* Address is not in the memory map */
 	else {
-		info.si_signo = SIGSEGV;
-		info.si_code = SEGV_ACCERR;
+		si_signo = SIGSEGV;
+		si_code  = SEGV_ACCERR;
 	}
-	info.si_errno = 0;
-	info.si_addr = (void __user *)address;
-	force_sig_info(info.si_signo, &info, current);
+	force_sig_fault(si_signo, si_code, (void __user *)address, current);
 	return;
 
 bad_area:
 	up_read(&mm->mmap_sem);
 
 	if (user_mode(regs)) {
-		info.si_signo = SIGSEGV;
-		info.si_errno = 0;
-		info.si_code = si_code;
-		info.si_addr = (void *)address;
-		force_sig_info(info.si_signo, &info, current);
+		force_sig_fault(SIGSEGV, si_code, (void __user *)address, current);
 		return;
 	}
 	/* Kernel-mode fault falls through */
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index bbe12a038d21..792437d526c6 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -29,7 +29,6 @@ config IA64
 	select HAVE_FUNCTION_TRACER
 	select TTY
 	select HAVE_ARCH_TRACEHOOK
-	select HAVE_DMA_API_DEBUG
 	select HAVE_MEMBLOCK
 	select HAVE_MEMBLOCK_NODE_MAP
 	select HAVE_VIRT_CPU_ACCOUNTING
@@ -54,6 +53,8 @@ config IA64
 	select MODULES_USE_ELF_RELA
 	select ARCH_USE_CMPXCHG_LOCKREF
 	select HAVE_ARCH_AUDITSYSCALL
+	select NEED_DMA_MAP_STATE
+	select NEED_SG_DMA_LENGTH
 	default y
 	help
 	  The Itanium Processor Family is Intel's 64-bit successor to
@@ -78,18 +79,6 @@ config MMU
 	bool
 	default y
 
-config ARCH_DMA_ADDR_T_64BIT
-	def_bool y
-
-config NEED_DMA_MAP_STATE
-	def_bool y
-
-config NEED_SG_DMA_LENGTH
-	def_bool y
-
-config SWIOTLB
-       bool
-
 config STACKTRACE_SUPPORT
 	def_bool y
 
@@ -146,7 +135,6 @@ config IA64_GENERIC
 	bool "generic"
 	select NUMA
 	select ACPI_NUMA
-	select DMA_DIRECT_OPS
 	select SWIOTLB
 	select PCI_MSI
 	help
@@ -167,7 +155,6 @@ config IA64_GENERIC
 
 config IA64_DIG
 	bool "DIG-compliant"
-	select DMA_DIRECT_OPS
 	select SWIOTLB
 
 config IA64_DIG_VTD
@@ -183,7 +170,6 @@ config IA64_HP_ZX1
 
 config IA64_HP_ZX1_SWIOTLB
 	bool "HP-zx1/sx1000 with software I/O TLB"
-	select DMA_DIRECT_OPS
 	select SWIOTLB
 	help
 	  Build a kernel that runs on HP zx1 and sx1000 systems even when they
@@ -207,7 +193,6 @@ config IA64_SGI_UV
 	bool "SGI-UV"
 	select NUMA
 	select ACPI_NUMA
-	select DMA_DIRECT_OPS
 	select SWIOTLB
 	help
 	  Selecting this option will optimize the kernel for use on UV based
@@ -218,7 +203,6 @@ config IA64_SGI_UV
 
 config IA64_HP_SIM
 	bool "Ski-simulator"
-	select DMA_DIRECT_OPS
 	select SWIOTLB
 	depends on !PM
 
@@ -397,7 +381,7 @@ config ARCH_DISCONTIGMEM_ENABLE
 	  Say Y to support efficient handling of discontiguous physical memory,
 	  for architectures which are either NUMA (Non-Uniform Memory Access)
 	  or have huge holes in the physical address space for other reasons.
- 	  See <file:Documentation/vm/numa> for more.
+	  See <file:Documentation/vm/numa.rst> for more.
 
 config ARCH_FLATMEM_ENABLE
 	def_bool y
@@ -613,6 +597,3 @@ source "security/Kconfig"
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
-
-config IOMMU_HELPER
-	def_bool (IA64_HP_ZX1 || IA64_HP_ZX1_SWIOTLB || IA64_GENERIC || SWIOTLB)
diff --git a/arch/ia64/hp/common/sba_iommu.c b/arch/ia64/hp/common/sba_iommu.c
index aec4a3354abe..ee5b652d320a 100644
--- a/arch/ia64/hp/common/sba_iommu.c
+++ b/arch/ia64/hp/common/sba_iommu.c
@@ -1845,9 +1845,6 @@ static void ioc_init(unsigned long hpa, struct ioc *ioc)
 	ioc_resource_init(ioc);
 	ioc_sac_init(ioc);
 
-	if ((long) ~iovp_mask > (long) ia64_max_iommu_merge_mask)
-		ia64_max_iommu_merge_mask = ~iovp_mask;
-
 	printk(KERN_INFO PFX
 		"%s %d.%d HPA 0x%lx IOVA space %dMb at 0x%lx\n",
 		ioc->name, (ioc->rev >> 4) & 0xF, ioc->rev & 0xF,
@@ -1942,19 +1939,6 @@ static const struct seq_operations ioc_seq_ops = {
 	.show  = ioc_show
 };
 
-static int
-ioc_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &ioc_seq_ops);
-}
-
-static const struct file_operations ioc_fops = {
-	.open    = ioc_open,
-	.read    = seq_read,
-	.llseek  = seq_lseek,
-	.release = seq_release
-};
-
 static void __init
 ioc_proc_init(void)
 {
@@ -1964,7 +1948,7 @@ ioc_proc_init(void)
 	if (!dir)
 		return;
 
-	proc_create(ioc_list->name, 0, dir, &ioc_fops);
+	proc_create_seq(ioc_list->name, 0, dir, &ioc_seq_ops);
 }
 #endif
 
diff --git a/arch/ia64/hp/sim/simserial.c b/arch/ia64/hp/sim/simserial.c
index a419ccf33cde..663388a73d4e 100644
--- a/arch/ia64/hp/sim/simserial.c
+++ b/arch/ia64/hp/sim/simserial.c
@@ -435,19 +435,6 @@ static int rs_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int rs_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, rs_proc_show, NULL);
-}
-
-static const struct file_operations rs_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= rs_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static const struct tty_operations hp_ops = {
 	.open = rs_open,
 	.close = rs_close,
@@ -462,7 +449,7 @@ static const struct tty_operations hp_ops = {
 	.unthrottle = rs_unthrottle,
 	.send_xchar = rs_send_xchar,
 	.hangup = rs_hangup,
-	.proc_fops = &rs_proc_fops,
+	.proc_show = rs_proc_show,
 };
 
 static const struct tty_port_operations hp_port_ops = {
diff --git a/arch/ia64/include/asm/Kbuild b/arch/ia64/include/asm/Kbuild
index 6dd867873364..557bbc8ba9f5 100644
--- a/arch/ia64/include/asm/Kbuild
+++ b/arch/ia64/include/asm/Kbuild
@@ -1,3 +1,4 @@
+generic-y += compat.h
 generic-y += exec.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
diff --git a/arch/ia64/include/asm/hardirq.h b/arch/ia64/include/asm/hardirq.h
index bdc4669c71c3..ccde7c2ba00f 100644
--- a/arch/ia64/include/asm/hardirq.h
+++ b/arch/ia64/include/asm/hardirq.h
@@ -13,7 +13,7 @@
 
 #define __ARCH_IRQ_STAT	1
 
-#define local_softirq_pending()		(local_cpu_data->softirq_pending)
+#define local_softirq_pending_ref	ia64_cpu_info.softirq_pending
 
 #include <linux/threads.h>
 #include <linux/irq.h>
diff --git a/arch/ia64/include/asm/pci.h b/arch/ia64/include/asm/pci.h
index b1d04e8bafc8..780e8744ba85 100644
--- a/arch/ia64/include/asm/pci.h
+++ b/arch/ia64/include/asm/pci.h
@@ -30,23 +30,6 @@ struct pci_vector_struct {
 #define PCIBIOS_MIN_IO		0x1000
 #define PCIBIOS_MIN_MEM		0x10000000
 
-/*
- * PCI_DMA_BUS_IS_PHYS should be set to 1 if there is _necessarily_ a direct
- * correspondence between device bus addresses and CPU physical addresses.
- * Platforms with a hardware I/O MMU _must_ turn this off to suppress the
- * bounce buffer handling code in the block and network device layers.
- * Platforms with separate bus address spaces _must_ turn this off and provide
- * a device DMA mapping implementation that takes care of the necessary
- * address translation.
- *
- * For now, the ia64 platforms which may have separate/multiple bus address
- * spaces all have I/O MMUs which support the merging of physically
- * discontiguous buffers, so we can use that as the sole factor to determine
- * the setting of PCI_DMA_BUS_IS_PHYS.
- */
-extern unsigned long ia64_max_iommu_merge_mask;
-#define PCI_DMA_BUS_IS_PHYS	(ia64_max_iommu_merge_mask == ~0UL)
-
 #define HAVE_PCI_MMAP
 #define ARCH_GENERIC_PCI_MMAP_RESOURCE
 #define arch_can_pci_mmap_wc()	1
diff --git a/arch/ia64/include/uapi/asm/Kbuild b/arch/ia64/include/uapi/asm/Kbuild
index c0527cfc48f0..3982e673e967 100644
--- a/arch/ia64/include/uapi/asm/Kbuild
+++ b/arch/ia64/include/uapi/asm/Kbuild
@@ -2,5 +2,9 @@
 include include/uapi/asm-generic/Kbuild.asm
 
 generic-y += bpf_perf_event.h
+generic-y += ipcbuf.h
 generic-y += kvm_para.h
+generic-y += msgbuf.h
 generic-y += poll.h
+generic-y += sembuf.h
+generic-y += shmbuf.h
diff --git a/arch/ia64/include/uapi/asm/ipcbuf.h b/arch/ia64/include/uapi/asm/ipcbuf.h
deleted file mode 100644
index 90d6445a14df..000000000000
--- a/arch/ia64/include/uapi/asm/ipcbuf.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#include <asm-generic/ipcbuf.h>
diff --git a/arch/ia64/include/uapi/asm/msgbuf.h b/arch/ia64/include/uapi/asm/msgbuf.h
deleted file mode 100644
index aa25df92d9dc..000000000000
--- a/arch/ia64/include/uapi/asm/msgbuf.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _ASM_IA64_MSGBUF_H
-#define _ASM_IA64_MSGBUF_H
-
-/*
- * The msqid64_ds structure for IA-64 architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space is left for:
- * - 2 miscellaneous 64-bit values
- */
-
-struct msqid64_ds {
-	struct ipc64_perm msg_perm;
-	__kernel_time_t msg_stime;	/* last msgsnd time */
-	__kernel_time_t msg_rtime;	/* last msgrcv time */
-	__kernel_time_t msg_ctime;	/* last change time */
-	unsigned long  msg_cbytes;	/* current number of bytes on queue */
-	unsigned long  msg_qnum;	/* number of messages in queue */
-	unsigned long  msg_qbytes;	/* max number of bytes on queue */
-	__kernel_pid_t msg_lspid;	/* pid of last msgsnd */
-	__kernel_pid_t msg_lrpid;	/* last receive pid */
-	unsigned long  __unused1;
-	unsigned long  __unused2;
-};
-
-#endif /* _ASM_IA64_MSGBUF_H */
diff --git a/arch/ia64/include/uapi/asm/sembuf.h b/arch/ia64/include/uapi/asm/sembuf.h
deleted file mode 100644
index 6ed058760afc..000000000000
--- a/arch/ia64/include/uapi/asm/sembuf.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _ASM_IA64_SEMBUF_H
-#define _ASM_IA64_SEMBUF_H
-
-/*
- * The semid64_ds structure for IA-64 architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space is left for:
- * - 2 miscellaneous 64-bit values
- */
-
-struct semid64_ds {
-	struct ipc64_perm sem_perm;		/* permissions .. see ipc.h */
-	__kernel_time_t	sem_otime;		/* last semop time */
-	__kernel_time_t	sem_ctime;		/* last change time */
-	unsigned long	sem_nsems;		/* no. of semaphores in array */
-	unsigned long	__unused1;
-	unsigned long	__unused2;
-};
-
-#endif /* _ASM_IA64_SEMBUF_H */
diff --git a/arch/ia64/include/uapi/asm/shmbuf.h b/arch/ia64/include/uapi/asm/shmbuf.h
deleted file mode 100644
index 6ef57cb70dee..000000000000
--- a/arch/ia64/include/uapi/asm/shmbuf.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _ASM_IA64_SHMBUF_H
-#define _ASM_IA64_SHMBUF_H
-
-/*
- * The shmid64_ds structure for IA-64 architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space is left for:
- * - 2 miscellaneous 64-bit values
- */
-
-struct shmid64_ds {
-	struct ipc64_perm	shm_perm;	/* operation perms */
-	size_t			shm_segsz;	/* size of segment (bytes) */
-	__kernel_time_t		shm_atime;	/* last attach time */
-	__kernel_time_t		shm_dtime;	/* last detach time */
-	__kernel_time_t		shm_ctime;	/* last change time */
-	__kernel_pid_t		shm_cpid;	/* pid of creator */
-	__kernel_pid_t		shm_lpid;	/* pid of last operator */
-	unsigned long		shm_nattch;	/* no. of current attaches */
-	unsigned long		__unused1;
-	unsigned long		__unused2;
-};
-
-struct shminfo64 {
-	unsigned long	shmmax;
-	unsigned long	shmmin;
-	unsigned long	shmmni;
-	unsigned long	shmseg;
-	unsigned long	shmall;
-	unsigned long	__unused1;
-	unsigned long	__unused2;
-	unsigned long	__unused3;
-	unsigned long	__unused4;
-};
-
-#endif /* _ASM_IA64_SHMBUF_H */
diff --git a/arch/ia64/include/uapi/asm/siginfo.h b/arch/ia64/include/uapi/asm/siginfo.h
index 5aa454ed89db..52b5af424511 100644
--- a/arch/ia64/include/uapi/asm/siginfo.h
+++ b/arch/ia64/include/uapi/asm/siginfo.h
@@ -27,11 +27,4 @@
 #define __ISR_VALID_BIT	0
 #define __ISR_VALID	(1 << __ISR_VALID_BIT)
 
-/*
- * SIGFPE si_codes
- */
-#ifdef __KERNEL__
-#define FPE_FIXME	0	/* Broken dup of SI_USER */
-#endif /* __KERNEL__ */
-
 #endif /* _UAPI_ASM_IA64_SIGINFO_H */
diff --git a/arch/ia64/kernel/brl_emu.c b/arch/ia64/kernel/brl_emu.c
index 9bcc908bc85e..a61f6c6a36f8 100644
--- a/arch/ia64/kernel/brl_emu.c
+++ b/arch/ia64/kernel/brl_emu.c
@@ -62,6 +62,7 @@ ia64_emulate_brl (struct pt_regs *regs, unsigned long ar_ec)
 	struct illegal_op_return rv;
 	long tmp_taken, unimplemented_address;
 
+	clear_siginfo(&siginfo);
 	rv.fkt = (unsigned long) -1;
 
 	/*
diff --git a/arch/ia64/kernel/dma-mapping.c b/arch/ia64/kernel/dma-mapping.c
index f2d57e66fd86..7a471d8d67d4 100644
--- a/arch/ia64/kernel/dma-mapping.c
+++ b/arch/ia64/kernel/dma-mapping.c
@@ -9,16 +9,6 @@ int iommu_detected __read_mostly;
 const struct dma_map_ops *dma_ops;
 EXPORT_SYMBOL(dma_ops);
 
-#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
-
-static int __init dma_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-
-	return 0;
-}
-fs_initcall(dma_init);
-
 const struct dma_map_ops *dma_get_ops(struct device *dev)
 {
 	return dma_ops;
diff --git a/arch/ia64/kernel/palinfo.c b/arch/ia64/kernel/palinfo.c
index b6e597860888..f4a94241265c 100644
--- a/arch/ia64/kernel/palinfo.c
+++ b/arch/ia64/kernel/palinfo.c
@@ -920,18 +920,6 @@ static int proc_palinfo_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int proc_palinfo_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_palinfo_show, PDE_DATA(inode));
-}
-
-static const struct file_operations proc_palinfo_fops = {
-	.open		= proc_palinfo_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int palinfo_add_proc(unsigned int cpu)
 {
 	pal_func_cpu_u_t f;
@@ -948,8 +936,8 @@ static int palinfo_add_proc(unsigned int cpu)
 
 	for (j=0; j < NR_PALINFO_ENTRIES; j++) {
 		f.func_id = j;
-		proc_create_data(palinfo_entries[j].name, 0, cpu_dir,
-				 &proc_palinfo_fops, (void *)f.value);
+		proc_create_single_data(palinfo_entries[j].name, 0, cpu_dir,
+				proc_palinfo_show, (void *)f.value);
 	}
 	return 0;
 }
diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c
index 8fb280e33114..3b38c717008a 100644
--- a/arch/ia64/kernel/perfmon.c
+++ b/arch/ia64/kernel/perfmon.c
@@ -5708,13 +5708,6 @@ const struct seq_operations pfm_seq_ops = {
  	.show =		pfm_proc_show
 };
 
-static int
-pfm_proc_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &pfm_seq_ops);
-}
-
-
 /*
  * we come here as soon as local_cpu_data->pfm_syst_wide is set. this happens
  * during pfm_enable() hence before pfm_start(). We cannot assume monitoring
@@ -6537,13 +6530,6 @@ pfm_probe_pmu(void)
 	return 0;
 }
 
-static const struct file_operations pfm_proc_fops = {
-	.open		= pfm_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
 int __init
 pfm_init(void)
 {
@@ -6615,7 +6601,7 @@ pfm_init(void)
 	/*
 	 * create /proc/perfmon (mostly for debugging purposes)
 	 */
-	perfmon_dir = proc_create("perfmon", S_IRUGO, NULL, &pfm_proc_fops);
+	perfmon_dir = proc_create_seq("perfmon", S_IRUGO, NULL, &pfm_seq_ops);
 	if (perfmon_dir == NULL) {
 		printk(KERN_ERR "perfmon: cannot create /proc entry, perfmon disabled\n");
 		pmu_conf = NULL;
diff --git a/arch/ia64/kernel/salinfo.c b/arch/ia64/kernel/salinfo.c
index 52c404b08904..aba1f463a8dd 100644
--- a/arch/ia64/kernel/salinfo.c
+++ b/arch/ia64/kernel/salinfo.c
@@ -54,8 +54,6 @@ MODULE_AUTHOR("Jesse Barnes <jbarnes@sgi.com>");
 MODULE_DESCRIPTION("/proc interface to IA-64 SAL features");
 MODULE_LICENSE("GPL");
 
-static const struct file_operations proc_salinfo_fops;
-
 typedef struct {
 	const char		*name;		/* name of the proc entry */
 	unsigned long           feature;        /* feature bit */
@@ -578,6 +576,17 @@ static int salinfo_cpu_pre_down(unsigned int cpu)
 	return 0;
 }
 
+/*
+ * 'data' contains an integer that corresponds to the feature we're
+ * testing
+ */
+static int proc_salinfo_show(struct seq_file *m, void *v)
+{
+	unsigned long data = (unsigned long)v;
+	seq_puts(m, (sal_platform_features & data) ? "1\n" : "0\n");
+	return 0;
+}
+
 static int __init
 salinfo_init(void)
 {
@@ -593,9 +602,9 @@ salinfo_init(void)
 
 	for (i=0; i < NR_SALINFO_ENTRIES; i++) {
 		/* pass the feature bit in question as misc data */
-		*sdir++ = proc_create_data(salinfo_entries[i].name, 0, salinfo_dir,
-					   &proc_salinfo_fops,
-					   (void *)salinfo_entries[i].feature);
+		*sdir++ = proc_create_single_data(salinfo_entries[i].name, 0,
+				salinfo_dir, proc_salinfo_show,
+				(void *)salinfo_entries[i].feature);
 	}
 
 	for (i = 0; i < ARRAY_SIZE(salinfo_log_name); i++) {
@@ -633,27 +642,4 @@ salinfo_init(void)
 	return 0;
 }
 
-/*
- * 'data' contains an integer that corresponds to the feature we're
- * testing
- */
-static int proc_salinfo_show(struct seq_file *m, void *v)
-{
-	unsigned long data = (unsigned long)v;
-	seq_puts(m, (sal_platform_features & data) ? "1\n" : "0\n");
-	return 0;
-}
-
-static int proc_salinfo_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_salinfo_show, PDE_DATA(inode));
-}
-
-static const struct file_operations proc_salinfo_fops = {
-	.open		= proc_salinfo_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 module_init(salinfo_init);
diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c
index dee56bcb993d..ad43cbf70628 100644
--- a/arch/ia64/kernel/setup.c
+++ b/arch/ia64/kernel/setup.c
@@ -123,18 +123,6 @@ unsigned long ia64_i_cache_stride_shift = ~0;
 #define	CACHE_STRIDE_SHIFT	5
 unsigned long ia64_cache_stride_shift = ~0;
 
-/*
- * The merge_mask variable needs to be set to (max(iommu_page_size(iommu)) - 1).  This
- * mask specifies a mask of address bits that must be 0 in order for two buffers to be
- * mergeable by the I/O MMU (i.e., the end address of the first buffer and the start
- * address of the second buffer must be aligned to (merge_mask+1) in order to be
- * mergeable).  By default, we assume there is no I/O MMU which can merge physically
- * discontiguous buffers, so we set the merge_mask to ~0UL, which corresponds to a iommu
- * page-size of 2^64.
- */
-unsigned long ia64_max_iommu_merge_mask = ~0UL;
-EXPORT_SYMBOL(ia64_max_iommu_merge_mask);
-
 /*
  * We use a special marker for the end of memory and it uses the extra (+1) slot
  */
diff --git a/arch/ia64/kernel/signal.c b/arch/ia64/kernel/signal.c
index 54547c7cf8a2..d1234a5ba4c5 100644
--- a/arch/ia64/kernel/signal.c
+++ b/arch/ia64/kernel/signal.c
@@ -153,6 +153,7 @@ ia64_rt_sigreturn (struct sigscratch *scr)
 	return retval;
 
   give_sigsegv:
+	clear_siginfo(&si);
 	si.si_signo = SIGSEGV;
 	si.si_errno = 0;
 	si.si_code = SI_KERNEL;
@@ -236,6 +237,7 @@ force_sigsegv_info (int sig, void __user *addr)
 	unsigned long flags;
 	struct siginfo si;
 
+	clear_siginfo(&si);
 	if (sig == SIGSEGV) {
 		/*
 		 * Acquiring siglock around the sa_handler-update is almost
diff --git a/arch/ia64/kernel/traps.c b/arch/ia64/kernel/traps.c
index 6d4e76a4267f..c6f4932073a1 100644
--- a/arch/ia64/kernel/traps.c
+++ b/arch/ia64/kernel/traps.c
@@ -104,6 +104,7 @@ __kprobes ia64_bad_break (unsigned long break_num, struct pt_regs *regs)
 	int sig, code;
 
 	/* SIGILL, SIGFPE, SIGSEGV, and SIGBUS want these field initialized: */
+	clear_siginfo(&siginfo);
 	siginfo.si_addr = (void __user *) (regs->cr_iip + ia64_psr(regs)->ri);
 	siginfo.si_imm = break_num;
 	siginfo.si_flags = 0;		/* clear __ISR_VALID */
@@ -293,7 +294,6 @@ handle_fpu_swa (int fp_fault, struct pt_regs *regs, unsigned long isr)
 {
 	long exception, bundle[2];
 	unsigned long fault_ip;
-	struct siginfo siginfo;
 
 	fault_ip = regs->cr_iip;
 	if (!fp_fault && (ia64_psr(regs)->ri == 0))
@@ -344,13 +344,16 @@ handle_fpu_swa (int fp_fault, struct pt_regs *regs, unsigned long isr)
 			printk(KERN_ERR "handle_fpu_swa: fp_emulate() returned -1\n");
 			return -1;
 		} else {
+			struct siginfo siginfo;
+
 			/* is next instruction a trap? */
 			if (exception & 2) {
 				ia64_increment_ip(regs);
 			}
+			clear_siginfo(&siginfo);
 			siginfo.si_signo = SIGFPE;
 			siginfo.si_errno = 0;
-			siginfo.si_code = FPE_FIXME;	/* default code */
+			siginfo.si_code = FPE_FLTUNK;	/* default code */
 			siginfo.si_addr = (void __user *) (regs->cr_iip + ia64_psr(regs)->ri);
 			if (isr & 0x11) {
 				siginfo.si_code = FPE_FLTINV;
@@ -372,9 +375,12 @@ handle_fpu_swa (int fp_fault, struct pt_regs *regs, unsigned long isr)
 			return -1;
 		} else if (exception != 0) {
 			/* raise exception */
+			struct siginfo siginfo;
+
+			clear_siginfo(&siginfo);
 			siginfo.si_signo = SIGFPE;
 			siginfo.si_errno = 0;
-			siginfo.si_code = FPE_FIXME;	/* default code */
+			siginfo.si_code = FPE_FLTUNK;	/* default code */
 			siginfo.si_addr = (void __user *) (regs->cr_iip + ia64_psr(regs)->ri);
 			if (isr & 0x880) {
 				siginfo.si_code = FPE_FLTOVF;
@@ -420,7 +426,7 @@ ia64_illegal_op_fault (unsigned long ec, long arg1, long arg2, long arg3,
 	if (die_if_kernel(buf, &regs, 0))
 		return rv;
 
-	memset(&si, 0, sizeof(si));
+	clear_siginfo(&si);
 	si.si_signo = SIGILL;
 	si.si_code = ILL_ILLOPC;
 	si.si_addr = (void __user *) (regs.cr_iip + ia64_psr(&regs)->ri);
@@ -434,7 +440,6 @@ ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa,
 	    long arg7, struct pt_regs regs)
 {
 	unsigned long code, error = isr, iip;
-	struct siginfo siginfo;
 	char buf[128];
 	int result, sig;
 	static const char *reason[] = {
@@ -485,6 +490,7 @@ ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa,
 
 	      case 26: /* NaT Consumption */
 		if (user_mode(&regs)) {
+			struct siginfo siginfo;
 			void __user *addr;
 
 			if (((isr >> 4) & 0xf) == 2) {
@@ -499,6 +505,7 @@ ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa,
 				addr = (void __user *) (regs.cr_iip
 							+ ia64_psr(&regs)->ri);
 			}
+			clear_siginfo(&siginfo);
 			siginfo.si_signo = sig;
 			siginfo.si_code = code;
 			siginfo.si_errno = 0;
@@ -515,6 +522,9 @@ ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa,
 
 	      case 31: /* Unsupported Data Reference */
 		if (user_mode(&regs)) {
+			struct siginfo siginfo;
+
+			clear_siginfo(&siginfo);
 			siginfo.si_signo = SIGILL;
 			siginfo.si_code = ILL_ILLOPN;
 			siginfo.si_errno = 0;
@@ -531,6 +541,10 @@ ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa,
 	      case 29: /* Debug */
 	      case 35: /* Taken Branch Trap */
 	      case 36: /* Single Step Trap */
+	      {
+		struct siginfo siginfo;
+
+		clear_siginfo(&siginfo);
 		if (fsys_mode(current, &regs)) {
 			extern char __kernel_syscall_via_break[];
 			/*
@@ -578,11 +592,15 @@ ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa,
 		siginfo.si_isr   = isr;
 		force_sig_info(SIGTRAP, &siginfo, current);
 		return;
+	      }
 
 	      case 32: /* fp fault */
 	      case 33: /* fp trap */
 		result = handle_fpu_swa((vector == 32) ? 1 : 0, &regs, isr);
 		if ((result < 0) || (current->thread.flags & IA64_THREAD_FPEMU_SIGFPE)) {
+			struct siginfo siginfo;
+
+			clear_siginfo(&siginfo);
 			siginfo.si_signo = SIGFPE;
 			siginfo.si_errno = 0;
 			siginfo.si_code = FPE_FLTINV;
@@ -616,6 +634,9 @@ ia64_fault (unsigned long vector, unsigned long isr, unsigned long ifa,
 		} else {
 			/* Unimplemented Instr. Address Trap */
 			if (user_mode(&regs)) {
+				struct siginfo siginfo;
+
+				clear_siginfo(&siginfo);
 				siginfo.si_signo = SIGILL;
 				siginfo.si_code = ILL_BADIADDR;
 				siginfo.si_errno = 0;
diff --git a/arch/ia64/kernel/unaligned.c b/arch/ia64/kernel/unaligned.c
index 72e9b4242564..e309f9859acc 100644
--- a/arch/ia64/kernel/unaligned.c
+++ b/arch/ia64/kernel/unaligned.c
@@ -1537,6 +1537,7 @@ ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs)
 		/* NOT_REACHED */
 	}
   force_sigbus:
+	clear_siginfo(&si);
 	si.si_signo = SIGBUS;
 	si.si_errno = 0;
 	si.si_code = BUS_ADRALN;
diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c
index dfdc152d6737..817fa120645f 100644
--- a/arch/ia64/mm/fault.c
+++ b/arch/ia64/mm/fault.c
@@ -85,7 +85,6 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re
 	int signal = SIGSEGV, code = SEGV_MAPERR;
 	struct vm_area_struct *vma, *prev_vma;
 	struct mm_struct *mm = current->mm;
-	struct siginfo si;
 	unsigned long mask;
 	int fault;
 	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
@@ -249,6 +248,9 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re
 		return;
 	}
 	if (user_mode(regs)) {
+		struct siginfo si;
+
+		clear_siginfo(&si);
 		si.si_signo = signal;
 		si.si_errno = 0;
 		si.si_code = code;
diff --git a/arch/ia64/sn/kernel/io_common.c b/arch/ia64/sn/kernel/io_common.c
index 11f2275570fb..8479e9a7ce16 100644
--- a/arch/ia64/sn/kernel/io_common.c
+++ b/arch/ia64/sn/kernel/io_common.c
@@ -480,11 +480,6 @@ sn_io_early_init(void)
 	tioca_init_provider();
 	tioce_init_provider();
 
-	/*
-	 * This is needed to avoid bounce limit checks in the blk layer
-	 */
-	ia64_max_iommu_merge_mask = ~PAGE_MASK;
-
 	sn_irq_lh_init();
 	INIT_LIST_HEAD(&sn_sysdata_list);
 	sn_init_cpei_timer();
diff --git a/arch/ia64/sn/kernel/sn2/prominfo_proc.c b/arch/ia64/sn/kernel/sn2/prominfo_proc.c
index ec4de2b09653..e15457bf21ac 100644
--- a/arch/ia64/sn/kernel/sn2/prominfo_proc.c
+++ b/arch/ia64/sn/kernel/sn2/prominfo_proc.c
@@ -140,18 +140,6 @@ static int proc_fit_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int proc_fit_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_fit_show, PDE_DATA(inode));
-}
-
-static const struct file_operations proc_fit_fops = {
-	.open		= proc_fit_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int proc_version_show(struct seq_file *m, void *v)
 {
 	unsigned long nasid = (unsigned long)m->private;
@@ -174,18 +162,6 @@ static int proc_version_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int proc_version_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_version_show, PDE_DATA(inode));
-}
-
-static const struct file_operations proc_version_fops = {
-	.open		= proc_version_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 /* module entry points */
 int __init prominfo_init(void);
 void __exit prominfo_exit(void);
@@ -217,10 +193,10 @@ int __init prominfo_init(void)
 		if (!dir)
 			continue;
 		nasid = cnodeid_to_nasid(cnodeid);
-		proc_create_data("fit", 0, dir,
-				 &proc_fit_fops, (void *)nasid);
-		proc_create_data("version", 0, dir,
-				 &proc_version_fops, (void *)nasid);
+		proc_create_single_data("fit", 0, dir, proc_fit_show, 
+				(void *)nasid);
+		proc_create_single_data("version", 0, dir, proc_version_show,
+				(void *)nasid);
 	}
 	return 0;
 }
diff --git a/arch/ia64/sn/kernel/sn2/sn_proc_fs.c b/arch/ia64/sn/kernel/sn2/sn_proc_fs.c
index 29cf8f8c08e9..c2a4d84297b0 100644
--- a/arch/ia64/sn/kernel/sn2/sn_proc_fs.c
+++ b/arch/ia64/sn/kernel/sn2/sn_proc_fs.c
@@ -18,33 +18,18 @@ static int partition_id_show(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int partition_id_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, partition_id_show, NULL);
-}
-
 static int system_serial_number_show(struct seq_file *s, void *p)
 {
 	seq_printf(s, "%s\n", sn_system_serial_number());
 	return 0;
 }
 
-static int system_serial_number_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, system_serial_number_show, NULL);
-}
-
 static int licenseID_show(struct seq_file *s, void *p)
 {
 	seq_printf(s, "0x%llx\n", sn_partition_serial_number_val());
 	return 0;
 }
 
-static int licenseID_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, licenseID_show, NULL);
-}
-
 static int coherence_id_show(struct seq_file *s, void *p)
 {
 	seq_printf(s, "%d\n", partition_coherence_id());
@@ -52,43 +37,10 @@ static int coherence_id_show(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int coherence_id_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, coherence_id_show, NULL);
-}
-
 /* /proc/sgi_sn/sn_topology uses seq_file, see sn_hwperf.c */
 extern int sn_topology_open(struct inode *, struct file *);
 extern int sn_topology_release(struct inode *, struct file *);
 
-static const struct file_operations proc_partition_id_fops = {
-	.open		= partition_id_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-static const struct file_operations proc_system_sn_fops = {
-	.open		= system_serial_number_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-static const struct file_operations proc_license_id_fops = {
-	.open		= licenseID_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-static const struct file_operations proc_coherence_id_fops = {
-	.open		= coherence_id_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static const struct file_operations proc_sn_topo_fops = {
 	.open		= sn_topology_open,
 	.read		= seq_read,
@@ -104,13 +56,13 @@ void register_sn_procfs(void)
 	if (!(sgi_proc_dir = proc_mkdir("sgi_sn", NULL)))
 		return;
 
-	proc_create("partition_id", 0444, sgi_proc_dir,
-		    &proc_partition_id_fops);
-	proc_create("system_serial_number", 0444, sgi_proc_dir,
-		    &proc_system_sn_fops);
-	proc_create("licenseID", 0444, sgi_proc_dir, &proc_license_id_fops);
-	proc_create("coherence_id", 0444, sgi_proc_dir,
-		    &proc_coherence_id_fops);
+	proc_create_single("partition_id", 0444, sgi_proc_dir,
+			partition_id_show);
+	proc_create_single("system_serial_number", 0444, sgi_proc_dir,
+			system_serial_number_show);
+	proc_create_single("licenseID", 0444, sgi_proc_dir, licenseID_show);
+	proc_create_single("coherence_id", 0444, sgi_proc_dir,
+			coherence_id_show);
 	proc_create("sn_topology", 0444, sgi_proc_dir, &proc_sn_topo_fops);
 }
 
diff --git a/arch/m68k/68000/timers.c b/arch/m68k/68000/timers.c
index 252455bce144..71ddb4c98726 100644
--- a/arch/m68k/68000/timers.c
+++ b/arch/m68k/68000/timers.c
@@ -125,7 +125,9 @@ int m68328_hwclk(int set, struct rtc_time *t)
 {
 	if (!set) {
 		long now = RTCTIME;
-		t->tm_year = t->tm_mon = t->tm_mday = 1;
+		t->tm_year = 1;
+		t->tm_mon = 0;
+		t->tm_mday = 1;
 		t->tm_hour = (now >> 24) % 24;
 		t->tm_min = (now >> 16) % 60;
 		t->tm_sec = now % 60;
diff --git a/arch/m68k/Kconfig.bus b/arch/m68k/Kconfig.bus
index d5e66ec136db..aef698fa50e5 100644
--- a/arch/m68k/Kconfig.bus
+++ b/arch/m68k/Kconfig.bus
@@ -59,6 +59,10 @@ config ATARI_ROM_ISA
 config GENERIC_ISA_DMA
 	def_bool ISA
 
+source "drivers/zorro/Kconfig"
+
+endif
+
 config PCI
 	bool "PCI support"
 	depends on M54xx
@@ -66,10 +70,8 @@ config PCI
 	  Enable the PCI bus. Support for the PCI bus hardware built into the
 	  ColdFire 547x and 548x processors.
 
+if PCI
 source "drivers/pci/Kconfig"
-
-source "drivers/zorro/Kconfig"
-
 endif
 
 if !MMU
diff --git a/arch/m68k/apollo/config.c b/arch/m68k/apollo/config.c
index 0d27706f14d4..b2a6bc63f8cd 100644
--- a/arch/m68k/apollo/config.c
+++ b/arch/m68k/apollo/config.c
@@ -221,8 +221,10 @@ int dn_dummy_hwclk(int op, struct rtc_time *t) {
     t->tm_hour=rtc->hours;
     t->tm_mday=rtc->day_of_month;
     t->tm_wday=rtc->day_of_week;
-    t->tm_mon=rtc->month;
+    t->tm_mon = rtc->month - 1;
     t->tm_year=rtc->year;
+    if (t->tm_year < 70)
+	t->tm_year += 100;
   } else {
     rtc->second=t->tm_sec;
     rtc->minute=t->tm_min;
@@ -230,8 +232,8 @@ int dn_dummy_hwclk(int op, struct rtc_time *t) {
     rtc->day_of_month=t->tm_mday;
     if(t->tm_wday!=-1)
       rtc->day_of_week=t->tm_wday;
-    rtc->month=t->tm_mon;
-    rtc->year=t->tm_year;
+    rtc->month = t->tm_mon + 1;
+    rtc->year = t->tm_year % 100;
   }
 
   return 0;
diff --git a/arch/m68k/coldfire/pci.c b/arch/m68k/coldfire/pci.c
index 3097fa2ca746..62b0eb6cf69a 100644
--- a/arch/m68k/coldfire/pci.c
+++ b/arch/m68k/coldfire/pci.c
@@ -23,20 +23,10 @@
 
 /*
  * Memory and IO mappings. We use a 1:1 mapping for local host memory to
- * PCI bus memory (no reason not to really). IO space doesn't matter, we
- * always use access functions for that. The device configuration space is
- * mapped over the IO map space when we enable it in the PCICAR register.
+ * PCI bus memory (no reason not to really). IO space is mapped in its own
+ * separate address region. The device configuration space is mapped over
+ * the IO map space when we enable it in the PCICAR register.
  */
-#define	PCI_MEM_PA	0xf0000000		/* Host physical address */
-#define	PCI_MEM_BA	0xf0000000		/* Bus physical address */
-#define	PCI_MEM_SIZE	0x08000000		/* 128 MB */
-#define	PCI_MEM_MASK	(PCI_MEM_SIZE - 1)
-
-#define	PCI_IO_PA	0xf8000000		/* Host physical address */
-#define	PCI_IO_BA	0x00000000		/* Bus physical address */
-#define	PCI_IO_SIZE	0x00010000		/* 64k */
-#define	PCI_IO_MASK	(PCI_IO_SIZE - 1)
-
 static struct pci_bus *rootbus;
 static unsigned long iospace;
 
@@ -56,13 +46,6 @@ static unsigned char mcf_host_irq[] = {
 	0, 69, 69, 71, 71,
 };
 
-
-static inline void syncio(void)
-{
-	/* The ColdFire "nop" instruction waits for all bus IO to complete */
-	__asm__ __volatile__ ("nop");
-}
-
 /*
  * Configuration space access functions. Configuration space access is
  * through the IO mapping window, enabling it via the PCICAR register.
@@ -84,9 +67,9 @@ static int mcf_pci_readconfig(struct pci_bus *bus, unsigned int devfn,
 			return PCIBIOS_SUCCESSFUL;
 	}
 
-	syncio();
 	addr = mcf_mk_pcicar(bus->number, devfn, where);
 	__raw_writel(PCICAR_E | addr, PCICAR);
+	__raw_readl(PCICAR);
 	addr = iospace + (where & 0x3);
 
 	switch (size) {
@@ -101,8 +84,8 @@ static int mcf_pci_readconfig(struct pci_bus *bus, unsigned int devfn,
 		break;
 	}
 
-	syncio();
 	__raw_writel(0, PCICAR);
+	__raw_readl(PCICAR);
 	return PCIBIOS_SUCCESSFUL;
 }
 
@@ -116,9 +99,9 @@ static int mcf_pci_writeconfig(struct pci_bus *bus, unsigned int devfn,
 			return PCIBIOS_SUCCESSFUL;
 	}
 
-	syncio();
 	addr = mcf_mk_pcicar(bus->number, devfn, where);
 	__raw_writel(PCICAR_E | addr, PCICAR);
+	__raw_readl(PCICAR);
 	addr = iospace + (where & 0x3);
 
 	switch (size) {
@@ -133,8 +116,8 @@ static int mcf_pci_writeconfig(struct pci_bus *bus, unsigned int devfn,
 		break;
 	}
 
-	syncio();
 	__raw_writel(0, PCICAR);
+	__raw_readl(PCICAR);
 	return PCIBIOS_SUCCESSFUL;
 }
 
@@ -143,89 +126,6 @@ static struct pci_ops mcf_pci_ops = {
 	.write	= mcf_pci_writeconfig,
 };
 
-/*
- *	IO address space access functions. Pretty strait forward, these are
- *	directly mapped in to the IO mapping window. And that is mapped into
- *	virtual address space.
- */
-u8 mcf_pci_inb(u32 addr)
-{
-	return __raw_readb(iospace + (addr & PCI_IO_MASK));
-}
-EXPORT_SYMBOL(mcf_pci_inb);
-
-u16 mcf_pci_inw(u32 addr)
-{
-	return le16_to_cpu(__raw_readw(iospace + (addr & PCI_IO_MASK)));
-}
-EXPORT_SYMBOL(mcf_pci_inw);
-
-u32 mcf_pci_inl(u32 addr)
-{
-	return le32_to_cpu(__raw_readl(iospace + (addr & PCI_IO_MASK)));
-}
-EXPORT_SYMBOL(mcf_pci_inl);
-
-void mcf_pci_insb(u32 addr, u8 *buf, u32 len)
-{
-	for (; len; len--)
-		*buf++ = mcf_pci_inb(addr);
-}
-EXPORT_SYMBOL(mcf_pci_insb);
-
-void mcf_pci_insw(u32 addr, u16 *buf, u32 len)
-{
-	for (; len; len--)
-		*buf++ = mcf_pci_inw(addr);
-}
-EXPORT_SYMBOL(mcf_pci_insw);
-
-void mcf_pci_insl(u32 addr, u32 *buf, u32 len)
-{
-	for (; len; len--)
-		*buf++ = mcf_pci_inl(addr);
-}
-EXPORT_SYMBOL(mcf_pci_insl);
-
-void mcf_pci_outb(u8 v, u32 addr)
-{
-	__raw_writeb(v, iospace + (addr & PCI_IO_MASK));
-}
-EXPORT_SYMBOL(mcf_pci_outb);
-
-void mcf_pci_outw(u16 v, u32 addr)
-{
-	__raw_writew(cpu_to_le16(v), iospace + (addr & PCI_IO_MASK));
-}
-EXPORT_SYMBOL(mcf_pci_outw);
-
-void mcf_pci_outl(u32 v, u32 addr)
-{
-	__raw_writel(cpu_to_le32(v), iospace + (addr & PCI_IO_MASK));
-}
-EXPORT_SYMBOL(mcf_pci_outl);
-
-void mcf_pci_outsb(u32 addr, const u8 *buf, u32 len)
-{
-	for (; len; len--)
-		mcf_pci_outb(*buf++, addr);
-}
-EXPORT_SYMBOL(mcf_pci_outsb);
-
-void mcf_pci_outsw(u32 addr, const u16 *buf, u32 len)
-{
-	for (; len; len--)
-		mcf_pci_outw(*buf++, addr);
-}
-EXPORT_SYMBOL(mcf_pci_outsw);
-
-void mcf_pci_outsl(u32 addr, const u32 *buf, u32 len)
-{
-	for (; len; len--)
-		mcf_pci_outl(*buf++, addr);
-}
-EXPORT_SYMBOL(mcf_pci_outsl);
-
 /*
  * Initialize the PCI bus registers, and scan the bus.
  */
diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig
index 37a8e5ab8728..a874e54404d1 100644
--- a/arch/m68k/configs/amiga_defconfig
+++ b/arch/m68k/configs/amiga_defconfig
@@ -98,8 +98,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -204,7 +204,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -233,12 +233,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -259,7 +259,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -310,7 +310,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -414,6 +413,7 @@ CONFIG_ARIADNE=y
 # CONFIG_NET_VENDOR_MARVELL is not set
 # CONFIG_NET_VENDOR_MICREL is not set
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 CONFIG_HYDRA=y
 CONFIG_APNE=y
 CONFIG_ZORRO8390=y
@@ -485,6 +485,7 @@ CONFIG_RTC_DRV_MSM6242=m
 CONFIG_RTC_DRV_RP5C01=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_HEARTBEAT=y
 CONFIG_PROC_HARDWARE=y
 CONFIG_AMIGA_BUILTIN_SERIAL=y
@@ -621,6 +622,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -647,6 +649,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig
index 6a466266b852..8ce39e23aa42 100644
--- a/arch/m68k/configs/apollo_defconfig
+++ b/arch/m68k/configs/apollo_defconfig
@@ -96,8 +96,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -202,7 +202,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -231,12 +231,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -257,7 +257,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -308,7 +308,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -392,6 +391,7 @@ CONFIG_VETH=m
 # CONFIG_NET_VENDOR_MICREL is not set
 # CONFIG_NET_VENDOR_NATSEMI is not set
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 # CONFIG_NET_VENDOR_QUALCOMM is not set
 # CONFIG_NET_VENDOR_RENESAS is not set
 # CONFIG_NET_VENDOR_ROCKER is not set
@@ -446,6 +446,7 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_HEARTBEAT=y
 CONFIG_PROC_HARDWARE=y
 CONFIG_EXT4_FS=y
@@ -580,6 +581,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -606,6 +608,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig
index b0691a7a3345..346c4e75edf8 100644
--- a/arch/m68k/configs/atari_defconfig
+++ b/arch/m68k/configs/atari_defconfig
@@ -96,8 +96,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -202,7 +202,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -231,12 +231,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -257,7 +257,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -308,7 +308,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -401,6 +400,7 @@ CONFIG_ATARILANCE=y
 # CONFIG_NET_VENDOR_MARVELL is not set
 # CONFIG_NET_VENDOR_MICREL is not set
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 CONFIG_NE2000=y
 # CONFIG_NET_VENDOR_QUALCOMM is not set
 # CONFIG_NET_VENDOR_RENESAS is not set
@@ -461,6 +461,7 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_HEARTBEAT=y
 CONFIG_PROC_HARDWARE=y
 CONFIG_NATFEAT=y
@@ -602,6 +603,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -628,6 +630,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig
index 6f6470fa9a50..fca9c7aa71a3 100644
--- a/arch/m68k/configs/bvme6000_defconfig
+++ b/arch/m68k/configs/bvme6000_defconfig
@@ -94,8 +94,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -200,7 +200,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -229,12 +229,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -255,7 +255,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -306,7 +306,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -391,6 +390,7 @@ CONFIG_BVME6000_NET=y
 # CONFIG_NET_VENDOR_MICREL is not set
 # CONFIG_NET_VENDOR_NATSEMI is not set
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 # CONFIG_NET_VENDOR_QUALCOMM is not set
 # CONFIG_NET_VENDOR_RENESAS is not set
 # CONFIG_NET_VENDOR_ROCKER is not set
@@ -439,6 +439,7 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_PROC_HARDWARE=y
 CONFIG_EXT4_FS=y
 CONFIG_REISERFS_FS=m
@@ -572,6 +573,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -598,6 +600,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig
index 31a1a2b5e860..f9eab174915c 100644
--- a/arch/m68k/configs/hp300_defconfig
+++ b/arch/m68k/configs/hp300_defconfig
@@ -96,8 +96,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -202,7 +202,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -231,12 +231,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -257,7 +257,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -308,7 +308,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -393,6 +392,7 @@ CONFIG_HPLANCE=y
 # CONFIG_NET_VENDOR_MICREL is not set
 # CONFIG_NET_VENDOR_NATSEMI is not set
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 # CONFIG_NET_VENDOR_QUALCOMM is not set
 # CONFIG_NET_VENDOR_RENESAS is not set
 # CONFIG_NET_VENDOR_ROCKER is not set
@@ -449,6 +449,7 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_PROC_HARDWARE=y
 CONFIG_EXT4_FS=y
 CONFIG_REISERFS_FS=m
@@ -582,6 +583,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -608,6 +610,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig
index 390d4a87441c..b52e597899eb 100644
--- a/arch/m68k/configs/mac_defconfig
+++ b/arch/m68k/configs/mac_defconfig
@@ -95,8 +95,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -201,7 +201,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -230,12 +230,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -256,7 +256,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -310,7 +310,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -410,6 +409,7 @@ CONFIG_MAC89x0=y
 # CONFIG_NET_VENDOR_MICREL is not set
 CONFIG_MACSONIC=y
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 CONFIG_MAC8390=y
 # CONFIG_NET_VENDOR_QUALCOMM is not set
 # CONFIG_NET_VENDOR_RENESAS is not set
@@ -471,6 +471,7 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_PROC_HARDWARE=y
 CONFIG_EXT4_FS=y
 CONFIG_REISERFS_FS=m
@@ -604,6 +605,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -630,6 +632,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig
index 77be97d82dc3..2a84eeec5b02 100644
--- a/arch/m68k/configs/multi_defconfig
+++ b/arch/m68k/configs/multi_defconfig
@@ -105,8 +105,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -211,7 +211,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -240,12 +240,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -266,7 +266,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -320,7 +320,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -452,6 +451,7 @@ CONFIG_MVME16x_NET=y
 # CONFIG_NET_VENDOR_MICREL is not set
 CONFIG_MACSONIC=y
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 CONFIG_HYDRA=y
 CONFIG_MAC8390=y
 CONFIG_NE2000=y
@@ -541,6 +541,7 @@ CONFIG_RTC_DRV_RP5C01=m
 CONFIG_RTC_DRV_GENERIC=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_HEARTBEAT=y
 CONFIG_PROC_HARDWARE=y
 CONFIG_NATFEAT=y
@@ -684,6 +685,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -710,6 +712,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig
index 2ca140757b0f..476e69994340 100644
--- a/arch/m68k/configs/mvme147_defconfig
+++ b/arch/m68k/configs/mvme147_defconfig
@@ -93,8 +93,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -199,7 +199,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -228,12 +228,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -254,7 +254,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -305,7 +305,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -391,6 +390,7 @@ CONFIG_MVME147_NET=y
 # CONFIG_NET_VENDOR_MICREL is not set
 # CONFIG_NET_VENDOR_NATSEMI is not set
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 # CONFIG_NET_VENDOR_QUALCOMM is not set
 # CONFIG_NET_VENDOR_RENESAS is not set
 # CONFIG_NET_VENDOR_ROCKER is not set
@@ -439,6 +439,7 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_PROC_HARDWARE=y
 CONFIG_EXT4_FS=y
 CONFIG_REISERFS_FS=m
@@ -572,6 +573,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -598,6 +600,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig
index 6a3b4dcc5aab..1477cda9146e 100644
--- a/arch/m68k/configs/mvme16x_defconfig
+++ b/arch/m68k/configs/mvme16x_defconfig
@@ -94,8 +94,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -200,7 +200,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -229,12 +229,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -255,7 +255,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -306,7 +306,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -391,6 +390,7 @@ CONFIG_MVME16x_NET=y
 # CONFIG_NET_VENDOR_MICREL is not set
 # CONFIG_NET_VENDOR_NATSEMI is not set
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 # CONFIG_NET_VENDOR_QUALCOMM is not set
 # CONFIG_NET_VENDOR_RENESAS is not set
 # CONFIG_NET_VENDOR_ROCKER is not set
@@ -439,6 +439,7 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_PROC_HARDWARE=y
 CONFIG_EXT4_FS=y
 CONFIG_REISERFS_FS=m
@@ -572,6 +573,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -598,6 +600,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig
index 2a3e29c97652..b3a543dc48a0 100644
--- a/arch/m68k/configs/q40_defconfig
+++ b/arch/m68k/configs/q40_defconfig
@@ -94,8 +94,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -200,7 +200,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -229,12 +229,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -255,7 +255,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -306,7 +306,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -400,6 +399,7 @@ CONFIG_VETH=m
 # CONFIG_NET_VENDOR_MARVELL is not set
 # CONFIG_NET_VENDOR_MICREL is not set
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 CONFIG_NE2000=y
 # CONFIG_NET_VENDOR_QUALCOMM is not set
 # CONFIG_NET_VENDOR_RENESAS is not set
@@ -461,6 +461,7 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_HEARTBEAT=y
 CONFIG_PROC_HARDWARE=y
 CONFIG_EXT4_FS=y
@@ -595,6 +596,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -621,6 +623,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig
index cba2494c99b2..d543ed5dfa96 100644
--- a/arch/m68k/configs/sun3_defconfig
+++ b/arch/m68k/configs/sun3_defconfig
@@ -91,8 +91,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -197,7 +197,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -226,12 +226,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -252,7 +252,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -303,7 +303,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -388,6 +387,7 @@ CONFIG_SUN3_82586=y
 # CONFIG_NET_VENDOR_MICREL is not set
 # CONFIG_NET_VENDOR_NATSEMI is not set
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 # CONFIG_NET_VENDOR_QUALCOMM is not set
 # CONFIG_NET_VENDOR_RENESAS is not set
 # CONFIG_NET_VENDOR_ROCKER is not set
@@ -441,6 +441,7 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_PROC_HARDWARE=y
 CONFIG_EXT4_FS=y
 CONFIG_REISERFS_FS=m
@@ -573,6 +574,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -599,6 +601,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig
index d911561137fd..a67e54246023 100644
--- a/arch/m68k/configs/sun3x_defconfig
+++ b/arch/m68k/configs/sun3x_defconfig
@@ -91,8 +91,8 @@ CONFIG_NF_CONNTRACK_SANE=m
 CONFIG_NF_CONNTRACK_SIP=m
 CONFIG_NF_CONNTRACK_TFTP=m
 CONFIG_NF_TABLES=m
-CONFIG_NF_TABLES_INET=m
-CONFIG_NF_TABLES_NETDEV=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
 CONFIG_NFT_EXTHDR=m
 CONFIG_NFT_META=m
 CONFIG_NFT_RT=m
@@ -197,7 +197,7 @@ CONFIG_NF_SOCKET_IPV4=m
 CONFIG_NFT_CHAIN_ROUTE_IPV4=m
 CONFIG_NFT_DUP_IPV4=m
 CONFIG_NFT_FIB_IPV4=m
-CONFIG_NF_TABLES_ARP=m
+CONFIG_NF_TABLES_ARP=y
 CONFIG_NF_FLOW_TABLE_IPV4=m
 CONFIG_NF_LOG_ARP=m
 CONFIG_NFT_CHAIN_NAT_IPV4=m
@@ -226,12 +226,12 @@ CONFIG_IP_NF_ARP_MANGLE=m
 CONFIG_NF_CONNTRACK_IPV6=m
 CONFIG_NF_SOCKET_IPV6=m
 CONFIG_NFT_CHAIN_ROUTE_IPV6=m
-CONFIG_NFT_DUP_IPV6=m
-CONFIG_NFT_FIB_IPV6=m
-CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_NFT_CHAIN_NAT_IPV6=m
 CONFIG_NFT_MASQ_IPV6=m
 CONFIG_NFT_REDIR_IPV6=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_NF_FLOW_TABLE_IPV6=m
 CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP6_NF_MATCH_AH=m
 CONFIG_IP6_NF_MATCH_EUI64=m
@@ -252,7 +252,7 @@ CONFIG_IP6_NF_RAW=m
 CONFIG_IP6_NF_NAT=m
 CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_IP6_NF_TARGET_NPT=m
-CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NF_TABLES_BRIDGE=y
 CONFIG_NFT_BRIDGE_META=m
 CONFIG_NFT_BRIDGE_REJECT=m
 CONFIG_NF_LOG_BRIDGE=m
@@ -303,7 +303,6 @@ CONFIG_NET_MPLS_GSO=m
 CONFIG_MPLS_ROUTING=m
 CONFIG_MPLS_IPTUNNEL=m
 CONFIG_NET_NSH=m
-CONFIG_NET_L3_MASTER_DEV=y
 CONFIG_AF_KCM=m
 # CONFIG_WIRELESS is not set
 CONFIG_PSAMPLE=m
@@ -389,6 +388,7 @@ CONFIG_SUN3LANCE=y
 # CONFIG_NET_VENDOR_MICREL is not set
 # CONFIG_NET_VENDOR_NATSEMI is not set
 # CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NI is not set
 # CONFIG_NET_VENDOR_QUALCOMM is not set
 # CONFIG_NET_VENDOR_RENESAS is not set
 # CONFIG_NET_VENDOR_ROCKER is not set
@@ -441,6 +441,7 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=m
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_IOMMU_SUPPORT is not set
+CONFIG_DAX=m
 CONFIG_PROC_HARDWARE=y
 CONFIG_EXT4_FS=y
 CONFIG_REISERFS_FS=m
@@ -574,6 +575,7 @@ CONFIG_CRYPTO_CRYPTD=m
 CONFIG_CRYPTO_MCRYPTD=m
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CFB=m
 CONFIG_CRYPTO_LRW=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_KEYWRAP=m
@@ -600,6 +602,8 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_SM4=m
+CONFIG_CRYPTO_SPECK=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild
index 88a9d27df1ac..4d8d68c4e3dd 100644
--- a/arch/m68k/include/asm/Kbuild
+++ b/arch/m68k/include/asm/Kbuild
@@ -1,4 +1,5 @@
 generic-y += barrier.h
+generic-y += compat.h
 generic-y += device.h
 generic-y += emergency-restart.h
 generic-y += exec.h
diff --git a/arch/m68k/include/asm/atarihw.h b/arch/m68k/include/asm/atarihw.h
index 972c8f33f055..9000b249d225 100644
--- a/arch/m68k/include/asm/atarihw.h
+++ b/arch/m68k/include/asm/atarihw.h
@@ -23,6 +23,7 @@
 #include <linux/types.h>
 #include <asm/bootinfo-atari.h>
 #include <asm/raw_io.h>
+#include <asm/kmap.h>
 
 extern u_long atari_mch_cookie;
 extern u_long atari_mch_type;
diff --git a/arch/m68k/include/asm/delay.h b/arch/m68k/include/asm/delay.h
index 7f474121e4ca..751712f8beea 100644
--- a/arch/m68k/include/asm/delay.h
+++ b/arch/m68k/include/asm/delay.h
@@ -49,8 +49,6 @@ extern void __bad_udelay(void);
  * The simpler m68k and ColdFire processors do not have a 32*32->64
  * multiply instruction. So we need to handle them a little differently.
  * We use a bit of shifting and a single 32*32->32 multiply to get close.
- * This is a macro so that the const version can factor out the first
- * multiply and shift.
  */
 #define	HZSCALE		(268435456 / (1000000 / HZ))
 
@@ -115,6 +113,13 @@ static inline void __udelay(unsigned long usecs)
  */
 #define	HZSCALE		(268435456 / (1000000 / HZ))
 
-#define ndelay(n) __delay(DIV_ROUND_UP((n) * ((((HZSCALE) >> 11) * (loops_per_jiffy >> 11)) >> 6), 1000))
+static inline void ndelay(unsigned long nsec)
+{
+	__delay(DIV_ROUND_UP(nsec *
+			     ((((HZSCALE) >> 11) *
+			       (loops_per_jiffy >> 11)) >> 6),
+			     1000));
+}
+#define ndelay(n) ndelay(n)
 
 #endif /* defined(_M68K_DELAY_H) */
diff --git a/arch/m68k/include/asm/io.h b/arch/m68k/include/asm/io.h
index 756089cc019c..ca2849afb087 100644
--- a/arch/m68k/include/asm/io.h
+++ b/arch/m68k/include/asm/io.h
@@ -1,14 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-#ifdef __uClinux__
+#if defined(__uClinux__) || defined(CONFIG_COLDFIRE)
 #include <asm/io_no.h>
 #else
 #include <asm/io_mm.h>
 #endif
-
-#define readb_relaxed(addr)	readb(addr)
-#define readw_relaxed(addr)	readw(addr)
-#define readl_relaxed(addr)	readl(addr)
-
-#define writeb_relaxed(b, addr)	writeb(b, addr)
-#define writew_relaxed(b, addr)	writew(b, addr)
-#define writel_relaxed(b, addr)	writel(b, addr)
diff --git a/arch/m68k/include/asm/io_mm.h b/arch/m68k/include/asm/io_mm.h
index ed5333e87879..fe485f4f5fac 100644
--- a/arch/m68k/include/asm/io_mm.h
+++ b/arch/m68k/include/asm/io_mm.h
@@ -26,6 +26,7 @@
 #include <linux/compiler.h>
 #include <asm/raw_io.h>
 #include <asm/virtconvert.h>
+#include <asm/kmap.h>
 
 #include <asm-generic/iomap.h>
 
@@ -85,53 +86,7 @@
 #endif /* ATARI_ROM_ISA */
 
 
-#if defined(CONFIG_PCI) && defined(CONFIG_COLDFIRE)
-
-#define HAVE_ARCH_PIO_SIZE
-#define PIO_OFFSET	0
-#define PIO_MASK	0xffff
-#define PIO_RESERVED	0x10000
-
-u8 mcf_pci_inb(u32 addr);
-u16 mcf_pci_inw(u32 addr);
-u32 mcf_pci_inl(u32 addr);
-void mcf_pci_insb(u32 addr, u8 *buf, u32 len);
-void mcf_pci_insw(u32 addr, u16 *buf, u32 len);
-void mcf_pci_insl(u32 addr, u32 *buf, u32 len);
-
-void mcf_pci_outb(u8 v, u32 addr);
-void mcf_pci_outw(u16 v, u32 addr);
-void mcf_pci_outl(u32 v, u32 addr);
-void mcf_pci_outsb(u32 addr, const u8 *buf, u32 len);
-void mcf_pci_outsw(u32 addr, const u16 *buf, u32 len);
-void mcf_pci_outsl(u32 addr, const u32 *buf, u32 len);
-
-#define	inb	mcf_pci_inb
-#define	inb_p	mcf_pci_inb
-#define	inw	mcf_pci_inw
-#define	inw_p	mcf_pci_inw
-#define	inl	mcf_pci_inl
-#define	inl_p	mcf_pci_inl
-#define	insb	mcf_pci_insb
-#define	insw	mcf_pci_insw
-#define	insl	mcf_pci_insl
-
-#define	outb	mcf_pci_outb
-#define	outb_p	mcf_pci_outb
-#define	outw	mcf_pci_outw
-#define	outw_p	mcf_pci_outw
-#define	outl	mcf_pci_outl
-#define	outl_p	mcf_pci_outl
-#define	outsb	mcf_pci_outsb
-#define	outsw	mcf_pci_outsw
-#define	outsl	mcf_pci_outsl
-
-#define readb(addr)	in_8(addr)
-#define writeb(v, addr)	out_8((addr), (v))
-#define readw(addr)	in_le16(addr)
-#define writew(v, addr)	out_le16((addr), (v))
-
-#elif defined(CONFIG_ISA) || defined(CONFIG_ATARI_ROM_ISA)
+#if defined(CONFIG_ISA) || defined(CONFIG_ATARI_ROM_ISA)
 
 #if MULTI_ISA == 0
 #undef MULTI_ISA
@@ -414,8 +369,7 @@ static inline void isa_delay(void)
 #define writew(val, addr)	out_le16((addr), (val))
 #endif /* CONFIG_ATARI_ROM_ISA */
 
-#if !defined(CONFIG_ISA) && !defined(CONFIG_ATARI_ROM_ISA) && \
-    !(defined(CONFIG_PCI) && defined(CONFIG_COLDFIRE))
+#if !defined(CONFIG_ISA) && !defined(CONFIG_ATARI_ROM_ISA)
 /*
  * We need to define dummy functions for GENERIC_IOMAP support.
  */
@@ -461,39 +415,6 @@ static inline void isa_delay(void)
 
 #define mmiowb()
 
-static inline void __iomem *ioremap(unsigned long physaddr, unsigned long size)
-{
-	return __ioremap(physaddr, size, IOMAP_NOCACHE_SER);
-}
-static inline void __iomem *ioremap_nocache(unsigned long physaddr, unsigned long size)
-{
-	return __ioremap(physaddr, size, IOMAP_NOCACHE_SER);
-}
-#define ioremap_uc ioremap_nocache
-static inline void __iomem *ioremap_wt(unsigned long physaddr,
-					 unsigned long size)
-{
-	return __ioremap(physaddr, size, IOMAP_WRITETHROUGH);
-}
-static inline void __iomem *ioremap_fullcache(unsigned long physaddr,
-				      unsigned long size)
-{
-	return __ioremap(physaddr, size, IOMAP_FULL_CACHING);
-}
-
-static inline void memset_io(volatile void __iomem *addr, unsigned char val, int count)
-{
-	__builtin_memset((void __force *) addr, val, count);
-}
-static inline void memcpy_fromio(void *dst, const volatile void __iomem *src, int count)
-{
-	__builtin_memcpy(dst, (void __force *) src, count);
-}
-static inline void memcpy_toio(volatile void __iomem *dst, const void *src, int count)
-{
-	__builtin_memcpy((void __force *) dst, src, count);
-}
-
 #ifndef CONFIG_SUN3
 #define IO_SPACE_LIMIT 0xffff
 #else
@@ -515,13 +436,12 @@ static inline void memcpy_toio(volatile void __iomem *dst, const void *src, int
  */
 #define xlate_dev_kmem_ptr(p)	p
 
-static inline void __iomem *ioport_map(unsigned long port, unsigned int nr)
-{
-	return (void __iomem *) port;
-}
+#define readb_relaxed(addr)	readb(addr)
+#define readw_relaxed(addr)	readw(addr)
+#define readl_relaxed(addr)	readl(addr)
 
-static inline void ioport_unmap(void __iomem *p)
-{
-}
+#define writeb_relaxed(b, addr)	writeb(b, addr)
+#define writew_relaxed(b, addr)	writew(b, addr)
+#define writel_relaxed(b, addr)	writel(b, addr)
 
 #endif /* _IO_H */
diff --git a/arch/m68k/include/asm/io_no.h b/arch/m68k/include/asm/io_no.h
index 86f45b403bcc..83a0a6d449f4 100644
--- a/arch/m68k/include/asm/io_no.h
+++ b/arch/m68k/include/asm/io_no.h
@@ -2,191 +2,148 @@
 #ifndef _M68KNOMMU_IO_H
 #define _M68KNOMMU_IO_H
 
-#ifdef __KERNEL__
-
-#define ARCH_HAS_IOREMAP_WT
-
-#include <asm/virtconvert.h>
-#include <asm-generic/iomap.h>
-
 /*
- * These are for ISA/PCI shared memory _only_ and should never be used
- * on any other type of memory, including Zorro memory. They are meant to
- * access the bus in the bus byte order which is little-endian!.
- *
- * readX/writeX() are used to access memory mapped devices. On some
- * architectures the memory mapped IO stuff needs to be accessed
- * differently. On the m68k architecture, we just read/write the
- * memory location directly.
- */
-/* ++roman: The assignments to temp. vars avoid that gcc sometimes generates
- * two accesses to memory, which may be undesirable for some devices.
+ * Convert a physical memory address into a IO memory address.
+ * For us this is trivially a type cast.
  */
+#define iomem(a)	((void __iomem *) (a))
 
 /*
- * swap functions are sometimes needed to interface little-endian hardware
+ * The non-MMU m68k and ColdFire IO and memory mapped hardware access
+ * functions have always worked in CPU native endian. We need to define
+ * that behavior here first before we include asm-generic/io.h.
  */
-static inline unsigned short _swapw(volatile unsigned short v)
-{
-    return ((v << 8) | (v >> 8));
-}
-
-static inline unsigned int _swapl(volatile unsigned long v)
-{
-    return ((v << 24) | ((v & 0xff00) << 8) | ((v & 0xff0000) >> 8) | (v >> 24));
-}
-
-#define readb(addr) \
+#define __raw_readb(addr) \
     ({ unsigned char __v = (*(volatile unsigned char *) (addr)); __v; })
-#define readw(addr) \
+#define __raw_readw(addr) \
     ({ unsigned short __v = (*(volatile unsigned short *) (addr)); __v; })
-#define readl(addr) \
+#define __raw_readl(addr) \
     ({ unsigned int __v = (*(volatile unsigned int *) (addr)); __v; })
 
-#define writeb(b,addr) (void)((*(volatile unsigned char *) (addr)) = (b))
-#define writew(b,addr) (void)((*(volatile unsigned short *) (addr)) = (b))
-#define writel(b,addr) (void)((*(volatile unsigned int *) (addr)) = (b))
+#define __raw_writeb(b, addr) (void)((*(volatile unsigned char *) (addr)) = (b))
+#define __raw_writew(b, addr) (void)((*(volatile unsigned short *) (addr)) = (b))
+#define __raw_writel(b, addr) (void)((*(volatile unsigned int *) (addr)) = (b))
 
-#define __raw_readb readb
-#define __raw_readw readw
-#define __raw_readl readl
-#define __raw_writeb writeb
-#define __raw_writew writew
-#define __raw_writel writel
+#if defined(CONFIG_COLDFIRE)
+/*
+ * For ColdFire platforms we may need to do some extra checks for what
+ * type of address range we are accessing. Include the ColdFire platform
+ * definitions so we can figure out if need to do something special.
+ */
+#include <asm/byteorder.h>
+#include <asm/coldfire.h>
+#include <asm/mcfsim.h>
+#endif /* CONFIG_COLDFIRE */
 
-static inline void io_outsb(unsigned int addr, const void *buf, int len)
+#if defined(IOMEMBASE)
+/*
+ * The ColdFire SoC internal peripherals are mapped into virtual address
+ * space using the ACR registers of the cache control unit. This means we
+ * are using a 1:1 physical:virtual mapping for them. We can quickly
+ * determine if we are accessing an internal peripheral device given the
+ * physical or vitrual address using the same range check. This check logic
+ * applies just the same of there is no MMU but something like a PCI bus
+ * is present.
+ */
+static int __cf_internalio(unsigned long addr)
 {
-	volatile unsigned char *ap = (volatile unsigned char *) addr;
-	unsigned char *bp = (unsigned char *) buf;
-	while (len--)
-		*ap = *bp++;
+	return (addr >= IOMEMBASE) && (addr <= IOMEMBASE + IOMEMSIZE - 1);
 }
 
-static inline void io_outsw(unsigned int addr, const void *buf, int len)
+static int cf_internalio(const volatile void __iomem *addr)
 {
-	volatile unsigned short *ap = (volatile unsigned short *) addr;
-	unsigned short *bp = (unsigned short *) buf;
-	while (len--)
-		*ap = _swapw(*bp++);
+	return __cf_internalio((unsigned long) addr);
 }
 
-static inline void io_outsl(unsigned int addr, const void *buf, int len)
+/*
+ * We need to treat built-in peripherals and bus based address ranges
+ * differently. Local built-in peripherals (and the ColdFire SoC parts
+ * have quite a lot of them) are always native endian - which is big
+ * endian on m68k/ColdFire. Bus based address ranges, like the PCI bus,
+ * are accessed little endian - so we need to byte swap those.
+ */
+#define readw readw
+static inline u16 readw(const volatile void __iomem *addr)
 {
-	volatile unsigned int *ap = (volatile unsigned int *) addr;
-	unsigned int *bp = (unsigned int *) buf;
-	while (len--)
-		*ap = _swapl(*bp++);
+	if (cf_internalio(addr))
+		return __raw_readw(addr);
+	return __le16_to_cpu(__raw_readw(addr));
 }
 
-static inline void io_insb(unsigned int addr, void *buf, int len)
+#define readl readl
+static inline u32 readl(const volatile void __iomem *addr)
 {
-	volatile unsigned char *ap = (volatile unsigned char *) addr;
-	unsigned char *bp = (unsigned char *) buf;
-	while (len--)
-		*bp++ = *ap;
+	if (cf_internalio(addr))
+		return __raw_readl(addr);
+	return __le32_to_cpu(__raw_readl(addr));
 }
 
-static inline void io_insw(unsigned int addr, void *buf, int len)
+#define writew writew
+static inline void writew(u16 value, volatile void __iomem *addr)
 {
-	volatile unsigned short *ap = (volatile unsigned short *) addr;
-	unsigned short *bp = (unsigned short *) buf;
-	while (len--)
-		*bp++ = _swapw(*ap);
+	if (cf_internalio(addr))
+		__raw_writew(value, addr);
+	else
+		__raw_writew(__cpu_to_le16(value), addr);
 }
 
-static inline void io_insl(unsigned int addr, void *buf, int len)
+#define writel writel
+static inline void writel(u32 value, volatile void __iomem *addr)
 {
-	volatile unsigned int *ap = (volatile unsigned int *) addr;
-	unsigned int *bp = (unsigned int *) buf;
-	while (len--)
-		*bp++ = _swapl(*ap);
+	if (cf_internalio(addr))
+		__raw_writel(value, addr);
+	else
+		__raw_writel(__cpu_to_le32(value), addr);
 }
 
-#define mmiowb()
-
-/*
- *	make the short names macros so specific devices
- *	can override them as required
- */
-
-#define memset_io(a,b,c)	memset((void *)(a),(b),(c))
-#define memcpy_fromio(a,b,c)	memcpy((a),(void *)(b),(c))
-#define memcpy_toio(a,b,c)	memcpy((void *)(a),(b),(c))
-
-#define inb(addr)    readb(addr)
-#define inw(addr)    readw(addr)
-#define inl(addr)    readl(addr)
-#define outb(x,addr) ((void) writeb(x,addr))
-#define outw(x,addr) ((void) writew(x,addr))
-#define outl(x,addr) ((void) writel(x,addr))
-
-#define inb_p(addr)    inb(addr)
-#define inw_p(addr)    inw(addr)
-#define inl_p(addr)    inl(addr)
-#define outb_p(x,addr) outb(x,addr)
-#define outw_p(x,addr) outw(x,addr)
-#define outl_p(x,addr) outl(x,addr)
+#else
 
-#define outsb(a,b,l) io_outsb(a,b,l)
-#define outsw(a,b,l) io_outsw(a,b,l)
-#define outsl(a,b,l) io_outsl(a,b,l)
+#define readb __raw_readb
+#define readw __raw_readw
+#define readl __raw_readl
+#define writeb __raw_writeb
+#define writew __raw_writew
+#define writel __raw_writel
 
-#define insb(a,b,l) io_insb(a,b,l)
-#define insw(a,b,l) io_insw(a,b,l)
-#define insl(a,b,l) io_insl(a,b,l)
-
-#define IO_SPACE_LIMIT 0xffffffff
-
-
-/* Values for nocacheflag and cmode */
-#define IOMAP_FULL_CACHING		0
-#define IOMAP_NOCACHE_SER		1
-#define IOMAP_NOCACHE_NONSER		2
-#define IOMAP_WRITETHROUGH		3
-
-static inline void *__ioremap(unsigned long physaddr, unsigned long size, int cacheflag)
-{
-	return (void *) physaddr;
-}
-static inline void *ioremap(unsigned long physaddr, unsigned long size)
-{
-	return __ioremap(physaddr, size, IOMAP_NOCACHE_SER);
-}
-static inline void *ioremap_nocache(unsigned long physaddr, unsigned long size)
-{
-	return __ioremap(physaddr, size, IOMAP_NOCACHE_SER);
-}
-static inline void *ioremap_wt(unsigned long physaddr, unsigned long size)
-{
-	return __ioremap(physaddr, size, IOMAP_WRITETHROUGH);
-}
-static inline void *ioremap_fullcache(unsigned long physaddr, unsigned long size)
-{
-	return __ioremap(physaddr, size, IOMAP_FULL_CACHING);
-}
-
-#define	iounmap(addr)	do { } while(0)
+#endif /* IOMEMBASE */
 
+#if defined(CONFIG_PCI)
 /*
- * Convert a physical pointer to a virtual kernel pointer for /dev/mem
- * access
+ * Support for PCI bus access uses the asm-generic access functions.
+ * We need to supply the base address and masks for the normal memory
+ * and IO address space mappings.
  */
-#define xlate_dev_mem_ptr(p)	__va(p)
+#define PCI_MEM_PA	0xf0000000		/* Host physical address */
+#define PCI_MEM_BA	0xf0000000		/* Bus physical address */
+#define PCI_MEM_SIZE	0x08000000		/* 128 MB */
+#define PCI_MEM_MASK	(PCI_MEM_SIZE - 1)
+
+#define PCI_IO_PA	0xf8000000		/* Host physical address */
+#define PCI_IO_BA	0x00000000		/* Bus physical address */
+#define PCI_IO_SIZE	0x00010000		/* 64k */
+#define PCI_IO_MASK	(PCI_IO_SIZE - 1)
+
+#define HAVE_ARCH_PIO_SIZE
+#define PIO_OFFSET	0
+#define PIO_MASK	0xffff
+#define PIO_RESERVED	0x10000
+#define PCI_IOBASE	((void __iomem *) PCI_IO_PA)
+#define PCI_SPACE_LIMIT	PCI_IO_MASK
+#endif /* CONFIG_PCI */
 
 /*
- * Convert a virtual cached pointer to an uncached pointer
+ * These are defined in kmap.h as static inline functions. To maintain
+ * previous behavior we put these define guards here so io_mm.h doesn't
+ * see them.
  */
-#define xlate_dev_kmem_ptr(p)	p
+#ifdef CONFIG_MMU
+#define memset_io memset_io
+#define memcpy_fromio memcpy_fromio
+#define memcpy_toio memcpy_toio
+#endif
 
-static inline void __iomem *ioport_map(unsigned long port, unsigned int nr)
-{
-	return (void __iomem *) port;
-}
-
-static inline void ioport_unmap(void __iomem *p)
-{
-}
-
-#endif /* __KERNEL__ */
+#include <asm/kmap.h>
+#include <asm/virtconvert.h>
+#include <asm-generic/io.h>
 
 #endif /* _M68KNOMMU_IO_H */
diff --git a/arch/m68k/include/asm/kmap.h b/arch/m68k/include/asm/kmap.h
new file mode 100644
index 000000000000..84b8333db8ad
--- /dev/null
+++ b/arch/m68k/include/asm/kmap.h
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _KMAP_H
+#define _KMAP_H
+
+#ifdef CONFIG_MMU
+
+/* Values for nocacheflag and cmode */
+#define IOMAP_FULL_CACHING		0
+#define IOMAP_NOCACHE_SER		1
+#define IOMAP_NOCACHE_NONSER		2
+#define IOMAP_WRITETHROUGH		3
+
+/*
+ * These functions exported by arch/m68k/mm/kmap.c.
+ * Only needed on MMU enabled systems.
+ */
+extern void __iomem *__ioremap(unsigned long physaddr, unsigned long size,
+			       int cacheflag);
+extern void iounmap(void __iomem *addr);
+extern void __iounmap(void *addr, unsigned long size);
+
+#define ioremap ioremap
+static inline void __iomem *ioremap(unsigned long physaddr, unsigned long size)
+{
+	return __ioremap(physaddr, size, IOMAP_NOCACHE_SER);
+}
+
+#define ioremap_nocache ioremap_nocache
+static inline void __iomem *ioremap_nocache(unsigned long physaddr,
+					    unsigned long size)
+{
+	return __ioremap(physaddr, size, IOMAP_NOCACHE_SER);
+}
+
+#define ioremap_uc ioremap_nocache
+static inline void __iomem *ioremap_wt(unsigned long physaddr,
+				       unsigned long size)
+{
+	return __ioremap(physaddr, size, IOMAP_WRITETHROUGH);
+}
+
+#define ioremap_fillcache ioremap_fullcache
+static inline void __iomem *ioremap_fullcache(unsigned long physaddr,
+					      unsigned long size)
+{
+	return __ioremap(physaddr, size, IOMAP_FULL_CACHING);
+}
+
+static inline void memset_io(volatile void __iomem *addr, unsigned char val,
+			     int count)
+{
+	__builtin_memset((void __force *) addr, val, count);
+}
+
+static inline void memcpy_fromio(void *dst, const volatile void __iomem *src,
+				 int count)
+{
+	__builtin_memcpy(dst, (void __force *) src, count);
+}
+
+static inline void memcpy_toio(volatile void __iomem *dst, const void *src,
+			       int count)
+{
+	__builtin_memcpy((void __force *) dst, src, count);
+}
+
+#endif /* CONFIG_MMU */
+
+#define ioport_map ioport_map
+static inline void __iomem *ioport_map(unsigned long port, unsigned int nr)
+{
+	return (void __iomem *) port;
+}
+
+#define ioport_unmap ioport_unmap
+static inline void ioport_unmap(void __iomem *p)
+{
+}
+
+#endif /* _KMAP_H */
diff --git a/arch/m68k/include/asm/nubus.h b/arch/m68k/include/asm/nubus.h
index d0d2039e434e..c2281da6c51a 100644
--- a/arch/m68k/include/asm/nubus.h
+++ b/arch/m68k/include/asm/nubus.h
@@ -3,6 +3,7 @@
 #define _ASM_M68K_NUBUS_H
 
 #include <asm/raw_io.h>
+#include <asm/kmap.h>
 
 #define nubus_readb raw_inb
 #define nubus_readw raw_inw
diff --git a/arch/m68k/include/asm/pci.h b/arch/m68k/include/asm/pci.h
index ef26fae8cf0b..5a4bc223743b 100644
--- a/arch/m68k/include/asm/pci.h
+++ b/arch/m68k/include/asm/pci.h
@@ -4,12 +4,6 @@
 
 #include <asm-generic/pci.h>
 
-/* The PCI address space does equal the physical memory
- * address space.  The networking and block device layers use
- * this boolean for bounce buffer decisions.
- */
-#define PCI_DMA_BUS_IS_PHYS	(1)
-
 #define	pcibios_assign_all_busses()	1
 
 #define	PCIBIOS_MIN_IO		0x00000100
diff --git a/arch/m68k/include/asm/q40_master.h b/arch/m68k/include/asm/q40_master.h
index 3a89c088800c..9b00fb8079e6 100644
--- a/arch/m68k/include/asm/q40_master.h
+++ b/arch/m68k/include/asm/q40_master.h
@@ -8,7 +8,7 @@
 #define _Q40_MASTER_H
 
 #include <asm/raw_io.h>
-
+#include <asm/kmap.h>
 
 #define q40_master_addr 0xff000000
 
diff --git a/arch/m68k/include/asm/raw_io.h b/arch/m68k/include/asm/raw_io.h
index 05e940c29b54..85761255dde5 100644
--- a/arch/m68k/include/asm/raw_io.h
+++ b/arch/m68k/include/asm/raw_io.h
@@ -13,20 +13,6 @@
 
 #include <asm/byteorder.h>
 
-
-/* Values for nocacheflag and cmode */
-#define IOMAP_FULL_CACHING		0
-#define IOMAP_NOCACHE_SER		1
-#define IOMAP_NOCACHE_NONSER		2
-#define IOMAP_WRITETHROUGH		3
-
-extern void iounmap(void __iomem *addr);
-
-extern void __iomem *__ioremap(unsigned long physaddr, unsigned long size,
-		       int cacheflag);
-extern void __iounmap(void *addr, unsigned long size);
-
-
 /* ++roman: The assignments to temp. vars avoid that gcc sometimes generates
  * two accesses to memory, which may be undesirable for some devices.
  */
diff --git a/arch/m68k/include/asm/uaccess_mm.h b/arch/m68k/include/asm/uaccess_mm.h
index 75c172e909ac..c4cb889660aa 100644
--- a/arch/m68k/include/asm/uaccess_mm.h
+++ b/arch/m68k/include/asm/uaccess_mm.h
@@ -141,10 +141,12 @@ asm volatile ("\n"					\
 	case 4:								\
 		__get_user_asm(__gu_err, x, ptr, u32, l, r, -EFAULT);	\
 		break;							\
-/*	case 8:	disabled because gcc-4.1 has a broken typeof		\
- 	    {								\
- 		const void *__gu_ptr = (ptr);				\
- 		u64 __gu_val;						\
+	case 8: {							\
+		const void *__gu_ptr = (ptr);				\
+		union {							\
+			u64 l;						\
+			__typeof__(*(ptr)) t;				\
+		} __gu_val;						\
 		asm volatile ("\n"					\
 			"1:	"MOVES".l	(%2)+,%1\n"		\
 			"2:	"MOVES".l	(%2),%R1\n"		\
@@ -162,13 +164,13 @@ asm volatile ("\n"					\
 			"	.long	1b,10b\n"			\
 			"	.long	2b,10b\n"			\
 			"	.previous"				\
-			: "+d" (__gu_err), "=&r" (__gu_val),		\
+			: "+d" (__gu_err), "=&r" (__gu_val.l),		\
 			  "+a" (__gu_ptr)				\
 			: "i" (-EFAULT)					\
 			: "memory");					\
-		(x) = (__force typeof(*(ptr)))__gu_val;			\
+		(x) = __gu_val.t;					\
 		break;							\
-	    }	*/							\
+	}								\
 	default:							\
 		__gu_err = __get_user_bad();				\
 		break;							\
diff --git a/arch/m68k/include/asm/vga.h b/arch/m68k/include/asm/vga.h
index 010a624d1b39..4742e6bc3ab8 100644
--- a/arch/m68k/include/asm/vga.h
+++ b/arch/m68k/include/asm/vga.h
@@ -2,7 +2,15 @@
 #ifndef _ASM_M68K_VGA_H
 #define _ASM_M68K_VGA_H
 
+/*
+ * Some ColdFire platforms do in fact have a PCI bus. So for those we want
+ * to use the real IO access functions, don't fake them out or redirect them
+ * for that case.
+ */
+#ifndef CONFIG_PCI
+
 #include <asm/raw_io.h>
+#include <asm/kmap.h>
 
 /*
  * FIXME
@@ -25,4 +33,5 @@
 #define writeb			raw_outb
 #define writew			raw_outw
 
+#endif /* CONFIG_PCI */
 #endif /* _ASM_M68K_VGA_H */
diff --git a/arch/m68k/include/asm/virtconvert.h b/arch/m68k/include/asm/virtconvert.h
index 4aea6be7b220..dfe43083b579 100644
--- a/arch/m68k/include/asm/virtconvert.h
+++ b/arch/m68k/include/asm/virtconvert.h
@@ -16,11 +16,13 @@
 /*
  * Change virtual addresses to physical addresses and vv.
  */
+#define virt_to_phys virt_to_phys
 static inline unsigned long virt_to_phys(void *address)
 {
 	return __pa(address);
 }
 
+#define phys_to_virt phys_to_virt
 static inline void *phys_to_virt(unsigned long address)
 {
 	return __va(address);
diff --git a/arch/m68k/include/asm/zorro.h b/arch/m68k/include/asm/zorro.h
index 96f64bf7bcaa..60fc4b6f294d 100644
--- a/arch/m68k/include/asm/zorro.h
+++ b/arch/m68k/include/asm/zorro.h
@@ -3,6 +3,7 @@
 #define _ASM_M68K_ZORRO_H
 
 #include <asm/raw_io.h>
+#include <asm/kmap.h>
 
 #define z_readb raw_inb
 #define z_readw raw_inw
diff --git a/arch/m68k/kernel/dma.c b/arch/m68k/kernel/dma.c
index c01b9b8f97bf..463572c4943f 100644
--- a/arch/m68k/kernel/dma.c
+++ b/arch/m68k/kernel/dma.c
@@ -9,6 +9,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/device.h>
 #include <linux/kernel.h>
+#include <linux/platform_device.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
@@ -165,3 +166,12 @@ const struct dma_map_ops m68k_dma_ops = {
 	.sync_sg_for_device	= m68k_dma_sync_sg_for_device,
 };
 EXPORT_SYMBOL(m68k_dma_ops);
+
+void arch_setup_pdev_archdata(struct platform_device *pdev)
+{
+	if (pdev->dev.coherent_dma_mask == DMA_MASK_NONE &&
+	    pdev->dev.dma_mask == NULL) {
+		pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
+		pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
+	}
+}
diff --git a/arch/m68k/kernel/setup_mm.c b/arch/m68k/kernel/setup_mm.c
index dd25bfc22fb4..f35e3ebd6331 100644
--- a/arch/m68k/kernel/setup_mm.c
+++ b/arch/m68k/kernel/setup_mm.c
@@ -527,21 +527,9 @@ static int hardware_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int hardware_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, hardware_proc_show, NULL);
-}
-
-static const struct file_operations hardware_proc_fops = {
-	.open		= hardware_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int __init proc_hardware_init(void)
 {
-	proc_create("hardware", 0, NULL, &hardware_proc_fops);
+	proc_create_single("hardware", 0, NULL, hardware_proc_show);
 	return 0;
 }
 module_init(proc_hardware_init);
diff --git a/arch/m68k/kernel/signal.c b/arch/m68k/kernel/signal.c
index f7cd5ecfacd3..72850b85ecf8 100644
--- a/arch/m68k/kernel/signal.c
+++ b/arch/m68k/kernel/signal.c
@@ -576,41 +576,42 @@ static inline int rt_save_fpu_state(struct ucontext __user *uc, struct pt_regs *
 
 static inline void siginfo_build_tests(void)
 {
-	/* This needs to be tested on m68k as it has a lesser
-	 * alignment requirment than x86 and that can cause surprises.
+	/*
+	 * This needs to be tested on m68k as it has a lesser
+	 * alignment requirement than x86 and that can cause surprises.
 	 */
 
 	/* This is part of the ABI and can never change in size: */
 	BUILD_BUG_ON(sizeof(siginfo_t) != 128);
 
-	/* Ensure the know fields never change in location */
+	/* Ensure the known fields never change in location */
 	BUILD_BUG_ON(offsetof(siginfo_t, si_signo) != 0);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_errno) != 4);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_code)  != 8);
 
 	/* _kill */
-	BUILD_BUG_ON(offsetof(siginfo_t, si_pid) != 0x0C);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_pid) != 0x0c);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_uid) != 0x10);
 
 	/* _timer */
-	BUILD_BUG_ON(offsetof(siginfo_t, si_tid)     != 0x0C);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_tid)     != 0x0c);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_overrun) != 0x10);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_value)   != 0x14);
 
 	/* _rt */
-	BUILD_BUG_ON(offsetof(siginfo_t, si_pid)   != 0x0C);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_pid)   != 0x0c);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_uid)   != 0x10);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_value) != 0x14);
 
 	/* _sigchld */
-	BUILD_BUG_ON(offsetof(siginfo_t, si_pid)    != 0x0C);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_pid)    != 0x0c);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_uid)    != 0x10);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_status) != 0x14);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_utime)  != 0x18);
-	BUILD_BUG_ON(offsetof(siginfo_t, si_stime)  != 0x1C);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_stime)  != 0x1c);
 
 	/* _sigfault */
-	BUILD_BUG_ON(offsetof(siginfo_t, si_addr) != 0x0C);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_addr) != 0x0c);
 
 	/* _sigfault._mcerr */
 	BUILD_BUG_ON(offsetof(siginfo_t, si_addr_lsb) != 0x10);
@@ -623,11 +624,11 @@ static inline void siginfo_build_tests(void)
 	BUILD_BUG_ON(offsetof(siginfo_t, si_pkey) != 0x12);
 
 	/* _sigpoll */
-	BUILD_BUG_ON(offsetof(siginfo_t, si_band)   != 0x0C);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_band)   != 0x0c);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_fd)     != 0x10);
 
 	/* _sigsys */
-	BUILD_BUG_ON(offsetof(siginfo_t, si_call_addr) != 0x0C);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_call_addr) != 0x0c);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_syscall)   != 0x10);
 	BUILD_BUG_ON(offsetof(siginfo_t, si_arch)      != 0x14);
 
diff --git a/arch/m68k/kernel/time.c b/arch/m68k/kernel/time.c
index 97dd4e26f234..3a8b47f8f97b 100644
--- a/arch/m68k/kernel/time.c
+++ b/arch/m68k/kernel/time.c
@@ -71,23 +71,26 @@ static irqreturn_t timer_interrupt(int irq, void *dummy)
 	return IRQ_HANDLED;
 }
 
-void read_persistent_clock(struct timespec *ts)
+#ifdef CONFIG_M68KCLASSIC
+#if !IS_BUILTIN(CONFIG_RTC_DRV_GENERIC)
+void read_persistent_clock64(struct timespec64 *ts)
 {
 	struct rtc_time time;
+
 	ts->tv_sec = 0;
 	ts->tv_nsec = 0;
 
-	if (mach_hwclk) {
-		mach_hwclk(0, &time);
+	if (!mach_hwclk)
+		return;
 
-		if ((time.tm_year += 1900) < 1970)
-			time.tm_year += 100;
-		ts->tv_sec = mktime(time.tm_year, time.tm_mon, time.tm_mday,
-				      time.tm_hour, time.tm_min, time.tm_sec);
-	}
+	mach_hwclk(0, &time);
+
+	ts->tv_sec = mktime64(time.tm_year + 1900, time.tm_mon + 1, time.tm_mday,
+			      time.tm_hour, time.tm_min, time.tm_sec);
 }
+#endif
 
-#if defined(CONFIG_ARCH_USES_GETTIMEOFFSET) && IS_ENABLED(CONFIG_RTC_DRV_GENERIC)
+#if IS_ENABLED(CONFIG_RTC_DRV_GENERIC)
 static int rtc_generic_get_time(struct device *dev, struct rtc_time *tm)
 {
 	mach_hwclk(0, tm);
@@ -145,8 +148,8 @@ static int __init rtc_init(void)
 }
 
 module_init(rtc_init);
-
-#endif /* CONFIG_ARCH_USES_GETTIMEOFFSET */
+#endif /* CONFIG_RTC_DRV_GENERIC */
+#endif /* CONFIG M68KCLASSIC */
 
 void __init time_init(void)
 {
diff --git a/arch/m68k/kernel/traps.c b/arch/m68k/kernel/traps.c
index c1cc4e99aa94..b2fd000b9285 100644
--- a/arch/m68k/kernel/traps.c
+++ b/arch/m68k/kernel/traps.c
@@ -1007,9 +1007,9 @@ void bad_super_trap (struct frame *fp)
 
 asmlinkage void trap_c(struct frame *fp)
 {
-	int sig;
+	int sig, si_code;
+	void __user *addr;
 	int vector = (fp->ptregs.vector >> 2) & 0xff;
-	siginfo_t info;
 
 	if (fp->ptregs.sr & PS_S) {
 		if (vector == VEC_TRACE) {
@@ -1029,21 +1029,21 @@ asmlinkage void trap_c(struct frame *fp)
 	/* send the appropriate signal to the user program */
 	switch (vector) {
 	    case VEC_ADDRERR:
-		info.si_code = BUS_ADRALN;
+		si_code = BUS_ADRALN;
 		sig = SIGBUS;
 		break;
 	    case VEC_ILLEGAL:
 	    case VEC_LINE10:
 	    case VEC_LINE11:
-		info.si_code = ILL_ILLOPC;
+		si_code = ILL_ILLOPC;
 		sig = SIGILL;
 		break;
 	    case VEC_PRIV:
-		info.si_code = ILL_PRVOPC;
+		si_code = ILL_PRVOPC;
 		sig = SIGILL;
 		break;
 	    case VEC_COPROC:
-		info.si_code = ILL_COPROC;
+		si_code = ILL_COPROC;
 		sig = SIGILL;
 		break;
 	    case VEC_TRAP1:
@@ -1060,76 +1060,74 @@ asmlinkage void trap_c(struct frame *fp)
 	    case VEC_TRAP12:
 	    case VEC_TRAP13:
 	    case VEC_TRAP14:
-		info.si_code = ILL_ILLTRP;
+		si_code = ILL_ILLTRP;
 		sig = SIGILL;
 		break;
 	    case VEC_FPBRUC:
 	    case VEC_FPOE:
 	    case VEC_FPNAN:
-		info.si_code = FPE_FLTINV;
+		si_code = FPE_FLTINV;
 		sig = SIGFPE;
 		break;
 	    case VEC_FPIR:
-		info.si_code = FPE_FLTRES;
+		si_code = FPE_FLTRES;
 		sig = SIGFPE;
 		break;
 	    case VEC_FPDIVZ:
-		info.si_code = FPE_FLTDIV;
+		si_code = FPE_FLTDIV;
 		sig = SIGFPE;
 		break;
 	    case VEC_FPUNDER:
-		info.si_code = FPE_FLTUND;
+		si_code = FPE_FLTUND;
 		sig = SIGFPE;
 		break;
 	    case VEC_FPOVER:
-		info.si_code = FPE_FLTOVF;
+		si_code = FPE_FLTOVF;
 		sig = SIGFPE;
 		break;
 	    case VEC_ZERODIV:
-		info.si_code = FPE_INTDIV;
+		si_code = FPE_INTDIV;
 		sig = SIGFPE;
 		break;
 	    case VEC_CHK:
 	    case VEC_TRAP:
-		info.si_code = FPE_INTOVF;
+		si_code = FPE_INTOVF;
 		sig = SIGFPE;
 		break;
 	    case VEC_TRACE:		/* ptrace single step */
-		info.si_code = TRAP_TRACE;
+		si_code = TRAP_TRACE;
 		sig = SIGTRAP;
 		break;
 	    case VEC_TRAP15:		/* breakpoint */
-		info.si_code = TRAP_BRKPT;
+		si_code = TRAP_BRKPT;
 		sig = SIGTRAP;
 		break;
 	    default:
-		info.si_code = ILL_ILLOPC;
+		si_code = ILL_ILLOPC;
 		sig = SIGILL;
 		break;
 	}
-	info.si_signo = sig;
-	info.si_errno = 0;
 	switch (fp->ptregs.format) {
 	    default:
-		info.si_addr = (void *) fp->ptregs.pc;
+		addr = (void __user *) fp->ptregs.pc;
 		break;
 	    case 2:
-		info.si_addr = (void *) fp->un.fmt2.iaddr;
+		addr = (void __user *) fp->un.fmt2.iaddr;
 		break;
 	    case 7:
-		info.si_addr = (void *) fp->un.fmt7.effaddr;
+		addr = (void __user *) fp->un.fmt7.effaddr;
 		break;
 	    case 9:
-		info.si_addr = (void *) fp->un.fmt9.iaddr;
+		addr = (void __user *) fp->un.fmt9.iaddr;
 		break;
 	    case 10:
-		info.si_addr = (void *) fp->un.fmta.daddr;
+		addr = (void __user *) fp->un.fmta.daddr;
 		break;
 	    case 11:
-		info.si_addr = (void *) fp->un.fmtb.daddr;
+		addr = (void __user*) fp->un.fmtb.daddr;
 		break;
 	}
-	force_sig_info (sig, &info, current);
+	force_sig_fault(sig, si_code, addr, current);
 }
 
 void die_if_kernel (char *str, struct pt_regs *fp, int nr)
@@ -1161,12 +1159,6 @@ asmlinkage void fpsp040_die(void)
 #ifdef CONFIG_M68KFPU_EMU
 asmlinkage void fpemu_signal(int signal, int code, void *addr)
 {
-	siginfo_t info;
-
-	info.si_signo = signal;
-	info.si_errno = 0;
-	info.si_code = code;
-	info.si_addr = addr;
-	force_sig_info(signal, &info, current);
+	force_sig_fault(signal, code, addr, current);
 }
 #endif
diff --git a/arch/m68k/mac/config.c b/arch/m68k/mac/config.c
index 0c3275aa0197..e522307db47c 100644
--- a/arch/m68k/mac/config.c
+++ b/arch/m68k/mac/config.c
@@ -1005,7 +1005,7 @@ int __init mac_platform_init(void)
 		struct resource swim_rsrc = {
 			.flags = IORESOURCE_MEM,
 			.start = (resource_size_t)swim_base,
-			.end   = (resource_size_t)swim_base + 0x2000,
+			.end   = (resource_size_t)swim_base + 0x1FFF,
 		};
 
 		platform_device_register_simple("swim", -1, &swim_rsrc, 1);
diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c
index 03253c4f8e6a..f2ff3779875a 100644
--- a/arch/m68k/mm/fault.c
+++ b/arch/m68k/mm/fault.c
@@ -21,35 +21,32 @@ extern void die_if_kernel(char *, struct pt_regs *, long);
 
 int send_fault_sig(struct pt_regs *regs)
 {
-	siginfo_t siginfo;
+	int signo, si_code;
+	void __user *addr;
 
-	clear_siginfo(&siginfo);
-	siginfo.si_signo = current->thread.signo;
-	siginfo.si_code = current->thread.code;
-	siginfo.si_addr = (void *)current->thread.faddr;
-	pr_debug("send_fault_sig: %p,%d,%d\n", siginfo.si_addr,
-		 siginfo.si_signo, siginfo.si_code);
+	signo = current->thread.signo;
+	si_code = current->thread.code;
+	addr = (void __user *)current->thread.faddr;
+	pr_debug("send_fault_sig: %p,%d,%d\n", addr, signo, si_code);
 
 	if (user_mode(regs)) {
-		force_sig_info(siginfo.si_signo,
-			       &siginfo, current);
+		force_sig_fault(signo, si_code, addr, current);
 	} else {
 		if (fixup_exception(regs))
 			return -1;
 
-		//if (siginfo.si_signo == SIGBUS)
-		//	force_sig_info(siginfo.si_signo,
-		//		       &siginfo, current);
+		//if (signo == SIGBUS)
+		//	force_sig_fault(si_signo, si_code, addr, current);
 
 		/*
 		 * Oops. The kernel tried to access some bad page. We'll have to
 		 * terminate things with extreme prejudice.
 		 */
-		if ((unsigned long)siginfo.si_addr < PAGE_SIZE)
+		if ((unsigned long)addr < PAGE_SIZE)
 			pr_alert("Unable to handle kernel NULL pointer dereference");
 		else
 			pr_alert("Unable to handle kernel access");
-		pr_cont(" at virtual address %p\n", siginfo.si_addr);
+		pr_cont(" at virtual address %p\n", addr);
 		die_if_kernel("Oops", regs, 0 /*error_code*/);
 		do_exit(SIGKILL);
 	}
diff --git a/arch/m68k/mm/kmap.c b/arch/m68k/mm/kmap.c
index c2a38321c96d..40a3b327da07 100644
--- a/arch/m68k/mm/kmap.c
+++ b/arch/m68k/mm/kmap.c
@@ -89,7 +89,8 @@ static inline void free_io_area(void *addr)
 	for (p = &iolist ; (tmp = *p) ; p = &tmp->next) {
 		if (tmp->addr == addr) {
 			*p = tmp->next;
-			__iounmap(tmp->addr, tmp->size);
+			/* remove gap added in get_io_area() */
+			__iounmap(tmp->addr, tmp->size - IO_SIZE);
 			kfree(tmp);
 			return;
 		}
@@ -125,6 +126,10 @@ void __iomem *__ioremap(unsigned long physaddr, unsigned long size, int cachefla
 			return (void __iomem *)physaddr;
 	}
 #endif
+#ifdef CONFIG_COLDFIRE
+	if (__cf_internalio(physaddr))
+		return (void __iomem *) physaddr;
+#endif
 
 #ifdef DEBUG
 	printk("ioremap: 0x%lx,0x%lx(%d) - ", physaddr, size, cacheflag);
@@ -235,6 +240,10 @@ void iounmap(void __iomem *addr)
 	     ((unsigned long)addr > 0x60000000)))
 			free_io_area((__force void *)addr);
 #else
+#ifdef CONFIG_COLDFIRE
+	if (cf_internalio(addr))
+		return;
+#endif
 	free_io_area((__force void *)addr);
 #endif
 }
diff --git a/arch/m68k/mvme147/config.c b/arch/m68k/mvme147/config.c
index 8778612d1f31..f8a710fd84cd 100644
--- a/arch/m68k/mvme147/config.c
+++ b/arch/m68k/mvme147/config.c
@@ -153,12 +153,14 @@ int mvme147_hwclk(int op, struct rtc_time *t)
 	if (!op) {
 		m147_rtc->ctrl = RTC_READ;
 		t->tm_year = bcd2int (m147_rtc->bcd_year);
-		t->tm_mon  = bcd2int (m147_rtc->bcd_mth);
+		t->tm_mon  = bcd2int(m147_rtc->bcd_mth) - 1;
 		t->tm_mday = bcd2int (m147_rtc->bcd_dom);
 		t->tm_hour = bcd2int (m147_rtc->bcd_hr);
 		t->tm_min  = bcd2int (m147_rtc->bcd_min);
 		t->tm_sec  = bcd2int (m147_rtc->bcd_sec);
 		m147_rtc->ctrl = 0;
+		if (t->tm_year < 70)
+			t->tm_year += 100;
 	}
 	return 0;
 }
diff --git a/arch/m68k/mvme16x/config.c b/arch/m68k/mvme16x/config.c
index 6fa06d4d16bf..4ffd9ef98de4 100644
--- a/arch/m68k/mvme16x/config.c
+++ b/arch/m68k/mvme16x/config.c
@@ -400,12 +400,14 @@ int mvme16x_hwclk(int op, struct rtc_time *t)
 	if (!op) {
 		rtc->ctrl = RTC_READ;
 		t->tm_year = bcd2int (rtc->bcd_year);
-		t->tm_mon  = bcd2int (rtc->bcd_mth);
+		t->tm_mon  = bcd2int(rtc->bcd_mth) - 1;
 		t->tm_mday = bcd2int (rtc->bcd_dom);
 		t->tm_hour = bcd2int (rtc->bcd_hr);
 		t->tm_min  = bcd2int (rtc->bcd_min);
 		t->tm_sec  = bcd2int (rtc->bcd_sec);
 		rtc->ctrl = 0;
+		if (t->tm_year < 70)
+			t->tm_year += 100;
 	}
 	return 0;
 }
diff --git a/arch/m68k/sun3/intersil.c b/arch/m68k/sun3/intersil.c
index 2cd0bcbe6f30..d911070af02a 100644
--- a/arch/m68k/sun3/intersil.c
+++ b/arch/m68k/sun3/intersil.c
@@ -48,9 +48,9 @@ int sun3_hwclk(int set, struct rtc_time *t)
 		todintersil->hour = t->tm_hour;
 		todintersil->minute = t->tm_min;
 		todintersil->second = t->tm_sec;
-		todintersil->month = t->tm_mon;
+		todintersil->month = t->tm_mon + 1;
 		todintersil->day = t->tm_mday;
-		todintersil->year = t->tm_year - 68;
+		todintersil->year = (t->tm_year - 68) % 100;
 		todintersil->weekday = t->tm_wday;
 	} else {
 		/* read clock */
@@ -58,10 +58,12 @@ int sun3_hwclk(int set, struct rtc_time *t)
 		t->tm_hour = todintersil->hour;
 		t->tm_min = todintersil->minute;
 		t->tm_sec = todintersil->second;
-		t->tm_mon = todintersil->month;
+		t->tm_mon = todintersil->month - 1;
 		t->tm_mday = todintersil->day;
 		t->tm_year = todintersil->year + 68;
 		t->tm_wday = todintersil->weekday;
+		if (t->tm_year < 70)
+			t->tm_year += 100;
 	}
 
 	intersil_clock->cmd_reg = START_VAL;
diff --git a/arch/m68k/sun3x/time.c b/arch/m68k/sun3x/time.c
index 7a2c53d9f779..047e2bcee3d7 100644
--- a/arch/m68k/sun3x/time.c
+++ b/arch/m68k/sun3x/time.c
@@ -52,8 +52,8 @@ int sun3x_hwclk(int set, struct rtc_time *t)
 		h->hour = bin2bcd(t->tm_hour);
 		h->wday = bin2bcd(t->tm_wday);
 		h->mday = bin2bcd(t->tm_mday);
-		h->month = bin2bcd(t->tm_mon);
-		h->year = bin2bcd(t->tm_year);
+		h->month = bin2bcd(t->tm_mon + 1);
+		h->year = bin2bcd(t->tm_year % 100);
 		h->csr &= ~C_WRITE;
 	} else {
 		h->csr |= C_READ;
@@ -62,9 +62,11 @@ int sun3x_hwclk(int set, struct rtc_time *t)
 		t->tm_hour = bcd2bin(h->hour);
 		t->tm_wday = bcd2bin(h->wday);
 		t->tm_mday = bcd2bin(h->mday);
-		t->tm_mon = bcd2bin(h->month);
+		t->tm_mon = bcd2bin(h->month) - 1;
 		t->tm_year = bcd2bin(h->year);
 		h->csr &= ~C_READ;
+		if (t->tm_year < 70)
+			t->tm_year += 100;
 	}
 
 	local_irq_restore(flags);
diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig
index 3817a3e2146c..d14782100088 100644
--- a/arch/microblaze/Kconfig
+++ b/arch/microblaze/Kconfig
@@ -19,7 +19,6 @@ config MICROBLAZE
 	select HAVE_ARCH_HASH
 	select HAVE_ARCH_KGDB
 	select HAVE_DEBUG_KMEMLEAK
-	select HAVE_DMA_API_DEBUG
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_FUNCTION_GRAPH_TRACER
diff --git a/arch/microblaze/Kconfig.debug b/arch/microblaze/Kconfig.debug
index 012e377330cd..331a3bb66297 100644
--- a/arch/microblaze/Kconfig.debug
+++ b/arch/microblaze/Kconfig.debug
@@ -8,14 +8,6 @@ config TRACE_IRQFLAGS_SUPPORT
 
 source "lib/Kconfig.debug"
 
-config EARLY_PRINTK
-	bool "Early printk function for kernel"
-	depends on SERIAL_UARTLITE_CONSOLE || SERIAL_8250_CONSOLE
-	default n
-	help
-	  This option turns on/off early printk messages to console.
-	  First Uartlite node is taken.
-
 config HEART_BEAT
 	bool "Heart beat function for kernel"
 	default n
diff --git a/arch/microblaze/boot/Makefile b/arch/microblaze/boot/Makefile
index fd46385a4c97..600e5a198bd2 100644
--- a/arch/microblaze/boot/Makefile
+++ b/arch/microblaze/boot/Makefile
@@ -22,17 +22,19 @@ $(obj)/linux.bin.gz: $(obj)/linux.bin FORCE
 quiet_cmd_cp = CP      $< $@$2
 	cmd_cp = cat $< >$@$2 || (rm -f $@ && echo false)
 
-quiet_cmd_strip = STRIP   $@
+quiet_cmd_strip = STRIP   $< $@$2
 	cmd_strip = $(STRIP) -K microblaze_start -K _end -K __log_buf \
-				-K _fdt_start vmlinux -o $@
+				-K _fdt_start $< -o $@$2
 
 UIMAGE_LOADADDR = $(CONFIG_KERNEL_BASE_ADDR)
+UIMAGE_IN = $@
+UIMAGE_OUT = $@.ub
 
 $(obj)/simpleImage.%: vmlinux FORCE
 	$(call if_changed,cp,.unstrip)
 	$(call if_changed,objcopy)
 	$(call if_changed,uimage)
-	$(call if_changed,strip)
-	@echo 'Kernel: $@ is ready' ' (#'`cat .version`')'
+	$(call if_changed,strip,.strip)
+	@echo 'Kernel: $(UIMAGE_OUT) is ready' ' (#'`cat .version`')'
 
 clean-files += simpleImage.*.unstrip linux.bin.ub
diff --git a/arch/microblaze/boot/dts/system.dts b/arch/microblaze/boot/dts/system.dts
index b620da23febb..8a420c6702eb 100644
--- a/arch/microblaze/boot/dts/system.dts
+++ b/arch/microblaze/boot/dts/system.dts
@@ -44,7 +44,7 @@
 	} ;
 	chosen {
 		bootargs = "console=ttyUL0,115200 highres=on";
-		linux,stdout-path = "/plb@0/serial@84000000";
+		stdout-path = "/plb@0/serial@84000000";
 	} ;
 	cpus {
 		#address-cells = <1>;
diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild
index 3c80a5a308ed..fe6a6c6e5003 100644
--- a/arch/microblaze/include/asm/Kbuild
+++ b/arch/microblaze/include/asm/Kbuild
@@ -2,6 +2,7 @@ generic-y += barrier.h
 generic-y += bitops.h
 generic-y += bug.h
 generic-y += bugs.h
+generic-y += compat.h
 generic-y += device.h
 generic-y += div64.h
 generic-y += emergency-restart.h
diff --git a/arch/microblaze/include/asm/cpuinfo.h b/arch/microblaze/include/asm/cpuinfo.h
index 3337417fcdca..8f4996730552 100644
--- a/arch/microblaze/include/asm/cpuinfo.h
+++ b/arch/microblaze/include/asm/cpuinfo.h
@@ -13,7 +13,7 @@
 #ifndef _ASM_MICROBLAZE_CPUINFO_H
 #define _ASM_MICROBLAZE_CPUINFO_H
 
-#include <asm/prom.h>
+#include <linux/of.h>
 
 /* CPU Version and FPGA Family code conversion table type */
 struct cpu_ver_key {
diff --git a/arch/microblaze/include/asm/pci.h b/arch/microblaze/include/asm/pci.h
index 5de871eb4a59..859c19828dd4 100644
--- a/arch/microblaze/include/asm/pci.h
+++ b/arch/microblaze/include/asm/pci.h
@@ -19,7 +19,6 @@
 #include <linux/scatterlist.h>
 
 #include <asm/io.h>
-#include <asm/prom.h>
 #include <asm/pci-bridge.h>
 
 #define PCIBIOS_MIN_IO		0x1000
@@ -62,16 +61,6 @@ extern int pci_mmap_legacy_page_range(struct pci_bus *bus,
 
 #define HAVE_PCI_LEGACY	1
 
-/* The PCI address space does equal the physical memory
- * address space (no IOMMU).  The IDE and SCSI device layers use
- * this boolean for bounce buffer decisions.
- */
-#define PCI_DMA_BUS_IS_PHYS     (1)
-
-extern void pcibios_claim_one_bus(struct pci_bus *b);
-
-extern void pcibios_finish_adding_to_bus(struct pci_bus *bus);
-
 extern void pcibios_resource_survey(void);
 
 struct file;
diff --git a/arch/microblaze/include/asm/prom.h b/arch/microblaze/include/asm/prom.h
deleted file mode 100644
index 2f03ac815851..000000000000
--- a/arch/microblaze/include/asm/prom.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Definitions for talking to the Open Firmware PROM on
- * Power Macintosh computers.
- *
- * Copyright (C) 1996-2005 Paul Mackerras.
- *
- * Updates for PPC64 by Peter Bergner & David Engebretsen, IBM Corp.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-#ifndef _ASM_MICROBLAZE_PROM_H
-#define _ASM_MICROBLAZE_PROM_H
-
-#include <linux/of.h>
-
-/* Other Prototypes */
-enum early_consoles {
-	UARTLITE = 1,
-	UART16550 = 2,
-};
-
-extern int of_early_console(void *version);
-
-#endif /* _ASM_MICROBLAZE_PROM_H */
diff --git a/arch/microblaze/include/asm/setup.h b/arch/microblaze/include/asm/setup.h
index 7c968c1d1729..d5384f6f36f7 100644
--- a/arch/microblaze/include/asm/setup.h
+++ b/arch/microblaze/include/asm/setup.h
@@ -19,16 +19,11 @@ extern char cmd_line[COMMAND_LINE_SIZE];
 
 extern char *klimit;
 
-int setup_early_printk(char *opt);
-void remap_early_printk(void);
-void disable_early_printk(void);
-
 void microblaze_heartbeat(void);
 void microblaze_setup_heartbeat(void);
 
 #   ifdef CONFIG_MMU
 extern void mmu_reset(void);
-extern void early_console_reg_tlb_alloc(unsigned int addr);
 #   endif /* CONFIG_MMU */
 
 extern void of_platform_reset_gpio_probe(void);
diff --git a/arch/microblaze/kernel/Makefile b/arch/microblaze/kernel/Makefile
index 0da76fa1ab17..7e99cf6984a1 100644
--- a/arch/microblaze/kernel/Makefile
+++ b/arch/microblaze/kernel/Makefile
@@ -22,7 +22,6 @@ obj-y += dma.o exceptions.o \
 
 obj-y += cpu/
 
-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
 obj-$(CONFIG_HEART_BEAT)	+= heartbeat.o
 obj-$(CONFIG_MODULES)		+= microblaze_ksyms.o module.o
 obj-$(CONFIG_MMU)		+= misc.o
diff --git a/arch/microblaze/kernel/dma.c b/arch/microblaze/kernel/dma.c
index c91e8cef98dd..3145e7dc8ab1 100644
--- a/arch/microblaze/kernel/dma.c
+++ b/arch/microblaze/kernel/dma.c
@@ -184,14 +184,3 @@ const struct dma_map_ops dma_nommu_ops = {
 	.sync_sg_for_device	= dma_nommu_sync_sg_for_device,
 };
 EXPORT_SYMBOL(dma_nommu_ops);
-
-/* Number of entries preallocated for DMA-API debugging */
-#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
-
-static int __init dma_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-
-	return 0;
-}
-fs_initcall(dma_init);
diff --git a/arch/microblaze/kernel/early_printk.c b/arch/microblaze/kernel/early_printk.c
deleted file mode 100644
index 365f2d53f1b2..000000000000
--- a/arch/microblaze/kernel/early_printk.c
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Early printk support for Microblaze.
- *
- * Copyright (C) 2007-2009 Michal Simek <monstr@monstr.eu>
- * Copyright (C) 2007-2009 PetaLogix
- * Copyright (C) 2003-2006 Yasushi SHOJI <yashi@atmark-techno.com>
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- */
-
-#include <linux/console.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/string.h>
-#include <linux/tty.h>
-#include <linux/io.h>
-#include <asm/processor.h>
-#include <linux/fcntl.h>
-#include <asm/setup.h>
-#include <asm/prom.h>
-
-static u32 base_addr;
-
-#ifdef CONFIG_SERIAL_UARTLITE_CONSOLE
-static void early_printk_uartlite_putc(char c)
-{
-	/*
-	 * Limit how many times we'll spin waiting for TX FIFO status.
-	 * This will prevent lockups if the base address is incorrectly
-	 * set, or any other issue on the UARTLITE.
-	 * This limit is pretty arbitrary, unless we are at about 10 baud
-	 * we'll never timeout on a working UART.
-	 */
-
-	unsigned retries = 1000000;
-	/* read status bit - 0x8 offset */
-	while (--retries && (in_be32(base_addr + 8) & (1 << 3)))
-		;
-
-	/* Only attempt the iowrite if we didn't timeout */
-	/* write to TX_FIFO - 0x4 offset */
-	if (retries)
-		out_be32(base_addr + 4, c & 0xff);
-}
-
-static void early_printk_uartlite_write(struct console *unused,
-					const char *s, unsigned n)
-{
-	while (*s && n-- > 0) {
-		if (*s == '\n')
-			early_printk_uartlite_putc('\r');
-		early_printk_uartlite_putc(*s);
-		s++;
-	}
-}
-
-static struct console early_serial_uartlite_console = {
-	.name = "earlyser",
-	.write = early_printk_uartlite_write,
-	.flags = CON_PRINTBUFFER | CON_BOOT,
-	.index = -1,
-};
-#endif /* CONFIG_SERIAL_UARTLITE_CONSOLE */
-
-#ifdef CONFIG_SERIAL_8250_CONSOLE
-static void early_printk_uart16550_putc(char c)
-{
-	/*
-	 * Limit how many times we'll spin waiting for TX FIFO status.
-	 * This will prevent lockups if the base address is incorrectly
-	 * set, or any other issue on the UARTLITE.
-	 * This limit is pretty arbitrary, unless we are at about 10 baud
-	 * we'll never timeout on a working UART.
-	 */
-
-	#define UART_LSR_TEMT	0x40 /* Transmitter empty */
-	#define UART_LSR_THRE	0x20 /* Transmit-hold-register empty */
-	#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE)
-
-	unsigned retries = 10000;
-
-	while (--retries &&
-		!((in_be32(base_addr + 0x14) & BOTH_EMPTY) == BOTH_EMPTY))
-		;
-
-	if (retries)
-		out_be32(base_addr, c & 0xff);
-}
-
-static void early_printk_uart16550_write(struct console *unused,
-					const char *s, unsigned n)
-{
-	while (*s && n-- > 0) {
-		if (*s == '\n')
-			early_printk_uart16550_putc('\r');
-		early_printk_uart16550_putc(*s);
-		s++;
-	}
-}
-
-static struct console early_serial_uart16550_console = {
-	.name = "earlyser",
-	.write = early_printk_uart16550_write,
-	.flags = CON_PRINTBUFFER | CON_BOOT,
-	.index = -1,
-};
-#endif /* CONFIG_SERIAL_8250_CONSOLE */
-
-int __init setup_early_printk(char *opt)
-{
-	int version = 0;
-
-	if (early_console)
-		return 1;
-
-	base_addr = of_early_console(&version);
-	if (base_addr) {
-#ifdef CONFIG_MMU
-		early_console_reg_tlb_alloc(base_addr);
-#endif
-		switch (version) {
-#ifdef CONFIG_SERIAL_UARTLITE_CONSOLE
-		case UARTLITE:
-			pr_info("Early console on uartlite at 0x%08x\n",
-								base_addr);
-			early_console = &early_serial_uartlite_console;
-			break;
-#endif
-#ifdef CONFIG_SERIAL_8250_CONSOLE
-		case UART16550:
-			pr_info("Early console on uart16650 at 0x%08x\n",
-								base_addr);
-			early_console = &early_serial_uart16550_console;
-			break;
-#endif
-		default:
-			pr_info("Unsupported early console %d\n",
-								version);
-			return 1;
-		}
-
-		register_console(early_console);
-		return 0;
-	}
-	return 1;
-}
-
-/* Remap early console to virtual address and do not allocate one TLB
- * only for early console because of performance degression */
-void __init remap_early_printk(void)
-{
-	if (!early_console)
-		return;
-	pr_info("early_printk_console remapping from 0x%x to ", base_addr);
-	base_addr = (u32) ioremap(base_addr, PAGE_SIZE);
-	pr_cont("0x%x\n", base_addr);
-
-#ifdef CONFIG_MMU
-	/*
-	 * Early console is on the top of skipped TLB entries
-	 * decrease tlb_skip size ensure that hardcoded TLB entry will be
-	 * used by generic algorithm
-	 * FIXME check if early console mapping is on the top by rereading
-	 * TLB entry and compare baseaddr
-	 *  mts  rtlbx, (tlb_skip - 1)
-	 *  nop
-	 *  mfs  rX, rtlblo
-	 *  nop
-	 *  cmp rX, orig_base_addr
-	 */
-	tlb_skip -= 1;
-#endif
-}
-
-void __init disable_early_printk(void)
-{
-	if (!early_console)
-		return;
-	pr_warn("disabling early console\n");
-	unregister_console(early_console);
-	early_console = NULL;
-}
diff --git a/arch/microblaze/kernel/exceptions.c b/arch/microblaze/kernel/exceptions.c
index e6f338d0496b..eafff21fcb0e 100644
--- a/arch/microblaze/kernel/exceptions.c
+++ b/arch/microblaze/kernel/exceptions.c
@@ -60,16 +60,10 @@ asmlinkage void sw_exception(struct pt_regs *regs)
 
 void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr)
 {
-	siginfo_t info;
-
 	if (kernel_mode(regs))
 		die("Exception in kernel mode", regs, signr);
 
-	info.si_signo = signr;
-	info.si_errno = 0;
-	info.si_code = code;
-	info.si_addr = (void __user *) addr;
-	force_sig_info(signr, &info, current);
+	force_sig_fault(signr, code, (void __user *)addr, current);
 }
 
 asmlinkage void full_exception(struct pt_regs *regs, unsigned int type,
diff --git a/arch/microblaze/kernel/misc.S b/arch/microblaze/kernel/misc.S
index 1dafddeb8a0b..6759af688451 100644
--- a/arch/microblaze/kernel/misc.S
+++ b/arch/microblaze/kernel/misc.S
@@ -63,38 +63,3 @@ _tlbie_1:
 	nop
 
 	.size  _tlbie, . - _tlbie
-
-/*
- * Allocate TLB entry for early console
- */
-.globl early_console_reg_tlb_alloc;
-.type  early_console_reg_tlb_alloc, @function
-.align 4;
-early_console_reg_tlb_alloc:
-	/*
-	 * Load a TLB entry for the UART, so that microblaze_progress() can use
-	 * the UARTs nice and early.  We use a 4k real==virtual mapping.
-	 */
-	lwi	r4, r0, tlb_skip
-	mts	rtlbx, r4 /* TLB slot 63 */
-
-	or	r4,r5,r0
-	andi	r4,r4,0xfffff000
-	ori	r4,r4,(TLB_WR|TLB_I|TLB_M|TLB_G)
-
-	andi	r5,r5,0xfffff000
-	ori	r5,r5,(TLB_VALID | TLB_PAGESZ(PAGESZ_4K))
-
-	mts	rtlblo,r4 /* Load the data portion of the entry */
-	nop
-	mts	rtlbhi,r5 /* Load the tag portion of the entry */
-	nop
-
-	lwi	r5, r0, tlb_skip
-	addik	r5, r5, 1
-	swi	r5, r0, tlb_skip
-
-	rtsd	r15, 8
-	nop
-
-	.size  early_console_reg_tlb_alloc, . - early_console_reg_tlb_alloc
diff --git a/arch/microblaze/kernel/platform.c b/arch/microblaze/kernel/platform.c
index b9529caa507a..2540d60610d9 100644
--- a/arch/microblaze/kernel/platform.c
+++ b/arch/microblaze/kernel/platform.c
@@ -12,7 +12,6 @@
 
 #include <linux/init.h>
 #include <linux/of_platform.h>
-#include <asm/prom.h>
 #include <asm/setup.h>
 
 static struct of_device_id xilinx_of_bus_ids[] __initdata = {
diff --git a/arch/microblaze/kernel/prom.c b/arch/microblaze/kernel/prom.c
index 68f099960ebc..c76c93b90b79 100644
--- a/arch/microblaze/kernel/prom.c
+++ b/arch/microblaze/kernel/prom.c
@@ -13,91 +13,11 @@
  *      2 of the License, or (at your option) any later version.
  */
 
-#include <stdarg.h>
-#include <linux/export.h>
 #include <linux/kernel.h>
 #include <linux/string.h>
-#include <linux/init.h>
-#include <linux/threads.h>
-#include <linux/spinlock.h>
-#include <linux/types.h>
-#include <linux/pci.h>
-#include <linux/stringify.h>
-#include <linux/delay.h>
-#include <linux/initrd.h>
-#include <linux/bitops.h>
-#include <linux/kexec.h>
-#include <linux/debugfs.h>
-#include <linux/irq.h>
 #include <linux/memblock.h>
 #include <linux/of_fdt.h>
 
-#include <asm/prom.h>
-#include <asm/page.h>
-#include <asm/processor.h>
-#include <asm/irq.h>
-#include <linux/io.h>
-#include <asm/mmu.h>
-#include <asm/pgtable.h>
-#include <asm/sections.h>
-#include <asm/pci-bridge.h>
-
-#ifdef CONFIG_EARLY_PRINTK
-static const char *stdout;
-
-static int __init early_init_dt_scan_chosen_serial(unsigned long node,
-				const char *uname, int depth, void *data)
-{
-	int l;
-	const char *p;
-
-	pr_debug("%s: depth: %d, uname: %s\n", __func__, depth, uname);
-
-	if (depth == 1 && (strcmp(uname, "chosen") == 0 ||
-				strcmp(uname, "chosen@0") == 0)) {
-		p = of_get_flat_dt_prop(node, "linux,stdout-path", &l);
-		if (p != NULL && l > 0)
-			stdout = p; /* store pointer to stdout-path */
-	}
-
-	if (stdout && strstr(stdout, uname)) {
-		p = of_get_flat_dt_prop(node, "compatible", &l);
-		pr_debug("Compatible string: %s\n", p);
-
-		if ((strncmp(p, "xlnx,xps-uart16550", 18) == 0) ||
-			(strncmp(p, "xlnx,axi-uart16550", 18) == 0)) {
-			unsigned int addr;
-
-			*(u32 *)data = UART16550;
-
-			addr = *(u32 *)of_get_flat_dt_prop(node, "reg", &l);
-			addr += *(u32 *)of_get_flat_dt_prop(node,
-							"reg-offset", &l);
-			/* clear register offset */
-			return be32_to_cpu(addr) & ~3;
-		}
-		if ((strncmp(p, "xlnx,xps-uartlite", 17) == 0) ||
-				(strncmp(p, "xlnx,opb-uartlite", 17) == 0) ||
-				(strncmp(p, "xlnx,axi-uartlite", 17) == 0) ||
-				(strncmp(p, "xlnx,mdm", 8) == 0)) {
-			const unsigned int *addrp;
-
-			*(u32 *)data = UARTLITE;
-
-			addrp = of_get_flat_dt_prop(node, "reg", &l);
-			return be32_to_cpup(addrp); /* return address */
-		}
-	}
-	return 0;
-}
-
-/* this function is looking for early console - Microblaze specific */
-int __init of_early_console(void *version)
-{
-	return of_scan_flat_dt(early_init_dt_scan_chosen_serial, version);
-}
-#endif
-
 void __init early_init_devtree(void *params)
 {
 	pr_debug(" -> early_init_devtree(%p)\n", params);
@@ -106,8 +26,6 @@ void __init early_init_devtree(void *params)
 	if (!strlen(boot_command_line))
 		strlcpy(boot_command_line, cmd_line, COMMAND_LINE_SIZE);
 
-	parse_early_param();
-
 	memblock_allow_resize();
 
 	pr_debug("Phys. mem: %lx\n", (unsigned long) memblock_phys_mem_size());
diff --git a/arch/microblaze/kernel/setup.c b/arch/microblaze/kernel/setup.c
index be98ffe28ca8..bbd6968ce55b 100644
--- a/arch/microblaze/kernel/setup.c
+++ b/arch/microblaze/kernel/setup.c
@@ -27,13 +27,12 @@
 #include <linux/param.h>
 #include <linux/pci.h>
 #include <linux/cache.h>
-#include <linux/of_platform.h>
+#include <linux/of.h>
 #include <linux/dma-mapping.h>
 #include <asm/cacheflush.h>
 #include <asm/entry.h>
 #include <asm/cpuinfo.h>
 
-#include <asm/prom.h>
 #include <asm/pgtable.h>
 
 DEFINE_PER_CPU(unsigned int, KSP);	/* Saved kernel stack pointer */
@@ -54,6 +53,9 @@ void __init setup_arch(char **cmdline_p)
 {
 	*cmdline_p = boot_command_line;
 
+	setup_memory();
+	parse_early_param();
+
 	console_verbose();
 
 	unflatten_device_tree();
@@ -62,13 +64,6 @@ void __init setup_arch(char **cmdline_p)
 
 	microblaze_cache_init();
 
-	setup_memory();
-
-#ifdef CONFIG_EARLY_PRINTK
-	/* remap early console to virtual address */
-	remap_early_printk();
-#endif
-
 	xilinx_pci_init();
 
 #if defined(CONFIG_DUMMY_CONSOLE)
@@ -133,10 +128,6 @@ void __init machine_early_init(const char *cmdline, unsigned int ram,
 /* initialize device tree for usage in early_printk */
 	early_init_devtree(_fdt_start);
 
-#ifdef CONFIG_EARLY_PRINTK
-	setup_early_printk(NULL);
-#endif
-
 	/* setup kernel_tlb after BSS cleaning
 	 * Maybe worth to move to asm code */
 	kernel_tlb = tlb0 + tlb1;
diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c
index f91b30f8aaa8..af607447c683 100644
--- a/arch/microblaze/mm/fault.c
+++ b/arch/microblaze/mm/fault.c
@@ -88,7 +88,6 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
 {
 	struct vm_area_struct *vma;
 	struct mm_struct *mm = current->mm;
-	siginfo_t info;
 	int code = SEGV_MAPERR;
 	int is_write = error_code & ESR_S;
 	int fault;
@@ -269,11 +268,6 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
 	/* User mode accesses cause a SIGSEGV */
 	if (user_mode(regs)) {
 		_exception(SIGSEGV, regs, code, address);
-/*		info.si_signo = SIGSEGV;
-		info.si_errno = 0;
-		info.si_code = code;
-		info.si_addr = (void *) address;
-		force_sig_info(SIGSEGV, &info, current);*/
 		return;
 	}
 
@@ -295,11 +289,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
 do_sigbus:
 	up_read(&mm->mmap_sem);
 	if (user_mode(regs)) {
-		info.si_signo = SIGBUS;
-		info.si_errno = 0;
-		info.si_code = BUS_ADRERR;
-		info.si_addr = (void __user *)address;
-		force_sig_info(SIGBUS, &info, current);
+		force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address, current);
 		return;
 	}
 	bad_page_fault(regs, address, SIGBUS);
diff --git a/arch/microblaze/pci/indirect_pci.c b/arch/microblaze/pci/indirect_pci.c
index ae4fca46c9f6..24030837a425 100644
--- a/arch/microblaze/pci/indirect_pci.c
+++ b/arch/microblaze/pci/indirect_pci.c
@@ -16,7 +16,6 @@
 #include <linux/init.h>
 
 #include <linux/io.h>
-#include <asm/prom.h>
 #include <asm/pci-bridge.h>
 
 static int
diff --git a/arch/microblaze/pci/pci-common.c b/arch/microblaze/pci/pci-common.c
index 161f9758c631..f34346d56095 100644
--- a/arch/microblaze/pci/pci-common.c
+++ b/arch/microblaze/pci/pci-common.c
@@ -915,67 +915,6 @@ void __init pcibios_resource_survey(void)
 	pci_assign_unassigned_resources();
 }
 
-/* This is used by the PCI hotplug driver to allocate resource
- * of newly plugged busses. We can try to consolidate with the
- * rest of the code later, for now, keep it as-is as our main
- * resource allocation function doesn't deal with sub-trees yet.
- */
-void pcibios_claim_one_bus(struct pci_bus *bus)
-{
-	struct pci_dev *dev;
-	struct pci_bus *child_bus;
-
-	list_for_each_entry(dev, &bus->devices, bus_list) {
-		int i;
-
-		for (i = 0; i < PCI_NUM_RESOURCES; i++) {
-			struct resource *r = &dev->resource[i];
-
-			if (r->parent || !r->start || !r->flags)
-				continue;
-
-			pr_debug("PCI: Claiming %s: ", pci_name(dev));
-			pr_debug("Resource %d: %016llx..%016llx [%x]\n",
-				 i, (unsigned long long)r->start,
-				 (unsigned long long)r->end,
-				 (unsigned int)r->flags);
-
-			if (pci_claim_resource(dev, i) == 0)
-				continue;
-
-			pci_claim_bridge_resource(dev, i);
-		}
-	}
-
-	list_for_each_entry(child_bus, &bus->children, node)
-		pcibios_claim_one_bus(child_bus);
-}
-EXPORT_SYMBOL_GPL(pcibios_claim_one_bus);
-
-
-/* pcibios_finish_adding_to_bus
- *
- * This is to be called by the hotplug code after devices have been
- * added to a bus, this include calling it for a PHB that is just
- * being added
- */
-void pcibios_finish_adding_to_bus(struct pci_bus *bus)
-{
-	pr_debug("PCI: Finishing adding to hotplug bus %04x:%02x\n",
-		 pci_domain_nr(bus), bus->number);
-
-	/* Allocate bus and devices resources */
-	pcibios_allocate_bus_resources(bus);
-	pcibios_claim_one_bus(bus);
-
-	/* Add new devices to global lists.  Register in proc, sysfs. */
-	pci_bus_add_devices(bus);
-
-	/* Fixup EEH */
-	/* eeh_add_device_tree_late(bus); */
-}
-EXPORT_SYMBOL_GPL(pcibios_finish_adding_to_bus);
-
 static void pcibios_setup_phb_resources(struct pci_controller *hose,
 					struct list_head *resources)
 {
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 225c95da23ce..fe98e459a416 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -22,6 +22,11 @@ config MIPS
 	select GENERIC_CPU_AUTOPROBE
 	select GENERIC_IRQ_PROBE
 	select GENERIC_IRQ_SHOW
+	select GENERIC_LIB_ASHLDI3
+	select GENERIC_LIB_ASHRDI3
+	select GENERIC_LIB_CMPDI2
+	select GENERIC_LIB_LSHRDI3
+	select GENERIC_LIB_UCMPDI2
 	select GENERIC_PCI_IOMAP
 	select GENERIC_SCHED_CLOCK if !CAVIUM_OCTEON_SOC
 	select GENERIC_SMP_IDLE_THREAD
@@ -42,7 +47,6 @@ config MIPS
 	select HAVE_C_RECORDMCOUNT
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DEBUG_STACKOVERFLOW
-	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_CONTIGUOUS
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_EXIT_THREAD
@@ -132,7 +136,7 @@ config MIPS_GENERIC
 
 config MIPS_ALCHEMY
 	bool "Alchemy processor based machines"
-	select ARCH_PHYS_ADDR_T_64BIT
+	select PHYS_ADDR_T_64BIT
 	select CEVT_R4K
 	select CSRC_R4K
 	select IRQ_MIPS_CPU
@@ -890,7 +894,7 @@ config CAVIUM_OCTEON_SOC
 	bool "Cavium Networks Octeon SoC based boards"
 	select CEVT_R4K
 	select ARCH_HAS_PHYS_TO_DMA
-	select ARCH_PHYS_ADDR_T_64BIT
+	select PHYS_ADDR_T_64BIT
 	select DMA_COHERENT
 	select SYS_SUPPORTS_64BIT_KERNEL
 	select SYS_SUPPORTS_BIG_ENDIAN
@@ -912,6 +916,7 @@ config CAVIUM_OCTEON_SOC
 	select MIPS_NR_CPU_NR_MAP_1024
 	select BUILTIN_DTB
 	select MTD_COMPLEX_MAPPINGS
+	select SWIOTLB
 	select SYS_SUPPORTS_RELOCATABLE
 	help
 	  This option supports all of the Octeon reference boards from Cavium
@@ -936,7 +941,7 @@ config NLM_XLR_BOARD
 	select SWAP_IO_SPACE
 	select SYS_SUPPORTS_32BIT_KERNEL
 	select SYS_SUPPORTS_64BIT_KERNEL
-	select ARCH_PHYS_ADDR_T_64BIT
+	select PHYS_ADDR_T_64BIT
 	select SYS_SUPPORTS_BIG_ENDIAN
 	select SYS_SUPPORTS_HIGHMEM
 	select DMA_COHERENT
@@ -962,7 +967,7 @@ config NLM_XLP_BOARD
 	select HW_HAS_PCI
 	select SYS_SUPPORTS_32BIT_KERNEL
 	select SYS_SUPPORTS_64BIT_KERNEL
-	select ARCH_PHYS_ADDR_T_64BIT
+	select PHYS_ADDR_T_64BIT
 	select GPIOLIB
 	select SYS_SUPPORTS_BIG_ENDIAN
 	select SYS_SUPPORTS_LITTLE_ENDIAN
@@ -1101,9 +1106,6 @@ config GPIO_TXX9
 config FW_CFE
 	bool
 
-config ARCH_DMA_ADDR_T_64BIT
-	def_bool (HIGHMEM && ARCH_PHYS_ADDR_T_64BIT) || 64BIT
-
 config ARCH_SUPPORTS_UPROBES
 	bool
 
@@ -1122,9 +1124,6 @@ config DMA_NONCOHERENT
 	bool
 	select NEED_DMA_MAP_STATE
 
-config NEED_DMA_MAP_STATE
-	bool
-
 config SYS_HAS_EARLY_PRINTK
 	bool
 
@@ -1373,6 +1372,7 @@ config CPU_LOONGSON3
 	select MIPS_PGD_C0_CONTEXT
 	select MIPS_L1_CACHE_SHIFT_6
 	select GPIOLIB
+	select SWIOTLB
 	help
 		The Loongson 3 processor implements the MIPS64R2 instruction
 		set with many extensions.
@@ -1770,7 +1770,7 @@ config CPU_MIPS32_R5_XPA
 	depends on SYS_SUPPORTS_HIGHMEM
 	select XPA
 	select HIGHMEM
-	select ARCH_PHYS_ADDR_T_64BIT
+	select PHYS_ADDR_T_64BIT
 	default n
 	help
 	  Choose this option if you want to enable the Extended Physical
@@ -2402,9 +2402,6 @@ config SB1_PASS_2_1_WORKAROUNDS
 	default y
 
 
-config ARCH_PHYS_ADDR_T_64BIT
-       bool
-
 choice
 	prompt "SmartMIPS or microMIPS ASE support"
 
@@ -2556,7 +2553,7 @@ config ARCH_DISCONTIGMEM_ENABLE
 	  Say Y to support efficient handling of discontiguous physical memory,
 	  for architectures which are either NUMA (Non-Uniform Memory Access)
 	  or have huge holes in the physical address space for other reasons.
-	  See <file:Documentation/vm/numa> for more.
+	  See <file:Documentation/vm/numa.rst> for more.
 
 config ARCH_SPARSEMEM_ENABLE
 	bool
diff --git a/arch/mips/alchemy/board-gpr.c b/arch/mips/alchemy/board-gpr.c
index 4e79dbd54a33..fa75d75b5ba9 100644
--- a/arch/mips/alchemy/board-gpr.c
+++ b/arch/mips/alchemy/board-gpr.c
@@ -29,7 +29,7 @@
 #include <linux/leds.h>
 #include <linux/gpio.h>
 #include <linux/i2c.h>
-#include <linux/i2c-gpio.h>
+#include <linux/platform_data/i2c-gpio.h>
 #include <linux/gpio/machine.h>
 #include <asm/bootinfo.h>
 #include <asm/idle.h>
diff --git a/arch/mips/bcm47xx/board.c b/arch/mips/bcm47xx/board.c
index edfaef0d73a4..a80910d2738c 100644
--- a/arch/mips/bcm47xx/board.c
+++ b/arch/mips/bcm47xx/board.c
@@ -172,6 +172,8 @@ struct bcm47xx_board_type_list1 bcm47xx_board_list_board_id[] __initconst = {
 	{{BCM47XX_BOARD_NETGEAR_WNDR4000, "Netgear WNDR4000"}, "U12H181T00_NETGEAR"},
 	{{BCM47XX_BOARD_NETGEAR_WNDR4500V1, "Netgear WNDR4500 V1"}, "U12H189T00_NETGEAR"},
 	{{BCM47XX_BOARD_NETGEAR_WNDR4500V2, "Netgear WNDR4500 V2"}, "U12H224T00_NETGEAR"},
+	{{BCM47XX_BOARD_NETGEAR_WNR1000_V3, "Netgear WNR1000 V3"}, "U12H139T00_NETGEAR"},
+	{{BCM47XX_BOARD_NETGEAR_WNR1000_V3, "Netgear WNR1000 V3"}, "U12H139T50_NETGEAR"},
 	{{BCM47XX_BOARD_NETGEAR_WNR2000, "Netgear WNR2000"}, "U12H114T00_NETGEAR"},
 	{{BCM47XX_BOARD_NETGEAR_WNR3500L, "Netgear WNR3500L"}, "U12H136T99_NETGEAR"},
 	{{BCM47XX_BOARD_NETGEAR_WNR3500U, "Netgear WNR3500U"}, "U12H136T00_NETGEAR"},
diff --git a/arch/mips/bcm47xx/buttons.c b/arch/mips/bcm47xx/buttons.c
index 88d400d256c4..977990a609ba 100644
--- a/arch/mips/bcm47xx/buttons.c
+++ b/arch/mips/bcm47xx/buttons.c
@@ -411,6 +411,12 @@ bcm47xx_buttons_netgear_wndr4500v1[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
+static const struct gpio_keys_button
+bcm47xx_buttons_netgear_wnr1000_v3[] __initconst = {
+	BCM47XX_GPIO_KEY(2, KEY_WPS_BUTTON),
+	BCM47XX_GPIO_KEY(3, KEY_RESTART),
+};
+
 static const struct gpio_keys_button
 bcm47xx_buttons_netgear_wnr3500lv1[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_RESTART),
@@ -670,6 +676,9 @@ int __init bcm47xx_buttons_register(void)
 	case BCM47XX_BOARD_NETGEAR_WNDR4500V1:
 		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wndr4500v1);
 		break;
+	case BCM47XX_BOARD_NETGEAR_WNR1000_V3:
+		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wnr1000_v3);
+		break;
 	case BCM47XX_BOARD_NETGEAR_WNR3500L:
 		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wnr3500lv1);
 		break;
diff --git a/arch/mips/bcm47xx/leds.c b/arch/mips/bcm47xx/leds.c
index 34a7b3fbdfd9..d85fcdac8bf0 100644
--- a/arch/mips/bcm47xx/leds.c
+++ b/arch/mips/bcm47xx/leds.c
@@ -497,6 +497,12 @@ bcm47xx_leds_netgear_wndr4500v1[] __initconst = {
 	BCM47XX_GPIO_LED(14, "green", "usb2", 1, LEDS_GPIO_DEFSTATE_OFF),
 };
 
+static const struct gpio_led
+bcm47xx_leds_netgear_wnr1000_v3[] __initconst = {
+	BCM47XX_GPIO_LED(0, "blue", "wlan", 0, LEDS_GPIO_DEFSTATE_OFF),
+	BCM47XX_GPIO_LED(1, "green", "wps", 0, LEDS_GPIO_DEFSTATE_OFF),
+};
+
 static const struct gpio_led
 bcm47xx_leds_netgear_wnr3500lv1[] __initconst = {
 	BCM47XX_GPIO_LED(0, "blue", "wlan", 1, LEDS_GPIO_DEFSTATE_OFF),
@@ -532,7 +538,7 @@ bcm47xx_leds_simpletech_simpleshare[] __initconst = {
  * Init
  **************************************************/
 
-static struct gpio_led_platform_data bcm47xx_leds_pdata;
+static struct gpio_led_platform_data bcm47xx_leds_pdata __initdata;
 
 #define bcm47xx_set_pdata(dev_leds) do {				\
 	bcm47xx_leds_pdata.leds = dev_leds;				\
@@ -758,6 +764,9 @@ void __init bcm47xx_leds_register(void)
 	case BCM47XX_BOARD_NETGEAR_WNDR4500V1:
 		bcm47xx_set_pdata(bcm47xx_leds_netgear_wndr4500v1);
 		break;
+	case BCM47XX_BOARD_NETGEAR_WNR1000_V3:
+		bcm47xx_set_pdata(bcm47xx_leds_netgear_wnr1000_v3);
+		break;
 	case BCM47XX_BOARD_NETGEAR_WNR3500L:
 		bcm47xx_set_pdata(bcm47xx_leds_netgear_wnr3500lv1);
 		break;
diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile
index adce180f3ee4..abe77add8789 100644
--- a/arch/mips/boot/compressed/Makefile
+++ b/arch/mips/boot/compressed/Makefile
@@ -46,10 +46,13 @@ $(obj)/uart-ath79.c: $(srctree)/arch/mips/ath79/early_printk.c
 
 vmlinuzobjs-$(CONFIG_KERNEL_XZ) += $(obj)/ashldi3.o $(obj)/bswapsi.o
 
-extra-y += ashldi3.c bswapsi.c
-$(obj)/ashldi3.o $(obj)/bswapsi.o: KBUILD_CFLAGS += -I$(srctree)/arch/mips/lib
-$(obj)/ashldi3.c $(obj)/bswapsi.c: $(obj)/%.c: $(srctree)/arch/mips/lib/%.c
-	$(call cmd,shipped)
+extra-y += ashldi3.c
+$(obj)/ashldi3.c: $(obj)/%.c: $(srctree)/lib/%.c FORCE
+	$(call if_changed,shipped)
+
+extra-y += bswapsi.c
+$(obj)/bswapsi.c: $(obj)/%.c: $(srctree)/arch/mips/lib/%.c FORCE
+	$(call if_changed,shipped)
 
 targets := $(notdir $(vmlinuzobjs-y))
 
diff --git a/arch/mips/boot/dts/brcm/Makefile b/arch/mips/boot/dts/brcm/Makefile
index d8787c9a499e..d85f446cc0ce 100644
--- a/arch/mips/boot/dts/brcm/Makefile
+++ b/arch/mips/boot/dts/brcm/Makefile
@@ -34,4 +34,4 @@ dtb-$(CONFIG_DT_NONE) += \
 	bcm97425svmb.dtb \
 	bcm97435svmb.dtb
 
-obj-y				+= $(patsubst %.dtb, %.dtb.o, $(dtb-y))
+obj-$(CONFIG_BUILTIN_DTB)	+= $(addsuffix .o, $(dtb-y))
diff --git a/arch/mips/boot/dts/cavium-octeon/Makefile b/arch/mips/boot/dts/cavium-octeon/Makefile
index 24a8efcd7b03..17aef35f311b 100644
--- a/arch/mips/boot/dts/cavium-octeon/Makefile
+++ b/arch/mips/boot/dts/cavium-octeon/Makefile
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
 dtb-$(CONFIG_CAVIUM_OCTEON_SOC)	+= octeon_3xxx.dtb octeon_68xx.dtb
 
-obj-y				+= $(patsubst %.dtb, %.dtb.o, $(dtb-y))
+obj-$(CONFIG_BUILTIN_DTB)	+= $(addsuffix .o, $(dtb-y))
diff --git a/arch/mips/boot/dts/ingenic/Makefile b/arch/mips/boot/dts/ingenic/Makefile
index 5b1361a89e02..9cc48441eb71 100644
--- a/arch/mips/boot/dts/ingenic/Makefile
+++ b/arch/mips/boot/dts/ingenic/Makefile
@@ -3,4 +3,4 @@ dtb-$(CONFIG_JZ4740_QI_LB60)	+= qi_lb60.dtb
 dtb-$(CONFIG_JZ4770_GCW0)	+= gcw0.dtb
 dtb-$(CONFIG_JZ4780_CI20)	+= ci20.dtb
 
-obj-y				+= $(patsubst %.dtb, %.dtb.o, $(dtb-y))
+obj-$(CONFIG_BUILTIN_DTB)	+= $(addsuffix .o, $(dtb-y))
diff --git a/arch/mips/boot/dts/ingenic/ci20.dts b/arch/mips/boot/dts/ingenic/ci20.dts
index 38078594cf97..50cff3cbcc6d 100644
--- a/arch/mips/boot/dts/ingenic/ci20.dts
+++ b/arch/mips/boot/dts/ingenic/ci20.dts
@@ -36,6 +36,28 @@
 	clock-frequency = <48000000>;
 };
 
+&mmc0 {
+	status = "okay";
+
+	bus-width = <4>;
+	max-frequency = <50000000>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pins_mmc0>;
+
+	cd-gpios = <&gpf 20 GPIO_ACTIVE_LOW>;
+};
+
+&mmc1 {
+	status = "okay";
+
+	bus-width = <4>;
+	max-frequency = <50000000>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pins_mmc1>;
+};
+
 &uart0 {
 	status = "okay";
 
@@ -203,4 +225,16 @@
 		groups = "nemc-cs6";
 		bias-disable;
 	};
+
+	pins_mmc0: mmc0 {
+		function = "mmc0";
+		groups = "mmc0-1bit-e", "mmc0-4bit-e";
+		bias-disable;
+	};
+
+	pins_mmc1: mmc1 {
+		function = "mmc1";
+		groups = "mmc1-1bit-d", "mmc1-4bit-d";
+		bias-disable;
+	};
 };
diff --git a/arch/mips/boot/dts/ingenic/jz4740.dtsi b/arch/mips/boot/dts/ingenic/jz4740.dtsi
index cd5185bb90ae..26c6b561d6f7 100644
--- a/arch/mips/boot/dts/ingenic/jz4740.dtsi
+++ b/arch/mips/boot/dts/ingenic/jz4740.dtsi
@@ -45,6 +45,14 @@
 		#clock-cells = <1>;
 	};
 
+	watchdog: watchdog@10002000 {
+		compatible = "ingenic,jz4740-watchdog";
+		reg = <0x10002000 0x10>;
+
+		clocks = <&cgu JZ4740_CLK_RTC>;
+		clock-names = "rtc";
+	};
+
 	rtc_dev: rtc@10003000 {
 		compatible = "ingenic,jz4740-rtc";
 		reg = <0x10003000 0x40>;
diff --git a/arch/mips/boot/dts/ingenic/jz4780.dtsi b/arch/mips/boot/dts/ingenic/jz4780.dtsi
index 9b5794667aee..aa4e8f75ff5d 100644
--- a/arch/mips/boot/dts/ingenic/jz4780.dtsi
+++ b/arch/mips/boot/dts/ingenic/jz4780.dtsi
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <dt-bindings/clock/jz4780-cgu.h>
+#include <dt-bindings/dma/jz4780-dma.h>
 
 / {
 	#address-cells = <1>;
@@ -221,7 +222,10 @@
 
 	watchdog: watchdog@10002000 {
 		compatible = "ingenic,jz4780-watchdog";
-		reg = <0x10002000 0x100>;
+		reg = <0x10002000 0x10>;
+
+		clocks = <&cgu JZ4780_CLK_RTCLK>;
+		clock-names = "rtc";
 	};
 
 	nemc: nemc@13410000 {
@@ -241,6 +245,57 @@
 		status = "disabled";
 	};
 
+	dma: dma@13420000 {
+		compatible = "ingenic,jz4780-dma";
+		reg = <0x13420000 0x10000>;
+		#dma-cells = <2>;
+
+		interrupt-parent = <&intc>;
+		interrupts = <10>;
+
+		clocks = <&cgu JZ4780_CLK_PDMA>;
+	};
+
+	mmc0: mmc@13450000 {
+		compatible = "ingenic,jz4780-mmc";
+		reg = <0x13450000 0x1000>;
+
+		interrupt-parent = <&intc>;
+		interrupts = <37>;
+
+		clocks = <&cgu JZ4780_CLK_MSC0>;
+		clock-names = "mmc";
+
+		cap-sd-highspeed;
+		cap-mmc-highspeed;
+		cap-sdio-irq;
+		dmas = <&dma JZ4780_DMA_MSC0_RX 0xffffffff>,
+		       <&dma JZ4780_DMA_MSC0_TX 0xffffffff>;
+		dma-names = "rx", "tx";
+
+		status = "disabled";
+	};
+
+	mmc1: mmc@13460000 {
+		compatible = "ingenic,jz4780-mmc";
+		reg = <0x13460000 0x1000>;
+
+		interrupt-parent = <&intc>;
+		interrupts = <36>;
+
+		clocks = <&cgu JZ4780_CLK_MSC1>;
+		clock-names = "mmc";
+
+		cap-sd-highspeed;
+		cap-mmc-highspeed;
+		cap-sdio-irq;
+		dmas = <&dma JZ4780_DMA_MSC1_RX 0xffffffff>,
+		       <&dma JZ4780_DMA_MSC1_TX 0xffffffff>;
+		dma-names = "rx", "tx";
+
+		status = "disabled";
+	};
+
 	bch: bch@134d0000 {
 		compatible = "ingenic,jz4780-bch";
 		reg = <0x134d0000 0x10000>;
diff --git a/arch/mips/boot/dts/lantiq/Makefile b/arch/mips/boot/dts/lantiq/Makefile
index 51ab9c1dff42..f5dfc06242b9 100644
--- a/arch/mips/boot/dts/lantiq/Makefile
+++ b/arch/mips/boot/dts/lantiq/Makefile
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
 dtb-$(CONFIG_DT_EASY50712)	+= easy50712.dtb
 
-obj-y				+= $(patsubst %.dtb, %.dtb.o, $(dtb-y))
+obj-$(CONFIG_BUILTIN_DTB)	+= $(addsuffix .o, $(dtb-y))
diff --git a/arch/mips/boot/dts/mscc/Makefile b/arch/mips/boot/dts/mscc/Makefile
index c51164537c02..3c6aed9f5439 100644
--- a/arch/mips/boot/dts/mscc/Makefile
+++ b/arch/mips/boot/dts/mscc/Makefile
@@ -1,3 +1,3 @@
 dtb-$(CONFIG_LEGACY_BOARD_OCELOT)	+= ocelot_pcb123.dtb
 
-obj-y				+= $(patsubst %.dtb, %.dtb.o, $(dtb-y))
+obj-$(CONFIG_BUILTIN_DTB)	+= $(addsuffix .o, $(dtb-y))
diff --git a/arch/mips/boot/dts/mscc/ocelot.dtsi b/arch/mips/boot/dts/mscc/ocelot.dtsi
index dd239cab2f9d..4f33dbc67348 100644
--- a/arch/mips/boot/dts/mscc/ocelot.dtsi
+++ b/arch/mips/boot/dts/mscc/ocelot.dtsi
@@ -91,6 +91,72 @@
 			status = "disabled";
 		};
 
+		switch@1010000 {
+			compatible = "mscc,vsc7514-switch";
+			reg = <0x1010000 0x10000>,
+			      <0x1030000 0x10000>,
+			      <0x1080000 0x100>,
+			      <0x10d0000 0x10000>,
+			      <0x11e0000 0x100>,
+			      <0x11f0000 0x100>,
+			      <0x1200000 0x100>,
+			      <0x1210000 0x100>,
+			      <0x1220000 0x100>,
+			      <0x1230000 0x100>,
+			      <0x1240000 0x100>,
+			      <0x1250000 0x100>,
+			      <0x1260000 0x100>,
+			      <0x1270000 0x100>,
+			      <0x1280000 0x100>,
+			      <0x1800000 0x80000>,
+			      <0x1880000 0x10000>;
+			reg-names = "sys", "rew", "qs", "hsio", "port0",
+				    "port1", "port2", "port3", "port4", "port5",
+				    "port6", "port7", "port8", "port9", "port10",
+				    "qsys", "ana";
+			interrupts = <21 22>;
+			interrupt-names = "xtr", "inj";
+
+			ethernet-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port0: port@0 {
+					reg = <0>;
+				};
+				port1: port@1 {
+					reg = <1>;
+				};
+				port2: port@2 {
+					reg = <2>;
+				};
+				port3: port@3 {
+					reg = <3>;
+				};
+				port4: port@4 {
+					reg = <4>;
+				};
+				port5: port@5 {
+					reg = <5>;
+				};
+				port6: port@6 {
+					reg = <6>;
+				};
+				port7: port@7 {
+					reg = <7>;
+				};
+				port8: port@8 {
+					reg = <8>;
+				};
+				port9: port@9 {
+					reg = <9>;
+				};
+				port10: port@10 {
+					reg = <10>;
+				};
+			};
+		};
+
 		reset@1070008 {
 			compatible = "mscc,ocelot-chip-reset";
 			reg = <0x1070008 0x4>;
@@ -113,5 +179,27 @@
 				function = "uart2";
 			};
 		};
+
+		mdio0: mdio@107009c {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "mscc,ocelot-miim";
+			reg = <0x107009c 0x36>, <0x10700f0 0x8>;
+			interrupts = <14>;
+			status = "disabled";
+
+			phy0: ethernet-phy@0 {
+				reg = <0>;
+			};
+			phy1: ethernet-phy@1 {
+				reg = <1>;
+			};
+			phy2: ethernet-phy@2 {
+				reg = <2>;
+			};
+			phy3: ethernet-phy@3 {
+				reg = <3>;
+			};
+		};
 	};
 };
diff --git a/arch/mips/boot/dts/mscc/ocelot_pcb123.dts b/arch/mips/boot/dts/mscc/ocelot_pcb123.dts
index 29d6414f8886..4ccd65379059 100644
--- a/arch/mips/boot/dts/mscc/ocelot_pcb123.dts
+++ b/arch/mips/boot/dts/mscc/ocelot_pcb123.dts
@@ -25,3 +25,23 @@
 &uart2 {
 	status = "okay";
 };
+
+&mdio0 {
+	status = "okay";
+};
+
+&port0 {
+	phy-handle = <&phy0>;
+};
+
+&port1 {
+	phy-handle = <&phy1>;
+};
+
+&port2 {
+	phy-handle = <&phy2>;
+};
+
+&port3 {
+	phy-handle = <&phy3>;
+};
diff --git a/arch/mips/boot/dts/mti/Makefile b/arch/mips/boot/dts/mti/Makefile
index 3508720cb6d9..b5f7426998b1 100644
--- a/arch/mips/boot/dts/mti/Makefile
+++ b/arch/mips/boot/dts/mti/Makefile
@@ -2,4 +2,4 @@
 dtb-$(CONFIG_MIPS_MALTA)	+= malta.dtb
 dtb-$(CONFIG_LEGACY_BOARD_SEAD3)	+= sead3.dtb
 
-obj-y				+= $(patsubst %.dtb, %.dtb.o, $(dtb-y))
+obj-$(CONFIG_BUILTIN_DTB)	+= $(addsuffix .o, $(dtb-y))
diff --git a/arch/mips/boot/dts/netlogic/Makefile b/arch/mips/boot/dts/netlogic/Makefile
index d630b27950f0..45af4224494f 100644
--- a/arch/mips/boot/dts/netlogic/Makefile
+++ b/arch/mips/boot/dts/netlogic/Makefile
@@ -5,4 +5,4 @@ dtb-$(CONFIG_DT_XLP_FVP)	+= xlp_fvp.dtb
 dtb-$(CONFIG_DT_XLP_GVP)	+= xlp_gvp.dtb
 dtb-$(CONFIG_DT_XLP_RVP)	+= xlp_rvp.dtb
 
-obj-y				+= $(patsubst %.dtb, %.dtb.o, $(dtb-y))
+obj-$(CONFIG_BUILTIN_DTB)	+= $(addsuffix .o, $(dtb-y))
diff --git a/arch/mips/boot/dts/pic32/Makefile b/arch/mips/boot/dts/pic32/Makefile
index ba9bcef8fde9..fb57f36324db 100644
--- a/arch/mips/boot/dts/pic32/Makefile
+++ b/arch/mips/boot/dts/pic32/Makefile
@@ -4,4 +4,4 @@ dtb-$(CONFIG_DTB_PIC32_MZDA_SK)		+= pic32mzda_sk.dtb
 dtb-$(CONFIG_DTB_PIC32_NONE)		+= \
 					pic32mzda_sk.dtb
 
-obj-y				+= $(patsubst %.dtb, %.dtb.o, $(dtb-y))
+obj-$(CONFIG_BUILTIN_DTB)	+= $(addsuffix .o, $(dtb-y))
diff --git a/arch/mips/boot/dts/ralink/Makefile b/arch/mips/boot/dts/ralink/Makefile
index 94bee5b38b53..6c26dfa0a903 100644
--- a/arch/mips/boot/dts/ralink/Makefile
+++ b/arch/mips/boot/dts/ralink/Makefile
@@ -6,4 +6,4 @@ dtb-$(CONFIG_DTB_MT7620A_EVAL)	+= mt7620a_eval.dtb
 dtb-$(CONFIG_DTB_OMEGA2P)	+= omega2p.dtb
 dtb-$(CONFIG_DTB_VOCORE2)	+= vocore2.dtb
 
-obj-y				+= $(patsubst %.dtb, %.dtb.o, $(dtb-y))
+obj-$(CONFIG_BUILTIN_DTB)	+= $(addsuffix .o, $(dtb-y))
diff --git a/arch/mips/cavium-octeon/Kconfig b/arch/mips/cavium-octeon/Kconfig
index b5eee1a57d6c..3a74b4a6a09b 100644
--- a/arch/mips/cavium-octeon/Kconfig
+++ b/arch/mips/cavium-octeon/Kconfig
@@ -11,21 +11,26 @@ config CAVIUM_CN63XXP1
 	  non-CN63XXP1 hardware, so it is recommended to select "n"
 	  unless it is known the workarounds are needed.
 
-config CAVIUM_OCTEON_CVMSEG_SIZE
-	int "Number of L1 cache lines reserved for CVMSEG memory"
-	range 0 54
-	default 1
-	help
-	  CVMSEG LM is a segment that accesses portions of the dcache as a
-	  local memory; the larger CVMSEG is, the smaller the cache is.
-	  This selects the size of CVMSEG LM, which is in cache blocks. The
-	  legally range is from zero to 54 cache blocks (i.e. CVMSEG LM is
-	  between zero and 6192 bytes).
-
 endif # CPU_CAVIUM_OCTEON
 
 if CAVIUM_OCTEON_SOC
 
+config CAVIUM_OCTEON_EXTRA_CVMSEG
+	int "Number of extra L1 cache lines reserved for CVMSEG memory"
+	range 0 50
+	default 0
+	help
+	  CVMSEG LM is a segment that accesses portions of the dcache
+	  as a local memory; the larger CVMSEG is, the smaller the
+	  cache is.  The kernel uses two or three blocks (one for TLB
+	  exception handlers, one for driver IOBDMA operations, and on
+	  models that need it, one for LMTDMA operations). This
+	  selects an optional extra number of CVMSEG lines for use by
+	  other software.
+
+	  Normally no extra lines are required, and this parameter
+	  should be set to zero.
+
 config CAVIUM_OCTEON_LOCK_L2
 	bool "Lock often used kernel code in the L2"
 	default "y"
@@ -67,18 +72,6 @@ config CAVIUM_OCTEON_LOCK_L2_MEMCPY
 	help
 	  Lock the kernel's implementation of memcpy() into L2.
 
-config IOMMU_HELPER
-	bool
-
-config NEED_SG_DMA_LENGTH
-	bool
-
-config SWIOTLB
-	def_bool y
-	select DMA_DIRECT_OPS
-	select IOMMU_HELPER
-	select NEED_SG_DMA_LENGTH
-
 config OCTEON_ILM
 	tristate "Module to measure interrupt latency using Octeon CIU Timer"
 	help
diff --git a/arch/mips/cavium-octeon/Makefile b/arch/mips/cavium-octeon/Makefile
index 7c02e542959a..28c0bb75d1a4 100644
--- a/arch/mips/cavium-octeon/Makefile
+++ b/arch/mips/cavium-octeon/Makefile
@@ -10,6 +10,7 @@
 #
 
 obj-y := cpu.o setup.o octeon-platform.o octeon-irq.o csrc-octeon.o
+obj-y += resource-mgr.o
 obj-y += dma-octeon.o
 obj-y += octeon-memcpy.o
 obj-y += executive/
diff --git a/arch/mips/cavium-octeon/resource-mgr.c b/arch/mips/cavium-octeon/resource-mgr.c
new file mode 100644
index 000000000000..74efda5420ff
--- /dev/null
+++ b/arch/mips/cavium-octeon/resource-mgr.c
@@ -0,0 +1,351 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Resource manager for Octeon.
+ *
+ * Copyright (C) 2017 Cavium, Inc.
+ */
+#include <linux/module.h>
+
+#include <asm/octeon/octeon.h>
+#include <asm/octeon/cvmx-bootmem.h>
+
+#define RESOURCE_MGR_BLOCK_NAME		"cvmx-global-resources"
+#define MAX_RESOURCES			128
+#define INST_AVAILABLE			-88
+#define OWNER				0xbadc0de
+
+struct global_resource_entry {
+	struct global_resource_tag tag;
+	u64 phys_addr;
+	u64 size;
+};
+
+struct global_resources {
+#ifdef __LITTLE_ENDIAN_BITFIELD
+	u32 rlock;
+	u32 pad;
+#else
+	u32 pad;
+	u32 rlock;
+#endif
+	u64 entry_cnt;
+	struct global_resource_entry resource_entry[];
+};
+
+static struct global_resources *res_mgr_info;
+
+
+/*
+ * The resource manager interacts with software running outside of the
+ * Linux kernel, which necessitates locking to maintain data structure
+ * consistency.  These custom locking functions implement the locking
+ * protocol, and cannot be replaced by kernel locking functions that
+ * may use different in-memory structures.
+ */
+
+static void res_mgr_lock(void)
+{
+	while (cmpxchg(&res_mgr_info->rlock, 0, 1))
+		; /* Loop while not zero */
+	rmb();
+}
+
+static void res_mgr_unlock(void)
+{
+	/* Wait until all resource operations finish before unlocking. */
+	wmb();
+	WRITE_ONCE(res_mgr_info->rlock, 0);
+	/* Force a write buffer flush. */
+	wmb();
+}
+
+static int res_mgr_find_resource(struct global_resource_tag tag)
+{
+	struct global_resource_entry *res_entry;
+	int i;
+
+	for (i = 0; i < res_mgr_info->entry_cnt; i++) {
+		res_entry = &res_mgr_info->resource_entry[i];
+		if (res_entry->tag.lo == tag.lo && res_entry->tag.hi == tag.hi)
+			return i;
+	}
+	return -1;
+}
+
+/**
+ * res_mgr_create_resource() - Create a resource.
+ * @tag: Identifies the resource.
+ * @inst_cnt: Number of resource instances to create.
+ *
+ * Returns 0 if the source was created successfully.
+ * Returns < 0 for error codes.
+ */
+int res_mgr_create_resource(struct global_resource_tag tag, int inst_cnt)
+{
+	struct global_resource_entry *res_entry;
+	u64 size;
+	u64 *res_addr;
+	int res_index, i, rc = 0;
+
+	res_mgr_lock();
+
+	/* Make sure resource doesn't already exist. */
+	res_index = res_mgr_find_resource(tag);
+	if (res_index >= 0) {
+		rc = -EEXIST;
+		goto err;
+	}
+
+	if (res_mgr_info->entry_cnt >= MAX_RESOURCES) {
+		pr_err("Resource max limit reached, not created\n");
+		rc = -ENOSPC;
+		goto err;
+	}
+
+	/*
+	 * Each instance is kept in an array of u64s. The first array element
+	 * holds the number of allocated instances.
+	 */
+	size = sizeof(u64) * (inst_cnt + 1);
+	res_addr = cvmx_bootmem_alloc_range(size, CVMX_CACHE_LINE_SIZE, 0, 0);
+	if (!res_addr) {
+		pr_err("Failed to allocate resource. not created\n");
+		rc = -ENOMEM;
+		goto err;
+	}
+
+	/* Initialize the newly created resource. */
+	*res_addr = inst_cnt;
+	for (i = 1; i <= inst_cnt; i++)
+		res_addr[i] = INST_AVAILABLE;
+
+	res_index = res_mgr_info->entry_cnt;
+	res_entry = &res_mgr_info->resource_entry[res_index];
+	res_entry->tag = tag;
+	res_entry->phys_addr = virt_to_phys(res_addr);
+	res_entry->size = size;
+	res_mgr_info->entry_cnt++;
+
+err:
+	res_mgr_unlock();
+
+	return rc;
+}
+EXPORT_SYMBOL(res_mgr_create_resource);
+
+/**
+ * res_mgr_alloc_range() - Allocate a range of resource instances.
+ * @tag: Identifies the resource.
+ * @req_inst: Requested start of instance range to allocate.
+ *	      Range instances are guaranteed to be sequential
+ *	      (-1 for don't care).
+ * @req_cnt: Number of instances to allocate.
+ * @use_last_avail: Set to request the last available instance.
+ * @inst: Updated with the allocated instances.
+ *
+ * Returns 0 if the source was created successfully.
+ * Returns < 0 for error codes.
+ */
+int res_mgr_alloc_range(struct global_resource_tag tag, int req_inst,
+			int req_cnt, bool use_last_avail, int *inst)
+{
+	struct global_resource_entry *res_entry;
+	int res_index;
+	u64 *res_addr;
+	u64 inst_cnt;
+	int alloc_cnt, i, rc = -ENOENT;
+
+	/* Start with no instances allocated. */
+	for (i = 0; i < req_cnt; i++)
+		inst[i] = INST_AVAILABLE;
+
+	res_mgr_lock();
+
+	/* Find the resource. */
+	res_index = res_mgr_find_resource(tag);
+	if (res_index < 0) {
+		pr_err("Resource not found, can't allocate instance\n");
+		goto err;
+	}
+
+	/* Get resource data. */
+	res_entry = &res_mgr_info->resource_entry[res_index];
+	res_addr = phys_to_virt(res_entry->phys_addr);
+	inst_cnt = *res_addr;
+
+	/* Allocate the requested instances. */
+	if (req_inst >= 0) {
+		/* Specific instance range requested. */
+		if (req_inst + req_cnt >= inst_cnt) {
+			pr_err("Requested instance out of range\n");
+			goto err;
+		}
+
+		for (i = 0; i < req_cnt; i++) {
+			if (*(res_addr + req_inst + 1 + i) == INST_AVAILABLE) {
+				inst[i] = req_inst + i;
+			} else {
+				inst[0] = INST_AVAILABLE;
+				break;
+			}
+		}
+	} else if (use_last_avail) {
+		/* Last available instance requested. */
+		alloc_cnt = 0;
+		for (i = inst_cnt; i > 0; i--) {
+			if (*(res_addr + i) == INST_AVAILABLE) {
+				/*
+				 * Instance off by 1 (first element holds the
+				 * count).
+				 */
+				inst[alloc_cnt] = i - 1;
+
+				alloc_cnt++;
+				if (alloc_cnt == req_cnt)
+					break;
+			}
+		}
+
+		if (i == 0)
+			inst[0] = INST_AVAILABLE;
+	} else {
+		/* Next available instance requested. */
+		alloc_cnt = 0;
+		for (i = 1; i <= inst_cnt; i++) {
+			if (*(res_addr + i) == INST_AVAILABLE) {
+				/*
+				 * Instance off by 1 (first element holds the
+				 * count).
+				 */
+				inst[alloc_cnt] = i - 1;
+
+				alloc_cnt++;
+				if (alloc_cnt == req_cnt)
+					break;
+			}
+		}
+
+		if (i > inst_cnt)
+			inst[0] = INST_AVAILABLE;
+	}
+
+	if (inst[0] != INST_AVAILABLE) {
+		for (i = 0; i < req_cnt; i++)
+			*(res_addr + inst[i] + 1) = OWNER;
+		rc = 0;
+	}
+
+err:
+	res_mgr_unlock();
+
+	return rc;
+}
+EXPORT_SYMBOL(res_mgr_alloc_range);
+
+/**
+ * res_mgr_alloc() - Allocate a resource instance.
+ * @tag: Identifies the resource.
+ * @req_inst: Requested instance to allocate (-1 for don't care).
+ * @use_last_avail: Set to request the last available instance.
+ *
+ * Returns: Allocated resource instance if successful.
+ * Returns <0 for error codes.
+ */
+int res_mgr_alloc(struct global_resource_tag tag, int req_inst,
+		  bool use_last_avail)
+{
+	int inst, rc;
+
+	rc = res_mgr_alloc_range(tag, req_inst, 1, use_last_avail, &inst);
+	if (!rc)
+		return inst;
+	return rc;
+}
+EXPORT_SYMBOL(res_mgr_alloc);
+
+/**
+ * res_mgr_free_range() - Free a resource instance range.
+ * @tag: Identifies the resource.
+ * @inst: Requested instance to free.
+ * @req_cnt: Number of instances to free.
+ */
+void res_mgr_free_range(struct global_resource_tag tag, const int *inst,
+			int req_cnt)
+{
+	struct global_resource_entry *res_entry;
+	int res_index, i;
+	u64 *res_addr;
+
+	res_mgr_lock();
+
+	/* Find the resource. */
+	res_index = res_mgr_find_resource(tag);
+	if (res_index < 0) {
+		pr_err("Resource not found, can't free instance\n");
+		goto err;
+	}
+
+	/* Get the resource data. */
+	res_entry = &res_mgr_info->resource_entry[res_index];
+	res_addr = phys_to_virt(res_entry->phys_addr);
+
+	/* Free the resource instances. */
+	for (i = 0; i < req_cnt; i++) {
+		/* Instance off by 1 (first element holds the count). */
+		*(res_addr + inst[i] + 1) = INST_AVAILABLE;
+	}
+
+err:
+	res_mgr_unlock();
+}
+EXPORT_SYMBOL(res_mgr_free_range);
+
+/**
+ * res_mgr_free() - Free a resource instance.
+ * @tag: Identifies the resource.
+ * @req_inst: Requested instance to free.
+ */
+void res_mgr_free(struct global_resource_tag tag, int inst)
+{
+	res_mgr_free_range(tag, &inst, 1);
+}
+EXPORT_SYMBOL(res_mgr_free);
+
+static int __init res_mgr_init(void)
+{
+	struct cvmx_bootmem_named_block_desc *block;
+	int block_size;
+	u64 addr;
+
+	cvmx_bootmem_lock();
+
+	/* Search for the resource manager data in boot memory. */
+	block = cvmx_bootmem_phy_named_block_find(RESOURCE_MGR_BLOCK_NAME,
+						  CVMX_BOOTMEM_FLAG_NO_LOCKING);
+	if (block) {
+		/* Found. */
+		res_mgr_info = phys_to_virt(block->base_addr);
+	} else {
+		/* Create it. */
+		block_size = sizeof(struct global_resources) +
+			sizeof(struct global_resource_entry) * MAX_RESOURCES;
+		addr = cvmx_bootmem_phy_named_block_alloc(block_size, 0, 0,
+				CVMX_CACHE_LINE_SIZE, RESOURCE_MGR_BLOCK_NAME,
+				CVMX_BOOTMEM_FLAG_NO_LOCKING);
+		if (!addr) {
+			pr_err("Failed to allocate name block %s\n",
+			       RESOURCE_MGR_BLOCK_NAME);
+		} else {
+			res_mgr_info = phys_to_virt(addr);
+			memset(res_mgr_info, 0, block_size);
+		}
+	}
+
+	cvmx_bootmem_unlock();
+
+	return 0;
+}
+device_initcall(res_mgr_init);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Cavium, Inc. Octeon resource manager");
diff --git a/arch/mips/cavium-octeon/setup.c b/arch/mips/cavium-octeon/setup.c
index a8034d0dcade..51c4d3c3cada 100644
--- a/arch/mips/cavium-octeon/setup.c
+++ b/arch/mips/cavium-octeon/setup.c
@@ -68,6 +68,12 @@ extern void pci_console_init(const char *arg);
 static unsigned long long max_memory = ULLONG_MAX;
 static unsigned long long reserve_low_mem;
 
+/*
+ * modified in hernel-entry-init.h, must have an initial value to keep
+ * it from being clobbered when bss is zeroed.
+ */
+u32 octeon_cvmseg_lines = 2;
+
 DEFINE_SEMAPHORE(octeon_bootbus_sem);
 EXPORT_SYMBOL(octeon_bootbus_sem);
 
@@ -604,11 +610,13 @@ void octeon_user_io_init(void)
 
 	/* R/W If set, CVMSEG is available for loads/stores in
 	 * kernel/debug mode. */
-#if CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
 	cvmmemctl.s.cvmsegenak = 1;
-#else
-	cvmmemctl.s.cvmsegenak = 0;
-#endif
+	if (OCTEON_IS_OCTEON3()) {
+		/* Enable LMTDMA */
+		cvmmemctl.s.lmtena = 1;
+		/* Scratch line to use for LMT operation */
+		cvmmemctl.s.lmtline = 2;
+	}
 	/* R/W If set, CVMSEG is available for loads/stores in
 	 * supervisor mode. */
 	cvmmemctl.s.cvmsegenas = 0;
@@ -620,9 +628,9 @@ void octeon_user_io_init(void)
 
 	/* Setup of CVMSEG is done in kernel-entry-init.h */
 	if (smp_processor_id() == 0)
-		pr_notice("CVMSEG size: %d cache lines (%d bytes)\n",
-			  CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE,
-			  CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128);
+		pr_notice("CVMSEG size: %u cache lines (%u bytes)\n",
+			  octeon_cvmseg_lines,
+			  octeon_cvmseg_lines * 128);
 
 	if (octeon_has_feature(OCTEON_FEATURE_FAU)) {
 		union cvmx_iob_fau_timeout fau_timeout;
diff --git a/arch/mips/configs/ci20_defconfig b/arch/mips/configs/ci20_defconfig
index b5f4ad8f2c45..be23fd25eeaa 100644
--- a/arch/mips/configs/ci20_defconfig
+++ b/arch/mips/configs/ci20_defconfig
@@ -104,10 +104,14 @@ CONFIG_REGULATOR_FIXED_VOLTAGE=y
 # CONFIG_HID is not set
 # CONFIG_USB_SUPPORT is not set
 CONFIG_MMC=y
+CONFIG_MMC_JZ4740=y
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_JZ4740=y
+CONFIG_DMADEVICES=y
+CONFIG_DMA_JZ4780=y
 # CONFIG_IOMMU_SUPPORT is not set
 CONFIG_MEMORY=y
+CONFIG_EXT4_FS=y
 # CONFIG_DNOTIFY is not set
 CONFIG_PROC_KCORE=y
 # CONFIG_PROC_PAGE_MONITOR is not set
diff --git a/arch/mips/configs/qi_lb60_defconfig b/arch/mips/configs/qi_lb60_defconfig
index 3b02ff9a7c64..d8b7211a7b0f 100644
--- a/arch/mips/configs/qi_lb60_defconfig
+++ b/arch/mips/configs/qi_lb60_defconfig
@@ -72,6 +72,8 @@ CONFIG_POWER_SUPPLY=y
 CONFIG_BATTERY_JZ4740=y
 CONFIG_CHARGER_GPIO=y
 # CONFIG_HWMON is not set
+CONFIG_WATCHDOG=y
+CONFIG_JZ4740_WDT=y
 CONFIG_MFD_JZ4740_ADC=y
 CONFIG_REGULATOR=y
 CONFIG_REGULATOR_FIXED_VOLTAGE=y
diff --git a/arch/mips/dec/time.c b/arch/mips/dec/time.c
index a2a150e4fbc2..c38686f89a18 100644
--- a/arch/mips/dec/time.c
+++ b/arch/mips/dec/time.c
@@ -19,7 +19,7 @@
 #include <asm/dec/ioasic.h>
 #include <asm/dec/machtype.h>
 
-void read_persistent_clock(struct timespec *ts)
+void read_persistent_clock64(struct timespec64 *ts)
 {
 	unsigned int year, mon, day, hour, min, sec, real_year;
 	unsigned long flags;
@@ -54,19 +54,20 @@ void read_persistent_clock(struct timespec *ts)
 
 	year += real_year - 72 + 2000;
 
-	ts->tv_sec = mktime(year, mon, day, hour, min, sec);
+	ts->tv_sec = mktime64(year, mon, day, hour, min, sec);
 	ts->tv_nsec = 0;
 }
 
 /*
- * In order to set the CMOS clock precisely, rtc_mips_set_mmss has to
+ * In order to set the CMOS clock precisely, update_persistent_clock64 has to
  * be called 500 ms after the second nowtime has started, because when
  * nowtime is written into the registers of the CMOS clock, it will
  * jump to the next second precisely 500 ms later.  Check the Dallas
  * DS1287 data sheet for details.
  */
-int rtc_mips_set_mmss(unsigned long nowtime)
+int update_persistent_clock64(struct timespec64 now)
 {
+	time64_t nowtime = now.tv_sec;
 	int retval = 0;
 	int real_seconds, real_minutes, cmos_minutes;
 	unsigned char save_control, save_freq_select;
@@ -91,8 +92,7 @@ int rtc_mips_set_mmss(unsigned long nowtime)
 	 * messing with unknown time zones but requires your
 	 * RTC not to be off by more than 15 minutes
 	 */
-	real_seconds = nowtime % 60;
-	real_minutes = nowtime / 60;
+	real_minutes = div_s64_rem(nowtime, 60, &real_seconds);
 	if (((abs(real_minutes - cmos_minutes) + 15) / 30) & 1)
 		real_minutes += 30;	/* correct for half hour time zone */
 	real_minutes %= 60;
diff --git a/arch/mips/include/asm/barrier.h b/arch/mips/include/asm/barrier.h
index a5eb1bb199a7..0e8e6afbf80a 100644
--- a/arch/mips/include/asm/barrier.h
+++ b/arch/mips/include/asm/barrier.h
@@ -216,7 +216,7 @@
 #else
 #define smp_mb__before_llsc() smp_llsc_mb()
 #define __smp_mb__before_llsc() smp_llsc_mb()
-#define nudge_writes() mb()
+#define nudge_writes() do { } while (0)
 #endif
 
 #define __smp_mb__before_atomic()	__smp_mb__before_llsc()
diff --git a/arch/mips/include/asm/compat.h b/arch/mips/include/asm/compat.h
index 9a0fa66b81ac..78675f19440f 100644
--- a/arch/mips/include/asm/compat.h
+++ b/arch/mips/include/asm/compat.h
@@ -14,7 +14,6 @@
 
 typedef u32		compat_size_t;
 typedef s32		compat_ssize_t;
-typedef s32		compat_time_t;
 typedef s32		compat_clock_t;
 typedef s32		compat_suseconds_t;
 
@@ -38,24 +37,16 @@ typedef struct {
 typedef s32		compat_timer_t;
 typedef s32		compat_key_t;
 
+typedef s16		compat_short_t;
 typedef s32		compat_int_t;
 typedef s32		compat_long_t;
 typedef s64		compat_s64;
+typedef u16		compat_ushort_t;
 typedef u32		compat_uint_t;
 typedef u32		compat_ulong_t;
 typedef u64		compat_u64;
 typedef u32		compat_uptr_t;
 
-struct compat_timespec {
-	compat_time_t	tv_sec;
-	s32		tv_nsec;
-};
-
-struct compat_timeval {
-	compat_time_t	tv_sec;
-	s32		tv_usec;
-};
-
 struct compat_stat {
 	compat_dev_t	st_dev;
 	s32		st_pad1[3];
@@ -168,35 +159,35 @@ struct compat_ipc64_perm {
 
 struct compat_semid64_ds {
 	struct compat_ipc64_perm sem_perm;
-	compat_time_t	sem_otime;
-	compat_time_t	sem_ctime;
+	compat_ulong_t	sem_otime;
+	compat_ulong_t	sem_ctime;
 	compat_ulong_t	sem_nsems;
-	compat_ulong_t	__unused1;
-	compat_ulong_t	__unused2;
+	compat_ulong_t	sem_otime_high;
+	compat_ulong_t	sem_ctime_high;
 };
 
 struct compat_msqid64_ds {
 	struct compat_ipc64_perm msg_perm;
 #ifndef CONFIG_CPU_LITTLE_ENDIAN
-	compat_ulong_t	__unused1;
+	compat_ulong_t	msg_stime_high;
 #endif
-	compat_time_t	msg_stime;
+	compat_ulong_t	msg_stime;
 #ifdef CONFIG_CPU_LITTLE_ENDIAN
-	compat_ulong_t	__unused1;
+	compat_ulong_t	msg_stime_high;
 #endif
 #ifndef CONFIG_CPU_LITTLE_ENDIAN
-	compat_ulong_t	__unused2;
+	compat_ulong_t	msg_rtime_high;
 #endif
-	compat_time_t	msg_rtime;
+	compat_ulong_t	msg_rtime;
 #ifdef CONFIG_CPU_LITTLE_ENDIAN
-	compat_ulong_t	__unused2;
+	compat_ulong_t	msg_rtime_high;
 #endif
 #ifndef CONFIG_CPU_LITTLE_ENDIAN
-	compat_ulong_t	__unused3;
+	compat_ulong_t	msg_ctime_high;
 #endif
-	compat_time_t	msg_ctime;
+	compat_ulong_t	msg_ctime;
 #ifdef CONFIG_CPU_LITTLE_ENDIAN
-	compat_ulong_t	__unused3;
+	compat_ulong_t	msg_ctime_high;
 #endif
 	compat_ulong_t	msg_cbytes;
 	compat_ulong_t	msg_qnum;
@@ -210,14 +201,16 @@ struct compat_msqid64_ds {
 struct compat_shmid64_ds {
 	struct compat_ipc64_perm shm_perm;
 	compat_size_t	shm_segsz;
-	compat_time_t	shm_atime;
-	compat_time_t	shm_dtime;
-	compat_time_t	shm_ctime;
+	compat_ulong_t	shm_atime;
+	compat_ulong_t	shm_dtime;
+	compat_ulong_t	shm_ctime;
 	compat_pid_t	shm_cpid;
 	compat_pid_t	shm_lpid;
 	compat_ulong_t	shm_nattch;
-	compat_ulong_t	__unused1;
-	compat_ulong_t	__unused2;
+	compat_ushort_t	shm_atime_high;
+	compat_ushort_t	shm_dtime_high;
+	compat_ushort_t	shm_ctime_high;
+	compat_ushort_t	__unused2;
 };
 
 /* MIPS has unusual order of fields in stack_t */
diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h
index 5f74590e0bea..9cdb4e4ce258 100644
--- a/arch/mips/include/asm/cpu-features.h
+++ b/arch/mips/include/asm/cpu-features.h
@@ -535,6 +535,13 @@
 # define cpu_has_shared_ftlb_entries 0
 #endif
 
+#ifdef CONFIG_MIPS_MT_SMP
+# define cpu_has_mipsmt_pertccounters \
+	(cpu_data[0].options & MIPS_CPU_MT_PER_TC_PERF_COUNTERS)
+#else
+# define cpu_has_mipsmt_pertccounters 0
+#endif /* CONFIG_MIPS_MT_SMP */
+
 /*
  * Guest capabilities
  */
diff --git a/arch/mips/include/asm/cpu.h b/arch/mips/include/asm/cpu.h
index d39324c4adf1..5b9d02ef4f60 100644
--- a/arch/mips/include/asm/cpu.h
+++ b/arch/mips/include/asm/cpu.h
@@ -418,6 +418,8 @@ enum cpu_type_enum {
 				MBIT_ULL(54)	/* CPU shares FTLB RAM with another */
 #define MIPS_CPU_SHARED_FTLB_ENTRIES \
 				MBIT_ULL(55)	/* CPU shares FTLB entries with another */
+#define MIPS_CPU_MT_PER_TC_PERF_COUNTERS \
+				MBIT_ULL(56)	/* CPU has perf counters implemented per TC (MIPSMT ASE) */
 
 /*
  * CPU ASE encodings
diff --git a/arch/mips/include/asm/mach-bcm47xx/bcm47xx_board.h b/arch/mips/include/asm/mach-bcm47xx/bcm47xx_board.h
index cbf9da7f2f94..0ef8893e07f8 100644
--- a/arch/mips/include/asm/mach-bcm47xx/bcm47xx_board.h
+++ b/arch/mips/include/asm/mach-bcm47xx/bcm47xx_board.h
@@ -110,6 +110,7 @@ enum bcm47xx_board {
 	BCM47XX_BOARD_NETGEAR_WNDR4000,
 	BCM47XX_BOARD_NETGEAR_WNDR4500V1,
 	BCM47XX_BOARD_NETGEAR_WNDR4500V2,
+	BCM47XX_BOARD_NETGEAR_WNR1000_V3,
 	BCM47XX_BOARD_NETGEAR_WNR2000,
 	BCM47XX_BOARD_NETGEAR_WNR3500L,
 	BCM47XX_BOARD_NETGEAR_WNR3500U,
diff --git a/arch/mips/include/asm/mach-cavium-octeon/kernel-entry-init.h b/arch/mips/include/asm/mach-cavium-octeon/kernel-entry-init.h
index c38b38ce5a3d..cdcca60978a2 100644
--- a/arch/mips/include/asm/mach-cavium-octeon/kernel-entry-init.h
+++ b/arch/mips/include/asm/mach-cavium-octeon/kernel-entry-init.h
@@ -26,11 +26,18 @@
 	# a3 = address of boot descriptor block
 	.set push
 	.set arch=octeon
+	mfc0	v1, CP0_PRID_REG
+	andi	v1, 0xff00
+	li	v0, 0x9500		# cn78XX or later
+	subu	v1, v1, v0
+	li	t2, 2 + CONFIG_CAVIUM_OCTEON_EXTRA_CVMSEG
+	bltz	v1, 1f
+	addiu	t2, 1			# t2 has cvmseg_size
+1:
 	# Read the cavium mem control register
 	dmfc0	v0, CP0_CVMMEMCTL_REG
 	# Clear the lower 6 bits, the CVMSEG size
-	dins	v0, $0, 0, 6
-	ori	v0, CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE
+	dins	v0, t2, 0, 6
 	dmtc0	v0, CP0_CVMMEMCTL_REG	# Write the cavium mem control register
 	dmfc0	v0, CP0_CVMCTL_REG	# Read the cavium control register
 	# Disable unaligned load/store support but leave HW fixup enabled
@@ -70,7 +77,7 @@
 	# Flush dcache after config change
 	cache	9, 0($0)
 	# Zero all of CVMSEG to make sure parity is correct
-	dli	v0, CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE
+	move	v0, t2
 	dsll	v0, 7
 	beqz	v0, 2f
 1:	dsubu	v0, 8
@@ -126,12 +133,7 @@
 	LONG_L	sp, (t0)
 	# Set the SP global variable to zero so the master knows we've started
 	LONG_S	zero, (t0)
-#ifdef __OCTEON__
-	syncw
-	syncw
-#else
 	sync
-#endif
 	# Jump to the normal Linux SMP entry point
 	j   smp_bootstrap
 	nop
@@ -148,6 +150,8 @@
 
 #endif /* CONFIG_SMP */
 octeon_main_processor:
+	dla	v0, octeon_cvmseg_lines
+	sw	t2, 0(v0)
 	.set pop
 .endm
 
diff --git a/arch/mips/include/asm/mach-jz4740/platform.h b/arch/mips/include/asm/mach-jz4740/platform.h
index 3645974b7f65..c0c932ac72a7 100644
--- a/arch/mips/include/asm/mach-jz4740/platform.h
+++ b/arch/mips/include/asm/mach-jz4740/platform.h
@@ -29,7 +29,6 @@ extern struct platform_device jz4740_i2s_device;
 extern struct platform_device jz4740_pcm_device;
 extern struct platform_device jz4740_codec_device;
 extern struct platform_device jz4740_adc_device;
-extern struct platform_device jz4740_wdt_device;
 extern struct platform_device jz4740_pwm_device;
 extern struct platform_device jz4740_dma_device;
 
diff --git a/arch/mips/include/asm/mc146818-time.h b/arch/mips/include/asm/mc146818-time.h
index 9e1ad26abdc0..cbf5cec345f1 100644
--- a/arch/mips/include/asm/mc146818-time.h
+++ b/arch/mips/include/asm/mc146818-time.h
@@ -86,7 +86,7 @@ static inline int mc146818_set_rtc_mmss(unsigned long nowtime)
 	return retval;
 }
 
-static inline unsigned long mc146818_get_cmos_time(void)
+static inline time64_t mc146818_get_cmos_time(void)
 {
 	unsigned int year, mon, day, hour, min, sec;
 	unsigned long flags;
@@ -113,7 +113,7 @@ static inline unsigned long mc146818_get_cmos_time(void)
 	spin_unlock_irqrestore(&rtc_lock, flags);
 	year = mc146818_decode_year(year);
 
-	return mktime(year, mon, day, hour, min, sec);
+	return mktime64(year, mon, day, hour, min, sec);
 }
 
 #endif /* __ASM_MC146818_TIME_H */
diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h
index f65859784a4c..ef5fc14780d8 100644
--- a/arch/mips/include/asm/mipsregs.h
+++ b/arch/mips/include/asm/mipsregs.h
@@ -685,6 +685,11 @@
 #define MIPS_CONF7_IAR		(_ULCAST_(1) << 10)
 #define MIPS_CONF7_AR		(_ULCAST_(1) << 16)
 
+/* Config7 Bits specific to MIPS Technologies. */
+
+/* Performance counters implemented Per TC */
+#define MTI_CONF7_PTC		(_ULCAST_(1) << 19)
+
 /* WatchLo* register definitions */
 #define MIPS_WATCHLO_IRW	(_ULCAST_(0x7) << 0)
 
@@ -1127,6 +1132,8 @@
 #define FPU_CSR_RD	0x3	/* towards -Infinity */
 
 
+#define CAVIUM_OCTEON_SCRATCH_OFFSET (2 * 128 - 16 - 32768)
+
 #ifndef __ASSEMBLY__
 
 /*
diff --git a/arch/mips/include/asm/octeon/octeon.h b/arch/mips/include/asm/octeon/octeon.h
index c99c4b6a79f4..4dafeaf262b5 100644
--- a/arch/mips/include/asm/octeon/octeon.h
+++ b/arch/mips/include/asm/octeon/octeon.h
@@ -179,7 +179,15 @@ union octeon_cvmemctl {
 		/* RO 1 = BIST fail, 0 = BIST pass */
 		__BITFIELD_FIELD(uint64_t wbfbist:1,
 		/* Reserved */
-		__BITFIELD_FIELD(uint64_t reserved:17,
+		__BITFIELD_FIELD(uint64_t reserved_52_57:6,
+		/* When set, LMTDMA/LMTST operations are permitted */
+		__BITFIELD_FIELD(uint64_t lmtena:1,
+		/* Selects the CVMSEG LM cacheline used by LMTDMA
+		 * LMTST and wide atomic store operations.
+		 */
+		__BITFIELD_FIELD(uint64_t lmtline:6,
+		/* Reserved */
+		__BITFIELD_FIELD(uint64_t reserved_41_44:4,
 		/* OCTEON II - TLB replacement policy: 0 = bitmask LRU; 1 = NLU.
 		 * This field selects between the TLB replacement policies:
 		 * bitmask LRU or NLU. Bitmask LRU maintains a mask of
@@ -275,7 +283,7 @@ union octeon_cvmemctl {
 		/* R/W Size of local memory in cache blocks, 54 (6912
 		 * bytes) is max legal value. */
 		__BITFIELD_FIELD(uint64_t lmemsz:6,
-		;)))))))))))))))))))))))))))))))))
+		;))))))))))))))))))))))))))))))))))))
 	} s;
 };
 
@@ -338,6 +346,24 @@ void octeon_mult_restore3_end(void);
 void octeon_mult_restore2(void);
 void octeon_mult_restore2_end(void);
 
+/*
+ * This definition must be kept in sync with the one in
+ * cvmx-global-resources.c
+ */
+struct global_resource_tag {
+	uint64_t lo;
+	uint64_t hi;
+};
+
+void res_mgr_free(struct global_resource_tag tag, int inst);
+void res_mgr_free_range(struct global_resource_tag tag, const int *inst,
+			int req_cnt);
+int res_mgr_alloc(struct global_resource_tag tag, int req_inst,
+		  bool use_last_avail);
+int res_mgr_alloc_range(struct global_resource_tag tag, int req_inst,
+			int req_cnt, bool use_last_avail, int *inst);
+int res_mgr_create_resource(struct global_resource_tag tag, int inst_cnt);
+
 /**
  * Read a 32bit value from the Octeon NPI register space
  *
@@ -351,6 +377,8 @@ static inline uint32_t octeon_npi_read32(uint64_t address)
 
 extern struct cvmx_bootinfo *octeon_bootinfo;
 
+extern u32 octeon_cvmseg_lines;
+
 extern uint64_t octeon_bootloader_entry_addr;
 
 extern void (*octeon_irq_setup_secondary)(void);
diff --git a/arch/mips/include/asm/pci.h b/arch/mips/include/asm/pci.h
index 2339f42f047a..436099883022 100644
--- a/arch/mips/include/asm/pci.h
+++ b/arch/mips/include/asm/pci.h
@@ -121,13 +121,6 @@ extern unsigned long PCIBIOS_MIN_MEM;
 #include <linux/string.h>
 #include <asm/io.h>
 
-/*
- * The PCI address space does equal the physical memory address space.
- * The networking and block device layers use this boolean for bounce
- * buffer decisions.
- */
-#define PCI_DMA_BUS_IS_PHYS     (1)
-
 #ifdef CONFIG_PCI_DOMAINS_GENERIC
 static inline int pci_proc_domain(struct pci_bus *bus)
 {
diff --git a/arch/mips/include/asm/processor.h b/arch/mips/include/asm/processor.h
index af34afbc32d9..1a20f9c5509f 100644
--- a/arch/mips/include/asm/processor.h
+++ b/arch/mips/include/asm/processor.h
@@ -216,7 +216,7 @@ struct octeon_cop2_state {
 	.cp2			= {0,},
 
 struct octeon_cvmseg_state {
-	unsigned long cvmseg[CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE]
+	unsigned long cvmseg[CONFIG_CAVIUM_OCTEON_EXTRA_CVMSEG + 3]
 			    [cpu_dcache_line_size() / sizeof(unsigned long)];
 };
 
diff --git a/arch/mips/include/asm/time.h b/arch/mips/include/asm/time.h
index 17d4cd20f18c..b85ec64ee7e9 100644
--- a/arch/mips/include/asm/time.h
+++ b/arch/mips/include/asm/time.h
@@ -21,15 +21,6 @@
 
 extern spinlock_t rtc_lock;
 
-/*
- * RTC ops.  By default, they point to weak no-op RTC functions.
- *	rtc_mips_set_time - reverse the above translation and set time to RTC.
- *	rtc_mips_set_mmss - similar to rtc_set_time, but only min and sec need
- *			to be set.  Used by RTC sync-up.
- */
-extern int rtc_mips_set_time(unsigned long);
-extern int rtc_mips_set_mmss(unsigned long);
-
 /*
  * board specific routines required by time_init().
  */
diff --git a/arch/mips/include/uapi/asm/msgbuf.h b/arch/mips/include/uapi/asm/msgbuf.h
index eb4d0f9d7364..46aa15b13e4e 100644
--- a/arch/mips/include/uapi/asm/msgbuf.h
+++ b/arch/mips/include/uapi/asm/msgbuf.h
@@ -9,33 +9,15 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - extension of time_t to 64-bit on 32-bitsystem to solve the y2038 problem
  * - 2 miscellaneous unsigned long values
  */
 
+#if defined(__mips64)
 struct msqid64_ds {
 	struct ipc64_perm msg_perm;
-#if !defined(__mips64) && defined(__MIPSEB__)
-	unsigned long	__unused1;
-#endif
 	__kernel_time_t msg_stime;	/* last msgsnd time */
-#if !defined(__mips64) && defined(__MIPSEL__)
-	unsigned long	__unused1;
-#endif
-#if !defined(__mips64) && defined(__MIPSEB__)
-	unsigned long	__unused2;
-#endif
 	__kernel_time_t msg_rtime;	/* last msgrcv time */
-#if !defined(__mips64) && defined(__MIPSEL__)
-	unsigned long	__unused2;
-#endif
-#if !defined(__mips64) && defined(__MIPSEB__)
-	unsigned long	__unused3;
-#endif
 	__kernel_time_t msg_ctime;	/* last change time */
-#if !defined(__mips64) && defined(__MIPSEL__)
-	unsigned long	__unused3;
-#endif
 	unsigned long  msg_cbytes;	/* current number of bytes on queue */
 	unsigned long  msg_qnum;	/* number of messages in queue */
 	unsigned long  msg_qbytes;	/* max number of bytes on queue */
@@ -44,5 +26,42 @@ struct msqid64_ds {
 	unsigned long  __unused4;
 	unsigned long  __unused5;
 };
+#elif defined (__MIPSEB__)
+struct msqid64_ds {
+	struct ipc64_perm msg_perm;
+	unsigned long  msg_stime_high;
+	unsigned long  msg_stime;	/* last msgsnd time */
+	unsigned long  msg_rtime_high;
+	unsigned long  msg_rtime;	/* last msgrcv time */
+	unsigned long  msg_ctime_high;
+	unsigned long  msg_ctime;	/* last change time */
+	unsigned long  msg_cbytes;	/* current number of bytes on queue */
+	unsigned long  msg_qnum;	/* number of messages in queue */
+	unsigned long  msg_qbytes;	/* max number of bytes on queue */
+	__kernel_pid_t msg_lspid;	/* pid of last msgsnd */
+	__kernel_pid_t msg_lrpid;	/* last receive pid */
+	unsigned long  __unused4;
+	unsigned long  __unused5;
+};
+#elif defined (__MIPSEL__)
+struct msqid64_ds {
+	struct ipc64_perm msg_perm;
+	unsigned long  msg_stime;	/* last msgsnd time */
+	unsigned long  msg_stime_high;
+	unsigned long  msg_rtime;	/* last msgrcv time */
+	unsigned long  msg_rtime_high;
+	unsigned long  msg_ctime;	/* last change time */
+	unsigned long  msg_ctime_high;
+	unsigned long  msg_cbytes;	/* current number of bytes on queue */
+	unsigned long  msg_qnum;	/* number of messages in queue */
+	unsigned long  msg_qbytes;	/* max number of bytes on queue */
+	__kernel_pid_t msg_lspid;	/* pid of last msgsnd */
+	__kernel_pid_t msg_lrpid;	/* last receive pid */
+	unsigned long  __unused4;
+	unsigned long  __unused5;
+};
+#else
+#warning no endianess set
+#endif
 
 #endif /* _ASM_MSGBUF_H */
diff --git a/arch/mips/include/uapi/asm/sembuf.h b/arch/mips/include/uapi/asm/sembuf.h
index 2c0f507ab7d1..60c89e6cb25b 100644
--- a/arch/mips/include/uapi/asm/sembuf.h
+++ b/arch/mips/include/uapi/asm/sembuf.h
@@ -7,10 +7,11 @@
  * Note extra padding because this structure is passed back and forth
  * between kernel and user space.
  *
- * Pad space is left for:
- * - 2 miscellaneous 64-bit values
+ * Pad space is left for 2 miscellaneous 64-bit values on mips64,
+ * but used for the upper 32 bit of the time values on mips32.
  */
 
+#ifdef __mips64
 struct semid64_ds {
 	struct ipc64_perm sem_perm;		/* permissions .. see ipc.h */
 	__kernel_time_t sem_otime;		/* last semop time */
@@ -19,5 +20,15 @@ struct semid64_ds {
 	unsigned long	__unused1;
 	unsigned long	__unused2;
 };
+#else
+struct semid64_ds {
+	struct ipc64_perm sem_perm;		/* permissions .. see ipc.h */
+	unsigned long   sem_otime;		/* last semop time */
+	unsigned long   sem_ctime;		/* last change time */
+	unsigned long	sem_nsems;		/* no. of semaphores in array */
+	unsigned long	sem_otime_high;
+	unsigned long	sem_ctime_high;
+};
+#endif
 
 #endif /* _ASM_SEMBUF_H */
diff --git a/arch/mips/include/uapi/asm/shmbuf.h b/arch/mips/include/uapi/asm/shmbuf.h
index 379e6bca518b..9b9bba3401f2 100644
--- a/arch/mips/include/uapi/asm/shmbuf.h
+++ b/arch/mips/include/uapi/asm/shmbuf.h
@@ -7,10 +7,13 @@
  * Note extra padding because this structure is passed back and forth
  * between kernel and user space.
  *
- * Pad space is left for:
- * - 2 miscellaneous 32-bit rsp. 64-bit values
+ * As MIPS was lacking proper padding after shm_?time, we use 48 bits
+ * of the padding at the end to store a few additional bits of the time.
+ * libc implementations need to take care to convert this into a proper
+ * data structure when moving to 64-bit time_t.
  */
 
+#ifdef __mips64
 struct shmid64_ds {
 	struct ipc64_perm	shm_perm;	/* operation perms */
 	size_t			shm_segsz;	/* size of segment (bytes) */
@@ -23,6 +26,22 @@ struct shmid64_ds {
 	unsigned long		__unused1;
 	unsigned long		__unused2;
 };
+#else
+struct shmid64_ds {
+	struct ipc64_perm	shm_perm;	/* operation perms */
+	size_t			shm_segsz;	/* size of segment (bytes) */
+	unsigned long		shm_atime;	/* last attach time */
+	unsigned long		shm_dtime;	/* last detach time */
+	unsigned long		shm_ctime;	/* last change time */
+	__kernel_pid_t		shm_cpid;	/* pid of creator */
+	__kernel_pid_t		shm_lpid;	/* pid of last operator */
+	unsigned long		shm_nattch;	/* no. of current attaches */
+	unsigned short		shm_atime_high;
+	unsigned short		shm_dtime_high;
+	unsigned short		shm_ctime_high;
+	unsigned short		__unused1;
+};
+#endif
 
 struct shminfo64 {
 	unsigned long	shmmax;
diff --git a/arch/mips/jz4740/platform.c b/arch/mips/jz4740/platform.c
index 5b7cdd67a9d9..cbc5f8e87230 100644
--- a/arch/mips/jz4740/platform.c
+++ b/arch/mips/jz4740/platform.c
@@ -233,22 +233,6 @@ struct platform_device jz4740_adc_device = {
 	.resource	= jz4740_adc_resources,
 };
 
-/* Watchdog */
-static struct resource jz4740_wdt_resources[] = {
-	{
-		.start = JZ4740_WDT_BASE_ADDR,
-		.end   = JZ4740_WDT_BASE_ADDR + 0x10 - 1,
-		.flags = IORESOURCE_MEM,
-	},
-};
-
-struct platform_device jz4740_wdt_device = {
-	.name	       = "jz4740-wdt",
-	.id	       = -1,
-	.num_resources = ARRAY_SIZE(jz4740_wdt_resources),
-	.resource      = jz4740_wdt_resources,
-};
-
 /* PWM */
 struct platform_device jz4740_pwm_device = {
 	.name = "jz4740-pwm",
diff --git a/arch/mips/jz4740/reset.c b/arch/mips/jz4740/reset.c
index 67780c4b6573..5bf0cf44b55f 100644
--- a/arch/mips/jz4740/reset.c
+++ b/arch/mips/jz4740/reset.c
@@ -12,18 +12,9 @@
  *
  */
 
-#include <linux/clk.h>
-#include <linux/io.h>
-#include <linux/kernel.h>
-#include <linux/pm.h>
-
 #include <asm/reboot.h>
 
-#include <asm/mach-jz4740/base.h>
-#include <asm/mach-jz4740/timer.h>
-
 #include "reset.h"
-#include "clock.h"
 
 static void jz4740_halt(void)
 {
@@ -36,29 +27,7 @@ static void jz4740_halt(void)
 	}
 }
 
-#define JZ_REG_WDT_DATA 0x00
-#define JZ_REG_WDT_COUNTER_ENABLE 0x04
-#define JZ_REG_WDT_COUNTER 0x08
-#define JZ_REG_WDT_CTRL 0x0c
-
-static void jz4740_restart(char *command)
-{
-	void __iomem *wdt_base = ioremap(JZ4740_WDT_BASE_ADDR, 0x0f);
-
-	jz4740_timer_enable_watchdog();
-
-	writeb(0, wdt_base + JZ_REG_WDT_COUNTER_ENABLE);
-
-	writew(0, wdt_base + JZ_REG_WDT_COUNTER);
-	writew(0, wdt_base + JZ_REG_WDT_DATA);
-	writew(BIT(2), wdt_base + JZ_REG_WDT_CTRL);
-
-	writeb(1, wdt_base + JZ_REG_WDT_COUNTER_ENABLE);
-	jz4740_halt();
-}
-
 void jz4740_reset_init(void)
 {
-	_machine_restart = jz4740_restart;
 	_machine_halt = jz4740_halt;
 }
diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c
index 6b07b739f914..b2509c19cfb5 100644
--- a/arch/mips/kernel/cpu-probe.c
+++ b/arch/mips/kernel/cpu-probe.c
@@ -414,6 +414,14 @@ static int __init ftlb_disable(char *s)
 
 __setup("noftlb", ftlb_disable);
 
+/*
+ * Check if the CPU has per tc perf counters
+ */
+static inline void cpu_set_mt_per_tc_perf(struct cpuinfo_mips *c)
+{
+	if (read_c0_config7() & MTI_CONF7_PTC)
+		c->options |= MIPS_CPU_MT_PER_TC_PERF_COUNTERS;
+}
 
 static inline void check_errata(void)
 {
@@ -1572,6 +1580,7 @@ static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu)
 		c->cputype = CPU_34K;
 		c->writecombine = _CACHE_UNCACHED;
 		__cpu_name[cpu] = "MIPS 34Kc";
+		cpu_set_mt_per_tc_perf(c);
 		break;
 	case PRID_IMP_74K:
 		c->cputype = CPU_74K;
@@ -1592,6 +1601,7 @@ static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu)
 		c->cputype = CPU_1004K;
 		c->writecombine = _CACHE_UNCACHED;
 		__cpu_name[cpu] = "MIPS 1004Kc";
+		cpu_set_mt_per_tc_perf(c);
 		break;
 	case PRID_IMP_1074K:
 		c->cputype = CPU_1074K;
@@ -1601,10 +1611,12 @@ static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu)
 	case PRID_IMP_INTERAPTIV_UP:
 		c->cputype = CPU_INTERAPTIV;
 		__cpu_name[cpu] = "MIPS interAptiv";
+		cpu_set_mt_per_tc_perf(c);
 		break;
 	case PRID_IMP_INTERAPTIV_MP:
 		c->cputype = CPU_INTERAPTIV;
 		__cpu_name[cpu] = "MIPS interAptiv (multi)";
+		cpu_set_mt_per_tc_perf(c);
 		break;
 	case PRID_IMP_PROAPTIV_UP:
 		c->cputype = CPU_PROAPTIV;
diff --git a/arch/mips/kernel/octeon_switch.S b/arch/mips/kernel/octeon_switch.S
index e42113fe2762..4f56902d5ee7 100644
--- a/arch/mips/kernel/octeon_switch.S
+++ b/arch/mips/kernel/octeon_switch.S
@@ -29,7 +29,6 @@
 	cpu_save_nonscratch a0
 	LONG_S	ra, THREAD_REG31(a0)
 
-#if CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
 	/* Check if we need to store CVMSEG state */
 	dmfc0	t0, $11,7	/* CvmMemCtl */
 	bbit0	t0, 6, 3f	/* Is user access enabled? */
@@ -58,7 +57,6 @@
 	dmfc0	t0, $11,7	/* CvmMemCtl */
 	xori	t0, t0, 0x40	/* Bit 6 is CVMSEG user enable */
 	dmtc0	t0, $11,7	/* CvmMemCtl */
-#endif
 3:
 
 #if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP)
diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c
index ee73550f0b9a..413863508f6f 100644
--- a/arch/mips/kernel/perf_event_mipsxx.c
+++ b/arch/mips/kernel/perf_event_mipsxx.c
@@ -129,20 +129,14 @@ static struct mips_pmu mipspmu;
 
 
 #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
-static int cpu_has_mipsmt_pertccounters;
-
 static DEFINE_RWLOCK(pmuint_rwlock);
 
 #if defined(CONFIG_CPU_BMIPS5000)
 #define vpe_id()	(cpu_has_mipsmt_pertccounters ? \
 			 0 : (smp_processor_id() & MIPS_CPUID_TO_COUNTER_MASK))
 #else
-/*
- * FIXME: For VSMP, vpe_id() is redefined for Perf-events, because
- * cpu_data[cpuid].vpe_id reports 0 for _both_ CPUs.
- */
 #define vpe_id()	(cpu_has_mipsmt_pertccounters ? \
-			 0 : smp_processor_id())
+			 0 : cpu_vpe_id(&current_cpu_data))
 #endif
 
 /* Copied from op_model_mipsxx.c */
@@ -329,7 +323,11 @@ static int mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc,
 
 static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx)
 {
+	struct perf_event *event = container_of(evt, struct perf_event, hw);
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+#ifdef CONFIG_MIPS_MT_SMP
+	unsigned int range = evt->event_base >> 24;
+#endif /* CONFIG_MIPS_MT_SMP */
 
 	WARN_ON(idx < 0 || idx >= mipspmu.num_counters);
 
@@ -337,11 +335,37 @@ static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx)
 		(evt->config_base & M_PERFCTL_CONFIG_MASK) |
 		/* Make sure interrupt enabled. */
 		MIPS_PERFCTRL_IE;
-	if (IS_ENABLED(CONFIG_CPU_BMIPS5000))
+
+#ifdef CONFIG_CPU_BMIPS5000
+	{
 		/* enable the counter for the calling thread */
 		cpuc->saved_ctrl[idx] |=
 			(1 << (12 + vpe_id())) | BRCM_PERFCTRL_TC;
+	}
+#else
+#ifdef CONFIG_MIPS_MT_SMP
+	if (range > V) {
+		/* The counter is processor wide. Set it up to count all TCs. */
+		pr_debug("Enabling perf counter for all TCs\n");
+		cpuc->saved_ctrl[idx] |= M_TC_EN_ALL;
+	} else
+#endif /* CONFIG_MIPS_MT_SMP */
+	{
+		unsigned int cpu, ctrl;
+
+		/*
+		 * Set up the counter for a particular CPU when event->cpu is
+		 * a valid CPU number. Otherwise set up the counter for the CPU
+		 * scheduling this thread.
+		 */
+		cpu = (event->cpu >= 0) ? event->cpu : smp_processor_id();
 
+		ctrl = M_PERFCTL_VPEID(cpu_vpe_id(&cpu_data[cpu]));
+		ctrl |= M_TC_EN_VPE;
+		cpuc->saved_ctrl[idx] |= ctrl;
+		pr_debug("Enabling perf counter for CPU%d\n", cpu);
+	}
+#endif /* CONFIG_CPU_BMIPS5000 */
 	/*
 	 * We do not actually let the counter run. Leave it until start().
 	 */
@@ -655,13 +679,14 @@ static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev)
  * event_id.
  */
 #ifdef CONFIG_MIPS_MT_SMP
-	return ((unsigned int)pev->range << 24) |
-		(pev->cntr_mask & 0xffff00) |
-		(pev->event_id & 0xff);
-#else
-	return (pev->cntr_mask & 0xffff00) |
-		(pev->event_id & 0xff);
-#endif
+	if (num_possible_cpus() > 1)
+		return ((unsigned int)pev->range << 24) |
+			(pev->cntr_mask & 0xffff00) |
+			(pev->event_id & 0xff);
+	else
+#endif /* CONFIG_MIPS_MT_SMP */
+		return ((pev->cntr_mask & 0xffff00) |
+			(pev->event_id & 0xff));
 }
 
 static const struct mips_perf_event *mipspmu_map_general_event(int idx)
@@ -1265,37 +1290,6 @@ static const struct mips_perf_event xlp_cache_map
 },
 };
 
-#ifdef CONFIG_MIPS_MT_SMP
-static void check_and_calc_range(struct perf_event *event,
-				 const struct mips_perf_event *pev)
-{
-	struct hw_perf_event *hwc = &event->hw;
-
-	if (event->cpu >= 0) {
-		if (pev->range > V) {
-			/*
-			 * The user selected an event that is processor
-			 * wide, while expecting it to be VPE wide.
-			 */
-			hwc->config_base |= M_TC_EN_ALL;
-		} else {
-			/*
-			 * FIXME: cpu_data[event->cpu].vpe_id reports 0
-			 * for both CPUs.
-			 */
-			hwc->config_base |= M_PERFCTL_VPEID(event->cpu);
-			hwc->config_base |= M_TC_EN_VPE;
-		}
-	} else
-		hwc->config_base |= M_TC_EN_ALL;
-}
-#else
-static void check_and_calc_range(struct perf_event *event,
-				 const struct mips_perf_event *pev)
-{
-}
-#endif
-
 static int __hw_perf_event_init(struct perf_event *event)
 {
 	struct perf_event_attr *attr = &event->attr;
@@ -1331,10 +1325,6 @@ static int __hw_perf_event_init(struct perf_event *event)
 	 */
 	hwc->config_base = MIPS_PERFCTRL_IE;
 
-	/* Calculate range bits and validate it. */
-	if (num_possible_cpus() > 1)
-		check_and_calc_range(event, pev);
-
 	hwc->event_base = mipspmu_perf_event_encode(pev);
 	if (PERF_TYPE_RAW == event->attr.type)
 		mutex_unlock(&raw_event_mutex);
@@ -1723,7 +1713,6 @@ init_hw_perf_events(void)
 	}
 
 #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
-	cpu_has_mipsmt_pertccounters = read_c0_config7() & (1<<19);
 	if (!cpu_has_mipsmt_pertccounters)
 		counters = counters_total_to_per_cpu(counters);
 #endif
diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index 563188ac6fa2..2c96c0c68116 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -93,7 +93,7 @@ void __init add_memory_region(phys_addr_t start, phys_addr_t size, long type)
 	 * If the region reaches the top of the physical address space, adjust
 	 * the size slightly so that (start + size) doesn't overflow
 	 */
-	if (start + size - 1 == (phys_addr_t)ULLONG_MAX)
+	if (start + size - 1 == PHYS_ADDR_MAX)
 		--size;
 
 	/* Sanity check */
@@ -376,7 +376,7 @@ static void __init bootmem_init(void)
 	unsigned long reserved_end;
 	unsigned long mapstart = ~0UL;
 	unsigned long bootmap_size;
-	phys_addr_t ramstart = (phys_addr_t)ULLONG_MAX;
+	phys_addr_t ramstart = PHYS_ADDR_MAX;
 	bool bootmap_valid = false;
 	int i;
 
diff --git a/arch/mips/kernel/signal32.c b/arch/mips/kernel/signal32.c
index c4db910a8794..b5d9e1784aff 100644
--- a/arch/mips/kernel/signal32.c
+++ b/arch/mips/kernel/signal32.c
@@ -8,13 +8,13 @@
  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
  * Copyright (C) 2016, Imagination Technologies Ltd.
  */
+#include <linux/compat.h>
 #include <linux/compiler.h>
 #include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/signal.h>
 #include <linux/syscalls.h>
 
-#include <asm/compat.h>
 #include <asm/compat-signal.h>
 #include <linux/uaccess.h>
 #include <asm/unistd.h>
diff --git a/arch/mips/kernel/time.c b/arch/mips/kernel/time.c
index a6ebc8135112..bfe02ded25d1 100644
--- a/arch/mips/kernel/time.c
+++ b/arch/mips/kernel/time.c
@@ -34,21 +34,6 @@
 DEFINE_SPINLOCK(rtc_lock);
 EXPORT_SYMBOL(rtc_lock);
 
-int __weak rtc_mips_set_time(unsigned long sec)
-{
-	return -ENODEV;
-}
-
-int __weak rtc_mips_set_mmss(unsigned long nowtime)
-{
-	return rtc_mips_set_time(nowtime);
-}
-
-int update_persistent_clock(struct timespec now)
-{
-	return rtc_mips_set_mmss(now.tv_sec);
-}
-
 static int null_perf_irq(void)
 {
 	return 0;
diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index 967e9e4e795e..d67fa74622ee 100644
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -699,17 +699,11 @@ static int simulate_sync(struct pt_regs *regs, unsigned int opcode)
 asmlinkage void do_ov(struct pt_regs *regs)
 {
 	enum ctx_state prev_state;
-	siginfo_t info;
-
-	clear_siginfo(&info);
-	info.si_signo = SIGFPE;
-	info.si_code = FPE_INTOVF;
-	info.si_addr = (void __user *)regs->cp0_epc;
 
 	prev_state = exception_enter();
 	die_if_kernel("Integer overflow", regs);
 
-	force_sig_info(SIGFPE, &info, current);
+	force_sig_fault(SIGFPE, FPE_INTOVF, (void __user *)regs->cp0_epc, current);
 	exception_exit(prev_state);
 }
 
@@ -722,32 +716,27 @@ asmlinkage void do_ov(struct pt_regs *regs)
 void force_fcr31_sig(unsigned long fcr31, void __user *fault_addr,
 		     struct task_struct *tsk)
 {
-	struct siginfo si;
-
-	clear_siginfo(&si);
-	si.si_addr = fault_addr;
-	si.si_signo = SIGFPE;
+	int si_code = FPE_FLTUNK;
 
 	if (fcr31 & FPU_CSR_INV_X)
-		si.si_code = FPE_FLTINV;
+		si_code = FPE_FLTINV;
 	else if (fcr31 & FPU_CSR_DIV_X)
-		si.si_code = FPE_FLTDIV;
+		si_code = FPE_FLTDIV;
 	else if (fcr31 & FPU_CSR_OVF_X)
-		si.si_code = FPE_FLTOVF;
+		si_code = FPE_FLTOVF;
 	else if (fcr31 & FPU_CSR_UDF_X)
-		si.si_code = FPE_FLTUND;
+		si_code = FPE_FLTUND;
 	else if (fcr31 & FPU_CSR_INE_X)
-		si.si_code = FPE_FLTRES;
+		si_code = FPE_FLTRES;
 
-	force_sig_info(SIGFPE, &si, tsk);
+	force_sig_fault(SIGFPE, si_code, fault_addr, tsk);
 }
 
 int process_fpemu_return(int sig, void __user *fault_addr, unsigned long fcr31)
 {
-	struct siginfo si;
+	int si_code;
 	struct vm_area_struct *vma;
 
-	clear_siginfo(&si);
 	switch (sig) {
 	case 0:
 		return 0;
@@ -757,23 +746,18 @@ int process_fpemu_return(int sig, void __user *fault_addr, unsigned long fcr31)
 		return 1;
 
 	case SIGBUS:
-		si.si_addr = fault_addr;
-		si.si_signo = sig;
-		si.si_code = BUS_ADRERR;
-		force_sig_info(sig, &si, current);
+		force_sig_fault(SIGBUS, BUS_ADRERR, fault_addr, current);
 		return 1;
 
 	case SIGSEGV:
-		si.si_addr = fault_addr;
-		si.si_signo = sig;
 		down_read(&current->mm->mmap_sem);
 		vma = find_vma(current->mm, (unsigned long)fault_addr);
 		if (vma && (vma->vm_start <= (unsigned long)fault_addr))
-			si.si_code = SEGV_ACCERR;
+			si_code = SEGV_ACCERR;
 		else
-			si.si_code = SEGV_MAPERR;
+			si_code = SEGV_MAPERR;
 		up_read(&current->mm->mmap_sem);
-		force_sig_info(sig, &si, current);
+		force_sig_fault(SIGSEGV, si_code, fault_addr, current);
 		return 1;
 
 	default:
@@ -896,10 +880,8 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
 void do_trap_or_bp(struct pt_regs *regs, unsigned int code, int si_code,
 	const char *str)
 {
-	siginfo_t info;
 	char b[40];
 
-	clear_siginfo(&info);
 #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
 	if (kgdb_ll_trap(DIE_TRAP, str, regs, code, current->thread.trap_nr,
 			 SIGTRAP) == NOTIFY_STOP)
@@ -921,13 +903,9 @@ void do_trap_or_bp(struct pt_regs *regs, unsigned int code, int si_code,
 	case BRK_DIVZERO:
 		scnprintf(b, sizeof(b), "%s instruction in kernel code", str);
 		die_if_kernel(b, regs);
-		if (code == BRK_DIVZERO)
-			info.si_code = FPE_INTDIV;
-		else
-			info.si_code = FPE_INTOVF;
-		info.si_signo = SIGFPE;
-		info.si_addr = (void __user *) regs->cp0_epc;
-		force_sig_info(SIGFPE, &info, current);
+		force_sig_fault(SIGFPE,
+				code == BRK_DIVZERO ? FPE_INTDIV : FPE_INTOVF,
+				(void __user *) regs->cp0_epc, current);
 		break;
 	case BRK_BUG:
 		die_if_kernel("Kernel bug detected", regs);
@@ -952,9 +930,7 @@ void do_trap_or_bp(struct pt_regs *regs, unsigned int code, int si_code,
 		scnprintf(b, sizeof(b), "%s instruction in kernel code", str);
 		die_if_kernel(b, regs);
 		if (si_code) {
-			info.si_signo = SIGTRAP;
-			info.si_code = si_code;
-			force_sig_info(SIGTRAP, &info, current);
+			force_sig_fault(SIGTRAP, si_code, NULL,	current);
 		} else {
 			force_sig(SIGTRAP, current);
 		}
@@ -1506,13 +1482,8 @@ asmlinkage void do_mdmx(struct pt_regs *regs)
  */
 asmlinkage void do_watch(struct pt_regs *regs)
 {
-	siginfo_t info;
 	enum ctx_state prev_state;
 
-	clear_siginfo(&info);
-	info.si_signo = SIGTRAP;
-	info.si_code = TRAP_HWBKPT;
-
 	prev_state = exception_enter();
 	/*
 	 * Clear WP (bit 22) bit of cause register so we don't loop
@@ -1528,7 +1499,7 @@ asmlinkage void do_watch(struct pt_regs *regs)
 	if (test_tsk_thread_flag(current, TIF_LOAD_WATCH)) {
 		mips_read_watch_registers();
 		local_irq_enable();
-		force_sig_info(SIGTRAP, &info, current);
+		force_sig_fault(SIGTRAP, TRAP_HWBKPT, NULL, current);
 	} else {
 		mips_clear_watch_registers();
 		local_irq_enable();
diff --git a/arch/mips/kernel/vpe.c b/arch/mips/kernel/vpe.c
index 544ea21bfef9..0bef238d2c0c 100644
--- a/arch/mips/kernel/vpe.c
+++ b/arch/mips/kernel/vpe.c
@@ -872,7 +872,7 @@ static ssize_t vpe_write(struct file *file, const char __user *buffer,
 		return -ENODEV;
 
 	if ((count + v->len) > v->plen) {
-		pr_warn("VPE loader: elf size too big. Perhaps strip uneeded symbols\n");
+		pr_warn("VPE loader: elf size too big. Perhaps strip unneeded symbols\n");
 		return -ENOMEM;
 	}
 
diff --git a/arch/mips/lasat/ds1603.c b/arch/mips/lasat/ds1603.c
index 8bd5cf820eed..e6ce39fefa78 100644
--- a/arch/mips/lasat/ds1603.c
+++ b/arch/mips/lasat/ds1603.c
@@ -136,7 +136,7 @@ static void rtc_end_op(void)
 	lasat_ndelay(1000);
 }
 
-void read_persistent_clock(struct timespec *ts)
+void read_persistent_clock64(struct timespec64 *ts)
 {
 	unsigned long word;
 	unsigned long flags;
@@ -152,14 +152,19 @@ void read_persistent_clock(struct timespec *ts)
 	ts->tv_nsec = 0;
 }
 
-int rtc_mips_set_mmss(unsigned long time)
+int update_persistent_clock64(struct timespec64 now)
 {
+	time64_t time = now.tv_sec;
 	unsigned long flags;
 
 	spin_lock_irqsave(&rtc_lock, flags);
 	rtc_init_op();
 	rtc_write_byte(SET_TIME_CMD);
-	rtc_write_word(time);
+	/*
+	 * Due to the hardware limitation, we cast to 'unsigned long' type,
+	 * so it will overflow in year 2106 on 32-bit machine.
+	 */
+	rtc_write_word((unsigned long)time);
 	rtc_end_op();
 	spin_unlock_irqrestore(&rtc_lock, flags);
 
diff --git a/arch/mips/lasat/sysctl.c b/arch/mips/lasat/sysctl.c
index 6f7422400f32..ead07c243c6a 100644
--- a/arch/mips/lasat/sysctl.c
+++ b/arch/mips/lasat/sysctl.c
@@ -73,8 +73,16 @@ int proc_dolasatrtc(struct ctl_table *table, int write,
 	if (r)
 		return r;
 
-	if (write)
-		rtc_mips_set_mmss(rtctmp);
+	if (write) {
+		/*
+		 * Due to the RTC hardware limitation, we can not actually
+		 * use the full 64-bit range here.
+		 */
+		ts.tv_sec = rtctmp;
+		ts.tv_nsec = 0;
+
+		update_persistent_clock64(ts);
+	}
 
 	return 0;
 }
diff --git a/arch/mips/lib/Makefile b/arch/mips/lib/Makefile
index e84e12655fa8..6537e022ef62 100644
--- a/arch/mips/lib/Makefile
+++ b/arch/mips/lib/Makefile
@@ -16,5 +16,4 @@ obj-$(CONFIG_CPU_R3000)		+= r3k_dump_tlb.o
 obj-$(CONFIG_CPU_TX39XX)	+= r3k_dump_tlb.o
 
 # libgcc-style stuff needed in the kernel
-obj-y += ashldi3.o ashrdi3.o bswapsi.o bswapdi.o cmpdi2.o lshrdi3.o multi3.o \
-	 ucmpdi2.o
+obj-y += bswapsi.o bswapdi.o multi3.o
diff --git a/arch/mips/lib/ashldi3.c b/arch/mips/lib/ashldi3.c
deleted file mode 100644
index 24cd6903e797..000000000000
--- a/arch/mips/lib/ashldi3.c
+++ /dev/null
@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/export.h>
-
-#include "libgcc.h"
-
-long long notrace __ashldi3(long long u, word_type b)
-{
-	DWunion uu, w;
-	word_type bm;
-
-	if (b == 0)
-		return u;
-
-	uu.ll = u;
-	bm = 32 - b;
-
-	if (bm <= 0) {
-		w.s.low = 0;
-		w.s.high = (unsigned int) uu.s.low << -bm;
-	} else {
-		const unsigned int carries = (unsigned int) uu.s.low >> bm;
-
-		w.s.low = (unsigned int) uu.s.low << b;
-		w.s.high = ((unsigned int) uu.s.high << b) | carries;
-	}
-
-	return w.ll;
-}
-
-EXPORT_SYMBOL(__ashldi3);
diff --git a/arch/mips/lib/ashrdi3.c b/arch/mips/lib/ashrdi3.c
deleted file mode 100644
index 23f5295af51e..000000000000
--- a/arch/mips/lib/ashrdi3.c
+++ /dev/null
@@ -1,32 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/export.h>
-
-#include "libgcc.h"
-
-long long notrace __ashrdi3(long long u, word_type b)
-{
-	DWunion uu, w;
-	word_type bm;
-
-	if (b == 0)
-		return u;
-
-	uu.ll = u;
-	bm = 32 - b;
-
-	if (bm <= 0) {
-		/* w.s.high = 1..1 or 0..0 */
-		w.s.high =
-		    uu.s.high >> 31;
-		w.s.low = uu.s.high >> -bm;
-	} else {
-		const unsigned int carries = (unsigned int) uu.s.high << bm;
-
-		w.s.high = uu.s.high >> b;
-		w.s.low = ((unsigned int) uu.s.low >> b) | carries;
-	}
-
-	return w.ll;
-}
-
-EXPORT_SYMBOL(__ashrdi3);
diff --git a/arch/mips/lib/cmpdi2.c b/arch/mips/lib/cmpdi2.c
deleted file mode 100644
index 93cfc785927d..000000000000
--- a/arch/mips/lib/cmpdi2.c
+++ /dev/null
@@ -1,28 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/export.h>
-
-#include "libgcc.h"
-
-word_type notrace __cmpdi2(long long a, long long b)
-{
-	const DWunion au = {
-		.ll = a
-	};
-	const DWunion bu = {
-		.ll = b
-	};
-
-	if (au.s.high < bu.s.high)
-		return 0;
-	else if (au.s.high > bu.s.high)
-		return 2;
-
-	if ((unsigned int) au.s.low < (unsigned int) bu.s.low)
-		return 0;
-	else if ((unsigned int) au.s.low > (unsigned int) bu.s.low)
-		return 2;
-
-	return 1;
-}
-
-EXPORT_SYMBOL(__cmpdi2);
diff --git a/arch/mips/lib/lshrdi3.c b/arch/mips/lib/lshrdi3.c
deleted file mode 100644
index 914b971aca3b..000000000000
--- a/arch/mips/lib/lshrdi3.c
+++ /dev/null
@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/export.h>
-
-#include "libgcc.h"
-
-long long notrace __lshrdi3(long long u, word_type b)
-{
-	DWunion uu, w;
-	word_type bm;
-
-	if (b == 0)
-		return u;
-
-	uu.ll = u;
-	bm = 32 - b;
-
-	if (bm <= 0) {
-		w.s.high = 0;
-		w.s.low = (unsigned int) uu.s.high >> -bm;
-	} else {
-		const unsigned int carries = (unsigned int) uu.s.high << bm;
-
-		w.s.high = (unsigned int) uu.s.high >> b;
-		w.s.low = ((unsigned int) uu.s.low >> b) | carries;
-	}
-
-	return w.ll;
-}
-
-EXPORT_SYMBOL(__lshrdi3);
diff --git a/arch/mips/lib/memset.S b/arch/mips/lib/memset.S
index f7327979a8f8..1cc306520a55 100644
--- a/arch/mips/lib/memset.S
+++ b/arch/mips/lib/memset.S
@@ -95,7 +95,7 @@
 
 	sltiu		t0, a2, STORSIZE	/* very small region? */
 	bnez		t0, .Lsmall_memset\@
-	andi		t0, a0, STORMASK	/* aligned? */
+	 andi		t0, a0, STORMASK	/* aligned? */
 
 #ifdef CONFIG_CPU_MICROMIPS
 	move		t8, a1			/* used by 'swp' instruction */
@@ -103,12 +103,12 @@
 #endif
 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
 	beqz		t0, 1f
-	PTR_SUBU	t0, STORSIZE		/* alignment in bytes */
+	 PTR_SUBU	t0, STORSIZE		/* alignment in bytes */
 #else
 	.set		noat
 	li		AT, STORSIZE
 	beqz		t0, 1f
-	PTR_SUBU	t0, AT			/* alignment in bytes */
+	 PTR_SUBU	t0, AT			/* alignment in bytes */
 	.set		at
 #endif
 
@@ -149,7 +149,7 @@
 1:	ori		t1, a2, 0x3f		/* # of full blocks */
 	xori		t1, 0x3f
 	beqz		t1, .Lmemset_partial\@	/* no block to fill */
-	andi		t0, a2, 0x40-STORSIZE
+	 andi		t0, a2, 0x40-STORSIZE
 
 	PTR_ADDU	t1, a0			/* end address */
 	.set		reorder
@@ -174,7 +174,7 @@
 	.set		at
 #endif
 	jr		t1
-	PTR_ADDU	a0, t0			/* dest ptr */
+	 PTR_ADDU	a0, t0			/* dest ptr */
 
 	.set		push
 	.set		noreorder
@@ -186,7 +186,7 @@
 
 	beqz		a2, 1f
 #ifndef CONFIG_CPU_MIPSR6
-	PTR_ADDU	a0, a2			/* What's left */
+	 PTR_ADDU	a0, a2			/* What's left */
 	R10KCBARRIER(0(ra))
 #ifdef __MIPSEB__
 	EX(LONG_S_R, a1, -1(a0), .Llast_fixup\@)
@@ -194,7 +194,7 @@
 	EX(LONG_S_L, a1, -1(a0), .Llast_fixup\@)
 #endif
 #else
-	PTR_SUBU	t0, $0, a2
+	 PTR_SUBU	t0, $0, a2
 	PTR_ADDIU	t0, 1
 	STORE_BYTE(0)
 	STORE_BYTE(1)
@@ -210,11 +210,11 @@
 0:
 #endif
 1:	jr		ra
-	move		a2, zero
+	 move		a2, zero
 
 .Lsmall_memset\@:
 	beqz		a2, 2f
-	PTR_ADDU	t1, a0, a2
+	 PTR_ADDU	t1, a0, a2
 
 1:	PTR_ADDIU	a0, 1			/* fill bytewise */
 	R10KCBARRIER(0(ra))
@@ -222,7 +222,7 @@
 	 EX(sb, a1, -1(a0), .Lsmall_fixup\@)
 
 2:	jr		ra			/* done */
-	move		a2, zero
+	 move		a2, zero
 	.if __memset == 1
 	END(memset)
 	.set __memset, 0
@@ -238,7 +238,7 @@
 
 .Lfirst_fixup\@:
 	jr	ra
-	nop
+	 nop
 
 .Lfwd_fixup\@:
 	PTR_L		t0, TI_TASK($28)
@@ -246,7 +246,7 @@
 	LONG_L		t0, THREAD_BUADDR(t0)
 	LONG_ADDU	a2, t1
 	jr		ra
-	LONG_SUBU	a2, t0
+	 LONG_SUBU	a2, t0
 
 .Lpartial_fixup\@:
 	PTR_L		t0, TI_TASK($28)
@@ -254,7 +254,7 @@
 	LONG_L		t0, THREAD_BUADDR(t0)
 	LONG_ADDU	a2, a0
 	jr		ra
-	LONG_SUBU	a2, t0
+	 LONG_SUBU	a2, t0
 
 .Llast_fixup\@:
 	jr		ra
@@ -278,7 +278,7 @@
 LEAF(memset)
 EXPORT_SYMBOL(memset)
 	beqz		a1, 1f
-	move		v0, a0			/* result */
+	 move		v0, a0			/* result */
 
 	andi		a1, 0xff		/* spread fillword */
 	LONG_SLL		t1, a1, 8
diff --git a/arch/mips/lib/ucmpdi2.c b/arch/mips/lib/ucmpdi2.c
deleted file mode 100644
index c31c78ca4175..000000000000
--- a/arch/mips/lib/ucmpdi2.c
+++ /dev/null
@@ -1,22 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/export.h>
-
-#include "libgcc.h"
-
-word_type notrace __ucmpdi2(unsigned long long a, unsigned long long b)
-{
-	const DWunion au = {.ll = a};
-	const DWunion bu = {.ll = b};
-
-	if ((unsigned int) au.s.high < (unsigned int) bu.s.high)
-		return 0;
-	else if ((unsigned int) au.s.high > (unsigned int) bu.s.high)
-		return 2;
-	if ((unsigned int) au.s.low < (unsigned int) bu.s.low)
-		return 0;
-	else if ((unsigned int) au.s.low > (unsigned int) bu.s.low)
-		return 2;
-	return 1;
-}
-
-EXPORT_SYMBOL(__ucmpdi2);
diff --git a/arch/mips/loongson64/Kconfig b/arch/mips/loongson64/Kconfig
index 72af0c183969..c79e6a565572 100644
--- a/arch/mips/loongson64/Kconfig
+++ b/arch/mips/loongson64/Kconfig
@@ -130,21 +130,6 @@ config LOONGSON_UART_BASE
 	default y
 	depends on EARLY_PRINTK || SERIAL_8250
 
-config IOMMU_HELPER
-	bool
-
-config NEED_SG_DMA_LENGTH
-	bool
-
-config SWIOTLB
-	bool "Soft IOMMU Support for All-Memory DMA"
-	default y
-	depends on CPU_LOONGSON3
-	select DMA_DIRECT_OPS
-	select IOMMU_HELPER
-	select NEED_SG_DMA_LENGTH
-	select NEED_DMA_MAP_STATE
-
 config PHYS48_TO_HT40
 	bool
 	default y if CPU_LOONGSON3
diff --git a/arch/mips/loongson64/common/time.c b/arch/mips/loongson64/common/time.c
index e1a5382ad47e..0ba53c55ff33 100644
--- a/arch/mips/loongson64/common/time.c
+++ b/arch/mips/loongson64/common/time.c
@@ -29,7 +29,7 @@ void __init plat_time_init(void)
 #endif
 }
 
-void read_persistent_clock(struct timespec *ts)
+void read_persistent_clock64(struct timespec64 *ts)
 {
 	ts->tv_sec = mc146818_get_cmos_time();
 	ts->tv_nsec = 0;
diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c
index dcafa43613b6..f9fef0028ca2 100644
--- a/arch/mips/mm/dma-default.c
+++ b/arch/mips/mm/dma-default.c
@@ -402,13 +402,3 @@ static const struct dma_map_ops mips_default_dma_map_ops = {
 
 const struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
 EXPORT_SYMBOL(mips_dma_map_ops);
-
-#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
-
-static int __init mips_dma_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-
-	return 0;
-}
-fs_initcall(mips_dma_init);
diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
index 4f8f5bf46977..5f71f2b903b7 100644
--- a/arch/mips/mm/fault.c
+++ b/arch/mips/mm/fault.c
@@ -42,7 +42,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
 	struct task_struct *tsk = current;
 	struct mm_struct *mm = tsk->mm;
 	const int field = sizeof(unsigned long) * 2;
-	siginfo_t info;
+	int si_code;
 	int fault;
 	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 
@@ -63,7 +63,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
 		return;
 #endif
 
-	info.si_code = SEGV_MAPERR;
+	si_code = SEGV_MAPERR;
 
 	/*
 	 * We fault-in kernel-space virtual memory on-demand. The
@@ -112,7 +112,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
  * we can handle it..
  */
 good_area:
-	info.si_code = SEGV_ACCERR;
+	si_code = SEGV_ACCERR;
 
 	if (write) {
 		if (!(vma->vm_flags & VM_WRITE))
@@ -223,11 +223,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
 			pr_cont("\n");
 		}
 		current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f;
-		info.si_signo = SIGSEGV;
-		info.si_errno = 0;
-		/* info.si_code has been set above */
-		info.si_addr = (void __user *) address;
-		force_sig_info(SIGSEGV, &info, tsk);
+		force_sig_fault(SIGSEGV, si_code, (void __user *)address, tsk);
 		return;
 	}
 
@@ -283,11 +279,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
 #endif
 	current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f;
 	tsk->thread.cp0_badvaddr = address;
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRERR;
-	info.si_addr = (void __user *) address;
-	force_sig_info(SIGBUS, &info, tsk);
+	force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address, tsk);
 
 	return;
 #ifndef CONFIG_64BIT
diff --git a/arch/mips/mm/sc-debugfs.c b/arch/mips/mm/sc-debugfs.c
index 2e2132d3f5c7..2a116084216f 100644
--- a/arch/mips/mm/sc-debugfs.c
+++ b/arch/mips/mm/sc-debugfs.c
@@ -31,17 +31,10 @@ static ssize_t sc_prefetch_write(struct file *file,
 				 const char __user *user_buf,
 				 size_t count, loff_t *ppos)
 {
-	char buf[32];
-	ssize_t buf_size;
 	bool enabled;
 	int err;
 
-	buf_size = min(count, sizeof(buf) - 1);
-	if (copy_from_user(buf, user_buf, buf_size))
-		return -EFAULT;
-
-	buf[buf_size] = '\0';
-	err = strtobool(buf, &enabled);
+	err = kstrtobool_from_user(user_buf, count, &enabled);
 	if (err)
 		return err;
 
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 79b9f2ad3ff5..3d3dfba465ae 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -115,33 +115,17 @@ static int use_lwx_insns(void)
 		return 0;
 	}
 }
-#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \
-    CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
-static bool scratchpad_available(void)
-{
-	return true;
-}
-static int scratchpad_offset(int i)
-{
-	/*
-	 * CVMSEG starts at address -32768 and extends for
-	 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines.
-	 */
-	i += 1; /* Kernel use starts at the top and works down. */
-	return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768;
-}
-#else
-static bool scratchpad_available(void)
-{
-	return false;
-}
+
 static int scratchpad_offset(int i)
 {
+	if (IS_ENABLED(CONFIG_CPU_CAVIUM_OCTEON))
+		return (CAVIUM_OCTEON_SCRATCH_OFFSET - (8 * i));
+
 	BUG();
 	/* Really unreachable, but evidently some GCC want this. */
 	return 0;
 }
-#endif
+
 /*
  * Found by experiment: At least some revisions of the 4kc throw under
  * some circumstances a machine check exception, triggered by invalid
@@ -1302,7 +1286,8 @@ static void build_r4000_tlb_refill_handler(void)
 	memset(relocs, 0, sizeof(relocs));
 	memset(final_handler, 0, sizeof(final_handler));
 
-	if (IS_ENABLED(CONFIG_64BIT) && (scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) {
+	if (IS_ENABLED(CONFIG_64BIT) && use_bbit_insns() &&
+	   (scratch_reg >= 0 || IS_ENABLED(CONFIG_CPU_CAVIUM_OCTEON))) {
 		htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
 							  scratch_reg);
 		vmalloc_mode = refill_scratch;
diff --git a/arch/mips/mti-malta/malta-time.c b/arch/mips/mti-malta/malta-time.c
index 66c866740ff2..d22b7edc3886 100644
--- a/arch/mips/mti-malta/malta-time.c
+++ b/arch/mips/mti-malta/malta-time.c
@@ -134,7 +134,7 @@ static void __init estimate_frequencies(void)
 	}
 }
 
-void read_persistent_clock(struct timespec *ts)
+void read_persistent_clock64(struct timespec64 *ts)
 {
 	ts->tv_sec = mc146818_get_cmos_time();
 	ts->tv_nsec = 0;
diff --git a/arch/mips/net/ebpf_jit.c b/arch/mips/net/ebpf_jit.c
index 3e2798bfea4f..aeb7b1b0f202 100644
--- a/arch/mips/net/ebpf_jit.c
+++ b/arch/mips/net/ebpf_jit.c
@@ -95,7 +95,6 @@ enum reg_val_type {
  * struct jit_ctx - JIT context
  * @skf:		The sk_filter
  * @stack_size:		eBPF stack size
- * @tmp_offset:		eBPF $sp offset to 8-byte temporary memory
  * @idx:		Instruction index
  * @flags:		JIT flags
  * @offsets:		Instruction offsets
@@ -105,7 +104,6 @@ enum reg_val_type {
 struct jit_ctx {
 	const struct bpf_prog *skf;
 	int stack_size;
-	int tmp_offset;
 	u32 idx;
 	u32 flags;
 	u32 *offsets;
@@ -293,7 +291,6 @@ static int gen_int_prologue(struct jit_ctx *ctx)
 	locals_size = (ctx->flags & EBPF_SEEN_FP) ? MAX_BPF_STACK : 0;
 
 	stack_adjust += locals_size;
-	ctx->tmp_offset = locals_size;
 
 	ctx->stack_size = stack_adjust;
 
@@ -399,7 +396,6 @@ static void gen_imm_to_reg(const struct bpf_insn *insn, int reg,
 		emit_instr(ctx, lui, reg, upper >> 16);
 		emit_instr(ctx, addiu, reg, reg, lower);
 	}
-
 }
 
 static int gen_imm_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
@@ -547,28 +543,6 @@ static int gen_imm_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 	return 0;
 }
 
-static void * __must_check
-ool_skb_header_pointer(const struct sk_buff *skb, int offset,
-		       int len, void *buffer)
-{
-	return skb_header_pointer(skb, offset, len, buffer);
-}
-
-static int size_to_len(const struct bpf_insn *insn)
-{
-	switch (BPF_SIZE(insn->code)) {
-	case BPF_B:
-		return 1;
-	case BPF_H:
-		return 2;
-	case BPF_W:
-		return 4;
-	case BPF_DW:
-		return 8;
-	}
-	return 0;
-}
-
 static void emit_const_to_reg(struct jit_ctx *ctx, int dst, u64 value)
 {
 	if (value >= 0xffffffffffff8000ull || value < 0x8000ull) {
@@ -1267,110 +1241,6 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 			return -EINVAL;
 		break;
 
-	case BPF_LD | BPF_B | BPF_ABS:
-	case BPF_LD | BPF_H | BPF_ABS:
-	case BPF_LD | BPF_W | BPF_ABS:
-	case BPF_LD | BPF_DW | BPF_ABS:
-		ctx->flags |= EBPF_SAVE_RA;
-
-		gen_imm_to_reg(insn, MIPS_R_A1, ctx);
-		emit_instr(ctx, addiu, MIPS_R_A2, MIPS_R_ZERO, size_to_len(insn));
-
-		if (insn->imm < 0) {
-			emit_const_to_reg(ctx, MIPS_R_T9, (u64)bpf_internal_load_pointer_neg_helper);
-		} else {
-			emit_const_to_reg(ctx, MIPS_R_T9, (u64)ool_skb_header_pointer);
-			emit_instr(ctx, daddiu, MIPS_R_A3, MIPS_R_SP, ctx->tmp_offset);
-		}
-		goto ld_skb_common;
-
-	case BPF_LD | BPF_B | BPF_IND:
-	case BPF_LD | BPF_H | BPF_IND:
-	case BPF_LD | BPF_W | BPF_IND:
-	case BPF_LD | BPF_DW | BPF_IND:
-		ctx->flags |= EBPF_SAVE_RA;
-		src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
-		if (src < 0)
-			return src;
-		ts = get_reg_val_type(ctx, this_idx, insn->src_reg);
-		if (ts == REG_32BIT_ZERO_EX) {
-			/* sign extend */
-			emit_instr(ctx, sll, MIPS_R_A1, src, 0);
-			src = MIPS_R_A1;
-		}
-		if (insn->imm >= S16_MIN && insn->imm <= S16_MAX) {
-			emit_instr(ctx, daddiu, MIPS_R_A1, src, insn->imm);
-		} else {
-			gen_imm_to_reg(insn, MIPS_R_AT, ctx);
-			emit_instr(ctx, daddu, MIPS_R_A1, MIPS_R_AT, src);
-		}
-		/* truncate to 32-bit int */
-		emit_instr(ctx, sll, MIPS_R_A1, MIPS_R_A1, 0);
-		emit_instr(ctx, daddiu, MIPS_R_A3, MIPS_R_SP, ctx->tmp_offset);
-		emit_instr(ctx, slt, MIPS_R_AT, MIPS_R_A1, MIPS_R_ZERO);
-
-		emit_const_to_reg(ctx, MIPS_R_T8, (u64)bpf_internal_load_pointer_neg_helper);
-		emit_const_to_reg(ctx, MIPS_R_T9, (u64)ool_skb_header_pointer);
-		emit_instr(ctx, addiu, MIPS_R_A2, MIPS_R_ZERO, size_to_len(insn));
-		emit_instr(ctx, movn, MIPS_R_T9, MIPS_R_T8, MIPS_R_AT);
-
-ld_skb_common:
-		emit_instr(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
-		/* delay slot move */
-		emit_instr(ctx, daddu, MIPS_R_A0, MIPS_R_S0, MIPS_R_ZERO);
-
-		/* Check the error value */
-		b_off = b_imm(exit_idx, ctx);
-		if (is_bad_offset(b_off)) {
-			target = j_target(ctx, exit_idx);
-			if (target == (unsigned int)-1)
-				return -E2BIG;
-
-			if (!(ctx->offsets[this_idx] & OFFSETS_B_CONV)) {
-				ctx->offsets[this_idx] |= OFFSETS_B_CONV;
-				ctx->long_b_conversion = 1;
-			}
-			emit_instr(ctx, bne, MIPS_R_V0, MIPS_R_ZERO, 4 * 3);
-			emit_instr(ctx, nop);
-			emit_instr(ctx, j, target);
-			emit_instr(ctx, nop);
-		} else {
-			emit_instr(ctx, beq, MIPS_R_V0, MIPS_R_ZERO, b_off);
-			emit_instr(ctx, nop);
-		}
-
-#ifdef __BIG_ENDIAN
-		need_swap = false;
-#else
-		need_swap = true;
-#endif
-		dst = MIPS_R_V0;
-		switch (BPF_SIZE(insn->code)) {
-		case BPF_B:
-			emit_instr(ctx, lbu, dst, 0, MIPS_R_V0);
-			break;
-		case BPF_H:
-			emit_instr(ctx, lhu, dst, 0, MIPS_R_V0);
-			if (need_swap)
-				emit_instr(ctx, wsbh, dst, dst);
-			break;
-		case BPF_W:
-			emit_instr(ctx, lw, dst, 0, MIPS_R_V0);
-			if (need_swap) {
-				emit_instr(ctx, wsbh, dst, dst);
-				emit_instr(ctx, rotr, dst, dst, 16);
-			}
-			break;
-		case BPF_DW:
-			emit_instr(ctx, ld, dst, 0, MIPS_R_V0);
-			if (need_swap) {
-				emit_instr(ctx, dsbh, dst, dst);
-				emit_instr(ctx, dshd, dst, dst);
-			}
-			break;
-		}
-
-		break;
 	case BPF_ALU | BPF_END | BPF_FROM_BE:
 	case BPF_ALU | BPF_END | BPF_FROM_LE:
 		dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
diff --git a/arch/mips/netlogic/Kconfig b/arch/mips/netlogic/Kconfig
index 7fcfc7fe9f14..412351c5acc6 100644
--- a/arch/mips/netlogic/Kconfig
+++ b/arch/mips/netlogic/Kconfig
@@ -83,10 +83,4 @@ endif
 config NLM_COMMON
 	bool
 
-config IOMMU_HELPER
-	bool
-
-config NEED_SG_DMA_LENGTH
-	bool
-
 endif
diff --git a/arch/mips/oprofile/op_model_mipsxx.c b/arch/mips/oprofile/op_model_mipsxx.c
index c3e4c18ef8d4..7c04b17f4a48 100644
--- a/arch/mips/oprofile/op_model_mipsxx.c
+++ b/arch/mips/oprofile/op_model_mipsxx.c
@@ -36,7 +36,6 @@ static int perfcount_irq;
 #endif
 
 #ifdef CONFIG_MIPS_MT_SMP
-static int cpu_has_mipsmt_pertccounters;
 #define WHAT		(MIPS_PERFCTRL_MT_EN_VPE | \
 			 M_PERFCTL_VPEID(cpu_vpe_id(&current_cpu_data)))
 #define vpe_id()	(cpu_has_mipsmt_pertccounters ? \
@@ -326,7 +325,6 @@ static int __init mipsxx_init(void)
 	}
 
 #ifdef CONFIG_MIPS_MT_SMP
-	cpu_has_mipsmt_pertccounters = read_c0_config7() & (1<<19);
 	if (!cpu_has_mipsmt_pertccounters)
 		counters = counters_total_to_per_cpu(counters);
 #endif
diff --git a/arch/mips/pci/ops-pmcmsp.c b/arch/mips/pci/ops-pmcmsp.c
index dd2d9f7e9412..7649372103af 100644
--- a/arch/mips/pci/ops-pmcmsp.c
+++ b/arch/mips/pci/ops-pmcmsp.c
@@ -83,18 +83,6 @@ static int show_msp_pci_counts(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int msp_pci_rd_cnt_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, show_msp_pci_counts, NULL);
-}
-
-static const struct file_operations msp_pci_rd_cnt_fops = {
-	.open		= msp_pci_rd_cnt_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 /*****************************************************************************
  *
  *  FUNCTION: gen_pci_cfg_wr_show
@@ -160,18 +148,6 @@ static int gen_pci_cfg_wr_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int gen_pci_cfg_wr_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, gen_pci_cfg_wr_show, NULL);
-}
-
-static const struct file_operations gen_pci_cfg_wr_fops = {
-	.open		= gen_pci_cfg_wr_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 /*****************************************************************************
  *
  *  FUNCTION: pci_proc_init
@@ -188,8 +164,8 @@ static const struct file_operations gen_pci_cfg_wr_fops = {
  ****************************************************************************/
 static void pci_proc_init(void)
 {
-	proc_create("pmc_msp_pci_rd_cnt", 0, NULL, &msp_pci_rd_cnt_fops);
-	proc_create("pmc_msp_pci_cfg_wr", 0, NULL, &gen_pci_cfg_wr_fops);
+	proc_create_single("pmc_msp_pci_rd_cnt", 0, NULL, show_msp_pci_counts);
+	proc_create_single("pmc_msp_pci_cfg_wr", 0, NULL, gen_pci_cfg_wr_show);
 }
 #endif /* CONFIG_PROC_FS && PCI_COUNTERS */
 
diff --git a/arch/mips/pci/pci-legacy.c b/arch/mips/pci/pci-legacy.c
index 0c65c38e05d6..f1e92bf743c2 100644
--- a/arch/mips/pci/pci-legacy.c
+++ b/arch/mips/pci/pci-legacy.c
@@ -263,9 +263,8 @@ static int pcibios_enable_resources(struct pci_dev *dev, int mask)
 				(!(r->flags & IORESOURCE_ROM_ENABLE)))
 			continue;
 		if (!r->start && r->end) {
-			printk(KERN_ERR "PCI: Device %s not available "
-			       "because of resource collisions\n",
-			       pci_name(dev));
+			pci_err(dev,
+				"can't enable device: resource collisions\n");
 			return -EINVAL;
 		}
 		if (r->flags & IORESOURCE_IO)
@@ -274,8 +273,7 @@ static int pcibios_enable_resources(struct pci_dev *dev, int mask)
 			cmd |= PCI_COMMAND_MEMORY;
 	}
 	if (cmd != old_cmd) {
-		printk("PCI: Enabling device %s (%04x -> %04x)\n",
-		       pci_name(dev), old_cmd, cmd);
+		pci_info(dev, "enabling device (%04x -> %04x)\n", old_cmd, cmd);
 		pci_write_config_word(dev, PCI_COMMAND, cmd);
 	}
 	return 0;
diff --git a/arch/mips/sibyte/common/bus_watcher.c b/arch/mips/sibyte/common/bus_watcher.c
index a4e55999ecb4..4bb85de9229b 100644
--- a/arch/mips/sibyte/common/bus_watcher.c
+++ b/arch/mips/sibyte/common/bus_watcher.c
@@ -142,24 +142,12 @@ static int bw_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int bw_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, bw_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations bw_proc_fops = {
-	.open		= bw_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static void create_proc_decoder(struct bw_stats_struct *stats)
 {
 	struct proc_dir_entry *ent;
 
-	ent = proc_create_data("bus_watcher", S_IWUSR | S_IRUGO, NULL,
-			       &bw_proc_fops, stats);
+	ent = proc_create_single_data("bus_watcher", S_IWUSR | S_IRUGO, NULL,
+			bw_proc_show, stats);
 	if (!ent) {
 		printk(KERN_INFO "Unable to initialize bus_watcher /proc entry\n");
 		return;
diff --git a/arch/mips/sibyte/swarm/rtc_m41t81.c b/arch/mips/sibyte/swarm/rtc_m41t81.c
index e62466445f08..4ac8ccdf56bb 100644
--- a/arch/mips/sibyte/swarm/rtc_m41t81.c
+++ b/arch/mips/sibyte/swarm/rtc_m41t81.c
@@ -141,13 +141,13 @@ static int m41t81_write(uint8_t addr, int b)
 	return 0;
 }
 
-int m41t81_set_time(unsigned long t)
+int m41t81_set_time(time64_t t)
 {
 	struct rtc_time tm;
 	unsigned long flags;
 
 	/* Note we don't care about the century */
-	rtc_time_to_tm(t, &tm);
+	rtc_time64_to_tm(t, &tm);
 
 	/*
 	 * Note the write order matters as it ensures the correctness.
@@ -188,7 +188,7 @@ int m41t81_set_time(unsigned long t)
 	return 0;
 }
 
-unsigned long m41t81_get_time(void)
+time64_t m41t81_get_time(void)
 {
 	unsigned int year, mon, day, hour, min, sec;
 	unsigned long flags;
@@ -218,7 +218,7 @@ unsigned long m41t81_get_time(void)
 
 	year += 2000;
 
-	return mktime(year, mon, day, hour, min, sec);
+	return mktime64(year, mon, day, hour, min, sec);
 }
 
 int m41t81_probe(void)
diff --git a/arch/mips/sibyte/swarm/rtc_xicor1241.c b/arch/mips/sibyte/swarm/rtc_xicor1241.c
index 50a82c495427..2dcaaa7e3bfa 100644
--- a/arch/mips/sibyte/swarm/rtc_xicor1241.c
+++ b/arch/mips/sibyte/swarm/rtc_xicor1241.c
@@ -109,13 +109,13 @@ static int xicor_write(uint8_t addr, int b)
 	}
 }
 
-int xicor_set_time(unsigned long t)
+int xicor_set_time(time64_t t)
 {
 	struct rtc_time tm;
 	int tmp;
 	unsigned long flags;
 
-	rtc_time_to_tm(t, &tm);
+	rtc_time64_to_tm(t, &tm);
 	tm.tm_year += 1900;
 
 	spin_lock_irqsave(&rtc_lock, flags);
@@ -168,7 +168,7 @@ int xicor_set_time(unsigned long t)
 	return 0;
 }
 
-unsigned long xicor_get_time(void)
+time64_t xicor_get_time(void)
 {
 	unsigned int year, mon, day, hour, min, sec, y2k;
 	unsigned long flags;
@@ -201,7 +201,7 @@ unsigned long xicor_get_time(void)
 
 	year += (y2k * 100);
 
-	return mktime(year, mon, day, hour, min, sec);
+	return mktime64(year, mon, day, hour, min, sec);
 }
 
 int xicor_probe(void)
diff --git a/arch/mips/sibyte/swarm/setup.c b/arch/mips/sibyte/swarm/setup.c
index 494fb0a475ac..152ca71cc2d7 100644
--- a/arch/mips/sibyte/swarm/setup.c
+++ b/arch/mips/sibyte/swarm/setup.c
@@ -57,12 +57,12 @@ extern void sb1250_setup(void);
 #endif
 
 extern int xicor_probe(void);
-extern int xicor_set_time(unsigned long);
-extern unsigned long xicor_get_time(void);
+extern int xicor_set_time(time64_t);
+extern time64_t xicor_get_time(void);
 
 extern int m41t81_probe(void);
-extern int m41t81_set_time(unsigned long);
-extern unsigned long m41t81_get_time(void);
+extern int m41t81_set_time(time64_t);
+extern time64_t m41t81_get_time(void);
 
 const char *get_system_type(void)
 {
@@ -87,9 +87,9 @@ enum swarm_rtc_type {
 
 enum swarm_rtc_type swarm_rtc_type;
 
-void read_persistent_clock(struct timespec *ts)
+void read_persistent_clock64(struct timespec64 *ts)
 {
-	unsigned long sec;
+	time64_t sec;
 
 	switch (swarm_rtc_type) {
 	case RTC_XICOR:
@@ -102,15 +102,17 @@ void read_persistent_clock(struct timespec *ts)
 
 	case RTC_NONE:
 	default:
-		sec = mktime(2000, 1, 1, 0, 0, 0);
+		sec = mktime64(2000, 1, 1, 0, 0, 0);
 		break;
 	}
 	ts->tv_sec = sec;
 	ts->tv_nsec = 0;
 }
 
-int rtc_mips_set_time(unsigned long sec)
+int update_persistent_clock64(struct timespec64 now)
 {
+	time64_t sec = now.tv_sec;
+
 	switch (swarm_rtc_type) {
 	case RTC_XICOR:
 		return xicor_set_time(sec);
diff --git a/arch/mips/sni/time.c b/arch/mips/sni/time.c
index 0eb7d1e8821b..dbace1f3e1a9 100644
--- a/arch/mips/sni/time.c
+++ b/arch/mips/sni/time.c
@@ -171,9 +171,3 @@ void __init plat_time_init(void)
 	}
 	setup_pit_timer();
 }
-
-void read_persistent_clock(struct timespec *ts)
-{
-	ts->tv_sec = -1;
-	ts->tv_nsec = 0;
-}
diff --git a/arch/nds32/Kconfig b/arch/nds32/Kconfig
index 249f38d3388f..6aed974276d8 100644
--- a/arch/nds32/Kconfig
+++ b/arch/nds32/Kconfig
@@ -5,10 +5,19 @@
 
 config NDS32
         def_bool y
+	select ARCH_HAS_SYNC_DMA_FOR_CPU
+	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
 	select ARCH_WANT_FRAME_POINTERS if FTRACE
 	select CLKSRC_MMIO
 	select CLONE_BACKWARDS
 	select COMMON_CLK
+	select DMA_NONCOHERENT_OPS
+	select GENERIC_ASHLDI3
+	select GENERIC_ASHRDI3
+	select GENERIC_LSHRDI3
+	select GENERIC_CMPDI2
+	select GENERIC_MULDI3
+	select GENERIC_UCMPDI2
 	select GENERIC_ATOMIC64
 	select GENERIC_CPU_DEVICES
 	select GENERIC_CLOCKEVENTS
@@ -82,6 +91,7 @@ endmenu
 
 menu "Kernel Features"
 source "kernel/Kconfig.preempt"
+source "kernel/Kconfig.freezer"
 source "mm/Kconfig"
 source "kernel/Kconfig.hz"
 endmenu
diff --git a/arch/nds32/Kconfig.cpu b/arch/nds32/Kconfig.cpu
index ba44cc539da9..b8c8984d1456 100644
--- a/arch/nds32/Kconfig.cpu
+++ b/arch/nds32/Kconfig.cpu
@@ -1,10 +1,11 @@
 comment "Processor Features"
 
 config CPU_BIG_ENDIAN
-	bool "Big endian"
+	def_bool !CPU_LITTLE_ENDIAN
 
 config CPU_LITTLE_ENDIAN
-        def_bool !CPU_BIG_ENDIAN
+	bool "Little endian"
+	default y
 
 config HWZOL
 	bool "hardware zero overhead loop support"
diff --git a/arch/nds32/Makefile b/arch/nds32/Makefile
index 91f933d5a962..513bb2e9baf9 100644
--- a/arch/nds32/Makefile
+++ b/arch/nds32/Makefile
@@ -23,9 +23,6 @@ export	TEXTADDR
 # If we have a machine-specific directory, then include it in the build.
 core-y				+= arch/nds32/kernel/ arch/nds32/mm/
 libs-y				+= arch/nds32/lib/
-LIBGCC_PATH     		:= \
-  $(shell $(CC) $(KBUILD_CFLAGS) $(KCFLAGS) -print-libgcc-file-name)
-libs-y				+= $(LIBGCC_PATH)
 
 ifneq '$(CONFIG_NDS32_BUILTIN_DTB)' '""'
 BUILTIN_DTB := y
@@ -35,8 +32,12 @@ endif
 
 ifdef CONFIG_CPU_LITTLE_ENDIAN
 KBUILD_CFLAGS   += $(call cc-option, -EL)
+KBUILD_AFLAGS   += $(call cc-option, -EL)
+LDFLAGS         += $(call cc-option, -EL)
 else
 KBUILD_CFLAGS   += $(call cc-option, -EB)
+KBUILD_AFLAGS   += $(call cc-option, -EB)
+LDFLAGS         += $(call cc-option, -EB)
 endif
 
 boot := arch/nds32/boot
diff --git a/arch/nds32/include/asm/Kbuild b/arch/nds32/include/asm/Kbuild
index 06bdf8167f5a..dbc4e5422550 100644
--- a/arch/nds32/include/asm/Kbuild
+++ b/arch/nds32/include/asm/Kbuild
@@ -9,13 +9,16 @@ generic-y += checksum.h
 generic-y += clkdev.h
 generic-y += cmpxchg.h
 generic-y += cmpxchg-local.h
+generic-y += compat.h
 generic-y += cputime.h
 generic-y += device.h
 generic-y += div64.h
 generic-y += dma.h
+generic-y += dma-mapping.h
 generic-y += emergency-restart.h
 generic-y += errno.h
 generic-y += exec.h
+generic-y += export.h
 generic-y += fb.h
 generic-y += fcntl.h
 generic-y += ftrace.h
@@ -49,6 +52,7 @@ generic-y += switch_to.h
 generic-y += timex.h
 generic-y += topology.h
 generic-y += trace_clock.h
+generic-y += xor.h
 generic-y += unaligned.h
 generic-y += user.h
 generic-y += vga.h
diff --git a/arch/nds32/include/asm/bitfield.h b/arch/nds32/include/asm/bitfield.h
index c73f71d67744..8e84fc385b94 100644
--- a/arch/nds32/include/asm/bitfield.h
+++ b/arch/nds32/include/asm/bitfield.h
@@ -336,7 +336,7 @@
 #define INT_MASK_mskIDIVZE	( 0x1  << INT_MASK_offIDIVZE )
 #define INT_MASK_mskDSSIM	( 0x1  << INT_MASK_offDSSIM )
 
-#define INT_MASK_INITAIAL_VAL	0x10003
+#define INT_MASK_INITAIAL_VAL	(INT_MASK_mskDSSIM|INT_MASK_mskIDIVZE)
 
 /******************************************************************************
  * ir15: INT_PEND (Interrupt Pending Register)
@@ -396,6 +396,7 @@
 #define MMU_CTL_D8KB		1
 #define MMU_CTL_UNA		( 0x1  << MMU_CTL_offUNA )
 
+#define MMU_CTL_CACHEABLE_NON   0
 #define MMU_CTL_CACHEABLE_WB	2
 #define MMU_CTL_CACHEABLE_WT	3
 
diff --git a/arch/nds32/include/asm/cacheflush.h b/arch/nds32/include/asm/cacheflush.h
index 1240f148ec0f..10b48f0d8e85 100644
--- a/arch/nds32/include/asm/cacheflush.h
+++ b/arch/nds32/include/asm/cacheflush.h
@@ -32,6 +32,8 @@ void flush_anon_page(struct vm_area_struct *vma,
 
 #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
 void flush_kernel_dcache_page(struct page *page);
+void flush_kernel_vmap_range(void *addr, int size);
+void invalidate_kernel_vmap_range(void *addr, int size);
 void flush_icache_range(unsigned long start, unsigned long end);
 void flush_icache_page(struct vm_area_struct *vma, struct page *page);
 #define flush_dcache_mmap_lock(mapping)   xa_lock_irq(&(mapping)->i_pages)
diff --git a/arch/nds32/include/asm/dma-mapping.h b/arch/nds32/include/asm/dma-mapping.h
deleted file mode 100644
index 2dd47d245c25..000000000000
--- a/arch/nds32/include/asm/dma-mapping.h
+++ /dev/null
@@ -1,14 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-// Copyright (C) 2005-2017 Andes Technology Corporation
-
-#ifndef ASMNDS32_DMA_MAPPING_H
-#define ASMNDS32_DMA_MAPPING_H
-
-extern struct dma_map_ops nds32_dma_ops;
-
-static inline struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
-{
-	return &nds32_dma_ops;
-}
-
-#endif
diff --git a/arch/nds32/include/asm/io.h b/arch/nds32/include/asm/io.h
index 966e71b3c960..71cd226d6863 100644
--- a/arch/nds32/include/asm/io.h
+++ b/arch/nds32/include/asm/io.h
@@ -4,6 +4,8 @@
 #ifndef __ASM_NDS32_IO_H
 #define __ASM_NDS32_IO_H
 
+#include <linux/types.h>
+
 extern void iounmap(volatile void __iomem *addr);
 #define __raw_writeb __raw_writeb
 static inline void __raw_writeb(u8 val, volatile void __iomem *addr)
diff --git a/arch/nds32/include/asm/page.h b/arch/nds32/include/asm/page.h
index e27365c097b6..947f0491c9a7 100644
--- a/arch/nds32/include/asm/page.h
+++ b/arch/nds32/include/asm/page.h
@@ -27,6 +27,9 @@ extern void copy_user_highpage(struct page *to, struct page *from,
 			       unsigned long vaddr, struct vm_area_struct *vma);
 extern void clear_user_highpage(struct page *page, unsigned long vaddr);
 
+void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
+		    struct page *to);
+void clear_user_page(void *addr, unsigned long vaddr, struct page *page);
 #define __HAVE_ARCH_COPY_USER_HIGHPAGE
 #define clear_user_highpage	clear_user_highpage
 #else
diff --git a/arch/nds32/include/asm/pgtable.h b/arch/nds32/include/asm/pgtable.h
index 6783937edbeb..d3e19a55cf53 100644
--- a/arch/nds32/include/asm/pgtable.h
+++ b/arch/nds32/include/asm/pgtable.h
@@ -152,6 +152,7 @@ extern void __pgd_error(const char *file, int line, unsigned long val);
 #define PAGE_CACHE_L1	__pgprot(_HAVE_PAGE_L | _PAGE_V | _PAGE_M_KRW | _PAGE_D | _PAGE_E | _PAGE_G | _PAGE_CACHE)
 #define PAGE_MEMORY	__pgprot(_HAVE_PAGE_L | _PAGE_V | _PAGE_M_KRW | _PAGE_D | _PAGE_E | _PAGE_G | _PAGE_CACHE_SHRD)
 #define PAGE_KERNEL	__pgprot(_PAGE_V | _PAGE_M_KRW | _PAGE_D | _PAGE_E | _PAGE_G | _PAGE_CACHE_SHRD)
+#define PAGE_SHARED	__pgprot(_PAGE_V | _PAGE_M_URW_KRW | _PAGE_D | _PAGE_CACHE_SHRD)
 #define PAGE_DEVICE    __pgprot(_PAGE_V | _PAGE_M_KRW | _PAGE_D | _PAGE_G | _PAGE_C_DEV)
 #endif /* __ASSEMBLY__ */
 
diff --git a/arch/nds32/kernel/dma.c b/arch/nds32/kernel/dma.c
index d291800fc621..d0dbd4fe9645 100644
--- a/arch/nds32/kernel/dma.c
+++ b/arch/nds32/kernel/dma.c
@@ -3,17 +3,14 @@
 
 #include <linux/types.h>
 #include <linux/mm.h>
-#include <linux/export.h>
 #include <linux/string.h>
-#include <linux/scatterlist.h>
-#include <linux/dma-mapping.h>
+#include <linux/dma-noncoherent.h>
 #include <linux/io.h>
 #include <linux/cache.h>
 #include <linux/highmem.h>
 #include <linux/slab.h>
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
-#include <asm/dma-mapping.h>
 #include <asm/proc-fns.h>
 
 /*
@@ -22,11 +19,6 @@
 static pte_t *consistent_pte;
 static DEFINE_RAW_SPINLOCK(consistent_lock);
 
-enum master_type {
-	FOR_CPU = 0,
-	FOR_DEVICE = 1,
-};
-
 /*
  * VM region handling support.
  *
@@ -124,10 +116,8 @@ static struct arch_vm_region *vm_region_find(struct arch_vm_region *head,
 	return c;
 }
 
-/* FIXME: attrs is not used. */
-static void *nds32_dma_alloc_coherent(struct device *dev, size_t size,
-				      dma_addr_t * handle, gfp_t gfp,
-				      unsigned long attrs)
+void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
+		gfp_t gfp, unsigned long attrs)
 {
 	struct page *page;
 	struct arch_vm_region *c;
@@ -232,8 +222,8 @@ static void *nds32_dma_alloc_coherent(struct device *dev, size_t size,
 	return NULL;
 }
 
-static void nds32_dma_free(struct device *dev, size_t size, void *cpu_addr,
-			   dma_addr_t handle, unsigned long attrs)
+void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
+		dma_addr_t handle, unsigned long attrs)
 {
 	struct arch_vm_region *c;
 	unsigned long flags, addr;
@@ -333,145 +323,69 @@ static int __init consistent_init(void)
 }
 
 core_initcall(consistent_init);
-static void consistent_sync(void *vaddr, size_t size, int direction, int master_type);
-static dma_addr_t nds32_dma_map_page(struct device *dev, struct page *page,
-				     unsigned long offset, size_t size,
-				     enum dma_data_direction dir,
-				     unsigned long attrs)
-{
-	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
-		consistent_sync((void *)(page_address(page) + offset), size, dir, FOR_DEVICE);
-	return page_to_phys(page) + offset;
-}
-
-static void nds32_dma_unmap_page(struct device *dev, dma_addr_t handle,
-				 size_t size, enum dma_data_direction dir,
-				 unsigned long attrs)
-{
-	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
-		consistent_sync(phys_to_virt(handle), size, dir, FOR_CPU);
-}
-
-/*
- * Make an area consistent for devices.
- */
-static void consistent_sync(void *vaddr, size_t size, int direction, int master_type)
-{
-	unsigned long start = (unsigned long)vaddr;
-	unsigned long end = start + size;
-
-	if (master_type == FOR_CPU) {
-		switch (direction) {
-		case DMA_TO_DEVICE:
-			break;
-		case DMA_FROM_DEVICE:
-		case DMA_BIDIRECTIONAL:
-			cpu_dma_inval_range(start, end);
-			break;
-		default:
-			BUG();
-		}
-	} else {
-		/* FOR_DEVICE */
-		switch (direction) {
-		case DMA_FROM_DEVICE:
-			break;
-		case DMA_TO_DEVICE:
-		case DMA_BIDIRECTIONAL:
-			cpu_dma_wb_range(start, end);
-			break;
-		default:
-			BUG();
-		}
-	}
-}
 
-static int nds32_dma_map_sg(struct device *dev, struct scatterlist *sg,
-			    int nents, enum dma_data_direction dir,
-			    unsigned long attrs)
+static inline void cache_op(phys_addr_t paddr, size_t size,
+		void (*fn)(unsigned long start, unsigned long end))
 {
-	int i;
+	struct page *page = pfn_to_page(paddr >> PAGE_SHIFT);
+	unsigned offset = paddr & ~PAGE_MASK;
+	size_t left = size;
+	unsigned long start;
 
-	for (i = 0; i < nents; i++, sg++) {
-		void *virt;
-		unsigned long pfn;
-		struct page *page = sg_page(sg);
+	do {
+		size_t len = left;
 
-		sg->dma_address = sg_phys(sg);
-		pfn = page_to_pfn(page) + sg->offset / PAGE_SIZE;
-		page = pfn_to_page(pfn);
 		if (PageHighMem(page)) {
-			virt = kmap_atomic(page);
-			consistent_sync(virt, sg->length, dir, FOR_CPU);
-			kunmap_atomic(virt);
+			void *addr;
+
+			if (offset + len > PAGE_SIZE) {
+				if (offset >= PAGE_SIZE) {
+					page += offset >> PAGE_SHIFT;
+					offset &= ~PAGE_MASK;
+				}
+				len = PAGE_SIZE - offset;
+			}
+
+			addr = kmap_atomic(page);
+			start = (unsigned long)(addr + offset);
+			fn(start, start + len);
+			kunmap_atomic(addr);
 		} else {
-			if (sg->offset > PAGE_SIZE)
-				panic("sg->offset:%08x > PAGE_SIZE\n",
-				      sg->offset);
-			virt = page_address(page) + sg->offset;
-			consistent_sync(virt, sg->length, dir, FOR_CPU);
+			start = (unsigned long)phys_to_virt(paddr);
+			fn(start, start + size);
 		}
-	}
-	return nents;
+		offset = 0;
+		page++;
+		left -= len;
+	} while (left);
 }
 
-static void nds32_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
-			       int nhwentries, enum dma_data_direction dir,
-			       unsigned long attrs)
+void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
+		size_t size, enum dma_data_direction dir)
 {
-}
-
-static void
-nds32_dma_sync_single_for_cpu(struct device *dev, dma_addr_t handle,
-			      size_t size, enum dma_data_direction dir)
-{
-	consistent_sync((void *)phys_to_virt(handle), size, dir, FOR_CPU);
-}
-
-static void
-nds32_dma_sync_single_for_device(struct device *dev, dma_addr_t handle,
-				 size_t size, enum dma_data_direction dir)
-{
-	consistent_sync((void *)phys_to_virt(handle), size, dir, FOR_DEVICE);
-}
-
-static void
-nds32_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nents,
-			  enum dma_data_direction dir)
-{
-	int i;
-
-	for (i = 0; i < nents; i++, sg++) {
-		char *virt =
-		    page_address((struct page *)sg->page_link) + sg->offset;
-		consistent_sync(virt, sg->length, dir, FOR_CPU);
+	switch (dir) {
+	case DMA_FROM_DEVICE:
+		break;
+	case DMA_TO_DEVICE:
+	case DMA_BIDIRECTIONAL:
+		cache_op(paddr, size, cpu_dma_wb_range);
+		break;
+	default:
+		BUG();
 	}
 }
 
-static void
-nds32_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
-			     int nents, enum dma_data_direction dir)
+void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
+		size_t size, enum dma_data_direction dir)
 {
-	int i;
-
-	for (i = 0; i < nents; i++, sg++) {
-		char *virt =
-		    page_address((struct page *)sg->page_link) + sg->offset;
-		consistent_sync(virt, sg->length, dir, FOR_DEVICE);
+	switch (dir) {
+	case DMA_TO_DEVICE:
+		break;
+	case DMA_FROM_DEVICE:
+	case DMA_BIDIRECTIONAL:
+		cache_op(paddr, size, cpu_dma_inval_range);
+		break;
+	default:
+		BUG();
 	}
 }
-
-struct dma_map_ops nds32_dma_ops = {
-	.alloc = nds32_dma_alloc_coherent,
-	.free = nds32_dma_free,
-	.map_page = nds32_dma_map_page,
-	.unmap_page = nds32_dma_unmap_page,
-	.map_sg = nds32_dma_map_sg,
-	.unmap_sg = nds32_dma_unmap_sg,
-	.sync_single_for_device = nds32_dma_sync_single_for_device,
-	.sync_single_for_cpu = nds32_dma_sync_single_for_cpu,
-	.sync_sg_for_cpu = nds32_dma_sync_sg_for_cpu,
-	.sync_sg_for_device = nds32_dma_sync_sg_for_device,
-};
-
-EXPORT_SYMBOL(nds32_dma_ops);
diff --git a/arch/nds32/kernel/ex-entry.S b/arch/nds32/kernel/ex-entry.S
index a72e83d804f5..b8ae4e9a6b93 100644
--- a/arch/nds32/kernel/ex-entry.S
+++ b/arch/nds32/kernel/ex-entry.S
@@ -118,7 +118,7 @@ common_exception_handler:
 	/* interrupt */
 2:
 #ifdef CONFIG_TRACE_IRQFLAGS
-	jal     arch_trace_hardirqs_off
+	jal     trace_hardirqs_off
 #endif
 	move	$r0, $sp
 	sethi	$lp, hi20(ret_from_intr)
diff --git a/arch/nds32/kernel/head.S b/arch/nds32/kernel/head.S
index 71f57bd70f3b..c5fdae174ced 100644
--- a/arch/nds32/kernel/head.S
+++ b/arch/nds32/kernel/head.S
@@ -57,14 +57,32 @@ _nodtb:
 	isb
 	mtsr    $r4, $L1_PPTB       ! load page table pointer\n"
 
-/* set NTC0 cacheable/writeback, mutliple page size in use */
+#ifdef CONFIG_CPU_DCACHE_DISABLE
+	#define MMU_CTL_NTCC MMU_CTL_CACHEABLE_NON
+#else
+	#ifdef CONFIG_CPU_DCACHE_WRITETHROUGH
+		#define MMU_CTL_NTCC MMU_CTL_CACHEABLE_WT
+	#else
+		#define MMU_CTL_NTCC MMU_CTL_CACHEABLE_WB
+	#endif
+#endif
+
+/* set NTC cacheability, mutliple page size in use */
 	mfsr    $r3, $MMU_CTL
-	li      $r0, #~MMU_CTL_mskNTC0
-	and     $r3, $r3, $r0
+#if CONFIG_MEMORY_START >= 0xc0000000
+	ori     $r3, $r3, (MMU_CTL_NTCC << MMU_CTL_offNTC3)
+#elif CONFIG_MEMORY_START >= 0x80000000
+	ori     $r3, $r3, (MMU_CTL_NTCC << MMU_CTL_offNTC2)
+#elif CONFIG_MEMORY_START >= 0x40000000
+	ori     $r3, $r3, (MMU_CTL_NTCC << MMU_CTL_offNTC1)
+#else
+	ori     $r3, $r3, (MMU_CTL_NTCC << MMU_CTL_offNTC0)
+#endif
+
 #ifdef CONFIG_ANDES_PAGE_SIZE_4KB
-	ori     $r3, $r3, #(MMU_CTL_mskMPZIU|(MMU_CTL_CACHEABLE_WB << MMU_CTL_offNTC0))
+	ori     $r3, $r3, #(MMU_CTL_mskMPZIU)
 #else
-	ori     $r3, $r3, #(MMU_CTL_mskMPZIU|(MMU_CTL_CACHEABLE_WB << MMU_CTL_offNTC0)|MMU_CTL_D8KB)
+	ori     $r3, $r3, #(MMU_CTL_mskMPZIU|MMU_CTL_D8KB)
 #endif
 #ifdef CONFIG_HW_SUPPORT_UNALIGNMENT_ACCESS
 	li      $r0, #MMU_CTL_UNA
diff --git a/arch/nds32/kernel/setup.c b/arch/nds32/kernel/setup.c
index ba910e9e4ecb..2f5b2ccebe47 100644
--- a/arch/nds32/kernel/setup.c
+++ b/arch/nds32/kernel/setup.c
@@ -293,6 +293,9 @@ void __init setup_arch(char **cmdline_p)
 	/* paging_init() sets up the MMU and marks all pages as reserved */
 	paging_init();
 
+	/* invalidate all TLB entries because the new mapping is created */
+	__nds32__tlbop_flua();
+
 	/* use generic way to parse */
 	parse_early_param();
 
diff --git a/arch/nds32/kernel/stacktrace.c b/arch/nds32/kernel/stacktrace.c
index bc70113c0e84..8b231e910ea6 100644
--- a/arch/nds32/kernel/stacktrace.c
+++ b/arch/nds32/kernel/stacktrace.c
@@ -9,6 +9,7 @@ void save_stack_trace(struct stack_trace *trace)
 {
 	save_stack_trace_tsk(current, trace);
 }
+EXPORT_SYMBOL_GPL(save_stack_trace);
 
 void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
 {
@@ -45,3 +46,4 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
 		fpn = (unsigned long *)fpp;
 	}
 }
+EXPORT_SYMBOL_GPL(save_stack_trace_tsk);
diff --git a/arch/nds32/kernel/traps.c b/arch/nds32/kernel/traps.c
index 6e34eb9824a4..a6205fd4db52 100644
--- a/arch/nds32/kernel/traps.c
+++ b/arch/nds32/kernel/traps.c
@@ -222,19 +222,13 @@ void die_if_kernel(const char *str, struct pt_regs *regs, int err)
 
 int bad_syscall(int n, struct pt_regs *regs)
 {
-	siginfo_t info;
-
 	if (current->personality != PER_LINUX) {
 		send_sig(SIGSEGV, current, 1);
 		return regs->uregs[0];
 	}
 
-	info.si_signo = SIGILL;
-	info.si_errno = 0;
-	info.si_code = ILL_ILLTRP;
-	info.si_addr = (void __user *)instruction_pointer(regs) - 4;
-
-	force_sig_info(SIGILL, &info, current);
+	force_sig_fault(SIGILL, ILL_ILLTRP,
+			(void __user *)instruction_pointer(regs) - 4, current);
 	die_if_kernel("Oops - bad syscall", regs, n);
 	return regs->uregs[0];
 }
@@ -287,16 +281,11 @@ void __init early_trap_init(void)
 void send_sigtrap(struct task_struct *tsk, struct pt_regs *regs,
 		  int error_code, int si_code)
 {
-	struct siginfo info;
-
 	tsk->thread.trap_no = ENTRY_DEBUG_RELATED;
 	tsk->thread.error_code = error_code;
 
-	memset(&info, 0, sizeof(info));
-	info.si_signo = SIGTRAP;
-	info.si_code = si_code;
-	info.si_addr = (void __user *)instruction_pointer(regs);
-	force_sig_info(SIGTRAP, &info, tsk);
+	force_sig_fault(SIGTRAP, si_code,
+			(void __user *)instruction_pointer(regs), tsk);
 }
 
 void do_debug_trap(unsigned long entry, unsigned long addr,
@@ -318,29 +307,22 @@ void do_debug_trap(unsigned long entry, unsigned long addr,
 
 void unhandled_interruption(struct pt_regs *regs)
 {
-	siginfo_t si;
 	pr_emerg("unhandled_interruption\n");
 	show_regs(regs);
 	if (!user_mode(regs))
 		do_exit(SIGKILL);
-	si.si_signo = SIGKILL;
-	si.si_errno = 0;
-	force_sig_info(SIGKILL, &si, current);
+	force_sig(SIGKILL, current);
 }
 
 void unhandled_exceptions(unsigned long entry, unsigned long addr,
 			  unsigned long type, struct pt_regs *regs)
 {
-	siginfo_t si;
 	pr_emerg("Unhandled Exception: entry: %lx addr:%lx itype:%lx\n", entry,
 		 addr, type);
 	show_regs(regs);
 	if (!user_mode(regs))
 		do_exit(SIGKILL);
-	si.si_signo = SIGKILL;
-	si.si_errno = 0;
-	si.si_addr = (void *)addr;
-	force_sig_info(SIGKILL, &si, current);
+	force_sig(SIGKILL, current);
 }
 
 extern int do_page_fault(unsigned long entry, unsigned long addr,
@@ -363,14 +345,11 @@ void do_dispatch_tlb_misc(unsigned long entry, unsigned long addr,
 
 void do_revinsn(struct pt_regs *regs)
 {
-	siginfo_t si;
 	pr_emerg("Reserved Instruction\n");
 	show_regs(regs);
 	if (!user_mode(regs))
 		do_exit(SIGILL);
-	si.si_signo = SIGILL;
-	si.si_errno = 0;
-	force_sig_info(SIGILL, &si, current);
+	force_sig(SIGILL, current);
 }
 
 #ifdef CONFIG_ALIGNMENT_TRAP
diff --git a/arch/nds32/kernel/vdso.c b/arch/nds32/kernel/vdso.c
index f1198d7a5654..016f15891f6d 100644
--- a/arch/nds32/kernel/vdso.c
+++ b/arch/nds32/kernel/vdso.c
@@ -23,7 +23,7 @@
 #include <asm/vdso_timer_info.h>
 #include <asm/cache_info.h>
 extern struct cache_info L1_cache_info[2];
-extern char vdso_start, vdso_end;
+extern char vdso_start[], vdso_end[];
 static unsigned long vdso_pages __ro_after_init;
 static unsigned long timer_mapping_base;
 
@@ -66,16 +66,16 @@ static int __init vdso_init(void)
 	int i;
 	struct page **vdso_pagelist;
 
-	if (memcmp(&vdso_start, "\177ELF", 4)) {
+	if (memcmp(vdso_start, "\177ELF", 4)) {
 		pr_err("vDSO is not a valid ELF object!\n");
 		return -EINVAL;
 	}
 	/* Creat a timer io mapping to get clock cycles counter */
 	get_timer_node_info();
 
-	vdso_pages = (&vdso_end - &vdso_start) >> PAGE_SHIFT;
+	vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT;
 	pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n",
-		vdso_pages + 1, vdso_pages, &vdso_start, 1L, vdso_data);
+		vdso_pages + 1, vdso_pages, vdso_start, 1L, vdso_data);
 
 	/* Allocate the vDSO pagelist */
 	vdso_pagelist = kcalloc(vdso_pages, sizeof(struct page *), GFP_KERNEL);
@@ -83,7 +83,7 @@ static int __init vdso_init(void)
 		return -ENOMEM;
 
 	for (i = 0; i < vdso_pages; i++)
-		vdso_pagelist[i] = virt_to_page(&vdso_start + i * PAGE_SIZE);
+		vdso_pagelist[i] = virt_to_page(vdso_start + i * PAGE_SIZE);
 	vdso_spec[1].pages = &vdso_pagelist[0];
 
 	return 0;
diff --git a/arch/nds32/lib/copy_page.S b/arch/nds32/lib/copy_page.S
index 4a2ff85f17ee..f8701ed161a8 100644
--- a/arch/nds32/lib/copy_page.S
+++ b/arch/nds32/lib/copy_page.S
@@ -2,6 +2,7 @@
 // Copyright (C) 2005-2017 Andes Technology Corporation
 
 #include <linux/linkage.h>
+#include <asm/export.h>
 #include <asm/page.h>
 
 	.text
@@ -16,6 +17,7 @@ ENTRY(copy_page)
 	popm	$r2, $r10
 	ret
 ENDPROC(copy_page)
+EXPORT_SYMBOL(copy_page)
 
 ENTRY(clear_page)
 	pushm	$r1, $r9
@@ -35,3 +37,4 @@ ENTRY(clear_page)
 	popm	$r1, $r9
         ret
 ENDPROC(clear_page)
+EXPORT_SYMBOL(clear_page)
diff --git a/arch/nds32/mm/alignment.c b/arch/nds32/mm/alignment.c
index b96a01b10ca7..e1aed9dc692d 100644
--- a/arch/nds32/mm/alignment.c
+++ b/arch/nds32/mm/alignment.c
@@ -19,7 +19,7 @@
 #define RA(inst)	(((inst) >> 15) & 0x1FUL)
 #define RB(inst)	(((inst) >> 10) & 0x1FUL)
 #define SV(inst)	(((inst) >> 8) & 0x3UL)
-#define IMM(inst)	(((inst) >> 0) & 0x3FFFUL)
+#define IMM(inst)	(((inst) >> 0) & 0x7FFFUL)
 
 #define RA3(inst)	(((inst) >> 3) & 0x7UL)
 #define RT3(inst)	(((inst) >> 6) & 0x7UL)
@@ -28,6 +28,9 @@
 #define RA5(inst)	(((inst) >> 0) & 0x1FUL)
 #define RT4(inst)	(((inst) >> 5) & 0xFUL)
 
+#define GET_IMMSVAL(imm_value) \
+	(((imm_value >> 14) & 0x1) ? (imm_value - 0x8000) : imm_value)
+
 #define __get8_data(val,addr,err)	\
 	__asm__(					\
 	"1:	lbi.bi	%1, [%2], #1\n"			\
@@ -467,7 +470,7 @@ static inline int do_32(unsigned long inst, struct pt_regs *regs)
 	}
 
 	if (imm)
-		shift = IMM(inst) * len;
+		shift = GET_IMMSVAL(IMM(inst)) * len;
 	else
 		shift = *idx_to_addr(regs, RB(inst)) << SV(inst);
 
@@ -552,7 +555,7 @@ static struct ctl_table alignment_tbl[3] = {
 
 static struct ctl_table nds32_sysctl_table[2] = {
 	{
-	 .procname = "unaligned_acess",
+	 .procname = "unaligned_access",
 	 .mode = 0555,
 	 .child = alignment_tbl},
 	{}
diff --git a/arch/nds32/mm/cacheflush.c b/arch/nds32/mm/cacheflush.c
index 6eb786a399a2..ce8fd34497bf 100644
--- a/arch/nds32/mm/cacheflush.c
+++ b/arch/nds32/mm/cacheflush.c
@@ -147,6 +147,25 @@ void flush_cache_vunmap(unsigned long start, unsigned long end)
 	cpu_icache_inval_all();
 }
 
+void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
+		    struct page *to)
+{
+	cpu_dcache_wbinval_page((unsigned long)vaddr);
+	cpu_icache_inval_page((unsigned long)vaddr);
+	copy_page(vto, vfrom);
+	cpu_dcache_wbinval_page((unsigned long)vto);
+	cpu_icache_inval_page((unsigned long)vto);
+}
+
+void clear_user_page(void *addr, unsigned long vaddr, struct page *page)
+{
+	cpu_dcache_wbinval_page((unsigned long)vaddr);
+	cpu_icache_inval_page((unsigned long)vaddr);
+	clear_page(addr);
+	cpu_dcache_wbinval_page((unsigned long)addr);
+	cpu_icache_inval_page((unsigned long)addr);
+}
+
 void copy_user_highpage(struct page *to, struct page *from,
 			unsigned long vaddr, struct vm_area_struct *vma)
 {
@@ -156,11 +175,9 @@ void copy_user_highpage(struct page *to, struct page *from,
 	pto = page_to_phys(to);
 	pfrom = page_to_phys(from);
 
+	local_irq_save(flags);
 	if (aliasing(vaddr, (unsigned long)kfrom))
 		cpu_dcache_wb_page((unsigned long)kfrom);
-	if (aliasing(vaddr, (unsigned long)kto))
-		cpu_dcache_inval_page((unsigned long)kto);
-	local_irq_save(flags);
 	vto = kremap0(vaddr, pto);
 	vfrom = kremap1(vaddr, pfrom);
 	copy_page((void *)vto, (void *)vfrom);
@@ -198,21 +215,25 @@ void flush_dcache_page(struct page *page)
 	if (mapping && !mapping_mapped(mapping))
 		set_bit(PG_dcache_dirty, &page->flags);
 	else {
-		int i, pc;
-		unsigned long vto, kaddr, flags;
+		unsigned long kaddr, flags;
+
 		kaddr = (unsigned long)page_address(page);
-		cpu_dcache_wbinval_page(kaddr);
-		pc = CACHE_SET(DCACHE) * CACHE_LINE_SIZE(DCACHE) / PAGE_SIZE;
 		local_irq_save(flags);
-		for (i = 0; i < pc; i++) {
-			vto =
-			    kremap0(kaddr + i * PAGE_SIZE, page_to_phys(page));
-			cpu_dcache_wbinval_page(vto);
-			kunmap01(vto);
+		cpu_dcache_wbinval_page(kaddr);
+		if (mapping) {
+			unsigned long vaddr, kto;
+
+			vaddr = page->index << PAGE_SHIFT;
+			if (aliasing(vaddr, kaddr)) {
+				kto = kremap0(vaddr, page_to_phys(page));
+				cpu_dcache_wbinval_page(kto);
+				kunmap01(kto);
+			}
 		}
 		local_irq_restore(flags);
 	}
 }
+EXPORT_SYMBOL(flush_dcache_page);
 
 void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
 		       unsigned long vaddr, void *dst, void *src, int len)
@@ -251,7 +272,7 @@ void copy_from_user_page(struct vm_area_struct *vma, struct page *page,
 void flush_anon_page(struct vm_area_struct *vma,
 		     struct page *page, unsigned long vaddr)
 {
-	unsigned long flags;
+	unsigned long kaddr, flags, ktmp;
 	if (!PageAnon(page))
 		return;
 
@@ -261,7 +282,12 @@ void flush_anon_page(struct vm_area_struct *vma,
 	local_irq_save(flags);
 	if (vma->vm_flags & VM_EXEC)
 		cpu_icache_inval_page(vaddr & PAGE_MASK);
-	cpu_dcache_wbinval_page((unsigned long)page_address(page));
+	kaddr = (unsigned long)page_address(page);
+	if (aliasing(vaddr, kaddr)) {
+		ktmp = kremap0(vaddr, page_to_phys(page));
+		cpu_dcache_wbinval_page(ktmp);
+		kunmap01(ktmp);
+	}
 	local_irq_restore(flags);
 }
 
@@ -272,6 +298,25 @@ void flush_kernel_dcache_page(struct page *page)
 	cpu_dcache_wbinval_page((unsigned long)page_address(page));
 	local_irq_restore(flags);
 }
+EXPORT_SYMBOL(flush_kernel_dcache_page);
+
+void flush_kernel_vmap_range(void *addr, int size)
+{
+	unsigned long flags;
+	local_irq_save(flags);
+	cpu_dcache_wb_range((unsigned long)addr, (unsigned long)addr +  size);
+	local_irq_restore(flags);
+}
+EXPORT_SYMBOL(flush_kernel_vmap_range);
+
+void invalidate_kernel_vmap_range(void *addr, int size)
+{
+	unsigned long flags;
+	local_irq_save(flags);
+	cpu_dcache_inval_range((unsigned long)addr, (unsigned long)addr + size);
+	local_irq_restore(flags);
+}
+EXPORT_SYMBOL(invalidate_kernel_vmap_range);
 
 void flush_icache_range(unsigned long start, unsigned long end)
 {
@@ -283,6 +328,7 @@ void flush_icache_range(unsigned long start, unsigned long end)
 	cpu_cache_wbinval_range(start, end, 1);
 	local_irq_restore(flags);
 }
+EXPORT_SYMBOL(flush_icache_range);
 
 void flush_icache_page(struct vm_area_struct *vma, struct page *page)
 {
diff --git a/arch/nds32/mm/fault.c b/arch/nds32/mm/fault.c
index 3a246fb8098c..9bdb7c3ecbb6 100644
--- a/arch/nds32/mm/fault.c
+++ b/arch/nds32/mm/fault.c
@@ -72,7 +72,7 @@ void do_page_fault(unsigned long entry, unsigned long addr,
 	struct task_struct *tsk;
 	struct mm_struct *mm;
 	struct vm_area_struct *vma;
-	siginfo_t info;
+	int si_code;
 	int fault;
 	unsigned int mask = VM_READ | VM_WRITE | VM_EXEC;
 	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
@@ -80,7 +80,7 @@ void do_page_fault(unsigned long entry, unsigned long addr,
 	error_code = error_code & (ITYPE_mskINST | ITYPE_mskETYPE);
 	tsk = current;
 	mm = tsk->mm;
-	info.si_code = SEGV_MAPERR;
+	si_code = SEGV_MAPERR;
 	/*
 	 * We fault-in kernel-space virtual memory on-demand. The
 	 * 'reference' page table is init_mm.pgd.
@@ -161,7 +161,7 @@ void do_page_fault(unsigned long entry, unsigned long addr,
 	 */
 
 good_area:
-	info.si_code = SEGV_ACCERR;
+	si_code = SEGV_ACCERR;
 
 	/* first do some preliminary protection checks */
 	if (entry == ENTRY_PTE_NOT_PRESENT) {
@@ -266,11 +266,7 @@ void do_page_fault(unsigned long entry, unsigned long addr,
 		tsk->thread.address = addr;
 		tsk->thread.error_code = error_code;
 		tsk->thread.trap_no = entry;
-		info.si_signo = SIGSEGV;
-		info.si_errno = 0;
-		/* info.si_code has been set above */
-		info.si_addr = (void *)addr;
-		force_sig_info(SIGSEGV, &info, tsk);
+		force_sig_fault(SIGSEGV, si_code, (void __user *)addr, tsk);
 		return;
 	}
 
@@ -339,11 +335,7 @@ void do_page_fault(unsigned long entry, unsigned long addr,
 	tsk->thread.address = addr;
 	tsk->thread.error_code = error_code;
 	tsk->thread.trap_no = entry;
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRERR;
-	info.si_addr = (void *)addr;
-	force_sig_info(SIGBUS, &info, tsk);
+	force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)addr, tsk);
 
 	return;
 
diff --git a/arch/nds32/mm/init.c b/arch/nds32/mm/init.c
index 93ee0160720b..c713d2ad55dc 100644
--- a/arch/nds32/mm/init.c
+++ b/arch/nds32/mm/init.c
@@ -30,6 +30,7 @@ extern unsigned long phys_initrd_size;
  * zero-initialized data and COW.
  */
 struct page *empty_zero_page;
+EXPORT_SYMBOL(empty_zero_page);
 
 static void __init zone_sizes_init(void)
 {
diff --git a/arch/nios2/include/asm/Kbuild b/arch/nios2/include/asm/Kbuild
index d232da2cbb38..64ed3d656956 100644
--- a/arch/nios2/include/asm/Kbuild
+++ b/arch/nios2/include/asm/Kbuild
@@ -4,6 +4,7 @@ generic-y += bitops.h
 generic-y += bug.h
 generic-y += bugs.h
 generic-y += cmpxchg.h
+generic-y += compat.h
 generic-y += current.h
 generic-y += device.h
 generic-y += div64.h
diff --git a/arch/nios2/kernel/traps.c b/arch/nios2/kernel/traps.c
index 8184e7d6b385..3bc3cd22b750 100644
--- a/arch/nios2/kernel/traps.c
+++ b/arch/nios2/kernel/traps.c
@@ -26,13 +26,7 @@ static DEFINE_SPINLOCK(die_lock);
 
 static void _send_sig(int signo, int code, unsigned long addr)
 {
-	siginfo_t info;
-
-	info.si_signo = signo;
-	info.si_errno = 0;
-	info.si_code = code;
-	info.si_addr = (void __user *) addr;
-	force_sig_info(signo, &info, current);
+	force_sig_fault(signo, code, (void __user *) addr, current);
 }
 
 void die(const char *str, struct pt_regs *regs, long err)
diff --git a/arch/openrisc/Makefile b/arch/openrisc/Makefile
index cf8802962864..89076a66eee2 100644
--- a/arch/openrisc/Makefile
+++ b/arch/openrisc/Makefile
@@ -25,7 +25,6 @@ LDFLAGS_vmlinux :=
 LIBGCC 		:= $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name)
 
 KBUILD_CFLAGS	+= -pipe -ffixed-r10 -D__linux__
-CHECKFLAGS	+= -mbig-endian
 
 ifeq ($(CONFIG_OPENRISC_HAVE_INST_MUL),y)
 	KBUILD_CFLAGS += $(call cc-option,-mhard-mul)
diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild
index f05c722a21f8..65964d390b10 100644
--- a/arch/openrisc/include/asm/Kbuild
+++ b/arch/openrisc/include/asm/Kbuild
@@ -2,6 +2,7 @@ generic-y += barrier.h
 generic-y += bug.h
 generic-y += bugs.h
 generic-y += checksum.h
+generic-y += compat.h
 generic-y += current.h
 generic-y += device.h
 generic-y += div64.h
diff --git a/arch/openrisc/kernel/dma.c b/arch/openrisc/kernel/dma.c
index a945f00011b4..ec7fd45704d2 100644
--- a/arch/openrisc/kernel/dma.c
+++ b/arch/openrisc/kernel/dma.c
@@ -247,14 +247,3 @@ const struct dma_map_ops or1k_dma_map_ops = {
 	.sync_single_for_device = or1k_sync_single_for_device,
 };
 EXPORT_SYMBOL(or1k_dma_map_ops);
-
-/* Number of entries preallocated for DMA-API debugging */
-#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
-
-static int __init dma_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-
-	return 0;
-}
-fs_initcall(dma_init);
diff --git a/arch/openrisc/kernel/traps.c b/arch/openrisc/kernel/traps.c
index 113c175fe469..fac246e6f37a 100644
--- a/arch/openrisc/kernel/traps.c
+++ b/arch/openrisc/kernel/traps.c
@@ -250,27 +250,16 @@ void __init trap_init(void)
 
 asmlinkage void do_trap(struct pt_regs *regs, unsigned long address)
 {
-	siginfo_t info;
-	memset(&info, 0, sizeof(info));
-	info.si_signo = SIGTRAP;
-	info.si_code = TRAP_TRACE;
-	info.si_addr = (void *)address;
-	force_sig_info(SIGTRAP, &info, current);
+	force_sig_fault(SIGTRAP, TRAP_TRACE, (void __user *)address, current);
 
 	regs->pc += 4;
 }
 
 asmlinkage void do_unaligned_access(struct pt_regs *regs, unsigned long address)
 {
-	siginfo_t info;
-
 	if (user_mode(regs)) {
 		/* Send a SIGBUS */
-		info.si_signo = SIGBUS;
-		info.si_errno = 0;
-		info.si_code = BUS_ADRALN;
-		info.si_addr = (void __user *)address;
-		force_sig_info(SIGBUS, &info, current);
+		force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)address, current);
 	} else {
 		printk("KERNEL: Unaligned Access 0x%.8lx\n", address);
 		show_registers(regs);
@@ -281,15 +270,9 @@ asmlinkage void do_unaligned_access(struct pt_regs *regs, unsigned long address)
 
 asmlinkage void do_bus_fault(struct pt_regs *regs, unsigned long address)
 {
-	siginfo_t info;
-
 	if (user_mode(regs)) {
 		/* Send a SIGBUS */
-		info.si_signo = SIGBUS;
-		info.si_errno = 0;
-		info.si_code = BUS_ADRERR;
-		info.si_addr = (void *)address;
-		force_sig_info(SIGBUS, &info, current);
+		force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address, current);
 	} else {		/* Kernel mode */
 		printk("KERNEL: Bus error (SIGBUS) 0x%.8lx\n", address);
 		show_registers(regs);
@@ -464,7 +447,6 @@ static inline void simulate_swa(struct pt_regs *regs, unsigned long address,
 asmlinkage void do_illegal_instruction(struct pt_regs *regs,
 				       unsigned long address)
 {
-	siginfo_t info;
 	unsigned int op;
 	unsigned int insn = *((unsigned int *)address);
 
@@ -485,11 +467,7 @@ asmlinkage void do_illegal_instruction(struct pt_regs *regs,
 
 	if (user_mode(regs)) {
 		/* Send a SIGILL */
-		info.si_signo = SIGILL;
-		info.si_errno = 0;
-		info.si_code = ILL_ILLOPC;
-		info.si_addr = (void *)address;
-		force_sig_info(SIGBUS, &info, current);
+		force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)address, current);
 	} else {		/* Kernel mode */
 		printk("KERNEL: Illegal instruction (SIGILL) 0x%.8lx\n",
 		       address);
diff --git a/arch/openrisc/mm/fault.c b/arch/openrisc/mm/fault.c
index d0021dfae20a..9f011d16cc46 100644
--- a/arch/openrisc/mm/fault.c
+++ b/arch/openrisc/mm/fault.c
@@ -52,7 +52,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address,
 	struct task_struct *tsk;
 	struct mm_struct *mm;
 	struct vm_area_struct *vma;
-	siginfo_t info;
+	int si_code;
 	int fault;
 	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 
@@ -97,7 +97,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address,
 	}
 
 	mm = tsk->mm;
-	info.si_code = SEGV_MAPERR;
+	si_code = SEGV_MAPERR;
 
 	/*
 	 * If we're in an interrupt or have no user
@@ -139,7 +139,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address,
 	 */
 
 good_area:
-	info.si_code = SEGV_ACCERR;
+	si_code = SEGV_ACCERR;
 
 	/* first do some preliminary protection checks */
 
@@ -213,11 +213,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address,
 	/* User mode accesses just cause a SIGSEGV */
 
 	if (user_mode(regs)) {
-		info.si_signo = SIGSEGV;
-		info.si_errno = 0;
-		/* info.si_code has been set above */
-		info.si_addr = (void *)address;
-		force_sig_info(SIGSEGV, &info, tsk);
+		force_sig_fault(SIGSEGV, si_code, (void __user *)address, tsk);
 		return;
 	}
 
@@ -282,11 +278,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address,
 	 * Send a sigbus, regardless of whether we were in kernel
 	 * or user mode.
 	 */
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRERR;
-	info.si_addr = (void *)address;
-	force_sig_info(SIGBUS, &info, tsk);
+	force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address, tsk);
 
 	/* Kernel mode? Handle exceptions or die */
 	if (!user_mode(regs))
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
index fc5a574c3482..4d8f64d48597 100644
--- a/arch/parisc/Kconfig
+++ b/arch/parisc/Kconfig
@@ -51,6 +51,8 @@ config PARISC
 	select GENERIC_CLOCKEVENTS
 	select ARCH_NO_COHERENT_DMA_MMAP
 	select CPU_NO_EFFICIENT_FFS
+	select NEED_DMA_MAP_STATE
+	select NEED_SG_DMA_LENGTH
 
 	help
 	  The PA-RISC microprocessor is designed by Hewlett-Packard and used
@@ -111,12 +113,6 @@ config PM
 config STACKTRACE_SUPPORT
 	def_bool y
 
-config NEED_DMA_MAP_STATE
-	def_bool y
-
-config NEED_SG_DMA_LENGTH
-	def_bool y
-
 config ISA_DMA_API
 	bool
 
diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
index 34ac503e28ad..b88e2d0825d8 100644
--- a/arch/parisc/Makefile
+++ b/arch/parisc/Makefile
@@ -22,7 +22,7 @@ KBUILD_IMAGE := vmlinuz
 KBUILD_DEFCONFIG := default_defconfig
 
 NM		= sh $(srctree)/arch/parisc/nm
-CHECKFLAGS	+= -D__hppa__=1 -mbig-endian
+CHECKFLAGS	+= -D__hppa__=1
 LIBGCC		= $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name)
 export LIBGCC
 
@@ -65,10 +65,6 @@ endif
 # kernel.
 cflags-y	+= -mdisable-fpregs
 
-# Without this, "ld -r" results in .text sections that are too big
-# (> 0x40000) for branches to reach stubs.
-cflags-y	+= -ffunction-sections
-
 # Use long jumps instead of long branches (needed if your linker fails to
 # link a too big vmlinux executable). Not enabled for building modules.
 ifdef CONFIG_MLONGCALLS
diff --git a/arch/parisc/include/asm/compat.h b/arch/parisc/include/asm/compat.h
index 57b8b2a2fd4e..ab8a54771507 100644
--- a/arch/parisc/include/asm/compat.h
+++ b/arch/parisc/include/asm/compat.h
@@ -13,7 +13,6 @@
 
 typedef u32	compat_size_t;
 typedef s32	compat_ssize_t;
-typedef s32	compat_time_t;
 typedef s32	compat_clock_t;
 typedef s32	compat_pid_t;
 typedef u32	__compat_uid_t;
@@ -40,16 +39,6 @@ typedef u32	compat_ulong_t;
 typedef u64	compat_u64;
 typedef u32	compat_uptr_t;
 
-struct compat_timespec {
-	compat_time_t		tv_sec;
-	s32			tv_nsec;
-};
-
-struct compat_timeval {
-	compat_time_t		tv_sec;
-	s32			tv_usec;
-};
-
 struct compat_stat {
 	compat_dev_t		st_dev;	/* dev_t is 32 bits on parisc */
 	compat_ino_t		st_ino;	/* 32 bits */
@@ -149,10 +138,10 @@ struct compat_ipc64_perm {
 
 struct compat_semid64_ds {
 	struct compat_ipc64_perm sem_perm;
-	unsigned int __unused1;
-	compat_time_t sem_otime;
-	unsigned int __unused2;
-	compat_time_t sem_ctime;
+	unsigned int sem_otime_high;
+	unsigned int sem_otime;
+	unsigned int sem_ctime_high;
+	unsigned int sem_ctime;
 	compat_ulong_t sem_nsems;
 	compat_ulong_t __unused3;
 	compat_ulong_t __unused4;
@@ -160,12 +149,12 @@ struct compat_semid64_ds {
 
 struct compat_msqid64_ds {
 	struct compat_ipc64_perm msg_perm;
-	unsigned int __unused1;
-	compat_time_t msg_stime;
-	unsigned int __unused2;
-	compat_time_t msg_rtime;
-	unsigned int __unused3;
-	compat_time_t msg_ctime;
+	unsigned int msg_stime_high;
+	unsigned int msg_stime;
+	unsigned int msg_rtime_high;
+	unsigned int msg_rtime;
+	unsigned int msg_ctime_high;
+	unsigned int msg_ctime;
 	compat_ulong_t msg_cbytes;
 	compat_ulong_t msg_qnum;
 	compat_ulong_t msg_qbytes;
@@ -177,12 +166,12 @@ struct compat_msqid64_ds {
 
 struct compat_shmid64_ds {
 	struct compat_ipc64_perm shm_perm;
-	unsigned int __unused1;
-	compat_time_t shm_atime;
-	unsigned int __unused2;
-	compat_time_t shm_dtime;
-	unsigned int __unused3;
-	compat_time_t shm_ctime;
+	unsigned int shm_atime_high;
+	unsigned int shm_atime;
+	unsigned int shm_dtime_high;
+	unsigned int shm_dtime;
+	unsigned int shm_ctime_high;
+	unsigned int shm_ctime;
 	unsigned int __unused4;
 	compat_size_t shm_segsz;
 	compat_pid_t shm_cpid;
diff --git a/arch/parisc/include/asm/hardirq.h b/arch/parisc/include/asm/hardirq.h
index 077815169258..1a1235a9d533 100644
--- a/arch/parisc/include/asm/hardirq.h
+++ b/arch/parisc/include/asm/hardirq.h
@@ -34,14 +34,6 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
 #define __IRQ_STAT(cpu, member) (irq_stat[cpu].member)
 #define inc_irq_stat(member)	this_cpu_inc(irq_stat.member)
 #define __inc_irq_stat(member)	__this_cpu_inc(irq_stat.member)
-#define local_softirq_pending()	this_cpu_read(irq_stat.__softirq_pending)
-
-#define __ARCH_SET_SOFTIRQ_PENDING
-
-#define set_softirq_pending(x)	\
-		this_cpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x)	this_cpu_or(irq_stat.__softirq_pending, (x))
-
 #define ack_bad_irq(irq) WARN(1, "unexpected IRQ trap at vector %02x\n", irq)
 
 #endif /* _PARISC_HARDIRQ_H */
diff --git a/arch/parisc/include/asm/pci.h b/arch/parisc/include/asm/pci.h
index 96b7deec512d..3328fd17c19d 100644
--- a/arch/parisc/include/asm/pci.h
+++ b/arch/parisc/include/asm/pci.h
@@ -87,29 +87,6 @@ struct pci_hba_data {
 #define PCI_F_EXTEND		0UL
 #endif /* !CONFIG_64BIT */
 
-/*
- * If the PCI device's view of memory is the same as the CPU's view of memory,
- * PCI_DMA_BUS_IS_PHYS is true.  The networking and block device layers use
- * this boolean for bounce buffer decisions.
- */
-#ifdef CONFIG_PA20
-/* All PA-2.0 machines have an IOMMU. */
-#define PCI_DMA_BUS_IS_PHYS	0
-#define parisc_has_iommu()	do { } while (0)
-#else
-
-#if defined(CONFIG_IOMMU_CCIO) || defined(CONFIG_IOMMU_SBA)
-extern int parisc_bus_is_phys; 	/* in arch/parisc/kernel/setup.c */
-#define PCI_DMA_BUS_IS_PHYS	parisc_bus_is_phys
-#define parisc_has_iommu()	do { parisc_bus_is_phys = 0; } while (0)
-#else
-#define PCI_DMA_BUS_IS_PHYS	1
-#define parisc_has_iommu()	do { } while (0)
-#endif
-
-#endif	/* !CONFIG_PA20 */
-
-
 /*
 ** Most PCI devices (eg Tulip, NCR720) also export the same registers
 ** to both MMIO and I/O port space.  Due to poor performance of I/O Port
diff --git a/arch/parisc/include/asm/signal.h b/arch/parisc/include/asm/signal.h
index eeb5c8858663..715c96ba2ec8 100644
--- a/arch/parisc/include/asm/signal.h
+++ b/arch/parisc/include/asm/signal.h
@@ -21,14 +21,6 @@ typedef struct {
 	unsigned long sig[_NSIG_WORDS];
 } sigset_t;
 
-#ifndef __KERNEL__
-struct sigaction {
-	__sighandler_t sa_handler;
-	unsigned long sa_flags;
-	sigset_t sa_mask;		/* mask last for extensibility */
-};
-#endif
-
 #include <asm/sigcontext.h>
 
 #endif /* !__ASSEMBLY */
diff --git a/arch/parisc/include/uapi/asm/msgbuf.h b/arch/parisc/include/uapi/asm/msgbuf.h
index b48b810e626b..6a2e9ab2ef8d 100644
--- a/arch/parisc/include/uapi/asm/msgbuf.h
+++ b/arch/parisc/include/uapi/asm/msgbuf.h
@@ -10,31 +10,30 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
  * - 2 miscellaneous 32-bit values
  */
 
 struct msqid64_ds {
 	struct ipc64_perm msg_perm;
-#if __BITS_PER_LONG != 64
-	unsigned int   __pad1;
-#endif
+#if __BITS_PER_LONG == 64
 	__kernel_time_t msg_stime;	/* last msgsnd time */
-#if __BITS_PER_LONG != 64
-	unsigned int   __pad2;
-#endif
 	__kernel_time_t msg_rtime;	/* last msgrcv time */
-#if __BITS_PER_LONG != 64
-	unsigned int   __pad3;
-#endif
 	__kernel_time_t msg_ctime;	/* last change time */
-	unsigned long msg_cbytes;	/* current number of bytes on queue */
-	unsigned long msg_qnum;		/* number of messages in queue */
-	unsigned long msg_qbytes;	/* max number of bytes on queue */
-	__kernel_pid_t msg_lspid;	/* pid of last msgsnd */
-	__kernel_pid_t msg_lrpid;	/* last receive pid */
-	unsigned long __unused1;
-	unsigned long __unused2;
+#else
+	unsigned long	msg_stime_high;
+	unsigned long	msg_stime;	/* last msgsnd time */
+	unsigned long	msg_rtime_high;
+	unsigned long	msg_rtime;	/* last msgrcv time */
+	unsigned long	msg_ctime_high;
+	unsigned long	msg_ctime;	/* last change time */
+#endif
+	unsigned long	msg_cbytes;	/* current number of bytes on queue */
+	unsigned long	msg_qnum;	/* number of messages in queue */
+	unsigned long	msg_qbytes;	/* max number of bytes on queue */
+	__kernel_pid_t	msg_lspid;	/* pid of last msgsnd */
+	__kernel_pid_t	msg_lrpid;	/* last receive pid */
+	unsigned long	__unused1;
+	unsigned long	__unused2;
 };
 
 #endif /* _PARISC_MSGBUF_H */
diff --git a/arch/parisc/include/uapi/asm/sembuf.h b/arch/parisc/include/uapi/asm/sembuf.h
index 746c5d86a9b1..3c31163b1241 100644
--- a/arch/parisc/include/uapi/asm/sembuf.h
+++ b/arch/parisc/include/uapi/asm/sembuf.h
@@ -10,21 +10,21 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
  * - 2 miscellaneous 32-bit values
  */
 
 struct semid64_ds {
 	struct ipc64_perm sem_perm;		/* permissions .. see ipc.h */
-#if __BITS_PER_LONG != 64
-	unsigned int	__pad1;
-#endif
+#if __BITS_PER_LONG == 64
 	__kernel_time_t	sem_otime;		/* last semop time */
-#if __BITS_PER_LONG != 64
-	unsigned int	__pad2;
-#endif
 	__kernel_time_t	sem_ctime;		/* last change time */
-	unsigned long 	sem_nsems;		/* no. of semaphores in array */
+#else
+	unsigned long	sem_otime_high;
+	unsigned long	sem_otime;		/* last semop time */
+	unsigned long	sem_ctime_high;
+	unsigned long	sem_ctime;		/* last change time */
+#endif
+	unsigned long	sem_nsems;		/* no. of semaphores in array */
 	unsigned long	__unused1;
 	unsigned long	__unused2;
 };
diff --git a/arch/parisc/include/uapi/asm/shmbuf.h b/arch/parisc/include/uapi/asm/shmbuf.h
index cd4dbce55d0b..c89b3dd8db21 100644
--- a/arch/parisc/include/uapi/asm/shmbuf.h
+++ b/arch/parisc/include/uapi/asm/shmbuf.h
@@ -10,25 +10,22 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
  * - 2 miscellaneous 32-bit values
  */
 
 struct shmid64_ds {
 	struct ipc64_perm	shm_perm;	/* operation perms */
-#if __BITS_PER_LONG != 64
-	unsigned int		__pad1;
-#endif
+#if __BITS_PER_LONG == 64
 	__kernel_time_t		shm_atime;	/* last attach time */
-#if __BITS_PER_LONG != 64
-	unsigned int		__pad2;
-#endif
 	__kernel_time_t		shm_dtime;	/* last detach time */
-#if __BITS_PER_LONG != 64
-	unsigned int		__pad3;
-#endif
 	__kernel_time_t		shm_ctime;	/* last change time */
-#if __BITS_PER_LONG != 64
+#else
+	unsigned long		shm_atime_high;
+	unsigned long		shm_atime;	/* last attach time */
+	unsigned long		shm_dtime_high;
+	unsigned long		shm_dtime;	/* last detach time */
+	unsigned long		shm_ctime_high;
+	unsigned long		shm_ctime;	/* last change time */
 	unsigned int		__pad4;
 #endif
 	__kernel_size_t		shm_segsz;	/* size of segment (bytes) */
diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c
index 91bc0cac03a1..6df07ce4f3c2 100644
--- a/arch/parisc/kernel/pci-dma.c
+++ b/arch/parisc/kernel/pci-dma.c
@@ -367,19 +367,6 @@ static int proc_pcxl_dma_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int proc_pcxl_dma_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_pcxl_dma_show, NULL);
-}
-
-static const struct file_operations proc_pcxl_dma_ops = {
-	.owner		= THIS_MODULE,
-	.open		= proc_pcxl_dma_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int __init
 pcxl_dma_init(void)
 {
@@ -397,8 +384,8 @@ pcxl_dma_init(void)
 			"pcxl_dma_init: Unable to create gsc /proc dir entry\n");
 	else {
 		struct proc_dir_entry* ent;
-		ent = proc_create("pcxl_dma", 0, proc_gsc_root,
-				  &proc_pcxl_dma_ops);
+		ent = proc_create_single("pcxl_dma", 0, proc_gsc_root,
+				proc_pcxl_dma_show);
 		if (!ent)
 			printk(KERN_WARNING
 				"pci-dma.c: Unable to create pcxl_dma /proc entry.\n");
diff --git a/arch/parisc/kernel/pdc_chassis.c b/arch/parisc/kernel/pdc_chassis.c
index 3e04242de5a7..28e07482b0f1 100644
--- a/arch/parisc/kernel/pdc_chassis.c
+++ b/arch/parisc/kernel/pdc_chassis.c
@@ -266,18 +266,6 @@ static int pdc_chassis_warn_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int pdc_chassis_warn_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, pdc_chassis_warn_show, NULL);
-}
-
-static const struct file_operations pdc_chassis_warn_fops = {
-	.open		= pdc_chassis_warn_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int __init pdc_chassis_create_procfs(void)
 {
 	unsigned long test;
@@ -292,7 +280,7 @@ static int __init pdc_chassis_create_procfs(void)
 
 	printk(KERN_INFO "Enabling PDC chassis warnings support v%s\n",
 			PDC_CHASSIS_VER);
-	proc_create("chassis", 0400, NULL, &pdc_chassis_warn_fops);
+	proc_create_single("chassis", 0400, NULL, pdc_chassis_warn_show);
 	return 0;
 }
 
diff --git a/arch/parisc/kernel/ptrace.c b/arch/parisc/kernel/ptrace.c
index 1a2be6e639b5..7aa1d4d0d444 100644
--- a/arch/parisc/kernel/ptrace.c
+++ b/arch/parisc/kernel/ptrace.c
@@ -76,8 +76,6 @@ void user_enable_single_step(struct task_struct *task)
 	set_tsk_thread_flag(task, TIF_SINGLESTEP);
 
 	if (pa_psw(task)->n) {
-		struct siginfo si;
-
 		/* Nullified, just crank over the queue. */
 		task_regs(task)->iaoq[0] = task_regs(task)->iaoq[1];
 		task_regs(task)->iasq[0] = task_regs(task)->iasq[1];
@@ -90,11 +88,9 @@ void user_enable_single_step(struct task_struct *task)
 		ptrace_disable(task);
 		/* Don't wake up the task, but let the
 		   parent know something happened. */
-		si.si_code = TRAP_TRACE;
-		si.si_addr = (void __user *) (task_regs(task)->iaoq[0] & ~3);
-		si.si_signo = SIGTRAP;
-		si.si_errno = 0;
-		force_sig_info(SIGTRAP, &si, task);
+		force_sig_fault(SIGTRAP, TRAP_TRACE,
+				(void __user *) (task_regs(task)->iaoq[0] & ~3),
+				task);
 		/* notify_parent(task, SIGCHLD); */
 		return;
 	}
diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c
index 0e9675f857a5..8d3a7b80ac42 100644
--- a/arch/parisc/kernel/setup.c
+++ b/arch/parisc/kernel/setup.c
@@ -58,11 +58,6 @@ struct proc_dir_entry * proc_runway_root __read_mostly = NULL;
 struct proc_dir_entry * proc_gsc_root __read_mostly = NULL;
 struct proc_dir_entry * proc_mckinley_root __read_mostly = NULL;
 
-#if !defined(CONFIG_PA20) && (defined(CONFIG_IOMMU_CCIO) || defined(CONFIG_IOMMU_SBA))
-int parisc_bus_is_phys __read_mostly = 1;	/* Assume no IOMMU is present */
-EXPORT_SYMBOL(parisc_bus_is_phys);
-#endif
-
 void __init setup_cmdline(char **cmdline_p)
 {
 	extern unsigned int boot_args[];
diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c
index 71d31274d782..4309ad31a874 100644
--- a/arch/parisc/kernel/traps.c
+++ b/arch/parisc/kernel/traps.c
@@ -297,13 +297,8 @@ void die_if_kernel(char *str, struct pt_regs *regs, long err)
 #define GDB_BREAK_INSN 0x10004
 static void handle_gdb_break(struct pt_regs *regs, int wot)
 {
-	struct siginfo si;
-
-	si.si_signo = SIGTRAP;
-	si.si_errno = 0;
-	si.si_code = wot;
-	si.si_addr = (void __user *) (regs->iaoq[0] & ~3);
-	force_sig_info(SIGTRAP, &si, current);
+	force_sig_fault(SIGTRAP, wot,
+			(void __user *) (regs->iaoq[0] & ~3), current);
 }
 
 static void handle_break(struct pt_regs *regs)
@@ -487,7 +482,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
 {
 	unsigned long fault_address = 0;
 	unsigned long fault_space = 0;
-	struct siginfo si;
+	int si_code;
 
 	if (code == 1)
 	    pdc_console_restart();  /* switch back to pdc if HPMC */
@@ -571,7 +566,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
 	case  8:
 		/* Illegal instruction trap */
 		die_if_kernel("Illegal instruction", regs, code);
-		si.si_code = ILL_ILLOPC;
+		si_code = ILL_ILLOPC;
 		goto give_sigill;
 
 	case  9:
@@ -582,7 +577,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
 	case 10:
 		/* Privileged operation trap */
 		die_if_kernel("Privileged operation", regs, code);
-		si.si_code = ILL_PRVOPC;
+		si_code = ILL_PRVOPC;
 		goto give_sigill;
 
 	case 11:
@@ -605,20 +600,16 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
 		}
 
 		die_if_kernel("Privileged register usage", regs, code);
-		si.si_code = ILL_PRVREG;
+		si_code = ILL_PRVREG;
 	give_sigill:
-		si.si_signo = SIGILL;
-		si.si_errno = 0;
-		si.si_addr = (void __user *) regs->iaoq[0];
-		force_sig_info(SIGILL, &si, current);
+		force_sig_fault(SIGILL, si_code,
+				(void __user *) regs->iaoq[0], current);
 		return;
 
 	case 12:
 		/* Overflow Trap, let the userland signal handler do the cleanup */
-		si.si_signo = SIGFPE;
-		si.si_code = FPE_INTOVF;
-		si.si_addr = (void __user *) regs->iaoq[0];
-		force_sig_info(SIGFPE, &si, current);
+		force_sig_fault(SIGFPE, FPE_INTOVF,
+				(void __user *) regs->iaoq[0], current);
 		return;
 		
 	case 13:
@@ -626,13 +617,11 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
 		   The condition succeeds in an instruction which traps
 		   on condition  */
 		if(user_mode(regs)){
-			si.si_signo = SIGFPE;
 			/* Let userspace app figure it out from the insn pointed
 			 * to by si_addr.
 			 */
-			si.si_code = FPE_CONDTRAP;
-			si.si_addr = (void __user *) regs->iaoq[0];
-			force_sig_info(SIGFPE, &si, current);
+			force_sig_fault(SIGFPE, FPE_CONDTRAP,
+					(void __user *) regs->iaoq[0], current);
 			return;
 		} 
 		/* The kernel doesn't want to handle condition codes */
@@ -741,14 +730,10 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
 			return;
 
 		die_if_kernel("Protection id trap", regs, code);
-		si.si_code = SEGV_MAPERR;
-		si.si_signo = SIGSEGV;
-		si.si_errno = 0;
-		if (code == 7)
-		    si.si_addr = (void __user *) regs->iaoq[0];
-		else
-		    si.si_addr = (void __user *) regs->ior;
-		force_sig_info(SIGSEGV, &si, current);
+		force_sig_fault(SIGSEGV, SEGV_MAPERR,
+				(code == 7)?
+				((void __user *) regs->iaoq[0]) :
+				((void __user *) regs->ior), current);
 		return;
 
 	case 28: 
@@ -762,11 +747,8 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
 				"handle_interruption() pid=%d command='%s'\n",
 				task_pid_nr(current), current->comm);
 			/* SIGBUS, for lack of a better one. */
-			si.si_signo = SIGBUS;
-			si.si_code = BUS_OBJERR;
-			si.si_errno = 0;
-			si.si_addr = (void __user *) regs->ior;
-			force_sig_info(SIGBUS, &si, current);
+			force_sig_fault(SIGBUS, BUS_OBJERR,
+					(void __user *)regs->ior, current);
 			return;
 		}
 		pdc_chassis_send_status(PDC_CHASSIS_DIRECT_PANIC);
@@ -781,11 +763,8 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
 				"User fault %d on space 0x%08lx, pid=%d command='%s'\n",
 				code, fault_space,
 				task_pid_nr(current), current->comm);
-		si.si_signo = SIGSEGV;
-		si.si_errno = 0;
-		si.si_code = SEGV_MAPERR;
-		si.si_addr = (void __user *) regs->ior;
-		force_sig_info(SIGSEGV, &si, current);
+		force_sig_fault(SIGSEGV, SEGV_MAPERR,
+				(void __user *)regs->ior, current);
 		return;
 	    }
 	}
diff --git a/arch/parisc/kernel/unaligned.c b/arch/parisc/kernel/unaligned.c
index e36f7b75ab07..932bfc0b7cd8 100644
--- a/arch/parisc/kernel/unaligned.c
+++ b/arch/parisc/kernel/unaligned.c
@@ -452,7 +452,6 @@ void handle_unaligned(struct pt_regs *regs)
 	unsigned long newbase = R1(regs->iir)?regs->gr[R1(regs->iir)]:0;
 	int modify = 0;
 	int ret = ERR_NOTHANDLED;
-	struct siginfo si;
 	register int flop=0;	/* true if this is a flop */
 
 	__inc_irq_stat(irq_unaligned_count);
@@ -690,21 +689,15 @@ void handle_unaligned(struct pt_regs *regs)
 
 		if (ret == ERR_PAGEFAULT)
 		{
-			si.si_signo = SIGSEGV;
-			si.si_errno = 0;
-			si.si_code = SEGV_MAPERR;
-			si.si_addr = (void __user *)regs->ior;
-			force_sig_info(SIGSEGV, &si, current);
+			force_sig_fault(SIGSEGV, SEGV_MAPERR,
+					(void __user *)regs->ior, current);
 		}
 		else
 		{
 force_sigbus:
 			/* couldn't handle it ... */
-			si.si_signo = SIGBUS;
-			si.si_errno = 0;
-			si.si_code = BUS_ADRALN;
-			si.si_addr = (void __user *)regs->ior;
-			force_sig_info(SIGBUS, &si, current);
+			force_sig_fault(SIGBUS, BUS_ADRALN,
+					(void __user *)regs->ior, current);
 		}
 		
 		return;
diff --git a/arch/parisc/math-emu/driver.c b/arch/parisc/math-emu/driver.c
index 2fb59d2e2b29..0590e05571d1 100644
--- a/arch/parisc/math-emu/driver.c
+++ b/arch/parisc/math-emu/driver.c
@@ -81,7 +81,6 @@ int
 handle_fpe(struct pt_regs *regs)
 {
 	extern void printbinary(unsigned long x, int nbits);
-	struct siginfo si;
 	unsigned int orig_sw, sw;
 	int signalcode;
 	/* need an intermediate copy of float regs because FPU emulation
@@ -117,11 +116,8 @@ handle_fpe(struct pt_regs *regs)
 
 	memcpy(regs->fr, frcopy, sizeof regs->fr);
 	if (signalcode != 0) {
-	    si.si_signo = signalcode >> 24;
-	    si.si_errno = 0;
-	    si.si_code = signalcode & 0xffffff;
-	    si.si_addr = (void __user *) regs->iaoq[0];
-	    force_sig_info(si.si_signo, &si, current);
+	    force_sig_fault(signalcode >> 24, signalcode & 0xffffff,
+			    (void __user *) regs->iaoq[0], current);
 	    return -1;
 	}
 
diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c
index e247edbca68e..a80117980fc2 100644
--- a/arch/parisc/mm/fault.c
+++ b/arch/parisc/mm/fault.c
@@ -353,23 +353,22 @@ void do_page_fault(struct pt_regs *regs, unsigned long code,
 	up_read(&mm->mmap_sem);
 
 	if (user_mode(regs)) {
-		struct siginfo si;
-		unsigned int lsb = 0;
+		int signo, si_code;
 
 		switch (code) {
 		case 15:	/* Data TLB miss fault/Data page fault */
 			/* send SIGSEGV when outside of vma */
 			if (!vma ||
 			    address < vma->vm_start || address >= vma->vm_end) {
-				si.si_signo = SIGSEGV;
-				si.si_code = SEGV_MAPERR;
+				signo = SIGSEGV;
+				si_code = SEGV_MAPERR;
 				break;
 			}
 
 			/* send SIGSEGV for wrong permissions */
 			if ((vma->vm_flags & acc_type) != acc_type) {
-				si.si_signo = SIGSEGV;
-				si.si_code = SEGV_ACCERR;
+				signo = SIGSEGV;
+				si_code = SEGV_ACCERR;
 				break;
 			}
 
@@ -377,43 +376,40 @@ void do_page_fault(struct pt_regs *regs, unsigned long code,
 			/* fall through */
 		case 17:	/* NA data TLB miss / page fault */
 		case 18:	/* Unaligned access - PCXS only */
-			si.si_signo = SIGBUS;
-			si.si_code = (code == 18) ? BUS_ADRALN : BUS_ADRERR;
+			signo = SIGBUS;
+			si_code = (code == 18) ? BUS_ADRALN : BUS_ADRERR;
 			break;
 		case 16:	/* Non-access instruction TLB miss fault */
 		case 26:	/* PCXL: Data memory access rights trap */
 		default:
-			si.si_signo = SIGSEGV;
-			si.si_code = (code == 26) ? SEGV_ACCERR : SEGV_MAPERR;
+			signo = SIGSEGV;
+			si_code = (code == 26) ? SEGV_ACCERR : SEGV_MAPERR;
 			break;
 		}
-
 #ifdef CONFIG_MEMORY_FAILURE
 		if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
+			unsigned int lsb = 0;
 			printk(KERN_ERR
 	"MCE: Killing %s:%d due to hardware memory corruption fault at %08lx\n",
 			tsk->comm, tsk->pid, address);
-			si.si_signo = SIGBUS;
-			si.si_code = BUS_MCEERR_AR;
+			/*
+			 * Either small page or large page may be poisoned.
+			 * In other words, VM_FAULT_HWPOISON_LARGE and
+			 * VM_FAULT_HWPOISON are mutually exclusive.
+			 */
+			if (fault & VM_FAULT_HWPOISON_LARGE)
+				lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
+			else if (fault & VM_FAULT_HWPOISON)
+				lsb = PAGE_SHIFT;
+
+			force_sig_mceerr(BUS_MCEERR_AR, (void __user *) address,
+					 lsb, current);
+			return;
 		}
 #endif
+		show_signal_msg(regs, code, address, tsk, vma);
 
-		/*
-		 * Either small page or large page may be poisoned.
-		 * In other words, VM_FAULT_HWPOISON_LARGE and
-		 * VM_FAULT_HWPOISON are mutually exclusive.
-		 */
-		if (fault & VM_FAULT_HWPOISON_LARGE)
-			lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
-		else if (fault & VM_FAULT_HWPOISON)
-			lsb = PAGE_SHIFT;
-		else
-			show_signal_msg(regs, code, address, tsk, vma);
-		si.si_addr_lsb = lsb;
-
-		si.si_errno = 0;
-		si.si_addr = (void __user *) address;
-		force_sig_info(si.si_signo, &si, current);
+		force_sig_fault(signo, si_code, (void __user *) address, current);
 		return;
 	}
 
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index c32a181a7cbb..ea433bd20d38 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -13,12 +13,6 @@ config 64BIT
 	bool
 	default y if PPC64
 
-config ARCH_PHYS_ADDR_T_64BIT
-       def_bool PPC64 || PHYS_64BIT
-
-config ARCH_DMA_ADDR_T_64BIT
-	def_bool ARCH_PHYS_ADDR_T_64BIT
-
 config MMU
 	bool
 	default y
@@ -141,6 +135,7 @@ config PPC
 	select ARCH_HAS_GCOV_PROFILE_ALL
 	select ARCH_HAS_PHYS_TO_DMA
 	select ARCH_HAS_PMEM_API                if PPC64
+	select ARCH_HAS_PTE_SPECIAL
 	select ARCH_HAS_MEMBARRIER_CALLBACKS
 	select ARCH_HAS_SCALED_CPUTIME		if VIRT_CPU_ACCOUNTING_NATIVE
 	select ARCH_HAS_SG_CHAIN
@@ -162,6 +157,7 @@ config PPC
 	select BUILDTIME_EXTABLE_SORT
 	select CLONE_BACKWARDS
 	select DCACHE_WORD_ACCESS		if PPC64 && CPU_LITTLE_ENDIAN
+	select DYNAMIC_FTRACE			if FUNCTION_TRACER
 	select EDAC_ATOMIC_SCRUB
 	select EDAC_SUPPORT
 	select GENERIC_ATOMIC64			if PPC32
@@ -187,7 +183,6 @@ config PPC
 	select HAVE_CONTEXT_TRACKING		if PPC64
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DEBUG_STACKOVERFLOW
-	select HAVE_DMA_API_DEBUG
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_DYNAMIC_FTRACE_WITH_REGS	if MPROFILE_KERNEL
 	select HAVE_EBPF_JIT			if PPC64
@@ -195,7 +190,7 @@ config PPC
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_FUNCTION_GRAPH_TRACER
 	select HAVE_FUNCTION_TRACER
-	select HAVE_GCC_PLUGINS
+	select HAVE_GCC_PLUGINS			if GCC_VERSION >= 50200   # plugin support on gcc <= 5.1 is buggy on PPC
 	select HAVE_GENERIC_GUP
 	select HAVE_HW_BREAKPOINT		if PERF_EVENTS && (PPC_BOOK3S || PPC_8xx)
 	select HAVE_IDE
@@ -205,6 +200,7 @@ config PPC
 	select HAVE_KPROBES
 	select HAVE_KPROBES_ON_FTRACE
 	select HAVE_KRETPROBES
+	select HAVE_LD_DEAD_CODE_DATA_ELIMINATION
 	select HAVE_LIVEPATCH			if HAVE_DYNAMIC_FTRACE_WITH_REGS
 	select HAVE_MEMBLOCK
 	select HAVE_MEMBLOCK_NODE_MAP
@@ -220,12 +216,15 @@ config PPC
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_RCU_TABLE_FREE		if SMP
 	select HAVE_REGS_AND_STACK_ACCESS_API
+	select HAVE_RELIABLE_STACKTRACE		if PPC64 && CPU_LITTLE_ENDIAN
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_VIRT_CPU_ACCOUNTING
 	select HAVE_IRQ_TIME_ACCOUNTING
+	select IOMMU_HELPER			if PPC64
 	select IRQ_DOMAIN
 	select IRQ_FORCED_THREADING
 	select MODULES_USE_ELF_RELA
+	select NEED_SG_DMA_LENGTH
 	select NO_BOOTMEM
 	select OF
 	select OF_EARLY_FLATTREE
@@ -460,36 +459,9 @@ config LD_HEAD_STUB_CATCH
 
 	  If unsure, say "N".
 
-config DISABLE_MPROFILE_KERNEL
-	bool "Disable use of mprofile-kernel for kernel tracing"
-	depends on PPC64 && CPU_LITTLE_ENDIAN
-	default y
-	help
-	  Selecting this options disables use of the mprofile-kernel ABI for
-	  kernel tracing. That will cause options such as live patching
-	  (CONFIG_LIVEPATCH) which depend on CONFIG_DYNAMIC_FTRACE_WITH_REGS to
-	  be disabled also.
-
-	  If you have a toolchain which supports mprofile-kernel, then you can
-	  disable this. Otherwise leave it enabled. If you're not sure, say
-	  "Y".
-
 config MPROFILE_KERNEL
 	depends on PPC64 && CPU_LITTLE_ENDIAN
-	def_bool !DISABLE_MPROFILE_KERNEL
-
-config IOMMU_HELPER
-	def_bool PPC64
-
-config SWIOTLB
-	bool "SWIOTLB support"
-	default n
-	select IOMMU_HELPER
-	---help---
-	  Support for IO bounce buffering for systems without an IOMMU.
-	  This allows us to DMA to the full physical address space on
-	  platforms where the size of a physical address is larger
-	  than the bus address.  Not all platforms support this.
+	def_bool $(success,$(srctree)/arch/powerpc/tools/gcc-check-mprofile-kernel.sh $(CC) -I$(srctree)/include -D__KERNEL__)
 
 config HOTPLUG_CPU
 	bool "Support for enabling/disabling CPUs"
@@ -883,7 +855,7 @@ config PPC_MEM_KEYS
 	  page-based protections, but without requiring modification of the
 	  page tables when an application changes protection domains.
 
-	  For details, see Documentation/vm/protection-keys.txt
+	  For details, see Documentation/vm/protection-keys.rst
 
 	  If unsure, say y.
 
@@ -913,9 +885,6 @@ config ZONE_DMA
 config NEED_DMA_MAP_STATE
 	def_bool (PPC64 || NOT_COHERENT_CACHE)
 
-config NEED_SG_DMA_LENGTH
-	def_bool y
-
 config GENERIC_ISA_DMA
 	bool
 	depends on ISA_DMA_API
diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile
index 95813df90801..8eda91e0b8df 100644
--- a/arch/powerpc/Makefile
+++ b/arch/powerpc/Makefile
@@ -17,17 +17,18 @@ HAS_BIARCH	:= $(call cc-option-yn, -m32)
 # Set default 32 bits cross compilers for vdso and boot wrapper
 CROSS32_COMPILE ?=
 
-CROSS32CC		:= $(CROSS32_COMPILE)gcc
-CROSS32AR		:= $(CROSS32_COMPILE)ar
-
 ifeq ($(HAS_BIARCH),y)
 ifeq ($(CROSS32_COMPILE),)
-CROSS32CC	:= $(CC) -m32
-KBUILD_ARFLAGS	+= --target=elf32-powerpc
+ifdef CONFIG_PPC32
+# These options will be overridden by any -mcpu option that the CPU
+# or platform code sets later on the command line, but they are needed
+# to set a sane 32-bit cpu target for the 64-bit cross compiler which
+# may default to the wrong ISA.
+KBUILD_CFLAGS		+= -mcpu=powerpc
+KBUILD_AFLAGS		+= -mcpu=powerpc
+endif
 endif
 endif
-
-export CROSS32CC CROSS32AR
 
 ifeq ($(CROSS_COMPILE),)
 KBUILD_DEFCONFIG := $(shell uname -m)_defconfig
@@ -74,13 +75,15 @@ endif
 endif
 
 ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y)
-override LD	+= -EL
+KBUILD_CFLAGS	+= -mlittle-endian
+LDFLAGS		+= -EL
 LDEMULATION	:= lppc
 GNUTARGET	:= powerpcle
 MULTIPLEWORD	:= -mno-multiple
 KBUILD_CFLAGS_MODULE += $(call cc-option,-mno-save-toc-indirect)
 else
-override LD	+= -EB
+KBUILD_CFLAGS += $(call cc-option,-mbig-endian)
+LDFLAGS		+= -EB
 LDEMULATION	:= ppc
 GNUTARGET	:= powerpc
 MULTIPLEWORD	:= -mmultiple
@@ -93,19 +96,19 @@ aflags-$(CONFIG_CPU_BIG_ENDIAN)		+= $(call cc-option,-mabi=elfv1)
 aflags-$(CONFIG_CPU_LITTLE_ENDIAN)	+= -mabi=elfv2
 endif
 
-cflags-$(CONFIG_CPU_LITTLE_ENDIAN)	+= -mlittle-endian
-cflags-$(CONFIG_CPU_BIG_ENDIAN)		+= $(call cc-option,-mbig-endian)
 ifneq ($(cc-name),clang)
   cflags-$(CONFIG_CPU_LITTLE_ENDIAN)	+= -mno-strict-align
 endif
 
+cflags-$(CONFIG_CPU_BIG_ENDIAN)		+= $(call cc-option,-mbig-endian)
+cflags-$(CONFIG_CPU_LITTLE_ENDIAN)	+= -mlittle-endian
 aflags-$(CONFIG_CPU_BIG_ENDIAN)		+= $(call cc-option,-mbig-endian)
 aflags-$(CONFIG_CPU_LITTLE_ENDIAN)	+= -mlittle-endian
 
 ifeq ($(HAS_BIARCH),y)
-override AS	+= -a$(BITS)
-override LD	+= -m elf$(BITS)$(LDEMULATION)
-override CC	+= -m$(BITS)
+KBUILD_CFLAGS	+= -m$(BITS)
+KBUILD_AFLAGS	+= -m$(BITS) -Wl,-a$(BITS)
+LDFLAGS		+= -m elf$(BITS)$(LDEMULATION)
 KBUILD_ARFLAGS	+= --target=elf$(BITS)-$(GNUTARGET)
 endif
 
@@ -158,18 +161,7 @@ CFLAGS-$(CONFIG_GENERIC_CPU) += -mcpu=powerpc64
 endif
 
 ifdef CONFIG_MPROFILE_KERNEL
-    ifeq ($(shell $(srctree)/arch/powerpc/tools/gcc-check-mprofile-kernel.sh $(CC) -I$(srctree)/include -D__KERNEL__),OK)
-        CC_FLAGS_FTRACE := -pg -mprofile-kernel
-        KBUILD_CPPFLAGS += -DCC_USING_MPROFILE_KERNEL
-    else
-        # If the user asked for mprofile-kernel but the toolchain doesn't
-        # support it, emit a warning and deliberately break the build later
-        # with mprofile-kernel-not-supported. We would prefer to make this an
-        # error right here, but then the user would never be able to run
-        # oldconfig to change their configuration.
-        $(warning Compiler does not support mprofile-kernel, set CONFIG_DISABLE_MPROFILE_KERNEL)
-        CC_FLAGS_FTRACE := -mprofile-kernel-not-supported
-    endif
+	CC_FLAGS_FTRACE := -pg -mprofile-kernel
 endif
 
 CFLAGS-$(CONFIG_CELL_CPU) += $(call cc-option,-mcpu=cell)
diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile
index 26d5d2a5b8e9..49767e06202c 100644
--- a/arch/powerpc/boot/Makefile
+++ b/arch/powerpc/boot/Makefile
@@ -23,19 +23,23 @@ all: $(obj)/zImage
 compress-$(CONFIG_KERNEL_GZIP) := CONFIG_KERNEL_GZIP
 compress-$(CONFIG_KERNEL_XZ)   := CONFIG_KERNEL_XZ
 
+ifdef CROSS32_COMPILE
+    BOOTCC := $(CROSS32_COMPILE)gcc
+    BOOTAR := $(CROSS32_COMPILE)ar
+else
+    BOOTCC := $(CC)
+    BOOTAR := $(AR)
+endif
+
 BOOTCFLAGS    := -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \
 		 -fno-strict-aliasing -Os -msoft-float -pipe \
 		 -fomit-frame-pointer -fno-builtin -fPIC -nostdinc \
 		 -D$(compress-y)
 
-BOOTCC := $(CC)
 ifdef CONFIG_PPC64_BOOT_WRAPPER
 BOOTCFLAGS	+= -m64
 else
 BOOTCFLAGS	+= -m32
-ifdef CROSS32_COMPILE
-    BOOTCC := $(CROSS32_COMPILE)gcc
-endif
 endif
 
 BOOTCFLAGS	+= -isystem $(shell $(BOOTCC) -print-file-name=include)
@@ -49,6 +53,8 @@ endif
 
 BOOTAFLAGS	:= -D__ASSEMBLY__ $(BOOTCFLAGS) -traditional -nostdinc
 
+BOOTARFLAGS	:= -cr$(KBUILD_ARFLAGS)
+
 ifdef CONFIG_DEBUG_INFO
 BOOTCFLAGS	+= -g
 endif
@@ -202,7 +208,7 @@ quiet_cmd_bootas = BOOTAS  $@
       cmd_bootas = $(BOOTCC) -Wp,-MD,$(depfile) $(BOOTAFLAGS) -c -o $@ $<
 
 quiet_cmd_bootar = BOOTAR  $@
-      cmd_bootar = $(CROSS32AR) -cr$(KBUILD_ARFLAGS) $@.$$$$ $(filter-out FORCE,$^); mv $@.$$$$ $@
+      cmd_bootar = $(BOOTAR) $(BOOTARFLAGS) $@.$$$$ $(filter-out FORCE,$^); mv $@.$$$$ $@
 
 $(obj-libfdt): $(obj)/%.o: $(srctree)/scripts/dtc/libfdt/%.c FORCE
 	$(call if_changed_dep,bootcc)
diff --git a/arch/powerpc/boot/dts/fsl/t104xqds.dtsi b/arch/powerpc/boot/dts/fsl/t104xqds.dtsi
index 2fd4cbe7098f..615479732252 100644
--- a/arch/powerpc/boot/dts/fsl/t104xqds.dtsi
+++ b/arch/powerpc/boot/dts/fsl/t104xqds.dtsi
@@ -269,7 +269,7 @@
 
 		i2c@118000 {
 			pca9547@77 {
-				compatible = "philips,pca9547";
+				compatible = "nxp,pca9547";
 				reg = <0x77>;
 			};
 			rtc@68 {
diff --git a/arch/powerpc/boot/dts/sbc8349.dts b/arch/powerpc/boot/dts/sbc8349.dts
deleted file mode 100644
index fc89e00b765c..000000000000
--- a/arch/powerpc/boot/dts/sbc8349.dts
+++ /dev/null
@@ -1,331 +0,0 @@
-/*
- * SBC8349E Device Tree Source
- *
- * Copyright 2007 Wind River Inc.
- *
- * Paul Gortmaker (see MAINTAINERS for contact information)
- *
- *	-based largely on the Freescale MPC834x_MDS dts.
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-/dts-v1/;
-
-/ {
-	model = "SBC8349E";
-	compatible = "SBC834xE";
-	#address-cells = <1>;
-	#size-cells = <1>;
-
-	aliases {
-		ethernet0 = &enet0;
-		ethernet1 = &enet1;
-		serial0 = &serial0;
-		serial1 = &serial1;
-		pci0 = &pci0;
-	};
-
-	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		PowerPC,8349@0 {
-			device_type = "cpu";
-			reg = <0x0>;
-			d-cache-line-size = <32>;
-			i-cache-line-size = <32>;
-			d-cache-size = <32768>;
-			i-cache-size = <32768>;
-			timebase-frequency = <0>;	// from bootloader
-			bus-frequency = <0>;		// from bootloader
-			clock-frequency = <0>;		// from bootloader
-		};
-	};
-
-	memory {
-		device_type = "memory";
-		reg = <0x00000000 0x10000000>;	// 256MB at 0
-	};
-
-	soc8349@e0000000 {
-		#address-cells = <1>;
-		#size-cells = <1>;
-		device_type = "soc";
-		ranges = <0x0 0xe0000000 0x00100000>;
-		reg = <0xe0000000 0x00000200>;
-		bus-frequency = <0>;
-
-		wdt@200 {
-			compatible = "mpc83xx_wdt";
-			reg = <0x200 0x100>;
-		};
-
-		i2c@3000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			cell-index = <0>;
-			compatible = "fsl-i2c";
-			reg = <0x3000 0x100>;
-			interrupts = <14 0x8>;
-			interrupt-parent = <&ipic>;
-			dfsrr;
-		};
-
-		i2c@3100 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			cell-index = <1>;
-			compatible = "fsl-i2c";
-			reg = <0x3100 0x100>;
-			interrupts = <15 0x8>;
-			interrupt-parent = <&ipic>;
-			dfsrr;
-		};
-
-		spi@7000 {
-			cell-index = <0>;
-			compatible = "fsl,spi";
-			reg = <0x7000 0x1000>;
-			interrupts = <16 0x8>;
-			interrupt-parent = <&ipic>;
-			mode = "cpu";
-		};
-
-		dma@82a8 {
-			#address-cells = <1>;
-			#size-cells = <1>;
-			compatible = "fsl,mpc8349-dma", "fsl,elo-dma";
-			reg = <0x82a8 4>;
-			ranges = <0 0x8100 0x1a8>;
-			interrupt-parent = <&ipic>;
-			interrupts = <71 8>;
-			cell-index = <0>;
-			dma-channel@0 {
-				compatible = "fsl,mpc8349-dma-channel", "fsl,elo-dma-channel";
-				reg = <0 0x80>;
-				cell-index = <0>;
-				interrupt-parent = <&ipic>;
-				interrupts = <71 8>;
-			};
-			dma-channel@80 {
-				compatible = "fsl,mpc8349-dma-channel", "fsl,elo-dma-channel";
-				reg = <0x80 0x80>;
-				cell-index = <1>;
-				interrupt-parent = <&ipic>;
-				interrupts = <71 8>;
-			};
-			dma-channel@100 {
-				compatible = "fsl,mpc8349-dma-channel", "fsl,elo-dma-channel";
-				reg = <0x100 0x80>;
-				cell-index = <2>;
-				interrupt-parent = <&ipic>;
-				interrupts = <71 8>;
-			};
-			dma-channel@180 {
-				compatible = "fsl,mpc8349-dma-channel", "fsl,elo-dma-channel";
-				reg = <0x180 0x28>;
-				cell-index = <3>;
-				interrupt-parent = <&ipic>;
-				interrupts = <71 8>;
-			};
-		};
-
-		/* phy type (ULPI or SERIAL) are only types supported for MPH */
-		/* port = 0 or 1 */
-		usb@22000 {
-			compatible = "fsl-usb2-mph";
-			reg = <0x22000 0x1000>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			interrupt-parent = <&ipic>;
-			interrupts = <39 0x8>;
-			phy_type = "ulpi";
-			port0;
-		};
-
-		enet0: ethernet@24000 {
-			#address-cells = <1>;
-			#size-cells = <1>;
-			cell-index = <0>;
-			device_type = "network";
-			model = "TSEC";
-			compatible = "gianfar";
-			reg = <0x24000 0x1000>;
-			ranges = <0x0 0x24000 0x1000>;
-			local-mac-address = [ 00 00 00 00 00 00 ];
-			interrupts = <32 0x8 33 0x8 34 0x8>;
-			interrupt-parent = <&ipic>;
-			tbi-handle = <&tbi0>;
-			phy-handle = <&phy0>;
-			linux,network-index = <0>;
-
-			mdio@520 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				compatible = "fsl,gianfar-mdio";
-				reg = <0x520 0x20>;
-
-				phy0: ethernet-phy@19 {
-					interrupt-parent = <&ipic>;
-					interrupts = <20 0x8>;
-					reg = <0x19>;
-				};
-
-				phy1: ethernet-phy@1a {
-					interrupt-parent = <&ipic>;
-					interrupts = <21 0x8>;
-					reg = <0x1a>;
-				};
-
-				tbi0: tbi-phy@11 {
-					reg = <0x11>;
-					device_type = "tbi-phy";
-				};
-			};
-		};
-
-		enet1: ethernet@25000 {
-			#address-cells = <1>;
-			#size-cells = <1>;
-			cell-index = <1>;
-			device_type = "network";
-			model = "TSEC";
-			compatible = "gianfar";
-			reg = <0x25000 0x1000>;
-			ranges = <0x0 0x25000 0x1000>;
-			local-mac-address = [ 00 00 00 00 00 00 ];
-			interrupts = <35 0x8 36 0x8 37 0x8>;
-			interrupt-parent = <&ipic>;
-			tbi-handle = <&tbi1>;
-			phy-handle = <&phy1>;
-			linux,network-index = <1>;
-
-			mdio@520 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				compatible = "fsl,gianfar-tbi";
-				reg = <0x520 0x20>;
-
-				tbi1: tbi-phy@11 {
-					reg = <0x11>;
-					device_type = "tbi-phy";
-				};
-			};
-		};
-
-		serial0: serial@4500 {
-			cell-index = <0>;
-			device_type = "serial";
-			compatible = "fsl,ns16550", "ns16550";
-			reg = <0x4500 0x100>;
-			clock-frequency = <0>;
-			interrupts = <9 0x8>;
-			interrupt-parent = <&ipic>;
-		};
-
-		serial1: serial@4600 {
-			cell-index = <1>;
-			device_type = "serial";
-			compatible = "fsl,ns16550", "ns16550";
-			reg = <0x4600 0x100>;
-			clock-frequency = <0>;
-			interrupts = <10 0x8>;
-			interrupt-parent = <&ipic>;
-		};
-
-		crypto@30000 {
-			compatible = "fsl,sec2.0";
-			reg = <0x30000 0x10000>;
-			interrupts = <11 0x8>;
-			interrupt-parent = <&ipic>;
-			fsl,num-channels = <4>;
-			fsl,channel-fifo-len = <24>;
-			fsl,exec-units-mask = <0x7e>;
-			fsl,descriptor-types-mask = <0x01010ebf>;
-		};
-
-		/* IPIC
-		 * interrupts cell = <intr #, sense>
-		 * sense values match linux IORESOURCE_IRQ_* defines:
-		 * sense == 8: Level, low assertion
-		 * sense == 2: Edge, high-to-low change
-		 */
-		ipic: pic@700 {
-			interrupt-controller;
-			#address-cells = <0>;
-			#interrupt-cells = <2>;
-			reg = <0x700 0x100>;
-			device_type = "ipic";
-		};
-	};
-
-	localbus@e0005000 {
-		#address-cells = <2>;
-		#size-cells = <1>;
-		compatible = "fsl,mpc8349-localbus", "simple-bus";
-		reg = <0xe0005000 0x1000>;
-		interrupts = <77 0x8>;
-		interrupt-parent = <&ipic>;
-		ranges = <0x0 0x0 0xff800000 0x00800000		/* 8MB Flash */
-			  0x1 0x0 0xf8000000 0x00002000		/* 8KB EEPROM */
-			  0x2 0x0 0x10000000 0x04000000		/* 64MB SDRAM */
-			  0x3 0x0 0x10000000 0x04000000>;	/* 64MB SDRAM */
-
-		flash@0,0 {
-			#address-cells = <1>;
-			#size-cells = <1>;
-			compatible = "intel,28F640J3A", "cfi-flash";
-			reg = <0x0 0x0 0x800000>;
-			bank-width = <2>;
-			device-width = <1>;
-
-			partition@0 {
-				label = "u-boot";
-				reg = <0x00000000 0x00040000>;
-				read-only;
-			};
-
-			partition@40000 {
-				label = "user";
-				reg = <0x00040000 0x006c0000>;
-			};
-
-			partition@700000 {
-				label = "legacy u-boot";
-				reg = <0x00700000 0x00100000>;
-				read-only;
-			};
-
-		};
-	};
-
-	pci0: pci@e0008500 {
-		interrupt-map-mask = <0xf800 0x0 0x0 0x7>;
-		interrupt-map = <
-
-				/* IDSEL 0x11 */
-				 0x8800 0x0 0x0 0x1 &ipic 48 0x8
-				 0x8800 0x0 0x0 0x2 &ipic 17 0x8
-				 0x8800 0x0 0x0 0x3 &ipic 18 0x8
-				 0x8800 0x0 0x0 0x4 &ipic 19 0x8>;
-
-		interrupt-parent = <&ipic>;
-		interrupts = <0x42 0x8>;
-		bus-range = <0 0>;
-		ranges = <0x02000000 0x0 0x90000000 0x90000000 0x0 0x10000000
-			  0x42000000 0x0 0x80000000 0x80000000 0x0 0x10000000
-			  0x01000000 0x0 0x00000000 0xe2000000 0x0 0x00100000>;
-		clock-frequency = <66666666>;
-		#interrupt-cells = <1>;
-		#size-cells = <2>;
-		#address-cells = <3>;
-		reg = <0xe0008500 0x100		/* internal registers */
-		       0xe0008300 0x8>;		/* config space access registers */
-		compatible = "fsl,mpc8349-pci";
-		device_type = "pci";
-	};
-};
diff --git a/arch/powerpc/configs/83xx/sbc834x_defconfig b/arch/powerpc/configs/83xx/sbc834x_defconfig
deleted file mode 100644
index 7d74699334da..000000000000
--- a/arch/powerpc/configs/83xx/sbc834x_defconfig
+++ /dev/null
@@ -1,74 +0,0 @@
-CONFIG_SYSVIPC=y
-CONFIG_LOG_BUF_SHIFT=14
-CONFIG_BLK_DEV_INITRD=y
-CONFIG_EXPERT=y
-# CONFIG_KALLSYMS is not set
-CONFIG_SLAB=y
-CONFIG_MODULES=y
-CONFIG_MODULE_UNLOAD=y
-# CONFIG_BLK_DEV_BSG is not set
-# CONFIG_PPC_CHRP is not set
-# CONFIG_PPC_PMAC is not set
-CONFIG_PPC_83xx=y
-CONFIG_SBC834x=y
-CONFIG_GEN_RTC=y
-CONFIG_PCI=y
-CONFIG_NET=y
-CONFIG_PACKET=y
-CONFIG_UNIX=y
-CONFIG_XFRM_USER=m
-CONFIG_INET=y
-CONFIG_IP_MULTICAST=y
-CONFIG_IP_PNP=y
-CONFIG_IP_PNP_DHCP=y
-CONFIG_IP_PNP_BOOTP=y
-CONFIG_SYN_COOKIES=y
-# CONFIG_IPV6 is not set
-CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FW_LOADER is not set
-CONFIG_MTD=y
-CONFIG_MTD_CMDLINE_PARTS=y
-CONFIG_MTD_BLOCK=y
-CONFIG_MTD_CFI=y
-CONFIG_MTD_CFI_INTELEXT=y
-CONFIG_MTD_PHYSMAP_OF=y
-CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_RAM=y
-CONFIG_BLK_DEV_RAM_SIZE=32768
-CONFIG_SCSI=y
-# CONFIG_SCSI_PROC_FS is not set
-CONFIG_BLK_DEV_SD=y
-# CONFIG_SCSI_LOWLEVEL is not set
-CONFIG_NETDEVICES=y
-CONFIG_GIANFAR=y
-CONFIG_BROADCOM_PHY=y
-# CONFIG_INPUT_KEYBOARD is not set
-# CONFIG_INPUT_MOUSE is not set
-# CONFIG_SERIO is not set
-# CONFIG_VT is not set
-CONFIG_SERIAL_8250=y
-CONFIG_SERIAL_8250_CONSOLE=y
-# CONFIG_SERIAL_8250_PCI is not set
-CONFIG_SERIAL_8250_NR_UARTS=2
-CONFIG_SERIAL_8250_RUNTIME_UARTS=2
-# CONFIG_HW_RANDOM is not set
-CONFIG_I2C=y
-CONFIG_I2C_CHARDEV=y
-CONFIG_I2C_MPC=y
-CONFIG_WATCHDOG=y
-# CONFIG_USB_HID is not set
-CONFIG_USB=y
-CONFIG_USB_MON=y
-CONFIG_USB_EHCI_HCD=y
-CONFIG_USB_EHCI_FSL=y
-CONFIG_USB_STORAGE=y
-CONFIG_EXT2_FS=y
-CONFIG_EXT4_FS=y
-CONFIG_PROC_KCORE=y
-CONFIG_TMPFS=y
-CONFIG_NFS_FS=y
-CONFIG_NFS_V4=y
-CONFIG_ROOT_NFS=y
-CONFIG_CRYPTO_ECB=m
-CONFIG_CRYPTO_PCBC=m
-# CONFIG_CRYPTO_HW is not set
diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig
index 9e92aa6a52ba..6ab34e60495f 100644
--- a/arch/powerpc/configs/powernv_defconfig
+++ b/arch/powerpc/configs/powernv_defconfig
@@ -38,7 +38,9 @@ CONFIG_MODULE_UNLOAD=y
 CONFIG_MODVERSIONS=y
 CONFIG_MODULE_SRCVERSION_ALL=y
 CONFIG_PARTITION_ADVANCED=y
+CONFIG_SCOM_DEBUGFS=y
 CONFIG_OPAL_PRD=y
+CONFIG_PPC_MEMTRACE=y
 # CONFIG_PPC_PSERIES is not set
 # CONFIG_PPC_OF_BOOT_TRAMPOLINE is not set
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
@@ -54,7 +56,10 @@ CONFIG_NUMA=y
 CONFIG_MEMORY_HOTPLUG=y
 CONFIG_MEMORY_HOTREMOVE=y
 CONFIG_KSM=y
+CONFIG_MEMORY_FAILURE=y
+CONFIG_HWPOISON_INJECT=m
 CONFIG_TRANSPARENT_HUGEPAGE=y
+CONFIG_DEFERRED_STRUCT_PAGE_INIT=y
 CONFIG_PPC_64K_PAGES=y
 CONFIG_PPC_SUBPAGE_PROT=y
 CONFIG_SCHED_SMT=y
@@ -72,7 +77,13 @@ CONFIG_SYN_COOKIES=y
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
 CONFIG_INET_IPCOMP=m
-# CONFIG_IPV6 is not set
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_IPCOMP=m
+CONFIG_INET6_XFRM_MODE_TRANSPORT=m
+CONFIG_INET6_XFRM_MODE_TUNNEL=m
+CONFIG_INET6_XFRM_MODE_BEET=m
+CONFIG_IPV6_SIT=m
 CONFIG_NETFILTER=y
 # CONFIG_NETFILTER_ADVANCED is not set
 CONFIG_BRIDGE=m
@@ -81,33 +92,28 @@ CONFIG_NET_SCHED=y
 CONFIG_NET_CLS_BPF=m
 CONFIG_NET_CLS_ACT=y
 CONFIG_NET_ACT_BPF=m
+CONFIG_DNS_RESOLVER=y
 CONFIG_BPF_JIT=y
+# CONFIG_WIRELESS is not set
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 CONFIG_MTD=y
 CONFIG_MTD_BLOCK=y
 CONFIG_MTD_POWERNV_FLASH=y
-CONFIG_PARPORT=m
-CONFIG_PARPORT_PC=m
-CONFIG_BLK_DEV_FD=m
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_NBD=m
-CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM=m
 CONFIG_BLK_DEV_RAM_SIZE=65536
-CONFIG_VIRTIO_BLK=m
 CONFIG_BLK_DEV_NVME=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_AMD74XX=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=m
-CONFIG_BLK_DEV_SR=y
+CONFIG_BLK_DEV_SR=m
 CONFIG_BLK_DEV_SR_VENDOR=y
-CONFIG_CHR_DEV_SG=y
+CONFIG_CHR_DEV_SG=m
 CONFIG_SCSI_CONSTANTS=y
-CONFIG_SCSI_FC_ATTRS=y
+CONFIG_SCSI_SCAN_ASYNC=y
+CONFIG_SCSI_FC_ATTRS=m
 CONFIG_SCSI_SRP_ATTRS=y
 CONFIG_SCSI_CXGB3_ISCSI=m
 CONFIG_SCSI_CXGB4_ISCSI=m
@@ -121,7 +127,6 @@ CONFIG_SCSI_IPR=y
 CONFIG_SCSI_QLA_FC=m
 CONFIG_SCSI_QLA_ISCSI=m
 CONFIG_SCSI_LPFC=m
-CONFIG_SCSI_VIRTIO=m
 CONFIG_SCSI_DH=y
 CONFIG_SCSI_DH_RDAC=m
 CONFIG_SCSI_DH_ALUA=m
@@ -152,16 +157,16 @@ CONFIG_DUMMY=m
 CONFIG_MACVLAN=m
 CONFIG_MACVTAP=m
 CONFIG_VXLAN=m
-CONFIG_NETCONSOLE=y
+CONFIG_NETCONSOLE=m
 CONFIG_TUN=m
 CONFIG_VETH=m
-CONFIG_VIRTIO_NET=m
 CONFIG_VORTEX=m
 CONFIG_ACENIC=m
 CONFIG_ACENIC_OMIT_TIGON_I=y
 CONFIG_PCNET32=m
 CONFIG_TIGON3=y
 CONFIG_BNX2X=m
+# CONFIG_CAVIUM_PTP is not set
 CONFIG_CHELSIO_T1=m
 CONFIG_BE2NET=m
 CONFIG_S2IO=m
@@ -172,46 +177,62 @@ CONFIG_IXGB=m
 CONFIG_IXGBE=m
 CONFIG_I40E=m
 CONFIG_MLX4_EN=m
+CONFIG_MLX5_CORE=m
+CONFIG_MLX5_FPGA=y
+CONFIG_MLX5_CORE_EN=y
+CONFIG_MLX5_CORE_IPOIB=y
 CONFIG_MYRI10GE=m
 CONFIG_QLGE=m
 CONFIG_NETXEN_NIC=m
-CONFIG_PPP=m
-CONFIG_PPP_BSDCOMP=m
-CONFIG_PPP_DEFLATE=m
-CONFIG_PPPOE=m
-CONFIG_PPP_ASYNC=m
-CONFIG_PPP_SYNC_TTY=m
+CONFIG_USB_NET_DRIVERS=m
+# CONFIG_WLAN is not set
 CONFIG_INPUT_EVDEV=m
-CONFIG_INPUT_MISC=y
-# CONFIG_SERIO_SERPORT is not set
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+# CONFIG_SERIO is not set
+# CONFIG_LEGACY_PTYS is not set
 CONFIG_SERIAL_8250=y
 CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_PCI=m
 CONFIG_SERIAL_JSM=m
-CONFIG_VIRTIO_CONSOLE=m
 CONFIG_IPMI_HANDLER=y
 CONFIG_IPMI_DEVICE_INTERFACE=y
 CONFIG_IPMI_POWERNV=y
-CONFIG_RAW_DRIVER=y
-CONFIG_MAX_RAW_DEVS=1024
+# CONFIG_DEVPORT is not set
 CONFIG_I2C_CHARDEV=y
+# CONFIG_PTP_1588_CLOCK is not set
 CONFIG_DRM=y
 CONFIG_DRM_AST=y
 CONFIG_FIRMWARE_EDID=y
 CONFIG_FB_OF=y
-CONFIG_FB_MATROX=y
+CONFIG_FB_MATROX=m
 CONFIG_FB_MATROX_MILLENIUM=y
 CONFIG_FB_MATROX_MYSTIQUE=y
 CONFIG_FB_MATROX_G=y
-CONFIG_FB_RADEON=y
-CONFIG_FB_IBM_GXT4500=y
+CONFIG_FB_RADEON=m
+CONFIG_FB_IBM_GXT4500=m
 CONFIG_LCD_PLATFORM=m
+CONFIG_BACKLIGHT_GENERIC=m
 # CONFIG_VGA_CONSOLE is not set
 CONFIG_LOGO=y
-CONFIG_HID_GYRATION=y
-CONFIG_HID_PANTHERLORD=y
-CONFIG_HID_PETALYNX=y
-CONFIG_HID_SAMSUNG=y
-CONFIG_HID_SUNPLUS=y
+CONFIG_HID_A4TECH=m
+CONFIG_HID_APPLE=m
+CONFIG_HID_BELKIN=m
+CONFIG_HID_CHERRY=m
+CONFIG_HID_CHICONY=m
+CONFIG_HID_CYPRESS=m
+CONFIG_HID_EZKEY=m
+CONFIG_HID_GYRATION=m
+CONFIG_HID_ITE=m
+CONFIG_HID_KENSINGTON=m
+CONFIG_HID_LOGITECH=m
+CONFIG_HID_MICROSOFT=m
+CONFIG_HID_MONTEREY=m
+CONFIG_HID_PANTHERLORD=m
+CONFIG_HID_PETALYNX=m
+CONFIG_HID_SAMSUNG=m
+CONFIG_HID_SUNPLUS=m
+CONFIG_USB_HID=m
 CONFIG_USB_HIDDEV=y
 CONFIG_USB=y
 CONFIG_USB_MON=m
@@ -219,6 +240,7 @@ CONFIG_USB_XHCI_HCD=y
 CONFIG_USB_EHCI_HCD=y
 # CONFIG_USB_EHCI_HCD_PPC_OF is not set
 CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PCI=m
 CONFIG_USB_STORAGE=y
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=m
@@ -236,8 +258,9 @@ CONFIG_INFINIBAND_SRP=m
 CONFIG_INFINIBAND_ISER=m
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=y
-CONFIG_VIRTIO_PCI=m
-CONFIG_VIRTIO_BALLOON=m
+# CONFIG_VIRTIO_MENU is not set
+CONFIG_LIBNVDIMM=y
+# CONFIG_ND_BLK is not set
 CONFIG_EXT2_FS=y
 CONFIG_EXT2_FS_XATTR=y
 CONFIG_EXT2_FS_POSIX_ACL=y
@@ -253,12 +276,13 @@ CONFIG_XFS_POSIX_ACL=y
 CONFIG_BTRFS_FS=m
 CONFIG_BTRFS_FS_POSIX_ACL=y
 CONFIG_NILFS2_FS=m
+CONFIG_FANOTIFY=y
 CONFIG_AUTOFS4_FS=m
 CONFIG_FUSE_FS=m
 CONFIG_OVERLAY_FS=m
 CONFIG_ISO9660_FS=y
 CONFIG_UDF_FS=m
-CONFIG_MSDOS_FS=y
+CONFIG_MSDOS_FS=m
 CONFIG_VFAT_FS=m
 CONFIG_PROC_KCORE=y
 CONFIG_TMPFS=y
@@ -270,9 +294,9 @@ CONFIG_SQUASHFS_XATTR=y
 CONFIG_SQUASHFS_LZO=y
 CONFIG_SQUASHFS_XZ=y
 CONFIG_PSTORE=y
-CONFIG_NFS_FS=y
+CONFIG_NFS_FS=m
 CONFIG_NFS_V3_ACL=y
-CONFIG_NFS_V4=y
+CONFIG_NFS_V4=m
 CONFIG_NFSD=m
 CONFIG_NFSD_V3_ACL=y
 CONFIG_NFSD_V4=y
@@ -291,9 +315,7 @@ CONFIG_DEBUG_STACKOVERFLOW=y
 CONFIG_SOFTLOCKUP_DETECTOR=y
 CONFIG_HARDLOCKUP_DETECTOR=y
 CONFIG_LATENCYTOP=y
-CONFIG_FTRACE=y
 CONFIG_FUNCTION_TRACER=y
-CONFIG_FUNCTION_GRAPH_TRACER=y
 CONFIG_SCHED_TRACER=y
 CONFIG_FTRACE_SYSCALLS=y
 CONFIG_BLK_DEV_IO_TRACE=y
@@ -303,10 +325,10 @@ CONFIG_FTR_FIXUP_SELFTEST=y
 CONFIG_MSI_BITMAP_SELFTEST=y
 CONFIG_XMON=y
 CONFIG_CRYPTO_TEST=m
-CONFIG_CRYPTO_CCM=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_CRC32C_VPMSUM=m
+CONFIG_CRYPTO_CRCT10DIF_VPMSUM=m
 CONFIG_CRYPTO_MD5_PPC=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_SHA1_PPC=m
diff --git a/arch/powerpc/configs/wii_defconfig b/arch/powerpc/configs/wii_defconfig
index 0b0f78823a1b..10940533da71 100644
--- a/arch/powerpc/configs/wii_defconfig
+++ b/arch/powerpc/configs/wii_defconfig
@@ -49,7 +49,9 @@ CONFIG_BLK_DEV_RAM_COUNT=2
 CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_NETDEVICES=y
+# CONFIG_ETHERNET is not set
 CONFIG_B43=y
+CONFIG_B43_BUSES_SSB=y
 CONFIG_B43_SDIO=y
 # CONFIG_B43_PHY_LP is not set
 CONFIG_B43_DEBUG=y
@@ -57,6 +59,7 @@ CONFIG_INPUT_FF_MEMLESS=m
 CONFIG_INPUT_JOYDEV=y
 CONFIG_INPUT_EVDEV=y
 # CONFIG_KEYBOARD_ATKBD is not set
+CONFIG_KEYBOARD_GPIO=y
 # CONFIG_MOUSE_PS2 is not set
 CONFIG_INPUT_JOYSTICK=y
 CONFIG_INPUT_MISC=y
@@ -71,6 +74,9 @@ CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_GPIO=y
 CONFIG_GPIOLIB=y
 CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_HLWD=y
+CONFIG_POWER_RESET=y
+CONFIG_POWER_RESET_GPIO=y
 # CONFIG_HWMON is not set
 CONFIG_SSB_DEBUG=y
 CONFIG_FB=y
@@ -88,6 +94,14 @@ CONFIG_HID_APPLE=m
 CONFIG_HID_WACOM=m
 CONFIG_MMC=y
 CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_HLWD=y
+CONFIG_NEW_LEDS=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_TRIGGERS=y
+CONFIG_LEDS_TRIGGER_HEARTBEAT=y
+CONFIG_LEDS_TRIGGER_PANIC=y
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_GENERIC=y
 CONFIG_EXT2_FS=y
diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
index d9713ad62e3c..6f661e3757c8 100644
--- a/arch/powerpc/include/asm/asm-prototypes.h
+++ b/arch/powerpc/include/asm/asm-prototypes.h
@@ -36,8 +36,7 @@ void kexec_copy_flush(struct kimage *image);
 /* pseries hcall tracing */
 extern struct static_key hcall_tracepoint_key;
 void __trace_hcall_entry(unsigned long opcode, unsigned long *args);
-void __trace_hcall_exit(long opcode, unsigned long retval,
-			unsigned long *retbuf);
+void __trace_hcall_exit(long opcode, long retval, unsigned long *retbuf);
 /* OPAL tracing */
 #ifdef HAVE_JUMP_LABEL
 extern struct static_key opal_tracepoint_key;
@@ -81,18 +80,12 @@ void machine_check_exception(struct pt_regs *regs);
 void emulation_assist_interrupt(struct pt_regs *regs);
 
 /* signals, syscalls and interrupts */
-#ifdef CONFIG_PPC64
-int sys_swapcontext(struct ucontext __user *old_ctx,
-		    struct ucontext __user *new_ctx,
-		    long ctx_size, long r6, long r7, long r8, struct pt_regs *regs);
-#else
 long sys_swapcontext(struct ucontext __user *old_ctx,
 		    struct ucontext __user *new_ctx,
-		    int ctx_size, int r6, int r7, int r8, struct pt_regs *regs);
-int sys_debug_setcontext(struct ucontext __user *ctx,
-			 int ndbg, struct sig_dbg_op __user *dbg,
-			 int r6, int r7, int r8,
-			 struct pt_regs *regs);
+		    long ctx_size);
+#ifdef CONFIG_PPC32
+long sys_debug_setcontext(struct ucontext __user *ctx,
+			  int ndbg, struct sig_dbg_op __user *dbg);
 int
 ppc_select(int n, fd_set __user *inp, fd_set __user *outp, fd_set __user *exp, struct timeval __user *tvp);
 unsigned long __init early_init(unsigned long dt_ptr);
diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h
index 4b5423030d4b..9a3798660cef 100644
--- a/arch/powerpc/include/asm/book3s/64/hash-4k.h
+++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h
@@ -38,8 +38,12 @@
 #define H_PAGE_4K_PFN	0x0
 #define H_PAGE_THP_HUGE 0x0
 #define H_PAGE_COMBO	0x0
-#define H_PTE_FRAG_NR	0
-#define H_PTE_FRAG_SIZE_SHIFT  0
+
+/* 8 bytes per each pte entry */
+#define H_PTE_FRAG_SIZE_SHIFT  (H_PTE_INDEX_SIZE + 3)
+#define H_PTE_FRAG_NR	(PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT)
+#define H_PMD_FRAG_SIZE_SHIFT  (H_PMD_INDEX_SIZE + 3)
+#define H_PMD_FRAG_NR	(PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT)
 
 /* memory key bits, only 8 keys supported */
 #define H_PTE_PKEY_BIT0	0
diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h
index cc82745355b3..c81793d47af9 100644
--- a/arch/powerpc/include/asm/book3s/64/hash-64k.h
+++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h
@@ -46,6 +46,13 @@
 #define H_PTE_FRAG_SIZE_SHIFT  (H_PTE_INDEX_SIZE + 3 + 1)
 #define H_PTE_FRAG_NR	(PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT)
 
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
+#define H_PMD_FRAG_SIZE_SHIFT  (H_PMD_INDEX_SIZE + 3 + 1)
+#else
+#define H_PMD_FRAG_SIZE_SHIFT  (H_PMD_INDEX_SIZE + 3)
+#endif
+#define H_PMD_FRAG_NR	(PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT)
+
 #ifndef __ASSEMBLY__
 #include <asm/errno.h>
 
diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h
index cc8cd656ccfe..0387b155f13d 100644
--- a/arch/powerpc/include/asm/book3s/64/hash.h
+++ b/arch/powerpc/include/asm/book3s/64/hash.h
@@ -23,16 +23,6 @@
 				 H_PUD_INDEX_SIZE + H_PGD_INDEX_SIZE + PAGE_SHIFT)
 #define H_PGTABLE_RANGE		(ASM_CONST(1) << H_PGTABLE_EADDR_SIZE)
 
-#if (defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)) && \
-	defined(CONFIG_PPC_64K_PAGES)
-/*
- * only with hash 64k we need to use the second half of pmd page table
- * to store pointer to deposited pgtable_t
- */
-#define H_PMD_CACHE_INDEX	(H_PMD_INDEX_SIZE + 1)
-#else
-#define H_PMD_CACHE_INDEX	H_PMD_INDEX_SIZE
-#endif
 /*
  * We store the slot details in the second half of page table.
  * Increase the pud level table so that hugetlb ptes can be stored
diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h
index 5094696eecd6..9c8c669a6b6a 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu.h
@@ -134,10 +134,11 @@ typedef struct {
 #ifdef CONFIG_PPC_SUBPAGE_PROT
 	struct subpage_prot_table spt;
 #endif /* CONFIG_PPC_SUBPAGE_PROT */
-#ifdef CONFIG_PPC_64K_PAGES
-	/* for 4K PTE fragment support */
+	/*
+	 * pagetable fragment support
+	 */
 	void *pte_frag;
-#endif
+	void *pmd_frag;
 #ifdef CONFIG_SPAPR_TCE_IOMMU
 	struct list_head iommu_group_mem_list;
 #endif
diff --git a/arch/powerpc/include/asm/book3s/64/pgalloc.h b/arch/powerpc/include/asm/book3s/64/pgalloc.h
index 558a159600ad..01ee40f11f3a 100644
--- a/arch/powerpc/include/asm/book3s/64/pgalloc.h
+++ b/arch/powerpc/include/asm/book3s/64/pgalloc.h
@@ -42,7 +42,9 @@ extern struct kmem_cache *pgtable_cache[];
 		})
 
 extern pte_t *pte_fragment_alloc(struct mm_struct *, unsigned long, int);
+extern pmd_t *pmd_fragment_alloc(struct mm_struct *, unsigned long);
 extern void pte_fragment_free(unsigned long *, int);
+extern void pmd_fragment_free(unsigned long *);
 extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
 #ifdef CONFIG_SMP
 extern void __tlb_remove_table(void *_table);
@@ -88,8 +90,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 	 * need to do this for 4k.
 	 */
 #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_64K_PAGES) && \
-	((H_PGD_INDEX_SIZE == H_PUD_CACHE_INDEX) ||		     \
-	 (H_PGD_INDEX_SIZE == H_PMD_CACHE_INDEX))
+	(H_PGD_INDEX_SIZE == H_PUD_CACHE_INDEX)
 	memset(pgd, 0, PGD_TABLE_SIZE);
 #endif
 	return pgd;
@@ -124,36 +125,35 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
 }
 
 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
-                                  unsigned long address)
+				  unsigned long address)
 {
 	/*
 	 * By now all the pud entries should be none entries. So go
 	 * ahead and flush the page walk cache
 	 */
 	flush_tlb_pgtable(tlb, address);
-	pgtable_free_tlb(tlb, pud, PUD_CACHE_INDEX);
+	pgtable_free_tlb(tlb, pud, PUD_INDEX);
 }
 
 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
 {
-	return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
-		pgtable_gfp_flags(mm, GFP_KERNEL));
+	return pmd_fragment_alloc(mm, addr);
 }
 
 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
 {
-	kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd);
+	pmd_fragment_free((unsigned long *)pmd);
 }
 
 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
-                                  unsigned long address)
+				  unsigned long address)
 {
 	/*
 	 * By now all the pud entries should be none entries. So go
 	 * ahead and flush the page walk cache
 	 */
 	flush_tlb_pgtable(tlb, address);
-        return pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX);
+	return pgtable_free_tlb(tlb, pmd, PMD_INDEX);
 }
 
 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
@@ -173,31 +173,6 @@ static inline pgtable_t pmd_pgtable(pmd_t pmd)
 	return (pgtable_t)pmd_page_vaddr(pmd);
 }
 
-#ifdef CONFIG_PPC_4K_PAGES
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
-					  unsigned long address)
-{
-	return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
-}
-
-static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
-				      unsigned long address)
-{
-	struct page *page;
-	pte_t *pte;
-
-	pte = (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT);
-	if (!pte)
-		return NULL;
-	page = virt_to_page(pte);
-	if (!pgtable_page_ctor(page)) {
-		__free_page(page);
-		return NULL;
-	}
-	return pte;
-}
-#else /* if CONFIG_PPC_64K_PAGES */
-
 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
 					  unsigned long address)
 {
@@ -209,7 +184,6 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
 {
 	return (pgtable_t)pte_fragment_alloc(mm, address, 0);
 }
-#endif
 
 static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
 {
@@ -229,7 +203,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
 	 * ahead and flush the page walk cache
 	 */
 	flush_tlb_pgtable(tlb, address);
-	pgtable_free_tlb(tlb, table, 0);
+	pgtable_free_tlb(tlb, table, PTE_INDEX);
 }
 
 #define check_pgt_cache()	do { } while (0)
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index 47b5ffc8715d..77ab65f32d8e 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -212,13 +212,13 @@ extern unsigned long __pte_index_size;
 extern unsigned long __pmd_index_size;
 extern unsigned long __pud_index_size;
 extern unsigned long __pgd_index_size;
-extern unsigned long __pmd_cache_index;
 extern unsigned long __pud_cache_index;
 #define PTE_INDEX_SIZE  __pte_index_size
 #define PMD_INDEX_SIZE  __pmd_index_size
 #define PUD_INDEX_SIZE  __pud_index_size
 #define PGD_INDEX_SIZE  __pgd_index_size
-#define PMD_CACHE_INDEX __pmd_cache_index
+/* pmd table use page table fragments */
+#define PMD_CACHE_INDEX  0
 #define PUD_CACHE_INDEX __pud_cache_index
 /*
  * Because of use of pte fragments and THP, size of page table
@@ -246,6 +246,12 @@ extern unsigned long __pte_frag_size_shift;
 #define PTE_FRAG_SIZE_SHIFT __pte_frag_size_shift
 #define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
 
+extern unsigned long __pmd_frag_nr;
+#define PMD_FRAG_NR __pmd_frag_nr
+extern unsigned long __pmd_frag_size_shift;
+#define PMD_FRAG_SIZE_SHIFT __pmd_frag_size_shift
+#define PMD_FRAG_SIZE (1UL << PMD_FRAG_SIZE_SHIFT)
+
 #define PTRS_PER_PTE	(1 << PTE_INDEX_SIZE)
 #define PTRS_PER_PMD	(1 << PMD_INDEX_SIZE)
 #define PTRS_PER_PUD	(1 << PUD_INDEX_SIZE)
@@ -273,6 +279,16 @@ extern unsigned long __pte_frag_size_shift;
 /* Bits to mask out from a PGD to get to the PUD page */
 #define PGD_MASKED_BITS		0xc0000000000000ffUL
 
+/*
+ * Used as an indicator for rcu callback functions
+ */
+enum pgtable_index {
+	PTE_INDEX = 0,
+	PMD_INDEX,
+	PUD_INDEX,
+	PGD_INDEX,
+};
+
 extern unsigned long __vmalloc_start;
 extern unsigned long __vmalloc_end;
 #define VMALLOC_START	__vmalloc_start
@@ -319,9 +335,6 @@ extern unsigned long pci_io_base;
 /* Advertise special mapping type for AGP */
 #define HAVE_PAGE_AGP
 
-/* Advertise support for _PAGE_SPECIAL */
-#define __HAVE_ARCH_PTE_SPECIAL
-
 #ifndef __ASSEMBLY__
 
 /*
diff --git a/arch/powerpc/include/asm/book3s/64/radix-4k.h b/arch/powerpc/include/asm/book3s/64/radix-4k.h
index ca366ec86310..863c3e8286fb 100644
--- a/arch/powerpc/include/asm/book3s/64/radix-4k.h
+++ b/arch/powerpc/include/asm/book3s/64/radix-4k.h
@@ -15,4 +15,7 @@
 #define RADIX_PTE_FRAG_SIZE_SHIFT  (RADIX_PTE_INDEX_SIZE + 3)
 #define RADIX_PTE_FRAG_NR	(PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT)
 
+#define RADIX_PMD_FRAG_SIZE_SHIFT  (RADIX_PMD_INDEX_SIZE + 3)
+#define RADIX_PMD_FRAG_NR	(PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT)
+
 #endif /* _ASM_POWERPC_PGTABLE_RADIX_4K_H */
diff --git a/arch/powerpc/include/asm/book3s/64/radix-64k.h b/arch/powerpc/include/asm/book3s/64/radix-64k.h
index 830082496876..ccb78ca9d0c5 100644
--- a/arch/powerpc/include/asm/book3s/64/radix-64k.h
+++ b/arch/powerpc/include/asm/book3s/64/radix-64k.h
@@ -16,4 +16,8 @@
  */
 #define RADIX_PTE_FRAG_SIZE_SHIFT  (RADIX_PTE_INDEX_SIZE + 3)
 #define RADIX_PTE_FRAG_NR	(PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT)
+
+#define RADIX_PMD_FRAG_SIZE_SHIFT  (RADIX_PMD_INDEX_SIZE + 3)
+#define RADIX_PMD_FRAG_NR	(PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT)
+
 #endif /* _ASM_POWERPC_PGTABLE_RADIX_64K_H */
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 19b45ba6caf9..ef5c3f2994c9 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -51,4 +51,11 @@ extern void radix__flush_tlb_all(void);
 extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
 					unsigned long address);
 
+extern void radix__flush_tlb_lpid_page(unsigned int lpid,
+					unsigned long addr,
+					unsigned long page_size);
+extern void radix__flush_pwc_lpid(unsigned int lpid);
+extern void radix__local_flush_tlb_lpid(unsigned int lpid);
+extern void radix__local_flush_tlb_lpid_guest(unsigned int lpid);
+
 #endif
diff --git a/arch/powerpc/include/asm/checksum.h b/arch/powerpc/include/asm/checksum.h
index 842124b199b5..4e63787dc3be 100644
--- a/arch/powerpc/include/asm/checksum.h
+++ b/arch/powerpc/include/asm/checksum.h
@@ -112,7 +112,7 @@ static inline __wsum csum_add(__wsum csum, __wsum addend)
 
 #ifdef __powerpc64__
 	res += (__force u64)addend;
-	return (__force __wsum) from64to32(res);
+	return (__force __wsum)((u32)res + (res >> 32));
 #else
 	asm("addc %0,%0,%1;"
 	    "addze %0,%0;"
diff --git a/arch/powerpc/include/asm/compat.h b/arch/powerpc/include/asm/compat.h
index 62168e1158f1..85c8af2bb272 100644
--- a/arch/powerpc/include/asm/compat.h
+++ b/arch/powerpc/include/asm/compat.h
@@ -17,7 +17,6 @@
 
 typedef u32		compat_size_t;
 typedef s32		compat_ssize_t;
-typedef s32		compat_time_t;
 typedef s32		compat_clock_t;
 typedef s32		compat_pid_t;
 typedef u32		__compat_uid_t;
@@ -45,16 +44,6 @@ typedef u32		compat_ulong_t;
 typedef u64		compat_u64;
 typedef u32		compat_uptr_t;
 
-struct compat_timespec {
-	compat_time_t	tv_sec;
-	s32		tv_nsec;
-};
-
-struct compat_timeval {
-	compat_time_t	tv_sec;
-	s32		tv_usec;
-};
-
 struct compat_stat {
 	compat_dev_t	st_dev;
 	compat_ino_t	st_ino;
@@ -173,10 +162,10 @@ struct compat_ipc64_perm {
 
 struct compat_semid64_ds {
 	struct compat_ipc64_perm sem_perm;
-	unsigned int __unused1;
-	compat_time_t sem_otime;
-	unsigned int __unused2;
-	compat_time_t sem_ctime;
+	unsigned int sem_otime_high;
+	unsigned int sem_otime;
+	unsigned int sem_ctime_high;
+	unsigned int sem_ctime;
 	compat_ulong_t sem_nsems;
 	compat_ulong_t __unused3;
 	compat_ulong_t __unused4;
@@ -184,12 +173,12 @@ struct compat_semid64_ds {
 
 struct compat_msqid64_ds {
 	struct compat_ipc64_perm msg_perm;
-	unsigned int __unused1;
-	compat_time_t msg_stime;
-	unsigned int __unused2;
-	compat_time_t msg_rtime;
-	unsigned int __unused3;
-	compat_time_t msg_ctime;
+	unsigned int msg_stime_high;
+	unsigned int msg_stime;
+	unsigned int msg_rtime_high;
+	unsigned int msg_rtime;
+	unsigned int msg_ctime_high;
+	unsigned int msg_ctime;
 	compat_ulong_t msg_cbytes;
 	compat_ulong_t msg_qnum;
 	compat_ulong_t msg_qbytes;
@@ -201,12 +190,12 @@ struct compat_msqid64_ds {
 
 struct compat_shmid64_ds {
 	struct compat_ipc64_perm shm_perm;
-	unsigned int __unused1;
-	compat_time_t shm_atime;
-	unsigned int __unused2;
-	compat_time_t shm_dtime;
-	unsigned int __unused3;
-	compat_time_t shm_ctime;
+	unsigned int shm_atime_high;
+	unsigned int shm_atime;
+	unsigned int shm_dtime_high;
+	unsigned int shm_dtime;
+	unsigned int shm_ctime_high;
+	unsigned int shm_ctime;
 	unsigned int __unused4;
 	compat_size_t shm_segsz;
 	compat_pid_t shm_cpid;
diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h
index b2dabd06659d..9ad759a792a8 100644
--- a/arch/powerpc/include/asm/ftrace.h
+++ b/arch/powerpc/include/asm/ftrace.h
@@ -48,9 +48,6 @@
 #else /* !__ASSEMBLY__ */
 extern void _mcount(void);
 
-#ifdef CONFIG_DYNAMIC_FTRACE
-# define FTRACE_ADDR ((unsigned long)ftrace_caller)
-# define FTRACE_REGS_ADDR FTRACE_ADDR
 static inline unsigned long ftrace_call_adjust(unsigned long addr)
 {
        /* reloction of mcount call site is the same as the address */
@@ -60,15 +57,15 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)
 struct dyn_arch_ftrace {
 	struct module *mod;
 };
-#endif /*  CONFIG_DYNAMIC_FTRACE */
 #endif /* __ASSEMBLY__ */
 
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 #define ARCH_SUPPORTS_FTRACE_OPS 1
 #endif
-#endif
+#endif /* CONFIG_FUNCTION_TRACER */
 
-#if defined(CONFIG_FTRACE_SYSCALLS) && !defined(__ASSEMBLY__)
+#ifndef __ASSEMBLY__
+#if defined(CONFIG_FTRACE_SYSCALLS)
 /*
  * Some syscall entry functions on powerpc start with "ppc_" (fork and clone,
  * for instance) or ppc32_/ppc64_. We should also match the sys_ variant with
@@ -94,7 +91,25 @@ static inline bool arch_syscall_match_sym_name(const char *sym, const char *name
 		(!strncmp(sym, "ppc32_", 6) && !strcmp(sym + 6, name + 4)) ||
 		(!strncmp(sym, "ppc64_", 6) && !strcmp(sym + 6, name + 4));
 }
-#endif
-#endif /* CONFIG_FTRACE_SYSCALLS && !__ASSEMBLY__ */
+#endif /* PPC64_ELF_ABI_v1 */
+#endif /* CONFIG_FTRACE_SYSCALLS */
+
+#ifdef CONFIG_PPC64
+#include <asm/paca.h>
+
+static inline void this_cpu_disable_ftrace(void)
+{
+	get_paca()->ftrace_enabled = 0;
+}
+
+static inline void this_cpu_enable_ftrace(void)
+{
+	get_paca()->ftrace_enabled = 1;
+}
+#else /* CONFIG_PPC64 */
+static inline void this_cpu_disable_ftrace(void) { }
+static inline void this_cpu_enable_ftrace(void) { }
+#endif /* CONFIG_PPC64 */
+#endif /* !__ASSEMBLY__ */
 
 #endif /* _ASM_POWERPC_FTRACE */
diff --git a/arch/powerpc/include/asm/hardirq.h b/arch/powerpc/include/asm/hardirq.h
index 5986d473722b..383f628acbf8 100644
--- a/arch/powerpc/include/asm/hardirq.h
+++ b/arch/powerpc/include/asm/hardirq.h
@@ -25,15 +25,8 @@ typedef struct {
 DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
 
 #define __ARCH_IRQ_STAT
-
-#define local_softirq_pending()	__this_cpu_read(irq_stat.__softirq_pending)
-
-#define __ARCH_SET_SOFTIRQ_PENDING
 #define __ARCH_IRQ_EXIT_IRQS_DISABLED
 
-#define set_softirq_pending(x) __this_cpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x) __this_cpu_or(irq_stat.__softirq_pending, (x))
-
 static inline void ack_bad_irq(unsigned int irq)
 {
 	printk(KERN_CRIT "unexpected IRQ trap at vector %02x\n", irq);
diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h
index 78540c074d70..96444bc08034 100644
--- a/arch/powerpc/include/asm/hugetlb.h
+++ b/arch/powerpc/include/asm/hugetlb.h
@@ -202,7 +202,7 @@ static inline void flush_hugetlb_page(struct vm_area_struct *vma,
 static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr,
 				    unsigned pdshift)
 {
-	return 0;
+	return NULL;
 }
 #endif /* CONFIG_HUGETLB_PAGE */
 
diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h
index 2e2dddab5d65..662c8347d699 100644
--- a/arch/powerpc/include/asm/hvcall.h
+++ b/arch/powerpc/include/asm/hvcall.h
@@ -279,6 +279,7 @@
 #define H_GET_MPP_X		0x314
 #define H_SET_MODE		0x31C
 #define H_CLEAR_HPT		0x358
+#define H_REQUEST_VMC		0x360
 #define H_RESIZE_HPT_PREPARE	0x36C
 #define H_RESIZE_HPT_COMMIT	0x370
 #define H_REGISTER_PROC_TBL	0x37C
diff --git a/arch/powerpc/include/asm/io.h b/arch/powerpc/include/asm/io.h
index af074923d598..e0331e754568 100644
--- a/arch/powerpc/include/asm/io.h
+++ b/arch/powerpc/include/asm/io.h
@@ -367,6 +367,11 @@ static inline void __raw_writeq(unsigned long v, volatile void __iomem *addr)
 	*(volatile unsigned long __force *)PCI_FIX_ADDR(addr) = v;
 }
 
+static inline void __raw_writeq_be(unsigned long v, volatile void __iomem *addr)
+{
+	__raw_writeq((__force unsigned long)cpu_to_be64(v), addr);
+}
+
 /*
  * Real mode versions of the above. Those instructions are only supposed
  * to be used in hypervisor real mode as per the architecture spec.
@@ -395,6 +400,11 @@ static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr)
 		: : "r" (val), "r" (paddr) : "memory");
 }
 
+static inline void __raw_rm_writeq_be(u64 val, volatile void __iomem *paddr)
+{
+	__raw_rm_writeq((__force u64)cpu_to_be64(val), paddr);
+}
+
 static inline u8 __raw_rm_readb(volatile void __iomem *paddr)
 {
 	u8 ret;
diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h
index e7377b73cfec..20d3d5a87296 100644
--- a/arch/powerpc/include/asm/kvm_book3s.h
+++ b/arch/powerpc/include/asm/kvm_book3s.h
@@ -104,6 +104,7 @@ struct kvmppc_vcore {
 	ulong vtb;		/* virtual timebase */
 	ulong conferring_threads;
 	unsigned int halt_poll_ns;
+	atomic_t online_count;
 };
 
 struct kvmppc_vcpu_book3s {
@@ -274,12 +275,12 @@ static inline struct kvmppc_vcpu_book3s *to_book3s(struct kvm_vcpu *vcpu)
 
 static inline void kvmppc_set_gpr(struct kvm_vcpu *vcpu, int num, ulong val)
 {
-	vcpu->arch.gpr[num] = val;
+	vcpu->arch.regs.gpr[num] = val;
 }
 
 static inline ulong kvmppc_get_gpr(struct kvm_vcpu *vcpu, int num)
 {
-	return vcpu->arch.gpr[num];
+	return vcpu->arch.regs.gpr[num];
 }
 
 static inline void kvmppc_set_cr(struct kvm_vcpu *vcpu, u32 val)
@@ -294,42 +295,42 @@ static inline u32 kvmppc_get_cr(struct kvm_vcpu *vcpu)
 
 static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, ulong val)
 {
-	vcpu->arch.xer = val;
+	vcpu->arch.regs.xer = val;
 }
 
 static inline ulong kvmppc_get_xer(struct kvm_vcpu *vcpu)
 {
-	return vcpu->arch.xer;
+	return vcpu->arch.regs.xer;
 }
 
 static inline void kvmppc_set_ctr(struct kvm_vcpu *vcpu, ulong val)
 {
-	vcpu->arch.ctr = val;
+	vcpu->arch.regs.ctr = val;
 }
 
 static inline ulong kvmppc_get_ctr(struct kvm_vcpu *vcpu)
 {
-	return vcpu->arch.ctr;
+	return vcpu->arch.regs.ctr;
 }
 
 static inline void kvmppc_set_lr(struct kvm_vcpu *vcpu, ulong val)
 {
-	vcpu->arch.lr = val;
+	vcpu->arch.regs.link = val;
 }
 
 static inline ulong kvmppc_get_lr(struct kvm_vcpu *vcpu)
 {
-	return vcpu->arch.lr;
+	return vcpu->arch.regs.link;
 }
 
 static inline void kvmppc_set_pc(struct kvm_vcpu *vcpu, ulong val)
 {
-	vcpu->arch.pc = val;
+	vcpu->arch.regs.nip = val;
 }
 
 static inline ulong kvmppc_get_pc(struct kvm_vcpu *vcpu)
 {
-	return vcpu->arch.pc;
+	return vcpu->arch.regs.nip;
 }
 
 static inline u64 kvmppc_get_msr(struct kvm_vcpu *vcpu);
diff --git a/arch/powerpc/include/asm/kvm_book3s_64.h b/arch/powerpc/include/asm/kvm_book3s_64.h
index c424e44f4c00..dc435a5af7d6 100644
--- a/arch/powerpc/include/asm/kvm_book3s_64.h
+++ b/arch/powerpc/include/asm/kvm_book3s_64.h
@@ -483,15 +483,15 @@ static inline u64 sanitize_msr(u64 msr)
 static inline void copy_from_checkpoint(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.cr  = vcpu->arch.cr_tm;
-	vcpu->arch.xer = vcpu->arch.xer_tm;
-	vcpu->arch.lr  = vcpu->arch.lr_tm;
-	vcpu->arch.ctr = vcpu->arch.ctr_tm;
+	vcpu->arch.regs.xer = vcpu->arch.xer_tm;
+	vcpu->arch.regs.link  = vcpu->arch.lr_tm;
+	vcpu->arch.regs.ctr = vcpu->arch.ctr_tm;
 	vcpu->arch.amr = vcpu->arch.amr_tm;
 	vcpu->arch.ppr = vcpu->arch.ppr_tm;
 	vcpu->arch.dscr = vcpu->arch.dscr_tm;
 	vcpu->arch.tar = vcpu->arch.tar_tm;
-	memcpy(vcpu->arch.gpr, vcpu->arch.gpr_tm,
-	       sizeof(vcpu->arch.gpr));
+	memcpy(vcpu->arch.regs.gpr, vcpu->arch.gpr_tm,
+	       sizeof(vcpu->arch.regs.gpr));
 	vcpu->arch.fp  = vcpu->arch.fp_tm;
 	vcpu->arch.vr  = vcpu->arch.vr_tm;
 	vcpu->arch.vrsave = vcpu->arch.vrsave_tm;
@@ -500,15 +500,15 @@ static inline void copy_from_checkpoint(struct kvm_vcpu *vcpu)
 static inline void copy_to_checkpoint(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.cr_tm  = vcpu->arch.cr;
-	vcpu->arch.xer_tm = vcpu->arch.xer;
-	vcpu->arch.lr_tm  = vcpu->arch.lr;
-	vcpu->arch.ctr_tm = vcpu->arch.ctr;
+	vcpu->arch.xer_tm = vcpu->arch.regs.xer;
+	vcpu->arch.lr_tm  = vcpu->arch.regs.link;
+	vcpu->arch.ctr_tm = vcpu->arch.regs.ctr;
 	vcpu->arch.amr_tm = vcpu->arch.amr;
 	vcpu->arch.ppr_tm = vcpu->arch.ppr;
 	vcpu->arch.dscr_tm = vcpu->arch.dscr;
 	vcpu->arch.tar_tm = vcpu->arch.tar;
-	memcpy(vcpu->arch.gpr_tm, vcpu->arch.gpr,
-	       sizeof(vcpu->arch.gpr));
+	memcpy(vcpu->arch.gpr_tm, vcpu->arch.regs.gpr,
+	       sizeof(vcpu->arch.regs.gpr));
 	vcpu->arch.fp_tm  = vcpu->arch.fp;
 	vcpu->arch.vr_tm  = vcpu->arch.vr;
 	vcpu->arch.vrsave_tm = vcpu->arch.vrsave;
diff --git a/arch/powerpc/include/asm/kvm_booke.h b/arch/powerpc/include/asm/kvm_booke.h
index bc6e29e4dfd4..d513e3ed1c65 100644
--- a/arch/powerpc/include/asm/kvm_booke.h
+++ b/arch/powerpc/include/asm/kvm_booke.h
@@ -36,12 +36,12 @@
 
 static inline void kvmppc_set_gpr(struct kvm_vcpu *vcpu, int num, ulong val)
 {
-	vcpu->arch.gpr[num] = val;
+	vcpu->arch.regs.gpr[num] = val;
 }
 
 static inline ulong kvmppc_get_gpr(struct kvm_vcpu *vcpu, int num)
 {
-	return vcpu->arch.gpr[num];
+	return vcpu->arch.regs.gpr[num];
 }
 
 static inline void kvmppc_set_cr(struct kvm_vcpu *vcpu, u32 val)
@@ -56,12 +56,12 @@ static inline u32 kvmppc_get_cr(struct kvm_vcpu *vcpu)
 
 static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, ulong val)
 {
-	vcpu->arch.xer = val;
+	vcpu->arch.regs.xer = val;
 }
 
 static inline ulong kvmppc_get_xer(struct kvm_vcpu *vcpu)
 {
-	return vcpu->arch.xer;
+	return vcpu->arch.regs.xer;
 }
 
 static inline bool kvmppc_need_byteswap(struct kvm_vcpu *vcpu)
@@ -72,32 +72,32 @@ static inline bool kvmppc_need_byteswap(struct kvm_vcpu *vcpu)
 
 static inline void kvmppc_set_ctr(struct kvm_vcpu *vcpu, ulong val)
 {
-	vcpu->arch.ctr = val;
+	vcpu->arch.regs.ctr = val;
 }
 
 static inline ulong kvmppc_get_ctr(struct kvm_vcpu *vcpu)
 {
-	return vcpu->arch.ctr;
+	return vcpu->arch.regs.ctr;
 }
 
 static inline void kvmppc_set_lr(struct kvm_vcpu *vcpu, ulong val)
 {
-	vcpu->arch.lr = val;
+	vcpu->arch.regs.link = val;
 }
 
 static inline ulong kvmppc_get_lr(struct kvm_vcpu *vcpu)
 {
-	return vcpu->arch.lr;
+	return vcpu->arch.regs.link;
 }
 
 static inline void kvmppc_set_pc(struct kvm_vcpu *vcpu, ulong val)
 {
-	vcpu->arch.pc = val;
+	vcpu->arch.regs.nip = val;
 }
 
 static inline ulong kvmppc_get_pc(struct kvm_vcpu *vcpu)
 {
-	return vcpu->arch.pc;
+	return vcpu->arch.regs.nip;
 }
 
 static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu)
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index 17498e9a26e4..8dc5e439b387 100644
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -269,7 +269,6 @@ struct kvm_arch {
 	unsigned long host_lpcr;
 	unsigned long sdr1;
 	unsigned long host_sdr1;
-	int tlbie_lock;
 	unsigned long lpcr;
 	unsigned long vrma_slb_v;
 	int mmu_ready;
@@ -454,6 +453,12 @@ struct mmio_hpte_cache {
 #define KVMPPC_VSX_COPY_WORD		1
 #define KVMPPC_VSX_COPY_DWORD		2
 #define KVMPPC_VSX_COPY_DWORD_LOAD_DUMP	3
+#define KVMPPC_VSX_COPY_WORD_LOAD_DUMP	4
+
+#define KVMPPC_VMX_COPY_BYTE		8
+#define KVMPPC_VMX_COPY_HWORD		9
+#define KVMPPC_VMX_COPY_WORD		10
+#define KVMPPC_VMX_COPY_DWORD		11
 
 struct openpic;
 
@@ -486,7 +491,7 @@ struct kvm_vcpu_arch {
 	struct kvmppc_book3s_shadow_vcpu *shadow_vcpu;
 #endif
 
-	ulong gpr[32];
+	struct pt_regs regs;
 
 	struct thread_fp_state fp;
 
@@ -521,14 +526,10 @@ struct kvm_vcpu_arch {
 	u32 qpr[32];
 #endif
 
-	ulong pc;
-	ulong ctr;
-	ulong lr;
 #ifdef CONFIG_PPC_BOOK3S
 	ulong tar;
 #endif
 
-	ulong xer;
 	u32 cr;
 
 #ifdef CONFIG_PPC_BOOK3S
@@ -681,16 +682,17 @@ struct kvm_vcpu_arch {
 	 * Number of simulations for vsx.
 	 * If we use 2*8bytes to simulate 1*16bytes,
 	 * then the number should be 2 and
-	 * mmio_vsx_copy_type=KVMPPC_VSX_COPY_DWORD.
+	 * mmio_copy_type=KVMPPC_VSX_COPY_DWORD.
 	 * If we use 4*4bytes to simulate 1*16bytes,
 	 * the number should be 4 and
 	 * mmio_vsx_copy_type=KVMPPC_VSX_COPY_WORD.
 	 */
 	u8 mmio_vsx_copy_nums;
 	u8 mmio_vsx_offset;
-	u8 mmio_vsx_copy_type;
 	u8 mmio_vsx_tx_sx_enabled;
 	u8 mmio_vmx_copy_nums;
+	u8 mmio_vmx_offset;
+	u8 mmio_copy_type;
 	u8 osi_needed;
 	u8 osi_enabled;
 	u8 papr_enabled;
@@ -772,6 +774,8 @@ struct kvm_vcpu_arch {
 	u64 busy_preempt;
 
 	u32 emul_inst;
+
+	u32 online;
 #endif
 
 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h
index abe7032cdb54..e991821dd7fa 100644
--- a/arch/powerpc/include/asm/kvm_ppc.h
+++ b/arch/powerpc/include/asm/kvm_ppc.h
@@ -52,7 +52,7 @@ enum emulation_result {
 	EMULATE_EXIT_USER,    /* emulation requires exit to user-space */
 };
 
-enum instruction_type {
+enum instruction_fetch_type {
 	INST_GENERIC,
 	INST_SC,		/* system call */
 };
@@ -81,10 +81,10 @@ extern int kvmppc_handle_loads(struct kvm_run *run, struct kvm_vcpu *vcpu,
 extern int kvmppc_handle_vsx_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
 				unsigned int rt, unsigned int bytes,
 			int is_default_endian, int mmio_sign_extend);
-extern int kvmppc_handle_load128_by2x64(struct kvm_run *run,
-		struct kvm_vcpu *vcpu, unsigned int rt, int is_default_endian);
-extern int kvmppc_handle_store128_by2x64(struct kvm_run *run,
-		struct kvm_vcpu *vcpu, unsigned int rs, int is_default_endian);
+extern int kvmppc_handle_vmx_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
+		unsigned int rt, unsigned int bytes, int is_default_endian);
+extern int kvmppc_handle_vmx_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
+		unsigned int rs, unsigned int bytes, int is_default_endian);
 extern int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
 			       u64 val, unsigned int bytes,
 			       int is_default_endian);
@@ -93,7 +93,7 @@ extern int kvmppc_handle_vsx_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
 				int is_default_endian);
 
 extern int kvmppc_load_last_inst(struct kvm_vcpu *vcpu,
-				 enum instruction_type type, u32 *inst);
+				 enum instruction_fetch_type type, u32 *inst);
 
 extern int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr,
 		     bool data);
@@ -265,6 +265,8 @@ union kvmppc_one_reg {
 	vector128 vval;
 	u64	vsxval[2];
 	u32	vsx32val[4];
+	u16	vsx16val[8];
+	u8	vsx8val[16];
 	struct {
 		u64	addr;
 		u64	length;
@@ -324,13 +326,14 @@ struct kvmppc_ops {
 	int (*get_rmmu_info)(struct kvm *kvm, struct kvm_ppc_rmmu_info *info);
 	int (*set_smt_mode)(struct kvm *kvm, unsigned long mode,
 			    unsigned long flags);
+	void (*giveup_ext)(struct kvm_vcpu *vcpu, ulong msr);
 };
 
 extern struct kvmppc_ops *kvmppc_hv_ops;
 extern struct kvmppc_ops *kvmppc_pr_ops;
 
 static inline int kvmppc_get_last_inst(struct kvm_vcpu *vcpu,
-					enum instruction_type type, u32 *inst)
+				enum instruction_fetch_type type, u32 *inst)
 {
 	int ret = EMULATE_DONE;
 	u32 fetched_inst;
diff --git a/arch/powerpc/include/asm/mmu-book3e.h b/arch/powerpc/include/asm/mmu-book3e.h
index cda94a0f5146..e20072972e35 100644
--- a/arch/powerpc/include/asm/mmu-book3e.h
+++ b/arch/powerpc/include/asm/mmu-book3e.h
@@ -230,10 +230,6 @@ typedef struct {
 	unsigned int	id;
 	unsigned int	active;
 	unsigned long	vdso_base;
-#ifdef CONFIG_PPC_64K_PAGES
-	/* for 4K PTE fragment support */
-	void *pte_frag;
-#endif
 } mm_context_t;
 
 /* Page size definitions, common between 32 and 64-bit
@@ -275,8 +271,6 @@ static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize)
  */
 #if defined(CONFIG_PPC_4K_PAGES)
 #define mmu_virtual_psize	MMU_PAGE_4K
-#elif defined(CONFIG_PPC_64K_PAGES)
-#define mmu_virtual_psize	MMU_PAGE_64K
 #else
 #error Unsupported page size
 #endif
diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
index 1835ca1505d6..896efa559996 100644
--- a/arch/powerpc/include/asm/mmu_context.h
+++ b/arch/powerpc/include/asm/mmu_context.h
@@ -250,11 +250,6 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
 #define thread_pkey_regs_restore(new_thread, old_thread)
 #define thread_pkey_regs_init(thread)
 
-static inline int vma_pkey(struct vm_area_struct *vma)
-{
-	return 0;
-}
-
 static inline u64 pte_to_hpte_pkey_bits(u64 pteflags)
 {
 	return 0x0UL;
diff --git a/arch/powerpc/include/asm/module.h b/arch/powerpc/include/asm/module.h
index 4f6573934792..9cf925cf948f 100644
--- a/arch/powerpc/include/asm/module.h
+++ b/arch/powerpc/include/asm/module.h
@@ -14,7 +14,7 @@
 #include <asm-generic/module.h>
 
 
-#ifdef CC_USING_MPROFILE_KERNEL
+#ifdef CONFIG_MPROFILE_KERNEL
 #define MODULE_ARCH_VERMAGIC_FTRACE	"mprofile-kernel "
 #else
 #define MODULE_ARCH_VERMAGIC_FTRACE	""
@@ -53,6 +53,9 @@ struct mod_arch_specific {
 #ifdef CONFIG_DYNAMIC_FTRACE
 	unsigned long toc;
 	unsigned long tramp;
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+	unsigned long tramp_regs;
+#endif
 #endif
 
 	/* For module function descriptor dereference */
diff --git a/arch/powerpc/include/asm/mpc52xx.h b/arch/powerpc/include/asm/mpc52xx.h
index e94cede14522..ce1e0aabaa64 100644
--- a/arch/powerpc/include/asm/mpc52xx.h
+++ b/arch/powerpc/include/asm/mpc52xx.h
@@ -350,14 +350,14 @@ extern struct mpc52xx_suspend mpc52xx_suspend;
 extern int __init mpc52xx_pm_init(void);
 extern int mpc52xx_set_wakeup_gpio(u8 pin, u8 level);
 
-#ifdef CONFIG_PPC_LITE5200
-extern int __init lite5200_pm_init(void);
-
 /* lite5200 calls mpc5200 suspend functions, so here they are */
 extern int mpc52xx_pm_prepare(void);
 extern int mpc52xx_pm_enter(suspend_state_t);
 extern void mpc52xx_pm_finish(void);
 extern char saved_sram[0x4000]; /* reuse buffer from mpc52xx suspend */
+
+#ifdef CONFIG_PPC_LITE5200
+int __init lite5200_pm_init(void);
 #endif
 #endif /* CONFIG_PM */
 
diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h
index 03bbd1149530..987a658b18e1 100644
--- a/arch/powerpc/include/asm/nohash/32/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/32/pgtable.h
@@ -133,7 +133,7 @@ extern int icache_44x_need_flush;
 #ifndef __ASSEMBLY__
 
 #define pte_clear(mm, addr, ptep) \
-	do { pte_update(ptep, ~_PAGE_HASHPTE, 0); } while (0)
+	do { pte_update(ptep, ~0, 0); } while (0)
 
 #define pmd_none(pmd)		(!pmd_val(pmd))
 #define	pmd_bad(pmd)		(pmd_val(pmd) & _PMD_BAD)
@@ -145,21 +145,6 @@ static inline void pmd_clear(pmd_t *pmdp)
 
 
 
-/*
- * When flushing the tlb entry for a page, we also need to flush the hash
- * table entry.  flush_hash_pages is assembler (for speed) in hashtable.S.
- */
-extern int flush_hash_pages(unsigned context, unsigned long va,
-			    unsigned long pmdval, int count);
-
-/* Add an HPTE to the hash table */
-extern void add_hash_page(unsigned context, unsigned long va,
-			  unsigned long pmdval);
-
-/* Flush an entry from the TLB/hash table */
-extern void flush_hash_entry(struct mm_struct *mm, pte_t *ptep,
-			     unsigned long address);
-
 /*
  * PTE updates. This function is called whenever an existing
  * valid PTE is updated. This does -not- include set_pte_at()
@@ -246,12 +231,6 @@ static inline int __ptep_test_and_clear_young(unsigned int context, unsigned lon
 {
 	unsigned long old;
 	old = pte_update(ptep, _PAGE_ACCESSED, 0);
-#if _PAGE_HASHPTE != 0
-	if (old & _PAGE_HASHPTE) {
-		unsigned long ptephys = __pa(ptep) & PAGE_MASK;
-		flush_hash_pages(context, addr, ptephys, 1);
-	}
-#endif
 	return (old & _PAGE_ACCESSED) != 0;
 }
 #define ptep_test_and_clear_young(__vma, __addr, __ptep) \
@@ -261,7 +240,7 @@ static inline int __ptep_test_and_clear_young(unsigned int context, unsigned lon
 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
 				       pte_t *ptep)
 {
-	return __pte(pte_update(ptep, ~_PAGE_HASHPTE, 0));
+	return __pte(pte_update(ptep, ~0, 0));
 }
 
 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
@@ -288,8 +267,13 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm,
 	pte_update(ptep, clr, set);
 }
 
+static inline int pte_young(pte_t pte)
+{
+	return pte_val(pte) & _PAGE_ACCESSED;
+}
+
 #define __HAVE_ARCH_PTE_SAME
-#define pte_same(A,B)	(((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0)
+#define pte_same(A,B)	((pte_val(A) ^ pte_val(B)) == 0)
 
 /*
  * Note that on Book E processors, the pmd contains the kernel virtual
@@ -330,7 +314,7 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm,
 /*
  * Encode and decode a swap entry.
  * Note that the bits we use in a PTE for representing a swap entry
- * must not include the _PAGE_PRESENT bit or the _PAGE_HASHPTE bit (if used).
+ * must not include the _PAGE_PRESENT bit.
  *   -- paulus
  */
 #define __swp_type(entry)		((entry).val & 0x1f)
diff --git a/arch/powerpc/include/asm/nohash/32/pte-40x.h b/arch/powerpc/include/asm/nohash/32/pte-40x.h
index 124f9ac23a1e..bb4b3a4b92a0 100644
--- a/arch/powerpc/include/asm/nohash/32/pte-40x.h
+++ b/arch/powerpc/include/asm/nohash/32/pte-40x.h
@@ -52,12 +52,9 @@
 
 #define _PMD_PRESENT	0x400	/* PMD points to page of PTEs */
 #define _PMD_BAD	0x802
-#define _PMD_SIZE	0x0e0	/* size field, != 0 for large-page PMD entry */
 #define _PMD_SIZE_4M	0x0c0
 #define _PMD_SIZE_16M	0x0e0
 
-#define PMD_PAGE_SIZE(pmdval)	(1024 << (((pmdval) & _PMD_SIZE) >> 4))
-
 /* Until my rework is finished, 40x still needs atomic PTE updates */
 #define PTE_ATOMIC_UPDATES	1
 
diff --git a/arch/powerpc/include/asm/nohash/64/pgalloc.h b/arch/powerpc/include/asm/nohash/64/pgalloc.h
index 9721c7867b9c..21624ff1f065 100644
--- a/arch/powerpc/include/asm/nohash/64/pgalloc.h
+++ b/arch/powerpc/include/asm/nohash/64/pgalloc.h
@@ -52,8 +52,6 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 	kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd);
 }
 
-#ifndef CONFIG_PPC_64K_PAGES
-
 #define pgd_populate(MM, PGD, PUD)	pgd_set(PGD, (unsigned long)PUD)
 
 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
@@ -86,6 +84,18 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
 
 #define pmd_pgtable(pmd) pmd_page(pmd)
 
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+	return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
+			pgtable_gfp_flags(mm, GFP_KERNEL));
+}
+
+static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
+{
+	kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd);
+}
+
+
 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
 					  unsigned long address)
 {
@@ -120,84 +130,47 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
 	__free_page(ptepage);
 }
 
-extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
-#ifdef CONFIG_SMP
-extern void __tlb_remove_table(void *_table);
-#endif
-static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
-				  unsigned long address)
+static inline void pgtable_free(void *table, int shift)
 {
-	tlb_flush_pgtable(tlb, address);
-	pgtable_free_tlb(tlb, page_address(table), 0);
+	if (!shift) {
+		pgtable_page_dtor(table);
+		free_page((unsigned long)table);
+	} else {
+		BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
+		kmem_cache_free(PGT_CACHE(shift), table);
+	}
 }
 
-#else /* if CONFIG_PPC_64K_PAGES */
-
-extern pte_t *pte_fragment_alloc(struct mm_struct *, unsigned long, int);
-extern void pte_fragment_free(unsigned long *, int);
-extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
 #ifdef CONFIG_SMP
-extern void __tlb_remove_table(void *_table);
-#endif
-
-#define pud_populate(mm, pud, pmd)	pud_set(pud, (unsigned long)pmd)
-
-static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
-				       pte_t *pte)
+static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
 {
-	pmd_set(pmd, (unsigned long)pte);
-}
+	unsigned long pgf = (unsigned long)table;
 
-static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
-				pgtable_t pte_page)
-{
-	pmd_set(pmd, (unsigned long)pte_page);
+	BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
+	pgf |= shift;
+	tlb_remove_table(tlb, (void *)pgf);
 }
 
-static inline pgtable_t pmd_pgtable(pmd_t pmd)
+static inline void __tlb_remove_table(void *_table)
 {
-	return (pgtable_t)(pmd_val(pmd) & ~PMD_MASKED_BITS);
-}
+	void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
+	unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
 
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
-					  unsigned long address)
-{
-	return (pte_t *)pte_fragment_alloc(mm, address, 1);
+	pgtable_free(table, shift);
 }
 
-static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
-					unsigned long address)
-{
-	return (pgtable_t)pte_fragment_alloc(mm, address, 0);
-}
-
-static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
-{
-	pte_fragment_free((unsigned long *)pte, 1);
-}
-
-static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
+#else
+static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
 {
-	pte_fragment_free((unsigned long *)ptepage, 0);
+	pgtable_free(table, shift);
 }
+#endif
 
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
 				  unsigned long address)
 {
 	tlb_flush_pgtable(tlb, address);
-	pgtable_free_tlb(tlb, table, 0);
-}
-#endif /* CONFIG_PPC_64K_PAGES */
-
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
-{
-	return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
-			pgtable_gfp_flags(mm, GFP_KERNEL));
-}
-
-static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
-{
-	kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd);
+	pgtable_free_tlb(tlb, page_address(table), 0);
 }
 
 #define __pmd_free_tlb(tlb, pmd, addr)		      \
diff --git a/arch/powerpc/include/asm/nohash/64/pgtable-64k.h b/arch/powerpc/include/asm/nohash/64/pgtable-64k.h
deleted file mode 100644
index 7210c2818e41..000000000000
--- a/arch/powerpc/include/asm/nohash/64/pgtable-64k.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_64K_H
-#define _ASM_POWERPC_NOHASH_64_PGTABLE_64K_H
-
-#define __ARCH_USE_5LEVEL_HACK
-#include <asm-generic/pgtable-nopud.h>
-
-
-#define PTE_INDEX_SIZE  8
-#define PMD_INDEX_SIZE  10
-#define PUD_INDEX_SIZE	0
-#define PGD_INDEX_SIZE  12
-
-/*
- * we support 32 fragments per PTE page of 64K size
- */
-#define PTE_FRAG_NR	32
-/*
- * We use a 2K PTE page fragment and another 2K for storing
- * real_pte_t hash index
- */
-#define PTE_FRAG_SIZE_SHIFT  11
-#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
-
-#ifndef __ASSEMBLY__
-#define PTE_TABLE_SIZE	PTE_FRAG_SIZE
-#define PMD_TABLE_SIZE	(sizeof(pmd_t) << PMD_INDEX_SIZE)
-#define PUD_TABLE_SIZE	(0)
-#define PGD_TABLE_SIZE	(sizeof(pgd_t) << PGD_INDEX_SIZE)
-#endif	/* __ASSEMBLY__ */
-
-#define PTRS_PER_PTE	(1 << PTE_INDEX_SIZE)
-#define PTRS_PER_PMD	(1 << PMD_INDEX_SIZE)
-#define PTRS_PER_PGD	(1 << PGD_INDEX_SIZE)
-
-/* PMD_SHIFT determines what a second-level page table entry can map */
-#define PMD_SHIFT	(PAGE_SHIFT + PTE_INDEX_SIZE)
-#define PMD_SIZE	(1UL << PMD_SHIFT)
-#define PMD_MASK	(~(PMD_SIZE-1))
-
-/* PGDIR_SHIFT determines what a third-level page table entry can map */
-#define PGDIR_SHIFT	(PMD_SHIFT + PMD_INDEX_SIZE)
-#define PGDIR_SIZE	(1UL << PGDIR_SHIFT)
-#define PGDIR_MASK	(~(PGDIR_SIZE-1))
-
-/*
- * Bits to mask out from a PMD to get to the PTE page
- * PMDs point to PTE table fragments which are PTE_FRAG_SIZE aligned.
- */
-#define PMD_MASKED_BITS		(PTE_FRAG_SIZE - 1)
-/* Bits to mask out from a PGD/PUD to get to the PMD page */
-#define PUD_MASKED_BITS		0x1ff
-
-#define pgd_pte(pgd)	(pud_pte(((pud_t){ pgd })))
-#define pte_pgd(pte)	((pgd_t)pte_pud(pte))
-
-#endif /* _ASM_POWERPC_NOHASH_64_PGTABLE_64K_H */
diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h
index 5c5f75d005ad..de78eda5f841 100644
--- a/arch/powerpc/include/asm/nohash/64/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/64/pgtable.h
@@ -6,13 +6,13 @@
  * the ppc64 hashed page table.
  */
 
-#ifdef CONFIG_PPC_64K_PAGES
-#include <asm/nohash/64/pgtable-64k.h>
-#else
 #include <asm/nohash/64/pgtable-4k.h>
-#endif
 #include <asm/barrier.h>
 
+#ifdef CONFIG_PPC_64K_PAGES
+#error "Page size not supported"
+#endif
+
 #define FIRST_USER_ADDRESS	0UL
 
 /*
@@ -173,8 +173,6 @@ static inline void pgd_set(pgd_t *pgdp, unsigned long val)
 /* to find an entry in a kernel page-table-directory */
 /* This now only contains the vmalloc pages */
 #define pgd_offset_k(address) pgd_offset(&init_mm, address)
-extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
-			    pte_t *ptep, unsigned long pte, int huge);
 
 /* Atomic PTE updates */
 static inline unsigned long pte_update(struct mm_struct *mm,
@@ -188,14 +186,12 @@ static inline unsigned long pte_update(struct mm_struct *mm,
 
 	__asm__ __volatile__(
 	"1:	ldarx	%0,0,%3		# pte_update\n\
-	andi.	%1,%0,%6\n\
-	bne-	1b \n\
 	andc	%1,%0,%4 \n\
-	or	%1,%1,%7\n\
+	or	%1,%1,%6\n\
 	stdcx.	%1,0,%3 \n\
 	bne-	1b"
 	: "=&r" (old), "=&r" (tmp), "=m" (*ptep)
-	: "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY), "r" (set)
+	: "r" (ptep), "r" (clr), "m" (*ptep), "r" (set)
 	: "cc" );
 #else
 	unsigned long old = pte_val(*ptep);
@@ -205,20 +201,20 @@ static inline unsigned long pte_update(struct mm_struct *mm,
 	if (!huge)
 		assert_pte_locked(mm, addr);
 
-#ifdef CONFIG_PPC_BOOK3S_64
-	if (old & _PAGE_HASHPTE)
-		hpte_need_flush(mm, addr, ptep, old, huge);
-#endif
-
 	return old;
 }
 
+static inline int pte_young(pte_t pte)
+{
+	return pte_val(pte) & _PAGE_ACCESSED;
+}
+
 static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
 					      unsigned long addr, pte_t *ptep)
 {
 	unsigned long old;
 
-	if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
+	if (pte_young(*ptep))
 		return 0;
 	old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
 	return (old & _PAGE_ACCESSED) != 0;
@@ -297,13 +293,11 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm,
 
 	__asm__ __volatile__(
 	"1:	ldarx	%0,0,%4\n\
-		andi.	%1,%0,%6\n\
-		bne-	1b \n\
 		or	%0,%3,%0\n\
 		stdcx.	%0,0,%4\n\
 		bne-	1b"
 	:"=&r" (old), "=&r" (tmp), "=m" (*ptep)
-	:"r" (bits), "r" (ptep), "m" (*ptep), "i" (_PAGE_BUSY)
+	:"r" (bits), "r" (ptep), "m" (*ptep)
 	:"cc");
 #else
 	unsigned long old = pte_val(*ptep);
@@ -312,7 +306,7 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm,
 }
 
 #define __HAVE_ARCH_PTE_SAME
-#define pte_same(A,B)	(((pte_val(A) ^ pte_val(B)) & ~_PAGE_HPTEFLAGS) == 0)
+#define pte_same(A,B)	((pte_val(A) ^ pte_val(B)) == 0)
 
 #define pte_ERROR(e) \
 	pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
@@ -324,11 +318,6 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm,
 /* Encode and de-code a swap entry */
 #define MAX_SWAPFILES_CHECK() do { \
 	BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS); \
-	/*							\
-	 * Don't have overlapping bits with _PAGE_HPTEFLAGS	\
-	 * We filter HPTEFLAGS on set_pte.			\
-	 */							\
-	BUILD_BUG_ON(_PAGE_HPTEFLAGS & (0x1f << _PAGE_BIT_SWAP_TYPE)); \
 	} while (0)
 /*
  * on pte we don't need handle RADIX_TREE_EXCEPTIONAL_SHIFT;
diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h
index c56de1e8026f..2160be2e4339 100644
--- a/arch/powerpc/include/asm/nohash/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/pgtable.h
@@ -17,7 +17,6 @@ static inline int pte_write(pte_t pte)
 }
 static inline int pte_read(pte_t pte)		{ return 1; }
 static inline int pte_dirty(pte_t pte)		{ return pte_val(pte) & _PAGE_DIRTY; }
-static inline int pte_young(pte_t pte)		{ return pte_val(pte) & _PAGE_ACCESSED; }
 static inline int pte_special(pte_t pte)	{ return pte_val(pte) & _PAGE_SPECIAL; }
 static inline int pte_none(pte_t pte)		{ return (pte_val(pte) & ~_PTE_NONE_MASK) == 0; }
 static inline pgprot_t pte_pgprot(pte_t pte)	{ return __pgprot(pte_val(pte) & PAGE_PROT_BITS); }
@@ -148,70 +147,33 @@ extern void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
 static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
 				pte_t *ptep, pte_t pte, int percpu)
 {
-#if defined(CONFIG_PPC_STD_MMU_32) && defined(CONFIG_SMP) && !defined(CONFIG_PTE_64BIT)
-	/* First case is 32-bit Hash MMU in SMP mode with 32-bit PTEs. We use the
-	 * helper pte_update() which does an atomic update. We need to do that
-	 * because a concurrent invalidation can clear _PAGE_HASHPTE. If it's a
-	 * per-CPU PTE such as a kmap_atomic, we do a simple update preserving
-	 * the hash bits instead (ie, same as the non-SMP case)
-	 */
-	if (percpu)
-		*ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE)
-			      | (pte_val(pte) & ~_PAGE_HASHPTE));
-	else
-		pte_update(ptep, ~_PAGE_HASHPTE, pte_val(pte));
-
-#elif defined(CONFIG_PPC32) && defined(CONFIG_PTE_64BIT)
 	/* Second case is 32-bit with 64-bit PTE.  In this case, we
 	 * can just store as long as we do the two halves in the right order
-	 * with a barrier in between. This is possible because we take care,
-	 * in the hash code, to pre-invalidate if the PTE was already hashed,
-	 * which synchronizes us with any concurrent invalidation.
-	 * In the percpu case, we also fallback to the simple update preserving
-	 * the hash bits
+	 * with a barrier in between.
+	 * In the percpu case, we also fallback to the simple update
 	 */
-	if (percpu) {
-		*ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE)
-			      | (pte_val(pte) & ~_PAGE_HASHPTE));
+	if (IS_ENABLED(CONFIG_PPC32) && IS_ENABLED(CONFIG_PTE_64BIT) && !percpu) {
+		__asm__ __volatile__("\
+			stw%U0%X0 %2,%0\n\
+			eieio\n\
+			stw%U0%X0 %L2,%1"
+		: "=m" (*ptep), "=m" (*((unsigned char *)ptep+4))
+		: "r" (pte) : "memory");
 		return;
 	}
-#if _PAGE_HASHPTE != 0
-	if (pte_val(*ptep) & _PAGE_HASHPTE)
-		flush_hash_entry(mm, ptep, addr);
-#endif
-	__asm__ __volatile__("\
-		stw%U0%X0 %2,%0\n\
-		eieio\n\
-		stw%U0%X0 %L2,%1"
-	: "=m" (*ptep), "=m" (*((unsigned char *)ptep+4))
-	: "r" (pte) : "memory");
-
-#elif defined(CONFIG_PPC_STD_MMU_32)
-	/* Third case is 32-bit hash table in UP mode, we need to preserve
-	 * the _PAGE_HASHPTE bit since we may not have invalidated the previous
-	 * translation in the hash yet (done in a subsequent flush_tlb_xxx())
-	 * and see we need to keep track that this PTE needs invalidating
-	 */
-	*ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE)
-		      | (pte_val(pte) & ~_PAGE_HASHPTE));
-
-#else
 	/* Anything else just stores the PTE normally. That covers all 64-bit
 	 * cases, and 32-bit non-hash with 32-bit PTEs.
 	 */
 	*ptep = pte;
 
-#ifdef CONFIG_PPC_BOOK3E_64
 	/*
 	 * With hardware tablewalk, a sync is needed to ensure that
 	 * subsequent accesses see the PTE we just wrote.  Unlike userspace
 	 * mappings, we can't tolerate spurious faults, so make sure
 	 * the new PTE will be seen the first time.
 	 */
-	if (is_kernel_addr(addr))
+	if (IS_ENABLED(CONFIG_PPC_BOOK3E_64) && is_kernel_addr(addr))
 		mb();
-#endif
-#endif
 }
 
 
diff --git a/arch/powerpc/include/asm/nohash/pte-book3e.h b/arch/powerpc/include/asm/nohash/pte-book3e.h
index ccee8eb509bb..12730b81cd98 100644
--- a/arch/powerpc/include/asm/nohash/pte-book3e.h
+++ b/arch/powerpc/include/asm/nohash/pte-book3e.h
@@ -57,14 +57,8 @@
 #define _PAGE_USER		(_PAGE_BAP_UR | _PAGE_BAP_SR) /* Can be read */
 #define _PAGE_PRIVILEGED	(_PAGE_BAP_SR)
 
-#define _PAGE_HASHPTE	0
-#define _PAGE_BUSY	0
-
 #define _PAGE_SPECIAL	_PAGE_SW0
 
-/* Flags to be preserved on PTE modifications */
-#define _PAGE_HPTEFLAGS	_PAGE_BUSY
-
 /* Base page size */
 #ifdef CONFIG_PPC_64K_PAGES
 #define _PAGE_PSIZE	_PAGE_PSIZE_64K
diff --git a/arch/powerpc/include/asm/opal-api.h b/arch/powerpc/include/asm/opal-api.h
index d886a5b7ff21..f34d173a2ebf 100644
--- a/arch/powerpc/include/asm/opal-api.h
+++ b/arch/powerpc/include/asm/opal-api.h
@@ -204,6 +204,7 @@
 #define OPAL_NPU_SPA_SETUP			159
 #define OPAL_NPU_SPA_CLEAR_CACHE		160
 #define OPAL_NPU_TL_SET				161
+#define OPAL_SENSOR_READ_U64			162
 #define OPAL_PCI_GET_PBCQ_TUNNEL_BAR		164
 #define OPAL_PCI_SET_PBCQ_TUNNEL_BAR		165
 #define OPAL_LAST				165
diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h
index 03e1a920491e..3960def0e39c 100644
--- a/arch/powerpc/include/asm/opal.h
+++ b/arch/powerpc/include/asm/opal.h
@@ -201,6 +201,7 @@ int64_t opal_get_param(uint64_t token, uint32_t param_id, uint64_t buffer,
 int64_t opal_set_param(uint64_t token, uint32_t param_id, uint64_t buffer,
 		uint64_t length);
 int64_t opal_sensor_read(uint32_t sensor_hndl, int token, __be32 *sensor_data);
+int64_t opal_sensor_read_u64(u32 sensor_hndl, int token, __be64 *sensor_data);
 int64_t opal_handle_hmi(void);
 int64_t opal_register_dump_region(uint32_t id, uint64_t start, uint64_t end);
 int64_t opal_unregister_dump_region(uint32_t id);
@@ -323,6 +324,7 @@ extern int opal_async_wait_response(uint64_t token, struct opal_msg *msg);
 extern int opal_async_wait_response_interruptible(uint64_t token,
 		struct opal_msg *msg);
 extern int opal_get_sensor_data(u32 sensor_hndl, u32 *sensor_data);
+extern int opal_get_sensor_data_u64(u32 sensor_hndl, u64 *sensor_data);
 
 struct rtc_time;
 extern unsigned long opal_get_boot_time(void);
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 3f109a3e3edb..81471bd08f66 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -222,6 +222,7 @@ struct paca_struct {
 	u8 hmi_event_available;		/* HMI event is available */
 	u8 hmi_p9_special_emu;		/* HMI P9 special emulation */
 #endif
+	u8 ftrace_enabled;		/* Hard disable ftrace */
 
 	/* Stuff for accurate time accounting */
 	struct cpu_accounting_data accounting;
diff --git a/arch/powerpc/include/asm/page.h b/arch/powerpc/include/asm/page.h
index dec9ce5ba8af..db7be0779d55 100644
--- a/arch/powerpc/include/asm/page.h
+++ b/arch/powerpc/include/asm/page.h
@@ -39,6 +39,7 @@
 
 #ifndef __ASSEMBLY__
 #ifdef CONFIG_HUGETLB_PAGE
+extern bool hugetlb_disabled;
 extern unsigned int HPAGE_SHIFT;
 #else
 #define HPAGE_SHIFT PAGE_SHIFT
diff --git a/arch/powerpc/include/asm/pci.h b/arch/powerpc/include/asm/pci.h
index 401c62aad5e4..2af9ded80540 100644
--- a/arch/powerpc/include/asm/pci.h
+++ b/arch/powerpc/include/asm/pci.h
@@ -92,24 +92,6 @@ extern int pci_mmap_legacy_page_range(struct pci_bus *bus,
 
 #define HAVE_PCI_LEGACY	1
 
-#ifdef CONFIG_PPC64
-
-/* The PCI address space does not equal the physical memory address
- * space (we have an IOMMU).  The IDE and SCSI device layers use
- * this boolean for bounce buffer decisions.
- */
-#define PCI_DMA_BUS_IS_PHYS	(0)
-
-#else /* 32-bit */
-
-/* The PCI address space does equal the physical memory
- * address space (no IOMMU).  The IDE and SCSI device layers use
- * this boolean for bounce buffer decisions.
- */
-#define PCI_DMA_BUS_IS_PHYS     (1)
-
-#endif /* CONFIG_PPC64 */
-
 extern void pcibios_claim_one_bus(struct pci_bus *b);
 
 extern void pcibios_finish_adding_to_bus(struct pci_bus *bus);
diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h
index 0409c80c32c0..5ba80cffb505 100644
--- a/arch/powerpc/include/asm/pkeys.h
+++ b/arch/powerpc/include/asm/pkeys.h
@@ -15,19 +15,6 @@ DECLARE_STATIC_KEY_TRUE(pkey_disabled);
 extern int pkeys_total; /* total pkeys as per device tree */
 extern u32 initial_allocation_mask; /* bits set for reserved keys */
 
-/*
- * Define these here temporarily so we're not dependent on patching linux/mm.h.
- * Once it's updated we can drop these.
- */
-#ifndef VM_PKEY_BIT0
-# define VM_PKEY_SHIFT	VM_HIGH_ARCH_BIT_0
-# define VM_PKEY_BIT0	VM_HIGH_ARCH_0
-# define VM_PKEY_BIT1	VM_HIGH_ARCH_1
-# define VM_PKEY_BIT2	VM_HIGH_ARCH_2
-# define VM_PKEY_BIT3	VM_HIGH_ARCH_3
-# define VM_PKEY_BIT4	VM_HIGH_ARCH_4
-#endif
-
 #define ARCH_VM_PKEY_FLAGS (VM_PKEY_BIT0 | VM_PKEY_BIT1 | VM_PKEY_BIT2 | \
 			    VM_PKEY_BIT3 | VM_PKEY_BIT4)
 
diff --git a/arch/powerpc/include/asm/pmac_pfunc.h b/arch/powerpc/include/asm/pmac_pfunc.h
index 73bd8f28f2a8..cee4e9f5b8cf 100644
--- a/arch/powerpc/include/asm/pmac_pfunc.h
+++ b/arch/powerpc/include/asm/pmac_pfunc.h
@@ -245,6 +245,7 @@ extern void pmf_put_function(struct pmf_function *func);
 
 extern int pmf_call_one(struct pmf_function *func, struct pmf_args *args);
 
+int pmac_pfunc_base_install(void);
 
 /* Suspend/resume code called by via-pmu directly for now */
 extern void pmac_pfunc_base_suspend(void);
diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h
index 18883b8a6dac..4436887bc415 100644
--- a/arch/powerpc/include/asm/ppc-opcode.h
+++ b/arch/powerpc/include/asm/ppc-opcode.h
@@ -162,6 +162,7 @@
 /* VMX Vector Store Instructions */
 #define OP_31_XOP_STVX          231
 
+#define OP_31   31
 #define OP_LWZ  32
 #define OP_STFS 52
 #define OP_STFSU 53
diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h
index 13f7f4c0e1ea..75ece56dcd62 100644
--- a/arch/powerpc/include/asm/ppc_asm.h
+++ b/arch/powerpc/include/asm/ppc_asm.h
@@ -80,10 +80,8 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
 #else
 #define SAVE_GPR(n, base)	stw	n,GPR0+4*(n)(base)
 #define REST_GPR(n, base)	lwz	n,GPR0+4*(n)(base)
-#define SAVE_NVGPRS(base)	SAVE_GPR(13, base); SAVE_8GPRS(14, base); \
-				SAVE_10GPRS(22, base)
-#define REST_NVGPRS(base)	REST_GPR(13, base); REST_8GPRS(14, base); \
-				REST_10GPRS(22, base)
+#define SAVE_NVGPRS(base)	stmw	13, GPR0+4*13(base)
+#define REST_NVGPRS(base)	lmw	13, GPR0+4*13(base)
 #endif
 
 #define SAVE_2GPRS(n, base)	SAVE_GPR(n, base); SAVE_GPR(n+1, base)
diff --git a/arch/powerpc/include/asm/pte-common.h b/arch/powerpc/include/asm/pte-common.h
index c4a72c7a8c83..bef56141a549 100644
--- a/arch/powerpc/include/asm/pte-common.h
+++ b/arch/powerpc/include/asm/pte-common.h
@@ -60,10 +60,6 @@
 #ifndef _PMD_PRESENT_MASK
 #define _PMD_PRESENT_MASK	_PMD_PRESENT
 #endif
-#ifndef _PMD_SIZE
-#define _PMD_SIZE	0
-#define PMD_PAGE_SIZE(pmd)	bad_call_to_PMD_PAGE_SIZE()
-#endif
 #ifndef _PMD_USER
 #define _PMD_USER	0
 #endif
@@ -88,11 +84,7 @@
 #define _PTE_NONE_MASK	_PAGE_HPTEFLAGS
 #endif
 
-/* Make sure we get a link error if PMD_PAGE_SIZE is ever called on a
- * kernel without large page PMD support
- */
 #ifndef __ASSEMBLY__
-extern unsigned long bad_call_to_PMD_PAGE_SIZE(void);
 
 /*
  * Don't just check for any non zero bits in __PAGE_USER, since for book3e
@@ -216,9 +208,6 @@ static inline bool pte_user(pte_t pte)
 #define PAGE_AGP		(PAGE_KERNEL_NC)
 #define HAVE_PAGE_AGP
 
-/* Advertise support for _PAGE_SPECIAL */
-#define __HAVE_ARCH_PTE_SPECIAL
-
 #ifndef _PAGE_READ
 /* if not defined, we should not find _PAGE_WRITE too */
 #define _PAGE_READ 0
diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
index cb0f272ce123..44b2be4a65d1 100644
--- a/arch/powerpc/include/asm/reg.h
+++ b/arch/powerpc/include/asm/reg.h
@@ -365,6 +365,7 @@
 #define SPRN_PSSCR	0x357	/* Processor Stop Status and Control Register (ISA 3.0) */
 #define SPRN_PSSCR_PR	0x337	/* PSSCR ISA 3.0, privileged mode access */
 #define SPRN_PMCR	0x374	/* Power Management Control Register */
+#define SPRN_RWMR	0x375	/* Region-Weighting Mode Register */
 
 /* HFSCR and FSCR bit numbers are the same */
 #define FSCR_SCV_LG	12	/* Enable System Call Vectored */
diff --git a/arch/powerpc/include/asm/rheap.h b/arch/powerpc/include/asm/rheap.h
index 172381769cfc..8e83703d6736 100644
--- a/arch/powerpc/include/asm/rheap.h
+++ b/arch/powerpc/include/asm/rheap.h
@@ -83,6 +83,9 @@ extern int rh_get_stats(rh_info_t * info, int what, int max_stats,
 /* Simple dump of remote heap info */
 extern void rh_dump(rh_info_t * info);
 
+/* Simple dump of remote info block */
+void rh_dump_blk(rh_info_t *info, rh_block_t *blk);
+
 /* Set owner of taken block */
 extern int rh_set_owner(rh_info_t * info, unsigned long start, const char *owner);
 
diff --git a/arch/powerpc/include/asm/syscalls.h b/arch/powerpc/include/asm/syscalls.h
index 1b90a3516a35..398171fdcd9f 100644
--- a/arch/powerpc/include/asm/syscalls.h
+++ b/arch/powerpc/include/asm/syscalls.h
@@ -16,7 +16,7 @@ asmlinkage long sys_mmap2(unsigned long addr, size_t len,
 		unsigned long prot, unsigned long flags,
 		unsigned long fd, unsigned long pgoff);
 asmlinkage long ppc64_personality(unsigned long personality);
-asmlinkage int ppc_rtas(struct rtas_args __user *uargs);
+asmlinkage long sys_rtas(struct rtas_args __user *uargs);
 
 #endif /* __KERNEL__ */
 #endif /* __ASM_POWERPC_SYSCALLS_H */
diff --git a/arch/powerpc/include/asm/systbl.h b/arch/powerpc/include/asm/systbl.h
index d61f9c96d916..79a3b47e4839 100644
--- a/arch/powerpc/include/asm/systbl.h
+++ b/arch/powerpc/include/asm/systbl.h
@@ -147,7 +147,7 @@ SYSCALL_SPU(setfsuid)
 SYSCALL_SPU(setfsgid)
 SYSCALL_SPU(llseek)
 COMPAT_SYS_SPU(getdents)
-SYSX_SPU(sys_select,ppc32_select,sys_select)
+COMPAT_SPU_NEW(select)
 SYSCALL_SPU(flock)
 SYSCALL_SPU(msync)
 COMPAT_SYS_SPU(readv)
@@ -245,7 +245,7 @@ SYSCALL_SPU(epoll_create)
 SYSCALL_SPU(epoll_ctl)
 SYSCALL_SPU(epoll_wait)
 SYSCALL_SPU(remap_file_pages)
-SYSX_SPU(sys_timer_create,compat_sys_timer_create,sys_timer_create)
+COMPAT_SYS_SPU(timer_create)
 COMPAT_SYS_SPU(timer_settime)
 COMPAT_SYS_SPU(timer_gettime)
 SYSCALL_SPU(timer_getoverrun)
@@ -260,7 +260,7 @@ COMPAT_SYS_SPU(utimes)
 COMPAT_SYS_SPU(statfs64)
 COMPAT_SYS_SPU(fstatfs64)
 SYSX(sys_ni_syscall,ppc_fadvise64_64,ppc_fadvise64_64)
-PPC_SYS_SPU(rtas)
+SYSCALL_SPU(rtas)
 OLDSYS(debug_setcontext)
 SYSCALL(ni_syscall)
 COMPAT_SYS(migrate_pages)
diff --git a/arch/powerpc/include/asm/trace.h b/arch/powerpc/include/asm/trace.h
index 33f3b479138b..d018e8602694 100644
--- a/arch/powerpc/include/asm/trace.h
+++ b/arch/powerpc/include/asm/trace.h
@@ -81,8 +81,7 @@ TRACE_EVENT_FN_COND(hcall_entry,
 
 TRACE_EVENT_FN_COND(hcall_exit,
 
-	TP_PROTO(unsigned long opcode, unsigned long retval,
-		unsigned long *retbuf),
+	TP_PROTO(unsigned long opcode, long retval, unsigned long *retbuf),
 
 	TP_ARGS(opcode, retval, retbuf),
 
@@ -90,7 +89,7 @@ TRACE_EVENT_FN_COND(hcall_exit,
 
 	TP_STRUCT__entry(
 		__field(unsigned long, opcode)
-		__field(unsigned long, retval)
+		__field(long, retval)
 	),
 
 	TP_fast_assign(
@@ -98,7 +97,7 @@ TRACE_EVENT_FN_COND(hcall_exit,
 		__entry->retval = retval;
 	),
 
-	TP_printk("opcode=%lu retval=%lu", __entry->opcode, __entry->retval),
+	TP_printk("opcode=%lu retval=%ld", __entry->opcode, __entry->retval),
 
 	hcall_tracepoint_regfunc, hcall_tracepoint_unregfunc
 );
diff --git a/arch/powerpc/include/asm/xmon.h b/arch/powerpc/include/asm/xmon.h
index eb42a0c6e1d9..30ff69bd8f43 100644
--- a/arch/powerpc/include/asm/xmon.h
+++ b/arch/powerpc/include/asm/xmon.h
@@ -29,7 +29,7 @@ static inline void xmon_register_spus(struct list_head *list) { };
 extern int cpus_are_in_xmon(void);
 #endif
 
-extern void xmon_printf(const char *format, ...);
+extern __printf(1, 2) void xmon_printf(const char *format, ...);
 
 #endif /* __KERNEL __ */
 #endif /* __ASM_POWERPC_XMON_H */
diff --git a/arch/powerpc/include/asm/xor.h b/arch/powerpc/include/asm/xor.h
index a36c2069d8ed..7d6dc503349d 100644
--- a/arch/powerpc/include/asm/xor.h
+++ b/arch/powerpc/include/asm/xor.h
@@ -24,17 +24,7 @@
 
 #include <asm/cputable.h>
 #include <asm/cpu_has_feature.h>
-
-void xor_altivec_2(unsigned long bytes, unsigned long *v1_in,
-		   unsigned long *v2_in);
-void xor_altivec_3(unsigned long bytes, unsigned long *v1_in,
-		   unsigned long *v2_in, unsigned long *v3_in);
-void xor_altivec_4(unsigned long bytes, unsigned long *v1_in,
-		   unsigned long *v2_in, unsigned long *v3_in,
-		   unsigned long *v4_in);
-void xor_altivec_5(unsigned long bytes, unsigned long *v1_in,
-		   unsigned long *v2_in, unsigned long *v3_in,
-		   unsigned long *v4_in, unsigned long *v5_in);
+#include <asm/xor_altivec.h>
 
 static struct xor_block_template xor_block_altivec = {
 	.name = "altivec",
diff --git a/arch/powerpc/include/asm/xor_altivec.h b/arch/powerpc/include/asm/xor_altivec.h
new file mode 100644
index 000000000000..6ca923510b59
--- /dev/null
+++ b/arch/powerpc/include/asm/xor_altivec.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_POWERPC_XOR_ALTIVEC_H
+#define _ASM_POWERPC_XOR_ALTIVEC_H
+
+#ifdef CONFIG_ALTIVEC
+
+void xor_altivec_2(unsigned long bytes, unsigned long *v1_in,
+		   unsigned long *v2_in);
+void xor_altivec_3(unsigned long bytes, unsigned long *v1_in,
+		   unsigned long *v2_in, unsigned long *v3_in);
+void xor_altivec_4(unsigned long bytes, unsigned long *v1_in,
+		   unsigned long *v2_in, unsigned long *v3_in,
+		   unsigned long *v4_in);
+void xor_altivec_5(unsigned long bytes, unsigned long *v1_in,
+		   unsigned long *v2_in, unsigned long *v3_in,
+		   unsigned long *v4_in, unsigned long *v5_in);
+
+#endif
+#endif /* _ASM_POWERPC_XOR_ALTIVEC_H */
diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h
index 833ed9a16adf..1b32b56a03d3 100644
--- a/arch/powerpc/include/uapi/asm/kvm.h
+++ b/arch/powerpc/include/uapi/asm/kvm.h
@@ -633,6 +633,7 @@ struct kvm_ppc_cpu_char {
 #define KVM_REG_PPC_PSSCR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbd)
 
 #define KVM_REG_PPC_DEC_EXPIRY	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbe)
+#define KVM_REG_PPC_ONLINE	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xbf)
 
 /* Transactional Memory checkpointed state:
  * This is all GPRs, all VSX regs and a subset of SPRs
diff --git a/arch/powerpc/include/uapi/asm/msgbuf.h b/arch/powerpc/include/uapi/asm/msgbuf.h
index 65beb0942500..2b1b37797a47 100644
--- a/arch/powerpc/include/uapi/asm/msgbuf.h
+++ b/arch/powerpc/include/uapi/asm/msgbuf.h
@@ -10,18 +10,18 @@
 
 struct msqid64_ds {
 	struct ipc64_perm msg_perm;
-#ifndef __powerpc64__
-	unsigned int	__unused1;
-#endif
+#ifdef __powerpc64__
 	__kernel_time_t msg_stime;	/* last msgsnd time */
-#ifndef __powerpc64__
-	unsigned int	__unused2;
-#endif
 	__kernel_time_t msg_rtime;	/* last msgrcv time */
-#ifndef __powerpc64__
-	unsigned int	__unused3;
-#endif
 	__kernel_time_t msg_ctime;	/* last change time */
+#else
+	unsigned long  msg_stime_high;
+	unsigned long  msg_stime;	/* last msgsnd time */
+	unsigned long  msg_rtime_high;
+	unsigned long  msg_rtime;	/* last msgrcv time */
+	unsigned long  msg_ctime_high;
+	unsigned long  msg_ctime;	/* last change time */
+#endif
 	unsigned long  msg_cbytes;	/* current number of bytes on queue */
 	unsigned long  msg_qnum;	/* number of messages in queue */
 	unsigned long  msg_qbytes;	/* max number of bytes on queue */
diff --git a/arch/powerpc/include/uapi/asm/sembuf.h b/arch/powerpc/include/uapi/asm/sembuf.h
index 8f393d60f02d..3f60946f77e3 100644
--- a/arch/powerpc/include/uapi/asm/sembuf.h
+++ b/arch/powerpc/include/uapi/asm/sembuf.h
@@ -15,20 +15,20 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
- * - 2 miscellaneous 32-bit values
+ * - 2 miscellaneous 32/64-bit values
  */
 
 struct semid64_ds {
 	struct ipc64_perm sem_perm;	/* permissions .. see ipc.h */
 #ifndef __powerpc64__
-	unsigned long	__unused1;
-#endif
+	unsigned long	sem_otime_high;
+	unsigned long	sem_otime;	/* last semop time */
+	unsigned long	sem_ctime_high;
+	unsigned long	sem_ctime;	/* last change time */
+#else
 	__kernel_time_t	sem_otime;	/* last semop time */
-#ifndef __powerpc64__
-	unsigned long	__unused2;
-#endif
 	__kernel_time_t	sem_ctime;	/* last change time */
+#endif
 	unsigned long	sem_nsems;	/* no. of semaphores in array */
 	unsigned long	__unused3;
 	unsigned long	__unused4;
diff --git a/arch/powerpc/include/uapi/asm/shmbuf.h b/arch/powerpc/include/uapi/asm/shmbuf.h
index deb1c3e503d3..b591c4d7e4c5 100644
--- a/arch/powerpc/include/uapi/asm/shmbuf.h
+++ b/arch/powerpc/include/uapi/asm/shmbuf.h
@@ -16,25 +16,22 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
  * - 2 miscellaneous 32-bit values
  */
 
 struct shmid64_ds {
 	struct ipc64_perm	shm_perm;	/* operation perms */
-#ifndef __powerpc64__
-	unsigned long		__unused1;
-#endif
+#ifdef __powerpc64__
 	__kernel_time_t		shm_atime;	/* last attach time */
-#ifndef __powerpc64__
-	unsigned long		__unused2;
-#endif
 	__kernel_time_t		shm_dtime;	/* last detach time */
-#ifndef __powerpc64__
-	unsigned long		__unused3;
-#endif
 	__kernel_time_t		shm_ctime;	/* last change time */
-#ifndef __powerpc64__
+#else
+	unsigned long		shm_atime_high;
+	unsigned long		shm_atime;	/* last attach time */
+	unsigned long		shm_dtime_high;
+	unsigned long		shm_dtime;	/* last detach time */
+	unsigned long		shm_ctime_high;
+	unsigned long		shm_ctime;	/* last change time */
 	unsigned long		__unused4;
 #endif
 	size_t			shm_segsz;	/* size of segment (bytes) */
diff --git a/arch/powerpc/include/uapi/asm/siginfo.h b/arch/powerpc/include/uapi/asm/siginfo.h
index 9f142451a01f..1d51d9b88221 100644
--- a/arch/powerpc/include/uapi/asm/siginfo.h
+++ b/arch/powerpc/include/uapi/asm/siginfo.h
@@ -15,19 +15,4 @@
 
 #include <asm-generic/siginfo.h>
 
-/*
- * SIGFPE si_codes
- */
-#ifdef __KERNEL__
-#define FPE_FIXME	0	/* Broken dup of SI_USER */
-#endif /* __KERNEL__ */
-
-/*
- * SIGTRAP si_codes
- */
-#ifdef __KERNEL__
-#define TRAP_FIXME	0	/* Broken dup of SI_USER */
-#endif /* __KERNEL__ */
-
-
 #endif	/* _ASM_POWERPC_SIGINFO_H */
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 373dc1d6ef44..0a0544335950 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -13,6 +13,7 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#include <linux/compat.h>
 #include <linux/signal.h>
 #include <linux/sched.h>
 #include <linux/kernel.h>
@@ -42,7 +43,6 @@
 #include <asm/paca.h>
 #include <asm/lppaca.h>
 #include <asm/cache.h>
-#include <asm/compat.h>
 #include <asm/mmu.h>
 #include <asm/hvcall.h>
 #include <asm/xics.h>
@@ -180,6 +180,7 @@ int main(void)
 	OFFSET(PACAKMSR, paca_struct, kernel_msr);
 	OFFSET(PACAIRQSOFTMASK, paca_struct, irq_soft_mask);
 	OFFSET(PACAIRQHAPPENED, paca_struct, irq_happened);
+	OFFSET(PACA_FTRACE_ENABLED, paca_struct, ftrace_enabled);
 #ifdef CONFIG_PPC_BOOK3S
 	OFFSET(PACACONTEXTID, paca_struct, mm_ctx_id);
 #ifdef CONFIG_PPC_MM_SLICES
@@ -425,20 +426,20 @@ int main(void)
 	OFFSET(VCPU_HOST_STACK, kvm_vcpu, arch.host_stack);
 	OFFSET(VCPU_HOST_PID, kvm_vcpu, arch.host_pid);
 	OFFSET(VCPU_GUEST_PID, kvm_vcpu, arch.pid);
-	OFFSET(VCPU_GPRS, kvm_vcpu, arch.gpr);
+	OFFSET(VCPU_GPRS, kvm_vcpu, arch.regs.gpr);
 	OFFSET(VCPU_VRSAVE, kvm_vcpu, arch.vrsave);
 	OFFSET(VCPU_FPRS, kvm_vcpu, arch.fp.fpr);
 #ifdef CONFIG_ALTIVEC
 	OFFSET(VCPU_VRS, kvm_vcpu, arch.vr.vr);
 #endif
-	OFFSET(VCPU_XER, kvm_vcpu, arch.xer);
-	OFFSET(VCPU_CTR, kvm_vcpu, arch.ctr);
-	OFFSET(VCPU_LR, kvm_vcpu, arch.lr);
+	OFFSET(VCPU_XER, kvm_vcpu, arch.regs.xer);
+	OFFSET(VCPU_CTR, kvm_vcpu, arch.regs.ctr);
+	OFFSET(VCPU_LR, kvm_vcpu, arch.regs.link);
 #ifdef CONFIG_PPC_BOOK3S
 	OFFSET(VCPU_TAR, kvm_vcpu, arch.tar);
 #endif
 	OFFSET(VCPU_CR, kvm_vcpu, arch.cr);
-	OFFSET(VCPU_PC, kvm_vcpu, arch.pc);
+	OFFSET(VCPU_PC, kvm_vcpu, arch.regs.nip);
 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 	OFFSET(VCPU_MSR, kvm_vcpu, arch.shregs.msr);
 	OFFSET(VCPU_SRR0, kvm_vcpu, arch.shregs.srr0);
@@ -695,10 +696,10 @@ int main(void)
 
 #else /* CONFIG_PPC_BOOK3S */
 	OFFSET(VCPU_CR, kvm_vcpu, arch.cr);
-	OFFSET(VCPU_XER, kvm_vcpu, arch.xer);
-	OFFSET(VCPU_LR, kvm_vcpu, arch.lr);
-	OFFSET(VCPU_CTR, kvm_vcpu, arch.ctr);
-	OFFSET(VCPU_PC, kvm_vcpu, arch.pc);
+	OFFSET(VCPU_XER, kvm_vcpu, arch.regs.xer);
+	OFFSET(VCPU_LR, kvm_vcpu, arch.regs.link);
+	OFFSET(VCPU_CTR, kvm_vcpu, arch.regs.ctr);
+	OFFSET(VCPU_PC, kvm_vcpu, arch.regs.nip);
 	OFFSET(VCPU_SPRG9, kvm_vcpu, arch.sprg9);
 	OFFSET(VCPU_LAST_INST, kvm_vcpu, arch.last_inst);
 	OFFSET(VCPU_FAULT_DEAR, kvm_vcpu, arch.fault_dear);
diff --git a/arch/powerpc/kernel/btext.c b/arch/powerpc/kernel/btext.c
index 6537cba1a758..b2072d5bbf2b 100644
--- a/arch/powerpc/kernel/btext.c
+++ b/arch/powerpc/kernel/btext.c
@@ -157,20 +157,20 @@ void btext_map(void)
 
 	/* By default, we are no longer mapped */
 	boot_text_mapped = 0;
-	if (dispDeviceBase == 0)
+	if (!dispDeviceBase)
 		return;
 	base = ((unsigned long) dispDeviceBase) & 0xFFFFF000UL;
 	offset = ((unsigned long) dispDeviceBase) - base;
 	size = dispDeviceRowBytes * dispDeviceRect[3] + offset
 		+ dispDeviceRect[0];
 	vbase = __ioremap(base, size, pgprot_val(pgprot_noncached_wc(__pgprot(0))));
-	if (vbase == 0)
+	if (!vbase)
 		return;
 	logicalDisplayBase = vbase + offset;
 	boot_text_mapped = 1;
 }
 
-int btext_initialize(struct device_node *np)
+static int btext_initialize(struct device_node *np)
 {
 	unsigned int width, height, depth, pitch;
 	unsigned long address = 0;
@@ -270,7 +270,7 @@ static unsigned char * calc_base(int x, int y)
 	unsigned char *base;
 
 	base = logicalDisplayBase;
-	if (base == 0)
+	if (!base)
 		base = dispDeviceBase;
 	base += (x + dispDeviceRect[0]) * (dispDeviceDepth >> 3);
 	base += (y + dispDeviceRect[1]) * dispDeviceRowBytes;
@@ -281,7 +281,7 @@ static unsigned char * calc_base(int x, int y)
 void btext_update_display(unsigned long phys, int width, int height,
 			  int depth, int pitch)
 {
-	if (dispDeviceBase == 0)
+	if (!dispDeviceBase)
 		return;
 
 	/* check it's the same frame buffer (within 256MB) */
diff --git a/arch/powerpc/kernel/dma.c b/arch/powerpc/kernel/dma.c
index da20569de9d4..138157deeadf 100644
--- a/arch/powerpc/kernel/dma.c
+++ b/arch/powerpc/kernel/dma.c
@@ -309,8 +309,6 @@ int dma_set_coherent_mask(struct device *dev, u64 mask)
 }
 EXPORT_SYMBOL(dma_set_coherent_mask);
 
-#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
-
 int dma_set_mask(struct device *dev, u64 dma_mask)
 {
 	if (ppc_md.dma_set_mask)
@@ -361,7 +359,6 @@ EXPORT_SYMBOL_GPL(dma_get_required_mask);
 
 static int __init dma_init(void)
 {
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 #ifdef CONFIG_PCI
 	dma_debug_add_bus(&pci_bus_type);
 #endif
diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
index bc640e4c5ca5..90bb39b1a23c 100644
--- a/arch/powerpc/kernel/eeh.c
+++ b/arch/powerpc/kernel/eeh.c
@@ -1775,18 +1775,6 @@ static int proc_eeh_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int proc_eeh_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_eeh_show, NULL);
-}
-
-static const struct file_operations proc_eeh_operations = {
-	.open      = proc_eeh_open,
-	.read      = seq_read,
-	.llseek    = seq_lseek,
-	.release   = single_release,
-};
-
 #ifdef CONFIG_DEBUG_FS
 static int eeh_enable_dbgfs_set(void *data, u64 val)
 {
@@ -1828,7 +1816,7 @@ DEFINE_SIMPLE_ATTRIBUTE(eeh_freeze_dbgfs_ops, eeh_freeze_dbgfs_get,
 static int __init eeh_init_proc(void)
 {
 	if (machine_is(pseries) || machine_is(powernv)) {
-		proc_create("powerpc/eeh", 0, NULL, &proc_eeh_operations);
+		proc_create_single("powerpc/eeh", 0, NULL, proc_eeh_show);
 #ifdef CONFIG_DEBUG_FS
 		debugfs_create_file("eeh_enable", 0600,
                                     powerpc_debugfs_root, NULL,
diff --git a/arch/powerpc/kernel/fadump.c b/arch/powerpc/kernel/fadump.c
index 3c2c2688918f..07e8396d472b 100644
--- a/arch/powerpc/kernel/fadump.c
+++ b/arch/powerpc/kernel/fadump.c
@@ -335,6 +335,26 @@ static unsigned long get_fadump_area_size(void)
 	return size;
 }
 
+static void __init fadump_reserve_crash_area(unsigned long base,
+					     unsigned long size)
+{
+	struct memblock_region *reg;
+	unsigned long mstart, mend, msize;
+
+	for_each_memblock(memory, reg) {
+		mstart = max_t(unsigned long, base, reg->base);
+		mend = reg->base + reg->size;
+		mend = min(base + size, mend);
+
+		if (mstart < mend) {
+			msize = mend - mstart;
+			memblock_reserve(mstart, msize);
+			pr_info("Reserved %ldMB of memory at %#016lx for saving crash dump\n",
+				(msize >> 20), mstart);
+		}
+	}
+}
+
 int __init fadump_reserve_mem(void)
 {
 	unsigned long base, size, memory_boundary;
@@ -380,7 +400,16 @@ int __init fadump_reserve_mem(void)
 		memory_boundary = memblock_end_of_DRAM();
 
 	if (fw_dump.dump_active) {
-		printk(KERN_INFO "Firmware-assisted dump is active.\n");
+		pr_info("Firmware-assisted dump is active.\n");
+
+#ifdef CONFIG_HUGETLB_PAGE
+		/*
+		 * FADump capture kernel doesn't care much about hugepages.
+		 * In fact, handling hugepages in capture kernel is asking for
+		 * trouble. So, disable HugeTLB support when fadump is active.
+		 */
+		hugetlb_disabled = true;
+#endif
 		/*
 		 * If last boot has crashed then reserve all the memory
 		 * above boot_memory_size so that we don't touch it until
@@ -389,11 +418,7 @@ int __init fadump_reserve_mem(void)
 		 */
 		base = fw_dump.boot_memory_size;
 		size = memory_boundary - base;
-		memblock_reserve(base, size);
-		printk(KERN_INFO "Reserved %ldMB of memory at %ldMB "
-				"for saving crash dump\n",
-				(unsigned long)(size >> 20),
-				(unsigned long)(base >> 20));
+		fadump_reserve_crash_area(base, size);
 
 		fw_dump.fadumphdr_addr =
 				be64_to_cpu(fdm_active->rmr_region.destination_address) +
@@ -1155,6 +1180,9 @@ void fadump_cleanup(void)
 		init_fadump_mem_struct(&fdm,
 			be64_to_cpu(fdm_active->cpu_state_data.destination_address));
 		fadump_invalidate_dump(&fdm);
+	} else if (fw_dump.dump_registered) {
+		/* Un-register Firmware-assisted dump if it was registered. */
+		fadump_unregister_dump(&fdm);
 	}
 }
 
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index d8670a37d70c..6cab07e76732 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -913,7 +913,7 @@ start_here:
 	tovirt(r6,r6)
 	lis	r5, abatron_pteptrs@h
 	ori	r5, r5, abatron_pteptrs@l
-	stw	r5, 0xf0(r0)	/* Must match your Abatron config file */
+	stw	r5, 0xf0(0)	/* Must match your Abatron config file */
 	tophys(r5,r5)
 	stw	r6, 0(r5)
 
diff --git a/arch/powerpc/kernel/hw_breakpoint.c b/arch/powerpc/kernel/hw_breakpoint.c
index 4c1012b80d3b..80547dad37da 100644
--- a/arch/powerpc/kernel/hw_breakpoint.c
+++ b/arch/powerpc/kernel/hw_breakpoint.c
@@ -178,8 +178,8 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp)
 	if (cpu_has_feature(CPU_FTR_DAWR)) {
 		length_max = 512 ; /* 64 doublewords */
 		/* DAWR region can't cross 512 boundary */
-		if ((bp->attr.bp_addr >> 10) != 
-		    ((bp->attr.bp_addr + bp->attr.bp_len - 1) >> 10))
+		if ((bp->attr.bp_addr >> 9) !=
+		    ((bp->attr.bp_addr + bp->attr.bp_len - 1) >> 9))
 			return -EINVAL;
 	}
 	if (info->len >
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 061aa0f47bb1..bbf7ec582d60 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -89,7 +89,7 @@ atomic_t ppc_n_lost_interrupts;
 
 #ifdef CONFIG_TAU_INT
 extern int tau_initialized;
-extern int tau_interrupts(int);
+u32 tau_interrupts(unsigned long cpu);
 #endif
 #endif /* CONFIG_PPC32 */
 
diff --git a/arch/powerpc/kernel/kvm.c b/arch/powerpc/kernel/kvm.c
index 9ad37f827a97..683b5b3805bd 100644
--- a/arch/powerpc/kernel/kvm.c
+++ b/arch/powerpc/kernel/kvm.c
@@ -25,6 +25,7 @@
 #include <linux/kvm_para.h>
 #include <linux/slab.h>
 #include <linux/of.h>
+#include <linux/pagemap.h>
 
 #include <asm/reg.h>
 #include <asm/sections.h>
@@ -672,14 +673,13 @@ static void kvm_use_magic_page(void)
 {
 	u32 *p;
 	u32 *start, *end;
-	u32 tmp;
 	u32 features;
 
 	/* Tell the host to map the magic page to -4096 on all CPUs */
 	on_each_cpu(kvm_map_magic_page, &features, 1);
 
 	/* Quick self-test to see if the mapping works */
-	if (__get_user(tmp, (u32*)KVM_MAGIC_PAGE)) {
+	if (!fault_in_pages_readable((const char *)KVM_MAGIC_PAGE, sizeof(u32))) {
 		kvm_patching_worked = false;
 		return;
 	}
diff --git a/arch/powerpc/kernel/machine_kexec.c b/arch/powerpc/kernel/machine_kexec.c
index 2694d078741d..936c7e2d421e 100644
--- a/arch/powerpc/kernel/machine_kexec.c
+++ b/arch/powerpc/kernel/machine_kexec.c
@@ -98,12 +98,14 @@ void machine_kexec(struct kimage *image)
 	int save_ftrace_enabled;
 
 	save_ftrace_enabled = __ftrace_enabled_save();
+	this_cpu_disable_ftrace();
 
 	if (ppc_md.machine_kexec)
 		ppc_md.machine_kexec(image);
 	else
 		default_machine_kexec(image);
 
+	this_cpu_enable_ftrace();
 	__ftrace_enabled_restore(save_ftrace_enabled);
 
 	/* Fall back to normal restart if we're still alive. */
diff --git a/arch/powerpc/kernel/machine_kexec_64.c b/arch/powerpc/kernel/machine_kexec_64.c
index 1044bf15d5ed..a0f6f45005bd 100644
--- a/arch/powerpc/kernel/machine_kexec_64.c
+++ b/arch/powerpc/kernel/machine_kexec_64.c
@@ -231,16 +231,16 @@ static void kexec_prepare_cpus(void)
 	/* we are sure every CPU has IRQs off at this point */
 	kexec_all_irq_disabled = 1;
 
-	/* after we tell the others to go down */
-	if (ppc_md.kexec_cpu_down)
-		ppc_md.kexec_cpu_down(0, 0);
-
 	/*
 	 * Before removing MMU mappings make sure all CPUs have entered real
 	 * mode:
 	 */
 	kexec_prepare_cpus_wait(KEXEC_STATE_REAL_MODE);
 
+	/* after we tell the others to go down */
+	if (ppc_md.kexec_cpu_down)
+		ppc_md.kexec_cpu_down(0, 0);
+
 	put_cpu();
 }
 
diff --git a/arch/powerpc/kernel/misc.S b/arch/powerpc/kernel/misc.S
index 384357cb8bc0..2f18fc1368d0 100644
--- a/arch/powerpc/kernel/misc.S
+++ b/arch/powerpc/kernel/misc.S
@@ -60,6 +60,10 @@ _GLOBAL(setjmp)
 	PPC_STL	r0,0(r3)
 	PPC_STL	r1,SZL(r3)
 	PPC_STL	r2,2*SZL(r3)
+#ifdef CONFIG_PPC32
+	mfcr	r12
+	stmw	r12, 3*SZL(r3)
+#else
 	mfcr	r0
 	PPC_STL	r0,3*SZL(r3)
 	PPC_STL	r13,4*SZL(r3)
@@ -81,14 +85,16 @@ _GLOBAL(setjmp)
 	PPC_STL	r29,20*SZL(r3)
 	PPC_STL	r30,21*SZL(r3)
 	PPC_STL	r31,22*SZL(r3)
+#endif
 	li	r3,0
 	blr
 
 _GLOBAL(longjmp)
-	PPC_LCMPI r4,0
-	bne	1f
-	li	r4,1
-1:	PPC_LL	r13,4*SZL(r3)
+#ifdef CONFIG_PPC32
+	lmw	r12, 3*SZL(r3)
+	mtcrf	0x38, r12
+#else
+	PPC_LL	r13,4*SZL(r3)
 	PPC_LL	r14,5*SZL(r3)
 	PPC_LL	r15,6*SZL(r3)
 	PPC_LL	r16,7*SZL(r3)
@@ -109,11 +115,14 @@ _GLOBAL(longjmp)
 	PPC_LL	r31,22*SZL(r3)
 	PPC_LL	r0,3*SZL(r3)
 	mtcrf	0x38,r0
+#endif
 	PPC_LL	r0,0(r3)
 	PPC_LL	r1,SZL(r3)
 	PPC_LL	r2,2*SZL(r3)
 	mtlr	r0
-	mr	r3,r4
+	mr.	r3, r4
+	bnelr
+	li	r3, 1
 	blr
 
 _GLOBAL(current_stack_pointer)
diff --git a/arch/powerpc/kernel/module_32.c b/arch/powerpc/kernel/module_32.c
index 5a7a78f12562..88d83771f462 100644
--- a/arch/powerpc/kernel/module_32.c
+++ b/arch/powerpc/kernel/module_32.c
@@ -109,12 +109,12 @@ static unsigned long get_plt_size(const Elf32_Ehdr *hdr,
 	for (i = 1; i < hdr->e_shnum; i++) {
 		/* If it's called *.init*, and we're not init, we're
                    not interested */
-		if ((strstr(secstrings + sechdrs[i].sh_name, ".init") != 0)
+		if ((strstr(secstrings + sechdrs[i].sh_name, ".init") != NULL)
 		    != is_init)
 			continue;
 
 		/* We don't want to look at debug sections. */
-		if (strstr(secstrings + sechdrs[i].sh_name, ".debug") != 0)
+		if (strstr(secstrings + sechdrs[i].sh_name, ".debug"))
 			continue;
 
 		if (sechdrs[i].sh_type == SHT_RELA) {
diff --git a/arch/powerpc/kernel/module_64.c b/arch/powerpc/kernel/module_64.c
index a2636c250b7b..caaa0054417d 100644
--- a/arch/powerpc/kernel/module_64.c
+++ b/arch/powerpc/kernel/module_64.c
@@ -280,6 +280,10 @@ static unsigned long get_stubs_size(const Elf64_Ehdr *hdr,
 #ifdef CONFIG_DYNAMIC_FTRACE
 	/* make the trampoline to the ftrace_caller */
 	relocs++;
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+	/* an additional one for ftrace_regs_caller */
+	relocs++;
+#endif
 #endif
 
 	pr_debug("Looks like a total of %lu stubs, max\n", relocs);
@@ -462,9 +466,12 @@ static unsigned long stub_for_addr(const Elf64_Shdr *sechdrs,
 	return (unsigned long)&stubs[i];
 }
 
-#ifdef CC_USING_MPROFILE_KERNEL
-static bool is_early_mcount_callsite(u32 *instruction)
+#ifdef CONFIG_MPROFILE_KERNEL
+static bool is_mprofile_mcount_callsite(const char *name, u32 *instruction)
 {
+	if (strcmp("_mcount", name))
+		return false;
+
 	/*
 	 * Check if this is one of the -mprofile-kernel sequences.
 	 */
@@ -496,8 +503,7 @@ static void squash_toc_save_inst(const char *name, unsigned long addr)
 #else
 static void squash_toc_save_inst(const char *name, unsigned long addr) { }
 
-/* without -mprofile-kernel, mcount calls are never early */
-static bool is_early_mcount_callsite(u32 *instruction)
+static bool is_mprofile_mcount_callsite(const char *name, u32 *instruction)
 {
 	return false;
 }
@@ -505,11 +511,11 @@ static bool is_early_mcount_callsite(u32 *instruction)
 
 /* We expect a noop next: if it is, replace it with instruction to
    restore r2. */
-static int restore_r2(u32 *instruction, struct module *me)
+static int restore_r2(const char *name, u32 *instruction, struct module *me)
 {
 	u32 *prev_insn = instruction - 1;
 
-	if (is_early_mcount_callsite(prev_insn))
+	if (is_mprofile_mcount_callsite(name, prev_insn))
 		return 1;
 
 	/*
@@ -650,7 +656,8 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
 				value = stub_for_addr(sechdrs, value, me);
 				if (!value)
 					return -ENOENT;
-				if (!restore_r2((u32 *)location + 1, me))
+				if (!restore_r2(strtab + sym->st_name,
+							(u32 *)location + 1, me))
 					return -ENOEXEC;
 
 				squash_toc_save_inst(strtab + sym->st_name, value);
@@ -746,7 +753,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
 
 #ifdef CONFIG_DYNAMIC_FTRACE
 
-#ifdef CC_USING_MPROFILE_KERNEL
+#ifdef CONFIG_MPROFILE_KERNEL
 
 #define PACATOC offsetof(struct paca_struct, kernel_toc)
 
@@ -762,7 +769,8 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
  * via the paca (in r13). The target (ftrace_caller()) is responsible for
  * saving and restoring the toc before returning.
  */
-static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs, struct module *me)
+static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs,
+				struct module *me, unsigned long addr)
 {
 	struct ppc64_stub_entry *entry;
 	unsigned int i, num_stubs;
@@ -789,9 +797,10 @@ static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs, struct module
 	memcpy(entry->jump, stub_insns, sizeof(stub_insns));
 
 	/* Stub uses address relative to kernel toc (from the paca) */
-	reladdr = (unsigned long)ftrace_caller - kernel_toc_addr();
+	reladdr = addr - kernel_toc_addr();
 	if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
-		pr_err("%s: Address of ftrace_caller out of range of kernel_toc.\n", me->name);
+		pr_err("%s: Address of %ps out of range of kernel_toc.\n",
+							me->name, (void *)addr);
 		return 0;
 	}
 
@@ -799,22 +808,30 @@ static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs, struct module
 	entry->jump[2] |= PPC_LO(reladdr);
 
 	/* Eventhough we don't use funcdata in the stub, it's needed elsewhere. */
-	entry->funcdata = func_desc((unsigned long)ftrace_caller);
+	entry->funcdata = func_desc(addr);
 	entry->magic = STUB_MAGIC;
 
 	return (unsigned long)entry;
 }
 #else
-static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs, struct module *me)
+static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs,
+				struct module *me, unsigned long addr)
 {
-	return stub_for_addr(sechdrs, (unsigned long)ftrace_caller, me);
+	return stub_for_addr(sechdrs, addr, me);
 }
 #endif
 
 int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs)
 {
 	mod->arch.toc = my_r2(sechdrs, mod);
-	mod->arch.tramp = create_ftrace_stub(sechdrs, mod);
+	mod->arch.tramp = create_ftrace_stub(sechdrs, mod,
+					(unsigned long)ftrace_caller);
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+	mod->arch.tramp_regs = create_ftrace_stub(sechdrs, mod,
+					(unsigned long)ftrace_regs_caller);
+	if (!mod->arch.tramp_regs)
+		return -ENOENT;
+#endif
 
 	if (!mod->arch.tramp)
 		return -ENOENT;
diff --git a/arch/powerpc/kernel/pci_32.c b/arch/powerpc/kernel/pci_32.c
index 85ad2f78b889..d63b488d34d7 100644
--- a/arch/powerpc/kernel/pci_32.c
+++ b/arch/powerpc/kernel/pci_32.c
@@ -11,11 +11,13 @@
 #include <linux/sched.h>
 #include <linux/errno.h>
 #include <linux/bootmem.h>
+#include <linux/syscalls.h>
 #include <linux/irq.h>
 #include <linux/list.h>
 #include <linux/of.h>
 #include <linux/slab.h>
 #include <linux/export.h>
+#include <linux/syscalls.h>
 
 #include <asm/processor.h>
 #include <asm/io.h>
@@ -283,7 +285,8 @@ pci_bus_to_hose(int bus)
  * Note that the returned IO or memory base is a physical address
  */
 
-long sys_pciconfig_iobase(long which, unsigned long bus, unsigned long devfn)
+SYSCALL_DEFINE3(pciconfig_iobase, long, which,
+		unsigned long, bus, unsigned long, devfn)
 {
 	struct pci_controller* hose;
 	long result = -EOPNOTSUPP;
@@ -307,5 +310,3 @@ long sys_pciconfig_iobase(long which, unsigned long bus, unsigned long devfn)
 
 	return result;
 }
-
-
diff --git a/arch/powerpc/kernel/pci_64.c b/arch/powerpc/kernel/pci_64.c
index 15ce0306b092..dff28f903512 100644
--- a/arch/powerpc/kernel/pci_64.c
+++ b/arch/powerpc/kernel/pci_64.c
@@ -203,8 +203,8 @@ void pcibios_setup_phb_io_space(struct pci_controller *hose)
 #define IOBASE_ISA_IO		3
 #define IOBASE_ISA_MEM		4
 
-long sys_pciconfig_iobase(long which, unsigned long in_bus,
-			  unsigned long in_devfn)
+SYSCALL_DEFINE3(pciconfig_iobase, long, which, unsigned long, in_bus,
+			  unsigned long, in_devfn)
 {
 	struct pci_controller* hose;
 	struct pci_bus *tmp_bus, *bus = NULL;
diff --git a/arch/powerpc/kernel/ppc_save_regs.S b/arch/powerpc/kernel/ppc_save_regs.S
index 1b1787d52896..8afbe213d729 100644
--- a/arch/powerpc/kernel/ppc_save_regs.S
+++ b/arch/powerpc/kernel/ppc_save_regs.S
@@ -25,6 +25,9 @@
  */
 _GLOBAL(ppc_save_regs)
 	PPC_STL	r0,0*SZL(r3)
+#ifdef CONFIG_PPC32
+	stmw	r2, 2*SZL(r3)
+#else
 	PPC_STL	r2,2*SZL(r3)
 	PPC_STL	r3,3*SZL(r3)
 	PPC_STL	r4,4*SZL(r3)
@@ -55,6 +58,7 @@ _GLOBAL(ppc_save_regs)
 	PPC_STL	r29,29*SZL(r3)
 	PPC_STL	r30,30*SZL(r3)
 	PPC_STL	r31,31*SZL(r3)
+#endif
 	/* go up one stack frame for SP */
 	PPC_LL	r4,0(r1)
 	PPC_STL	r4,1*SZL(r3)
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 1237f13fed51..26ea9793d290 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -632,6 +632,7 @@ void do_break (struct pt_regs *regs, unsigned long address,
 	hw_breakpoint_disable();
 
 	/* Deliver the signal to userspace */
+	clear_siginfo(&info);
 	info.si_signo = SIGTRAP;
 	info.si_errno = 0;
 	info.si_code = TRAP_HWBKPT;
diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c
index 9dbed488aba1..05e7fb47a7a4 100644
--- a/arch/powerpc/kernel/prom.c
+++ b/arch/powerpc/kernel/prom.c
@@ -332,25 +332,10 @@ static int __init early_init_dt_scan_cpus(unsigned long node,
 	 * NOTE: This must match the parsing done in smp_setup_cpu_maps.
 	 */
 	for (i = 0; i < nthreads; i++) {
-		/*
-		 * version 2 of the kexec param format adds the phys cpuid of
-		 * booted proc.
-		 */
-		if (fdt_version(initial_boot_params) >= 2) {
-			if (be32_to_cpu(intserv[i]) ==
-			    fdt_boot_cpuid_phys(initial_boot_params)) {
-				found = boot_cpu_count;
-				found_thread = i;
-			}
-		} else {
-			/*
-			 * Check if it's the boot-cpu, set it's hw index now,
-			 * unfortunately this format did not support booting
-			 * off secondary threads.
-			 */
-			if (of_get_flat_dt_prop(node,
-					"linux,boot-cpu", NULL) != NULL)
-				found = boot_cpu_count;
+		if (be32_to_cpu(intserv[i]) ==
+			fdt_boot_cpuid_phys(initial_boot_params)) {
+			found = boot_cpu_count;
+			found_thread = i;
 		}
 #ifdef CONFIG_SMP
 		/* logical cpu id is always 0 on UP kernels */
diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c
index f9d6befb55a6..425992e393bc 100644
--- a/arch/powerpc/kernel/prom_init.c
+++ b/arch/powerpc/kernel/prom_init.c
@@ -103,7 +103,7 @@ int of_workarounds;
 #ifdef DEBUG_PROM
 #define prom_debug(x...)	prom_printf(x)
 #else
-#define prom_debug(x...)
+#define prom_debug(x...)	do { } while (0)
 #endif
 
 
@@ -334,6 +334,7 @@ static void __init prom_print_dec(unsigned long val)
 	call_prom("write", 3, 1, prom.stdout, buf+i, size);
 }
 
+__printf(1, 2)
 static void __init prom_printf(const char *format, ...)
 {
 	const char *p, *q, *s;
@@ -1160,7 +1161,7 @@ static void __init prom_send_capabilities(void)
 		 */
 
 		cores = DIV_ROUND_UP(NR_CPUS, prom_count_smt_threads());
-		prom_printf("Max number of cores passed to firmware: %lu (NR_CPUS = %lu)\n",
+		prom_printf("Max number of cores passed to firmware: %u (NR_CPUS = %d)\n",
 			    cores, NR_CPUS);
 
 		ibm_architecture_vec.vec5.max_cpus = cpu_to_be32(cores);
@@ -1242,7 +1243,7 @@ static unsigned long __init alloc_up(unsigned long size, unsigned long align)
 
 	if (align)
 		base = _ALIGN_UP(base, align);
-	prom_debug("alloc_up(%x, %x)\n", size, align);
+	prom_debug("%s(%lx, %lx)\n", __func__, size, align);
 	if (ram_top == 0)
 		prom_panic("alloc_up() called with mem not initialized\n");
 
@@ -1253,7 +1254,7 @@ static unsigned long __init alloc_up(unsigned long size, unsigned long align)
 
 	for(; (base + size) <= alloc_top; 
 	    base = _ALIGN_UP(base + 0x100000, align)) {
-		prom_debug("    trying: 0x%x\n\r", base);
+		prom_debug("    trying: 0x%lx\n\r", base);
 		addr = (unsigned long)prom_claim(base, size, 0);
 		if (addr != PROM_ERROR && addr != 0)
 			break;
@@ -1265,12 +1266,12 @@ static unsigned long __init alloc_up(unsigned long size, unsigned long align)
 		return 0;
 	alloc_bottom = addr + size;
 
-	prom_debug(" -> %x\n", addr);
-	prom_debug("  alloc_bottom : %x\n", alloc_bottom);
-	prom_debug("  alloc_top    : %x\n", alloc_top);
-	prom_debug("  alloc_top_hi : %x\n", alloc_top_high);
-	prom_debug("  rmo_top      : %x\n", rmo_top);
-	prom_debug("  ram_top      : %x\n", ram_top);
+	prom_debug(" -> %lx\n", addr);
+	prom_debug("  alloc_bottom : %lx\n", alloc_bottom);
+	prom_debug("  alloc_top    : %lx\n", alloc_top);
+	prom_debug("  alloc_top_hi : %lx\n", alloc_top_high);
+	prom_debug("  rmo_top      : %lx\n", rmo_top);
+	prom_debug("  ram_top      : %lx\n", ram_top);
 
 	return addr;
 }
@@ -1285,7 +1286,7 @@ static unsigned long __init alloc_down(unsigned long size, unsigned long align,
 {
 	unsigned long base, addr = 0;
 
-	prom_debug("alloc_down(%x, %x, %s)\n", size, align,
+	prom_debug("%s(%lx, %lx, %s)\n", __func__, size, align,
 		   highmem ? "(high)" : "(low)");
 	if (ram_top == 0)
 		prom_panic("alloc_down() called with mem not initialized\n");
@@ -1313,7 +1314,7 @@ static unsigned long __init alloc_down(unsigned long size, unsigned long align,
 	base = _ALIGN_DOWN(alloc_top - size, align);
 	for (; base > alloc_bottom;
 	     base = _ALIGN_DOWN(base - 0x100000, align))  {
-		prom_debug("    trying: 0x%x\n\r", base);
+		prom_debug("    trying: 0x%lx\n\r", base);
 		addr = (unsigned long)prom_claim(base, size, 0);
 		if (addr != PROM_ERROR && addr != 0)
 			break;
@@ -1324,12 +1325,12 @@ static unsigned long __init alloc_down(unsigned long size, unsigned long align,
 	alloc_top = addr;
 
  bail:
-	prom_debug(" -> %x\n", addr);
-	prom_debug("  alloc_bottom : %x\n", alloc_bottom);
-	prom_debug("  alloc_top    : %x\n", alloc_top);
-	prom_debug("  alloc_top_hi : %x\n", alloc_top_high);
-	prom_debug("  rmo_top      : %x\n", rmo_top);
-	prom_debug("  ram_top      : %x\n", ram_top);
+	prom_debug(" -> %lx\n", addr);
+	prom_debug("  alloc_bottom : %lx\n", alloc_bottom);
+	prom_debug("  alloc_top    : %lx\n", alloc_top);
+	prom_debug("  alloc_top_hi : %lx\n", alloc_top_high);
+	prom_debug("  rmo_top      : %lx\n", rmo_top);
+	prom_debug("  ram_top      : %lx\n", ram_top);
 
 	return addr;
 }
@@ -1455,7 +1456,7 @@ static void __init prom_init_mem(void)
 
 			if (size == 0)
 				continue;
-			prom_debug("    %x %x\n", base, size);
+			prom_debug("    %lx %lx\n", base, size);
 			if (base == 0 && (of_platform & PLATFORM_LPAR))
 				rmo_top = size;
 			if ((base + size) > ram_top)
@@ -1475,12 +1476,12 @@ static void __init prom_init_mem(void)
 
 	if (prom_memory_limit) {
 		if (prom_memory_limit <= alloc_bottom) {
-			prom_printf("Ignoring mem=%x <= alloc_bottom.\n",
-				prom_memory_limit);
+			prom_printf("Ignoring mem=%lx <= alloc_bottom.\n",
+				    prom_memory_limit);
 			prom_memory_limit = 0;
 		} else if (prom_memory_limit >= ram_top) {
-			prom_printf("Ignoring mem=%x >= ram_top.\n",
-				prom_memory_limit);
+			prom_printf("Ignoring mem=%lx >= ram_top.\n",
+				    prom_memory_limit);
 			prom_memory_limit = 0;
 		} else {
 			ram_top = prom_memory_limit;
@@ -1512,12 +1513,13 @@ static void __init prom_init_mem(void)
 		alloc_bottom = PAGE_ALIGN(prom_initrd_end);
 
 	prom_printf("memory layout at init:\n");
-	prom_printf("  memory_limit : %x (16 MB aligned)\n", prom_memory_limit);
-	prom_printf("  alloc_bottom : %x\n", alloc_bottom);
-	prom_printf("  alloc_top    : %x\n", alloc_top);
-	prom_printf("  alloc_top_hi : %x\n", alloc_top_high);
-	prom_printf("  rmo_top      : %x\n", rmo_top);
-	prom_printf("  ram_top      : %x\n", ram_top);
+	prom_printf("  memory_limit : %lx (16 MB aligned)\n",
+		    prom_memory_limit);
+	prom_printf("  alloc_bottom : %lx\n", alloc_bottom);
+	prom_printf("  alloc_top    : %lx\n", alloc_top);
+	prom_printf("  alloc_top_hi : %lx\n", alloc_top_high);
+	prom_printf("  rmo_top      : %lx\n", rmo_top);
+	prom_printf("  ram_top      : %lx\n", ram_top);
 }
 
 static void __init prom_close_stdin(void)
@@ -1578,7 +1580,7 @@ static void __init prom_instantiate_opal(void)
 		return;
 	}
 
-	prom_printf("instantiating opal at 0x%x...", base);
+	prom_printf("instantiating opal at 0x%llx...", base);
 
 	if (call_prom_ret("call-method", 4, 3, rets,
 			  ADDR("load-opal-runtime"),
@@ -1594,10 +1596,10 @@ static void __init prom_instantiate_opal(void)
 
 	reserve_mem(base, size);
 
-	prom_debug("opal base     = 0x%x\n", base);
-	prom_debug("opal align    = 0x%x\n", align);
-	prom_debug("opal entry    = 0x%x\n", entry);
-	prom_debug("opal size     = 0x%x\n", (long)size);
+	prom_debug("opal base     = 0x%llx\n", base);
+	prom_debug("opal align    = 0x%llx\n", align);
+	prom_debug("opal entry    = 0x%llx\n", entry);
+	prom_debug("opal size     = 0x%llx\n", size);
 
 	prom_setprop(opal_node, "/ibm,opal", "opal-base-address",
 		     &base, sizeof(base));
@@ -1674,7 +1676,7 @@ static void __init prom_instantiate_rtas(void)
 
 	prom_debug("rtas base     = 0x%x\n", base);
 	prom_debug("rtas entry    = 0x%x\n", entry);
-	prom_debug("rtas size     = 0x%x\n", (long)size);
+	prom_debug("rtas size     = 0x%x\n", size);
 
 	prom_debug("prom_instantiate_rtas: end...\n");
 }
@@ -1732,7 +1734,7 @@ static void __init prom_instantiate_sml(void)
 	if (base == 0)
 		prom_panic("Could not allocate memory for sml\n");
 
-	prom_printf("instantiating sml at 0x%x...", base);
+	prom_printf("instantiating sml at 0x%llx...", base);
 
 	memset((void *)base, 0, size);
 
@@ -1751,8 +1753,8 @@ static void __init prom_instantiate_sml(void)
 	prom_setprop(ibmvtpm_node, "/vdevice/vtpm", "linux,sml-size",
 		     &size, sizeof(size));
 
-	prom_debug("sml base     = 0x%x\n", base);
-	prom_debug("sml size     = 0x%x\n", (long)size);
+	prom_debug("sml base     = 0x%llx\n", base);
+	prom_debug("sml size     = 0x%x\n", size);
 
 	prom_debug("prom_instantiate_sml: end...\n");
 }
@@ -1845,7 +1847,7 @@ static void __init prom_initialize_tce_table(void)
 
 		prom_debug("TCE table: %s\n", path);
 		prom_debug("\tnode = 0x%x\n", node);
-		prom_debug("\tbase = 0x%x\n", base);
+		prom_debug("\tbase = 0x%llx\n", base);
 		prom_debug("\tsize = 0x%x\n", minsize);
 
 		/* Initialize the table to have a one-to-one mapping
@@ -1932,12 +1934,12 @@ static void __init prom_hold_cpus(void)
 	}
 
 	prom_debug("prom_hold_cpus: start...\n");
-	prom_debug("    1) spinloop       = 0x%x\n", (unsigned long)spinloop);
-	prom_debug("    1) *spinloop      = 0x%x\n", *spinloop);
-	prom_debug("    1) acknowledge    = 0x%x\n",
+	prom_debug("    1) spinloop       = 0x%lx\n", (unsigned long)spinloop);
+	prom_debug("    1) *spinloop      = 0x%lx\n", *spinloop);
+	prom_debug("    1) acknowledge    = 0x%lx\n",
 		   (unsigned long)acknowledge);
-	prom_debug("    1) *acknowledge   = 0x%x\n", *acknowledge);
-	prom_debug("    1) secondary_hold = 0x%x\n", secondary_hold);
+	prom_debug("    1) *acknowledge   = 0x%lx\n", *acknowledge);
+	prom_debug("    1) secondary_hold = 0x%lx\n", secondary_hold);
 
 	/* Set the common spinloop variable, so all of the secondary cpus
 	 * will block when they are awakened from their OF spinloop.
@@ -1965,7 +1967,7 @@ static void __init prom_hold_cpus(void)
 		prom_getprop(node, "reg", &reg, sizeof(reg));
 		cpu_no = be32_to_cpu(reg);
 
-		prom_debug("cpu hw idx   = %lu\n", cpu_no);
+		prom_debug("cpu hw idx   = %u\n", cpu_no);
 
 		/* Init the acknowledge var which will be reset by
 		 * the secondary cpu when it awakens from its OF
@@ -1975,7 +1977,7 @@ static void __init prom_hold_cpus(void)
 
 		if (cpu_no != prom.cpu) {
 			/* Primary Thread of non-boot cpu or any thread */
-			prom_printf("starting cpu hw idx %lu... ", cpu_no);
+			prom_printf("starting cpu hw idx %u... ", cpu_no);
 			call_prom("start-cpu", 3, 0, node,
 				  secondary_hold, cpu_no);
 
@@ -1986,11 +1988,11 @@ static void __init prom_hold_cpus(void)
 			if (*acknowledge == cpu_no)
 				prom_printf("done\n");
 			else
-				prom_printf("failed: %x\n", *acknowledge);
+				prom_printf("failed: %lx\n", *acknowledge);
 		}
 #ifdef CONFIG_SMP
 		else
-			prom_printf("boot cpu hw idx %lu\n", cpu_no);
+			prom_printf("boot cpu hw idx %u\n", cpu_no);
 #endif /* CONFIG_SMP */
 	}
 
@@ -2268,7 +2270,7 @@ static void __init *make_room(unsigned long *mem_start, unsigned long *mem_end,
 	while ((*mem_start + needed) > *mem_end) {
 		unsigned long room, chunk;
 
-		prom_debug("Chunk exhausted, claiming more at %x...\n",
+		prom_debug("Chunk exhausted, claiming more at %lx...\n",
 			   alloc_bottom);
 		room = alloc_top - alloc_bottom;
 		if (room > DEVTREE_CHUNK_SIZE)
@@ -2494,7 +2496,7 @@ static void __init flatten_device_tree(void)
 	room = alloc_top - alloc_bottom - 0x4000;
 	if (room > DEVTREE_CHUNK_SIZE)
 		room = DEVTREE_CHUNK_SIZE;
-	prom_debug("starting device tree allocs at %x\n", alloc_bottom);
+	prom_debug("starting device tree allocs at %lx\n", alloc_bottom);
 
 	/* Now try to claim that */
 	mem_start = (unsigned long)alloc_up(room, PAGE_SIZE);
@@ -2557,7 +2559,7 @@ static void __init flatten_device_tree(void)
 		int i;
 		prom_printf("reserved memory map:\n");
 		for (i = 0; i < mem_reserve_cnt; i++)
-			prom_printf("  %x - %x\n",
+			prom_printf("  %llx - %llx\n",
 				    be64_to_cpu(mem_reserve_map[i].base),
 				    be64_to_cpu(mem_reserve_map[i].size));
 	}
@@ -2567,9 +2569,9 @@ static void __init flatten_device_tree(void)
 	 */
 	mem_reserve_cnt = MEM_RESERVE_MAP_SIZE;
 
-	prom_printf("Device tree strings 0x%x -> 0x%x\n",
+	prom_printf("Device tree strings 0x%lx -> 0x%lx\n",
 		    dt_string_start, dt_string_end);
-	prom_printf("Device tree struct  0x%x -> 0x%x\n",
+	prom_printf("Device tree struct  0x%lx -> 0x%lx\n",
 		    dt_struct_start, dt_struct_end);
 }
 
@@ -3001,7 +3003,7 @@ static void __init prom_find_boot_cpu(void)
 	prom_getprop(cpu_pkg, "reg", &rval, sizeof(rval));
 	prom.cpu = be32_to_cpu(rval);
 
-	prom_debug("Booting CPU hw index = %lu\n", prom.cpu);
+	prom_debug("Booting CPU hw index = %d\n", prom.cpu);
 }
 
 static void __init prom_check_initrd(unsigned long r3, unsigned long r4)
@@ -3023,8 +3025,8 @@ static void __init prom_check_initrd(unsigned long r3, unsigned long r4)
 		reserve_mem(prom_initrd_start,
 			    prom_initrd_end - prom_initrd_start);
 
-		prom_debug("initrd_start=0x%x\n", prom_initrd_start);
-		prom_debug("initrd_end=0x%x\n", prom_initrd_end);
+		prom_debug("initrd_start=0x%lx\n", prom_initrd_start);
+		prom_debug("initrd_end=0x%lx\n", prom_initrd_end);
 	}
 #endif /* CONFIG_BLK_DEV_INITRD */
 }
@@ -3277,7 +3279,7 @@ unsigned long __init prom_init(unsigned long r3, unsigned long r4,
 	/* Don't print anything after quiesce under OPAL, it crashes OFW */
 	if (of_platform != PLATFORM_OPAL) {
 		prom_printf("Booting Linux via __start() @ 0x%lx ...\n", kbase);
-		prom_debug("->dt_header_start=0x%x\n", hdr);
+		prom_debug("->dt_header_start=0x%lx\n", hdr);
 	}
 
 #ifdef CONFIG_PPC32
diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c
index d23cf632edf0..0f63dd5972e9 100644
--- a/arch/powerpc/kernel/ptrace.c
+++ b/arch/powerpc/kernel/ptrace.c
@@ -2443,6 +2443,7 @@ static int ptrace_set_debugreg(struct task_struct *task, unsigned long addr,
 	/* Create a new breakpoint request if one doesn't exist already */
 	hw_breakpoint_init(&attr);
 	attr.bp_addr = hw_brk.address;
+	attr.bp_len = 8;
 	arch_bp_generic_fields(hw_brk.type,
 			       &attr.bp_type);
 
diff --git a/arch/powerpc/kernel/rtas-proc.c b/arch/powerpc/kernel/rtas-proc.c
index fb070d8cad07..669177fe3041 100644
--- a/arch/powerpc/kernel/rtas-proc.c
+++ b/arch/powerpc/kernel/rtas-proc.c
@@ -154,18 +154,6 @@ static ssize_t ppc_rtas_tone_volume_write(struct file *file,
 static int ppc_rtas_tone_volume_show(struct seq_file *m, void *v);
 static int ppc_rtas_rmo_buf_show(struct seq_file *m, void *v);
 
-static int sensors_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ppc_rtas_sensors_show, NULL);
-}
-
-static const struct file_operations ppc_rtas_sensors_operations = {
-	.open		= sensors_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int poweron_open(struct inode *inode, struct file *file)
 {
 	return single_open(file, ppc_rtas_poweron_show, NULL);
@@ -231,18 +219,6 @@ static const struct file_operations ppc_rtas_tone_volume_operations = {
 	.release	= single_release,
 };
 
-static int rmo_buf_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ppc_rtas_rmo_buf_show, NULL);
-}
-
-static const struct file_operations ppc_rtas_rmo_buf_ops = {
-	.open		= rmo_buf_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int ppc_rtas_find_all_sensors(void);
 static void ppc_rtas_process_sensor(struct seq_file *m,
 	struct individual_sensor *s, int state, int error, const char *loc);
@@ -267,14 +243,14 @@ static int __init proc_rtas_init(void)
 		    &ppc_rtas_clock_operations);
 	proc_create("powerpc/rtas/poweron", 0644, NULL,
 		    &ppc_rtas_poweron_operations);
-	proc_create("powerpc/rtas/sensors", 0444, NULL,
-		    &ppc_rtas_sensors_operations);
+	proc_create_single("powerpc/rtas/sensors", 0444, NULL,
+			ppc_rtas_sensors_show);
 	proc_create("powerpc/rtas/frequency", 0644, NULL,
 		    &ppc_rtas_tone_freq_operations);
 	proc_create("powerpc/rtas/volume", 0644, NULL,
 		    &ppc_rtas_tone_volume_operations);
-	proc_create("powerpc/rtas/rmo_buffer", 0400, NULL,
-		    &ppc_rtas_rmo_buf_ops);
+	proc_create_single("powerpc/rtas/rmo_buffer", 0400, NULL,
+			ppc_rtas_rmo_buf_show);
 	return 0;
 }
 
@@ -528,7 +504,7 @@ static void ppc_rtas_process_sensor(struct seq_file *m,
 		"EPOW power off" };
 	const char * battery_cyclestate[]  = { "None", "In progress", 
 						"Requested" };
-	const char * battery_charging[]    = { "Charging", "Discharching", 
+	const char * battery_charging[]    = { "Charging", "Discharging",
 						"No current flow" };
 	const char * ibm_drconnector[]     = { "Empty", "Present", "Unusable", 
 						"Exchange" };
diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c
index 3f1c4fcbe0aa..8afd146bc9c7 100644
--- a/arch/powerpc/kernel/rtas.c
+++ b/arch/powerpc/kernel/rtas.c
@@ -26,6 +26,7 @@
 #include <linux/memblock.h>
 #include <linux/slab.h>
 #include <linux/reboot.h>
+#include <linux/syscalls.h>
 
 #include <asm/prom.h>
 #include <asm/rtas.h>
@@ -1050,7 +1051,7 @@ struct pseries_errorlog *get_pseries_errorlog(struct rtas_error_log *log,
 }
 
 /* We assume to be passed big endian arguments */
-asmlinkage int ppc_rtas(struct rtas_args __user *uargs)
+SYSCALL_DEFINE1(rtas, struct rtas_args __user *, uargs)
 {
 	struct rtas_args args;
 	unsigned long flags;
diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c
index 0af5c11b9e78..62b1a40d8957 100644
--- a/arch/powerpc/kernel/setup-common.c
+++ b/arch/powerpc/kernel/setup-common.c
@@ -192,12 +192,6 @@ void machine_halt(void)
 	machine_hang();
 }
 
-
-#ifdef CONFIG_TAU
-extern u32 cpu_temp(unsigned long cpu);
-extern u32 cpu_temp_both(unsigned long cpu);
-#endif /* CONFIG_TAU */
-
 #ifdef CONFIG_SMP
 DEFINE_PER_CPU(unsigned int, cpu_pvr);
 #endif
diff --git a/arch/powerpc/kernel/setup.h b/arch/powerpc/kernel/setup.h
index d144df54ad40..c6a592b67386 100644
--- a/arch/powerpc/kernel/setup.h
+++ b/arch/powerpc/kernel/setup.h
@@ -62,4 +62,10 @@ void kvm_cma_reserve(void);
 static inline void kvm_cma_reserve(void) { };
 #endif
 
+#ifdef CONFIG_TAU
+u32 cpu_temp(unsigned long cpu);
+u32 cpu_temp_both(unsigned long cpu);
+u32 tau_interrupts(unsigned long cpu);
+#endif /* CONFIG_TAU */
+
 #endif /* __ARCH_POWERPC_KERNEL_SETUP_H */
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index b78f142a4148..7a7ce8ad455e 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -346,6 +346,13 @@ void __init early_setup(unsigned long dt_ptr)
 	 */
 	cpu_ready_for_interrupts();
 
+	/*
+	 * We enable ftrace here, but since we only support DYNAMIC_FTRACE, it
+	 * will only actually get enabled on the boot cpu much later once
+	 * ftrace itself has been initialized.
+	 */
+	this_cpu_enable_ftrace();
+
 	DBG(" <- early_setup()\n");
 
 #ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX
diff --git a/arch/powerpc/kernel/signal.h b/arch/powerpc/kernel/signal.h
index a6467f843acf..800433685888 100644
--- a/arch/powerpc/kernel/signal.h
+++ b/arch/powerpc/kernel/signal.h
@@ -49,10 +49,8 @@ extern int handle_rt_signal64(struct ksignal *ksig, sigset_t *set,
 
 #else /* CONFIG_PPC64 */
 
-extern long sys_rt_sigreturn(int r3, int r4, int r5, int r6, int r7, int r8,
-		     struct pt_regs *regs);
-extern long sys_sigreturn(int r3, int r4, int r5, int r6, int r7, int r8,
-		       struct pt_regs *regs);
+extern long sys_rt_sigreturn(void);
+extern long sys_sigreturn(void);
 
 static inline int handle_rt_signal64(struct ksignal *ksig, sigset_t *set,
 				     struct task_struct *tsk)
diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c
index 492f03451877..9cf8a03d3bc7 100644
--- a/arch/powerpc/kernel/signal_32.c
+++ b/arch/powerpc/kernel/signal_32.c
@@ -26,8 +26,8 @@
 #include <linux/elf.h>
 #include <linux/ptrace.h>
 #include <linux/ratelimit.h>
-#ifdef CONFIG_PPC64
 #include <linux/syscalls.h>
+#ifdef CONFIG_PPC64
 #include <linux/compat.h>
 #else
 #include <linux/wait.h>
@@ -57,10 +57,6 @@
 
 
 #ifdef CONFIG_PPC64
-#define sys_rt_sigreturn	compat_sys_rt_sigreturn
-#define sys_swapcontext	compat_sys_swapcontext
-#define sys_sigreturn	compat_sys_sigreturn
-
 #define old_sigaction	old_sigaction32
 #define sigcontext	sigcontext32
 #define mcontext	mcontext32
@@ -1041,10 +1037,15 @@ static int do_setcontext_tm(struct ucontext __user *ucp,
 }
 #endif
 
-long sys_swapcontext(struct ucontext __user *old_ctx,
-		     struct ucontext __user *new_ctx,
-		     int ctx_size, int r6, int r7, int r8, struct pt_regs *regs)
+#ifdef CONFIG_PPC64
+COMPAT_SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx,
+		       struct ucontext __user *, new_ctx, int, ctx_size)
+#else
+SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx,
+		       struct ucontext __user *, new_ctx, long, ctx_size)
+#endif
 {
+	struct pt_regs *regs = current_pt_regs();
 	unsigned char tmp __maybe_unused;
 	int ctx_has_vsx_region = 0;
 
@@ -1132,10 +1133,14 @@ long sys_swapcontext(struct ucontext __user *old_ctx,
 	return 0;
 }
 
-long sys_rt_sigreturn(int r3, int r4, int r5, int r6, int r7, int r8,
-		     struct pt_regs *regs)
+#ifdef CONFIG_PPC64
+COMPAT_SYSCALL_DEFINE0(rt_sigreturn)
+#else
+SYSCALL_DEFINE0(rt_sigreturn)
+#endif
 {
 	struct rt_sigframe __user *rt_sf;
+	struct pt_regs *regs = current_pt_regs();
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 	struct ucontext __user *uc_transact;
 	unsigned long msr_hi;
@@ -1224,11 +1229,10 @@ long sys_rt_sigreturn(int r3, int r4, int r5, int r6, int r7, int r8,
 }
 
 #ifdef CONFIG_PPC32
-int sys_debug_setcontext(struct ucontext __user *ctx,
-			 int ndbg, struct sig_dbg_op __user *dbg,
-			 int r6, int r7, int r8,
-			 struct pt_regs *regs)
+SYSCALL_DEFINE3(debug_setcontext, struct ucontext __user *, ctx,
+			 int, ndbg, struct sig_dbg_op __user *, dbg)
 {
+	struct pt_regs *regs = current_pt_regs();
 	struct sig_dbg_op op;
 	int i;
 	unsigned char tmp __maybe_unused;
@@ -1419,9 +1423,13 @@ int handle_signal32(struct ksignal *ksig, sigset_t *oldset,
 /*
  * Do a signal return; undo the signal stack.
  */
-long sys_sigreturn(int r3, int r4, int r5, int r6, int r7, int r8,
-		       struct pt_regs *regs)
+#ifdef CONFIG_PPC64
+COMPAT_SYSCALL_DEFINE0(sigreturn)
+#else
+SYSCALL_DEFINE0(sigreturn)
+#endif
 {
+	struct pt_regs *regs = current_pt_regs();
 	struct sigframe __user *sf;
 	struct sigcontext __user *sc;
 	struct sigcontext sigctx;
diff --git a/arch/powerpc/kernel/signal_64.c b/arch/powerpc/kernel/signal_64.c
index 720117690822..83d51bf586c7 100644
--- a/arch/powerpc/kernel/signal_64.c
+++ b/arch/powerpc/kernel/signal_64.c
@@ -24,6 +24,7 @@
 #include <linux/elf.h>
 #include <linux/ptrace.h>
 #include <linux/ratelimit.h>
+#include <linux/syscalls.h>
 
 #include <asm/sigcontext.h>
 #include <asm/ucontext.h>
@@ -624,17 +625,14 @@ static long setup_trampoline(unsigned int syscall, unsigned int __user *tramp)
 /*
  * Handle {get,set,swap}_context operations
  */
-int sys_swapcontext(struct ucontext __user *old_ctx,
-		    struct ucontext __user *new_ctx,
-		    long ctx_size, long r6, long r7, long r8, struct pt_regs *regs)
+SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx,
+		struct ucontext __user *, new_ctx, long, ctx_size)
 {
 	unsigned char tmp;
 	sigset_t set;
 	unsigned long new_msr = 0;
 	int ctx_has_vsx_region = 0;
 
-	BUG_ON(regs != current->thread.regs);
-
 	if (new_ctx &&
 	    get_user(new_msr, &new_ctx->uc_mcontext.gp_regs[PT_MSR]))
 		return -EFAULT;
@@ -698,18 +696,15 @@ int sys_swapcontext(struct ucontext __user *old_ctx,
  * Do a signal return; undo the signal stack.
  */
 
-int sys_rt_sigreturn(unsigned long r3, unsigned long r4, unsigned long r5,
-		     unsigned long r6, unsigned long r7, unsigned long r8,
-		     struct pt_regs *regs)
+SYSCALL_DEFINE0(rt_sigreturn)
 {
+	struct pt_regs *regs = current_pt_regs();
 	struct ucontext __user *uc = (struct ucontext __user *)regs->gpr[1];
 	sigset_t set;
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 	unsigned long msr;
 #endif
 
-	BUG_ON(current->thread.regs != regs);
-
 	/* Always make any pending restarted system calls return -EINTR */
 	current->restart_block.fn = do_no_restart_syscall;
 
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 9ca7148b5881..c96f8fbc1942 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -59,6 +59,7 @@
 #include <asm/kexec.h>
 #include <asm/asm-prototypes.h>
 #include <asm/cpu_has_feature.h>
+#include <asm/ftrace.h>
 
 #ifdef DEBUG
 #include <asm/udbg.h>
@@ -1066,6 +1067,9 @@ void start_secondary(void *unused)
 
 	local_irq_enable();
 
+	/* We can enable ftrace for secondary cpus now */
+	this_cpu_enable_ftrace();
+
 	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 
 	BUG();
@@ -1162,6 +1166,8 @@ int __cpu_disable(void)
 	if (!smp_ops->cpu_disable)
 		return -ENOSYS;
 
+	this_cpu_disable_ftrace();
+
 	err = smp_ops->cpu_disable();
 	if (err)
 		return err;
@@ -1180,6 +1186,12 @@ void __cpu_die(unsigned int cpu)
 
 void cpu_die(void)
 {
+	/*
+	 * Disable on the down path. This will be re-enabled by
+	 * start_secondary() via start_secondary_resume() below
+	 */
+	this_cpu_disable_ftrace();
+
 	if (ppc_md.cpu_die)
 		ppc_md.cpu_die();
 
diff --git a/arch/powerpc/kernel/stacktrace.c b/arch/powerpc/kernel/stacktrace.c
index d534ed901538..26a50603177c 100644
--- a/arch/powerpc/kernel/stacktrace.c
+++ b/arch/powerpc/kernel/stacktrace.c
@@ -2,7 +2,7 @@
  * Stack trace utility
  *
  * Copyright 2008 Christoph Hellwig, IBM Corp.
- *
+ * Copyright 2018 SUSE Linux GmbH
  *
  *      This program is free software; you can redistribute it and/or
  *      modify it under the terms of the GNU General Public License
@@ -11,11 +11,16 @@
  */
 
 #include <linux/export.h>
+#include <linux/kallsyms.h>
+#include <linux/module.h>
 #include <linux/sched.h>
 #include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
 #include <linux/stacktrace.h>
 #include <asm/ptrace.h>
 #include <asm/processor.h>
+#include <linux/ftrace.h>
+#include <asm/kprobes.h>
 
 /*
  * Save stack-backtrace addresses into a stack_trace buffer.
@@ -76,3 +81,115 @@ save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace)
 	save_context_stack(trace, regs->gpr[1], current, 0);
 }
 EXPORT_SYMBOL_GPL(save_stack_trace_regs);
+
+#ifdef CONFIG_HAVE_RELIABLE_STACKTRACE
+int
+save_stack_trace_tsk_reliable(struct task_struct *tsk,
+				struct stack_trace *trace)
+{
+	unsigned long sp;
+	unsigned long stack_page = (unsigned long)task_stack_page(tsk);
+	unsigned long stack_end;
+	int graph_idx = 0;
+
+	/*
+	 * The last frame (unwinding first) may not yet have saved
+	 * its LR onto the stack.
+	 */
+	int firstframe = 1;
+
+	if (tsk == current)
+		sp = current_stack_pointer();
+	else
+		sp = tsk->thread.ksp;
+
+	stack_end = stack_page + THREAD_SIZE;
+	if (!is_idle_task(tsk)) {
+		/*
+		 * For user tasks, this is the SP value loaded on
+		 * kernel entry, see "PACAKSAVE(r13)" in _switch() and
+		 * system_call_common()/EXCEPTION_PROLOG_COMMON().
+		 *
+		 * Likewise for non-swapper kernel threads,
+		 * this also happens to be the top of the stack
+		 * as setup by copy_thread().
+		 *
+		 * Note that stack backlinks are not properly setup by
+		 * copy_thread() and thus, a forked task() will have
+		 * an unreliable stack trace until it's been
+		 * _switch()'ed to for the first time.
+		 */
+		stack_end -= STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);
+	} else {
+		/*
+		 * idle tasks have a custom stack layout,
+		 * c.f. cpu_idle_thread_init().
+		 */
+		stack_end -= STACK_FRAME_OVERHEAD;
+	}
+
+	if (sp < stack_page + sizeof(struct thread_struct) ||
+	    sp > stack_end - STACK_FRAME_MIN_SIZE) {
+		return 1;
+	}
+
+	for (;;) {
+		unsigned long *stack = (unsigned long *) sp;
+		unsigned long newsp, ip;
+
+		/* sanity check: ABI requires SP to be aligned 16 bytes. */
+		if (sp & 0xF)
+			return 1;
+
+		/* Mark stacktraces with exception frames as unreliable. */
+		if (sp <= stack_end - STACK_INT_FRAME_SIZE &&
+		    stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
+			return 1;
+		}
+
+		newsp = stack[0];
+		/* Stack grows downwards; unwinder may only go up. */
+		if (newsp <= sp)
+			return 1;
+
+		if (newsp != stack_end &&
+		    newsp > stack_end - STACK_FRAME_MIN_SIZE) {
+			return 1; /* invalid backlink, too far up. */
+		}
+
+		/* Examine the saved LR: it must point into kernel code. */
+		ip = stack[STACK_FRAME_LR_SAVE];
+		if (!firstframe && !__kernel_text_address(ip))
+			return 1;
+		firstframe = 0;
+
+		/*
+		 * FIXME: IMHO these tests do not belong in
+		 * arch-dependent code, they are generic.
+		 */
+		ip = ftrace_graph_ret_addr(tsk, &graph_idx, ip, NULL);
+
+		/*
+		 * Mark stacktraces with kretprobed functions on them
+		 * as unreliable.
+		 */
+		if (ip == (unsigned long)kretprobe_trampoline)
+			return 1;
+
+		if (!trace->skip)
+			trace->entries[trace->nr_entries++] = ip;
+		else
+			trace->skip--;
+
+		if (newsp == stack_end)
+			break;
+
+		if (trace->nr_entries >= trace->max_entries)
+			return -E2BIG;
+
+		sp = newsp;
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(save_stack_trace_tsk_reliable);
+#endif /* CONFIG_HAVE_RELIABLE_STACKTRACE */
diff --git a/arch/powerpc/kernel/sys_ppc32.c b/arch/powerpc/kernel/sys_ppc32.c
index c11c73373691..bdf58ba1a94b 100644
--- a/arch/powerpc/kernel/sys_ppc32.c
+++ b/arch/powerpc/kernel/sys_ppc32.c
@@ -52,15 +52,6 @@
 #include <asm/syscalls.h>
 #include <asm/switch_to.h>
 
-
-asmlinkage long ppc32_select(u32 n, compat_ulong_t __user *inp,
-		compat_ulong_t __user *outp, compat_ulong_t __user *exp,
-		compat_uptr_t tvp_x)
-{
-	/* sign extend n */
-	return compat_sys_select((int)n, inp, outp, exp, compat_ptr(tvp_x));
-}
-
 unsigned long compat_sys_mmap2(unsigned long addr, size_t len,
 			  unsigned long prot, unsigned long flags,
 			  unsigned long fd, unsigned long pgoff)
diff --git a/arch/powerpc/kernel/systbl.S b/arch/powerpc/kernel/systbl.S
index 7ccb7f81f8db..919a32746ede 100644
--- a/arch/powerpc/kernel/systbl.S
+++ b/arch/powerpc/kernel/systbl.S
@@ -35,7 +35,7 @@
 #endif
 #define SYSCALL_SPU(func)	SYSCALL(func)
 #define COMPAT_SYS_SPU(func)	COMPAT_SYS(func)
-#define PPC_SYS_SPU(func)	PPC_SYS(func)
+#define COMPAT_SPU_NEW(func)	COMPAT_SYS(func)
 #define SYSX_SPU(f, f3264, f32)	SYSX(f, f3264, f32)
 
 .section .rodata,"a"
diff --git a/arch/powerpc/kernel/systbl_chk.c b/arch/powerpc/kernel/systbl_chk.c
index 55323a620cfe..4653258722ac 100644
--- a/arch/powerpc/kernel/systbl_chk.c
+++ b/arch/powerpc/kernel/systbl_chk.c
@@ -31,7 +31,7 @@
 
 #define SYSCALL_SPU(func)	SYSCALL(func)
 #define COMPAT_SYS_SPU(func)	COMPAT_SYS(func)
-#define PPC_SYS_SPU(func)	PPC_SYS(func)
+#define COMPAT_SPU_NEW(func)	COMPAT_SYS(_new##func)
 #define SYSX_SPU(f, f3264, f32)	SYSX(f, f3264, f32)
 
 /* Just insert a marker for ni_syscalls */
diff --git a/arch/powerpc/kernel/systbl_chk.sh b/arch/powerpc/kernel/systbl_chk.sh
index 31b6e7c358ca..f2e356c2a345 100644
--- a/arch/powerpc/kernel/systbl_chk.sh
+++ b/arch/powerpc/kernel/systbl_chk.sh
@@ -16,7 +16,7 @@ awk	'BEGIN { num = -1; }	# Ignore the beginning of the file
 	/^START_TABLE/ { num = 0; next; }
 	/^END_TABLE/ {
 		if (num != $2) {
-			printf "NR_syscalls (%s) is not one more than the last syscall (%s)\n",
+			printf "Error: NR_syscalls (%s) is not one more than the last syscall (%s)\n",
 				$2, num - 1;
 			exit(1);
 		}
@@ -25,7 +25,7 @@ awk	'BEGIN { num = -1; }	# Ignore the beginning of the file
 	{
 		if (num == -1) next;
 		if (($1 != -1) && ($1 != num)) {
-			printf "Syscall %s out of order (expected %s)\n",
+			printf "Error: Syscall %s out of order (expected %s)\n",
 				$1, num;
 			exit(1);
 		};
diff --git a/arch/powerpc/kernel/tau_6xx.c b/arch/powerpc/kernel/tau_6xx.c
index 8cdd852aedd1..e2ab8a111b69 100644
--- a/arch/powerpc/kernel/tau_6xx.c
+++ b/arch/powerpc/kernel/tau_6xx.c
@@ -27,6 +27,9 @@
 #include <asm/cache.h>
 #include <asm/8xx_immap.h>
 #include <asm/machdep.h>
+#include <asm/asm-prototypes.h>
+
+#include "setup.h"
 
 static struct tau_temp
 {
@@ -50,7 +53,7 @@ struct timer_list tau_timer;
 #define shrink_timer	2*HZ	/* period between shrinking the window */
 #define min_window	2	/* minimum window size, degrees C */
 
-void set_thresholds(unsigned long cpu)
+static void set_thresholds(unsigned long cpu)
 {
 #ifdef CONFIG_TAU_INT
 	/*
@@ -70,7 +73,7 @@ void set_thresholds(unsigned long cpu)
 #endif
 }
 
-void TAUupdate(int cpu)
+static void TAUupdate(int cpu)
 {
 	unsigned thrm;
 
@@ -205,7 +208,7 @@ static void tau_timeout_smp(struct timer_list *unused)
 
 int tau_initialized = 0;
 
-void __init TAU_init_smp(void * info)
+static void __init TAU_init_smp(void *info)
 {
 	unsigned long cpu = smp_processor_id();
 
@@ -217,7 +220,7 @@ void __init TAU_init_smp(void * info)
 	set_thresholds(cpu);
 }
 
-int __init TAU_init(void)
+static int __init TAU_init(void)
 {
 	/* We assume in SMP that if one CPU has TAU support, they
 	 * all have it --BenH
@@ -259,12 +262,12 @@ u32 cpu_temp_both(unsigned long cpu)
 	return ((tau[cpu].high << 16) | tau[cpu].low);
 }
 
-int cpu_temp(unsigned long cpu)
+u32 cpu_temp(unsigned long cpu)
 {
 	return ((tau[cpu].high + tau[cpu].low) / 2);
 }
 
-int tau_interrupts(unsigned long cpu)
+u32 tau_interrupts(unsigned long cpu)
 {
 	return (tau[cpu].interrupts);
 }
diff --git a/arch/powerpc/kernel/trace/ftrace.c b/arch/powerpc/kernel/trace/ftrace.c
index 4741fe112f05..4bfbb54dee51 100644
--- a/arch/powerpc/kernel/trace/ftrace.c
+++ b/arch/powerpc/kernel/trace/ftrace.c
@@ -144,7 +144,7 @@ __ftrace_make_nop(struct module *mod,
 		return -EINVAL;
 	}
 
-#ifdef CC_USING_MPROFILE_KERNEL
+#ifdef CONFIG_MPROFILE_KERNEL
 	/* When using -mkernel_profile there is no load to jump over */
 	pop = PPC_INST_NOP;
 
@@ -188,7 +188,7 @@ __ftrace_make_nop(struct module *mod,
 		pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, op);
 		return -EINVAL;
 	}
-#endif /* CC_USING_MPROFILE_KERNEL */
+#endif /* CONFIG_MPROFILE_KERNEL */
 
 	if (patch_instruction((unsigned int *)ip, pop)) {
 		pr_err("Patching NOP failed.\n");
@@ -324,7 +324,7 @@ int ftrace_make_nop(struct module *mod,
  * They should effectively be a NOP, and follow formal constraints,
  * depending on the ABI. Return false if they don't.
  */
-#ifndef CC_USING_MPROFILE_KERNEL
+#ifndef CONFIG_MPROFILE_KERNEL
 static int
 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
 {
@@ -357,6 +357,8 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 {
 	unsigned int op[2];
 	void *ip = (void *)rec->ip;
+	unsigned long entry, ptr, tramp;
+	struct module *mod = rec->arch.mod;
 
 	/* read where this goes */
 	if (probe_kernel_read(op, ip, sizeof(op)))
@@ -368,19 +370,44 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 		return -EINVAL;
 	}
 
-	/* If we never set up a trampoline to ftrace_caller, then bail */
-	if (!rec->arch.mod->arch.tramp) {
+	/* If we never set up ftrace trampoline(s), then bail */
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+	if (!mod->arch.tramp || !mod->arch.tramp_regs) {
+#else
+	if (!mod->arch.tramp) {
+#endif
 		pr_err("No ftrace trampoline\n");
 		return -EINVAL;
 	}
 
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+	if (rec->flags & FTRACE_FL_REGS)
+		tramp = mod->arch.tramp_regs;
+	else
+#endif
+		tramp = mod->arch.tramp;
+
+	if (module_trampoline_target(mod, tramp, &ptr)) {
+		pr_err("Failed to get trampoline target\n");
+		return -EFAULT;
+	}
+
+	pr_devel("trampoline target %lx", ptr);
+
+	entry = ppc_global_function_entry((void *)addr);
+	/* This should match what was called */
+	if (ptr != entry) {
+		pr_err("addr %lx does not match expected %lx\n", ptr, entry);
+		return -EINVAL;
+	}
+
 	/* Ensure branch is within 24 bits */
-	if (!create_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
+	if (!create_branch(ip, tramp, BRANCH_SET_LINK)) {
 		pr_err("Branch out of range\n");
 		return -EINVAL;
 	}
 
-	if (patch_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
+	if (patch_branch(ip, tramp, BRANCH_SET_LINK)) {
 		pr_err("REL24 out of range!\n");
 		return -EINVAL;
 	}
@@ -388,14 +415,6 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 	return 0;
 }
 
-#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
-int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
-			unsigned long addr)
-{
-	return ftrace_make_call(rec, addr);
-}
-#endif
-
 #else  /* !CONFIG_PPC64: */
 static int
 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
@@ -472,53 +491,158 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 #endif /* CONFIG_MODULES */
 }
 
-int ftrace_update_ftrace_func(ftrace_func_t func)
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+#ifdef CONFIG_MODULES
+static int
+__ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
+					unsigned long addr)
 {
-	unsigned long ip = (unsigned long)(&ftrace_call);
-	unsigned int old, new;
-	int ret;
+	unsigned int op;
+	unsigned long ip = rec->ip;
+	unsigned long entry, ptr, tramp;
+	struct module *mod = rec->arch.mod;
 
-	old = *(unsigned int *)&ftrace_call;
-	new = ftrace_call_replace(ip, (unsigned long)func, 1);
-	ret = ftrace_modify_code(ip, old, new);
+	/* If we never set up ftrace trampolines, then bail */
+	if (!mod->arch.tramp || !mod->arch.tramp_regs) {
+		pr_err("No ftrace trampoline\n");
+		return -EINVAL;
+	}
 
-	return ret;
-}
+	/* read where this goes */
+	if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
+		pr_err("Fetching opcode failed.\n");
+		return -EFAULT;
+	}
 
-static int __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
-{
-	unsigned long ftrace_addr = (unsigned long)FTRACE_ADDR;
-	int ret;
+	/* Make sure that that this is still a 24bit jump */
+	if (!is_bl_op(op)) {
+		pr_err("Not expected bl: opcode is %x\n", op);
+		return -EINVAL;
+	}
+
+	/* lets find where the pointer goes */
+	tramp = find_bl_target(ip, op);
+	entry = ppc_global_function_entry((void *)old_addr);
+
+	pr_devel("ip:%lx jumps to %lx", ip, tramp);
 
-	ret = ftrace_update_record(rec, enable);
+	if (tramp != entry) {
+		/* old_addr is not within range, so we must have used a trampoline */
+		if (module_trampoline_target(mod, tramp, &ptr)) {
+			pr_err("Failed to get trampoline target\n");
+			return -EFAULT;
+		}
+
+		pr_devel("trampoline target %lx", ptr);
+
+		/* This should match what was called */
+		if (ptr != entry) {
+			pr_err("addr %lx does not match expected %lx\n", ptr, entry);
+			return -EINVAL;
+		}
+	}
+
+	/* The new target may be within range */
+	if (test_24bit_addr(ip, addr)) {
+		/* within range */
+		if (patch_branch((unsigned int *)ip, addr, BRANCH_SET_LINK)) {
+			pr_err("REL24 out of range!\n");
+			return -EINVAL;
+		}
 
-	switch (ret) {
-	case FTRACE_UPDATE_IGNORE:
 		return 0;
-	case FTRACE_UPDATE_MAKE_CALL:
-		return ftrace_make_call(rec, ftrace_addr);
-	case FTRACE_UPDATE_MAKE_NOP:
-		return ftrace_make_nop(NULL, rec, ftrace_addr);
+	}
+
+	if (rec->flags & FTRACE_FL_REGS)
+		tramp = mod->arch.tramp_regs;
+	else
+		tramp = mod->arch.tramp;
+
+	if (module_trampoline_target(mod, tramp, &ptr)) {
+		pr_err("Failed to get trampoline target\n");
+		return -EFAULT;
+	}
+
+	pr_devel("trampoline target %lx", ptr);
+
+	entry = ppc_global_function_entry((void *)addr);
+	/* This should match what was called */
+	if (ptr != entry) {
+		pr_err("addr %lx does not match expected %lx\n", ptr, entry);
+		return -EINVAL;
+	}
+
+	/* Ensure branch is within 24 bits */
+	if (!create_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) {
+		pr_err("Branch out of range\n");
+		return -EINVAL;
+	}
+
+	if (patch_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) {
+		pr_err("REL24 out of range!\n");
+		return -EINVAL;
 	}
 
 	return 0;
 }
+#endif
 
-void ftrace_replace_code(int enable)
+int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
+			unsigned long addr)
 {
-	struct ftrace_rec_iter *iter;
-	struct dyn_ftrace *rec;
+	unsigned long ip = rec->ip;
+	unsigned int old, new;
+
+	/*
+	 * If the calling address is more that 24 bits away,
+	 * then we had to use a trampoline to make the call.
+	 * Otherwise just update the call site.
+	 */
+	if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) {
+		/* within range */
+		old = ftrace_call_replace(ip, old_addr, 1);
+		new = ftrace_call_replace(ip, addr, 1);
+		return ftrace_modify_code(ip, old, new);
+	}
+
+#ifdef CONFIG_MODULES
+	/*
+	 * Out of range jumps are called from modules.
+	 */
+	if (!rec->arch.mod) {
+		pr_err("No module loaded\n");
+		return -EINVAL;
+	}
+
+	return __ftrace_modify_call(rec, old_addr, addr);
+#else
+	/* We should not get here without modules */
+	return -EINVAL;
+#endif /* CONFIG_MODULES */
+}
+#endif
+
+int ftrace_update_ftrace_func(ftrace_func_t func)
+{
+	unsigned long ip = (unsigned long)(&ftrace_call);
+	unsigned int old, new;
 	int ret;
 
-	for (iter = ftrace_rec_iter_start(); iter;
-	     iter = ftrace_rec_iter_next(iter)) {
-		rec = ftrace_rec_iter_record(iter);
-		ret = __ftrace_replace_code(rec, enable);
-		if (ret) {
-			ftrace_bug(ret, rec);
-			return;
-		}
+	old = *(unsigned int *)&ftrace_call;
+	new = ftrace_call_replace(ip, (unsigned long)func, 1);
+	ret = ftrace_modify_code(ip, old, new);
+
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+	/* Also update the regs callback function */
+	if (!ret) {
+		ip = (unsigned long)(&ftrace_regs_call);
+		old = *(unsigned int *)&ftrace_regs_call;
+		new = ftrace_call_replace(ip, (unsigned long)func, 1);
+		ret = ftrace_modify_code(ip, old, new);
 	}
+#endif
+
+	return ret;
 }
 
 /*
@@ -538,7 +662,6 @@ int __init ftrace_dyn_arch_init(void)
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 
-#ifdef CONFIG_DYNAMIC_FTRACE
 extern void ftrace_graph_call(void);
 extern void ftrace_graph_stub(void);
 
@@ -567,7 +690,6 @@ int ftrace_disable_ftrace_graph_caller(void)
 
 	return ftrace_modify_code(ip, old, new);
 }
-#endif /* CONFIG_DYNAMIC_FTRACE */
 
 /*
  * Hook the return address and push it in the stack of return addrs
diff --git a/arch/powerpc/kernel/trace/ftrace_32.S b/arch/powerpc/kernel/trace/ftrace_32.S
index afef2c076282..2c29098f630f 100644
--- a/arch/powerpc/kernel/trace/ftrace_32.S
+++ b/arch/powerpc/kernel/trace/ftrace_32.S
@@ -14,7 +14,6 @@
 #include <asm/ftrace.h>
 #include <asm/export.h>
 
-#ifdef CONFIG_DYNAMIC_FTRACE
 _GLOBAL(mcount)
 _GLOBAL(_mcount)
 	/*
@@ -47,26 +46,7 @@ _GLOBAL(ftrace_graph_stub)
 	MCOUNT_RESTORE_FRAME
 	/* old link register ends up in ctr reg */
 	bctr
-#else
-_GLOBAL(mcount)
-_GLOBAL(_mcount)
-
-	MCOUNT_SAVE_FRAME
 
-	subi	r3, r3, MCOUNT_INSN_SIZE
-	LOAD_REG_ADDR(r5, ftrace_trace_function)
-	lwz	r5,0(r5)
-
-	mtctr	r5
-	bctrl
-	nop
-
-#ifdef CONFIG_FUNCTION_GRAPH_TRACER
-	b	ftrace_graph_caller
-#endif
-	MCOUNT_RESTORE_FRAME
-	bctr
-#endif
 EXPORT_SYMBOL(_mcount)
 
 _GLOBAL(ftrace_stub)
diff --git a/arch/powerpc/kernel/trace/ftrace_64.S b/arch/powerpc/kernel/trace/ftrace_64.S
index e5ccea19821e..e25f77c10a72 100644
--- a/arch/powerpc/kernel/trace/ftrace_64.S
+++ b/arch/powerpc/kernel/trace/ftrace_64.S
@@ -14,7 +14,6 @@
 #include <asm/ppc-opcode.h>
 #include <asm/export.h>
 
-#ifdef CONFIG_DYNAMIC_FTRACE
 _GLOBAL(mcount)
 _GLOBAL(_mcount)
 EXPORT_SYMBOL(_mcount)
@@ -23,34 +22,6 @@ EXPORT_SYMBOL(_mcount)
 	mtlr	r0
 	bctr
 
-#else /* CONFIG_DYNAMIC_FTRACE */
-_GLOBAL_TOC(_mcount)
-EXPORT_SYMBOL(_mcount)
-	/* Taken from output of objdump from lib64/glibc */
-	mflr	r3
-	ld	r11, 0(r1)
-	stdu	r1, -112(r1)
-	std	r3, 128(r1)
-	ld	r4, 16(r11)
-
-	subi	r3, r3, MCOUNT_INSN_SIZE
-	LOAD_REG_ADDR(r5,ftrace_trace_function)
-	ld	r5,0(r5)
-	ld	r5,0(r5)
-	mtctr	r5
-	bctrl
-	nop
-
-#ifdef CONFIG_FUNCTION_GRAPH_TRACER
-	b	ftrace_graph_caller
-#endif
-	ld	r0, 128(r1)
-	mtlr	r0
-	addi	r1, r1, 112
-_GLOBAL(ftrace_stub)
-	blr
-#endif /* CONFIG_DYNAMIC_FTRACE */
-
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 _GLOBAL(return_to_handler)
 	/* need to save return values */
diff --git a/arch/powerpc/kernel/trace/ftrace_64_mprofile.S b/arch/powerpc/kernel/trace/ftrace_64_mprofile.S
index 3f3e81852422..9a5b5a513604 100644
--- a/arch/powerpc/kernel/trace/ftrace_64_mprofile.S
+++ b/arch/powerpc/kernel/trace/ftrace_64_mprofile.S
@@ -17,11 +17,10 @@
 #include <asm/bug.h>
 #include <asm/ptrace.h>
 
-#ifdef CONFIG_DYNAMIC_FTRACE
 /*
  *
- * ftrace_caller() is the function that replaces _mcount() when ftrace is
- * active.
+ * ftrace_caller()/ftrace_regs_caller() is the function that replaces _mcount()
+ * when ftrace is active.
  *
  * We arrive here after a function A calls function B, and we are the trace
  * function for B. When we enter r1 points to A's stack frame, B has not yet
@@ -37,7 +36,7 @@
  * Our job is to save the register state into a struct pt_regs (on the stack)
  * and then arrange for the ftrace function to be called.
  */
-_GLOBAL(ftrace_caller)
+_GLOBAL(ftrace_regs_caller)
 	/* Save the original return address in A's stack frame */
 	std	r0,LRSAVE(r1)
 
@@ -47,6 +46,12 @@ _GLOBAL(ftrace_caller)
 	/* Save all gprs to pt_regs */
 	SAVE_GPR(0, r1)
 	SAVE_10GPRS(2, r1)
+
+	/* Ok to continue? */
+	lbz	r3, PACA_FTRACE_ENABLED(r13)
+	cmpdi	r3, 0
+	beq	ftrace_no_trace
+
 	SAVE_10GPRS(12, r1)
 	SAVE_10GPRS(22, r1)
 
@@ -94,8 +99,8 @@ _GLOBAL(ftrace_caller)
 	addi    r6, r1 ,STACK_FRAME_OVERHEAD
 
 	/* ftrace_call(r3, r4, r5, r6) */
-.globl ftrace_call
-ftrace_call:
+.globl ftrace_regs_call
+ftrace_regs_call:
 	bl	ftrace_stub
 	nop
 
@@ -156,6 +161,7 @@ ftrace_call:
 	bne-	livepatch_handler
 #endif
 
+ftrace_caller_common:
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 .globl ftrace_graph_call
 ftrace_graph_call:
@@ -168,6 +174,74 @@ _GLOBAL(ftrace_graph_stub)
 _GLOBAL(ftrace_stub)
 	blr
 
+ftrace_no_trace:
+	mflr	r3
+	mtctr	r3
+	REST_GPR(3, r1)
+	addi	r1, r1, SWITCH_FRAME_SIZE
+	mtlr	r0
+	bctr
+
+_GLOBAL(ftrace_caller)
+	/* Save the original return address in A's stack frame */
+	std	r0, LRSAVE(r1)
+
+	/* Create our stack frame + pt_regs */
+	stdu	r1, -SWITCH_FRAME_SIZE(r1)
+
+	/* Save all gprs to pt_regs */
+	SAVE_8GPRS(3, r1)
+
+	lbz	r3, PACA_FTRACE_ENABLED(r13)
+	cmpdi	r3, 0
+	beq	ftrace_no_trace
+
+	/* Get the _mcount() call site out of LR */
+	mflr	r7
+	std     r7, _NIP(r1)
+
+	/* Save callee's TOC in the ABI compliant location */
+	std	r2, 24(r1)
+	ld	r2, PACATOC(r13)	/* get kernel TOC in r2 */
+
+	addis	r3, r2, function_trace_op@toc@ha
+	addi	r3, r3, function_trace_op@toc@l
+	ld	r5, 0(r3)
+
+	/* Calculate ip from nip-4 into r3 for call below */
+	subi    r3, r7, MCOUNT_INSN_SIZE
+
+	/* Put the original return address in r4 as parent_ip */
+	mr	r4, r0
+
+	/* Set pt_regs to NULL */
+	li	r6, 0
+
+	/* ftrace_call(r3, r4, r5, r6) */
+.globl ftrace_call
+ftrace_call:
+	bl	ftrace_stub
+	nop
+
+	ld	r3, _NIP(r1)
+	mtctr	r3
+
+	/* Restore gprs */
+	REST_8GPRS(3,r1)
+
+	/* Restore callee's TOC */
+	ld	r2, 24(r1)
+
+	/* Pop our stack frame */
+	addi	r1, r1, SWITCH_FRAME_SIZE
+
+	/* Reload original LR */
+	ld	r0, LRSAVE(r1)
+	mtlr	r0
+
+	/* Handle function_graph or go back */
+	b	ftrace_caller_common
+
 #ifdef CONFIG_LIVEPATCH
 	/*
 	 * This function runs in the mcount context, between two functions. As
@@ -236,8 +310,6 @@ livepatch_handler:
 	blr
 #endif /* CONFIG_LIVEPATCH */
 
-#endif /* CONFIG_DYNAMIC_FTRACE */
-
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 _GLOBAL(ftrace_graph_caller)
 	stdu	r1, -112(r1)
diff --git a/arch/powerpc/kernel/trace/ftrace_64_pg.S b/arch/powerpc/kernel/trace/ftrace_64_pg.S
index f095358da96e..4c515c4023de 100644
--- a/arch/powerpc/kernel/trace/ftrace_64_pg.S
+++ b/arch/powerpc/kernel/trace/ftrace_64_pg.S
@@ -14,8 +14,11 @@
 #include <asm/ppc-opcode.h>
 #include <asm/export.h>
 
-#ifdef CONFIG_DYNAMIC_FTRACE
 _GLOBAL_TOC(ftrace_caller)
+	lbz	r3, PACA_FTRACE_ENABLED(r13)
+	cmpdi	r3, 0
+	beqlr
+
 	/* Taken from output of objdump from lib64/glibc */
 	mflr	r3
 	ld	r11, 0(r1)
@@ -39,7 +42,6 @@ _GLOBAL(ftrace_graph_stub)
 
 _GLOBAL(ftrace_stub)
 	blr
-#endif /* CONFIG_DYNAMIC_FTRACE */
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 _GLOBAL(ftrace_graph_caller)
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 0904492e7032..0e17dcb48720 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -296,7 +296,6 @@ NOKPROBE_SYMBOL(die);
 void user_single_step_siginfo(struct task_struct *tsk,
 				struct pt_regs *regs, siginfo_t *info)
 {
-	memset(info, 0, sizeof(*info));
 	info->si_signo = SIGTRAP;
 	info->si_code = TRAP_TRACE;
 	info->si_addr = (void __user *)regs->nip;
@@ -334,7 +333,7 @@ void _exception_pkey(int signr, struct pt_regs *regs, int code,
 	 */
 	thread_pkey_regs_save(&current->thread);
 
-	memset(&info, 0, sizeof(info));
+	clear_siginfo(&info);
 	info.si_signo = signr;
 	info.si_code = code;
 	info.si_addr = (void __user *) addr;
@@ -970,7 +969,7 @@ void unknown_exception(struct pt_regs *regs)
 	printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
 	       regs->nip, regs->msr, regs->trap);
 
-	_exception(SIGTRAP, regs, TRAP_FIXME, 0);
+	_exception(SIGTRAP, regs, TRAP_UNK, 0);
 
 	exception_exit(prev_state);
 }
@@ -992,7 +991,7 @@ void instruction_breakpoint_exception(struct pt_regs *regs)
 
 void RunModeException(struct pt_regs *regs)
 {
-	_exception(SIGTRAP, regs, TRAP_FIXME, 0);
+	_exception(SIGTRAP, regs, TRAP_UNK, 0);
 }
 
 void single_step_exception(struct pt_regs *regs)
@@ -1032,7 +1031,7 @@ static void emulate_single_step(struct pt_regs *regs)
 
 static inline int __parse_fpscr(unsigned long fpscr)
 {
-	int ret = FPE_FIXME;
+	int ret = FPE_FLTUNK;
 
 	/* Invalid operation */
 	if ((fpscr & FPSCR_VE) && (fpscr & FPSCR_VX))
@@ -1973,7 +1972,7 @@ void SPEFloatingPointException(struct pt_regs *regs)
 	extern int do_spe_mathemu(struct pt_regs *regs);
 	unsigned long spefscr;
 	int fpexc_mode;
-	int code = FPE_FIXME;
+	int code = FPE_FLTUNK;
 	int err;
 
 	flush_spe_to_thread(current);
@@ -2042,7 +2041,7 @@ void SPEFloatingPointRoundException(struct pt_regs *regs)
 		printk(KERN_ERR "unrecognized spe instruction "
 		       "in %s at %lx\n", current->comm, regs->nip);
 	} else {
-		_exception(SIGFPE, regs, FPE_FIXME, regs->nip);
+		_exception(SIGFPE, regs, FPE_FLTUNK, regs->nip);
 		return;
 	}
 }
diff --git a/arch/powerpc/kernel/vdso32/Makefile b/arch/powerpc/kernel/vdso32/Makefile
index b8c434d1d459..50112d4473bb 100644
--- a/arch/powerpc/kernel/vdso32/Makefile
+++ b/arch/powerpc/kernel/vdso32/Makefile
@@ -8,8 +8,15 @@ obj-vdso32 = sigtramp.o gettimeofday.o datapage.o cacheflush.o note.o \
 
 # Build rules
 
-ifeq ($(CONFIG_PPC32),y)
-CROSS32CC := $(CC)
+ifdef CROSS32_COMPILE
+    VDSOCC := $(CROSS32_COMPILE)gcc
+else
+    VDSOCC := $(CC)
+endif
+
+CC32FLAGS :=
+ifdef CONFIG_PPC64
+CC32FLAGS += -m32
 endif
 
 targets := $(obj-vdso32) vdso32.so vdso32.so.dbg
@@ -45,9 +52,9 @@ $(obj-vdso32): %.o: %.S FORCE
 
 # actual build commands
 quiet_cmd_vdso32ld = VDSO32L $@
-      cmd_vdso32ld = $(CROSS32CC) $(c_flags) -o $@ -Wl,-T$(filter %.lds,$^) $(filter %.o,$^)
+      cmd_vdso32ld = $(VDSOCC) $(c_flags) $(CC32FLAGS) -o $@ -Wl,-T$(filter %.lds,$^) $(filter %.o,$^)
 quiet_cmd_vdso32as = VDSO32A $@
-      cmd_vdso32as = $(CROSS32CC) $(a_flags) -c -o $@ $<
+      cmd_vdso32as = $(VDSOCC) $(a_flags) $(CC32FLAGS) -c -o $@ $<
 
 # install commands for the unstripped file
 quiet_cmd_vdso_install = INSTALL $@
diff --git a/arch/powerpc/kernel/vecemu.c b/arch/powerpc/kernel/vecemu.c
index 8812085883fd..4acd3fb2b38e 100644
--- a/arch/powerpc/kernel/vecemu.c
+++ b/arch/powerpc/kernel/vecemu.c
@@ -8,6 +8,7 @@
 #include <linux/sched.h>
 #include <asm/ptrace.h>
 #include <asm/processor.h>
+#include <asm/switch_to.h>
 #include <linux/uaccess.h>
 
 /* Functions in vector.S */
diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S
index b8d82678f8b4..dd10e6f1d1b7 100644
--- a/arch/powerpc/kernel/vmlinux.lds.S
+++ b/arch/powerpc/kernel/vmlinux.lds.S
@@ -89,7 +89,7 @@ SECTIONS
 	 */
 	.text BLOCK(0) : AT(ADDR(.text) - LOAD_OFFSET) {
 #ifdef CONFIG_LD_HEAD_STUB_CATCH
-		*(.linker_stub_catch);
+		KEEP(*(.linker_stub_catch));
 		. = . ;
 #endif
 
@@ -98,7 +98,7 @@ SECTIONS
 		ALIGN_FUNCTION();
 #endif
 		/* careful! __ftr_alt_* sections need to be close to .text */
-		*(.text.hot .text .text.fixup .text.unlikely .fixup __ftr_alt_* .ref.text);
+		*(.text.hot TEXT_MAIN .text.fixup .text.unlikely .fixup __ftr_alt_* .ref.text);
 		SCHED_TEXT
 		CPUIDLE_TEXT
 		LOCK_TEXT
@@ -184,10 +184,10 @@ SECTIONS
 	.init.data : AT(ADDR(.init.data) - LOAD_OFFSET) {
 		INIT_DATA
 		__vtop_table_begin = .;
-		*(.vtop_fixup);
+		KEEP(*(.vtop_fixup));
 		__vtop_table_end = .;
 		__ptov_table_begin = .;
-		*(.ptov_fixup);
+		KEEP(*(.ptov_fixup));
 		__ptov_table_end = .;
 	}
 
@@ -208,26 +208,26 @@ SECTIONS
 	. = ALIGN(8);
 	__ftr_fixup : AT(ADDR(__ftr_fixup) - LOAD_OFFSET) {
 		__start___ftr_fixup = .;
-		*(__ftr_fixup)
+		KEEP(*(__ftr_fixup))
 		__stop___ftr_fixup = .;
 	}
 	. = ALIGN(8);
 	__mmu_ftr_fixup : AT(ADDR(__mmu_ftr_fixup) - LOAD_OFFSET) {
 		__start___mmu_ftr_fixup = .;
-		*(__mmu_ftr_fixup)
+		KEEP(*(__mmu_ftr_fixup))
 		__stop___mmu_ftr_fixup = .;
 	}
 	. = ALIGN(8);
 	__lwsync_fixup : AT(ADDR(__lwsync_fixup) - LOAD_OFFSET) {
 		__start___lwsync_fixup = .;
-		*(__lwsync_fixup)
+		KEEP(*(__lwsync_fixup))
 		__stop___lwsync_fixup = .;
 	}
 #ifdef CONFIG_PPC64
 	. = ALIGN(8);
 	__fw_ftr_fixup : AT(ADDR(__fw_ftr_fixup) - LOAD_OFFSET) {
 		__start___fw_ftr_fixup = .;
-		*(__fw_ftr_fixup)
+		KEEP(*(__fw_ftr_fixup))
 		__stop___fw_ftr_fixup = .;
 	}
 #endif
@@ -240,7 +240,7 @@ SECTIONS
 	. = ALIGN(8);
 	.machine.desc : AT(ADDR(.machine.desc) - LOAD_OFFSET) {
 		__machine_desc_start = . ;
-		*(.machine.desc)
+		KEEP(*(.machine.desc))
 		__machine_desc_end = . ;
 	}
 #ifdef CONFIG_RELOCATABLE
@@ -288,7 +288,7 @@ SECTIONS
 	.data : AT(ADDR(.data) - LOAD_OFFSET) {
 		DATA_DATA
 		*(.data.rel*)
-		*(.sdata)
+		*(SDATA_MAIN)
 		*(.sdata2)
 		*(.got.plt) *(.got)
 		*(.plt)
@@ -303,7 +303,7 @@ SECTIONS
 
 	.opd : AT(ADDR(.opd) - LOAD_OFFSET) {
 		__start_opd = .;
-		*(.opd)
+		KEEP(*(.opd))
 		__end_opd = .;
 	}
 
diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c
index 6256dc3b0087..1d82274f7e9f 100644
--- a/arch/powerpc/kernel/watchdog.c
+++ b/arch/powerpc/kernel/watchdog.c
@@ -64,7 +64,7 @@
  * means the CPU(s) with their bit still set in the pending mask have had
  * their heartbeat stop, and action is taken.
  *
- * Some platforms implement true NMI IPIs, which can by used by the SMP
+ * Some platforms implement true NMI IPIs, which can be used by the SMP
  * watchdog to detect an unresponsive CPU and pull it out of its stuck
  * state with the NMI IPI, to get crash/debug data from it. This way the
  * SMP watchdog can detect hardware interrupts off lockups.
@@ -111,7 +111,13 @@ static inline void wd_smp_unlock(unsigned long *flags)
 
 static void wd_lockup_ipi(struct pt_regs *regs)
 {
-	pr_emerg("CPU %d Hard LOCKUP\n", raw_smp_processor_id());
+	int cpu = raw_smp_processor_id();
+	u64 tb = get_tb();
+
+	pr_emerg("CPU %d Hard LOCKUP\n", cpu);
+	pr_emerg("CPU %d TB:%lld, last heartbeat TB:%lld (%lldms ago)\n",
+		 cpu, tb, per_cpu(wd_timer_tb, cpu),
+		 tb_to_ns(tb - per_cpu(wd_timer_tb, cpu)) / 1000000);
 	print_modules();
 	print_irqtrace_events(current);
 	if (regs)
@@ -154,6 +160,9 @@ static void watchdog_smp_panic(int cpu, u64 tb)
 
 	pr_emerg("CPU %d detected hard LOCKUP on other CPUs %*pbl\n",
 		 cpu, cpumask_pr_args(&wd_smp_cpus_pending));
+	pr_emerg("CPU %d TB:%lld, last SMP heartbeat TB:%lld (%lldms ago)\n",
+		 cpu, tb, wd_smp_last_reset_tb,
+		 tb_to_ns(tb - wd_smp_last_reset_tb) / 1000000);
 
 	if (!sysctl_hardlockup_all_cpu_backtrace) {
 		/*
@@ -194,10 +203,19 @@ static void wd_smp_clear_cpu_pending(int cpu, u64 tb)
 {
 	if (!cpumask_test_cpu(cpu, &wd_smp_cpus_pending)) {
 		if (unlikely(cpumask_test_cpu(cpu, &wd_smp_cpus_stuck))) {
+			struct pt_regs *regs = get_irq_regs();
 			unsigned long flags;
 
-			pr_emerg("CPU %d became unstuck\n", cpu);
 			wd_smp_lock(&flags);
+
+			pr_emerg("CPU %d became unstuck TB:%lld\n",
+				 cpu, tb);
+			print_irqtrace_events(current);
+			if (regs)
+				show_regs(regs);
+			else
+				dump_stack();
+
 			cpumask_clear_cpu(cpu, &wd_smp_cpus_stuck);
 			wd_smp_unlock(&flags);
 		}
@@ -245,8 +263,6 @@ void soft_nmi_interrupt(struct pt_regs *regs)
 
 	tb = get_tb();
 	if (tb - per_cpu(wd_timer_tb, cpu) >= wd_panic_timeout_tb) {
-		per_cpu(wd_timer_tb, cpu) = tb;
-
 		wd_smp_lock(&flags);
 		if (cpumask_test_cpu(cpu, &wd_smp_cpus_stuck)) {
 			wd_smp_unlock(&flags);
@@ -254,7 +270,11 @@ void soft_nmi_interrupt(struct pt_regs *regs)
 		}
 		set_cpu_stuck(cpu, tb);
 
-		pr_emerg("CPU %d self-detected hard LOCKUP @ %pS\n", cpu, (void *)regs->nip);
+		pr_emerg("CPU %d self-detected hard LOCKUP @ %pS\n",
+			 cpu, (void *)regs->nip);
+		pr_emerg("CPU %d TB:%lld, last heartbeat TB:%lld (%lldms ago)\n",
+			 cpu, tb, per_cpu(wd_timer_tb, cpu),
+			 tb_to_ns(tb - per_cpu(wd_timer_tb, cpu)) / 1000000);
 		print_modules();
 		print_irqtrace_events(current);
 		show_regs(regs);
diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c
index 97d4a112648f..320cdcf84591 100644
--- a/arch/powerpc/kvm/book3s.c
+++ b/arch/powerpc/kvm/book3s.c
@@ -450,8 +450,8 @@ int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, enum xlate_instdata xlid,
 	return r;
 }
 
-int kvmppc_load_last_inst(struct kvm_vcpu *vcpu, enum instruction_type type,
-					 u32 *inst)
+int kvmppc_load_last_inst(struct kvm_vcpu *vcpu,
+		enum instruction_fetch_type type, u32 *inst)
 {
 	ulong pc = kvmppc_get_pc(vcpu);
 	int r;
diff --git a/arch/powerpc/kvm/book3s_32_mmu.c b/arch/powerpc/kvm/book3s_32_mmu.c
index 1992676c7a94..45c8ea4a0487 100644
--- a/arch/powerpc/kvm/book3s_32_mmu.c
+++ b/arch/powerpc/kvm/book3s_32_mmu.c
@@ -52,7 +52,7 @@
 static inline bool check_debug_ip(struct kvm_vcpu *vcpu)
 {
 #ifdef DEBUG_MMU_PTE_IP
-	return vcpu->arch.pc == DEBUG_MMU_PTE_IP;
+	return vcpu->arch.regs.nip == DEBUG_MMU_PTE_IP;
 #else
 	return true;
 #endif
diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c
index a670fa5fbe50..37cd6434d1c8 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_hv.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c
@@ -272,6 +272,9 @@ int kvmppc_mmu_hv_init(void)
 	if (!cpu_has_feature(CPU_FTR_HVMODE))
 		return -EINVAL;
 
+	if (!mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE))
+		return -EINVAL;
+
 	/* POWER7 has 10-bit LPIDs (12-bit in POWER8) */
 	host_lpid = mfspr(SPRN_LPID);
 	rsvd_lpid = LPID_RSVD;
diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c
index 361f42c8c73e..176f911ee983 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -139,44 +139,24 @@ int kvmppc_mmu_radix_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
 	return 0;
 }
 
-#ifdef CONFIG_PPC_64K_PAGES
-#define MMU_BASE_PSIZE	MMU_PAGE_64K
-#else
-#define MMU_BASE_PSIZE	MMU_PAGE_4K
-#endif
-
 static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
 				    unsigned int pshift)
 {
-	int psize = MMU_BASE_PSIZE;
-
-	if (pshift >= PUD_SHIFT)
-		psize = MMU_PAGE_1G;
-	else if (pshift >= PMD_SHIFT)
-		psize = MMU_PAGE_2M;
-	addr &= ~0xfffUL;
-	addr |= mmu_psize_defs[psize].ap << 5;
-	asm volatile("ptesync": : :"memory");
-	asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
-		     : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
-	if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG))
-		asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
-			     : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
-	asm volatile("eieio ; tlbsync ; ptesync": : :"memory");
+	unsigned long psize = PAGE_SIZE;
+
+	if (pshift)
+		psize = 1UL << pshift;
+
+	addr &= ~(psize - 1);
+	radix__flush_tlb_lpid_page(kvm->arch.lpid, addr, psize);
 }
 
-static void kvmppc_radix_flush_pwc(struct kvm *kvm, unsigned long addr)
+static void kvmppc_radix_flush_pwc(struct kvm *kvm)
 {
-	unsigned long rb = 0x2 << PPC_BITLSHIFT(53); /* IS = 2 */
-
-	asm volatile("ptesync": : :"memory");
-	/* RIC=1 PRS=0 R=1 IS=2 */
-	asm volatile(PPC_TLBIE_5(%0, %1, 1, 0, 1)
-		     : : "r" (rb), "r" (kvm->arch.lpid) : "memory");
-	asm volatile("eieio ; tlbsync ; ptesync": : :"memory");
+	radix__flush_pwc_lpid(kvm->arch.lpid);
 }
 
-unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep,
+static unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep,
 				      unsigned long clr, unsigned long set,
 				      unsigned long addr, unsigned int shift)
 {
@@ -200,6 +180,7 @@ void kvmppc_radix_set_pte_at(struct kvm *kvm, unsigned long addr,
 }
 
 static struct kmem_cache *kvm_pte_cache;
+static struct kmem_cache *kvm_pmd_cache;
 
 static pte_t *kvmppc_pte_alloc(void)
 {
@@ -217,6 +198,177 @@ static inline int pmd_is_leaf(pmd_t pmd)
 	return !!(pmd_val(pmd) & _PAGE_PTE);
 }
 
+static pmd_t *kvmppc_pmd_alloc(void)
+{
+	return kmem_cache_alloc(kvm_pmd_cache, GFP_KERNEL);
+}
+
+static void kvmppc_pmd_free(pmd_t *pmdp)
+{
+	kmem_cache_free(kvm_pmd_cache, pmdp);
+}
+
+static void kvmppc_unmap_pte(struct kvm *kvm, pte_t *pte,
+			     unsigned long gpa, unsigned int shift)
+
+{
+	unsigned long page_size = 1ul << shift;
+	unsigned long old;
+
+	old = kvmppc_radix_update_pte(kvm, pte, ~0UL, 0, gpa, shift);
+	kvmppc_radix_tlbie_page(kvm, gpa, shift);
+	if (old & _PAGE_DIRTY) {
+		unsigned long gfn = gpa >> PAGE_SHIFT;
+		struct kvm_memory_slot *memslot;
+
+		memslot = gfn_to_memslot(kvm, gfn);
+		if (memslot && memslot->dirty_bitmap)
+			kvmppc_update_dirty_map(memslot, gfn, page_size);
+	}
+}
+
+/*
+ * kvmppc_free_p?d are used to free existing page tables, and recursively
+ * descend and clear and free children.
+ * Callers are responsible for flushing the PWC.
+ *
+ * When page tables are being unmapped/freed as part of page fault path
+ * (full == false), ptes are not expected. There is code to unmap them
+ * and emit a warning if encountered, but there may already be data
+ * corruption due to the unexpected mappings.
+ */
+static void kvmppc_unmap_free_pte(struct kvm *kvm, pte_t *pte, bool full)
+{
+	if (full) {
+		memset(pte, 0, sizeof(long) << PTE_INDEX_SIZE);
+	} else {
+		pte_t *p = pte;
+		unsigned long it;
+
+		for (it = 0; it < PTRS_PER_PTE; ++it, ++p) {
+			if (pte_val(*p) == 0)
+				continue;
+			WARN_ON_ONCE(1);
+			kvmppc_unmap_pte(kvm, p,
+					 pte_pfn(*p) << PAGE_SHIFT,
+					 PAGE_SHIFT);
+		}
+	}
+
+	kvmppc_pte_free(pte);
+}
+
+static void kvmppc_unmap_free_pmd(struct kvm *kvm, pmd_t *pmd, bool full)
+{
+	unsigned long im;
+	pmd_t *p = pmd;
+
+	for (im = 0; im < PTRS_PER_PMD; ++im, ++p) {
+		if (!pmd_present(*p))
+			continue;
+		if (pmd_is_leaf(*p)) {
+			if (full) {
+				pmd_clear(p);
+			} else {
+				WARN_ON_ONCE(1);
+				kvmppc_unmap_pte(kvm, (pte_t *)p,
+					 pte_pfn(*(pte_t *)p) << PAGE_SHIFT,
+					 PMD_SHIFT);
+			}
+		} else {
+			pte_t *pte;
+
+			pte = pte_offset_map(p, 0);
+			kvmppc_unmap_free_pte(kvm, pte, full);
+			pmd_clear(p);
+		}
+	}
+	kvmppc_pmd_free(pmd);
+}
+
+static void kvmppc_unmap_free_pud(struct kvm *kvm, pud_t *pud)
+{
+	unsigned long iu;
+	pud_t *p = pud;
+
+	for (iu = 0; iu < PTRS_PER_PUD; ++iu, ++p) {
+		if (!pud_present(*p))
+			continue;
+		if (pud_huge(*p)) {
+			pud_clear(p);
+		} else {
+			pmd_t *pmd;
+
+			pmd = pmd_offset(p, 0);
+			kvmppc_unmap_free_pmd(kvm, pmd, true);
+			pud_clear(p);
+		}
+	}
+	pud_free(kvm->mm, pud);
+}
+
+void kvmppc_free_radix(struct kvm *kvm)
+{
+	unsigned long ig;
+	pgd_t *pgd;
+
+	if (!kvm->arch.pgtable)
+		return;
+	pgd = kvm->arch.pgtable;
+	for (ig = 0; ig < PTRS_PER_PGD; ++ig, ++pgd) {
+		pud_t *pud;
+
+		if (!pgd_present(*pgd))
+			continue;
+		pud = pud_offset(pgd, 0);
+		kvmppc_unmap_free_pud(kvm, pud);
+		pgd_clear(pgd);
+	}
+	pgd_free(kvm->mm, kvm->arch.pgtable);
+	kvm->arch.pgtable = NULL;
+}
+
+static void kvmppc_unmap_free_pmd_entry_table(struct kvm *kvm, pmd_t *pmd,
+					      unsigned long gpa)
+{
+	pte_t *pte = pte_offset_kernel(pmd, 0);
+
+	/*
+	 * Clearing the pmd entry then flushing the PWC ensures that the pte
+	 * page no longer be cached by the MMU, so can be freed without
+	 * flushing the PWC again.
+	 */
+	pmd_clear(pmd);
+	kvmppc_radix_flush_pwc(kvm);
+
+	kvmppc_unmap_free_pte(kvm, pte, false);
+}
+
+static void kvmppc_unmap_free_pud_entry_table(struct kvm *kvm, pud_t *pud,
+					unsigned long gpa)
+{
+	pmd_t *pmd = pmd_offset(pud, 0);
+
+	/*
+	 * Clearing the pud entry then flushing the PWC ensures that the pmd
+	 * page and any children pte pages will no longer be cached by the MMU,
+	 * so can be freed without flushing the PWC again.
+	 */
+	pud_clear(pud);
+	kvmppc_radix_flush_pwc(kvm);
+
+	kvmppc_unmap_free_pmd(kvm, pmd, false);
+}
+
+/*
+ * There are a number of bits which may differ between different faults to
+ * the same partition scope entry. RC bits, in the course of cleaning and
+ * aging. And the write bit can change, either the access could have been
+ * upgraded, or a read fault could happen concurrently with a write fault
+ * that sets those bits first.
+ */
+#define PTE_BITS_MUST_MATCH (~(_PAGE_WRITE | _PAGE_DIRTY | _PAGE_ACCESSED))
+
 static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 			     unsigned int level, unsigned long mmu_seq)
 {
@@ -224,7 +376,6 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 	pud_t *pud, *new_pud = NULL;
 	pmd_t *pmd, *new_pmd = NULL;
 	pte_t *ptep, *new_ptep = NULL;
-	unsigned long old;
 	int ret;
 
 	/* Traverse the guest's 2nd-level tree, allocate new levels needed */
@@ -239,7 +390,7 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 	if (pud && pud_present(*pud) && !pud_huge(*pud))
 		pmd = pmd_offset(pud, gpa);
 	else if (level <= 1)
-		new_pmd = pmd_alloc_one(kvm->mm, gpa);
+		new_pmd = kvmppc_pmd_alloc();
 
 	if (level == 0 && !(pmd && pmd_present(*pmd) && !pmd_is_leaf(*pmd)))
 		new_ptep = kvmppc_pte_alloc();
@@ -262,42 +413,39 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 	if (pud_huge(*pud)) {
 		unsigned long hgpa = gpa & PUD_MASK;
 
+		/* Check if we raced and someone else has set the same thing */
+		if (level == 2) {
+			if (pud_raw(*pud) == pte_raw(pte)) {
+				ret = 0;
+				goto out_unlock;
+			}
+			/* Valid 1GB page here already, add our extra bits */
+			WARN_ON_ONCE((pud_val(*pud) ^ pte_val(pte)) &
+							PTE_BITS_MUST_MATCH);
+			kvmppc_radix_update_pte(kvm, (pte_t *)pud,
+					      0, pte_val(pte), hgpa, PUD_SHIFT);
+			ret = 0;
+			goto out_unlock;
+		}
 		/*
 		 * If we raced with another CPU which has just put
 		 * a 1GB pte in after we saw a pmd page, try again.
 		 */
-		if (level <= 1 && !new_pmd) {
+		if (!new_pmd) {
 			ret = -EAGAIN;
 			goto out_unlock;
 		}
-		/* Check if we raced and someone else has set the same thing */
-		if (level == 2 && pud_raw(*pud) == pte_raw(pte)) {
-			ret = 0;
-			goto out_unlock;
-		}
 		/* Valid 1GB page here already, remove it */
-		old = kvmppc_radix_update_pte(kvm, (pte_t *)pud,
-					      ~0UL, 0, hgpa, PUD_SHIFT);
-		kvmppc_radix_tlbie_page(kvm, hgpa, PUD_SHIFT);
-		if (old & _PAGE_DIRTY) {
-			unsigned long gfn = hgpa >> PAGE_SHIFT;
-			struct kvm_memory_slot *memslot;
-			memslot = gfn_to_memslot(kvm, gfn);
-			if (memslot && memslot->dirty_bitmap)
-				kvmppc_update_dirty_map(memslot,
-							gfn, PUD_SIZE);
-		}
+		kvmppc_unmap_pte(kvm, (pte_t *)pud, hgpa, PUD_SHIFT);
 	}
 	if (level == 2) {
 		if (!pud_none(*pud)) {
 			/*
 			 * There's a page table page here, but we wanted to
 			 * install a large page, so remove and free the page
-			 * table page.  new_pmd will be NULL since level == 2.
+			 * table page.
 			 */
-			new_pmd = pmd_offset(pud, 0);
-			pud_clear(pud);
-			kvmppc_radix_flush_pwc(kvm, gpa);
+			kvmppc_unmap_free_pud_entry_table(kvm, pud, gpa);
 		}
 		kvmppc_radix_set_pte_at(kvm, gpa, (pte_t *)pud, pte);
 		ret = 0;
@@ -313,42 +461,40 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 	if (pmd_is_leaf(*pmd)) {
 		unsigned long lgpa = gpa & PMD_MASK;
 
+		/* Check if we raced and someone else has set the same thing */
+		if (level == 1) {
+			if (pmd_raw(*pmd) == pte_raw(pte)) {
+				ret = 0;
+				goto out_unlock;
+			}
+			/* Valid 2MB page here already, add our extra bits */
+			WARN_ON_ONCE((pmd_val(*pmd) ^ pte_val(pte)) &
+							PTE_BITS_MUST_MATCH);
+			kvmppc_radix_update_pte(kvm, pmdp_ptep(pmd),
+					      0, pte_val(pte), lgpa, PMD_SHIFT);
+			ret = 0;
+			goto out_unlock;
+		}
+
 		/*
 		 * If we raced with another CPU which has just put
 		 * a 2MB pte in after we saw a pte page, try again.
 		 */
-		if (level == 0 && !new_ptep) {
+		if (!new_ptep) {
 			ret = -EAGAIN;
 			goto out_unlock;
 		}
-		/* Check if we raced and someone else has set the same thing */
-		if (level == 1 && pmd_raw(*pmd) == pte_raw(pte)) {
-			ret = 0;
-			goto out_unlock;
-		}
 		/* Valid 2MB page here already, remove it */
-		old = kvmppc_radix_update_pte(kvm, pmdp_ptep(pmd),
-					      ~0UL, 0, lgpa, PMD_SHIFT);
-		kvmppc_radix_tlbie_page(kvm, lgpa, PMD_SHIFT);
-		if (old & _PAGE_DIRTY) {
-			unsigned long gfn = lgpa >> PAGE_SHIFT;
-			struct kvm_memory_slot *memslot;
-			memslot = gfn_to_memslot(kvm, gfn);
-			if (memslot && memslot->dirty_bitmap)
-				kvmppc_update_dirty_map(memslot,
-							gfn, PMD_SIZE);
-		}
+		kvmppc_unmap_pte(kvm, pmdp_ptep(pmd), lgpa, PMD_SHIFT);
 	}
 	if (level == 1) {
 		if (!pmd_none(*pmd)) {
 			/*
 			 * There's a page table page here, but we wanted to
 			 * install a large page, so remove and free the page
-			 * table page.  new_ptep will be NULL since level == 1.
+			 * table page.
 			 */
-			new_ptep = pte_offset_kernel(pmd, 0);
-			pmd_clear(pmd);
-			kvmppc_radix_flush_pwc(kvm, gpa);
+			kvmppc_unmap_free_pmd_entry_table(kvm, pmd, gpa);
 		}
 		kvmppc_radix_set_pte_at(kvm, gpa, pmdp_ptep(pmd), pte);
 		ret = 0;
@@ -367,12 +513,12 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 			ret = 0;
 			goto out_unlock;
 		}
-		/* PTE was previously valid, so invalidate it */
-		old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_PRESENT,
-					      0, gpa, 0);
-		kvmppc_radix_tlbie_page(kvm, gpa, 0);
-		if (old & _PAGE_DIRTY)
-			mark_page_dirty(kvm, gpa >> PAGE_SHIFT);
+		/* Valid page here already, add our extra bits */
+		WARN_ON_ONCE((pte_val(*ptep) ^ pte_val(pte)) &
+							PTE_BITS_MUST_MATCH);
+		kvmppc_radix_update_pte(kvm, ptep, 0, pte_val(pte), gpa, 0);
+		ret = 0;
+		goto out_unlock;
 	}
 	kvmppc_radix_set_pte_at(kvm, gpa, ptep, pte);
 	ret = 0;
@@ -382,7 +528,7 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 	if (new_pud)
 		pud_free(kvm->mm, new_pud);
 	if (new_pmd)
-		pmd_free(kvm->mm, new_pmd);
+		kvmppc_pmd_free(new_pmd);
 	if (new_ptep)
 		kvmppc_pte_free(new_ptep);
 	return ret;
@@ -554,9 +700,13 @@ int kvmppc_book3s_radix_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 			unsigned long mask = (1ul << shift) - PAGE_SIZE;
 			pte = __pte(pte_val(pte) | (hva & mask));
 		}
-		if (!(writing || upgrade_write))
-			pte = __pte(pte_val(pte) & ~ _PAGE_WRITE);
-		pte = __pte(pte_val(pte) | _PAGE_EXEC);
+		pte = __pte(pte_val(pte) | _PAGE_EXEC | _PAGE_ACCESSED);
+		if (writing || upgrade_write) {
+			if (pte_val(pte) & _PAGE_WRITE)
+				pte = __pte(pte_val(pte) | _PAGE_DIRTY);
+		} else {
+			pte = __pte(pte_val(pte) & ~(_PAGE_WRITE | _PAGE_DIRTY));
+		}
 	}
 
 	/* Allocate space in the tree and write the PTE */
@@ -723,67 +873,37 @@ int kvmppc_init_vm_radix(struct kvm *kvm)
 	return 0;
 }
 
-void kvmppc_free_radix(struct kvm *kvm)
+static void pte_ctor(void *addr)
 {
-	unsigned long ig, iu, im;
-	pte_t *pte;
-	pmd_t *pmd;
-	pud_t *pud;
-	pgd_t *pgd;
-
-	if (!kvm->arch.pgtable)
-		return;
-	pgd = kvm->arch.pgtable;
-	for (ig = 0; ig < PTRS_PER_PGD; ++ig, ++pgd) {
-		if (!pgd_present(*pgd))
-			continue;
-		pud = pud_offset(pgd, 0);
-		for (iu = 0; iu < PTRS_PER_PUD; ++iu, ++pud) {
-			if (!pud_present(*pud))
-				continue;
-			if (pud_huge(*pud)) {
-				pud_clear(pud);
-				continue;
-			}
-			pmd = pmd_offset(pud, 0);
-			for (im = 0; im < PTRS_PER_PMD; ++im, ++pmd) {
-				if (pmd_is_leaf(*pmd)) {
-					pmd_clear(pmd);
-					continue;
-				}
-				if (!pmd_present(*pmd))
-					continue;
-				pte = pte_offset_map(pmd, 0);
-				memset(pte, 0, sizeof(long) << PTE_INDEX_SIZE);
-				kvmppc_pte_free(pte);
-				pmd_clear(pmd);
-			}
-			pmd_free(kvm->mm, pmd_offset(pud, 0));
-			pud_clear(pud);
-		}
-		pud_free(kvm->mm, pud_offset(pgd, 0));
-		pgd_clear(pgd);
-	}
-	pgd_free(kvm->mm, kvm->arch.pgtable);
-	kvm->arch.pgtable = NULL;
+	memset(addr, 0, RADIX_PTE_TABLE_SIZE);
 }
 
-static void pte_ctor(void *addr)
+static void pmd_ctor(void *addr)
 {
-	memset(addr, 0, PTE_TABLE_SIZE);
+	memset(addr, 0, RADIX_PMD_TABLE_SIZE);
 }
 
 int kvmppc_radix_init(void)
 {
-	unsigned long size = sizeof(void *) << PTE_INDEX_SIZE;
+	unsigned long size = sizeof(void *) << RADIX_PTE_INDEX_SIZE;
 
 	kvm_pte_cache = kmem_cache_create("kvm-pte", size, size, 0, pte_ctor);
 	if (!kvm_pte_cache)
 		return -ENOMEM;
+
+	size = sizeof(void *) << RADIX_PMD_INDEX_SIZE;
+
+	kvm_pmd_cache = kmem_cache_create("kvm-pmd", size, size, 0, pmd_ctor);
+	if (!kvm_pmd_cache) {
+		kmem_cache_destroy(kvm_pte_cache);
+		return -ENOMEM;
+	}
+
 	return 0;
 }
 
 void kvmppc_radix_exit(void)
 {
 	kmem_cache_destroy(kvm_pte_cache);
+	kmem_cache_destroy(kvm_pmd_cache);
 }
diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c
index 4dffa611376d..d066e37551ec 100644
--- a/arch/powerpc/kvm/book3s_64_vio.c
+++ b/arch/powerpc/kvm/book3s_64_vio.c
@@ -176,14 +176,12 @@ extern long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd,
 
 		if (!tbltmp)
 			continue;
-		/*
-		 * Make sure hardware table parameters are exactly the same;
-		 * this is used in the TCE handlers where boundary checks
-		 * use only the first attached table.
-		 */
-		if ((tbltmp->it_page_shift == stt->page_shift) &&
-				(tbltmp->it_offset == stt->offset) &&
-				(tbltmp->it_size == stt->size)) {
+		/* Make sure hardware table parameters are compatible */
+		if ((tbltmp->it_page_shift <= stt->page_shift) &&
+				(tbltmp->it_offset << tbltmp->it_page_shift ==
+				 stt->offset << stt->page_shift) &&
+				(tbltmp->it_size << tbltmp->it_page_shift ==
+				 stt->size << stt->page_shift)) {
 			/*
 			 * Reference the table to avoid races with
 			 * add/remove DMA windows.
@@ -237,7 +235,7 @@ static void release_spapr_tce_table(struct rcu_head *head)
 	kfree(stt);
 }
 
-static int kvm_spapr_tce_fault(struct vm_fault *vmf)
+static vm_fault_t kvm_spapr_tce_fault(struct vm_fault *vmf)
 {
 	struct kvmppc_spapr_tce_table *stt = vmf->vma->vm_file->private_data;
 	struct page *page;
@@ -302,7 +300,8 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 	int ret = -ENOMEM;
 	int i;
 
-	if (!args->size)
+	if (!args->size || args->page_shift < 12 || args->page_shift > 34 ||
+		(args->offset + args->size > (ULLONG_MAX >> args->page_shift)))
 		return -EINVAL;
 
 	size = _ALIGN_UP(args->size, PAGE_SIZE >> 3);
@@ -396,7 +395,7 @@ static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm,
 	return H_SUCCESS;
 }
 
-static long kvmppc_tce_iommu_unmap(struct kvm *kvm,
+static long kvmppc_tce_iommu_do_unmap(struct kvm *kvm,
 		struct iommu_table *tbl, unsigned long entry)
 {
 	enum dma_data_direction dir = DMA_NONE;
@@ -416,7 +415,24 @@ static long kvmppc_tce_iommu_unmap(struct kvm *kvm,
 	return ret;
 }
 
-long kvmppc_tce_iommu_map(struct kvm *kvm, struct iommu_table *tbl,
+static long kvmppc_tce_iommu_unmap(struct kvm *kvm,
+		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,
+		unsigned long entry)
+{
+	unsigned long i, ret = H_SUCCESS;
+	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);
+	unsigned long io_entry = entry * subpages;
+
+	for (i = 0; i < subpages; ++i) {
+		ret = kvmppc_tce_iommu_do_unmap(kvm, tbl, io_entry + i);
+		if (ret != H_SUCCESS)
+			break;
+	}
+
+	return ret;
+}
+
+long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
 		unsigned long entry, unsigned long ua,
 		enum dma_data_direction dir)
 {
@@ -453,6 +469,27 @@ long kvmppc_tce_iommu_map(struct kvm *kvm, struct iommu_table *tbl,
 	return 0;
 }
 
+static long kvmppc_tce_iommu_map(struct kvm *kvm,
+		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,
+		unsigned long entry, unsigned long ua,
+		enum dma_data_direction dir)
+{
+	unsigned long i, pgoff, ret = H_SUCCESS;
+	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);
+	unsigned long io_entry = entry * subpages;
+
+	for (i = 0, pgoff = 0; i < subpages;
+			++i, pgoff += IOMMU_PAGE_SIZE(tbl)) {
+
+		ret = kvmppc_tce_iommu_do_map(kvm, tbl,
+				io_entry + i, ua + pgoff, dir);
+		if (ret != H_SUCCESS)
+			break;
+	}
+
+	return ret;
+}
+
 long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
 		      unsigned long ioba, unsigned long tce)
 {
@@ -491,10 +528,10 @@ long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
 
 	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
 		if (dir == DMA_NONE)
-			ret = kvmppc_tce_iommu_unmap(vcpu->kvm,
+			ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt,
 					stit->tbl, entry);
 		else
-			ret = kvmppc_tce_iommu_map(vcpu->kvm, stit->tbl,
+			ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl,
 					entry, ua, dir);
 
 		if (ret == H_SUCCESS)
@@ -570,7 +607,7 @@ long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu,
 			return H_PARAMETER;
 
 		list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
-			ret = kvmppc_tce_iommu_map(vcpu->kvm,
+			ret = kvmppc_tce_iommu_map(vcpu->kvm, stt,
 					stit->tbl, entry + i, ua,
 					iommu_tce_direction(tce));
 
@@ -615,10 +652,10 @@ long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu,
 		return H_PARAMETER;
 
 	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
-		unsigned long entry = ioba >> stit->tbl->it_page_shift;
+		unsigned long entry = ioba >> stt->page_shift;
 
 		for (i = 0; i < npages; ++i) {
-			ret = kvmppc_tce_iommu_unmap(vcpu->kvm,
+			ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt,
 					stit->tbl, entry + i);
 
 			if (ret == H_SUCCESS)
diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c
index 6651f736a0b1..925fc316a104 100644
--- a/arch/powerpc/kvm/book3s_64_vio_hv.c
+++ b/arch/powerpc/kvm/book3s_64_vio_hv.c
@@ -221,7 +221,7 @@ static long kvmppc_rm_tce_iommu_mapped_dec(struct kvm *kvm,
 	return H_SUCCESS;
 }
 
-static long kvmppc_rm_tce_iommu_unmap(struct kvm *kvm,
+static long kvmppc_rm_tce_iommu_do_unmap(struct kvm *kvm,
 		struct iommu_table *tbl, unsigned long entry)
 {
 	enum dma_data_direction dir = DMA_NONE;
@@ -245,7 +245,24 @@ static long kvmppc_rm_tce_iommu_unmap(struct kvm *kvm,
 	return ret;
 }
 
-static long kvmppc_rm_tce_iommu_map(struct kvm *kvm, struct iommu_table *tbl,
+static long kvmppc_rm_tce_iommu_unmap(struct kvm *kvm,
+		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,
+		unsigned long entry)
+{
+	unsigned long i, ret = H_SUCCESS;
+	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);
+	unsigned long io_entry = entry * subpages;
+
+	for (i = 0; i < subpages; ++i) {
+		ret = kvmppc_rm_tce_iommu_do_unmap(kvm, tbl, io_entry + i);
+		if (ret != H_SUCCESS)
+			break;
+	}
+
+	return ret;
+}
+
+static long kvmppc_rm_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
 		unsigned long entry, unsigned long ua,
 		enum dma_data_direction dir)
 {
@@ -290,6 +307,27 @@ static long kvmppc_rm_tce_iommu_map(struct kvm *kvm, struct iommu_table *tbl,
 	return 0;
 }
 
+static long kvmppc_rm_tce_iommu_map(struct kvm *kvm,
+		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,
+		unsigned long entry, unsigned long ua,
+		enum dma_data_direction dir)
+{
+	unsigned long i, pgoff, ret = H_SUCCESS;
+	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);
+	unsigned long io_entry = entry * subpages;
+
+	for (i = 0, pgoff = 0; i < subpages;
+			++i, pgoff += IOMMU_PAGE_SIZE(tbl)) {
+
+		ret = kvmppc_rm_tce_iommu_do_map(kvm, tbl,
+				io_entry + i, ua + pgoff, dir);
+		if (ret != H_SUCCESS)
+			break;
+	}
+
+	return ret;
+}
+
 long kvmppc_rm_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
 		unsigned long ioba, unsigned long tce)
 {
@@ -327,10 +365,10 @@ long kvmppc_rm_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
 
 	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
 		if (dir == DMA_NONE)
-			ret = kvmppc_rm_tce_iommu_unmap(vcpu->kvm,
+			ret = kvmppc_rm_tce_iommu_unmap(vcpu->kvm, stt,
 					stit->tbl, entry);
 		else
-			ret = kvmppc_rm_tce_iommu_map(vcpu->kvm,
+			ret = kvmppc_rm_tce_iommu_map(vcpu->kvm, stt,
 					stit->tbl, entry, ua, dir);
 
 		if (ret == H_SUCCESS)
@@ -477,7 +515,7 @@ long kvmppc_rm_h_put_tce_indirect(struct kvm_vcpu *vcpu,
 			return H_PARAMETER;
 
 		list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
-			ret = kvmppc_rm_tce_iommu_map(vcpu->kvm,
+			ret = kvmppc_rm_tce_iommu_map(vcpu->kvm, stt,
 					stit->tbl, entry + i, ua,
 					iommu_tce_direction(tce));
 
@@ -526,10 +564,10 @@ long kvmppc_rm_h_stuff_tce(struct kvm_vcpu *vcpu,
 		return H_PARAMETER;
 
 	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
-		unsigned long entry = ioba >> stit->tbl->it_page_shift;
+		unsigned long entry = ioba >> stt->page_shift;
 
 		for (i = 0; i < npages; ++i) {
-			ret = kvmppc_rm_tce_iommu_unmap(vcpu->kvm,
+			ret = kvmppc_rm_tce_iommu_unmap(vcpu->kvm, stt,
 					stit->tbl, entry + i);
 
 			if (ret == H_SUCCESS)
@@ -571,7 +609,7 @@ long kvmppc_h_get_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
 	page = stt->pages[idx / TCES_PER_PAGE];
 	tbl = (u64 *)page_address(page);
 
-	vcpu->arch.gpr[4] = tbl[idx % TCES_PER_PAGE];
+	vcpu->arch.regs.gpr[4] = tbl[idx % TCES_PER_PAGE];
 
 	return H_SUCCESS;
 }
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index 9963f65c212b..b14d65b5b9dd 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -123,6 +123,32 @@ static bool no_mixing_hpt_and_radix;
 static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
 
+/*
+ * RWMR values for POWER8.  These control the rate at which PURR
+ * and SPURR count and should be set according to the number of
+ * online threads in the vcore being run.
+ */
+#define RWMR_RPA_P8_1THREAD	0x164520C62609AECA
+#define RWMR_RPA_P8_2THREAD	0x7FFF2908450D8DA9
+#define RWMR_RPA_P8_3THREAD	0x164520C62609AECA
+#define RWMR_RPA_P8_4THREAD	0x199A421245058DA9
+#define RWMR_RPA_P8_5THREAD	0x164520C62609AECA
+#define RWMR_RPA_P8_6THREAD	0x164520C62609AECA
+#define RWMR_RPA_P8_7THREAD	0x164520C62609AECA
+#define RWMR_RPA_P8_8THREAD	0x164520C62609AECA
+
+static unsigned long p8_rwmr_values[MAX_SMT_THREADS + 1] = {
+	RWMR_RPA_P8_1THREAD,
+	RWMR_RPA_P8_1THREAD,
+	RWMR_RPA_P8_2THREAD,
+	RWMR_RPA_P8_3THREAD,
+	RWMR_RPA_P8_4THREAD,
+	RWMR_RPA_P8_5THREAD,
+	RWMR_RPA_P8_6THREAD,
+	RWMR_RPA_P8_7THREAD,
+	RWMR_RPA_P8_8THREAD,
+};
+
 static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc,
 		int *ip)
 {
@@ -371,13 +397,13 @@ static void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
 
 	pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
 	pr_err("pc  = %.16lx  msr = %.16llx  trap = %x\n",
-	       vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap);
+	       vcpu->arch.regs.nip, vcpu->arch.shregs.msr, vcpu->arch.trap);
 	for (r = 0; r < 16; ++r)
 		pr_err("r%2d = %.16lx  r%d = %.16lx\n",
 		       r, kvmppc_get_gpr(vcpu, r),
 		       r+16, kvmppc_get_gpr(vcpu, r+16));
 	pr_err("ctr = %.16lx  lr  = %.16lx\n",
-	       vcpu->arch.ctr, vcpu->arch.lr);
+	       vcpu->arch.regs.ctr, vcpu->arch.regs.link);
 	pr_err("srr0 = %.16llx srr1 = %.16llx\n",
 	       vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
 	pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
@@ -385,7 +411,7 @@ static void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
 	pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
 	       vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
 	pr_err("cr = %.8x  xer = %.16lx  dsisr = %.8x\n",
-	       vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr);
+	       vcpu->arch.cr, vcpu->arch.regs.xer, vcpu->arch.shregs.dsisr);
 	pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
 	pr_err("fault dar = %.16lx dsisr = %.8x\n",
 	       vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
@@ -1526,6 +1552,9 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
 		*val = get_reg_val(id, vcpu->arch.dec_expires +
 				   vcpu->arch.vcore->tb_offset);
 		break;
+	case KVM_REG_PPC_ONLINE:
+		*val = get_reg_val(id, vcpu->arch.online);
+		break;
 	default:
 		r = -EINVAL;
 		break;
@@ -1757,6 +1786,14 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
 		vcpu->arch.dec_expires = set_reg_val(id, *val) -
 			vcpu->arch.vcore->tb_offset;
 		break;
+	case KVM_REG_PPC_ONLINE:
+		i = set_reg_val(id, *val);
+		if (i && !vcpu->arch.online)
+			atomic_inc(&vcpu->arch.vcore->online_count);
+		else if (!i && vcpu->arch.online)
+			atomic_dec(&vcpu->arch.vcore->online_count);
+		vcpu->arch.online = i;
+		break;
 	default:
 		r = -EINVAL;
 		break;
@@ -2850,6 +2887,25 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 		}
 	}
 
+	/*
+	 * On POWER8, set RWMR register.
+	 * Since it only affects PURR and SPURR, it doesn't affect
+	 * the host, so we don't save/restore the host value.
+	 */
+	if (is_power8) {
+		unsigned long rwmr_val = RWMR_RPA_P8_8THREAD;
+		int n_online = atomic_read(&vc->online_count);
+
+		/*
+		 * Use the 8-thread value if we're doing split-core
+		 * or if the vcore's online count looks bogus.
+		 */
+		if (split == 1 && threads_per_subcore == MAX_SMT_THREADS &&
+		    n_online >= 1 && n_online <= MAX_SMT_THREADS)
+			rwmr_val = p8_rwmr_values[n_online];
+		mtspr(SPRN_RWMR, rwmr_val);
+	}
+
 	/* Start all the threads */
 	active = 0;
 	for (sub = 0; sub < core_info.n_subcores; ++sub) {
@@ -2902,6 +2958,32 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 	for (sub = 0; sub < core_info.n_subcores; ++sub)
 		spin_unlock(&core_info.vc[sub]->lock);
 
+	if (kvm_is_radix(vc->kvm)) {
+		int tmp = pcpu;
+
+		/*
+		 * Do we need to flush the process scoped TLB for the LPAR?
+		 *
+		 * On POWER9, individual threads can come in here, but the
+		 * TLB is shared between the 4 threads in a core, hence
+		 * invalidating on one thread invalidates for all.
+		 * Thus we make all 4 threads use the same bit here.
+		 *
+		 * Hash must be flushed in realmode in order to use tlbiel.
+		 */
+		mtspr(SPRN_LPID, vc->kvm->arch.lpid);
+		isync();
+
+		if (cpu_has_feature(CPU_FTR_ARCH_300))
+			tmp &= ~0x3UL;
+
+		if (cpumask_test_cpu(tmp, &vc->kvm->arch.need_tlb_flush)) {
+			radix__local_flush_tlb_lpid_guest(vc->kvm->arch.lpid);
+			/* Clear the bit after the TLB flush */
+			cpumask_clear_cpu(tmp, &vc->kvm->arch.need_tlb_flush);
+		}
+	}
+
 	/*
 	 * Interrupts will be enabled once we get into the guest,
 	 * so tell lockdep that we're about to enable interrupts.
@@ -2912,8 +2994,12 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 
 	srcu_idx = srcu_read_lock(&vc->kvm->srcu);
 
+	this_cpu_disable_ftrace();
+
 	trap = __kvmppc_vcore_entry();
 
+	this_cpu_enable_ftrace();
+
 	srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
 
 	trace_hardirqs_off();
@@ -3352,6 +3438,15 @@ static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu)
 	}
 #endif
 
+	/*
+	 * Force online to 1 for the sake of old userspace which doesn't
+	 * set it.
+	 */
+	if (!vcpu->arch.online) {
+		atomic_inc(&vcpu->arch.vcore->online_count);
+		vcpu->arch.online = 1;
+	}
+
 	kvmppc_core_prepare_to_enter(vcpu);
 
 	/* No need to go into the guest when all we'll do is come back out */
diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c
index de18299f92b7..d4a3f4da409b 100644
--- a/arch/powerpc/kvm/book3s_hv_builtin.c
+++ b/arch/powerpc/kvm/book3s_hv_builtin.c
@@ -18,6 +18,7 @@
 #include <linux/cma.h>
 #include <linux/bitops.h>
 
+#include <asm/asm-prototypes.h>
 #include <asm/cputable.h>
 #include <asm/kvm_ppc.h>
 #include <asm/kvm_book3s.h>
@@ -211,9 +212,9 @@ long kvmppc_h_random(struct kvm_vcpu *vcpu)
 
 	/* Only need to do the expensive mfmsr() on radix */
 	if (kvm_is_radix(vcpu->kvm) && (mfmsr() & MSR_IR))
-		r = powernv_get_random_long(&vcpu->arch.gpr[4]);
+		r = powernv_get_random_long(&vcpu->arch.regs.gpr[4]);
 	else
-		r = powernv_get_random_real_mode(&vcpu->arch.gpr[4]);
+		r = powernv_get_random_real_mode(&vcpu->arch.regs.gpr[4]);
 	if (r)
 		return H_SUCCESS;
 
@@ -562,7 +563,7 @@ unsigned long kvmppc_rm_h_xirr_x(struct kvm_vcpu *vcpu)
 {
 	if (!kvmppc_xics_enabled(vcpu))
 		return H_TOO_HARD;
-	vcpu->arch.gpr[5] = get_tb();
+	vcpu->arch.regs.gpr[5] = get_tb();
 	if (xive_enabled()) {
 		if (is_rm())
 			return xive_rm_h_xirr(vcpu);
@@ -633,7 +634,19 @@ int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
 
 void kvmppc_bad_interrupt(struct pt_regs *regs)
 {
-	die("Bad interrupt in KVM entry/exit code", regs, SIGABRT);
+	/*
+	 * 100 could happen at any time, 200 can happen due to invalid real
+	 * address access for example (or any time due to a hardware problem).
+	 */
+	if (TRAP(regs) == 0x100) {
+		get_paca()->in_nmi++;
+		system_reset_exception(regs);
+		get_paca()->in_nmi--;
+	} else if (TRAP(regs) == 0x200) {
+		machine_check_exception(regs);
+	} else {
+		die("Bad interrupt in KVM entry/exit code", regs, SIGABRT);
+	}
 	panic("Bad KVM trap");
 }
 
diff --git a/arch/powerpc/kvm/book3s_hv_interrupts.S b/arch/powerpc/kvm/book3s_hv_interrupts.S
index 0e8493033288..82f2ff9410b6 100644
--- a/arch/powerpc/kvm/book3s_hv_interrupts.S
+++ b/arch/powerpc/kvm/book3s_hv_interrupts.S
@@ -137,7 +137,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 /*
  * We return here in virtual mode after the guest exits
  * with something that we can't handle in real mode.
- * Interrupts are enabled again at this point.
+ * Interrupts are still hard-disabled.
  */
 
 	/*
diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
index 78e6a392330f..1f22d9e977d4 100644
--- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
@@ -418,7 +418,8 @@ long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags,
 		    long pte_index, unsigned long pteh, unsigned long ptel)
 {
 	return kvmppc_do_h_enter(vcpu->kvm, flags, pte_index, pteh, ptel,
-				 vcpu->arch.pgdir, true, &vcpu->arch.gpr[4]);
+				 vcpu->arch.pgdir, true,
+				 &vcpu->arch.regs.gpr[4]);
 }
 
 #ifdef __BIG_ENDIAN__
@@ -434,24 +435,6 @@ static inline int is_mmio_hpte(unsigned long v, unsigned long r)
 		(HPTE_R_KEY_HI | HPTE_R_KEY_LO));
 }
 
-static inline int try_lock_tlbie(unsigned int *lock)
-{
-	unsigned int tmp, old;
-	unsigned int token = LOCK_TOKEN;
-
-	asm volatile("1:lwarx	%1,0,%2\n"
-		     "	cmpwi	cr0,%1,0\n"
-		     "	bne	2f\n"
-		     "  stwcx.	%3,0,%2\n"
-		     "	bne-	1b\n"
-		     "  isync\n"
-		     "2:"
-		     : "=&r" (tmp), "=&r" (old)
-		     : "r" (lock), "r" (token)
-		     : "cc", "memory");
-	return old == 0;
-}
-
 static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
 		      long npages, int global, bool need_sync)
 {
@@ -463,8 +446,6 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
 	 * the RS field, this is backwards-compatible with P7 and P8.
 	 */
 	if (global) {
-		while (!try_lock_tlbie(&kvm->arch.tlbie_lock))
-			cpu_relax();
 		if (need_sync)
 			asm volatile("ptesync" : : : "memory");
 		for (i = 0; i < npages; ++i) {
@@ -483,7 +464,6 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
 		}
 
 		asm volatile("eieio; tlbsync; ptesync" : : : "memory");
-		kvm->arch.tlbie_lock = 0;
 	} else {
 		if (need_sync)
 			asm volatile("ptesync" : : : "memory");
@@ -561,13 +541,13 @@ long kvmppc_h_remove(struct kvm_vcpu *vcpu, unsigned long flags,
 		     unsigned long pte_index, unsigned long avpn)
 {
 	return kvmppc_do_h_remove(vcpu->kvm, flags, pte_index, avpn,
-				  &vcpu->arch.gpr[4]);
+				  &vcpu->arch.regs.gpr[4]);
 }
 
 long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
 {
 	struct kvm *kvm = vcpu->kvm;
-	unsigned long *args = &vcpu->arch.gpr[4];
+	unsigned long *args = &vcpu->arch.regs.gpr[4];
 	__be64 *hp, *hptes[4];
 	unsigned long tlbrb[4];
 	long int i, j, k, n, found, indexes[4];
@@ -787,8 +767,8 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags,
 			r = rev[i].guest_rpte | (r & (HPTE_R_R | HPTE_R_C));
 			r &= ~HPTE_GR_RESERVED;
 		}
-		vcpu->arch.gpr[4 + i * 2] = v;
-		vcpu->arch.gpr[5 + i * 2] = r;
+		vcpu->arch.regs.gpr[4 + i * 2] = v;
+		vcpu->arch.regs.gpr[5 + i * 2] = r;
 	}
 	return H_SUCCESS;
 }
@@ -834,7 +814,7 @@ long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags,
 			}
 		}
 	}
-	vcpu->arch.gpr[4] = gr;
+	vcpu->arch.regs.gpr[4] = gr;
 	ret = H_SUCCESS;
  out:
 	unlock_hpte(hpte, v & ~HPTE_V_HVLOCK);
@@ -881,7 +861,7 @@ long kvmppc_h_clear_mod(struct kvm_vcpu *vcpu, unsigned long flags,
 			kvmppc_set_dirty_from_hpte(kvm, v, gr);
 		}
 	}
-	vcpu->arch.gpr[4] = gr;
+	vcpu->arch.regs.gpr[4] = gr;
 	ret = H_SUCCESS;
  out:
 	unlock_hpte(hpte, v & ~HPTE_V_HVLOCK);
diff --git a/arch/powerpc/kvm/book3s_hv_rm_xics.c b/arch/powerpc/kvm/book3s_hv_rm_xics.c
index 2a862618f072..758d1d23215e 100644
--- a/arch/powerpc/kvm/book3s_hv_rm_xics.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_xics.c
@@ -517,7 +517,7 @@ unsigned long xics_rm_h_xirr(struct kvm_vcpu *vcpu)
 	} while (!icp_rm_try_update(icp, old_state, new_state));
 
 	/* Return the result in GPR4 */
-	vcpu->arch.gpr[4] = xirr;
+	vcpu->arch.regs.gpr[4] = xirr;
 
 	return check_too_hard(xics, icp);
 }
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index 07ca1b2a7966..9a3b00eda67b 100644
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -342,6 +342,9 @@ kvm_start_guest:
 
 	ld	r2,PACATOC(r13)
 
+	li	r0,0
+	stb	r0,PACA_FTRACE_ENABLED(r13)
+
 	li	r0,KVM_HWTHREAD_IN_KVM
 	stb	r0,HSTATE_HWTHREAD_STATE(r13)
 
@@ -636,6 +639,10 @@ kvmppc_hv_entry:
 	/* Primary thread switches to guest partition. */
 	cmpwi	r6,0
 	bne	10f
+
+	/* Radix has already switched LPID and flushed core TLB */
+	bne	cr7, 22f
+
 	lwz	r7,KVM_LPID(r9)
 BEGIN_FTR_SECTION
 	ld	r6,KVM_SDR1(r9)
@@ -647,7 +654,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
 	mtspr	SPRN_LPID,r7
 	isync
 
-	/* See if we need to flush the TLB */
+	/* See if we need to flush the TLB. Hash has to be done in RM */
 	lhz	r6,PACAPACAINDEX(r13)	/* test_bit(cpu, need_tlb_flush) */
 BEGIN_FTR_SECTION
 	/*
@@ -674,15 +681,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 	li	r7,0x800		/* IS field = 0b10 */
 	ptesync
 	li	r0,0			/* RS for P9 version of tlbiel */
-	bne	cr7, 29f
 28:	tlbiel	r7			/* On P9, rs=0, RIC=0, PRS=0, R=0 */
 	addi	r7,r7,0x1000
 	bdnz	28b
-	b	30f
-29:	PPC_TLBIEL(7,0,2,1,1)		/* for radix, RIC=2, PRS=1, R=1 */
-	addi	r7,r7,0x1000
-	bdnz	29b
-30:	ptesync
+	ptesync
 23:	ldarx	r7,0,r6			/* clear the bit after TLB flushed */
 	andc	r7,r7,r8
 	stdcx.	r7,0,r6
@@ -3569,6 +3571,8 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
 	bcl	20, 31, .+4
 5:	mflr	r3
 	addi	r3, r3, 9f - 5b
+	li	r4, -1
+	rldimi	r3, r4, 62, 0	/* ensure 0xc000000000000000 bits are set */
 	ld	r4, PACAKMSR(r13)
 	mtspr	SPRN_SRR0, r3
 	mtspr	SPRN_SRR1, r4
diff --git a/arch/powerpc/kvm/book3s_hv_tm.c b/arch/powerpc/kvm/book3s_hv_tm.c
index bf710ad3a6d7..008285058f9b 100644
--- a/arch/powerpc/kvm/book3s_hv_tm.c
+++ b/arch/powerpc/kvm/book3s_hv_tm.c
@@ -19,7 +19,7 @@ static void emulate_tx_failure(struct kvm_vcpu *vcpu, u64 failure_cause)
 	u64 texasr, tfiar;
 	u64 msr = vcpu->arch.shregs.msr;
 
-	tfiar = vcpu->arch.pc & ~0x3ull;
+	tfiar = vcpu->arch.regs.nip & ~0x3ull;
 	texasr = (failure_cause << 56) | TEXASR_ABORT | TEXASR_FS | TEXASR_EXACT;
 	if (MSR_TM_SUSPENDED(vcpu->arch.shregs.msr))
 		texasr |= TEXASR_SUSP;
@@ -57,8 +57,8 @@ int kvmhv_p9_tm_emulation(struct kvm_vcpu *vcpu)
 			       (newmsr & MSR_TM)));
 		newmsr = sanitize_msr(newmsr);
 		vcpu->arch.shregs.msr = newmsr;
-		vcpu->arch.cfar = vcpu->arch.pc - 4;
-		vcpu->arch.pc = vcpu->arch.shregs.srr0;
+		vcpu->arch.cfar = vcpu->arch.regs.nip - 4;
+		vcpu->arch.regs.nip = vcpu->arch.shregs.srr0;
 		return RESUME_GUEST;
 
 	case PPC_INST_RFEBB:
@@ -90,8 +90,8 @@ int kvmhv_p9_tm_emulation(struct kvm_vcpu *vcpu)
 		vcpu->arch.bescr = bescr;
 		msr = (msr & ~MSR_TS_MASK) | MSR_TS_T;
 		vcpu->arch.shregs.msr = msr;
-		vcpu->arch.cfar = vcpu->arch.pc - 4;
-		vcpu->arch.pc = vcpu->arch.ebbrr;
+		vcpu->arch.cfar = vcpu->arch.regs.nip - 4;
+		vcpu->arch.regs.nip = vcpu->arch.ebbrr;
 		return RESUME_GUEST;
 
 	case PPC_INST_MTMSRD:
diff --git a/arch/powerpc/kvm/book3s_hv_tm_builtin.c b/arch/powerpc/kvm/book3s_hv_tm_builtin.c
index d98ccfd2b88c..b2c7c6fca4f9 100644
--- a/arch/powerpc/kvm/book3s_hv_tm_builtin.c
+++ b/arch/powerpc/kvm/book3s_hv_tm_builtin.c
@@ -35,8 +35,8 @@ int kvmhv_p9_tm_emulation_early(struct kvm_vcpu *vcpu)
 			return 0;
 		newmsr = sanitize_msr(newmsr);
 		vcpu->arch.shregs.msr = newmsr;
-		vcpu->arch.cfar = vcpu->arch.pc - 4;
-		vcpu->arch.pc = vcpu->arch.shregs.srr0;
+		vcpu->arch.cfar = vcpu->arch.regs.nip - 4;
+		vcpu->arch.regs.nip = vcpu->arch.shregs.srr0;
 		return 1;
 
 	case PPC_INST_RFEBB:
@@ -58,8 +58,8 @@ int kvmhv_p9_tm_emulation_early(struct kvm_vcpu *vcpu)
 		mtspr(SPRN_BESCR, bescr);
 		msr = (msr & ~MSR_TS_MASK) | MSR_TS_T;
 		vcpu->arch.shregs.msr = msr;
-		vcpu->arch.cfar = vcpu->arch.pc - 4;
-		vcpu->arch.pc = mfspr(SPRN_EBBRR);
+		vcpu->arch.cfar = vcpu->arch.regs.nip - 4;
+		vcpu->arch.regs.nip = mfspr(SPRN_EBBRR);
 		return 1;
 
 	case PPC_INST_MTMSRD:
@@ -103,7 +103,7 @@ int kvmhv_p9_tm_emulation_early(struct kvm_vcpu *vcpu)
 void kvmhv_emulate_tm_rollback(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.shregs.msr &= ~MSR_TS_MASK;	/* go to N state */
-	vcpu->arch.pc = vcpu->arch.tfhar;
+	vcpu->arch.regs.nip = vcpu->arch.tfhar;
 	copy_from_checkpoint(vcpu);
 	vcpu->arch.cr = (vcpu->arch.cr & 0x0fffffff) | 0xa0000000;
 }
diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c
index d3f304d06adf..c74a8885427d 100644
--- a/arch/powerpc/kvm/book3s_pr.c
+++ b/arch/powerpc/kvm/book3s_pr.c
@@ -147,25 +147,25 @@ void kvmppc_copy_to_svcpu(struct kvm_vcpu *vcpu)
 {
 	struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu);
 
-	svcpu->gpr[0] = vcpu->arch.gpr[0];
-	svcpu->gpr[1] = vcpu->arch.gpr[1];
-	svcpu->gpr[2] = vcpu->arch.gpr[2];
-	svcpu->gpr[3] = vcpu->arch.gpr[3];
-	svcpu->gpr[4] = vcpu->arch.gpr[4];
-	svcpu->gpr[5] = vcpu->arch.gpr[5];
-	svcpu->gpr[6] = vcpu->arch.gpr[6];
-	svcpu->gpr[7] = vcpu->arch.gpr[7];
-	svcpu->gpr[8] = vcpu->arch.gpr[8];
-	svcpu->gpr[9] = vcpu->arch.gpr[9];
-	svcpu->gpr[10] = vcpu->arch.gpr[10];
-	svcpu->gpr[11] = vcpu->arch.gpr[11];
-	svcpu->gpr[12] = vcpu->arch.gpr[12];
-	svcpu->gpr[13] = vcpu->arch.gpr[13];
+	svcpu->gpr[0] = vcpu->arch.regs.gpr[0];
+	svcpu->gpr[1] = vcpu->arch.regs.gpr[1];
+	svcpu->gpr[2] = vcpu->arch.regs.gpr[2];
+	svcpu->gpr[3] = vcpu->arch.regs.gpr[3];
+	svcpu->gpr[4] = vcpu->arch.regs.gpr[4];
+	svcpu->gpr[5] = vcpu->arch.regs.gpr[5];
+	svcpu->gpr[6] = vcpu->arch.regs.gpr[6];
+	svcpu->gpr[7] = vcpu->arch.regs.gpr[7];
+	svcpu->gpr[8] = vcpu->arch.regs.gpr[8];
+	svcpu->gpr[9] = vcpu->arch.regs.gpr[9];
+	svcpu->gpr[10] = vcpu->arch.regs.gpr[10];
+	svcpu->gpr[11] = vcpu->arch.regs.gpr[11];
+	svcpu->gpr[12] = vcpu->arch.regs.gpr[12];
+	svcpu->gpr[13] = vcpu->arch.regs.gpr[13];
 	svcpu->cr  = vcpu->arch.cr;
-	svcpu->xer = vcpu->arch.xer;
-	svcpu->ctr = vcpu->arch.ctr;
-	svcpu->lr  = vcpu->arch.lr;
-	svcpu->pc  = vcpu->arch.pc;
+	svcpu->xer = vcpu->arch.regs.xer;
+	svcpu->ctr = vcpu->arch.regs.ctr;
+	svcpu->lr  = vcpu->arch.regs.link;
+	svcpu->pc  = vcpu->arch.regs.nip;
 #ifdef CONFIG_PPC_BOOK3S_64
 	svcpu->shadow_fscr = vcpu->arch.shadow_fscr;
 #endif
@@ -194,25 +194,25 @@ void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu)
 	if (!svcpu->in_use)
 		goto out;
 
-	vcpu->arch.gpr[0] = svcpu->gpr[0];
-	vcpu->arch.gpr[1] = svcpu->gpr[1];
-	vcpu->arch.gpr[2] = svcpu->gpr[2];
-	vcpu->arch.gpr[3] = svcpu->gpr[3];
-	vcpu->arch.gpr[4] = svcpu->gpr[4];
-	vcpu->arch.gpr[5] = svcpu->gpr[5];
-	vcpu->arch.gpr[6] = svcpu->gpr[6];
-	vcpu->arch.gpr[7] = svcpu->gpr[7];
-	vcpu->arch.gpr[8] = svcpu->gpr[8];
-	vcpu->arch.gpr[9] = svcpu->gpr[9];
-	vcpu->arch.gpr[10] = svcpu->gpr[10];
-	vcpu->arch.gpr[11] = svcpu->gpr[11];
-	vcpu->arch.gpr[12] = svcpu->gpr[12];
-	vcpu->arch.gpr[13] = svcpu->gpr[13];
+	vcpu->arch.regs.gpr[0] = svcpu->gpr[0];
+	vcpu->arch.regs.gpr[1] = svcpu->gpr[1];
+	vcpu->arch.regs.gpr[2] = svcpu->gpr[2];
+	vcpu->arch.regs.gpr[3] = svcpu->gpr[3];
+	vcpu->arch.regs.gpr[4] = svcpu->gpr[4];
+	vcpu->arch.regs.gpr[5] = svcpu->gpr[5];
+	vcpu->arch.regs.gpr[6] = svcpu->gpr[6];
+	vcpu->arch.regs.gpr[7] = svcpu->gpr[7];
+	vcpu->arch.regs.gpr[8] = svcpu->gpr[8];
+	vcpu->arch.regs.gpr[9] = svcpu->gpr[9];
+	vcpu->arch.regs.gpr[10] = svcpu->gpr[10];
+	vcpu->arch.regs.gpr[11] = svcpu->gpr[11];
+	vcpu->arch.regs.gpr[12] = svcpu->gpr[12];
+	vcpu->arch.regs.gpr[13] = svcpu->gpr[13];
 	vcpu->arch.cr  = svcpu->cr;
-	vcpu->arch.xer = svcpu->xer;
-	vcpu->arch.ctr = svcpu->ctr;
-	vcpu->arch.lr  = svcpu->lr;
-	vcpu->arch.pc  = svcpu->pc;
+	vcpu->arch.regs.xer = svcpu->xer;
+	vcpu->arch.regs.ctr = svcpu->ctr;
+	vcpu->arch.regs.link  = svcpu->lr;
+	vcpu->arch.regs.nip  = svcpu->pc;
 	vcpu->arch.shadow_srr1 = svcpu->shadow_srr1;
 	vcpu->arch.fault_dar   = svcpu->fault_dar;
 	vcpu->arch.fault_dsisr = svcpu->fault_dsisr;
@@ -1735,9 +1735,16 @@ static void kvmppc_core_destroy_vm_pr(struct kvm *kvm)
 static int kvmppc_core_check_processor_compat_pr(void)
 {
 	/*
-	 * Disable KVM for Power9 untill the required bits merged.
+	 * PR KVM can work on POWER9 inside a guest partition
+	 * running in HPT mode.  It can't work if we are using
+	 * radix translation (because radix provides no way for
+	 * a process to have unique translations in quadrant 3)
+	 * or in a bare-metal HPT-mode host (because POWER9
+	 * uses a modified HPTE format which the PR KVM code
+	 * has not been adapted to use).
 	 */
-	if (cpu_has_feature(CPU_FTR_ARCH_300))
+	if (cpu_has_feature(CPU_FTR_ARCH_300) &&
+	    (radix_enabled() || cpu_has_feature(CPU_FTR_HVMODE)))
 		return -EIO;
 	return 0;
 }
@@ -1782,6 +1789,7 @@ static struct kvmppc_ops kvm_ops_pr = {
 #ifdef CONFIG_PPC_BOOK3S_64
 	.hcall_implemented = kvmppc_hcall_impl_pr,
 #endif
+	.giveup_ext = kvmppc_giveup_ext,
 };
 
 
diff --git a/arch/powerpc/kvm/book3s_xive_template.c b/arch/powerpc/kvm/book3s_xive_template.c
index 99c3620b40d9..6e41ba7ec8f4 100644
--- a/arch/powerpc/kvm/book3s_xive_template.c
+++ b/arch/powerpc/kvm/book3s_xive_template.c
@@ -334,7 +334,7 @@ X_STATIC unsigned long GLUE(X_PFX,h_xirr)(struct kvm_vcpu *vcpu)
 	 */
 
 	/* Return interrupt and old CPPR in GPR4 */
-	vcpu->arch.gpr[4] = hirq | (old_cppr << 24);
+	vcpu->arch.regs.gpr[4] = hirq | (old_cppr << 24);
 
 	return H_SUCCESS;
 }
@@ -369,7 +369,7 @@ X_STATIC unsigned long GLUE(X_PFX,h_ipoll)(struct kvm_vcpu *vcpu, unsigned long
 	hirq = GLUE(X_PFX,scan_interrupts)(xc, pending, scan_poll);
 
 	/* Return interrupt and old CPPR in GPR4 */
-	vcpu->arch.gpr[4] = hirq | (xc->cppr << 24);
+	vcpu->arch.regs.gpr[4] = hirq | (xc->cppr << 24);
 
 	return H_SUCCESS;
 }
diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c
index 876d4f294fdd..a9ca016da670 100644
--- a/arch/powerpc/kvm/booke.c
+++ b/arch/powerpc/kvm/booke.c
@@ -77,8 +77,10 @@ void kvmppc_dump_vcpu(struct kvm_vcpu *vcpu)
 {
 	int i;
 
-	printk("pc:   %08lx msr:  %08llx\n", vcpu->arch.pc, vcpu->arch.shared->msr);
-	printk("lr:   %08lx ctr:  %08lx\n", vcpu->arch.lr, vcpu->arch.ctr);
+	printk("pc:   %08lx msr:  %08llx\n", vcpu->arch.regs.nip,
+			vcpu->arch.shared->msr);
+	printk("lr:   %08lx ctr:  %08lx\n", vcpu->arch.regs.link,
+			vcpu->arch.regs.ctr);
 	printk("srr0: %08llx srr1: %08llx\n", vcpu->arch.shared->srr0,
 					    vcpu->arch.shared->srr1);
 
@@ -491,24 +493,25 @@ static int kvmppc_booke_irqprio_deliver(struct kvm_vcpu *vcpu,
 	if (allowed) {
 		switch (int_class) {
 		case INT_CLASS_NONCRIT:
-			set_guest_srr(vcpu, vcpu->arch.pc,
+			set_guest_srr(vcpu, vcpu->arch.regs.nip,
 				      vcpu->arch.shared->msr);
 			break;
 		case INT_CLASS_CRIT:
-			set_guest_csrr(vcpu, vcpu->arch.pc,
+			set_guest_csrr(vcpu, vcpu->arch.regs.nip,
 				       vcpu->arch.shared->msr);
 			break;
 		case INT_CLASS_DBG:
-			set_guest_dsrr(vcpu, vcpu->arch.pc,
+			set_guest_dsrr(vcpu, vcpu->arch.regs.nip,
 				       vcpu->arch.shared->msr);
 			break;
 		case INT_CLASS_MC:
-			set_guest_mcsrr(vcpu, vcpu->arch.pc,
+			set_guest_mcsrr(vcpu, vcpu->arch.regs.nip,
 					vcpu->arch.shared->msr);
 			break;
 		}
 
-		vcpu->arch.pc = vcpu->arch.ivpr | vcpu->arch.ivor[priority];
+		vcpu->arch.regs.nip = vcpu->arch.ivpr |
+					vcpu->arch.ivor[priority];
 		if (update_esr == true)
 			kvmppc_set_esr(vcpu, vcpu->arch.queued_esr);
 		if (update_dear == true)
@@ -826,7 +829,7 @@ static int emulation_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
 
 	case EMULATE_FAIL:
 		printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
-		       __func__, vcpu->arch.pc, vcpu->arch.last_inst);
+		       __func__, vcpu->arch.regs.nip, vcpu->arch.last_inst);
 		/* For debugging, encode the failing instruction and
 		 * report it to userspace. */
 		run->hw.hardware_exit_reason = ~0ULL << 32;
@@ -875,7 +878,7 @@ static int kvmppc_handle_debug(struct kvm_run *run, struct kvm_vcpu *vcpu)
 	 */
 	vcpu->arch.dbsr = 0;
 	run->debug.arch.status = 0;
-	run->debug.arch.address = vcpu->arch.pc;
+	run->debug.arch.address = vcpu->arch.regs.nip;
 
 	if (dbsr & (DBSR_IAC1 | DBSR_IAC2 | DBSR_IAC3 | DBSR_IAC4)) {
 		run->debug.arch.status |= KVMPPC_DEBUG_BREAKPOINT;
@@ -971,7 +974,7 @@ static int kvmppc_resume_inst_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
 
 	case EMULATE_FAIL:
 		pr_debug("%s: load instruction from guest address %lx failed\n",
-		       __func__, vcpu->arch.pc);
+		       __func__, vcpu->arch.regs.nip);
 		/* For debugging, encode the failing instruction and
 		 * report it to userspace. */
 		run->hw.hardware_exit_reason = ~0ULL << 32;
@@ -1169,7 +1172,7 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	case BOOKE_INTERRUPT_SPE_FP_DATA:
 	case BOOKE_INTERRUPT_SPE_FP_ROUND:
 		printk(KERN_CRIT "%s: unexpected SPE interrupt %u at %08lx\n",
-		       __func__, exit_nr, vcpu->arch.pc);
+		       __func__, exit_nr, vcpu->arch.regs.nip);
 		run->hw.hardware_exit_reason = exit_nr;
 		r = RESUME_HOST;
 		break;
@@ -1299,7 +1302,7 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	}
 
 	case BOOKE_INTERRUPT_ITLB_MISS: {
-		unsigned long eaddr = vcpu->arch.pc;
+		unsigned long eaddr = vcpu->arch.regs.nip;
 		gpa_t gpaddr;
 		gfn_t gfn;
 		int gtlb_index;
@@ -1391,7 +1394,7 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 	int i;
 	int r;
 
-	vcpu->arch.pc = 0;
+	vcpu->arch.regs.nip = 0;
 	vcpu->arch.shared->pir = vcpu->vcpu_id;
 	kvmppc_set_gpr(vcpu, 1, (16<<20) - 8); /* -8 for the callee-save LR slot */
 	kvmppc_set_msr(vcpu, 0);
@@ -1440,10 +1443,10 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 
 	vcpu_load(vcpu);
 
-	regs->pc = vcpu->arch.pc;
+	regs->pc = vcpu->arch.regs.nip;
 	regs->cr = kvmppc_get_cr(vcpu);
-	regs->ctr = vcpu->arch.ctr;
-	regs->lr = vcpu->arch.lr;
+	regs->ctr = vcpu->arch.regs.ctr;
+	regs->lr = vcpu->arch.regs.link;
 	regs->xer = kvmppc_get_xer(vcpu);
 	regs->msr = vcpu->arch.shared->msr;
 	regs->srr0 = kvmppc_get_srr0(vcpu);
@@ -1471,10 +1474,10 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 
 	vcpu_load(vcpu);
 
-	vcpu->arch.pc = regs->pc;
+	vcpu->arch.regs.nip = regs->pc;
 	kvmppc_set_cr(vcpu, regs->cr);
-	vcpu->arch.ctr = regs->ctr;
-	vcpu->arch.lr = regs->lr;
+	vcpu->arch.regs.ctr = regs->ctr;
+	vcpu->arch.regs.link = regs->lr;
 	kvmppc_set_xer(vcpu, regs->xer);
 	kvmppc_set_msr(vcpu, regs->msr);
 	kvmppc_set_srr0(vcpu, regs->srr0);
diff --git a/arch/powerpc/kvm/booke_emulate.c b/arch/powerpc/kvm/booke_emulate.c
index a82f64502de1..d23e582f0fee 100644
--- a/arch/powerpc/kvm/booke_emulate.c
+++ b/arch/powerpc/kvm/booke_emulate.c
@@ -34,19 +34,19 @@
 
 static void kvmppc_emul_rfi(struct kvm_vcpu *vcpu)
 {
-	vcpu->arch.pc = vcpu->arch.shared->srr0;
+	vcpu->arch.regs.nip = vcpu->arch.shared->srr0;
 	kvmppc_set_msr(vcpu, vcpu->arch.shared->srr1);
 }
 
 static void kvmppc_emul_rfdi(struct kvm_vcpu *vcpu)
 {
-	vcpu->arch.pc = vcpu->arch.dsrr0;
+	vcpu->arch.regs.nip = vcpu->arch.dsrr0;
 	kvmppc_set_msr(vcpu, vcpu->arch.dsrr1);
 }
 
 static void kvmppc_emul_rfci(struct kvm_vcpu *vcpu)
 {
-	vcpu->arch.pc = vcpu->arch.csrr0;
+	vcpu->arch.regs.nip = vcpu->arch.csrr0;
 	kvmppc_set_msr(vcpu, vcpu->arch.csrr1);
 }
 
diff --git a/arch/powerpc/kvm/e500_emulate.c b/arch/powerpc/kvm/e500_emulate.c
index 990db69a1d0b..3f8189eb56ed 100644
--- a/arch/powerpc/kvm/e500_emulate.c
+++ b/arch/powerpc/kvm/e500_emulate.c
@@ -53,7 +53,7 @@ static int dbell2prio(ulong param)
 
 static int kvmppc_e500_emul_msgclr(struct kvm_vcpu *vcpu, int rb)
 {
-	ulong param = vcpu->arch.gpr[rb];
+	ulong param = vcpu->arch.regs.gpr[rb];
 	int prio = dbell2prio(param);
 
 	if (prio < 0)
@@ -65,7 +65,7 @@ static int kvmppc_e500_emul_msgclr(struct kvm_vcpu *vcpu, int rb)
 
 static int kvmppc_e500_emul_msgsnd(struct kvm_vcpu *vcpu, int rb)
 {
-	ulong param = vcpu->arch.gpr[rb];
+	ulong param = vcpu->arch.regs.gpr[rb];
 	int prio = dbell2prio(rb);
 	int pir = param & PPC_DBELL_PIR_MASK;
 	int i;
@@ -94,7 +94,7 @@ static int kvmppc_e500_emul_ehpriv(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	switch (get_oc(inst)) {
 	case EHPRIV_OC_DEBUG:
 		run->exit_reason = KVM_EXIT_DEBUG;
-		run->debug.arch.address = vcpu->arch.pc;
+		run->debug.arch.address = vcpu->arch.regs.nip;
 		run->debug.arch.status = 0;
 		kvmppc_account_exit(vcpu, DEBUG_EXITS);
 		emulated = EMULATE_EXIT_USER;
diff --git a/arch/powerpc/kvm/e500_mmu.c b/arch/powerpc/kvm/e500_mmu.c
index ddbf8f0284c0..24296f4cadc6 100644
--- a/arch/powerpc/kvm/e500_mmu.c
+++ b/arch/powerpc/kvm/e500_mmu.c
@@ -513,7 +513,7 @@ void kvmppc_mmu_itlb_miss(struct kvm_vcpu *vcpu)
 {
 	unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS);
 
-	kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.pc, as);
+	kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.regs.nip, as);
 }
 
 void kvmppc_mmu_dtlb_miss(struct kvm_vcpu *vcpu)
diff --git a/arch/powerpc/kvm/e500_mmu_host.c b/arch/powerpc/kvm/e500_mmu_host.c
index c878b4ffb86f..8f2985e46f6f 100644
--- a/arch/powerpc/kvm/e500_mmu_host.c
+++ b/arch/powerpc/kvm/e500_mmu_host.c
@@ -625,8 +625,8 @@ void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 eaddr, gpa_t gpaddr,
 }
 
 #ifdef CONFIG_KVM_BOOKE_HV
-int kvmppc_load_last_inst(struct kvm_vcpu *vcpu, enum instruction_type type,
-			  u32 *instr)
+int kvmppc_load_last_inst(struct kvm_vcpu *vcpu,
+		enum instruction_fetch_type type, u32 *instr)
 {
 	gva_t geaddr;
 	hpa_t addr;
@@ -715,8 +715,8 @@ int kvmppc_load_last_inst(struct kvm_vcpu *vcpu, enum instruction_type type,
 	return EMULATE_DONE;
 }
 #else
-int kvmppc_load_last_inst(struct kvm_vcpu *vcpu, enum instruction_type type,
-			  u32 *instr)
+int kvmppc_load_last_inst(struct kvm_vcpu *vcpu,
+		enum instruction_fetch_type type, u32 *instr)
 {
 	return EMULATE_AGAIN;
 }
diff --git a/arch/powerpc/kvm/emulate_loadstore.c b/arch/powerpc/kvm/emulate_loadstore.c
index a382e15135e6..afde788be141 100644
--- a/arch/powerpc/kvm/emulate_loadstore.c
+++ b/arch/powerpc/kvm/emulate_loadstore.c
@@ -31,6 +31,7 @@
 #include <asm/kvm_ppc.h>
 #include <asm/disassemble.h>
 #include <asm/ppc-opcode.h>
+#include <asm/sstep.h>
 #include "timing.h"
 #include "trace.h"
 
@@ -84,8 +85,9 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu)
 	struct kvm_run *run = vcpu->run;
 	u32 inst;
 	int ra, rs, rt;
-	enum emulation_result emulated;
+	enum emulation_result emulated = EMULATE_FAIL;
 	int advance = 1;
+	struct instruction_op op;
 
 	/* this default type might be overwritten by subcategories */
 	kvmppc_set_exit_type(vcpu, EMULATED_INST_EXITS);
@@ -107,580 +109,276 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu)
 	vcpu->arch.mmio_vsx_tx_sx_enabled = get_tx_or_sx(inst);
 	vcpu->arch.mmio_vsx_copy_nums = 0;
 	vcpu->arch.mmio_vsx_offset = 0;
-	vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_NONE;
+	vcpu->arch.mmio_copy_type = KVMPPC_VSX_COPY_NONE;
 	vcpu->arch.mmio_sp64_extend = 0;
 	vcpu->arch.mmio_sign_extend = 0;
 	vcpu->arch.mmio_vmx_copy_nums = 0;
+	vcpu->arch.mmio_vmx_offset = 0;
+	vcpu->arch.mmio_host_swabbed = 0;
 
-	switch (get_op(inst)) {
-	case 31:
-		switch (get_xop(inst)) {
-		case OP_31_XOP_LWZX:
-			emulated = kvmppc_handle_load(run, vcpu, rt, 4, 1);
-			break;
-
-		case OP_31_XOP_LWZUX:
-			emulated = kvmppc_handle_load(run, vcpu, rt, 4, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
+	emulated = EMULATE_FAIL;
+	vcpu->arch.regs.msr = vcpu->arch.shared->msr;
+	vcpu->arch.regs.ccr = vcpu->arch.cr;
+	if (analyse_instr(&op, &vcpu->arch.regs, inst) == 0) {
+		int type = op.type & INSTR_TYPE_MASK;
+		int size = GETSIZE(op.type);
 
-		case OP_31_XOP_LBZX:
-			emulated = kvmppc_handle_load(run, vcpu, rt, 1, 1);
-			break;
-
-		case OP_31_XOP_LBZUX:
-			emulated = kvmppc_handle_load(run, vcpu, rt, 1, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
+		switch (type) {
+		case LOAD:  {
+			int instr_byte_swap = op.type & BYTEREV;
 
-		case OP_31_XOP_STDX:
-			emulated = kvmppc_handle_store(run, vcpu,
-					kvmppc_get_gpr(vcpu, rs), 8, 1);
-			break;
+			if (op.type & SIGNEXT)
+				emulated = kvmppc_handle_loads(run, vcpu,
+						op.reg, size, !instr_byte_swap);
+			else
+				emulated = kvmppc_handle_load(run, vcpu,
+						op.reg, size, !instr_byte_swap);
 
-		case OP_31_XOP_STDUX:
-			emulated = kvmppc_handle_store(run, vcpu,
-					kvmppc_get_gpr(vcpu, rs), 8, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
+			if ((op.type & UPDATE) && (emulated != EMULATE_FAIL))
+				kvmppc_set_gpr(vcpu, op.update_reg, op.ea);
 
-		case OP_31_XOP_STWX:
-			emulated = kvmppc_handle_store(run, vcpu,
-					kvmppc_get_gpr(vcpu, rs), 4, 1);
-			break;
-
-		case OP_31_XOP_STWUX:
-			emulated = kvmppc_handle_store(run, vcpu,
-					kvmppc_get_gpr(vcpu, rs), 4, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
-
-		case OP_31_XOP_STBX:
-			emulated = kvmppc_handle_store(run, vcpu,
-					kvmppc_get_gpr(vcpu, rs), 1, 1);
-			break;
-
-		case OP_31_XOP_STBUX:
-			emulated = kvmppc_handle_store(run, vcpu,
-					kvmppc_get_gpr(vcpu, rs), 1, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
-
-		case OP_31_XOP_LHAX:
-			emulated = kvmppc_handle_loads(run, vcpu, rt, 2, 1);
-			break;
-
-		case OP_31_XOP_LHAUX:
-			emulated = kvmppc_handle_loads(run, vcpu, rt, 2, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
-
-		case OP_31_XOP_LHZX:
-			emulated = kvmppc_handle_load(run, vcpu, rt, 2, 1);
-			break;
-
-		case OP_31_XOP_LHZUX:
-			emulated = kvmppc_handle_load(run, vcpu, rt, 2, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
-
-		case OP_31_XOP_STHX:
-			emulated = kvmppc_handle_store(run, vcpu,
-					kvmppc_get_gpr(vcpu, rs), 2, 1);
-			break;
-
-		case OP_31_XOP_STHUX:
-			emulated = kvmppc_handle_store(run, vcpu,
-					kvmppc_get_gpr(vcpu, rs), 2, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
-
-		case OP_31_XOP_DCBST:
-		case OP_31_XOP_DCBF:
-		case OP_31_XOP_DCBI:
-			/* Do nothing. The guest is performing dcbi because
-			 * hardware DMA is not snooped by the dcache, but
-			 * emulated DMA either goes through the dcache as
-			 * normal writes, or the host kernel has handled dcache
-			 * coherence. */
-			break;
-
-		case OP_31_XOP_LWBRX:
-			emulated = kvmppc_handle_load(run, vcpu, rt, 4, 0);
-			break;
-
-		case OP_31_XOP_STWBRX:
-			emulated = kvmppc_handle_store(run, vcpu,
-					kvmppc_get_gpr(vcpu, rs), 4, 0);
 			break;
-
-		case OP_31_XOP_LHBRX:
-			emulated = kvmppc_handle_load(run, vcpu, rt, 2, 0);
-			break;
-
-		case OP_31_XOP_STHBRX:
-			emulated = kvmppc_handle_store(run, vcpu,
-					kvmppc_get_gpr(vcpu, rs), 2, 0);
-			break;
-
-		case OP_31_XOP_LDBRX:
-			emulated = kvmppc_handle_load(run, vcpu, rt, 8, 0);
-			break;
-
-		case OP_31_XOP_STDBRX:
-			emulated = kvmppc_handle_store(run, vcpu,
-					kvmppc_get_gpr(vcpu, rs), 8, 0);
-			break;
-
-		case OP_31_XOP_LDX:
-			emulated = kvmppc_handle_load(run, vcpu, rt, 8, 1);
-			break;
-
-		case OP_31_XOP_LDUX:
-			emulated = kvmppc_handle_load(run, vcpu, rt, 8, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
-
-		case OP_31_XOP_LWAX:
-			emulated = kvmppc_handle_loads(run, vcpu, rt, 4, 1);
-			break;
-
-		case OP_31_XOP_LWAUX:
-			emulated = kvmppc_handle_loads(run, vcpu, rt, 4, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
-
+		}
 #ifdef CONFIG_PPC_FPU
-		case OP_31_XOP_LFSX:
-			if (kvmppc_check_fp_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_sp64_extend = 1;
-			emulated = kvmppc_handle_load(run, vcpu,
-				KVM_MMIO_REG_FPR|rt, 4, 1);
-			break;
-
-		case OP_31_XOP_LFSUX:
+		case LOAD_FP:
 			if (kvmppc_check_fp_disabled(vcpu))
 				return EMULATE_DONE;
-			vcpu->arch.mmio_sp64_extend = 1;
-			emulated = kvmppc_handle_load(run, vcpu,
-				KVM_MMIO_REG_FPR|rt, 4, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
 
-		case OP_31_XOP_LFDX:
-			if (kvmppc_check_fp_disabled(vcpu))
-				return EMULATE_DONE;
-			emulated = kvmppc_handle_load(run, vcpu,
-				KVM_MMIO_REG_FPR|rt, 8, 1);
-			break;
+			if (op.type & FPCONV)
+				vcpu->arch.mmio_sp64_extend = 1;
 
-		case OP_31_XOP_LFDUX:
-			if (kvmppc_check_fp_disabled(vcpu))
-				return EMULATE_DONE;
-			emulated = kvmppc_handle_load(run, vcpu,
-				KVM_MMIO_REG_FPR|rt, 8, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
-
-		case OP_31_XOP_LFIWAX:
-			if (kvmppc_check_fp_disabled(vcpu))
-				return EMULATE_DONE;
-			emulated = kvmppc_handle_loads(run, vcpu,
-				KVM_MMIO_REG_FPR|rt, 4, 1);
-			break;
+			if (op.type & SIGNEXT)
+				emulated = kvmppc_handle_loads(run, vcpu,
+					     KVM_MMIO_REG_FPR|op.reg, size, 1);
+			else
+				emulated = kvmppc_handle_load(run, vcpu,
+					     KVM_MMIO_REG_FPR|op.reg, size, 1);
 
-		case OP_31_XOP_LFIWZX:
-			if (kvmppc_check_fp_disabled(vcpu))
-				return EMULATE_DONE;
-			emulated = kvmppc_handle_load(run, vcpu,
-				KVM_MMIO_REG_FPR|rt, 4, 1);
-			break;
+			if ((op.type & UPDATE) && (emulated != EMULATE_FAIL))
+				kvmppc_set_gpr(vcpu, op.update_reg, op.ea);
 
-		case OP_31_XOP_STFSX:
-			if (kvmppc_check_fp_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_sp64_extend = 1;
-			emulated = kvmppc_handle_store(run, vcpu,
-				VCPU_FPR(vcpu, rs), 4, 1);
 			break;
-
-		case OP_31_XOP_STFSUX:
-			if (kvmppc_check_fp_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_sp64_extend = 1;
-			emulated = kvmppc_handle_store(run, vcpu,
-				VCPU_FPR(vcpu, rs), 4, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
-
-		case OP_31_XOP_STFDX:
-			if (kvmppc_check_fp_disabled(vcpu))
-				return EMULATE_DONE;
-			emulated = kvmppc_handle_store(run, vcpu,
-				VCPU_FPR(vcpu, rs), 8, 1);
-			break;
-
-		case OP_31_XOP_STFDUX:
-			if (kvmppc_check_fp_disabled(vcpu))
+#endif
+#ifdef CONFIG_ALTIVEC
+		case LOAD_VMX:
+			if (kvmppc_check_altivec_disabled(vcpu))
 				return EMULATE_DONE;
-			emulated = kvmppc_handle_store(run, vcpu,
-				VCPU_FPR(vcpu, rs), 8, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
 
-		case OP_31_XOP_STFIWX:
-			if (kvmppc_check_fp_disabled(vcpu))
-				return EMULATE_DONE;
-			emulated = kvmppc_handle_store(run, vcpu,
-				VCPU_FPR(vcpu, rs), 4, 1);
+			/* Hardware enforces alignment of VMX accesses */
+			vcpu->arch.vaddr_accessed &= ~((unsigned long)size - 1);
+			vcpu->arch.paddr_accessed &= ~((unsigned long)size - 1);
+
+			if (size == 16) { /* lvx */
+				vcpu->arch.mmio_copy_type =
+						KVMPPC_VMX_COPY_DWORD;
+			} else if (size == 4) { /* lvewx  */
+				vcpu->arch.mmio_copy_type =
+						KVMPPC_VMX_COPY_WORD;
+			} else if (size == 2) { /* lvehx  */
+				vcpu->arch.mmio_copy_type =
+						KVMPPC_VMX_COPY_HWORD;
+			} else if (size == 1) { /* lvebx  */
+				vcpu->arch.mmio_copy_type =
+						KVMPPC_VMX_COPY_BYTE;
+			} else
+				break;
+
+			vcpu->arch.mmio_vmx_offset =
+				(vcpu->arch.vaddr_accessed & 0xf)/size;
+
+			if (size == 16) {
+				vcpu->arch.mmio_vmx_copy_nums = 2;
+				emulated = kvmppc_handle_vmx_load(run,
+						vcpu, KVM_MMIO_REG_VMX|op.reg,
+						8, 1);
+			} else {
+				vcpu->arch.mmio_vmx_copy_nums = 1;
+				emulated = kvmppc_handle_vmx_load(run, vcpu,
+						KVM_MMIO_REG_VMX|op.reg,
+						size, 1);
+			}
 			break;
 #endif
-
 #ifdef CONFIG_VSX
-		case OP_31_XOP_LXSDX:
-			if (kvmppc_check_vsx_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_copy_nums = 1;
-			vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_DWORD;
-			emulated = kvmppc_handle_vsx_load(run, vcpu,
-				KVM_MMIO_REG_VSX|rt, 8, 1, 0);
-			break;
-
-		case OP_31_XOP_LXSSPX:
-			if (kvmppc_check_vsx_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_copy_nums = 1;
-			vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_DWORD;
-			vcpu->arch.mmio_sp64_extend = 1;
-			emulated = kvmppc_handle_vsx_load(run, vcpu,
-				KVM_MMIO_REG_VSX|rt, 4, 1, 0);
-			break;
+		case LOAD_VSX: {
+			int io_size_each;
+
+			if (op.vsx_flags & VSX_CHECK_VEC) {
+				if (kvmppc_check_altivec_disabled(vcpu))
+					return EMULATE_DONE;
+			} else {
+				if (kvmppc_check_vsx_disabled(vcpu))
+					return EMULATE_DONE;
+			}
+
+			if (op.vsx_flags & VSX_FPCONV)
+				vcpu->arch.mmio_sp64_extend = 1;
+
+			if (op.element_size == 8)  {
+				if (op.vsx_flags & VSX_SPLAT)
+					vcpu->arch.mmio_copy_type =
+						KVMPPC_VSX_COPY_DWORD_LOAD_DUMP;
+				else
+					vcpu->arch.mmio_copy_type =
+						KVMPPC_VSX_COPY_DWORD;
+			} else if (op.element_size == 4) {
+				if (op.vsx_flags & VSX_SPLAT)
+					vcpu->arch.mmio_copy_type =
+						KVMPPC_VSX_COPY_WORD_LOAD_DUMP;
+				else
+					vcpu->arch.mmio_copy_type =
+						KVMPPC_VSX_COPY_WORD;
+			} else
+				break;
+
+			if (size < op.element_size) {
+				/* precision convert case: lxsspx, etc */
+				vcpu->arch.mmio_vsx_copy_nums = 1;
+				io_size_each = size;
+			} else { /* lxvw4x, lxvd2x, etc */
+				vcpu->arch.mmio_vsx_copy_nums =
+					size/op.element_size;
+				io_size_each = op.element_size;
+			}
 
-		case OP_31_XOP_LXSIWAX:
-			if (kvmppc_check_vsx_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_copy_nums = 1;
-			vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_DWORD;
 			emulated = kvmppc_handle_vsx_load(run, vcpu,
-				KVM_MMIO_REG_VSX|rt, 4, 1, 1);
-			break;
-
-		case OP_31_XOP_LXSIWZX:
-			if (kvmppc_check_vsx_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_copy_nums = 1;
-			vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_DWORD;
-			emulated = kvmppc_handle_vsx_load(run, vcpu,
-				KVM_MMIO_REG_VSX|rt, 4, 1, 0);
+					KVM_MMIO_REG_VSX | (op.reg & 0x1f),
+					io_size_each, 1, op.type & SIGNEXT);
 			break;
+		}
+#endif
+		case STORE:
+			/* if need byte reverse, op.val has been reversed by
+			 * analyse_instr().
+			 */
+			emulated = kvmppc_handle_store(run, vcpu, op.val,
+					size, 1);
 
-		case OP_31_XOP_LXVD2X:
-		/*
-		 * In this case, the official load/store process is like this:
-		 * Step1, exit from vm by page fault isr, then kvm save vsr.
-		 * Please see guest_exit_cont->store_fp_state->SAVE_32VSRS
-		 * as reference.
-		 *
-		 * Step2, copy data between memory and VCPU
-		 * Notice: for LXVD2X/STXVD2X/LXVW4X/STXVW4X, we use
-		 * 2copies*8bytes or 4copies*4bytes
-		 * to simulate one copy of 16bytes.
-		 * Also there is an endian issue here, we should notice the
-		 * layout of memory.
-		 * Please see MARCO of LXVD2X_ROT/STXVD2X_ROT as more reference.
-		 * If host is little-endian, kvm will call XXSWAPD for
-		 * LXVD2X_ROT/STXVD2X_ROT.
-		 * So, if host is little-endian,
-		 * the postion of memeory should be swapped.
-		 *
-		 * Step3, return to guest, kvm reset register.
-		 * Please see kvmppc_hv_entry->load_fp_state->REST_32VSRS
-		 * as reference.
-		 */
-			if (kvmppc_check_vsx_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_copy_nums = 2;
-			vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_DWORD;
-			emulated = kvmppc_handle_vsx_load(run, vcpu,
-				KVM_MMIO_REG_VSX|rt, 8, 1, 0);
-			break;
+			if ((op.type & UPDATE) && (emulated != EMULATE_FAIL))
+				kvmppc_set_gpr(vcpu, op.update_reg, op.ea);
 
-		case OP_31_XOP_LXVW4X:
-			if (kvmppc_check_vsx_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_copy_nums = 4;
-			vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_WORD;
-			emulated = kvmppc_handle_vsx_load(run, vcpu,
-				KVM_MMIO_REG_VSX|rt, 4, 1, 0);
 			break;
-
-		case OP_31_XOP_LXVDSX:
-			if (kvmppc_check_vsx_disabled(vcpu))
+#ifdef CONFIG_PPC_FPU
+		case STORE_FP:
+			if (kvmppc_check_fp_disabled(vcpu))
 				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_copy_nums = 1;
-			vcpu->arch.mmio_vsx_copy_type =
-				 KVMPPC_VSX_COPY_DWORD_LOAD_DUMP;
-			emulated = kvmppc_handle_vsx_load(run, vcpu,
-				KVM_MMIO_REG_VSX|rt, 8, 1, 0);
-			break;
 
-		case OP_31_XOP_STXSDX:
-			if (kvmppc_check_vsx_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_copy_nums = 1;
-			vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_DWORD;
-			emulated = kvmppc_handle_vsx_store(run, vcpu,
-						 rs, 8, 1);
-			break;
+			/* The FP registers need to be flushed so that
+			 * kvmppc_handle_store() can read actual FP vals
+			 * from vcpu->arch.
+			 */
+			if (vcpu->kvm->arch.kvm_ops->giveup_ext)
+				vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu,
+						MSR_FP);
 
-		case OP_31_XOP_STXSSPX:
-			if (kvmppc_check_vsx_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_copy_nums = 1;
-			vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_DWORD;
-			vcpu->arch.mmio_sp64_extend = 1;
-			emulated = kvmppc_handle_vsx_store(run, vcpu,
-						 rs, 4, 1);
-			break;
+			if (op.type & FPCONV)
+				vcpu->arch.mmio_sp64_extend = 1;
 
-		case OP_31_XOP_STXSIWX:
-			if (kvmppc_check_vsx_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_offset = 1;
-			vcpu->arch.mmio_vsx_copy_nums = 1;
-			vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_WORD;
-			emulated = kvmppc_handle_vsx_store(run, vcpu,
-							 rs, 4, 1);
-			break;
+			emulated = kvmppc_handle_store(run, vcpu,
+					VCPU_FPR(vcpu, op.reg), size, 1);
 
-		case OP_31_XOP_STXVD2X:
-			if (kvmppc_check_vsx_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_copy_nums = 2;
-			vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_DWORD;
-			emulated = kvmppc_handle_vsx_store(run, vcpu,
-							 rs, 8, 1);
-			break;
+			if ((op.type & UPDATE) && (emulated != EMULATE_FAIL))
+				kvmppc_set_gpr(vcpu, op.update_reg, op.ea);
 
-		case OP_31_XOP_STXVW4X:
-			if (kvmppc_check_vsx_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.mmio_vsx_copy_nums = 4;
-			vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_WORD;
-			emulated = kvmppc_handle_vsx_store(run, vcpu,
-							 rs, 4, 1);
 			break;
-#endif /* CONFIG_VSX */
-
+#endif
 #ifdef CONFIG_ALTIVEC
-		case OP_31_XOP_LVX:
+		case STORE_VMX:
 			if (kvmppc_check_altivec_disabled(vcpu))
 				return EMULATE_DONE;
-			vcpu->arch.vaddr_accessed &= ~0xFULL;
-			vcpu->arch.paddr_accessed &= ~0xFULL;
-			vcpu->arch.mmio_vmx_copy_nums = 2;
-			emulated = kvmppc_handle_load128_by2x64(run, vcpu,
-					KVM_MMIO_REG_VMX|rt, 1);
-			break;
 
-		case OP_31_XOP_STVX:
-			if (kvmppc_check_altivec_disabled(vcpu))
-				return EMULATE_DONE;
-			vcpu->arch.vaddr_accessed &= ~0xFULL;
-			vcpu->arch.paddr_accessed &= ~0xFULL;
-			vcpu->arch.mmio_vmx_copy_nums = 2;
-			emulated = kvmppc_handle_store128_by2x64(run, vcpu,
-					rs, 1);
-			break;
-#endif /* CONFIG_ALTIVEC */
+			/* Hardware enforces alignment of VMX accesses. */
+			vcpu->arch.vaddr_accessed &= ~((unsigned long)size - 1);
+			vcpu->arch.paddr_accessed &= ~((unsigned long)size - 1);
+
+			if (vcpu->kvm->arch.kvm_ops->giveup_ext)
+				vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu,
+						MSR_VEC);
+			if (size == 16) { /* stvx */
+				vcpu->arch.mmio_copy_type =
+						KVMPPC_VMX_COPY_DWORD;
+			} else if (size == 4) { /* stvewx  */
+				vcpu->arch.mmio_copy_type =
+						KVMPPC_VMX_COPY_WORD;
+			} else if (size == 2) { /* stvehx  */
+				vcpu->arch.mmio_copy_type =
+						KVMPPC_VMX_COPY_HWORD;
+			} else if (size == 1) { /* stvebx  */
+				vcpu->arch.mmio_copy_type =
+						KVMPPC_VMX_COPY_BYTE;
+			} else
+				break;
+
+			vcpu->arch.mmio_vmx_offset =
+				(vcpu->arch.vaddr_accessed & 0xf)/size;
+
+			if (size == 16) {
+				vcpu->arch.mmio_vmx_copy_nums = 2;
+				emulated = kvmppc_handle_vmx_store(run,
+						vcpu, op.reg, 8, 1);
+			} else {
+				vcpu->arch.mmio_vmx_copy_nums = 1;
+				emulated = kvmppc_handle_vmx_store(run,
+						vcpu, op.reg, size, 1);
+			}
 
-		default:
-			emulated = EMULATE_FAIL;
 			break;
-		}
-		break;
-
-	case OP_LWZ:
-		emulated = kvmppc_handle_load(run, vcpu, rt, 4, 1);
-		break;
-
-#ifdef CONFIG_PPC_FPU
-	case OP_STFS:
-		if (kvmppc_check_fp_disabled(vcpu))
-			return EMULATE_DONE;
-		vcpu->arch.mmio_sp64_extend = 1;
-		emulated = kvmppc_handle_store(run, vcpu,
-			VCPU_FPR(vcpu, rs),
-			4, 1);
-		break;
-
-	case OP_STFSU:
-		if (kvmppc_check_fp_disabled(vcpu))
-			return EMULATE_DONE;
-		vcpu->arch.mmio_sp64_extend = 1;
-		emulated = kvmppc_handle_store(run, vcpu,
-			VCPU_FPR(vcpu, rs),
-			4, 1);
-		kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-		break;
-
-	case OP_STFD:
-		if (kvmppc_check_fp_disabled(vcpu))
-			return EMULATE_DONE;
-		emulated = kvmppc_handle_store(run, vcpu,
-			VCPU_FPR(vcpu, rs),
-	                               8, 1);
-		break;
-
-	case OP_STFDU:
-		if (kvmppc_check_fp_disabled(vcpu))
-			return EMULATE_DONE;
-		emulated = kvmppc_handle_store(run, vcpu,
-			VCPU_FPR(vcpu, rs),
-	                               8, 1);
-		kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-		break;
 #endif
+#ifdef CONFIG_VSX
+		case STORE_VSX: {
+			int io_size_each;
+
+			if (op.vsx_flags & VSX_CHECK_VEC) {
+				if (kvmppc_check_altivec_disabled(vcpu))
+					return EMULATE_DONE;
+			} else {
+				if (kvmppc_check_vsx_disabled(vcpu))
+					return EMULATE_DONE;
+			}
+
+			if (vcpu->kvm->arch.kvm_ops->giveup_ext)
+				vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu,
+						MSR_VSX);
+
+			if (op.vsx_flags & VSX_FPCONV)
+				vcpu->arch.mmio_sp64_extend = 1;
+
+			if (op.element_size == 8)
+				vcpu->arch.mmio_copy_type =
+						KVMPPC_VSX_COPY_DWORD;
+			else if (op.element_size == 4)
+				vcpu->arch.mmio_copy_type =
+						KVMPPC_VSX_COPY_WORD;
+			else
+				break;
+
+			if (size < op.element_size) {
+				/* precise conversion case, like stxsspx */
+				vcpu->arch.mmio_vsx_copy_nums = 1;
+				io_size_each = size;
+			} else { /* stxvw4x, stxvd2x, etc */
+				vcpu->arch.mmio_vsx_copy_nums =
+						size/op.element_size;
+				io_size_each = op.element_size;
+			}
 
-	case OP_LD:
-		rt = get_rt(inst);
-		switch (inst & 3) {
-		case 0:	/* ld */
-			emulated = kvmppc_handle_load(run, vcpu, rt, 8, 1);
-			break;
-		case 1: /* ldu */
-			emulated = kvmppc_handle_load(run, vcpu, rt, 8, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-			break;
-		case 2:	/* lwa */
-			emulated = kvmppc_handle_loads(run, vcpu, rt, 4, 1);
+			emulated = kvmppc_handle_vsx_store(run, vcpu,
+					op.reg & 0x1f, io_size_each, 1);
 			break;
-		default:
-			emulated = EMULATE_FAIL;
 		}
-		break;
-
-	case OP_LWZU:
-		emulated = kvmppc_handle_load(run, vcpu, rt, 4, 1);
-		kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-		break;
-
-	case OP_LBZ:
-		emulated = kvmppc_handle_load(run, vcpu, rt, 1, 1);
-		break;
-
-	case OP_LBZU:
-		emulated = kvmppc_handle_load(run, vcpu, rt, 1, 1);
-		kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-		break;
-
-	case OP_STW:
-		emulated = kvmppc_handle_store(run, vcpu,
-					       kvmppc_get_gpr(vcpu, rs),
-		                               4, 1);
-		break;
-
-	case OP_STD:
-		rs = get_rs(inst);
-		switch (inst & 3) {
-		case 0:	/* std */
-			emulated = kvmppc_handle_store(run, vcpu,
-				kvmppc_get_gpr(vcpu, rs), 8, 1);
-			break;
-		case 1: /* stdu */
-			emulated = kvmppc_handle_store(run, vcpu,
-				kvmppc_get_gpr(vcpu, rs), 8, 1);
-			kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
+#endif
+		case CACHEOP:
+			/* Do nothing. The guest is performing dcbi because
+			 * hardware DMA is not snooped by the dcache, but
+			 * emulated DMA either goes through the dcache as
+			 * normal writes, or the host kernel has handled dcache
+			 * coherence.
+			 */
+			emulated = EMULATE_DONE;
 			break;
 		default:
-			emulated = EMULATE_FAIL;
+			break;
 		}
-		break;
-
-	case OP_STWU:
-		emulated = kvmppc_handle_store(run, vcpu,
-				kvmppc_get_gpr(vcpu, rs), 4, 1);
-		kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-		break;
-
-	case OP_STB:
-		emulated = kvmppc_handle_store(run, vcpu,
-				kvmppc_get_gpr(vcpu, rs), 1, 1);
-		break;
-
-	case OP_STBU:
-		emulated = kvmppc_handle_store(run, vcpu,
-				kvmppc_get_gpr(vcpu, rs), 1, 1);
-		kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-		break;
-
-	case OP_LHZ:
-		emulated = kvmppc_handle_load(run, vcpu, rt, 2, 1);
-		break;
-
-	case OP_LHZU:
-		emulated = kvmppc_handle_load(run, vcpu, rt, 2, 1);
-		kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-		break;
-
-	case OP_LHA:
-		emulated = kvmppc_handle_loads(run, vcpu, rt, 2, 1);
-		break;
-
-	case OP_LHAU:
-		emulated = kvmppc_handle_loads(run, vcpu, rt, 2, 1);
-		kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-		break;
-
-	case OP_STH:
-		emulated = kvmppc_handle_store(run, vcpu,
-				kvmppc_get_gpr(vcpu, rs), 2, 1);
-		break;
-
-	case OP_STHU:
-		emulated = kvmppc_handle_store(run, vcpu,
-				kvmppc_get_gpr(vcpu, rs), 2, 1);
-		kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-		break;
-
-#ifdef CONFIG_PPC_FPU
-	case OP_LFS:
-		if (kvmppc_check_fp_disabled(vcpu))
-			return EMULATE_DONE;
-		vcpu->arch.mmio_sp64_extend = 1;
-		emulated = kvmppc_handle_load(run, vcpu,
-			KVM_MMIO_REG_FPR|rt, 4, 1);
-		break;
-
-	case OP_LFSU:
-		if (kvmppc_check_fp_disabled(vcpu))
-			return EMULATE_DONE;
-		vcpu->arch.mmio_sp64_extend = 1;
-		emulated = kvmppc_handle_load(run, vcpu,
-			KVM_MMIO_REG_FPR|rt, 4, 1);
-		kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-		break;
-
-	case OP_LFD:
-		if (kvmppc_check_fp_disabled(vcpu))
-			return EMULATE_DONE;
-		emulated = kvmppc_handle_load(run, vcpu,
-			KVM_MMIO_REG_FPR|rt, 8, 1);
-		break;
-
-	case OP_LFDU:
-		if (kvmppc_check_fp_disabled(vcpu))
-			return EMULATE_DONE;
-		emulated = kvmppc_handle_load(run, vcpu,
-			KVM_MMIO_REG_FPR|rt, 8, 1);
-		kvmppc_set_gpr(vcpu, ra, vcpu->arch.vaddr_accessed);
-		break;
-#endif
-
-	default:
-		emulated = EMULATE_FAIL;
-		break;
 	}
 
 	if (emulated == EMULATE_FAIL) {
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 4e387647b5af..05eccdc10fdd 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -907,6 +907,26 @@ static inline void kvmppc_set_vsr_dword_dump(struct kvm_vcpu *vcpu,
 	}
 }
 
+static inline void kvmppc_set_vsr_word_dump(struct kvm_vcpu *vcpu,
+	u32 gpr)
+{
+	union kvmppc_one_reg val;
+	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
+
+	if (vcpu->arch.mmio_vsx_tx_sx_enabled) {
+		val.vsx32val[0] = gpr;
+		val.vsx32val[1] = gpr;
+		val.vsx32val[2] = gpr;
+		val.vsx32val[3] = gpr;
+		VCPU_VSX_VR(vcpu, index) = val.vval;
+	} else {
+		val.vsx32val[0] = gpr;
+		val.vsx32val[1] = gpr;
+		VCPU_VSX_FPR(vcpu, index, 0) = val.vsxval[0];
+		VCPU_VSX_FPR(vcpu, index, 1) = val.vsxval[0];
+	}
+}
+
 static inline void kvmppc_set_vsr_word(struct kvm_vcpu *vcpu,
 	u32 gpr32)
 {
@@ -933,30 +953,110 @@ static inline void kvmppc_set_vsr_word(struct kvm_vcpu *vcpu,
 #endif /* CONFIG_VSX */
 
 #ifdef CONFIG_ALTIVEC
+static inline int kvmppc_get_vmx_offset_generic(struct kvm_vcpu *vcpu,
+		int index, int element_size)
+{
+	int offset;
+	int elts = sizeof(vector128)/element_size;
+
+	if ((index < 0) || (index >= elts))
+		return -1;
+
+	if (kvmppc_need_byteswap(vcpu))
+		offset = elts - index - 1;
+	else
+		offset = index;
+
+	return offset;
+}
+
+static inline int kvmppc_get_vmx_dword_offset(struct kvm_vcpu *vcpu,
+		int index)
+{
+	return kvmppc_get_vmx_offset_generic(vcpu, index, 8);
+}
+
+static inline int kvmppc_get_vmx_word_offset(struct kvm_vcpu *vcpu,
+		int index)
+{
+	return kvmppc_get_vmx_offset_generic(vcpu, index, 4);
+}
+
+static inline int kvmppc_get_vmx_hword_offset(struct kvm_vcpu *vcpu,
+		int index)
+{
+	return kvmppc_get_vmx_offset_generic(vcpu, index, 2);
+}
+
+static inline int kvmppc_get_vmx_byte_offset(struct kvm_vcpu *vcpu,
+		int index)
+{
+	return kvmppc_get_vmx_offset_generic(vcpu, index, 1);
+}
+
+
 static inline void kvmppc_set_vmx_dword(struct kvm_vcpu *vcpu,
-		u64 gpr)
+	u64 gpr)
 {
+	union kvmppc_one_reg val;
+	int offset = kvmppc_get_vmx_dword_offset(vcpu,
+			vcpu->arch.mmio_vmx_offset);
 	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
-	u32 hi, lo;
-	u32 di;
 
-#ifdef __BIG_ENDIAN
-	hi = gpr >> 32;
-	lo = gpr & 0xffffffff;
-#else
-	lo = gpr >> 32;
-	hi = gpr & 0xffffffff;
-#endif
+	if (offset == -1)
+		return;
+
+	val.vval = VCPU_VSX_VR(vcpu, index);
+	val.vsxval[offset] = gpr;
+	VCPU_VSX_VR(vcpu, index) = val.vval;
+}
+
+static inline void kvmppc_set_vmx_word(struct kvm_vcpu *vcpu,
+	u32 gpr32)
+{
+	union kvmppc_one_reg val;
+	int offset = kvmppc_get_vmx_word_offset(vcpu,
+			vcpu->arch.mmio_vmx_offset);
+	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
 
-	di = 2 - vcpu->arch.mmio_vmx_copy_nums;		/* doubleword index */
-	if (di > 1)
+	if (offset == -1)
 		return;
 
-	if (vcpu->arch.mmio_host_swabbed)
-		di = 1 - di;
+	val.vval = VCPU_VSX_VR(vcpu, index);
+	val.vsx32val[offset] = gpr32;
+	VCPU_VSX_VR(vcpu, index) = val.vval;
+}
+
+static inline void kvmppc_set_vmx_hword(struct kvm_vcpu *vcpu,
+	u16 gpr16)
+{
+	union kvmppc_one_reg val;
+	int offset = kvmppc_get_vmx_hword_offset(vcpu,
+			vcpu->arch.mmio_vmx_offset);
+	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
 
-	VCPU_VSX_VR(vcpu, index).u[di * 2] = hi;
-	VCPU_VSX_VR(vcpu, index).u[di * 2 + 1] = lo;
+	if (offset == -1)
+		return;
+
+	val.vval = VCPU_VSX_VR(vcpu, index);
+	val.vsx16val[offset] = gpr16;
+	VCPU_VSX_VR(vcpu, index) = val.vval;
+}
+
+static inline void kvmppc_set_vmx_byte(struct kvm_vcpu *vcpu,
+	u8 gpr8)
+{
+	union kvmppc_one_reg val;
+	int offset = kvmppc_get_vmx_byte_offset(vcpu,
+			vcpu->arch.mmio_vmx_offset);
+	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
+
+	if (offset == -1)
+		return;
+
+	val.vval = VCPU_VSX_VR(vcpu, index);
+	val.vsx8val[offset] = gpr8;
+	VCPU_VSX_VR(vcpu, index) = val.vval;
 }
 #endif /* CONFIG_ALTIVEC */
 
@@ -1041,6 +1141,9 @@ static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu,
 		kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, gpr);
 		break;
 	case KVM_MMIO_REG_FPR:
+		if (vcpu->kvm->arch.kvm_ops->giveup_ext)
+			vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_FP);
+
 		VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr;
 		break;
 #ifdef CONFIG_PPC_BOOK3S
@@ -1054,18 +1157,36 @@ static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu,
 #endif
 #ifdef CONFIG_VSX
 	case KVM_MMIO_REG_VSX:
-		if (vcpu->arch.mmio_vsx_copy_type == KVMPPC_VSX_COPY_DWORD)
+		if (vcpu->kvm->arch.kvm_ops->giveup_ext)
+			vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VSX);
+
+		if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_DWORD)
 			kvmppc_set_vsr_dword(vcpu, gpr);
-		else if (vcpu->arch.mmio_vsx_copy_type == KVMPPC_VSX_COPY_WORD)
+		else if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_WORD)
 			kvmppc_set_vsr_word(vcpu, gpr);
-		else if (vcpu->arch.mmio_vsx_copy_type ==
+		else if (vcpu->arch.mmio_copy_type ==
 				KVMPPC_VSX_COPY_DWORD_LOAD_DUMP)
 			kvmppc_set_vsr_dword_dump(vcpu, gpr);
+		else if (vcpu->arch.mmio_copy_type ==
+				KVMPPC_VSX_COPY_WORD_LOAD_DUMP)
+			kvmppc_set_vsr_word_dump(vcpu, gpr);
 		break;
 #endif
 #ifdef CONFIG_ALTIVEC
 	case KVM_MMIO_REG_VMX:
-		kvmppc_set_vmx_dword(vcpu, gpr);
+		if (vcpu->kvm->arch.kvm_ops->giveup_ext)
+			vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VEC);
+
+		if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_DWORD)
+			kvmppc_set_vmx_dword(vcpu, gpr);
+		else if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_WORD)
+			kvmppc_set_vmx_word(vcpu, gpr);
+		else if (vcpu->arch.mmio_copy_type ==
+				KVMPPC_VMX_COPY_HWORD)
+			kvmppc_set_vmx_hword(vcpu, gpr);
+		else if (vcpu->arch.mmio_copy_type ==
+				KVMPPC_VMX_COPY_BYTE)
+			kvmppc_set_vmx_byte(vcpu, gpr);
 		break;
 #endif
 	default:
@@ -1228,7 +1349,7 @@ static inline int kvmppc_get_vsr_data(struct kvm_vcpu *vcpu, int rs, u64 *val)
 	u32 dword_offset, word_offset;
 	union kvmppc_one_reg reg;
 	int vsx_offset = 0;
-	int copy_type = vcpu->arch.mmio_vsx_copy_type;
+	int copy_type = vcpu->arch.mmio_copy_type;
 	int result = 0;
 
 	switch (copy_type) {
@@ -1344,14 +1465,16 @@ static int kvmppc_emulate_mmio_vsx_loadstore(struct kvm_vcpu *vcpu,
 #endif /* CONFIG_VSX */
 
 #ifdef CONFIG_ALTIVEC
-/* handle quadword load access in two halves */
-int kvmppc_handle_load128_by2x64(struct kvm_run *run, struct kvm_vcpu *vcpu,
-		unsigned int rt, int is_default_endian)
+int kvmppc_handle_vmx_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
+		unsigned int rt, unsigned int bytes, int is_default_endian)
 {
 	enum emulation_result emulated = EMULATE_DONE;
 
+	if (vcpu->arch.mmio_vsx_copy_nums > 2)
+		return EMULATE_FAIL;
+
 	while (vcpu->arch.mmio_vmx_copy_nums) {
-		emulated = __kvmppc_handle_load(run, vcpu, rt, 8,
+		emulated = __kvmppc_handle_load(run, vcpu, rt, bytes,
 				is_default_endian, 0);
 
 		if (emulated != EMULATE_DONE)
@@ -1359,55 +1482,127 @@ int kvmppc_handle_load128_by2x64(struct kvm_run *run, struct kvm_vcpu *vcpu,
 
 		vcpu->arch.paddr_accessed += run->mmio.len;
 		vcpu->arch.mmio_vmx_copy_nums--;
+		vcpu->arch.mmio_vmx_offset++;
 	}
 
 	return emulated;
 }
 
-static inline int kvmppc_get_vmx_data(struct kvm_vcpu *vcpu, int rs, u64 *val)
+int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val)
 {
-	vector128 vrs = VCPU_VSX_VR(vcpu, rs);
-	u32 di;
-	u64 w0, w1;
+	union kvmppc_one_reg reg;
+	int vmx_offset = 0;
+	int result = 0;
+
+	vmx_offset =
+		kvmppc_get_vmx_dword_offset(vcpu, vcpu->arch.mmio_vmx_offset);
 
-	di = 2 - vcpu->arch.mmio_vmx_copy_nums;		/* doubleword index */
-	if (di > 1)
+	if (vmx_offset == -1)
 		return -1;
 
-	if (vcpu->arch.mmio_host_swabbed)
-		di = 1 - di;
+	reg.vval = VCPU_VSX_VR(vcpu, index);
+	*val = reg.vsxval[vmx_offset];
 
-	w0 = vrs.u[di * 2];
-	w1 = vrs.u[di * 2 + 1];
+	return result;
+}
 
-#ifdef __BIG_ENDIAN
-	*val = (w0 << 32) | w1;
-#else
-	*val = (w1 << 32) | w0;
-#endif
-	return 0;
+int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val)
+{
+	union kvmppc_one_reg reg;
+	int vmx_offset = 0;
+	int result = 0;
+
+	vmx_offset =
+		kvmppc_get_vmx_word_offset(vcpu, vcpu->arch.mmio_vmx_offset);
+
+	if (vmx_offset == -1)
+		return -1;
+
+	reg.vval = VCPU_VSX_VR(vcpu, index);
+	*val = reg.vsx32val[vmx_offset];
+
+	return result;
+}
+
+int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val)
+{
+	union kvmppc_one_reg reg;
+	int vmx_offset = 0;
+	int result = 0;
+
+	vmx_offset =
+		kvmppc_get_vmx_hword_offset(vcpu, vcpu->arch.mmio_vmx_offset);
+
+	if (vmx_offset == -1)
+		return -1;
+
+	reg.vval = VCPU_VSX_VR(vcpu, index);
+	*val = reg.vsx16val[vmx_offset];
+
+	return result;
+}
+
+int kvmppc_get_vmx_byte(struct kvm_vcpu *vcpu, int index, u64 *val)
+{
+	union kvmppc_one_reg reg;
+	int vmx_offset = 0;
+	int result = 0;
+
+	vmx_offset =
+		kvmppc_get_vmx_byte_offset(vcpu, vcpu->arch.mmio_vmx_offset);
+
+	if (vmx_offset == -1)
+		return -1;
+
+	reg.vval = VCPU_VSX_VR(vcpu, index);
+	*val = reg.vsx8val[vmx_offset];
+
+	return result;
 }
 
-/* handle quadword store in two halves */
-int kvmppc_handle_store128_by2x64(struct kvm_run *run, struct kvm_vcpu *vcpu,
-		unsigned int rs, int is_default_endian)
+int kvmppc_handle_vmx_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
+		unsigned int rs, unsigned int bytes, int is_default_endian)
 {
 	u64 val = 0;
+	unsigned int index = rs & KVM_MMIO_REG_MASK;
 	enum emulation_result emulated = EMULATE_DONE;
 
+	if (vcpu->arch.mmio_vsx_copy_nums > 2)
+		return EMULATE_FAIL;
+
 	vcpu->arch.io_gpr = rs;
 
 	while (vcpu->arch.mmio_vmx_copy_nums) {
-		if (kvmppc_get_vmx_data(vcpu, rs, &val) == -1)
+		switch (vcpu->arch.mmio_copy_type) {
+		case KVMPPC_VMX_COPY_DWORD:
+			if (kvmppc_get_vmx_dword(vcpu, index, &val) == -1)
+				return EMULATE_FAIL;
+
+			break;
+		case KVMPPC_VMX_COPY_WORD:
+			if (kvmppc_get_vmx_word(vcpu, index, &val) == -1)
+				return EMULATE_FAIL;
+			break;
+		case KVMPPC_VMX_COPY_HWORD:
+			if (kvmppc_get_vmx_hword(vcpu, index, &val) == -1)
+				return EMULATE_FAIL;
+			break;
+		case KVMPPC_VMX_COPY_BYTE:
+			if (kvmppc_get_vmx_byte(vcpu, index, &val) == -1)
+				return EMULATE_FAIL;
+			break;
+		default:
 			return EMULATE_FAIL;
+		}
 
-		emulated = kvmppc_handle_store(run, vcpu, val, 8,
+		emulated = kvmppc_handle_store(run, vcpu, val, bytes,
 				is_default_endian);
 		if (emulated != EMULATE_DONE)
 			break;
 
 		vcpu->arch.paddr_accessed += run->mmio.len;
 		vcpu->arch.mmio_vmx_copy_nums--;
+		vcpu->arch.mmio_vmx_offset++;
 	}
 
 	return emulated;
@@ -1422,11 +1617,11 @@ static int kvmppc_emulate_mmio_vmx_loadstore(struct kvm_vcpu *vcpu,
 	vcpu->arch.paddr_accessed += run->mmio.len;
 
 	if (!vcpu->mmio_is_write) {
-		emulated = kvmppc_handle_load128_by2x64(run, vcpu,
-				vcpu->arch.io_gpr, 1);
+		emulated = kvmppc_handle_vmx_load(run, vcpu,
+				vcpu->arch.io_gpr, run->mmio.len, 1);
 	} else {
-		emulated = kvmppc_handle_store128_by2x64(run, vcpu,
-				vcpu->arch.io_gpr, 1);
+		emulated = kvmppc_handle_vmx_store(run, vcpu,
+				vcpu->arch.io_gpr, run->mmio.len, 1);
 	}
 
 	switch (emulated) {
@@ -1570,8 +1765,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 		}
 #endif
 #ifdef CONFIG_ALTIVEC
-		if (vcpu->arch.mmio_vmx_copy_nums > 0)
+		if (vcpu->arch.mmio_vmx_copy_nums > 0) {
 			vcpu->arch.mmio_vmx_copy_nums--;
+			vcpu->arch.mmio_vmx_offset++;
+		}
 
 		if (vcpu->arch.mmio_vmx_copy_nums > 0) {
 			r = kvmppc_emulate_mmio_vmx_loadstore(vcpu, run);
diff --git a/arch/powerpc/lib/feature-fixups-test.S b/arch/powerpc/lib/feature-fixups-test.S
index f4613118132e..f16cec989506 100644
--- a/arch/powerpc/lib/feature-fixups-test.S
+++ b/arch/powerpc/lib/feature-fixups-test.S
@@ -167,16 +167,52 @@ globl(ftr_fixup_test6_expected)
 	blt	2b
 	b	3f
 	b	1b
-2:	or	1,1,1
+3:	or	1,1,1
+	or	2,2,2
+	or	3,3,3
+
+globl(ftr_fixup_test7)
+	or	1,1,1
+BEGIN_FTR_SECTION
+	or	2,2,2
+	or	2,2,2
 	or	2,2,2
-3:	or	3,3,3
+	or	2,2,2
+	or	2,2,2
+	or	2,2,2
+	or	2,2,2
+FTR_SECTION_ELSE
+2:	b	3f
+3:	or	5,5,5
+	beq	3b
+	b	1f
+	or	6,6,6
+	b	2b
+	bdnz	3b
+1:
+ALT_FTR_SECTION_END(0, 1)
+	or	1,1,1
+	or	1,1,1
+
+globl(end_ftr_fixup_test7)
+	nop
 
+globl(ftr_fixup_test7_expected)
+	or	1,1,1
+2:	b	3f
+3:	or	5,5,5
+	beq	3b
+	b	1f
+	or	6,6,6
+	b	2b
+	bdnz	3b
+1:	or	1,1,1
 
 #if 0
 /* Test that if we have a larger else case the assembler spots it and
  * reports an error. #if 0'ed so as not to break the build normally.
  */
-ftr_fixup_test7:
+ftr_fixup_test_too_big:
 	or	1,1,1
 BEGIN_FTR_SECTION
 	or	2,2,2
diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c
index e1bcdc32a851..8d445bffc778 100644
--- a/arch/powerpc/lib/feature-fixups.c
+++ b/arch/powerpc/lib/feature-fixups.c
@@ -400,7 +400,7 @@ static void test_basic_patching(void)
 	extern unsigned int end_ftr_fixup_test1[];
 	extern unsigned int ftr_fixup_test1_orig[];
 	extern unsigned int ftr_fixup_test1_expected[];
-	int size = end_ftr_fixup_test1 - ftr_fixup_test1;
+	int size = 4 * (end_ftr_fixup_test1 - ftr_fixup_test1);
 
 	fixup.value = fixup.mask = 8;
 	fixup.start_off = calc_offset(&fixup, ftr_fixup_test1 + 1);
@@ -432,7 +432,7 @@ static void test_alternative_patching(void)
 	extern unsigned int ftr_fixup_test2_orig[];
 	extern unsigned int ftr_fixup_test2_alt[];
 	extern unsigned int ftr_fixup_test2_expected[];
-	int size = end_ftr_fixup_test2 - ftr_fixup_test2;
+	int size = 4 * (end_ftr_fixup_test2 - ftr_fixup_test2);
 
 	fixup.value = fixup.mask = 0xF;
 	fixup.start_off = calc_offset(&fixup, ftr_fixup_test2 + 1);
@@ -464,7 +464,7 @@ static void test_alternative_case_too_big(void)
 	extern unsigned int end_ftr_fixup_test3[];
 	extern unsigned int ftr_fixup_test3_orig[];
 	extern unsigned int ftr_fixup_test3_alt[];
-	int size = end_ftr_fixup_test3 - ftr_fixup_test3;
+	int size = 4 * (end_ftr_fixup_test3 - ftr_fixup_test3);
 
 	fixup.value = fixup.mask = 0xC;
 	fixup.start_off = calc_offset(&fixup, ftr_fixup_test3 + 1);
@@ -491,7 +491,7 @@ static void test_alternative_case_too_small(void)
 	extern unsigned int ftr_fixup_test4_orig[];
 	extern unsigned int ftr_fixup_test4_alt[];
 	extern unsigned int ftr_fixup_test4_expected[];
-	int size = end_ftr_fixup_test4 - ftr_fixup_test4;
+	int size = 4 * (end_ftr_fixup_test4 - ftr_fixup_test4);
 	unsigned long flag;
 
 	/* Check a high-bit flag */
@@ -525,7 +525,7 @@ static void test_alternative_case_with_branch(void)
 	extern unsigned int ftr_fixup_test5[];
 	extern unsigned int end_ftr_fixup_test5[];
 	extern unsigned int ftr_fixup_test5_expected[];
-	int size = end_ftr_fixup_test5 - ftr_fixup_test5;
+	int size = 4 * (end_ftr_fixup_test5 - ftr_fixup_test5);
 
 	check(memcmp(ftr_fixup_test5, ftr_fixup_test5_expected, size) == 0);
 }
@@ -535,11 +535,21 @@ static void test_alternative_case_with_external_branch(void)
 	extern unsigned int ftr_fixup_test6[];
 	extern unsigned int end_ftr_fixup_test6[];
 	extern unsigned int ftr_fixup_test6_expected[];
-	int size = end_ftr_fixup_test6 - ftr_fixup_test6;
+	int size = 4 * (end_ftr_fixup_test6 - ftr_fixup_test6);
 
 	check(memcmp(ftr_fixup_test6, ftr_fixup_test6_expected, size) == 0);
 }
 
+static void test_alternative_case_with_branch_to_end(void)
+{
+	extern unsigned int ftr_fixup_test7[];
+	extern unsigned int end_ftr_fixup_test7[];
+	extern unsigned int ftr_fixup_test7_expected[];
+	int size = 4 * (end_ftr_fixup_test7 - ftr_fixup_test7);
+
+	check(memcmp(ftr_fixup_test7, ftr_fixup_test7_expected, size) == 0);
+}
+
 static void test_cpu_macros(void)
 {
 	extern u8 ftr_fixup_test_FTR_macros[];
@@ -595,6 +605,7 @@ static int __init test_feature_fixups(void)
 	test_alternative_case_too_small();
 	test_alternative_case_with_branch();
 	test_alternative_case_with_external_branch();
+	test_alternative_case_with_branch_to_end();
 	test_cpu_macros();
 	test_fw_macros();
 	test_lwsync_macros();
diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
index 34d68f1b1b40..49427a3ee104 100644
--- a/arch/powerpc/lib/sstep.c
+++ b/arch/powerpc/lib/sstep.c
@@ -1065,9 +1065,10 @@ static nokprobe_inline void do_popcnt(const struct pt_regs *regs,
 {
 	unsigned long long out = v1;
 
-	out -= (out >> 1) & 0x5555555555555555;
-	out = (0x3333333333333333 & out) + (0x3333333333333333 & (out >> 2));
-	out = (out + (out >> 4)) & 0x0f0f0f0f0f0f0f0f;
+	out -= (out >> 1) & 0x5555555555555555ULL;
+	out = (0x3333333333333333ULL & out) +
+	      (0x3333333333333333ULL & (out >> 2));
+	out = (out + (out >> 4)) & 0x0f0f0f0f0f0f0f0fULL;
 
 	if (size == 8) {	/* popcntb */
 		op->val = out;
@@ -1076,7 +1077,7 @@ static nokprobe_inline void do_popcnt(const struct pt_regs *regs,
 	out += out >> 8;
 	out += out >> 16;
 	if (size == 32) {	/* popcntw */
-		op->val = out & 0x0000003f0000003f;
+		op->val = out & 0x0000003f0000003fULL;
 		return;
 	}
 
@@ -1114,7 +1115,7 @@ static nokprobe_inline void do_prty(const struct pt_regs *regs,
 
 	res ^= res >> 16;
 	if (size == 32) {		/* prtyw */
-		op->val = res & 0x0000000100000001;
+		op->val = res & 0x0000000100000001ULL;
 		return;
 	}
 
diff --git a/arch/powerpc/lib/xor_vmx_glue.c b/arch/powerpc/lib/xor_vmx_glue.c
index 6521fe5e8cef..dab2b6bfcf36 100644
--- a/arch/powerpc/lib/xor_vmx_glue.c
+++ b/arch/powerpc/lib/xor_vmx_glue.c
@@ -13,6 +13,7 @@
 #include <linux/export.h>
 #include <linux/sched.h>
 #include <asm/switch_to.h>
+#include <asm/xor_altivec.h>
 #include "xor_vmx.h"
 
 void xor_altivec_2(unsigned long bytes, unsigned long *v1_in,
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index c01d627e687a..b1ca7a0974e3 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -22,6 +22,7 @@
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/types.h>
+#include <linux/pagemap.h>
 #include <linux/ptrace.h>
 #include <linux/mman.h>
 #include <linux/mm.h>
@@ -66,37 +67,33 @@ static inline bool notify_page_fault(struct pt_regs *regs)
 }
 
 /*
- * Check whether the instruction at regs->nip is a store using
+ * Check whether the instruction inst is a store using
  * an update addressing form which will update r1.
  */
-static bool store_updates_sp(struct pt_regs *regs)
+static bool store_updates_sp(unsigned int inst)
 {
-	unsigned int inst;
-
-	if (get_user(inst, (unsigned int __user *)regs->nip))
-		return false;
 	/* check for 1 in the rA field */
 	if (((inst >> 16) & 0x1f) != 1)
 		return false;
 	/* check major opcode */
 	switch (inst >> 26) {
-	case 37:	/* stwu */
-	case 39:	/* stbu */
-	case 45:	/* sthu */
-	case 53:	/* stfsu */
-	case 55:	/* stfdu */
+	case OP_STWU:
+	case OP_STBU:
+	case OP_STHU:
+	case OP_STFSU:
+	case OP_STFDU:
 		return true;
-	case 62:	/* std or stdu */
+	case OP_STD:	/* std or stdu */
 		return (inst & 3) == 1;
-	case 31:
+	case OP_31:
 		/* check minor opcode */
 		switch ((inst >> 1) & 0x3ff) {
-		case 181:	/* stdux */
-		case 183:	/* stwux */
-		case 247:	/* stbux */
-		case 439:	/* sthux */
-		case 695:	/* stfsux */
-		case 759:	/* stfdux */
+		case OP_31_XOP_STDUX:
+		case OP_31_XOP_STWUX:
+		case OP_31_XOP_STBUX:
+		case OP_31_XOP_STHUX:
+		case OP_31_XOP_STFSUX:
+		case OP_31_XOP_STFDUX:
 			return true;
 		}
 	}
@@ -168,6 +165,7 @@ static int do_sigbus(struct pt_regs *regs, unsigned long address,
 		return SIGBUS;
 
 	current->thread.trap_nr = BUS_ADRERR;
+	clear_siginfo(&info);
 	info.si_signo = SIGBUS;
 	info.si_errno = 0;
 	info.si_code = BUS_ADRERR;
@@ -234,8 +232,8 @@ static bool bad_kernel_fault(bool is_exec, unsigned long error_code,
 }
 
 static bool bad_stack_expansion(struct pt_regs *regs, unsigned long address,
-				struct vm_area_struct *vma,
-				bool store_update_sp)
+				struct vm_area_struct *vma, unsigned int flags,
+				bool *must_retry)
 {
 	/*
 	 * N.B. The POWER/Open ABI allows programs to access up to
@@ -247,6 +245,7 @@ static bool bad_stack_expansion(struct pt_regs *regs, unsigned long address,
 	 * expand to 1MB without further checks.
 	 */
 	if (address + 0x100000 < vma->vm_end) {
+		unsigned int __user *nip = (unsigned int __user *)regs->nip;
 		/* get user regs even if this fault is in kernel mode */
 		struct pt_regs *uregs = current->thread.regs;
 		if (uregs == NULL)
@@ -264,8 +263,22 @@ static bool bad_stack_expansion(struct pt_regs *regs, unsigned long address,
 		 * between the last mapped region and the stack will
 		 * expand the stack rather than segfaulting.
 		 */
-		if (address + 2048 < uregs->gpr[1] && !store_update_sp)
-			return true;
+		if (address + 2048 >= uregs->gpr[1])
+			return false;
+
+		if ((flags & FAULT_FLAG_WRITE) && (flags & FAULT_FLAG_USER) &&
+		    access_ok(VERIFY_READ, nip, sizeof(*nip))) {
+			unsigned int inst;
+			int res;
+
+			pagefault_disable();
+			res = __get_user_inatomic(inst, nip);
+			pagefault_enable();
+			if (!res)
+				return !store_updates_sp(inst);
+			*must_retry = true;
+		}
+		return true;
 	}
 	return false;
 }
@@ -403,7 +416,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
 	int is_user = user_mode(regs);
 	int is_write = page_fault_is_write(error_code);
 	int fault, major = 0;
-	bool store_update_sp = false;
+	bool must_retry = false;
 
 	if (notify_page_fault(regs))
 		return 0;
@@ -454,9 +467,6 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
 	 * can result in fault, which will cause a deadlock when called with
 	 * mmap_sem held
 	 */
-	if (is_write && is_user)
-		store_update_sp = store_updates_sp(regs);
-
 	if (is_user)
 		flags |= FAULT_FLAG_USER;
 	if (is_write)
@@ -503,8 +513,17 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
 		return bad_area(regs, address);
 
 	/* The stack is being expanded, check if it's valid */
-	if (unlikely(bad_stack_expansion(regs, address, vma, store_update_sp)))
-		return bad_area(regs, address);
+	if (unlikely(bad_stack_expansion(regs, address, vma, flags,
+					 &must_retry))) {
+		if (!must_retry)
+			return bad_area(regs, address);
+
+		up_read(&mm->mmap_sem);
+		if (fault_in_pages_readable((const char __user *)regs->nip,
+					    sizeof(unsigned int)))
+			return bad_area_nosemaphore(regs, address);
+		goto retry;
+	}
 
 	/* Try to expand it */
 	if (unlikely(expand_stack(vma, address)))
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 0bd3790d35df..3dc3dd3ea0b6 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -572,8 +572,10 @@ static void __init htab_scan_page_sizes(void)
 	}
 
 #ifdef CONFIG_HUGETLB_PAGE
-	/* Reserve 16G huge page memory sections for huge pages */
-	of_scan_flat_dt(htab_dt_scan_hugepage_blocks, NULL);
+	if (!hugetlb_disabled) {
+		/* Reserve 16G huge page memory sections for huge pages */
+		of_scan_flat_dt(htab_dt_scan_hugepage_blocks, NULL);
+	}
 #endif /* CONFIG_HUGETLB_PAGE */
 }
 
@@ -1010,13 +1012,14 @@ void __init hash__early_init_mmu(void)
 	 */
 	__pte_frag_nr = H_PTE_FRAG_NR;
 	__pte_frag_size_shift = H_PTE_FRAG_SIZE_SHIFT;
+	__pmd_frag_nr = H_PMD_FRAG_NR;
+	__pmd_frag_size_shift = H_PMD_FRAG_SIZE_SHIFT;
 
 	__pte_index_size = H_PTE_INDEX_SIZE;
 	__pmd_index_size = H_PMD_INDEX_SIZE;
 	__pud_index_size = H_PUD_INDEX_SIZE;
 	__pgd_index_size = H_PGD_INDEX_SIZE;
 	__pud_cache_index = H_PUD_CACHE_INDEX;
-	__pmd_cache_index = H_PMD_CACHE_INDEX;
 	__pte_table_size = H_PTE_TABLE_SIZE;
 	__pmd_table_size = H_PMD_TABLE_SIZE;
 	__pud_table_size = H_PUD_TABLE_SIZE;
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index f1153f8254e3..2a4b1bf8bde6 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -35,6 +35,8 @@
 #define PAGE_SHIFT_16M	24
 #define PAGE_SHIFT_16G	34
 
+bool hugetlb_disabled = false;
+
 unsigned int HPAGE_SHIFT;
 EXPORT_SYMBOL(HPAGE_SHIFT);
 
@@ -651,6 +653,11 @@ static int __init hugetlbpage_init(void)
 {
 	int psize;
 
+	if (hugetlb_disabled) {
+		pr_info("HugeTLB support is disabled!\n");
+		return 0;
+	}
+
 #if !defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_PPC_8xx)
 	if (!radix_enabled() && !mmu_has_feature(MMU_FTR_16M_PAGE))
 		return -ENODEV;
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index c3c39b02b2ba..404fb6cba3ff 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -215,7 +215,7 @@ void __init mem_topology_setup(void)
 	/* Place all memblock_regions in the same node and merge contiguous
 	 * memblock_regions
 	 */
-	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);
+	memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0);
 }
 
 void __init initmem_init(void)
diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
index b75194dff64c..f3d4b4a0e561 100644
--- a/arch/powerpc/mm/mmu_context_book3s64.c
+++ b/arch/powerpc/mm/mmu_context_book3s64.c
@@ -159,9 +159,8 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 
 	mm->context.id = index;
 
-#ifdef CONFIG_PPC_64K_PAGES
 	mm->context.pte_frag = NULL;
-#endif
+	mm->context.pmd_frag = NULL;
 #ifdef CONFIG_SPAPR_TCE_IOMMU
 	mm_iommu_init(mm);
 #endif
@@ -192,17 +191,11 @@ static void destroy_contexts(mm_context_t *ctx)
 	spin_unlock(&mmu_context_lock);
 }
 
-#ifdef CONFIG_PPC_64K_PAGES
-static void destroy_pagetable_page(struct mm_struct *mm)
+static void pte_frag_destroy(void *pte_frag)
 {
 	int count;
-	void *pte_frag;
 	struct page *page;
 
-	pte_frag = mm->context.pte_frag;
-	if (!pte_frag)
-		return;
-
 	page = virt_to_page(pte_frag);
 	/* drop all the pending references */
 	count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
@@ -213,12 +206,34 @@ static void destroy_pagetable_page(struct mm_struct *mm)
 	}
 }
 
-#else
-static inline void destroy_pagetable_page(struct mm_struct *mm)
+static void pmd_frag_destroy(void *pmd_frag)
 {
+	int count;
+	struct page *page;
+
+	page = virt_to_page(pmd_frag);
+	/* drop all the pending references */
+	count = ((unsigned long)pmd_frag & ~PAGE_MASK) >> PMD_FRAG_SIZE_SHIFT;
+	/* We allow PTE_FRAG_NR fragments from a PTE page */
+	if (page_ref_sub_and_test(page, PMD_FRAG_NR - count)) {
+		pgtable_pmd_page_dtor(page);
+		free_unref_page(page);
+	}
+}
+
+static void destroy_pagetable_page(struct mm_struct *mm)
+{
+	void *frag;
+
+	frag = mm->context.pte_frag;
+	if (frag)
+		pte_frag_destroy(frag);
+
+	frag = mm->context.pmd_frag;
+	if (frag)
+		pmd_frag_destroy(frag);
 	return;
 }
-#endif
 
 void destroy_context(struct mm_struct *mm)
 {
diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c
index 518518fb7c45..abda2b92f1ba 100644
--- a/arch/powerpc/mm/pgtable-book3s64.c
+++ b/arch/powerpc/mm/pgtable-book3s64.c
@@ -9,14 +9,22 @@
 
 #include <linux/sched.h>
 #include <linux/mm_types.h>
+#include <linux/memblock.h>
 #include <misc/cxl-base.h>
 
 #include <asm/pgalloc.h>
 #include <asm/tlb.h>
+#include <asm/trace.h>
+#include <asm/powernv.h>
 
 #include "mmu_decl.h"
 #include <trace/events/thp.h>
 
+unsigned long __pmd_frag_nr;
+EXPORT_SYMBOL(__pmd_frag_nr);
+unsigned long __pmd_frag_size_shift;
+EXPORT_SYMBOL(__pmd_frag_size_shift);
+
 int (*register_process_table)(unsigned long base, unsigned long page_size,
 			      unsigned long tbl_size);
 
@@ -34,7 +42,7 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
 	int changed;
 #ifdef CONFIG_DEBUG_VM
 	WARN_ON(!pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
-	assert_spin_locked(&vma->vm_mm->page_table_lock);
+	assert_spin_locked(pmd_lockptr(vma->vm_mm, pmdp));
 #endif
 	changed = !pmd_same(*(pmdp), entry);
 	if (changed) {
@@ -59,7 +67,7 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
 {
 #ifdef CONFIG_DEBUG_VM
 	WARN_ON(pte_present(pmd_pte(*pmdp)) && !pte_protnone(pmd_pte(*pmdp)));
-	assert_spin_locked(&mm->page_table_lock);
+	assert_spin_locked(pmd_lockptr(mm, pmdp));
 	WARN_ON(!(pmd_trans_huge(pmd) || pmd_devmap(pmd)));
 #endif
 	trace_hugepage_set_pmd(addr, pmd_val(pmd));
@@ -171,3 +179,258 @@ int __meminit remove_section_mapping(unsigned long start, unsigned long end)
 	return hash__remove_section_mapping(start, end);
 }
 #endif /* CONFIG_MEMORY_HOTPLUG */
+
+void __init mmu_partition_table_init(void)
+{
+	unsigned long patb_size = 1UL << PATB_SIZE_SHIFT;
+	unsigned long ptcr;
+
+	BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 36), "Partition table size too large.");
+	partition_tb = __va(memblock_alloc_base(patb_size, patb_size,
+						MEMBLOCK_ALLOC_ANYWHERE));
+
+	/* Initialize the Partition Table with no entries */
+	memset((void *)partition_tb, 0, patb_size);
+
+	/*
+	 * update partition table control register,
+	 * 64 K size.
+	 */
+	ptcr = __pa(partition_tb) | (PATB_SIZE_SHIFT - 12);
+	mtspr(SPRN_PTCR, ptcr);
+	powernv_set_nmmu_ptcr(ptcr);
+}
+
+void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
+				   unsigned long dw1)
+{
+	unsigned long old = be64_to_cpu(partition_tb[lpid].patb0);
+
+	partition_tb[lpid].patb0 = cpu_to_be64(dw0);
+	partition_tb[lpid].patb1 = cpu_to_be64(dw1);
+
+	/*
+	 * Global flush of TLBs and partition table caches for this lpid.
+	 * The type of flush (hash or radix) depends on what the previous
+	 * use of this partition ID was, not the new use.
+	 */
+	asm volatile("ptesync" : : : "memory");
+	if (old & PATB_HR) {
+		asm volatile(PPC_TLBIE_5(%0,%1,2,0,1) : :
+			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
+		asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
+			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
+		trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 1);
+	} else {
+		asm volatile(PPC_TLBIE_5(%0,%1,2,0,0) : :
+			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
+		trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 0);
+	}
+	/* do we need fixup here ?*/
+	asm volatile("eieio; tlbsync; ptesync" : : : "memory");
+}
+EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);
+
+static pmd_t *get_pmd_from_cache(struct mm_struct *mm)
+{
+	void *pmd_frag, *ret;
+
+	spin_lock(&mm->page_table_lock);
+	ret = mm->context.pmd_frag;
+	if (ret) {
+		pmd_frag = ret + PMD_FRAG_SIZE;
+		/*
+		 * If we have taken up all the fragments mark PTE page NULL
+		 */
+		if (((unsigned long)pmd_frag & ~PAGE_MASK) == 0)
+			pmd_frag = NULL;
+		mm->context.pmd_frag = pmd_frag;
+	}
+	spin_unlock(&mm->page_table_lock);
+	return (pmd_t *)ret;
+}
+
+static pmd_t *__alloc_for_pmdcache(struct mm_struct *mm)
+{
+	void *ret = NULL;
+	struct page *page;
+	gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO;
+
+	if (mm == &init_mm)
+		gfp &= ~__GFP_ACCOUNT;
+	page = alloc_page(gfp);
+	if (!page)
+		return NULL;
+	if (!pgtable_pmd_page_ctor(page)) {
+		__free_pages(page, 0);
+		return NULL;
+	}
+
+	ret = page_address(page);
+	/*
+	 * if we support only one fragment just return the
+	 * allocated page.
+	 */
+	if (PMD_FRAG_NR == 1)
+		return ret;
+
+	spin_lock(&mm->page_table_lock);
+	/*
+	 * If we find pgtable_page set, we return
+	 * the allocated page with single fragement
+	 * count.
+	 */
+	if (likely(!mm->context.pmd_frag)) {
+		set_page_count(page, PMD_FRAG_NR);
+		mm->context.pmd_frag = ret + PMD_FRAG_SIZE;
+	}
+	spin_unlock(&mm->page_table_lock);
+
+	return (pmd_t *)ret;
+}
+
+pmd_t *pmd_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr)
+{
+	pmd_t *pmd;
+
+	pmd = get_pmd_from_cache(mm);
+	if (pmd)
+		return pmd;
+
+	return __alloc_for_pmdcache(mm);
+}
+
+void pmd_fragment_free(unsigned long *pmd)
+{
+	struct page *page = virt_to_page(pmd);
+
+	if (put_page_testzero(page)) {
+		pgtable_pmd_page_dtor(page);
+		free_unref_page(page);
+	}
+}
+
+static pte_t *get_pte_from_cache(struct mm_struct *mm)
+{
+	void *pte_frag, *ret;
+
+	spin_lock(&mm->page_table_lock);
+	ret = mm->context.pte_frag;
+	if (ret) {
+		pte_frag = ret + PTE_FRAG_SIZE;
+		/*
+		 * If we have taken up all the fragments mark PTE page NULL
+		 */
+		if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
+			pte_frag = NULL;
+		mm->context.pte_frag = pte_frag;
+	}
+	spin_unlock(&mm->page_table_lock);
+	return (pte_t *)ret;
+}
+
+static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel)
+{
+	void *ret = NULL;
+	struct page *page;
+
+	if (!kernel) {
+		page = alloc_page(PGALLOC_GFP | __GFP_ACCOUNT);
+		if (!page)
+			return NULL;
+		if (!pgtable_page_ctor(page)) {
+			__free_page(page);
+			return NULL;
+		}
+	} else {
+		page = alloc_page(PGALLOC_GFP);
+		if (!page)
+			return NULL;
+	}
+
+
+	ret = page_address(page);
+	/*
+	 * if we support only one fragment just return the
+	 * allocated page.
+	 */
+	if (PTE_FRAG_NR == 1)
+		return ret;
+	spin_lock(&mm->page_table_lock);
+	/*
+	 * If we find pgtable_page set, we return
+	 * the allocated page with single fragement
+	 * count.
+	 */
+	if (likely(!mm->context.pte_frag)) {
+		set_page_count(page, PTE_FRAG_NR);
+		mm->context.pte_frag = ret + PTE_FRAG_SIZE;
+	}
+	spin_unlock(&mm->page_table_lock);
+
+	return (pte_t *)ret;
+}
+
+pte_t *pte_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel)
+{
+	pte_t *pte;
+
+	pte = get_pte_from_cache(mm);
+	if (pte)
+		return pte;
+
+	return __alloc_for_ptecache(mm, kernel);
+}
+
+void pte_fragment_free(unsigned long *table, int kernel)
+{
+	struct page *page = virt_to_page(table);
+
+	if (put_page_testzero(page)) {
+		if (!kernel)
+			pgtable_page_dtor(page);
+		free_unref_page(page);
+	}
+}
+
+static inline void pgtable_free(void *table, int index)
+{
+	switch (index) {
+	case PTE_INDEX:
+		pte_fragment_free(table, 0);
+		break;
+	case PMD_INDEX:
+		pmd_fragment_free(table);
+		break;
+	case PUD_INDEX:
+		kmem_cache_free(PGT_CACHE(PUD_CACHE_INDEX), table);
+		break;
+		/* We don't free pgd table via RCU callback */
+	default:
+		BUG();
+	}
+}
+
+#ifdef CONFIG_SMP
+void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int index)
+{
+	unsigned long pgf = (unsigned long)table;
+
+	BUG_ON(index > MAX_PGTABLE_INDEX_SIZE);
+	pgf |= index;
+	tlb_remove_table(tlb, (void *)pgf);
+}
+
+void __tlb_remove_table(void *_table)
+{
+	void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
+	unsigned int index = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
+
+	return pgtable_free(table, index);
+}
+#else
+void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int index)
+{
+	return pgtable_free(table, index);
+}
+#endif
diff --git a/arch/powerpc/mm/pgtable-hash64.c b/arch/powerpc/mm/pgtable-hash64.c
index 199bfda5f0d9..692bfc9e372c 100644
--- a/arch/powerpc/mm/pgtable-hash64.c
+++ b/arch/powerpc/mm/pgtable-hash64.c
@@ -193,7 +193,7 @@ unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr
 
 #ifdef CONFIG_DEBUG_VM
 	WARN_ON(!hash__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
-	assert_spin_locked(&mm->page_table_lock);
+	assert_spin_locked(pmd_lockptr(mm, pmdp));
 #endif
 
 	__asm__ __volatile__(
@@ -265,7 +265,8 @@ void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 				  pgtable_t pgtable)
 {
 	pgtable_t *pgtable_slot;
-	assert_spin_locked(&mm->page_table_lock);
+
+	assert_spin_locked(pmd_lockptr(mm, pmdp));
 	/*
 	 * we store the pgtable in the second half of PMD
 	 */
@@ -285,7 +286,8 @@ pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
 	pgtable_t pgtable;
 	pgtable_t *pgtable_slot;
 
-	assert_spin_locked(&mm->page_table_lock);
+	assert_spin_locked(pmd_lockptr(mm, pmdp));
+
 	pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
 	pgtable = *pgtable_slot;
 	/*
diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
index f1891e215e39..ce24d72ea679 100644
--- a/arch/powerpc/mm/pgtable-radix.c
+++ b/arch/powerpc/mm/pgtable-radix.c
@@ -617,7 +617,6 @@ void __init radix__early_init_mmu(void)
 	__pud_index_size = RADIX_PUD_INDEX_SIZE;
 	__pgd_index_size = RADIX_PGD_INDEX_SIZE;
 	__pud_cache_index = RADIX_PUD_INDEX_SIZE;
-	__pmd_cache_index = RADIX_PMD_INDEX_SIZE;
 	__pte_table_size = RADIX_PTE_TABLE_SIZE;
 	__pmd_table_size = RADIX_PMD_TABLE_SIZE;
 	__pud_table_size = RADIX_PUD_TABLE_SIZE;
@@ -640,6 +639,8 @@ void __init radix__early_init_mmu(void)
 #endif
 	__pte_frag_nr = RADIX_PTE_FRAG_NR;
 	__pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
+	__pmd_frag_nr = RADIX_PMD_FRAG_NR;
+	__pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
 
 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 		radix_init_native();
@@ -975,7 +976,7 @@ unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long add
 
 #ifdef CONFIG_DEBUG_VM
 	WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
-	assert_spin_locked(&mm->page_table_lock);
+	assert_spin_locked(pmd_lockptr(mm, pmdp));
 #endif
 
 	old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index 9bf659d5078c..53e9eeecd5d4 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -33,7 +33,6 @@
 #include <linux/swap.h>
 #include <linux/stddef.h>
 #include <linux/vmalloc.h>
-#include <linux/memblock.h>
 #include <linux/slab.h>
 #include <linux/hugetlb.h>
 
@@ -47,13 +46,11 @@
 #include <asm/smp.h>
 #include <asm/machdep.h>
 #include <asm/tlb.h>
-#include <asm/trace.h>
 #include <asm/processor.h>
 #include <asm/cputable.h>
 #include <asm/sections.h>
 #include <asm/firmware.h>
 #include <asm/dma.h>
-#include <asm/powernv.h>
 
 #include "mmu_decl.h"
 
@@ -75,8 +72,6 @@ unsigned long __pud_index_size;
 EXPORT_SYMBOL(__pud_index_size);
 unsigned long __pgd_index_size;
 EXPORT_SYMBOL(__pgd_index_size);
-unsigned long __pmd_cache_index;
-EXPORT_SYMBOL(__pmd_cache_index);
 unsigned long __pud_cache_index;
 EXPORT_SYMBOL(__pud_cache_index);
 unsigned long __pte_table_size;
@@ -316,172 +311,6 @@ struct page *pmd_page(pmd_t pmd)
 	return virt_to_page(pmd_page_vaddr(pmd));
 }
 
-#ifdef CONFIG_PPC_64K_PAGES
-static pte_t *get_from_cache(struct mm_struct *mm)
-{
-	void *pte_frag, *ret;
-
-	spin_lock(&mm->page_table_lock);
-	ret = mm->context.pte_frag;
-	if (ret) {
-		pte_frag = ret + PTE_FRAG_SIZE;
-		/*
-		 * If we have taken up all the fragments mark PTE page NULL
-		 */
-		if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
-			pte_frag = NULL;
-		mm->context.pte_frag = pte_frag;
-	}
-	spin_unlock(&mm->page_table_lock);
-	return (pte_t *)ret;
-}
-
-static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
-{
-	void *ret = NULL;
-	struct page *page;
-
-	if (!kernel) {
-		page = alloc_page(PGALLOC_GFP | __GFP_ACCOUNT);
-		if (!page)
-			return NULL;
-		if (!pgtable_page_ctor(page)) {
-			__free_page(page);
-			return NULL;
-		}
-	} else {
-		page = alloc_page(PGALLOC_GFP);
-		if (!page)
-			return NULL;
-	}
-
-	ret = page_address(page);
-	spin_lock(&mm->page_table_lock);
-	/*
-	 * If we find pgtable_page set, we return
-	 * the allocated page with single fragement
-	 * count.
-	 */
-	if (likely(!mm->context.pte_frag)) {
-		set_page_count(page, PTE_FRAG_NR);
-		mm->context.pte_frag = ret + PTE_FRAG_SIZE;
-	}
-	spin_unlock(&mm->page_table_lock);
-
-	return (pte_t *)ret;
-}
-
-pte_t *pte_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel)
-{
-	pte_t *pte;
-
-	pte = get_from_cache(mm);
-	if (pte)
-		return pte;
-
-	return __alloc_for_cache(mm, kernel);
-}
-#endif /* CONFIG_PPC_64K_PAGES */
-
-void pte_fragment_free(unsigned long *table, int kernel)
-{
-	struct page *page = virt_to_page(table);
-	if (put_page_testzero(page)) {
-		if (!kernel)
-			pgtable_page_dtor(page);
-		free_unref_page(page);
-	}
-}
-
-#ifdef CONFIG_SMP
-void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
-{
-	unsigned long pgf = (unsigned long)table;
-
-	BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
-	pgf |= shift;
-	tlb_remove_table(tlb, (void *)pgf);
-}
-
-void __tlb_remove_table(void *_table)
-{
-	void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
-	unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
-
-	if (!shift)
-		/* PTE page needs special handling */
-		pte_fragment_free(table, 0);
-	else {
-		BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
-		kmem_cache_free(PGT_CACHE(shift), table);
-	}
-}
-#else
-void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
-{
-	if (!shift) {
-		/* PTE page needs special handling */
-		pte_fragment_free(table, 0);
-	} else {
-		BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
-		kmem_cache_free(PGT_CACHE(shift), table);
-	}
-}
-#endif
-
-#ifdef CONFIG_PPC_BOOK3S_64
-void __init mmu_partition_table_init(void)
-{
-	unsigned long patb_size = 1UL << PATB_SIZE_SHIFT;
-	unsigned long ptcr;
-
-	BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 36), "Partition table size too large.");
-	partition_tb = __va(memblock_alloc_base(patb_size, patb_size,
-						MEMBLOCK_ALLOC_ANYWHERE));
-
-	/* Initialize the Partition Table with no entries */
-	memset((void *)partition_tb, 0, patb_size);
-
-	/*
-	 * update partition table control register,
-	 * 64 K size.
-	 */
-	ptcr = __pa(partition_tb) | (PATB_SIZE_SHIFT - 12);
-	mtspr(SPRN_PTCR, ptcr);
-	powernv_set_nmmu_ptcr(ptcr);
-}
-
-void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
-				   unsigned long dw1)
-{
-	unsigned long old = be64_to_cpu(partition_tb[lpid].patb0);
-
-	partition_tb[lpid].patb0 = cpu_to_be64(dw0);
-	partition_tb[lpid].patb1 = cpu_to_be64(dw1);
-
-	/*
-	 * Global flush of TLBs and partition table caches for this lpid.
-	 * The type of flush (hash or radix) depends on what the previous
-	 * use of this partition ID was, not the new use.
-	 */
-	asm volatile("ptesync" : : : "memory");
-	if (old & PATB_HR) {
-		asm volatile(PPC_TLBIE_5(%0,%1,2,0,1) : :
-			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
-		asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
-			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
-		trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 1);
-	} else {
-		asm volatile(PPC_TLBIE_5(%0,%1,2,0,0) : :
-			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
-		trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 0);
-	}
-	/* do we need fixup here ?*/
-	asm volatile("eieio; tlbsync; ptesync" : : : "memory");
-}
-EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);
-#endif /* CONFIG_PPC_BOOK3S_64 */
-
 #ifdef CONFIG_STRICT_KERNEL_RWX
 void mark_rodata_ro(void)
 {
diff --git a/arch/powerpc/mm/ppc_mmu_32.c b/arch/powerpc/mm/ppc_mmu_32.c
index 2a049fb8523d..bea6c544e38f 100644
--- a/arch/powerpc/mm/ppc_mmu_32.c
+++ b/arch/powerpc/mm/ppc_mmu_32.c
@@ -167,7 +167,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea,
 {
 	pmd_t *pmd;
 
-	if (Hash == 0)
+	if (!Hash)
 		return;
 	pmd = pmd_offset(pud_offset(pgd_offset(mm, ea), ea), ea);
 	if (!pmd_none(*pmd))
diff --git a/arch/powerpc/mm/subpage-prot.c b/arch/powerpc/mm/subpage-prot.c
index f14a07c2fb90..9d16ee251fc0 100644
--- a/arch/powerpc/mm/subpage-prot.c
+++ b/arch/powerpc/mm/subpage-prot.c
@@ -13,6 +13,7 @@
 #include <linux/types.h>
 #include <linux/mm.h>
 #include <linux/hugetlb.h>
+#include <linux/syscalls.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -185,7 +186,8 @@ static void subpage_mark_vma_nohuge(struct mm_struct *mm, unsigned long addr,
  * in a 2-bit field won't allow writes to a page that is otherwise
  * write-protected.
  */
-long sys_subpage_prot(unsigned long addr, unsigned long len, u32 __user *map)
+SYSCALL_DEFINE3(subpage_prot, unsigned long, addr,
+		unsigned long, len, u32 __user *, map)
 {
 	struct mm_struct *mm = current->mm;
 	struct subpage_prot_table *spt = &mm->context.spt;
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index a5d7309c2d05..5ac3206c51cc 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -118,6 +118,53 @@ static inline void __tlbie_pid(unsigned long pid, unsigned long ric)
 	trace_tlbie(0, 0, rb, rs, ric, prs, r);
 }
 
+static inline void __tlbiel_lpid(unsigned long lpid, int set,
+				unsigned long ric)
+{
+	unsigned long rb,rs,prs,r;
+
+	rb = PPC_BIT(52); /* IS = 2 */
+	rb |= set << PPC_BITLSHIFT(51);
+	rs = 0;  /* LPID comes from LPIDR */
+	prs = 0; /* partition scoped */
+	r = 1;   /* radix format */
+
+	asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
+		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+	trace_tlbie(lpid, 1, rb, rs, ric, prs, r);
+}
+
+static inline void __tlbie_lpid(unsigned long lpid, unsigned long ric)
+{
+	unsigned long rb,rs,prs,r;
+
+	rb = PPC_BIT(52); /* IS = 2 */
+	rs = lpid;
+	prs = 0; /* partition scoped */
+	r = 1;   /* radix format */
+
+	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
+		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+	trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
+}
+
+static inline void __tlbiel_lpid_guest(unsigned long lpid, int set,
+				unsigned long ric)
+{
+	unsigned long rb,rs,prs,r;
+
+	rb = PPC_BIT(52); /* IS = 2 */
+	rb |= set << PPC_BITLSHIFT(51);
+	rs = 0;  /* LPID comes from LPIDR */
+	prs = 1; /* process scoped */
+	r = 1;   /* radix format */
+
+	asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
+		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+	trace_tlbie(lpid, 1, rb, rs, ric, prs, r);
+}
+
+
 static inline void __tlbiel_va(unsigned long va, unsigned long pid,
 			       unsigned long ap, unsigned long ric)
 {
@@ -150,6 +197,22 @@ static inline void __tlbie_va(unsigned long va, unsigned long pid,
 	trace_tlbie(0, 0, rb, rs, ric, prs, r);
 }
 
+static inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid,
+			      unsigned long ap, unsigned long ric)
+{
+	unsigned long rb,rs,prs,r;
+
+	rb = va & ~(PPC_BITMASK(52, 63));
+	rb |= ap << PPC_BITLSHIFT(58);
+	rs = lpid;
+	prs = 0; /* partition scoped */
+	r = 1;   /* radix format */
+
+	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
+		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+	trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
+}
+
 static inline void fixup_tlbie(void)
 {
 	unsigned long pid = 0;
@@ -161,6 +224,16 @@ static inline void fixup_tlbie(void)
 	}
 }
 
+static inline void fixup_tlbie_lpid(unsigned long lpid)
+{
+	unsigned long va = ((1UL << 52) - 1);
+
+	if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
+		asm volatile("ptesync": : :"memory");
+		__tlbie_lpid_va(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
+	}
+}
+
 /*
  * We use 128 set in radix mode and 256 set in hpt mode.
  */
@@ -214,6 +287,86 @@ static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
 	asm volatile("eieio; tlbsync; ptesync": : :"memory");
 }
 
+static inline void _tlbiel_lpid(unsigned long lpid, unsigned long ric)
+{
+	int set;
+
+	VM_BUG_ON(mfspr(SPRN_LPID) != lpid);
+
+	asm volatile("ptesync": : :"memory");
+
+	/*
+	 * Flush the first set of the TLB, and if we're doing a RIC_FLUSH_ALL,
+	 * also flush the entire Page Walk Cache.
+	 */
+	__tlbiel_lpid(lpid, 0, ric);
+
+	/* For PWC, only one flush is needed */
+	if (ric == RIC_FLUSH_PWC) {
+		asm volatile("ptesync": : :"memory");
+		return;
+	}
+
+	/* For the remaining sets, just flush the TLB */
+	for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++)
+		__tlbiel_lpid(lpid, set, RIC_FLUSH_TLB);
+
+	asm volatile("ptesync": : :"memory");
+	asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
+}
+
+static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric)
+{
+	asm volatile("ptesync": : :"memory");
+
+	/*
+	 * Workaround the fact that the "ric" argument to __tlbie_pid
+	 * must be a compile-time contraint to match the "i" constraint
+	 * in the asm statement.
+	 */
+	switch (ric) {
+	case RIC_FLUSH_TLB:
+		__tlbie_lpid(lpid, RIC_FLUSH_TLB);
+		break;
+	case RIC_FLUSH_PWC:
+		__tlbie_lpid(lpid, RIC_FLUSH_PWC);
+		break;
+	case RIC_FLUSH_ALL:
+	default:
+		__tlbie_lpid(lpid, RIC_FLUSH_ALL);
+	}
+	fixup_tlbie_lpid(lpid);
+	asm volatile("eieio; tlbsync; ptesync": : :"memory");
+}
+
+static inline void _tlbiel_lpid_guest(unsigned long lpid, unsigned long ric)
+{
+	int set;
+
+	VM_BUG_ON(mfspr(SPRN_LPID) != lpid);
+
+	asm volatile("ptesync": : :"memory");
+
+	/*
+	 * Flush the first set of the TLB, and if we're doing a RIC_FLUSH_ALL,
+	 * also flush the entire Page Walk Cache.
+	 */
+	__tlbiel_lpid_guest(lpid, 0, ric);
+
+	/* For PWC, only one flush is needed */
+	if (ric == RIC_FLUSH_PWC) {
+		asm volatile("ptesync": : :"memory");
+		return;
+	}
+
+	/* For the remaining sets, just flush the TLB */
+	for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++)
+		__tlbiel_lpid_guest(lpid, set, RIC_FLUSH_TLB);
+
+	asm volatile("ptesync": : :"memory");
+}
+
+
 static inline void __tlbiel_va_range(unsigned long start, unsigned long end,
 				    unsigned long pid, unsigned long page_size,
 				    unsigned long psize)
@@ -268,6 +421,17 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid,
 	asm volatile("eieio; tlbsync; ptesync": : :"memory");
 }
 
+static inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid,
+			      unsigned long psize, unsigned long ric)
+{
+	unsigned long ap = mmu_get_ap(psize);
+
+	asm volatile("ptesync": : :"memory");
+	__tlbie_lpid_va(va, lpid, ap, ric);
+	fixup_tlbie_lpid(lpid);
+	asm volatile("eieio; tlbsync; ptesync": : :"memory");
+}
+
 static inline void _tlbie_va_range(unsigned long start, unsigned long end,
 				    unsigned long pid, unsigned long page_size,
 				    unsigned long psize, bool also_pwc)
@@ -534,6 +698,49 @@ static int radix_get_mmu_psize(int page_size)
 	return psize;
 }
 
+/*
+ * Flush partition scoped LPID address translation for all CPUs.
+ */
+void radix__flush_tlb_lpid_page(unsigned int lpid,
+					unsigned long addr,
+					unsigned long page_size)
+{
+	int psize = radix_get_mmu_psize(page_size);
+
+	_tlbie_lpid_va(addr, lpid, psize, RIC_FLUSH_TLB);
+}
+EXPORT_SYMBOL_GPL(radix__flush_tlb_lpid_page);
+
+/*
+ * Flush partition scoped PWC from LPID for all CPUs.
+ */
+void radix__flush_pwc_lpid(unsigned int lpid)
+{
+	_tlbie_lpid(lpid, RIC_FLUSH_PWC);
+}
+EXPORT_SYMBOL_GPL(radix__flush_pwc_lpid);
+
+/*
+ * Flush partition scoped translations from LPID (=LPIDR)
+ */
+void radix__local_flush_tlb_lpid(unsigned int lpid)
+{
+	_tlbiel_lpid(lpid, RIC_FLUSH_ALL);
+}
+EXPORT_SYMBOL_GPL(radix__local_flush_tlb_lpid);
+
+/*
+ * Flush process scoped translations from LPID (=LPIDR).
+ * Important difference, the guest normally manages its own translations,
+ * but some cases e.g., vCPU CPU migration require KVM to flush.
+ */
+void radix__local_flush_tlb_lpid_guest(unsigned int lpid)
+{
+	_tlbiel_lpid_guest(lpid, RIC_FLUSH_ALL);
+}
+EXPORT_SYMBOL_GPL(radix__local_flush_tlb_lpid_guest);
+
+
 static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
 				  unsigned long end, int psize);
 
diff --git a/arch/powerpc/mm/tlb_hash32.c b/arch/powerpc/mm/tlb_hash32.c
index 702d7689d714..cf8472cf3d59 100644
--- a/arch/powerpc/mm/tlb_hash32.c
+++ b/arch/powerpc/mm/tlb_hash32.c
@@ -41,7 +41,7 @@ void flush_hash_entry(struct mm_struct *mm, pte_t *ptep, unsigned long addr)
 {
 	unsigned long ptephys;
 
-	if (Hash != 0) {
+	if (Hash) {
 		ptephys = __pa(ptep) & PAGE_MASK;
 		flush_hash_pages(mm->context.id, addr, ptephys, 1);
 	}
@@ -54,7 +54,7 @@ EXPORT_SYMBOL(flush_hash_entry);
  */
 void tlb_flush(struct mmu_gather *tlb)
 {
-	if (Hash == 0) {
+	if (!Hash) {
 		/*
 		 * 603 needs to flush the whole TLB here since
 		 * it doesn't use a hash table.
@@ -84,7 +84,7 @@ static void flush_range(struct mm_struct *mm, unsigned long start,
 	int count;
 	unsigned int ctx = mm->context.id;
 
-	if (Hash == 0) {
+	if (!Hash) {
 		_tlbia();
 		return;
 	}
@@ -124,7 +124,7 @@ void flush_tlb_mm(struct mm_struct *mm)
 {
 	struct vm_area_struct *mp;
 
-	if (Hash == 0) {
+	if (!Hash) {
 		_tlbia();
 		return;
 	}
@@ -145,7 +145,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 	struct mm_struct *mm;
 	pmd_t *pmd;
 
-	if (Hash == 0) {
+	if (!Hash) {
 		_tlbie(vmaddr);
 		return;
 	}
diff --git a/arch/powerpc/net/Makefile b/arch/powerpc/net/Makefile
index 02d369ca6a53..809f019d3cba 100644
--- a/arch/powerpc/net/Makefile
+++ b/arch/powerpc/net/Makefile
@@ -3,7 +3,7 @@
 # Arch-specific network modules
 #
 ifeq ($(CONFIG_PPC64),y)
-obj-$(CONFIG_BPF_JIT) += bpf_jit_asm64.o bpf_jit_comp64.o
+obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o
 else
 obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o
 endif
diff --git a/arch/powerpc/net/bpf_jit64.h b/arch/powerpc/net/bpf_jit64.h
index 8bdef7ed28a8..3609be4692b3 100644
--- a/arch/powerpc/net/bpf_jit64.h
+++ b/arch/powerpc/net/bpf_jit64.h
@@ -20,7 +20,7 @@
  * with our redzone usage.
  *
  *		[	prev sp		] <-------------
- *		[   nv gpr save area	] 8*8		|
+ *		[   nv gpr save area	] 6*8		|
  *		[    tail_call_cnt	] 8		|
  *		[    local_tmp_var	] 8		|
  * fp (r31) -->	[   ebpf stack space	] upto 512	|
@@ -28,8 +28,8 @@
  * sp (r1) --->	[    stack pointer	] --------------
  */
 
-/* for gpr non volatile registers BPG_REG_6 to 10, plus skb cache registers */
-#define BPF_PPC_STACK_SAVE	(8*8)
+/* for gpr non volatile registers BPG_REG_6 to 10 */
+#define BPF_PPC_STACK_SAVE	(6*8)
 /* for bpf JIT code internal usage */
 #define BPF_PPC_STACK_LOCALS	16
 /* stack frame excluding BPF stack, ensure this is quadword aligned */
@@ -39,10 +39,8 @@
 #ifndef __ASSEMBLY__
 
 /* BPF register usage */
-#define SKB_HLEN_REG	(MAX_BPF_JIT_REG + 0)
-#define SKB_DATA_REG	(MAX_BPF_JIT_REG + 1)
-#define TMP_REG_1	(MAX_BPF_JIT_REG + 2)
-#define TMP_REG_2	(MAX_BPF_JIT_REG + 3)
+#define TMP_REG_1	(MAX_BPF_JIT_REG + 0)
+#define TMP_REG_2	(MAX_BPF_JIT_REG + 1)
 
 /* BPF to ppc register mappings */
 static const int b2p[] = {
@@ -63,40 +61,23 @@ static const int b2p[] = {
 	[BPF_REG_FP] = 31,
 	/* eBPF jit internal registers */
 	[BPF_REG_AX] = 2,
-	[SKB_HLEN_REG] = 25,
-	[SKB_DATA_REG] = 26,
 	[TMP_REG_1] = 9,
 	[TMP_REG_2] = 10
 };
 
-/* PPC NVR range -- update this if we ever use NVRs below r24 */
-#define BPF_PPC_NVR_MIN		24
-
-/* Assembly helpers */
-#define DECLARE_LOAD_FUNC(func)	u64 func(u64 r3, u64 r4);			\
-				u64 func##_negative_offset(u64 r3, u64 r4);	\
-				u64 func##_positive_offset(u64 r3, u64 r4);
-
-DECLARE_LOAD_FUNC(sk_load_word);
-DECLARE_LOAD_FUNC(sk_load_half);
-DECLARE_LOAD_FUNC(sk_load_byte);
-
-#define CHOOSE_LOAD_FUNC(imm, func)						\
-			(imm < 0 ?						\
-			(imm >= SKF_LL_OFF ? func##_negative_offset : func) :	\
-			func##_positive_offset)
+/* PPC NVR range -- update this if we ever use NVRs below r27 */
+#define BPF_PPC_NVR_MIN		27
 
 #define SEEN_FUNC	0x1000 /* might call external helpers */
 #define SEEN_STACK	0x2000 /* uses BPF stack */
-#define SEEN_SKB	0x4000 /* uses sk_buff */
-#define SEEN_TAILCALL	0x8000 /* uses tail calls */
+#define SEEN_TAILCALL	0x4000 /* uses tail calls */
 
 struct codegen_context {
 	/*
 	 * This is used to track register usage as well
 	 * as calls to external helpers.
 	 * - register usage is tracked with corresponding
-	 *   bits (r3-r10 and r25-r31)
+	 *   bits (r3-r10 and r27-r31)
 	 * - rest of the bits can be used to track other
 	 *   things -- for now, we use bits 16 to 23
 	 *   encoded in SEEN_* macros above
diff --git a/arch/powerpc/net/bpf_jit_asm64.S b/arch/powerpc/net/bpf_jit_asm64.S
deleted file mode 100644
index 7e4c51430b84..000000000000
--- a/arch/powerpc/net/bpf_jit_asm64.S
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * bpf_jit_asm64.S: Packet/header access helper functions
- * for PPC64 BPF compiler.
- *
- * Copyright 2016, Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
- * 		   IBM Corporation
- *
- * Based on bpf_jit_asm.S by Matt Evans
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; version 2
- * of the License.
- */
-
-#include <asm/ppc_asm.h>
-#include <asm/ptrace.h>
-#include "bpf_jit64.h"
-
-/*
- * All of these routines are called directly from generated code,
- * with the below register usage:
- * r27		skb pointer (ctx)
- * r25		skb header length
- * r26		skb->data pointer
- * r4		offset
- *
- * Result is passed back in:
- * r8		data read in host endian format (accumulator)
- *
- * r9 is used as a temporary register
- */
-
-#define r_skb	r27
-#define r_hlen	r25
-#define r_data	r26
-#define r_off	r4
-#define r_val	r8
-#define r_tmp	r9
-
-_GLOBAL_TOC(sk_load_word)
-	cmpdi	r_off, 0
-	blt	bpf_slow_path_word_neg
-	b	sk_load_word_positive_offset
-
-_GLOBAL_TOC(sk_load_word_positive_offset)
-	/* Are we accessing past headlen? */
-	subi	r_tmp, r_hlen, 4
-	cmpd	r_tmp, r_off
-	blt	bpf_slow_path_word
-	/* Nope, just hitting the header.  cr0 here is eq or gt! */
-	LWZX_BE	r_val, r_data, r_off
-	blr	/* Return success, cr0 != LT */
-
-_GLOBAL_TOC(sk_load_half)
-	cmpdi	r_off, 0
-	blt	bpf_slow_path_half_neg
-	b	sk_load_half_positive_offset
-
-_GLOBAL_TOC(sk_load_half_positive_offset)
-	subi	r_tmp, r_hlen, 2
-	cmpd	r_tmp, r_off
-	blt	bpf_slow_path_half
-	LHZX_BE	r_val, r_data, r_off
-	blr
-
-_GLOBAL_TOC(sk_load_byte)
-	cmpdi	r_off, 0
-	blt	bpf_slow_path_byte_neg
-	b	sk_load_byte_positive_offset
-
-_GLOBAL_TOC(sk_load_byte_positive_offset)
-	cmpd	r_hlen, r_off
-	ble	bpf_slow_path_byte
-	lbzx	r_val, r_data, r_off
-	blr
-
-/*
- * Call out to skb_copy_bits:
- * Allocate a new stack frame here to remain ABI-compliant in
- * stashing LR.
- */
-#define bpf_slow_path_common(SIZE)					\
-	mflr	r0;							\
-	std	r0, PPC_LR_STKOFF(r1);					\
-	stdu	r1, -(STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS)(r1);	\
-	mr	r3, r_skb;						\
-	/* r4 = r_off as passed */					\
-	addi	r5, r1, STACK_FRAME_MIN_SIZE;				\
-	li	r6, SIZE;						\
-	bl	skb_copy_bits;						\
-	nop;								\
-	/* save r5 */							\
-	addi	r5, r1, STACK_FRAME_MIN_SIZE;				\
-	/* r3 = 0 on success */						\
-	addi	r1, r1, STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS;	\
-	ld	r0, PPC_LR_STKOFF(r1);					\
-	mtlr	r0;							\
-	cmpdi	r3, 0;							\
-	blt	bpf_error;	/* cr0 = LT */
-
-bpf_slow_path_word:
-	bpf_slow_path_common(4)
-	/* Data value is on stack, and cr0 != LT */
-	LWZX_BE	r_val, 0, r5
-	blr
-
-bpf_slow_path_half:
-	bpf_slow_path_common(2)
-	LHZX_BE	r_val, 0, r5
-	blr
-
-bpf_slow_path_byte:
-	bpf_slow_path_common(1)
-	lbzx	r_val, 0, r5
-	blr
-
-/*
- * Call out to bpf_internal_load_pointer_neg_helper
- */
-#define sk_negative_common(SIZE)				\
-	mflr	r0;						\
-	std	r0, PPC_LR_STKOFF(r1);				\
-	stdu	r1, -STACK_FRAME_MIN_SIZE(r1);			\
-	mr	r3, r_skb;					\
-	/* r4 = r_off, as passed */				\
-	li	r5, SIZE;					\
-	bl	bpf_internal_load_pointer_neg_helper;		\
-	nop;							\
-	addi	r1, r1, STACK_FRAME_MIN_SIZE;			\
-	ld	r0, PPC_LR_STKOFF(r1);				\
-	mtlr	r0;						\
-	/* R3 != 0 on success */				\
-	cmpldi	r3, 0;						\
-	beq	bpf_error_slow;	/* cr0 = EQ */
-
-bpf_slow_path_word_neg:
-	lis     r_tmp, -32	/* SKF_LL_OFF */
-	cmpd	r_off, r_tmp	/* addr < SKF_* */
-	blt	bpf_error	/* cr0 = LT */
-	b	sk_load_word_negative_offset
-
-_GLOBAL_TOC(sk_load_word_negative_offset)
-	sk_negative_common(4)
-	LWZX_BE	r_val, 0, r3
-	blr
-
-bpf_slow_path_half_neg:
-	lis     r_tmp, -32	/* SKF_LL_OFF */
-	cmpd	r_off, r_tmp	/* addr < SKF_* */
-	blt	bpf_error	/* cr0 = LT */
-	b	sk_load_half_negative_offset
-
-_GLOBAL_TOC(sk_load_half_negative_offset)
-	sk_negative_common(2)
-	LHZX_BE	r_val, 0, r3
-	blr
-
-bpf_slow_path_byte_neg:
-	lis     r_tmp, -32	/* SKF_LL_OFF */
-	cmpd	r_off, r_tmp	/* addr < SKF_* */
-	blt	bpf_error	/* cr0 = LT */
-	b	sk_load_byte_negative_offset
-
-_GLOBAL_TOC(sk_load_byte_negative_offset)
-	sk_negative_common(1)
-	lbzx	r_val, 0, r3
-	blr
-
-bpf_error_slow:
-	/* fabricate a cr0 = lt */
-	li	r_tmp, -1
-	cmpdi	r_tmp, 0
-bpf_error:
-	/*
-	 * Entered with cr0 = lt
-	 * Generated code will 'blt epilogue', returning 0.
-	 */
-	li	r_val, 0
-	blr
diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index 0ef3d9580e98..f1c95779843b 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -59,7 +59,7 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
  *		[	prev sp		] <-------------
  *		[	  ...       	] 		|
  * sp (r1) --->	[    stack pointer	] --------------
- *		[   nv gpr save area	] 8*8
+ *		[   nv gpr save area	] 6*8
  *		[    tail_call_cnt	] 8
  *		[    local_tmp_var	] 8
  *		[   unused red zone	] 208 bytes protected
@@ -88,21 +88,6 @@ static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
 	BUG();
 }
 
-static void bpf_jit_emit_skb_loads(u32 *image, struct codegen_context *ctx)
-{
-	/*
-	 * Load skb->len and skb->data_len
-	 * r3 points to skb
-	 */
-	PPC_LWZ(b2p[SKB_HLEN_REG], 3, offsetof(struct sk_buff, len));
-	PPC_LWZ(b2p[TMP_REG_1], 3, offsetof(struct sk_buff, data_len));
-	/* header_len = len - data_len */
-	PPC_SUB(b2p[SKB_HLEN_REG], b2p[SKB_HLEN_REG], b2p[TMP_REG_1]);
-
-	/* skb->data pointer */
-	PPC_BPF_LL(b2p[SKB_DATA_REG], 3, offsetof(struct sk_buff, data));
-}
-
 static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
 {
 	int i;
@@ -145,18 +130,6 @@ static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
 		if (bpf_is_seen_register(ctx, i))
 			PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
 
-	/*
-	 * Save additional non-volatile regs if we cache skb
-	 * Also, setup skb data
-	 */
-	if (ctx->seen & SEEN_SKB) {
-		PPC_BPF_STL(b2p[SKB_HLEN_REG], 1,
-				bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG]));
-		PPC_BPF_STL(b2p[SKB_DATA_REG], 1,
-				bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG]));
-		bpf_jit_emit_skb_loads(image, ctx);
-	}
-
 	/* Setup frame pointer to point to the bpf stack area */
 	if (bpf_is_seen_register(ctx, BPF_REG_FP))
 		PPC_ADDI(b2p[BPF_REG_FP], 1,
@@ -172,14 +145,6 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx
 		if (bpf_is_seen_register(ctx, i))
 			PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
 
-	/* Restore non-volatile registers used for skb cache */
-	if (ctx->seen & SEEN_SKB) {
-		PPC_BPF_LL(b2p[SKB_HLEN_REG], 1,
-				bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG]));
-		PPC_BPF_LL(b2p[SKB_DATA_REG], 1,
-				bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG]));
-	}
-
 	/* Tear down our stack frame */
 	if (bpf_has_stack_frame(ctx)) {
 		PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size);
@@ -202,25 +167,37 @@ static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
 
 static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 func)
 {
+	unsigned int i, ctx_idx = ctx->idx;
+
+	/* Load function address into r12 */
+	PPC_LI64(12, func);
+
+	/* For bpf-to-bpf function calls, the callee's address is unknown
+	 * until the last extra pass. As seen above, we use PPC_LI64() to
+	 * load the callee's address, but this may optimize the number of
+	 * instructions required based on the nature of the address.
+	 *
+	 * Since we don't want the number of instructions emitted to change,
+	 * we pad the optimized PPC_LI64() call with NOPs to guarantee that
+	 * we always have a five-instruction sequence, which is the maximum
+	 * that PPC_LI64() can emit.
+	 */
+	for (i = ctx->idx - ctx_idx; i < 5; i++)
+		PPC_NOP();
+
 #ifdef PPC64_ELF_ABI_v1
-	/* func points to the function descriptor */
-	PPC_LI64(b2p[TMP_REG_2], func);
-	/* Load actual entry point from function descriptor */
-	PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0);
-	/* ... and move it to LR */
-	PPC_MTLR(b2p[TMP_REG_1]);
 	/*
 	 * Load TOC from function descriptor at offset 8.
 	 * We can clobber r2 since we get called through a
 	 * function pointer (so caller will save/restore r2)
 	 * and since we don't use a TOC ourself.
 	 */
-	PPC_BPF_LL(2, b2p[TMP_REG_2], 8);
-#else
-	/* We can clobber r12 */
-	PPC_FUNC_ADDR(12, func);
-	PPC_MTLR(12);
+	PPC_BPF_LL(2, 12, 8);
+	/* Load actual entry point from function descriptor */
+	PPC_BPF_LL(12, 12, 0);
 #endif
+
+	PPC_MTLR(12);
 	PPC_BLRL();
 }
 
@@ -291,7 +268,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32
 /* Assemble the body code between the prologue & epilogue */
 static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
 			      struct codegen_context *ctx,
-			      u32 *addrs)
+			      u32 *addrs, bool extra_pass)
 {
 	const struct bpf_insn *insn = fp->insnsi;
 	int flen = fp->len;
@@ -747,29 +724,30 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
 			break;
 
 		/*
-		 * Call kernel helper
+		 * Call kernel helper or bpf function
 		 */
 		case BPF_JMP | BPF_CALL:
 			ctx->seen |= SEEN_FUNC;
-			func = (u8 *) __bpf_call_base + imm;
 
-			/* Save skb pointer if we need to re-cache skb data */
-			if ((ctx->seen & SEEN_SKB) &&
-			    bpf_helper_changes_pkt_data(func))
-				PPC_BPF_STL(3, 1, bpf_jit_stack_local(ctx));
+			/* bpf function call */
+			if (insn[i].src_reg == BPF_PSEUDO_CALL)
+				if (!extra_pass)
+					func = NULL;
+				else if (fp->aux->func && off < fp->aux->func_cnt)
+					/* use the subprog id from the off
+					 * field to lookup the callee address
+					 */
+					func = (u8 *) fp->aux->func[off]->bpf_func;
+				else
+					return -EINVAL;
+			/* kernel helper call */
+			else
+				func = (u8 *) __bpf_call_base + imm;
 
 			bpf_jit_emit_func_call(image, ctx, (u64)func);
 
 			/* move return value from r3 to BPF_REG_0 */
 			PPC_MR(b2p[BPF_REG_0], 3);
-
-			/* refresh skb cache */
-			if ((ctx->seen & SEEN_SKB) &&
-			    bpf_helper_changes_pkt_data(func)) {
-				/* reload skb pointer to r3 */
-				PPC_BPF_LL(3, 1, bpf_jit_stack_local(ctx));
-				bpf_jit_emit_skb_loads(image, ctx);
-			}
 			break;
 
 		/*
@@ -886,65 +864,6 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
 			PPC_BCC(true_cond, addrs[i + 1 + off]);
 			break;
 
-		/*
-		 * Loads from packet header/data
-		 * Assume 32-bit input value in imm and X (src_reg)
-		 */
-
-		/* Absolute loads */
-		case BPF_LD | BPF_W | BPF_ABS:
-			func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_word);
-			goto common_load_abs;
-		case BPF_LD | BPF_H | BPF_ABS:
-			func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_half);
-			goto common_load_abs;
-		case BPF_LD | BPF_B | BPF_ABS:
-			func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_byte);
-common_load_abs:
-			/*
-			 * Load from [imm]
-			 * Load into r4, which can just be passed onto
-			 *  skb load helpers as the second parameter
-			 */
-			PPC_LI32(4, imm);
-			goto common_load;
-
-		/* Indirect loads */
-		case BPF_LD | BPF_W | BPF_IND:
-			func = (u8 *)sk_load_word;
-			goto common_load_ind;
-		case BPF_LD | BPF_H | BPF_IND:
-			func = (u8 *)sk_load_half;
-			goto common_load_ind;
-		case BPF_LD | BPF_B | BPF_IND:
-			func = (u8 *)sk_load_byte;
-common_load_ind:
-			/*
-			 * Load from [src_reg + imm]
-			 * Treat src_reg as a 32-bit value
-			 */
-			PPC_EXTSW(4, src_reg);
-			if (imm) {
-				if (imm >= -32768 && imm < 32768)
-					PPC_ADDI(4, 4, IMM_L(imm));
-				else {
-					PPC_LI32(b2p[TMP_REG_1], imm);
-					PPC_ADD(4, 4, b2p[TMP_REG_1]);
-				}
-			}
-
-common_load:
-			ctx->seen |= SEEN_SKB;
-			ctx->seen |= SEEN_FUNC;
-			bpf_jit_emit_func_call(image, ctx, (u64)func);
-
-			/*
-			 * Helper returns 'lt' condition on error, and an
-			 * appropriate return value in BPF_REG_0
-			 */
-			PPC_BCC(COND_LT, exit_addr);
-			break;
-
 		/*
 		 * Tail call
 		 */
@@ -971,6 +890,14 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
 	return 0;
 }
 
+struct powerpc64_jit_data {
+	struct bpf_binary_header *header;
+	u32 *addrs;
+	u8 *image;
+	u32 proglen;
+	struct codegen_context ctx;
+};
+
 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 {
 	u32 proglen;
@@ -978,6 +905,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 	u8 *image = NULL;
 	u32 *code_base;
 	u32 *addrs;
+	struct powerpc64_jit_data *jit_data;
 	struct codegen_context cgctx;
 	int pass;
 	int flen;
@@ -985,6 +913,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 	struct bpf_prog *org_fp = fp;
 	struct bpf_prog *tmp_fp;
 	bool bpf_blinded = false;
+	bool extra_pass = false;
 
 	if (!fp->jit_requested)
 		return org_fp;
@@ -998,11 +927,32 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 		fp = tmp_fp;
 	}
 
+	jit_data = fp->aux->jit_data;
+	if (!jit_data) {
+		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+		if (!jit_data) {
+			fp = org_fp;
+			goto out;
+		}
+		fp->aux->jit_data = jit_data;
+	}
+
 	flen = fp->len;
+	addrs = jit_data->addrs;
+	if (addrs) {
+		cgctx = jit_data->ctx;
+		image = jit_data->image;
+		bpf_hdr = jit_data->header;
+		proglen = jit_data->proglen;
+		alloclen = proglen + FUNCTION_DESCR_SIZE;
+		extra_pass = true;
+		goto skip_init_ctx;
+	}
+
 	addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL);
 	if (addrs == NULL) {
 		fp = org_fp;
-		goto out;
+		goto out_addrs;
 	}
 
 	memset(&cgctx, 0, sizeof(struct codegen_context));
@@ -1011,10 +961,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 	cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
 
 	/* Scouting faux-generate pass 0 */
-	if (bpf_jit_build_body(fp, 0, &cgctx, addrs)) {
+	if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
 		/* We hit something illegal or unsupported. */
 		fp = org_fp;
-		goto out;
+		goto out_addrs;
 	}
 
 	/*
@@ -1032,9 +982,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 			bpf_jit_fill_ill_insns);
 	if (!bpf_hdr) {
 		fp = org_fp;
-		goto out;
+		goto out_addrs;
 	}
 
+skip_init_ctx:
 	code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
 
 	/* Code generation passes 1-2 */
@@ -1042,7 +993,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 		/* Now build the prologue, body code & epilogue for real. */
 		cgctx.idx = 0;
 		bpf_jit_build_prologue(code_base, &cgctx);
-		bpf_jit_build_body(fp, code_base, &cgctx, addrs);
+		bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass);
 		bpf_jit_build_epilogue(code_base, &cgctx);
 
 		if (bpf_jit_enable > 1)
@@ -1068,10 +1019,20 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 	fp->jited_len = alloclen;
 
 	bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE));
+	if (!fp->is_func || extra_pass) {
+out_addrs:
+		kfree(addrs);
+		kfree(jit_data);
+		fp->aux->jit_data = NULL;
+	} else {
+		jit_data->addrs = addrs;
+		jit_data->ctx = cgctx;
+		jit_data->proglen = proglen;
+		jit_data->image = image;
+		jit_data->header = bpf_hdr;
+	}
 
 out:
-	kfree(addrs);
-
 	if (bpf_blinded)
 		bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
 
diff --git a/arch/powerpc/oprofile/backtrace.c b/arch/powerpc/oprofile/backtrace.c
index ecc66d5f02c9..ad054dd0d666 100644
--- a/arch/powerpc/oprofile/backtrace.c
+++ b/arch/powerpc/oprofile/backtrace.c
@@ -7,6 +7,7 @@
  * 2 of the License, or (at your option) any later version.
 **/
 
+#include <linux/compat_time.h>
 #include <linux/oprofile.h>
 #include <linux/sched.h>
 #include <asm/processor.h>
diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c
index d7532e7b9ab5..75fb23c24ee8 100644
--- a/arch/powerpc/perf/imc-pmu.c
+++ b/arch/powerpc/perf/imc-pmu.c
@@ -1146,7 +1146,7 @@ static int init_nest_pmu_ref(void)
 
 static void cleanup_all_core_imc_memory(void)
 {
-	int i, nr_cores = DIV_ROUND_UP(num_present_cpus(), threads_per_core);
+	int i, nr_cores = DIV_ROUND_UP(num_possible_cpus(), threads_per_core);
 	struct imc_mem_info *ptr = core_imc_pmu->mem_info;
 	int size = core_imc_pmu->counter_mem_size;
 
@@ -1264,7 +1264,7 @@ static int imc_mem_init(struct imc_pmu *pmu_ptr, struct device_node *parent,
 		if (!pmu_ptr->pmu.name)
 			return -ENOMEM;
 
-		nr_cores = DIV_ROUND_UP(num_present_cpus(), threads_per_core);
+		nr_cores = DIV_ROUND_UP(num_possible_cpus(), threads_per_core);
 		pmu_ptr->mem_info = kcalloc(nr_cores, sizeof(struct imc_mem_info),
 								GFP_KERNEL);
 
diff --git a/arch/powerpc/platforms/83xx/Kconfig b/arch/powerpc/platforms/83xx/Kconfig
index 7e38b7b71a5a..071f53b0c0a0 100644
--- a/arch/powerpc/platforms/83xx/Kconfig
+++ b/arch/powerpc/platforms/83xx/Kconfig
@@ -90,13 +90,6 @@ config MPC837x_RDB
 	help
 	  This option enables support for the MPC837x RDB and WLAN Boards.
 
-config SBC834x
-	bool "Wind River SBC834x"
-	select DEFAULT_UIMAGE
-	select PPC_MPC834x
-	help
-	  This option enables support for the Wind River SBC834x board.
-
 config ASP834x
 	bool "Analogue & Micro ASP 834x"
 	select PPC_MPC834x
diff --git a/arch/powerpc/platforms/83xx/Makefile b/arch/powerpc/platforms/83xx/Makefile
index bb4720897f6a..41cb5f842eff 100644
--- a/arch/powerpc/platforms/83xx/Makefile
+++ b/arch/powerpc/platforms/83xx/Makefile
@@ -14,7 +14,6 @@ obj-$(CONFIG_MPC836x_MDS)	+= mpc836x_mds.o
 obj-$(CONFIG_MPC836x_RDK)	+= mpc836x_rdk.o
 obj-$(CONFIG_MPC832x_MDS)	+= mpc832x_mds.o
 obj-$(CONFIG_MPC837x_MDS)	+= mpc837x_mds.o
-obj-$(CONFIG_SBC834x)		+= sbc834x.o
 obj-$(CONFIG_MPC837x_RDB)	+= mpc837x_rdb.o
 obj-$(CONFIG_ASP834x)		+= asp834x.o
 obj-$(CONFIG_KMETER1)		+= km83xx.o
diff --git a/arch/powerpc/platforms/83xx/sbc834x.c b/arch/powerpc/platforms/83xx/sbc834x.c
deleted file mode 100644
index cb4bdabfdf1c..000000000000
--- a/arch/powerpc/platforms/83xx/sbc834x.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * arch/powerpc/platforms/83xx/sbc834x.c
- *
- * Wind River SBC834x board specific routines
- *
- * By Paul Gortmaker (see MAINTAINERS for contact information)
- *
- * Based largely on the mpc834x_mds.c support by Kumar Gala.
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-#include <linux/stddef.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/errno.h>
-#include <linux/reboot.h>
-#include <linux/pci.h>
-#include <linux/kdev_t.h>
-#include <linux/major.h>
-#include <linux/console.h>
-#include <linux/delay.h>
-#include <linux/seq_file.h>
-#include <linux/root_dev.h>
-#include <linux/of_platform.h>
-
-#include <linux/atomic.h>
-#include <asm/time.h>
-#include <asm/io.h>
-#include <asm/machdep.h>
-#include <asm/ipic.h>
-#include <asm/irq.h>
-#include <asm/prom.h>
-#include <asm/udbg.h>
-#include <sysdev/fsl_soc.h>
-#include <sysdev/fsl_pci.h>
-
-#include "mpc83xx.h"
-
-/* ************************************************************************
- *
- * Setup the architecture
- *
- */
-static void __init sbc834x_setup_arch(void)
-{
-	mpc83xx_setup_arch();
-}
-
-machine_device_initcall(sbc834x, mpc83xx_declare_of_platform_devices);
-
-/*
- * Called very early, MMU is off, device-tree isn't unflattened
- */
-static int __init sbc834x_probe(void)
-{
-	return of_machine_is_compatible("SBC834xE");
-}
-
-define_machine(sbc834x) {
-	.name			= "SBC834xE",
-	.probe			= sbc834x_probe,
-	.setup_arch		= sbc834x_setup_arch,
-	.init_IRQ		= mpc83xx_ipic_init_IRQ,
-	.get_irq		= ipic_get_irq,
-	.restart		= mpc83xx_restart,
-	.time_init		= mpc83xx_time_init,
-	.calibrate_decr		= generic_calibrate_decr,
-	.progress		= udbg_progress,
-};
diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index 67d3125d0610..e6a1de521319 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype
@@ -222,6 +222,7 @@ config PTE_64BIT
 config PHYS_64BIT
 	bool 'Large physical address support' if E500 || PPC_86xx
 	depends on (44x || E500 || PPC_86xx) && !PPC_83xx && !PPC_82xx
+	select PHYS_ADDR_T_64BIT
 	---help---
 	  This option enables kernel support for larger than 32-bit physical
 	  addresses.  This feature may not be available on all cores.
@@ -292,6 +293,10 @@ config PPC_STD_MMU_32
 	def_bool y
 	depends on PPC_STD_MMU && PPC32
 
+config ARCH_ENABLE_SPLIT_PMD_PTLOCK
+	def_bool y
+	depends on PPC_BOOK3S_64
+
 config PPC_RADIX_MMU
 	bool "Radix MMU Support"
 	depends on PPC_BOOK3S_64
diff --git a/arch/powerpc/platforms/cell/spu_callbacks.c b/arch/powerpc/platforms/cell/spu_callbacks.c
index a494028b2cdf..8ae86200ef6c 100644
--- a/arch/powerpc/platforms/cell/spu_callbacks.c
+++ b/arch/powerpc/platforms/cell/spu_callbacks.c
@@ -44,7 +44,7 @@ static void *spu_syscall_table[] = {
 
 #define SYSCALL_SPU(func)	sys_##func,
 #define COMPAT_SYS_SPU(func)	sys_##func,
-#define PPC_SYS_SPU(func)	ppc_##func,
+#define COMPAT_SPU_NEW(func)	sys_##func,
 #define SYSX_SPU(f, f3264, f32)	f,
 
 #include <asm/systbl.h>
diff --git a/arch/powerpc/platforms/cell/spu_syscalls.c b/arch/powerpc/platforms/cell/spu_syscalls.c
index 5e6e0bad6db6..263413a34823 100644
--- a/arch/powerpc/platforms/cell/spu_syscalls.c
+++ b/arch/powerpc/platforms/cell/spu_syscalls.c
@@ -26,6 +26,7 @@
 #include <linux/syscalls.h>
 #include <linux/rcupdate.h>
 #include <linux/binfmts.h>
+#include <linux/syscalls.h>
 
 #include <asm/spu.h>
 
@@ -90,7 +91,7 @@ SYSCALL_DEFINE4(spu_create, const char __user *, name, unsigned int, flags,
 	return ret;
 }
 
-asmlinkage long sys_spu_run(int fd, __u32 __user *unpc, __u32 __user *ustatus)
+SYSCALL_DEFINE3(spu_run,int, fd, __u32 __user *, unpc, __u32 __user *, ustatus)
 {
 	long ret;
 	struct fd arg;
diff --git a/arch/powerpc/platforms/cell/spufs/fault.c b/arch/powerpc/platforms/cell/spufs/fault.c
index 870c0a82d560..1e002e94d0f6 100644
--- a/arch/powerpc/platforms/cell/spufs/fault.c
+++ b/arch/powerpc/platforms/cell/spufs/fault.c
@@ -44,7 +44,7 @@ static void spufs_handle_event(struct spu_context *ctx,
 		return;
 	}
 
-	memset(&info, 0, sizeof(info));
+	clear_siginfo(&info);
 
 	switch (type) {
 	case SPE_EVENT_INVALID_DMA:
diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c
index 469bdd0b748f..43e7b93f27c7 100644
--- a/arch/powerpc/platforms/cell/spufs/file.c
+++ b/arch/powerpc/platforms/cell/spufs/file.c
@@ -232,12 +232,13 @@ spufs_mem_write(struct file *file, const char __user *buffer,
 	return size;
 }
 
-static int
+static vm_fault_t
 spufs_mem_mmap_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct spu_context *ctx	= vma->vm_file->private_data;
 	unsigned long pfn, offset;
+	vm_fault_t ret;
 
 	offset = vmf->pgoff << PAGE_SHIFT;
 	if (offset >= LS_SIZE)
@@ -256,11 +257,11 @@ spufs_mem_mmap_fault(struct vm_fault *vmf)
 		vma->vm_page_prot = pgprot_noncached_wc(vma->vm_page_prot);
 		pfn = (ctx->spu->local_store_phys + offset) >> PAGE_SHIFT;
 	}
-	vm_insert_pfn(vma, vmf->address, pfn);
+	ret = vmf_insert_pfn(vma, vmf->address, pfn);
 
 	spu_release(ctx);
 
-	return VM_FAULT_NOPAGE;
+	return ret;
 }
 
 static int spufs_mem_mmap_access(struct vm_area_struct *vma,
@@ -312,13 +313,14 @@ static const struct file_operations spufs_mem_fops = {
 	.mmap			= spufs_mem_mmap,
 };
 
-static int spufs_ps_fault(struct vm_fault *vmf,
+static vm_fault_t spufs_ps_fault(struct vm_fault *vmf,
 				    unsigned long ps_offs,
 				    unsigned long ps_size)
 {
 	struct spu_context *ctx = vmf->vma->vm_file->private_data;
 	unsigned long area, offset = vmf->pgoff << PAGE_SHIFT;
-	int ret = 0;
+	int err = 0;
+	vm_fault_t ret = VM_FAULT_NOPAGE;
 
 	spu_context_nospu_trace(spufs_ps_fault__enter, ctx);
 
@@ -349,25 +351,26 @@ static int spufs_ps_fault(struct vm_fault *vmf,
 	if (ctx->state == SPU_STATE_SAVED) {
 		up_read(&current->mm->mmap_sem);
 		spu_context_nospu_trace(spufs_ps_fault__sleep, ctx);
-		ret = spufs_wait(ctx->run_wq, ctx->state == SPU_STATE_RUNNABLE);
+		err = spufs_wait(ctx->run_wq, ctx->state == SPU_STATE_RUNNABLE);
 		spu_context_trace(spufs_ps_fault__wake, ctx, ctx->spu);
 		down_read(&current->mm->mmap_sem);
 	} else {
 		area = ctx->spu->problem_phys + ps_offs;
-		vm_insert_pfn(vmf->vma, vmf->address, (area + offset) >> PAGE_SHIFT);
+		ret = vmf_insert_pfn(vmf->vma, vmf->address,
+				(area + offset) >> PAGE_SHIFT);
 		spu_context_trace(spufs_ps_fault__insert, ctx, ctx->spu);
 	}
 
-	if (!ret)
+	if (!err)
 		spu_release(ctx);
 
 refault:
 	put_spu_context(ctx);
-	return VM_FAULT_NOPAGE;
+	return ret;
 }
 
 #if SPUFS_MMAP_4K
-static int spufs_cntl_mmap_fault(struct vm_fault *vmf)
+static vm_fault_t spufs_cntl_mmap_fault(struct vm_fault *vmf)
 {
 	return spufs_ps_fault(vmf, 0x4000, SPUFS_CNTL_MAP_SIZE);
 }
@@ -1040,7 +1043,7 @@ static ssize_t spufs_signal1_write(struct file *file, const char __user *buf,
 	return 4;
 }
 
-static int
+static vm_fault_t
 spufs_signal1_mmap_fault(struct vm_fault *vmf)
 {
 #if SPUFS_SIGNAL_MAP_SIZE == 0x1000
@@ -1178,7 +1181,7 @@ static ssize_t spufs_signal2_write(struct file *file, const char __user *buf,
 }
 
 #if SPUFS_MMAP_4K
-static int
+static vm_fault_t
 spufs_signal2_mmap_fault(struct vm_fault *vmf)
 {
 #if SPUFS_SIGNAL_MAP_SIZE == 0x1000
@@ -1307,7 +1310,7 @@ DEFINE_SPUFS_ATTRIBUTE(spufs_signal2_type, spufs_signal2_type_get,
 		       spufs_signal2_type_set, "%llu\n", SPU_ATTR_ACQUIRE);
 
 #if SPUFS_MMAP_4K
-static int
+static vm_fault_t
 spufs_mss_mmap_fault(struct vm_fault *vmf)
 {
 	return spufs_ps_fault(vmf, 0x0000, SPUFS_MSS_MAP_SIZE);
@@ -1369,7 +1372,7 @@ static const struct file_operations spufs_mss_fops = {
 	.llseek  = no_llseek,
 };
 
-static int
+static vm_fault_t
 spufs_psmap_mmap_fault(struct vm_fault *vmf)
 {
 	return spufs_ps_fault(vmf, 0x0000, SPUFS_PS_MAP_SIZE);
@@ -1429,7 +1432,7 @@ static const struct file_operations spufs_psmap_fops = {
 
 
 #if SPUFS_MMAP_4K
-static int
+static vm_fault_t
 spufs_mfc_mmap_fault(struct vm_fault *vmf)
 {
 	return spufs_ps_fault(vmf, 0x3000, SPUFS_MFC_MAP_SIZE);
diff --git a/arch/powerpc/platforms/cell/spufs/sched.c b/arch/powerpc/platforms/cell/spufs/sched.c
index ccc421503363..c9ef3c532169 100644
--- a/arch/powerpc/platforms/cell/spufs/sched.c
+++ b/arch/powerpc/platforms/cell/spufs/sched.c
@@ -1095,18 +1095,6 @@ static int show_spu_loadavg(struct seq_file *s, void *private)
 		atomic_read(&nr_spu_contexts),
 		idr_get_cursor(&task_active_pid_ns(current)->idr) - 1);
 	return 0;
-}
-
-static int spu_loadavg_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, show_spu_loadavg, NULL);
-}
-
-static const struct file_operations spu_loadavg_fops = {
-	.open		= spu_loadavg_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
 };
 
 int __init spu_sched_init(void)
@@ -1135,7 +1123,7 @@ int __init spu_sched_init(void)
 
 	mod_timer(&spuloadavg_timer, 0);
 
-	entry = proc_create("spu_loadavg", 0, NULL, &spu_loadavg_fops);
+	entry = proc_create_single("spu_loadavg", 0, NULL, show_spu_loadavg);
 	if (!entry)
 		goto out_stop_kthread;
 
diff --git a/arch/powerpc/platforms/chrp/pci.c b/arch/powerpc/platforms/chrp/pci.c
index 0f512d35f7c5..5ddb57b82921 100644
--- a/arch/powerpc/platforms/chrp/pci.c
+++ b/arch/powerpc/platforms/chrp/pci.c
@@ -31,7 +31,7 @@ void __iomem *gg2_pci_config_base;
  * limit the bus number to 3 bits
  */
 
-int gg2_read_config(struct pci_bus *bus, unsigned int devfn, int off,
+static int gg2_read_config(struct pci_bus *bus, unsigned int devfn, int off,
 			   int len, u32 *val)
 {
 	volatile void __iomem *cfg_data;
@@ -58,7 +58,7 @@ int gg2_read_config(struct pci_bus *bus, unsigned int devfn, int off,
 	return PCIBIOS_SUCCESSFUL;
 }
 
-int gg2_write_config(struct pci_bus *bus, unsigned int devfn, int off,
+static int gg2_write_config(struct pci_bus *bus, unsigned int devfn, int off,
 			    int len, u32 val)
 {
 	volatile void __iomem *cfg_data;
@@ -94,8 +94,8 @@ static struct pci_ops gg2_pci_ops =
 /*
  * Access functions for PCI config space using RTAS calls.
  */
-int rtas_read_config(struct pci_bus *bus, unsigned int devfn, int offset,
-		     int len, u32 *val)
+static int rtas_read_config(struct pci_bus *bus, unsigned int devfn, int offset,
+			    int len, u32 *val)
 {
 	struct pci_controller *hose = pci_bus_to_host(bus);
 	unsigned long addr = (offset & 0xff) | ((devfn & 0xff) << 8)
@@ -109,8 +109,8 @@ int rtas_read_config(struct pci_bus *bus, unsigned int devfn, int offset,
 	return rval? PCIBIOS_DEVICE_NOT_FOUND: PCIBIOS_SUCCESSFUL;
 }
 
-int rtas_write_config(struct pci_bus *bus, unsigned int devfn, int offset,
-		      int len, u32 val)
+static int rtas_write_config(struct pci_bus *bus, unsigned int devfn, int offset,
+			     int len, u32 val)
 {
 	struct pci_controller *hose = pci_bus_to_host(bus);
 	unsigned long addr = (offset & 0xff) | ((devfn & 0xff) << 8)
diff --git a/arch/powerpc/platforms/chrp/setup.c b/arch/powerpc/platforms/chrp/setup.c
index 481ed133e04b..d6d8ffc0271e 100644
--- a/arch/powerpc/platforms/chrp/setup.c
+++ b/arch/powerpc/platforms/chrp/setup.c
@@ -94,7 +94,7 @@ static const char *chrp_names[] = {
 	"Total Impact Briq"
 };
 
-void chrp_show_cpuinfo(struct seq_file *m)
+static void chrp_show_cpuinfo(struct seq_file *m)
 {
 	int i, sdramen;
 	unsigned int t;
@@ -299,7 +299,7 @@ static __init void chrp_init(void)
 	of_node_put(node);
 }
 
-void __init chrp_setup_arch(void)
+static void __init chrp_setup_arch(void)
 {
 	struct device_node *root = of_find_node_by_path("/");
 	const char *machine = NULL;
@@ -382,7 +382,7 @@ static void __init chrp_find_openpic(void)
 {
 	struct device_node *np, *root;
 	int len, i, j;
-	int isu_size, idu_size;
+	int isu_size;
 	const unsigned int *iranges, *opprop = NULL;
 	int oplen = 0;
 	unsigned long opaddr;
@@ -427,11 +427,9 @@ static void __init chrp_find_openpic(void)
 	}
 
 	isu_size = 0;
-	idu_size = 0;
 	if (len > 0 && iranges[1] != 0) {
 		printk(KERN_INFO "OpenPIC irqs %d..%d in IDU\n",
 		       iranges[0], iranges[0] + iranges[1] - 1);
-		idu_size = iranges[1];
 	}
 	if (len > 1)
 		isu_size = iranges[3];
@@ -523,7 +521,7 @@ static void __init chrp_find_8259(void)
 	}
 }
 
-void __init chrp_init_IRQ(void)
+static void __init chrp_init_IRQ(void)
 {
 #if defined(CONFIG_VT) && defined(CONFIG_INPUT_ADBHID) && defined(CONFIG_XMON)
 	struct device_node *kbd;
@@ -555,7 +553,7 @@ void __init chrp_init_IRQ(void)
 #endif
 }
 
-void __init
+static void __init
 chrp_init2(void)
 {
 #ifdef CONFIG_NVRAM
diff --git a/arch/powerpc/platforms/chrp/time.c b/arch/powerpc/platforms/chrp/time.c
index 03d115aaa191..acde7bbe0716 100644
--- a/arch/powerpc/platforms/chrp/time.c
+++ b/arch/powerpc/platforms/chrp/time.c
@@ -28,6 +28,8 @@
 #include <asm/sections.h>
 #include <asm/time.h>
 
+#include <platforms/chrp/chrp.h>
+
 extern spinlock_t rtc_lock;
 
 #define NVRAM_AS0  0x74
@@ -63,7 +65,7 @@ long __init chrp_time_init(void)
 	return 0;
 }
 
-int chrp_cmos_clock_read(int addr)
+static int chrp_cmos_clock_read(int addr)
 {
 	if (nvram_as1 != 0)
 		outb(addr>>8, nvram_as1);
@@ -71,7 +73,7 @@ int chrp_cmos_clock_read(int addr)
 	return (inb(nvram_data));
 }
 
-void chrp_cmos_clock_write(unsigned long val, int addr)
+static void chrp_cmos_clock_write(unsigned long val, int addr)
 {
 	if (nvram_as1 != 0)
 		outb(addr>>8, nvram_as1);
diff --git a/arch/powerpc/platforms/embedded6xx/flipper-pic.c b/arch/powerpc/platforms/embedded6xx/flipper-pic.c
index 7206f3f573d4..db0be007fd06 100644
--- a/arch/powerpc/platforms/embedded6xx/flipper-pic.c
+++ b/arch/powerpc/platforms/embedded6xx/flipper-pic.c
@@ -108,16 +108,8 @@ static int flipper_pic_map(struct irq_domain *h, unsigned int virq,
 	return 0;
 }
 
-static int flipper_pic_match(struct irq_domain *h, struct device_node *np,
-			     enum irq_domain_bus_token bus_token)
-{
-	return 1;
-}
-
-
 static const struct irq_domain_ops flipper_irq_domain_ops = {
 	.map = flipper_pic_map,
-	.match = flipper_pic_match,
 };
 
 /*
diff --git a/arch/powerpc/platforms/embedded6xx/hlwd-pic.c b/arch/powerpc/platforms/embedded6xx/hlwd-pic.c
index 89c54de88b7a..8112b39879d6 100644
--- a/arch/powerpc/platforms/embedded6xx/hlwd-pic.c
+++ b/arch/powerpc/platforms/embedded6xx/hlwd-pic.c
@@ -35,6 +35,8 @@
  */
 #define HW_BROADWAY_ICR		0x00
 #define HW_BROADWAY_IMR		0x04
+#define HW_STARLET_ICR		0x08
+#define HW_STARLET_IMR		0x0c
 
 
 /*
@@ -74,6 +76,9 @@ static void hlwd_pic_unmask(struct irq_data *d)
 	void __iomem *io_base = irq_data_get_irq_chip_data(d);
 
 	setbits32(io_base + HW_BROADWAY_IMR, 1 << irq);
+
+	/* Make sure the ARM (aka. Starlet) doesn't handle this interrupt. */
+	clrbits32(io_base + HW_STARLET_IMR, 1 << irq);
 }
 
 
@@ -155,7 +160,7 @@ static void __hlwd_quiesce(void __iomem *io_base)
 	out_be32(io_base + HW_BROADWAY_ICR, 0xffffffff);
 }
 
-struct irq_domain *hlwd_pic_init(struct device_node *np)
+static struct irq_domain *hlwd_pic_init(struct device_node *np)
 {
 	struct irq_domain *irq_domain;
 	struct resource res;
diff --git a/arch/powerpc/platforms/powermac/bootx_init.c b/arch/powerpc/platforms/powermac/bootx_init.c
index c3c9bbb3573a..ba0964c17620 100644
--- a/arch/powerpc/platforms/powermac/bootx_init.c
+++ b/arch/powerpc/platforms/powermac/bootx_init.c
@@ -468,7 +468,7 @@ void __init bootx_init(unsigned long r3, unsigned long r4)
 	boot_infos_t *bi = (boot_infos_t *) r4;
 	unsigned long hdr;
 	unsigned long space;
-	unsigned long ptr, x;
+	unsigned long ptr;
 	char *model;
 	unsigned long offset = reloc_offset();
 
@@ -562,6 +562,8 @@ void __init bootx_init(unsigned long r3, unsigned long r4)
 	 * MMU switched OFF, so this should not be useful anymore.
 	 */
 	if (bi->version < 4) {
+		unsigned long x __maybe_unused;
+
 		bootx_printf("Touching pages...\n");
 
 		/*
diff --git a/arch/powerpc/platforms/powermac/pci.c b/arch/powerpc/platforms/powermac/pci.c
index 0b8174a79993..df762bb3c735 100644
--- a/arch/powerpc/platforms/powermac/pci.c
+++ b/arch/powerpc/platforms/powermac/pci.c
@@ -62,7 +62,7 @@ struct device_node *k2_skiplist[2];
 
 static int __init fixup_one_level_bus_range(struct device_node *node, int higher)
 {
-	for (; node != 0;node = node->sibling) {
+	for (; node; node = node->sibling) {
 		const int * bus_range;
 		const unsigned int *class_code;
 		int len;
@@ -1219,7 +1219,7 @@ static void fixup_u4_pcie(struct pci_dev* dev)
 			region = r;
 	}
 	/* Nothing found, bail */
-	if (region == 0)
+	if (!region)
 		return;
 
 	/* Print things out */
diff --git a/arch/powerpc/platforms/powermac/setup.c b/arch/powerpc/platforms/powermac/setup.c
index ab668cb72263..3a529fcdae97 100644
--- a/arch/powerpc/platforms/powermac/setup.c
+++ b/arch/powerpc/platforms/powermac/setup.c
@@ -152,7 +152,7 @@ static void pmac_show_cpuinfo(struct seq_file *m)
 			of_get_property(np, "d-cache-size", NULL);
 		seq_printf(m, "L2 cache\t:");
 		has_l2cache = 1;
-		if (of_get_property(np, "cache-unified", NULL) != 0 && dc) {
+		if (of_get_property(np, "cache-unified", NULL) && dc) {
 			seq_printf(m, " %dK unified", *dc / 1024);
 		} else {
 			if (ic)
@@ -244,12 +244,12 @@ static void __init l2cr_init(void)
 	/* Checks "l2cr-value" property in the registry */
 	if (cpu_has_feature(CPU_FTR_L2CR)) {
 		struct device_node *np = of_find_node_by_name(NULL, "cpus");
-		if (np == 0)
+		if (!np)
 			np = of_find_node_by_type(NULL, "cpu");
-		if (np != 0) {
+		if (np) {
 			const unsigned int *l2cr =
 				of_get_property(np, "l2cr-value", NULL);
-			if (l2cr != 0) {
+			if (l2cr) {
 				ppc_override_l2cr = 1;
 				ppc_override_l2cr_value = *l2cr;
 				_set_L2CR(0);
@@ -352,6 +352,7 @@ static int pmac_late_init(void)
 }
 machine_late_initcall(powermac, pmac_late_init);
 
+void note_bootable_part(dev_t dev, int part, int goodness);
 /*
  * This is __ref because we check for "initializing" before
  * touching any of the __init sensitive things and "initializing"
diff --git a/arch/powerpc/platforms/powermac/smp.c b/arch/powerpc/platforms/powermac/smp.c
index 95275e0e2efa..447da6db450a 100644
--- a/arch/powerpc/platforms/powermac/smp.c
+++ b/arch/powerpc/platforms/powermac/smp.c
@@ -65,7 +65,6 @@
 #endif
 
 extern void __secondary_start_pmac_0(void);
-extern int pmac_pfunc_base_install(void);
 
 static void (*pmac_tb_freeze)(int freeze);
 static u64 timebase;
diff --git a/arch/powerpc/platforms/powernv/memtrace.c b/arch/powerpc/platforms/powernv/memtrace.c
index fc222a0c2ac4..b99283df8584 100644
--- a/arch/powerpc/platforms/powernv/memtrace.c
+++ b/arch/powerpc/platforms/powernv/memtrace.c
@@ -131,7 +131,7 @@ static u64 memtrace_alloc_node(u32 nid, u64 size)
 	u64 start_pfn, end_pfn, nr_pages;
 	u64 base_pfn;
 
-	if (!NODE_DATA(nid) || !node_spanned_pages(nid))
+	if (!node_spanned_pages(nid))
 		return 0;
 
 	start_pfn = node_start_pfn(nid);
diff --git a/arch/powerpc/platforms/powernv/npu-dma.c b/arch/powerpc/platforms/powernv/npu-dma.c
index 525e966dce34..8cdf91f5d3a4 100644
--- a/arch/powerpc/platforms/powernv/npu-dma.c
+++ b/arch/powerpc/platforms/powernv/npu-dma.c
@@ -459,10 +459,9 @@ static void mmio_launch_invalidate(struct mmio_atsd_reg *mmio_atsd_reg,
 	struct npu *npu = mmio_atsd_reg->npu;
 	int reg = mmio_atsd_reg->reg;
 
-	__raw_writeq(cpu_to_be64(va),
-		npu->mmio_atsd_regs[reg] + XTS_ATSD_AVA);
+	__raw_writeq_be(va, npu->mmio_atsd_regs[reg] + XTS_ATSD_AVA);
 	eieio();
-	__raw_writeq(cpu_to_be64(launch), npu->mmio_atsd_regs[reg]);
+	__raw_writeq_be(launch, npu->mmio_atsd_regs[reg]);
 }
 
 static void mmio_invalidate_pid(struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS],
diff --git a/arch/powerpc/platforms/powernv/opal-irqchip.c b/arch/powerpc/platforms/powernv/opal-irqchip.c
index 9d1b8c0aaf93..05ffe05f0fdc 100644
--- a/arch/powerpc/platforms/powernv/opal-irqchip.c
+++ b/arch/powerpc/platforms/powernv/opal-irqchip.c
@@ -177,7 +177,7 @@ void opal_event_shutdown(void)
 		if (!opal_irqs[i])
 			continue;
 
-		if (in_interrupt())
+		if (in_interrupt() || irqs_disabled())
 			disable_irq_nosync(opal_irqs[i]);
 		else
 			free_irq(opal_irqs[i], NULL);
diff --git a/arch/powerpc/platforms/powernv/opal-sensor.c b/arch/powerpc/platforms/powernv/opal-sensor.c
index 0a7074bb91dc..35a5f4b9aeb5 100644
--- a/arch/powerpc/platforms/powernv/opal-sensor.c
+++ b/arch/powerpc/platforms/powernv/opal-sensor.c
@@ -72,6 +72,59 @@ int opal_get_sensor_data(u32 sensor_hndl, u32 *sensor_data)
 }
 EXPORT_SYMBOL_GPL(opal_get_sensor_data);
 
+int opal_get_sensor_data_u64(u32 sensor_hndl, u64 *sensor_data)
+{
+	int ret, token;
+	struct opal_msg msg;
+	__be64 data;
+
+	if (!opal_check_token(OPAL_SENSOR_READ_U64)) {
+		u32 sdata;
+
+		ret = opal_get_sensor_data(sensor_hndl, &sdata);
+		if (!ret)
+			*sensor_data = sdata;
+		return ret;
+	}
+
+	token = opal_async_get_token_interruptible();
+	if (token < 0)
+		return token;
+
+	ret = opal_sensor_read_u64(sensor_hndl, token, &data);
+	switch (ret) {
+	case OPAL_ASYNC_COMPLETION:
+		ret = opal_async_wait_response(token, &msg);
+		if (ret) {
+			pr_err("%s: Failed to wait for the async response, %d\n",
+			       __func__, ret);
+			goto out_token;
+		}
+
+		ret = opal_error_code(opal_get_async_rc(msg));
+		*sensor_data = be64_to_cpu(data);
+		break;
+
+	case OPAL_SUCCESS:
+		ret = 0;
+		*sensor_data = be64_to_cpu(data);
+		break;
+
+	case OPAL_WRONG_STATE:
+		ret = -EIO;
+		break;
+
+	default:
+		ret = opal_error_code(ret);
+		break;
+	}
+
+out_token:
+	opal_async_release_token(token);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(opal_get_sensor_data_u64);
+
 int __init opal_sensor_init(void)
 {
 	struct platform_device *pdev;
diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S
index 3da30c2f26b4..8482df255969 100644
--- a/arch/powerpc/platforms/powernv/opal-wrappers.S
+++ b/arch/powerpc/platforms/powernv/opal-wrappers.S
@@ -325,3 +325,4 @@ OPAL_CALL(opal_npu_spa_clear_cache,		OPAL_NPU_SPA_CLEAR_CACHE);
 OPAL_CALL(opal_npu_tl_set,			OPAL_NPU_TL_SET);
 OPAL_CALL(opal_pci_get_pbcq_tunnel_bar,		OPAL_PCI_GET_PBCQ_TUNNEL_BAR);
 OPAL_CALL(opal_pci_set_pbcq_tunnel_bar,		OPAL_PCI_SET_PBCQ_TUNNEL_BAR);
+OPAL_CALL(opal_sensor_read_u64,			OPAL_SENSOR_READ_U64);
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 3f9c69d7623a..1b02f86dfec1 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1976,9 +1976,10 @@ static void pnv_pci_p7ioc_tce_invalidate(struct iommu_table *tbl,
         mb(); /* Ensure above stores are visible */
         while (start <= end) {
 		if (rm)
-			__raw_rm_writeq(cpu_to_be64(start), invalidate);
+			__raw_rm_writeq_be(start, invalidate);
 		else
-			__raw_writeq(cpu_to_be64(start), invalidate);
+			__raw_writeq_be(start, invalidate);
+
                 start += inc;
         }
 
@@ -2055,9 +2056,9 @@ static void pnv_pci_phb3_tce_invalidate_entire(struct pnv_phb *phb, bool rm)
 
 	mb(); /* Ensure previous TCE table stores are visible */
 	if (rm)
-		__raw_rm_writeq(cpu_to_be64(val), invalidate);
+		__raw_rm_writeq_be(val, invalidate);
 	else
-		__raw_writeq(cpu_to_be64(val), invalidate);
+		__raw_writeq_be(val, invalidate);
 }
 
 static inline void pnv_pci_phb3_tce_invalidate_pe(struct pnv_ioda_pe *pe)
@@ -2067,7 +2068,7 @@ static inline void pnv_pci_phb3_tce_invalidate_pe(struct pnv_ioda_pe *pe)
 	unsigned long val = PHB3_TCE_KILL_INVAL_PE | (pe->pe_number & 0xFF);
 
 	mb(); /* Ensure above stores are visible */
-	__raw_writeq(cpu_to_be64(val), invalidate);
+	__raw_writeq_be(val, invalidate);
 }
 
 static void pnv_pci_phb3_tce_invalidate(struct pnv_ioda_pe *pe, bool rm,
@@ -2090,9 +2091,9 @@ static void pnv_pci_phb3_tce_invalidate(struct pnv_ioda_pe *pe, bool rm,
 
 	while (start <= end) {
 		if (rm)
-			__raw_rm_writeq(cpu_to_be64(start), invalidate);
+			__raw_rm_writeq_be(start, invalidate);
 		else
-			__raw_writeq(cpu_to_be64(start), invalidate);
+			__raw_writeq_be(start, invalidate);
 		start += inc;
 	}
 }
@@ -2910,6 +2911,34 @@ static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
 			tbl->it_indirect_levels);
 }
 
+static unsigned long pnv_ioda_parse_tce_sizes(struct pnv_phb *phb)
+{
+	struct pci_controller *hose = phb->hose;
+	struct device_node *dn = hose->dn;
+	unsigned long mask = 0;
+	int i, rc, count;
+	u32 val;
+
+	count = of_property_count_u32_elems(dn, "ibm,supported-tce-sizes");
+	if (count <= 0) {
+		mask = SZ_4K | SZ_64K;
+		/* Add 16M for POWER8 by default */
+		if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
+				!cpu_has_feature(CPU_FTR_ARCH_300))
+			mask |= SZ_16M;
+		return mask;
+	}
+
+	for (i = 0; i < count; i++) {
+		rc = of_property_read_u32_index(dn, "ibm,supported-tce-sizes",
+						i, &val);
+		if (rc == 0)
+			mask |= 1ULL << val;
+	}
+
+	return mask;
+}
+
 static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
 				       struct pnv_ioda_pe *pe)
 {
@@ -2934,7 +2963,7 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
 	pe->table_group.max_dynamic_windows_supported =
 			IOMMU_TABLE_GROUP_MAX_TABLES;
 	pe->table_group.max_levels = POWERNV_IOMMU_MAX_LEVELS;
-	pe->table_group.pgsizes = SZ_4K | SZ_64K | SZ_16M;
+	pe->table_group.pgsizes = pnv_ioda_parse_tce_sizes(phb);
 #ifdef CONFIG_IOMMU_API
 	pe->table_group.ops = &pnv_pci_ioda2_ops;
 #endif
diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c
index a6648ec99ca7..1ab6dc70b5a4 100644
--- a/arch/powerpc/platforms/powernv/setup.c
+++ b/arch/powerpc/platforms/powernv/setup.c
@@ -357,15 +357,7 @@ static void pnv_kexec_cpu_down(int crash_shutdown, int secondary)
 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
 static unsigned long pnv_memory_block_size(void)
 {
-	/*
-	 * We map the kernel linear region with 1GB large pages on radix. For
-	 * memory hot unplug to work our memory block size must be at least
-	 * this size.
-	 */
-	if (radix_enabled())
-		return 1UL * 1024 * 1024 * 1024;
-	else
-		return 256UL * 1024 * 1024;
+	return 256UL * 1024 * 1024;
 }
 #endif
 
diff --git a/arch/powerpc/platforms/pseries/hvCall_inst.c b/arch/powerpc/platforms/pseries/hvCall_inst.c
index 89b7ce807e70..6da320c786cd 100644
--- a/arch/powerpc/platforms/pseries/hvCall_inst.c
+++ b/arch/powerpc/platforms/pseries/hvCall_inst.c
@@ -125,7 +125,7 @@ static void probe_hcall_entry(void *ignored, unsigned long opcode, unsigned long
 	h->purr_start = mfspr(SPRN_PURR);
 }
 
-static void probe_hcall_exit(void *ignored, unsigned long opcode, unsigned long retval,
+static void probe_hcall_exit(void *ignored, unsigned long opcode, long retval,
 			     unsigned long *retbuf)
 {
 	struct hcall_stats *h;
diff --git a/arch/powerpc/platforms/pseries/kexec.c b/arch/powerpc/platforms/pseries/kexec.c
index 3fe126796975..46fbaef69a59 100644
--- a/arch/powerpc/platforms/pseries/kexec.c
+++ b/arch/powerpc/platforms/pseries/kexec.c
@@ -57,8 +57,11 @@ void pseries_kexec_cpu_down(int crash_shutdown, int secondary)
 		}
 	}
 
-	if (xive_enabled())
+	if (xive_enabled()) {
 		xive_kexec_teardown_cpu(secondary);
-	else
+
+		if (!secondary)
+			xive_shutdown();
+	} else
 		xics_kexec_teardown_cpu(secondary);
 }
diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
index adb996ed51e1..5a392e40f3d2 100644
--- a/arch/powerpc/platforms/pseries/lpar.c
+++ b/arch/powerpc/platforms/pseries/lpar.c
@@ -902,8 +902,7 @@ void __trace_hcall_entry(unsigned long opcode, unsigned long *args)
 	local_irq_restore(flags);
 }
 
-void __trace_hcall_exit(long opcode, unsigned long retval,
-			unsigned long *retbuf)
+void __trace_hcall_exit(long opcode, long retval, unsigned long *retbuf)
 {
 	unsigned long flags;
 	unsigned int *depth;
diff --git a/arch/powerpc/sysdev/xive/spapr.c b/arch/powerpc/sysdev/xive/spapr.c
index 091f1d0d0af1..575db3b06a6b 100644
--- a/arch/powerpc/sysdev/xive/spapr.c
+++ b/arch/powerpc/sysdev/xive/spapr.c
@@ -19,6 +19,7 @@
 #include <linux/spinlock.h>
 #include <linux/cpumask.h>
 #include <linux/mm.h>
+#include <linux/delay.h>
 
 #include <asm/prom.h>
 #include <asm/io.h>
@@ -108,6 +109,51 @@ static void xive_irq_bitmap_free(int irq)
 	}
 }
 
+
+/* Based on the similar routines in RTAS */
+static unsigned int plpar_busy_delay_time(long rc)
+{
+	unsigned int ms = 0;
+
+	if (H_IS_LONG_BUSY(rc)) {
+		ms = get_longbusy_msecs(rc);
+	} else if (rc == H_BUSY) {
+		ms = 10; /* seems appropriate for XIVE hcalls */
+	}
+
+	return ms;
+}
+
+static unsigned int plpar_busy_delay(int rc)
+{
+	unsigned int ms;
+
+	ms = plpar_busy_delay_time(rc);
+	if (ms)
+		mdelay(ms);
+
+	return ms;
+}
+
+/*
+ * Note: this call has a partition wide scope and can take a while to
+ * complete. If it returns H_LONG_BUSY_* it should be retried
+ * periodically.
+ */
+static long plpar_int_reset(unsigned long flags)
+{
+	long rc;
+
+	do {
+		rc = plpar_hcall_norets(H_INT_RESET, flags);
+	} while (plpar_busy_delay(rc));
+
+	if (rc)
+		pr_err("H_INT_RESET failed %ld\n", rc);
+
+	return rc;
+}
+
 static long plpar_int_get_source_info(unsigned long flags,
 				      unsigned long lisn,
 				      unsigned long *src_flags,
@@ -118,7 +164,10 @@ static long plpar_int_get_source_info(unsigned long flags,
 	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
 	long rc;
 
-	rc = plpar_hcall(H_INT_GET_SOURCE_INFO, retbuf, flags, lisn);
+	do {
+		rc = plpar_hcall(H_INT_GET_SOURCE_INFO, retbuf, flags, lisn);
+	} while (plpar_busy_delay(rc));
+
 	if (rc) {
 		pr_err("H_INT_GET_SOURCE_INFO lisn=%ld failed %ld\n", lisn, rc);
 		return rc;
@@ -151,8 +200,11 @@ static long plpar_int_set_source_config(unsigned long flags,
 		flags, lisn, target, prio, sw_irq);
 
 
-	rc = plpar_hcall_norets(H_INT_SET_SOURCE_CONFIG, flags, lisn,
-				target, prio, sw_irq);
+	do {
+		rc = plpar_hcall_norets(H_INT_SET_SOURCE_CONFIG, flags, lisn,
+					target, prio, sw_irq);
+	} while (plpar_busy_delay(rc));
+
 	if (rc) {
 		pr_err("H_INT_SET_SOURCE_CONFIG lisn=%ld target=%lx prio=%lx failed %ld\n",
 		       lisn, target, prio, rc);
@@ -171,7 +223,11 @@ static long plpar_int_get_queue_info(unsigned long flags,
 	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
 	long rc;
 
-	rc = plpar_hcall(H_INT_GET_QUEUE_INFO, retbuf, flags, target, priority);
+	do {
+		rc = plpar_hcall(H_INT_GET_QUEUE_INFO, retbuf, flags, target,
+				 priority);
+	} while (plpar_busy_delay(rc));
+
 	if (rc) {
 		pr_err("H_INT_GET_QUEUE_INFO cpu=%ld prio=%ld failed %ld\n",
 		       target, priority, rc);
@@ -200,8 +256,11 @@ static long plpar_int_set_queue_config(unsigned long flags,
 	pr_devel("H_INT_SET_QUEUE_CONFIG flags=%lx target=%lx priority=%lx qpage=%lx qsize=%lx\n",
 		flags,  target, priority, qpage, qsize);
 
-	rc = plpar_hcall_norets(H_INT_SET_QUEUE_CONFIG, flags, target,
-				priority, qpage, qsize);
+	do {
+		rc = plpar_hcall_norets(H_INT_SET_QUEUE_CONFIG, flags, target,
+					priority, qpage, qsize);
+	} while (plpar_busy_delay(rc));
+
 	if (rc) {
 		pr_err("H_INT_SET_QUEUE_CONFIG cpu=%ld prio=%ld qpage=%lx returned %ld\n",
 		       target, priority, qpage, rc);
@@ -215,7 +274,10 @@ static long plpar_int_sync(unsigned long flags, unsigned long lisn)
 {
 	long rc;
 
-	rc = plpar_hcall_norets(H_INT_SYNC, flags, lisn);
+	do {
+		rc = plpar_hcall_norets(H_INT_SYNC, flags, lisn);
+	} while (plpar_busy_delay(rc));
+
 	if (rc) {
 		pr_err("H_INT_SYNC lisn=%ld returned %ld\n", lisn, rc);
 		return  rc;
@@ -238,7 +300,11 @@ static long plpar_int_esb(unsigned long flags,
 	pr_devel("H_INT_ESB flags=%lx lisn=%lx offset=%lx in=%lx\n",
 		flags,  lisn, offset, in_data);
 
-	rc = plpar_hcall(H_INT_ESB, retbuf, flags, lisn, offset, in_data);
+	do {
+		rc = plpar_hcall(H_INT_ESB, retbuf, flags, lisn, offset,
+				 in_data);
+	} while (plpar_busy_delay(rc));
+
 	if (rc) {
 		pr_err("H_INT_ESB lisn=%ld offset=%ld returned %ld\n",
 		       lisn, offset, rc);
@@ -445,11 +511,7 @@ static void xive_spapr_put_ipi(unsigned int cpu, struct xive_cpu *xc)
 
 static void xive_spapr_shutdown(void)
 {
-	long rc;
-
-	rc = plpar_hcall_norets(H_INT_RESET, 0);
-	if (rc)
-		pr_err("H_INT_RESET failed %ld\n", rc);
+	plpar_int_reset(0);
 }
 
 /*
diff --git a/arch/powerpc/tools/gcc-check-mprofile-kernel.sh b/arch/powerpc/tools/gcc-check-mprofile-kernel.sh
index 061f8035bdbe..137f3376ac2b 100755
--- a/arch/powerpc/tools/gcc-check-mprofile-kernel.sh
+++ b/arch/powerpc/tools/gcc-check-mprofile-kernel.sh
@@ -7,18 +7,21 @@ set -o pipefail
 # To debug, uncomment the following line
 # set -x
 
+# -mprofile-kernel is only supported on 64le, so this should not be invoked
+# for other targets. Therefore we can pass in -m64 and -mlittle-endian
+# explicitly, to take care of toolchains defaulting to other targets.
+
 # Test whether the compile option -mprofile-kernel exists and generates
 # profiling code (ie. a call to _mcount()).
 echo "int func() { return 0; }" | \
-    $* -S -x c -O2 -p -mprofile-kernel - -o - 2> /dev/null | \
-    grep -q "_mcount"
+    $* -m64 -mlittle-endian -S -x c -O2 -p -mprofile-kernel - -o - \
+    2> /dev/null | grep -q "_mcount"
 
 # Test whether the notrace attribute correctly suppresses calls to _mcount().
 
 echo -e "#include <linux/compiler.h>\nnotrace int func() { return 0; }" | \
-    $* -S -x c -O2 -p -mprofile-kernel - -o - 2> /dev/null | \
-    grep -q "_mcount" && \
+    $* -m64 -mlittle-endian -S -x c -O2 -p -mprofile-kernel - -o - \
+    2> /dev/null | grep -q "_mcount" && \
     exit 1
 
-echo "OK"
 exit 0
diff --git a/arch/powerpc/xmon/nonstdio.h b/arch/powerpc/xmon/nonstdio.h
index 2202ec61972c..e8deac6c84e2 100644
--- a/arch/powerpc/xmon/nonstdio.h
+++ b/arch/powerpc/xmon/nonstdio.h
@@ -1,13 +1,13 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #define EOF	(-1)
 
-#define printf	xmon_printf
-#define putchar	xmon_putchar
-
 extern void xmon_set_pagination_lpp(unsigned long lpp);
 extern void xmon_start_pagination(void);
 extern void xmon_end_pagination(void);
 extern int xmon_putchar(int c);
 extern void xmon_puts(const char *);
 extern char *xmon_gets(char *, int);
-extern void xmon_printf(const char *, ...);
+extern __printf(1, 2) void xmon_printf(const char *fmt, ...);
+
+#define printf	xmon_printf
+#define putchar	xmon_putchar
diff --git a/arch/powerpc/xmon/spu-dis.c b/arch/powerpc/xmon/spu-dis.c
index e5f89837c82e..4cbc7da88524 100644
--- a/arch/powerpc/xmon/spu-dis.c
+++ b/arch/powerpc/xmon/spu-dis.c
@@ -102,7 +102,7 @@ print_insn_spu (unsigned long insn, unsigned long memaddr)
 
   if (index == 0)
     {
-      printf(".long 0x%x", insn);
+      printf(".long 0x%lx", insn);
     }
   else
     {
@@ -134,27 +134,27 @@ print_insn_spu (unsigned long insn, unsigned long memaddr)
 	  switch (arg)
 	    {
 	    case A_T:
-	      printf("$%d",
+	      printf("$%lu",
 				     DECODE_INSN_RT (insn));
 	      break;
 	    case A_A:
-	      printf("$%d",
+	      printf("$%lu",
 				     DECODE_INSN_RA (insn));
 	      break;
 	    case A_B:
-	      printf("$%d",
+	      printf("$%lu",
 				     DECODE_INSN_RB (insn));
 	      break;
 	    case A_C:
-	      printf("$%d",
+	      printf("$%lu",
 				     DECODE_INSN_RC (insn));
 	      break;
 	    case A_S:
-	      printf("$sp%d",
+	      printf("$sp%lu",
 				     DECODE_INSN_RA (insn));
 	      break;
 	    case A_H:
-	      printf("$ch%d",
+	      printf("$ch%lu",
 				     DECODE_INSN_RA (insn));
 	      break;
 	    case A_P:
@@ -162,11 +162,11 @@ print_insn_spu (unsigned long insn, unsigned long memaddr)
 	      printf("(");
 	      break;
 	    case A_U7A:
-	      printf("%d",
+	      printf("%lu",
 				     173 - DECODE_INSN_U8 (insn));
 	      break;
 	    case A_U7B:
-	      printf("%d",
+	      printf("%lu",
 				     155 - DECODE_INSN_U8 (insn));
 	      break;
 	    case A_S3:
diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index a0842f1ff72c..c2e9270728c7 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -515,7 +515,7 @@ static int xmon_core(struct pt_regs *regs, int fromipi)
 		get_output_lock();
 		excprint(regs);
 		if (bp) {
-			printf("cpu 0x%x stopped at breakpoint 0x%lx (",
+			printf("cpu 0x%x stopped at breakpoint 0x%tx (",
 			       cpu, BP_NUM(bp));
 			xmon_print_symbol(regs->nip, " ", ")\n");
 		}
@@ -622,7 +622,7 @@ static int xmon_core(struct pt_regs *regs, int fromipi)
 		excprint(regs);
 		bp = at_breakpoint(regs->nip);
 		if (bp) {
-			printf("Stopped at breakpoint %lx (", BP_NUM(bp));
+			printf("Stopped at breakpoint %tx (", BP_NUM(bp));
 			xmon_print_symbol(regs->nip, " ", ")\n");
 		}
 		if (unrecoverable_excp(regs))
@@ -778,6 +778,16 @@ static int xmon_fault_handler(struct pt_regs *regs)
 	return 0;
 }
 
+/* Force enable xmon if not already enabled */
+static inline void force_enable_xmon(void)
+{
+	/* Enable xmon hooks if needed */
+	if (!xmon_on) {
+		printf("xmon: Enabling debugger hooks\n");
+		xmon_on = 1;
+	}
+}
+
 static struct bpt *at_breakpoint(unsigned long pc)
 {
 	int i;
@@ -1094,6 +1104,7 @@ static int do_step(struct pt_regs *regs)
 	unsigned int instr;
 	int stepped;
 
+	force_enable_xmon();
 	/* check we are in 64-bit kernel mode, translation enabled */
 	if ((regs->msr & (MSR_64BIT|MSR_PR|MSR_IR)) == (MSR_64BIT|MSR_IR)) {
 		if (mread(regs->nip, &instr, 4) == 4) {
@@ -1160,7 +1171,7 @@ static int cpu_cmd(void)
 	}
 	/* try to switch to cpu specified */
 	if (!cpumask_test_cpu(cpu, &cpus_in_xmon)) {
-		printf("cpu 0x%x isn't in xmon\n", cpu);
+		printf("cpu 0x%lx isn't in xmon\n", cpu);
 		return 0;
 	}
 	xmon_taken = 0;
@@ -1174,7 +1185,7 @@ static int cpu_cmd(void)
 			/* take control back */
 			mb();
 			xmon_owner = smp_processor_id();
-			printf("cpu 0x%x didn't take control\n", cpu);
+			printf("cpu 0x%lx didn't take control\n", cpu);
 			return 0;
 		}
 		barrier();
@@ -1268,16 +1279,6 @@ static long check_bp_loc(unsigned long addr)
 	return 1;
 }
 
-/* Force enable xmon if not already enabled */
-static inline void force_enable_xmon(void)
-{
-	/* Enable xmon hooks if needed */
-	if (!xmon_on) {
-		printf("xmon: Enabling debugger hooks\n");
-		xmon_on = 1;
-	}
-}
-
 static char *breakpoint_help_string =
     "Breakpoint command usage:\n"
     "b                show breakpoints\n"
@@ -1374,7 +1375,7 @@ bpt_cmds(void)
 			}
 		}
 
-		printf("Cleared breakpoint %lx (", BP_NUM(bp));
+		printf("Cleared breakpoint %tx (", BP_NUM(bp));
 		xmon_print_symbol(bp->address, " ", ")\n");
 		bp->enabled = 0;
 		break;
@@ -1401,7 +1402,7 @@ bpt_cmds(void)
 			for (bp = bpts; bp < &bpts[NBPTS]; ++bp) {
 				if (!bp->enabled)
 					continue;
-				printf("%2x %s   ", BP_NUM(bp),
+				printf("%tx %s   ", BP_NUM(bp),
 				    (bp->enabled & BP_CIABR) ? "inst": "trap");
 				xmon_print_symbol(bp->address, "  ", "\n");
 			}
@@ -1618,11 +1619,11 @@ static void excprint(struct pt_regs *fp)
 #endif /* CONFIG_SMP */
 
 	trap = TRAP(fp);
-	printf("Vector: %lx %s at [%lx]\n", fp->trap, getvecname(trap), fp);
+	printf("Vector: %lx %s at [%px]\n", fp->trap, getvecname(trap), fp);
 	printf("    pc: ");
 	xmon_print_symbol(fp->nip, ": ", "\n");
 
-	printf("    lr: ", fp->link);
+	printf("    lr: ");
 	xmon_print_symbol(fp->link, ": ", "\n");
 
 	printf("    sp: %lx\n", fp->gpr[1]);
@@ -1634,13 +1635,13 @@ static void excprint(struct pt_regs *fp)
 			printf(" dsisr: %lx\n", fp->dsisr);
 	}
 
-	printf("  current = 0x%lx\n", current);
+	printf("  current = 0x%px\n", current);
 #ifdef CONFIG_PPC64
-	printf("  paca    = 0x%lx\t softe: %d\t irq_happened: 0x%02x\n",
+	printf("  paca    = 0x%px\t softe: %d\t irq_happened: 0x%02x\n",
 	       local_paca, local_paca->irq_soft_mask, local_paca->irq_happened);
 #endif
 	if (current) {
-		printf("    pid   = %ld, comm = %s\n",
+		printf("    pid   = %d, comm = %s\n",
 		       current->pid, current->comm);
 	}
 
@@ -1676,16 +1677,16 @@ static void prregs(struct pt_regs *fp)
 #ifdef CONFIG_PPC64
 	if (FULL_REGS(fp)) {
 		for (n = 0; n < 16; ++n)
-			printf("R%.2ld = "REG"   R%.2ld = "REG"\n",
+			printf("R%.2d = "REG"   R%.2d = "REG"\n",
 			       n, fp->gpr[n], n+16, fp->gpr[n+16]);
 	} else {
 		for (n = 0; n < 7; ++n)
-			printf("R%.2ld = "REG"   R%.2ld = "REG"\n",
+			printf("R%.2d = "REG"   R%.2d = "REG"\n",
 			       n, fp->gpr[n], n+7, fp->gpr[n+7]);
 	}
 #else
 	for (n = 0; n < 32; ++n) {
-		printf("R%.2d = %.8x%s", n, fp->gpr[n],
+		printf("R%.2d = %.8lx%s", n, fp->gpr[n],
 		       (n & 3) == 3? "\n": "   ");
 		if (n == 12 && !FULL_REGS(fp)) {
 			printf("\n");
@@ -1789,9 +1790,9 @@ static void dump_206_sprs(void)
 
 	/* Actually some of these pre-date 2.06, but whatevs */
 
-	printf("srr0   = %.16lx  srr1  = %.16lx dsisr  = %.8x\n",
+	printf("srr0   = %.16lx  srr1  = %.16lx dsisr  = %.8lx\n",
 		mfspr(SPRN_SRR0), mfspr(SPRN_SRR1), mfspr(SPRN_DSISR));
-	printf("dscr   = %.16lx  ppr   = %.16lx pir    = %.8x\n",
+	printf("dscr   = %.16lx  ppr   = %.16lx pir    = %.8lx\n",
 		mfspr(SPRN_DSCR), mfspr(SPRN_PPR), mfspr(SPRN_PIR));
 	printf("amr    = %.16lx  uamor = %.16lx\n",
 		mfspr(SPRN_AMR), mfspr(SPRN_UAMOR));
@@ -1799,11 +1800,11 @@ static void dump_206_sprs(void)
 	if (!(mfmsr() & MSR_HV))
 		return;
 
-	printf("sdr1   = %.16lx  hdar  = %.16lx hdsisr = %.8x\n",
+	printf("sdr1   = %.16lx  hdar  = %.16lx hdsisr = %.8lx\n",
 		mfspr(SPRN_SDR1), mfspr(SPRN_HDAR), mfspr(SPRN_HDSISR));
 	printf("hsrr0  = %.16lx hsrr1  = %.16lx hdec   = %.16lx\n",
 		mfspr(SPRN_HSRR0), mfspr(SPRN_HSRR1), mfspr(SPRN_HDEC));
-	printf("lpcr   = %.16lx  pcr   = %.16lx lpidr  = %.8x\n",
+	printf("lpcr   = %.16lx  pcr   = %.16lx lpidr  = %.8lx\n",
 		mfspr(SPRN_LPCR), mfspr(SPRN_PCR), mfspr(SPRN_LPID));
 	printf("hsprg0 = %.16lx hsprg1 = %.16lx amor   = %.16lx\n",
 		mfspr(SPRN_HSPRG0), mfspr(SPRN_HSPRG1), mfspr(SPRN_AMOR));
@@ -1820,10 +1821,10 @@ static void dump_207_sprs(void)
 	if (!cpu_has_feature(CPU_FTR_ARCH_207S))
 		return;
 
-	printf("dpdes  = %.16lx  tir   = %.16lx cir    = %.8x\n",
+	printf("dpdes  = %.16lx  tir   = %.16lx cir    = %.8lx\n",
 		mfspr(SPRN_DPDES), mfspr(SPRN_TIR), mfspr(SPRN_CIR));
 
-	printf("fscr   = %.16lx  tar   = %.16lx pspb   = %.8x\n",
+	printf("fscr   = %.16lx  tar   = %.16lx pspb   = %.8lx\n",
 		mfspr(SPRN_FSCR), mfspr(SPRN_TAR), mfspr(SPRN_PSPB));
 
 	msr = mfmsr();
@@ -1836,12 +1837,12 @@ static void dump_207_sprs(void)
 
 	printf("mmcr0  = %.16lx  mmcr1 = %.16lx mmcr2  = %.16lx\n",
 		mfspr(SPRN_MMCR0), mfspr(SPRN_MMCR1), mfspr(SPRN_MMCR2));
-	printf("pmc1   = %.8x pmc2 = %.8x  pmc3 = %.8x  pmc4   = %.8x\n",
+	printf("pmc1   = %.8lx pmc2 = %.8lx  pmc3 = %.8lx  pmc4   = %.8lx\n",
 		mfspr(SPRN_PMC1), mfspr(SPRN_PMC2),
 		mfspr(SPRN_PMC3), mfspr(SPRN_PMC4));
-	printf("mmcra  = %.16lx   siar = %.16lx pmc5   = %.8x\n",
+	printf("mmcra  = %.16lx   siar = %.16lx pmc5   = %.8lx\n",
 		mfspr(SPRN_MMCRA), mfspr(SPRN_SIAR), mfspr(SPRN_PMC5));
-	printf("sdar   = %.16lx   sier = %.16lx pmc6   = %.8x\n",
+	printf("sdar   = %.16lx   sier = %.16lx pmc6   = %.8lx\n",
 		mfspr(SPRN_SDAR), mfspr(SPRN_SIER), mfspr(SPRN_PMC6));
 	printf("ebbhr  = %.16lx  ebbrr = %.16lx bescr  = %.16lx\n",
 		mfspr(SPRN_EBBHR), mfspr(SPRN_EBBRR), mfspr(SPRN_BESCR));
@@ -2345,31 +2346,31 @@ static void dump_one_paca(int cpu)
 
 	printf("paca for cpu 0x%x @ %px:\n", cpu, p);
 
-	printf(" %-*s = %s\n", 20, "possible", cpu_possible(cpu) ? "yes" : "no");
-	printf(" %-*s = %s\n", 20, "present", cpu_present(cpu) ? "yes" : "no");
-	printf(" %-*s = %s\n", 20, "online", cpu_online(cpu) ? "yes" : "no");
+	printf(" %-*s = %s\n", 25, "possible", cpu_possible(cpu) ? "yes" : "no");
+	printf(" %-*s = %s\n", 25, "present", cpu_present(cpu) ? "yes" : "no");
+	printf(" %-*s = %s\n", 25, "online", cpu_online(cpu) ? "yes" : "no");
 
-#define DUMP(paca, name, format) \
-	printf(" %-*s = %#-*"format"\t(0x%lx)\n", 20, #name, 18, paca->name, \
+#define DUMP(paca, name, format)				\
+	printf(" %-*s = "format"\t(0x%lx)\n", 25, #name, 18, paca->name, \
 		offsetof(struct paca_struct, name));
 
-	DUMP(p, lock_token, "x");
-	DUMP(p, paca_index, "x");
-	DUMP(p, kernel_toc, "lx");
-	DUMP(p, kernelbase, "lx");
-	DUMP(p, kernel_msr, "lx");
-	DUMP(p, emergency_sp, "px");
+	DUMP(p, lock_token, "%#-*x");
+	DUMP(p, paca_index, "%#-*x");
+	DUMP(p, kernel_toc, "%#-*llx");
+	DUMP(p, kernelbase, "%#-*llx");
+	DUMP(p, kernel_msr, "%#-*llx");
+	DUMP(p, emergency_sp, "%-*px");
 #ifdef CONFIG_PPC_BOOK3S_64
-	DUMP(p, nmi_emergency_sp, "px");
-	DUMP(p, mc_emergency_sp, "px");
-	DUMP(p, in_nmi, "x");
-	DUMP(p, in_mce, "x");
-	DUMP(p, hmi_event_available, "x");
+	DUMP(p, nmi_emergency_sp, "%-*px");
+	DUMP(p, mc_emergency_sp, "%-*px");
+	DUMP(p, in_nmi, "%#-*x");
+	DUMP(p, in_mce, "%#-*x");
+	DUMP(p, hmi_event_available, "%#-*x");
 #endif
-	DUMP(p, data_offset, "lx");
-	DUMP(p, hw_cpu_id, "x");
-	DUMP(p, cpu_start, "x");
-	DUMP(p, kexec_state, "x");
+	DUMP(p, data_offset, "%#-*llx");
+	DUMP(p, hw_cpu_id, "%#-*x");
+	DUMP(p, cpu_start, "%#-*x");
+	DUMP(p, kexec_state, "%#-*x");
 #ifdef CONFIG_PPC_BOOK3S_64
 	for (i = 0; i < SLB_NUM_BOLTED; i++) {
 		u64 esid, vsid;
@@ -2381,58 +2382,69 @@ static void dump_one_paca(int cpu)
 		vsid = be64_to_cpu(p->slb_shadow_ptr->save_area[i].vsid);
 
 		if (esid || vsid) {
-			printf(" slb_shadow[%d]:       = 0x%016lx 0x%016lx\n",
-				i, esid, vsid);
+			printf(" %-*s[%d] = 0x%016llx 0x%016llx\n",
+			       22, "slb_shadow", i, esid, vsid);
 		}
 	}
-	DUMP(p, vmalloc_sllp, "x");
-	DUMP(p, slb_cache_ptr, "x");
+	DUMP(p, vmalloc_sllp, "%#-*x");
+	DUMP(p, slb_cache_ptr, "%#-*x");
 	for (i = 0; i < SLB_CACHE_ENTRIES; i++)
-		printf(" slb_cache[%d]:        = 0x%016lx\n", i, p->slb_cache[i]);
+		printf(" %-*s[%d] = 0x%016x\n",
+		       22, "slb_cache", i, p->slb_cache[i]);
 
-	DUMP(p, rfi_flush_fallback_area, "px");
+	DUMP(p, rfi_flush_fallback_area, "%-*px");
 #endif
-	DUMP(p, dscr_default, "llx");
+	DUMP(p, dscr_default, "%#-*llx");
 #ifdef CONFIG_PPC_BOOK3E
-	DUMP(p, pgd, "px");
-	DUMP(p, kernel_pgd, "px");
-	DUMP(p, tcd_ptr, "px");
-	DUMP(p, mc_kstack, "px");
-	DUMP(p, crit_kstack, "px");
-	DUMP(p, dbg_kstack, "px");
+	DUMP(p, pgd, "%-*px");
+	DUMP(p, kernel_pgd, "%-*px");
+	DUMP(p, tcd_ptr, "%-*px");
+	DUMP(p, mc_kstack, "%-*px");
+	DUMP(p, crit_kstack, "%-*px");
+	DUMP(p, dbg_kstack, "%-*px");
 #endif
-	DUMP(p, __current, "px");
-	DUMP(p, kstack, "lx");
-	printf(" kstack_base          = 0x%016lx\n", p->kstack & ~(THREAD_SIZE - 1));
-	DUMP(p, stab_rr, "lx");
-	DUMP(p, saved_r1, "lx");
-	DUMP(p, trap_save, "x");
-	DUMP(p, irq_soft_mask, "x");
-	DUMP(p, irq_happened, "x");
-	DUMP(p, io_sync, "x");
-	DUMP(p, irq_work_pending, "x");
-	DUMP(p, nap_state_lost, "x");
-	DUMP(p, sprg_vdso, "llx");
+	DUMP(p, __current, "%-*px");
+	DUMP(p, kstack, "%#-*llx");
+	printf(" %-*s = 0x%016llx\n", 25, "kstack_base", p->kstack & ~(THREAD_SIZE - 1));
+	DUMP(p, stab_rr, "%#-*llx");
+	DUMP(p, saved_r1, "%#-*llx");
+	DUMP(p, trap_save, "%#-*x");
+	DUMP(p, irq_soft_mask, "%#-*x");
+	DUMP(p, irq_happened, "%#-*x");
+	DUMP(p, io_sync, "%#-*x");
+	DUMP(p, irq_work_pending, "%#-*x");
+	DUMP(p, nap_state_lost, "%#-*x");
+	DUMP(p, sprg_vdso, "%#-*llx");
 
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
-	DUMP(p, tm_scratch, "llx");
+	DUMP(p, tm_scratch, "%#-*llx");
 #endif
 
 #ifdef CONFIG_PPC_POWERNV
-	DUMP(p, core_idle_state_ptr, "px");
-	DUMP(p, thread_idle_state, "x");
-	DUMP(p, thread_mask, "x");
-	DUMP(p, subcore_sibling_mask, "x");
+	DUMP(p, core_idle_state_ptr, "%-*px");
+	DUMP(p, thread_idle_state, "%#-*x");
+	DUMP(p, thread_mask, "%#-*x");
+	DUMP(p, subcore_sibling_mask, "%#-*x");
+	DUMP(p, thread_sibling_pacas, "%-*px");
+	DUMP(p, requested_psscr, "%#-*llx");
+	DUMP(p, stop_sprs.pid, "%#-*llx");
+	DUMP(p, stop_sprs.ldbar, "%#-*llx");
+	DUMP(p, stop_sprs.fscr, "%#-*llx");
+	DUMP(p, stop_sprs.hfscr, "%#-*llx");
+	DUMP(p, stop_sprs.mmcr1, "%#-*llx");
+	DUMP(p, stop_sprs.mmcr2, "%#-*llx");
+	DUMP(p, stop_sprs.mmcra, "%#-*llx");
+	DUMP(p, dont_stop.counter, "%#-*x");
 #endif
 
-	DUMP(p, accounting.utime, "llx");
-	DUMP(p, accounting.stime, "llx");
-	DUMP(p, accounting.utime_scaled, "llx");
-	DUMP(p, accounting.starttime, "llx");
-	DUMP(p, accounting.starttime_user, "llx");
-	DUMP(p, accounting.startspurr, "llx");
-	DUMP(p, accounting.utime_sspurr, "llx");
-	DUMP(p, accounting.steal_time, "llx");
+	DUMP(p, accounting.utime, "%#-*lx");
+	DUMP(p, accounting.stime, "%#-*lx");
+	DUMP(p, accounting.utime_scaled, "%#-*lx");
+	DUMP(p, accounting.starttime, "%#-*lx");
+	DUMP(p, accounting.starttime_user, "%#-*lx");
+	DUMP(p, accounting.startspurr, "%#-*lx");
+	DUMP(p, accounting.utime_sspurr, "%#-*lx");
+	DUMP(p, accounting.steal_time, "%#-*lx");
 #undef DUMP
 
 	catch_memory_errors = 0;
@@ -2578,7 +2590,7 @@ static void dump_by_size(unsigned long addr, long count, int size)
 			default: val = 0;
 			}
 
-			printf("%0*lx", size * 2, val);
+			printf("%0*llx", size * 2, val);
 		}
 		printf("\n");
 	}
@@ -2742,7 +2754,7 @@ generic_inst_dump(unsigned long adr, long count, int praddr,
 		dotted = 0;
 		last_inst = inst;
 		if (praddr)
-			printf(REG"  %.8x", adr, inst);
+			printf(REG"  %.8lx", adr, inst);
 		printf("\t");
 		dump_func(inst, adr);
 		printf("\n");
@@ -2874,7 +2886,7 @@ memdiffs(unsigned char *p1, unsigned char *p2, unsigned nb, unsigned maxpr)
 	for( n = nb; n > 0; --n )
 		if( *p1++ != *p2++ )
 			if( ++prt <= maxpr )
-				printf("%.16x %.2x # %.16x %.2x\n", p1 - 1,
+				printf("%px %.2x # %px %.2x\n", p1 - 1,
 					p1[-1], p2 - 1, p2[-1]);
 	if( prt > maxpr )
 		printf("Total of %d differences\n", prt);
@@ -2934,13 +2946,13 @@ memzcan(void)
 		if (ok && !ook) {
 			printf("%.8x .. ", a);
 		} else if (!ok && ook)
-			printf("%.8x\n", a - mskip);
+			printf("%.8lx\n", a - mskip);
 		ook = ok;
 		if (a + mskip < a)
 			break;
 	}
 	if (ook)
-		printf("%.8x\n", a - mskip);
+		printf("%.8lx\n", a - mskip);
 }
 
 static void show_task(struct task_struct *tsk)
@@ -3024,13 +3036,13 @@ static void show_pte(unsigned long addr)
 		return;
 	}
 
-	printf("pgd  @ 0x%016lx\n", pgdir);
+	printf("pgd  @ 0x%px\n", pgdir);
 
 	if (pgd_huge(*pgdp)) {
 		format_pte(pgdp, pgd_val(*pgdp));
 		return;
 	}
-	printf("pgdp @ 0x%016lx = 0x%016lx\n", pgdp, pgd_val(*pgdp));
+	printf("pgdp @ 0x%px = 0x%016lx\n", pgdp, pgd_val(*pgdp));
 
 	pudp = pud_offset(pgdp, addr);
 
@@ -3044,7 +3056,7 @@ static void show_pte(unsigned long addr)
 		return;
 	}
 
-	printf("pudp @ 0x%016lx = 0x%016lx\n", pudp, pud_val(*pudp));
+	printf("pudp @ 0x%px = 0x%016lx\n", pudp, pud_val(*pudp));
 
 	pmdp = pmd_offset(pudp, addr);
 
@@ -3057,7 +3069,7 @@ static void show_pte(unsigned long addr)
 		format_pte(pmdp, pmd_val(*pmdp));
 		return;
 	}
-	printf("pmdp @ 0x%016lx = 0x%016lx\n", pmdp, pmd_val(*pmdp));
+	printf("pmdp @ 0x%px = 0x%016lx\n", pmdp, pmd_val(*pmdp));
 
 	ptep = pte_offset_map(pmdp, addr);
 	if (pte_none(*ptep)) {
@@ -3456,9 +3468,9 @@ static void dump_tlb_44x(void)
 		asm volatile("tlbre  %0,%1,0" : "=r" (w0) : "r" (i));
 		asm volatile("tlbre  %0,%1,1" : "=r" (w1) : "r" (i));
 		asm volatile("tlbre  %0,%1,2" : "=r" (w2) : "r" (i));
-		printf("[%02x] %08x %08x %08x ", i, w0, w1, w2);
+		printf("[%02x] %08lx %08lx %08lx ", i, w0, w1, w2);
 		if (w0 & PPC44x_TLB_VALID) {
-			printf("V %08x -> %01x%08x %c%c%c%c%c",
+			printf("V %08lx -> %01lx%08lx %c%c%c%c%c",
 			       w0 & PPC44x_TLB_EPN_MASK,
 			       w1 & PPC44x_TLB_ERPN_MASK,
 			       w1 & PPC44x_TLB_RPN_MASK,
@@ -3882,19 +3894,19 @@ static void dump_spu_fields(struct spu *spu)
 	DUMP_FIELD(spu, "0x%lx", ls_size);
 	DUMP_FIELD(spu, "0x%x", node);
 	DUMP_FIELD(spu, "0x%lx", flags);
-	DUMP_FIELD(spu, "%d", class_0_pending);
-	DUMP_FIELD(spu, "0x%lx", class_0_dar);
-	DUMP_FIELD(spu, "0x%lx", class_1_dar);
-	DUMP_FIELD(spu, "0x%lx", class_1_dsisr);
-	DUMP_FIELD(spu, "0x%lx", irqs[0]);
-	DUMP_FIELD(spu, "0x%lx", irqs[1]);
-	DUMP_FIELD(spu, "0x%lx", irqs[2]);
+	DUMP_FIELD(spu, "%llu", class_0_pending);
+	DUMP_FIELD(spu, "0x%llx", class_0_dar);
+	DUMP_FIELD(spu, "0x%llx", class_1_dar);
+	DUMP_FIELD(spu, "0x%llx", class_1_dsisr);
+	DUMP_FIELD(spu, "0x%x", irqs[0]);
+	DUMP_FIELD(spu, "0x%x", irqs[1]);
+	DUMP_FIELD(spu, "0x%x", irqs[2]);
 	DUMP_FIELD(spu, "0x%x", slb_replace);
 	DUMP_FIELD(spu, "%d", pid);
 	DUMP_FIELD(spu, "0x%p", mm);
 	DUMP_FIELD(spu, "0x%p", ctx);
 	DUMP_FIELD(spu, "0x%p", rq);
-	DUMP_FIELD(spu, "0x%p", timestamp);
+	DUMP_FIELD(spu, "0x%llx", timestamp);
 	DUMP_FIELD(spu, "0x%lx", problem_phys);
 	DUMP_FIELD(spu, "0x%p", problem);
 	DUMP_VALUE("0x%x", problem->spu_runcntl_RW,
@@ -3925,7 +3937,7 @@ static void dump_spu_ls(unsigned long num, int subcmd)
 		__delay(200);
 	} else {
 		catch_memory_errors = 0;
-		printf("*** Error: accessing spu info for spu %d\n", num);
+		printf("*** Error: accessing spu info for spu %ld\n", num);
 		return;
 	}
 	catch_memory_errors = 0;
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index cd4fd85fde84..42e581a268e1 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -3,8 +3,16 @@
 # see Documentation/kbuild/kconfig-language.txt.
 #
 
+config 64BIT
+	bool
+
+config 32BIT
+	bool
+
 config RISCV
 	def_bool y
+	# even on 32-bit, physical (and DMA) addresses are > 32-bits
+	select PHYS_ADDR_T_64BIT
 	select OF
 	select OF_EARLY_FLATTREE
 	select OF_IRQ
@@ -22,7 +30,6 @@ config RISCV
 	select GENERIC_ATOMIC64 if !64BIT || !RISCV_ISA_A
 	select HAVE_MEMBLOCK
 	select HAVE_MEMBLOCK_NODE_MAP
-	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_CONTIGUOUS
 	select HAVE_GENERIC_DMA_COHERENT
 	select IRQ_DOMAIN
@@ -35,20 +42,14 @@ config RISCV
 	select THREAD_INFO_IN_TASK
 	select RISCV_TIMER
 	select GENERIC_IRQ_MULTI_HANDLER
+	select ARCH_HAS_PTE_SPECIAL
 
 config MMU
 	def_bool y
 
-# even on 32-bit, physical (and DMA) addresses are > 32-bits
-config ARCH_PHYS_ADDR_T_64BIT
-	def_bool y
-
 config ZONE_DMA32
 	bool
-	default y
-
-config ARCH_DMA_ADDR_T_64BIT
-	def_bool y
+	default y if 64BIT
 
 config PAGE_OFFSET
 	hex
@@ -101,21 +102,20 @@ choice
 
 config ARCH_RV32I
 	bool "RV32I"
-	select CPU_SUPPORTS_32BIT_KERNEL
 	select 32BIT
-	select GENERIC_ASHLDI3
-	select GENERIC_ASHRDI3
-	select GENERIC_LSHRDI3
+	select GENERIC_LIB_ASHLDI3
+	select GENERIC_LIB_ASHRDI3
+	select GENERIC_LIB_LSHRDI3
 
 config ARCH_RV64I
 	bool "RV64I"
-	select CPU_SUPPORTS_64BIT_KERNEL
 	select 64BIT
 	select HAVE_FUNCTION_TRACER
 	select HAVE_FUNCTION_GRAPH_TRACER
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_DYNAMIC_FTRACE_WITH_REGS
+	select SWIOTLB
 
 endchoice
 
@@ -171,11 +171,6 @@ config NR_CPUS
 	depends on SMP
 	default "8"
 
-config CPU_SUPPORTS_32BIT_KERNEL
-	bool
-config CPU_SUPPORTS_64BIT_KERNEL
-	bool
-
 choice
 	prompt "CPU Tuning"
 	default TUNE_GENERIC
@@ -202,24 +197,6 @@ endmenu
 
 menu "Kernel type"
 
-choice
-	prompt "Kernel code model"
-	default 64BIT
-
-config 32BIT
-	bool "32-bit kernel"
-	depends on CPU_SUPPORTS_32BIT_KERNEL
-	help
-	  Select this option to build a 32-bit kernel.
-
-config 64BIT
-	bool "64-bit kernel"
-	depends on CPU_SUPPORTS_64BIT_KERNEL
-	help
-	  Select this option to build a 64-bit kernel.
-
-endchoice
-
 source "mm/Kconfig"
 
 source "kernel/Kconfig.preempt"
diff --git a/arch/riscv/include/asm/dma-mapping.h b/arch/riscv/include/asm/dma-mapping.h
new file mode 100644
index 000000000000..8facc1c8fa05
--- /dev/null
+++ b/arch/riscv/include/asm/dma-mapping.h
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+#ifndef _RISCV_ASM_DMA_MAPPING_H
+#define _RISCV_ASM_DMA_MAPPING_H 1
+
+#ifdef CONFIG_SWIOTLB
+#include <linux/swiotlb.h>
+static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
+{
+	return &swiotlb_dma_ops;
+}
+#else
+#include <asm-generic/dma-mapping.h>
+#endif /* CONFIG_SWIOTLB */
+
+#endif /* _RISCV_ASM_DMA_MAPPING_H */
diff --git a/arch/riscv/include/asm/pci.h b/arch/riscv/include/asm/pci.h
index 0f2fc9ef20fc..b3638c505728 100644
--- a/arch/riscv/include/asm/pci.h
+++ b/arch/riscv/include/asm/pci.h
@@ -26,9 +26,6 @@
 /* RISC-V shim does not initialize PCI bus */
 #define pcibios_assign_all_busses() 1
 
-/* We do not have an IOMMU */
-#define PCI_DMA_BUS_IS_PHYS 1
-
 extern int isa_dma_bridge_buggy;
 
 #ifdef CONFIG_PCI
diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h
index 997ddbb1d370..2fa2942be221 100644
--- a/arch/riscv/include/asm/pgtable-bits.h
+++ b/arch/riscv/include/asm/pgtable-bits.h
@@ -42,7 +42,4 @@
 					  _PAGE_WRITE | _PAGE_EXEC |	\
 					  _PAGE_USER | _PAGE_GLOBAL))
 
-/* Advertise support for _PAGE_SPECIAL */
-#define __HAVE_ARCH_PTE_SPECIAL
-
 #endif /* _ASM_RISCV_PGTABLE_BITS_H */
diff --git a/arch/riscv/kernel/cacheinfo.c b/arch/riscv/kernel/cacheinfo.c
index 10ed2749e246..0bc86e5f8f3f 100644
--- a/arch/riscv/kernel/cacheinfo.c
+++ b/arch/riscv/kernel/cacheinfo.c
@@ -20,7 +20,6 @@ static void ci_leaf_init(struct cacheinfo *this_leaf,
 			 struct device_node *node,
 			 enum cache_type type, unsigned int level)
 {
-	this_leaf->of_node = node;
 	this_leaf->level = level;
 	this_leaf->type = type;
 	/* not a sector cache */
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index c11f40c1b2a8..ee44a48faf79 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -29,6 +29,7 @@
 #include <linux/of_fdt.h>
 #include <linux/of_platform.h>
 #include <linux/sched/task.h>
+#include <linux/swiotlb.h>
 
 #include <asm/setup.h>
 #include <asm/sections.h>
@@ -206,6 +207,7 @@ void __init setup_arch(char **cmdline_p)
 	setup_bootmem();
 	paging_init();
 	unflatten_device_tree();
+	swiotlb_init(1);
 
 #ifdef CONFIG_SMP
 	setup_smp();
diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
index 93132cb59184..b99d9dd21fd0 100644
--- a/arch/riscv/kernel/traps.c
+++ b/arch/riscv/kernel/traps.c
@@ -63,18 +63,6 @@ void die(struct pt_regs *regs, const char *str)
 		do_exit(SIGSEGV);
 }
 
-static inline void do_trap_siginfo(int signo, int code,
-	unsigned long addr, struct task_struct *tsk)
-{
-	siginfo_t info;
-
-	info.si_signo = signo;
-	info.si_errno = 0;
-	info.si_code = code;
-	info.si_addr = (void __user *)addr;
-	force_sig_info(signo, &info, tsk);
-}
-
 void do_trap(struct pt_regs *regs, int signo, int code,
 	unsigned long addr, struct task_struct *tsk)
 {
@@ -87,7 +75,7 @@ void do_trap(struct pt_regs *regs, int signo, int code,
 		show_regs(regs);
 	}
 
-	do_trap_siginfo(signo, code, addr, tsk);
+	force_sig_fault(signo, code, (void __user *)addr, tsk);
 }
 
 static void do_trap_error(struct pt_regs *regs, int signo, int code,
@@ -149,7 +137,7 @@ asmlinkage void do_trap_break(struct pt_regs *regs)
 	}
 #endif /* CONFIG_GENERIC_BUG */
 
-	do_trap_siginfo(SIGTRAP, TRAP_BRKPT, regs->sepc, current);
+	force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)(regs->sepc), current);
 	regs->sepc += 0x4;
 }
 
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 199ac3e4da1d..baed39772c84 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -35,9 +35,6 @@ config GENERIC_BUG
 config GENERIC_BUG_RELATIVE_POINTERS
 	def_bool y
 
-config ARCH_DMA_ADDR_T_64BIT
-	def_bool y
-
 config GENERIC_LOCKBREAK
 	def_bool y if SMP && PREEMPT
 
@@ -68,6 +65,7 @@ config S390
 	select ARCH_HAS_GCOV_PROFILE_ALL
 	select ARCH_HAS_GIGANTIC_PAGE if (MEMORY_ISOLATION && COMPACTION) || CMA
 	select ARCH_HAS_KCOV
+	select ARCH_HAS_PTE_SPECIAL
 	select ARCH_HAS_SET_MEMORY
 	select ARCH_HAS_SG_CHAIN
 	select ARCH_HAS_STRICT_KERNEL_RWX
@@ -133,7 +131,6 @@ config S390
 	select HAVE_CMPXCHG_LOCAL
 	select HAVE_COPY_THREAD_TLS
 	select HAVE_DEBUG_KMEMLEAK
-	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_CONTIGUOUS
 	select DMA_DIRECT_OPS
 	select HAVE_DYNAMIC_FTRACE
@@ -709,7 +706,11 @@ config QDIO
 menuconfig PCI
 	bool "PCI support"
 	select PCI_MSI
+	select IOMMU_HELPER
 	select IOMMU_SUPPORT
+	select NEED_DMA_MAP_STATE
+	select NEED_SG_DMA_LENGTH
+
 	help
 	  Enable PCI support.
 
@@ -733,15 +734,6 @@ config PCI_DOMAINS
 config HAS_IOMEM
 	def_bool PCI
 
-config IOMMU_HELPER
-	def_bool PCI
-
-config NEED_SG_DMA_LENGTH
-	def_bool PCI
-
-config NEED_DMA_MAP_STATE
-	def_bool PCI
-
 config CHSC_SCH
 	def_tristate m
 	prompt "Support for CHSC subchannels"
@@ -847,6 +839,10 @@ config CCW
 
 source "drivers/Kconfig"
 
+config HAVE_PNETID
+	tristate
+	default (SMC || CCWGROUP)
+
 source "fs/Kconfig"
 
 source "arch/s390/Kconfig.debug"
diff --git a/arch/s390/Makefile b/arch/s390/Makefile
index c79936d02f7b..68a690442be0 100644
--- a/arch/s390/Makefile
+++ b/arch/s390/Makefile
@@ -18,7 +18,7 @@ KBUILD_CFLAGS	+= -m64
 KBUILD_AFLAGS	+= -m64
 UTS_MACHINE	:= s390x
 STACK_SIZE	:= 16384
-CHECKFLAGS	+= -D__s390__ -D__s390x__ -mbig-endian
+CHECKFLAGS	+= -D__s390__ -D__s390x__
 
 export LD_BFD
 
diff --git a/arch/s390/crypto/arch_random.c b/arch/s390/crypto/arch_random.c
index 8720e9203ecf..dd95cdbd22ce 100644
--- a/arch/s390/crypto/arch_random.c
+++ b/arch/s390/crypto/arch_random.c
@@ -2,14 +2,37 @@
 /*
  * s390 arch random implementation.
  *
- * Copyright IBM Corp. 2017
- * Author(s): Harald Freudenberger <freude@de.ibm.com>
+ * Copyright IBM Corp. 2017, 2018
+ * Author(s): Harald Freudenberger
+ *
+ * The s390_arch_random_generate() function may be called from random.c
+ * in interrupt context. So this implementation does the best to be very
+ * fast. There is a buffer of random data which is asynchronously checked
+ * and filled by a workqueue thread.
+ * If there are enough bytes in the buffer the s390_arch_random_generate()
+ * just delivers these bytes. Otherwise false is returned until the
+ * worker thread refills the buffer.
+ * The worker fills the rng buffer by pulling fresh entropy from the
+ * high quality (but slow) true hardware random generator. This entropy
+ * is then spread over the buffer with an pseudo random generator PRNG.
+ * As the arch_get_random_seed_long() fetches 8 bytes and the calling
+ * function add_interrupt_randomness() counts this as 1 bit entropy the
+ * distribution needs to make sure there is in fact 1 bit entropy contained
+ * in 8 bytes of the buffer. The current values pull 32 byte entropy
+ * and scatter this into a 2048 byte buffer. So 8 byte in the buffer
+ * will contain 1 bit of entropy.
+ * The worker thread is rescheduled based on the charge level of the
+ * buffer but at least with 500 ms delay to avoid too much CPU consumption.
+ * So the max. amount of rng data delivered via arch_get_random_seed is
+ * limited to 4k bytes per second.
  */
 
 #include <linux/kernel.h>
 #include <linux/atomic.h>
 #include <linux/random.h>
+#include <linux/slab.h>
 #include <linux/static_key.h>
+#include <linux/workqueue.h>
 #include <asm/cpacf.h>
 
 DEFINE_STATIC_KEY_FALSE(s390_arch_random_available);
@@ -17,11 +40,83 @@ DEFINE_STATIC_KEY_FALSE(s390_arch_random_available);
 atomic64_t s390_arch_random_counter = ATOMIC64_INIT(0);
 EXPORT_SYMBOL(s390_arch_random_counter);
 
+#define ARCH_REFILL_TICKS (HZ/2)
+#define ARCH_PRNG_SEED_SIZE 32
+#define ARCH_RNG_BUF_SIZE 2048
+
+static DEFINE_SPINLOCK(arch_rng_lock);
+static u8 *arch_rng_buf;
+static unsigned int arch_rng_buf_idx;
+
+static void arch_rng_refill_buffer(struct work_struct *);
+static DECLARE_DELAYED_WORK(arch_rng_work, arch_rng_refill_buffer);
+
+bool s390_arch_random_generate(u8 *buf, unsigned int nbytes)
+{
+	/* lock rng buffer */
+	if (!spin_trylock(&arch_rng_lock))
+		return false;
+
+	/* try to resolve the requested amount of bytes from the buffer */
+	arch_rng_buf_idx -= nbytes;
+	if (arch_rng_buf_idx < ARCH_RNG_BUF_SIZE) {
+		memcpy(buf, arch_rng_buf + arch_rng_buf_idx, nbytes);
+		atomic64_add(nbytes, &s390_arch_random_counter);
+		spin_unlock(&arch_rng_lock);
+		return true;
+	}
+
+	/* not enough bytes in rng buffer, refill is done asynchronously */
+	spin_unlock(&arch_rng_lock);
+
+	return false;
+}
+EXPORT_SYMBOL(s390_arch_random_generate);
+
+static void arch_rng_refill_buffer(struct work_struct *unused)
+{
+	unsigned int delay = ARCH_REFILL_TICKS;
+
+	spin_lock(&arch_rng_lock);
+	if (arch_rng_buf_idx > ARCH_RNG_BUF_SIZE) {
+		/* buffer is exhausted and needs refill */
+		u8 seed[ARCH_PRNG_SEED_SIZE];
+		u8 prng_wa[240];
+		/* fetch ARCH_PRNG_SEED_SIZE bytes of entropy */
+		cpacf_trng(NULL, 0, seed, sizeof(seed));
+		/* blow this entropy up to ARCH_RNG_BUF_SIZE with PRNG */
+		memset(prng_wa, 0, sizeof(prng_wa));
+		cpacf_prno(CPACF_PRNO_SHA512_DRNG_SEED,
+			   &prng_wa, NULL, 0, seed, sizeof(seed));
+		cpacf_prno(CPACF_PRNO_SHA512_DRNG_GEN,
+			   &prng_wa, arch_rng_buf, ARCH_RNG_BUF_SIZE, NULL, 0);
+		arch_rng_buf_idx = ARCH_RNG_BUF_SIZE;
+	}
+	delay += (ARCH_REFILL_TICKS * arch_rng_buf_idx) / ARCH_RNG_BUF_SIZE;
+	spin_unlock(&arch_rng_lock);
+
+	/* kick next check */
+	queue_delayed_work(system_long_wq, &arch_rng_work, delay);
+}
+
 static int __init s390_arch_random_init(void)
 {
-	/* check if subfunction CPACF_PRNO_TRNG is available */
-	if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG))
+	/* all the needed PRNO subfunctions available ? */
+	if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG) &&
+	    cpacf_query_func(CPACF_PRNO, CPACF_PRNO_SHA512_DRNG_GEN)) {
+
+		/* alloc arch random working buffer */
+		arch_rng_buf = kmalloc(ARCH_RNG_BUF_SIZE, GFP_KERNEL);
+		if (!arch_rng_buf)
+			return -ENOMEM;
+
+		/* kick worker queue job to fill the random buffer */
+		queue_delayed_work(system_long_wq,
+				   &arch_rng_work, ARCH_REFILL_TICKS);
+
+		/* enable arch random to the outside world */
 		static_branch_enable(&s390_arch_random_available);
+	}
 
 	return 0;
 }
diff --git a/arch/s390/hypfs/hypfs_sprp.c b/arch/s390/hypfs/hypfs_sprp.c
index ae0ed8dd5f1b..5d85a039391c 100644
--- a/arch/s390/hypfs/hypfs_sprp.c
+++ b/arch/s390/hypfs/hypfs_sprp.c
@@ -13,7 +13,6 @@
 #include <linux/string.h>
 #include <linux/types.h>
 #include <linux/uaccess.h>
-#include <asm/compat.h>
 #include <asm/diag.h>
 #include <asm/sclp.h>
 #include "hypfs.h"
diff --git a/arch/s390/include/asm/archrandom.h b/arch/s390/include/asm/archrandom.h
index 09aed1095336..c67b82dfa558 100644
--- a/arch/s390/include/asm/archrandom.h
+++ b/arch/s390/include/asm/archrandom.h
@@ -15,16 +15,11 @@
 
 #include <linux/static_key.h>
 #include <linux/atomic.h>
-#include <asm/cpacf.h>
 
 DECLARE_STATIC_KEY_FALSE(s390_arch_random_available);
 extern atomic64_t s390_arch_random_counter;
 
-static void s390_arch_random_generate(u8 *buf, unsigned int nbytes)
-{
-	cpacf_trng(NULL, 0, buf, nbytes);
-	atomic64_add(nbytes, &s390_arch_random_counter);
-}
+bool s390_arch_random_generate(u8 *buf, unsigned int nbytes);
 
 static inline bool arch_has_random(void)
 {
@@ -51,8 +46,7 @@ static inline bool arch_get_random_int(unsigned int *v)
 static inline bool arch_get_random_seed_long(unsigned long *v)
 {
 	if (static_branch_likely(&s390_arch_random_available)) {
-		s390_arch_random_generate((u8 *)v, sizeof(*v));
-		return true;
+		return s390_arch_random_generate((u8 *)v, sizeof(*v));
 	}
 	return false;
 }
@@ -60,8 +54,7 @@ static inline bool arch_get_random_seed_long(unsigned long *v)
 static inline bool arch_get_random_seed_int(unsigned int *v)
 {
 	if (static_branch_likely(&s390_arch_random_available)) {
-		s390_arch_random_generate((u8 *)v, sizeof(*v));
-		return true;
+		return s390_arch_random_generate((u8 *)v, sizeof(*v));
 	}
 	return false;
 }
diff --git a/arch/s390/include/asm/ccwdev.h b/arch/s390/include/asm/ccwdev.h
index 20bce136b2e5..a29dd430fb40 100644
--- a/arch/s390/include/asm/ccwdev.h
+++ b/arch/s390/include/asm/ccwdev.h
@@ -231,4 +231,5 @@ int ccw_device_siosl(struct ccw_device *);
 extern void ccw_device_get_schid(struct ccw_device *, struct subchannel_id *);
 
 struct channel_path_desc_fmt0 *ccw_device_get_chp_desc(struct ccw_device *, int);
+u8 *ccw_device_get_util_str(struct ccw_device *cdev, int chp_idx);
 #endif /* _S390_CCWDEV_H_ */
diff --git a/arch/s390/include/asm/ccwgroup.h b/arch/s390/include/asm/ccwgroup.h
index 99aa817dad32..860cab7479c3 100644
--- a/arch/s390/include/asm/ccwgroup.h
+++ b/arch/s390/include/asm/ccwgroup.h
@@ -73,4 +73,14 @@ extern void ccwgroup_remove_ccwdev(struct ccw_device *cdev);
 
 #define to_ccwgroupdev(x) container_of((x), struct ccwgroup_device, dev)
 #define to_ccwgroupdrv(x) container_of((x), struct ccwgroup_driver, driver)
+
+#if IS_ENABLED(CONFIG_CCWGROUP)
+bool dev_is_ccwgroup(struct device *dev);
+#else /* CONFIG_CCWGROUP */
+static inline bool dev_is_ccwgroup(struct device *dev)
+{
+	return false;
+}
+#endif /* CONFIG_CCWGROUP */
+
 #endif
diff --git a/arch/s390/include/asm/compat.h b/arch/s390/include/asm/compat.h
index 9830fb6b076e..97db2fba546a 100644
--- a/arch/s390/include/asm/compat.h
+++ b/arch/s390/include/asm/compat.h
@@ -53,7 +53,6 @@
 
 typedef u32		compat_size_t;
 typedef s32		compat_ssize_t;
-typedef s32		compat_time_t;
 typedef s32		compat_clock_t;
 typedef s32		compat_pid_t;
 typedef u16		__compat_uid_t;
@@ -97,16 +96,6 @@ typedef struct {
 	u32 gprs_high[NUM_GPRS];
 } s390_compat_regs_high;
 
-struct compat_timespec {
-	compat_time_t	tv_sec;
-	s32		tv_nsec;
-};
-
-struct compat_timeval {
-	compat_time_t	tv_sec;
-	s32		tv_usec;
-};
-
 struct compat_stat {
 	compat_dev_t	st_dev;
 	u16		__pad1;
@@ -243,10 +232,10 @@ struct compat_ipc64_perm {
 
 struct compat_semid64_ds {
 	struct compat_ipc64_perm sem_perm;
-	compat_time_t  sem_otime;
-	compat_ulong_t __pad1;
-	compat_time_t  sem_ctime;
-	compat_ulong_t __pad2;
+	compat_ulong_t sem_otime;
+	compat_ulong_t sem_otime_high;
+	compat_ulong_t sem_ctime;
+	compat_ulong_t sem_ctime_high;
 	compat_ulong_t sem_nsems;
 	compat_ulong_t __unused1;
 	compat_ulong_t __unused2;
@@ -254,12 +243,12 @@ struct compat_semid64_ds {
 
 struct compat_msqid64_ds {
 	struct compat_ipc64_perm msg_perm;
-	compat_time_t   msg_stime;
-	compat_ulong_t __pad1;
-	compat_time_t   msg_rtime;
-	compat_ulong_t __pad2;
-	compat_time_t   msg_ctime;
-	compat_ulong_t __pad3;
+	compat_ulong_t msg_stime;
+	compat_ulong_t msg_stime_high;
+	compat_ulong_t msg_rtime;
+	compat_ulong_t msg_rtime_high;
+	compat_ulong_t msg_ctime;
+	compat_ulong_t msg_ctime_high;
 	compat_ulong_t msg_cbytes;
 	compat_ulong_t msg_qnum;
 	compat_ulong_t msg_qbytes;
@@ -272,12 +261,12 @@ struct compat_msqid64_ds {
 struct compat_shmid64_ds {
 	struct compat_ipc64_perm shm_perm;
 	compat_size_t  shm_segsz;
-	compat_time_t  shm_atime;
-	compat_ulong_t __pad1;
-	compat_time_t  shm_dtime;
-	compat_ulong_t __pad2;
-	compat_time_t  shm_ctime;
-	compat_ulong_t __pad3;
+	compat_ulong_t shm_atime;
+	compat_ulong_t shm_atime_high;
+	compat_ulong_t shm_dtime;
+	compat_ulong_t shm_dtime_high;
+	compat_ulong_t shm_ctime;
+	compat_ulong_t shm_ctime_high;
 	compat_pid_t   shm_cpid;
 	compat_pid_t   shm_lpid;
 	compat_ulong_t shm_nattch;
diff --git a/arch/s390/include/asm/cpu_mf.h b/arch/s390/include/asm/cpu_mf.h
index f58d17e9dd65..de023a9a88ca 100644
--- a/arch/s390/include/asm/cpu_mf.h
+++ b/arch/s390/include/asm/cpu_mf.h
@@ -113,7 +113,7 @@ struct hws_basic_entry {
 
 struct hws_diag_entry {
 	unsigned int def:16;	    /* 0-15  Data Entry Format		 */
-	unsigned int R:14;	    /* 16-19 and 20-30 reserved		 */
+	unsigned int R:15;	    /* 16-19 and 20-30 reserved		 */
 	unsigned int I:1;	    /* 31 entry valid or invalid	 */
 	u8	     data[];	    /* Machine-dependent sample data	 */
 } __packed;
@@ -129,7 +129,9 @@ struct hws_trailer_entry {
 			unsigned int f:1;	/* 0 - Block Full Indicator   */
 			unsigned int a:1;	/* 1 - Alert request control  */
 			unsigned int t:1;	/* 2 - Timestamp format	      */
-			unsigned long long:61;	/* 3 - 63: Reserved	      */
+			unsigned int :29;	/* 3 - 31: Reserved	      */
+			unsigned int bsdes:16;	/* 32-47: size of basic SDE   */
+			unsigned int dsdes:16;	/* 48-63: size of diagnostic SDE */
 		};
 		unsigned long long flags;	/* 0 - 63: All indicators     */
 	};
diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctl_reg.h
index 99c93d0346f9..4600453536c2 100644
--- a/arch/s390/include/asm/ctl_reg.h
+++ b/arch/s390/include/asm/ctl_reg.h
@@ -10,8 +10,20 @@
 
 #include <linux/const.h>
 
+#define CR0_CLOCK_COMPARATOR_SIGN	_BITUL(63 - 10)
+#define CR0_EMERGENCY_SIGNAL_SUBMASK	_BITUL(63 - 49)
+#define CR0_EXTERNAL_CALL_SUBMASK	_BITUL(63 - 50)
+#define CR0_CLOCK_COMPARATOR_SUBMASK	_BITUL(63 - 52)
+#define CR0_CPU_TIMER_SUBMASK		_BITUL(63 - 53)
+#define CR0_SERVICE_SIGNAL_SUBMASK	_BITUL(63 - 54)
+#define CR0_UNUSED_56			_BITUL(63 - 56)
+#define CR0_INTERRUPT_KEY_SUBMASK	_BITUL(63 - 57)
+#define CR0_MEASUREMENT_ALERT_SUBMASK	_BITUL(63 - 58)
+
 #define CR2_GUARDED_STORAGE		_BITUL(63 - 59)
 
+#define CR14_UNUSED_32			_BITUL(63 - 32)
+#define CR14_UNUSED_33			_BITUL(63 - 33)
 #define CR14_CHANNEL_REPORT_SUBMASK	_BITUL(63 - 35)
 #define CR14_RECOVERY_SUBMASK		_BITUL(63 - 36)
 #define CR14_DEGRADATION_SUBMASK	_BITUL(63 - 37)
diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h
index 1a61b1b997f2..7d22a474a040 100644
--- a/arch/s390/include/asm/elf.h
+++ b/arch/s390/include/asm/elf.h
@@ -125,8 +125,9 @@
  * ELF register definitions..
  */
 
+#include <linux/compat.h>
+
 #include <asm/ptrace.h>
-#include <asm/compat.h>
 #include <asm/syscall.h>
 #include <asm/user.h>
 
@@ -136,7 +137,6 @@ typedef s390_regs elf_gregset_t;
 typedef s390_fp_regs compat_elf_fpregset_t;
 typedef s390_compat_regs compat_elf_gregset_t;
 
-#include <linux/compat.h>
 #include <linux/sched/mm.h>	/* for task_struct */
 #include <asm/mmu_context.h>
 
diff --git a/arch/s390/include/asm/hardirq.h b/arch/s390/include/asm/hardirq.h
index a296c6acfd07..dfbc3c6c0674 100644
--- a/arch/s390/include/asm/hardirq.h
+++ b/arch/s390/include/asm/hardirq.h
@@ -14,6 +14,8 @@
 #include <asm/lowcore.h>
 
 #define local_softirq_pending() (S390_lowcore.softirq_pending)
+#define set_softirq_pending(x) (S390_lowcore.softirq_pending = (x))
+#define or_softirq_pending(x)  (S390_lowcore.softirq_pending |= (x))
 
 #define __ARCH_IRQ_STAT
 #define __ARCH_HAS_DO_SOFTIRQ
diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index 81cdb6b55118..a2188e309bd6 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -812,6 +812,7 @@ struct kvm_arch{
 	int use_irqchip;
 	int use_cmma;
 	int use_pfmfi;
+	int use_skf;
 	int user_cpu_state_ctrl;
 	int user_sigp;
 	int user_stsi;
diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h
index c639c95850e4..f5ff9dbad8ac 100644
--- a/arch/s390/include/asm/mmu.h
+++ b/arch/s390/include/asm/mmu.h
@@ -21,7 +21,7 @@ typedef struct {
 	/* The mmu context uses extended page tables. */
 	unsigned int has_pgste:1;
 	/* The mmu context uses storage keys. */
-	unsigned int use_skey:1;
+	unsigned int uses_skeys:1;
 	/* The mmu context uses CMM. */
 	unsigned int uses_cmm:1;
 } mm_context_t;
diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h
index 324f6f452982..d16bc79c30bb 100644
--- a/arch/s390/include/asm/mmu_context.h
+++ b/arch/s390/include/asm/mmu_context.h
@@ -30,7 +30,7 @@ static inline int init_new_context(struct task_struct *tsk,
 		test_thread_flag(TIF_PGSTE) ||
 		(current->mm && current->mm->context.alloc_pgste);
 	mm->context.has_pgste = 0;
-	mm->context.use_skey = 0;
+	mm->context.uses_skeys = 0;
 	mm->context.uses_cmm = 0;
 #endif
 	switch (mm->context.asce_limit) {
diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
index 12fe3591034f..94f8db468c9b 100644
--- a/arch/s390/include/asm/pci.h
+++ b/arch/s390/include/asm/pci.h
@@ -2,8 +2,6 @@
 #ifndef __ASM_S390_PCI_H
 #define __ASM_S390_PCI_H
 
-/* must be set before including asm-generic/pci.h */
-#define PCI_DMA_BUS_IS_PHYS (0)
 /* must be set before including pci_clp.h */
 #define PCI_BAR_COUNT	6
 
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 2d24d33bf188..5ab636089c60 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -171,7 +171,6 @@ static inline int is_module_addr(void *addr)
 #define _PAGE_WRITE	0x020		/* SW pte write bit */
 #define _PAGE_SPECIAL	0x040		/* SW associated with special page */
 #define _PAGE_UNUSED	0x080		/* SW bit for pgste usage state */
-#define __HAVE_ARCH_PTE_SPECIAL
 
 #ifdef CONFIG_MEM_SOFT_DIRTY
 #define _PAGE_SOFT_DIRTY 0x002		/* SW pte soft dirty bit */
@@ -507,10 +506,10 @@ static inline int mm_alloc_pgste(struct mm_struct *mm)
  * faults should no longer be backed by zero pages
  */
 #define mm_forbids_zeropage mm_has_pgste
-static inline int mm_use_skey(struct mm_struct *mm)
+static inline int mm_uses_skeys(struct mm_struct *mm)
 {
 #ifdef CONFIG_PGSTE
-	if (mm->context.use_skey)
+	if (mm->context.uses_skeys)
 		return 1;
 #endif
 	return 0;
diff --git a/arch/s390/include/asm/pnet.h b/arch/s390/include/asm/pnet.h
new file mode 100644
index 000000000000..6e278584f8f1
--- /dev/null
+++ b/arch/s390/include/asm/pnet.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *  IBM System z PNET ID Support
+ *
+ *    Copyright IBM Corp. 2018
+ */
+
+#ifndef _ASM_S390_PNET_H
+#define _ASM_S390_PNET_H
+
+#include <linux/device.h>
+#include <linux/types.h>
+
+#define PNETIDS_LEN		64	/* Total utility string length in bytes
+					 * to cover up to 4 PNETIDs of 16 bytes
+					 * for up to 4 device ports
+					 */
+#define MAX_PNETID_LEN		16	/* Max.length of a single port PNETID */
+#define MAX_PNETID_PORTS	(PNETIDS_LEN / MAX_PNETID_LEN)
+					/* Max. # of ports with a PNETID */
+
+int pnet_id_by_dev_port(struct device *dev, unsigned short port, u8 *pnetid);
+#endif /* _ASM_S390_PNET_H */
diff --git a/arch/s390/include/uapi/asm/Kbuild b/arch/s390/include/uapi/asm/Kbuild
index faef3f7e8353..e364873e0d10 100644
--- a/arch/s390/include/uapi/asm/Kbuild
+++ b/arch/s390/include/uapi/asm/Kbuild
@@ -9,9 +9,12 @@ generic-y += errno.h
 generic-y += fcntl.h
 generic-y += ioctl.h
 generic-y += mman.h
+generic-y += msgbuf.h
 generic-y += param.h
 generic-y += poll.h
 generic-y += resource.h
+generic-y += sembuf.h
+generic-y += shmbuf.h
 generic-y += sockios.h
 generic-y += swab.h
 generic-y += termbits.h
diff --git a/arch/s390/include/uapi/asm/msgbuf.h b/arch/s390/include/uapi/asm/msgbuf.h
deleted file mode 100644
index 604f847cd68c..000000000000
--- a/arch/s390/include/uapi/asm/msgbuf.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _S390_MSGBUF_H
-#define _S390_MSGBUF_H
-
-/* 
- * The msqid64_ds structure for S/390 architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
- * - 2 miscellaneous 32-bit values
- */
-
-struct msqid64_ds {
-	struct ipc64_perm msg_perm;
-	__kernel_time_t msg_stime;	/* last msgsnd time */
-#ifndef __s390x__
-	unsigned long	__unused1;
-#endif /* ! __s390x__ */
-	__kernel_time_t msg_rtime;	/* last msgrcv time */
-#ifndef __s390x__
-	unsigned long	__unused2;
-#endif /* ! __s390x__ */
-	__kernel_time_t msg_ctime;	/* last change time */
-#ifndef __s390x__
-	unsigned long	__unused3;
-#endif /* ! __s390x__ */
-	unsigned long  msg_cbytes;	/* current number of bytes on queue */
-	unsigned long  msg_qnum;	/* number of messages in queue */
-	unsigned long  msg_qbytes;	/* max number of bytes on queue */
-	__kernel_pid_t msg_lspid;	/* pid of last msgsnd */
-	__kernel_pid_t msg_lrpid;	/* last receive pid */
-	unsigned long  __unused4;
-	unsigned long  __unused5;
-};
-
-#endif /* _S390_MSGBUF_H */
diff --git a/arch/s390/include/uapi/asm/sembuf.h b/arch/s390/include/uapi/asm/sembuf.h
deleted file mode 100644
index 3e917697b668..000000000000
--- a/arch/s390/include/uapi/asm/sembuf.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _S390_SEMBUF_H
-#define _S390_SEMBUF_H
-
-/* 
- * The semid64_ds structure for S/390 architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem (for !__s390x__)
- * - 2 miscellaneous 32-bit values
- */
-
-struct semid64_ds {
-	struct ipc64_perm sem_perm;		/* permissions .. see ipc.h */
-	__kernel_time_t	sem_otime;		/* last semop time */
-#ifndef __s390x__
-	unsigned long	__unused1;
-#endif /* ! __s390x__ */
-	__kernel_time_t	sem_ctime;		/* last change time */
-#ifndef __s390x__
-	unsigned long	__unused2;
-#endif /* ! __s390x__ */
-	unsigned long	sem_nsems;		/* no. of semaphores in array */
-	unsigned long	__unused3;
-	unsigned long	__unused4;
-};
-
-#endif /* _S390_SEMBUF_H */
diff --git a/arch/s390/include/uapi/asm/shmbuf.h b/arch/s390/include/uapi/asm/shmbuf.h
deleted file mode 100644
index 9cdce8d7ce60..000000000000
--- a/arch/s390/include/uapi/asm/shmbuf.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _S390_SHMBUF_H
-#define _S390_SHMBUF_H
-
-/* 
- * The shmid64_ds structure for S/390 architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem (for !__s390x__)
- * - 2 miscellaneous 32-bit values
- */
-
-struct shmid64_ds {
-	struct ipc64_perm	shm_perm;	/* operation perms */
-	size_t			shm_segsz;	/* size of segment (bytes) */
-	__kernel_time_t		shm_atime;	/* last attach time */
-#ifndef __s390x__
-	unsigned long		__unused1;
-#endif /* ! __s390x__ */
-	__kernel_time_t		shm_dtime;	/* last detach time */
-#ifndef __s390x__
-	unsigned long		__unused2;
-#endif /* ! __s390x__ */
-	__kernel_time_t		shm_ctime;	/* last change time */
-#ifndef __s390x__
-	unsigned long		__unused3;
-#endif /* ! __s390x__ */
-	__kernel_pid_t		shm_cpid;	/* pid of creator */
-	__kernel_pid_t		shm_lpid;	/* pid of last operator */
-	unsigned long		shm_nattch;	/* no. of current attaches */
-	unsigned long		__unused4;
-	unsigned long		__unused5;
-};
-
-struct shminfo64 {
-	unsigned long	shmmax;
-	unsigned long	shmmin;
-	unsigned long	shmmni;
-	unsigned long	shmseg;
-	unsigned long	shmall;
-	unsigned long	__unused1;
-	unsigned long	__unused2;
-	unsigned long	__unused3;
-	unsigned long	__unused4;
-};
-
-#endif /* _S390_SHMBUF_H */
diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile
index f92dd8ed3884..2fed39b26b42 100644
--- a/arch/s390/kernel/Makefile
+++ b/arch/s390/kernel/Makefile
@@ -6,22 +6,26 @@
 ifdef CONFIG_FUNCTION_TRACER
 
 # Do not trace tracer code
-CFLAGS_REMOVE_ftrace.o	= $(CC_FLAGS_FTRACE)
+CFLAGS_REMOVE_ftrace.o		= $(CC_FLAGS_FTRACE)
 
 # Do not trace early setup code
-CFLAGS_REMOVE_als.o	= $(CC_FLAGS_FTRACE)
-CFLAGS_REMOVE_early.o	= $(CC_FLAGS_FTRACE)
+CFLAGS_REMOVE_als.o		= $(CC_FLAGS_FTRACE)
+CFLAGS_REMOVE_early.o		= $(CC_FLAGS_FTRACE)
+CFLAGS_REMOVE_early_nobss.o	= $(CC_FLAGS_FTRACE)
 
 endif
 
-GCOV_PROFILE_als.o	:= n
-GCOV_PROFILE_early.o	:= n
+GCOV_PROFILE_als.o		:= n
+GCOV_PROFILE_early.o		:= n
+GCOV_PROFILE_early_nobss.o	:= n
 
-KCOV_INSTRUMENT_als.o	:= n
-KCOV_INSTRUMENT_early.o	:= n
+KCOV_INSTRUMENT_als.o		:= n
+KCOV_INSTRUMENT_early.o		:= n
+KCOV_INSTRUMENT_early_nobss.o	:= n
 
-UBSAN_SANITIZE_als.o	:= n
-UBSAN_SANITIZE_early.o	:= n
+UBSAN_SANITIZE_als.o		:= n
+UBSAN_SANITIZE_early.o		:= n
+UBSAN_SANITIZE_early_nobss.o	:= n
 
 #
 # Use -march=z900 for als.c to be able to print an error
@@ -57,7 +61,7 @@ CFLAGS_ptrace.o		+= -DUTS_MACHINE='"$(UTS_MACHINE)"'
 
 obj-y	:= traps.o time.o process.o base.o early.o setup.o idle.o vtime.o
 obj-y	+= processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o
-obj-y	+= debug.o irq.o ipl.o dis.o diag.o vdso.o als.o
+obj-y	+= debug.o irq.o ipl.o dis.o diag.o vdso.o als.o early_nobss.o
 obj-y	+= sysinfo.o jump_label.o lgr.o os_info.o machine_kexec.o pgm_check.o
 obj-y	+= runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o
 obj-y	+= entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o
@@ -94,3 +98,6 @@ obj-$(CONFIG_TRACEPOINTS)	+= trace.o
 # vdso
 obj-y				+= vdso64/
 obj-$(CONFIG_COMPAT)		+= vdso32/
+
+chkbss := head.o head64.o als.o early_nobss.o
+include $(srctree)/arch/s390/scripts/Makefile.chkbss
diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c
index 32daa0f84325..827699eb48fa 100644
--- a/arch/s390/kernel/early.c
+++ b/arch/s390/kernel/early.c
@@ -33,32 +33,6 @@
 
 static void __init setup_boot_command_line(void);
 
-/*
- * Get the TOD clock running.
- */
-static void __init reset_tod_clock(void)
-{
-	u64 time;
-
-	if (store_tod_clock(&time) == 0)
-		return;
-	/* TOD clock not running. Set the clock to Unix Epoch. */
-	if (set_tod_clock(TOD_UNIX_EPOCH) != 0 || store_tod_clock(&time) != 0)
-		disabled_wait(0);
-
-	memset(tod_clock_base, 0, 16);
-	*(__u64 *) &tod_clock_base[1] = TOD_UNIX_EPOCH;
-	S390_lowcore.last_update_clock = TOD_UNIX_EPOCH;
-}
-
-/*
- * Clear bss memory
- */
-static noinline __init void clear_bss_section(void)
-{
-	memset(__bss_start, 0, __bss_stop - __bss_start);
-}
-
 /*
  * Initialize storage key for kernel pages
  */
@@ -310,57 +284,6 @@ static int __init cad_setup(char *str)
 }
 early_param("cad", cad_setup);
 
-static __init void memmove_early(void *dst, const void *src, size_t n)
-{
-	unsigned long addr;
-	long incr;
-	psw_t old;
-
-	if (!n)
-		return;
-	incr = 1;
-	if (dst > src) {
-		incr = -incr;
-		dst += n - 1;
-		src += n - 1;
-	}
-	old = S390_lowcore.program_new_psw;
-	S390_lowcore.program_new_psw.mask = __extract_psw();
-	asm volatile(
-		"	larl	%[addr],1f\n"
-		"	stg	%[addr],%[psw_pgm_addr]\n"
-		"0:     mvc	0(1,%[dst]),0(%[src])\n"
-		"	agr	%[dst],%[incr]\n"
-		"	agr	%[src],%[incr]\n"
-		"	brctg	%[n],0b\n"
-		"1:\n"
-		: [addr] "=&d" (addr),
-		  [psw_pgm_addr] "=Q" (S390_lowcore.program_new_psw.addr),
-		  [dst] "+&a" (dst), [src] "+&a" (src),  [n] "+d" (n)
-		: [incr] "d" (incr)
-		: "cc", "memory");
-	S390_lowcore.program_new_psw = old;
-}
-
-static __init noinline void rescue_initrd(void)
-{
-#ifdef CONFIG_BLK_DEV_INITRD
-	unsigned long min_initrd_addr = (unsigned long) _end + (4UL << 20);
-	/*
-	 * Just like in case of IPL from VM reader we make sure there is a
-	 * gap of 4MB between end of kernel and start of initrd.
-	 * That way we can also be sure that saving an NSS will succeed,
-	 * which however only requires different segments.
-	 */
-	if (!INITRD_START || !INITRD_SIZE)
-		return;
-	if (INITRD_START >= min_initrd_addr)
-		return;
-	memmove_early((void *) min_initrd_addr, (void *) INITRD_START, INITRD_SIZE);
-	INITRD_START = min_initrd_addr;
-#endif
-}
-
 /* Set up boot command line */
 static void __init append_to_cmdline(size_t (*ipl_data)(char *, size_t))
 {
@@ -410,9 +333,6 @@ static void __init setup_boot_command_line(void)
 
 void __init startup_init(void)
 {
-	reset_tod_clock();
-	rescue_initrd();
-	clear_bss_section();
 	time_early_init();
 	init_kernel_storage_key();
 	lockdep_off();
diff --git a/arch/s390/kernel/early_nobss.c b/arch/s390/kernel/early_nobss.c
new file mode 100644
index 000000000000..2d84fc48df3a
--- /dev/null
+++ b/arch/s390/kernel/early_nobss.c
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *    Copyright IBM Corp. 2007, 2018
+ */
+
+/*
+ * Early setup functions which may not rely on an initialized bss
+ * section. The last thing that is supposed to happen here is
+ * initialization of the bss section.
+ */
+
+#include <linux/processor.h>
+#include <linux/string.h>
+#include <asm/sections.h>
+#include <asm/lowcore.h>
+#include <asm/setup.h>
+#include <asm/timex.h>
+#include "entry.h"
+
+static void __init reset_tod_clock(void)
+{
+	u64 time;
+
+	if (store_tod_clock(&time) == 0)
+		return;
+	/* TOD clock not running. Set the clock to Unix Epoch. */
+	if (set_tod_clock(TOD_UNIX_EPOCH) != 0 || store_tod_clock(&time) != 0)
+		disabled_wait(0);
+
+	memset(tod_clock_base, 0, 16);
+	*(__u64 *) &tod_clock_base[1] = TOD_UNIX_EPOCH;
+	S390_lowcore.last_update_clock = TOD_UNIX_EPOCH;
+}
+
+static void __init rescue_initrd(void)
+{
+	unsigned long min_initrd_addr = (unsigned long) _end + (4UL << 20);
+
+	/*
+	 * Just like in case of IPL from VM reader we make sure there is a
+	 * gap of 4MB between end of kernel and start of initrd.
+	 * That way we can also be sure that saving an NSS will succeed,
+	 * which however only requires different segments.
+	 */
+	if (!IS_ENABLED(CONFIG_BLK_DEV_INITRD))
+		return;
+	if (!INITRD_START || !INITRD_SIZE)
+		return;
+	if (INITRD_START >= min_initrd_addr)
+		return;
+	memmove((void *) min_initrd_addr, (void *) INITRD_START, INITRD_SIZE);
+	INITRD_START = min_initrd_addr;
+}
+
+static void __init clear_bss_section(void)
+{
+	memset(__bss_start, 0, __bss_stop - __bss_start);
+}
+
+void __init startup_init_nobss(void)
+{
+	reset_tod_clock();
+	rescue_initrd();
+	clear_bss_section();
+}
diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h
index e87758f8fbdc..961abfac2c5f 100644
--- a/arch/s390/kernel/entry.h
+++ b/arch/s390/kernel/entry.h
@@ -58,6 +58,7 @@ void do_notify_resume(struct pt_regs *regs);
 void __init init_IRQ(void);
 void do_IRQ(struct pt_regs *regs, int irq);
 void do_restart(void);
+void __init startup_init_nobss(void);
 void __init startup_init(void);
 void die(struct pt_regs *regs, const char *str);
 int setup_profiling_timer(unsigned int multiplier);
diff --git a/arch/s390/kernel/head64.S b/arch/s390/kernel/head64.S
index 38a973ccf501..791cb9000e86 100644
--- a/arch/s390/kernel/head64.S
+++ b/arch/s390/kernel/head64.S
@@ -40,8 +40,12 @@ ENTRY(startup_continue)
 	stg	%r15,__LC_KERNEL_STACK	# set end of kernel stack
 	aghi	%r15,-160
 #
-# Save ipl parameters, clear bss memory, initialize storage key for kernel pages,
-# and create a kernel NSS if the SAVESYS= parm is defined
+# Early setup functions that may not rely on an initialized bss section,
+# like moving the initrd. Returns with an initialized bss section.
+#
+	brasl	%r14,startup_init_nobss
+#
+# Early machine initialization and detection functions.
 #
 	brasl	%r14,startup_init
 	lpswe	.Lentry-.LPG1(13)	# jump to _stext in primary-space,
diff --git a/arch/s390/kernel/nospec-branch.c b/arch/s390/kernel/nospec-branch.c
index 8ad6a7128b3a..18ae7b9c71d6 100644
--- a/arch/s390/kernel/nospec-branch.c
+++ b/arch/s390/kernel/nospec-branch.c
@@ -36,9 +36,9 @@ early_param("nospec", nospec_setup_early);
 static int __init nospec_report(void)
 {
 	if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable)
-		pr_info("Spectre V2 mitigation: execute trampolines.\n");
+		pr_info("Spectre V2 mitigation: execute trampolines\n");
 	if (__test_facility(82, S390_lowcore.alt_stfle_fac_list))
-		pr_info("Spectre V2 mitigation: limited branch prediction.\n");
+		pr_info("Spectre V2 mitigation: limited branch prediction\n");
 	return 0;
 }
 arch_initcall(nospec_report);
diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c
index fc7e04c2195b..54f5496913fa 100644
--- a/arch/s390/kernel/sysinfo.c
+++ b/arch/s390/kernel/sysinfo.c
@@ -294,21 +294,9 @@ static int sysinfo_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int sysinfo_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, sysinfo_show, NULL);
-}
-
-static const struct file_operations sysinfo_fops = {
-	.open		= sysinfo_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int __init sysinfo_create_proc(void)
 {
-	proc_create("sysinfo", 0444, NULL, &sysinfo_fops);
+	proc_create_single("sysinfo", 0444, NULL, sysinfo_show);
 	return 0;
 }
 device_initcall(sysinfo_create_proc);
@@ -386,18 +374,6 @@ static const struct seq_operations service_level_seq_ops = {
 	.show		= service_level_show
 };
 
-static int service_level_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &service_level_seq_ops);
-}
-
-static const struct file_operations service_level_ops = {
-	.open		= service_level_open,
-	.read		= seq_read,
-	.llseek 	= seq_lseek,
-	.release	= seq_release
-};
-
 static void service_level_vm_print(struct seq_file *m,
 				   struct service_level *slr)
 {
@@ -420,7 +396,7 @@ static struct service_level service_level_vm = {
 
 static __init int create_proc_service_level(void)
 {
-	proc_create("service_levels", 0, NULL, &service_level_ops);
+	proc_create_seq("service_levels", 0, NULL, &service_level_seq_ops);
 	if (MACHINE_IS_VM)
 		register_service_level(&service_level_vm);
 	return 0;
diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c
index a5297a22bc1e..8003b38c1688 100644
--- a/arch/s390/kernel/traps.c
+++ b/arch/s390/kernel/traps.c
@@ -44,14 +44,8 @@ int is_valid_bugaddr(unsigned long addr)
 
 void do_report_trap(struct pt_regs *regs, int si_signo, int si_code, char *str)
 {
-	siginfo_t info;
-
 	if (user_mode(regs)) {
-		info.si_signo = si_signo;
-		info.si_errno = 0;
-		info.si_code = si_code;
-		info.si_addr = get_trap_ip(regs);
-		force_sig_info(si_signo, &info, current);
+		force_sig_fault(si_signo, si_code, get_trap_ip(regs), current);
 		report_user_fault(regs, si_signo, 0);
         } else {
                 const struct exception_table_entry *fixup;
@@ -80,18 +74,12 @@ NOKPROBE_SYMBOL(do_trap);
 
 void do_per_trap(struct pt_regs *regs)
 {
-	siginfo_t info;
-
 	if (notify_die(DIE_SSTEP, "sstep", regs, 0, 0, SIGTRAP) == NOTIFY_STOP)
 		return;
 	if (!current->ptrace)
 		return;
-	info.si_signo = SIGTRAP;
-	info.si_errno = 0;
-	info.si_code = TRAP_HWBKPT;
-	info.si_addr =
-		(void __force __user *) current->thread.per_event.address;
-	force_sig_info(SIGTRAP, &info, current);
+	force_sig_fault(SIGTRAP, TRAP_HWBKPT,
+		(void __force __user *) current->thread.per_event.address, current);
 }
 NOKPROBE_SYMBOL(do_per_trap);
 
@@ -165,7 +153,6 @@ void translation_exception(struct pt_regs *regs)
 
 void illegal_op(struct pt_regs *regs)
 {
-	siginfo_t info;
         __u8 opcode[6];
 	__u16 __user *location;
 	int is_uprobe_insn = 0;
@@ -177,13 +164,9 @@ void illegal_op(struct pt_regs *regs)
 		if (get_user(*((__u16 *) opcode), (__u16 __user *) location))
 			return;
 		if (*((__u16 *) opcode) == S390_BREAKPOINT_U16) {
-			if (current->ptrace) {
-				info.si_signo = SIGTRAP;
-				info.si_errno = 0;
-				info.si_code = TRAP_BRKPT;
-				info.si_addr = location;
-				force_sig_info(SIGTRAP, &info, current);
-			} else
+			if (current->ptrace)
+				force_sig_fault(SIGTRAP, TRAP_BRKPT, location, current);
+			else
 				signal = SIGILL;
 #ifdef CONFIG_UPROBES
 		} else if (*((__u16 *) opcode) == UPROBE_SWBP_INSN) {
diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S
index 08d12cfaf091..f0414f52817b 100644
--- a/arch/s390/kernel/vmlinux.lds.S
+++ b/arch/s390/kernel/vmlinux.lds.S
@@ -82,10 +82,10 @@ SECTIONS
 
 	. = ALIGN(PAGE_SIZE);
 	.init.text : AT(ADDR(.init.text) - LOAD_OFFSET) {
-		VMLINUX_SYMBOL(_sinittext) = . ;
+		_sinittext = .;
 		INIT_TEXT
 		. = ALIGN(PAGE_SIZE);
-		VMLINUX_SYMBOL(_einittext) = . ;
+		_einittext = .;
 	}
 
 	/*
diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/guestdbg.c
index b5f3e82006d0..394a5f53805b 100644
--- a/arch/s390/kvm/guestdbg.c
+++ b/arch/s390/kvm/guestdbg.c
@@ -153,7 +153,7 @@ void kvm_s390_patch_guest_per_regs(struct kvm_vcpu *vcpu)
 
 	if (guestdbg_sstep_enabled(vcpu)) {
 		/* disable timer (clock-comparator) interrupts */
-		vcpu->arch.sie_block->gcr[0] &= ~0x800ul;
+		vcpu->arch.sie_block->gcr[0] &= ~CR0_CLOCK_COMPARATOR_SUBMASK;
 		vcpu->arch.sie_block->gcr[9] |= PER_EVENT_IFETCH;
 		vcpu->arch.sie_block->gcr[10] = 0;
 		vcpu->arch.sie_block->gcr[11] = -1UL;
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index 37d06e022238..daa09f89ca2d 100644
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -159,7 +159,7 @@ static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
 {
 	if (psw_extint_disabled(vcpu) ||
-	    !(vcpu->arch.sie_block->gcr[0] & 0x800ul))
+	    !(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 		return 0;
 	if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
 		/* No timer interrupts when single stepping */
@@ -172,7 +172,7 @@ static int ckc_irq_pending(struct kvm_vcpu *vcpu)
 	const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
 	const u64 ckc = vcpu->arch.sie_block->ckc;
 
-	if (vcpu->arch.sie_block->gcr[0] & 0x0020000000000000ul) {
+	if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
 		if ((s64)ckc >= (s64)now)
 			return 0;
 	} else if (ckc >= now) {
@@ -184,7 +184,7 @@ static int ckc_irq_pending(struct kvm_vcpu *vcpu)
 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
 {
 	return !psw_extint_disabled(vcpu) &&
-	       (vcpu->arch.sie_block->gcr[0] & 0x400ul);
+	       (vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK);
 }
 
 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
@@ -285,15 +285,15 @@ static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
 		active_mask &= ~IRQ_PEND_IO_MASK;
 	else
 		active_mask = disable_iscs(vcpu, active_mask);
-	if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul))
+	if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
 		__clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
-	if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul))
+	if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK))
 		__clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
-	if (!(vcpu->arch.sie_block->gcr[0] & 0x800ul))
+	if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 		__clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
-	if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul))
+	if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK))
 		__clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
-	if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
+	if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
 		__clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
 	if (psw_mchk_disabled(vcpu))
 		active_mask &= ~IRQ_PEND_MCHK_MASK;
@@ -1042,7 +1042,7 @@ int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
 	/* external call pending and deliverable */
 	if (kvm_s390_ext_call_pending(vcpu) &&
 	    !psw_extint_disabled(vcpu) &&
-	    (vcpu->arch.sie_block->gcr[0] & 0x2000ul))
+	    (vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
 		return 1;
 
 	if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
@@ -1062,7 +1062,7 @@ static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
 	u64 cputm, sltime = 0;
 
 	if (ckc_interrupts_enabled(vcpu)) {
-		if (vcpu->arch.sie_block->gcr[0] & 0x0020000000000000ul) {
+		if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
 			if ((s64)now < (s64)ckc)
 				sltime = tod_to_ns((s64)ckc - (s64)now);
 		} else if (now < ckc) {
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 64c986243018..e521f7699032 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -791,11 +791,21 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att
 
 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 
-static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
+void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
 {
 	struct kvm_vcpu *vcpu;
 	int i;
 
+	kvm_s390_vcpu_block_all(kvm);
+
+	kvm_for_each_vcpu(i, vcpu, kvm)
+		kvm_s390_vcpu_crypto_setup(vcpu);
+
+	kvm_s390_vcpu_unblock_all(kvm);
+}
+
+static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
+{
 	if (!test_kvm_facility(kvm, 76))
 		return -EINVAL;
 
@@ -832,10 +842,7 @@ static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 		return -ENXIO;
 	}
 
-	kvm_for_each_vcpu(i, vcpu, kvm) {
-		kvm_s390_vcpu_crypto_setup(vcpu);
-		exit_sie(vcpu);
-	}
+	kvm_s390_vcpu_crypto_reset_all(kvm);
 	mutex_unlock(&kvm->lock);
 	return 0;
 }
@@ -1033,8 +1040,8 @@ static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 	return ret;
 }
 
-static void kvm_s390_get_tod_clock_ext(struct kvm *kvm,
-					struct kvm_s390_vm_tod_clock *gtod)
+static void kvm_s390_get_tod_clock(struct kvm *kvm,
+				   struct kvm_s390_vm_tod_clock *gtod)
 {
 	struct kvm_s390_tod_clock_ext htod;
 
@@ -1043,10 +1050,12 @@ static void kvm_s390_get_tod_clock_ext(struct kvm *kvm,
 	get_tod_clock_ext((char *)&htod);
 
 	gtod->tod = htod.tod + kvm->arch.epoch;
-	gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
-
-	if (gtod->tod < htod.tod)
-		gtod->epoch_idx += 1;
+	gtod->epoch_idx = 0;
+	if (test_kvm_facility(kvm, 139)) {
+		gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
+		if (gtod->tod < htod.tod)
+			gtod->epoch_idx += 1;
+	}
 
 	preempt_enable();
 }
@@ -1056,12 +1065,7 @@ static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
 	struct kvm_s390_vm_tod_clock gtod;
 
 	memset(&gtod, 0, sizeof(gtod));
-
-	if (test_kvm_facility(kvm, 139))
-		kvm_s390_get_tod_clock_ext(kvm, &gtod);
-	else
-		gtod.tod = kvm_s390_get_tod_clock_fast(kvm);
-
+	kvm_s390_get_tod_clock(kvm, &gtod);
 	if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 		return -EFAULT;
 
@@ -1493,7 +1497,7 @@ static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
 		return -EINVAL;
 
 	/* Is this guest using storage keys? */
-	if (!mm_use_skey(current->mm))
+	if (!mm_uses_skeys(current->mm))
 		return KVM_S390_GET_SKEYS_NONE;
 
 	/* Enforce sane limit on memory allocation */
@@ -1982,10 +1986,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 
 	rc = -ENOMEM;
 
-	kvm->arch.use_esca = 0; /* start with basic SCA */
 	if (!sclp.has_64bscao)
 		alloc_flags |= GFP_DMA;
 	rwlock_init(&kvm->arch.sca_lock);
+	/* start with basic SCA */
 	kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
 	if (!kvm->arch.sca)
 		goto out_err;
@@ -2036,8 +2040,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	kvm_s390_crypto_init(kvm);
 
 	mutex_init(&kvm->arch.float_int.ais_lock);
-	kvm->arch.float_int.simm = 0;
-	kvm->arch.float_int.nimm = 0;
 	spin_lock_init(&kvm->arch.float_int.lock);
 	for (i = 0; i < FIRQ_LIST_COUNT; i++)
 		INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
@@ -2063,11 +2065,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 		kvm->arch.gmap->pfault_enabled = 0;
 	}
 
-	kvm->arch.css_support = 0;
-	kvm->arch.use_irqchip = 0;
 	kvm->arch.use_pfmfi = sclp.has_pfmfi;
-	kvm->arch.epoch = 0;
-
+	kvm->arch.use_skf = sclp.has_skey;
 	spin_lock_init(&kvm->arch.start_stop_lock);
 	kvm_s390_vsie_init(kvm);
 	kvm_s390_gisa_init(kvm);
@@ -2433,8 +2432,12 @@ static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
 	vcpu->arch.sie_block->ckc       = 0UL;
 	vcpu->arch.sie_block->todpr     = 0;
 	memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
-	vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
-	vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
+	vcpu->arch.sie_block->gcr[0]  = CR0_UNUSED_56 |
+					CR0_INTERRUPT_KEY_SUBMASK |
+					CR0_MEASUREMENT_ALERT_SUBMASK;
+	vcpu->arch.sie_block->gcr[14] = CR14_UNUSED_32 |
+					CR14_UNUSED_33 |
+					CR14_EXTERNAL_DAMAGE_SUBMASK;
 	/* make sure the new fpc will be lazily loaded */
 	save_fpu_regs();
 	current->thread.fpu.fpc = 0;
@@ -3192,7 +3195,7 @@ static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
 		return 0;
 	if (kvm_s390_vcpu_has_irq(vcpu, 0))
 		return 0;
-	if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
+	if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
 		return 0;
 	if (!vcpu->arch.gmap->pfault_enabled)
 		return 0;
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index 1b5621f4fe5b..981e3ba97461 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -410,4 +410,17 @@ static inline int kvm_s390_use_sca_entries(void)
 }
 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
 				     struct mcck_volatile_info *mcck_info);
+
+/**
+ * kvm_s390_vcpu_crypto_reset_all
+ *
+ * Reset the crypto attributes for each vcpu. This can be done while the vcpus
+ * are running as each vcpu will be removed from SIE before resetting the crypt
+ * attributes and restored to SIE afterward.
+ *
+ * Note: The kvm->lock must be held while calling this function
+ *
+ * @kvm: the KVM guest
+ */
+void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm);
 #endif
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
index ebfa0442e569..eb0eb60c7be6 100644
--- a/arch/s390/kvm/priv.c
+++ b/arch/s390/kvm/priv.c
@@ -26,7 +26,6 @@
 #include <asm/gmap.h>
 #include <asm/io.h>
 #include <asm/ptrace.h>
-#include <asm/compat.h>
 #include <asm/sclp.h>
 #include "gaccess.h"
 #include "kvm-s390.h"
@@ -205,24 +204,28 @@ static int handle_store_cpu_address(struct kvm_vcpu *vcpu)
 
 int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu)
 {
-	int rc = 0;
+	int rc;
 	struct kvm_s390_sie_block *sie_block = vcpu->arch.sie_block;
 
 	trace_kvm_s390_skey_related_inst(vcpu);
-	if (!(sie_block->ictl & (ICTL_ISKE | ICTL_SSKE | ICTL_RRBE)) &&
+	/* Already enabled? */
+	if (vcpu->kvm->arch.use_skf &&
+	    !(sie_block->ictl & (ICTL_ISKE | ICTL_SSKE | ICTL_RRBE)) &&
 	    !kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
-		return rc;
+		return 0;
 
 	rc = s390_enable_skey();
 	VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc);
-	if (!rc) {
-		if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
-			kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS);
-		else
-			sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE |
-					     ICTL_RRBE);
-	}
-	return rc;
+	if (rc)
+		return rc;
+
+	if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
+		kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS);
+	if (!vcpu->kvm->arch.use_skf)
+		sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
+	else
+		sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE | ICTL_RRBE);
+	return 0;
 }
 
 static int try_handle_skey(struct kvm_vcpu *vcpu)
@@ -232,7 +235,7 @@ static int try_handle_skey(struct kvm_vcpu *vcpu)
 	rc = kvm_s390_skey_check_enable(vcpu);
 	if (rc)
 		return rc;
-	if (sclp.has_skey) {
+	if (vcpu->kvm->arch.use_skf) {
 		/* with storage-key facility, SIE interprets it for us */
 		kvm_s390_retry_instr(vcpu);
 		VCPU_EVENT(vcpu, 4, "%s", "retrying storage key operation");
diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
index 969882b54266..84c89cb9636f 100644
--- a/arch/s390/kvm/vsie.c
+++ b/arch/s390/kvm/vsie.c
@@ -557,7 +557,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 	if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_64BSCAO))
 		gpa |= (u64) READ_ONCE(scb_o->scaoh) << 32;
 	if (gpa) {
-		if (!(gpa & ~0x1fffUL))
+		if (gpa < 2 * PAGE_SIZE)
 			rc = set_validity_icpt(scb_s, 0x0038U);
 		else if ((gpa & ~0x1fffUL) == kvm_s390_get_prefix(vcpu))
 			rc = set_validity_icpt(scb_s, 0x0011U);
@@ -578,7 +578,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 
 	gpa = READ_ONCE(scb_o->itdba) & ~0xffUL;
 	if (gpa && (scb_s->ecb & ECB_TE)) {
-		if (!(gpa & ~0x1fffUL)) {
+		if (gpa < 2 * PAGE_SIZE) {
 			rc = set_validity_icpt(scb_s, 0x0080U);
 			goto unpin;
 		}
@@ -594,7 +594,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 
 	gpa = READ_ONCE(scb_o->gvrd) & ~0x1ffUL;
 	if (gpa && (scb_s->eca & ECA_VX) && !(scb_s->ecd & ECD_HOSTREGMGMT)) {
-		if (!(gpa & ~0x1fffUL)) {
+		if (gpa < 2 * PAGE_SIZE) {
 			rc = set_validity_icpt(scb_s, 0x1310U);
 			goto unpin;
 		}
@@ -613,7 +613,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 
 	gpa = READ_ONCE(scb_o->riccbd) & ~0x3fUL;
 	if (gpa && (scb_s->ecb3 & ECB3_RI)) {
-		if (!(gpa & ~0x1fffUL)) {
+		if (gpa < 2 * PAGE_SIZE) {
 			rc = set_validity_icpt(scb_s, 0x0043U);
 			goto unpin;
 		}
@@ -632,7 +632,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 
 		gpa = READ_ONCE(scb_o->sdnxo) & ~0xfUL;
 		sdnxc = READ_ONCE(scb_o->sdnxo) & 0xfUL;
-		if (!gpa || !(gpa & ~0x1fffUL)) {
+		if (!gpa || gpa < 2 * PAGE_SIZE) {
 			rc = set_validity_icpt(scb_s, 0x10b0U);
 			goto unpin;
 		}
diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile
index 9bfe0802684b..57ab40188d4b 100644
--- a/arch/s390/lib/Makefile
+++ b/arch/s390/lib/Makefile
@@ -8,3 +8,6 @@ obj-y += mem.o xor.o
 lib-$(CONFIG_SMP) += spinlock.o
 lib-$(CONFIG_KPROBES) += probes.o
 lib-$(CONFIG_UPROBES) += probes.o
+
+chkbss := mem.o
+include $(srctree)/arch/s390/scripts/Makefile.chkbss
diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index 93faeca52284..e074480d3598 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -265,14 +265,10 @@ void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault)
  */
 static noinline void do_sigsegv(struct pt_regs *regs, int si_code)
 {
-	struct siginfo si;
-
 	report_user_fault(regs, SIGSEGV, 1);
-	si.si_signo = SIGSEGV;
-	si.si_errno = 0;
-	si.si_code = si_code;
-	si.si_addr = (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK);
-	force_sig_info(SIGSEGV, &si, current);
+	force_sig_fault(SIGSEGV, si_code,
+			(void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK),
+			current);
 }
 
 static noinline void do_no_context(struct pt_regs *regs)
@@ -316,18 +312,13 @@ static noinline void do_low_address(struct pt_regs *regs)
 
 static noinline void do_sigbus(struct pt_regs *regs)
 {
-	struct task_struct *tsk = current;
-	struct siginfo si;
-
 	/*
 	 * Send a sigbus, regardless of whether we were in kernel
 	 * or user mode.
 	 */
-	si.si_signo = SIGBUS;
-	si.si_errno = 0;
-	si.si_code = BUS_ADRERR;
-	si.si_addr = (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK);
-	force_sig_info(SIGBUS, &si, tsk);
+	force_sig_fault(SIGBUS, BUS_ADRERR,
+			(void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK),
+			current);
 }
 
 static noinline int signal_return(struct pt_regs *regs)
diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c
index 2c55a2b9d6c6..bc56ec8abcf7 100644
--- a/arch/s390/mm/gmap.c
+++ b/arch/s390/mm/gmap.c
@@ -2184,14 +2184,14 @@ int s390_enable_skey(void)
 	int rc = 0;
 
 	down_write(&mm->mmap_sem);
-	if (mm_use_skey(mm))
+	if (mm_uses_skeys(mm))
 		goto out_up;
 
-	mm->context.use_skey = 1;
+	mm->context.uses_skeys = 1;
 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
 		if (ksm_madvise(vma, vma->vm_start, vma->vm_end,
 				MADV_UNMERGEABLE, &vma->vm_flags)) {
-			mm->context.use_skey = 0;
+			mm->context.uses_skeys = 0;
 			rc = -ENOMEM;
 			goto out_up;
 		}
diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c
index 562f72955956..84bd6329a88d 100644
--- a/arch/s390/mm/pgalloc.c
+++ b/arch/s390/mm/pgalloc.c
@@ -190,14 +190,15 @@ unsigned long *page_table_alloc(struct mm_struct *mm)
 		if (!list_empty(&mm->context.pgtable_list)) {
 			page = list_first_entry(&mm->context.pgtable_list,
 						struct page, lru);
-			mask = atomic_read(&page->_mapcount);
+			mask = atomic_read(&page->_refcount) >> 24;
 			mask = (mask | (mask >> 4)) & 3;
 			if (mask != 3) {
 				table = (unsigned long *) page_to_phys(page);
 				bit = mask & 1;		/* =1 -> second 2K */
 				if (bit)
 					table += PTRS_PER_PTE;
-				atomic_xor_bits(&page->_mapcount, 1U << bit);
+				atomic_xor_bits(&page->_refcount,
+							1U << (bit + 24));
 				list_del(&page->lru);
 			}
 		}
@@ -218,12 +219,12 @@ unsigned long *page_table_alloc(struct mm_struct *mm)
 	table = (unsigned long *) page_to_phys(page);
 	if (mm_alloc_pgste(mm)) {
 		/* Return 4K page table with PGSTEs */
-		atomic_set(&page->_mapcount, 3);
+		atomic_xor_bits(&page->_refcount, 3 << 24);
 		memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);
 		memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);
 	} else {
 		/* Return the first 2K fragment of the page */
-		atomic_set(&page->_mapcount, 1);
+		atomic_xor_bits(&page->_refcount, 1 << 24);
 		memset64((u64 *)table, _PAGE_INVALID, 2 * PTRS_PER_PTE);
 		spin_lock_bh(&mm->context.lock);
 		list_add(&page->lru, &mm->context.pgtable_list);
@@ -242,7 +243,8 @@ void page_table_free(struct mm_struct *mm, unsigned long *table)
 		/* Free 2K page table fragment of a 4K page */
 		bit = (__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t));
 		spin_lock_bh(&mm->context.lock);
-		mask = atomic_xor_bits(&page->_mapcount, 1U << bit);
+		mask = atomic_xor_bits(&page->_refcount, 1U << (bit + 24));
+		mask >>= 24;
 		if (mask & 3)
 			list_add(&page->lru, &mm->context.pgtable_list);
 		else
@@ -253,7 +255,6 @@ void page_table_free(struct mm_struct *mm, unsigned long *table)
 	}
 
 	pgtable_page_dtor(page);
-	atomic_set(&page->_mapcount, -1);
 	__free_page(page);
 }
 
@@ -274,7 +275,8 @@ void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table,
 	}
 	bit = (__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t));
 	spin_lock_bh(&mm->context.lock);
-	mask = atomic_xor_bits(&page->_mapcount, 0x11U << bit);
+	mask = atomic_xor_bits(&page->_refcount, 0x11U << (bit + 24));
+	mask >>= 24;
 	if (mask & 3)
 		list_add_tail(&page->lru, &mm->context.pgtable_list);
 	else
@@ -296,12 +298,13 @@ static void __tlb_remove_table(void *_table)
 		break;
 	case 1:		/* lower 2K of a 4K page table */
 	case 2:		/* higher 2K of a 4K page table */
-		if (atomic_xor_bits(&page->_mapcount, mask << 4) != 0)
+		mask = atomic_xor_bits(&page->_refcount, mask << (4 + 24));
+		mask >>= 24;
+		if (mask != 0)
 			break;
 		/* fallthrough */
 	case 3:		/* 4K page table with pgstes */
 		pgtable_page_dtor(page);
-		atomic_set(&page->_mapcount, -1);
 		__free_page(page);
 		break;
 	}
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 4f2b65d01a70..301e466e4263 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -158,7 +158,7 @@ static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste,
 #ifdef CONFIG_PGSTE
 	unsigned long address, bits, skey;
 
-	if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID)
+	if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID)
 		return pgste;
 	address = pte_val(pte) & PAGE_MASK;
 	skey = (unsigned long) page_get_storage_key(address);
@@ -180,7 +180,7 @@ static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
 	unsigned long address;
 	unsigned long nkey;
 
-	if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID)
+	if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID)
 		return;
 	VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
 	address = pte_val(entry) & PAGE_MASK;
diff --git a/arch/s390/net/Makefile b/arch/s390/net/Makefile
index e0d5f245e42b..8cab6deb0403 100644
--- a/arch/s390/net/Makefile
+++ b/arch/s390/net/Makefile
@@ -2,4 +2,5 @@
 #
 # Arch-specific network modules
 #
-obj-$(CONFIG_BPF_JIT) += bpf_jit.o bpf_jit_comp.o
+obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o
+obj-$(CONFIG_HAVE_PNETID) += pnet.o
diff --git a/arch/s390/net/bpf_jit.S b/arch/s390/net/bpf_jit.S
deleted file mode 100644
index 9f794869c1b0..000000000000
--- a/arch/s390/net/bpf_jit.S
+++ /dev/null
@@ -1,120 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * BPF Jit compiler for s390, help functions.
- *
- * Copyright IBM Corp. 2012,2015
- *
- * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
- *	      Michael Holzheu <holzheu@linux.vnet.ibm.com>
- */
-
-#include <linux/linkage.h>
-#include <asm/nospec-insn.h>
-#include "bpf_jit.h"
-
-/*
- * Calling convention:
- * registers %r7-%r10, %r11,%r13, and %r15 are call saved
- *
- * Input (64 bit):
- *   %r3 (%b2) = offset into skb data
- *   %r6 (%b5) = return address
- *   %r7 (%b6) = skb pointer
- *   %r12      = skb data pointer
- *
- * Output:
- *   %r14= %b0 = return value (read skb value)
- *
- * Work registers: %r2,%r4,%r5,%r14
- *
- * skb_copy_bits takes 4 parameters:
- *   %r2 = skb pointer
- *   %r3 = offset into skb data
- *   %r4 = pointer to temp buffer
- *   %r5 = length to copy
- *   Return value in %r2: 0 = ok
- *
- * bpf_internal_load_pointer_neg_helper takes 3 parameters:
- *   %r2 = skb pointer
- *   %r3 = offset into data
- *   %r4 = length to copy
- *   Return value in %r2: Pointer to data
- */
-
-#define SKF_MAX_NEG_OFF	-0x200000	/* SKF_LL_OFF from filter.h */
-
-/*
- * Load SIZE bytes from SKB
- */
-#define sk_load_common(NAME, SIZE, LOAD)				\
-ENTRY(sk_load_##NAME);							\
-	ltgr	%r3,%r3;		/* Is offset negative? */	\
-	jl	sk_load_##NAME##_slow_neg;				\
-ENTRY(sk_load_##NAME##_pos);						\
-	aghi	%r3,SIZE;		/* Offset + SIZE */		\
-	clg	%r3,STK_OFF_HLEN(%r15);	/* Offset + SIZE > hlen? */	\
-	jh	sk_load_##NAME##_slow;					\
-	LOAD	%r14,-SIZE(%r3,%r12);	/* Get data from skb */		\
-	B_EX	OFF_OK,%r6;		/* Return */			\
-									\
-sk_load_##NAME##_slow:;							\
-	lgr	%r2,%r7;		/* Arg1 = skb pointer */	\
-	aghi	%r3,-SIZE;		/* Arg2 = offset */		\
-	la	%r4,STK_OFF_TMP(%r15);	/* Arg3 = temp bufffer */	\
-	lghi	%r5,SIZE;		/* Arg4 = size */		\
-	brasl	%r14,skb_copy_bits;	/* Get data from skb */		\
-	LOAD	%r14,STK_OFF_TMP(%r15);	/* Load from temp bufffer */	\
-	ltgr	%r2,%r2;		/* Set cc to (%r2 != 0) */	\
-	BR_EX	%r6;			/* Return */
-
-sk_load_common(word, 4, llgf)	/* r14 = *(u32 *) (skb->data+offset) */
-sk_load_common(half, 2, llgh)	/* r14 = *(u16 *) (skb->data+offset) */
-
-	GEN_BR_THUNK %r6
-	GEN_B_THUNK OFF_OK,%r6
-
-/*
- * Load 1 byte from SKB (optimized version)
- */
-	/* r14 = *(u8 *) (skb->data+offset) */
-ENTRY(sk_load_byte)
-	ltgr	%r3,%r3			# Is offset negative?
-	jl	sk_load_byte_slow_neg
-ENTRY(sk_load_byte_pos)
-	clg	%r3,STK_OFF_HLEN(%r15)	# Offset >= hlen?
-	jnl	sk_load_byte_slow
-	llgc	%r14,0(%r3,%r12)	# Get byte from skb
-	B_EX	OFF_OK,%r6		# Return OK
-
-sk_load_byte_slow:
-	lgr	%r2,%r7			# Arg1 = skb pointer
-					# Arg2 = offset
-	la	%r4,STK_OFF_TMP(%r15)	# Arg3 = pointer to temp buffer
-	lghi	%r5,1			# Arg4 = size (1 byte)
-	brasl	%r14,skb_copy_bits	# Get data from skb
-	llgc	%r14,STK_OFF_TMP(%r15)	# Load result from temp buffer
-	ltgr	%r2,%r2			# Set cc to (%r2 != 0)
-	BR_EX	%r6			# Return cc
-
-#define sk_negative_common(NAME, SIZE, LOAD)				\
-sk_load_##NAME##_slow_neg:;						\
-	cgfi	%r3,SKF_MAX_NEG_OFF;					\
-	jl	bpf_error;						\
-	lgr	%r2,%r7;		/* Arg1 = skb pointer */	\
-					/* Arg2 = offset */		\
-	lghi	%r4,SIZE;		/* Arg3 = size */		\
-	brasl	%r14,bpf_internal_load_pointer_neg_helper;		\
-	ltgr	%r2,%r2;						\
-	jz	bpf_error;						\
-	LOAD	%r14,0(%r2);		/* Get data from pointer */	\
-	xr	%r3,%r3;		/* Set cc to zero */		\
-	BR_EX	%r6;			/* Return cc */
-
-sk_negative_common(word, 4, llgf)
-sk_negative_common(half, 2, llgh)
-sk_negative_common(byte, 1, llgc)
-
-bpf_error:
-# force a return 0 from jit handler
-	ltgr	%r15,%r15	# Set condition code
-	BR_EX	%r6
diff --git a/arch/s390/net/bpf_jit.h b/arch/s390/net/bpf_jit.h
index 5e1e5133132d..7822ea92e54a 100644
--- a/arch/s390/net/bpf_jit.h
+++ b/arch/s390/net/bpf_jit.h
@@ -16,9 +16,6 @@
 #include <linux/filter.h>
 #include <linux/types.h>
 
-extern u8 sk_load_word_pos[], sk_load_half_pos[], sk_load_byte_pos[];
-extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
-
 #endif /* __ASSEMBLY__ */
 
 /*
@@ -36,15 +33,6 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
  *	      |		      |     |
  *	      |   BPF stack   |     |
  *	      |		      |     |
- *	      +---------------+     |
- *	      | 8 byte skbp   |     |
- * R15+176 -> +---------------+     |
- *	      | 8 byte hlen   |     |
- * R15+168 -> +---------------+     |
- *	      | 4 byte align  |     |
- *	      +---------------+     |
- *	      | 4 byte temp   |     |
- *	      | for bpf_jit.S |     |
  * R15+160 -> +---------------+     |
  *	      | new backchain |     |
  * R15+152 -> +---------------+     |
@@ -57,17 +45,11 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
  * The stack size used by the BPF program ("BPF stack" above) is passed
  * via "aux->stack_depth".
  */
-#define STK_SPACE_ADD (8 + 8 + 4 + 4 + 160)
+#define STK_SPACE_ADD	(160)
 #define STK_160_UNUSED	(160 - 12 * 8)
 #define STK_OFF		(STK_SPACE_ADD - STK_160_UNUSED)
-#define STK_OFF_TMP	160	/* Offset of tmp buffer on stack */
-#define STK_OFF_HLEN	168	/* Offset of SKB header length on stack */
-#define STK_OFF_SKBP	176	/* Offset of SKB pointer on stack */
 
 #define STK_OFF_R6	(160 - 11 * 8)	/* Offset of r6 on stack */
 #define STK_OFF_TCCNT	(160 - 12 * 8)	/* Offset of tail_call_cnt on stack */
 
-/* Offset to skip condition code check */
-#define OFF_OK		4
-
 #endif /* __ARCH_S390_NET_BPF_JIT_H */
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index dd2bcf0e7d00..d2db8acb1a55 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -51,23 +51,21 @@ struct bpf_jit {
 
 #define BPF_SIZE_MAX	0xffff	/* Max size for program (16 bit branches) */
 
-#define SEEN_SKB	1	/* skb access */
-#define SEEN_MEM	2	/* use mem[] for temporary storage */
-#define SEEN_RET0	4	/* ret0_ip points to a valid return 0 */
-#define SEEN_LITERAL	8	/* code uses literals */
-#define SEEN_FUNC	16	/* calls C functions */
-#define SEEN_TAIL_CALL	32	/* code uses tail calls */
-#define SEEN_REG_AX	64	/* code uses constant blinding */
-#define SEEN_STACK	(SEEN_FUNC | SEEN_MEM | SEEN_SKB)
+#define SEEN_MEM	(1 << 0)	/* use mem[] for temporary storage */
+#define SEEN_RET0	(1 << 1)	/* ret0_ip points to a valid return 0 */
+#define SEEN_LITERAL	(1 << 2)	/* code uses literals */
+#define SEEN_FUNC	(1 << 3)	/* calls C functions */
+#define SEEN_TAIL_CALL	(1 << 4)	/* code uses tail calls */
+#define SEEN_REG_AX	(1 << 5)	/* code uses constant blinding */
+#define SEEN_STACK	(SEEN_FUNC | SEEN_MEM)
 
 /*
  * s390 registers
  */
 #define REG_W0		(MAX_BPF_JIT_REG + 0)	/* Work register 1 (even) */
 #define REG_W1		(MAX_BPF_JIT_REG + 1)	/* Work register 2 (odd) */
-#define REG_SKB_DATA	(MAX_BPF_JIT_REG + 2)	/* SKB data register */
-#define REG_L		(MAX_BPF_JIT_REG + 3)	/* Literal pool register */
-#define REG_15		(MAX_BPF_JIT_REG + 4)	/* Register 15 */
+#define REG_L		(MAX_BPF_JIT_REG + 2)	/* Literal pool register */
+#define REG_15		(MAX_BPF_JIT_REG + 3)	/* Register 15 */
 #define REG_0		REG_W0			/* Register 0 */
 #define REG_1		REG_W1			/* Register 1 */
 #define REG_2		BPF_REG_1		/* Register 2 */
@@ -92,10 +90,8 @@ static const int reg2hex[] = {
 	[BPF_REG_9]	= 10,
 	/* BPF stack pointer */
 	[BPF_REG_FP]	= 13,
-	/* Register for blinding (shared with REG_SKB_DATA) */
+	/* Register for blinding */
 	[BPF_REG_AX]	= 12,
-	/* SKB data pointer */
-	[REG_SKB_DATA]	= 12,
 	/* Work registers for s390x backend */
 	[REG_W0]	= 0,
 	[REG_W1]	= 1,
@@ -401,27 +397,6 @@ static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth)
 	} while (re <= 15);
 }
 
-/*
- * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S"
- * we store the SKB header length on the stack and the SKB data
- * pointer in REG_SKB_DATA if BPF_REG_AX is not used.
- */
-static void emit_load_skb_data_hlen(struct bpf_jit *jit)
-{
-	/* Header length: llgf %w1,<len>(%b1) */
-	EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1,
-		      offsetof(struct sk_buff, len));
-	/* s %w1,<data_len>(%b1) */
-	EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1,
-		   offsetof(struct sk_buff, data_len));
-	/* stg %w1,ST_OFF_HLEN(%r0,%r15) */
-	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, STK_OFF_HLEN);
-	if (!(jit->seen & SEEN_REG_AX))
-		/* lg %skb_data,data_off(%b1) */
-		EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
-			      BPF_REG_1, offsetof(struct sk_buff, data));
-}
-
 /*
  * Emit function prologue
  *
@@ -462,12 +437,6 @@ static void bpf_jit_prologue(struct bpf_jit *jit, u32 stack_depth)
 			EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
 				      REG_15, 152);
 	}
-	if (jit->seen & SEEN_SKB) {
-		emit_load_skb_data_hlen(jit);
-		/* stg %b1,ST_OFF_SKBP(%r0,%r15) */
-		EMIT6_DISP_LH(0xe3000000, 0x0024, BPF_REG_1, REG_0, REG_15,
-			      STK_OFF_SKBP);
-	}
 }
 
 /*
@@ -537,12 +506,12 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
 {
 	struct bpf_insn *insn = &fp->insnsi[i];
 	int jmp_off, last, insn_count = 1;
-	unsigned int func_addr, mask;
 	u32 dst_reg = insn->dst_reg;
 	u32 src_reg = insn->src_reg;
 	u32 *addrs = jit->addrs;
 	s32 imm = insn->imm;
 	s16 off = insn->off;
+	unsigned int mask;
 
 	if (dst_reg == BPF_REG_AX || src_reg == BPF_REG_AX)
 		jit->seen |= SEEN_REG_AX;
@@ -1029,13 +998,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
 		}
 		/* lgr %b0,%r2: load return value into %b0 */
 		EMIT4(0xb9040000, BPF_REG_0, REG_2);
-		if ((jit->seen & SEEN_SKB) &&
-		    bpf_helper_changes_pkt_data((void *)func)) {
-			/* lg %b1,ST_OFF_SKBP(%r15) */
-			EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_1, REG_0,
-				      REG_15, STK_OFF_SKBP);
-			emit_load_skb_data_hlen(jit);
-		}
 		break;
 	}
 	case BPF_JMP | BPF_TAIL_CALL:
@@ -1235,73 +1197,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
 		jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4);
 		EMIT4_PCREL(0xa7040000 | mask << 8, jmp_off);
 		break;
-	/*
-	 * BPF_LD
-	 */
-	case BPF_LD | BPF_ABS | BPF_B: /* b0 = *(u8 *) (skb->data+imm) */
-	case BPF_LD | BPF_IND | BPF_B: /* b0 = *(u8 *) (skb->data+imm+src) */
-		if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
-			func_addr = __pa(sk_load_byte_pos);
-		else
-			func_addr = __pa(sk_load_byte);
-		goto call_fn;
-	case BPF_LD | BPF_ABS | BPF_H: /* b0 = *(u16 *) (skb->data+imm) */
-	case BPF_LD | BPF_IND | BPF_H: /* b0 = *(u16 *) (skb->data+imm+src) */
-		if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
-			func_addr = __pa(sk_load_half_pos);
-		else
-			func_addr = __pa(sk_load_half);
-		goto call_fn;
-	case BPF_LD | BPF_ABS | BPF_W: /* b0 = *(u32 *) (skb->data+imm) */
-	case BPF_LD | BPF_IND | BPF_W: /* b0 = *(u32 *) (skb->data+imm+src) */
-		if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
-			func_addr = __pa(sk_load_word_pos);
-		else
-			func_addr = __pa(sk_load_word);
-		goto call_fn;
-call_fn:
-		jit->seen |= SEEN_SKB | SEEN_RET0 | SEEN_FUNC;
-		REG_SET_SEEN(REG_14); /* Return address of possible func call */
-
-		/*
-		 * Implicit input:
-		 *  BPF_REG_6	 (R7) : skb pointer
-		 *  REG_SKB_DATA (R12): skb data pointer (if no BPF_REG_AX)
-		 *
-		 * Calculated input:
-		 *  BPF_REG_2	 (R3) : offset of byte(s) to fetch in skb
-		 *  BPF_REG_5	 (R6) : return address
-		 *
-		 * Output:
-		 *  BPF_REG_0	 (R14): data read from skb
-		 *
-		 * Scratch registers (BPF_REG_1-5)
-		 */
-
-		/* Call function: llilf %w1,func_addr  */
-		EMIT6_IMM(0xc00f0000, REG_W1, func_addr);
-
-		/* Offset: lgfi %b2,imm */
-		EMIT6_IMM(0xc0010000, BPF_REG_2, imm);
-		if (BPF_MODE(insn->code) == BPF_IND)
-			/* agfr %b2,%src (%src is s32 here) */
-			EMIT4(0xb9180000, BPF_REG_2, src_reg);
-
-		/* Reload REG_SKB_DATA if BPF_REG_AX is used */
-		if (jit->seen & SEEN_REG_AX)
-			/* lg %skb_data,data_off(%b6) */
-			EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
-				      BPF_REG_6, offsetof(struct sk_buff, data));
-		/* basr %b5,%w1 (%b5 is call saved) */
-		EMIT2(0x0d00, BPF_REG_5, REG_W1);
-
-		/*
-		 * Note: For fast access we jump directly after the
-		 * jnz instruction from bpf_jit.S
-		 */
-		/* jnz <ret0> */
-		EMIT4_PCREL(0xa7740000, jit->ret0_ip - jit->prg);
-		break;
 	default: /* too complex, give up */
 		pr_err("Unknown opcode %02x\n", insn->code);
 		return -1;
diff --git a/arch/s390/net/pnet.c b/arch/s390/net/pnet.c
new file mode 100644
index 000000000000..e22f1b10a6c7
--- /dev/null
+++ b/arch/s390/net/pnet.c
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  IBM System z PNET ID Support
+ *
+ *    Copyright IBM Corp. 2018
+ */
+
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/types.h>
+#include <asm/ccwgroup.h>
+#include <asm/ccwdev.h>
+#include <asm/pnet.h>
+
+/*
+ * Get the PNETIDs from a device.
+ * s390 hardware supports the definition of a so-called Physical Network
+ * Identifier (short PNETID) per network device port. These PNETIDs can be
+ * used to identify network devices that are attached to the same physical
+ * network (broadcast domain).
+ *
+ * The device can be
+ * - a ccwgroup device with all bundled subchannels having the same PNETID
+ * - a PCI attached network device
+ *
+ * Returns:
+ * 0:		PNETIDs extracted from device.
+ * -ENOMEM:	No memory to extract utility string.
+ * -EOPNOTSUPP: Device type without utility string support
+ */
+static int pnet_ids_by_device(struct device *dev, u8 *pnetids)
+{
+	memset(pnetids, 0, PNETIDS_LEN);
+	if (dev_is_ccwgroup(dev)) {
+		struct ccwgroup_device *gdev = to_ccwgroupdev(dev);
+		u8 *util_str;
+
+		util_str = ccw_device_get_util_str(gdev->cdev[0], 0);
+		if (!util_str)
+			return -ENOMEM;
+		memcpy(pnetids, util_str, PNETIDS_LEN);
+		kfree(util_str);
+		return 0;
+	}
+	if (dev_is_pci(dev)) {
+		struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
+
+		memcpy(pnetids, zdev->util_str, sizeof(zdev->util_str));
+		return 0;
+	}
+	return -EOPNOTSUPP;
+}
+
+/*
+ * Extract the pnetid for a device port.
+ *
+ * Return 0 if a pnetid is found and -ENOENT otherwise.
+ */
+int pnet_id_by_dev_port(struct device *dev, unsigned short port, u8 *pnetid)
+{
+	u8 pnetids[MAX_PNETID_PORTS][MAX_PNETID_LEN];
+	static const u8 zero[MAX_PNETID_LEN] = { 0 };
+	int rc = 0;
+
+	if (!dev || port >= MAX_PNETID_PORTS)
+		return -ENOENT;
+
+	if (!pnet_ids_by_device(dev, (u8 *)pnetids) &&
+	    memcmp(pnetids[port], zero, MAX_PNETID_LEN))
+		memcpy(pnetid, pnetids[port], MAX_PNETID_LEN);
+	else
+		rc = -ENOENT;
+
+	return rc;
+}
+EXPORT_SYMBOL_GPL(pnet_id_by_dev_port);
+
+MODULE_DESCRIPTION("pnetid determination from utility strings");
+MODULE_LICENSE("GPL");
diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c
index 93cd0f1ca12b..19b2d2a9b43d 100644
--- a/arch/s390/pci/pci_clp.c
+++ b/arch/s390/pci/pci_clp.c
@@ -19,7 +19,6 @@
 #include <linux/uaccess.h>
 #include <asm/pci_debug.h>
 #include <asm/pci_clp.h>
-#include <asm/compat.h>
 #include <asm/clp.h>
 #include <uapi/asm/clp.h>
 
diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c
index 2d15d84c20ed..d387a0fbdd7e 100644
--- a/arch/s390/pci/pci_dma.c
+++ b/arch/s390/pci/pci_dma.c
@@ -668,15 +668,6 @@ void zpci_dma_exit(void)
 	kmem_cache_destroy(dma_region_table_cache);
 }
 
-#define PREALLOC_DMA_DEBUG_ENTRIES	(1 << 16)
-
-static int __init dma_debug_do_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-	return 0;
-}
-fs_initcall(dma_debug_do_init);
-
 const struct dma_map_ops s390_pci_dma_ops = {
 	.alloc		= s390_dma_alloc,
 	.free		= s390_dma_free,
@@ -685,8 +676,6 @@ const struct dma_map_ops s390_pci_dma_ops = {
 	.map_page	= s390_dma_map_pages,
 	.unmap_page	= s390_dma_unmap_pages,
 	.mapping_error	= s390_mapping_error,
-	/* if we support direct DMA this must be conditional */
-	.is_phys	= 0,
 	/* dma_supported is unconditionally true without a callback */
 };
 EXPORT_SYMBOL_GPL(s390_pci_dma_ops);
diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile
index e9525bc1b4a6..1ace023cbdce 100644
--- a/arch/s390/purgatory/Makefile
+++ b/arch/s390/purgatory/Makefile
@@ -21,7 +21,7 @@ LDFLAGS_purgatory.ro += -z nodefaultlib
 KBUILD_CFLAGS := -fno-strict-aliasing -Wall -Wstrict-prototypes
 KBUILD_CFLAGS += -Wno-pointer-sign -Wno-sign-compare
 KBUILD_CFLAGS += -fno-zero-initialized-in-bss -fno-builtin -ffreestanding
-KBUILD_CFLAGS += -c -MD -Os -m64
+KBUILD_CFLAGS += -c -MD -Os -m64 -msoft-float
 KBUILD_CFLAGS += $(call cc-option,-fno-PIE)
 
 $(obj)/purgatory.ro: $(PURGATORY_OBJS) FORCE
diff --git a/arch/s390/scripts/Makefile.chkbss b/arch/s390/scripts/Makefile.chkbss
new file mode 100644
index 000000000000..d92f2d94a5d9
--- /dev/null
+++ b/arch/s390/scripts/Makefile.chkbss
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0
+
+quiet_cmd_chkbss = CHKBSS  $<
+define cmd_chkbss
+	if ! $(OBJDUMP) -j .bss -w -h $< | awk 'END { if ($$3) exit 1 }'; then \
+		echo "error: $< .bss section is not empty" >&2; exit 1; \
+	fi; \
+	touch $@;
+endef
+
+$(obj)/built-in.a: $(patsubst %, $(obj)/%.chkbss, $(chkbss))
+
+%.o.chkbss: %.o
+	$(call cmd,chkbss)
diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig
index 1851eaeee131..84048fbf6f8a 100644
--- a/arch/sh/Kconfig
+++ b/arch/sh/Kconfig
@@ -14,7 +14,6 @@ config SUPERH
 	select HAVE_OPROFILE
 	select HAVE_GENERIC_DMA_COHERENT
 	select HAVE_ARCH_TRACEHOOK
-	select HAVE_DMA_API_DEBUG
 	select HAVE_PERF_EVENTS
 	select HAVE_DEBUG_BUGVERBOSE
 	select ARCH_HAVE_CUSTOM_GPIO_H
@@ -34,6 +33,8 @@ config SUPERH
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select MAY_HAVE_SPARSE_IRQ
 	select IRQ_FORCED_THREADING
+	select NEED_DMA_MAP_STATE
+	select NEED_SG_DMA_LENGTH
 	select RTC_LIB
 	select GENERIC_ATOMIC64
 	select GENERIC_IRQ_SHOW
@@ -51,6 +52,8 @@ config SUPERH
 	select HAVE_ARCH_AUDITSYSCALL
 	select HAVE_FUTEX_CMPXCHG if FUTEX
 	select HAVE_NMI
+	select ARCH_HAS_PTE_SPECIAL
+
 	help
 	  The SuperH is a RISC processor targeted for use in embedded systems
 	  and consumer electronics; it was also used in the Sega Dreamcast
@@ -58,7 +61,7 @@ config SUPERH
 	  <http://www.linux-sh.org/>.
 
 config SUPERH32
-	def_bool ARCH = "sh"
+	def_bool "$(ARCH)" = "sh"
 	select HAVE_KPROBES
 	select HAVE_KRETPROBES
 	select HAVE_IOREMAP_PROT if MMU && !X2TLB
@@ -77,7 +80,7 @@ config SUPERH32
 	select HAVE_CC_STACKPROTECTOR
 
 config SUPERH64
-	def_bool ARCH = "sh64"
+	def_bool "$(ARCH)" = "sh64"
 	select HAVE_EXIT_THREAD
 	select KALLSYMS
 
@@ -161,12 +164,6 @@ config DMA_COHERENT
 config DMA_NONCOHERENT
 	def_bool !DMA_COHERENT
 
-config NEED_DMA_MAP_STATE
-	def_bool DMA_NONCOHERENT
-
-config NEED_SG_DMA_LENGTH
-	def_bool y
-
 config PGTABLE_LEVELS
 	default 3 if X2TLB
 	default 2
diff --git a/arch/sh/boards/board-sh7785lcr.c b/arch/sh/boards/board-sh7785lcr.c
index d7d232dea33e..3cba60ff7aab 100644
--- a/arch/sh/boards/board-sh7785lcr.c
+++ b/arch/sh/boards/board-sh7785lcr.c
@@ -17,7 +17,7 @@
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/i2c.h>
-#include <linux/i2c-pca-platform.h>
+#include <linux/platform_data/i2c-pca-platform.h>
 #include <linux/i2c-algo-pca.h>
 #include <linux/usb/r8a66597.h>
 #include <linux/sh_intc.h>
diff --git a/arch/sh/boards/mach-ecovec24/setup.c b/arch/sh/boards/mach-ecovec24/setup.c
index adc61d14172c..c296b5c399b7 100644
--- a/arch/sh/boards/mach-ecovec24/setup.c
+++ b/arch/sh/boards/mach-ecovec24/setup.c
@@ -633,7 +633,6 @@ static struct regulator_init_data cn12_power_init_data = {
 static struct fixed_voltage_config cn12_power_info = {
 	.supply_name = "CN12 SD/MMC Vdd",
 	.microvolts = 3300000,
-	.gpio = GPIO_PTB7,
 	.enable_high = 1,
 	.init_data = &cn12_power_init_data,
 };
@@ -646,6 +645,16 @@ static struct platform_device cn12_power = {
 	},
 };
 
+static struct gpiod_lookup_table cn12_power_gpiod_table = {
+	.dev_id = "reg-fixed-voltage.0",
+	.table = {
+		/* Offset 7 on port B */
+		GPIO_LOOKUP("sh7724_pfc", GPIO_PTB7,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 #if defined(CONFIG_MMC_SDHI) || defined(CONFIG_MMC_SDHI_MODULE)
 /* SDHI0 */
 static struct regulator_consumer_supply sdhi0_power_consumers[] =
@@ -665,7 +674,6 @@ static struct regulator_init_data sdhi0_power_init_data = {
 static struct fixed_voltage_config sdhi0_power_info = {
 	.supply_name = "CN11 SD/MMC Vdd",
 	.microvolts = 3300000,
-	.gpio = GPIO_PTB6,
 	.enable_high = 1,
 	.init_data = &sdhi0_power_init_data,
 };
@@ -678,6 +686,16 @@ static struct platform_device sdhi0_power = {
 	},
 };
 
+static struct gpiod_lookup_table sdhi0_power_gpiod_table = {
+	.dev_id = "reg-fixed-voltage.1",
+	.table = {
+		/* Offset 6 on port B */
+		GPIO_LOOKUP("sh7724_pfc", GPIO_PTB6,
+			    "enable", GPIO_ACTIVE_HIGH),
+		{ },
+	},
+};
+
 static struct tmio_mmc_data sdhi0_info = {
 	.chan_priv_tx	= (void *)SHDMA_SLAVE_SDHI0_TX,
 	.chan_priv_rx	= (void *)SHDMA_SLAVE_SDHI0_RX,
diff --git a/arch/sh/drivers/dma/dma-api.c b/arch/sh/drivers/dma/dma-api.c
index c0eec08d8f95..b05be597b19f 100644
--- a/arch/sh/drivers/dma/dma-api.c
+++ b/arch/sh/drivers/dma/dma-api.c
@@ -339,18 +339,6 @@ static int dma_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int dma_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, dma_proc_show, NULL);
-}
-
-static const struct file_operations dma_proc_fops = {
-	.open		= dma_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 int register_dmac(struct dma_info *info)
 {
 	unsigned int total_channels, i;
@@ -423,7 +411,7 @@ EXPORT_SYMBOL(unregister_dmac);
 static int __init dma_api_init(void)
 {
 	printk(KERN_NOTICE "DMA: Registering DMA API.\n");
-	return proc_create("dma", 0, NULL, &dma_proc_fops) ? 0 : -ENOMEM;
+	return proc_create_single("dma", 0, NULL, dma_proc_show) ? 0 : -ENOMEM;
 }
 subsys_initcall(dma_api_init);
 
diff --git a/arch/sh/include/asm/Kbuild b/arch/sh/include/asm/Kbuild
index 1efcce74997b..46dd82ab2c29 100644
--- a/arch/sh/include/asm/Kbuild
+++ b/arch/sh/include/asm/Kbuild
@@ -1,3 +1,4 @@
+generic-y += compat.h
 generic-y += current.h
 generic-y += delay.h
 generic-y += div64.h
diff --git a/arch/sh/include/asm/pci.h b/arch/sh/include/asm/pci.h
index 0033f0df2b3b..10a36b1cf2ea 100644
--- a/arch/sh/include/asm/pci.h
+++ b/arch/sh/include/asm/pci.h
@@ -71,12 +71,6 @@ extern unsigned long PCIBIOS_MIN_IO, PCIBIOS_MIN_MEM;
  * SuperH has everything mapped statically like x86.
  */
 
-/* The PCI address space does equal the physical memory
- * address space.  The networking and block device layers use
- * this boolean for bounce buffer decisions.
- */
-#define PCI_DMA_BUS_IS_PHYS	(dma_ops->is_phys)
-
 #ifdef CONFIG_PCI
 /*
  * None of the SH PCI controllers support MWI, it is always treated as a
diff --git a/arch/sh/include/asm/pgtable.h b/arch/sh/include/asm/pgtable.h
index 89c513a982fc..f6abfe2bca93 100644
--- a/arch/sh/include/asm/pgtable.h
+++ b/arch/sh/include/asm/pgtable.h
@@ -156,8 +156,6 @@ extern void page_table_range_init(unsigned long start, unsigned long end,
 #define HAVE_ARCH_UNMAPPED_AREA
 #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
 
-#define __HAVE_ARCH_PTE_SPECIAL
-
 #include <asm-generic/pgtable.h>
 
 #endif /* __ASM_SH_PGTABLE_H */
diff --git a/arch/sh/kernel/dma-nommu.c b/arch/sh/kernel/dma-nommu.c
index 178457d7620c..3e3a32fc676e 100644
--- a/arch/sh/kernel/dma-nommu.c
+++ b/arch/sh/kernel/dma-nommu.c
@@ -78,7 +78,6 @@ const struct dma_map_ops nommu_dma_ops = {
 	.sync_single_for_device	= nommu_sync_single_for_device,
 	.sync_sg_for_device	= nommu_sync_sg_for_device,
 #endif
-	.is_phys		= 1,
 };
 
 void __init no_iommu_init(void)
diff --git a/arch/sh/kernel/hw_breakpoint.c b/arch/sh/kernel/hw_breakpoint.c
index afe965712a69..8648ed05ccf0 100644
--- a/arch/sh/kernel/hw_breakpoint.c
+++ b/arch/sh/kernel/hw_breakpoint.c
@@ -347,13 +347,8 @@ static int __kprobes hw_breakpoint_handler(struct die_args *args)
 
 		/* Deliver the signal to userspace */
 		if (!arch_check_bp_in_kernelspace(bp)) {
-			siginfo_t info;
-
-			info.si_signo = args->signr;
-			info.si_errno = notifier_to_errno(rc);
-			info.si_code = TRAP_HWBKPT;
-
-			force_sig_info(args->signr, &info, current);
+			force_sig_fault(SIGTRAP, TRAP_HWBKPT,
+					(void __user *)NULL, current);
 		}
 
 		rcu_read_unlock();
diff --git a/arch/sh/kernel/irq.c b/arch/sh/kernel/irq.c
index 245dbeb20afe..5717c7cbdd97 100644
--- a/arch/sh/kernel/irq.c
+++ b/arch/sh/kernel/irq.c
@@ -44,7 +44,7 @@ int arch_show_interrupts(struct seq_file *p, int prec)
 
 	seq_printf(p, "%*s: ", prec, "NMI");
 	for_each_online_cpu(j)
-		seq_printf(p, "%10u ", irq_stat[j].__nmi_count);
+		seq_printf(p, "%10u ", nmi_count(j));
 	seq_printf(p, "  Non-maskable interrupts\n");
 
 	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
diff --git a/arch/sh/kernel/traps_32.c b/arch/sh/kernel/traps_32.c
index b3770bb26211..60709ad17fc7 100644
--- a/arch/sh/kernel/traps_32.c
+++ b/arch/sh/kernel/traps_32.c
@@ -477,7 +477,6 @@ asmlinkage void do_address_error(struct pt_regs *regs,
 {
 	unsigned long error_code = 0;
 	mm_segment_t oldfs;
-	siginfo_t info;
 	insn_size_t instruction;
 	int tmp;
 
@@ -537,11 +536,7 @@ asmlinkage void do_address_error(struct pt_regs *regs,
 		       "access (PC %lx PR %lx)\n", current->comm, regs->pc,
 		       regs->pr);
 
-		info.si_signo = SIGBUS;
-		info.si_errno = 0;
-		info.si_code = si_code;
-		info.si_addr = (void __user *)address;
-		force_sig_info(SIGBUS, &info, current);
+		force_sig_fault(SIGBUS, si_code, (void __user *)address, current);
 	} else {
 		inc_unaligned_kernel_access();
 
@@ -598,19 +593,20 @@ int is_dsp_inst(struct pt_regs *regs)
 #ifdef CONFIG_CPU_SH2A
 asmlinkage void do_divide_error(unsigned long r4)
 {
-	siginfo_t info;
+	int code;
 
 	switch (r4) {
 	case TRAP_DIVZERO_ERROR:
-		info.si_code = FPE_INTDIV;
+		code = FPE_INTDIV;
 		break;
 	case TRAP_DIVOVF_ERROR:
-		info.si_code = FPE_INTOVF;
+		code = FPE_INTOVF;
 		break;
+	default:
+		/* Let gcc know unhandled cases don't make it past here */
+		return;
 	}
-
-	info.si_signo = SIGFPE;
-	force_sig_info(info.si_signo, &info, current);
+	force_sig_fault(SIGFPE, code, NULL, current);
 }
 #endif
 
diff --git a/arch/sh/math-emu/math.c b/arch/sh/math-emu/math.c
index c86f4360c6ce..a0fa8fc88739 100644
--- a/arch/sh/math-emu/math.c
+++ b/arch/sh/math-emu/math.c
@@ -507,7 +507,6 @@ static int ieee_fpe_handler(struct pt_regs *regs)
 	unsigned short insn = *(unsigned short *)regs->pc;
 	unsigned short finsn;
 	unsigned long nextpc;
-	siginfo_t info;
 	int nib[4] = {
 		(insn >> 12) & 0xf,
 		(insn >> 8) & 0xf,
@@ -560,11 +559,8 @@ static int ieee_fpe_handler(struct pt_regs *regs)
 				~(FPSCR_CAUSE_MASK | FPSCR_FLAG_MASK);
 			task_thread_info(tsk)->status |= TS_USEDFPU;
 		} else {
-			info.si_signo = SIGFPE;
-			info.si_errno = 0;
-			info.si_code = FPE_FLTINV;
-			info.si_addr = (void __user *)regs->pc;
-			force_sig_info(SIGFPE, &info, tsk);
+			force_sig_fault(SIGFPE, FPE_FLTINV,
+					(void __user *)regs->pc, tsk);
 		}
 
 		regs->pc = nextpc;
diff --git a/arch/sh/mm/consistent.c b/arch/sh/mm/consistent.c
index f1b44697ad68..fceb2adfcac7 100644
--- a/arch/sh/mm/consistent.c
+++ b/arch/sh/mm/consistent.c
@@ -20,18 +20,9 @@
 #include <asm/cacheflush.h>
 #include <asm/addrspace.h>
 
-#define PREALLOC_DMA_DEBUG_ENTRIES	4096
-
 const struct dma_map_ops *dma_ops;
 EXPORT_SYMBOL(dma_ops);
 
-static int __init dma_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-	return 0;
-}
-fs_initcall(dma_init);
-
 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 				 dma_addr_t *dma_handle, gfp_t gfp,
 				 unsigned long attrs)
diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c
index 6fd1bf7481c7..b8e7bb84b6b1 100644
--- a/arch/sh/mm/fault.c
+++ b/arch/sh/mm/fault.c
@@ -42,14 +42,7 @@ static void
 force_sig_info_fault(int si_signo, int si_code, unsigned long address,
 		     struct task_struct *tsk)
 {
-	siginfo_t info;
-
-	info.si_signo	= si_signo;
-	info.si_errno	= 0;
-	info.si_code	= si_code;
-	info.si_addr	= (void __user *)address;
-
-	force_sig_info(si_signo, &info, tsk);
+	force_sig_fault(si_signo, si_code, (void __user *)address, tsk);
 }
 
 /*
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 8767e45f1b2b..9a2b8877f174 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -1,6 +1,6 @@
 config 64BIT
-	bool "64-bit kernel" if ARCH = "sparc"
-	default ARCH = "sparc64"
+	bool "64-bit kernel" if "$(ARCH)" = "sparc"
+	default "$(ARCH)" = "sparc64"
 	help
 	  SPARC is a family of RISC microprocessors designed and marketed by
 	  Sun Microsystems, incorporated.  They are very widely found in Sun
@@ -25,7 +25,6 @@ config SPARC
 	select RTC_CLASS
 	select RTC_DRV_M48T59
 	select RTC_SYSTOHC
-	select HAVE_DMA_API_DEBUG
 	select HAVE_ARCH_JUMP_LABEL if SPARC64
 	select GENERIC_IRQ_SHOW
 	select ARCH_WANT_IPC_PARSE_VERSION
@@ -44,6 +43,8 @@ config SPARC
 	select ARCH_HAS_SG_CHAIN
 	select CPU_NO_EFFICIENT_FFS
 	select LOCKDEP_SMALL if LOCKDEP
+	select NEED_DMA_MAP_STATE
+	select NEED_SG_DMA_LENGTH
 
 config SPARC32
 	def_bool !64BIT
@@ -67,6 +68,7 @@ config SPARC64
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_CONTEXT_TRACKING
 	select HAVE_DEBUG_KMEMLEAK
+	select IOMMU_HELPER
 	select SPARSE_IRQ
 	select RTC_DRV_CMOS
 	select RTC_DRV_BQ4802
@@ -86,6 +88,7 @@ config SPARC64
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select GENERIC_TIME_VSYSCALL
 	select ARCH_CLOCKSOURCE_DATA
+	select ARCH_HAS_PTE_SPECIAL
 
 config ARCH_DEFCONFIG
 	string
@@ -102,14 +105,6 @@ config ARCH_ATU
 	bool
 	default y if SPARC64
 
-config ARCH_DMA_ADDR_T_64BIT
-	bool
-	default y if ARCH_ATU
-
-config IOMMU_HELPER
-	bool
-	default y if SPARC64
-
 config STACKTRACE_SUPPORT
 	bool
 	default y if SPARC64
@@ -146,12 +141,6 @@ config ZONE_DMA
 	bool
 	default y if SPARC32
 
-config NEED_DMA_MAP_STATE
-	def_bool y
-
-config NEED_SG_DMA_LENGTH
-	def_bool y
-
 config GENERIC_ISA_DMA
 	bool
 	default y if SPARC32
diff --git a/arch/sparc/include/asm/compat.h b/arch/sparc/include/asm/compat.h
index 615283e16f22..4eb51d2dae98 100644
--- a/arch/sparc/include/asm/compat.h
+++ b/arch/sparc/include/asm/compat.h
@@ -11,7 +11,6 @@
 
 typedef u32		compat_size_t;
 typedef s32		compat_ssize_t;
-typedef s32		compat_time_t;
 typedef s32		compat_clock_t;
 typedef s32		compat_pid_t;
 typedef u16		__compat_uid_t;
@@ -39,16 +38,6 @@ typedef u32		compat_ulong_t;
 typedef u64		compat_u64;
 typedef u32		compat_uptr_t;
 
-struct compat_timespec {
-	compat_time_t	tv_sec;
-	s32		tv_nsec;
-};
-
-struct compat_timeval {
-	compat_time_t	tv_sec;
-	s32		tv_usec;
-};
-
 struct compat_stat {
 	compat_dev_t	st_dev;
 	compat_ino_t	st_ino;
@@ -168,6 +157,7 @@ static inline compat_uptr_t ptr_to_compat(void __user *uptr)
 	return (u32)(unsigned long)uptr;
 }
 
+#ifdef CONFIG_COMPAT
 static inline void __user *arch_compat_alloc_user_space(long len)
 {
 	struct pt_regs *regs = current_thread_info()->kregs;
@@ -184,6 +174,7 @@ static inline void __user *arch_compat_alloc_user_space(long len)
 
 	return (void __user *) usp;
 }
+#endif
 
 struct compat_ipc64_perm {
 	compat_key_t key;
@@ -201,10 +192,10 @@ struct compat_ipc64_perm {
 
 struct compat_semid64_ds {
 	struct compat_ipc64_perm sem_perm;
-	unsigned int	__pad1;
-	compat_time_t	sem_otime;
-	unsigned int	__pad2;
-	compat_time_t	sem_ctime;
+	unsigned int	sem_otime_high;
+	unsigned int	sem_otime;
+	unsigned int	sem_ctime_high;
+	unsigned int	sem_ctime;
 	u32		sem_nsems;
 	u32		__unused1;
 	u32		__unused2;
@@ -212,12 +203,12 @@ struct compat_semid64_ds {
 
 struct compat_msqid64_ds {
 	struct compat_ipc64_perm msg_perm;
-	unsigned int	__pad1;
-	compat_time_t	msg_stime;
-	unsigned int	__pad2;
-	compat_time_t	msg_rtime;
-	unsigned int	__pad3;
-	compat_time_t	msg_ctime;
+	unsigned int	msg_stime_high;
+	unsigned int	msg_stime;
+	unsigned int	msg_rtime_high;
+	unsigned int	msg_rtime;
+	unsigned int	msg_ctime_high;
+	unsigned int	msg_ctime;
 	unsigned int	msg_cbytes;
 	unsigned int	msg_qnum;
 	unsigned int	msg_qbytes;
@@ -229,12 +220,12 @@ struct compat_msqid64_ds {
 
 struct compat_shmid64_ds {
 	struct compat_ipc64_perm shm_perm;
-	unsigned int	__pad1;
-	compat_time_t	shm_atime;
-	unsigned int	__pad2;
-	compat_time_t	shm_dtime;
-	unsigned int	__pad3;
-	compat_time_t	shm_ctime;
+	unsigned int	shm_atime_high;
+	unsigned int	shm_atime;
+	unsigned int	shm_dtime_high;
+	unsigned int	shm_dtime;
+	unsigned int	shm_ctime_high;
+	unsigned int	shm_ctime;
 	compat_size_t	shm_segsz;
 	compat_pid_t	shm_cpid;
 	compat_pid_t	shm_lpid;
@@ -243,6 +234,7 @@ struct compat_shmid64_ds {
 	unsigned int	__unused2;
 };
 
+#ifdef CONFIG_COMPAT
 static inline int is_compat_task(void)
 {
 	return test_thread_flag(TIF_32BIT);
@@ -254,5 +246,6 @@ static inline bool in_compat_syscall(void)
 	return pt_regs_trap_type(current_pt_regs()) == 0x110;
 }
 #define in_compat_syscall in_compat_syscall
+#endif
 
 #endif /* _ASM_SPARC64_COMPAT_H */
diff --git a/arch/sparc/include/asm/hardirq_64.h b/arch/sparc/include/asm/hardirq_64.h
index f56540271993..75b92bfe04b5 100644
--- a/arch/sparc/include/asm/hardirq_64.h
+++ b/arch/sparc/include/asm/hardirq_64.h
@@ -10,8 +10,9 @@
 #include <asm/cpudata.h>
 
 #define __ARCH_IRQ_STAT
-#define local_softirq_pending() \
-	(local_cpu_data().__softirq_pending)
+
+#define local_softirq_pending_ref \
+	__cpu_data.__softirq_pending
 
 void ack_bad_irq(unsigned int irq);
 
diff --git a/arch/sparc/include/asm/iommu-common.h b/arch/sparc/include/asm/iommu-common.h
new file mode 100644
index 000000000000..802c90c79d1f
--- /dev/null
+++ b/arch/sparc/include/asm/iommu-common.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_IOMMU_COMMON_H
+#define _LINUX_IOMMU_COMMON_H
+
+#include <linux/spinlock_types.h>
+#include <linux/device.h>
+#include <asm/page.h>
+
+#define IOMMU_POOL_HASHBITS     4
+#define IOMMU_NR_POOLS          (1 << IOMMU_POOL_HASHBITS)
+#define IOMMU_ERROR_CODE	(~(unsigned long) 0)
+
+struct iommu_pool {
+	unsigned long	start;
+	unsigned long	end;
+	unsigned long	hint;
+	spinlock_t	lock;
+};
+
+struct iommu_map_table {
+	unsigned long		table_map_base;
+	unsigned long		table_shift;
+	unsigned long		nr_pools;
+	void			(*lazy_flush)(struct iommu_map_table *);
+	unsigned long		poolsize;
+	struct iommu_pool	pools[IOMMU_NR_POOLS];
+	u32			flags;
+#define	IOMMU_HAS_LARGE_POOL	0x00000001
+#define	IOMMU_NO_SPAN_BOUND	0x00000002
+#define	IOMMU_NEED_FLUSH	0x00000004
+	struct iommu_pool	large_pool;
+	unsigned long		*map;
+};
+
+extern void iommu_tbl_pool_init(struct iommu_map_table *iommu,
+				unsigned long num_entries,
+				u32 table_shift,
+				void (*lazy_flush)(struct iommu_map_table *),
+				bool large_pool, u32 npools,
+				bool skip_span_boundary_check);
+
+extern unsigned long iommu_tbl_range_alloc(struct device *dev,
+					   struct iommu_map_table *iommu,
+					   unsigned long npages,
+					   unsigned long *handle,
+					   unsigned long mask,
+					   unsigned int align_order);
+
+extern void iommu_tbl_range_free(struct iommu_map_table *iommu,
+				 u64 dma_addr, unsigned long npages,
+				 unsigned long entry);
+
+#endif
diff --git a/arch/sparc/include/asm/iommu_64.h b/arch/sparc/include/asm/iommu_64.h
index 9ed6b54caa4b..0ef6dedf747e 100644
--- a/arch/sparc/include/asm/iommu_64.h
+++ b/arch/sparc/include/asm/iommu_64.h
@@ -17,7 +17,7 @@
 #define IOPTE_WRITE   0x0000000000000002UL
 
 #define IOMMU_NUM_CTXS	4096
-#include <linux/iommu-common.h>
+#include <asm/iommu-common.h>
 
 struct iommu_arena {
 	unsigned long	*map;
diff --git a/arch/sparc/include/asm/pci_32.h b/arch/sparc/include/asm/pci_32.h
index 98917e48727d..cfc0ee9476c6 100644
--- a/arch/sparc/include/asm/pci_32.h
+++ b/arch/sparc/include/asm/pci_32.h
@@ -17,10 +17,6 @@
 
 #define PCI_IRQ_NONE		0xffffffff
 
-/* Dynamic DMA mapping stuff.
- */
-#define PCI_DMA_BUS_IS_PHYS	(0)
-
 #endif /* __KERNEL__ */
 
 #ifndef CONFIG_LEON_PCI
diff --git a/arch/sparc/include/asm/pci_64.h b/arch/sparc/include/asm/pci_64.h
index 671274e36cfa..fac77813402c 100644
--- a/arch/sparc/include/asm/pci_64.h
+++ b/arch/sparc/include/asm/pci_64.h
@@ -17,12 +17,6 @@
 
 #define PCI_IRQ_NONE		0xffffffff
 
-/* The PCI address space does not equal the physical memory
- * address space.  The networking and block device layers use
- * this boolean for bounce buffer decisions.
- */
-#define PCI_DMA_BUS_IS_PHYS	(0)
-
 /* PCI IOMMU mapping bypass support. */
 
 /* PCI 64-bit addressing works for all slots on all controller
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 44d6ac47e035..1393a8ac596b 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -117,9 +117,6 @@ bool kern_addr_valid(unsigned long addr);
 #define _PAGE_PMD_HUGE    _AC(0x0100000000000000,UL) /* Huge page            */
 #define _PAGE_PUD_HUGE    _PAGE_PMD_HUGE
 
-/* Advertise support for _PAGE_SPECIAL */
-#define __HAVE_ARCH_PTE_SPECIAL
-
 /* SUN4U pte bits... */
 #define _PAGE_SZ4MB_4U	  _AC(0x6000000000000000,UL) /* 4MB Page             */
 #define _PAGE_SZ512K_4U	  _AC(0x4000000000000000,UL) /* 512K Page            */
diff --git a/arch/sparc/include/uapi/asm/jsflash.h b/arch/sparc/include/uapi/asm/jsflash.h
deleted file mode 100644
index 68c98a54281a..000000000000
--- a/arch/sparc/include/uapi/asm/jsflash.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-/*
- * jsflash.h: OS Flash SIMM support for JavaStations.
- *
- * Copyright (C) 1999  Pete Zaitcev
- */
-
-#ifndef _SPARC_JSFLASH_H
-#define _SPARC_JSFLASH_H
-
-#ifndef _SPARC_TYPES_H
-#include <linux/types.h>
-#endif
-
-/*
- * Semantics of the offset is a full address.
- * Hardcode it or get it from probe ioctl.
- *
- * We use full bus address, so that we would be
- * automatically compatible with possible future systems.
- */
-
-#define JSFLASH_IDENT   (('F'<<8)|54)
-struct jsflash_ident_arg {
-	__u64 off;                /* 0x20000000 is included */
-	__u32 size;
-	char name[32];		/* With trailing zero */
-};
-
-#define JSFLASH_ERASE   (('F'<<8)|55)
-/* Put 0 as argument, may be flags or sector number... */
-
-#define JSFLASH_PROGRAM (('F'<<8)|56)
-struct jsflash_program_arg {
-	__u64 data;		/* char* for sparc and sparc64 */
-	__u64 off;
-	__u32 size;
-};
-
-#endif /* _SPARC_JSFLASH_H */
diff --git a/arch/sparc/include/uapi/asm/msgbuf.h b/arch/sparc/include/uapi/asm/msgbuf.h
index b601c4f4d956..ffc46c211d6d 100644
--- a/arch/sparc/include/uapi/asm/msgbuf.h
+++ b/arch/sparc/include/uapi/asm/msgbuf.h
@@ -8,25 +8,22 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
  * - 2 miscellaneous 32-bit values
  */
-
-#if defined(__sparc__) && defined(__arch64__)
-# define PADDING(x)
-#else
-# define PADDING(x) unsigned int x;
-#endif
-
-
 struct msqid64_ds {
 	struct ipc64_perm msg_perm;
-	PADDING(__pad1)
+#if defined(__sparc__) && defined(__arch64__)
 	__kernel_time_t msg_stime;	/* last msgsnd time */
-	PADDING(__pad2)
 	__kernel_time_t msg_rtime;	/* last msgrcv time */
-	PADDING(__pad3)
 	__kernel_time_t msg_ctime;	/* last change time */
+#else
+	unsigned long msg_stime_high;
+	unsigned long msg_stime;	/* last msgsnd time */
+	unsigned long msg_rtime_high;
+	unsigned long msg_rtime;	/* last msgrcv time */
+	unsigned long msg_ctime_high;
+	unsigned long msg_ctime;	/* last change time */
+#endif
 	unsigned long  msg_cbytes;	/* current number of bytes on queue */
 	unsigned long  msg_qnum;	/* number of messages in queue */
 	unsigned long  msg_qbytes;	/* max number of bytes on queue */
@@ -35,5 +32,4 @@ struct msqid64_ds {
 	unsigned long  __unused1;
 	unsigned long  __unused2;
 };
-#undef PADDING
 #endif /* _SPARC_MSGBUF_H */
diff --git a/arch/sparc/include/uapi/asm/sembuf.h b/arch/sparc/include/uapi/asm/sembuf.h
index f49b0ffa0ab8..f3d309c2e1cd 100644
--- a/arch/sparc/include/uapi/asm/sembuf.h
+++ b/arch/sparc/include/uapi/asm/sembuf.h
@@ -8,25 +8,23 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
  * - 2 miscellaneous 32-bit values
  */
-#if defined(__sparc__) && defined(__arch64__)
-# define PADDING(x)
-#else
-# define PADDING(x) unsigned int x;
-#endif
 
 struct semid64_ds {
 	struct ipc64_perm sem_perm;		/* permissions .. see ipc.h */
-	PADDING(__pad1)
+#if defined(__sparc__) && defined(__arch64__)
 	__kernel_time_t	sem_otime;		/* last semop time */
-	PADDING(__pad2)
 	__kernel_time_t	sem_ctime;		/* last change time */
+#else
+	unsigned long	sem_otime_high;
+	unsigned long	sem_otime;		/* last semop time */
+	unsigned long	sem_ctime_high;
+	unsigned long	sem_ctime;		/* last change time */
+#endif
 	unsigned long	sem_nsems;		/* no. of semaphores in array */
 	unsigned long	__unused1;
 	unsigned long	__unused2;
 };
-#undef PADDING
 
 #endif /* _SPARC64_SEMBUF_H */
diff --git a/arch/sparc/include/uapi/asm/shmbuf.h b/arch/sparc/include/uapi/asm/shmbuf.h
index 286631db705c..06618b84822d 100644
--- a/arch/sparc/include/uapi/asm/shmbuf.h
+++ b/arch/sparc/include/uapi/asm/shmbuf.h
@@ -8,24 +8,23 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
  * - 2 miscellaneous 32-bit values
  */
 
-#if defined(__sparc__) && defined(__arch64__)
-# define PADDING(x)
-#else
-# define PADDING(x) unsigned int x;
-#endif
-
 struct shmid64_ds {
 	struct ipc64_perm	shm_perm;	/* operation perms */
-	PADDING(__pad1)
+#if defined(__sparc__) && defined(__arch64__)
 	__kernel_time_t		shm_atime;	/* last attach time */
-	PADDING(__pad2)
 	__kernel_time_t		shm_dtime;	/* last detach time */
-	PADDING(__pad3)
 	__kernel_time_t		shm_ctime;	/* last change time */
+#else
+	unsigned long		shm_atime_high;
+	unsigned long		shm_atime;	/* last attach time */
+	unsigned long		shm_dtime_high;
+	unsigned long		shm_dtime;	/* last detach time */
+	unsigned long		shm_ctime_high;
+	unsigned long		shm_ctime;	/* last change time */
+#endif
 	size_t			shm_segsz;	/* size of segment (bytes) */
 	__kernel_pid_t		shm_cpid;	/* pid of creator */
 	__kernel_pid_t		shm_lpid;	/* pid of last operator */
@@ -46,6 +45,4 @@ struct shminfo64 {
 	unsigned long	__unused4;
 };
 
-#undef PADDING
-
 #endif /* _SPARC_SHMBUF_H */
diff --git a/arch/sparc/include/uapi/asm/siginfo.h b/arch/sparc/include/uapi/asm/siginfo.h
index 896ce447d16a..e7049550ac82 100644
--- a/arch/sparc/include/uapi/asm/siginfo.h
+++ b/arch/sparc/include/uapi/asm/siginfo.h
@@ -17,13 +17,6 @@
 
 #define SI_NOINFO	32767		/* no information in siginfo_t */
 
-/*
- * SIGFPE si_codes
- */
-#ifdef __KERNEL__
-#define FPE_FIXME	0	/* Broken dup of SI_USER */
-#endif /* __KERNEL__ */
-
 /*
  * SIGEMT si_codes
  */
diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile
index 76cb57750dda..cf8640841b7a 100644
--- a/arch/sparc/kernel/Makefile
+++ b/arch/sparc/kernel/Makefile
@@ -59,7 +59,7 @@ obj-$(CONFIG_SPARC32)   += leon_pmc.o
 
 obj-$(CONFIG_SPARC64)   += reboot.o
 obj-$(CONFIG_SPARC64)   += sysfs.o
-obj-$(CONFIG_SPARC64)   += iommu.o
+obj-$(CONFIG_SPARC64)   += iommu.o iommu-common.o
 obj-$(CONFIG_SPARC64)   += central.o
 obj-$(CONFIG_SPARC64)   += starfire.o
 obj-$(CONFIG_SPARC64)   += power.o
@@ -74,8 +74,6 @@ obj-$(CONFIG_SPARC64)	+= pcr.o
 obj-$(CONFIG_SPARC64)	+= nmi.o
 obj-$(CONFIG_SPARC64_SMP) += cpumap.o
 
-obj-y                     += dma.o
-
 obj-$(CONFIG_PCIC_PCI)    += pcic.o
 obj-$(CONFIG_LEON_PCI)    += leon_pci.o
 obj-$(CONFIG_SPARC_GRPCI2)+= leon_pci_grpci2.o
diff --git a/arch/sparc/kernel/dma.c b/arch/sparc/kernel/dma.c
deleted file mode 100644
index f73e7597c971..000000000000
--- a/arch/sparc/kernel/dma.c
+++ /dev/null
@@ -1,13 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/kernel.h>
-#include <linux/dma-mapping.h>
-#include <linux/dma-debug.h>
-
-#define PREALLOC_DMA_DEBUG_ENTRIES       (1 << 15)
-
-static int __init dma_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-	return 0;
-}
-fs_initcall(dma_init);
diff --git a/arch/sparc/kernel/iommu-common.c b/arch/sparc/kernel/iommu-common.c
new file mode 100644
index 000000000000..59cb16691322
--- /dev/null
+++ b/arch/sparc/kernel/iommu-common.c
@@ -0,0 +1,264 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * IOMMU mmap management and range allocation functions.
+ * Based almost entirely upon the powerpc iommu allocator.
+ */
+
+#include <linux/export.h>
+#include <linux/bitmap.h>
+#include <linux/bug.h>
+#include <linux/iommu-helper.h>
+#include <linux/dma-mapping.h>
+#include <linux/hash.h>
+#include <asm/iommu-common.h>
+
+static unsigned long iommu_large_alloc = 15;
+
+static	DEFINE_PER_CPU(unsigned int, iommu_hash_common);
+
+static inline bool need_flush(struct iommu_map_table *iommu)
+{
+	return ((iommu->flags & IOMMU_NEED_FLUSH) != 0);
+}
+
+static inline void set_flush(struct iommu_map_table *iommu)
+{
+	iommu->flags |= IOMMU_NEED_FLUSH;
+}
+
+static inline void clear_flush(struct iommu_map_table *iommu)
+{
+	iommu->flags &= ~IOMMU_NEED_FLUSH;
+}
+
+static void setup_iommu_pool_hash(void)
+{
+	unsigned int i;
+	static bool do_once;
+
+	if (do_once)
+		return;
+	do_once = true;
+	for_each_possible_cpu(i)
+		per_cpu(iommu_hash_common, i) = hash_32(i, IOMMU_POOL_HASHBITS);
+}
+
+/*
+ * Initialize iommu_pool entries for the iommu_map_table. `num_entries'
+ * is the number of table entries. If `large_pool' is set to true,
+ * the top 1/4 of the table will be set aside for pool allocations
+ * of more than iommu_large_alloc pages.
+ */
+void iommu_tbl_pool_init(struct iommu_map_table *iommu,
+			 unsigned long num_entries,
+			 u32 table_shift,
+			 void (*lazy_flush)(struct iommu_map_table *),
+			 bool large_pool, u32 npools,
+			 bool skip_span_boundary_check)
+{
+	unsigned int start, i;
+	struct iommu_pool *p = &(iommu->large_pool);
+
+	setup_iommu_pool_hash();
+	if (npools == 0)
+		iommu->nr_pools = IOMMU_NR_POOLS;
+	else
+		iommu->nr_pools = npools;
+	BUG_ON(npools > IOMMU_NR_POOLS);
+
+	iommu->table_shift = table_shift;
+	iommu->lazy_flush = lazy_flush;
+	start = 0;
+	if (skip_span_boundary_check)
+		iommu->flags |= IOMMU_NO_SPAN_BOUND;
+	if (large_pool)
+		iommu->flags |= IOMMU_HAS_LARGE_POOL;
+
+	if (!large_pool)
+		iommu->poolsize = num_entries/iommu->nr_pools;
+	else
+		iommu->poolsize = (num_entries * 3 / 4)/iommu->nr_pools;
+	for (i = 0; i < iommu->nr_pools; i++) {
+		spin_lock_init(&(iommu->pools[i].lock));
+		iommu->pools[i].start = start;
+		iommu->pools[i].hint = start;
+		start += iommu->poolsize; /* start for next pool */
+		iommu->pools[i].end = start - 1;
+	}
+	if (!large_pool)
+		return;
+	/* initialize large_pool */
+	spin_lock_init(&(p->lock));
+	p->start = start;
+	p->hint = p->start;
+	p->end = num_entries;
+}
+
+unsigned long iommu_tbl_range_alloc(struct device *dev,
+				struct iommu_map_table *iommu,
+				unsigned long npages,
+				unsigned long *handle,
+				unsigned long mask,
+				unsigned int align_order)
+{
+	unsigned int pool_hash = __this_cpu_read(iommu_hash_common);
+	unsigned long n, end, start, limit, boundary_size;
+	struct iommu_pool *pool;
+	int pass = 0;
+	unsigned int pool_nr;
+	unsigned int npools = iommu->nr_pools;
+	unsigned long flags;
+	bool large_pool = ((iommu->flags & IOMMU_HAS_LARGE_POOL) != 0);
+	bool largealloc = (large_pool && npages > iommu_large_alloc);
+	unsigned long shift;
+	unsigned long align_mask = 0;
+
+	if (align_order > 0)
+		align_mask = ~0ul >> (BITS_PER_LONG - align_order);
+
+	/* Sanity check */
+	if (unlikely(npages == 0)) {
+		WARN_ON_ONCE(1);
+		return IOMMU_ERROR_CODE;
+	}
+
+	if (largealloc) {
+		pool = &(iommu->large_pool);
+		pool_nr = 0; /* to keep compiler happy */
+	} else {
+		/* pick out pool_nr */
+		pool_nr =  pool_hash & (npools - 1);
+		pool = &(iommu->pools[pool_nr]);
+	}
+	spin_lock_irqsave(&pool->lock, flags);
+
+ again:
+	if (pass == 0 && handle && *handle &&
+	    (*handle >= pool->start) && (*handle < pool->end))
+		start = *handle;
+	else
+		start = pool->hint;
+
+	limit = pool->end;
+
+	/* The case below can happen if we have a small segment appended
+	 * to a large, or when the previous alloc was at the very end of
+	 * the available space. If so, go back to the beginning. If a
+	 * flush is needed, it will get done based on the return value
+	 * from iommu_area_alloc() below.
+	 */
+	if (start >= limit)
+		start = pool->start;
+	shift = iommu->table_map_base >> iommu->table_shift;
+	if (limit + shift > mask) {
+		limit = mask - shift + 1;
+		/* If we're constrained on address range, first try
+		 * at the masked hint to avoid O(n) search complexity,
+		 * but on second pass, start at 0 in pool 0.
+		 */
+		if ((start & mask) >= limit || pass > 0) {
+			spin_unlock(&(pool->lock));
+			pool = &(iommu->pools[0]);
+			spin_lock(&(pool->lock));
+			start = pool->start;
+		} else {
+			start &= mask;
+		}
+	}
+
+	if (dev)
+		boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
+				      1 << iommu->table_shift);
+	else
+		boundary_size = ALIGN(1ULL << 32, 1 << iommu->table_shift);
+
+	boundary_size = boundary_size >> iommu->table_shift;
+	/*
+	 * if the skip_span_boundary_check had been set during init, we set
+	 * things up so that iommu_is_span_boundary() merely checks if the
+	 * (index + npages) < num_tsb_entries
+	 */
+	if ((iommu->flags & IOMMU_NO_SPAN_BOUND) != 0) {
+		shift = 0;
+		boundary_size = iommu->poolsize * iommu->nr_pools;
+	}
+	n = iommu_area_alloc(iommu->map, limit, start, npages, shift,
+			     boundary_size, align_mask);
+	if (n == -1) {
+		if (likely(pass == 0)) {
+			/* First failure, rescan from the beginning.  */
+			pool->hint = pool->start;
+			set_flush(iommu);
+			pass++;
+			goto again;
+		} else if (!largealloc && pass <= iommu->nr_pools) {
+			spin_unlock(&(pool->lock));
+			pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1);
+			pool = &(iommu->pools[pool_nr]);
+			spin_lock(&(pool->lock));
+			pool->hint = pool->start;
+			set_flush(iommu);
+			pass++;
+			goto again;
+		} else {
+			/* give up */
+			n = IOMMU_ERROR_CODE;
+			goto bail;
+		}
+	}
+	if (iommu->lazy_flush &&
+	    (n < pool->hint || need_flush(iommu))) {
+		clear_flush(iommu);
+		iommu->lazy_flush(iommu);
+	}
+
+	end = n + npages;
+	pool->hint = end;
+
+	/* Update handle for SG allocations */
+	if (handle)
+		*handle = end;
+bail:
+	spin_unlock_irqrestore(&(pool->lock), flags);
+
+	return n;
+}
+
+static struct iommu_pool *get_pool(struct iommu_map_table *tbl,
+				   unsigned long entry)
+{
+	struct iommu_pool *p;
+	unsigned long largepool_start = tbl->large_pool.start;
+	bool large_pool = ((tbl->flags & IOMMU_HAS_LARGE_POOL) != 0);
+
+	/* The large pool is the last pool at the top of the table */
+	if (large_pool && entry >= largepool_start) {
+		p = &tbl->large_pool;
+	} else {
+		unsigned int pool_nr = entry / tbl->poolsize;
+
+		BUG_ON(pool_nr >= tbl->nr_pools);
+		p = &tbl->pools[pool_nr];
+	}
+	return p;
+}
+
+/* Caller supplies the index of the entry into the iommu map table
+ * itself when the mapping from dma_addr to the entry is not the
+ * default addr->entry mapping below.
+ */
+void iommu_tbl_range_free(struct iommu_map_table *iommu, u64 dma_addr,
+			  unsigned long npages, unsigned long entry)
+{
+	struct iommu_pool *pool;
+	unsigned long flags;
+	unsigned long shift = iommu->table_shift;
+
+	if (entry == IOMMU_ERROR_CODE) /* use default addr->entry mapping */
+		entry = (dma_addr - iommu->table_map_base) >> shift;
+	pool = get_pool(iommu, entry);
+
+	spin_lock_irqsave(&(pool->lock), flags);
+	bitmap_clear(iommu->map, entry, npages);
+	spin_unlock_irqrestore(&(pool->lock), flags);
+}
diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c
index b08dc3416f06..40d008b0bd3e 100644
--- a/arch/sparc/kernel/iommu.c
+++ b/arch/sparc/kernel/iommu.c
@@ -14,7 +14,7 @@
 #include <linux/errno.h>
 #include <linux/iommu-helper.h>
 #include <linux/bitmap.h>
-#include <linux/iommu-common.h>
+#include <asm/iommu-common.h>
 
 #ifdef CONFIG_PCI
 #include <linux/pci.h>
diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c
index 3bcef9ce74df..cca9134cfa7d 100644
--- a/arch/sparc/kernel/ioport.c
+++ b/arch/sparc/kernel/ioport.c
@@ -678,25 +678,14 @@ static int sparc_io_proc_show(struct seq_file *m, void *v)
 
 	return 0;
 }
-
-static int sparc_io_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, sparc_io_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations sparc_io_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= sparc_io_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
 #endif /* CONFIG_PROC_FS */
 
 static void register_proc_sparc_ioport(void)
 {
 #ifdef CONFIG_PROC_FS
-	proc_create_data("io_map", 0, NULL, &sparc_io_proc_fops, &sparc_iomap);
-	proc_create_data("dvma_map", 0, NULL, &sparc_io_proc_fops, &_sparc_dvma);
+	proc_create_single_data("io_map", 0, NULL, sparc_io_proc_show,
+			&sparc_iomap);
+	proc_create_single_data("dvma_map", 0, NULL, sparc_io_proc_show,
+			&_sparc_dvma);
 #endif
 }
diff --git a/arch/sparc/kernel/ldc.c b/arch/sparc/kernel/ldc.c
index 86b625f9d8dc..c0fa3ef6cf01 100644
--- a/arch/sparc/kernel/ldc.c
+++ b/arch/sparc/kernel/ldc.c
@@ -16,7 +16,7 @@
 #include <linux/list.h>
 #include <linux/init.h>
 #include <linux/bitmap.h>
-#include <linux/iommu-common.h>
+#include <asm/iommu-common.h>
 
 #include <asm/hypervisor.h>
 #include <asm/iommu.h>
diff --git a/arch/sparc/kernel/leon_pci.c b/arch/sparc/kernel/leon_pci.c
index 15b59169c535..e5e5ff6b9a5c 100644
--- a/arch/sparc/kernel/leon_pci.c
+++ b/arch/sparc/kernel/leon_pci.c
@@ -60,50 +60,30 @@ void leon_pci_init(struct platform_device *ofdev, struct leon_pci_info *info)
 	pci_bus_add_devices(root_bus);
 }
 
-void pcibios_fixup_bus(struct pci_bus *pbus)
+int pcibios_enable_device(struct pci_dev *dev, int mask)
 {
-	struct pci_dev *dev;
-	int i, has_io, has_mem;
-	u16 cmd;
+	u16 cmd, oldcmd;
+	int i;
 
-	list_for_each_entry(dev, &pbus->devices, bus_list) {
-		/*
-		 * We can not rely on that the bootloader has enabled I/O
-		 * or memory access to PCI devices. Instead we enable it here
-		 * if the device has BARs of respective type.
-		 */
-		has_io = has_mem = 0;
-		for (i = 0; i < PCI_ROM_RESOURCE; i++) {
-			unsigned long f = dev->resource[i].flags;
-			if (f & IORESOURCE_IO)
-				has_io = 1;
-			else if (f & IORESOURCE_MEM)
-				has_mem = 1;
-		}
-		/* ROM BARs are mapped into 32-bit memory space */
-		if (dev->resource[PCI_ROM_RESOURCE].end != 0) {
-			dev->resource[PCI_ROM_RESOURCE].flags |=
-							IORESOURCE_ROM_ENABLE;
-			has_mem = 1;
-		}
-		pci_bus_read_config_word(pbus, dev->devfn, PCI_COMMAND, &cmd);
-		if (has_io && !(cmd & PCI_COMMAND_IO)) {
-#ifdef CONFIG_PCI_DEBUG
-			printk(KERN_INFO "LEONPCI: Enabling I/O for dev %s\n",
-					 pci_name(dev));
-#endif
+	pci_read_config_word(dev, PCI_COMMAND, &cmd);
+	oldcmd = cmd;
+
+	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
+		struct resource *res = &dev->resource[i];
+
+		/* Only set up the requested stuff */
+		if (!(mask & (1<<i)))
+			continue;
+
+		if (res->flags & IORESOURCE_IO)
 			cmd |= PCI_COMMAND_IO;
-			pci_bus_write_config_word(pbus, dev->devfn, PCI_COMMAND,
-									cmd);
-		}
-		if (has_mem && !(cmd & PCI_COMMAND_MEMORY)) {
-#ifdef CONFIG_PCI_DEBUG
-			printk(KERN_INFO "LEONPCI: Enabling MEMORY for dev"
-					 "%s\n", pci_name(dev));
-#endif
+		if (res->flags & IORESOURCE_MEM)
 			cmd |= PCI_COMMAND_MEMORY;
-			pci_bus_write_config_word(pbus, dev->devfn, PCI_COMMAND,
-									cmd);
-		}
 	}
+
+	if (cmd != oldcmd) {
+		pci_info(dev, "enabling device (%04x -> %04x)\n", oldcmd, cmd);
+		pci_write_config_word(dev, PCI_COMMAND, cmd);
+	}
+	return 0;
 }
diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c
index 41b20edb427d..17ea16a1337c 100644
--- a/arch/sparc/kernel/pci.c
+++ b/arch/sparc/kernel/pci.c
@@ -214,8 +214,8 @@ static void pci_parse_of_addrs(struct platform_device *op,
 	if (!addrs)
 		return;
 	if (ofpci_verbose)
-		printk("    parse addresses (%d bytes) @ %p\n",
-		       proplen, addrs);
+		pci_info(dev, "    parse addresses (%d bytes) @ %p\n",
+			 proplen, addrs);
 	op_res = &op->resource[0];
 	for (; proplen >= 20; proplen -= 20, addrs += 5, op_res++) {
 		struct resource *res;
@@ -227,8 +227,8 @@ static void pci_parse_of_addrs(struct platform_device *op,
 			continue;
 		i = addrs[0] & 0xff;
 		if (ofpci_verbose)
-			printk("  start: %llx, end: %llx, i: %x\n",
-			       op_res->start, op_res->end, i);
+			pci_info(dev, "  start: %llx, end: %llx, i: %x\n",
+				 op_res->start, op_res->end, i);
 
 		if (PCI_BASE_ADDRESS_0 <= i && i <= PCI_BASE_ADDRESS_5) {
 			res = &dev->resource[(i - PCI_BASE_ADDRESS_0) >> 2];
@@ -236,13 +236,15 @@ static void pci_parse_of_addrs(struct platform_device *op,
 			res = &dev->resource[PCI_ROM_RESOURCE];
 			flags |= IORESOURCE_READONLY | IORESOURCE_SIZEALIGN;
 		} else {
-			printk(KERN_ERR "PCI: bad cfg reg num 0x%x\n", i);
+			pci_err(dev, "bad cfg reg num 0x%x\n", i);
 			continue;
 		}
 		res->start = op_res->start;
 		res->end = op_res->end;
 		res->flags = flags;
 		res->name = pci_name(dev);
+
+		pci_info(dev, "reg 0x%x: %pR\n", i, res);
 	}
 }
 
@@ -289,8 +291,8 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm,
 		type = "";
 
 	if (ofpci_verbose)
-		printk("    create device, devfn: %x, type: %s\n",
-		       devfn, type);
+		pci_info(bus,"    create device, devfn: %x, type: %s\n",
+			 devfn, type);
 
 	dev->sysdata = node;
 	dev->dev.parent = bus->bridge;
@@ -323,10 +325,6 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm,
 	dev_set_name(&dev->dev, "%04x:%02x:%02x.%d", pci_domain_nr(bus),
 		dev->bus->number, PCI_SLOT(devfn), PCI_FUNC(devfn));
 
-	if (ofpci_verbose)
-		printk("    class: 0x%x device name: %s\n",
-		       dev->class, pci_name(dev));
-
 	/* I have seen IDE devices which will not respond to
 	 * the bmdma simplex check reads if bus mastering is
 	 * disabled.
@@ -353,10 +351,13 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm,
 			dev->irq = PCI_IRQ_NONE;
 	}
 
+	pci_info(dev, "[%04x:%04x] type %02x class %#08x\n",
+		 dev->vendor, dev->device, dev->hdr_type, dev->class);
+
 	pci_parse_of_addrs(sd->op, node, dev);
 
 	if (ofpci_verbose)
-		printk("    adding to system ...\n");
+		pci_info(dev, "    adding to system ...\n");
 
 	pci_device_add(dev, bus);
 
@@ -430,19 +431,19 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm,
 	u64 size;
 
 	if (ofpci_verbose)
-		printk("of_scan_pci_bridge(%s)\n", node->full_name);
+		pci_info(dev, "of_scan_pci_bridge(%s)\n", node->full_name);
 
 	/* parse bus-range property */
 	busrange = of_get_property(node, "bus-range", &len);
 	if (busrange == NULL || len != 8) {
-		printk(KERN_DEBUG "Can't get bus-range for PCI-PCI bridge %s\n",
+		pci_info(dev, "Can't get bus-range for PCI-PCI bridge %s\n",
 		       node->full_name);
 		return;
 	}
 
 	if (ofpci_verbose)
-		printk("    Bridge bus range [%u --> %u]\n",
-		       busrange[0], busrange[1]);
+		pci_info(dev, "    Bridge bus range [%u --> %u]\n",
+			 busrange[0], busrange[1]);
 
 	ranges = of_get_property(node, "ranges", &len);
 	simba = 0;
@@ -454,8 +455,8 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm,
 
 	bus = pci_add_new_bus(dev->bus, dev, busrange[0]);
 	if (!bus) {
-		printk(KERN_ERR "Failed to create pci bus for %s\n",
-		       node->full_name);
+		pci_err(dev, "Failed to create pci bus for %s\n",
+			node->full_name);
 		return;
 	}
 
@@ -464,8 +465,8 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm,
 	bus->bridge_ctl = 0;
 
 	if (ofpci_verbose)
-		printk("    Bridge ranges[%p] simba[%d]\n",
-		       ranges, simba);
+		pci_info(dev, "    Bridge ranges[%p] simba[%d]\n",
+			 ranges, simba);
 
 	/* parse ranges property, or cook one up by hand for Simba */
 	/* PCI #address-cells == 3 and #size-cells == 2 always */
@@ -487,10 +488,10 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm,
 		u64 start;
 
 		if (ofpci_verbose)
-			printk("    RAW Range[%08x:%08x:%08x:%08x:%08x:%08x:"
-			       "%08x:%08x]\n",
-			       ranges[0], ranges[1], ranges[2], ranges[3],
-			       ranges[4], ranges[5], ranges[6], ranges[7]);
+			pci_info(dev, "    RAW Range[%08x:%08x:%08x:%08x:%08x:%08x:"
+				 "%08x:%08x]\n",
+				 ranges[0], ranges[1], ranges[2], ranges[3],
+				 ranges[4], ranges[5], ranges[6], ranges[7]);
 
 		flags = pci_parse_of_flags(ranges[0]);
 		size = GET_64BIT(ranges, 6);
@@ -510,14 +511,14 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm,
 		if (flags & IORESOURCE_IO) {
 			res = bus->resource[0];
 			if (res->flags) {
-				printk(KERN_ERR "PCI: ignoring extra I/O range"
-				       " for bridge %s\n", node->full_name);
+				pci_err(dev, "ignoring extra I/O range"
+					" for bridge %s\n", node->full_name);
 				continue;
 			}
 		} else {
 			if (i >= PCI_NUM_RESOURCES - PCI_BRIDGE_RESOURCES) {
-				printk(KERN_ERR "PCI: too many memory ranges"
-				       " for bridge %s\n", node->full_name);
+				pci_err(dev, "too many memory ranges"
+					" for bridge %s\n", node->full_name);
 				continue;
 			}
 			res = bus->resource[i];
@@ -529,8 +530,8 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm,
 		region.end = region.start + size - 1;
 
 		if (ofpci_verbose)
-			printk("      Using flags[%08x] start[%016llx] size[%016llx]\n",
-			       flags, start, size);
+			pci_info(dev, "      Using flags[%08x] start[%016llx] size[%016llx]\n",
+				 flags, start, size);
 
 		pcibios_bus_to_resource(dev->bus, res, &region);
 	}
@@ -538,7 +539,7 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm,
 	sprintf(bus->name, "PCI Bus %04x:%02x", pci_domain_nr(bus),
 		bus->number);
 	if (ofpci_verbose)
-		printk("    bus name: %s\n", bus->name);
+		pci_info(dev, "    bus name: %s\n", bus->name);
 
 	pci_of_scan_bus(pbm, node, bus);
 }
@@ -553,14 +554,14 @@ static void pci_of_scan_bus(struct pci_pbm_info *pbm,
 	struct pci_dev *dev;
 
 	if (ofpci_verbose)
-		printk("PCI: scan_bus[%s] bus no %d\n",
-		       node->full_name, bus->number);
+		pci_info(bus, "scan_bus[%s] bus no %d\n",
+			 node->full_name, bus->number);
 
 	child = NULL;
 	prev_devfn = -1;
 	while ((child = of_get_next_child(node, child)) != NULL) {
 		if (ofpci_verbose)
-			printk("  * %s\n", child->full_name);
+			pci_info(bus, "  * %s\n", child->full_name);
 		reg = of_get_property(child, "reg", &reglen);
 		if (reg == NULL || reglen < 20)
 			continue;
@@ -581,8 +582,7 @@ static void pci_of_scan_bus(struct pci_pbm_info *pbm,
 		if (!dev)
 			continue;
 		if (ofpci_verbose)
-			printk("PCI: dev header type: %x\n",
-			       dev->hdr_type);
+			pci_info(dev, "dev header type: %x\n", dev->hdr_type);
 
 		if (pci_is_bridge(dev))
 			of_scan_pci_bridge(pbm, child, dev);
@@ -624,6 +624,45 @@ static void pci_bus_register_of_sysfs(struct pci_bus *bus)
 		pci_bus_register_of_sysfs(child_bus);
 }
 
+static void pci_claim_legacy_resources(struct pci_dev *dev)
+{
+	struct pci_bus_region region;
+	struct resource *p, *root, *conflict;
+
+	if ((dev->class >> 8) != PCI_CLASS_DISPLAY_VGA)
+		return;
+
+	p = kzalloc(sizeof(*p), GFP_KERNEL);
+	if (!p)
+		return;
+
+	p->name = "Video RAM area";
+	p->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
+
+	region.start = 0xa0000UL;
+	region.end = region.start + 0x1ffffUL;
+	pcibios_bus_to_resource(dev->bus, p, &region);
+
+	root = pci_find_parent_resource(dev, p);
+	if (!root) {
+		pci_info(dev, "can't claim VGA legacy %pR: no compatible bridge window\n", p);
+		goto err;
+	}
+
+	conflict = request_resource_conflict(root, p);
+	if (conflict) {
+		pci_info(dev, "can't claim VGA legacy %pR: address conflict with %s %pR\n",
+			 p, conflict->name, conflict);
+		goto err;
+	}
+
+	pci_info(dev, "VGA legacy framebuffer %pR\n", p);
+	return;
+
+err:
+	kfree(p);
+}
+
 static void pci_claim_bus_resources(struct pci_bus *bus)
 {
 	struct pci_bus *child_bus;
@@ -639,15 +678,13 @@ static void pci_claim_bus_resources(struct pci_bus *bus)
 				continue;
 
 			if (ofpci_verbose)
-				printk("PCI: Claiming %s: "
-				       "Resource %d: %016llx..%016llx [%x]\n",
-				       pci_name(dev), i,
-				       (unsigned long long)r->start,
-				       (unsigned long long)r->end,
-				       (unsigned int)r->flags);
+				pci_info(dev, "Claiming Resource %d: %pR\n",
+					 i, r);
 
 			pci_claim_resource(dev, i);
 		}
+
+		pci_claim_legacy_resources(dev);
 	}
 
 	list_for_each_entry(child_bus, &bus->children, node)
@@ -687,6 +724,7 @@ struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm,
 	pci_bus_register_of_sysfs(bus);
 
 	pci_claim_bus_resources(bus);
+
 	pci_bus_add_devices(bus);
 	return bus;
 }
@@ -713,9 +751,7 @@ int pcibios_enable_device(struct pci_dev *dev, int mask)
 	}
 
 	if (cmd != oldcmd) {
-		printk(KERN_DEBUG "PCI: Enabling device: (%s), cmd %x\n",
-		       pci_name(dev), cmd);
-                /* Enable the appropriate bits in the PCI command register.  */
+		pci_info(dev, "enabling device (%04x -> %04x)\n", oldcmd, cmd);
 		pci_write_config_word(dev, PCI_COMMAND, cmd);
 	}
 	return 0;
@@ -1075,8 +1111,8 @@ static void pci_bus_slot_names(struct device_node *node, struct pci_bus *bus)
 	sp = prop->names;
 
 	if (ofpci_verbose)
-		printk("PCI: Making slots for [%s] mask[0x%02x]\n",
-		       node->full_name, mask);
+		pci_info(bus, "Making slots for [%s] mask[0x%02x]\n",
+			 node->full_name, mask);
 
 	i = 0;
 	while (mask) {
@@ -1089,12 +1125,12 @@ static void pci_bus_slot_names(struct device_node *node, struct pci_bus *bus)
 		}
 
 		if (ofpci_verbose)
-			printk("PCI: Making slot [%s]\n", sp);
+			pci_info(bus, "Making slot [%s]\n", sp);
 
 		pci_slot = pci_create_slot(bus, i, sp, NULL);
 		if (IS_ERR(pci_slot))
-			printk(KERN_ERR "PCI: pci_create_slot returned %ld\n",
-			       PTR_ERR(pci_slot));
+			pci_err(bus, "pci_create_slot returned %ld\n",
+				PTR_ERR(pci_slot));
 
 		sp += strlen(sp) + 1;
 		mask &= ~this_bit;
diff --git a/arch/sparc/kernel/pci_common.c b/arch/sparc/kernel/pci_common.c
index 38d46bcc8634..4759ccd542fe 100644
--- a/arch/sparc/kernel/pci_common.c
+++ b/arch/sparc/kernel/pci_common.c
@@ -329,23 +329,6 @@ void pci_get_pbm_props(struct pci_pbm_info *pbm)
 	}
 }
 
-static void pci_register_legacy_regions(struct resource *io_res,
-					struct resource *mem_res)
-{
-	struct resource *p;
-
-	/* VGA Video RAM. */
-	p = kzalloc(sizeof(*p), GFP_KERNEL);
-	if (!p)
-		return;
-
-	p->name = "Video RAM area";
-	p->start = mem_res->start + 0xa0000UL;
-	p->end = p->start + 0x1ffffUL;
-	p->flags = IORESOURCE_BUSY;
-	request_resource(mem_res, p);
-}
-
 static void pci_register_iommu_region(struct pci_pbm_info *pbm)
 {
 	const u32 *vdma = of_get_property(pbm->op->dev.of_node, "virtual-dma",
@@ -487,8 +470,6 @@ void pci_determine_mem_io_space(struct pci_pbm_info *pbm)
 	if (pbm->mem64_space.flags)
 		request_resource(&iomem_resource, &pbm->mem64_space);
 
-	pci_register_legacy_regions(&pbm->io_space,
-				    &pbm->mem_space);
 	pci_register_iommu_region(pbm);
 }
 
@@ -508,8 +489,8 @@ void pci_scan_for_target_abort(struct pci_pbm_info *pbm,
 				   PCI_STATUS_REC_TARGET_ABORT));
 		if (error_bits) {
 			pci_write_config_word(pdev, PCI_STATUS, error_bits);
-			printk("%s: Device %s saw Target Abort [%016x]\n",
-			       pbm->name, pci_name(pdev), status);
+			pci_info(pdev, "%s: Device saw Target Abort [%016x]\n",
+				 pbm->name, status);
 		}
 	}
 
@@ -531,8 +512,8 @@ void pci_scan_for_master_abort(struct pci_pbm_info *pbm,
 			(status & (PCI_STATUS_REC_MASTER_ABORT));
 		if (error_bits) {
 			pci_write_config_word(pdev, PCI_STATUS, error_bits);
-			printk("%s: Device %s received Master Abort [%016x]\n",
-			       pbm->name, pci_name(pdev), status);
+			pci_info(pdev, "%s: Device received Master Abort "
+				 "[%016x]\n", pbm->name, status);
 		}
 	}
 
@@ -555,8 +536,8 @@ void pci_scan_for_parity_error(struct pci_pbm_info *pbm,
 				   PCI_STATUS_DETECTED_PARITY));
 		if (error_bits) {
 			pci_write_config_word(pdev, PCI_STATUS, error_bits);
-			printk("%s: Device %s saw Parity Error [%016x]\n",
-			       pbm->name, pci_name(pdev), status);
+			pci_info(pdev, "%s: Device saw Parity Error [%016x]\n",
+				 pbm->name, status);
 		}
 	}
 
diff --git a/arch/sparc/kernel/pci_msi.c b/arch/sparc/kernel/pci_msi.c
index 1994d7638406..fb5899cbfa51 100644
--- a/arch/sparc/kernel/pci_msi.c
+++ b/arch/sparc/kernel/pci_msi.c
@@ -191,8 +191,8 @@ static void sparc64_teardown_msi_irq(unsigned int irq,
 			break;
 	}
 	if (i >= pbm->msi_num) {
-		printk(KERN_ERR "%s: teardown: No MSI for irq %u\n",
-		       pbm->name, irq);
+		pci_err(pdev, "%s: teardown: No MSI for irq %u\n", pbm->name,
+			irq);
 		return;
 	}
 
@@ -201,9 +201,9 @@ static void sparc64_teardown_msi_irq(unsigned int irq,
 
 	err = ops->msi_teardown(pbm, msi_num);
 	if (err) {
-		printk(KERN_ERR "%s: teardown: ops->teardown() on MSI %u, "
-		       "irq %u, gives error %d\n",
-		       pbm->name, msi_num, irq, err);
+		pci_err(pdev, "%s: teardown: ops->teardown() on MSI %u, "
+			"irq %u, gives error %d\n", pbm->name, msi_num, irq,
+			err);
 		return;
 	}
 
diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c
index 249367228c33..565d9ac883d0 100644
--- a/arch/sparc/kernel/pci_sun4v.c
+++ b/arch/sparc/kernel/pci_sun4v.c
@@ -16,7 +16,7 @@
 #include <linux/export.h>
 #include <linux/log2.h>
 #include <linux/of_device.h>
-#include <linux/iommu-common.h>
+#include <asm/iommu-common.h>
 
 #include <asm/iommu.h>
 #include <asm/irq.h>
diff --git a/arch/sparc/kernel/pcic.c b/arch/sparc/kernel/pcic.c
index 22f8774977d5..ee4c9a9a171c 100644
--- a/arch/sparc/kernel/pcic.c
+++ b/arch/sparc/kernel/pcic.c
@@ -518,10 +518,10 @@ static void pcic_map_pci_device(struct linux_pcic *pcic,
 				 * board in a PCI slot. We must remap it
 				 * under 64K but it is not done yet. XXX
 				 */
-				printk("PCIC: Skipping I/O space at 0x%lx, "
-				    "this will Oops if a driver attaches "
-				    "device '%s' at %02x:%02x)\n", address,
-				    namebuf, dev->bus->number, dev->devfn);
+				pci_info(dev, "PCIC: Skipping I/O space at "
+					 "0x%lx, this will Oops if a driver "
+					 "attaches device '%s'\n", address,
+					 namebuf);
 			}
 		}
 	}
@@ -551,8 +551,8 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node)
 		p++;
 	}
 	if (i >= pcic->pcic_imdim) {
-		printk("PCIC: device %s devfn %02x:%02x not found in %d\n",
-		    namebuf, dev->bus->number, dev->devfn, pcic->pcic_imdim);
+		pci_info(dev, "PCIC: device %s not found in %d\n", namebuf,
+			 pcic->pcic_imdim);
 		dev->irq = 0;
 		return;
 	}
@@ -565,7 +565,7 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node)
 		ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_HI);
 		real_irq = ivec >> ((i-4) << 2) & 0xF;
 	} else {					/* Corrupted map */
-		printk("PCIC: BAD PIN %d\n", i); for (;;) {}
+		pci_info(dev, "PCIC: BAD PIN %d\n", i); for (;;) {}
 	}
 /* P3 */ /* printk("PCIC: device %s pin %d ivec 0x%x irq %x\n", namebuf, i, ivec, dev->irq); */
 
@@ -574,10 +574,10 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node)
 	 */
 	if (real_irq == 0 || p->force) {
 		if (p->irq == 0 || p->irq >= 15) {	/* Corrupted map */
-			printk("PCIC: BAD IRQ %d\n", p->irq); for (;;) {}
+			pci_info(dev, "PCIC: BAD IRQ %d\n", p->irq); for (;;) {}
 		}
-		printk("PCIC: setting irq %d at pin %d for device %02x:%02x\n",
-		    p->irq, p->pin, dev->bus->number, dev->devfn);
+		pci_info(dev, "PCIC: setting irq %d at pin %d\n", p->irq,
+			 p->pin);
 		real_irq = p->irq;
 
 		i = p->pin;
@@ -602,15 +602,13 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node)
 void pcibios_fixup_bus(struct pci_bus *bus)
 {
 	struct pci_dev *dev;
-	int i, has_io, has_mem;
-	unsigned int cmd = 0;
 	struct linux_pcic *pcic;
 	/* struct linux_pbm_info* pbm = &pcic->pbm; */
 	int node;
 	struct pcidev_cookie *pcp;
 
 	if (!pcic0_up) {
-		printk("pcibios_fixup_bus: no PCIC\n");
+		pci_info(bus, "pcibios_fixup_bus: no PCIC\n");
 		return;
 	}
 	pcic = &pcic0;
@@ -619,44 +617,12 @@ void pcibios_fixup_bus(struct pci_bus *bus)
 	 * Next crud is an equivalent of pbm = pcic_bus_to_pbm(bus);
 	 */
 	if (bus->number != 0) {
-		printk("pcibios_fixup_bus: nonzero bus 0x%x\n", bus->number);
+		pci_info(bus, "pcibios_fixup_bus: nonzero bus 0x%x\n",
+			 bus->number);
 		return;
 	}
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
-
-		/*
-		 * Comment from i386 branch:
-		 *     There are buggy BIOSes that forget to enable I/O and memory
-		 *     access to PCI devices. We try to fix this, but we need to
-		 *     be sure that the BIOS didn't forget to assign an address
-		 *     to the device. [mj]
-		 * OBP is a case of such BIOS :-)
-		 */
-		has_io = has_mem = 0;
-		for(i=0; i<6; i++) {
-			unsigned long f = dev->resource[i].flags;
-			if (f & IORESOURCE_IO) {
-				has_io = 1;
-			} else if (f & IORESOURCE_MEM)
-				has_mem = 1;
-		}
-		pcic_read_config(dev->bus, dev->devfn, PCI_COMMAND, 2, &cmd);
-		if (has_io && !(cmd & PCI_COMMAND_IO)) {
-			printk("PCIC: Enabling I/O for device %02x:%02x\n",
-				dev->bus->number, dev->devfn);
-			cmd |= PCI_COMMAND_IO;
-			pcic_write_config(dev->bus, dev->devfn,
-			    PCI_COMMAND, 2, cmd);
-		}
-		if (has_mem && !(cmd & PCI_COMMAND_MEMORY)) {
-			printk("PCIC: Enabling memory for device %02x:%02x\n",
-				dev->bus->number, dev->devfn);
-			cmd |= PCI_COMMAND_MEMORY;
-			pcic_write_config(dev->bus, dev->devfn,
-			    PCI_COMMAND, 2, cmd);
-		}
-
 		node = pdev_to_pnode(&pcic->pbm, dev);
 		if(node == 0)
 			node = -1;
@@ -675,6 +641,34 @@ void pcibios_fixup_bus(struct pci_bus *bus)
 	}
 }
 
+int pcibios_enable_device(struct pci_dev *dev, int mask)
+{
+	u16 cmd, oldcmd;
+	int i;
+
+	pci_read_config_word(dev, PCI_COMMAND, &cmd);
+	oldcmd = cmd;
+
+	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
+		struct resource *res = &dev->resource[i];
+
+		/* Only set up the requested stuff */
+		if (!(mask & (1<<i)))
+			continue;
+
+		if (res->flags & IORESOURCE_IO)
+			cmd |= PCI_COMMAND_IO;
+		if (res->flags & IORESOURCE_MEM)
+			cmd |= PCI_COMMAND_MEMORY;
+	}
+
+	if (cmd != oldcmd) {
+		pci_info(dev, "enabling device (%04x -> %04x)\n", oldcmd, cmd);
+		pci_write_config_word(dev, PCI_COMMAND, cmd);
+	}
+	return 0;
+}
+
 /* Makes compiler happy */
 static volatile int pcic_timer_dummy;
 
@@ -747,17 +741,11 @@ static void watchdog_reset() {
 }
 #endif
 
-int pcibios_enable_device(struct pci_dev *pdev, int mask)
-{
-	return 0;
-}
-
 /*
  * NMI
  */
 void pcic_nmi(unsigned int pend, struct pt_regs *regs)
 {
-
 	pend = swab32(pend);
 
 	if (!pcic_speculative || (pend & PCI_SYS_INT_PENDING_PIO) == 0) {
diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c
index 454a8af28f13..6c086086ca8f 100644
--- a/arch/sparc/kernel/process_64.c
+++ b/arch/sparc/kernel/process_64.c
@@ -518,14 +518,7 @@ void synchronize_user_stack(void)
 
 static void stack_unaligned(unsigned long sp)
 {
-	siginfo_t info;
-
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRALN;
-	info.si_addr = (void __user *) sp;
-	info.si_trapno = 0;
-	force_sig_info(SIGBUS, &info, current);
+	force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) sp, 0, current);
 }
 
 void fault_in_user_windows(void)
diff --git a/arch/sparc/kernel/sys_sparc_32.c b/arch/sparc/kernel/sys_sparc_32.c
index e8c3cb6b6d08..7f3d9c59719a 100644
--- a/arch/sparc/kernel/sys_sparc_32.c
+++ b/arch/sparc/kernel/sys_sparc_32.c
@@ -147,17 +147,11 @@ SYSCALL_DEFINE0(nis_syscall)
 asmlinkage void
 sparc_breakpoint (struct pt_regs *regs)
 {
-	siginfo_t info;
 
 #ifdef DEBUG_SPARC_BREAKPOINT
         printk ("TRAP: Entering kernel PC=%x, nPC=%x\n", regs->pc, regs->npc);
 #endif
-	info.si_signo = SIGTRAP;
-	info.si_errno = 0;
-	info.si_code = TRAP_BRKPT;
-	info.si_addr = (void __user *)regs->pc;
-	info.si_trapno = 0;
-	force_sig_info(SIGTRAP, &info, current);
+	force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc, 0, current);
 
 #ifdef DEBUG_SPARC_BREAKPOINT
 	printk ("TRAP: Returning to space: PC=%x nPC=%x\n", regs->pc, regs->npc);
diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c
index 9ef8de63f28b..7e49bbc925a5 100644
--- a/arch/sparc/kernel/sys_sparc_64.c
+++ b/arch/sparc/kernel/sys_sparc_64.c
@@ -502,7 +502,6 @@ SYSCALL_DEFINE0(nis_syscall)
 asmlinkage void sparc_breakpoint(struct pt_regs *regs)
 {
 	enum ctx_state prev_state = exception_enter();
-	siginfo_t info;
 
 	if (test_thread_flag(TIF_32BIT)) {
 		regs->tpc &= 0xffffffff;
@@ -511,12 +510,7 @@ asmlinkage void sparc_breakpoint(struct pt_regs *regs)
 #ifdef DEBUG_SPARC_BREAKPOINT
         printk ("TRAP: Entering kernel PC=%lx, nPC=%lx\n", regs->tpc, regs->tnpc);
 #endif
-	info.si_signo = SIGTRAP;
-	info.si_errno = 0;
-	info.si_code = TRAP_BRKPT;
-	info.si_addr = (void __user *)regs->tpc;
-	info.si_trapno = 0;
-	force_sig_info(SIGTRAP, &info, current);
+	force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->tpc, 0, current);
 #ifdef DEBUG_SPARC_BREAKPOINT
 	printk ("TRAP: Returning to space: PC=%lx nPC=%lx\n", regs->tpc, regs->tnpc);
 #endif
diff --git a/arch/sparc/kernel/traps_32.c b/arch/sparc/kernel/traps_32.c
index b1ed763e4787..bcdfc6168dd5 100644
--- a/arch/sparc/kernel/traps_32.c
+++ b/arch/sparc/kernel/traps_32.c
@@ -93,8 +93,6 @@ void __noreturn die_if_kernel(char *str, struct pt_regs *regs)
 
 void do_hw_interrupt(struct pt_regs *regs, unsigned long type)
 {
-	siginfo_t info;
-
 	if(type < 0x80) {
 		/* Sun OS's puke from bad traps, Linux survives! */
 		printk("Unimplemented Sparc TRAP, type = %02lx\n", type);
@@ -104,19 +102,13 @@ void do_hw_interrupt(struct pt_regs *regs, unsigned long type)
 	if(regs->psr & PSR_PS)
 		die_if_kernel("Kernel bad trap", regs);
 
-	info.si_signo = SIGILL;
-	info.si_errno = 0;
-	info.si_code = ILL_ILLTRP;
-	info.si_addr = (void __user *)regs->pc;
-	info.si_trapno = type - 0x80;
-	force_sig_info(SIGILL, &info, current);
+	force_sig_fault(SIGILL, ILL_ILLTRP,
+			(void __user *)regs->pc, type - 0x80, current);
 }
 
 void do_illegal_instruction(struct pt_regs *regs, unsigned long pc, unsigned long npc,
 			    unsigned long psr)
 {
-	siginfo_t info;
-
 	if(psr & PSR_PS)
 		die_if_kernel("Kernel illegal instruction", regs);
 #ifdef TRAP_DEBUG
@@ -124,27 +116,15 @@ void do_illegal_instruction(struct pt_regs *regs, unsigned long pc, unsigned lon
 	       regs->pc, *(unsigned long *)regs->pc);
 #endif
 
-	info.si_signo = SIGILL;
-	info.si_errno = 0;
-	info.si_code = ILL_ILLOPC;
-	info.si_addr = (void __user *)pc;
-	info.si_trapno = 0;
-	send_sig_info(SIGILL, &info, current);
+	send_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)pc, 0, current);
 }
 
 void do_priv_instruction(struct pt_regs *regs, unsigned long pc, unsigned long npc,
 			 unsigned long psr)
 {
-	siginfo_t info;
-
 	if(psr & PSR_PS)
 		die_if_kernel("Penguin instruction from Penguin mode??!?!", regs);
-	info.si_signo = SIGILL;
-	info.si_errno = 0;
-	info.si_code = ILL_PRVOPC;
-	info.si_addr = (void __user *)pc;
-	info.si_trapno = 0;
-	send_sig_info(SIGILL, &info, current);
+	send_sig_fault(SIGILL, ILL_PRVOPC, (void __user *)pc, 0, current);
 }
 
 /* XXX User may want to be allowed to do this. XXX */
@@ -152,8 +132,6 @@ void do_priv_instruction(struct pt_regs *regs, unsigned long pc, unsigned long n
 void do_memaccess_unaligned(struct pt_regs *regs, unsigned long pc, unsigned long npc,
 			    unsigned long psr)
 {
-	siginfo_t info;
-
 	if(regs->psr & PSR_PS) {
 		printk("KERNEL MNA at pc %08lx npc %08lx called by %08lx\n", pc, npc,
 		       regs->u_regs[UREG_RETPC]);
@@ -165,12 +143,9 @@ void do_memaccess_unaligned(struct pt_regs *regs, unsigned long pc, unsigned lon
 	instruction_dump ((unsigned long *) regs->pc);
 	printk ("do_MNA!\n");
 #endif
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRALN;
-	info.si_addr = /* FIXME: Should dig out mna address */ (void *)0;
-	info.si_trapno = 0;
-	send_sig_info(SIGBUS, &info, current);
+	send_sig_fault(SIGBUS, BUS_ADRALN,
+		       /* FIXME: Should dig out mna address */ (void *)0,
+		       0, current);
 }
 
 static unsigned long init_fsr = 0x0UL;
@@ -226,9 +201,9 @@ void do_fpe_trap(struct pt_regs *regs, unsigned long pc, unsigned long npc,
 		 unsigned long psr)
 {
 	static int calls;
-	siginfo_t info;
 	unsigned long fsr;
 	int ret = 0;
+	int code;
 #ifndef CONFIG_SMP
 	struct task_struct *fpt = last_task_used_math;
 #else
@@ -303,24 +278,20 @@ void do_fpe_trap(struct pt_regs *regs, unsigned long pc, unsigned long npc,
 	}
 
 	fsr = fpt->thread.fsr;
-	info.si_signo = SIGFPE;
-	info.si_errno = 0;
-	info.si_addr = (void __user *)pc;
-	info.si_trapno = 0;
-	info.si_code = FPE_FIXME;
+	code = FPE_FLTUNK;
 	if ((fsr & 0x1c000) == (1 << 14)) {
 		if (fsr & 0x10)
-			info.si_code = FPE_FLTINV;
+			code = FPE_FLTINV;
 		else if (fsr & 0x08)
-			info.si_code = FPE_FLTOVF;
+			code = FPE_FLTOVF;
 		else if (fsr & 0x04)
-			info.si_code = FPE_FLTUND;
+			code = FPE_FLTUND;
 		else if (fsr & 0x02)
-			info.si_code = FPE_FLTDIV;
+			code = FPE_FLTDIV;
 		else if (fsr & 0x01)
-			info.si_code = FPE_FLTRES;
+			code = FPE_FLTRES;
 	}
-	send_sig_info(SIGFPE, &info, fpt);
+	send_sig_fault(SIGFPE, code, (void __user *)pc, 0, fpt);
 #ifndef CONFIG_SMP
 	last_task_used_math = NULL;
 #endif
@@ -332,16 +303,9 @@ void do_fpe_trap(struct pt_regs *regs, unsigned long pc, unsigned long npc,
 void handle_tag_overflow(struct pt_regs *regs, unsigned long pc, unsigned long npc,
 			 unsigned long psr)
 {
-	siginfo_t info;
-
 	if(psr & PSR_PS)
 		die_if_kernel("Penguin overflow trap from kernel mode", regs);
-	info.si_signo = SIGEMT;
-	info.si_errno = 0;
-	info.si_code = EMT_TAGOVF;
-	info.si_addr = (void __user *)pc;
-	info.si_trapno = 0;
-	send_sig_info(SIGEMT, &info, current);
+	send_sig_fault(SIGEMT, EMT_TAGOVF, (void __user *)pc, 0, current);
 }
 
 void handle_watchpoint(struct pt_regs *regs, unsigned long pc, unsigned long npc,
@@ -359,61 +323,33 @@ void handle_watchpoint(struct pt_regs *regs, unsigned long pc, unsigned long npc
 void handle_reg_access(struct pt_regs *regs, unsigned long pc, unsigned long npc,
 		       unsigned long psr)
 {
-	siginfo_t info;
-
 #ifdef TRAP_DEBUG
 	printk("Register Access Exception at PC %08lx NPC %08lx PSR %08lx\n",
 	       pc, npc, psr);
 #endif
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_OBJERR;
-	info.si_addr = (void __user *)pc;
-	info.si_trapno = 0;
-	force_sig_info(SIGBUS, &info, current);
+	force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)pc, 0, current);
 }
 
 void handle_cp_disabled(struct pt_regs *regs, unsigned long pc, unsigned long npc,
 			unsigned long psr)
 {
-	siginfo_t info;
-
-	info.si_signo = SIGILL;
-	info.si_errno = 0;
-	info.si_code = ILL_COPROC;
-	info.si_addr = (void __user *)pc;
-	info.si_trapno = 0;
-	send_sig_info(SIGILL, &info, current);
+	send_sig_fault(SIGILL, ILL_COPROC, (void __user *)pc, 0, current);
 }
 
 void handle_cp_exception(struct pt_regs *regs, unsigned long pc, unsigned long npc,
 			 unsigned long psr)
 {
-	siginfo_t info;
-
 #ifdef TRAP_DEBUG
 	printk("Co-Processor Exception at PC %08lx NPC %08lx PSR %08lx\n",
 	       pc, npc, psr);
 #endif
-	info.si_signo = SIGILL;
-	info.si_errno = 0;
-	info.si_code = ILL_COPROC;
-	info.si_addr = (void __user *)pc;
-	info.si_trapno = 0;
-	send_sig_info(SIGILL, &info, current);
+	send_sig_fault(SIGILL, ILL_COPROC, (void __user *)pc, 0, current);
 }
 
 void handle_hw_divzero(struct pt_regs *regs, unsigned long pc, unsigned long npc,
 		       unsigned long psr)
 {
-	siginfo_t info;
-
-	info.si_signo = SIGFPE;
-	info.si_errno = 0;
-	info.si_code = FPE_INTDIV;
-	info.si_addr = (void __user *)pc;
-	info.si_trapno = 0;
-	send_sig_info(SIGFPE, &info, current);
+	send_sig_fault(SIGFPE, FPE_INTDIV, (void __user *)pc, 0, current);
 }
 
 #ifdef CONFIG_DEBUG_BUGVERBOSE
diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c
index 462a21abd105..aa624ed79db1 100644
--- a/arch/sparc/kernel/traps_64.c
+++ b/arch/sparc/kernel/traps_64.c
@@ -87,7 +87,6 @@ static void dump_tl1_traplog(struct tl1_traplog *p)
 void bad_trap(struct pt_regs *regs, long lvl)
 {
 	char buffer[36];
-	siginfo_t info;
 
 	if (notify_die(DIE_TRAP, "bad trap", regs,
 		       0, lvl, SIGTRAP) == NOTIFY_STOP)
@@ -107,12 +106,8 @@ void bad_trap(struct pt_regs *regs, long lvl)
 		regs->tpc &= 0xffffffff;
 		regs->tnpc &= 0xffffffff;
 	}
-	info.si_signo = SIGILL;
-	info.si_errno = 0;
-	info.si_code = ILL_ILLTRP;
-	info.si_addr = (void __user *)regs->tpc;
-	info.si_trapno = lvl;
-	force_sig_info(SIGILL, &info, current);
+	force_sig_fault(SIGILL, ILL_ILLTRP,
+			(void __user *)regs->tpc, lvl, current);
 }
 
 void bad_trap_tl1(struct pt_regs *regs, long lvl)
@@ -191,7 +186,6 @@ EXPORT_SYMBOL_GPL(unregister_dimm_printer);
 void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
 {
 	enum ctx_state prev_state = exception_enter();
-	siginfo_t info;
 
 	if (notify_die(DIE_TRAP, "instruction access exception", regs,
 		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
@@ -206,12 +200,8 @@ void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, un
 		regs->tpc &= 0xffffffff;
 		regs->tnpc &= 0xffffffff;
 	}
-	info.si_signo = SIGSEGV;
-	info.si_errno = 0;
-	info.si_code = SEGV_MAPERR;
-	info.si_addr = (void __user *)regs->tpc;
-	info.si_trapno = 0;
-	force_sig_info(SIGSEGV, &info, current);
+	force_sig_fault(SIGSEGV, SEGV_MAPERR,
+			(void __user *)regs->tpc, 0, current);
 out:
 	exception_exit(prev_state);
 }
@@ -230,7 +220,6 @@ void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsig
 {
 	unsigned short type = (type_ctx >> 16);
 	unsigned short ctx  = (type_ctx & 0xffff);
-	siginfo_t info;
 
 	if (notify_die(DIE_TRAP, "instruction access exception", regs,
 		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
@@ -247,12 +236,7 @@ void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsig
 		regs->tpc &= 0xffffffff;
 		regs->tnpc &= 0xffffffff;
 	}
-	info.si_signo = SIGSEGV;
-	info.si_errno = 0;
-	info.si_code = SEGV_MAPERR;
-	info.si_addr = (void __user *) addr;
-	info.si_trapno = 0;
-	force_sig_info(SIGSEGV, &info, current);
+	force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *) addr, 0, current);
 }
 
 void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
@@ -307,7 +291,6 @@ bool is_no_fault_exception(struct pt_regs *regs)
 void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
 {
 	enum ctx_state prev_state = exception_enter();
-	siginfo_t info;
 
 	if (notify_die(DIE_TRAP, "data access exception", regs,
 		       0, 0x30, SIGTRAP) == NOTIFY_STOP)
@@ -338,12 +321,7 @@ void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, un
 	if (is_no_fault_exception(regs))
 		return;
 
-	info.si_signo = SIGSEGV;
-	info.si_errno = 0;
-	info.si_code = SEGV_MAPERR;
-	info.si_addr = (void __user *)sfar;
-	info.si_trapno = 0;
-	force_sig_info(SIGSEGV, &info, current);
+	force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)sfar, 0, current);
 out:
 	exception_exit(prev_state);
 }
@@ -559,8 +537,6 @@ static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned lo
 
 static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs)
 {
-	siginfo_t info;
-
 	printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] "
 	       "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n",
 	       smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1);
@@ -595,12 +571,7 @@ static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned lon
 		regs->tpc &= 0xffffffff;
 		regs->tnpc &= 0xffffffff;
 	}
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_OBJERR;
-	info.si_addr = (void *)0;
-	info.si_trapno = 0;
-	force_sig_info(SIGBUS, &info, current);
+	force_sig_fault(SIGBUS, BUS_OBJERR, (void *)0, 0, current);
 }
 
 void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar)
@@ -2190,7 +2161,6 @@ bool sun4v_nonresum_error_user_handled(struct pt_regs *regs,
 
 	if (attrs & SUN4V_ERR_ATTRS_MEMORY) {
 		unsigned long addr = ent->err_raddr;
-		siginfo_t info;
 
 		if (addr == ~(u64)0) {
 			/* This seems highly unlikely to ever occur */
@@ -2211,21 +2181,13 @@ bool sun4v_nonresum_error_user_handled(struct pt_regs *regs,
 				addr += PAGE_SIZE;
 			}
 		}
-		info.si_signo = SIGKILL;
-		info.si_errno = 0;
-		info.si_trapno = 0;
-		force_sig_info(info.si_signo, &info, current);
+		force_sig(SIGKILL, current);
 
 		return true;
 	}
 	if (attrs & SUN4V_ERR_ATTRS_PIO) {
-		siginfo_t info;
-
-		info.si_signo = SIGBUS;
-		info.si_code = BUS_ADRERR;
-		info.si_addr = (void __user *)sun4v_get_vaddr(regs);
-		force_sig_info(info.si_signo, &info, current);
-
+		force_sig_fault(SIGBUS, BUS_ADRERR,
+				(void __user *)sun4v_get_vaddr(regs), 0, current);
 		return true;
 	}
 
@@ -2362,30 +2324,27 @@ static void do_fpe_common(struct pt_regs *regs)
 		regs->tnpc += 4;
 	} else {
 		unsigned long fsr = current_thread_info()->xfsr[0];
-		siginfo_t info;
+		int code;
 
 		if (test_thread_flag(TIF_32BIT)) {
 			regs->tpc &= 0xffffffff;
 			regs->tnpc &= 0xffffffff;
 		}
-		info.si_signo = SIGFPE;
-		info.si_errno = 0;
-		info.si_addr = (void __user *)regs->tpc;
-		info.si_trapno = 0;
-		info.si_code = FPE_FIXME;
+		code = FPE_FLTUNK;
 		if ((fsr & 0x1c000) == (1 << 14)) {
 			if (fsr & 0x10)
-				info.si_code = FPE_FLTINV;
+				code = FPE_FLTINV;
 			else if (fsr & 0x08)
-				info.si_code = FPE_FLTOVF;
+				code = FPE_FLTOVF;
 			else if (fsr & 0x04)
-				info.si_code = FPE_FLTUND;
+				code = FPE_FLTUND;
 			else if (fsr & 0x02)
-				info.si_code = FPE_FLTDIV;
+				code = FPE_FLTDIV;
 			else if (fsr & 0x01)
-				info.si_code = FPE_FLTRES;
+				code = FPE_FLTRES;
 		}
-		force_sig_info(SIGFPE, &info, current);
+		force_sig_fault(SIGFPE, code,
+				(void __user *)regs->tpc, 0, current);
 	}
 }
 
@@ -2428,7 +2387,6 @@ void do_fpother(struct pt_regs *regs)
 void do_tof(struct pt_regs *regs)
 {
 	enum ctx_state prev_state = exception_enter();
-	siginfo_t info;
 
 	if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs,
 		       0, 0x26, SIGEMT) == NOTIFY_STOP)
@@ -2440,12 +2398,8 @@ void do_tof(struct pt_regs *regs)
 		regs->tpc &= 0xffffffff;
 		regs->tnpc &= 0xffffffff;
 	}
-	info.si_signo = SIGEMT;
-	info.si_errno = 0;
-	info.si_code = EMT_TAGOVF;
-	info.si_addr = (void __user *)regs->tpc;
-	info.si_trapno = 0;
-	force_sig_info(SIGEMT, &info, current);
+	force_sig_fault(SIGEMT, EMT_TAGOVF,
+			(void __user *)regs->tpc, 0, current);
 out:
 	exception_exit(prev_state);
 }
@@ -2453,7 +2407,6 @@ void do_tof(struct pt_regs *regs)
 void do_div0(struct pt_regs *regs)
 {
 	enum ctx_state prev_state = exception_enter();
-	siginfo_t info;
 
 	if (notify_die(DIE_TRAP, "integer division by zero", regs,
 		       0, 0x28, SIGFPE) == NOTIFY_STOP)
@@ -2465,12 +2418,8 @@ void do_div0(struct pt_regs *regs)
 		regs->tpc &= 0xffffffff;
 		regs->tnpc &= 0xffffffff;
 	}
-	info.si_signo = SIGFPE;
-	info.si_errno = 0;
-	info.si_code = FPE_INTDIV;
-	info.si_addr = (void __user *)regs->tpc;
-	info.si_trapno = 0;
-	force_sig_info(SIGFPE, &info, current);
+	force_sig_fault(SIGFPE, FPE_INTDIV,
+			(void __user *)regs->tpc, 0, current);
 out:
 	exception_exit(prev_state);
 }
@@ -2632,7 +2581,6 @@ void do_illegal_instruction(struct pt_regs *regs)
 	unsigned long pc = regs->tpc;
 	unsigned long tstate = regs->tstate;
 	u32 insn;
-	siginfo_t info;
 
 	if (notify_die(DIE_TRAP, "illegal instruction", regs,
 		       0, 0x10, SIGILL) == NOTIFY_STOP)
@@ -2666,12 +2614,7 @@ void do_illegal_instruction(struct pt_regs *regs)
 			}
 		}
 	}
-	info.si_signo = SIGILL;
-	info.si_errno = 0;
-	info.si_code = ILL_ILLOPC;
-	info.si_addr = (void __user *)pc;
-	info.si_trapno = 0;
-	force_sig_info(SIGILL, &info, current);
+	force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)pc, 0, current);
 out:
 	exception_exit(prev_state);
 }
@@ -2679,7 +2622,6 @@ void do_illegal_instruction(struct pt_regs *regs)
 void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
 {
 	enum ctx_state prev_state = exception_enter();
-	siginfo_t info;
 
 	if (notify_die(DIE_TRAP, "memory address unaligned", regs,
 		       0, 0x34, SIGSEGV) == NOTIFY_STOP)
@@ -2692,20 +2634,13 @@ void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned lo
 	if (is_no_fault_exception(regs))
 		return;
 
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRALN;
-	info.si_addr = (void __user *)sfar;
-	info.si_trapno = 0;
-	force_sig_info(SIGBUS, &info, current);
+	force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)sfar, 0, current);
 out:
 	exception_exit(prev_state);
 }
 
 void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
 {
-	siginfo_t info;
-
 	if (notify_die(DIE_TRAP, "memory address unaligned", regs,
 		       0, 0x34, SIGSEGV) == NOTIFY_STOP)
 		return;
@@ -2717,12 +2652,7 @@ void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_c
 	if (is_no_fault_exception(regs))
 		return;
 
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRALN;
-	info.si_addr = (void __user *) addr;
-	info.si_trapno = 0;
-	force_sig_info(SIGBUS, &info, current);
+	force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) addr, 0, current);
 }
 
 /* sun4v_mem_corrupt_detect_precise() - Handle precise exception on an ADI
@@ -2775,7 +2705,6 @@ void sun4v_mem_corrupt_detect_precise(struct pt_regs *regs, unsigned long addr,
 void do_privop(struct pt_regs *regs)
 {
 	enum ctx_state prev_state = exception_enter();
-	siginfo_t info;
 
 	if (notify_die(DIE_TRAP, "privileged operation", regs,
 		       0, 0x11, SIGILL) == NOTIFY_STOP)
@@ -2785,12 +2714,8 @@ void do_privop(struct pt_regs *regs)
 		regs->tpc &= 0xffffffff;
 		regs->tnpc &= 0xffffffff;
 	}
-	info.si_signo = SIGILL;
-	info.si_errno = 0;
-	info.si_code = ILL_PRVOPC;
-	info.si_addr = (void __user *)regs->tpc;
-	info.si_trapno = 0;
-	force_sig_info(SIGILL, &info, current);
+	force_sig_fault(SIGILL, ILL_PRVOPC,
+			(void __user *)regs->tpc, 0, current);
 out:
 	exception_exit(prev_state);
 }
diff --git a/arch/sparc/kernel/unaligned_32.c b/arch/sparc/kernel/unaligned_32.c
index 7642d7e4f0d9..64ac8c0c1429 100644
--- a/arch/sparc/kernel/unaligned_32.c
+++ b/arch/sparc/kernel/unaligned_32.c
@@ -311,14 +311,9 @@ static inline int ok_for_user(struct pt_regs *regs, unsigned int insn,
 
 static void user_mna_trap_fault(struct pt_regs *regs, unsigned int insn)
 {
-	siginfo_t info;
-
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRALN;
-	info.si_addr = (void __user *)safe_compute_effective_address(regs, insn);
-	info.si_trapno = 0;
-	send_sig_info(SIGBUS, &info, current);
+	send_sig_fault(SIGBUS, BUS_ADRALN,
+		       (void __user *)safe_compute_effective_address(regs, insn),
+		       0, current);
 }
 
 asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn)
diff --git a/arch/sparc/lib/NG4memset.S b/arch/sparc/lib/NG4memset.S
index f81ee5419e2c..d0c4d195fd40 100644
--- a/arch/sparc/lib/NG4memset.S
+++ b/arch/sparc/lib/NG4memset.S
@@ -14,14 +14,14 @@
 	.globl		NG4memset
 NG4memset:
 	andcc		%o1, 0xff, %o4
-	be,pt		%icc, 1f
+	be,pt		%xcc, 1f
 	 mov		%o2, %o1
 	sllx		%o4, 8, %g1
 	or		%g1, %o4, %o2
 	sllx		%o2, 16, %g1
 	or		%g1, %o2, %o2
 	sllx		%o2, 32, %g1
-	ba,pt		%icc, 1f
+	ba,pt		%xcc, 1f
 	 or		%g1, %o2, %o4
 	.size		NG4memset,.-NG4memset
 
@@ -30,7 +30,7 @@ NG4memset:
 NG4bzero:
 	clr		%o4
 1:	cmp		%o1, 16
-	ble		%icc, .Ltiny
+	ble		%xcc, .Ltiny
 	 mov		%o0, %o3
 	sub		%g0, %o0, %g1
 	and		%g1, 0x7, %g1
@@ -38,7 +38,7 @@ NG4bzero:
 	 sub		%o1, %g1, %o1
 1:	stb		%o4, [%o0 + 0x00]
 	subcc		%g1, 1, %g1
-	bne,pt		%icc, 1b
+	bne,pt		%xcc, 1b
 	 add		%o0, 1, %o0
 .Laligned8:
 	cmp		%o1, 64 + (64 - 8)
@@ -49,7 +49,7 @@ NG4bzero:
 	 sub		%o1, %g1, %o1
 1:	stx		%o4, [%o0 + 0x00]
 	subcc		%g1, 8, %g1
-	bne,pt		%icc, 1b
+	bne,pt		%xcc, 1b
 	 add		%o0, 0x8, %o0
 .Laligned64:
 	andn		%o1, 64 - 1, %g1
@@ -59,30 +59,30 @@ NG4bzero:
 1:	stxa		%o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P
 	subcc		%g1, 0x40, %g1
 	stxa		%o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P
-	bne,pt		%icc, 1b
+	bne,pt		%xcc, 1b
 	 add		%o0, 0x40, %o0
 .Lpostloop:
 	cmp		%o1, 8
-	bl,pn		%icc, .Ltiny
+	bl,pn		%xcc, .Ltiny
 	 membar		#StoreStore|#StoreLoad
 .Lmedium:
 	andn		%o1, 0x7, %g1
 	sub		%o1, %g1, %o1
 1:	stx		%o4, [%o0 + 0x00]
 	subcc		%g1, 0x8, %g1
-	bne,pt		%icc, 1b
+	bne,pt		%xcc, 1b
 	 add		%o0, 0x08, %o0
 	andcc		%o1, 0x4, %g1
-	be,pt		%icc, .Ltiny
+	be,pt		%xcc, .Ltiny
 	 sub		%o1, %g1, %o1
 	stw		%o4, [%o0 + 0x00]
 	add		%o0, 0x4, %o0
 .Ltiny:
 	cmp		%o1, 0
-	be,pn		%icc, .Lexit
+	be,pn		%xcc, .Lexit
 1:	 subcc		%o1, 1, %o1
 	stb		%o4, [%o0 + 0x00]
-	bne,pt		%icc, 1b
+	bne,pt		%xcc, 1b
 	 add		%o0, 1, %o0
 .Lexit:
 	retl
@@ -100,8 +100,8 @@ NG4bzero:
 	stxa		%o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P
 	stxa		%o4, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P
 	stxa		%o4, [%o0 + %o5] ASI_BLK_INIT_QUAD_LDD_P
-	bne,pt		%icc, 1b
+	bne,pt		%xcc, 1b
 	 add		%o0, 0x30, %o0
-	ba,a,pt		%icc, .Lpostloop
+	ba,a,pt		%xcc, .Lpostloop
 	 nop
 	.size		NG4bzero,.-NG4bzero
diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c
index a8103a84b4ac..9f75b6444bf1 100644
--- a/arch/sparc/mm/fault_32.c
+++ b/arch/sparc/mm/fault_32.c
@@ -127,19 +127,11 @@ show_signal_msg(struct pt_regs *regs, int sig, int code,
 static void __do_fault_siginfo(int code, int sig, struct pt_regs *regs,
 			       unsigned long addr)
 {
-	siginfo_t info;
-
-	info.si_signo = sig;
-	info.si_code = code;
-	info.si_errno = 0;
-	info.si_addr = (void __user *) addr;
-	info.si_trapno = 0;
-
 	if (unlikely(show_unhandled_signals))
-		show_signal_msg(regs, sig, info.si_code,
+		show_signal_msg(regs, sig, code,
 				addr, current);
 
-	force_sig_info (sig, &info, current);
+	force_sig_fault(sig, code, (void __user *) addr, 0, current);
 }
 
 static unsigned long compute_si_addr(struct pt_regs *regs, int text_fault)
diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 41363f46797b..63166fcf9e25 100644
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -170,11 +170,7 @@ static void do_fault_siginfo(int code, int sig, struct pt_regs *regs,
 			     int fault_code)
 {
 	unsigned long addr;
-	siginfo_t info;
 
-	info.si_code = code;
-	info.si_signo = sig;
-	info.si_errno = 0;
 	if (fault_code & FAULT_CODE_ITLB) {
 		addr = regs->tpc;
 	} else {
@@ -187,13 +183,11 @@ static void do_fault_siginfo(int code, int sig, struct pt_regs *regs,
 		else
 			addr = fault_addr;
 	}
-	info.si_addr = (void __user *) addr;
-	info.si_trapno = 0;
 
 	if (unlikely(show_unhandled_signals))
 		show_signal_msg(regs, sig, code, addr, current);
 
-	force_sig_info(sig, &info, current);
+	force_sig_fault(sig, code, (void __user *) addr, 0, current);
 }
 
 static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn)
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 8aeb1aabe76e..f396048a0d68 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -1620,7 +1620,7 @@ static void __init bootmem_init_nonnuma(void)
 	       (top_of_ram - total_ram) >> 20);
 
 	init_node_masks_nonnuma();
-	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);
+	memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0);
 	allocate_node_data(0);
 	node_set_online(0);
 }
diff --git a/arch/sparc/net/Makefile b/arch/sparc/net/Makefile
index 76fa8e95b721..d32aac3a25b8 100644
--- a/arch/sparc/net/Makefile
+++ b/arch/sparc/net/Makefile
@@ -1,4 +1,7 @@
 #
 # Arch-specific network modules
 #
-obj-$(CONFIG_BPF_JIT) += bpf_jit_asm_$(BITS).o bpf_jit_comp_$(BITS).o
+obj-$(CONFIG_BPF_JIT) += bpf_jit_comp_$(BITS).o
+ifeq ($(BITS),32)
+obj-$(CONFIG_BPF_JIT) += bpf_jit_asm_32.o
+endif
diff --git a/arch/sparc/net/bpf_jit_64.h b/arch/sparc/net/bpf_jit_64.h
index 428f7fd19175..fbc836f1c51c 100644
--- a/arch/sparc/net/bpf_jit_64.h
+++ b/arch/sparc/net/bpf_jit_64.h
@@ -33,35 +33,6 @@
 #define I5		0x1d
 #define FP		0x1e
 #define I7		0x1f
-
-#define r_SKB		L0
-#define r_HEADLEN	L4
-#define r_SKB_DATA	L5
-#define r_TMP		G1
-#define r_TMP2		G3
-
-/* assembly code in arch/sparc/net/bpf_jit_asm_64.S */
-extern u32 bpf_jit_load_word[];
-extern u32 bpf_jit_load_half[];
-extern u32 bpf_jit_load_byte[];
-extern u32 bpf_jit_load_byte_msh[];
-extern u32 bpf_jit_load_word_positive_offset[];
-extern u32 bpf_jit_load_half_positive_offset[];
-extern u32 bpf_jit_load_byte_positive_offset[];
-extern u32 bpf_jit_load_byte_msh_positive_offset[];
-extern u32 bpf_jit_load_word_negative_offset[];
-extern u32 bpf_jit_load_half_negative_offset[];
-extern u32 bpf_jit_load_byte_negative_offset[];
-extern u32 bpf_jit_load_byte_msh_negative_offset[];
-
-#else
-#define r_RESULT	%o0
-#define r_SKB		%o0
-#define r_OFF		%o1
-#define r_HEADLEN	%l4
-#define r_SKB_DATA	%l5
-#define r_TMP		%g1
-#define r_TMP2		%g3
 #endif
 
 #endif /* _BPF_JIT_H */
diff --git a/arch/sparc/net/bpf_jit_asm_64.S b/arch/sparc/net/bpf_jit_asm_64.S
deleted file mode 100644
index 7177867052a1..000000000000
--- a/arch/sparc/net/bpf_jit_asm_64.S
+++ /dev/null
@@ -1,162 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#include <asm/ptrace.h>
-
-#include "bpf_jit_64.h"
-
-#define SAVE_SZ		176
-#define SCRATCH_OFF	STACK_BIAS + 128
-#define BE_PTR(label)	be,pn %xcc, label
-#define SIGN_EXTEND(reg)	sra reg, 0, reg
-
-#define SKF_MAX_NEG_OFF	(-0x200000) /* SKF_LL_OFF from filter.h */
-
-	.text
-	.globl	bpf_jit_load_word
-bpf_jit_load_word:
-	cmp	r_OFF, 0
-	bl	bpf_slow_path_word_neg
-	 nop
-	.globl	bpf_jit_load_word_positive_offset
-bpf_jit_load_word_positive_offset:
-	sub	r_HEADLEN, r_OFF, r_TMP
-	cmp	r_TMP, 3
-	ble	bpf_slow_path_word
-	 add	r_SKB_DATA, r_OFF, r_TMP
-	andcc	r_TMP, 3, %g0
-	bne	load_word_unaligned
-	 nop
-	retl
-	 ld	[r_TMP], r_RESULT
-load_word_unaligned:
-	ldub	[r_TMP + 0x0], r_OFF
-	ldub	[r_TMP + 0x1], r_TMP2
-	sll	r_OFF, 8, r_OFF
-	or	r_OFF, r_TMP2, r_OFF
-	ldub	[r_TMP + 0x2], r_TMP2
-	sll	r_OFF, 8, r_OFF
-	or	r_OFF, r_TMP2, r_OFF
-	ldub	[r_TMP + 0x3], r_TMP2
-	sll	r_OFF, 8, r_OFF
-	retl
-	 or	r_OFF, r_TMP2, r_RESULT
-
-	.globl	bpf_jit_load_half
-bpf_jit_load_half:
-	cmp	r_OFF, 0
-	bl	bpf_slow_path_half_neg
-	 nop
-	.globl	bpf_jit_load_half_positive_offset
-bpf_jit_load_half_positive_offset:
-	sub	r_HEADLEN, r_OFF, r_TMP
-	cmp	r_TMP, 1
-	ble	bpf_slow_path_half
-	 add	r_SKB_DATA, r_OFF, r_TMP
-	andcc	r_TMP, 1, %g0
-	bne	load_half_unaligned
-	 nop
-	retl
-	 lduh	[r_TMP], r_RESULT
-load_half_unaligned:
-	ldub	[r_TMP + 0x0], r_OFF
-	ldub	[r_TMP + 0x1], r_TMP2
-	sll	r_OFF, 8, r_OFF
-	retl
-	 or	r_OFF, r_TMP2, r_RESULT
-
-	.globl	bpf_jit_load_byte
-bpf_jit_load_byte:
-	cmp	r_OFF, 0
-	bl	bpf_slow_path_byte_neg
-	 nop
-	.globl	bpf_jit_load_byte_positive_offset
-bpf_jit_load_byte_positive_offset:
-	cmp	r_OFF, r_HEADLEN
-	bge	bpf_slow_path_byte
-	 nop
-	retl
-	 ldub	[r_SKB_DATA + r_OFF], r_RESULT
-
-#define bpf_slow_path_common(LEN)	\
-	save	%sp, -SAVE_SZ, %sp;	\
-	mov	%i0, %o0;		\
-	mov	%i1, %o1;		\
-	add	%fp, SCRATCH_OFF, %o2;	\
-	call	skb_copy_bits;		\
-	 mov	(LEN), %o3;		\
-	cmp	%o0, 0;			\
-	restore;
-
-bpf_slow_path_word:
-	bpf_slow_path_common(4)
-	bl	bpf_error
-	 ld	[%sp + SCRATCH_OFF], r_RESULT
-	retl
-	 nop
-bpf_slow_path_half:
-	bpf_slow_path_common(2)
-	bl	bpf_error
-	 lduh	[%sp + SCRATCH_OFF], r_RESULT
-	retl
-	 nop
-bpf_slow_path_byte:
-	bpf_slow_path_common(1)
-	bl	bpf_error
-	 ldub	[%sp + SCRATCH_OFF], r_RESULT
-	retl
-	 nop
-
-#define bpf_negative_common(LEN)			\
-	save	%sp, -SAVE_SZ, %sp;			\
-	mov	%i0, %o0;				\
-	mov	%i1, %o1;				\
-	SIGN_EXTEND(%o1);				\
-	call	bpf_internal_load_pointer_neg_helper;	\
-	 mov	(LEN), %o2;				\
-	mov	%o0, r_TMP;				\
-	cmp	%o0, 0;					\
-	BE_PTR(bpf_error);				\
-	 restore;
-
-bpf_slow_path_word_neg:
-	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
-	cmp	r_OFF, r_TMP
-	bl	bpf_error
-	 nop
-	.globl	bpf_jit_load_word_negative_offset
-bpf_jit_load_word_negative_offset:
-	bpf_negative_common(4)
-	andcc	r_TMP, 3, %g0
-	bne	load_word_unaligned
-	 nop
-	retl
-	 ld	[r_TMP], r_RESULT
-
-bpf_slow_path_half_neg:
-	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
-	cmp	r_OFF, r_TMP
-	bl	bpf_error
-	 nop
-	.globl	bpf_jit_load_half_negative_offset
-bpf_jit_load_half_negative_offset:
-	bpf_negative_common(2)
-	andcc	r_TMP, 1, %g0
-	bne	load_half_unaligned
-	 nop
-	retl
-	 lduh	[r_TMP], r_RESULT
-
-bpf_slow_path_byte_neg:
-	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
-	cmp	r_OFF, r_TMP
-	bl	bpf_error
-	 nop
-	.globl	bpf_jit_load_byte_negative_offset
-bpf_jit_load_byte_negative_offset:
-	bpf_negative_common(1)
-	retl
-	 ldub	[r_TMP], r_RESULT
-
-bpf_error:
-	/* Make the JIT program itself return zero. */
-	ret
-	restore	%g0, %g0, %o0
diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c
index 48a25869349b..222785af550b 100644
--- a/arch/sparc/net/bpf_jit_comp_64.c
+++ b/arch/sparc/net/bpf_jit_comp_64.c
@@ -48,10 +48,6 @@ static void bpf_flush_icache(void *start_, void *end_)
 	}
 }
 
-#define SEEN_DATAREF 1 /* might call external helpers */
-#define SEEN_XREG    2 /* ebx is used */
-#define SEEN_MEM     4 /* use mem[] for temporary storage */
-
 #define S13(X)		((X) & 0x1fff)
 #define S5(X)		((X) & 0x1f)
 #define IMMED		0x00002000
@@ -198,7 +194,6 @@ struct jit_ctx {
 	bool 			tmp_1_used;
 	bool 			tmp_2_used;
 	bool 			tmp_3_used;
-	bool			saw_ld_abs_ind;
 	bool			saw_frame_pointer;
 	bool			saw_call;
 	bool			saw_tail_call;
@@ -207,9 +202,7 @@ struct jit_ctx {
 
 #define TMP_REG_1	(MAX_BPF_JIT_REG + 0)
 #define TMP_REG_2	(MAX_BPF_JIT_REG + 1)
-#define SKB_HLEN_REG	(MAX_BPF_JIT_REG + 2)
-#define SKB_DATA_REG	(MAX_BPF_JIT_REG + 3)
-#define TMP_REG_3	(MAX_BPF_JIT_REG + 4)
+#define TMP_REG_3	(MAX_BPF_JIT_REG + 2)
 
 /* Map BPF registers to SPARC registers */
 static const int bpf2sparc[] = {
@@ -238,9 +231,6 @@ static const int bpf2sparc[] = {
 	[TMP_REG_1] = G1,
 	[TMP_REG_2] = G2,
 	[TMP_REG_3] = G3,
-
-	[SKB_HLEN_REG] = L4,
-	[SKB_DATA_REG] = L5,
 };
 
 static void emit(const u32 insn, struct jit_ctx *ctx)
@@ -800,25 +790,6 @@ static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
 	return 0;
 }
 
-static void load_skb_regs(struct jit_ctx *ctx, u8 r_skb)
-{
-	const u8 r_headlen = bpf2sparc[SKB_HLEN_REG];
-	const u8 r_data = bpf2sparc[SKB_DATA_REG];
-	const u8 r_tmp = bpf2sparc[TMP_REG_1];
-	unsigned int off;
-
-	off = offsetof(struct sk_buff, len);
-	emit(LD32I | RS1(r_skb) | S13(off) | RD(r_headlen), ctx);
-
-	off = offsetof(struct sk_buff, data_len);
-	emit(LD32I | RS1(r_skb) | S13(off) | RD(r_tmp), ctx);
-
-	emit(SUB | RS1(r_headlen) | RS2(r_tmp) | RD(r_headlen), ctx);
-
-	off = offsetof(struct sk_buff, data);
-	emit(LDPTRI | RS1(r_skb) | S13(off) | RD(r_data), ctx);
-}
-
 /* Just skip the save instruction and the ctx register move.  */
 #define BPF_TAILCALL_PROLOGUE_SKIP	16
 #define BPF_TAILCALL_CNT_SP_OFF		(STACK_BIAS + 128)
@@ -857,9 +828,6 @@ static void build_prologue(struct jit_ctx *ctx)
 
 	emit_reg_move(I0, O0, ctx);
 	/* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
-
-	if (ctx->saw_ld_abs_ind)
-		load_skb_regs(ctx, bpf2sparc[BPF_REG_1]);
 }
 
 static void build_epilogue(struct jit_ctx *ctx)
@@ -926,7 +894,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	const int i = insn - ctx->prog->insnsi;
 	const s16 off = insn->off;
 	const s32 imm = insn->imm;
-	u32 *func;
 
 	if (insn->src_reg == BPF_REG_FP)
 		ctx->saw_frame_pointer = true;
@@ -1225,16 +1192,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 		u8 *func = ((u8 *)__bpf_call_base) + imm;
 
 		ctx->saw_call = true;
-		if (ctx->saw_ld_abs_ind && bpf_helper_changes_pkt_data(func))
-			emit_reg_move(bpf2sparc[BPF_REG_1], L7, ctx);
 
 		emit_call((u32 *)func, ctx);
 		emit_nop(ctx);
 
 		emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
-
-		if (ctx->saw_ld_abs_ind && bpf_helper_changes_pkt_data(func))
-			load_skb_regs(ctx, L7);
 		break;
 	}
 
@@ -1412,43 +1374,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 		emit_nop(ctx);
 		break;
 	}
-#define CHOOSE_LOAD_FUNC(K, func) \
-		((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
-
-	/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
-	case BPF_LD | BPF_ABS | BPF_W:
-		func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_word);
-		goto common_load;
-	case BPF_LD | BPF_ABS | BPF_H:
-		func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_half);
-		goto common_load;
-	case BPF_LD | BPF_ABS | BPF_B:
-		func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_byte);
-		goto common_load;
-	/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
-	case BPF_LD | BPF_IND | BPF_W:
-		func = bpf_jit_load_word;
-		goto common_load;
-	case BPF_LD | BPF_IND | BPF_H:
-		func = bpf_jit_load_half;
-		goto common_load;
-
-	case BPF_LD | BPF_IND | BPF_B:
-		func = bpf_jit_load_byte;
-	common_load:
-		ctx->saw_ld_abs_ind = true;
-
-		emit_reg_move(bpf2sparc[BPF_REG_6], O0, ctx);
-		emit_loadimm(imm, O1, ctx);
-
-		if (BPF_MODE(code) == BPF_IND)
-			emit_alu(ADD, src, O1, ctx);
-
-		emit_call(func, ctx);
-		emit_alu_K(SRA, O1, 0, ctx);
-
-		emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
-		break;
 
 	default:
 		pr_err_once("unknown opcode %02x\n", code);
@@ -1583,12 +1508,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		build_epilogue(&ctx);
 
 		if (bpf_jit_enable > 1)
-			pr_info("Pass %d: shrink = %d, seen = [%c%c%c%c%c%c%c]\n", pass,
+			pr_info("Pass %d: shrink = %d, seen = [%c%c%c%c%c%c]\n", pass,
 				image_size - (ctx.idx * 4),
 				ctx.tmp_1_used ? '1' : ' ',
 				ctx.tmp_2_used ? '2' : ' ',
 				ctx.tmp_3_used ? '3' : ' ',
-				ctx.saw_ld_abs_ind ? 'L' : ' ',
 				ctx.saw_frame_pointer ? 'F' : ' ',
 				ctx.saw_call ? 'C' : ' ',
 				ctx.saw_tail_call ? 'T' : ' ');
diff --git a/arch/um/Kconfig.common b/arch/um/Kconfig.common
index c68add8df3ae..07f84c842cc3 100644
--- a/arch/um/Kconfig.common
+++ b/arch/um/Kconfig.common
@@ -54,10 +54,6 @@ config HZ
 	int
 	default 100
 
-config SUBARCH
-	string
-	option env="SUBARCH"
-
 config NR_CPUS
 	int
 	range 1 1
diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c
index d4e8c497ae86..dcf5ea28a281 100644
--- a/arch/um/drivers/ubd_kern.c
+++ b/arch/um/drivers/ubd_kern.c
@@ -208,19 +208,6 @@ static int fake_ide_media_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int fake_ide_media_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, fake_ide_media_proc_show, NULL);
-}
-
-static const struct file_operations fake_ide_media_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= fake_ide_media_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static void make_ide_entries(const char *dev_name)
 {
 	struct proc_dir_entry *dir, *ent;
@@ -231,7 +218,8 @@ static void make_ide_entries(const char *dev_name)
 	dir = proc_mkdir(dev_name, proc_ide);
 	if(!dir) return;
 
-	ent = proc_create("media", S_IRUGO, dir, &fake_ide_media_proc_fops);
+	ent = proc_create_single("media", S_IRUGO, dir,
+			fake_ide_media_proc_show);
 	if(!ent) return;
 	snprintf(name, sizeof(name), "ide0/%s", dev_name);
 	proc_symlink(dev_name, proc_ide_root, name);
diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild
index bb5a196c3061..b10dde6cb793 100644
--- a/arch/um/include/asm/Kbuild
+++ b/arch/um/include/asm/Kbuild
@@ -1,6 +1,7 @@
 generic-y += barrier.h
 generic-y += bpf_perf_event.h
 generic-y += bug.h
+generic-y += compat.h
 generic-y += current.h
 generic-y += delay.h
 generic-y += device.h
diff --git a/arch/um/kernel/ptrace.c b/arch/um/kernel/ptrace.c
index bc2a516c190f..1a1d88a4d940 100644
--- a/arch/um/kernel/ptrace.c
+++ b/arch/um/kernel/ptrace.c
@@ -115,17 +115,10 @@ long arch_ptrace(struct task_struct *child, long request,
 static void send_sigtrap(struct task_struct *tsk, struct uml_pt_regs *regs,
 		  int error_code)
 {
-	struct siginfo info;
-
-	memset(&info, 0, sizeof(info));
-	info.si_signo = SIGTRAP;
-	info.si_code = TRAP_BRKPT;
-
-	/* User-mode eip? */
-	info.si_addr = UPT_IS_USER(regs) ? (void __user *) UPT_IP(regs) : NULL;
-
 	/* Send us the fake SIGTRAP */
-	force_sig_info(SIGTRAP, &info, tsk);
+	force_sig_fault(SIGTRAP, TRAP_BRKPT,
+			/* User-mode eip? */
+			UPT_IS_USER(regs) ? (void __user *) UPT_IP(regs) : NULL, tsk);
 }
 
 /*
diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c
index b2b02df9896e..ec9a42c14c56 100644
--- a/arch/um/kernel/trap.c
+++ b/arch/um/kernel/trap.c
@@ -162,13 +162,9 @@ static void show_segv_info(struct uml_pt_regs *regs)
 
 static void bad_segv(struct faultinfo fi, unsigned long ip)
 {
-	struct siginfo si;
-
-	si.si_signo = SIGSEGV;
-	si.si_code = SEGV_ACCERR;
-	si.si_addr = (void __user *) FAULT_ADDRESS(fi);
 	current->thread.arch.faultinfo = fi;
-	force_sig_info(SIGSEGV, &si, current);
+	force_sig_fault(SIGSEGV, SEGV_ACCERR, (void __user *) FAULT_ADDRESS(fi),
+			current);
 }
 
 void fatal_sigsegv(void)
@@ -214,8 +210,8 @@ void segv_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs *regs)
 unsigned long segv(struct faultinfo fi, unsigned long ip, int is_user,
 		   struct uml_pt_regs *regs)
 {
-	struct siginfo si;
 	jmp_buf *catcher;
+	int si_code;
 	int err;
 	int is_write = FAULT_WRITE(fi);
 	unsigned long address = FAULT_ADDRESS(fi);
@@ -239,7 +235,7 @@ unsigned long segv(struct faultinfo fi, unsigned long ip, int is_user,
 
 	if (SEGV_IS_FIXABLE(&fi))
 		err = handle_page_fault(address, ip, is_write, is_user,
-					&si.si_code);
+					&si_code);
 	else {
 		err = -EFAULT;
 		/*
@@ -271,18 +267,14 @@ unsigned long segv(struct faultinfo fi, unsigned long ip, int is_user,
 	show_segv_info(regs);
 
 	if (err == -EACCES) {
-		si.si_signo = SIGBUS;
-		si.si_errno = 0;
-		si.si_code = BUS_ADRERR;
-		si.si_addr = (void __user *)address;
 		current->thread.arch.faultinfo = fi;
-		force_sig_info(SIGBUS, &si, current);
+		force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address,
+				current);
 	} else {
 		BUG_ON(err != -EFAULT);
-		si.si_signo = SIGSEGV;
-		si.si_addr = (void __user *) address;
 		current->thread.arch.faultinfo = fi;
-		force_sig_info(SIGSEGV, &si, current);
+		force_sig_fault(SIGSEGV, si_code, (void __user *) address,
+				current);
 	}
 
 out:
@@ -294,9 +286,7 @@ unsigned long segv(struct faultinfo fi, unsigned long ip, int is_user,
 
 void relay_signal(int sig, struct siginfo *si, struct uml_pt_regs *regs)
 {
-	struct faultinfo *fi;
-	struct siginfo clean_si;
-
+	int code, err;
 	if (!UPT_IS_USER(regs)) {
 		if (sig == SIGBUS)
 			printk(KERN_ERR "Bus error - the host /dev/shm or /tmp "
@@ -306,29 +296,21 @@ void relay_signal(int sig, struct siginfo *si, struct uml_pt_regs *regs)
 
 	arch_examine_signal(sig, regs);
 
-	clear_siginfo(&clean_si);
-	clean_si.si_signo = si->si_signo;
-	clean_si.si_errno = si->si_errno;
-	clean_si.si_code = si->si_code;
-	switch (sig) {
-	case SIGILL:
-	case SIGFPE:
-	case SIGSEGV:
-	case SIGBUS:
-	case SIGTRAP:
-		fi = UPT_FAULTINFO(regs);
-		clean_si.si_addr = (void __user *) FAULT_ADDRESS(*fi);
+	/* Is the signal layout for the signal known?
+	 * Signal data must be scrubbed to prevent information leaks.
+	 */
+	code = si->si_code;
+	err = si->si_errno;
+	if ((err == 0) && (siginfo_layout(sig, code) == SIL_FAULT)) {
+		struct faultinfo *fi = UPT_FAULTINFO(regs);
 		current->thread.arch.faultinfo = *fi;
-#ifdef __ARCH_SI_TRAPNO
-		clean_si.si_trapno = si->si_trapno;
-#endif
-		break;
-	default:
-		printk(KERN_ERR "Attempted to relay unknown signal %d (si_code = %d)\n",
-			sig, si->si_code);
+		force_sig_fault(sig, code, (void __user *)FAULT_ADDRESS(*fi),
+				current);
+	} else {
+		printk(KERN_ERR "Attempted to relay unknown signal %d (si_code = %d) with errno %d\n",
+		       sig, code, err);
+		force_sig(sig, current);
 	}
-
-	force_sig_info(sig, &clean_si, current);
 }
 
 void bus_handler(int sig, struct siginfo *si, struct uml_pt_regs *regs)
diff --git a/arch/unicore32/Kconfig b/arch/unicore32/Kconfig
index 462e59a7ae78..03f991e44288 100644
--- a/arch/unicore32/Kconfig
+++ b/arch/unicore32/Kconfig
@@ -19,6 +19,8 @@ config UNICORE32
 	select ARCH_WANT_FRAME_POINTERS
 	select GENERIC_IOMAP
 	select MODULES_USE_ELF_REL
+	select NEED_DMA_MAP_STATE
+	select SWIOTLB
 	help
 	  UniCore-32 is 32-bit Instruction Set Architecture,
 	  including a series of low-power-consumption RISC chip
@@ -61,9 +63,6 @@ config ARCH_MAY_HAVE_PC_FDC
 config ZONE_DMA
 	def_bool y
 
-config NEED_DMA_MAP_STATE
-       def_bool y
-
 source "init/Kconfig"
 
 source "kernel/Kconfig.freezer"
diff --git a/arch/unicore32/include/asm/Kbuild b/arch/unicore32/include/asm/Kbuild
index 6f70c76c81fc..bfc7abe77905 100644
--- a/arch/unicore32/include/asm/Kbuild
+++ b/arch/unicore32/include/asm/Kbuild
@@ -1,5 +1,6 @@
 generic-y += atomic.h
 generic-y += bugs.h
+generic-y += compat.h
 generic-y += current.h
 generic-y += device.h
 generic-y += div64.h
diff --git a/arch/unicore32/kernel/fpu-ucf64.c b/arch/unicore32/kernel/fpu-ucf64.c
index 12c8c9527b8e..8594b168f25e 100644
--- a/arch/unicore32/kernel/fpu-ucf64.c
+++ b/arch/unicore32/kernel/fpu-ucf64.c
@@ -52,14 +52,14 @@
  * Raise a SIGFPE for the current process.
  * sicode describes the signal being raised.
  */
-void ucf64_raise_sigfpe(unsigned int sicode, struct pt_regs *regs)
+void ucf64_raise_sigfpe(struct pt_regs *regs)
 {
 	siginfo_t info;
 
-	memset(&info, 0, sizeof(info));
+	clear_siginfo(&info);
 
 	info.si_signo = SIGFPE;
-	info.si_code = sicode;
+	info.si_code = FPE_FLTUNK;
 	info.si_addr = (void __user *)(instruction_pointer(regs) - 4);
 
 	/*
@@ -94,7 +94,7 @@ void ucf64_exchandler(u32 inst, u32 fpexc, struct pt_regs *regs)
 		pr_debug("UniCore-F64 FPSCR 0x%08x INST 0x%08x\n",
 				cff(FPSCR), inst);
 
-		ucf64_raise_sigfpe(0, regs);
+		ucf64_raise_sigfpe(regs);
 		return;
 	}
 
diff --git a/arch/unicore32/mm/Kconfig b/arch/unicore32/mm/Kconfig
index e9154a59d561..82759b6aba67 100644
--- a/arch/unicore32/mm/Kconfig
+++ b/arch/unicore32/mm/Kconfig
@@ -39,14 +39,3 @@ config CPU_TLB_SINGLE_ENTRY_DISABLE
 	default y
 	help
 	  Say Y here to disable the TLB single entry operations.
-
-config SWIOTLB
-	def_bool y
-	select DMA_DIRECT_OPS
-
-config IOMMU_HELPER
-	def_bool SWIOTLB
-
-config NEED_SG_DMA_LENGTH
-	def_bool SWIOTLB
-
diff --git a/arch/unicore32/mm/fault.c b/arch/unicore32/mm/fault.c
index bbefcc46a45e..381473412937 100644
--- a/arch/unicore32/mm/fault.c
+++ b/arch/unicore32/mm/fault.c
@@ -125,6 +125,7 @@ static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
 	tsk->thread.address = addr;
 	tsk->thread.error_code = fsr;
 	tsk->thread.trap_no = 14;
+	clear_siginfo(&si);
 	si.si_signo = sig;
 	si.si_errno = 0;
 	si.si_code = code;
@@ -472,6 +473,7 @@ asmlinkage void do_DataAbort(unsigned long addr, unsigned int fsr,
 	printk(KERN_ALERT "Unhandled fault: %s (0x%03x) at 0x%08lx\n",
 	       inf->name, fsr, addr);
 
+	clear_siginfo(&info);
 	info.si_signo = inf->sig;
 	info.si_errno = 0;
 	info.si_code = inf->code;
@@ -491,6 +493,7 @@ asmlinkage void do_PrefetchAbort(unsigned long addr,
 	printk(KERN_ALERT "Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
 	       inf->name, ifsr, addr);
 
+	clear_siginfo(&info);
 	info.si_signo = inf->sig;
 	info.si_errno = 0;
 	info.si_code = inf->code;
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index c07f492b871a..4849f148946e 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 # Select 32 or 64 bit
 config 64BIT
-	bool "64-bit kernel" if ARCH = "x86"
-	default ARCH != "i386"
+	bool "64-bit kernel" if "$(ARCH)" = "x86"
+	default "$(ARCH)" != "i386"
 	---help---
 	  Say yes to build a 64-bit kernel - formerly known as x86_64
 	  Say no to build a 32-bit kernel - formerly known as i386
@@ -28,6 +28,8 @@ config X86_64
 	select ARCH_USE_CMPXCHG_LOCKREF
 	select HAVE_ARCH_SOFT_DIRTY
 	select MODULES_USE_ELF_RELA
+	select NEED_DMA_MAP_STATE
+	select SWIOTLB
 	select X86_DEV_DMA_OPS
 	select ARCH_HAS_SYSCALL_WRAPPER
 
@@ -58,8 +60,10 @@ config X86
 	select ARCH_HAS_KCOV			if X86_64
 	select ARCH_HAS_MEMBARRIER_SYNC_CORE
 	select ARCH_HAS_PMEM_API		if X86_64
+	select ARCH_HAS_PTE_SPECIAL
 	select ARCH_HAS_REFCOUNT
 	select ARCH_HAS_UACCESS_FLUSHCACHE	if X86_64
+	select ARCH_HAS_UACCESS_MCSAFE		if X86_64
 	select ARCH_HAS_SET_MEMORY
 	select ARCH_HAS_SG_CHAIN
 	select ARCH_HAS_STRICT_KERNEL_RWX
@@ -126,7 +130,7 @@ config X86
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD if X86_64
 	select HAVE_ARCH_VMAP_STACK		if X86_64
 	select HAVE_ARCH_WITHIN_STACK_FRAMES
-	select HAVE_CC_STACKPROTECTOR
+	select HAVE_CC_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR
 	select HAVE_CMPXCHG_DOUBLE
 	select HAVE_CMPXCHG_LOCAL
 	select HAVE_CONTEXT_TRACKING		if X86_64
@@ -134,11 +138,10 @@ config X86
 	select HAVE_C_RECORDMCOUNT
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DEBUG_STACKOVERFLOW
-	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_CONTIGUOUS
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_DYNAMIC_FTRACE_WITH_REGS
-	select HAVE_EBPF_JIT			if X86_64
+	select HAVE_EBPF_JIT
 	select HAVE_EFFICIENT_UNALIGNED_ACCESS
 	select HAVE_EXIT_THREAD
 	select HAVE_FENTRY			if X86_64 || DYNAMIC_FTRACE
@@ -184,6 +187,7 @@ config X86
 	select HAVE_UNSTABLE_SCHED_CLOCK
 	select HAVE_USER_RETURN_NOTIFIER
 	select IRQ_FORCED_THREADING
+	select NEED_SG_DMA_LENGTH
 	select PCI_LOCKLESS_CONFIG
 	select PERF_EVENTS
 	select RTC_LIB
@@ -236,13 +240,6 @@ config ARCH_MMAP_RND_COMPAT_BITS_MAX
 config SBUS
 	bool
 
-config NEED_DMA_MAP_STATE
-	def_bool y
-	depends on X86_64 || INTEL_IOMMU || DMA_API_DEBUG || SWIOTLB
-
-config NEED_SG_DMA_LENGTH
-	def_bool y
-
 config GENERIC_ISA_DMA
 	def_bool y
 	depends on ISA_DMA_API
@@ -345,6 +342,15 @@ config PGTABLE_LEVELS
 	default 2
 
 source "init/Kconfig"
+
+config CC_HAS_SANE_STACKPROTECTOR
+	bool
+	default $(success,$(srctree)/scripts/gcc-x86_64-has-stack-protector.sh $(CC)) if 64BIT
+	default $(success,$(srctree)/scripts/gcc-x86_32-has-stack-protector.sh $(CC))
+	help
+	   We have to make sure stack protector is unconditionally disabled if
+	   the compiler produces broken code.
+
 source "kernel/Kconfig.freezer"
 
 menu "Processor type and features"
@@ -875,6 +881,7 @@ config DMI
 
 config GART_IOMMU
 	bool "Old AMD GART IOMMU support"
+	select IOMMU_HELPER
 	select SWIOTLB
 	depends on X86_64 && PCI && AMD_NB
 	---help---
@@ -896,6 +903,7 @@ config GART_IOMMU
 
 config CALGARY_IOMMU
 	bool "IBM Calgary IOMMU support"
+	select IOMMU_HELPER
 	select SWIOTLB
 	depends on X86_64 && PCI
 	---help---
@@ -923,20 +931,6 @@ config CALGARY_IOMMU_ENABLED_BY_DEFAULT
 	  Calgary anyway, pass 'iommu=calgary' on the kernel command line.
 	  If unsure, say Y.
 
-# need this always selected by IOMMU for the VIA workaround
-config SWIOTLB
-	def_bool y if X86_64
-	---help---
-	  Support for software bounce buffers used on x86-64 systems
-	  which don't have a hardware IOMMU. Using this PCI devices
-	  which can only access 32-bits of memory can be used on systems
-	  with more than 3 GB of memory.
-	  If unsure, say Y.
-
-config IOMMU_HELPER
-	def_bool y
-	depends on CALGARY_IOMMU || GART_IOMMU || SWIOTLB || AMD_IOMMU
-
 config MAXSMP
 	bool "Enable Maximum number of SMP Processors and NUMA Nodes"
 	depends on X86_64 && SMP && DEBUG_KERNEL
@@ -1458,6 +1452,7 @@ config HIGHMEM
 config X86_PAE
 	bool "PAE (Physical Address Extension) Support"
 	depends on X86_32 && !HIGHMEM4G
+	select PHYS_ADDR_T_64BIT
 	select SWIOTLB
 	---help---
 	  PAE is required for NX support, and furthermore enables
@@ -1485,14 +1480,6 @@ config X86_5LEVEL
 
 	  Say N if unsure.
 
-config ARCH_PHYS_ADDR_T_64BIT
-	def_bool y
-	depends on X86_64 || X86_PAE
-
-config ARCH_DMA_ADDR_T_64BIT
-	def_bool y
-	depends on X86_64 || HIGHMEM64G
-
 config X86_DIRECT_GBPAGES
 	def_bool y
 	depends on X86_64 && !DEBUG_PAGEALLOC
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 192e4d2f9efc..c6dd1d980081 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -72,6 +72,9 @@ config EARLY_PRINTK_USB_XDBC
 	  You should normally say N here, unless you want to debug early
 	  crashes or need a very simple printk logging facility.
 
+config MCSAFE_TEST
+	def_bool n
+
 config X86_PTDUMP_CORE
 	def_bool n
 
diff --git a/arch/x86/boot/compressed/eboot.c b/arch/x86/boot/compressed/eboot.c
index 09f36c0d9d4f..a8a8642d2b0b 100644
--- a/arch/x86/boot/compressed/eboot.c
+++ b/arch/x86/boot/compressed/eboot.c
@@ -109,23 +109,34 @@ void efi_char16_printk(efi_system_table_t *table, efi_char16_t *str)
 }
 
 static efi_status_t
-__setup_efi_pci32(efi_pci_io_protocol_32 *pci, struct pci_setup_rom **__rom)
+__setup_efi_pci(efi_pci_io_protocol_t *pci, struct pci_setup_rom **__rom)
 {
 	struct pci_setup_rom *rom = NULL;
 	efi_status_t status;
 	unsigned long size;
-	uint64_t attributes;
+	uint64_t attributes, romsize;
+	void *romimage;
 
-	status = efi_early->call(pci->attributes, pci,
-				 EfiPciIoAttributeOperationGet, 0, 0,
-				 &attributes);
+	status = efi_call_proto(efi_pci_io_protocol, attributes, pci,
+				EfiPciIoAttributeOperationGet, 0, 0,
+				&attributes);
 	if (status != EFI_SUCCESS)
 		return status;
 
-	if (!pci->romimage || !pci->romsize)
+	/*
+	 * Some firmware images contain EFI function pointers at the place where the
+	 * romimage and romsize fields are supposed to be. Typically the EFI
+	 * code is mapped at high addresses, translating to an unrealistically
+	 * large romsize. The UEFI spec limits the size of option ROMs to 16
+	 * MiB so we reject any ROMs over 16 MiB in size to catch this.
+	 */
+	romimage = (void *)(unsigned long)efi_table_attr(efi_pci_io_protocol,
+							 romimage, pci);
+	romsize = efi_table_attr(efi_pci_io_protocol, romsize, pci);
+	if (!romimage || !romsize || romsize > SZ_16M)
 		return EFI_INVALID_PARAMETER;
 
-	size = pci->romsize + sizeof(*rom);
+	size = romsize + sizeof(*rom);
 
 	status = efi_call_early(allocate_pool, EFI_LOADER_DATA, size, &rom);
 	if (status != EFI_SUCCESS) {
@@ -141,30 +152,32 @@ __setup_efi_pci32(efi_pci_io_protocol_32 *pci, struct pci_setup_rom **__rom)
 	rom->pcilen = pci->romsize;
 	*__rom = rom;
 
-	status = efi_early->call(pci->pci.read, pci, EfiPciIoWidthUint16,
-				 PCI_VENDOR_ID, 1, &(rom->vendor));
+	status = efi_call_proto(efi_pci_io_protocol, pci.read, pci,
+				EfiPciIoWidthUint16, PCI_VENDOR_ID, 1,
+				&rom->vendor);
 
 	if (status != EFI_SUCCESS) {
 		efi_printk(sys_table, "Failed to read rom->vendor\n");
 		goto free_struct;
 	}
 
-	status = efi_early->call(pci->pci.read, pci, EfiPciIoWidthUint16,
-				 PCI_DEVICE_ID, 1, &(rom->devid));
+	status = efi_call_proto(efi_pci_io_protocol, pci.read, pci,
+				EfiPciIoWidthUint16, PCI_DEVICE_ID, 1,
+				&rom->devid);
 
 	if (status != EFI_SUCCESS) {
 		efi_printk(sys_table, "Failed to read rom->devid\n");
 		goto free_struct;
 	}
 
-	status = efi_early->call(pci->get_location, pci, &(rom->segment),
-				 &(rom->bus), &(rom->device), &(rom->function));
+	status = efi_call_proto(efi_pci_io_protocol, get_location, pci,
+				&rom->segment, &rom->bus, &rom->device,
+				&rom->function);
 
 	if (status != EFI_SUCCESS)
 		goto free_struct;
 
-	memcpy(rom->romdata, (void *)(unsigned long)pci->romimage,
-	       pci->romsize);
+	memcpy(rom->romdata, romimage, romsize);
 	return status;
 
 free_struct:
@@ -176,7 +189,7 @@ static void
 setup_efi_pci32(struct boot_params *params, void **pci_handle,
 		unsigned long size)
 {
-	efi_pci_io_protocol_32 *pci = NULL;
+	efi_pci_io_protocol_t *pci = NULL;
 	efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID;
 	u32 *handles = (u32 *)(unsigned long)pci_handle;
 	efi_status_t status;
@@ -203,7 +216,7 @@ setup_efi_pci32(struct boot_params *params, void **pci_handle,
 		if (!pci)
 			continue;
 
-		status = __setup_efi_pci32(pci, &rom);
+		status = __setup_efi_pci(pci, &rom);
 		if (status != EFI_SUCCESS)
 			continue;
 
@@ -217,74 +230,11 @@ setup_efi_pci32(struct boot_params *params, void **pci_handle,
 	}
 }
 
-static efi_status_t
-__setup_efi_pci64(efi_pci_io_protocol_64 *pci, struct pci_setup_rom **__rom)
-{
-	struct pci_setup_rom *rom;
-	efi_status_t status;
-	unsigned long size;
-	uint64_t attributes;
-
-	status = efi_early->call(pci->attributes, pci,
-				 EfiPciIoAttributeOperationGet, 0,
-				 &attributes);
-	if (status != EFI_SUCCESS)
-		return status;
-
-	if (!pci->romimage || !pci->romsize)
-		return EFI_INVALID_PARAMETER;
-
-	size = pci->romsize + sizeof(*rom);
-
-	status = efi_call_early(allocate_pool, EFI_LOADER_DATA, size, &rom);
-	if (status != EFI_SUCCESS) {
-		efi_printk(sys_table, "Failed to alloc mem for rom\n");
-		return status;
-	}
-
-	rom->data.type = SETUP_PCI;
-	rom->data.len = size - sizeof(struct setup_data);
-	rom->data.next = 0;
-	rom->pcilen = pci->romsize;
-	*__rom = rom;
-
-	status = efi_early->call(pci->pci.read, pci, EfiPciIoWidthUint16,
-				 PCI_VENDOR_ID, 1, &(rom->vendor));
-
-	if (status != EFI_SUCCESS) {
-		efi_printk(sys_table, "Failed to read rom->vendor\n");
-		goto free_struct;
-	}
-
-	status = efi_early->call(pci->pci.read, pci, EfiPciIoWidthUint16,
-				 PCI_DEVICE_ID, 1, &(rom->devid));
-
-	if (status != EFI_SUCCESS) {
-		efi_printk(sys_table, "Failed to read rom->devid\n");
-		goto free_struct;
-	}
-
-	status = efi_early->call(pci->get_location, pci, &(rom->segment),
-				 &(rom->bus), &(rom->device), &(rom->function));
-
-	if (status != EFI_SUCCESS)
-		goto free_struct;
-
-	memcpy(rom->romdata, (void *)(unsigned long)pci->romimage,
-	       pci->romsize);
-	return status;
-
-free_struct:
-	efi_call_early(free_pool, rom);
-	return status;
-
-}
-
 static void
 setup_efi_pci64(struct boot_params *params, void **pci_handle,
 		unsigned long size)
 {
-	efi_pci_io_protocol_64 *pci = NULL;
+	efi_pci_io_protocol_t *pci = NULL;
 	efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID;
 	u64 *handles = (u64 *)(unsigned long)pci_handle;
 	efi_status_t status;
@@ -311,7 +261,7 @@ setup_efi_pci64(struct boot_params *params, void **pci_handle,
 		if (!pci)
 			continue;
 
-		status = __setup_efi_pci64(pci, &rom);
+		status = __setup_efi_pci(pci, &rom);
 		if (status != EFI_SUCCESS)
 			continue;
 
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 5f07333bb224..a450ad573dcb 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -15,7 +15,6 @@ obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o
 
 obj-$(CONFIG_CRYPTO_AES_586) += aes-i586.o
 obj-$(CONFIG_CRYPTO_TWOFISH_586) += twofish-i586.o
-obj-$(CONFIG_CRYPTO_SALSA20_586) += salsa20-i586.o
 obj-$(CONFIG_CRYPTO_SERPENT_SSE2_586) += serpent-sse2-i586.o
 
 obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o
@@ -24,7 +23,6 @@ obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o
 obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o
 obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o
 obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o
-obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o
 obj-$(CONFIG_CRYPTO_CHACHA20_X86_64) += chacha20-x86_64.o
 obj-$(CONFIG_CRYPTO_SERPENT_SSE2_X86_64) += serpent-sse2-x86_64.o
 obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
@@ -38,6 +36,16 @@ obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o
 obj-$(CONFIG_CRYPTO_CRCT10DIF_PCLMUL) += crct10dif-pclmul.o
 obj-$(CONFIG_CRYPTO_POLY1305_X86_64) += poly1305-x86_64.o
 
+obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o
+obj-$(CONFIG_CRYPTO_AEGIS128L_AESNI_SSE2) += aegis128l-aesni.o
+obj-$(CONFIG_CRYPTO_AEGIS256_AESNI_SSE2) += aegis256-aesni.o
+
+obj-$(CONFIG_CRYPTO_MORUS640_GLUE) += morus640_glue.o
+obj-$(CONFIG_CRYPTO_MORUS1280_GLUE) += morus1280_glue.o
+
+obj-$(CONFIG_CRYPTO_MORUS640_SSE2) += morus640-sse2.o
+obj-$(CONFIG_CRYPTO_MORUS1280_SSE2) += morus1280-sse2.o
+
 # These modules require assembler to support AVX.
 ifeq ($(avx_supported),yes)
 	obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX_X86_64) += \
@@ -55,11 +63,12 @@ ifeq ($(avx2_supported),yes)
 	obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb/
 	obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb/
 	obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb/
+
+	obj-$(CONFIG_CRYPTO_MORUS1280_AVX2) += morus1280-avx2.o
 endif
 
 aes-i586-y := aes-i586-asm_32.o aes_glue.o
 twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o
-salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o
 serpent-sse2-i586-y := serpent-sse2-i586-asm_32.o serpent_sse2_glue.o
 
 aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o
@@ -68,10 +77,16 @@ camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o
 blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o
 twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o
 twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o
-salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o
 chacha20-x86_64-y := chacha20-ssse3-x86_64.o chacha20_glue.o
 serpent-sse2-x86_64-y := serpent-sse2-x86_64-asm_64.o serpent_sse2_glue.o
 
+aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o
+aegis128l-aesni-y := aegis128l-aesni-asm.o aegis128l-aesni-glue.o
+aegis256-aesni-y := aegis256-aesni-asm.o aegis256-aesni-glue.o
+
+morus640-sse2-y := morus640-sse2-asm.o morus640-sse2-glue.o
+morus1280-sse2-y := morus1280-sse2-asm.o morus1280-sse2-glue.o
+
 ifeq ($(avx_supported),yes)
 	camellia-aesni-avx-x86_64-y := camellia-aesni-avx-asm_64.o \
 					camellia_aesni_avx_glue.o
@@ -87,6 +102,8 @@ ifeq ($(avx2_supported),yes)
 	camellia-aesni-avx2-y := camellia-aesni-avx2-asm_64.o camellia_aesni_avx2_glue.o
 	chacha20-x86_64-y += chacha20-avx2-x86_64.o
 	serpent-avx2-y := serpent-avx2-asm_64.o serpent_avx2_glue.o
+
+	morus1280-avx2-y := morus1280-avx2-asm.o morus1280-avx2-glue.o
 endif
 
 aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o
diff --git a/arch/x86/crypto/aegis128-aesni-asm.S b/arch/x86/crypto/aegis128-aesni-asm.S
new file mode 100644
index 000000000000..9254e0b6cc06
--- /dev/null
+++ b/arch/x86/crypto/aegis128-aesni-asm.S
@@ -0,0 +1,749 @@
+/*
+ * AES-NI + SSE2 implementation of AEGIS-128
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/frame.h>
+
+#define STATE0	%xmm0
+#define STATE1	%xmm1
+#define STATE2	%xmm2
+#define STATE3	%xmm3
+#define STATE4	%xmm4
+#define KEY	%xmm5
+#define MSG	%xmm5
+#define T0	%xmm6
+#define T1	%xmm7
+
+#define STATEP	%rdi
+#define LEN	%rsi
+#define SRC	%rdx
+#define DST	%rcx
+
+.section .rodata.cst16.aegis128_const, "aM", @progbits, 32
+.align 16
+.Laegis128_const_0:
+	.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
+	.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
+.Laegis128_const_1:
+	.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
+	.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
+
+.section .rodata.cst16.aegis128_counter, "aM", @progbits, 16
+.align 16
+.Laegis128_counter:
+	.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
+	.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
+
+.text
+
+/*
+ * aegis128_update
+ * input:
+ *   STATE[0-4] - input state
+ * output:
+ *   STATE[0-4] - output state (shifted positions)
+ * changed:
+ *   T0
+ */
+.macro aegis128_update
+	movdqa STATE4, T0
+	aesenc STATE0, STATE4
+	aesenc STATE1, STATE0
+	aesenc STATE2, STATE1
+	aesenc STATE3, STATE2
+	aesenc T0,     STATE3
+.endm
+
+/*
+ * __load_partial: internal ABI
+ * input:
+ *   LEN - bytes
+ *   SRC - src
+ * output:
+ *   MSG  - message block
+ * changed:
+ *   T0
+ *   %r8
+ *   %r9
+ */
+__load_partial:
+	xor %r9, %r9
+	pxor MSG, MSG
+
+	mov LEN, %r8
+	and $0x1, %r8
+	jz .Lld_partial_1
+
+	mov LEN, %r8
+	and $0x1E, %r8
+	add SRC, %r8
+	mov (%r8), %r9b
+
+.Lld_partial_1:
+	mov LEN, %r8
+	and $0x2, %r8
+	jz .Lld_partial_2
+
+	mov LEN, %r8
+	and $0x1C, %r8
+	add SRC, %r8
+	shl $0x10, %r9
+	mov (%r8), %r9w
+
+.Lld_partial_2:
+	mov LEN, %r8
+	and $0x4, %r8
+	jz .Lld_partial_4
+
+	mov LEN, %r8
+	and $0x18, %r8
+	add SRC, %r8
+	shl $32, %r9
+	mov (%r8), %r8d
+	xor %r8, %r9
+
+.Lld_partial_4:
+	movq %r9, MSG
+
+	mov LEN, %r8
+	and $0x8, %r8
+	jz .Lld_partial_8
+
+	mov LEN, %r8
+	and $0x10, %r8
+	add SRC, %r8
+	pslldq $8, MSG
+	movq (%r8), T0
+	pxor T0, MSG
+
+.Lld_partial_8:
+	ret
+ENDPROC(__load_partial)
+
+/*
+ * __store_partial: internal ABI
+ * input:
+ *   LEN - bytes
+ *   DST - dst
+ * output:
+ *   T0   - message block
+ * changed:
+ *   %r8
+ *   %r9
+ *   %r10
+ */
+__store_partial:
+	mov LEN, %r8
+	mov DST, %r9
+
+	movq T0, %r10
+
+	cmp $8, %r8
+	jl .Lst_partial_8
+
+	mov %r10, (%r9)
+	psrldq $8, T0
+	movq T0, %r10
+
+	sub $8, %r8
+	add $8, %r9
+
+.Lst_partial_8:
+	cmp $4, %r8
+	jl .Lst_partial_4
+
+	mov %r10d, (%r9)
+	shr $32, %r10
+
+	sub $4, %r8
+	add $4, %r9
+
+.Lst_partial_4:
+	cmp $2, %r8
+	jl .Lst_partial_2
+
+	mov %r10w, (%r9)
+	shr $0x10, %r10
+
+	sub $2, %r8
+	add $2, %r9
+
+.Lst_partial_2:
+	cmp $1, %r8
+	jl .Lst_partial_1
+
+	mov %r10b, (%r9)
+
+.Lst_partial_1:
+	ret
+ENDPROC(__store_partial)
+
+/*
+ * void crypto_aegis128_aesni_init(void *state, const void *key, const void *iv);
+ */
+ENTRY(crypto_aegis128_aesni_init)
+	FRAME_BEGIN
+
+	/* load IV: */
+	movdqu (%rdx), T1
+
+	/* load key: */
+	movdqa (%rsi), KEY
+	pxor KEY, T1
+	movdqa T1, STATE0
+	movdqa KEY, STATE3
+	movdqa KEY, STATE4
+
+	/* load the constants: */
+	movdqa .Laegis128_const_0, STATE2
+	movdqa .Laegis128_const_1, STATE1
+	pxor STATE2, STATE3
+	pxor STATE1, STATE4
+
+	/* update 10 times with KEY / KEY xor IV: */
+	aegis128_update; pxor KEY, STATE4
+	aegis128_update; pxor T1,  STATE3
+	aegis128_update; pxor KEY, STATE2
+	aegis128_update; pxor T1,  STATE1
+	aegis128_update; pxor KEY, STATE0
+	aegis128_update; pxor T1,  STATE4
+	aegis128_update; pxor KEY, STATE3
+	aegis128_update; pxor T1,  STATE2
+	aegis128_update; pxor KEY, STATE1
+	aegis128_update; pxor T1,  STATE0
+
+	/* store the state: */
+	movdqu STATE0, 0x00(STATEP)
+	movdqu STATE1, 0x10(STATEP)
+	movdqu STATE2, 0x20(STATEP)
+	movdqu STATE3, 0x30(STATEP)
+	movdqu STATE4, 0x40(STATEP)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128_aesni_init)
+
+/*
+ * void crypto_aegis128_aesni_ad(void *state, unsigned int length,
+ *                               const void *data);
+ */
+ENTRY(crypto_aegis128_aesni_ad)
+	FRAME_BEGIN
+
+	cmp $0x10, LEN
+	jb .Lad_out
+
+	/* load the state: */
+	movdqu 0x00(STATEP), STATE0
+	movdqu 0x10(STATEP), STATE1
+	movdqu 0x20(STATEP), STATE2
+	movdqu 0x30(STATEP), STATE3
+	movdqu 0x40(STATEP), STATE4
+
+	mov SRC, %r8
+	and $0xF, %r8
+	jnz .Lad_u_loop
+
+.align 8
+.Lad_a_loop:
+	movdqa 0x00(SRC), MSG
+	aegis128_update
+	pxor MSG, STATE4
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lad_out_1
+
+	movdqa 0x10(SRC), MSG
+	aegis128_update
+	pxor MSG, STATE3
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lad_out_2
+
+	movdqa 0x20(SRC), MSG
+	aegis128_update
+	pxor MSG, STATE2
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lad_out_3
+
+	movdqa 0x30(SRC), MSG
+	aegis128_update
+	pxor MSG, STATE1
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lad_out_4
+
+	movdqa 0x40(SRC), MSG
+	aegis128_update
+	pxor MSG, STATE0
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lad_out_0
+
+	add $0x50, SRC
+	jmp .Lad_a_loop
+
+.align 8
+.Lad_u_loop:
+	movdqu 0x00(SRC), MSG
+	aegis128_update
+	pxor MSG, STATE4
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lad_out_1
+
+	movdqu 0x10(SRC), MSG
+	aegis128_update
+	pxor MSG, STATE3
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lad_out_2
+
+	movdqu 0x20(SRC), MSG
+	aegis128_update
+	pxor MSG, STATE2
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lad_out_3
+
+	movdqu 0x30(SRC), MSG
+	aegis128_update
+	pxor MSG, STATE1
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lad_out_4
+
+	movdqu 0x40(SRC), MSG
+	aegis128_update
+	pxor MSG, STATE0
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lad_out_0
+
+	add $0x50, SRC
+	jmp .Lad_u_loop
+
+	/* store the state: */
+.Lad_out_0:
+	movdqu STATE0, 0x00(STATEP)
+	movdqu STATE1, 0x10(STATEP)
+	movdqu STATE2, 0x20(STATEP)
+	movdqu STATE3, 0x30(STATEP)
+	movdqu STATE4, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Lad_out_1:
+	movdqu STATE4, 0x00(STATEP)
+	movdqu STATE0, 0x10(STATEP)
+	movdqu STATE1, 0x20(STATEP)
+	movdqu STATE2, 0x30(STATEP)
+	movdqu STATE3, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Lad_out_2:
+	movdqu STATE3, 0x00(STATEP)
+	movdqu STATE4, 0x10(STATEP)
+	movdqu STATE0, 0x20(STATEP)
+	movdqu STATE1, 0x30(STATEP)
+	movdqu STATE2, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Lad_out_3:
+	movdqu STATE2, 0x00(STATEP)
+	movdqu STATE3, 0x10(STATEP)
+	movdqu STATE4, 0x20(STATEP)
+	movdqu STATE0, 0x30(STATEP)
+	movdqu STATE1, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Lad_out_4:
+	movdqu STATE1, 0x00(STATEP)
+	movdqu STATE2, 0x10(STATEP)
+	movdqu STATE3, 0x20(STATEP)
+	movdqu STATE4, 0x30(STATEP)
+	movdqu STATE0, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Lad_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128_aesni_ad)
+
+.macro encrypt_block a s0 s1 s2 s3 s4 i
+	movdq\a (\i * 0x10)(SRC), MSG
+	movdqa MSG, T0
+	pxor \s1, T0
+	pxor \s4, T0
+	movdqa \s2, T1
+	pand \s3, T1
+	pxor T1, T0
+	movdq\a T0, (\i * 0x10)(DST)
+
+	aegis128_update
+	pxor MSG, \s4
+
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lenc_out_\i
+.endm
+
+/*
+ * void crypto_aegis128_aesni_enc(void *state, unsigned int length,
+ *                                const void *src, void *dst);
+ */
+ENTRY(crypto_aegis128_aesni_enc)
+	FRAME_BEGIN
+
+	cmp $0x10, LEN
+	jb .Lenc_out
+
+	/* load the state: */
+	movdqu 0x00(STATEP), STATE0
+	movdqu 0x10(STATEP), STATE1
+	movdqu 0x20(STATEP), STATE2
+	movdqu 0x30(STATEP), STATE3
+	movdqu 0x40(STATEP), STATE4
+
+	mov  SRC,  %r8
+	or   DST,  %r8
+	and $0xF, %r8
+	jnz .Lenc_u_loop
+
+.align 8
+.Lenc_a_loop:
+	encrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0
+	encrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1
+	encrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2
+	encrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3
+	encrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4
+
+	add $0x50, SRC
+	add $0x50, DST
+	jmp .Lenc_a_loop
+
+.align 8
+.Lenc_u_loop:
+	encrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0
+	encrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1
+	encrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2
+	encrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3
+	encrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4
+
+	add $0x50, SRC
+	add $0x50, DST
+	jmp .Lenc_u_loop
+
+	/* store the state: */
+.Lenc_out_0:
+	movdqu STATE4, 0x00(STATEP)
+	movdqu STATE0, 0x10(STATEP)
+	movdqu STATE1, 0x20(STATEP)
+	movdqu STATE2, 0x30(STATEP)
+	movdqu STATE3, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Lenc_out_1:
+	movdqu STATE3, 0x00(STATEP)
+	movdqu STATE4, 0x10(STATEP)
+	movdqu STATE0, 0x20(STATEP)
+	movdqu STATE1, 0x30(STATEP)
+	movdqu STATE2, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Lenc_out_2:
+	movdqu STATE2, 0x00(STATEP)
+	movdqu STATE3, 0x10(STATEP)
+	movdqu STATE4, 0x20(STATEP)
+	movdqu STATE0, 0x30(STATEP)
+	movdqu STATE1, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Lenc_out_3:
+	movdqu STATE1, 0x00(STATEP)
+	movdqu STATE2, 0x10(STATEP)
+	movdqu STATE3, 0x20(STATEP)
+	movdqu STATE4, 0x30(STATEP)
+	movdqu STATE0, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Lenc_out_4:
+	movdqu STATE0, 0x00(STATEP)
+	movdqu STATE1, 0x10(STATEP)
+	movdqu STATE2, 0x20(STATEP)
+	movdqu STATE3, 0x30(STATEP)
+	movdqu STATE4, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Lenc_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128_aesni_enc)
+
+/*
+ * void crypto_aegis128_aesni_enc_tail(void *state, unsigned int length,
+ *                                     const void *src, void *dst);
+ */
+ENTRY(crypto_aegis128_aesni_enc_tail)
+	FRAME_BEGIN
+
+	/* load the state: */
+	movdqu 0x00(STATEP), STATE0
+	movdqu 0x10(STATEP), STATE1
+	movdqu 0x20(STATEP), STATE2
+	movdqu 0x30(STATEP), STATE3
+	movdqu 0x40(STATEP), STATE4
+
+	/* encrypt message: */
+	call __load_partial
+
+	movdqa MSG, T0
+	pxor STATE1, T0
+	pxor STATE4, T0
+	movdqa STATE2, T1
+	pand STATE3, T1
+	pxor T1, T0
+
+	call __store_partial
+
+	aegis128_update
+	pxor MSG, STATE4
+
+	/* store the state: */
+	movdqu STATE4, 0x00(STATEP)
+	movdqu STATE0, 0x10(STATEP)
+	movdqu STATE1, 0x20(STATEP)
+	movdqu STATE2, 0x30(STATEP)
+	movdqu STATE3, 0x40(STATEP)
+
+	FRAME_END
+ENDPROC(crypto_aegis128_aesni_enc_tail)
+
+.macro decrypt_block a s0 s1 s2 s3 s4 i
+	movdq\a (\i * 0x10)(SRC), MSG
+	pxor \s1, MSG
+	pxor \s4, MSG
+	movdqa \s2, T1
+	pand \s3, T1
+	pxor T1, MSG
+	movdq\a MSG, (\i * 0x10)(DST)
+
+	aegis128_update
+	pxor MSG, \s4
+
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Ldec_out_\i
+.endm
+
+/*
+ * void crypto_aegis128_aesni_dec(void *state, unsigned int length,
+ *                                const void *src, void *dst);
+ */
+ENTRY(crypto_aegis128_aesni_dec)
+	FRAME_BEGIN
+
+	cmp $0x10, LEN
+	jb .Ldec_out
+
+	/* load the state: */
+	movdqu 0x00(STATEP), STATE0
+	movdqu 0x10(STATEP), STATE1
+	movdqu 0x20(STATEP), STATE2
+	movdqu 0x30(STATEP), STATE3
+	movdqu 0x40(STATEP), STATE4
+
+	mov  SRC, %r8
+	or   DST, %r8
+	and $0xF, %r8
+	jnz .Ldec_u_loop
+
+.align 8
+.Ldec_a_loop:
+	decrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0
+	decrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1
+	decrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2
+	decrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3
+	decrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4
+
+	add $0x50, SRC
+	add $0x50, DST
+	jmp .Ldec_a_loop
+
+.align 8
+.Ldec_u_loop:
+	decrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0
+	decrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1
+	decrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2
+	decrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3
+	decrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4
+
+	add $0x50, SRC
+	add $0x50, DST
+	jmp .Ldec_u_loop
+
+	/* store the state: */
+.Ldec_out_0:
+	movdqu STATE4, 0x00(STATEP)
+	movdqu STATE0, 0x10(STATEP)
+	movdqu STATE1, 0x20(STATEP)
+	movdqu STATE2, 0x30(STATEP)
+	movdqu STATE3, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Ldec_out_1:
+	movdqu STATE3, 0x00(STATEP)
+	movdqu STATE4, 0x10(STATEP)
+	movdqu STATE0, 0x20(STATEP)
+	movdqu STATE1, 0x30(STATEP)
+	movdqu STATE2, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Ldec_out_2:
+	movdqu STATE2, 0x00(STATEP)
+	movdqu STATE3, 0x10(STATEP)
+	movdqu STATE4, 0x20(STATEP)
+	movdqu STATE0, 0x30(STATEP)
+	movdqu STATE1, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Ldec_out_3:
+	movdqu STATE1, 0x00(STATEP)
+	movdqu STATE2, 0x10(STATEP)
+	movdqu STATE3, 0x20(STATEP)
+	movdqu STATE4, 0x30(STATEP)
+	movdqu STATE0, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Ldec_out_4:
+	movdqu STATE0, 0x00(STATEP)
+	movdqu STATE1, 0x10(STATEP)
+	movdqu STATE2, 0x20(STATEP)
+	movdqu STATE3, 0x30(STATEP)
+	movdqu STATE4, 0x40(STATEP)
+	FRAME_END
+	ret
+
+.Ldec_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128_aesni_dec)
+
+/*
+ * void crypto_aegis128_aesni_dec_tail(void *state, unsigned int length,
+ *                                     const void *src, void *dst);
+ */
+ENTRY(crypto_aegis128_aesni_dec_tail)
+	FRAME_BEGIN
+
+	/* load the state: */
+	movdqu 0x00(STATEP), STATE0
+	movdqu 0x10(STATEP), STATE1
+	movdqu 0x20(STATEP), STATE2
+	movdqu 0x30(STATEP), STATE3
+	movdqu 0x40(STATEP), STATE4
+
+	/* decrypt message: */
+	call __load_partial
+
+	pxor STATE1, MSG
+	pxor STATE4, MSG
+	movdqa STATE2, T1
+	pand STATE3, T1
+	pxor T1, MSG
+
+	movdqa MSG, T0
+	call __store_partial
+
+	/* mask with byte count: */
+	movq LEN, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	movdqa .Laegis128_counter, T1
+	pcmpgtb T1, T0
+	pand T0, MSG
+
+	aegis128_update
+	pxor MSG, STATE4
+
+	/* store the state: */
+	movdqu STATE4, 0x00(STATEP)
+	movdqu STATE0, 0x10(STATEP)
+	movdqu STATE1, 0x20(STATEP)
+	movdqu STATE2, 0x30(STATEP)
+	movdqu STATE3, 0x40(STATEP)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128_aesni_dec_tail)
+
+/*
+ * void crypto_aegis128_aesni_final(void *state, void *tag_xor,
+ *                                  u64 assoclen, u64 cryptlen);
+ */
+ENTRY(crypto_aegis128_aesni_final)
+	FRAME_BEGIN
+
+	/* load the state: */
+	movdqu 0x00(STATEP), STATE0
+	movdqu 0x10(STATEP), STATE1
+	movdqu 0x20(STATEP), STATE2
+	movdqu 0x30(STATEP), STATE3
+	movdqu 0x40(STATEP), STATE4
+
+	/* prepare length block: */
+	movq %rdx, MSG
+	movq %rcx, T0
+	pslldq $8, T0
+	pxor T0, MSG
+	psllq $3, MSG /* multiply by 8 (to get bit count) */
+
+	pxor STATE3, MSG
+
+	/* update state: */
+	aegis128_update; pxor MSG, STATE4
+	aegis128_update; pxor MSG, STATE3
+	aegis128_update; pxor MSG, STATE2
+	aegis128_update; pxor MSG, STATE1
+	aegis128_update; pxor MSG, STATE0
+	aegis128_update; pxor MSG, STATE4
+	aegis128_update; pxor MSG, STATE3
+
+	/* xor tag: */
+	movdqu (%rsi), MSG
+
+	pxor STATE0, MSG
+	pxor STATE1, MSG
+	pxor STATE2, MSG
+	pxor STATE3, MSG
+	pxor STATE4, MSG
+
+	movdqu MSG, (%rsi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128_aesni_final)
diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c
new file mode 100644
index 000000000000..5de7c0d46edf
--- /dev/null
+++ b/arch/x86/crypto/aegis128-aesni-glue.c
@@ -0,0 +1,407 @@
+/*
+ * The AEGIS-128 Authenticated-Encryption Algorithm
+ *   Glue for AES-NI + SSE2 implementation
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/cryptd.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
+#include <linux/module.h>
+#include <asm/fpu/api.h>
+#include <asm/cpu_device_id.h>
+
+#define AEGIS128_BLOCK_ALIGN 16
+#define AEGIS128_BLOCK_SIZE 16
+#define AEGIS128_NONCE_SIZE 16
+#define AEGIS128_STATE_BLOCKS 5
+#define AEGIS128_KEY_SIZE 16
+#define AEGIS128_MIN_AUTH_SIZE 8
+#define AEGIS128_MAX_AUTH_SIZE 16
+
+asmlinkage void crypto_aegis128_aesni_init(void *state, void *key, void *iv);
+
+asmlinkage void crypto_aegis128_aesni_ad(
+		void *state, unsigned int length, const void *data);
+
+asmlinkage void crypto_aegis128_aesni_enc(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis128_aesni_dec(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis128_aesni_enc_tail(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis128_aesni_dec_tail(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis128_aesni_final(
+		void *state, void *tag_xor, unsigned int cryptlen,
+		unsigned int assoclen);
+
+struct aegis_block {
+	u8 bytes[AEGIS128_BLOCK_SIZE] __aligned(AEGIS128_BLOCK_ALIGN);
+};
+
+struct aegis_state {
+	struct aegis_block blocks[AEGIS128_STATE_BLOCKS];
+};
+
+struct aegis_ctx {
+	struct aegis_block key;
+};
+
+struct aegis_crypt_ops {
+	int (*skcipher_walk_init)(struct skcipher_walk *walk,
+				  struct aead_request *req, bool atomic);
+
+	void (*crypt_blocks)(void *state, unsigned int length, const void *src,
+			     void *dst);
+	void (*crypt_tail)(void *state, unsigned int length, const void *src,
+			   void *dst);
+};
+
+static void crypto_aegis128_aesni_process_ad(
+		struct aegis_state *state, struct scatterlist *sg_src,
+		unsigned int assoclen)
+{
+	struct scatter_walk walk;
+	struct aegis_block buf;
+	unsigned int pos = 0;
+
+	scatterwalk_start(&walk, sg_src);
+	while (assoclen != 0) {
+		unsigned int size = scatterwalk_clamp(&walk, assoclen);
+		unsigned int left = size;
+		void *mapped = scatterwalk_map(&walk);
+		const u8 *src = (const u8 *)mapped;
+
+		if (pos + size >= AEGIS128_BLOCK_SIZE) {
+			if (pos > 0) {
+				unsigned int fill = AEGIS128_BLOCK_SIZE - pos;
+				memcpy(buf.bytes + pos, src, fill);
+				crypto_aegis128_aesni_ad(state,
+							 AEGIS128_BLOCK_SIZE,
+							 buf.bytes);
+				pos = 0;
+				left -= fill;
+				src += fill;
+			}
+
+			crypto_aegis128_aesni_ad(state, left, src);
+
+			src += left & ~(AEGIS128_BLOCK_SIZE - 1);
+			left &= AEGIS128_BLOCK_SIZE - 1;
+		}
+
+		memcpy(buf.bytes + pos, src, left);
+		pos += left;
+		assoclen -= size;
+
+		scatterwalk_unmap(mapped);
+		scatterwalk_advance(&walk, size);
+		scatterwalk_done(&walk, 0, assoclen);
+	}
+
+	if (pos > 0) {
+		memset(buf.bytes + pos, 0, AEGIS128_BLOCK_SIZE - pos);
+		crypto_aegis128_aesni_ad(state, AEGIS128_BLOCK_SIZE, buf.bytes);
+	}
+}
+
+static void crypto_aegis128_aesni_process_crypt(
+		struct aegis_state *state, struct aead_request *req,
+		const struct aegis_crypt_ops *ops)
+{
+	struct skcipher_walk walk;
+	u8 *src, *dst;
+	unsigned int chunksize, base;
+
+	ops->skcipher_walk_init(&walk, req, false);
+
+	while (walk.nbytes) {
+		src = walk.src.virt.addr;
+		dst = walk.dst.virt.addr;
+		chunksize = walk.nbytes;
+
+		ops->crypt_blocks(state, chunksize, src, dst);
+
+		base = chunksize & ~(AEGIS128_BLOCK_SIZE - 1);
+		src += base;
+		dst += base;
+		chunksize &= AEGIS128_BLOCK_SIZE - 1;
+
+		if (chunksize > 0)
+			ops->crypt_tail(state, chunksize, src, dst);
+
+		skcipher_walk_done(&walk, 0);
+	}
+}
+
+static struct aegis_ctx *crypto_aegis128_aesni_ctx(struct crypto_aead *aead)
+{
+	u8 *ctx = crypto_aead_ctx(aead);
+	ctx = PTR_ALIGN(ctx, __alignof__(struct aegis_ctx));
+	return (void *)ctx;
+}
+
+static int crypto_aegis128_aesni_setkey(struct crypto_aead *aead, const u8 *key,
+					unsigned int keylen)
+{
+	struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(aead);
+
+	if (keylen != AEGIS128_KEY_SIZE) {
+		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	memcpy(ctx->key.bytes, key, AEGIS128_KEY_SIZE);
+
+	return 0;
+}
+
+static int crypto_aegis128_aesni_setauthsize(struct crypto_aead *tfm,
+						unsigned int authsize)
+{
+	if (authsize > AEGIS128_MAX_AUTH_SIZE)
+		return -EINVAL;
+	if (authsize < AEGIS128_MIN_AUTH_SIZE)
+		return -EINVAL;
+	return 0;
+}
+
+static void crypto_aegis128_aesni_crypt(struct aead_request *req,
+					struct aegis_block *tag_xor,
+					unsigned int cryptlen,
+					const struct aegis_crypt_ops *ops)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(tfm);
+	struct aegis_state state;
+
+	kernel_fpu_begin();
+
+	crypto_aegis128_aesni_init(&state, ctx->key.bytes, req->iv);
+	crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen);
+	crypto_aegis128_aesni_process_crypt(&state, req, ops);
+	crypto_aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
+
+	kernel_fpu_end();
+}
+
+static int crypto_aegis128_aesni_encrypt(struct aead_request *req)
+{
+	static const struct aegis_crypt_ops OPS = {
+		.skcipher_walk_init = skcipher_walk_aead_encrypt,
+		.crypt_blocks = crypto_aegis128_aesni_enc,
+		.crypt_tail = crypto_aegis128_aesni_enc_tail,
+	};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_block tag = {};
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen;
+
+	crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);
+
+	scatterwalk_map_and_copy(tag.bytes, req->dst,
+				 req->assoclen + cryptlen, authsize, 1);
+	return 0;
+}
+
+static int crypto_aegis128_aesni_decrypt(struct aead_request *req)
+{
+	static const struct aegis_block zeros = {};
+
+	static const struct aegis_crypt_ops OPS = {
+		.skcipher_walk_init = skcipher_walk_aead_decrypt,
+		.crypt_blocks = crypto_aegis128_aesni_dec,
+		.crypt_tail = crypto_aegis128_aesni_dec_tail,
+	};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_block tag;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen - authsize;
+
+	scatterwalk_map_and_copy(tag.bytes, req->src,
+				 req->assoclen + cryptlen, authsize, 0);
+
+	crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);
+
+	return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
+}
+
+static int crypto_aegis128_aesni_init_tfm(struct crypto_aead *aead)
+{
+	return 0;
+}
+
+static void crypto_aegis128_aesni_exit_tfm(struct crypto_aead *aead)
+{
+}
+
+static int cryptd_aegis128_aesni_setkey(struct crypto_aead *aead,
+					const u8 *key, unsigned int keylen)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
+}
+
+static int cryptd_aegis128_aesni_setauthsize(struct crypto_aead *aead,
+					     unsigned int authsize)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
+}
+
+static int cryptd_aegis128_aesni_encrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	aead = &cryptd_tfm->base;
+	if (irq_fpu_usable() && (!in_atomic() ||
+				 !cryptd_aead_queued(cryptd_tfm)))
+		aead = cryptd_aead_child(cryptd_tfm);
+
+	aead_request_set_tfm(req, aead);
+
+	return crypto_aead_encrypt(req);
+}
+
+static int cryptd_aegis128_aesni_decrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	aead = &cryptd_tfm->base;
+	if (irq_fpu_usable() && (!in_atomic() ||
+				 !cryptd_aead_queued(cryptd_tfm)))
+		aead = cryptd_aead_child(cryptd_tfm);
+
+	aead_request_set_tfm(req, aead);
+
+	return crypto_aead_decrypt(req);
+}
+
+static int cryptd_aegis128_aesni_init_tfm(struct crypto_aead *aead)
+{
+	struct cryptd_aead *cryptd_tfm;
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+
+	cryptd_tfm = cryptd_alloc_aead("__aegis128-aesni", CRYPTO_ALG_INTERNAL,
+				       CRYPTO_ALG_INTERNAL);
+	if (IS_ERR(cryptd_tfm))
+		return PTR_ERR(cryptd_tfm);
+
+	*ctx = cryptd_tfm;
+	crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
+	return 0;
+}
+
+static void cryptd_aegis128_aesni_exit_tfm(struct crypto_aead *aead)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+
+	cryptd_free_aead(*ctx);
+}
+
+static struct aead_alg crypto_aegis128_aesni_alg[] = {
+	{
+		.setkey = crypto_aegis128_aesni_setkey,
+		.setauthsize = crypto_aegis128_aesni_setauthsize,
+		.encrypt = crypto_aegis128_aesni_encrypt,
+		.decrypt = crypto_aegis128_aesni_decrypt,
+		.init = crypto_aegis128_aesni_init_tfm,
+		.exit = crypto_aegis128_aesni_exit_tfm,
+
+		.ivsize = AEGIS128_NONCE_SIZE,
+		.maxauthsize = AEGIS128_MAX_AUTH_SIZE,
+		.chunksize = AEGIS128_BLOCK_SIZE,
+
+		.base = {
+			.cra_flags = CRYPTO_ALG_INTERNAL,
+			.cra_blocksize = 1,
+			.cra_ctxsize = sizeof(struct aegis_ctx) +
+				__alignof__(struct aegis_ctx),
+			.cra_alignmask = 0,
+
+			.cra_name = "__aegis128",
+			.cra_driver_name = "__aegis128-aesni",
+
+			.cra_module = THIS_MODULE,
+		}
+	}, {
+		.setkey = cryptd_aegis128_aesni_setkey,
+		.setauthsize = cryptd_aegis128_aesni_setauthsize,
+		.encrypt = cryptd_aegis128_aesni_encrypt,
+		.decrypt = cryptd_aegis128_aesni_decrypt,
+		.init = cryptd_aegis128_aesni_init_tfm,
+		.exit = cryptd_aegis128_aesni_exit_tfm,
+
+		.ivsize = AEGIS128_NONCE_SIZE,
+		.maxauthsize = AEGIS128_MAX_AUTH_SIZE,
+		.chunksize = AEGIS128_BLOCK_SIZE,
+
+		.base = {
+			.cra_flags = CRYPTO_ALG_ASYNC,
+			.cra_blocksize = 1,
+			.cra_ctxsize = sizeof(struct cryptd_aead *),
+			.cra_alignmask = 0,
+
+			.cra_priority = 400,
+
+			.cra_name = "aegis128",
+			.cra_driver_name = "aegis128-aesni",
+
+			.cra_module = THIS_MODULE,
+		}
+	}
+};
+
+static const struct x86_cpu_id aesni_cpu_id[] = {
+	X86_FEATURE_MATCH(X86_FEATURE_AES),
+	X86_FEATURE_MATCH(X86_FEATURE_XMM2),
+	{}
+};
+MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
+
+static int __init crypto_aegis128_aesni_module_init(void)
+{
+	if (!x86_match_cpu(aesni_cpu_id))
+		return -ENODEV;
+
+	return crypto_register_aeads(crypto_aegis128_aesni_alg,
+				     ARRAY_SIZE(crypto_aegis128_aesni_alg));
+}
+
+static void __exit crypto_aegis128_aesni_module_exit(void)
+{
+	crypto_unregister_aeads(crypto_aegis128_aesni_alg,
+				ARRAY_SIZE(crypto_aegis128_aesni_alg));
+}
+
+module_init(crypto_aegis128_aesni_module_init);
+module_exit(crypto_aegis128_aesni_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm -- AESNI+SSE2 implementation");
+MODULE_ALIAS_CRYPTO("aegis128");
+MODULE_ALIAS_CRYPTO("aegis128-aesni");
diff --git a/arch/x86/crypto/aegis128l-aesni-asm.S b/arch/x86/crypto/aegis128l-aesni-asm.S
new file mode 100644
index 000000000000..9263c344f2c7
--- /dev/null
+++ b/arch/x86/crypto/aegis128l-aesni-asm.S
@@ -0,0 +1,825 @@
+/*
+ * AES-NI + SSE2 implementation of AEGIS-128L
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/frame.h>
+
+#define STATE0	%xmm0
+#define STATE1	%xmm1
+#define STATE2	%xmm2
+#define STATE3	%xmm3
+#define STATE4	%xmm4
+#define STATE5	%xmm5
+#define STATE6	%xmm6
+#define STATE7	%xmm7
+#define MSG0	%xmm8
+#define MSG1	%xmm9
+#define T0	%xmm10
+#define T1	%xmm11
+#define T2	%xmm12
+#define T3	%xmm13
+
+#define STATEP	%rdi
+#define LEN	%rsi
+#define SRC	%rdx
+#define DST	%rcx
+
+.section .rodata.cst16.aegis128l_const, "aM", @progbits, 32
+.align 16
+.Laegis128l_const_0:
+	.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
+	.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
+.Laegis128l_const_1:
+	.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
+	.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
+
+.section .rodata.cst16.aegis128l_counter, "aM", @progbits, 16
+.align 16
+.Laegis128l_counter0:
+	.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
+	.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
+.Laegis128l_counter1:
+	.byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
+	.byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f
+
+.text
+
+/*
+ * __load_partial: internal ABI
+ * input:
+ *   LEN - bytes
+ *   SRC - src
+ * output:
+ *   MSG0 - first message block
+ *   MSG1 - second message block
+ * changed:
+ *   T0
+ *   %r8
+ *   %r9
+ */
+__load_partial:
+	xor %r9, %r9
+	pxor MSG0, MSG0
+	pxor MSG1, MSG1
+
+	mov LEN, %r8
+	and $0x1, %r8
+	jz .Lld_partial_1
+
+	mov LEN, %r8
+	and $0x1E, %r8
+	add SRC, %r8
+	mov (%r8), %r9b
+
+.Lld_partial_1:
+	mov LEN, %r8
+	and $0x2, %r8
+	jz .Lld_partial_2
+
+	mov LEN, %r8
+	and $0x1C, %r8
+	add SRC, %r8
+	shl $0x10, %r9
+	mov (%r8), %r9w
+
+.Lld_partial_2:
+	mov LEN, %r8
+	and $0x4, %r8
+	jz .Lld_partial_4
+
+	mov LEN, %r8
+	and $0x18, %r8
+	add SRC, %r8
+	shl $32, %r9
+	mov (%r8), %r8d
+	xor %r8, %r9
+
+.Lld_partial_4:
+	movq %r9, MSG0
+
+	mov LEN, %r8
+	and $0x8, %r8
+	jz .Lld_partial_8
+
+	mov LEN, %r8
+	and $0x10, %r8
+	add SRC, %r8
+	pslldq $8, MSG0
+	movq (%r8), T0
+	pxor T0, MSG0
+
+.Lld_partial_8:
+	mov LEN, %r8
+	and $0x10, %r8
+	jz .Lld_partial_16
+
+	movdqa MSG0, MSG1
+	movdqu (SRC), MSG0
+
+.Lld_partial_16:
+	ret
+ENDPROC(__load_partial)
+
+/*
+ * __store_partial: internal ABI
+ * input:
+ *   LEN - bytes
+ *   DST - dst
+ * output:
+ *   T0   - first message block
+ *   T1   - second message block
+ * changed:
+ *   %r8
+ *   %r9
+ *   %r10
+ */
+__store_partial:
+	mov LEN, %r8
+	mov DST, %r9
+
+	cmp $16, %r8
+	jl .Lst_partial_16
+
+	movdqu T0, (%r9)
+	movdqa T1, T0
+
+	sub $16, %r8
+	add $16, %r9
+
+.Lst_partial_16:
+	movq T0, %r10
+
+	cmp $8, %r8
+	jl .Lst_partial_8
+
+	mov %r10, (%r9)
+	psrldq $8, T0
+	movq T0, %r10
+
+	sub $8, %r8
+	add $8, %r9
+
+.Lst_partial_8:
+	cmp $4, %r8
+	jl .Lst_partial_4
+
+	mov %r10d, (%r9)
+	shr $32, %r10
+
+	sub $4, %r8
+	add $4, %r9
+
+.Lst_partial_4:
+	cmp $2, %r8
+	jl .Lst_partial_2
+
+	mov %r10w, (%r9)
+	shr $0x10, %r10
+
+	sub $2, %r8
+	add $2, %r9
+
+.Lst_partial_2:
+	cmp $1, %r8
+	jl .Lst_partial_1
+
+	mov %r10b, (%r9)
+
+.Lst_partial_1:
+	ret
+ENDPROC(__store_partial)
+
+.macro update
+	movdqa STATE7, T0
+	aesenc STATE0, STATE7
+	aesenc STATE1, STATE0
+	aesenc STATE2, STATE1
+	aesenc STATE3, STATE2
+	aesenc STATE4, STATE3
+	aesenc STATE5, STATE4
+	aesenc STATE6, STATE5
+	aesenc T0,     STATE6
+.endm
+
+.macro update0
+	update
+	pxor MSG0, STATE7
+	pxor MSG1, STATE3
+.endm
+
+.macro update1
+	update
+	pxor MSG0, STATE6
+	pxor MSG1, STATE2
+.endm
+
+.macro update2
+	update
+	pxor MSG0, STATE5
+	pxor MSG1, STATE1
+.endm
+
+.macro update3
+	update
+	pxor MSG0, STATE4
+	pxor MSG1, STATE0
+.endm
+
+.macro update4
+	update
+	pxor MSG0, STATE3
+	pxor MSG1, STATE7
+.endm
+
+.macro update5
+	update
+	pxor MSG0, STATE2
+	pxor MSG1, STATE6
+.endm
+
+.macro update6
+	update
+	pxor MSG0, STATE1
+	pxor MSG1, STATE5
+.endm
+
+.macro update7
+	update
+	pxor MSG0, STATE0
+	pxor MSG1, STATE4
+.endm
+
+.macro state_load
+	movdqu 0x00(STATEP), STATE0
+	movdqu 0x10(STATEP), STATE1
+	movdqu 0x20(STATEP), STATE2
+	movdqu 0x30(STATEP), STATE3
+	movdqu 0x40(STATEP), STATE4
+	movdqu 0x50(STATEP), STATE5
+	movdqu 0x60(STATEP), STATE6
+	movdqu 0x70(STATEP), STATE7
+.endm
+
+.macro state_store s0 s1 s2 s3 s4 s5 s6 s7
+	movdqu \s7, 0x00(STATEP)
+	movdqu \s0, 0x10(STATEP)
+	movdqu \s1, 0x20(STATEP)
+	movdqu \s2, 0x30(STATEP)
+	movdqu \s3, 0x40(STATEP)
+	movdqu \s4, 0x50(STATEP)
+	movdqu \s5, 0x60(STATEP)
+	movdqu \s6, 0x70(STATEP)
+.endm
+
+.macro state_store0
+	state_store STATE0 STATE1 STATE2 STATE3 STATE4 STATE5 STATE6 STATE7
+.endm
+
+.macro state_store1
+	state_store STATE7 STATE0 STATE1 STATE2 STATE3 STATE4 STATE5 STATE6
+.endm
+
+.macro state_store2
+	state_store STATE6 STATE7 STATE0 STATE1 STATE2 STATE3 STATE4 STATE5
+.endm
+
+.macro state_store3
+	state_store STATE5 STATE6 STATE7 STATE0 STATE1 STATE2 STATE3 STATE4
+.endm
+
+.macro state_store4
+	state_store STATE4 STATE5 STATE6 STATE7 STATE0 STATE1 STATE2 STATE3
+.endm
+
+.macro state_store5
+	state_store STATE3 STATE4 STATE5 STATE6 STATE7 STATE0 STATE1 STATE2
+.endm
+
+.macro state_store6
+	state_store STATE2 STATE3 STATE4 STATE5 STATE6 STATE7 STATE0 STATE1
+.endm
+
+.macro state_store7
+	state_store STATE1 STATE2 STATE3 STATE4 STATE5 STATE6 STATE7 STATE0
+.endm
+
+/*
+ * void crypto_aegis128l_aesni_init(void *state, const void *key, const void *iv);
+ */
+ENTRY(crypto_aegis128l_aesni_init)
+	FRAME_BEGIN
+
+	/* load key: */
+	movdqa (%rsi), MSG1
+	movdqa MSG1, STATE0
+	movdqa MSG1, STATE4
+	movdqa MSG1, STATE5
+	movdqa MSG1, STATE6
+	movdqa MSG1, STATE7
+
+	/* load IV: */
+	movdqu (%rdx), MSG0
+	pxor MSG0, STATE0
+	pxor MSG0, STATE4
+
+	/* load the constants: */
+	movdqa .Laegis128l_const_0, STATE2
+	movdqa .Laegis128l_const_1, STATE1
+	movdqa STATE1, STATE3
+	pxor STATE2, STATE5
+	pxor STATE1, STATE6
+	pxor STATE2, STATE7
+
+	/* update 10 times with IV and KEY: */
+	update0
+	update1
+	update2
+	update3
+	update4
+	update5
+	update6
+	update7
+	update0
+	update1
+
+	state_store1
+
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128l_aesni_init)
+
+.macro ad_block a i
+	movdq\a (\i * 0x20 + 0x00)(SRC), MSG0
+	movdq\a (\i * 0x20 + 0x10)(SRC), MSG1
+	update\i
+	sub $0x20, LEN
+	cmp $0x20, LEN
+	jl .Lad_out_\i
+.endm
+
+/*
+ * void crypto_aegis128l_aesni_ad(void *state, unsigned int length,
+ *                                const void *data);
+ */
+ENTRY(crypto_aegis128l_aesni_ad)
+	FRAME_BEGIN
+
+	cmp $0x20, LEN
+	jb .Lad_out
+
+	state_load
+
+	mov  SRC, %r8
+	and $0xf, %r8
+	jnz .Lad_u_loop
+
+.align 8
+.Lad_a_loop:
+	ad_block a 0
+	ad_block a 1
+	ad_block a 2
+	ad_block a 3
+	ad_block a 4
+	ad_block a 5
+	ad_block a 6
+	ad_block a 7
+
+	add $0x100, SRC
+	jmp .Lad_a_loop
+
+.align 8
+.Lad_u_loop:
+	ad_block u 0
+	ad_block u 1
+	ad_block u 2
+	ad_block u 3
+	ad_block u 4
+	ad_block u 5
+	ad_block u 6
+	ad_block u 7
+
+	add $0x100, SRC
+	jmp .Lad_u_loop
+
+.Lad_out_0:
+	state_store0
+	FRAME_END
+	ret
+
+.Lad_out_1:
+	state_store1
+	FRAME_END
+	ret
+
+.Lad_out_2:
+	state_store2
+	FRAME_END
+	ret
+
+.Lad_out_3:
+	state_store3
+	FRAME_END
+	ret
+
+.Lad_out_4:
+	state_store4
+	FRAME_END
+	ret
+
+.Lad_out_5:
+	state_store5
+	FRAME_END
+	ret
+
+.Lad_out_6:
+	state_store6
+	FRAME_END
+	ret
+
+.Lad_out_7:
+	state_store7
+	FRAME_END
+	ret
+
+.Lad_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128l_aesni_ad)
+
+.macro crypt m0 m1 s0 s1 s2 s3 s4 s5 s6 s7
+	pxor \s1, \m0
+	pxor \s6, \m0
+	movdqa \s2, T3
+	pand \s3, T3
+	pxor T3, \m0
+
+	pxor \s2, \m1
+	pxor \s5, \m1
+	movdqa \s6, T3
+	pand \s7, T3
+	pxor T3, \m1
+.endm
+
+.macro crypt0 m0 m1
+	crypt \m0 \m1 STATE0 STATE1 STATE2 STATE3 STATE4 STATE5 STATE6 STATE7
+.endm
+
+.macro crypt1 m0 m1
+	crypt \m0 \m1 STATE7 STATE0 STATE1 STATE2 STATE3 STATE4 STATE5 STATE6
+.endm
+
+.macro crypt2 m0 m1
+	crypt \m0 \m1 STATE6 STATE7 STATE0 STATE1 STATE2 STATE3 STATE4 STATE5
+.endm
+
+.macro crypt3 m0 m1
+	crypt \m0 \m1 STATE5 STATE6 STATE7 STATE0 STATE1 STATE2 STATE3 STATE4
+.endm
+
+.macro crypt4 m0 m1
+	crypt \m0 \m1 STATE4 STATE5 STATE6 STATE7 STATE0 STATE1 STATE2 STATE3
+.endm
+
+.macro crypt5 m0 m1
+	crypt \m0 \m1 STATE3 STATE4 STATE5 STATE6 STATE7 STATE0 STATE1 STATE2
+.endm
+
+.macro crypt6 m0 m1
+	crypt \m0 \m1 STATE2 STATE3 STATE4 STATE5 STATE6 STATE7 STATE0 STATE1
+.endm
+
+.macro crypt7 m0 m1
+	crypt \m0 \m1 STATE1 STATE2 STATE3 STATE4 STATE5 STATE6 STATE7 STATE0
+.endm
+
+.macro encrypt_block a i
+	movdq\a (\i * 0x20 + 0x00)(SRC), MSG0
+	movdq\a (\i * 0x20 + 0x10)(SRC), MSG1
+	movdqa MSG0, T0
+	movdqa MSG1, T1
+	crypt\i T0, T1
+	movdq\a T0, (\i * 0x20 + 0x00)(DST)
+	movdq\a T1, (\i * 0x20 + 0x10)(DST)
+
+	update\i
+
+	sub $0x20, LEN
+	cmp $0x20, LEN
+	jl .Lenc_out_\i
+.endm
+
+.macro decrypt_block a i
+	movdq\a (\i * 0x20 + 0x00)(SRC), MSG0
+	movdq\a (\i * 0x20 + 0x10)(SRC), MSG1
+	crypt\i MSG0, MSG1
+	movdq\a MSG0, (\i * 0x20 + 0x00)(DST)
+	movdq\a MSG1, (\i * 0x20 + 0x10)(DST)
+
+	update\i
+
+	sub $0x20, LEN
+	cmp $0x20, LEN
+	jl .Ldec_out_\i
+.endm
+
+/*
+ * void crypto_aegis128l_aesni_enc(void *state, unsigned int length,
+ *                                 const void *src, void *dst);
+ */
+ENTRY(crypto_aegis128l_aesni_enc)
+	FRAME_BEGIN
+
+	cmp $0x20, LEN
+	jb .Lenc_out
+
+	state_load
+
+	mov  SRC, %r8
+	or   DST, %r8
+	and $0xf, %r8
+	jnz .Lenc_u_loop
+
+.align 8
+.Lenc_a_loop:
+	encrypt_block a 0
+	encrypt_block a 1
+	encrypt_block a 2
+	encrypt_block a 3
+	encrypt_block a 4
+	encrypt_block a 5
+	encrypt_block a 6
+	encrypt_block a 7
+
+	add $0x100, SRC
+	add $0x100, DST
+	jmp .Lenc_a_loop
+
+.align 8
+.Lenc_u_loop:
+	encrypt_block u 0
+	encrypt_block u 1
+	encrypt_block u 2
+	encrypt_block u 3
+	encrypt_block u 4
+	encrypt_block u 5
+	encrypt_block u 6
+	encrypt_block u 7
+
+	add $0x100, SRC
+	add $0x100, DST
+	jmp .Lenc_u_loop
+
+.Lenc_out_0:
+	state_store0
+	FRAME_END
+	ret
+
+.Lenc_out_1:
+	state_store1
+	FRAME_END
+	ret
+
+.Lenc_out_2:
+	state_store2
+	FRAME_END
+	ret
+
+.Lenc_out_3:
+	state_store3
+	FRAME_END
+	ret
+
+.Lenc_out_4:
+	state_store4
+	FRAME_END
+	ret
+
+.Lenc_out_5:
+	state_store5
+	FRAME_END
+	ret
+
+.Lenc_out_6:
+	state_store6
+	FRAME_END
+	ret
+
+.Lenc_out_7:
+	state_store7
+	FRAME_END
+	ret
+
+.Lenc_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128l_aesni_enc)
+
+/*
+ * void crypto_aegis128l_aesni_enc_tail(void *state, unsigned int length,
+ *                                      const void *src, void *dst);
+ */
+ENTRY(crypto_aegis128l_aesni_enc_tail)
+	FRAME_BEGIN
+
+	state_load
+
+	/* encrypt message: */
+	call __load_partial
+
+	movdqa MSG0, T0
+	movdqa MSG1, T1
+	crypt0 T0, T1
+
+	call __store_partial
+
+	update0
+
+	state_store0
+
+	FRAME_END
+ENDPROC(crypto_aegis128l_aesni_enc_tail)
+
+/*
+ * void crypto_aegis128l_aesni_dec(void *state, unsigned int length,
+ *                                 const void *src, void *dst);
+ */
+ENTRY(crypto_aegis128l_aesni_dec)
+	FRAME_BEGIN
+
+	cmp $0x20, LEN
+	jb .Ldec_out
+
+	state_load
+
+	mov  SRC, %r8
+	or   DST, %r8
+	and $0xF, %r8
+	jnz .Ldec_u_loop
+
+.align 8
+.Ldec_a_loop:
+	decrypt_block a 0
+	decrypt_block a 1
+	decrypt_block a 2
+	decrypt_block a 3
+	decrypt_block a 4
+	decrypt_block a 5
+	decrypt_block a 6
+	decrypt_block a 7
+
+	add $0x100, SRC
+	add $0x100, DST
+	jmp .Ldec_a_loop
+
+.align 8
+.Ldec_u_loop:
+	decrypt_block u 0
+	decrypt_block u 1
+	decrypt_block u 2
+	decrypt_block u 3
+	decrypt_block u 4
+	decrypt_block u 5
+	decrypt_block u 6
+	decrypt_block u 7
+
+	add $0x100, SRC
+	add $0x100, DST
+	jmp .Ldec_u_loop
+
+.Ldec_out_0:
+	state_store0
+	FRAME_END
+	ret
+
+.Ldec_out_1:
+	state_store1
+	FRAME_END
+	ret
+
+.Ldec_out_2:
+	state_store2
+	FRAME_END
+	ret
+
+.Ldec_out_3:
+	state_store3
+	FRAME_END
+	ret
+
+.Ldec_out_4:
+	state_store4
+	FRAME_END
+	ret
+
+.Ldec_out_5:
+	state_store5
+	FRAME_END
+	ret
+
+.Ldec_out_6:
+	state_store6
+	FRAME_END
+	ret
+
+.Ldec_out_7:
+	state_store7
+	FRAME_END
+	ret
+
+.Ldec_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128l_aesni_dec)
+
+/*
+ * void crypto_aegis128l_aesni_dec_tail(void *state, unsigned int length,
+ *                                      const void *src, void *dst);
+ */
+ENTRY(crypto_aegis128l_aesni_dec_tail)
+	FRAME_BEGIN
+
+	state_load
+
+	/* decrypt message: */
+	call __load_partial
+
+	crypt0 MSG0, MSG1
+
+	movdqa MSG0, T0
+	movdqa MSG1, T1
+	call __store_partial
+
+	/* mask with byte count: */
+	movq LEN, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	movdqa T0, T1
+	movdqa .Laegis128l_counter0, T2
+	movdqa .Laegis128l_counter1, T3
+	pcmpgtb T2, T0
+	pcmpgtb T3, T1
+	pand T0, MSG0
+	pand T1, MSG1
+
+	update0
+
+	state_store0
+
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128l_aesni_dec_tail)
+
+/*
+ * void crypto_aegis128l_aesni_final(void *state, void *tag_xor,
+ *                                   u64 assoclen, u64 cryptlen);
+ */
+ENTRY(crypto_aegis128l_aesni_final)
+	FRAME_BEGIN
+
+	state_load
+
+	/* prepare length block: */
+	movq %rdx, MSG0
+	movq %rcx, T0
+	pslldq $8, T0
+	pxor T0, MSG0
+	psllq $3, MSG0 /* multiply by 8 (to get bit count) */
+
+	pxor STATE2, MSG0
+	movdqa MSG0, MSG1
+
+	/* update state: */
+	update0
+	update1
+	update2
+	update3
+	update4
+	update5
+	update6
+
+	/* xor tag: */
+	movdqu (%rsi), T0
+
+	pxor STATE1, T0
+	pxor STATE2, T0
+	pxor STATE3, T0
+	pxor STATE4, T0
+	pxor STATE5, T0
+	pxor STATE6, T0
+	pxor STATE7, T0
+
+	movdqu T0, (%rsi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis128l_aesni_final)
diff --git a/arch/x86/crypto/aegis128l-aesni-glue.c b/arch/x86/crypto/aegis128l-aesni-glue.c
new file mode 100644
index 000000000000..876e4866e633
--- /dev/null
+++ b/arch/x86/crypto/aegis128l-aesni-glue.c
@@ -0,0 +1,407 @@
+/*
+ * The AEGIS-128L Authenticated-Encryption Algorithm
+ *   Glue for AES-NI + SSE2 implementation
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/cryptd.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
+#include <linux/module.h>
+#include <asm/fpu/api.h>
+#include <asm/cpu_device_id.h>
+
+#define AEGIS128L_BLOCK_ALIGN 16
+#define AEGIS128L_BLOCK_SIZE 32
+#define AEGIS128L_NONCE_SIZE 16
+#define AEGIS128L_STATE_BLOCKS 8
+#define AEGIS128L_KEY_SIZE 16
+#define AEGIS128L_MIN_AUTH_SIZE 8
+#define AEGIS128L_MAX_AUTH_SIZE 16
+
+asmlinkage void crypto_aegis128l_aesni_init(void *state, void *key, void *iv);
+
+asmlinkage void crypto_aegis128l_aesni_ad(
+		void *state, unsigned int length, const void *data);
+
+asmlinkage void crypto_aegis128l_aesni_enc(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis128l_aesni_dec(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis128l_aesni_enc_tail(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis128l_aesni_dec_tail(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis128l_aesni_final(
+		void *state, void *tag_xor, unsigned int cryptlen,
+		unsigned int assoclen);
+
+struct aegis_block {
+	u8 bytes[AEGIS128L_BLOCK_SIZE] __aligned(AEGIS128L_BLOCK_ALIGN);
+};
+
+struct aegis_state {
+	struct aegis_block blocks[AEGIS128L_STATE_BLOCKS];
+};
+
+struct aegis_ctx {
+	struct aegis_block key;
+};
+
+struct aegis_crypt_ops {
+	int (*skcipher_walk_init)(struct skcipher_walk *walk,
+				  struct aead_request *req, bool atomic);
+
+	void (*crypt_blocks)(void *state, unsigned int length, const void *src,
+			     void *dst);
+	void (*crypt_tail)(void *state, unsigned int length, const void *src,
+			   void *dst);
+};
+
+static void crypto_aegis128l_aesni_process_ad(
+		struct aegis_state *state, struct scatterlist *sg_src,
+		unsigned int assoclen)
+{
+	struct scatter_walk walk;
+	struct aegis_block buf;
+	unsigned int pos = 0;
+
+	scatterwalk_start(&walk, sg_src);
+	while (assoclen != 0) {
+		unsigned int size = scatterwalk_clamp(&walk, assoclen);
+		unsigned int left = size;
+		void *mapped = scatterwalk_map(&walk);
+		const u8 *src = (const u8 *)mapped;
+
+		if (pos + size >= AEGIS128L_BLOCK_SIZE) {
+			if (pos > 0) {
+				unsigned int fill = AEGIS128L_BLOCK_SIZE - pos;
+				memcpy(buf.bytes + pos, src, fill);
+				crypto_aegis128l_aesni_ad(state,
+							  AEGIS128L_BLOCK_SIZE,
+							  buf.bytes);
+				pos = 0;
+				left -= fill;
+				src += fill;
+			}
+
+			crypto_aegis128l_aesni_ad(state, left, src);
+
+			src += left & ~(AEGIS128L_BLOCK_SIZE - 1);
+			left &= AEGIS128L_BLOCK_SIZE - 1;
+		}
+
+		memcpy(buf.bytes + pos, src, left);
+		pos += left;
+		assoclen -= size;
+
+		scatterwalk_unmap(mapped);
+		scatterwalk_advance(&walk, size);
+		scatterwalk_done(&walk, 0, assoclen);
+	}
+
+	if (pos > 0) {
+		memset(buf.bytes + pos, 0, AEGIS128L_BLOCK_SIZE - pos);
+		crypto_aegis128l_aesni_ad(state, AEGIS128L_BLOCK_SIZE, buf.bytes);
+	}
+}
+
+static void crypto_aegis128l_aesni_process_crypt(
+		struct aegis_state *state, struct aead_request *req,
+		const struct aegis_crypt_ops *ops)
+{
+	struct skcipher_walk walk;
+	u8 *src, *dst;
+	unsigned int chunksize, base;
+
+	ops->skcipher_walk_init(&walk, req, false);
+
+	while (walk.nbytes) {
+		src = walk.src.virt.addr;
+		dst = walk.dst.virt.addr;
+		chunksize = walk.nbytes;
+
+		ops->crypt_blocks(state, chunksize, src, dst);
+
+		base = chunksize & ~(AEGIS128L_BLOCK_SIZE - 1);
+		src += base;
+		dst += base;
+		chunksize &= AEGIS128L_BLOCK_SIZE - 1;
+
+		if (chunksize > 0)
+			ops->crypt_tail(state, chunksize, src, dst);
+
+		skcipher_walk_done(&walk, 0);
+	}
+}
+
+static struct aegis_ctx *crypto_aegis128l_aesni_ctx(struct crypto_aead *aead)
+{
+	u8 *ctx = crypto_aead_ctx(aead);
+	ctx = PTR_ALIGN(ctx, __alignof__(struct aegis_ctx));
+	return (void *)ctx;
+}
+
+static int crypto_aegis128l_aesni_setkey(struct crypto_aead *aead,
+					 const u8 *key, unsigned int keylen)
+{
+	struct aegis_ctx *ctx = crypto_aegis128l_aesni_ctx(aead);
+
+	if (keylen != AEGIS128L_KEY_SIZE) {
+		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	memcpy(ctx->key.bytes, key, AEGIS128L_KEY_SIZE);
+
+	return 0;
+}
+
+static int crypto_aegis128l_aesni_setauthsize(struct crypto_aead *tfm,
+					      unsigned int authsize)
+{
+	if (authsize > AEGIS128L_MAX_AUTH_SIZE)
+		return -EINVAL;
+	if (authsize < AEGIS128L_MIN_AUTH_SIZE)
+		return -EINVAL;
+	return 0;
+}
+
+static void crypto_aegis128l_aesni_crypt(struct aead_request *req,
+					 struct aegis_block *tag_xor,
+					 unsigned int cryptlen,
+					 const struct aegis_crypt_ops *ops)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_ctx *ctx = crypto_aegis128l_aesni_ctx(tfm);
+	struct aegis_state state;
+
+	kernel_fpu_begin();
+
+	crypto_aegis128l_aesni_init(&state, ctx->key.bytes, req->iv);
+	crypto_aegis128l_aesni_process_ad(&state, req->src, req->assoclen);
+	crypto_aegis128l_aesni_process_crypt(&state, req, ops);
+	crypto_aegis128l_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
+
+	kernel_fpu_end();
+}
+
+static int crypto_aegis128l_aesni_encrypt(struct aead_request *req)
+{
+	static const struct aegis_crypt_ops OPS = {
+		.skcipher_walk_init = skcipher_walk_aead_encrypt,
+		.crypt_blocks = crypto_aegis128l_aesni_enc,
+		.crypt_tail = crypto_aegis128l_aesni_enc_tail,
+	};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_block tag = {};
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen;
+
+	crypto_aegis128l_aesni_crypt(req, &tag, cryptlen, &OPS);
+
+	scatterwalk_map_and_copy(tag.bytes, req->dst,
+				 req->assoclen + cryptlen, authsize, 1);
+	return 0;
+}
+
+static int crypto_aegis128l_aesni_decrypt(struct aead_request *req)
+{
+	static const struct aegis_block zeros = {};
+
+	static const struct aegis_crypt_ops OPS = {
+		.skcipher_walk_init = skcipher_walk_aead_decrypt,
+		.crypt_blocks = crypto_aegis128l_aesni_dec,
+		.crypt_tail = crypto_aegis128l_aesni_dec_tail,
+	};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_block tag;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen - authsize;
+
+	scatterwalk_map_and_copy(tag.bytes, req->src,
+				 req->assoclen + cryptlen, authsize, 0);
+
+	crypto_aegis128l_aesni_crypt(req, &tag, cryptlen, &OPS);
+
+	return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
+}
+
+static int crypto_aegis128l_aesni_init_tfm(struct crypto_aead *aead)
+{
+	return 0;
+}
+
+static void crypto_aegis128l_aesni_exit_tfm(struct crypto_aead *aead)
+{
+}
+
+static int cryptd_aegis128l_aesni_setkey(struct crypto_aead *aead,
+					 const u8 *key, unsigned int keylen)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
+}
+
+static int cryptd_aegis128l_aesni_setauthsize(struct crypto_aead *aead,
+					      unsigned int authsize)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
+}
+
+static int cryptd_aegis128l_aesni_encrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	aead = &cryptd_tfm->base;
+	if (irq_fpu_usable() && (!in_atomic() ||
+				 !cryptd_aead_queued(cryptd_tfm)))
+		aead = cryptd_aead_child(cryptd_tfm);
+
+	aead_request_set_tfm(req, aead);
+
+	return crypto_aead_encrypt(req);
+}
+
+static int cryptd_aegis128l_aesni_decrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	aead = &cryptd_tfm->base;
+	if (irq_fpu_usable() && (!in_atomic() ||
+				 !cryptd_aead_queued(cryptd_tfm)))
+		aead = cryptd_aead_child(cryptd_tfm);
+
+	aead_request_set_tfm(req, aead);
+
+	return crypto_aead_decrypt(req);
+}
+
+static int cryptd_aegis128l_aesni_init_tfm(struct crypto_aead *aead)
+{
+	struct cryptd_aead *cryptd_tfm;
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+
+	cryptd_tfm = cryptd_alloc_aead("__aegis128l-aesni", CRYPTO_ALG_INTERNAL,
+				       CRYPTO_ALG_INTERNAL);
+	if (IS_ERR(cryptd_tfm))
+		return PTR_ERR(cryptd_tfm);
+
+	*ctx = cryptd_tfm;
+	crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
+	return 0;
+}
+
+static void cryptd_aegis128l_aesni_exit_tfm(struct crypto_aead *aead)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+
+	cryptd_free_aead(*ctx);
+}
+
+static struct aead_alg crypto_aegis128l_aesni_alg[] = {
+	{
+		.setkey = crypto_aegis128l_aesni_setkey,
+		.setauthsize = crypto_aegis128l_aesni_setauthsize,
+		.encrypt = crypto_aegis128l_aesni_encrypt,
+		.decrypt = crypto_aegis128l_aesni_decrypt,
+		.init = crypto_aegis128l_aesni_init_tfm,
+		.exit = crypto_aegis128l_aesni_exit_tfm,
+
+		.ivsize = AEGIS128L_NONCE_SIZE,
+		.maxauthsize = AEGIS128L_MAX_AUTH_SIZE,
+		.chunksize = AEGIS128L_BLOCK_SIZE,
+
+		.base = {
+			.cra_flags = CRYPTO_ALG_INTERNAL,
+			.cra_blocksize = 1,
+			.cra_ctxsize = sizeof(struct aegis_ctx) +
+				__alignof__(struct aegis_ctx),
+			.cra_alignmask = 0,
+
+			.cra_name = "__aegis128l",
+			.cra_driver_name = "__aegis128l-aesni",
+
+			.cra_module = THIS_MODULE,
+		}
+	}, {
+		.setkey = cryptd_aegis128l_aesni_setkey,
+		.setauthsize = cryptd_aegis128l_aesni_setauthsize,
+		.encrypt = cryptd_aegis128l_aesni_encrypt,
+		.decrypt = cryptd_aegis128l_aesni_decrypt,
+		.init = cryptd_aegis128l_aesni_init_tfm,
+		.exit = cryptd_aegis128l_aesni_exit_tfm,
+
+		.ivsize = AEGIS128L_NONCE_SIZE,
+		.maxauthsize = AEGIS128L_MAX_AUTH_SIZE,
+		.chunksize = AEGIS128L_BLOCK_SIZE,
+
+		.base = {
+			.cra_flags = CRYPTO_ALG_ASYNC,
+			.cra_blocksize = 1,
+			.cra_ctxsize = sizeof(struct cryptd_aead *),
+			.cra_alignmask = 0,
+
+			.cra_priority = 400,
+
+			.cra_name = "aegis128l",
+			.cra_driver_name = "aegis128l-aesni",
+
+			.cra_module = THIS_MODULE,
+		}
+	}
+};
+
+static const struct x86_cpu_id aesni_cpu_id[] = {
+	X86_FEATURE_MATCH(X86_FEATURE_AES),
+	X86_FEATURE_MATCH(X86_FEATURE_XMM2),
+	{}
+};
+MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
+
+static int __init crypto_aegis128l_aesni_module_init(void)
+{
+	if (!x86_match_cpu(aesni_cpu_id))
+		return -ENODEV;
+
+	return crypto_register_aeads(crypto_aegis128l_aesni_alg,
+				     ARRAY_SIZE(crypto_aegis128l_aesni_alg));
+}
+
+static void __exit crypto_aegis128l_aesni_module_exit(void)
+{
+	crypto_unregister_aeads(crypto_aegis128l_aesni_alg,
+				ARRAY_SIZE(crypto_aegis128l_aesni_alg));
+}
+
+module_init(crypto_aegis128l_aesni_module_init);
+module_exit(crypto_aegis128l_aesni_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("AEGIS-128L AEAD algorithm -- AESNI+SSE2 implementation");
+MODULE_ALIAS_CRYPTO("aegis128l");
+MODULE_ALIAS_CRYPTO("aegis128l-aesni");
diff --git a/arch/x86/crypto/aegis256-aesni-asm.S b/arch/x86/crypto/aegis256-aesni-asm.S
new file mode 100644
index 000000000000..1d977d515bf9
--- /dev/null
+++ b/arch/x86/crypto/aegis256-aesni-asm.S
@@ -0,0 +1,702 @@
+/*
+ * AES-NI + SSE2 implementation of AEGIS-128L
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/frame.h>
+
+#define STATE0	%xmm0
+#define STATE1	%xmm1
+#define STATE2	%xmm2
+#define STATE3	%xmm3
+#define STATE4	%xmm4
+#define STATE5	%xmm5
+#define MSG	%xmm6
+#define T0	%xmm7
+#define T1	%xmm8
+#define T2	%xmm9
+#define T3	%xmm10
+
+#define STATEP	%rdi
+#define LEN	%rsi
+#define SRC	%rdx
+#define DST	%rcx
+
+.section .rodata.cst16.aegis256_const, "aM", @progbits, 32
+.align 16
+.Laegis256_const_0:
+	.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
+	.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
+.Laegis256_const_1:
+	.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
+	.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
+
+.section .rodata.cst16.aegis256_counter, "aM", @progbits, 16
+.align 16
+.Laegis256_counter:
+	.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
+	.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
+
+.text
+
+/*
+ * __load_partial: internal ABI
+ * input:
+ *   LEN - bytes
+ *   SRC - src
+ * output:
+ *   MSG  - message block
+ * changed:
+ *   T0
+ *   %r8
+ *   %r9
+ */
+__load_partial:
+	xor %r9, %r9
+	pxor MSG, MSG
+
+	mov LEN, %r8
+	and $0x1, %r8
+	jz .Lld_partial_1
+
+	mov LEN, %r8
+	and $0x1E, %r8
+	add SRC, %r8
+	mov (%r8), %r9b
+
+.Lld_partial_1:
+	mov LEN, %r8
+	and $0x2, %r8
+	jz .Lld_partial_2
+
+	mov LEN, %r8
+	and $0x1C, %r8
+	add SRC, %r8
+	shl $0x10, %r9
+	mov (%r8), %r9w
+
+.Lld_partial_2:
+	mov LEN, %r8
+	and $0x4, %r8
+	jz .Lld_partial_4
+
+	mov LEN, %r8
+	and $0x18, %r8
+	add SRC, %r8
+	shl $32, %r9
+	mov (%r8), %r8d
+	xor %r8, %r9
+
+.Lld_partial_4:
+	movq %r9, MSG
+
+	mov LEN, %r8
+	and $0x8, %r8
+	jz .Lld_partial_8
+
+	mov LEN, %r8
+	and $0x10, %r8
+	add SRC, %r8
+	pslldq $8, MSG
+	movq (%r8), T0
+	pxor T0, MSG
+
+.Lld_partial_8:
+	ret
+ENDPROC(__load_partial)
+
+/*
+ * __store_partial: internal ABI
+ * input:
+ *   LEN - bytes
+ *   DST - dst
+ * output:
+ *   T0   - message block
+ * changed:
+ *   %r8
+ *   %r9
+ *   %r10
+ */
+__store_partial:
+	mov LEN, %r8
+	mov DST, %r9
+
+	movq T0, %r10
+
+	cmp $8, %r8
+	jl .Lst_partial_8
+
+	mov %r10, (%r9)
+	psrldq $8, T0
+	movq T0, %r10
+
+	sub $8, %r8
+	add $8, %r9
+
+.Lst_partial_8:
+	cmp $4, %r8
+	jl .Lst_partial_4
+
+	mov %r10d, (%r9)
+	shr $32, %r10
+
+	sub $4, %r8
+	add $4, %r9
+
+.Lst_partial_4:
+	cmp $2, %r8
+	jl .Lst_partial_2
+
+	mov %r10w, (%r9)
+	shr $0x10, %r10
+
+	sub $2, %r8
+	add $2, %r9
+
+.Lst_partial_2:
+	cmp $1, %r8
+	jl .Lst_partial_1
+
+	mov %r10b, (%r9)
+
+.Lst_partial_1:
+	ret
+ENDPROC(__store_partial)
+
+.macro update
+	movdqa STATE5, T0
+	aesenc STATE0, STATE5
+	aesenc STATE1, STATE0
+	aesenc STATE2, STATE1
+	aesenc STATE3, STATE2
+	aesenc STATE4, STATE3
+	aesenc T0,     STATE4
+.endm
+
+.macro update0 m
+	update
+	pxor \m, STATE5
+.endm
+
+.macro update1 m
+	update
+	pxor \m, STATE4
+.endm
+
+.macro update2 m
+	update
+	pxor \m, STATE3
+.endm
+
+.macro update3 m
+	update
+	pxor \m, STATE2
+.endm
+
+.macro update4 m
+	update
+	pxor \m, STATE1
+.endm
+
+.macro update5 m
+	update
+	pxor \m, STATE0
+.endm
+
+.macro state_load
+	movdqu 0x00(STATEP), STATE0
+	movdqu 0x10(STATEP), STATE1
+	movdqu 0x20(STATEP), STATE2
+	movdqu 0x30(STATEP), STATE3
+	movdqu 0x40(STATEP), STATE4
+	movdqu 0x50(STATEP), STATE5
+.endm
+
+.macro state_store s0 s1 s2 s3 s4 s5
+	movdqu \s5, 0x00(STATEP)
+	movdqu \s0, 0x10(STATEP)
+	movdqu \s1, 0x20(STATEP)
+	movdqu \s2, 0x30(STATEP)
+	movdqu \s3, 0x40(STATEP)
+	movdqu \s4, 0x50(STATEP)
+.endm
+
+.macro state_store0
+	state_store STATE0 STATE1 STATE2 STATE3 STATE4 STATE5
+.endm
+
+.macro state_store1
+	state_store STATE5 STATE0 STATE1 STATE2 STATE3 STATE4
+.endm
+
+.macro state_store2
+	state_store STATE4 STATE5 STATE0 STATE1 STATE2 STATE3
+.endm
+
+.macro state_store3
+	state_store STATE3 STATE4 STATE5 STATE0 STATE1 STATE2
+.endm
+
+.macro state_store4
+	state_store STATE2 STATE3 STATE4 STATE5 STATE0 STATE1
+.endm
+
+.macro state_store5
+	state_store STATE1 STATE2 STATE3 STATE4 STATE5 STATE0
+.endm
+
+/*
+ * void crypto_aegis256_aesni_init(void *state, const void *key, const void *iv);
+ */
+ENTRY(crypto_aegis256_aesni_init)
+	FRAME_BEGIN
+
+	/* load key: */
+	movdqa 0x00(%rsi), MSG
+	movdqa 0x10(%rsi), T1
+	movdqa MSG, STATE4
+	movdqa T1, STATE5
+
+	/* load IV: */
+	movdqu 0x00(%rdx), T2
+	movdqu 0x10(%rdx), T3
+	pxor MSG, T2
+	pxor T1, T3
+	movdqa T2, STATE0
+	movdqa T3, STATE1
+
+	/* load the constants: */
+	movdqa .Laegis256_const_0, STATE3
+	movdqa .Laegis256_const_1, STATE2
+	pxor STATE3, STATE4
+	pxor STATE2, STATE5
+
+	/* update 10 times with IV and KEY: */
+	update0 MSG
+	update1 T1
+	update2 T2
+	update3 T3
+	update4 MSG
+	update5 T1
+	update0 T2
+	update1 T3
+	update2 MSG
+	update3 T1
+	update4 T2
+	update5 T3
+	update0 MSG
+	update1 T1
+	update2 T2
+	update3 T3
+
+	state_store3
+
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis256_aesni_init)
+
+.macro ad_block a i
+	movdq\a (\i * 0x10)(SRC), MSG
+	update\i MSG
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lad_out_\i
+.endm
+
+/*
+ * void crypto_aegis256_aesni_ad(void *state, unsigned int length,
+ *                               const void *data);
+ */
+ENTRY(crypto_aegis256_aesni_ad)
+	FRAME_BEGIN
+
+	cmp $0x10, LEN
+	jb .Lad_out
+
+	state_load
+
+	mov  SRC, %r8
+	and $0xf, %r8
+	jnz .Lad_u_loop
+
+.align 8
+.Lad_a_loop:
+	ad_block a 0
+	ad_block a 1
+	ad_block a 2
+	ad_block a 3
+	ad_block a 4
+	ad_block a 5
+
+	add $0x60, SRC
+	jmp .Lad_a_loop
+
+.align 8
+.Lad_u_loop:
+	ad_block u 0
+	ad_block u 1
+	ad_block u 2
+	ad_block u 3
+	ad_block u 4
+	ad_block u 5
+
+	add $0x60, SRC
+	jmp .Lad_u_loop
+
+.Lad_out_0:
+	state_store0
+	FRAME_END
+	ret
+
+.Lad_out_1:
+	state_store1
+	FRAME_END
+	ret
+
+.Lad_out_2:
+	state_store2
+	FRAME_END
+	ret
+
+.Lad_out_3:
+	state_store3
+	FRAME_END
+	ret
+
+.Lad_out_4:
+	state_store4
+	FRAME_END
+	ret
+
+.Lad_out_5:
+	state_store5
+	FRAME_END
+	ret
+
+.Lad_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis256_aesni_ad)
+
+.macro crypt m s0 s1 s2 s3 s4 s5
+	pxor \s1, \m
+	pxor \s4, \m
+	pxor \s5, \m
+	movdqa \s2, T3
+	pand \s3, T3
+	pxor T3, \m
+.endm
+
+.macro crypt0 m
+	crypt \m STATE0 STATE1 STATE2 STATE3 STATE4 STATE5
+.endm
+
+.macro crypt1 m
+	crypt \m STATE5 STATE0 STATE1 STATE2 STATE3 STATE4
+.endm
+
+.macro crypt2 m
+	crypt \m STATE4 STATE5 STATE0 STATE1 STATE2 STATE3
+.endm
+
+.macro crypt3 m
+	crypt \m STATE3 STATE4 STATE5 STATE0 STATE1 STATE2
+.endm
+
+.macro crypt4 m
+	crypt \m STATE2 STATE3 STATE4 STATE5 STATE0 STATE1
+.endm
+
+.macro crypt5 m
+	crypt \m STATE1 STATE2 STATE3 STATE4 STATE5 STATE0
+.endm
+
+.macro encrypt_block a i
+	movdq\a (\i * 0x10)(SRC), MSG
+	movdqa MSG, T0
+	crypt\i T0
+	movdq\a T0, (\i * 0x10)(DST)
+
+	update\i MSG
+
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Lenc_out_\i
+.endm
+
+.macro decrypt_block a i
+	movdq\a (\i * 0x10)(SRC), MSG
+	crypt\i MSG
+	movdq\a MSG, (\i * 0x10)(DST)
+
+	update\i MSG
+
+	sub $0x10, LEN
+	cmp $0x10, LEN
+	jl .Ldec_out_\i
+.endm
+
+/*
+ * void crypto_aegis256_aesni_enc(void *state, unsigned int length,
+ *                                const void *src, void *dst);
+ */
+ENTRY(crypto_aegis256_aesni_enc)
+	FRAME_BEGIN
+
+	cmp $0x10, LEN
+	jb .Lenc_out
+
+	state_load
+
+	mov  SRC, %r8
+	or   DST, %r8
+	and $0xf, %r8
+	jnz .Lenc_u_loop
+
+.align 8
+.Lenc_a_loop:
+	encrypt_block a 0
+	encrypt_block a 1
+	encrypt_block a 2
+	encrypt_block a 3
+	encrypt_block a 4
+	encrypt_block a 5
+
+	add $0x60, SRC
+	add $0x60, DST
+	jmp .Lenc_a_loop
+
+.align 8
+.Lenc_u_loop:
+	encrypt_block u 0
+	encrypt_block u 1
+	encrypt_block u 2
+	encrypt_block u 3
+	encrypt_block u 4
+	encrypt_block u 5
+
+	add $0x60, SRC
+	add $0x60, DST
+	jmp .Lenc_u_loop
+
+.Lenc_out_0:
+	state_store0
+	FRAME_END
+	ret
+
+.Lenc_out_1:
+	state_store1
+	FRAME_END
+	ret
+
+.Lenc_out_2:
+	state_store2
+	FRAME_END
+	ret
+
+.Lenc_out_3:
+	state_store3
+	FRAME_END
+	ret
+
+.Lenc_out_4:
+	state_store4
+	FRAME_END
+	ret
+
+.Lenc_out_5:
+	state_store5
+	FRAME_END
+	ret
+
+.Lenc_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis256_aesni_enc)
+
+/*
+ * void crypto_aegis256_aesni_enc_tail(void *state, unsigned int length,
+ *                                     const void *src, void *dst);
+ */
+ENTRY(crypto_aegis256_aesni_enc_tail)
+	FRAME_BEGIN
+
+	state_load
+
+	/* encrypt message: */
+	call __load_partial
+
+	movdqa MSG, T0
+	crypt0 T0
+
+	call __store_partial
+
+	update0 MSG
+
+	state_store0
+
+	FRAME_END
+ENDPROC(crypto_aegis256_aesni_enc_tail)
+
+/*
+ * void crypto_aegis256_aesni_dec(void *state, unsigned int length,
+ *                                const void *src, void *dst);
+ */
+ENTRY(crypto_aegis256_aesni_dec)
+	FRAME_BEGIN
+
+	cmp $0x10, LEN
+	jb .Ldec_out
+
+	state_load
+
+	mov  SRC, %r8
+	or   DST, %r8
+	and $0xF, %r8
+	jnz .Ldec_u_loop
+
+.align 8
+.Ldec_a_loop:
+	decrypt_block a 0
+	decrypt_block a 1
+	decrypt_block a 2
+	decrypt_block a 3
+	decrypt_block a 4
+	decrypt_block a 5
+
+	add $0x60, SRC
+	add $0x60, DST
+	jmp .Ldec_a_loop
+
+.align 8
+.Ldec_u_loop:
+	decrypt_block u 0
+	decrypt_block u 1
+	decrypt_block u 2
+	decrypt_block u 3
+	decrypt_block u 4
+	decrypt_block u 5
+
+	add $0x60, SRC
+	add $0x60, DST
+	jmp .Ldec_u_loop
+
+.Ldec_out_0:
+	state_store0
+	FRAME_END
+	ret
+
+.Ldec_out_1:
+	state_store1
+	FRAME_END
+	ret
+
+.Ldec_out_2:
+	state_store2
+	FRAME_END
+	ret
+
+.Ldec_out_3:
+	state_store3
+	FRAME_END
+	ret
+
+.Ldec_out_4:
+	state_store4
+	FRAME_END
+	ret
+
+.Ldec_out_5:
+	state_store5
+	FRAME_END
+	ret
+
+.Ldec_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis256_aesni_dec)
+
+/*
+ * void crypto_aegis256_aesni_dec_tail(void *state, unsigned int length,
+ *                                     const void *src, void *dst);
+ */
+ENTRY(crypto_aegis256_aesni_dec_tail)
+	FRAME_BEGIN
+
+	state_load
+
+	/* decrypt message: */
+	call __load_partial
+
+	crypt0 MSG
+
+	movdqa MSG, T0
+	call __store_partial
+
+	/* mask with byte count: */
+	movq LEN, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	movdqa .Laegis256_counter, T1
+	pcmpgtb T1, T0
+	pand T0, MSG
+
+	update0 MSG
+
+	state_store0
+
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis256_aesni_dec_tail)
+
+/*
+ * void crypto_aegis256_aesni_final(void *state, void *tag_xor,
+ *                                  u64 assoclen, u64 cryptlen);
+ */
+ENTRY(crypto_aegis256_aesni_final)
+	FRAME_BEGIN
+
+	state_load
+
+	/* prepare length block: */
+	movq %rdx, MSG
+	movq %rcx, T0
+	pslldq $8, T0
+	pxor T0, MSG
+	psllq $3, MSG /* multiply by 8 (to get bit count) */
+
+	pxor STATE3, MSG
+
+	/* update state: */
+	update0 MSG
+	update1 MSG
+	update2 MSG
+	update3 MSG
+	update4 MSG
+	update5 MSG
+	update0 MSG
+
+	/* xor tag: */
+	movdqu (%rsi), MSG
+
+	pxor STATE0, MSG
+	pxor STATE1, MSG
+	pxor STATE2, MSG
+	pxor STATE3, MSG
+	pxor STATE4, MSG
+	pxor STATE5, MSG
+
+	movdqu MSG, (%rsi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_aegis256_aesni_final)
diff --git a/arch/x86/crypto/aegis256-aesni-glue.c b/arch/x86/crypto/aegis256-aesni-glue.c
new file mode 100644
index 000000000000..2b5dd3af8f4d
--- /dev/null
+++ b/arch/x86/crypto/aegis256-aesni-glue.c
@@ -0,0 +1,407 @@
+/*
+ * The AEGIS-256 Authenticated-Encryption Algorithm
+ *   Glue for AES-NI + SSE2 implementation
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/cryptd.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
+#include <linux/module.h>
+#include <asm/fpu/api.h>
+#include <asm/cpu_device_id.h>
+
+#define AEGIS256_BLOCK_ALIGN 16
+#define AEGIS256_BLOCK_SIZE 16
+#define AEGIS256_NONCE_SIZE 32
+#define AEGIS256_STATE_BLOCKS 6
+#define AEGIS256_KEY_SIZE 32
+#define AEGIS256_MIN_AUTH_SIZE 8
+#define AEGIS256_MAX_AUTH_SIZE 16
+
+asmlinkage void crypto_aegis256_aesni_init(void *state, void *key, void *iv);
+
+asmlinkage void crypto_aegis256_aesni_ad(
+		void *state, unsigned int length, const void *data);
+
+asmlinkage void crypto_aegis256_aesni_enc(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis256_aesni_dec(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis256_aesni_enc_tail(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis256_aesni_dec_tail(
+		void *state, unsigned int length, const void *src, void *dst);
+
+asmlinkage void crypto_aegis256_aesni_final(
+		void *state, void *tag_xor, unsigned int cryptlen,
+		unsigned int assoclen);
+
+struct aegis_block {
+	u8 bytes[AEGIS256_BLOCK_SIZE] __aligned(AEGIS256_BLOCK_ALIGN);
+};
+
+struct aegis_state {
+	struct aegis_block blocks[AEGIS256_STATE_BLOCKS];
+};
+
+struct aegis_ctx {
+	struct aegis_block key[AEGIS256_KEY_SIZE / AEGIS256_BLOCK_SIZE];
+};
+
+struct aegis_crypt_ops {
+	int (*skcipher_walk_init)(struct skcipher_walk *walk,
+				  struct aead_request *req, bool atomic);
+
+	void (*crypt_blocks)(void *state, unsigned int length, const void *src,
+			     void *dst);
+	void (*crypt_tail)(void *state, unsigned int length, const void *src,
+			   void *dst);
+};
+
+static void crypto_aegis256_aesni_process_ad(
+		struct aegis_state *state, struct scatterlist *sg_src,
+		unsigned int assoclen)
+{
+	struct scatter_walk walk;
+	struct aegis_block buf;
+	unsigned int pos = 0;
+
+	scatterwalk_start(&walk, sg_src);
+	while (assoclen != 0) {
+		unsigned int size = scatterwalk_clamp(&walk, assoclen);
+		unsigned int left = size;
+		void *mapped = scatterwalk_map(&walk);
+		const u8 *src = (const u8 *)mapped;
+
+		if (pos + size >= AEGIS256_BLOCK_SIZE) {
+			if (pos > 0) {
+				unsigned int fill = AEGIS256_BLOCK_SIZE - pos;
+				memcpy(buf.bytes + pos, src, fill);
+				crypto_aegis256_aesni_ad(state,
+							 AEGIS256_BLOCK_SIZE,
+							 buf.bytes);
+				pos = 0;
+				left -= fill;
+				src += fill;
+			}
+
+			crypto_aegis256_aesni_ad(state, left, src);
+
+			src += left & ~(AEGIS256_BLOCK_SIZE - 1);
+			left &= AEGIS256_BLOCK_SIZE - 1;
+		}
+
+		memcpy(buf.bytes + pos, src, left);
+		pos += left;
+		assoclen -= size;
+
+		scatterwalk_unmap(mapped);
+		scatterwalk_advance(&walk, size);
+		scatterwalk_done(&walk, 0, assoclen);
+	}
+
+	if (pos > 0) {
+		memset(buf.bytes + pos, 0, AEGIS256_BLOCK_SIZE - pos);
+		crypto_aegis256_aesni_ad(state, AEGIS256_BLOCK_SIZE, buf.bytes);
+	}
+}
+
+static void crypto_aegis256_aesni_process_crypt(
+		struct aegis_state *state, struct aead_request *req,
+		const struct aegis_crypt_ops *ops)
+{
+	struct skcipher_walk walk;
+	u8 *src, *dst;
+	unsigned int chunksize, base;
+
+	ops->skcipher_walk_init(&walk, req, false);
+
+	while (walk.nbytes) {
+		src = walk.src.virt.addr;
+		dst = walk.dst.virt.addr;
+		chunksize = walk.nbytes;
+
+		ops->crypt_blocks(state, chunksize, src, dst);
+
+		base = chunksize & ~(AEGIS256_BLOCK_SIZE - 1);
+		src += base;
+		dst += base;
+		chunksize &= AEGIS256_BLOCK_SIZE - 1;
+
+		if (chunksize > 0)
+			ops->crypt_tail(state, chunksize, src, dst);
+
+		skcipher_walk_done(&walk, 0);
+	}
+}
+
+static struct aegis_ctx *crypto_aegis256_aesni_ctx(struct crypto_aead *aead)
+{
+	u8 *ctx = crypto_aead_ctx(aead);
+	ctx = PTR_ALIGN(ctx, __alignof__(struct aegis_ctx));
+	return (void *)ctx;
+}
+
+static int crypto_aegis256_aesni_setkey(struct crypto_aead *aead, const u8 *key,
+					unsigned int keylen)
+{
+	struct aegis_ctx *ctx = crypto_aegis256_aesni_ctx(aead);
+
+	if (keylen != AEGIS256_KEY_SIZE) {
+		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	memcpy(ctx->key, key, AEGIS256_KEY_SIZE);
+
+	return 0;
+}
+
+static int crypto_aegis256_aesni_setauthsize(struct crypto_aead *tfm,
+						unsigned int authsize)
+{
+	if (authsize > AEGIS256_MAX_AUTH_SIZE)
+		return -EINVAL;
+	if (authsize < AEGIS256_MIN_AUTH_SIZE)
+		return -EINVAL;
+	return 0;
+}
+
+static void crypto_aegis256_aesni_crypt(struct aead_request *req,
+					struct aegis_block *tag_xor,
+					unsigned int cryptlen,
+					const struct aegis_crypt_ops *ops)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_ctx *ctx = crypto_aegis256_aesni_ctx(tfm);
+	struct aegis_state state;
+
+	kernel_fpu_begin();
+
+	crypto_aegis256_aesni_init(&state, ctx->key, req->iv);
+	crypto_aegis256_aesni_process_ad(&state, req->src, req->assoclen);
+	crypto_aegis256_aesni_process_crypt(&state, req, ops);
+	crypto_aegis256_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
+
+	kernel_fpu_end();
+}
+
+static int crypto_aegis256_aesni_encrypt(struct aead_request *req)
+{
+	static const struct aegis_crypt_ops OPS = {
+		.skcipher_walk_init = skcipher_walk_aead_encrypt,
+		.crypt_blocks = crypto_aegis256_aesni_enc,
+		.crypt_tail = crypto_aegis256_aesni_enc_tail,
+	};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_block tag = {};
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen;
+
+	crypto_aegis256_aesni_crypt(req, &tag, cryptlen, &OPS);
+
+	scatterwalk_map_and_copy(tag.bytes, req->dst,
+				 req->assoclen + cryptlen, authsize, 1);
+	return 0;
+}
+
+static int crypto_aegis256_aesni_decrypt(struct aead_request *req)
+{
+	static const struct aegis_block zeros = {};
+
+	static const struct aegis_crypt_ops OPS = {
+		.skcipher_walk_init = skcipher_walk_aead_decrypt,
+		.crypt_blocks = crypto_aegis256_aesni_dec,
+		.crypt_tail = crypto_aegis256_aesni_dec_tail,
+	};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_block tag;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen - authsize;
+
+	scatterwalk_map_and_copy(tag.bytes, req->src,
+				 req->assoclen + cryptlen, authsize, 0);
+
+	crypto_aegis256_aesni_crypt(req, &tag, cryptlen, &OPS);
+
+	return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
+}
+
+static int crypto_aegis256_aesni_init_tfm(struct crypto_aead *aead)
+{
+	return 0;
+}
+
+static void crypto_aegis256_aesni_exit_tfm(struct crypto_aead *aead)
+{
+}
+
+static int cryptd_aegis256_aesni_setkey(struct crypto_aead *aead,
+					const u8 *key, unsigned int keylen)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
+}
+
+static int cryptd_aegis256_aesni_setauthsize(struct crypto_aead *aead,
+					     unsigned int authsize)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
+}
+
+static int cryptd_aegis256_aesni_encrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	aead = &cryptd_tfm->base;
+	if (irq_fpu_usable() && (!in_atomic() ||
+				 !cryptd_aead_queued(cryptd_tfm)))
+		aead = cryptd_aead_child(cryptd_tfm);
+
+	aead_request_set_tfm(req, aead);
+
+	return crypto_aead_encrypt(req);
+}
+
+static int cryptd_aegis256_aesni_decrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	aead = &cryptd_tfm->base;
+	if (irq_fpu_usable() && (!in_atomic() ||
+				 !cryptd_aead_queued(cryptd_tfm)))
+		aead = cryptd_aead_child(cryptd_tfm);
+
+	aead_request_set_tfm(req, aead);
+
+	return crypto_aead_decrypt(req);
+}
+
+static int cryptd_aegis256_aesni_init_tfm(struct crypto_aead *aead)
+{
+	struct cryptd_aead *cryptd_tfm;
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+
+	cryptd_tfm = cryptd_alloc_aead("__aegis256-aesni", CRYPTO_ALG_INTERNAL,
+				       CRYPTO_ALG_INTERNAL);
+	if (IS_ERR(cryptd_tfm))
+		return PTR_ERR(cryptd_tfm);
+
+	*ctx = cryptd_tfm;
+	crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
+	return 0;
+}
+
+static void cryptd_aegis256_aesni_exit_tfm(struct crypto_aead *aead)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+
+	cryptd_free_aead(*ctx);
+}
+
+static struct aead_alg crypto_aegis256_aesni_alg[] = {
+	{
+		.setkey = crypto_aegis256_aesni_setkey,
+		.setauthsize = crypto_aegis256_aesni_setauthsize,
+		.encrypt = crypto_aegis256_aesni_encrypt,
+		.decrypt = crypto_aegis256_aesni_decrypt,
+		.init = crypto_aegis256_aesni_init_tfm,
+		.exit = crypto_aegis256_aesni_exit_tfm,
+
+		.ivsize = AEGIS256_NONCE_SIZE,
+		.maxauthsize = AEGIS256_MAX_AUTH_SIZE,
+		.chunksize = AEGIS256_BLOCK_SIZE,
+
+		.base = {
+			.cra_flags = CRYPTO_ALG_INTERNAL,
+			.cra_blocksize = 1,
+			.cra_ctxsize = sizeof(struct aegis_ctx) +
+				__alignof__(struct aegis_ctx),
+			.cra_alignmask = 0,
+
+			.cra_name = "__aegis256",
+			.cra_driver_name = "__aegis256-aesni",
+
+			.cra_module = THIS_MODULE,
+		}
+	}, {
+		.setkey = cryptd_aegis256_aesni_setkey,
+		.setauthsize = cryptd_aegis256_aesni_setauthsize,
+		.encrypt = cryptd_aegis256_aesni_encrypt,
+		.decrypt = cryptd_aegis256_aesni_decrypt,
+		.init = cryptd_aegis256_aesni_init_tfm,
+		.exit = cryptd_aegis256_aesni_exit_tfm,
+
+		.ivsize = AEGIS256_NONCE_SIZE,
+		.maxauthsize = AEGIS256_MAX_AUTH_SIZE,
+		.chunksize = AEGIS256_BLOCK_SIZE,
+
+		.base = {
+			.cra_flags = CRYPTO_ALG_ASYNC,
+			.cra_blocksize = 1,
+			.cra_ctxsize = sizeof(struct cryptd_aead *),
+			.cra_alignmask = 0,
+
+			.cra_priority = 400,
+
+			.cra_name = "aegis256",
+			.cra_driver_name = "aegis256-aesni",
+
+			.cra_module = THIS_MODULE,
+		}
+	}
+};
+
+static const struct x86_cpu_id aesni_cpu_id[] = {
+	X86_FEATURE_MATCH(X86_FEATURE_AES),
+	X86_FEATURE_MATCH(X86_FEATURE_XMM2),
+	{}
+};
+MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
+
+static int __init crypto_aegis256_aesni_module_init(void)
+{
+	if (!x86_match_cpu(aesni_cpu_id))
+		return -ENODEV;
+
+	return crypto_register_aeads(crypto_aegis256_aesni_alg,
+				    ARRAY_SIZE(crypto_aegis256_aesni_alg));
+}
+
+static void __exit crypto_aegis256_aesni_module_exit(void)
+{
+	crypto_unregister_aeads(crypto_aegis256_aesni_alg,
+				ARRAY_SIZE(crypto_aegis256_aesni_alg));
+}
+
+module_init(crypto_aegis256_aesni_module_init);
+module_exit(crypto_aegis256_aesni_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("AEGIS-256 AEAD algorithm -- AESNI+SSE2 implementation");
+MODULE_ALIAS_CRYPTO("aegis256");
+MODULE_ALIAS_CRYPTO("aegis256-aesni");
diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c
index 0420bab19efb..2ddbe3a1868b 100644
--- a/arch/x86/crypto/ghash-clmulni-intel_glue.c
+++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c
@@ -364,5 +364,5 @@ module_exit(ghash_pclmulqdqni_mod_exit);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("GHASH Message Digest Algorithm, "
-		   "acclerated by PCLMULQDQ-NI");
+		   "accelerated by PCLMULQDQ-NI");
 MODULE_ALIAS_CRYPTO("ghash");
diff --git a/arch/x86/crypto/morus1280-avx2-asm.S b/arch/x86/crypto/morus1280-avx2-asm.S
new file mode 100644
index 000000000000..37d422e77931
--- /dev/null
+++ b/arch/x86/crypto/morus1280-avx2-asm.S
@@ -0,0 +1,621 @@
+/*
+ * AVX2 implementation of MORUS-1280
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/frame.h>
+
+#define SHUFFLE_MASK(i0, i1, i2, i3) \
+	(i0 | (i1 << 2) | (i2 << 4) | (i3 << 6))
+
+#define MASK1 SHUFFLE_MASK(3, 0, 1, 2)
+#define MASK2 SHUFFLE_MASK(2, 3, 0, 1)
+#define MASK3 SHUFFLE_MASK(1, 2, 3, 0)
+
+#define STATE0		%ymm0
+#define STATE0_LOW	%xmm0
+#define STATE1		%ymm1
+#define STATE2		%ymm2
+#define STATE3		%ymm3
+#define STATE4		%ymm4
+#define KEY		%ymm5
+#define MSG		%ymm5
+#define MSG_LOW		%xmm5
+#define T0		%ymm6
+#define T0_LOW		%xmm6
+#define T1		%ymm7
+
+.section .rodata.cst32.morus1280_const, "aM", @progbits, 32
+.align 32
+.Lmorus1280_const:
+	.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
+	.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
+	.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
+	.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
+
+.section .rodata.cst32.morus1280_counter, "aM", @progbits, 32
+.align 32
+.Lmorus1280_counter:
+	.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
+	.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
+	.byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
+	.byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f
+
+.text
+
+.macro morus1280_round s0, s1, s2, s3, s4, b, w
+	vpand \s1, \s2, T0
+	vpxor T0, \s0, \s0
+	vpxor \s3, \s0, \s0
+	vpsllq $\b, \s0, T0
+	vpsrlq $(64 - \b), \s0, \s0
+	vpxor T0, \s0, \s0
+	vpermq $\w, \s3, \s3
+.endm
+
+/*
+ * __morus1280_update: internal ABI
+ * input:
+ *   STATE[0-4] - input state
+ *   MSG        - message block
+ * output:
+ *   STATE[0-4] - output state
+ * changed:
+ *   T0
+ */
+__morus1280_update:
+	morus1280_round STATE0, STATE1, STATE2, STATE3, STATE4, 13, MASK1
+	vpxor MSG, STATE1, STATE1
+	morus1280_round STATE1, STATE2, STATE3, STATE4, STATE0, 46, MASK2
+	vpxor MSG, STATE2, STATE2
+	morus1280_round STATE2, STATE3, STATE4, STATE0, STATE1, 38, MASK3
+	vpxor MSG, STATE3, STATE3
+	morus1280_round STATE3, STATE4, STATE0, STATE1, STATE2,  7, MASK2
+	vpxor MSG, STATE4, STATE4
+	morus1280_round STATE4, STATE0, STATE1, STATE2, STATE3,  4, MASK1
+	ret
+ENDPROC(__morus1280_update)
+
+/*
+ * __morus1280_update_zero: internal ABI
+ * input:
+ *   STATE[0-4] - input state
+ * output:
+ *   STATE[0-4] - output state
+ * changed:
+ *   T0
+ */
+__morus1280_update_zero:
+	morus1280_round STATE0, STATE1, STATE2, STATE3, STATE4, 13, MASK1
+	morus1280_round STATE1, STATE2, STATE3, STATE4, STATE0, 46, MASK2
+	morus1280_round STATE2, STATE3, STATE4, STATE0, STATE1, 38, MASK3
+	morus1280_round STATE3, STATE4, STATE0, STATE1, STATE2,  7, MASK2
+	morus1280_round STATE4, STATE0, STATE1, STATE2, STATE3,  4, MASK1
+	ret
+ENDPROC(__morus1280_update_zero)
+
+/*
+ * __load_partial: internal ABI
+ * input:
+ *   %rsi - src
+ *   %rcx - bytes
+ * output:
+ *   MSG  - message block
+ * changed:
+ *   %r8
+ *   %r9
+ */
+__load_partial:
+	xor %r9, %r9
+	vpxor MSG, MSG, MSG
+
+	mov %rcx, %r8
+	and $0x1, %r8
+	jz .Lld_partial_1
+
+	mov %rcx, %r8
+	and $0x1E, %r8
+	add %rsi, %r8
+	mov (%r8), %r9b
+
+.Lld_partial_1:
+	mov %rcx, %r8
+	and $0x2, %r8
+	jz .Lld_partial_2
+
+	mov %rcx, %r8
+	and $0x1C, %r8
+	add %rsi, %r8
+	shl $16, %r9
+	mov (%r8), %r9w
+
+.Lld_partial_2:
+	mov %rcx, %r8
+	and $0x4, %r8
+	jz .Lld_partial_4
+
+	mov %rcx, %r8
+	and $0x18, %r8
+	add %rsi, %r8
+	shl $32, %r9
+	mov (%r8), %r8d
+	xor %r8, %r9
+
+.Lld_partial_4:
+	movq %r9, MSG_LOW
+
+	mov %rcx, %r8
+	and $0x8, %r8
+	jz .Lld_partial_8
+
+	mov %rcx, %r8
+	and $0x10, %r8
+	add %rsi, %r8
+	pshufd $MASK2, MSG_LOW, MSG_LOW
+	pinsrq $0, (%r8), MSG_LOW
+
+.Lld_partial_8:
+	mov %rcx, %r8
+	and $0x10, %r8
+	jz .Lld_partial_16
+
+	vpermq $MASK2, MSG, MSG
+	movdqu (%rsi), MSG_LOW
+
+.Lld_partial_16:
+	ret
+ENDPROC(__load_partial)
+
+/*
+ * __store_partial: internal ABI
+ * input:
+ *   %rdx - dst
+ *   %rcx - bytes
+ * output:
+ *   T0   - message block
+ * changed:
+ *   %r8
+ *   %r9
+ *   %r10
+ */
+__store_partial:
+	mov %rcx, %r8
+	mov %rdx, %r9
+
+	cmp $16, %r8
+	jl .Lst_partial_16
+
+	movdqu T0_LOW, (%r9)
+	vpermq $MASK2, T0, T0
+
+	sub $16, %r8
+	add $16, %r9
+
+.Lst_partial_16:
+	movq T0_LOW, %r10
+
+	cmp $8, %r8
+	jl .Lst_partial_8
+
+	mov %r10, (%r9)
+	pextrq $1, T0_LOW, %r10
+
+	sub $8, %r8
+	add $8, %r9
+
+.Lst_partial_8:
+	cmp $4, %r8
+	jl .Lst_partial_4
+
+	mov %r10d, (%r9)
+	shr $32, %r10
+
+	sub $4, %r8
+	add $4, %r9
+
+.Lst_partial_4:
+	cmp $2, %r8
+	jl .Lst_partial_2
+
+	mov %r10w, (%r9)
+	shr $16, %r10
+
+	sub $2, %r8
+	add $2, %r9
+
+.Lst_partial_2:
+	cmp $1, %r8
+	jl .Lst_partial_1
+
+	mov %r10b, (%r9)
+
+.Lst_partial_1:
+	ret
+ENDPROC(__store_partial)
+
+/*
+ * void crypto_morus1280_avx2_init(void *state, const void *key,
+ *                                 const void *iv);
+ */
+ENTRY(crypto_morus1280_avx2_init)
+	FRAME_BEGIN
+
+	/* load IV: */
+	vpxor STATE0, STATE0, STATE0
+	movdqu (%rdx), STATE0_LOW
+	/* load key: */
+	vmovdqu (%rsi), KEY
+	vmovdqa KEY, STATE1
+	/* load all ones: */
+	vpcmpeqd STATE2, STATE2, STATE2
+	/* load all zeros: */
+	vpxor STATE3, STATE3, STATE3
+	/* load the constant: */
+	vmovdqa .Lmorus1280_const, STATE4
+
+	/* update 16 times with zero: */
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+
+	/* xor-in the key again after updates: */
+	vpxor KEY, STATE1, STATE1
+
+	/* store the state: */
+	vmovdqu STATE0, (0 * 32)(%rdi)
+	vmovdqu STATE1, (1 * 32)(%rdi)
+	vmovdqu STATE2, (2 * 32)(%rdi)
+	vmovdqu STATE3, (3 * 32)(%rdi)
+	vmovdqu STATE4, (4 * 32)(%rdi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_avx2_init)
+
+/*
+ * void crypto_morus1280_avx2_ad(void *state, const void *data,
+ *                               unsigned int length);
+ */
+ENTRY(crypto_morus1280_avx2_ad)
+	FRAME_BEGIN
+
+	cmp $32, %rdx
+	jb .Lad_out
+
+	/* load the state: */
+	vmovdqu (0 * 32)(%rdi), STATE0
+	vmovdqu (1 * 32)(%rdi), STATE1
+	vmovdqu (2 * 32)(%rdi), STATE2
+	vmovdqu (3 * 32)(%rdi), STATE3
+	vmovdqu (4 * 32)(%rdi), STATE4
+
+	mov %rsi,  %r8
+	and $0x1F, %r8
+	jnz .Lad_u_loop
+
+.align 4
+.Lad_a_loop:
+	vmovdqa (%rsi), MSG
+	call __morus1280_update
+	sub $32, %rdx
+	add $32, %rsi
+	cmp $32, %rdx
+	jge .Lad_a_loop
+
+	jmp .Lad_cont
+.align 4
+.Lad_u_loop:
+	vmovdqu (%rsi), MSG
+	call __morus1280_update
+	sub $32, %rdx
+	add $32, %rsi
+	cmp $32, %rdx
+	jge .Lad_u_loop
+
+.Lad_cont:
+	/* store the state: */
+	vmovdqu STATE0, (0 * 32)(%rdi)
+	vmovdqu STATE1, (1 * 32)(%rdi)
+	vmovdqu STATE2, (2 * 32)(%rdi)
+	vmovdqu STATE3, (3 * 32)(%rdi)
+	vmovdqu STATE4, (4 * 32)(%rdi)
+
+.Lad_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_avx2_ad)
+
+/*
+ * void crypto_morus1280_avx2_enc(void *state, const void *src, void *dst,
+ *                                unsigned int length);
+ */
+ENTRY(crypto_morus1280_avx2_enc)
+	FRAME_BEGIN
+
+	cmp $32, %rcx
+	jb .Lenc_out
+
+	/* load the state: */
+	vmovdqu (0 * 32)(%rdi), STATE0
+	vmovdqu (1 * 32)(%rdi), STATE1
+	vmovdqu (2 * 32)(%rdi), STATE2
+	vmovdqu (3 * 32)(%rdi), STATE3
+	vmovdqu (4 * 32)(%rdi), STATE4
+
+	mov %rsi,  %r8
+	or  %rdx,  %r8
+	and $0x1F, %r8
+	jnz .Lenc_u_loop
+
+.align 4
+.Lenc_a_loop:
+	vmovdqa (%rsi), MSG
+	vmovdqa MSG, T0
+	vpxor STATE0, T0, T0
+	vpermq $MASK3, STATE1, T1
+	vpxor T1, T0, T0
+	vpand STATE2, STATE3, T1
+	vpxor T1, T0, T0
+	vmovdqa T0, (%rdx)
+
+	call __morus1280_update
+	sub $32, %rcx
+	add $32, %rsi
+	add $32, %rdx
+	cmp $32, %rcx
+	jge .Lenc_a_loop
+
+	jmp .Lenc_cont
+.align 4
+.Lenc_u_loop:
+	vmovdqu (%rsi), MSG
+	vmovdqa MSG, T0
+	vpxor STATE0, T0, T0
+	vpermq $MASK3, STATE1, T1
+	vpxor T1, T0, T0
+	vpand STATE2, STATE3, T1
+	vpxor T1, T0, T0
+	vmovdqu T0, (%rdx)
+
+	call __morus1280_update
+	sub $32, %rcx
+	add $32, %rsi
+	add $32, %rdx
+	cmp $32, %rcx
+	jge .Lenc_u_loop
+
+.Lenc_cont:
+	/* store the state: */
+	vmovdqu STATE0, (0 * 32)(%rdi)
+	vmovdqu STATE1, (1 * 32)(%rdi)
+	vmovdqu STATE2, (2 * 32)(%rdi)
+	vmovdqu STATE3, (3 * 32)(%rdi)
+	vmovdqu STATE4, (4 * 32)(%rdi)
+
+.Lenc_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_avx2_enc)
+
+/*
+ * void crypto_morus1280_avx2_enc_tail(void *state, const void *src, void *dst,
+ *                                     unsigned int length);
+ */
+ENTRY(crypto_morus1280_avx2_enc_tail)
+	FRAME_BEGIN
+
+	/* load the state: */
+	vmovdqu (0 * 32)(%rdi), STATE0
+	vmovdqu (1 * 32)(%rdi), STATE1
+	vmovdqu (2 * 32)(%rdi), STATE2
+	vmovdqu (3 * 32)(%rdi), STATE3
+	vmovdqu (4 * 32)(%rdi), STATE4
+
+	/* encrypt message: */
+	call __load_partial
+
+	vmovdqa MSG, T0
+	vpxor STATE0, T0, T0
+	vpermq $MASK3, STATE1, T1
+	vpxor T1, T0, T0
+	vpand STATE2, STATE3, T1
+	vpxor T1, T0, T0
+
+	call __store_partial
+
+	call __morus1280_update
+
+	/* store the state: */
+	vmovdqu STATE0, (0 * 32)(%rdi)
+	vmovdqu STATE1, (1 * 32)(%rdi)
+	vmovdqu STATE2, (2 * 32)(%rdi)
+	vmovdqu STATE3, (3 * 32)(%rdi)
+	vmovdqu STATE4, (4 * 32)(%rdi)
+
+	FRAME_END
+ENDPROC(crypto_morus1280_avx2_enc_tail)
+
+/*
+ * void crypto_morus1280_avx2_dec(void *state, const void *src, void *dst,
+ *                                unsigned int length);
+ */
+ENTRY(crypto_morus1280_avx2_dec)
+	FRAME_BEGIN
+
+	cmp $32, %rcx
+	jb .Ldec_out
+
+	/* load the state: */
+	vmovdqu (0 * 32)(%rdi), STATE0
+	vmovdqu (1 * 32)(%rdi), STATE1
+	vmovdqu (2 * 32)(%rdi), STATE2
+	vmovdqu (3 * 32)(%rdi), STATE3
+	vmovdqu (4 * 32)(%rdi), STATE4
+
+	mov %rsi,  %r8
+	or  %rdx,  %r8
+	and $0x1F, %r8
+	jnz .Ldec_u_loop
+
+.align 4
+.Ldec_a_loop:
+	vmovdqa (%rsi), MSG
+	vpxor STATE0, MSG, MSG
+	vpermq $MASK3, STATE1, T0
+	vpxor T0, MSG, MSG
+	vpand STATE2, STATE3, T0
+	vpxor T0, MSG, MSG
+	vmovdqa MSG, (%rdx)
+
+	call __morus1280_update
+	sub $32, %rcx
+	add $32, %rsi
+	add $32, %rdx
+	cmp $32, %rcx
+	jge .Ldec_a_loop
+
+	jmp .Ldec_cont
+.align 4
+.Ldec_u_loop:
+	vmovdqu (%rsi), MSG
+	vpxor STATE0, MSG, MSG
+	vpermq $MASK3, STATE1, T0
+	vpxor T0, MSG, MSG
+	vpand STATE2, STATE3, T0
+	vpxor T0, MSG, MSG
+	vmovdqu MSG, (%rdx)
+
+	call __morus1280_update
+	sub $32, %rcx
+	add $32, %rsi
+	add $32, %rdx
+	cmp $32, %rcx
+	jge .Ldec_u_loop
+
+.Ldec_cont:
+	/* store the state: */
+	vmovdqu STATE0, (0 * 32)(%rdi)
+	vmovdqu STATE1, (1 * 32)(%rdi)
+	vmovdqu STATE2, (2 * 32)(%rdi)
+	vmovdqu STATE3, (3 * 32)(%rdi)
+	vmovdqu STATE4, (4 * 32)(%rdi)
+
+.Ldec_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_avx2_dec)
+
+/*
+ * void crypto_morus1280_avx2_dec_tail(void *state, const void *src, void *dst,
+ *                                     unsigned int length);
+ */
+ENTRY(crypto_morus1280_avx2_dec_tail)
+	FRAME_BEGIN
+
+	/* load the state: */
+	vmovdqu (0 * 32)(%rdi), STATE0
+	vmovdqu (1 * 32)(%rdi), STATE1
+	vmovdqu (2 * 32)(%rdi), STATE2
+	vmovdqu (3 * 32)(%rdi), STATE3
+	vmovdqu (4 * 32)(%rdi), STATE4
+
+	/* decrypt message: */
+	call __load_partial
+
+	vpxor STATE0, MSG, MSG
+	vpermq $MASK3, STATE1, T0
+	vpxor T0, MSG, MSG
+	vpand STATE2, STATE3, T0
+	vpxor T0, MSG, MSG
+	vmovdqa MSG, T0
+
+	call __store_partial
+
+	/* mask with byte count: */
+	movq %rcx, T0_LOW
+	vpbroadcastb T0_LOW, T0
+	vmovdqa .Lmorus1280_counter, T1
+	vpcmpgtb T1, T0, T0
+	vpand T0, MSG, MSG
+
+	call __morus1280_update
+
+	/* store the state: */
+	vmovdqu STATE0, (0 * 32)(%rdi)
+	vmovdqu STATE1, (1 * 32)(%rdi)
+	vmovdqu STATE2, (2 * 32)(%rdi)
+	vmovdqu STATE3, (3 * 32)(%rdi)
+	vmovdqu STATE4, (4 * 32)(%rdi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_avx2_dec_tail)
+
+/*
+ * void crypto_morus1280_avx2_final(void *state, void *tag_xor,
+ *                                  u64 assoclen, u64 cryptlen);
+ */
+ENTRY(crypto_morus1280_avx2_final)
+	FRAME_BEGIN
+
+	/* load the state: */
+	vmovdqu (0 * 32)(%rdi), STATE0
+	vmovdqu (1 * 32)(%rdi), STATE1
+	vmovdqu (2 * 32)(%rdi), STATE2
+	vmovdqu (3 * 32)(%rdi), STATE3
+	vmovdqu (4 * 32)(%rdi), STATE4
+
+	/* xor state[0] into state[4]: */
+	vpxor STATE0, STATE4, STATE4
+
+	/* prepare length block: */
+	vpxor MSG, MSG, MSG
+	vpinsrq $0, %rdx, MSG_LOW, MSG_LOW
+	vpinsrq $1, %rcx, MSG_LOW, MSG_LOW
+	vpsllq $3, MSG, MSG /* multiply by 8 (to get bit count) */
+
+	/* update state: */
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+
+	/* xor tag: */
+	vmovdqu (%rsi), MSG
+
+	vpxor STATE0, MSG, MSG
+	vpermq $MASK3, STATE1, T0
+	vpxor T0, MSG, MSG
+	vpand STATE2, STATE3, T0
+	vpxor T0, MSG, MSG
+	vmovdqu MSG, (%rsi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_avx2_final)
diff --git a/arch/x86/crypto/morus1280-avx2-glue.c b/arch/x86/crypto/morus1280-avx2-glue.c
new file mode 100644
index 000000000000..f111f36d26dc
--- /dev/null
+++ b/arch/x86/crypto/morus1280-avx2-glue.c
@@ -0,0 +1,68 @@
+/*
+ * The MORUS-1280 Authenticated-Encryption Algorithm
+ *   Glue for AVX2 implementation
+ *
+ * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/internal/aead.h>
+#include <crypto/morus1280_glue.h>
+#include <linux/module.h>
+#include <asm/fpu/api.h>
+#include <asm/cpu_device_id.h>
+
+asmlinkage void crypto_morus1280_avx2_init(void *state, const void *key,
+					   const void *iv);
+asmlinkage void crypto_morus1280_avx2_ad(void *state, const void *data,
+					 unsigned int length);
+
+asmlinkage void crypto_morus1280_avx2_enc(void *state, const void *src,
+					  void *dst, unsigned int length);
+asmlinkage void crypto_morus1280_avx2_dec(void *state, const void *src,
+					  void *dst, unsigned int length);
+
+asmlinkage void crypto_morus1280_avx2_enc_tail(void *state, const void *src,
+					       void *dst, unsigned int length);
+asmlinkage void crypto_morus1280_avx2_dec_tail(void *state, const void *src,
+					       void *dst, unsigned int length);
+
+asmlinkage void crypto_morus1280_avx2_final(void *state, void *tag_xor,
+					    u64 assoclen, u64 cryptlen);
+
+MORUS1280_DECLARE_ALGS(avx2, "morus1280-avx2", 400);
+
+static const struct x86_cpu_id avx2_cpu_id[] = {
+    X86_FEATURE_MATCH(X86_FEATURE_AVX2),
+    {}
+};
+MODULE_DEVICE_TABLE(x86cpu, avx2_cpu_id);
+
+static int __init crypto_morus1280_avx2_module_init(void)
+{
+	if (!x86_match_cpu(avx2_cpu_id))
+		return -ENODEV;
+
+	return crypto_register_aeads(crypto_morus1280_avx2_algs,
+				     ARRAY_SIZE(crypto_morus1280_avx2_algs));
+}
+
+static void __exit crypto_morus1280_avx2_module_exit(void)
+{
+	crypto_unregister_aeads(crypto_morus1280_avx2_algs,
+				ARRAY_SIZE(crypto_morus1280_avx2_algs));
+}
+
+module_init(crypto_morus1280_avx2_module_init);
+module_exit(crypto_morus1280_avx2_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm -- AVX2 implementation");
+MODULE_ALIAS_CRYPTO("morus1280");
+MODULE_ALIAS_CRYPTO("morus1280-avx2");
diff --git a/arch/x86/crypto/morus1280-sse2-asm.S b/arch/x86/crypto/morus1280-sse2-asm.S
new file mode 100644
index 000000000000..1fe637c7be9d
--- /dev/null
+++ b/arch/x86/crypto/morus1280-sse2-asm.S
@@ -0,0 +1,895 @@
+/*
+ * SSE2 implementation of MORUS-1280
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/frame.h>
+
+#define SHUFFLE_MASK(i0, i1, i2, i3) \
+	(i0 | (i1 << 2) | (i2 << 4) | (i3 << 6))
+
+#define MASK2 SHUFFLE_MASK(2, 3, 0, 1)
+
+#define STATE0_LO	%xmm0
+#define STATE0_HI	%xmm1
+#define STATE1_LO	%xmm2
+#define STATE1_HI	%xmm3
+#define STATE2_LO	%xmm4
+#define STATE2_HI	%xmm5
+#define STATE3_LO	%xmm6
+#define STATE3_HI	%xmm7
+#define STATE4_LO	%xmm8
+#define STATE4_HI	%xmm9
+#define KEY_LO		%xmm10
+#define KEY_HI		%xmm11
+#define MSG_LO		%xmm10
+#define MSG_HI		%xmm11
+#define T0_LO		%xmm12
+#define T0_HI		%xmm13
+#define T1_LO		%xmm14
+#define T1_HI		%xmm15
+
+.section .rodata.cst16.morus640_const, "aM", @progbits, 16
+.align 16
+.Lmorus640_const_0:
+	.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
+	.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
+.Lmorus640_const_1:
+	.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
+	.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
+
+.section .rodata.cst16.morus640_counter, "aM", @progbits, 16
+.align 16
+.Lmorus640_counter_0:
+	.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
+	.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
+.Lmorus640_counter_1:
+	.byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
+	.byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f
+
+.text
+
+.macro rol1 hi, lo
+	/*
+	 * HI_1 | HI_0 || LO_1 | LO_0
+	 *  ==>
+	 * HI_0 | HI_1 || LO_1 | LO_0
+	 *  ==>
+	 * HI_0 | LO_1 || LO_0 | HI_1
+	 */
+	pshufd $MASK2, \hi, \hi
+	movdqa \hi, T0_LO
+	punpcklqdq \lo, T0_LO
+	punpckhqdq \hi, \lo
+	movdqa \lo, \hi
+	movdqa T0_LO, \lo
+.endm
+
+.macro rol2 hi, lo
+	movdqa \lo, T0_LO
+	movdqa \hi, \lo
+	movdqa T0_LO, \hi
+.endm
+
+.macro rol3 hi, lo
+	/*
+	 * HI_1 | HI_0 || LO_1 | LO_0
+	 *  ==>
+	 * HI_0 | HI_1 || LO_1 | LO_0
+	 *  ==>
+	 * LO_0 | HI_1 || HI_0 | LO_1
+	 */
+	pshufd $MASK2, \hi, \hi
+	movdqa \lo, T0_LO
+	punpckhqdq \hi, T0_LO
+	punpcklqdq \lo, \hi
+	movdqa T0_LO, \lo
+.endm
+
+.macro morus1280_round s0_l, s0_h, s1_l, s1_h, s2_l, s2_h, s3_l, s3_h, s4_l, s4_h, b, w
+	movdqa \s1_l, T0_LO
+	pand \s2_l, T0_LO
+	pxor T0_LO, \s0_l
+
+	movdqa \s1_h, T0_LO
+	pand \s2_h, T0_LO
+	pxor T0_LO, \s0_h
+
+	pxor \s3_l, \s0_l
+	pxor \s3_h, \s0_h
+
+	movdqa \s0_l, T0_LO
+	psllq $\b, T0_LO
+	psrlq $(64 - \b), \s0_l
+	pxor T0_LO, \s0_l
+
+	movdqa \s0_h, T0_LO
+	psllq $\b, T0_LO
+	psrlq $(64 - \b), \s0_h
+	pxor T0_LO, \s0_h
+
+	\w \s3_h, \s3_l
+.endm
+
+/*
+ * __morus1280_update: internal ABI
+ * input:
+ *   STATE[0-4] - input state
+ *   MSG        - message block
+ * output:
+ *   STATE[0-4] - output state
+ * changed:
+ *   T0
+ */
+__morus1280_update:
+	morus1280_round \
+		STATE0_LO, STATE0_HI, \
+		STATE1_LO, STATE1_HI, \
+		STATE2_LO, STATE2_HI, \
+		STATE3_LO, STATE3_HI, \
+		STATE4_LO, STATE4_HI, \
+		13, rol1
+	pxor MSG_LO, STATE1_LO
+	pxor MSG_HI, STATE1_HI
+	morus1280_round \
+		STATE1_LO, STATE1_HI, \
+		STATE2_LO, STATE2_HI, \
+		STATE3_LO, STATE3_HI, \
+		STATE4_LO, STATE4_HI, \
+		STATE0_LO, STATE0_HI, \
+		46, rol2
+	pxor MSG_LO, STATE2_LO
+	pxor MSG_HI, STATE2_HI
+	morus1280_round \
+		STATE2_LO, STATE2_HI, \
+		STATE3_LO, STATE3_HI, \
+		STATE4_LO, STATE4_HI, \
+		STATE0_LO, STATE0_HI, \
+		STATE1_LO, STATE1_HI, \
+		38, rol3
+	pxor MSG_LO, STATE3_LO
+	pxor MSG_HI, STATE3_HI
+	morus1280_round \
+		STATE3_LO, STATE3_HI, \
+		STATE4_LO, STATE4_HI, \
+		STATE0_LO, STATE0_HI, \
+		STATE1_LO, STATE1_HI, \
+		STATE2_LO, STATE2_HI, \
+		7, rol2
+	pxor MSG_LO, STATE4_LO
+	pxor MSG_HI, STATE4_HI
+	morus1280_round \
+		STATE4_LO, STATE4_HI, \
+		STATE0_LO, STATE0_HI, \
+		STATE1_LO, STATE1_HI, \
+		STATE2_LO, STATE2_HI, \
+		STATE3_LO, STATE3_HI, \
+		4, rol1
+	ret
+ENDPROC(__morus1280_update)
+
+/*
+ * __morus1280_update_zero: internal ABI
+ * input:
+ *   STATE[0-4] - input state
+ * output:
+ *   STATE[0-4] - output state
+ * changed:
+ *   T0
+ */
+__morus1280_update_zero:
+	morus1280_round \
+		STATE0_LO, STATE0_HI, \
+		STATE1_LO, STATE1_HI, \
+		STATE2_LO, STATE2_HI, \
+		STATE3_LO, STATE3_HI, \
+		STATE4_LO, STATE4_HI, \
+		13, rol1
+	morus1280_round \
+		STATE1_LO, STATE1_HI, \
+		STATE2_LO, STATE2_HI, \
+		STATE3_LO, STATE3_HI, \
+		STATE4_LO, STATE4_HI, \
+		STATE0_LO, STATE0_HI, \
+		46, rol2
+	morus1280_round \
+		STATE2_LO, STATE2_HI, \
+		STATE3_LO, STATE3_HI, \
+		STATE4_LO, STATE4_HI, \
+		STATE0_LO, STATE0_HI, \
+		STATE1_LO, STATE1_HI, \
+		38, rol3
+	morus1280_round \
+		STATE3_LO, STATE3_HI, \
+		STATE4_LO, STATE4_HI, \
+		STATE0_LO, STATE0_HI, \
+		STATE1_LO, STATE1_HI, \
+		STATE2_LO, STATE2_HI, \
+		7, rol2
+	morus1280_round \
+		STATE4_LO, STATE4_HI, \
+		STATE0_LO, STATE0_HI, \
+		STATE1_LO, STATE1_HI, \
+		STATE2_LO, STATE2_HI, \
+		STATE3_LO, STATE3_HI, \
+		4, rol1
+	ret
+ENDPROC(__morus1280_update_zero)
+
+/*
+ * __load_partial: internal ABI
+ * input:
+ *   %rsi - src
+ *   %rcx - bytes
+ * output:
+ *   MSG  - message block
+ * changed:
+ *   %r8
+ *   %r9
+ */
+__load_partial:
+	xor %r9, %r9
+	pxor MSG_LO, MSG_LO
+	pxor MSG_HI, MSG_HI
+
+	mov %rcx, %r8
+	and $0x1, %r8
+	jz .Lld_partial_1
+
+	mov %rcx, %r8
+	and $0x1E, %r8
+	add %rsi, %r8
+	mov (%r8), %r9b
+
+.Lld_partial_1:
+	mov %rcx, %r8
+	and $0x2, %r8
+	jz .Lld_partial_2
+
+	mov %rcx, %r8
+	and $0x1C, %r8
+	add %rsi, %r8
+	shl $16, %r9
+	mov (%r8), %r9w
+
+.Lld_partial_2:
+	mov %rcx, %r8
+	and $0x4, %r8
+	jz .Lld_partial_4
+
+	mov %rcx, %r8
+	and $0x18, %r8
+	add %rsi, %r8
+	shl $32, %r9
+	mov (%r8), %r8d
+	xor %r8, %r9
+
+.Lld_partial_4:
+	movq %r9, MSG_LO
+
+	mov %rcx, %r8
+	and $0x8, %r8
+	jz .Lld_partial_8
+
+	mov %rcx, %r8
+	and $0x10, %r8
+	add %rsi, %r8
+	pslldq $8, MSG_LO
+	movq (%r8), T0_LO
+	pxor T0_LO, MSG_LO
+
+.Lld_partial_8:
+	mov %rcx, %r8
+	and $0x10, %r8
+	jz .Lld_partial_16
+
+	movdqa MSG_LO, MSG_HI
+	movdqu (%rsi), MSG_LO
+
+.Lld_partial_16:
+	ret
+ENDPROC(__load_partial)
+
+/*
+ * __store_partial: internal ABI
+ * input:
+ *   %rdx - dst
+ *   %rcx - bytes
+ * output:
+ *   T0   - message block
+ * changed:
+ *   %r8
+ *   %r9
+ *   %r10
+ */
+__store_partial:
+	mov %rcx, %r8
+	mov %rdx, %r9
+
+	cmp $16, %r8
+	jl .Lst_partial_16
+
+	movdqu T0_LO, (%r9)
+	movdqa T0_HI, T0_LO
+
+	sub $16, %r8
+	add $16, %r9
+
+.Lst_partial_16:
+	movq T0_LO, %r10
+
+	cmp $8, %r8
+	jl .Lst_partial_8
+
+	mov %r10, (%r9)
+	psrldq $8, T0_LO
+	movq T0_LO, %r10
+
+	sub $8, %r8
+	add $8, %r9
+
+.Lst_partial_8:
+	cmp $4, %r8
+	jl .Lst_partial_4
+
+	mov %r10d, (%r9)
+	shr $32, %r10
+
+	sub $4, %r8
+	add $4, %r9
+
+.Lst_partial_4:
+	cmp $2, %r8
+	jl .Lst_partial_2
+
+	mov %r10w, (%r9)
+	shr $16, %r10
+
+	sub $2, %r8
+	add $2, %r9
+
+.Lst_partial_2:
+	cmp $1, %r8
+	jl .Lst_partial_1
+
+	mov %r10b, (%r9)
+
+.Lst_partial_1:
+	ret
+ENDPROC(__store_partial)
+
+/*
+ * void crypto_morus1280_sse2_init(void *state, const void *key,
+ *                                 const void *iv);
+ */
+ENTRY(crypto_morus1280_sse2_init)
+	FRAME_BEGIN
+
+	/* load IV: */
+	pxor STATE0_HI, STATE0_HI
+	movdqu (%rdx), STATE0_LO
+	/* load key: */
+	movdqu  0(%rsi), KEY_LO
+	movdqu 16(%rsi), KEY_HI
+	movdqa KEY_LO, STATE1_LO
+	movdqa KEY_HI, STATE1_HI
+	/* load all ones: */
+	pcmpeqd STATE2_LO, STATE2_LO
+	pcmpeqd STATE2_HI, STATE2_HI
+	/* load all zeros: */
+	pxor STATE3_LO, STATE3_LO
+	pxor STATE3_HI, STATE3_HI
+	/* load the constant: */
+	movdqa .Lmorus640_const_0, STATE4_LO
+	movdqa .Lmorus640_const_1, STATE4_HI
+
+	/* update 16 times with zero: */
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+	call __morus1280_update_zero
+
+	/* xor-in the key again after updates: */
+	pxor KEY_LO, STATE1_LO
+	pxor KEY_HI, STATE1_HI
+
+	/* store the state: */
+	movdqu STATE0_LO, (0 * 16)(%rdi)
+	movdqu STATE0_HI, (1 * 16)(%rdi)
+	movdqu STATE1_LO, (2 * 16)(%rdi)
+	movdqu STATE1_HI, (3 * 16)(%rdi)
+	movdqu STATE2_LO, (4 * 16)(%rdi)
+	movdqu STATE2_HI, (5 * 16)(%rdi)
+	movdqu STATE3_LO, (6 * 16)(%rdi)
+	movdqu STATE3_HI, (7 * 16)(%rdi)
+	movdqu STATE4_LO, (8 * 16)(%rdi)
+	movdqu STATE4_HI, (9 * 16)(%rdi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_sse2_init)
+
+/*
+ * void crypto_morus1280_sse2_ad(void *state, const void *data,
+ *                               unsigned int length);
+ */
+ENTRY(crypto_morus1280_sse2_ad)
+	FRAME_BEGIN
+
+	cmp $32, %rdx
+	jb .Lad_out
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0_LO
+	movdqu (1 * 16)(%rdi), STATE0_HI
+	movdqu (2 * 16)(%rdi), STATE1_LO
+	movdqu (3 * 16)(%rdi), STATE1_HI
+	movdqu (4 * 16)(%rdi), STATE2_LO
+	movdqu (5 * 16)(%rdi), STATE2_HI
+	movdqu (6 * 16)(%rdi), STATE3_LO
+	movdqu (7 * 16)(%rdi), STATE3_HI
+	movdqu (8 * 16)(%rdi), STATE4_LO
+	movdqu (9 * 16)(%rdi), STATE4_HI
+
+	mov %rsi, %r8
+	and $0xF, %r8
+	jnz .Lad_u_loop
+
+.align 4
+.Lad_a_loop:
+	movdqa  0(%rsi), MSG_LO
+	movdqa 16(%rsi), MSG_HI
+	call __morus1280_update
+	sub $32, %rdx
+	add $32, %rsi
+	cmp $32, %rdx
+	jge .Lad_a_loop
+
+	jmp .Lad_cont
+.align 4
+.Lad_u_loop:
+	movdqu  0(%rsi), MSG_LO
+	movdqu 16(%rsi), MSG_HI
+	call __morus1280_update
+	sub $32, %rdx
+	add $32, %rsi
+	cmp $32, %rdx
+	jge .Lad_u_loop
+
+.Lad_cont:
+	/* store the state: */
+	movdqu STATE0_LO, (0 * 16)(%rdi)
+	movdqu STATE0_HI, (1 * 16)(%rdi)
+	movdqu STATE1_LO, (2 * 16)(%rdi)
+	movdqu STATE1_HI, (3 * 16)(%rdi)
+	movdqu STATE2_LO, (4 * 16)(%rdi)
+	movdqu STATE2_HI, (5 * 16)(%rdi)
+	movdqu STATE3_LO, (6 * 16)(%rdi)
+	movdqu STATE3_HI, (7 * 16)(%rdi)
+	movdqu STATE4_LO, (8 * 16)(%rdi)
+	movdqu STATE4_HI, (9 * 16)(%rdi)
+
+.Lad_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_sse2_ad)
+
+/*
+ * void crypto_morus1280_sse2_enc(void *state, const void *src, void *dst,
+ *                                unsigned int length);
+ */
+ENTRY(crypto_morus1280_sse2_enc)
+	FRAME_BEGIN
+
+	cmp $32, %rcx
+	jb .Lenc_out
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0_LO
+	movdqu (1 * 16)(%rdi), STATE0_HI
+	movdqu (2 * 16)(%rdi), STATE1_LO
+	movdqu (3 * 16)(%rdi), STATE1_HI
+	movdqu (4 * 16)(%rdi), STATE2_LO
+	movdqu (5 * 16)(%rdi), STATE2_HI
+	movdqu (6 * 16)(%rdi), STATE3_LO
+	movdqu (7 * 16)(%rdi), STATE3_HI
+	movdqu (8 * 16)(%rdi), STATE4_LO
+	movdqu (9 * 16)(%rdi), STATE4_HI
+
+	mov %rsi, %r8
+	or  %rdx, %r8
+	and $0xF, %r8
+	jnz .Lenc_u_loop
+
+.align 4
+.Lenc_a_loop:
+	movdqa  0(%rsi), MSG_LO
+	movdqa 16(%rsi), MSG_HI
+	movdqa STATE1_LO, T1_LO
+	movdqa STATE1_HI, T1_HI
+	rol3 T1_HI, T1_LO
+	movdqa MSG_LO, T0_LO
+	movdqa MSG_HI, T0_HI
+	pxor T1_LO, T0_LO
+	pxor T1_HI, T0_HI
+	pxor STATE0_LO, T0_LO
+	pxor STATE0_HI, T0_HI
+	movdqa STATE2_LO, T1_LO
+	movdqa STATE2_HI, T1_HI
+	pand STATE3_LO, T1_LO
+	pand STATE3_HI, T1_HI
+	pxor T1_LO, T0_LO
+	pxor T1_HI, T0_HI
+	movdqa T0_LO,  0(%rdx)
+	movdqa T0_HI, 16(%rdx)
+
+	call __morus1280_update
+	sub $32, %rcx
+	add $32, %rsi
+	add $32, %rdx
+	cmp $32, %rcx
+	jge .Lenc_a_loop
+
+	jmp .Lenc_cont
+.align 4
+.Lenc_u_loop:
+	movdqu  0(%rsi), MSG_LO
+	movdqu 16(%rsi), MSG_HI
+	movdqa STATE1_LO, T1_LO
+	movdqa STATE1_HI, T1_HI
+	rol3 T1_HI, T1_LO
+	movdqa MSG_LO, T0_LO
+	movdqa MSG_HI, T0_HI
+	pxor T1_LO, T0_LO
+	pxor T1_HI, T0_HI
+	pxor STATE0_LO, T0_LO
+	pxor STATE0_HI, T0_HI
+	movdqa STATE2_LO, T1_LO
+	movdqa STATE2_HI, T1_HI
+	pand STATE3_LO, T1_LO
+	pand STATE3_HI, T1_HI
+	pxor T1_LO, T0_LO
+	pxor T1_HI, T0_HI
+	movdqu T0_LO,  0(%rdx)
+	movdqu T0_HI, 16(%rdx)
+
+	call __morus1280_update
+	sub $32, %rcx
+	add $32, %rsi
+	add $32, %rdx
+	cmp $32, %rcx
+	jge .Lenc_u_loop
+
+.Lenc_cont:
+	/* store the state: */
+	movdqu STATE0_LO, (0 * 16)(%rdi)
+	movdqu STATE0_HI, (1 * 16)(%rdi)
+	movdqu STATE1_LO, (2 * 16)(%rdi)
+	movdqu STATE1_HI, (3 * 16)(%rdi)
+	movdqu STATE2_LO, (4 * 16)(%rdi)
+	movdqu STATE2_HI, (5 * 16)(%rdi)
+	movdqu STATE3_LO, (6 * 16)(%rdi)
+	movdqu STATE3_HI, (7 * 16)(%rdi)
+	movdqu STATE4_LO, (8 * 16)(%rdi)
+	movdqu STATE4_HI, (9 * 16)(%rdi)
+
+.Lenc_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_sse2_enc)
+
+/*
+ * void crypto_morus1280_sse2_enc_tail(void *state, const void *src, void *dst,
+ *                                     unsigned int length);
+ */
+ENTRY(crypto_morus1280_sse2_enc_tail)
+	FRAME_BEGIN
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0_LO
+	movdqu (1 * 16)(%rdi), STATE0_HI
+	movdqu (2 * 16)(%rdi), STATE1_LO
+	movdqu (3 * 16)(%rdi), STATE1_HI
+	movdqu (4 * 16)(%rdi), STATE2_LO
+	movdqu (5 * 16)(%rdi), STATE2_HI
+	movdqu (6 * 16)(%rdi), STATE3_LO
+	movdqu (7 * 16)(%rdi), STATE3_HI
+	movdqu (8 * 16)(%rdi), STATE4_LO
+	movdqu (9 * 16)(%rdi), STATE4_HI
+
+	/* encrypt message: */
+	call __load_partial
+
+	movdqa STATE1_LO, T1_LO
+	movdqa STATE1_HI, T1_HI
+	rol3 T1_HI, T1_LO
+	movdqa MSG_LO, T0_LO
+	movdqa MSG_HI, T0_HI
+	pxor T1_LO, T0_LO
+	pxor T1_HI, T0_HI
+	pxor STATE0_LO, T0_LO
+	pxor STATE0_HI, T0_HI
+	movdqa STATE2_LO, T1_LO
+	movdqa STATE2_HI, T1_HI
+	pand STATE3_LO, T1_LO
+	pand STATE3_HI, T1_HI
+	pxor T1_LO, T0_LO
+	pxor T1_HI, T0_HI
+
+	call __store_partial
+
+	call __morus1280_update
+
+	/* store the state: */
+	movdqu STATE0_LO, (0 * 16)(%rdi)
+	movdqu STATE0_HI, (1 * 16)(%rdi)
+	movdqu STATE1_LO, (2 * 16)(%rdi)
+	movdqu STATE1_HI, (3 * 16)(%rdi)
+	movdqu STATE2_LO, (4 * 16)(%rdi)
+	movdqu STATE2_HI, (5 * 16)(%rdi)
+	movdqu STATE3_LO, (6 * 16)(%rdi)
+	movdqu STATE3_HI, (7 * 16)(%rdi)
+	movdqu STATE4_LO, (8 * 16)(%rdi)
+	movdqu STATE4_HI, (9 * 16)(%rdi)
+
+	FRAME_END
+ENDPROC(crypto_morus1280_sse2_enc_tail)
+
+/*
+ * void crypto_morus1280_sse2_dec(void *state, const void *src, void *dst,
+ *                                unsigned int length);
+ */
+ENTRY(crypto_morus1280_sse2_dec)
+	FRAME_BEGIN
+
+	cmp $32, %rcx
+	jb .Ldec_out
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0_LO
+	movdqu (1 * 16)(%rdi), STATE0_HI
+	movdqu (2 * 16)(%rdi), STATE1_LO
+	movdqu (3 * 16)(%rdi), STATE1_HI
+	movdqu (4 * 16)(%rdi), STATE2_LO
+	movdqu (5 * 16)(%rdi), STATE2_HI
+	movdqu (6 * 16)(%rdi), STATE3_LO
+	movdqu (7 * 16)(%rdi), STATE3_HI
+	movdqu (8 * 16)(%rdi), STATE4_LO
+	movdqu (9 * 16)(%rdi), STATE4_HI
+
+	mov %rsi, %r8
+	or  %rdx, %r8
+	and $0xF, %r8
+	jnz .Ldec_u_loop
+
+.align 4
+.Ldec_a_loop:
+	movdqa  0(%rsi), MSG_LO
+	movdqa 16(%rsi), MSG_HI
+	pxor STATE0_LO, MSG_LO
+	pxor STATE0_HI, MSG_HI
+	movdqa STATE1_LO, T1_LO
+	movdqa STATE1_HI, T1_HI
+	rol3 T1_HI, T1_LO
+	pxor T1_LO, MSG_LO
+	pxor T1_HI, MSG_HI
+	movdqa STATE2_LO, T1_LO
+	movdqa STATE2_HI, T1_HI
+	pand STATE3_LO, T1_LO
+	pand STATE3_HI, T1_HI
+	pxor T1_LO, MSG_LO
+	pxor T1_HI, MSG_HI
+	movdqa MSG_LO,  0(%rdx)
+	movdqa MSG_HI, 16(%rdx)
+
+	call __morus1280_update
+	sub $32, %rcx
+	add $32, %rsi
+	add $32, %rdx
+	cmp $32, %rcx
+	jge .Ldec_a_loop
+
+	jmp .Ldec_cont
+.align 4
+.Ldec_u_loop:
+	movdqu  0(%rsi), MSG_LO
+	movdqu 16(%rsi), MSG_HI
+	pxor STATE0_LO, MSG_LO
+	pxor STATE0_HI, MSG_HI
+	movdqa STATE1_LO, T1_LO
+	movdqa STATE1_HI, T1_HI
+	rol3 T1_HI, T1_LO
+	pxor T1_LO, MSG_LO
+	pxor T1_HI, MSG_HI
+	movdqa STATE2_LO, T1_LO
+	movdqa STATE2_HI, T1_HI
+	pand STATE3_LO, T1_LO
+	pand STATE3_HI, T1_HI
+	pxor T1_LO, MSG_LO
+	pxor T1_HI, MSG_HI
+	movdqu MSG_LO,  0(%rdx)
+	movdqu MSG_HI, 16(%rdx)
+
+	call __morus1280_update
+	sub $32, %rcx
+	add $32, %rsi
+	add $32, %rdx
+	cmp $32, %rcx
+	jge .Ldec_u_loop
+
+.Ldec_cont:
+	/* store the state: */
+	movdqu STATE0_LO, (0 * 16)(%rdi)
+	movdqu STATE0_HI, (1 * 16)(%rdi)
+	movdqu STATE1_LO, (2 * 16)(%rdi)
+	movdqu STATE1_HI, (3 * 16)(%rdi)
+	movdqu STATE2_LO, (4 * 16)(%rdi)
+	movdqu STATE2_HI, (5 * 16)(%rdi)
+	movdqu STATE3_LO, (6 * 16)(%rdi)
+	movdqu STATE3_HI, (7 * 16)(%rdi)
+	movdqu STATE4_LO, (8 * 16)(%rdi)
+	movdqu STATE4_HI, (9 * 16)(%rdi)
+
+.Ldec_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_sse2_dec)
+
+/*
+ * void crypto_morus1280_sse2_dec_tail(void *state, const void *src, void *dst,
+ *                                     unsigned int length);
+ */
+ENTRY(crypto_morus1280_sse2_dec_tail)
+	FRAME_BEGIN
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0_LO
+	movdqu (1 * 16)(%rdi), STATE0_HI
+	movdqu (2 * 16)(%rdi), STATE1_LO
+	movdqu (3 * 16)(%rdi), STATE1_HI
+	movdqu (4 * 16)(%rdi), STATE2_LO
+	movdqu (5 * 16)(%rdi), STATE2_HI
+	movdqu (6 * 16)(%rdi), STATE3_LO
+	movdqu (7 * 16)(%rdi), STATE3_HI
+	movdqu (8 * 16)(%rdi), STATE4_LO
+	movdqu (9 * 16)(%rdi), STATE4_HI
+
+	/* decrypt message: */
+	call __load_partial
+
+	pxor STATE0_LO, MSG_LO
+	pxor STATE0_HI, MSG_HI
+	movdqa STATE1_LO, T1_LO
+	movdqa STATE1_HI, T1_HI
+	rol3 T1_HI, T1_LO
+	pxor T1_LO, MSG_LO
+	pxor T1_HI, MSG_HI
+	movdqa STATE2_LO, T1_LO
+	movdqa STATE2_HI, T1_HI
+	pand STATE3_LO, T1_LO
+	pand STATE3_HI, T1_HI
+	pxor T1_LO, MSG_LO
+	pxor T1_HI, MSG_HI
+	movdqa MSG_LO, T0_LO
+	movdqa MSG_HI, T0_HI
+
+	call __store_partial
+
+	/* mask with byte count: */
+	movq %rcx, T0_LO
+	punpcklbw T0_LO, T0_LO
+	punpcklbw T0_LO, T0_LO
+	punpcklbw T0_LO, T0_LO
+	punpcklbw T0_LO, T0_LO
+	movdqa T0_LO, T0_HI
+	movdqa .Lmorus640_counter_0, T1_LO
+	movdqa .Lmorus640_counter_1, T1_HI
+	pcmpgtb T1_LO, T0_LO
+	pcmpgtb T1_HI, T0_HI
+	pand T0_LO, MSG_LO
+	pand T0_HI, MSG_HI
+
+	call __morus1280_update
+
+	/* store the state: */
+	movdqu STATE0_LO, (0 * 16)(%rdi)
+	movdqu STATE0_HI, (1 * 16)(%rdi)
+	movdqu STATE1_LO, (2 * 16)(%rdi)
+	movdqu STATE1_HI, (3 * 16)(%rdi)
+	movdqu STATE2_LO, (4 * 16)(%rdi)
+	movdqu STATE2_HI, (5 * 16)(%rdi)
+	movdqu STATE3_LO, (6 * 16)(%rdi)
+	movdqu STATE3_HI, (7 * 16)(%rdi)
+	movdqu STATE4_LO, (8 * 16)(%rdi)
+	movdqu STATE4_HI, (9 * 16)(%rdi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_sse2_dec_tail)
+
+/*
+ * void crypto_morus1280_sse2_final(void *state, void *tag_xor,
+ *                                  u64 assoclen, u64 cryptlen);
+ */
+ENTRY(crypto_morus1280_sse2_final)
+	FRAME_BEGIN
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0_LO
+	movdqu (1 * 16)(%rdi), STATE0_HI
+	movdqu (2 * 16)(%rdi), STATE1_LO
+	movdqu (3 * 16)(%rdi), STATE1_HI
+	movdqu (4 * 16)(%rdi), STATE2_LO
+	movdqu (5 * 16)(%rdi), STATE2_HI
+	movdqu (6 * 16)(%rdi), STATE3_LO
+	movdqu (7 * 16)(%rdi), STATE3_HI
+	movdqu (8 * 16)(%rdi), STATE4_LO
+	movdqu (9 * 16)(%rdi), STATE4_HI
+
+	/* xor state[0] into state[4]: */
+	pxor STATE0_LO, STATE4_LO
+	pxor STATE0_HI, STATE4_HI
+
+	/* prepare length block: */
+	movq %rdx, MSG_LO
+	movq %rcx, T0_LO
+	pslldq $8, T0_LO
+	pxor T0_LO, MSG_LO
+	psllq $3, MSG_LO /* multiply by 8 (to get bit count) */
+	pxor MSG_HI, MSG_HI
+
+	/* update state: */
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+	call __morus1280_update
+
+	/* xor tag: */
+	movdqu  0(%rsi), MSG_LO
+	movdqu 16(%rsi), MSG_HI
+
+	pxor STATE0_LO, MSG_LO
+	pxor STATE0_HI, MSG_HI
+	movdqa STATE1_LO, T0_LO
+	movdqa STATE1_HI, T0_HI
+	rol3 T0_HI, T0_LO
+	pxor T0_LO, MSG_LO
+	pxor T0_HI, MSG_HI
+	movdqa STATE2_LO, T0_LO
+	movdqa STATE2_HI, T0_HI
+	pand STATE3_LO, T0_LO
+	pand STATE3_HI, T0_HI
+	pxor T0_LO, MSG_LO
+	pxor T0_HI, MSG_HI
+
+	movdqu MSG_LO,  0(%rsi)
+	movdqu MSG_HI, 16(%rsi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_morus1280_sse2_final)
diff --git a/arch/x86/crypto/morus1280-sse2-glue.c b/arch/x86/crypto/morus1280-sse2-glue.c
new file mode 100644
index 000000000000..839270aa713c
--- /dev/null
+++ b/arch/x86/crypto/morus1280-sse2-glue.c
@@ -0,0 +1,68 @@
+/*
+ * The MORUS-1280 Authenticated-Encryption Algorithm
+ *   Glue for SSE2 implementation
+ *
+ * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/internal/aead.h>
+#include <crypto/morus1280_glue.h>
+#include <linux/module.h>
+#include <asm/fpu/api.h>
+#include <asm/cpu_device_id.h>
+
+asmlinkage void crypto_morus1280_sse2_init(void *state, const void *key,
+					   const void *iv);
+asmlinkage void crypto_morus1280_sse2_ad(void *state, const void *data,
+					 unsigned int length);
+
+asmlinkage void crypto_morus1280_sse2_enc(void *state, const void *src,
+					  void *dst, unsigned int length);
+asmlinkage void crypto_morus1280_sse2_dec(void *state, const void *src,
+					  void *dst, unsigned int length);
+
+asmlinkage void crypto_morus1280_sse2_enc_tail(void *state, const void *src,
+					       void *dst, unsigned int length);
+asmlinkage void crypto_morus1280_sse2_dec_tail(void *state, const void *src,
+					       void *dst, unsigned int length);
+
+asmlinkage void crypto_morus1280_sse2_final(void *state, void *tag_xor,
+					    u64 assoclen, u64 cryptlen);
+
+MORUS1280_DECLARE_ALGS(sse2, "morus1280-sse2", 350);
+
+static const struct x86_cpu_id sse2_cpu_id[] = {
+    X86_FEATURE_MATCH(X86_FEATURE_XMM2),
+    {}
+};
+MODULE_DEVICE_TABLE(x86cpu, sse2_cpu_id);
+
+static int __init crypto_morus1280_sse2_module_init(void)
+{
+	if (!x86_match_cpu(sse2_cpu_id))
+		return -ENODEV;
+
+	return crypto_register_aeads(crypto_morus1280_sse2_algs,
+				     ARRAY_SIZE(crypto_morus1280_sse2_algs));
+}
+
+static void __exit crypto_morus1280_sse2_module_exit(void)
+{
+	crypto_unregister_aeads(crypto_morus1280_sse2_algs,
+				ARRAY_SIZE(crypto_morus1280_sse2_algs));
+}
+
+module_init(crypto_morus1280_sse2_module_init);
+module_exit(crypto_morus1280_sse2_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm -- SSE2 implementation");
+MODULE_ALIAS_CRYPTO("morus1280");
+MODULE_ALIAS_CRYPTO("morus1280-sse2");
diff --git a/arch/x86/crypto/morus1280_glue.c b/arch/x86/crypto/morus1280_glue.c
new file mode 100644
index 000000000000..0dccdda1eb3a
--- /dev/null
+++ b/arch/x86/crypto/morus1280_glue.c
@@ -0,0 +1,302 @@
+/*
+ * The MORUS-1280 Authenticated-Encryption Algorithm
+ *   Common x86 SIMD glue skeleton
+ *
+ * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/cryptd.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/morus1280_glue.h>
+#include <crypto/scatterwalk.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <asm/fpu/api.h>
+
+struct morus1280_state {
+	struct morus1280_block s[MORUS_STATE_BLOCKS];
+};
+
+struct morus1280_ops {
+	int (*skcipher_walk_init)(struct skcipher_walk *walk,
+				  struct aead_request *req, bool atomic);
+
+	void (*crypt_blocks)(void *state, const void *src, void *dst,
+			     unsigned int length);
+	void (*crypt_tail)(void *state, const void *src, void *dst,
+			   unsigned int length);
+};
+
+static void crypto_morus1280_glue_process_ad(
+		struct morus1280_state *state,
+		const struct morus1280_glue_ops *ops,
+		struct scatterlist *sg_src, unsigned int assoclen)
+{
+	struct scatter_walk walk;
+	struct morus1280_block buf;
+	unsigned int pos = 0;
+
+	scatterwalk_start(&walk, sg_src);
+	while (assoclen != 0) {
+		unsigned int size = scatterwalk_clamp(&walk, assoclen);
+		unsigned int left = size;
+		void *mapped = scatterwalk_map(&walk);
+		const u8 *src = (const u8 *)mapped;
+
+		if (pos + size >= MORUS1280_BLOCK_SIZE) {
+			if (pos > 0) {
+				unsigned int fill = MORUS1280_BLOCK_SIZE - pos;
+				memcpy(buf.bytes + pos, src, fill);
+				ops->ad(state, buf.bytes, MORUS1280_BLOCK_SIZE);
+				pos = 0;
+				left -= fill;
+				src += fill;
+			}
+
+			ops->ad(state, src, left);
+			src += left & ~(MORUS1280_BLOCK_SIZE - 1);
+			left &= MORUS1280_BLOCK_SIZE - 1;
+		}
+
+		memcpy(buf.bytes + pos, src, left);
+
+		pos += left;
+		assoclen -= size;
+		scatterwalk_unmap(mapped);
+		scatterwalk_advance(&walk, size);
+		scatterwalk_done(&walk, 0, assoclen);
+	}
+
+	if (pos > 0) {
+		memset(buf.bytes + pos, 0, MORUS1280_BLOCK_SIZE - pos);
+		ops->ad(state, buf.bytes, MORUS1280_BLOCK_SIZE);
+	}
+}
+
+static void crypto_morus1280_glue_process_crypt(struct morus1280_state *state,
+						struct morus1280_ops ops,
+						struct aead_request *req)
+{
+	struct skcipher_walk walk;
+	u8 *cursor_src, *cursor_dst;
+	unsigned int chunksize, base;
+
+	ops.skcipher_walk_init(&walk, req, false);
+
+	while (walk.nbytes) {
+		cursor_src = walk.src.virt.addr;
+		cursor_dst = walk.dst.virt.addr;
+		chunksize = walk.nbytes;
+
+		ops.crypt_blocks(state, cursor_src, cursor_dst, chunksize);
+
+		base = chunksize & ~(MORUS1280_BLOCK_SIZE - 1);
+		cursor_src += base;
+		cursor_dst += base;
+		chunksize &= MORUS1280_BLOCK_SIZE - 1;
+
+		if (chunksize > 0)
+			ops.crypt_tail(state, cursor_src, cursor_dst,
+				       chunksize);
+
+		skcipher_walk_done(&walk, 0);
+	}
+}
+
+int crypto_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key,
+				 unsigned int keylen)
+{
+	struct morus1280_ctx *ctx = crypto_aead_ctx(aead);
+
+	if (keylen == MORUS1280_BLOCK_SIZE) {
+		memcpy(ctx->key.bytes, key, MORUS1280_BLOCK_SIZE);
+	} else if (keylen == MORUS1280_BLOCK_SIZE / 2) {
+		memcpy(ctx->key.bytes, key, keylen);
+		memcpy(ctx->key.bytes + keylen, key, keylen);
+	} else {
+		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_morus1280_glue_setkey);
+
+int crypto_morus1280_glue_setauthsize(struct crypto_aead *tfm,
+				      unsigned int authsize)
+{
+	return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL;
+}
+EXPORT_SYMBOL_GPL(crypto_morus1280_glue_setauthsize);
+
+static void crypto_morus1280_glue_crypt(struct aead_request *req,
+					struct morus1280_ops ops,
+					unsigned int cryptlen,
+					struct morus1280_block *tag_xor)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
+	struct morus1280_state state;
+
+	kernel_fpu_begin();
+
+	ctx->ops->init(&state, &ctx->key, req->iv);
+	crypto_morus1280_glue_process_ad(&state, ctx->ops, req->src, req->assoclen);
+	crypto_morus1280_glue_process_crypt(&state, ops, req);
+	ctx->ops->final(&state, tag_xor, req->assoclen, cryptlen);
+
+	kernel_fpu_end();
+}
+
+int crypto_morus1280_glue_encrypt(struct aead_request *req)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
+	struct morus1280_ops OPS = {
+		.skcipher_walk_init = skcipher_walk_aead_encrypt,
+		.crypt_blocks = ctx->ops->enc,
+		.crypt_tail = ctx->ops->enc_tail,
+	};
+
+	struct morus1280_block tag = {};
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen;
+
+	crypto_morus1280_glue_crypt(req, OPS, cryptlen, &tag);
+
+	scatterwalk_map_and_copy(tag.bytes, req->dst,
+				 req->assoclen + cryptlen, authsize, 1);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_morus1280_glue_encrypt);
+
+int crypto_morus1280_glue_decrypt(struct aead_request *req)
+{
+	static const u8 zeros[MORUS1280_BLOCK_SIZE] = {};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
+	struct morus1280_ops OPS = {
+		.skcipher_walk_init = skcipher_walk_aead_decrypt,
+		.crypt_blocks = ctx->ops->dec,
+		.crypt_tail = ctx->ops->dec_tail,
+	};
+
+	struct morus1280_block tag;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen - authsize;
+
+	scatterwalk_map_and_copy(tag.bytes, req->src,
+				 req->assoclen + cryptlen, authsize, 0);
+
+	crypto_morus1280_glue_crypt(req, OPS, cryptlen, &tag);
+
+	return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;
+}
+EXPORT_SYMBOL_GPL(crypto_morus1280_glue_decrypt);
+
+void crypto_morus1280_glue_init_ops(struct crypto_aead *aead,
+				    const struct morus1280_glue_ops *ops)
+{
+	struct morus1280_ctx *ctx = crypto_aead_ctx(aead);
+	ctx->ops = ops;
+}
+EXPORT_SYMBOL_GPL(crypto_morus1280_glue_init_ops);
+
+int cryptd_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key,
+				 unsigned int keylen)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
+}
+EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_setkey);
+
+int cryptd_morus1280_glue_setauthsize(struct crypto_aead *aead,
+				      unsigned int authsize)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
+}
+EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_setauthsize);
+
+int cryptd_morus1280_glue_encrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	aead = &cryptd_tfm->base;
+	if (irq_fpu_usable() && (!in_atomic() ||
+				 !cryptd_aead_queued(cryptd_tfm)))
+		aead = cryptd_aead_child(cryptd_tfm);
+
+	aead_request_set_tfm(req, aead);
+
+	return crypto_aead_encrypt(req);
+}
+EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_encrypt);
+
+int cryptd_morus1280_glue_decrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	aead = &cryptd_tfm->base;
+	if (irq_fpu_usable() && (!in_atomic() ||
+				 !cryptd_aead_queued(cryptd_tfm)))
+		aead = cryptd_aead_child(cryptd_tfm);
+
+	aead_request_set_tfm(req, aead);
+
+	return crypto_aead_decrypt(req);
+}
+EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_decrypt);
+
+int cryptd_morus1280_glue_init_tfm(struct crypto_aead *aead)
+{
+	struct cryptd_aead *cryptd_tfm;
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	const char *name = crypto_aead_alg(aead)->base.cra_driver_name;
+	char internal_name[CRYPTO_MAX_ALG_NAME];
+
+	if (snprintf(internal_name, CRYPTO_MAX_ALG_NAME, "__%s", name)
+			>= CRYPTO_MAX_ALG_NAME)
+		return -ENAMETOOLONG;
+
+	cryptd_tfm = cryptd_alloc_aead(internal_name, CRYPTO_ALG_INTERNAL,
+				       CRYPTO_ALG_INTERNAL);
+	if (IS_ERR(cryptd_tfm))
+		return PTR_ERR(cryptd_tfm);
+
+	*ctx = cryptd_tfm;
+	crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_init_tfm);
+
+void cryptd_morus1280_glue_exit_tfm(struct crypto_aead *aead)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+
+	cryptd_free_aead(*ctx);
+}
+EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_exit_tfm);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("MORUS-1280 AEAD mode -- glue for x86 optimizations");
diff --git a/arch/x86/crypto/morus640-sse2-asm.S b/arch/x86/crypto/morus640-sse2-asm.S
new file mode 100644
index 000000000000..71c72a0a0862
--- /dev/null
+++ b/arch/x86/crypto/morus640-sse2-asm.S
@@ -0,0 +1,614 @@
+/*
+ * SSE2 implementation of MORUS-640
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/frame.h>
+
+#define SHUFFLE_MASK(i0, i1, i2, i3) \
+	(i0 | (i1 << 2) | (i2 << 4) | (i3 << 6))
+
+#define MASK1 SHUFFLE_MASK(3, 0, 1, 2)
+#define MASK2 SHUFFLE_MASK(2, 3, 0, 1)
+#define MASK3 SHUFFLE_MASK(1, 2, 3, 0)
+
+#define STATE0	%xmm0
+#define STATE1	%xmm1
+#define STATE2	%xmm2
+#define STATE3	%xmm3
+#define STATE4	%xmm4
+#define KEY	%xmm5
+#define MSG	%xmm5
+#define T0	%xmm6
+#define T1	%xmm7
+
+.section .rodata.cst16.morus640_const, "aM", @progbits, 32
+.align 16
+.Lmorus640_const_0:
+	.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
+	.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
+.Lmorus640_const_1:
+	.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
+	.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
+
+.section .rodata.cst16.morus640_counter, "aM", @progbits, 16
+.align 16
+.Lmorus640_counter:
+	.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
+	.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
+
+.text
+
+.macro morus640_round s0, s1, s2, s3, s4, b, w
+	movdqa \s1, T0
+	pand \s2, T0
+	pxor T0, \s0
+	pxor \s3, \s0
+	movdqa \s0, T0
+	pslld $\b, T0
+	psrld $(32 - \b), \s0
+	pxor T0, \s0
+	pshufd $\w, \s3, \s3
+.endm
+
+/*
+ * __morus640_update: internal ABI
+ * input:
+ *   STATE[0-4] - input state
+ *   MSG        - message block
+ * output:
+ *   STATE[0-4] - output state
+ * changed:
+ *   T0
+ */
+__morus640_update:
+	morus640_round STATE0, STATE1, STATE2, STATE3, STATE4,  5, MASK1
+	pxor MSG, STATE1
+	morus640_round STATE1, STATE2, STATE3, STATE4, STATE0, 31, MASK2
+	pxor MSG, STATE2
+	morus640_round STATE2, STATE3, STATE4, STATE0, STATE1,  7, MASK3
+	pxor MSG, STATE3
+	morus640_round STATE3, STATE4, STATE0, STATE1, STATE2, 22, MASK2
+	pxor MSG, STATE4
+	morus640_round STATE4, STATE0, STATE1, STATE2, STATE3, 13, MASK1
+	ret
+ENDPROC(__morus640_update)
+
+
+/*
+ * __morus640_update_zero: internal ABI
+ * input:
+ *   STATE[0-4] - input state
+ * output:
+ *   STATE[0-4] - output state
+ * changed:
+ *   T0
+ */
+__morus640_update_zero:
+	morus640_round STATE0, STATE1, STATE2, STATE3, STATE4,  5, MASK1
+	morus640_round STATE1, STATE2, STATE3, STATE4, STATE0, 31, MASK2
+	morus640_round STATE2, STATE3, STATE4, STATE0, STATE1,  7, MASK3
+	morus640_round STATE3, STATE4, STATE0, STATE1, STATE2, 22, MASK2
+	morus640_round STATE4, STATE0, STATE1, STATE2, STATE3, 13, MASK1
+	ret
+ENDPROC(__morus640_update_zero)
+
+/*
+ * __load_partial: internal ABI
+ * input:
+ *   %rsi - src
+ *   %rcx - bytes
+ * output:
+ *   MSG  - message block
+ * changed:
+ *   T0
+ *   %r8
+ *   %r9
+ */
+__load_partial:
+	xor %r9, %r9
+	pxor MSG, MSG
+
+	mov %rcx, %r8
+	and $0x1, %r8
+	jz .Lld_partial_1
+
+	mov %rcx, %r8
+	and $0x1E, %r8
+	add %rsi, %r8
+	mov (%r8), %r9b
+
+.Lld_partial_1:
+	mov %rcx, %r8
+	and $0x2, %r8
+	jz .Lld_partial_2
+
+	mov %rcx, %r8
+	and $0x1C, %r8
+	add %rsi, %r8
+	shl $16, %r9
+	mov (%r8), %r9w
+
+.Lld_partial_2:
+	mov %rcx, %r8
+	and $0x4, %r8
+	jz .Lld_partial_4
+
+	mov %rcx, %r8
+	and $0x18, %r8
+	add %rsi, %r8
+	shl $32, %r9
+	mov (%r8), %r8d
+	xor %r8, %r9
+
+.Lld_partial_4:
+	movq %r9, MSG
+
+	mov %rcx, %r8
+	and $0x8, %r8
+	jz .Lld_partial_8
+
+	mov %rcx, %r8
+	and $0x10, %r8
+	add %rsi, %r8
+	pslldq $8, MSG
+	movq (%r8), T0
+	pxor T0, MSG
+
+.Lld_partial_8:
+	ret
+ENDPROC(__load_partial)
+
+/*
+ * __store_partial: internal ABI
+ * input:
+ *   %rdx - dst
+ *   %rcx - bytes
+ * output:
+ *   T0   - message block
+ * changed:
+ *   %r8
+ *   %r9
+ *   %r10
+ */
+__store_partial:
+	mov %rcx, %r8
+	mov %rdx, %r9
+
+	movq T0, %r10
+
+	cmp $8, %r8
+	jl .Lst_partial_8
+
+	mov %r10, (%r9)
+	psrldq $8, T0
+	movq T0, %r10
+
+	sub $8, %r8
+	add $8, %r9
+
+.Lst_partial_8:
+	cmp $4, %r8
+	jl .Lst_partial_4
+
+	mov %r10d, (%r9)
+	shr $32, %r10
+
+	sub $4, %r8
+	add $4, %r9
+
+.Lst_partial_4:
+	cmp $2, %r8
+	jl .Lst_partial_2
+
+	mov %r10w, (%r9)
+	shr $16, %r10
+
+	sub $2, %r8
+	add $2, %r9
+
+.Lst_partial_2:
+	cmp $1, %r8
+	jl .Lst_partial_1
+
+	mov %r10b, (%r9)
+
+.Lst_partial_1:
+	ret
+ENDPROC(__store_partial)
+
+/*
+ * void crypto_morus640_sse2_init(void *state, const void *key, const void *iv);
+ */
+ENTRY(crypto_morus640_sse2_init)
+	FRAME_BEGIN
+
+	/* load IV: */
+	movdqu (%rdx), STATE0
+	/* load key: */
+	movdqu (%rsi), KEY
+	movdqa KEY, STATE1
+	/* load all ones: */
+	pcmpeqd STATE2, STATE2
+	/* load the constants: */
+	movdqa .Lmorus640_const_0, STATE3
+	movdqa .Lmorus640_const_1, STATE4
+
+	/* update 16 times with zero: */
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+	call __morus640_update_zero
+
+	/* xor-in the key again after updates: */
+	pxor KEY, STATE1
+
+	/* store the state: */
+	movdqu STATE0, (0 * 16)(%rdi)
+	movdqu STATE1, (1 * 16)(%rdi)
+	movdqu STATE2, (2 * 16)(%rdi)
+	movdqu STATE3, (3 * 16)(%rdi)
+	movdqu STATE4, (4 * 16)(%rdi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_morus640_sse2_init)
+
+/*
+ * void crypto_morus640_sse2_ad(void *state, const void *data,
+ *                              unsigned int length);
+ */
+ENTRY(crypto_morus640_sse2_ad)
+	FRAME_BEGIN
+
+	cmp $16, %rdx
+	jb .Lad_out
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0
+	movdqu (1 * 16)(%rdi), STATE1
+	movdqu (2 * 16)(%rdi), STATE2
+	movdqu (3 * 16)(%rdi), STATE3
+	movdqu (4 * 16)(%rdi), STATE4
+
+	mov %rsi, %r8
+	and $0xF, %r8
+	jnz .Lad_u_loop
+
+.align 4
+.Lad_a_loop:
+	movdqa (%rsi), MSG
+	call __morus640_update
+	sub $16, %rdx
+	add $16, %rsi
+	cmp $16, %rdx
+	jge .Lad_a_loop
+
+	jmp .Lad_cont
+.align 4
+.Lad_u_loop:
+	movdqu (%rsi), MSG
+	call __morus640_update
+	sub $16, %rdx
+	add $16, %rsi
+	cmp $16, %rdx
+	jge .Lad_u_loop
+
+.Lad_cont:
+	/* store the state: */
+	movdqu STATE0, (0 * 16)(%rdi)
+	movdqu STATE1, (1 * 16)(%rdi)
+	movdqu STATE2, (2 * 16)(%rdi)
+	movdqu STATE3, (3 * 16)(%rdi)
+	movdqu STATE4, (4 * 16)(%rdi)
+
+.Lad_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_morus640_sse2_ad)
+
+/*
+ * void crypto_morus640_sse2_enc(void *state, const void *src, void *dst,
+ *                               unsigned int length);
+ */
+ENTRY(crypto_morus640_sse2_enc)
+	FRAME_BEGIN
+
+	cmp $16, %rcx
+	jb .Lenc_out
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0
+	movdqu (1 * 16)(%rdi), STATE1
+	movdqu (2 * 16)(%rdi), STATE2
+	movdqu (3 * 16)(%rdi), STATE3
+	movdqu (4 * 16)(%rdi), STATE4
+
+	mov %rsi, %r8
+	or  %rdx, %r8
+	and $0xF, %r8
+	jnz .Lenc_u_loop
+
+.align 4
+.Lenc_a_loop:
+	movdqa (%rsi), MSG
+	movdqa MSG, T0
+	pxor STATE0, T0
+	pshufd $MASK3, STATE1, T1
+	pxor T1, T0
+	movdqa STATE2, T1
+	pand STATE3, T1
+	pxor T1, T0
+	movdqa T0, (%rdx)
+
+	call __morus640_update
+	sub $16, %rcx
+	add $16, %rsi
+	add $16, %rdx
+	cmp $16, %rcx
+	jge .Lenc_a_loop
+
+	jmp .Lenc_cont
+.align 4
+.Lenc_u_loop:
+	movdqu (%rsi), MSG
+	movdqa MSG, T0
+	pxor STATE0, T0
+	pshufd $MASK3, STATE1, T1
+	pxor T1, T0
+	movdqa STATE2, T1
+	pand STATE3, T1
+	pxor T1, T0
+	movdqu T0, (%rdx)
+
+	call __morus640_update
+	sub $16, %rcx
+	add $16, %rsi
+	add $16, %rdx
+	cmp $16, %rcx
+	jge .Lenc_u_loop
+
+.Lenc_cont:
+	/* store the state: */
+	movdqu STATE0, (0 * 16)(%rdi)
+	movdqu STATE1, (1 * 16)(%rdi)
+	movdqu STATE2, (2 * 16)(%rdi)
+	movdqu STATE3, (3 * 16)(%rdi)
+	movdqu STATE4, (4 * 16)(%rdi)
+
+.Lenc_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_morus640_sse2_enc)
+
+/*
+ * void crypto_morus640_sse2_enc_tail(void *state, const void *src, void *dst,
+ *                                    unsigned int length);
+ */
+ENTRY(crypto_morus640_sse2_enc_tail)
+	FRAME_BEGIN
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0
+	movdqu (1 * 16)(%rdi), STATE1
+	movdqu (2 * 16)(%rdi), STATE2
+	movdqu (3 * 16)(%rdi), STATE3
+	movdqu (4 * 16)(%rdi), STATE4
+
+	/* encrypt message: */
+	call __load_partial
+
+	movdqa MSG, T0
+	pxor STATE0, T0
+	pshufd $MASK3, STATE1, T1
+	pxor T1, T0
+	movdqa STATE2, T1
+	pand STATE3, T1
+	pxor T1, T0
+
+	call __store_partial
+
+	call __morus640_update
+
+	/* store the state: */
+	movdqu STATE0, (0 * 16)(%rdi)
+	movdqu STATE1, (1 * 16)(%rdi)
+	movdqu STATE2, (2 * 16)(%rdi)
+	movdqu STATE3, (3 * 16)(%rdi)
+	movdqu STATE4, (4 * 16)(%rdi)
+
+	FRAME_END
+ENDPROC(crypto_morus640_sse2_enc_tail)
+
+/*
+ * void crypto_morus640_sse2_dec(void *state, const void *src, void *dst,
+ *                               unsigned int length);
+ */
+ENTRY(crypto_morus640_sse2_dec)
+	FRAME_BEGIN
+
+	cmp $16, %rcx
+	jb .Ldec_out
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0
+	movdqu (1 * 16)(%rdi), STATE1
+	movdqu (2 * 16)(%rdi), STATE2
+	movdqu (3 * 16)(%rdi), STATE3
+	movdqu (4 * 16)(%rdi), STATE4
+
+	mov %rsi, %r8
+	or  %rdx, %r8
+	and $0xF, %r8
+	jnz .Ldec_u_loop
+
+.align 4
+.Ldec_a_loop:
+	movdqa (%rsi), MSG
+	pxor STATE0, MSG
+	pshufd $MASK3, STATE1, T0
+	pxor T0, MSG
+	movdqa STATE2, T0
+	pand STATE3, T0
+	pxor T0, MSG
+	movdqa MSG, (%rdx)
+
+	call __morus640_update
+	sub $16, %rcx
+	add $16, %rsi
+	add $16, %rdx
+	cmp $16, %rcx
+	jge .Ldec_a_loop
+
+	jmp .Ldec_cont
+.align 4
+.Ldec_u_loop:
+	movdqu (%rsi), MSG
+	pxor STATE0, MSG
+	pshufd $MASK3, STATE1, T0
+	pxor T0, MSG
+	movdqa STATE2, T0
+	pand STATE3, T0
+	pxor T0, MSG
+	movdqu MSG, (%rdx)
+
+	call __morus640_update
+	sub $16, %rcx
+	add $16, %rsi
+	add $16, %rdx
+	cmp $16, %rcx
+	jge .Ldec_u_loop
+
+.Ldec_cont:
+	/* store the state: */
+	movdqu STATE0, (0 * 16)(%rdi)
+	movdqu STATE1, (1 * 16)(%rdi)
+	movdqu STATE2, (2 * 16)(%rdi)
+	movdqu STATE3, (3 * 16)(%rdi)
+	movdqu STATE4, (4 * 16)(%rdi)
+
+.Ldec_out:
+	FRAME_END
+	ret
+ENDPROC(crypto_morus640_sse2_dec)
+
+/*
+ * void crypto_morus640_sse2_dec_tail(void *state, const void *src, void *dst,
+ *                                    unsigned int length);
+ */
+ENTRY(crypto_morus640_sse2_dec_tail)
+	FRAME_BEGIN
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0
+	movdqu (1 * 16)(%rdi), STATE1
+	movdqu (2 * 16)(%rdi), STATE2
+	movdqu (3 * 16)(%rdi), STATE3
+	movdqu (4 * 16)(%rdi), STATE4
+
+	/* decrypt message: */
+	call __load_partial
+
+	pxor STATE0, MSG
+	pshufd $MASK3, STATE1, T0
+	pxor T0, MSG
+	movdqa STATE2, T0
+	pand STATE3, T0
+	pxor T0, MSG
+	movdqa MSG, T0
+
+	call __store_partial
+
+	/* mask with byte count: */
+	movq %rcx, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	punpcklbw T0, T0
+	movdqa .Lmorus640_counter, T1
+	pcmpgtb T1, T0
+	pand T0, MSG
+
+	call __morus640_update
+
+	/* store the state: */
+	movdqu STATE0, (0 * 16)(%rdi)
+	movdqu STATE1, (1 * 16)(%rdi)
+	movdqu STATE2, (2 * 16)(%rdi)
+	movdqu STATE3, (3 * 16)(%rdi)
+	movdqu STATE4, (4 * 16)(%rdi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_morus640_sse2_dec_tail)
+
+/*
+ * void crypto_morus640_sse2_final(void *state, void *tag_xor,
+ *	                           u64 assoclen, u64 cryptlen);
+ */
+ENTRY(crypto_morus640_sse2_final)
+	FRAME_BEGIN
+
+	/* load the state: */
+	movdqu (0 * 16)(%rdi), STATE0
+	movdqu (1 * 16)(%rdi), STATE1
+	movdqu (2 * 16)(%rdi), STATE2
+	movdqu (3 * 16)(%rdi), STATE3
+	movdqu (4 * 16)(%rdi), STATE4
+
+	/* xor state[0] into state[4]: */
+	pxor STATE0, STATE4
+
+	/* prepare length block: */
+	movq %rdx, MSG
+	movq %rcx, T0
+	pslldq $8, T0
+	pxor T0, MSG
+	psllq $3, MSG /* multiply by 8 (to get bit count) */
+
+	/* update state: */
+	call __morus640_update
+	call __morus640_update
+	call __morus640_update
+	call __morus640_update
+	call __morus640_update
+	call __morus640_update
+	call __morus640_update
+	call __morus640_update
+	call __morus640_update
+	call __morus640_update
+
+	/* xor tag: */
+	movdqu (%rsi), MSG
+
+	pxor STATE0, MSG
+	pshufd $MASK3, STATE1, T0
+	pxor T0, MSG
+	movdqa STATE2, T0
+	pand STATE3, T0
+	pxor T0, MSG
+
+	movdqu MSG, (%rsi)
+
+	FRAME_END
+	ret
+ENDPROC(crypto_morus640_sse2_final)
diff --git a/arch/x86/crypto/morus640-sse2-glue.c b/arch/x86/crypto/morus640-sse2-glue.c
new file mode 100644
index 000000000000..26b47e2db8d2
--- /dev/null
+++ b/arch/x86/crypto/morus640-sse2-glue.c
@@ -0,0 +1,68 @@
+/*
+ * The MORUS-640 Authenticated-Encryption Algorithm
+ *   Glue for SSE2 implementation
+ *
+ * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/internal/aead.h>
+#include <crypto/morus640_glue.h>
+#include <linux/module.h>
+#include <asm/fpu/api.h>
+#include <asm/cpu_device_id.h>
+
+asmlinkage void crypto_morus640_sse2_init(void *state, const void *key,
+					  const void *iv);
+asmlinkage void crypto_morus640_sse2_ad(void *state, const void *data,
+					unsigned int length);
+
+asmlinkage void crypto_morus640_sse2_enc(void *state, const void *src,
+					 void *dst, unsigned int length);
+asmlinkage void crypto_morus640_sse2_dec(void *state, const void *src,
+					 void *dst, unsigned int length);
+
+asmlinkage void crypto_morus640_sse2_enc_tail(void *state, const void *src,
+					      void *dst, unsigned int length);
+asmlinkage void crypto_morus640_sse2_dec_tail(void *state, const void *src,
+					      void *dst, unsigned int length);
+
+asmlinkage void crypto_morus640_sse2_final(void *state, void *tag_xor,
+					   u64 assoclen, u64 cryptlen);
+
+MORUS640_DECLARE_ALGS(sse2, "morus640-sse2", 400);
+
+static const struct x86_cpu_id sse2_cpu_id[] = {
+    X86_FEATURE_MATCH(X86_FEATURE_XMM2),
+    {}
+};
+MODULE_DEVICE_TABLE(x86cpu, sse2_cpu_id);
+
+static int __init crypto_morus640_sse2_module_init(void)
+{
+	if (!x86_match_cpu(sse2_cpu_id))
+		return -ENODEV;
+
+	return crypto_register_aeads(crypto_morus640_sse2_algs,
+				     ARRAY_SIZE(crypto_morus640_sse2_algs));
+}
+
+static void __exit crypto_morus640_sse2_module_exit(void)
+{
+	crypto_unregister_aeads(crypto_morus640_sse2_algs,
+				ARRAY_SIZE(crypto_morus640_sse2_algs));
+}
+
+module_init(crypto_morus640_sse2_module_init);
+module_exit(crypto_morus640_sse2_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("MORUS-640 AEAD algorithm -- SSE2 implementation");
+MODULE_ALIAS_CRYPTO("morus640");
+MODULE_ALIAS_CRYPTO("morus640-sse2");
diff --git a/arch/x86/crypto/morus640_glue.c b/arch/x86/crypto/morus640_glue.c
new file mode 100644
index 000000000000..7b58fe4d9bd1
--- /dev/null
+++ b/arch/x86/crypto/morus640_glue.c
@@ -0,0 +1,298 @@
+/*
+ * The MORUS-640 Authenticated-Encryption Algorithm
+ *   Common x86 SIMD glue skeleton
+ *
+ * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/cryptd.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/morus640_glue.h>
+#include <crypto/scatterwalk.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <asm/fpu/api.h>
+
+struct morus640_state {
+	struct morus640_block s[MORUS_STATE_BLOCKS];
+};
+
+struct morus640_ops {
+	int (*skcipher_walk_init)(struct skcipher_walk *walk,
+				  struct aead_request *req, bool atomic);
+
+	void (*crypt_blocks)(void *state, const void *src, void *dst,
+			     unsigned int length);
+	void (*crypt_tail)(void *state, const void *src, void *dst,
+			   unsigned int length);
+};
+
+static void crypto_morus640_glue_process_ad(
+		struct morus640_state *state,
+		const struct morus640_glue_ops *ops,
+		struct scatterlist *sg_src, unsigned int assoclen)
+{
+	struct scatter_walk walk;
+	struct morus640_block buf;
+	unsigned int pos = 0;
+
+	scatterwalk_start(&walk, sg_src);
+	while (assoclen != 0) {
+		unsigned int size = scatterwalk_clamp(&walk, assoclen);
+		unsigned int left = size;
+		void *mapped = scatterwalk_map(&walk);
+		const u8 *src = (const u8 *)mapped;
+
+		if (pos + size >= MORUS640_BLOCK_SIZE) {
+			if (pos > 0) {
+				unsigned int fill = MORUS640_BLOCK_SIZE - pos;
+				memcpy(buf.bytes + pos, src, fill);
+				ops->ad(state, buf.bytes, MORUS640_BLOCK_SIZE);
+				pos = 0;
+				left -= fill;
+				src += fill;
+			}
+
+			ops->ad(state, src, left);
+			src += left & ~(MORUS640_BLOCK_SIZE - 1);
+			left &= MORUS640_BLOCK_SIZE - 1;
+		}
+
+		memcpy(buf.bytes + pos, src, left);
+
+		pos += left;
+		assoclen -= size;
+		scatterwalk_unmap(mapped);
+		scatterwalk_advance(&walk, size);
+		scatterwalk_done(&walk, 0, assoclen);
+	}
+
+	if (pos > 0) {
+		memset(buf.bytes + pos, 0, MORUS640_BLOCK_SIZE - pos);
+		ops->ad(state, buf.bytes, MORUS640_BLOCK_SIZE);
+	}
+}
+
+static void crypto_morus640_glue_process_crypt(struct morus640_state *state,
+					       struct morus640_ops ops,
+					       struct aead_request *req)
+{
+	struct skcipher_walk walk;
+	u8 *cursor_src, *cursor_dst;
+	unsigned int chunksize, base;
+
+	ops.skcipher_walk_init(&walk, req, false);
+
+	while (walk.nbytes) {
+		cursor_src = walk.src.virt.addr;
+		cursor_dst = walk.dst.virt.addr;
+		chunksize = walk.nbytes;
+
+		ops.crypt_blocks(state, cursor_src, cursor_dst, chunksize);
+
+		base = chunksize & ~(MORUS640_BLOCK_SIZE - 1);
+		cursor_src += base;
+		cursor_dst += base;
+		chunksize &= MORUS640_BLOCK_SIZE - 1;
+
+		if (chunksize > 0)
+			ops.crypt_tail(state, cursor_src, cursor_dst,
+				       chunksize);
+
+		skcipher_walk_done(&walk, 0);
+	}
+}
+
+int crypto_morus640_glue_setkey(struct crypto_aead *aead, const u8 *key,
+				unsigned int keylen)
+{
+	struct morus640_ctx *ctx = crypto_aead_ctx(aead);
+
+	if (keylen != MORUS640_BLOCK_SIZE) {
+		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	memcpy(ctx->key.bytes, key, MORUS640_BLOCK_SIZE);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_morus640_glue_setkey);
+
+int crypto_morus640_glue_setauthsize(struct crypto_aead *tfm,
+				     unsigned int authsize)
+{
+	return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL;
+}
+EXPORT_SYMBOL_GPL(crypto_morus640_glue_setauthsize);
+
+static void crypto_morus640_glue_crypt(struct aead_request *req,
+				       struct morus640_ops ops,
+				       unsigned int cryptlen,
+				       struct morus640_block *tag_xor)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
+	struct morus640_state state;
+
+	kernel_fpu_begin();
+
+	ctx->ops->init(&state, &ctx->key, req->iv);
+	crypto_morus640_glue_process_ad(&state, ctx->ops, req->src, req->assoclen);
+	crypto_morus640_glue_process_crypt(&state, ops, req);
+	ctx->ops->final(&state, tag_xor, req->assoclen, cryptlen);
+
+	kernel_fpu_end();
+}
+
+int crypto_morus640_glue_encrypt(struct aead_request *req)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
+	struct morus640_ops OPS = {
+		.skcipher_walk_init = skcipher_walk_aead_encrypt,
+		.crypt_blocks = ctx->ops->enc,
+		.crypt_tail = ctx->ops->enc_tail,
+	};
+
+	struct morus640_block tag = {};
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen;
+
+	crypto_morus640_glue_crypt(req, OPS, cryptlen, &tag);
+
+	scatterwalk_map_and_copy(tag.bytes, req->dst,
+				 req->assoclen + cryptlen, authsize, 1);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_morus640_glue_encrypt);
+
+int crypto_morus640_glue_decrypt(struct aead_request *req)
+{
+	static const u8 zeros[MORUS640_BLOCK_SIZE] = {};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
+	struct morus640_ops OPS = {
+		.skcipher_walk_init = skcipher_walk_aead_decrypt,
+		.crypt_blocks = ctx->ops->dec,
+		.crypt_tail = ctx->ops->dec_tail,
+	};
+
+	struct morus640_block tag;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen - authsize;
+
+	scatterwalk_map_and_copy(tag.bytes, req->src,
+				 req->assoclen + cryptlen, authsize, 0);
+
+	crypto_morus640_glue_crypt(req, OPS, cryptlen, &tag);
+
+	return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;
+}
+EXPORT_SYMBOL_GPL(crypto_morus640_glue_decrypt);
+
+void crypto_morus640_glue_init_ops(struct crypto_aead *aead,
+				   const struct morus640_glue_ops *ops)
+{
+	struct morus640_ctx *ctx = crypto_aead_ctx(aead);
+	ctx->ops = ops;
+}
+EXPORT_SYMBOL_GPL(crypto_morus640_glue_init_ops);
+
+int cryptd_morus640_glue_setkey(struct crypto_aead *aead, const u8 *key,
+				unsigned int keylen)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
+}
+EXPORT_SYMBOL_GPL(cryptd_morus640_glue_setkey);
+
+int cryptd_morus640_glue_setauthsize(struct crypto_aead *aead,
+				     unsigned int authsize)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
+}
+EXPORT_SYMBOL_GPL(cryptd_morus640_glue_setauthsize);
+
+int cryptd_morus640_glue_encrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	aead = &cryptd_tfm->base;
+	if (irq_fpu_usable() && (!in_atomic() ||
+				 !cryptd_aead_queued(cryptd_tfm)))
+		aead = cryptd_aead_child(cryptd_tfm);
+
+	aead_request_set_tfm(req, aead);
+
+	return crypto_aead_encrypt(req);
+}
+EXPORT_SYMBOL_GPL(cryptd_morus640_glue_encrypt);
+
+int cryptd_morus640_glue_decrypt(struct aead_request *req)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	struct cryptd_aead *cryptd_tfm = *ctx;
+
+	aead = &cryptd_tfm->base;
+	if (irq_fpu_usable() && (!in_atomic() ||
+				 !cryptd_aead_queued(cryptd_tfm)))
+		aead = cryptd_aead_child(cryptd_tfm);
+
+	aead_request_set_tfm(req, aead);
+
+	return crypto_aead_decrypt(req);
+}
+EXPORT_SYMBOL_GPL(cryptd_morus640_glue_decrypt);
+
+int cryptd_morus640_glue_init_tfm(struct crypto_aead *aead)
+{
+	struct cryptd_aead *cryptd_tfm;
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+	const char *name = crypto_aead_alg(aead)->base.cra_driver_name;
+	char internal_name[CRYPTO_MAX_ALG_NAME];
+
+	if (snprintf(internal_name, CRYPTO_MAX_ALG_NAME, "__%s", name)
+			>= CRYPTO_MAX_ALG_NAME)
+		return -ENAMETOOLONG;
+
+	cryptd_tfm = cryptd_alloc_aead(internal_name, CRYPTO_ALG_INTERNAL,
+				       CRYPTO_ALG_INTERNAL);
+	if (IS_ERR(cryptd_tfm))
+		return PTR_ERR(cryptd_tfm);
+
+	*ctx = cryptd_tfm;
+	crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cryptd_morus640_glue_init_tfm);
+
+void cryptd_morus640_glue_exit_tfm(struct crypto_aead *aead)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+
+	cryptd_free_aead(*ctx);
+}
+EXPORT_SYMBOL_GPL(cryptd_morus640_glue_exit_tfm);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("MORUS-640 AEAD mode -- glue for x86 optimizations");
diff --git a/arch/x86/crypto/salsa20-i586-asm_32.S b/arch/x86/crypto/salsa20-i586-asm_32.S
deleted file mode 100644
index 6014b7b9e52a..000000000000
--- a/arch/x86/crypto/salsa20-i586-asm_32.S
+++ /dev/null
@@ -1,938 +0,0 @@
-# Derived from:
-#	salsa20_pm.s version 20051229
-#	D. J. Bernstein
-#	Public domain.
-
-#include <linux/linkage.h>
-
-.text
-
-# enter salsa20_encrypt_bytes
-ENTRY(salsa20_encrypt_bytes)
-	mov	%esp,%eax
-	and	$31,%eax
-	add	$256,%eax
-	sub	%eax,%esp
-	# eax_stack = eax
-	movl	%eax,80(%esp)
-	# ebx_stack = ebx
-	movl	%ebx,84(%esp)
-	# esi_stack = esi
-	movl	%esi,88(%esp)
-	# edi_stack = edi
-	movl	%edi,92(%esp)
-	# ebp_stack = ebp
-	movl	%ebp,96(%esp)
-	# x = arg1
-	movl	4(%esp,%eax),%edx
-	# m = arg2
-	movl	8(%esp,%eax),%esi
-	# out = arg3
-	movl	12(%esp,%eax),%edi
-	# bytes = arg4
-	movl	16(%esp,%eax),%ebx
-	# bytes -= 0
-	sub	$0,%ebx
-	# goto done if unsigned<=
-	jbe	._done
-._start:
-	# in0 = *(uint32 *) (x + 0)
-	movl	0(%edx),%eax
-	# in1 = *(uint32 *) (x + 4)
-	movl	4(%edx),%ecx
-	# in2 = *(uint32 *) (x + 8)
-	movl	8(%edx),%ebp
-	# j0 = in0
-	movl	%eax,164(%esp)
-	# in3 = *(uint32 *) (x + 12)
-	movl	12(%edx),%eax
-	# j1 = in1
-	movl	%ecx,168(%esp)
-	# in4 = *(uint32 *) (x + 16)
-	movl	16(%edx),%ecx
-	# j2 = in2
-	movl	%ebp,172(%esp)
-	# in5 = *(uint32 *) (x + 20)
-	movl	20(%edx),%ebp
-	# j3 = in3
-	movl	%eax,176(%esp)
-	# in6 = *(uint32 *) (x + 24)
-	movl	24(%edx),%eax
-	# j4 = in4
-	movl	%ecx,180(%esp)
-	# in7 = *(uint32 *) (x + 28)
-	movl	28(%edx),%ecx
-	# j5 = in5
-	movl	%ebp,184(%esp)
-	# in8 = *(uint32 *) (x + 32)
-	movl	32(%edx),%ebp
-	# j6 = in6
-	movl	%eax,188(%esp)
-	# in9 = *(uint32 *) (x + 36)
-	movl	36(%edx),%eax
-	# j7 = in7
-	movl	%ecx,192(%esp)
-	# in10 = *(uint32 *) (x + 40)
-	movl	40(%edx),%ecx
-	# j8 = in8
-	movl	%ebp,196(%esp)
-	# in11 = *(uint32 *) (x + 44)
-	movl	44(%edx),%ebp
-	# j9 = in9
-	movl	%eax,200(%esp)
-	# in12 = *(uint32 *) (x + 48)
-	movl	48(%edx),%eax
-	# j10 = in10
-	movl	%ecx,204(%esp)
-	# in13 = *(uint32 *) (x + 52)
-	movl	52(%edx),%ecx
-	# j11 = in11
-	movl	%ebp,208(%esp)
-	# in14 = *(uint32 *) (x + 56)
-	movl	56(%edx),%ebp
-	# j12 = in12
-	movl	%eax,212(%esp)
-	# in15 = *(uint32 *) (x + 60)
-	movl	60(%edx),%eax
-	# j13 = in13
-	movl	%ecx,216(%esp)
-	# j14 = in14
-	movl	%ebp,220(%esp)
-	# j15 = in15
-	movl	%eax,224(%esp)
-	# x_backup = x
-	movl	%edx,64(%esp)
-._bytesatleast1:
-	#   bytes - 64
-	cmp	$64,%ebx
-	#   goto nocopy if unsigned>=
-	jae	._nocopy
-	#     ctarget = out
-	movl	%edi,228(%esp)
-	#     out = &tmp
-	leal	0(%esp),%edi
-	#     i = bytes
-	mov	%ebx,%ecx
-	#     while (i) { *out++ = *m++; --i }
-	rep	movsb
-	#     out = &tmp
-	leal	0(%esp),%edi
-	#     m = &tmp
-	leal	0(%esp),%esi
-._nocopy:
-	#   out_backup = out
-	movl	%edi,72(%esp)
-	#   m_backup = m
-	movl	%esi,68(%esp)
-	#   bytes_backup = bytes
-	movl	%ebx,76(%esp)
-	#   in0 = j0
-	movl	164(%esp),%eax
-	#   in1 = j1
-	movl	168(%esp),%ecx
-	#   in2 = j2
-	movl	172(%esp),%edx
-	#   in3 = j3
-	movl	176(%esp),%ebx
-	#   x0 = in0
-	movl	%eax,100(%esp)
-	#   x1 = in1
-	movl	%ecx,104(%esp)
-	#   x2 = in2
-	movl	%edx,108(%esp)
-	#   x3 = in3
-	movl	%ebx,112(%esp)
-	#   in4 = j4
-	movl	180(%esp),%eax
-	#   in5 = j5
-	movl	184(%esp),%ecx
-	#   in6 = j6
-	movl	188(%esp),%edx
-	#   in7 = j7
-	movl	192(%esp),%ebx
-	#   x4 = in4
-	movl	%eax,116(%esp)
-	#   x5 = in5
-	movl	%ecx,120(%esp)
-	#   x6 = in6
-	movl	%edx,124(%esp)
-	#   x7 = in7
-	movl	%ebx,128(%esp)
-	#   in8 = j8
-	movl	196(%esp),%eax
-	#   in9 = j9
-	movl	200(%esp),%ecx
-	#   in10 = j10
-	movl	204(%esp),%edx
-	#   in11 = j11
-	movl	208(%esp),%ebx
-	#   x8 = in8
-	movl	%eax,132(%esp)
-	#   x9 = in9
-	movl	%ecx,136(%esp)
-	#   x10 = in10
-	movl	%edx,140(%esp)
-	#   x11 = in11
-	movl	%ebx,144(%esp)
-	#   in12 = j12
-	movl	212(%esp),%eax
-	#   in13 = j13
-	movl	216(%esp),%ecx
-	#   in14 = j14
-	movl	220(%esp),%edx
-	#   in15 = j15
-	movl	224(%esp),%ebx
-	#   x12 = in12
-	movl	%eax,148(%esp)
-	#   x13 = in13
-	movl	%ecx,152(%esp)
-	#   x14 = in14
-	movl	%edx,156(%esp)
-	#   x15 = in15
-	movl	%ebx,160(%esp)
-	#   i = 20
-	mov	$20,%ebp
-	# p = x0
-	movl	100(%esp),%eax
-	# s = x5
-	movl	120(%esp),%ecx
-	# t = x10
-	movl	140(%esp),%edx
-	# w = x15
-	movl	160(%esp),%ebx
-._mainloop:
-	# x0 = p
-	movl	%eax,100(%esp)
-	# 				x10 = t
-	movl	%edx,140(%esp)
-	# p += x12
-	addl	148(%esp),%eax
-	# 		x5 = s
-	movl	%ecx,120(%esp)
-	# 				t += x6
-	addl	124(%esp),%edx
-	# 						x15 = w
-	movl	%ebx,160(%esp)
-	# 		r = x1
-	movl	104(%esp),%esi
-	# 		r += s
-	add	%ecx,%esi
-	# 						v = x11
-	movl	144(%esp),%edi
-	# 						v += w
-	add	%ebx,%edi
-	# p <<<= 7
-	rol	$7,%eax
-	# p ^= x4
-	xorl	116(%esp),%eax
-	# 				t <<<= 7
-	rol	$7,%edx
-	# 				t ^= x14
-	xorl	156(%esp),%edx
-	# 		r <<<= 7
-	rol	$7,%esi
-	# 		r ^= x9
-	xorl	136(%esp),%esi
-	# 						v <<<= 7
-	rol	$7,%edi
-	# 						v ^= x3
-	xorl	112(%esp),%edi
-	# x4 = p
-	movl	%eax,116(%esp)
-	# 				x14 = t
-	movl	%edx,156(%esp)
-	# p += x0
-	addl	100(%esp),%eax
-	# 		x9 = r
-	movl	%esi,136(%esp)
-	# 				t += x10
-	addl	140(%esp),%edx
-	# 						x3 = v
-	movl	%edi,112(%esp)
-	# p <<<= 9
-	rol	$9,%eax
-	# p ^= x8
-	xorl	132(%esp),%eax
-	# 				t <<<= 9
-	rol	$9,%edx
-	# 				t ^= x2
-	xorl	108(%esp),%edx
-	# 		s += r
-	add	%esi,%ecx
-	# 		s <<<= 9
-	rol	$9,%ecx
-	# 		s ^= x13
-	xorl	152(%esp),%ecx
-	# 						w += v
-	add	%edi,%ebx
-	# 						w <<<= 9
-	rol	$9,%ebx
-	# 						w ^= x7
-	xorl	128(%esp),%ebx
-	# x8 = p
-	movl	%eax,132(%esp)
-	# 				x2 = t
-	movl	%edx,108(%esp)
-	# p += x4
-	addl	116(%esp),%eax
-	# 		x13 = s
-	movl	%ecx,152(%esp)
-	# 				t += x14
-	addl	156(%esp),%edx
-	# 						x7 = w
-	movl	%ebx,128(%esp)
-	# p <<<= 13
-	rol	$13,%eax
-	# p ^= x12
-	xorl	148(%esp),%eax
-	# 				t <<<= 13
-	rol	$13,%edx
-	# 				t ^= x6
-	xorl	124(%esp),%edx
-	# 		r += s
-	add	%ecx,%esi
-	# 		r <<<= 13
-	rol	$13,%esi
-	# 		r ^= x1
-	xorl	104(%esp),%esi
-	# 						v += w
-	add	%ebx,%edi
-	# 						v <<<= 13
-	rol	$13,%edi
-	# 						v ^= x11
-	xorl	144(%esp),%edi
-	# x12 = p
-	movl	%eax,148(%esp)
-	# 				x6 = t
-	movl	%edx,124(%esp)
-	# p += x8
-	addl	132(%esp),%eax
-	# 		x1 = r
-	movl	%esi,104(%esp)
-	# 				t += x2
-	addl	108(%esp),%edx
-	# 						x11 = v
-	movl	%edi,144(%esp)
-	# p <<<= 18
-	rol	$18,%eax
-	# p ^= x0
-	xorl	100(%esp),%eax
-	# 				t <<<= 18
-	rol	$18,%edx
-	# 				t ^= x10
-	xorl	140(%esp),%edx
-	# 		s += r
-	add	%esi,%ecx
-	# 		s <<<= 18
-	rol	$18,%ecx
-	# 		s ^= x5
-	xorl	120(%esp),%ecx
-	# 						w += v
-	add	%edi,%ebx
-	# 						w <<<= 18
-	rol	$18,%ebx
-	# 						w ^= x15
-	xorl	160(%esp),%ebx
-	# x0 = p
-	movl	%eax,100(%esp)
-	# 				x10 = t
-	movl	%edx,140(%esp)
-	# p += x3
-	addl	112(%esp),%eax
-	# p <<<= 7
-	rol	$7,%eax
-	# 		x5 = s
-	movl	%ecx,120(%esp)
-	# 				t += x9
-	addl	136(%esp),%edx
-	# 						x15 = w
-	movl	%ebx,160(%esp)
-	# 		r = x4
-	movl	116(%esp),%esi
-	# 		r += s
-	add	%ecx,%esi
-	# 						v = x14
-	movl	156(%esp),%edi
-	# 						v += w
-	add	%ebx,%edi
-	# p ^= x1
-	xorl	104(%esp),%eax
-	# 				t <<<= 7
-	rol	$7,%edx
-	# 				t ^= x11
-	xorl	144(%esp),%edx
-	# 		r <<<= 7
-	rol	$7,%esi
-	# 		r ^= x6
-	xorl	124(%esp),%esi
-	# 						v <<<= 7
-	rol	$7,%edi
-	# 						v ^= x12
-	xorl	148(%esp),%edi
-	# x1 = p
-	movl	%eax,104(%esp)
-	# 				x11 = t
-	movl	%edx,144(%esp)
-	# p += x0
-	addl	100(%esp),%eax
-	# 		x6 = r
-	movl	%esi,124(%esp)
-	# 				t += x10
-	addl	140(%esp),%edx
-	# 						x12 = v
-	movl	%edi,148(%esp)
-	# p <<<= 9
-	rol	$9,%eax
-	# p ^= x2
-	xorl	108(%esp),%eax
-	# 				t <<<= 9
-	rol	$9,%edx
-	# 				t ^= x8
-	xorl	132(%esp),%edx
-	# 		s += r
-	add	%esi,%ecx
-	# 		s <<<= 9
-	rol	$9,%ecx
-	# 		s ^= x7
-	xorl	128(%esp),%ecx
-	# 						w += v
-	add	%edi,%ebx
-	# 						w <<<= 9
-	rol	$9,%ebx
-	# 						w ^= x13
-	xorl	152(%esp),%ebx
-	# x2 = p
-	movl	%eax,108(%esp)
-	# 				x8 = t
-	movl	%edx,132(%esp)
-	# p += x1
-	addl	104(%esp),%eax
-	# 		x7 = s
-	movl	%ecx,128(%esp)
-	# 				t += x11
-	addl	144(%esp),%edx
-	# 						x13 = w
-	movl	%ebx,152(%esp)
-	# p <<<= 13
-	rol	$13,%eax
-	# p ^= x3
-	xorl	112(%esp),%eax
-	# 				t <<<= 13
-	rol	$13,%edx
-	# 				t ^= x9
-	xorl	136(%esp),%edx
-	# 		r += s
-	add	%ecx,%esi
-	# 		r <<<= 13
-	rol	$13,%esi
-	# 		r ^= x4
-	xorl	116(%esp),%esi
-	# 						v += w
-	add	%ebx,%edi
-	# 						v <<<= 13
-	rol	$13,%edi
-	# 						v ^= x14
-	xorl	156(%esp),%edi
-	# x3 = p
-	movl	%eax,112(%esp)
-	# 				x9 = t
-	movl	%edx,136(%esp)
-	# p += x2
-	addl	108(%esp),%eax
-	# 		x4 = r
-	movl	%esi,116(%esp)
-	# 				t += x8
-	addl	132(%esp),%edx
-	# 						x14 = v
-	movl	%edi,156(%esp)
-	# p <<<= 18
-	rol	$18,%eax
-	# p ^= x0
-	xorl	100(%esp),%eax
-	# 				t <<<= 18
-	rol	$18,%edx
-	# 				t ^= x10
-	xorl	140(%esp),%edx
-	# 		s += r
-	add	%esi,%ecx
-	# 		s <<<= 18
-	rol	$18,%ecx
-	# 		s ^= x5
-	xorl	120(%esp),%ecx
-	# 						w += v
-	add	%edi,%ebx
-	# 						w <<<= 18
-	rol	$18,%ebx
-	# 						w ^= x15
-	xorl	160(%esp),%ebx
-	# x0 = p
-	movl	%eax,100(%esp)
-	# 				x10 = t
-	movl	%edx,140(%esp)
-	# p += x12
-	addl	148(%esp),%eax
-	# 		x5 = s
-	movl	%ecx,120(%esp)
-	# 				t += x6
-	addl	124(%esp),%edx
-	# 						x15 = w
-	movl	%ebx,160(%esp)
-	# 		r = x1
-	movl	104(%esp),%esi
-	# 		r += s
-	add	%ecx,%esi
-	# 						v = x11
-	movl	144(%esp),%edi
-	# 						v += w
-	add	%ebx,%edi
-	# p <<<= 7
-	rol	$7,%eax
-	# p ^= x4
-	xorl	116(%esp),%eax
-	# 				t <<<= 7
-	rol	$7,%edx
-	# 				t ^= x14
-	xorl	156(%esp),%edx
-	# 		r <<<= 7
-	rol	$7,%esi
-	# 		r ^= x9
-	xorl	136(%esp),%esi
-	# 						v <<<= 7
-	rol	$7,%edi
-	# 						v ^= x3
-	xorl	112(%esp),%edi
-	# x4 = p
-	movl	%eax,116(%esp)
-	# 				x14 = t
-	movl	%edx,156(%esp)
-	# p += x0
-	addl	100(%esp),%eax
-	# 		x9 = r
-	movl	%esi,136(%esp)
-	# 				t += x10
-	addl	140(%esp),%edx
-	# 						x3 = v
-	movl	%edi,112(%esp)
-	# p <<<= 9
-	rol	$9,%eax
-	# p ^= x8
-	xorl	132(%esp),%eax
-	# 				t <<<= 9
-	rol	$9,%edx
-	# 				t ^= x2
-	xorl	108(%esp),%edx
-	# 		s += r
-	add	%esi,%ecx
-	# 		s <<<= 9
-	rol	$9,%ecx
-	# 		s ^= x13
-	xorl	152(%esp),%ecx
-	# 						w += v
-	add	%edi,%ebx
-	# 						w <<<= 9
-	rol	$9,%ebx
-	# 						w ^= x7
-	xorl	128(%esp),%ebx
-	# x8 = p
-	movl	%eax,132(%esp)
-	# 				x2 = t
-	movl	%edx,108(%esp)
-	# p += x4
-	addl	116(%esp),%eax
-	# 		x13 = s
-	movl	%ecx,152(%esp)
-	# 				t += x14
-	addl	156(%esp),%edx
-	# 						x7 = w
-	movl	%ebx,128(%esp)
-	# p <<<= 13
-	rol	$13,%eax
-	# p ^= x12
-	xorl	148(%esp),%eax
-	# 				t <<<= 13
-	rol	$13,%edx
-	# 				t ^= x6
-	xorl	124(%esp),%edx
-	# 		r += s
-	add	%ecx,%esi
-	# 		r <<<= 13
-	rol	$13,%esi
-	# 		r ^= x1
-	xorl	104(%esp),%esi
-	# 						v += w
-	add	%ebx,%edi
-	# 						v <<<= 13
-	rol	$13,%edi
-	# 						v ^= x11
-	xorl	144(%esp),%edi
-	# x12 = p
-	movl	%eax,148(%esp)
-	# 				x6 = t
-	movl	%edx,124(%esp)
-	# p += x8
-	addl	132(%esp),%eax
-	# 		x1 = r
-	movl	%esi,104(%esp)
-	# 				t += x2
-	addl	108(%esp),%edx
-	# 						x11 = v
-	movl	%edi,144(%esp)
-	# p <<<= 18
-	rol	$18,%eax
-	# p ^= x0
-	xorl	100(%esp),%eax
-	# 				t <<<= 18
-	rol	$18,%edx
-	# 				t ^= x10
-	xorl	140(%esp),%edx
-	# 		s += r
-	add	%esi,%ecx
-	# 		s <<<= 18
-	rol	$18,%ecx
-	# 		s ^= x5
-	xorl	120(%esp),%ecx
-	# 						w += v
-	add	%edi,%ebx
-	# 						w <<<= 18
-	rol	$18,%ebx
-	# 						w ^= x15
-	xorl	160(%esp),%ebx
-	# x0 = p
-	movl	%eax,100(%esp)
-	# 				x10 = t
-	movl	%edx,140(%esp)
-	# p += x3
-	addl	112(%esp),%eax
-	# p <<<= 7
-	rol	$7,%eax
-	# 		x5 = s
-	movl	%ecx,120(%esp)
-	# 				t += x9
-	addl	136(%esp),%edx
-	# 						x15 = w
-	movl	%ebx,160(%esp)
-	# 		r = x4
-	movl	116(%esp),%esi
-	# 		r += s
-	add	%ecx,%esi
-	# 						v = x14
-	movl	156(%esp),%edi
-	# 						v += w
-	add	%ebx,%edi
-	# p ^= x1
-	xorl	104(%esp),%eax
-	# 				t <<<= 7
-	rol	$7,%edx
-	# 				t ^= x11
-	xorl	144(%esp),%edx
-	# 		r <<<= 7
-	rol	$7,%esi
-	# 		r ^= x6
-	xorl	124(%esp),%esi
-	# 						v <<<= 7
-	rol	$7,%edi
-	# 						v ^= x12
-	xorl	148(%esp),%edi
-	# x1 = p
-	movl	%eax,104(%esp)
-	# 				x11 = t
-	movl	%edx,144(%esp)
-	# p += x0
-	addl	100(%esp),%eax
-	# 		x6 = r
-	movl	%esi,124(%esp)
-	# 				t += x10
-	addl	140(%esp),%edx
-	# 						x12 = v
-	movl	%edi,148(%esp)
-	# p <<<= 9
-	rol	$9,%eax
-	# p ^= x2
-	xorl	108(%esp),%eax
-	# 				t <<<= 9
-	rol	$9,%edx
-	# 				t ^= x8
-	xorl	132(%esp),%edx
-	# 		s += r
-	add	%esi,%ecx
-	# 		s <<<= 9
-	rol	$9,%ecx
-	# 		s ^= x7
-	xorl	128(%esp),%ecx
-	# 						w += v
-	add	%edi,%ebx
-	# 						w <<<= 9
-	rol	$9,%ebx
-	# 						w ^= x13
-	xorl	152(%esp),%ebx
-	# x2 = p
-	movl	%eax,108(%esp)
-	# 				x8 = t
-	movl	%edx,132(%esp)
-	# p += x1
-	addl	104(%esp),%eax
-	# 		x7 = s
-	movl	%ecx,128(%esp)
-	# 				t += x11
-	addl	144(%esp),%edx
-	# 						x13 = w
-	movl	%ebx,152(%esp)
-	# p <<<= 13
-	rol	$13,%eax
-	# p ^= x3
-	xorl	112(%esp),%eax
-	# 				t <<<= 13
-	rol	$13,%edx
-	# 				t ^= x9
-	xorl	136(%esp),%edx
-	# 		r += s
-	add	%ecx,%esi
-	# 		r <<<= 13
-	rol	$13,%esi
-	# 		r ^= x4
-	xorl	116(%esp),%esi
-	# 						v += w
-	add	%ebx,%edi
-	# 						v <<<= 13
-	rol	$13,%edi
-	# 						v ^= x14
-	xorl	156(%esp),%edi
-	# x3 = p
-	movl	%eax,112(%esp)
-	# 				x9 = t
-	movl	%edx,136(%esp)
-	# p += x2
-	addl	108(%esp),%eax
-	# 		x4 = r
-	movl	%esi,116(%esp)
-	# 				t += x8
-	addl	132(%esp),%edx
-	# 						x14 = v
-	movl	%edi,156(%esp)
-	# p <<<= 18
-	rol	$18,%eax
-	# p ^= x0
-	xorl	100(%esp),%eax
-	# 				t <<<= 18
-	rol	$18,%edx
-	# 				t ^= x10
-	xorl	140(%esp),%edx
-	# 		s += r
-	add	%esi,%ecx
-	# 		s <<<= 18
-	rol	$18,%ecx
-	# 		s ^= x5
-	xorl	120(%esp),%ecx
-	# 						w += v
-	add	%edi,%ebx
-	# 						w <<<= 18
-	rol	$18,%ebx
-	# 						w ^= x15
-	xorl	160(%esp),%ebx
-	# i -= 4
-	sub	$4,%ebp
-	# goto mainloop if unsigned >
-	ja	._mainloop
-	# x0 = p
-	movl	%eax,100(%esp)
-	# x5 = s
-	movl	%ecx,120(%esp)
-	# x10 = t
-	movl	%edx,140(%esp)
-	# x15 = w
-	movl	%ebx,160(%esp)
-	#   out = out_backup
-	movl	72(%esp),%edi
-	#   m = m_backup
-	movl	68(%esp),%esi
-	#   in0 = x0
-	movl	100(%esp),%eax
-	#   in1 = x1
-	movl	104(%esp),%ecx
-	#   in0 += j0
-	addl	164(%esp),%eax
-	#   in1 += j1
-	addl	168(%esp),%ecx
-	#   in0 ^= *(uint32 *) (m + 0)
-	xorl	0(%esi),%eax
-	#   in1 ^= *(uint32 *) (m + 4)
-	xorl	4(%esi),%ecx
-	#   *(uint32 *) (out + 0) = in0
-	movl	%eax,0(%edi)
-	#   *(uint32 *) (out + 4) = in1
-	movl	%ecx,4(%edi)
-	#   in2 = x2
-	movl	108(%esp),%eax
-	#   in3 = x3
-	movl	112(%esp),%ecx
-	#   in2 += j2
-	addl	172(%esp),%eax
-	#   in3 += j3
-	addl	176(%esp),%ecx
-	#   in2 ^= *(uint32 *) (m + 8)
-	xorl	8(%esi),%eax
-	#   in3 ^= *(uint32 *) (m + 12)
-	xorl	12(%esi),%ecx
-	#   *(uint32 *) (out + 8) = in2
-	movl	%eax,8(%edi)
-	#   *(uint32 *) (out + 12) = in3
-	movl	%ecx,12(%edi)
-	#   in4 = x4
-	movl	116(%esp),%eax
-	#   in5 = x5
-	movl	120(%esp),%ecx
-	#   in4 += j4
-	addl	180(%esp),%eax
-	#   in5 += j5
-	addl	184(%esp),%ecx
-	#   in4 ^= *(uint32 *) (m + 16)
-	xorl	16(%esi),%eax
-	#   in5 ^= *(uint32 *) (m + 20)
-	xorl	20(%esi),%ecx
-	#   *(uint32 *) (out + 16) = in4
-	movl	%eax,16(%edi)
-	#   *(uint32 *) (out + 20) = in5
-	movl	%ecx,20(%edi)
-	#   in6 = x6
-	movl	124(%esp),%eax
-	#   in7 = x7
-	movl	128(%esp),%ecx
-	#   in6 += j6
-	addl	188(%esp),%eax
-	#   in7 += j7
-	addl	192(%esp),%ecx
-	#   in6 ^= *(uint32 *) (m + 24)
-	xorl	24(%esi),%eax
-	#   in7 ^= *(uint32 *) (m + 28)
-	xorl	28(%esi),%ecx
-	#   *(uint32 *) (out + 24) = in6
-	movl	%eax,24(%edi)
-	#   *(uint32 *) (out + 28) = in7
-	movl	%ecx,28(%edi)
-	#   in8 = x8
-	movl	132(%esp),%eax
-	#   in9 = x9
-	movl	136(%esp),%ecx
-	#   in8 += j8
-	addl	196(%esp),%eax
-	#   in9 += j9
-	addl	200(%esp),%ecx
-	#   in8 ^= *(uint32 *) (m + 32)
-	xorl	32(%esi),%eax
-	#   in9 ^= *(uint32 *) (m + 36)
-	xorl	36(%esi),%ecx
-	#   *(uint32 *) (out + 32) = in8
-	movl	%eax,32(%edi)
-	#   *(uint32 *) (out + 36) = in9
-	movl	%ecx,36(%edi)
-	#   in10 = x10
-	movl	140(%esp),%eax
-	#   in11 = x11
-	movl	144(%esp),%ecx
-	#   in10 += j10
-	addl	204(%esp),%eax
-	#   in11 += j11
-	addl	208(%esp),%ecx
-	#   in10 ^= *(uint32 *) (m + 40)
-	xorl	40(%esi),%eax
-	#   in11 ^= *(uint32 *) (m + 44)
-	xorl	44(%esi),%ecx
-	#   *(uint32 *) (out + 40) = in10
-	movl	%eax,40(%edi)
-	#   *(uint32 *) (out + 44) = in11
-	movl	%ecx,44(%edi)
-	#   in12 = x12
-	movl	148(%esp),%eax
-	#   in13 = x13
-	movl	152(%esp),%ecx
-	#   in12 += j12
-	addl	212(%esp),%eax
-	#   in13 += j13
-	addl	216(%esp),%ecx
-	#   in12 ^= *(uint32 *) (m + 48)
-	xorl	48(%esi),%eax
-	#   in13 ^= *(uint32 *) (m + 52)
-	xorl	52(%esi),%ecx
-	#   *(uint32 *) (out + 48) = in12
-	movl	%eax,48(%edi)
-	#   *(uint32 *) (out + 52) = in13
-	movl	%ecx,52(%edi)
-	#   in14 = x14
-	movl	156(%esp),%eax
-	#   in15 = x15
-	movl	160(%esp),%ecx
-	#   in14 += j14
-	addl	220(%esp),%eax
-	#   in15 += j15
-	addl	224(%esp),%ecx
-	#   in14 ^= *(uint32 *) (m + 56)
-	xorl	56(%esi),%eax
-	#   in15 ^= *(uint32 *) (m + 60)
-	xorl	60(%esi),%ecx
-	#   *(uint32 *) (out + 56) = in14
-	movl	%eax,56(%edi)
-	#   *(uint32 *) (out + 60) = in15
-	movl	%ecx,60(%edi)
-	#   bytes = bytes_backup
-	movl	76(%esp),%ebx
-	#   in8 = j8
-	movl	196(%esp),%eax
-	#   in9 = j9
-	movl	200(%esp),%ecx
-	#   in8 += 1
-	add	$1,%eax
-	#   in9 += 0 + carry
-	adc	$0,%ecx
-	#   j8 = in8
-	movl	%eax,196(%esp)
-	#   j9 = in9
-	movl	%ecx,200(%esp)
-	#   bytes - 64
-	cmp	$64,%ebx
-	#   goto bytesatleast65 if unsigned>
-	ja	._bytesatleast65
-	#     goto bytesatleast64 if unsigned>=
-	jae	._bytesatleast64
-	#       m = out
-	mov	%edi,%esi
-	#       out = ctarget
-	movl	228(%esp),%edi
-	#       i = bytes
-	mov	%ebx,%ecx
-	#       while (i) { *out++ = *m++; --i }
-	rep	movsb
-._bytesatleast64:
-	#     x = x_backup
-	movl	64(%esp),%eax
-	#     in8 = j8
-	movl	196(%esp),%ecx
-	#     in9 = j9
-	movl	200(%esp),%edx
-	#     *(uint32 *) (x + 32) = in8
-	movl	%ecx,32(%eax)
-	#     *(uint32 *) (x + 36) = in9
-	movl	%edx,36(%eax)
-._done:
-	#     eax = eax_stack
-	movl	80(%esp),%eax
-	#     ebx = ebx_stack
-	movl	84(%esp),%ebx
-	#     esi = esi_stack
-	movl	88(%esp),%esi
-	#     edi = edi_stack
-	movl	92(%esp),%edi
-	#     ebp = ebp_stack
-	movl	96(%esp),%ebp
-	#     leave
-	add	%eax,%esp
-	ret
-._bytesatleast65:
-	#   bytes -= 64
-	sub	$64,%ebx
-	#   out += 64
-	add	$64,%edi
-	#   m += 64
-	add	$64,%esi
-	# goto bytesatleast1
-	jmp	._bytesatleast1
-ENDPROC(salsa20_encrypt_bytes)
diff --git a/arch/x86/crypto/salsa20-x86_64-asm_64.S b/arch/x86/crypto/salsa20-x86_64-asm_64.S
deleted file mode 100644
index 03a4918f41ee..000000000000
--- a/arch/x86/crypto/salsa20-x86_64-asm_64.S
+++ /dev/null
@@ -1,805 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#include <linux/linkage.h>
-
-# enter salsa20_encrypt_bytes
-ENTRY(salsa20_encrypt_bytes)
-	mov	%rsp,%r11
-	and	$31,%r11
-	add	$256,%r11
-	sub	%r11,%rsp
-	# x = arg1
-	mov	%rdi,%r8
-	# m = arg2
-	mov	%rsi,%rsi
-	# out = arg3
-	mov	%rdx,%rdi
-	# bytes = arg4
-	mov	%rcx,%rdx
-	#               unsigned>? bytes - 0
-	cmp	$0,%rdx
-	# comment:fp stack unchanged by jump
-	# goto done if !unsigned>
-	jbe	._done
-	# comment:fp stack unchanged by fallthrough
-# start:
-._start:
-	# r11_stack = r11
-	movq	%r11,0(%rsp)
-	# r12_stack = r12
-	movq	%r12,8(%rsp)
-	# r13_stack = r13
-	movq	%r13,16(%rsp)
-	# r14_stack = r14
-	movq	%r14,24(%rsp)
-	# r15_stack = r15
-	movq	%r15,32(%rsp)
-	# rbx_stack = rbx
-	movq	%rbx,40(%rsp)
-	# rbp_stack = rbp
-	movq	%rbp,48(%rsp)
-	# in0 = *(uint64 *) (x + 0)
-	movq	0(%r8),%rcx
-	# in2 = *(uint64 *) (x + 8)
-	movq	8(%r8),%r9
-	# in4 = *(uint64 *) (x + 16)
-	movq	16(%r8),%rax
-	# in6 = *(uint64 *) (x + 24)
-	movq	24(%r8),%r10
-	# in8 = *(uint64 *) (x + 32)
-	movq	32(%r8),%r11
-	# in10 = *(uint64 *) (x + 40)
-	movq	40(%r8),%r12
-	# in12 = *(uint64 *) (x + 48)
-	movq	48(%r8),%r13
-	# in14 = *(uint64 *) (x + 56)
-	movq	56(%r8),%r14
-	# j0 = in0
-	movq	%rcx,56(%rsp)
-	# j2 = in2
-	movq	%r9,64(%rsp)
-	# j4 = in4
-	movq	%rax,72(%rsp)
-	# j6 = in6
-	movq	%r10,80(%rsp)
-	# j8 = in8
-	movq	%r11,88(%rsp)
-	# j10 = in10
-	movq	%r12,96(%rsp)
-	# j12 = in12
-	movq	%r13,104(%rsp)
-	# j14 = in14
-	movq	%r14,112(%rsp)
-	# x_backup = x
-	movq	%r8,120(%rsp)
-# bytesatleast1:
-._bytesatleast1:
-	#                   unsigned<? bytes - 64
-	cmp	$64,%rdx
-	# comment:fp stack unchanged by jump
-	#   goto nocopy if !unsigned<
-	jae	._nocopy
-	#     ctarget = out
-	movq	%rdi,128(%rsp)
-	#     out = &tmp
-	leaq	192(%rsp),%rdi
-	#     i = bytes
-	mov	%rdx,%rcx
-	#     while (i) { *out++ = *m++; --i }
-	rep	movsb
-	#     out = &tmp
-	leaq	192(%rsp),%rdi
-	#     m = &tmp
-	leaq	192(%rsp),%rsi
-	# comment:fp stack unchanged by fallthrough
-#   nocopy:
-._nocopy:
-	#   out_backup = out
-	movq	%rdi,136(%rsp)
-	#   m_backup = m
-	movq	%rsi,144(%rsp)
-	#   bytes_backup = bytes
-	movq	%rdx,152(%rsp)
-	#   x1 = j0
-	movq	56(%rsp),%rdi
-	#   x0 = x1
-	mov	%rdi,%rdx
-	#   (uint64) x1 >>= 32
-	shr	$32,%rdi
-	#   		x3 = j2
-	movq	64(%rsp),%rsi
-	#   		x2 = x3
-	mov	%rsi,%rcx
-	#   		(uint64) x3 >>= 32
-	shr	$32,%rsi
-	#   x5 = j4
-	movq	72(%rsp),%r8
-	#   x4 = x5
-	mov	%r8,%r9
-	#   (uint64) x5 >>= 32
-	shr	$32,%r8
-	#   x5_stack = x5
-	movq	%r8,160(%rsp)
-	#   		x7 = j6
-	movq	80(%rsp),%r8
-	#   		x6 = x7
-	mov	%r8,%rax
-	#   		(uint64) x7 >>= 32
-	shr	$32,%r8
-	#   x9 = j8
-	movq	88(%rsp),%r10
-	#   x8 = x9
-	mov	%r10,%r11
-	#   (uint64) x9 >>= 32
-	shr	$32,%r10
-	#   		x11 = j10
-	movq	96(%rsp),%r12
-	#   		x10 = x11
-	mov	%r12,%r13
-	#   		x10_stack = x10
-	movq	%r13,168(%rsp)
-	#   		(uint64) x11 >>= 32
-	shr	$32,%r12
-	#   x13 = j12
-	movq	104(%rsp),%r13
-	#   x12 = x13
-	mov	%r13,%r14
-	#   (uint64) x13 >>= 32
-	shr	$32,%r13
-	#   		x15 = j14
-	movq	112(%rsp),%r15
-	#   		x14 = x15
-	mov	%r15,%rbx
-	#   		(uint64) x15 >>= 32
-	shr	$32,%r15
-	#   		x15_stack = x15
-	movq	%r15,176(%rsp)
-	#   i = 20
-	mov	$20,%r15
-#   mainloop:
-._mainloop:
-	#   i_backup = i
-	movq	%r15,184(%rsp)
-	# 		x5 = x5_stack
-	movq	160(%rsp),%r15
-	# a = x12 + x0
-	lea	(%r14,%rdx),%rbp
-	# (uint32) a <<<= 7
-	rol	$7,%ebp
-	# x4 ^= a
-	xor	%rbp,%r9
-	# 		b = x1 + x5
-	lea	(%rdi,%r15),%rbp
-	# 		(uint32) b <<<= 7
-	rol	$7,%ebp
-	# 		x9 ^= b
-	xor	%rbp,%r10
-	# a = x0 + x4
-	lea	(%rdx,%r9),%rbp
-	# (uint32) a <<<= 9
-	rol	$9,%ebp
-	# x8 ^= a
-	xor	%rbp,%r11
-	# 		b = x5 + x9
-	lea	(%r15,%r10),%rbp
-	# 		(uint32) b <<<= 9
-	rol	$9,%ebp
-	# 		x13 ^= b
-	xor	%rbp,%r13
-	# a = x4 + x8
-	lea	(%r9,%r11),%rbp
-	# (uint32) a <<<= 13
-	rol	$13,%ebp
-	# x12 ^= a
-	xor	%rbp,%r14
-	# 		b = x9 + x13
-	lea	(%r10,%r13),%rbp
-	# 		(uint32) b <<<= 13
-	rol	$13,%ebp
-	# 		x1 ^= b
-	xor	%rbp,%rdi
-	# a = x8 + x12
-	lea	(%r11,%r14),%rbp
-	# (uint32) a <<<= 18
-	rol	$18,%ebp
-	# x0 ^= a
-	xor	%rbp,%rdx
-	# 		b = x13 + x1
-	lea	(%r13,%rdi),%rbp
-	# 		(uint32) b <<<= 18
-	rol	$18,%ebp
-	# 		x5 ^= b
-	xor	%rbp,%r15
-	# 				x10 = x10_stack
-	movq	168(%rsp),%rbp
-	# 		x5_stack = x5
-	movq	%r15,160(%rsp)
-	# 				c = x6 + x10
-	lea	(%rax,%rbp),%r15
-	# 				(uint32) c <<<= 7
-	rol	$7,%r15d
-	# 				x14 ^= c
-	xor	%r15,%rbx
-	# 				c = x10 + x14
-	lea	(%rbp,%rbx),%r15
-	# 				(uint32) c <<<= 9
-	rol	$9,%r15d
-	# 				x2 ^= c
-	xor	%r15,%rcx
-	# 				c = x14 + x2
-	lea	(%rbx,%rcx),%r15
-	# 				(uint32) c <<<= 13
-	rol	$13,%r15d
-	# 				x6 ^= c
-	xor	%r15,%rax
-	# 				c = x2 + x6
-	lea	(%rcx,%rax),%r15
-	# 				(uint32) c <<<= 18
-	rol	$18,%r15d
-	# 				x10 ^= c
-	xor	%r15,%rbp
-	# 						x15 = x15_stack
-	movq	176(%rsp),%r15
-	# 				x10_stack = x10
-	movq	%rbp,168(%rsp)
-	# 						d = x11 + x15
-	lea	(%r12,%r15),%rbp
-	# 						(uint32) d <<<= 7
-	rol	$7,%ebp
-	# 						x3 ^= d
-	xor	%rbp,%rsi
-	# 						d = x15 + x3
-	lea	(%r15,%rsi),%rbp
-	# 						(uint32) d <<<= 9
-	rol	$9,%ebp
-	# 						x7 ^= d
-	xor	%rbp,%r8
-	# 						d = x3 + x7
-	lea	(%rsi,%r8),%rbp
-	# 						(uint32) d <<<= 13
-	rol	$13,%ebp
-	# 						x11 ^= d
-	xor	%rbp,%r12
-	# 						d = x7 + x11
-	lea	(%r8,%r12),%rbp
-	# 						(uint32) d <<<= 18
-	rol	$18,%ebp
-	# 						x15 ^= d
-	xor	%rbp,%r15
-	# 						x15_stack = x15
-	movq	%r15,176(%rsp)
-	# 		x5 = x5_stack
-	movq	160(%rsp),%r15
-	# a = x3 + x0
-	lea	(%rsi,%rdx),%rbp
-	# (uint32) a <<<= 7
-	rol	$7,%ebp
-	# x1 ^= a
-	xor	%rbp,%rdi
-	# 		b = x4 + x5
-	lea	(%r9,%r15),%rbp
-	# 		(uint32) b <<<= 7
-	rol	$7,%ebp
-	# 		x6 ^= b
-	xor	%rbp,%rax
-	# a = x0 + x1
-	lea	(%rdx,%rdi),%rbp
-	# (uint32) a <<<= 9
-	rol	$9,%ebp
-	# x2 ^= a
-	xor	%rbp,%rcx
-	# 		b = x5 + x6
-	lea	(%r15,%rax),%rbp
-	# 		(uint32) b <<<= 9
-	rol	$9,%ebp
-	# 		x7 ^= b
-	xor	%rbp,%r8
-	# a = x1 + x2
-	lea	(%rdi,%rcx),%rbp
-	# (uint32) a <<<= 13
-	rol	$13,%ebp
-	# x3 ^= a
-	xor	%rbp,%rsi
-	# 		b = x6 + x7
-	lea	(%rax,%r8),%rbp
-	# 		(uint32) b <<<= 13
-	rol	$13,%ebp
-	# 		x4 ^= b
-	xor	%rbp,%r9
-	# a = x2 + x3
-	lea	(%rcx,%rsi),%rbp
-	# (uint32) a <<<= 18
-	rol	$18,%ebp
-	# x0 ^= a
-	xor	%rbp,%rdx
-	# 		b = x7 + x4
-	lea	(%r8,%r9),%rbp
-	# 		(uint32) b <<<= 18
-	rol	$18,%ebp
-	# 		x5 ^= b
-	xor	%rbp,%r15
-	# 				x10 = x10_stack
-	movq	168(%rsp),%rbp
-	# 		x5_stack = x5
-	movq	%r15,160(%rsp)
-	# 				c = x9 + x10
-	lea	(%r10,%rbp),%r15
-	# 				(uint32) c <<<= 7
-	rol	$7,%r15d
-	# 				x11 ^= c
-	xor	%r15,%r12
-	# 				c = x10 + x11
-	lea	(%rbp,%r12),%r15
-	# 				(uint32) c <<<= 9
-	rol	$9,%r15d
-	# 				x8 ^= c
-	xor	%r15,%r11
-	# 				c = x11 + x8
-	lea	(%r12,%r11),%r15
-	# 				(uint32) c <<<= 13
-	rol	$13,%r15d
-	# 				x9 ^= c
-	xor	%r15,%r10
-	# 				c = x8 + x9
-	lea	(%r11,%r10),%r15
-	# 				(uint32) c <<<= 18
-	rol	$18,%r15d
-	# 				x10 ^= c
-	xor	%r15,%rbp
-	# 						x15 = x15_stack
-	movq	176(%rsp),%r15
-	# 				x10_stack = x10
-	movq	%rbp,168(%rsp)
-	# 						d = x14 + x15
-	lea	(%rbx,%r15),%rbp
-	# 						(uint32) d <<<= 7
-	rol	$7,%ebp
-	# 						x12 ^= d
-	xor	%rbp,%r14
-	# 						d = x15 + x12
-	lea	(%r15,%r14),%rbp
-	# 						(uint32) d <<<= 9
-	rol	$9,%ebp
-	# 						x13 ^= d
-	xor	%rbp,%r13
-	# 						d = x12 + x13
-	lea	(%r14,%r13),%rbp
-	# 						(uint32) d <<<= 13
-	rol	$13,%ebp
-	# 						x14 ^= d
-	xor	%rbp,%rbx
-	# 						d = x13 + x14
-	lea	(%r13,%rbx),%rbp
-	# 						(uint32) d <<<= 18
-	rol	$18,%ebp
-	# 						x15 ^= d
-	xor	%rbp,%r15
-	# 						x15_stack = x15
-	movq	%r15,176(%rsp)
-	# 		x5 = x5_stack
-	movq	160(%rsp),%r15
-	# a = x12 + x0
-	lea	(%r14,%rdx),%rbp
-	# (uint32) a <<<= 7
-	rol	$7,%ebp
-	# x4 ^= a
-	xor	%rbp,%r9
-	# 		b = x1 + x5
-	lea	(%rdi,%r15),%rbp
-	# 		(uint32) b <<<= 7
-	rol	$7,%ebp
-	# 		x9 ^= b
-	xor	%rbp,%r10
-	# a = x0 + x4
-	lea	(%rdx,%r9),%rbp
-	# (uint32) a <<<= 9
-	rol	$9,%ebp
-	# x8 ^= a
-	xor	%rbp,%r11
-	# 		b = x5 + x9
-	lea	(%r15,%r10),%rbp
-	# 		(uint32) b <<<= 9
-	rol	$9,%ebp
-	# 		x13 ^= b
-	xor	%rbp,%r13
-	# a = x4 + x8
-	lea	(%r9,%r11),%rbp
-	# (uint32) a <<<= 13
-	rol	$13,%ebp
-	# x12 ^= a
-	xor	%rbp,%r14
-	# 		b = x9 + x13
-	lea	(%r10,%r13),%rbp
-	# 		(uint32) b <<<= 13
-	rol	$13,%ebp
-	# 		x1 ^= b
-	xor	%rbp,%rdi
-	# a = x8 + x12
-	lea	(%r11,%r14),%rbp
-	# (uint32) a <<<= 18
-	rol	$18,%ebp
-	# x0 ^= a
-	xor	%rbp,%rdx
-	# 		b = x13 + x1
-	lea	(%r13,%rdi),%rbp
-	# 		(uint32) b <<<= 18
-	rol	$18,%ebp
-	# 		x5 ^= b
-	xor	%rbp,%r15
-	# 				x10 = x10_stack
-	movq	168(%rsp),%rbp
-	# 		x5_stack = x5
-	movq	%r15,160(%rsp)
-	# 				c = x6 + x10
-	lea	(%rax,%rbp),%r15
-	# 				(uint32) c <<<= 7
-	rol	$7,%r15d
-	# 				x14 ^= c
-	xor	%r15,%rbx
-	# 				c = x10 + x14
-	lea	(%rbp,%rbx),%r15
-	# 				(uint32) c <<<= 9
-	rol	$9,%r15d
-	# 				x2 ^= c
-	xor	%r15,%rcx
-	# 				c = x14 + x2
-	lea	(%rbx,%rcx),%r15
-	# 				(uint32) c <<<= 13
-	rol	$13,%r15d
-	# 				x6 ^= c
-	xor	%r15,%rax
-	# 				c = x2 + x6
-	lea	(%rcx,%rax),%r15
-	# 				(uint32) c <<<= 18
-	rol	$18,%r15d
-	# 				x10 ^= c
-	xor	%r15,%rbp
-	# 						x15 = x15_stack
-	movq	176(%rsp),%r15
-	# 				x10_stack = x10
-	movq	%rbp,168(%rsp)
-	# 						d = x11 + x15
-	lea	(%r12,%r15),%rbp
-	# 						(uint32) d <<<= 7
-	rol	$7,%ebp
-	# 						x3 ^= d
-	xor	%rbp,%rsi
-	# 						d = x15 + x3
-	lea	(%r15,%rsi),%rbp
-	# 						(uint32) d <<<= 9
-	rol	$9,%ebp
-	# 						x7 ^= d
-	xor	%rbp,%r8
-	# 						d = x3 + x7
-	lea	(%rsi,%r8),%rbp
-	# 						(uint32) d <<<= 13
-	rol	$13,%ebp
-	# 						x11 ^= d
-	xor	%rbp,%r12
-	# 						d = x7 + x11
-	lea	(%r8,%r12),%rbp
-	# 						(uint32) d <<<= 18
-	rol	$18,%ebp
-	# 						x15 ^= d
-	xor	%rbp,%r15
-	# 						x15_stack = x15
-	movq	%r15,176(%rsp)
-	# 		x5 = x5_stack
-	movq	160(%rsp),%r15
-	# a = x3 + x0
-	lea	(%rsi,%rdx),%rbp
-	# (uint32) a <<<= 7
-	rol	$7,%ebp
-	# x1 ^= a
-	xor	%rbp,%rdi
-	# 		b = x4 + x5
-	lea	(%r9,%r15),%rbp
-	# 		(uint32) b <<<= 7
-	rol	$7,%ebp
-	# 		x6 ^= b
-	xor	%rbp,%rax
-	# a = x0 + x1
-	lea	(%rdx,%rdi),%rbp
-	# (uint32) a <<<= 9
-	rol	$9,%ebp
-	# x2 ^= a
-	xor	%rbp,%rcx
-	# 		b = x5 + x6
-	lea	(%r15,%rax),%rbp
-	# 		(uint32) b <<<= 9
-	rol	$9,%ebp
-	# 		x7 ^= b
-	xor	%rbp,%r8
-	# a = x1 + x2
-	lea	(%rdi,%rcx),%rbp
-	# (uint32) a <<<= 13
-	rol	$13,%ebp
-	# x3 ^= a
-	xor	%rbp,%rsi
-	# 		b = x6 + x7
-	lea	(%rax,%r8),%rbp
-	# 		(uint32) b <<<= 13
-	rol	$13,%ebp
-	# 		x4 ^= b
-	xor	%rbp,%r9
-	# a = x2 + x3
-	lea	(%rcx,%rsi),%rbp
-	# (uint32) a <<<= 18
-	rol	$18,%ebp
-	# x0 ^= a
-	xor	%rbp,%rdx
-	# 		b = x7 + x4
-	lea	(%r8,%r9),%rbp
-	# 		(uint32) b <<<= 18
-	rol	$18,%ebp
-	# 		x5 ^= b
-	xor	%rbp,%r15
-	# 				x10 = x10_stack
-	movq	168(%rsp),%rbp
-	# 		x5_stack = x5
-	movq	%r15,160(%rsp)
-	# 				c = x9 + x10
-	lea	(%r10,%rbp),%r15
-	# 				(uint32) c <<<= 7
-	rol	$7,%r15d
-	# 				x11 ^= c
-	xor	%r15,%r12
-	# 				c = x10 + x11
-	lea	(%rbp,%r12),%r15
-	# 				(uint32) c <<<= 9
-	rol	$9,%r15d
-	# 				x8 ^= c
-	xor	%r15,%r11
-	# 				c = x11 + x8
-	lea	(%r12,%r11),%r15
-	# 				(uint32) c <<<= 13
-	rol	$13,%r15d
-	# 				x9 ^= c
-	xor	%r15,%r10
-	# 				c = x8 + x9
-	lea	(%r11,%r10),%r15
-	# 				(uint32) c <<<= 18
-	rol	$18,%r15d
-	# 				x10 ^= c
-	xor	%r15,%rbp
-	# 						x15 = x15_stack
-	movq	176(%rsp),%r15
-	# 				x10_stack = x10
-	movq	%rbp,168(%rsp)
-	# 						d = x14 + x15
-	lea	(%rbx,%r15),%rbp
-	# 						(uint32) d <<<= 7
-	rol	$7,%ebp
-	# 						x12 ^= d
-	xor	%rbp,%r14
-	# 						d = x15 + x12
-	lea	(%r15,%r14),%rbp
-	# 						(uint32) d <<<= 9
-	rol	$9,%ebp
-	# 						x13 ^= d
-	xor	%rbp,%r13
-	# 						d = x12 + x13
-	lea	(%r14,%r13),%rbp
-	# 						(uint32) d <<<= 13
-	rol	$13,%ebp
-	# 						x14 ^= d
-	xor	%rbp,%rbx
-	# 						d = x13 + x14
-	lea	(%r13,%rbx),%rbp
-	# 						(uint32) d <<<= 18
-	rol	$18,%ebp
-	# 						x15 ^= d
-	xor	%rbp,%r15
-	# 						x15_stack = x15
-	movq	%r15,176(%rsp)
-	#   i = i_backup
-	movq	184(%rsp),%r15
-	#                  unsigned>? i -= 4
-	sub	$4,%r15
-	# comment:fp stack unchanged by jump
-	# goto mainloop if unsigned>
-	ja	._mainloop
-	#   (uint32) x2 += j2
-	addl	64(%rsp),%ecx
-	#   x3 <<= 32
-	shl	$32,%rsi
-	#   x3 += j2
-	addq	64(%rsp),%rsi
-	#   (uint64) x3 >>= 32
-	shr	$32,%rsi
-	#   x3 <<= 32
-	shl	$32,%rsi
-	#   x2 += x3
-	add	%rsi,%rcx
-	#   (uint32) x6 += j6
-	addl	80(%rsp),%eax
-	#   x7 <<= 32
-	shl	$32,%r8
-	#   x7 += j6
-	addq	80(%rsp),%r8
-	#   (uint64) x7 >>= 32
-	shr	$32,%r8
-	#   x7 <<= 32
-	shl	$32,%r8
-	#   x6 += x7
-	add	%r8,%rax
-	#   (uint32) x8 += j8
-	addl	88(%rsp),%r11d
-	#   x9 <<= 32
-	shl	$32,%r10
-	#   x9 += j8
-	addq	88(%rsp),%r10
-	#   (uint64) x9 >>= 32
-	shr	$32,%r10
-	#   x9 <<= 32
-	shl	$32,%r10
-	#   x8 += x9
-	add	%r10,%r11
-	#   (uint32) x12 += j12
-	addl	104(%rsp),%r14d
-	#   x13 <<= 32
-	shl	$32,%r13
-	#   x13 += j12
-	addq	104(%rsp),%r13
-	#   (uint64) x13 >>= 32
-	shr	$32,%r13
-	#   x13 <<= 32
-	shl	$32,%r13
-	#   x12 += x13
-	add	%r13,%r14
-	#   (uint32) x0 += j0
-	addl	56(%rsp),%edx
-	#   x1 <<= 32
-	shl	$32,%rdi
-	#   x1 += j0
-	addq	56(%rsp),%rdi
-	#   (uint64) x1 >>= 32
-	shr	$32,%rdi
-	#   x1 <<= 32
-	shl	$32,%rdi
-	#   x0 += x1
-	add	%rdi,%rdx
-	#   x5 = x5_stack
-	movq	160(%rsp),%rdi
-	#   (uint32) x4 += j4
-	addl	72(%rsp),%r9d
-	#   x5 <<= 32
-	shl	$32,%rdi
-	#   x5 += j4
-	addq	72(%rsp),%rdi
-	#   (uint64) x5 >>= 32
-	shr	$32,%rdi
-	#   x5 <<= 32
-	shl	$32,%rdi
-	#   x4 += x5
-	add	%rdi,%r9
-	#   x10 = x10_stack
-	movq	168(%rsp),%r8
-	#   (uint32) x10 += j10
-	addl	96(%rsp),%r8d
-	#   x11 <<= 32
-	shl	$32,%r12
-	#   x11 += j10
-	addq	96(%rsp),%r12
-	#   (uint64) x11 >>= 32
-	shr	$32,%r12
-	#   x11 <<= 32
-	shl	$32,%r12
-	#   x10 += x11
-	add	%r12,%r8
-	#   x15 = x15_stack
-	movq	176(%rsp),%rdi
-	#   (uint32) x14 += j14
-	addl	112(%rsp),%ebx
-	#   x15 <<= 32
-	shl	$32,%rdi
-	#   x15 += j14
-	addq	112(%rsp),%rdi
-	#   (uint64) x15 >>= 32
-	shr	$32,%rdi
-	#   x15 <<= 32
-	shl	$32,%rdi
-	#   x14 += x15
-	add	%rdi,%rbx
-	#   out = out_backup
-	movq	136(%rsp),%rdi
-	#   m = m_backup
-	movq	144(%rsp),%rsi
-	#   x0 ^= *(uint64 *) (m + 0)
-	xorq	0(%rsi),%rdx
-	#   *(uint64 *) (out + 0) = x0
-	movq	%rdx,0(%rdi)
-	#   x2 ^= *(uint64 *) (m + 8)
-	xorq	8(%rsi),%rcx
-	#   *(uint64 *) (out + 8) = x2
-	movq	%rcx,8(%rdi)
-	#   x4 ^= *(uint64 *) (m + 16)
-	xorq	16(%rsi),%r9
-	#   *(uint64 *) (out + 16) = x4
-	movq	%r9,16(%rdi)
-	#   x6 ^= *(uint64 *) (m + 24)
-	xorq	24(%rsi),%rax
-	#   *(uint64 *) (out + 24) = x6
-	movq	%rax,24(%rdi)
-	#   x8 ^= *(uint64 *) (m + 32)
-	xorq	32(%rsi),%r11
-	#   *(uint64 *) (out + 32) = x8
-	movq	%r11,32(%rdi)
-	#   x10 ^= *(uint64 *) (m + 40)
-	xorq	40(%rsi),%r8
-	#   *(uint64 *) (out + 40) = x10
-	movq	%r8,40(%rdi)
-	#   x12 ^= *(uint64 *) (m + 48)
-	xorq	48(%rsi),%r14
-	#   *(uint64 *) (out + 48) = x12
-	movq	%r14,48(%rdi)
-	#   x14 ^= *(uint64 *) (m + 56)
-	xorq	56(%rsi),%rbx
-	#   *(uint64 *) (out + 56) = x14
-	movq	%rbx,56(%rdi)
-	#   bytes = bytes_backup
-	movq	152(%rsp),%rdx
-	#   in8 = j8
-	movq	88(%rsp),%rcx
-	#   in8 += 1
-	add	$1,%rcx
-	#   j8 = in8
-	movq	%rcx,88(%rsp)
-	#                          unsigned>? unsigned<? bytes - 64
-	cmp	$64,%rdx
-	# comment:fp stack unchanged by jump
-	#   goto bytesatleast65 if unsigned>
-	ja	._bytesatleast65
-	# comment:fp stack unchanged by jump
-	#     goto bytesatleast64 if !unsigned<
-	jae	._bytesatleast64
-	#       m = out
-	mov	%rdi,%rsi
-	#       out = ctarget
-	movq	128(%rsp),%rdi
-	#       i = bytes
-	mov	%rdx,%rcx
-	#       while (i) { *out++ = *m++; --i }
-	rep	movsb
-	# comment:fp stack unchanged by fallthrough
-#     bytesatleast64:
-._bytesatleast64:
-	#     x = x_backup
-	movq	120(%rsp),%rdi
-	#     in8 = j8
-	movq	88(%rsp),%rsi
-	#     *(uint64 *) (x + 32) = in8
-	movq	%rsi,32(%rdi)
-	#     r11 = r11_stack
-	movq	0(%rsp),%r11
-	#     r12 = r12_stack
-	movq	8(%rsp),%r12
-	#     r13 = r13_stack
-	movq	16(%rsp),%r13
-	#     r14 = r14_stack
-	movq	24(%rsp),%r14
-	#     r15 = r15_stack
-	movq	32(%rsp),%r15
-	#     rbx = rbx_stack
-	movq	40(%rsp),%rbx
-	#     rbp = rbp_stack
-	movq	48(%rsp),%rbp
-	# comment:fp stack unchanged by fallthrough
-#     done:
-._done:
-	#     leave
-	add	%r11,%rsp
-	mov	%rdi,%rax
-	mov	%rsi,%rdx
-	ret
-#   bytesatleast65:
-._bytesatleast65:
-	#   bytes -= 64
-	sub	$64,%rdx
-	#   out += 64
-	add	$64,%rdi
-	#   m += 64
-	add	$64,%rsi
-	# comment:fp stack unchanged by jump
-	# goto bytesatleast1
-	jmp	._bytesatleast1
-ENDPROC(salsa20_encrypt_bytes)
diff --git a/arch/x86/crypto/salsa20_glue.c b/arch/x86/crypto/salsa20_glue.c
deleted file mode 100644
index b07d7d959806..000000000000
--- a/arch/x86/crypto/salsa20_glue.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Glue code for optimized assembly version of  Salsa20.
- *
- * Copyright (c) 2007 Tan Swee Heng <thesweeheng@gmail.com>
- *
- * The assembly codes are public domain assembly codes written by Daniel. J.
- * Bernstein <djb@cr.yp.to>. The codes are modified to include indentation
- * and to remove extraneous comments and functions that are not needed.
- * - i586 version, renamed as salsa20-i586-asm_32.S
- *   available from <http://cr.yp.to/snuffle/salsa20/x86-pm/salsa20.s>
- * - x86-64 version, renamed as salsa20-x86_64-asm_64.S
- *   available from <http://cr.yp.to/snuffle/salsa20/amd64-3/salsa20.s>
- *
- * Also modified to set up the initial state using the generic C code rather
- * than in assembly.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- */
-
-#include <asm/unaligned.h>
-#include <crypto/internal/skcipher.h>
-#include <crypto/salsa20.h>
-#include <linux/module.h>
-
-asmlinkage void salsa20_encrypt_bytes(u32 state[16], const u8 *src, u8 *dst,
-				      u32 bytes);
-
-static int salsa20_asm_crypt(struct skcipher_request *req)
-{
-	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-	const struct salsa20_ctx *ctx = crypto_skcipher_ctx(tfm);
-	struct skcipher_walk walk;
-	u32 state[16];
-	int err;
-
-	err = skcipher_walk_virt(&walk, req, true);
-
-	crypto_salsa20_init(state, ctx, walk.iv);
-
-	while (walk.nbytes > 0) {
-		unsigned int nbytes = walk.nbytes;
-
-		if (nbytes < walk.total)
-			nbytes = round_down(nbytes, walk.stride);
-
-		salsa20_encrypt_bytes(state, walk.src.virt.addr,
-				      walk.dst.virt.addr, nbytes);
-		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
-	}
-
-	return err;
-}
-
-static struct skcipher_alg alg = {
-	.base.cra_name		= "salsa20",
-	.base.cra_driver_name	= "salsa20-asm",
-	.base.cra_priority	= 200,
-	.base.cra_blocksize	= 1,
-	.base.cra_ctxsize	= sizeof(struct salsa20_ctx),
-	.base.cra_module	= THIS_MODULE,
-
-	.min_keysize		= SALSA20_MIN_KEY_SIZE,
-	.max_keysize		= SALSA20_MAX_KEY_SIZE,
-	.ivsize			= SALSA20_IV_SIZE,
-	.chunksize		= SALSA20_BLOCK_SIZE,
-	.setkey			= crypto_salsa20_setkey,
-	.encrypt		= salsa20_asm_crypt,
-	.decrypt		= salsa20_asm_crypt,
-};
-
-static int __init init(void)
-{
-	return crypto_register_skcipher(&alg);
-}
-
-static void __exit fini(void)
-{
-	crypto_unregister_skcipher(&alg);
-}
-
-module_init(init);
-module_exit(fini);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION ("Salsa20 stream cipher algorithm (optimized assembly version)");
-MODULE_ALIAS_CRYPTO("salsa20");
-MODULE_ALIAS_CRYPTO("salsa20-asm");
diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl
index d6b27dab1b30..14a2f996e543 100644
--- a/arch/x86/entry/syscalls/syscall_32.tbl
+++ b/arch/x86/entry/syscalls/syscall_32.tbl
@@ -396,3 +396,4 @@
 382	i386	pkey_free		sys_pkey_free			__ia32_sys_pkey_free
 383	i386	statx			sys_statx			__ia32_sys_statx
 384	i386	arch_prctl		sys_arch_prctl			__ia32_compat_sys_arch_prctl
+385	i386	io_pgetevents		sys_io_pgetevents		__ia32_compat_sys_io_pgetevents
diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl
index 4dfe42666d0c..cd36232ab62f 100644
--- a/arch/x86/entry/syscalls/syscall_64.tbl
+++ b/arch/x86/entry/syscalls/syscall_64.tbl
@@ -341,6 +341,7 @@
 330	common	pkey_alloc		__x64_sys_pkey_alloc
 331	common	pkey_free		__x64_sys_pkey_free
 332	common	statx			__x64_sys_statx
+333	common	io_pgetevents		__x64_sys_io_pgetevents
 
 #
 # x32-specific system call numbers start at 512 to avoid cache impact
diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile
index d998a487c9b1..261802b1cc50 100644
--- a/arch/x86/entry/vdso/Makefile
+++ b/arch/x86/entry/vdso/Makefile
@@ -44,14 +44,14 @@ obj-y += $(vdso_img_objs)
 targets += $(vdso_img_cfiles)
 targets += $(vdso_img_sodbg) $(vdso_img-y:%=vdso%.so)
 
-export CPPFLAGS_vdso.lds += -P -C
+CPPFLAGS_vdso.lds += -P -C
 
 VDSO_LDFLAGS_vdso.lds = -m64 -Wl,-soname=linux-vdso.so.1 \
 			-Wl,--no-undefined \
 			-Wl,-z,max-page-size=4096 -Wl,-z,common-page-size=4096 \
 			$(DISABLE_LTO)
 
-$(obj)/vdso64.so.dbg: $(src)/vdso.lds $(vobjs) FORCE
+$(obj)/vdso64.so.dbg: $(obj)/vdso.lds $(vobjs) FORCE
 	$(call if_changed,vdso)
 
 HOST_EXTRACFLAGS += -I$(srctree)/tools/include -I$(srctree)/include/uapi -I$(srctree)/arch/$(SUBARCH)/include/uapi
@@ -100,11 +100,8 @@ VDSO_LDFLAGS_vdsox32.lds = -Wl,-m,elf32_x86_64 \
 			   -Wl,-z,max-page-size=4096 \
 			   -Wl,-z,common-page-size=4096
 
-# 64-bit objects to re-brand as x32
-vobjs64-for-x32 := $(filter-out $(vobjs-nox32),$(vobjs-y))
-
 # x32-rebranded versions
-vobjx32s-y := $(vobjs64-for-x32:.o=-x32.o)
+vobjx32s-y := $(vobjs-y:.o=-x32.o)
 
 # same thing, but in the output directory
 vobjx32s := $(foreach F,$(vobjx32s-y),$(obj)/$F)
@@ -122,7 +119,7 @@ $(obj)/%.so: OBJCOPYFLAGS := -S
 $(obj)/%.so: $(obj)/%.so.dbg
 	$(call if_changed,objcopy)
 
-$(obj)/vdsox32.so.dbg: $(src)/vdsox32.lds $(vobjx32s) FORCE
+$(obj)/vdsox32.so.dbg: $(obj)/vdsox32.lds $(vobjx32s) FORCE
 	$(call if_changed,vdso)
 
 CPPFLAGS_vdso32.lds = $(CPPFLAGS_vdso.lds)
diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c
index 70b7845434cb..7782cdbcd67d 100644
--- a/arch/x86/entry/vsyscall/vsyscall_64.c
+++ b/arch/x86/entry/vsyscall/vsyscall_64.c
@@ -107,7 +107,7 @@ static bool write_ok_or_segv(unsigned long ptr, size_t size)
 		thread->cr2		= ptr;
 		thread->trap_nr		= X86_TRAP_PF;
 
-		memset(&info, 0, sizeof(info));
+		clear_siginfo(&info);
 		info.si_signo		= SIGSEGV;
 		info.si_errno		= 0;
 		info.si_code		= SEGV_MAPERR;
diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c
index 786fd875de92..4b98101209a1 100644
--- a/arch/x86/events/amd/ibs.c
+++ b/arch/x86/events/amd/ibs.c
@@ -889,7 +889,7 @@ static void force_ibs_eilvt_setup(void)
 	if (!ibs_eilvt_valid())
 		goto out;
 
-	pr_info("IBS: LVT offset %d assigned\n", offset);
+	pr_info("LVT offset %d assigned\n", offset);
 
 	return;
 out:
diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c
index f5cbbba99283..981ba5e8241b 100644
--- a/arch/x86/events/amd/uncore.c
+++ b/arch/x86/events/amd/uncore.c
@@ -19,6 +19,7 @@
 #include <asm/cpufeature.h>
 #include <asm/perf_event.h>
 #include <asm/msr.h>
+#include <asm/smp.h>
 
 #define NUM_COUNTERS_NB		4
 #define NUM_COUNTERS_L2		4
@@ -399,26 +400,8 @@ static int amd_uncore_cpu_starting(unsigned int cpu)
 	}
 
 	if (amd_uncore_llc) {
-		unsigned int apicid = cpu_data(cpu).apicid;
-		unsigned int nshared, subleaf, prev_eax = 0;
-
 		uncore = *per_cpu_ptr(amd_uncore_llc, cpu);
-		/*
-		 * Iterate over Cache Topology Definition leaves until no
-		 * more cache descriptions are available.
-		 */
-		for (subleaf = 0; subleaf < 5; subleaf++) {
-			cpuid_count(0x8000001d, subleaf, &eax, &ebx, &ecx, &edx);
-
-			/* EAX[0:4] gives type of cache */
-			if (!(eax & 0x1f))
-				break;
-
-			prev_eax = eax;
-		}
-		nshared = ((prev_eax >> 14) & 0xfff) + 1;
-
-		uncore->id = apicid - (apicid % nshared);
+		uncore->id = per_cpu(cpu_llc_id, cpu);
 
 		uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_llc);
 		*per_cpu_ptr(amd_uncore_llc, cpu) = uncore;
diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index 45b2b1c93d04..6e461fb1e0d4 100644
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -2397,7 +2397,7 @@ static unsigned long get_segment_base(unsigned int segment)
 
 #ifdef CONFIG_IA32_EMULATION
 
-#include <asm/compat.h>
+#include <linux/compat.h>
 
 static inline int
 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
diff --git a/arch/x86/include/asm/cacheinfo.h b/arch/x86/include/asm/cacheinfo.h
new file mode 100644
index 000000000000..e958e28f7ab5
--- /dev/null
+++ b/arch/x86/include/asm/cacheinfo.h
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_CACHEINFO_H
+#define _ASM_X86_CACHEINFO_H
+
+void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id);
+
+#endif /* _ASM_X86_CACHEINFO_H */
diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h
index e1c8dab86670..fb97cf7c4137 100644
--- a/arch/x86/include/asm/compat.h
+++ b/arch/x86/include/asm/compat.h
@@ -17,7 +17,6 @@
 
 typedef u32		compat_size_t;
 typedef s32		compat_ssize_t;
-typedef s32		compat_time_t;
 typedef s32		compat_clock_t;
 typedef s32		compat_pid_t;
 typedef u16		__compat_uid_t;
@@ -46,16 +45,6 @@ typedef u32		compat_u32;
 typedef u64 __attribute__((aligned(4))) compat_u64;
 typedef u32		compat_uptr_t;
 
-struct compat_timespec {
-	compat_time_t	tv_sec;
-	s32		tv_nsec;
-};
-
-struct compat_timeval {
-	compat_time_t	tv_sec;
-	s32		tv_usec;
-};
-
 struct compat_stat {
 	compat_dev_t	st_dev;
 	u16		__pad1;
@@ -145,10 +134,10 @@ struct compat_ipc64_perm {
 
 struct compat_semid64_ds {
 	struct compat_ipc64_perm sem_perm;
-	compat_time_t  sem_otime;
-	compat_ulong_t __unused1;
-	compat_time_t  sem_ctime;
-	compat_ulong_t __unused2;
+	compat_ulong_t sem_otime;
+	compat_ulong_t sem_otime_high;
+	compat_ulong_t sem_ctime;
+	compat_ulong_t sem_ctime_high;
 	compat_ulong_t sem_nsems;
 	compat_ulong_t __unused3;
 	compat_ulong_t __unused4;
@@ -156,12 +145,12 @@ struct compat_semid64_ds {
 
 struct compat_msqid64_ds {
 	struct compat_ipc64_perm msg_perm;
-	compat_time_t  msg_stime;
-	compat_ulong_t __unused1;
-	compat_time_t  msg_rtime;
-	compat_ulong_t __unused2;
-	compat_time_t  msg_ctime;
-	compat_ulong_t __unused3;
+	compat_ulong_t msg_stime;
+	compat_ulong_t msg_stime_high;
+	compat_ulong_t msg_rtime;
+	compat_ulong_t msg_rtime_high;
+	compat_ulong_t msg_ctime;
+	compat_ulong_t msg_ctime_high;
 	compat_ulong_t msg_cbytes;
 	compat_ulong_t msg_qnum;
 	compat_ulong_t msg_qbytes;
@@ -174,12 +163,12 @@ struct compat_msqid64_ds {
 struct compat_shmid64_ds {
 	struct compat_ipc64_perm shm_perm;
 	compat_size_t  shm_segsz;
-	compat_time_t  shm_atime;
-	compat_ulong_t __unused1;
-	compat_time_t  shm_dtime;
-	compat_ulong_t __unused2;
-	compat_time_t  shm_ctime;
-	compat_ulong_t __unused3;
+	compat_ulong_t shm_atime;
+	compat_ulong_t shm_atime_high;
+	compat_ulong_t shm_dtime;
+	compat_ulong_t shm_dtime_high;
+	compat_ulong_t shm_ctime;
+	compat_ulong_t shm_ctime_high;
 	compat_pid_t   shm_cpid;
 	compat_pid_t   shm_lpid;
 	compat_ulong_t shm_nattch;
diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h
index 89ce4bfd241f..ce4d176b3d13 100644
--- a/arch/x86/include/asm/dma-mapping.h
+++ b/arch/x86/include/asm/dma-mapping.h
@@ -30,10 +30,7 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
 	return dma_ops;
 }
 
-int arch_dma_supported(struct device *dev, u64 mask);
-#define arch_dma_supported arch_dma_supported
-
-bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp);
+bool arch_dma_alloc_attrs(struct device **dev);
 #define arch_dma_alloc_attrs arch_dma_alloc_attrs
 
 #endif
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index cc8f8fcf9b4a..c18ed65287d5 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -63,7 +63,7 @@ static inline bool arch_syscall_match_sym_name(const char *sym, const char *name
 #ifndef COMPILE_OFFSETS
 
 #if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_IA32_EMULATION)
-#include <asm/compat.h>
+#include <linux/compat.h>
 
 /*
  * Because ia32 syscalls do not map to x86_64 syscall numbers
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 5ea2afd4c871..740a428acf1e 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -50,14 +50,6 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
 
 #define inc_irq_stat(member)	this_cpu_inc(irq_stat.member)
 
-#define local_softirq_pending()	this_cpu_read(irq_stat.__softirq_pending)
-
-#define __ARCH_SET_SOFTIRQ_PENDING
-
-#define set_softirq_pending(x)	\
-		this_cpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x)	this_cpu_or(irq_stat.__softirq_pending, (x))
-
 extern void ack_bad_irq(unsigned int irq);
 
 extern u64 arch_irq_stat_cpu(unsigned int cpu);
diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
index 416cb0e0c496..a8897615354e 100644
--- a/arch/x86/include/asm/hyperv-tlfs.h
+++ b/arch/x86/include/asm/hyperv-tlfs.h
@@ -303,6 +303,9 @@ struct ms_hyperv_tsc_page {
 /* TSC emulation after migration */
 #define HV_X64_MSR_REENLIGHTENMENT_CONTROL	0x40000106
 
+/* Nested features (CPUID 0x4000000A) EAX */
+#define HV_X64_NESTED_MSR_BITMAP		BIT(19)
+
 struct hv_reenlightenment_control {
 	__u64 vector:8;
 	__u64 reserved1:8;
@@ -668,7 +671,11 @@ struct hv_enlightened_vmcs {
 	u32 hv_clean_fields;
 	u32 hv_padding_32;
 	u32 hv_synthetic_controls;
-	u32 hv_enlightenments_control;
+	struct {
+		u32 nested_flush_hypercall:1;
+		u32 msr_bitmap:1;
+		u32 reserved:30;
+	} hv_enlightenments_control;
 	u32 hv_vp_id;
 
 	u64 hv_vm_id;
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index f4b2588865e9..130874077c93 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -258,7 +258,8 @@ union kvm_mmu_page_role {
 		unsigned smep_andnot_wp:1;
 		unsigned smap_andnot_wp:1;
 		unsigned ad_disabled:1;
-		unsigned :7;
+		unsigned guest_mode:1;
+		unsigned :6;
 
 		/*
 		 * This is left at the top of the word so that
@@ -995,7 +996,7 @@ struct kvm_x86_ops {
 	void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr);
 	void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr);
 	void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
-	void (*set_virtual_x2apic_mode)(struct kvm_vcpu *vcpu, bool set);
+	void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu);
 	void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu, hpa_t hpa);
 	void (*deliver_posted_interrupt)(struct kvm_vcpu *vcpu, int vector);
 	int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu);
@@ -1277,6 +1278,7 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);
 int kvm_mmu_load(struct kvm_vcpu *vcpu);
 void kvm_mmu_unload(struct kvm_vcpu *vcpu);
 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
+void kvm_mmu_free_roots(struct kvm_vcpu *vcpu);
 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
 			   struct x86_exception *exception);
 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h
index 14caa9d9fb7f..c0b70bc1e659 100644
--- a/arch/x86/include/asm/linkage.h
+++ b/arch/x86/include/asm/linkage.h
@@ -18,7 +18,7 @@
 	name:
 
 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_ALIGNMENT_16)
-#define __ALIGN		.p2align 4, 0x90
+#define __ALIGN		.p2align 4, 0xCC
 #define __ALIGN_STR	__stringify(__ALIGN)
 #endif
 
diff --git a/arch/x86/include/asm/mcsafe_test.h b/arch/x86/include/asm/mcsafe_test.h
new file mode 100644
index 000000000000..eb59804b6201
--- /dev/null
+++ b/arch/x86/include/asm/mcsafe_test.h
@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _MCSAFE_TEST_H_
+#define _MCSAFE_TEST_H_
+
+#ifndef __ASSEMBLY__
+#ifdef CONFIG_MCSAFE_TEST
+extern unsigned long mcsafe_test_src;
+extern unsigned long mcsafe_test_dst;
+
+static inline void mcsafe_inject_src(void *addr)
+{
+	if (addr)
+		mcsafe_test_src = (unsigned long) addr;
+	else
+		mcsafe_test_src = ~0UL;
+}
+
+static inline void mcsafe_inject_dst(void *addr)
+{
+	if (addr)
+		mcsafe_test_dst = (unsigned long) addr;
+	else
+		mcsafe_test_dst = ~0UL;
+}
+#else /* CONFIG_MCSAFE_TEST */
+static inline void mcsafe_inject_src(void *addr)
+{
+}
+
+static inline void mcsafe_inject_dst(void *addr)
+{
+}
+#endif /* CONFIG_MCSAFE_TEST */
+
+#else /* __ASSEMBLY__ */
+#include <asm/export.h>
+
+#ifdef CONFIG_MCSAFE_TEST
+.macro MCSAFE_TEST_CTL
+	.pushsection .data
+	.align 8
+	.globl mcsafe_test_src
+	mcsafe_test_src:
+		.quad 0
+	EXPORT_SYMBOL_GPL(mcsafe_test_src)
+	.globl mcsafe_test_dst
+	mcsafe_test_dst:
+		.quad 0
+	EXPORT_SYMBOL_GPL(mcsafe_test_dst)
+	.popsection
+.endm
+
+.macro MCSAFE_TEST_SRC reg count target
+	leaq \count(\reg), %r9
+	cmp mcsafe_test_src, %r9
+	ja \target
+.endm
+
+.macro MCSAFE_TEST_DST reg count target
+	leaq \count(\reg), %r9
+	cmp mcsafe_test_dst, %r9
+	ja \target
+.endm
+#else
+.macro MCSAFE_TEST_CTL
+.endm
+
+.macro MCSAFE_TEST_SRC reg count target
+.endm
+
+.macro MCSAFE_TEST_DST reg count target
+.endm
+#endif /* CONFIG_MCSAFE_TEST */
+#endif /* __ASSEMBLY__ */
+#endif /* _MCSAFE_TEST_H_ */
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index cf9911b5a53c..bbc796eb0a3b 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -288,21 +288,6 @@ static inline void arch_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
 		mpx_notify_unmap(mm, vma, start, end);
 }
 
-#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
-static inline int vma_pkey(struct vm_area_struct *vma)
-{
-	unsigned long vma_pkey_mask = VM_PKEY_BIT0 | VM_PKEY_BIT1 |
-				      VM_PKEY_BIT2 | VM_PKEY_BIT3;
-
-	return (vma->vm_flags & vma_pkey_mask) >> VM_PKEY_SHIFT;
-}
-#else
-static inline int vma_pkey(struct vm_area_struct *vma)
-{
-	return 0;
-}
-#endif
-
 /*
  * We only want to enforce protection keys on the current process
  * because we effectively have no access to PKRU for other
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 8b38df98548e..f6f6c63da62f 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -308,16 +308,20 @@ do {									\
  *    lfence
  *    jmp spec_trap
  *  do_rop:
- *    mov %rax,(%rsp)
+ *    mov %rax,(%rsp) for x86_64
+ *    mov %edx,(%esp) for x86_32
  *    retq
  *
  * Without retpolines configured:
  *
- *    jmp *%rax
+ *    jmp *%rax for x86_64
+ *    jmp *%edx for x86_32
  */
 #ifdef CONFIG_RETPOLINE
-# define RETPOLINE_RAX_BPF_JIT_SIZE	17
-# define RETPOLINE_RAX_BPF_JIT()				\
+# ifdef CONFIG_X86_64
+#  define RETPOLINE_RAX_BPF_JIT_SIZE	17
+#  define RETPOLINE_RAX_BPF_JIT()				\
+do {								\
 	EMIT1_off32(0xE8, 7);	 /* callq do_rop */		\
 	/* spec_trap: */					\
 	EMIT2(0xF3, 0x90);       /* pause */			\
@@ -325,11 +329,30 @@ do {									\
 	EMIT2(0xEB, 0xF9);       /* jmp spec_trap */		\
 	/* do_rop: */						\
 	EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */	\
-	EMIT1(0xC3);             /* retq */
-#else
-# define RETPOLINE_RAX_BPF_JIT_SIZE	2
-# define RETPOLINE_RAX_BPF_JIT()				\
-	EMIT2(0xFF, 0xE0);	 /* jmp *%rax */
+	EMIT1(0xC3);             /* retq */			\
+} while (0)
+# else /* !CONFIG_X86_64 */
+#  define RETPOLINE_EDX_BPF_JIT()				\
+do {								\
+	EMIT1_off32(0xE8, 7);	 /* call do_rop */		\
+	/* spec_trap: */					\
+	EMIT2(0xF3, 0x90);       /* pause */			\
+	EMIT3(0x0F, 0xAE, 0xE8); /* lfence */			\
+	EMIT2(0xEB, 0xF9);       /* jmp spec_trap */		\
+	/* do_rop: */						\
+	EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */		\
+	EMIT1(0xC3);             /* ret */			\
+} while (0)
+# endif
+#else /* !CONFIG_RETPOLINE */
+# ifdef CONFIG_X86_64
+#  define RETPOLINE_RAX_BPF_JIT_SIZE	2
+#  define RETPOLINE_RAX_BPF_JIT()				\
+	EMIT2(0xFF, 0xE0);       /* jmp *%rax */
+# else /* !CONFIG_X86_64 */
+#  define RETPOLINE_EDX_BPF_JIT()				\
+	EMIT2(0xFF, 0xE2)        /* jmp *%edx */
+# endif
 #endif
 
 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h
index d32175e30259..662963681ea6 100644
--- a/arch/x86/include/asm/pci.h
+++ b/arch/x86/include/asm/pci.h
@@ -117,9 +117,6 @@ void native_restore_msi_irqs(struct pci_dev *dev);
 #define native_setup_msi_irqs		NULL
 #define native_teardown_msi_irq		NULL
 #endif
-
-#define PCI_DMA_BUS_IS_PHYS (dma_ops->is_phys)
-
 #endif  /* __KERNEL__ */
 
 #ifdef CONFIG_X86_64
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index 1e5a40673953..99fff853c944 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -65,7 +65,6 @@
 #define _PAGE_PKEY_BIT2	(_AT(pteval_t, 0))
 #define _PAGE_PKEY_BIT3	(_AT(pteval_t, 0))
 #endif
-#define __HAVE_ARCH_PTE_SPECIAL
 
 #define _PAGE_PKEY_MASK (_PAGE_PKEY_BIT0 | \
 			 _PAGE_PKEY_BIT1 | \
diff --git a/arch/x86/include/asm/pkeys.h b/arch/x86/include/asm/pkeys.h
index 851c04b7a092..19b137f1b3be 100644
--- a/arch/x86/include/asm/pkeys.h
+++ b/arch/x86/include/asm/pkeys.h
@@ -9,6 +9,11 @@
 extern int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
 		unsigned long init_val);
 
+static inline bool arch_pkeys_enabled(void)
+{
+	return boot_cpu_has(X86_FEATURE_OSPKE);
+}
+
 /*
  * Try to dedicate one of the protection keys to be used as an
  * execute-only protection key.
@@ -116,4 +121,12 @@ extern int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
 		unsigned long init_val);
 extern void copy_init_pkru_to_fpregs(void);
 
+static inline int vma_pkey(struct vm_area_struct *vma)
+{
+	unsigned long vma_pkey_mask = VM_PKEY_BIT0 | VM_PKEY_BIT1 |
+				      VM_PKEY_BIT2 | VM_PKEY_BIT3;
+
+	return (vma->vm_flags & vma_pkey_mask) >> VM_PKEY_SHIFT;
+}
+
 #endif /*_ASM_X86_PKEYS_H */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 21a114914ba4..e28add6b791f 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -186,15 +186,6 @@ extern void identify_boot_cpu(void);
 extern void identify_secondary_cpu(struct cpuinfo_x86 *);
 extern void print_cpu_info(struct cpuinfo_x86 *);
 void print_cpu_msr(struct cpuinfo_x86 *);
-extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
-extern u32 get_scattered_cpuid_leaf(unsigned int level,
-				    unsigned int sub_leaf,
-				    enum cpuid_regs_idx reg);
-extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
-extern void init_amd_cacheinfo(struct cpuinfo_x86 *c);
-
-extern void detect_extended_topology(struct cpuinfo_x86 *c);
-extern void detect_ht(struct cpuinfo_x86 *c);
 
 #ifdef CONFIG_X86_32
 extern int have_cpuid_p(void);
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
index 5e16b5d40d32..3e70bed8a978 100644
--- a/arch/x86/include/asm/qspinlock.h
+++ b/arch/x86/include/asm/qspinlock.h
@@ -7,6 +7,14 @@
 #include <asm-generic/qspinlock_types.h>
 #include <asm/paravirt.h>
 
+#define _Q_PENDING_LOOPS	(1 << 9)
+
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+extern void __pv_init_lock_hash(void);
+extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock);
+
 #define	queued_spin_unlock queued_spin_unlock
 /**
  * queued_spin_unlock - release a queued spinlock
@@ -16,15 +24,9 @@
  */
 static inline void native_queued_spin_unlock(struct qspinlock *lock)
 {
-	smp_store_release((u8 *)lock, 0);
+	smp_store_release(&lock->locked, 0);
 }
 
-#ifdef CONFIG_PARAVIRT_SPINLOCKS
-extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
-extern void __pv_init_lock_hash(void);
-extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
-extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock);
-
 static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
 {
 	pv_queued_spin_lock_slowpath(lock, val);
@@ -40,11 +42,6 @@ static inline bool vcpu_is_preempted(long cpu)
 {
 	return pv_vcpu_is_preempted(cpu);
 }
-#else
-static inline void queued_spin_unlock(struct qspinlock *lock)
-{
-	native_queued_spin_unlock(lock);
-}
 #endif
 
 #ifdef CONFIG_PARAVIRT
diff --git a/arch/x86/include/asm/qspinlock_paravirt.h b/arch/x86/include/asm/qspinlock_paravirt.h
index 923307ea11c7..9ef5ee03d2d7 100644
--- a/arch/x86/include/asm/qspinlock_paravirt.h
+++ b/arch/x86/include/asm/qspinlock_paravirt.h
@@ -22,8 +22,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
  *
  * void __pv_queued_spin_unlock(struct qspinlock *lock)
  * {
- *	struct __qspinlock *l = (void *)lock;
- *	u8 lockval = cmpxchg(&l->locked, _Q_LOCKED_VAL, 0);
+ *	u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
  *
  *	if (likely(lockval == _Q_LOCKED_VAL))
  *		return;
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index f75bff8f9d82..547c4fe50711 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -171,7 +171,6 @@ static inline int wbinvd_on_all_cpus(void)
 	wbinvd();
 	return 0;
 }
-#define smp_num_siblings	1
 #endif /* CONFIG_SMP */
 
 extern unsigned disabled_cpus;
diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h
index 133d9425fced..b6dc698f992a 100644
--- a/arch/x86/include/asm/stacktrace.h
+++ b/arch/x86/include/asm/stacktrace.h
@@ -111,4 +111,6 @@ static inline unsigned long caller_frame_pointer(void)
 	return (unsigned long)frame;
 }
 
+void show_opcodes(u8 *rip, const char *loglvl);
+void show_ip(struct pt_regs *regs, const char *loglvl);
 #endif /* _ASM_X86_STACKTRACE_H */
diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h
index 533f74c300c2..f286c8e82dbc 100644
--- a/arch/x86/include/asm/string_64.h
+++ b/arch/x86/include/asm/string_64.h
@@ -116,7 +116,8 @@ int strcmp(const char *cs, const char *ct);
 #endif
 
 #define __HAVE_ARCH_MEMCPY_MCSAFE 1
-__must_check int memcpy_mcsafe_unrolled(void *dst, const void *src, size_t cnt);
+__must_check unsigned long __memcpy_mcsafe(void *dst, const void *src,
+		size_t cnt);
 DECLARE_STATIC_KEY_FALSE(mcsafe_key);
 
 /**
@@ -131,14 +132,15 @@ DECLARE_STATIC_KEY_FALSE(mcsafe_key);
  * actually do machine check recovery. Everyone else can just
  * use memcpy().
  *
- * Return 0 for success, -EFAULT for fail
+ * Return 0 for success, or number of bytes not copied if there was an
+ * exception.
  */
-static __always_inline __must_check int
+static __always_inline __must_check unsigned long
 memcpy_mcsafe(void *dst, const void *src, size_t cnt)
 {
 #ifdef CONFIG_X86_MCE
 	if (static_branch_unlikely(&mcsafe_key))
-		return memcpy_mcsafe_unrolled(dst, src, cnt);
+		return __memcpy_mcsafe(dst, src, cnt);
 	else
 #endif
 		memcpy(dst, src, cnt);
@@ -147,7 +149,33 @@ memcpy_mcsafe(void *dst, const void *src, size_t cnt)
 
 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1
-void memcpy_flushcache(void *dst, const void *src, size_t cnt);
+void __memcpy_flushcache(void *dst, const void *src, size_t cnt);
+static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t cnt)
+{
+	if (__builtin_constant_p(cnt)) {
+		switch (cnt) {
+		case 4:
+			asm volatile("movntil %1, %0"
+				     : "=m" (*(u32 *)dst)
+				     : "r" (*(u32 *)src));
+			return;
+		case 8:
+			asm volatile("movntiq %1, %0"
+				     : "=m" (*(u64 *)dst)
+				     : "r" (*(u64 *)src));
+			return;
+		case 16:
+			asm volatile("movntiq %1, %0"
+				     : "=m" (*(u64 *)dst)
+				     : "r" (*(u64 *)src));
+			asm volatile("movntiq %1, %0"
+				     : "=m" (*(u64 *)(dst + 8))
+				     : "r" (*(u64 *)(src + 8)));
+			return;
+		}
+	}
+	__memcpy_flushcache(dst, src, cnt);
+}
 #endif
 
 #endif /* __KERNEL__ */
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 62546b3a398e..62acb613114b 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -46,6 +46,17 @@ copy_user_generic(void *to, const void *from, unsigned len)
 	return ret;
 }
 
+static __always_inline __must_check unsigned long
+copy_to_user_mcsafe(void *to, const void *from, unsigned len)
+{
+	unsigned long ret;
+
+	__uaccess_begin();
+	ret = memcpy_mcsafe(to, from, len);
+	__uaccess_end();
+	return ret;
+}
+
 static __always_inline __must_check unsigned long
 raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
 {
@@ -194,4 +205,7 @@ __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
 unsigned long
 copy_user_handle_tail(char *to, char *from, unsigned len);
 
+unsigned long
+mcsafe_handle_tail(char *to, char *from, unsigned len);
+
 #endif /* _ASM_X86_UACCESS_64_H */
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 5db8b0b10766..425e6b8b9547 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -207,7 +207,9 @@ enum vmcs_field {
 	EPTP_LIST_ADDRESS               = 0x00002024,
 	EPTP_LIST_ADDRESS_HIGH          = 0x00002025,
 	VMREAD_BITMAP                   = 0x00002026,
+	VMREAD_BITMAP_HIGH              = 0x00002027,
 	VMWRITE_BITMAP                  = 0x00002028,
+	VMWRITE_BITMAP_HIGH             = 0x00002029,
 	XSS_EXIT_BITMAP                 = 0x0000202C,
 	XSS_EXIT_BITMAP_HIGH            = 0x0000202D,
 	TSC_MULTIPLIER                  = 0x00002032,
diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
index ce8b4da07e35..db98e3ab3295 100644
--- a/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@ -301,5 +301,6 @@ extern struct x86_apic_ops x86_apic_ops;
 extern void x86_early_init_platform_quirks(void);
 extern void x86_init_noop(void);
 extern void x86_init_uint_noop(unsigned int unused);
+extern bool x86_pnpbios_disabled(void);
 
 #endif
diff --git a/arch/x86/include/uapi/asm/sembuf.h b/arch/x86/include/uapi/asm/sembuf.h
index cabd7476bd6c..89de6cd9f0a7 100644
--- a/arch/x86/include/uapi/asm/sembuf.h
+++ b/arch/x86/include/uapi/asm/sembuf.h
@@ -8,15 +8,24 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
  * - 2 miscellaneous 32-bit values
+ *
+ * x86_64 and x32 incorrectly added padding here, so the structures
+ * are still incompatible with the padding on x86.
  */
 struct semid64_ds {
 	struct ipc64_perm sem_perm;	/* permissions .. see ipc.h */
+#ifdef __i386__
+	unsigned long	sem_otime;	/* last semop time */
+	unsigned long	sem_otime_high;
+	unsigned long	sem_ctime;	/* last change time */
+	unsigned long	sem_ctime_high;
+#else
 	__kernel_time_t	sem_otime;	/* last semop time */
 	__kernel_ulong_t __unused1;
 	__kernel_time_t	sem_ctime;	/* last change time */
 	__kernel_ulong_t __unused2;
+#endif
 	__kernel_ulong_t sem_nsems;	/* no. of semaphores in array */
 	__kernel_ulong_t __unused3;
 	__kernel_ulong_t __unused4;
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c
index dfcbe6924eaf..cadeafabf167 100644
--- a/arch/x86/kernel/apm_32.c
+++ b/arch/x86/kernel/apm_32.c
@@ -1715,19 +1715,6 @@ static int proc_apm_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int proc_apm_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_apm_show, NULL);
-}
-
-static const struct file_operations apm_file_ops = {
-	.owner		= THIS_MODULE,
-	.open		= proc_apm_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int apm(void *unused)
 {
 	unsigned short	bx;
@@ -2360,7 +2347,7 @@ static int __init apm_init(void)
 	set_desc_base(&gdt[APM_DS >> 3],
 		 (unsigned long)__va((unsigned long)apm_info.bios.dseg << 4));
 
-	proc_create("apm", 0, NULL, &apm_file_ops);
+	proc_create_single("apm", 0, NULL, proc_apm_show);
 
 	kapmd_task = kthread_create(apm, NULL, "kapmd");
 	if (IS_ERR(kapmd_task)) {
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index a66229f51b12..7a40196967cb 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -17,7 +17,7 @@ KCOV_INSTRUMENT_perf_event.o := n
 nostackp := $(call cc-option, -fno-stack-protector)
 CFLAGS_common.o		:= $(nostackp)
 
-obj-y			:= intel_cacheinfo.o scattered.o topology.o
+obj-y			:= cacheinfo.o scattered.o topology.o
 obj-y			+= common.o
 obj-y			+= rdrand.o
 obj-y			+= match.o
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 1b18be3f35a8..082d7875cef8 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -9,6 +9,7 @@
 #include <linux/random.h>
 #include <asm/processor.h>
 #include <asm/apic.h>
+#include <asm/cacheinfo.h>
 #include <asm/cpu.h>
 #include <asm/spec-ctrl.h>
 #include <asm/smp.h>
@@ -298,7 +299,6 @@ static int nearby_node(int apicid)
 }
 #endif
 
-#ifdef CONFIG_SMP
 /*
  * Fix up cpu_core_id for pre-F17h systems to be in the
  * [0 .. cores_per_node - 1] range. Not really needed but
@@ -328,6 +328,7 @@ static void amd_get_topology(struct cpuinfo_x86 *c)
 
 	/* get information required for multi-node processors */
 	if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
+		int err;
 		u32 eax, ebx, ecx, edx;
 
 		cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
@@ -346,21 +347,15 @@ static void amd_get_topology(struct cpuinfo_x86 *c)
 		}
 
 		/*
-		 * We may have multiple LLCs if L3 caches exist, so check if we
-		 * have an L3 cache by looking at the L3 cache CPUID leaf.
+		 * In case leaf B is available, use it to derive
+		 * topology information.
 		 */
-		if (cpuid_edx(0x80000006)) {
-			if (c->x86 == 0x17) {
-				/*
-				 * LLC is at the core complex level.
-				 * Core complex id is ApicId[3].
-				 */
-				per_cpu(cpu_llc_id, cpu) = c->apicid >> 3;
-			} else {
-				/* LLC is at the node level. */
-				per_cpu(cpu_llc_id, cpu) = node_id;
-			}
-		}
+		err = detect_extended_topology(c);
+		if (!err)
+			c->x86_coreid_bits = get_count_order(c->x86_max_cores);
+
+		cacheinfo_amd_init_llc_id(c, cpu, node_id);
+
 	} else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) {
 		u64 value;
 
@@ -376,7 +371,6 @@ static void amd_get_topology(struct cpuinfo_x86 *c)
 		legacy_fixup_core_id(c);
 	}
 }
-#endif
 
 /*
  * On a AMD dual core setup the lower bits of the APIC id distinguish the cores.
@@ -384,7 +378,6 @@ static void amd_get_topology(struct cpuinfo_x86 *c)
  */
 static void amd_detect_cmp(struct cpuinfo_x86 *c)
 {
-#ifdef CONFIG_SMP
 	unsigned bits;
 	int cpu = smp_processor_id();
 
@@ -395,17 +388,11 @@ static void amd_detect_cmp(struct cpuinfo_x86 *c)
 	c->phys_proc_id = c->initial_apicid >> bits;
 	/* use socket ID also for last level cache */
 	per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
-	amd_get_topology(c);
-#endif
 }
 
 u16 amd_get_nb_id(int cpu)
 {
-	u16 id = 0;
-#ifdef CONFIG_SMP
-	id = per_cpu(cpu_llc_id, cpu);
-#endif
-	return id;
+	return per_cpu(cpu_llc_id, cpu);
 }
 EXPORT_SYMBOL_GPL(amd_get_nb_id);
 
@@ -864,6 +851,7 @@ static void init_amd(struct cpuinfo_x86 *c)
 	/* Multi core CPU? */
 	if (c->extended_cpuid_level >= 0x80000008) {
 		amd_detect_cmp(c);
+		amd_get_topology(c);
 		srat_detect_node(c);
 	}
 
diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c
new file mode 100644
index 000000000000..38354c66df81
--- /dev/null
+++ b/arch/x86/kernel/cpu/cacheinfo.c
@@ -0,0 +1,1010 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *	Routines to identify caches on Intel CPU.
+ *
+ *	Changes:
+ *	Venkatesh Pallipadi	: Adding cache identification through cpuid(4)
+ *	Ashok Raj <ashok.raj@intel.com>: Work with CPU hotplug infrastructure.
+ *	Andi Kleen / Andreas Herrmann	: CPUID4 emulation on AMD.
+ */
+
+#include <linux/slab.h>
+#include <linux/cacheinfo.h>
+#include <linux/cpu.h>
+#include <linux/sched.h>
+#include <linux/capability.h>
+#include <linux/sysfs.h>
+#include <linux/pci.h>
+
+#include <asm/cpufeature.h>
+#include <asm/amd_nb.h>
+#include <asm/smp.h>
+
+#include "cpu.h"
+
+#define LVL_1_INST	1
+#define LVL_1_DATA	2
+#define LVL_2		3
+#define LVL_3		4
+#define LVL_TRACE	5
+
+struct _cache_table {
+	unsigned char descriptor;
+	char cache_type;
+	short size;
+};
+
+#define MB(x)	((x) * 1024)
+
+/* All the cache descriptor types we care about (no TLB or
+   trace cache entries) */
+
+static const struct _cache_table cache_table[] =
+{
+	{ 0x06, LVL_1_INST, 8 },	/* 4-way set assoc, 32 byte line size */
+	{ 0x08, LVL_1_INST, 16 },	/* 4-way set assoc, 32 byte line size */
+	{ 0x09, LVL_1_INST, 32 },	/* 4-way set assoc, 64 byte line size */
+	{ 0x0a, LVL_1_DATA, 8 },	/* 2 way set assoc, 32 byte line size */
+	{ 0x0c, LVL_1_DATA, 16 },	/* 4-way set assoc, 32 byte line size */
+	{ 0x0d, LVL_1_DATA, 16 },	/* 4-way set assoc, 64 byte line size */
+	{ 0x0e, LVL_1_DATA, 24 },	/* 6-way set assoc, 64 byte line size */
+	{ 0x21, LVL_2,      256 },	/* 8-way set assoc, 64 byte line size */
+	{ 0x22, LVL_3,      512 },	/* 4-way set assoc, sectored cache, 64 byte line size */
+	{ 0x23, LVL_3,      MB(1) },	/* 8-way set assoc, sectored cache, 64 byte line size */
+	{ 0x25, LVL_3,      MB(2) },	/* 8-way set assoc, sectored cache, 64 byte line size */
+	{ 0x29, LVL_3,      MB(4) },	/* 8-way set assoc, sectored cache, 64 byte line size */
+	{ 0x2c, LVL_1_DATA, 32 },	/* 8-way set assoc, 64 byte line size */
+	{ 0x30, LVL_1_INST, 32 },	/* 8-way set assoc, 64 byte line size */
+	{ 0x39, LVL_2,      128 },	/* 4-way set assoc, sectored cache, 64 byte line size */
+	{ 0x3a, LVL_2,      192 },	/* 6-way set assoc, sectored cache, 64 byte line size */
+	{ 0x3b, LVL_2,      128 },	/* 2-way set assoc, sectored cache, 64 byte line size */
+	{ 0x3c, LVL_2,      256 },	/* 4-way set assoc, sectored cache, 64 byte line size */
+	{ 0x3d, LVL_2,      384 },	/* 6-way set assoc, sectored cache, 64 byte line size */
+	{ 0x3e, LVL_2,      512 },	/* 4-way set assoc, sectored cache, 64 byte line size */
+	{ 0x3f, LVL_2,      256 },	/* 2-way set assoc, 64 byte line size */
+	{ 0x41, LVL_2,      128 },	/* 4-way set assoc, 32 byte line size */
+	{ 0x42, LVL_2,      256 },	/* 4-way set assoc, 32 byte line size */
+	{ 0x43, LVL_2,      512 },	/* 4-way set assoc, 32 byte line size */
+	{ 0x44, LVL_2,      MB(1) },	/* 4-way set assoc, 32 byte line size */
+	{ 0x45, LVL_2,      MB(2) },	/* 4-way set assoc, 32 byte line size */
+	{ 0x46, LVL_3,      MB(4) },	/* 4-way set assoc, 64 byte line size */
+	{ 0x47, LVL_3,      MB(8) },	/* 8-way set assoc, 64 byte line size */
+	{ 0x48, LVL_2,      MB(3) },	/* 12-way set assoc, 64 byte line size */
+	{ 0x49, LVL_3,      MB(4) },	/* 16-way set assoc, 64 byte line size */
+	{ 0x4a, LVL_3,      MB(6) },	/* 12-way set assoc, 64 byte line size */
+	{ 0x4b, LVL_3,      MB(8) },	/* 16-way set assoc, 64 byte line size */
+	{ 0x4c, LVL_3,      MB(12) },	/* 12-way set assoc, 64 byte line size */
+	{ 0x4d, LVL_3,      MB(16) },	/* 16-way set assoc, 64 byte line size */
+	{ 0x4e, LVL_2,      MB(6) },	/* 24-way set assoc, 64 byte line size */
+	{ 0x60, LVL_1_DATA, 16 },	/* 8-way set assoc, sectored cache, 64 byte line size */
+	{ 0x66, LVL_1_DATA, 8 },	/* 4-way set assoc, sectored cache, 64 byte line size */
+	{ 0x67, LVL_1_DATA, 16 },	/* 4-way set assoc, sectored cache, 64 byte line size */
+	{ 0x68, LVL_1_DATA, 32 },	/* 4-way set assoc, sectored cache, 64 byte line size */
+	{ 0x70, LVL_TRACE,  12 },	/* 8-way set assoc */
+	{ 0x71, LVL_TRACE,  16 },	/* 8-way set assoc */
+	{ 0x72, LVL_TRACE,  32 },	/* 8-way set assoc */
+	{ 0x73, LVL_TRACE,  64 },	/* 8-way set assoc */
+	{ 0x78, LVL_2,      MB(1) },	/* 4-way set assoc, 64 byte line size */
+	{ 0x79, LVL_2,      128 },	/* 8-way set assoc, sectored cache, 64 byte line size */
+	{ 0x7a, LVL_2,      256 },	/* 8-way set assoc, sectored cache, 64 byte line size */
+	{ 0x7b, LVL_2,      512 },	/* 8-way set assoc, sectored cache, 64 byte line size */
+	{ 0x7c, LVL_2,      MB(1) },	/* 8-way set assoc, sectored cache, 64 byte line size */
+	{ 0x7d, LVL_2,      MB(2) },	/* 8-way set assoc, 64 byte line size */
+	{ 0x7f, LVL_2,      512 },	/* 2-way set assoc, 64 byte line size */
+	{ 0x80, LVL_2,      512 },	/* 8-way set assoc, 64 byte line size */
+	{ 0x82, LVL_2,      256 },	/* 8-way set assoc, 32 byte line size */
+	{ 0x83, LVL_2,      512 },	/* 8-way set assoc, 32 byte line size */
+	{ 0x84, LVL_2,      MB(1) },	/* 8-way set assoc, 32 byte line size */
+	{ 0x85, LVL_2,      MB(2) },	/* 8-way set assoc, 32 byte line size */
+	{ 0x86, LVL_2,      512 },	/* 4-way set assoc, 64 byte line size */
+	{ 0x87, LVL_2,      MB(1) },	/* 8-way set assoc, 64 byte line size */
+	{ 0xd0, LVL_3,      512 },	/* 4-way set assoc, 64 byte line size */
+	{ 0xd1, LVL_3,      MB(1) },	/* 4-way set assoc, 64 byte line size */
+	{ 0xd2, LVL_3,      MB(2) },	/* 4-way set assoc, 64 byte line size */
+	{ 0xd6, LVL_3,      MB(1) },	/* 8-way set assoc, 64 byte line size */
+	{ 0xd7, LVL_3,      MB(2) },	/* 8-way set assoc, 64 byte line size */
+	{ 0xd8, LVL_3,      MB(4) },	/* 12-way set assoc, 64 byte line size */
+	{ 0xdc, LVL_3,      MB(2) },	/* 12-way set assoc, 64 byte line size */
+	{ 0xdd, LVL_3,      MB(4) },	/* 12-way set assoc, 64 byte line size */
+	{ 0xde, LVL_3,      MB(8) },	/* 12-way set assoc, 64 byte line size */
+	{ 0xe2, LVL_3,      MB(2) },	/* 16-way set assoc, 64 byte line size */
+	{ 0xe3, LVL_3,      MB(4) },	/* 16-way set assoc, 64 byte line size */
+	{ 0xe4, LVL_3,      MB(8) },	/* 16-way set assoc, 64 byte line size */
+	{ 0xea, LVL_3,      MB(12) },	/* 24-way set assoc, 64 byte line size */
+	{ 0xeb, LVL_3,      MB(18) },	/* 24-way set assoc, 64 byte line size */
+	{ 0xec, LVL_3,      MB(24) },	/* 24-way set assoc, 64 byte line size */
+	{ 0x00, 0, 0}
+};
+
+
+enum _cache_type {
+	CTYPE_NULL = 0,
+	CTYPE_DATA = 1,
+	CTYPE_INST = 2,
+	CTYPE_UNIFIED = 3
+};
+
+union _cpuid4_leaf_eax {
+	struct {
+		enum _cache_type	type:5;
+		unsigned int		level:3;
+		unsigned int		is_self_initializing:1;
+		unsigned int		is_fully_associative:1;
+		unsigned int		reserved:4;
+		unsigned int		num_threads_sharing:12;
+		unsigned int		num_cores_on_die:6;
+	} split;
+	u32 full;
+};
+
+union _cpuid4_leaf_ebx {
+	struct {
+		unsigned int		coherency_line_size:12;
+		unsigned int		physical_line_partition:10;
+		unsigned int		ways_of_associativity:10;
+	} split;
+	u32 full;
+};
+
+union _cpuid4_leaf_ecx {
+	struct {
+		unsigned int		number_of_sets:32;
+	} split;
+	u32 full;
+};
+
+struct _cpuid4_info_regs {
+	union _cpuid4_leaf_eax eax;
+	union _cpuid4_leaf_ebx ebx;
+	union _cpuid4_leaf_ecx ecx;
+	unsigned int id;
+	unsigned long size;
+	struct amd_northbridge *nb;
+};
+
+static unsigned short num_cache_leaves;
+
+/* AMD doesn't have CPUID4. Emulate it here to report the same
+   information to the user.  This makes some assumptions about the machine:
+   L2 not shared, no SMT etc. that is currently true on AMD CPUs.
+
+   In theory the TLBs could be reported as fake type (they are in "dummy").
+   Maybe later */
+union l1_cache {
+	struct {
+		unsigned line_size:8;
+		unsigned lines_per_tag:8;
+		unsigned assoc:8;
+		unsigned size_in_kb:8;
+	};
+	unsigned val;
+};
+
+union l2_cache {
+	struct {
+		unsigned line_size:8;
+		unsigned lines_per_tag:4;
+		unsigned assoc:4;
+		unsigned size_in_kb:16;
+	};
+	unsigned val;
+};
+
+union l3_cache {
+	struct {
+		unsigned line_size:8;
+		unsigned lines_per_tag:4;
+		unsigned assoc:4;
+		unsigned res:2;
+		unsigned size_encoded:14;
+	};
+	unsigned val;
+};
+
+static const unsigned short assocs[] = {
+	[1] = 1,
+	[2] = 2,
+	[4] = 4,
+	[6] = 8,
+	[8] = 16,
+	[0xa] = 32,
+	[0xb] = 48,
+	[0xc] = 64,
+	[0xd] = 96,
+	[0xe] = 128,
+	[0xf] = 0xffff /* fully associative - no way to show this currently */
+};
+
+static const unsigned char levels[] = { 1, 1, 2, 3 };
+static const unsigned char types[] = { 1, 2, 3, 3 };
+
+static const enum cache_type cache_type_map[] = {
+	[CTYPE_NULL] = CACHE_TYPE_NOCACHE,
+	[CTYPE_DATA] = CACHE_TYPE_DATA,
+	[CTYPE_INST] = CACHE_TYPE_INST,
+	[CTYPE_UNIFIED] = CACHE_TYPE_UNIFIED,
+};
+
+static void
+amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax,
+		     union _cpuid4_leaf_ebx *ebx,
+		     union _cpuid4_leaf_ecx *ecx)
+{
+	unsigned dummy;
+	unsigned line_size, lines_per_tag, assoc, size_in_kb;
+	union l1_cache l1i, l1d;
+	union l2_cache l2;
+	union l3_cache l3;
+	union l1_cache *l1 = &l1d;
+
+	eax->full = 0;
+	ebx->full = 0;
+	ecx->full = 0;
+
+	cpuid(0x80000005, &dummy, &dummy, &l1d.val, &l1i.val);
+	cpuid(0x80000006, &dummy, &dummy, &l2.val, &l3.val);
+
+	switch (leaf) {
+	case 1:
+		l1 = &l1i;
+	case 0:
+		if (!l1->val)
+			return;
+		assoc = assocs[l1->assoc];
+		line_size = l1->line_size;
+		lines_per_tag = l1->lines_per_tag;
+		size_in_kb = l1->size_in_kb;
+		break;
+	case 2:
+		if (!l2.val)
+			return;
+		assoc = assocs[l2.assoc];
+		line_size = l2.line_size;
+		lines_per_tag = l2.lines_per_tag;
+		/* cpu_data has errata corrections for K7 applied */
+		size_in_kb = __this_cpu_read(cpu_info.x86_cache_size);
+		break;
+	case 3:
+		if (!l3.val)
+			return;
+		assoc = assocs[l3.assoc];
+		line_size = l3.line_size;
+		lines_per_tag = l3.lines_per_tag;
+		size_in_kb = l3.size_encoded * 512;
+		if (boot_cpu_has(X86_FEATURE_AMD_DCM)) {
+			size_in_kb = size_in_kb >> 1;
+			assoc = assoc >> 1;
+		}
+		break;
+	default:
+		return;
+	}
+
+	eax->split.is_self_initializing = 1;
+	eax->split.type = types[leaf];
+	eax->split.level = levels[leaf];
+	eax->split.num_threads_sharing = 0;
+	eax->split.num_cores_on_die = __this_cpu_read(cpu_info.x86_max_cores) - 1;
+
+
+	if (assoc == 0xffff)
+		eax->split.is_fully_associative = 1;
+	ebx->split.coherency_line_size = line_size - 1;
+	ebx->split.ways_of_associativity = assoc - 1;
+	ebx->split.physical_line_partition = lines_per_tag - 1;
+	ecx->split.number_of_sets = (size_in_kb * 1024) / line_size /
+		(ebx->split.ways_of_associativity + 1) - 1;
+}
+
+#if defined(CONFIG_AMD_NB) && defined(CONFIG_SYSFS)
+
+/*
+ * L3 cache descriptors
+ */
+static void amd_calc_l3_indices(struct amd_northbridge *nb)
+{
+	struct amd_l3_cache *l3 = &nb->l3_cache;
+	unsigned int sc0, sc1, sc2, sc3;
+	u32 val = 0;
+
+	pci_read_config_dword(nb->misc, 0x1C4, &val);
+
+	/* calculate subcache sizes */
+	l3->subcaches[0] = sc0 = !(val & BIT(0));
+	l3->subcaches[1] = sc1 = !(val & BIT(4));
+
+	if (boot_cpu_data.x86 == 0x15) {
+		l3->subcaches[0] = sc0 += !(val & BIT(1));
+		l3->subcaches[1] = sc1 += !(val & BIT(5));
+	}
+
+	l3->subcaches[2] = sc2 = !(val & BIT(8))  + !(val & BIT(9));
+	l3->subcaches[3] = sc3 = !(val & BIT(12)) + !(val & BIT(13));
+
+	l3->indices = (max(max3(sc0, sc1, sc2), sc3) << 10) - 1;
+}
+
+/*
+ * check whether a slot used for disabling an L3 index is occupied.
+ * @l3: L3 cache descriptor
+ * @slot: slot number (0..1)
+ *
+ * @returns: the disabled index if used or negative value if slot free.
+ */
+static int amd_get_l3_disable_slot(struct amd_northbridge *nb, unsigned slot)
+{
+	unsigned int reg = 0;
+
+	pci_read_config_dword(nb->misc, 0x1BC + slot * 4, &reg);
+
+	/* check whether this slot is activated already */
+	if (reg & (3UL << 30))
+		return reg & 0xfff;
+
+	return -1;
+}
+
+static ssize_t show_cache_disable(struct cacheinfo *this_leaf, char *buf,
+				  unsigned int slot)
+{
+	int index;
+	struct amd_northbridge *nb = this_leaf->priv;
+
+	index = amd_get_l3_disable_slot(nb, slot);
+	if (index >= 0)
+		return sprintf(buf, "%d\n", index);
+
+	return sprintf(buf, "FREE\n");
+}
+
+#define SHOW_CACHE_DISABLE(slot)					\
+static ssize_t								\
+cache_disable_##slot##_show(struct device *dev,				\
+			    struct device_attribute *attr, char *buf)	\
+{									\
+	struct cacheinfo *this_leaf = dev_get_drvdata(dev);		\
+	return show_cache_disable(this_leaf, buf, slot);		\
+}
+SHOW_CACHE_DISABLE(0)
+SHOW_CACHE_DISABLE(1)
+
+static void amd_l3_disable_index(struct amd_northbridge *nb, int cpu,
+				 unsigned slot, unsigned long idx)
+{
+	int i;
+
+	idx |= BIT(30);
+
+	/*
+	 *  disable index in all 4 subcaches
+	 */
+	for (i = 0; i < 4; i++) {
+		u32 reg = idx | (i << 20);
+
+		if (!nb->l3_cache.subcaches[i])
+			continue;
+
+		pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg);
+
+		/*
+		 * We need to WBINVD on a core on the node containing the L3
+		 * cache which indices we disable therefore a simple wbinvd()
+		 * is not sufficient.
+		 */
+		wbinvd_on_cpu(cpu);
+
+		reg |= BIT(31);
+		pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg);
+	}
+}
+
+/*
+ * disable a L3 cache index by using a disable-slot
+ *
+ * @l3:    L3 cache descriptor
+ * @cpu:   A CPU on the node containing the L3 cache
+ * @slot:  slot number (0..1)
+ * @index: index to disable
+ *
+ * @return: 0 on success, error status on failure
+ */
+static int amd_set_l3_disable_slot(struct amd_northbridge *nb, int cpu,
+			    unsigned slot, unsigned long index)
+{
+	int ret = 0;
+
+	/*  check if @slot is already used or the index is already disabled */
+	ret = amd_get_l3_disable_slot(nb, slot);
+	if (ret >= 0)
+		return -EEXIST;
+
+	if (index > nb->l3_cache.indices)
+		return -EINVAL;
+
+	/* check whether the other slot has disabled the same index already */
+	if (index == amd_get_l3_disable_slot(nb, !slot))
+		return -EEXIST;
+
+	amd_l3_disable_index(nb, cpu, slot, index);
+
+	return 0;
+}
+
+static ssize_t store_cache_disable(struct cacheinfo *this_leaf,
+				   const char *buf, size_t count,
+				   unsigned int slot)
+{
+	unsigned long val = 0;
+	int cpu, err = 0;
+	struct amd_northbridge *nb = this_leaf->priv;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	cpu = cpumask_first(&this_leaf->shared_cpu_map);
+
+	if (kstrtoul(buf, 10, &val) < 0)
+		return -EINVAL;
+
+	err = amd_set_l3_disable_slot(nb, cpu, slot, val);
+	if (err) {
+		if (err == -EEXIST)
+			pr_warn("L3 slot %d in use/index already disabled!\n",
+				   slot);
+		return err;
+	}
+	return count;
+}
+
+#define STORE_CACHE_DISABLE(slot)					\
+static ssize_t								\
+cache_disable_##slot##_store(struct device *dev,			\
+			     struct device_attribute *attr,		\
+			     const char *buf, size_t count)		\
+{									\
+	struct cacheinfo *this_leaf = dev_get_drvdata(dev);		\
+	return store_cache_disable(this_leaf, buf, count, slot);	\
+}
+STORE_CACHE_DISABLE(0)
+STORE_CACHE_DISABLE(1)
+
+static ssize_t subcaches_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	struct cacheinfo *this_leaf = dev_get_drvdata(dev);
+	int cpu = cpumask_first(&this_leaf->shared_cpu_map);
+
+	return sprintf(buf, "%x\n", amd_get_subcaches(cpu));
+}
+
+static ssize_t subcaches_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf, size_t count)
+{
+	struct cacheinfo *this_leaf = dev_get_drvdata(dev);
+	int cpu = cpumask_first(&this_leaf->shared_cpu_map);
+	unsigned long val;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	if (kstrtoul(buf, 16, &val) < 0)
+		return -EINVAL;
+
+	if (amd_set_subcaches(cpu, val))
+		return -EINVAL;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(cache_disable_0);
+static DEVICE_ATTR_RW(cache_disable_1);
+static DEVICE_ATTR_RW(subcaches);
+
+static umode_t
+cache_private_attrs_is_visible(struct kobject *kobj,
+			       struct attribute *attr, int unused)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct cacheinfo *this_leaf = dev_get_drvdata(dev);
+	umode_t mode = attr->mode;
+
+	if (!this_leaf->priv)
+		return 0;
+
+	if ((attr == &dev_attr_subcaches.attr) &&
+	    amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
+		return mode;
+
+	if ((attr == &dev_attr_cache_disable_0.attr ||
+	     attr == &dev_attr_cache_disable_1.attr) &&
+	    amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE))
+		return mode;
+
+	return 0;
+}
+
+static struct attribute_group cache_private_group = {
+	.is_visible = cache_private_attrs_is_visible,
+};
+
+static void init_amd_l3_attrs(void)
+{
+	int n = 1;
+	static struct attribute **amd_l3_attrs;
+
+	if (amd_l3_attrs) /* already initialized */
+		return;
+
+	if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE))
+		n += 2;
+	if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
+		n += 1;
+
+	amd_l3_attrs = kcalloc(n, sizeof(*amd_l3_attrs), GFP_KERNEL);
+	if (!amd_l3_attrs)
+		return;
+
+	n = 0;
+	if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE)) {
+		amd_l3_attrs[n++] = &dev_attr_cache_disable_0.attr;
+		amd_l3_attrs[n++] = &dev_attr_cache_disable_1.attr;
+	}
+	if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
+		amd_l3_attrs[n++] = &dev_attr_subcaches.attr;
+
+	cache_private_group.attrs = amd_l3_attrs;
+}
+
+const struct attribute_group *
+cache_get_priv_group(struct cacheinfo *this_leaf)
+{
+	struct amd_northbridge *nb = this_leaf->priv;
+
+	if (this_leaf->level < 3 || !nb)
+		return NULL;
+
+	if (nb && nb->l3_cache.indices)
+		init_amd_l3_attrs();
+
+	return &cache_private_group;
+}
+
+static void amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index)
+{
+	int node;
+
+	/* only for L3, and not in virtualized environments */
+	if (index < 3)
+		return;
+
+	node = amd_get_nb_id(smp_processor_id());
+	this_leaf->nb = node_to_amd_nb(node);
+	if (this_leaf->nb && !this_leaf->nb->l3_cache.indices)
+		amd_calc_l3_indices(this_leaf->nb);
+}
+#else
+#define amd_init_l3_cache(x, y)
+#endif  /* CONFIG_AMD_NB && CONFIG_SYSFS */
+
+static int
+cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *this_leaf)
+{
+	union _cpuid4_leaf_eax	eax;
+	union _cpuid4_leaf_ebx	ebx;
+	union _cpuid4_leaf_ecx	ecx;
+	unsigned		edx;
+
+	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
+		if (boot_cpu_has(X86_FEATURE_TOPOEXT))
+			cpuid_count(0x8000001d, index, &eax.full,
+				    &ebx.full, &ecx.full, &edx);
+		else
+			amd_cpuid4(index, &eax, &ebx, &ecx);
+		amd_init_l3_cache(this_leaf, index);
+	} else {
+		cpuid_count(4, index, &eax.full, &ebx.full, &ecx.full, &edx);
+	}
+
+	if (eax.split.type == CTYPE_NULL)
+		return -EIO; /* better error ? */
+
+	this_leaf->eax = eax;
+	this_leaf->ebx = ebx;
+	this_leaf->ecx = ecx;
+	this_leaf->size = (ecx.split.number_of_sets          + 1) *
+			  (ebx.split.coherency_line_size     + 1) *
+			  (ebx.split.physical_line_partition + 1) *
+			  (ebx.split.ways_of_associativity   + 1);
+	return 0;
+}
+
+static int find_num_cache_leaves(struct cpuinfo_x86 *c)
+{
+	unsigned int		eax, ebx, ecx, edx, op;
+	union _cpuid4_leaf_eax	cache_eax;
+	int 			i = -1;
+
+	if (c->x86_vendor == X86_VENDOR_AMD)
+		op = 0x8000001d;
+	else
+		op = 4;
+
+	do {
+		++i;
+		/* Do cpuid(op) loop to find out num_cache_leaves */
+		cpuid_count(op, i, &eax, &ebx, &ecx, &edx);
+		cache_eax.full = eax;
+	} while (cache_eax.split.type != CTYPE_NULL);
+	return i;
+}
+
+void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id)
+{
+	/*
+	 * We may have multiple LLCs if L3 caches exist, so check if we
+	 * have an L3 cache by looking at the L3 cache CPUID leaf.
+	 */
+	if (!cpuid_edx(0x80000006))
+		return;
+
+	if (c->x86 < 0x17) {
+		/* LLC is at the node level. */
+		per_cpu(cpu_llc_id, cpu) = node_id;
+	} else if (c->x86 == 0x17 &&
+		   c->x86_model >= 0 && c->x86_model <= 0x1F) {
+		/*
+		 * LLC is at the core complex level.
+		 * Core complex ID is ApicId[3] for these processors.
+		 */
+		per_cpu(cpu_llc_id, cpu) = c->apicid >> 3;
+	} else {
+		/*
+		 * LLC ID is calculated from the number of threads sharing the
+		 * cache.
+		 * */
+		u32 eax, ebx, ecx, edx, num_sharing_cache = 0;
+		u32 llc_index = find_num_cache_leaves(c) - 1;
+
+		cpuid_count(0x8000001d, llc_index, &eax, &ebx, &ecx, &edx);
+		if (eax)
+			num_sharing_cache = ((eax >> 14) & 0xfff) + 1;
+
+		if (num_sharing_cache) {
+			int bits = get_count_order(num_sharing_cache) - 1;
+
+			per_cpu(cpu_llc_id, cpu) = c->apicid >> bits;
+		}
+	}
+}
+
+void init_amd_cacheinfo(struct cpuinfo_x86 *c)
+{
+
+	if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
+		num_cache_leaves = find_num_cache_leaves(c);
+	} else if (c->extended_cpuid_level >= 0x80000006) {
+		if (cpuid_edx(0x80000006) & 0xf000)
+			num_cache_leaves = 4;
+		else
+			num_cache_leaves = 3;
+	}
+}
+
+void init_intel_cacheinfo(struct cpuinfo_x86 *c)
+{
+	/* Cache sizes */
+	unsigned int trace = 0, l1i = 0, l1d = 0, l2 = 0, l3 = 0;
+	unsigned int new_l1d = 0, new_l1i = 0; /* Cache sizes from cpuid(4) */
+	unsigned int new_l2 = 0, new_l3 = 0, i; /* Cache sizes from cpuid(4) */
+	unsigned int l2_id = 0, l3_id = 0, num_threads_sharing, index_msb;
+#ifdef CONFIG_SMP
+	unsigned int cpu = c->cpu_index;
+#endif
+
+	if (c->cpuid_level > 3) {
+		static int is_initialized;
+
+		if (is_initialized == 0) {
+			/* Init num_cache_leaves from boot CPU */
+			num_cache_leaves = find_num_cache_leaves(c);
+			is_initialized++;
+		}
+
+		/*
+		 * Whenever possible use cpuid(4), deterministic cache
+		 * parameters cpuid leaf to find the cache details
+		 */
+		for (i = 0; i < num_cache_leaves; i++) {
+			struct _cpuid4_info_regs this_leaf = {};
+			int retval;
+
+			retval = cpuid4_cache_lookup_regs(i, &this_leaf);
+			if (retval < 0)
+				continue;
+
+			switch (this_leaf.eax.split.level) {
+			case 1:
+				if (this_leaf.eax.split.type == CTYPE_DATA)
+					new_l1d = this_leaf.size/1024;
+				else if (this_leaf.eax.split.type == CTYPE_INST)
+					new_l1i = this_leaf.size/1024;
+				break;
+			case 2:
+				new_l2 = this_leaf.size/1024;
+				num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing;
+				index_msb = get_count_order(num_threads_sharing);
+				l2_id = c->apicid & ~((1 << index_msb) - 1);
+				break;
+			case 3:
+				new_l3 = this_leaf.size/1024;
+				num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing;
+				index_msb = get_count_order(num_threads_sharing);
+				l3_id = c->apicid & ~((1 << index_msb) - 1);
+				break;
+			default:
+				break;
+			}
+		}
+	}
+	/*
+	 * Don't use cpuid2 if cpuid4 is supported. For P4, we use cpuid2 for
+	 * trace cache
+	 */
+	if ((num_cache_leaves == 0 || c->x86 == 15) && c->cpuid_level > 1) {
+		/* supports eax=2  call */
+		int j, n;
+		unsigned int regs[4];
+		unsigned char *dp = (unsigned char *)regs;
+		int only_trace = 0;
+
+		if (num_cache_leaves != 0 && c->x86 == 15)
+			only_trace = 1;
+
+		/* Number of times to iterate */
+		n = cpuid_eax(2) & 0xFF;
+
+		for (i = 0 ; i < n ; i++) {
+			cpuid(2, &regs[0], &regs[1], &regs[2], &regs[3]);
+
+			/* If bit 31 is set, this is an unknown format */
+			for (j = 0 ; j < 3 ; j++)
+				if (regs[j] & (1 << 31))
+					regs[j] = 0;
+
+			/* Byte 0 is level count, not a descriptor */
+			for (j = 1 ; j < 16 ; j++) {
+				unsigned char des = dp[j];
+				unsigned char k = 0;
+
+				/* look up this descriptor in the table */
+				while (cache_table[k].descriptor != 0) {
+					if (cache_table[k].descriptor == des) {
+						if (only_trace && cache_table[k].cache_type != LVL_TRACE)
+							break;
+						switch (cache_table[k].cache_type) {
+						case LVL_1_INST:
+							l1i += cache_table[k].size;
+							break;
+						case LVL_1_DATA:
+							l1d += cache_table[k].size;
+							break;
+						case LVL_2:
+							l2 += cache_table[k].size;
+							break;
+						case LVL_3:
+							l3 += cache_table[k].size;
+							break;
+						case LVL_TRACE:
+							trace += cache_table[k].size;
+							break;
+						}
+
+						break;
+					}
+
+					k++;
+				}
+			}
+		}
+	}
+
+	if (new_l1d)
+		l1d = new_l1d;
+
+	if (new_l1i)
+		l1i = new_l1i;
+
+	if (new_l2) {
+		l2 = new_l2;
+#ifdef CONFIG_SMP
+		per_cpu(cpu_llc_id, cpu) = l2_id;
+#endif
+	}
+
+	if (new_l3) {
+		l3 = new_l3;
+#ifdef CONFIG_SMP
+		per_cpu(cpu_llc_id, cpu) = l3_id;
+#endif
+	}
+
+#ifdef CONFIG_SMP
+	/*
+	 * If cpu_llc_id is not yet set, this means cpuid_level < 4 which in
+	 * turns means that the only possibility is SMT (as indicated in
+	 * cpuid1). Since cpuid2 doesn't specify shared caches, and we know
+	 * that SMT shares all caches, we can unconditionally set cpu_llc_id to
+	 * c->phys_proc_id.
+	 */
+	if (per_cpu(cpu_llc_id, cpu) == BAD_APICID)
+		per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
+#endif
+
+	c->x86_cache_size = l3 ? l3 : (l2 ? l2 : (l1i+l1d));
+
+	if (!l2)
+		cpu_detect_cache_sizes(c);
+}
+
+static int __cache_amd_cpumap_setup(unsigned int cpu, int index,
+				    struct _cpuid4_info_regs *base)
+{
+	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
+	struct cacheinfo *this_leaf;
+	int i, sibling;
+
+	/*
+	 * For L3, always use the pre-calculated cpu_llc_shared_mask
+	 * to derive shared_cpu_map.
+	 */
+	if (index == 3) {
+		for_each_cpu(i, cpu_llc_shared_mask(cpu)) {
+			this_cpu_ci = get_cpu_cacheinfo(i);
+			if (!this_cpu_ci->info_list)
+				continue;
+			this_leaf = this_cpu_ci->info_list + index;
+			for_each_cpu(sibling, cpu_llc_shared_mask(cpu)) {
+				if (!cpu_online(sibling))
+					continue;
+				cpumask_set_cpu(sibling,
+						&this_leaf->shared_cpu_map);
+			}
+		}
+	} else if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
+		unsigned int apicid, nshared, first, last;
+
+		nshared = base->eax.split.num_threads_sharing + 1;
+		apicid = cpu_data(cpu).apicid;
+		first = apicid - (apicid % nshared);
+		last = first + nshared - 1;
+
+		for_each_online_cpu(i) {
+			this_cpu_ci = get_cpu_cacheinfo(i);
+			if (!this_cpu_ci->info_list)
+				continue;
+
+			apicid = cpu_data(i).apicid;
+			if ((apicid < first) || (apicid > last))
+				continue;
+
+			this_leaf = this_cpu_ci->info_list + index;
+
+			for_each_online_cpu(sibling) {
+				apicid = cpu_data(sibling).apicid;
+				if ((apicid < first) || (apicid > last))
+					continue;
+				cpumask_set_cpu(sibling,
+						&this_leaf->shared_cpu_map);
+			}
+		}
+	} else
+		return 0;
+
+	return 1;
+}
+
+static void __cache_cpumap_setup(unsigned int cpu, int index,
+				 struct _cpuid4_info_regs *base)
+{
+	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
+	struct cacheinfo *this_leaf, *sibling_leaf;
+	unsigned long num_threads_sharing;
+	int index_msb, i;
+	struct cpuinfo_x86 *c = &cpu_data(cpu);
+
+	if (c->x86_vendor == X86_VENDOR_AMD) {
+		if (__cache_amd_cpumap_setup(cpu, index, base))
+			return;
+	}
+
+	this_leaf = this_cpu_ci->info_list + index;
+	num_threads_sharing = 1 + base->eax.split.num_threads_sharing;
+
+	cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map);
+	if (num_threads_sharing == 1)
+		return;
+
+	index_msb = get_count_order(num_threads_sharing);
+
+	for_each_online_cpu(i)
+		if (cpu_data(i).apicid >> index_msb == c->apicid >> index_msb) {
+			struct cpu_cacheinfo *sib_cpu_ci = get_cpu_cacheinfo(i);
+
+			if (i == cpu || !sib_cpu_ci->info_list)
+				continue;/* skip if itself or no cacheinfo */
+			sibling_leaf = sib_cpu_ci->info_list + index;
+			cpumask_set_cpu(i, &this_leaf->shared_cpu_map);
+			cpumask_set_cpu(cpu, &sibling_leaf->shared_cpu_map);
+		}
+}
+
+static void ci_leaf_init(struct cacheinfo *this_leaf,
+			 struct _cpuid4_info_regs *base)
+{
+	this_leaf->id = base->id;
+	this_leaf->attributes = CACHE_ID;
+	this_leaf->level = base->eax.split.level;
+	this_leaf->type = cache_type_map[base->eax.split.type];
+	this_leaf->coherency_line_size =
+				base->ebx.split.coherency_line_size + 1;
+	this_leaf->ways_of_associativity =
+				base->ebx.split.ways_of_associativity + 1;
+	this_leaf->size = base->size;
+	this_leaf->number_of_sets = base->ecx.split.number_of_sets + 1;
+	this_leaf->physical_line_partition =
+				base->ebx.split.physical_line_partition + 1;
+	this_leaf->priv = base->nb;
+}
+
+static int __init_cache_level(unsigned int cpu)
+{
+	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
+
+	if (!num_cache_leaves)
+		return -ENOENT;
+	if (!this_cpu_ci)
+		return -EINVAL;
+	this_cpu_ci->num_levels = 3;
+	this_cpu_ci->num_leaves = num_cache_leaves;
+	return 0;
+}
+
+/*
+ * The max shared threads number comes from CPUID.4:EAX[25-14] with input
+ * ECX as cache index. Then right shift apicid by the number's order to get
+ * cache id for this cache node.
+ */
+static void get_cache_id(int cpu, struct _cpuid4_info_regs *id4_regs)
+{
+	struct cpuinfo_x86 *c = &cpu_data(cpu);
+	unsigned long num_threads_sharing;
+	int index_msb;
+
+	num_threads_sharing = 1 + id4_regs->eax.split.num_threads_sharing;
+	index_msb = get_count_order(num_threads_sharing);
+	id4_regs->id = c->apicid >> index_msb;
+}
+
+static int __populate_cache_leaves(unsigned int cpu)
+{
+	unsigned int idx, ret;
+	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
+	struct cacheinfo *this_leaf = this_cpu_ci->info_list;
+	struct _cpuid4_info_regs id4_regs = {};
+
+	for (idx = 0; idx < this_cpu_ci->num_leaves; idx++) {
+		ret = cpuid4_cache_lookup_regs(idx, &id4_regs);
+		if (ret)
+			return ret;
+		get_cache_id(cpu, &id4_regs);
+		ci_leaf_init(this_leaf++, &id4_regs);
+		__cache_cpumap_setup(cpu, idx, &id4_regs);
+	}
+	this_cpu_ci->cpu_map_populated = true;
+
+	return 0;
+}
+
+DEFINE_SMP_CALL_CACHE_FUNCTION(init_cache_level)
+DEFINE_SMP_CALL_CACHE_FUNCTION(populate_cache_leaves)
diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c
index e5ec0f11c0de..14433ff5b828 100644
--- a/arch/x86/kernel/cpu/centaur.c
+++ b/arch/x86/kernel/cpu/centaur.c
@@ -18,6 +18,13 @@
 #define RNG_ENABLED	(1 << 3)
 #define RNG_ENABLE	(1 << 6)	/* MSR_VIA_RNG */
 
+#define X86_VMX_FEATURE_PROC_CTLS_TPR_SHADOW	0x00200000
+#define X86_VMX_FEATURE_PROC_CTLS_VNMI		0x00400000
+#define X86_VMX_FEATURE_PROC_CTLS_2ND_CTLS	0x80000000
+#define X86_VMX_FEATURE_PROC_CTLS2_VIRT_APIC	0x00000001
+#define X86_VMX_FEATURE_PROC_CTLS2_EPT		0x00000002
+#define X86_VMX_FEATURE_PROC_CTLS2_VPID		0x00000020
+
 static void init_c3(struct cpuinfo_x86 *c)
 {
 	u32  lo, hi;
@@ -112,6 +119,31 @@ static void early_init_centaur(struct cpuinfo_x86 *c)
 	}
 }
 
+static void centaur_detect_vmx_virtcap(struct cpuinfo_x86 *c)
+{
+	u32 vmx_msr_low, vmx_msr_high, msr_ctl, msr_ctl2;
+
+	rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, vmx_msr_low, vmx_msr_high);
+	msr_ctl = vmx_msr_high | vmx_msr_low;
+
+	if (msr_ctl & X86_VMX_FEATURE_PROC_CTLS_TPR_SHADOW)
+		set_cpu_cap(c, X86_FEATURE_TPR_SHADOW);
+	if (msr_ctl & X86_VMX_FEATURE_PROC_CTLS_VNMI)
+		set_cpu_cap(c, X86_FEATURE_VNMI);
+	if (msr_ctl & X86_VMX_FEATURE_PROC_CTLS_2ND_CTLS) {
+		rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
+		      vmx_msr_low, vmx_msr_high);
+		msr_ctl2 = vmx_msr_high | vmx_msr_low;
+		if ((msr_ctl2 & X86_VMX_FEATURE_PROC_CTLS2_VIRT_APIC) &&
+		    (msr_ctl & X86_VMX_FEATURE_PROC_CTLS_TPR_SHADOW))
+			set_cpu_cap(c, X86_FEATURE_FLEXPRIORITY);
+		if (msr_ctl2 & X86_VMX_FEATURE_PROC_CTLS2_EPT)
+			set_cpu_cap(c, X86_FEATURE_EPT);
+		if (msr_ctl2 & X86_VMX_FEATURE_PROC_CTLS2_VPID)
+			set_cpu_cap(c, X86_FEATURE_VPID);
+	}
+}
+
 static void init_centaur(struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_X86_32
@@ -128,6 +160,24 @@ static void init_centaur(struct cpuinfo_x86 *c)
 	clear_cpu_cap(c, 0*32+31);
 #endif
 	early_init_centaur(c);
+	init_intel_cacheinfo(c);
+	detect_num_cpu_cores(c);
+#ifdef CONFIG_X86_32
+	detect_ht(c);
+#endif
+
+	if (c->cpuid_level > 9) {
+		unsigned int eax = cpuid_eax(10);
+
+		/*
+		 * Check for version and the number of counters
+		 * Version(eax[7:0]) can't be 0;
+		 * Counters(eax[15:8]) should be greater than 1;
+		 */
+		if ((eax & 0xff) && (((eax >> 8) & 0xff) > 1))
+			set_cpu_cap(c, X86_FEATURE_ARCH_PERFMON);
+	}
+
 	switch (c->x86) {
 #ifdef CONFIG_X86_32
 	case 5:
@@ -199,6 +249,9 @@ static void init_centaur(struct cpuinfo_x86 *c)
 #ifdef CONFIG_X86_64
 	set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
 #endif
+
+	if (cpu_has(c, X86_FEATURE_VMX))
+		centaur_detect_vmx_virtcap(c);
 }
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 38276f58d3bf..a7f5b4c81c68 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -66,6 +66,13 @@ cpumask_var_t cpu_callin_mask;
 /* representing cpus for which sibling maps can be computed */
 cpumask_var_t cpu_sibling_setup_mask;
 
+/* Number of siblings per CPU package */
+int smp_num_siblings = 1;
+EXPORT_SYMBOL(smp_num_siblings);
+
+/* Last level cache ID of each logical CPU */
+DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
+
 /* correctly size the local cpu masks */
 void __init setup_cpu_local_masks(void)
 {
@@ -577,6 +584,19 @@ static void get_model_name(struct cpuinfo_x86 *c)
 	*(s + 1) = '\0';
 }
 
+void detect_num_cpu_cores(struct cpuinfo_x86 *c)
+{
+	unsigned int eax, ebx, ecx, edx;
+
+	c->x86_max_cores = 1;
+	if (!IS_ENABLED(CONFIG_SMP) || c->cpuid_level < 4)
+		return;
+
+	cpuid_count(4, 0, &eax, &ebx, &ecx, &edx);
+	if (eax & 0x1f)
+		c->x86_max_cores = (eax >> 26) + 1;
+}
+
 void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
 {
 	unsigned int n, dummy, ebx, ecx, edx, l2size;
diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h
index 37672d299e35..38216f678fc3 100644
--- a/arch/x86/kernel/cpu/cpu.h
+++ b/arch/x86/kernel/cpu/cpu.h
@@ -47,6 +47,16 @@ extern const struct cpu_dev *const __x86_cpu_dev_start[],
 
 extern void get_cpu_cap(struct cpuinfo_x86 *c);
 extern void cpu_detect_cache_sizes(struct cpuinfo_x86 *c);
+extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
+extern u32 get_scattered_cpuid_leaf(unsigned int level,
+				    unsigned int sub_leaf,
+				    enum cpuid_regs_idx reg);
+extern void init_intel_cacheinfo(struct cpuinfo_x86 *c);
+extern void init_amd_cacheinfo(struct cpuinfo_x86 *c);
+
+extern void detect_num_cpu_cores(struct cpuinfo_x86 *c);
+extern int detect_extended_topology(struct cpuinfo_x86 *c);
+extern void detect_ht(struct cpuinfo_x86 *c);
 
 unsigned int aperfmperf_get_khz(int cpu);
 
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 577e7f7ae273..eb75564f2d25 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -456,24 +456,6 @@ static void srat_detect_node(struct cpuinfo_x86 *c)
 #endif
 }
 
-/*
- * find out the number of processor cores on the die
- */
-static int intel_num_cpu_cores(struct cpuinfo_x86 *c)
-{
-	unsigned int eax, ebx, ecx, edx;
-
-	if (!IS_ENABLED(CONFIG_SMP) || c->cpuid_level < 4)
-		return 1;
-
-	/* Intel has a non-standard dependency on %ecx for this CPUID level. */
-	cpuid_count(4, 0, &eax, &ebx, &ecx, &edx);
-	if (eax & 0x1f)
-		return (eax >> 26) + 1;
-	else
-		return 1;
-}
-
 static void detect_vmx_virtcap(struct cpuinfo_x86 *c)
 {
 	/* Intel VMX MSR indicated features */
@@ -656,8 +638,6 @@ static void init_intel_misc_features(struct cpuinfo_x86 *c)
 
 static void init_intel(struct cpuinfo_x86 *c)
 {
-	unsigned int l2 = 0;
-
 	early_init_intel(c);
 
 	intel_workarounds(c);
@@ -674,19 +654,13 @@ static void init_intel(struct cpuinfo_x86 *c)
 		 * let's use the legacy cpuid vector 0x1 and 0x4 for topology
 		 * detection.
 		 */
-		c->x86_max_cores = intel_num_cpu_cores(c);
+		detect_num_cpu_cores(c);
 #ifdef CONFIG_X86_32
 		detect_ht(c);
 #endif
 	}
 
-	l2 = init_intel_cacheinfo(c);
-
-	/* Detect legacy cache sizes if init_intel_cacheinfo did not */
-	if (l2 == 0) {
-		cpu_detect_cache_sizes(c);
-		l2 = c->x86_cache_size;
-	}
+	init_intel_cacheinfo(c);
 
 	if (c->cpuid_level > 9) {
 		unsigned eax = cpuid_eax(10);
@@ -699,7 +673,8 @@ static void init_intel(struct cpuinfo_x86 *c)
 		set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
 
 	if (boot_cpu_has(X86_FEATURE_DS)) {
-		unsigned int l1;
+		unsigned int l1, l2;
+
 		rdmsr(MSR_IA32_MISC_ENABLE, l1, l2);
 		if (!(l1 & (1<<11)))
 			set_cpu_cap(c, X86_FEATURE_BTS);
@@ -727,6 +702,7 @@ static void init_intel(struct cpuinfo_x86 *c)
 	 * Dixon is NOT a Celeron.
 	 */
 	if (c->x86 == 6) {
+		unsigned int l2 = c->x86_cache_size;
 		char *p = NULL;
 
 		switch (c->x86_model) {
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
deleted file mode 100644
index 54d04d574148..000000000000
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ /dev/null
@@ -1,968 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- *	Routines to identify caches on Intel CPU.
- *
- *	Changes:
- *	Venkatesh Pallipadi	: Adding cache identification through cpuid(4)
- *	Ashok Raj <ashok.raj@intel.com>: Work with CPU hotplug infrastructure.
- *	Andi Kleen / Andreas Herrmann	: CPUID4 emulation on AMD.
- */
-
-#include <linux/slab.h>
-#include <linux/cacheinfo.h>
-#include <linux/cpu.h>
-#include <linux/sched.h>
-#include <linux/capability.h>
-#include <linux/sysfs.h>
-#include <linux/pci.h>
-
-#include <asm/cpufeature.h>
-#include <asm/amd_nb.h>
-#include <asm/smp.h>
-
-#define LVL_1_INST	1
-#define LVL_1_DATA	2
-#define LVL_2		3
-#define LVL_3		4
-#define LVL_TRACE	5
-
-struct _cache_table {
-	unsigned char descriptor;
-	char cache_type;
-	short size;
-};
-
-#define MB(x)	((x) * 1024)
-
-/* All the cache descriptor types we care about (no TLB or
-   trace cache entries) */
-
-static const struct _cache_table cache_table[] =
-{
-	{ 0x06, LVL_1_INST, 8 },	/* 4-way set assoc, 32 byte line size */
-	{ 0x08, LVL_1_INST, 16 },	/* 4-way set assoc, 32 byte line size */
-	{ 0x09, LVL_1_INST, 32 },	/* 4-way set assoc, 64 byte line size */
-	{ 0x0a, LVL_1_DATA, 8 },	/* 2 way set assoc, 32 byte line size */
-	{ 0x0c, LVL_1_DATA, 16 },	/* 4-way set assoc, 32 byte line size */
-	{ 0x0d, LVL_1_DATA, 16 },	/* 4-way set assoc, 64 byte line size */
-	{ 0x0e, LVL_1_DATA, 24 },	/* 6-way set assoc, 64 byte line size */
-	{ 0x21, LVL_2,      256 },	/* 8-way set assoc, 64 byte line size */
-	{ 0x22, LVL_3,      512 },	/* 4-way set assoc, sectored cache, 64 byte line size */
-	{ 0x23, LVL_3,      MB(1) },	/* 8-way set assoc, sectored cache, 64 byte line size */
-	{ 0x25, LVL_3,      MB(2) },	/* 8-way set assoc, sectored cache, 64 byte line size */
-	{ 0x29, LVL_3,      MB(4) },	/* 8-way set assoc, sectored cache, 64 byte line size */
-	{ 0x2c, LVL_1_DATA, 32 },	/* 8-way set assoc, 64 byte line size */
-	{ 0x30, LVL_1_INST, 32 },	/* 8-way set assoc, 64 byte line size */
-	{ 0x39, LVL_2,      128 },	/* 4-way set assoc, sectored cache, 64 byte line size */
-	{ 0x3a, LVL_2,      192 },	/* 6-way set assoc, sectored cache, 64 byte line size */
-	{ 0x3b, LVL_2,      128 },	/* 2-way set assoc, sectored cache, 64 byte line size */
-	{ 0x3c, LVL_2,      256 },	/* 4-way set assoc, sectored cache, 64 byte line size */
-	{ 0x3d, LVL_2,      384 },	/* 6-way set assoc, sectored cache, 64 byte line size */
-	{ 0x3e, LVL_2,      512 },	/* 4-way set assoc, sectored cache, 64 byte line size */
-	{ 0x3f, LVL_2,      256 },	/* 2-way set assoc, 64 byte line size */
-	{ 0x41, LVL_2,      128 },	/* 4-way set assoc, 32 byte line size */
-	{ 0x42, LVL_2,      256 },	/* 4-way set assoc, 32 byte line size */
-	{ 0x43, LVL_2,      512 },	/* 4-way set assoc, 32 byte line size */
-	{ 0x44, LVL_2,      MB(1) },	/* 4-way set assoc, 32 byte line size */
-	{ 0x45, LVL_2,      MB(2) },	/* 4-way set assoc, 32 byte line size */
-	{ 0x46, LVL_3,      MB(4) },	/* 4-way set assoc, 64 byte line size */
-	{ 0x47, LVL_3,      MB(8) },	/* 8-way set assoc, 64 byte line size */
-	{ 0x48, LVL_2,      MB(3) },	/* 12-way set assoc, 64 byte line size */
-	{ 0x49, LVL_3,      MB(4) },	/* 16-way set assoc, 64 byte line size */
-	{ 0x4a, LVL_3,      MB(6) },	/* 12-way set assoc, 64 byte line size */
-	{ 0x4b, LVL_3,      MB(8) },	/* 16-way set assoc, 64 byte line size */
-	{ 0x4c, LVL_3,      MB(12) },	/* 12-way set assoc, 64 byte line size */
-	{ 0x4d, LVL_3,      MB(16) },	/* 16-way set assoc, 64 byte line size */
-	{ 0x4e, LVL_2,      MB(6) },	/* 24-way set assoc, 64 byte line size */
-	{ 0x60, LVL_1_DATA, 16 },	/* 8-way set assoc, sectored cache, 64 byte line size */
-	{ 0x66, LVL_1_DATA, 8 },	/* 4-way set assoc, sectored cache, 64 byte line size */
-	{ 0x67, LVL_1_DATA, 16 },	/* 4-way set assoc, sectored cache, 64 byte line size */
-	{ 0x68, LVL_1_DATA, 32 },	/* 4-way set assoc, sectored cache, 64 byte line size */
-	{ 0x70, LVL_TRACE,  12 },	/* 8-way set assoc */
-	{ 0x71, LVL_TRACE,  16 },	/* 8-way set assoc */
-	{ 0x72, LVL_TRACE,  32 },	/* 8-way set assoc */
-	{ 0x73, LVL_TRACE,  64 },	/* 8-way set assoc */
-	{ 0x78, LVL_2,      MB(1) },	/* 4-way set assoc, 64 byte line size */
-	{ 0x79, LVL_2,      128 },	/* 8-way set assoc, sectored cache, 64 byte line size */
-	{ 0x7a, LVL_2,      256 },	/* 8-way set assoc, sectored cache, 64 byte line size */
-	{ 0x7b, LVL_2,      512 },	/* 8-way set assoc, sectored cache, 64 byte line size */
-	{ 0x7c, LVL_2,      MB(1) },	/* 8-way set assoc, sectored cache, 64 byte line size */
-	{ 0x7d, LVL_2,      MB(2) },	/* 8-way set assoc, 64 byte line size */
-	{ 0x7f, LVL_2,      512 },	/* 2-way set assoc, 64 byte line size */
-	{ 0x80, LVL_2,      512 },	/* 8-way set assoc, 64 byte line size */
-	{ 0x82, LVL_2,      256 },	/* 8-way set assoc, 32 byte line size */
-	{ 0x83, LVL_2,      512 },	/* 8-way set assoc, 32 byte line size */
-	{ 0x84, LVL_2,      MB(1) },	/* 8-way set assoc, 32 byte line size */
-	{ 0x85, LVL_2,      MB(2) },	/* 8-way set assoc, 32 byte line size */
-	{ 0x86, LVL_2,      512 },	/* 4-way set assoc, 64 byte line size */
-	{ 0x87, LVL_2,      MB(1) },	/* 8-way set assoc, 64 byte line size */
-	{ 0xd0, LVL_3,      512 },	/* 4-way set assoc, 64 byte line size */
-	{ 0xd1, LVL_3,      MB(1) },	/* 4-way set assoc, 64 byte line size */
-	{ 0xd2, LVL_3,      MB(2) },	/* 4-way set assoc, 64 byte line size */
-	{ 0xd6, LVL_3,      MB(1) },	/* 8-way set assoc, 64 byte line size */
-	{ 0xd7, LVL_3,      MB(2) },	/* 8-way set assoc, 64 byte line size */
-	{ 0xd8, LVL_3,      MB(4) },	/* 12-way set assoc, 64 byte line size */
-	{ 0xdc, LVL_3,      MB(2) },	/* 12-way set assoc, 64 byte line size */
-	{ 0xdd, LVL_3,      MB(4) },	/* 12-way set assoc, 64 byte line size */
-	{ 0xde, LVL_3,      MB(8) },	/* 12-way set assoc, 64 byte line size */
-	{ 0xe2, LVL_3,      MB(2) },	/* 16-way set assoc, 64 byte line size */
-	{ 0xe3, LVL_3,      MB(4) },	/* 16-way set assoc, 64 byte line size */
-	{ 0xe4, LVL_3,      MB(8) },	/* 16-way set assoc, 64 byte line size */
-	{ 0xea, LVL_3,      MB(12) },	/* 24-way set assoc, 64 byte line size */
-	{ 0xeb, LVL_3,      MB(18) },	/* 24-way set assoc, 64 byte line size */
-	{ 0xec, LVL_3,      MB(24) },	/* 24-way set assoc, 64 byte line size */
-	{ 0x00, 0, 0}
-};
-
-
-enum _cache_type {
-	CTYPE_NULL = 0,
-	CTYPE_DATA = 1,
-	CTYPE_INST = 2,
-	CTYPE_UNIFIED = 3
-};
-
-union _cpuid4_leaf_eax {
-	struct {
-		enum _cache_type	type:5;
-		unsigned int		level:3;
-		unsigned int		is_self_initializing:1;
-		unsigned int		is_fully_associative:1;
-		unsigned int		reserved:4;
-		unsigned int		num_threads_sharing:12;
-		unsigned int		num_cores_on_die:6;
-	} split;
-	u32 full;
-};
-
-union _cpuid4_leaf_ebx {
-	struct {
-		unsigned int		coherency_line_size:12;
-		unsigned int		physical_line_partition:10;
-		unsigned int		ways_of_associativity:10;
-	} split;
-	u32 full;
-};
-
-union _cpuid4_leaf_ecx {
-	struct {
-		unsigned int		number_of_sets:32;
-	} split;
-	u32 full;
-};
-
-struct _cpuid4_info_regs {
-	union _cpuid4_leaf_eax eax;
-	union _cpuid4_leaf_ebx ebx;
-	union _cpuid4_leaf_ecx ecx;
-	unsigned int id;
-	unsigned long size;
-	struct amd_northbridge *nb;
-};
-
-static unsigned short num_cache_leaves;
-
-/* AMD doesn't have CPUID4. Emulate it here to report the same
-   information to the user.  This makes some assumptions about the machine:
-   L2 not shared, no SMT etc. that is currently true on AMD CPUs.
-
-   In theory the TLBs could be reported as fake type (they are in "dummy").
-   Maybe later */
-union l1_cache {
-	struct {
-		unsigned line_size:8;
-		unsigned lines_per_tag:8;
-		unsigned assoc:8;
-		unsigned size_in_kb:8;
-	};
-	unsigned val;
-};
-
-union l2_cache {
-	struct {
-		unsigned line_size:8;
-		unsigned lines_per_tag:4;
-		unsigned assoc:4;
-		unsigned size_in_kb:16;
-	};
-	unsigned val;
-};
-
-union l3_cache {
-	struct {
-		unsigned line_size:8;
-		unsigned lines_per_tag:4;
-		unsigned assoc:4;
-		unsigned res:2;
-		unsigned size_encoded:14;
-	};
-	unsigned val;
-};
-
-static const unsigned short assocs[] = {
-	[1] = 1,
-	[2] = 2,
-	[4] = 4,
-	[6] = 8,
-	[8] = 16,
-	[0xa] = 32,
-	[0xb] = 48,
-	[0xc] = 64,
-	[0xd] = 96,
-	[0xe] = 128,
-	[0xf] = 0xffff /* fully associative - no way to show this currently */
-};
-
-static const unsigned char levels[] = { 1, 1, 2, 3 };
-static const unsigned char types[] = { 1, 2, 3, 3 };
-
-static const enum cache_type cache_type_map[] = {
-	[CTYPE_NULL] = CACHE_TYPE_NOCACHE,
-	[CTYPE_DATA] = CACHE_TYPE_DATA,
-	[CTYPE_INST] = CACHE_TYPE_INST,
-	[CTYPE_UNIFIED] = CACHE_TYPE_UNIFIED,
-};
-
-static void
-amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax,
-		     union _cpuid4_leaf_ebx *ebx,
-		     union _cpuid4_leaf_ecx *ecx)
-{
-	unsigned dummy;
-	unsigned line_size, lines_per_tag, assoc, size_in_kb;
-	union l1_cache l1i, l1d;
-	union l2_cache l2;
-	union l3_cache l3;
-	union l1_cache *l1 = &l1d;
-
-	eax->full = 0;
-	ebx->full = 0;
-	ecx->full = 0;
-
-	cpuid(0x80000005, &dummy, &dummy, &l1d.val, &l1i.val);
-	cpuid(0x80000006, &dummy, &dummy, &l2.val, &l3.val);
-
-	switch (leaf) {
-	case 1:
-		l1 = &l1i;
-	case 0:
-		if (!l1->val)
-			return;
-		assoc = assocs[l1->assoc];
-		line_size = l1->line_size;
-		lines_per_tag = l1->lines_per_tag;
-		size_in_kb = l1->size_in_kb;
-		break;
-	case 2:
-		if (!l2.val)
-			return;
-		assoc = assocs[l2.assoc];
-		line_size = l2.line_size;
-		lines_per_tag = l2.lines_per_tag;
-		/* cpu_data has errata corrections for K7 applied */
-		size_in_kb = __this_cpu_read(cpu_info.x86_cache_size);
-		break;
-	case 3:
-		if (!l3.val)
-			return;
-		assoc = assocs[l3.assoc];
-		line_size = l3.line_size;
-		lines_per_tag = l3.lines_per_tag;
-		size_in_kb = l3.size_encoded * 512;
-		if (boot_cpu_has(X86_FEATURE_AMD_DCM)) {
-			size_in_kb = size_in_kb >> 1;
-			assoc = assoc >> 1;
-		}
-		break;
-	default:
-		return;
-	}
-
-	eax->split.is_self_initializing = 1;
-	eax->split.type = types[leaf];
-	eax->split.level = levels[leaf];
-	eax->split.num_threads_sharing = 0;
-	eax->split.num_cores_on_die = __this_cpu_read(cpu_info.x86_max_cores) - 1;
-
-
-	if (assoc == 0xffff)
-		eax->split.is_fully_associative = 1;
-	ebx->split.coherency_line_size = line_size - 1;
-	ebx->split.ways_of_associativity = assoc - 1;
-	ebx->split.physical_line_partition = lines_per_tag - 1;
-	ecx->split.number_of_sets = (size_in_kb * 1024) / line_size /
-		(ebx->split.ways_of_associativity + 1) - 1;
-}
-
-#if defined(CONFIG_AMD_NB) && defined(CONFIG_SYSFS)
-
-/*
- * L3 cache descriptors
- */
-static void amd_calc_l3_indices(struct amd_northbridge *nb)
-{
-	struct amd_l3_cache *l3 = &nb->l3_cache;
-	unsigned int sc0, sc1, sc2, sc3;
-	u32 val = 0;
-
-	pci_read_config_dword(nb->misc, 0x1C4, &val);
-
-	/* calculate subcache sizes */
-	l3->subcaches[0] = sc0 = !(val & BIT(0));
-	l3->subcaches[1] = sc1 = !(val & BIT(4));
-
-	if (boot_cpu_data.x86 == 0x15) {
-		l3->subcaches[0] = sc0 += !(val & BIT(1));
-		l3->subcaches[1] = sc1 += !(val & BIT(5));
-	}
-
-	l3->subcaches[2] = sc2 = !(val & BIT(8))  + !(val & BIT(9));
-	l3->subcaches[3] = sc3 = !(val & BIT(12)) + !(val & BIT(13));
-
-	l3->indices = (max(max3(sc0, sc1, sc2), sc3) << 10) - 1;
-}
-
-/*
- * check whether a slot used for disabling an L3 index is occupied.
- * @l3: L3 cache descriptor
- * @slot: slot number (0..1)
- *
- * @returns: the disabled index if used or negative value if slot free.
- */
-static int amd_get_l3_disable_slot(struct amd_northbridge *nb, unsigned slot)
-{
-	unsigned int reg = 0;
-
-	pci_read_config_dword(nb->misc, 0x1BC + slot * 4, &reg);
-
-	/* check whether this slot is activated already */
-	if (reg & (3UL << 30))
-		return reg & 0xfff;
-
-	return -1;
-}
-
-static ssize_t show_cache_disable(struct cacheinfo *this_leaf, char *buf,
-				  unsigned int slot)
-{
-	int index;
-	struct amd_northbridge *nb = this_leaf->priv;
-
-	index = amd_get_l3_disable_slot(nb, slot);
-	if (index >= 0)
-		return sprintf(buf, "%d\n", index);
-
-	return sprintf(buf, "FREE\n");
-}
-
-#define SHOW_CACHE_DISABLE(slot)					\
-static ssize_t								\
-cache_disable_##slot##_show(struct device *dev,				\
-			    struct device_attribute *attr, char *buf)	\
-{									\
-	struct cacheinfo *this_leaf = dev_get_drvdata(dev);		\
-	return show_cache_disable(this_leaf, buf, slot);		\
-}
-SHOW_CACHE_DISABLE(0)
-SHOW_CACHE_DISABLE(1)
-
-static void amd_l3_disable_index(struct amd_northbridge *nb, int cpu,
-				 unsigned slot, unsigned long idx)
-{
-	int i;
-
-	idx |= BIT(30);
-
-	/*
-	 *  disable index in all 4 subcaches
-	 */
-	for (i = 0; i < 4; i++) {
-		u32 reg = idx | (i << 20);
-
-		if (!nb->l3_cache.subcaches[i])
-			continue;
-
-		pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg);
-
-		/*
-		 * We need to WBINVD on a core on the node containing the L3
-		 * cache which indices we disable therefore a simple wbinvd()
-		 * is not sufficient.
-		 */
-		wbinvd_on_cpu(cpu);
-
-		reg |= BIT(31);
-		pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg);
-	}
-}
-
-/*
- * disable a L3 cache index by using a disable-slot
- *
- * @l3:    L3 cache descriptor
- * @cpu:   A CPU on the node containing the L3 cache
- * @slot:  slot number (0..1)
- * @index: index to disable
- *
- * @return: 0 on success, error status on failure
- */
-static int amd_set_l3_disable_slot(struct amd_northbridge *nb, int cpu,
-			    unsigned slot, unsigned long index)
-{
-	int ret = 0;
-
-	/*  check if @slot is already used or the index is already disabled */
-	ret = amd_get_l3_disable_slot(nb, slot);
-	if (ret >= 0)
-		return -EEXIST;
-
-	if (index > nb->l3_cache.indices)
-		return -EINVAL;
-
-	/* check whether the other slot has disabled the same index already */
-	if (index == amd_get_l3_disable_slot(nb, !slot))
-		return -EEXIST;
-
-	amd_l3_disable_index(nb, cpu, slot, index);
-
-	return 0;
-}
-
-static ssize_t store_cache_disable(struct cacheinfo *this_leaf,
-				   const char *buf, size_t count,
-				   unsigned int slot)
-{
-	unsigned long val = 0;
-	int cpu, err = 0;
-	struct amd_northbridge *nb = this_leaf->priv;
-
-	if (!capable(CAP_SYS_ADMIN))
-		return -EPERM;
-
-	cpu = cpumask_first(&this_leaf->shared_cpu_map);
-
-	if (kstrtoul(buf, 10, &val) < 0)
-		return -EINVAL;
-
-	err = amd_set_l3_disable_slot(nb, cpu, slot, val);
-	if (err) {
-		if (err == -EEXIST)
-			pr_warn("L3 slot %d in use/index already disabled!\n",
-				   slot);
-		return err;
-	}
-	return count;
-}
-
-#define STORE_CACHE_DISABLE(slot)					\
-static ssize_t								\
-cache_disable_##slot##_store(struct device *dev,			\
-			     struct device_attribute *attr,		\
-			     const char *buf, size_t count)		\
-{									\
-	struct cacheinfo *this_leaf = dev_get_drvdata(dev);		\
-	return store_cache_disable(this_leaf, buf, count, slot);	\
-}
-STORE_CACHE_DISABLE(0)
-STORE_CACHE_DISABLE(1)
-
-static ssize_t subcaches_show(struct device *dev,
-			      struct device_attribute *attr, char *buf)
-{
-	struct cacheinfo *this_leaf = dev_get_drvdata(dev);
-	int cpu = cpumask_first(&this_leaf->shared_cpu_map);
-
-	return sprintf(buf, "%x\n", amd_get_subcaches(cpu));
-}
-
-static ssize_t subcaches_store(struct device *dev,
-			       struct device_attribute *attr,
-			       const char *buf, size_t count)
-{
-	struct cacheinfo *this_leaf = dev_get_drvdata(dev);
-	int cpu = cpumask_first(&this_leaf->shared_cpu_map);
-	unsigned long val;
-
-	if (!capable(CAP_SYS_ADMIN))
-		return -EPERM;
-
-	if (kstrtoul(buf, 16, &val) < 0)
-		return -EINVAL;
-
-	if (amd_set_subcaches(cpu, val))
-		return -EINVAL;
-
-	return count;
-}
-
-static DEVICE_ATTR_RW(cache_disable_0);
-static DEVICE_ATTR_RW(cache_disable_1);
-static DEVICE_ATTR_RW(subcaches);
-
-static umode_t
-cache_private_attrs_is_visible(struct kobject *kobj,
-			       struct attribute *attr, int unused)
-{
-	struct device *dev = kobj_to_dev(kobj);
-	struct cacheinfo *this_leaf = dev_get_drvdata(dev);
-	umode_t mode = attr->mode;
-
-	if (!this_leaf->priv)
-		return 0;
-
-	if ((attr == &dev_attr_subcaches.attr) &&
-	    amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
-		return mode;
-
-	if ((attr == &dev_attr_cache_disable_0.attr ||
-	     attr == &dev_attr_cache_disable_1.attr) &&
-	    amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE))
-		return mode;
-
-	return 0;
-}
-
-static struct attribute_group cache_private_group = {
-	.is_visible = cache_private_attrs_is_visible,
-};
-
-static void init_amd_l3_attrs(void)
-{
-	int n = 1;
-	static struct attribute **amd_l3_attrs;
-
-	if (amd_l3_attrs) /* already initialized */
-		return;
-
-	if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE))
-		n += 2;
-	if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
-		n += 1;
-
-	amd_l3_attrs = kcalloc(n, sizeof(*amd_l3_attrs), GFP_KERNEL);
-	if (!amd_l3_attrs)
-		return;
-
-	n = 0;
-	if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE)) {
-		amd_l3_attrs[n++] = &dev_attr_cache_disable_0.attr;
-		amd_l3_attrs[n++] = &dev_attr_cache_disable_1.attr;
-	}
-	if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
-		amd_l3_attrs[n++] = &dev_attr_subcaches.attr;
-
-	cache_private_group.attrs = amd_l3_attrs;
-}
-
-const struct attribute_group *
-cache_get_priv_group(struct cacheinfo *this_leaf)
-{
-	struct amd_northbridge *nb = this_leaf->priv;
-
-	if (this_leaf->level < 3 || !nb)
-		return NULL;
-
-	if (nb && nb->l3_cache.indices)
-		init_amd_l3_attrs();
-
-	return &cache_private_group;
-}
-
-static void amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index)
-{
-	int node;
-
-	/* only for L3, and not in virtualized environments */
-	if (index < 3)
-		return;
-
-	node = amd_get_nb_id(smp_processor_id());
-	this_leaf->nb = node_to_amd_nb(node);
-	if (this_leaf->nb && !this_leaf->nb->l3_cache.indices)
-		amd_calc_l3_indices(this_leaf->nb);
-}
-#else
-#define amd_init_l3_cache(x, y)
-#endif  /* CONFIG_AMD_NB && CONFIG_SYSFS */
-
-static int
-cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *this_leaf)
-{
-	union _cpuid4_leaf_eax	eax;
-	union _cpuid4_leaf_ebx	ebx;
-	union _cpuid4_leaf_ecx	ecx;
-	unsigned		edx;
-
-	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
-		if (boot_cpu_has(X86_FEATURE_TOPOEXT))
-			cpuid_count(0x8000001d, index, &eax.full,
-				    &ebx.full, &ecx.full, &edx);
-		else
-			amd_cpuid4(index, &eax, &ebx, &ecx);
-		amd_init_l3_cache(this_leaf, index);
-	} else {
-		cpuid_count(4, index, &eax.full, &ebx.full, &ecx.full, &edx);
-	}
-
-	if (eax.split.type == CTYPE_NULL)
-		return -EIO; /* better error ? */
-
-	this_leaf->eax = eax;
-	this_leaf->ebx = ebx;
-	this_leaf->ecx = ecx;
-	this_leaf->size = (ecx.split.number_of_sets          + 1) *
-			  (ebx.split.coherency_line_size     + 1) *
-			  (ebx.split.physical_line_partition + 1) *
-			  (ebx.split.ways_of_associativity   + 1);
-	return 0;
-}
-
-static int find_num_cache_leaves(struct cpuinfo_x86 *c)
-{
-	unsigned int		eax, ebx, ecx, edx, op;
-	union _cpuid4_leaf_eax	cache_eax;
-	int 			i = -1;
-
-	if (c->x86_vendor == X86_VENDOR_AMD)
-		op = 0x8000001d;
-	else
-		op = 4;
-
-	do {
-		++i;
-		/* Do cpuid(op) loop to find out num_cache_leaves */
-		cpuid_count(op, i, &eax, &ebx, &ecx, &edx);
-		cache_eax.full = eax;
-	} while (cache_eax.split.type != CTYPE_NULL);
-	return i;
-}
-
-void init_amd_cacheinfo(struct cpuinfo_x86 *c)
-{
-
-	if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
-		num_cache_leaves = find_num_cache_leaves(c);
-	} else if (c->extended_cpuid_level >= 0x80000006) {
-		if (cpuid_edx(0x80000006) & 0xf000)
-			num_cache_leaves = 4;
-		else
-			num_cache_leaves = 3;
-	}
-}
-
-unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c)
-{
-	/* Cache sizes */
-	unsigned int trace = 0, l1i = 0, l1d = 0, l2 = 0, l3 = 0;
-	unsigned int new_l1d = 0, new_l1i = 0; /* Cache sizes from cpuid(4) */
-	unsigned int new_l2 = 0, new_l3 = 0, i; /* Cache sizes from cpuid(4) */
-	unsigned int l2_id = 0, l3_id = 0, num_threads_sharing, index_msb;
-#ifdef CONFIG_SMP
-	unsigned int cpu = c->cpu_index;
-#endif
-
-	if (c->cpuid_level > 3) {
-		static int is_initialized;
-
-		if (is_initialized == 0) {
-			/* Init num_cache_leaves from boot CPU */
-			num_cache_leaves = find_num_cache_leaves(c);
-			is_initialized++;
-		}
-
-		/*
-		 * Whenever possible use cpuid(4), deterministic cache
-		 * parameters cpuid leaf to find the cache details
-		 */
-		for (i = 0; i < num_cache_leaves; i++) {
-			struct _cpuid4_info_regs this_leaf = {};
-			int retval;
-
-			retval = cpuid4_cache_lookup_regs(i, &this_leaf);
-			if (retval < 0)
-				continue;
-
-			switch (this_leaf.eax.split.level) {
-			case 1:
-				if (this_leaf.eax.split.type == CTYPE_DATA)
-					new_l1d = this_leaf.size/1024;
-				else if (this_leaf.eax.split.type == CTYPE_INST)
-					new_l1i = this_leaf.size/1024;
-				break;
-			case 2:
-				new_l2 = this_leaf.size/1024;
-				num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing;
-				index_msb = get_count_order(num_threads_sharing);
-				l2_id = c->apicid & ~((1 << index_msb) - 1);
-				break;
-			case 3:
-				new_l3 = this_leaf.size/1024;
-				num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing;
-				index_msb = get_count_order(num_threads_sharing);
-				l3_id = c->apicid & ~((1 << index_msb) - 1);
-				break;
-			default:
-				break;
-			}
-		}
-	}
-	/*
-	 * Don't use cpuid2 if cpuid4 is supported. For P4, we use cpuid2 for
-	 * trace cache
-	 */
-	if ((num_cache_leaves == 0 || c->x86 == 15) && c->cpuid_level > 1) {
-		/* supports eax=2  call */
-		int j, n;
-		unsigned int regs[4];
-		unsigned char *dp = (unsigned char *)regs;
-		int only_trace = 0;
-
-		if (num_cache_leaves != 0 && c->x86 == 15)
-			only_trace = 1;
-
-		/* Number of times to iterate */
-		n = cpuid_eax(2) & 0xFF;
-
-		for (i = 0 ; i < n ; i++) {
-			cpuid(2, &regs[0], &regs[1], &regs[2], &regs[3]);
-
-			/* If bit 31 is set, this is an unknown format */
-			for (j = 0 ; j < 3 ; j++)
-				if (regs[j] & (1 << 31))
-					regs[j] = 0;
-
-			/* Byte 0 is level count, not a descriptor */
-			for (j = 1 ; j < 16 ; j++) {
-				unsigned char des = dp[j];
-				unsigned char k = 0;
-
-				/* look up this descriptor in the table */
-				while (cache_table[k].descriptor != 0) {
-					if (cache_table[k].descriptor == des) {
-						if (only_trace && cache_table[k].cache_type != LVL_TRACE)
-							break;
-						switch (cache_table[k].cache_type) {
-						case LVL_1_INST:
-							l1i += cache_table[k].size;
-							break;
-						case LVL_1_DATA:
-							l1d += cache_table[k].size;
-							break;
-						case LVL_2:
-							l2 += cache_table[k].size;
-							break;
-						case LVL_3:
-							l3 += cache_table[k].size;
-							break;
-						case LVL_TRACE:
-							trace += cache_table[k].size;
-							break;
-						}
-
-						break;
-					}
-
-					k++;
-				}
-			}
-		}
-	}
-
-	if (new_l1d)
-		l1d = new_l1d;
-
-	if (new_l1i)
-		l1i = new_l1i;
-
-	if (new_l2) {
-		l2 = new_l2;
-#ifdef CONFIG_SMP
-		per_cpu(cpu_llc_id, cpu) = l2_id;
-#endif
-	}
-
-	if (new_l3) {
-		l3 = new_l3;
-#ifdef CONFIG_SMP
-		per_cpu(cpu_llc_id, cpu) = l3_id;
-#endif
-	}
-
-#ifdef CONFIG_SMP
-	/*
-	 * If cpu_llc_id is not yet set, this means cpuid_level < 4 which in
-	 * turns means that the only possibility is SMT (as indicated in
-	 * cpuid1). Since cpuid2 doesn't specify shared caches, and we know
-	 * that SMT shares all caches, we can unconditionally set cpu_llc_id to
-	 * c->phys_proc_id.
-	 */
-	if (per_cpu(cpu_llc_id, cpu) == BAD_APICID)
-		per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
-#endif
-
-	c->x86_cache_size = l3 ? l3 : (l2 ? l2 : (l1i+l1d));
-
-	return l2;
-}
-
-static int __cache_amd_cpumap_setup(unsigned int cpu, int index,
-				    struct _cpuid4_info_regs *base)
-{
-	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
-	struct cacheinfo *this_leaf;
-	int i, sibling;
-
-	/*
-	 * For L3, always use the pre-calculated cpu_llc_shared_mask
-	 * to derive shared_cpu_map.
-	 */
-	if (index == 3) {
-		for_each_cpu(i, cpu_llc_shared_mask(cpu)) {
-			this_cpu_ci = get_cpu_cacheinfo(i);
-			if (!this_cpu_ci->info_list)
-				continue;
-			this_leaf = this_cpu_ci->info_list + index;
-			for_each_cpu(sibling, cpu_llc_shared_mask(cpu)) {
-				if (!cpu_online(sibling))
-					continue;
-				cpumask_set_cpu(sibling,
-						&this_leaf->shared_cpu_map);
-			}
-		}
-	} else if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
-		unsigned int apicid, nshared, first, last;
-
-		nshared = base->eax.split.num_threads_sharing + 1;
-		apicid = cpu_data(cpu).apicid;
-		first = apicid - (apicid % nshared);
-		last = first + nshared - 1;
-
-		for_each_online_cpu(i) {
-			this_cpu_ci = get_cpu_cacheinfo(i);
-			if (!this_cpu_ci->info_list)
-				continue;
-
-			apicid = cpu_data(i).apicid;
-			if ((apicid < first) || (apicid > last))
-				continue;
-
-			this_leaf = this_cpu_ci->info_list + index;
-
-			for_each_online_cpu(sibling) {
-				apicid = cpu_data(sibling).apicid;
-				if ((apicid < first) || (apicid > last))
-					continue;
-				cpumask_set_cpu(sibling,
-						&this_leaf->shared_cpu_map);
-			}
-		}
-	} else
-		return 0;
-
-	return 1;
-}
-
-static void __cache_cpumap_setup(unsigned int cpu, int index,
-				 struct _cpuid4_info_regs *base)
-{
-	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
-	struct cacheinfo *this_leaf, *sibling_leaf;
-	unsigned long num_threads_sharing;
-	int index_msb, i;
-	struct cpuinfo_x86 *c = &cpu_data(cpu);
-
-	if (c->x86_vendor == X86_VENDOR_AMD) {
-		if (__cache_amd_cpumap_setup(cpu, index, base))
-			return;
-	}
-
-	this_leaf = this_cpu_ci->info_list + index;
-	num_threads_sharing = 1 + base->eax.split.num_threads_sharing;
-
-	cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map);
-	if (num_threads_sharing == 1)
-		return;
-
-	index_msb = get_count_order(num_threads_sharing);
-
-	for_each_online_cpu(i)
-		if (cpu_data(i).apicid >> index_msb == c->apicid >> index_msb) {
-			struct cpu_cacheinfo *sib_cpu_ci = get_cpu_cacheinfo(i);
-
-			if (i == cpu || !sib_cpu_ci->info_list)
-				continue;/* skip if itself or no cacheinfo */
-			sibling_leaf = sib_cpu_ci->info_list + index;
-			cpumask_set_cpu(i, &this_leaf->shared_cpu_map);
-			cpumask_set_cpu(cpu, &sibling_leaf->shared_cpu_map);
-		}
-}
-
-static void ci_leaf_init(struct cacheinfo *this_leaf,
-			 struct _cpuid4_info_regs *base)
-{
-	this_leaf->id = base->id;
-	this_leaf->attributes = CACHE_ID;
-	this_leaf->level = base->eax.split.level;
-	this_leaf->type = cache_type_map[base->eax.split.type];
-	this_leaf->coherency_line_size =
-				base->ebx.split.coherency_line_size + 1;
-	this_leaf->ways_of_associativity =
-				base->ebx.split.ways_of_associativity + 1;
-	this_leaf->size = base->size;
-	this_leaf->number_of_sets = base->ecx.split.number_of_sets + 1;
-	this_leaf->physical_line_partition =
-				base->ebx.split.physical_line_partition + 1;
-	this_leaf->priv = base->nb;
-}
-
-static int __init_cache_level(unsigned int cpu)
-{
-	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
-
-	if (!num_cache_leaves)
-		return -ENOENT;
-	if (!this_cpu_ci)
-		return -EINVAL;
-	this_cpu_ci->num_levels = 3;
-	this_cpu_ci->num_leaves = num_cache_leaves;
-	return 0;
-}
-
-/*
- * The max shared threads number comes from CPUID.4:EAX[25-14] with input
- * ECX as cache index. Then right shift apicid by the number's order to get
- * cache id for this cache node.
- */
-static void get_cache_id(int cpu, struct _cpuid4_info_regs *id4_regs)
-{
-	struct cpuinfo_x86 *c = &cpu_data(cpu);
-	unsigned long num_threads_sharing;
-	int index_msb;
-
-	num_threads_sharing = 1 + id4_regs->eax.split.num_threads_sharing;
-	index_msb = get_count_order(num_threads_sharing);
-	id4_regs->id = c->apicid >> index_msb;
-}
-
-static int __populate_cache_leaves(unsigned int cpu)
-{
-	unsigned int idx, ret;
-	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
-	struct cacheinfo *this_leaf = this_cpu_ci->info_list;
-	struct _cpuid4_info_regs id4_regs = {};
-
-	for (idx = 0; idx < this_cpu_ci->num_leaves; idx++) {
-		ret = cpuid4_cache_lookup_regs(idx, &id4_regs);
-		if (ret)
-			return ret;
-		get_cache_id(cpu, &id4_regs);
-		ci_leaf_init(this_leaf++, &id4_regs);
-		__cache_cpumap_setup(cpu, idx, &id4_regs);
-	}
-	this_cpu_ci->cpu_map_populated = true;
-
-	return 0;
-}
-
-DEFINE_SMP_CALL_CACHE_FUNCTION(init_cache_level)
-DEFINE_SMP_CALL_CACHE_FUNCTION(populate_cache_leaves)
diff --git a/arch/x86/kernel/cpu/mcheck/mce-inject.c b/arch/x86/kernel/cpu/mcheck/mce-inject.c
index 475cb4f5f14f..c805a06e14c3 100644
--- a/arch/x86/kernel/cpu/mcheck/mce-inject.c
+++ b/arch/x86/kernel/cpu/mcheck/mce-inject.c
@@ -48,7 +48,7 @@ static struct dentry *dfs_inj;
 
 static u8 n_banks;
 
-#define MAX_FLAG_OPT_SIZE	3
+#define MAX_FLAG_OPT_SIZE	4
 #define NBCFG			0x44
 
 enum injection_type {
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 42cf2880d0ed..cd76380af79f 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -1727,6 +1727,21 @@ static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c)
 	}
 }
 
+static void mce_centaur_feature_init(struct cpuinfo_x86 *c)
+{
+	struct mca_config *cfg = &mca_cfg;
+
+	 /*
+	  * All newer Centaur CPUs support MCE broadcasting. Enable
+	  * synchronization with a one second timeout.
+	  */
+	if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) ||
+	     c->x86 > 6) {
+		if (cfg->monarch_timeout < 0)
+			cfg->monarch_timeout = USEC_PER_SEC;
+	}
+}
+
 static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
 {
 	switch (c->x86_vendor) {
@@ -1739,6 +1754,9 @@ static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
 		mce_amd_feature_init(c);
 		break;
 		}
+	case X86_VENDOR_CENTAUR:
+		mce_centaur_feature_init(c);
+		break;
 
 	default:
 		break;
diff --git a/arch/x86/kernel/cpu/mtrr/Makefile b/arch/x86/kernel/cpu/mtrr/Makefile
index ad9e5ed81181..2ad9107ee980 100644
--- a/arch/x86/kernel/cpu/mtrr/Makefile
+++ b/arch/x86/kernel/cpu/mtrr/Makefile
@@ -1,3 +1,3 @@
-obj-y		:= main.o if.o generic.o cleanup.o
+obj-y		:= mtrr.o if.o generic.o cleanup.o
 obj-$(CONFIG_X86_32) += amd.o cyrix.o centaur.o
 
diff --git a/arch/x86/kernel/cpu/mtrr/if.c b/arch/x86/kernel/cpu/mtrr/if.c
index 558444b23923..c610f47373e4 100644
--- a/arch/x86/kernel/cpu/mtrr/if.c
+++ b/arch/x86/kernel/cpu/mtrr/if.c
@@ -106,17 +106,9 @@ mtrr_write(struct file *file, const char __user *buf, size_t len, loff_t * ppos)
 
 	memset(line, 0, LINE_SIZE);
 
-	length = len;
-	length--;
-
-	if (length > LINE_SIZE - 1)
-		length = LINE_SIZE - 1;
-
+	length = strncpy_from_user(line, buf, LINE_SIZE - 1);
 	if (length < 0)
-		return -EINVAL;
-
-	if (copy_from_user(line, buf, length))
-		return -EFAULT;
+		return length;
 
 	linelen = strlen(line);
 	ptr = line + linelen - 1;
@@ -149,17 +141,16 @@ mtrr_write(struct file *file, const char __user *buf, size_t len, loff_t * ppos)
 		return -EINVAL;
 	ptr = skip_spaces(ptr + 5);
 
-	for (i = 0; i < MTRR_NUM_TYPES; ++i) {
-		if (strcmp(ptr, mtrr_strings[i]))
-			continue;
-		base >>= PAGE_SHIFT;
-		size >>= PAGE_SHIFT;
-		err = mtrr_add_page((unsigned long)base, (unsigned long)size, i, true);
-		if (err < 0)
-			return err;
-		return len;
-	}
-	return -EINVAL;
+	i = match_string(mtrr_strings, MTRR_NUM_TYPES, ptr);
+	if (i < 0)
+		return i;
+
+	base >>= PAGE_SHIFT;
+	size >>= PAGE_SHIFT;
+	err = mtrr_add_page((unsigned long)base, (unsigned long)size, i, true);
+	if (err < 0)
+		return err;
+	return len;
 }
 
 static long
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
deleted file mode 100644
index 7468de429087..000000000000
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ /dev/null
@@ -1,887 +0,0 @@
-/*  Generic MTRR (Memory Type Range Register) driver.
-
-    Copyright (C) 1997-2000  Richard Gooch
-    Copyright (c) 2002	     Patrick Mochel
-
-    This library is free software; you can redistribute it and/or
-    modify it under the terms of the GNU Library General Public
-    License as published by the Free Software Foundation; either
-    version 2 of the License, or (at your option) any later version.
-
-    This library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-    Library General Public License for more details.
-
-    You should have received a copy of the GNU Library General Public
-    License along with this library; if not, write to the Free
-    Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-    Richard Gooch may be reached by email at  rgooch@atnf.csiro.au
-    The postal address is:
-      Richard Gooch, c/o ATNF, P. O. Box 76, Epping, N.S.W., 2121, Australia.
-
-    Source: "Pentium Pro Family Developer's Manual, Volume 3:
-    Operating System Writer's Guide" (Intel document number 242692),
-    section 11.11.7
-
-    This was cleaned and made readable by Patrick Mochel <mochel@osdl.org>
-    on 6-7 March 2002.
-    Source: Intel Architecture Software Developers Manual, Volume 3:
-    System Programming Guide; Section 9.11. (1997 edition - PPro).
-*/
-
-#define DEBUG
-
-#include <linux/types.h> /* FIXME: kvm_para.h needs this */
-
-#include <linux/stop_machine.h>
-#include <linux/kvm_para.h>
-#include <linux/uaccess.h>
-#include <linux/export.h>
-#include <linux/mutex.h>
-#include <linux/init.h>
-#include <linux/sort.h>
-#include <linux/cpu.h>
-#include <linux/pci.h>
-#include <linux/smp.h>
-#include <linux/syscore_ops.h>
-
-#include <asm/cpufeature.h>
-#include <asm/e820/api.h>
-#include <asm/mtrr.h>
-#include <asm/msr.h>
-#include <asm/pat.h>
-
-#include "mtrr.h"
-
-/* arch_phys_wc_add returns an MTRR register index plus this offset. */
-#define MTRR_TO_PHYS_WC_OFFSET 1000
-
-u32 num_var_ranges;
-static bool __mtrr_enabled;
-
-static bool mtrr_enabled(void)
-{
-	return __mtrr_enabled;
-}
-
-unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
-static DEFINE_MUTEX(mtrr_mutex);
-
-u64 size_or_mask, size_and_mask;
-static bool mtrr_aps_delayed_init;
-
-static const struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM] __ro_after_init;
-
-const struct mtrr_ops *mtrr_if;
-
-static void set_mtrr(unsigned int reg, unsigned long base,
-		     unsigned long size, mtrr_type type);
-
-void __init set_mtrr_ops(const struct mtrr_ops *ops)
-{
-	if (ops->vendor && ops->vendor < X86_VENDOR_NUM)
-		mtrr_ops[ops->vendor] = ops;
-}
-
-/*  Returns non-zero if we have the write-combining memory type  */
-static int have_wrcomb(void)
-{
-	struct pci_dev *dev;
-
-	dev = pci_get_class(PCI_CLASS_BRIDGE_HOST << 8, NULL);
-	if (dev != NULL) {
-		/*
-		 * ServerWorks LE chipsets < rev 6 have problems with
-		 * write-combining. Don't allow it and leave room for other
-		 * chipsets to be tagged
-		 */
-		if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS &&
-		    dev->device == PCI_DEVICE_ID_SERVERWORKS_LE &&
-		    dev->revision <= 5) {
-			pr_info("mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n");
-			pci_dev_put(dev);
-			return 0;
-		}
-		/*
-		 * Intel 450NX errata # 23. Non ascending cacheline evictions to
-		 * write combining memory may resulting in data corruption
-		 */
-		if (dev->vendor == PCI_VENDOR_ID_INTEL &&
-		    dev->device == PCI_DEVICE_ID_INTEL_82451NX) {
-			pr_info("mtrr: Intel 450NX MMC detected. Write-combining disabled.\n");
-			pci_dev_put(dev);
-			return 0;
-		}
-		pci_dev_put(dev);
-	}
-	return mtrr_if->have_wrcomb ? mtrr_if->have_wrcomb() : 0;
-}
-
-/*  This function returns the number of variable MTRRs  */
-static void __init set_num_var_ranges(void)
-{
-	unsigned long config = 0, dummy;
-
-	if (use_intel())
-		rdmsr(MSR_MTRRcap, config, dummy);
-	else if (is_cpu(AMD))
-		config = 2;
-	else if (is_cpu(CYRIX) || is_cpu(CENTAUR))
-		config = 8;
-
-	num_var_ranges = config & 0xff;
-}
-
-static void __init init_table(void)
-{
-	int i, max;
-
-	max = num_var_ranges;
-	for (i = 0; i < max; i++)
-		mtrr_usage_table[i] = 1;
-}
-
-struct set_mtrr_data {
-	unsigned long	smp_base;
-	unsigned long	smp_size;
-	unsigned int	smp_reg;
-	mtrr_type	smp_type;
-};
-
-/**
- * mtrr_rendezvous_handler - Work done in the synchronization handler. Executed
- * by all the CPUs.
- * @info: pointer to mtrr configuration data
- *
- * Returns nothing.
- */
-static int mtrr_rendezvous_handler(void *info)
-{
-	struct set_mtrr_data *data = info;
-
-	/*
-	 * We use this same function to initialize the mtrrs during boot,
-	 * resume, runtime cpu online and on an explicit request to set a
-	 * specific MTRR.
-	 *
-	 * During boot or suspend, the state of the boot cpu's mtrrs has been
-	 * saved, and we want to replicate that across all the cpus that come
-	 * online (either at the end of boot or resume or during a runtime cpu
-	 * online). If we're doing that, @reg is set to something special and on
-	 * all the cpu's we do mtrr_if->set_all() (On the logical cpu that
-	 * started the boot/resume sequence, this might be a duplicate
-	 * set_all()).
-	 */
-	if (data->smp_reg != ~0U) {
-		mtrr_if->set(data->smp_reg, data->smp_base,
-			     data->smp_size, data->smp_type);
-	} else if (mtrr_aps_delayed_init || !cpu_online(smp_processor_id())) {
-		mtrr_if->set_all();
-	}
-	return 0;
-}
-
-static inline int types_compatible(mtrr_type type1, mtrr_type type2)
-{
-	return type1 == MTRR_TYPE_UNCACHABLE ||
-	       type2 == MTRR_TYPE_UNCACHABLE ||
-	       (type1 == MTRR_TYPE_WRTHROUGH && type2 == MTRR_TYPE_WRBACK) ||
-	       (type1 == MTRR_TYPE_WRBACK && type2 == MTRR_TYPE_WRTHROUGH);
-}
-
-/**
- * set_mtrr - update mtrrs on all processors
- * @reg:	mtrr in question
- * @base:	mtrr base
- * @size:	mtrr size
- * @type:	mtrr type
- *
- * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly:
- *
- * 1. Queue work to do the following on all processors:
- * 2. Disable Interrupts
- * 3. Wait for all procs to do so
- * 4. Enter no-fill cache mode
- * 5. Flush caches
- * 6. Clear PGE bit
- * 7. Flush all TLBs
- * 8. Disable all range registers
- * 9. Update the MTRRs
- * 10. Enable all range registers
- * 11. Flush all TLBs and caches again
- * 12. Enter normal cache mode and reenable caching
- * 13. Set PGE
- * 14. Wait for buddies to catch up
- * 15. Enable interrupts.
- *
- * What does that mean for us? Well, stop_machine() will ensure that
- * the rendezvous handler is started on each CPU. And in lockstep they
- * do the state transition of disabling interrupts, updating MTRR's
- * (the CPU vendors may each do it differently, so we call mtrr_if->set()
- * callback and let them take care of it.) and enabling interrupts.
- *
- * Note that the mechanism is the same for UP systems, too; all the SMP stuff
- * becomes nops.
- */
-static void
-set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type)
-{
-	struct set_mtrr_data data = { .smp_reg = reg,
-				      .smp_base = base,
-				      .smp_size = size,
-				      .smp_type = type
-				    };
-
-	stop_machine(mtrr_rendezvous_handler, &data, cpu_online_mask);
-}
-
-static void set_mtrr_cpuslocked(unsigned int reg, unsigned long base,
-				unsigned long size, mtrr_type type)
-{
-	struct set_mtrr_data data = { .smp_reg = reg,
-				      .smp_base = base,
-				      .smp_size = size,
-				      .smp_type = type
-				    };
-
-	stop_machine_cpuslocked(mtrr_rendezvous_handler, &data, cpu_online_mask);
-}
-
-static void set_mtrr_from_inactive_cpu(unsigned int reg, unsigned long base,
-				      unsigned long size, mtrr_type type)
-{
-	struct set_mtrr_data data = { .smp_reg = reg,
-				      .smp_base = base,
-				      .smp_size = size,
-				      .smp_type = type
-				    };
-
-	stop_machine_from_inactive_cpu(mtrr_rendezvous_handler, &data,
-				       cpu_callout_mask);
-}
-
-/**
- * mtrr_add_page - Add a memory type region
- * @base: Physical base address of region in pages (in units of 4 kB!)
- * @size: Physical size of region in pages (4 kB)
- * @type: Type of MTRR desired
- * @increment: If this is true do usage counting on the region
- *
- * Memory type region registers control the caching on newer Intel and
- * non Intel processors. This function allows drivers to request an
- * MTRR is added. The details and hardware specifics of each processor's
- * implementation are hidden from the caller, but nevertheless the
- * caller should expect to need to provide a power of two size on an
- * equivalent power of two boundary.
- *
- * If the region cannot be added either because all regions are in use
- * or the CPU cannot support it a negative value is returned. On success
- * the register number for this entry is returned, but should be treated
- * as a cookie only.
- *
- * On a multiprocessor machine the changes are made to all processors.
- * This is required on x86 by the Intel processors.
- *
- * The available types are
- *
- * %MTRR_TYPE_UNCACHABLE - No caching
- *
- * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
- *
- * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
- *
- * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
- *
- * BUGS: Needs a quiet flag for the cases where drivers do not mind
- * failures and do not wish system log messages to be sent.
- */
-int mtrr_add_page(unsigned long base, unsigned long size,
-		  unsigned int type, bool increment)
-{
-	unsigned long lbase, lsize;
-	int i, replace, error;
-	mtrr_type ltype;
-
-	if (!mtrr_enabled())
-		return -ENXIO;
-
-	error = mtrr_if->validate_add_page(base, size, type);
-	if (error)
-		return error;
-
-	if (type >= MTRR_NUM_TYPES) {
-		pr_warn("mtrr: type: %u invalid\n", type);
-		return -EINVAL;
-	}
-
-	/* If the type is WC, check that this processor supports it */
-	if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) {
-		pr_warn("mtrr: your processor doesn't support write-combining\n");
-		return -ENOSYS;
-	}
-
-	if (!size) {
-		pr_warn("mtrr: zero sized request\n");
-		return -EINVAL;
-	}
-
-	if ((base | (base + size - 1)) >>
-	    (boot_cpu_data.x86_phys_bits - PAGE_SHIFT)) {
-		pr_warn("mtrr: base or size exceeds the MTRR width\n");
-		return -EINVAL;
-	}
-
-	error = -EINVAL;
-	replace = -1;
-
-	/* No CPU hotplug when we change MTRR entries */
-	get_online_cpus();
-
-	/* Search for existing MTRR  */
-	mutex_lock(&mtrr_mutex);
-	for (i = 0; i < num_var_ranges; ++i) {
-		mtrr_if->get(i, &lbase, &lsize, &ltype);
-		if (!lsize || base > lbase + lsize - 1 ||
-		    base + size - 1 < lbase)
-			continue;
-		/*
-		 * At this point we know there is some kind of
-		 * overlap/enclosure
-		 */
-		if (base < lbase || base + size - 1 > lbase + lsize - 1) {
-			if (base <= lbase &&
-			    base + size - 1 >= lbase + lsize - 1) {
-				/*  New region encloses an existing region  */
-				if (type == ltype) {
-					replace = replace == -1 ? i : -2;
-					continue;
-				} else if (types_compatible(type, ltype))
-					continue;
-			}
-			pr_warn("mtrr: 0x%lx000,0x%lx000 overlaps existing"
-				" 0x%lx000,0x%lx000\n", base, size, lbase,
-				lsize);
-			goto out;
-		}
-		/* New region is enclosed by an existing region */
-		if (ltype != type) {
-			if (types_compatible(type, ltype))
-				continue;
-			pr_warn("mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n",
-				base, size, mtrr_attrib_to_str(ltype),
-				mtrr_attrib_to_str(type));
-			goto out;
-		}
-		if (increment)
-			++mtrr_usage_table[i];
-		error = i;
-		goto out;
-	}
-	/* Search for an empty MTRR */
-	i = mtrr_if->get_free_region(base, size, replace);
-	if (i >= 0) {
-		set_mtrr_cpuslocked(i, base, size, type);
-		if (likely(replace < 0)) {
-			mtrr_usage_table[i] = 1;
-		} else {
-			mtrr_usage_table[i] = mtrr_usage_table[replace];
-			if (increment)
-				mtrr_usage_table[i]++;
-			if (unlikely(replace != i)) {
-				set_mtrr_cpuslocked(replace, 0, 0, 0);
-				mtrr_usage_table[replace] = 0;
-			}
-		}
-	} else {
-		pr_info("mtrr: no more MTRRs available\n");
-	}
-	error = i;
- out:
-	mutex_unlock(&mtrr_mutex);
-	put_online_cpus();
-	return error;
-}
-
-static int mtrr_check(unsigned long base, unsigned long size)
-{
-	if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
-		pr_warn("mtrr: size and base must be multiples of 4 kiB\n");
-		pr_debug("mtrr: size: 0x%lx  base: 0x%lx\n", size, base);
-		dump_stack();
-		return -1;
-	}
-	return 0;
-}
-
-/**
- * mtrr_add - Add a memory type region
- * @base: Physical base address of region
- * @size: Physical size of region
- * @type: Type of MTRR desired
- * @increment: If this is true do usage counting on the region
- *
- * Memory type region registers control the caching on newer Intel and
- * non Intel processors. This function allows drivers to request an
- * MTRR is added. The details and hardware specifics of each processor's
- * implementation are hidden from the caller, but nevertheless the
- * caller should expect to need to provide a power of two size on an
- * equivalent power of two boundary.
- *
- * If the region cannot be added either because all regions are in use
- * or the CPU cannot support it a negative value is returned. On success
- * the register number for this entry is returned, but should be treated
- * as a cookie only.
- *
- * On a multiprocessor machine the changes are made to all processors.
- * This is required on x86 by the Intel processors.
- *
- * The available types are
- *
- * %MTRR_TYPE_UNCACHABLE - No caching
- *
- * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
- *
- * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
- *
- * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
- *
- * BUGS: Needs a quiet flag for the cases where drivers do not mind
- * failures and do not wish system log messages to be sent.
- */
-int mtrr_add(unsigned long base, unsigned long size, unsigned int type,
-	     bool increment)
-{
-	if (!mtrr_enabled())
-		return -ENODEV;
-	if (mtrr_check(base, size))
-		return -EINVAL;
-	return mtrr_add_page(base >> PAGE_SHIFT, size >> PAGE_SHIFT, type,
-			     increment);
-}
-
-/**
- * mtrr_del_page - delete a memory type region
- * @reg: Register returned by mtrr_add
- * @base: Physical base address
- * @size: Size of region
- *
- * If register is supplied then base and size are ignored. This is
- * how drivers should call it.
- *
- * Releases an MTRR region. If the usage count drops to zero the
- * register is freed and the region returns to default state.
- * On success the register is returned, on failure a negative error
- * code.
- */
-int mtrr_del_page(int reg, unsigned long base, unsigned long size)
-{
-	int i, max;
-	mtrr_type ltype;
-	unsigned long lbase, lsize;
-	int error = -EINVAL;
-
-	if (!mtrr_enabled())
-		return -ENODEV;
-
-	max = num_var_ranges;
-	/* No CPU hotplug when we change MTRR entries */
-	get_online_cpus();
-	mutex_lock(&mtrr_mutex);
-	if (reg < 0) {
-		/*  Search for existing MTRR  */
-		for (i = 0; i < max; ++i) {
-			mtrr_if->get(i, &lbase, &lsize, &ltype);
-			if (lbase == base && lsize == size) {
-				reg = i;
-				break;
-			}
-		}
-		if (reg < 0) {
-			pr_debug("mtrr: no MTRR for %lx000,%lx000 found\n",
-				 base, size);
-			goto out;
-		}
-	}
-	if (reg >= max) {
-		pr_warn("mtrr: register: %d too big\n", reg);
-		goto out;
-	}
-	mtrr_if->get(reg, &lbase, &lsize, &ltype);
-	if (lsize < 1) {
-		pr_warn("mtrr: MTRR %d not used\n", reg);
-		goto out;
-	}
-	if (mtrr_usage_table[reg] < 1) {
-		pr_warn("mtrr: reg: %d has count=0\n", reg);
-		goto out;
-	}
-	if (--mtrr_usage_table[reg] < 1)
-		set_mtrr_cpuslocked(reg, 0, 0, 0);
-	error = reg;
- out:
-	mutex_unlock(&mtrr_mutex);
-	put_online_cpus();
-	return error;
-}
-
-/**
- * mtrr_del - delete a memory type region
- * @reg: Register returned by mtrr_add
- * @base: Physical base address
- * @size: Size of region
- *
- * If register is supplied then base and size are ignored. This is
- * how drivers should call it.
- *
- * Releases an MTRR region. If the usage count drops to zero the
- * register is freed and the region returns to default state.
- * On success the register is returned, on failure a negative error
- * code.
- */
-int mtrr_del(int reg, unsigned long base, unsigned long size)
-{
-	if (!mtrr_enabled())
-		return -ENODEV;
-	if (mtrr_check(base, size))
-		return -EINVAL;
-	return mtrr_del_page(reg, base >> PAGE_SHIFT, size >> PAGE_SHIFT);
-}
-
-/**
- * arch_phys_wc_add - add a WC MTRR and handle errors if PAT is unavailable
- * @base: Physical base address
- * @size: Size of region
- *
- * If PAT is available, this does nothing.  If PAT is unavailable, it
- * attempts to add a WC MTRR covering size bytes starting at base and
- * logs an error if this fails.
- *
- * The called should provide a power of two size on an equivalent
- * power of two boundary.
- *
- * Drivers must store the return value to pass to mtrr_del_wc_if_needed,
- * but drivers should not try to interpret that return value.
- */
-int arch_phys_wc_add(unsigned long base, unsigned long size)
-{
-	int ret;
-
-	if (pat_enabled() || !mtrr_enabled())
-		return 0;  /* Success!  (We don't need to do anything.) */
-
-	ret = mtrr_add(base, size, MTRR_TYPE_WRCOMB, true);
-	if (ret < 0) {
-		pr_warn("Failed to add WC MTRR for [%p-%p]; performance may suffer.",
-			(void *)base, (void *)(base + size - 1));
-		return ret;
-	}
-	return ret + MTRR_TO_PHYS_WC_OFFSET;
-}
-EXPORT_SYMBOL(arch_phys_wc_add);
-
-/*
- * arch_phys_wc_del - undoes arch_phys_wc_add
- * @handle: Return value from arch_phys_wc_add
- *
- * This cleans up after mtrr_add_wc_if_needed.
- *
- * The API guarantees that mtrr_del_wc_if_needed(error code) and
- * mtrr_del_wc_if_needed(0) do nothing.
- */
-void arch_phys_wc_del(int handle)
-{
-	if (handle >= 1) {
-		WARN_ON(handle < MTRR_TO_PHYS_WC_OFFSET);
-		mtrr_del(handle - MTRR_TO_PHYS_WC_OFFSET, 0, 0);
-	}
-}
-EXPORT_SYMBOL(arch_phys_wc_del);
-
-/*
- * arch_phys_wc_index - translates arch_phys_wc_add's return value
- * @handle: Return value from arch_phys_wc_add
- *
- * This will turn the return value from arch_phys_wc_add into an mtrr
- * index suitable for debugging.
- *
- * Note: There is no legitimate use for this function, except possibly
- * in printk line.  Alas there is an illegitimate use in some ancient
- * drm ioctls.
- */
-int arch_phys_wc_index(int handle)
-{
-	if (handle < MTRR_TO_PHYS_WC_OFFSET)
-		return -1;
-	else
-		return handle - MTRR_TO_PHYS_WC_OFFSET;
-}
-EXPORT_SYMBOL_GPL(arch_phys_wc_index);
-
-/*
- * HACK ALERT!
- * These should be called implicitly, but we can't yet until all the initcall
- * stuff is done...
- */
-static void __init init_ifs(void)
-{
-#ifndef CONFIG_X86_64
-	amd_init_mtrr();
-	cyrix_init_mtrr();
-	centaur_init_mtrr();
-#endif
-}
-
-/* The suspend/resume methods are only for CPU without MTRR. CPU using generic
- * MTRR driver doesn't require this
- */
-struct mtrr_value {
-	mtrr_type	ltype;
-	unsigned long	lbase;
-	unsigned long	lsize;
-};
-
-static struct mtrr_value mtrr_value[MTRR_MAX_VAR_RANGES];
-
-static int mtrr_save(void)
-{
-	int i;
-
-	for (i = 0; i < num_var_ranges; i++) {
-		mtrr_if->get(i, &mtrr_value[i].lbase,
-				&mtrr_value[i].lsize,
-				&mtrr_value[i].ltype);
-	}
-	return 0;
-}
-
-static void mtrr_restore(void)
-{
-	int i;
-
-	for (i = 0; i < num_var_ranges; i++) {
-		if (mtrr_value[i].lsize) {
-			set_mtrr(i, mtrr_value[i].lbase,
-				    mtrr_value[i].lsize,
-				    mtrr_value[i].ltype);
-		}
-	}
-}
-
-
-
-static struct syscore_ops mtrr_syscore_ops = {
-	.suspend	= mtrr_save,
-	.resume		= mtrr_restore,
-};
-
-int __initdata changed_by_mtrr_cleanup;
-
-#define SIZE_OR_MASK_BITS(n)  (~((1ULL << ((n) - PAGE_SHIFT)) - 1))
-/**
- * mtrr_bp_init - initialize mtrrs on the boot CPU
- *
- * This needs to be called early; before any of the other CPUs are
- * initialized (i.e. before smp_init()).
- *
- */
-void __init mtrr_bp_init(void)
-{
-	u32 phys_addr;
-
-	init_ifs();
-
-	phys_addr = 32;
-
-	if (boot_cpu_has(X86_FEATURE_MTRR)) {
-		mtrr_if = &generic_mtrr_ops;
-		size_or_mask = SIZE_OR_MASK_BITS(36);
-		size_and_mask = 0x00f00000;
-		phys_addr = 36;
-
-		/*
-		 * This is an AMD specific MSR, but we assume(hope?) that
-		 * Intel will implement it too when they extend the address
-		 * bus of the Xeon.
-		 */
-		if (cpuid_eax(0x80000000) >= 0x80000008) {
-			phys_addr = cpuid_eax(0x80000008) & 0xff;
-			/* CPUID workaround for Intel 0F33/0F34 CPU */
-			if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
-			    boot_cpu_data.x86 == 0xF &&
-			    boot_cpu_data.x86_model == 0x3 &&
-			    (boot_cpu_data.x86_stepping == 0x3 ||
-			     boot_cpu_data.x86_stepping == 0x4))
-				phys_addr = 36;
-
-			size_or_mask = SIZE_OR_MASK_BITS(phys_addr);
-			size_and_mask = ~size_or_mask & 0xfffff00000ULL;
-		} else if (boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR &&
-			   boot_cpu_data.x86 == 6) {
-			/*
-			 * VIA C* family have Intel style MTRRs,
-			 * but don't support PAE
-			 */
-			size_or_mask = SIZE_OR_MASK_BITS(32);
-			size_and_mask = 0;
-			phys_addr = 32;
-		}
-	} else {
-		switch (boot_cpu_data.x86_vendor) {
-		case X86_VENDOR_AMD:
-			if (cpu_feature_enabled(X86_FEATURE_K6_MTRR)) {
-				/* Pre-Athlon (K6) AMD CPU MTRRs */
-				mtrr_if = mtrr_ops[X86_VENDOR_AMD];
-				size_or_mask = SIZE_OR_MASK_BITS(32);
-				size_and_mask = 0;
-			}
-			break;
-		case X86_VENDOR_CENTAUR:
-			if (cpu_feature_enabled(X86_FEATURE_CENTAUR_MCR)) {
-				mtrr_if = mtrr_ops[X86_VENDOR_CENTAUR];
-				size_or_mask = SIZE_OR_MASK_BITS(32);
-				size_and_mask = 0;
-			}
-			break;
-		case X86_VENDOR_CYRIX:
-			if (cpu_feature_enabled(X86_FEATURE_CYRIX_ARR)) {
-				mtrr_if = mtrr_ops[X86_VENDOR_CYRIX];
-				size_or_mask = SIZE_OR_MASK_BITS(32);
-				size_and_mask = 0;
-			}
-			break;
-		default:
-			break;
-		}
-	}
-
-	if (mtrr_if) {
-		__mtrr_enabled = true;
-		set_num_var_ranges();
-		init_table();
-		if (use_intel()) {
-			/* BIOS may override */
-			__mtrr_enabled = get_mtrr_state();
-
-			if (mtrr_enabled())
-				mtrr_bp_pat_init();
-
-			if (mtrr_cleanup(phys_addr)) {
-				changed_by_mtrr_cleanup = 1;
-				mtrr_if->set_all();
-			}
-		}
-	}
-
-	if (!mtrr_enabled()) {
-		pr_info("MTRR: Disabled\n");
-
-		/*
-		 * PAT initialization relies on MTRR's rendezvous handler.
-		 * Skip PAT init until the handler can initialize both
-		 * features independently.
-		 */
-		pat_disable("MTRRs disabled, skipping PAT initialization too.");
-	}
-}
-
-void mtrr_ap_init(void)
-{
-	if (!mtrr_enabled())
-		return;
-
-	if (!use_intel() || mtrr_aps_delayed_init)
-		return;
-	/*
-	 * Ideally we should hold mtrr_mutex here to avoid mtrr entries
-	 * changed, but this routine will be called in cpu boot time,
-	 * holding the lock breaks it.
-	 *
-	 * This routine is called in two cases:
-	 *
-	 *   1. very earily time of software resume, when there absolutely
-	 *      isn't mtrr entry changes;
-	 *
-	 *   2. cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug
-	 *      lock to prevent mtrr entry changes
-	 */
-	set_mtrr_from_inactive_cpu(~0U, 0, 0, 0);
-}
-
-/**
- * Save current fixed-range MTRR state of the first cpu in cpu_online_mask.
- */
-void mtrr_save_state(void)
-{
-	int first_cpu;
-
-	if (!mtrr_enabled())
-		return;
-
-	first_cpu = cpumask_first(cpu_online_mask);
-	smp_call_function_single(first_cpu, mtrr_save_fixed_ranges, NULL, 1);
-}
-
-void set_mtrr_aps_delayed_init(void)
-{
-	if (!mtrr_enabled())
-		return;
-	if (!use_intel())
-		return;
-
-	mtrr_aps_delayed_init = true;
-}
-
-/*
- * Delayed MTRR initialization for all AP's
- */
-void mtrr_aps_init(void)
-{
-	if (!use_intel() || !mtrr_enabled())
-		return;
-
-	/*
-	 * Check if someone has requested the delay of AP MTRR initialization,
-	 * by doing set_mtrr_aps_delayed_init(), prior to this point. If not,
-	 * then we are done.
-	 */
-	if (!mtrr_aps_delayed_init)
-		return;
-
-	set_mtrr(~0U, 0, 0, 0);
-	mtrr_aps_delayed_init = false;
-}
-
-void mtrr_bp_restore(void)
-{
-	if (!use_intel() || !mtrr_enabled())
-		return;
-
-	mtrr_if->set_all();
-}
-
-static int __init mtrr_init_finialize(void)
-{
-	if (!mtrr_enabled())
-		return 0;
-
-	if (use_intel()) {
-		if (!changed_by_mtrr_cleanup)
-			mtrr_state_warn();
-		return 0;
-	}
-
-	/*
-	 * The CPU has no MTRR and seems to not support SMP. They have
-	 * specific drivers, we use a tricky method to support
-	 * suspend/resume for them.
-	 *
-	 * TBD: is there any system with such CPU which supports
-	 * suspend/resume? If no, we should remove the code.
-	 */
-	register_syscore_ops(&mtrr_syscore_ops);
-
-	return 0;
-}
-subsys_initcall(mtrr_init_finialize);
diff --git a/arch/x86/kernel/cpu/mtrr/mtrr.c b/arch/x86/kernel/cpu/mtrr/mtrr.c
new file mode 100644
index 000000000000..9a19c800fe40
--- /dev/null
+++ b/arch/x86/kernel/cpu/mtrr/mtrr.c
@@ -0,0 +1,890 @@
+/*  Generic MTRR (Memory Type Range Register) driver.
+
+    Copyright (C) 1997-2000  Richard Gooch
+    Copyright (c) 2002	     Patrick Mochel
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Library General Public
+    License as published by the Free Software Foundation; either
+    version 2 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Library General Public License for more details.
+
+    You should have received a copy of the GNU Library General Public
+    License along with this library; if not, write to the Free
+    Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    Richard Gooch may be reached by email at  rgooch@atnf.csiro.au
+    The postal address is:
+      Richard Gooch, c/o ATNF, P. O. Box 76, Epping, N.S.W., 2121, Australia.
+
+    Source: "Pentium Pro Family Developer's Manual, Volume 3:
+    Operating System Writer's Guide" (Intel document number 242692),
+    section 11.11.7
+
+    This was cleaned and made readable by Patrick Mochel <mochel@osdl.org>
+    on 6-7 March 2002.
+    Source: Intel Architecture Software Developers Manual, Volume 3:
+    System Programming Guide; Section 9.11. (1997 edition - PPro).
+*/
+
+#define DEBUG
+
+#include <linux/types.h> /* FIXME: kvm_para.h needs this */
+
+#include <linux/stop_machine.h>
+#include <linux/kvm_para.h>
+#include <linux/uaccess.h>
+#include <linux/export.h>
+#include <linux/mutex.h>
+#include <linux/init.h>
+#include <linux/sort.h>
+#include <linux/cpu.h>
+#include <linux/pci.h>
+#include <linux/smp.h>
+#include <linux/syscore_ops.h>
+#include <linux/rcupdate.h>
+
+#include <asm/cpufeature.h>
+#include <asm/e820/api.h>
+#include <asm/mtrr.h>
+#include <asm/msr.h>
+#include <asm/pat.h>
+
+#include "mtrr.h"
+
+/* arch_phys_wc_add returns an MTRR register index plus this offset. */
+#define MTRR_TO_PHYS_WC_OFFSET 1000
+
+u32 num_var_ranges;
+static bool __mtrr_enabled;
+
+static bool mtrr_enabled(void)
+{
+	return __mtrr_enabled;
+}
+
+unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
+static DEFINE_MUTEX(mtrr_mutex);
+
+u64 size_or_mask, size_and_mask;
+static bool mtrr_aps_delayed_init;
+
+static const struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM] __ro_after_init;
+
+const struct mtrr_ops *mtrr_if;
+
+static void set_mtrr(unsigned int reg, unsigned long base,
+		     unsigned long size, mtrr_type type);
+
+void __init set_mtrr_ops(const struct mtrr_ops *ops)
+{
+	if (ops->vendor && ops->vendor < X86_VENDOR_NUM)
+		mtrr_ops[ops->vendor] = ops;
+}
+
+/*  Returns non-zero if we have the write-combining memory type  */
+static int have_wrcomb(void)
+{
+	struct pci_dev *dev;
+
+	dev = pci_get_class(PCI_CLASS_BRIDGE_HOST << 8, NULL);
+	if (dev != NULL) {
+		/*
+		 * ServerWorks LE chipsets < rev 6 have problems with
+		 * write-combining. Don't allow it and leave room for other
+		 * chipsets to be tagged
+		 */
+		if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS &&
+		    dev->device == PCI_DEVICE_ID_SERVERWORKS_LE &&
+		    dev->revision <= 5) {
+			pr_info("Serverworks LE rev < 6 detected. Write-combining disabled.\n");
+			pci_dev_put(dev);
+			return 0;
+		}
+		/*
+		 * Intel 450NX errata # 23. Non ascending cacheline evictions to
+		 * write combining memory may resulting in data corruption
+		 */
+		if (dev->vendor == PCI_VENDOR_ID_INTEL &&
+		    dev->device == PCI_DEVICE_ID_INTEL_82451NX) {
+			pr_info("Intel 450NX MMC detected. Write-combining disabled.\n");
+			pci_dev_put(dev);
+			return 0;
+		}
+		pci_dev_put(dev);
+	}
+	return mtrr_if->have_wrcomb ? mtrr_if->have_wrcomb() : 0;
+}
+
+/*  This function returns the number of variable MTRRs  */
+static void __init set_num_var_ranges(void)
+{
+	unsigned long config = 0, dummy;
+
+	if (use_intel())
+		rdmsr(MSR_MTRRcap, config, dummy);
+	else if (is_cpu(AMD))
+		config = 2;
+	else if (is_cpu(CYRIX) || is_cpu(CENTAUR))
+		config = 8;
+
+	num_var_ranges = config & 0xff;
+}
+
+static void __init init_table(void)
+{
+	int i, max;
+
+	max = num_var_ranges;
+	for (i = 0; i < max; i++)
+		mtrr_usage_table[i] = 1;
+}
+
+struct set_mtrr_data {
+	unsigned long	smp_base;
+	unsigned long	smp_size;
+	unsigned int	smp_reg;
+	mtrr_type	smp_type;
+};
+
+/**
+ * mtrr_rendezvous_handler - Work done in the synchronization handler. Executed
+ * by all the CPUs.
+ * @info: pointer to mtrr configuration data
+ *
+ * Returns nothing.
+ */
+static int mtrr_rendezvous_handler(void *info)
+{
+	struct set_mtrr_data *data = info;
+
+	/*
+	 * We use this same function to initialize the mtrrs during boot,
+	 * resume, runtime cpu online and on an explicit request to set a
+	 * specific MTRR.
+	 *
+	 * During boot or suspend, the state of the boot cpu's mtrrs has been
+	 * saved, and we want to replicate that across all the cpus that come
+	 * online (either at the end of boot or resume or during a runtime cpu
+	 * online). If we're doing that, @reg is set to something special and on
+	 * all the cpu's we do mtrr_if->set_all() (On the logical cpu that
+	 * started the boot/resume sequence, this might be a duplicate
+	 * set_all()).
+	 */
+	if (data->smp_reg != ~0U) {
+		mtrr_if->set(data->smp_reg, data->smp_base,
+			     data->smp_size, data->smp_type);
+	} else if (mtrr_aps_delayed_init || !cpu_online(smp_processor_id())) {
+		mtrr_if->set_all();
+	}
+	return 0;
+}
+
+static inline int types_compatible(mtrr_type type1, mtrr_type type2)
+{
+	return type1 == MTRR_TYPE_UNCACHABLE ||
+	       type2 == MTRR_TYPE_UNCACHABLE ||
+	       (type1 == MTRR_TYPE_WRTHROUGH && type2 == MTRR_TYPE_WRBACK) ||
+	       (type1 == MTRR_TYPE_WRBACK && type2 == MTRR_TYPE_WRTHROUGH);
+}
+
+/**
+ * set_mtrr - update mtrrs on all processors
+ * @reg:	mtrr in question
+ * @base:	mtrr base
+ * @size:	mtrr size
+ * @type:	mtrr type
+ *
+ * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly:
+ *
+ * 1. Queue work to do the following on all processors:
+ * 2. Disable Interrupts
+ * 3. Wait for all procs to do so
+ * 4. Enter no-fill cache mode
+ * 5. Flush caches
+ * 6. Clear PGE bit
+ * 7. Flush all TLBs
+ * 8. Disable all range registers
+ * 9. Update the MTRRs
+ * 10. Enable all range registers
+ * 11. Flush all TLBs and caches again
+ * 12. Enter normal cache mode and reenable caching
+ * 13. Set PGE
+ * 14. Wait for buddies to catch up
+ * 15. Enable interrupts.
+ *
+ * What does that mean for us? Well, stop_machine() will ensure that
+ * the rendezvous handler is started on each CPU. And in lockstep they
+ * do the state transition of disabling interrupts, updating MTRR's
+ * (the CPU vendors may each do it differently, so we call mtrr_if->set()
+ * callback and let them take care of it.) and enabling interrupts.
+ *
+ * Note that the mechanism is the same for UP systems, too; all the SMP stuff
+ * becomes nops.
+ */
+static void
+set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type)
+{
+	struct set_mtrr_data data = { .smp_reg = reg,
+				      .smp_base = base,
+				      .smp_size = size,
+				      .smp_type = type
+				    };
+
+	stop_machine(mtrr_rendezvous_handler, &data, cpu_online_mask);
+}
+
+static void set_mtrr_cpuslocked(unsigned int reg, unsigned long base,
+				unsigned long size, mtrr_type type)
+{
+	struct set_mtrr_data data = { .smp_reg = reg,
+				      .smp_base = base,
+				      .smp_size = size,
+				      .smp_type = type
+				    };
+
+	stop_machine_cpuslocked(mtrr_rendezvous_handler, &data, cpu_online_mask);
+}
+
+static void set_mtrr_from_inactive_cpu(unsigned int reg, unsigned long base,
+				      unsigned long size, mtrr_type type)
+{
+	struct set_mtrr_data data = { .smp_reg = reg,
+				      .smp_base = base,
+				      .smp_size = size,
+				      .smp_type = type
+				    };
+
+	stop_machine_from_inactive_cpu(mtrr_rendezvous_handler, &data,
+				       cpu_callout_mask);
+}
+
+/**
+ * mtrr_add_page - Add a memory type region
+ * @base: Physical base address of region in pages (in units of 4 kB!)
+ * @size: Physical size of region in pages (4 kB)
+ * @type: Type of MTRR desired
+ * @increment: If this is true do usage counting on the region
+ *
+ * Memory type region registers control the caching on newer Intel and
+ * non Intel processors. This function allows drivers to request an
+ * MTRR is added. The details and hardware specifics of each processor's
+ * implementation are hidden from the caller, but nevertheless the
+ * caller should expect to need to provide a power of two size on an
+ * equivalent power of two boundary.
+ *
+ * If the region cannot be added either because all regions are in use
+ * or the CPU cannot support it a negative value is returned. On success
+ * the register number for this entry is returned, but should be treated
+ * as a cookie only.
+ *
+ * On a multiprocessor machine the changes are made to all processors.
+ * This is required on x86 by the Intel processors.
+ *
+ * The available types are
+ *
+ * %MTRR_TYPE_UNCACHABLE - No caching
+ *
+ * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
+ *
+ * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
+ *
+ * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
+ *
+ * BUGS: Needs a quiet flag for the cases where drivers do not mind
+ * failures and do not wish system log messages to be sent.
+ */
+int mtrr_add_page(unsigned long base, unsigned long size,
+		  unsigned int type, bool increment)
+{
+	unsigned long lbase, lsize;
+	int i, replace, error;
+	mtrr_type ltype;
+
+	if (!mtrr_enabled())
+		return -ENXIO;
+
+	error = mtrr_if->validate_add_page(base, size, type);
+	if (error)
+		return error;
+
+	if (type >= MTRR_NUM_TYPES) {
+		pr_warn("type: %u invalid\n", type);
+		return -EINVAL;
+	}
+
+	/* If the type is WC, check that this processor supports it */
+	if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) {
+		pr_warn("your processor doesn't support write-combining\n");
+		return -ENOSYS;
+	}
+
+	if (!size) {
+		pr_warn("zero sized request\n");
+		return -EINVAL;
+	}
+
+	if ((base | (base + size - 1)) >>
+	    (boot_cpu_data.x86_phys_bits - PAGE_SHIFT)) {
+		pr_warn("base or size exceeds the MTRR width\n");
+		return -EINVAL;
+	}
+
+	error = -EINVAL;
+	replace = -1;
+
+	/* No CPU hotplug when we change MTRR entries */
+	get_online_cpus();
+
+	/* Search for existing MTRR  */
+	mutex_lock(&mtrr_mutex);
+	for (i = 0; i < num_var_ranges; ++i) {
+		mtrr_if->get(i, &lbase, &lsize, &ltype);
+		if (!lsize || base > lbase + lsize - 1 ||
+		    base + size - 1 < lbase)
+			continue;
+		/*
+		 * At this point we know there is some kind of
+		 * overlap/enclosure
+		 */
+		if (base < lbase || base + size - 1 > lbase + lsize - 1) {
+			if (base <= lbase &&
+			    base + size - 1 >= lbase + lsize - 1) {
+				/*  New region encloses an existing region  */
+				if (type == ltype) {
+					replace = replace == -1 ? i : -2;
+					continue;
+				} else if (types_compatible(type, ltype))
+					continue;
+			}
+			pr_warn("0x%lx000,0x%lx000 overlaps existing 0x%lx000,0x%lx000\n", base, size, lbase,
+				lsize);
+			goto out;
+		}
+		/* New region is enclosed by an existing region */
+		if (ltype != type) {
+			if (types_compatible(type, ltype))
+				continue;
+			pr_warn("type mismatch for %lx000,%lx000 old: %s new: %s\n",
+				base, size, mtrr_attrib_to_str(ltype),
+				mtrr_attrib_to_str(type));
+			goto out;
+		}
+		if (increment)
+			++mtrr_usage_table[i];
+		error = i;
+		goto out;
+	}
+	/* Search for an empty MTRR */
+	i = mtrr_if->get_free_region(base, size, replace);
+	if (i >= 0) {
+		set_mtrr_cpuslocked(i, base, size, type);
+		if (likely(replace < 0)) {
+			mtrr_usage_table[i] = 1;
+		} else {
+			mtrr_usage_table[i] = mtrr_usage_table[replace];
+			if (increment)
+				mtrr_usage_table[i]++;
+			if (unlikely(replace != i)) {
+				set_mtrr_cpuslocked(replace, 0, 0, 0);
+				mtrr_usage_table[replace] = 0;
+			}
+		}
+	} else {
+		pr_info("no more MTRRs available\n");
+	}
+	error = i;
+ out:
+	mutex_unlock(&mtrr_mutex);
+	put_online_cpus();
+	return error;
+}
+
+static int mtrr_check(unsigned long base, unsigned long size)
+{
+	if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
+		pr_warn("size and base must be multiples of 4 kiB\n");
+		pr_debug("size: 0x%lx  base: 0x%lx\n", size, base);
+		dump_stack();
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * mtrr_add - Add a memory type region
+ * @base: Physical base address of region
+ * @size: Physical size of region
+ * @type: Type of MTRR desired
+ * @increment: If this is true do usage counting on the region
+ *
+ * Memory type region registers control the caching on newer Intel and
+ * non Intel processors. This function allows drivers to request an
+ * MTRR is added. The details and hardware specifics of each processor's
+ * implementation are hidden from the caller, but nevertheless the
+ * caller should expect to need to provide a power of two size on an
+ * equivalent power of two boundary.
+ *
+ * If the region cannot be added either because all regions are in use
+ * or the CPU cannot support it a negative value is returned. On success
+ * the register number for this entry is returned, but should be treated
+ * as a cookie only.
+ *
+ * On a multiprocessor machine the changes are made to all processors.
+ * This is required on x86 by the Intel processors.
+ *
+ * The available types are
+ *
+ * %MTRR_TYPE_UNCACHABLE - No caching
+ *
+ * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
+ *
+ * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
+ *
+ * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
+ *
+ * BUGS: Needs a quiet flag for the cases where drivers do not mind
+ * failures and do not wish system log messages to be sent.
+ */
+int mtrr_add(unsigned long base, unsigned long size, unsigned int type,
+	     bool increment)
+{
+	if (!mtrr_enabled())
+		return -ENODEV;
+	if (mtrr_check(base, size))
+		return -EINVAL;
+	return mtrr_add_page(base >> PAGE_SHIFT, size >> PAGE_SHIFT, type,
+			     increment);
+}
+
+/**
+ * mtrr_del_page - delete a memory type region
+ * @reg: Register returned by mtrr_add
+ * @base: Physical base address
+ * @size: Size of region
+ *
+ * If register is supplied then base and size are ignored. This is
+ * how drivers should call it.
+ *
+ * Releases an MTRR region. If the usage count drops to zero the
+ * register is freed and the region returns to default state.
+ * On success the register is returned, on failure a negative error
+ * code.
+ */
+int mtrr_del_page(int reg, unsigned long base, unsigned long size)
+{
+	int i, max;
+	mtrr_type ltype;
+	unsigned long lbase, lsize;
+	int error = -EINVAL;
+
+	if (!mtrr_enabled())
+		return -ENODEV;
+
+	max = num_var_ranges;
+	/* No CPU hotplug when we change MTRR entries */
+	get_online_cpus();
+	mutex_lock(&mtrr_mutex);
+	if (reg < 0) {
+		/*  Search for existing MTRR  */
+		for (i = 0; i < max; ++i) {
+			mtrr_if->get(i, &lbase, &lsize, &ltype);
+			if (lbase == base && lsize == size) {
+				reg = i;
+				break;
+			}
+		}
+		if (reg < 0) {
+			pr_debug("no MTRR for %lx000,%lx000 found\n",
+				 base, size);
+			goto out;
+		}
+	}
+	if (reg >= max) {
+		pr_warn("register: %d too big\n", reg);
+		goto out;
+	}
+	mtrr_if->get(reg, &lbase, &lsize, &ltype);
+	if (lsize < 1) {
+		pr_warn("MTRR %d not used\n", reg);
+		goto out;
+	}
+	if (mtrr_usage_table[reg] < 1) {
+		pr_warn("reg: %d has count=0\n", reg);
+		goto out;
+	}
+	if (--mtrr_usage_table[reg] < 1)
+		set_mtrr_cpuslocked(reg, 0, 0, 0);
+	error = reg;
+ out:
+	mutex_unlock(&mtrr_mutex);
+	put_online_cpus();
+	return error;
+}
+
+/**
+ * mtrr_del - delete a memory type region
+ * @reg: Register returned by mtrr_add
+ * @base: Physical base address
+ * @size: Size of region
+ *
+ * If register is supplied then base and size are ignored. This is
+ * how drivers should call it.
+ *
+ * Releases an MTRR region. If the usage count drops to zero the
+ * register is freed and the region returns to default state.
+ * On success the register is returned, on failure a negative error
+ * code.
+ */
+int mtrr_del(int reg, unsigned long base, unsigned long size)
+{
+	if (!mtrr_enabled())
+		return -ENODEV;
+	if (mtrr_check(base, size))
+		return -EINVAL;
+	return mtrr_del_page(reg, base >> PAGE_SHIFT, size >> PAGE_SHIFT);
+}
+
+/**
+ * arch_phys_wc_add - add a WC MTRR and handle errors if PAT is unavailable
+ * @base: Physical base address
+ * @size: Size of region
+ *
+ * If PAT is available, this does nothing.  If PAT is unavailable, it
+ * attempts to add a WC MTRR covering size bytes starting at base and
+ * logs an error if this fails.
+ *
+ * The called should provide a power of two size on an equivalent
+ * power of two boundary.
+ *
+ * Drivers must store the return value to pass to mtrr_del_wc_if_needed,
+ * but drivers should not try to interpret that return value.
+ */
+int arch_phys_wc_add(unsigned long base, unsigned long size)
+{
+	int ret;
+
+	if (pat_enabled() || !mtrr_enabled())
+		return 0;  /* Success!  (We don't need to do anything.) */
+
+	ret = mtrr_add(base, size, MTRR_TYPE_WRCOMB, true);
+	if (ret < 0) {
+		pr_warn("Failed to add WC MTRR for [%p-%p]; performance may suffer.",
+			(void *)base, (void *)(base + size - 1));
+		return ret;
+	}
+	return ret + MTRR_TO_PHYS_WC_OFFSET;
+}
+EXPORT_SYMBOL(arch_phys_wc_add);
+
+/*
+ * arch_phys_wc_del - undoes arch_phys_wc_add
+ * @handle: Return value from arch_phys_wc_add
+ *
+ * This cleans up after mtrr_add_wc_if_needed.
+ *
+ * The API guarantees that mtrr_del_wc_if_needed(error code) and
+ * mtrr_del_wc_if_needed(0) do nothing.
+ */
+void arch_phys_wc_del(int handle)
+{
+	if (handle >= 1) {
+		WARN_ON(handle < MTRR_TO_PHYS_WC_OFFSET);
+		mtrr_del(handle - MTRR_TO_PHYS_WC_OFFSET, 0, 0);
+	}
+}
+EXPORT_SYMBOL(arch_phys_wc_del);
+
+/*
+ * arch_phys_wc_index - translates arch_phys_wc_add's return value
+ * @handle: Return value from arch_phys_wc_add
+ *
+ * This will turn the return value from arch_phys_wc_add into an mtrr
+ * index suitable for debugging.
+ *
+ * Note: There is no legitimate use for this function, except possibly
+ * in printk line.  Alas there is an illegitimate use in some ancient
+ * drm ioctls.
+ */
+int arch_phys_wc_index(int handle)
+{
+	if (handle < MTRR_TO_PHYS_WC_OFFSET)
+		return -1;
+	else
+		return handle - MTRR_TO_PHYS_WC_OFFSET;
+}
+EXPORT_SYMBOL_GPL(arch_phys_wc_index);
+
+/*
+ * HACK ALERT!
+ * These should be called implicitly, but we can't yet until all the initcall
+ * stuff is done...
+ */
+static void __init init_ifs(void)
+{
+#ifndef CONFIG_X86_64
+	amd_init_mtrr();
+	cyrix_init_mtrr();
+	centaur_init_mtrr();
+#endif
+}
+
+/* The suspend/resume methods are only for CPU without MTRR. CPU using generic
+ * MTRR driver doesn't require this
+ */
+struct mtrr_value {
+	mtrr_type	ltype;
+	unsigned long	lbase;
+	unsigned long	lsize;
+};
+
+static struct mtrr_value mtrr_value[MTRR_MAX_VAR_RANGES];
+
+static int mtrr_save(void)
+{
+	int i;
+
+	for (i = 0; i < num_var_ranges; i++) {
+		mtrr_if->get(i, &mtrr_value[i].lbase,
+				&mtrr_value[i].lsize,
+				&mtrr_value[i].ltype);
+	}
+	return 0;
+}
+
+static void mtrr_restore(void)
+{
+	int i;
+
+	for (i = 0; i < num_var_ranges; i++) {
+		if (mtrr_value[i].lsize) {
+			set_mtrr(i, mtrr_value[i].lbase,
+				    mtrr_value[i].lsize,
+				    mtrr_value[i].ltype);
+		}
+	}
+}
+
+
+
+static struct syscore_ops mtrr_syscore_ops = {
+	.suspend	= mtrr_save,
+	.resume		= mtrr_restore,
+};
+
+int __initdata changed_by_mtrr_cleanup;
+
+#define SIZE_OR_MASK_BITS(n)  (~((1ULL << ((n) - PAGE_SHIFT)) - 1))
+/**
+ * mtrr_bp_init - initialize mtrrs on the boot CPU
+ *
+ * This needs to be called early; before any of the other CPUs are
+ * initialized (i.e. before smp_init()).
+ *
+ */
+void __init mtrr_bp_init(void)
+{
+	u32 phys_addr;
+
+	init_ifs();
+
+	phys_addr = 32;
+
+	if (boot_cpu_has(X86_FEATURE_MTRR)) {
+		mtrr_if = &generic_mtrr_ops;
+		size_or_mask = SIZE_OR_MASK_BITS(36);
+		size_and_mask = 0x00f00000;
+		phys_addr = 36;
+
+		/*
+		 * This is an AMD specific MSR, but we assume(hope?) that
+		 * Intel will implement it too when they extend the address
+		 * bus of the Xeon.
+		 */
+		if (cpuid_eax(0x80000000) >= 0x80000008) {
+			phys_addr = cpuid_eax(0x80000008) & 0xff;
+			/* CPUID workaround for Intel 0F33/0F34 CPU */
+			if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
+			    boot_cpu_data.x86 == 0xF &&
+			    boot_cpu_data.x86_model == 0x3 &&
+			    (boot_cpu_data.x86_stepping == 0x3 ||
+			     boot_cpu_data.x86_stepping == 0x4))
+				phys_addr = 36;
+
+			size_or_mask = SIZE_OR_MASK_BITS(phys_addr);
+			size_and_mask = ~size_or_mask & 0xfffff00000ULL;
+		} else if (boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR &&
+			   boot_cpu_data.x86 == 6) {
+			/*
+			 * VIA C* family have Intel style MTRRs,
+			 * but don't support PAE
+			 */
+			size_or_mask = SIZE_OR_MASK_BITS(32);
+			size_and_mask = 0;
+			phys_addr = 32;
+		}
+	} else {
+		switch (boot_cpu_data.x86_vendor) {
+		case X86_VENDOR_AMD:
+			if (cpu_feature_enabled(X86_FEATURE_K6_MTRR)) {
+				/* Pre-Athlon (K6) AMD CPU MTRRs */
+				mtrr_if = mtrr_ops[X86_VENDOR_AMD];
+				size_or_mask = SIZE_OR_MASK_BITS(32);
+				size_and_mask = 0;
+			}
+			break;
+		case X86_VENDOR_CENTAUR:
+			if (cpu_feature_enabled(X86_FEATURE_CENTAUR_MCR)) {
+				mtrr_if = mtrr_ops[X86_VENDOR_CENTAUR];
+				size_or_mask = SIZE_OR_MASK_BITS(32);
+				size_and_mask = 0;
+			}
+			break;
+		case X86_VENDOR_CYRIX:
+			if (cpu_feature_enabled(X86_FEATURE_CYRIX_ARR)) {
+				mtrr_if = mtrr_ops[X86_VENDOR_CYRIX];
+				size_or_mask = SIZE_OR_MASK_BITS(32);
+				size_and_mask = 0;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+
+	if (mtrr_if) {
+		__mtrr_enabled = true;
+		set_num_var_ranges();
+		init_table();
+		if (use_intel()) {
+			/* BIOS may override */
+			__mtrr_enabled = get_mtrr_state();
+
+			if (mtrr_enabled())
+				mtrr_bp_pat_init();
+
+			if (mtrr_cleanup(phys_addr)) {
+				changed_by_mtrr_cleanup = 1;
+				mtrr_if->set_all();
+			}
+		}
+	}
+
+	if (!mtrr_enabled()) {
+		pr_info("Disabled\n");
+
+		/*
+		 * PAT initialization relies on MTRR's rendezvous handler.
+		 * Skip PAT init until the handler can initialize both
+		 * features independently.
+		 */
+		pat_disable("MTRRs disabled, skipping PAT initialization too.");
+	}
+}
+
+void mtrr_ap_init(void)
+{
+	if (!mtrr_enabled())
+		return;
+
+	if (!use_intel() || mtrr_aps_delayed_init)
+		return;
+
+	rcu_cpu_starting(smp_processor_id());
+
+	/*
+	 * Ideally we should hold mtrr_mutex here to avoid mtrr entries
+	 * changed, but this routine will be called in cpu boot time,
+	 * holding the lock breaks it.
+	 *
+	 * This routine is called in two cases:
+	 *
+	 *   1. very earily time of software resume, when there absolutely
+	 *      isn't mtrr entry changes;
+	 *
+	 *   2. cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug
+	 *      lock to prevent mtrr entry changes
+	 */
+	set_mtrr_from_inactive_cpu(~0U, 0, 0, 0);
+}
+
+/**
+ * Save current fixed-range MTRR state of the first cpu in cpu_online_mask.
+ */
+void mtrr_save_state(void)
+{
+	int first_cpu;
+
+	if (!mtrr_enabled())
+		return;
+
+	first_cpu = cpumask_first(cpu_online_mask);
+	smp_call_function_single(first_cpu, mtrr_save_fixed_ranges, NULL, 1);
+}
+
+void set_mtrr_aps_delayed_init(void)
+{
+	if (!mtrr_enabled())
+		return;
+	if (!use_intel())
+		return;
+
+	mtrr_aps_delayed_init = true;
+}
+
+/*
+ * Delayed MTRR initialization for all AP's
+ */
+void mtrr_aps_init(void)
+{
+	if (!use_intel() || !mtrr_enabled())
+		return;
+
+	/*
+	 * Check if someone has requested the delay of AP MTRR initialization,
+	 * by doing set_mtrr_aps_delayed_init(), prior to this point. If not,
+	 * then we are done.
+	 */
+	if (!mtrr_aps_delayed_init)
+		return;
+
+	set_mtrr(~0U, 0, 0, 0);
+	mtrr_aps_delayed_init = false;
+}
+
+void mtrr_bp_restore(void)
+{
+	if (!use_intel() || !mtrr_enabled())
+		return;
+
+	mtrr_if->set_all();
+}
+
+static int __init mtrr_init_finialize(void)
+{
+	if (!mtrr_enabled())
+		return 0;
+
+	if (use_intel()) {
+		if (!changed_by_mtrr_cleanup)
+			mtrr_state_warn();
+		return 0;
+	}
+
+	/*
+	 * The CPU has no MTRR and seems to not support SMP. They have
+	 * specific drivers, we use a tricky method to support
+	 * suspend/resume for them.
+	 *
+	 * TBD: is there any system with such CPU which supports
+	 * suspend/resume? If no, we should remove the code.
+	 */
+	register_syscore_ops(&mtrr_syscore_ops);
+
+	return 0;
+}
+subsys_initcall(mtrr_init_finialize);
diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c
index b099024d339c..81c0afb39d0a 100644
--- a/arch/x86/kernel/cpu/topology.c
+++ b/arch/x86/kernel/cpu/topology.c
@@ -27,7 +27,7 @@
  * exists, use it for populating initial_apicid and cpu topology
  * detection.
  */
-void detect_extended_topology(struct cpuinfo_x86 *c)
+int detect_extended_topology(struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_SMP
 	unsigned int eax, ebx, ecx, edx, sub_index;
@@ -36,7 +36,7 @@ void detect_extended_topology(struct cpuinfo_x86 *c)
 	static bool printed;
 
 	if (c->cpuid_level < 0xb)
-		return;
+		return -1;
 
 	cpuid_count(0xb, SMT_LEVEL, &eax, &ebx, &ecx, &edx);
 
@@ -44,7 +44,7 @@ void detect_extended_topology(struct cpuinfo_x86 *c)
 	 * check if the cpuid leaf 0xb is actually implemented.
 	 */
 	if (ebx == 0 || (LEAFB_SUBTYPE(ecx) != SMT_TYPE))
-		return;
+		return -1;
 
 	set_cpu_cap(c, X86_FEATURE_XTOPOLOGY);
 
@@ -95,6 +95,6 @@ void detect_extended_topology(struct cpuinfo_x86 *c)
 			       c->cpu_core_id);
 		printed = 1;
 	}
-	return;
 #endif
+	return 0;
 }
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index 18fa9d74c182..666a284116ac 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -22,11 +22,14 @@
 #include <asm/stacktrace.h>
 #include <asm/unwind.h>
 
+#define OPCODE_BUFSIZE 64
+
 int panic_on_unrecovered_nmi;
 int panic_on_io_nmi;
-static unsigned int code_bytes = 64;
 static int die_counter;
 
+static struct pt_regs exec_summary_regs;
+
 bool in_task_stack(unsigned long *stack, struct task_struct *task,
 		   struct stack_info *info)
 {
@@ -69,9 +72,62 @@ static void printk_stack_address(unsigned long address, int reliable,
 	printk("%s %s%pB\n", log_lvl, reliable ? "" : "? ", (void *)address);
 }
 
+/*
+ * There are a couple of reasons for the 2/3rd prologue, courtesy of Linus:
+ *
+ * In case where we don't have the exact kernel image (which, if we did, we can
+ * simply disassemble and navigate to the RIP), the purpose of the bigger
+ * prologue is to have more context and to be able to correlate the code from
+ * the different toolchains better.
+ *
+ * In addition, it helps in recreating the register allocation of the failing
+ * kernel and thus make sense of the register dump.
+ *
+ * What is more, the additional complication of a variable length insn arch like
+ * x86 warrants having longer byte sequence before rIP so that the disassembler
+ * can "sync" up properly and find instruction boundaries when decoding the
+ * opcode bytes.
+ *
+ * Thus, the 2/3rds prologue and 64 byte OPCODE_BUFSIZE is just a random
+ * guesstimate in attempt to achieve all of the above.
+ */
+void show_opcodes(u8 *rip, const char *loglvl)
+{
+	unsigned int code_prologue = OPCODE_BUFSIZE * 2 / 3;
+	u8 opcodes[OPCODE_BUFSIZE];
+	u8 *ip;
+	int i;
+
+	printk("%sCode: ", loglvl);
+
+	ip = (u8 *)rip - code_prologue;
+	if (probe_kernel_read(opcodes, ip, OPCODE_BUFSIZE)) {
+		pr_cont("Bad RIP value.\n");
+		return;
+	}
+
+	for (i = 0; i < OPCODE_BUFSIZE; i++, ip++) {
+		if (ip == rip)
+			pr_cont("<%02x> ", opcodes[i]);
+		else
+			pr_cont("%02x ", opcodes[i]);
+	}
+	pr_cont("\n");
+}
+
+void show_ip(struct pt_regs *regs, const char *loglvl)
+{
+#ifdef CONFIG_X86_32
+	printk("%sEIP: %pS\n", loglvl, (void *)regs->ip);
+#else
+	printk("%sRIP: %04x:%pS\n", loglvl, (int)regs->cs, (void *)regs->ip);
+#endif
+	show_opcodes((u8 *)regs->ip, loglvl);
+}
+
 void show_iret_regs(struct pt_regs *regs)
 {
-	printk(KERN_DEFAULT "RIP: %04x:%pS\n", (int)regs->cs, (void *)regs->ip);
+	show_ip(regs, KERN_DEFAULT);
 	printk(KERN_DEFAULT "RSP: %04x:%016lx EFLAGS: %08lx", (int)regs->ss,
 		regs->sp, regs->flags);
 }
@@ -267,7 +323,6 @@ unsigned long oops_begin(void)
 	bust_spinlocks(1);
 	return flags;
 }
-EXPORT_SYMBOL_GPL(oops_begin);
 NOKPROBE_SYMBOL(oops_begin);
 
 void __noreturn rewind_stack_do_exit(int signr);
@@ -287,6 +342,9 @@ void oops_end(unsigned long flags, struct pt_regs *regs, int signr)
 	raw_local_irq_restore(flags);
 	oops_exit();
 
+	/* Executive summary in case the oops scrolled away */
+	__show_regs(&exec_summary_regs, true);
+
 	if (!signr)
 		return;
 	if (in_interrupt())
@@ -305,10 +363,10 @@ NOKPROBE_SYMBOL(oops_end);
 
 int __die(const char *str, struct pt_regs *regs, long err)
 {
-#ifdef CONFIG_X86_32
-	unsigned short ss;
-	unsigned long sp;
-#endif
+	/* Save the regs of the first oops for the executive summary later. */
+	if (!die_counter)
+		exec_summary_regs = *regs;
+
 	printk(KERN_DEFAULT
 	       "%s: %04lx [#%d]%s%s%s%s%s\n", str, err & 0xffff, ++die_counter,
 	       IS_ENABLED(CONFIG_PREEMPT) ? " PREEMPT"         : "",
@@ -318,26 +376,13 @@ int __die(const char *str, struct pt_regs *regs, long err)
 	       IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION) ?
 	       (boot_cpu_has(X86_FEATURE_PTI) ? " PTI" : " NOPTI") : "");
 
+	show_regs(regs);
+	print_modules();
+
 	if (notify_die(DIE_OOPS, str, regs, err,
 			current->thread.trap_nr, SIGSEGV) == NOTIFY_STOP)
 		return 1;
 
-	print_modules();
-	show_regs(regs);
-#ifdef CONFIG_X86_32
-	if (user_mode(regs)) {
-		sp = regs->sp;
-		ss = regs->ss;
-	} else {
-		sp = kernel_stack_pointer(regs);
-		savesegment(ss, ss);
-	}
-	printk(KERN_EMERG "EIP: %pS SS:ESP: %04x:%08lx\n",
-	       (void *)regs->ip, ss, sp);
-#else
-	/* Executive summary in case the oops scrolled away */
-	printk(KERN_ALERT "RIP: %pS RSP: %016lx\n", (void *)regs->ip, regs->sp);
-#endif
 	return 0;
 }
 NOKPROBE_SYMBOL(__die);
@@ -356,30 +401,9 @@ void die(const char *str, struct pt_regs *regs, long err)
 	oops_end(flags, regs, sig);
 }
 
-static int __init code_bytes_setup(char *s)
-{
-	ssize_t ret;
-	unsigned long val;
-
-	if (!s)
-		return -EINVAL;
-
-	ret = kstrtoul(s, 0, &val);
-	if (ret)
-		return ret;
-
-	code_bytes = val;
-	if (code_bytes > 8192)
-		code_bytes = 8192;
-
-	return 1;
-}
-__setup("code_bytes=", code_bytes_setup);
-
 void show_regs(struct pt_regs *regs)
 {
 	bool all = true;
-	int i;
 
 	show_regs_print_info(KERN_DEFAULT);
 
@@ -389,36 +413,8 @@ void show_regs(struct pt_regs *regs)
 	__show_regs(regs, all);
 
 	/*
-	 * When in-kernel, we also print out the stack and code at the
-	 * time of the fault..
+	 * When in-kernel, we also print out the stack at the time of the fault..
 	 */
-	if (!user_mode(regs)) {
-		unsigned int code_prologue = code_bytes * 43 / 64;
-		unsigned int code_len = code_bytes;
-		unsigned char c;
-		u8 *ip;
-
+	if (!user_mode(regs))
 		show_trace_log_lvl(current, regs, NULL, KERN_DEFAULT);
-
-		printk(KERN_DEFAULT "Code: ");
-
-		ip = (u8 *)regs->ip - code_prologue;
-		if (ip < (u8 *)PAGE_OFFSET || probe_kernel_address(ip, c)) {
-			/* try starting at IP */
-			ip = (u8 *)regs->ip;
-			code_len = code_len - code_prologue + 1;
-		}
-		for (i = 0; i < code_len; i++, ip++) {
-			if (ip < (u8 *)PAGE_OFFSET ||
-					probe_kernel_address(ip, c)) {
-				pr_cont(" Bad RIP value.");
-				break;
-			}
-			if (ip == (u8 *)regs->ip)
-				pr_cont("<%02x> ", c);
-			else
-				pr_cont("%02x ", c);
-		}
-	}
-	pr_cont("\n");
 }
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
index 6a2cb1442e05..d1f25c831447 100644
--- a/arch/x86/kernel/e820.c
+++ b/arch/x86/kernel/e820.c
@@ -155,7 +155,8 @@ static void __init __e820__range_add(struct e820_table *table, u64 start, u64 si
 	int x = table->nr_entries;
 
 	if (x >= ARRAY_SIZE(table->entries)) {
-		pr_err("e820: too many entries; ignoring [mem %#010llx-%#010llx]\n", start, start + size - 1);
+		pr_err("too many entries; ignoring [mem %#010llx-%#010llx]\n",
+		       start, start + size - 1);
 		return;
 	}
 
@@ -190,9 +191,10 @@ void __init e820__print_table(char *who)
 	int i;
 
 	for (i = 0; i < e820_table->nr_entries; i++) {
-		pr_info("%s: [mem %#018Lx-%#018Lx] ", who,
-		       e820_table->entries[i].addr,
-		       e820_table->entries[i].addr + e820_table->entries[i].size - 1);
+		pr_info("%s: [mem %#018Lx-%#018Lx] ",
+			who,
+			e820_table->entries[i].addr,
+			e820_table->entries[i].addr + e820_table->entries[i].size - 1);
 
 		e820_print_type(e820_table->entries[i].type);
 		pr_cont("\n");
@@ -574,7 +576,7 @@ void __init e820__update_table_print(void)
 	if (e820__update_table(e820_table))
 		return;
 
-	pr_info("e820: modified physical RAM map:\n");
+	pr_info("modified physical RAM map:\n");
 	e820__print_table("modified");
 }
 
@@ -636,9 +638,8 @@ __init void e820__setup_pci_gap(void)
 	if (!found) {
 #ifdef CONFIG_X86_64
 		gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024;
-		pr_err(
-			"e820: Cannot find an available gap in the 32-bit address range\n"
-			"e820: PCI devices with unassigned 32-bit BARs may not work!\n");
+		pr_err("Cannot find an available gap in the 32-bit address range\n");
+		pr_err("PCI devices with unassigned 32-bit BARs may not work!\n");
 #else
 		gapstart = 0x10000000;
 #endif
@@ -649,7 +650,8 @@ __init void e820__setup_pci_gap(void)
 	 */
 	pci_mem_start = gapstart;
 
-	pr_info("e820: [mem %#010lx-%#010lx] available for PCI devices\n", gapstart, gapstart + gapsize - 1);
+	pr_info("[mem %#010lx-%#010lx] available for PCI devices\n",
+		gapstart, gapstart + gapsize - 1);
 }
 
 /*
@@ -711,7 +713,7 @@ void __init e820__memory_setup_extended(u64 phys_addr, u32 data_len)
 	memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware));
 
 	early_memunmap(sdata, data_len);
-	pr_info("e820: extended physical RAM map:\n");
+	pr_info("extended physical RAM map:\n");
 	e820__print_table("extended");
 }
 
@@ -780,7 +782,7 @@ u64 __init e820__memblock_alloc_reserved(u64 size, u64 align)
 	addr = __memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE);
 	if (addr) {
 		e820__range_update_kexec(addr, size, E820_TYPE_RAM, E820_TYPE_RESERVED);
-		pr_info("e820: update e820_table_kexec for e820__memblock_alloc_reserved()\n");
+		pr_info("update e820_table_kexec for e820__memblock_alloc_reserved()\n");
 		e820__update_table_kexec();
 	}
 
@@ -830,8 +832,8 @@ static unsigned long __init e820_end_pfn(unsigned long limit_pfn, enum e820_type
 	if (last_pfn > max_arch_pfn)
 		last_pfn = max_arch_pfn;
 
-	pr_info("e820: last_pfn = %#lx max_arch_pfn = %#lx\n",
-			 last_pfn, max_arch_pfn);
+	pr_info("last_pfn = %#lx max_arch_pfn = %#lx\n",
+		last_pfn, max_arch_pfn);
 	return last_pfn;
 }
 
@@ -1005,7 +1007,7 @@ void __init e820__finish_early_params(void)
 		if (e820__update_table(e820_table) < 0)
 			early_panic("Invalid user supplied memory map");
 
-		pr_info("e820: user-defined physical RAM map:\n");
+		pr_info("user-defined physical RAM map:\n");
 		e820__print_table("user");
 	}
 }
@@ -1238,7 +1240,7 @@ void __init e820__memory_setup(void)
 	memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec));
 	memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware));
 
-	pr_info("e820: BIOS-provided physical RAM map:\n");
+	pr_info("BIOS-provided physical RAM map:\n");
 	e820__print_table(who);
 }
 
diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c
index bae0d32e327b..da5d8ac60062 100644
--- a/arch/x86/kernel/early-quirks.c
+++ b/arch/x86/kernel/early-quirks.c
@@ -28,8 +28,6 @@
 #include <asm/irq_remapping.h>
 #include <asm/early_ioremap.h>
 
-#define dev_err(msg)  pr_err("pci 0000:%02x:%02x.%d: %s", bus, slot, func, msg)
-
 static void __init fix_hypertransport_config(int num, int slot, int func)
 {
 	u32 htcfg;
@@ -617,7 +615,8 @@ static void __init apple_airport_reset(int bus, int slot, int func)
 
 		pmcsr = read_pci_config_16(bus, slot, func, BCM4331_PM_CAP + PCI_PM_CTRL);
 		if ((pmcsr & PCI_PM_CTRL_STATE_MASK) != PCI_D0) {
-			dev_err("Cannot power up Apple AirPort card\n");
+			pr_err("pci 0000:%02x:%02x.%d: Cannot power up Apple AirPort card\n",
+			       bus, slot, func);
 			return;
 		}
 	}
@@ -628,7 +627,8 @@ static void __init apple_airport_reset(int bus, int slot, int func)
 
 	mmio = early_ioremap(addr, BCM4331_MMIO_SIZE);
 	if (!mmio) {
-		dev_err("Cannot iomap Apple AirPort card\n");
+		pr_err("pci 0000:%02x:%02x.%d: Cannot iomap Apple AirPort card\n",
+		       bus, slot, func);
 		return;
 	}
 
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c
index 8ce4212e2b8d..b6be34ee88e9 100644
--- a/arch/x86/kernel/hpet.c
+++ b/arch/x86/kernel/hpet.c
@@ -975,8 +975,7 @@ int __init hpet_enable(void)
 	cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY);
 	hpet_writel(cfg, HPET_CFG);
 	if (cfg)
-		pr_warn("HPET: Unrecognized bits %#x set in global cfg\n",
-			cfg);
+		pr_warn("Unrecognized bits %#x set in global cfg\n", cfg);
 
 	for (i = 0; i <= last; ++i) {
 		cfg = hpet_readl(HPET_Tn_CFG(i));
@@ -988,7 +987,7 @@ int __init hpet_enable(void)
 			 | HPET_TN_64BIT_CAP | HPET_TN_32BIT | HPET_TN_ROUTE
 			 | HPET_TN_FSB | HPET_TN_FSB_CAP);
 		if (cfg)
-			pr_warn("HPET: Unrecognized bits %#x set in cfg#%u\n",
+			pr_warn("Unrecognized bits %#x set in cfg#%u\n",
 				cfg, i);
 	}
 	hpet_print_config();
diff --git a/arch/x86/kernel/i8237.c b/arch/x86/kernel/i8237.c
index 8eeaa81de066..0a3e70fd00d6 100644
--- a/arch/x86/kernel/i8237.c
+++ b/arch/x86/kernel/i8237.c
@@ -9,10 +9,12 @@
  * your option) any later version.
  */
 
+#include <linux/dmi.h>
 #include <linux/init.h>
 #include <linux/syscore_ops.h>
 
 #include <asm/dma.h>
+#include <asm/x86_init.h>
 
 /*
  * This module just handles suspend/resume issues with the
@@ -49,6 +51,29 @@ static struct syscore_ops i8237_syscore_ops = {
 
 static int __init i8237A_init_ops(void)
 {
+	/*
+	 * From SKL PCH onwards, the legacy DMA device is removed in which the
+	 * I/O ports (81h-83h, 87h, 89h-8Bh, 8Fh) related to it are removed
+	 * as well. All removed ports must return 0xff for a inb() request.
+	 *
+	 * Note: DMA_PAGE_2 (port 0x81) should not be checked for detecting
+	 * the presence of DMA device since it may be used by BIOS to decode
+	 * LPC traffic for POST codes. Original LPC only decodes one byte of
+	 * port 0x80 but some BIOS may choose to enhance PCH LPC port 0x8x
+	 * decoding.
+	 */
+	if (dma_inb(DMA_PAGE_0) == 0xFF)
+		return -ENODEV;
+
+	/*
+	 * It is not required to load this driver as newer SoC may not
+	 * support 8237 DMA or bus mastering from LPC. Platform firmware
+	 * must announce the support for such legacy devices via
+	 * ACPI_FADT_LEGACY_DEVICES field in FADT table.
+	 */
+	if (x86_pnpbios_disabled() && dmi_get_bios_year() >= 2017)
+		return -ENODEV;
+
 	register_syscore_ops(&i8237_syscore_ops);
 	return 0;
 }
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index 77625b60a510..ab5d9dd668d2 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -15,13 +15,11 @@
 #include <asm/x86_init.h>
 #include <asm/iommu_table.h>
 
-static int forbid_dac __read_mostly;
+static bool disable_dac_quirk __read_mostly;
 
 const struct dma_map_ops *dma_ops = &dma_direct_ops;
 EXPORT_SYMBOL(dma_ops);
 
-static int iommu_sac_force __read_mostly;
-
 #ifdef CONFIG_IOMMU_DEBUG
 int panic_on_overflow __read_mostly = 1;
 int force_iommu __read_mostly = 1;
@@ -55,9 +53,6 @@ struct device x86_dma_fallback_dev = {
 };
 EXPORT_SYMBOL(x86_dma_fallback_dev);
 
-/* Number of entries preallocated for DMA-API debugging */
-#define PREALLOC_DMA_DEBUG_ENTRIES       65536
-
 void __init pci_iommu_alloc(void)
 {
 	struct iommu_table_entry *p;
@@ -76,7 +71,7 @@ void __init pci_iommu_alloc(void)
 	}
 }
 
-bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp)
+bool arch_dma_alloc_attrs(struct device **dev)
 {
 	if (!*dev)
 		*dev = &x86_dma_fallback_dev;
@@ -125,13 +120,13 @@ static __init int iommu_setup(char *p)
 		if (!strncmp(p, "nomerge", 7))
 			iommu_merge = 0;
 		if (!strncmp(p, "forcesac", 8))
-			iommu_sac_force = 1;
+			pr_warn("forcesac option ignored.\n");
 		if (!strncmp(p, "allowdac", 8))
-			forbid_dac = 0;
+			pr_warn("allowdac option ignored.\n");
 		if (!strncmp(p, "nodac", 5))
-			forbid_dac = 1;
+			pr_warn("nodac option ignored.\n");
 		if (!strncmp(p, "usedac", 6)) {
-			forbid_dac = -1;
+			disable_dac_quirk = true;
 			return 1;
 		}
 #ifdef CONFIG_SWIOTLB
@@ -156,40 +151,9 @@ static __init int iommu_setup(char *p)
 }
 early_param("iommu", iommu_setup);
 
-int arch_dma_supported(struct device *dev, u64 mask)
-{
-#ifdef CONFIG_PCI
-	if (mask > 0xffffffff && forbid_dac > 0) {
-		dev_info(dev, "PCI: Disallowing DAC for device\n");
-		return 0;
-	}
-#endif
-
-	/* Tell the device to use SAC when IOMMU force is on.  This
-	   allows the driver to use cheaper accesses in some cases.
-
-	   Problem with this is that if we overflow the IOMMU area and
-	   return DAC as fallback address the device may not handle it
-	   correctly.
-
-	   As a special case some controllers have a 39bit address
-	   mode that is as efficient as 32bit (aic79xx). Don't force
-	   SAC for these.  Assume all masks <= 40 bits are of this
-	   type. Normally this doesn't make any difference, but gives
-	   more gentle handling of IOMMU overflow. */
-	if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
-		dev_info(dev, "Force SAC with mask %Lx\n", mask);
-		return 0;
-	}
-
-	return 1;
-}
-EXPORT_SYMBOL(arch_dma_supported);
-
 static int __init pci_iommu_init(void)
 {
 	struct iommu_table_entry *p;
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 
 #ifdef CONFIG_PCI
 	dma_debug_add_bus(&pci_bus_type);
@@ -209,11 +173,17 @@ rootfs_initcall(pci_iommu_init);
 #ifdef CONFIG_PCI
 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 
+static int via_no_dac_cb(struct pci_dev *pdev, void *data)
+{
+	pdev->dev.dma_32bit_limit = true;
+	return 0;
+}
+
 static void via_no_dac(struct pci_dev *dev)
 {
-	if (forbid_dac == 0) {
+	if (!disable_dac_quirk) {
 		dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
-		forbid_dac = 1;
+		pci_walk_bus(dev->subordinate, via_no_dac_cb, NULL);
 	}
 }
 DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID,
diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c
index 235fe6008ac8..b348a672f71d 100644
--- a/arch/x86/kernel/platform-quirks.c
+++ b/arch/x86/kernel/platform-quirks.c
@@ -33,9 +33,14 @@ void __init x86_early_init_platform_quirks(void)
 		x86_platform.set_legacy_features();
 }
 
+bool __init x86_pnpbios_disabled(void)
+{
+	return x86_platform.legacy.devices.pnpbios == 0;
+}
+
 #if defined(CONFIG_PNPBIOS)
 bool __init arch_pnpbios_disabled(void)
 {
-	return x86_platform.legacy.devices.pnpbios == 0;
+	return x86_pnpbios_disabled();
 }
 #endif
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 5224c6099184..0ae659de21eb 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -76,16 +76,14 @@ void __show_regs(struct pt_regs *regs, int all)
 		savesegment(gs, gs);
 	}
 
-	printk(KERN_DEFAULT "EIP: %pS\n", (void *)regs->ip);
-	printk(KERN_DEFAULT "EFLAGS: %08lx CPU: %d\n", regs->flags,
-		raw_smp_processor_id());
+	show_ip(regs, KERN_DEFAULT);
 
 	printk(KERN_DEFAULT "EAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n",
 		regs->ax, regs->bx, regs->cx, regs->dx);
 	printk(KERN_DEFAULT "ESI: %08lx EDI: %08lx EBP: %08lx ESP: %08lx\n",
 		regs->si, regs->di, regs->bp, sp);
-	printk(KERN_DEFAULT " DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x\n",
-	       (u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, ss);
+	printk(KERN_DEFAULT "DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x EFLAGS: %08lx\n",
+	       (u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, ss, regs->flags);
 
 	if (!all)
 		return;
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index ed5c4cdf0a34..e2ee403865eb 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -1377,7 +1377,6 @@ static void fill_sigtrap_info(struct task_struct *tsk,
 	tsk->thread.trap_nr = X86_TRAP_DB;
 	tsk->thread.error_code = error_code;
 
-	memset(info, 0, sizeof(*info));
 	info->si_signo = SIGTRAP;
 	info->si_code = si_code;
 	info->si_addr = user_mode(regs) ? (void __user *)regs->ip : NULL;
@@ -1395,6 +1394,7 @@ void send_sigtrap(struct task_struct *tsk, struct pt_regs *regs,
 {
 	struct siginfo info;
 
+	clear_siginfo(&info);
 	fill_sigtrap_info(tsk, regs, error_code, si_code, &info);
 	/* Send us the fake SIGTRAP */
 	force_sig_info(SIGTRAP, &info, tsk);
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 5c623dfe39d1..2f86d883dd95 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -1312,11 +1312,3 @@ static int __init register_kernel_offset_dumper(void)
 	return 0;
 }
 __initcall(register_kernel_offset_dumper);
-
-void arch_show_smap(struct seq_file *m, struct vm_area_struct *vma)
-{
-	if (!boot_cpu_has(X86_FEATURE_OSPKE))
-		return;
-
-	seq_printf(m, "ProtectionKey:  %8u\n", vma_pkey(vma));
-}
diff --git a/arch/x86/kernel/signal_compat.c b/arch/x86/kernel/signal_compat.c
index 14c057f29979..9ccbf0576cd0 100644
--- a/arch/x86/kernel/signal_compat.c
+++ b/arch/x86/kernel/signal_compat.c
@@ -29,7 +29,7 @@ static inline void signal_compat_build_tests(void)
 	BUILD_BUG_ON(NSIGFPE  != 15);
 	BUILD_BUG_ON(NSIGSEGV != 7);
 	BUILD_BUG_ON(NSIGBUS  != 5);
-	BUILD_BUG_ON(NSIGTRAP != 4);
+	BUILD_BUG_ON(NSIGTRAP != 5);
 	BUILD_BUG_ON(NSIGCHLD != 6);
 	BUILD_BUG_ON(NSIGSYS  != 1);
 
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 9dd324ae4832..c2f7d1d2a5c3 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -81,13 +81,6 @@
 #include <asm/cpu_device_id.h>
 #include <asm/spec-ctrl.h>
 
-/* Number of siblings per CPU package */
-int smp_num_siblings = 1;
-EXPORT_SYMBOL(smp_num_siblings);
-
-/* Last level cache ID of each logical CPU */
-DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
-
 /* representing HT siblings of each logical CPU */
 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c
index a3f15ed545b5..6a78d4b36a79 100644
--- a/arch/x86/kernel/sys_x86_64.c
+++ b/arch/x86/kernel/sys_x86_64.c
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
+#include <linux/compat.h>
 #include <linux/errno.h>
 #include <linux/sched.h>
 #include <linux/sched/mm.h>
@@ -19,7 +20,6 @@
 #include <linux/elf.h>
 
 #include <asm/elf.h>
-#include <asm/compat.h>
 #include <asm/ia32.h>
 #include <asm/syscalls.h>
 #include <asm/mpx.h>
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 03f3d7695dac..a535dd64de63 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -299,6 +299,7 @@ static void do_error_trap(struct pt_regs *regs, long error_code, char *str,
 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=
 			NOTIFY_STOP) {
 		cond_local_irq_enable(regs);
+		clear_siginfo(&info);
 		do_trap(trapnr, signr, str, regs, error_code,
 			fill_trap_info(regs, signr, trapnr, &info));
 	}
@@ -854,6 +855,7 @@ static void math_error(struct pt_regs *regs, int error_code, int trapnr)
 
 	task->thread.trap_nr	= trapnr;
 	task->thread.error_code = error_code;
+	clear_siginfo(&info);
 	info.si_signo		= SIGFPE;
 	info.si_errno		= 0;
 	info.si_addr		= (void __user *)uprobe_get_trap_addr(regs);
@@ -929,6 +931,7 @@ dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
 	RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
 	local_irq_enable();
 
+	clear_siginfo(&info);
 	info.si_signo = SIGILL;
 	info.si_errno = 0;
 	info.si_code = ILL_BADSTK;
diff --git a/arch/x86/kernel/umip.c b/arch/x86/kernel/umip.c
index f44ce0fb3583..ff20b35e98dd 100644
--- a/arch/x86/kernel/umip.c
+++ b/arch/x86/kernel/umip.c
@@ -278,6 +278,7 @@ static void force_sig_info_umip_fault(void __user *addr, struct pt_regs *regs)
 	tsk->thread.error_code	= X86_PF_USER | X86_PF_WRITE;
 	tsk->thread.trap_nr	= X86_TRAP_PF;
 
+	clear_siginfo(&info);
 	info.si_signo	= SIGSEGV;
 	info.si_errno	= 0;
 	info.si_code	= SEGV_MAPERR;
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index c84bb5396958..58d8d800875d 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -1083,8 +1083,8 @@ arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs
 		return orig_ret_vaddr;
 
 	if (nleft != rasize) {
-		pr_err("uprobe: return address clobbered: pid=%d, %%sp=%#lx, "
-			"%%ip=%#lx\n", current->pid, regs->sp, regs->ip);
+		pr_err("return address clobbered: pid=%d, %%sp=%#lx, %%ip=%#lx\n",
+		       current->pid, regs->sp, regs->ip);
 
 		force_sig_info(SIGSEGV, SEND_SIG_FORCED, current);
 	}
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 795f3a80e576..5e1458f609a1 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -117,11 +117,11 @@ SECTIONS
 
 #ifdef CONFIG_X86_64
 		. = ALIGN(PAGE_SIZE);
-		VMLINUX_SYMBOL(__entry_trampoline_start) = .;
+		__entry_trampoline_start = .;
 		_entry_trampoline = .;
 		*(.entry_trampoline)
 		. = ALIGN(PAGE_SIZE);
-		VMLINUX_SYMBOL(__entry_trampoline_end) = .;
+		__entry_trampoline_end = .;
 		ASSERT(. - _entry_trampoline == PAGE_SIZE, "entry trampoline is too big");
 #endif
 
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 92bf2f2e7cdd..15ba2b53bb68 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -403,7 +403,8 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 	const u32 kvm_cpuid_7_0_ecx_x86_features =
 		F(AVX512VBMI) | F(LA57) | F(PKU) | 0 /*OSPKE*/ |
 		F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) |
-		F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG);
+		F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG) |
+		F(CLDEMOTE);
 
 	/* cpuid 7.0.edx*/
 	const u32 kvm_cpuid_7_0_edx_x86_features =
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index b3705ae52824..143b7ae52624 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -4189,7 +4189,9 @@ static int check_cr_write(struct x86_emulate_ctxt *ctxt)
 				maxphyaddr = eax & 0xff;
 			else
 				maxphyaddr = 36;
-			rsvd = rsvd_bits(maxphyaddr, 62);
+			rsvd = rsvd_bits(maxphyaddr, 63);
+			if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
+				rsvd &= ~CR3_PCID_INVD;
 		}
 
 		if (new_val & rsvd)
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 3773c4625114..b5cd8465d44f 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -2002,13 +2002,11 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
 		}
 	}
 
-	if ((old_value ^ value) & X2APIC_ENABLE) {
-		if (value & X2APIC_ENABLE) {
-			kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
-			kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true);
-		} else
-			kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false);
-	}
+	if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE))
+		kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
+
+	if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
+		kvm_x86_ops->set_virtual_apic_mode(vcpu);
 
 	apic->base_address = apic->vcpu->arch.apic_base &
 			     MSR_IA32_APICBASE_BASE;
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index edce055e9fd7..ed0ed39abd36 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -16,6 +16,13 @@
 #define APIC_BUS_CYCLE_NS       1
 #define APIC_BUS_FREQUENCY      (1000000000ULL / APIC_BUS_CYCLE_NS)
 
+enum lapic_mode {
+	LAPIC_MODE_DISABLED = 0,
+	LAPIC_MODE_INVALID = X2APIC_ENABLE,
+	LAPIC_MODE_XAPIC = MSR_IA32_APICBASE_ENABLE,
+	LAPIC_MODE_X2APIC = MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE,
+};
+
 struct kvm_timer {
 	struct hrtimer timer;
 	s64 period; 				/* unit: ns */
@@ -89,6 +96,7 @@ u64 kvm_get_apic_base(struct kvm_vcpu *vcpu);
 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info);
 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s);
 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s);
+enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu);
 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu);
 
 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu);
@@ -220,4 +228,10 @@ void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu);
 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu);
 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu);
 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu);
+
+static inline enum lapic_mode kvm_apic_mode(u64 apic_base)
+{
+	return apic_base & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
+}
+
 #endif
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 8494dbae41b9..d594690d8b95 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -222,7 +222,6 @@ static const u64 shadow_acc_track_saved_bits_mask = PT64_EPT_READABLE_MASK |
 static const u64 shadow_acc_track_saved_bits_shift = PT64_SECOND_AVAIL_BITS_SHIFT;
 
 static void mmu_spte_set(u64 *sptep, u64 spte);
-static void mmu_free_roots(struct kvm_vcpu *vcpu);
 
 void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value)
 {
@@ -3007,6 +3006,7 @@ static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *
 {
 	siginfo_t info;
 
+	clear_siginfo(&info);
 	info.si_signo	= SIGBUS;
 	info.si_errno	= 0;
 	info.si_code	= BUS_MCEERR_AR;
@@ -3342,51 +3342,48 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
 	return RET_PF_RETRY;
 }
 
-
-static void mmu_free_roots(struct kvm_vcpu *vcpu)
+static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa,
+			       struct list_head *invalid_list)
 {
-	int i;
 	struct kvm_mmu_page *sp;
-	LIST_HEAD(invalid_list);
 
-	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
+	if (!VALID_PAGE(*root_hpa))
 		return;
 
-	if (vcpu->arch.mmu.shadow_root_level >= PT64_ROOT_4LEVEL &&
-	    (vcpu->arch.mmu.root_level >= PT64_ROOT_4LEVEL ||
-	     vcpu->arch.mmu.direct_map)) {
-		hpa_t root = vcpu->arch.mmu.root_hpa;
+	sp = page_header(*root_hpa & PT64_BASE_ADDR_MASK);
+	--sp->root_count;
+	if (!sp->root_count && sp->role.invalid)
+		kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
 
-		spin_lock(&vcpu->kvm->mmu_lock);
-		sp = page_header(root);
-		--sp->root_count;
-		if (!sp->root_count && sp->role.invalid) {
-			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
-			kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
-		}
-		spin_unlock(&vcpu->kvm->mmu_lock);
-		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
+	*root_hpa = INVALID_PAGE;
+}
+
+void kvm_mmu_free_roots(struct kvm_vcpu *vcpu)
+{
+	int i;
+	LIST_HEAD(invalid_list);
+	struct kvm_mmu *mmu = &vcpu->arch.mmu;
+
+	if (!VALID_PAGE(mmu->root_hpa))
 		return;
-	}
 
 	spin_lock(&vcpu->kvm->mmu_lock);
-	for (i = 0; i < 4; ++i) {
-		hpa_t root = vcpu->arch.mmu.pae_root[i];
 
-		if (root) {
-			root &= PT64_BASE_ADDR_MASK;
-			sp = page_header(root);
-			--sp->root_count;
-			if (!sp->root_count && sp->role.invalid)
-				kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
-							 &invalid_list);
-		}
-		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
+	if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL &&
+	    (mmu->root_level >= PT64_ROOT_4LEVEL || mmu->direct_map)) {
+		mmu_free_root_page(vcpu->kvm, &mmu->root_hpa, &invalid_list);
+	} else {
+		for (i = 0; i < 4; ++i)
+			if (mmu->pae_root[i] != 0)
+				mmu_free_root_page(vcpu->kvm, &mmu->pae_root[i],
+						   &invalid_list);
+		mmu->root_hpa = INVALID_PAGE;
 	}
+
 	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
 	spin_unlock(&vcpu->kvm->mmu_lock);
-	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
 }
+EXPORT_SYMBOL_GPL(kvm_mmu_free_roots);
 
 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
 {
@@ -3719,7 +3716,6 @@ static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
 	 */
 	return RET_PF_RETRY;
 }
-EXPORT_SYMBOL_GPL(handle_mmio_page_fault);
 
 static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
 					 u32 error_code, gfn_t gfn)
@@ -3811,6 +3807,14 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
 	struct kvm_memory_slot *slot;
 	bool async;
 
+	/*
+	 * Don't expose private memslots to L2.
+	 */
+	if (is_guest_mode(vcpu) && !kvm_is_visible_gfn(vcpu->kvm, gfn)) {
+		*pfn = KVM_PFN_NOSLOT;
+		return false;
+	}
+
 	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
 	async = false;
 	*pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable);
@@ -3950,7 +3954,7 @@ static void nonpaging_init_context(struct kvm_vcpu *vcpu,
 
 void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu)
 {
-	mmu_free_roots(vcpu);
+	kvm_mmu_free_roots(vcpu);
 }
 
 static unsigned long get_cr3(struct kvm_vcpu *vcpu)
@@ -4472,6 +4476,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
 	struct kvm_mmu *context = &vcpu->arch.mmu;
 
 	context->base_role.word = 0;
+	context->base_role.guest_mode = is_guest_mode(vcpu);
 	context->base_role.smm = is_smm(vcpu);
 	context->base_role.ad_disabled = (shadow_accessed_mask == 0);
 	context->page_fault = tdp_page_fault;
@@ -4538,6 +4543,7 @@ void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu)
 		= smep && !is_write_protection(vcpu);
 	context->base_role.smap_andnot_wp
 		= smap && !is_write_protection(vcpu);
+	context->base_role.guest_mode = is_guest_mode(vcpu);
 	context->base_role.smm = is_smm(vcpu);
 	reset_shadow_zero_bits_mask(vcpu, context);
 }
@@ -4563,7 +4569,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
 	context->root_hpa = INVALID_PAGE;
 	context->direct_map = false;
 	context->base_role.ad_disabled = !accessed_dirty;
-
+	context->base_role.guest_mode = 1;
 	update_permission_bitmask(vcpu, context, true);
 	update_pkru_bitmask(vcpu, context, true);
 	update_last_nonleaf_level(vcpu, context);
@@ -4663,7 +4669,7 @@ EXPORT_SYMBOL_GPL(kvm_mmu_load);
 
 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
 {
-	mmu_free_roots(vcpu);
+	kvm_mmu_free_roots(vcpu);
 	WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_unload);
@@ -4824,6 +4830,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
 	mask.smep_andnot_wp = 1;
 	mask.smap_andnot_wp = 1;
 	mask.smm = 1;
+	mask.guest_mode = 1;
 	mask.ad_disabled = 1;
 
 	/*
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 26110c202b19..b2e7140f23ea 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -1768,7 +1768,10 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
 	unsigned long npages, npinned, size;
 	unsigned long locked, lock_limit;
 	struct page **pages;
-	int first, last;
+	unsigned long first, last;
+
+	if (ulen == 0 || uaddr + ulen < uaddr)
+		return NULL;
 
 	/* Calculate number of pages. */
 	first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
@@ -5060,7 +5063,7 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
 		set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
 }
 
-static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
+static void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
 {
 	return;
 }
@@ -6947,6 +6950,9 @@ static int svm_register_enc_region(struct kvm *kvm,
 	if (!sev_guest(kvm))
 		return -ENOTTY;
 
+	if (range->addr > ULONG_MAX || range->size > ULONG_MAX)
+		return -EINVAL;
+
 	region = kzalloc(sizeof(*region), GFP_KERNEL);
 	if (!region)
 		return -ENOMEM;
@@ -7098,7 +7104,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
 	.enable_nmi_window = enable_nmi_window,
 	.enable_irq_window = enable_irq_window,
 	.update_cr8_intercept = update_cr8_intercept,
-	.set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
+	.set_virtual_apic_mode = svm_set_virtual_apic_mode,
 	.get_enable_apicv = svm_get_enable_apicv,
 	.refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
 	.load_eoi_exitmap = svm_load_eoi_exitmap,
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 40aa29204baf..549df83278b2 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -242,7 +242,11 @@ struct shared_msr_entry {
  * underlying hardware which will be used to run L2.
  * This structure is packed to ensure that its layout is identical across
  * machines (necessary for live migration).
- * If there are changes in this struct, VMCS12_REVISION must be changed.
+ *
+ * IMPORTANT: Changing the layout of existing fields in this structure
+ * will break save/restore compatibility with older kvm releases. When
+ * adding new fields, either use space in the reserved padding* arrays
+ * or add the new fields to the end of the structure.
  */
 typedef u64 natural_width;
 struct __packed vmcs12 {
@@ -265,17 +269,14 @@ struct __packed vmcs12 {
 	u64 virtual_apic_page_addr;
 	u64 apic_access_addr;
 	u64 posted_intr_desc_addr;
-	u64 vm_function_control;
 	u64 ept_pointer;
 	u64 eoi_exit_bitmap0;
 	u64 eoi_exit_bitmap1;
 	u64 eoi_exit_bitmap2;
 	u64 eoi_exit_bitmap3;
-	u64 eptp_list_address;
 	u64 xss_exit_bitmap;
 	u64 guest_physical_address;
 	u64 vmcs_link_pointer;
-	u64 pml_address;
 	u64 guest_ia32_debugctl;
 	u64 guest_ia32_pat;
 	u64 guest_ia32_efer;
@@ -288,7 +289,12 @@ struct __packed vmcs12 {
 	u64 host_ia32_pat;
 	u64 host_ia32_efer;
 	u64 host_ia32_perf_global_ctrl;
-	u64 padding64[8]; /* room for future expansion */
+	u64 vmread_bitmap;
+	u64 vmwrite_bitmap;
+	u64 vm_function_control;
+	u64 eptp_list_address;
+	u64 pml_address;
+	u64 padding64[3]; /* room for future expansion */
 	/*
 	 * To allow migration of L1 (complete with its L2 guests) between
 	 * machines of different natural widths (32 or 64 bit), we cannot have
@@ -397,7 +403,6 @@ struct __packed vmcs12 {
 	u16 guest_ldtr_selector;
 	u16 guest_tr_selector;
 	u16 guest_intr_status;
-	u16 guest_pml_index;
 	u16 host_es_selector;
 	u16 host_cs_selector;
 	u16 host_ss_selector;
@@ -405,12 +410,172 @@ struct __packed vmcs12 {
 	u16 host_fs_selector;
 	u16 host_gs_selector;
 	u16 host_tr_selector;
+	u16 guest_pml_index;
 };
 
+/*
+ * For save/restore compatibility, the vmcs12 field offsets must not change.
+ */
+#define CHECK_OFFSET(field, loc)				\
+	BUILD_BUG_ON_MSG(offsetof(struct vmcs12, field) != (loc),	\
+		"Offset of " #field " in struct vmcs12 has changed.")
+
+static inline void vmx_check_vmcs12_offsets(void) {
+	CHECK_OFFSET(revision_id, 0);
+	CHECK_OFFSET(abort, 4);
+	CHECK_OFFSET(launch_state, 8);
+	CHECK_OFFSET(io_bitmap_a, 40);
+	CHECK_OFFSET(io_bitmap_b, 48);
+	CHECK_OFFSET(msr_bitmap, 56);
+	CHECK_OFFSET(vm_exit_msr_store_addr, 64);
+	CHECK_OFFSET(vm_exit_msr_load_addr, 72);
+	CHECK_OFFSET(vm_entry_msr_load_addr, 80);
+	CHECK_OFFSET(tsc_offset, 88);
+	CHECK_OFFSET(virtual_apic_page_addr, 96);
+	CHECK_OFFSET(apic_access_addr, 104);
+	CHECK_OFFSET(posted_intr_desc_addr, 112);
+	CHECK_OFFSET(ept_pointer, 120);
+	CHECK_OFFSET(eoi_exit_bitmap0, 128);
+	CHECK_OFFSET(eoi_exit_bitmap1, 136);
+	CHECK_OFFSET(eoi_exit_bitmap2, 144);
+	CHECK_OFFSET(eoi_exit_bitmap3, 152);
+	CHECK_OFFSET(xss_exit_bitmap, 160);
+	CHECK_OFFSET(guest_physical_address, 168);
+	CHECK_OFFSET(vmcs_link_pointer, 176);
+	CHECK_OFFSET(guest_ia32_debugctl, 184);
+	CHECK_OFFSET(guest_ia32_pat, 192);
+	CHECK_OFFSET(guest_ia32_efer, 200);
+	CHECK_OFFSET(guest_ia32_perf_global_ctrl, 208);
+	CHECK_OFFSET(guest_pdptr0, 216);
+	CHECK_OFFSET(guest_pdptr1, 224);
+	CHECK_OFFSET(guest_pdptr2, 232);
+	CHECK_OFFSET(guest_pdptr3, 240);
+	CHECK_OFFSET(guest_bndcfgs, 248);
+	CHECK_OFFSET(host_ia32_pat, 256);
+	CHECK_OFFSET(host_ia32_efer, 264);
+	CHECK_OFFSET(host_ia32_perf_global_ctrl, 272);
+	CHECK_OFFSET(vmread_bitmap, 280);
+	CHECK_OFFSET(vmwrite_bitmap, 288);
+	CHECK_OFFSET(vm_function_control, 296);
+	CHECK_OFFSET(eptp_list_address, 304);
+	CHECK_OFFSET(pml_address, 312);
+	CHECK_OFFSET(cr0_guest_host_mask, 344);
+	CHECK_OFFSET(cr4_guest_host_mask, 352);
+	CHECK_OFFSET(cr0_read_shadow, 360);
+	CHECK_OFFSET(cr4_read_shadow, 368);
+	CHECK_OFFSET(cr3_target_value0, 376);
+	CHECK_OFFSET(cr3_target_value1, 384);
+	CHECK_OFFSET(cr3_target_value2, 392);
+	CHECK_OFFSET(cr3_target_value3, 400);
+	CHECK_OFFSET(exit_qualification, 408);
+	CHECK_OFFSET(guest_linear_address, 416);
+	CHECK_OFFSET(guest_cr0, 424);
+	CHECK_OFFSET(guest_cr3, 432);
+	CHECK_OFFSET(guest_cr4, 440);
+	CHECK_OFFSET(guest_es_base, 448);
+	CHECK_OFFSET(guest_cs_base, 456);
+	CHECK_OFFSET(guest_ss_base, 464);
+	CHECK_OFFSET(guest_ds_base, 472);
+	CHECK_OFFSET(guest_fs_base, 480);
+	CHECK_OFFSET(guest_gs_base, 488);
+	CHECK_OFFSET(guest_ldtr_base, 496);
+	CHECK_OFFSET(guest_tr_base, 504);
+	CHECK_OFFSET(guest_gdtr_base, 512);
+	CHECK_OFFSET(guest_idtr_base, 520);
+	CHECK_OFFSET(guest_dr7, 528);
+	CHECK_OFFSET(guest_rsp, 536);
+	CHECK_OFFSET(guest_rip, 544);
+	CHECK_OFFSET(guest_rflags, 552);
+	CHECK_OFFSET(guest_pending_dbg_exceptions, 560);
+	CHECK_OFFSET(guest_sysenter_esp, 568);
+	CHECK_OFFSET(guest_sysenter_eip, 576);
+	CHECK_OFFSET(host_cr0, 584);
+	CHECK_OFFSET(host_cr3, 592);
+	CHECK_OFFSET(host_cr4, 600);
+	CHECK_OFFSET(host_fs_base, 608);
+	CHECK_OFFSET(host_gs_base, 616);
+	CHECK_OFFSET(host_tr_base, 624);
+	CHECK_OFFSET(host_gdtr_base, 632);
+	CHECK_OFFSET(host_idtr_base, 640);
+	CHECK_OFFSET(host_ia32_sysenter_esp, 648);
+	CHECK_OFFSET(host_ia32_sysenter_eip, 656);
+	CHECK_OFFSET(host_rsp, 664);
+	CHECK_OFFSET(host_rip, 672);
+	CHECK_OFFSET(pin_based_vm_exec_control, 744);
+	CHECK_OFFSET(cpu_based_vm_exec_control, 748);
+	CHECK_OFFSET(exception_bitmap, 752);
+	CHECK_OFFSET(page_fault_error_code_mask, 756);
+	CHECK_OFFSET(page_fault_error_code_match, 760);
+	CHECK_OFFSET(cr3_target_count, 764);
+	CHECK_OFFSET(vm_exit_controls, 768);
+	CHECK_OFFSET(vm_exit_msr_store_count, 772);
+	CHECK_OFFSET(vm_exit_msr_load_count, 776);
+	CHECK_OFFSET(vm_entry_controls, 780);
+	CHECK_OFFSET(vm_entry_msr_load_count, 784);
+	CHECK_OFFSET(vm_entry_intr_info_field, 788);
+	CHECK_OFFSET(vm_entry_exception_error_code, 792);
+	CHECK_OFFSET(vm_entry_instruction_len, 796);
+	CHECK_OFFSET(tpr_threshold, 800);
+	CHECK_OFFSET(secondary_vm_exec_control, 804);
+	CHECK_OFFSET(vm_instruction_error, 808);
+	CHECK_OFFSET(vm_exit_reason, 812);
+	CHECK_OFFSET(vm_exit_intr_info, 816);
+	CHECK_OFFSET(vm_exit_intr_error_code, 820);
+	CHECK_OFFSET(idt_vectoring_info_field, 824);
+	CHECK_OFFSET(idt_vectoring_error_code, 828);
+	CHECK_OFFSET(vm_exit_instruction_len, 832);
+	CHECK_OFFSET(vmx_instruction_info, 836);
+	CHECK_OFFSET(guest_es_limit, 840);
+	CHECK_OFFSET(guest_cs_limit, 844);
+	CHECK_OFFSET(guest_ss_limit, 848);
+	CHECK_OFFSET(guest_ds_limit, 852);
+	CHECK_OFFSET(guest_fs_limit, 856);
+	CHECK_OFFSET(guest_gs_limit, 860);
+	CHECK_OFFSET(guest_ldtr_limit, 864);
+	CHECK_OFFSET(guest_tr_limit, 868);
+	CHECK_OFFSET(guest_gdtr_limit, 872);
+	CHECK_OFFSET(guest_idtr_limit, 876);
+	CHECK_OFFSET(guest_es_ar_bytes, 880);
+	CHECK_OFFSET(guest_cs_ar_bytes, 884);
+	CHECK_OFFSET(guest_ss_ar_bytes, 888);
+	CHECK_OFFSET(guest_ds_ar_bytes, 892);
+	CHECK_OFFSET(guest_fs_ar_bytes, 896);
+	CHECK_OFFSET(guest_gs_ar_bytes, 900);
+	CHECK_OFFSET(guest_ldtr_ar_bytes, 904);
+	CHECK_OFFSET(guest_tr_ar_bytes, 908);
+	CHECK_OFFSET(guest_interruptibility_info, 912);
+	CHECK_OFFSET(guest_activity_state, 916);
+	CHECK_OFFSET(guest_sysenter_cs, 920);
+	CHECK_OFFSET(host_ia32_sysenter_cs, 924);
+	CHECK_OFFSET(vmx_preemption_timer_value, 928);
+	CHECK_OFFSET(virtual_processor_id, 960);
+	CHECK_OFFSET(posted_intr_nv, 962);
+	CHECK_OFFSET(guest_es_selector, 964);
+	CHECK_OFFSET(guest_cs_selector, 966);
+	CHECK_OFFSET(guest_ss_selector, 968);
+	CHECK_OFFSET(guest_ds_selector, 970);
+	CHECK_OFFSET(guest_fs_selector, 972);
+	CHECK_OFFSET(guest_gs_selector, 974);
+	CHECK_OFFSET(guest_ldtr_selector, 976);
+	CHECK_OFFSET(guest_tr_selector, 978);
+	CHECK_OFFSET(guest_intr_status, 980);
+	CHECK_OFFSET(host_es_selector, 982);
+	CHECK_OFFSET(host_cs_selector, 984);
+	CHECK_OFFSET(host_ss_selector, 986);
+	CHECK_OFFSET(host_ds_selector, 988);
+	CHECK_OFFSET(host_fs_selector, 990);
+	CHECK_OFFSET(host_gs_selector, 992);
+	CHECK_OFFSET(host_tr_selector, 994);
+	CHECK_OFFSET(guest_pml_index, 996);
+}
+
 /*
  * VMCS12_REVISION is an arbitrary id that should be changed if the content or
  * layout of struct vmcs12 is changed. MSR_IA32_VMX_BASIC returns this id, and
  * VMPTRLD verifies that the VMCS region that L1 is loading contains this id.
+ *
+ * IMPORTANT: Changing this value will break save/restore compatibility with
+ * older kvm releases.
  */
 #define VMCS12_REVISION 0x11e57ed0
 
@@ -481,7 +646,8 @@ struct nested_vmx {
 	bool sync_shadow_vmcs;
 	bool dirty_vmcs12;
 
-	bool change_vmcs01_virtual_x2apic_mode;
+	bool change_vmcs01_virtual_apic_mode;
+
 	/* L2 must run next, and mustn't decide to exit to L1. */
 	bool nested_run_pending;
 
@@ -761,6 +927,7 @@ static const unsigned short vmcs_field_to_offset_table[] = {
 	FIELD64(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr),
 	FIELD64(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr),
 	FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr),
+	FIELD64(PML_ADDRESS, pml_address),
 	FIELD64(TSC_OFFSET, tsc_offset),
 	FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr),
 	FIELD64(APIC_ACCESS_ADDR, apic_access_addr),
@@ -772,10 +939,11 @@ static const unsigned short vmcs_field_to_offset_table[] = {
 	FIELD64(EOI_EXIT_BITMAP2, eoi_exit_bitmap2),
 	FIELD64(EOI_EXIT_BITMAP3, eoi_exit_bitmap3),
 	FIELD64(EPTP_LIST_ADDRESS, eptp_list_address),
+	FIELD64(VMREAD_BITMAP, vmread_bitmap),
+	FIELD64(VMWRITE_BITMAP, vmwrite_bitmap),
 	FIELD64(XSS_EXIT_BITMAP, xss_exit_bitmap),
 	FIELD64(GUEST_PHYSICAL_ADDRESS, guest_physical_address),
 	FIELD64(VMCS_LINK_POINTER, vmcs_link_pointer),
-	FIELD64(PML_ADDRESS, pml_address),
 	FIELD64(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl),
 	FIELD64(GUEST_IA32_PAT, guest_ia32_pat),
 	FIELD64(GUEST_IA32_EFER, guest_ia32_efer),
@@ -1089,6 +1257,16 @@ static inline u16 evmcs_read16(unsigned long field)
 	return *(u16 *)((char *)current_evmcs + offset);
 }
 
+static inline void evmcs_touch_msr_bitmap(void)
+{
+	if (unlikely(!current_evmcs))
+		return;
+
+	if (current_evmcs->hv_enlightenments_control.msr_bitmap)
+		current_evmcs->hv_clean_fields &=
+			~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP;
+}
+
 static void evmcs_load(u64 phys_addr)
 {
 	struct hv_vp_assist_page *vp_ap =
@@ -1173,6 +1351,7 @@ static inline u32 evmcs_read32(unsigned long field) { return 0; }
 static inline u16 evmcs_read16(unsigned long field) { return 0; }
 static inline void evmcs_load(u64 phys_addr) {}
 static inline void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf) {}
+static inline void evmcs_touch_msr_bitmap(void) {}
 #endif /* IS_ENABLED(CONFIG_HYPERV) */
 
 static inline bool is_exception_n(u32 intr_info, u8 vector)
@@ -1393,6 +1572,11 @@ static inline bool cpu_has_vmx_invept_global(void)
 	return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
 }
 
+static inline bool cpu_has_vmx_invvpid_individual_addr(void)
+{
+	return vmx_capability.vpid & VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT;
+}
+
 static inline bool cpu_has_vmx_invvpid_single(void)
 {
 	return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
@@ -4216,6 +4400,14 @@ static int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
 		if (!loaded_vmcs->msr_bitmap)
 			goto out_vmcs;
 		memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE);
+
+		if (static_branch_unlikely(&enable_evmcs) &&
+		    (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) {
+			struct hv_enlightened_vmcs *evmcs =
+				(struct hv_enlightened_vmcs *)loaded_vmcs->vmcs;
+
+			evmcs->hv_enlightenments_control.msr_bitmap = 1;
+		}
 	}
 	return 0;
 
@@ -5329,6 +5521,9 @@ static void __always_inline vmx_disable_intercept_for_msr(unsigned long *msr_bit
 	if (!cpu_has_vmx_msr_bitmap())
 		return;
 
+	if (static_branch_unlikely(&enable_evmcs))
+		evmcs_touch_msr_bitmap();
+
 	/*
 	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
 	 * have the write-low and read-high bitmap offsets the wrong way round.
@@ -5364,6 +5559,9 @@ static void __always_inline vmx_enable_intercept_for_msr(unsigned long *msr_bitm
 	if (!cpu_has_vmx_msr_bitmap())
 		return;
 
+	if (static_branch_unlikely(&enable_evmcs))
+		evmcs_touch_msr_bitmap();
+
 	/*
 	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
 	 * have the write-low and read-high bitmap offsets the wrong way round.
@@ -8317,12 +8515,19 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
 
 	switch (type) {
 	case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
-		if (is_noncanonical_address(operand.gla, vcpu)) {
+		if (!operand.vpid ||
+		    is_noncanonical_address(operand.gla, vcpu)) {
 			nested_vmx_failValid(vcpu,
 				VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
 			return kvm_skip_emulated_instruction(vcpu);
 		}
-		/* fall through */
+		if (cpu_has_vmx_invvpid_individual_addr() &&
+		    vmx->nested.vpid02) {
+			__invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR,
+				vmx->nested.vpid02, operand.gla);
+		} else
+			__vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
+		break;
 	case VMX_VPID_EXTENT_SINGLE_CONTEXT:
 	case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
 		if (!operand.vpid) {
@@ -8330,15 +8535,16 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
 				VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
 			return kvm_skip_emulated_instruction(vcpu);
 		}
+		__vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
 		break;
 	case VMX_VPID_EXTENT_ALL_CONTEXT:
+		__vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
 		break;
 	default:
 		WARN_ON_ONCE(1);
 		return kvm_skip_emulated_instruction(vcpu);
 	}
 
-	__vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
 	nested_vmx_succeed(vcpu);
 
 	return kvm_skip_emulated_instruction(vcpu);
@@ -8842,11 +9048,13 @@ static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason)
 	case EXIT_REASON_TPR_BELOW_THRESHOLD:
 		return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
 	case EXIT_REASON_APIC_ACCESS:
-		return nested_cpu_has2(vmcs12,
-			SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
 	case EXIT_REASON_APIC_WRITE:
 	case EXIT_REASON_EOI_INDUCED:
-		/* apic_write and eoi_induced should exit unconditionally. */
+		/*
+		 * The controls for "virtualize APIC accesses," "APIC-
+		 * register virtualization," and "virtual-interrupt
+		 * delivery" only come from vmcs12.
+		 */
 		return true;
 	case EXIT_REASON_EPT_VIOLATION:
 		/*
@@ -9253,31 +9461,43 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
 	vmcs_write32(TPR_THRESHOLD, irr);
 }
 
-static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
+static void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
 {
 	u32 sec_exec_control;
 
+	if (!lapic_in_kernel(vcpu))
+		return;
+
 	/* Postpone execution until vmcs01 is the current VMCS. */
 	if (is_guest_mode(vcpu)) {
-		to_vmx(vcpu)->nested.change_vmcs01_virtual_x2apic_mode = true;
+		to_vmx(vcpu)->nested.change_vmcs01_virtual_apic_mode = true;
 		return;
 	}
 
-	if (!cpu_has_vmx_virtualize_x2apic_mode())
-		return;
-
 	if (!cpu_need_tpr_shadow(vcpu))
 		return;
 
 	sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
+	sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
+			      SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
 
-	if (set) {
-		sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
-		sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
-	} else {
-		sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
-		sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
-		vmx_flush_tlb(vcpu, true);
+	switch (kvm_get_apic_mode(vcpu)) {
+	case LAPIC_MODE_INVALID:
+		WARN_ONCE(true, "Invalid local APIC state");
+	case LAPIC_MODE_DISABLED:
+		break;
+	case LAPIC_MODE_XAPIC:
+		if (flexpriority_enabled) {
+			sec_exec_control |=
+				SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
+			vmx_flush_tlb(vcpu, true);
+		}
+		break;
+	case LAPIC_MODE_X2APIC:
+		if (cpu_has_vmx_virtualize_x2apic_mode())
+			sec_exec_control |=
+				SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
+		break;
 	}
 	vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);
 
@@ -9286,24 +9506,7 @@ static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
 
 static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa)
 {
-	struct vcpu_vmx *vmx = to_vmx(vcpu);
-
-	/*
-	 * Currently we do not handle the nested case where L2 has an
-	 * APIC access page of its own; that page is still pinned.
-	 * Hence, we skip the case where the VCPU is in guest mode _and_
-	 * L1 prepared an APIC access page for L2.
-	 *
-	 * For the case where L1 and L2 share the same APIC access page
-	 * (flexpriority=Y but SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES clear
-	 * in the vmcs12), this function will only update either the vmcs01
-	 * or the vmcs02.  If the former, the vmcs02 will be updated by
-	 * prepare_vmcs02.  If the latter, the vmcs01 will be updated in
-	 * the next L2->L1 exit.
-	 */
-	if (!is_guest_mode(vcpu) ||
-	    !nested_cpu_has2(get_vmcs12(&vmx->vcpu),
-			     SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
+	if (!is_guest_mode(vcpu)) {
 		vmcs_write64(APIC_ACCESS_ADDR, hpa);
 		vmx_flush_tlb(vcpu, true);
 	}
@@ -10387,11 +10590,6 @@ static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
 			vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
 					SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
 		}
-	} else if (!(nested_cpu_has_virt_x2apic_mode(vmcs12)) &&
-		   cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
-		vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
-			      SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
-		kvm_vcpu_reload_apic_access_page(vcpu);
 	}
 
 	if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
@@ -10871,8 +11069,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne
 	return 0;
 }
 
-static void prepare_vmcs02_full(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
-			       bool from_vmentry)
+static void prepare_vmcs02_full(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 
@@ -11006,13 +11203,13 @@ static void prepare_vmcs02_full(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
  * is assigned to entry_failure_code on failure.
  */
 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
-			  bool from_vmentry, u32 *entry_failure_code)
+			  u32 *entry_failure_code)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	u32 exec_control, vmcs12_exec_ctrl;
 
 	if (vmx->nested.dirty_vmcs12) {
-		prepare_vmcs02_full(vcpu, vmcs12, from_vmentry);
+		prepare_vmcs02_full(vcpu, vmcs12);
 		vmx->nested.dirty_vmcs12 = false;
 	}
 
@@ -11032,7 +11229,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	 * HOST_FS_BASE, HOST_GS_BASE.
 	 */
 
-	if (from_vmentry &&
+	if (vmx->nested.nested_run_pending &&
 	    (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) {
 		kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
 		vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
@@ -11040,7 +11237,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 		kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
 		vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
 	}
-	if (from_vmentry) {
+	if (vmx->nested.nested_run_pending) {
 		vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
 			     vmcs12->vm_entry_intr_info_field);
 		vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
@@ -11172,7 +11369,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 			~VM_ENTRY_IA32E_MODE) |
 		(vmcs_config.vmentry_ctrl & ~VM_ENTRY_IA32E_MODE));
 
-	if (from_vmentry &&
+	if (vmx->nested.nested_run_pending &&
 	    (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) {
 		vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
 		vcpu->arch.pat = vmcs12->guest_ia32_pat;
@@ -11197,7 +11394,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 		if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) {
 			if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
 				vmx->nested.last_vpid = vmcs12->virtual_processor_id;
-				__vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02, true);
+				__vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
 			}
 		} else {
 			vmx_flush_tlb(vcpu, true);
@@ -11240,7 +11437,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	vmx_set_cr4(vcpu, vmcs12->guest_cr4);
 	vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
 
-	if (from_vmentry &&
+	if (vmx->nested.nested_run_pending &&
 	    (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER))
 		vcpu->arch.efer = vmcs12->guest_ia32_efer;
 	else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
@@ -11418,7 +11615,7 @@ static int check_vmentry_postreqs(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	return 0;
 }
 
-static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry)
+static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
@@ -11438,7 +11635,7 @@ static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry)
 		vcpu->arch.tsc_offset += vmcs12->tsc_offset;
 
 	r = EXIT_REASON_INVALID_STATE;
-	if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &exit_qual))
+	if (prepare_vmcs02(vcpu, vmcs12, &exit_qual))
 		goto fail;
 
 	nested_get_vmcs12_pages(vcpu, vmcs12);
@@ -11540,20 +11737,22 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
 	 * the nested entry.
 	 */
 
-	ret = enter_vmx_non_root_mode(vcpu, true);
-	if (ret)
+	vmx->nested.nested_run_pending = 1;
+	ret = enter_vmx_non_root_mode(vcpu);
+	if (ret) {
+		vmx->nested.nested_run_pending = 0;
 		return ret;
+	}
 
 	/*
 	 * If we're entering a halted L2 vcpu and the L2 vcpu won't be woken
 	 * by event injection, halt vcpu.
 	 */
 	if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) &&
-	    !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK))
+	    !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK)) {
+		vmx->nested.nested_run_pending = 0;
 		return kvm_vcpu_halt(vcpu);
-
-	vmx->nested.nested_run_pending = 1;
-
+	}
 	return 1;
 
 out:
@@ -11925,12 +12124,20 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
 
 	load_vmcs12_mmu_host_state(vcpu, vmcs12);
 
-	if (enable_vpid) {
-		/*
-		 * Trivially support vpid by letting L2s share their parent
-		 * L1's vpid. TODO: move to a more elaborate solution, giving
-		 * each L2 its own vpid and exposing the vpid feature to L1.
-		 */
+	/*
+	 * If vmcs01 don't use VPID, CPU flushes TLB on every
+	 * VMEntry/VMExit. Thus, no need to flush TLB.
+	 *
+	 * If vmcs12 uses VPID, TLB entries populated by L2 are
+	 * tagged with vmx->nested.vpid02 while L1 entries are tagged
+	 * with vmx->vpid. Thus, no need to flush TLB.
+	 *
+	 * Therefore, flush TLB only in case vmcs01 uses VPID and
+	 * vmcs12 don't use VPID as in this case L1 & L2 TLB entries
+	 * are both tagged with vmx->vpid.
+	 */
+	if (enable_vpid &&
+	    !(nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02)) {
 		vmx_flush_tlb(vcpu, true);
 	}
 
@@ -12069,10 +12276,9 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
 	if (kvm_has_tsc_control)
 		decache_tsc_multiplier(vmx);
 
-	if (vmx->nested.change_vmcs01_virtual_x2apic_mode) {
-		vmx->nested.change_vmcs01_virtual_x2apic_mode = false;
-		vmx_set_virtual_x2apic_mode(vcpu,
-				vcpu->arch.apic_base & X2APIC_ENABLE);
+	if (vmx->nested.change_vmcs01_virtual_apic_mode) {
+		vmx->nested.change_vmcs01_virtual_apic_mode = false;
+		vmx_set_virtual_apic_mode(vcpu);
 	} else if (!nested_cpu_has_ept(vmcs12) &&
 		   nested_cpu_has2(vmcs12,
 				   SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
@@ -12615,7 +12821,7 @@ static int vmx_pre_leave_smm(struct kvm_vcpu *vcpu, u64 smbase)
 
 	if (vmx->nested.smm.guest_mode) {
 		vcpu->arch.hflags &= ~HF_SMM_MASK;
-		ret = enter_vmx_non_root_mode(vcpu, false);
+		ret = enter_vmx_non_root_mode(vcpu);
 		vcpu->arch.hflags |= HF_SMM_MASK;
 		if (ret)
 			return ret;
@@ -12700,7 +12906,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
 	.enable_nmi_window = enable_nmi_window,
 	.enable_irq_window = enable_irq_window,
 	.update_cr8_intercept = update_cr8_intercept,
-	.set_virtual_x2apic_mode = vmx_set_virtual_x2apic_mode,
+	.set_virtual_apic_mode = vmx_set_virtual_apic_mode,
 	.set_apic_access_page_addr = vmx_set_apic_access_page_addr,
 	.get_enable_apicv = vmx_get_enable_apicv,
 	.refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
@@ -12812,6 +13018,7 @@ static int __init vmx_init(void)
 	rcu_assign_pointer(crash_vmclear_loaded_vmcss,
 			   crash_vmclear_local_loaded_vmcss);
 #endif
+	vmx_check_vmcs12_offsets();
 
 	return 0;
 }
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 71e7cda6d014..db055b4fb06d 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -318,23 +318,27 @@ u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
 }
 EXPORT_SYMBOL_GPL(kvm_get_apic_base);
 
+enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
+{
+	return kvm_apic_mode(kvm_get_apic_base(vcpu));
+}
+EXPORT_SYMBOL_GPL(kvm_get_apic_mode);
+
 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 {
-	u64 old_state = vcpu->arch.apic_base &
-		(MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
-	u64 new_state = msr_info->data &
-		(MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
+	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
+	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
 	u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 0x2ff |
 		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
 
-	if ((msr_info->data & reserved_bits) || new_state == X2APIC_ENABLE)
-		return 1;
-	if (!msr_info->host_initiated &&
-	    ((new_state == MSR_IA32_APICBASE_ENABLE &&
-	      old_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) ||
-	     (new_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE) &&
-	      old_state == 0)))
+	if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
 		return 1;
+	if (!msr_info->host_initiated) {
+		if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
+			return 1;
+		if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
+			return 1;
+	}
 
 	kvm_lapic_set_base(vcpu, msr_info->data);
 	return 0;
@@ -856,7 +860,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 	}
 
 	if (is_long_mode(vcpu) &&
-	    (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 62)))
+	    (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63)))
 		return 1;
 	else if (is_pae(vcpu) && is_paging(vcpu) &&
 		   !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
@@ -1761,7 +1765,7 @@ static int do_monotonic_boot(s64 *t, u64 *tsc_timestamp)
 	return mode;
 }
 
-static int do_realtime(struct timespec *ts, u64 *tsc_timestamp)
+static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
 {
 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
 	unsigned long seq;
@@ -1794,7 +1798,7 @@ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
 }
 
 /* returns true if host is using TSC based clocksource */
-static bool kvm_get_walltime_and_clockread(struct timespec *ts,
+static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
 					   u64 *tsc_timestamp)
 {
 	/* checked again under seqlock below */
@@ -6617,7 +6621,7 @@ static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
 			        unsigned long clock_type)
 {
 	struct kvm_clock_pairing clock_pairing;
-	struct timespec ts;
+	struct timespec64 ts;
 	u64 cycle;
 	int ret;
 
diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S
index 9a53a06e5a3e..298ef1479240 100644
--- a/arch/x86/lib/memcpy_64.S
+++ b/arch/x86/lib/memcpy_64.S
@@ -3,6 +3,7 @@
 #include <linux/linkage.h>
 #include <asm/errno.h>
 #include <asm/cpufeatures.h>
+#include <asm/mcsafe_test.h>
 #include <asm/alternative-asm.h>
 #include <asm/export.h>
 
@@ -183,12 +184,15 @@ ENTRY(memcpy_orig)
 ENDPROC(memcpy_orig)
 
 #ifndef CONFIG_UML
+
+MCSAFE_TEST_CTL
+
 /*
- * memcpy_mcsafe_unrolled - memory copy with machine check exception handling
+ * __memcpy_mcsafe - memory copy with machine check exception handling
  * Note that we only catch machine checks when reading the source addresses.
  * Writes to target are posted and don't generate machine checks.
  */
-ENTRY(memcpy_mcsafe_unrolled)
+ENTRY(__memcpy_mcsafe)
 	cmpl $8, %edx
 	/* Less than 8 bytes? Go to byte copy loop */
 	jb .L_no_whole_words
@@ -204,58 +208,33 @@ ENTRY(memcpy_mcsafe_unrolled)
 	subl $8, %ecx
 	negl %ecx
 	subl %ecx, %edx
-.L_copy_leading_bytes:
+.L_read_leading_bytes:
 	movb (%rsi), %al
+	MCSAFE_TEST_SRC %rsi 1 .E_leading_bytes
+	MCSAFE_TEST_DST %rdi 1 .E_leading_bytes
+.L_write_leading_bytes:
 	movb %al, (%rdi)
 	incq %rsi
 	incq %rdi
 	decl %ecx
-	jnz .L_copy_leading_bytes
+	jnz .L_read_leading_bytes
 
 .L_8byte_aligned:
-	/* Figure out how many whole cache lines (64-bytes) to copy */
-	movl %edx, %ecx
-	andl $63, %edx
-	shrl $6, %ecx
-	jz .L_no_whole_cache_lines
-
-	/* Loop copying whole cache lines */
-.L_cache_w0: movq (%rsi), %r8
-.L_cache_w1: movq 1*8(%rsi), %r9
-.L_cache_w2: movq 2*8(%rsi), %r10
-.L_cache_w3: movq 3*8(%rsi), %r11
-	movq %r8, (%rdi)
-	movq %r9, 1*8(%rdi)
-	movq %r10, 2*8(%rdi)
-	movq %r11, 3*8(%rdi)
-.L_cache_w4: movq 4*8(%rsi), %r8
-.L_cache_w5: movq 5*8(%rsi), %r9
-.L_cache_w6: movq 6*8(%rsi), %r10
-.L_cache_w7: movq 7*8(%rsi), %r11
-	movq %r8, 4*8(%rdi)
-	movq %r9, 5*8(%rdi)
-	movq %r10, 6*8(%rdi)
-	movq %r11, 7*8(%rdi)
-	leaq 64(%rsi), %rsi
-	leaq 64(%rdi), %rdi
-	decl %ecx
-	jnz .L_cache_w0
-
-	/* Are there any trailing 8-byte words? */
-.L_no_whole_cache_lines:
 	movl %edx, %ecx
 	andl $7, %edx
 	shrl $3, %ecx
 	jz .L_no_whole_words
 
-	/* Copy trailing words */
-.L_copy_trailing_words:
+.L_read_words:
 	movq (%rsi), %r8
-	mov %r8, (%rdi)
-	leaq 8(%rsi), %rsi
-	leaq 8(%rdi), %rdi
+	MCSAFE_TEST_SRC %rsi 8 .E_read_words
+	MCSAFE_TEST_DST %rdi 8 .E_write_words
+.L_write_words:
+	movq %r8, (%rdi)
+	addq $8, %rsi
+	addq $8, %rdi
 	decl %ecx
-	jnz .L_copy_trailing_words
+	jnz .L_read_words
 
 	/* Any trailing bytes? */
 .L_no_whole_words:
@@ -264,38 +243,55 @@ ENTRY(memcpy_mcsafe_unrolled)
 
 	/* Copy trailing bytes */
 	movl %edx, %ecx
-.L_copy_trailing_bytes:
+.L_read_trailing_bytes:
 	movb (%rsi), %al
+	MCSAFE_TEST_SRC %rsi 1 .E_trailing_bytes
+	MCSAFE_TEST_DST %rdi 1 .E_trailing_bytes
+.L_write_trailing_bytes:
 	movb %al, (%rdi)
 	incq %rsi
 	incq %rdi
 	decl %ecx
-	jnz .L_copy_trailing_bytes
+	jnz .L_read_trailing_bytes
 
 	/* Copy successful. Return zero */
 .L_done_memcpy_trap:
 	xorq %rax, %rax
 	ret
-ENDPROC(memcpy_mcsafe_unrolled)
-EXPORT_SYMBOL_GPL(memcpy_mcsafe_unrolled)
+ENDPROC(__memcpy_mcsafe)
+EXPORT_SYMBOL_GPL(__memcpy_mcsafe)
 
 	.section .fixup, "ax"
-	/* Return -EFAULT for any failure */
-.L_memcpy_mcsafe_fail:
-	mov	$-EFAULT, %rax
+	/*
+	 * Return number of bytes not copied for any failure. Note that
+	 * there is no "tail" handling since the source buffer is 8-byte
+	 * aligned and poison is cacheline aligned.
+	 */
+.E_read_words:
+	shll	$3, %ecx
+.E_leading_bytes:
+	addl	%edx, %ecx
+.E_trailing_bytes:
+	mov	%ecx, %eax
 	ret
 
+	/*
+	 * For write fault handling, given the destination is unaligned,
+	 * we handle faults on multi-byte writes with a byte-by-byte
+	 * copy up to the write-protected page.
+	 */
+.E_write_words:
+	shll	$3, %ecx
+	addl	%edx, %ecx
+	movl	%ecx, %edx
+	jmp mcsafe_handle_tail
+
 	.previous
 
-	_ASM_EXTABLE_FAULT(.L_copy_leading_bytes, .L_memcpy_mcsafe_fail)
-	_ASM_EXTABLE_FAULT(.L_cache_w0, .L_memcpy_mcsafe_fail)
-	_ASM_EXTABLE_FAULT(.L_cache_w1, .L_memcpy_mcsafe_fail)
-	_ASM_EXTABLE_FAULT(.L_cache_w2, .L_memcpy_mcsafe_fail)
-	_ASM_EXTABLE_FAULT(.L_cache_w3, .L_memcpy_mcsafe_fail)
-	_ASM_EXTABLE_FAULT(.L_cache_w4, .L_memcpy_mcsafe_fail)
-	_ASM_EXTABLE_FAULT(.L_cache_w5, .L_memcpy_mcsafe_fail)
-	_ASM_EXTABLE_FAULT(.L_cache_w6, .L_memcpy_mcsafe_fail)
-	_ASM_EXTABLE_FAULT(.L_cache_w7, .L_memcpy_mcsafe_fail)
-	_ASM_EXTABLE_FAULT(.L_copy_trailing_words, .L_memcpy_mcsafe_fail)
-	_ASM_EXTABLE_FAULT(.L_copy_trailing_bytes, .L_memcpy_mcsafe_fail)
+	_ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes)
+	_ASM_EXTABLE_FAULT(.L_read_words, .E_read_words)
+	_ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes)
+	_ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)
+	_ASM_EXTABLE(.L_write_words, .E_write_words)
+	_ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)
 #endif
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 75d3776123cc..c50a1d815a37 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -23,13 +23,13 @@ unsigned long __clear_user(void __user *addr, unsigned long size)
 	asm volatile(
 		"	testq  %[size8],%[size8]\n"
 		"	jz     4f\n"
-		"0:	movq %[zero],(%[dst])\n"
-		"	addq   %[eight],%[dst]\n"
+		"0:	movq $0,(%[dst])\n"
+		"	addq   $8,%[dst]\n"
 		"	decl %%ecx ; jnz   0b\n"
 		"4:	movq  %[size1],%%rcx\n"
 		"	testl %%ecx,%%ecx\n"
 		"	jz     2f\n"
-		"1:	movb   %b[zero],(%[dst])\n"
+		"1:	movb   $0,(%[dst])\n"
 		"	incq   %[dst]\n"
 		"	decl %%ecx ; jnz  1b\n"
 		"2:\n"
@@ -40,8 +40,7 @@ unsigned long __clear_user(void __user *addr, unsigned long size)
 		_ASM_EXTABLE(0b,3b)
 		_ASM_EXTABLE(1b,2b)
 		: [size8] "=&c"(size), [dst] "=&D" (__d0)
-		: [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr),
-		  [zero] "r" (0UL), [eight] "r" (8UL));
+		: [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr));
 	clac();
 	return size;
 }
@@ -75,6 +74,27 @@ copy_user_handle_tail(char *to, char *from, unsigned len)
 	return len;
 }
 
+/*
+ * Similar to copy_user_handle_tail, probe for the write fault point,
+ * but reuse __memcpy_mcsafe in case a new read error is encountered.
+ * clac() is handled in _copy_to_iter_mcsafe().
+ */
+__visible unsigned long
+mcsafe_handle_tail(char *to, char *from, unsigned len)
+{
+	for (; len; --len, to++, from++) {
+		/*
+		 * Call the assembly routine back directly since
+		 * memcpy_mcsafe() may silently fallback to memcpy.
+		 */
+		unsigned long rem = __memcpy_mcsafe(to, from, 1);
+
+		if (rem)
+			break;
+	}
+	return len;
+}
+
 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 /**
  * clean_cache_range - write back a cache range with CLWB
@@ -133,7 +153,7 @@ long __copy_user_flushcache(void *dst, const void __user *src, unsigned size)
 	return rc;
 }
 
-void memcpy_flushcache(void *_dst, const void *_src, size_t size)
+void __memcpy_flushcache(void *_dst, const void *_src, size_t size)
 {
 	unsigned long dest = (unsigned long) _dst;
 	unsigned long source = (unsigned long) _src;
@@ -196,7 +216,7 @@ void memcpy_flushcache(void *_dst, const void *_src, size_t size)
 		clean_cache_range((void *) dest, size);
 	}
 }
-EXPORT_SYMBOL_GPL(memcpy_flushcache);
+EXPORT_SYMBOL_GPL(__memcpy_flushcache);
 
 void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
 		size_t len)
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 73bd8c95ac71..fd84edf82252 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -209,6 +209,7 @@ force_sig_info_fault(int si_signo, int si_code, unsigned long address,
 	unsigned lsb = 0;
 	siginfo_t info;
 
+	clear_siginfo(&info);
 	info.si_signo	= si_signo;
 	info.si_errno	= 0;
 	info.si_code	= si_code;
@@ -828,6 +829,8 @@ static inline void
 show_signal_msg(struct pt_regs *regs, unsigned long error_code,
 		unsigned long address, struct task_struct *tsk)
 {
+	const char *loglvl = task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG;
+
 	if (!unhandled_signal(tsk, SIGSEGV))
 		return;
 
@@ -835,13 +838,14 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code,
 		return;
 
 	printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx",
-		task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
-		tsk->comm, task_pid_nr(tsk), address,
+		loglvl, tsk->comm, task_pid_nr(tsk), address,
 		(void *)regs->ip, (void *)regs->sp, error_code);
 
 	print_vma_addr(KERN_CONT " in ", regs->ip);
 
 	printk(KERN_CONT "\n");
+
+	show_opcodes((u8 *)regs->ip, loglvl);
 }
 
 static void
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index c893c6a3d707..979e0a02cbe1 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -692,7 +692,7 @@ void __init initmem_init(void)
 	high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1;
 #endif
 
-	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);
+	memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0);
 	sparse_memory_present_with_active_regions(0);
 
 #ifdef CONFIG_FLATMEM
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 0a400606dea0..765a50fb6364 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -742,7 +742,7 @@ kernel_physical_mapping_init(unsigned long paddr_start,
 #ifndef CONFIG_NUMA
 void __init initmem_init(void)
 {
-	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);
+	memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0);
 }
 #endif
 
diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c
index 25504d5aa816..fa150855647c 100644
--- a/arch/x86/mm/numa.c
+++ b/arch/x86/mm/numa.c
@@ -136,13 +136,13 @@ static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
 
 	/* whine about and ignore invalid blks */
 	if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
-		pr_warning("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
-			   nid, start, end - 1);
+		pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
+			nid, start, end - 1);
 		return 0;
 	}
 
 	if (mi->nr_blks >= NR_NODE_MEMBLKS) {
-		pr_err("NUMA: too many memblk ranges\n");
+		pr_err("too many memblk ranges\n");
 		return -EINVAL;
 	}
 
@@ -267,14 +267,14 @@ int __init numa_cleanup_meminfo(struct numa_meminfo *mi)
 			 */
 			if (bi->end > bj->start && bi->start < bj->end) {
 				if (bi->nid != bj->nid) {
-					pr_err("NUMA: node %d [mem %#010Lx-%#010Lx] overlaps with node %d [mem %#010Lx-%#010Lx]\n",
+					pr_err("node %d [mem %#010Lx-%#010Lx] overlaps with node %d [mem %#010Lx-%#010Lx]\n",
 					       bi->nid, bi->start, bi->end - 1,
 					       bj->nid, bj->start, bj->end - 1);
 					return -EINVAL;
 				}
-				pr_warning("NUMA: Warning: node %d [mem %#010Lx-%#010Lx] overlaps with itself [mem %#010Lx-%#010Lx]\n",
-					   bi->nid, bi->start, bi->end - 1,
-					   bj->start, bj->end - 1);
+				pr_warn("Warning: node %d [mem %#010Lx-%#010Lx] overlaps with itself [mem %#010Lx-%#010Lx]\n",
+					bi->nid, bi->start, bi->end - 1,
+					bj->start, bj->end - 1);
 			}
 
 			/*
@@ -364,7 +364,7 @@ static int __init numa_alloc_distance(void)
 	phys = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped),
 				      size, PAGE_SIZE);
 	if (!phys) {
-		pr_warning("NUMA: Warning: can't allocate distance table!\n");
+		pr_warn("Warning: can't allocate distance table!\n");
 		/* don't retry until explicitly reset */
 		numa_distance = (void *)1LU;
 		return -ENOMEM;
@@ -410,14 +410,14 @@ void __init numa_set_distance(int from, int to, int distance)
 
 	if (from >= numa_distance_cnt || to >= numa_distance_cnt ||
 			from < 0 || to < 0) {
-		pr_warn_once("NUMA: Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
-			    from, to, distance);
+		pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
+			     from, to, distance);
 		return;
 	}
 
 	if ((u8)distance != distance ||
 	    (from == to && distance != LOCAL_DISTANCE)) {
-		pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
+		pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
 			     from, to, distance);
 		return;
 	}
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index ffc8c13c50e4..938dbcd46b97 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -114,13 +114,12 @@ static inline void pgd_list_del(pgd_t *pgd)
 
 static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
 {
-	BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
-	virt_to_page(pgd)->index = (pgoff_t)mm;
+	virt_to_page(pgd)->pt_mm = mm;
 }
 
 struct mm_struct *pgd_page_get_mm(struct page *page)
 {
-	return (struct mm_struct *)page->index;
+	return page->pt_mm;
 }
 
 static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)
diff --git a/arch/x86/net/Makefile b/arch/x86/net/Makefile
index fefb4b619598..59e123da580c 100644
--- a/arch/x86/net/Makefile
+++ b/arch/x86/net/Makefile
@@ -1,6 +1,9 @@
 #
 # Arch-specific network modules
 #
-OBJECT_FILES_NON_STANDARD_bpf_jit.o += y
 
-obj-$(CONFIG_BPF_JIT) += bpf_jit.o bpf_jit_comp.o
+ifeq ($(CONFIG_X86_32),y)
+        obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o
+else
+        obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o
+endif
diff --git a/arch/x86/net/bpf_jit.S b/arch/x86/net/bpf_jit.S
deleted file mode 100644
index b33093f84528..000000000000
--- a/arch/x86/net/bpf_jit.S
+++ /dev/null
@@ -1,154 +0,0 @@
-/* bpf_jit.S : BPF JIT helper functions
- *
- * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; version 2
- * of the License.
- */
-#include <linux/linkage.h>
-#include <asm/frame.h>
-
-/*
- * Calling convention :
- * rbx : skb pointer (callee saved)
- * esi : offset of byte(s) to fetch in skb (can be scratched)
- * r10 : copy of skb->data
- * r9d : hlen = skb->len - skb->data_len
- */
-#define SKBDATA	%r10
-#define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
-
-#define FUNC(name) \
-	.globl name; \
-	.type name, @function; \
-	name:
-
-FUNC(sk_load_word)
-	test	%esi,%esi
-	js	bpf_slow_path_word_neg
-
-FUNC(sk_load_word_positive_offset)
-	mov	%r9d,%eax		# hlen
-	sub	%esi,%eax		# hlen - offset
-	cmp	$3,%eax
-	jle	bpf_slow_path_word
-	mov     (SKBDATA,%rsi),%eax
-	bswap   %eax  			/* ntohl() */
-	ret
-
-FUNC(sk_load_half)
-	test	%esi,%esi
-	js	bpf_slow_path_half_neg
-
-FUNC(sk_load_half_positive_offset)
-	mov	%r9d,%eax
-	sub	%esi,%eax		#	hlen - offset
-	cmp	$1,%eax
-	jle	bpf_slow_path_half
-	movzwl	(SKBDATA,%rsi),%eax
-	rol	$8,%ax			# ntohs()
-	ret
-
-FUNC(sk_load_byte)
-	test	%esi,%esi
-	js	bpf_slow_path_byte_neg
-
-FUNC(sk_load_byte_positive_offset)
-	cmp	%esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
-	jle	bpf_slow_path_byte
-	movzbl	(SKBDATA,%rsi),%eax
-	ret
-
-/* rsi contains offset and can be scratched */
-#define bpf_slow_path_common(LEN)		\
-	lea	32(%rbp), %rdx;\
-	FRAME_BEGIN;				\
-	mov	%rbx, %rdi; /* arg1 == skb */	\
-	push	%r9;				\
-	push	SKBDATA;			\
-/* rsi already has offset */			\
-	mov	$LEN,%ecx;	/* len */	\
-	call	skb_copy_bits;			\
-	test    %eax,%eax;			\
-	pop	SKBDATA;			\
-	pop	%r9;				\
-	FRAME_END
-
-
-bpf_slow_path_word:
-	bpf_slow_path_common(4)
-	js	bpf_error
-	mov	32(%rbp),%eax
-	bswap	%eax
-	ret
-
-bpf_slow_path_half:
-	bpf_slow_path_common(2)
-	js	bpf_error
-	mov	32(%rbp),%ax
-	rol	$8,%ax
-	movzwl	%ax,%eax
-	ret
-
-bpf_slow_path_byte:
-	bpf_slow_path_common(1)
-	js	bpf_error
-	movzbl	32(%rbp),%eax
-	ret
-
-#define sk_negative_common(SIZE)				\
-	FRAME_BEGIN;						\
-	mov	%rbx, %rdi; /* arg1 == skb */			\
-	push	%r9;						\
-	push	SKBDATA;					\
-/* rsi already has offset */					\
-	mov	$SIZE,%edx;	/* size */			\
-	call	bpf_internal_load_pointer_neg_helper;		\
-	test	%rax,%rax;					\
-	pop	SKBDATA;					\
-	pop	%r9;						\
-	FRAME_END;						\
-	jz	bpf_error
-
-bpf_slow_path_word_neg:
-	cmp	SKF_MAX_NEG_OFF, %esi	/* test range */
-	jl	bpf_error	/* offset lower -> error  */
-
-FUNC(sk_load_word_negative_offset)
-	sk_negative_common(4)
-	mov	(%rax), %eax
-	bswap	%eax
-	ret
-
-bpf_slow_path_half_neg:
-	cmp	SKF_MAX_NEG_OFF, %esi
-	jl	bpf_error
-
-FUNC(sk_load_half_negative_offset)
-	sk_negative_common(2)
-	mov	(%rax),%ax
-	rol	$8,%ax
-	movzwl	%ax,%eax
-	ret
-
-bpf_slow_path_byte_neg:
-	cmp	SKF_MAX_NEG_OFF, %esi
-	jl	bpf_error
-
-FUNC(sk_load_byte_negative_offset)
-	sk_negative_common(1)
-	movzbl	(%rax), %eax
-	ret
-
-bpf_error:
-# force a return 0 from jit handler
-	xor	%eax,%eax
-	mov	(%rbp),%rbx
-	mov	8(%rbp),%r13
-	mov	16(%rbp),%r14
-	mov	24(%rbp),%r15
-	add	$40, %rbp
-	leaveq
-	ret
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 263c8453815e..8fca446aaef6 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1,4 +1,5 @@
-/* bpf_jit_comp.c : BPF JIT compiler
+/*
+ * bpf_jit_comp.c: BPF JIT compiler
  *
  * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com)
  * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
@@ -16,15 +17,6 @@
 #include <asm/set_memory.h>
 #include <asm/nospec-branch.h>
 
-/*
- * assembly code in arch/x86/net/bpf_jit.S
- */
-extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
-extern u8 sk_load_word_positive_offset[], sk_load_half_positive_offset[];
-extern u8 sk_load_byte_positive_offset[];
-extern u8 sk_load_word_negative_offset[], sk_load_half_negative_offset[];
-extern u8 sk_load_byte_negative_offset[];
-
 static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
 {
 	if (len == 1)
@@ -45,14 +37,15 @@ static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
 #define EMIT2(b1, b2)		EMIT((b1) + ((b2) << 8), 2)
 #define EMIT3(b1, b2, b3)	EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
 #define EMIT4(b1, b2, b3, b4)   EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)
+
 #define EMIT1_off32(b1, off) \
-	do {EMIT1(b1); EMIT(off, 4); } while (0)
+	do { EMIT1(b1); EMIT(off, 4); } while (0)
 #define EMIT2_off32(b1, b2, off) \
-	do {EMIT2(b1, b2); EMIT(off, 4); } while (0)
+	do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
 #define EMIT3_off32(b1, b2, b3, off) \
-	do {EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
+	do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
 #define EMIT4_off32(b1, b2, b3, b4, off) \
-	do {EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
+	do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
 
 static bool is_imm8(int value)
 {
@@ -70,9 +63,10 @@ static bool is_uimm32(u64 value)
 }
 
 /* mov dst, src */
-#define EMIT_mov(DST, SRC) \
-	do {if (DST != SRC) \
-		EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
+#define EMIT_mov(DST, SRC)								 \
+	do {										 \
+		if (DST != SRC)								 \
+			EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
 	} while (0)
 
 static int bpf_size_to_x86_bytes(int bpf_size)
@@ -89,7 +83,8 @@ static int bpf_size_to_x86_bytes(int bpf_size)
 		return 0;
 }
 
-/* list of x86 cond jumps opcodes (. + s8)
+/*
+ * List of x86 cond jumps opcodes (. + s8)
  * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
  */
 #define X86_JB  0x72
@@ -103,38 +98,37 @@ static int bpf_size_to_x86_bytes(int bpf_size)
 #define X86_JLE 0x7E
 #define X86_JG  0x7F
 
-#define CHOOSE_LOAD_FUNC(K, func) \
-	((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
-
-/* pick a register outside of BPF range for JIT internal work */
+/* Pick a register outside of BPF range for JIT internal work */
 #define AUX_REG (MAX_BPF_JIT_REG + 1)
 
-/* The following table maps BPF registers to x64 registers.
+/*
+ * The following table maps BPF registers to x86-64 registers.
  *
- * x64 register r12 is unused, since if used as base address
+ * x86-64 register R12 is unused, since if used as base address
  * register in load/store instructions, it always needs an
  * extra byte of encoding and is callee saved.
  *
- *  r9 caches skb->len - skb->data_len
- * r10 caches skb->data, and used for blinding (if enabled)
+ * Also x86-64 register R9 is unused. x86-64 register R10 is
+ * used for blinding (if enabled).
  */
 static const int reg2hex[] = {
-	[BPF_REG_0] = 0,  /* rax */
-	[BPF_REG_1] = 7,  /* rdi */
-	[BPF_REG_2] = 6,  /* rsi */
-	[BPF_REG_3] = 2,  /* rdx */
-	[BPF_REG_4] = 1,  /* rcx */
-	[BPF_REG_5] = 0,  /* r8 */
-	[BPF_REG_6] = 3,  /* rbx callee saved */
-	[BPF_REG_7] = 5,  /* r13 callee saved */
-	[BPF_REG_8] = 6,  /* r14 callee saved */
-	[BPF_REG_9] = 7,  /* r15 callee saved */
-	[BPF_REG_FP] = 5, /* rbp readonly */
-	[BPF_REG_AX] = 2, /* r10 temp register */
-	[AUX_REG] = 3,    /* r11 temp register */
+	[BPF_REG_0] = 0,  /* RAX */
+	[BPF_REG_1] = 7,  /* RDI */
+	[BPF_REG_2] = 6,  /* RSI */
+	[BPF_REG_3] = 2,  /* RDX */
+	[BPF_REG_4] = 1,  /* RCX */
+	[BPF_REG_5] = 0,  /* R8  */
+	[BPF_REG_6] = 3,  /* RBX callee saved */
+	[BPF_REG_7] = 5,  /* R13 callee saved */
+	[BPF_REG_8] = 6,  /* R14 callee saved */
+	[BPF_REG_9] = 7,  /* R15 callee saved */
+	[BPF_REG_FP] = 5, /* RBP readonly */
+	[BPF_REG_AX] = 2, /* R10 temp register */
+	[AUX_REG] = 3,    /* R11 temp register */
 };
 
-/* is_ereg() == true if BPF register 'reg' maps to x64 r8..r15
+/*
+ * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15
  * which need extra byte of encoding.
  * rax,rcx,...,rbp have simpler encoding
  */
@@ -153,7 +147,7 @@ static bool is_axreg(u32 reg)
 	return reg == BPF_REG_0;
 }
 
-/* add modifiers if 'reg' maps to x64 registers r8..r15 */
+/* Add modifiers if 'reg' maps to x86-64 registers R8..R15 */
 static u8 add_1mod(u8 byte, u32 reg)
 {
 	if (is_ereg(reg))
@@ -170,13 +164,13 @@ static u8 add_2mod(u8 byte, u32 r1, u32 r2)
 	return byte;
 }
 
-/* encode 'dst_reg' register into x64 opcode 'byte' */
+/* Encode 'dst_reg' register into x86-64 opcode 'byte' */
 static u8 add_1reg(u8 byte, u32 dst_reg)
 {
 	return byte + reg2hex[dst_reg];
 }
 
-/* encode 'dst_reg' and 'src_reg' registers into x64 opcode 'byte' */
+/* Encode 'dst_reg' and 'src_reg' registers into x86-64 opcode 'byte' */
 static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)
 {
 	return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3);
@@ -184,27 +178,24 @@ static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)
 
 static void jit_fill_hole(void *area, unsigned int size)
 {
-	/* fill whole space with int3 instructions */
+	/* Fill whole space with INT3 instructions */
 	memset(area, 0xcc, size);
 }
 
 struct jit_context {
-	int cleanup_addr; /* epilogue code offset */
-	bool seen_ld_abs;
-	bool seen_ax_reg;
+	int cleanup_addr; /* Epilogue code offset */
 };
 
-/* maximum number of bytes emitted while JITing one eBPF insn */
+/* Maximum number of bytes emitted while JITing one eBPF insn */
 #define BPF_MAX_INSN_SIZE	128
 #define BPF_INSN_SAFETY		64
 
-#define AUX_STACK_SPACE \
-	(32 /* space for rbx, r13, r14, r15 */ + \
-	 8 /* space for skb_copy_bits() buffer */)
+#define AUX_STACK_SPACE		40 /* Space for RBX, R13, R14, R15, tailcnt */
 
-#define PROLOGUE_SIZE 37
+#define PROLOGUE_SIZE		37
 
-/* emit x64 prologue code for BPF program and check it's size.
+/*
+ * Emit x86-64 prologue code for BPF program and check its size.
  * bpf_tail_call helper will skip it while jumping into another program
  */
 static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
@@ -212,8 +203,11 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
 	u8 *prog = *pprog;
 	int cnt = 0;
 
-	EMIT1(0x55); /* push rbp */
-	EMIT3(0x48, 0x89, 0xE5); /* mov rbp,rsp */
+	/* push rbp */
+	EMIT1(0x55);
+
+	/* mov rbp,rsp */
+	EMIT3(0x48, 0x89, 0xE5);
 
 	/* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */
 	EMIT3_off32(0x48, 0x81, 0xEC,
@@ -222,19 +216,8 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
 	/* sub rbp, AUX_STACK_SPACE */
 	EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE);
 
-	/* all classic BPF filters use R6(rbx) save it */
-
 	/* mov qword ptr [rbp+0],rbx */
 	EMIT4(0x48, 0x89, 0x5D, 0);
-
-	/* bpf_convert_filter() maps classic BPF register X to R7 and uses R8
-	 * as temporary, so all tcpdump filters need to spill/fill R7(r13) and
-	 * R8(r14). R9(r15) spill could be made conditional, but there is only
-	 * one 'bpf_error' return path out of helper functions inside bpf_jit.S
-	 * The overhead of extra spill is negligible for any filter other
-	 * than synthetic ones. Therefore not worth adding complexity.
-	 */
-
 	/* mov qword ptr [rbp+8],r13 */
 	EMIT4(0x4C, 0x89, 0x6D, 8);
 	/* mov qword ptr [rbp+16],r14 */
@@ -243,9 +226,10 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
 	EMIT4(0x4C, 0x89, 0x7D, 24);
 
 	if (!ebpf_from_cbpf) {
-		/* Clear the tail call counter (tail_call_cnt): for eBPF tail
+		/*
+		 * Clear the tail call counter (tail_call_cnt): for eBPF tail
 		 * calls we need to reset the counter to 0. It's done in two
-		 * instructions, resetting rax register to 0, and moving it
+		 * instructions, resetting RAX register to 0, and moving it
 		 * to the counter location.
 		 */
 
@@ -260,7 +244,9 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
 	*pprog = prog;
 }
 
-/* generate the following code:
+/*
+ * Generate the following code:
+ *
  * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ...
  *   if (index >= array->map.max_entries)
  *     goto out;
@@ -278,23 +264,26 @@ static void emit_bpf_tail_call(u8 **pprog)
 	int label1, label2, label3;
 	int cnt = 0;
 
-	/* rdi - pointer to ctx
+	/*
+	 * rdi - pointer to ctx
 	 * rsi - pointer to bpf_array
 	 * rdx - index in bpf_array
 	 */
 
-	/* if (index >= array->map.max_entries)
-	 *   goto out;
+	/*
+	 * if (index >= array->map.max_entries)
+	 *	goto out;
 	 */
 	EMIT2(0x89, 0xD2);                        /* mov edx, edx */
 	EMIT3(0x39, 0x56,                         /* cmp dword ptr [rsi + 16], edx */
 	      offsetof(struct bpf_array, map.max_entries));
-#define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* number of bytes to jump */
+#define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* Number of bytes to jump */
 	EMIT2(X86_JBE, OFFSET1);                  /* jbe out */
 	label1 = cnt;
 
-	/* if (tail_call_cnt > MAX_TAIL_CALL_CNT)
-	 *   goto out;
+	/*
+	 * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
+	 *	goto out;
 	 */
 	EMIT2_off32(0x8B, 0x85, 36);              /* mov eax, dword ptr [rbp + 36] */
 	EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT);     /* cmp eax, MAX_TAIL_CALL_CNT */
@@ -308,8 +297,9 @@ static void emit_bpf_tail_call(u8 **pprog)
 	EMIT4_off32(0x48, 0x8B, 0x84, 0xD6,       /* mov rax, [rsi + rdx * 8 + offsetof(...)] */
 		    offsetof(struct bpf_array, ptrs));
 
-	/* if (prog == NULL)
-	 *   goto out;
+	/*
+	 * if (prog == NULL)
+	 *	goto out;
 	 */
 	EMIT3(0x48, 0x85, 0xC0);		  /* test rax,rax */
 #define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE)
@@ -321,7 +311,8 @@ static void emit_bpf_tail_call(u8 **pprog)
 	      offsetof(struct bpf_prog, bpf_func));
 	EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE);   /* add rax, prologue_size */
 
-	/* now we're ready to jump into next BPF program
+	/*
+	 * Wow we're ready to jump into next BPF program
 	 * rdi == ctx (1st arg)
 	 * rax == prog->bpf_func + prologue_size
 	 */
@@ -334,26 +325,6 @@ static void emit_bpf_tail_call(u8 **pprog)
 	*pprog = prog;
 }
 
-
-static void emit_load_skb_data_hlen(u8 **pprog)
-{
-	u8 *prog = *pprog;
-	int cnt = 0;
-
-	/* r9d = skb->len - skb->data_len (headlen)
-	 * r10 = skb->data
-	 */
-	/* mov %r9d, off32(%rdi) */
-	EMIT3_off32(0x44, 0x8b, 0x8f, offsetof(struct sk_buff, len));
-
-	/* sub %r9d, off32(%rdi) */
-	EMIT3_off32(0x44, 0x2b, 0x8f, offsetof(struct sk_buff, data_len));
-
-	/* mov %r10, off32(%rdi) */
-	EMIT3_off32(0x4c, 0x8b, 0x97, offsetof(struct sk_buff, data));
-	*pprog = prog;
-}
-
 static void emit_mov_imm32(u8 **pprog, bool sign_propagate,
 			   u32 dst_reg, const u32 imm32)
 {
@@ -361,7 +332,8 @@ static void emit_mov_imm32(u8 **pprog, bool sign_propagate,
 	u8 b1, b2, b3;
 	int cnt = 0;
 
-	/* optimization: if imm32 is positive, use 'mov %eax, imm32'
+	/*
+	 * Optimization: if imm32 is positive, use 'mov %eax, imm32'
 	 * (which zero-extends imm32) to save 2 bytes.
 	 */
 	if (sign_propagate && (s32)imm32 < 0) {
@@ -373,7 +345,8 @@ static void emit_mov_imm32(u8 **pprog, bool sign_propagate,
 		goto done;
 	}
 
-	/* optimization: if imm32 is zero, use 'xor %eax, %eax'
+	/*
+	 * Optimization: if imm32 is zero, use 'xor %eax, %eax'
 	 * to save 3 bytes.
 	 */
 	if (imm32 == 0) {
@@ -400,7 +373,8 @@ static void emit_mov_imm64(u8 **pprog, u32 dst_reg,
 	int cnt = 0;
 
 	if (is_uimm32(((u64)imm32_hi << 32) | (u32)imm32_lo)) {
-		/* For emitting plain u32, where sign bit must not be
+		/*
+		 * For emitting plain u32, where sign bit must not be
 		 * propagated LLVM tends to load imm64 over mov32
 		 * directly, so save couple of bytes by just doing
 		 * 'mov %eax, imm32' instead.
@@ -439,8 +413,6 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 {
 	struct bpf_insn *insn = bpf_prog->insnsi;
 	int insn_cnt = bpf_prog->len;
-	bool seen_ld_abs = ctx->seen_ld_abs | (oldproglen == 0);
-	bool seen_ax_reg = ctx->seen_ax_reg | (oldproglen == 0);
 	bool seen_exit = false;
 	u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
 	int i, cnt = 0;
@@ -450,9 +422,6 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 	emit_prologue(&prog, bpf_prog->aux->stack_depth,
 		      bpf_prog_was_classic(bpf_prog));
 
-	if (seen_ld_abs)
-		emit_load_skb_data_hlen(&prog);
-
 	for (i = 0; i < insn_cnt; i++, insn++) {
 		const s32 imm32 = insn->imm;
 		u32 dst_reg = insn->dst_reg;
@@ -460,13 +429,9 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 		u8 b2 = 0, b3 = 0;
 		s64 jmp_offset;
 		u8 jmp_cond;
-		bool reload_skb_data;
 		int ilen;
 		u8 *func;
 
-		if (dst_reg == BPF_REG_AX || src_reg == BPF_REG_AX)
-			ctx->seen_ax_reg = seen_ax_reg = true;
-
 		switch (insn->code) {
 			/* ALU */
 		case BPF_ALU | BPF_ADD | BPF_X:
@@ -525,7 +490,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 			else if (is_ereg(dst_reg))
 				EMIT1(add_1mod(0x40, dst_reg));
 
-			/* b3 holds 'normal' opcode, b2 short form only valid
+			/*
+			 * b3 holds 'normal' opcode, b2 short form only valid
 			 * in case dst is eax/rax.
 			 */
 			switch (BPF_OP(insn->code)) {
@@ -593,7 +559,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 			/* mov rax, dst_reg */
 			EMIT_mov(BPF_REG_0, dst_reg);
 
-			/* xor edx, edx
+			/*
+			 * xor edx, edx
 			 * equivalent to 'xor rdx, rdx', but one byte less
 			 */
 			EMIT2(0x31, 0xd2);
@@ -655,7 +622,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 			}
 			break;
 		}
-			/* shifts */
+			/* Shifts */
 		case BPF_ALU | BPF_LSH | BPF_K:
 		case BPF_ALU | BPF_RSH | BPF_K:
 		case BPF_ALU | BPF_ARSH | BPF_K:
@@ -686,7 +653,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 		case BPF_ALU64 | BPF_RSH | BPF_X:
 		case BPF_ALU64 | BPF_ARSH | BPF_X:
 
-			/* check for bad case when dst_reg == rcx */
+			/* Check for bad case when dst_reg == rcx */
 			if (dst_reg == BPF_REG_4) {
 				/* mov r11, dst_reg */
 				EMIT_mov(AUX_REG, dst_reg);
@@ -724,13 +691,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 		case BPF_ALU | BPF_END | BPF_FROM_BE:
 			switch (imm32) {
 			case 16:
-				/* emit 'ror %ax, 8' to swap lower 2 bytes */
+				/* Emit 'ror %ax, 8' to swap lower 2 bytes */
 				EMIT1(0x66);
 				if (is_ereg(dst_reg))
 					EMIT1(0x41);
 				EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8);
 
-				/* emit 'movzwl eax, ax' */
+				/* Emit 'movzwl eax, ax' */
 				if (is_ereg(dst_reg))
 					EMIT3(0x45, 0x0F, 0xB7);
 				else
@@ -738,7 +705,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 				EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
 				break;
 			case 32:
-				/* emit 'bswap eax' to swap lower 4 bytes */
+				/* Emit 'bswap eax' to swap lower 4 bytes */
 				if (is_ereg(dst_reg))
 					EMIT2(0x41, 0x0F);
 				else
@@ -746,7 +713,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 				EMIT1(add_1reg(0xC8, dst_reg));
 				break;
 			case 64:
-				/* emit 'bswap rax' to swap 8 bytes */
+				/* Emit 'bswap rax' to swap 8 bytes */
 				EMIT3(add_1mod(0x48, dst_reg), 0x0F,
 				      add_1reg(0xC8, dst_reg));
 				break;
@@ -756,7 +723,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 		case BPF_ALU | BPF_END | BPF_FROM_LE:
 			switch (imm32) {
 			case 16:
-				/* emit 'movzwl eax, ax' to zero extend 16-bit
+				/*
+				 * Emit 'movzwl eax, ax' to zero extend 16-bit
 				 * into 64 bit
 				 */
 				if (is_ereg(dst_reg))
@@ -766,7 +734,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 				EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
 				break;
 			case 32:
-				/* emit 'mov eax, eax' to clear upper 32-bits */
+				/* Emit 'mov eax, eax' to clear upper 32-bits */
 				if (is_ereg(dst_reg))
 					EMIT1(0x45);
 				EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg));
@@ -809,9 +777,9 @@ st:			if (is_imm8(insn->off))
 
 			/* STX: *(u8*)(dst_reg + off) = src_reg */
 		case BPF_STX | BPF_MEM | BPF_B:
-			/* emit 'mov byte ptr [rax + off], al' */
+			/* Emit 'mov byte ptr [rax + off], al' */
 			if (is_ereg(dst_reg) || is_ereg(src_reg) ||
-			    /* have to add extra byte for x86 SIL, DIL regs */
+			    /* We have to add extra byte for x86 SIL, DIL regs */
 			    src_reg == BPF_REG_1 || src_reg == BPF_REG_2)
 				EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);
 			else
@@ -840,25 +808,26 @@ stx:			if (is_imm8(insn->off))
 
 			/* LDX: dst_reg = *(u8*)(src_reg + off) */
 		case BPF_LDX | BPF_MEM | BPF_B:
-			/* emit 'movzx rax, byte ptr [rax + off]' */
+			/* Emit 'movzx rax, byte ptr [rax + off]' */
 			EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);
 			goto ldx;
 		case BPF_LDX | BPF_MEM | BPF_H:
-			/* emit 'movzx rax, word ptr [rax + off]' */
+			/* Emit 'movzx rax, word ptr [rax + off]' */
 			EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);
 			goto ldx;
 		case BPF_LDX | BPF_MEM | BPF_W:
-			/* emit 'mov eax, dword ptr [rax+0x14]' */
+			/* Emit 'mov eax, dword ptr [rax+0x14]' */
 			if (is_ereg(dst_reg) || is_ereg(src_reg))
 				EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);
 			else
 				EMIT1(0x8B);
 			goto ldx;
 		case BPF_LDX | BPF_MEM | BPF_DW:
-			/* emit 'mov rax, qword ptr [rax+0x14]' */
+			/* Emit 'mov rax, qword ptr [rax+0x14]' */
 			EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);
-ldx:			/* if insn->off == 0 we can save one extra byte, but
-			 * special case of x86 r13 which always needs an offset
+ldx:			/*
+			 * If insn->off == 0 we can save one extra byte, but
+			 * special case of x86 R13 which always needs an offset
 			 * is not worth the hassle
 			 */
 			if (is_imm8(insn->off))
@@ -870,7 +839,7 @@ stx:			if (is_imm8(insn->off))
 
 			/* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */
 		case BPF_STX | BPF_XADD | BPF_W:
-			/* emit 'lock add dword ptr [rax + off], eax' */
+			/* Emit 'lock add dword ptr [rax + off], eax' */
 			if (is_ereg(dst_reg) || is_ereg(src_reg))
 				EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01);
 			else
@@ -889,35 +858,12 @@ xadd:			if (is_imm8(insn->off))
 		case BPF_JMP | BPF_CALL:
 			func = (u8 *) __bpf_call_base + imm32;
 			jmp_offset = func - (image + addrs[i]);
-			if (seen_ld_abs) {
-				reload_skb_data = bpf_helper_changes_pkt_data(func);
-				if (reload_skb_data) {
-					EMIT1(0x57); /* push %rdi */
-					jmp_offset += 22; /* pop, mov, sub, mov */
-				} else {
-					EMIT2(0x41, 0x52); /* push %r10 */
-					EMIT2(0x41, 0x51); /* push %r9 */
-					/* need to adjust jmp offset, since
-					 * pop %r9, pop %r10 take 4 bytes after call insn
-					 */
-					jmp_offset += 4;
-				}
-			}
 			if (!imm32 || !is_simm32(jmp_offset)) {
-				pr_err("unsupported bpf func %d addr %p image %p\n",
+				pr_err("unsupported BPF func %d addr %p image %p\n",
 				       imm32, func, image);
 				return -EINVAL;
 			}
 			EMIT1_off32(0xE8, jmp_offset);
-			if (seen_ld_abs) {
-				if (reload_skb_data) {
-					EMIT1(0x5F); /* pop %rdi */
-					emit_load_skb_data_hlen(&prog);
-				} else {
-					EMIT2(0x41, 0x59); /* pop %r9 */
-					EMIT2(0x41, 0x5A); /* pop %r10 */
-				}
-			}
 			break;
 
 		case BPF_JMP | BPF_TAIL_CALL:
@@ -970,7 +916,7 @@ xadd:			if (is_imm8(insn->off))
 			else
 				EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32);
 
-emit_cond_jmp:		/* convert BPF opcode to x86 */
+emit_cond_jmp:		/* Convert BPF opcode to x86 */
 			switch (BPF_OP(insn->code)) {
 			case BPF_JEQ:
 				jmp_cond = X86_JE;
@@ -996,22 +942,22 @@ xadd:			if (is_imm8(insn->off))
 				jmp_cond = X86_JBE;
 				break;
 			case BPF_JSGT:
-				/* signed '>', GT in x86 */
+				/* Signed '>', GT in x86 */
 				jmp_cond = X86_JG;
 				break;
 			case BPF_JSLT:
-				/* signed '<', LT in x86 */
+				/* Signed '<', LT in x86 */
 				jmp_cond = X86_JL;
 				break;
 			case BPF_JSGE:
-				/* signed '>=', GE in x86 */
+				/* Signed '>=', GE in x86 */
 				jmp_cond = X86_JGE;
 				break;
 			case BPF_JSLE:
-				/* signed '<=', LE in x86 */
+				/* Signed '<=', LE in x86 */
 				jmp_cond = X86_JLE;
 				break;
-			default: /* to silence gcc warning */
+			default: /* to silence GCC warning */
 				return -EFAULT;
 			}
 			jmp_offset = addrs[i + insn->off] - addrs[i];
@@ -1039,7 +985,7 @@ xadd:			if (is_imm8(insn->off))
 				jmp_offset = addrs[i + insn->off] - addrs[i];
 
 			if (!jmp_offset)
-				/* optimize out nop jumps */
+				/* Optimize out nop jumps */
 				break;
 emit_jmp:
 			if (is_imm8(jmp_offset)) {
@@ -1052,66 +998,13 @@ xadd:			if (is_imm8(insn->off))
 			}
 			break;
 
-		case BPF_LD | BPF_IND | BPF_W:
-			func = sk_load_word;
-			goto common_load;
-		case BPF_LD | BPF_ABS | BPF_W:
-			func = CHOOSE_LOAD_FUNC(imm32, sk_load_word);
-common_load:
-			ctx->seen_ld_abs = seen_ld_abs = true;
-			jmp_offset = func - (image + addrs[i]);
-			if (!func || !is_simm32(jmp_offset)) {
-				pr_err("unsupported bpf func %d addr %p image %p\n",
-				       imm32, func, image);
-				return -EINVAL;
-			}
-			if (BPF_MODE(insn->code) == BPF_ABS) {
-				/* mov %esi, imm32 */
-				EMIT1_off32(0xBE, imm32);
-			} else {
-				/* mov %rsi, src_reg */
-				EMIT_mov(BPF_REG_2, src_reg);
-				if (imm32) {
-					if (is_imm8(imm32))
-						/* add %esi, imm8 */
-						EMIT3(0x83, 0xC6, imm32);
-					else
-						/* add %esi, imm32 */
-						EMIT2_off32(0x81, 0xC6, imm32);
-				}
-			}
-			/* skb pointer is in R6 (%rbx), it will be copied into
-			 * %rdi if skb_copy_bits() call is necessary.
-			 * sk_load_* helpers also use %r10 and %r9d.
-			 * See bpf_jit.S
-			 */
-			if (seen_ax_reg)
-				/* r10 = skb->data, mov %r10, off32(%rbx) */
-				EMIT3_off32(0x4c, 0x8b, 0x93,
-					    offsetof(struct sk_buff, data));
-			EMIT1_off32(0xE8, jmp_offset); /* call */
-			break;
-
-		case BPF_LD | BPF_IND | BPF_H:
-			func = sk_load_half;
-			goto common_load;
-		case BPF_LD | BPF_ABS | BPF_H:
-			func = CHOOSE_LOAD_FUNC(imm32, sk_load_half);
-			goto common_load;
-		case BPF_LD | BPF_IND | BPF_B:
-			func = sk_load_byte;
-			goto common_load;
-		case BPF_LD | BPF_ABS | BPF_B:
-			func = CHOOSE_LOAD_FUNC(imm32, sk_load_byte);
-			goto common_load;
-
 		case BPF_JMP | BPF_EXIT:
 			if (seen_exit) {
 				jmp_offset = ctx->cleanup_addr - addrs[i];
 				goto emit_jmp;
 			}
 			seen_exit = true;
-			/* update cleanup_addr */
+			/* Update cleanup_addr */
 			ctx->cleanup_addr = proglen;
 			/* mov rbx, qword ptr [rbp+0] */
 			EMIT4(0x48, 0x8B, 0x5D, 0);
@@ -1129,10 +1022,11 @@ xadd:			if (is_imm8(insn->off))
 			break;
 
 		default:
-			/* By design x64 JIT should support all BPF instructions
+			/*
+			 * By design x86-64 JIT should support all BPF instructions.
 			 * This error will be seen if new instruction was added
-			 * to interpreter, but not to JIT
-			 * or if there is junk in bpf_prog
+			 * to the interpreter, but not to the JIT, or if there is
+			 * junk in bpf_prog.
 			 */
 			pr_err("bpf_jit: unknown opcode %02x\n", insn->code);
 			return -EINVAL;
@@ -1184,7 +1078,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		return orig_prog;
 
 	tmp = bpf_jit_blind_constants(prog);
-	/* If blinding was requested and we failed during blinding,
+	/*
+	 * If blinding was requested and we failed during blinding,
 	 * we must fall back to the interpreter.
 	 */
 	if (IS_ERR(tmp))
@@ -1218,8 +1113,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		goto out_addrs;
 	}
 
-	/* Before first pass, make a rough estimation of addrs[]
-	 * each bpf instruction is translated to less than 64 bytes
+	/*
+	 * Before first pass, make a rough estimation of addrs[]
+	 * each BPF instruction is translated to less than 64 bytes
 	 */
 	for (proglen = 0, i = 0; i < prog->len; i++) {
 		proglen += 64;
@@ -1228,10 +1124,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	ctx.cleanup_addr = proglen;
 skip_init_addrs:
 
-	/* JITed image shrinks with every pass and the loop iterates
-	 * until the image stops shrinking. Very large bpf programs
+	/*
+	 * JITed image shrinks with every pass and the loop iterates
+	 * until the image stops shrinking. Very large BPF programs
 	 * may converge on the last pass. In such case do one more
-	 * pass to emit the final image
+	 * pass to emit the final image.
 	 */
 	for (pass = 0; pass < 20 || image; pass++) {
 		proglen = do_jit(prog, addrs, image, oldproglen, &ctx);
diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
new file mode 100644
index 000000000000..0cc04e30adc1
--- /dev/null
+++ b/arch/x86/net/bpf_jit_comp32.c
@@ -0,0 +1,2419 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Just-In-Time compiler for eBPF filters on IA32 (32bit x86)
+ *
+ * Author: Wang YanQing (udknight@gmail.com)
+ * The code based on code and ideas from:
+ * Eric Dumazet (eric.dumazet@gmail.com)
+ * and from:
+ * Shubham Bansal <illusionist.neo@gmail.com>
+ */
+
+#include <linux/netdevice.h>
+#include <linux/filter.h>
+#include <linux/if_vlan.h>
+#include <asm/cacheflush.h>
+#include <asm/set_memory.h>
+#include <asm/nospec-branch.h>
+#include <linux/bpf.h>
+
+/*
+ * eBPF prog stack layout:
+ *
+ *                         high
+ * original ESP =>        +-----+
+ *                        |     | callee saved registers
+ *                        +-----+
+ *                        | ... | eBPF JIT scratch space
+ * BPF_FP,IA32_EBP  =>    +-----+
+ *                        | ... | eBPF prog stack
+ *                        +-----+
+ *                        |RSVD | JIT scratchpad
+ * current ESP =>         +-----+
+ *                        |     |
+ *                        | ... | Function call stack
+ *                        |     |
+ *                        +-----+
+ *                          low
+ *
+ * The callee saved registers:
+ *
+ *                                high
+ * original ESP =>        +------------------+ \
+ *                        |        ebp       | |
+ * current EBP =>         +------------------+ } callee saved registers
+ *                        |    ebx,esi,edi   | |
+ *                        +------------------+ /
+ *                                low
+ */
+
+static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
+{
+	if (len == 1)
+		*ptr = bytes;
+	else if (len == 2)
+		*(u16 *)ptr = bytes;
+	else {
+		*(u32 *)ptr = bytes;
+		barrier();
+	}
+	return ptr + len;
+}
+
+#define EMIT(bytes, len) \
+	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
+
+#define EMIT1(b1)		EMIT(b1, 1)
+#define EMIT2(b1, b2)		EMIT((b1) + ((b2) << 8), 2)
+#define EMIT3(b1, b2, b3)	EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
+#define EMIT4(b1, b2, b3, b4)   \
+	EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)
+
+#define EMIT1_off32(b1, off) \
+	do { EMIT1(b1); EMIT(off, 4); } while (0)
+#define EMIT2_off32(b1, b2, off) \
+	do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
+#define EMIT3_off32(b1, b2, b3, off) \
+	do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
+#define EMIT4_off32(b1, b2, b3, b4, off) \
+	do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
+
+#define jmp_label(label, jmp_insn_len) (label - cnt - jmp_insn_len)
+
+static bool is_imm8(int value)
+{
+	return value <= 127 && value >= -128;
+}
+
+static bool is_simm32(s64 value)
+{
+	return value == (s64) (s32) value;
+}
+
+#define STACK_OFFSET(k)	(k)
+#define TCALL_CNT	(MAX_BPF_JIT_REG + 0)	/* Tail Call Count */
+
+#define IA32_EAX	(0x0)
+#define IA32_EBX	(0x3)
+#define IA32_ECX	(0x1)
+#define IA32_EDX	(0x2)
+#define IA32_ESI	(0x6)
+#define IA32_EDI	(0x7)
+#define IA32_EBP	(0x5)
+#define IA32_ESP	(0x4)
+
+/*
+ * List of x86 cond jumps opcodes (. + s8)
+ * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
+ */
+#define IA32_JB  0x72
+#define IA32_JAE 0x73
+#define IA32_JE  0x74
+#define IA32_JNE 0x75
+#define IA32_JBE 0x76
+#define IA32_JA  0x77
+#define IA32_JL  0x7C
+#define IA32_JGE 0x7D
+#define IA32_JLE 0x7E
+#define IA32_JG  0x7F
+
+/*
+ * Map eBPF registers to IA32 32bit registers or stack scratch space.
+ *
+ * 1. All the registers, R0-R10, are mapped to scratch space on stack.
+ * 2. We need two 64 bit temp registers to do complex operations on eBPF
+ *    registers.
+ * 3. For performance reason, the BPF_REG_AX for blinding constant, is
+ *    mapped to real hardware register pair, IA32_ESI and IA32_EDI.
+ *
+ * As the eBPF registers are all 64 bit registers and IA32 has only 32 bit
+ * registers, we have to map each eBPF registers with two IA32 32 bit regs
+ * or scratch memory space and we have to build eBPF 64 bit register from those.
+ *
+ * We use IA32_EAX, IA32_EDX, IA32_ECX, IA32_EBX as temporary registers.
+ */
+static const u8 bpf2ia32[][2] = {
+	/* Return value from in-kernel function, and exit value from eBPF */
+	[BPF_REG_0] = {STACK_OFFSET(0), STACK_OFFSET(4)},
+
+	/* The arguments from eBPF program to in-kernel function */
+	/* Stored on stack scratch space */
+	[BPF_REG_1] = {STACK_OFFSET(8), STACK_OFFSET(12)},
+	[BPF_REG_2] = {STACK_OFFSET(16), STACK_OFFSET(20)},
+	[BPF_REG_3] = {STACK_OFFSET(24), STACK_OFFSET(28)},
+	[BPF_REG_4] = {STACK_OFFSET(32), STACK_OFFSET(36)},
+	[BPF_REG_5] = {STACK_OFFSET(40), STACK_OFFSET(44)},
+
+	/* Callee saved registers that in-kernel function will preserve */
+	/* Stored on stack scratch space */
+	[BPF_REG_6] = {STACK_OFFSET(48), STACK_OFFSET(52)},
+	[BPF_REG_7] = {STACK_OFFSET(56), STACK_OFFSET(60)},
+	[BPF_REG_8] = {STACK_OFFSET(64), STACK_OFFSET(68)},
+	[BPF_REG_9] = {STACK_OFFSET(72), STACK_OFFSET(76)},
+
+	/* Read only Frame Pointer to access Stack */
+	[BPF_REG_FP] = {STACK_OFFSET(80), STACK_OFFSET(84)},
+
+	/* Temporary register for blinding constants. */
+	[BPF_REG_AX] = {IA32_ESI, IA32_EDI},
+
+	/* Tail call count. Stored on stack scratch space. */
+	[TCALL_CNT] = {STACK_OFFSET(88), STACK_OFFSET(92)},
+};
+
+#define dst_lo	dst[0]
+#define dst_hi	dst[1]
+#define src_lo	src[0]
+#define src_hi	src[1]
+
+#define STACK_ALIGNMENT	8
+/*
+ * Stack space for BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4,
+ * BPF_REG_5, BPF_REG_6, BPF_REG_7, BPF_REG_8, BPF_REG_9,
+ * BPF_REG_FP, BPF_REG_AX and Tail call counts.
+ */
+#define SCRATCH_SIZE 96
+
+/* Total stack size used in JITed code */
+#define _STACK_SIZE	(stack_depth + SCRATCH_SIZE)
+
+#define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT)
+
+/* Get the offset of eBPF REGISTERs stored on scratch space. */
+#define STACK_VAR(off) (off)
+
+/* Encode 'dst_reg' register into IA32 opcode 'byte' */
+static u8 add_1reg(u8 byte, u32 dst_reg)
+{
+	return byte + dst_reg;
+}
+
+/* Encode 'dst_reg' and 'src_reg' registers into IA32 opcode 'byte' */
+static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)
+{
+	return byte + dst_reg + (src_reg << 3);
+}
+
+static void jit_fill_hole(void *area, unsigned int size)
+{
+	/* Fill whole space with int3 instructions */
+	memset(area, 0xcc, size);
+}
+
+static inline void emit_ia32_mov_i(const u8 dst, const u32 val, bool dstk,
+				   u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+
+	if (dstk) {
+		if (val == 0) {
+			/* xor eax,eax */
+			EMIT2(0x33, add_2reg(0xC0, IA32_EAX, IA32_EAX));
+			/* mov dword ptr [ebp+off],eax */
+			EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
+			      STACK_VAR(dst));
+		} else {
+			EMIT3_off32(0xC7, add_1reg(0x40, IA32_EBP),
+				    STACK_VAR(dst), val);
+		}
+	} else {
+		if (val == 0)
+			EMIT2(0x33, add_2reg(0xC0, dst, dst));
+		else
+			EMIT2_off32(0xC7, add_1reg(0xC0, dst),
+				    val);
+	}
+	*pprog = prog;
+}
+
+/* dst = imm (4 bytes)*/
+static inline void emit_ia32_mov_r(const u8 dst, const u8 src, bool dstk,
+				   bool sstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u8 sreg = sstk ? IA32_EAX : src;
+
+	if (sstk)
+		/* mov eax,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src));
+	if (dstk)
+		/* mov dword ptr [ebp+off],eax */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, sreg), STACK_VAR(dst));
+	else
+		/* mov dst,sreg */
+		EMIT2(0x89, add_2reg(0xC0, dst, sreg));
+
+	*pprog = prog;
+}
+
+/* dst = src */
+static inline void emit_ia32_mov_r64(const bool is64, const u8 dst[],
+				     const u8 src[], bool dstk,
+				     bool sstk, u8 **pprog)
+{
+	emit_ia32_mov_r(dst_lo, src_lo, dstk, sstk, pprog);
+	if (is64)
+		/* complete 8 byte move */
+		emit_ia32_mov_r(dst_hi, src_hi, dstk, sstk, pprog);
+	else
+		/* zero out high 4 bytes */
+		emit_ia32_mov_i(dst_hi, 0, dstk, pprog);
+}
+
+/* Sign extended move */
+static inline void emit_ia32_mov_i64(const bool is64, const u8 dst[],
+				     const u32 val, bool dstk, u8 **pprog)
+{
+	u32 hi = 0;
+
+	if (is64 && (val & (1<<31)))
+		hi = (u32)~0;
+	emit_ia32_mov_i(dst_lo, val, dstk, pprog);
+	emit_ia32_mov_i(dst_hi, hi, dstk, pprog);
+}
+
+/*
+ * ALU operation (32 bit)
+ * dst = dst * src
+ */
+static inline void emit_ia32_mul_r(const u8 dst, const u8 src, bool dstk,
+				   bool sstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u8 sreg = sstk ? IA32_ECX : src;
+
+	if (sstk)
+		/* mov ecx,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src));
+
+	if (dstk)
+		/* mov eax,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst));
+	else
+		/* mov eax,dst */
+		EMIT2(0x8B, add_2reg(0xC0, dst, IA32_EAX));
+
+
+	EMIT2(0xF7, add_1reg(0xE0, sreg));
+
+	if (dstk)
+		/* mov dword ptr [ebp+off],eax */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst));
+	else
+		/* mov dst,eax */
+		EMIT2(0x89, add_2reg(0xC0, dst, IA32_EAX));
+
+	*pprog = prog;
+}
+
+static inline void emit_ia32_to_le_r64(const u8 dst[], s32 val,
+					 bool dstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+
+	if (dstk && val != 64) {
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+		      STACK_VAR(dst_hi));
+	}
+	switch (val) {
+	case 16:
+		/*
+		 * Emit 'movzwl eax,ax' to zero extend 16-bit
+		 * into 64 bit
+		 */
+		EMIT2(0x0F, 0xB7);
+		EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo));
+		/* xor dreg_hi,dreg_hi */
+		EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
+		break;
+	case 32:
+		/* xor dreg_hi,dreg_hi */
+		EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
+		break;
+	case 64:
+		/* nop */
+		break;
+	}
+
+	if (dstk && val != 64) {
+		/* mov dword ptr [ebp+off],dreg_lo */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
+		      STACK_VAR(dst_lo));
+		/* mov dword ptr [ebp+off],dreg_hi */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
+		      STACK_VAR(dst_hi));
+	}
+	*pprog = prog;
+}
+
+static inline void emit_ia32_to_be_r64(const u8 dst[], s32 val,
+				       bool dstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+
+	if (dstk) {
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+		      STACK_VAR(dst_hi));
+	}
+	switch (val) {
+	case 16:
+		/* Emit 'ror %ax, 8' to swap lower 2 bytes */
+		EMIT1(0x66);
+		EMIT3(0xC1, add_1reg(0xC8, dreg_lo), 8);
+
+		EMIT2(0x0F, 0xB7);
+		EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo));
+
+		/* xor dreg_hi,dreg_hi */
+		EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
+		break;
+	case 32:
+		/* Emit 'bswap eax' to swap lower 4 bytes */
+		EMIT1(0x0F);
+		EMIT1(add_1reg(0xC8, dreg_lo));
+
+		/* xor dreg_hi,dreg_hi */
+		EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
+		break;
+	case 64:
+		/* Emit 'bswap eax' to swap lower 4 bytes */
+		EMIT1(0x0F);
+		EMIT1(add_1reg(0xC8, dreg_lo));
+
+		/* Emit 'bswap edx' to swap lower 4 bytes */
+		EMIT1(0x0F);
+		EMIT1(add_1reg(0xC8, dreg_hi));
+
+		/* mov ecx,dreg_hi */
+		EMIT2(0x89, add_2reg(0xC0, IA32_ECX, dreg_hi));
+		/* mov dreg_hi,dreg_lo */
+		EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo));
+		/* mov dreg_lo,ecx */
+		EMIT2(0x89, add_2reg(0xC0, dreg_lo, IA32_ECX));
+
+		break;
+	}
+	if (dstk) {
+		/* mov dword ptr [ebp+off],dreg_lo */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
+		      STACK_VAR(dst_lo));
+		/* mov dword ptr [ebp+off],dreg_hi */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
+		      STACK_VAR(dst_hi));
+	}
+	*pprog = prog;
+}
+
+/*
+ * ALU operation (32 bit)
+ * dst = dst (div|mod) src
+ */
+static inline void emit_ia32_div_mod_r(const u8 op, const u8 dst, const u8 src,
+				       bool dstk, bool sstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+
+	if (sstk)
+		/* mov ecx,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
+		      STACK_VAR(src));
+	else if (src != IA32_ECX)
+		/* mov ecx,src */
+		EMIT2(0x8B, add_2reg(0xC0, src, IA32_ECX));
+
+	if (dstk)
+		/* mov eax,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst));
+	else
+		/* mov eax,dst */
+		EMIT2(0x8B, add_2reg(0xC0, dst, IA32_EAX));
+
+	/* xor edx,edx */
+	EMIT2(0x31, add_2reg(0xC0, IA32_EDX, IA32_EDX));
+	/* div ecx */
+	EMIT2(0xF7, add_1reg(0xF0, IA32_ECX));
+
+	if (op == BPF_MOD) {
+		if (dstk)
+			EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),
+			      STACK_VAR(dst));
+		else
+			EMIT2(0x89, add_2reg(0xC0, dst, IA32_EDX));
+	} else {
+		if (dstk)
+			EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
+			      STACK_VAR(dst));
+		else
+			EMIT2(0x89, add_2reg(0xC0, dst, IA32_EAX));
+	}
+	*pprog = prog;
+}
+
+/*
+ * ALU operation (32 bit)
+ * dst = dst (shift) src
+ */
+static inline void emit_ia32_shift_r(const u8 op, const u8 dst, const u8 src,
+				     bool dstk, bool sstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u8 dreg = dstk ? IA32_EAX : dst;
+	u8 b2;
+
+	if (dstk)
+		/* mov eax,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst));
+
+	if (sstk)
+		/* mov ecx,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src));
+	else if (src != IA32_ECX)
+		/* mov ecx,src */
+		EMIT2(0x8B, add_2reg(0xC0, src, IA32_ECX));
+
+	switch (op) {
+	case BPF_LSH:
+		b2 = 0xE0; break;
+	case BPF_RSH:
+		b2 = 0xE8; break;
+	case BPF_ARSH:
+		b2 = 0xF8; break;
+	default:
+		return;
+	}
+	EMIT2(0xD3, add_1reg(b2, dreg));
+
+	if (dstk)
+		/* mov dword ptr [ebp+off],dreg */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), STACK_VAR(dst));
+	*pprog = prog;
+}
+
+/*
+ * ALU operation (32 bit)
+ * dst = dst (op) src
+ */
+static inline void emit_ia32_alu_r(const bool is64, const bool hi, const u8 op,
+				   const u8 dst, const u8 src, bool dstk,
+				   bool sstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u8 sreg = sstk ? IA32_EAX : src;
+	u8 dreg = dstk ? IA32_EDX : dst;
+
+	if (sstk)
+		/* mov eax,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src));
+
+	if (dstk)
+		/* mov eax,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(dst));
+
+	switch (BPF_OP(op)) {
+	/* dst = dst + src */
+	case BPF_ADD:
+		if (hi && is64)
+			EMIT2(0x11, add_2reg(0xC0, dreg, sreg));
+		else
+			EMIT2(0x01, add_2reg(0xC0, dreg, sreg));
+		break;
+	/* dst = dst - src */
+	case BPF_SUB:
+		if (hi && is64)
+			EMIT2(0x19, add_2reg(0xC0, dreg, sreg));
+		else
+			EMIT2(0x29, add_2reg(0xC0, dreg, sreg));
+		break;
+	/* dst = dst | src */
+	case BPF_OR:
+		EMIT2(0x09, add_2reg(0xC0, dreg, sreg));
+		break;
+	/* dst = dst & src */
+	case BPF_AND:
+		EMIT2(0x21, add_2reg(0xC0, dreg, sreg));
+		break;
+	/* dst = dst ^ src */
+	case BPF_XOR:
+		EMIT2(0x31, add_2reg(0xC0, dreg, sreg));
+		break;
+	}
+
+	if (dstk)
+		/* mov dword ptr [ebp+off],dreg */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg),
+		      STACK_VAR(dst));
+	*pprog = prog;
+}
+
+/* ALU operation (64 bit) */
+static inline void emit_ia32_alu_r64(const bool is64, const u8 op,
+				     const u8 dst[], const u8 src[],
+				     bool dstk,  bool sstk,
+				     u8 **pprog)
+{
+	u8 *prog = *pprog;
+
+	emit_ia32_alu_r(is64, false, op, dst_lo, src_lo, dstk, sstk, &prog);
+	if (is64)
+		emit_ia32_alu_r(is64, true, op, dst_hi, src_hi, dstk, sstk,
+				&prog);
+	else
+		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
+	*pprog = prog;
+}
+
+/*
+ * ALU operation (32 bit)
+ * dst = dst (op) val
+ */
+static inline void emit_ia32_alu_i(const bool is64, const bool hi, const u8 op,
+				   const u8 dst, const s32 val, bool dstk,
+				   u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u8 dreg = dstk ? IA32_EAX : dst;
+	u8 sreg = IA32_EDX;
+
+	if (dstk)
+		/* mov eax,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst));
+
+	if (!is_imm8(val))
+		/* mov edx,imm32*/
+		EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EDX), val);
+
+	switch (op) {
+	/* dst = dst + val */
+	case BPF_ADD:
+		if (hi && is64) {
+			if (is_imm8(val))
+				EMIT3(0x83, add_1reg(0xD0, dreg), val);
+			else
+				EMIT2(0x11, add_2reg(0xC0, dreg, sreg));
+		} else {
+			if (is_imm8(val))
+				EMIT3(0x83, add_1reg(0xC0, dreg), val);
+			else
+				EMIT2(0x01, add_2reg(0xC0, dreg, sreg));
+		}
+		break;
+	/* dst = dst - val */
+	case BPF_SUB:
+		if (hi && is64) {
+			if (is_imm8(val))
+				EMIT3(0x83, add_1reg(0xD8, dreg), val);
+			else
+				EMIT2(0x19, add_2reg(0xC0, dreg, sreg));
+		} else {
+			if (is_imm8(val))
+				EMIT3(0x83, add_1reg(0xE8, dreg), val);
+			else
+				EMIT2(0x29, add_2reg(0xC0, dreg, sreg));
+		}
+		break;
+	/* dst = dst | val */
+	case BPF_OR:
+		if (is_imm8(val))
+			EMIT3(0x83, add_1reg(0xC8, dreg), val);
+		else
+			EMIT2(0x09, add_2reg(0xC0, dreg, sreg));
+		break;
+	/* dst = dst & val */
+	case BPF_AND:
+		if (is_imm8(val))
+			EMIT3(0x83, add_1reg(0xE0, dreg), val);
+		else
+			EMIT2(0x21, add_2reg(0xC0, dreg, sreg));
+		break;
+	/* dst = dst ^ val */
+	case BPF_XOR:
+		if (is_imm8(val))
+			EMIT3(0x83, add_1reg(0xF0, dreg), val);
+		else
+			EMIT2(0x31, add_2reg(0xC0, dreg, sreg));
+		break;
+	case BPF_NEG:
+		EMIT2(0xF7, add_1reg(0xD8, dreg));
+		break;
+	}
+
+	if (dstk)
+		/* mov dword ptr [ebp+off],dreg */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg),
+		      STACK_VAR(dst));
+	*pprog = prog;
+}
+
+/* ALU operation (64 bit) */
+static inline void emit_ia32_alu_i64(const bool is64, const u8 op,
+				     const u8 dst[], const u32 val,
+				     bool dstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	u32 hi = 0;
+
+	if (is64 && (val & (1<<31)))
+		hi = (u32)~0;
+
+	emit_ia32_alu_i(is64, false, op, dst_lo, val, dstk, &prog);
+	if (is64)
+		emit_ia32_alu_i(is64, true, op, dst_hi, hi, dstk, &prog);
+	else
+		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
+
+	*pprog = prog;
+}
+
+/* dst = ~dst (64 bit) */
+static inline void emit_ia32_neg64(const u8 dst[], bool dstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+
+	if (dstk) {
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+		      STACK_VAR(dst_hi));
+	}
+
+	/* xor ecx,ecx */
+	EMIT2(0x31, add_2reg(0xC0, IA32_ECX, IA32_ECX));
+	/* sub dreg_lo,ecx */
+	EMIT2(0x2B, add_2reg(0xC0, dreg_lo, IA32_ECX));
+	/* mov dreg_lo,ecx */
+	EMIT2(0x89, add_2reg(0xC0, dreg_lo, IA32_ECX));
+
+	/* xor ecx,ecx */
+	EMIT2(0x31, add_2reg(0xC0, IA32_ECX, IA32_ECX));
+	/* sbb dreg_hi,ecx */
+	EMIT2(0x19, add_2reg(0xC0, dreg_hi, IA32_ECX));
+	/* mov dreg_hi,ecx */
+	EMIT2(0x89, add_2reg(0xC0, dreg_hi, IA32_ECX));
+
+	if (dstk) {
+		/* mov dword ptr [ebp+off],dreg_lo */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
+		      STACK_VAR(dst_lo));
+		/* mov dword ptr [ebp+off],dreg_hi */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
+		      STACK_VAR(dst_hi));
+	}
+	*pprog = prog;
+}
+
+/* dst = dst << src */
+static inline void emit_ia32_lsh_r64(const u8 dst[], const u8 src[],
+				     bool dstk, bool sstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	static int jmp_label1 = -1;
+	static int jmp_label2 = -1;
+	static int jmp_label3 = -1;
+	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+
+	if (dstk) {
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+		      STACK_VAR(dst_hi));
+	}
+
+	if (sstk)
+		/* mov ecx,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
+		      STACK_VAR(src_lo));
+	else
+		/* mov ecx,src_lo */
+		EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX));
+
+	/* cmp ecx,32 */
+	EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32);
+	/* Jumps when >= 32 */
+	if (is_imm8(jmp_label(jmp_label1, 2)))
+		EMIT2(IA32_JAE, jmp_label(jmp_label1, 2));
+	else
+		EMIT2_off32(0x0F, IA32_JAE + 0x10, jmp_label(jmp_label1, 6));
+
+	/* < 32 */
+	/* shl dreg_hi,cl */
+	EMIT2(0xD3, add_1reg(0xE0, dreg_hi));
+	/* mov ebx,dreg_lo */
+	EMIT2(0x8B, add_2reg(0xC0, dreg_lo, IA32_EBX));
+	/* shl dreg_lo,cl */
+	EMIT2(0xD3, add_1reg(0xE0, dreg_lo));
+
+	/* IA32_ECX = -IA32_ECX + 32 */
+	/* neg ecx */
+	EMIT2(0xF7, add_1reg(0xD8, IA32_ECX));
+	/* add ecx,32 */
+	EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 32);
+
+	/* shr ebx,cl */
+	EMIT2(0xD3, add_1reg(0xE8, IA32_EBX));
+	/* or dreg_hi,ebx */
+	EMIT2(0x09, add_2reg(0xC0, dreg_hi, IA32_EBX));
+
+	/* goto out; */
+	if (is_imm8(jmp_label(jmp_label3, 2)))
+		EMIT2(0xEB, jmp_label(jmp_label3, 2));
+	else
+		EMIT1_off32(0xE9, jmp_label(jmp_label3, 5));
+
+	/* >= 32 */
+	if (jmp_label1 == -1)
+		jmp_label1 = cnt;
+
+	/* cmp ecx,64 */
+	EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 64);
+	/* Jumps when >= 64 */
+	if (is_imm8(jmp_label(jmp_label2, 2)))
+		EMIT2(IA32_JAE, jmp_label(jmp_label2, 2));
+	else
+		EMIT2_off32(0x0F, IA32_JAE + 0x10, jmp_label(jmp_label2, 6));
+
+	/* >= 32 && < 64 */
+	/* sub ecx,32 */
+	EMIT3(0x83, add_1reg(0xE8, IA32_ECX), 32);
+	/* shl dreg_lo,cl */
+	EMIT2(0xD3, add_1reg(0xE0, dreg_lo));
+	/* mov dreg_hi,dreg_lo */
+	EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo));
+
+	/* xor dreg_lo,dreg_lo */
+	EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo));
+
+	/* goto out; */
+	if (is_imm8(jmp_label(jmp_label3, 2)))
+		EMIT2(0xEB, jmp_label(jmp_label3, 2));
+	else
+		EMIT1_off32(0xE9, jmp_label(jmp_label3, 5));
+
+	/* >= 64 */
+	if (jmp_label2 == -1)
+		jmp_label2 = cnt;
+	/* xor dreg_lo,dreg_lo */
+	EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo));
+	/* xor dreg_hi,dreg_hi */
+	EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
+
+	if (jmp_label3 == -1)
+		jmp_label3 = cnt;
+
+	if (dstk) {
+		/* mov dword ptr [ebp+off],dreg_lo */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
+		      STACK_VAR(dst_lo));
+		/* mov dword ptr [ebp+off],dreg_hi */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
+		      STACK_VAR(dst_hi));
+	}
+	/* out: */
+	*pprog = prog;
+}
+
+/* dst = dst >> src (signed)*/
+static inline void emit_ia32_arsh_r64(const u8 dst[], const u8 src[],
+				      bool dstk, bool sstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	static int jmp_label1 = -1;
+	static int jmp_label2 = -1;
+	static int jmp_label3 = -1;
+	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+
+	if (dstk) {
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+		      STACK_VAR(dst_hi));
+	}
+
+	if (sstk)
+		/* mov ecx,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
+		      STACK_VAR(src_lo));
+	else
+		/* mov ecx,src_lo */
+		EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX));
+
+	/* cmp ecx,32 */
+	EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32);
+	/* Jumps when >= 32 */
+	if (is_imm8(jmp_label(jmp_label1, 2)))
+		EMIT2(IA32_JAE, jmp_label(jmp_label1, 2));
+	else
+		EMIT2_off32(0x0F, IA32_JAE + 0x10, jmp_label(jmp_label1, 6));
+
+	/* < 32 */
+	/* lshr dreg_lo,cl */
+	EMIT2(0xD3, add_1reg(0xE8, dreg_lo));
+	/* mov ebx,dreg_hi */
+	EMIT2(0x8B, add_2reg(0xC0, dreg_hi, IA32_EBX));
+	/* ashr dreg_hi,cl */
+	EMIT2(0xD3, add_1reg(0xF8, dreg_hi));
+
+	/* IA32_ECX = -IA32_ECX + 32 */
+	/* neg ecx */
+	EMIT2(0xF7, add_1reg(0xD8, IA32_ECX));
+	/* add ecx,32 */
+	EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 32);
+
+	/* shl ebx,cl */
+	EMIT2(0xD3, add_1reg(0xE0, IA32_EBX));
+	/* or dreg_lo,ebx */
+	EMIT2(0x09, add_2reg(0xC0, dreg_lo, IA32_EBX));
+
+	/* goto out; */
+	if (is_imm8(jmp_label(jmp_label3, 2)))
+		EMIT2(0xEB, jmp_label(jmp_label3, 2));
+	else
+		EMIT1_off32(0xE9, jmp_label(jmp_label3, 5));
+
+	/* >= 32 */
+	if (jmp_label1 == -1)
+		jmp_label1 = cnt;
+
+	/* cmp ecx,64 */
+	EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 64);
+	/* Jumps when >= 64 */
+	if (is_imm8(jmp_label(jmp_label2, 2)))
+		EMIT2(IA32_JAE, jmp_label(jmp_label2, 2));
+	else
+		EMIT2_off32(0x0F, IA32_JAE + 0x10, jmp_label(jmp_label2, 6));
+
+	/* >= 32 && < 64 */
+	/* sub ecx,32 */
+	EMIT3(0x83, add_1reg(0xE8, IA32_ECX), 32);
+	/* ashr dreg_hi,cl */
+	EMIT2(0xD3, add_1reg(0xF8, dreg_hi));
+	/* mov dreg_lo,dreg_hi */
+	EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi));
+
+	/* ashr dreg_hi,imm8 */
+	EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31);
+
+	/* goto out; */
+	if (is_imm8(jmp_label(jmp_label3, 2)))
+		EMIT2(0xEB, jmp_label(jmp_label3, 2));
+	else
+		EMIT1_off32(0xE9, jmp_label(jmp_label3, 5));
+
+	/* >= 64 */
+	if (jmp_label2 == -1)
+		jmp_label2 = cnt;
+	/* ashr dreg_hi,imm8 */
+	EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31);
+	/* mov dreg_lo,dreg_hi */
+	EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi));
+
+	if (jmp_label3 == -1)
+		jmp_label3 = cnt;
+
+	if (dstk) {
+		/* mov dword ptr [ebp+off],dreg_lo */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
+		      STACK_VAR(dst_lo));
+		/* mov dword ptr [ebp+off],dreg_hi */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
+		      STACK_VAR(dst_hi));
+	}
+	/* out: */
+	*pprog = prog;
+}
+
+/* dst = dst >> src */
+static inline void emit_ia32_rsh_r64(const u8 dst[], const u8 src[], bool dstk,
+				     bool sstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	static int jmp_label1 = -1;
+	static int jmp_label2 = -1;
+	static int jmp_label3 = -1;
+	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+
+	if (dstk) {
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+		      STACK_VAR(dst_hi));
+	}
+
+	if (sstk)
+		/* mov ecx,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
+		      STACK_VAR(src_lo));
+	else
+		/* mov ecx,src_lo */
+		EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX));
+
+	/* cmp ecx,32 */
+	EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32);
+	/* Jumps when >= 32 */
+	if (is_imm8(jmp_label(jmp_label1, 2)))
+		EMIT2(IA32_JAE, jmp_label(jmp_label1, 2));
+	else
+		EMIT2_off32(0x0F, IA32_JAE + 0x10, jmp_label(jmp_label1, 6));
+
+	/* < 32 */
+	/* lshr dreg_lo,cl */
+	EMIT2(0xD3, add_1reg(0xE8, dreg_lo));
+	/* mov ebx,dreg_hi */
+	EMIT2(0x8B, add_2reg(0xC0, dreg_hi, IA32_EBX));
+	/* shr dreg_hi,cl */
+	EMIT2(0xD3, add_1reg(0xE8, dreg_hi));
+
+	/* IA32_ECX = -IA32_ECX + 32 */
+	/* neg ecx */
+	EMIT2(0xF7, add_1reg(0xD8, IA32_ECX));
+	/* add ecx,32 */
+	EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 32);
+
+	/* shl ebx,cl */
+	EMIT2(0xD3, add_1reg(0xE0, IA32_EBX));
+	/* or dreg_lo,ebx */
+	EMIT2(0x09, add_2reg(0xC0, dreg_lo, IA32_EBX));
+
+	/* goto out; */
+	if (is_imm8(jmp_label(jmp_label3, 2)))
+		EMIT2(0xEB, jmp_label(jmp_label3, 2));
+	else
+		EMIT1_off32(0xE9, jmp_label(jmp_label3, 5));
+
+	/* >= 32 */
+	if (jmp_label1 == -1)
+		jmp_label1 = cnt;
+	/* cmp ecx,64 */
+	EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 64);
+	/* Jumps when >= 64 */
+	if (is_imm8(jmp_label(jmp_label2, 2)))
+		EMIT2(IA32_JAE, jmp_label(jmp_label2, 2));
+	else
+		EMIT2_off32(0x0F, IA32_JAE + 0x10, jmp_label(jmp_label2, 6));
+
+	/* >= 32 && < 64 */
+	/* sub ecx,32 */
+	EMIT3(0x83, add_1reg(0xE8, IA32_ECX), 32);
+	/* shr dreg_hi,cl */
+	EMIT2(0xD3, add_1reg(0xE8, dreg_hi));
+	/* mov dreg_lo,dreg_hi */
+	EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi));
+	/* xor dreg_hi,dreg_hi */
+	EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
+
+	/* goto out; */
+	if (is_imm8(jmp_label(jmp_label3, 2)))
+		EMIT2(0xEB, jmp_label(jmp_label3, 2));
+	else
+		EMIT1_off32(0xE9, jmp_label(jmp_label3, 5));
+
+	/* >= 64 */
+	if (jmp_label2 == -1)
+		jmp_label2 = cnt;
+	/* xor dreg_lo,dreg_lo */
+	EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo));
+	/* xor dreg_hi,dreg_hi */
+	EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
+
+	if (jmp_label3 == -1)
+		jmp_label3 = cnt;
+
+	if (dstk) {
+		/* mov dword ptr [ebp+off],dreg_lo */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
+		      STACK_VAR(dst_lo));
+		/* mov dword ptr [ebp+off],dreg_hi */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
+		      STACK_VAR(dst_hi));
+	}
+	/* out: */
+	*pprog = prog;
+}
+
+/* dst = dst << val */
+static inline void emit_ia32_lsh_i64(const u8 dst[], const u32 val,
+				     bool dstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+
+	if (dstk) {
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+		      STACK_VAR(dst_hi));
+	}
+	/* Do LSH operation */
+	if (val < 32) {
+		/* shl dreg_hi,imm8 */
+		EMIT3(0xC1, add_1reg(0xE0, dreg_hi), val);
+		/* mov ebx,dreg_lo */
+		EMIT2(0x8B, add_2reg(0xC0, dreg_lo, IA32_EBX));
+		/* shl dreg_lo,imm8 */
+		EMIT3(0xC1, add_1reg(0xE0, dreg_lo), val);
+
+		/* IA32_ECX = 32 - val */
+		/* mov ecx,val */
+		EMIT2(0xB1, val);
+		/* movzx ecx,ecx */
+		EMIT3(0x0F, 0xB6, add_2reg(0xC0, IA32_ECX, IA32_ECX));
+		/* neg ecx */
+		EMIT2(0xF7, add_1reg(0xD8, IA32_ECX));
+		/* add ecx,32 */
+		EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 32);
+
+		/* shr ebx,cl */
+		EMIT2(0xD3, add_1reg(0xE8, IA32_EBX));
+		/* or dreg_hi,ebx */
+		EMIT2(0x09, add_2reg(0xC0, dreg_hi, IA32_EBX));
+	} else if (val >= 32 && val < 64) {
+		u32 value = val - 32;
+
+		/* shl dreg_lo,imm8 */
+		EMIT3(0xC1, add_1reg(0xE0, dreg_lo), value);
+		/* mov dreg_hi,dreg_lo */
+		EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo));
+		/* xor dreg_lo,dreg_lo */
+		EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo));
+	} else {
+		/* xor dreg_lo,dreg_lo */
+		EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo));
+		/* xor dreg_hi,dreg_hi */
+		EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
+	}
+
+	if (dstk) {
+		/* mov dword ptr [ebp+off],dreg_lo */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
+		      STACK_VAR(dst_lo));
+		/* mov dword ptr [ebp+off],dreg_hi */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
+		      STACK_VAR(dst_hi));
+	}
+	*pprog = prog;
+}
+
+/* dst = dst >> val */
+static inline void emit_ia32_rsh_i64(const u8 dst[], const u32 val,
+				     bool dstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+
+	if (dstk) {
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+		      STACK_VAR(dst_hi));
+	}
+
+	/* Do RSH operation */
+	if (val < 32) {
+		/* shr dreg_lo,imm8 */
+		EMIT3(0xC1, add_1reg(0xE8, dreg_lo), val);
+		/* mov ebx,dreg_hi */
+		EMIT2(0x8B, add_2reg(0xC0, dreg_hi, IA32_EBX));
+		/* shr dreg_hi,imm8 */
+		EMIT3(0xC1, add_1reg(0xE8, dreg_hi), val);
+
+		/* IA32_ECX = 32 - val */
+		/* mov ecx,val */
+		EMIT2(0xB1, val);
+		/* movzx ecx,ecx */
+		EMIT3(0x0F, 0xB6, add_2reg(0xC0, IA32_ECX, IA32_ECX));
+		/* neg ecx */
+		EMIT2(0xF7, add_1reg(0xD8, IA32_ECX));
+		/* add ecx,32 */
+		EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 32);
+
+		/* shl ebx,cl */
+		EMIT2(0xD3, add_1reg(0xE0, IA32_EBX));
+		/* or dreg_lo,ebx */
+		EMIT2(0x09, add_2reg(0xC0, dreg_lo, IA32_EBX));
+	} else if (val >= 32 && val < 64) {
+		u32 value = val - 32;
+
+		/* shr dreg_hi,imm8 */
+		EMIT3(0xC1, add_1reg(0xE8, dreg_hi), value);
+		/* mov dreg_lo,dreg_hi */
+		EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi));
+		/* xor dreg_hi,dreg_hi */
+		EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
+	} else {
+		/* xor dreg_lo,dreg_lo */
+		EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo));
+		/* xor dreg_hi,dreg_hi */
+		EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
+	}
+
+	if (dstk) {
+		/* mov dword ptr [ebp+off],dreg_lo */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
+		      STACK_VAR(dst_lo));
+		/* mov dword ptr [ebp+off],dreg_hi */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
+		      STACK_VAR(dst_hi));
+	}
+	*pprog = prog;
+}
+
+/* dst = dst >> val (signed) */
+static inline void emit_ia32_arsh_i64(const u8 dst[], const u32 val,
+				      bool dstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+
+	if (dstk) {
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+		      STACK_VAR(dst_hi));
+	}
+	/* Do RSH operation */
+	if (val < 32) {
+		/* shr dreg_lo,imm8 */
+		EMIT3(0xC1, add_1reg(0xE8, dreg_lo), val);
+		/* mov ebx,dreg_hi */
+		EMIT2(0x8B, add_2reg(0xC0, dreg_hi, IA32_EBX));
+		/* ashr dreg_hi,imm8 */
+		EMIT3(0xC1, add_1reg(0xF8, dreg_hi), val);
+
+		/* IA32_ECX = 32 - val */
+		/* mov ecx,val */
+		EMIT2(0xB1, val);
+		/* movzx ecx,ecx */
+		EMIT3(0x0F, 0xB6, add_2reg(0xC0, IA32_ECX, IA32_ECX));
+		/* neg ecx */
+		EMIT2(0xF7, add_1reg(0xD8, IA32_ECX));
+		/* add ecx,32 */
+		EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 32);
+
+		/* shl ebx,cl */
+		EMIT2(0xD3, add_1reg(0xE0, IA32_EBX));
+		/* or dreg_lo,ebx */
+		EMIT2(0x09, add_2reg(0xC0, dreg_lo, IA32_EBX));
+	} else if (val >= 32 && val < 64) {
+		u32 value = val - 32;
+
+		/* ashr dreg_hi,imm8 */
+		EMIT3(0xC1, add_1reg(0xF8, dreg_hi), value);
+		/* mov dreg_lo,dreg_hi */
+		EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi));
+
+		/* ashr dreg_hi,imm8 */
+		EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31);
+	} else {
+		/* ashr dreg_hi,imm8 */
+		EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31);
+		/* mov dreg_lo,dreg_hi */
+		EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi));
+	}
+
+	if (dstk) {
+		/* mov dword ptr [ebp+off],dreg_lo */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
+		      STACK_VAR(dst_lo));
+		/* mov dword ptr [ebp+off],dreg_hi */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
+		      STACK_VAR(dst_hi));
+	}
+	*pprog = prog;
+}
+
+static inline void emit_ia32_mul_r64(const u8 dst[], const u8 src[], bool dstk,
+				     bool sstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+
+	if (dstk)
+		/* mov eax,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_hi));
+	else
+		/* mov eax,dst_hi */
+		EMIT2(0x8B, add_2reg(0xC0, dst_hi, IA32_EAX));
+
+	if (sstk)
+		/* mul dword ptr [ebp+off] */
+		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo));
+	else
+		/* mul src_lo */
+		EMIT2(0xF7, add_1reg(0xE0, src_lo));
+
+	/* mov ecx,eax */
+	EMIT2(0x89, add_2reg(0xC0, IA32_ECX, IA32_EAX));
+
+	if (dstk)
+		/* mov eax,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+	else
+		/* mov eax,dst_lo */
+		EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX));
+
+	if (sstk)
+		/* mul dword ptr [ebp+off] */
+		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_hi));
+	else
+		/* mul src_hi */
+		EMIT2(0xF7, add_1reg(0xE0, src_hi));
+
+	/* add eax,eax */
+	EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EAX));
+
+	if (dstk)
+		/* mov eax,dword ptr [ebp+off] */
+		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+	else
+		/* mov eax,dst_lo */
+		EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX));
+
+	if (sstk)
+		/* mul dword ptr [ebp+off] */
+		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo));
+	else
+		/* mul src_lo */
+		EMIT2(0xF7, add_1reg(0xE0, src_lo));
+
+	/* add ecx,edx */
+	EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EDX));
+
+	if (dstk) {
+		/* mov dword ptr [ebp+off],eax */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+		/* mov dword ptr [ebp+off],ecx */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX),
+		      STACK_VAR(dst_hi));
+	} else {
+		/* mov dst_lo,eax */
+		EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EAX));
+		/* mov dst_hi,ecx */
+		EMIT2(0x89, add_2reg(0xC0, dst_hi, IA32_ECX));
+	}
+
+	*pprog = prog;
+}
+
+static inline void emit_ia32_mul_i64(const u8 dst[], const u32 val,
+				     bool dstk, u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	u32 hi;
+
+	hi = val & (1<<31) ? (u32)~0 : 0;
+	/* movl eax,imm32 */
+	EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), val);
+	if (dstk)
+		/* mul dword ptr [ebp+off] */
+		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_hi));
+	else
+		/* mul dst_hi */
+		EMIT2(0xF7, add_1reg(0xE0, dst_hi));
+
+	/* mov ecx,eax */
+	EMIT2(0x89, add_2reg(0xC0, IA32_ECX, IA32_EAX));
+
+	/* movl eax,imm32 */
+	EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), hi);
+	if (dstk)
+		/* mul dword ptr [ebp+off] */
+		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo));
+	else
+		/* mul dst_lo */
+		EMIT2(0xF7, add_1reg(0xE0, dst_lo));
+	/* add ecx,eax */
+	EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EAX));
+
+	/* movl eax,imm32 */
+	EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), val);
+	if (dstk)
+		/* mul dword ptr [ebp+off] */
+		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo));
+	else
+		/* mul dst_lo */
+		EMIT2(0xF7, add_1reg(0xE0, dst_lo));
+
+	/* add ecx,edx */
+	EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EDX));
+
+	if (dstk) {
+		/* mov dword ptr [ebp+off],eax */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
+		      STACK_VAR(dst_lo));
+		/* mov dword ptr [ebp+off],ecx */
+		EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX),
+		      STACK_VAR(dst_hi));
+	} else {
+		/* mov dword ptr [ebp+off],eax */
+		EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EAX));
+		/* mov dword ptr [ebp+off],ecx */
+		EMIT2(0x89, add_2reg(0xC0, dst_hi, IA32_ECX));
+	}
+
+	*pprog = prog;
+}
+
+static int bpf_size_to_x86_bytes(int bpf_size)
+{
+	if (bpf_size == BPF_W)
+		return 4;
+	else if (bpf_size == BPF_H)
+		return 2;
+	else if (bpf_size == BPF_B)
+		return 1;
+	else if (bpf_size == BPF_DW)
+		return 4; /* imm32 */
+	else
+		return 0;
+}
+
+struct jit_context {
+	int cleanup_addr; /* Epilogue code offset */
+};
+
+/* Maximum number of bytes emitted while JITing one eBPF insn */
+#define BPF_MAX_INSN_SIZE	128
+#define BPF_INSN_SAFETY		64
+
+#define PROLOGUE_SIZE 35
+
+/*
+ * Emit prologue code for BPF program and check it's size.
+ * bpf_tail_call helper will skip it while jumping into another program.
+ */
+static void emit_prologue(u8 **pprog, u32 stack_depth)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	const u8 *r1 = bpf2ia32[BPF_REG_1];
+	const u8 fplo = bpf2ia32[BPF_REG_FP][0];
+	const u8 fphi = bpf2ia32[BPF_REG_FP][1];
+	const u8 *tcc = bpf2ia32[TCALL_CNT];
+
+	/* push ebp */
+	EMIT1(0x55);
+	/* mov ebp,esp */
+	EMIT2(0x89, 0xE5);
+	/* push edi */
+	EMIT1(0x57);
+	/* push esi */
+	EMIT1(0x56);
+	/* push ebx */
+	EMIT1(0x53);
+
+	/* sub esp,STACK_SIZE */
+	EMIT2_off32(0x81, 0xEC, STACK_SIZE);
+	/* sub ebp,SCRATCH_SIZE+4+12*/
+	EMIT3(0x83, add_1reg(0xE8, IA32_EBP), SCRATCH_SIZE + 16);
+	/* xor ebx,ebx */
+	EMIT2(0x31, add_2reg(0xC0, IA32_EBX, IA32_EBX));
+
+	/* Set up BPF prog stack base register */
+	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBP), STACK_VAR(fplo));
+	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(fphi));
+
+	/* Move BPF_CTX (EAX) to BPF_REG_R1 */
+	/* mov dword ptr [ebp+off],eax */
+	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0]));
+	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(r1[1]));
+
+	/* Initialize Tail Count */
+	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[0]));
+	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1]));
+
+	BUILD_BUG_ON(cnt != PROLOGUE_SIZE);
+	*pprog = prog;
+}
+
+/* Emit epilogue code for BPF program */
+static void emit_epilogue(u8 **pprog, u32 stack_depth)
+{
+	u8 *prog = *pprog;
+	const u8 *r0 = bpf2ia32[BPF_REG_0];
+	int cnt = 0;
+
+	/* mov eax,dword ptr [ebp+off]*/
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r0[0]));
+	/* mov edx,dword ptr [ebp+off]*/
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r0[1]));
+
+	/* add ebp,SCRATCH_SIZE+4+12*/
+	EMIT3(0x83, add_1reg(0xC0, IA32_EBP), SCRATCH_SIZE + 16);
+
+	/* mov ebx,dword ptr [ebp-12]*/
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), -12);
+	/* mov esi,dword ptr [ebp-8]*/
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ESI), -8);
+	/* mov edi,dword ptr [ebp-4]*/
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDI), -4);
+
+	EMIT1(0xC9); /* leave */
+	EMIT1(0xC3); /* ret */
+	*pprog = prog;
+}
+
+/*
+ * Generate the following code:
+ * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ...
+ *   if (index >= array->map.max_entries)
+ *     goto out;
+ *   if (++tail_call_cnt > MAX_TAIL_CALL_CNT)
+ *     goto out;
+ *   prog = array->ptrs[index];
+ *   if (prog == NULL)
+ *     goto out;
+ *   goto *(prog->bpf_func + prologue_size);
+ * out:
+ */
+static void emit_bpf_tail_call(u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+	const u8 *r1 = bpf2ia32[BPF_REG_1];
+	const u8 *r2 = bpf2ia32[BPF_REG_2];
+	const u8 *r3 = bpf2ia32[BPF_REG_3];
+	const u8 *tcc = bpf2ia32[TCALL_CNT];
+	u32 lo, hi;
+	static int jmp_label1 = -1;
+
+	/*
+	 * if (index >= array->map.max_entries)
+	 *     goto out;
+	 */
+	/* mov eax,dword ptr [ebp+off] */
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r2[0]));
+	/* mov edx,dword ptr [ebp+off] */
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r3[0]));
+
+	/* cmp dword ptr [eax+off],edx */
+	EMIT3(0x39, add_2reg(0x40, IA32_EAX, IA32_EDX),
+	      offsetof(struct bpf_array, map.max_entries));
+	/* jbe out */
+	EMIT2(IA32_JBE, jmp_label(jmp_label1, 2));
+
+	/*
+	 * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
+	 *     goto out;
+	 */
+	lo = (u32)MAX_TAIL_CALL_CNT;
+	hi = (u32)((u64)MAX_TAIL_CALL_CNT >> 32);
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0]));
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1]));
+
+	/* cmp edx,hi */
+	EMIT3(0x83, add_1reg(0xF8, IA32_EBX), hi);
+	EMIT2(IA32_JNE, 3);
+	/* cmp ecx,lo */
+	EMIT3(0x83, add_1reg(0xF8, IA32_ECX), lo);
+
+	/* ja out */
+	EMIT2(IA32_JAE, jmp_label(jmp_label1, 2));
+
+	/* add eax,0x1 */
+	EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 0x01);
+	/* adc ebx,0x0 */
+	EMIT3(0x83, add_1reg(0xD0, IA32_EBX), 0x00);
+
+	/* mov dword ptr [ebp+off],eax */
+	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0]));
+	/* mov dword ptr [ebp+off],edx */
+	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1]));
+
+	/* prog = array->ptrs[index]; */
+	/* mov edx, [eax + edx * 4 + offsetof(...)] */
+	EMIT3_off32(0x8B, 0x94, 0x90, offsetof(struct bpf_array, ptrs));
+
+	/*
+	 * if (prog == NULL)
+	 *     goto out;
+	 */
+	/* test edx,edx */
+	EMIT2(0x85, add_2reg(0xC0, IA32_EDX, IA32_EDX));
+	/* je out */
+	EMIT2(IA32_JE, jmp_label(jmp_label1, 2));
+
+	/* goto *(prog->bpf_func + prologue_size); */
+	/* mov edx, dword ptr [edx + 32] */
+	EMIT3(0x8B, add_2reg(0x40, IA32_EDX, IA32_EDX),
+	      offsetof(struct bpf_prog, bpf_func));
+	/* add edx,prologue_size */
+	EMIT3(0x83, add_1reg(0xC0, IA32_EDX), PROLOGUE_SIZE);
+
+	/* mov eax,dword ptr [ebp+off] */
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0]));
+
+	/*
+	 * Now we're ready to jump into next BPF program:
+	 * eax == ctx (1st arg)
+	 * edx == prog->bpf_func + prologue_size
+	 */
+	RETPOLINE_EDX_BPF_JIT();
+
+	if (jmp_label1 == -1)
+		jmp_label1 = cnt;
+
+	/* out: */
+	*pprog = prog;
+}
+
+/* Push the scratch stack register on top of the stack. */
+static inline void emit_push_r64(const u8 src[], u8 **pprog)
+{
+	u8 *prog = *pprog;
+	int cnt = 0;
+
+	/* mov ecx,dword ptr [ebp+off] */
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_hi));
+	/* push ecx */
+	EMIT1(0x51);
+
+	/* mov ecx,dword ptr [ebp+off] */
+	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_lo));
+	/* push ecx */
+	EMIT1(0x51);
+
+	*pprog = prog;
+}
+
+static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
+		  int oldproglen, struct jit_context *ctx)
+{
+	struct bpf_insn *insn = bpf_prog->insnsi;
+	int insn_cnt = bpf_prog->len;
+	bool seen_exit = false;
+	u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
+	int i, cnt = 0;
+	int proglen = 0;
+	u8 *prog = temp;
+
+	emit_prologue(&prog, bpf_prog->aux->stack_depth);
+
+	for (i = 0; i < insn_cnt; i++, insn++) {
+		const s32 imm32 = insn->imm;
+		const bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
+		const bool dstk = insn->dst_reg == BPF_REG_AX ? false : true;
+		const bool sstk = insn->src_reg == BPF_REG_AX ? false : true;
+		const u8 code = insn->code;
+		const u8 *dst = bpf2ia32[insn->dst_reg];
+		const u8 *src = bpf2ia32[insn->src_reg];
+		const u8 *r0 = bpf2ia32[BPF_REG_0];
+		s64 jmp_offset;
+		u8 jmp_cond;
+		int ilen;
+		u8 *func;
+
+		switch (code) {
+		/* ALU operations */
+		/* dst = src */
+		case BPF_ALU | BPF_MOV | BPF_K:
+		case BPF_ALU | BPF_MOV | BPF_X:
+		case BPF_ALU64 | BPF_MOV | BPF_K:
+		case BPF_ALU64 | BPF_MOV | BPF_X:
+			switch (BPF_SRC(code)) {
+			case BPF_X:
+				emit_ia32_mov_r64(is64, dst, src, dstk,
+						  sstk, &prog);
+				break;
+			case BPF_K:
+				/* Sign-extend immediate value to dst reg */
+				emit_ia32_mov_i64(is64, dst, imm32,
+						  dstk, &prog);
+				break;
+			}
+			break;
+		/* dst = dst + src/imm */
+		/* dst = dst - src/imm */
+		/* dst = dst | src/imm */
+		/* dst = dst & src/imm */
+		/* dst = dst ^ src/imm */
+		/* dst = dst * src/imm */
+		/* dst = dst << src */
+		/* dst = dst >> src */
+		case BPF_ALU | BPF_ADD | BPF_K:
+		case BPF_ALU | BPF_ADD | BPF_X:
+		case BPF_ALU | BPF_SUB | BPF_K:
+		case BPF_ALU | BPF_SUB | BPF_X:
+		case BPF_ALU | BPF_OR | BPF_K:
+		case BPF_ALU | BPF_OR | BPF_X:
+		case BPF_ALU | BPF_AND | BPF_K:
+		case BPF_ALU | BPF_AND | BPF_X:
+		case BPF_ALU | BPF_XOR | BPF_K:
+		case BPF_ALU | BPF_XOR | BPF_X:
+		case BPF_ALU64 | BPF_ADD | BPF_K:
+		case BPF_ALU64 | BPF_ADD | BPF_X:
+		case BPF_ALU64 | BPF_SUB | BPF_K:
+		case BPF_ALU64 | BPF_SUB | BPF_X:
+		case BPF_ALU64 | BPF_OR | BPF_K:
+		case BPF_ALU64 | BPF_OR | BPF_X:
+		case BPF_ALU64 | BPF_AND | BPF_K:
+		case BPF_ALU64 | BPF_AND | BPF_X:
+		case BPF_ALU64 | BPF_XOR | BPF_K:
+		case BPF_ALU64 | BPF_XOR | BPF_X:
+			switch (BPF_SRC(code)) {
+			case BPF_X:
+				emit_ia32_alu_r64(is64, BPF_OP(code), dst,
+						  src, dstk, sstk, &prog);
+				break;
+			case BPF_K:
+				emit_ia32_alu_i64(is64, BPF_OP(code), dst,
+						  imm32, dstk, &prog);
+				break;
+			}
+			break;
+		case BPF_ALU | BPF_MUL | BPF_K:
+		case BPF_ALU | BPF_MUL | BPF_X:
+			switch (BPF_SRC(code)) {
+			case BPF_X:
+				emit_ia32_mul_r(dst_lo, src_lo, dstk,
+						sstk, &prog);
+				break;
+			case BPF_K:
+				/* mov ecx,imm32*/
+				EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX),
+					    imm32);
+				emit_ia32_mul_r(dst_lo, IA32_ECX, dstk,
+						false, &prog);
+				break;
+			}
+			emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
+			break;
+		case BPF_ALU | BPF_LSH | BPF_X:
+		case BPF_ALU | BPF_RSH | BPF_X:
+		case BPF_ALU | BPF_ARSH | BPF_K:
+		case BPF_ALU | BPF_ARSH | BPF_X:
+			switch (BPF_SRC(code)) {
+			case BPF_X:
+				emit_ia32_shift_r(BPF_OP(code), dst_lo, src_lo,
+						  dstk, sstk, &prog);
+				break;
+			case BPF_K:
+				/* mov ecx,imm32*/
+				EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX),
+					    imm32);
+				emit_ia32_shift_r(BPF_OP(code), dst_lo,
+						  IA32_ECX, dstk, false,
+						  &prog);
+				break;
+			}
+			emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
+			break;
+		/* dst = dst / src(imm) */
+		/* dst = dst % src(imm) */
+		case BPF_ALU | BPF_DIV | BPF_K:
+		case BPF_ALU | BPF_DIV | BPF_X:
+		case BPF_ALU | BPF_MOD | BPF_K:
+		case BPF_ALU | BPF_MOD | BPF_X:
+			switch (BPF_SRC(code)) {
+			case BPF_X:
+				emit_ia32_div_mod_r(BPF_OP(code), dst_lo,
+						    src_lo, dstk, sstk, &prog);
+				break;
+			case BPF_K:
+				/* mov ecx,imm32*/
+				EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX),
+					    imm32);
+				emit_ia32_div_mod_r(BPF_OP(code), dst_lo,
+						    IA32_ECX, dstk, false,
+						    &prog);
+				break;
+			}
+			emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
+			break;
+		case BPF_ALU64 | BPF_DIV | BPF_K:
+		case BPF_ALU64 | BPF_DIV | BPF_X:
+		case BPF_ALU64 | BPF_MOD | BPF_K:
+		case BPF_ALU64 | BPF_MOD | BPF_X:
+			goto notyet;
+		/* dst = dst >> imm */
+		/* dst = dst << imm */
+		case BPF_ALU | BPF_RSH | BPF_K:
+		case BPF_ALU | BPF_LSH | BPF_K:
+			if (unlikely(imm32 > 31))
+				return -EINVAL;
+			/* mov ecx,imm32*/
+			EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32);
+			emit_ia32_shift_r(BPF_OP(code), dst_lo, IA32_ECX, dstk,
+					  false, &prog);
+			emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
+			break;
+		/* dst = dst << imm */
+		case BPF_ALU64 | BPF_LSH | BPF_K:
+			if (unlikely(imm32 > 63))
+				return -EINVAL;
+			emit_ia32_lsh_i64(dst, imm32, dstk, &prog);
+			break;
+		/* dst = dst >> imm */
+		case BPF_ALU64 | BPF_RSH | BPF_K:
+			if (unlikely(imm32 > 63))
+				return -EINVAL;
+			emit_ia32_rsh_i64(dst, imm32, dstk, &prog);
+			break;
+		/* dst = dst << src */
+		case BPF_ALU64 | BPF_LSH | BPF_X:
+			emit_ia32_lsh_r64(dst, src, dstk, sstk, &prog);
+			break;
+		/* dst = dst >> src */
+		case BPF_ALU64 | BPF_RSH | BPF_X:
+			emit_ia32_rsh_r64(dst, src, dstk, sstk, &prog);
+			break;
+		/* dst = dst >> src (signed) */
+		case BPF_ALU64 | BPF_ARSH | BPF_X:
+			emit_ia32_arsh_r64(dst, src, dstk, sstk, &prog);
+			break;
+		/* dst = dst >> imm (signed) */
+		case BPF_ALU64 | BPF_ARSH | BPF_K:
+			if (unlikely(imm32 > 63))
+				return -EINVAL;
+			emit_ia32_arsh_i64(dst, imm32, dstk, &prog);
+			break;
+		/* dst = ~dst */
+		case BPF_ALU | BPF_NEG:
+			emit_ia32_alu_i(is64, false, BPF_OP(code),
+					dst_lo, 0, dstk, &prog);
+			emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
+			break;
+		/* dst = ~dst (64 bit) */
+		case BPF_ALU64 | BPF_NEG:
+			emit_ia32_neg64(dst, dstk, &prog);
+			break;
+		/* dst = dst * src/imm */
+		case BPF_ALU64 | BPF_MUL | BPF_X:
+		case BPF_ALU64 | BPF_MUL | BPF_K:
+			switch (BPF_SRC(code)) {
+			case BPF_X:
+				emit_ia32_mul_r64(dst, src, dstk, sstk, &prog);
+				break;
+			case BPF_K:
+				emit_ia32_mul_i64(dst, imm32, dstk, &prog);
+				break;
+			}
+			break;
+		/* dst = htole(dst) */
+		case BPF_ALU | BPF_END | BPF_FROM_LE:
+			emit_ia32_to_le_r64(dst, imm32, dstk, &prog);
+			break;
+		/* dst = htobe(dst) */
+		case BPF_ALU | BPF_END | BPF_FROM_BE:
+			emit_ia32_to_be_r64(dst, imm32, dstk, &prog);
+			break;
+		/* dst = imm64 */
+		case BPF_LD | BPF_IMM | BPF_DW: {
+			s32 hi, lo = imm32;
+
+			hi = insn[1].imm;
+			emit_ia32_mov_i(dst_lo, lo, dstk, &prog);
+			emit_ia32_mov_i(dst_hi, hi, dstk, &prog);
+			insn++;
+			i++;
+			break;
+		}
+		/* ST: *(u8*)(dst_reg + off) = imm */
+		case BPF_ST | BPF_MEM | BPF_H:
+		case BPF_ST | BPF_MEM | BPF_B:
+		case BPF_ST | BPF_MEM | BPF_W:
+		case BPF_ST | BPF_MEM | BPF_DW:
+			if (dstk)
+				/* mov eax,dword ptr [ebp+off] */
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+				      STACK_VAR(dst_lo));
+			else
+				/* mov eax,dst_lo */
+				EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX));
+
+			switch (BPF_SIZE(code)) {
+			case BPF_B:
+				EMIT(0xC6, 1); break;
+			case BPF_H:
+				EMIT2(0x66, 0xC7); break;
+			case BPF_W:
+			case BPF_DW:
+				EMIT(0xC7, 1); break;
+			}
+
+			if (is_imm8(insn->off))
+				EMIT2(add_1reg(0x40, IA32_EAX), insn->off);
+			else
+				EMIT1_off32(add_1reg(0x80, IA32_EAX),
+					    insn->off);
+			EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(code)));
+
+			if (BPF_SIZE(code) == BPF_DW) {
+				u32 hi;
+
+				hi = imm32 & (1<<31) ? (u32)~0 : 0;
+				EMIT2_off32(0xC7, add_1reg(0x80, IA32_EAX),
+					    insn->off + 4);
+				EMIT(hi, 4);
+			}
+			break;
+
+		/* STX: *(u8*)(dst_reg + off) = src_reg */
+		case BPF_STX | BPF_MEM | BPF_B:
+		case BPF_STX | BPF_MEM | BPF_H:
+		case BPF_STX | BPF_MEM | BPF_W:
+		case BPF_STX | BPF_MEM | BPF_DW:
+			if (dstk)
+				/* mov eax,dword ptr [ebp+off] */
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+				      STACK_VAR(dst_lo));
+			else
+				/* mov eax,dst_lo */
+				EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX));
+
+			if (sstk)
+				/* mov edx,dword ptr [ebp+off] */
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+				      STACK_VAR(src_lo));
+			else
+				/* mov edx,src_lo */
+				EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_EDX));
+
+			switch (BPF_SIZE(code)) {
+			case BPF_B:
+				EMIT(0x88, 1); break;
+			case BPF_H:
+				EMIT2(0x66, 0x89); break;
+			case BPF_W:
+			case BPF_DW:
+				EMIT(0x89, 1); break;
+			}
+
+			if (is_imm8(insn->off))
+				EMIT2(add_2reg(0x40, IA32_EAX, IA32_EDX),
+				      insn->off);
+			else
+				EMIT1_off32(add_2reg(0x80, IA32_EAX, IA32_EDX),
+					    insn->off);
+
+			if (BPF_SIZE(code) == BPF_DW) {
+				if (sstk)
+					/* mov edi,dword ptr [ebp+off] */
+					EMIT3(0x8B, add_2reg(0x40, IA32_EBP,
+							     IA32_EDX),
+					      STACK_VAR(src_hi));
+				else
+					/* mov edi,src_hi */
+					EMIT2(0x8B, add_2reg(0xC0, src_hi,
+							     IA32_EDX));
+				EMIT1(0x89);
+				if (is_imm8(insn->off + 4)) {
+					EMIT2(add_2reg(0x40, IA32_EAX,
+						       IA32_EDX),
+					      insn->off + 4);
+				} else {
+					EMIT1(add_2reg(0x80, IA32_EAX,
+						       IA32_EDX));
+					EMIT(insn->off + 4, 4);
+				}
+			}
+			break;
+
+		/* LDX: dst_reg = *(u8*)(src_reg + off) */
+		case BPF_LDX | BPF_MEM | BPF_B:
+		case BPF_LDX | BPF_MEM | BPF_H:
+		case BPF_LDX | BPF_MEM | BPF_W:
+		case BPF_LDX | BPF_MEM | BPF_DW:
+			if (sstk)
+				/* mov eax,dword ptr [ebp+off] */
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+				      STACK_VAR(src_lo));
+			else
+				/* mov eax,dword ptr [ebp+off] */
+				EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_EAX));
+
+			switch (BPF_SIZE(code)) {
+			case BPF_B:
+				EMIT2(0x0F, 0xB6); break;
+			case BPF_H:
+				EMIT2(0x0F, 0xB7); break;
+			case BPF_W:
+			case BPF_DW:
+				EMIT(0x8B, 1); break;
+			}
+
+			if (is_imm8(insn->off))
+				EMIT2(add_2reg(0x40, IA32_EAX, IA32_EDX),
+				      insn->off);
+			else
+				EMIT1_off32(add_2reg(0x80, IA32_EAX, IA32_EDX),
+					    insn->off);
+
+			if (dstk)
+				/* mov dword ptr [ebp+off],edx */
+				EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),
+				      STACK_VAR(dst_lo));
+			else
+				/* mov dst_lo,edx */
+				EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EDX));
+			switch (BPF_SIZE(code)) {
+			case BPF_B:
+			case BPF_H:
+			case BPF_W:
+				if (dstk) {
+					EMIT3(0xC7, add_1reg(0x40, IA32_EBP),
+					      STACK_VAR(dst_hi));
+					EMIT(0x0, 4);
+				} else {
+					EMIT3(0xC7, add_1reg(0xC0, dst_hi), 0);
+				}
+				break;
+			case BPF_DW:
+				EMIT2_off32(0x8B,
+					    add_2reg(0x80, IA32_EAX, IA32_EDX),
+					    insn->off + 4);
+				if (dstk)
+					EMIT3(0x89,
+					      add_2reg(0x40, IA32_EBP,
+						       IA32_EDX),
+					      STACK_VAR(dst_hi));
+				else
+					EMIT2(0x89,
+					      add_2reg(0xC0, dst_hi, IA32_EDX));
+				break;
+			default:
+				break;
+			}
+			break;
+		/* call */
+		case BPF_JMP | BPF_CALL:
+		{
+			const u8 *r1 = bpf2ia32[BPF_REG_1];
+			const u8 *r2 = bpf2ia32[BPF_REG_2];
+			const u8 *r3 = bpf2ia32[BPF_REG_3];
+			const u8 *r4 = bpf2ia32[BPF_REG_4];
+			const u8 *r5 = bpf2ia32[BPF_REG_5];
+
+			if (insn->src_reg == BPF_PSEUDO_CALL)
+				goto notyet;
+
+			func = (u8 *) __bpf_call_base + imm32;
+			jmp_offset = func - (image + addrs[i]);
+
+			if (!imm32 || !is_simm32(jmp_offset)) {
+				pr_err("unsupported BPF func %d addr %p image %p\n",
+				       imm32, func, image);
+				return -EINVAL;
+			}
+
+			/* mov eax,dword ptr [ebp+off] */
+			EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+			      STACK_VAR(r1[0]));
+			/* mov edx,dword ptr [ebp+off] */
+			EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+			      STACK_VAR(r1[1]));
+
+			emit_push_r64(r5, &prog);
+			emit_push_r64(r4, &prog);
+			emit_push_r64(r3, &prog);
+			emit_push_r64(r2, &prog);
+
+			EMIT1_off32(0xE8, jmp_offset + 9);
+
+			/* mov dword ptr [ebp+off],eax */
+			EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
+			      STACK_VAR(r0[0]));
+			/* mov dword ptr [ebp+off],edx */
+			EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),
+			      STACK_VAR(r0[1]));
+
+			/* add esp,32 */
+			EMIT3(0x83, add_1reg(0xC0, IA32_ESP), 32);
+			break;
+		}
+		case BPF_JMP | BPF_TAIL_CALL:
+			emit_bpf_tail_call(&prog);
+			break;
+
+		/* cond jump */
+		case BPF_JMP | BPF_JEQ | BPF_X:
+		case BPF_JMP | BPF_JNE | BPF_X:
+		case BPF_JMP | BPF_JGT | BPF_X:
+		case BPF_JMP | BPF_JLT | BPF_X:
+		case BPF_JMP | BPF_JGE | BPF_X:
+		case BPF_JMP | BPF_JLE | BPF_X:
+		case BPF_JMP | BPF_JSGT | BPF_X:
+		case BPF_JMP | BPF_JSLE | BPF_X:
+		case BPF_JMP | BPF_JSLT | BPF_X:
+		case BPF_JMP | BPF_JSGE | BPF_X: {
+			u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+			u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+			u8 sreg_lo = sstk ? IA32_ECX : src_lo;
+			u8 sreg_hi = sstk ? IA32_EBX : src_hi;
+
+			if (dstk) {
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+				      STACK_VAR(dst_lo));
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+				      STACK_VAR(dst_hi));
+			}
+
+			if (sstk) {
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
+				      STACK_VAR(src_lo));
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX),
+				      STACK_VAR(src_hi));
+			}
+
+			/* cmp dreg_hi,sreg_hi */
+			EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi));
+			EMIT2(IA32_JNE, 2);
+			/* cmp dreg_lo,sreg_lo */
+			EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo));
+			goto emit_cond_jmp;
+		}
+		case BPF_JMP | BPF_JSET | BPF_X: {
+			u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+			u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+			u8 sreg_lo = sstk ? IA32_ECX : src_lo;
+			u8 sreg_hi = sstk ? IA32_EBX : src_hi;
+
+			if (dstk) {
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+				      STACK_VAR(dst_lo));
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+				      STACK_VAR(dst_hi));
+			}
+
+			if (sstk) {
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
+				      STACK_VAR(src_lo));
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX),
+				      STACK_VAR(src_hi));
+			}
+			/* and dreg_lo,sreg_lo */
+			EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo));
+			/* and dreg_hi,sreg_hi */
+			EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi));
+			/* or dreg_lo,dreg_hi */
+			EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi));
+			goto emit_cond_jmp;
+		}
+		case BPF_JMP | BPF_JSET | BPF_K: {
+			u32 hi;
+			u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+			u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+			u8 sreg_lo = IA32_ECX;
+			u8 sreg_hi = IA32_EBX;
+
+			if (dstk) {
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+				      STACK_VAR(dst_lo));
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+				      STACK_VAR(dst_hi));
+			}
+			hi = imm32 & (1<<31) ? (u32)~0 : 0;
+
+			/* mov ecx,imm32 */
+			EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32);
+			/* mov ebx,imm32 */
+			EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi);
+
+			/* and dreg_lo,sreg_lo */
+			EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo));
+			/* and dreg_hi,sreg_hi */
+			EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi));
+			/* or dreg_lo,dreg_hi */
+			EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi));
+			goto emit_cond_jmp;
+		}
+		case BPF_JMP | BPF_JEQ | BPF_K:
+		case BPF_JMP | BPF_JNE | BPF_K:
+		case BPF_JMP | BPF_JGT | BPF_K:
+		case BPF_JMP | BPF_JLT | BPF_K:
+		case BPF_JMP | BPF_JGE | BPF_K:
+		case BPF_JMP | BPF_JLE | BPF_K:
+		case BPF_JMP | BPF_JSGT | BPF_K:
+		case BPF_JMP | BPF_JSLE | BPF_K:
+		case BPF_JMP | BPF_JSLT | BPF_K:
+		case BPF_JMP | BPF_JSGE | BPF_K: {
+			u32 hi;
+			u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
+			u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+			u8 sreg_lo = IA32_ECX;
+			u8 sreg_hi = IA32_EBX;
+
+			if (dstk) {
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
+				      STACK_VAR(dst_lo));
+				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
+				      STACK_VAR(dst_hi));
+			}
+
+			hi = imm32 & (1<<31) ? (u32)~0 : 0;
+			/* mov ecx,imm32 */
+			EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32);
+			/* mov ebx,imm32 */
+			EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi);
+
+			/* cmp dreg_hi,sreg_hi */
+			EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi));
+			EMIT2(IA32_JNE, 2);
+			/* cmp dreg_lo,sreg_lo */
+			EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo));
+
+emit_cond_jmp:		/* Convert BPF opcode to x86 */
+			switch (BPF_OP(code)) {
+			case BPF_JEQ:
+				jmp_cond = IA32_JE;
+				break;
+			case BPF_JSET:
+			case BPF_JNE:
+				jmp_cond = IA32_JNE;
+				break;
+			case BPF_JGT:
+				/* GT is unsigned '>', JA in x86 */
+				jmp_cond = IA32_JA;
+				break;
+			case BPF_JLT:
+				/* LT is unsigned '<', JB in x86 */
+				jmp_cond = IA32_JB;
+				break;
+			case BPF_JGE:
+				/* GE is unsigned '>=', JAE in x86 */
+				jmp_cond = IA32_JAE;
+				break;
+			case BPF_JLE:
+				/* LE is unsigned '<=', JBE in x86 */
+				jmp_cond = IA32_JBE;
+				break;
+			case BPF_JSGT:
+				/* Signed '>', GT in x86 */
+				jmp_cond = IA32_JG;
+				break;
+			case BPF_JSLT:
+				/* Signed '<', LT in x86 */
+				jmp_cond = IA32_JL;
+				break;
+			case BPF_JSGE:
+				/* Signed '>=', GE in x86 */
+				jmp_cond = IA32_JGE;
+				break;
+			case BPF_JSLE:
+				/* Signed '<=', LE in x86 */
+				jmp_cond = IA32_JLE;
+				break;
+			default: /* to silence GCC warning */
+				return -EFAULT;
+			}
+			jmp_offset = addrs[i + insn->off] - addrs[i];
+			if (is_imm8(jmp_offset)) {
+				EMIT2(jmp_cond, jmp_offset);
+			} else if (is_simm32(jmp_offset)) {
+				EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
+			} else {
+				pr_err("cond_jmp gen bug %llx\n", jmp_offset);
+				return -EFAULT;
+			}
+
+			break;
+		}
+		case BPF_JMP | BPF_JA:
+			if (insn->off == -1)
+				/* -1 jmp instructions will always jump
+				 * backwards two bytes. Explicitly handling
+				 * this case avoids wasting too many passes
+				 * when there are long sequences of replaced
+				 * dead code.
+				 */
+				jmp_offset = -2;
+			else
+				jmp_offset = addrs[i + insn->off] - addrs[i];
+
+			if (!jmp_offset)
+				/* Optimize out nop jumps */
+				break;
+emit_jmp:
+			if (is_imm8(jmp_offset)) {
+				EMIT2(0xEB, jmp_offset);
+			} else if (is_simm32(jmp_offset)) {
+				EMIT1_off32(0xE9, jmp_offset);
+			} else {
+				pr_err("jmp gen bug %llx\n", jmp_offset);
+				return -EFAULT;
+			}
+			break;
+		/* STX XADD: lock *(u32 *)(dst + off) += src */
+		case BPF_STX | BPF_XADD | BPF_W:
+		/* STX XADD: lock *(u64 *)(dst + off) += src */
+		case BPF_STX | BPF_XADD | BPF_DW:
+			goto notyet;
+		case BPF_JMP | BPF_EXIT:
+			if (seen_exit) {
+				jmp_offset = ctx->cleanup_addr - addrs[i];
+				goto emit_jmp;
+			}
+			seen_exit = true;
+			/* Update cleanup_addr */
+			ctx->cleanup_addr = proglen;
+			emit_epilogue(&prog, bpf_prog->aux->stack_depth);
+			break;
+notyet:
+			pr_info_once("*** NOT YET: opcode %02x ***\n", code);
+			return -EFAULT;
+		default:
+			/*
+			 * This error will be seen if new instruction was added
+			 * to interpreter, but not to JIT or if there is junk in
+			 * bpf_prog
+			 */
+			pr_err("bpf_jit: unknown opcode %02x\n", code);
+			return -EINVAL;
+		}
+
+		ilen = prog - temp;
+		if (ilen > BPF_MAX_INSN_SIZE) {
+			pr_err("bpf_jit: fatal insn size error\n");
+			return -EFAULT;
+		}
+
+		if (image) {
+			if (unlikely(proglen + ilen > oldproglen)) {
+				pr_err("bpf_jit: fatal error\n");
+				return -EFAULT;
+			}
+			memcpy(image + proglen, temp, ilen);
+		}
+		proglen += ilen;
+		addrs[i] = proglen;
+		prog = temp;
+	}
+	return proglen;
+}
+
+struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+{
+	struct bpf_binary_header *header = NULL;
+	struct bpf_prog *tmp, *orig_prog = prog;
+	int proglen, oldproglen = 0;
+	struct jit_context ctx = {};
+	bool tmp_blinded = false;
+	u8 *image = NULL;
+	int *addrs;
+	int pass;
+	int i;
+
+	if (!prog->jit_requested)
+		return orig_prog;
+
+	tmp = bpf_jit_blind_constants(prog);
+	/*
+	 * If blinding was requested and we failed during blinding,
+	 * we must fall back to the interpreter.
+	 */
+	if (IS_ERR(tmp))
+		return orig_prog;
+	if (tmp != prog) {
+		tmp_blinded = true;
+		prog = tmp;
+	}
+
+	addrs = kmalloc(prog->len * sizeof(*addrs), GFP_KERNEL);
+	if (!addrs) {
+		prog = orig_prog;
+		goto out;
+	}
+
+	/*
+	 * Before first pass, make a rough estimation of addrs[]
+	 * each BPF instruction is translated to less than 64 bytes
+	 */
+	for (proglen = 0, i = 0; i < prog->len; i++) {
+		proglen += 64;
+		addrs[i] = proglen;
+	}
+	ctx.cleanup_addr = proglen;
+
+	/*
+	 * JITed image shrinks with every pass and the loop iterates
+	 * until the image stops shrinking. Very large BPF programs
+	 * may converge on the last pass. In such case do one more
+	 * pass to emit the final image.
+	 */
+	for (pass = 0; pass < 20 || image; pass++) {
+		proglen = do_jit(prog, addrs, image, oldproglen, &ctx);
+		if (proglen <= 0) {
+out_image:
+			image = NULL;
+			if (header)
+				bpf_jit_binary_free(header);
+			prog = orig_prog;
+			goto out_addrs;
+		}
+		if (image) {
+			if (proglen != oldproglen) {
+				pr_err("bpf_jit: proglen=%d != oldproglen=%d\n",
+				       proglen, oldproglen);
+				goto out_image;
+			}
+			break;
+		}
+		if (proglen == oldproglen) {
+			header = bpf_jit_binary_alloc(proglen, &image,
+						      1, jit_fill_hole);
+			if (!header) {
+				prog = orig_prog;
+				goto out_addrs;
+			}
+		}
+		oldproglen = proglen;
+		cond_resched();
+	}
+
+	if (bpf_jit_enable > 1)
+		bpf_jit_dump(prog->len, proglen, pass + 1, image);
+
+	if (image) {
+		bpf_jit_binary_lock_ro(header);
+		prog->bpf_func = (void *)image;
+		prog->jited = 1;
+		prog->jited_len = proglen;
+	} else {
+		prog = orig_prog;
+	}
+
+out_addrs:
+	kfree(addrs);
+out:
+	if (tmp_blinded)
+		bpf_jit_prog_release_other(prog, prog == orig_prog ?
+					   tmp : orig_prog);
+	return prog;
+}
diff --git a/arch/x86/pci/early.c b/arch/x86/pci/early.c
index f0114007e915..e5f753cbb1c3 100644
--- a/arch/x86/pci/early.c
+++ b/arch/x86/pci/early.c
@@ -59,24 +59,15 @@ int early_pci_allowed(void)
 
 void early_dump_pci_device(u8 bus, u8 slot, u8 func)
 {
+	u32 value[256 / 4];
 	int i;
-	int j;
-	u32 val;
 
-	printk(KERN_INFO "pci 0000:%02x:%02x.%d config space:",
-	       bus, slot, func);
+	pr_info("pci 0000:%02x:%02x.%d config space:\n", bus, slot, func);
 
-	for (i = 0; i < 256; i += 4) {
-		if (!(i & 0x0f))
-			printk("\n  %02x:",i);
+	for (i = 0; i < 256; i += 4)
+		value[i / 4] = read_pci_config(bus, slot, func, i);
 
-		val = read_pci_config(bus, slot, func, i);
-		for (j = 0; j < 4; j++) {
-			printk(" %02x", val & 0xff);
-			val >>= 8;
-		}
-	}
-	printk("\n");
+	print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1, value, 256, false);
 }
 
 void early_dump_pci_devices(void)
diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c
index 54ef19e90705..13f4485ca388 100644
--- a/arch/x86/pci/fixup.c
+++ b/arch/x86/pci/fixup.c
@@ -636,6 +636,10 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2030, quirk_no_aersid);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2031, quirk_no_aersid);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2032, quirk_no_aersid);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2033, quirk_no_aersid);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x334a, quirk_no_aersid);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x334b, quirk_no_aersid);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x334c, quirk_no_aersid);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x334d, quirk_no_aersid);
 
 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 
diff --git a/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c b/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c
index 4392c15ed9e0..fc77d69e51d7 100644
--- a/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c
+++ b/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c
@@ -10,7 +10,7 @@
  * of the License.
  */
 
-#include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/machine.h>
 #include <linux/regulator/fixed.h>
@@ -43,7 +43,6 @@ static struct fixed_voltage_config bcm43xx_vmmc = {
 	 * real voltage and signaling are still 1.8V.
 	 */
 	.microvolts		= 2000000,		/* 1.8V */
-	.gpio			= -EINVAL,
 	.startup_delay		= 250 * 1000,		/* 250ms */
 	.enable_high		= 1,			/* active high */
 	.enabled_at_boot	= 0,			/* disabled at boot */
@@ -58,11 +57,23 @@ static struct platform_device bcm43xx_vmmc_regulator = {
 	},
 };
 
+static struct gpiod_lookup_table bcm43xx_vmmc_gpio_table = {
+	.dev_id	= "reg-fixed-voltage.0",
+	.table	= {
+		GPIO_LOOKUP("0000:00:0c.0", -1, "enable", GPIO_ACTIVE_LOW),
+		{}
+	},
+};
+
 static int __init bcm43xx_regulator_register(void)
 {
+	struct gpiod_lookup_table *table = &bcm43xx_vmmc_gpio_table;
+	struct gpiod_lookup *lookup = table->table;
 	int ret;
 
-	bcm43xx_vmmc.gpio = get_gpio_by_name(WLAN_SFI_GPIO_ENABLE_NAME);
+	lookup[0].chip_hwnum = get_gpio_by_name(WLAN_SFI_GPIO_ENABLE_NAME);
+	gpiod_add_lookup_table(table);
+
 	ret = platform_device_register(&bcm43xx_vmmc_regulator);
 	if (ret) {
 		pr_err("%s: vmmc regulator register failed\n", __func__);
diff --git a/arch/x86/um/Kconfig b/arch/x86/um/Kconfig
index 13ed827c7c66..9d529f22fd9d 100644
--- a/arch/x86/um/Kconfig
+++ b/arch/x86/um/Kconfig
@@ -1,5 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0
-mainmenu "User Mode Linux/$SUBARCH $KERNELVERSION Kernel Configuration"
+mainmenu "User Mode Linux/$(SUBARCH) $(KERNELVERSION) Kernel Configuration"
+
+comment "Compiler: $(CC_VERSION_TEXT)"
+
+source "scripts/Kconfig.include"
 
 source "arch/um/Kconfig.common"
 
@@ -16,8 +20,8 @@ config UML_X86
 	select GENERIC_FIND_FIRST_BIT
 
 config 64BIT
-	bool "64-bit kernel" if SUBARCH = "x86"
-	default SUBARCH != "i386"
+	bool "64-bit kernel" if "$(SUBARCH)" = "x86"
+	default "$(SUBARCH)" != "i386"
 
 config X86_32
 	def_bool !64BIT
diff --git a/arch/x86/um/vdso/Makefile b/arch/x86/um/vdso/Makefile
index 10003359e633..b2d6967262b2 100644
--- a/arch/x86/um/vdso/Makefile
+++ b/arch/x86/um/vdso/Makefile
@@ -23,14 +23,14 @@ $(obj)/vdso.o: $(obj)/vdso.so
 
 targets += vdso.so vdso.so.dbg vdso.lds $(vobjs-y)
 
-export CPPFLAGS_vdso.lds += -P -C
+CPPFLAGS_vdso.lds += -P -C
 
 VDSO_LDFLAGS_vdso.lds = -m64 -Wl,-soname=linux-vdso.so.1 \
        -Wl,-z,max-page-size=4096 -Wl,-z,common-page-size=4096
 
 $(obj)/vdso.o: $(src)/vdso.S $(obj)/vdso.so
 
-$(obj)/vdso.so.dbg: $(src)/vdso.lds $(vobjs) FORCE
+$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(vobjs) FORCE
 	$(call if_changed,vdso)
 
 $(obj)/%.so: OBJCOPYFLAGS := -S
diff --git a/arch/x86/xen/efi.c b/arch/x86/xen/efi.c
index a18703be9ead..1804b27f9632 100644
--- a/arch/x86/xen/efi.c
+++ b/arch/x86/xen/efi.c
@@ -115,6 +115,61 @@ static efi_system_table_t __init *xen_efi_probe(void)
 	return &efi_systab_xen;
 }
 
+/*
+ * Determine whether we're in secure boot mode.
+ *
+ * Please keep the logic in sync with
+ * drivers/firmware/efi/libstub/secureboot.c:efi_get_secureboot().
+ */
+static enum efi_secureboot_mode xen_efi_get_secureboot(void)
+{
+	static efi_guid_t efi_variable_guid = EFI_GLOBAL_VARIABLE_GUID;
+	static efi_guid_t shim_guid = EFI_SHIM_LOCK_GUID;
+	efi_status_t status;
+	u8 moksbstate, secboot, setupmode;
+	unsigned long size;
+
+	size = sizeof(secboot);
+	status = efi.get_variable(L"SecureBoot", &efi_variable_guid,
+				  NULL, &size, &secboot);
+
+	if (status == EFI_NOT_FOUND)
+		return efi_secureboot_mode_disabled;
+
+	if (status != EFI_SUCCESS)
+		goto out_efi_err;
+
+	size = sizeof(setupmode);
+	status = efi.get_variable(L"SetupMode", &efi_variable_guid,
+				  NULL, &size, &setupmode);
+
+	if (status != EFI_SUCCESS)
+		goto out_efi_err;
+
+	if (secboot == 0 || setupmode == 1)
+		return efi_secureboot_mode_disabled;
+
+	/* See if a user has put the shim into insecure mode. */
+	size = sizeof(moksbstate);
+	status = efi.get_variable(L"MokSBStateRT", &shim_guid,
+				  NULL, &size, &moksbstate);
+
+	/* If it fails, we don't care why. Default to secure. */
+	if (status != EFI_SUCCESS)
+		goto secure_boot_enabled;
+
+	if (moksbstate == 1)
+		return efi_secureboot_mode_disabled;
+
+ secure_boot_enabled:
+	pr_info("UEFI Secure Boot is enabled.\n");
+	return efi_secureboot_mode_enabled;
+
+ out_efi_err:
+	pr_err("Could not determine UEFI Secure Boot status.\n");
+	return efi_secureboot_mode_unknown;
+}
+
 void __init xen_efi_init(void)
 {
 	efi_system_table_t *efi_systab_xen;
@@ -129,6 +184,8 @@ void __init xen_efi_init(void)
 	boot_params.efi_info.efi_systab = (__u32)__pa(efi_systab_xen);
 	boot_params.efi_info.efi_systab_hi = (__u32)(__pa(efi_systab_xen) >> 32);
 
+	boot_params.secure_boot = xen_efi_get_secureboot();
+
 	set_bit(EFI_BOOT, &efi.flags);
 	set_bit(EFI_PARAVIRT, &efi.flags);
 	set_bit(EFI_64BIT, &efi.flags);
diff --git a/arch/x86/xen/xen-pvh.S b/arch/x86/xen/xen-pvh.S
index e1a5fbeae08d..ca2d3b2bf2af 100644
--- a/arch/x86/xen/xen-pvh.S
+++ b/arch/x86/xen/xen-pvh.S
@@ -54,12 +54,19 @@
  * charge of setting up it's own stack, GDT and IDT.
  */
 
+#define PVH_GDT_ENTRY_CS	1
+#define PVH_GDT_ENTRY_DS	2
+#define PVH_GDT_ENTRY_CANARY	3
+#define PVH_CS_SEL		(PVH_GDT_ENTRY_CS * 8)
+#define PVH_DS_SEL		(PVH_GDT_ENTRY_DS * 8)
+#define PVH_CANARY_SEL		(PVH_GDT_ENTRY_CANARY * 8)
+
 ENTRY(pvh_start_xen)
 	cld
 
 	lgdt (_pa(gdt))
 
-	mov $(__BOOT_DS),%eax
+	mov $PVH_DS_SEL,%eax
 	mov %eax,%ds
 	mov %eax,%es
 	mov %eax,%ss
@@ -93,11 +100,17 @@ ENTRY(pvh_start_xen)
 	mov %eax, %cr0
 
 	/* Jump to 64-bit mode. */
-	ljmp $__KERNEL_CS, $_pa(1f)
+	ljmp $PVH_CS_SEL, $_pa(1f)
 
 	/* 64-bit entry point. */
 	.code64
 1:
+	/* Set base address in stack canary descriptor. */
+	mov $MSR_GS_BASE,%ecx
+	mov $_pa(canary), %eax
+	xor %edx, %edx
+	wrmsr
+
 	call xen_prepare_pvh
 
 	/* startup_64 expects boot_params in %rsi. */
@@ -107,6 +120,17 @@ ENTRY(pvh_start_xen)
 
 #else /* CONFIG_X86_64 */
 
+	/* Set base address in stack canary descriptor. */
+	movl $_pa(gdt_start),%eax
+	movl $_pa(canary),%ecx
+	movw %cx, (PVH_GDT_ENTRY_CANARY * 8) + 2(%eax)
+	shrl $16, %ecx
+	movb %cl, (PVH_GDT_ENTRY_CANARY * 8) + 4(%eax)
+	movb %ch, (PVH_GDT_ENTRY_CANARY * 8) + 7(%eax)
+
+	mov $PVH_CANARY_SEL,%eax
+	mov %eax,%gs
+
 	call mk_early_pgtbl_32
 
 	mov $_pa(initial_page_table), %eax
@@ -116,13 +140,13 @@ ENTRY(pvh_start_xen)
 	or $(X86_CR0_PG | X86_CR0_PE), %eax
 	mov %eax, %cr0
 
-	ljmp $__BOOT_CS, $1f
+	ljmp $PVH_CS_SEL, $1f
 1:
 	call xen_prepare_pvh
 	mov $_pa(pvh_bootparams), %esi
 
 	/* startup_32 doesn't expect paging and PAE to be on. */
-	ljmp $__BOOT_CS, $_pa(2f)
+	ljmp $PVH_CS_SEL, $_pa(2f)
 2:
 	mov %cr0, %eax
 	and $~X86_CR0_PG, %eax
@@ -131,7 +155,7 @@ ENTRY(pvh_start_xen)
 	and $~X86_CR4_PAE, %eax
 	mov %eax, %cr4
 
-	ljmp $__BOOT_CS, $_pa(startup_32)
+	ljmp $PVH_CS_SEL, $_pa(startup_32)
 #endif
 END(pvh_start_xen)
 
@@ -143,16 +167,19 @@ gdt:
 	.word 0
 gdt_start:
 	.quad 0x0000000000000000            /* NULL descriptor */
-	.quad 0x0000000000000000            /* reserved */
 #ifdef CONFIG_X86_64
-	.quad GDT_ENTRY(0xa09a, 0, 0xfffff) /* __KERNEL_CS */
+	.quad GDT_ENTRY(0xa09a, 0, 0xfffff) /* PVH_CS_SEL */
 #else
-	.quad GDT_ENTRY(0xc09a, 0, 0xfffff) /* __KERNEL_CS */
+	.quad GDT_ENTRY(0xc09a, 0, 0xfffff) /* PVH_CS_SEL */
 #endif
-	.quad GDT_ENTRY(0xc092, 0, 0xfffff) /* __KERNEL_DS */
+	.quad GDT_ENTRY(0xc092, 0, 0xfffff) /* PVH_DS_SEL */
+	.quad GDT_ENTRY(0x4090, 0, 0x18)    /* PVH_CANARY_SEL */
 gdt_end:
 
-	.balign 4
+	.balign 16
+canary:
+	.fill 48, 1, 0
+
 early_stack:
 	.fill 256, 1, 0
 early_stack_end:
diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig
index c921e8bccdc8..17df332269b2 100644
--- a/arch/xtensa/Kconfig
+++ b/arch/xtensa/Kconfig
@@ -19,7 +19,6 @@ config XTENSA
 	select HAVE_ARCH_KASAN if MMU
 	select HAVE_CC_STACKPROTECTOR
 	select HAVE_DEBUG_KMEMLEAK
-	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_CONTIGUOUS
 	select HAVE_EXIT_THREAD
 	select HAVE_FUNCTION_TRACER
diff --git a/arch/xtensa/include/asm/Kbuild b/arch/xtensa/include/asm/Kbuild
index 436b20337168..e5e1e61c538c 100644
--- a/arch/xtensa/include/asm/Kbuild
+++ b/arch/xtensa/include/asm/Kbuild
@@ -1,4 +1,5 @@
 generic-y += bug.h
+generic-y += compat.h
 generic-y += device.h
 generic-y += div64.h
 generic-y += dma-contiguous.h
diff --git a/arch/xtensa/include/asm/pci.h b/arch/xtensa/include/asm/pci.h
index d5a82153a7c5..883024054b05 100644
--- a/arch/xtensa/include/asm/pci.h
+++ b/arch/xtensa/include/asm/pci.h
@@ -20,8 +20,6 @@
 
 #define pcibios_assign_all_busses()	0
 
-extern struct pci_controller* pcibios_alloc_controller(void);
-
 /* Assume some values. (We should revise them, if necessary) */
 
 #define PCIBIOS_MIN_IO		0x2000
@@ -42,8 +40,6 @@ extern struct pci_controller* pcibios_alloc_controller(void);
  * decisions.
  */
 
-#define PCI_DMA_BUS_IS_PHYS	(1)
-
 /* Tell PCI code what kind of PCI resource mappings we support */
 #define HAVE_PCI_MMAP			1
 #define ARCH_GENERIC_PCI_MMAP_RESOURCE	1
diff --git a/arch/xtensa/include/uapi/asm/msgbuf.h b/arch/xtensa/include/uapi/asm/msgbuf.h
index 36e2e103ca38..d6915e9f071c 100644
--- a/arch/xtensa/include/uapi/asm/msgbuf.h
+++ b/arch/xtensa/include/uapi/asm/msgbuf.h
@@ -7,7 +7,6 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
  * - 2 miscellaneous 32-bit values
  *
  * This file is subject to the terms and conditions of the GNU General
@@ -21,19 +20,19 @@
 struct msqid64_ds {
 	struct ipc64_perm msg_perm;
 #ifdef __XTENSA_EB__
-	unsigned int	__unused1;
-	__kernel_time_t msg_stime;	/* last msgsnd time */
-	unsigned int	__unused2;
-	__kernel_time_t msg_rtime;	/* last msgrcv time */
-	unsigned int	__unused3;
-	__kernel_time_t msg_ctime;	/* last change time */
+	unsigned long  msg_stime_high;
+	unsigned long  msg_stime;	/* last msgsnd time */
+	unsigned long  msg_rtime_high;
+	unsigned long  msg_rtime;	/* last msgrcv time */
+	unsigned long  msg_ctime_high;
+	unsigned long  msg_ctime;	/* last change time */
 #elif defined(__XTENSA_EL__)
-	__kernel_time_t msg_stime;	/* last msgsnd time */
-	unsigned int	__unused1;
-	__kernel_time_t msg_rtime;	/* last msgrcv time */
-	unsigned int	__unused2;
-	__kernel_time_t msg_ctime;	/* last change time */
-	unsigned int	__unused3;
+	unsigned long  msg_stime;	/* last msgsnd time */
+	unsigned long  msg_stime_high;
+	unsigned long  msg_rtime;	/* last msgrcv time */
+	unsigned long  msg_rtime_high;
+	unsigned long  msg_ctime;	/* last change time */
+	unsigned long  msg_ctime_high;
 #else
 # error processor byte order undefined!
 #endif
diff --git a/arch/xtensa/include/uapi/asm/sembuf.h b/arch/xtensa/include/uapi/asm/sembuf.h
index f61b6331a10c..09f348d643f1 100644
--- a/arch/xtensa/include/uapi/asm/sembuf.h
+++ b/arch/xtensa/include/uapi/asm/sembuf.h
@@ -14,7 +14,6 @@
  * between kernel and user space.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
  * - 2 miscellaneous 32-bit values
  *
  */
@@ -27,15 +26,15 @@
 struct semid64_ds {
 	struct ipc64_perm sem_perm;		/* permissions .. see ipc.h */
 #ifdef __XTENSA_EL__
-	__kernel_time_t	sem_otime;		/* last semop time */
-	unsigned long	__unused1;
-	__kernel_time_t	sem_ctime;		/* last change time */
-	unsigned long	__unused2;
+	unsigned long	sem_otime;		/* last semop time */
+	unsigned long	sem_otime_high;
+	unsigned long	sem_ctime;		/* last change time */
+	unsigned long	sem_ctime_high;
 #else
-	unsigned long	__unused1;
-	__kernel_time_t	sem_otime;		/* last semop time */
-	unsigned long	__unused2;
-	__kernel_time_t	sem_ctime;		/* last change time */
+	unsigned long	sem_otime_high;
+	unsigned long	sem_otime;		/* last semop time */
+	unsigned long	sem_ctime_high;
+	unsigned long	sem_ctime;		/* last change time */
 #endif
 	unsigned long	sem_nsems;		/* no. of semaphores in array */
 	unsigned long	__unused3;
diff --git a/arch/xtensa/include/uapi/asm/shmbuf.h b/arch/xtensa/include/uapi/asm/shmbuf.h
index 26550bdc8430..554a57a6a90f 100644
--- a/arch/xtensa/include/uapi/asm/shmbuf.h
+++ b/arch/xtensa/include/uapi/asm/shmbuf.h
@@ -4,10 +4,10 @@
  *
  * The shmid64_ds structure for Xtensa architecture.
  * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
+ * between kernel and user space, but the padding is on the wrong
+ * side for big-endian xtensa, for historic reasons.
  *
  * Pad space is left for:
- * - 64-bit time_t to solve y2038 problem
  * - 2 miscellaneous 32-bit values
  *
  * This file is subject to the terms and conditions of the GNU General Public
@@ -20,42 +20,21 @@
 #ifndef _XTENSA_SHMBUF_H
 #define _XTENSA_SHMBUF_H
 
-#if defined (__XTENSA_EL__)
 struct shmid64_ds {
 	struct ipc64_perm	shm_perm;	/* operation perms */
 	size_t			shm_segsz;	/* size of segment (bytes) */
-	__kernel_time_t		shm_atime;	/* last attach time */
-	unsigned long		__unused1;
-	__kernel_time_t		shm_dtime;	/* last detach time */
-	unsigned long		__unused2;
-	__kernel_time_t		shm_ctime;	/* last change time */
-	unsigned long		__unused3;
+	unsigned long		shm_atime;	/* last attach time */
+	unsigned long		shm_atime_high;
+	unsigned long		shm_dtime;	/* last detach time */
+	unsigned long		shm_dtime_high;
+	unsigned long		shm_ctime;	/* last change time */
+	unsigned long		shm_ctime_high;
 	__kernel_pid_t		shm_cpid;	/* pid of creator */
 	__kernel_pid_t		shm_lpid;	/* pid of last operator */
 	unsigned long		shm_nattch;	/* no. of current attaches */
 	unsigned long		__unused4;
 	unsigned long		__unused5;
 };
-#elif defined (__XTENSA_EB__)
-struct shmid64_ds {
-	struct ipc64_perm	shm_perm;	/* operation perms */
-	size_t			shm_segsz;	/* size of segment (bytes) */
-	__kernel_time_t		shm_atime;	/* last attach time */
-	unsigned long		__unused1;
-	__kernel_time_t		shm_dtime;	/* last detach time */
-	unsigned long		__unused2;
-	__kernel_time_t		shm_ctime;	/* last change time */
-	unsigned long		__unused3;
-	__kernel_pid_t		shm_cpid;	/* pid of creator */
-	__kernel_pid_t		shm_lpid;	/* pid of last operator */
-	unsigned long		shm_nattch;	/* no. of current attaches */
-	unsigned long		__unused4;
-	unsigned long		__unused5;
-};
-#else
-# error endian order not defined
-#endif
-
 
 struct shminfo64 {
 	unsigned long	shmmax;
diff --git a/arch/xtensa/kernel/pci-dma.c b/arch/xtensa/kernel/pci-dma.c
index 732631ce250f..ba4640cc0093 100644
--- a/arch/xtensa/kernel/pci-dma.c
+++ b/arch/xtensa/kernel/pci-dma.c
@@ -147,6 +147,10 @@ static void *xtensa_dma_alloc(struct device *dev, size_t size,
 
 	*handle = phys_to_dma(dev, page_to_phys(page));
 
+	if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) {
+		return page;
+	}
+
 #ifdef CONFIG_MMU
 	if (PageHighMem(page)) {
 		void *p;
@@ -178,8 +182,10 @@ static void xtensa_dma_free(struct device *dev, size_t size, void *vaddr,
 	unsigned long addr = (unsigned long)vaddr;
 	struct page *page;
 
-	if (addr >= XCHAL_KSEG_BYPASS_VADDR &&
-	    addr - XCHAL_KSEG_BYPASS_VADDR < XCHAL_KSEG_SIZE) {
+	if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) {
+		page = vaddr;
+	} else if (addr >= XCHAL_KSEG_BYPASS_VADDR &&
+		   addr - XCHAL_KSEG_BYPASS_VADDR < XCHAL_KSEG_SIZE) {
 		addr += XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR;
 		page = virt_to_page(addr);
 	} else {
@@ -261,12 +267,3 @@ const struct dma_map_ops xtensa_dma_map_ops = {
 	.mapping_error = xtensa_dma_mapping_error,
 };
 EXPORT_SYMBOL(xtensa_dma_map_ops);
-
-#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
-
-static int __init xtensa_dma_init(void)
-{
-	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
-	return 0;
-}
-fs_initcall(xtensa_dma_init);
diff --git a/arch/xtensa/kernel/pci.c b/arch/xtensa/kernel/pci.c
index b7c7a60c7000..21f13e9aabe1 100644
--- a/arch/xtensa/kernel/pci.c
+++ b/arch/xtensa/kernel/pci.c
@@ -41,8 +41,8 @@
  * pci_bus_add_device
  */
 
-struct pci_controller* pci_ctrl_head;
-struct pci_controller** pci_ctrl_tail = &pci_ctrl_head;
+static struct pci_controller *pci_ctrl_head;
+static struct pci_controller **pci_ctrl_tail = &pci_ctrl_head;
 
 static int pci_bus_count;
 
@@ -80,50 +80,6 @@ pcibios_align_resource(void *data, const struct resource *res,
 	return start;
 }
 
-int
-pcibios_enable_resources(struct pci_dev *dev, int mask)
-{
-	u16 cmd, old_cmd;
-	int idx;
-	struct resource *r;
-
-	pci_read_config_word(dev, PCI_COMMAND, &cmd);
-	old_cmd = cmd;
-	for(idx=0; idx<6; idx++) {
-		r = &dev->resource[idx];
-		if (!r->start && r->end) {
-			pr_err("PCI: Device %s not available because "
-			       "of resource collisions\n", pci_name(dev));
-			return -EINVAL;
-		}
-		if (r->flags & IORESOURCE_IO)
-			cmd |= PCI_COMMAND_IO;
-		if (r->flags & IORESOURCE_MEM)
-			cmd |= PCI_COMMAND_MEMORY;
-	}
-	if (dev->resource[PCI_ROM_RESOURCE].start)
-		cmd |= PCI_COMMAND_MEMORY;
-	if (cmd != old_cmd) {
-		pr_info("PCI: Enabling device %s (%04x -> %04x)\n",
-			pci_name(dev), old_cmd, cmd);
-		pci_write_config_word(dev, PCI_COMMAND, cmd);
-	}
-	return 0;
-}
-
-struct pci_controller * __init pcibios_alloc_controller(void)
-{
-	struct pci_controller *pci_ctrl;
-
-	pci_ctrl = (struct pci_controller *)alloc_bootmem(sizeof(*pci_ctrl));
-	memset(pci_ctrl, 0, sizeof(struct pci_controller));
-
-	*pci_ctrl_tail = pci_ctrl;
-	pci_ctrl_tail = &pci_ctrl->next;
-
-	return pci_ctrl;
-}
-
 static void __init pci_controller_apertures(struct pci_controller *pci_ctrl,
 					    struct list_head *resources)
 {
@@ -223,8 +179,7 @@ int pcibios_enable_device(struct pci_dev *dev, int mask)
 	for (idx=0; idx<6; idx++) {
 		r = &dev->resource[idx];
 		if (!r->start && r->end) {
-			pr_err("PCI: Device %s not available because "
-			       "of resource collisions\n", pci_name(dev));
+			pci_err(dev, "can't enable device: resource collisions\n");
 			return -EINVAL;
 		}
 		if (r->flags & IORESOURCE_IO)
@@ -233,29 +188,13 @@ int pcibios_enable_device(struct pci_dev *dev, int mask)
 			cmd |= PCI_COMMAND_MEMORY;
 	}
 	if (cmd != old_cmd) {
-		pr_info("PCI: Enabling device %s (%04x -> %04x)\n",
-			pci_name(dev), old_cmd, cmd);
+		pci_info(dev, "enabling device (%04x -> %04x)\n", old_cmd, cmd);
 		pci_write_config_word(dev, PCI_COMMAND, cmd);
 	}
 
 	return 0;
 }
 
-#ifdef CONFIG_PROC_FS
-
-/*
- * Return the index of the PCI controller for device pdev.
- */
-
-int
-pci_controller_num(struct pci_dev *dev)
-{
-	struct pci_controller *pci_ctrl = (struct pci_controller*) dev->sysdata;
-	return pci_ctrl->index;
-}
-
-#endif /* CONFIG_PROC_FS */
-
 /*
  * Platform support for /proc/bus/pci/X/Y mmap()s.
  *  -- paulus.
diff --git a/arch/xtensa/kernel/traps.c b/arch/xtensa/kernel/traps.c
index 32c5207f1226..86507fa7c2d7 100644
--- a/arch/xtensa/kernel/traps.c
+++ b/arch/xtensa/kernel/traps.c
@@ -323,8 +323,6 @@ do_illegal_instruction(struct pt_regs *regs)
 void
 do_unaligned_user (struct pt_regs *regs)
 {
-	siginfo_t info;
-
 	__die_if_kernel("Unhandled unaligned exception in kernel",
 			regs, SIGKILL);
 
@@ -334,12 +332,7 @@ do_unaligned_user (struct pt_regs *regs)
 			    "(pid = %d, pc = %#010lx)\n",
 			    regs->excvaddr, current->comm,
 			    task_pid_nr(current), regs->pc);
-	info.si_signo = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRALN;
-	info.si_addr = (void *) regs->excvaddr;
-	force_sig_info(SIGSEGV, &info, current);
-
+	force_sig_fault(SIGBUS, BUS_ADRALN, (void *) regs->excvaddr, current);
 }
 #endif
 
diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c
index 8b9b6f44bb06..c111a833205a 100644
--- a/arch/xtensa/mm/fault.c
+++ b/arch/xtensa/mm/fault.c
@@ -39,13 +39,13 @@ void do_page_fault(struct pt_regs *regs)
 	struct mm_struct *mm = current->mm;
 	unsigned int exccause = regs->exccause;
 	unsigned int address = regs->excvaddr;
-	siginfo_t info;
+	int code;
 
 	int is_write, is_exec;
 	int fault;
 	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 
-	info.si_code = SEGV_MAPERR;
+	code = SEGV_MAPERR;
 
 	/* We fault-in kernel-space virtual memory on-demand. The
 	 * 'reference' page table is init_mm.pgd.
@@ -91,7 +91,7 @@ void do_page_fault(struct pt_regs *regs)
 	 */
 
 good_area:
-	info.si_code = SEGV_ACCERR;
+	code = SEGV_ACCERR;
 
 	if (is_write) {
 		if (!(vma->vm_flags & VM_WRITE))
@@ -157,11 +157,7 @@ void do_page_fault(struct pt_regs *regs)
 	if (user_mode(regs)) {
 		current->thread.bad_vaddr = address;
 		current->thread.error_code = is_write;
-		info.si_signo = SIGSEGV;
-		info.si_errno = 0;
-		/* info.si_code has been set above */
-		info.si_addr = (void *) address;
-		force_sig_info(SIGSEGV, &info, current);
+		force_sig_fault(SIGSEGV, code, (void *) address, current);
 		return;
 	}
 	bad_page_fault(regs, address, SIGSEGV);
@@ -186,11 +182,7 @@ void do_page_fault(struct pt_regs *regs)
 	 * or user mode.
 	 */
 	current->thread.bad_vaddr = address;
-	info.si_code = SIGBUS;
-	info.si_errno = 0;
-	info.si_code = BUS_ADRERR;
-	info.si_addr = (void *) address;
-	force_sig_info(SIGBUS, &info, current);
+	force_sig_fault(SIGBUS, BUS_ADRERR, (void *) address, current);
 
 	/* Kernel mode? Handle exceptions or die */
 	if (!user_mode(regs))
diff --git a/arch/xtensa/platforms/iss/console.c b/arch/xtensa/platforms/iss/console.c
index 92f567f9a21e..af81a62faba6 100644
--- a/arch/xtensa/platforms/iss/console.c
+++ b/arch/xtensa/platforms/iss/console.c
@@ -153,19 +153,6 @@ static int rs_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int rs_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, rs_proc_show, NULL);
-}
-
-static const struct file_operations rs_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= rs_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static const struct tty_operations serial_ops = {
 	.open = rs_open,
 	.close = rs_close,
@@ -176,7 +163,7 @@ static const struct tty_operations serial_ops = {
 	.chars_in_buffer = rs_chars_in_buffer,
 	.hangup = rs_hangup,
 	.wait_until_sent = rs_wait_until_sent,
-	.proc_fops = &rs_proc_fops,
+	.proc_show = rs_proc_show,
 };
 
 int __init rs_init(void)
diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
index d819dc77fe65..a9e8633388f4 100644
--- a/block/bfq-cgroup.c
+++ b/block/bfq-cgroup.c
@@ -55,13 +55,13 @@ BFQG_FLAG_FNS(empty)
 /* This should be called with the scheduler lock held. */
 static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats)
 {
-	unsigned long long now;
+	u64 now;
 
 	if (!bfqg_stats_waiting(stats))
 		return;
 
-	now = sched_clock();
-	if (time_after64(now, stats->start_group_wait_time))
+	now = ktime_get_ns();
+	if (now > stats->start_group_wait_time)
 		blkg_stat_add(&stats->group_wait_time,
 			      now - stats->start_group_wait_time);
 	bfqg_stats_clear_waiting(stats);
@@ -77,20 +77,20 @@ static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,
 		return;
 	if (bfqg == curr_bfqg)
 		return;
-	stats->start_group_wait_time = sched_clock();
+	stats->start_group_wait_time = ktime_get_ns();
 	bfqg_stats_mark_waiting(stats);
 }
 
 /* This should be called with the scheduler lock held. */
 static void bfqg_stats_end_empty_time(struct bfqg_stats *stats)
 {
-	unsigned long long now;
+	u64 now;
 
 	if (!bfqg_stats_empty(stats))
 		return;
 
-	now = sched_clock();
-	if (time_after64(now, stats->start_empty_time))
+	now = ktime_get_ns();
+	if (now > stats->start_empty_time)
 		blkg_stat_add(&stats->empty_time,
 			      now - stats->start_empty_time);
 	bfqg_stats_clear_empty(stats);
@@ -116,7 +116,7 @@ void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)
 	if (bfqg_stats_empty(stats))
 		return;
 
-	stats->start_empty_time = sched_clock();
+	stats->start_empty_time = ktime_get_ns();
 	bfqg_stats_mark_empty(stats);
 }
 
@@ -125,9 +125,9 @@ void bfqg_stats_update_idle_time(struct bfq_group *bfqg)
 	struct bfqg_stats *stats = &bfqg->stats;
 
 	if (bfqg_stats_idling(stats)) {
-		unsigned long long now = sched_clock();
+		u64 now = ktime_get_ns();
 
-		if (time_after64(now, stats->start_idle_time))
+		if (now > stats->start_idle_time)
 			blkg_stat_add(&stats->idle_time,
 				      now - stats->start_idle_time);
 		bfqg_stats_clear_idling(stats);
@@ -138,7 +138,7 @@ void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)
 {
 	struct bfqg_stats *stats = &bfqg->stats;
 
-	stats->start_idle_time = sched_clock();
+	stats->start_idle_time = ktime_get_ns();
 	bfqg_stats_mark_idling(stats);
 }
 
@@ -171,18 +171,18 @@ void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op)
 	blkg_rwstat_add(&bfqg->stats.merged, op, 1);
 }
 
-void bfqg_stats_update_completion(struct bfq_group *bfqg, uint64_t start_time,
-				  uint64_t io_start_time, unsigned int op)
+void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
+				  u64 io_start_time_ns, unsigned int op)
 {
 	struct bfqg_stats *stats = &bfqg->stats;
-	unsigned long long now = sched_clock();
+	u64 now = ktime_get_ns();
 
-	if (time_after64(now, io_start_time))
+	if (now > io_start_time_ns)
 		blkg_rwstat_add(&stats->service_time, op,
-				now - io_start_time);
-	if (time_after64(io_start_time, start_time))
+				now - io_start_time_ns);
+	if (io_start_time_ns > start_time_ns)
 		blkg_rwstat_add(&stats->wait_time, op,
-				io_start_time - start_time);
+				io_start_time_ns - start_time_ns);
 }
 
 #else /* CONFIG_BFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
@@ -191,8 +191,8 @@ void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
 			      unsigned int op) { }
 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op) { }
 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op) { }
-void bfqg_stats_update_completion(struct bfq_group *bfqg, uint64_t start_time,
-				  uint64_t io_start_time, unsigned int op) { }
+void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
+				  u64 io_start_time_ns, unsigned int op) { }
 void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }
 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { }
 void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { }
diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index 771ae9730ac6..b622e73a326a 100644
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -486,46 +486,6 @@ static struct request *bfq_choose_req(struct bfq_data *bfqd,
 	}
 }
 
-/*
- * See the comments on bfq_limit_depth for the purpose of
- * the depths set in the function.
- */
-static void bfq_update_depths(struct bfq_data *bfqd, struct sbitmap_queue *bt)
-{
-	bfqd->sb_shift = bt->sb.shift;
-
-	/*
-	 * In-word depths if no bfq_queue is being weight-raised:
-	 * leaving 25% of tags only for sync reads.
-	 *
-	 * In next formulas, right-shift the value
-	 * (1U<<bfqd->sb_shift), instead of computing directly
-	 * (1U<<(bfqd->sb_shift - something)), to be robust against
-	 * any possible value of bfqd->sb_shift, without having to
-	 * limit 'something'.
-	 */
-	/* no more than 50% of tags for async I/O */
-	bfqd->word_depths[0][0] = max((1U<<bfqd->sb_shift)>>1, 1U);
-	/*
-	 * no more than 75% of tags for sync writes (25% extra tags
-	 * w.r.t. async I/O, to prevent async I/O from starving sync
-	 * writes)
-	 */
-	bfqd->word_depths[0][1] = max(((1U<<bfqd->sb_shift) * 3)>>2, 1U);
-
-	/*
-	 * In-word depths in case some bfq_queue is being weight-
-	 * raised: leaving ~63% of tags for sync reads. This is the
-	 * highest percentage for which, in our tests, application
-	 * start-up times didn't suffer from any regression due to tag
-	 * shortage.
-	 */
-	/* no more than ~18% of tags for async I/O */
-	bfqd->word_depths[1][0] = max(((1U<<bfqd->sb_shift) * 3)>>4, 1U);
-	/* no more than ~37% of tags for sync writes (~20% extra tags) */
-	bfqd->word_depths[1][1] = max(((1U<<bfqd->sb_shift) * 6)>>4, 1U);
-}
-
 /*
  * Async I/O can easily starve sync I/O (both sync reads and sync
  * writes), by consuming all tags. Similarly, storms of sync writes,
@@ -535,25 +495,11 @@ static void bfq_update_depths(struct bfq_data *bfqd, struct sbitmap_queue *bt)
  */
 static void bfq_limit_depth(unsigned int op, struct blk_mq_alloc_data *data)
 {
-	struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
 	struct bfq_data *bfqd = data->q->elevator->elevator_data;
-	struct sbitmap_queue *bt;
 
 	if (op_is_sync(op) && !op_is_write(op))
 		return;
 
-	if (data->flags & BLK_MQ_REQ_RESERVED) {
-		if (unlikely(!tags->nr_reserved_tags)) {
-			WARN_ON_ONCE(1);
-			return;
-		}
-		bt = &tags->breserved_tags;
-	} else
-		bt = &tags->bitmap_tags;
-
-	if (unlikely(bfqd->sb_shift != bt->sb.shift))
-		bfq_update_depths(bfqd, bt);
-
 	data->shallow_depth =
 		bfqd->word_depths[!!bfqd->wr_busy_queues][op_is_sync(op)];
 
@@ -1858,6 +1804,8 @@ static int bfq_request_merge(struct request_queue *q, struct request **req,
 	return ELEVATOR_NO_MERGE;
 }
 
+static struct bfq_queue *bfq_init_rq(struct request *rq);
+
 static void bfq_request_merged(struct request_queue *q, struct request *req,
 			       enum elv_merge type)
 {
@@ -1866,7 +1814,7 @@ static void bfq_request_merged(struct request_queue *q, struct request *req,
 	    blk_rq_pos(req) <
 	    blk_rq_pos(container_of(rb_prev(&req->rb_node),
 				    struct request, rb_node))) {
-		struct bfq_queue *bfqq = RQ_BFQQ(req);
+		struct bfq_queue *bfqq = bfq_init_rq(req);
 		struct bfq_data *bfqd = bfqq->bfqd;
 		struct request *prev, *next_rq;
 
@@ -1894,7 +1842,8 @@ static void bfq_request_merged(struct request_queue *q, struct request *req,
 static void bfq_requests_merged(struct request_queue *q, struct request *rq,
 				struct request *next)
 {
-	struct bfq_queue *bfqq = RQ_BFQQ(rq), *next_bfqq = RQ_BFQQ(next);
+	struct bfq_queue *bfqq = bfq_init_rq(rq),
+		*next_bfqq = bfq_init_rq(next);
 
 	if (!RB_EMPTY_NODE(&rq->rb_node))
 		goto end;
@@ -4540,14 +4489,12 @@ static inline void bfq_update_insert_stats(struct request_queue *q,
 					   unsigned int cmd_flags) {}
 #endif
 
-static void bfq_prepare_request(struct request *rq, struct bio *bio);
-
 static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
 			       bool at_head)
 {
 	struct request_queue *q = hctx->queue;
 	struct bfq_data *bfqd = q->elevator->elevator_data;
-	struct bfq_queue *bfqq = RQ_BFQQ(rq);
+	struct bfq_queue *bfqq;
 	bool idle_timer_disabled = false;
 	unsigned int cmd_flags;
 
@@ -4562,24 +4509,13 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
 	blk_mq_sched_request_inserted(rq);
 
 	spin_lock_irq(&bfqd->lock);
+	bfqq = bfq_init_rq(rq);
 	if (at_head || blk_rq_is_passthrough(rq)) {
 		if (at_head)
 			list_add(&rq->queuelist, &bfqd->dispatch);
 		else
 			list_add_tail(&rq->queuelist, &bfqd->dispatch);
-	} else {
-		if (WARN_ON_ONCE(!bfqq)) {
-			/*
-			 * This should never happen. Most likely rq is
-			 * a requeued regular request, being
-			 * re-inserted without being first
-			 * re-prepared. Do a prepare, to avoid
-			 * failure.
-			 */
-			bfq_prepare_request(rq, rq->bio);
-			bfqq = RQ_BFQQ(rq);
-		}
-
+	} else { /* bfqq is assumed to be non null here */
 		idle_timer_disabled = __bfq_insert_request(bfqd, rq);
 		/*
 		 * Update bfqq, because, if a queue merge has occurred
@@ -4778,8 +4714,8 @@ static void bfq_finish_requeue_request(struct request *rq)
 
 	if (rq->rq_flags & RQF_STARTED)
 		bfqg_stats_update_completion(bfqq_group(bfqq),
-					     rq_start_time_ns(rq),
-					     rq_io_start_time_ns(rq),
+					     rq->start_time_ns,
+					     rq->io_start_time_ns,
 					     rq->cmd_flags);
 
 	if (likely(rq->rq_flags & RQF_STARTED)) {
@@ -4922,11 +4858,48 @@ static struct bfq_queue *bfq_get_bfqq_handle_split(struct bfq_data *bfqd,
 }
 
 /*
- * Allocate bfq data structures associated with this request.
+ * Only reset private fields. The actual request preparation will be
+ * performed by bfq_init_rq, when rq is either inserted or merged. See
+ * comments on bfq_init_rq for the reason behind this delayed
+ * preparation.
  */
 static void bfq_prepare_request(struct request *rq, struct bio *bio)
+{
+	/*
+	 * Regardless of whether we have an icq attached, we have to
+	 * clear the scheduler pointers, as they might point to
+	 * previously allocated bic/bfqq structs.
+	 */
+	rq->elv.priv[0] = rq->elv.priv[1] = NULL;
+}
+
+/*
+ * If needed, init rq, allocate bfq data structures associated with
+ * rq, and increment reference counters in the destination bfq_queue
+ * for rq. Return the destination bfq_queue for rq, or NULL is rq is
+ * not associated with any bfq_queue.
+ *
+ * This function is invoked by the functions that perform rq insertion
+ * or merging. One may have expected the above preparation operations
+ * to be performed in bfq_prepare_request, and not delayed to when rq
+ * is inserted or merged. The rationale behind this delayed
+ * preparation is that, after the prepare_request hook is invoked for
+ * rq, rq may still be transformed into a request with no icq, i.e., a
+ * request not associated with any queue. No bfq hook is invoked to
+ * signal this tranformation. As a consequence, should these
+ * preparation operations be performed when the prepare_request hook
+ * is invoked, and should rq be transformed one moment later, bfq
+ * would end up in an inconsistent state, because it would have
+ * incremented some queue counters for an rq destined to
+ * transformation, without any chance to correctly lower these
+ * counters back. In contrast, no transformation can still happen for
+ * rq after rq has been inserted or merged. So, it is safe to execute
+ * these preparation operations when rq is finally inserted or merged.
+ */
+static struct bfq_queue *bfq_init_rq(struct request *rq)
 {
 	struct request_queue *q = rq->q;
+	struct bio *bio = rq->bio;
 	struct bfq_data *bfqd = q->elevator->elevator_data;
 	struct bfq_io_cq *bic;
 	const int is_sync = rq_is_sync(rq);
@@ -4934,20 +4907,21 @@ static void bfq_prepare_request(struct request *rq, struct bio *bio)
 	bool new_queue = false;
 	bool bfqq_already_existing = false, split = false;
 
+	if (unlikely(!rq->elv.icq))
+		return NULL;
+
 	/*
-	 * Even if we don't have an icq attached, we should still clear
-	 * the scheduler pointers, as they might point to previously
-	 * allocated bic/bfqq structs.
+	 * Assuming that elv.priv[1] is set only if everything is set
+	 * for this rq. This holds true, because this function is
+	 * invoked only for insertion or merging, and, after such
+	 * events, a request cannot be manipulated any longer before
+	 * being removed from bfq.
 	 */
-	if (!rq->elv.icq) {
-		rq->elv.priv[0] = rq->elv.priv[1] = NULL;
-		return;
-	}
+	if (rq->elv.priv[1])
+		return rq->elv.priv[1];
 
 	bic = icq_to_bic(rq->elv.icq);
 
-	spin_lock_irq(&bfqd->lock);
-
 	bfq_check_ioprio_change(bic, bio);
 
 	bfq_bic_update_cgroup(bic, bio);
@@ -5006,7 +4980,7 @@ static void bfq_prepare_request(struct request *rq, struct bio *bio)
 	if (unlikely(bfq_bfqq_just_created(bfqq)))
 		bfq_handle_burst(bfqd, bfqq);
 
-	spin_unlock_irq(&bfqd->lock);
+	return bfqq;
 }
 
 static void bfq_idle_slice_timer_body(struct bfq_queue *bfqq)
@@ -5105,6 +5079,64 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg)
 	__bfq_put_async_bfqq(bfqd, &bfqg->async_idle_bfqq);
 }
 
+/*
+ * See the comments on bfq_limit_depth for the purpose of
+ * the depths set in the function. Return minimum shallow depth we'll use.
+ */
+static unsigned int bfq_update_depths(struct bfq_data *bfqd,
+				      struct sbitmap_queue *bt)
+{
+	unsigned int i, j, min_shallow = UINT_MAX;
+
+	/*
+	 * In-word depths if no bfq_queue is being weight-raised:
+	 * leaving 25% of tags only for sync reads.
+	 *
+	 * In next formulas, right-shift the value
+	 * (1U<<bt->sb.shift), instead of computing directly
+	 * (1U<<(bt->sb.shift - something)), to be robust against
+	 * any possible value of bt->sb.shift, without having to
+	 * limit 'something'.
+	 */
+	/* no more than 50% of tags for async I/O */
+	bfqd->word_depths[0][0] = max((1U << bt->sb.shift) >> 1, 1U);
+	/*
+	 * no more than 75% of tags for sync writes (25% extra tags
+	 * w.r.t. async I/O, to prevent async I/O from starving sync
+	 * writes)
+	 */
+	bfqd->word_depths[0][1] = max(((1U << bt->sb.shift) * 3) >> 2, 1U);
+
+	/*
+	 * In-word depths in case some bfq_queue is being weight-
+	 * raised: leaving ~63% of tags for sync reads. This is the
+	 * highest percentage for which, in our tests, application
+	 * start-up times didn't suffer from any regression due to tag
+	 * shortage.
+	 */
+	/* no more than ~18% of tags for async I/O */
+	bfqd->word_depths[1][0] = max(((1U << bt->sb.shift) * 3) >> 4, 1U);
+	/* no more than ~37% of tags for sync writes (~20% extra tags) */
+	bfqd->word_depths[1][1] = max(((1U << bt->sb.shift) * 6) >> 4, 1U);
+
+	for (i = 0; i < 2; i++)
+		for (j = 0; j < 2; j++)
+			min_shallow = min(min_shallow, bfqd->word_depths[i][j]);
+
+	return min_shallow;
+}
+
+static int bfq_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int index)
+{
+	struct bfq_data *bfqd = hctx->queue->elevator->elevator_data;
+	struct blk_mq_tags *tags = hctx->sched_tags;
+	unsigned int min_shallow;
+
+	min_shallow = bfq_update_depths(bfqd, &tags->bitmap_tags);
+	sbitmap_queue_min_shallow_depth(&tags->bitmap_tags, min_shallow);
+	return 0;
+}
+
 static void bfq_exit_queue(struct elevator_queue *e)
 {
 	struct bfq_data *bfqd = e->elevator_data;
@@ -5526,6 +5558,7 @@ static struct elevator_type iosched_bfq_mq = {
 		.requests_merged	= bfq_requests_merged,
 		.request_merged		= bfq_request_merged,
 		.has_work		= bfq_has_work,
+		.init_hctx		= bfq_init_hctx,
 		.init_sched		= bfq_init_queue,
 		.exit_sched		= bfq_exit_queue,
 	},
diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
index ae2f3dadec44..faac509cb35e 100644
--- a/block/bfq-iosched.h
+++ b/block/bfq-iosched.h
@@ -635,12 +635,6 @@ struct bfq_data {
 	/* bfqq associated with the task issuing current bio for merging */
 	struct bfq_queue *bio_bfqq;
 
-	/*
-	 * Cached sbitmap shift, used to compute depth limits in
-	 * bfq_update_depths.
-	 */
-	unsigned int sb_shift;
-
 	/*
 	 * Depth limits used in bfq_limit_depth (see comments on the
 	 * function)
@@ -732,9 +726,9 @@ struct bfqg_stats {
 	/* total time with empty current active q with other requests queued */
 	struct blkg_stat		empty_time;
 	/* fields after this shouldn't be cleared on stat reset */
-	uint64_t			start_group_wait_time;
-	uint64_t			start_idle_time;
-	uint64_t			start_empty_time;
+	u64				start_group_wait_time;
+	u64				start_idle_time;
+	u64				start_empty_time;
 	uint16_t			flags;
 #endif	/* CONFIG_BFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
 };
@@ -856,8 +850,8 @@ void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
 			      unsigned int op);
 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op);
 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op);
-void bfqg_stats_update_completion(struct bfq_group *bfqg, uint64_t start_time,
-				  uint64_t io_start_time, unsigned int op);
+void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
+				  u64 io_start_time_ns, unsigned int op);
 void bfqg_stats_update_dequeue(struct bfq_group *bfqg);
 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg);
 void bfqg_stats_update_idle_time(struct bfq_group *bfqg);
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 9cfdd6c83b5b..add7c7c85335 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -56,12 +56,12 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
 	struct bio_set *bs = bio->bi_pool;
 	unsigned inline_vecs;
 
-	if (!bs || !bs->bio_integrity_pool) {
+	if (!bs || !mempool_initialized(&bs->bio_integrity_pool)) {
 		bip = kmalloc(sizeof(struct bio_integrity_payload) +
 			      sizeof(struct bio_vec) * nr_vecs, gfp_mask);
 		inline_vecs = nr_vecs;
 	} else {
-		bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask);
+		bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask);
 		inline_vecs = BIP_INLINE_VECS;
 	}
 
@@ -74,7 +74,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
 		unsigned long idx = 0;
 
 		bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
-					  bs->bvec_integrity_pool);
+					  &bs->bvec_integrity_pool);
 		if (!bip->bip_vec)
 			goto err;
 		bip->bip_max_vcnt = bvec_nr_vecs(idx);
@@ -90,7 +90,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
 
 	return bip;
 err:
-	mempool_free(bip, bs->bio_integrity_pool);
+	mempool_free(bip, &bs->bio_integrity_pool);
 	return ERR_PTR(-ENOMEM);
 }
 EXPORT_SYMBOL(bio_integrity_alloc);
@@ -111,10 +111,10 @@ static void bio_integrity_free(struct bio *bio)
 		kfree(page_address(bip->bip_vec->bv_page) +
 		      bip->bip_vec->bv_offset);
 
-	if (bs && bs->bio_integrity_pool) {
-		bvec_free(bs->bvec_integrity_pool, bip->bip_vec, bip->bip_slab);
+	if (bs && mempool_initialized(&bs->bio_integrity_pool)) {
+		bvec_free(&bs->bvec_integrity_pool, bip->bip_vec, bip->bip_slab);
 
-		mempool_free(bip, bs->bio_integrity_pool);
+		mempool_free(bip, &bs->bio_integrity_pool);
 	} else {
 		kfree(bip);
 	}
@@ -465,16 +465,15 @@ EXPORT_SYMBOL(bio_integrity_clone);
 
 int bioset_integrity_create(struct bio_set *bs, int pool_size)
 {
-	if (bs->bio_integrity_pool)
+	if (mempool_initialized(&bs->bio_integrity_pool))
 		return 0;
 
-	bs->bio_integrity_pool = mempool_create_slab_pool(pool_size, bip_slab);
-	if (!bs->bio_integrity_pool)
+	if (mempool_init_slab_pool(&bs->bio_integrity_pool,
+				   pool_size, bip_slab))
 		return -1;
 
-	bs->bvec_integrity_pool = biovec_create_pool(pool_size);
-	if (!bs->bvec_integrity_pool) {
-		mempool_destroy(bs->bio_integrity_pool);
+	if (biovec_init_pool(&bs->bvec_integrity_pool, pool_size)) {
+		mempool_exit(&bs->bio_integrity_pool);
 		return -1;
 	}
 
@@ -484,8 +483,8 @@ EXPORT_SYMBOL(bioset_integrity_create);
 
 void bioset_integrity_free(struct bio_set *bs)
 {
-	mempool_destroy(bs->bio_integrity_pool);
-	mempool_destroy(bs->bvec_integrity_pool);
+	mempool_exit(&bs->bio_integrity_pool);
+	mempool_exit(&bs->bvec_integrity_pool);
 }
 EXPORT_SYMBOL(bioset_integrity_free);
 
diff --git a/block/bio.c b/block/bio.c
index 53e0f0a1ed94..595663e0281a 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -53,7 +53,7 @@ static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
  * fs_bio_set is the bio_set containing bio and iovec memory pools used by
  * IO code that does not need private memory pools.
  */
-struct bio_set *fs_bio_set;
+struct bio_set fs_bio_set;
 EXPORT_SYMBOL(fs_bio_set);
 
 /*
@@ -254,7 +254,7 @@ static void bio_free(struct bio *bio)
 	bio_uninit(bio);
 
 	if (bs) {
-		bvec_free(bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
+		bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
 
 		/*
 		 * If we have front padding, adjust the bio pointer before freeing
@@ -262,7 +262,7 @@ static void bio_free(struct bio *bio)
 		p = bio;
 		p -= bs->front_pad;
 
-		mempool_free(p, bs->bio_pool);
+		mempool_free(p, &bs->bio_pool);
 	} else {
 		/* Bio was allocated by bio_kmalloc() */
 		kfree(bio);
@@ -454,7 +454,8 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
 		inline_vecs = nr_iovecs;
 	} else {
 		/* should not use nobvec bioset for nr_iovecs > 0 */
-		if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0))
+		if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&
+				 nr_iovecs > 0))
 			return NULL;
 		/*
 		 * generic_make_request() converts recursion to iteration; this
@@ -483,11 +484,11 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
 		    bs->rescue_workqueue)
 			gfp_mask &= ~__GFP_DIRECT_RECLAIM;
 
-		p = mempool_alloc(bs->bio_pool, gfp_mask);
+		p = mempool_alloc(&bs->bio_pool, gfp_mask);
 		if (!p && gfp_mask != saved_gfp) {
 			punt_bios_to_rescuer(bs);
 			gfp_mask = saved_gfp;
-			p = mempool_alloc(bs->bio_pool, gfp_mask);
+			p = mempool_alloc(&bs->bio_pool, gfp_mask);
 		}
 
 		front_pad = bs->front_pad;
@@ -503,11 +504,11 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
 	if (nr_iovecs > inline_vecs) {
 		unsigned long idx = 0;
 
-		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
+		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
 		if (!bvl && gfp_mask != saved_gfp) {
 			punt_bios_to_rescuer(bs);
 			gfp_mask = saved_gfp;
-			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
+			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
 		}
 
 		if (unlikely(!bvl))
@@ -524,25 +525,25 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
 	return bio;
 
 err_free:
-	mempool_free(p, bs->bio_pool);
+	mempool_free(p, &bs->bio_pool);
 	return NULL;
 }
 EXPORT_SYMBOL(bio_alloc_bioset);
 
-void zero_fill_bio(struct bio *bio)
+void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
 {
 	unsigned long flags;
 	struct bio_vec bv;
 	struct bvec_iter iter;
 
-	bio_for_each_segment(bv, bio, iter) {
+	__bio_for_each_segment(bv, bio, iter, start) {
 		char *data = bvec_kmap_irq(&bv, &flags);
 		memset(data, 0, bv.bv_len);
 		flush_dcache_page(bv.bv_page);
 		bvec_kunmap_irq(data, &flags);
 	}
 }
-EXPORT_SYMBOL(zero_fill_bio);
+EXPORT_SYMBOL(zero_fill_bio_iter);
 
 /**
  * bio_put - release a reference to a bio
@@ -970,27 +971,68 @@ void bio_advance(struct bio *bio, unsigned bytes)
 }
 EXPORT_SYMBOL(bio_advance);
 
+void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
+			struct bio *src, struct bvec_iter *src_iter)
+{
+	struct bio_vec src_bv, dst_bv;
+	void *src_p, *dst_p;
+	unsigned bytes;
+
+	while (src_iter->bi_size && dst_iter->bi_size) {
+		src_bv = bio_iter_iovec(src, *src_iter);
+		dst_bv = bio_iter_iovec(dst, *dst_iter);
+
+		bytes = min(src_bv.bv_len, dst_bv.bv_len);
+
+		src_p = kmap_atomic(src_bv.bv_page);
+		dst_p = kmap_atomic(dst_bv.bv_page);
+
+		memcpy(dst_p + dst_bv.bv_offset,
+		       src_p + src_bv.bv_offset,
+		       bytes);
+
+		kunmap_atomic(dst_p);
+		kunmap_atomic(src_p);
+
+		flush_dcache_page(dst_bv.bv_page);
+
+		bio_advance_iter(src, src_iter, bytes);
+		bio_advance_iter(dst, dst_iter, bytes);
+	}
+}
+EXPORT_SYMBOL(bio_copy_data_iter);
+
 /**
- * bio_copy_data - copy contents of data buffers from one chain of bios to
- * another
- * @src: source bio list
- * @dst: destination bio list
- *
- * If @src and @dst are single bios, bi_next must be NULL - otherwise, treats
- * @src and @dst as linked lists of bios.
+ * bio_copy_data - copy contents of data buffers from one bio to another
+ * @src: source bio
+ * @dst: destination bio
  *
  * Stops when it reaches the end of either @src or @dst - that is, copies
  * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
  */
 void bio_copy_data(struct bio *dst, struct bio *src)
 {
-	struct bvec_iter src_iter, dst_iter;
-	struct bio_vec src_bv, dst_bv;
-	void *src_p, *dst_p;
-	unsigned bytes;
+	struct bvec_iter src_iter = src->bi_iter;
+	struct bvec_iter dst_iter = dst->bi_iter;
 
-	src_iter = src->bi_iter;
-	dst_iter = dst->bi_iter;
+	bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
+}
+EXPORT_SYMBOL(bio_copy_data);
+
+/**
+ * bio_list_copy_data - copy contents of data buffers from one chain of bios to
+ * another
+ * @src: source bio list
+ * @dst: destination bio list
+ *
+ * Stops when it reaches the end of either the @src list or @dst list - that is,
+ * copies min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of
+ * bios).
+ */
+void bio_list_copy_data(struct bio *dst, struct bio *src)
+{
+	struct bvec_iter src_iter = src->bi_iter;
+	struct bvec_iter dst_iter = dst->bi_iter;
 
 	while (1) {
 		if (!src_iter.bi_size) {
@@ -1009,26 +1051,10 @@ void bio_copy_data(struct bio *dst, struct bio *src)
 			dst_iter = dst->bi_iter;
 		}
 
-		src_bv = bio_iter_iovec(src, src_iter);
-		dst_bv = bio_iter_iovec(dst, dst_iter);
-
-		bytes = min(src_bv.bv_len, dst_bv.bv_len);
-
-		src_p = kmap_atomic(src_bv.bv_page);
-		dst_p = kmap_atomic(dst_bv.bv_page);
-
-		memcpy(dst_p + dst_bv.bv_offset,
-		       src_p + src_bv.bv_offset,
-		       bytes);
-
-		kunmap_atomic(dst_p);
-		kunmap_atomic(src_p);
-
-		bio_advance_iter(src, &src_iter, bytes);
-		bio_advance_iter(dst, &dst_iter, bytes);
+		bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
 	}
 }
-EXPORT_SYMBOL(bio_copy_data);
+EXPORT_SYMBOL(bio_list_copy_data);
 
 struct bio_map_data {
 	int is_our_pages;
@@ -1584,6 +1610,7 @@ void bio_set_pages_dirty(struct bio *bio)
 			set_page_dirty_lock(page);
 	}
 }
+EXPORT_SYMBOL_GPL(bio_set_pages_dirty);
 
 static void bio_release_pages(struct bio *bio)
 {
@@ -1667,6 +1694,7 @@ void bio_check_pages_dirty(struct bio *bio)
 		bio_put(bio);
 	}
 }
+EXPORT_SYMBOL_GPL(bio_check_pages_dirty);
 
 void generic_start_io_acct(struct request_queue *q, int rw,
 			   unsigned long sectors, struct hd_struct *part)
@@ -1749,6 +1777,9 @@ void bio_endio(struct bio *bio)
 	if (!bio_integrity_endio(bio))
 		return;
 
+	if (WARN_ONCE(bio->bi_next, "driver left bi_next not NULL"))
+		bio->bi_next = NULL;
+
 	/*
 	 * Need to have a real endio function for chained bios, otherwise
 	 * various corner cases will break (like stacking block devices that
@@ -1848,30 +1879,38 @@ EXPORT_SYMBOL_GPL(bio_trim);
  * create memory pools for biovec's in a bio_set.
  * use the global biovec slabs created for general use.
  */
-mempool_t *biovec_create_pool(int pool_entries)
+int biovec_init_pool(mempool_t *pool, int pool_entries)
 {
 	struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;
 
-	return mempool_create_slab_pool(pool_entries, bp->slab);
+	return mempool_init_slab_pool(pool, pool_entries, bp->slab);
 }
 
-void bioset_free(struct bio_set *bs)
+/*
+ * bioset_exit - exit a bioset initialized with bioset_init()
+ *
+ * May be called on a zeroed but uninitialized bioset (i.e. allocated with
+ * kzalloc()).
+ */
+void bioset_exit(struct bio_set *bs)
 {
 	if (bs->rescue_workqueue)
 		destroy_workqueue(bs->rescue_workqueue);
+	bs->rescue_workqueue = NULL;
 
-	mempool_destroy(bs->bio_pool);
-	mempool_destroy(bs->bvec_pool);
+	mempool_exit(&bs->bio_pool);
+	mempool_exit(&bs->bvec_pool);
 
 	bioset_integrity_free(bs);
-	bio_put_slab(bs);
-
-	kfree(bs);
+	if (bs->bio_slab)
+		bio_put_slab(bs);
+	bs->bio_slab = NULL;
 }
-EXPORT_SYMBOL(bioset_free);
+EXPORT_SYMBOL(bioset_exit);
 
 /**
- * bioset_create  - Create a bio_set
+ * bioset_init - Initialize a bio_set
+ * @bs:		pool to initialize
  * @pool_size:	Number of bio and bio_vecs to cache in the mempool
  * @front_pad:	Number of bytes to allocate in front of the returned bio
  * @flags:	Flags to modify behavior, currently %BIOSET_NEED_BVECS
@@ -1890,16 +1929,12 @@ EXPORT_SYMBOL(bioset_free);
  *    dispatch queued requests when the mempool runs out of space.
  *
  */
-struct bio_set *bioset_create(unsigned int pool_size,
-			      unsigned int front_pad,
-			      int flags)
+int bioset_init(struct bio_set *bs,
+		unsigned int pool_size,
+		unsigned int front_pad,
+		int flags)
 {
 	unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
-	struct bio_set *bs;
-
-	bs = kzalloc(sizeof(*bs), GFP_KERNEL);
-	if (!bs)
-		return NULL;
 
 	bs->front_pad = front_pad;
 
@@ -1908,34 +1943,29 @@ struct bio_set *bioset_create(unsigned int pool_size,
 	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
 
 	bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
-	if (!bs->bio_slab) {
-		kfree(bs);
-		return NULL;
-	}
+	if (!bs->bio_slab)
+		return -ENOMEM;
 
-	bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab);
-	if (!bs->bio_pool)
+	if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))
 		goto bad;
 
-	if (flags & BIOSET_NEED_BVECS) {
-		bs->bvec_pool = biovec_create_pool(pool_size);
-		if (!bs->bvec_pool)
-			goto bad;
-	}
+	if ((flags & BIOSET_NEED_BVECS) &&
+	    biovec_init_pool(&bs->bvec_pool, pool_size))
+		goto bad;
 
 	if (!(flags & BIOSET_NEED_RESCUER))
-		return bs;
+		return 0;
 
 	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
 	if (!bs->rescue_workqueue)
 		goto bad;
 
-	return bs;
+	return 0;
 bad:
-	bioset_free(bs);
-	return NULL;
+	bioset_exit(bs);
+	return -ENOMEM;
 }
-EXPORT_SYMBOL(bioset_create);
+EXPORT_SYMBOL(bioset_init);
 
 #ifdef CONFIG_BLK_CGROUP
 
@@ -2020,11 +2050,10 @@ static int __init init_bio(void)
 	bio_integrity_init();
 	biovec_init_slabs();
 
-	fs_bio_set = bioset_create(BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
-	if (!fs_bio_set)
+	if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS))
 		panic("bio: can't allocate bios\n");
 
-	if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE))
+	if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE))
 		panic("bio: can't create integrity pool\n");
 
 	return 0;
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index eb85cb87c40f..7c6acb9cf72b 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -45,6 +45,7 @@ struct blkcg blkcg_root;
 EXPORT_SYMBOL_GPL(blkcg_root);
 
 struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
+EXPORT_SYMBOL_GPL(blkcg_root_css);
 
 static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
 
diff --git a/block/blk-core.c b/block/blk-core.c
index 85909b431eb0..a295b3c159b2 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -196,15 +196,8 @@ void blk_rq_init(struct request_queue *q, struct request *rq)
 	RB_CLEAR_NODE(&rq->rb_node);
 	rq->tag = -1;
 	rq->internal_tag = -1;
-	rq->start_time = jiffies;
-	set_start_time_ns(rq);
+	rq->start_time_ns = ktime_get_ns();
 	rq->part = NULL;
-	seqcount_init(&rq->gstate_seq);
-	u64_stats_init(&rq->aborted_gstate_sync);
-	/*
-	 * See comment of blk_mq_init_request
-	 */
-	WRITE_ONCE(rq->gstate, MQ_RQ_GEN_INC);
 }
 EXPORT_SYMBOL(blk_rq_init);
 
@@ -280,6 +273,10 @@ static void req_bio_endio(struct request *rq, struct bio *bio,
 	bio_advance(bio, nbytes);
 
 	/* don't actually finish bio if it's part of flush sequence */
+	/*
+	 * XXX this code looks suspicious - it's not consistent with advancing
+	 * req->bio in caller
+	 */
 	if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
 		bio_endio(bio);
 }
@@ -360,7 +357,6 @@ EXPORT_SYMBOL(blk_start_queue_async);
 void blk_start_queue(struct request_queue *q)
 {
 	lockdep_assert_held(q->queue_lock);
-	WARN_ON(!in_interrupt() && !irqs_disabled());
 	WARN_ON_ONCE(q->mq_ops);
 
 	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
@@ -996,6 +992,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
 					   spinlock_t *lock)
 {
 	struct request_queue *q;
+	int ret;
 
 	q = kmem_cache_alloc_node(blk_requestq_cachep,
 				gfp_mask | __GFP_ZERO, node_id);
@@ -1006,8 +1003,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
 	if (q->id < 0)
 		goto fail_q;
 
-	q->bio_split = bioset_create(BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
-	if (!q->bio_split)
+	ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
+	if (ret)
 		goto fail_id;
 
 	q->backing_dev_info = bdi_alloc_node(gfp_mask, node_id);
@@ -1079,7 +1076,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
 fail_stats:
 	bdi_put(q->backing_dev_info);
 fail_split:
-	bioset_free(q->bio_split);
+	bioset_exit(&q->bio_split);
 fail_id:
 	ida_simple_remove(&blk_queue_ida, q->id);
 fail_q:
@@ -1334,6 +1331,7 @@ int blk_update_nr_requests(struct request_queue *q, unsigned int nr)
  * @op: operation and flags
  * @bio: bio to allocate request for (can be %NULL)
  * @flags: BLQ_MQ_REQ_* flags
+ * @gfp_mask: allocator flags
  *
  * Get a free request from @q.  This function may fail under memory
  * pressure or if @q is dead.
@@ -1343,7 +1341,7 @@ int blk_update_nr_requests(struct request_queue *q, unsigned int nr)
  * Returns request pointer on success, with @q->queue_lock *not held*.
  */
 static struct request *__get_request(struct request_list *rl, unsigned int op,
-				     struct bio *bio, blk_mq_req_flags_t flags)
+		struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp_mask)
 {
 	struct request_queue *q = rl->q;
 	struct request *rq;
@@ -1352,8 +1350,6 @@ static struct request *__get_request(struct request_list *rl, unsigned int op,
 	struct io_cq *icq = NULL;
 	const bool is_sync = op_is_sync(op);
 	int may_queue;
-	gfp_t gfp_mask = flags & BLK_MQ_REQ_NOWAIT ? GFP_ATOMIC :
-			 __GFP_DIRECT_RECLAIM;
 	req_flags_t rq_flags = RQF_ALLOCED;
 
 	lockdep_assert_held(q->queue_lock);
@@ -1517,8 +1513,9 @@ static struct request *__get_request(struct request_list *rl, unsigned int op,
  * @op: operation and flags
  * @bio: bio to allocate request for (can be %NULL)
  * @flags: BLK_MQ_REQ_* flags.
+ * @gfp: allocator flags
  *
- * Get a free request from @q.  If %__GFP_DIRECT_RECLAIM is set in @gfp_mask,
+ * Get a free request from @q.  If %BLK_MQ_REQ_NOWAIT is set in @flags,
  * this function keeps retrying under memory pressure and fails iff @q is dead.
  *
  * Must be called with @q->queue_lock held and,
@@ -1526,7 +1523,7 @@ static struct request *__get_request(struct request_list *rl, unsigned int op,
  * Returns request pointer on success, with @q->queue_lock *not held*.
  */
 static struct request *get_request(struct request_queue *q, unsigned int op,
-				   struct bio *bio, blk_mq_req_flags_t flags)
+		struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp)
 {
 	const bool is_sync = op_is_sync(op);
 	DEFINE_WAIT(wait);
@@ -1538,7 +1535,7 @@ static struct request *get_request(struct request_queue *q, unsigned int op,
 
 	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */
 retry:
-	rq = __get_request(rl, op, bio, flags);
+	rq = __get_request(rl, op, bio, flags, gfp);
 	if (!IS_ERR(rq))
 		return rq;
 
@@ -1579,8 +1576,7 @@ static struct request *blk_old_get_request(struct request_queue *q,
 				unsigned int op, blk_mq_req_flags_t flags)
 {
 	struct request *rq;
-	gfp_t gfp_mask = flags & BLK_MQ_REQ_NOWAIT ? GFP_ATOMIC :
-			 __GFP_DIRECT_RECLAIM;
+	gfp_t gfp_mask = flags & BLK_MQ_REQ_NOWAIT ? GFP_ATOMIC : GFP_NOIO;
 	int ret = 0;
 
 	WARN_ON_ONCE(q->mq_ops);
@@ -1592,7 +1588,7 @@ static struct request *blk_old_get_request(struct request_queue *q,
 	if (ret)
 		return ERR_PTR(ret);
 	spin_lock_irq(q->queue_lock);
-	rq = get_request(q, op, NULL, flags);
+	rq = get_request(q, op, NULL, flags, gfp_mask);
 	if (IS_ERR(rq)) {
 		spin_unlock_irq(q->queue_lock);
 		blk_queue_exit(q);
@@ -1607,13 +1603,13 @@ static struct request *blk_old_get_request(struct request_queue *q,
 }
 
 /**
- * blk_get_request_flags - allocate a request
+ * blk_get_request - allocate a request
  * @q: request queue to allocate a request for
  * @op: operation (REQ_OP_*) and REQ_* flags, e.g. REQ_SYNC.
  * @flags: BLK_MQ_REQ_* flags, e.g. BLK_MQ_REQ_NOWAIT.
  */
-struct request *blk_get_request_flags(struct request_queue *q, unsigned int op,
-				      blk_mq_req_flags_t flags)
+struct request *blk_get_request(struct request_queue *q, unsigned int op,
+				blk_mq_req_flags_t flags)
 {
 	struct request *req;
 
@@ -1632,14 +1628,6 @@ struct request *blk_get_request_flags(struct request_queue *q, unsigned int op,
 
 	return req;
 }
-EXPORT_SYMBOL(blk_get_request_flags);
-
-struct request *blk_get_request(struct request_queue *q, unsigned int op,
-				gfp_t gfp_mask)
-{
-	return blk_get_request_flags(q, op, gfp_mask & __GFP_DIRECT_RECLAIM ?
-				     0 : BLK_MQ_REQ_NOWAIT);
-}
 EXPORT_SYMBOL(blk_get_request);
 
 /**
@@ -1660,7 +1648,7 @@ void blk_requeue_request(struct request_queue *q, struct request *rq)
 	blk_delete_timer(rq);
 	blk_clear_rq_complete(rq);
 	trace_block_rq_requeue(q, rq);
-	wbt_requeue(q->rq_wb, &rq->issue_stat);
+	wbt_requeue(q->rq_wb, rq);
 
 	if (rq->rq_flags & RQF_QUEUED)
 		blk_queue_end_tag(q, rq);
@@ -1767,7 +1755,7 @@ void __blk_put_request(struct request_queue *q, struct request *req)
 	/* this is a bio leak */
 	WARN_ON(req->bio != NULL);
 
-	wbt_done(q->rq_wb, &req->issue_stat);
+	wbt_done(q->rq_wb, req);
 
 	/*
 	 * Request may not have originated from ll_rw_blk. if not,
@@ -2066,7 +2054,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
 	 * Returns with the queue unlocked.
 	 */
 	blk_queue_enter_live(q);
-	req = get_request(q, bio->bi_opf, bio, 0);
+	req = get_request(q, bio->bi_opf, bio, 0, GFP_NOIO);
 	if (IS_ERR(req)) {
 		blk_queue_exit(q);
 		__wbt_done(q->rq_wb, wb_acct);
@@ -2078,7 +2066,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
 		goto out_unlock;
 	}
 
-	wbt_track(&req->issue_stat, wb_acct);
+	wbt_track(req, wb_acct);
 
 	/*
 	 * After dropping the lock and possibly sleeping here, our request
@@ -2727,7 +2715,7 @@ void blk_account_io_completion(struct request *req, unsigned int bytes)
 	}
 }
 
-void blk_account_io_done(struct request *req)
+void blk_account_io_done(struct request *req, u64 now)
 {
 	/*
 	 * Account IO completion.  flush_rq isn't accounted as a
@@ -2735,11 +2723,12 @@ void blk_account_io_done(struct request *req)
 	 * containing request is enough.
 	 */
 	if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {
-		unsigned long duration = jiffies - req->start_time;
+		unsigned long duration;
 		const int rw = rq_data_dir(req);
 		struct hd_struct *part;
 		int cpu;
 
+		duration = nsecs_to_jiffies(now - req->start_time_ns);
 		cpu = part_stat_lock();
 		part = req->part;
 
@@ -2970,10 +2959,8 @@ static void blk_dequeue_request(struct request *rq)
 	 * and to it is freed is accounted as io that is in progress at
 	 * the driver side.
 	 */
-	if (blk_account_rq(rq)) {
+	if (blk_account_rq(rq))
 		q->in_flight[rq_is_sync(rq)]++;
-		set_io_start_time_ns(rq);
-	}
 }
 
 /**
@@ -2992,9 +2979,12 @@ void blk_start_request(struct request *req)
 	blk_dequeue_request(req);
 
 	if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
-		blk_stat_set_issue(&req->issue_stat, blk_rq_sectors(req));
+		req->io_start_time_ns = ktime_get_ns();
+#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
+		req->throtl_size = blk_rq_sectors(req);
+#endif
 		req->rq_flags |= RQF_STATS;
-		wbt_issue(req->q->rq_wb, &req->issue_stat);
+		wbt_issue(req->q->rq_wb, req);
 	}
 
 	BUG_ON(blk_rq_is_complete(req));
@@ -3092,8 +3082,10 @@ bool blk_update_request(struct request *req, blk_status_t error,
 		struct bio *bio = req->bio;
 		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
 
-		if (bio_bytes == bio->bi_iter.bi_size)
+		if (bio_bytes == bio->bi_iter.bi_size) {
 			req->bio = bio->bi_next;
+			bio->bi_next = NULL;
+		}
 
 		/* Completion has already been traced */
 		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
@@ -3190,12 +3182,13 @@ EXPORT_SYMBOL_GPL(blk_unprep_request);
 void blk_finish_request(struct request *req, blk_status_t error)
 {
 	struct request_queue *q = req->q;
+	u64 now = ktime_get_ns();
 
 	lockdep_assert_held(req->q->queue_lock);
 	WARN_ON_ONCE(q->mq_ops);
 
 	if (req->rq_flags & RQF_STATS)
-		blk_stat_add(req);
+		blk_stat_add(req, now);
 
 	if (req->rq_flags & RQF_QUEUED)
 		blk_queue_end_tag(q, req);
@@ -3210,10 +3203,10 @@ void blk_finish_request(struct request *req, blk_status_t error)
 	if (req->rq_flags & RQF_DONTPREP)
 		blk_unprep_request(req);
 
-	blk_account_io_done(req);
+	blk_account_io_done(req, now);
 
 	if (req->end_io) {
-		wbt_done(req->q->rq_wb, &req->issue_stat);
+		wbt_done(req->q->rq_wb, req);
 		req->end_io(req, error);
 	} else {
 		if (blk_bidi_rq(req))
@@ -3519,7 +3512,7 @@ int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
 	struct bio *bio, *bio_src;
 
 	if (!bs)
-		bs = fs_bio_set;
+		bs = &fs_bio_set;
 
 	__rq_for_each_bio(bio_src, rq_src) {
 		bio = bio_clone_fast(bio_src, gfp_mask, bs);
@@ -3630,7 +3623,7 @@ static void queue_unplugged(struct request_queue *q, unsigned int depth,
 		blk_run_queue_async(q);
 	else
 		__blk_run_queue(q);
-	spin_unlock(q->queue_lock);
+	spin_unlock_irq(q->queue_lock);
 }
 
 static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
@@ -3678,7 +3671,6 @@ EXPORT_SYMBOL(blk_check_plugged);
 void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 {
 	struct request_queue *q;
-	unsigned long flags;
 	struct request *rq;
 	LIST_HEAD(list);
 	unsigned int depth;
@@ -3698,11 +3690,6 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 	q = NULL;
 	depth = 0;
 
-	/*
-	 * Save and disable interrupts here, to avoid doing it for every
-	 * queue lock we have to take.
-	 */
-	local_irq_save(flags);
 	while (!list_empty(&list)) {
 		rq = list_entry_rq(list.next);
 		list_del_init(&rq->queuelist);
@@ -3715,7 +3702,7 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 				queue_unplugged(q, depth, from_schedule);
 			q = rq->q;
 			depth = 0;
-			spin_lock(q->queue_lock);
+			spin_lock_irq(q->queue_lock);
 		}
 
 		/*
@@ -3742,8 +3729,6 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 	 */
 	if (q)
 		queue_unplugged(q, depth, from_schedule);
-
-	local_irq_restore(flags);
 }
 
 void blk_finish_plug(struct blk_plug *plug)
diff --git a/block/blk-integrity.c b/block/blk-integrity.c
index feb30570eaf5..6121611e1316 100644
--- a/block/blk-integrity.c
+++ b/block/blk-integrity.c
@@ -333,34 +333,34 @@ static ssize_t integrity_device_show(struct blk_integrity *bi, char *page)
 }
 
 static struct integrity_sysfs_entry integrity_format_entry = {
-	.attr = { .name = "format", .mode = S_IRUGO },
+	.attr = { .name = "format", .mode = 0444 },
 	.show = integrity_format_show,
 };
 
 static struct integrity_sysfs_entry integrity_tag_size_entry = {
-	.attr = { .name = "tag_size", .mode = S_IRUGO },
+	.attr = { .name = "tag_size", .mode = 0444 },
 	.show = integrity_tag_size_show,
 };
 
 static struct integrity_sysfs_entry integrity_interval_entry = {
-	.attr = { .name = "protection_interval_bytes", .mode = S_IRUGO },
+	.attr = { .name = "protection_interval_bytes", .mode = 0444 },
 	.show = integrity_interval_show,
 };
 
 static struct integrity_sysfs_entry integrity_verify_entry = {
-	.attr = { .name = "read_verify", .mode = S_IRUGO | S_IWUSR },
+	.attr = { .name = "read_verify", .mode = 0644 },
 	.show = integrity_verify_show,
 	.store = integrity_verify_store,
 };
 
 static struct integrity_sysfs_entry integrity_generate_entry = {
-	.attr = { .name = "write_generate", .mode = S_IRUGO | S_IWUSR },
+	.attr = { .name = "write_generate", .mode = 0644 },
 	.show = integrity_generate_show,
 	.store = integrity_generate_store,
 };
 
 static struct integrity_sysfs_entry integrity_device_entry = {
-	.attr = { .name = "device_is_integrity_capable", .mode = S_IRUGO },
+	.attr = { .name = "device_is_integrity_capable", .mode = 0444 },
 	.show = integrity_device_show,
 };
 
diff --git a/block/blk-lib.c b/block/blk-lib.c
index a676084d4740..8faa70f26fcd 100644
--- a/block/blk-lib.c
+++ b/block/blk-lib.c
@@ -62,10 +62,16 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
 		unsigned int req_sects;
 		sector_t end_sect, tmp;
 
-		/* Make sure bi_size doesn't overflow */
-		req_sects = min_t(sector_t, nr_sects, UINT_MAX >> 9);
+		/*
+		 * Issue in chunks of the user defined max discard setting,
+		 * ensuring that bi_size doesn't overflow
+		 */
+		req_sects = min_t(sector_t, nr_sects,
+					q->limits.max_discard_sectors);
+		if (req_sects > UINT_MAX >> 9)
+			req_sects = UINT_MAX >> 9;
 
-		/**
+		/*
 		 * If splitting a request, and the next starting sector would be
 		 * misaligned, stop the discard at the previous aligned sector.
 		 */
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 782940c65d8a..d70ab08820e5 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -188,16 +188,16 @@ void blk_queue_split(struct request_queue *q, struct bio **bio)
 	switch (bio_op(*bio)) {
 	case REQ_OP_DISCARD:
 	case REQ_OP_SECURE_ERASE:
-		split = blk_bio_discard_split(q, *bio, q->bio_split, &nsegs);
+		split = blk_bio_discard_split(q, *bio, &q->bio_split, &nsegs);
 		break;
 	case REQ_OP_WRITE_ZEROES:
-		split = blk_bio_write_zeroes_split(q, *bio, q->bio_split, &nsegs);
+		split = blk_bio_write_zeroes_split(q, *bio, &q->bio_split, &nsegs);
 		break;
 	case REQ_OP_WRITE_SAME:
-		split = blk_bio_write_same_split(q, *bio, q->bio_split, &nsegs);
+		split = blk_bio_write_same_split(q, *bio, &q->bio_split, &nsegs);
 		break;
 	default:
-		split = blk_bio_segment_split(q, *bio, q->bio_split, &nsegs);
+		split = blk_bio_segment_split(q, *bio, &q->bio_split, &nsegs);
 		break;
 	}
 
@@ -724,13 +724,12 @@ static struct request *attempt_merge(struct request_queue *q,
 	}
 
 	/*
-	 * At this point we have either done a back merge
-	 * or front merge. We need the smaller start_time of
-	 * the merged requests to be the current request
-	 * for accounting purposes.
+	 * At this point we have either done a back merge or front merge. We
+	 * need the smaller start_time_ns of the merged requests to be the
+	 * current request for accounting purposes.
 	 */
-	if (time_after(req->start_time, next->start_time))
-		req->start_time = next->start_time;
+	if (next->start_time_ns < req->start_time_ns)
+		req->start_time_ns = next->start_time_ns;
 
 	req->biotail->bi_next = next->bio;
 	req->biotail = next->biotail;
diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c
index 3080e18cb859..ffa622366922 100644
--- a/block/blk-mq-debugfs.c
+++ b/block/blk-mq-debugfs.c
@@ -344,7 +344,6 @@ static const char *const rqf_name[] = {
 	RQF_NAME(STATS),
 	RQF_NAME(SPECIAL_PAYLOAD),
 	RQF_NAME(ZONE_WRITE_LOCKED),
-	RQF_NAME(MQ_TIMEOUT_EXPIRED),
 	RQF_NAME(MQ_POLL_SLEPT),
 };
 #undef RQF_NAME
diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index 25c14c58385c..b0f2c2a40a0c 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -268,19 +268,16 @@ bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
 EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
 
 /*
- * Reverse check our software queue for entries that we could potentially
- * merge with. Currently includes a hand-wavy stop count of 8, to not spend
- * too much time checking for merges.
+ * Iterate list of requests and see if we can merge this bio with any
+ * of them.
  */
-static bool blk_mq_attempt_merge(struct request_queue *q,
-				 struct blk_mq_ctx *ctx, struct bio *bio)
+bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list,
+			   struct bio *bio)
 {
 	struct request *rq;
 	int checked = 8;
 
-	lockdep_assert_held(&ctx->lock);
-
-	list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
+	list_for_each_entry_reverse(rq, list, queuelist) {
 		bool merged = false;
 
 		if (!checked--)
@@ -305,13 +302,30 @@ static bool blk_mq_attempt_merge(struct request_queue *q,
 			continue;
 		}
 
-		if (merged)
-			ctx->rq_merged++;
 		return merged;
 	}
 
 	return false;
 }
+EXPORT_SYMBOL_GPL(blk_mq_bio_list_merge);
+
+/*
+ * Reverse check our software queue for entries that we could potentially
+ * merge with. Currently includes a hand-wavy stop count of 8, to not spend
+ * too much time checking for merges.
+ */
+static bool blk_mq_attempt_merge(struct request_queue *q,
+				 struct blk_mq_ctx *ctx, struct bio *bio)
+{
+	lockdep_assert_held(&ctx->lock);
+
+	if (blk_mq_bio_list_merge(q, &ctx->rq_list, bio)) {
+		ctx->rq_merged++;
+		return true;
+	}
+
+	return false;
+}
 
 bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio)
 {
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
index a54b4b070f1c..aafb44224c89 100644
--- a/block/blk-mq-sysfs.c
+++ b/block/blk-mq-sysfs.c
@@ -166,15 +166,15 @@ static struct attribute *default_ctx_attrs[] = {
 };
 
 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
-	.attr = {.name = "nr_tags", .mode = S_IRUGO },
+	.attr = {.name = "nr_tags", .mode = 0444 },
 	.show = blk_mq_hw_sysfs_nr_tags_show,
 };
 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
-	.attr = {.name = "nr_reserved_tags", .mode = S_IRUGO },
+	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
 	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
 };
 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
-	.attr = {.name = "cpu_list", .mode = S_IRUGO },
+	.attr = {.name = "cpu_list", .mode = 0444 },
 	.show = blk_mq_hw_sysfs_cpus_show,
 };
 
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 336dde07b230..70356a2a11ab 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -134,6 +134,8 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
 	ws = bt_wait_ptr(bt, data->hctx);
 	drop_ctx = data->ctx == NULL;
 	do {
+		struct sbitmap_queue *bt_prev;
+
 		/*
 		 * We're out of tags on this hardware queue, kick any
 		 * pending IO submits before going to sleep waiting for
@@ -159,6 +161,7 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
 		if (data->ctx)
 			blk_mq_put_ctx(data->ctx);
 
+		bt_prev = bt;
 		io_schedule();
 
 		data->ctx = blk_mq_get_ctx(data->q);
@@ -170,6 +173,15 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
 			bt = &tags->bitmap_tags;
 
 		finish_wait(&ws->wait, &wait);
+
+		/*
+		 * If destination hw queue is changed, fake wake up on
+		 * previous queue for compensating the wake up miss, so
+		 * other allocations on previous queue won't be starved.
+		 */
+		if (bt != bt_prev)
+			sbitmap_queue_wake_up(bt_prev);
+
 		ws = bt_wait_ptr(bt, data->hctx);
 	} while (1);
 
@@ -259,7 +271,7 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
 	 * test and set the bit before assining ->rqs[].
 	 */
 	rq = tags->rqs[bitnr];
-	if (rq)
+	if (rq && blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
 		iter_data->fn(rq, iter_data->data, reserved);
 
 	return true;
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 9ce9cac16c3f..3d3952301b3e 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -309,7 +309,8 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
 	RB_CLEAR_NODE(&rq->rb_node);
 	rq->rq_disk = NULL;
 	rq->part = NULL;
-	rq->start_time = jiffies;
+	rq->start_time_ns = ktime_get_ns();
+	rq->io_start_time_ns = 0;
 	rq->nr_phys_segments = 0;
 #if defined(CONFIG_BLK_DEV_INTEGRITY)
 	rq->nr_integrity_segments = 0;
@@ -328,11 +329,10 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
 
 #ifdef CONFIG_BLK_CGROUP
 	rq->rl = NULL;
-	set_start_time_ns(rq);
-	rq->io_start_time_ns = 0;
 #endif
 
 	data->ctx->rq_dispatched[op_is_sync(op)]++;
+	refcount_set(&rq->ref, 1);
 	return rq;
 }
 
@@ -361,9 +361,11 @@ static struct request *blk_mq_get_request(struct request_queue *q,
 
 		/*
 		 * Flush requests are special and go directly to the
-		 * dispatch list.
+		 * dispatch list. Don't include reserved tags in the
+		 * limiting, as it isn't useful.
 		 */
-		if (!op_is_flush(op) && e->type->ops.mq.limit_depth)
+		if (!op_is_flush(op) && e->type->ops.mq.limit_depth &&
+		    !(data->flags & BLK_MQ_REQ_RESERVED))
 			e->type->ops.mq.limit_depth(op, data);
 	}
 
@@ -464,13 +466,27 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
 }
 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
 
+static void __blk_mq_free_request(struct request *rq)
+{
+	struct request_queue *q = rq->q;
+	struct blk_mq_ctx *ctx = rq->mq_ctx;
+	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
+	const int sched_tag = rq->internal_tag;
+
+	if (rq->tag != -1)
+		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
+	if (sched_tag != -1)
+		blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);
+	blk_mq_sched_restart(hctx);
+	blk_queue_exit(q);
+}
+
 void blk_mq_free_request(struct request *rq)
 {
 	struct request_queue *q = rq->q;
 	struct elevator_queue *e = q->elevator;
 	struct blk_mq_ctx *ctx = rq->mq_ctx;
 	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
-	const int sched_tag = rq->internal_tag;
 
 	if (rq->rq_flags & RQF_ELVPRIV) {
 		if (e && e->type->ops.mq.finish_request)
@@ -488,27 +504,30 @@ void blk_mq_free_request(struct request *rq)
 	if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
 		laptop_io_completion(q->backing_dev_info);
 
-	wbt_done(q->rq_wb, &rq->issue_stat);
+	wbt_done(q->rq_wb, rq);
 
 	if (blk_rq_rl(rq))
 		blk_put_rl(blk_rq_rl(rq));
 
-	blk_mq_rq_update_state(rq, MQ_RQ_IDLE);
-	if (rq->tag != -1)
-		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
-	if (sched_tag != -1)
-		blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);
-	blk_mq_sched_restart(hctx);
-	blk_queue_exit(q);
+	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
+	if (refcount_dec_and_test(&rq->ref))
+		__blk_mq_free_request(rq);
 }
 EXPORT_SYMBOL_GPL(blk_mq_free_request);
 
 inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
 {
-	blk_account_io_done(rq);
+	u64 now = ktime_get_ns();
+
+	if (rq->rq_flags & RQF_STATS) {
+		blk_mq_poll_stats_start(rq->q);
+		blk_stat_add(rq, now);
+	}
+
+	blk_account_io_done(rq, now);
 
 	if (rq->end_io) {
-		wbt_done(rq->q->rq_wb, &rq->issue_stat);
+		wbt_done(rq->q->rq_wb, rq);
 		rq->end_io(rq, error);
 	} else {
 		if (unlikely(blk_bidi_rq(rq)))
@@ -539,15 +558,12 @@ static void __blk_mq_complete_request(struct request *rq)
 	bool shared = false;
 	int cpu;
 
-	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT);
-	blk_mq_rq_update_state(rq, MQ_RQ_COMPLETE);
+	if (cmpxchg(&rq->state, MQ_RQ_IN_FLIGHT, MQ_RQ_COMPLETE) !=
+			MQ_RQ_IN_FLIGHT)
+		return;
 
 	if (rq->internal_tag != -1)
 		blk_mq_sched_completed_request(rq);
-	if (rq->rq_flags & RQF_STATS) {
-		blk_mq_poll_stats_start(rq->q);
-		blk_stat_add(rq);
-	}
 
 	if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
 		rq->q->softirq_done_fn(rq);
@@ -589,36 +605,6 @@ static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx)
 		*srcu_idx = srcu_read_lock(hctx->srcu);
 }
 
-static void blk_mq_rq_update_aborted_gstate(struct request *rq, u64 gstate)
-{
-	unsigned long flags;
-
-	/*
-	 * blk_mq_rq_aborted_gstate() is used from the completion path and
-	 * can thus be called from irq context.  u64_stats_fetch in the
-	 * middle of update on the same CPU leads to lockup.  Disable irq
-	 * while updating.
-	 */
-	local_irq_save(flags);
-	u64_stats_update_begin(&rq->aborted_gstate_sync);
-	rq->aborted_gstate = gstate;
-	u64_stats_update_end(&rq->aborted_gstate_sync);
-	local_irq_restore(flags);
-}
-
-static u64 blk_mq_rq_aborted_gstate(struct request *rq)
-{
-	unsigned int start;
-	u64 aborted_gstate;
-
-	do {
-		start = u64_stats_fetch_begin(&rq->aborted_gstate_sync);
-		aborted_gstate = rq->aborted_gstate;
-	} while (u64_stats_fetch_retry(&rq->aborted_gstate_sync, start));
-
-	return aborted_gstate;
-}
-
 /**
  * blk_mq_complete_request - end I/O on a request
  * @rq:		the request being processed
@@ -629,28 +615,9 @@ static u64 blk_mq_rq_aborted_gstate(struct request *rq)
  **/
 void blk_mq_complete_request(struct request *rq)
 {
-	struct request_queue *q = rq->q;
-	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, rq->mq_ctx->cpu);
-	int srcu_idx;
-
-	if (unlikely(blk_should_fake_timeout(q)))
+	if (unlikely(blk_should_fake_timeout(rq->q)))
 		return;
-
-	/*
-	 * If @rq->aborted_gstate equals the current instance, timeout is
-	 * claiming @rq and we lost.  This is synchronized through
-	 * hctx_lock().  See blk_mq_timeout_work() for details.
-	 *
-	 * Completion path never blocks and we can directly use RCU here
-	 * instead of hctx_lock() which can be either RCU or SRCU.
-	 * However, that would complicate paths which want to synchronize
-	 * against us.  Let stay in sync with the issue path so that
-	 * hctx_lock() covers both issue and completion paths.
-	 */
-	hctx_lock(hctx, &srcu_idx);
-	if (blk_mq_rq_aborted_gstate(rq) != rq->gstate)
-		__blk_mq_complete_request(rq);
-	hctx_unlock(hctx, srcu_idx);
+	__blk_mq_complete_request(rq);
 }
 EXPORT_SYMBOL(blk_mq_complete_request);
 
@@ -669,32 +636,18 @@ void blk_mq_start_request(struct request *rq)
 	trace_block_rq_issue(q, rq);
 
 	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
-		blk_stat_set_issue(&rq->issue_stat, blk_rq_sectors(rq));
+		rq->io_start_time_ns = ktime_get_ns();
+#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
+		rq->throtl_size = blk_rq_sectors(rq);
+#endif
 		rq->rq_flags |= RQF_STATS;
-		wbt_issue(q->rq_wb, &rq->issue_stat);
+		wbt_issue(q->rq_wb, rq);
 	}
 
 	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
 
-	/*
-	 * Mark @rq in-flight which also advances the generation number,
-	 * and register for timeout.  Protect with a seqcount to allow the
-	 * timeout path to read both @rq->gstate and @rq->deadline
-	 * coherently.
-	 *
-	 * This is the only place where a request is marked in-flight.  If
-	 * the timeout path reads an in-flight @rq->gstate, the
-	 * @rq->deadline it reads together under @rq->gstate_seq is
-	 * guaranteed to be the matching one.
-	 */
-	preempt_disable();
-	write_seqcount_begin(&rq->gstate_seq);
-
-	blk_mq_rq_update_state(rq, MQ_RQ_IN_FLIGHT);
 	blk_add_timer(rq);
-
-	write_seqcount_end(&rq->gstate_seq);
-	preempt_enable();
+	WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
 
 	if (q->dma_drain_size && blk_rq_bytes(rq)) {
 		/*
@@ -707,11 +660,6 @@ void blk_mq_start_request(struct request *rq)
 }
 EXPORT_SYMBOL(blk_mq_start_request);
 
-/*
- * When we reach here because queue is busy, it's safe to change the state
- * to IDLE without checking @rq->aborted_gstate because we should still be
- * holding the RCU read lock and thus protected against timeout.
- */
 static void __blk_mq_requeue_request(struct request *rq)
 {
 	struct request_queue *q = rq->q;
@@ -719,10 +667,10 @@ static void __blk_mq_requeue_request(struct request *rq)
 	blk_mq_put_driver_tag(rq);
 
 	trace_block_rq_requeue(q, rq);
-	wbt_requeue(q->rq_wb, &rq->issue_stat);
+	wbt_requeue(q->rq_wb, rq);
 
-	if (blk_mq_rq_state(rq) != MQ_RQ_IDLE) {
-		blk_mq_rq_update_state(rq, MQ_RQ_IDLE);
+	if (blk_mq_request_started(rq)) {
+		WRITE_ONCE(rq->state, MQ_RQ_IDLE);
 		if (q->dma_drain_size && blk_rq_bytes(rq))
 			rq->nr_phys_segments--;
 	}
@@ -820,101 +768,79 @@ struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
 }
 EXPORT_SYMBOL(blk_mq_tag_to_rq);
 
-struct blk_mq_timeout_data {
-	unsigned long next;
-	unsigned int next_set;
-	unsigned int nr_expired;
-};
-
 static void blk_mq_rq_timed_out(struct request *req, bool reserved)
 {
-	const struct blk_mq_ops *ops = req->q->mq_ops;
-	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
-
-	req->rq_flags |= RQF_MQ_TIMEOUT_EXPIRED;
-
-	if (ops->timeout)
-		ret = ops->timeout(req, reserved);
+	if (req->q->mq_ops->timeout) {
+		enum blk_eh_timer_return ret;
 
-	switch (ret) {
-	case BLK_EH_HANDLED:
-		__blk_mq_complete_request(req);
-		break;
-	case BLK_EH_RESET_TIMER:
-		/*
-		 * As nothing prevents from completion happening while
-		 * ->aborted_gstate is set, this may lead to ignored
-		 * completions and further spurious timeouts.
-		 */
-		blk_mq_rq_update_aborted_gstate(req, 0);
-		blk_add_timer(req);
-		break;
-	case BLK_EH_NOT_HANDLED:
-		break;
-	default:
-		printk(KERN_ERR "block: bad eh return: %d\n", ret);
-		break;
+		ret = req->q->mq_ops->timeout(req, reserved);
+		if (ret == BLK_EH_DONE)
+			return;
+		WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
 	}
+
+	blk_add_timer(req);
 }
 
-static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
-		struct request *rq, void *priv, bool reserved)
+static bool blk_mq_req_expired(struct request *rq, unsigned long *next)
 {
-	struct blk_mq_timeout_data *data = priv;
-	unsigned long gstate, deadline;
-	int start;
+	unsigned long deadline;
 
-	might_sleep();
-
-	if (rq->rq_flags & RQF_MQ_TIMEOUT_EXPIRED)
-		return;
+	if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
+		return false;
 
-	/* read coherent snapshots of @rq->state_gen and @rq->deadline */
-	while (true) {
-		start = read_seqcount_begin(&rq->gstate_seq);
-		gstate = READ_ONCE(rq->gstate);
-		deadline = blk_rq_deadline(rq);
-		if (!read_seqcount_retry(&rq->gstate_seq, start))
-			break;
-		cond_resched();
-	}
+	deadline = blk_rq_deadline(rq);
+	if (time_after_eq(jiffies, deadline))
+		return true;
 
-	/* if in-flight && overdue, mark for abortion */
-	if ((gstate & MQ_RQ_STATE_MASK) == MQ_RQ_IN_FLIGHT &&
-	    time_after_eq(jiffies, deadline)) {
-		blk_mq_rq_update_aborted_gstate(rq, gstate);
-		data->nr_expired++;
-		hctx->nr_expired++;
-	} else if (!data->next_set || time_after(data->next, deadline)) {
-		data->next = deadline;
-		data->next_set = 1;
-	}
+	if (*next == 0)
+		*next = deadline;
+	else if (time_after(*next, deadline))
+		*next = deadline;
+	return false;
 }
 
-static void blk_mq_terminate_expired(struct blk_mq_hw_ctx *hctx,
+static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
 		struct request *rq, void *priv, bool reserved)
 {
+	unsigned long *next = priv;
+
 	/*
-	 * We marked @rq->aborted_gstate and waited for RCU.  If there were
-	 * completions that we lost to, they would have finished and
-	 * updated @rq->gstate by now; otherwise, the completion path is
-	 * now guaranteed to see @rq->aborted_gstate and yield.  If
-	 * @rq->aborted_gstate still matches @rq->gstate, @rq is ours.
+	 * Just do a quick check if it is expired before locking the request in
+	 * so we're not unnecessarilly synchronizing across CPUs.
+	 */
+	if (!blk_mq_req_expired(rq, next))
+		return;
+
+	/*
+	 * We have reason to believe the request may be expired. Take a
+	 * reference on the request to lock this request lifetime into its
+	 * currently allocated context to prevent it from being reallocated in
+	 * the event the completion by-passes this timeout handler.
+	 *
+	 * If the reference was already released, then the driver beat the
+	 * timeout handler to posting a natural completion.
 	 */
-	if (!(rq->rq_flags & RQF_MQ_TIMEOUT_EXPIRED) &&
-	    READ_ONCE(rq->gstate) == rq->aborted_gstate)
+	if (!refcount_inc_not_zero(&rq->ref))
+		return;
+
+	/*
+	 * The request is now locked and cannot be reallocated underneath the
+	 * timeout handler's processing. Re-verify this exact request is truly
+	 * expired; if it is not expired, then the request was completed and
+	 * reallocated as a new request.
+	 */
+	if (blk_mq_req_expired(rq, next))
 		blk_mq_rq_timed_out(rq, reserved);
+	if (refcount_dec_and_test(&rq->ref))
+		__blk_mq_free_request(rq);
 }
 
 static void blk_mq_timeout_work(struct work_struct *work)
 {
 	struct request_queue *q =
 		container_of(work, struct request_queue, timeout_work);
-	struct blk_mq_timeout_data data = {
-		.next		= 0,
-		.next_set	= 0,
-		.nr_expired	= 0,
-	};
+	unsigned long next = 0;
 	struct blk_mq_hw_ctx *hctx;
 	int i;
 
@@ -934,39 +860,10 @@ static void blk_mq_timeout_work(struct work_struct *work)
 	if (!percpu_ref_tryget(&q->q_usage_counter))
 		return;
 
-	/* scan for the expired ones and set their ->aborted_gstate */
-	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
-
-	if (data.nr_expired) {
-		bool has_rcu = false;
-
-		/*
-		 * Wait till everyone sees ->aborted_gstate.  The
-		 * sequential waits for SRCUs aren't ideal.  If this ever
-		 * becomes a problem, we can add per-hw_ctx rcu_head and
-		 * wait in parallel.
-		 */
-		queue_for_each_hw_ctx(q, hctx, i) {
-			if (!hctx->nr_expired)
-				continue;
-
-			if (!(hctx->flags & BLK_MQ_F_BLOCKING))
-				has_rcu = true;
-			else
-				synchronize_srcu(hctx->srcu);
-
-			hctx->nr_expired = 0;
-		}
-		if (has_rcu)
-			synchronize_rcu();
-
-		/* terminate the ones we won */
-		blk_mq_queue_tag_busy_iter(q, blk_mq_terminate_expired, NULL);
-	}
+	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);
 
-	if (data.next_set) {
-		data.next = blk_rq_timeout(round_jiffies_up(data.next));
-		mod_timer(&q->timeout, data.next);
+	if (next != 0) {
+		mod_timer(&q->timeout, next);
 	} else {
 		/*
 		 * Request timeouts are handled as a forward rolling timer. If
@@ -1029,7 +926,7 @@ static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
 	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
 
 	spin_lock(&ctx->lock);
-	if (unlikely(!list_empty(&ctx->rq_list))) {
+	if (!list_empty(&ctx->rq_list)) {
 		dispatch_data->rq = list_entry_rq(ctx->rq_list.next);
 		list_del_init(&dispatch_data->rq->queuelist);
 		if (list_empty(&ctx->rq_list))
@@ -1716,15 +1613,6 @@ static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
 	blk_account_io_start(rq, true);
 }
 
-static inline void blk_mq_queue_io(struct blk_mq_hw_ctx *hctx,
-				   struct blk_mq_ctx *ctx,
-				   struct request *rq)
-{
-	spin_lock(&ctx->lock);
-	__blk_mq_insert_request(hctx, rq, false);
-	spin_unlock(&ctx->lock);
-}
-
 static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
 {
 	if (rq->tag != -1)
@@ -1882,7 +1770,7 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
 		return BLK_QC_T_NONE;
 	}
 
-	wbt_track(&rq->issue_stat, wb_acct);
+	wbt_track(rq, wb_acct);
 
 	cookie = request_to_qc_t(data.hctx, rq);
 
@@ -1949,15 +1837,10 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
 		blk_mq_put_ctx(data.ctx);
 		blk_mq_bio_to_request(rq, bio);
 		blk_mq_try_issue_directly(data.hctx, rq, &cookie);
-	} else if (q->elevator) {
-		blk_mq_put_ctx(data.ctx);
-		blk_mq_bio_to_request(rq, bio);
-		blk_mq_sched_insert_request(rq, false, true, true);
 	} else {
 		blk_mq_put_ctx(data.ctx);
 		blk_mq_bio_to_request(rq, bio);
-		blk_mq_queue_io(data.hctx, data.ctx, rq);
-		blk_mq_run_hw_queue(data.hctx, true);
+		blk_mq_sched_insert_request(rq, false, true, true);
 	}
 
 	return cookie;
@@ -2056,15 +1939,7 @@ static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
 			return ret;
 	}
 
-	seqcount_init(&rq->gstate_seq);
-	u64_stats_init(&rq->aborted_gstate_sync);
-	/*
-	 * start gstate with gen 1 instead of 0, otherwise it will be equal
-	 * to aborted_gstate, and be identified timed out by
-	 * blk_mq_terminate_expired.
-	 */
-	WRITE_ONCE(rq->gstate, MQ_RQ_GEN_INC);
-
+	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
 	return 0;
 }
 
@@ -2365,6 +2240,7 @@ static void blk_mq_map_swqueue(struct request_queue *q)
 	queue_for_each_hw_ctx(q, hctx, i) {
 		cpumask_clear(hctx->cpumask);
 		hctx->nr_ctx = 0;
+		hctx->dispatch_from = NULL;
 	}
 
 	/*
diff --git a/block/blk-mq.h b/block/blk-mq.h
index e1bb420dc5d6..89231e439b2f 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -30,20 +30,6 @@ struct blk_mq_ctx {
 	struct kobject		kobj;
 } ____cacheline_aligned_in_smp;
 
-/*
- * Bits for request->gstate.  The lower two bits carry MQ_RQ_* state value
- * and the upper bits the generation number.
- */
-enum mq_rq_state {
-	MQ_RQ_IDLE		= 0,
-	MQ_RQ_IN_FLIGHT		= 1,
-	MQ_RQ_COMPLETE		= 2,
-
-	MQ_RQ_STATE_BITS	= 2,
-	MQ_RQ_STATE_MASK	= (1 << MQ_RQ_STATE_BITS) - 1,
-	MQ_RQ_GEN_INC		= 1 << MQ_RQ_STATE_BITS,
-};
-
 void blk_mq_freeze_queue(struct request_queue *q);
 void blk_mq_free_queue(struct request_queue *q);
 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
@@ -107,33 +93,9 @@ void blk_mq_release(struct request_queue *q);
  * blk_mq_rq_state() - read the current MQ_RQ_* state of a request
  * @rq: target request.
  */
-static inline int blk_mq_rq_state(struct request *rq)
+static inline enum mq_rq_state blk_mq_rq_state(struct request *rq)
 {
-	return READ_ONCE(rq->gstate) & MQ_RQ_STATE_MASK;
-}
-
-/**
- * blk_mq_rq_update_state() - set the current MQ_RQ_* state of a request
- * @rq: target request.
- * @state: new state to set.
- *
- * Set @rq's state to @state.  The caller is responsible for ensuring that
- * there are no other updaters.  A request can transition into IN_FLIGHT
- * only from IDLE and doing so increments the generation number.
- */
-static inline void blk_mq_rq_update_state(struct request *rq,
-					  enum mq_rq_state state)
-{
-	u64 old_val = READ_ONCE(rq->gstate);
-	u64 new_val = (old_val & ~MQ_RQ_STATE_MASK) | state;
-
-	if (state == MQ_RQ_IN_FLIGHT) {
-		WARN_ON_ONCE((old_val & MQ_RQ_STATE_MASK) != MQ_RQ_IDLE);
-		new_val += MQ_RQ_GEN_INC;
-	}
-
-	/* avoid exposing interim values */
-	WRITE_ONCE(rq->gstate, new_val);
+	return READ_ONCE(rq->state);
 }
 
 static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
diff --git a/block/blk-stat.c b/block/blk-stat.c
index bd365a95fcf8..175c143ac5b9 100644
--- a/block/blk-stat.c
+++ b/block/blk-stat.c
@@ -47,19 +47,15 @@ static void __blk_stat_add(struct blk_rq_stat *stat, u64 value)
 	stat->nr_samples++;
 }
 
-void blk_stat_add(struct request *rq)
+void blk_stat_add(struct request *rq, u64 now)
 {
 	struct request_queue *q = rq->q;
 	struct blk_stat_callback *cb;
 	struct blk_rq_stat *stat;
 	int bucket;
-	u64 now, value;
+	u64 value;
 
-	now = __blk_stat_time(ktime_to_ns(ktime_get()));
-	if (now < blk_stat_time(&rq->issue_stat))
-		return;
-
-	value = now - blk_stat_time(&rq->issue_stat);
+	value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
 
 	blk_throtl_stat_add(rq, value);
 
diff --git a/block/blk-stat.h b/block/blk-stat.h
index 2dd36347252a..78399cdde9c9 100644
--- a/block/blk-stat.h
+++ b/block/blk-stat.h
@@ -8,21 +8,6 @@
 #include <linux/rcupdate.h>
 #include <linux/timer.h>
 
-/*
- * from upper:
- * 3 bits: reserved for other usage
- * 12 bits: size
- * 49 bits: time
- */
-#define BLK_STAT_RES_BITS	3
-#define BLK_STAT_SIZE_BITS	12
-#define BLK_STAT_RES_SHIFT	(64 - BLK_STAT_RES_BITS)
-#define BLK_STAT_SIZE_SHIFT	(BLK_STAT_RES_SHIFT - BLK_STAT_SIZE_BITS)
-#define BLK_STAT_TIME_MASK	((1ULL << BLK_STAT_SIZE_SHIFT) - 1)
-#define BLK_STAT_SIZE_MASK	\
-	(((1ULL << BLK_STAT_SIZE_BITS) - 1) << BLK_STAT_SIZE_SHIFT)
-#define BLK_STAT_RES_MASK	(~((1ULL << BLK_STAT_RES_SHIFT) - 1))
-
 /**
  * struct blk_stat_callback - Block statistics callback.
  *
@@ -80,35 +65,7 @@ struct blk_stat_callback {
 struct blk_queue_stats *blk_alloc_queue_stats(void);
 void blk_free_queue_stats(struct blk_queue_stats *);
 
-void blk_stat_add(struct request *);
-
-static inline u64 __blk_stat_time(u64 time)
-{
-	return time & BLK_STAT_TIME_MASK;
-}
-
-static inline u64 blk_stat_time(struct blk_issue_stat *stat)
-{
-	return __blk_stat_time(stat->stat);
-}
-
-static inline sector_t blk_capped_size(sector_t size)
-{
-	return size & ((1ULL << BLK_STAT_SIZE_BITS) - 1);
-}
-
-static inline sector_t blk_stat_size(struct blk_issue_stat *stat)
-{
-	return (stat->stat & BLK_STAT_SIZE_MASK) >> BLK_STAT_SIZE_SHIFT;
-}
-
-static inline void blk_stat_set_issue(struct blk_issue_stat *stat,
-	sector_t size)
-{
-	stat->stat = (stat->stat & BLK_STAT_RES_MASK) |
-		(ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK) |
-		(((u64)blk_capped_size(size)) << BLK_STAT_SIZE_SHIFT);
-}
+void blk_stat_add(struct request *rq, u64 now);
 
 /* record time/size info in request but not add a callback */
 void blk_stat_enable_accounting(struct request_queue *q);
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index d00d1b0ec109..94987b1f69e1 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -491,188 +491,198 @@ static ssize_t queue_wc_store(struct request_queue *q, const char *page,
 	return count;
 }
 
+static ssize_t queue_fua_show(struct request_queue *q, char *page)
+{
+	return sprintf(page, "%u\n", test_bit(QUEUE_FLAG_FUA, &q->queue_flags));
+}
+
 static ssize_t queue_dax_show(struct request_queue *q, char *page)
 {
 	return queue_var_show(blk_queue_dax(q), page);
 }
 
 static struct queue_sysfs_entry queue_requests_entry = {
-	.attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "nr_requests", .mode = 0644 },
 	.show = queue_requests_show,
 	.store = queue_requests_store,
 };
 
 static struct queue_sysfs_entry queue_ra_entry = {
-	.attr = {.name = "read_ahead_kb", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "read_ahead_kb", .mode = 0644 },
 	.show = queue_ra_show,
 	.store = queue_ra_store,
 };
 
 static struct queue_sysfs_entry queue_max_sectors_entry = {
-	.attr = {.name = "max_sectors_kb", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "max_sectors_kb", .mode = 0644 },
 	.show = queue_max_sectors_show,
 	.store = queue_max_sectors_store,
 };
 
 static struct queue_sysfs_entry queue_max_hw_sectors_entry = {
-	.attr = {.name = "max_hw_sectors_kb", .mode = S_IRUGO },
+	.attr = {.name = "max_hw_sectors_kb", .mode = 0444 },
 	.show = queue_max_hw_sectors_show,
 };
 
 static struct queue_sysfs_entry queue_max_segments_entry = {
-	.attr = {.name = "max_segments", .mode = S_IRUGO },
+	.attr = {.name = "max_segments", .mode = 0444 },
 	.show = queue_max_segments_show,
 };
 
 static struct queue_sysfs_entry queue_max_discard_segments_entry = {
-	.attr = {.name = "max_discard_segments", .mode = S_IRUGO },
+	.attr = {.name = "max_discard_segments", .mode = 0444 },
 	.show = queue_max_discard_segments_show,
 };
 
 static struct queue_sysfs_entry queue_max_integrity_segments_entry = {
-	.attr = {.name = "max_integrity_segments", .mode = S_IRUGO },
+	.attr = {.name = "max_integrity_segments", .mode = 0444 },
 	.show = queue_max_integrity_segments_show,
 };
 
 static struct queue_sysfs_entry queue_max_segment_size_entry = {
-	.attr = {.name = "max_segment_size", .mode = S_IRUGO },
+	.attr = {.name = "max_segment_size", .mode = 0444 },
 	.show = queue_max_segment_size_show,
 };
 
 static struct queue_sysfs_entry queue_iosched_entry = {
-	.attr = {.name = "scheduler", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "scheduler", .mode = 0644 },
 	.show = elv_iosched_show,
 	.store = elv_iosched_store,
 };
 
 static struct queue_sysfs_entry queue_hw_sector_size_entry = {
-	.attr = {.name = "hw_sector_size", .mode = S_IRUGO },
+	.attr = {.name = "hw_sector_size", .mode = 0444 },
 	.show = queue_logical_block_size_show,
 };
 
 static struct queue_sysfs_entry queue_logical_block_size_entry = {
-	.attr = {.name = "logical_block_size", .mode = S_IRUGO },
+	.attr = {.name = "logical_block_size", .mode = 0444 },
 	.show = queue_logical_block_size_show,
 };
 
 static struct queue_sysfs_entry queue_physical_block_size_entry = {
-	.attr = {.name = "physical_block_size", .mode = S_IRUGO },
+	.attr = {.name = "physical_block_size", .mode = 0444 },
 	.show = queue_physical_block_size_show,
 };
 
 static struct queue_sysfs_entry queue_chunk_sectors_entry = {
-	.attr = {.name = "chunk_sectors", .mode = S_IRUGO },
+	.attr = {.name = "chunk_sectors", .mode = 0444 },
 	.show = queue_chunk_sectors_show,
 };
 
 static struct queue_sysfs_entry queue_io_min_entry = {
-	.attr = {.name = "minimum_io_size", .mode = S_IRUGO },
+	.attr = {.name = "minimum_io_size", .mode = 0444 },
 	.show = queue_io_min_show,
 };
 
 static struct queue_sysfs_entry queue_io_opt_entry = {
-	.attr = {.name = "optimal_io_size", .mode = S_IRUGO },
+	.attr = {.name = "optimal_io_size", .mode = 0444 },
 	.show = queue_io_opt_show,
 };
 
 static struct queue_sysfs_entry queue_discard_granularity_entry = {
-	.attr = {.name = "discard_granularity", .mode = S_IRUGO },
+	.attr = {.name = "discard_granularity", .mode = 0444 },
 	.show = queue_discard_granularity_show,
 };
 
 static struct queue_sysfs_entry queue_discard_max_hw_entry = {
-	.attr = {.name = "discard_max_hw_bytes", .mode = S_IRUGO },
+	.attr = {.name = "discard_max_hw_bytes", .mode = 0444 },
 	.show = queue_discard_max_hw_show,
 };
 
 static struct queue_sysfs_entry queue_discard_max_entry = {
-	.attr = {.name = "discard_max_bytes", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "discard_max_bytes", .mode = 0644 },
 	.show = queue_discard_max_show,
 	.store = queue_discard_max_store,
 };
 
 static struct queue_sysfs_entry queue_discard_zeroes_data_entry = {
-	.attr = {.name = "discard_zeroes_data", .mode = S_IRUGO },
+	.attr = {.name = "discard_zeroes_data", .mode = 0444 },
 	.show = queue_discard_zeroes_data_show,
 };
 
 static struct queue_sysfs_entry queue_write_same_max_entry = {
-	.attr = {.name = "write_same_max_bytes", .mode = S_IRUGO },
+	.attr = {.name = "write_same_max_bytes", .mode = 0444 },
 	.show = queue_write_same_max_show,
 };
 
 static struct queue_sysfs_entry queue_write_zeroes_max_entry = {
-	.attr = {.name = "write_zeroes_max_bytes", .mode = S_IRUGO },
+	.attr = {.name = "write_zeroes_max_bytes", .mode = 0444 },
 	.show = queue_write_zeroes_max_show,
 };
 
 static struct queue_sysfs_entry queue_nonrot_entry = {
-	.attr = {.name = "rotational", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "rotational", .mode = 0644 },
 	.show = queue_show_nonrot,
 	.store = queue_store_nonrot,
 };
 
 static struct queue_sysfs_entry queue_zoned_entry = {
-	.attr = {.name = "zoned", .mode = S_IRUGO },
+	.attr = {.name = "zoned", .mode = 0444 },
 	.show = queue_zoned_show,
 };
 
 static struct queue_sysfs_entry queue_nomerges_entry = {
-	.attr = {.name = "nomerges", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "nomerges", .mode = 0644 },
 	.show = queue_nomerges_show,
 	.store = queue_nomerges_store,
 };
 
 static struct queue_sysfs_entry queue_rq_affinity_entry = {
-	.attr = {.name = "rq_affinity", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "rq_affinity", .mode = 0644 },
 	.show = queue_rq_affinity_show,
 	.store = queue_rq_affinity_store,
 };
 
 static struct queue_sysfs_entry queue_iostats_entry = {
-	.attr = {.name = "iostats", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "iostats", .mode = 0644 },
 	.show = queue_show_iostats,
 	.store = queue_store_iostats,
 };
 
 static struct queue_sysfs_entry queue_random_entry = {
-	.attr = {.name = "add_random", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "add_random", .mode = 0644 },
 	.show = queue_show_random,
 	.store = queue_store_random,
 };
 
 static struct queue_sysfs_entry queue_poll_entry = {
-	.attr = {.name = "io_poll", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "io_poll", .mode = 0644 },
 	.show = queue_poll_show,
 	.store = queue_poll_store,
 };
 
 static struct queue_sysfs_entry queue_poll_delay_entry = {
-	.attr = {.name = "io_poll_delay", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "io_poll_delay", .mode = 0644 },
 	.show = queue_poll_delay_show,
 	.store = queue_poll_delay_store,
 };
 
 static struct queue_sysfs_entry queue_wc_entry = {
-	.attr = {.name = "write_cache", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "write_cache", .mode = 0644 },
 	.show = queue_wc_show,
 	.store = queue_wc_store,
 };
 
+static struct queue_sysfs_entry queue_fua_entry = {
+	.attr = {.name = "fua", .mode = 0444 },
+	.show = queue_fua_show,
+};
+
 static struct queue_sysfs_entry queue_dax_entry = {
-	.attr = {.name = "dax", .mode = S_IRUGO },
+	.attr = {.name = "dax", .mode = 0444 },
 	.show = queue_dax_show,
 };
 
 static struct queue_sysfs_entry queue_wb_lat_entry = {
-	.attr = {.name = "wbt_lat_usec", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "wbt_lat_usec", .mode = 0644 },
 	.show = queue_wb_lat_show,
 	.store = queue_wb_lat_store,
 };
 
 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
 static struct queue_sysfs_entry throtl_sample_time_entry = {
-	.attr = {.name = "throttle_sample_time", .mode = S_IRUGO | S_IWUSR },
+	.attr = {.name = "throttle_sample_time", .mode = 0644 },
 	.show = blk_throtl_sample_time_show,
 	.store = blk_throtl_sample_time_store,
 };
@@ -708,6 +718,7 @@ static struct attribute *default_attrs[] = {
 	&queue_random_entry.attr,
 	&queue_poll_entry.attr,
 	&queue_wc_entry.attr,
+	&queue_fua_entry.attr,
 	&queue_dax_entry.attr,
 	&queue_wb_lat_entry.attr,
 	&queue_poll_delay_entry.attr,
@@ -813,8 +824,7 @@ static void __blk_release_queue(struct work_struct *work)
 	if (q->mq_ops)
 		blk_mq_debugfs_unregister(q);
 
-	if (q->bio_split)
-		bioset_free(q->bio_split);
+	bioset_exit(&q->bio_split);
 
 	ida_simple_remove(&blk_queue_ida, q->id);
 	call_rcu(&q->rcu_head, blk_free_queue_rcu);
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index c5a131673733..82282e6fdcf8 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -36,8 +36,6 @@ static int throtl_quantum = 32;
  */
 #define LATENCY_FILTERED_HD (1000L) /* 1ms */
 
-#define SKIP_LATENCY (((u64)1) << BLK_STAT_RES_SHIFT)
-
 static struct blkcg_policy blkcg_policy_throtl;
 
 /* A workqueue to queue throttle related work */
@@ -821,7 +819,7 @@ static bool throtl_slice_used(struct throtl_grp *tg, bool rw)
 	if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw]))
 		return false;
 
-	return 1;
+	return true;
 }
 
 /* Trim the used slices and adjust slice start accordingly */
@@ -931,7 +929,7 @@ static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio,
 
 	if (wait)
 		*wait = jiffy_wait;
-	return 0;
+	return false;
 }
 
 static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,
@@ -974,7 +972,7 @@ static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,
 	jiffy_wait = jiffy_wait + (jiffy_elapsed_rnd - jiffy_elapsed);
 	if (wait)
 		*wait = jiffy_wait;
-	return 0;
+	return false;
 }
 
 /*
@@ -1024,7 +1022,7 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio,
 	    tg_with_in_iops_limit(tg, bio, &iops_wait)) {
 		if (wait)
 			*wait = 0;
-		return 1;
+		return true;
 	}
 
 	max_wait = max(bps_wait, iops_wait);
@@ -1035,7 +1033,7 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio,
 	if (time_before(tg->slice_end[rw], jiffies + max_wait))
 		throtl_extend_slice(tg, rw, jiffies + max_wait);
 
-	return 0;
+	return false;
 }
 
 static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)
@@ -1209,7 +1207,7 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq)
 
 	while (1) {
 		struct throtl_grp *tg = throtl_rb_first(parent_sq);
-		struct throtl_service_queue *sq = &tg->service_queue;
+		struct throtl_service_queue *sq;
 
 		if (!tg)
 			break;
@@ -1221,6 +1219,7 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq)
 
 		nr_disp += throtl_dispatch_tg(tg);
 
+		sq = &tg->service_queue;
 		if (sq->nr_queued[0] || sq->nr_queued[1])
 			tg_update_disptime(tg);
 
@@ -2139,7 +2138,7 @@ static void blk_throtl_assoc_bio(struct throtl_grp *tg, struct bio *bio)
 		bio->bi_cg_private = tg;
 		blkg_get(tg_to_blkg(tg));
 	}
-	blk_stat_set_issue(&bio->bi_issue_stat, bio_sectors(bio));
+	bio_issue_init(&bio->bi_issue, bio_sectors(bio));
 #endif
 }
 
@@ -2251,7 +2250,7 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
 
 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
 	if (throttled || !td->track_bio_latency)
-		bio->bi_issue_stat.stat |= SKIP_LATENCY;
+		bio->bi_issue.value |= BIO_ISSUE_THROTL_SKIP_LATENCY;
 #endif
 	return throttled;
 }
@@ -2281,8 +2280,7 @@ void blk_throtl_stat_add(struct request *rq, u64 time_ns)
 	struct request_queue *q = rq->q;
 	struct throtl_data *td = q->td;
 
-	throtl_track_latency(td, blk_stat_size(&rq->issue_stat),
-		req_op(rq), time_ns >> 10);
+	throtl_track_latency(td, rq->throtl_size, req_op(rq), time_ns >> 10);
 }
 
 void blk_throtl_bio_endio(struct bio *bio)
@@ -2302,8 +2300,8 @@ void blk_throtl_bio_endio(struct bio *bio)
 	finish_time_ns = ktime_get_ns();
 	tg->last_finish_time = finish_time_ns >> 10;
 
-	start_time = blk_stat_time(&bio->bi_issue_stat) >> 10;
-	finish_time = __blk_stat_time(finish_time_ns) >> 10;
+	start_time = bio_issue_time(&bio->bi_issue) >> 10;
+	finish_time = __bio_issue_time(finish_time_ns) >> 10;
 	if (!start_time || finish_time <= start_time) {
 		blkg_put(tg_to_blkg(tg));
 		return;
@@ -2311,16 +2309,15 @@ void blk_throtl_bio_endio(struct bio *bio)
 
 	lat = finish_time - start_time;
 	/* this is only for bio based driver */
-	if (!(bio->bi_issue_stat.stat & SKIP_LATENCY))
-		throtl_track_latency(tg->td, blk_stat_size(&bio->bi_issue_stat),
-			bio_op(bio), lat);
+	if (!(bio->bi_issue.value & BIO_ISSUE_THROTL_SKIP_LATENCY))
+		throtl_track_latency(tg->td, bio_issue_size(&bio->bi_issue),
+				     bio_op(bio), lat);
 
 	if (tg->latency_target && lat >= tg->td->filtered_latency) {
 		int bucket;
 		unsigned int threshold;
 
-		bucket = request_bucket_index(
-			blk_stat_size(&bio->bi_issue_stat));
+		bucket = request_bucket_index(bio_issue_size(&bio->bi_issue));
 		threshold = tg->td->avg_buckets[rw][bucket].latency +
 			tg->latency_target;
 		if (lat > threshold)
diff --git a/block/blk-timeout.c b/block/blk-timeout.c
index 652d4d4d3e97..4b8a48d48ba1 100644
--- a/block/blk-timeout.c
+++ b/block/blk-timeout.c
@@ -86,14 +86,11 @@ static void blk_rq_timed_out(struct request *req)
 	if (q->rq_timed_out_fn)
 		ret = q->rq_timed_out_fn(req);
 	switch (ret) {
-	case BLK_EH_HANDLED:
-		__blk_complete_request(req);
-		break;
 	case BLK_EH_RESET_TIMER:
 		blk_add_timer(req);
 		blk_clear_rq_complete(req);
 		break;
-	case BLK_EH_NOT_HANDLED:
+	case BLK_EH_DONE:
 		/*
 		 * LLD handles this for now but in the future
 		 * we can send a request msg to abort the command
@@ -214,7 +211,6 @@ void blk_add_timer(struct request *req)
 		req->timeout = q->rq_timeout;
 
 	blk_rq_set_deadline(req, jiffies + req->timeout);
-	req->rq_flags &= ~RQF_MQ_TIMEOUT_EXPIRED;
 
 	/*
 	 * Only the non-mq case needs to add the request to a protected list.
diff --git a/block/blk-wbt.c b/block/blk-wbt.c
index f92fc84b5e2c..4f89b28fa652 100644
--- a/block/blk-wbt.c
+++ b/block/blk-wbt.c
@@ -29,6 +29,26 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/wbt.h>
 
+static inline void wbt_clear_state(struct request *rq)
+{
+	rq->wbt_flags = 0;
+}
+
+static inline enum wbt_flags wbt_flags(struct request *rq)
+{
+	return rq->wbt_flags;
+}
+
+static inline bool wbt_is_tracked(struct request *rq)
+{
+	return rq->wbt_flags & WBT_TRACKED;
+}
+
+static inline bool wbt_is_read(struct request *rq)
+{
+	return rq->wbt_flags & WBT_READ;
+}
+
 enum {
 	/*
 	 * Default setting, we'll scale up (to 75% of QD max) or down (min 1)
@@ -101,9 +121,15 @@ static bool wb_recent_wait(struct rq_wb *rwb)
 	return time_before(jiffies, wb->dirty_sleep + HZ);
 }
 
-static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb, bool is_kswapd)
+static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb,
+					  enum wbt_flags wb_acct)
 {
-	return &rwb->rq_wait[is_kswapd];
+	if (wb_acct & WBT_KSWAPD)
+		return &rwb->rq_wait[WBT_RWQ_KSWAPD];
+	else if (wb_acct & WBT_DISCARD)
+		return &rwb->rq_wait[WBT_RWQ_DISCARD];
+
+	return &rwb->rq_wait[WBT_RWQ_BG];
 }
 
 static void rwb_wake_all(struct rq_wb *rwb)
@@ -126,7 +152,7 @@ void __wbt_done(struct rq_wb *rwb, enum wbt_flags wb_acct)
 	if (!(wb_acct & WBT_TRACKED))
 		return;
 
-	rqw = get_rq_wait(rwb, wb_acct & WBT_KSWAPD);
+	rqw = get_rq_wait(rwb, wb_acct);
 	inflight = atomic_dec_return(&rqw->inflight);
 
 	/*
@@ -139,10 +165,13 @@ void __wbt_done(struct rq_wb *rwb, enum wbt_flags wb_acct)
 	}
 
 	/*
-	 * If the device does write back caching, drop further down
-	 * before we wake people up.
+	 * For discards, our limit is always the background. For writes, if
+	 * the device does write back caching, drop further down before we
+	 * wake people up.
 	 */
-	if (rwb->wc && !wb_recent_wait(rwb))
+	if (wb_acct & WBT_DISCARD)
+		limit = rwb->wb_background;
+	else if (rwb->wc && !wb_recent_wait(rwb))
 		limit = 0;
 	else
 		limit = rwb->wb_normal;
@@ -165,24 +194,24 @@ void __wbt_done(struct rq_wb *rwb, enum wbt_flags wb_acct)
  * Called on completion of a request. Note that it's also called when
  * a request is merged, when the request gets freed.
  */
-void wbt_done(struct rq_wb *rwb, struct blk_issue_stat *stat)
+void wbt_done(struct rq_wb *rwb, struct request *rq)
 {
 	if (!rwb)
 		return;
 
-	if (!wbt_is_tracked(stat)) {
-		if (rwb->sync_cookie == stat) {
+	if (!wbt_is_tracked(rq)) {
+		if (rwb->sync_cookie == rq) {
 			rwb->sync_issue = 0;
 			rwb->sync_cookie = NULL;
 		}
 
-		if (wbt_is_read(stat))
+		if (wbt_is_read(rq))
 			wb_timestamp(rwb, &rwb->last_comp);
 	} else {
-		WARN_ON_ONCE(stat == rwb->sync_cookie);
-		__wbt_done(rwb, wbt_stat_to_mask(stat));
+		WARN_ON_ONCE(rq == rwb->sync_cookie);
+		__wbt_done(rwb, wbt_flags(rq));
 	}
-	wbt_clear_state(stat);
+	wbt_clear_state(rq);
 }
 
 /*
@@ -479,6 +508,9 @@ static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw)
 {
 	unsigned int limit;
 
+	if ((rw & REQ_OP_MASK) == REQ_OP_DISCARD)
+		return rwb->wb_background;
+
 	/*
 	 * At this point we know it's a buffered write. If this is
 	 * kswapd trying to free memory, or REQ_SYNC is set, then
@@ -529,11 +561,12 @@ static inline bool may_queue(struct rq_wb *rwb, struct rq_wait *rqw,
  * Block if we will exceed our limit, or if we are currently waiting for
  * the timer to kick off queuing again.
  */
-static void __wbt_wait(struct rq_wb *rwb, unsigned long rw, spinlock_t *lock)
+static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct,
+		       unsigned long rw, spinlock_t *lock)
 	__releases(lock)
 	__acquires(lock)
 {
-	struct rq_wait *rqw = get_rq_wait(rwb, current_is_kswapd());
+	struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);
 	DEFINE_WAIT(wait);
 
 	if (may_queue(rwb, rqw, &wait, rw))
@@ -559,21 +592,20 @@ static void __wbt_wait(struct rq_wb *rwb, unsigned long rw, spinlock_t *lock)
 
 static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)
 {
-	const int op = bio_op(bio);
-
-	/*
-	 * If not a WRITE, do nothing
-	 */
-	if (op != REQ_OP_WRITE)
-		return false;
-
-	/*
-	 * Don't throttle WRITE_ODIRECT
-	 */
-	if ((bio->bi_opf & (REQ_SYNC | REQ_IDLE)) == (REQ_SYNC | REQ_IDLE))
+	switch (bio_op(bio)) {
+	case REQ_OP_WRITE:
+		/*
+		 * Don't throttle WRITE_ODIRECT
+		 */
+		if ((bio->bi_opf & (REQ_SYNC | REQ_IDLE)) ==
+		    (REQ_SYNC | REQ_IDLE))
+			return false;
+		/* fallthrough */
+	case REQ_OP_DISCARD:
+		return true;
+	default:
 		return false;
-
-	return true;
+	}
 }
 
 /*
@@ -584,7 +616,7 @@ static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)
  */
 enum wbt_flags wbt_wait(struct rq_wb *rwb, struct bio *bio, spinlock_t *lock)
 {
-	unsigned int ret = 0;
+	enum wbt_flags ret = 0;
 
 	if (!rwb_enabled(rwb))
 		return 0;
@@ -598,41 +630,42 @@ enum wbt_flags wbt_wait(struct rq_wb *rwb, struct bio *bio, spinlock_t *lock)
 		return ret;
 	}
 
-	__wbt_wait(rwb, bio->bi_opf, lock);
+	if (current_is_kswapd())
+		ret |= WBT_KSWAPD;
+	if (bio_op(bio) == REQ_OP_DISCARD)
+		ret |= WBT_DISCARD;
+
+	__wbt_wait(rwb, ret, bio->bi_opf, lock);
 
 	if (!blk_stat_is_active(rwb->cb))
 		rwb_arm_timer(rwb);
 
-	if (current_is_kswapd())
-		ret |= WBT_KSWAPD;
-
 	return ret | WBT_TRACKED;
 }
 
-void wbt_issue(struct rq_wb *rwb, struct blk_issue_stat *stat)
+void wbt_issue(struct rq_wb *rwb, struct request *rq)
 {
 	if (!rwb_enabled(rwb))
 		return;
 
 	/*
-	 * Track sync issue, in case it takes a long time to complete. Allows
-	 * us to react quicker, if a sync IO takes a long time to complete.
-	 * Note that this is just a hint. 'stat' can go away when the
-	 * request completes, so it's important we never dereference it. We
-	 * only use the address to compare with, which is why we store the
-	 * sync_issue time locally.
+	 * Track sync issue, in case it takes a long time to complete. Allows us
+	 * to react quicker, if a sync IO takes a long time to complete. Note
+	 * that this is just a hint. The request can go away when it completes,
+	 * so it's important we never dereference it. We only use the address to
+	 * compare with, which is why we store the sync_issue time locally.
 	 */
-	if (wbt_is_read(stat) && !rwb->sync_issue) {
-		rwb->sync_cookie = stat;
-		rwb->sync_issue = blk_stat_time(stat);
+	if (wbt_is_read(rq) && !rwb->sync_issue) {
+		rwb->sync_cookie = rq;
+		rwb->sync_issue = rq->io_start_time_ns;
 	}
 }
 
-void wbt_requeue(struct rq_wb *rwb, struct blk_issue_stat *stat)
+void wbt_requeue(struct rq_wb *rwb, struct request *rq)
 {
 	if (!rwb_enabled(rwb))
 		return;
-	if (stat == rwb->sync_cookie) {
+	if (rq == rwb->sync_cookie) {
 		rwb->sync_issue = 0;
 		rwb->sync_cookie = NULL;
 	}
@@ -701,7 +734,7 @@ static int wbt_data_dir(const struct request *rq)
 
 	if (op == REQ_OP_READ)
 		return READ;
-	else if (op == REQ_OP_WRITE || op == REQ_OP_FLUSH)
+	else if (op_is_write(op))
 		return WRITE;
 
 	/* don't account */
@@ -713,8 +746,6 @@ int wbt_init(struct request_queue *q)
 	struct rq_wb *rwb;
 	int i;
 
-	BUILD_BUG_ON(WBT_NR_BITS > BLK_STAT_RES_BITS);
-
 	rwb = kzalloc(sizeof(*rwb), GFP_KERNEL);
 	if (!rwb)
 		return -ENOMEM;
diff --git a/block/blk-wbt.h b/block/blk-wbt.h
index a232c98fbf4d..300df531d0a6 100644
--- a/block/blk-wbt.h
+++ b/block/blk-wbt.h
@@ -14,12 +14,16 @@ enum wbt_flags {
 	WBT_TRACKED		= 1,	/* write, tracked for throttling */
 	WBT_READ		= 2,	/* read */
 	WBT_KSWAPD		= 4,	/* write, from kswapd */
+	WBT_DISCARD		= 8,	/* discard */
 
-	WBT_NR_BITS		= 3,	/* number of bits */
+	WBT_NR_BITS		= 4,	/* number of bits */
 };
 
 enum {
-	WBT_NUM_RWQ		= 2,
+	WBT_RWQ_BG		= 0,
+	WBT_RWQ_KSWAPD,
+	WBT_RWQ_DISCARD,
+	WBT_NUM_RWQ,
 };
 
 /*
@@ -31,31 +35,6 @@ enum {
 	WBT_STATE_ON_MANUAL	= 2,
 };
 
-static inline void wbt_clear_state(struct blk_issue_stat *stat)
-{
-	stat->stat &= ~BLK_STAT_RES_MASK;
-}
-
-static inline enum wbt_flags wbt_stat_to_mask(struct blk_issue_stat *stat)
-{
-	return (stat->stat & BLK_STAT_RES_MASK) >> BLK_STAT_RES_SHIFT;
-}
-
-static inline void wbt_track(struct blk_issue_stat *stat, enum wbt_flags wb_acct)
-{
-	stat->stat |= ((u64) wb_acct) << BLK_STAT_RES_SHIFT;
-}
-
-static inline bool wbt_is_tracked(struct blk_issue_stat *stat)
-{
-	return (stat->stat >> BLK_STAT_RES_SHIFT) & WBT_TRACKED;
-}
-
-static inline bool wbt_is_read(struct blk_issue_stat *stat)
-{
-	return (stat->stat >> BLK_STAT_RES_SHIFT) & WBT_READ;
-}
-
 struct rq_wait {
 	wait_queue_head_t wait;
 	atomic_t inflight;
@@ -84,7 +63,7 @@ struct rq_wb {
 
 	struct blk_stat_callback *cb;
 
-	s64 sync_issue;
+	u64 sync_issue;
 	void *sync_cookie;
 
 	unsigned int wc;
@@ -109,14 +88,19 @@ static inline unsigned int wbt_inflight(struct rq_wb *rwb)
 
 #ifdef CONFIG_BLK_WBT
 
+static inline void wbt_track(struct request *rq, enum wbt_flags flags)
+{
+	rq->wbt_flags |= flags;
+}
+
 void __wbt_done(struct rq_wb *, enum wbt_flags);
-void wbt_done(struct rq_wb *, struct blk_issue_stat *);
+void wbt_done(struct rq_wb *, struct request *);
 enum wbt_flags wbt_wait(struct rq_wb *, struct bio *, spinlock_t *);
 int wbt_init(struct request_queue *);
 void wbt_exit(struct request_queue *);
 void wbt_update_limits(struct rq_wb *);
-void wbt_requeue(struct rq_wb *, struct blk_issue_stat *);
-void wbt_issue(struct rq_wb *, struct blk_issue_stat *);
+void wbt_requeue(struct rq_wb *, struct request *);
+void wbt_issue(struct rq_wb *, struct request *);
 void wbt_disable_default(struct request_queue *);
 void wbt_enable_default(struct request_queue *);
 
@@ -127,10 +111,13 @@ u64 wbt_default_latency_nsec(struct request_queue *);
 
 #else
 
+static inline void wbt_track(struct request *rq, enum wbt_flags flags)
+{
+}
 static inline void __wbt_done(struct rq_wb *rwb, enum wbt_flags flags)
 {
 }
-static inline void wbt_done(struct rq_wb *rwb, struct blk_issue_stat *stat)
+static inline void wbt_done(struct rq_wb *rwb, struct request *rq)
 {
 }
 static inline enum wbt_flags wbt_wait(struct rq_wb *rwb, struct bio *bio,
@@ -148,10 +135,10 @@ static inline void wbt_exit(struct request_queue *q)
 static inline void wbt_update_limits(struct rq_wb *rwb)
 {
 }
-static inline void wbt_requeue(struct rq_wb *rwb, struct blk_issue_stat *stat)
+static inline void wbt_requeue(struct rq_wb *rwb, struct request *rq)
 {
 }
-static inline void wbt_issue(struct rq_wb *rwb, struct blk_issue_stat *stat)
+static inline void wbt_issue(struct rq_wb *rwb, struct request *rq)
 {
 }
 static inline void wbt_disable_default(struct request_queue *q)
diff --git a/block/blk-zoned.c b/block/blk-zoned.c
index 08e84ef2bc05..3d08dc84db16 100644
--- a/block/blk-zoned.c
+++ b/block/blk-zoned.c
@@ -328,7 +328,11 @@ int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
 	if (!rep.nr_zones)
 		return -EINVAL;
 
-	zones = kcalloc(rep.nr_zones, sizeof(struct blk_zone), GFP_KERNEL);
+	if (rep.nr_zones > INT_MAX / sizeof(struct blk_zone))
+		return -ERANGE;
+
+	zones = kvmalloc(rep.nr_zones * sizeof(struct blk_zone),
+			GFP_KERNEL | __GFP_ZERO);
 	if (!zones)
 		return -ENOMEM;
 
@@ -350,7 +354,7 @@ int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
 	}
 
  out:
-	kfree(zones);
+	kvfree(zones);
 
 	return ret;
 }
diff --git a/block/blk.h b/block/blk.h
index b034fd2460c4..eaf1a8e87d11 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -186,7 +186,7 @@ unsigned int blk_plug_queued_count(struct request_queue *q);
 
 void blk_account_io_start(struct request *req, bool new_io);
 void blk_account_io_completion(struct request *req, unsigned int bytes);
-void blk_account_io_done(struct request *req);
+void blk_account_io_done(struct request *req, u64 now);
 
 /*
  * EH timer and IO completion will both attempt to 'grab' the request, make
diff --git a/block/bounce.c b/block/bounce.c
index dd0b93f2a871..fd31347b7836 100644
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -28,28 +28,29 @@
 #define POOL_SIZE	64
 #define ISA_POOL_SIZE	16
 
-static struct bio_set *bounce_bio_set, *bounce_bio_split;
-static mempool_t *page_pool, *isa_page_pool;
+static struct bio_set bounce_bio_set, bounce_bio_split;
+static mempool_t page_pool, isa_page_pool;
 
 #if defined(CONFIG_HIGHMEM)
 static __init int init_emergency_pool(void)
 {
+	int ret;
 #if defined(CONFIG_HIGHMEM) && !defined(CONFIG_MEMORY_HOTPLUG)
 	if (max_pfn <= max_low_pfn)
 		return 0;
 #endif
 
-	page_pool = mempool_create_page_pool(POOL_SIZE, 0);
-	BUG_ON(!page_pool);
+	ret = mempool_init_page_pool(&page_pool, POOL_SIZE, 0);
+	BUG_ON(ret);
 	pr_info("pool size: %d pages\n", POOL_SIZE);
 
-	bounce_bio_set = bioset_create(BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
-	BUG_ON(!bounce_bio_set);
-	if (bioset_integrity_create(bounce_bio_set, BIO_POOL_SIZE))
+	ret = bioset_init(&bounce_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
+	BUG_ON(ret);
+	if (bioset_integrity_create(&bounce_bio_set, BIO_POOL_SIZE))
 		BUG_ON(1);
 
-	bounce_bio_split = bioset_create(BIO_POOL_SIZE, 0, 0);
-	BUG_ON(!bounce_bio_split);
+	ret = bioset_init(&bounce_bio_split, BIO_POOL_SIZE, 0, 0);
+	BUG_ON(ret);
 
 	return 0;
 }
@@ -63,14 +64,11 @@ __initcall(init_emergency_pool);
  */
 static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom)
 {
-	unsigned long flags;
 	unsigned char *vto;
 
-	local_irq_save(flags);
 	vto = kmap_atomic(to->bv_page);
 	memcpy(vto + to->bv_offset, vfrom, to->bv_len);
 	kunmap_atomic(vto);
-	local_irq_restore(flags);
 }
 
 #else /* CONFIG_HIGHMEM */
@@ -94,12 +92,14 @@ static void *mempool_alloc_pages_isa(gfp_t gfp_mask, void *data)
  */
 int init_emergency_isa_pool(void)
 {
-	if (isa_page_pool)
+	int ret;
+
+	if (mempool_initialized(&isa_page_pool))
 		return 0;
 
-	isa_page_pool = mempool_create(ISA_POOL_SIZE, mempool_alloc_pages_isa,
-				       mempool_free_pages, (void *) 0);
-	BUG_ON(!isa_page_pool);
+	ret = mempool_init(&isa_page_pool, ISA_POOL_SIZE, mempool_alloc_pages_isa,
+			   mempool_free_pages, (void *) 0);
+	BUG_ON(ret);
 
 	pr_info("isa pool size: %d pages\n", ISA_POOL_SIZE);
 	return 0;
@@ -166,13 +166,13 @@ static void bounce_end_io(struct bio *bio, mempool_t *pool)
 
 static void bounce_end_io_write(struct bio *bio)
 {
-	bounce_end_io(bio, page_pool);
+	bounce_end_io(bio, &page_pool);
 }
 
 static void bounce_end_io_write_isa(struct bio *bio)
 {
 
-	bounce_end_io(bio, isa_page_pool);
+	bounce_end_io(bio, &isa_page_pool);
 }
 
 static void __bounce_end_io_read(struct bio *bio, mempool_t *pool)
@@ -187,12 +187,12 @@ static void __bounce_end_io_read(struct bio *bio, mempool_t *pool)
 
 static void bounce_end_io_read(struct bio *bio)
 {
-	__bounce_end_io_read(bio, page_pool);
+	__bounce_end_io_read(bio, &page_pool);
 }
 
 static void bounce_end_io_read_isa(struct bio *bio)
 {
-	__bounce_end_io_read(bio, isa_page_pool);
+	__bounce_end_io_read(bio, &isa_page_pool);
 }
 
 static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
@@ -217,13 +217,13 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
 		return;
 
 	if (!passthrough && sectors < bio_sectors(*bio_orig)) {
-		bio = bio_split(*bio_orig, sectors, GFP_NOIO, bounce_bio_split);
+		bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split);
 		bio_chain(bio, *bio_orig);
 		generic_make_request(*bio_orig);
 		*bio_orig = bio;
 	}
 	bio = bio_clone_bioset(*bio_orig, GFP_NOIO, passthrough ? NULL :
-			bounce_bio_set);
+			&bounce_bio_set);
 
 	bio_for_each_segment_all(to, bio, i) {
 		struct page *page = to->bv_page;
@@ -250,7 +250,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
 
 	bio->bi_flags |= (1 << BIO_BOUNCED);
 
-	if (pool == page_pool) {
+	if (pool == &page_pool) {
 		bio->bi_end_io = bounce_end_io_write;
 		if (rw == READ)
 			bio->bi_end_io = bounce_end_io_read;
@@ -282,10 +282,10 @@ void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
 	if (!(q->bounce_gfp & GFP_DMA)) {
 		if (q->limits.bounce_pfn >= blk_max_pfn)
 			return;
-		pool = page_pool;
+		pool = &page_pool;
 	} else {
-		BUG_ON(!isa_page_pool);
-		pool = isa_page_pool;
+		BUG_ON(!mempool_initialized(&isa_page_pool));
+		pool = &isa_page_pool;
 	}
 
 	/*
diff --git a/block/bsg-lib.c b/block/bsg-lib.c
index fc2e5ff2c4b9..9419def8c017 100644
--- a/block/bsg-lib.c
+++ b/block/bsg-lib.c
@@ -303,11 +303,9 @@ static void bsg_exit_rq(struct request_queue *q, struct request *req)
  * @name: device to give bsg device
  * @job_fn: bsg job handler
  * @dd_job_size: size of LLD data needed for each job
- * @release: @dev release function
  */
 struct request_queue *bsg_setup_queue(struct device *dev, const char *name,
-		bsg_job_fn *job_fn, int dd_job_size,
-		void (*release)(struct device *))
+		bsg_job_fn *job_fn, int dd_job_size)
 {
 	struct request_queue *q;
 	int ret;
@@ -331,7 +329,7 @@ struct request_queue *bsg_setup_queue(struct device *dev, const char *name,
 	blk_queue_softirq_done(q, bsg_softirq_done);
 	blk_queue_rq_timeout(q, BLK_DEFAULT_SG_TIMEOUT);
 
-	ret = bsg_register_queue(q, dev, name, &bsg_transport_ops, release);
+	ret = bsg_register_queue(q, dev, name, &bsg_transport_ops);
 	if (ret) {
 		printk(KERN_ERR "%s: bsg interface failed to "
 		       "initialize - register queue\n", dev->kobj.name);
diff --git a/block/bsg.c b/block/bsg.c
index defa06c11858..132e657e2d91 100644
--- a/block/bsg.c
+++ b/block/bsg.c
@@ -226,8 +226,7 @@ bsg_map_hdr(struct request_queue *q, struct sg_io_v4 *hdr, fmode_t mode)
 		return ERR_PTR(ret);
 
 	rq = blk_get_request(q, hdr->dout_xfer_len ?
-			REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN,
-			GFP_KERNEL);
+			REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, 0);
 	if (IS_ERR(rq))
 		return rq;
 
@@ -249,7 +248,7 @@ bsg_map_hdr(struct request_queue *q, struct sg_io_v4 *hdr, fmode_t mode)
 			goto out;
 		}
 
-		next_rq = blk_get_request(q, REQ_OP_SCSI_IN, GFP_KERNEL);
+		next_rq = blk_get_request(q, REQ_OP_SCSI_IN, 0);
 		if (IS_ERR(next_rq)) {
 			ret = PTR_ERR(next_rq);
 			goto out;
@@ -650,18 +649,6 @@ static struct bsg_device *bsg_alloc_device(void)
 	return bd;
 }
 
-static void bsg_kref_release_function(struct kref *kref)
-{
-	struct bsg_class_device *bcd =
-		container_of(kref, struct bsg_class_device, ref);
-	struct device *parent = bcd->parent;
-
-	if (bcd->release)
-		bcd->release(bcd->parent);
-
-	put_device(parent);
-}
-
 static int bsg_put_device(struct bsg_device *bd)
 {
 	int ret = 0, do_free;
@@ -694,7 +681,6 @@ static int bsg_put_device(struct bsg_device *bd)
 
 	kfree(bd);
 out:
-	kref_put(&q->bsg_dev.ref, bsg_kref_release_function);
 	if (do_free)
 		blk_put_queue(q);
 	return ret;
@@ -760,8 +746,6 @@ static struct bsg_device *bsg_get_device(struct inode *inode, struct file *file)
 	 */
 	mutex_lock(&bsg_mutex);
 	bcd = idr_find(&bsg_minor_idr, iminor(inode));
-	if (bcd)
-		kref_get(&bcd->ref);
 	mutex_unlock(&bsg_mutex);
 
 	if (!bcd)
@@ -772,8 +756,6 @@ static struct bsg_device *bsg_get_device(struct inode *inode, struct file *file)
 		return bd;
 
 	bd = bsg_add_device(inode, bcd->queue, file);
-	if (IS_ERR(bd))
-		kref_put(&bcd->ref, bsg_kref_release_function);
 
 	return bd;
 }
@@ -913,25 +895,17 @@ void bsg_unregister_queue(struct request_queue *q)
 		sysfs_remove_link(&q->kobj, "bsg");
 	device_unregister(bcd->class_dev);
 	bcd->class_dev = NULL;
-	kref_put(&bcd->ref, bsg_kref_release_function);
 	mutex_unlock(&bsg_mutex);
 }
 EXPORT_SYMBOL_GPL(bsg_unregister_queue);
 
 int bsg_register_queue(struct request_queue *q, struct device *parent,
-		const char *name, const struct bsg_ops *ops,
-		void (*release)(struct device *))
+		const char *name, const struct bsg_ops *ops)
 {
 	struct bsg_class_device *bcd;
 	dev_t dev;
 	int ret;
 	struct device *class_dev = NULL;
-	const char *devname;
-
-	if (name)
-		devname = name;
-	else
-		devname = dev_name(parent);
 
 	/*
 	 * we need a proper transport to send commands, not a stacked device
@@ -955,15 +929,12 @@ int bsg_register_queue(struct request_queue *q, struct device *parent,
 
 	bcd->minor = ret;
 	bcd->queue = q;
-	bcd->parent = get_device(parent);
-	bcd->release = release;
 	bcd->ops = ops;
-	kref_init(&bcd->ref);
 	dev = MKDEV(bsg_major, bcd->minor);
-	class_dev = device_create(bsg_class, parent, dev, NULL, "%s", devname);
+	class_dev = device_create(bsg_class, parent, dev, NULL, "%s", name);
 	if (IS_ERR(class_dev)) {
 		ret = PTR_ERR(class_dev);
-		goto put_dev;
+		goto idr_remove;
 	}
 	bcd->class_dev = class_dev;
 
@@ -978,8 +949,7 @@ int bsg_register_queue(struct request_queue *q, struct device *parent,
 
 unregister_class_dev:
 	device_unregister(class_dev);
-put_dev:
-	put_device(parent);
+idr_remove:
 	idr_remove(&bsg_minor_idr, bcd->minor);
 unlock:
 	mutex_unlock(&bsg_mutex);
@@ -993,7 +963,7 @@ int bsg_scsi_register_queue(struct request_queue *q, struct device *parent)
 		return -EINVAL;
 	}
 
-	return bsg_register_queue(q, parent, NULL, &bsg_scsi_ops, NULL);
+	return bsg_register_queue(q, parent, dev_name(parent), &bsg_scsi_ops);
 }
 EXPORT_SYMBOL_GPL(bsg_scsi_register_queue);
 
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 9f342ef1ad42..82b6c27b3245 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -210,9 +210,9 @@ struct cfqg_stats {
 	/* total time with empty current active q with other requests queued */
 	struct blkg_stat		empty_time;
 	/* fields after this shouldn't be cleared on stat reset */
-	uint64_t			start_group_wait_time;
-	uint64_t			start_idle_time;
-	uint64_t			start_empty_time;
+	u64				start_group_wait_time;
+	u64				start_idle_time;
+	u64				start_empty_time;
 	uint16_t			flags;
 #endif	/* CONFIG_DEBUG_BLK_CGROUP */
 #endif	/* CONFIG_CFQ_GROUP_IOSCHED */
@@ -491,13 +491,13 @@ CFQG_FLAG_FNS(empty)
 /* This should be called with the queue_lock held. */
 static void cfqg_stats_update_group_wait_time(struct cfqg_stats *stats)
 {
-	unsigned long long now;
+	u64 now;
 
 	if (!cfqg_stats_waiting(stats))
 		return;
 
-	now = sched_clock();
-	if (time_after64(now, stats->start_group_wait_time))
+	now = ktime_get_ns();
+	if (now > stats->start_group_wait_time)
 		blkg_stat_add(&stats->group_wait_time,
 			      now - stats->start_group_wait_time);
 	cfqg_stats_clear_waiting(stats);
@@ -513,20 +513,20 @@ static void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg,
 		return;
 	if (cfqg == curr_cfqg)
 		return;
-	stats->start_group_wait_time = sched_clock();
+	stats->start_group_wait_time = ktime_get_ns();
 	cfqg_stats_mark_waiting(stats);
 }
 
 /* This should be called with the queue_lock held. */
 static void cfqg_stats_end_empty_time(struct cfqg_stats *stats)
 {
-	unsigned long long now;
+	u64 now;
 
 	if (!cfqg_stats_empty(stats))
 		return;
 
-	now = sched_clock();
-	if (time_after64(now, stats->start_empty_time))
+	now = ktime_get_ns();
+	if (now > stats->start_empty_time)
 		blkg_stat_add(&stats->empty_time,
 			      now - stats->start_empty_time);
 	cfqg_stats_clear_empty(stats);
@@ -552,7 +552,7 @@ static void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg)
 	if (cfqg_stats_empty(stats))
 		return;
 
-	stats->start_empty_time = sched_clock();
+	stats->start_empty_time = ktime_get_ns();
 	cfqg_stats_mark_empty(stats);
 }
 
@@ -561,9 +561,9 @@ static void cfqg_stats_update_idle_time(struct cfq_group *cfqg)
 	struct cfqg_stats *stats = &cfqg->stats;
 
 	if (cfqg_stats_idling(stats)) {
-		unsigned long long now = sched_clock();
+		u64 now = ktime_get_ns();
 
-		if (time_after64(now, stats->start_idle_time))
+		if (now > stats->start_idle_time)
 			blkg_stat_add(&stats->idle_time,
 				      now - stats->start_idle_time);
 		cfqg_stats_clear_idling(stats);
@@ -576,7 +576,7 @@ static void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg)
 
 	BUG_ON(cfqg_stats_idling(stats));
 
-	stats->start_idle_time = sched_clock();
+	stats->start_idle_time = ktime_get_ns();
 	cfqg_stats_mark_idling(stats);
 }
 
@@ -701,17 +701,19 @@ static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg,
 }
 
 static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
-			uint64_t start_time, uint64_t io_start_time,
-			unsigned int op)
+						u64 start_time_ns,
+						u64 io_start_time_ns,
+						unsigned int op)
 {
 	struct cfqg_stats *stats = &cfqg->stats;
-	unsigned long long now = sched_clock();
+	u64 now = ktime_get_ns();
 
-	if (time_after64(now, io_start_time))
-		blkg_rwstat_add(&stats->service_time, op, now - io_start_time);
-	if (time_after64(io_start_time, start_time))
+	if (now > io_start_time_ns)
+		blkg_rwstat_add(&stats->service_time, op,
+				now - io_start_time_ns);
+	if (io_start_time_ns > start_time_ns)
 		blkg_rwstat_add(&stats->wait_time, op,
-				io_start_time - start_time);
+				io_start_time_ns - start_time_ns);
 }
 
 /* @stats = 0 */
@@ -797,8 +799,9 @@ static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg,
 static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg,
 			unsigned int op) { }
 static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
-			uint64_t start_time, uint64_t io_start_time,
-			unsigned int op) { }
+						u64 start_time_ns,
+						u64 io_start_time_ns,
+						unsigned int op) { }
 
 #endif	/* CONFIG_CFQ_GROUP_IOSCHED */
 
@@ -4225,8 +4228,8 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
 	cfqd->rq_in_driver--;
 	cfqq->dispatched--;
 	(RQ_CFQG(rq))->dispatched--;
-	cfqg_stats_update_completion(cfqq->cfqg, rq_start_time_ns(rq),
-				     rq_io_start_time_ns(rq), rq->cmd_flags);
+	cfqg_stats_update_completion(cfqq->cfqg, rq->start_time_ns,
+				     rq->io_start_time_ns, rq->cmd_flags);
 
 	cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--;
 
@@ -4242,16 +4245,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
 					cfqq_type(cfqq));
 
 		st->ttime.last_end_request = now;
-		/*
-		 * We have to do this check in jiffies since start_time is in
-		 * jiffies and it is not trivial to convert to ns. If
-		 * cfq_fifo_expire[1] ever comes close to 1 jiffie, this test
-		 * will become problematic but so far we are fine (the default
-		 * is 128 ms).
-		 */
-		if (!time_after(rq->start_time +
-				  nsecs_to_jiffies(cfqd->cfq_fifo_expire[1]),
-				jiffies))
+		if (rq->start_time_ns + cfqd->cfq_fifo_expire[1] <= now)
 			cfqd->last_delayed_sync = now;
 	}
 
@@ -4792,7 +4786,7 @@ USEC_STORE_FUNCTION(cfq_target_latency_us_store, &cfqd->cfq_target_latency, 1, U
 #undef USEC_STORE_FUNCTION
 
 #define CFQ_ATTR(name) \
-	__ATTR(name, S_IRUGO|S_IWUSR, cfq_##name##_show, cfq_##name##_store)
+	__ATTR(name, 0644, cfq_##name##_show, cfq_##name##_store)
 
 static struct elv_fs_entry cfq_attrs[] = {
 	CFQ_ATTR(quantum),
diff --git a/block/deadline-iosched.c b/block/deadline-iosched.c
index 9de9f156e203..ef2f1f09e9b3 100644
--- a/block/deadline-iosched.c
+++ b/block/deadline-iosched.c
@@ -512,8 +512,7 @@ STORE_FUNCTION(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX, 0);
 #undef STORE_FUNCTION
 
 #define DD_ATTR(name) \
-	__ATTR(name, S_IRUGO|S_IWUSR, deadline_##name##_show, \
-				      deadline_##name##_store)
+	__ATTR(name, 0644, deadline_##name##_show, deadline_##name##_store)
 
 static struct elv_fs_entry deadline_attrs[] = {
 	DD_ATTR(read_expire),
diff --git a/block/genhd.c b/block/genhd.c
index c4513fe1adda..cb65a3013480 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -1003,7 +1003,7 @@ static int show_partition(struct seq_file *seqf, void *v)
 	char buf[BDEVNAME_SIZE];
 
 	/* Don't show non-partitionable removeable devices or empty devices */
-	if (!get_capacity(sgp) || (!disk_max_parts(sgp) &&
+	if (!get_capacity(sgp) || (!(disk_max_parts(sgp) > 1) &&
 				   (sgp->flags & GENHD_FL_REMOVABLE)))
 		return 0;
 	if (sgp->flags & GENHD_FL_SUPPRESS_PARTITION_INFO)
@@ -1027,18 +1027,6 @@ static const struct seq_operations partitions_op = {
 	.stop	= disk_seqf_stop,
 	.show	= show_partition
 };
-
-static int partitions_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &partitions_op);
-}
-
-static const struct file_operations proc_partitions_operations = {
-	.open		= partitions_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
 #endif
 
 
@@ -1139,28 +1127,25 @@ static ssize_t disk_discard_alignment_show(struct device *dev,
 	return sprintf(buf, "%d\n", queue_discard_alignment(disk->queue));
 }
 
-static DEVICE_ATTR(range, S_IRUGO, disk_range_show, NULL);
-static DEVICE_ATTR(ext_range, S_IRUGO, disk_ext_range_show, NULL);
-static DEVICE_ATTR(removable, S_IRUGO, disk_removable_show, NULL);
-static DEVICE_ATTR(hidden, S_IRUGO, disk_hidden_show, NULL);
-static DEVICE_ATTR(ro, S_IRUGO, disk_ro_show, NULL);
-static DEVICE_ATTR(size, S_IRUGO, part_size_show, NULL);
-static DEVICE_ATTR(alignment_offset, S_IRUGO, disk_alignment_offset_show, NULL);
-static DEVICE_ATTR(discard_alignment, S_IRUGO, disk_discard_alignment_show,
-		   NULL);
-static DEVICE_ATTR(capability, S_IRUGO, disk_capability_show, NULL);
-static DEVICE_ATTR(stat, S_IRUGO, part_stat_show, NULL);
-static DEVICE_ATTR(inflight, S_IRUGO, part_inflight_show, NULL);
-static DEVICE_ATTR(badblocks, S_IRUGO | S_IWUSR, disk_badblocks_show,
-		disk_badblocks_store);
+static DEVICE_ATTR(range, 0444, disk_range_show, NULL);
+static DEVICE_ATTR(ext_range, 0444, disk_ext_range_show, NULL);
+static DEVICE_ATTR(removable, 0444, disk_removable_show, NULL);
+static DEVICE_ATTR(hidden, 0444, disk_hidden_show, NULL);
+static DEVICE_ATTR(ro, 0444, disk_ro_show, NULL);
+static DEVICE_ATTR(size, 0444, part_size_show, NULL);
+static DEVICE_ATTR(alignment_offset, 0444, disk_alignment_offset_show, NULL);
+static DEVICE_ATTR(discard_alignment, 0444, disk_discard_alignment_show, NULL);
+static DEVICE_ATTR(capability, 0444, disk_capability_show, NULL);
+static DEVICE_ATTR(stat, 0444, part_stat_show, NULL);
+static DEVICE_ATTR(inflight, 0444, part_inflight_show, NULL);
+static DEVICE_ATTR(badblocks, 0644, disk_badblocks_show, disk_badblocks_store);
 #ifdef CONFIG_FAIL_MAKE_REQUEST
 static struct device_attribute dev_attr_fail =
-	__ATTR(make-it-fail, S_IRUGO|S_IWUSR, part_fail_show, part_fail_store);
+	__ATTR(make-it-fail, 0644, part_fail_show, part_fail_store);
 #endif
 #ifdef CONFIG_FAIL_IO_TIMEOUT
 static struct device_attribute dev_attr_fail_timeout =
-	__ATTR(io-timeout-fail,  S_IRUGO|S_IWUSR, part_timeout_show,
-		part_timeout_store);
+	__ATTR(io-timeout-fail, 0644, part_timeout_show, part_timeout_store);
 #endif
 
 static struct attribute *disk_attrs[] = {
@@ -1377,22 +1362,10 @@ static const struct seq_operations diskstats_op = {
 	.show	= diskstats_show
 };
 
-static int diskstats_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &diskstats_op);
-}
-
-static const struct file_operations proc_diskstats_operations = {
-	.open		= diskstats_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
 static int __init proc_genhd_init(void)
 {
-	proc_create("diskstats", 0, NULL, &proc_diskstats_operations);
-	proc_create("partitions", 0, NULL, &proc_partitions_operations);
+	proc_create_seq("diskstats", 0, NULL, &diskstats_op);
+	proc_create_seq("partitions", 0, NULL, &partitions_op);
 	return 0;
 }
 module_init(proc_genhd_init);
@@ -1924,9 +1897,9 @@ static ssize_t disk_events_poll_msecs_store(struct device *dev,
 	return count;
 }
 
-static const DEVICE_ATTR(events, S_IRUGO, disk_events_show, NULL);
-static const DEVICE_ATTR(events_async, S_IRUGO, disk_events_async_show, NULL);
-static const DEVICE_ATTR(events_poll_msecs, S_IRUGO|S_IWUSR,
+static const DEVICE_ATTR(events, 0444, disk_events_show, NULL);
+static const DEVICE_ATTR(events_async, 0444, disk_events_async_show, NULL);
+static const DEVICE_ATTR(events_poll_msecs, 0644,
 			 disk_events_poll_msecs_show,
 			 disk_events_poll_msecs_store);
 
diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c
index 0d6d25e32e1f..a1660bafc912 100644
--- a/block/kyber-iosched.c
+++ b/block/kyber-iosched.c
@@ -72,6 +72,19 @@ static const unsigned int kyber_batch_size[] = {
 	[KYBER_OTHER] = 8,
 };
 
+/*
+ * There is a same mapping between ctx & hctx and kcq & khd,
+ * we use request->mq_ctx->index_hw to index the kcq in khd.
+ */
+struct kyber_ctx_queue {
+	/*
+	 * Used to ensure operations on rq_list and kcq_map to be an atmoic one.
+	 * Also protect the rqs on rq_list when merge.
+	 */
+	spinlock_t lock;
+	struct list_head rq_list[KYBER_NUM_DOMAINS];
+} ____cacheline_aligned_in_smp;
+
 struct kyber_queue_data {
 	struct request_queue *q;
 
@@ -99,6 +112,8 @@ struct kyber_hctx_data {
 	struct list_head rqs[KYBER_NUM_DOMAINS];
 	unsigned int cur_domain;
 	unsigned int batching;
+	struct kyber_ctx_queue *kcqs;
+	struct sbitmap kcq_map[KYBER_NUM_DOMAINS];
 	wait_queue_entry_t domain_wait[KYBER_NUM_DOMAINS];
 	struct sbq_wait_state *domain_ws[KYBER_NUM_DOMAINS];
 	atomic_t wait_index[KYBER_NUM_DOMAINS];
@@ -107,10 +122,8 @@ struct kyber_hctx_data {
 static int kyber_domain_wake(wait_queue_entry_t *wait, unsigned mode, int flags,
 			     void *key);
 
-static int rq_sched_domain(const struct request *rq)
+static unsigned int kyber_sched_domain(unsigned int op)
 {
-	unsigned int op = rq->cmd_flags;
-
 	if ((op & REQ_OP_MASK) == REQ_OP_READ)
 		return KYBER_READ;
 	else if ((op & REQ_OP_MASK) == REQ_OP_WRITE && op_is_sync(op))
@@ -284,6 +297,11 @@ static unsigned int kyber_sched_tags_shift(struct kyber_queue_data *kqd)
 	return kqd->q->queue_hw_ctx[0]->sched_tags->bitmap_tags.sb.shift;
 }
 
+static int kyber_bucket_fn(const struct request *rq)
+{
+	return kyber_sched_domain(rq->cmd_flags);
+}
+
 static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q)
 {
 	struct kyber_queue_data *kqd;
@@ -297,7 +315,7 @@ static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q)
 		goto err;
 	kqd->q = q;
 
-	kqd->cb = blk_stat_alloc_callback(kyber_stat_timer_fn, rq_sched_domain,
+	kqd->cb = blk_stat_alloc_callback(kyber_stat_timer_fn, kyber_bucket_fn,
 					  KYBER_NUM_DOMAINS, kqd);
 	if (!kqd->cb)
 		goto err_kqd;
@@ -376,8 +394,18 @@ static void kyber_exit_sched(struct elevator_queue *e)
 	kfree(kqd);
 }
 
+static void kyber_ctx_queue_init(struct kyber_ctx_queue *kcq)
+{
+	unsigned int i;
+
+	spin_lock_init(&kcq->lock);
+	for (i = 0; i < KYBER_NUM_DOMAINS; i++)
+		INIT_LIST_HEAD(&kcq->rq_list[i]);
+}
+
 static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
 {
+	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;
 	struct kyber_hctx_data *khd;
 	int i;
 
@@ -385,6 +413,24 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
 	if (!khd)
 		return -ENOMEM;
 
+	khd->kcqs = kmalloc_array_node(hctx->nr_ctx,
+				       sizeof(struct kyber_ctx_queue),
+				       GFP_KERNEL, hctx->numa_node);
+	if (!khd->kcqs)
+		goto err_khd;
+
+	for (i = 0; i < hctx->nr_ctx; i++)
+		kyber_ctx_queue_init(&khd->kcqs[i]);
+
+	for (i = 0; i < KYBER_NUM_DOMAINS; i++) {
+		if (sbitmap_init_node(&khd->kcq_map[i], hctx->nr_ctx,
+				      ilog2(8), GFP_KERNEL, hctx->numa_node)) {
+			while (--i >= 0)
+				sbitmap_free(&khd->kcq_map[i]);
+			goto err_kcqs;
+		}
+	}
+
 	spin_lock_init(&khd->lock);
 
 	for (i = 0; i < KYBER_NUM_DOMAINS; i++) {
@@ -400,12 +446,26 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
 	khd->batching = 0;
 
 	hctx->sched_data = khd;
+	sbitmap_queue_min_shallow_depth(&hctx->sched_tags->bitmap_tags,
+					kqd->async_depth);
 
 	return 0;
+
+err_kcqs:
+	kfree(khd->kcqs);
+err_khd:
+	kfree(khd);
+	return -ENOMEM;
 }
 
 static void kyber_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
 {
+	struct kyber_hctx_data *khd = hctx->sched_data;
+	int i;
+
+	for (i = 0; i < KYBER_NUM_DOMAINS; i++)
+		sbitmap_free(&khd->kcq_map[i]);
+	kfree(khd->kcqs);
 	kfree(hctx->sched_data);
 }
 
@@ -427,7 +487,7 @@ static void rq_clear_domain_token(struct kyber_queue_data *kqd,
 
 	nr = rq_get_domain_token(rq);
 	if (nr != -1) {
-		sched_domain = rq_sched_domain(rq);
+		sched_domain = kyber_sched_domain(rq->cmd_flags);
 		sbitmap_queue_clear(&kqd->domain_tokens[sched_domain], nr,
 				    rq->mq_ctx->cpu);
 	}
@@ -446,11 +506,51 @@ static void kyber_limit_depth(unsigned int op, struct blk_mq_alloc_data *data)
 	}
 }
 
+static bool kyber_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio)
+{
+	struct kyber_hctx_data *khd = hctx->sched_data;
+	struct blk_mq_ctx *ctx = blk_mq_get_ctx(hctx->queue);
+	struct kyber_ctx_queue *kcq = &khd->kcqs[ctx->index_hw];
+	unsigned int sched_domain = kyber_sched_domain(bio->bi_opf);
+	struct list_head *rq_list = &kcq->rq_list[sched_domain];
+	bool merged;
+
+	spin_lock(&kcq->lock);
+	merged = blk_mq_bio_list_merge(hctx->queue, rq_list, bio);
+	spin_unlock(&kcq->lock);
+	blk_mq_put_ctx(ctx);
+
+	return merged;
+}
+
 static void kyber_prepare_request(struct request *rq, struct bio *bio)
 {
 	rq_set_domain_token(rq, -1);
 }
 
+static void kyber_insert_requests(struct blk_mq_hw_ctx *hctx,
+				  struct list_head *rq_list, bool at_head)
+{
+	struct kyber_hctx_data *khd = hctx->sched_data;
+	struct request *rq, *next;
+
+	list_for_each_entry_safe(rq, next, rq_list, queuelist) {
+		unsigned int sched_domain = kyber_sched_domain(rq->cmd_flags);
+		struct kyber_ctx_queue *kcq = &khd->kcqs[rq->mq_ctx->index_hw];
+		struct list_head *head = &kcq->rq_list[sched_domain];
+
+		spin_lock(&kcq->lock);
+		if (at_head)
+			list_move(&rq->queuelist, head);
+		else
+			list_move_tail(&rq->queuelist, head);
+		sbitmap_set_bit(&khd->kcq_map[sched_domain],
+				rq->mq_ctx->index_hw);
+		blk_mq_sched_request_inserted(rq);
+		spin_unlock(&kcq->lock);
+	}
+}
+
 static void kyber_finish_request(struct request *rq)
 {
 	struct kyber_queue_data *kqd = rq->q->elevator->elevator_data;
@@ -469,7 +569,7 @@ static void kyber_completed_request(struct request *rq)
 	 * Check if this request met our latency goal. If not, quickly gather
 	 * some statistics and start throttling.
 	 */
-	sched_domain = rq_sched_domain(rq);
+	sched_domain = kyber_sched_domain(rq->cmd_flags);
 	switch (sched_domain) {
 	case KYBER_READ:
 		target = kqd->read_lat_nsec;
@@ -485,29 +585,48 @@ static void kyber_completed_request(struct request *rq)
 	if (blk_stat_is_active(kqd->cb))
 		return;
 
-	now = __blk_stat_time(ktime_to_ns(ktime_get()));
-	if (now < blk_stat_time(&rq->issue_stat))
+	now = ktime_get_ns();
+	if (now < rq->io_start_time_ns)
 		return;
 
-	latency = now - blk_stat_time(&rq->issue_stat);
+	latency = now - rq->io_start_time_ns;
 
 	if (latency > target)
 		blk_stat_activate_msecs(kqd->cb, 10);
 }
 
-static void kyber_flush_busy_ctxs(struct kyber_hctx_data *khd,
-				  struct blk_mq_hw_ctx *hctx)
+struct flush_kcq_data {
+	struct kyber_hctx_data *khd;
+	unsigned int sched_domain;
+	struct list_head *list;
+};
+
+static bool flush_busy_kcq(struct sbitmap *sb, unsigned int bitnr, void *data)
 {
-	LIST_HEAD(rq_list);
-	struct request *rq, *next;
+	struct flush_kcq_data *flush_data = data;
+	struct kyber_ctx_queue *kcq = &flush_data->khd->kcqs[bitnr];
 
-	blk_mq_flush_busy_ctxs(hctx, &rq_list);
-	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
-		unsigned int sched_domain;
+	spin_lock(&kcq->lock);
+	list_splice_tail_init(&kcq->rq_list[flush_data->sched_domain],
+			      flush_data->list);
+	sbitmap_clear_bit(sb, bitnr);
+	spin_unlock(&kcq->lock);
 
-		sched_domain = rq_sched_domain(rq);
-		list_move_tail(&rq->queuelist, &khd->rqs[sched_domain]);
-	}
+	return true;
+}
+
+static void kyber_flush_busy_kcqs(struct kyber_hctx_data *khd,
+				  unsigned int sched_domain,
+				  struct list_head *list)
+{
+	struct flush_kcq_data data = {
+		.khd = khd,
+		.sched_domain = sched_domain,
+		.list = list,
+	};
+
+	sbitmap_for_each_set(&khd->kcq_map[sched_domain],
+			     flush_busy_kcq, &data);
 }
 
 static int kyber_domain_wake(wait_queue_entry_t *wait, unsigned mode, int flags,
@@ -570,26 +689,23 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
 static struct request *
 kyber_dispatch_cur_domain(struct kyber_queue_data *kqd,
 			  struct kyber_hctx_data *khd,
-			  struct blk_mq_hw_ctx *hctx,
-			  bool *flushed)
+			  struct blk_mq_hw_ctx *hctx)
 {
 	struct list_head *rqs;
 	struct request *rq;
 	int nr;
 
 	rqs = &khd->rqs[khd->cur_domain];
-	rq = list_first_entry_or_null(rqs, struct request, queuelist);
 
 	/*
-	 * If there wasn't already a pending request and we haven't flushed the
-	 * software queues yet, flush the software queues and check again.
+	 * If we already have a flushed request, then we just need to get a
+	 * token for it. Otherwise, if there are pending requests in the kcqs,
+	 * flush the kcqs, but only if we can get a token. If not, we should
+	 * leave the requests in the kcqs so that they can be merged. Note that
+	 * khd->lock serializes the flushes, so if we observed any bit set in
+	 * the kcq_map, we will always get a request.
 	 */
-	if (!rq && !*flushed) {
-		kyber_flush_busy_ctxs(khd, hctx);
-		*flushed = true;
-		rq = list_first_entry_or_null(rqs, struct request, queuelist);
-	}
-
+	rq = list_first_entry_or_null(rqs, struct request, queuelist);
 	if (rq) {
 		nr = kyber_get_domain_token(kqd, khd, hctx);
 		if (nr >= 0) {
@@ -598,6 +714,16 @@ kyber_dispatch_cur_domain(struct kyber_queue_data *kqd,
 			list_del_init(&rq->queuelist);
 			return rq;
 		}
+	} else if (sbitmap_any_bit_set(&khd->kcq_map[khd->cur_domain])) {
+		nr = kyber_get_domain_token(kqd, khd, hctx);
+		if (nr >= 0) {
+			kyber_flush_busy_kcqs(khd, khd->cur_domain, rqs);
+			rq = list_first_entry(rqs, struct request, queuelist);
+			khd->batching++;
+			rq_set_domain_token(rq, nr);
+			list_del_init(&rq->queuelist);
+			return rq;
+		}
 	}
 
 	/* There were either no pending requests or no tokens. */
@@ -608,7 +734,6 @@ static struct request *kyber_dispatch_request(struct blk_mq_hw_ctx *hctx)
 {
 	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;
 	struct kyber_hctx_data *khd = hctx->sched_data;
-	bool flushed = false;
 	struct request *rq;
 	int i;
 
@@ -619,7 +744,7 @@ static struct request *kyber_dispatch_request(struct blk_mq_hw_ctx *hctx)
 	 * from the batch.
 	 */
 	if (khd->batching < kyber_batch_size[khd->cur_domain]) {
-		rq = kyber_dispatch_cur_domain(kqd, khd, hctx, &flushed);
+		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);
 		if (rq)
 			goto out;
 	}
@@ -640,7 +765,7 @@ static struct request *kyber_dispatch_request(struct blk_mq_hw_ctx *hctx)
 		else
 			khd->cur_domain++;
 
-		rq = kyber_dispatch_cur_domain(kqd, khd, hctx, &flushed);
+		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);
 		if (rq)
 			goto out;
 	}
@@ -657,10 +782,12 @@ static bool kyber_has_work(struct blk_mq_hw_ctx *hctx)
 	int i;
 
 	for (i = 0; i < KYBER_NUM_DOMAINS; i++) {
-		if (!list_empty_careful(&khd->rqs[i]))
+		if (!list_empty_careful(&khd->rqs[i]) ||
+		    sbitmap_any_bit_set(&khd->kcq_map[i]))
 			return true;
 	}
-	return sbitmap_any_bit_set(&hctx->ctx_map);
+
+	return false;
 }
 
 #define KYBER_LAT_SHOW_STORE(op)					\
@@ -831,7 +958,9 @@ static struct elevator_type kyber_sched = {
 		.init_hctx = kyber_init_hctx,
 		.exit_hctx = kyber_exit_hctx,
 		.limit_depth = kyber_limit_depth,
+		.bio_merge = kyber_bio_merge,
 		.prepare_request = kyber_prepare_request,
+		.insert_requests = kyber_insert_requests,
 		.finish_request = kyber_finish_request,
 		.requeue_request = kyber_finish_request,
 		.completed_request = kyber_completed_request,
diff --git a/block/mq-deadline.c b/block/mq-deadline.c
index 8ec0ba9f5386..099a9e05854c 100644
--- a/block/mq-deadline.c
+++ b/block/mq-deadline.c
@@ -630,8 +630,7 @@ STORE_FUNCTION(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX, 0);
 #undef STORE_FUNCTION
 
 #define DD_ATTR(name) \
-	__ATTR(name, S_IRUGO|S_IWUSR, deadline_##name##_show, \
-				      deadline_##name##_store)
+	__ATTR(name, 0644, deadline_##name##_show, deadline_##name##_store)
 
 static struct elv_fs_entry deadline_attrs[] = {
 	DD_ATTR(read_expire),
diff --git a/block/partition-generic.c b/block/partition-generic.c
index db57cced9b98..3dcfd4ec0e11 100644
--- a/block/partition-generic.c
+++ b/block/partition-generic.c
@@ -179,18 +179,17 @@ ssize_t part_fail_store(struct device *dev,
 }
 #endif
 
-static DEVICE_ATTR(partition, S_IRUGO, part_partition_show, NULL);
-static DEVICE_ATTR(start, S_IRUGO, part_start_show, NULL);
-static DEVICE_ATTR(size, S_IRUGO, part_size_show, NULL);
-static DEVICE_ATTR(ro, S_IRUGO, part_ro_show, NULL);
-static DEVICE_ATTR(alignment_offset, S_IRUGO, part_alignment_offset_show, NULL);
-static DEVICE_ATTR(discard_alignment, S_IRUGO, part_discard_alignment_show,
-		   NULL);
-static DEVICE_ATTR(stat, S_IRUGO, part_stat_show, NULL);
-static DEVICE_ATTR(inflight, S_IRUGO, part_inflight_show, NULL);
+static DEVICE_ATTR(partition, 0444, part_partition_show, NULL);
+static DEVICE_ATTR(start, 0444, part_start_show, NULL);
+static DEVICE_ATTR(size, 0444, part_size_show, NULL);
+static DEVICE_ATTR(ro, 0444, part_ro_show, NULL);
+static DEVICE_ATTR(alignment_offset, 0444, part_alignment_offset_show, NULL);
+static DEVICE_ATTR(discard_alignment, 0444, part_discard_alignment_show, NULL);
+static DEVICE_ATTR(stat, 0444, part_stat_show, NULL);
+static DEVICE_ATTR(inflight, 0444, part_inflight_show, NULL);
 #ifdef CONFIG_FAIL_MAKE_REQUEST
 static struct device_attribute dev_attr_fail =
-	__ATTR(make-it-fail, S_IRUGO|S_IWUSR, part_fail_show, part_fail_store);
+	__ATTR(make-it-fail, 0644, part_fail_show, part_fail_store);
 #endif
 
 static struct attribute *part_attrs[] = {
@@ -291,8 +290,7 @@ static ssize_t whole_disk_show(struct device *dev,
 {
 	return 0;
 }
-static DEVICE_ATTR(whole_disk, S_IRUSR | S_IRGRP | S_IROTH,
-		   whole_disk_show, NULL);
+static DEVICE_ATTR(whole_disk, 0444, whole_disk_show, NULL);
 
 /*
  * Must be called either with bd_mutex held, before a disk can be opened or
@@ -518,7 +516,7 @@ int rescan_partitions(struct gendisk *disk, struct block_device *bdev)
 
 	if (disk->fops->revalidate_disk)
 		disk->fops->revalidate_disk(disk);
-	check_disk_size_change(disk, bdev);
+	check_disk_size_change(disk, bdev, true);
 	bdev->bd_invalidated = 0;
 	if (!get_capacity(disk) || !(state = check_partition(disk, bdev)))
 		return 0;
@@ -643,7 +641,7 @@ int invalidate_partitions(struct gendisk *disk, struct block_device *bdev)
 		return res;
 
 	set_capacity(disk, 0);
-	check_disk_size_change(disk, bdev);
+	check_disk_size_change(disk, bdev, false);
 	bdev->bd_invalidated = 0;
 	/* tell userspace that the media / partition table may have changed */
 	kobject_uevent(&disk_to_dev(disk)->kobj, KOBJ_CHANGE);
diff --git a/block/scsi_ioctl.c b/block/scsi_ioctl.c
index 60b471f8621b..533f4aee8567 100644
--- a/block/scsi_ioctl.c
+++ b/block/scsi_ioctl.c
@@ -321,8 +321,7 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk,
 		at_head = 1;
 
 	ret = -ENOMEM;
-	rq = blk_get_request(q, writing ? REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN,
-			GFP_KERNEL);
+	rq = blk_get_request(q, writing ? REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, 0);
 	if (IS_ERR(rq))
 		return PTR_ERR(rq);
 	req = scsi_req(rq);
@@ -449,8 +448,7 @@ int sg_scsi_ioctl(struct request_queue *q, struct gendisk *disk, fmode_t mode,
 
 	}
 
-	rq = blk_get_request(q, in_len ? REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN,
-			__GFP_RECLAIM);
+	rq = blk_get_request(q, in_len ? REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, 0);
 	if (IS_ERR(rq)) {
 		err = PTR_ERR(rq);
 		goto error_free_buffer;
@@ -501,7 +499,7 @@ int sg_scsi_ioctl(struct request_queue *q, struct gendisk *disk, fmode_t mode,
 		break;
 	}
 
-	if (bytes && blk_rq_map_kern(q, rq, buffer, bytes, __GFP_RECLAIM)) {
+	if (bytes && blk_rq_map_kern(q, rq, buffer, bytes, GFP_NOIO)) {
 		err = DRIVER_ERROR << 24;
 		goto error;
 	}
@@ -538,7 +536,7 @@ static int __blk_send_generic(struct request_queue *q, struct gendisk *bd_disk,
 	struct request *rq;
 	int err;
 
-	rq = blk_get_request(q, REQ_OP_SCSI_OUT, __GFP_RECLAIM);
+	rq = blk_get_request(q, REQ_OP_SCSI_OUT, 0);
 	if (IS_ERR(rq))
 		return PTR_ERR(rq);
 	rq->timeout = BLK_DEFAULT_SG_TIMEOUT;
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 76e8c88c97b4..f3e40ac56d93 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -289,6 +289,107 @@ config CRYPTO_CHACHA20POLY1305
 	  with the Poly1305 authenticator. It is defined in RFC7539 for use in
 	  IETF protocols.
 
+config CRYPTO_AEGIS128
+	tristate "AEGIS-128 AEAD algorithm"
+	select CRYPTO_AEAD
+	select CRYPTO_AES  # for AES S-box tables
+	help
+	 Support for the AEGIS-128 dedicated AEAD algorithm.
+
+config CRYPTO_AEGIS128L
+	tristate "AEGIS-128L AEAD algorithm"
+	select CRYPTO_AEAD
+	select CRYPTO_AES  # for AES S-box tables
+	help
+	 Support for the AEGIS-128L dedicated AEAD algorithm.
+
+config CRYPTO_AEGIS256
+	tristate "AEGIS-256 AEAD algorithm"
+	select CRYPTO_AEAD
+	select CRYPTO_AES  # for AES S-box tables
+	help
+	 Support for the AEGIS-256 dedicated AEAD algorithm.
+
+config CRYPTO_AEGIS128_AESNI_SSE2
+	tristate "AEGIS-128 AEAD algorithm (x86_64 AESNI+SSE2 implementation)"
+	depends on X86 && 64BIT
+	select CRYPTO_AEAD
+	select CRYPTO_CRYPTD
+	help
+	 AESNI+SSE2 implementation of the AEGSI-128 dedicated AEAD algorithm.
+
+config CRYPTO_AEGIS128L_AESNI_SSE2
+	tristate "AEGIS-128L AEAD algorithm (x86_64 AESNI+SSE2 implementation)"
+	depends on X86 && 64BIT
+	select CRYPTO_AEAD
+	select CRYPTO_CRYPTD
+	help
+	 AESNI+SSE2 implementation of the AEGSI-128L dedicated AEAD algorithm.
+
+config CRYPTO_AEGIS256_AESNI_SSE2
+	tristate "AEGIS-256 AEAD algorithm (x86_64 AESNI+SSE2 implementation)"
+	depends on X86 && 64BIT
+	select CRYPTO_AEAD
+	select CRYPTO_CRYPTD
+	help
+	 AESNI+SSE2 implementation of the AEGSI-256 dedicated AEAD algorithm.
+
+config CRYPTO_MORUS640
+	tristate "MORUS-640 AEAD algorithm"
+	select CRYPTO_AEAD
+	help
+	  Support for the MORUS-640 dedicated AEAD algorithm.
+
+config CRYPTO_MORUS640_GLUE
+	tristate
+	depends on X86
+	select CRYPTO_AEAD
+	select CRYPTO_CRYPTD
+	help
+	  Common glue for SIMD optimizations of the MORUS-640 dedicated AEAD
+	  algorithm.
+
+config CRYPTO_MORUS640_SSE2
+	tristate "MORUS-640 AEAD algorithm (x86_64 SSE2 implementation)"
+	depends on X86 && 64BIT
+	select CRYPTO_AEAD
+	select CRYPTO_MORUS640_GLUE
+	help
+	  SSE2 implementation of the MORUS-640 dedicated AEAD algorithm.
+
+config CRYPTO_MORUS1280
+	tristate "MORUS-1280 AEAD algorithm"
+	select CRYPTO_AEAD
+	help
+	  Support for the MORUS-1280 dedicated AEAD algorithm.
+
+config CRYPTO_MORUS1280_GLUE
+	tristate
+	depends on X86
+	select CRYPTO_AEAD
+	select CRYPTO_CRYPTD
+	help
+	  Common glue for SIMD optimizations of the MORUS-1280 dedicated AEAD
+	  algorithm.
+
+config CRYPTO_MORUS1280_SSE2
+	tristate "MORUS-1280 AEAD algorithm (x86_64 SSE2 implementation)"
+	depends on X86 && 64BIT
+	select CRYPTO_AEAD
+	select CRYPTO_MORUS1280_GLUE
+	help
+	  SSE2 optimizedimplementation of the MORUS-1280 dedicated AEAD
+	  algorithm.
+
+config CRYPTO_MORUS1280_AVX2
+	tristate "MORUS-1280 AEAD algorithm (x86_64 AVX2 implementation)"
+	depends on X86 && 64BIT
+	select CRYPTO_AEAD
+	select CRYPTO_MORUS1280_GLUE
+	help
+	  AVX2 optimized implementation of the MORUS-1280 dedicated AEAD
+	  algorithm.
+
 config CRYPTO_SEQIV
 	tristate "Sequence Number IV Generator"
 	select CRYPTO_AEAD
@@ -1335,34 +1436,6 @@ config CRYPTO_SALSA20
 	  The Salsa20 stream cipher algorithm is designed by Daniel J.
 	  Bernstein <djb@cr.yp.to>. See <http://cr.yp.to/snuffle.html>
 
-config CRYPTO_SALSA20_586
-	tristate "Salsa20 stream cipher algorithm (i586)"
-	depends on (X86 || UML_X86) && !64BIT
-	select CRYPTO_BLKCIPHER
-	select CRYPTO_SALSA20
-	help
-	  Salsa20 stream cipher algorithm.
-
-	  Salsa20 is a stream cipher submitted to eSTREAM, the ECRYPT
-	  Stream Cipher Project. See <http://www.ecrypt.eu.org/stream/>
-
-	  The Salsa20 stream cipher algorithm is designed by Daniel J.
-	  Bernstein <djb@cr.yp.to>. See <http://cr.yp.to/snuffle.html>
-
-config CRYPTO_SALSA20_X86_64
-	tristate "Salsa20 stream cipher algorithm (x86_64)"
-	depends on (X86 || UML_X86) && 64BIT
-	select CRYPTO_BLKCIPHER
-	select CRYPTO_SALSA20
-	help
-	  Salsa20 stream cipher algorithm.
-
-	  Salsa20 is a stream cipher submitted to eSTREAM, the ECRYPT
-	  Stream Cipher Project. See <http://www.ecrypt.eu.org/stream/>
-
-	  The Salsa20 stream cipher algorithm is designed by Daniel J.
-	  Bernstein <djb@cr.yp.to>. See <http://cr.yp.to/snuffle.html>
-
 config CRYPTO_CHACHA20
 	tristate "ChaCha20 cipher algorithm"
 	select CRYPTO_BLKCIPHER
@@ -1695,6 +1768,15 @@ config CRYPTO_LZ4HC
 	help
 	  This is the LZ4 high compression mode algorithm.
 
+config CRYPTO_ZSTD
+	tristate "Zstd compression algorithm"
+	select CRYPTO_ALGAPI
+	select CRYPTO_ACOMP2
+	select ZSTD_COMPRESS
+	select ZSTD_DECOMPRESS
+	help
+	  This is the zstd algorithm.
+
 comment "Random Number Generation"
 
 config CRYPTO_ANSI_CPRNG
diff --git a/crypto/Makefile b/crypto/Makefile
index 3a5f01616f74..6d1d40eeb964 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -86,6 +86,11 @@ obj-$(CONFIG_CRYPTO_KEYWRAP) += keywrap.o
 obj-$(CONFIG_CRYPTO_GCM) += gcm.o
 obj-$(CONFIG_CRYPTO_CCM) += ccm.o
 obj-$(CONFIG_CRYPTO_CHACHA20POLY1305) += chacha20poly1305.o
+obj-$(CONFIG_CRYPTO_AEGIS128) += aegis128.o
+obj-$(CONFIG_CRYPTO_AEGIS128L) += aegis128l.o
+obj-$(CONFIG_CRYPTO_AEGIS256) += aegis256.o
+obj-$(CONFIG_CRYPTO_MORUS640) += morus640.o
+obj-$(CONFIG_CRYPTO_MORUS1280) += morus1280.o
 obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o
 obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o
 obj-$(CONFIG_CRYPTO_MCRYPTD) += mcryptd.o
@@ -137,6 +142,7 @@ obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o
 obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o
 obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o
 obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o
+obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o
 
 ecdh_generic-y := ecc.o
 ecdh_generic-y += ecdh.o
diff --git a/crypto/aegis.h b/crypto/aegis.h
new file mode 100644
index 000000000000..f1c6900ddb80
--- /dev/null
+++ b/crypto/aegis.h
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * AEGIS common definitions
+ *
+ * Copyright (c) 2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (c) 2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#ifndef _CRYPTO_AEGIS_H
+#define _CRYPTO_AEGIS_H
+
+#include <crypto/aes.h>
+#include <linux/types.h>
+
+#define AEGIS_BLOCK_SIZE 16
+
+union aegis_block {
+	__le64 words64[AEGIS_BLOCK_SIZE / sizeof(__le64)];
+	u32 words32[AEGIS_BLOCK_SIZE / sizeof(u32)];
+	u8 bytes[AEGIS_BLOCK_SIZE];
+};
+
+#define AEGIS_BLOCK_ALIGN (__alignof__(union aegis_block))
+#define AEGIS_ALIGNED(p) IS_ALIGNED((uintptr_t)p, AEGIS_BLOCK_ALIGN)
+
+static const union aegis_block crypto_aegis_const[2] = {
+	{ .words64 = {
+		cpu_to_le64(U64_C(0x0d08050302010100)),
+		cpu_to_le64(U64_C(0x6279e99059372215)),
+	} },
+	{ .words64 = {
+		cpu_to_le64(U64_C(0xf12fc26d55183ddb)),
+		cpu_to_le64(U64_C(0xdd28b57342311120)),
+	} },
+};
+
+static void crypto_aegis_block_xor(union aegis_block *dst,
+				   const union aegis_block *src)
+{
+	dst->words64[0] ^= src->words64[0];
+	dst->words64[1] ^= src->words64[1];
+}
+
+static void crypto_aegis_block_and(union aegis_block *dst,
+				   const union aegis_block *src)
+{
+	dst->words64[0] &= src->words64[0];
+	dst->words64[1] &= src->words64[1];
+}
+
+static void crypto_aegis_aesenc(union aegis_block *dst,
+				const union aegis_block *src,
+				const union aegis_block *key)
+{
+	u32 *d = dst->words32;
+	const u8  *s  = src->bytes;
+	const u32 *k  = key->words32;
+	const u32 *t0 = crypto_ft_tab[0];
+	const u32 *t1 = crypto_ft_tab[1];
+	const u32 *t2 = crypto_ft_tab[2];
+	const u32 *t3 = crypto_ft_tab[3];
+	u32 d0, d1, d2, d3;
+
+	d0 = t0[s[ 0]] ^ t1[s[ 5]] ^ t2[s[10]] ^ t3[s[15]] ^ k[0];
+	d1 = t0[s[ 4]] ^ t1[s[ 9]] ^ t2[s[14]] ^ t3[s[ 3]] ^ k[1];
+	d2 = t0[s[ 8]] ^ t1[s[13]] ^ t2[s[ 2]] ^ t3[s[ 7]] ^ k[2];
+	d3 = t0[s[12]] ^ t1[s[ 1]] ^ t2[s[ 6]] ^ t3[s[11]] ^ k[3];
+
+	d[0] = d0;
+	d[1] = d1;
+	d[2] = d2;
+	d[3] = d3;
+}
+
+#endif /* _CRYPTO_AEGIS_H */
diff --git a/crypto/aegis128.c b/crypto/aegis128.c
new file mode 100644
index 000000000000..38271303ce16
--- /dev/null
+++ b/crypto/aegis128.c
@@ -0,0 +1,463 @@
+/*
+ * The AEGIS-128 Authenticated-Encryption Algorithm
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/algapi.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+
+#include "aegis.h"
+
+#define AEGIS128_NONCE_SIZE 16
+#define AEGIS128_STATE_BLOCKS 5
+#define AEGIS128_KEY_SIZE 16
+#define AEGIS128_MIN_AUTH_SIZE 8
+#define AEGIS128_MAX_AUTH_SIZE 16
+
+struct aegis_state {
+	union aegis_block blocks[AEGIS128_STATE_BLOCKS];
+};
+
+struct aegis_ctx {
+	union aegis_block key;
+};
+
+struct aegis128_ops {
+	int (*skcipher_walk_init)(struct skcipher_walk *walk,
+				  struct aead_request *req, bool atomic);
+
+	void (*crypt_chunk)(struct aegis_state *state, u8 *dst,
+			    const u8 *src, unsigned int size);
+};
+
+static void crypto_aegis128_update(struct aegis_state *state)
+{
+	union aegis_block tmp;
+	unsigned int i;
+
+	tmp = state->blocks[AEGIS128_STATE_BLOCKS - 1];
+	for (i = AEGIS128_STATE_BLOCKS - 1; i > 0; i--)
+		crypto_aegis_aesenc(&state->blocks[i], &state->blocks[i - 1],
+				    &state->blocks[i]);
+	crypto_aegis_aesenc(&state->blocks[0], &tmp, &state->blocks[0]);
+}
+
+static void crypto_aegis128_update_a(struct aegis_state *state,
+				     const union aegis_block *msg)
+{
+	crypto_aegis128_update(state);
+	crypto_aegis_block_xor(&state->blocks[0], msg);
+}
+
+static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg)
+{
+	crypto_aegis128_update(state);
+	crypto_xor(state->blocks[0].bytes, msg, AEGIS_BLOCK_SIZE);
+}
+
+static void crypto_aegis128_init(struct aegis_state *state,
+				 const union aegis_block *key,
+				 const u8 *iv)
+{
+	union aegis_block key_iv;
+	unsigned int i;
+
+	key_iv = *key;
+	crypto_xor(key_iv.bytes, iv, AEGIS_BLOCK_SIZE);
+
+	state->blocks[0] = key_iv;
+	state->blocks[1] = crypto_aegis_const[1];
+	state->blocks[2] = crypto_aegis_const[0];
+	state->blocks[3] = *key;
+	state->blocks[4] = *key;
+
+	crypto_aegis_block_xor(&state->blocks[3], &crypto_aegis_const[0]);
+	crypto_aegis_block_xor(&state->blocks[4], &crypto_aegis_const[1]);
+
+	for (i = 0; i < 5; i++) {
+		crypto_aegis128_update_a(state, key);
+		crypto_aegis128_update_a(state, &key_iv);
+	}
+}
+
+static void crypto_aegis128_ad(struct aegis_state *state,
+			       const u8 *src, unsigned int size)
+{
+	if (AEGIS_ALIGNED(src)) {
+		const union aegis_block *src_blk =
+				(const union aegis_block *)src;
+
+		while (size >= AEGIS_BLOCK_SIZE) {
+			crypto_aegis128_update_a(state, src_blk);
+
+			size -= AEGIS_BLOCK_SIZE;
+			src_blk++;
+		}
+	} else {
+		while (size >= AEGIS_BLOCK_SIZE) {
+			crypto_aegis128_update_u(state, src);
+
+			size -= AEGIS_BLOCK_SIZE;
+			src += AEGIS_BLOCK_SIZE;
+		}
+	}
+}
+
+static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst,
+					  const u8 *src, unsigned int size)
+{
+	union aegis_block tmp;
+
+	if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) {
+		while (size >= AEGIS_BLOCK_SIZE) {
+			union aegis_block *dst_blk =
+					(union aegis_block *)dst;
+			const union aegis_block *src_blk =
+					(const union aegis_block *)src;
+
+			tmp = state->blocks[2];
+			crypto_aegis_block_and(&tmp, &state->blocks[3]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+			crypto_aegis_block_xor(&tmp, src_blk);
+
+			crypto_aegis128_update_a(state, src_blk);
+
+			*dst_blk = tmp;
+
+			size -= AEGIS_BLOCK_SIZE;
+			src += AEGIS_BLOCK_SIZE;
+			dst += AEGIS_BLOCK_SIZE;
+		}
+	} else {
+		while (size >= AEGIS_BLOCK_SIZE) {
+			tmp = state->blocks[2];
+			crypto_aegis_block_and(&tmp, &state->blocks[3]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+			crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE);
+
+			crypto_aegis128_update_u(state, src);
+
+			memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE);
+
+			size -= AEGIS_BLOCK_SIZE;
+			src += AEGIS_BLOCK_SIZE;
+			dst += AEGIS_BLOCK_SIZE;
+		}
+	}
+
+	if (size > 0) {
+		union aegis_block msg = {};
+		memcpy(msg.bytes, src, size);
+
+		tmp = state->blocks[2];
+		crypto_aegis_block_and(&tmp, &state->blocks[3]);
+		crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+		crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+
+		crypto_aegis128_update_a(state, &msg);
+
+		crypto_aegis_block_xor(&msg, &tmp);
+
+		memcpy(dst, msg.bytes, size);
+	}
+}
+
+static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst,
+					  const u8 *src, unsigned int size)
+{
+	union aegis_block tmp;
+
+	if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) {
+		while (size >= AEGIS_BLOCK_SIZE) {
+			union aegis_block *dst_blk =
+					(union aegis_block *)dst;
+			const union aegis_block *src_blk =
+					(const union aegis_block *)src;
+
+			tmp = state->blocks[2];
+			crypto_aegis_block_and(&tmp, &state->blocks[3]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+			crypto_aegis_block_xor(&tmp, src_blk);
+
+			crypto_aegis128_update_a(state, &tmp);
+
+			*dst_blk = tmp;
+
+			size -= AEGIS_BLOCK_SIZE;
+			src += AEGIS_BLOCK_SIZE;
+			dst += AEGIS_BLOCK_SIZE;
+		}
+	} else {
+		while (size >= AEGIS_BLOCK_SIZE) {
+			tmp = state->blocks[2];
+			crypto_aegis_block_and(&tmp, &state->blocks[3]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+			crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE);
+
+			crypto_aegis128_update_a(state, &tmp);
+
+			memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE);
+
+			size -= AEGIS_BLOCK_SIZE;
+			src += AEGIS_BLOCK_SIZE;
+			dst += AEGIS_BLOCK_SIZE;
+		}
+	}
+
+	if (size > 0) {
+		union aegis_block msg = {};
+		memcpy(msg.bytes, src, size);
+
+		tmp = state->blocks[2];
+		crypto_aegis_block_and(&tmp, &state->blocks[3]);
+		crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+		crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+		crypto_aegis_block_xor(&msg, &tmp);
+
+		memset(msg.bytes + size, 0, AEGIS_BLOCK_SIZE - size);
+
+		crypto_aegis128_update_a(state, &msg);
+
+		memcpy(dst, msg.bytes, size);
+	}
+}
+
+static void crypto_aegis128_process_ad(struct aegis_state *state,
+				       struct scatterlist *sg_src,
+				       unsigned int assoclen)
+{
+	struct scatter_walk walk;
+	union aegis_block buf;
+	unsigned int pos = 0;
+
+	scatterwalk_start(&walk, sg_src);
+	while (assoclen != 0) {
+		unsigned int size = scatterwalk_clamp(&walk, assoclen);
+		unsigned int left = size;
+		void *mapped = scatterwalk_map(&walk);
+		const u8 *src = (const u8 *)mapped;
+
+		if (pos + size >= AEGIS_BLOCK_SIZE) {
+			if (pos > 0) {
+				unsigned int fill = AEGIS_BLOCK_SIZE - pos;
+				memcpy(buf.bytes + pos, src, fill);
+				crypto_aegis128_update_a(state, &buf);
+				pos = 0;
+				left -= fill;
+				src += fill;
+			}
+
+			crypto_aegis128_ad(state, src, left);
+			src += left & ~(AEGIS_BLOCK_SIZE - 1);
+			left &= AEGIS_BLOCK_SIZE - 1;
+		}
+
+		memcpy(buf.bytes + pos, src, left);
+
+		pos += left;
+		assoclen -= size;
+		scatterwalk_unmap(mapped);
+		scatterwalk_advance(&walk, size);
+		scatterwalk_done(&walk, 0, assoclen);
+	}
+
+	if (pos > 0) {
+		memset(buf.bytes + pos, 0, AEGIS_BLOCK_SIZE - pos);
+		crypto_aegis128_update_a(state, &buf);
+	}
+}
+
+static void crypto_aegis128_process_crypt(struct aegis_state *state,
+					  struct aead_request *req,
+					  const struct aegis128_ops *ops)
+{
+	struct skcipher_walk walk;
+	u8 *src, *dst;
+	unsigned int chunksize;
+
+	ops->skcipher_walk_init(&walk, req, false);
+
+	while (walk.nbytes) {
+		src = walk.src.virt.addr;
+		dst = walk.dst.virt.addr;
+		chunksize = walk.nbytes;
+
+		ops->crypt_chunk(state, dst, src, chunksize);
+
+		skcipher_walk_done(&walk, 0);
+	}
+}
+
+static void crypto_aegis128_final(struct aegis_state *state,
+				  union aegis_block *tag_xor,
+				  u64 assoclen, u64 cryptlen)
+{
+	u64 assocbits = assoclen * 8;
+	u64 cryptbits = cryptlen * 8;
+
+	union aegis_block tmp;
+	unsigned int i;
+
+	tmp.words64[0] = cpu_to_le64(assocbits);
+	tmp.words64[1] = cpu_to_le64(cryptbits);
+
+	crypto_aegis_block_xor(&tmp, &state->blocks[3]);
+
+	for (i = 0; i < 7; i++)
+		crypto_aegis128_update_a(state, &tmp);
+
+	for (i = 0; i < AEGIS128_STATE_BLOCKS; i++)
+		crypto_aegis_block_xor(tag_xor, &state->blocks[i]);
+}
+
+static int crypto_aegis128_setkey(struct crypto_aead *aead, const u8 *key,
+				  unsigned int keylen)
+{
+	struct aegis_ctx *ctx = crypto_aead_ctx(aead);
+
+	if (keylen != AEGIS128_KEY_SIZE) {
+		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	memcpy(ctx->key.bytes, key, AEGIS128_KEY_SIZE);
+	return 0;
+}
+
+static int crypto_aegis128_setauthsize(struct crypto_aead *tfm,
+				       unsigned int authsize)
+{
+	if (authsize > AEGIS128_MAX_AUTH_SIZE)
+		return -EINVAL;
+	if (authsize < AEGIS128_MIN_AUTH_SIZE)
+		return -EINVAL;
+	return 0;
+}
+
+static void crypto_aegis128_crypt(struct aead_request *req,
+				  union aegis_block *tag_xor,
+				  unsigned int cryptlen,
+				  const struct aegis128_ops *ops)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
+	struct aegis_state state;
+
+	crypto_aegis128_init(&state, &ctx->key, req->iv);
+	crypto_aegis128_process_ad(&state, req->src, req->assoclen);
+	crypto_aegis128_process_crypt(&state, req, ops);
+	crypto_aegis128_final(&state, tag_xor, req->assoclen, cryptlen);
+}
+
+static int crypto_aegis128_encrypt(struct aead_request *req)
+{
+	static const struct aegis128_ops ops = {
+		.skcipher_walk_init = skcipher_walk_aead_encrypt,
+		.crypt_chunk = crypto_aegis128_encrypt_chunk,
+	};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	union aegis_block tag = {};
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen;
+
+	crypto_aegis128_crypt(req, &tag, cryptlen, &ops);
+
+	scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,
+				 authsize, 1);
+	return 0;
+}
+
+static int crypto_aegis128_decrypt(struct aead_request *req)
+{
+	static const struct aegis128_ops ops = {
+		.skcipher_walk_init = skcipher_walk_aead_decrypt,
+		.crypt_chunk = crypto_aegis128_decrypt_chunk,
+	};
+	static const u8 zeros[AEGIS128_MAX_AUTH_SIZE] = {};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	union aegis_block tag;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen - authsize;
+
+	scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,
+				 authsize, 0);
+
+	crypto_aegis128_crypt(req, &tag, cryptlen, &ops);
+
+	return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;
+}
+
+static int crypto_aegis128_init_tfm(struct crypto_aead *tfm)
+{
+	return 0;
+}
+
+static void crypto_aegis128_exit_tfm(struct crypto_aead *tfm)
+{
+}
+
+static struct aead_alg crypto_aegis128_alg = {
+	.setkey = crypto_aegis128_setkey,
+	.setauthsize = crypto_aegis128_setauthsize,
+	.encrypt = crypto_aegis128_encrypt,
+	.decrypt = crypto_aegis128_decrypt,
+	.init = crypto_aegis128_init_tfm,
+	.exit = crypto_aegis128_exit_tfm,
+
+	.ivsize = AEGIS128_NONCE_SIZE,
+	.maxauthsize = AEGIS128_MAX_AUTH_SIZE,
+	.chunksize = AEGIS_BLOCK_SIZE,
+
+	.base = {
+		.cra_flags = CRYPTO_ALG_TYPE_AEAD,
+		.cra_blocksize = 1,
+		.cra_ctxsize = sizeof(struct aegis_ctx),
+		.cra_alignmask = 0,
+
+		.cra_priority = 100,
+
+		.cra_name = "aegis128",
+		.cra_driver_name = "aegis128-generic",
+
+		.cra_module = THIS_MODULE,
+	}
+};
+
+static int __init crypto_aegis128_module_init(void)
+{
+	return crypto_register_aead(&crypto_aegis128_alg);
+}
+
+static void __exit crypto_aegis128_module_exit(void)
+{
+	crypto_unregister_aead(&crypto_aegis128_alg);
+}
+
+module_init(crypto_aegis128_module_init);
+module_exit(crypto_aegis128_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm");
+MODULE_ALIAS_CRYPTO("aegis128");
+MODULE_ALIAS_CRYPTO("aegis128-generic");
diff --git a/crypto/aegis128l.c b/crypto/aegis128l.c
new file mode 100644
index 000000000000..0cc1a7525c85
--- /dev/null
+++ b/crypto/aegis128l.c
@@ -0,0 +1,527 @@
+/*
+ * The AEGIS-128L Authenticated-Encryption Algorithm
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/algapi.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+
+#include "aegis.h"
+
+#define AEGIS128L_CHUNK_BLOCKS 2
+#define AEGIS128L_CHUNK_SIZE (AEGIS128L_CHUNK_BLOCKS * AEGIS_BLOCK_SIZE)
+#define AEGIS128L_NONCE_SIZE 16
+#define AEGIS128L_STATE_BLOCKS 8
+#define AEGIS128L_KEY_SIZE 16
+#define AEGIS128L_MIN_AUTH_SIZE 8
+#define AEGIS128L_MAX_AUTH_SIZE 16
+
+union aegis_chunk {
+	union aegis_block blocks[AEGIS128L_CHUNK_BLOCKS];
+	u8 bytes[AEGIS128L_CHUNK_SIZE];
+};
+
+struct aegis_state {
+	union aegis_block blocks[AEGIS128L_STATE_BLOCKS];
+};
+
+struct aegis_ctx {
+	union aegis_block key;
+};
+
+struct aegis128l_ops {
+	int (*skcipher_walk_init)(struct skcipher_walk *walk,
+				  struct aead_request *req, bool atomic);
+
+	void (*crypt_chunk)(struct aegis_state *state, u8 *dst,
+			    const u8 *src, unsigned int size);
+};
+
+static void crypto_aegis128l_update(struct aegis_state *state)
+{
+	union aegis_block tmp;
+	unsigned int i;
+
+	tmp = state->blocks[AEGIS128L_STATE_BLOCKS - 1];
+	for (i = AEGIS128L_STATE_BLOCKS - 1; i > 0; i--)
+		crypto_aegis_aesenc(&state->blocks[i], &state->blocks[i - 1],
+				    &state->blocks[i]);
+	crypto_aegis_aesenc(&state->blocks[0], &tmp, &state->blocks[0]);
+}
+
+static void crypto_aegis128l_update_a(struct aegis_state *state,
+				      const union aegis_chunk *msg)
+{
+	crypto_aegis128l_update(state);
+	crypto_aegis_block_xor(&state->blocks[0], &msg->blocks[0]);
+	crypto_aegis_block_xor(&state->blocks[4], &msg->blocks[1]);
+}
+
+static void crypto_aegis128l_update_u(struct aegis_state *state,
+				      const void *msg)
+{
+	crypto_aegis128l_update(state);
+	crypto_xor(state->blocks[0].bytes, msg + 0 * AEGIS_BLOCK_SIZE,
+			AEGIS_BLOCK_SIZE);
+	crypto_xor(state->blocks[4].bytes, msg + 1 * AEGIS_BLOCK_SIZE,
+			AEGIS_BLOCK_SIZE);
+}
+
+static void crypto_aegis128l_init(struct aegis_state *state,
+				  const union aegis_block *key,
+				  const u8 *iv)
+{
+	union aegis_block key_iv;
+	union aegis_chunk chunk;
+	unsigned int i;
+
+	memcpy(chunk.blocks[0].bytes, iv, AEGIS_BLOCK_SIZE);
+	chunk.blocks[1] = *key;
+
+	key_iv = *key;
+	crypto_aegis_block_xor(&key_iv, &chunk.blocks[0]);
+
+	state->blocks[0] = key_iv;
+	state->blocks[1] = crypto_aegis_const[1];
+	state->blocks[2] = crypto_aegis_const[0];
+	state->blocks[3] = crypto_aegis_const[1];
+	state->blocks[4] = key_iv;
+	state->blocks[5] = *key;
+	state->blocks[6] = *key;
+	state->blocks[7] = *key;
+
+	crypto_aegis_block_xor(&state->blocks[5], &crypto_aegis_const[0]);
+	crypto_aegis_block_xor(&state->blocks[6], &crypto_aegis_const[1]);
+	crypto_aegis_block_xor(&state->blocks[7], &crypto_aegis_const[0]);
+
+	for (i = 0; i < 10; i++) {
+		crypto_aegis128l_update_a(state, &chunk);
+	}
+}
+
+static void crypto_aegis128l_ad(struct aegis_state *state,
+				const u8 *src, unsigned int size)
+{
+	if (AEGIS_ALIGNED(src)) {
+		const union aegis_chunk *src_chunk =
+				(const union aegis_chunk *)src;
+
+		while (size >= AEGIS128L_CHUNK_SIZE) {
+                    crypto_aegis128l_update_a(state, src_chunk);
+
+			size -= AEGIS128L_CHUNK_SIZE;
+			src_chunk += 1;
+		}
+	} else {
+		while (size >= AEGIS128L_CHUNK_SIZE) {
+			crypto_aegis128l_update_u(state, src);
+
+			size -= AEGIS128L_CHUNK_SIZE;
+			src += AEGIS128L_CHUNK_SIZE;
+		}
+	}
+}
+
+static void crypto_aegis128l_encrypt_chunk(struct aegis_state *state, u8 *dst,
+					   const u8 *src, unsigned int size)
+{
+	union aegis_chunk tmp;
+	union aegis_block *tmp0 = &tmp.blocks[0];
+	union aegis_block *tmp1 = &tmp.blocks[1];
+
+	if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) {
+		while (size >= AEGIS128L_CHUNK_SIZE) {
+			union aegis_chunk *dst_blk =
+					(union aegis_chunk *)dst;
+			const union aegis_chunk *src_blk =
+					(const union aegis_chunk *)src;
+
+			*tmp0 = state->blocks[2];
+			crypto_aegis_block_and(tmp0, &state->blocks[3]);
+			crypto_aegis_block_xor(tmp0, &state->blocks[6]);
+			crypto_aegis_block_xor(tmp0, &state->blocks[1]);
+			crypto_aegis_block_xor(tmp0, &src_blk->blocks[0]);
+
+			*tmp1 = state->blocks[6];
+			crypto_aegis_block_and(tmp1, &state->blocks[7]);
+			crypto_aegis_block_xor(tmp1, &state->blocks[5]);
+			crypto_aegis_block_xor(tmp1, &state->blocks[2]);
+			crypto_aegis_block_xor(tmp1, &src_blk->blocks[1]);
+
+			crypto_aegis128l_update_a(state, src_blk);
+
+			*dst_blk = tmp;
+
+			size -= AEGIS128L_CHUNK_SIZE;
+			src += AEGIS128L_CHUNK_SIZE;
+			dst += AEGIS128L_CHUNK_SIZE;
+		}
+	} else {
+		while (size >= AEGIS128L_CHUNK_SIZE) {
+			*tmp0 = state->blocks[2];
+			crypto_aegis_block_and(tmp0, &state->blocks[3]);
+			crypto_aegis_block_xor(tmp0, &state->blocks[6]);
+			crypto_aegis_block_xor(tmp0, &state->blocks[1]);
+			crypto_xor(tmp0->bytes, src + 0 * AEGIS_BLOCK_SIZE,
+				   AEGIS_BLOCK_SIZE);
+
+			*tmp1 = state->blocks[6];
+			crypto_aegis_block_and(tmp1, &state->blocks[7]);
+			crypto_aegis_block_xor(tmp1, &state->blocks[5]);
+			crypto_aegis_block_xor(tmp1, &state->blocks[2]);
+			crypto_xor(tmp1->bytes, src + 1 * AEGIS_BLOCK_SIZE,
+				   AEGIS_BLOCK_SIZE);
+
+			crypto_aegis128l_update_u(state, src);
+
+			memcpy(dst, tmp.bytes, AEGIS128L_CHUNK_SIZE);
+
+			size -= AEGIS128L_CHUNK_SIZE;
+			src += AEGIS128L_CHUNK_SIZE;
+			dst += AEGIS128L_CHUNK_SIZE;
+		}
+	}
+
+	if (size > 0) {
+		union aegis_chunk msg = {};
+		memcpy(msg.bytes, src, size);
+
+		*tmp0 = state->blocks[2];
+		crypto_aegis_block_and(tmp0, &state->blocks[3]);
+		crypto_aegis_block_xor(tmp0, &state->blocks[6]);
+		crypto_aegis_block_xor(tmp0, &state->blocks[1]);
+
+		*tmp1 = state->blocks[6];
+		crypto_aegis_block_and(tmp1, &state->blocks[7]);
+		crypto_aegis_block_xor(tmp1, &state->blocks[5]);
+		crypto_aegis_block_xor(tmp1, &state->blocks[2]);
+
+		crypto_aegis128l_update_a(state, &msg);
+
+		crypto_aegis_block_xor(&msg.blocks[0], tmp0);
+		crypto_aegis_block_xor(&msg.blocks[1], tmp1);
+
+		memcpy(dst, msg.bytes, size);
+	}
+}
+
+static void crypto_aegis128l_decrypt_chunk(struct aegis_state *state, u8 *dst,
+					   const u8 *src, unsigned int size)
+{
+	union aegis_chunk tmp;
+	union aegis_block *tmp0 = &tmp.blocks[0];
+	union aegis_block *tmp1 = &tmp.blocks[1];
+
+	if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) {
+		while (size >= AEGIS128L_CHUNK_SIZE) {
+			union aegis_chunk *dst_blk =
+					(union aegis_chunk *)dst;
+			const union aegis_chunk *src_blk =
+					(const union aegis_chunk *)src;
+
+			*tmp0 = state->blocks[2];
+			crypto_aegis_block_and(tmp0, &state->blocks[3]);
+			crypto_aegis_block_xor(tmp0, &state->blocks[6]);
+			crypto_aegis_block_xor(tmp0, &state->blocks[1]);
+			crypto_aegis_block_xor(tmp0, &src_blk->blocks[0]);
+
+			*tmp1 = state->blocks[6];
+			crypto_aegis_block_and(tmp1, &state->blocks[7]);
+			crypto_aegis_block_xor(tmp1, &state->blocks[5]);
+			crypto_aegis_block_xor(tmp1, &state->blocks[2]);
+			crypto_aegis_block_xor(tmp1, &src_blk->blocks[1]);
+
+			crypto_aegis128l_update_a(state, &tmp);
+
+			*dst_blk = tmp;
+
+			size -= AEGIS128L_CHUNK_SIZE;
+			src += AEGIS128L_CHUNK_SIZE;
+			dst += AEGIS128L_CHUNK_SIZE;
+		}
+	} else {
+		while (size >= AEGIS128L_CHUNK_SIZE) {
+			*tmp0 = state->blocks[2];
+			crypto_aegis_block_and(tmp0, &state->blocks[3]);
+			crypto_aegis_block_xor(tmp0, &state->blocks[6]);
+			crypto_aegis_block_xor(tmp0, &state->blocks[1]);
+			crypto_xor(tmp0->bytes, src + 0 * AEGIS_BLOCK_SIZE,
+				   AEGIS_BLOCK_SIZE);
+
+			*tmp1 = state->blocks[6];
+			crypto_aegis_block_and(tmp1, &state->blocks[7]);
+			crypto_aegis_block_xor(tmp1, &state->blocks[5]);
+			crypto_aegis_block_xor(tmp1, &state->blocks[2]);
+			crypto_xor(tmp1->bytes, src + 1 * AEGIS_BLOCK_SIZE,
+				   AEGIS_BLOCK_SIZE);
+
+			crypto_aegis128l_update_a(state, &tmp);
+
+			memcpy(dst, tmp.bytes, AEGIS128L_CHUNK_SIZE);
+
+			size -= AEGIS128L_CHUNK_SIZE;
+			src += AEGIS128L_CHUNK_SIZE;
+			dst += AEGIS128L_CHUNK_SIZE;
+		}
+	}
+
+	if (size > 0) {
+		union aegis_chunk msg = {};
+		memcpy(msg.bytes, src, size);
+
+		*tmp0 = state->blocks[2];
+		crypto_aegis_block_and(tmp0, &state->blocks[3]);
+		crypto_aegis_block_xor(tmp0, &state->blocks[6]);
+		crypto_aegis_block_xor(tmp0, &state->blocks[1]);
+		crypto_aegis_block_xor(&msg.blocks[0], tmp0);
+
+		*tmp1 = state->blocks[6];
+		crypto_aegis_block_and(tmp1, &state->blocks[7]);
+		crypto_aegis_block_xor(tmp1, &state->blocks[5]);
+		crypto_aegis_block_xor(tmp1, &state->blocks[2]);
+		crypto_aegis_block_xor(&msg.blocks[1], tmp1);
+
+		memset(msg.bytes + size, 0, AEGIS128L_CHUNK_SIZE - size);
+
+		crypto_aegis128l_update_a(state, &msg);
+
+		memcpy(dst, msg.bytes, size);
+	}
+}
+
+static void crypto_aegis128l_process_ad(struct aegis_state *state,
+					struct scatterlist *sg_src,
+					unsigned int assoclen)
+{
+	struct scatter_walk walk;
+	union aegis_chunk buf;
+	unsigned int pos = 0;
+
+	scatterwalk_start(&walk, sg_src);
+	while (assoclen != 0) {
+		unsigned int size = scatterwalk_clamp(&walk, assoclen);
+		unsigned int left = size;
+		void *mapped = scatterwalk_map(&walk);
+		const u8 *src = (const u8 *)mapped;
+
+		if (pos + size >= AEGIS128L_CHUNK_SIZE) {
+			if (pos > 0) {
+				unsigned int fill = AEGIS128L_CHUNK_SIZE - pos;
+				memcpy(buf.bytes + pos, src, fill);
+				crypto_aegis128l_update_a(state, &buf);
+				pos = 0;
+				left -= fill;
+				src += fill;
+			}
+
+			crypto_aegis128l_ad(state, src, left);
+			src += left & ~(AEGIS128L_CHUNK_SIZE - 1);
+			left &= AEGIS128L_CHUNK_SIZE - 1;
+		}
+
+		memcpy(buf.bytes + pos, src, left);
+
+		pos += left;
+		assoclen -= size;
+		scatterwalk_unmap(mapped);
+		scatterwalk_advance(&walk, size);
+		scatterwalk_done(&walk, 0, assoclen);
+	}
+
+	if (pos > 0) {
+		memset(buf.bytes + pos, 0, AEGIS128L_CHUNK_SIZE - pos);
+		crypto_aegis128l_update_a(state, &buf);
+	}
+}
+
+static void crypto_aegis128l_process_crypt(struct aegis_state *state,
+					   struct aead_request *req,
+					   const struct aegis128l_ops *ops)
+{
+	struct skcipher_walk walk;
+	u8 *src, *dst;
+	unsigned int chunksize;
+
+	ops->skcipher_walk_init(&walk, req, false);
+
+	while (walk.nbytes) {
+		src = walk.src.virt.addr;
+		dst = walk.dst.virt.addr;
+		chunksize = walk.nbytes;
+
+		ops->crypt_chunk(state, dst, src, chunksize);
+
+		skcipher_walk_done(&walk, 0);
+	}
+}
+
+static void crypto_aegis128l_final(struct aegis_state *state,
+				   union aegis_block *tag_xor,
+				   u64 assoclen, u64 cryptlen)
+{
+	u64 assocbits = assoclen * 8;
+	u64 cryptbits = cryptlen * 8;
+
+	union aegis_chunk tmp;
+	unsigned int i;
+
+	tmp.blocks[0].words64[0] = cpu_to_le64(assocbits);
+	tmp.blocks[0].words64[1] = cpu_to_le64(cryptbits);
+
+	crypto_aegis_block_xor(&tmp.blocks[0], &state->blocks[2]);
+
+	tmp.blocks[1] = tmp.blocks[0];
+	for (i = 0; i < 7; i++)
+		crypto_aegis128l_update_a(state, &tmp);
+
+	for (i = 0; i < 7; i++)
+		crypto_aegis_block_xor(tag_xor, &state->blocks[i]);
+}
+
+static int crypto_aegis128l_setkey(struct crypto_aead *aead, const u8 *key,
+				   unsigned int keylen)
+{
+	struct aegis_ctx *ctx = crypto_aead_ctx(aead);
+
+	if (keylen != AEGIS128L_KEY_SIZE) {
+		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	memcpy(ctx->key.bytes, key, AEGIS128L_KEY_SIZE);
+	return 0;
+}
+
+static int crypto_aegis128l_setauthsize(struct crypto_aead *tfm,
+					unsigned int authsize)
+{
+	if (authsize > AEGIS128L_MAX_AUTH_SIZE)
+		return -EINVAL;
+	if (authsize < AEGIS128L_MIN_AUTH_SIZE)
+		return -EINVAL;
+	return 0;
+}
+
+static void crypto_aegis128l_crypt(struct aead_request *req,
+				   union aegis_block *tag_xor,
+				   unsigned int cryptlen,
+				   const struct aegis128l_ops *ops)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
+	struct aegis_state state;
+
+	crypto_aegis128l_init(&state, &ctx->key, req->iv);
+	crypto_aegis128l_process_ad(&state, req->src, req->assoclen);
+	crypto_aegis128l_process_crypt(&state, req, ops);
+	crypto_aegis128l_final(&state, tag_xor, req->assoclen, cryptlen);
+}
+
+static int crypto_aegis128l_encrypt(struct aead_request *req)
+{
+	static const struct aegis128l_ops ops = {
+		.skcipher_walk_init = skcipher_walk_aead_encrypt,
+		.crypt_chunk = crypto_aegis128l_encrypt_chunk,
+	};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	union aegis_block tag = {};
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen;
+
+	crypto_aegis128l_crypt(req, &tag, cryptlen, &ops);
+
+	scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,
+				 authsize, 1);
+	return 0;
+}
+
+static int crypto_aegis128l_decrypt(struct aead_request *req)
+{
+	static const struct aegis128l_ops ops = {
+		.skcipher_walk_init = skcipher_walk_aead_decrypt,
+		.crypt_chunk = crypto_aegis128l_decrypt_chunk,
+	};
+	static const u8 zeros[AEGIS128L_MAX_AUTH_SIZE] = {};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	union aegis_block tag;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen - authsize;
+
+	scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,
+				 authsize, 0);
+
+	crypto_aegis128l_crypt(req, &tag, cryptlen, &ops);
+
+	return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;
+}
+
+static int crypto_aegis128l_init_tfm(struct crypto_aead *tfm)
+{
+	return 0;
+}
+
+static void crypto_aegis128l_exit_tfm(struct crypto_aead *tfm)
+{
+}
+
+static struct aead_alg crypto_aegis128l_alg = {
+	.setkey = crypto_aegis128l_setkey,
+	.setauthsize = crypto_aegis128l_setauthsize,
+	.encrypt = crypto_aegis128l_encrypt,
+	.decrypt = crypto_aegis128l_decrypt,
+	.init = crypto_aegis128l_init_tfm,
+	.exit = crypto_aegis128l_exit_tfm,
+
+	.ivsize = AEGIS128L_NONCE_SIZE,
+	.maxauthsize = AEGIS128L_MAX_AUTH_SIZE,
+	.chunksize = AEGIS128L_CHUNK_SIZE,
+
+	.base = {
+		.cra_flags = CRYPTO_ALG_TYPE_AEAD,
+		.cra_blocksize = 1,
+		.cra_ctxsize = sizeof(struct aegis_ctx),
+		.cra_alignmask = 0,
+
+		.cra_priority = 100,
+
+		.cra_name = "aegis128l",
+		.cra_driver_name = "aegis128l-generic",
+
+		.cra_module = THIS_MODULE,
+	}
+};
+
+static int __init crypto_aegis128l_module_init(void)
+{
+	return crypto_register_aead(&crypto_aegis128l_alg);
+}
+
+static void __exit crypto_aegis128l_module_exit(void)
+{
+	crypto_unregister_aead(&crypto_aegis128l_alg);
+}
+
+module_init(crypto_aegis128l_module_init);
+module_exit(crypto_aegis128l_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("AEGIS-128L AEAD algorithm");
+MODULE_ALIAS_CRYPTO("aegis128l");
+MODULE_ALIAS_CRYPTO("aegis128l-generic");
diff --git a/crypto/aegis256.c b/crypto/aegis256.c
new file mode 100644
index 000000000000..a489d741d33a
--- /dev/null
+++ b/crypto/aegis256.c
@@ -0,0 +1,478 @@
+/*
+ * The AEGIS-256 Authenticated-Encryption Algorithm
+ *
+ * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/algapi.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+
+#include "aegis.h"
+
+#define AEGIS256_NONCE_SIZE 32
+#define AEGIS256_STATE_BLOCKS 6
+#define AEGIS256_KEY_SIZE 32
+#define AEGIS256_MIN_AUTH_SIZE 8
+#define AEGIS256_MAX_AUTH_SIZE 16
+
+struct aegis_state {
+	union aegis_block blocks[AEGIS256_STATE_BLOCKS];
+};
+
+struct aegis_ctx {
+	union aegis_block key[AEGIS256_KEY_SIZE / AEGIS_BLOCK_SIZE];
+};
+
+struct aegis256_ops {
+	int (*skcipher_walk_init)(struct skcipher_walk *walk,
+				  struct aead_request *req, bool atomic);
+
+	void (*crypt_chunk)(struct aegis_state *state, u8 *dst,
+			    const u8 *src, unsigned int size);
+};
+
+static void crypto_aegis256_update(struct aegis_state *state)
+{
+	union aegis_block tmp;
+	unsigned int i;
+
+	tmp = state->blocks[AEGIS256_STATE_BLOCKS - 1];
+	for (i = AEGIS256_STATE_BLOCKS - 1; i > 0; i--)
+		crypto_aegis_aesenc(&state->blocks[i], &state->blocks[i - 1],
+				    &state->blocks[i]);
+	crypto_aegis_aesenc(&state->blocks[0], &tmp, &state->blocks[0]);
+}
+
+static void crypto_aegis256_update_a(struct aegis_state *state,
+				     const union aegis_block *msg)
+{
+	crypto_aegis256_update(state);
+	crypto_aegis_block_xor(&state->blocks[0], msg);
+}
+
+static void crypto_aegis256_update_u(struct aegis_state *state, const void *msg)
+{
+	crypto_aegis256_update(state);
+	crypto_xor(state->blocks[0].bytes, msg, AEGIS_BLOCK_SIZE);
+}
+
+static void crypto_aegis256_init(struct aegis_state *state,
+				 const union aegis_block *key,
+				 const u8 *iv)
+{
+	union aegis_block key_iv[2];
+	unsigned int i;
+
+	key_iv[0] = key[0];
+	key_iv[1] = key[1];
+	crypto_xor(key_iv[0].bytes, iv + 0 * AEGIS_BLOCK_SIZE,
+			AEGIS_BLOCK_SIZE);
+	crypto_xor(key_iv[1].bytes, iv + 1 * AEGIS_BLOCK_SIZE,
+			AEGIS_BLOCK_SIZE);
+
+	state->blocks[0] = key_iv[0];
+	state->blocks[1] = key_iv[1];
+	state->blocks[2] = crypto_aegis_const[1];
+	state->blocks[3] = crypto_aegis_const[0];
+	state->blocks[4] = key[0];
+	state->blocks[5] = key[1];
+
+	crypto_aegis_block_xor(&state->blocks[4], &crypto_aegis_const[0]);
+	crypto_aegis_block_xor(&state->blocks[5], &crypto_aegis_const[1]);
+
+	for (i = 0; i < 4; i++) {
+		crypto_aegis256_update_a(state, &key[0]);
+		crypto_aegis256_update_a(state, &key[1]);
+		crypto_aegis256_update_a(state, &key_iv[0]);
+		crypto_aegis256_update_a(state, &key_iv[1]);
+	}
+}
+
+static void crypto_aegis256_ad(struct aegis_state *state,
+			       const u8 *src, unsigned int size)
+{
+	if (AEGIS_ALIGNED(src)) {
+		const union aegis_block *src_blk =
+				(const union aegis_block *)src;
+
+		while (size >= AEGIS_BLOCK_SIZE) {
+			crypto_aegis256_update_a(state, src_blk);
+
+			size -= AEGIS_BLOCK_SIZE;
+			src_blk++;
+		}
+	} else {
+		while (size >= AEGIS_BLOCK_SIZE) {
+			crypto_aegis256_update_u(state, src);
+
+			size -= AEGIS_BLOCK_SIZE;
+			src += AEGIS_BLOCK_SIZE;
+		}
+	}
+}
+
+static void crypto_aegis256_encrypt_chunk(struct aegis_state *state, u8 *dst,
+					  const u8 *src, unsigned int size)
+{
+	union aegis_block tmp;
+
+	if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) {
+		while (size >= AEGIS_BLOCK_SIZE) {
+			union aegis_block *dst_blk =
+					(union aegis_block *)dst;
+			const union aegis_block *src_blk =
+					(const union aegis_block *)src;
+
+			tmp = state->blocks[2];
+			crypto_aegis_block_and(&tmp, &state->blocks[3]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[5]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+			crypto_aegis_block_xor(&tmp, src_blk);
+
+			crypto_aegis256_update_a(state, src_blk);
+
+			*dst_blk = tmp;
+
+			size -= AEGIS_BLOCK_SIZE;
+			src += AEGIS_BLOCK_SIZE;
+			dst += AEGIS_BLOCK_SIZE;
+		}
+	} else {
+		while (size >= AEGIS_BLOCK_SIZE) {
+			tmp = state->blocks[2];
+			crypto_aegis_block_and(&tmp, &state->blocks[3]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[5]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+			crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE);
+
+			crypto_aegis256_update_u(state, src);
+
+			memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE);
+
+			size -= AEGIS_BLOCK_SIZE;
+			src += AEGIS_BLOCK_SIZE;
+			dst += AEGIS_BLOCK_SIZE;
+		}
+	}
+
+	if (size > 0) {
+		union aegis_block msg = {};
+		memcpy(msg.bytes, src, size);
+
+		tmp = state->blocks[2];
+		crypto_aegis_block_and(&tmp, &state->blocks[3]);
+		crypto_aegis_block_xor(&tmp, &state->blocks[5]);
+		crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+		crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+
+		crypto_aegis256_update_a(state, &msg);
+
+		crypto_aegis_block_xor(&msg, &tmp);
+
+		memcpy(dst, msg.bytes, size);
+	}
+}
+
+static void crypto_aegis256_decrypt_chunk(struct aegis_state *state, u8 *dst,
+					  const u8 *src, unsigned int size)
+{
+	union aegis_block tmp;
+
+	if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) {
+		while (size >= AEGIS_BLOCK_SIZE) {
+			union aegis_block *dst_blk =
+					(union aegis_block *)dst;
+			const union aegis_block *src_blk =
+					(const union aegis_block *)src;
+
+			tmp = state->blocks[2];
+			crypto_aegis_block_and(&tmp, &state->blocks[3]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[5]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+			crypto_aegis_block_xor(&tmp, src_blk);
+
+			crypto_aegis256_update_a(state, &tmp);
+
+			*dst_blk = tmp;
+
+			size -= AEGIS_BLOCK_SIZE;
+			src += AEGIS_BLOCK_SIZE;
+			dst += AEGIS_BLOCK_SIZE;
+		}
+	} else {
+		while (size >= AEGIS_BLOCK_SIZE) {
+			tmp = state->blocks[2];
+			crypto_aegis_block_and(&tmp, &state->blocks[3]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[5]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+			crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE);
+
+			crypto_aegis256_update_a(state, &tmp);
+
+			memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE);
+
+			size -= AEGIS_BLOCK_SIZE;
+			src += AEGIS_BLOCK_SIZE;
+			dst += AEGIS_BLOCK_SIZE;
+		}
+	}
+
+	if (size > 0) {
+		union aegis_block msg = {};
+		memcpy(msg.bytes, src, size);
+
+		tmp = state->blocks[2];
+		crypto_aegis_block_and(&tmp, &state->blocks[3]);
+		crypto_aegis_block_xor(&tmp, &state->blocks[5]);
+		crypto_aegis_block_xor(&tmp, &state->blocks[4]);
+		crypto_aegis_block_xor(&tmp, &state->blocks[1]);
+		crypto_aegis_block_xor(&msg, &tmp);
+
+		memset(msg.bytes + size, 0, AEGIS_BLOCK_SIZE - size);
+
+		crypto_aegis256_update_a(state, &msg);
+
+		memcpy(dst, msg.bytes, size);
+	}
+}
+
+static void crypto_aegis256_process_ad(struct aegis_state *state,
+				       struct scatterlist *sg_src,
+				       unsigned int assoclen)
+{
+	struct scatter_walk walk;
+	union aegis_block buf;
+	unsigned int pos = 0;
+
+	scatterwalk_start(&walk, sg_src);
+	while (assoclen != 0) {
+		unsigned int size = scatterwalk_clamp(&walk, assoclen);
+		unsigned int left = size;
+		void *mapped = scatterwalk_map(&walk);
+		const u8 *src = (const u8 *)mapped;
+
+		if (pos + size >= AEGIS_BLOCK_SIZE) {
+			if (pos > 0) {
+				unsigned int fill = AEGIS_BLOCK_SIZE - pos;
+				memcpy(buf.bytes + pos, src, fill);
+				crypto_aegis256_update_a(state, &buf);
+				pos = 0;
+				left -= fill;
+				src += fill;
+			}
+
+			crypto_aegis256_ad(state, src, left);
+			src += left & ~(AEGIS_BLOCK_SIZE - 1);
+			left &= AEGIS_BLOCK_SIZE - 1;
+		}
+
+		memcpy(buf.bytes + pos, src, left);
+
+		pos += left;
+		assoclen -= size;
+		scatterwalk_unmap(mapped);
+		scatterwalk_advance(&walk, size);
+		scatterwalk_done(&walk, 0, assoclen);
+	}
+
+	if (pos > 0) {
+		memset(buf.bytes + pos, 0, AEGIS_BLOCK_SIZE - pos);
+		crypto_aegis256_update_a(state, &buf);
+	}
+}
+
+static void crypto_aegis256_process_crypt(struct aegis_state *state,
+					  struct aead_request *req,
+					  const struct aegis256_ops *ops)
+{
+	struct skcipher_walk walk;
+	u8 *src, *dst;
+	unsigned int chunksize;
+
+	ops->skcipher_walk_init(&walk, req, false);
+
+	while (walk.nbytes) {
+		src = walk.src.virt.addr;
+		dst = walk.dst.virt.addr;
+		chunksize = walk.nbytes;
+
+		ops->crypt_chunk(state, dst, src, chunksize);
+
+		skcipher_walk_done(&walk, 0);
+	}
+}
+
+static void crypto_aegis256_final(struct aegis_state *state,
+				  union aegis_block *tag_xor,
+				  u64 assoclen, u64 cryptlen)
+{
+	u64 assocbits = assoclen * 8;
+	u64 cryptbits = cryptlen * 8;
+
+	union aegis_block tmp;
+	unsigned int i;
+
+	tmp.words64[0] = cpu_to_le64(assocbits);
+	tmp.words64[1] = cpu_to_le64(cryptbits);
+
+	crypto_aegis_block_xor(&tmp, &state->blocks[3]);
+
+	for (i = 0; i < 7; i++)
+		crypto_aegis256_update_a(state, &tmp);
+
+	for (i = 0; i < AEGIS256_STATE_BLOCKS; i++)
+		crypto_aegis_block_xor(tag_xor, &state->blocks[i]);
+}
+
+static int crypto_aegis256_setkey(struct crypto_aead *aead, const u8 *key,
+				  unsigned int keylen)
+{
+	struct aegis_ctx *ctx = crypto_aead_ctx(aead);
+
+	if (keylen != AEGIS256_KEY_SIZE) {
+		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	memcpy(ctx->key[0].bytes, key, AEGIS_BLOCK_SIZE);
+	memcpy(ctx->key[1].bytes, key + AEGIS_BLOCK_SIZE,
+			AEGIS_BLOCK_SIZE);
+	return 0;
+}
+
+static int crypto_aegis256_setauthsize(struct crypto_aead *tfm,
+				       unsigned int authsize)
+{
+	if (authsize > AEGIS256_MAX_AUTH_SIZE)
+		return -EINVAL;
+	if (authsize < AEGIS256_MIN_AUTH_SIZE)
+		return -EINVAL;
+	return 0;
+}
+
+static void crypto_aegis256_crypt(struct aead_request *req,
+				  union aegis_block *tag_xor,
+				  unsigned int cryptlen,
+				  const struct aegis256_ops *ops)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
+	struct aegis_state state;
+
+	crypto_aegis256_init(&state, ctx->key, req->iv);
+	crypto_aegis256_process_ad(&state, req->src, req->assoclen);
+	crypto_aegis256_process_crypt(&state, req, ops);
+	crypto_aegis256_final(&state, tag_xor, req->assoclen, cryptlen);
+}
+
+static int crypto_aegis256_encrypt(struct aead_request *req)
+{
+	static const struct aegis256_ops ops = {
+		.skcipher_walk_init = skcipher_walk_aead_encrypt,
+		.crypt_chunk = crypto_aegis256_encrypt_chunk,
+	};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	union aegis_block tag = {};
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen;
+
+	crypto_aegis256_crypt(req, &tag, cryptlen, &ops);
+
+	scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,
+				 authsize, 1);
+	return 0;
+}
+
+static int crypto_aegis256_decrypt(struct aead_request *req)
+{
+	static const struct aegis256_ops ops = {
+		.skcipher_walk_init = skcipher_walk_aead_decrypt,
+		.crypt_chunk = crypto_aegis256_decrypt_chunk,
+	};
+	static const u8 zeros[AEGIS256_MAX_AUTH_SIZE] = {};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	union aegis_block tag;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen - authsize;
+
+	scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,
+				 authsize, 0);
+
+	crypto_aegis256_crypt(req, &tag, cryptlen, &ops);
+
+	return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;
+}
+
+static int crypto_aegis256_init_tfm(struct crypto_aead *tfm)
+{
+	return 0;
+}
+
+static void crypto_aegis256_exit_tfm(struct crypto_aead *tfm)
+{
+}
+
+static struct aead_alg crypto_aegis256_alg = {
+	.setkey = crypto_aegis256_setkey,
+	.setauthsize = crypto_aegis256_setauthsize,
+	.encrypt = crypto_aegis256_encrypt,
+	.decrypt = crypto_aegis256_decrypt,
+	.init = crypto_aegis256_init_tfm,
+	.exit = crypto_aegis256_exit_tfm,
+
+	.ivsize = AEGIS256_NONCE_SIZE,
+	.maxauthsize = AEGIS256_MAX_AUTH_SIZE,
+	.chunksize = AEGIS_BLOCK_SIZE,
+
+	.base = {
+		.cra_flags = CRYPTO_ALG_TYPE_AEAD,
+		.cra_blocksize = 1,
+		.cra_ctxsize = sizeof(struct aegis_ctx),
+		.cra_alignmask = 0,
+
+		.cra_priority = 100,
+
+		.cra_name = "aegis256",
+		.cra_driver_name = "aegis256-generic",
+
+		.cra_module = THIS_MODULE,
+	}
+};
+
+static int __init crypto_aegis256_module_init(void)
+{
+	return crypto_register_aead(&crypto_aegis256_alg);
+}
+
+static void __exit crypto_aegis256_module_exit(void)
+{
+	crypto_unregister_aead(&crypto_aegis256_alg);
+}
+
+module_init(crypto_aegis256_module_init);
+module_exit(crypto_aegis256_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("AEGIS-256 AEAD algorithm");
+MODULE_ALIAS_CRYPTO("aegis256");
+MODULE_ALIAS_CRYPTO("aegis256-generic");
diff --git a/crypto/af_alg.c b/crypto/af_alg.c
index 7846c0c20cfe..89ed613c017e 100644
--- a/crypto/af_alg.c
+++ b/crypto/af_alg.c
@@ -347,7 +347,6 @@ static const struct proto_ops alg_proto_ops = {
 	.sendpage	=	sock_no_sendpage,
 	.sendmsg	=	sock_no_sendmsg,
 	.recvmsg	=	sock_no_recvmsg,
-	.poll		=	sock_no_poll,
 
 	.bind		=	alg_bind,
 	.release	=	af_alg_release,
@@ -1061,19 +1060,12 @@ void af_alg_async_cb(struct crypto_async_request *_req, int err)
 }
 EXPORT_SYMBOL_GPL(af_alg_async_cb);
 
-/**
- * af_alg_poll - poll system call handler
- */
-__poll_t af_alg_poll(struct file *file, struct socket *sock,
-			 poll_table *wait)
+__poll_t af_alg_poll_mask(struct socket *sock, __poll_t events)
 {
 	struct sock *sk = sock->sk;
 	struct alg_sock *ask = alg_sk(sk);
 	struct af_alg_ctx *ctx = ask->private;
-	__poll_t mask;
-
-	sock_poll_wait(file, sk_sleep(sk), wait);
-	mask = 0;
+	__poll_t mask = 0;
 
 	if (!ctx->more || ctx->used)
 		mask |= EPOLLIN | EPOLLRDNORM;
@@ -1083,7 +1075,7 @@ __poll_t af_alg_poll(struct file *file, struct socket *sock,
 
 	return mask;
 }
-EXPORT_SYMBOL_GPL(af_alg_poll);
+EXPORT_SYMBOL_GPL(af_alg_poll_mask);
 
 /**
  * af_alg_alloc_areq - allocate struct af_alg_async_req
diff --git a/crypto/algapi.c b/crypto/algapi.c
index 2a0271b5f62a..c0755cf4f53f 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -10,6 +10,7 @@
  *
  */
 
+#include <crypto/algapi.h>
 #include <linux/err.h>
 #include <linux/errno.h>
 #include <linux/fips.h>
@@ -59,6 +60,15 @@ static int crypto_check_alg(struct crypto_alg *alg)
 	if (alg->cra_blocksize > PAGE_SIZE / 8)
 		return -EINVAL;
 
+	if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
+			       CRYPTO_ALG_TYPE_CIPHER) {
+		if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK)
+			return -EINVAL;
+
+		if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE)
+			return -EINVAL;
+	}
+
 	if (alg->cra_priority < 0)
 		return -EINVAL;
 
diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c
index 4b07edd5a9ff..330cf9f2b767 100644
--- a/crypto/algif_aead.c
+++ b/crypto/algif_aead.c
@@ -375,7 +375,7 @@ static struct proto_ops algif_aead_ops = {
 	.sendmsg	=	aead_sendmsg,
 	.sendpage	=	af_alg_sendpage,
 	.recvmsg	=	aead_recvmsg,
-	.poll		=	af_alg_poll,
+	.poll_mask	=	af_alg_poll_mask,
 };
 
 static int aead_check_key(struct socket *sock)
@@ -471,7 +471,7 @@ static struct proto_ops algif_aead_ops_nokey = {
 	.sendmsg	=	aead_sendmsg_nokey,
 	.sendpage	=	aead_sendpage_nokey,
 	.recvmsg	=	aead_recvmsg_nokey,
-	.poll		=	af_alg_poll,
+	.poll_mask	=	af_alg_poll_mask,
 };
 
 static void *aead_bind(const char *name, u32 type, u32 mask)
diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c
index 6c9b1927a520..bfcf595fd8f9 100644
--- a/crypto/algif_hash.c
+++ b/crypto/algif_hash.c
@@ -288,7 +288,6 @@ static struct proto_ops algif_hash_ops = {
 	.mmap		=	sock_no_mmap,
 	.bind		=	sock_no_bind,
 	.setsockopt	=	sock_no_setsockopt,
-	.poll		=	sock_no_poll,
 
 	.release	=	af_alg_release,
 	.sendmsg	=	hash_sendmsg,
@@ -396,7 +395,6 @@ static struct proto_ops algif_hash_ops_nokey = {
 	.mmap		=	sock_no_mmap,
 	.bind		=	sock_no_bind,
 	.setsockopt	=	sock_no_setsockopt,
-	.poll		=	sock_no_poll,
 
 	.release	=	af_alg_release,
 	.sendmsg	=	hash_sendmsg_nokey,
diff --git a/crypto/algif_rng.c b/crypto/algif_rng.c
index 150c2b6480ed..22df3799a17b 100644
--- a/crypto/algif_rng.c
+++ b/crypto/algif_rng.c
@@ -106,7 +106,6 @@ static struct proto_ops algif_rng_ops = {
 	.bind		=	sock_no_bind,
 	.accept		=	sock_no_accept,
 	.setsockopt	=	sock_no_setsockopt,
-	.poll		=	sock_no_poll,
 	.sendmsg	=	sock_no_sendmsg,
 	.sendpage	=	sock_no_sendpage,
 
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
index c4e885df4564..15cf3c5222e0 100644
--- a/crypto/algif_skcipher.c
+++ b/crypto/algif_skcipher.c
@@ -205,7 +205,7 @@ static struct proto_ops algif_skcipher_ops = {
 	.sendmsg	=	skcipher_sendmsg,
 	.sendpage	=	af_alg_sendpage,
 	.recvmsg	=	skcipher_recvmsg,
-	.poll		=	af_alg_poll,
+	.poll_mask	=	af_alg_poll_mask,
 };
 
 static int skcipher_check_key(struct socket *sock)
@@ -301,7 +301,7 @@ static struct proto_ops algif_skcipher_ops_nokey = {
 	.sendmsg	=	skcipher_sendmsg_nokey,
 	.sendpage	=	skcipher_sendpage_nokey,
 	.recvmsg	=	skcipher_recvmsg_nokey,
-	.poll		=	af_alg_poll,
+	.poll_mask	=	af_alg_poll_mask,
 };
 
 static void *skcipher_bind(const char *name, u32 type, u32 mask)
diff --git a/crypto/authenc.c b/crypto/authenc.c
index d3d6d72fe649..4fa8d40d947b 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -108,6 +108,7 @@ static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
 				       CRYPTO_TFM_RES_MASK);
 
 out:
+	memzero_explicit(&keys, sizeof(keys));
 	return err;
 
 badkey:
diff --git a/crypto/authencesn.c b/crypto/authencesn.c
index 15f91ddd7f0e..50b804747e20 100644
--- a/crypto/authencesn.c
+++ b/crypto/authencesn.c
@@ -90,6 +90,7 @@ static int crypto_authenc_esn_setkey(struct crypto_aead *authenc_esn, const u8 *
 					   CRYPTO_TFM_RES_MASK);
 
 out:
+	memzero_explicit(&keys, sizeof(keys));
 	return err;
 
 badkey:
diff --git a/crypto/cfb.c b/crypto/cfb.c
index 94ee39bed758..a0d68c09e1b9 100644
--- a/crypto/cfb.c
+++ b/crypto/cfb.c
@@ -53,9 +53,8 @@ static void crypto_cfb_encrypt_one(struct crypto_skcipher *tfm,
 static void crypto_cfb_final(struct skcipher_walk *walk,
 			     struct crypto_skcipher *tfm)
 {
-	const unsigned int bsize = crypto_cfb_bsize(tfm);
 	const unsigned long alignmask = crypto_skcipher_alignmask(tfm);
-	u8 tmp[bsize + alignmask];
+	u8 tmp[MAX_CIPHER_BLOCKSIZE + MAX_CIPHER_ALIGNMASK];
 	u8 *stream = PTR_ALIGN(tmp + 0, alignmask + 1);
 	u8 *src = walk->src.virt.addr;
 	u8 *dst = walk->dst.virt.addr;
@@ -94,7 +93,7 @@ static int crypto_cfb_encrypt_inplace(struct skcipher_walk *walk,
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *iv = walk->iv;
-	u8 tmp[bsize];
+	u8 tmp[MAX_CIPHER_BLOCKSIZE];
 
 	do {
 		crypto_cfb_encrypt_one(tfm, iv, tmp);
@@ -164,7 +163,7 @@ static int crypto_cfb_decrypt_inplace(struct skcipher_walk *walk,
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *iv = walk->iv;
-	u8 tmp[bsize];
+	u8 tmp[MAX_CIPHER_BLOCKSIZE];
 
 	do {
 		crypto_cfb_encrypt_one(tfm, iv, tmp);
diff --git a/crypto/cipher.c b/crypto/cipher.c
index 94fa3551476b..57836c30a49a 100644
--- a/crypto/cipher.c
+++ b/crypto/cipher.c
@@ -13,6 +13,7 @@
  *
  */
 
+#include <crypto/algapi.h>
 #include <linux/kernel.h>
 #include <linux/crypto.h>
 #include <linux/errno.h>
@@ -67,7 +68,7 @@ static void cipher_crypt_unaligned(void (*fn)(struct crypto_tfm *, u8 *,
 {
 	unsigned long alignmask = crypto_tfm_alg_alignmask(tfm);
 	unsigned int size = crypto_tfm_alg_blocksize(tfm);
-	u8 buffer[size + alignmask];
+	u8 buffer[MAX_CIPHER_BLOCKSIZE + MAX_CIPHER_ALIGNMASK];
 	u8 *tmp = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1);
 
 	memcpy(tmp, src, size);
diff --git a/crypto/crc32_generic.c b/crypto/crc32_generic.c
index 718cbce8d169..00facd27bcc2 100644
--- a/crypto/crc32_generic.c
+++ b/crypto/crc32_generic.c
@@ -29,6 +29,7 @@
  * This is crypto api shash wrappers to crc32_le.
  */
 
+#include <asm/unaligned.h>
 #include <linux/crc32.h>
 #include <crypto/internal/hash.h>
 #include <linux/init.h>
@@ -39,11 +40,6 @@
 #define CHKSUM_BLOCK_SIZE	1
 #define CHKSUM_DIGEST_SIZE	4
 
-static u32 __crc32_le(u32 crc, unsigned char const *p, size_t len)
-{
-	return crc32_le(crc, p, len);
-}
-
 /** No default init with ~0 */
 static int crc32_cra_init(struct crypto_tfm *tfm)
 {
@@ -54,7 +50,6 @@ static int crc32_cra_init(struct crypto_tfm *tfm)
 	return 0;
 }
 
-
 /*
  * Setting the seed allows arbitrary accumulators and flexible XOR policy
  * If your algorithm starts with ~0, then XOR with ~0 before you set
@@ -69,7 +64,7 @@ static int crc32_setkey(struct crypto_shash *hash, const u8 *key,
 		crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
 		return -EINVAL;
 	}
-	*mctx = le32_to_cpup((__le32 *)key);
+	*mctx = get_unaligned_le32(key);
 	return 0;
 }
 
@@ -88,7 +83,7 @@ static int crc32_update(struct shash_desc *desc, const u8 *data,
 {
 	u32 *crcp = shash_desc_ctx(desc);
 
-	*crcp = __crc32_le(*crcp, data, len);
+	*crcp = crc32_le(*crcp, data, len);
 	return 0;
 }
 
@@ -96,7 +91,7 @@ static int crc32_update(struct shash_desc *desc, const u8 *data,
 static int __crc32_finup(u32 *crcp, const u8 *data, unsigned int len,
 			 u8 *out)
 {
-	*(__le32 *)out = cpu_to_le32(__crc32_le(*crcp, data, len));
+	put_unaligned_le32(crc32_le(*crcp, data, len), out);
 	return 0;
 }
 
@@ -110,7 +105,7 @@ static int crc32_final(struct shash_desc *desc, u8 *out)
 {
 	u32 *crcp = shash_desc_ctx(desc);
 
-	*(__le32 *)out = cpu_to_le32p(crcp);
+	put_unaligned_le32(*crcp, out);
 	return 0;
 }
 
diff --git a/crypto/crc32c_generic.c b/crypto/crc32c_generic.c
index 372320399622..7283066ecc98 100644
--- a/crypto/crc32c_generic.c
+++ b/crypto/crc32c_generic.c
@@ -35,6 +35,7 @@
  *
  */
 
+#include <asm/unaligned.h>
 #include <crypto/internal/hash.h>
 #include <linux/init.h>
 #include <linux/module.h>
@@ -82,7 +83,7 @@ static int chksum_setkey(struct crypto_shash *tfm, const u8 *key,
 		crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 		return -EINVAL;
 	}
-	mctx->key = le32_to_cpu(*(__le32 *)key);
+	mctx->key = get_unaligned_le32(key);
 	return 0;
 }
 
@@ -99,13 +100,13 @@ static int chksum_final(struct shash_desc *desc, u8 *out)
 {
 	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 
-	*(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
+	put_unaligned_le32(~ctx->crc, out);
 	return 0;
 }
 
 static int __chksum_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out)
 {
-	*(__le32 *)out = ~cpu_to_le32(__crc32c_le(*crcp, data, len));
+	put_unaligned_le32(~__crc32c_le(*crcp, data, len), out);
 	return 0;
 }
 
@@ -148,7 +149,6 @@ static struct shash_alg alg = {
 		.cra_priority		=	100,
 		.cra_flags		=	CRYPTO_ALG_OPTIONAL_KEY,
 		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,
-		.cra_alignmask		=	3,
 		.cra_ctxsize		=	sizeof(struct chksum_ctx),
 		.cra_module		=	THIS_MODULE,
 		.cra_init		=	crc32c_cra_init,
diff --git a/crypto/ctr.c b/crypto/ctr.c
index 854d924f9d8e..435b75bd619e 100644
--- a/crypto/ctr.c
+++ b/crypto/ctr.c
@@ -58,7 +58,7 @@ static void crypto_ctr_crypt_final(struct blkcipher_walk *walk,
 	unsigned int bsize = crypto_cipher_blocksize(tfm);
 	unsigned long alignmask = crypto_cipher_alignmask(tfm);
 	u8 *ctrblk = walk->iv;
-	u8 tmp[bsize + alignmask];
+	u8 tmp[MAX_CIPHER_BLOCKSIZE + MAX_CIPHER_ALIGNMASK];
 	u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1);
 	u8 *src = walk->src.virt.addr;
 	u8 *dst = walk->dst.virt.addr;
@@ -106,7 +106,7 @@ static int crypto_ctr_crypt_inplace(struct blkcipher_walk *walk,
 	unsigned int nbytes = walk->nbytes;
 	u8 *ctrblk = walk->iv;
 	u8 *src = walk->src.virt.addr;
-	u8 tmp[bsize + alignmask];
+	u8 tmp[MAX_CIPHER_BLOCKSIZE + MAX_CIPHER_ALIGNMASK];
 	u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1);
 
 	do {
diff --git a/crypto/cts.c b/crypto/cts.c
index 4773c188e6d9..4e28d83ae37d 100644
--- a/crypto/cts.c
+++ b/crypto/cts.c
@@ -40,6 +40,7 @@
  * rfc3962 includes errata information in its Appendix A.
  */
 
+#include <crypto/algapi.h>
 #include <crypto/internal/skcipher.h>
 #include <linux/err.h>
 #include <linux/init.h>
@@ -104,7 +105,7 @@ static int cts_cbc_encrypt(struct skcipher_request *req)
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 	struct skcipher_request *subreq = &rctx->subreq;
 	int bsize = crypto_skcipher_blocksize(tfm);
-	u8 d[bsize * 2] __aligned(__alignof__(u32));
+	u8 d[MAX_CIPHER_BLOCKSIZE * 2] __aligned(__alignof__(u32));
 	struct scatterlist *sg;
 	unsigned int offset;
 	int lastn;
@@ -183,7 +184,7 @@ static int cts_cbc_decrypt(struct skcipher_request *req)
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 	struct skcipher_request *subreq = &rctx->subreq;
 	int bsize = crypto_skcipher_blocksize(tfm);
-	u8 d[bsize * 2] __aligned(__alignof__(u32));
+	u8 d[MAX_CIPHER_BLOCKSIZE * 2] __aligned(__alignof__(u32));
 	struct scatterlist *sg;
 	unsigned int offset;
 	u8 *space;
diff --git a/crypto/ecc.c b/crypto/ecc.c
index 9c066b5ac12d..815541309a95 100644
--- a/crypto/ecc.c
+++ b/crypto/ecc.c
@@ -515,7 +515,7 @@ static void vli_mmod_fast_256(u64 *result, const u64 *product,
 static bool vli_mmod_fast(u64 *result, u64 *product,
 			  const u64 *curve_prime, unsigned int ndigits)
 {
-	u64 tmp[2 * ndigits];
+	u64 tmp[2 * ECC_MAX_DIGITS];
 
 	switch (ndigits) {
 	case 3:
@@ -536,7 +536,7 @@ static bool vli_mmod_fast(u64 *result, u64 *product,
 static void vli_mod_mult_fast(u64 *result, const u64 *left, const u64 *right,
 			      const u64 *curve_prime, unsigned int ndigits)
 {
-	u64 product[2 * ndigits];
+	u64 product[2 * ECC_MAX_DIGITS];
 
 	vli_mult(product, left, right, ndigits);
 	vli_mmod_fast(result, product, curve_prime, ndigits);
@@ -546,7 +546,7 @@ static void vli_mod_mult_fast(u64 *result, const u64 *left, const u64 *right,
 static void vli_mod_square_fast(u64 *result, const u64 *left,
 				const u64 *curve_prime, unsigned int ndigits)
 {
-	u64 product[2 * ndigits];
+	u64 product[2 * ECC_MAX_DIGITS];
 
 	vli_square(product, left, ndigits);
 	vli_mmod_fast(result, product, curve_prime, ndigits);
@@ -560,8 +560,8 @@ static void vli_mod_square_fast(u64 *result, const u64 *left,
 static void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod,
 			unsigned int ndigits)
 {
-	u64 a[ndigits], b[ndigits];
-	u64 u[ndigits], v[ndigits];
+	u64 a[ECC_MAX_DIGITS], b[ECC_MAX_DIGITS];
+	u64 u[ECC_MAX_DIGITS], v[ECC_MAX_DIGITS];
 	u64 carry;
 	int cmp_result;
 
@@ -649,8 +649,8 @@ static void ecc_point_double_jacobian(u64 *x1, u64 *y1, u64 *z1,
 				      u64 *curve_prime, unsigned int ndigits)
 {
 	/* t1 = x, t2 = y, t3 = z */
-	u64 t4[ndigits];
-	u64 t5[ndigits];
+	u64 t4[ECC_MAX_DIGITS];
+	u64 t5[ECC_MAX_DIGITS];
 
 	if (vli_is_zero(z1, ndigits))
 		return;
@@ -711,7 +711,7 @@ static void ecc_point_double_jacobian(u64 *x1, u64 *y1, u64 *z1,
 static void apply_z(u64 *x1, u64 *y1, u64 *z, u64 *curve_prime,
 		    unsigned int ndigits)
 {
-	u64 t1[ndigits];
+	u64 t1[ECC_MAX_DIGITS];
 
 	vli_mod_square_fast(t1, z, curve_prime, ndigits);    /* z^2 */
 	vli_mod_mult_fast(x1, x1, t1, curve_prime, ndigits); /* x1 * z^2 */
@@ -724,7 +724,7 @@ static void xycz_initial_double(u64 *x1, u64 *y1, u64 *x2, u64 *y2,
 				u64 *p_initial_z, u64 *curve_prime,
 				unsigned int ndigits)
 {
-	u64 z[ndigits];
+	u64 z[ECC_MAX_DIGITS];
 
 	vli_set(x2, x1, ndigits);
 	vli_set(y2, y1, ndigits);
@@ -750,7 +750,7 @@ static void xycz_add(u64 *x1, u64 *y1, u64 *x2, u64 *y2, u64 *curve_prime,
 		     unsigned int ndigits)
 {
 	/* t1 = X1, t2 = Y1, t3 = X2, t4 = Y2 */
-	u64 t5[ndigits];
+	u64 t5[ECC_MAX_DIGITS];
 
 	/* t5 = x2 - x1 */
 	vli_mod_sub(t5, x2, x1, curve_prime, ndigits);
@@ -791,9 +791,9 @@ static void xycz_add_c(u64 *x1, u64 *y1, u64 *x2, u64 *y2, u64 *curve_prime,
 		       unsigned int ndigits)
 {
 	/* t1 = X1, t2 = Y1, t3 = X2, t4 = Y2 */
-	u64 t5[ndigits];
-	u64 t6[ndigits];
-	u64 t7[ndigits];
+	u64 t5[ECC_MAX_DIGITS];
+	u64 t6[ECC_MAX_DIGITS];
+	u64 t7[ECC_MAX_DIGITS];
 
 	/* t5 = x2 - x1 */
 	vli_mod_sub(t5, x2, x1, curve_prime, ndigits);
@@ -846,9 +846,9 @@ static void ecc_point_mult(struct ecc_point *result,
 			   unsigned int ndigits)
 {
 	/* R0 and R1 */
-	u64 rx[2][ndigits];
-	u64 ry[2][ndigits];
-	u64 z[ndigits];
+	u64 rx[2][ECC_MAX_DIGITS];
+	u64 ry[2][ECC_MAX_DIGITS];
+	u64 z[ECC_MAX_DIGITS];
 	int i, nb;
 	int num_bits = vli_num_bits(scalar, ndigits);
 
@@ -943,13 +943,13 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
 int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits, u64 *privkey)
 {
 	const struct ecc_curve *curve = ecc_get_curve(curve_id);
-	u64 priv[ndigits];
+	u64 priv[ECC_MAX_DIGITS];
 	unsigned int nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
 	unsigned int nbits = vli_num_bits(curve->n, ndigits);
 	int err;
 
 	/* Check that N is included in Table 1 of FIPS 186-4, section 6.1.1 */
-	if (nbits < 160)
+	if (nbits < 160 || ndigits > ARRAY_SIZE(priv))
 		return -EINVAL;
 
 	/*
@@ -988,10 +988,10 @@ int ecc_make_pub_key(unsigned int curve_id, unsigned int ndigits,
 {
 	int ret = 0;
 	struct ecc_point *pk;
-	u64 priv[ndigits];
+	u64 priv[ECC_MAX_DIGITS];
 	const struct ecc_curve *curve = ecc_get_curve(curve_id);
 
-	if (!private_key || !curve) {
+	if (!private_key || !curve || ndigits > ARRAY_SIZE(priv)) {
 		ret = -EINVAL;
 		goto out;
 	}
@@ -1025,30 +1025,25 @@ int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
 {
 	int ret = 0;
 	struct ecc_point *product, *pk;
-	u64 *priv, *rand_z;
+	u64 priv[ECC_MAX_DIGITS];
+	u64 rand_z[ECC_MAX_DIGITS];
+	unsigned int nbytes;
 	const struct ecc_curve *curve = ecc_get_curve(curve_id);
 
-	if (!private_key || !public_key || !curve) {
+	if (!private_key || !public_key || !curve ||
+	    ndigits > ARRAY_SIZE(priv) || ndigits > ARRAY_SIZE(rand_z)) {
 		ret = -EINVAL;
 		goto out;
 	}
 
-	priv = kmalloc_array(ndigits, sizeof(*priv), GFP_KERNEL);
-	if (!priv) {
-		ret = -ENOMEM;
-		goto out;
-	}
+	nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
 
-	rand_z = kmalloc_array(ndigits, sizeof(*rand_z), GFP_KERNEL);
-	if (!rand_z) {
-		ret = -ENOMEM;
-		goto kfree_out;
-	}
+	get_random_bytes(rand_z, nbytes);
 
 	pk = ecc_alloc_point(ndigits);
 	if (!pk) {
 		ret = -ENOMEM;
-		goto kfree_out;
+		goto out;
 	}
 
 	product = ecc_alloc_point(ndigits);
@@ -1057,8 +1052,6 @@ int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
 		goto err_alloc_product;
 	}
 
-	get_random_bytes(rand_z, ndigits << ECC_DIGITS_TO_BYTES_SHIFT);
-
 	ecc_swap_digits(public_key, pk->x, ndigits);
 	ecc_swap_digits(&public_key[ndigits], pk->y, ndigits);
 	ecc_swap_digits(private_key, priv, ndigits);
@@ -1073,9 +1066,6 @@ int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
 	ecc_free_point(product);
 err_alloc_product:
 	ecc_free_point(pk);
-kfree_out:
-	kzfree(priv);
-	kzfree(rand_z);
 out:
 	return ret;
 }
diff --git a/crypto/ecc.h b/crypto/ecc.h
index e4fd4492c765..f75a86baa3bd 100644
--- a/crypto/ecc.h
+++ b/crypto/ecc.h
@@ -26,7 +26,9 @@
 #ifndef _CRYPTO_ECC_H
 #define _CRYPTO_ECC_H
 
-#define ECC_MAX_DIGITS	4 /* 256 */
+#define ECC_CURVE_NIST_P192_DIGITS  3
+#define ECC_CURVE_NIST_P256_DIGITS  4
+#define ECC_MAX_DIGITS              ECC_CURVE_NIST_P256_DIGITS
 
 #define ECC_DIGITS_TO_BYTES_SHIFT 3
 
diff --git a/crypto/ecdh.c b/crypto/ecdh.c
index d2ec33f0e098..bf6300175b9c 100644
--- a/crypto/ecdh.c
+++ b/crypto/ecdh.c
@@ -30,8 +30,8 @@ static inline struct ecdh_ctx *ecdh_get_ctx(struct crypto_kpp *tfm)
 static unsigned int ecdh_supported_curve(unsigned int curve_id)
 {
 	switch (curve_id) {
-	case ECC_CURVE_NIST_P192: return 3;
-	case ECC_CURVE_NIST_P256: return 4;
+	case ECC_CURVE_NIST_P192: return ECC_CURVE_NIST_P192_DIGITS;
+	case ECC_CURVE_NIST_P256: return ECC_CURVE_NIST_P256_DIGITS;
 	default: return 0;
 	}
 }
diff --git a/crypto/morus1280.c b/crypto/morus1280.c
new file mode 100644
index 000000000000..6180b2557836
--- /dev/null
+++ b/crypto/morus1280.c
@@ -0,0 +1,549 @@
+/*
+ * The MORUS-1280 Authenticated-Encryption Algorithm
+ *
+ * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <asm/unaligned.h>
+#include <crypto/algapi.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/morus_common.h>
+#include <crypto/scatterwalk.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+
+#define MORUS1280_WORD_SIZE 8
+#define MORUS1280_BLOCK_SIZE (MORUS_BLOCK_WORDS * MORUS1280_WORD_SIZE)
+#define MORUS1280_BLOCK_ALIGN (__alignof__(__le64))
+#define MORUS1280_ALIGNED(p) IS_ALIGNED((uintptr_t)p, MORUS1280_BLOCK_ALIGN)
+
+struct morus1280_block {
+	u64 words[MORUS_BLOCK_WORDS];
+};
+
+union morus1280_block_in {
+	__le64 words[MORUS_BLOCK_WORDS];
+	u8 bytes[MORUS1280_BLOCK_SIZE];
+};
+
+struct morus1280_state {
+	struct morus1280_block s[MORUS_STATE_BLOCKS];
+};
+
+struct morus1280_ctx {
+	struct morus1280_block key;
+};
+
+struct morus1280_ops {
+	int (*skcipher_walk_init)(struct skcipher_walk *walk,
+				  struct aead_request *req, bool atomic);
+
+	void (*crypt_chunk)(struct morus1280_state *state,
+			    u8 *dst, const u8 *src, unsigned int size);
+};
+
+static const struct morus1280_block crypto_morus1280_const[1] = {
+	{ .words = {
+		U64_C(0x0d08050302010100),
+		U64_C(0x6279e99059372215),
+		U64_C(0xf12fc26d55183ddb),
+		U64_C(0xdd28b57342311120),
+	} },
+};
+
+static void crypto_morus1280_round(struct morus1280_block *b0,
+				   struct morus1280_block *b1,
+				   struct morus1280_block *b2,
+				   struct morus1280_block *b3,
+				   struct morus1280_block *b4,
+				   const struct morus1280_block *m,
+				   unsigned int b, unsigned int w)
+{
+	unsigned int i;
+	struct morus1280_block tmp;
+
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		b0->words[i] ^= b1->words[i] & b2->words[i];
+		b0->words[i] ^= b3->words[i];
+		b0->words[i] ^= m->words[i];
+		b0->words[i] = rol64(b0->words[i], b);
+	}
+
+	tmp = *b3;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++)
+		b3->words[(i + w) % MORUS_BLOCK_WORDS] = tmp.words[i];
+}
+
+static void crypto_morus1280_update(struct morus1280_state *state,
+				    const struct morus1280_block *m)
+{
+	static const struct morus1280_block z = {};
+
+	struct morus1280_block *s = state->s;
+
+	crypto_morus1280_round(&s[0], &s[1], &s[2], &s[3], &s[4], &z, 13, 1);
+	crypto_morus1280_round(&s[1], &s[2], &s[3], &s[4], &s[0], m,  46, 2);
+	crypto_morus1280_round(&s[2], &s[3], &s[4], &s[0], &s[1], m,  38, 3);
+	crypto_morus1280_round(&s[3], &s[4], &s[0], &s[1], &s[2], m,   7, 2);
+	crypto_morus1280_round(&s[4], &s[0], &s[1], &s[2], &s[3], m,   4, 1);
+}
+
+static void crypto_morus1280_load_a(struct morus1280_block *dst, const u8 *src)
+{
+	unsigned int i;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		dst->words[i] = le64_to_cpu(*(const __le64 *)src);
+		src += MORUS1280_WORD_SIZE;
+	}
+}
+
+static void crypto_morus1280_load_u(struct morus1280_block *dst, const u8 *src)
+{
+	unsigned int i;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		dst->words[i] = get_unaligned_le64(src);
+		src += MORUS1280_WORD_SIZE;
+	}
+}
+
+static void crypto_morus1280_load(struct morus1280_block *dst, const u8 *src)
+{
+	if (MORUS1280_ALIGNED(src))
+		crypto_morus1280_load_a(dst, src);
+	else
+		crypto_morus1280_load_u(dst, src);
+}
+
+static void crypto_morus1280_store_a(u8 *dst, const struct morus1280_block *src)
+{
+	unsigned int i;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		*(__le64 *)dst = cpu_to_le64(src->words[i]);
+		dst += MORUS1280_WORD_SIZE;
+	}
+}
+
+static void crypto_morus1280_store_u(u8 *dst, const struct morus1280_block *src)
+{
+	unsigned int i;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		put_unaligned_le64(src->words[i], dst);
+		dst += MORUS1280_WORD_SIZE;
+	}
+}
+
+static void crypto_morus1280_store(u8 *dst, const struct morus1280_block *src)
+{
+	if (MORUS1280_ALIGNED(dst))
+		crypto_morus1280_store_a(dst, src);
+	else
+		crypto_morus1280_store_u(dst, src);
+}
+
+static void crypto_morus1280_ad(struct morus1280_state *state, const u8 *src,
+				unsigned int size)
+{
+	struct morus1280_block m;
+
+	if (MORUS1280_ALIGNED(src)) {
+		while (size >= MORUS1280_BLOCK_SIZE) {
+			crypto_morus1280_load_a(&m, src);
+			crypto_morus1280_update(state, &m);
+
+			size -= MORUS1280_BLOCK_SIZE;
+			src += MORUS1280_BLOCK_SIZE;
+		}
+	} else {
+		while (size >= MORUS1280_BLOCK_SIZE) {
+			crypto_morus1280_load_u(&m, src);
+			crypto_morus1280_update(state, &m);
+
+			size -= MORUS1280_BLOCK_SIZE;
+			src += MORUS1280_BLOCK_SIZE;
+		}
+	}
+}
+
+static void crypto_morus1280_core(const struct morus1280_state *state,
+				  struct morus1280_block *blk)
+{
+	unsigned int i;
+
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++)
+		blk->words[(i + 3) % MORUS_BLOCK_WORDS] ^= state->s[1].words[i];
+
+        for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		blk->words[i] ^= state->s[0].words[i];
+		blk->words[i] ^= state->s[2].words[i] & state->s[3].words[i];
+	}
+}
+
+static void crypto_morus1280_encrypt_chunk(struct morus1280_state *state,
+					   u8 *dst, const u8 *src,
+					   unsigned int size)
+{
+	struct morus1280_block c, m;
+
+	if (MORUS1280_ALIGNED(src) && MORUS1280_ALIGNED(dst)) {
+		while (size >= MORUS1280_BLOCK_SIZE) {
+			crypto_morus1280_load_a(&m, src);
+			c = m;
+			crypto_morus1280_core(state, &c);
+			crypto_morus1280_store_a(dst, &c);
+			crypto_morus1280_update(state, &m);
+
+			src += MORUS1280_BLOCK_SIZE;
+			dst += MORUS1280_BLOCK_SIZE;
+			size -= MORUS1280_BLOCK_SIZE;
+		}
+	} else {
+		while (size >= MORUS1280_BLOCK_SIZE) {
+			crypto_morus1280_load_u(&m, src);
+			c = m;
+			crypto_morus1280_core(state, &c);
+			crypto_morus1280_store_u(dst, &c);
+			crypto_morus1280_update(state, &m);
+
+			src += MORUS1280_BLOCK_SIZE;
+			dst += MORUS1280_BLOCK_SIZE;
+			size -= MORUS1280_BLOCK_SIZE;
+		}
+	}
+
+	if (size > 0) {
+		union morus1280_block_in tail;
+
+		memcpy(tail.bytes, src, size);
+		memset(tail.bytes + size, 0, MORUS1280_BLOCK_SIZE - size);
+
+		crypto_morus1280_load_a(&m, tail.bytes);
+		c = m;
+		crypto_morus1280_core(state, &c);
+		crypto_morus1280_store_a(tail.bytes, &c);
+		crypto_morus1280_update(state, &m);
+
+		memcpy(dst, tail.bytes, size);
+	}
+}
+
+static void crypto_morus1280_decrypt_chunk(struct morus1280_state *state,
+					   u8 *dst, const u8 *src,
+					   unsigned int size)
+{
+	struct morus1280_block m;
+
+	if (MORUS1280_ALIGNED(src) && MORUS1280_ALIGNED(dst)) {
+		while (size >= MORUS1280_BLOCK_SIZE) {
+			crypto_morus1280_load_a(&m, src);
+			crypto_morus1280_core(state, &m);
+			crypto_morus1280_store_a(dst, &m);
+			crypto_morus1280_update(state, &m);
+
+			src += MORUS1280_BLOCK_SIZE;
+			dst += MORUS1280_BLOCK_SIZE;
+			size -= MORUS1280_BLOCK_SIZE;
+		}
+	} else {
+		while (size >= MORUS1280_BLOCK_SIZE) {
+			crypto_morus1280_load_u(&m, src);
+			crypto_morus1280_core(state, &m);
+			crypto_morus1280_store_u(dst, &m);
+			crypto_morus1280_update(state, &m);
+
+			src += MORUS1280_BLOCK_SIZE;
+			dst += MORUS1280_BLOCK_SIZE;
+			size -= MORUS1280_BLOCK_SIZE;
+		}
+	}
+
+	if (size > 0) {
+		union morus1280_block_in tail;
+
+		memcpy(tail.bytes, src, size);
+		memset(tail.bytes + size, 0, MORUS1280_BLOCK_SIZE - size);
+
+		crypto_morus1280_load_a(&m, tail.bytes);
+		crypto_morus1280_core(state, &m);
+		crypto_morus1280_store_a(tail.bytes, &m);
+		memset(tail.bytes + size, 0, MORUS1280_BLOCK_SIZE - size);
+		crypto_morus1280_load_a(&m, tail.bytes);
+		crypto_morus1280_update(state, &m);
+
+		memcpy(dst, tail.bytes, size);
+	}
+}
+
+static void crypto_morus1280_init(struct morus1280_state *state,
+				  const struct morus1280_block *key,
+				  const u8 *iv)
+{
+	static const struct morus1280_block z = {};
+
+	union morus1280_block_in tmp;
+	unsigned int i;
+
+	memcpy(tmp.bytes, iv, MORUS_NONCE_SIZE);
+	memset(tmp.bytes + MORUS_NONCE_SIZE, 0,
+	       MORUS1280_BLOCK_SIZE - MORUS_NONCE_SIZE);
+
+	crypto_morus1280_load(&state->s[0], tmp.bytes);
+	state->s[1] = *key;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++)
+		state->s[2].words[i] = U64_C(0xFFFFFFFFFFFFFFFF);
+	state->s[3] = z;
+	state->s[4] = crypto_morus1280_const[0];
+
+	for (i = 0; i < 16; i++)
+		crypto_morus1280_update(state, &z);
+
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++)
+		state->s[1].words[i] ^= key->words[i];
+}
+
+static void crypto_morus1280_process_ad(struct morus1280_state *state,
+					struct scatterlist *sg_src,
+					unsigned int assoclen)
+{
+	struct scatter_walk walk;
+	struct morus1280_block m;
+	union morus1280_block_in buf;
+	unsigned int pos = 0;
+
+	scatterwalk_start(&walk, sg_src);
+	while (assoclen != 0) {
+		unsigned int size = scatterwalk_clamp(&walk, assoclen);
+		unsigned int left = size;
+		void *mapped = scatterwalk_map(&walk);
+		const u8 *src = (const u8 *)mapped;
+
+		if (pos + size >= MORUS1280_BLOCK_SIZE) {
+			if (pos > 0) {
+				unsigned int fill = MORUS1280_BLOCK_SIZE - pos;
+				memcpy(buf.bytes + pos, src, fill);
+
+				crypto_morus1280_load_a(&m, buf.bytes);
+				crypto_morus1280_update(state, &m);
+
+				pos = 0;
+				left -= fill;
+				src += fill;
+			}
+
+			crypto_morus1280_ad(state, src, left);
+			src += left & ~(MORUS1280_BLOCK_SIZE - 1);
+			left &= MORUS1280_BLOCK_SIZE - 1;
+		}
+
+		memcpy(buf.bytes + pos, src, left);
+
+		pos += left;
+		assoclen -= size;
+		scatterwalk_unmap(mapped);
+		scatterwalk_advance(&walk, size);
+		scatterwalk_done(&walk, 0, assoclen);
+	}
+
+	if (pos > 0) {
+		memset(buf.bytes + pos, 0, MORUS1280_BLOCK_SIZE - pos);
+
+		crypto_morus1280_load_a(&m, buf.bytes);
+		crypto_morus1280_update(state, &m);
+	}
+}
+
+static void crypto_morus1280_process_crypt(struct morus1280_state *state,
+					   struct aead_request *req,
+					   const struct morus1280_ops *ops)
+{
+	struct skcipher_walk walk;
+	u8 *dst;
+	const u8 *src;
+
+	ops->skcipher_walk_init(&walk, req, false);
+
+	while (walk.nbytes) {
+		src = walk.src.virt.addr;
+		dst = walk.dst.virt.addr;
+
+		ops->crypt_chunk(state, dst, src, walk.nbytes);
+
+		skcipher_walk_done(&walk, 0);
+	}
+}
+
+static void crypto_morus1280_final(struct morus1280_state *state,
+				   struct morus1280_block *tag_xor,
+				   u64 assoclen, u64 cryptlen)
+{
+	u64 assocbits = assoclen * 8;
+	u64 cryptbits = cryptlen * 8;
+
+	struct morus1280_block tmp;
+	unsigned int i;
+
+	tmp.words[0] = cpu_to_le64(assocbits);
+	tmp.words[1] = cpu_to_le64(cryptbits);
+	tmp.words[2] = 0;
+	tmp.words[3] = 0;
+
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++)
+		state->s[4].words[i] ^= state->s[0].words[i];
+
+	for (i = 0; i < 10; i++)
+		crypto_morus1280_update(state, &tmp);
+
+	crypto_morus1280_core(state, tag_xor);
+}
+
+static int crypto_morus1280_setkey(struct crypto_aead *aead, const u8 *key,
+				   unsigned int keylen)
+{
+	struct morus1280_ctx *ctx = crypto_aead_ctx(aead);
+	union morus1280_block_in tmp;
+
+	if (keylen == MORUS1280_BLOCK_SIZE)
+		crypto_morus1280_load(&ctx->key, key);
+	else if (keylen == MORUS1280_BLOCK_SIZE / 2) {
+		memcpy(tmp.bytes, key, keylen);
+		memcpy(tmp.bytes + keylen, key, keylen);
+
+		crypto_morus1280_load(&ctx->key, tmp.bytes);
+	} else {
+		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int crypto_morus1280_setauthsize(struct crypto_aead *tfm,
+					unsigned int authsize)
+{
+	return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL;
+}
+
+static void crypto_morus1280_crypt(struct aead_request *req,
+				   struct morus1280_block *tag_xor,
+				   unsigned int cryptlen,
+				   const struct morus1280_ops *ops)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
+	struct morus1280_state state;
+
+	crypto_morus1280_init(&state, &ctx->key, req->iv);
+	crypto_morus1280_process_ad(&state, req->src, req->assoclen);
+	crypto_morus1280_process_crypt(&state, req, ops);
+	crypto_morus1280_final(&state, tag_xor, req->assoclen, cryptlen);
+}
+
+static int crypto_morus1280_encrypt(struct aead_request *req)
+{
+	static const struct morus1280_ops ops = {
+		.skcipher_walk_init = skcipher_walk_aead_encrypt,
+		.crypt_chunk = crypto_morus1280_encrypt_chunk,
+	};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct morus1280_block tag = {};
+	union morus1280_block_in tag_out;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen;
+
+	crypto_morus1280_crypt(req, &tag, cryptlen, &ops);
+	crypto_morus1280_store(tag_out.bytes, &tag);
+
+	scatterwalk_map_and_copy(tag_out.bytes, req->dst,
+				 req->assoclen + cryptlen, authsize, 1);
+	return 0;
+}
+
+static int crypto_morus1280_decrypt(struct aead_request *req)
+{
+	static const struct morus1280_ops ops = {
+		.skcipher_walk_init = skcipher_walk_aead_decrypt,
+		.crypt_chunk = crypto_morus1280_decrypt_chunk,
+	};
+	static const u8 zeros[MORUS1280_BLOCK_SIZE] = {};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	union morus1280_block_in tag_in;
+	struct morus1280_block tag;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen - authsize;
+
+	scatterwalk_map_and_copy(tag_in.bytes, req->src,
+				 req->assoclen + cryptlen, authsize, 0);
+
+	crypto_morus1280_load(&tag, tag_in.bytes);
+	crypto_morus1280_crypt(req, &tag, cryptlen, &ops);
+	crypto_morus1280_store(tag_in.bytes, &tag);
+
+	return crypto_memneq(tag_in.bytes, zeros, authsize) ? -EBADMSG : 0;
+}
+
+static int crypto_morus1280_init_tfm(struct crypto_aead *tfm)
+{
+	return 0;
+}
+
+static void crypto_morus1280_exit_tfm(struct crypto_aead *tfm)
+{
+}
+
+static struct aead_alg crypto_morus1280_alg = {
+	.setkey = crypto_morus1280_setkey,
+	.setauthsize = crypto_morus1280_setauthsize,
+	.encrypt = crypto_morus1280_encrypt,
+	.decrypt = crypto_morus1280_decrypt,
+	.init = crypto_morus1280_init_tfm,
+	.exit = crypto_morus1280_exit_tfm,
+
+	.ivsize = MORUS_NONCE_SIZE,
+	.maxauthsize = MORUS_MAX_AUTH_SIZE,
+	.chunksize = MORUS1280_BLOCK_SIZE,
+
+	.base = {
+		.cra_flags = CRYPTO_ALG_TYPE_AEAD,
+		.cra_blocksize = 1,
+		.cra_ctxsize = sizeof(struct morus1280_ctx),
+		.cra_alignmask = 0,
+
+		.cra_priority = 100,
+
+		.cra_name = "morus1280",
+		.cra_driver_name = "morus1280-generic",
+
+		.cra_module = THIS_MODULE,
+	}
+};
+
+
+static int __init crypto_morus1280_module_init(void)
+{
+	return crypto_register_aead(&crypto_morus1280_alg);
+}
+
+static void __exit crypto_morus1280_module_exit(void)
+{
+	crypto_unregister_aead(&crypto_morus1280_alg);
+}
+
+module_init(crypto_morus1280_module_init);
+module_exit(crypto_morus1280_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm");
+MODULE_ALIAS_CRYPTO("morus1280");
+MODULE_ALIAS_CRYPTO("morus1280-generic");
diff --git a/crypto/morus640.c b/crypto/morus640.c
new file mode 100644
index 000000000000..9fbcde307daf
--- /dev/null
+++ b/crypto/morus640.c
@@ -0,0 +1,544 @@
+/*
+ * The MORUS-640 Authenticated-Encryption Algorithm
+ *
+ * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
+ * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <asm/unaligned.h>
+#include <crypto/algapi.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/morus_common.h>
+#include <crypto/scatterwalk.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+
+#define MORUS640_WORD_SIZE 4
+#define MORUS640_BLOCK_SIZE (MORUS_BLOCK_WORDS * MORUS640_WORD_SIZE)
+#define MORUS640_BLOCK_ALIGN (__alignof__(__le32))
+#define MORUS640_ALIGNED(p) IS_ALIGNED((uintptr_t)p, MORUS640_BLOCK_ALIGN)
+
+struct morus640_block {
+	u32 words[MORUS_BLOCK_WORDS];
+};
+
+union morus640_block_in {
+	__le32 words[MORUS_BLOCK_WORDS];
+	u8 bytes[MORUS640_BLOCK_SIZE];
+};
+
+struct morus640_state {
+	struct morus640_block s[MORUS_STATE_BLOCKS];
+};
+
+struct morus640_ctx {
+	struct morus640_block key;
+};
+
+struct morus640_ops {
+	int (*skcipher_walk_init)(struct skcipher_walk *walk,
+				  struct aead_request *req, bool atomic);
+
+	void (*crypt_chunk)(struct morus640_state *state,
+			    u8 *dst, const u8 *src, unsigned int size);
+};
+
+static const struct morus640_block crypto_morus640_const[2] = {
+	{ .words = {
+		U32_C(0x02010100),
+		U32_C(0x0d080503),
+		U32_C(0x59372215),
+		U32_C(0x6279e990),
+	} },
+	{ .words = {
+		U32_C(0x55183ddb),
+		U32_C(0xf12fc26d),
+		U32_C(0x42311120),
+		U32_C(0xdd28b573),
+	} },
+};
+
+static void crypto_morus640_round(struct morus640_block *b0,
+				  struct morus640_block *b1,
+				  struct morus640_block *b2,
+				  struct morus640_block *b3,
+				  struct morus640_block *b4,
+				  const struct morus640_block *m,
+				  unsigned int b, unsigned int w)
+{
+	unsigned int i;
+	struct morus640_block tmp;
+
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		b0->words[i] ^= b1->words[i] & b2->words[i];
+		b0->words[i] ^= b3->words[i];
+		b0->words[i] ^= m->words[i];
+		b0->words[i] = rol32(b0->words[i], b);
+	}
+
+	tmp = *b3;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++)
+		b3->words[(i + w) % MORUS_BLOCK_WORDS] = tmp.words[i];
+}
+
+static void crypto_morus640_update(struct morus640_state *state,
+				   const struct morus640_block *m)
+{
+	static const struct morus640_block z = {};
+
+	struct morus640_block *s = state->s;
+
+	crypto_morus640_round(&s[0], &s[1], &s[2], &s[3], &s[4], &z,  5, 1);
+	crypto_morus640_round(&s[1], &s[2], &s[3], &s[4], &s[0], m,  31, 2);
+	crypto_morus640_round(&s[2], &s[3], &s[4], &s[0], &s[1], m,   7, 3);
+	crypto_morus640_round(&s[3], &s[4], &s[0], &s[1], &s[2], m,  22, 2);
+	crypto_morus640_round(&s[4], &s[0], &s[1], &s[2], &s[3], m,  13, 1);
+}
+
+static void crypto_morus640_load_a(struct morus640_block *dst, const u8 *src)
+{
+	unsigned int i;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		dst->words[i] = le32_to_cpu(*(const __le32 *)src);
+		src += MORUS640_WORD_SIZE;
+	}
+}
+
+static void crypto_morus640_load_u(struct morus640_block *dst, const u8 *src)
+{
+	unsigned int i;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		dst->words[i] = get_unaligned_le32(src);
+		src += MORUS640_WORD_SIZE;
+	}
+}
+
+static void crypto_morus640_load(struct morus640_block *dst, const u8 *src)
+{
+	if (MORUS640_ALIGNED(src))
+		crypto_morus640_load_a(dst, src);
+	else
+		crypto_morus640_load_u(dst, src);
+}
+
+static void crypto_morus640_store_a(u8 *dst, const struct morus640_block *src)
+{
+	unsigned int i;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		*(__le32 *)dst = cpu_to_le32(src->words[i]);
+		dst += MORUS640_WORD_SIZE;
+	}
+}
+
+static void crypto_morus640_store_u(u8 *dst, const struct morus640_block *src)
+{
+	unsigned int i;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		put_unaligned_le32(src->words[i], dst);
+		dst += MORUS640_WORD_SIZE;
+	}
+}
+
+static void crypto_morus640_store(u8 *dst, const struct morus640_block *src)
+{
+	if (MORUS640_ALIGNED(dst))
+		crypto_morus640_store_a(dst, src);
+	else
+		crypto_morus640_store_u(dst, src);
+}
+
+static void crypto_morus640_ad(struct morus640_state *state, const u8 *src,
+			       unsigned int size)
+{
+	struct morus640_block m;
+
+	if (MORUS640_ALIGNED(src)) {
+		while (size >= MORUS640_BLOCK_SIZE) {
+			crypto_morus640_load_a(&m, src);
+			crypto_morus640_update(state, &m);
+
+			size -= MORUS640_BLOCK_SIZE;
+			src += MORUS640_BLOCK_SIZE;
+		}
+	} else {
+		while (size >= MORUS640_BLOCK_SIZE) {
+			crypto_morus640_load_u(&m, src);
+			crypto_morus640_update(state, &m);
+
+			size -= MORUS640_BLOCK_SIZE;
+			src += MORUS640_BLOCK_SIZE;
+		}
+	}
+}
+
+static void crypto_morus640_core(const struct morus640_state *state,
+				 struct morus640_block *blk)
+{
+	unsigned int i;
+
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++)
+		blk->words[(i + 3) % MORUS_BLOCK_WORDS] ^= state->s[1].words[i];
+
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++) {
+		blk->words[i] ^= state->s[0].words[i];
+		blk->words[i] ^= state->s[2].words[i] & state->s[3].words[i];
+	}
+}
+
+static void crypto_morus640_encrypt_chunk(struct morus640_state *state, u8 *dst,
+					  const u8 *src, unsigned int size)
+{
+	struct morus640_block c, m;
+
+	if (MORUS640_ALIGNED(src) && MORUS640_ALIGNED(dst)) {
+		while (size >= MORUS640_BLOCK_SIZE) {
+			crypto_morus640_load_a(&m, src);
+			c = m;
+			crypto_morus640_core(state, &c);
+			crypto_morus640_store_a(dst, &c);
+			crypto_morus640_update(state, &m);
+
+			src += MORUS640_BLOCK_SIZE;
+			dst += MORUS640_BLOCK_SIZE;
+			size -= MORUS640_BLOCK_SIZE;
+		}
+	} else {
+		while (size >= MORUS640_BLOCK_SIZE) {
+			crypto_morus640_load_u(&m, src);
+			c = m;
+			crypto_morus640_core(state, &c);
+			crypto_morus640_store_u(dst, &c);
+			crypto_morus640_update(state, &m);
+
+			src += MORUS640_BLOCK_SIZE;
+			dst += MORUS640_BLOCK_SIZE;
+			size -= MORUS640_BLOCK_SIZE;
+		}
+	}
+
+	if (size > 0) {
+		union morus640_block_in tail;
+
+		memcpy(tail.bytes, src, size);
+		memset(tail.bytes + size, 0, MORUS640_BLOCK_SIZE - size);
+
+		crypto_morus640_load_a(&m, tail.bytes);
+		c = m;
+		crypto_morus640_core(state, &c);
+		crypto_morus640_store_a(tail.bytes, &c);
+		crypto_morus640_update(state, &m);
+
+		memcpy(dst, tail.bytes, size);
+	}
+}
+
+static void crypto_morus640_decrypt_chunk(struct morus640_state *state, u8 *dst,
+					  const u8 *src, unsigned int size)
+{
+	struct morus640_block m;
+
+	if (MORUS640_ALIGNED(src) && MORUS640_ALIGNED(dst)) {
+		while (size >= MORUS640_BLOCK_SIZE) {
+			crypto_morus640_load_a(&m, src);
+			crypto_morus640_core(state, &m);
+			crypto_morus640_store_a(dst, &m);
+			crypto_morus640_update(state, &m);
+
+			src += MORUS640_BLOCK_SIZE;
+			dst += MORUS640_BLOCK_SIZE;
+			size -= MORUS640_BLOCK_SIZE;
+		}
+	} else {
+		while (size >= MORUS640_BLOCK_SIZE) {
+			crypto_morus640_load_u(&m, src);
+			crypto_morus640_core(state, &m);
+			crypto_morus640_store_u(dst, &m);
+			crypto_morus640_update(state, &m);
+
+			src += MORUS640_BLOCK_SIZE;
+			dst += MORUS640_BLOCK_SIZE;
+			size -= MORUS640_BLOCK_SIZE;
+		}
+	}
+
+	if (size > 0) {
+		union morus640_block_in tail;
+
+		memcpy(tail.bytes, src, size);
+
+		crypto_morus640_load_a(&m, src);
+		crypto_morus640_core(state, &m);
+		crypto_morus640_store_a(tail.bytes, &m);
+		memset(tail.bytes + size, 0, MORUS640_BLOCK_SIZE - size);
+		crypto_morus640_load_a(&m, tail.bytes);
+		crypto_morus640_update(state, &m);
+
+		memcpy(dst, tail.bytes, size);
+	}
+}
+
+static void crypto_morus640_init(struct morus640_state *state,
+				 const struct morus640_block *key,
+				 const u8 *iv)
+{
+	static const struct morus640_block z = {};
+
+	unsigned int i;
+
+	crypto_morus640_load(&state->s[0], iv);
+	state->s[1] = *key;
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++)
+		state->s[2].words[i] = U32_C(0xFFFFFFFF);
+	state->s[3] = crypto_morus640_const[0];
+	state->s[4] = crypto_morus640_const[1];
+
+	for (i = 0; i < 16; i++)
+		crypto_morus640_update(state, &z);
+
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++)
+		state->s[1].words[i] ^= key->words[i];
+}
+
+static void crypto_morus640_process_ad(struct morus640_state *state,
+				       struct scatterlist *sg_src,
+				       unsigned int assoclen)
+{
+	struct scatter_walk walk;
+	struct morus640_block m;
+	union morus640_block_in buf;
+	unsigned int pos = 0;
+
+	scatterwalk_start(&walk, sg_src);
+	while (assoclen != 0) {
+		unsigned int size = scatterwalk_clamp(&walk, assoclen);
+		unsigned int left = size;
+		void *mapped = scatterwalk_map(&walk);
+		const u8 *src = (const u8 *)mapped;
+
+		if (pos + size >= MORUS640_BLOCK_SIZE) {
+			if (pos > 0) {
+				unsigned int fill = MORUS640_BLOCK_SIZE - pos;
+				memcpy(buf.bytes + pos, src, fill);
+
+				crypto_morus640_load_a(&m, buf.bytes);
+				crypto_morus640_update(state, &m);
+
+				pos = 0;
+				left -= fill;
+				src += fill;
+			}
+
+			crypto_morus640_ad(state, src, left);
+			src += left & ~(MORUS640_BLOCK_SIZE - 1);
+			left &= MORUS640_BLOCK_SIZE - 1;
+		}
+
+		memcpy(buf.bytes + pos, src, left);
+
+		pos += left;
+		assoclen -= size;
+		scatterwalk_unmap(mapped);
+		scatterwalk_advance(&walk, size);
+		scatterwalk_done(&walk, 0, assoclen);
+	}
+
+	if (pos > 0) {
+		memset(buf.bytes + pos, 0, MORUS640_BLOCK_SIZE - pos);
+
+		crypto_morus640_load_a(&m, buf.bytes);
+		crypto_morus640_update(state, &m);
+	}
+}
+
+static void crypto_morus640_process_crypt(struct morus640_state *state,
+					  struct aead_request *req,
+					  const struct morus640_ops *ops)
+{
+	struct skcipher_walk walk;
+	u8 *dst;
+	const u8 *src;
+
+	ops->skcipher_walk_init(&walk, req, false);
+
+	while (walk.nbytes) {
+		src = walk.src.virt.addr;
+		dst = walk.dst.virt.addr;
+
+		ops->crypt_chunk(state, dst, src, walk.nbytes);
+
+		skcipher_walk_done(&walk, 0);
+	}
+}
+
+static void crypto_morus640_final(struct morus640_state *state,
+				  struct morus640_block *tag_xor,
+				  u64 assoclen, u64 cryptlen)
+{
+	u64 assocbits = assoclen * 8;
+	u64 cryptbits = cryptlen * 8;
+
+	u32 assocbits_lo = (u32)assocbits;
+	u32 assocbits_hi = (u32)(assocbits >> 32);
+	u32 cryptbits_lo = (u32)cryptbits;
+	u32 cryptbits_hi = (u32)(cryptbits >> 32);
+
+	struct morus640_block tmp;
+	unsigned int i;
+
+	tmp.words[0] = cpu_to_le32(assocbits_lo);
+	tmp.words[1] = cpu_to_le32(assocbits_hi);
+	tmp.words[2] = cpu_to_le32(cryptbits_lo);
+	tmp.words[3] = cpu_to_le32(cryptbits_hi);
+
+	for (i = 0; i < MORUS_BLOCK_WORDS; i++)
+		state->s[4].words[i] ^= state->s[0].words[i];
+
+	for (i = 0; i < 10; i++)
+		crypto_morus640_update(state, &tmp);
+
+	crypto_morus640_core(state, tag_xor);
+}
+
+static int crypto_morus640_setkey(struct crypto_aead *aead, const u8 *key,
+				  unsigned int keylen)
+{
+	struct morus640_ctx *ctx = crypto_aead_ctx(aead);
+
+	if (keylen != MORUS640_BLOCK_SIZE) {
+		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	crypto_morus640_load(&ctx->key, key);
+	return 0;
+}
+
+static int crypto_morus640_setauthsize(struct crypto_aead *tfm,
+				       unsigned int authsize)
+{
+	return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL;
+}
+
+static void crypto_morus640_crypt(struct aead_request *req,
+				  struct morus640_block *tag_xor,
+				  unsigned int cryptlen,
+				  const struct morus640_ops *ops)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
+	struct morus640_state state;
+
+	crypto_morus640_init(&state, &ctx->key, req->iv);
+	crypto_morus640_process_ad(&state, req->src, req->assoclen);
+	crypto_morus640_process_crypt(&state, req, ops);
+	crypto_morus640_final(&state, tag_xor, req->assoclen, cryptlen);
+}
+
+static int crypto_morus640_encrypt(struct aead_request *req)
+{
+	static const struct morus640_ops ops = {
+		.skcipher_walk_init = skcipher_walk_aead_encrypt,
+		.crypt_chunk = crypto_morus640_encrypt_chunk,
+	};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct morus640_block tag = {};
+	union morus640_block_in tag_out;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen;
+
+	crypto_morus640_crypt(req, &tag, cryptlen, &ops);
+	crypto_morus640_store(tag_out.bytes, &tag);
+
+	scatterwalk_map_and_copy(tag_out.bytes, req->dst,
+				 req->assoclen + cryptlen, authsize, 1);
+	return 0;
+}
+
+static int crypto_morus640_decrypt(struct aead_request *req)
+{
+	static const struct morus640_ops ops = {
+		.skcipher_walk_init = skcipher_walk_aead_decrypt,
+		.crypt_chunk = crypto_morus640_decrypt_chunk,
+	};
+	static const u8 zeros[MORUS640_BLOCK_SIZE] = {};
+
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	union morus640_block_in tag_in;
+	struct morus640_block tag;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned int cryptlen = req->cryptlen - authsize;
+
+	scatterwalk_map_and_copy(tag_in.bytes, req->src,
+				 req->assoclen + cryptlen, authsize, 0);
+
+	crypto_morus640_load(&tag, tag_in.bytes);
+	crypto_morus640_crypt(req, &tag, cryptlen, &ops);
+	crypto_morus640_store(tag_in.bytes, &tag);
+
+	return crypto_memneq(tag_in.bytes, zeros, authsize) ? -EBADMSG : 0;
+}
+
+static int crypto_morus640_init_tfm(struct crypto_aead *tfm)
+{
+	return 0;
+}
+
+static void crypto_morus640_exit_tfm(struct crypto_aead *tfm)
+{
+}
+
+static struct aead_alg crypto_morus640_alg = {
+	.setkey = crypto_morus640_setkey,
+	.setauthsize = crypto_morus640_setauthsize,
+	.encrypt = crypto_morus640_encrypt,
+	.decrypt = crypto_morus640_decrypt,
+	.init = crypto_morus640_init_tfm,
+	.exit = crypto_morus640_exit_tfm,
+
+	.ivsize = MORUS_NONCE_SIZE,
+	.maxauthsize = MORUS_MAX_AUTH_SIZE,
+	.chunksize = MORUS640_BLOCK_SIZE,
+
+	.base = {
+		.cra_flags = CRYPTO_ALG_TYPE_AEAD,
+		.cra_blocksize = 1,
+		.cra_ctxsize = sizeof(struct morus640_ctx),
+		.cra_alignmask = 0,
+
+		.cra_priority = 100,
+
+		.cra_name = "morus640",
+		.cra_driver_name = "morus640-generic",
+
+		.cra_module = THIS_MODULE,
+	}
+};
+
+static int __init crypto_morus640_module_init(void)
+{
+	return crypto_register_aead(&crypto_morus640_alg);
+}
+
+static void __exit crypto_morus640_module_exit(void)
+{
+	crypto_unregister_aead(&crypto_morus640_alg);
+}
+
+module_init(crypto_morus640_module_init);
+module_exit(crypto_morus640_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
+MODULE_DESCRIPTION("MORUS-640 AEAD algorithm");
+MODULE_ALIAS_CRYPTO("morus640");
+MODULE_ALIAS_CRYPTO("morus640-generic");
diff --git a/crypto/pcbc.c b/crypto/pcbc.c
index d9e45a958720..ef802f6e9642 100644
--- a/crypto/pcbc.c
+++ b/crypto/pcbc.c
@@ -14,6 +14,7 @@
  *
  */
 
+#include <crypto/algapi.h>
 #include <crypto/internal/skcipher.h>
 #include <linux/err.h>
 #include <linux/init.h>
@@ -72,7 +73,7 @@ static int crypto_pcbc_encrypt_inplace(struct skcipher_request *req,
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *iv = walk->iv;
-	u8 tmpbuf[bsize];
+	u8 tmpbuf[MAX_CIPHER_BLOCKSIZE];
 
 	do {
 		memcpy(tmpbuf, src, bsize);
@@ -144,7 +145,7 @@ static int crypto_pcbc_decrypt_inplace(struct skcipher_request *req,
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *iv = walk->iv;
-	u8 tmpbuf[bsize] __aligned(__alignof__(u32));
+	u8 tmpbuf[MAX_CIPHER_BLOCKSIZE] __aligned(__alignof__(u32));
 
 	do {
 		memcpy(tmpbuf, src, bsize);
diff --git a/crypto/proc.c b/crypto/proc.c
index 822fcef6d91c..f4eb6139973e 100644
--- a/crypto/proc.c
+++ b/crypto/proc.c
@@ -94,21 +94,9 @@ static const struct seq_operations crypto_seq_ops = {
 	.show		= c_show
 };
 
-static int crypto_info_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &crypto_seq_ops);
-}
-        
-static const struct file_operations proc_crypto_ops = {
-	.open		= crypto_info_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release
-};
-
 void __init crypto_init_proc(void)
 {
-	proc_create("crypto", 0, NULL, &proc_crypto_ops);
+	proc_create_seq("crypto", 0, NULL, &crypto_seq_ops);
 }
 
 void __exit crypto_exit_proc(void)
diff --git a/crypto/rsa.c b/crypto/rsa.c
index b067f3a93880..4167980c243d 100644
--- a/crypto/rsa.c
+++ b/crypto/rsa.c
@@ -215,7 +215,6 @@ static int rsa_verify(struct akcipher_request *req)
 		goto err_free_m;
 	}
 
-	ret = -ENOMEM;
 	s = mpi_read_raw_from_sgl(req->src, req->src_len);
 	if (!s) {
 		ret = -ENOMEM;
diff --git a/crypto/salsa20_generic.c b/crypto/salsa20_generic.c
index 5074006a56c3..8c77bc78a09f 100644
--- a/crypto/salsa20_generic.c
+++ b/crypto/salsa20_generic.c
@@ -21,9 +21,17 @@
 
 #include <asm/unaligned.h>
 #include <crypto/internal/skcipher.h>
-#include <crypto/salsa20.h>
 #include <linux/module.h>
 
+#define SALSA20_IV_SIZE        8
+#define SALSA20_MIN_KEY_SIZE  16
+#define SALSA20_MAX_KEY_SIZE  32
+#define SALSA20_BLOCK_SIZE    64
+
+struct salsa20_ctx {
+	u32 initial_state[16];
+};
+
 static void salsa20_block(u32 *state, __le32 *stream)
 {
 	u32 x[16];
@@ -93,16 +101,15 @@ static void salsa20_docrypt(u32 *state, u8 *dst, const u8 *src,
 	}
 }
 
-void crypto_salsa20_init(u32 *state, const struct salsa20_ctx *ctx,
+static void salsa20_init(u32 *state, const struct salsa20_ctx *ctx,
 			 const u8 *iv)
 {
 	memcpy(state, ctx->initial_state, sizeof(ctx->initial_state));
 	state[6] = get_unaligned_le32(iv + 0);
 	state[7] = get_unaligned_le32(iv + 4);
 }
-EXPORT_SYMBOL_GPL(crypto_salsa20_init);
 
-int crypto_salsa20_setkey(struct crypto_skcipher *tfm, const u8 *key,
+static int salsa20_setkey(struct crypto_skcipher *tfm, const u8 *key,
 			  unsigned int keysize)
 {
 	static const char sigma[16] = "expand 32-byte k";
@@ -143,7 +150,6 @@ int crypto_salsa20_setkey(struct crypto_skcipher *tfm, const u8 *key,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(crypto_salsa20_setkey);
 
 static int salsa20_crypt(struct skcipher_request *req)
 {
@@ -155,7 +161,7 @@ static int salsa20_crypt(struct skcipher_request *req)
 
 	err = skcipher_walk_virt(&walk, req, true);
 
-	crypto_salsa20_init(state, ctx, walk.iv);
+	salsa20_init(state, ctx, walk.iv);
 
 	while (walk.nbytes > 0) {
 		unsigned int nbytes = walk.nbytes;
@@ -183,7 +189,7 @@ static struct skcipher_alg alg = {
 	.max_keysize		= SALSA20_MAX_KEY_SIZE,
 	.ivsize			= SALSA20_IV_SIZE,
 	.chunksize		= SALSA20_BLOCK_SIZE,
-	.setkey			= crypto_salsa20_setkey,
+	.setkey			= salsa20_setkey,
 	.encrypt		= salsa20_crypt,
 	.decrypt		= salsa20_crypt,
 };
diff --git a/crypto/sm4_generic.c b/crypto/sm4_generic.c
index f537a2766c55..c18eebfd5edd 100644
--- a/crypto/sm4_generic.c
+++ b/crypto/sm4_generic.c
@@ -190,21 +190,23 @@ static void sm4_do_crypt(const u32 *rk, u32 *out, const u32 *in)
 
 /* encrypt a block of text */
 
-static void sm4_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+void crypto_sm4_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 {
 	const struct crypto_sm4_ctx *ctx = crypto_tfm_ctx(tfm);
 
 	sm4_do_crypt(ctx->rkey_enc, (u32 *)out, (u32 *)in);
 }
+EXPORT_SYMBOL_GPL(crypto_sm4_encrypt);
 
 /* decrypt a block of text */
 
-static void sm4_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+void crypto_sm4_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 {
 	const struct crypto_sm4_ctx *ctx = crypto_tfm_ctx(tfm);
 
 	sm4_do_crypt(ctx->rkey_dec, (u32 *)out, (u32 *)in);
 }
+EXPORT_SYMBOL_GPL(crypto_sm4_decrypt);
 
 static struct crypto_alg sm4_alg = {
 	.cra_name		=	"sm4",
@@ -219,8 +221,8 @@ static struct crypto_alg sm4_alg = {
 			.cia_min_keysize	=	SM4_KEY_SIZE,
 			.cia_max_keysize	=	SM4_KEY_SIZE,
 			.cia_setkey		=	crypto_sm4_set_key,
-			.cia_encrypt		=	sm4_encrypt,
-			.cia_decrypt		=	sm4_decrypt
+			.cia_encrypt		=	crypto_sm4_encrypt,
+			.cia_decrypt		=	crypto_sm4_decrypt
 		}
 	}
 };
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index 51fe7c8744ae..d5bcdd905007 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -158,9 +158,9 @@ struct test_mb_aead_data {
 };
 
 static int do_mult_aead_op(struct test_mb_aead_data *data, int enc,
-				u32 num_mb)
+				u32 num_mb, int *rc)
 {
-	int i, rc[num_mb], err = 0;
+	int i, err = 0;
 
 	/* Fire up a bunch of concurrent requests */
 	for (i = 0; i < num_mb; i++) {
@@ -188,18 +188,26 @@ static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc,
 {
 	unsigned long start, end;
 	int bcount;
-	int ret;
+	int ret = 0;
+	int *rc;
+
+	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
+	if (!rc)
+		return -ENOMEM;
 
 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
 	     time_before(jiffies, end); bcount++) {
-		ret = do_mult_aead_op(data, enc, num_mb);
+		ret = do_mult_aead_op(data, enc, num_mb, rc);
 		if (ret)
-			return ret;
+			goto out;
 	}
 
 	pr_cont("%d operations in %d seconds (%ld bytes)\n",
 		bcount * num_mb, secs, (long)bcount * blen * num_mb);
-	return 0;
+
+out:
+	kfree(rc);
+	return ret;
 }
 
 static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc,
@@ -208,10 +216,15 @@ static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc,
 	unsigned long cycles = 0;
 	int ret = 0;
 	int i;
+	int *rc;
+
+	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
+	if (!rc)
+		return -ENOMEM;
 
 	/* Warm-up run. */
 	for (i = 0; i < 4; i++) {
-		ret = do_mult_aead_op(data, enc, num_mb);
+		ret = do_mult_aead_op(data, enc, num_mb, rc);
 		if (ret)
 			goto out;
 	}
@@ -221,7 +234,7 @@ static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc,
 		cycles_t start, end;
 
 		start = get_cycles();
-		ret = do_mult_aead_op(data, enc, num_mb);
+		ret = do_mult_aead_op(data, enc, num_mb, rc);
 		end = get_cycles();
 
 		if (ret)
@@ -230,11 +243,11 @@ static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc,
 		cycles += end - start;
 	}
 
-out:
-	if (ret == 0)
-		pr_cont("1 operation in %lu cycles (%d bytes)\n",
-			(cycles + 4) / (8 * num_mb), blen);
+	pr_cont("1 operation in %lu cycles (%d bytes)\n",
+		(cycles + 4) / (8 * num_mb), blen);
 
+out:
+	kfree(rc);
 	return ret;
 }
 
@@ -705,9 +718,10 @@ struct test_mb_ahash_data {
 	char *xbuf[XBUFSIZE];
 };
 
-static inline int do_mult_ahash_op(struct test_mb_ahash_data *data, u32 num_mb)
+static inline int do_mult_ahash_op(struct test_mb_ahash_data *data, u32 num_mb,
+				   int *rc)
 {
-	int i, rc[num_mb], err = 0;
+	int i, err = 0;
 
 	/* Fire up a bunch of concurrent requests */
 	for (i = 0; i < num_mb; i++)
@@ -731,18 +745,26 @@ static int test_mb_ahash_jiffies(struct test_mb_ahash_data *data, int blen,
 {
 	unsigned long start, end;
 	int bcount;
-	int ret;
+	int ret = 0;
+	int *rc;
+
+	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
+	if (!rc)
+		return -ENOMEM;
 
 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
 	     time_before(jiffies, end); bcount++) {
-		ret = do_mult_ahash_op(data, num_mb);
+		ret = do_mult_ahash_op(data, num_mb, rc);
 		if (ret)
-			return ret;
+			goto out;
 	}
 
 	pr_cont("%d operations in %d seconds (%ld bytes)\n",
 		bcount * num_mb, secs, (long)bcount * blen * num_mb);
-	return 0;
+
+out:
+	kfree(rc);
+	return ret;
 }
 
 static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen,
@@ -751,10 +773,15 @@ static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen,
 	unsigned long cycles = 0;
 	int ret = 0;
 	int i;
+	int *rc;
+
+	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
+	if (!rc)
+		return -ENOMEM;
 
 	/* Warm-up run. */
 	for (i = 0; i < 4; i++) {
-		ret = do_mult_ahash_op(data, num_mb);
+		ret = do_mult_ahash_op(data, num_mb, rc);
 		if (ret)
 			goto out;
 	}
@@ -764,7 +791,7 @@ static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen,
 		cycles_t start, end;
 
 		start = get_cycles();
-		ret = do_mult_ahash_op(data, num_mb);
+		ret = do_mult_ahash_op(data, num_mb, rc);
 		end = get_cycles();
 
 		if (ret)
@@ -773,11 +800,11 @@ static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen,
 		cycles += end - start;
 	}
 
-out:
-	if (ret == 0)
-		pr_cont("1 operation in %lu cycles (%d bytes)\n",
-			(cycles + 4) / (8 * num_mb), blen);
+	pr_cont("1 operation in %lu cycles (%d bytes)\n",
+		(cycles + 4) / (8 * num_mb), blen);
 
+out:
+	kfree(rc);
 	return ret;
 }
 
@@ -1118,9 +1145,9 @@ struct test_mb_skcipher_data {
 };
 
 static int do_mult_acipher_op(struct test_mb_skcipher_data *data, int enc,
-				u32 num_mb)
+				u32 num_mb, int *rc)
 {
-	int i, rc[num_mb], err = 0;
+	int i, err = 0;
 
 	/* Fire up a bunch of concurrent requests */
 	for (i = 0; i < num_mb; i++) {
@@ -1148,18 +1175,26 @@ static int test_mb_acipher_jiffies(struct test_mb_skcipher_data *data, int enc,
 {
 	unsigned long start, end;
 	int bcount;
-	int ret;
+	int ret = 0;
+	int *rc;
+
+	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
+	if (!rc)
+		return -ENOMEM;
 
 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
 	     time_before(jiffies, end); bcount++) {
-		ret = do_mult_acipher_op(data, enc, num_mb);
+		ret = do_mult_acipher_op(data, enc, num_mb, rc);
 		if (ret)
-			return ret;
+			goto out;
 	}
 
 	pr_cont("%d operations in %d seconds (%ld bytes)\n",
 		bcount * num_mb, secs, (long)bcount * blen * num_mb);
-	return 0;
+
+out:
+	kfree(rc);
+	return ret;
 }
 
 static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc,
@@ -1168,10 +1203,15 @@ static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc,
 	unsigned long cycles = 0;
 	int ret = 0;
 	int i;
+	int *rc;
+
+	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
+	if (!rc)
+		return -ENOMEM;
 
 	/* Warm-up run. */
 	for (i = 0; i < 4; i++) {
-		ret = do_mult_acipher_op(data, enc, num_mb);
+		ret = do_mult_acipher_op(data, enc, num_mb, rc);
 		if (ret)
 			goto out;
 	}
@@ -1181,7 +1221,7 @@ static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc,
 		cycles_t start, end;
 
 		start = get_cycles();
-		ret = do_mult_acipher_op(data, enc, num_mb);
+		ret = do_mult_acipher_op(data, enc, num_mb, rc);
 		end = get_cycles();
 
 		if (ret)
@@ -1190,11 +1230,11 @@ static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc,
 		cycles += end - start;
 	}
 
-out:
-	if (ret == 0)
-		pr_cont("1 operation in %lu cycles (%d bytes)\n",
-			(cycles + 4) / (8 * num_mb), blen);
+	pr_cont("1 operation in %lu cycles (%d bytes)\n",
+		(cycles + 4) / (8 * num_mb), blen);
 
+out:
+	kfree(rc);
 	return ret;
 }
 
@@ -1606,7 +1646,7 @@ static inline int tcrypt_test(const char *alg)
 	return ret;
 }
 
-static int do_test(const char *alg, u32 type, u32 mask, int m)
+static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
 {
 	int i;
 	int ret = 0;
@@ -1621,7 +1661,7 @@ static int do_test(const char *alg, u32 type, u32 mask, int m)
 		}
 
 		for (i = 1; i < 200; i++)
-			ret += do_test(NULL, 0, 0, i);
+			ret += do_test(NULL, 0, 0, i, num_mb);
 		break;
 
 	case 1:
@@ -1902,10 +1942,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m)
 		ret += tcrypt_test("vmac(aes)");
 		break;
 
-	case 110:
-		ret += tcrypt_test("hmac(crc32)");
-		break;
-
 	case 111:
 		ret += tcrypt_test("hmac(sha3-224)");
 		break;
@@ -2903,7 +2939,7 @@ static int __init tcrypt_mod_init(void)
 			goto err_free_tv;
 	}
 
-	err = do_test(alg, type, mask, mode);
+	err = do_test(alg, type, mask, mode, num_mb);
 
 	if (err) {
 		printk(KERN_ERR "tcrypt: one or more tests failed!\n");
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index af4a01c5037b..d1d99843cce4 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -84,10 +84,8 @@ struct aead_test_suite {
 };
 
 struct cipher_test_suite {
-	struct {
-		const struct cipher_testvec *vecs;
-		unsigned int count;
-	} enc, dec;
+	const struct cipher_testvec *vecs;
+	unsigned int count;
 };
 
 struct comp_test_suite {
@@ -988,6 +986,7 @@ static int test_cipher(struct crypto_cipher *tfm, int enc,
 	unsigned int i, j, k;
 	char *q;
 	const char *e;
+	const char *input, *result;
 	void *data;
 	char *xbuf[XBUFSIZE];
 	int ret = -ENOMEM;
@@ -1008,14 +1007,16 @@ static int test_cipher(struct crypto_cipher *tfm, int enc,
 		if (fips_enabled && template[i].fips_skip)
 			continue;
 
+		input  = enc ? template[i].ptext : template[i].ctext;
+		result = enc ? template[i].ctext : template[i].ptext;
 		j++;
 
 		ret = -EINVAL;
-		if (WARN_ON(template[i].ilen > PAGE_SIZE))
+		if (WARN_ON(template[i].len > PAGE_SIZE))
 			goto out;
 
 		data = xbuf[0];
-		memcpy(data, template[i].input, template[i].ilen);
+		memcpy(data, input, template[i].len);
 
 		crypto_cipher_clear_flags(tfm, ~0);
 		if (template[i].wk)
@@ -1031,7 +1032,7 @@ static int test_cipher(struct crypto_cipher *tfm, int enc,
 		} else if (ret)
 			continue;
 
-		for (k = 0; k < template[i].ilen;
+		for (k = 0; k < template[i].len;
 		     k += crypto_cipher_blocksize(tfm)) {
 			if (enc)
 				crypto_cipher_encrypt_one(tfm, data + k,
@@ -1042,10 +1043,10 @@ static int test_cipher(struct crypto_cipher *tfm, int enc,
 		}
 
 		q = data;
-		if (memcmp(q, template[i].result, template[i].rlen)) {
+		if (memcmp(q, result, template[i].len)) {
 			printk(KERN_ERR "alg: cipher: Test %d failed "
 			       "on %s for %s\n", j, e, algo);
-			hexdump(q, template[i].rlen);
+			hexdump(q, template[i].len);
 			ret = -EINVAL;
 			goto out;
 		}
@@ -1073,6 +1074,7 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc,
 	struct scatterlist sgout[8];
 	const char *e, *d;
 	struct crypto_wait wait;
+	const char *input, *result;
 	void *data;
 	char iv[MAX_IVLEN];
 	char *xbuf[XBUFSIZE];
@@ -1116,19 +1118,21 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc,
 		if (fips_enabled && template[i].fips_skip)
 			continue;
 
-		if (template[i].iv)
+		if (template[i].iv && !(template[i].generates_iv && enc))
 			memcpy(iv, template[i].iv, ivsize);
 		else
 			memset(iv, 0, MAX_IVLEN);
 
+		input  = enc ? template[i].ptext : template[i].ctext;
+		result = enc ? template[i].ctext : template[i].ptext;
 		j++;
 		ret = -EINVAL;
-		if (WARN_ON(align_offset + template[i].ilen > PAGE_SIZE))
+		if (WARN_ON(align_offset + template[i].len > PAGE_SIZE))
 			goto out;
 
 		data = xbuf[0];
 		data += align_offset;
-		memcpy(data, template[i].input, template[i].ilen);
+		memcpy(data, input, template[i].len);
 
 		crypto_skcipher_clear_flags(tfm, ~0);
 		if (template[i].wk)
@@ -1144,15 +1148,15 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc,
 		} else if (ret)
 			continue;
 
-		sg_init_one(&sg[0], data, template[i].ilen);
+		sg_init_one(&sg[0], data, template[i].len);
 		if (diff_dst) {
 			data = xoutbuf[0];
 			data += align_offset;
-			sg_init_one(&sgout[0], data, template[i].ilen);
+			sg_init_one(&sgout[0], data, template[i].len);
 		}
 
 		skcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
-					   template[i].ilen, iv);
+					   template[i].len, iv);
 		ret = crypto_wait_req(enc ? crypto_skcipher_encrypt(req) :
 				      crypto_skcipher_decrypt(req), &wait);
 
@@ -1163,17 +1167,16 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc,
 		}
 
 		q = data;
-		if (memcmp(q, template[i].result, template[i].rlen)) {
+		if (memcmp(q, result, template[i].len)) {
 			pr_err("alg: skcipher%s: Test %d failed (invalid result) on %s for %s\n",
 			       d, j, e, algo);
-			hexdump(q, template[i].rlen);
+			hexdump(q, template[i].len);
 			ret = -EINVAL;
 			goto out;
 		}
 
-		if (template[i].iv_out &&
-		    memcmp(iv, template[i].iv_out,
-			   crypto_skcipher_ivsize(tfm))) {
+		if (template[i].generates_iv && enc &&
+		    memcmp(iv, template[i].iv, crypto_skcipher_ivsize(tfm))) {
 			pr_err("alg: skcipher%s: Test %d failed (invalid output IV) on %s for %s\n",
 			       d, j, e, algo);
 			hexdump(iv, crypto_skcipher_ivsize(tfm));
@@ -1194,11 +1197,13 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc,
 		if (fips_enabled && template[i].fips_skip)
 			continue;
 
-		if (template[i].iv)
+		if (template[i].iv && !(template[i].generates_iv && enc))
 			memcpy(iv, template[i].iv, ivsize);
 		else
 			memset(iv, 0, MAX_IVLEN);
 
+		input  = enc ? template[i].ptext : template[i].ctext;
+		result = enc ? template[i].ctext : template[i].ptext;
 		j++;
 		crypto_skcipher_clear_flags(tfm, ~0);
 		if (template[i].wk)
@@ -1226,7 +1231,7 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc,
 
 			q = xbuf[IDX[k] >> PAGE_SHIFT] + offset_in_page(IDX[k]);
 
-			memcpy(q, template[i].input + temp, template[i].tap[k]);
+			memcpy(q, input + temp, template[i].tap[k]);
 
 			if (offset_in_page(q) + template[i].tap[k] < PAGE_SIZE)
 				q[template[i].tap[k]] = 0;
@@ -1248,7 +1253,7 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc,
 		}
 
 		skcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
-					   template[i].ilen, iv);
+					   template[i].len, iv);
 
 		ret = crypto_wait_req(enc ? crypto_skcipher_encrypt(req) :
 				      crypto_skcipher_decrypt(req), &wait);
@@ -1269,8 +1274,7 @@ static int __test_skcipher(struct crypto_skcipher *tfm, int enc,
 				q = xbuf[IDX[k] >> PAGE_SHIFT] +
 				    offset_in_page(IDX[k]);
 
-			if (memcmp(q, template[i].result + temp,
-				   template[i].tap[k])) {
+			if (memcmp(q, result + temp, template[i].tap[k])) {
 				pr_err("alg: skcipher%s: Chunk test %d failed on %s at page %u for %s\n",
 				       d, j, e, k, algo);
 				hexdump(q, template[i].tap[k]);
@@ -1342,19 +1346,30 @@ static int test_comp(struct crypto_comp *tfm,
 		     int ctcount, int dtcount)
 {
 	const char *algo = crypto_tfm_alg_driver_name(crypto_comp_tfm(tfm));
+	char *output, *decomp_output;
 	unsigned int i;
-	char result[COMP_BUF_SIZE];
 	int ret;
 
+	output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
+	if (!output)
+		return -ENOMEM;
+
+	decomp_output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
+	if (!decomp_output) {
+		kfree(output);
+		return -ENOMEM;
+	}
+
 	for (i = 0; i < ctcount; i++) {
 		int ilen;
 		unsigned int dlen = COMP_BUF_SIZE;
 
-		memset(result, 0, sizeof (result));
+		memset(output, 0, sizeof(COMP_BUF_SIZE));
+		memset(decomp_output, 0, sizeof(COMP_BUF_SIZE));
 
 		ilen = ctemplate[i].inlen;
 		ret = crypto_comp_compress(tfm, ctemplate[i].input,
-		                           ilen, result, &dlen);
+					   ilen, output, &dlen);
 		if (ret) {
 			printk(KERN_ERR "alg: comp: compression failed "
 			       "on test %d for %s: ret=%d\n", i + 1, algo,
@@ -1362,7 +1377,17 @@ static int test_comp(struct crypto_comp *tfm,
 			goto out;
 		}
 
-		if (dlen != ctemplate[i].outlen) {
+		ilen = dlen;
+		dlen = COMP_BUF_SIZE;
+		ret = crypto_comp_decompress(tfm, output,
+					     ilen, decomp_output, &dlen);
+		if (ret) {
+			pr_err("alg: comp: compression failed: decompress: on test %d for %s failed: ret=%d\n",
+			       i + 1, algo, -ret);
+			goto out;
+		}
+
+		if (dlen != ctemplate[i].inlen) {
 			printk(KERN_ERR "alg: comp: Compression test %d "
 			       "failed for %s: output len = %d\n", i + 1, algo,
 			       dlen);
@@ -1370,10 +1395,11 @@ static int test_comp(struct crypto_comp *tfm,
 			goto out;
 		}
 
-		if (memcmp(result, ctemplate[i].output, dlen)) {
-			printk(KERN_ERR "alg: comp: Compression test %d "
-			       "failed for %s\n", i + 1, algo);
-			hexdump(result, dlen);
+		if (memcmp(decomp_output, ctemplate[i].input,
+			   ctemplate[i].inlen)) {
+			pr_err("alg: comp: compression failed: output differs: on test %d for %s\n",
+			       i + 1, algo);
+			hexdump(decomp_output, dlen);
 			ret = -EINVAL;
 			goto out;
 		}
@@ -1383,11 +1409,11 @@ static int test_comp(struct crypto_comp *tfm,
 		int ilen;
 		unsigned int dlen = COMP_BUF_SIZE;
 
-		memset(result, 0, sizeof (result));
+		memset(decomp_output, 0, sizeof(COMP_BUF_SIZE));
 
 		ilen = dtemplate[i].inlen;
 		ret = crypto_comp_decompress(tfm, dtemplate[i].input,
-		                             ilen, result, &dlen);
+					     ilen, decomp_output, &dlen);
 		if (ret) {
 			printk(KERN_ERR "alg: comp: decompression failed "
 			       "on test %d for %s: ret=%d\n", i + 1, algo,
@@ -1403,10 +1429,10 @@ static int test_comp(struct crypto_comp *tfm,
 			goto out;
 		}
 
-		if (memcmp(result, dtemplate[i].output, dlen)) {
+		if (memcmp(decomp_output, dtemplate[i].output, dlen)) {
 			printk(KERN_ERR "alg: comp: Decompression test %d "
 			       "failed for %s\n", i + 1, algo);
-			hexdump(result, dlen);
+			hexdump(decomp_output, dlen);
 			ret = -EINVAL;
 			goto out;
 		}
@@ -1415,11 +1441,13 @@ static int test_comp(struct crypto_comp *tfm,
 	ret = 0;
 
 out:
+	kfree(decomp_output);
+	kfree(output);
 	return ret;
 }
 
 static int test_acomp(struct crypto_acomp *tfm,
-		      const struct comp_testvec *ctemplate,
+			      const struct comp_testvec *ctemplate,
 		      const struct comp_testvec *dtemplate,
 		      int ctcount, int dtcount)
 {
@@ -1681,8 +1709,9 @@ static int alg_test_aead(const struct alg_test_desc *desc, const char *driver,
 static int alg_test_cipher(const struct alg_test_desc *desc,
 			   const char *driver, u32 type, u32 mask)
 {
+	const struct cipher_test_suite *suite = &desc->suite.cipher;
 	struct crypto_cipher *tfm;
-	int err = 0;
+	int err;
 
 	tfm = crypto_alloc_cipher(driver, type, mask);
 	if (IS_ERR(tfm)) {
@@ -1691,18 +1720,10 @@ static int alg_test_cipher(const struct alg_test_desc *desc,
 		return PTR_ERR(tfm);
 	}
 
-	if (desc->suite.cipher.enc.vecs) {
-		err = test_cipher(tfm, ENCRYPT, desc->suite.cipher.enc.vecs,
-				  desc->suite.cipher.enc.count);
-		if (err)
-			goto out;
-	}
-
-	if (desc->suite.cipher.dec.vecs)
-		err = test_cipher(tfm, DECRYPT, desc->suite.cipher.dec.vecs,
-				  desc->suite.cipher.dec.count);
+	err = test_cipher(tfm, ENCRYPT, suite->vecs, suite->count);
+	if (!err)
+		err = test_cipher(tfm, DECRYPT, suite->vecs, suite->count);
 
-out:
 	crypto_free_cipher(tfm);
 	return err;
 }
@@ -1710,8 +1731,9 @@ static int alg_test_cipher(const struct alg_test_desc *desc,
 static int alg_test_skcipher(const struct alg_test_desc *desc,
 			     const char *driver, u32 type, u32 mask)
 {
+	const struct cipher_test_suite *suite = &desc->suite.cipher;
 	struct crypto_skcipher *tfm;
-	int err = 0;
+	int err;
 
 	tfm = crypto_alloc_skcipher(driver, type, mask);
 	if (IS_ERR(tfm)) {
@@ -1720,18 +1742,10 @@ static int alg_test_skcipher(const struct alg_test_desc *desc,
 		return PTR_ERR(tfm);
 	}
 
-	if (desc->suite.cipher.enc.vecs) {
-		err = test_skcipher(tfm, ENCRYPT, desc->suite.cipher.enc.vecs,
-				    desc->suite.cipher.enc.count);
-		if (err)
-			goto out;
-	}
-
-	if (desc->suite.cipher.dec.vecs)
-		err = test_skcipher(tfm, DECRYPT, desc->suite.cipher.dec.vecs,
-				    desc->suite.cipher.dec.count);
+	err = test_skcipher(tfm, ENCRYPT, suite->vecs, suite->count);
+	if (!err)
+		err = test_skcipher(tfm, DECRYPT, suite->vecs, suite->count);
 
-out:
 	crypto_free_skcipher(tfm);
 	return err;
 }
@@ -1774,8 +1788,9 @@ static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,
 	return err;
 }
 
-static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
-			 u32 type, u32 mask)
+static int __alg_test_hash(const struct hash_testvec *template,
+			   unsigned int tcount, const char *driver,
+			   u32 type, u32 mask)
 {
 	struct crypto_ahash *tfm;
 	int err;
@@ -1787,16 +1802,51 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
 		return PTR_ERR(tfm);
 	}
 
-	err = test_hash(tfm, desc->suite.hash.vecs,
-			desc->suite.hash.count, true);
+	err = test_hash(tfm, template, tcount, true);
 	if (!err)
-		err = test_hash(tfm, desc->suite.hash.vecs,
-				desc->suite.hash.count, false);
-
+		err = test_hash(tfm, template, tcount, false);
 	crypto_free_ahash(tfm);
 	return err;
 }
 
+static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
+			 u32 type, u32 mask)
+{
+	const struct hash_testvec *template = desc->suite.hash.vecs;
+	unsigned int tcount = desc->suite.hash.count;
+	unsigned int nr_unkeyed, nr_keyed;
+	int err;
+
+	/*
+	 * For OPTIONAL_KEY algorithms, we have to do all the unkeyed tests
+	 * first, before setting a key on the tfm.  To make this easier, we
+	 * require that the unkeyed test vectors (if any) are listed first.
+	 */
+
+	for (nr_unkeyed = 0; nr_unkeyed < tcount; nr_unkeyed++) {
+		if (template[nr_unkeyed].ksize)
+			break;
+	}
+	for (nr_keyed = 0; nr_unkeyed + nr_keyed < tcount; nr_keyed++) {
+		if (!template[nr_unkeyed + nr_keyed].ksize) {
+			pr_err("alg: hash: test vectors for %s out of order, "
+			       "unkeyed ones must come first\n", desc->alg);
+			return -EINVAL;
+		}
+	}
+
+	err = 0;
+	if (nr_unkeyed) {
+		err = __alg_test_hash(template, nr_unkeyed, driver, type, mask);
+		template += nr_unkeyed;
+	}
+
+	if (!err && nr_keyed)
+		err = __alg_test_hash(template, nr_keyed, driver, type, mask);
+
+	return err;
+}
+
 static int alg_test_crc32c(const struct alg_test_desc *desc,
 			   const char *driver, u32 type, u32 mask)
 {
@@ -2316,6 +2366,33 @@ static int alg_test_null(const struct alg_test_desc *desc,
 /* Please keep this list sorted by algorithm name. */
 static const struct alg_test_desc alg_test_descs[] = {
 	{
+		.alg = "aegis128",
+		.test = alg_test_aead,
+		.suite = {
+			.aead = {
+				.enc = __VECS(aegis128_enc_tv_template),
+				.dec = __VECS(aegis128_dec_tv_template),
+			}
+		}
+	}, {
+		.alg = "aegis128l",
+		.test = alg_test_aead,
+		.suite = {
+			.aead = {
+				.enc = __VECS(aegis128l_enc_tv_template),
+				.dec = __VECS(aegis128l_dec_tv_template),
+			}
+		}
+	}, {
+		.alg = "aegis256",
+		.test = alg_test_aead,
+		.suite = {
+			.aead = {
+				.enc = __VECS(aegis256_enc_tv_template),
+				.dec = __VECS(aegis256_dec_tv_template),
+			}
+		}
+	}, {
 		.alg = "ansi_cprng",
 		.test = alg_test_cprng,
 		.suite = {
@@ -2488,93 +2565,70 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(aes_cbc_enc_tv_template),
-				.dec = __VECS(aes_cbc_dec_tv_template)
-			}
-		}
+			.cipher = __VECS(aes_cbc_tv_template)
+		},
 	}, {
 		.alg = "cbc(anubis)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(anubis_cbc_enc_tv_template),
-				.dec = __VECS(anubis_cbc_dec_tv_template)
-			}
-		}
+			.cipher = __VECS(anubis_cbc_tv_template)
+		},
 	}, {
 		.alg = "cbc(blowfish)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(bf_cbc_enc_tv_template),
-				.dec = __VECS(bf_cbc_dec_tv_template)
-			}
-		}
+			.cipher = __VECS(bf_cbc_tv_template)
+		},
 	}, {
 		.alg = "cbc(camellia)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(camellia_cbc_enc_tv_template),
-				.dec = __VECS(camellia_cbc_dec_tv_template)
-			}
-		}
+			.cipher = __VECS(camellia_cbc_tv_template)
+		},
 	}, {
 		.alg = "cbc(cast5)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(cast5_cbc_enc_tv_template),
-				.dec = __VECS(cast5_cbc_dec_tv_template)
-			}
-		}
+			.cipher = __VECS(cast5_cbc_tv_template)
+		},
 	}, {
 		.alg = "cbc(cast6)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(cast6_cbc_enc_tv_template),
-				.dec = __VECS(cast6_cbc_dec_tv_template)
-			}
-		}
+			.cipher = __VECS(cast6_cbc_tv_template)
+		},
 	}, {
 		.alg = "cbc(des)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(des_cbc_enc_tv_template),
-				.dec = __VECS(des_cbc_dec_tv_template)
-			}
-		}
+			.cipher = __VECS(des_cbc_tv_template)
+		},
 	}, {
 		.alg = "cbc(des3_ede)",
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(des3_ede_cbc_enc_tv_template),
-				.dec = __VECS(des3_ede_cbc_dec_tv_template)
-			}
-		}
+			.cipher = __VECS(des3_ede_cbc_tv_template)
+		},
+	}, {
+		/* Same as cbc(aes) except the key is stored in
+		 * hardware secure memory which we reference by index
+		 */
+		.alg = "cbc(paes)",
+		.test = alg_test_null,
+		.fips_allowed = 1,
 	}, {
 		.alg = "cbc(serpent)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(serpent_cbc_enc_tv_template),
-				.dec = __VECS(serpent_cbc_dec_tv_template)
-			}
-		}
+			.cipher = __VECS(serpent_cbc_tv_template)
+		},
 	}, {
 		.alg = "cbc(twofish)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(tf_cbc_enc_tv_template),
-				.dec = __VECS(tf_cbc_dec_tv_template)
-			}
-		}
+			.cipher = __VECS(tf_cbc_tv_template)
+		},
 	}, {
 		.alg = "cbcmac(aes)",
 		.fips_allowed = 1,
@@ -2596,11 +2650,8 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.alg = "chacha20",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(chacha20_enc_tv_template),
-				.dec = __VECS(chacha20_enc_tv_template),
-			}
-		}
+			.cipher = __VECS(chacha20_tv_template)
+		},
 	}, {
 		.alg = "cmac(aes)",
 		.fips_allowed = 1,
@@ -2643,92 +2694,69 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(aes_ctr_enc_tv_template),
-				.dec = __VECS(aes_ctr_dec_tv_template)
-			}
+			.cipher = __VECS(aes_ctr_tv_template)
 		}
 	}, {
 		.alg = "ctr(blowfish)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(bf_ctr_enc_tv_template),
-				.dec = __VECS(bf_ctr_dec_tv_template)
-			}
+			.cipher = __VECS(bf_ctr_tv_template)
 		}
 	}, {
 		.alg = "ctr(camellia)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(camellia_ctr_enc_tv_template),
-				.dec = __VECS(camellia_ctr_dec_tv_template)
-			}
+			.cipher = __VECS(camellia_ctr_tv_template)
 		}
 	}, {
 		.alg = "ctr(cast5)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(cast5_ctr_enc_tv_template),
-				.dec = __VECS(cast5_ctr_dec_tv_template)
-			}
+			.cipher = __VECS(cast5_ctr_tv_template)
 		}
 	}, {
 		.alg = "ctr(cast6)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(cast6_ctr_enc_tv_template),
-				.dec = __VECS(cast6_ctr_dec_tv_template)
-			}
+			.cipher = __VECS(cast6_ctr_tv_template)
 		}
 	}, {
 		.alg = "ctr(des)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(des_ctr_enc_tv_template),
-				.dec = __VECS(des_ctr_dec_tv_template)
-			}
+			.cipher = __VECS(des_ctr_tv_template)
 		}
 	}, {
 		.alg = "ctr(des3_ede)",
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(des3_ede_ctr_enc_tv_template),
-				.dec = __VECS(des3_ede_ctr_dec_tv_template)
-			}
+			.cipher = __VECS(des3_ede_ctr_tv_template)
 		}
+	}, {
+		/* Same as ctr(aes) except the key is stored in
+		 * hardware secure memory which we reference by index
+		 */
+		.alg = "ctr(paes)",
+		.test = alg_test_null,
+		.fips_allowed = 1,
 	}, {
 		.alg = "ctr(serpent)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(serpent_ctr_enc_tv_template),
-				.dec = __VECS(serpent_ctr_dec_tv_template)
-			}
+			.cipher = __VECS(serpent_ctr_tv_template)
 		}
 	}, {
 		.alg = "ctr(twofish)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(tf_ctr_enc_tv_template),
-				.dec = __VECS(tf_ctr_dec_tv_template)
-			}
+			.cipher = __VECS(tf_ctr_tv_template)
 		}
 	}, {
 		.alg = "cts(cbc(aes))",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(cts_mode_enc_tv_template),
-				.dec = __VECS(cts_mode_dec_tv_template)
-			}
+			.cipher = __VECS(cts_mode_tv_template)
 		}
 	}, {
 		.alg = "deflate",
@@ -2876,64 +2904,43 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(aes_enc_tv_template),
-				.dec = __VECS(aes_dec_tv_template)
-			}
+			.cipher = __VECS(aes_tv_template)
 		}
 	}, {
 		.alg = "ecb(anubis)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(anubis_enc_tv_template),
-				.dec = __VECS(anubis_dec_tv_template)
-			}
+			.cipher = __VECS(anubis_tv_template)
 		}
 	}, {
 		.alg = "ecb(arc4)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(arc4_enc_tv_template),
-				.dec = __VECS(arc4_dec_tv_template)
-			}
+			.cipher = __VECS(arc4_tv_template)
 		}
 	}, {
 		.alg = "ecb(blowfish)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(bf_enc_tv_template),
-				.dec = __VECS(bf_dec_tv_template)
-			}
+			.cipher = __VECS(bf_tv_template)
 		}
 	}, {
 		.alg = "ecb(camellia)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(camellia_enc_tv_template),
-				.dec = __VECS(camellia_dec_tv_template)
-			}
+			.cipher = __VECS(camellia_tv_template)
 		}
 	}, {
 		.alg = "ecb(cast5)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(cast5_enc_tv_template),
-				.dec = __VECS(cast5_dec_tv_template)
-			}
+			.cipher = __VECS(cast5_tv_template)
 		}
 	}, {
 		.alg = "ecb(cast6)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(cast6_enc_tv_template),
-				.dec = __VECS(cast6_dec_tv_template)
-			}
+			.cipher = __VECS(cast6_tv_template)
 		}
 	}, {
 		.alg = "ecb(cipher_null)",
@@ -2943,134 +2950,96 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.alg = "ecb(des)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(des_enc_tv_template),
-				.dec = __VECS(des_dec_tv_template)
-			}
+			.cipher = __VECS(des_tv_template)
 		}
 	}, {
 		.alg = "ecb(des3_ede)",
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(des3_ede_enc_tv_template),
-				.dec = __VECS(des3_ede_dec_tv_template)
-			}
+			.cipher = __VECS(des3_ede_tv_template)
 		}
 	}, {
 		.alg = "ecb(fcrypt)",
 		.test = alg_test_skcipher,
 		.suite = {
 			.cipher = {
-				.enc = {
-					.vecs = fcrypt_pcbc_enc_tv_template,
-					.count = 1
-				},
-				.dec = {
-					.vecs = fcrypt_pcbc_dec_tv_template,
-					.count = 1
-				}
+				.vecs = fcrypt_pcbc_tv_template,
+				.count = 1
 			}
 		}
 	}, {
 		.alg = "ecb(khazad)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(khazad_enc_tv_template),
-				.dec = __VECS(khazad_dec_tv_template)
-			}
+			.cipher = __VECS(khazad_tv_template)
 		}
+	}, {
+		/* Same as ecb(aes) except the key is stored in
+		 * hardware secure memory which we reference by index
+		 */
+		.alg = "ecb(paes)",
+		.test = alg_test_null,
+		.fips_allowed = 1,
 	}, {
 		.alg = "ecb(seed)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(seed_enc_tv_template),
-				.dec = __VECS(seed_dec_tv_template)
-			}
+			.cipher = __VECS(seed_tv_template)
 		}
 	}, {
 		.alg = "ecb(serpent)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(serpent_enc_tv_template),
-				.dec = __VECS(serpent_dec_tv_template)
-			}
+			.cipher = __VECS(serpent_tv_template)
 		}
 	}, {
 		.alg = "ecb(sm4)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(sm4_enc_tv_template),
-				.dec = __VECS(sm4_dec_tv_template)
-			}
+			.cipher = __VECS(sm4_tv_template)
 		}
 	}, {
 		.alg = "ecb(speck128)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(speck128_enc_tv_template),
-				.dec = __VECS(speck128_dec_tv_template)
-			}
+			.cipher = __VECS(speck128_tv_template)
 		}
 	}, {
 		.alg = "ecb(speck64)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(speck64_enc_tv_template),
-				.dec = __VECS(speck64_dec_tv_template)
-			}
+			.cipher = __VECS(speck64_tv_template)
 		}
 	}, {
 		.alg = "ecb(tea)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(tea_enc_tv_template),
-				.dec = __VECS(tea_dec_tv_template)
-			}
+			.cipher = __VECS(tea_tv_template)
 		}
 	}, {
 		.alg = "ecb(tnepres)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(tnepres_enc_tv_template),
-				.dec = __VECS(tnepres_dec_tv_template)
-			}
+			.cipher = __VECS(tnepres_tv_template)
 		}
 	}, {
 		.alg = "ecb(twofish)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(tf_enc_tv_template),
-				.dec = __VECS(tf_dec_tv_template)
-			}
+			.cipher = __VECS(tf_tv_template)
 		}
 	}, {
 		.alg = "ecb(xeta)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(xeta_enc_tv_template),
-				.dec = __VECS(xeta_dec_tv_template)
-			}
+			.cipher = __VECS(xeta_tv_template)
 		}
 	}, {
 		.alg = "ecb(xtea)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(xtea_enc_tv_template),
-				.dec = __VECS(xtea_dec_tv_template)
-			}
+			.cipher = __VECS(xtea_tv_template)
 		}
 	}, {
 		.alg = "ecdh",
@@ -3096,12 +3065,6 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.suite = {
 			.hash = __VECS(ghash_tv_template)
 		}
-	}, {
-		.alg = "hmac(crc32)",
-		.test = alg_test_hash,
-		.suite = {
-			.hash = __VECS(bfin_crc_tv_template)
-		}
 	}, {
 		.alg = "hmac(md5)",
 		.test = alg_test_hash,
@@ -3192,55 +3155,37 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(aes_kw_enc_tv_template),
-				.dec = __VECS(aes_kw_dec_tv_template)
-			}
+			.cipher = __VECS(aes_kw_tv_template)
 		}
 	}, {
 		.alg = "lrw(aes)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(aes_lrw_enc_tv_template),
-				.dec = __VECS(aes_lrw_dec_tv_template)
-			}
+			.cipher = __VECS(aes_lrw_tv_template)
 		}
 	}, {
 		.alg = "lrw(camellia)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(camellia_lrw_enc_tv_template),
-				.dec = __VECS(camellia_lrw_dec_tv_template)
-			}
+			.cipher = __VECS(camellia_lrw_tv_template)
 		}
 	}, {
 		.alg = "lrw(cast6)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(cast6_lrw_enc_tv_template),
-				.dec = __VECS(cast6_lrw_dec_tv_template)
-			}
+			.cipher = __VECS(cast6_lrw_tv_template)
 		}
 	}, {
 		.alg = "lrw(serpent)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(serpent_lrw_enc_tv_template),
-				.dec = __VECS(serpent_lrw_dec_tv_template)
-			}
+			.cipher = __VECS(serpent_lrw_tv_template)
 		}
 	}, {
 		.alg = "lrw(twofish)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(tf_lrw_enc_tv_template),
-				.dec = __VECS(tf_lrw_dec_tv_template)
-			}
+			.cipher = __VECS(tf_lrw_tv_template)
 		}
 	}, {
 		.alg = "lz4",
@@ -3290,24 +3235,43 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.suite = {
 			.hash = __VECS(michael_mic_tv_template)
 		}
+	}, {
+		.alg = "morus1280",
+		.test = alg_test_aead,
+		.suite = {
+			.aead = {
+				.enc = __VECS(morus1280_enc_tv_template),
+				.dec = __VECS(morus1280_dec_tv_template),
+			}
+		}
+	}, {
+		.alg = "morus640",
+		.test = alg_test_aead,
+		.suite = {
+			.aead = {
+				.enc = __VECS(morus640_enc_tv_template),
+				.dec = __VECS(morus640_dec_tv_template),
+			}
+		}
 	}, {
 		.alg = "ofb(aes)",
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(aes_ofb_enc_tv_template),
-				.dec = __VECS(aes_ofb_dec_tv_template)
-			}
+			.cipher = __VECS(aes_ofb_tv_template)
 		}
+	}, {
+		/* Same as ofb(aes) except the key is stored in
+		 * hardware secure memory which we reference by index
+		 */
+		.alg = "ofb(paes)",
+		.test = alg_test_null,
+		.fips_allowed = 1,
 	}, {
 		.alg = "pcbc(fcrypt)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(fcrypt_pcbc_enc_tv_template),
-				.dec = __VECS(fcrypt_pcbc_dec_tv_template)
-			}
+			.cipher = __VECS(fcrypt_pcbc_tv_template)
 		}
 	}, {
 		.alg = "pkcs1pad(rsa,sha224)",
@@ -3339,10 +3303,7 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(aes_ctr_rfc3686_enc_tv_template),
-				.dec = __VECS(aes_ctr_rfc3686_dec_tv_template)
-			}
+			.cipher = __VECS(aes_ctr_rfc3686_tv_template)
 		}
 	}, {
 		.alg = "rfc4106(gcm(aes))",
@@ -3426,9 +3387,7 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.alg = "salsa20",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(salsa20_stream_enc_tv_template)
-			}
+			.cipher = __VECS(salsa20_stream_tv_template)
 		}
 	}, {
 		.alg = "sha1",
@@ -3552,65 +3511,59 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(aes_xts_enc_tv_template),
-				.dec = __VECS(aes_xts_dec_tv_template)
-			}
+			.cipher = __VECS(aes_xts_tv_template)
 		}
 	}, {
 		.alg = "xts(camellia)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(camellia_xts_enc_tv_template),
-				.dec = __VECS(camellia_xts_dec_tv_template)
-			}
+			.cipher = __VECS(camellia_xts_tv_template)
 		}
 	}, {
 		.alg = "xts(cast6)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(cast6_xts_enc_tv_template),
-				.dec = __VECS(cast6_xts_dec_tv_template)
-			}
+			.cipher = __VECS(cast6_xts_tv_template)
 		}
+	}, {
+		/* Same as xts(aes) except the key is stored in
+		 * hardware secure memory which we reference by index
+		 */
+		.alg = "xts(paes)",
+		.test = alg_test_null,
+		.fips_allowed = 1,
 	}, {
 		.alg = "xts(serpent)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(serpent_xts_enc_tv_template),
-				.dec = __VECS(serpent_xts_dec_tv_template)
-			}
+			.cipher = __VECS(serpent_xts_tv_template)
 		}
 	}, {
 		.alg = "xts(speck128)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(speck128_xts_enc_tv_template),
-				.dec = __VECS(speck128_xts_dec_tv_template)
-			}
+			.cipher = __VECS(speck128_xts_tv_template)
 		}
 	}, {
 		.alg = "xts(speck64)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(speck64_xts_enc_tv_template),
-				.dec = __VECS(speck64_xts_dec_tv_template)
-			}
+			.cipher = __VECS(speck64_xts_tv_template)
 		}
 	}, {
 		.alg = "xts(twofish)",
 		.test = alg_test_skcipher,
 		.suite = {
-			.cipher = {
-				.enc = __VECS(tf_xts_enc_tv_template),
-				.dec = __VECS(tf_xts_dec_tv_template)
-			}
+			.cipher = __VECS(tf_xts_tv_template)
 		}
+	}, {
+		.alg = "xts4096(paes)",
+		.test = alg_test_null,
+		.fips_allowed = 1,
+	}, {
+		.alg = "xts512(paes)",
+		.test = alg_test_null,
+		.fips_allowed = 1,
 	}, {
 		.alg = "zlib-deflate",
 		.test = alg_test_comp,
@@ -3621,6 +3574,16 @@ static const struct alg_test_desc alg_test_descs[] = {
 				.decomp = __VECS(zlib_deflate_decomp_tv_template)
 			}
 		}
+	}, {
+		.alg = "zstd",
+		.test = alg_test_comp,
+		.fips_allowed = 1,
+		.suite = {
+			.comp = {
+				.comp = __VECS(zstd_comp_tv_template),
+				.decomp = __VECS(zstd_decomp_tv_template)
+			}
+		}
 	}
 };
 
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 004c0a0f8004..b950aa234e43 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -44,14 +44,13 @@ struct hash_testvec {
 };
 
 /*
- * cipher_testvec:	structure to describe a cipher test
- * @key:	A pointer to a key used by the test
- * @klen:	The length of @key
- * @iv:		A pointer to the IV used by the test
- * @input:	A pointer to data used as input
- * @ilen	The length of data in @input
- * @result:	A pointer to what the test need to produce
- * @rlen:	The length of data in @result
+ * cipher_testvec:	structure to describe a symmetric cipher test
+ * @key:	Pointer to key
+ * @klen:	Length of @key in bytes
+ * @iv:		Pointer to IV (optional for some ciphers)
+ * @ptext:	Pointer to plaintext
+ * @ctext:	Pointer to ciphertext
+ * @len:	Length of @ptext and @ctext in bytes
  * @fail:	If set to one, the test need to fail
  * @wk:		Does the test need CRYPTO_TFM_REQ_WEAK_KEY
  * 		( e.g. test needs to fail due to a weak key )
@@ -60,23 +59,23 @@ struct hash_testvec {
  * @also_non_np: 	if set to 1, the test will be also done without
  * 			splitting data in @np SGs
  * @fips_skip:	Skip the test vector in FIPS mode
+ * @generates_iv: Encryption should ignore the given IV, and output @iv.
+ *		  Decryption takes @iv.  Needed for AES Keywrap ("kw(aes)").
  */
-
 struct cipher_testvec {
 	const char *key;
 	const char *iv;
-	const char *iv_out;
-	const char *input;
-	const char *result;
+	const char *ptext;
+	const char *ctext;
 	unsigned short tap[MAX_TAP];
 	int np;
 	unsigned char also_non_np;
 	bool fail;
 	unsigned char wk; /* weak key flag */
 	unsigned char klen;
-	unsigned short ilen;
-	unsigned short rlen;
+	unsigned short len;
 	bool fips_skip;
+	bool generates_iv;
 };
 
 struct aead_testvec {
@@ -5542,111 +5541,121 @@ static const struct hash_testvec poly1305_tv_template[] = {
 /*
  * DES test vectors.
  */
-static const struct cipher_testvec des_enc_tv_template[] = {
+static const struct cipher_testvec des_tv_template[] = {
 	{ /* From Applied Cryptography */
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xe7",
-		.ilen	= 8,
-		.result	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d",
-		.rlen	= 8,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xe7",
+		.ctext	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d",
+		.len	= 8,
 	}, { /* Same key, different plaintext block */
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
-		.input	= "\x22\x33\x44\x55\x66\x77\x88\x99",
-		.ilen	= 8,
-		.result	= "\xf7\x9c\x89\x2a\x33\x8f\x4a\x8b",
-		.rlen	= 8,
+		.ptext	= "\x22\x33\x44\x55\x66\x77\x88\x99",
+		.ctext	= "\xf7\x9c\x89\x2a\x33\x8f\x4a\x8b",
+		.len	= 8,
 	}, { /* Sbox test from NBS */
 		.key	= "\x7c\xa1\x10\x45\x4a\x1a\x6e\x57",
 		.klen	= 8,
-		.input	= "\x01\xa1\xd6\xd0\x39\x77\x67\x42",
-		.ilen	= 8,
-		.result	= "\x69\x0f\x5b\x0d\x9a\x26\x93\x9b",
-		.rlen	= 8,
+		.ptext	= "\x01\xa1\xd6\xd0\x39\x77\x67\x42",
+		.ctext	= "\x69\x0f\x5b\x0d\x9a\x26\x93\x9b",
+		.len	= 8,
 	}, { /* Three blocks */
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
 			  "\x22\x33\x44\x55\x66\x77\x88\x99"
 			  "\xca\xfe\xba\xbe\xfe\xed\xbe\xef",
-		.ilen	= 24,
-		.result	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
+		.ctext	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
 			  "\xf7\x9c\x89\x2a\x33\x8f\x4a\x8b"
 			  "\xb4\x99\x26\xf7\x1f\xe1\xd4\x90",
-		.rlen	= 24,
+		.len	= 24,
 	}, { /* Weak key */
 		.fail	= true,
 		.wk	= 1,
 		.key	= "\x01\x01\x01\x01\x01\x01\x01\x01",
 		.klen	= 8,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xe7",
-		.ilen	= 8,
-		.result	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d",
-		.rlen	= 8,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xe7",
+		.ctext	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d",
+		.len	= 8,
 	}, { /* Two blocks -- for testing encryption across pages */
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
 			  "\x22\x33\x44\x55\x66\x77\x88\x99",
-		.ilen	= 16,
-		.result	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
+		.ctext	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
 			  "\xf7\x9c\x89\x2a\x33\x8f\x4a\x8b",
-		.rlen	= 16,
+		.len	= 16,
+		.np	= 2,
+		.tap	= { 8, 8 }
+	}, {
+		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.klen	= 8,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
+			  "\xa3\x99\x7b\xca\xaf\x69\xa0\xf5",
+		.ctext	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
+			  "\x69\x0f\x5b\x0d\x9a\x26\x93\x9b",
+		.len	= 16,
 		.np	= 2,
 		.tap	= { 8, 8 }
+	}, {
+		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.klen	= 8,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
+			  "\xa3\x99\x7b\xca\xaf\x69\xa0\xf5",
+		.ctext	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
+			  "\x69\x0f\x5b\x0d\x9a\x26\x93\x9b",
+		.len	= 16,
+		.np	= 3,
+		.tap	= { 3, 12, 1 }
 	}, { /* Four blocks -- for testing encryption with chunking */
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
 			  "\x22\x33\x44\x55\x66\x77\x88\x99"
 			  "\xca\xfe\xba\xbe\xfe\xed\xbe\xef"
 			  "\x22\x33\x44\x55\x66\x77\x88\x99",
-		.ilen	= 32,
-		.result	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
+		.ctext	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
 			  "\xf7\x9c\x89\x2a\x33\x8f\x4a\x8b"
 			  "\xb4\x99\x26\xf7\x1f\xe1\xd4\x90"
 			  "\xf7\x9c\x89\x2a\x33\x8f\x4a\x8b",
-		.rlen	= 32,
+		.len	= 32,
 		.np	= 3,
 		.tap	= { 14, 10, 8 }
 	}, {
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
 			  "\x22\x33\x44\x55\x66\x77\x88\x99"
 			  "\xca\xfe\xba\xbe\xfe\xed\xbe\xef",
-		.ilen	= 24,
-		.result	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
+		.ctext	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
 			  "\xf7\x9c\x89\x2a\x33\x8f\x4a\x8b"
 			  "\xb4\x99\x26\xf7\x1f\xe1\xd4\x90",
-		.rlen	= 24,
+		.len	= 24,
 		.np	= 4,
 		.tap	= { 2, 1, 3, 18 }
 	}, {
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
 			  "\x22\x33\x44\x55\x66\x77\x88\x99",
-		.ilen	= 16,
-		.result	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
+		.ctext	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
 			  "\xf7\x9c\x89\x2a\x33\x8f\x4a\x8b",
-		.rlen	= 16,
+		.len	= 16,
 		.np	= 5,
 		.tap	= { 2, 2, 2, 2, 8 }
 	}, {
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xe7",
-		.ilen	= 8,
-		.result	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d",
-		.rlen	= 8,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xe7",
+		.ctext	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d",
+		.len	= 8,
 		.np	= 8,
 		.tap	= { 1, 1, 1, 1, 1, 1, 1, 1 }
 	}, { /* Generated with Crypto++ */
 		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55",
 		.klen	= 8,
-		.input	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
+		.ptext	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
 			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
 			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
 			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
@@ -5677,86 +5686,7 @@ static const struct cipher_testvec des_enc_tv_template[] = {
 			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
 			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
 			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB",
-		.ilen	= 248,
-		.result	= "\x88\xCB\x1F\xAB\x2F\x2A\x49\x57"
-			  "\x92\xB9\x77\xFF\x2F\x47\x58\xDD"
-			  "\xD7\x8A\x91\x95\x26\x33\x78\xB2"
-			  "\x33\xBA\xB2\x3E\x02\xF5\x1F\xEF"
-			  "\x98\xC5\xA6\xD2\x7D\x79\xEC\xB3"
-			  "\x45\xF3\x4C\x61\xAC\x6C\xC2\x55"
-			  "\xE5\xD3\x06\x58\x8A\x42\x3E\xDD"
-			  "\x3D\x20\x45\xE9\x6F\x0D\x25\xA8"
-			  "\xA5\xC7\x69\xCE\xD5\x3B\x7B\xC9"
-			  "\x9E\x65\xE7\xA3\xF2\xE4\x18\x94"
-			  "\xD2\x81\xE9\x33\x2B\x2D\x49\xC4"
-			  "\xFE\xDA\x7F\xE2\xF2\x8C\x9C\xDC"
-			  "\x73\x58\x11\x1F\x81\xD7\x21\x1A"
-			  "\x80\xD0\x0D\xE8\x45\xD6\xD8\xD5"
-			  "\x2E\x51\x16\xCA\x09\x89\x54\x62"
-			  "\xF7\x04\x3D\x75\xB9\xA3\x84\xF4"
-			  "\x62\xF0\x02\x58\x83\xAF\x30\x87"
-			  "\x85\x3F\x01\xCD\x8E\x58\x42\xC4"
-			  "\x41\x73\xE0\x15\x0A\xE6\x2E\x80"
-			  "\x94\xF8\x5B\x3A\x4E\xDF\x51\xB2"
-			  "\x9D\xE4\xC4\x9D\xF7\x3F\xF8\x8E"
-			  "\x37\x22\x4D\x00\x2A\xEF\xC1\x0F"
-			  "\x14\xA0\x66\xAB\x79\x39\xD0\x8E"
-			  "\xE9\x95\x61\x74\x12\xED\x07\xD7"
-			  "\xDD\x95\xDC\x7B\x57\x25\x27\x9C"
-			  "\x51\x96\x16\xF7\x94\x61\xB8\x87"
-			  "\xF0\x21\x1B\x32\xFB\x07\x0F\x29"
-			  "\x56\xBD\x9D\x22\xA2\x9F\xA2\xB9"
-			  "\x46\x31\x4C\x5E\x2E\x95\x61\xEF"
-			  "\xE1\x58\x39\x09\xB4\x8B\x40\xAC"
-			  "\x5F\x62\xC7\x72\xD9\xFC\xCB\x9A",
-		.rlen	= 248,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 248 - 10, 2, 8 },
-	},
-};
-
-static const struct cipher_testvec des_dec_tv_template[] = {
-	{ /* From Applied Cryptography */
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.input	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d",
-		.ilen	= 8,
-		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xe7",
-		.rlen	= 8,
-	}, { /* Sbox test from NBS */
-		.key	= "\x7c\xa1\x10\x45\x4a\x1a\x6e\x57",
-		.klen	= 8,
-		.input	= "\x69\x0f\x5b\x0d\x9a\x26\x93\x9b",
-		.ilen	= 8,
-		.result	= "\x01\xa1\xd6\xd0\x39\x77\x67\x42",
-		.rlen	= 8,
-	}, { /* Two blocks, for chunking test */
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.input	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
-			  "\x69\x0f\x5b\x0d\x9a\x26\x93\x9b",
-		.ilen	= 16,
-		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
-			  "\xa3\x99\x7b\xca\xaf\x69\xa0\xf5",
-		.rlen	= 16,
-		.np	= 2,
-		.tap	= { 8, 8 }
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.input	= "\xc9\x57\x44\x25\x6a\x5e\xd3\x1d"
-			  "\x69\x0f\x5b\x0d\x9a\x26\x93\x9b",
-		.ilen	= 16,
-		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xe7"
-			  "\xa3\x99\x7b\xca\xaf\x69\xa0\xf5",
-		.rlen	= 16,
-		.np	= 3,
-		.tap	= { 3, 12, 1 }
-	}, { /* Generated with Crypto++ */
-		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55",
-		.klen	= 8,
-		.input	= "\x88\xCB\x1F\xAB\x2F\x2A\x49\x57"
+		.ctext	= "\x88\xCB\x1F\xAB\x2F\x2A\x49\x57"
 			  "\x92\xB9\x77\xFF\x2F\x47\x58\xDD"
 			  "\xD7\x8A\x91\x95\x26\x33\x78\xB2"
 			  "\x33\xBA\xB2\x3E\x02\xF5\x1F\xEF"
@@ -5787,102 +5717,68 @@ static const struct cipher_testvec des_dec_tv_template[] = {
 			  "\x46\x31\x4C\x5E\x2E\x95\x61\xEF"
 			  "\xE1\x58\x39\x09\xB4\x8B\x40\xAC"
 			  "\x5F\x62\xC7\x72\xD9\xFC\xCB\x9A",
-		.ilen	= 248,
-		.result	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB",
-		.rlen	= 248,
+		.len	= 248,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 248 - 10, 2, 8 },
 	},
 };
 
-static const struct cipher_testvec des_cbc_enc_tv_template[] = {
+static const struct cipher_testvec des_cbc_tv_template[] = {
 	{ /* From OpenSSL */
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
 		.iv	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.input	= "\x37\x36\x35\x34\x33\x32\x31\x20"
+		.ptext	= "\x37\x36\x35\x34\x33\x32\x31\x20"
 			  "\x4e\x6f\x77\x20\x69\x73\x20\x74"
 			  "\x68\x65\x20\x74\x69\x6d\x65\x20",
-		.ilen	= 24,
-		.result	= "\xcc\xd1\x73\xff\xab\x20\x39\xf4"
+		.ctext	= "\xcc\xd1\x73\xff\xab\x20\x39\xf4"
 			  "\xac\xd8\xae\xfd\xdf\xd8\xa1\xeb"
 			  "\x46\x8e\x91\x15\x78\x88\xba\x68",
-		.rlen	= 24,
+		.len	= 24,
 	}, { /* FIPS Pub 81 */
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
 		.iv	= "\x12\x34\x56\x78\x90\xab\xcd\xef",
-		.input	= "\x4e\x6f\x77\x20\x69\x73\x20\x74",
-		.ilen	= 8,
-		.result	= "\xe5\xc7\xcd\xde\x87\x2b\xf2\x7c",
-		.rlen	= 8,
+		.ptext	= "\x4e\x6f\x77\x20\x69\x73\x20\x74",
+		.ctext	= "\xe5\xc7\xcd\xde\x87\x2b\xf2\x7c",
+		.len	= 8,
 	}, {
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
 		.iv	= "\xe5\xc7\xcd\xde\x87\x2b\xf2\x7c",
-		.input	= "\x68\x65\x20\x74\x69\x6d\x65\x20",
-		.ilen	= 8,
-		.result	= "\x43\xe9\x34\x00\x8c\x38\x9c\x0f",
-		.rlen	= 8,
+		.ptext	= "\x68\x65\x20\x74\x69\x6d\x65\x20",
+		.ctext	= "\x43\xe9\x34\x00\x8c\x38\x9c\x0f",
+		.len	= 8,
 	}, {
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
 		.iv	= "\x43\xe9\x34\x00\x8c\x38\x9c\x0f",
-		.input	= "\x66\x6f\x72\x20\x61\x6c\x6c\x20",
-		.ilen	= 8,
-		.result	= "\x68\x37\x88\x49\x9a\x7c\x05\xf6",
-		.rlen	= 8,
+		.ptext	= "\x66\x6f\x72\x20\x61\x6c\x6c\x20",
+		.ctext	= "\x68\x37\x88\x49\x9a\x7c\x05\xf6",
+		.len	= 8,
+		.np	= 2,
+		.tap	= { 4, 4 },
+		.also_non_np = 1,
 	}, { /* Copy of openssl vector for chunk testing */
 	     /* From OpenSSL */
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.klen	= 8,
 		.iv	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.input	= "\x37\x36\x35\x34\x33\x32\x31\x20"
+		.ptext	= "\x37\x36\x35\x34\x33\x32\x31\x20"
 			  "\x4e\x6f\x77\x20\x69\x73\x20\x74"
 			  "\x68\x65\x20\x74\x69\x6d\x65\x20",
-		.ilen	= 24,
-		.result	= "\xcc\xd1\x73\xff\xab\x20\x39\xf4"
+		.ctext	= "\xcc\xd1\x73\xff\xab\x20\x39\xf4"
 			  "\xac\xd8\xae\xfd\xdf\xd8\xa1\xeb"
 			  "\x46\x8e\x91\x15\x78\x88\xba\x68",
-		.rlen	= 24,
+		.len	= 24,
 		.np	= 2,
 		.tap	= { 13, 11 }
 	}, { /* Generated with Crypto++ */
 		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55",
 		.klen	= 8,
 		.iv	= "\xE7\x82\x1D\xB8\x53\x11\xAC\x47",
-		.input	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
+		.ptext	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
 			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
 			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
 			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
@@ -5913,85 +5809,7 @@ static const struct cipher_testvec des_cbc_enc_tv_template[] = {
 			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
 			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
 			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB",
-		.ilen	= 248,
-		.result	= "\x71\xCC\x56\x1C\x87\x2C\x43\x20"
-			  "\x1C\x20\x13\x09\xF9\x2B\x40\x47"
-			  "\x99\x10\xD1\x1B\x65\x33\x33\xBA"
-			  "\x88\x0D\xA2\xD1\x86\xFF\x4D\xF4"
-			  "\x5A\x0C\x12\x96\x32\x57\xAA\x26"
-			  "\xA7\xF4\x32\x8D\xBC\x10\x31\x9E"
-			  "\x81\x72\x74\xDE\x30\x19\x69\x49"
-			  "\x54\x9C\xC3\xEB\x0B\x97\xDD\xD1"
-			  "\xE8\x6D\x0D\x05\x83\xA5\x12\x08"
-			  "\x47\xF8\x88\x03\x86\x51\x3C\xEF"
-			  "\xE7\x11\x73\x4D\x44\x2B\xE2\x16"
-			  "\xE8\xA5\x06\x50\x66\x70\x0E\x14"
-			  "\xBA\x21\x3B\xD5\x23\x5B\xA7\x8F"
-			  "\x56\xB6\xA7\x44\xDB\x86\xAB\x69"
-			  "\x33\x3C\xBE\x64\xC4\x22\xD3\xFE"
-			  "\x49\x90\x88\x6A\x09\x8F\x76\x59"
-			  "\xCB\xB7\xA0\x2D\x79\x75\x92\x8A"
-			  "\x82\x1D\xC2\xFE\x09\x1F\x78\x6B"
-			  "\x2F\xD6\xA4\x87\x1E\xC4\x53\x63"
-			  "\x80\x02\x61\x2F\xE3\x46\xB6\xB5"
-			  "\xAA\x95\xF4\xEE\xA7\x64\x2B\x4F"
-			  "\x20\xCF\xD2\x47\x4E\x39\x65\xB3"
-			  "\x11\x87\xA2\x6C\x49\x7E\x36\xC7"
-			  "\x62\x8B\x48\x0D\x6A\x64\x00\xBD"
-			  "\x71\x91\x8C\xE9\x70\x19\x01\x4F"
-			  "\x4E\x68\x23\xBA\xDA\x24\x2E\x45"
-			  "\x02\x14\x33\x21\xAE\x58\x4B\xCF"
-			  "\x3B\x4B\xE8\xF8\xF6\x4F\x34\x93"
-			  "\xD7\x07\x8A\xD7\x18\x92\x36\x8C"
-			  "\x82\xA9\xBD\x6A\x31\x91\x39\x11"
-			  "\xC6\x4A\xF3\x55\xC7\x29\x2E\x63",
-		.rlen	= 248,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 248 - 10, 2, 8 },
-	},
-};
-
-static const struct cipher_testvec des_cbc_dec_tv_template[] = {
-	{ /* FIPS Pub 81 */
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.iv	= "\x12\x34\x56\x78\x90\xab\xcd\xef",
-		.input	= "\xe5\xc7\xcd\xde\x87\x2b\xf2\x7c",
-		.ilen	= 8,
-		.result	= "\x4e\x6f\x77\x20\x69\x73\x20\x74",
-		.rlen	= 8,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.iv	= "\xe5\xc7\xcd\xde\x87\x2b\xf2\x7c",
-		.input	= "\x43\xe9\x34\x00\x8c\x38\x9c\x0f",
-		.ilen	= 8,
-		.result	= "\x68\x65\x20\x74\x69\x6d\x65\x20",
-		.rlen	= 8,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.iv	= "\x43\xe9\x34\x00\x8c\x38\x9c\x0f",
-		.input	= "\x68\x37\x88\x49\x9a\x7c\x05\xf6",
-		.ilen	= 8,
-		.result	= "\x66\x6f\x72\x20\x61\x6c\x6c\x20",
-		.rlen	= 8,
-	}, { /* Copy of above, for chunk testing */
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.iv	= "\x43\xe9\x34\x00\x8c\x38\x9c\x0f",
-		.input	= "\x68\x37\x88\x49\x9a\x7c\x05\xf6",
-		.ilen	= 8,
-		.result	= "\x66\x6f\x72\x20\x61\x6c\x6c\x20",
-		.rlen	= 8,
-		.np	= 2,
-		.tap	= { 4, 4 }
-	}, { /* Generated with Crypto++ */
-		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55",
-		.klen	= 8,
-		.iv	= "\xE7\x82\x1D\xB8\x53\x11\xAC\x47",
-		.input	= "\x71\xCC\x56\x1C\x87\x2C\x43\x20"
+		.ctext	= "\x71\xCC\x56\x1C\x87\x2C\x43\x20"
 			  "\x1C\x20\x13\x09\xF9\x2B\x40\x47"
 			  "\x99\x10\xD1\x1B\x65\x33\x33\xBA"
 			  "\x88\x0D\xA2\xD1\x86\xFF\x4D\xF4"
@@ -6022,51 +5840,19 @@ static const struct cipher_testvec des_cbc_dec_tv_template[] = {
 			  "\xD7\x07\x8A\xD7\x18\x92\x36\x8C"
 			  "\x82\xA9\xBD\x6A\x31\x91\x39\x11"
 			  "\xC6\x4A\xF3\x55\xC7\x29\x2E\x63",
-		.ilen	= 248,
-		.result	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB",
-		.rlen	= 248,
+		.len	= 248,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 248 - 10, 2, 8 },
 	},
 };
 
-static const struct cipher_testvec des_ctr_enc_tv_template[] = {
+static const struct cipher_testvec des_ctr_tv_template[] = {
 	{ /* Generated with Crypto++ */
 		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55",
 		.klen	= 8,
 		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
+		.ptext	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
 			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
 			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
 			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
@@ -6097,8 +5883,7 @@ static const struct cipher_testvec des_ctr_enc_tv_template[] = {
 			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
 			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
 			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB",
-		.ilen	= 248,
-		.result	= "\x2F\x96\x06\x0F\x50\xC9\x68\x03"
+		.ctext	= "\x2F\x96\x06\x0F\x50\xC9\x68\x03"
 			  "\x0F\x31\xD4\x64\xA5\x29\x77\x35"
 			  "\xBC\x7A\x9F\x19\xE7\x0D\x33\x3E"
 			  "\x12\x0B\x8C\xAE\x48\xAE\xD9\x02"
@@ -6129,7 +5914,7 @@ static const struct cipher_testvec des_ctr_enc_tv_template[] = {
 			  "\x5C\xC4\x15\xC9\x9A\x21\xC5\xCD"
 			  "\x19\x7F\x99\x19\x53\xCE\x1D\x14"
 			  "\x69\x74\xA1\x06\x46\x0F\x4E\x75",
-		.rlen	= 248,
+		.len	= 248,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 248 - 10, 2, 8 },
@@ -6137,7 +5922,7 @@ static const struct cipher_testvec des_ctr_enc_tv_template[] = {
 		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55",
 		.klen	= 8,
 		.iv	= "\xE7\x82\x1D\xB8\x53\x11\xAC\x47",
-		.input	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
+		.ptext	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
 			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
 			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
 			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
@@ -6168,8 +5953,7 @@ static const struct cipher_testvec des_ctr_enc_tv_template[] = {
 			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
 			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
 			  "\xC6\x2F\xBB\x24\x8D\x19\x82",
-		.ilen	= 247,
-		.result	= "\x62\xE5\xF4\xDC\x99\xE7\x89\xE3"
+		.ctext	= "\x62\xE5\xF4\xDC\x99\xE7\x89\xE3"
 			  "\xF4\x10\xCC\x21\x99\xEB\xDC\x15"
 			  "\x19\x13\x93\x27\x9D\xB6\x6F\x45"
 			  "\x17\x55\x61\x72\xC8\xD3\x7F\xA5"
@@ -6200,193 +5984,44 @@ static const struct cipher_testvec des_ctr_enc_tv_template[] = {
 			  "\xDF\x85\x2D\xE1\xB2\xD6\xAB\x94"
 			  "\xA5\xA6\xE7\xB0\x51\x36\x52\x37"
 			  "\x91\x45\x05\x3E\x58\xBF\x32",
-		.rlen	= 247,
+		.len	= 247,
 		.also_non_np = 1,
 		.np	= 2,
 		.tap	= { 247 - 8, 8 },
 	},
 };
 
-static const struct cipher_testvec des_ctr_dec_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55",
-		.klen	= 8,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x2F\x96\x06\x0F\x50\xC9\x68\x03"
-			  "\x0F\x31\xD4\x64\xA5\x29\x77\x35"
-			  "\xBC\x7A\x9F\x19\xE7\x0D\x33\x3E"
-			  "\x12\x0B\x8C\xAE\x48\xAE\xD9\x02"
-			  "\x0A\xD4\xB0\xD6\x37\xB2\x65\x1C"
-			  "\x4B\x65\xEB\x24\xB5\x8E\xAD\x47"
-			  "\x0D\xDA\x79\x77\xA0\x29\xA0\x2B"
-			  "\xC8\x0F\x85\xDC\x03\x13\xA9\x04"
-			  "\x19\x40\xBE\xBE\x5C\x49\x4A\x69"
-			  "\xED\xE8\xE1\x9E\x14\x43\x74\xDE"
-			  "\xEC\x6E\x11\x3F\x36\xEF\x7B\xFB"
-			  "\xBE\x4C\x91\x43\x22\x65\x72\x48"
-			  "\xE2\x12\xED\x88\xAC\xA7\xC9\x91"
-			  "\x14\xA2\x36\x1C\x29\xFF\xC8\x4F"
-			  "\x72\x5C\x4B\xB0\x1E\x93\xC2\xFA"
-			  "\x9D\x53\x86\xA0\xAE\xC6\xB7\x3C"
-			  "\x59\x0C\xD0\x8F\xA6\xD8\xA4\x31"
-			  "\xB7\x30\x1C\x21\x38\xFB\x68\x8C"
-			  "\x2E\xF5\x6E\x73\xC3\x16\x5F\x12"
-			  "\x0C\x33\xB9\x1E\x7B\x70\xDE\x86"
-			  "\x32\xB3\xC1\x16\xAB\xD9\x49\x0B"
-			  "\x96\x28\x72\x6B\xF3\x30\xA9\xEB"
-			  "\x69\xE2\x1E\x58\x46\xA2\x8E\xC7"
-			  "\xC0\xEF\x07\xB7\x77\x2C\x00\x05"
-			  "\x46\xBD\xFE\x53\x81\x8B\xA4\x03"
-			  "\x20\x0F\xDB\x78\x0B\x1F\x53\x04"
-			  "\x4C\x60\x4C\xC3\x2A\x86\x86\x7E"
-			  "\x13\xD2\x26\xED\x5D\x3E\x9C\xF2"
-			  "\x5C\xC4\x15\xC9\x9A\x21\xC5\xCD"
-			  "\x19\x7F\x99\x19\x53\xCE\x1D\x14"
-			  "\x69\x74\xA1\x06\x46\x0F\x4E\x75",
-		.ilen	= 248,
-		.result	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB",
-		.rlen	= 248,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 248 - 10, 2, 8 },
-	}, { /* Generated with Crypto++ */
-		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55",
-		.klen	= 8,
-		.iv	= "\xE7\x82\x1D\xB8\x53\x11\xAC\x47",
-		.input	= "\x62\xE5\xF4\xDC\x99\xE7\x89\xE3"
-			  "\xF4\x10\xCC\x21\x99\xEB\xDC\x15"
-			  "\x19\x13\x93\x27\x9D\xB6\x6F\x45"
-			  "\x17\x55\x61\x72\xC8\xD3\x7F\xA5"
-			  "\x32\xD0\xD3\x02\x15\xA4\x05\x23"
-			  "\x9C\x23\x61\x60\x77\x7B\x6C\x95"
-			  "\x26\x49\x42\x2E\xF3\xC1\x8C\x6D"
-			  "\xC8\x47\xD5\x94\xE7\x53\xC8\x23"
-			  "\x1B\xA5\x0B\xCB\x12\xD3\x7A\x12"
-			  "\xA4\x42\x15\x34\xF7\x5F\xDC\x58"
-			  "\x5B\x58\x4C\xAD\xD1\x33\x8E\xE6"
-			  "\xE5\xA0\xDA\x4D\x94\x3D\x63\xA8"
-			  "\x02\x82\xBB\x16\xB8\xDC\xB5\x58"
-			  "\xC3\x2D\x79\xE4\x25\x79\x43\xF9"
-			  "\x6D\xD3\xCA\xC0\xE8\x12\xD4\x7E"
-			  "\x04\x25\x79\xFD\x27\xFB\xC4\xEA"
-			  "\x32\x94\x48\x92\xF3\x68\x1A\x7F"
-			  "\x36\x33\x43\x79\xF7\xCA\xC2\x38"
-			  "\xC0\x68\xD4\x53\xA9\xCC\x43\x0C"
-			  "\x40\x57\x3E\xED\x00\x9F\x22\x6E"
-			  "\x80\x99\x0B\xCC\x40\x63\x46\x8A"
-			  "\xE8\xC4\x9B\x6D\x7A\x08\x6E\xA9"
-			  "\x6F\x84\xBC\xB3\xF4\x95\x0B\x2D"
-			  "\x6A\xBA\x37\x50\xC3\xCF\x9F\x7C"
-			  "\x59\x5E\xDE\x0B\x30\xFA\x34\x8A"
-			  "\xF8\xD1\xA2\xF8\x4E\xBD\x5D\x5E"
-			  "\x7D\x71\x99\xE0\xF6\xE5\x7C\xE0"
-			  "\x6D\xEE\x82\x89\x92\xD4\xF5\xD7"
-			  "\xDF\x85\x2D\xE1\xB2\xD6\xAB\x94"
-			  "\xA5\xA6\xE7\xB0\x51\x36\x52\x37"
-			  "\x91\x45\x05\x3E\x58\xBF\x32",
-		.ilen	= 247,
-		.result	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82",
-		.rlen	= 247,
-		.also_non_np = 1,
-		.np	= 2,
-		.tap	= { 247 - 8, 8 },
-	},
-};
-
-static const struct cipher_testvec des3_ede_enc_tv_template[] = {
+static const struct cipher_testvec des3_ede_tv_template[] = {
 	{ /* These are from openssl */
 		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
 			  "\x55\x55\x55\x55\x55\x55\x55\x55"
 			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
 		.klen	= 24,
-		.input	= "\x73\x6f\x6d\x65\x64\x61\x74\x61",
-		.ilen	= 8,
-		.result	= "\x18\xd7\x48\xe5\x63\x62\x05\x72",
-		.rlen	= 8,
+		.ptext	= "\x73\x6f\x6d\x65\x64\x61\x74\x61",
+		.ctext	= "\x18\xd7\x48\xe5\x63\x62\x05\x72",
+		.len	= 8,
 	}, {
 		.key	= "\x03\x52\x02\x07\x67\x20\x82\x17"
 			  "\x86\x02\x87\x66\x59\x08\x21\x98"
 			  "\x64\x05\x6a\xbd\xfe\xa9\x34\x57",
 		.klen	= 24,
-		.input	= "\x73\x71\x75\x69\x67\x67\x6c\x65",
-		.ilen	= 8,
-		.result	= "\xc0\x7d\x2a\x0f\xa5\x66\xfa\x30",
-		.rlen	= 8,
+		.ptext	= "\x73\x71\x75\x69\x67\x67\x6c\x65",
+		.ctext	= "\xc0\x7d\x2a\x0f\xa5\x66\xfa\x30",
+		.len	= 8,
 	}, {
 		.key	= "\x10\x46\x10\x34\x89\x98\x80\x20"
 			  "\x91\x07\xd0\x15\x89\x19\x01\x01"
 			  "\x19\x07\x92\x10\x98\x1a\x01\x01",
 		.klen	= 24,
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 8,
-		.result	= "\xe1\xef\x62\xc3\x32\xfe\x82\x5b",
-		.rlen	= 8,
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\xe1\xef\x62\xc3\x32\xfe\x82\x5b",
+		.len	= 8,
 	}, { /* Generated with Crypto++ */
 		.key	= "\xF3\x9C\xD6\xF3\x9C\xB9\x5A\x67"
 			  "\x00\x5A\x67\x00\x2D\xCE\xEB\x2D"
 			  "\xCE\xEB\xB4\x51\x72\xB4\x51\x72",
 		.klen	= 24,
-		.input	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
+		.ptext	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
 			  "\x8B\x12\x86\x69\xF0\x5B\xCF\x56"
 			  "\x39\xAD\x34\x9F\x66\xEA\x7D\xC4"
 			  "\x48\xD3\xBA\x0D\xB1\x18\xE3\x4A"
@@ -6448,8 +6083,7 @@ static const struct cipher_testvec des3_ede_enc_tv_template[] = {
 			  "\xFB\x42\xF6\x59\x20\x54\x3F\x86"
 			  "\x69\x9D\x64\xCF\x56\xDA\xAD\x34"
 			  "\xB8\x03\xEA\x7D\xE1\x48\xD3\x47",
-		.ilen	= 496,
-		.result	= "\x4E\x9A\x40\x3D\x61\x7D\x17\xFA"
+		.ctext	= "\x4E\x9A\x40\x3D\x61\x7D\x17\xFA"
 			  "\x16\x86\x88\x0B\xD8\xAE\xF8\xE4"
 			  "\x81\x01\x04\x00\x76\xFA\xED\xD3"
 			  "\x44\x7E\x21\x9D\xF0\xFB\x2B\x64"
@@ -6511,110 +6145,61 @@ static const struct cipher_testvec des3_ede_enc_tv_template[] = {
 			  "\x12\xE1\x71\x4A\xF9\x2A\xF5\xF6"
 			  "\x93\x03\xD7\x51\x09\xFA\xBE\x68"
 			  "\xD8\x45\xFF\x33\xBA\xBB\x2B\x63",
-		.rlen	= 496,
+		.len	= 496,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 496 - 20, 4, 16 },
 	},
 };
 
-static const struct cipher_testvec des3_ede_dec_tv_template[] = {
-	{ /* These are from openssl */
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\x55\x55\x55\x55\x55\x55\x55\x55"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.klen	= 24,
-		.input	= "\x18\xd7\x48\xe5\x63\x62\x05\x72",
-		.ilen	= 8,
-		.result	= "\x73\x6f\x6d\x65\x64\x61\x74\x61",
-		.rlen	= 8,
-	}, {
-		.key	= "\x03\x52\x02\x07\x67\x20\x82\x17"
-			  "\x86\x02\x87\x66\x59\x08\x21\x98"
-			  "\x64\x05\x6a\xbd\xfe\xa9\x34\x57",
-		.klen	= 24,
-		.input	= "\xc0\x7d\x2a\x0f\xa5\x66\xfa\x30",
-		.ilen	= 8,
-		.result	= "\x73\x71\x75\x69\x67\x67\x6c\x65",
-		.rlen	= 8,
-	}, {
-		.key	= "\x10\x46\x10\x34\x89\x98\x80\x20"
-			  "\x91\x07\xd0\x15\x89\x19\x01\x01"
-			  "\x19\x07\x92\x10\x98\x1a\x01\x01",
+static const struct cipher_testvec des3_ede_cbc_tv_template[] = {
+	{ /* Generated from openssl */
+		.key	= "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
+			  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
+			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
 		.klen	= 24,
-		.input	= "\xe1\xef\x62\xc3\x32\xfe\x82\x5b",
-		.ilen	= 8,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen	= 8,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.ptext	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.ctext	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
+			  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
+			  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
+			  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
+			  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
+			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
+			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
+			  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
+			  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
+			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
+			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
+			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
+			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
+			  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
+			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
+			  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19",
+		.len	= 128,
 	}, { /* Generated with Crypto++ */
-		.key	= "\xF3\x9C\xD6\xF3\x9C\xB9\x5A\x67"
-			  "\x00\x5A\x67\x00\x2D\xCE\xEB\x2D"
-			  "\xCE\xEB\xB4\x51\x72\xB4\x51\x72",
+		.key	= "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00"
+			  "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE"
+			  "\xEB\xB4\x51\x72\xB4\x51\x72\x1F",
 		.klen	= 24,
-		.input	= "\x4E\x9A\x40\x3D\x61\x7D\x17\xFA"
-			  "\x16\x86\x88\x0B\xD8\xAE\xF8\xE4"
-			  "\x81\x01\x04\x00\x76\xFA\xED\xD3"
-			  "\x44\x7E\x21\x9D\xF0\xFB\x2B\x64"
-			  "\xCA\x4E\x90\xE0\xC0\x63\x28\x92"
-			  "\xF3\x1F\xA4\x53\x2C\x77\xCC\x77"
-			  "\x69\x56\xD0\x19\xAD\x00\x2D\x97"
-			  "\xBC\xDE\x49\x6A\x82\xBC\x16\xE2"
-			  "\x2F\x3E\x72\xEE\xD1\xCE\xFC\x1B"
-			  "\xEA\x32\x56\xE4\x0B\xAF\x27\x36"
-			  "\xAF\x08\xB9\x61\xB7\x48\x23\x27"
-			  "\xEE\x4D\xC8\x79\x56\x06\xEB\xC7"
-			  "\x5B\xCA\x0A\xC6\x5E\x5C\xCB\xB6"
-			  "\x9D\xDA\x04\x59\xE2\x09\x48\x7E"
-			  "\x6B\x37\xC6\xFE\x92\xA9\x1E\x6E"
-			  "\x0D\x19\xFA\x33\x0F\xEE\x36\x68"
-			  "\x11\xBB\xF9\x5A\x73\xAB\x3A\xEA"
-			  "\xAC\x28\xD8\xD5\x27\xE8\x6B\x16"
-			  "\x45\x86\x50\x01\x70\x35\x99\x92"
-			  "\xDF\x0C\x07\x88\x8B\x7F\x9E\x4B"
-			  "\xD2\x04\x84\x90\xC4\x27\xDF\x0A"
-			  "\x49\xA8\xA7\x1A\x6D\x78\x16\xCA"
-			  "\xB3\x18\x5C\xC3\x93\x63\x5A\x68"
-			  "\x77\x02\xBA\xED\x62\x71\xB1\xD9"
-			  "\x5E\xE5\x6F\x1A\xCC\x1D\xBE\x2E"
-			  "\x11\xF3\xA6\x97\xCA\x8E\xBF\xB4"
-			  "\x56\xA1\x36\x6B\xB1\x0A\x3E\x70"
-			  "\xEA\xD7\xCD\x72\x7B\x79\xC8\xAD"
-			  "\x6B\xFE\xFB\xBA\x64\xAE\x19\xC1"
-			  "\x82\xCF\x8A\xA1\x50\x17\x7F\xB2"
-			  "\x6F\x7B\x0F\x52\xC5\x3E\x4A\x52"
-			  "\x3F\xD9\x3F\x01\xA6\x41\x1A\xB3"
-			  "\xB3\x7A\x0E\x8E\x75\xB2\xB1\x5F"
-			  "\xDB\xEA\x84\x13\x26\x6C\x85\x4E"
-			  "\xAE\x6B\xDC\xE7\xE7\xAD\xB0\x06"
-			  "\x5C\xBA\x92\xD0\x30\xBB\x8D\xD2"
-			  "\xAE\x4C\x70\x85\xA0\x07\xE3\x2C"
-			  "\xD1\x27\x9C\xCF\xDB\x13\xB7\xE5"
-			  "\xF9\x6A\x02\xD0\x39\x9D\xB6\xE7"
-			  "\xD1\x17\x25\x08\xF9\xA9\xA6\x67"
-			  "\x38\x80\xD1\x22\xAB\x1A\xD7\x26"
-			  "\xAD\xCA\x19\x1B\xFA\x18\xA7\x57"
-			  "\x31\xEC\xC9\xED\xDB\x79\xC0\x48"
-			  "\xAC\x31\x9F\x03\x8B\x62\x5B\x7E"
-			  "\x0E\xA6\xD0\x64\xEE\xEA\x00\xFC"
-			  "\x58\xC8\xDE\x51\x4E\x17\x15\x11"
-			  "\x66\x58\xB6\x90\xDC\xDF\xA1\x49"
-			  "\xCA\x79\xE9\x31\x31\x42\xDC\x56"
-			  "\x0B\xCD\xB6\x0D\xC7\x64\xF7\x19"
-			  "\xD9\x42\x05\x7F\xBC\x2F\xFC\x90"
-			  "\xAE\x29\x86\xAA\x43\x7A\x4F\x6B"
-			  "\xCE\xEA\xBC\x31\x8D\x65\x9D\x46"
-			  "\xEA\x77\xB4\xF9\x58\xEA\x5D\x84"
-			  "\xE4\xDC\x14\xBB\xBD\x15\x0E\xDA"
-			  "\xD8\xE4\xA4\x5D\x61\xF9\x58\x0F"
-			  "\xE4\x82\x77\xCE\x87\xC0\x09\xF0"
-			  "\xD6\x10\x9E\x34\xE1\x0C\x67\x55"
-			  "\x7B\x6D\xD5\x51\x4B\x00\xEE\xBA"
-			  "\xF2\x7B\xBE\x75\x07\x42\x9D\x99"
-			  "\x12\xE1\x71\x4A\xF9\x2A\xF5\xF6"
-			  "\x93\x03\xD7\x51\x09\xFA\xBE\x68"
-			  "\xD8\x45\xFF\x33\xBA\xBB\x2B\x63",
-		.ilen	= 496,
-		.result	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
+		.iv	= "\xB2\xD7\x48\xED\x06\x44\xF9\x12"
+			  "\xB7\x28\x4D\x83\x24\x59\xF2\x17",
+		.ptext	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
 			  "\x8B\x12\x86\x69\xF0\x5B\xCF\x56"
 			  "\x39\xAD\x34\x9F\x66\xEA\x7D\xC4"
 			  "\x48\xD3\xBA\x0D\xB1\x18\xE3\x4A"
@@ -6676,125 +6261,7 @@ static const struct cipher_testvec des3_ede_dec_tv_template[] = {
 			  "\xFB\x42\xF6\x59\x20\x54\x3F\x86"
 			  "\x69\x9D\x64\xCF\x56\xDA\xAD\x34"
 			  "\xB8\x03\xEA\x7D\xE1\x48\xD3\x47",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec des3_ede_cbc_enc_tv_template[] = {
-	{ /* Generated from openssl */
-		.key	= "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
-			  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
-			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
-		.klen	= 24,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.ilen	= 128,
-		.result	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
-			  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
-			  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
-			  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
-			  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
-			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
-			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
-			  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
-			  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
-			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
-			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
-			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
-			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
-			  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
-			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
-			  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19",
-		.rlen	= 128,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00"
-			  "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE"
-			  "\xEB\xB4\x51\x72\xB4\x51\x72\x1F",
-		.klen	= 24,
-		.iv	= "\xB2\xD7\x48\xED\x06\x44\xF9\x12"
-			  "\xB7\x28\x4D\x83\x24\x59\xF2\x17",
-		.input	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
-			  "\x8B\x12\x86\x69\xF0\x5B\xCF\x56"
-			  "\x39\xAD\x34\x9F\x66\xEA\x7D\xC4"
-			  "\x48\xD3\xBA\x0D\xB1\x18\xE3\x4A"
-			  "\xFE\x41\x28\x5C\x27\x8E\x11\x85"
-			  "\x6C\xF7\x5E\xC2\x55\x3C\xA0\x0B"
-			  "\x92\x65\xE9\x70\xDB\x4F\xD6\xB9"
-			  "\x00\xB4\x1F\xE6\x49\xFD\x44\x2F"
-			  "\x53\x3A\x8D\x14\x98\x63\xCA\x5D"
-			  "\xC1\xA8\x33\xA7\x0E\x91\x78\xEC"
-			  "\x77\xDE\x42\xD5\xBC\x07\x8B\x12"
-			  "\xE5\x4C\xF0\x5B\x22\x56\x39\x80"
-			  "\x6B\x9F\x66\xC9\x50\xC4\xAF\x36"
-			  "\xBA\x0D\x94\x7F\xE3\x4A\xDD\x41"
-			  "\x28\xB3\x1A\x8E\x11\xF8\x43\xF7"
-			  "\x5E\x21\x55\x3C\x87\x6E\x92\x65"
-			  "\xCC\x57\xDB\xA2\x35\xB9\x00\xEB"
-			  "\x72\xE6\x49\xD0\x44\x2F\xB6\x19"
-			  "\x8D\x14\xFF\x46\xCA\x5D\x24\xA8"
-			  "\x33\x9A\x6D\x91\x78\xC3\x77\xDE"
-			  "\xA1\x08\xBC\x07\xEE\x71\xE5\x4C"
-			  "\xD7\x5B\x22\xB5\x1C\x80\x6B\xF2"
-			  "\x45\xC9\x50\x3B\xAF\x36\x99\x60"
-			  "\x94\x7F\xC6\x4A\xDD\xA4\x0F\xB3"
-			  "\x1A\xED\x74\xF8\x43\x2A\x5E\x21"
-			  "\x88\x13\x87\x6E\xF1\x58\xCC\x57"
-			  "\x3E\xA2\x35\x9C\x67\xEB\x72\xC5"
-			  "\x49\xD0\xBB\x02\xB6\x19\xE0\x4B"
-			  "\xFF\x46\x29\x5D\x24\x8F\x16\x9A"
-			  "\x6D\xF4\x5F\xC3\xAA\x3D\xA1\x08"
-			  "\x93\x7A\xEE\x71\xD8\x4C\xD7\xBE"
-			  "\x01\xB5\x1C\xE7\x4E\xF2\x45\x2C"
-			  "\x50\x3B\x82\x15\x99\x60\xCB\x52"
-			  "\xC6\xA9\x30\xA4\x0F\x96\x79\xED"
-			  "\x74\xDF\x43\x2A\xBD\x04\x88\x13"
-			  "\xFA\x4D\xF1\x58\x23\x57\x3E\x81"
-			  "\x68\x9C\x67\xCE\x51\xC5\xAC\x37"
-			  "\xBB\x02\x95\x7C\xE0\x4B\xD2\x46"
-			  "\x29\xB0\x1B\x8F\x16\xF9\x40\xF4"
-			  "\x5F\x26\xAA\x3D\x84\x6F\x93\x7A"
-			  "\xCD\x54\xD8\xA3\x0A\xBE\x01\xE8"
-			  "\x73\xE7\x4E\xD1\x45\x2C\xB7\x1E"
-			  "\x82\x15\xFC\x47\xCB\x52\x25\xA9"
-			  "\x30\x9B\x62\x96\x79\xC0\x74\xDF"
-			  "\xA6\x09\xBD\x04\xEF\x76\xFA\x4D"
-			  "\xD4\x58\x23\x8A\x1D\x81\x68\xF3"
-			  "\x5A\xCE\x51\x38\xAC\x37\x9E\x61"
-			  "\x95\x7C\xC7\x4B\xD2\xA5\x0C\xB0"
-			  "\x1B\xE2\x75\xF9\x40\x2B\x5F\x26"
-			  "\x89\x10\x84\x6F\xF6\x59\xCD\x54"
-			  "\x3F\xA3\x0A\x9D\x64\xE8\x73\xDA"
-			  "\x4E\xD1\xB8\x03\xB7\x1E\xE1\x48"
-			  "\xFC\x47\x2E\x52\x25\x8C\x17\x9B"
-			  "\x62\xF5\x5C\xC0\xAB\x32\xA6\x09"
-			  "\x90\x7B\xEF\x76\xD9\x4D\xD4\xBF"
-			  "\x06\x8A\x1D\xE4\x4F\xF3\x5A\x2D"
-			  "\x51\x38\x83\x6A\x9E\x61\xC8\x53"
-			  "\xC7\xAE\x31\xA5\x0C\x97\x7E\xE2"
-			  "\x75\xDC\x40\x2B\xB2\x05\x89\x10"
-			  "\xFB\x42\xF6\x59\x20\x54\x3F\x86"
-			  "\x69\x9D\x64\xCF\x56\xDA\xAD\x34"
-			  "\xB8\x03\xEA\x7D\xE1\x48\xD3\x47",
-		.ilen	= 496,
-		.result	= "\xF8\xF6\xB5\x60\x5C\x5A\x75\x84"
+		.ctext	= "\xF8\xF6\xB5\x60\x5C\x5A\x75\x84"
 			  "\x87\x81\x53\xBA\xC9\x6F\xEC\xD5"
 			  "\x1E\x68\x8E\x85\x12\x86\x1D\x38"
 			  "\x1C\x91\x40\xCC\x69\x6A\xD5\x35"
@@ -6856,125 +6323,22 @@ static const struct cipher_testvec des3_ede_cbc_enc_tv_template[] = {
 			  "\x90\xE9\xFA\x4B\x00\x10\xAC\x58"
 			  "\x83\x70\xFF\x86\xE6\xAA\x0F\x1F"
 			  "\x95\x63\x73\xA2\x44\xAC\xF8\xA5",
-		.rlen	= 496,
+		.len	= 496,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 496 - 20, 4, 16 },
 	},
 };
 
-static const struct cipher_testvec des3_ede_cbc_dec_tv_template[] = {
-	{ /* Generated from openssl */
-		.key	= "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
-			  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
-			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
-		.klen	= 24,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.input	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
-			  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
-			  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
-			  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
-			  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
-			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
-			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
-			  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
-			  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
-			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
-			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
-			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
-			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
-			  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
-			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
-			  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19",
-		.ilen	= 128,
-		.result	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.rlen	= 128,
-	}, { /* Generated with Crypto++ */
+static const struct cipher_testvec des3_ede_ctr_tv_template[] = {
+	{ /* Generated with Crypto++ */
 		.key	= "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00"
 			  "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE"
 			  "\xEB\xB4\x51\x72\xB4\x51\x72\x1F",
 		.klen	= 24,
-		.iv	= "\xB2\xD7\x48\xED\x06\x44\xF9\x12"
-			  "\xB7\x28\x4D\x83\x24\x59\xF2\x17",
-		.input	= "\xF8\xF6\xB5\x60\x5C\x5A\x75\x84"
-			  "\x87\x81\x53\xBA\xC9\x6F\xEC\xD5"
-			  "\x1E\x68\x8E\x85\x12\x86\x1D\x38"
-			  "\x1C\x91\x40\xCC\x69\x6A\xD5\x35"
-			  "\x0D\x7C\xB5\x07\x7C\x7B\x2A\xAF"
-			  "\x32\xBC\xA1\xB3\x84\x31\x1B\x3C"
-			  "\x0A\x2B\xFA\xD3\x9F\xB0\x8C\x37"
-			  "\x8F\x9D\xA7\x6D\x6C\xFA\xD7\x90"
-			  "\xE3\x69\x54\xED\x3A\xC4\xF1\x6B"
-			  "\xB1\xCC\xFB\x7D\xD8\x8E\x17\x0B"
-			  "\x9C\xF6\x4C\xD6\xFF\x03\x4E\xD9"
-			  "\xE6\xA5\xAD\x25\xE6\x17\x69\x63"
-			  "\x11\x35\x61\x94\x88\x7B\x1C\x48"
-			  "\xF1\x24\x20\x29\x6B\x93\x1A\x8E"
-			  "\x43\x03\x89\xD8\xB1\xDA\x47\x7B"
-			  "\x79\x3A\x83\x76\xDA\xAE\xC6\xBB"
-			  "\x22\xF8\xE8\x3D\x9A\x65\x54\xD8"
-			  "\x4C\xE9\xE7\xE4\x63\x2F\x5C\x73"
-			  "\x5A\xC3\xAE\x46\xA8\xCD\x57\xE6"
-			  "\x67\x88\xA5\x20\x6F\x5F\x97\xC7"
-			  "\xCC\x15\xA2\x0A\x93\xEA\x33\xE7"
-			  "\x03\x5F\xEC\x64\x30\x6F\xEE\xD7"
-			  "\x7E\xDF\xD6\xE9\x6F\x3F\xD6\x1E"
-			  "\xBE\x67\x6C\x5B\x97\xA0\x09\xE6"
-			  "\xEE\xFE\x55\xA3\x29\x65\xE0\x12"
-			  "\xA1\x6A\x8A\x6F\xF2\xE6\xF1\x96"
-			  "\x87\xFB\x9C\x05\xDD\x80\xEC\xFF"
-			  "\xC5\xED\x50\xFE\xFC\x91\xCD\xCE"
-			  "\x25\x2C\x5F\xD9\xAD\x95\x7D\x99"
-			  "\xF0\x05\xC4\x71\x46\x5F\xF9\x0D"
-			  "\xD2\x63\xDF\x9B\x96\x2E\x2B\xA6"
-			  "\x2B\x1C\xD5\xFB\x96\x24\x60\x60"
-			  "\x54\x40\xB8\x62\xA4\xF8\x46\x95"
-			  "\x73\x28\xA3\xA6\x16\x2B\x17\xE7"
-			  "\x7A\xF8\x62\x54\x3B\x64\x69\xE1"
-			  "\x71\x34\x29\x5B\x4E\x05\x9B\xFA"
-			  "\x5E\xF1\x96\xB7\xCE\x16\x9B\x59"
-			  "\xF1\x1A\x4C\x51\x26\xFD\x79\xE2"
-			  "\x3B\x8E\x71\x69\x6A\x91\xB6\x65"
-			  "\x32\x09\xB8\xE4\x09\x1F\xEA\x39"
-			  "\xCE\x20\x65\x9F\xD6\xD1\xC7\xF0"
-			  "\x73\x50\x08\x56\x20\x9B\x94\x23"
-			  "\x14\x39\xB7\x2B\xB1\x2D\x6D\x6F"
-			  "\x41\x5B\xCC\xE2\x18\xAE\x62\x89"
-			  "\x78\x8E\x67\x23\xD0\xFB\x2B\xE5"
-			  "\x25\xC9\x48\x97\xB5\xD3\x17\xD5"
-			  "\x6A\x9F\xA7\x48\x0C\x2B\x73\x3B"
-			  "\x57\x08\xAE\x91\xF2\xB7\x57\x89"
-			  "\xF4\xD0\xB0\x07\xB0\x42\x6C\xAF"
-			  "\x98\x1A\xE7\xD1\xAC\x1E\xB5\x02"
-			  "\xD4\x56\x42\x79\x79\x7F\x2A\x77"
-			  "\x25\xE9\x7D\xC1\x88\x19\x2B\x49"
-			  "\x6F\x46\x59\xAB\x56\x1F\x61\xE0"
-			  "\x0C\x24\x9C\xC9\x5B\x63\xA9\x12"
-			  "\xCF\x88\x96\xB6\xA8\x24\xC6\xA8"
-			  "\x21\x85\x1A\x62\x7E\x34\xBB\xEB"
-			  "\xBD\x02\x2A\xC7\xD8\x89\x80\xC5"
-			  "\xB1\xBB\x60\xA5\x22\xFC\x6F\x38"
-			  "\x02\x80\xA3\x28\x22\x75\xE1\xE9"
-			  "\x90\xE9\xFA\x4B\x00\x10\xAC\x58"
-			  "\x83\x70\xFF\x86\xE6\xAA\x0F\x1F"
-			  "\x95\x63\x73\xA2\x44\xAC\xF8\xA5",
-		.ilen	= 496,
-		.result	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.ptext	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
 			  "\x8B\x12\x86\x69\xF0\x5B\xCF\x56"
 			  "\x39\xAD\x34\x9F\x66\xEA\x7D\xC4"
 			  "\x48\xD3\xBA\x0D\xB1\x18\xE3\x4A"
@@ -7036,147 +6400,69 @@ static const struct cipher_testvec des3_ede_cbc_dec_tv_template[] = {
 			  "\xFB\x42\xF6\x59\x20\x54\x3F\x86"
 			  "\x69\x9D\x64\xCF\x56\xDA\xAD\x34"
 			  "\xB8\x03\xEA\x7D\xE1\x48\xD3\x47",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec des3_ede_ctr_enc_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00"
-			  "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE"
-			  "\xEB\xB4\x51\x72\xB4\x51\x72\x1F",
-		.klen	= 24,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
-			  "\x8B\x12\x86\x69\xF0\x5B\xCF\x56"
-			  "\x39\xAD\x34\x9F\x66\xEA\x7D\xC4"
-			  "\x48\xD3\xBA\x0D\xB1\x18\xE3\x4A"
-			  "\xFE\x41\x28\x5C\x27\x8E\x11\x85"
-			  "\x6C\xF7\x5E\xC2\x55\x3C\xA0\x0B"
-			  "\x92\x65\xE9\x70\xDB\x4F\xD6\xB9"
-			  "\x00\xB4\x1F\xE6\x49\xFD\x44\x2F"
-			  "\x53\x3A\x8D\x14\x98\x63\xCA\x5D"
-			  "\xC1\xA8\x33\xA7\x0E\x91\x78\xEC"
-			  "\x77\xDE\x42\xD5\xBC\x07\x8B\x12"
-			  "\xE5\x4C\xF0\x5B\x22\x56\x39\x80"
-			  "\x6B\x9F\x66\xC9\x50\xC4\xAF\x36"
-			  "\xBA\x0D\x94\x7F\xE3\x4A\xDD\x41"
-			  "\x28\xB3\x1A\x8E\x11\xF8\x43\xF7"
-			  "\x5E\x21\x55\x3C\x87\x6E\x92\x65"
-			  "\xCC\x57\xDB\xA2\x35\xB9\x00\xEB"
-			  "\x72\xE6\x49\xD0\x44\x2F\xB6\x19"
-			  "\x8D\x14\xFF\x46\xCA\x5D\x24\xA8"
-			  "\x33\x9A\x6D\x91\x78\xC3\x77\xDE"
-			  "\xA1\x08\xBC\x07\xEE\x71\xE5\x4C"
-			  "\xD7\x5B\x22\xB5\x1C\x80\x6B\xF2"
-			  "\x45\xC9\x50\x3B\xAF\x36\x99\x60"
-			  "\x94\x7F\xC6\x4A\xDD\xA4\x0F\xB3"
-			  "\x1A\xED\x74\xF8\x43\x2A\x5E\x21"
-			  "\x88\x13\x87\x6E\xF1\x58\xCC\x57"
-			  "\x3E\xA2\x35\x9C\x67\xEB\x72\xC5"
-			  "\x49\xD0\xBB\x02\xB6\x19\xE0\x4B"
-			  "\xFF\x46\x29\x5D\x24\x8F\x16\x9A"
-			  "\x6D\xF4\x5F\xC3\xAA\x3D\xA1\x08"
-			  "\x93\x7A\xEE\x71\xD8\x4C\xD7\xBE"
-			  "\x01\xB5\x1C\xE7\x4E\xF2\x45\x2C"
-			  "\x50\x3B\x82\x15\x99\x60\xCB\x52"
-			  "\xC6\xA9\x30\xA4\x0F\x96\x79\xED"
-			  "\x74\xDF\x43\x2A\xBD\x04\x88\x13"
-			  "\xFA\x4D\xF1\x58\x23\x57\x3E\x81"
-			  "\x68\x9C\x67\xCE\x51\xC5\xAC\x37"
-			  "\xBB\x02\x95\x7C\xE0\x4B\xD2\x46"
-			  "\x29\xB0\x1B\x8F\x16\xF9\x40\xF4"
-			  "\x5F\x26\xAA\x3D\x84\x6F\x93\x7A"
-			  "\xCD\x54\xD8\xA3\x0A\xBE\x01\xE8"
-			  "\x73\xE7\x4E\xD1\x45\x2C\xB7\x1E"
-			  "\x82\x15\xFC\x47\xCB\x52\x25\xA9"
-			  "\x30\x9B\x62\x96\x79\xC0\x74\xDF"
-			  "\xA6\x09\xBD\x04\xEF\x76\xFA\x4D"
-			  "\xD4\x58\x23\x8A\x1D\x81\x68\xF3"
-			  "\x5A\xCE\x51\x38\xAC\x37\x9E\x61"
-			  "\x95\x7C\xC7\x4B\xD2\xA5\x0C\xB0"
-			  "\x1B\xE2\x75\xF9\x40\x2B\x5F\x26"
-			  "\x89\x10\x84\x6F\xF6\x59\xCD\x54"
-			  "\x3F\xA3\x0A\x9D\x64\xE8\x73\xDA"
-			  "\x4E\xD1\xB8\x03\xB7\x1E\xE1\x48"
-			  "\xFC\x47\x2E\x52\x25\x8C\x17\x9B"
-			  "\x62\xF5\x5C\xC0\xAB\x32\xA6\x09"
-			  "\x90\x7B\xEF\x76\xD9\x4D\xD4\xBF"
-			  "\x06\x8A\x1D\xE4\x4F\xF3\x5A\x2D"
-			  "\x51\x38\x83\x6A\x9E\x61\xC8\x53"
-			  "\xC7\xAE\x31\xA5\x0C\x97\x7E\xE2"
-			  "\x75\xDC\x40\x2B\xB2\x05\x89\x10"
-			  "\xFB\x42\xF6\x59\x20\x54\x3F\x86"
-			  "\x69\x9D\x64\xCF\x56\xDA\xAD\x34"
-			  "\xB8\x03\xEA\x7D\xE1\x48\xD3\x47",
-		.ilen	= 496,
-		.result	= "\x07\xC2\x08\x20\x72\x1F\x49\xEF"
-			  "\x19\xCD\x6F\x32\x53\x05\x22\x15"
-			  "\xA2\x85\x2B\xDB\x85\xD2\xD8\xB9"
-			  "\xDD\x0D\x1B\x45\xCB\x69\x11\xD4"
-			  "\xEA\xBE\xB2\x45\x5D\x0C\xAE\xBE"
-			  "\xA0\xC1\x27\xAC\x65\x9F\x53\x7E"
-			  "\xAF\xC2\x1B\xB5\xB8\x6D\x36\x0C"
-			  "\x25\xC0\xF8\x6D\x0B\x29\x01\xDA"
-			  "\x13\x78\xDC\x89\x12\x12\x43\xFA"
-			  "\xF6\x12\xEF\x8D\x87\x62\x78\x83"
-			  "\xE2\xBE\x41\x20\x4C\x6D\x35\x1B"
-			  "\xD1\x0C\x30\xCF\xE2\xDE\x2B\x03"
-			  "\xBF\x45\x73\xD4\xE5\x59\x95\xD1"
-			  "\xB3\x9B\x27\x62\x97\xBD\xDE\x7F"
-			  "\xA4\xD2\x39\x80\xAA\x50\x23\xF0"
-			  "\x74\x88\x3D\xA8\x6A\x18\x79\x3B"
-			  "\xC4\x96\x6C\x8D\x22\x40\x92\x6E"
-			  "\xD6\xAD\x2A\x1F\xDE\x63\xC0\xE7"
-			  "\x07\xF7\x2D\xF7\xB5\xF3\xF0\xCC"
-			  "\x01\x7C\x2A\x9B\xC2\x10\xCA\xAA"
-			  "\xFD\x2B\x3F\xC5\xF3\xF6\xFC\x9B"
-			  "\x45\xDB\x53\xE4\x5B\xF3\xC9\x7B"
-			  "\x8E\x52\xFF\xC8\x02\xB8\xAC\x9D"
-			  "\xA1\x00\x39\xDA\x3D\x2D\x0E\x01"
-			  "\x09\x7D\x8D\x5E\xBE\x53\xB9\xB0"
-			  "\x8E\xE7\xE2\x96\x6A\xB2\x78\xEA"
-			  "\xDE\x23\x8B\xA5\xFA\x5C\xE3\xDA"
-			  "\xBF\x8E\x31\x6A\x55\xD1\x6A\xB2"
-			  "\xB5\x46\x6F\xA5\xF0\xEE\xBA\x1F"
-			  "\x9F\x98\xB0\x66\x4F\xD0\x3F\xA9"
-			  "\xDF\x5F\x58\xC4\xF4\xFF\x75\x5C"
-			  "\x40\x3A\x09\x7E\x6E\x1C\x97\xD4"
-			  "\xCC\xE7\xE7\x71\xCF\x0B\x15\x08"
-			  "\x71\xFA\x07\x97\xCD\xE6\xCA\x1D"
-			  "\x14\x28\x0C\xCF\x99\x13\x7A\xF1"
-			  "\xEB\xFA\xFA\x92\x07\xDE\x1D\xA1"
-			  "\xD3\x36\x69\xFE\x51\x4D\x9F\x2E"
-			  "\x83\x37\x4F\x1F\x48\x30\xED\x04"
-			  "\x4D\xA4\xEF\x3A\xCA\x76\xF4\x1C"
-			  "\x41\x8F\x63\x37\x78\x2F\x86\xA6"
-			  "\xEF\x41\x7E\xD2\xAF\x88\xAB\x67"
-			  "\x52\x71\xC3\x8E\xF8\x26\x93\x72"
-			  "\xAA\xD6\x0E\xE7\x0B\x46\xB1\x3A"
-			  "\xB4\x08\xA9\xA8\xA0\xCF\x20\x0C"
-			  "\x52\xBC\x8B\x05\x56\xB2\xBC\x31"
-			  "\x9B\x74\xB9\x29\x29\x96\x9A\x50"
-			  "\xDC\x45\xDC\x1A\xEB\x0C\x64\xD4"
-			  "\xD3\x05\x7E\x59\x55\xC3\xF4\x90"
-			  "\xC2\xAB\xF8\x9B\x8A\xDA\xCE\xA1"
-			  "\xC3\xF4\xAD\x77\xDD\x44\xC8\xAC"
-			  "\xA3\xF1\xC9\xD2\x19\x5C\xB0\xCA"
-			  "\xA2\x34\xC1\xF7\x6C\xFD\xAC\x65"
-			  "\x32\xDC\x48\xC4\xF2\x00\x6B\x77"
-			  "\xF1\x7D\x76\xAC\xC0\x31\x63\x2A"
-			  "\xA5\x3A\x62\xC8\x91\xB1\x03\x65"
-			  "\xCB\x43\xD1\x06\xDF\xC3\x67\xBC"
-			  "\xDC\xE0\xCD\x35\xCE\x49\x65\xA0"
-			  "\x52\x7B\xA7\x0D\x07\xA9\x1B\xB0"
-			  "\x40\x77\x72\xC2\xEA\x0E\x3A\x78"
-			  "\x46\xB9\x91\xB6\xE7\x3D\x51\x42"
-			  "\xFD\x51\xB0\xC6\x2C\x63\x13\x78"
-			  "\x5C\xEE\xFC\xCF\xC4\x70\x00\x34",
-		.rlen	= 496,
+		.ctext	= "\x07\xC2\x08\x20\x72\x1F\x49\xEF"
+			  "\x19\xCD\x6F\x32\x53\x05\x22\x15"
+			  "\xA2\x85\x2B\xDB\x85\xD2\xD8\xB9"
+			  "\xDD\x0D\x1B\x45\xCB\x69\x11\xD4"
+			  "\xEA\xBE\xB2\x45\x5D\x0C\xAE\xBE"
+			  "\xA0\xC1\x27\xAC\x65\x9F\x53\x7E"
+			  "\xAF\xC2\x1B\xB5\xB8\x6D\x36\x0C"
+			  "\x25\xC0\xF8\x6D\x0B\x29\x01\xDA"
+			  "\x13\x78\xDC\x89\x12\x12\x43\xFA"
+			  "\xF6\x12\xEF\x8D\x87\x62\x78\x83"
+			  "\xE2\xBE\x41\x20\x4C\x6D\x35\x1B"
+			  "\xD1\x0C\x30\xCF\xE2\xDE\x2B\x03"
+			  "\xBF\x45\x73\xD4\xE5\x59\x95\xD1"
+			  "\xB3\x9B\x27\x62\x97\xBD\xDE\x7F"
+			  "\xA4\xD2\x39\x80\xAA\x50\x23\xF0"
+			  "\x74\x88\x3D\xA8\x6A\x18\x79\x3B"
+			  "\xC4\x96\x6C\x8D\x22\x40\x92\x6E"
+			  "\xD6\xAD\x2A\x1F\xDE\x63\xC0\xE7"
+			  "\x07\xF7\x2D\xF7\xB5\xF3\xF0\xCC"
+			  "\x01\x7C\x2A\x9B\xC2\x10\xCA\xAA"
+			  "\xFD\x2B\x3F\xC5\xF3\xF6\xFC\x9B"
+			  "\x45\xDB\x53\xE4\x5B\xF3\xC9\x7B"
+			  "\x8E\x52\xFF\xC8\x02\xB8\xAC\x9D"
+			  "\xA1\x00\x39\xDA\x3D\x2D\x0E\x01"
+			  "\x09\x7D\x8D\x5E\xBE\x53\xB9\xB0"
+			  "\x8E\xE7\xE2\x96\x6A\xB2\x78\xEA"
+			  "\xDE\x23\x8B\xA5\xFA\x5C\xE3\xDA"
+			  "\xBF\x8E\x31\x6A\x55\xD1\x6A\xB2"
+			  "\xB5\x46\x6F\xA5\xF0\xEE\xBA\x1F"
+			  "\x9F\x98\xB0\x66\x4F\xD0\x3F\xA9"
+			  "\xDF\x5F\x58\xC4\xF4\xFF\x75\x5C"
+			  "\x40\x3A\x09\x7E\x6E\x1C\x97\xD4"
+			  "\xCC\xE7\xE7\x71\xCF\x0B\x15\x08"
+			  "\x71\xFA\x07\x97\xCD\xE6\xCA\x1D"
+			  "\x14\x28\x0C\xCF\x99\x13\x7A\xF1"
+			  "\xEB\xFA\xFA\x92\x07\xDE\x1D\xA1"
+			  "\xD3\x36\x69\xFE\x51\x4D\x9F\x2E"
+			  "\x83\x37\x4F\x1F\x48\x30\xED\x04"
+			  "\x4D\xA4\xEF\x3A\xCA\x76\xF4\x1C"
+			  "\x41\x8F\x63\x37\x78\x2F\x86\xA6"
+			  "\xEF\x41\x7E\xD2\xAF\x88\xAB\x67"
+			  "\x52\x71\xC3\x8E\xF8\x26\x93\x72"
+			  "\xAA\xD6\x0E\xE7\x0B\x46\xB1\x3A"
+			  "\xB4\x08\xA9\xA8\xA0\xCF\x20\x0C"
+			  "\x52\xBC\x8B\x05\x56\xB2\xBC\x31"
+			  "\x9B\x74\xB9\x29\x29\x96\x9A\x50"
+			  "\xDC\x45\xDC\x1A\xEB\x0C\x64\xD4"
+			  "\xD3\x05\x7E\x59\x55\xC3\xF4\x90"
+			  "\xC2\xAB\xF8\x9B\x8A\xDA\xCE\xA1"
+			  "\xC3\xF4\xAD\x77\xDD\x44\xC8\xAC"
+			  "\xA3\xF1\xC9\xD2\x19\x5C\xB0\xCA"
+			  "\xA2\x34\xC1\xF7\x6C\xFD\xAC\x65"
+			  "\x32\xDC\x48\xC4\xF2\x00\x6B\x77"
+			  "\xF1\x7D\x76\xAC\xC0\x31\x63\x2A"
+			  "\xA5\x3A\x62\xC8\x91\xB1\x03\x65"
+			  "\xCB\x43\xD1\x06\xDF\xC3\x67\xBC"
+			  "\xDC\xE0\xCD\x35\xCE\x49\x65\xA0"
+			  "\x52\x7B\xA7\x0D\x07\xA9\x1B\xB0"
+			  "\x40\x77\x72\xC2\xEA\x0E\x3A\x78"
+			  "\x46\xB9\x91\xB6\xE7\x3D\x51\x42"
+			  "\xFD\x51\xB0\xC6\x2C\x63\x13\x78"
+			  "\x5C\xEE\xFC\xCF\xC4\x70\x00\x34",
+		.len	= 496,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 496 - 20, 4, 16 },
@@ -7187,7 +6473,7 @@ static const struct cipher_testvec des3_ede_ctr_enc_tv_template[] = {
 		.klen	= 24,
 		.iv	= "\xB2\xD7\x48\xED\x06\x44\xF9\x12"
 			  "\xB7\x28\x4D\x83\x24\x59\xF2\x17",
-		.input	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
+		.ptext	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
 			  "\x8B\x12\x86\x69\xF0\x5B\xCF\x56"
 			  "\x39\xAD\x34\x9F\x66\xEA\x7D\xC4"
 			  "\x48\xD3\xBA\x0D\xB1\x18\xE3\x4A"
@@ -7250,8 +6536,7 @@ static const struct cipher_testvec des3_ede_ctr_enc_tv_template[] = {
 			  "\x69\x9D\x64\xCF\x56\xDA\xAD\x34"
 			  "\xB8\x03\xEA\x7D\xE1\x48\xD3\x47"
 			  "\x2E\xB1\x18",
-		.ilen	= 499,
-		.result	= "\x23\xFF\x5C\x99\x75\xBB\x1F\xD4"
+		.ctext	= "\x23\xFF\x5C\x99\x75\xBB\x1F\xD4"
 			  "\xBC\x27\x9D\x36\x60\xA9\xC9\xF7"
 			  "\x94\x9D\x1B\xFF\x8E\x95\x57\x89"
 			  "\x8C\x2E\x33\x70\x43\x61\xE6\xD2"
@@ -7314,285 +6599,7 @@ static const struct cipher_testvec des3_ede_ctr_enc_tv_template[] = {
 			  "\x5C\x60\x1F\x6E\xA7\xE2\xDC\xE7"
 			  "\x32\x0F\x05\x2F\xF2\x4C\x95\x3B"
 			  "\xF2\x79\xD9",
-		.rlen	= 499,
-		.also_non_np = 1,
-		.np	= 2,
-		.tap	= { 499 - 16, 16 },
-	},
-};
-
-static const struct cipher_testvec des3_ede_ctr_dec_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00"
-			  "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE"
-			  "\xEB\xB4\x51\x72\xB4\x51\x72\x1F",
-		.klen	= 24,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x07\xC2\x08\x20\x72\x1F\x49\xEF"
-			  "\x19\xCD\x6F\x32\x53\x05\x22\x15"
-			  "\xA2\x85\x2B\xDB\x85\xD2\xD8\xB9"
-			  "\xDD\x0D\x1B\x45\xCB\x69\x11\xD4"
-			  "\xEA\xBE\xB2\x45\x5D\x0C\xAE\xBE"
-			  "\xA0\xC1\x27\xAC\x65\x9F\x53\x7E"
-			  "\xAF\xC2\x1B\xB5\xB8\x6D\x36\x0C"
-			  "\x25\xC0\xF8\x6D\x0B\x29\x01\xDA"
-			  "\x13\x78\xDC\x89\x12\x12\x43\xFA"
-			  "\xF6\x12\xEF\x8D\x87\x62\x78\x83"
-			  "\xE2\xBE\x41\x20\x4C\x6D\x35\x1B"
-			  "\xD1\x0C\x30\xCF\xE2\xDE\x2B\x03"
-			  "\xBF\x45\x73\xD4\xE5\x59\x95\xD1"
-			  "\xB3\x9B\x27\x62\x97\xBD\xDE\x7F"
-			  "\xA4\xD2\x39\x80\xAA\x50\x23\xF0"
-			  "\x74\x88\x3D\xA8\x6A\x18\x79\x3B"
-			  "\xC4\x96\x6C\x8D\x22\x40\x92\x6E"
-			  "\xD6\xAD\x2A\x1F\xDE\x63\xC0\xE7"
-			  "\x07\xF7\x2D\xF7\xB5\xF3\xF0\xCC"
-			  "\x01\x7C\x2A\x9B\xC2\x10\xCA\xAA"
-			  "\xFD\x2B\x3F\xC5\xF3\xF6\xFC\x9B"
-			  "\x45\xDB\x53\xE4\x5B\xF3\xC9\x7B"
-			  "\x8E\x52\xFF\xC8\x02\xB8\xAC\x9D"
-			  "\xA1\x00\x39\xDA\x3D\x2D\x0E\x01"
-			  "\x09\x7D\x8D\x5E\xBE\x53\xB9\xB0"
-			  "\x8E\xE7\xE2\x96\x6A\xB2\x78\xEA"
-			  "\xDE\x23\x8B\xA5\xFA\x5C\xE3\xDA"
-			  "\xBF\x8E\x31\x6A\x55\xD1\x6A\xB2"
-			  "\xB5\x46\x6F\xA5\xF0\xEE\xBA\x1F"
-			  "\x9F\x98\xB0\x66\x4F\xD0\x3F\xA9"
-			  "\xDF\x5F\x58\xC4\xF4\xFF\x75\x5C"
-			  "\x40\x3A\x09\x7E\x6E\x1C\x97\xD4"
-			  "\xCC\xE7\xE7\x71\xCF\x0B\x15\x08"
-			  "\x71\xFA\x07\x97\xCD\xE6\xCA\x1D"
-			  "\x14\x28\x0C\xCF\x99\x13\x7A\xF1"
-			  "\xEB\xFA\xFA\x92\x07\xDE\x1D\xA1"
-			  "\xD3\x36\x69\xFE\x51\x4D\x9F\x2E"
-			  "\x83\x37\x4F\x1F\x48\x30\xED\x04"
-			  "\x4D\xA4\xEF\x3A\xCA\x76\xF4\x1C"
-			  "\x41\x8F\x63\x37\x78\x2F\x86\xA6"
-			  "\xEF\x41\x7E\xD2\xAF\x88\xAB\x67"
-			  "\x52\x71\xC3\x8E\xF8\x26\x93\x72"
-			  "\xAA\xD6\x0E\xE7\x0B\x46\xB1\x3A"
-			  "\xB4\x08\xA9\xA8\xA0\xCF\x20\x0C"
-			  "\x52\xBC\x8B\x05\x56\xB2\xBC\x31"
-			  "\x9B\x74\xB9\x29\x29\x96\x9A\x50"
-			  "\xDC\x45\xDC\x1A\xEB\x0C\x64\xD4"
-			  "\xD3\x05\x7E\x59\x55\xC3\xF4\x90"
-			  "\xC2\xAB\xF8\x9B\x8A\xDA\xCE\xA1"
-			  "\xC3\xF4\xAD\x77\xDD\x44\xC8\xAC"
-			  "\xA3\xF1\xC9\xD2\x19\x5C\xB0\xCA"
-			  "\xA2\x34\xC1\xF7\x6C\xFD\xAC\x65"
-			  "\x32\xDC\x48\xC4\xF2\x00\x6B\x77"
-			  "\xF1\x7D\x76\xAC\xC0\x31\x63\x2A"
-			  "\xA5\x3A\x62\xC8\x91\xB1\x03\x65"
-			  "\xCB\x43\xD1\x06\xDF\xC3\x67\xBC"
-			  "\xDC\xE0\xCD\x35\xCE\x49\x65\xA0"
-			  "\x52\x7B\xA7\x0D\x07\xA9\x1B\xB0"
-			  "\x40\x77\x72\xC2\xEA\x0E\x3A\x78"
-			  "\x46\xB9\x91\xB6\xE7\x3D\x51\x42"
-			  "\xFD\x51\xB0\xC6\x2C\x63\x13\x78"
-			  "\x5C\xEE\xFC\xCF\xC4\x70\x00\x34",
-		.ilen	= 496,
-		.result	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
-			  "\x8B\x12\x86\x69\xF0\x5B\xCF\x56"
-			  "\x39\xAD\x34\x9F\x66\xEA\x7D\xC4"
-			  "\x48\xD3\xBA\x0D\xB1\x18\xE3\x4A"
-			  "\xFE\x41\x28\x5C\x27\x8E\x11\x85"
-			  "\x6C\xF7\x5E\xC2\x55\x3C\xA0\x0B"
-			  "\x92\x65\xE9\x70\xDB\x4F\xD6\xB9"
-			  "\x00\xB4\x1F\xE6\x49\xFD\x44\x2F"
-			  "\x53\x3A\x8D\x14\x98\x63\xCA\x5D"
-			  "\xC1\xA8\x33\xA7\x0E\x91\x78\xEC"
-			  "\x77\xDE\x42\xD5\xBC\x07\x8B\x12"
-			  "\xE5\x4C\xF0\x5B\x22\x56\x39\x80"
-			  "\x6B\x9F\x66\xC9\x50\xC4\xAF\x36"
-			  "\xBA\x0D\x94\x7F\xE3\x4A\xDD\x41"
-			  "\x28\xB3\x1A\x8E\x11\xF8\x43\xF7"
-			  "\x5E\x21\x55\x3C\x87\x6E\x92\x65"
-			  "\xCC\x57\xDB\xA2\x35\xB9\x00\xEB"
-			  "\x72\xE6\x49\xD0\x44\x2F\xB6\x19"
-			  "\x8D\x14\xFF\x46\xCA\x5D\x24\xA8"
-			  "\x33\x9A\x6D\x91\x78\xC3\x77\xDE"
-			  "\xA1\x08\xBC\x07\xEE\x71\xE5\x4C"
-			  "\xD7\x5B\x22\xB5\x1C\x80\x6B\xF2"
-			  "\x45\xC9\x50\x3B\xAF\x36\x99\x60"
-			  "\x94\x7F\xC6\x4A\xDD\xA4\x0F\xB3"
-			  "\x1A\xED\x74\xF8\x43\x2A\x5E\x21"
-			  "\x88\x13\x87\x6E\xF1\x58\xCC\x57"
-			  "\x3E\xA2\x35\x9C\x67\xEB\x72\xC5"
-			  "\x49\xD0\xBB\x02\xB6\x19\xE0\x4B"
-			  "\xFF\x46\x29\x5D\x24\x8F\x16\x9A"
-			  "\x6D\xF4\x5F\xC3\xAA\x3D\xA1\x08"
-			  "\x93\x7A\xEE\x71\xD8\x4C\xD7\xBE"
-			  "\x01\xB5\x1C\xE7\x4E\xF2\x45\x2C"
-			  "\x50\x3B\x82\x15\x99\x60\xCB\x52"
-			  "\xC6\xA9\x30\xA4\x0F\x96\x79\xED"
-			  "\x74\xDF\x43\x2A\xBD\x04\x88\x13"
-			  "\xFA\x4D\xF1\x58\x23\x57\x3E\x81"
-			  "\x68\x9C\x67\xCE\x51\xC5\xAC\x37"
-			  "\xBB\x02\x95\x7C\xE0\x4B\xD2\x46"
-			  "\x29\xB0\x1B\x8F\x16\xF9\x40\xF4"
-			  "\x5F\x26\xAA\x3D\x84\x6F\x93\x7A"
-			  "\xCD\x54\xD8\xA3\x0A\xBE\x01\xE8"
-			  "\x73\xE7\x4E\xD1\x45\x2C\xB7\x1E"
-			  "\x82\x15\xFC\x47\xCB\x52\x25\xA9"
-			  "\x30\x9B\x62\x96\x79\xC0\x74\xDF"
-			  "\xA6\x09\xBD\x04\xEF\x76\xFA\x4D"
-			  "\xD4\x58\x23\x8A\x1D\x81\x68\xF3"
-			  "\x5A\xCE\x51\x38\xAC\x37\x9E\x61"
-			  "\x95\x7C\xC7\x4B\xD2\xA5\x0C\xB0"
-			  "\x1B\xE2\x75\xF9\x40\x2B\x5F\x26"
-			  "\x89\x10\x84\x6F\xF6\x59\xCD\x54"
-			  "\x3F\xA3\x0A\x9D\x64\xE8\x73\xDA"
-			  "\x4E\xD1\xB8\x03\xB7\x1E\xE1\x48"
-			  "\xFC\x47\x2E\x52\x25\x8C\x17\x9B"
-			  "\x62\xF5\x5C\xC0\xAB\x32\xA6\x09"
-			  "\x90\x7B\xEF\x76\xD9\x4D\xD4\xBF"
-			  "\x06\x8A\x1D\xE4\x4F\xF3\x5A\x2D"
-			  "\x51\x38\x83\x6A\x9E\x61\xC8\x53"
-			  "\xC7\xAE\x31\xA5\x0C\x97\x7E\xE2"
-			  "\x75\xDC\x40\x2B\xB2\x05\x89\x10"
-			  "\xFB\x42\xF6\x59\x20\x54\x3F\x86"
-			  "\x69\x9D\x64\xCF\x56\xDA\xAD\x34"
-			  "\xB8\x03\xEA\x7D\xE1\x48\xD3\x47",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	}, { /* Generated with Crypto++ */
-		.key	= "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00"
-			  "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE"
-			  "\xEB\xB4\x51\x72\xB4\x51\x72\x1F",
-		.klen	= 24,
-		.iv	= "\xB2\xD7\x48\xED\x06\x44\xF9\x12"
-			  "\xB7\x28\x4D\x83\x24\x59\xF2\x17",
-		.input	= "\x23\xFF\x5C\x99\x75\xBB\x1F\xD4"
-			  "\xBC\x27\x9D\x36\x60\xA9\xC9\xF7"
-			  "\x94\x9D\x1B\xFF\x8E\x95\x57\x89"
-			  "\x8C\x2E\x33\x70\x43\x61\xE6\xD2"
-			  "\x82\x33\x63\xB6\xC4\x34\x5E\xF8"
-			  "\x96\x07\xA7\xD2\x3B\x8E\xC9\xAA"
-			  "\x7C\xA0\x55\x89\x2E\xE1\x85\x25"
-			  "\x14\x04\xDA\x6B\xE0\xEE\x56\xCF"
-			  "\x08\x2E\x69\xD4\x54\xDE\x22\x84"
-			  "\x69\xA6\xA7\xD3\x3A\x9A\xE8\x05"
-			  "\x63\xDB\xBF\x46\x3A\x26\x2E\x0F"
-			  "\x58\x5C\x46\xEA\x07\x40\xDA\xE1"
-			  "\x14\x1D\xCD\x4F\x06\xC0\xCA\x54"
-			  "\x1E\xC9\x45\x85\x67\x7C\xC2\xB5"
-			  "\x97\x5D\x61\x78\x2E\x46\xEC\x6A"
-			  "\x53\xF4\xD0\xAE\xFA\xB4\x86\x29"
-			  "\x9F\x17\x33\x24\xD8\xB9\xB2\x05"
-			  "\x93\x88\xEA\xF7\xA0\x70\x69\x49"
-			  "\x88\x6B\x73\x40\x41\x8D\xD9\xD9"
-			  "\x7E\x78\xE9\xBE\x6C\x14\x22\x7A"
-			  "\x66\xE1\xDA\xED\x10\xFF\x69\x1D"
-			  "\xB9\xAA\xF2\x56\x72\x1B\x23\xE2"
-			  "\x45\x54\x8B\xA3\x70\x23\xB4\x5E"
-			  "\x8E\x96\xC9\x05\x00\xB3\xB6\xC2"
-			  "\x2A\x02\x43\x7A\x62\xD5\xC8\xD2"
-			  "\xC2\xD0\xE4\x78\xA1\x7B\x3E\xE8"
-			  "\x9F\x7F\x7D\x40\x54\x30\x3B\xC0"
-			  "\xA5\x54\xFD\xCA\x25\xEC\x44\x3E"
-			  "\x1A\x54\x7F\x88\xD0\xE1\xFE\x71"
-			  "\xCE\x05\x49\x89\xBA\xD6\x72\xE7"
-			  "\xD6\x5D\x3F\xA2\xD9\xAB\xC5\x02"
-			  "\xD6\x43\x22\xAF\xA2\xE4\x80\x85"
-			  "\xD7\x87\xB9\xEA\x43\xDB\xC8\xEF"
-			  "\x5C\x82\x2E\x98\x0D\x30\x41\x6B"
-			  "\x08\x48\x8D\xF0\xF8\x60\xD7\x9D"
-			  "\xE9\xDE\x40\xAD\x0D\xAD\x0D\x58"
-			  "\x2A\x98\x35\xFE\xF7\xDD\x4B\x40"
-			  "\xDE\xB0\x05\xD9\x7B\x09\x4D\xBC"
-			  "\x42\xC0\xF1\x15\x0B\xFA\x26\x6B"
-			  "\xC6\x12\x13\x4F\xCB\x35\xBA\x35"
-			  "\xDD\x7A\x36\x9C\x12\x57\x55\x83"
-			  "\x78\x58\x09\xD0\xB0\xCF\x7C\x5C"
-			  "\x38\xCF\xBD\x79\x5B\x13\x4D\x97"
-			  "\xC1\x85\x6F\x97\xC9\xE8\xC2\xA4"
-			  "\x98\xE2\xBD\x77\x6B\x53\x39\x1A"
-			  "\x28\x10\xE7\xE0\xE7\xDE\x9D\x69"
-			  "\x78\x6F\x8E\xD2\xD9\x5D\xD2\x15"
-			  "\x9E\xB5\x4D\x8C\xC0\x78\x22\x2F"
-			  "\x17\x11\x2E\x99\xD7\xE3\xA4\x4F"
-			  "\x65\xA5\x6B\x03\x2C\x35\x6F\xDA"
-			  "\x8A\x19\x08\xE1\x08\x48\x59\x51"
-			  "\x53\x4B\xD1\xDF\xDA\x14\x50\x5F"
-			  "\xDF\xB5\x8C\xDF\xC6\xFD\x85\xFA"
-			  "\xD4\xF9\x64\x45\x65\x0D\x7D\xF4"
-			  "\xC8\xCD\x3F\x32\xAF\xDD\x30\xED"
-			  "\x7B\xAA\xAC\xF0\xDA\x7F\xDF\x75"
-			  "\x1C\xA4\xF1\xCB\x5E\x4F\x0B\xB4"
-			  "\x97\x73\x28\xDE\xCF\xAF\x82\xBD"
-			  "\xC4\xBA\xB4\x9C\x0D\x16\x77\x42"
-			  "\x42\x39\x7C\x53\xA4\xD4\xDD\x40"
-			  "\x5C\x60\x1F\x6E\xA7\xE2\xDC\xE7"
-			  "\x32\x0F\x05\x2F\xF2\x4C\x95\x3B"
-			  "\xF2\x79\xD9",
-		.ilen	= 499,
-		.result	= "\x05\xEC\x77\xFB\x42\xD5\x59\x20"
-			  "\x8B\x12\x86\x69\xF0\x5B\xCF\x56"
-			  "\x39\xAD\x34\x9F\x66\xEA\x7D\xC4"
-			  "\x48\xD3\xBA\x0D\xB1\x18\xE3\x4A"
-			  "\xFE\x41\x28\x5C\x27\x8E\x11\x85"
-			  "\x6C\xF7\x5E\xC2\x55\x3C\xA0\x0B"
-			  "\x92\x65\xE9\x70\xDB\x4F\xD6\xB9"
-			  "\x00\xB4\x1F\xE6\x49\xFD\x44\x2F"
-			  "\x53\x3A\x8D\x14\x98\x63\xCA\x5D"
-			  "\xC1\xA8\x33\xA7\x0E\x91\x78\xEC"
-			  "\x77\xDE\x42\xD5\xBC\x07\x8B\x12"
-			  "\xE5\x4C\xF0\x5B\x22\x56\x39\x80"
-			  "\x6B\x9F\x66\xC9\x50\xC4\xAF\x36"
-			  "\xBA\x0D\x94\x7F\xE3\x4A\xDD\x41"
-			  "\x28\xB3\x1A\x8E\x11\xF8\x43\xF7"
-			  "\x5E\x21\x55\x3C\x87\x6E\x92\x65"
-			  "\xCC\x57\xDB\xA2\x35\xB9\x00\xEB"
-			  "\x72\xE6\x49\xD0\x44\x2F\xB6\x19"
-			  "\x8D\x14\xFF\x46\xCA\x5D\x24\xA8"
-			  "\x33\x9A\x6D\x91\x78\xC3\x77\xDE"
-			  "\xA1\x08\xBC\x07\xEE\x71\xE5\x4C"
-			  "\xD7\x5B\x22\xB5\x1C\x80\x6B\xF2"
-			  "\x45\xC9\x50\x3B\xAF\x36\x99\x60"
-			  "\x94\x7F\xC6\x4A\xDD\xA4\x0F\xB3"
-			  "\x1A\xED\x74\xF8\x43\x2A\x5E\x21"
-			  "\x88\x13\x87\x6E\xF1\x58\xCC\x57"
-			  "\x3E\xA2\x35\x9C\x67\xEB\x72\xC5"
-			  "\x49\xD0\xBB\x02\xB6\x19\xE0\x4B"
-			  "\xFF\x46\x29\x5D\x24\x8F\x16\x9A"
-			  "\x6D\xF4\x5F\xC3\xAA\x3D\xA1\x08"
-			  "\x93\x7A\xEE\x71\xD8\x4C\xD7\xBE"
-			  "\x01\xB5\x1C\xE7\x4E\xF2\x45\x2C"
-			  "\x50\x3B\x82\x15\x99\x60\xCB\x52"
-			  "\xC6\xA9\x30\xA4\x0F\x96\x79\xED"
-			  "\x74\xDF\x43\x2A\xBD\x04\x88\x13"
-			  "\xFA\x4D\xF1\x58\x23\x57\x3E\x81"
-			  "\x68\x9C\x67\xCE\x51\xC5\xAC\x37"
-			  "\xBB\x02\x95\x7C\xE0\x4B\xD2\x46"
-			  "\x29\xB0\x1B\x8F\x16\xF9\x40\xF4"
-			  "\x5F\x26\xAA\x3D\x84\x6F\x93\x7A"
-			  "\xCD\x54\xD8\xA3\x0A\xBE\x01\xE8"
-			  "\x73\xE7\x4E\xD1\x45\x2C\xB7\x1E"
-			  "\x82\x15\xFC\x47\xCB\x52\x25\xA9"
-			  "\x30\x9B\x62\x96\x79\xC0\x74\xDF"
-			  "\xA6\x09\xBD\x04\xEF\x76\xFA\x4D"
-			  "\xD4\x58\x23\x8A\x1D\x81\x68\xF3"
-			  "\x5A\xCE\x51\x38\xAC\x37\x9E\x61"
-			  "\x95\x7C\xC7\x4B\xD2\xA5\x0C\xB0"
-			  "\x1B\xE2\x75\xF9\x40\x2B\x5F\x26"
-			  "\x89\x10\x84\x6F\xF6\x59\xCD\x54"
-			  "\x3F\xA3\x0A\x9D\x64\xE8\x73\xDA"
-			  "\x4E\xD1\xB8\x03\xB7\x1E\xE1\x48"
-			  "\xFC\x47\x2E\x52\x25\x8C\x17\x9B"
-			  "\x62\xF5\x5C\xC0\xAB\x32\xA6\x09"
-			  "\x90\x7B\xEF\x76\xD9\x4D\xD4\xBF"
-			  "\x06\x8A\x1D\xE4\x4F\xF3\x5A\x2D"
-			  "\x51\x38\x83\x6A\x9E\x61\xC8\x53"
-			  "\xC7\xAE\x31\xA5\x0C\x97\x7E\xE2"
-			  "\x75\xDC\x40\x2B\xB2\x05\x89\x10"
-			  "\xFB\x42\xF6\x59\x20\x54\x3F\x86"
-			  "\x69\x9D\x64\xCF\x56\xDA\xAD\x34"
-			  "\xB8\x03\xEA\x7D\xE1\x48\xD3\x47"
-			  "\x2E\xB1\x18",
-		.rlen	= 499,
+		.len	= 499,
 		.also_non_np = 1,
 		.np	= 2,
 		.tap	= { 499 - 16, 16 },
@@ -7602,45 +6609,40 @@ static const struct cipher_testvec des3_ede_ctr_dec_tv_template[] = {
 /*
  * Blowfish test vectors.
  */
-static const struct cipher_testvec bf_enc_tv_template[] = {
+static const struct cipher_testvec bf_tv_template[] = {
 	{ /* DES test vectors from OpenSSL */
 		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00",
 		.klen	= 8,
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 8,
-		.result	= "\x4e\xf9\x97\x45\x61\x98\xdd\x78",
-		.rlen	= 8,
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\x4e\xf9\x97\x45\x61\x98\xdd\x78",
+		.len	= 8,
 	}, {
 		.key	= "\x1f\x1f\x1f\x1f\x0e\x0e\x0e\x0e",
 		.klen	= 8,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.ilen	= 8,
-		.result	= "\xa7\x90\x79\x51\x08\xea\x3c\xae",
-		.rlen	= 8,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.ctext	= "\xa7\x90\x79\x51\x08\xea\x3c\xae",
+		.len	= 8,
 	}, {
 		.key	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
 		.klen	= 8,
-		.input	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.ilen	= 8,
-		.result	= "\xe8\x7a\x24\x4e\x2c\xc8\x5e\x82",
-		.rlen	= 8,
+		.ptext	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.ctext	= "\xe8\x7a\x24\x4e\x2c\xc8\x5e\x82",
+		.len	= 8,
 	}, { /* Vary the keylength... */
 		.key	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87"
 			  "\x78\x69\x5a\x4b\x3c\x2d\x1e\x0f",
 		.klen	= 16,
-		.input	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.ilen	= 8,
-		.result	= "\x93\x14\x28\x87\xee\x3b\xe1\x5c",
-		.rlen	= 8,
+		.ptext	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.ctext	= "\x93\x14\x28\x87\xee\x3b\xe1\x5c",
+		.len	= 8,
 	}, {
 		.key	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87"
 			  "\x78\x69\x5a\x4b\x3c\x2d\x1e\x0f"
 			  "\x00\x11\x22\x33\x44",
 		.klen	= 21,
-		.input	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.ilen	= 8,
-		.result	= "\xe6\xf5\x1e\xd7\x9b\x9d\xb2\x1f",
-		.rlen	= 8,
+		.ptext	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.ctext	= "\xe6\xf5\x1e\xd7\x9b\x9d\xb2\x1f",
+		.len	= 8,
 	}, { /* Generated with bf488 */
 		.key	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87"
 			  "\x78\x69\x5a\x4b\x3c\x2d\x1e\x0f"
@@ -7650,17 +6652,16 @@ static const struct cipher_testvec bf_enc_tv_template[] = {
 			  "\x1f\x1f\x1f\x1f\x0e\x0e\x0e\x0e"
 			  "\xff\xff\xff\xff\xff\xff\xff\xff",
 		.klen	= 56,
-		.input	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.ilen	= 8,
-		.result	= "\xc0\x45\x04\x01\x2e\x4e\x1f\x53",
-		.rlen	= 8,
+		.ptext	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.ctext	= "\xc0\x45\x04\x01\x2e\x4e\x1f\x53",
+		.len	= 8,
 	}, { /* Generated with Crypto++ */
 		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
 			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
 			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
 			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -7723,8 +6724,7 @@ static const struct cipher_testvec bf_enc_tv_template[] = {
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
 			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
 			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06",
-		.ilen	= 504,
-		.result	= "\x96\x87\x3D\x0C\x7B\xFB\xBD\x1F"
+		.ctext	= "\x96\x87\x3D\x0C\x7B\xFB\xBD\x1F"
 			  "\xE3\xC1\x99\x6D\x39\xD4\xC2\x7D"
 			  "\xD7\x87\xA1\xF2\xDF\x51\x71\x26"
 			  "\xC2\xF4\x6D\xFF\xF6\xCD\x6B\x40"
@@ -7787,136 +6787,36 @@ static const struct cipher_testvec bf_enc_tv_template[] = {
 			  "\x54\x59\x09\xA5\x3D\xC5\x84\x68"
 			  "\x56\xEB\x36\x77\x3D\xAA\xB8\xF5"
 			  "\xC9\x1A\xFB\x5D\xDE\xBB\x43\xF4",
-		.rlen	= 504,
+		.len	= 504,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 504 - 10, 2, 8 },
 	},
 };
 
-static const struct cipher_testvec bf_dec_tv_template[] = {
-	{ /* DES test vectors from OpenSSL */
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 8,
-		.input	= "\x4e\xf9\x97\x45\x61\x98\xdd\x78",
-		.ilen	= 8,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen	= 8,
-	}, {
-		.key	= "\x1f\x1f\x1f\x1f\x0e\x0e\x0e\x0e",
-		.klen	= 8,
-		.input	= "\xa7\x90\x79\x51\x08\xea\x3c\xae",
-		.ilen	= 8,
-		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.rlen	= 8,
-	}, {
-		.key	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
-		.klen	= 8,
-		.input	= "\xe8\x7a\x24\x4e\x2c\xc8\x5e\x82",
-		.ilen	= 8,
-		.result	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.rlen	= 8,
-	}, { /* Vary the keylength... */
-		.key	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87"
-			  "\x78\x69\x5a\x4b\x3c\x2d\x1e\x0f",
+static const struct cipher_testvec bf_cbc_tv_template[] = {
+	{ /* From OpenSSL */
+		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
 		.klen	= 16,
-		.input	= "\x93\x14\x28\x87\xee\x3b\xe1\x5c",
-		.ilen	= 8,
-		.result	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.rlen	= 8,
-	}, {
-		.key	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87"
-			  "\x78\x69\x5a\x4b\x3c\x2d\x1e\x0f"
-			  "\x00\x11\x22\x33\x44",
-		.klen	= 21,
-		.input	= "\xe6\xf5\x1e\xd7\x9b\x9d\xb2\x1f",
-		.ilen	= 8,
-		.result	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.rlen	= 8,
-	}, { /* Generated with bf488, using OpenSSL, Libgcrypt and Nettle */
-		.key	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87"
-			  "\x78\x69\x5a\x4b\x3c\x2d\x1e\x0f"
-			  "\x00\x11\x22\x33\x44\x55\x66\x77"
-			  "\x04\x68\x91\x04\xc2\xfd\x3b\x2f"
-			  "\x58\x40\x23\x64\x1a\xba\x61\x76"
-			  "\x1f\x1f\x1f\x1f\x0e\x0e\x0e\x0e"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff",
-		.klen	= 56,
-		.input	= "\xc0\x45\x04\x01\x2e\x4e\x1f\x53",
-		.ilen	= 8,
-		.result	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.rlen	= 8,
+		.iv	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.ptext	= "\x37\x36\x35\x34\x33\x32\x31\x20"
+			  "\x4e\x6f\x77\x20\x69\x73\x20\x74"
+			  "\x68\x65\x20\x74\x69\x6d\x65\x20"
+			  "\x66\x6f\x72\x20\x00\x00\x00\x00",
+		.ctext	= "\x6b\x77\xb4\xd6\x30\x06\xde\xe6"
+			  "\x05\xb1\x56\xe2\x74\x03\x97\x93"
+			  "\x58\xde\xb9\xe7\x15\x46\x16\xd9"
+			  "\x59\xf1\x65\x2b\xd5\xff\x92\xcc",
+		.len	= 32,
 	}, { /* Generated with Crypto++ */
 		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
 			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
 			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
 			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
-		.input	= "\x96\x87\x3D\x0C\x7B\xFB\xBD\x1F"
-			  "\xE3\xC1\x99\x6D\x39\xD4\xC2\x7D"
-			  "\xD7\x87\xA1\xF2\xDF\x51\x71\x26"
-			  "\xC2\xF4\x6D\xFF\xF6\xCD\x6B\x40"
-			  "\xE1\xB3\xBF\xD4\x38\x2B\xC8\x3B"
-			  "\xD3\xB2\xD4\x61\xC7\x9F\x06\xE9"
-			  "\xCD\xF3\x88\x39\x39\x7A\xDF\x19"
-			  "\xE8\x03\x2A\x0B\x9E\xA0\x2B\x86"
-			  "\x31\xF8\x9D\xB1\xEE\x78\x9D\xB5"
-			  "\xCD\x8B\x7C\x2E\xF5\xA2\x2D\x5D"
-			  "\x6E\x66\xAF\x38\x6C\xD3\x13\xED"
-			  "\x14\xEA\x5D\xD0\x17\x77\x0F\x4A"
-			  "\x50\xF2\xD0\x0F\xC8\xF7\x1E\x7B"
-			  "\x9D\x5B\x54\x65\x4F\x16\x8A\x97"
-			  "\xF3\xF6\xD4\xAA\x87\x36\x77\x72"
-			  "\x99\x4A\xB5\x5E\x88\xC3\xCD\x7D"
-			  "\x1D\x97\xF9\x11\xBD\xE0\x1F\x1F"
-			  "\x96\x3E\x4B\x22\xF4\xC0\xE6\xB8"
-			  "\x47\x82\x98\x23\x33\x36\xBC\x1B"
-			  "\x36\xE7\xF6\xCF\x97\x37\x16\xC0"
-			  "\x87\x31\x8B\xB0\xDB\x19\x42\xA5"
-			  "\x1F\x90\x7E\x66\x34\xDD\x5E\xE9"
-			  "\x4F\xB2\x2B\x9A\xDE\xB3\x5D\x71"
-			  "\x4D\x68\xF0\xDC\xA6\xEA\xE3\x9B"
-			  "\x60\x00\x55\x57\x06\x8B\xD5\xB3"
-			  "\x86\x30\x78\xDA\x33\x9A\x9D\xCC"
-			  "\xBA\x0B\x81\x06\x77\x43\xC7\xC9"
-			  "\xDB\x37\x60\x11\x45\x59\x6D\x2D"
-			  "\x90\x3D\x65\x3E\xD0\x13\xC6\x3C"
-			  "\x0E\x78\x7D\x9A\x00\xD6\x2F\x0B"
-			  "\x3B\x53\x19\x1E\xA8\x9B\x11\xD9"
-			  "\x98\xE4\x7F\xC3\x6E\x51\x24\x70"
-			  "\x9F\x04\x9C\xC2\x9E\x44\x84\xE3"
-			  "\xE0\x8A\x44\xA2\x5C\x94\x74\x34"
-			  "\x37\x52\x7C\x03\xE8\x8E\x97\xE1"
-			  "\x5B\x5C\x0E\xB0\x70\xFE\x54\x3F"
-			  "\xD8\x65\xA9\xC5\xCD\xEC\xF4\x45"
-			  "\x55\xC5\xA7\xA3\x19\x80\x28\x51"
-			  "\xBE\x64\x4A\xC1\xD4\xE1\xBE\xEB"
-			  "\x73\x4C\xB6\xF9\x5F\x6D\x82\xBC"
-			  "\x3E\x42\x14\x49\x88\x51\xBF\x68"
-			  "\x45\x75\x27\x1B\x0A\x72\xED\xAF"
-			  "\xDA\xC4\x4D\x67\x0D\xEE\x75\xE3"
-			  "\x34\xDD\x91\x19\x42\x3A\xCB\xDA"
-			  "\x38\xFA\x3C\x93\x62\xF2\xE3\x81"
-			  "\xB3\xE4\xBB\xF6\x0D\x0B\x1D\x09"
-			  "\x9C\x52\x0D\x50\x63\xA4\xB2\xD2"
-			  "\x82\xA0\x23\x3F\x1F\xB6\xED\x6E"
-			  "\xC2\x9C\x1C\xD0\x9A\x40\xB6\xFC"
-			  "\x36\x56\x6E\x85\x73\xD7\x52\xBA"
-			  "\x35\x5E\x32\x89\x5D\x42\xF5\x36"
-			  "\x52\x8D\x46\x7D\xC8\x71\xAD\x33"
-			  "\xE1\xAF\x6A\xA8\xEC\xBA\x1C\xDC"
-			  "\xFE\x88\xE6\x16\xE4\xC8\x13\x00"
-			  "\x3C\xDA\x59\x32\x38\x19\xD5\xEB"
-			  "\xB6\x7F\x78\x45\x1B\x8E\x07\x8C"
-			  "\x66\x52\x75\xFF\xAF\xCE\x2D\x2B"
-			  "\x22\x29\xCA\xB3\x5F\x7F\xE3\x29"
-			  "\xB2\xB8\x9D\xEB\x16\xC8\xC5\x1D"
-			  "\xC9\x0D\x59\x82\x27\x57\x9D\x42"
-			  "\x54\x59\x09\xA5\x3D\xC5\x84\x68"
-			  "\x56\xEB\x36\x77\x3D\xAA\xB8\xF5"
-			  "\xC9\x1A\xFB\x5D\xDE\xBB\x43\xF4",
-		.ilen	= 504,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -7979,194 +6879,7 @@ static const struct cipher_testvec bf_dec_tv_template[] = {
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
 			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
 			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06",
-		.rlen	= 504,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 504 - 10, 2, 8 },
-	},
-};
-
-static const struct cipher_testvec bf_cbc_enc_tv_template[] = {
-	{ /* From OpenSSL */
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
-		.klen	= 16,
-		.iv	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.input	= "\x37\x36\x35\x34\x33\x32\x31\x20"
-			  "\x4e\x6f\x77\x20\x69\x73\x20\x74"
-			  "\x68\x65\x20\x74\x69\x6d\x65\x20"
-			  "\x66\x6f\x72\x20\x00\x00\x00\x00",
-		.ilen	= 32,
-		.result	= "\x6b\x77\xb4\xd6\x30\x06\xde\xe6"
-			  "\x05\xb1\x56\xe2\x74\x03\x97\x93"
-			  "\x58\xde\xb9\xe7\x15\x46\x16\xd9"
-			  "\x59\xf1\x65\x2b\xd5\xff\x92\xcc",
-		.rlen	= 32,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06",
-		.ilen	= 504,
-		.result	= "\xB4\xFE\xA5\xBB\x3D\x2C\x27\x06"
-			  "\x06\x2B\x3A\x92\xB2\xF5\x5E\x62"
-			  "\x84\xCD\xF7\x66\x7E\x41\x6C\x8E"
-			  "\x1B\xD9\x02\xB6\x48\xB0\x87\x25"
-			  "\x01\x9C\x93\x63\x51\x60\x82\xD2"
-			  "\x4D\xE5\xC2\xB7\xAE\x60\xD8\xAD"
-			  "\x9F\xAB\x6C\xFA\x20\x05\xDA\x6F"
-			  "\x1F\xD1\xD8\x36\x0F\xB5\x16\x69"
-			  "\x3C\xAF\xB3\x30\x18\x33\xE6\xB5"
-			  "\x43\x29\x9D\x94\xF4\x2F\x0A\x65"
-			  "\x40\xB2\xB2\xB2\x42\x89\xEE\x8A"
-			  "\x60\xD3\x52\xA8\xED\x91\xDF\xE1"
-			  "\x91\x73\x7C\x28\xA1\x14\xC3\x4C"
-			  "\x82\x72\x4B\x7D\x7D\x32\xD5\x19"
-			  "\xE8\xB8\x6B\x30\x21\x09\x0E\x27"
-			  "\x10\x9D\x2D\x3A\x6A\x4B\x7B\xE6"
-			  "\x8D\x4E\x02\x32\xFF\x7F\x8E\x13"
-			  "\xB0\x96\xF4\xC2\xA1\x60\x8A\x69"
-			  "\xEF\x0F\x86\xD0\x25\x13\x1A\x7C"
-			  "\x6E\xF0\x41\xA3\xFB\xB3\xAB\x40"
-			  "\x7D\x19\xA0\x11\x4F\x3E\x1D\x43"
-			  "\x65\xFE\x15\x40\xD0\x62\x41\x02"
-			  "\xEA\x0C\x7A\xC3\x84\xEE\xB0\xBE"
-			  "\xBE\xC8\x57\x51\xCD\x4F\xAD\x5C"
-			  "\xCC\x79\xBA\x0D\x85\x3A\xED\x6B"
-			  "\xAC\x6B\xA3\x4D\xBC\xE8\x02\x6A"
-			  "\xC2\x6D\xBD\x5E\x89\x95\x86\x43"
-			  "\x2C\x17\x4B\xC6\x40\xA2\xBD\x24"
-			  "\x04\xF0\x86\x08\x78\x18\x42\xE0"
-			  "\x39\x1B\x22\x9E\x89\x4C\x04\x6B"
-			  "\x65\xC5\xB6\x0E\xF6\x63\xFC\xD7"
-			  "\xAE\x9E\x87\x13\xCC\xD3\x1A\xEC"
-			  "\xF0\x51\xCC\x93\x68\xFC\xE9\x19"
-			  "\x7C\x4E\x9B\xCC\x17\xAD\xD2\xFC"
-			  "\x97\x18\x92\xFF\x15\x11\xCE\xED"
-			  "\x04\x41\x05\xA3\x92\xFF\x3B\xE6"
-			  "\xB6\x8C\x90\xC6\xCD\x15\xA0\x04"
-			  "\x25\x8B\x5D\x5B\x5F\xDB\xAE\x68"
-			  "\xEF\xB3\x61\x18\xDB\x83\x9B\x39"
-			  "\xCA\x82\xD1\x88\xF0\xA2\x5C\x02"
-			  "\x87\xBD\x8D\x8F\xBB\x62\xF0\x35"
-			  "\x75\x6F\x06\x81\x0A\x97\x4D\xF0"
-			  "\x43\x12\x73\x77\xDB\x91\x83\x5B"
-			  "\xE7\x3A\xA6\x07\x7B\xBF\x2C\x50"
-			  "\x94\xDE\x7B\x65\xDA\x1C\xF1\x9F"
-			  "\x7E\x12\x40\xB2\x3E\x19\x23\xF1"
-			  "\x7C\x1B\x5F\xA8\xF3\xAC\x63\x87"
-			  "\xEB\x3E\x0C\xBE\xA3\x63\x97\x88"
-			  "\x8D\x27\xC6\x2A\xF8\xF2\x67\x9A"
-			  "\x0D\x14\x16\x2B\x6F\xCB\xD4\x76"
-			  "\x14\x48\x2E\xDE\x2A\x44\x5E\x45"
-			  "\xF1\x97\x82\xEF\xB7\xAE\xED\x3A"
-			  "\xED\x73\xD3\x79\xF7\x38\x1D\xD0"
-			  "\xC5\xF8\x69\x83\x28\x84\x87\x56"
-			  "\x3F\xAE\x81\x04\x79\x1F\xD1\x09"
-			  "\xC5\xE5\x05\x0D\x64\x16\xCE\x42"
-			  "\xC5\xF8\xDB\x57\x89\x33\x22\xFC"
-			  "\xB4\xD7\x94\xB9\xF3\xCC\x02\x90"
-			  "\x02\xBA\x55\x1E\x24\x3E\x02\x1D"
-			  "\xC6\xCD\x8F\xD9\xBD\xED\xB0\x51"
-			  "\xCD\xE9\xD5\x0C\xFE\x12\x39\xA9"
-			  "\x93\x9B\xEE\xB5\x97\x41\xD2\xA0"
-			  "\xB4\x98\xD8\x6B\x74\xE7\x65\xF4",
-		.rlen	= 504,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 504 - 10, 2, 8 },
-	},
-};
-
-static const struct cipher_testvec bf_cbc_dec_tv_template[] = {
-	{ /* From OpenSSL */
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
-		.klen	= 16,
-		.iv	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.input	= "\x6b\x77\xb4\xd6\x30\x06\xde\xe6"
-			  "\x05\xb1\x56\xe2\x74\x03\x97\x93"
-			  "\x58\xde\xb9\xe7\x15\x46\x16\xd9"
-			  "\x59\xf1\x65\x2b\xd5\xff\x92\xcc",
-		.ilen	= 32,
-		.result	= "\x37\x36\x35\x34\x33\x32\x31\x20"
-			  "\x4e\x6f\x77\x20\x69\x73\x20\x74"
-			  "\x68\x65\x20\x74\x69\x6d\x65\x20"
-			  "\x66\x6f\x72\x20\x00\x00\x00\x00",
-		.rlen	= 32,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\xB4\xFE\xA5\xBB\x3D\x2C\x27\x06"
+		.ctext	= "\xB4\xFE\xA5\xBB\x3D\x2C\x27\x06"
 			  "\x06\x2B\x3A\x92\xB2\xF5\x5E\x62"
 			  "\x84\xCD\xF7\x66\x7E\x41\x6C\x8E"
 			  "\x1B\xD9\x02\xB6\x48\xB0\x87\x25"
@@ -8229,78 +6942,14 @@ static const struct cipher_testvec bf_cbc_dec_tv_template[] = {
 			  "\xCD\xE9\xD5\x0C\xFE\x12\x39\xA9"
 			  "\x93\x9B\xEE\xB5\x97\x41\xD2\xA0"
 			  "\xB4\x98\xD8\x6B\x74\xE7\x65\xF4",
-		.ilen	= 504,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06",
-		.rlen	= 504,
+		.len	= 504,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 504 - 10, 2, 8 },
 	},
 };
 
-static const struct cipher_testvec bf_ctr_enc_tv_template[] = {
+static const struct cipher_testvec bf_ctr_tv_template[] = {
 	{ /* Generated with Crypto++ */
 		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
 			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
@@ -8308,7 +6957,7 @@ static const struct cipher_testvec bf_ctr_enc_tv_template[] = {
 			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
 		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -8371,8 +7020,7 @@ static const struct cipher_testvec bf_ctr_enc_tv_template[] = {
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
 			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
 			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06",
-		.ilen	= 504,
-		.result	= "\xC7\xA3\xDF\xB9\x05\xF4\x9E\x8D"
+		.ctext	= "\xC7\xA3\xDF\xB9\x05\xF4\x9E\x8D"
 			  "\x9E\xDF\x38\x18\x83\x07\xEF\xC1"
 			  "\x93\x3C\xAA\xAA\xFE\x06\x42\xCC"
 			  "\x0D\x70\x86\x5A\x44\xAD\x85\x17"
@@ -8435,7 +7083,7 @@ static const struct cipher_testvec bf_ctr_enc_tv_template[] = {
 			  "\x64\xBB\x15\xB8\x56\xCF\xEE\xE5"
 			  "\x32\x44\x96\x1C\xD8\xEB\x95\xD2"
 			  "\xF3\x71\xEF\xEB\x4E\xBB\x4D\x29",
-		.rlen	= 504,
+		.len	= 504,
 	}, { /* Generated with Crypto++ */
 		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
 			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
@@ -8443,7 +7091,7 @@ static const struct cipher_testvec bf_ctr_enc_tv_template[] = {
 			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
 		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -8506,8 +7154,7 @@ static const struct cipher_testvec bf_ctr_enc_tv_template[] = {
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
 			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
 			  "\x2B\xC2\x59\xF0\x64\xFB\x92",
-		.ilen	= 503,
-		.result	= "\xC7\xA3\xDF\xB9\x05\xF4\x9E\x8D"
+		.ctext	= "\xC7\xA3\xDF\xB9\x05\xF4\x9E\x8D"
 			  "\x9E\xDF\x38\x18\x83\x07\xEF\xC1"
 			  "\x93\x3C\xAA\xAA\xFE\x06\x42\xCC"
 			  "\x0D\x70\x86\x5A\x44\xAD\x85\x17"
@@ -8570,7 +7217,7 @@ static const struct cipher_testvec bf_ctr_enc_tv_template[] = {
 			  "\x64\xBB\x15\xB8\x56\xCF\xEE\xE5"
 			  "\x32\x44\x96\x1C\xD8\xEB\x95\xD2"
 			  "\xF3\x71\xEF\xEB\x4E\xBB\x4D",
-		.rlen	= 503,
+		.len	= 503,
 		.also_non_np = 1,
 		.np	= 2,
 		.tap	= { 503 - 8, 8 },
@@ -8581,7 +7228,7 @@ static const struct cipher_testvec bf_ctr_enc_tv_template[] = {
 			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
 		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -8644,8 +7291,7 @@ static const struct cipher_testvec bf_ctr_enc_tv_template[] = {
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
 			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
 			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06",
-		.ilen	= 504,
-		.result	= "\x5F\x58\x6E\x60\x51\x6E\xDC\x3D"
+		.ctext	= "\x5F\x58\x6E\x60\x51\x6E\xDC\x3D"
 			  "\xD1\xBB\xF7\xB7\xFD\x04\x44\x82"
 			  "\xDC\x9F\x4B\x02\xF1\xD2\x5A\x6F"
 			  "\x25\xF9\x27\x21\xF2\xD2\x9A\x01"
@@ -8708,83 +7354,47 @@ static const struct cipher_testvec bf_ctr_enc_tv_template[] = {
 			  "\xB5\x77\xBA\x1C\xEE\x71\xFA\xB0"
 			  "\x16\x4C\x18\x6B\xF2\x69\xA0\x07"
 			  "\xEF\xBE\xEC\x69\xAC\xA8\x63\x9E",
-		.rlen	= 504,
+		.len	= 504,
 	},
 };
 
-static const struct cipher_testvec bf_ctr_dec_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\xC7\xA3\xDF\xB9\x05\xF4\x9E\x8D"
-			  "\x9E\xDF\x38\x18\x83\x07\xEF\xC1"
-			  "\x93\x3C\xAA\xAA\xFE\x06\x42\xCC"
-			  "\x0D\x70\x86\x5A\x44\xAD\x85\x17"
-			  "\xE4\x1F\x5E\xA5\x89\xAC\x32\xBC"
-			  "\x3D\xA7\xE9\x0A\x5C\x70\x4D\xDE"
-			  "\x99\x38\x07\xCA\x1D\x21\xC1\x11"
-			  "\x97\xEB\x98\x75\xC4\x73\x45\x83"
-			  "\x46\x1C\x9C\x91\x87\xC1\xA0\x56"
-			  "\x98\xA1\x8B\xDB\x22\x76\xBD\x62"
-			  "\xA4\xBC\xE8\x86\xDA\xD2\x51\x13"
-			  "\x13\xD2\x96\x68\x69\x10\x67\x0C"
-			  "\xD0\x17\x25\x7C\xB2\xAE\x4F\x93"
-			  "\xA6\x82\x20\xCF\x0F\xA6\x47\x79"
-			  "\x88\x09\x40\x59\xBD\x12\x64\xB5"
-			  "\x19\x38\x0D\xFF\x86\xD9\x42\x20"
-			  "\x81\x0D\x96\x99\xAF\x22\x1F\x94"
-			  "\x5C\x6E\xEC\xEA\xA3\x39\xCB\x09"
-			  "\x43\x19\x7F\xD0\xBB\x10\xC2\x49"
-			  "\xF7\xE9\xF2\xEE\xBF\xF7\xF8\xB3"
-			  "\x0E\x1A\xF1\x8D\x70\x82\x0C\x04"
-			  "\xFD\x29\x1A\xAC\xC0\x92\x48\x34"
-			  "\x6A\xE3\x1D\x4F\xFC\x1C\x72\x6A"
-			  "\x57\xCB\xAD\xD0\x98\xAB\xB1\x01"
-			  "\x03\x6A\x45\xDD\x07\x71\x5F\x5B"
-			  "\xB5\x4A\xE4\xE5\xB9\xB9\xBC\xAC"
-			  "\x44\xF7\x41\xA4\x5F\x2E\xE9\x28"
-			  "\xE3\x05\xD2\x94\x78\x4C\x33\x1B"
-			  "\xBD\xC1\x6E\x51\xD9\xAD\xD9\x86"
-			  "\x15\x4A\x78\xAE\x7B\xAD\x3B\xBC"
-			  "\x2F\xE0\x0E\xC5\x7B\x54\x97\x5F"
-			  "\x60\x51\x14\x65\xF9\x91\xE9\xDA"
-			  "\x9A\xBC\xFC\x19\x29\x67\xAA\x63"
-			  "\x5E\xF2\x48\x88\xEB\x79\xE1\xE4"
-			  "\xF7\xF6\x4C\xA9\xE2\x8C\x3B\xE0"
-			  "\xED\x52\xAE\x90\x8F\x5B\x98\x34"
-			  "\x29\x94\x34\x7F\xF9\x6C\x1E\xB6"
-			  "\xA4\xE7\x2D\x06\x54\x9D\xC3\x02"
-			  "\xC1\x90\xA4\x72\x31\x6B\x24\x51"
-			  "\x0B\xB3\x7C\x63\x15\xBA\xAF\x5D"
-			  "\x41\xE0\x37\x6D\xBE\x41\x58\xDE"
-			  "\xF2\x07\x62\x99\xBE\xC1\x8C\x0F"
-			  "\x0F\x28\xFB\x8F\x0E\x1D\x91\xE2"
-			  "\xDA\x99\x5C\x49\xBA\x9C\xA8\x86"
-			  "\x82\x63\x11\xB3\x54\x49\x00\x08"
-			  "\x07\xF2\xE8\x1F\x34\x49\x61\xF4"
-			  "\x81\xE9\xF6\xA9\x5A\x28\x60\x1F"
-			  "\x66\x99\x08\x06\xF2\xE8\x2D\xD1"
-			  "\xD0\x67\xBA\x32\x1F\x02\x86\x7B"
-			  "\xFB\x79\x3D\xC5\xB1\x7F\x15\xAF"
-			  "\xD7\xBF\x31\x46\x22\x7F\xAE\x5B"
-			  "\x8B\x95\x47\xC2\xB1\x62\xA1\xCE"
-			  "\x52\xAC\x9C\x8B\xC2\x49\x7F\xBC"
-			  "\x9C\x89\xB8\xB6\xCA\xE3\x8F\xEA"
-			  "\xAC\xB4\x5D\xE4\x50\xDC\x3A\xB5"
-			  "\x91\x04\x94\x99\x03\x3B\x42\x6D"
-			  "\x9C\x4A\x02\xF5\xB5\x38\x98\xA8"
-			  "\x5C\x97\x2E\x4D\x79\x67\x71\xAF"
-			  "\xF0\x70\x77\xFF\x2D\xDA\xA0\x9E"
-			  "\x23\x8D\xD6\xA6\x68\x10\x78\x9A"
-			  "\x64\xBB\x15\xB8\x56\xCF\xEE\xE5"
-			  "\x32\x44\x96\x1C\xD8\xEB\x95\xD2"
-			  "\xF3\x71\xEF\xEB\x4E\xBB\x4D\x29",
-		.ilen	= 504,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+/*
+ * Twofish test vectors.
+ */
+static const struct cipher_testvec tf_tv_template[] = {
+	{
+		.key	= zeroed_string,
+		.klen	= 16,
+		.ptext	= zeroed_string,
+		.ctext	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
+			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a",
+		.len	= 16,
+	}, {
+		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
+			  "\x00\x11\x22\x33\x44\x55\x66\x77",
+		.klen	= 24,
+		.ptext	= zeroed_string,
+		.ctext	= "\xcf\xd1\xd2\xe5\xa9\xbe\x9c\xdf"
+			  "\x50\x1f\x13\xb8\x92\xbd\x22\x48",
+		.len	= 16,
+	}, {
+		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
+			  "\x00\x11\x22\x33\x44\x55\x66\x77"
+			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
+		.klen	= 32,
+		.ptext	= zeroed_string,
+		.ctext	= "\x37\x52\x7b\xe0\x05\x23\x34\xb8"
+			  "\x9f\x0c\xfc\xca\xe8\x7c\xfa\x20",
+		.len	= 16,
+	}, { /* Generated with Crypto++ */
+		.key	= "\x3F\x85\x62\x3F\x1C\xF9\xD6\x1C"
+			  "\xF9\xD6\xB3\x90\x6D\x4A\x90\x6D"
+			  "\x4A\x27\x04\xE1\x27\x04\xE1\xBE"
+			  "\x9B\x78\xBE\x9B\x78\x55\x32\x0F",
+		.klen	= 32,
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -8845,81 +7455,124 @@ static const struct cipher_testvec bf_ctr_dec_tv_template[] = {
 			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
 			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06",
-		.rlen	= 504,
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\x88\xCB\x1E\xC2\xAF\x8A\x97\xFF"
+			  "\xF6\x90\x46\x9C\x4A\x0F\x08\xDC"
+			  "\xDE\xAB\xAD\xFA\xFC\xA8\xC2\x3D"
+			  "\xE0\xE4\x8B\x3F\xD5\xA3\xF7\x14"
+			  "\x34\x9E\xB6\x08\xB2\xDD\xA8\xF5"
+			  "\xDF\xFA\xC7\xE8\x09\x50\x76\x08"
+			  "\xA2\xB6\x6A\x59\xC0\x2B\x6D\x05"
+			  "\x89\xF6\x82\xF0\xD3\xDB\x06\x02"
+			  "\xB5\x11\x5C\x5E\x79\x1A\xAC\x43"
+			  "\x5C\xC0\x30\x4B\x6B\x16\xA1\x40"
+			  "\x80\x27\x88\xBA\x2C\x74\x42\xE0"
+			  "\x1B\xA5\x85\x08\xB9\xE6\x22\x7A"
+			  "\x36\x3B\x0D\x9F\xA0\x22\x6C\x2A"
+			  "\x91\x75\x47\xBC\x67\x21\x4E\xF9"
+			  "\xEA\xFF\xD9\xD5\xC0\xFC\x9E\x2C"
+			  "\x3E\xAD\xC6\x61\x0E\x93\x7A\x22"
+			  "\x09\xC8\x8D\xC1\x8E\xB4\x8B\x5C"
+			  "\xC6\x24\x42\xB8\x23\x66\x80\xA9"
+			  "\x32\x0B\x7A\x29\xBF\xB3\x0B\x63"
+			  "\x43\x27\x13\xA9\xBE\xEB\xBD\xF3"
+			  "\x33\x62\x70\xE2\x1B\x86\x7A\xA1"
+			  "\x51\x4A\x16\xFE\x29\x63\x7E\xD0"
+			  "\x7A\xA4\x6E\x2C\xF8\xC1\xDB\xE8"
+			  "\xCB\x4D\xD2\x8C\x04\x14\xB4\x66"
+			  "\x41\xB7\x3A\x96\x16\x7C\x1D\x5B"
+			  "\xB6\x41\x42\x64\x43\xEE\x6E\x7C"
+			  "\x8B\xAF\x01\x9C\xA4\x6E\x75\x8F"
+			  "\xDE\x10\x9F\xA6\xE7\xD6\x44\x97"
+			  "\x66\xA3\x96\x0F\x1C\x25\x60\xF5"
+			  "\x3C\x2E\x32\x69\x0E\x82\xFF\x27"
+			  "\x0F\xB5\x06\xDA\xD8\x31\x15\x6C"
+			  "\xDF\x18\x6C\x87\xF5\x3B\x11\x9A"
+			  "\x1B\x42\x1F\x5B\x29\x19\x96\x13"
+			  "\x68\x2E\x5E\x08\x1C\x8F\x32\x4B"
+			  "\x81\x77\x6D\xF4\xA0\x01\x42\xEC"
+			  "\xDD\x5B\xFD\x3A\x8E\x6A\x14\xFB"
+			  "\x83\x54\xDF\x0F\x86\xB7\xEA\x40"
+			  "\x46\x39\xF7\x2A\x89\x8D\x4E\x96"
+			  "\x5F\x5F\x6D\x76\xC6\x13\x9D\x3D"
+			  "\x1D\x5F\x0C\x7D\xE2\xBC\xC2\x16"
+			  "\x16\xBE\x89\x3E\xB0\x61\xA2\x5D"
+			  "\xAF\xD1\x40\x5F\x1A\xB8\x26\x41"
+			  "\xC6\xBD\x36\xEF\xED\x29\x50\x6D"
+			  "\x10\xEF\x26\xE8\xA8\x93\x11\x3F"
+			  "\x2D\x1F\x88\x20\x77\x45\xF5\x66"
+			  "\x08\xB9\xF1\xEF\xB1\x93\xA8\x81"
+			  "\x65\xC5\xCD\x3E\x8C\x06\x60\x2C"
+			  "\xB2\x10\x7A\xCA\x05\x25\x59\xDB"
+			  "\xC7\x28\xF5\x20\x35\x52\x9E\x62"
+			  "\xF8\x88\x24\x1C\x4D\x84\x12\x39"
+			  "\x39\xE4\x2E\xF4\xD4\x9D\x2B\xBC"
+			  "\x87\x66\xE6\xC0\x6B\x31\x9A\x66"
+			  "\x03\xDC\x95\xD8\x6B\xD0\x30\x8F"
+			  "\xDF\x8F\x8D\xFA\xEC\x1F\x08\xBD"
+			  "\xA3\x63\xE2\x71\x4F\x03\x94\x87"
+			  "\x50\xDF\x15\x1F\xED\x3A\xA3\x7F"
+			  "\x1F\x2A\xB5\xA1\x69\xAC\x4B\x0D"
+			  "\x84\x9B\x2A\xE9\x55\xDD\x46\x91"
+			  "\x15\x33\xF3\x2B\x9B\x46\x97\x00"
+			  "\xF0\x29\xD8\x59\x5D\x33\x37\xF9"
+			  "\x58\x33\x9B\x78\xC7\x58\x48\x6B"
+			  "\x2C\x75\x64\xC4\xCA\xC1\x7E\xD5",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec tf_cbc_tv_template[] = {
+	{ /* Generated with Nettle */
+		.key	= zeroed_string,
+		.klen	= 16,
+		.iv	= zeroed_string,
+		.ptext	= zeroed_string,
+		.ctext	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
+			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a",
+		.len	= 16,
+	}, {
+		.key	= zeroed_string,
+		.klen	= 16,
+		.iv	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
+			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a",
+		.ptext	= zeroed_string,
+		.ctext	= "\xd4\x91\xdb\x16\xe7\xb1\xc3\x9e"
+			  "\x86\xcb\x08\x6b\x78\x9f\x54\x19",
+		.len	= 16,
+	}, {
+		.key	= zeroed_string,
+		.klen	= 16,
+		.iv	= "\xd4\x91\xdb\x16\xe7\xb1\xc3\x9e"
+			  "\x86\xcb\x08\x6b\x78\x9f\x54\x19",
+		.ptext	= zeroed_string,
+		.ctext	= "\x05\xef\x8c\x61\xa8\x11\x58\x26"
+			  "\x34\xba\x5c\xb7\x10\x6a\xa6\x41",
+		.len	= 16,
+	}, {
+		.key	= zeroed_string,
+		.klen	= 16,
+		.iv	= zeroed_string,
+		.ptext	= zeroed_string,
+		.ctext	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
+			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a"
+			  "\xd4\x91\xdb\x16\xe7\xb1\xc3\x9e"
+			  "\x86\xcb\x08\x6b\x78\x9f\x54\x19"
+			  "\x05\xef\x8c\x61\xa8\x11\x58\x26"
+			  "\x34\xba\x5c\xb7\x10\x6a\xa6\x41",
+		.len	= 48,
 	}, { /* Generated with Crypto++ */
 		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
 			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
 			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
 			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\xC7\xA3\xDF\xB9\x05\xF4\x9E\x8D"
-			  "\x9E\xDF\x38\x18\x83\x07\xEF\xC1"
-			  "\x93\x3C\xAA\xAA\xFE\x06\x42\xCC"
-			  "\x0D\x70\x86\x5A\x44\xAD\x85\x17"
-			  "\xE4\x1F\x5E\xA5\x89\xAC\x32\xBC"
-			  "\x3D\xA7\xE9\x0A\x5C\x70\x4D\xDE"
-			  "\x99\x38\x07\xCA\x1D\x21\xC1\x11"
-			  "\x97\xEB\x98\x75\xC4\x73\x45\x83"
-			  "\x46\x1C\x9C\x91\x87\xC1\xA0\x56"
-			  "\x98\xA1\x8B\xDB\x22\x76\xBD\x62"
-			  "\xA4\xBC\xE8\x86\xDA\xD2\x51\x13"
-			  "\x13\xD2\x96\x68\x69\x10\x67\x0C"
-			  "\xD0\x17\x25\x7C\xB2\xAE\x4F\x93"
-			  "\xA6\x82\x20\xCF\x0F\xA6\x47\x79"
-			  "\x88\x09\x40\x59\xBD\x12\x64\xB5"
-			  "\x19\x38\x0D\xFF\x86\xD9\x42\x20"
-			  "\x81\x0D\x96\x99\xAF\x22\x1F\x94"
-			  "\x5C\x6E\xEC\xEA\xA3\x39\xCB\x09"
-			  "\x43\x19\x7F\xD0\xBB\x10\xC2\x49"
-			  "\xF7\xE9\xF2\xEE\xBF\xF7\xF8\xB3"
-			  "\x0E\x1A\xF1\x8D\x70\x82\x0C\x04"
-			  "\xFD\x29\x1A\xAC\xC0\x92\x48\x34"
-			  "\x6A\xE3\x1D\x4F\xFC\x1C\x72\x6A"
-			  "\x57\xCB\xAD\xD0\x98\xAB\xB1\x01"
-			  "\x03\x6A\x45\xDD\x07\x71\x5F\x5B"
-			  "\xB5\x4A\xE4\xE5\xB9\xB9\xBC\xAC"
-			  "\x44\xF7\x41\xA4\x5F\x2E\xE9\x28"
-			  "\xE3\x05\xD2\x94\x78\x4C\x33\x1B"
-			  "\xBD\xC1\x6E\x51\xD9\xAD\xD9\x86"
-			  "\x15\x4A\x78\xAE\x7B\xAD\x3B\xBC"
-			  "\x2F\xE0\x0E\xC5\x7B\x54\x97\x5F"
-			  "\x60\x51\x14\x65\xF9\x91\xE9\xDA"
-			  "\x9A\xBC\xFC\x19\x29\x67\xAA\x63"
-			  "\x5E\xF2\x48\x88\xEB\x79\xE1\xE4"
-			  "\xF7\xF6\x4C\xA9\xE2\x8C\x3B\xE0"
-			  "\xED\x52\xAE\x90\x8F\x5B\x98\x34"
-			  "\x29\x94\x34\x7F\xF9\x6C\x1E\xB6"
-			  "\xA4\xE7\x2D\x06\x54\x9D\xC3\x02"
-			  "\xC1\x90\xA4\x72\x31\x6B\x24\x51"
-			  "\x0B\xB3\x7C\x63\x15\xBA\xAF\x5D"
-			  "\x41\xE0\x37\x6D\xBE\x41\x58\xDE"
-			  "\xF2\x07\x62\x99\xBE\xC1\x8C\x0F"
-			  "\x0F\x28\xFB\x8F\x0E\x1D\x91\xE2"
-			  "\xDA\x99\x5C\x49\xBA\x9C\xA8\x86"
-			  "\x82\x63\x11\xB3\x54\x49\x00\x08"
-			  "\x07\xF2\xE8\x1F\x34\x49\x61\xF4"
-			  "\x81\xE9\xF6\xA9\x5A\x28\x60\x1F"
-			  "\x66\x99\x08\x06\xF2\xE8\x2D\xD1"
-			  "\xD0\x67\xBA\x32\x1F\x02\x86\x7B"
-			  "\xFB\x79\x3D\xC5\xB1\x7F\x15\xAF"
-			  "\xD7\xBF\x31\x46\x22\x7F\xAE\x5B"
-			  "\x8B\x95\x47\xC2\xB1\x62\xA1\xCE"
-			  "\x52\xAC\x9C\x8B\xC2\x49\x7F\xBC"
-			  "\x9C\x89\xB8\xB6\xCA\xE3\x8F\xEA"
-			  "\xAC\xB4\x5D\xE4\x50\xDC\x3A\xB5"
-			  "\x91\x04\x94\x99\x03\x3B\x42\x6D"
-			  "\x9C\x4A\x02\xF5\xB5\x38\x98\xA8"
-			  "\x5C\x97\x2E\x4D\x79\x67\x71\xAF"
-			  "\xF0\x70\x77\xFF\x2D\xDA\xA0\x9E"
-			  "\x23\x8D\xD6\xA6\x68\x10\x78\x9A"
-			  "\x64\xBB\x15\xB8\x56\xCF\xEE\xE5"
-			  "\x32\x44\x96\x1C\xD8\xEB\x95\xD2"
-			  "\xF3\x71\xEF\xEB\x4E\xBB\x4D",
-		.ilen	= 503,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -8980,84 +7633,86 @@ static const struct cipher_testvec bf_ctr_dec_tv_template[] = {
 			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
 			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59\xF0\x64\xFB\x92",
-		.rlen	= 503,
-		.also_non_np = 1,
-		.np	= 2,
-		.tap	= { 503 - 8, 8 },
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x5F\x58\x6E\x60\x51\x6E\xDC\x3D"
-			  "\xD1\xBB\xF7\xB7\xFD\x04\x44\x82"
-			  "\xDC\x9F\x4B\x02\xF1\xD2\x5A\x6F"
-			  "\x25\xF9\x27\x21\xF2\xD2\x9A\x01"
-			  "\xBD\xAD\x3D\x93\x87\xCA\x0D\xFE"
-			  "\xB7\x2C\x17\x1F\x42\x8C\x13\xB2"
-			  "\x62\x44\x72\xB9\x5D\xC0\xF8\x37"
-			  "\xDF\xEA\x78\x81\x8F\xA6\x34\xB2"
-			  "\x07\x09\x7C\xB9\x3A\xA0\x2B\x18"
-			  "\x34\x6A\x9D\x3D\xA5\xEB\xF4\x60"
-			  "\xF8\x98\xA2\x39\x81\x23\x6C\xA9"
-			  "\x70\xCA\xCC\x45\xD8\x1F\xDF\x44"
-			  "\x2A\x67\x7A\x88\x28\xDC\x36\x83"
-			  "\x18\xD7\x48\x43\x17\x2B\x1B\xE6"
-			  "\x0B\x82\x59\x14\x26\x67\x08\x09"
-			  "\x5B\x5D\x38\xD0\x81\xCE\x54\x2A"
-			  "\xCD\x22\x94\x42\xF5\xBA\x74\x7E"
-			  "\xD9\x00\x40\xA9\x0D\x0B\xBD\x8E"
-			  "\xC4\x8E\x5E\x17\x8F\x48\xE2\xB8"
-			  "\xF4\xCC\x19\x76\xAB\x48\x29\xAA"
-			  "\x81\xD5\xCE\xD5\x8A\x3B\xC9\x21"
-			  "\xEF\x50\x4F\x04\x02\xBF\xE1\x1F"
-			  "\x59\x28\x1A\xE4\x18\x16\xA0\x29"
-			  "\xBF\x34\xA9\x2D\x28\x83\xC0\x5E"
-			  "\xEA\x44\xC4\x6E\xAB\x24\x79\x9D"
-			  "\x2D\xA1\xE8\x55\xCA\x74\xFC\xBD"
-			  "\xFE\xDD\xDA\xA5\xFB\x34\x90\x31"
-			  "\x0E\x62\x28\x9B\xDC\xD7\xA1\xBB"
-			  "\xF0\x1A\xB3\xE2\xD0\xFA\xBD\xE8"
-			  "\x5C\x5A\x10\x67\xF6\x6A\x17\x3F"
-			  "\xC5\xE9\x09\x08\xDD\x22\x77\x42"
-			  "\x26\x6A\x6A\x7A\x3F\x87\x80\x0C"
-			  "\xF0\xFF\x15\x8E\x84\x86\xC0\x10"
-			  "\x0F\x8D\x33\x06\xB8\x72\xA4\x47"
-			  "\x6B\xED\x2E\x05\x94\x6C\x5C\x5B"
-			  "\x13\xF6\x77\xEE\x3B\x16\xDF\xC2"
-			  "\x63\x66\x07\x6D\x3F\x6C\x51\x7C"
-			  "\x1C\xAC\x80\xB6\x58\x48\xB7\x9D"
-			  "\xB4\x19\xD8\x19\x45\x66\x27\x02"
-			  "\xA1\xA9\x99\xF3\x1F\xE5\xA7\x1D"
-			  "\x31\xE7\x1B\x0D\xFF\xBB\xB5\xA1"
-			  "\xF5\x9C\x45\x1E\x18\x19\xA1\xE7"
-			  "\xC2\xF1\xBF\x68\xC3\xEC\xCF\x53"
-			  "\x67\xA6\x2B\x7D\x3C\x6D\x24\xC3"
-			  "\xE8\xE6\x07\x5A\x09\xE0\x32\xA8"
-			  "\x52\xF6\xE9\xED\x0E\xC6\x0A\x6A"
-			  "\xFC\x60\x2A\xE0\x93\xCE\xB8\x2E"
-			  "\xA2\xA8\x0E\x79\x9E\x34\x5D\x37"
-			  "\x6F\x12\xFE\x48\x7B\xE7\xB9\x22"
-			  "\x29\xE8\xD7\xBE\x5D\xD1\x8B\xD9"
-			  "\x91\x51\x4E\x71\xF2\x98\x85\x16"
-			  "\x25\x7A\x76\x8A\x51\x0E\x65\x14"
-			  "\x81\xB5\x3A\x37\xFD\xEC\xB5\x8A"
-			  "\xE1\xCF\x41\x72\x14\x29\x4C\xF0"
-			  "\x20\xD9\x9A\xC5\x66\xA4\x03\x76"
-			  "\x5B\xA4\x15\x4F\x0E\x64\x39\x40"
-			  "\x25\xF9\x20\x22\xF5\x88\xF5\xBA"
-			  "\xE4\xDF\x45\x61\xBF\x8D\x7A\x24"
-			  "\x4B\x92\x71\xD9\x2F\x77\xA7\x95"
-			  "\xA8\x7F\x61\xD5\xA4\x57\xB0\xFB"
-			  "\xB5\x77\xBA\x1C\xEE\x71\xFA\xB0"
-			  "\x16\x4C\x18\x6B\xF2\x69\xA0\x07"
-			  "\xEF\xBE\xEC\x69\xAC\xA8\x63\x9E",
-		.ilen	= 504,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\xC8\xFF\xF2\x53\xA6\x27\x09\xD1"
+			  "\x33\x38\xC2\xC0\x0C\x14\x7E\xB5"
+			  "\x26\x1B\x05\x0C\x05\x12\x3F\xC0"
+			  "\xF9\x1C\x02\x28\x40\x96\x6F\xD0"
+			  "\x3D\x32\xDF\xDA\x56\x00\x6E\xEE"
+			  "\x5B\x2A\x72\x9D\xC2\x4D\x19\xBC"
+			  "\x8C\x53\xFA\x87\x6F\xDD\x81\xA3"
+			  "\xB1\xD3\x44\x65\xDF\xE7\x63\x38"
+			  "\x4A\xFC\xDC\xEC\x3F\x26\x8E\xB8"
+			  "\x43\xFC\xFE\x18\xB5\x11\x6D\x31"
+			  "\x81\x8B\x0D\x75\xF6\x80\xEC\x84"
+			  "\x04\xB9\xE6\x09\x63\xED\x39\xDB"
+			  "\xC3\xF6\x14\xD6\x6E\x5E\x8B\xBD"
+			  "\x3E\xFA\xD7\x98\x50\x6F\xD9\x63"
+			  "\x02\xCD\x0D\x39\x4B\x0D\xEC\x80"
+			  "\xE3\x6A\x17\xF4\xCC\xAD\xFF\x68"
+			  "\x45\xDD\xC8\x83\x1D\x41\x96\x0D"
+			  "\x91\x2E\x05\xD3\x59\x82\xE0\x43"
+			  "\x90\x4F\xB9\xF7\xAD\x6B\x2E\xAF"
+			  "\xA7\x84\x00\x53\xCD\x6F\xD1\x0C"
+			  "\x4E\xF9\x5A\x23\xFB\xCA\xC7\xD3"
+			  "\xA9\xAA\x9D\xB2\x3F\x66\xF1\xAC"
+			  "\x25\x21\x8F\xF7\xEF\xF2\x6A\xDF"
+			  "\xE8\xDA\x75\x1A\x8A\xF1\xDD\x38"
+			  "\x1F\xF9\x3D\x68\x4A\xBB\x9E\x34"
+			  "\x1F\x66\x1F\x9C\x2B\x54\xFF\x60"
+			  "\x7F\x29\x4B\x55\x80\x8F\x4E\xA7"
+			  "\xA6\x9A\x0A\xD9\x0D\x19\x00\xF8"
+			  "\x1F\xBC\x0C\x40\x6B\xEC\x99\x25"
+			  "\x94\x70\x74\x0E\x1D\xC5\xBC\x12"
+			  "\xF3\x42\xBE\x95\xBF\xFB\x4E\x55"
+			  "\x9A\xB9\xCE\x14\x16\x5B\xDC\xD3"
+			  "\x75\x42\x62\x04\x31\x1F\x95\x7C"
+			  "\x66\x1A\x97\xDC\x2F\x40\x5C\x39"
+			  "\x78\xE6\x02\xDB\x49\xE1\xC6\x47"
+			  "\xC2\x78\x9A\xBB\xF3\xBE\xCB\x93"
+			  "\xD8\xB8\xE8\xBB\x8C\xB3\x9B\xA7"
+			  "\xC2\x89\xF3\x91\x88\x83\x3D\xF0"
+			  "\x29\xA2\xCD\xB5\x79\x16\xC2\x40"
+			  "\x11\x03\x8E\x9C\xFD\xC9\x43\xC4"
+			  "\xC2\x19\xF0\x4A\x32\xEF\x0C\x2B"
+			  "\xD3\x2B\xE9\xD4\x4C\xDE\x95\xCF"
+			  "\x04\x03\xD3\x2C\x7F\x82\xC8\xFA"
+			  "\x0F\xD8\x7A\x39\x7B\x01\x41\x9C"
+			  "\x78\xB6\xC9\xBF\xF9\x78\x57\x88"
+			  "\xB1\xA5\xE1\xE0\xD9\x16\xD4\xC8"
+			  "\xEE\xC4\xBE\x7B\x55\x59\x00\x48"
+			  "\x1B\xBC\x14\xFA\x2A\x9D\xC9\x1C"
+			  "\xFB\x28\x3F\x95\xDD\xB7\xD6\xCE"
+			  "\x3A\x7F\x09\x0C\x0E\x69\x30\x7D"
+			  "\xBC\x68\x9C\x91\x2A\x59\x57\x04"
+			  "\xED\x1A\x1E\x00\xB1\x85\x92\x04"
+			  "\x28\x8C\x0C\x3C\xC1\xD5\x12\xF7"
+			  "\x4C\x3E\xB0\xE7\x86\x62\x68\x91"
+			  "\xFC\xC4\xE2\xCE\xA6\xDC\x5E\x93"
+			  "\x5D\x8D\x8C\x68\xB3\xB2\xB9\x64"
+			  "\x16\xB8\xC8\x6F\xD8\xEE\x21\xBD"
+			  "\xAC\x18\x0C\x7D\x0D\x05\xAB\xF1"
+			  "\xFA\xDD\xE2\x48\xDF\x4C\x02\x39"
+			  "\x69\xA1\x62\xBD\x49\x3A\x9D\x91"
+			  "\x30\x70\x56\xA4\x37\xDD\x7C\xC0"
+			  "\x0A\xA3\x30\x10\x26\x25\x41\x2C",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec tf_ctr_tv_template[] = {
+	{ /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -9118,52 +7773,79 @@ static const struct cipher_testvec bf_ctr_dec_tv_template[] = {
 			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
 			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06",
-		.rlen	= 504,
-	},
-};
-
-/*
- * Twofish test vectors.
- */
-static const struct cipher_testvec tf_enc_tv_template[] = {
-	{
-		.key	= zeroed_string,
-		.klen	= 16,
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.result	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
-			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a",
-		.rlen	= 16,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
-			  "\x00\x11\x22\x33\x44\x55\x66\x77",
-		.klen	= 24,
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.result	= "\xcf\xd1\xd2\xe5\xa9\xbe\x9c\xdf"
-			  "\x50\x1f\x13\xb8\x92\xbd\x22\x48",
-		.rlen	= 16,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
-			  "\x00\x11\x22\x33\x44\x55\x66\x77"
-			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
-		.klen	= 32,
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.result	= "\x37\x52\x7b\xe0\x05\x23\x34\xb8"
-			  "\x9f\x0c\xfc\xca\xe8\x7c\xfa\x20",
-		.rlen	= 16,
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\xDF\xDD\x69\xFA\xB0\x2E\xFD\xFE"
+			  "\x70\x9E\xC5\x4B\xC9\xD4\xA1\x30"
+			  "\x26\x9B\x89\xA1\xEE\x43\xE0\x52"
+			  "\x55\x17\x4E\xC7\x0E\x33\x1F\xF1"
+			  "\x9F\x8D\x40\x9F\x24\xFD\x92\xA0"
+			  "\xBC\x8F\x35\xDD\x67\x38\xD8\xAA"
+			  "\xCF\xF8\x48\xCA\xFB\xE4\x5C\x60"
+			  "\x01\x41\x21\x12\x38\xAB\x52\x4F"
+			  "\xA8\x57\x20\xE0\x21\x6A\x17\x0D"
+			  "\x0E\xF9\x8E\x49\x42\x00\x3C\x94"
+			  "\x14\xC0\xD0\x8D\x8A\x98\xEB\x29"
+			  "\xEC\xAE\x96\x44\xC0\x3C\x48\xDC"
+			  "\x29\x35\x25\x2F\xE7\x11\x6C\x68"
+			  "\xC8\x67\x0A\x2F\xF4\x07\xBE\xF9"
+			  "\x2C\x31\x87\x40\xAB\xB2\xB6\xFA"
+			  "\xD2\xC9\x6D\x5C\x50\xE9\xE6\x7E"
+			  "\xE3\x0A\xD2\xD5\x6D\x8D\x64\x9E"
+			  "\x70\xCE\x03\x76\xDD\xE0\xF0\x8C"
+			  "\x84\x86\x8B\x6A\xFE\xC7\xF9\x69"
+			  "\x2E\xFE\xFC\xC2\xC4\x1A\x55\x58"
+			  "\xB3\xBE\xE2\x7E\xED\x39\x42\x6C"
+			  "\xB4\x42\x97\x9A\xEC\xE1\x0A\x06"
+			  "\x02\xC5\x03\x9D\xC4\x48\x15\x66"
+			  "\x35\x6A\xC2\xC9\xA2\x26\x30\xBB"
+			  "\xDB\x2D\xC8\x08\x2B\xA0\x29\x1A"
+			  "\x23\x61\x48\xEA\x80\x04\x27\xAA"
+			  "\x69\x49\xE8\xE8\x4A\x83\x6B\x5A"
+			  "\xCA\x7C\xD3\xB1\xB5\x0B\xCC\x23"
+			  "\x74\x1F\xA9\x87\xCD\xED\xC0\x2D"
+			  "\xBF\xEB\xCF\x16\x2D\x2A\x2E\x1D"
+			  "\x96\xBA\x36\x11\x45\x41\xDA\xCE"
+			  "\xA4\x48\x80\x8B\x06\xF4\x98\x89"
+			  "\x8B\x23\x08\x53\xF4\xD4\x5A\x24"
+			  "\x8B\xF8\x43\x73\xD1\xEE\xC4\xB0"
+			  "\xF8\xFE\x09\x0C\x75\x05\x38\x0B"
+			  "\x7C\x81\xDE\x9D\xE4\x61\x37\x63"
+			  "\x63\xAD\x12\xD2\x04\xB9\xCE\x45"
+			  "\x5A\x1A\x6E\xB3\x78\x2A\xA4\x74"
+			  "\x86\xD0\xE3\xFF\xDA\x38\x9C\xB5"
+			  "\xB8\xB1\xDB\x38\x2F\xC5\x6A\xB4"
+			  "\xEB\x6E\x96\xE8\x43\x80\xB5\x51"
+			  "\x61\x2D\x48\xAA\x07\x65\x11\x8C"
+			  "\x48\xE3\x90\x7E\x78\x3A\xEC\x97"
+			  "\x05\x3D\x84\xE7\x90\x2B\xAA\xBD"
+			  "\x83\x29\x0E\x1A\x81\x73\x7B\xE0"
+			  "\x7A\x01\x4A\x37\x3B\x77\x7F\x8D"
+			  "\x49\xA4\x2F\x6E\xBE\x68\x99\x08"
+			  "\x99\xAA\x4C\x12\x04\xAE\x1F\x77"
+			  "\x35\x88\xF1\x65\x06\x0A\x0B\x4D"
+			  "\x47\xF9\x50\x38\x5D\x71\xF9\x6E"
+			  "\xDE\xEC\x61\x35\x2C\x4C\x96\x50"
+			  "\xE8\x28\x93\x9C\x7E\x01\xC6\x04"
+			  "\xB2\xD6\xBC\x6C\x17\xEB\xC1\x7D"
+			  "\x11\xE9\x43\x83\x76\xAA\x53\x37"
+			  "\x0C\x1D\x39\x89\x53\x72\x09\x7E"
+			  "\xD9\x85\x16\x04\xA5\x2C\x05\x6F"
+			  "\x17\x0C\x6E\x66\xAA\x84\xA7\xD9"
+			  "\xE2\xD9\xC4\xEB\x43\x3E\xB1\x8D"
+			  "\x7C\x36\xC7\x71\x70\x9C\x10\xD8"
+			  "\xE8\x47\x2A\x4D\xFD\xA1\xBC\xE3"
+			  "\xB9\x32\xE2\xC1\x82\xAC\xFE\xCC"
+			  "\xC5\xC9\x7F\x9E\xCF\x33\x7A\xDF",
+		.len	= 496,
 	}, { /* Generated with Crypto++ */
-		.key	= "\x3F\x85\x62\x3F\x1C\xF9\xD6\x1C"
-			  "\xF9\xD6\xB3\x90\x6D\x4A\x90\x6D"
-			  "\x4A\x27\x04\xE1\x27\x04\xE1\xBE"
-			  "\x9B\x78\xBE\x9B\x78\x55\x32\x0F",
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -9225,176 +7907,78 @@ static const struct cipher_testvec tf_enc_tv_template[] = {
 			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
 			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\x88\xCB\x1E\xC2\xAF\x8A\x97\xFF"
-			  "\xF6\x90\x46\x9C\x4A\x0F\x08\xDC"
-			  "\xDE\xAB\xAD\xFA\xFC\xA8\xC2\x3D"
-			  "\xE0\xE4\x8B\x3F\xD5\xA3\xF7\x14"
-			  "\x34\x9E\xB6\x08\xB2\xDD\xA8\xF5"
-			  "\xDF\xFA\xC7\xE8\x09\x50\x76\x08"
-			  "\xA2\xB6\x6A\x59\xC0\x2B\x6D\x05"
-			  "\x89\xF6\x82\xF0\xD3\xDB\x06\x02"
-			  "\xB5\x11\x5C\x5E\x79\x1A\xAC\x43"
-			  "\x5C\xC0\x30\x4B\x6B\x16\xA1\x40"
-			  "\x80\x27\x88\xBA\x2C\x74\x42\xE0"
-			  "\x1B\xA5\x85\x08\xB9\xE6\x22\x7A"
-			  "\x36\x3B\x0D\x9F\xA0\x22\x6C\x2A"
-			  "\x91\x75\x47\xBC\x67\x21\x4E\xF9"
-			  "\xEA\xFF\xD9\xD5\xC0\xFC\x9E\x2C"
-			  "\x3E\xAD\xC6\x61\x0E\x93\x7A\x22"
-			  "\x09\xC8\x8D\xC1\x8E\xB4\x8B\x5C"
-			  "\xC6\x24\x42\xB8\x23\x66\x80\xA9"
-			  "\x32\x0B\x7A\x29\xBF\xB3\x0B\x63"
-			  "\x43\x27\x13\xA9\xBE\xEB\xBD\xF3"
-			  "\x33\x62\x70\xE2\x1B\x86\x7A\xA1"
-			  "\x51\x4A\x16\xFE\x29\x63\x7E\xD0"
-			  "\x7A\xA4\x6E\x2C\xF8\xC1\xDB\xE8"
-			  "\xCB\x4D\xD2\x8C\x04\x14\xB4\x66"
-			  "\x41\xB7\x3A\x96\x16\x7C\x1D\x5B"
-			  "\xB6\x41\x42\x64\x43\xEE\x6E\x7C"
-			  "\x8B\xAF\x01\x9C\xA4\x6E\x75\x8F"
-			  "\xDE\x10\x9F\xA6\xE7\xD6\x44\x97"
-			  "\x66\xA3\x96\x0F\x1C\x25\x60\xF5"
-			  "\x3C\x2E\x32\x69\x0E\x82\xFF\x27"
-			  "\x0F\xB5\x06\xDA\xD8\x31\x15\x6C"
-			  "\xDF\x18\x6C\x87\xF5\x3B\x11\x9A"
-			  "\x1B\x42\x1F\x5B\x29\x19\x96\x13"
-			  "\x68\x2E\x5E\x08\x1C\x8F\x32\x4B"
-			  "\x81\x77\x6D\xF4\xA0\x01\x42\xEC"
-			  "\xDD\x5B\xFD\x3A\x8E\x6A\x14\xFB"
-			  "\x83\x54\xDF\x0F\x86\xB7\xEA\x40"
-			  "\x46\x39\xF7\x2A\x89\x8D\x4E\x96"
-			  "\x5F\x5F\x6D\x76\xC6\x13\x9D\x3D"
-			  "\x1D\x5F\x0C\x7D\xE2\xBC\xC2\x16"
-			  "\x16\xBE\x89\x3E\xB0\x61\xA2\x5D"
-			  "\xAF\xD1\x40\x5F\x1A\xB8\x26\x41"
-			  "\xC6\xBD\x36\xEF\xED\x29\x50\x6D"
-			  "\x10\xEF\x26\xE8\xA8\x93\x11\x3F"
-			  "\x2D\x1F\x88\x20\x77\x45\xF5\x66"
-			  "\x08\xB9\xF1\xEF\xB1\x93\xA8\x81"
-			  "\x65\xC5\xCD\x3E\x8C\x06\x60\x2C"
-			  "\xB2\x10\x7A\xCA\x05\x25\x59\xDB"
-			  "\xC7\x28\xF5\x20\x35\x52\x9E\x62"
-			  "\xF8\x88\x24\x1C\x4D\x84\x12\x39"
-			  "\x39\xE4\x2E\xF4\xD4\x9D\x2B\xBC"
-			  "\x87\x66\xE6\xC0\x6B\x31\x9A\x66"
-			  "\x03\xDC\x95\xD8\x6B\xD0\x30\x8F"
-			  "\xDF\x8F\x8D\xFA\xEC\x1F\x08\xBD"
-			  "\xA3\x63\xE2\x71\x4F\x03\x94\x87"
-			  "\x50\xDF\x15\x1F\xED\x3A\xA3\x7F"
-			  "\x1F\x2A\xB5\xA1\x69\xAC\x4B\x0D"
-			  "\x84\x9B\x2A\xE9\x55\xDD\x46\x91"
-			  "\x15\x33\xF3\x2B\x9B\x46\x97\x00"
-			  "\xF0\x29\xD8\x59\x5D\x33\x37\xF9"
-			  "\x58\x33\x9B\x78\xC7\x58\x48\x6B"
-			  "\x2C\x75\x64\xC4\xCA\xC1\x7E\xD5",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec tf_dec_tv_template[] = {
-	{
-		.key	= zeroed_string,
-		.klen	= 16,
-		.input	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
-			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
-			  "\x00\x11\x22\x33\x44\x55\x66\x77",
-		.klen	= 24,
-		.input	= "\xcf\xd1\xd2\xe5\xa9\xbe\x9c\xdf"
-			  "\x50\x1f\x13\xb8\x92\xbd\x22\x48",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
-			  "\x00\x11\x22\x33\x44\x55\x66\x77"
-			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
-		.klen	= 32,
-		.input	= "\x37\x52\x7b\xe0\x05\x23\x34\xb8"
-			  "\x9f\x0c\xfc\xca\xe8\x7c\xfa\x20",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
+		.ctext	= "\xEB\x44\xAF\x49\x27\xB8\xFB\x44"
+			  "\x4C\xA6\xC3\x0C\x8B\xD0\x01\x0C"
+			  "\x53\xC8\x16\x38\xDE\x40\x4F\x91"
+			  "\x25\x6D\x4C\xA0\x9A\x87\x1E\xDA"
+			  "\x88\x7E\x89\xE9\x67\x2B\x83\xA2"
+			  "\x5F\x2E\x23\x3E\x45\xB9\x77\x7B"
+			  "\xA6\x7E\x47\x36\x81\x9F\x9B\xF3"
+			  "\xE0\xF0\xD7\x47\xA9\xC8\xEF\x33"
+			  "\x0C\x43\xFE\x67\x50\x0A\x2C\x3E"
+			  "\xA0\xE1\x25\x8E\x80\x07\x4A\xC0"
+			  "\x64\x89\x9F\x6A\x27\x96\x07\xA6"
+			  "\x9B\xC8\x1B\x21\x60\xAE\x5D\x01"
+			  "\xE2\xCD\xC8\xAA\x6C\x9D\x1C\x34"
+			  "\x39\x18\x09\xA4\x82\x59\x78\xE7"
+			  "\xFC\x59\x65\xF2\x94\xFF\xFB\xE2"
+			  "\x3C\xDA\xB1\x90\x95\xBF\x91\xE3"
+			  "\xE6\x87\x31\x9E\x16\x85\xAD\xB1"
+			  "\x4C\xAE\x43\x4D\x19\x58\xB5\x5E"
+			  "\x2E\xF5\x09\xAA\x39\xF4\xC0\xB3"
+			  "\xD4\x4D\xDB\x73\x7A\xD4\xF1\xBF"
+			  "\x89\x16\x4D\x2D\xA2\x26\x33\x72"
+			  "\x18\x33\x7E\xD6\xD2\x16\xA4\x54"
+			  "\xF4\x8C\xB3\x52\xDF\x21\x9C\xEB"
+			  "\xBF\x49\xD3\xF9\x05\x06\xCB\xD2"
+			  "\xA9\xD2\x3B\x6E\x19\x8C\xBC\x19"
+			  "\xAB\x89\xD6\xD8\xCD\x56\x89\x5E"
+			  "\xAC\x00\xE3\x50\x63\x4A\x80\x9A"
+			  "\x05\xBC\x50\x39\xD3\x32\xD9\x0D"
+			  "\xE3\x20\x0D\x75\x54\xEC\xE6\x31"
+			  "\x14\xB9\x3A\x59\x00\x43\x37\x8E"
+			  "\x8C\x5A\x79\x62\x14\x76\x8A\xAE"
+			  "\x8F\xCC\xA1\x6C\x38\x78\xDD\x2D"
+			  "\x8B\x6D\xEA\xBD\x7B\x25\xFF\x60"
+			  "\xC9\x87\xB1\x79\x1E\xA5\x86\x68"
+			  "\x81\xB4\xE2\xC1\x05\x7D\x3A\x73"
+			  "\xD0\xDA\x75\x77\x9E\x05\x27\xF1"
+			  "\x08\xA9\x66\x64\x6C\xBC\x82\x17"
+			  "\x2C\x23\x5F\x62\x4D\x02\x1A\x58"
+			  "\xE7\xB7\x23\x6D\xE2\x20\xDA\xEF"
+			  "\xB4\xB3\x3F\xB2\x2B\x69\x98\x83"
+			  "\x95\x87\x13\x57\x60\xD7\xB5\xB1"
+			  "\xEE\x0A\x2F\x95\x36\x4C\x76\x5D"
+			  "\x5F\xD9\x19\xED\xB9\xA5\x48\xBF"
+			  "\xC8\xAB\x0F\x71\xCC\x61\x8E\x0A"
+			  "\xD0\x29\x44\xA8\xB9\xC1\xE8\xC8"
+			  "\xC9\xA8\x28\x81\xFB\x50\xF2\xF0"
+			  "\x26\xAE\x39\xB8\x91\xCD\xA8\xAC"
+			  "\xDE\x55\x1B\x50\x14\x53\x44\x17"
+			  "\x54\x46\xFC\xB1\xE4\x07\x6B\x9A"
+			  "\x01\x14\xF0\x2E\x2E\xDB\x46\x1B"
+			  "\x1A\x09\x97\xA9\xB6\x97\x79\x06"
+			  "\xFB\xCB\x85\xCF\xDD\xA1\x41\xB1"
+			  "\x00\xAA\xF7\xE0\x89\x73\xFB\xE5"
+			  "\xBF\x84\xDB\xC9\xCD\xC4\xA2\x0D"
+			  "\x3B\xAC\xF9\xDF\x96\xBF\x88\x23"
+			  "\x41\x67\xA1\x24\x99\x7E\xCC\x9B"
+			  "\x02\x8F\x6A\x49\xF6\x25\xBA\x7A"
+			  "\xF4\x78\xFD\x79\x62\x63\x4F\x14"
+			  "\xD6\x11\x11\x04\x05\x5F\x7E\xEA"
+			  "\x4C\xB6\xF8\xF4\x5F\x48\x52\x54"
+			  "\x94\x63\xA8\x4E\xCF\xD2\x1B\x1B"
+			  "\x22\x18\x6A\xAF\x6E\x3E\xE1\x0D",
+		.len	= 496,
 	}, { /* Generated with Crypto++ */
-		.key	= "\x3F\x85\x62\x3F\x1C\xF9\xD6\x1C"
-			  "\xF9\xD6\xB3\x90\x6D\x4A\x90\x6D"
-			  "\x4A\x27\x04\xE1\x27\x04\xE1\xBE"
-			  "\x9B\x78\xBE\x9B\x78\x55\x32\x0F",
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
-		.input	= "\x88\xCB\x1E\xC2\xAF\x8A\x97\xFF"
-			  "\xF6\x90\x46\x9C\x4A\x0F\x08\xDC"
-			  "\xDE\xAB\xAD\xFA\xFC\xA8\xC2\x3D"
-			  "\xE0\xE4\x8B\x3F\xD5\xA3\xF7\x14"
-			  "\x34\x9E\xB6\x08\xB2\xDD\xA8\xF5"
-			  "\xDF\xFA\xC7\xE8\x09\x50\x76\x08"
-			  "\xA2\xB6\x6A\x59\xC0\x2B\x6D\x05"
-			  "\x89\xF6\x82\xF0\xD3\xDB\x06\x02"
-			  "\xB5\x11\x5C\x5E\x79\x1A\xAC\x43"
-			  "\x5C\xC0\x30\x4B\x6B\x16\xA1\x40"
-			  "\x80\x27\x88\xBA\x2C\x74\x42\xE0"
-			  "\x1B\xA5\x85\x08\xB9\xE6\x22\x7A"
-			  "\x36\x3B\x0D\x9F\xA0\x22\x6C\x2A"
-			  "\x91\x75\x47\xBC\x67\x21\x4E\xF9"
-			  "\xEA\xFF\xD9\xD5\xC0\xFC\x9E\x2C"
-			  "\x3E\xAD\xC6\x61\x0E\x93\x7A\x22"
-			  "\x09\xC8\x8D\xC1\x8E\xB4\x8B\x5C"
-			  "\xC6\x24\x42\xB8\x23\x66\x80\xA9"
-			  "\x32\x0B\x7A\x29\xBF\xB3\x0B\x63"
-			  "\x43\x27\x13\xA9\xBE\xEB\xBD\xF3"
-			  "\x33\x62\x70\xE2\x1B\x86\x7A\xA1"
-			  "\x51\x4A\x16\xFE\x29\x63\x7E\xD0"
-			  "\x7A\xA4\x6E\x2C\xF8\xC1\xDB\xE8"
-			  "\xCB\x4D\xD2\x8C\x04\x14\xB4\x66"
-			  "\x41\xB7\x3A\x96\x16\x7C\x1D\x5B"
-			  "\xB6\x41\x42\x64\x43\xEE\x6E\x7C"
-			  "\x8B\xAF\x01\x9C\xA4\x6E\x75\x8F"
-			  "\xDE\x10\x9F\xA6\xE7\xD6\x44\x97"
-			  "\x66\xA3\x96\x0F\x1C\x25\x60\xF5"
-			  "\x3C\x2E\x32\x69\x0E\x82\xFF\x27"
-			  "\x0F\xB5\x06\xDA\xD8\x31\x15\x6C"
-			  "\xDF\x18\x6C\x87\xF5\x3B\x11\x9A"
-			  "\x1B\x42\x1F\x5B\x29\x19\x96\x13"
-			  "\x68\x2E\x5E\x08\x1C\x8F\x32\x4B"
-			  "\x81\x77\x6D\xF4\xA0\x01\x42\xEC"
-			  "\xDD\x5B\xFD\x3A\x8E\x6A\x14\xFB"
-			  "\x83\x54\xDF\x0F\x86\xB7\xEA\x40"
-			  "\x46\x39\xF7\x2A\x89\x8D\x4E\x96"
-			  "\x5F\x5F\x6D\x76\xC6\x13\x9D\x3D"
-			  "\x1D\x5F\x0C\x7D\xE2\xBC\xC2\x16"
-			  "\x16\xBE\x89\x3E\xB0\x61\xA2\x5D"
-			  "\xAF\xD1\x40\x5F\x1A\xB8\x26\x41"
-			  "\xC6\xBD\x36\xEF\xED\x29\x50\x6D"
-			  "\x10\xEF\x26\xE8\xA8\x93\x11\x3F"
-			  "\x2D\x1F\x88\x20\x77\x45\xF5\x66"
-			  "\x08\xB9\xF1\xEF\xB1\x93\xA8\x81"
-			  "\x65\xC5\xCD\x3E\x8C\x06\x60\x2C"
-			  "\xB2\x10\x7A\xCA\x05\x25\x59\xDB"
-			  "\xC7\x28\xF5\x20\x35\x52\x9E\x62"
-			  "\xF8\x88\x24\x1C\x4D\x84\x12\x39"
-			  "\x39\xE4\x2E\xF4\xD4\x9D\x2B\xBC"
-			  "\x87\x66\xE6\xC0\x6B\x31\x9A\x66"
-			  "\x03\xDC\x95\xD8\x6B\xD0\x30\x8F"
-			  "\xDF\x8F\x8D\xFA\xEC\x1F\x08\xBD"
-			  "\xA3\x63\xE2\x71\x4F\x03\x94\x87"
-			  "\x50\xDF\x15\x1F\xED\x3A\xA3\x7F"
-			  "\x1F\x2A\xB5\xA1\x69\xAC\x4B\x0D"
-			  "\x84\x9B\x2A\xE9\x55\xDD\x46\x91"
-			  "\x15\x33\xF3\x2B\x9B\x46\x97\x00"
-			  "\xF0\x29\xD8\x59\x5D\x33\x37\xF9"
-			  "\x58\x33\x9B\x78\xC7\x58\x48\x6B"
-			  "\x2C\x75\x64\xC4\xCA\xC1\x7E\xD5",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -9455,20657 +8039,1063 @@ static const struct cipher_testvec tf_dec_tv_template[] = {
 			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
 			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec tf_cbc_enc_tv_template[] = {
-	{ /* Generated with Nettle */
-		.key	= zeroed_string,
-		.klen	= 16,
-		.iv	= zeroed_string,
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.result	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
-			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a",
-		.rlen	= 16,
-	}, {
-		.key	= zeroed_string,
-		.klen	= 16,
-		.iv	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
-			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a",
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.result	= "\xd4\x91\xdb\x16\xe7\xb1\xc3\x9e"
-			  "\x86\xcb\x08\x6b\x78\x9f\x54\x19",
-		.rlen	= 16,
-	}, {
-		.key	= zeroed_string,
-		.klen	= 16,
-		.iv	= "\xd4\x91\xdb\x16\xe7\xb1\xc3\x9e"
-			  "\x86\xcb\x08\x6b\x78\x9f\x54\x19",
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.result	= "\x05\xef\x8c\x61\xa8\x11\x58\x26"
-			  "\x34\xba\x5c\xb7\x10\x6a\xa6\x41",
-		.rlen	= 16,
-	}, {
-		.key	= zeroed_string,
-		.klen	= 16,
-		.iv	= zeroed_string,
-		.input	= zeroed_string,
-		.ilen	= 48,
-		.result	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
-			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a"
-			  "\xd4\x91\xdb\x16\xe7\xb1\xc3\x9e"
-			  "\x86\xcb\x08\x6b\x78\x9f\x54\x19"
-			  "\x05\xef\x8c\x61\xa8\x11\x58\x26"
-			  "\x34\xba\x5c\xb7\x10\x6a\xa6\x41",
-		.rlen	= 48,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\xC8\xFF\xF2\x53\xA6\x27\x09\xD1"
-			  "\x33\x38\xC2\xC0\x0C\x14\x7E\xB5"
-			  "\x26\x1B\x05\x0C\x05\x12\x3F\xC0"
-			  "\xF9\x1C\x02\x28\x40\x96\x6F\xD0"
-			  "\x3D\x32\xDF\xDA\x56\x00\x6E\xEE"
-			  "\x5B\x2A\x72\x9D\xC2\x4D\x19\xBC"
-			  "\x8C\x53\xFA\x87\x6F\xDD\x81\xA3"
-			  "\xB1\xD3\x44\x65\xDF\xE7\x63\x38"
-			  "\x4A\xFC\xDC\xEC\x3F\x26\x8E\xB8"
-			  "\x43\xFC\xFE\x18\xB5\x11\x6D\x31"
-			  "\x81\x8B\x0D\x75\xF6\x80\xEC\x84"
-			  "\x04\xB9\xE6\x09\x63\xED\x39\xDB"
-			  "\xC3\xF6\x14\xD6\x6E\x5E\x8B\xBD"
-			  "\x3E\xFA\xD7\x98\x50\x6F\xD9\x63"
-			  "\x02\xCD\x0D\x39\x4B\x0D\xEC\x80"
-			  "\xE3\x6A\x17\xF4\xCC\xAD\xFF\x68"
-			  "\x45\xDD\xC8\x83\x1D\x41\x96\x0D"
-			  "\x91\x2E\x05\xD3\x59\x82\xE0\x43"
-			  "\x90\x4F\xB9\xF7\xAD\x6B\x2E\xAF"
-			  "\xA7\x84\x00\x53\xCD\x6F\xD1\x0C"
-			  "\x4E\xF9\x5A\x23\xFB\xCA\xC7\xD3"
-			  "\xA9\xAA\x9D\xB2\x3F\x66\xF1\xAC"
-			  "\x25\x21\x8F\xF7\xEF\xF2\x6A\xDF"
-			  "\xE8\xDA\x75\x1A\x8A\xF1\xDD\x38"
-			  "\x1F\xF9\x3D\x68\x4A\xBB\x9E\x34"
-			  "\x1F\x66\x1F\x9C\x2B\x54\xFF\x60"
-			  "\x7F\x29\x4B\x55\x80\x8F\x4E\xA7"
-			  "\xA6\x9A\x0A\xD9\x0D\x19\x00\xF8"
-			  "\x1F\xBC\x0C\x40\x6B\xEC\x99\x25"
-			  "\x94\x70\x74\x0E\x1D\xC5\xBC\x12"
-			  "\xF3\x42\xBE\x95\xBF\xFB\x4E\x55"
-			  "\x9A\xB9\xCE\x14\x16\x5B\xDC\xD3"
-			  "\x75\x42\x62\x04\x31\x1F\x95\x7C"
-			  "\x66\x1A\x97\xDC\x2F\x40\x5C\x39"
-			  "\x78\xE6\x02\xDB\x49\xE1\xC6\x47"
-			  "\xC2\x78\x9A\xBB\xF3\xBE\xCB\x93"
-			  "\xD8\xB8\xE8\xBB\x8C\xB3\x9B\xA7"
-			  "\xC2\x89\xF3\x91\x88\x83\x3D\xF0"
-			  "\x29\xA2\xCD\xB5\x79\x16\xC2\x40"
-			  "\x11\x03\x8E\x9C\xFD\xC9\x43\xC4"
-			  "\xC2\x19\xF0\x4A\x32\xEF\x0C\x2B"
-			  "\xD3\x2B\xE9\xD4\x4C\xDE\x95\xCF"
-			  "\x04\x03\xD3\x2C\x7F\x82\xC8\xFA"
-			  "\x0F\xD8\x7A\x39\x7B\x01\x41\x9C"
-			  "\x78\xB6\xC9\xBF\xF9\x78\x57\x88"
-			  "\xB1\xA5\xE1\xE0\xD9\x16\xD4\xC8"
-			  "\xEE\xC4\xBE\x7B\x55\x59\x00\x48"
-			  "\x1B\xBC\x14\xFA\x2A\x9D\xC9\x1C"
-			  "\xFB\x28\x3F\x95\xDD\xB7\xD6\xCE"
-			  "\x3A\x7F\x09\x0C\x0E\x69\x30\x7D"
-			  "\xBC\x68\x9C\x91\x2A\x59\x57\x04"
-			  "\xED\x1A\x1E\x00\xB1\x85\x92\x04"
-			  "\x28\x8C\x0C\x3C\xC1\xD5\x12\xF7"
-			  "\x4C\x3E\xB0\xE7\x86\x62\x68\x91"
-			  "\xFC\xC4\xE2\xCE\xA6\xDC\x5E\x93"
-			  "\x5D\x8D\x8C\x68\xB3\xB2\xB9\x64"
-			  "\x16\xB8\xC8\x6F\xD8\xEE\x21\xBD"
-			  "\xAC\x18\x0C\x7D\x0D\x05\xAB\xF1"
-			  "\xFA\xDD\xE2\x48\xDF\x4C\x02\x39"
-			  "\x69\xA1\x62\xBD\x49\x3A\x9D\x91"
-			  "\x30\x70\x56\xA4\x37\xDD\x7C\xC0"
-			  "\x0A\xA3\x30\x10\x26\x25\x41\x2C",
-		.rlen	= 496,
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
+			  "\x2B\xC2\x59",
+		.ctext	= "\xDF\xDD\x69\xFA\xB0\x2E\xFD\xFE"
+			  "\x70\x9E\xC5\x4B\xC9\xD4\xA1\x30"
+			  "\x26\x9B\x89\xA1\xEE\x43\xE0\x52"
+			  "\x55\x17\x4E\xC7\x0E\x33\x1F\xF1"
+			  "\x9F\x8D\x40\x9F\x24\xFD\x92\xA0"
+			  "\xBC\x8F\x35\xDD\x67\x38\xD8\xAA"
+			  "\xCF\xF8\x48\xCA\xFB\xE4\x5C\x60"
+			  "\x01\x41\x21\x12\x38\xAB\x52\x4F"
+			  "\xA8\x57\x20\xE0\x21\x6A\x17\x0D"
+			  "\x0E\xF9\x8E\x49\x42\x00\x3C\x94"
+			  "\x14\xC0\xD0\x8D\x8A\x98\xEB\x29"
+			  "\xEC\xAE\x96\x44\xC0\x3C\x48\xDC"
+			  "\x29\x35\x25\x2F\xE7\x11\x6C\x68"
+			  "\xC8\x67\x0A\x2F\xF4\x07\xBE\xF9"
+			  "\x2C\x31\x87\x40\xAB\xB2\xB6\xFA"
+			  "\xD2\xC9\x6D\x5C\x50\xE9\xE6\x7E"
+			  "\xE3\x0A\xD2\xD5\x6D\x8D\x64\x9E"
+			  "\x70\xCE\x03\x76\xDD\xE0\xF0\x8C"
+			  "\x84\x86\x8B\x6A\xFE\xC7\xF9\x69"
+			  "\x2E\xFE\xFC\xC2\xC4\x1A\x55\x58"
+			  "\xB3\xBE\xE2\x7E\xED\x39\x42\x6C"
+			  "\xB4\x42\x97\x9A\xEC\xE1\x0A\x06"
+			  "\x02\xC5\x03\x9D\xC4\x48\x15\x66"
+			  "\x35\x6A\xC2\xC9\xA2\x26\x30\xBB"
+			  "\xDB\x2D\xC8\x08\x2B\xA0\x29\x1A"
+			  "\x23\x61\x48\xEA\x80\x04\x27\xAA"
+			  "\x69\x49\xE8\xE8\x4A\x83\x6B\x5A"
+			  "\xCA\x7C\xD3\xB1\xB5\x0B\xCC\x23"
+			  "\x74\x1F\xA9\x87\xCD\xED\xC0\x2D"
+			  "\xBF\xEB\xCF\x16\x2D\x2A\x2E\x1D"
+			  "\x96\xBA\x36\x11\x45\x41\xDA\xCE"
+			  "\xA4\x48\x80\x8B\x06\xF4\x98\x89"
+			  "\x8B\x23\x08\x53\xF4\xD4\x5A\x24"
+			  "\x8B\xF8\x43\x73\xD1\xEE\xC4\xB0"
+			  "\xF8\xFE\x09\x0C\x75\x05\x38\x0B"
+			  "\x7C\x81\xDE\x9D\xE4\x61\x37\x63"
+			  "\x63\xAD\x12\xD2\x04\xB9\xCE\x45"
+			  "\x5A\x1A\x6E\xB3\x78\x2A\xA4\x74"
+			  "\x86\xD0\xE3\xFF\xDA\x38\x9C\xB5"
+			  "\xB8\xB1\xDB\x38\x2F\xC5\x6A\xB4"
+			  "\xEB\x6E\x96\xE8\x43\x80\xB5\x51"
+			  "\x61\x2D\x48\xAA\x07\x65\x11\x8C"
+			  "\x48\xE3\x90\x7E\x78\x3A\xEC\x97"
+			  "\x05\x3D\x84\xE7\x90\x2B\xAA\xBD"
+			  "\x83\x29\x0E\x1A\x81\x73\x7B\xE0"
+			  "\x7A\x01\x4A\x37\x3B\x77\x7F\x8D"
+			  "\x49\xA4\x2F\x6E\xBE\x68\x99\x08"
+			  "\x99\xAA\x4C\x12\x04\xAE\x1F\x77"
+			  "\x35\x88\xF1\x65\x06\x0A\x0B\x4D"
+			  "\x47\xF9\x50\x38\x5D\x71\xF9\x6E"
+			  "\xDE\xEC\x61\x35\x2C\x4C\x96\x50"
+			  "\xE8\x28\x93\x9C\x7E\x01\xC6\x04"
+			  "\xB2\xD6\xBC\x6C\x17\xEB\xC1\x7D"
+			  "\x11\xE9\x43\x83\x76\xAA\x53\x37"
+			  "\x0C\x1D\x39\x89\x53\x72\x09\x7E"
+			  "\xD9\x85\x16\x04\xA5\x2C\x05\x6F"
+			  "\x17\x0C\x6E\x66\xAA\x84\xA7\xD9"
+			  "\xE2\xD9\xC4\xEB\x43\x3E\xB1\x8D"
+			  "\x7C\x36\xC7\x71\x70\x9C\x10\xD8"
+			  "\xE8\x47\x2A\x4D\xFD\xA1\xBC\xE3"
+			  "\xB9\x32\xE2\xC1\x82\xAC\xFE\xCC"
+			  "\xC5\xC9\x7F\x9E\xCF\x33\x7A\xDF"
+			  "\x6C\x82\x9D",
+		.len	= 499,
 		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
+		.np	= 2,
+		.tap	= { 499 - 16, 16 },
 	},
 };
 
-static const struct cipher_testvec tf_cbc_dec_tv_template[] = {
-	{ /* Reverse of the first four above */
-		.key	= zeroed_string,
-		.klen	= 16,
-		.iv	= zeroed_string,
-		.input	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
-			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
+static const struct cipher_testvec tf_lrw_tv_template[] = {
+	/* Generated from AES-LRW test vectors */
+	{
+		.key	= "\x45\x62\xac\x25\xf8\x28\x17\x6d"
+			  "\x4c\x26\x84\x14\xb5\x68\x01\x85"
+			  "\x25\x8e\x2a\x05\xe7\x3e\x9d\x03"
+			  "\xee\x5a\x83\x0c\xcc\x09\x4c\x87",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\xa1\x6c\x50\x69\x26\xa4\xef\x7b"
+			  "\x7c\xc6\x91\xeb\x72\xdd\x9b\xee",
+		.len	= 16,
 	}, {
-		.key	= zeroed_string,
-		.klen	= 16,
-		.iv	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
-			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a",
-		.input	= "\xd4\x91\xdb\x16\xe7\xb1\xc3\x9e"
-			  "\x86\xcb\x08\x6b\x78\x9f\x54\x19",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
+		.key	= "\x59\x70\x47\x14\xf5\x57\x47\x8c"
+			  "\xd7\x79\xe8\x0f\x54\x88\x79\x44"
+			  "\x0d\x48\xf0\xb7\xb1\x5a\x53\xea"
+			  "\x1c\xaa\x6b\x29\xc2\xca\xfb\xaf",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x02",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\xab\x72\x0a\xad\x3b\x0c\xf0\xc9"
+			  "\x42\x2f\xf1\xae\xf1\x3c\xb1\xbd",
+		.len	= 16,
 	}, {
-		.key	= zeroed_string,
-		.klen	= 16,
-		.iv	= "\xd4\x91\xdb\x16\xe7\xb1\xc3\x9e"
-			  "\x86\xcb\x08\x6b\x78\x9f\x54\x19",
-		.input	= "\x05\xef\x8c\x61\xa8\x11\x58\x26"
-			  "\x34\xba\x5c\xb7\x10\x6a\xa6\x41",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
+		.key	= "\xd8\x2a\x91\x34\xb2\x6a\x56\x50"
+			  "\x30\xfe\x69\xe2\x37\x7f\x98\x47"
+			  "\xcd\xf9\x0b\x16\x0c\x64\x8f\xb6"
+			  "\xb0\x0d\x0d\x1b\xae\x85\x87\x1f",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x02\x00\x00\x00\x00",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x85\xa7\x56\x67\x08\xfa\x42\xe1"
+			  "\x22\xe6\x82\xfc\xd9\xb4\xd7\xd4",
+		.len	= 16,
 	}, {
-		.key	= zeroed_string,
-		.klen	= 16,
-		.iv	= zeroed_string,
-		.input	= "\x9f\x58\x9f\x5c\xf6\x12\x2c\x32"
-			  "\xb6\xbf\xec\x2f\x2a\xe8\xc3\x5a"
-			  "\xd4\x91\xdb\x16\xe7\xb1\xc3\x9e"
-			  "\x86\xcb\x08\x6b\x78\x9f\x54\x19"
-			  "\x05\xef\x8c\x61\xa8\x11\x58\x26"
-			  "\x34\xba\x5c\xb7\x10\x6a\xa6\x41",
-		.ilen	= 48,
-		.result	= zeroed_string,
-		.rlen	= 48,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.key	= "\x0f\x6a\xef\xf8\xd3\xd2\xbb\x15"
+			  "\x25\x83\xf7\x3c\x1f\x01\x28\x74"
+			  "\xca\xc6\xbc\x35\x4d\x4a\x65\x54"
+			  "\x90\xae\x61\xcf\x7b\xae\xbd\xcc"
+			  "\xad\xe4\x94\xc5\x4a\x29\xae\x70",
+		.klen	= 40,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\xd2\xaf\x69\x35\x24\x1d\x0e\x1c"
+			  "\x84\x8b\x05\xe4\xa2\x2f\x16\xf5",
+		.len	= 16,
+	}, {
+		.key	= "\x8a\xd4\xee\x10\x2f\xbd\x81\xff"
+			  "\xf8\x86\xce\xac\x93\xc5\xad\xc6"
+			  "\xa0\x19\x07\xc0\x9d\xf7\xbb\xdd"
+			  "\x52\x13\xb2\xb7\xf0\xff\x11\xd8"
+			  "\xd6\x08\xd0\xcd\x2e\xb1\x17\x6f",
+		.klen	= 40,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x02\x00\x00\x00\x00",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x4a\x23\x56\xd7\xff\x90\xd0\x9a"
+			  "\x0d\x7c\x26\xfc\xf0\xf0\xf6\xe4",
+		.len	= 16,
+	}, {
+		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
+			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
+			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
+			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
+			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
+			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x30\xaf\x26\x05\x9d\x5d\x0a\x58"
+			  "\xe2\xe7\xce\x8a\xb2\x56\x6d\x76",
+		.len	= 16,
+	}, {
+		.key	= "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
+			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
+			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
+			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
+			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
+			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x02\x00\x00\x00\x00",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\xdf\xcf\xdc\xd2\xe1\xcf\x86\x75"
+			  "\x17\x66\x5e\x0c\x14\xa1\x3d\x40",
+		.len	= 16,
+	}, {
+		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
+			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
+			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
+			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
+			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
+			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
+			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
+			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
+			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
+			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
+			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
+			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
+			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
+			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
+			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
+			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
+			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
+			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
+			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
+			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
+			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
+			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
+			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
+			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
+			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
+			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
+			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
+			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
+			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
+			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
+			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
+			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
+			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
+			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
+			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
+			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
+			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
+			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
+			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
+			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
+			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
+			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
+			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
+			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
+			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
+			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
+			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
+			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
+			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
+			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
+			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
+			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
+			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
+			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
+			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
+			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
+			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
+			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
+			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
+			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
+			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
+			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
+			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
+			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
+			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
+			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
+			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
+			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
+			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
+		.ctext	= "\x30\x38\xeb\xaf\x12\x43\x1a\x89"
+			  "\x62\xa2\x36\xe5\xcf\x77\x1e\xd9"
+			  "\x08\xc3\x0d\xdd\x95\xab\x19\x96"
+			  "\x27\x52\x41\xc3\xca\xfb\xf6\xee"
+			  "\x40\x2d\xdf\xdd\x00\x0c\xb9\x0a"
+			  "\x3a\xf0\xc0\xd1\xda\x63\x9e\x45"
+			  "\x42\xe9\x29\xc0\xb4\x07\xb4\x31"
+			  "\x66\x77\x72\xb5\xb6\xb3\x57\x46"
+			  "\x34\x9a\xfe\x03\xaf\x6b\x36\x07"
+			  "\x63\x8e\xc2\x5d\xa6\x0f\xb6\x7d"
+			  "\xfb\x6d\x82\x51\xb6\x98\xd0\x71"
+			  "\xe7\x10\x7a\xdf\xb2\xbd\xf1\x1d"
+			  "\x72\x2b\x54\x13\xe3\x6d\x79\x37"
+			  "\xa9\x39\x2c\xdf\x21\xab\x87\xd5"
+			  "\xee\xef\x9a\x12\x50\x39\x2e\x1b"
+			  "\x7d\xe6\x6a\x27\x48\xb9\xe7\xac"
+			  "\xaa\xcd\x79\x5f\xf2\xf3\xa0\x08"
+			  "\x6f\x2c\xf4\x0e\xd1\xb8\x89\x25"
+			  "\x31\x9d\xef\xb1\x1d\x27\x55\x04"
+			  "\xc9\x8c\xb7\x68\xdc\xb6\x67\x8a"
+			  "\xdb\xcf\x22\xf2\x3b\x6f\xce\xbb"
+			  "\x26\xbe\x4f\x27\x04\x42\xd1\x44"
+			  "\x4c\x08\xa3\x95\x4c\x7f\x1a\xaf"
+			  "\x1d\x28\x14\xfd\xb1\x1a\x34\x18"
+			  "\xf5\x1e\x28\x69\x95\x6a\x5a\xba"
+			  "\x8e\xb2\x58\x1d\x28\x17\x13\x3d"
+			  "\x38\x7d\x14\x8d\xab\x5d\xf9\xe8"
+			  "\x3c\x0f\x2b\x0d\x2b\x08\xb4\x4b"
+			  "\x6b\x0d\xc8\xa7\x84\xc2\x3a\x1a"
+			  "\xb7\xbd\xda\x92\x29\xb8\x5b\x5a"
+			  "\x63\xa5\x99\x82\x09\x72\x8f\xc6"
+			  "\xa4\x62\x24\x69\x8c\x2d\x26\x00"
+			  "\x99\x83\x91\xd6\xc6\xcf\x57\x67"
+			  "\x38\xea\xf2\xfc\x29\xe0\x73\x39"
+			  "\xf9\x13\x94\x6d\xe2\x58\x28\x75"
+			  "\x3e\xae\x71\x90\x07\x70\x1c\x38"
+			  "\x5b\x4c\x1e\xb5\xa5\x3b\x20\xef"
+			  "\xb1\x4c\x3e\x1a\x72\x62\xbb\x22"
+			  "\x82\x09\xe3\x18\x3f\x4f\x48\xfc"
+			  "\xdd\xac\xfc\xb6\x09\xdb\xd2\x7b"
+			  "\xd6\xb7\x7e\x41\x2f\x14\xf5\x0e"
+			  "\xc3\xac\x4a\xed\xe7\x82\xef\x31"
+			  "\x1f\x1a\x51\x1e\x29\x60\xc8\x98"
+			  "\x93\x51\x1d\x3d\x62\x59\x83\x82"
+			  "\x0c\xf1\xd7\x8d\xac\x33\x44\x81"
+			  "\x3c\x59\xb7\xd4\x5b\x65\x82\xc4"
+			  "\xec\xdc\x24\xfd\x0e\x1a\x79\x94"
+			  "\x34\xb0\x62\xfa\x98\x49\x26\x1f"
+			  "\xf4\x9e\x40\x44\x5b\x1f\xf8\xbe"
+			  "\x36\xff\xc6\xc6\x9d\xf2\xd6\xcc"
+			  "\x63\x93\x29\xb9\x0b\x6d\xd7\x6c"
+			  "\xdb\xf6\x21\x80\xf7\x5a\x37\x15"
+			  "\x0c\xe3\x36\xc8\x74\x75\x20\x91"
+			  "\xdf\x52\x2d\x0c\xe7\x45\xff\x46"
+			  "\xb3\xf4\xec\xc2\xbd\xd3\x37\xb6"
+			  "\x26\xa2\x5d\x7d\x61\xbf\x10\x46"
+			  "\x57\x8d\x05\x96\x70\x0b\xd6\x41"
+			  "\x5c\xe9\xd3\x54\x81\x39\x3a\xdd"
+			  "\x5f\x92\x81\x6e\x35\x03\xd4\x72"
+			  "\x3d\x5a\xe7\xb9\x3b\x0c\x84\x23"
+			  "\x45\x5d\xec\x72\xc1\x52\xef\x2e"
+			  "\x81\x00\xd3\xfe\x4c\x3c\x05\x61"
+			  "\x80\x18\xc4\x6c\x03\xd3\xb7\xba"
+			  "\x11\xd7\xb8\x6e\xea\xe1\x80\x30",
+		.len	= 512,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 512 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec tf_xts_tv_template[] = {
+	/* Generated from AES-XTS test vectors */
+{
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
 		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\xC8\xFF\xF2\x53\xA6\x27\x09\xD1"
-			  "\x33\x38\xC2\xC0\x0C\x14\x7E\xB5"
-			  "\x26\x1B\x05\x0C\x05\x12\x3F\xC0"
-			  "\xF9\x1C\x02\x28\x40\x96\x6F\xD0"
-			  "\x3D\x32\xDF\xDA\x56\x00\x6E\xEE"
-			  "\x5B\x2A\x72\x9D\xC2\x4D\x19\xBC"
-			  "\x8C\x53\xFA\x87\x6F\xDD\x81\xA3"
-			  "\xB1\xD3\x44\x65\xDF\xE7\x63\x38"
-			  "\x4A\xFC\xDC\xEC\x3F\x26\x8E\xB8"
-			  "\x43\xFC\xFE\x18\xB5\x11\x6D\x31"
-			  "\x81\x8B\x0D\x75\xF6\x80\xEC\x84"
-			  "\x04\xB9\xE6\x09\x63\xED\x39\xDB"
-			  "\xC3\xF6\x14\xD6\x6E\x5E\x8B\xBD"
-			  "\x3E\xFA\xD7\x98\x50\x6F\xD9\x63"
-			  "\x02\xCD\x0D\x39\x4B\x0D\xEC\x80"
-			  "\xE3\x6A\x17\xF4\xCC\xAD\xFF\x68"
-			  "\x45\xDD\xC8\x83\x1D\x41\x96\x0D"
-			  "\x91\x2E\x05\xD3\x59\x82\xE0\x43"
-			  "\x90\x4F\xB9\xF7\xAD\x6B\x2E\xAF"
-			  "\xA7\x84\x00\x53\xCD\x6F\xD1\x0C"
-			  "\x4E\xF9\x5A\x23\xFB\xCA\xC7\xD3"
-			  "\xA9\xAA\x9D\xB2\x3F\x66\xF1\xAC"
-			  "\x25\x21\x8F\xF7\xEF\xF2\x6A\xDF"
-			  "\xE8\xDA\x75\x1A\x8A\xF1\xDD\x38"
-			  "\x1F\xF9\x3D\x68\x4A\xBB\x9E\x34"
-			  "\x1F\x66\x1F\x9C\x2B\x54\xFF\x60"
-			  "\x7F\x29\x4B\x55\x80\x8F\x4E\xA7"
-			  "\xA6\x9A\x0A\xD9\x0D\x19\x00\xF8"
-			  "\x1F\xBC\x0C\x40\x6B\xEC\x99\x25"
-			  "\x94\x70\x74\x0E\x1D\xC5\xBC\x12"
-			  "\xF3\x42\xBE\x95\xBF\xFB\x4E\x55"
-			  "\x9A\xB9\xCE\x14\x16\x5B\xDC\xD3"
-			  "\x75\x42\x62\x04\x31\x1F\x95\x7C"
-			  "\x66\x1A\x97\xDC\x2F\x40\x5C\x39"
-			  "\x78\xE6\x02\xDB\x49\xE1\xC6\x47"
-			  "\xC2\x78\x9A\xBB\xF3\xBE\xCB\x93"
-			  "\xD8\xB8\xE8\xBB\x8C\xB3\x9B\xA7"
-			  "\xC2\x89\xF3\x91\x88\x83\x3D\xF0"
-			  "\x29\xA2\xCD\xB5\x79\x16\xC2\x40"
-			  "\x11\x03\x8E\x9C\xFD\xC9\x43\xC4"
-			  "\xC2\x19\xF0\x4A\x32\xEF\x0C\x2B"
-			  "\xD3\x2B\xE9\xD4\x4C\xDE\x95\xCF"
-			  "\x04\x03\xD3\x2C\x7F\x82\xC8\xFA"
-			  "\x0F\xD8\x7A\x39\x7B\x01\x41\x9C"
-			  "\x78\xB6\xC9\xBF\xF9\x78\x57\x88"
-			  "\xB1\xA5\xE1\xE0\xD9\x16\xD4\xC8"
-			  "\xEE\xC4\xBE\x7B\x55\x59\x00\x48"
-			  "\x1B\xBC\x14\xFA\x2A\x9D\xC9\x1C"
-			  "\xFB\x28\x3F\x95\xDD\xB7\xD6\xCE"
-			  "\x3A\x7F\x09\x0C\x0E\x69\x30\x7D"
-			  "\xBC\x68\x9C\x91\x2A\x59\x57\x04"
-			  "\xED\x1A\x1E\x00\xB1\x85\x92\x04"
-			  "\x28\x8C\x0C\x3C\xC1\xD5\x12\xF7"
-			  "\x4C\x3E\xB0\xE7\x86\x62\x68\x91"
-			  "\xFC\xC4\xE2\xCE\xA6\xDC\x5E\x93"
-			  "\x5D\x8D\x8C\x68\xB3\xB2\xB9\x64"
-			  "\x16\xB8\xC8\x6F\xD8\xEE\x21\xBD"
-			  "\xAC\x18\x0C\x7D\x0D\x05\xAB\xF1"
-			  "\xFA\xDD\xE2\x48\xDF\x4C\x02\x39"
-			  "\x69\xA1\x62\xBD\x49\x3A\x9D\x91"
-			  "\x30\x70\x56\xA4\x37\xDD\x7C\xC0"
-			  "\x0A\xA3\x30\x10\x26\x25\x41\x2C",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec tf_ctr_enc_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\x4b\xc9\x44\x4a\x11\xa3\xef\xac"
+			  "\x30\x74\xe4\x44\x52\x77\x97\x43"
+			  "\xa7\x60\xb2\x45\x2e\xf9\x00\x90"
+			  "\x9f\xaa\xfd\x89\x6e\x9d\x4a\xe0",
+		.len	= 32,
+	}, {
+		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
+			  "\x11\x11\x11\x11\x11\x11\x11\x11"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22",
 		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\xDF\xDD\x69\xFA\xB0\x2E\xFD\xFE"
-			  "\x70\x9E\xC5\x4B\xC9\xD4\xA1\x30"
-			  "\x26\x9B\x89\xA1\xEE\x43\xE0\x52"
-			  "\x55\x17\x4E\xC7\x0E\x33\x1F\xF1"
-			  "\x9F\x8D\x40\x9F\x24\xFD\x92\xA0"
-			  "\xBC\x8F\x35\xDD\x67\x38\xD8\xAA"
-			  "\xCF\xF8\x48\xCA\xFB\xE4\x5C\x60"
-			  "\x01\x41\x21\x12\x38\xAB\x52\x4F"
-			  "\xA8\x57\x20\xE0\x21\x6A\x17\x0D"
-			  "\x0E\xF9\x8E\x49\x42\x00\x3C\x94"
-			  "\x14\xC0\xD0\x8D\x8A\x98\xEB\x29"
-			  "\xEC\xAE\x96\x44\xC0\x3C\x48\xDC"
-			  "\x29\x35\x25\x2F\xE7\x11\x6C\x68"
-			  "\xC8\x67\x0A\x2F\xF4\x07\xBE\xF9"
-			  "\x2C\x31\x87\x40\xAB\xB2\xB6\xFA"
-			  "\xD2\xC9\x6D\x5C\x50\xE9\xE6\x7E"
-			  "\xE3\x0A\xD2\xD5\x6D\x8D\x64\x9E"
-			  "\x70\xCE\x03\x76\xDD\xE0\xF0\x8C"
-			  "\x84\x86\x8B\x6A\xFE\xC7\xF9\x69"
-			  "\x2E\xFE\xFC\xC2\xC4\x1A\x55\x58"
-			  "\xB3\xBE\xE2\x7E\xED\x39\x42\x6C"
-			  "\xB4\x42\x97\x9A\xEC\xE1\x0A\x06"
-			  "\x02\xC5\x03\x9D\xC4\x48\x15\x66"
-			  "\x35\x6A\xC2\xC9\xA2\x26\x30\xBB"
-			  "\xDB\x2D\xC8\x08\x2B\xA0\x29\x1A"
-			  "\x23\x61\x48\xEA\x80\x04\x27\xAA"
-			  "\x69\x49\xE8\xE8\x4A\x83\x6B\x5A"
-			  "\xCA\x7C\xD3\xB1\xB5\x0B\xCC\x23"
-			  "\x74\x1F\xA9\x87\xCD\xED\xC0\x2D"
-			  "\xBF\xEB\xCF\x16\x2D\x2A\x2E\x1D"
-			  "\x96\xBA\x36\x11\x45\x41\xDA\xCE"
-			  "\xA4\x48\x80\x8B\x06\xF4\x98\x89"
-			  "\x8B\x23\x08\x53\xF4\xD4\x5A\x24"
-			  "\x8B\xF8\x43\x73\xD1\xEE\xC4\xB0"
-			  "\xF8\xFE\x09\x0C\x75\x05\x38\x0B"
-			  "\x7C\x81\xDE\x9D\xE4\x61\x37\x63"
-			  "\x63\xAD\x12\xD2\x04\xB9\xCE\x45"
-			  "\x5A\x1A\x6E\xB3\x78\x2A\xA4\x74"
-			  "\x86\xD0\xE3\xFF\xDA\x38\x9C\xB5"
-			  "\xB8\xB1\xDB\x38\x2F\xC5\x6A\xB4"
-			  "\xEB\x6E\x96\xE8\x43\x80\xB5\x51"
-			  "\x61\x2D\x48\xAA\x07\x65\x11\x8C"
-			  "\x48\xE3\x90\x7E\x78\x3A\xEC\x97"
-			  "\x05\x3D\x84\xE7\x90\x2B\xAA\xBD"
-			  "\x83\x29\x0E\x1A\x81\x73\x7B\xE0"
-			  "\x7A\x01\x4A\x37\x3B\x77\x7F\x8D"
-			  "\x49\xA4\x2F\x6E\xBE\x68\x99\x08"
-			  "\x99\xAA\x4C\x12\x04\xAE\x1F\x77"
-			  "\x35\x88\xF1\x65\x06\x0A\x0B\x4D"
-			  "\x47\xF9\x50\x38\x5D\x71\xF9\x6E"
-			  "\xDE\xEC\x61\x35\x2C\x4C\x96\x50"
-			  "\xE8\x28\x93\x9C\x7E\x01\xC6\x04"
-			  "\xB2\xD6\xBC\x6C\x17\xEB\xC1\x7D"
-			  "\x11\xE9\x43\x83\x76\xAA\x53\x37"
-			  "\x0C\x1D\x39\x89\x53\x72\x09\x7E"
-			  "\xD9\x85\x16\x04\xA5\x2C\x05\x6F"
-			  "\x17\x0C\x6E\x66\xAA\x84\xA7\xD9"
-			  "\xE2\xD9\xC4\xEB\x43\x3E\xB1\x8D"
-			  "\x7C\x36\xC7\x71\x70\x9C\x10\xD8"
-			  "\xE8\x47\x2A\x4D\xFD\xA1\xBC\xE3"
-			  "\xB9\x32\xE2\xC1\x82\xAC\xFE\xCC"
-			  "\xC5\xC9\x7F\x9E\xCF\x33\x7A\xDF",
-		.rlen	= 496,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44",
+		.ctext	= "\x57\x0e\x8f\xe5\x2a\x35\x61\x4f"
+			  "\x32\xd3\xbd\x36\x05\x15\x44\x2c"
+			  "\x58\x06\xf7\xf8\x00\xa8\xb6\xd5"
+			  "\xc6\x28\x92\xdb\xd8\x34\xa2\xe9",
+		.len	= 32,
+	}, {
+		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
+			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22",
 		.klen	= 32,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\xEB\x44\xAF\x49\x27\xB8\xFB\x44"
-			  "\x4C\xA6\xC3\x0C\x8B\xD0\x01\x0C"
-			  "\x53\xC8\x16\x38\xDE\x40\x4F\x91"
-			  "\x25\x6D\x4C\xA0\x9A\x87\x1E\xDA"
-			  "\x88\x7E\x89\xE9\x67\x2B\x83\xA2"
-			  "\x5F\x2E\x23\x3E\x45\xB9\x77\x7B"
-			  "\xA6\x7E\x47\x36\x81\x9F\x9B\xF3"
-			  "\xE0\xF0\xD7\x47\xA9\xC8\xEF\x33"
-			  "\x0C\x43\xFE\x67\x50\x0A\x2C\x3E"
-			  "\xA0\xE1\x25\x8E\x80\x07\x4A\xC0"
-			  "\x64\x89\x9F\x6A\x27\x96\x07\xA6"
-			  "\x9B\xC8\x1B\x21\x60\xAE\x5D\x01"
-			  "\xE2\xCD\xC8\xAA\x6C\x9D\x1C\x34"
-			  "\x39\x18\x09\xA4\x82\x59\x78\xE7"
-			  "\xFC\x59\x65\xF2\x94\xFF\xFB\xE2"
-			  "\x3C\xDA\xB1\x90\x95\xBF\x91\xE3"
-			  "\xE6\x87\x31\x9E\x16\x85\xAD\xB1"
-			  "\x4C\xAE\x43\x4D\x19\x58\xB5\x5E"
-			  "\x2E\xF5\x09\xAA\x39\xF4\xC0\xB3"
-			  "\xD4\x4D\xDB\x73\x7A\xD4\xF1\xBF"
-			  "\x89\x16\x4D\x2D\xA2\x26\x33\x72"
-			  "\x18\x33\x7E\xD6\xD2\x16\xA4\x54"
-			  "\xF4\x8C\xB3\x52\xDF\x21\x9C\xEB"
-			  "\xBF\x49\xD3\xF9\x05\x06\xCB\xD2"
-			  "\xA9\xD2\x3B\x6E\x19\x8C\xBC\x19"
-			  "\xAB\x89\xD6\xD8\xCD\x56\x89\x5E"
-			  "\xAC\x00\xE3\x50\x63\x4A\x80\x9A"
-			  "\x05\xBC\x50\x39\xD3\x32\xD9\x0D"
-			  "\xE3\x20\x0D\x75\x54\xEC\xE6\x31"
-			  "\x14\xB9\x3A\x59\x00\x43\x37\x8E"
-			  "\x8C\x5A\x79\x62\x14\x76\x8A\xAE"
-			  "\x8F\xCC\xA1\x6C\x38\x78\xDD\x2D"
-			  "\x8B\x6D\xEA\xBD\x7B\x25\xFF\x60"
-			  "\xC9\x87\xB1\x79\x1E\xA5\x86\x68"
-			  "\x81\xB4\xE2\xC1\x05\x7D\x3A\x73"
-			  "\xD0\xDA\x75\x77\x9E\x05\x27\xF1"
-			  "\x08\xA9\x66\x64\x6C\xBC\x82\x17"
-			  "\x2C\x23\x5F\x62\x4D\x02\x1A\x58"
-			  "\xE7\xB7\x23\x6D\xE2\x20\xDA\xEF"
-			  "\xB4\xB3\x3F\xB2\x2B\x69\x98\x83"
-			  "\x95\x87\x13\x57\x60\xD7\xB5\xB1"
-			  "\xEE\x0A\x2F\x95\x36\x4C\x76\x5D"
-			  "\x5F\xD9\x19\xED\xB9\xA5\x48\xBF"
-			  "\xC8\xAB\x0F\x71\xCC\x61\x8E\x0A"
-			  "\xD0\x29\x44\xA8\xB9\xC1\xE8\xC8"
-			  "\xC9\xA8\x28\x81\xFB\x50\xF2\xF0"
-			  "\x26\xAE\x39\xB8\x91\xCD\xA8\xAC"
-			  "\xDE\x55\x1B\x50\x14\x53\x44\x17"
-			  "\x54\x46\xFC\xB1\xE4\x07\x6B\x9A"
-			  "\x01\x14\xF0\x2E\x2E\xDB\x46\x1B"
-			  "\x1A\x09\x97\xA9\xB6\x97\x79\x06"
-			  "\xFB\xCB\x85\xCF\xDD\xA1\x41\xB1"
-			  "\x00\xAA\xF7\xE0\x89\x73\xFB\xE5"
-			  "\xBF\x84\xDB\xC9\xCD\xC4\xA2\x0D"
-			  "\x3B\xAC\xF9\xDF\x96\xBF\x88\x23"
-			  "\x41\x67\xA1\x24\x99\x7E\xCC\x9B"
-			  "\x02\x8F\x6A\x49\xF6\x25\xBA\x7A"
-			  "\xF4\x78\xFD\x79\x62\x63\x4F\x14"
-			  "\xD6\x11\x11\x04\x05\x5F\x7E\xEA"
-			  "\x4C\xB6\xF8\xF4\x5F\x48\x52\x54"
-			  "\x94\x63\xA8\x4E\xCF\xD2\x1B\x1B"
-			  "\x22\x18\x6A\xAF\x6E\x3E\xE1\x0D",
-		.rlen	= 496,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59",
-		.ilen	= 499,
-		.result	= "\xDF\xDD\x69\xFA\xB0\x2E\xFD\xFE"
-			  "\x70\x9E\xC5\x4B\xC9\xD4\xA1\x30"
-			  "\x26\x9B\x89\xA1\xEE\x43\xE0\x52"
-			  "\x55\x17\x4E\xC7\x0E\x33\x1F\xF1"
-			  "\x9F\x8D\x40\x9F\x24\xFD\x92\xA0"
-			  "\xBC\x8F\x35\xDD\x67\x38\xD8\xAA"
-			  "\xCF\xF8\x48\xCA\xFB\xE4\x5C\x60"
-			  "\x01\x41\x21\x12\x38\xAB\x52\x4F"
-			  "\xA8\x57\x20\xE0\x21\x6A\x17\x0D"
-			  "\x0E\xF9\x8E\x49\x42\x00\x3C\x94"
-			  "\x14\xC0\xD0\x8D\x8A\x98\xEB\x29"
-			  "\xEC\xAE\x96\x44\xC0\x3C\x48\xDC"
-			  "\x29\x35\x25\x2F\xE7\x11\x6C\x68"
-			  "\xC8\x67\x0A\x2F\xF4\x07\xBE\xF9"
-			  "\x2C\x31\x87\x40\xAB\xB2\xB6\xFA"
-			  "\xD2\xC9\x6D\x5C\x50\xE9\xE6\x7E"
-			  "\xE3\x0A\xD2\xD5\x6D\x8D\x64\x9E"
-			  "\x70\xCE\x03\x76\xDD\xE0\xF0\x8C"
-			  "\x84\x86\x8B\x6A\xFE\xC7\xF9\x69"
-			  "\x2E\xFE\xFC\xC2\xC4\x1A\x55\x58"
-			  "\xB3\xBE\xE2\x7E\xED\x39\x42\x6C"
-			  "\xB4\x42\x97\x9A\xEC\xE1\x0A\x06"
-			  "\x02\xC5\x03\x9D\xC4\x48\x15\x66"
-			  "\x35\x6A\xC2\xC9\xA2\x26\x30\xBB"
-			  "\xDB\x2D\xC8\x08\x2B\xA0\x29\x1A"
-			  "\x23\x61\x48\xEA\x80\x04\x27\xAA"
-			  "\x69\x49\xE8\xE8\x4A\x83\x6B\x5A"
-			  "\xCA\x7C\xD3\xB1\xB5\x0B\xCC\x23"
-			  "\x74\x1F\xA9\x87\xCD\xED\xC0\x2D"
-			  "\xBF\xEB\xCF\x16\x2D\x2A\x2E\x1D"
-			  "\x96\xBA\x36\x11\x45\x41\xDA\xCE"
-			  "\xA4\x48\x80\x8B\x06\xF4\x98\x89"
-			  "\x8B\x23\x08\x53\xF4\xD4\x5A\x24"
-			  "\x8B\xF8\x43\x73\xD1\xEE\xC4\xB0"
-			  "\xF8\xFE\x09\x0C\x75\x05\x38\x0B"
-			  "\x7C\x81\xDE\x9D\xE4\x61\x37\x63"
-			  "\x63\xAD\x12\xD2\x04\xB9\xCE\x45"
-			  "\x5A\x1A\x6E\xB3\x78\x2A\xA4\x74"
-			  "\x86\xD0\xE3\xFF\xDA\x38\x9C\xB5"
-			  "\xB8\xB1\xDB\x38\x2F\xC5\x6A\xB4"
-			  "\xEB\x6E\x96\xE8\x43\x80\xB5\x51"
-			  "\x61\x2D\x48\xAA\x07\x65\x11\x8C"
-			  "\x48\xE3\x90\x7E\x78\x3A\xEC\x97"
-			  "\x05\x3D\x84\xE7\x90\x2B\xAA\xBD"
-			  "\x83\x29\x0E\x1A\x81\x73\x7B\xE0"
-			  "\x7A\x01\x4A\x37\x3B\x77\x7F\x8D"
-			  "\x49\xA4\x2F\x6E\xBE\x68\x99\x08"
-			  "\x99\xAA\x4C\x12\x04\xAE\x1F\x77"
-			  "\x35\x88\xF1\x65\x06\x0A\x0B\x4D"
-			  "\x47\xF9\x50\x38\x5D\x71\xF9\x6E"
-			  "\xDE\xEC\x61\x35\x2C\x4C\x96\x50"
-			  "\xE8\x28\x93\x9C\x7E\x01\xC6\x04"
-			  "\xB2\xD6\xBC\x6C\x17\xEB\xC1\x7D"
-			  "\x11\xE9\x43\x83\x76\xAA\x53\x37"
-			  "\x0C\x1D\x39\x89\x53\x72\x09\x7E"
-			  "\xD9\x85\x16\x04\xA5\x2C\x05\x6F"
-			  "\x17\x0C\x6E\x66\xAA\x84\xA7\xD9"
-			  "\xE2\xD9\xC4\xEB\x43\x3E\xB1\x8D"
-			  "\x7C\x36\xC7\x71\x70\x9C\x10\xD8"
-			  "\xE8\x47\x2A\x4D\xFD\xA1\xBC\xE3"
-			  "\xB9\x32\xE2\xC1\x82\xAC\xFE\xCC"
-			  "\xC5\xC9\x7F\x9E\xCF\x33\x7A\xDF"
-			  "\x6C\x82\x9D",
-		.rlen	= 499,
-		.also_non_np = 1,
-		.np	= 2,
-		.tap	= { 499 - 16, 16 },
-	},
-};
-
-static const struct cipher_testvec tf_ctr_dec_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44",
+		.ctext	= "\x96\x45\x8f\x8d\x7a\x75\xb1\xde"
+			  "\x40\x0c\x89\x56\xf6\x4d\xa7\x07"
+			  "\x38\xbb\x5b\xe9\xcd\x84\xae\xb2"
+			  "\x7b\x6a\x62\xf4\x8c\xb5\x37\xea",
+		.len	= 32,
+	}, {
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95",
 		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\xDF\xDD\x69\xFA\xB0\x2E\xFD\xFE"
-			  "\x70\x9E\xC5\x4B\xC9\xD4\xA1\x30"
-			  "\x26\x9B\x89\xA1\xEE\x43\xE0\x52"
-			  "\x55\x17\x4E\xC7\x0E\x33\x1F\xF1"
-			  "\x9F\x8D\x40\x9F\x24\xFD\x92\xA0"
-			  "\xBC\x8F\x35\xDD\x67\x38\xD8\xAA"
-			  "\xCF\xF8\x48\xCA\xFB\xE4\x5C\x60"
-			  "\x01\x41\x21\x12\x38\xAB\x52\x4F"
-			  "\xA8\x57\x20\xE0\x21\x6A\x17\x0D"
-			  "\x0E\xF9\x8E\x49\x42\x00\x3C\x94"
-			  "\x14\xC0\xD0\x8D\x8A\x98\xEB\x29"
-			  "\xEC\xAE\x96\x44\xC0\x3C\x48\xDC"
-			  "\x29\x35\x25\x2F\xE7\x11\x6C\x68"
-			  "\xC8\x67\x0A\x2F\xF4\x07\xBE\xF9"
-			  "\x2C\x31\x87\x40\xAB\xB2\xB6\xFA"
-			  "\xD2\xC9\x6D\x5C\x50\xE9\xE6\x7E"
-			  "\xE3\x0A\xD2\xD5\x6D\x8D\x64\x9E"
-			  "\x70\xCE\x03\x76\xDD\xE0\xF0\x8C"
-			  "\x84\x86\x8B\x6A\xFE\xC7\xF9\x69"
-			  "\x2E\xFE\xFC\xC2\xC4\x1A\x55\x58"
-			  "\xB3\xBE\xE2\x7E\xED\x39\x42\x6C"
-			  "\xB4\x42\x97\x9A\xEC\xE1\x0A\x06"
-			  "\x02\xC5\x03\x9D\xC4\x48\x15\x66"
-			  "\x35\x6A\xC2\xC9\xA2\x26\x30\xBB"
-			  "\xDB\x2D\xC8\x08\x2B\xA0\x29\x1A"
-			  "\x23\x61\x48\xEA\x80\x04\x27\xAA"
-			  "\x69\x49\xE8\xE8\x4A\x83\x6B\x5A"
-			  "\xCA\x7C\xD3\xB1\xB5\x0B\xCC\x23"
-			  "\x74\x1F\xA9\x87\xCD\xED\xC0\x2D"
-			  "\xBF\xEB\xCF\x16\x2D\x2A\x2E\x1D"
-			  "\x96\xBA\x36\x11\x45\x41\xDA\xCE"
-			  "\xA4\x48\x80\x8B\x06\xF4\x98\x89"
-			  "\x8B\x23\x08\x53\xF4\xD4\x5A\x24"
-			  "\x8B\xF8\x43\x73\xD1\xEE\xC4\xB0"
-			  "\xF8\xFE\x09\x0C\x75\x05\x38\x0B"
-			  "\x7C\x81\xDE\x9D\xE4\x61\x37\x63"
-			  "\x63\xAD\x12\xD2\x04\xB9\xCE\x45"
-			  "\x5A\x1A\x6E\xB3\x78\x2A\xA4\x74"
-			  "\x86\xD0\xE3\xFF\xDA\x38\x9C\xB5"
-			  "\xB8\xB1\xDB\x38\x2F\xC5\x6A\xB4"
-			  "\xEB\x6E\x96\xE8\x43\x80\xB5\x51"
-			  "\x61\x2D\x48\xAA\x07\x65\x11\x8C"
-			  "\x48\xE3\x90\x7E\x78\x3A\xEC\x97"
-			  "\x05\x3D\x84\xE7\x90\x2B\xAA\xBD"
-			  "\x83\x29\x0E\x1A\x81\x73\x7B\xE0"
-			  "\x7A\x01\x4A\x37\x3B\x77\x7F\x8D"
-			  "\x49\xA4\x2F\x6E\xBE\x68\x99\x08"
-			  "\x99\xAA\x4C\x12\x04\xAE\x1F\x77"
-			  "\x35\x88\xF1\x65\x06\x0A\x0B\x4D"
-			  "\x47\xF9\x50\x38\x5D\x71\xF9\x6E"
-			  "\xDE\xEC\x61\x35\x2C\x4C\x96\x50"
-			  "\xE8\x28\x93\x9C\x7E\x01\xC6\x04"
-			  "\xB2\xD6\xBC\x6C\x17\xEB\xC1\x7D"
-			  "\x11\xE9\x43\x83\x76\xAA\x53\x37"
-			  "\x0C\x1D\x39\x89\x53\x72\x09\x7E"
-			  "\xD9\x85\x16\x04\xA5\x2C\x05\x6F"
-			  "\x17\x0C\x6E\x66\xAA\x84\xA7\xD9"
-			  "\xE2\xD9\xC4\xEB\x43\x3E\xB1\x8D"
-			  "\x7C\x36\xC7\x71\x70\x9C\x10\xD8"
-			  "\xE8\x47\x2A\x4D\xFD\xA1\xBC\xE3"
-			  "\xB9\x32\xE2\xC1\x82\xAC\xFE\xCC"
-			  "\xC5\xC9\x7F\x9E\xCF\x33\x7A\xDF",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\xEB\x44\xAF\x49\x27\xB8\xFB\x44"
-			  "\x4C\xA6\xC3\x0C\x8B\xD0\x01\x0C"
-			  "\x53\xC8\x16\x38\xDE\x40\x4F\x91"
-			  "\x25\x6D\x4C\xA0\x9A\x87\x1E\xDA"
-			  "\x88\x7E\x89\xE9\x67\x2B\x83\xA2"
-			  "\x5F\x2E\x23\x3E\x45\xB9\x77\x7B"
-			  "\xA6\x7E\x47\x36\x81\x9F\x9B\xF3"
-			  "\xE0\xF0\xD7\x47\xA9\xC8\xEF\x33"
-			  "\x0C\x43\xFE\x67\x50\x0A\x2C\x3E"
-			  "\xA0\xE1\x25\x8E\x80\x07\x4A\xC0"
-			  "\x64\x89\x9F\x6A\x27\x96\x07\xA6"
-			  "\x9B\xC8\x1B\x21\x60\xAE\x5D\x01"
-			  "\xE2\xCD\xC8\xAA\x6C\x9D\x1C\x34"
-			  "\x39\x18\x09\xA4\x82\x59\x78\xE7"
-			  "\xFC\x59\x65\xF2\x94\xFF\xFB\xE2"
-			  "\x3C\xDA\xB1\x90\x95\xBF\x91\xE3"
-			  "\xE6\x87\x31\x9E\x16\x85\xAD\xB1"
-			  "\x4C\xAE\x43\x4D\x19\x58\xB5\x5E"
-			  "\x2E\xF5\x09\xAA\x39\xF4\xC0\xB3"
-			  "\xD4\x4D\xDB\x73\x7A\xD4\xF1\xBF"
-			  "\x89\x16\x4D\x2D\xA2\x26\x33\x72"
-			  "\x18\x33\x7E\xD6\xD2\x16\xA4\x54"
-			  "\xF4\x8C\xB3\x52\xDF\x21\x9C\xEB"
-			  "\xBF\x49\xD3\xF9\x05\x06\xCB\xD2"
-			  "\xA9\xD2\x3B\x6E\x19\x8C\xBC\x19"
-			  "\xAB\x89\xD6\xD8\xCD\x56\x89\x5E"
-			  "\xAC\x00\xE3\x50\x63\x4A\x80\x9A"
-			  "\x05\xBC\x50\x39\xD3\x32\xD9\x0D"
-			  "\xE3\x20\x0D\x75\x54\xEC\xE6\x31"
-			  "\x14\xB9\x3A\x59\x00\x43\x37\x8E"
-			  "\x8C\x5A\x79\x62\x14\x76\x8A\xAE"
-			  "\x8F\xCC\xA1\x6C\x38\x78\xDD\x2D"
-			  "\x8B\x6D\xEA\xBD\x7B\x25\xFF\x60"
-			  "\xC9\x87\xB1\x79\x1E\xA5\x86\x68"
-			  "\x81\xB4\xE2\xC1\x05\x7D\x3A\x73"
-			  "\xD0\xDA\x75\x77\x9E\x05\x27\xF1"
-			  "\x08\xA9\x66\x64\x6C\xBC\x82\x17"
-			  "\x2C\x23\x5F\x62\x4D\x02\x1A\x58"
-			  "\xE7\xB7\x23\x6D\xE2\x20\xDA\xEF"
-			  "\xB4\xB3\x3F\xB2\x2B\x69\x98\x83"
-			  "\x95\x87\x13\x57\x60\xD7\xB5\xB1"
-			  "\xEE\x0A\x2F\x95\x36\x4C\x76\x5D"
-			  "\x5F\xD9\x19\xED\xB9\xA5\x48\xBF"
-			  "\xC8\xAB\x0F\x71\xCC\x61\x8E\x0A"
-			  "\xD0\x29\x44\xA8\xB9\xC1\xE8\xC8"
-			  "\xC9\xA8\x28\x81\xFB\x50\xF2\xF0"
-			  "\x26\xAE\x39\xB8\x91\xCD\xA8\xAC"
-			  "\xDE\x55\x1B\x50\x14\x53\x44\x17"
-			  "\x54\x46\xFC\xB1\xE4\x07\x6B\x9A"
-			  "\x01\x14\xF0\x2E\x2E\xDB\x46\x1B"
-			  "\x1A\x09\x97\xA9\xB6\x97\x79\x06"
-			  "\xFB\xCB\x85\xCF\xDD\xA1\x41\xB1"
-			  "\x00\xAA\xF7\xE0\x89\x73\xFB\xE5"
-			  "\xBF\x84\xDB\xC9\xCD\xC4\xA2\x0D"
-			  "\x3B\xAC\xF9\xDF\x96\xBF\x88\x23"
-			  "\x41\x67\xA1\x24\x99\x7E\xCC\x9B"
-			  "\x02\x8F\x6A\x49\xF6\x25\xBA\x7A"
-			  "\xF4\x78\xFD\x79\x62\x63\x4F\x14"
-			  "\xD6\x11\x11\x04\x05\x5F\x7E\xEA"
-			  "\x4C\xB6\xF8\xF4\x5F\x48\x52\x54"
-			  "\x94\x63\xA8\x4E\xCF\xD2\x1B\x1B"
-			  "\x22\x18\x6A\xAF\x6E\x3E\xE1\x0D",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\xDF\xDD\x69\xFA\xB0\x2E\xFD\xFE"
-			  "\x70\x9E\xC5\x4B\xC9\xD4\xA1\x30"
-			  "\x26\x9B\x89\xA1\xEE\x43\xE0\x52"
-			  "\x55\x17\x4E\xC7\x0E\x33\x1F\xF1"
-			  "\x9F\x8D\x40\x9F\x24\xFD\x92\xA0"
-			  "\xBC\x8F\x35\xDD\x67\x38\xD8\xAA"
-			  "\xCF\xF8\x48\xCA\xFB\xE4\x5C\x60"
-			  "\x01\x41\x21\x12\x38\xAB\x52\x4F"
-			  "\xA8\x57\x20\xE0\x21\x6A\x17\x0D"
-			  "\x0E\xF9\x8E\x49\x42\x00\x3C\x94"
-			  "\x14\xC0\xD0\x8D\x8A\x98\xEB\x29"
-			  "\xEC\xAE\x96\x44\xC0\x3C\x48\xDC"
-			  "\x29\x35\x25\x2F\xE7\x11\x6C\x68"
-			  "\xC8\x67\x0A\x2F\xF4\x07\xBE\xF9"
-			  "\x2C\x31\x87\x40\xAB\xB2\xB6\xFA"
-			  "\xD2\xC9\x6D\x5C\x50\xE9\xE6\x7E"
-			  "\xE3\x0A\xD2\xD5\x6D\x8D\x64\x9E"
-			  "\x70\xCE\x03\x76\xDD\xE0\xF0\x8C"
-			  "\x84\x86\x8B\x6A\xFE\xC7\xF9\x69"
-			  "\x2E\xFE\xFC\xC2\xC4\x1A\x55\x58"
-			  "\xB3\xBE\xE2\x7E\xED\x39\x42\x6C"
-			  "\xB4\x42\x97\x9A\xEC\xE1\x0A\x06"
-			  "\x02\xC5\x03\x9D\xC4\x48\x15\x66"
-			  "\x35\x6A\xC2\xC9\xA2\x26\x30\xBB"
-			  "\xDB\x2D\xC8\x08\x2B\xA0\x29\x1A"
-			  "\x23\x61\x48\xEA\x80\x04\x27\xAA"
-			  "\x69\x49\xE8\xE8\x4A\x83\x6B\x5A"
-			  "\xCA\x7C\xD3\xB1\xB5\x0B\xCC\x23"
-			  "\x74\x1F\xA9\x87\xCD\xED\xC0\x2D"
-			  "\xBF\xEB\xCF\x16\x2D\x2A\x2E\x1D"
-			  "\x96\xBA\x36\x11\x45\x41\xDA\xCE"
-			  "\xA4\x48\x80\x8B\x06\xF4\x98\x89"
-			  "\x8B\x23\x08\x53\xF4\xD4\x5A\x24"
-			  "\x8B\xF8\x43\x73\xD1\xEE\xC4\xB0"
-			  "\xF8\xFE\x09\x0C\x75\x05\x38\x0B"
-			  "\x7C\x81\xDE\x9D\xE4\x61\x37\x63"
-			  "\x63\xAD\x12\xD2\x04\xB9\xCE\x45"
-			  "\x5A\x1A\x6E\xB3\x78\x2A\xA4\x74"
-			  "\x86\xD0\xE3\xFF\xDA\x38\x9C\xB5"
-			  "\xB8\xB1\xDB\x38\x2F\xC5\x6A\xB4"
-			  "\xEB\x6E\x96\xE8\x43\x80\xB5\x51"
-			  "\x61\x2D\x48\xAA\x07\x65\x11\x8C"
-			  "\x48\xE3\x90\x7E\x78\x3A\xEC\x97"
-			  "\x05\x3D\x84\xE7\x90\x2B\xAA\xBD"
-			  "\x83\x29\x0E\x1A\x81\x73\x7B\xE0"
-			  "\x7A\x01\x4A\x37\x3B\x77\x7F\x8D"
-			  "\x49\xA4\x2F\x6E\xBE\x68\x99\x08"
-			  "\x99\xAA\x4C\x12\x04\xAE\x1F\x77"
-			  "\x35\x88\xF1\x65\x06\x0A\x0B\x4D"
-			  "\x47\xF9\x50\x38\x5D\x71\xF9\x6E"
-			  "\xDE\xEC\x61\x35\x2C\x4C\x96\x50"
-			  "\xE8\x28\x93\x9C\x7E\x01\xC6\x04"
-			  "\xB2\xD6\xBC\x6C\x17\xEB\xC1\x7D"
-			  "\x11\xE9\x43\x83\x76\xAA\x53\x37"
-			  "\x0C\x1D\x39\x89\x53\x72\x09\x7E"
-			  "\xD9\x85\x16\x04\xA5\x2C\x05\x6F"
-			  "\x17\x0C\x6E\x66\xAA\x84\xA7\xD9"
-			  "\xE2\xD9\xC4\xEB\x43\x3E\xB1\x8D"
-			  "\x7C\x36\xC7\x71\x70\x9C\x10\xD8"
-			  "\xE8\x47\x2A\x4D\xFD\xA1\xBC\xE3"
-			  "\xB9\x32\xE2\xC1\x82\xAC\xFE\xCC"
-			  "\xC5\xC9\x7F\x9E\xCF\x33\x7A\xDF"
-			  "\x6C\x82\x9D",
-		.ilen	= 499,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59",
-		.rlen	= 499,
-		.also_non_np = 1,
-		.np	= 2,
-		.tap	= { 499 - 16, 16 },
-	},
-};
-
-static const struct cipher_testvec tf_lrw_enc_tv_template[] = {
-	/* Generated from AES-LRW test vectors */
-	{
-		.key	= "\x45\x62\xac\x25\xf8\x28\x17\x6d"
-			  "\x4c\x26\x84\x14\xb5\x68\x01\x85"
-			  "\x25\x8e\x2a\x05\xe7\x3e\x9d\x03"
-			  "\xee\x5a\x83\x0c\xcc\x09\x4c\x87",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\xa1\x6c\x50\x69\x26\xa4\xef\x7b"
-			  "\x7c\xc6\x91\xeb\x72\xdd\x9b\xee",
-		.rlen	= 16,
-	}, {
-		.key	= "\x59\x70\x47\x14\xf5\x57\x47\x8c"
-			  "\xd7\x79\xe8\x0f\x54\x88\x79\x44"
-			  "\x0d\x48\xf0\xb7\xb1\x5a\x53\xea"
-			  "\x1c\xaa\x6b\x29\xc2\xca\xfb\xaf",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x02",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\xab\x72\x0a\xad\x3b\x0c\xf0\xc9"
-			  "\x42\x2f\xf1\xae\xf1\x3c\xb1\xbd",
-		.rlen	= 16,
-	}, {
-		.key	= "\xd8\x2a\x91\x34\xb2\x6a\x56\x50"
-			  "\x30\xfe\x69\xe2\x37\x7f\x98\x47"
-			  "\xcd\xf9\x0b\x16\x0c\x64\x8f\xb6"
-			  "\xb0\x0d\x0d\x1b\xae\x85\x87\x1f",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x85\xa7\x56\x67\x08\xfa\x42\xe1"
-			  "\x22\xe6\x82\xfc\xd9\xb4\xd7\xd4",
-		.rlen	= 16,
-	}, {
-		.key	= "\x0f\x6a\xef\xf8\xd3\xd2\xbb\x15"
-			  "\x25\x83\xf7\x3c\x1f\x01\x28\x74"
-			  "\xca\xc6\xbc\x35\x4d\x4a\x65\x54"
-			  "\x90\xae\x61\xcf\x7b\xae\xbd\xcc"
-			  "\xad\xe4\x94\xc5\x4a\x29\xae\x70",
-		.klen	= 40,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\xd2\xaf\x69\x35\x24\x1d\x0e\x1c"
-			  "\x84\x8b\x05\xe4\xa2\x2f\x16\xf5",
-		.rlen	= 16,
-	}, {
-		.key	= "\x8a\xd4\xee\x10\x2f\xbd\x81\xff"
-			  "\xf8\x86\xce\xac\x93\xc5\xad\xc6"
-			  "\xa0\x19\x07\xc0\x9d\xf7\xbb\xdd"
-			  "\x52\x13\xb2\xb7\xf0\xff\x11\xd8"
-			  "\xd6\x08\xd0\xcd\x2e\xb1\x17\x6f",
-		.klen	= 40,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x4a\x23\x56\xd7\xff\x90\xd0\x9a"
-			  "\x0d\x7c\x26\xfc\xf0\xf0\xf6\xe4",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x30\xaf\x26\x05\x9d\x5d\x0a\x58"
-			  "\xe2\xe7\xce\x8a\xb2\x56\x6d\x76",
-		.rlen	= 16,
-	}, {
-		.key	= "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
-			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
-			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
-			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
-			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
-			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\xdf\xcf\xdc\xd2\xe1\xcf\x86\x75"
-			  "\x17\x66\x5e\x0c\x14\xa1\x3d\x40",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
-			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
-			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
-			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
-			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
-			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
-			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
-			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
-			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
-			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
-			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
-			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
-			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
-			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
-			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
-			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
-			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
-			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
-			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
-			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
-			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
-			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
-			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
-			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
-			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
-			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
-			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
-			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
-			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
-			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
-			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
-			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
-			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
-			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
-			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
-			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
-			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
-			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
-			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
-			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
-			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
-			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
-			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
-			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
-			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
-			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
-			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
-			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
-			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
-			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
-			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
-			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
-			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
-			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
-			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
-			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
-			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
-			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
-			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
-			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
-			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
-			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
-			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
-			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
-		.ilen	= 512,
-		.result	= "\x30\x38\xeb\xaf\x12\x43\x1a\x89"
-			  "\x62\xa2\x36\xe5\xcf\x77\x1e\xd9"
-			  "\x08\xc3\x0d\xdd\x95\xab\x19\x96"
-			  "\x27\x52\x41\xc3\xca\xfb\xf6\xee"
-			  "\x40\x2d\xdf\xdd\x00\x0c\xb9\x0a"
-			  "\x3a\xf0\xc0\xd1\xda\x63\x9e\x45"
-			  "\x42\xe9\x29\xc0\xb4\x07\xb4\x31"
-			  "\x66\x77\x72\xb5\xb6\xb3\x57\x46"
-			  "\x34\x9a\xfe\x03\xaf\x6b\x36\x07"
-			  "\x63\x8e\xc2\x5d\xa6\x0f\xb6\x7d"
-			  "\xfb\x6d\x82\x51\xb6\x98\xd0\x71"
-			  "\xe7\x10\x7a\xdf\xb2\xbd\xf1\x1d"
-			  "\x72\x2b\x54\x13\xe3\x6d\x79\x37"
-			  "\xa9\x39\x2c\xdf\x21\xab\x87\xd5"
-			  "\xee\xef\x9a\x12\x50\x39\x2e\x1b"
-			  "\x7d\xe6\x6a\x27\x48\xb9\xe7\xac"
-			  "\xaa\xcd\x79\x5f\xf2\xf3\xa0\x08"
-			  "\x6f\x2c\xf4\x0e\xd1\xb8\x89\x25"
-			  "\x31\x9d\xef\xb1\x1d\x27\x55\x04"
-			  "\xc9\x8c\xb7\x68\xdc\xb6\x67\x8a"
-			  "\xdb\xcf\x22\xf2\x3b\x6f\xce\xbb"
-			  "\x26\xbe\x4f\x27\x04\x42\xd1\x44"
-			  "\x4c\x08\xa3\x95\x4c\x7f\x1a\xaf"
-			  "\x1d\x28\x14\xfd\xb1\x1a\x34\x18"
-			  "\xf5\x1e\x28\x69\x95\x6a\x5a\xba"
-			  "\x8e\xb2\x58\x1d\x28\x17\x13\x3d"
-			  "\x38\x7d\x14\x8d\xab\x5d\xf9\xe8"
-			  "\x3c\x0f\x2b\x0d\x2b\x08\xb4\x4b"
-			  "\x6b\x0d\xc8\xa7\x84\xc2\x3a\x1a"
-			  "\xb7\xbd\xda\x92\x29\xb8\x5b\x5a"
-			  "\x63\xa5\x99\x82\x09\x72\x8f\xc6"
-			  "\xa4\x62\x24\x69\x8c\x2d\x26\x00"
-			  "\x99\x83\x91\xd6\xc6\xcf\x57\x67"
-			  "\x38\xea\xf2\xfc\x29\xe0\x73\x39"
-			  "\xf9\x13\x94\x6d\xe2\x58\x28\x75"
-			  "\x3e\xae\x71\x90\x07\x70\x1c\x38"
-			  "\x5b\x4c\x1e\xb5\xa5\x3b\x20\xef"
-			  "\xb1\x4c\x3e\x1a\x72\x62\xbb\x22"
-			  "\x82\x09\xe3\x18\x3f\x4f\x48\xfc"
-			  "\xdd\xac\xfc\xb6\x09\xdb\xd2\x7b"
-			  "\xd6\xb7\x7e\x41\x2f\x14\xf5\x0e"
-			  "\xc3\xac\x4a\xed\xe7\x82\xef\x31"
-			  "\x1f\x1a\x51\x1e\x29\x60\xc8\x98"
-			  "\x93\x51\x1d\x3d\x62\x59\x83\x82"
-			  "\x0c\xf1\xd7\x8d\xac\x33\x44\x81"
-			  "\x3c\x59\xb7\xd4\x5b\x65\x82\xc4"
-			  "\xec\xdc\x24\xfd\x0e\x1a\x79\x94"
-			  "\x34\xb0\x62\xfa\x98\x49\x26\x1f"
-			  "\xf4\x9e\x40\x44\x5b\x1f\xf8\xbe"
-			  "\x36\xff\xc6\xc6\x9d\xf2\xd6\xcc"
-			  "\x63\x93\x29\xb9\x0b\x6d\xd7\x6c"
-			  "\xdb\xf6\x21\x80\xf7\x5a\x37\x15"
-			  "\x0c\xe3\x36\xc8\x74\x75\x20\x91"
-			  "\xdf\x52\x2d\x0c\xe7\x45\xff\x46"
-			  "\xb3\xf4\xec\xc2\xbd\xd3\x37\xb6"
-			  "\x26\xa2\x5d\x7d\x61\xbf\x10\x46"
-			  "\x57\x8d\x05\x96\x70\x0b\xd6\x41"
-			  "\x5c\xe9\xd3\x54\x81\x39\x3a\xdd"
-			  "\x5f\x92\x81\x6e\x35\x03\xd4\x72"
-			  "\x3d\x5a\xe7\xb9\x3b\x0c\x84\x23"
-			  "\x45\x5d\xec\x72\xc1\x52\xef\x2e"
-			  "\x81\x00\xd3\xfe\x4c\x3c\x05\x61"
-			  "\x80\x18\xc4\x6c\x03\xd3\xb7\xba"
-			  "\x11\xd7\xb8\x6e\xea\xe1\x80\x30",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec tf_lrw_dec_tv_template[] = {
-	/* Generated from AES-LRW test vectors */
-	/* same as enc vectors with input and result reversed */
-	{
-		.key	= "\x45\x62\xac\x25\xf8\x28\x17\x6d"
-			  "\x4c\x26\x84\x14\xb5\x68\x01\x85"
-			  "\x25\x8e\x2a\x05\xe7\x3e\x9d\x03"
-			  "\xee\x5a\x83\x0c\xcc\x09\x4c\x87",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\xa1\x6c\x50\x69\x26\xa4\xef\x7b"
-			  "\x7c\xc6\x91\xeb\x72\xdd\x9b\xee",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\x59\x70\x47\x14\xf5\x57\x47\x8c"
-			  "\xd7\x79\xe8\x0f\x54\x88\x79\x44"
-			  "\x0d\x48\xf0\xb7\xb1\x5a\x53\xea"
-			  "\x1c\xaa\x6b\x29\xc2\xca\xfb\xaf",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x02",
-		.input	= "\xab\x72\x0a\xad\x3b\x0c\xf0\xc9"
-			  "\x42\x2f\xf1\xae\xf1\x3c\xb1\xbd",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\xd8\x2a\x91\x34\xb2\x6a\x56\x50"
-			  "\x30\xfe\x69\xe2\x37\x7f\x98\x47"
-			  "\xcd\xf9\x0b\x16\x0c\x64\x8f\xb6"
-			  "\xb0\x0d\x0d\x1b\xae\x85\x87\x1f",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x85\xa7\x56\x67\x08\xfa\x42\xe1"
-			  "\x22\xe6\x82\xfc\xd9\xb4\xd7\xd4",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\x0f\x6a\xef\xf8\xd3\xd2\xbb\x15"
-			  "\x25\x83\xf7\x3c\x1f\x01\x28\x74"
-			  "\xca\xc6\xbc\x35\x4d\x4a\x65\x54"
-			  "\x90\xae\x61\xcf\x7b\xae\xbd\xcc"
-			  "\xad\xe4\x94\xc5\x4a\x29\xae\x70",
-		.klen	= 40,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\xd2\xaf\x69\x35\x24\x1d\x0e\x1c"
-			  "\x84\x8b\x05\xe4\xa2\x2f\x16\xf5",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\x8a\xd4\xee\x10\x2f\xbd\x81\xff"
-			  "\xf8\x86\xce\xac\x93\xc5\xad\xc6"
-			  "\xa0\x19\x07\xc0\x9d\xf7\xbb\xdd"
-			  "\x52\x13\xb2\xb7\xf0\xff\x11\xd8"
-			  "\xd6\x08\xd0\xcd\x2e\xb1\x17\x6f",
-		.klen	= 40,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x4a\x23\x56\xd7\xff\x90\xd0\x9a"
-			  "\x0d\x7c\x26\xfc\xf0\xf0\xf6\xe4",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x30\xaf\x26\x05\x9d\x5d\x0a\x58"
-			  "\xe2\xe7\xce\x8a\xb2\x56\x6d\x76",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
-			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
-			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
-			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
-			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
-			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\xdf\xcf\xdc\xd2\xe1\xcf\x86\x75"
-			  "\x17\x66\x5e\x0c\x14\xa1\x3d\x40",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x30\x38\xeb\xaf\x12\x43\x1a\x89"
-			  "\x62\xa2\x36\xe5\xcf\x77\x1e\xd9"
-			  "\x08\xc3\x0d\xdd\x95\xab\x19\x96"
-			  "\x27\x52\x41\xc3\xca\xfb\xf6\xee"
-			  "\x40\x2d\xdf\xdd\x00\x0c\xb9\x0a"
-			  "\x3a\xf0\xc0\xd1\xda\x63\x9e\x45"
-			  "\x42\xe9\x29\xc0\xb4\x07\xb4\x31"
-			  "\x66\x77\x72\xb5\xb6\xb3\x57\x46"
-			  "\x34\x9a\xfe\x03\xaf\x6b\x36\x07"
-			  "\x63\x8e\xc2\x5d\xa6\x0f\xb6\x7d"
-			  "\xfb\x6d\x82\x51\xb6\x98\xd0\x71"
-			  "\xe7\x10\x7a\xdf\xb2\xbd\xf1\x1d"
-			  "\x72\x2b\x54\x13\xe3\x6d\x79\x37"
-			  "\xa9\x39\x2c\xdf\x21\xab\x87\xd5"
-			  "\xee\xef\x9a\x12\x50\x39\x2e\x1b"
-			  "\x7d\xe6\x6a\x27\x48\xb9\xe7\xac"
-			  "\xaa\xcd\x79\x5f\xf2\xf3\xa0\x08"
-			  "\x6f\x2c\xf4\x0e\xd1\xb8\x89\x25"
-			  "\x31\x9d\xef\xb1\x1d\x27\x55\x04"
-			  "\xc9\x8c\xb7\x68\xdc\xb6\x67\x8a"
-			  "\xdb\xcf\x22\xf2\x3b\x6f\xce\xbb"
-			  "\x26\xbe\x4f\x27\x04\x42\xd1\x44"
-			  "\x4c\x08\xa3\x95\x4c\x7f\x1a\xaf"
-			  "\x1d\x28\x14\xfd\xb1\x1a\x34\x18"
-			  "\xf5\x1e\x28\x69\x95\x6a\x5a\xba"
-			  "\x8e\xb2\x58\x1d\x28\x17\x13\x3d"
-			  "\x38\x7d\x14\x8d\xab\x5d\xf9\xe8"
-			  "\x3c\x0f\x2b\x0d\x2b\x08\xb4\x4b"
-			  "\x6b\x0d\xc8\xa7\x84\xc2\x3a\x1a"
-			  "\xb7\xbd\xda\x92\x29\xb8\x5b\x5a"
-			  "\x63\xa5\x99\x82\x09\x72\x8f\xc6"
-			  "\xa4\x62\x24\x69\x8c\x2d\x26\x00"
-			  "\x99\x83\x91\xd6\xc6\xcf\x57\x67"
-			  "\x38\xea\xf2\xfc\x29\xe0\x73\x39"
-			  "\xf9\x13\x94\x6d\xe2\x58\x28\x75"
-			  "\x3e\xae\x71\x90\x07\x70\x1c\x38"
-			  "\x5b\x4c\x1e\xb5\xa5\x3b\x20\xef"
-			  "\xb1\x4c\x3e\x1a\x72\x62\xbb\x22"
-			  "\x82\x09\xe3\x18\x3f\x4f\x48\xfc"
-			  "\xdd\xac\xfc\xb6\x09\xdb\xd2\x7b"
-			  "\xd6\xb7\x7e\x41\x2f\x14\xf5\x0e"
-			  "\xc3\xac\x4a\xed\xe7\x82\xef\x31"
-			  "\x1f\x1a\x51\x1e\x29\x60\xc8\x98"
-			  "\x93\x51\x1d\x3d\x62\x59\x83\x82"
-			  "\x0c\xf1\xd7\x8d\xac\x33\x44\x81"
-			  "\x3c\x59\xb7\xd4\x5b\x65\x82\xc4"
-			  "\xec\xdc\x24\xfd\x0e\x1a\x79\x94"
-			  "\x34\xb0\x62\xfa\x98\x49\x26\x1f"
-			  "\xf4\x9e\x40\x44\x5b\x1f\xf8\xbe"
-			  "\x36\xff\xc6\xc6\x9d\xf2\xd6\xcc"
-			  "\x63\x93\x29\xb9\x0b\x6d\xd7\x6c"
-			  "\xdb\xf6\x21\x80\xf7\x5a\x37\x15"
-			  "\x0c\xe3\x36\xc8\x74\x75\x20\x91"
-			  "\xdf\x52\x2d\x0c\xe7\x45\xff\x46"
-			  "\xb3\xf4\xec\xc2\xbd\xd3\x37\xb6"
-			  "\x26\xa2\x5d\x7d\x61\xbf\x10\x46"
-			  "\x57\x8d\x05\x96\x70\x0b\xd6\x41"
-			  "\x5c\xe9\xd3\x54\x81\x39\x3a\xdd"
-			  "\x5f\x92\x81\x6e\x35\x03\xd4\x72"
-			  "\x3d\x5a\xe7\xb9\x3b\x0c\x84\x23"
-			  "\x45\x5d\xec\x72\xc1\x52\xef\x2e"
-			  "\x81\x00\xd3\xfe\x4c\x3c\x05\x61"
-			  "\x80\x18\xc4\x6c\x03\xd3\xb7\xba"
-			  "\x11\xd7\xb8\x6e\xea\xe1\x80\x30",
-		.ilen	= 512,
-		.result	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
-			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
-			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
-			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
-			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
-			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
-			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
-			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
-			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
-			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
-			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
-			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
-			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
-			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
-			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
-			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
-			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
-			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
-			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
-			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
-			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
-			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
-			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
-			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
-			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
-			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
-			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
-			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
-			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
-			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
-			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
-			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
-			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
-			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
-			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
-			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
-			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
-			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
-			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
-			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
-			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
-			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
-			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
-			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
-			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
-			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
-			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
-			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
-			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
-			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
-			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
-			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
-			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
-			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
-			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
-			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
-			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
-			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
-			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
-			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
-			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
-			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
-			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
-			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec tf_xts_enc_tv_template[] = {
-	/* Generated from AES-XTS test vectors */
-{
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 32,
-		.result	= "\x4b\xc9\x44\x4a\x11\xa3\xef\xac"
-			  "\x30\x74\xe4\x44\x52\x77\x97\x43"
-			  "\xa7\x60\xb2\x45\x2e\xf9\x00\x90"
-			  "\x9f\xaa\xfd\x89\x6e\x9d\x4a\xe0",
-		.rlen	= 32,
-	}, {
-		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen	= 32,
-		.result	= "\x57\x0e\x8f\xe5\x2a\x35\x61\x4f"
-			  "\x32\xd3\xbd\x36\x05\x15\x44\x2c"
-			  "\x58\x06\xf7\xf8\x00\xa8\xb6\xd5"
-			  "\xc6\x28\x92\xdb\xd8\x34\xa2\xe9",
-		.rlen	= 32,
-	}, {
-		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen	= 32,
-		.result	= "\x96\x45\x8f\x8d\x7a\x75\xb1\xde"
-			  "\x40\x0c\x89\x56\xf6\x4d\xa7\x07"
-			  "\x38\xbb\x5b\xe9\xcd\x84\xae\xb2"
-			  "\x7b\x6a\x62\xf4\x8c\xb5\x37\xea",
-		.rlen	= 32,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\xa9\x78\xae\x1e\xea\xa2\x44\x4c"
-			  "\xa2\x7a\x64\x1f\xaf\x46\xc1\xe0"
-			  "\x6c\xb2\xf3\x92\x9a\xd6\x7d\x58"
-			  "\xb8\x2d\xb9\x5d\x58\x07\x66\x50"
-			  "\xea\x35\x35\x8c\xb2\x46\x61\x06"
-			  "\x5d\x65\xfc\x57\x8f\x69\x74\xab"
-			  "\x8a\x06\x69\xb5\x6c\xda\x66\xc7"
-			  "\x52\x90\xbb\x8e\x6d\x8b\xb5\xa2"
-			  "\x78\x1d\xc2\xa9\xc2\x73\x00\xc3"
-			  "\x32\x36\x7c\x97\x6b\x4e\x8a\x50"
-			  "\xe4\x91\x83\x96\x8f\xf4\x94\x1a"
-			  "\xa6\x27\xe1\x33\xcb\x91\xc6\x5f"
-			  "\x94\x75\xbc\xd7\x3e\x3e\x6f\x9e"
-			  "\xa9\x31\x80\x5e\xe5\xdb\xc8\x53"
-			  "\x01\x73\x68\x32\x25\x19\xfa\xfb"
-			  "\xe4\xcf\xb9\x3e\xa2\xa0\x8f\x31"
-			  "\xbf\x54\x06\x93\xa8\xb1\x0f\xb6"
-			  "\x7c\x3c\xde\x6f\x0f\xfb\x0c\x11"
-			  "\x39\x80\x39\x09\x97\x65\xf2\x83"
-			  "\xae\xe6\xa1\x6f\x47\xb8\x49\xde"
-			  "\x99\x36\x20\x7d\x97\x3b\xec\xfa"
-			  "\xb4\x33\x6e\x7a\xc7\x46\x84\x49"
-			  "\x91\xcd\xe1\x57\x0d\xed\x40\x08"
-			  "\x13\xf1\x4e\x3e\xa4\xa4\x5c\xe6"
-			  "\xd2\x0c\x20\x8f\x3e\xdf\x3f\x47"
-			  "\x9a\x2f\xde\x6d\x66\xc9\x99\x4a"
-			  "\x2d\x9e\x9d\x4b\x1a\x27\xa2\x12"
-			  "\x99\xf0\xf8\xb1\xb6\xf6\x57\xc3"
-			  "\xca\x1c\xa3\x8e\xed\x39\x28\xb5"
-			  "\x10\x1b\x4b\x08\x42\x00\x4a\xd3"
-			  "\xad\x5a\xc6\x8e\xc8\xbb\x95\xc4"
-			  "\x4b\xaa\xfe\xd5\x42\xa8\xa3\x6d"
-			  "\x3c\xf3\x34\x91\x2d\xb4\xdd\x20"
-			  "\x0c\x90\x6d\xa3\x9b\x66\x9d\x24"
-			  "\x02\xa6\xa9\x3f\x3f\x58\x5d\x47"
-			  "\x24\x65\x63\x7e\xbd\x8c\xe6\x52"
-			  "\x7d\xef\x33\x53\x63\xec\xaa\x0b"
-			  "\x64\x15\xa9\xa6\x1f\x10\x00\x38"
-			  "\x35\xa8\xe7\xbe\x23\x70\x22\xe0"
-			  "\xd3\xb9\xe6\xfd\xe6\xaa\x03\x50"
-			  "\xf3\x3c\x27\x36\x8b\xcc\xfe\x9c"
-			  "\x9c\xa3\xb3\xe7\x68\x9b\xa2\x71"
-			  "\xe0\x07\xd9\x1f\x68\x1f\xac\x5e"
-			  "\x7a\x74\x85\xa9\x6a\x90\xab\x2c"
-			  "\x38\x51\xbc\x1f\x43\x4a\x56\x1c"
-			  "\xf8\x47\x03\x4e\x67\xa8\x1f\x99"
-			  "\x04\x39\x73\x32\xb2\x86\x79\xe7"
-			  "\x14\x28\x70\xb8\xe2\x7d\x69\x85"
-			  "\xb6\x0f\xc5\xd0\xd0\x01\x5c\xe6"
-			  "\x09\x0f\x75\xf7\xb6\x81\xd2\x11"
-			  "\x20\x9c\xa1\xee\x11\x44\x79\xd0"
-			  "\xb2\x34\x77\xda\x10\x9a\x6f\x6f"
-			  "\xef\x7c\xd9\xdc\x35\xb7\x61\xdd"
-			  "\xf1\xa4\xc6\x1c\xbf\x05\x22\xac"
-			  "\xfe\x2f\x85\x00\x44\xdf\x33\x16"
-			  "\x35\xb6\xa3\xd3\x70\xdf\x69\x35"
-			  "\x6a\xc7\xb4\x99\x45\x27\xc8\x8e"
-			  "\x5a\x14\x30\xd0\x55\x3e\x4f\x64"
-			  "\x0d\x38\xe3\xdf\x8b\xa8\x93\x26"
-			  "\x75\xae\xf6\xb5\x23\x0b\x17\x31"
-			  "\xbf\x27\xb8\xb5\x94\x31\xa7\x8f"
-			  "\x43\xc4\x46\x24\x22\x4f\x8f\x7e"
-			  "\xe5\xf4\x6d\x1e\x0e\x18\x7a\xbb"
-			  "\xa6\x8f\xfb\x49\x49\xd8\x7e\x5a",
-		.rlen	= 512,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\xd7\x4b\x93\x7d\x13\xa2\xa2\xe1"
-			  "\x35\x39\x71\x88\x76\x1e\xc9\xea"
-			  "\x86\xad\xf3\x14\x48\x3d\x5e\xe9"
-			  "\xe9\x2d\xb2\x56\x59\x35\x9d\xec"
-			  "\x84\xfa\x7e\x9d\x6d\x33\x36\x8f"
-			  "\xce\xf4\xa9\x21\x0b\x5f\x96\xec"
-			  "\xcb\xf9\x57\x68\x33\x88\x39\xbf"
-			  "\x2f\xbb\x59\x03\xbd\x66\x8b\x11"
-			  "\x11\x65\x51\x2e\xb8\x67\x05\xd1"
-			  "\x27\x11\x5c\xd4\xcc\x97\xc2\xb3"
-			  "\xa9\x55\xaf\x07\x56\xd1\xdc\xf5"
-			  "\x85\xdc\x46\xe6\xf0\x24\xeb\x93"
-			  "\x4d\xf0\x9b\xf5\x73\x1c\xda\x03"
-			  "\x22\xc8\x3a\x4f\xb4\x19\x91\x09"
-			  "\x54\x0b\xf6\xfe\x17\x3d\x1a\x53"
-			  "\x72\x60\x79\xcb\x0e\x32\x8a\x77"
-			  "\xd5\xed\xdb\x33\xd7\x62\x16\x69"
-			  "\x63\xe0\xab\xb5\xf6\x9c\x5f\x3d"
-			  "\x69\x35\x61\x86\xf8\x86\xb9\x89"
-			  "\x6e\x59\x35\xac\xf6\x6b\x33\xa0"
-			  "\xea\xef\x96\x62\xd8\xa9\xcf\x56"
-			  "\xbf\xdb\x8a\xfd\xa1\x82\x77\x73"
-			  "\x3d\x94\x4a\x49\x42\x6d\x08\x60"
-			  "\xa1\xea\xab\xb6\x88\x13\x94\xb8"
-			  "\x51\x98\xdb\x35\x85\xdf\xf6\xb9"
-			  "\x8f\xcd\xdf\x80\xd3\x40\x2d\x72"
-			  "\xb8\xb2\x6c\x02\x43\x35\x22\x2a"
-			  "\x31\xed\xcd\x16\x19\xdf\x62\x0f"
-			  "\x29\xcf\x87\x04\xec\x02\x4f\xe4"
-			  "\xa2\xed\x73\xc6\x69\xd3\x7e\x89"
-			  "\x0b\x76\x10\x7c\xd6\xf9\x6a\x25"
-			  "\xed\xcc\x60\x5d\x61\x20\xc1\x97"
-			  "\x56\x91\x57\x28\xbe\x71\x0d\xcd"
-			  "\xde\xc4\x9e\x55\x91\xbe\xd1\x28"
-			  "\x9b\x90\xeb\x73\xf3\x68\x51\xc6"
-			  "\xdf\x82\xcc\xd8\x1f\xce\x5b\x27"
-			  "\xc0\x60\x5e\x33\xd6\xa7\x20\xea"
-			  "\xb2\x54\xc7\x5d\x6a\x3b\x67\x47"
-			  "\xcf\xa0\xe3\xab\x86\xaf\xc1\x42"
-			  "\xe6\xb0\x23\x4a\xaf\x53\xdf\xa0"
-			  "\xad\x12\x32\x31\x03\xf7\x21\xbe"
-			  "\x2d\xd5\x82\x42\xb6\x4a\x3d\xcd"
-			  "\xd8\x81\x77\xa9\x49\x98\x6c\x09"
-			  "\xc5\xa3\x61\x12\x62\x85\x6b\xcd"
-			  "\xb3\xf4\x20\x0c\x41\xc4\x05\x37"
-			  "\x46\x5f\xeb\x71\x8b\xf1\xaf\x6e"
-			  "\xba\xf3\x50\x2e\xfe\xa8\x37\xeb"
-			  "\xe8\x8c\x4f\xa4\x0c\xf1\x31\xc8"
-			  "\x6e\x71\x4f\xa5\xd7\x97\x73\xe0"
-			  "\x93\x4a\x2f\xda\x7b\xe0\x20\x54"
-			  "\x1f\x8d\x85\x79\x0b\x7b\x5e\x75"
-			  "\xb9\x07\x67\xcc\xc8\xe7\x21\x15"
-			  "\xa7\xc8\x98\xff\x4b\x80\x1c\x12"
-			  "\xa8\x54\xe1\x38\x52\xe6\x74\x81"
-			  "\x97\x47\xa1\x41\x0e\xc0\x50\xe3"
-			  "\x55\x0e\xc3\xa7\x70\x77\xce\x07"
-			  "\xed\x8c\x88\xe6\xa1\x5b\x14\xec"
-			  "\xe6\xde\x06\x6d\x74\xc5\xd9\xfa"
-			  "\xe5\x2f\x5a\xff\xc8\x05\xee\x27"
-			  "\x35\x61\xbf\x0b\x19\x78\x9b\xd2"
-			  "\x04\xc7\x05\xb1\x79\xb4\xff\x5f"
-			  "\xf3\xea\x67\x52\x78\xc2\xce\x70"
-			  "\xa4\x05\x0b\xb2\xb3\xa8\x30\x97"
-			  "\x37\x30\xe1\x91\x8d\xb3\x2a\xff",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec tf_xts_dec_tv_template[] = {
-	/* Generated from AES-XTS test vectors */
-	/* same as enc vectors with input and result reversed */
-	{
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x4b\xc9\x44\x4a\x11\xa3\xef\xac"
-			  "\x30\x74\xe4\x44\x52\x77\x97\x43"
-			  "\xa7\x60\xb2\x45\x2e\xf9\x00\x90"
-			  "\x9f\xaa\xfd\x89\x6e\x9d\x4a\xe0",
-		.ilen	= 32,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen	= 32,
-	}, {
-		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x57\x0e\x8f\xe5\x2a\x35\x61\x4f"
-			  "\x32\xd3\xbd\x36\x05\x15\x44\x2c"
-			  "\x58\x06\xf7\xf8\x00\xa8\xb6\xd5"
-			  "\xc6\x28\x92\xdb\xd8\x34\xa2\xe9",
-		.ilen	= 32,
-		.result	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen	= 32,
-	}, {
-		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x96\x45\x8f\x8d\x7a\x75\xb1\xde"
-			  "\x40\x0c\x89\x56\xf6\x4d\xa7\x07"
-			  "\x38\xbb\x5b\xe9\xcd\x84\xae\xb2"
-			  "\x7b\x6a\x62\xf4\x8c\xb5\x37\xea",
-		.ilen	= 32,
-		.result	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen	= 32,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xa9\x78\xae\x1e\xea\xa2\x44\x4c"
-			  "\xa2\x7a\x64\x1f\xaf\x46\xc1\xe0"
-			  "\x6c\xb2\xf3\x92\x9a\xd6\x7d\x58"
-			  "\xb8\x2d\xb9\x5d\x58\x07\x66\x50"
-			  "\xea\x35\x35\x8c\xb2\x46\x61\x06"
-			  "\x5d\x65\xfc\x57\x8f\x69\x74\xab"
-			  "\x8a\x06\x69\xb5\x6c\xda\x66\xc7"
-			  "\x52\x90\xbb\x8e\x6d\x8b\xb5\xa2"
-			  "\x78\x1d\xc2\xa9\xc2\x73\x00\xc3"
-			  "\x32\x36\x7c\x97\x6b\x4e\x8a\x50"
-			  "\xe4\x91\x83\x96\x8f\xf4\x94\x1a"
-			  "\xa6\x27\xe1\x33\xcb\x91\xc6\x5f"
-			  "\x94\x75\xbc\xd7\x3e\x3e\x6f\x9e"
-			  "\xa9\x31\x80\x5e\xe5\xdb\xc8\x53"
-			  "\x01\x73\x68\x32\x25\x19\xfa\xfb"
-			  "\xe4\xcf\xb9\x3e\xa2\xa0\x8f\x31"
-			  "\xbf\x54\x06\x93\xa8\xb1\x0f\xb6"
-			  "\x7c\x3c\xde\x6f\x0f\xfb\x0c\x11"
-			  "\x39\x80\x39\x09\x97\x65\xf2\x83"
-			  "\xae\xe6\xa1\x6f\x47\xb8\x49\xde"
-			  "\x99\x36\x20\x7d\x97\x3b\xec\xfa"
-			  "\xb4\x33\x6e\x7a\xc7\x46\x84\x49"
-			  "\x91\xcd\xe1\x57\x0d\xed\x40\x08"
-			  "\x13\xf1\x4e\x3e\xa4\xa4\x5c\xe6"
-			  "\xd2\x0c\x20\x8f\x3e\xdf\x3f\x47"
-			  "\x9a\x2f\xde\x6d\x66\xc9\x99\x4a"
-			  "\x2d\x9e\x9d\x4b\x1a\x27\xa2\x12"
-			  "\x99\xf0\xf8\xb1\xb6\xf6\x57\xc3"
-			  "\xca\x1c\xa3\x8e\xed\x39\x28\xb5"
-			  "\x10\x1b\x4b\x08\x42\x00\x4a\xd3"
-			  "\xad\x5a\xc6\x8e\xc8\xbb\x95\xc4"
-			  "\x4b\xaa\xfe\xd5\x42\xa8\xa3\x6d"
-			  "\x3c\xf3\x34\x91\x2d\xb4\xdd\x20"
-			  "\x0c\x90\x6d\xa3\x9b\x66\x9d\x24"
-			  "\x02\xa6\xa9\x3f\x3f\x58\x5d\x47"
-			  "\x24\x65\x63\x7e\xbd\x8c\xe6\x52"
-			  "\x7d\xef\x33\x53\x63\xec\xaa\x0b"
-			  "\x64\x15\xa9\xa6\x1f\x10\x00\x38"
-			  "\x35\xa8\xe7\xbe\x23\x70\x22\xe0"
-			  "\xd3\xb9\xe6\xfd\xe6\xaa\x03\x50"
-			  "\xf3\x3c\x27\x36\x8b\xcc\xfe\x9c"
-			  "\x9c\xa3\xb3\xe7\x68\x9b\xa2\x71"
-			  "\xe0\x07\xd9\x1f\x68\x1f\xac\x5e"
-			  "\x7a\x74\x85\xa9\x6a\x90\xab\x2c"
-			  "\x38\x51\xbc\x1f\x43\x4a\x56\x1c"
-			  "\xf8\x47\x03\x4e\x67\xa8\x1f\x99"
-			  "\x04\x39\x73\x32\xb2\x86\x79\xe7"
-			  "\x14\x28\x70\xb8\xe2\x7d\x69\x85"
-			  "\xb6\x0f\xc5\xd0\xd0\x01\x5c\xe6"
-			  "\x09\x0f\x75\xf7\xb6\x81\xd2\x11"
-			  "\x20\x9c\xa1\xee\x11\x44\x79\xd0"
-			  "\xb2\x34\x77\xda\x10\x9a\x6f\x6f"
-			  "\xef\x7c\xd9\xdc\x35\xb7\x61\xdd"
-			  "\xf1\xa4\xc6\x1c\xbf\x05\x22\xac"
-			  "\xfe\x2f\x85\x00\x44\xdf\x33\x16"
-			  "\x35\xb6\xa3\xd3\x70\xdf\x69\x35"
-			  "\x6a\xc7\xb4\x99\x45\x27\xc8\x8e"
-			  "\x5a\x14\x30\xd0\x55\x3e\x4f\x64"
-			  "\x0d\x38\xe3\xdf\x8b\xa8\x93\x26"
-			  "\x75\xae\xf6\xb5\x23\x0b\x17\x31"
-			  "\xbf\x27\xb8\xb5\x94\x31\xa7\x8f"
-			  "\x43\xc4\x46\x24\x22\x4f\x8f\x7e"
-			  "\xe5\xf4\x6d\x1e\x0e\x18\x7a\xbb"
-			  "\xa6\x8f\xfb\x49\x49\xd8\x7e\x5a",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xd7\x4b\x93\x7d\x13\xa2\xa2\xe1"
-			  "\x35\x39\x71\x88\x76\x1e\xc9\xea"
-			  "\x86\xad\xf3\x14\x48\x3d\x5e\xe9"
-			  "\xe9\x2d\xb2\x56\x59\x35\x9d\xec"
-			  "\x84\xfa\x7e\x9d\x6d\x33\x36\x8f"
-			  "\xce\xf4\xa9\x21\x0b\x5f\x96\xec"
-			  "\xcb\xf9\x57\x68\x33\x88\x39\xbf"
-			  "\x2f\xbb\x59\x03\xbd\x66\x8b\x11"
-			  "\x11\x65\x51\x2e\xb8\x67\x05\xd1"
-			  "\x27\x11\x5c\xd4\xcc\x97\xc2\xb3"
-			  "\xa9\x55\xaf\x07\x56\xd1\xdc\xf5"
-			  "\x85\xdc\x46\xe6\xf0\x24\xeb\x93"
-			  "\x4d\xf0\x9b\xf5\x73\x1c\xda\x03"
-			  "\x22\xc8\x3a\x4f\xb4\x19\x91\x09"
-			  "\x54\x0b\xf6\xfe\x17\x3d\x1a\x53"
-			  "\x72\x60\x79\xcb\x0e\x32\x8a\x77"
-			  "\xd5\xed\xdb\x33\xd7\x62\x16\x69"
-			  "\x63\xe0\xab\xb5\xf6\x9c\x5f\x3d"
-			  "\x69\x35\x61\x86\xf8\x86\xb9\x89"
-			  "\x6e\x59\x35\xac\xf6\x6b\x33\xa0"
-			  "\xea\xef\x96\x62\xd8\xa9\xcf\x56"
-			  "\xbf\xdb\x8a\xfd\xa1\x82\x77\x73"
-			  "\x3d\x94\x4a\x49\x42\x6d\x08\x60"
-			  "\xa1\xea\xab\xb6\x88\x13\x94\xb8"
-			  "\x51\x98\xdb\x35\x85\xdf\xf6\xb9"
-			  "\x8f\xcd\xdf\x80\xd3\x40\x2d\x72"
-			  "\xb8\xb2\x6c\x02\x43\x35\x22\x2a"
-			  "\x31\xed\xcd\x16\x19\xdf\x62\x0f"
-			  "\x29\xcf\x87\x04\xec\x02\x4f\xe4"
-			  "\xa2\xed\x73\xc6\x69\xd3\x7e\x89"
-			  "\x0b\x76\x10\x7c\xd6\xf9\x6a\x25"
-			  "\xed\xcc\x60\x5d\x61\x20\xc1\x97"
-			  "\x56\x91\x57\x28\xbe\x71\x0d\xcd"
-			  "\xde\xc4\x9e\x55\x91\xbe\xd1\x28"
-			  "\x9b\x90\xeb\x73\xf3\x68\x51\xc6"
-			  "\xdf\x82\xcc\xd8\x1f\xce\x5b\x27"
-			  "\xc0\x60\x5e\x33\xd6\xa7\x20\xea"
-			  "\xb2\x54\xc7\x5d\x6a\x3b\x67\x47"
-			  "\xcf\xa0\xe3\xab\x86\xaf\xc1\x42"
-			  "\xe6\xb0\x23\x4a\xaf\x53\xdf\xa0"
-			  "\xad\x12\x32\x31\x03\xf7\x21\xbe"
-			  "\x2d\xd5\x82\x42\xb6\x4a\x3d\xcd"
-			  "\xd8\x81\x77\xa9\x49\x98\x6c\x09"
-			  "\xc5\xa3\x61\x12\x62\x85\x6b\xcd"
-			  "\xb3\xf4\x20\x0c\x41\xc4\x05\x37"
-			  "\x46\x5f\xeb\x71\x8b\xf1\xaf\x6e"
-			  "\xba\xf3\x50\x2e\xfe\xa8\x37\xeb"
-			  "\xe8\x8c\x4f\xa4\x0c\xf1\x31\xc8"
-			  "\x6e\x71\x4f\xa5\xd7\x97\x73\xe0"
-			  "\x93\x4a\x2f\xda\x7b\xe0\x20\x54"
-			  "\x1f\x8d\x85\x79\x0b\x7b\x5e\x75"
-			  "\xb9\x07\x67\xcc\xc8\xe7\x21\x15"
-			  "\xa7\xc8\x98\xff\x4b\x80\x1c\x12"
-			  "\xa8\x54\xe1\x38\x52\xe6\x74\x81"
-			  "\x97\x47\xa1\x41\x0e\xc0\x50\xe3"
-			  "\x55\x0e\xc3\xa7\x70\x77\xce\x07"
-			  "\xed\x8c\x88\xe6\xa1\x5b\x14\xec"
-			  "\xe6\xde\x06\x6d\x74\xc5\xd9\xfa"
-			  "\xe5\x2f\x5a\xff\xc8\x05\xee\x27"
-			  "\x35\x61\xbf\x0b\x19\x78\x9b\xd2"
-			  "\x04\xc7\x05\xb1\x79\xb4\xff\x5f"
-			  "\xf3\xea\x67\x52\x78\xc2\xce\x70"
-			  "\xa4\x05\x0b\xb2\xb3\xa8\x30\x97"
-			  "\x37\x30\xe1\x91\x8d\xb3\x2a\xff",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-/*
- * Serpent test vectors.  These are backwards because Serpent writes
- * octet sequences in right-to-left mode.
- */
-static const struct cipher_testvec serpent_enc_tv_template[] = {
-	{
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.ilen	= 16,
-		.result	= "\x12\x07\xfc\xce\x9b\xd0\xd6\x47"
-			  "\x6a\xe9\x8f\xbe\xd1\x43\xa0\xe2",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.klen	= 16,
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.ilen	= 16,
-		.result	= "\x4c\x7d\x8a\x32\x80\x72\xa2\x2c"
-			  "\x82\x3e\x4a\x1f\x3a\xcd\xa1\x6d",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.klen	= 32,
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.ilen	= 16,
-		.result	= "\xde\x26\x9f\xf8\x33\xe4\x32\xb8"
-			  "\x5b\x2e\x88\xd2\x70\x1c\xe7\x5c",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80",
-		.klen	= 16,
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.result	= "\xdd\xd2\x6b\x98\xa5\xff\xd8\x2c"
-			  "\x05\x34\x5a\x9d\xad\xbf\xaf\x49",
-		.rlen	= 16,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\xFB\xB0\x5D\xDE\xC0\xFE\xFC\xEB"
-			  "\xB1\x80\x10\x43\xDE\x62\x70\xBD"
-			  "\xFA\x8A\x93\xEA\x6B\xF7\xC5\xD7"
-			  "\x0C\xD1\xBB\x29\x25\x14\x4C\x22"
-			  "\x77\xA6\x38\x00\xDB\xB9\xE2\x07"
-			  "\xD1\xAC\x82\xBA\xEA\x67\xAA\x39"
-			  "\x99\x34\x89\x5B\x54\xE9\x12\x13"
-			  "\x3B\x04\xE5\x12\x42\xC5\x79\xAB"
-			  "\x0D\xC7\x3C\x58\x2D\xA3\x98\xF6"
-			  "\xE4\x61\x9E\x17\x0B\xCE\xE8\xAA"
-			  "\xB5\x6C\x1A\x3A\x67\x52\x81\x6A"
-			  "\x04\xFF\x8A\x1B\x96\xFE\xE6\x87"
-			  "\x3C\xD4\x39\x7D\x36\x9B\x03\xD5"
-			  "\xB6\xA0\x75\x3C\x83\xE6\x1C\x73"
-			  "\x9D\x74\x2B\x77\x53\x2D\xE5\xBD"
-			  "\x69\xDA\x7A\x01\xF5\x6A\x70\x39"
-			  "\x30\xD4\x2C\xF2\x8E\x06\x4B\x39"
-			  "\xB3\x12\x1D\xB3\x17\x46\xE6\xD6"
-			  "\xB6\x31\x36\x34\x38\x3C\x1D\x69"
-			  "\x9F\x47\x28\x9A\x1D\x96\x70\x54"
-			  "\x8E\x88\xCB\xE0\xF5\x6A\xAE\x0A"
-			  "\x3C\xD5\x93\x1C\x21\xC9\x14\x3A"
-			  "\x23\x9C\x9B\x79\xC7\x75\xC8\x39"
-			  "\xA6\xAC\x65\x9A\x99\x37\xAF\x6D"
-			  "\xBD\xB5\x32\xFD\xD8\x9C\x95\x7B"
-			  "\xC6\x6A\x80\x64\xEA\xEF\x6D\x3F"
-			  "\xA9\xFE\x5B\x16\xA3\xCF\x32\xC8"
-			  "\xEF\x50\x22\x20\x93\x30\xBE\xE2"
-			  "\x38\x05\x65\xAF\xBA\xB6\xE4\x72"
-			  "\xA9\xEE\x05\x42\x88\xBD\x9D\x49"
-			  "\xAD\x93\xCA\x4D\x45\x11\x43\x4D"
-			  "\xB8\xF5\x74\x2B\x48\xE7\x21\xE4"
-			  "\x4E\x3A\x4C\xDE\x65\x7A\x5A\xAD"
-			  "\x86\xE6\x23\xEC\x6B\xA7\x17\xE6"
-			  "\xF6\xA1\xAC\x29\xAE\xF9\x9B\x69"
-			  "\x73\x65\x65\x51\xD6\x0B\x4E\x8C"
-			  "\x17\x15\x9D\xB0\xCF\xB2\x42\x2B"
-			  "\x51\xC3\x03\xE8\xB7\x7D\x2D\x39"
-			  "\xE8\x10\x93\x16\xC8\x68\x4C\x60"
-			  "\x87\x70\x14\xD0\x01\x57\xCB\x42"
-			  "\x13\x59\xB1\x7F\x12\x4F\xBB\xC7"
-			  "\xBD\x2B\xD4\xA9\x12\x26\x4F\xDE"
-			  "\xFD\x72\xEC\xD7\x6F\x97\x14\x90"
-			  "\x0E\x37\x13\xE6\x67\x1D\xE5\xFE"
-			  "\x9E\x18\x3C\x8F\x3A\x3F\x59\x9B"
-			  "\x71\x80\x05\x35\x3F\x40\x0B\x21"
-			  "\x76\xE5\xEF\x42\x6C\xDB\x31\x05"
-			  "\x5F\x05\xCF\x14\xE3\xF0\x61\xA2"
-			  "\x49\x03\x5E\x77\x2E\x20\xBA\xA1"
-			  "\xAF\x46\x51\xC0\x2B\xC4\x64\x1E"
-			  "\x65\xCC\x51\x58\x0A\xDF\xF0\x5F"
-			  "\x75\x9F\x48\xCD\x81\xEC\xC3\xF6"
-			  "\xED\xC9\x4B\x7B\x4E\x26\x23\xE1"
-			  "\xBB\xE9\x83\x0B\xCF\xE4\xDE\x00"
-			  "\x48\xFF\xBF\x6C\xB4\x72\x16\xEF"
-			  "\xC7\x46\xEE\x48\x8C\xB8\xAF\x45"
-			  "\x91\x76\xE7\x6E\x65\x3D\x15\x86"
-			  "\x10\xF8\xDB\x66\x97\x7C\x43\x4D"
-			  "\x79\x12\x4E\xCE\x06\xD1\xD1\x6A"
-			  "\x34\xC1\xC9\xF2\x28\x4A\xCD\x02"
-			  "\x75\x55\x9B\xFF\x36\x73\xAB\x7C"
-			  "\xF4\x46\x2E\xEB\xAC\xF3\xD2\xB7",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec tnepres_enc_tv_template[] = {
-	{ /* KeySize=128, PT=0, I=1 */
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.key    = "\x80\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen   = 16,
-		.ilen	= 16,
-		.result	= "\x49\xaf\xbf\xad\x9d\x5a\x34\x05"
-			  "\x2c\xd8\xff\xa5\x98\x6b\xd2\xdd",
-		.rlen	= 16,
-	}, { /* KeySize=192, PT=0, I=1 */
-		.key	= "\x80\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 24,
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 16,
-		.result	= "\xe7\x8e\x54\x02\xc7\x19\x55\x68"
-			  "\xac\x36\x78\xf7\xa3\xf6\x0c\x66",
-		.rlen	= 16,
-	}, { /* KeySize=256, PT=0, I=1 */
-		.key	= "\x80\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 32,
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 16,
-		.result	= "\xab\xed\x96\xe7\x66\xbf\x28\xcb"
-			  "\xc0\xeb\xd2\x1a\x82\xef\x08\x19",
-		.rlen	= 16,
-	}, { /* KeySize=256, I=257 */
-		.key	= "\x1f\x1e\x1d\x1c\x1b\x1a\x19\x18"
-			  "\x17\x16\x15\x14\x13\x12\x11\x10"
-			  "\x0f\x0e\x0d\x0c\x0b\x0a\x09\x08"
-			  "\x07\x06\x05\x04\x03\x02\x01\x00",
-		.klen	= 32,
-		.input	= "\x0f\x0e\x0d\x0c\x0b\x0a\x09\x08"
-			  "\x07\x06\x05\x04\x03\x02\x01\x00",
-		.ilen	= 16,
-		.result	= "\x5c\xe7\x1c\x70\xd2\x88\x2e\x5b"
-			  "\xb8\x32\xe4\x33\xf8\x9f\x26\xde",
-		.rlen	= 16,
-	},
-};
-
-
-static const struct cipher_testvec serpent_dec_tv_template[] = {
-	{
-		.input	= "\x12\x07\xfc\xce\x9b\xd0\xd6\x47"
-			  "\x6a\xe9\x8f\xbe\xd1\x43\xa0\xe2",
-		.ilen	= 16,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.klen	= 16,
-		.input	= "\x4c\x7d\x8a\x32\x80\x72\xa2\x2c"
-			  "\x82\x3e\x4a\x1f\x3a\xcd\xa1\x6d",
-		.ilen	= 16,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.klen	= 32,
-		.input	= "\xde\x26\x9f\xf8\x33\xe4\x32\xb8"
-			  "\x5b\x2e\x88\xd2\x70\x1c\xe7\x5c",
-		.ilen	= 16,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80",
-		.klen	= 16,
-		.input	= "\xdd\xd2\x6b\x98\xa5\xff\xd8\x2c"
-			  "\x05\x34\x5a\x9d\xad\xbf\xaf\x49",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.input	= "\xFB\xB0\x5D\xDE\xC0\xFE\xFC\xEB"
-			  "\xB1\x80\x10\x43\xDE\x62\x70\xBD"
-			  "\xFA\x8A\x93\xEA\x6B\xF7\xC5\xD7"
-			  "\x0C\xD1\xBB\x29\x25\x14\x4C\x22"
-			  "\x77\xA6\x38\x00\xDB\xB9\xE2\x07"
-			  "\xD1\xAC\x82\xBA\xEA\x67\xAA\x39"
-			  "\x99\x34\x89\x5B\x54\xE9\x12\x13"
-			  "\x3B\x04\xE5\x12\x42\xC5\x79\xAB"
-			  "\x0D\xC7\x3C\x58\x2D\xA3\x98\xF6"
-			  "\xE4\x61\x9E\x17\x0B\xCE\xE8\xAA"
-			  "\xB5\x6C\x1A\x3A\x67\x52\x81\x6A"
-			  "\x04\xFF\x8A\x1B\x96\xFE\xE6\x87"
-			  "\x3C\xD4\x39\x7D\x36\x9B\x03\xD5"
-			  "\xB6\xA0\x75\x3C\x83\xE6\x1C\x73"
-			  "\x9D\x74\x2B\x77\x53\x2D\xE5\xBD"
-			  "\x69\xDA\x7A\x01\xF5\x6A\x70\x39"
-			  "\x30\xD4\x2C\xF2\x8E\x06\x4B\x39"
-			  "\xB3\x12\x1D\xB3\x17\x46\xE6\xD6"
-			  "\xB6\x31\x36\x34\x38\x3C\x1D\x69"
-			  "\x9F\x47\x28\x9A\x1D\x96\x70\x54"
-			  "\x8E\x88\xCB\xE0\xF5\x6A\xAE\x0A"
-			  "\x3C\xD5\x93\x1C\x21\xC9\x14\x3A"
-			  "\x23\x9C\x9B\x79\xC7\x75\xC8\x39"
-			  "\xA6\xAC\x65\x9A\x99\x37\xAF\x6D"
-			  "\xBD\xB5\x32\xFD\xD8\x9C\x95\x7B"
-			  "\xC6\x6A\x80\x64\xEA\xEF\x6D\x3F"
-			  "\xA9\xFE\x5B\x16\xA3\xCF\x32\xC8"
-			  "\xEF\x50\x22\x20\x93\x30\xBE\xE2"
-			  "\x38\x05\x65\xAF\xBA\xB6\xE4\x72"
-			  "\xA9\xEE\x05\x42\x88\xBD\x9D\x49"
-			  "\xAD\x93\xCA\x4D\x45\x11\x43\x4D"
-			  "\xB8\xF5\x74\x2B\x48\xE7\x21\xE4"
-			  "\x4E\x3A\x4C\xDE\x65\x7A\x5A\xAD"
-			  "\x86\xE6\x23\xEC\x6B\xA7\x17\xE6"
-			  "\xF6\xA1\xAC\x29\xAE\xF9\x9B\x69"
-			  "\x73\x65\x65\x51\xD6\x0B\x4E\x8C"
-			  "\x17\x15\x9D\xB0\xCF\xB2\x42\x2B"
-			  "\x51\xC3\x03\xE8\xB7\x7D\x2D\x39"
-			  "\xE8\x10\x93\x16\xC8\x68\x4C\x60"
-			  "\x87\x70\x14\xD0\x01\x57\xCB\x42"
-			  "\x13\x59\xB1\x7F\x12\x4F\xBB\xC7"
-			  "\xBD\x2B\xD4\xA9\x12\x26\x4F\xDE"
-			  "\xFD\x72\xEC\xD7\x6F\x97\x14\x90"
-			  "\x0E\x37\x13\xE6\x67\x1D\xE5\xFE"
-			  "\x9E\x18\x3C\x8F\x3A\x3F\x59\x9B"
-			  "\x71\x80\x05\x35\x3F\x40\x0B\x21"
-			  "\x76\xE5\xEF\x42\x6C\xDB\x31\x05"
-			  "\x5F\x05\xCF\x14\xE3\xF0\x61\xA2"
-			  "\x49\x03\x5E\x77\x2E\x20\xBA\xA1"
-			  "\xAF\x46\x51\xC0\x2B\xC4\x64\x1E"
-			  "\x65\xCC\x51\x58\x0A\xDF\xF0\x5F"
-			  "\x75\x9F\x48\xCD\x81\xEC\xC3\xF6"
-			  "\xED\xC9\x4B\x7B\x4E\x26\x23\xE1"
-			  "\xBB\xE9\x83\x0B\xCF\xE4\xDE\x00"
-			  "\x48\xFF\xBF\x6C\xB4\x72\x16\xEF"
-			  "\xC7\x46\xEE\x48\x8C\xB8\xAF\x45"
-			  "\x91\x76\xE7\x6E\x65\x3D\x15\x86"
-			  "\x10\xF8\xDB\x66\x97\x7C\x43\x4D"
-			  "\x79\x12\x4E\xCE\x06\xD1\xD1\x6A"
-			  "\x34\xC1\xC9\xF2\x28\x4A\xCD\x02"
-			  "\x75\x55\x9B\xFF\x36\x73\xAB\x7C"
-			  "\xF4\x46\x2E\xEB\xAC\xF3\xD2\xB7",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec tnepres_dec_tv_template[] = {
-	{
-		.input	= "\x41\xcc\x6b\x31\x59\x31\x45\x97"
-			  "\x6d\x6f\xbb\x38\x4b\x37\x21\x28",
-		.ilen	= 16,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.klen	= 16,
-		.input	= "\xea\xf4\xd7\xfc\xd8\x01\x34\x47"
-			  "\x81\x45\x0b\xfa\x0c\xd6\xad\x6e",
-		.ilen	= 16,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.klen	= 32,
-		.input	= "\x64\xa9\x1a\x37\xed\x9f\xe7\x49"
-			  "\xa8\x4e\x76\xd6\xf5\x0d\x78\xee",
-		.ilen	= 16,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.rlen	= 16,
-	}, { /* KeySize=128, I=121 */
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80",
-		.klen	= 16,
-		.input	= "\x3d\xda\xbf\xc0\x06\xda\xab\x06"
-			  "\x46\x2a\xf4\xef\x81\x54\x4e\x26",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
-	},
-};
-
-static const struct cipher_testvec serpent_cbc_enc_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\x80\xCF\x11\x41\x1A\xB9\x4B\x9C"
-			  "\xFF\xB7\x6C\xEA\xF0\xAF\x77\x6E"
-			  "\x71\x75\x95\x9D\x4E\x1C\xCF\xAD"
-			  "\x81\x34\xE9\x8F\xAE\x5A\x91\x1C"
-			  "\x38\x63\x35\x7E\x79\x18\x0A\xE8"
-			  "\x67\x06\x76\xD5\xFF\x22\x2F\xDA"
-			  "\xB6\x2D\x57\x13\xB6\x3C\xBC\x97"
-			  "\xFE\x53\x75\x35\x97\x7F\x51\xEA"
-			  "\xDF\x5D\xE8\x9D\xCC\xD9\xAE\xE7"
-			  "\x62\x67\xFF\x04\xC2\x18\x22\x5F"
-			  "\x2E\x06\xC1\xE2\x26\xCD\xC6\x1E"
-			  "\xE5\x2C\x4E\x87\x23\xDD\xF0\x41"
-			  "\x08\xA5\xB4\x3E\x07\x1E\x0B\xBB"
-			  "\x72\x84\xF8\x0A\x3F\x38\x5E\x91"
-			  "\x15\x26\xE1\xDB\xA4\x3D\x74\xD2"
-			  "\x41\x1E\x3F\xA9\xC6\x7D\x2A\xAB"
-			  "\x27\xDF\x89\x1D\x86\x3E\xF7\x5A"
-			  "\xF6\xE3\x0F\xC7\x6B\x4C\x96\x7C"
-			  "\x2D\x12\xA5\x05\x92\xCB\xD7\x4A"
-			  "\x4D\x1E\x88\x21\xE1\x63\xB4\xFC"
-			  "\x4A\xF2\xCD\x35\xB9\xD7\x70\x97"
-			  "\x5A\x5E\x7E\x96\x52\x20\xDC\x25"
-			  "\xE9\x6B\x36\xB4\xE0\x98\x85\x2C"
-			  "\x3C\xD2\xF7\x78\x8A\x73\x26\x9B"
-			  "\xAF\x0B\x11\xE8\x4D\x67\x23\xE9"
-			  "\x77\xDF\x58\xF6\x6F\x9E\xA4\xC5"
-			  "\x10\xA1\x82\x0E\x80\xA0\x8F\x4B"
-			  "\xA1\xC0\x12\x54\x4E\xC9\x20\x92"
-			  "\x11\x00\x10\x4E\xB3\x7C\xCA\x63"
-			  "\xE5\x3F\xD3\x41\x37\xCD\x74\xB7"
-			  "\xA5\x7C\x61\xB8\x0B\x7A\x7F\x4D"
-			  "\xFE\x96\x7D\x1B\xBE\x60\x37\xB7"
-			  "\x81\x92\x66\x67\x15\x1E\x39\x98"
-			  "\x52\xC0\xF4\x69\xC0\x99\x4F\x5A"
-			  "\x2E\x32\xAD\x7C\x8B\xE9\xAD\x05"
-			  "\x55\xF9\x0A\x1F\x97\x5C\xFA\x2B"
-			  "\xF4\x99\x76\x3A\x6E\x4D\xE1\x4C"
-			  "\x14\x4E\x6F\x87\xEE\x1A\x85\xA3"
-			  "\x96\xC6\x66\x49\xDA\x0D\x71\xAC"
-			  "\x04\x05\x46\xD3\x90\x0F\x64\x64"
-			  "\x01\x66\x2C\x62\x5D\x34\xD1\xCB"
-			  "\x3A\x24\xCE\x95\xEF\xAE\x2C\x97"
-			  "\x0E\x0C\x1D\x36\x49\xEB\xE9\x3D"
-			  "\x62\xA6\x19\x28\x9E\x26\xB4\x3F"
-			  "\xD7\x55\x42\x3C\xCD\x72\x0A\xF0"
-			  "\x7D\xE9\x95\x45\x86\xED\xB1\xE0"
-			  "\x8D\xE9\xC5\x86\x13\x24\x28\x7D"
-			  "\x74\xEF\xCA\x50\x12\x7E\x64\x8F"
-			  "\x1B\xF5\x5B\xFE\xE2\xAC\xFA\xE7"
-			  "\xBD\x38\x8C\x11\x20\xEF\xB1\xAA"
-			  "\x7B\xE5\xE5\x78\xAD\x9D\x2D\xA2"
-			  "\x8E\xDD\x48\xB3\xEF\x18\x92\x7E"
-			  "\xE6\x75\x0D\x54\x64\x11\xA3\x3A"
-			  "\xDB\x97\x0F\xD3\xDF\x07\xD3\x7E"
-			  "\x1E\xD1\x87\xE4\x74\xBB\x46\xF4"
-			  "\xBA\x23\x2D\x8D\x29\x07\x12\xCF"
-			  "\x34\xCD\x72\x7F\x01\x30\xE7\xA0"
-			  "\xF8\xDD\xA8\x08\xF0\xBC\xB1\xA2"
-			  "\xCC\xE1\x6B\x5F\xBE\xEA\xF1\xE4"
-			  "\x02\xC4\xAF\xFA\xAD\x31\xF4\xBF"
-			  "\xFC\x66\xAA\x37\xF2\x37\x39\x6B"
-			  "\xBC\x08\x3A\xA2\x29\xB3\xDF\xD1",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec serpent_cbc_dec_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x80\xCF\x11\x41\x1A\xB9\x4B\x9C"
-			  "\xFF\xB7\x6C\xEA\xF0\xAF\x77\x6E"
-			  "\x71\x75\x95\x9D\x4E\x1C\xCF\xAD"
-			  "\x81\x34\xE9\x8F\xAE\x5A\x91\x1C"
-			  "\x38\x63\x35\x7E\x79\x18\x0A\xE8"
-			  "\x67\x06\x76\xD5\xFF\x22\x2F\xDA"
-			  "\xB6\x2D\x57\x13\xB6\x3C\xBC\x97"
-			  "\xFE\x53\x75\x35\x97\x7F\x51\xEA"
-			  "\xDF\x5D\xE8\x9D\xCC\xD9\xAE\xE7"
-			  "\x62\x67\xFF\x04\xC2\x18\x22\x5F"
-			  "\x2E\x06\xC1\xE2\x26\xCD\xC6\x1E"
-			  "\xE5\x2C\x4E\x87\x23\xDD\xF0\x41"
-			  "\x08\xA5\xB4\x3E\x07\x1E\x0B\xBB"
-			  "\x72\x84\xF8\x0A\x3F\x38\x5E\x91"
-			  "\x15\x26\xE1\xDB\xA4\x3D\x74\xD2"
-			  "\x41\x1E\x3F\xA9\xC6\x7D\x2A\xAB"
-			  "\x27\xDF\x89\x1D\x86\x3E\xF7\x5A"
-			  "\xF6\xE3\x0F\xC7\x6B\x4C\x96\x7C"
-			  "\x2D\x12\xA5\x05\x92\xCB\xD7\x4A"
-			  "\x4D\x1E\x88\x21\xE1\x63\xB4\xFC"
-			  "\x4A\xF2\xCD\x35\xB9\xD7\x70\x97"
-			  "\x5A\x5E\x7E\x96\x52\x20\xDC\x25"
-			  "\xE9\x6B\x36\xB4\xE0\x98\x85\x2C"
-			  "\x3C\xD2\xF7\x78\x8A\x73\x26\x9B"
-			  "\xAF\x0B\x11\xE8\x4D\x67\x23\xE9"
-			  "\x77\xDF\x58\xF6\x6F\x9E\xA4\xC5"
-			  "\x10\xA1\x82\x0E\x80\xA0\x8F\x4B"
-			  "\xA1\xC0\x12\x54\x4E\xC9\x20\x92"
-			  "\x11\x00\x10\x4E\xB3\x7C\xCA\x63"
-			  "\xE5\x3F\xD3\x41\x37\xCD\x74\xB7"
-			  "\xA5\x7C\x61\xB8\x0B\x7A\x7F\x4D"
-			  "\xFE\x96\x7D\x1B\xBE\x60\x37\xB7"
-			  "\x81\x92\x66\x67\x15\x1E\x39\x98"
-			  "\x52\xC0\xF4\x69\xC0\x99\x4F\x5A"
-			  "\x2E\x32\xAD\x7C\x8B\xE9\xAD\x05"
-			  "\x55\xF9\x0A\x1F\x97\x5C\xFA\x2B"
-			  "\xF4\x99\x76\x3A\x6E\x4D\xE1\x4C"
-			  "\x14\x4E\x6F\x87\xEE\x1A\x85\xA3"
-			  "\x96\xC6\x66\x49\xDA\x0D\x71\xAC"
-			  "\x04\x05\x46\xD3\x90\x0F\x64\x64"
-			  "\x01\x66\x2C\x62\x5D\x34\xD1\xCB"
-			  "\x3A\x24\xCE\x95\xEF\xAE\x2C\x97"
-			  "\x0E\x0C\x1D\x36\x49\xEB\xE9\x3D"
-			  "\x62\xA6\x19\x28\x9E\x26\xB4\x3F"
-			  "\xD7\x55\x42\x3C\xCD\x72\x0A\xF0"
-			  "\x7D\xE9\x95\x45\x86\xED\xB1\xE0"
-			  "\x8D\xE9\xC5\x86\x13\x24\x28\x7D"
-			  "\x74\xEF\xCA\x50\x12\x7E\x64\x8F"
-			  "\x1B\xF5\x5B\xFE\xE2\xAC\xFA\xE7"
-			  "\xBD\x38\x8C\x11\x20\xEF\xB1\xAA"
-			  "\x7B\xE5\xE5\x78\xAD\x9D\x2D\xA2"
-			  "\x8E\xDD\x48\xB3\xEF\x18\x92\x7E"
-			  "\xE6\x75\x0D\x54\x64\x11\xA3\x3A"
-			  "\xDB\x97\x0F\xD3\xDF\x07\xD3\x7E"
-			  "\x1E\xD1\x87\xE4\x74\xBB\x46\xF4"
-			  "\xBA\x23\x2D\x8D\x29\x07\x12\xCF"
-			  "\x34\xCD\x72\x7F\x01\x30\xE7\xA0"
-			  "\xF8\xDD\xA8\x08\xF0\xBC\xB1\xA2"
-			  "\xCC\xE1\x6B\x5F\xBE\xEA\xF1\xE4"
-			  "\x02\xC4\xAF\xFA\xAD\x31\xF4\xBF"
-			  "\xFC\x66\xAA\x37\xF2\x37\x39\x6B"
-			  "\xBC\x08\x3A\xA2\x29\xB3\xDF\xD1",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec serpent_ctr_enc_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\x84\x68\xEC\xF2\x1C\x88\x20\xCA"
-			  "\x37\x69\xE3\x3A\x22\x85\x48\x46"
-			  "\x70\xAA\x25\xB4\xCD\x8B\x04\x4E"
-			  "\x8D\x15\x2B\x98\xDF\x7B\x6D\xB9"
-			  "\xE0\x4A\x73\x00\x65\xB6\x1A\x0D"
-			  "\x5C\x60\xDF\x34\xDC\x60\x4C\xDF"
-			  "\xB5\x1F\x26\x8C\xDA\xC1\x11\xA8"
-			  "\x80\xFA\x37\x7A\x89\xAA\xAE\x7B"
-			  "\x92\x6E\xB9\xDC\xC9\x62\x4F\x88"
-			  "\x0A\x5D\x97\x2F\x6B\xAC\x03\x7C"
-			  "\x22\xF6\x55\x5A\xFA\x35\xA5\x17"
-			  "\xA1\x5C\x5E\x2B\x63\x2D\xB9\x91"
-			  "\x3E\x83\x26\x00\x4E\xD5\xBE\xCE"
-			  "\x79\xC4\x3D\xFC\x70\xA0\xAD\x96"
-			  "\xBA\x58\x2A\x1C\xDF\xC2\x3A\xA5"
-			  "\x7C\xB5\x12\x89\xED\xBF\xB6\x09"
-			  "\x13\x4F\x7D\x61\x3C\x5C\x27\xFC"
-			  "\x5D\xE1\x4F\xA1\xEA\xB3\xCA\xB9"
-			  "\xE6\xD0\x97\x81\xDE\xD1\xFB\x8A"
-			  "\x30\xDB\xA3\x5D\xEC\x25\x0B\x86"
-			  "\x71\xC8\xA7\x67\xE8\xBC\x7D\x4C"
-			  "\xAE\x82\xD3\x73\x31\x09\xCB\xB3"
-			  "\x4D\xD4\xC0\x8A\x2B\xFA\xA6\x55"
-			  "\x39\x0A\xBC\x6E\x75\xAB\xC2\xE2"
-			  "\x8A\xF2\x26\xCD\x63\x38\x35\xF7"
-			  "\xAE\x12\x83\xCD\x8A\x9E\x7E\x4C"
-			  "\xFE\x4D\xD7\xCE\x5C\x6E\x4C\xAF"
-			  "\xE3\xCD\x76\xA7\x87\xA1\x54\x7C"
-			  "\xEC\x32\xC7\x83\x2A\xFF\xF8\xEA"
-			  "\x87\xB2\x47\xA3\x9D\xC2\x9C\xA2"
-			  "\xB7\x2C\x7C\x1A\x24\xCB\x88\x61"
-			  "\xFF\xA7\x1A\x16\x01\xDD\x4B\xFC"
-			  "\x2E\xE0\x48\x67\x09\x42\xCC\x91"
-			  "\xBE\x20\x38\xC0\x5E\x3B\x95\x00"
-			  "\xA1\x96\x66\x0B\x8A\xE9\x9E\xF7"
-			  "\x6B\x34\x0A\x51\xC0\x3B\xEB\x71"
-			  "\x07\x97\x38\x4B\x5C\x56\x98\x67"
-			  "\x78\x9C\xD0\x0E\x2B\xB5\x67\x90"
-			  "\x75\xF8\xFE\x6D\x4E\x85\xCC\x0D"
-			  "\x18\x06\x15\x9D\x5A\x10\x13\x37"
-			  "\xA3\xD6\x68\xA2\xDF\x7E\xC7\x12"
-			  "\xC9\x0D\x4D\x91\xB0\x2A\x55\xFF"
-			  "\x6F\x73\x13\xDF\x28\xB5\x2A\x2C"
-			  "\xE4\xFC\x20\xD9\xF1\x7A\x82\xB1"
-			  "\xCB\x57\xB6\x3D\x8C\xF4\x8E\x27"
-			  "\x37\xDC\x35\xF3\x79\x01\x53\xA4"
-			  "\x7B\x37\xDE\x7C\x04\xAE\x50\xDB"
-			  "\x9B\x1E\x8C\x07\xA7\x52\x49\x50"
-			  "\x34\x25\x65\xDD\xA9\x8F\x7E\xBD"
-			  "\x7A\xC9\x36\xAE\xDE\x21\x48\x64"
-			  "\xC2\x02\xBA\xBE\x11\x1E\x3D\x9C"
-			  "\x98\x52\xCC\x04\xBD\x5E\x61\x26"
-			  "\x10\xD3\x21\xD9\x6E\x25\x98\x77"
-			  "\x8E\x98\x63\xF6\xF6\x52\xFB\x13"
-			  "\xAA\x30\xF2\xB9\xA4\x43\x53\x39"
-			  "\x1C\x97\x07\x7E\x6B\xFF\x3D\x43"
-			  "\xA6\x71\x6B\x66\x8F\x58\x3F\x71"
-			  "\x90\x47\x40\x92\xE6\x69\xD1\x96"
-			  "\x34\xB3\x3B\xE5\x43\xE4\xD5\x56"
-			  "\xB2\xE6\x7E\x86\x7A\x12\x17\x5B"
-			  "\x30\xF3\x9B\x0D\xFA\x57\xE4\x50"
-			  "\x40\x53\x77\x8C\x15\xF8\x8D\x13",
-		.rlen	= 496,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59",
-		.ilen	= 499,
-		.result	= "\x84\x68\xEC\xF2\x1C\x88\x20\xCA"
-			  "\x37\x69\xE3\x3A\x22\x85\x48\x46"
-			  "\x70\xAA\x25\xB4\xCD\x8B\x04\x4E"
-			  "\x8D\x15\x2B\x98\xDF\x7B\x6D\xB9"
-			  "\xE0\x4A\x73\x00\x65\xB6\x1A\x0D"
-			  "\x5C\x60\xDF\x34\xDC\x60\x4C\xDF"
-			  "\xB5\x1F\x26\x8C\xDA\xC1\x11\xA8"
-			  "\x80\xFA\x37\x7A\x89\xAA\xAE\x7B"
-			  "\x92\x6E\xB9\xDC\xC9\x62\x4F\x88"
-			  "\x0A\x5D\x97\x2F\x6B\xAC\x03\x7C"
-			  "\x22\xF6\x55\x5A\xFA\x35\xA5\x17"
-			  "\xA1\x5C\x5E\x2B\x63\x2D\xB9\x91"
-			  "\x3E\x83\x26\x00\x4E\xD5\xBE\xCE"
-			  "\x79\xC4\x3D\xFC\x70\xA0\xAD\x96"
-			  "\xBA\x58\x2A\x1C\xDF\xC2\x3A\xA5"
-			  "\x7C\xB5\x12\x89\xED\xBF\xB6\x09"
-			  "\x13\x4F\x7D\x61\x3C\x5C\x27\xFC"
-			  "\x5D\xE1\x4F\xA1\xEA\xB3\xCA\xB9"
-			  "\xE6\xD0\x97\x81\xDE\xD1\xFB\x8A"
-			  "\x30\xDB\xA3\x5D\xEC\x25\x0B\x86"
-			  "\x71\xC8\xA7\x67\xE8\xBC\x7D\x4C"
-			  "\xAE\x82\xD3\x73\x31\x09\xCB\xB3"
-			  "\x4D\xD4\xC0\x8A\x2B\xFA\xA6\x55"
-			  "\x39\x0A\xBC\x6E\x75\xAB\xC2\xE2"
-			  "\x8A\xF2\x26\xCD\x63\x38\x35\xF7"
-			  "\xAE\x12\x83\xCD\x8A\x9E\x7E\x4C"
-			  "\xFE\x4D\xD7\xCE\x5C\x6E\x4C\xAF"
-			  "\xE3\xCD\x76\xA7\x87\xA1\x54\x7C"
-			  "\xEC\x32\xC7\x83\x2A\xFF\xF8\xEA"
-			  "\x87\xB2\x47\xA3\x9D\xC2\x9C\xA2"
-			  "\xB7\x2C\x7C\x1A\x24\xCB\x88\x61"
-			  "\xFF\xA7\x1A\x16\x01\xDD\x4B\xFC"
-			  "\x2E\xE0\x48\x67\x09\x42\xCC\x91"
-			  "\xBE\x20\x38\xC0\x5E\x3B\x95\x00"
-			  "\xA1\x96\x66\x0B\x8A\xE9\x9E\xF7"
-			  "\x6B\x34\x0A\x51\xC0\x3B\xEB\x71"
-			  "\x07\x97\x38\x4B\x5C\x56\x98\x67"
-			  "\x78\x9C\xD0\x0E\x2B\xB5\x67\x90"
-			  "\x75\xF8\xFE\x6D\x4E\x85\xCC\x0D"
-			  "\x18\x06\x15\x9D\x5A\x10\x13\x37"
-			  "\xA3\xD6\x68\xA2\xDF\x7E\xC7\x12"
-			  "\xC9\x0D\x4D\x91\xB0\x2A\x55\xFF"
-			  "\x6F\x73\x13\xDF\x28\xB5\x2A\x2C"
-			  "\xE4\xFC\x20\xD9\xF1\x7A\x82\xB1"
-			  "\xCB\x57\xB6\x3D\x8C\xF4\x8E\x27"
-			  "\x37\xDC\x35\xF3\x79\x01\x53\xA4"
-			  "\x7B\x37\xDE\x7C\x04\xAE\x50\xDB"
-			  "\x9B\x1E\x8C\x07\xA7\x52\x49\x50"
-			  "\x34\x25\x65\xDD\xA9\x8F\x7E\xBD"
-			  "\x7A\xC9\x36\xAE\xDE\x21\x48\x64"
-			  "\xC2\x02\xBA\xBE\x11\x1E\x3D\x9C"
-			  "\x98\x52\xCC\x04\xBD\x5E\x61\x26"
-			  "\x10\xD3\x21\xD9\x6E\x25\x98\x77"
-			  "\x8E\x98\x63\xF6\xF6\x52\xFB\x13"
-			  "\xAA\x30\xF2\xB9\xA4\x43\x53\x39"
-			  "\x1C\x97\x07\x7E\x6B\xFF\x3D\x43"
-			  "\xA6\x71\x6B\x66\x8F\x58\x3F\x71"
-			  "\x90\x47\x40\x92\xE6\x69\xD1\x96"
-			  "\x34\xB3\x3B\xE5\x43\xE4\xD5\x56"
-			  "\xB2\xE6\x7E\x86\x7A\x12\x17\x5B"
-			  "\x30\xF3\x9B\x0D\xFA\x57\xE4\x50"
-			  "\x40\x53\x77\x8C\x15\xF8\x8D\x13"
-			  "\x38\xE2\xE5",
-		.rlen	= 499,
-		.also_non_np = 1,
-		.np	= 2,
-		.tap	= { 499 - 16, 16 },
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\x06\x9A\xF8\xB4\x53\x88\x62\xFC"
-			  "\x68\xB8\x2E\xDF\xC1\x05\x0F\x3D"
-			  "\xAF\x4D\x95\xAE\xC4\xE9\x1C\xDC"
-			  "\xF6\x2B\x8F\x90\x89\xF6\x7E\x1A"
-			  "\xA6\xB9\xE4\xF4\xFA\xCA\xE5\x7E"
-			  "\x71\x28\x06\x4F\xE8\x08\x39\xDA"
-			  "\xA5\x0E\xC8\xC0\xB8\x16\xE5\x69"
-			  "\xE5\xCA\xEC\x4F\x63\x2C\xC0\x9B"
-			  "\x9F\x3E\x39\x79\xF0\xCD\x64\x35"
-			  "\x4A\xD3\xC8\xA9\x31\xCD\x48\x5B"
-			  "\x92\x3D\x8F\x3F\x96\xBD\xB3\x18"
-			  "\x74\x2A\x5D\x29\x3F\x57\x8F\xE2"
-			  "\x67\x9A\xE0\xE5\xD4\x4A\xE2\x47"
-			  "\xBC\xF6\xEB\x14\xF3\x8C\x20\xC2"
-			  "\x7D\xE2\x43\x81\x86\x72\x2E\xB1"
-			  "\x39\xF6\x95\xE1\x1F\xCB\x76\x33"
-			  "\x5B\x7D\x23\x0F\x3A\x67\x2A\x2F"
-			  "\xB9\x37\x9D\xDD\x1F\x16\xA1\x3C"
-			  "\x70\xFE\x52\xAA\x93\x3C\xC4\x46"
-			  "\xB1\xE5\xFF\xDA\xAF\xE2\x84\xFE"
-			  "\x25\x92\xB2\x63\xBD\x49\x77\xB4"
-			  "\x22\xA4\x6A\xD5\x04\xE0\x45\x58"
-			  "\x1C\x34\x96\x7C\x03\x0C\x13\xA2"
-			  "\x05\x22\xE2\xCB\x5A\x35\x03\x09"
-			  "\x40\xD2\x82\x05\xCA\x58\x73\xF2"
-			  "\x29\x5E\x01\x47\x13\x32\x78\xBE"
-			  "\x06\xB0\x51\xDB\x6C\x31\xA0\x1C"
-			  "\x74\xBC\x8D\x25\xDF\xF8\x65\xD1"
-			  "\x38\x35\x11\x26\x4A\xB4\x06\x32"
-			  "\xFA\xD2\x07\x77\xB3\x74\x98\x80"
-			  "\x61\x59\xA8\x9F\xF3\x6F\x2A\xBF"
-			  "\xE6\xA5\x9A\xC4\x6B\xA6\x49\x6F"
-			  "\xBC\x47\xD9\xFB\xC6\xEF\x25\x65"
-			  "\x96\xAC\x9F\xE4\x81\x4B\xD8\xBA"
-			  "\xD6\x9B\xC9\x6D\x58\x40\x81\x02"
-			  "\x73\x44\x4E\x43\x6E\x37\xBB\x11"
-			  "\xE3\xF9\xB8\x2F\xEC\x76\x34\xEA"
-			  "\x90\xCD\xB7\x2E\x0E\x32\x71\xE8"
-			  "\xBB\x4E\x0B\x98\xA4\x17\x17\x5B"
-			  "\x07\xB5\x82\x3A\xC4\xE8\x42\x51"
-			  "\x5A\x4C\x4E\x7D\xBF\xC4\xC0\x4F"
-			  "\x68\xB8\xC6\x4A\x32\x6F\x0B\xD7"
-			  "\x85\xED\x6B\xFB\x72\xD2\xA5\x8F"
-			  "\xBF\xF9\xAC\x59\x50\xA8\x08\x70"
-			  "\xEC\xBD\x0A\xBF\xE5\x87\xA1\xC2"
-			  "\x92\x14\x78\xAF\xE8\xEA\x2E\xDD"
-			  "\xC1\x03\x9A\xAA\x89\x8B\x32\x46"
-			  "\x5B\x18\x27\xBA\x46\xAA\x64\xDE"
-			  "\xE3\xD5\xA3\xFC\x7B\x5B\x61\xDB"
-			  "\x7E\xDA\xEC\x30\x17\x19\xF8\x80"
-			  "\xB5\x5E\x27\xB5\x37\x3A\x1F\x28"
-			  "\x07\x73\xC3\x63\xCE\xFF\x8C\xFE"
-			  "\x81\x4E\xF8\x24\xF3\xB8\xC7\xE8"
-			  "\x16\x9A\xCC\x58\x2F\x88\x1C\x4B"
-			  "\xBB\x33\xA2\x73\xF0\x1C\x89\x0E"
-			  "\xDC\x34\x27\x89\x98\xCE\x1C\xA2"
-			  "\xD8\xB8\x90\xBE\xEC\x72\x28\x13"
-			  "\xAC\x7B\xF1\xD0\x7F\x7A\x28\x50"
-			  "\xB7\x99\x65\x8A\xC9\xC6\x21\x34"
-			  "\x7F\x67\x9D\xB7\x2C\xCC\xF5\x17"
-			  "\x2B\x89\xAC\xB0\xD7\x1E\x47\xB0"
-			  "\x61\xAF\xD4\x63\x6D\xB8\x2D\x20",
-		.rlen	= 496,
-	},
-};
-
-static const struct cipher_testvec serpent_ctr_dec_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x84\x68\xEC\xF2\x1C\x88\x20\xCA"
-			  "\x37\x69\xE3\x3A\x22\x85\x48\x46"
-			  "\x70\xAA\x25\xB4\xCD\x8B\x04\x4E"
-			  "\x8D\x15\x2B\x98\xDF\x7B\x6D\xB9"
-			  "\xE0\x4A\x73\x00\x65\xB6\x1A\x0D"
-			  "\x5C\x60\xDF\x34\xDC\x60\x4C\xDF"
-			  "\xB5\x1F\x26\x8C\xDA\xC1\x11\xA8"
-			  "\x80\xFA\x37\x7A\x89\xAA\xAE\x7B"
-			  "\x92\x6E\xB9\xDC\xC9\x62\x4F\x88"
-			  "\x0A\x5D\x97\x2F\x6B\xAC\x03\x7C"
-			  "\x22\xF6\x55\x5A\xFA\x35\xA5\x17"
-			  "\xA1\x5C\x5E\x2B\x63\x2D\xB9\x91"
-			  "\x3E\x83\x26\x00\x4E\xD5\xBE\xCE"
-			  "\x79\xC4\x3D\xFC\x70\xA0\xAD\x96"
-			  "\xBA\x58\x2A\x1C\xDF\xC2\x3A\xA5"
-			  "\x7C\xB5\x12\x89\xED\xBF\xB6\x09"
-			  "\x13\x4F\x7D\x61\x3C\x5C\x27\xFC"
-			  "\x5D\xE1\x4F\xA1\xEA\xB3\xCA\xB9"
-			  "\xE6\xD0\x97\x81\xDE\xD1\xFB\x8A"
-			  "\x30\xDB\xA3\x5D\xEC\x25\x0B\x86"
-			  "\x71\xC8\xA7\x67\xE8\xBC\x7D\x4C"
-			  "\xAE\x82\xD3\x73\x31\x09\xCB\xB3"
-			  "\x4D\xD4\xC0\x8A\x2B\xFA\xA6\x55"
-			  "\x39\x0A\xBC\x6E\x75\xAB\xC2\xE2"
-			  "\x8A\xF2\x26\xCD\x63\x38\x35\xF7"
-			  "\xAE\x12\x83\xCD\x8A\x9E\x7E\x4C"
-			  "\xFE\x4D\xD7\xCE\x5C\x6E\x4C\xAF"
-			  "\xE3\xCD\x76\xA7\x87\xA1\x54\x7C"
-			  "\xEC\x32\xC7\x83\x2A\xFF\xF8\xEA"
-			  "\x87\xB2\x47\xA3\x9D\xC2\x9C\xA2"
-			  "\xB7\x2C\x7C\x1A\x24\xCB\x88\x61"
-			  "\xFF\xA7\x1A\x16\x01\xDD\x4B\xFC"
-			  "\x2E\xE0\x48\x67\x09\x42\xCC\x91"
-			  "\xBE\x20\x38\xC0\x5E\x3B\x95\x00"
-			  "\xA1\x96\x66\x0B\x8A\xE9\x9E\xF7"
-			  "\x6B\x34\x0A\x51\xC0\x3B\xEB\x71"
-			  "\x07\x97\x38\x4B\x5C\x56\x98\x67"
-			  "\x78\x9C\xD0\x0E\x2B\xB5\x67\x90"
-			  "\x75\xF8\xFE\x6D\x4E\x85\xCC\x0D"
-			  "\x18\x06\x15\x9D\x5A\x10\x13\x37"
-			  "\xA3\xD6\x68\xA2\xDF\x7E\xC7\x12"
-			  "\xC9\x0D\x4D\x91\xB0\x2A\x55\xFF"
-			  "\x6F\x73\x13\xDF\x28\xB5\x2A\x2C"
-			  "\xE4\xFC\x20\xD9\xF1\x7A\x82\xB1"
-			  "\xCB\x57\xB6\x3D\x8C\xF4\x8E\x27"
-			  "\x37\xDC\x35\xF3\x79\x01\x53\xA4"
-			  "\x7B\x37\xDE\x7C\x04\xAE\x50\xDB"
-			  "\x9B\x1E\x8C\x07\xA7\x52\x49\x50"
-			  "\x34\x25\x65\xDD\xA9\x8F\x7E\xBD"
-			  "\x7A\xC9\x36\xAE\xDE\x21\x48\x64"
-			  "\xC2\x02\xBA\xBE\x11\x1E\x3D\x9C"
-			  "\x98\x52\xCC\x04\xBD\x5E\x61\x26"
-			  "\x10\xD3\x21\xD9\x6E\x25\x98\x77"
-			  "\x8E\x98\x63\xF6\xF6\x52\xFB\x13"
-			  "\xAA\x30\xF2\xB9\xA4\x43\x53\x39"
-			  "\x1C\x97\x07\x7E\x6B\xFF\x3D\x43"
-			  "\xA6\x71\x6B\x66\x8F\x58\x3F\x71"
-			  "\x90\x47\x40\x92\xE6\x69\xD1\x96"
-			  "\x34\xB3\x3B\xE5\x43\xE4\xD5\x56"
-			  "\xB2\xE6\x7E\x86\x7A\x12\x17\x5B"
-			  "\x30\xF3\x9B\x0D\xFA\x57\xE4\x50"
-			  "\x40\x53\x77\x8C\x15\xF8\x8D\x13",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x84\x68\xEC\xF2\x1C\x88\x20\xCA"
-			  "\x37\x69\xE3\x3A\x22\x85\x48\x46"
-			  "\x70\xAA\x25\xB4\xCD\x8B\x04\x4E"
-			  "\x8D\x15\x2B\x98\xDF\x7B\x6D\xB9"
-			  "\xE0\x4A\x73\x00\x65\xB6\x1A\x0D"
-			  "\x5C\x60\xDF\x34\xDC\x60\x4C\xDF"
-			  "\xB5\x1F\x26\x8C\xDA\xC1\x11\xA8"
-			  "\x80\xFA\x37\x7A\x89\xAA\xAE\x7B"
-			  "\x92\x6E\xB9\xDC\xC9\x62\x4F\x88"
-			  "\x0A\x5D\x97\x2F\x6B\xAC\x03\x7C"
-			  "\x22\xF6\x55\x5A\xFA\x35\xA5\x17"
-			  "\xA1\x5C\x5E\x2B\x63\x2D\xB9\x91"
-			  "\x3E\x83\x26\x00\x4E\xD5\xBE\xCE"
-			  "\x79\xC4\x3D\xFC\x70\xA0\xAD\x96"
-			  "\xBA\x58\x2A\x1C\xDF\xC2\x3A\xA5"
-			  "\x7C\xB5\x12\x89\xED\xBF\xB6\x09"
-			  "\x13\x4F\x7D\x61\x3C\x5C\x27\xFC"
-			  "\x5D\xE1\x4F\xA1\xEA\xB3\xCA\xB9"
-			  "\xE6\xD0\x97\x81\xDE\xD1\xFB\x8A"
-			  "\x30\xDB\xA3\x5D\xEC\x25\x0B\x86"
-			  "\x71\xC8\xA7\x67\xE8\xBC\x7D\x4C"
-			  "\xAE\x82\xD3\x73\x31\x09\xCB\xB3"
-			  "\x4D\xD4\xC0\x8A\x2B\xFA\xA6\x55"
-			  "\x39\x0A\xBC\x6E\x75\xAB\xC2\xE2"
-			  "\x8A\xF2\x26\xCD\x63\x38\x35\xF7"
-			  "\xAE\x12\x83\xCD\x8A\x9E\x7E\x4C"
-			  "\xFE\x4D\xD7\xCE\x5C\x6E\x4C\xAF"
-			  "\xE3\xCD\x76\xA7\x87\xA1\x54\x7C"
-			  "\xEC\x32\xC7\x83\x2A\xFF\xF8\xEA"
-			  "\x87\xB2\x47\xA3\x9D\xC2\x9C\xA2"
-			  "\xB7\x2C\x7C\x1A\x24\xCB\x88\x61"
-			  "\xFF\xA7\x1A\x16\x01\xDD\x4B\xFC"
-			  "\x2E\xE0\x48\x67\x09\x42\xCC\x91"
-			  "\xBE\x20\x38\xC0\x5E\x3B\x95\x00"
-			  "\xA1\x96\x66\x0B\x8A\xE9\x9E\xF7"
-			  "\x6B\x34\x0A\x51\xC0\x3B\xEB\x71"
-			  "\x07\x97\x38\x4B\x5C\x56\x98\x67"
-			  "\x78\x9C\xD0\x0E\x2B\xB5\x67\x90"
-			  "\x75\xF8\xFE\x6D\x4E\x85\xCC\x0D"
-			  "\x18\x06\x15\x9D\x5A\x10\x13\x37"
-			  "\xA3\xD6\x68\xA2\xDF\x7E\xC7\x12"
-			  "\xC9\x0D\x4D\x91\xB0\x2A\x55\xFF"
-			  "\x6F\x73\x13\xDF\x28\xB5\x2A\x2C"
-			  "\xE4\xFC\x20\xD9\xF1\x7A\x82\xB1"
-			  "\xCB\x57\xB6\x3D\x8C\xF4\x8E\x27"
-			  "\x37\xDC\x35\xF3\x79\x01\x53\xA4"
-			  "\x7B\x37\xDE\x7C\x04\xAE\x50\xDB"
-			  "\x9B\x1E\x8C\x07\xA7\x52\x49\x50"
-			  "\x34\x25\x65\xDD\xA9\x8F\x7E\xBD"
-			  "\x7A\xC9\x36\xAE\xDE\x21\x48\x64"
-			  "\xC2\x02\xBA\xBE\x11\x1E\x3D\x9C"
-			  "\x98\x52\xCC\x04\xBD\x5E\x61\x26"
-			  "\x10\xD3\x21\xD9\x6E\x25\x98\x77"
-			  "\x8E\x98\x63\xF6\xF6\x52\xFB\x13"
-			  "\xAA\x30\xF2\xB9\xA4\x43\x53\x39"
-			  "\x1C\x97\x07\x7E\x6B\xFF\x3D\x43"
-			  "\xA6\x71\x6B\x66\x8F\x58\x3F\x71"
-			  "\x90\x47\x40\x92\xE6\x69\xD1\x96"
-			  "\x34\xB3\x3B\xE5\x43\xE4\xD5\x56"
-			  "\xB2\xE6\x7E\x86\x7A\x12\x17\x5B"
-			  "\x30\xF3\x9B\x0D\xFA\x57\xE4\x50"
-			  "\x40\x53\x77\x8C\x15\xF8\x8D\x13"
-			  "\x38\xE2\xE5",
-		.ilen	= 499,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59",
-		.rlen	= 499,
-		.also_non_np = 1,
-		.np	= 2,
-		.tap	= { 499 - 16, 16 },
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x06\x9A\xF8\xB4\x53\x88\x62\xFC"
-			  "\x68\xB8\x2E\xDF\xC1\x05\x0F\x3D"
-			  "\xAF\x4D\x95\xAE\xC4\xE9\x1C\xDC"
-			  "\xF6\x2B\x8F\x90\x89\xF6\x7E\x1A"
-			  "\xA6\xB9\xE4\xF4\xFA\xCA\xE5\x7E"
-			  "\x71\x28\x06\x4F\xE8\x08\x39\xDA"
-			  "\xA5\x0E\xC8\xC0\xB8\x16\xE5\x69"
-			  "\xE5\xCA\xEC\x4F\x63\x2C\xC0\x9B"
-			  "\x9F\x3E\x39\x79\xF0\xCD\x64\x35"
-			  "\x4A\xD3\xC8\xA9\x31\xCD\x48\x5B"
-			  "\x92\x3D\x8F\x3F\x96\xBD\xB3\x18"
-			  "\x74\x2A\x5D\x29\x3F\x57\x8F\xE2"
-			  "\x67\x9A\xE0\xE5\xD4\x4A\xE2\x47"
-			  "\xBC\xF6\xEB\x14\xF3\x8C\x20\xC2"
-			  "\x7D\xE2\x43\x81\x86\x72\x2E\xB1"
-			  "\x39\xF6\x95\xE1\x1F\xCB\x76\x33"
-			  "\x5B\x7D\x23\x0F\x3A\x67\x2A\x2F"
-			  "\xB9\x37\x9D\xDD\x1F\x16\xA1\x3C"
-			  "\x70\xFE\x52\xAA\x93\x3C\xC4\x46"
-			  "\xB1\xE5\xFF\xDA\xAF\xE2\x84\xFE"
-			  "\x25\x92\xB2\x63\xBD\x49\x77\xB4"
-			  "\x22\xA4\x6A\xD5\x04\xE0\x45\x58"
-			  "\x1C\x34\x96\x7C\x03\x0C\x13\xA2"
-			  "\x05\x22\xE2\xCB\x5A\x35\x03\x09"
-			  "\x40\xD2\x82\x05\xCA\x58\x73\xF2"
-			  "\x29\x5E\x01\x47\x13\x32\x78\xBE"
-			  "\x06\xB0\x51\xDB\x6C\x31\xA0\x1C"
-			  "\x74\xBC\x8D\x25\xDF\xF8\x65\xD1"
-			  "\x38\x35\x11\x26\x4A\xB4\x06\x32"
-			  "\xFA\xD2\x07\x77\xB3\x74\x98\x80"
-			  "\x61\x59\xA8\x9F\xF3\x6F\x2A\xBF"
-			  "\xE6\xA5\x9A\xC4\x6B\xA6\x49\x6F"
-			  "\xBC\x47\xD9\xFB\xC6\xEF\x25\x65"
-			  "\x96\xAC\x9F\xE4\x81\x4B\xD8\xBA"
-			  "\xD6\x9B\xC9\x6D\x58\x40\x81\x02"
-			  "\x73\x44\x4E\x43\x6E\x37\xBB\x11"
-			  "\xE3\xF9\xB8\x2F\xEC\x76\x34\xEA"
-			  "\x90\xCD\xB7\x2E\x0E\x32\x71\xE8"
-			  "\xBB\x4E\x0B\x98\xA4\x17\x17\x5B"
-			  "\x07\xB5\x82\x3A\xC4\xE8\x42\x51"
-			  "\x5A\x4C\x4E\x7D\xBF\xC4\xC0\x4F"
-			  "\x68\xB8\xC6\x4A\x32\x6F\x0B\xD7"
-			  "\x85\xED\x6B\xFB\x72\xD2\xA5\x8F"
-			  "\xBF\xF9\xAC\x59\x50\xA8\x08\x70"
-			  "\xEC\xBD\x0A\xBF\xE5\x87\xA1\xC2"
-			  "\x92\x14\x78\xAF\xE8\xEA\x2E\xDD"
-			  "\xC1\x03\x9A\xAA\x89\x8B\x32\x46"
-			  "\x5B\x18\x27\xBA\x46\xAA\x64\xDE"
-			  "\xE3\xD5\xA3\xFC\x7B\x5B\x61\xDB"
-			  "\x7E\xDA\xEC\x30\x17\x19\xF8\x80"
-			  "\xB5\x5E\x27\xB5\x37\x3A\x1F\x28"
-			  "\x07\x73\xC3\x63\xCE\xFF\x8C\xFE"
-			  "\x81\x4E\xF8\x24\xF3\xB8\xC7\xE8"
-			  "\x16\x9A\xCC\x58\x2F\x88\x1C\x4B"
-			  "\xBB\x33\xA2\x73\xF0\x1C\x89\x0E"
-			  "\xDC\x34\x27\x89\x98\xCE\x1C\xA2"
-			  "\xD8\xB8\x90\xBE\xEC\x72\x28\x13"
-			  "\xAC\x7B\xF1\xD0\x7F\x7A\x28\x50"
-			  "\xB7\x99\x65\x8A\xC9\xC6\x21\x34"
-			  "\x7F\x67\x9D\xB7\x2C\xCC\xF5\x17"
-			  "\x2B\x89\xAC\xB0\xD7\x1E\x47\xB0"
-			  "\x61\xAF\xD4\x63\x6D\xB8\x2D\x20",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-	},
-};
-
-static const struct cipher_testvec serpent_lrw_enc_tv_template[] = {
-	/* Generated from AES-LRW test vectors */
-	{
-		.key	= "\x45\x62\xac\x25\xf8\x28\x17\x6d"
-			  "\x4c\x26\x84\x14\xb5\x68\x01\x85"
-			  "\x25\x8e\x2a\x05\xe7\x3e\x9d\x03"
-			  "\xee\x5a\x83\x0c\xcc\x09\x4c\x87",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x6f\xbf\xd4\xa4\x5d\x71\x16\x79"
-			  "\x63\x9c\xa6\x8e\x40\xbe\x0d\x8a",
-		.rlen	= 16,
-	}, {
-		.key	= "\x59\x70\x47\x14\xf5\x57\x47\x8c"
-			  "\xd7\x79\xe8\x0f\x54\x88\x79\x44"
-			  "\x0d\x48\xf0\xb7\xb1\x5a\x53\xea"
-			  "\x1c\xaa\x6b\x29\xc2\xca\xfb\xaf",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x02",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\xfd\xb2\x66\x98\x80\x96\x55\xad"
-			  "\x08\x94\x54\x9c\x21\x7c\x69\xe3",
-		.rlen	= 16,
-	}, {
-		.key	= "\xd8\x2a\x91\x34\xb2\x6a\x56\x50"
-			  "\x30\xfe\x69\xe2\x37\x7f\x98\x47"
-			  "\xcd\xf9\x0b\x16\x0c\x64\x8f\xb6"
-			  "\xb0\x0d\x0d\x1b\xae\x85\x87\x1f",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x14\x5e\x3d\x70\xc0\x6e\x9c\x34"
-			  "\x5b\x5e\xcf\x0f\xe4\x8c\x21\x5c",
-		.rlen	= 16,
-	}, {
-		.key	= "\x0f\x6a\xef\xf8\xd3\xd2\xbb\x15"
-			  "\x25\x83\xf7\x3c\x1f\x01\x28\x74"
-			  "\xca\xc6\xbc\x35\x4d\x4a\x65\x54"
-			  "\x90\xae\x61\xcf\x7b\xae\xbd\xcc"
-			  "\xad\xe4\x94\xc5\x4a\x29\xae\x70",
-		.klen	= 40,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x25\x39\xaa\xa5\xf0\x65\xc8\xdc"
-			  "\x5d\x45\x95\x30\x8f\xff\x2f\x1b",
-		.rlen	= 16,
-	}, {
-		.key	= "\x8a\xd4\xee\x10\x2f\xbd\x81\xff"
-			  "\xf8\x86\xce\xac\x93\xc5\xad\xc6"
-			  "\xa0\x19\x07\xc0\x9d\xf7\xbb\xdd"
-			  "\x52\x13\xb2\xb7\xf0\xff\x11\xd8"
-			  "\xd6\x08\xd0\xcd\x2e\xb1\x17\x6f",
-		.klen	= 40,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x0c\x20\x20\x63\xd6\x8b\xfc\x8f"
-			  "\xc0\xe2\x17\xbb\xd2\x59\x6f\x26",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\xc1\x35\x2e\x53\xf0\x96\x4d\x9c"
-			  "\x2e\x18\xe6\x99\xcd\xd3\x15\x68",
-		.rlen	= 16,
-	}, {
-		.key	= "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
-			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
-			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
-			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
-			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
-			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x86\x0a\xc6\xa9\x1a\x9f\xe7\xe6"
-			  "\x64\x3b\x33\xd6\xd5\x84\xd6\xdf",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
-			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
-			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
-			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
-			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
-			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
-			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
-			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
-			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
-			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
-			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
-			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
-			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
-			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
-			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
-			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
-			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
-			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
-			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
-			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
-			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
-			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
-			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
-			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
-			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
-			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
-			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
-			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
-			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
-			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
-			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
-			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
-			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
-			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
-			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
-			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
-			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
-			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
-			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
-			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
-			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
-			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
-			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
-			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
-			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
-			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
-			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
-			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
-			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
-			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
-			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
-			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
-			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
-			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
-			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
-			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
-			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
-			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
-			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
-			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
-			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
-			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
-			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
-			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
-		.ilen	= 512,
-		.result	= "\xe3\x5a\x38\x0f\x4d\x92\x3a\x74"
-			  "\x15\xb1\x50\x8c\x9a\xd8\x99\x1d"
-			  "\x82\xec\xf1\x5f\x03\x6d\x02\x58"
-			  "\x90\x67\xfc\xdd\x8d\xe1\x38\x08"
-			  "\x7b\xc9\x9b\x4b\x04\x09\x50\x15"
-			  "\xce\xab\xda\x33\x30\x20\x12\xfa"
-			  "\x83\xc4\xa6\x9a\x2e\x7d\x90\xd9"
-			  "\xa6\xa6\x67\x43\xb4\xa7\xa8\x5c"
-			  "\xbb\x6a\x49\x2b\x8b\xf8\xd0\x22"
-			  "\xe5\x9e\xba\xe8\x8c\x67\xb8\x5b"
-			  "\x60\xbc\xf5\xa4\x95\x4e\x66\xe5"
-			  "\x6d\x8e\xa9\xf6\x65\x2e\x04\xf5"
-			  "\xba\xb5\xdb\x88\xc2\xf6\x7a\x4b"
-			  "\x89\x58\x7c\x9a\xae\x26\xe8\xb7"
-			  "\xb7\x28\xcc\xd6\xcc\xa5\x98\x4d"
-			  "\xb9\x91\xcb\xb4\xe4\x8b\x96\x47"
-			  "\x5f\x03\x8b\xdd\x94\xd1\xee\x12"
-			  "\xa7\x83\x80\xf2\xc1\x15\x74\x4f"
-			  "\x49\xf9\xb0\x7e\x6f\xdc\x73\x2f"
-			  "\xe2\xcf\xe0\x1b\x34\xa5\xa0\x52"
-			  "\xfb\x3c\x5d\x85\x91\xe6\x6d\x98"
-			  "\x04\xd6\xdd\x4c\x00\x64\xd9\x54"
-			  "\x5c\x3c\x08\x1d\x4c\x06\x9f\xb8"
-			  "\x1c\x4d\x8d\xdc\xa4\x3c\xb9\x3b"
-			  "\x9e\x85\xce\xc3\xa8\x4a\x0c\xd9"
-			  "\x04\xc3\x6f\x17\x66\xa9\x1f\x59"
-			  "\xd9\xe2\x19\x36\xa3\x88\xb8\x0b"
-			  "\x0f\x4a\x4d\xf8\xc8\x6f\xd5\x43"
-			  "\xeb\xa0\xab\x1f\x61\xc0\x06\xeb"
-			  "\x93\xb7\xb8\x6f\x0d\xbd\x07\x49"
-			  "\xb3\xac\x5d\xcf\x31\xa0\x27\x26"
-			  "\x21\xbe\x94\x2e\x19\xea\xf4\xee"
-			  "\xb5\x13\x89\xf7\x94\x0b\xef\x59"
-			  "\x44\xc5\x78\x8b\x3c\x3b\x71\x20"
-			  "\xf9\x35\x0c\x70\x74\xdc\x5b\xc2"
-			  "\xb4\x11\x0e\x2c\x61\xa1\x52\x46"
-			  "\x18\x11\x16\xc6\x86\x44\xa7\xaf"
-			  "\xd5\x0c\x7d\xa6\x9e\x25\x2d\x1b"
-			  "\x9a\x8f\x0f\xf8\x6a\x61\xa0\xea"
-			  "\x3f\x0e\x90\xd6\x8f\x83\x30\x64"
-			  "\xb5\x51\x2d\x08\x3c\xcd\x99\x36"
-			  "\x96\xd4\xb1\xb5\x48\x30\xca\x48"
-			  "\xf7\x11\xa8\xf5\x97\x8a\x6a\x6d"
-			  "\x12\x33\x2f\xc0\xe8\xda\xec\x8a"
-			  "\xe1\x88\x72\x63\xde\x20\xa3\xe1"
-			  "\x8e\xac\x84\x37\x35\xf5\xf7\x3f"
-			  "\x00\x02\x0e\xe4\xc1\x53\x68\x3f"
-			  "\xaa\xd5\xac\x52\x3d\x20\x2f\x4d"
-			  "\x7c\x83\xd0\xbd\xaa\x97\x35\x36"
-			  "\x98\x88\x59\x5d\xe7\x24\xe3\x90"
-			  "\x9d\x30\x47\xa7\xc3\x60\x35\xf4"
-			  "\xd5\xdb\x0e\x4d\x44\xc1\x81\x8b"
-			  "\xfd\xbd\xc3\x2b\xba\x68\xfe\x8d"
-			  "\x49\x5a\x3c\x8a\xa3\x01\xae\x25"
-			  "\x42\xab\xd2\x87\x1b\x35\xd6\xd2"
-			  "\xd7\x70\x1c\x1f\x72\xd1\xe1\x39"
-			  "\x1c\x58\xa2\xb4\xd0\x78\x55\x72"
-			  "\x76\x59\xea\xd9\xd7\x6e\x63\x8b"
-			  "\xcc\x9b\xa7\x74\x89\xfc\xa3\x68"
-			  "\x86\x28\xd1\xbb\x54\x8d\x66\xad"
-			  "\x2a\x92\xf9\x4e\x04\x3d\xae\xfd"
-			  "\x1b\x2b\x7f\xc3\x2f\x1a\x78\x0a"
-			  "\x5c\xc6\x84\xfe\x7c\xcb\x26\xfd"
-			  "\xd9\x51\x0f\xd7\x94\x2f\xc5\xa7",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec serpent_lrw_dec_tv_template[] = {
-	/* Generated from AES-LRW test vectors */
-	/* same as enc vectors with input and result reversed */
-	{
-		.key	= "\x45\x62\xac\x25\xf8\x28\x17\x6d"
-			  "\x4c\x26\x84\x14\xb5\x68\x01\x85"
-			  "\x25\x8e\x2a\x05\xe7\x3e\x9d\x03"
-			  "\xee\x5a\x83\x0c\xcc\x09\x4c\x87",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x6f\xbf\xd4\xa4\x5d\x71\x16\x79"
-			  "\x63\x9c\xa6\x8e\x40\xbe\x0d\x8a",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\x59\x70\x47\x14\xf5\x57\x47\x8c"
-			  "\xd7\x79\xe8\x0f\x54\x88\x79\x44"
-			  "\x0d\x48\xf0\xb7\xb1\x5a\x53\xea"
-			  "\x1c\xaa\x6b\x29\xc2\xca\xfb\xaf",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x02",
-		.input	= "\xfd\xb2\x66\x98\x80\x96\x55\xad"
-			  "\x08\x94\x54\x9c\x21\x7c\x69\xe3",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\xd8\x2a\x91\x34\xb2\x6a\x56\x50"
-			  "\x30\xfe\x69\xe2\x37\x7f\x98\x47"
-			  "\xcd\xf9\x0b\x16\x0c\x64\x8f\xb6"
-			  "\xb0\x0d\x0d\x1b\xae\x85\x87\x1f",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x14\x5e\x3d\x70\xc0\x6e\x9c\x34"
-			  "\x5b\x5e\xcf\x0f\xe4\x8c\x21\x5c",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\x0f\x6a\xef\xf8\xd3\xd2\xbb\x15"
-			  "\x25\x83\xf7\x3c\x1f\x01\x28\x74"
-			  "\xca\xc6\xbc\x35\x4d\x4a\x65\x54"
-			  "\x90\xae\x61\xcf\x7b\xae\xbd\xcc"
-			  "\xad\xe4\x94\xc5\x4a\x29\xae\x70",
-		.klen	= 40,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x25\x39\xaa\xa5\xf0\x65\xc8\xdc"
-			  "\x5d\x45\x95\x30\x8f\xff\x2f\x1b",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\x8a\xd4\xee\x10\x2f\xbd\x81\xff"
-			  "\xf8\x86\xce\xac\x93\xc5\xad\xc6"
-			  "\xa0\x19\x07\xc0\x9d\xf7\xbb\xdd"
-			  "\x52\x13\xb2\xb7\xf0\xff\x11\xd8"
-			  "\xd6\x08\xd0\xcd\x2e\xb1\x17\x6f",
-		.klen	= 40,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x0c\x20\x20\x63\xd6\x8b\xfc\x8f"
-			  "\xc0\xe2\x17\xbb\xd2\x59\x6f\x26",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\xc1\x35\x2e\x53\xf0\x96\x4d\x9c"
-			  "\x2e\x18\xe6\x99\xcd\xd3\x15\x68",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
-			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
-			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
-			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
-			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
-			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x86\x0a\xc6\xa9\x1a\x9f\xe7\xe6"
-			  "\x64\x3b\x33\xd6\xd5\x84\xd6\xdf",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\xe3\x5a\x38\x0f\x4d\x92\x3a\x74"
-			  "\x15\xb1\x50\x8c\x9a\xd8\x99\x1d"
-			  "\x82\xec\xf1\x5f\x03\x6d\x02\x58"
-			  "\x90\x67\xfc\xdd\x8d\xe1\x38\x08"
-			  "\x7b\xc9\x9b\x4b\x04\x09\x50\x15"
-			  "\xce\xab\xda\x33\x30\x20\x12\xfa"
-			  "\x83\xc4\xa6\x9a\x2e\x7d\x90\xd9"
-			  "\xa6\xa6\x67\x43\xb4\xa7\xa8\x5c"
-			  "\xbb\x6a\x49\x2b\x8b\xf8\xd0\x22"
-			  "\xe5\x9e\xba\xe8\x8c\x67\xb8\x5b"
-			  "\x60\xbc\xf5\xa4\x95\x4e\x66\xe5"
-			  "\x6d\x8e\xa9\xf6\x65\x2e\x04\xf5"
-			  "\xba\xb5\xdb\x88\xc2\xf6\x7a\x4b"
-			  "\x89\x58\x7c\x9a\xae\x26\xe8\xb7"
-			  "\xb7\x28\xcc\xd6\xcc\xa5\x98\x4d"
-			  "\xb9\x91\xcb\xb4\xe4\x8b\x96\x47"
-			  "\x5f\x03\x8b\xdd\x94\xd1\xee\x12"
-			  "\xa7\x83\x80\xf2\xc1\x15\x74\x4f"
-			  "\x49\xf9\xb0\x7e\x6f\xdc\x73\x2f"
-			  "\xe2\xcf\xe0\x1b\x34\xa5\xa0\x52"
-			  "\xfb\x3c\x5d\x85\x91\xe6\x6d\x98"
-			  "\x04\xd6\xdd\x4c\x00\x64\xd9\x54"
-			  "\x5c\x3c\x08\x1d\x4c\x06\x9f\xb8"
-			  "\x1c\x4d\x8d\xdc\xa4\x3c\xb9\x3b"
-			  "\x9e\x85\xce\xc3\xa8\x4a\x0c\xd9"
-			  "\x04\xc3\x6f\x17\x66\xa9\x1f\x59"
-			  "\xd9\xe2\x19\x36\xa3\x88\xb8\x0b"
-			  "\x0f\x4a\x4d\xf8\xc8\x6f\xd5\x43"
-			  "\xeb\xa0\xab\x1f\x61\xc0\x06\xeb"
-			  "\x93\xb7\xb8\x6f\x0d\xbd\x07\x49"
-			  "\xb3\xac\x5d\xcf\x31\xa0\x27\x26"
-			  "\x21\xbe\x94\x2e\x19\xea\xf4\xee"
-			  "\xb5\x13\x89\xf7\x94\x0b\xef\x59"
-			  "\x44\xc5\x78\x8b\x3c\x3b\x71\x20"
-			  "\xf9\x35\x0c\x70\x74\xdc\x5b\xc2"
-			  "\xb4\x11\x0e\x2c\x61\xa1\x52\x46"
-			  "\x18\x11\x16\xc6\x86\x44\xa7\xaf"
-			  "\xd5\x0c\x7d\xa6\x9e\x25\x2d\x1b"
-			  "\x9a\x8f\x0f\xf8\x6a\x61\xa0\xea"
-			  "\x3f\x0e\x90\xd6\x8f\x83\x30\x64"
-			  "\xb5\x51\x2d\x08\x3c\xcd\x99\x36"
-			  "\x96\xd4\xb1\xb5\x48\x30\xca\x48"
-			  "\xf7\x11\xa8\xf5\x97\x8a\x6a\x6d"
-			  "\x12\x33\x2f\xc0\xe8\xda\xec\x8a"
-			  "\xe1\x88\x72\x63\xde\x20\xa3\xe1"
-			  "\x8e\xac\x84\x37\x35\xf5\xf7\x3f"
-			  "\x00\x02\x0e\xe4\xc1\x53\x68\x3f"
-			  "\xaa\xd5\xac\x52\x3d\x20\x2f\x4d"
-			  "\x7c\x83\xd0\xbd\xaa\x97\x35\x36"
-			  "\x98\x88\x59\x5d\xe7\x24\xe3\x90"
-			  "\x9d\x30\x47\xa7\xc3\x60\x35\xf4"
-			  "\xd5\xdb\x0e\x4d\x44\xc1\x81\x8b"
-			  "\xfd\xbd\xc3\x2b\xba\x68\xfe\x8d"
-			  "\x49\x5a\x3c\x8a\xa3\x01\xae\x25"
-			  "\x42\xab\xd2\x87\x1b\x35\xd6\xd2"
-			  "\xd7\x70\x1c\x1f\x72\xd1\xe1\x39"
-			  "\x1c\x58\xa2\xb4\xd0\x78\x55\x72"
-			  "\x76\x59\xea\xd9\xd7\x6e\x63\x8b"
-			  "\xcc\x9b\xa7\x74\x89\xfc\xa3\x68"
-			  "\x86\x28\xd1\xbb\x54\x8d\x66\xad"
-			  "\x2a\x92\xf9\x4e\x04\x3d\xae\xfd"
-			  "\x1b\x2b\x7f\xc3\x2f\x1a\x78\x0a"
-			  "\x5c\xc6\x84\xfe\x7c\xcb\x26\xfd"
-			  "\xd9\x51\x0f\xd7\x94\x2f\xc5\xa7",
-		.ilen	= 512,
-		.result	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
-			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
-			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
-			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
-			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
-			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
-			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
-			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
-			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
-			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
-			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
-			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
-			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
-			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
-			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
-			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
-			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
-			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
-			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
-			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
-			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
-			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
-			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
-			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
-			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
-			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
-			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
-			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
-			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
-			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
-			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
-			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
-			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
-			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
-			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
-			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
-			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
-			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
-			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
-			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
-			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
-			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
-			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
-			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
-			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
-			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
-			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
-			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
-			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
-			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
-			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
-			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
-			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
-			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
-			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
-			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
-			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
-			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
-			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
-			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
-			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
-			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
-			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
-			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec serpent_xts_enc_tv_template[] = {
-	/* Generated from AES-XTS test vectors */
-	{
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 32,
-		.result	= "\xe1\x08\xb8\x1d\x2c\xf5\x33\x64"
-			  "\xc8\x12\x04\xc7\xb3\x70\xe8\xc4"
-			  "\x6a\x31\xc5\xf3\x00\xca\xb9\x16"
-			  "\xde\xe2\x77\x66\xf7\xfe\x62\x08",
-		.rlen	= 32,
-	}, {
-		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen	= 32,
-		.result	= "\x1a\x0a\x09\x5f\xcd\x07\x07\x98"
-			  "\x41\x86\x12\xaf\xb3\xd7\x68\x13"
-			  "\xed\x81\xcd\x06\x87\x43\x1a\xbb"
-			  "\x13\x3d\xd6\x1e\x2b\xe1\x77\xbe",
-		.rlen	= 32,
-	}, {
-		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen	= 32,
-		.result	= "\xf9\x9b\x28\xb8\x5c\xaf\x8c\x61"
-			  "\xb6\x1c\x81\x8f\x2c\x87\x60\x89"
-			  "\x0d\x8d\x7a\xe8\x60\x48\xcc\x86"
-			  "\xc1\x68\x45\xaa\x00\xe9\x24\xc5",
-		.rlen	= 32,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\xfe\x47\x4a\xc8\x60\x7e\xb4\x8b"
-			  "\x0d\x10\xf4\xb0\x0d\xba\xf8\x53"
-			  "\x65\x6e\x38\x4b\xdb\xaa\xb1\x9e"
-			  "\x28\xca\xb0\x22\xb3\x85\x75\xf4"
-			  "\x00\x5c\x75\x14\x06\xd6\x25\x82"
-			  "\xe6\xcb\x08\xf7\x29\x90\x23\x8e"
-			  "\xa4\x68\x57\xe4\xf0\xd8\x32\xf3"
-			  "\x80\x51\x67\xb5\x0b\x85\x69\xe8"
-			  "\x19\xfe\xc4\xc7\x3e\xea\x90\xd3"
-			  "\x8f\xa3\xf2\x0a\xac\x17\x4b\xa0"
-			  "\x63\x5a\x16\x0f\xf0\xce\x66\x1f"
-			  "\x2c\x21\x07\xf1\xa4\x03\xa3\x44"
-			  "\x41\x61\x87\x5d\x6b\xb3\xef\xd4"
-			  "\xfc\xaa\x32\x7e\x55\x58\x04\x41"
-			  "\xc9\x07\x33\xc6\xa2\x68\xd6\x5a"
-			  "\x55\x79\x4b\x6f\xcf\x89\xb9\x19"
-			  "\xe5\x54\x13\x15\xb2\x1a\xfa\x15"
-			  "\xc2\xf0\x06\x59\xfa\xa0\x25\x05"
-			  "\x58\xfa\x43\x91\x16\x85\x40\xbb"
-			  "\x0d\x34\x4d\xc5\x1e\x20\xd5\x08"
-			  "\xcd\x22\x22\x41\x11\x9f\x6c\x7c"
-			  "\x8d\x57\xc9\xba\x57\xe8\x2c\xf7"
-			  "\xa0\x42\xa8\xde\xfc\xa3\xca\x98"
-			  "\x4b\x43\xb1\xce\x4b\xbf\x01\x67"
-			  "\x6e\x29\x60\xbd\x10\x14\x84\x82"
-			  "\x83\x82\x0c\x63\x73\x92\x02\x7c"
-			  "\x55\x37\x20\x80\x17\x51\xc8\xbc"
-			  "\x46\x02\xcb\x38\x07\x6d\xe2\x85"
-			  "\xaa\x29\xaf\x24\x58\x0d\xf0\x75"
-			  "\x08\x0a\xa5\x34\x25\x16\xf3\x74"
-			  "\xa7\x0b\x97\xbe\xc1\xa9\xdc\x29"
-			  "\x1a\x0a\x56\xc1\x1a\x91\x97\x8c"
-			  "\x0b\xc7\x16\xed\x5a\x22\xa6\x2e"
-			  "\x8c\x2b\x4f\x54\x76\x47\x53\x8e"
-			  "\xe8\x00\xec\x92\xb9\x55\xe6\xa2"
-			  "\xf3\xe2\x4f\x6a\x66\x60\xd0\x87"
-			  "\xe6\xd1\xcc\xe3\x6a\xc5\x2d\x21"
-			  "\xcc\x9d\x6a\xb6\x75\xaa\xe2\x19"
-			  "\x21\x9f\xa1\x5e\x4c\xfd\x72\xf9"
-			  "\x94\x4e\x63\xc7\xae\xfc\xed\x47"
-			  "\xe2\xfe\x7a\x63\x77\xfe\x97\x82"
-			  "\xb1\x10\x6e\x36\x1d\xe1\xc4\x80"
-			  "\xec\x69\x41\xec\xa7\x8a\xe0\x2f"
-			  "\xe3\x49\x26\xa2\x41\xb2\x08\x0f"
-			  "\x28\xb4\xa7\x39\xa1\x99\x2d\x1e"
-			  "\x43\x42\x35\xd0\xcf\xec\x77\x67"
-			  "\xb2\x3b\x9e\x1c\x35\xde\x4f\x5e"
-			  "\x73\x3f\x5d\x6f\x07\x4b\x2e\x50"
-			  "\xab\x6c\x6b\xff\xea\x00\x67\xaa"
-			  "\x0e\x82\x32\xdd\x3d\xb5\xe5\x76"
-			  "\x2b\x77\x3f\xbe\x12\x75\xfb\x92"
-			  "\xc6\x89\x67\x4d\xca\xf7\xd4\x50"
-			  "\xc0\x74\x47\xcc\xd9\x0a\xd4\xc6"
-			  "\x3b\x17\x2e\xe3\x35\xbb\x53\xb5"
-			  "\x86\xad\x51\xcc\xd5\x96\xb8\xdc"
-			  "\x03\x57\xe6\x98\x52\x2f\x61\x62"
-			  "\xc4\x5c\x9c\x36\x71\x07\xfb\x94"
-			  "\xe3\x02\xc4\x2b\x08\x75\xc7\x35"
-			  "\xfb\x2e\x88\x7b\xbb\x67\x00\xe1"
-			  "\xc9\xdd\x99\xb2\x13\x53\x1a\x4e"
-			  "\x76\x87\x19\x04\x1a\x2f\x38\x3e"
-			  "\xef\x91\x64\x1d\x18\x07\x4e\x31"
-			  "\x88\x21\x7c\xb0\xa5\x12\x4c\x3c"
-			  "\xb0\x20\xbd\xda\xdf\xf9\x7c\xdd",
-		.rlen	= 512,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\x2b\xc9\xb4\x6b\x10\x94\xa9\x32"
-			  "\xaa\xb0\x20\xc6\x44\x3d\x74\x1f"
-			  "\x75\x01\xa7\xf6\xf5\xf7\x62\x1b"
-			  "\x80\x1b\x82\xcb\x01\x59\x91\x7f"
-			  "\x80\x3a\x98\xf0\xd2\xca\xc4\xc3"
-			  "\x34\xfd\xe6\x11\xf9\x33\x45\x12"
-			  "\x48\xc5\x8c\x25\xf1\xc5\xc5\x23"
-			  "\xd3\x44\xb4\x73\xd5\x04\xc0\xb7"
-			  "\xca\x2f\xf5\xcd\xc5\xb4\xdd\xb0"
-			  "\xf4\x60\xe8\xfb\xc6\x9c\xc5\x78"
-			  "\xcd\xec\x7d\xdc\x19\x9c\x72\x64"
-			  "\x63\x0b\x38\x2e\x76\xdd\x2d\x36"
-			  "\x49\xb0\x1d\xea\x78\x9e\x00\xca"
-			  "\x20\xcc\x1b\x1e\x98\x74\xab\xed"
-			  "\x79\xf7\xd0\x6c\xd8\x93\x80\x29"
-			  "\xac\xa5\x5e\x34\xa9\xab\xa0\x55"
-			  "\x9a\xea\xaa\x95\x4d\x7b\xfe\x46"
-			  "\x26\x8a\xfd\x88\xa2\xa8\xa6\xae"
-			  "\x25\x42\x17\xbf\x76\x8f\x1c\x3d"
-			  "\xec\x9a\xda\x64\x96\xb5\x61\xff"
-			  "\x99\xeb\x12\x96\x85\x82\x9d\xd5"
-			  "\x81\x85\x14\xa8\x59\xac\x8c\x94"
-			  "\xbb\x3b\x85\x2b\xdf\xb3\x0c\xba"
-			  "\x82\xc6\x4d\xca\x86\xea\x53\x28"
-			  "\x4c\xe0\x4e\x31\xe3\x73\x2f\x79"
-			  "\x9d\x42\xe1\x03\xe3\x8b\xc4\xff"
-			  "\x05\xca\x81\x7b\xda\xa2\xde\x63"
-			  "\x3a\x10\xbe\xc2\xac\x32\xc4\x05"
-			  "\x47\x7e\xef\x67\xe2\x5f\x5b\xae"
-			  "\xed\xf1\x70\x34\x16\x9a\x07\x7b"
-			  "\xf2\x25\x2b\xb0\xf8\x3c\x15\x9a"
-			  "\xa6\x59\x55\x5f\xc1\xf4\x1e\xcd"
-			  "\x93\x1f\x06\xba\xd4\x9a\x22\x69"
-			  "\xfa\x8e\x95\x0d\xf3\x23\x59\x2c"
-			  "\xfe\x00\xba\xf0\x0e\xbc\x6d\xd6"
-			  "\x62\xf0\x7a\x0e\x83\x3e\xdb\x32"
-			  "\xfd\x43\x7d\xda\x42\x51\x87\x43"
-			  "\x9d\xf9\xef\xf4\x30\x97\xf8\x09"
-			  "\x88\xfc\x3f\x93\x70\xc1\x4a\xec"
-			  "\x27\x5f\x11\xac\x71\xc7\x48\x46"
-			  "\x2f\xf9\xdf\x8d\x9f\xf7\x2e\x56"
-			  "\x0d\x4e\xb0\x32\x76\xce\x86\x81"
-			  "\xcd\xdf\xe4\x00\xbf\xfd\x5f\x24"
-			  "\xaf\xf7\x9a\xde\xff\x18\xac\x14"
-			  "\x90\xc5\x01\x39\x34\x0f\x24\xf3"
-			  "\x13\x2f\x5e\x4f\x30\x9a\x36\x40"
-			  "\xec\xea\xbc\xcd\x9e\x0e\x5b\x23"
-			  "\x50\x88\x97\x40\x69\xb1\x37\xf5"
-			  "\xc3\x15\xf9\x3f\xb7\x79\x64\xe8"
-			  "\x7b\x10\x20\xb9\x2b\x46\x83\x5b"
-			  "\xd8\x39\xfc\xe4\xfa\x88\x52\xf2"
-			  "\x72\xb0\x97\x4e\x89\xb3\x48\x00"
-			  "\xc1\x16\x73\x50\x77\xba\xa6\x65"
-			  "\x20\x2d\xb0\x02\x27\x89\xda\x99"
-			  "\x45\xfb\xe9\xd3\x1d\x39\x2f\xd6"
-			  "\x2a\xda\x09\x12\x11\xaf\xe6\x57"
-			  "\x01\x04\x8a\xff\x86\x8b\xac\xf8"
-			  "\xee\xe4\x1c\x98\x5b\xcf\x6b\x76"
-			  "\xa3\x0e\x33\x74\x40\x18\x39\x72"
-			  "\x66\x50\x31\xfd\x70\xdf\xe8\x51"
-			  "\x96\x21\x36\xb2\x9b\xfa\x85\xd1"
-			  "\x30\x05\xc8\x92\x98\x80\xff\x7a"
-			  "\xaf\x43\x0b\xc5\x20\x41\x92\x20"
-			  "\xd4\xa0\x91\x98\x11\x5f\x4d\xb1",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec serpent_xts_dec_tv_template[] = {
-	/* Generated from AES-XTS test vectors */
-	/* same as enc vectors with input and result reversed */
-	{
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xe1\x08\xb8\x1d\x2c\xf5\x33\x64"
-			  "\xc8\x12\x04\xc7\xb3\x70\xe8\xc4"
-			  "\x6a\x31\xc5\xf3\x00\xca\xb9\x16"
-			  "\xde\xe2\x77\x66\xf7\xfe\x62\x08",
-		.ilen	= 32,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen	= 32,
-	}, {
-		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x1a\x0a\x09\x5f\xcd\x07\x07\x98"
-			  "\x41\x86\x12\xaf\xb3\xd7\x68\x13"
-			  "\xed\x81\xcd\x06\x87\x43\x1a\xbb"
-			  "\x13\x3d\xd6\x1e\x2b\xe1\x77\xbe",
-		.ilen	= 32,
-		.result	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen	= 32,
-	}, {
-		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xf9\x9b\x28\xb8\x5c\xaf\x8c\x61"
-			  "\xb6\x1c\x81\x8f\x2c\x87\x60\x89"
-			  "\x0d\x8d\x7a\xe8\x60\x48\xcc\x86"
-			  "\xc1\x68\x45\xaa\x00\xe9\x24\xc5",
-		.ilen	= 32,
-		.result	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen	= 32,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xfe\x47\x4a\xc8\x60\x7e\xb4\x8b"
-			  "\x0d\x10\xf4\xb0\x0d\xba\xf8\x53"
-			  "\x65\x6e\x38\x4b\xdb\xaa\xb1\x9e"
-			  "\x28\xca\xb0\x22\xb3\x85\x75\xf4"
-			  "\x00\x5c\x75\x14\x06\xd6\x25\x82"
-			  "\xe6\xcb\x08\xf7\x29\x90\x23\x8e"
-			  "\xa4\x68\x57\xe4\xf0\xd8\x32\xf3"
-			  "\x80\x51\x67\xb5\x0b\x85\x69\xe8"
-			  "\x19\xfe\xc4\xc7\x3e\xea\x90\xd3"
-			  "\x8f\xa3\xf2\x0a\xac\x17\x4b\xa0"
-			  "\x63\x5a\x16\x0f\xf0\xce\x66\x1f"
-			  "\x2c\x21\x07\xf1\xa4\x03\xa3\x44"
-			  "\x41\x61\x87\x5d\x6b\xb3\xef\xd4"
-			  "\xfc\xaa\x32\x7e\x55\x58\x04\x41"
-			  "\xc9\x07\x33\xc6\xa2\x68\xd6\x5a"
-			  "\x55\x79\x4b\x6f\xcf\x89\xb9\x19"
-			  "\xe5\x54\x13\x15\xb2\x1a\xfa\x15"
-			  "\xc2\xf0\x06\x59\xfa\xa0\x25\x05"
-			  "\x58\xfa\x43\x91\x16\x85\x40\xbb"
-			  "\x0d\x34\x4d\xc5\x1e\x20\xd5\x08"
-			  "\xcd\x22\x22\x41\x11\x9f\x6c\x7c"
-			  "\x8d\x57\xc9\xba\x57\xe8\x2c\xf7"
-			  "\xa0\x42\xa8\xde\xfc\xa3\xca\x98"
-			  "\x4b\x43\xb1\xce\x4b\xbf\x01\x67"
-			  "\x6e\x29\x60\xbd\x10\x14\x84\x82"
-			  "\x83\x82\x0c\x63\x73\x92\x02\x7c"
-			  "\x55\x37\x20\x80\x17\x51\xc8\xbc"
-			  "\x46\x02\xcb\x38\x07\x6d\xe2\x85"
-			  "\xaa\x29\xaf\x24\x58\x0d\xf0\x75"
-			  "\x08\x0a\xa5\x34\x25\x16\xf3\x74"
-			  "\xa7\x0b\x97\xbe\xc1\xa9\xdc\x29"
-			  "\x1a\x0a\x56\xc1\x1a\x91\x97\x8c"
-			  "\x0b\xc7\x16\xed\x5a\x22\xa6\x2e"
-			  "\x8c\x2b\x4f\x54\x76\x47\x53\x8e"
-			  "\xe8\x00\xec\x92\xb9\x55\xe6\xa2"
-			  "\xf3\xe2\x4f\x6a\x66\x60\xd0\x87"
-			  "\xe6\xd1\xcc\xe3\x6a\xc5\x2d\x21"
-			  "\xcc\x9d\x6a\xb6\x75\xaa\xe2\x19"
-			  "\x21\x9f\xa1\x5e\x4c\xfd\x72\xf9"
-			  "\x94\x4e\x63\xc7\xae\xfc\xed\x47"
-			  "\xe2\xfe\x7a\x63\x77\xfe\x97\x82"
-			  "\xb1\x10\x6e\x36\x1d\xe1\xc4\x80"
-			  "\xec\x69\x41\xec\xa7\x8a\xe0\x2f"
-			  "\xe3\x49\x26\xa2\x41\xb2\x08\x0f"
-			  "\x28\xb4\xa7\x39\xa1\x99\x2d\x1e"
-			  "\x43\x42\x35\xd0\xcf\xec\x77\x67"
-			  "\xb2\x3b\x9e\x1c\x35\xde\x4f\x5e"
-			  "\x73\x3f\x5d\x6f\x07\x4b\x2e\x50"
-			  "\xab\x6c\x6b\xff\xea\x00\x67\xaa"
-			  "\x0e\x82\x32\xdd\x3d\xb5\xe5\x76"
-			  "\x2b\x77\x3f\xbe\x12\x75\xfb\x92"
-			  "\xc6\x89\x67\x4d\xca\xf7\xd4\x50"
-			  "\xc0\x74\x47\xcc\xd9\x0a\xd4\xc6"
-			  "\x3b\x17\x2e\xe3\x35\xbb\x53\xb5"
-			  "\x86\xad\x51\xcc\xd5\x96\xb8\xdc"
-			  "\x03\x57\xe6\x98\x52\x2f\x61\x62"
-			  "\xc4\x5c\x9c\x36\x71\x07\xfb\x94"
-			  "\xe3\x02\xc4\x2b\x08\x75\xc7\x35"
-			  "\xfb\x2e\x88\x7b\xbb\x67\x00\xe1"
-			  "\xc9\xdd\x99\xb2\x13\x53\x1a\x4e"
-			  "\x76\x87\x19\x04\x1a\x2f\x38\x3e"
-			  "\xef\x91\x64\x1d\x18\x07\x4e\x31"
-			  "\x88\x21\x7c\xb0\xa5\x12\x4c\x3c"
-			  "\xb0\x20\xbd\xda\xdf\xf9\x7c\xdd",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x2b\xc9\xb4\x6b\x10\x94\xa9\x32"
-			  "\xaa\xb0\x20\xc6\x44\x3d\x74\x1f"
-			  "\x75\x01\xa7\xf6\xf5\xf7\x62\x1b"
-			  "\x80\x1b\x82\xcb\x01\x59\x91\x7f"
-			  "\x80\x3a\x98\xf0\xd2\xca\xc4\xc3"
-			  "\x34\xfd\xe6\x11\xf9\x33\x45\x12"
-			  "\x48\xc5\x8c\x25\xf1\xc5\xc5\x23"
-			  "\xd3\x44\xb4\x73\xd5\x04\xc0\xb7"
-			  "\xca\x2f\xf5\xcd\xc5\xb4\xdd\xb0"
-			  "\xf4\x60\xe8\xfb\xc6\x9c\xc5\x78"
-			  "\xcd\xec\x7d\xdc\x19\x9c\x72\x64"
-			  "\x63\x0b\x38\x2e\x76\xdd\x2d\x36"
-			  "\x49\xb0\x1d\xea\x78\x9e\x00\xca"
-			  "\x20\xcc\x1b\x1e\x98\x74\xab\xed"
-			  "\x79\xf7\xd0\x6c\xd8\x93\x80\x29"
-			  "\xac\xa5\x5e\x34\xa9\xab\xa0\x55"
-			  "\x9a\xea\xaa\x95\x4d\x7b\xfe\x46"
-			  "\x26\x8a\xfd\x88\xa2\xa8\xa6\xae"
-			  "\x25\x42\x17\xbf\x76\x8f\x1c\x3d"
-			  "\xec\x9a\xda\x64\x96\xb5\x61\xff"
-			  "\x99\xeb\x12\x96\x85\x82\x9d\xd5"
-			  "\x81\x85\x14\xa8\x59\xac\x8c\x94"
-			  "\xbb\x3b\x85\x2b\xdf\xb3\x0c\xba"
-			  "\x82\xc6\x4d\xca\x86\xea\x53\x28"
-			  "\x4c\xe0\x4e\x31\xe3\x73\x2f\x79"
-			  "\x9d\x42\xe1\x03\xe3\x8b\xc4\xff"
-			  "\x05\xca\x81\x7b\xda\xa2\xde\x63"
-			  "\x3a\x10\xbe\xc2\xac\x32\xc4\x05"
-			  "\x47\x7e\xef\x67\xe2\x5f\x5b\xae"
-			  "\xed\xf1\x70\x34\x16\x9a\x07\x7b"
-			  "\xf2\x25\x2b\xb0\xf8\x3c\x15\x9a"
-			  "\xa6\x59\x55\x5f\xc1\xf4\x1e\xcd"
-			  "\x93\x1f\x06\xba\xd4\x9a\x22\x69"
-			  "\xfa\x8e\x95\x0d\xf3\x23\x59\x2c"
-			  "\xfe\x00\xba\xf0\x0e\xbc\x6d\xd6"
-			  "\x62\xf0\x7a\x0e\x83\x3e\xdb\x32"
-			  "\xfd\x43\x7d\xda\x42\x51\x87\x43"
-			  "\x9d\xf9\xef\xf4\x30\x97\xf8\x09"
-			  "\x88\xfc\x3f\x93\x70\xc1\x4a\xec"
-			  "\x27\x5f\x11\xac\x71\xc7\x48\x46"
-			  "\x2f\xf9\xdf\x8d\x9f\xf7\x2e\x56"
-			  "\x0d\x4e\xb0\x32\x76\xce\x86\x81"
-			  "\xcd\xdf\xe4\x00\xbf\xfd\x5f\x24"
-			  "\xaf\xf7\x9a\xde\xff\x18\xac\x14"
-			  "\x90\xc5\x01\x39\x34\x0f\x24\xf3"
-			  "\x13\x2f\x5e\x4f\x30\x9a\x36\x40"
-			  "\xec\xea\xbc\xcd\x9e\x0e\x5b\x23"
-			  "\x50\x88\x97\x40\x69\xb1\x37\xf5"
-			  "\xc3\x15\xf9\x3f\xb7\x79\x64\xe8"
-			  "\x7b\x10\x20\xb9\x2b\x46\x83\x5b"
-			  "\xd8\x39\xfc\xe4\xfa\x88\x52\xf2"
-			  "\x72\xb0\x97\x4e\x89\xb3\x48\x00"
-			  "\xc1\x16\x73\x50\x77\xba\xa6\x65"
-			  "\x20\x2d\xb0\x02\x27\x89\xda\x99"
-			  "\x45\xfb\xe9\xd3\x1d\x39\x2f\xd6"
-			  "\x2a\xda\x09\x12\x11\xaf\xe6\x57"
-			  "\x01\x04\x8a\xff\x86\x8b\xac\xf8"
-			  "\xee\xe4\x1c\x98\x5b\xcf\x6b\x76"
-			  "\xa3\x0e\x33\x74\x40\x18\x39\x72"
-			  "\x66\x50\x31\xfd\x70\xdf\xe8\x51"
-			  "\x96\x21\x36\xb2\x9b\xfa\x85\xd1"
-			  "\x30\x05\xc8\x92\x98\x80\xff\x7a"
-			  "\xaf\x43\x0b\xc5\x20\x41\x92\x20"
-			  "\xd4\xa0\x91\x98\x11\x5f\x4d\xb1",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-/*
- * SM4 test vector taken from the draft RFC
- * https://tools.ietf.org/html/draft-crypto-sm4-00#ref-GBT.32907-2016
- */
-
-static const struct cipher_testvec sm4_enc_tv_template[] = {
-	{ /* SM4 Appendix A: Example Calculations. Example 1. */
-		.key	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
-			  "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
-		.klen	= 16,
-		.input	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
-			  "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
-		.ilen	= 16,
-		.result	= "\x68\x1E\xDF\x34\xD2\x06\x96\x5E"
-			  "\x86\xB3\xE9\x4F\x53\x6E\x42\x46",
-		.rlen	= 16,
-	}, { /*
-	      *  SM4 Appendix A: Example Calculations.
-	      *  Last 10 iterations of Example 2.
-	      */
-		.key    = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
-			  "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
-		.klen	= 16,
-		.input	= "\x99\x4a\xc3\xe7\xc3\x57\x89\x6a"
-			  "\x81\xfc\xa8\xe\x38\x3e\xef\x80"
-			  "\xb1\x98\xf2\xde\x3f\x4b\xae\xd1"
-			  "\xf0\xf1\x30\x4c\x1\x27\x5a\x8f"
-			  "\x45\xe1\x39\xb7\xae\xff\x1f\x27"
-			  "\xad\x57\x15\xab\x31\x5d\xc\xef"
-			  "\x8c\xc8\x80\xbd\x11\x98\xf3\x7b"
-			  "\xa2\xdd\x14\x20\xf9\xe8\xbb\x82"
-			  "\xf7\x32\xca\x4b\xa8\xf7\xb3\x4d"
-			  "\x27\xd1\xcd\xe6\xb6\x65\x5a\x23"
-			  "\xc2\xf3\x54\x84\x53\xe3\xb9\x20"
-			  "\xa5\x37\x0\xbe\xe7\x7b\x48\xfb"
-			  "\x21\x3d\x9e\x48\x1d\x9e\xf5\xbf"
-			  "\x77\xd5\xb4\x4a\x53\x71\x94\x7a"
-			  "\x88\xa6\x6e\x6\x93\xca\x43\xa5"
-			  "\xc4\xf6\xcd\x53\x4b\x7b\x8e\xfe"
-			  "\xb4\x28\x7c\x42\x29\x32\x5d\x88"
-			  "\xed\xce\x0\x19\xe\x16\x2\x6e"
-			  "\x87\xff\x2c\xac\xe8\xe7\xe9\xbf"
-			  "\x31\x51\xec\x47\xc3\x51\x83\xc1",
-		.ilen	= 160,
-		.result	= "\xb1\x98\xf2\xde\x3f\x4b\xae\xd1"
-			  "\xf0\xf1\x30\x4c\x1\x27\x5a\x8f"
-			  "\x45\xe1\x39\xb7\xae\xff\x1f\x27"
-			  "\xad\x57\x15\xab\x31\x5d\xc\xef"
-			  "\x8c\xc8\x80\xbd\x11\x98\xf3\x7b"
-			  "\xa2\xdd\x14\x20\xf9\xe8\xbb\x82"
-			  "\xf7\x32\xca\x4b\xa8\xf7\xb3\x4d"
-			  "\x27\xd1\xcd\xe6\xb6\x65\x5a\x23"
-			  "\xc2\xf3\x54\x84\x53\xe3\xb9\x20"
-			  "\xa5\x37\x0\xbe\xe7\x7b\x48\xfb"
-			  "\x21\x3d\x9e\x48\x1d\x9e\xf5\xbf"
-			  "\x77\xd5\xb4\x4a\x53\x71\x94\x7a"
-			  "\x88\xa6\x6e\x6\x93\xca\x43\xa5"
-			  "\xc4\xf6\xcd\x53\x4b\x7b\x8e\xfe"
-			  "\xb4\x28\x7c\x42\x29\x32\x5d\x88"
-			  "\xed\xce\x0\x19\xe\x16\x2\x6e"
-			  "\x87\xff\x2c\xac\xe8\xe7\xe9\xbf"
-			  "\x31\x51\xec\x47\xc3\x51\x83\xc1"
-			  "\x59\x52\x98\xc7\xc6\xfd\x27\x1f"
-			  "\x4\x2\xf8\x4\xc3\x3d\x3f\x66",
-		.rlen	= 160
-	}
-};
-
-static const struct cipher_testvec sm4_dec_tv_template[] = {
-	{ /* SM4 Appendix A: Example Calculations. Example 1. */
-		.key	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
-			  "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
-		.klen	= 16,
-		.input	= "\x68\x1E\xDF\x34\xD2\x06\x96\x5E"
-			  "\x86\xB3\xE9\x4F\x53\x6E\x42\x46",
-		.ilen	= 16,
-		.result	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
-			  "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
-		.rlen	= 16,
-	}, { /*
-	      *  SM4 Appendix A: Example Calculations.
-	      *  Last 10 iterations of Example 2.
-	      */
-		.key    = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
-			  "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
-		.klen	= 16,
-		.input	= "\xb1\x98\xf2\xde\x3f\x4b\xae\xd1"
-			  "\xf0\xf1\x30\x4c\x1\x27\x5a\x8f"
-			  "\x45\xe1\x39\xb7\xae\xff\x1f\x27"
-			  "\xad\x57\x15\xab\x31\x5d\xc\xef"
-			  "\x8c\xc8\x80\xbd\x11\x98\xf3\x7b"
-			  "\xa2\xdd\x14\x20\xf9\xe8\xbb\x82"
-			  "\xf7\x32\xca\x4b\xa8\xf7\xb3\x4d"
-			  "\x27\xd1\xcd\xe6\xb6\x65\x5a\x23"
-			  "\xc2\xf3\x54\x84\x53\xe3\xb9\x20"
-			  "\xa5\x37\x0\xbe\xe7\x7b\x48\xfb"
-			  "\x21\x3d\x9e\x48\x1d\x9e\xf5\xbf"
-			  "\x77\xd5\xb4\x4a\x53\x71\x94\x7a"
-			  "\x88\xa6\x6e\x6\x93\xca\x43\xa5"
-			  "\xc4\xf6\xcd\x53\x4b\x7b\x8e\xfe"
-			  "\xb4\x28\x7c\x42\x29\x32\x5d\x88"
-			  "\xed\xce\x0\x19\xe\x16\x2\x6e"
-			  "\x87\xff\x2c\xac\xe8\xe7\xe9\xbf"
-			  "\x31\x51\xec\x47\xc3\x51\x83\xc1"
-			  "\x59\x52\x98\xc7\xc6\xfd\x27\x1f"
-			  "\x4\x2\xf8\x4\xc3\x3d\x3f\x66",
-		.ilen	= 160,
-		.result	= "\x99\x4a\xc3\xe7\xc3\x57\x89\x6a"
-			  "\x81\xfc\xa8\xe\x38\x3e\xef\x80"
-			  "\xb1\x98\xf2\xde\x3f\x4b\xae\xd1"
-			  "\xf0\xf1\x30\x4c\x1\x27\x5a\x8f"
-			  "\x45\xe1\x39\xb7\xae\xff\x1f\x27"
-			  "\xad\x57\x15\xab\x31\x5d\xc\xef"
-			  "\x8c\xc8\x80\xbd\x11\x98\xf3\x7b"
-			  "\xa2\xdd\x14\x20\xf9\xe8\xbb\x82"
-			  "\xf7\x32\xca\x4b\xa8\xf7\xb3\x4d"
-			  "\x27\xd1\xcd\xe6\xb6\x65\x5a\x23"
-			  "\xc2\xf3\x54\x84\x53\xe3\xb9\x20"
-			  "\xa5\x37\x0\xbe\xe7\x7b\x48\xfb"
-			  "\x21\x3d\x9e\x48\x1d\x9e\xf5\xbf"
-			  "\x77\xd5\xb4\x4a\x53\x71\x94\x7a"
-			  "\x88\xa6\x6e\x6\x93\xca\x43\xa5"
-			  "\xc4\xf6\xcd\x53\x4b\x7b\x8e\xfe"
-			  "\xb4\x28\x7c\x42\x29\x32\x5d\x88"
-			  "\xed\xce\x0\x19\xe\x16\x2\x6e"
-			  "\x87\xff\x2c\xac\xe8\xe7\xe9\xbf"
-			  "\x31\x51\xec\x47\xc3\x51\x83\xc1",
-		.rlen	= 160
-	}
-};
-
-/*
- * Speck test vectors taken from the original paper:
- * "The Simon and Speck Families of Lightweight Block Ciphers"
- * https://eprint.iacr.org/2013/404.pdf
- *
- * Note that the paper does not make byte and word order clear.  But it was
- * confirmed with the authors that the intended orders are little endian byte
- * order and (y, x) word order.  Equivalently, the printed test vectors, when
- * looking at only the bytes (ignoring the whitespace that divides them into
- * words), are backwards: the left-most byte is actually the one with the
- * highest memory address, while the right-most byte is actually the one with
- * the lowest memory address.
- */
-
-static const struct cipher_testvec speck128_enc_tv_template[] = {
-	{ /* Speck128/128 */
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.klen	= 16,
-		.input	= "\x20\x6d\x61\x64\x65\x20\x69\x74"
-			  "\x20\x65\x71\x75\x69\x76\x61\x6c",
-		.ilen	= 16,
-		.result	= "\x18\x0d\x57\x5c\xdf\xfe\x60\x78"
-			  "\x65\x32\x78\x79\x51\x98\x5d\xa6",
-		.rlen	= 16,
-	}, { /* Speck128/192 */
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17",
-		.klen	= 24,
-		.input	= "\x65\x6e\x74\x20\x74\x6f\x20\x43"
-			  "\x68\x69\x65\x66\x20\x48\x61\x72",
-		.ilen	= 16,
-		.result	= "\x86\x18\x3c\xe0\x5d\x18\xbc\xf9"
-			  "\x66\x55\x13\x13\x3a\xcf\xe4\x1b",
-		.rlen	= 16,
-	}, { /* Speck128/256 */
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.klen	= 32,
-		.input	= "\x70\x6f\x6f\x6e\x65\x72\x2e\x20"
-			  "\x49\x6e\x20\x74\x68\x6f\x73\x65",
-		.ilen	= 16,
-		.result	= "\x43\x8f\x18\x9c\x8d\xb4\xee\x4e"
-			  "\x3e\xf5\xc0\x05\x04\x01\x09\x41",
-		.rlen	= 16,
-	},
-};
-
-static const struct cipher_testvec speck128_dec_tv_template[] = {
-	{ /* Speck128/128 */
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.klen	= 16,
-		.input	= "\x18\x0d\x57\x5c\xdf\xfe\x60\x78"
-			  "\x65\x32\x78\x79\x51\x98\x5d\xa6",
-		.ilen	= 16,
-		.result	= "\x20\x6d\x61\x64\x65\x20\x69\x74"
-			  "\x20\x65\x71\x75\x69\x76\x61\x6c",
-		.rlen	= 16,
-	}, { /* Speck128/192 */
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17",
-		.klen	= 24,
-		.input	= "\x86\x18\x3c\xe0\x5d\x18\xbc\xf9"
-			  "\x66\x55\x13\x13\x3a\xcf\xe4\x1b",
-		.ilen	= 16,
-		.result	= "\x65\x6e\x74\x20\x74\x6f\x20\x43"
-			  "\x68\x69\x65\x66\x20\x48\x61\x72",
-		.rlen	= 16,
-	}, { /* Speck128/256 */
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.klen	= 32,
-		.input	= "\x43\x8f\x18\x9c\x8d\xb4\xee\x4e"
-			  "\x3e\xf5\xc0\x05\x04\x01\x09\x41",
-		.ilen	= 16,
-		.result	= "\x70\x6f\x6f\x6e\x65\x72\x2e\x20"
-			  "\x49\x6e\x20\x74\x68\x6f\x73\x65",
-		.rlen	= 16,
-	},
-};
-
-/*
- * Speck128-XTS test vectors, taken from the AES-XTS test vectors with the
- * result recomputed with Speck128 as the cipher
- */
-
-static const struct cipher_testvec speck128_xts_enc_tv_template[] = {
-	{
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 32,
-		.result	= "\xbe\xa0\xe7\x03\xd7\xfe\xab\x62"
-			  "\x3b\x99\x4a\x64\x74\x77\xac\xed"
-			  "\xd8\xf4\xa6\xcf\xae\xb9\x07\x42"
-			  "\x51\xd9\xb6\x1d\xe0\x5e\xbc\x54",
-		.rlen	= 32,
-	}, {
-		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen	= 32,
-		.result	= "\xfb\x53\x81\x75\x6f\x9f\x34\xad"
-			  "\x7e\x01\xed\x7b\xcc\xda\x4e\x4a"
-			  "\xd4\x84\xa4\x53\xd5\x88\x73\x1b"
-			  "\xfd\xcb\xae\x0d\xf3\x04\xee\xe6",
-		.rlen	= 32,
-	}, {
-		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen	= 32,
-		.result	= "\x21\x52\x84\x15\xd1\xf7\x21\x55"
-			  "\xd9\x75\x4a\xd3\xc5\xdb\x9f\x7d"
-			  "\xda\x63\xb2\xf1\x82\xb0\x89\x59"
-			  "\x86\xd4\xaa\xaa\xdd\xff\x4f\x92",
-		.rlen	= 32,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\x57\xb5\xf8\x71\x6e\x6d\xdd\x82"
-			  "\x53\xd0\xed\x2d\x30\xc1\x20\xef"
-			  "\x70\x67\x5e\xff\x09\x70\xbb\xc1"
-			  "\x3a\x7b\x48\x26\xd9\x0b\xf4\x48"
-			  "\xbe\xce\xb1\xc7\xb2\x67\xc4\xa7"
-			  "\x76\xf8\x36\x30\xb7\xb4\x9a\xd9"
-			  "\xf5\x9d\xd0\x7b\xc1\x06\x96\x44"
-			  "\x19\xc5\x58\x84\x63\xb9\x12\x68"
-			  "\x68\xc7\xaa\x18\x98\xf2\x1f\x5c"
-			  "\x39\xa6\xd8\x32\x2b\xc3\x51\xfd"
-			  "\x74\x79\x2e\xb4\x44\xd7\x69\xc4"
-			  "\xfc\x29\xe6\xed\x26\x1e\xa6\x9d"
-			  "\x1c\xbe\x00\x0e\x7f\x3a\xca\xfb"
-			  "\x6d\x13\x65\xa0\xf9\x31\x12\xe2"
-			  "\x26\xd1\xec\x2b\x0a\x8b\x59\x99"
-			  "\xa7\x49\xa0\x0e\x09\x33\x85\x50"
-			  "\xc3\x23\xca\x7a\xdd\x13\x45\x5f"
-			  "\xde\x4c\xa7\xcb\x00\x8a\x66\x6f"
-			  "\xa2\xb6\xb1\x2e\xe1\xa0\x18\xf6"
-			  "\xad\xf3\xbd\xeb\xc7\xef\x55\x4f"
-			  "\x79\x91\x8d\x36\x13\x7b\xd0\x4a"
-			  "\x6c\x39\xfb\x53\xb8\x6f\x02\x51"
-			  "\xa5\x20\xac\x24\x1c\x73\x59\x73"
-			  "\x58\x61\x3a\x87\x58\xb3\x20\x56"
-			  "\x39\x06\x2b\x4d\xd3\x20\x2b\x89"
-			  "\x3f\xa2\xf0\x96\xeb\x7f\xa4\xcd"
-			  "\x11\xae\xbd\xcb\x3a\xb4\xd9\x91"
-			  "\x09\x35\x71\x50\x65\xac\x92\xe3"
-			  "\x7b\x32\xc0\x7a\xdd\xd4\xc3\x92"
-			  "\x6f\xeb\x79\xde\x6f\xd3\x25\xc9"
-			  "\xcd\x63\xf5\x1e\x7a\x3b\x26\x9d"
-			  "\x77\x04\x80\xa9\xbf\x38\xb5\xbd"
-			  "\xb8\x05\x07\xbd\xfd\xab\x7b\xf8"
-			  "\x2a\x26\xcc\x49\x14\x6d\x55\x01"
-			  "\x06\x94\xd8\xb2\x2d\x53\x83\x1b"
-			  "\x8f\xd4\xdd\x57\x12\x7e\x18\xba"
-			  "\x8e\xe2\x4d\x80\xef\x7e\x6b\x9d"
-			  "\x24\xa9\x60\xa4\x97\x85\x86\x2a"
-			  "\x01\x00\x09\xf1\xcb\x4a\x24\x1c"
-			  "\xd8\xf6\xe6\x5b\xe7\x5d\xf2\xc4"
-			  "\x97\x1c\x10\xc6\x4d\x66\x4f\x98"
-			  "\x87\x30\xac\xd5\xea\x73\x49\x10"
-			  "\x80\xea\xe5\x5f\x4d\x5f\x03\x33"
-			  "\x66\x02\x35\x3d\x60\x06\x36\x4f"
-			  "\x14\x1c\xd8\x07\x1f\x78\xd0\xf8"
-			  "\x4f\x6c\x62\x7c\x15\xa5\x7c\x28"
-			  "\x7c\xcc\xeb\x1f\xd1\x07\x90\x93"
-			  "\x7e\xc2\xa8\x3a\x80\xc0\xf5\x30"
-			  "\xcc\x75\xcf\x16\x26\xa9\x26\x3b"
-			  "\xe7\x68\x2f\x15\x21\x5b\xe4\x00"
-			  "\xbd\x48\x50\xcd\x75\x70\xc4\x62"
-			  "\xbb\x41\xfb\x89\x4a\x88\x3b\x3b"
-			  "\x51\x66\x02\x69\x04\x97\x36\xd4"
-			  "\x75\xae\x0b\xa3\x42\xf8\xca\x79"
-			  "\x8f\x93\xe9\xcc\x38\xbd\xd6\xd2"
-			  "\xf9\x70\x4e\xc3\x6a\x8e\x25\xbd"
-			  "\xea\x15\x5a\xa0\x85\x7e\x81\x0d"
-			  "\x03\xe7\x05\x39\xf5\x05\x26\xee"
-			  "\xec\xaa\x1f\x3d\xc9\x98\x76\x01"
-			  "\x2c\xf4\xfc\xa3\x88\x77\x38\xc4"
-			  "\x50\x65\x50\x6d\x04\x1f\xdf\x5a"
-			  "\xaa\xf2\x01\xa9\xc1\x8d\xee\xca"
-			  "\x47\x26\xef\x39\xb8\xb4\xf2\xd1"
-			  "\xd6\xbb\x1b\x2a\xc1\x34\x14\xcf",
-		.rlen	= 512,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\xc5\x85\x2a\x4b\x73\xe4\xf6\xf1"
-			  "\x7e\xf9\xf6\xe9\xa3\x73\x36\xcb"
-			  "\xaa\xb6\x22\xb0\x24\x6e\x3d\x73"
-			  "\x92\x99\xde\xd3\x76\xed\xcd\x63"
-			  "\x64\x3a\x22\x57\xc1\x43\x49\xd4"
-			  "\x79\x36\x31\x19\x62\xae\x10\x7e"
-			  "\x7d\xcf\x7a\xe2\x6b\xce\x27\xfa"
-			  "\xdc\x3d\xd9\x83\xd3\x42\x4c\xe0"
-			  "\x1b\xd6\x1d\x1a\x6f\xd2\x03\x00"
-			  "\xfc\x81\x99\x8a\x14\x62\xf5\x7e"
-			  "\x0d\xe7\x12\xe8\x17\x9d\x0b\xec"
-			  "\xe2\xf7\xc9\xa7\x63\xd1\x79\xb6"
-			  "\x62\x62\x37\xfe\x0a\x4c\x4a\x37"
-			  "\x70\xc7\x5e\x96\x5f\xbc\x8e\x9e"
-			  "\x85\x3c\x4f\x26\x64\x85\xbc\x68"
-			  "\xb0\xe0\x86\x5e\x26\x41\xce\x11"
-			  "\x50\xda\x97\x14\xe9\x9e\xc7\x6d"
-			  "\x3b\xdc\x43\xde\x2b\x27\x69\x7d"
-			  "\xfc\xb0\x28\xbd\x8f\xb1\xc6\x31"
-			  "\x14\x4d\xf0\x74\x37\xfd\x07\x25"
-			  "\x96\x55\xe5\xfc\x9e\x27\x2a\x74"
-			  "\x1b\x83\x4d\x15\x83\xac\x57\xa0"
-			  "\xac\xa5\xd0\x38\xef\x19\x56\x53"
-			  "\x25\x4b\xfc\xce\x04\x23\xe5\x6b"
-			  "\xf6\xc6\x6c\x32\x0b\xb3\x12\xc5"
-			  "\xed\x22\x34\x1c\x5d\xed\x17\x06"
-			  "\x36\xa3\xe6\x77\xb9\x97\x46\xb8"
-			  "\xe9\x3f\x7e\xc7\xbc\x13\x5c\xdc"
-			  "\x6e\x3f\x04\x5e\xd1\x59\xa5\x82"
-			  "\x35\x91\x3d\x1b\xe4\x97\x9f\x92"
-			  "\x1c\x5e\x5f\x6f\x41\xd4\x62\xa1"
-			  "\x8d\x39\xfc\x42\xfb\x38\x80\xb9"
-			  "\x0a\xe3\xcc\x6a\x93\xd9\x7a\xb1"
-			  "\xe9\x69\xaf\x0a\x6b\x75\x38\xa7"
-			  "\xa1\xbf\xf7\xda\x95\x93\x4b\x78"
-			  "\x19\xf5\x94\xf9\xd2\x00\x33\x37"
-			  "\xcf\xf5\x9e\x9c\xf3\xcc\xa6\xee"
-			  "\x42\xb2\x9e\x2c\x5f\x48\x23\x26"
-			  "\x15\x25\x17\x03\x3d\xfe\x2c\xfc"
-			  "\xeb\xba\xda\xe0\x00\x05\xb6\xa6"
-			  "\x07\xb3\xe8\x36\x5b\xec\x5b\xbf"
-			  "\xd6\x5b\x00\x74\xc6\x97\xf1\x6a"
-			  "\x49\xa1\xc3\xfa\x10\x52\xb9\x14"
-			  "\xad\xb7\x73\xf8\x78\x12\xc8\x59"
-			  "\x17\x80\x4c\x57\x39\xf1\x6d\x80"
-			  "\x25\x77\x0f\x5e\x7d\xf0\xaf\x21"
-			  "\xec\xce\xb7\xc8\x02\x8a\xed\x53"
-			  "\x2c\x25\x68\x2e\x1f\x85\x5e\x67"
-			  "\xd1\x07\x7a\x3a\x89\x08\xe0\x34"
-			  "\xdc\xdb\x26\xb4\x6b\x77\xfc\x40"
-			  "\x31\x15\x72\xa0\xf0\x73\xd9\x3b"
-			  "\xd5\xdb\xfe\xfc\x8f\xa9\x44\xa2"
-			  "\x09\x9f\xc6\x33\xe5\xe2\x88\xe8"
-			  "\xf3\xf0\x1a\xf4\xce\x12\x0f\xd6"
-			  "\xf7\x36\xe6\xa4\xf4\x7a\x10\x58"
-			  "\xcc\x1f\x48\x49\x65\x47\x75\xe9"
-			  "\x28\xe1\x65\x7b\xf2\xc4\xb5\x07"
-			  "\xf2\xec\x76\xd8\x8f\x09\xf3\x16"
-			  "\xa1\x51\x89\x3b\xeb\x96\x42\xac"
-			  "\x65\xe0\x67\x63\x29\xdc\xb4\x7d"
-			  "\xf2\x41\x51\x6a\xcb\xde\x3c\xfb"
-			  "\x66\x8d\x13\xca\xe0\x59\x2a\x00"
-			  "\xc9\x53\x4c\xe6\x9e\xe2\x73\xd5"
-			  "\x67\x19\xb2\xbd\x9a\x63\xd7\x5c",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	}
-};
-
-static const struct cipher_testvec speck128_xts_dec_tv_template[] = {
-	{
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xbe\xa0\xe7\x03\xd7\xfe\xab\x62"
-			  "\x3b\x99\x4a\x64\x74\x77\xac\xed"
-			  "\xd8\xf4\xa6\xcf\xae\xb9\x07\x42"
-			  "\x51\xd9\xb6\x1d\xe0\x5e\xbc\x54",
-		.ilen	= 32,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen	= 32,
-	}, {
-		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xfb\x53\x81\x75\x6f\x9f\x34\xad"
-			  "\x7e\x01\xed\x7b\xcc\xda\x4e\x4a"
-			  "\xd4\x84\xa4\x53\xd5\x88\x73\x1b"
-			  "\xfd\xcb\xae\x0d\xf3\x04\xee\xe6",
-		.ilen	= 32,
-		.result	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen	= 32,
-	}, {
-		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x21\x52\x84\x15\xd1\xf7\x21\x55"
-			  "\xd9\x75\x4a\xd3\xc5\xdb\x9f\x7d"
-			  "\xda\x63\xb2\xf1\x82\xb0\x89\x59"
-			  "\x86\xd4\xaa\xaa\xdd\xff\x4f\x92",
-		.ilen	= 32,
-		.result	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen	= 32,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x57\xb5\xf8\x71\x6e\x6d\xdd\x82"
-			  "\x53\xd0\xed\x2d\x30\xc1\x20\xef"
-			  "\x70\x67\x5e\xff\x09\x70\xbb\xc1"
-			  "\x3a\x7b\x48\x26\xd9\x0b\xf4\x48"
-			  "\xbe\xce\xb1\xc7\xb2\x67\xc4\xa7"
-			  "\x76\xf8\x36\x30\xb7\xb4\x9a\xd9"
-			  "\xf5\x9d\xd0\x7b\xc1\x06\x96\x44"
-			  "\x19\xc5\x58\x84\x63\xb9\x12\x68"
-			  "\x68\xc7\xaa\x18\x98\xf2\x1f\x5c"
-			  "\x39\xa6\xd8\x32\x2b\xc3\x51\xfd"
-			  "\x74\x79\x2e\xb4\x44\xd7\x69\xc4"
-			  "\xfc\x29\xe6\xed\x26\x1e\xa6\x9d"
-			  "\x1c\xbe\x00\x0e\x7f\x3a\xca\xfb"
-			  "\x6d\x13\x65\xa0\xf9\x31\x12\xe2"
-			  "\x26\xd1\xec\x2b\x0a\x8b\x59\x99"
-			  "\xa7\x49\xa0\x0e\x09\x33\x85\x50"
-			  "\xc3\x23\xca\x7a\xdd\x13\x45\x5f"
-			  "\xde\x4c\xa7\xcb\x00\x8a\x66\x6f"
-			  "\xa2\xb6\xb1\x2e\xe1\xa0\x18\xf6"
-			  "\xad\xf3\xbd\xeb\xc7\xef\x55\x4f"
-			  "\x79\x91\x8d\x36\x13\x7b\xd0\x4a"
-			  "\x6c\x39\xfb\x53\xb8\x6f\x02\x51"
-			  "\xa5\x20\xac\x24\x1c\x73\x59\x73"
-			  "\x58\x61\x3a\x87\x58\xb3\x20\x56"
-			  "\x39\x06\x2b\x4d\xd3\x20\x2b\x89"
-			  "\x3f\xa2\xf0\x96\xeb\x7f\xa4\xcd"
-			  "\x11\xae\xbd\xcb\x3a\xb4\xd9\x91"
-			  "\x09\x35\x71\x50\x65\xac\x92\xe3"
-			  "\x7b\x32\xc0\x7a\xdd\xd4\xc3\x92"
-			  "\x6f\xeb\x79\xde\x6f\xd3\x25\xc9"
-			  "\xcd\x63\xf5\x1e\x7a\x3b\x26\x9d"
-			  "\x77\x04\x80\xa9\xbf\x38\xb5\xbd"
-			  "\xb8\x05\x07\xbd\xfd\xab\x7b\xf8"
-			  "\x2a\x26\xcc\x49\x14\x6d\x55\x01"
-			  "\x06\x94\xd8\xb2\x2d\x53\x83\x1b"
-			  "\x8f\xd4\xdd\x57\x12\x7e\x18\xba"
-			  "\x8e\xe2\x4d\x80\xef\x7e\x6b\x9d"
-			  "\x24\xa9\x60\xa4\x97\x85\x86\x2a"
-			  "\x01\x00\x09\xf1\xcb\x4a\x24\x1c"
-			  "\xd8\xf6\xe6\x5b\xe7\x5d\xf2\xc4"
-			  "\x97\x1c\x10\xc6\x4d\x66\x4f\x98"
-			  "\x87\x30\xac\xd5\xea\x73\x49\x10"
-			  "\x80\xea\xe5\x5f\x4d\x5f\x03\x33"
-			  "\x66\x02\x35\x3d\x60\x06\x36\x4f"
-			  "\x14\x1c\xd8\x07\x1f\x78\xd0\xf8"
-			  "\x4f\x6c\x62\x7c\x15\xa5\x7c\x28"
-			  "\x7c\xcc\xeb\x1f\xd1\x07\x90\x93"
-			  "\x7e\xc2\xa8\x3a\x80\xc0\xf5\x30"
-			  "\xcc\x75\xcf\x16\x26\xa9\x26\x3b"
-			  "\xe7\x68\x2f\x15\x21\x5b\xe4\x00"
-			  "\xbd\x48\x50\xcd\x75\x70\xc4\x62"
-			  "\xbb\x41\xfb\x89\x4a\x88\x3b\x3b"
-			  "\x51\x66\x02\x69\x04\x97\x36\xd4"
-			  "\x75\xae\x0b\xa3\x42\xf8\xca\x79"
-			  "\x8f\x93\xe9\xcc\x38\xbd\xd6\xd2"
-			  "\xf9\x70\x4e\xc3\x6a\x8e\x25\xbd"
-			  "\xea\x15\x5a\xa0\x85\x7e\x81\x0d"
-			  "\x03\xe7\x05\x39\xf5\x05\x26\xee"
-			  "\xec\xaa\x1f\x3d\xc9\x98\x76\x01"
-			  "\x2c\xf4\xfc\xa3\x88\x77\x38\xc4"
-			  "\x50\x65\x50\x6d\x04\x1f\xdf\x5a"
-			  "\xaa\xf2\x01\xa9\xc1\x8d\xee\xca"
-			  "\x47\x26\xef\x39\xb8\xb4\xf2\xd1"
-			  "\xd6\xbb\x1b\x2a\xc1\x34\x14\xcf",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xc5\x85\x2a\x4b\x73\xe4\xf6\xf1"
-			  "\x7e\xf9\xf6\xe9\xa3\x73\x36\xcb"
-			  "\xaa\xb6\x22\xb0\x24\x6e\x3d\x73"
-			  "\x92\x99\xde\xd3\x76\xed\xcd\x63"
-			  "\x64\x3a\x22\x57\xc1\x43\x49\xd4"
-			  "\x79\x36\x31\x19\x62\xae\x10\x7e"
-			  "\x7d\xcf\x7a\xe2\x6b\xce\x27\xfa"
-			  "\xdc\x3d\xd9\x83\xd3\x42\x4c\xe0"
-			  "\x1b\xd6\x1d\x1a\x6f\xd2\x03\x00"
-			  "\xfc\x81\x99\x8a\x14\x62\xf5\x7e"
-			  "\x0d\xe7\x12\xe8\x17\x9d\x0b\xec"
-			  "\xe2\xf7\xc9\xa7\x63\xd1\x79\xb6"
-			  "\x62\x62\x37\xfe\x0a\x4c\x4a\x37"
-			  "\x70\xc7\x5e\x96\x5f\xbc\x8e\x9e"
-			  "\x85\x3c\x4f\x26\x64\x85\xbc\x68"
-			  "\xb0\xe0\x86\x5e\x26\x41\xce\x11"
-			  "\x50\xda\x97\x14\xe9\x9e\xc7\x6d"
-			  "\x3b\xdc\x43\xde\x2b\x27\x69\x7d"
-			  "\xfc\xb0\x28\xbd\x8f\xb1\xc6\x31"
-			  "\x14\x4d\xf0\x74\x37\xfd\x07\x25"
-			  "\x96\x55\xe5\xfc\x9e\x27\x2a\x74"
-			  "\x1b\x83\x4d\x15\x83\xac\x57\xa0"
-			  "\xac\xa5\xd0\x38\xef\x19\x56\x53"
-			  "\x25\x4b\xfc\xce\x04\x23\xe5\x6b"
-			  "\xf6\xc6\x6c\x32\x0b\xb3\x12\xc5"
-			  "\xed\x22\x34\x1c\x5d\xed\x17\x06"
-			  "\x36\xa3\xe6\x77\xb9\x97\x46\xb8"
-			  "\xe9\x3f\x7e\xc7\xbc\x13\x5c\xdc"
-			  "\x6e\x3f\x04\x5e\xd1\x59\xa5\x82"
-			  "\x35\x91\x3d\x1b\xe4\x97\x9f\x92"
-			  "\x1c\x5e\x5f\x6f\x41\xd4\x62\xa1"
-			  "\x8d\x39\xfc\x42\xfb\x38\x80\xb9"
-			  "\x0a\xe3\xcc\x6a\x93\xd9\x7a\xb1"
-			  "\xe9\x69\xaf\x0a\x6b\x75\x38\xa7"
-			  "\xa1\xbf\xf7\xda\x95\x93\x4b\x78"
-			  "\x19\xf5\x94\xf9\xd2\x00\x33\x37"
-			  "\xcf\xf5\x9e\x9c\xf3\xcc\xa6\xee"
-			  "\x42\xb2\x9e\x2c\x5f\x48\x23\x26"
-			  "\x15\x25\x17\x03\x3d\xfe\x2c\xfc"
-			  "\xeb\xba\xda\xe0\x00\x05\xb6\xa6"
-			  "\x07\xb3\xe8\x36\x5b\xec\x5b\xbf"
-			  "\xd6\x5b\x00\x74\xc6\x97\xf1\x6a"
-			  "\x49\xa1\xc3\xfa\x10\x52\xb9\x14"
-			  "\xad\xb7\x73\xf8\x78\x12\xc8\x59"
-			  "\x17\x80\x4c\x57\x39\xf1\x6d\x80"
-			  "\x25\x77\x0f\x5e\x7d\xf0\xaf\x21"
-			  "\xec\xce\xb7\xc8\x02\x8a\xed\x53"
-			  "\x2c\x25\x68\x2e\x1f\x85\x5e\x67"
-			  "\xd1\x07\x7a\x3a\x89\x08\xe0\x34"
-			  "\xdc\xdb\x26\xb4\x6b\x77\xfc\x40"
-			  "\x31\x15\x72\xa0\xf0\x73\xd9\x3b"
-			  "\xd5\xdb\xfe\xfc\x8f\xa9\x44\xa2"
-			  "\x09\x9f\xc6\x33\xe5\xe2\x88\xe8"
-			  "\xf3\xf0\x1a\xf4\xce\x12\x0f\xd6"
-			  "\xf7\x36\xe6\xa4\xf4\x7a\x10\x58"
-			  "\xcc\x1f\x48\x49\x65\x47\x75\xe9"
-			  "\x28\xe1\x65\x7b\xf2\xc4\xb5\x07"
-			  "\xf2\xec\x76\xd8\x8f\x09\xf3\x16"
-			  "\xa1\x51\x89\x3b\xeb\x96\x42\xac"
-			  "\x65\xe0\x67\x63\x29\xdc\xb4\x7d"
-			  "\xf2\x41\x51\x6a\xcb\xde\x3c\xfb"
-			  "\x66\x8d\x13\xca\xe0\x59\x2a\x00"
-			  "\xc9\x53\x4c\xe6\x9e\xe2\x73\xd5"
-			  "\x67\x19\xb2\xbd\x9a\x63\xd7\x5c",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	}
-};
-
-static const struct cipher_testvec speck64_enc_tv_template[] = {
-	{ /* Speck64/96 */
-		.key	= "\x00\x01\x02\x03\x08\x09\x0a\x0b"
-			  "\x10\x11\x12\x13",
-		.klen	= 12,
-		.input	= "\x65\x61\x6e\x73\x20\x46\x61\x74",
-		.ilen	= 8,
-		.result	= "\x6c\x94\x75\x41\xec\x52\x79\x9f",
-		.rlen	= 8,
-	}, { /* Speck64/128 */
-		.key	= "\x00\x01\x02\x03\x08\x09\x0a\x0b"
-			  "\x10\x11\x12\x13\x18\x19\x1a\x1b",
-		.klen	= 16,
-		.input	= "\x2d\x43\x75\x74\x74\x65\x72\x3b",
-		.ilen	= 8,
-		.result	= "\x8b\x02\x4e\x45\x48\xa5\x6f\x8c",
-		.rlen	= 8,
-	},
-};
-
-static const struct cipher_testvec speck64_dec_tv_template[] = {
-	{ /* Speck64/96 */
-		.key	= "\x00\x01\x02\x03\x08\x09\x0a\x0b"
-			  "\x10\x11\x12\x13",
-		.klen	= 12,
-		.input	= "\x6c\x94\x75\x41\xec\x52\x79\x9f",
-		.ilen	= 8,
-		.result	= "\x65\x61\x6e\x73\x20\x46\x61\x74",
-		.rlen	= 8,
-	}, { /* Speck64/128 */
-		.key	= "\x00\x01\x02\x03\x08\x09\x0a\x0b"
-			  "\x10\x11\x12\x13\x18\x19\x1a\x1b",
-		.klen	= 16,
-		.input	= "\x8b\x02\x4e\x45\x48\xa5\x6f\x8c",
-		.ilen	= 8,
-		.result	= "\x2d\x43\x75\x74\x74\x65\x72\x3b",
-		.rlen	= 8,
-	},
-};
-
-/*
- * Speck64-XTS test vectors, taken from the AES-XTS test vectors with the result
- * recomputed with Speck64 as the cipher, and key lengths adjusted
- */
-
-static const struct cipher_testvec speck64_xts_enc_tv_template[] = {
-	{
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 24,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 32,
-		.result	= "\x84\xaf\x54\x07\x19\xd4\x7c\xa6"
-			  "\xe4\xfe\xdf\xc4\x1f\x34\xc3\xc2"
-			  "\x80\xf5\x72\xe7\xcd\xf0\x99\x22"
-			  "\x35\xa7\x2f\x06\xef\xdc\x51\xaa",
-		.rlen	= 32,
-	}, {
-		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 24,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen	= 32,
-		.result	= "\x12\x56\x73\xcd\x15\x87\xa8\x59"
-			  "\xcf\x84\xae\xd9\x1c\x66\xd6\x9f"
-			  "\xb3\x12\x69\x7e\x36\xeb\x52\xff"
-			  "\x62\xdd\xba\x90\xb3\xe1\xee\x99",
-		.rlen	= 32,
-	}, {
-		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 24,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen	= 32,
-		.result	= "\x15\x1b\xe4\x2c\xa2\x5a\x2d\x2c"
-			  "\x27\x36\xc0\xbf\x5d\xea\x36\x37"
-			  "\x2d\x1a\x88\xbc\x66\xb5\xd0\x0b"
-			  "\xa1\xbc\x19\xb2\x0f\x3b\x75\x34",
-		.rlen	= 32,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93",
-		.klen	= 24,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\xaf\xa1\x81\xa6\x32\xbb\x15\x8e"
-			  "\xf8\x95\x2e\xd3\xe6\xee\x7e\x09"
-			  "\x0c\x1a\xf5\x02\x97\x8b\xe3\xb3"
-			  "\x11\xc7\x39\x96\xd0\x95\xf4\x56"
-			  "\xf4\xdd\x03\x38\x01\x44\x2c\xcf"
-			  "\x88\xae\x8e\x3c\xcd\xe7\xaa\x66"
-			  "\xfe\x3d\xc6\xfb\x01\x23\x51\x43"
-			  "\xd5\xd2\x13\x86\x94\x34\xe9\x62"
-			  "\xf9\x89\xe3\xd1\x7b\xbe\xf8\xef"
-			  "\x76\x35\x04\x3f\xdb\x23\x9d\x0b"
-			  "\x85\x42\xb9\x02\xd6\xcc\xdb\x96"
-			  "\xa7\x6b\x27\xb6\xd4\x45\x8f\x7d"
-			  "\xae\xd2\x04\xd5\xda\xc1\x7e\x24"
-			  "\x8c\x73\xbe\x48\x7e\xcf\x65\x28"
-			  "\x29\xe5\xbe\x54\x30\xcb\x46\x95"
-			  "\x4f\x2e\x8a\x36\xc8\x27\xc5\xbe"
-			  "\xd0\x1a\xaf\xab\x26\xcd\x9e\x69"
-			  "\xa1\x09\x95\x71\x26\xe9\xc4\xdf"
-			  "\xe6\x31\xc3\x46\xda\xaf\x0b\x41"
-			  "\x1f\xab\xb1\x8e\xd6\xfc\x0b\xb3"
-			  "\x82\xc0\x37\x27\xfc\x91\xa7\x05"
-			  "\xfb\xc5\xdc\x2b\x74\x96\x48\x43"
-			  "\x5d\x9c\x19\x0f\x60\x63\x3a\x1f"
-			  "\x6f\xf0\x03\xbe\x4d\xfd\xc8\x4a"
-			  "\xc6\xa4\x81\x6d\xc3\x12\x2a\x5c"
-			  "\x07\xff\xf3\x72\x74\x48\xb5\x40"
-			  "\x50\xb5\xdd\x90\x43\x31\x18\x15"
-			  "\x7b\xf2\xa6\xdb\x83\xc8\x4b\x4a"
-			  "\x29\x93\x90\x8b\xda\x07\xf0\x35"
-			  "\x6d\x90\x88\x09\x4e\x83\xf5\x5b"
-			  "\x94\x12\xbb\x33\x27\x1d\x3f\x23"
-			  "\x51\xa8\x7c\x07\xa2\xae\x77\xa6"
-			  "\x50\xfd\xcc\xc0\x4f\x80\x7a\x9f"
-			  "\x66\xdd\xcd\x75\x24\x8b\x33\xf7"
-			  "\x20\xdb\x83\x9b\x4f\x11\x63\x6e"
-			  "\xcf\x37\xef\xc9\x11\x01\x5c\x45"
-			  "\x32\x99\x7c\x3c\x9e\x42\x89\xe3"
-			  "\x70\x6d\x15\x9f\xb1\xe6\xb6\x05"
-			  "\xfe\x0c\xb9\x49\x2d\x90\x6d\xcc"
-			  "\x5d\x3f\xc1\xfe\x89\x0a\x2e\x2d"
-			  "\xa0\xa8\x89\x3b\x73\x39\xa5\x94"
-			  "\x4c\xa4\xa6\xbb\xa7\x14\x46\x89"
-			  "\x10\xff\xaf\xef\xca\xdd\x4f\x80"
-			  "\xb3\xdf\x3b\xab\xd4\xe5\x5a\xc7"
-			  "\x33\xca\x00\x8b\x8b\x3f\xea\xec"
-			  "\x68\x8a\xc2\x6d\xfd\xd4\x67\x0f"
-			  "\x22\x31\xe1\x0e\xfe\x5a\x04\xd5"
-			  "\x64\xa3\xf1\x1a\x76\x28\xcc\x35"
-			  "\x36\xa7\x0a\x74\xf7\x1c\x44\x9b"
-			  "\xc7\x1b\x53\x17\x02\xea\xd1\xad"
-			  "\x13\x51\x73\xc0\xa0\xb2\x05\x32"
-			  "\xa8\xa2\x37\x2e\xe1\x7a\x3a\x19"
-			  "\x26\xb4\x6c\x62\x5d\xb3\x1a\x1d"
-			  "\x59\xda\xee\x1a\x22\x18\xda\x0d"
-			  "\x88\x0f\x55\x8b\x72\x62\xfd\xc1"
-			  "\x69\x13\xcd\x0d\x5f\xc1\x09\x52"
-			  "\xee\xd6\xe3\x84\x4d\xee\xf6\x88"
-			  "\xaf\x83\xdc\x76\xf4\xc0\x93\x3f"
-			  "\x4a\x75\x2f\xb0\x0b\x3e\xc4\x54"
-			  "\x7d\x69\x8d\x00\x62\x77\x0d\x14"
-			  "\xbe\x7c\xa6\x7d\xc5\x24\x4f\xf3"
-			  "\x50\xf7\x5f\xf4\xc2\xca\x41\x97"
-			  "\x37\xbe\x75\x74\xcd\xf0\x75\x6e"
-			  "\x25\x23\x94\xbd\xda\x8d\xb0\xd4",
-		.rlen	= 512,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27",
-		.klen	= 32,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\x55\xed\x71\xd3\x02\x8e\x15\x3b"
-			  "\xc6\x71\x29\x2d\x3e\x89\x9f\x59"
-			  "\x68\x6a\xcc\x8a\x56\x97\xf3\x95"
-			  "\x4e\x51\x08\xda\x2a\xf8\x6f\x3c"
-			  "\x78\x16\xea\x80\xdb\x33\x75\x94"
-			  "\xf9\x29\xc4\x2b\x76\x75\x97\xc7"
-			  "\xf2\x98\x2c\xf9\xff\xc8\xd5\x2b"
-			  "\x18\xf1\xaf\xcf\x7c\xc5\x0b\xee"
-			  "\xad\x3c\x76\x7c\xe6\x27\xa2\x2a"
-			  "\xe4\x66\xe1\xab\xa2\x39\xfc\x7c"
-			  "\xf5\xec\x32\x74\xa3\xb8\x03\x88"
-			  "\x52\xfc\x2e\x56\x3f\xa1\xf0\x9f"
-			  "\x84\x5e\x46\xed\x20\x89\xb6\x44"
-			  "\x8d\xd0\xed\x54\x47\x16\xbe\x95"
-			  "\x8a\xb3\x6b\x72\xc4\x32\x52\x13"
-			  "\x1b\xb0\x82\xbe\xac\xf9\x70\xa6"
-			  "\x44\x18\xdd\x8c\x6e\xca\x6e\x45"
-			  "\x8f\x1e\x10\x07\x57\x25\x98\x7b"
-			  "\x17\x8c\x78\xdd\x80\xa7\xd9\xd8"
-			  "\x63\xaf\xb9\x67\x57\xfd\xbc\xdb"
-			  "\x44\xe9\xc5\x65\xd1\xc7\x3b\xff"
-			  "\x20\xa0\x80\x1a\xc3\x9a\xad\x5e"
-			  "\x5d\x3b\xd3\x07\xd9\xf5\xfd\x3d"
-			  "\x4a\x8b\xa8\xd2\x6e\x7a\x51\x65"
-			  "\x6c\x8e\x95\xe0\x45\xc9\x5f\x4a"
-			  "\x09\x3c\x3d\x71\x7f\x0c\x84\x2a"
-			  "\xc8\x48\x52\x1a\xc2\xd5\xd6\x78"
-			  "\x92\x1e\xa0\x90\x2e\xea\xf0\xf3"
-			  "\xdc\x0f\xb1\xaf\x0d\x9b\x06\x2e"
-			  "\x35\x10\x30\x82\x0d\xe7\xc5\x9b"
-			  "\xde\x44\x18\xbd\x9f\xd1\x45\xa9"
-			  "\x7b\x7a\x4a\xad\x35\x65\x27\xca"
-			  "\xb2\xc3\xd4\x9b\x71\x86\x70\xee"
-			  "\xf1\x89\x3b\x85\x4b\x5b\xaa\xaf"
-			  "\xfc\x42\xc8\x31\x59\xbe\x16\x60"
-			  "\x4f\xf9\xfa\x12\xea\xd0\xa7\x14"
-			  "\xf0\x7a\xf3\xd5\x8d\xbd\x81\xef"
-			  "\x52\x7f\x29\x51\x94\x20\x67\x3c"
-			  "\xd1\xaf\x77\x9f\x22\x5a\x4e\x63"
-			  "\xe7\xff\x73\x25\xd1\xdd\x96\x8a"
-			  "\x98\x52\x6d\xf3\xac\x3e\xf2\x18"
-			  "\x6d\xf6\x0a\x29\xa6\x34\x3d\xed"
-			  "\xe3\x27\x0d\x9d\x0a\x02\x44\x7e"
-			  "\x5a\x7e\x67\x0f\x0a\x9e\xd6\xad"
-			  "\x91\xe6\x4d\x81\x8c\x5c\x59\xaa"
-			  "\xfb\xeb\x56\x53\xd2\x7d\x4c\x81"
-			  "\x65\x53\x0f\x41\x11\xbd\x98\x99"
-			  "\xf9\xc6\xfa\x51\x2e\xa3\xdd\x8d"
-			  "\x84\x98\xf9\x34\xed\x33\x2a\x1f"
-			  "\x82\xed\xc1\x73\x98\xd3\x02\xdc"
-			  "\xe6\xc2\x33\x1d\xa2\xb4\xca\x76"
-			  "\x63\x51\x34\x9d\x96\x12\xae\xce"
-			  "\x83\xc9\x76\x5e\xa4\x1b\x53\x37"
-			  "\x17\xd5\xc0\x80\x1d\x62\xf8\x3d"
-			  "\x54\x27\x74\xbb\x10\x86\x57\x46"
-			  "\x68\xe1\xed\x14\xe7\x9d\xfc\x84"
-			  "\x47\xbc\xc2\xf8\x19\x4b\x99\xcf"
-			  "\x7a\xe9\xc4\xb8\x8c\x82\x72\x4d"
-			  "\x7b\x4f\x38\x55\x36\x71\x64\xc1"
-			  "\xfc\x5c\x75\x52\x33\x02\x18\xf8"
-			  "\x17\xe1\x2b\xc2\x43\x39\xbd\x76"
-			  "\x9b\x63\x76\x32\x2f\x19\x72\x10"
-			  "\x9f\x21\x0c\xf1\x66\x50\x7f\xa5"
-			  "\x0d\x1f\x46\xe0\xba\xd3\x2f\x3c",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	}
-};
-
-static const struct cipher_testvec speck64_xts_dec_tv_template[] = {
-	{
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 24,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x84\xaf\x54\x07\x19\xd4\x7c\xa6"
-			  "\xe4\xfe\xdf\xc4\x1f\x34\xc3\xc2"
-			  "\x80\xf5\x72\xe7\xcd\xf0\x99\x22"
-			  "\x35\xa7\x2f\x06\xef\xdc\x51\xaa",
-		.ilen	= 32,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen	= 32,
-	}, {
-		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 24,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x12\x56\x73\xcd\x15\x87\xa8\x59"
-			  "\xcf\x84\xae\xd9\x1c\x66\xd6\x9f"
-			  "\xb3\x12\x69\x7e\x36\xeb\x52\xff"
-			  "\x62\xdd\xba\x90\xb3\xe1\xee\x99",
-		.ilen	= 32,
-		.result	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen	= 32,
-	}, {
-		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 24,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x15\x1b\xe4\x2c\xa2\x5a\x2d\x2c"
-			  "\x27\x36\xc0\xbf\x5d\xea\x36\x37"
-			  "\x2d\x1a\x88\xbc\x66\xb5\xd0\x0b"
-			  "\xa1\xbc\x19\xb2\x0f\x3b\x75\x34",
-		.ilen	= 32,
-		.result	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen	= 32,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93",
-		.klen	= 24,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xaf\xa1\x81\xa6\x32\xbb\x15\x8e"
-			  "\xf8\x95\x2e\xd3\xe6\xee\x7e\x09"
-			  "\x0c\x1a\xf5\x02\x97\x8b\xe3\xb3"
-			  "\x11\xc7\x39\x96\xd0\x95\xf4\x56"
-			  "\xf4\xdd\x03\x38\x01\x44\x2c\xcf"
-			  "\x88\xae\x8e\x3c\xcd\xe7\xaa\x66"
-			  "\xfe\x3d\xc6\xfb\x01\x23\x51\x43"
-			  "\xd5\xd2\x13\x86\x94\x34\xe9\x62"
-			  "\xf9\x89\xe3\xd1\x7b\xbe\xf8\xef"
-			  "\x76\x35\x04\x3f\xdb\x23\x9d\x0b"
-			  "\x85\x42\xb9\x02\xd6\xcc\xdb\x96"
-			  "\xa7\x6b\x27\xb6\xd4\x45\x8f\x7d"
-			  "\xae\xd2\x04\xd5\xda\xc1\x7e\x24"
-			  "\x8c\x73\xbe\x48\x7e\xcf\x65\x28"
-			  "\x29\xe5\xbe\x54\x30\xcb\x46\x95"
-			  "\x4f\x2e\x8a\x36\xc8\x27\xc5\xbe"
-			  "\xd0\x1a\xaf\xab\x26\xcd\x9e\x69"
-			  "\xa1\x09\x95\x71\x26\xe9\xc4\xdf"
-			  "\xe6\x31\xc3\x46\xda\xaf\x0b\x41"
-			  "\x1f\xab\xb1\x8e\xd6\xfc\x0b\xb3"
-			  "\x82\xc0\x37\x27\xfc\x91\xa7\x05"
-			  "\xfb\xc5\xdc\x2b\x74\x96\x48\x43"
-			  "\x5d\x9c\x19\x0f\x60\x63\x3a\x1f"
-			  "\x6f\xf0\x03\xbe\x4d\xfd\xc8\x4a"
-			  "\xc6\xa4\x81\x6d\xc3\x12\x2a\x5c"
-			  "\x07\xff\xf3\x72\x74\x48\xb5\x40"
-			  "\x50\xb5\xdd\x90\x43\x31\x18\x15"
-			  "\x7b\xf2\xa6\xdb\x83\xc8\x4b\x4a"
-			  "\x29\x93\x90\x8b\xda\x07\xf0\x35"
-			  "\x6d\x90\x88\x09\x4e\x83\xf5\x5b"
-			  "\x94\x12\xbb\x33\x27\x1d\x3f\x23"
-			  "\x51\xa8\x7c\x07\xa2\xae\x77\xa6"
-			  "\x50\xfd\xcc\xc0\x4f\x80\x7a\x9f"
-			  "\x66\xdd\xcd\x75\x24\x8b\x33\xf7"
-			  "\x20\xdb\x83\x9b\x4f\x11\x63\x6e"
-			  "\xcf\x37\xef\xc9\x11\x01\x5c\x45"
-			  "\x32\x99\x7c\x3c\x9e\x42\x89\xe3"
-			  "\x70\x6d\x15\x9f\xb1\xe6\xb6\x05"
-			  "\xfe\x0c\xb9\x49\x2d\x90\x6d\xcc"
-			  "\x5d\x3f\xc1\xfe\x89\x0a\x2e\x2d"
-			  "\xa0\xa8\x89\x3b\x73\x39\xa5\x94"
-			  "\x4c\xa4\xa6\xbb\xa7\x14\x46\x89"
-			  "\x10\xff\xaf\xef\xca\xdd\x4f\x80"
-			  "\xb3\xdf\x3b\xab\xd4\xe5\x5a\xc7"
-			  "\x33\xca\x00\x8b\x8b\x3f\xea\xec"
-			  "\x68\x8a\xc2\x6d\xfd\xd4\x67\x0f"
-			  "\x22\x31\xe1\x0e\xfe\x5a\x04\xd5"
-			  "\x64\xa3\xf1\x1a\x76\x28\xcc\x35"
-			  "\x36\xa7\x0a\x74\xf7\x1c\x44\x9b"
-			  "\xc7\x1b\x53\x17\x02\xea\xd1\xad"
-			  "\x13\x51\x73\xc0\xa0\xb2\x05\x32"
-			  "\xa8\xa2\x37\x2e\xe1\x7a\x3a\x19"
-			  "\x26\xb4\x6c\x62\x5d\xb3\x1a\x1d"
-			  "\x59\xda\xee\x1a\x22\x18\xda\x0d"
-			  "\x88\x0f\x55\x8b\x72\x62\xfd\xc1"
-			  "\x69\x13\xcd\x0d\x5f\xc1\x09\x52"
-			  "\xee\xd6\xe3\x84\x4d\xee\xf6\x88"
-			  "\xaf\x83\xdc\x76\xf4\xc0\x93\x3f"
-			  "\x4a\x75\x2f\xb0\x0b\x3e\xc4\x54"
-			  "\x7d\x69\x8d\x00\x62\x77\x0d\x14"
-			  "\xbe\x7c\xa6\x7d\xc5\x24\x4f\xf3"
-			  "\x50\xf7\x5f\xf4\xc2\xca\x41\x97"
-			  "\x37\xbe\x75\x74\xcd\xf0\x75\x6e"
-			  "\x25\x23\x94\xbd\xda\x8d\xb0\xd4",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27",
-		.klen	= 32,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x55\xed\x71\xd3\x02\x8e\x15\x3b"
-			  "\xc6\x71\x29\x2d\x3e\x89\x9f\x59"
-			  "\x68\x6a\xcc\x8a\x56\x97\xf3\x95"
-			  "\x4e\x51\x08\xda\x2a\xf8\x6f\x3c"
-			  "\x78\x16\xea\x80\xdb\x33\x75\x94"
-			  "\xf9\x29\xc4\x2b\x76\x75\x97\xc7"
-			  "\xf2\x98\x2c\xf9\xff\xc8\xd5\x2b"
-			  "\x18\xf1\xaf\xcf\x7c\xc5\x0b\xee"
-			  "\xad\x3c\x76\x7c\xe6\x27\xa2\x2a"
-			  "\xe4\x66\xe1\xab\xa2\x39\xfc\x7c"
-			  "\xf5\xec\x32\x74\xa3\xb8\x03\x88"
-			  "\x52\xfc\x2e\x56\x3f\xa1\xf0\x9f"
-			  "\x84\x5e\x46\xed\x20\x89\xb6\x44"
-			  "\x8d\xd0\xed\x54\x47\x16\xbe\x95"
-			  "\x8a\xb3\x6b\x72\xc4\x32\x52\x13"
-			  "\x1b\xb0\x82\xbe\xac\xf9\x70\xa6"
-			  "\x44\x18\xdd\x8c\x6e\xca\x6e\x45"
-			  "\x8f\x1e\x10\x07\x57\x25\x98\x7b"
-			  "\x17\x8c\x78\xdd\x80\xa7\xd9\xd8"
-			  "\x63\xaf\xb9\x67\x57\xfd\xbc\xdb"
-			  "\x44\xe9\xc5\x65\xd1\xc7\x3b\xff"
-			  "\x20\xa0\x80\x1a\xc3\x9a\xad\x5e"
-			  "\x5d\x3b\xd3\x07\xd9\xf5\xfd\x3d"
-			  "\x4a\x8b\xa8\xd2\x6e\x7a\x51\x65"
-			  "\x6c\x8e\x95\xe0\x45\xc9\x5f\x4a"
-			  "\x09\x3c\x3d\x71\x7f\x0c\x84\x2a"
-			  "\xc8\x48\x52\x1a\xc2\xd5\xd6\x78"
-			  "\x92\x1e\xa0\x90\x2e\xea\xf0\xf3"
-			  "\xdc\x0f\xb1\xaf\x0d\x9b\x06\x2e"
-			  "\x35\x10\x30\x82\x0d\xe7\xc5\x9b"
-			  "\xde\x44\x18\xbd\x9f\xd1\x45\xa9"
-			  "\x7b\x7a\x4a\xad\x35\x65\x27\xca"
-			  "\xb2\xc3\xd4\x9b\x71\x86\x70\xee"
-			  "\xf1\x89\x3b\x85\x4b\x5b\xaa\xaf"
-			  "\xfc\x42\xc8\x31\x59\xbe\x16\x60"
-			  "\x4f\xf9\xfa\x12\xea\xd0\xa7\x14"
-			  "\xf0\x7a\xf3\xd5\x8d\xbd\x81\xef"
-			  "\x52\x7f\x29\x51\x94\x20\x67\x3c"
-			  "\xd1\xaf\x77\x9f\x22\x5a\x4e\x63"
-			  "\xe7\xff\x73\x25\xd1\xdd\x96\x8a"
-			  "\x98\x52\x6d\xf3\xac\x3e\xf2\x18"
-			  "\x6d\xf6\x0a\x29\xa6\x34\x3d\xed"
-			  "\xe3\x27\x0d\x9d\x0a\x02\x44\x7e"
-			  "\x5a\x7e\x67\x0f\x0a\x9e\xd6\xad"
-			  "\x91\xe6\x4d\x81\x8c\x5c\x59\xaa"
-			  "\xfb\xeb\x56\x53\xd2\x7d\x4c\x81"
-			  "\x65\x53\x0f\x41\x11\xbd\x98\x99"
-			  "\xf9\xc6\xfa\x51\x2e\xa3\xdd\x8d"
-			  "\x84\x98\xf9\x34\xed\x33\x2a\x1f"
-			  "\x82\xed\xc1\x73\x98\xd3\x02\xdc"
-			  "\xe6\xc2\x33\x1d\xa2\xb4\xca\x76"
-			  "\x63\x51\x34\x9d\x96\x12\xae\xce"
-			  "\x83\xc9\x76\x5e\xa4\x1b\x53\x37"
-			  "\x17\xd5\xc0\x80\x1d\x62\xf8\x3d"
-			  "\x54\x27\x74\xbb\x10\x86\x57\x46"
-			  "\x68\xe1\xed\x14\xe7\x9d\xfc\x84"
-			  "\x47\xbc\xc2\xf8\x19\x4b\x99\xcf"
-			  "\x7a\xe9\xc4\xb8\x8c\x82\x72\x4d"
-			  "\x7b\x4f\x38\x55\x36\x71\x64\xc1"
-			  "\xfc\x5c\x75\x52\x33\x02\x18\xf8"
-			  "\x17\xe1\x2b\xc2\x43\x39\xbd\x76"
-			  "\x9b\x63\x76\x32\x2f\x19\x72\x10"
-			  "\x9f\x21\x0c\xf1\x66\x50\x7f\xa5"
-			  "\x0d\x1f\x46\xe0\xba\xd3\x2f\x3c",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	}
-};
-
-/* Cast6 test vectors from RFC 2612 */
-static const struct cipher_testvec cast6_enc_tv_template[] = {
-	{
-		.key	= "\x23\x42\xbb\x9e\xfa\x38\x54\x2c"
-			  "\x0a\xf7\x56\x47\xf2\x9f\x61\x5d",
-		.klen	= 16,
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.result	= "\xc8\x42\xa0\x89\x72\xb4\x3d\x20"
-			  "\x83\x6c\x91\xd1\xb7\x53\x0f\x6b",
-		.rlen	= 16,
-	}, {
-		.key	= "\x23\x42\xbb\x9e\xfa\x38\x54\x2c"
-			  "\xbe\xd0\xac\x83\x94\x0a\xc2\x98"
-			  "\xba\xc7\x7a\x77\x17\x94\x28\x63",
-		.klen	= 24,
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.result	= "\x1b\x38\x6c\x02\x10\xdc\xad\xcb"
-			  "\xdd\x0e\x41\xaa\x08\xa7\xa7\xe8",
-		.rlen	= 16,
-	}, {
-		.key	= "\x23\x42\xbb\x9e\xfa\x38\x54\x2c"
-			  "\xbe\xd0\xac\x83\x94\x0a\xc2\x98"
-			  "\x8d\x7c\x47\xce\x26\x49\x08\x46"
-			  "\x1c\xc1\xb5\x13\x7a\xe6\xb6\x04",
-		.klen	= 32,
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.result	= "\x4f\x6a\x20\x38\x28\x68\x97\xb9"
-			  "\xc9\x87\x01\x36\x55\x33\x17\xfa",
-		.rlen	= 16,
-	}, { /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\xC3\x70\x22\x32\xF5\x80\xCB\x54"
-			  "\xFC\x30\xE0\xF6\xEB\x39\x57\xA6"
-			  "\xB6\xB9\xC5\xA4\x91\x55\x14\x97"
-			  "\xC1\x20\xFF\x6C\x5C\xF0\x67\xEA"
-			  "\x2F\xED\xD8\xC9\xFB\x38\x3F\xFE"
-			  "\x93\xBE\xDC\x00\xD3\x7F\xAD\x4C"
-			  "\x5A\x08\x92\xD1\x47\x0C\xFA\x6C"
-			  "\xD0\x6A\x99\x10\x72\xF8\x47\x62"
-			  "\x81\x42\xF8\xD8\xF5\xBB\x94\x08"
-			  "\xAA\x97\xA2\x8B\x69\xB3\xD2\x7E"
-			  "\xBC\xB5\x00\x0C\xE5\x44\x4B\x58"
-			  "\xE8\x63\xDC\xB3\xC4\xE5\x23\x12"
-			  "\x5A\x72\x85\x47\x8B\xEC\x9F\x26"
-			  "\x84\xB6\xED\x10\x33\x63\x9B\x5F"
-			  "\x4D\x53\xEE\x94\x45\x8B\x60\x58"
-			  "\x86\x20\xF9\x1E\x82\x08\x3E\x58"
-			  "\x60\x1B\x34\x19\x02\xBE\x4E\x09"
-			  "\xBB\x7C\x15\xCC\x60\x27\x55\x7A"
-			  "\x12\xB8\xD8\x08\x89\x3C\xA6\xF3"
-			  "\xF1\xDD\xA7\x07\xA3\x12\x85\x28"
-			  "\xE9\x57\xAC\x80\x0C\x5C\x0F\x3A"
-			  "\x5D\xC2\x91\xC7\x90\xE4\x8C\x43"
-			  "\x92\xE4\x7C\x26\x69\x4D\x83\x68"
-			  "\x14\x96\x42\x47\xBD\xA9\xE4\x8A"
-			  "\x33\x19\xEB\x54\x8E\x0D\x4B\x6E"
-			  "\x91\x51\xB5\x36\x08\xDE\x1C\x06"
-			  "\x03\xBD\xDE\x81\x26\xF7\x99\xC2"
-			  "\xBA\xF7\x6D\x87\x0D\xE4\xA6\xCF"
-			  "\xC1\xF5\x27\x05\xB8\x02\x57\x72"
-			  "\xE6\x42\x13\x0B\xC6\x47\x05\x74"
-			  "\x24\x15\xF7\x0D\xC2\x23\x9D\xB9"
-			  "\x3C\x77\x18\x93\xBA\xB4\xFC\x8C"
-			  "\x98\x82\x67\x67\xB4\xD7\xD3\x43"
-			  "\x23\x08\x02\xB7\x9B\x99\x05\xFB"
-			  "\xD3\xB5\x00\x0A\xA9\x9D\x66\xD6"
-			  "\x2E\x49\x58\xD0\xA8\x57\x29\x7F"
-			  "\x0A\x0E\x7D\xFC\x92\x83\xCC\x67"
-			  "\xA2\xB1\x70\x3A\x8F\x87\x4A\x8D"
-			  "\x17\xE2\x58\x2B\x88\x0D\x68\x62"
-			  "\xBF\x35\xD1\x6F\xC0\xF0\x18\x62"
-			  "\xB2\xC7\x2D\x58\xC7\x16\xDE\x08"
-			  "\xEB\x84\x1D\x25\xA7\x38\x94\x06"
-			  "\x93\x9D\xF8\xFE\x88\x71\xE7\x84"
-			  "\x2C\xA0\x38\xA3\x1D\x48\xCF\x29"
-			  "\x0B\xBC\xD8\x50\x99\x1A\x26\xFB"
-			  "\x8E\x75\x3D\x73\xEB\x6A\xED\x29"
-			  "\xE0\x8E\xED\xFC\xFE\x6F\xF6\xBA"
-			  "\x41\xE2\x10\x4C\x01\x8B\x69\x2B"
-			  "\x25\x3F\x4D\x70\x7B\x92\xD6\x3B"
-			  "\xAC\xF9\x77\x18\xD9\x6A\x30\xA6"
-			  "\x2E\xFA\x30\xFF\xC8\xD5\x1D\x06"
-			  "\x59\x28\x1D\x86\x43\x04\x5D\x3B"
-			  "\x99\x4C\x04\x5A\x21\x17\x8B\x76"
-			  "\x8F\x72\xCB\xA1\x9C\x29\x4C\xC3"
-			  "\x65\xA2\x58\x2A\xC5\x66\x24\xBF"
-			  "\xBA\xE6\x0C\xDD\x34\x24\x74\xC8"
-			  "\x84\x0A\x66\x2C\xBE\x8F\x32\xA9"
-			  "\xE7\xE4\xA1\xD7\xDA\xAB\x23\x1E"
-			  "\xEB\xEE\x6C\x94\x6F\x9C\x2E\xD1"
-			  "\x49\x2C\xF3\xD4\x90\xCC\x93\x4C"
-			  "\x84\x52\x6D\x68\xDE\xC6\x64\xB2"
-			  "\x11\x74\x93\x57\xB4\x7E\xC6\x00",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast6_dec_tv_template[] = {
-	{
-		.key	= "\x23\x42\xbb\x9e\xfa\x38\x54\x2c"
-			  "\x0a\xf7\x56\x47\xf2\x9f\x61\x5d",
-		.klen	= 16,
-		.input	= "\xc8\x42\xa0\x89\x72\xb4\x3d\x20"
-			  "\x83\x6c\x91\xd1\xb7\x53\x0f\x6b",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
-	}, {
-		.key	= "\x23\x42\xbb\x9e\xfa\x38\x54\x2c"
-			  "\xbe\xd0\xac\x83\x94\x0a\xc2\x98"
-			  "\xba\xc7\x7a\x77\x17\x94\x28\x63",
-		.klen	= 24,
-		.input	= "\x1b\x38\x6c\x02\x10\xdc\xad\xcb"
-			  "\xdd\x0e\x41\xaa\x08\xa7\xa7\xe8",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
-	}, {
-		.key	= "\x23\x42\xbb\x9e\xfa\x38\x54\x2c"
-			  "\xbe\xd0\xac\x83\x94\x0a\xc2\x98"
-			  "\x8d\x7c\x47\xce\x26\x49\x08\x46"
-			  "\x1c\xc1\xb5\x13\x7a\xe6\xb6\x04",
-		.klen	= 32,
-		.input	= "\x4f\x6a\x20\x38\x28\x68\x97\xb9"
-			  "\xc9\x87\x01\x36\x55\x33\x17\xfa",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
-	}, { /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\xC3\x70\x22\x32\xF5\x80\xCB\x54"
-			  "\xFC\x30\xE0\xF6\xEB\x39\x57\xA6"
-			  "\xB6\xB9\xC5\xA4\x91\x55\x14\x97"
-			  "\xC1\x20\xFF\x6C\x5C\xF0\x67\xEA"
-			  "\x2F\xED\xD8\xC9\xFB\x38\x3F\xFE"
-			  "\x93\xBE\xDC\x00\xD3\x7F\xAD\x4C"
-			  "\x5A\x08\x92\xD1\x47\x0C\xFA\x6C"
-			  "\xD0\x6A\x99\x10\x72\xF8\x47\x62"
-			  "\x81\x42\xF8\xD8\xF5\xBB\x94\x08"
-			  "\xAA\x97\xA2\x8B\x69\xB3\xD2\x7E"
-			  "\xBC\xB5\x00\x0C\xE5\x44\x4B\x58"
-			  "\xE8\x63\xDC\xB3\xC4\xE5\x23\x12"
-			  "\x5A\x72\x85\x47\x8B\xEC\x9F\x26"
-			  "\x84\xB6\xED\x10\x33\x63\x9B\x5F"
-			  "\x4D\x53\xEE\x94\x45\x8B\x60\x58"
-			  "\x86\x20\xF9\x1E\x82\x08\x3E\x58"
-			  "\x60\x1B\x34\x19\x02\xBE\x4E\x09"
-			  "\xBB\x7C\x15\xCC\x60\x27\x55\x7A"
-			  "\x12\xB8\xD8\x08\x89\x3C\xA6\xF3"
-			  "\xF1\xDD\xA7\x07\xA3\x12\x85\x28"
-			  "\xE9\x57\xAC\x80\x0C\x5C\x0F\x3A"
-			  "\x5D\xC2\x91\xC7\x90\xE4\x8C\x43"
-			  "\x92\xE4\x7C\x26\x69\x4D\x83\x68"
-			  "\x14\x96\x42\x47\xBD\xA9\xE4\x8A"
-			  "\x33\x19\xEB\x54\x8E\x0D\x4B\x6E"
-			  "\x91\x51\xB5\x36\x08\xDE\x1C\x06"
-			  "\x03\xBD\xDE\x81\x26\xF7\x99\xC2"
-			  "\xBA\xF7\x6D\x87\x0D\xE4\xA6\xCF"
-			  "\xC1\xF5\x27\x05\xB8\x02\x57\x72"
-			  "\xE6\x42\x13\x0B\xC6\x47\x05\x74"
-			  "\x24\x15\xF7\x0D\xC2\x23\x9D\xB9"
-			  "\x3C\x77\x18\x93\xBA\xB4\xFC\x8C"
-			  "\x98\x82\x67\x67\xB4\xD7\xD3\x43"
-			  "\x23\x08\x02\xB7\x9B\x99\x05\xFB"
-			  "\xD3\xB5\x00\x0A\xA9\x9D\x66\xD6"
-			  "\x2E\x49\x58\xD0\xA8\x57\x29\x7F"
-			  "\x0A\x0E\x7D\xFC\x92\x83\xCC\x67"
-			  "\xA2\xB1\x70\x3A\x8F\x87\x4A\x8D"
-			  "\x17\xE2\x58\x2B\x88\x0D\x68\x62"
-			  "\xBF\x35\xD1\x6F\xC0\xF0\x18\x62"
-			  "\xB2\xC7\x2D\x58\xC7\x16\xDE\x08"
-			  "\xEB\x84\x1D\x25\xA7\x38\x94\x06"
-			  "\x93\x9D\xF8\xFE\x88\x71\xE7\x84"
-			  "\x2C\xA0\x38\xA3\x1D\x48\xCF\x29"
-			  "\x0B\xBC\xD8\x50\x99\x1A\x26\xFB"
-			  "\x8E\x75\x3D\x73\xEB\x6A\xED\x29"
-			  "\xE0\x8E\xED\xFC\xFE\x6F\xF6\xBA"
-			  "\x41\xE2\x10\x4C\x01\x8B\x69\x2B"
-			  "\x25\x3F\x4D\x70\x7B\x92\xD6\x3B"
-			  "\xAC\xF9\x77\x18\xD9\x6A\x30\xA6"
-			  "\x2E\xFA\x30\xFF\xC8\xD5\x1D\x06"
-			  "\x59\x28\x1D\x86\x43\x04\x5D\x3B"
-			  "\x99\x4C\x04\x5A\x21\x17\x8B\x76"
-			  "\x8F\x72\xCB\xA1\x9C\x29\x4C\xC3"
-			  "\x65\xA2\x58\x2A\xC5\x66\x24\xBF"
-			  "\xBA\xE6\x0C\xDD\x34\x24\x74\xC8"
-			  "\x84\x0A\x66\x2C\xBE\x8F\x32\xA9"
-			  "\xE7\xE4\xA1\xD7\xDA\xAB\x23\x1E"
-			  "\xEB\xEE\x6C\x94\x6F\x9C\x2E\xD1"
-			  "\x49\x2C\xF3\xD4\x90\xCC\x93\x4C"
-			  "\x84\x52\x6D\x68\xDE\xC6\x64\xB2"
-			  "\x11\x74\x93\x57\xB4\x7E\xC6\x00",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast6_cbc_enc_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\xDF\x77\x68\x96\xC7\xBA\xF8\xE2"
-			  "\x0E\x24\x99\x1A\xAA\xF3\xC6\x9F"
-			  "\xA0\x73\xB3\x70\xC3\x68\x64\x70"
-			  "\xAD\x33\x02\xFB\x88\x74\xAA\x78"
-			  "\xC7\x47\x1A\x18\x61\x2D\xAC\x9F"
-			  "\x7E\x6F\xDF\x05\x13\x76\xA6\x72"
-			  "\xB7\x13\x09\x0F\x7D\x38\xDF\x25"
-			  "\x4E\xFD\x50\x45\xFA\x35\x6A\xC0"
-			  "\x57\x95\xE1\x21\x26\x10\x9A\x21"
-			  "\xA1\x8A\x51\x05\xD1\xB1\x78\x35"
-			  "\x98\xF5\xAE\xC0\xC1\x8B\x94\xFF"
-			  "\xD0\x69\x3F\x42\xC2\x01\xA7\x9B"
-			  "\x23\x16\x47\x72\x81\x13\x3A\x72"
-			  "\xEC\xD9\x40\x88\x00\x9C\xB0\xA8"
-			  "\x9C\xAC\xCE\x11\x73\x7B\x63\x3E"
-			  "\xA3\x63\x98\x7D\x35\xE4\xD9\x83"
-			  "\xE2\xD0\x52\x87\x0C\x1F\xB0\xB3"
-			  "\x41\x1A\x93\x8D\x76\x31\x9F\xF2"
-			  "\xFE\x09\xA3\x8F\x22\x6A\x3B\xB9"
-			  "\x6C\x9E\xE4\xA1\xA0\xC4\xE7\xA1"
-			  "\x21\x9C\x1A\xCA\x65\xDE\x44\x03"
-			  "\x99\xF2\xD2\x39\xE3\x3F\x0F\x37"
-			  "\x53\x50\x23\xA4\x81\x6E\xDA\xFB"
-			  "\xF8\x7B\x01\xD7\xB2\x32\x9C\xB8"
-			  "\xB1\x0E\x99\x17\xB5\x38\xF9\xD7"
-			  "\x86\x2D\x6E\x94\x5C\x99\x9D\xB3"
-			  "\xD3\x63\x4B\x2A\x7D\x44\x6A\xB2"
-			  "\xC1\x03\xE6\x5A\x37\xD8\x64\x18"
-			  "\xAA\x32\xCE\x29\xED\xC0\xA2\xCB"
-			  "\x8D\xAF\xCD\xBE\x8F\xB6\xEC\xB4"
-			  "\x89\x05\x81\x6E\x71\x4F\xC3\x28"
-			  "\x10\xC1\x62\xC4\x41\xE9\xD2\x39"
-			  "\xF3\x22\x39\x12\x2C\xC2\x95\x2D"
-			  "\xBF\x93\x58\x4B\x04\xD1\x8D\x57"
-			  "\xAE\xEB\x60\x03\x56\x35\xAD\x5A"
-			  "\xE9\xC3\xFF\x4E\x31\xE1\x37\xF8"
-			  "\x7D\xEE\x65\x8A\xB6\x88\x1A\x3E"
-			  "\x07\x09\x82\xBA\xF0\x80\x8A\xD0"
-			  "\xA0\x3F\x6A\xE9\x24\x87\x19\x65"
-			  "\x73\x3F\x12\x91\x47\x54\xBA\x39"
-			  "\x30\x5B\x1E\xE5\xC2\xF9\x3F\xEF"
-			  "\xD6\x75\xF9\xB8\x7C\x8B\x05\x76"
-			  "\xEE\xB7\x08\x25\x4B\xB6\x7B\x47"
-			  "\x72\xC0\x4C\xD4\xDA\xE0\x75\xF1"
-			  "\x7C\xE8\x94\x9E\x16\x6E\xB8\x12"
-			  "\xA1\xC1\x6E\x3B\x1C\x59\x41\x2D"
-			  "\x23\xFA\x7D\x77\xB8\x46\x75\xFE"
-			  "\x4F\x10\xD3\x09\x60\xA1\x36\x96"
-			  "\x5B\xC2\xDC\x6E\x84\x7D\x9B\x14"
-			  "\x80\x21\x83\x58\x3C\x76\xFD\x28"
-			  "\x1D\xF9\x93\x13\xD7\x0E\x62\x14"
-			  "\x5A\xC5\x4E\x08\xA5\x56\xA4\x3C"
-			  "\x68\x93\x44\x70\xDF\xCF\x4A\x51"
-			  "\x0B\x81\x29\x41\xE5\x62\x4D\x36"
-			  "\xB3\xEA\x94\xA6\xB9\xDD\x3F\x09"
-			  "\x62\x34\xA0\x6A\x7E\x7D\xF5\xF6"
-			  "\x01\x91\xB4\x27\xDA\x59\xD6\x17"
-			  "\x56\x4D\x82\x62\x37\xA3\x48\x01"
-			  "\x99\x91\x77\xB2\x08\x6B\x2C\x37"
-			  "\xC5\x5C\xAD\xB6\x07\xB6\x84\xF3"
-			  "\x4D\x59\x7D\xC5\x28\x69\xFA\x92"
-			  "\x22\x46\x89\x2D\x0F\x2B\x08\x24",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast6_cbc_dec_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\xDF\x77\x68\x96\xC7\xBA\xF8\xE2"
-			  "\x0E\x24\x99\x1A\xAA\xF3\xC6\x9F"
-			  "\xA0\x73\xB3\x70\xC3\x68\x64\x70"
-			  "\xAD\x33\x02\xFB\x88\x74\xAA\x78"
-			  "\xC7\x47\x1A\x18\x61\x2D\xAC\x9F"
-			  "\x7E\x6F\xDF\x05\x13\x76\xA6\x72"
-			  "\xB7\x13\x09\x0F\x7D\x38\xDF\x25"
-			  "\x4E\xFD\x50\x45\xFA\x35\x6A\xC0"
-			  "\x57\x95\xE1\x21\x26\x10\x9A\x21"
-			  "\xA1\x8A\x51\x05\xD1\xB1\x78\x35"
-			  "\x98\xF5\xAE\xC0\xC1\x8B\x94\xFF"
-			  "\xD0\x69\x3F\x42\xC2\x01\xA7\x9B"
-			  "\x23\x16\x47\x72\x81\x13\x3A\x72"
-			  "\xEC\xD9\x40\x88\x00\x9C\xB0\xA8"
-			  "\x9C\xAC\xCE\x11\x73\x7B\x63\x3E"
-			  "\xA3\x63\x98\x7D\x35\xE4\xD9\x83"
-			  "\xE2\xD0\x52\x87\x0C\x1F\xB0\xB3"
-			  "\x41\x1A\x93\x8D\x76\x31\x9F\xF2"
-			  "\xFE\x09\xA3\x8F\x22\x6A\x3B\xB9"
-			  "\x6C\x9E\xE4\xA1\xA0\xC4\xE7\xA1"
-			  "\x21\x9C\x1A\xCA\x65\xDE\x44\x03"
-			  "\x99\xF2\xD2\x39\xE3\x3F\x0F\x37"
-			  "\x53\x50\x23\xA4\x81\x6E\xDA\xFB"
-			  "\xF8\x7B\x01\xD7\xB2\x32\x9C\xB8"
-			  "\xB1\x0E\x99\x17\xB5\x38\xF9\xD7"
-			  "\x86\x2D\x6E\x94\x5C\x99\x9D\xB3"
-			  "\xD3\x63\x4B\x2A\x7D\x44\x6A\xB2"
-			  "\xC1\x03\xE6\x5A\x37\xD8\x64\x18"
-			  "\xAA\x32\xCE\x29\xED\xC0\xA2\xCB"
-			  "\x8D\xAF\xCD\xBE\x8F\xB6\xEC\xB4"
-			  "\x89\x05\x81\x6E\x71\x4F\xC3\x28"
-			  "\x10\xC1\x62\xC4\x41\xE9\xD2\x39"
-			  "\xF3\x22\x39\x12\x2C\xC2\x95\x2D"
-			  "\xBF\x93\x58\x4B\x04\xD1\x8D\x57"
-			  "\xAE\xEB\x60\x03\x56\x35\xAD\x5A"
-			  "\xE9\xC3\xFF\x4E\x31\xE1\x37\xF8"
-			  "\x7D\xEE\x65\x8A\xB6\x88\x1A\x3E"
-			  "\x07\x09\x82\xBA\xF0\x80\x8A\xD0"
-			  "\xA0\x3F\x6A\xE9\x24\x87\x19\x65"
-			  "\x73\x3F\x12\x91\x47\x54\xBA\x39"
-			  "\x30\x5B\x1E\xE5\xC2\xF9\x3F\xEF"
-			  "\xD6\x75\xF9\xB8\x7C\x8B\x05\x76"
-			  "\xEE\xB7\x08\x25\x4B\xB6\x7B\x47"
-			  "\x72\xC0\x4C\xD4\xDA\xE0\x75\xF1"
-			  "\x7C\xE8\x94\x9E\x16\x6E\xB8\x12"
-			  "\xA1\xC1\x6E\x3B\x1C\x59\x41\x2D"
-			  "\x23\xFA\x7D\x77\xB8\x46\x75\xFE"
-			  "\x4F\x10\xD3\x09\x60\xA1\x36\x96"
-			  "\x5B\xC2\xDC\x6E\x84\x7D\x9B\x14"
-			  "\x80\x21\x83\x58\x3C\x76\xFD\x28"
-			  "\x1D\xF9\x93\x13\xD7\x0E\x62\x14"
-			  "\x5A\xC5\x4E\x08\xA5\x56\xA4\x3C"
-			  "\x68\x93\x44\x70\xDF\xCF\x4A\x51"
-			  "\x0B\x81\x29\x41\xE5\x62\x4D\x36"
-			  "\xB3\xEA\x94\xA6\xB9\xDD\x3F\x09"
-			  "\x62\x34\xA0\x6A\x7E\x7D\xF5\xF6"
-			  "\x01\x91\xB4\x27\xDA\x59\xD6\x17"
-			  "\x56\x4D\x82\x62\x37\xA3\x48\x01"
-			  "\x99\x91\x77\xB2\x08\x6B\x2C\x37"
-			  "\xC5\x5C\xAD\xB6\x07\xB6\x84\xF3"
-			  "\x4D\x59\x7D\xC5\x28\x69\xFA\x92"
-			  "\x22\x46\x89\x2D\x0F\x2B\x08\x24",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast6_ctr_enc_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A",
-		.ilen	= 17,
-		.result	= "\x26\x0A\xF1\xE2\x3F\x8A\xEF\xA3"
-			  "\x53\x9A\x5E\x1B\x2A\x1A\xC6\x0A"
-			  "\x57",
-		.rlen	= 17,
-	}, { /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\x26\x0A\xF1\xE2\x3F\x8A\xEF\xA3"
-			  "\x53\x9A\x5E\x1B\x2A\x1A\xC6\x0A"
-			  "\x57\xA3\xEF\x47\x2A\xE8\x88\xA7"
-			  "\x3C\xD0\xEC\xB9\x94\x50\x7D\x56"
-			  "\xBC\xE1\xC1\xF5\xE1\xEE\x12\xF8"
-			  "\x4F\x03\x82\x3A\x93\x6B\x4C\xD3"
-			  "\xE3\xF3\xFA\xC2\x23\x55\x98\x20"
-			  "\x49\x76\x9B\x6B\xC1\x23\xBF\xE5"
-			  "\xD4\xC4\x2F\x61\xE1\x67\x2A\x30"
-			  "\x6F\x29\xCA\x54\xF8\x1B\xA6\x7D"
-			  "\x66\x45\xEE\xC8\x19\xBE\x50\xF0"
-			  "\x5F\x65\xF8\x1E\x4D\x07\x87\xD9"
-			  "\xD3\xD9\x1B\x09\x89\xFD\x42\xC5"
-			  "\xDB\xEB\x86\xF1\x67\x04\x0F\x5C"
-			  "\x81\xDF\x82\x12\xC7\x4C\x1B\x07"
-			  "\xDE\xE6\xFA\x29\x86\xD1\xB0\xBA"
-			  "\x3D\x6A\x69\x76\xEC\x0F\xB4\xE6"
-			  "\xCD\xA7\xF8\xA8\xB8\xE0\x33\xF5"
-			  "\x49\x61\x22\x52\x64\x8C\x46\x41"
-			  "\x1F\x48\x5F\x4F\xA2\x89\x36\x17"
-			  "\x20\xF8\x2F\x8F\x4B\xFA\xF2\xC0"
-			  "\x1E\x18\xA2\xF8\xB7\x6D\x98\xE3"
-			  "\x00\x14\x15\x59\xC1\x30\x64\xAF"
-			  "\xA8\x01\x38\xAB\xD4\x8B\xEC\x7C"
-			  "\x44\x9A\xC6\x2C\x2E\x2B\x2B\xF4"
-			  "\x02\x37\xC4\x69\xEF\x36\xC1\xF3"
-			  "\xA0\xFB\xFE\x29\xAD\x39\xCF\xD0"
-			  "\x51\x73\xA3\x22\x42\x41\xAB\xD2"
-			  "\x0F\x50\x14\xB9\x54\xD3\xD4\xFA"
-			  "\xBF\xC9\xBB\xCE\xC4\x1D\x2D\xAF"
-			  "\xC9\x3F\x07\x87\x42\x4B\x3A\x54"
-			  "\x34\x8E\x37\xA3\x03\x6F\x65\x66"
-			  "\xDB\x44\xC3\xE8\xD7\xDD\x7D\xDD"
-			  "\x61\xB4\x2B\x80\xA3\x98\x13\xF5"
-			  "\x5A\xD3\x34\x58\xC3\x6E\xF6\xB8"
-			  "\x0A\xC6\x50\x01\x8E\xD5\x6C\x7D"
-			  "\xFE\x16\xB6\xCF\xFC\x51\x40\xAE"
-			  "\xB3\x15\xAC\x90\x6F\x0B\x28\x3A"
-			  "\x60\x40\x38\x90\x20\x46\xC7\xB3"
-			  "\x0B\x12\x6D\x3B\x15\x14\xF9\xF4"
-			  "\x11\x41\x76\x6B\xB3\x60\x82\x3C"
-			  "\x84\xFB\x08\x2E\x92\x25\xCB\x79"
-			  "\x6F\x58\xC5\x94\x00\x00\x47\xB6"
-			  "\x9E\xDC\x0F\x29\x70\x46\x20\x76"
-			  "\x65\x75\x66\x5C\x00\x96\xB3\xE1"
-			  "\x0B\xA7\x11\x8B\x2E\x61\x4E\x45"
-			  "\x73\xFC\x91\xAB\x79\x41\x23\x14"
-			  "\x13\xB6\x72\x6C\x46\xB3\x03\x11"
-			  "\xE4\xF1\xEE\xC9\x7A\xCF\x96\x32"
-			  "\xB6\xF0\x8B\x97\xB4\xCF\x82\xB7"
-			  "\x15\x48\x44\x99\x09\xF6\xE0\xD7"
-			  "\xBC\xF1\x5B\x91\x4F\x30\x22\xA2"
-			  "\x45\xC4\x68\x55\xC2\xBE\xA7\xD2"
-			  "\x12\x53\x35\x9C\xF9\xE7\x35\x5D"
-			  "\x81\xE4\x86\x42\xC3\x58\xFB\xF0"
-			  "\x38\x9B\x8E\x5A\xEF\x83\x33\x0F"
-			  "\x00\x4E\x3F\x9F\xF5\x84\x62\xC4"
-			  "\x19\x35\x88\x22\x45\x59\x0E\x8F"
-			  "\xEC\x27\xDD\x4A\xA4\x1F\xBC\x41"
-			  "\x9B\x66\x8D\x32\xBA\x81\x34\x87"
-			  "\x0E\x74\x33\x30\x62\xB9\x89\xDF"
-			  "\xF9\xC5\xDD\x27\xB3\x39\xCB\xCB",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast6_ctr_dec_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x26\x0A\xF1\xE2\x3F\x8A\xEF\xA3"
-			  "\x53\x9A\x5E\x1B\x2A\x1A\xC6\x0A"
-			  "\x57",
-		.ilen	= 17,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A",
-		.rlen	= 17,
-	}, { /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x26\x0A\xF1\xE2\x3F\x8A\xEF\xA3"
-			  "\x53\x9A\x5E\x1B\x2A\x1A\xC6\x0A"
-			  "\x57\xA3\xEF\x47\x2A\xE8\x88\xA7"
-			  "\x3C\xD0\xEC\xB9\x94\x50\x7D\x56"
-			  "\xBC\xE1\xC1\xF5\xE1\xEE\x12\xF8"
-			  "\x4F\x03\x82\x3A\x93\x6B\x4C\xD3"
-			  "\xE3\xF3\xFA\xC2\x23\x55\x98\x20"
-			  "\x49\x76\x9B\x6B\xC1\x23\xBF\xE5"
-			  "\xD4\xC4\x2F\x61\xE1\x67\x2A\x30"
-			  "\x6F\x29\xCA\x54\xF8\x1B\xA6\x7D"
-			  "\x66\x45\xEE\xC8\x19\xBE\x50\xF0"
-			  "\x5F\x65\xF8\x1E\x4D\x07\x87\xD9"
-			  "\xD3\xD9\x1B\x09\x89\xFD\x42\xC5"
-			  "\xDB\xEB\x86\xF1\x67\x04\x0F\x5C"
-			  "\x81\xDF\x82\x12\xC7\x4C\x1B\x07"
-			  "\xDE\xE6\xFA\x29\x86\xD1\xB0\xBA"
-			  "\x3D\x6A\x69\x76\xEC\x0F\xB4\xE6"
-			  "\xCD\xA7\xF8\xA8\xB8\xE0\x33\xF5"
-			  "\x49\x61\x22\x52\x64\x8C\x46\x41"
-			  "\x1F\x48\x5F\x4F\xA2\x89\x36\x17"
-			  "\x20\xF8\x2F\x8F\x4B\xFA\xF2\xC0"
-			  "\x1E\x18\xA2\xF8\xB7\x6D\x98\xE3"
-			  "\x00\x14\x15\x59\xC1\x30\x64\xAF"
-			  "\xA8\x01\x38\xAB\xD4\x8B\xEC\x7C"
-			  "\x44\x9A\xC6\x2C\x2E\x2B\x2B\xF4"
-			  "\x02\x37\xC4\x69\xEF\x36\xC1\xF3"
-			  "\xA0\xFB\xFE\x29\xAD\x39\xCF\xD0"
-			  "\x51\x73\xA3\x22\x42\x41\xAB\xD2"
-			  "\x0F\x50\x14\xB9\x54\xD3\xD4\xFA"
-			  "\xBF\xC9\xBB\xCE\xC4\x1D\x2D\xAF"
-			  "\xC9\x3F\x07\x87\x42\x4B\x3A\x54"
-			  "\x34\x8E\x37\xA3\x03\x6F\x65\x66"
-			  "\xDB\x44\xC3\xE8\xD7\xDD\x7D\xDD"
-			  "\x61\xB4\x2B\x80\xA3\x98\x13\xF5"
-			  "\x5A\xD3\x34\x58\xC3\x6E\xF6\xB8"
-			  "\x0A\xC6\x50\x01\x8E\xD5\x6C\x7D"
-			  "\xFE\x16\xB6\xCF\xFC\x51\x40\xAE"
-			  "\xB3\x15\xAC\x90\x6F\x0B\x28\x3A"
-			  "\x60\x40\x38\x90\x20\x46\xC7\xB3"
-			  "\x0B\x12\x6D\x3B\x15\x14\xF9\xF4"
-			  "\x11\x41\x76\x6B\xB3\x60\x82\x3C"
-			  "\x84\xFB\x08\x2E\x92\x25\xCB\x79"
-			  "\x6F\x58\xC5\x94\x00\x00\x47\xB6"
-			  "\x9E\xDC\x0F\x29\x70\x46\x20\x76"
-			  "\x65\x75\x66\x5C\x00\x96\xB3\xE1"
-			  "\x0B\xA7\x11\x8B\x2E\x61\x4E\x45"
-			  "\x73\xFC\x91\xAB\x79\x41\x23\x14"
-			  "\x13\xB6\x72\x6C\x46\xB3\x03\x11"
-			  "\xE4\xF1\xEE\xC9\x7A\xCF\x96\x32"
-			  "\xB6\xF0\x8B\x97\xB4\xCF\x82\xB7"
-			  "\x15\x48\x44\x99\x09\xF6\xE0\xD7"
-			  "\xBC\xF1\x5B\x91\x4F\x30\x22\xA2"
-			  "\x45\xC4\x68\x55\xC2\xBE\xA7\xD2"
-			  "\x12\x53\x35\x9C\xF9\xE7\x35\x5D"
-			  "\x81\xE4\x86\x42\xC3\x58\xFB\xF0"
-			  "\x38\x9B\x8E\x5A\xEF\x83\x33\x0F"
-			  "\x00\x4E\x3F\x9F\xF5\x84\x62\xC4"
-			  "\x19\x35\x88\x22\x45\x59\x0E\x8F"
-			  "\xEC\x27\xDD\x4A\xA4\x1F\xBC\x41"
-			  "\x9B\x66\x8D\x32\xBA\x81\x34\x87"
-			  "\x0E\x74\x33\x30\x62\xB9\x89\xDF"
-			  "\xF9\xC5\xDD\x27\xB3\x39\xCB\xCB",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast6_lrw_enc_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
-			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
-			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
-			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
-			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
-			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
-			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
-			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
-			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
-			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
-			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
-			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
-			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
-			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
-			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
-			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
-			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
-			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
-			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
-			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
-			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
-			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
-			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
-			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
-			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
-			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
-			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
-			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
-			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
-			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
-			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
-			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
-			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
-			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
-			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
-			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
-			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
-			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
-			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
-			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
-			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
-			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
-			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
-			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
-			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
-			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
-			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
-			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
-			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
-			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
-			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
-			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
-			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
-			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
-			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
-			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
-			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
-			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
-			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
-			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
-			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
-			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
-			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
-			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
-		.ilen	= 512,
-		.result	= "\x55\x25\x09\x8B\xB5\xD5\xF8\xBF"
-			  "\x37\x4A\xFE\x3C\x47\xD8\xE6\xEB"
-			  "\xCA\xA4\x9B\xB0\xAB\x6D\x64\xCA"
-			  "\x58\xB6\x73\xF0\xD7\x52\x34\xEF"
-			  "\xFB\x3E\x96\x81\xB7\x71\x34\xA4"
-			  "\x55\x20\xBE\x39\x5A\x2B\xF9\xD1"
-			  "\x65\x0B\xDA\xD3\x7E\xB3\xA6\xF7"
-			  "\x2E\x0B\x5A\x52\xDB\x39\x8C\x9B"
-			  "\x61\x17\x5F\xAF\xB6\x5A\xC8\x08"
-			  "\xA7\xB7\x2A\x11\x7C\x97\x38\x9D"
-			  "\x59\x0E\x66\x59\x5E\xD8\x8B\xCE"
-			  "\x70\xE0\xC3\x42\xB0\x8C\x0F\xBA"
-			  "\xB2\x0D\x81\xB6\xBE\x61\x1C\x2D"
-			  "\x7E\xEA\x91\x25\xAC\xEC\xF8\x28"
-			  "\x80\x1D\xF0\x30\xBA\x62\x77\x7D"
-			  "\xDB\x15\x69\xDF\xFA\x2A\x81\x64"
-			  "\x95\x5B\xA4\x7F\x3E\x4F\xE3\x30"
-			  "\xB0\x5C\xC2\x05\xF8\xF0\x29\xE7"
-			  "\x0A\xA0\x66\xB2\x5D\x0F\x39\x2B"
-			  "\xB4\xB3\x00\xA9\xD0\xAB\x63\x61"
-			  "\x5E\xDB\xFC\x11\x74\x25\x96\x65"
-			  "\xE8\xE2\x34\x57\x77\x15\x5E\x70"
-			  "\xFF\x10\x90\xC3\x64\xF0\x11\x0A"
-			  "\x63\x3A\xD3\x55\x92\x15\x4B\x0C"
-			  "\xC7\x08\x89\x17\x3B\x99\xAD\x63"
-			  "\xE7\x06\xDF\x52\xBC\x15\x64\x45"
-			  "\x9D\x7A\xFB\x69\xBC\x2D\x6E\xA9"
-			  "\x35\xD9\xD8\xF5\x0C\xC4\xA2\x23"
-			  "\x9C\x18\x8B\xA8\x8C\xFE\xF8\x0E"
-			  "\xBD\xAB\x60\x1A\x51\x17\x54\x27"
-			  "\xB6\xE8\xBE\x0F\xA9\xA5\x82\x19"
-			  "\x2F\x6F\x20\xA7\x47\xED\x74\x6C"
-			  "\x4E\xC1\xF8\x8C\x14\xF3\xBB\x1F"
-			  "\xED\x4D\x8F\x7C\x37\xEF\x19\xA1"
-			  "\x07\x16\xDE\x76\xCC\x5E\x94\x02"
-			  "\xFB\xBF\xE4\x81\x50\xCE\xFC\x0F"
-			  "\x9E\xCF\x3D\xF6\x67\x00\xBF\xA7"
-			  "\x6E\x21\x58\x36\x06\xDE\xB3\xD4"
-			  "\xA2\xFA\xD8\x4E\xE0\xB9\x7F\x23"
-			  "\x51\x21\x2B\x32\x68\xAA\xF8\xA8"
-			  "\x93\x08\xB5\x6D\xE6\x43\x2C\xB7"
-			  "\x31\xB2\x0F\xD0\xA2\x51\xC0\x25"
-			  "\x30\xC7\x10\x3F\x97\x27\x01\x8E"
-			  "\xFA\xD8\x4F\x78\xD8\x2E\x1D\xEB"
-			  "\xA1\x37\x52\x0F\x7B\x5E\x87\xA8"
-			  "\x22\xE2\xE6\x92\xA7\x5F\x11\x32"
-			  "\xCC\x93\x34\xFC\xD1\x7E\xAE\x54"
-			  "\xBC\x6A\x1B\x91\xD1\x2E\x21\xEC"
-			  "\x5D\xF1\xC4\xF1\x55\x20\xBF\xE5"
-			  "\x96\x3D\x69\x91\x20\x4E\xF2\x61"
-			  "\xDA\x77\xFE\xEE\xC3\x74\x57\x2A"
-			  "\x78\x39\xB0\xE0\xCF\x12\x56\xD6"
-			  "\x05\xDC\xF9\x19\x66\x44\x1D\xF9"
-			  "\x82\x37\xD4\xC2\x60\xB6\x31\xDF"
-			  "\x0C\xAF\xBC\x8B\x55\x9A\xC8\x2D"
-			  "\xAB\xA7\x88\x7B\x41\xE8\x29\xC9"
-			  "\x9B\x8D\xA7\x00\x86\x25\xB6\x14"
-			  "\xF5\x13\x73\xD7\x4B\x6B\x83\xF3"
-			  "\xAF\x96\x00\xE4\xB7\x3C\x65\xA6"
-			  "\x15\xB7\x94\x7D\x4E\x70\x4C\x75"
-			  "\xF3\xB4\x02\xA9\x17\x1C\x7A\x0A"
-			  "\xC0\xD5\x33\x11\x56\xDE\xDC\xF5"
-			  "\x8D\xD9\xCD\x3B\x22\x67\x18\xC7"
-			  "\xC4\xF5\x99\x61\xBC\xBB\x5B\x46",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast6_lrw_dec_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x55\x25\x09\x8B\xB5\xD5\xF8\xBF"
-			  "\x37\x4A\xFE\x3C\x47\xD8\xE6\xEB"
-			  "\xCA\xA4\x9B\xB0\xAB\x6D\x64\xCA"
-			  "\x58\xB6\x73\xF0\xD7\x52\x34\xEF"
-			  "\xFB\x3E\x96\x81\xB7\x71\x34\xA4"
-			  "\x55\x20\xBE\x39\x5A\x2B\xF9\xD1"
-			  "\x65\x0B\xDA\xD3\x7E\xB3\xA6\xF7"
-			  "\x2E\x0B\x5A\x52\xDB\x39\x8C\x9B"
-			  "\x61\x17\x5F\xAF\xB6\x5A\xC8\x08"
-			  "\xA7\xB7\x2A\x11\x7C\x97\x38\x9D"
-			  "\x59\x0E\x66\x59\x5E\xD8\x8B\xCE"
-			  "\x70\xE0\xC3\x42\xB0\x8C\x0F\xBA"
-			  "\xB2\x0D\x81\xB6\xBE\x61\x1C\x2D"
-			  "\x7E\xEA\x91\x25\xAC\xEC\xF8\x28"
-			  "\x80\x1D\xF0\x30\xBA\x62\x77\x7D"
-			  "\xDB\x15\x69\xDF\xFA\x2A\x81\x64"
-			  "\x95\x5B\xA4\x7F\x3E\x4F\xE3\x30"
-			  "\xB0\x5C\xC2\x05\xF8\xF0\x29\xE7"
-			  "\x0A\xA0\x66\xB2\x5D\x0F\x39\x2B"
-			  "\xB4\xB3\x00\xA9\xD0\xAB\x63\x61"
-			  "\x5E\xDB\xFC\x11\x74\x25\x96\x65"
-			  "\xE8\xE2\x34\x57\x77\x15\x5E\x70"
-			  "\xFF\x10\x90\xC3\x64\xF0\x11\x0A"
-			  "\x63\x3A\xD3\x55\x92\x15\x4B\x0C"
-			  "\xC7\x08\x89\x17\x3B\x99\xAD\x63"
-			  "\xE7\x06\xDF\x52\xBC\x15\x64\x45"
-			  "\x9D\x7A\xFB\x69\xBC\x2D\x6E\xA9"
-			  "\x35\xD9\xD8\xF5\x0C\xC4\xA2\x23"
-			  "\x9C\x18\x8B\xA8\x8C\xFE\xF8\x0E"
-			  "\xBD\xAB\x60\x1A\x51\x17\x54\x27"
-			  "\xB6\xE8\xBE\x0F\xA9\xA5\x82\x19"
-			  "\x2F\x6F\x20\xA7\x47\xED\x74\x6C"
-			  "\x4E\xC1\xF8\x8C\x14\xF3\xBB\x1F"
-			  "\xED\x4D\x8F\x7C\x37\xEF\x19\xA1"
-			  "\x07\x16\xDE\x76\xCC\x5E\x94\x02"
-			  "\xFB\xBF\xE4\x81\x50\xCE\xFC\x0F"
-			  "\x9E\xCF\x3D\xF6\x67\x00\xBF\xA7"
-			  "\x6E\x21\x58\x36\x06\xDE\xB3\xD4"
-			  "\xA2\xFA\xD8\x4E\xE0\xB9\x7F\x23"
-			  "\x51\x21\x2B\x32\x68\xAA\xF8\xA8"
-			  "\x93\x08\xB5\x6D\xE6\x43\x2C\xB7"
-			  "\x31\xB2\x0F\xD0\xA2\x51\xC0\x25"
-			  "\x30\xC7\x10\x3F\x97\x27\x01\x8E"
-			  "\xFA\xD8\x4F\x78\xD8\x2E\x1D\xEB"
-			  "\xA1\x37\x52\x0F\x7B\x5E\x87\xA8"
-			  "\x22\xE2\xE6\x92\xA7\x5F\x11\x32"
-			  "\xCC\x93\x34\xFC\xD1\x7E\xAE\x54"
-			  "\xBC\x6A\x1B\x91\xD1\x2E\x21\xEC"
-			  "\x5D\xF1\xC4\xF1\x55\x20\xBF\xE5"
-			  "\x96\x3D\x69\x91\x20\x4E\xF2\x61"
-			  "\xDA\x77\xFE\xEE\xC3\x74\x57\x2A"
-			  "\x78\x39\xB0\xE0\xCF\x12\x56\xD6"
-			  "\x05\xDC\xF9\x19\x66\x44\x1D\xF9"
-			  "\x82\x37\xD4\xC2\x60\xB6\x31\xDF"
-			  "\x0C\xAF\xBC\x8B\x55\x9A\xC8\x2D"
-			  "\xAB\xA7\x88\x7B\x41\xE8\x29\xC9"
-			  "\x9B\x8D\xA7\x00\x86\x25\xB6\x14"
-			  "\xF5\x13\x73\xD7\x4B\x6B\x83\xF3"
-			  "\xAF\x96\x00\xE4\xB7\x3C\x65\xA6"
-			  "\x15\xB7\x94\x7D\x4E\x70\x4C\x75"
-			  "\xF3\xB4\x02\xA9\x17\x1C\x7A\x0A"
-			  "\xC0\xD5\x33\x11\x56\xDE\xDC\xF5"
-			  "\x8D\xD9\xCD\x3B\x22\x67\x18\xC7"
-			  "\xC4\xF5\x99\x61\xBC\xBB\x5B\x46",
-		.ilen	= 512,
-		.result	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
-			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
-			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
-			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
-			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
-			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
-			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
-			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
-			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
-			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
-			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
-			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
-			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
-			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
-			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
-			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
-			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
-			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
-			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
-			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
-			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
-			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
-			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
-			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
-			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
-			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
-			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
-			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
-			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
-			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
-			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
-			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
-			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
-			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
-			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
-			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
-			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
-			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
-			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
-			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
-			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
-			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
-			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
-			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
-			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
-			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
-			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
-			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
-			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
-			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
-			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
-			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
-			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
-			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
-			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
-			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
-			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
-			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
-			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
-			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
-			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
-			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
-			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
-			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast6_xts_enc_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\xDE\x6F\x22\xA5\xE8\x39\xE8\x78"
-			  "\x88\x5A\x4F\x8D\x82\x76\x52\x6D"
-			  "\xB2\x41\x16\xF4\x2B\xA6\xEB\xF6"
-			  "\xE2\xC5\x62\x8D\x61\xA1\x01\xED"
-			  "\xD9\x38\x01\xC1\x43\x63\x4E\x88"
-			  "\xC9\x4B\x5A\x88\x80\xB7\x5C\x71"
-			  "\x47\xEE\x11\xD8\xB7\x2D\x5D\x13"
-			  "\x1A\xB1\x68\x5B\x61\xA7\xA9\x81"
-			  "\x8B\x83\xA1\x6A\xAA\x36\xD6\xB6"
-			  "\x60\x54\x09\x32\xFE\x6A\x76\x2E"
-			  "\x28\xFF\xD5\xD6\xDD\x1D\x45\x7D"
-			  "\xF0\x8B\xF3\x32\x4E\x6C\x12\xCB"
-			  "\xB8\x25\x70\xF8\x40\xBC\x90\x1B"
-			  "\x11\xC3\x59\xAF\xF0\x2F\x92\xDD"
-			  "\xD3\x3B\xCF\x60\xA1\x78\x94\x57"
-			  "\xAF\x76\xC1\x67\xA6\x3C\xCD\x98"
-			  "\xB1\xF7\x27\xB9\xA3\xBD\x10\xEA"
-			  "\xCD\x8B\xC2\xF2\x14\xF2\xB2\x67"
-			  "\x05\xDD\x1D\x58\x6E\x2F\x95\x08"
-			  "\x3A\xF8\x78\x76\x82\x56\xA7\xEC"
-			  "\x51\x4B\x85\x77\xC2\x4C\x4A\x34"
-			  "\x71\x38\x17\x91\x44\xE8\xFC\x65"
-			  "\x99\x0D\x52\x91\xEE\xF8\xEF\x27"
-			  "\x2A\x9E\x6E\x78\xC4\x26\x87\xF4"
-			  "\x8A\xF0\x2D\x04\xE8\x14\x92\x5D"
-			  "\x59\x22\x9B\x29\x5C\x18\xF0\xC3"
-			  "\x47\xF3\x76\xD8\xE4\xF3\x1B\xD1"
-			  "\x70\xA3\x0D\xB5\x70\x02\x1D\xA3"
-			  "\x91\x3B\x49\x73\x18\xAB\xD4\xC9"
-			  "\xC3\x1E\xEF\x1F\xFE\xD5\x59\x8A"
-			  "\xD7\xF6\xC9\x71\x67\x79\xD7\x0E"
-			  "\xBE\x1F\x8E\xEC\x55\x7E\x4F\x24"
-			  "\xE6\x87\xEA\xFE\x96\x25\x67\x8E"
-			  "\x93\x03\xFA\xFF\xCE\xAF\xB2\x3C"
-			  "\x6F\xEB\x57\xFB\xD3\x28\x87\xA9"
-			  "\xCE\xC2\xF5\x9C\xC6\x67\xB5\x97"
-			  "\x49\xF7\x04\xCB\xEF\x84\x98\x33"
-			  "\xAF\x38\xD3\x04\x1C\x24\x71\x38"
-			  "\xC7\x71\xDD\x43\x0D\x12\x4A\x18"
-			  "\xBA\xC4\xAF\xBA\xB2\x5B\xEB\x95"
-			  "\x02\x43\x5D\xCE\x19\xCC\xCD\x66"
-			  "\x91\x0B\x8C\x7F\x51\xC4\xBF\x3C"
-			  "\x8B\xF1\xCC\xAA\x29\xD7\x87\xCB"
-			  "\x3E\xC5\xF3\xC9\x75\xE8\xA3\x5B"
-			  "\x30\x45\xA9\xB7\xAF\x80\x64\x6F"
-			  "\x75\x4A\xA7\xC0\x6D\x19\x6B\xDE"
-			  "\x17\xDE\x6D\xEA\x87\x9F\x95\xAE"
-			  "\xF5\x3C\xEE\x54\xB8\x27\x84\xF8"
-			  "\x97\xA3\xE1\x6F\x38\x24\x34\x88"
-			  "\xCE\xBD\x32\x52\xE0\x00\x6C\x94"
-			  "\xC9\xD7\x5D\x37\x81\x33\x2E\x7F"
-			  "\x4F\x7E\x2E\x0D\x94\xBD\xEA\x59"
-			  "\x34\x39\xA8\x35\x12\xB7\xBC\xAC"
-			  "\xEA\x52\x9C\x78\x02\x6D\x92\x36"
-			  "\xFB\x59\x2B\xA4\xEA\x7B\x1B\x83"
-			  "\xE1\x4D\x5E\x2A\x7E\x92\xB1\x64"
-			  "\xDE\xE0\x27\x4B\x0A\x6F\x4C\xE3"
-			  "\xB0\xEB\x31\xE4\x69\x95\xAB\x35"
-			  "\x8B\x2C\xF5\x6B\x7F\xF1\xA2\x82"
-			  "\xF8\xD9\x47\x82\xA9\x82\x03\x91"
-			  "\x69\x1F\xBE\x4C\xE7\xC7\x34\x2F"
-			  "\x45\x72\x80\x17\x81\xBD\x9D\x62"
-			  "\xA1\xAC\xE8\xCF\xC6\x74\xCF\xDC"
-			  "\x22\x60\x4E\xE8\xA4\x5D\x85\xB9",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast6_xts_dec_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xDE\x6F\x22\xA5\xE8\x39\xE8\x78"
-			  "\x88\x5A\x4F\x8D\x82\x76\x52\x6D"
-			  "\xB2\x41\x16\xF4\x2B\xA6\xEB\xF6"
-			  "\xE2\xC5\x62\x8D\x61\xA1\x01\xED"
-			  "\xD9\x38\x01\xC1\x43\x63\x4E\x88"
-			  "\xC9\x4B\x5A\x88\x80\xB7\x5C\x71"
-			  "\x47\xEE\x11\xD8\xB7\x2D\x5D\x13"
-			  "\x1A\xB1\x68\x5B\x61\xA7\xA9\x81"
-			  "\x8B\x83\xA1\x6A\xAA\x36\xD6\xB6"
-			  "\x60\x54\x09\x32\xFE\x6A\x76\x2E"
-			  "\x28\xFF\xD5\xD6\xDD\x1D\x45\x7D"
-			  "\xF0\x8B\xF3\x32\x4E\x6C\x12\xCB"
-			  "\xB8\x25\x70\xF8\x40\xBC\x90\x1B"
-			  "\x11\xC3\x59\xAF\xF0\x2F\x92\xDD"
-			  "\xD3\x3B\xCF\x60\xA1\x78\x94\x57"
-			  "\xAF\x76\xC1\x67\xA6\x3C\xCD\x98"
-			  "\xB1\xF7\x27\xB9\xA3\xBD\x10\xEA"
-			  "\xCD\x8B\xC2\xF2\x14\xF2\xB2\x67"
-			  "\x05\xDD\x1D\x58\x6E\x2F\x95\x08"
-			  "\x3A\xF8\x78\x76\x82\x56\xA7\xEC"
-			  "\x51\x4B\x85\x77\xC2\x4C\x4A\x34"
-			  "\x71\x38\x17\x91\x44\xE8\xFC\x65"
-			  "\x99\x0D\x52\x91\xEE\xF8\xEF\x27"
-			  "\x2A\x9E\x6E\x78\xC4\x26\x87\xF4"
-			  "\x8A\xF0\x2D\x04\xE8\x14\x92\x5D"
-			  "\x59\x22\x9B\x29\x5C\x18\xF0\xC3"
-			  "\x47\xF3\x76\xD8\xE4\xF3\x1B\xD1"
-			  "\x70\xA3\x0D\xB5\x70\x02\x1D\xA3"
-			  "\x91\x3B\x49\x73\x18\xAB\xD4\xC9"
-			  "\xC3\x1E\xEF\x1F\xFE\xD5\x59\x8A"
-			  "\xD7\xF6\xC9\x71\x67\x79\xD7\x0E"
-			  "\xBE\x1F\x8E\xEC\x55\x7E\x4F\x24"
-			  "\xE6\x87\xEA\xFE\x96\x25\x67\x8E"
-			  "\x93\x03\xFA\xFF\xCE\xAF\xB2\x3C"
-			  "\x6F\xEB\x57\xFB\xD3\x28\x87\xA9"
-			  "\xCE\xC2\xF5\x9C\xC6\x67\xB5\x97"
-			  "\x49\xF7\x04\xCB\xEF\x84\x98\x33"
-			  "\xAF\x38\xD3\x04\x1C\x24\x71\x38"
-			  "\xC7\x71\xDD\x43\x0D\x12\x4A\x18"
-			  "\xBA\xC4\xAF\xBA\xB2\x5B\xEB\x95"
-			  "\x02\x43\x5D\xCE\x19\xCC\xCD\x66"
-			  "\x91\x0B\x8C\x7F\x51\xC4\xBF\x3C"
-			  "\x8B\xF1\xCC\xAA\x29\xD7\x87\xCB"
-			  "\x3E\xC5\xF3\xC9\x75\xE8\xA3\x5B"
-			  "\x30\x45\xA9\xB7\xAF\x80\x64\x6F"
-			  "\x75\x4A\xA7\xC0\x6D\x19\x6B\xDE"
-			  "\x17\xDE\x6D\xEA\x87\x9F\x95\xAE"
-			  "\xF5\x3C\xEE\x54\xB8\x27\x84\xF8"
-			  "\x97\xA3\xE1\x6F\x38\x24\x34\x88"
-			  "\xCE\xBD\x32\x52\xE0\x00\x6C\x94"
-			  "\xC9\xD7\x5D\x37\x81\x33\x2E\x7F"
-			  "\x4F\x7E\x2E\x0D\x94\xBD\xEA\x59"
-			  "\x34\x39\xA8\x35\x12\xB7\xBC\xAC"
-			  "\xEA\x52\x9C\x78\x02\x6D\x92\x36"
-			  "\xFB\x59\x2B\xA4\xEA\x7B\x1B\x83"
-			  "\xE1\x4D\x5E\x2A\x7E\x92\xB1\x64"
-			  "\xDE\xE0\x27\x4B\x0A\x6F\x4C\xE3"
-			  "\xB0\xEB\x31\xE4\x69\x95\xAB\x35"
-			  "\x8B\x2C\xF5\x6B\x7F\xF1\xA2\x82"
-			  "\xF8\xD9\x47\x82\xA9\x82\x03\x91"
-			  "\x69\x1F\xBE\x4C\xE7\xC7\x34\x2F"
-			  "\x45\x72\x80\x17\x81\xBD\x9D\x62"
-			  "\xA1\xAC\xE8\xCF\xC6\x74\xCF\xDC"
-			  "\x22\x60\x4E\xE8\xA4\x5D\x85\xB9",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-
-/*
- * AES test vectors.
- */
-static const struct cipher_testvec aes_enc_tv_template[] = {
-	{ /* From FIPS-197 */
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.klen	= 16,
-		.input	= "\x00\x11\x22\x33\x44\x55\x66\x77"
-			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
-		.ilen	= 16,
-		.result	= "\x69\xc4\xe0\xd8\x6a\x7b\x04\x30"
-			  "\xd8\xcd\xb7\x80\x70\xb4\xc5\x5a",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17",
-		.klen	= 24,
-		.input	= "\x00\x11\x22\x33\x44\x55\x66\x77"
-			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
-		.ilen	= 16,
-		.result	= "\xdd\xa9\x7c\xa4\x86\x4c\xdf\xe0"
-			  "\x6e\xaf\x70\xa0\xec\x0d\x71\x91",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.klen	= 32,
-		.input	= "\x00\x11\x22\x33\x44\x55\x66\x77"
-			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
-		.ilen	= 16,
-		.result	= "\x8e\xa2\xb7\xca\x51\x67\x45\xbf"
-			  "\xea\xfc\x49\x90\x4b\x49\x60\x89",
-		.rlen	= 16,
-	}, { /* Generated with Crypto++ */
-		.key	= "\xA6\xC9\x83\xA6\xC9\xEC\x0F\x32"
-			  "\x55\x0F\x32\x55\x78\x9B\xBE\x78"
-			  "\x9B\xBE\xE1\x04\x27\xE1\x04\x27"
-			  "\x4A\x6D\x90\x4A\x6D\x90\xB3\xD6",
-		.klen	= 32,
-		.input	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
-			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
-			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
-			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
-			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
-			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
-			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
-			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
-			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
-			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
-			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
-			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
-			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
-			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
-			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
-			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
-			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
-			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
-			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
-			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
-			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
-			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
-			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
-			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
-			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
-			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
-			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
-			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
-			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
-			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
-			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
-			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12",
-		.ilen	= 496,
-		.result	= "\x71\x73\xF7\xDB\x24\x93\x21\x6D"
-			  "\x61\x1E\xBB\x63\x42\x79\xDB\x64"
-			  "\x6F\x82\xC0\xCA\xA3\x9B\xFA\x0B"
-			  "\xD9\x08\xC7\x4A\x90\xAE\x8F\x5F"
-			  "\x5E\x06\xF0\x5F\x31\x51\x18\x37"
-			  "\x45\xD7\xCA\x3A\xFD\x6C\x3F\xE1"
-			  "\xDD\x8D\x22\x65\x2B\x00\x50\xCE"
-			  "\xBA\x28\x67\xD7\xCE\x0E\x0D\xEA"
-			  "\x78\x69\x7F\xAE\x8F\x8B\x69\x37"
-			  "\x75\xE0\xDC\x96\xE0\xB7\xF4\x09"
-			  "\xCB\x6D\xA2\xFB\xDA\xAF\x09\xF8"
-			  "\x81\x82\x27\xFA\x45\x9C\x29\xA4"
-			  "\x22\x8B\x78\x69\x5B\x46\xF9\x39"
-			  "\x1B\xCC\xF9\x1D\x09\xEB\xBC\x5C"
-			  "\x41\x72\x51\x97\x1D\x07\x49\xA0"
-			  "\x1B\x8E\x65\x4B\xB2\x6A\x12\x03"
-			  "\x6A\x60\x95\xAC\xBD\xAC\x1A\x64"
-			  "\xDE\x5A\xA5\xF0\x83\x2F\xCB\xCA"
-			  "\x22\x74\xA6\x6C\x9B\x73\xCE\x3F"
-			  "\xE1\x8B\x22\x17\x59\x0C\x47\x89"
-			  "\x33\xA1\xD6\x47\x03\x19\x4F\xA8"
-			  "\x67\x69\xF0\x5B\xF0\x20\xAD\x06"
-			  "\x27\x81\x92\xD8\xC5\xBA\x98\x12"
-			  "\xBE\x24\xB5\x2F\x75\x02\xC2\xAD"
-			  "\x12\x2F\x07\x32\xEE\x39\xAF\x64"
-			  "\x05\x8F\xB3\xD4\xEB\x1B\x46\x6E"
-			  "\xD9\x21\xF9\xC4\xB7\xC9\x45\x68"
-			  "\xB4\xA1\x74\x9F\x82\x47\xEB\xCC"
-			  "\xBD\x0A\x14\x95\x0F\x8B\xA8\x2F"
-			  "\x4B\x1B\xA7\xBF\x82\xA6\x43\x0C"
-			  "\xB9\x39\x4A\xA8\x10\x6F\x50\x7B"
-			  "\x25\xFB\x26\x81\xE0\x2F\xF0\x96"
-			  "\x8D\x8B\xAC\x92\x0F\xF6\xED\x64"
-			  "\x63\x29\x4C\x8E\x18\x13\xC5\xBF"
-			  "\xFC\xA0\xD9\xBF\x7C\x3A\x0E\x29"
-			  "\x6F\xD1\x6C\x6F\xA5\xDA\xBF\xB1"
-			  "\x30\xEA\x44\x2D\xC3\x8F\x16\xE1"
-			  "\x66\xFA\xA3\x21\x3E\xFC\x13\xCA"
-			  "\xF0\xF6\xF0\x59\xBD\x8F\x38\x50"
-			  "\x31\xCB\x69\x3F\x96\x15\xD6\xF5"
-			  "\xAE\xFF\xF6\xAA\x41\x85\x4C\x10"
-			  "\x58\xE3\xF9\x44\xE6\x28\xDA\x9A"
-			  "\xDC\x6A\x80\x34\x73\x97\x1B\xC5"
-			  "\xCA\x26\x16\x77\x0E\x60\xAB\x89"
-			  "\x0F\x04\x27\xBD\xCE\x3E\x71\xB4"
-			  "\xA0\xD7\x22\x7E\xDB\xEB\x24\x70"
-			  "\x42\x71\x51\x78\x70\xB3\xE0\x3D"
-			  "\x84\x8E\x8D\x7B\xD0\x6D\xEA\x92"
-			  "\x11\x08\x42\x4F\xE5\xAD\x26\x92"
-			  "\xD2\x00\xAE\xA8\xE3\x4B\x37\x47"
-			  "\x22\xC1\x95\xC1\x63\x7F\xCB\x03"
-			  "\xF3\xE3\xD7\x9D\x60\xC7\xBC\xEA"
-			  "\x35\xA2\xFD\x45\x52\x39\x13\x6F"
-			  "\xC1\x53\xF3\x53\xDF\x33\x84\xD7"
-			  "\xD2\xC8\x37\xB0\x75\xE3\x41\x46"
-			  "\xB3\xC7\x83\x2E\x8A\xBB\xA4\xE5"
-			  "\x7F\x3C\xFD\x8B\xEB\xEA\x63\xBD"
-			  "\xB7\x46\xE7\xBF\x09\x9C\x0D\x0F"
-			  "\x40\x86\x7F\x51\xE1\x11\x9C\xCB"
-			  "\x88\xE6\x68\x47\xE3\x2B\xC5\xFF"
-			  "\x09\x79\xA0\x43\x5C\x0D\x08\x58"
-			  "\x17\xBB\xC0\x6B\x62\x3F\x56\xE9",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec aes_dec_tv_template[] = {
-	{ /* From FIPS-197 */
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.klen	= 16,
-		.input	= "\x69\xc4\xe0\xd8\x6a\x7b\x04\x30"
-			  "\xd8\xcd\xb7\x80\x70\xb4\xc5\x5a",
-		.ilen	= 16,
-		.result	= "\x00\x11\x22\x33\x44\x55\x66\x77"
-			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17",
-		.klen	= 24,
-		.input	= "\xdd\xa9\x7c\xa4\x86\x4c\xdf\xe0"
-			  "\x6e\xaf\x70\xa0\xec\x0d\x71\x91",
-		.ilen	= 16,
-		.result	= "\x00\x11\x22\x33\x44\x55\x66\x77"
-			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.klen	= 32,
-		.input	= "\x8e\xa2\xb7\xca\x51\x67\x45\xbf"
-			  "\xea\xfc\x49\x90\x4b\x49\x60\x89",
-		.ilen	= 16,
-		.result	= "\x00\x11\x22\x33\x44\x55\x66\x77"
-			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
-		.rlen	= 16,
-	}, { /* Generated with Crypto++ */
-		.key	= "\xA6\xC9\x83\xA6\xC9\xEC\x0F\x32"
-			  "\x55\x0F\x32\x55\x78\x9B\xBE\x78"
-			  "\x9B\xBE\xE1\x04\x27\xE1\x04\x27"
-			  "\x4A\x6D\x90\x4A\x6D\x90\xB3\xD6",
-		.klen	= 32,
-		.input	= "\x71\x73\xF7\xDB\x24\x93\x21\x6D"
-			  "\x61\x1E\xBB\x63\x42\x79\xDB\x64"
-			  "\x6F\x82\xC0\xCA\xA3\x9B\xFA\x0B"
-			  "\xD9\x08\xC7\x4A\x90\xAE\x8F\x5F"
-			  "\x5E\x06\xF0\x5F\x31\x51\x18\x37"
-			  "\x45\xD7\xCA\x3A\xFD\x6C\x3F\xE1"
-			  "\xDD\x8D\x22\x65\x2B\x00\x50\xCE"
-			  "\xBA\x28\x67\xD7\xCE\x0E\x0D\xEA"
-			  "\x78\x69\x7F\xAE\x8F\x8B\x69\x37"
-			  "\x75\xE0\xDC\x96\xE0\xB7\xF4\x09"
-			  "\xCB\x6D\xA2\xFB\xDA\xAF\x09\xF8"
-			  "\x81\x82\x27\xFA\x45\x9C\x29\xA4"
-			  "\x22\x8B\x78\x69\x5B\x46\xF9\x39"
-			  "\x1B\xCC\xF9\x1D\x09\xEB\xBC\x5C"
-			  "\x41\x72\x51\x97\x1D\x07\x49\xA0"
-			  "\x1B\x8E\x65\x4B\xB2\x6A\x12\x03"
-			  "\x6A\x60\x95\xAC\xBD\xAC\x1A\x64"
-			  "\xDE\x5A\xA5\xF0\x83\x2F\xCB\xCA"
-			  "\x22\x74\xA6\x6C\x9B\x73\xCE\x3F"
-			  "\xE1\x8B\x22\x17\x59\x0C\x47\x89"
-			  "\x33\xA1\xD6\x47\x03\x19\x4F\xA8"
-			  "\x67\x69\xF0\x5B\xF0\x20\xAD\x06"
-			  "\x27\x81\x92\xD8\xC5\xBA\x98\x12"
-			  "\xBE\x24\xB5\x2F\x75\x02\xC2\xAD"
-			  "\x12\x2F\x07\x32\xEE\x39\xAF\x64"
-			  "\x05\x8F\xB3\xD4\xEB\x1B\x46\x6E"
-			  "\xD9\x21\xF9\xC4\xB7\xC9\x45\x68"
-			  "\xB4\xA1\x74\x9F\x82\x47\xEB\xCC"
-			  "\xBD\x0A\x14\x95\x0F\x8B\xA8\x2F"
-			  "\x4B\x1B\xA7\xBF\x82\xA6\x43\x0C"
-			  "\xB9\x39\x4A\xA8\x10\x6F\x50\x7B"
-			  "\x25\xFB\x26\x81\xE0\x2F\xF0\x96"
-			  "\x8D\x8B\xAC\x92\x0F\xF6\xED\x64"
-			  "\x63\x29\x4C\x8E\x18\x13\xC5\xBF"
-			  "\xFC\xA0\xD9\xBF\x7C\x3A\x0E\x29"
-			  "\x6F\xD1\x6C\x6F\xA5\xDA\xBF\xB1"
-			  "\x30\xEA\x44\x2D\xC3\x8F\x16\xE1"
-			  "\x66\xFA\xA3\x21\x3E\xFC\x13\xCA"
-			  "\xF0\xF6\xF0\x59\xBD\x8F\x38\x50"
-			  "\x31\xCB\x69\x3F\x96\x15\xD6\xF5"
-			  "\xAE\xFF\xF6\xAA\x41\x85\x4C\x10"
-			  "\x58\xE3\xF9\x44\xE6\x28\xDA\x9A"
-			  "\xDC\x6A\x80\x34\x73\x97\x1B\xC5"
-			  "\xCA\x26\x16\x77\x0E\x60\xAB\x89"
-			  "\x0F\x04\x27\xBD\xCE\x3E\x71\xB4"
-			  "\xA0\xD7\x22\x7E\xDB\xEB\x24\x70"
-			  "\x42\x71\x51\x78\x70\xB3\xE0\x3D"
-			  "\x84\x8E\x8D\x7B\xD0\x6D\xEA\x92"
-			  "\x11\x08\x42\x4F\xE5\xAD\x26\x92"
-			  "\xD2\x00\xAE\xA8\xE3\x4B\x37\x47"
-			  "\x22\xC1\x95\xC1\x63\x7F\xCB\x03"
-			  "\xF3\xE3\xD7\x9D\x60\xC7\xBC\xEA"
-			  "\x35\xA2\xFD\x45\x52\x39\x13\x6F"
-			  "\xC1\x53\xF3\x53\xDF\x33\x84\xD7"
-			  "\xD2\xC8\x37\xB0\x75\xE3\x41\x46"
-			  "\xB3\xC7\x83\x2E\x8A\xBB\xA4\xE5"
-			  "\x7F\x3C\xFD\x8B\xEB\xEA\x63\xBD"
-			  "\xB7\x46\xE7\xBF\x09\x9C\x0D\x0F"
-			  "\x40\x86\x7F\x51\xE1\x11\x9C\xCB"
-			  "\x88\xE6\x68\x47\xE3\x2B\xC5\xFF"
-			  "\x09\x79\xA0\x43\x5C\x0D\x08\x58"
-			  "\x17\xBB\xC0\x6B\x62\x3F\x56\xE9",
-		.ilen	= 496,
-		.result	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
-			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
-			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
-			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
-			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
-			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
-			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
-			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
-			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
-			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
-			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
-			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
-			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
-			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
-			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
-			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
-			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
-			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
-			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
-			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
-			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
-			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
-			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
-			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
-			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
-			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
-			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
-			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
-			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
-			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
-			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
-			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec aes_cbc_enc_tv_template[] = {
-	{ /* From RFC 3602 */
-		.key    = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
-			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
-		.klen   = 16,
-		.iv	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.input	= "Single block msg",
-		.ilen   = 16,
-		.result = "\xe3\x53\x77\x9c\x10\x79\xae\xb8"
-			  "\x27\x08\x94\x2d\xbe\x77\x18\x1a",
-		.rlen   = 16,
-		.also_non_np = 1,
-		.np	= 8,
-		.tap	= { 3, 2, 3, 2, 3, 1, 1, 1 },
-	}, {
-		.key    = "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
-			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
-		.klen   = 16,
-		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.input  = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.ilen   = 32,
-		.result = "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a"
-			  "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a"
-			  "\x75\x86\x60\x2d\x25\x3c\xff\xf9"
-			  "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1",
-		.rlen   = 32,
-	}, { /* From NIST SP800-38A */
-		.key	= "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
-			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
-			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
-		.klen	= 24,
-		.iv	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.input	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen	= 64,
-		.result	= "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d"
-			  "\x71\x78\x18\x3a\x9f\xa0\x71\xe8"
-			  "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4"
-			  "\xe5\xe7\x38\x76\x3f\x69\x14\x5a"
-			  "\x57\x1b\x24\x20\x12\xfb\x7a\xe0"
-			  "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0"
-			  "\x08\xb0\xe2\x79\x88\x59\x88\x81"
-			  "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd",
-		.rlen	= 64,
-	}, {
-		.key	= "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
-			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
-			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
-			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
-		.klen	= 32,
-		.iv	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.input	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen	= 64,
-		.result	= "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba"
-			  "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6"
-			  "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d"
-			  "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d"
-			  "\x39\xf2\x33\x69\xa9\xd9\xba\xcf"
-			  "\xa5\x30\xe2\x63\x04\x23\x14\x61"
-			  "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc"
-			  "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b",
-		.rlen	= 64,
-	}, { /* Generated with Crypto++ */
-		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55"
-			  "\x0F\x32\x55\x78\x9B\xBE\x78\x9B"
-			  "\xBE\xE1\x04\x27\xE1\x04\x27\x4A"
-			  "\x6D\x90\x4A\x6D\x90\xB3\xD6\xF9",
-		.klen	= 32,
-		.iv	= "\xE7\x82\x1D\xB8\x53\x11\xAC\x47"
-			  "\xE2\x7D\x18\xD6\x71\x0C\xA7\x42",
-		.input	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
-			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
-			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
-			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
-			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
-			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
-			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
-			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
-			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
-			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
-			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
-			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
-			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
-			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
-			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
-			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
-			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
-			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
-			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
-			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
-			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
-			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
-			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
-			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
-			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
-			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
-			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
-			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
-			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
-			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
-			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
-			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12",
-		.ilen	= 496,
-		.result	= "\xEA\x65\x8A\x19\xB0\x66\xC1\x3F"
-			  "\xCE\xF1\x97\x75\xC1\xFD\xB5\xAF"
-			  "\x52\x65\xF7\xFF\xBC\xD8\x2D\x9F"
-			  "\x2F\xB9\x26\x9B\x6F\x10\xB7\xB8"
-			  "\x26\xA1\x02\x46\xA2\xAD\xC6\xC0"
-			  "\x11\x15\xFF\x6D\x1E\x82\x04\xA6"
-			  "\xB1\x74\xD1\x08\x13\xFD\x90\x7C"
-			  "\xF5\xED\xD3\xDB\x5A\x0A\x0C\x2F"
-			  "\x0A\x70\xF1\x88\x07\xCF\x21\x26"
-			  "\x40\x40\x8A\xF5\x53\xF7\x24\x4F"
-			  "\x83\x38\x43\x5F\x08\x99\xEB\xE3"
-			  "\xDC\x02\x64\x67\x50\x6E\x15\xC3"
-			  "\x01\x1A\xA0\x81\x13\x65\xA6\x73"
-			  "\x71\xA6\x3B\x91\x83\x77\xBE\xFA"
-			  "\xDB\x71\x73\xA6\xC1\xAE\x43\xC3"
-			  "\x36\xCE\xD6\xEB\xF9\x30\x1C\x4F"
-			  "\x80\x38\x5E\x9C\x6E\xAB\x98\x2F"
-			  "\x53\xAF\xCF\xC8\x9A\xB8\x86\x43"
-			  "\x3E\x86\xE7\xA1\xF4\x2F\x30\x40"
-			  "\x03\xA8\x6C\x50\x42\x9F\x77\x59"
-			  "\x89\xA0\xC5\xEC\x9A\xB8\xDD\x99"
-			  "\x16\x24\x02\x07\x48\xAE\xF2\x31"
-			  "\x34\x0E\xC3\x85\xFE\x1C\x95\x99"
-			  "\x87\x58\x98\x8B\xE7\xC6\xC5\x70"
-			  "\x73\x81\x07\x7C\x56\x2F\xD8\x1B"
-			  "\xB7\xB9\x2B\xAB\xE3\x01\x87\x0F"
-			  "\xD8\xBB\xC0\x0D\xAC\x2C\x2F\x98"
-			  "\x3C\x0B\xA2\x99\x4A\x8C\xF7\x04"
-			  "\xE0\xE0\xCF\xD1\x81\x5B\xFE\xF5"
-			  "\x24\x04\xFD\xB8\xDF\x13\xD8\xCD"
-			  "\xF1\xE3\x3D\x98\x50\x02\x77\x9E"
-			  "\xBC\x22\xAB\xFA\xC2\x43\x1F\x66"
-			  "\x20\x02\x23\xDA\xDF\xA0\x89\xF6"
-			  "\xD8\xF3\x45\x24\x53\x6F\x16\x77"
-			  "\x02\x3E\x7B\x36\x5F\xA0\x3B\x78"
-			  "\x63\xA2\xBD\xB5\xA4\xCA\x1E\xD3"
-			  "\x57\xBC\x0B\x9F\x43\x51\x28\x4F"
-			  "\x07\x50\x6C\x68\x12\x07\xCF\xFA"
-			  "\x6B\x72\x0B\xEB\xF8\x88\x90\x2C"
-			  "\x7E\xF5\x91\xD1\x03\xD8\xD5\xBD"
-			  "\x22\x39\x7B\x16\x03\x01\x69\xAF"
-			  "\x3D\x38\x66\x28\x0C\xBE\x5B\xC5"
-			  "\x03\xB4\x2F\x51\x8A\x56\x17\x2B"
-			  "\x88\x42\x6D\x40\x68\x8F\xD0\x11"
-			  "\x19\xF9\x1F\x43\x79\x95\x31\xFA"
-			  "\x28\x7A\x3D\xF7\x66\xEB\xEF\xAC"
-			  "\x06\xB2\x01\xAD\xDB\x68\xDB\xEC"
-			  "\x8D\x53\x6E\x72\x68\xA3\xC7\x63"
-			  "\x43\x2B\x78\xE0\x04\x29\x8F\x72"
-			  "\xB2\x2C\xE6\x84\x03\x30\x6D\xCD"
-			  "\x26\x92\x37\xE1\x2F\xBB\x8B\x9D"
-			  "\xE4\x4C\xF6\x93\xBC\xD9\xAD\x44"
-			  "\x52\x65\xC7\xB0\x0E\x3F\x0E\x61"
-			  "\x56\x5D\x1C\x6D\xA7\x05\x2E\xBC"
-			  "\x58\x08\x15\xAB\x12\xAB\x17\x4A"
-			  "\x5E\x1C\xF2\xCD\xB8\xA2\xAE\xFB"
-			  "\x9B\x2E\x0E\x85\x34\x80\x0E\x3F"
-			  "\x4C\xB8\xDB\xCE\x1C\x90\xA1\x61"
-			  "\x6C\x69\x09\x35\x9E\xD4\xF4\xAD"
-			  "\xBC\x06\x41\xE3\x01\xB4\x4E\x0A"
-			  "\xE0\x1F\x91\xF8\x82\x96\x2D\x65"
-			  "\xA3\xAA\x13\xCC\x50\xFF\x7B\x02",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec aes_cbc_dec_tv_template[] = {
-	{ /* From RFC 3602 */
-		.key    = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
-			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
-		.klen   = 16,
-		.iv     = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.input  = "\xe3\x53\x77\x9c\x10\x79\xae\xb8"
-			  "\x27\x08\x94\x2d\xbe\x77\x18\x1a",
-		.ilen   = 16,
-		.result = "Single block msg",
-		.rlen   = 16,
-		.also_non_np = 1,
-		.np	= 8,
-		.tap	= { 3, 2, 3, 2, 3, 1, 1, 1 },
-	}, {
-		.key    = "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
-			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
-		.klen   = 16,
-		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.input  = "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a"
-			  "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a"
-			  "\x75\x86\x60\x2d\x25\x3c\xff\xf9"
-			  "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1",
-		.ilen   = 32,
-		.result = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.rlen   = 32,
-	}, { /* From NIST SP800-38A */
-		.key	= "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
-			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
-			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
-		.klen	= 24,
-		.iv	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.input	= "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d"
-			  "\x71\x78\x18\x3a\x9f\xa0\x71\xe8"
-			  "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4"
-			  "\xe5\xe7\x38\x76\x3f\x69\x14\x5a"
-			  "\x57\x1b\x24\x20\x12\xfb\x7a\xe0"
-			  "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0"
-			  "\x08\xb0\xe2\x79\x88\x59\x88\x81"
-			  "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd",
-		.ilen	= 64,
-		.result	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.rlen	= 64,
-	}, {
-		.key	= "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
-			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
-			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
-			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
-		.klen	= 32,
-		.iv	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.input	= "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba"
-			  "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6"
-			  "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d"
-			  "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d"
-			  "\x39\xf2\x33\x69\xa9\xd9\xba\xcf"
-			  "\xa5\x30\xe2\x63\x04\x23\x14\x61"
-			  "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc"
-			  "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b",
-		.ilen	= 64,
-		.result	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.rlen	= 64,
-	}, { /* Generated with Crypto++ */
-		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55"
-			  "\x0F\x32\x55\x78\x9B\xBE\x78\x9B"
-			  "\xBE\xE1\x04\x27\xE1\x04\x27\x4A"
-			  "\x6D\x90\x4A\x6D\x90\xB3\xD6\xF9",
-		.klen	= 32,
-		.iv	= "\xE7\x82\x1D\xB8\x53\x11\xAC\x47"
-			  "\xE2\x7D\x18\xD6\x71\x0C\xA7\x42",
-		.input	= "\xEA\x65\x8A\x19\xB0\x66\xC1\x3F"
-			  "\xCE\xF1\x97\x75\xC1\xFD\xB5\xAF"
-			  "\x52\x65\xF7\xFF\xBC\xD8\x2D\x9F"
-			  "\x2F\xB9\x26\x9B\x6F\x10\xB7\xB8"
-			  "\x26\xA1\x02\x46\xA2\xAD\xC6\xC0"
-			  "\x11\x15\xFF\x6D\x1E\x82\x04\xA6"
-			  "\xB1\x74\xD1\x08\x13\xFD\x90\x7C"
-			  "\xF5\xED\xD3\xDB\x5A\x0A\x0C\x2F"
-			  "\x0A\x70\xF1\x88\x07\xCF\x21\x26"
-			  "\x40\x40\x8A\xF5\x53\xF7\x24\x4F"
-			  "\x83\x38\x43\x5F\x08\x99\xEB\xE3"
-			  "\xDC\x02\x64\x67\x50\x6E\x15\xC3"
-			  "\x01\x1A\xA0\x81\x13\x65\xA6\x73"
-			  "\x71\xA6\x3B\x91\x83\x77\xBE\xFA"
-			  "\xDB\x71\x73\xA6\xC1\xAE\x43\xC3"
-			  "\x36\xCE\xD6\xEB\xF9\x30\x1C\x4F"
-			  "\x80\x38\x5E\x9C\x6E\xAB\x98\x2F"
-			  "\x53\xAF\xCF\xC8\x9A\xB8\x86\x43"
-			  "\x3E\x86\xE7\xA1\xF4\x2F\x30\x40"
-			  "\x03\xA8\x6C\x50\x42\x9F\x77\x59"
-			  "\x89\xA0\xC5\xEC\x9A\xB8\xDD\x99"
-			  "\x16\x24\x02\x07\x48\xAE\xF2\x31"
-			  "\x34\x0E\xC3\x85\xFE\x1C\x95\x99"
-			  "\x87\x58\x98\x8B\xE7\xC6\xC5\x70"
-			  "\x73\x81\x07\x7C\x56\x2F\xD8\x1B"
-			  "\xB7\xB9\x2B\xAB\xE3\x01\x87\x0F"
-			  "\xD8\xBB\xC0\x0D\xAC\x2C\x2F\x98"
-			  "\x3C\x0B\xA2\x99\x4A\x8C\xF7\x04"
-			  "\xE0\xE0\xCF\xD1\x81\x5B\xFE\xF5"
-			  "\x24\x04\xFD\xB8\xDF\x13\xD8\xCD"
-			  "\xF1\xE3\x3D\x98\x50\x02\x77\x9E"
-			  "\xBC\x22\xAB\xFA\xC2\x43\x1F\x66"
-			  "\x20\x02\x23\xDA\xDF\xA0\x89\xF6"
-			  "\xD8\xF3\x45\x24\x53\x6F\x16\x77"
-			  "\x02\x3E\x7B\x36\x5F\xA0\x3B\x78"
-			  "\x63\xA2\xBD\xB5\xA4\xCA\x1E\xD3"
-			  "\x57\xBC\x0B\x9F\x43\x51\x28\x4F"
-			  "\x07\x50\x6C\x68\x12\x07\xCF\xFA"
-			  "\x6B\x72\x0B\xEB\xF8\x88\x90\x2C"
-			  "\x7E\xF5\x91\xD1\x03\xD8\xD5\xBD"
-			  "\x22\x39\x7B\x16\x03\x01\x69\xAF"
-			  "\x3D\x38\x66\x28\x0C\xBE\x5B\xC5"
-			  "\x03\xB4\x2F\x51\x8A\x56\x17\x2B"
-			  "\x88\x42\x6D\x40\x68\x8F\xD0\x11"
-			  "\x19\xF9\x1F\x43\x79\x95\x31\xFA"
-			  "\x28\x7A\x3D\xF7\x66\xEB\xEF\xAC"
-			  "\x06\xB2\x01\xAD\xDB\x68\xDB\xEC"
-			  "\x8D\x53\x6E\x72\x68\xA3\xC7\x63"
-			  "\x43\x2B\x78\xE0\x04\x29\x8F\x72"
-			  "\xB2\x2C\xE6\x84\x03\x30\x6D\xCD"
-			  "\x26\x92\x37\xE1\x2F\xBB\x8B\x9D"
-			  "\xE4\x4C\xF6\x93\xBC\xD9\xAD\x44"
-			  "\x52\x65\xC7\xB0\x0E\x3F\x0E\x61"
-			  "\x56\x5D\x1C\x6D\xA7\x05\x2E\xBC"
-			  "\x58\x08\x15\xAB\x12\xAB\x17\x4A"
-			  "\x5E\x1C\xF2\xCD\xB8\xA2\xAE\xFB"
-			  "\x9B\x2E\x0E\x85\x34\x80\x0E\x3F"
-			  "\x4C\xB8\xDB\xCE\x1C\x90\xA1\x61"
-			  "\x6C\x69\x09\x35\x9E\xD4\xF4\xAD"
-			  "\xBC\x06\x41\xE3\x01\xB4\x4E\x0A"
-			  "\xE0\x1F\x91\xF8\x82\x96\x2D\x65"
-			  "\xA3\xAA\x13\xCC\x50\xFF\x7B\x02",
-		.ilen	= 496,
-		.result	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
-			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
-			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
-			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
-			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
-			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
-			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
-			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
-			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
-			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
-			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
-			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
-			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
-			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
-			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
-			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
-			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
-			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
-			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
-			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
-			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
-			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
-			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
-			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
-			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
-			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
-			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
-			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
-			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
-			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
-			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
-			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct aead_testvec hmac_md5_ecb_cipher_null_enc_tv_template[] = {
-	{ /* Input data from RFC 2410 Case 1 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x00"	/* enc key length */
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen   = 8 + 16 + 0,
-		.iv     = "",
-		.input  = "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.ilen   = 8,
-		.result = "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xaa\x42\xfe\x43\x8d\xea\xa3\x5a"
-			  "\xb9\x3d\x9f\xb1\xa3\x8e\x9b\xae",
-		.rlen   = 8 + 16,
-	}, { /* Input data from RFC 2410 Case 2 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x00"	/* enc key length */
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen   = 8 + 16 + 0,
-		.iv     = "",
-		.input  = "Network Security People Have A Strange Sense Of Humor",
-		.ilen   = 53,
-		.result = "Network Security People Have A Strange Sense Of Humor"
-			  "\x73\xa5\x3e\x1c\x08\x0e\x8a\x8a"
-			  "\x8e\xb5\x5f\x90\x8e\xfe\x13\x23",
-		.rlen   = 53 + 16,
-	},
-};
-
-static const struct aead_testvec hmac_md5_ecb_cipher_null_dec_tv_template[] = {
-	{
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x00"	/* enc key length */
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen   = 8 + 16 + 0,
-		.iv     = "",
-		.input  = "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xaa\x42\xfe\x43\x8d\xea\xa3\x5a"
-			  "\xb9\x3d\x9f\xb1\xa3\x8e\x9b\xae",
-		.ilen   = 8 + 16,
-		.result = "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.rlen   = 8,
-	}, {
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x00"	/* enc key length */
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen   = 8 + 16 + 0,
-		.iv     = "",
-		.input  = "Network Security People Have A Strange Sense Of Humor"
-			  "\x73\xa5\x3e\x1c\x08\x0e\x8a\x8a"
-			  "\x8e\xb5\x5f\x90\x8e\xfe\x13\x23",
-		.ilen   = 53 + 16,
-		.result = "Network Security People Have A Strange Sense Of Humor",
-		.rlen   = 53,
-	},
-};
-
-static const struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = {
-	{ /* RFC 3602 Case 1 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00"
-			  "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
-			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
-		.klen   = 8 + 20 + 16,
-		.iv     = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.assoc	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.alen	= 16,
-		.input  = "Single block msg",
-		.ilen   = 16,
-		.result = "\xe3\x53\x77\x9c\x10\x79\xae\xb8"
-			  "\x27\x08\x94\x2d\xbe\x77\x18\x1a"
-			  "\x1b\x13\xcb\xaf\x89\x5e\xe1\x2c"
-			  "\x13\xc5\x2e\xa3\xcc\xed\xdc\xb5"
-			  "\x03\x71\xa2\x06",
-		.rlen   = 16 + 20,
-	}, { /* RFC 3602 Case 2 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33"
-			  "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
-			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
-		.klen   = 8 + 20 + 16,
-		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.assoc	= "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.alen	= 16,
-		.input  = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.ilen   = 32,
-		.result = "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a"
-			  "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a"
-			  "\x75\x86\x60\x2d\x25\x3c\xff\xf9"
-			  "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1"
-			  "\xad\x9b\x4c\x5c\x85\xe1\xda\xae"
-			  "\xee\x81\x4e\xd7\xdb\x74\xcf\x58"
-			  "\x65\x39\xf8\xde",
-		.rlen   = 32 + 20,
-	}, { /* RFC 3602 Case 3 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"            /* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55"
-			  "\x6c\x3e\xa0\x47\x76\x30\xce\x21"
-			  "\xa2\xce\x33\x4a\xa7\x46\xc2\xcd",
-		.klen   = 8 + 20 + 16,
-		.iv     = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
-			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
-		.assoc	= "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
-			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
-		.alen	= 16,
-		.input  = "This is a 48-byte message (exactly 3 AES blocks)",
-		.ilen   = 48,
-		.result = "\xd0\xa0\x2b\x38\x36\x45\x17\x53"
-			  "\xd4\x93\x66\x5d\x33\xf0\xe8\x86"
-			  "\x2d\xea\x54\xcd\xb2\x93\xab\xc7"
-			  "\x50\x69\x39\x27\x67\x72\xf8\xd5"
-			  "\x02\x1c\x19\x21\x6b\xad\x52\x5c"
-			  "\x85\x79\x69\x5d\x83\xba\x26\x84"
-			  "\xc2\xec\x0c\xf8\x7f\x05\xba\xca"
-			  "\xff\xee\x4c\xd0\x93\xe6\x36\x7f"
-			  "\x8d\x62\xf2\x1e",
-		.rlen   = 48 + 20,
-	}, { /* RFC 3602 Case 4 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55"
-			  "\x56\xe4\x7a\x38\xc5\x59\x89\x74"
-			  "\xbc\x46\x90\x3d\xba\x29\x03\x49",
-		.klen   = 8 + 20 + 16,
-		.iv     = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
-			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
-		.assoc	= "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
-			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
-		.alen	= 16,
-		.input  = "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf",
-		.ilen   = 64,
-		.result = "\xc3\x0e\x32\xff\xed\xc0\x77\x4e"
-			  "\x6a\xff\x6a\xf0\x86\x9f\x71\xaa"
-			  "\x0f\x3a\xf0\x7a\x9a\x31\xa9\xc6"
-			  "\x84\xdb\x20\x7e\xb0\xef\x8e\x4e"
-			  "\x35\x90\x7a\xa6\x32\xc3\xff\xdf"
-			  "\x86\x8b\xb7\xb2\x9d\x3d\x46\xad"
-			  "\x83\xce\x9f\x9a\x10\x2e\xe9\x9d"
-			  "\x49\xa5\x3e\x87\xf4\xc3\xda\x55"
-			  "\x1c\x45\x57\xa9\x56\xcb\xa9\x2d"
-			  "\x18\xac\xf1\xc7\x5d\xd1\xcd\x0d"
-			  "\x1d\xbe\xc6\xe9",
-		.rlen   = 64 + 20,
-	}, { /* RFC 3602 Case 5 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"            /* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55"
-			  "\x90\xd3\x82\xb4\x10\xee\xba\x7a"
-			  "\xd9\x38\xc4\x6c\xec\x1a\x82\xbf",
-		.klen   = 8 + 20 + 16,
-		.iv     = "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
-			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
-			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
-		.alen   = 24,
-		.input  = "\x08\x00\x0e\xbd\xa7\x0a\x00\x00"
-			  "\x8e\x9c\x08\x3d\xb9\x5b\x07\x00"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x01\x02\x03\x04\x05\x06\x07\x08"
-			  "\x09\x0a\x0b\x0c\x0d\x0e\x0e\x01",
-		.ilen   = 80,
-		.result = "\xf6\x63\xc2\x5d\x32\x5c\x18\xc6"
-			  "\xa9\x45\x3e\x19\x4e\x12\x08\x49"
-			  "\xa4\x87\x0b\x66\xcc\x6b\x99\x65"
-			  "\x33\x00\x13\xb4\x89\x8d\xc8\x56"
-			  "\xa4\x69\x9e\x52\x3a\x55\xdb\x08"
-			  "\x0b\x59\xec\x3a\x8e\x4b\x7e\x52"
-			  "\x77\x5b\x07\xd1\xdb\x34\xed\x9c"
-			  "\x53\x8a\xb5\x0c\x55\x1b\x87\x4a"
-			  "\xa2\x69\xad\xd0\x47\xad\x2d\x59"
-			  "\x13\xac\x19\xb7\xcf\xba\xd4\xa6"
-			  "\x58\xc6\x84\x75\xe4\xe9\x6b\x0c"
-			  "\xe1\xc5\x0b\x73\x4d\x82\x55\xa8"
-			  "\x85\xe1\x59\xf7",
-		.rlen   = 80 + 20,
-       }, { /* NIST SP800-38A F.2.3 CBC-AES192.Encrypt */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"            /* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x18"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55"
-			  "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
-			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
-			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
-		.klen   = 8 + 20 + 24,
-		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.alen	= 16,
-		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen   = 64,
-		.result = "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d"
-			  "\x71\x78\x18\x3a\x9f\xa0\x71\xe8"
-			  "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4"
-			  "\xe5\xe7\x38\x76\x3f\x69\x14\x5a"
-			  "\x57\x1b\x24\x20\x12\xfb\x7a\xe0"
-			  "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0"
-			  "\x08\xb0\xe2\x79\x88\x59\x88\x81"
-			  "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd"
-			  "\x73\xe3\x19\x3f\x8b\xc9\xc6\xf4"
-			  "\x5a\xf1\x5b\xa8\x98\x07\xc5\x36"
-			  "\x47\x4c\xfc\x36",
-		.rlen   = 64 + 20,
-	}, { /* NIST SP800-38A F.2.5 CBC-AES256.Encrypt */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x20"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55"
-			  "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
-			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
-			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
-			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
-		.klen   = 8 + 20 + 32,
-		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.alen	= 16,
-		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen   = 64,
-		.result = "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba"
-			  "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6"
-			  "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d"
-			  "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d"
-			  "\x39\xf2\x33\x69\xa9\xd9\xba\xcf"
-			  "\xa5\x30\xe2\x63\x04\x23\x14\x61"
-			  "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc"
-			  "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b"
-			  "\xa3\xe8\x9b\x17\xe3\xf4\x7f\xde"
-			  "\x1b\x9f\xc6\x81\x26\x43\x4a\x87"
-			  "\x51\xee\xd6\x4e",
-		.rlen   = 64 + 20,
-	},
-};
-
-static const struct aead_testvec hmac_sha1_ecb_cipher_null_enc_tv_temp[] = {
-	{ /* Input data from RFC 2410 Case 1 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x00"	/* enc key length */
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00",
-		.klen   = 8 + 20 + 0,
-		.iv     = "",
-		.input  = "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.ilen   = 8,
-		.result = "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\x40\xc3\x0a\xa1\xc9\xa0\x28\xab"
-			  "\x99\x5e\x19\x04\xd1\x72\xef\xb8"
-			  "\x8c\x5e\xe4\x08",
-		.rlen   = 8 + 20,
-	}, { /* Input data from RFC 2410 Case 2 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x00"	/* enc key length */
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00",
-		.klen   = 8 + 20 + 0,
-		.iv     = "",
-		.input  = "Network Security People Have A Strange Sense Of Humor",
-		.ilen   = 53,
-		.result = "Network Security People Have A Strange Sense Of Humor"
-			  "\x75\x6f\x42\x1e\xf8\x50\x21\xd2"
-			  "\x65\x47\xee\x8e\x1a\xef\x16\xf6"
-			  "\x91\x56\xe4\xd6",
-		.rlen   = 53 + 20,
-	},
-};
-
-static const struct aead_testvec hmac_sha1_ecb_cipher_null_dec_tv_temp[] = {
-	{
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x00"	/* enc key length */
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00",
-		.klen   = 8 + 20 + 0,
-		.iv     = "",
-		.input  = "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\x40\xc3\x0a\xa1\xc9\xa0\x28\xab"
-			  "\x99\x5e\x19\x04\xd1\x72\xef\xb8"
-			  "\x8c\x5e\xe4\x08",
-		.ilen   = 8 + 20,
-		.result = "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.rlen   = 8,
-	}, {
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x00"	/* enc key length */
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00",
-		.klen   = 8 + 20 + 0,
-		.iv     = "",
-		.input  = "Network Security People Have A Strange Sense Of Humor"
-			  "\x75\x6f\x42\x1e\xf8\x50\x21\xd2"
-			  "\x65\x47\xee\x8e\x1a\xef\x16\xf6"
-			  "\x91\x56\xe4\xd6",
-		.ilen   = 53 + 20,
-		.result = "Network Security People Have A Strange Sense Of Humor",
-		.rlen   = 53,
-	},
-};
-
-static const struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = {
-	{ /* RFC 3602 Case 1 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
-			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
-		.klen   = 8 + 32 + 16,
-		.iv     = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.assoc	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.alen	= 16,
-		.input  = "Single block msg",
-		.ilen   = 16,
-		.result = "\xe3\x53\x77\x9c\x10\x79\xae\xb8"
-			  "\x27\x08\x94\x2d\xbe\x77\x18\x1a"
-			  "\xcc\xde\x2d\x6a\xae\xf1\x0b\xcc"
-			  "\x38\x06\x38\x51\xb4\xb8\xf3\x5b"
-			  "\x5c\x34\xa6\xa3\x6e\x0b\x05\xe5"
-			  "\x6a\x6d\x44\xaa\x26\xa8\x44\xa5",
-		.rlen   = 16 + 32,
-	}, { /* RFC 3602 Case 2 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
-			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
-		.klen   = 8 + 32 + 16,
-		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.assoc	= "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.alen	= 16,
-		.input  = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.ilen   = 32,
-		.result = "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a"
-			  "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a"
-			  "\x75\x86\x60\x2d\x25\x3c\xff\xf9"
-			  "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1"
-			  "\xf5\x33\x53\xf3\x68\x85\x2a\x99"
-			  "\x0e\x06\x58\x8f\xba\xf6\x06\xda"
-			  "\x49\x69\x0d\x5b\xd4\x36\x06\x62"
-			  "\x35\x5e\x54\x58\x53\x4d\xdf\xbf",
-		.rlen   = 32 + 32,
-	}, { /* RFC 3602 Case 3 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"            /* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x6c\x3e\xa0\x47\x76\x30\xce\x21"
-			  "\xa2\xce\x33\x4a\xa7\x46\xc2\xcd",
-		.klen   = 8 + 32 + 16,
-		.iv     = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
-			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
-		.assoc	= "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
-			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
-		.alen	= 16,
-		.input  = "This is a 48-byte message (exactly 3 AES blocks)",
-		.ilen   = 48,
-		.result = "\xd0\xa0\x2b\x38\x36\x45\x17\x53"
-			  "\xd4\x93\x66\x5d\x33\xf0\xe8\x86"
-			  "\x2d\xea\x54\xcd\xb2\x93\xab\xc7"
-			  "\x50\x69\x39\x27\x67\x72\xf8\xd5"
-			  "\x02\x1c\x19\x21\x6b\xad\x52\x5c"
-			  "\x85\x79\x69\x5d\x83\xba\x26\x84"
-			  "\x68\xb9\x3e\x90\x38\xa0\x88\x01"
-			  "\xe7\xc6\xce\x10\x31\x2f\x9b\x1d"
-			  "\x24\x78\xfb\xbe\x02\xe0\x4f\x40"
-			  "\x10\xbd\xaa\xc6\xa7\x79\xe0\x1a",
-		.rlen   = 48 + 32,
-	}, { /* RFC 3602 Case 4 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x56\xe4\x7a\x38\xc5\x59\x89\x74"
-			  "\xbc\x46\x90\x3d\xba\x29\x03\x49",
-		.klen   = 8 + 32 + 16,
-		.iv     = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
-			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
-		.assoc	= "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
-			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
-		.alen	= 16,
-		.input  = "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf",
-		.ilen   = 64,
-		.result = "\xc3\x0e\x32\xff\xed\xc0\x77\x4e"
-			  "\x6a\xff\x6a\xf0\x86\x9f\x71\xaa"
-			  "\x0f\x3a\xf0\x7a\x9a\x31\xa9\xc6"
-			  "\x84\xdb\x20\x7e\xb0\xef\x8e\x4e"
-			  "\x35\x90\x7a\xa6\x32\xc3\xff\xdf"
-			  "\x86\x8b\xb7\xb2\x9d\x3d\x46\xad"
-			  "\x83\xce\x9f\x9a\x10\x2e\xe9\x9d"
-			  "\x49\xa5\x3e\x87\xf4\xc3\xda\x55"
-			  "\x7a\x1b\xd4\x3c\xdb\x17\x95\xe2"
-			  "\xe0\x93\xec\xc9\x9f\xf7\xce\xd8"
-			  "\x3f\x54\xe2\x49\x39\xe3\x71\x25"
-			  "\x2b\x6c\xe9\x5d\xec\xec\x2b\x64",
-		.rlen   = 64 + 32,
-	}, { /* RFC 3602 Case 5 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"            /* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x90\xd3\x82\xb4\x10\xee\xba\x7a"
-			  "\xd9\x38\xc4\x6c\xec\x1a\x82\xbf",
-		.klen   = 8 + 32 + 16,
-		.iv     = "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
-			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
-			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
-		.alen   = 24,
-		.input  = "\x08\x00\x0e\xbd\xa7\x0a\x00\x00"
-			  "\x8e\x9c\x08\x3d\xb9\x5b\x07\x00"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x01\x02\x03\x04\x05\x06\x07\x08"
-			  "\x09\x0a\x0b\x0c\x0d\x0e\x0e\x01",
-		.ilen   = 80,
-		.result = "\xf6\x63\xc2\x5d\x32\x5c\x18\xc6"
-			  "\xa9\x45\x3e\x19\x4e\x12\x08\x49"
-			  "\xa4\x87\x0b\x66\xcc\x6b\x99\x65"
-			  "\x33\x00\x13\xb4\x89\x8d\xc8\x56"
-			  "\xa4\x69\x9e\x52\x3a\x55\xdb\x08"
-			  "\x0b\x59\xec\x3a\x8e\x4b\x7e\x52"
-			  "\x77\x5b\x07\xd1\xdb\x34\xed\x9c"
-			  "\x53\x8a\xb5\x0c\x55\x1b\x87\x4a"
-			  "\xa2\x69\xad\xd0\x47\xad\x2d\x59"
-			  "\x13\xac\x19\xb7\xcf\xba\xd4\xa6"
-			  "\xbb\xd4\x0f\xbe\xa3\x3b\x4c\xb8"
-			  "\x3a\xd2\xe1\x03\x86\xa5\x59\xb7"
-			  "\x73\xc3\x46\x20\x2c\xb1\xef\x68"
-			  "\xbb\x8a\x32\x7e\x12\x8c\x69\xcf",
-		.rlen   = 80 + 32,
-       }, { /* NIST SP800-38A F.2.3 CBC-AES192.Encrypt */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"            /* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x18"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
-			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
-			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
-		.klen   = 8 + 32 + 24,
-		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.alen   = 16,
-		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen   = 64,
-		.result = "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d"
-			  "\x71\x78\x18\x3a\x9f\xa0\x71\xe8"
-			  "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4"
-			  "\xe5\xe7\x38\x76\x3f\x69\x14\x5a"
-			  "\x57\x1b\x24\x20\x12\xfb\x7a\xe0"
-			  "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0"
-			  "\x08\xb0\xe2\x79\x88\x59\x88\x81"
-			  "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd"
-			  "\x2f\xee\x5f\xdb\x66\xfe\x79\x09"
-			  "\x61\x81\x31\xea\x5b\x3d\x8e\xfb"
-			  "\xca\x71\x85\x93\xf7\x85\x55\x8b"
-			  "\x7a\xe4\x94\xca\x8b\xba\x19\x33",
-		.rlen   = 64 + 32,
-	}, { /* NIST SP800-38A F.2.5 CBC-AES256.Encrypt */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x20"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
-			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
-			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
-			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
-		.klen   = 8 + 32 + 32,
-		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.alen   = 16,
-		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen   = 64,
-		.result = "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba"
-			  "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6"
-			  "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d"
-			  "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d"
-			  "\x39\xf2\x33\x69\xa9\xd9\xba\xcf"
-			  "\xa5\x30\xe2\x63\x04\x23\x14\x61"
-			  "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc"
-			  "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b"
-			  "\x24\x29\xed\xc2\x31\x49\xdb\xb1"
-			  "\x8f\x74\xbd\x17\x92\x03\xbe\x8f"
-			  "\xf3\x61\xde\x1c\xe9\xdb\xcd\xd0"
-			  "\xcc\xce\xe9\x85\x57\xcf\x6f\x5f",
-		.rlen   = 64 + 32,
-	},
-};
-
-static const struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = {
-	{ /* RFC 3602 Case 1 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
-			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
-		.klen   = 8 + 64 + 16,
-		.iv     = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.assoc	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.alen   = 16,
-		.input  = "Single block msg",
-		.ilen   = 16,
-		.result = "\xe3\x53\x77\x9c\x10\x79\xae\xb8"
-			  "\x27\x08\x94\x2d\xbe\x77\x18\x1a"
-			  "\x3f\xdc\xad\x90\x03\x63\x5e\x68"
-			  "\xc3\x13\xdd\xa4\x5c\x4d\x54\xa7"
-			  "\x19\x6e\x03\x75\x2b\xa1\x62\xce"
-			  "\xe0\xc6\x96\x75\xb2\x14\xca\x96"
-			  "\xec\xbd\x50\x08\x07\x64\x1a\x49"
-			  "\xe8\x9a\x7c\x06\x3d\xcb\xff\xb2"
-			  "\xfa\x20\x89\xdd\x9c\xac\x9e\x16"
-			  "\x18\x8a\xa0\x6d\x01\x6c\xa3\x3a",
-		.rlen   = 16 + 64,
-	}, { /* RFC 3602 Case 2 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
-			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
-		.klen   = 8 + 64 + 16,
-		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.assoc	= "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.alen   = 16,
-		.input  = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.ilen   = 32,
-		.result = "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a"
-			  "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a"
-			  "\x75\x86\x60\x2d\x25\x3c\xff\xf9"
-			  "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1"
-			  "\xda\xb2\x0c\xb2\x26\xc4\xd5\xef"
-			  "\x60\x38\xa4\x5e\x9a\x8c\x1b\x41"
-			  "\x03\x9f\xc4\x64\x7f\x01\x42\x9b"
-			  "\x0e\x1b\xea\xef\xbc\x88\x19\x5e"
-			  "\x31\x7e\xc2\x95\xfc\x09\x32\x0a"
-			  "\x46\x32\x7c\x41\x9c\x59\x3e\xe9"
-			  "\x8f\x9f\xd4\x31\xd6\x22\xbd\xf8"
-			  "\xf7\x0a\x94\xe5\xa9\xc3\xf6\x9d",
-		.rlen   = 32 + 64,
-	}, { /* RFC 3602 Case 3 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"            /* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
-			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
-			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
-			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
-			  "\x6c\x3e\xa0\x47\x76\x30\xce\x21"
-			  "\xa2\xce\x33\x4a\xa7\x46\xc2\xcd",
-		.klen   = 8 + 64 + 16,
-		.iv     = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
-			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
-		.assoc	= "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
-			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
-		.alen   = 16,
-		.input  = "This is a 48-byte message (exactly 3 AES blocks)",
-		.ilen   = 48,
-		.result = "\xd0\xa0\x2b\x38\x36\x45\x17\x53"
-			  "\xd4\x93\x66\x5d\x33\xf0\xe8\x86"
-			  "\x2d\xea\x54\xcd\xb2\x93\xab\xc7"
-			  "\x50\x69\x39\x27\x67\x72\xf8\xd5"
-			  "\x02\x1c\x19\x21\x6b\xad\x52\x5c"
-			  "\x85\x79\x69\x5d\x83\xba\x26\x84"
-			  "\x64\x19\x17\x5b\x57\xe0\x21\x0f"
-			  "\xca\xdb\xa1\x26\x38\x14\xa2\x69"
-			  "\xdb\x54\x67\x80\xc0\x54\xe0\xfd"
-			  "\x3e\x91\xe7\x91\x7f\x13\x38\x44"
-			  "\xb7\xb1\xd6\xc8\x7d\x48\x8d\x41"
-			  "\x08\xea\x29\x6c\x74\x67\x3f\xb0"
-			  "\xac\x7f\x5c\x1d\xf5\xee\x22\x66"
-			  "\x27\xa6\xb6\x13\xba\xba\xf0\xc2",
-		.rlen   = 48 + 64,
-	}, { /* RFC 3602 Case 4 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
-			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
-			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
-			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
-			  "\x56\xe4\x7a\x38\xc5\x59\x89\x74"
-			  "\xbc\x46\x90\x3d\xba\x29\x03\x49",
-		.klen   = 8 + 64 + 16,
-		.iv     = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
-			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
-		.assoc	= "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
-			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
-		.alen   = 16,
-		.input  = "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf",
-		.ilen   = 64,
-		.result = "\xc3\x0e\x32\xff\xed\xc0\x77\x4e"
-			  "\x6a\xff\x6a\xf0\x86\x9f\x71\xaa"
-			  "\x0f\x3a\xf0\x7a\x9a\x31\xa9\xc6"
-			  "\x84\xdb\x20\x7e\xb0\xef\x8e\x4e"
-			  "\x35\x90\x7a\xa6\x32\xc3\xff\xdf"
-			  "\x86\x8b\xb7\xb2\x9d\x3d\x46\xad"
-			  "\x83\xce\x9f\x9a\x10\x2e\xe9\x9d"
-			  "\x49\xa5\x3e\x87\xf4\xc3\xda\x55"
-			  "\x82\xcd\x42\x28\x21\x20\x15\xcc"
-			  "\xb7\xb2\x48\x40\xc7\x64\x41\x3a"
-			  "\x61\x32\x82\x85\xcf\x27\xed\xb4"
-			  "\xe4\x68\xa2\xf5\x79\x26\x27\xb2"
-			  "\x51\x67\x6a\xc4\xf0\x66\x55\x50"
-			  "\xbc\x6f\xed\xd5\x8d\xde\x23\x7c"
-			  "\x62\x98\x14\xd7\x2f\x37\x8d\xdf"
-			  "\xf4\x33\x80\xeb\x8e\xb4\xa4\xda",
-		.rlen   = 64 + 64,
-	}, { /* RFC 3602 Case 5 */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"            /* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
-			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
-			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
-			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
-			  "\x90\xd3\x82\xb4\x10\xee\xba\x7a"
-			  "\xd9\x38\xc4\x6c\xec\x1a\x82\xbf",
-		.klen   = 8 + 64 + 16,
-		.iv     = "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
-			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
-			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
-		.alen   = 24,
-		.input  = "\x08\x00\x0e\xbd\xa7\x0a\x00\x00"
-			  "\x8e\x9c\x08\x3d\xb9\x5b\x07\x00"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x01\x02\x03\x04\x05\x06\x07\x08"
-			  "\x09\x0a\x0b\x0c\x0d\x0e\x0e\x01",
-		.ilen   = 80,
-		.result = "\xf6\x63\xc2\x5d\x32\x5c\x18\xc6"
-			  "\xa9\x45\x3e\x19\x4e\x12\x08\x49"
-			  "\xa4\x87\x0b\x66\xcc\x6b\x99\x65"
-			  "\x33\x00\x13\xb4\x89\x8d\xc8\x56"
-			  "\xa4\x69\x9e\x52\x3a\x55\xdb\x08"
-			  "\x0b\x59\xec\x3a\x8e\x4b\x7e\x52"
-			  "\x77\x5b\x07\xd1\xdb\x34\xed\x9c"
-			  "\x53\x8a\xb5\x0c\x55\x1b\x87\x4a"
-			  "\xa2\x69\xad\xd0\x47\xad\x2d\x59"
-			  "\x13\xac\x19\xb7\xcf\xba\xd4\xa6"
-			  "\x74\x84\x94\xe2\xd7\x7a\xf9\xbf"
-			  "\x00\x8a\xa2\xd5\xb7\xf3\x60\xcf"
-			  "\xa0\x47\xdf\x4e\x09\xf4\xb1\x7f"
-			  "\x14\xd9\x3d\x53\x8e\x12\xb3\x00"
-			  "\x4c\x0a\x4e\x32\x40\x43\x88\xce"
-			  "\x92\x26\xc1\x76\x20\x11\xeb\xba"
-			  "\x62\x4f\x9a\x62\x25\xc3\x75\x80"
-			  "\xb7\x0a\x17\xf5\xd7\x94\xb4\x14",
-		.rlen   = 80 + 64,
-       }, { /* NIST SP800-38A F.2.3 CBC-AES192.Encrypt */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"            /* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x18"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
-			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
-			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
-			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
-			  "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
-			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
-			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
-		.klen   = 8 + 64 + 24,
-		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.alen   = 16,
-		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen   = 64,
-		.result = "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d"
-			  "\x71\x78\x18\x3a\x9f\xa0\x71\xe8"
-			  "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4"
-			  "\xe5\xe7\x38\x76\x3f\x69\x14\x5a"
-			  "\x57\x1b\x24\x20\x12\xfb\x7a\xe0"
-			  "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0"
-			  "\x08\xb0\xe2\x79\x88\x59\x88\x81"
-			  "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd"
-			  "\x77\x4b\x69\x9d\x3a\x0d\xb4\x99"
-			  "\x8f\xc6\x8e\x0e\x72\x58\xe3\x56"
-			  "\xbb\x21\xd2\x7d\x93\x11\x17\x91"
-			  "\xc4\x83\xfd\x0a\xea\x71\xfe\x77"
-			  "\xae\x6f\x0a\xa5\xf0\xcf\xe1\x35"
-			  "\xba\x03\xd5\x32\xfa\x5f\x41\x58"
-			  "\x8d\x43\x98\xa7\x94\x16\x07\x02"
-			  "\x0f\xb6\x81\x50\x28\x95\x2e\x75",
-		.rlen   = 64 + 64,
-	}, { /* NIST SP800-38A F.2.5 CBC-AES256.Encrypt */
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
-#endif
-			  "\x00\x00\x00\x20"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
-			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
-			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
-			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
-			  "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
-			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
-			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
-			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
-		.klen   = 8 + 64 + 32,
-		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.alen   = 16,
-		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen   = 64,
-		.result = "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba"
-			  "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6"
-			  "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d"
-			  "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d"
-			  "\x39\xf2\x33\x69\xa9\xd9\xba\xcf"
-			  "\xa5\x30\xe2\x63\x04\x23\x14\x61"
-			  "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc"
-			  "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b"
-			  "\xb2\x27\x69\x7f\x45\x64\x79\x2b"
-			  "\xb7\xb8\x4c\xd4\x75\x94\x68\x40"
-			  "\x2a\xea\x91\xc7\x3f\x7c\xed\x7b"
-			  "\x95\x2c\x9b\xa8\xf5\xe5\x52\x8d"
-			  "\x6b\xe1\xae\xf1\x74\xfa\x0d\x0c"
-			  "\xe3\x8d\x64\xc3\x8d\xff\x7c\x8c"
-			  "\xdb\xbf\xa0\xb4\x01\xa2\xa8\xa2"
-			  "\x2c\xb1\x62\x2c\x10\xca\xf1\x21",
-		.rlen   = 64 + 64,
-	},
-};
-
-static const struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = {
-	{ /*Generated with cryptopp*/
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-	.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x08"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-		  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55"
-			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24",
-		.klen	= 8 + 20 + 8,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.alen   = 16,
-		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.ilen	= 128,
-		.result	= "\x70\xd6\xde\x64\x87\x17\xf1\xe8"
-			  "\x54\x31\x85\x37\xed\x6b\x01\x8d"
-			  "\xe3\xcc\xe0\x1d\x5e\xf3\xfe\xf1"
-			  "\x41\xaa\x33\x91\xa7\x7d\x99\x88"
-			  "\x4d\x85\x6e\x2f\xa3\x69\xf5\x82"
-			  "\x3a\x6f\x25\xcb\x7d\x58\x1f\x9b"
-			  "\xaa\x9c\x11\xd5\x76\x67\xce\xde"
-			  "\x56\xd7\x5a\x80\x69\xea\x3a\x02"
-			  "\xf0\xc7\x7c\xe3\xcb\x40\xe5\x52"
-			  "\xd1\x10\x92\x78\x0b\x8e\x5b\xf1"
-			  "\xe3\x26\x1f\xe1\x15\x41\xc7\xba"
-			  "\x99\xdb\x08\x51\x1c\xd3\x01\xf4"
-			  "\x87\x47\x39\xb8\xd2\xdd\xbd\xfb"
-			  "\x66\x13\xdf\x1c\x01\x44\xf0\x7a"
-			  "\x1a\x6b\x13\xf5\xd5\x0b\xb8\xba"
-			  "\x53\xba\xe1\x76\xe3\x82\x07\x86"
-			  "\x95\x16\x20\x09\xf5\x95\x19\xfd"
-			  "\x3c\xc7\xe0\x42\xc0\x14\x69\xfa"
-			  "\x5c\x44\xa9\x37",
-			  .rlen	= 128 + 20,
-	},
-};
-
-static const struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = {
-	{ /*Generated with cryptopp*/
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x08"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-		  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24",
-		.klen	= 8 + 24 + 8,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.alen   = 16,
-		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.ilen	= 128,
-		.result	= "\x70\xd6\xde\x64\x87\x17\xf1\xe8"
-			  "\x54\x31\x85\x37\xed\x6b\x01\x8d"
-			  "\xe3\xcc\xe0\x1d\x5e\xf3\xfe\xf1"
-			  "\x41\xaa\x33\x91\xa7\x7d\x99\x88"
-			  "\x4d\x85\x6e\x2f\xa3\x69\xf5\x82"
-			  "\x3a\x6f\x25\xcb\x7d\x58\x1f\x9b"
-			  "\xaa\x9c\x11\xd5\x76\x67\xce\xde"
-			  "\x56\xd7\x5a\x80\x69\xea\x3a\x02"
-			  "\xf0\xc7\x7c\xe3\xcb\x40\xe5\x52"
-			  "\xd1\x10\x92\x78\x0b\x8e\x5b\xf1"
-			  "\xe3\x26\x1f\xe1\x15\x41\xc7\xba"
-			  "\x99\xdb\x08\x51\x1c\xd3\x01\xf4"
-		  "\x87\x47\x39\xb8\xd2\xdd\xbd\xfb"
-			  "\x66\x13\xdf\x1c\x01\x44\xf0\x7a"
-			  "\x1a\x6b\x13\xf5\xd5\x0b\xb8\xba"
-			  "\x53\xba\xe1\x76\xe3\x82\x07\x86"
-			  "\x9c\x2d\x7e\xee\x20\x34\x55\x0a"
-			  "\xce\xb5\x4e\x64\x53\xe7\xbf\x91"
-			  "\xab\xd4\xd9\xda\xc9\x12\xae\xf7",
-		.rlen	= 128 + 24,
-	},
-};
-
-static const struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = {
-	{ /*Generated with cryptopp*/
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x08"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24",
-		.klen	= 8 + 32 + 8,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.alen   = 16,
-		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.ilen	= 128,
-		.result	= "\x70\xd6\xde\x64\x87\x17\xf1\xe8"
-			  "\x54\x31\x85\x37\xed\x6b\x01\x8d"
-			  "\xe3\xcc\xe0\x1d\x5e\xf3\xfe\xf1"
-			  "\x41\xaa\x33\x91\xa7\x7d\x99\x88"
-			  "\x4d\x85\x6e\x2f\xa3\x69\xf5\x82"
-			  "\x3a\x6f\x25\xcb\x7d\x58\x1f\x9b"
-			  "\xaa\x9c\x11\xd5\x76\x67\xce\xde"
-			  "\x56\xd7\x5a\x80\x69\xea\x3a\x02"
-			  "\xf0\xc7\x7c\xe3\xcb\x40\xe5\x52"
-			  "\xd1\x10\x92\x78\x0b\x8e\x5b\xf1"
-			  "\xe3\x26\x1f\xe1\x15\x41\xc7\xba"
-		  "\x99\xdb\x08\x51\x1c\xd3\x01\xf4"
-			  "\x87\x47\x39\xb8\xd2\xdd\xbd\xfb"
-		  "\x66\x13\xdf\x1c\x01\x44\xf0\x7a"
-		  "\x1a\x6b\x13\xf5\xd5\x0b\xb8\xba"
-			  "\x53\xba\xe1\x76\xe3\x82\x07\x86"
-			  "\xc6\x58\xa1\x60\x70\x91\x39\x36"
-			  "\x50\xf6\x5d\xab\x4b\x51\x4e\x5e"
-			  "\xde\x63\xde\x76\x52\xde\x9f\xba"
-			  "\x90\xcf\x15\xf2\xbb\x6e\x84\x00",
-		.rlen	= 128 + 32,
-	},
-};
-
-static const struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = {
-	{ /*Generated with cryptopp*/
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x08"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
-			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
-			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24",
-		.klen	= 8 + 48 + 8,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.alen   = 16,
-		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.ilen	= 128,
-		.result	= "\x70\xd6\xde\x64\x87\x17\xf1\xe8"
-			  "\x54\x31\x85\x37\xed\x6b\x01\x8d"
-			  "\xe3\xcc\xe0\x1d\x5e\xf3\xfe\xf1"
-			  "\x41\xaa\x33\x91\xa7\x7d\x99\x88"
-			  "\x4d\x85\x6e\x2f\xa3\x69\xf5\x82"
-			  "\x3a\x6f\x25\xcb\x7d\x58\x1f\x9b"
-			  "\xaa\x9c\x11\xd5\x76\x67\xce\xde"
-			  "\x56\xd7\x5a\x80\x69\xea\x3a\x02"
-			  "\xf0\xc7\x7c\xe3\xcb\x40\xe5\x52"
-			  "\xd1\x10\x92\x78\x0b\x8e\x5b\xf1"
-			  "\xe3\x26\x1f\xe1\x15\x41\xc7\xba"
-			  "\x99\xdb\x08\x51\x1c\xd3\x01\xf4"
-			  "\x87\x47\x39\xb8\xd2\xdd\xbd\xfb"
-			  "\x66\x13\xdf\x1c\x01\x44\xf0\x7a"
-			  "\x1a\x6b\x13\xf5\xd5\x0b\xb8\xba"
-			  "\x53\xba\xe1\x76\xe3\x82\x07\x86"
-			  "\xa8\x8e\x9c\x74\x8c\x2b\x99\xa0"
-			  "\xc8\x8c\xef\x25\x07\x83\x11\x3a"
-			  "\x31\x8d\xbe\x3b\x6a\xd7\x96\xfe"
-			  "\x5e\x67\xb5\x74\xe7\xe7\x85\x61"
-			  "\x6a\x95\x26\x75\xcc\x53\x89\xf3"
-			  "\x74\xc9\x2a\x76\x20\xa2\x64\x62",
-		.rlen	= 128 + 48,
-	},
-};
-
-static const struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = {
-	{ /*Generated with cryptopp*/
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-		  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x08"	/* enc key length */
-		  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
-			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
-			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
-			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
-			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24",
-		.klen	= 8 + 64 + 8,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.alen   = 16,
-		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.ilen	= 128,
-		.result	= "\x70\xd6\xde\x64\x87\x17\xf1\xe8"
-			  "\x54\x31\x85\x37\xed\x6b\x01\x8d"
-			  "\xe3\xcc\xe0\x1d\x5e\xf3\xfe\xf1"
-			  "\x41\xaa\x33\x91\xa7\x7d\x99\x88"
-			  "\x4d\x85\x6e\x2f\xa3\x69\xf5\x82"
-			  "\x3a\x6f\x25\xcb\x7d\x58\x1f\x9b"
-			  "\xaa\x9c\x11\xd5\x76\x67\xce\xde"
-		  "\x56\xd7\x5a\x80\x69\xea\x3a\x02"
-			  "\xf0\xc7\x7c\xe3\xcb\x40\xe5\x52"
-		  "\xd1\x10\x92\x78\x0b\x8e\x5b\xf1"
-			  "\xe3\x26\x1f\xe1\x15\x41\xc7\xba"
-			  "\x99\xdb\x08\x51\x1c\xd3\x01\xf4"
-			  "\x87\x47\x39\xb8\xd2\xdd\xbd\xfb"
-			  "\x66\x13\xdf\x1c\x01\x44\xf0\x7a"
-		  "\x1a\x6b\x13\xf5\xd5\x0b\xb8\xba"
-			  "\x53\xba\xe1\x76\xe3\x82\x07\x86"
-			  "\xc6\x2c\x73\x88\xb0\x9d\x5f\x3e"
-			  "\x5b\x78\xca\x0e\xab\x8a\xa3\xbb"
-			  "\xd9\x1d\xc3\xe3\x05\xac\x76\xfb"
-			  "\x58\x83\xda\x67\xfb\x21\x24\xa2"
-			  "\xb1\xa7\xd7\x66\xa6\x8d\xa6\x93"
-			  "\x97\xe2\xe3\xb8\xaa\x48\x85\xee"
-			  "\x8c\xf6\x07\x95\x1f\xa6\x6c\x96"
-			  "\x99\xc7\x5c\x8d\xd8\xb5\x68\x7b",
-		.rlen	= 128 + 64,
-	},
-};
-
-static const struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = {
-	{ /*Generated with cryptopp*/
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x18"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55"
-		  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
-			  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
-			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
-		.klen	= 8 + 20 + 24,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.alen   = 16,
-		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-		  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.ilen	= 128,
-		.result	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
-			  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
-		  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
-		  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
-		  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
-			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
-			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
-			  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
-		  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
-			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
-			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
-			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
-			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
-			  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
-			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
-			  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19"
-			  "\x67\x6d\xb1\xf5\xb8\x10\xdc\xc6"
-			  "\x75\x86\x96\x6b\xb1\xc5\xe4\xcf"
-			  "\xd1\x60\x91\xb3",
-			  .rlen	= 128 + 20,
-	},
-};
-
-static const struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = {
-	{ /*Generated with cryptopp*/
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x18"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
-			  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
-			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
-		.klen	= 8 + 24 + 24,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.alen   = 16,
-		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-		  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.ilen	= 128,
-		.result	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
-		  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
-			  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
-			  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
-			  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
-			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
-			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
-		  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
-			  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
-			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
-			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
-			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
-			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
-		  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
-			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
-		  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19"
-			  "\x15\x24\x7f\x5a\x45\x4a\x66\xce"
-			  "\x2b\x0b\x93\x99\x2f\x9d\x0c\x6c"
-			  "\x56\x1f\xe1\xa6\x41\xb2\x4c\xd0",
-			  .rlen	= 128 + 24,
-	},
-};
-
-static const struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = {
-	{ /*Generated with cryptopp*/
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x18"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
-			  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
-			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
-		.klen	= 8 + 32 + 24,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.alen   = 16,
-		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.ilen	= 128,
-		.result	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
-			  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
-			  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
-			  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
-			  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
-			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
-			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
-			  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
-			  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
-			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
-			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
-			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
-			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
-			  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
-			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
-			  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19"
-			  "\x73\xb0\xea\x9f\xe8\x18\x80\xd6"
-			  "\x56\x38\x44\xc0\xdb\xe3\x4f\x71"
-			  "\xf7\xce\xd1\xd3\xf8\xbd\x3e\x4f"
-			  "\xca\x43\x95\xdf\x80\x61\x81\xa9",
-		.rlen	= 128 + 32,
-	},
-};
-
-static const struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = {
-	{ /*Generated with cryptopp*/
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x18"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
-			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
-			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
-			  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
-			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
-		.klen	= 8 + 48 + 24,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.alen   = 16,
-		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.ilen	= 128,
-		.result	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
-			  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
-			  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
-			  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
-			  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
-			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
-			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
-			  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
-			  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
-			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
-			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
-			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
-			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
-			  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
-			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
-			  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19"
-			  "\x6d\x77\xfc\x80\x9d\x8a\x9c\xb7"
-		  "\x70\xe7\x93\xbf\x73\xe6\x9f\x83"
-			  "\x99\x62\x23\xe6\x5b\xd0\xda\x18"
-			  "\xa4\x32\x8a\x0b\x46\xd7\xf0\x39"
-			  "\x36\x5d\x13\x2f\x86\x10\x78\xd6"
-			  "\xd6\xbe\x5c\xb9\x15\x89\xf9\x1b",
-		.rlen	= 128 + 48,
-	},
-};
-
-static const struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = {
-	{ /*Generated with cryptopp*/
-#ifdef __LITTLE_ENDIAN
-		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"		/* rta type */
-#else
-		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"		/* rta type */
-#endif
-			  "\x00\x00\x00\x18"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
-			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
-			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
-			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
-			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
-		  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
-			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
-		.klen	= 8 + 64 + 24,
-		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
-			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
-		.alen   = 16,
-		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
-			  "\x53\x20\x63\x65\x65\x72\x73\x74"
-			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
-			  "\x20\x79\x65\x53\x72\x63\x74\x65"
-			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
-			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
-			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
-			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
-		  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
-			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
-			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
-			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
-			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
-			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
-			  "\x63\x65\x65\x72\x73\x74\x54\x20"
-			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
-		.ilen	= 128,
-		.result	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
-			  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
-			  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
-			  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
-			  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
-			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
-			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
-			  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
-			  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
-			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
-			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
-			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
-			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
-			  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
-			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
-			  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19"
-			  "\x41\xb5\x1f\xbb\xbd\x4e\xb8\x32"
-			  "\x22\x86\x4e\x57\x1b\x2a\xd8\x6e"
-			  "\xa9\xfb\xc8\xf3\xbf\x2d\xae\x2b"
-			  "\x3b\xbc\x41\xe8\x38\xbb\xf1\x60"
-			  "\x4c\x68\xa9\x4e\x8c\x73\xa7\xc0"
-			  "\x2a\x74\xd4\x65\x12\xcb\x55\xf2"
-			  "\xd5\x02\x6d\xe6\xaf\xc9\x2f\xf2"
-			  "\x57\xaa\x85\xf7\xf3\x6a\xcb\xdb",
-		.rlen	= 128 + 64,
-	},
-};
-
-static const struct cipher_testvec aes_lrw_enc_tv_template[] = {
-	/* from http://grouper.ieee.org/groups/1619/email/pdf00017.pdf */
-	{ /* LRW-32-AES 1 */
-		.key    = "\x45\x62\xac\x25\xf8\x28\x17\x6d"
-			  "\x4c\x26\x84\x14\xb5\x68\x01\x85"
-			  "\x25\x8e\x2a\x05\xe7\x3e\x9d\x03"
-			  "\xee\x5a\x83\x0c\xcc\x09\x4c\x87",
-		.klen   = 32,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input  = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen   = 16,
-		.result = "\xf1\xb2\x73\xcd\x65\xa3\xdf\x5f"
-			  "\xe9\x5d\x48\x92\x54\x63\x4e\xb8",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 2 */
-		.key    = "\x59\x70\x47\x14\xf5\x57\x47\x8c"
-			  "\xd7\x79\xe8\x0f\x54\x88\x79\x44"
-			  "\x0d\x48\xf0\xb7\xb1\x5a\x53\xea"
-			  "\x1c\xaa\x6b\x29\xc2\xca\xfb\xaf",
-		.klen   = 32,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x02",
-		.input  = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen   = 16,
-		.result = "\x00\xc8\x2b\xae\x95\xbb\xcd\xe5"
-			  "\x27\x4f\x07\x69\xb2\x60\xe1\x36",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 3 */
-		.key    = "\xd8\x2a\x91\x34\xb2\x6a\x56\x50"
-			  "\x30\xfe\x69\xe2\x37\x7f\x98\x47"
-			  "\xcd\xf9\x0b\x16\x0c\x64\x8f\xb6"
-			  "\xb0\x0d\x0d\x1b\xae\x85\x87\x1f",
-		.klen   = 32,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input  = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen   = 16,
-		.result = "\x76\x32\x21\x83\xed\x8f\xf1\x82"
-			  "\xf9\x59\x62\x03\x69\x0e\x5e\x01",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 4 */
-		.key    = "\x0f\x6a\xef\xf8\xd3\xd2\xbb\x15"
-			  "\x25\x83\xf7\x3c\x1f\x01\x28\x74"
-			  "\xca\xc6\xbc\x35\x4d\x4a\x65\x54"
-			  "\x90\xae\x61\xcf\x7b\xae\xbd\xcc"
-			  "\xad\xe4\x94\xc5\x4a\x29\xae\x70",
-		.klen   = 40,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input  = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen   = 16,
-		.result = "\x9c\x0f\x15\x2f\x55\xa2\xd8\xf0"
-			  "\xd6\x7b\x8f\x9e\x28\x22\xbc\x41",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 5 */
-		.key    = "\x8a\xd4\xee\x10\x2f\xbd\x81\xff"
-			  "\xf8\x86\xce\xac\x93\xc5\xad\xc6"
-			  "\xa0\x19\x07\xc0\x9d\xf7\xbb\xdd"
-			  "\x52\x13\xb2\xb7\xf0\xff\x11\xd8"
-			  "\xd6\x08\xd0\xcd\x2e\xb1\x17\x6f",
-		.klen   = 40,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input  = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen   = 16,
-		.result = "\xd4\x27\x6a\x7f\x14\x91\x3d\x65"
-			  "\xc8\x60\x48\x02\x87\xe3\x34\x06",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 6 */
-		.key    = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen   = 48,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input  = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen   = 16,
-		.result = "\xbd\x06\xb8\xe1\xdb\x98\x89\x9e"
-			  "\xc4\x98\xe4\x91\xcf\x1c\x70\x2b",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 7 */
-		.key    = "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
-			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
-			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
-			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
-			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
-			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
-		.klen   = 48,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input  = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen   = 16,
-		.result = "\x5b\x90\x8e\xc1\xab\xdd\x67\x5f"
-			  "\x3d\x69\x8a\x95\x53\xc8\x9c\xe5",
-		.rlen   = 16,
-	}, {
-/* http://www.mail-archive.com/stds-p1619@listserv.ieee.org/msg00173.html */
-		.key    = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen   = 48,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input  = "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
-			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
-			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
-			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
-			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
-			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
-			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
-			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
-			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
-			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
-			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
-			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
-			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
-			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
-			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
-			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
-			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
-			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
-			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
-			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
-			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
-			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
-			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
-			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
-			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
-			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
-			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
-			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
-			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
-			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
-			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
-			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
-			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
-			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
-			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
-			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
-			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
-			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
-			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
-			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
-			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
-			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
-			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
-			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
-			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
-			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
-			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
-			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
-			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
-			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
-			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
-			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
-			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
-			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
-			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
-			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
-			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
-			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
-			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
-			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
-			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
-			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
-			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
-			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
-		.ilen   = 512,
-		.result = "\x1a\x1d\xa9\x30\xad\xf9\x2f\x9b"
-			  "\xb6\x1d\xae\xef\xf0\x2f\xf8\x5a"
-			  "\x39\x3c\xbf\x2a\xb2\x45\xb2\x23"
-			  "\x1b\x63\x3c\xcf\xaa\xbe\xcf\x4e"
-			  "\xfa\xe8\x29\xc2\x20\x68\x2b\x3c"
-			  "\x2e\x8b\xf7\x6e\x25\xbd\xe3\x3d"
-			  "\x66\x27\xd6\xaf\xd6\x64\x3e\xe3"
-			  "\xe8\x58\x46\x97\x39\x51\x07\xde"
-			  "\xcb\x37\xbc\xa9\xc0\x5f\x75\xc3"
-			  "\x0e\x84\x23\x1d\x16\xd4\x1c\x59"
-			  "\x9c\x1a\x02\x55\xab\x3a\x97\x1d"
-			  "\xdf\xdd\xc7\x06\x51\xd7\x70\xae"
-			  "\x23\xc6\x8c\xf5\x1e\xa0\xe5\x82"
-			  "\xb8\xb2\xbf\x04\xa0\x32\x8e\x68"
-			  "\xeb\xaf\x6e\x2d\x94\x22\x2f\xce"
-			  "\x4c\xb5\x59\xe2\xa2\x2f\xa0\x98"
-			  "\x1a\x97\xc6\xd4\xb5\x00\x59\xf2"
-			  "\x84\x14\x72\xb1\x9a\x6e\xa3\x7f"
-			  "\xea\x20\xe7\xcb\x65\x77\x3a\xdf"
-			  "\xc8\x97\x67\x15\xc2\x2a\x27\xcc"
-			  "\x18\x55\xa1\x24\x0b\x24\x24\xaf"
-			  "\x5b\xec\x68\xb8\xc8\xf5\xba\x63"
-			  "\xff\xed\x89\xce\xd5\x3d\x88\xf3"
-			  "\x25\xef\x05\x7c\x3a\xef\xeb\xd8"
-			  "\x7a\x32\x0d\xd1\x1e\x58\x59\x99"
-			  "\x90\x25\xb5\x26\xb0\xe3\x2b\x6c"
-			  "\x4c\xa9\x8b\x84\x4f\x5e\x01\x50"
-			  "\x41\x30\x58\xc5\x62\x74\x52\x1d"
-			  "\x45\x24\x6a\x42\x64\x4f\x97\x1c"
-			  "\xa8\x66\xb5\x6d\x79\xd4\x0d\x48"
-			  "\xc5\x5f\xf3\x90\x32\xdd\xdd\xe1"
-			  "\xe4\xa9\x9f\xfc\xc3\x52\x5a\x46"
-			  "\xe4\x81\x84\x95\x36\x59\x7a\x6b"
-			  "\xaa\xb3\x60\xad\xce\x9f\x9f\x28"
-			  "\xe0\x01\x75\x22\xc4\x4e\xa9\x62"
-			  "\x5c\x62\x0d\x00\xcb\x13\xe8\x43"
-			  "\x72\xd4\x2d\x53\x46\xb5\xd1\x16"
-			  "\x22\x18\xdf\x34\x33\xf5\xd6\x1c"
-			  "\xb8\x79\x78\x97\x94\xff\x72\x13"
-			  "\x4c\x27\xfc\xcb\xbf\x01\x53\xa6"
-			  "\xb4\x50\x6e\xde\xdf\xb5\x43\xa4"
-			  "\x59\xdf\x52\xf9\x7c\xe0\x11\x6f"
-			  "\x2d\x14\x8e\x24\x61\x2c\xe1\x17"
-			  "\xcc\xce\x51\x0c\x19\x8a\x82\x30"
-			  "\x94\xd5\x3d\x6a\x53\x06\x5e\xbd"
-			  "\xb7\xeb\xfa\xfd\x27\x51\xde\x85"
-			  "\x1e\x86\x53\x11\x53\x94\x00\xee"
-			  "\x2b\x8c\x08\x2a\xbf\xdd\xae\x11"
-			  "\xcb\x1e\xa2\x07\x9a\x80\xcf\x62"
-			  "\x9b\x09\xdc\x95\x3c\x96\x8e\xb1"
-			  "\x09\xbd\xe4\xeb\xdb\xca\x70\x7a"
-			  "\x9e\xfa\x31\x18\x45\x3c\x21\x33"
-			  "\xb0\xb3\x2b\xea\xf3\x71\x2d\xe1"
-			  "\x03\xad\x1b\x48\xd4\x67\x27\xf0"
-			  "\x62\xe4\x3d\xfb\x9b\x08\x76\xe7"
-			  "\xdd\x2b\x01\x39\x04\x5a\x58\x7a"
-			  "\xf7\x11\x90\xec\xbd\x51\x5c\x32"
-			  "\x6b\xd7\x35\x39\x02\x6b\xf2\xa6"
-			  "\xd0\x0d\x07\xe1\x06\xc4\x5b\x7d"
-			  "\xe4\x6a\xd7\xee\x15\x1f\x83\xb4"
-			  "\xa3\xa7\x5e\xc3\x90\xb7\xef\xd3"
-			  "\xb7\x4f\xf8\x92\x4c\xb7\x3c\x29"
-			  "\xcd\x7e\x2b\x5d\x43\xea\x42\xe7"
-			  "\x74\x3f\x7d\x58\x88\x75\xde\x3e",
-		.rlen   = 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	}
-};
-
-static const struct cipher_testvec aes_lrw_dec_tv_template[] = {
-	/* from http://grouper.ieee.org/groups/1619/email/pdf00017.pdf */
-	/* same as enc vectors with input and result reversed */
-	{ /* LRW-32-AES 1 */
-		.key    = "\x45\x62\xac\x25\xf8\x28\x17\x6d"
-			  "\x4c\x26\x84\x14\xb5\x68\x01\x85"
-			  "\x25\x8e\x2a\x05\xe7\x3e\x9d\x03"
-			  "\xee\x5a\x83\x0c\xcc\x09\x4c\x87",
-		.klen   = 32,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input  = "\xf1\xb2\x73\xcd\x65\xa3\xdf\x5f"
-			  "\xe9\x5d\x48\x92\x54\x63\x4e\xb8",
-		.ilen   = 16,
-		.result = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 2 */
-		.key    = "\x59\x70\x47\x14\xf5\x57\x47\x8c"
-			  "\xd7\x79\xe8\x0f\x54\x88\x79\x44"
-			  "\x0d\x48\xf0\xb7\xb1\x5a\x53\xea"
-			  "\x1c\xaa\x6b\x29\xc2\xca\xfb\xaf",
-		.klen   = 32,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x02",
-		.input  = "\x00\xc8\x2b\xae\x95\xbb\xcd\xe5"
-			  "\x27\x4f\x07\x69\xb2\x60\xe1\x36",
-		.ilen   = 16,
-		.result  = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			   "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 3 */
-		.key    = "\xd8\x2a\x91\x34\xb2\x6a\x56\x50"
-			  "\x30\xfe\x69\xe2\x37\x7f\x98\x47"
-			  "\xcd\xf9\x0b\x16\x0c\x64\x8f\xb6"
-			  "\xb0\x0d\x0d\x1b\xae\x85\x87\x1f",
-		.klen   = 32,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input  = "\x76\x32\x21\x83\xed\x8f\xf1\x82"
-			  "\xf9\x59\x62\x03\x69\x0e\x5e\x01",
-		.ilen   = 16,
-		.result = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 4 */
-		.key    = "\x0f\x6a\xef\xf8\xd3\xd2\xbb\x15"
-			  "\x25\x83\xf7\x3c\x1f\x01\x28\x74"
-			  "\xca\xc6\xbc\x35\x4d\x4a\x65\x54"
-			  "\x90\xae\x61\xcf\x7b\xae\xbd\xcc"
-			  "\xad\xe4\x94\xc5\x4a\x29\xae\x70",
-		.klen   = 40,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input  = "\x9c\x0f\x15\x2f\x55\xa2\xd8\xf0"
-			  "\xd6\x7b\x8f\x9e\x28\x22\xbc\x41",
-		.ilen   = 16,
-		.result = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 5 */
-		.key    = "\x8a\xd4\xee\x10\x2f\xbd\x81\xff"
-			  "\xf8\x86\xce\xac\x93\xc5\xad\xc6"
-			  "\xa0\x19\x07\xc0\x9d\xf7\xbb\xdd"
-			  "\x52\x13\xb2\xb7\xf0\xff\x11\xd8"
-			  "\xd6\x08\xd0\xcd\x2e\xb1\x17\x6f",
-		.klen   = 40,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input  = "\xd4\x27\x6a\x7f\x14\x91\x3d\x65"
-			  "\xc8\x60\x48\x02\x87\xe3\x34\x06",
-		.ilen   = 16,
-		.result = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 6 */
-		.key    = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen   = 48,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input  = "\xbd\x06\xb8\xe1\xdb\x98\x89\x9e"
-			  "\xc4\x98\xe4\x91\xcf\x1c\x70\x2b",
-		.ilen   = 16,
-		.result = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen   = 16,
-	}, { /* LRW-32-AES 7 */
-		.key    = "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
-			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
-			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
-			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
-			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
-			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
-		.klen   = 48,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input  = "\x5b\x90\x8e\xc1\xab\xdd\x67\x5f"
-			  "\x3d\x69\x8a\x95\x53\xc8\x9c\xe5",
-		.ilen   = 16,
-		.result = "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen   = 16,
-	}, {
-/* http://www.mail-archive.com/stds-p1619@listserv.ieee.org/msg00173.html */
-		.key    = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen   = 48,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x1a\x1d\xa9\x30\xad\xf9\x2f\x9b"
-			  "\xb6\x1d\xae\xef\xf0\x2f\xf8\x5a"
-			  "\x39\x3c\xbf\x2a\xb2\x45\xb2\x23"
-			  "\x1b\x63\x3c\xcf\xaa\xbe\xcf\x4e"
-			  "\xfa\xe8\x29\xc2\x20\x68\x2b\x3c"
-			  "\x2e\x8b\xf7\x6e\x25\xbd\xe3\x3d"
-			  "\x66\x27\xd6\xaf\xd6\x64\x3e\xe3"
-			  "\xe8\x58\x46\x97\x39\x51\x07\xde"
-			  "\xcb\x37\xbc\xa9\xc0\x5f\x75\xc3"
-			  "\x0e\x84\x23\x1d\x16\xd4\x1c\x59"
-			  "\x9c\x1a\x02\x55\xab\x3a\x97\x1d"
-			  "\xdf\xdd\xc7\x06\x51\xd7\x70\xae"
-			  "\x23\xc6\x8c\xf5\x1e\xa0\xe5\x82"
-			  "\xb8\xb2\xbf\x04\xa0\x32\x8e\x68"
-			  "\xeb\xaf\x6e\x2d\x94\x22\x2f\xce"
-			  "\x4c\xb5\x59\xe2\xa2\x2f\xa0\x98"
-			  "\x1a\x97\xc6\xd4\xb5\x00\x59\xf2"
-			  "\x84\x14\x72\xb1\x9a\x6e\xa3\x7f"
-			  "\xea\x20\xe7\xcb\x65\x77\x3a\xdf"
-			  "\xc8\x97\x67\x15\xc2\x2a\x27\xcc"
-			  "\x18\x55\xa1\x24\x0b\x24\x24\xaf"
-			  "\x5b\xec\x68\xb8\xc8\xf5\xba\x63"
-			  "\xff\xed\x89\xce\xd5\x3d\x88\xf3"
-			  "\x25\xef\x05\x7c\x3a\xef\xeb\xd8"
-			  "\x7a\x32\x0d\xd1\x1e\x58\x59\x99"
-			  "\x90\x25\xb5\x26\xb0\xe3\x2b\x6c"
-			  "\x4c\xa9\x8b\x84\x4f\x5e\x01\x50"
-			  "\x41\x30\x58\xc5\x62\x74\x52\x1d"
-			  "\x45\x24\x6a\x42\x64\x4f\x97\x1c"
-			  "\xa8\x66\xb5\x6d\x79\xd4\x0d\x48"
-			  "\xc5\x5f\xf3\x90\x32\xdd\xdd\xe1"
-			  "\xe4\xa9\x9f\xfc\xc3\x52\x5a\x46"
-			  "\xe4\x81\x84\x95\x36\x59\x7a\x6b"
-			  "\xaa\xb3\x60\xad\xce\x9f\x9f\x28"
-			  "\xe0\x01\x75\x22\xc4\x4e\xa9\x62"
-			  "\x5c\x62\x0d\x00\xcb\x13\xe8\x43"
-			  "\x72\xd4\x2d\x53\x46\xb5\xd1\x16"
-			  "\x22\x18\xdf\x34\x33\xf5\xd6\x1c"
-			  "\xb8\x79\x78\x97\x94\xff\x72\x13"
-			  "\x4c\x27\xfc\xcb\xbf\x01\x53\xa6"
-			  "\xb4\x50\x6e\xde\xdf\xb5\x43\xa4"
-			  "\x59\xdf\x52\xf9\x7c\xe0\x11\x6f"
-			  "\x2d\x14\x8e\x24\x61\x2c\xe1\x17"
-			  "\xcc\xce\x51\x0c\x19\x8a\x82\x30"
-			  "\x94\xd5\x3d\x6a\x53\x06\x5e\xbd"
-			  "\xb7\xeb\xfa\xfd\x27\x51\xde\x85"
-			  "\x1e\x86\x53\x11\x53\x94\x00\xee"
-			  "\x2b\x8c\x08\x2a\xbf\xdd\xae\x11"
-			  "\xcb\x1e\xa2\x07\x9a\x80\xcf\x62"
-			  "\x9b\x09\xdc\x95\x3c\x96\x8e\xb1"
-			  "\x09\xbd\xe4\xeb\xdb\xca\x70\x7a"
-			  "\x9e\xfa\x31\x18\x45\x3c\x21\x33"
-			  "\xb0\xb3\x2b\xea\xf3\x71\x2d\xe1"
-			  "\x03\xad\x1b\x48\xd4\x67\x27\xf0"
-			  "\x62\xe4\x3d\xfb\x9b\x08\x76\xe7"
-			  "\xdd\x2b\x01\x39\x04\x5a\x58\x7a"
-			  "\xf7\x11\x90\xec\xbd\x51\x5c\x32"
-			  "\x6b\xd7\x35\x39\x02\x6b\xf2\xa6"
-			  "\xd0\x0d\x07\xe1\x06\xc4\x5b\x7d"
-			  "\xe4\x6a\xd7\xee\x15\x1f\x83\xb4"
-			  "\xa3\xa7\x5e\xc3\x90\xb7\xef\xd3"
-			  "\xb7\x4f\xf8\x92\x4c\xb7\x3c\x29"
-			  "\xcd\x7e\x2b\x5d\x43\xea\x42\xe7"
-			  "\x74\x3f\x7d\x58\x88\x75\xde\x3e",
-		.ilen   = 512,
-		.result	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
-			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
-			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
-			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
-			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
-			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
-			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
-			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
-			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
-			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
-			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
-			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
-			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
-			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
-			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
-			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
-			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
-			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
-			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
-			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
-			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
-			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
-			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
-			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
-			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
-			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
-			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
-			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
-			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
-			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
-			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
-			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
-			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
-			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
-			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
-			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
-			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
-			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
-			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
-			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
-			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
-			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
-			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
-			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
-			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
-			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
-			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
-			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
-			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
-			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
-			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
-			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
-			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
-			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
-			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
-			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
-			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
-			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
-			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
-			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
-			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
-			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
-			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
-			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
-		.rlen   = 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	}
-};
-
-static const struct cipher_testvec aes_xts_enc_tv_template[] = {
-	/* http://grouper.ieee.org/groups/1619/email/pdf00086.pdf */
-	{ /* XTS-AES 1 */
-		.key    = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen   = 32,
-		.fips_skip = 1,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input  = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen   = 32,
-		.result = "\x91\x7c\xf6\x9e\xbd\x68\xb2\xec"
-			  "\x9b\x9f\xe9\xa3\xea\xdd\xa6\x92"
-			  "\xcd\x43\xd2\xf5\x95\x98\xed\x85"
-			  "\x8c\x02\xc2\x65\x2f\xbf\x92\x2e",
-		.rlen   = 32,
-	}, { /* XTS-AES 2 */
-		.key    = "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen   = 32,
-		.iv     = "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input  = "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen   = 32,
-		.result = "\xc4\x54\x18\x5e\x6a\x16\x93\x6e"
-			  "\x39\x33\x40\x38\xac\xef\x83\x8b"
-			  "\xfb\x18\x6f\xff\x74\x80\xad\xc4"
-			  "\x28\x93\x82\xec\xd6\xd3\x94\xf0",
-		.rlen   = 32,
-	}, { /* XTS-AES 3 */
-		.key    = "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen   = 32,
-		.iv     = "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input  = "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen   = 32,
-		.result = "\xaf\x85\x33\x6b\x59\x7a\xfc\x1a"
-			  "\x90\x0b\x2e\xb2\x1e\xc9\x49\xd2"
-			  "\x92\xdf\x4c\x04\x7e\x0b\x21\x53"
-			  "\x21\x86\xa5\x97\x1a\x22\x7a\x89",
-		.rlen   = 32,
-	}, { /* XTS-AES 4 */
-		.key    = "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95",
-		.klen   = 32,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input  = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen   = 512,
-		.result = "\x27\xa7\x47\x9b\xef\xa1\xd4\x76"
-			  "\x48\x9f\x30\x8c\xd4\xcf\xa6\xe2"
-			  "\xa9\x6e\x4b\xbe\x32\x08\xff\x25"
-			  "\x28\x7d\xd3\x81\x96\x16\xe8\x9c"
-			  "\xc7\x8c\xf7\xf5\xe5\x43\x44\x5f"
-			  "\x83\x33\xd8\xfa\x7f\x56\x00\x00"
-			  "\x05\x27\x9f\xa5\xd8\xb5\xe4\xad"
-			  "\x40\xe7\x36\xdd\xb4\xd3\x54\x12"
-			  "\x32\x80\x63\xfd\x2a\xab\x53\xe5"
-			  "\xea\x1e\x0a\x9f\x33\x25\x00\xa5"
-			  "\xdf\x94\x87\xd0\x7a\x5c\x92\xcc"
-			  "\x51\x2c\x88\x66\xc7\xe8\x60\xce"
-			  "\x93\xfd\xf1\x66\xa2\x49\x12\xb4"
-			  "\x22\x97\x61\x46\xae\x20\xce\x84"
-			  "\x6b\xb7\xdc\x9b\xa9\x4a\x76\x7a"
-			  "\xae\xf2\x0c\x0d\x61\xad\x02\x65"
-			  "\x5e\xa9\x2d\xc4\xc4\xe4\x1a\x89"
-			  "\x52\xc6\x51\xd3\x31\x74\xbe\x51"
-			  "\xa1\x0c\x42\x11\x10\xe6\xd8\x15"
-			  "\x88\xed\xe8\x21\x03\xa2\x52\xd8"
-			  "\xa7\x50\xe8\x76\x8d\xef\xff\xed"
-			  "\x91\x22\x81\x0a\xae\xb9\x9f\x91"
-			  "\x72\xaf\x82\xb6\x04\xdc\x4b\x8e"
-			  "\x51\xbc\xb0\x82\x35\xa6\xf4\x34"
-			  "\x13\x32\xe4\xca\x60\x48\x2a\x4b"
-			  "\xa1\xa0\x3b\x3e\x65\x00\x8f\xc5"
-			  "\xda\x76\xb7\x0b\xf1\x69\x0d\xb4"
-			  "\xea\xe2\x9c\x5f\x1b\xad\xd0\x3c"
-			  "\x5c\xcf\x2a\x55\xd7\x05\xdd\xcd"
-			  "\x86\xd4\x49\x51\x1c\xeb\x7e\xc3"
-			  "\x0b\xf1\x2b\x1f\xa3\x5b\x91\x3f"
-			  "\x9f\x74\x7a\x8a\xfd\x1b\x13\x0e"
-			  "\x94\xbf\xf9\x4e\xff\xd0\x1a\x91"
-			  "\x73\x5c\xa1\x72\x6a\xcd\x0b\x19"
-			  "\x7c\x4e\x5b\x03\x39\x36\x97\xe1"
-			  "\x26\x82\x6f\xb6\xbb\xde\x8e\xcc"
-			  "\x1e\x08\x29\x85\x16\xe2\xc9\xed"
-			  "\x03\xff\x3c\x1b\x78\x60\xf6\xde"
-			  "\x76\xd4\xce\xcd\x94\xc8\x11\x98"
-			  "\x55\xef\x52\x97\xca\x67\xe9\xf3"
-			  "\xe7\xff\x72\xb1\xe9\x97\x85\xca"
-			  "\x0a\x7e\x77\x20\xc5\xb3\x6d\xc6"
-			  "\xd7\x2c\xac\x95\x74\xc8\xcb\xbc"
-			  "\x2f\x80\x1e\x23\xe5\x6f\xd3\x44"
-			  "\xb0\x7f\x22\x15\x4b\xeb\xa0\xf0"
-			  "\x8c\xe8\x89\x1e\x64\x3e\xd9\x95"
-			  "\xc9\x4d\x9a\x69\xc9\xf1\xb5\xf4"
-			  "\x99\x02\x7a\x78\x57\x2a\xee\xbd"
-			  "\x74\xd2\x0c\xc3\x98\x81\xc2\x13"
-			  "\xee\x77\x0b\x10\x10\xe4\xbe\xa7"
-			  "\x18\x84\x69\x77\xae\x11\x9f\x7a"
-			  "\x02\x3a\xb5\x8c\xca\x0a\xd7\x52"
-			  "\xaf\xe6\x56\xbb\x3c\x17\x25\x6a"
-			  "\x9f\x6e\x9b\xf1\x9f\xdd\x5a\x38"
-			  "\xfc\x82\xbb\xe8\x72\xc5\x53\x9e"
-			  "\xdb\x60\x9e\xf4\xf7\x9c\x20\x3e"
-			  "\xbb\x14\x0f\x2e\x58\x3c\xb2\xad"
-			  "\x15\xb4\xaa\x5b\x65\x50\x16\xa8"
-			  "\x44\x92\x77\xdb\xd4\x77\xef\x2c"
-			  "\x8d\x6c\x01\x7d\xb7\x38\xb1\x8d"
-			  "\xeb\x4a\x42\x7d\x19\x23\xce\x3f"
-			  "\xf2\x62\x73\x57\x79\xa4\x18\xf2"
-			  "\x0a\x28\x2d\xf9\x20\x14\x7b\xea"
-			  "\xbe\x42\x1e\xe5\x31\x9d\x05\x68",
-		.rlen   = 512,
-	}, { /* XTS-AES 10, XTS-AES-256, data unit 512 bytes */
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\x1c\x3b\x3a\x10\x2f\x77\x03\x86"
-			  "\xe4\x83\x6c\x99\xe3\x70\xcf\x9b"
-			  "\xea\x00\x80\x3f\x5e\x48\x23\x57"
-			  "\xa4\xae\x12\xd4\x14\xa3\xe6\x3b"
-			  "\x5d\x31\xe2\x76\xf8\xfe\x4a\x8d"
-			  "\x66\xb3\x17\xf9\xac\x68\x3f\x44"
-			  "\x68\x0a\x86\xac\x35\xad\xfc\x33"
-			  "\x45\xbe\xfe\xcb\x4b\xb1\x88\xfd"
-			  "\x57\x76\x92\x6c\x49\xa3\x09\x5e"
-			  "\xb1\x08\xfd\x10\x98\xba\xec\x70"
-			  "\xaa\xa6\x69\x99\xa7\x2a\x82\xf2"
-			  "\x7d\x84\x8b\x21\xd4\xa7\x41\xb0"
-			  "\xc5\xcd\x4d\x5f\xff\x9d\xac\x89"
-			  "\xae\xba\x12\x29\x61\xd0\x3a\x75"
-			  "\x71\x23\xe9\x87\x0f\x8a\xcf\x10"
-			  "\x00\x02\x08\x87\x89\x14\x29\xca"
-			  "\x2a\x3e\x7a\x7d\x7d\xf7\xb1\x03"
-			  "\x55\x16\x5c\x8b\x9a\x6d\x0a\x7d"
-			  "\xe8\xb0\x62\xc4\x50\x0d\xc4\xcd"
-			  "\x12\x0c\x0f\x74\x18\xda\xe3\xd0"
-			  "\xb5\x78\x1c\x34\x80\x3f\xa7\x54"
-			  "\x21\xc7\x90\xdf\xe1\xde\x18\x34"
-			  "\xf2\x80\xd7\x66\x7b\x32\x7f\x6c"
-			  "\x8c\xd7\x55\x7e\x12\xac\x3a\x0f"
-			  "\x93\xec\x05\xc5\x2e\x04\x93\xef"
-			  "\x31\xa1\x2d\x3d\x92\x60\xf7\x9a"
-			  "\x28\x9d\x6a\x37\x9b\xc7\x0c\x50"
-			  "\x84\x14\x73\xd1\xa8\xcc\x81\xec"
-			  "\x58\x3e\x96\x45\xe0\x7b\x8d\x96"
-			  "\x70\x65\x5b\xa5\xbb\xcf\xec\xc6"
-			  "\xdc\x39\x66\x38\x0a\xd8\xfe\xcb"
-			  "\x17\xb6\xba\x02\x46\x9a\x02\x0a"
-			  "\x84\xe1\x8e\x8f\x84\x25\x20\x70"
-			  "\xc1\x3e\x9f\x1f\x28\x9b\xe5\x4f"
-			  "\xbc\x48\x14\x57\x77\x8f\x61\x60"
-			  "\x15\xe1\x32\x7a\x02\xb1\x40\xf1"
-			  "\x50\x5e\xb3\x09\x32\x6d\x68\x37"
-			  "\x8f\x83\x74\x59\x5c\x84\x9d\x84"
-			  "\xf4\xc3\x33\xec\x44\x23\x88\x51"
-			  "\x43\xcb\x47\xbd\x71\xc5\xed\xae"
-			  "\x9b\xe6\x9a\x2f\xfe\xce\xb1\xbe"
-			  "\xc9\xde\x24\x4f\xbe\x15\x99\x2b"
-			  "\x11\xb7\x7c\x04\x0f\x12\xbd\x8f"
-			  "\x6a\x97\x5a\x44\xa0\xf9\x0c\x29"
-			  "\xa9\xab\xc3\xd4\xd8\x93\x92\x72"
-			  "\x84\xc5\x87\x54\xcc\xe2\x94\x52"
-			  "\x9f\x86\x14\xdc\xd2\xab\xa9\x91"
-			  "\x92\x5f\xed\xc4\xae\x74\xff\xac"
-			  "\x6e\x33\x3b\x93\xeb\x4a\xff\x04"
-			  "\x79\xda\x9a\x41\x0e\x44\x50\xe0"
-			  "\xdd\x7a\xe4\xc6\xe2\x91\x09\x00"
-			  "\x57\x5d\xa4\x01\xfc\x07\x05\x9f"
-			  "\x64\x5e\x8b\x7e\x9b\xfd\xef\x33"
-			  "\x94\x30\x54\xff\x84\x01\x14\x93"
-			  "\xc2\x7b\x34\x29\xea\xed\xb4\xed"
-			  "\x53\x76\x44\x1a\x77\xed\x43\x85"
-			  "\x1a\xd7\x7f\x16\xf5\x41\xdf\xd2"
-			  "\x69\xd5\x0d\x6a\x5f\x14\xfb\x0a"
-			  "\xab\x1c\xbb\x4c\x15\x50\xbe\x97"
-			  "\xf7\xab\x40\x66\x19\x3c\x4c\xaa"
-			  "\x77\x3d\xad\x38\x01\x4b\xd2\x09"
-			  "\x2f\xa7\x55\xc8\x24\xbb\x5e\x54"
-			  "\xc4\xf3\x6f\xfd\xa9\xfc\xea\x70"
-			  "\xb9\xc6\xe6\x93\xe1\x48\xc1\x51",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	}
-};
-
-static const struct cipher_testvec aes_xts_dec_tv_template[] = {
-	/* http://grouper.ieee.org/groups/1619/email/pdf00086.pdf */
-	{ /* XTS-AES 1 */
-		.key    = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen   = 32,
-		.fips_skip = 1,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input = "\x91\x7c\xf6\x9e\xbd\x68\xb2\xec"
-			 "\x9b\x9f\xe9\xa3\xea\xdd\xa6\x92"
-			 "\xcd\x43\xd2\xf5\x95\x98\xed\x85"
-			 "\x8c\x02\xc2\x65\x2f\xbf\x92\x2e",
-		.ilen   = 32,
-		.result  = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			   "\x00\x00\x00\x00\x00\x00\x00\x00"
-			   "\x00\x00\x00\x00\x00\x00\x00\x00"
-			   "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen   = 32,
-	}, { /* XTS-AES 2 */
-		.key    = "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen   = 32,
-		.iv     = "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input  = "\xc4\x54\x18\x5e\x6a\x16\x93\x6e"
-			  "\x39\x33\x40\x38\xac\xef\x83\x8b"
-			  "\xfb\x18\x6f\xff\x74\x80\xad\xc4"
-			  "\x28\x93\x82\xec\xd6\xd3\x94\xf0",
-		.ilen   = 32,
-		.result = "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen   = 32,
-	}, { /* XTS-AES 3 */
-		.key    = "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen   = 32,
-		.iv     = "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input = "\xaf\x85\x33\x6b\x59\x7a\xfc\x1a"
-			  "\x90\x0b\x2e\xb2\x1e\xc9\x49\xd2"
-			  "\x92\xdf\x4c\x04\x7e\x0b\x21\x53"
-			  "\x21\x86\xa5\x97\x1a\x22\x7a\x89",
-		.ilen   = 32,
-		.result  = "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen   = 32,
-	}, { /* XTS-AES 4 */
-		.key    = "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95",
-		.klen   = 32,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input  = "\x27\xa7\x47\x9b\xef\xa1\xd4\x76"
-			  "\x48\x9f\x30\x8c\xd4\xcf\xa6\xe2"
-			  "\xa9\x6e\x4b\xbe\x32\x08\xff\x25"
-			  "\x28\x7d\xd3\x81\x96\x16\xe8\x9c"
-			  "\xc7\x8c\xf7\xf5\xe5\x43\x44\x5f"
-			  "\x83\x33\xd8\xfa\x7f\x56\x00\x00"
-			  "\x05\x27\x9f\xa5\xd8\xb5\xe4\xad"
-			  "\x40\xe7\x36\xdd\xb4\xd3\x54\x12"
-			  "\x32\x80\x63\xfd\x2a\xab\x53\xe5"
-			  "\xea\x1e\x0a\x9f\x33\x25\x00\xa5"
-			  "\xdf\x94\x87\xd0\x7a\x5c\x92\xcc"
-			  "\x51\x2c\x88\x66\xc7\xe8\x60\xce"
-			  "\x93\xfd\xf1\x66\xa2\x49\x12\xb4"
-			  "\x22\x97\x61\x46\xae\x20\xce\x84"
-			  "\x6b\xb7\xdc\x9b\xa9\x4a\x76\x7a"
-			  "\xae\xf2\x0c\x0d\x61\xad\x02\x65"
-			  "\x5e\xa9\x2d\xc4\xc4\xe4\x1a\x89"
-			  "\x52\xc6\x51\xd3\x31\x74\xbe\x51"
-			  "\xa1\x0c\x42\x11\x10\xe6\xd8\x15"
-			  "\x88\xed\xe8\x21\x03\xa2\x52\xd8"
-			  "\xa7\x50\xe8\x76\x8d\xef\xff\xed"
-			  "\x91\x22\x81\x0a\xae\xb9\x9f\x91"
-			  "\x72\xaf\x82\xb6\x04\xdc\x4b\x8e"
-			  "\x51\xbc\xb0\x82\x35\xa6\xf4\x34"
-			  "\x13\x32\xe4\xca\x60\x48\x2a\x4b"
-			  "\xa1\xa0\x3b\x3e\x65\x00\x8f\xc5"
-			  "\xda\x76\xb7\x0b\xf1\x69\x0d\xb4"
-			  "\xea\xe2\x9c\x5f\x1b\xad\xd0\x3c"
-			  "\x5c\xcf\x2a\x55\xd7\x05\xdd\xcd"
-			  "\x86\xd4\x49\x51\x1c\xeb\x7e\xc3"
-			  "\x0b\xf1\x2b\x1f\xa3\x5b\x91\x3f"
-			  "\x9f\x74\x7a\x8a\xfd\x1b\x13\x0e"
-			  "\x94\xbf\xf9\x4e\xff\xd0\x1a\x91"
-			  "\x73\x5c\xa1\x72\x6a\xcd\x0b\x19"
-			  "\x7c\x4e\x5b\x03\x39\x36\x97\xe1"
-			  "\x26\x82\x6f\xb6\xbb\xde\x8e\xcc"
-			  "\x1e\x08\x29\x85\x16\xe2\xc9\xed"
-			  "\x03\xff\x3c\x1b\x78\x60\xf6\xde"
-			  "\x76\xd4\xce\xcd\x94\xc8\x11\x98"
-			  "\x55\xef\x52\x97\xca\x67\xe9\xf3"
-			  "\xe7\xff\x72\xb1\xe9\x97\x85\xca"
-			  "\x0a\x7e\x77\x20\xc5\xb3\x6d\xc6"
-			  "\xd7\x2c\xac\x95\x74\xc8\xcb\xbc"
-			  "\x2f\x80\x1e\x23\xe5\x6f\xd3\x44"
-			  "\xb0\x7f\x22\x15\x4b\xeb\xa0\xf0"
-			  "\x8c\xe8\x89\x1e\x64\x3e\xd9\x95"
-			  "\xc9\x4d\x9a\x69\xc9\xf1\xb5\xf4"
-			  "\x99\x02\x7a\x78\x57\x2a\xee\xbd"
-			  "\x74\xd2\x0c\xc3\x98\x81\xc2\x13"
-			  "\xee\x77\x0b\x10\x10\xe4\xbe\xa7"
-			  "\x18\x84\x69\x77\xae\x11\x9f\x7a"
-			  "\x02\x3a\xb5\x8c\xca\x0a\xd7\x52"
-			  "\xaf\xe6\x56\xbb\x3c\x17\x25\x6a"
-			  "\x9f\x6e\x9b\xf1\x9f\xdd\x5a\x38"
-			  "\xfc\x82\xbb\xe8\x72\xc5\x53\x9e"
-			  "\xdb\x60\x9e\xf4\xf7\x9c\x20\x3e"
-			  "\xbb\x14\x0f\x2e\x58\x3c\xb2\xad"
-			  "\x15\xb4\xaa\x5b\x65\x50\x16\xa8"
-			  "\x44\x92\x77\xdb\xd4\x77\xef\x2c"
-			  "\x8d\x6c\x01\x7d\xb7\x38\xb1\x8d"
-			  "\xeb\x4a\x42\x7d\x19\x23\xce\x3f"
-			  "\xf2\x62\x73\x57\x79\xa4\x18\xf2"
-			  "\x0a\x28\x2d\xf9\x20\x14\x7b\xea"
-			  "\xbe\x42\x1e\xe5\x31\x9d\x05\x68",
-		.ilen   = 512,
-		.result = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen   = 512,
-	}, { /* XTS-AES 10, XTS-AES-256, data unit 512 bytes */
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x1c\x3b\x3a\x10\x2f\x77\x03\x86"
-			  "\xe4\x83\x6c\x99\xe3\x70\xcf\x9b"
-			  "\xea\x00\x80\x3f\x5e\x48\x23\x57"
-			  "\xa4\xae\x12\xd4\x14\xa3\xe6\x3b"
-			  "\x5d\x31\xe2\x76\xf8\xfe\x4a\x8d"
-			  "\x66\xb3\x17\xf9\xac\x68\x3f\x44"
-			  "\x68\x0a\x86\xac\x35\xad\xfc\x33"
-			  "\x45\xbe\xfe\xcb\x4b\xb1\x88\xfd"
-			  "\x57\x76\x92\x6c\x49\xa3\x09\x5e"
-			  "\xb1\x08\xfd\x10\x98\xba\xec\x70"
-			  "\xaa\xa6\x69\x99\xa7\x2a\x82\xf2"
-			  "\x7d\x84\x8b\x21\xd4\xa7\x41\xb0"
-			  "\xc5\xcd\x4d\x5f\xff\x9d\xac\x89"
-			  "\xae\xba\x12\x29\x61\xd0\x3a\x75"
-			  "\x71\x23\xe9\x87\x0f\x8a\xcf\x10"
-			  "\x00\x02\x08\x87\x89\x14\x29\xca"
-			  "\x2a\x3e\x7a\x7d\x7d\xf7\xb1\x03"
-			  "\x55\x16\x5c\x8b\x9a\x6d\x0a\x7d"
-			  "\xe8\xb0\x62\xc4\x50\x0d\xc4\xcd"
-			  "\x12\x0c\x0f\x74\x18\xda\xe3\xd0"
-			  "\xb5\x78\x1c\x34\x80\x3f\xa7\x54"
-			  "\x21\xc7\x90\xdf\xe1\xde\x18\x34"
-			  "\xf2\x80\xd7\x66\x7b\x32\x7f\x6c"
-			  "\x8c\xd7\x55\x7e\x12\xac\x3a\x0f"
-			  "\x93\xec\x05\xc5\x2e\x04\x93\xef"
-			  "\x31\xa1\x2d\x3d\x92\x60\xf7\x9a"
-			  "\x28\x9d\x6a\x37\x9b\xc7\x0c\x50"
-			  "\x84\x14\x73\xd1\xa8\xcc\x81\xec"
-			  "\x58\x3e\x96\x45\xe0\x7b\x8d\x96"
-			  "\x70\x65\x5b\xa5\xbb\xcf\xec\xc6"
-			  "\xdc\x39\x66\x38\x0a\xd8\xfe\xcb"
-			  "\x17\xb6\xba\x02\x46\x9a\x02\x0a"
-			  "\x84\xe1\x8e\x8f\x84\x25\x20\x70"
-			  "\xc1\x3e\x9f\x1f\x28\x9b\xe5\x4f"
-			  "\xbc\x48\x14\x57\x77\x8f\x61\x60"
-			  "\x15\xe1\x32\x7a\x02\xb1\x40\xf1"
-			  "\x50\x5e\xb3\x09\x32\x6d\x68\x37"
-			  "\x8f\x83\x74\x59\x5c\x84\x9d\x84"
-			  "\xf4\xc3\x33\xec\x44\x23\x88\x51"
-			  "\x43\xcb\x47\xbd\x71\xc5\xed\xae"
-			  "\x9b\xe6\x9a\x2f\xfe\xce\xb1\xbe"
-			  "\xc9\xde\x24\x4f\xbe\x15\x99\x2b"
-			  "\x11\xb7\x7c\x04\x0f\x12\xbd\x8f"
-			  "\x6a\x97\x5a\x44\xa0\xf9\x0c\x29"
-			  "\xa9\xab\xc3\xd4\xd8\x93\x92\x72"
-			  "\x84\xc5\x87\x54\xcc\xe2\x94\x52"
-			  "\x9f\x86\x14\xdc\xd2\xab\xa9\x91"
-			  "\x92\x5f\xed\xc4\xae\x74\xff\xac"
-			  "\x6e\x33\x3b\x93\xeb\x4a\xff\x04"
-			  "\x79\xda\x9a\x41\x0e\x44\x50\xe0"
-			  "\xdd\x7a\xe4\xc6\xe2\x91\x09\x00"
-			  "\x57\x5d\xa4\x01\xfc\x07\x05\x9f"
-			  "\x64\x5e\x8b\x7e\x9b\xfd\xef\x33"
-			  "\x94\x30\x54\xff\x84\x01\x14\x93"
-			  "\xc2\x7b\x34\x29\xea\xed\xb4\xed"
-			  "\x53\x76\x44\x1a\x77\xed\x43\x85"
-			  "\x1a\xd7\x7f\x16\xf5\x41\xdf\xd2"
-			  "\x69\xd5\x0d\x6a\x5f\x14\xfb\x0a"
-			  "\xab\x1c\xbb\x4c\x15\x50\xbe\x97"
-			  "\xf7\xab\x40\x66\x19\x3c\x4c\xaa"
-			  "\x77\x3d\xad\x38\x01\x4b\xd2\x09"
-			  "\x2f\xa7\x55\xc8\x24\xbb\x5e\x54"
-			  "\xc4\xf3\x6f\xfd\xa9\xfc\xea\x70"
-			  "\xb9\xc6\xe6\x93\xe1\x48\xc1\x51",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	}
-};
-
-
-static const struct cipher_testvec aes_ctr_enc_tv_template[] = {
-	{ /* From NIST Special Publication 800-38A, Appendix F.5 */
-		.key	= "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
-			  "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
-		.klen	= 16,
-		.iv	= "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.input	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen	= 64,
-		.result	= "\x87\x4d\x61\x91\xb6\x20\xe3\x26"
-			  "\x1b\xef\x68\x64\x99\x0d\xb6\xce"
-			  "\x98\x06\xf6\x6b\x79\x70\xfd\xff"
-			  "\x86\x17\x18\x7b\xb9\xff\xfd\xff"
-			  "\x5a\xe4\xdf\x3e\xdb\xd5\xd3\x5e"
-			  "\x5b\x4f\x09\x02\x0d\xb0\x3e\xab"
-			  "\x1e\x03\x1d\xda\x2f\xbe\x03\xd1"
-			  "\x79\x21\x70\xa0\xf3\x00\x9c\xee",
-		.rlen	= 64,
-	}, {
-		.key	= "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
-			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
-			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
-		.klen	= 24,
-		.iv	= "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.input	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen	= 64,
-		.result	= "\x1a\xbc\x93\x24\x17\x52\x1c\xa2"
-			  "\x4f\x2b\x04\x59\xfe\x7e\x6e\x0b"
-			  "\x09\x03\x39\xec\x0a\xa6\xfa\xef"
-			  "\xd5\xcc\xc2\xc6\xf4\xce\x8e\x94"
-			  "\x1e\x36\xb2\x6b\xd1\xeb\xc6\x70"
-			  "\xd1\xbd\x1d\x66\x56\x20\xab\xf7"
-			  "\x4f\x78\xa7\xf6\xd2\x98\x09\x58"
-			  "\x5a\x97\xda\xec\x58\xc6\xb0\x50",
-		.rlen	= 64,
-	}, {
-		.key	= "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
-			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
-			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
-			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
-		.klen	= 32,
-		.iv	= "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.input	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen	= 64,
-		.result	= "\x60\x1e\xc3\x13\x77\x57\x89\xa5"
-			  "\xb7\xa7\xf5\x04\xbb\xf3\xd2\x28"
-			  "\xf4\x43\xe3\xca\x4d\x62\xb5\x9a"
-			  "\xca\x84\xe9\x90\xca\xca\xf5\xc5"
-			  "\x2b\x09\x30\xda\xa2\x3d\xe9\x4c"
-			  "\xe8\x70\x17\xba\x2d\x84\x98\x8d"
-			  "\xdf\xc9\xc5\x8d\xb6\x7a\xad\xa6"
-			  "\x13\xc2\xdd\x08\x45\x79\x41\xa6",
-		.rlen	= 64,
-	}, { /* Generated with Crypto++ */
-		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55"
-			  "\x0F\x32\x55\x78\x9B\xBE\x78\x9B"
-			  "\xBE\xE1\x04\x27\xE1\x04\x27\x4A"
-			  "\x6D\x90\x4A\x6D\x90\xB3\xD6\xF9",
-		.klen	= 32,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
-			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
-			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
-			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
-			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
-			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
-			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
-			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
-			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
-			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
-			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
-			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
-			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
-			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
-			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
-			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
-			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
-			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
-			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
-			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
-			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
-			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
-			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
-			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
-			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
-			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
-			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
-			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
-			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
-			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
-			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
-			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12",
-		.ilen	= 496,
-		.result	= "\x04\xF3\xD3\x88\x17\xEF\xDC\xEF"
-			  "\x8B\x04\xF8\x3A\x66\x8D\x1A\x53"
-			  "\x57\x1F\x4B\x23\xE4\xA0\xAF\xF9"
-			  "\x69\x95\x35\x98\x8D\x4D\x8C\xC1"
-			  "\xF0\xB2\x7F\x80\xBB\x54\x28\xA2"
-			  "\x7A\x1B\x9F\x77\xEC\x0E\x6E\xDE"
-			  "\xF0\xEC\xB8\xE4\x20\x62\xEE\xDB"
-			  "\x5D\xF5\xDD\xE3\x54\xFC\xDD\xEB"
-			  "\x6A\xEE\x65\xA1\x21\xD6\xD7\x81"
-			  "\x47\x61\x12\x4D\xC2\x8C\xFA\x78"
-			  "\x1F\x28\x02\x01\xC3\xFC\x1F\xEC"
-			  "\x0F\x10\x4F\xB3\x12\x45\xC6\x3B"
-			  "\x7E\x08\xF9\x5A\xD0\x5D\x73\x2D"
-			  "\x58\xA4\xE5\xCB\x1C\xB4\xCE\x74"
-			  "\x32\x41\x1F\x31\x9C\x08\xA2\x5D"
-			  "\x67\xEB\x72\x1D\xF8\xE7\x70\x54"
-			  "\x34\x4B\x31\x69\x84\x66\x96\x44"
-			  "\x56\xCC\x1E\xD9\xE6\x13\x6A\xB9"
-			  "\x2D\x0A\x05\x45\x2D\x90\xCC\xDF"
-			  "\x16\x5C\x5F\x79\x34\x52\x54\xFE"
-			  "\xFE\xCD\xAD\x04\x2E\xAD\x86\x06"
-			  "\x1F\x37\xE8\x28\xBC\xD3\x8F\x5B"
-			  "\x92\x66\x87\x3B\x8A\x0A\x1A\xCC"
-			  "\x6E\xAB\x9F\x0B\xFA\x5C\xE6\xFD"
-			  "\x3C\x98\x08\x12\xEC\xAA\x9E\x11"
-			  "\xCA\xB2\x1F\xCE\x5E\x5B\xB2\x72"
-			  "\x9C\xCC\x5D\xC5\xE0\x32\xC0\x56"
-			  "\xD5\x45\x16\xD2\xAF\x13\x66\xF7"
-			  "\x8C\x67\xAC\x79\xB2\xAF\x56\x27"
-			  "\x3F\xCC\xFE\xCB\x1E\xC0\x75\xF1"
-			  "\xA7\xC9\xC3\x1D\x8E\xDD\xF9\xD4"
-			  "\x42\xC8\x21\x08\x16\xF7\x01\xD7"
-			  "\xAC\x8E\x3F\x1D\x56\xC1\x06\xE4"
-			  "\x9C\x62\xD6\xA5\x6A\x50\x44\xB3"
-			  "\x35\x1C\x82\xB9\x10\xF9\x42\xA1"
-			  "\xFC\x74\x9B\x44\x4F\x25\x02\xE3"
-			  "\x08\xF5\xD4\x32\x39\x08\x11\xE8"
-			  "\xD2\x6B\x50\x53\xD4\x08\xD1\x6B"
-			  "\x3A\x4A\x68\x7B\x7C\xCD\x46\x5E"
-			  "\x0D\x07\x19\xDB\x67\xD7\x98\x91"
-			  "\xD7\x17\x10\x9B\x7B\x8A\x9B\x33"
-			  "\xAE\xF3\x00\xA6\xD4\x15\xD9\xEA"
-			  "\x85\x99\x22\xE8\x91\x38\x70\x83"
-			  "\x93\x01\x24\x6C\xFA\x9A\xB9\x07"
-			  "\xEA\x8D\x3B\xD9\x2A\x43\x59\x16"
-			  "\x2F\x69\xEE\x84\x36\x44\x76\x98"
-			  "\xF3\x04\x2A\x7C\x74\x3D\x29\x2B"
-			  "\x0D\xAD\x8F\x44\x82\x9E\x57\x8D"
-			  "\xAC\xED\x18\x1F\x50\xA4\xF5\x98"
-			  "\x1F\xBD\x92\x91\x1B\x2D\xA6\xD6"
-			  "\xD2\xE3\x02\xAA\x92\x3B\xC6\xB3"
-			  "\x1B\x39\x72\xD5\x26\xCA\x04\xE0"
-			  "\xFC\x58\x78\xBB\xB1\x3F\xA1\x9C"
-			  "\x42\x24\x3E\x2E\x22\xBB\x4B\xBA"
-			  "\xF4\x52\x0A\xE6\xAE\x47\xB4\x7D"
-			  "\x1D\xA8\xBE\x81\x1A\x75\xDA\xAC"
-			  "\xA6\x25\x1E\xEF\x3A\xC0\x6C\x63"
-			  "\xEF\xDC\xC9\x79\x10\x26\xE8\x61"
-			  "\x29\xFC\xA4\x05\xDF\x7D\x5C\x63"
-			  "\x10\x09\x9B\x46\x9B\xF2\x2C\x2B"
-			  "\xFA\x3A\x05\x4C\xFA\xD1\xFF\xFE"
-			  "\xF1\x4C\xE5\xB2\x91\x64\x0C\x51",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	}, { /* Generated with Crypto++ */
-		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55"
-			  "\x0F\x32\x55\x78\x9B\xBE\x78\x9B"
-			  "\xBE\xE1\x04\x27\xE1\x04\x27\x4A"
-			  "\x6D\x90\x4A\x6D\x90\xB3\xD6\xF9",
-		.klen	= 32,
-		.iv	= "\xE7\x82\x1D\xB8\x53\x11\xAC\x47"
-			  "\xE2\x7D\x18\xD6\x71\x0C\xA7\x42",
-		.input	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
-			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
-			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
-			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
-			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
-			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
-			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
-			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
-			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
-			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
-			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
-			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
-			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
-			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
-			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
-			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
-			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
-			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
-			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
-			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
-			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
-			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
-			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
-			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
-			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
-			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
-			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
-			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
-			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
-			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
-			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
-			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12"
-			  "\x7B\xE4\x4D",
-		.ilen	= 499,
-		.result	= "\xDA\x4E\x3F\xBC\xE8\xB6\x3A\xA2"
-			  "\xD5\x4D\x84\x4A\xA9\x0C\xE1\xA5"
-			  "\xB8\x73\xBC\xF9\xBB\x59\x2F\x44"
-			  "\x8B\xAB\x82\x6C\xB4\x32\x9A\xDE"
-			  "\x5A\x0B\xDB\x7A\x6B\xF2\x38\x9F"
-			  "\x06\xF7\xF7\xFF\xFF\xC0\x8A\x2E"
-			  "\x76\xEA\x06\x32\x23\xF3\x59\x2E"
-			  "\x75\xDE\x71\x86\x3C\x98\x23\x44"
-			  "\x5B\xF2\xFA\x6A\x00\xBB\xC1\xAD"
-			  "\x58\xBD\x3E\x6F\x2E\xB4\x19\x04"
-			  "\x70\x8B\x92\x55\x23\xE9\x6A\x3A"
-			  "\x78\x7A\x1B\x10\x85\x52\x9C\x12"
-			  "\xE4\x55\x81\x21\xCE\x53\xD0\x3B"
-			  "\x63\x77\x2C\x74\xD1\xF5\x60\xF3"
-			  "\xA1\xDE\x44\x3C\x8F\x4D\x2F\xDD"
-			  "\x8A\xFE\x3C\x42\x8E\xD3\xF2\x8E"
-			  "\xA8\x28\x69\x65\x31\xE1\x45\x83"
-			  "\xE4\x49\xC4\x9C\xA7\x28\xAA\x21"
-			  "\xCD\x5D\x0F\x15\xB7\x93\x07\x26"
-			  "\xB0\x65\x6D\x91\x90\x23\x7A\xC6"
-			  "\xDB\x68\xB0\xA1\x8E\xA4\x76\x4E"
-			  "\xC6\x91\x83\x20\x92\x4D\x63\x7A"
-			  "\x45\x18\x18\x74\x19\xAD\x71\x01"
-			  "\x6B\x23\xAD\x9D\x4E\xE4\x6E\x46"
-			  "\xC9\x73\x7A\xF9\x02\x95\xF4\x07"
-			  "\x0E\x7A\xA6\xC5\xAE\xFA\x15\x2C"
-			  "\x51\x71\xF1\xDC\x22\xB6\xAC\xD8"
-			  "\x19\x24\x44\xBC\x0C\xFB\x3C\x2D"
-			  "\xB1\x50\x47\x15\x0E\xDB\xB6\xD7"
-			  "\xE8\x61\xE5\x95\x52\x1E\x3E\x49"
-			  "\x70\xE9\x66\x04\x4C\xE1\xAF\xBD"
-			  "\xDD\x15\x3B\x20\x59\x24\xFF\xB0"
-			  "\x39\xAA\xE7\xBF\x23\xA3\x6E\xD5"
-			  "\x15\xF0\x61\x4F\xAE\x89\x10\x58"
-			  "\x5A\x33\x95\x52\x2A\xB5\x77\x9C"
-			  "\xA5\x43\x80\x40\x27\x2D\xAE\xD9"
-			  "\x3F\xE0\x80\x94\x78\x79\xCB\x7E"
-			  "\xAD\x12\x44\x4C\xEC\x27\xB0\xEE"
-			  "\x0B\x05\x2A\x82\x99\x58\xBB\x7A"
-			  "\x8D\x6D\x9D\x8E\xE2\x8E\xE7\x93"
-			  "\x2F\xB3\x09\x8D\x06\xD5\xEE\x70"
-			  "\x16\xAE\x35\xC5\x52\x0F\x46\x1F"
-			  "\x71\xF9\x5E\xF2\x67\xDC\x98\x2F"
-			  "\xA3\x23\xAA\xD5\xD0\x49\xF4\xA6"
-			  "\xF6\xB8\x32\xCD\xD6\x85\x73\x60"
-			  "\x59\x20\xE7\x55\x0E\x91\xE2\x0C"
-			  "\x3F\x1C\xEB\x3D\xDF\x52\x64\xF2"
-			  "\x7D\x8B\x5D\x63\x16\xB9\xB2\x5D"
-			  "\x5E\xAB\xB2\x97\xAB\x78\x44\xE7"
-			  "\xC6\x72\x20\xC5\x90\x9B\xDC\x5D"
-			  "\xB0\xEF\x44\xEF\x87\x31\x8D\xF4"
-			  "\xFB\x81\x5D\xF7\x96\x96\xD4\x50"
-			  "\x89\xA7\xF6\xB9\x67\x76\x40\x9E"
-			  "\x9D\x40\xD5\x2C\x30\xB8\x01\x8F"
-			  "\xE4\x7B\x71\x48\xA9\xA0\xA0\x1D"
-			  "\x87\x52\xA4\x91\xA9\xD7\xA9\x51"
-			  "\xD9\x59\xF7\xCC\x63\x22\xC1\x8D"
-			  "\x84\x7B\xD8\x22\x32\x5C\x6F\x1D"
-			  "\x6E\x9F\xFA\xDD\x49\x40\xDC\x37"
-			  "\x14\x8C\xE1\x80\x1B\xDD\x36\x2A"
-			  "\xD0\xE9\x54\x99\x5D\xBA\x3B\x11"
-			  "\xD8\xFE\xC9\x5B\x5C\x25\xE5\x76"
-			  "\xFB\xF2\x3F",
-		.rlen	= 499,
-		.also_non_np = 1,
-		.np	= 2,
-		.tap	= { 499 - 16, 16 },
-	},
-};
-
-static const struct cipher_testvec aes_ctr_dec_tv_template[] = {
-	{ /* From NIST Special Publication 800-38A, Appendix F.5 */
-		.key	= "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
-			  "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
-		.klen	= 16,
-		.iv	= "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.input	= "\x87\x4d\x61\x91\xb6\x20\xe3\x26"
-			  "\x1b\xef\x68\x64\x99\x0d\xb6\xce"
-			  "\x98\x06\xf6\x6b\x79\x70\xfd\xff"
-			  "\x86\x17\x18\x7b\xb9\xff\xfd\xff"
-			  "\x5a\xe4\xdf\x3e\xdb\xd5\xd3\x5e"
-			  "\x5b\x4f\x09\x02\x0d\xb0\x3e\xab"
-			  "\x1e\x03\x1d\xda\x2f\xbe\x03\xd1"
-			  "\x79\x21\x70\xa0\xf3\x00\x9c\xee",
-		.ilen	= 64,
-		.result	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.rlen	= 64,
-	}, {
-		.key	= "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
-			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
-			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
-		.klen	= 24,
-		.iv	= "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.input	= "\x1a\xbc\x93\x24\x17\x52\x1c\xa2"
-			  "\x4f\x2b\x04\x59\xfe\x7e\x6e\x0b"
-			  "\x09\x03\x39\xec\x0a\xa6\xfa\xef"
-			  "\xd5\xcc\xc2\xc6\xf4\xce\x8e\x94"
-			  "\x1e\x36\xb2\x6b\xd1\xeb\xc6\x70"
-			  "\xd1\xbd\x1d\x66\x56\x20\xab\xf7"
-			  "\x4f\x78\xa7\xf6\xd2\x98\x09\x58"
-			  "\x5a\x97\xda\xec\x58\xc6\xb0\x50",
-		.ilen	= 64,
-		.result	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.rlen	= 64,
-	}, {
-		.key	= "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
-			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
-			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
-			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
-		.klen	= 32,
-		.iv	= "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.input	= "\x60\x1e\xc3\x13\x77\x57\x89\xa5"
-			  "\xb7\xa7\xf5\x04\xbb\xf3\xd2\x28"
-			  "\xf4\x43\xe3\xca\x4d\x62\xb5\x9a"
-			  "\xca\x84\xe9\x90\xca\xca\xf5\xc5"
-			  "\x2b\x09\x30\xda\xa2\x3d\xe9\x4c"
-			  "\xe8\x70\x17\xba\x2d\x84\x98\x8d"
-			  "\xdf\xc9\xc5\x8d\xb6\x7a\xad\xa6"
-			  "\x13\xc2\xdd\x08\x45\x79\x41\xa6",
-		.ilen	= 64,
-		.result	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.rlen	= 64,
-	}, { /* Generated with Crypto++ */
-		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55"
-			  "\x0F\x32\x55\x78\x9B\xBE\x78\x9B"
-			  "\xBE\xE1\x04\x27\xE1\x04\x27\x4A"
-			  "\x6D\x90\x4A\x6D\x90\xB3\xD6\xF9",
-		.klen	= 32,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x04\xF3\xD3\x88\x17\xEF\xDC\xEF"
-			  "\x8B\x04\xF8\x3A\x66\x8D\x1A\x53"
-			  "\x57\x1F\x4B\x23\xE4\xA0\xAF\xF9"
-			  "\x69\x95\x35\x98\x8D\x4D\x8C\xC1"
-			  "\xF0\xB2\x7F\x80\xBB\x54\x28\xA2"
-			  "\x7A\x1B\x9F\x77\xEC\x0E\x6E\xDE"
-			  "\xF0\xEC\xB8\xE4\x20\x62\xEE\xDB"
-			  "\x5D\xF5\xDD\xE3\x54\xFC\xDD\xEB"
-			  "\x6A\xEE\x65\xA1\x21\xD6\xD7\x81"
-			  "\x47\x61\x12\x4D\xC2\x8C\xFA\x78"
-			  "\x1F\x28\x02\x01\xC3\xFC\x1F\xEC"
-			  "\x0F\x10\x4F\xB3\x12\x45\xC6\x3B"
-			  "\x7E\x08\xF9\x5A\xD0\x5D\x73\x2D"
-			  "\x58\xA4\xE5\xCB\x1C\xB4\xCE\x74"
-			  "\x32\x41\x1F\x31\x9C\x08\xA2\x5D"
-			  "\x67\xEB\x72\x1D\xF8\xE7\x70\x54"
-			  "\x34\x4B\x31\x69\x84\x66\x96\x44"
-			  "\x56\xCC\x1E\xD9\xE6\x13\x6A\xB9"
-			  "\x2D\x0A\x05\x45\x2D\x90\xCC\xDF"
-			  "\x16\x5C\x5F\x79\x34\x52\x54\xFE"
-			  "\xFE\xCD\xAD\x04\x2E\xAD\x86\x06"
-			  "\x1F\x37\xE8\x28\xBC\xD3\x8F\x5B"
-			  "\x92\x66\x87\x3B\x8A\x0A\x1A\xCC"
-			  "\x6E\xAB\x9F\x0B\xFA\x5C\xE6\xFD"
-			  "\x3C\x98\x08\x12\xEC\xAA\x9E\x11"
-			  "\xCA\xB2\x1F\xCE\x5E\x5B\xB2\x72"
-			  "\x9C\xCC\x5D\xC5\xE0\x32\xC0\x56"
-			  "\xD5\x45\x16\xD2\xAF\x13\x66\xF7"
-			  "\x8C\x67\xAC\x79\xB2\xAF\x56\x27"
-			  "\x3F\xCC\xFE\xCB\x1E\xC0\x75\xF1"
-			  "\xA7\xC9\xC3\x1D\x8E\xDD\xF9\xD4"
-			  "\x42\xC8\x21\x08\x16\xF7\x01\xD7"
-			  "\xAC\x8E\x3F\x1D\x56\xC1\x06\xE4"
-			  "\x9C\x62\xD6\xA5\x6A\x50\x44\xB3"
-			  "\x35\x1C\x82\xB9\x10\xF9\x42\xA1"
-			  "\xFC\x74\x9B\x44\x4F\x25\x02\xE3"
-			  "\x08\xF5\xD4\x32\x39\x08\x11\xE8"
-			  "\xD2\x6B\x50\x53\xD4\x08\xD1\x6B"
-			  "\x3A\x4A\x68\x7B\x7C\xCD\x46\x5E"
-			  "\x0D\x07\x19\xDB\x67\xD7\x98\x91"
-			  "\xD7\x17\x10\x9B\x7B\x8A\x9B\x33"
-			  "\xAE\xF3\x00\xA6\xD4\x15\xD9\xEA"
-			  "\x85\x99\x22\xE8\x91\x38\x70\x83"
-			  "\x93\x01\x24\x6C\xFA\x9A\xB9\x07"
-			  "\xEA\x8D\x3B\xD9\x2A\x43\x59\x16"
-			  "\x2F\x69\xEE\x84\x36\x44\x76\x98"
-			  "\xF3\x04\x2A\x7C\x74\x3D\x29\x2B"
-			  "\x0D\xAD\x8F\x44\x82\x9E\x57\x8D"
-			  "\xAC\xED\x18\x1F\x50\xA4\xF5\x98"
-			  "\x1F\xBD\x92\x91\x1B\x2D\xA6\xD6"
-			  "\xD2\xE3\x02\xAA\x92\x3B\xC6\xB3"
-			  "\x1B\x39\x72\xD5\x26\xCA\x04\xE0"
-			  "\xFC\x58\x78\xBB\xB1\x3F\xA1\x9C"
-			  "\x42\x24\x3E\x2E\x22\xBB\x4B\xBA"
-			  "\xF4\x52\x0A\xE6\xAE\x47\xB4\x7D"
-			  "\x1D\xA8\xBE\x81\x1A\x75\xDA\xAC"
-			  "\xA6\x25\x1E\xEF\x3A\xC0\x6C\x63"
-			  "\xEF\xDC\xC9\x79\x10\x26\xE8\x61"
-			  "\x29\xFC\xA4\x05\xDF\x7D\x5C\x63"
-			  "\x10\x09\x9B\x46\x9B\xF2\x2C\x2B"
-			  "\xFA\x3A\x05\x4C\xFA\xD1\xFF\xFE"
-			  "\xF1\x4C\xE5\xB2\x91\x64\x0C\x51",
-		.ilen	= 496,
-		.result	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
-			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
-			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
-			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
-			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
-			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
-			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
-			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
-			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
-			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
-			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
-			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
-			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
-			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
-			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
-			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
-			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
-			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
-			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
-			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
-			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
-			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
-			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
-			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
-			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
-			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
-			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
-			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
-			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
-			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
-			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
-			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	}, { /* Generated with Crypto++ */
-		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55"
-			  "\x0F\x32\x55\x78\x9B\xBE\x78\x9B"
-			  "\xBE\xE1\x04\x27\xE1\x04\x27\x4A"
-			  "\x6D\x90\x4A\x6D\x90\xB3\xD6\xF9",
-		.klen	= 32,
-		.iv	= "\xE7\x82\x1D\xB8\x53\x11\xAC\x47"
-			  "\xE2\x7D\x18\xD6\x71\x0C\xA7\x42",
-		.input	= "\xDA\x4E\x3F\xBC\xE8\xB6\x3A\xA2"
-			  "\xD5\x4D\x84\x4A\xA9\x0C\xE1\xA5"
-			  "\xB8\x73\xBC\xF9\xBB\x59\x2F\x44"
-			  "\x8B\xAB\x82\x6C\xB4\x32\x9A\xDE"
-			  "\x5A\x0B\xDB\x7A\x6B\xF2\x38\x9F"
-			  "\x06\xF7\xF7\xFF\xFF\xC0\x8A\x2E"
-			  "\x76\xEA\x06\x32\x23\xF3\x59\x2E"
-			  "\x75\xDE\x71\x86\x3C\x98\x23\x44"
-			  "\x5B\xF2\xFA\x6A\x00\xBB\xC1\xAD"
-			  "\x58\xBD\x3E\x6F\x2E\xB4\x19\x04"
-			  "\x70\x8B\x92\x55\x23\xE9\x6A\x3A"
-			  "\x78\x7A\x1B\x10\x85\x52\x9C\x12"
-			  "\xE4\x55\x81\x21\xCE\x53\xD0\x3B"
-			  "\x63\x77\x2C\x74\xD1\xF5\x60\xF3"
-			  "\xA1\xDE\x44\x3C\x8F\x4D\x2F\xDD"
-			  "\x8A\xFE\x3C\x42\x8E\xD3\xF2\x8E"
-			  "\xA8\x28\x69\x65\x31\xE1\x45\x83"
-			  "\xE4\x49\xC4\x9C\xA7\x28\xAA\x21"
-			  "\xCD\x5D\x0F\x15\xB7\x93\x07\x26"
-			  "\xB0\x65\x6D\x91\x90\x23\x7A\xC6"
-			  "\xDB\x68\xB0\xA1\x8E\xA4\x76\x4E"
-			  "\xC6\x91\x83\x20\x92\x4D\x63\x7A"
-			  "\x45\x18\x18\x74\x19\xAD\x71\x01"
-			  "\x6B\x23\xAD\x9D\x4E\xE4\x6E\x46"
-			  "\xC9\x73\x7A\xF9\x02\x95\xF4\x07"
-			  "\x0E\x7A\xA6\xC5\xAE\xFA\x15\x2C"
-			  "\x51\x71\xF1\xDC\x22\xB6\xAC\xD8"
-			  "\x19\x24\x44\xBC\x0C\xFB\x3C\x2D"
-			  "\xB1\x50\x47\x15\x0E\xDB\xB6\xD7"
-			  "\xE8\x61\xE5\x95\x52\x1E\x3E\x49"
-			  "\x70\xE9\x66\x04\x4C\xE1\xAF\xBD"
-			  "\xDD\x15\x3B\x20\x59\x24\xFF\xB0"
-			  "\x39\xAA\xE7\xBF\x23\xA3\x6E\xD5"
-			  "\x15\xF0\x61\x4F\xAE\x89\x10\x58"
-			  "\x5A\x33\x95\x52\x2A\xB5\x77\x9C"
-			  "\xA5\x43\x80\x40\x27\x2D\xAE\xD9"
-			  "\x3F\xE0\x80\x94\x78\x79\xCB\x7E"
-			  "\xAD\x12\x44\x4C\xEC\x27\xB0\xEE"
-			  "\x0B\x05\x2A\x82\x99\x58\xBB\x7A"
-			  "\x8D\x6D\x9D\x8E\xE2\x8E\xE7\x93"
-			  "\x2F\xB3\x09\x8D\x06\xD5\xEE\x70"
-			  "\x16\xAE\x35\xC5\x52\x0F\x46\x1F"
-			  "\x71\xF9\x5E\xF2\x67\xDC\x98\x2F"
-			  "\xA3\x23\xAA\xD5\xD0\x49\xF4\xA6"
-			  "\xF6\xB8\x32\xCD\xD6\x85\x73\x60"
-			  "\x59\x20\xE7\x55\x0E\x91\xE2\x0C"
-			  "\x3F\x1C\xEB\x3D\xDF\x52\x64\xF2"
-			  "\x7D\x8B\x5D\x63\x16\xB9\xB2\x5D"
-			  "\x5E\xAB\xB2\x97\xAB\x78\x44\xE7"
-			  "\xC6\x72\x20\xC5\x90\x9B\xDC\x5D"
-			  "\xB0\xEF\x44\xEF\x87\x31\x8D\xF4"
-			  "\xFB\x81\x5D\xF7\x96\x96\xD4\x50"
-			  "\x89\xA7\xF6\xB9\x67\x76\x40\x9E"
-			  "\x9D\x40\xD5\x2C\x30\xB8\x01\x8F"
-			  "\xE4\x7B\x71\x48\xA9\xA0\xA0\x1D"
-			  "\x87\x52\xA4\x91\xA9\xD7\xA9\x51"
-			  "\xD9\x59\xF7\xCC\x63\x22\xC1\x8D"
-			  "\x84\x7B\xD8\x22\x32\x5C\x6F\x1D"
-			  "\x6E\x9F\xFA\xDD\x49\x40\xDC\x37"
-			  "\x14\x8C\xE1\x80\x1B\xDD\x36\x2A"
-			  "\xD0\xE9\x54\x99\x5D\xBA\x3B\x11"
-			  "\xD8\xFE\xC9\x5B\x5C\x25\xE5\x76"
-			  "\xFB\xF2\x3F",
-		.ilen	= 499,
-		.result	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
-			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
-			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
-			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
-			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
-			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
-			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
-			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
-			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
-			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
-			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
-			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
-			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
-			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
-			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
-			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
-			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
-			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
-			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
-			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
-			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
-			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
-			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
-			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
-			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
-			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
-			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
-			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
-			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
-			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
-			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
-			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
-			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
-			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
-			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
-			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
-			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
-			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
-			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
-			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
-			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
-			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
-			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
-			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
-			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
-			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
-			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
-			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
-			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
-			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
-			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
-			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
-			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
-			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
-			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
-			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
-			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
-			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
-			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
-			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
-			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
-			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12"
-			  "\x7B\xE4\x4D",
-		.rlen	= 499,
-		.also_non_np = 1,
-		.np	= 2,
-		.tap	= { 499 - 16, 16 },
-	},
-};
-
-static const struct cipher_testvec aes_ctr_rfc3686_enc_tv_template[] = {
-	{ /* From RFC 3686 */
-		.key	= "\xae\x68\x52\xf8\x12\x10\x67\xcc"
-			  "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e"
-			  "\x00\x00\x00\x30",
-		.klen	= 20,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "Single block msg",
-		.ilen	= 16,
-		.result = "\xe4\x09\x5d\x4f\xb7\xa7\xb3\x79"
-			  "\x2d\x61\x75\xa3\x26\x13\x11\xb8",
-		.rlen	= 16,
-	}, {
-		.key	= "\x7e\x24\x06\x78\x17\xfa\xe0\xd7"
-			  "\x43\xd6\xce\x1f\x32\x53\x91\x63"
-			  "\x00\x6c\xb6\xdb",
-		.klen	= 20,
-		.iv	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.ilen	= 32,
-		.result = "\x51\x04\xa1\x06\x16\x8a\x72\xd9"
-			  "\x79\x0d\x41\xee\x8e\xda\xd3\x88"
-			  "\xeb\x2e\x1e\xfc\x46\xda\x57\xc8"
-			  "\xfc\xe6\x30\xdf\x91\x41\xbe\x28",
-		.rlen	= 32,
-	}, {
-		.key	= "\x16\xaf\x5b\x14\x5f\xc9\xf5\x79"
-			  "\xc1\x75\xf9\x3e\x3b\xfb\x0e\xed"
-			  "\x86\x3d\x06\xcc\xfd\xb7\x85\x15"
-			  "\x00\x00\x00\x48",
-		.klen	= 28,
-		.iv	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
-		.input	= "Single block msg",
-		.ilen	= 16,
-		.result	= "\x4b\x55\x38\x4f\xe2\x59\xc9\xc8"
-			  "\x4e\x79\x35\xa0\x03\xcb\xe9\x28",
-		.rlen	= 16,
-	}, {
-		.key	= "\x7c\x5c\xb2\x40\x1b\x3d\xc3\x3c"
-			  "\x19\xe7\x34\x08\x19\xe0\xf6\x9c"
-			  "\x67\x8c\x3d\xb8\xe6\xf6\xa9\x1a"
-			  "\x00\x96\xb0\x3b",
-		.klen	= 28,
-		.iv	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.ilen	= 32,
-		.result	= "\x45\x32\x43\xfc\x60\x9b\x23\x32"
-			  "\x7e\xdf\xaa\xfa\x71\x31\xcd\x9f"
-			  "\x84\x90\x70\x1c\x5a\xd4\xa7\x9c"
-			  "\xfc\x1f\xe0\xff\x42\xf4\xfb\x00",
-		.rlen	= 32,
-	}, {
-		.key	= "\x77\x6b\xef\xf2\x85\x1d\xb0\x6f"
-			  "\x4c\x8a\x05\x42\xc8\x69\x6f\x6c"
-			  "\x6a\x81\xaf\x1e\xec\x96\xb4\xd3"
-			  "\x7f\xc1\xd6\x89\xe6\xc1\xc1\x04"
-			  "\x00\x00\x00\x60",
-		.klen	= 36,
-		.iv	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
-		.input	= "Single block msg",
-		.ilen	= 16,
-		.result = "\x14\x5a\xd0\x1d\xbf\x82\x4e\xc7"
-			  "\x56\x08\x63\xdc\x71\xe3\xe0\xc0",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf6\xd6\x6d\x6b\xd5\x2d\x59\xbb"
-			  "\x07\x96\x36\x58\x79\xef\xf8\x86"
-			  "\xc6\x6d\xd5\x1a\x5b\x6a\x99\x74"
-			  "\x4b\x50\x59\x0c\x87\xa2\x38\x84"
-			  "\x00\xfa\xac\x24",
-		.klen	= 36,
-		.iv	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.ilen	= 32,
-		.result = "\xf0\x5e\x23\x1b\x38\x94\x61\x2c"
-			  "\x49\xee\x00\x0b\x80\x4e\xb2\xa9"
-			  "\xb8\x30\x6b\x50\x8f\x83\x9d\x6a"
-			  "\x55\x30\x83\x1d\x93\x44\xaf\x1c",
-		.rlen	= 32,
-	}, {
-	// generated using Crypto++
-		.key = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			"\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			"\x10\x11\x12\x13\x14\x15\x16\x17"
-			"\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			"\x00\x00\x00\x00",
-		.klen = 32 + 4,
-		.iv = "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input =
-			"\x00\x01\x02\x03\x04\x05\x06\x07"
-			"\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			"\x10\x11\x12\x13\x14\x15\x16\x17"
-			"\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			"\x20\x21\x22\x23\x24\x25\x26\x27"
-			"\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			"\x30\x31\x32\x33\x34\x35\x36\x37"
-			"\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			"\x40\x41\x42\x43\x44\x45\x46\x47"
-			"\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			"\x50\x51\x52\x53\x54\x55\x56\x57"
-			"\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			"\x60\x61\x62\x63\x64\x65\x66\x67"
-			"\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			"\x70\x71\x72\x73\x74\x75\x76\x77"
-			"\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			"\x80\x81\x82\x83\x84\x85\x86\x87"
-			"\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			"\x90\x91\x92\x93\x94\x95\x96\x97"
-			"\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			"\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			"\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			"\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			"\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			"\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			"\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			"\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			"\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			"\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			"\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			"\x00\x03\x06\x09\x0c\x0f\x12\x15"
-			"\x18\x1b\x1e\x21\x24\x27\x2a\x2d"
-			"\x30\x33\x36\x39\x3c\x3f\x42\x45"
-			"\x48\x4b\x4e\x51\x54\x57\x5a\x5d"
-			"\x60\x63\x66\x69\x6c\x6f\x72\x75"
-			"\x78\x7b\x7e\x81\x84\x87\x8a\x8d"
-			"\x90\x93\x96\x99\x9c\x9f\xa2\xa5"
-			"\xa8\xab\xae\xb1\xb4\xb7\xba\xbd"
-			"\xc0\xc3\xc6\xc9\xcc\xcf\xd2\xd5"
-			"\xd8\xdb\xde\xe1\xe4\xe7\xea\xed"
-			"\xf0\xf3\xf6\xf9\xfc\xff\x02\x05"
-			"\x08\x0b\x0e\x11\x14\x17\x1a\x1d"
-			"\x20\x23\x26\x29\x2c\x2f\x32\x35"
-			"\x38\x3b\x3e\x41\x44\x47\x4a\x4d"
-			"\x50\x53\x56\x59\x5c\x5f\x62\x65"
-			"\x68\x6b\x6e\x71\x74\x77\x7a\x7d"
-			"\x80\x83\x86\x89\x8c\x8f\x92\x95"
-			"\x98\x9b\x9e\xa1\xa4\xa7\xaa\xad"
-			"\xb0\xb3\xb6\xb9\xbc\xbf\xc2\xc5"
-			"\xc8\xcb\xce\xd1\xd4\xd7\xda\xdd"
-			"\xe0\xe3\xe6\xe9\xec\xef\xf2\xf5"
-			"\xf8\xfb\xfe\x01\x04\x07\x0a\x0d"
-			"\x10\x13\x16\x19\x1c\x1f\x22\x25"
-			"\x28\x2b\x2e\x31\x34\x37\x3a\x3d"
-			"\x40\x43\x46\x49\x4c\x4f\x52\x55"
-			"\x58\x5b\x5e\x61\x64\x67\x6a\x6d"
-			"\x70\x73\x76\x79\x7c\x7f\x82\x85"
-			"\x88\x8b\x8e\x91\x94\x97\x9a\x9d"
-			"\xa0\xa3\xa6\xa9\xac\xaf\xb2\xb5"
-			"\xb8\xbb\xbe\xc1\xc4\xc7\xca\xcd"
-			"\xd0\xd3\xd6\xd9\xdc\xdf\xe2\xe5"
-			"\xe8\xeb\xee\xf1\xf4\xf7\xfa\xfd"
-			"\x00\x05\x0a\x0f\x14\x19\x1e\x23"
-			"\x28\x2d\x32\x37\x3c\x41\x46\x4b"
-			"\x50\x55\x5a\x5f\x64\x69\x6e\x73"
-			"\x78\x7d\x82\x87\x8c\x91\x96\x9b"
-			"\xa0\xa5\xaa\xaf\xb4\xb9\xbe\xc3"
-			"\xc8\xcd\xd2\xd7\xdc\xe1\xe6\xeb"
-			"\xf0\xf5\xfa\xff\x04\x09\x0e\x13"
-			"\x18\x1d\x22\x27\x2c\x31\x36\x3b"
-			"\x40\x45\x4a\x4f\x54\x59\x5e\x63"
-			"\x68\x6d\x72\x77\x7c\x81\x86\x8b"
-			"\x90\x95\x9a\x9f\xa4\xa9\xae\xb3"
-			"\xb8\xbd\xc2\xc7\xcc\xd1\xd6\xdb"
-			"\xe0\xe5\xea\xef\xf4\xf9\xfe\x03"
-			"\x08\x0d\x12\x17\x1c\x21\x26\x2b"
-			"\x30\x35\x3a\x3f\x44\x49\x4e\x53"
-			"\x58\x5d\x62\x67\x6c\x71\x76\x7b"
-			"\x80\x85\x8a\x8f\x94\x99\x9e\xa3"
-			"\xa8\xad\xb2\xb7\xbc\xc1\xc6\xcb"
-			"\xd0\xd5\xda\xdf\xe4\xe9\xee\xf3"
-			"\xf8\xfd\x02\x07\x0c\x11\x16\x1b"
-			"\x20\x25\x2a\x2f\x34\x39\x3e\x43"
-			"\x48\x4d\x52\x57\x5c\x61\x66\x6b"
-			"\x70\x75\x7a\x7f\x84\x89\x8e\x93"
-			"\x98\x9d\xa2\xa7\xac\xb1\xb6\xbb"
-			"\xc0\xc5\xca\xcf\xd4\xd9\xde\xe3"
-			"\xe8\xed\xf2\xf7\xfc\x01\x06\x0b"
-			"\x10\x15\x1a\x1f\x24\x29\x2e\x33"
-			"\x38\x3d\x42\x47\x4c\x51\x56\x5b"
-			"\x60\x65\x6a\x6f\x74\x79\x7e\x83"
-			"\x88\x8d\x92\x97\x9c\xa1\xa6\xab"
-			"\xb0\xb5\xba\xbf\xc4\xc9\xce\xd3"
-			"\xd8\xdd\xe2\xe7\xec\xf1\xf6\xfb"
-			"\x00\x07\x0e\x15\x1c\x23\x2a\x31"
-			"\x38\x3f\x46\x4d\x54\x5b\x62\x69"
-			"\x70\x77\x7e\x85\x8c\x93\x9a\xa1"
-			"\xa8\xaf\xb6\xbd\xc4\xcb\xd2\xd9"
-			"\xe0\xe7\xee\xf5\xfc\x03\x0a\x11"
-			"\x18\x1f\x26\x2d\x34\x3b\x42\x49"
-			"\x50\x57\x5e\x65\x6c\x73\x7a\x81"
-			"\x88\x8f\x96\x9d\xa4\xab\xb2\xb9"
-			"\xc0\xc7\xce\xd5\xdc\xe3\xea\xf1"
-			"\xf8\xff\x06\x0d\x14\x1b\x22\x29"
-			"\x30\x37\x3e\x45\x4c\x53\x5a\x61"
-			"\x68\x6f\x76\x7d\x84\x8b\x92\x99"
-			"\xa0\xa7\xae\xb5\xbc\xc3\xca\xd1"
-			"\xd8\xdf\xe6\xed\xf4\xfb\x02\x09"
-			"\x10\x17\x1e\x25\x2c\x33\x3a\x41"
-			"\x48\x4f\x56\x5d\x64\x6b\x72\x79"
-			"\x80\x87\x8e\x95\x9c\xa3\xaa\xb1"
-			"\xb8\xbf\xc6\xcd\xd4\xdb\xe2\xe9"
-			"\xf0\xf7\xfe\x05\x0c\x13\x1a\x21"
-			"\x28\x2f\x36\x3d\x44\x4b\x52\x59"
-			"\x60\x67\x6e\x75\x7c\x83\x8a\x91"
-			"\x98\x9f\xa6\xad\xb4\xbb\xc2\xc9"
-			"\xd0\xd7\xde\xe5\xec\xf3\xfa\x01"
-			"\x08\x0f\x16\x1d\x24\x2b\x32\x39"
-			"\x40\x47\x4e\x55\x5c\x63\x6a\x71"
-			"\x78\x7f\x86\x8d\x94\x9b\xa2\xa9"
-			"\xb0\xb7\xbe\xc5\xcc\xd3\xda\xe1"
-			"\xe8\xef\xf6\xfd\x04\x0b\x12\x19"
-			"\x20\x27\x2e\x35\x3c\x43\x4a\x51"
-			"\x58\x5f\x66\x6d\x74\x7b\x82\x89"
-			"\x90\x97\x9e\xa5\xac\xb3\xba\xc1"
-			"\xc8\xcf\xd6\xdd\xe4\xeb\xf2\xf9"
-			"\x00\x09\x12\x1b\x24\x2d\x36\x3f"
-			"\x48\x51\x5a\x63\x6c\x75\x7e\x87"
-			"\x90\x99\xa2\xab\xb4\xbd\xc6\xcf"
-			"\xd8\xe1\xea\xf3\xfc\x05\x0e\x17"
-			"\x20\x29\x32\x3b\x44\x4d\x56\x5f"
-			"\x68\x71\x7a\x83\x8c\x95\x9e\xa7"
-			"\xb0\xb9\xc2\xcb\xd4\xdd\xe6\xef"
-			"\xf8\x01\x0a\x13\x1c\x25\x2e\x37"
-			"\x40\x49\x52\x5b\x64\x6d\x76\x7f"
-			"\x88\x91\x9a\xa3\xac\xb5\xbe\xc7"
-			"\xd0\xd9\xe2\xeb\xf4\xfd\x06\x0f"
-			"\x18\x21\x2a\x33\x3c\x45\x4e\x57"
-			"\x60\x69\x72\x7b\x84\x8d\x96\x9f"
-			"\xa8\xb1\xba\xc3\xcc\xd5\xde\xe7"
-			"\xf0\xf9\x02\x0b\x14\x1d\x26\x2f"
-			"\x38\x41\x4a\x53\x5c\x65\x6e\x77"
-			"\x80\x89\x92\x9b\xa4\xad\xb6\xbf"
-			"\xc8\xd1\xda\xe3\xec\xf5\xfe\x07"
-			"\x10\x19\x22\x2b\x34\x3d\x46\x4f"
-			"\x58\x61\x6a\x73\x7c\x85\x8e\x97"
-			"\xa0\xa9\xb2\xbb\xc4\xcd\xd6\xdf"
-			"\xe8\xf1\xfa\x03\x0c\x15\x1e\x27"
-			"\x30\x39\x42\x4b\x54\x5d\x66\x6f"
-			"\x78\x81\x8a\x93\x9c\xa5\xae\xb7"
-			"\xc0\xc9\xd2\xdb\xe4\xed\xf6\xff"
-			"\x08\x11\x1a\x23\x2c\x35\x3e\x47"
-			"\x50\x59\x62\x6b\x74\x7d\x86\x8f"
-			"\x98\xa1\xaa\xb3\xbc\xc5\xce\xd7"
-			"\xe0\xe9\xf2\xfb\x04\x0d\x16\x1f"
-			"\x28\x31\x3a\x43\x4c\x55\x5e\x67"
-			"\x70\x79\x82\x8b\x94\x9d\xa6\xaf"
-			"\xb8\xc1\xca\xd3\xdc\xe5\xee\xf7"
-			"\x00\x0b\x16\x21\x2c\x37\x42\x4d"
-			"\x58\x63\x6e\x79\x84\x8f\x9a\xa5"
-			"\xb0\xbb\xc6\xd1\xdc\xe7\xf2\xfd"
-			"\x08\x13\x1e\x29\x34\x3f\x4a\x55"
-			"\x60\x6b\x76\x81\x8c\x97\xa2\xad"
-			"\xb8\xc3\xce\xd9\xe4\xef\xfa\x05"
-			"\x10\x1b\x26\x31\x3c\x47\x52\x5d"
-			"\x68\x73\x7e\x89\x94\x9f\xaa\xb5"
-			"\xc0\xcb\xd6\xe1\xec\xf7\x02\x0d"
-			"\x18\x23\x2e\x39\x44\x4f\x5a\x65"
-			"\x70\x7b\x86\x91\x9c\xa7\xb2\xbd"
-			"\xc8\xd3\xde\xe9\xf4\xff\x0a\x15"
-			"\x20\x2b\x36\x41\x4c\x57\x62\x6d"
-			"\x78\x83\x8e\x99\xa4\xaf\xba\xc5"
-			"\xd0\xdb\xe6\xf1\xfc\x07\x12\x1d"
-			"\x28\x33\x3e\x49\x54\x5f\x6a\x75"
-			"\x80\x8b\x96\xa1\xac\xb7\xc2\xcd"
-			"\xd8\xe3\xee\xf9\x04\x0f\x1a\x25"
-			"\x30\x3b\x46\x51\x5c\x67\x72\x7d"
-			"\x88\x93\x9e\xa9\xb4\xbf\xca\xd5"
-			"\xe0\xeb\xf6\x01\x0c\x17\x22\x2d"
-			"\x38\x43\x4e\x59\x64\x6f\x7a\x85"
-			"\x90\x9b\xa6\xb1\xbc\xc7\xd2\xdd"
-			"\xe8\xf3\xfe\x09\x14\x1f\x2a\x35"
-			"\x40\x4b\x56\x61\x6c\x77\x82\x8d"
-			"\x98\xa3\xae\xb9\xc4\xcf\xda\xe5"
-			"\xf0\xfb\x06\x11\x1c\x27\x32\x3d"
-			"\x48\x53\x5e\x69\x74\x7f\x8a\x95"
-			"\xa0\xab\xb6\xc1\xcc\xd7\xe2\xed"
-			"\xf8\x03\x0e\x19\x24\x2f\x3a\x45"
-			"\x50\x5b\x66\x71\x7c\x87\x92\x9d"
-			"\xa8\xb3\xbe\xc9\xd4\xdf\xea\xf5"
-			"\x00\x0d\x1a\x27\x34\x41\x4e\x5b"
-			"\x68\x75\x82\x8f\x9c\xa9\xb6\xc3"
-			"\xd0\xdd\xea\xf7\x04\x11\x1e\x2b"
-			"\x38\x45\x52\x5f\x6c\x79\x86\x93"
-			"\xa0\xad\xba\xc7\xd4\xe1\xee\xfb"
-			"\x08\x15\x22\x2f\x3c\x49\x56\x63"
-			"\x70\x7d\x8a\x97\xa4\xb1\xbe\xcb"
-			"\xd8\xe5\xf2\xff\x0c\x19\x26\x33"
-			"\x40\x4d\x5a\x67\x74\x81\x8e\x9b"
-			"\xa8\xb5\xc2\xcf\xdc\xe9\xf6\x03"
-			"\x10\x1d\x2a\x37\x44\x51\x5e\x6b"
-			"\x78\x85\x92\x9f\xac\xb9\xc6\xd3"
-			"\xe0\xed\xfa\x07\x14\x21\x2e\x3b"
-			"\x48\x55\x62\x6f\x7c\x89\x96\xa3"
-			"\xb0\xbd\xca\xd7\xe4\xf1\xfe\x0b"
-			"\x18\x25\x32\x3f\x4c\x59\x66\x73"
-			"\x80\x8d\x9a\xa7\xb4\xc1\xce\xdb"
-			"\xe8\xf5\x02\x0f\x1c\x29\x36\x43"
-			"\x50\x5d\x6a\x77\x84\x91\x9e\xab"
-			"\xb8\xc5\xd2\xdf\xec\xf9\x06\x13"
-			"\x20\x2d\x3a\x47\x54\x61\x6e\x7b"
-			"\x88\x95\xa2\xaf\xbc\xc9\xd6\xe3"
-			"\xf0\xfd\x0a\x17\x24\x31\x3e\x4b"
-			"\x58\x65\x72\x7f\x8c\x99\xa6\xb3"
-			"\xc0\xcd\xda\xe7\xf4\x01\x0e\x1b"
-			"\x28\x35\x42\x4f\x5c\x69\x76\x83"
-			"\x90\x9d\xaa\xb7\xc4\xd1\xde\xeb"
-			"\xf8\x05\x12\x1f\x2c\x39\x46\x53"
-			"\x60\x6d\x7a\x87\x94\xa1\xae\xbb"
-			"\xc8\xd5\xe2\xef\xfc\x09\x16\x23"
-			"\x30\x3d\x4a\x57\x64\x71\x7e\x8b"
-			"\x98\xa5\xb2\xbf\xcc\xd9\xe6\xf3"
-			"\x00\x0f\x1e\x2d\x3c\x4b\x5a\x69"
-			"\x78\x87\x96\xa5\xb4\xc3\xd2\xe1"
-			"\xf0\xff\x0e\x1d\x2c\x3b\x4a\x59"
-			"\x68\x77\x86\x95\xa4\xb3\xc2\xd1"
-			"\xe0\xef\xfe\x0d\x1c\x2b\x3a\x49"
-			"\x58\x67\x76\x85\x94\xa3\xb2\xc1"
-			"\xd0\xdf\xee\xfd\x0c\x1b\x2a\x39"
-			"\x48\x57\x66\x75\x84\x93\xa2\xb1"
-			"\xc0\xcf\xde\xed\xfc\x0b\x1a\x29"
-			"\x38\x47\x56\x65\x74\x83\x92\xa1"
-			"\xb0\xbf\xce\xdd\xec\xfb\x0a\x19"
-			"\x28\x37\x46\x55\x64\x73\x82\x91"
-			"\xa0\xaf\xbe\xcd\xdc\xeb\xfa\x09"
-			"\x18\x27\x36\x45\x54\x63\x72\x81"
-			"\x90\x9f\xae\xbd\xcc\xdb\xea\xf9"
-			"\x08\x17\x26\x35\x44\x53\x62\x71"
-			"\x80\x8f\x9e\xad\xbc\xcb\xda\xe9"
-			"\xf8\x07\x16\x25\x34\x43\x52\x61"
-			"\x70\x7f\x8e\x9d\xac\xbb\xca\xd9"
-			"\xe8\xf7\x06\x15\x24\x33\x42\x51"
-			"\x60\x6f\x7e\x8d\x9c\xab\xba\xc9"
-			"\xd8\xe7\xf6\x05\x14\x23\x32\x41"
-			"\x50\x5f\x6e\x7d\x8c\x9b\xaa\xb9"
-			"\xc8\xd7\xe6\xf5\x04\x13\x22\x31"
-			"\x40\x4f\x5e\x6d\x7c\x8b\x9a\xa9"
-			"\xb8\xc7\xd6\xe5\xf4\x03\x12\x21"
-			"\x30\x3f\x4e\x5d\x6c\x7b\x8a\x99"
-			"\xa8\xb7\xc6\xd5\xe4\xf3\x02\x11"
-			"\x20\x2f\x3e\x4d\x5c\x6b\x7a\x89"
-			"\x98\xa7\xb6\xc5\xd4\xe3\xf2\x01"
-			"\x10\x1f\x2e\x3d\x4c\x5b\x6a\x79"
-			"\x88\x97\xa6\xb5\xc4\xd3\xe2\xf1"
-			"\x00\x11\x22\x33\x44\x55\x66\x77"
-			"\x88\x99\xaa\xbb\xcc\xdd\xee\xff"
-			"\x10\x21\x32\x43\x54\x65\x76\x87"
-			"\x98\xa9\xba\xcb\xdc\xed\xfe\x0f"
-			"\x20\x31\x42\x53\x64\x75\x86\x97"
-			"\xa8\xb9\xca\xdb\xec\xfd\x0e\x1f"
-			"\x30\x41\x52\x63\x74\x85\x96\xa7"
-			"\xb8\xc9\xda\xeb\xfc\x0d\x1e\x2f"
-			"\x40\x51\x62\x73\x84\x95\xa6\xb7"
-			"\xc8\xd9\xea\xfb\x0c\x1d\x2e\x3f"
-			"\x50\x61\x72\x83\x94\xa5\xb6\xc7"
-			"\xd8\xe9\xfa\x0b\x1c\x2d\x3e\x4f"
-			"\x60\x71\x82\x93\xa4\xb5\xc6\xd7"
-			"\xe8\xf9\x0a\x1b\x2c\x3d\x4e\x5f"
-			"\x70\x81\x92\xa3\xb4\xc5\xd6\xe7"
-			"\xf8\x09\x1a\x2b\x3c\x4d\x5e\x6f"
-			"\x80\x91\xa2\xb3\xc4\xd5\xe6\xf7"
-			"\x08\x19\x2a\x3b\x4c\x5d\x6e\x7f"
-			"\x90\xa1\xb2\xc3\xd4\xe5\xf6\x07"
-			"\x18\x29\x3a\x4b\x5c\x6d\x7e\x8f"
-			"\xa0\xb1\xc2\xd3\xe4\xf5\x06\x17"
-			"\x28\x39\x4a\x5b\x6c\x7d\x8e\x9f"
-			"\xb0\xc1\xd2\xe3\xf4\x05\x16\x27"
-			"\x38\x49\x5a\x6b\x7c\x8d\x9e\xaf"
-			"\xc0\xd1\xe2\xf3\x04\x15\x26\x37"
-			"\x48\x59\x6a\x7b\x8c\x9d\xae\xbf"
-			"\xd0\xe1\xf2\x03\x14\x25\x36\x47"
-			"\x58\x69\x7a\x8b\x9c\xad\xbe\xcf"
-			"\xe0\xf1\x02\x13\x24\x35\x46\x57"
-			"\x68\x79\x8a\x9b\xac\xbd\xce\xdf"
-			"\xf0\x01\x12\x23\x34\x45\x56\x67"
-			"\x78\x89\x9a\xab\xbc\xcd\xde\xef"
-			"\x00\x13\x26\x39\x4c\x5f\x72\x85"
-			"\x98\xab\xbe\xd1\xe4\xf7\x0a\x1d"
-			"\x30\x43\x56\x69\x7c\x8f\xa2\xb5"
-			"\xc8\xdb\xee\x01\x14\x27\x3a\x4d"
-			"\x60\x73\x86\x99\xac\xbf\xd2\xe5"
-			"\xf8\x0b\x1e\x31\x44\x57\x6a\x7d"
-			"\x90\xa3\xb6\xc9\xdc\xef\x02\x15"
-			"\x28\x3b\x4e\x61\x74\x87\x9a\xad"
-			"\xc0\xd3\xe6\xf9\x0c\x1f\x32\x45"
-			"\x58\x6b\x7e\x91\xa4\xb7\xca\xdd"
-			"\xf0\x03\x16\x29\x3c\x4f\x62\x75"
-			"\x88\x9b\xae\xc1\xd4\xe7\xfa\x0d"
-			"\x20\x33\x46\x59\x6c\x7f\x92\xa5"
-			"\xb8\xcb\xde\xf1\x04\x17\x2a\x3d"
-			"\x50\x63\x76\x89\x9c\xaf\xc2\xd5"
-			"\xe8\xfb\x0e\x21\x34\x47\x5a\x6d"
-			"\x80\x93\xa6\xb9\xcc\xdf\xf2\x05"
-			"\x18\x2b\x3e\x51\x64\x77\x8a\x9d"
-			"\xb0\xc3\xd6\xe9\xfc\x0f\x22\x35"
-			"\x48\x5b\x6e\x81\x94\xa7\xba\xcd"
-			"\xe0\xf3\x06\x19\x2c\x3f\x52\x65"
-			"\x78\x8b\x9e\xb1\xc4\xd7\xea\xfd"
-			"\x10\x23\x36\x49\x5c\x6f\x82\x95"
-			"\xa8\xbb\xce\xe1\xf4\x07\x1a\x2d"
-			"\x40\x53\x66\x79\x8c\x9f\xb2\xc5"
-			"\xd8\xeb\xfe\x11\x24\x37\x4a\x5d"
-			"\x70\x83\x96\xa9\xbc\xcf\xe2\xf5"
-			"\x08\x1b\x2e\x41\x54\x67\x7a\x8d"
-			"\xa0\xb3\xc6\xd9\xec\xff\x12\x25"
-			"\x38\x4b\x5e\x71\x84\x97\xaa\xbd"
-			"\xd0\xe3\xf6\x09\x1c\x2f\x42\x55"
-			"\x68\x7b\x8e\xa1\xb4\xc7\xda\xed"
-			"\x00\x15\x2a\x3f\x54\x69\x7e\x93"
-			"\xa8\xbd\xd2\xe7\xfc\x11\x26\x3b"
-			"\x50\x65\x7a\x8f\xa4\xb9\xce\xe3"
-			"\xf8\x0d\x22\x37\x4c\x61\x76\x8b"
-			"\xa0\xb5\xca\xdf\xf4\x09\x1e\x33"
-			"\x48\x5d\x72\x87\x9c\xb1\xc6\xdb"
-			"\xf0\x05\x1a\x2f\x44\x59\x6e\x83"
-			"\x98\xad\xc2\xd7\xec\x01\x16\x2b"
-			"\x40\x55\x6a\x7f\x94\xa9\xbe\xd3"
-			"\xe8\xfd\x12\x27\x3c\x51\x66\x7b"
-			"\x90\xa5\xba\xcf\xe4\xf9\x0e\x23"
-			"\x38\x4d\x62\x77\x8c\xa1\xb6\xcb"
-			"\xe0\xf5\x0a\x1f\x34\x49\x5e\x73"
-			"\x88\x9d\xb2\xc7\xdc\xf1\x06\x1b"
-			"\x30\x45\x5a\x6f\x84\x99\xae\xc3"
-			"\xd8\xed\x02\x17\x2c\x41\x56\x6b"
-			"\x80\x95\xaa\xbf\xd4\xe9\xfe\x13"
-			"\x28\x3d\x52\x67\x7c\x91\xa6\xbb"
-			"\xd0\xe5\xfa\x0f\x24\x39\x4e\x63"
-			"\x78\x8d\xa2\xb7\xcc\xe1\xf6\x0b"
-			"\x20\x35\x4a\x5f\x74\x89\x9e\xb3"
-			"\xc8\xdd\xf2\x07\x1c\x31\x46\x5b"
-			"\x70\x85\x9a\xaf\xc4\xd9\xee\x03"
-			"\x18\x2d\x42\x57\x6c\x81\x96\xab"
-			"\xc0\xd5\xea\xff\x14\x29\x3e\x53"
-			"\x68\x7d\x92\xa7\xbc\xd1\xe6\xfb"
-			"\x10\x25\x3a\x4f\x64\x79\x8e\xa3"
-			"\xb8\xcd\xe2\xf7\x0c\x21\x36\x4b"
-			"\x60\x75\x8a\x9f\xb4\xc9\xde\xf3"
-			"\x08\x1d\x32\x47\x5c\x71\x86\x9b"
-			"\xb0\xc5\xda\xef\x04\x19\x2e\x43"
-			"\x58\x6d\x82\x97\xac\xc1\xd6\xeb"
-			"\x00\x17\x2e\x45\x5c\x73\x8a\xa1"
-			"\xb8\xcf\xe6\xfd\x14\x2b\x42\x59"
-			"\x70\x87\x9e\xb5\xcc\xe3\xfa\x11"
-			"\x28\x3f\x56\x6d\x84\x9b\xb2\xc9"
-			"\xe0\xf7\x0e\x25\x3c\x53\x6a\x81"
-			"\x98\xaf\xc6\xdd\xf4\x0b\x22\x39"
-			"\x50\x67\x7e\x95\xac\xc3\xda\xf1"
-			"\x08\x1f\x36\x4d\x64\x7b\x92\xa9"
-			"\xc0\xd7\xee\x05\x1c\x33\x4a\x61"
-			"\x78\x8f\xa6\xbd\xd4\xeb\x02\x19"
-			"\x30\x47\x5e\x75\x8c\xa3\xba\xd1"
-			"\xe8\xff\x16\x2d\x44\x5b\x72\x89"
-			"\xa0\xb7\xce\xe5\xfc\x13\x2a\x41"
-			"\x58\x6f\x86\x9d\xb4\xcb\xe2\xf9"
-			"\x10\x27\x3e\x55\x6c\x83\x9a\xb1"
-			"\xc8\xdf\xf6\x0d\x24\x3b\x52\x69"
-			"\x80\x97\xae\xc5\xdc\xf3\x0a\x21"
-			"\x38\x4f\x66\x7d\x94\xab\xc2\xd9"
-			"\xf0\x07\x1e\x35\x4c\x63\x7a\x91"
-			"\xa8\xbf\xd6\xed\x04\x1b\x32\x49"
-			"\x60\x77\x8e\xa5\xbc\xd3\xea\x01"
-			"\x18\x2f\x46\x5d\x74\x8b\xa2\xb9"
-			"\xd0\xe7\xfe\x15\x2c\x43\x5a\x71"
-			"\x88\x9f\xb6\xcd\xe4\xfb\x12\x29"
-			"\x40\x57\x6e\x85\x9c\xb3\xca\xe1"
-			"\xf8\x0f\x26\x3d\x54\x6b\x82\x99"
-			"\xb0\xc7\xde\xf5\x0c\x23\x3a\x51"
-			"\x68\x7f\x96\xad\xc4\xdb\xf2\x09"
-			"\x20\x37\x4e\x65\x7c\x93\xaa\xc1"
-			"\xd8\xef\x06\x1d\x34\x4b\x62\x79"
-			"\x90\xa7\xbe\xd5\xec\x03\x1a\x31"
-			"\x48\x5f\x76\x8d\xa4\xbb\xd2\xe9"
-			"\x00\x19\x32\x4b\x64\x7d\x96\xaf"
-			"\xc8\xe1\xfa\x13\x2c\x45\x5e\x77"
-			"\x90\xa9\xc2\xdb\xf4\x0d\x26\x3f"
-			"\x58\x71\x8a\xa3\xbc\xd5\xee\x07"
-			"\x20\x39\x52\x6b\x84\x9d\xb6\xcf"
-			"\xe8\x01\x1a\x33\x4c\x65\x7e\x97"
-			"\xb0\xc9\xe2\xfb\x14\x2d\x46\x5f"
-			"\x78\x91\xaa\xc3\xdc\xf5\x0e\x27"
-			"\x40\x59\x72\x8b\xa4\xbd\xd6\xef"
-			"\x08\x21\x3a\x53\x6c\x85\x9e\xb7"
-			"\xd0\xe9\x02\x1b\x34\x4d\x66\x7f"
-			"\x98\xb1\xca\xe3\xfc\x15\x2e\x47"
-			"\x60\x79\x92\xab\xc4\xdd\xf6\x0f"
-			"\x28\x41\x5a\x73\x8c\xa5\xbe\xd7"
-			"\xf0\x09\x22\x3b\x54\x6d\x86\x9f"
-			"\xb8\xd1\xea\x03\x1c\x35\x4e\x67"
-			"\x80\x99\xb2\xcb\xe4\xfd\x16\x2f"
-			"\x48\x61\x7a\x93\xac\xc5\xde\xf7"
-			"\x10\x29\x42\x5b\x74\x8d\xa6\xbf"
-			"\xd8\xf1\x0a\x23\x3c\x55\x6e\x87"
-			"\xa0\xb9\xd2\xeb\x04\x1d\x36\x4f"
-			"\x68\x81\x9a\xb3\xcc\xe5\xfe\x17"
-			"\x30\x49\x62\x7b\x94\xad\xc6\xdf"
-			"\xf8\x11\x2a\x43\x5c\x75\x8e\xa7"
-			"\xc0\xd9\xf2\x0b\x24\x3d\x56\x6f"
-			"\x88\xa1\xba\xd3\xec\x05\x1e\x37"
-			"\x50\x69\x82\x9b\xb4\xcd\xe6\xff"
-			"\x18\x31\x4a\x63\x7c\x95\xae\xc7"
-			"\xe0\xf9\x12\x2b\x44\x5d\x76\x8f"
-			"\xa8\xc1\xda\xf3\x0c\x25\x3e\x57"
-			"\x70\x89\xa2\xbb\xd4\xed\x06\x1f"
-			"\x38\x51\x6a\x83\x9c\xb5\xce\xe7"
-			"\x00\x1b\x36\x51\x6c\x87\xa2\xbd"
-			"\xd8\xf3\x0e\x29\x44\x5f\x7a\x95"
-			"\xb0\xcb\xe6\x01\x1c\x37\x52\x6d"
-			"\x88\xa3\xbe\xd9\xf4\x0f\x2a\x45"
-			"\x60\x7b\x96\xb1\xcc\xe7\x02\x1d"
-			"\x38\x53\x6e\x89\xa4\xbf\xda\xf5"
-			"\x10\x2b\x46\x61\x7c\x97\xb2\xcd"
-			"\xe8\x03\x1e\x39\x54\x6f\x8a\xa5"
-			"\xc0\xdb\xf6\x11\x2c\x47\x62\x7d"
-			"\x98\xb3\xce\xe9\x04\x1f\x3a\x55"
-			"\x70\x8b\xa6\xc1\xdc\xf7\x12\x2d"
-			"\x48\x63\x7e\x99\xb4\xcf\xea\x05"
-			"\x20\x3b\x56\x71\x8c\xa7\xc2\xdd"
-			"\xf8\x13\x2e\x49\x64\x7f\x9a\xb5"
-			"\xd0\xeb\x06\x21\x3c\x57\x72\x8d"
-			"\xa8\xc3\xde\xf9\x14\x2f\x4a\x65"
-			"\x80\x9b\xb6\xd1\xec\x07\x22\x3d"
-			"\x58\x73\x8e\xa9\xc4\xdf\xfa\x15"
-			"\x30\x4b\x66\x81\x9c\xb7\xd2\xed"
-			"\x08\x23\x3e\x59\x74\x8f\xaa\xc5"
-			"\xe0\xfb\x16\x31\x4c\x67\x82\x9d"
-			"\xb8\xd3\xee\x09\x24\x3f\x5a\x75"
-			"\x90\xab\xc6\xe1\xfc\x17\x32\x4d"
-			"\x68\x83\x9e\xb9\xd4\xef\x0a\x25"
-			"\x40\x5b\x76\x91\xac\xc7\xe2\xfd"
-			"\x18\x33\x4e\x69\x84\x9f\xba\xd5"
-			"\xf0\x0b\x26\x41\x5c\x77\x92\xad"
-			"\xc8\xe3\xfe\x19\x34\x4f\x6a\x85"
-			"\xa0\xbb\xd6\xf1\x0c\x27\x42\x5d"
-			"\x78\x93\xae\xc9\xe4\xff\x1a\x35"
-			"\x50\x6b\x86\xa1\xbc\xd7\xf2\x0d"
-			"\x28\x43\x5e\x79\x94\xaf\xca\xe5"
-			"\x00\x1d\x3a\x57\x74\x91\xae\xcb"
-			"\xe8\x05\x22\x3f\x5c\x79\x96\xb3"
-			"\xd0\xed\x0a\x27\x44\x61\x7e\x9b"
-			"\xb8\xd5\xf2\x0f\x2c\x49\x66\x83"
-			"\xa0\xbd\xda\xf7\x14\x31\x4e\x6b"
-			"\x88\xa5\xc2\xdf\xfc\x19\x36\x53"
-			"\x70\x8d\xaa\xc7\xe4\x01\x1e\x3b"
-			"\x58\x75\x92\xaf\xcc\xe9\x06\x23"
-			"\x40\x5d\x7a\x97\xb4\xd1\xee\x0b"
-			"\x28\x45\x62\x7f\x9c\xb9\xd6\xf3"
-			"\x10\x2d\x4a\x67\x84\xa1\xbe\xdb"
-			"\xf8\x15\x32\x4f\x6c\x89\xa6\xc3"
-			"\xe0\xfd\x1a\x37\x54\x71\x8e\xab"
-			"\xc8\xe5\x02\x1f\x3c\x59\x76\x93"
-			"\xb0\xcd\xea\x07\x24\x41\x5e\x7b"
-			"\x98\xb5\xd2\xef\x0c\x29\x46\x63"
-			"\x80\x9d\xba\xd7\xf4\x11\x2e\x4b"
-			"\x68\x85\xa2\xbf\xdc\xf9\x16\x33"
-			"\x50\x6d\x8a\xa7\xc4\xe1\xfe\x1b"
-			"\x38\x55\x72\x8f\xac\xc9\xe6\x03"
-			"\x20\x3d\x5a\x77\x94\xb1\xce\xeb"
-			"\x08\x25\x42\x5f\x7c\x99\xb6\xd3"
-			"\xf0\x0d\x2a\x47\x64\x81\x9e\xbb"
-			"\xd8\xf5\x12\x2f\x4c\x69\x86\xa3"
-			"\xc0\xdd\xfa\x17\x34\x51\x6e\x8b"
-			"\xa8\xc5\xe2\xff\x1c\x39\x56\x73"
-			"\x90\xad\xca\xe7\x04\x21\x3e\x5b"
-			"\x78\x95\xb2\xcf\xec\x09\x26\x43"
-			"\x60\x7d\x9a\xb7\xd4\xf1\x0e\x2b"
-			"\x48\x65\x82\x9f\xbc\xd9\xf6\x13"
-			"\x30\x4d\x6a\x87\xa4\xc1\xde\xfb"
-			"\x18\x35\x52\x6f\x8c\xa9\xc6\xe3"
-			"\x00\x1f\x3e\x5d\x7c\x9b\xba\xd9"
-			"\xf8\x17\x36\x55\x74\x93\xb2\xd1"
-			"\xf0\x0f\x2e\x4d\x6c\x8b\xaa\xc9"
-			"\xe8\x07\x26\x45\x64\x83\xa2\xc1"
-			"\xe0\xff\x1e\x3d\x5c\x7b\x9a\xb9"
-			"\xd8\xf7\x16\x35\x54\x73\x92\xb1"
-			"\xd0\xef\x0e\x2d\x4c\x6b\x8a\xa9"
-			"\xc8\xe7\x06\x25\x44\x63\x82\xa1"
-			"\xc0\xdf\xfe\x1d\x3c\x5b\x7a\x99"
-			"\xb8\xd7\xf6\x15\x34\x53\x72\x91"
-			"\xb0\xcf\xee\x0d\x2c\x4b\x6a\x89"
-			"\xa8\xc7\xe6\x05\x24\x43\x62\x81"
-			"\xa0\xbf\xde\xfd\x1c\x3b\x5a\x79"
-			"\x98\xb7\xd6\xf5\x14\x33\x52\x71"
-			"\x90\xaf\xce\xed\x0c\x2b\x4a\x69"
-			"\x88\xa7\xc6\xe5\x04\x23\x42\x61"
-			"\x80\x9f\xbe\xdd\xfc\x1b\x3a\x59"
-			"\x78\x97\xb6\xd5\xf4\x13\x32\x51"
-			"\x70\x8f\xae\xcd\xec\x0b\x2a\x49"
-			"\x68\x87\xa6\xc5\xe4\x03\x22\x41"
-			"\x60\x7f\x9e\xbd\xdc\xfb\x1a\x39"
-			"\x58\x77\x96\xb5\xd4\xf3\x12\x31"
-			"\x50\x6f\x8e\xad\xcc\xeb\x0a\x29"
-			"\x48\x67\x86\xa5\xc4\xe3\x02\x21"
-			"\x40\x5f\x7e\x9d\xbc\xdb\xfa\x19"
-			"\x38\x57\x76\x95\xb4\xd3\xf2\x11"
-			"\x30\x4f\x6e\x8d\xac\xcb\xea\x09"
-			"\x28\x47\x66\x85\xa4\xc3\xe2\x01"
-			"\x20\x3f\x5e\x7d\x9c\xbb\xda\xf9"
-			"\x18\x37\x56\x75\x94\xb3\xd2\xf1"
-			"\x10\x2f\x4e\x6d\x8c\xab\xca\xe9"
-			"\x08\x27\x46\x65\x84\xa3\xc2\xe1"
-			"\x00\x21\x42\x63",
-		.ilen = 4100,
-		.result =
-			"\xf0\x5c\x74\xad\x4e\xbc\x99\xe2"
-			"\xae\xff\x91\x3a\x44\xcf\x38\x32"
-			"\x1e\xad\xa7\xcd\xa1\x39\x95\xaa"
-			"\x10\xb1\xb3\x2e\x04\x31\x8f\x86"
-			"\xf2\x62\x74\x70\x0c\xa4\x46\x08"
-			"\xa8\xb7\x99\xa8\xe9\xd2\x73\x79"
-			"\x7e\x6e\xd4\x8f\x1e\xc7\x8e\x31"
-			"\x0b\xfa\x4b\xce\xfd\xf3\x57\x71"
-			"\xe9\x46\x03\xa5\x3d\x34\x00\xe2"
-			"\x18\xff\x75\x6d\x06\x2d\x00\xab"
-			"\xb9\x3e\x6c\x59\xc5\x84\x06\xb5"
-			"\x8b\xd0\x89\x9c\x4a\x79\x16\xc6"
-			"\x3d\x74\x54\xfa\x44\xcd\x23\x26"
-			"\x5c\xcf\x7e\x28\x92\x32\xbf\xdf"
-			"\xa7\x20\x3c\x74\x58\x2a\x9a\xde"
-			"\x61\x00\x1c\x4f\xff\x59\xc4\x22"
-			"\xac\x3c\xd0\xe8\x6c\xf9\x97\x1b"
-			"\x58\x9b\xad\x71\xe8\xa9\xb5\x0d"
-			"\xee\x2f\x04\x1f\x7f\xbc\x99\xee"
-			"\x84\xff\x42\x60\xdc\x3a\x18\xa5"
-			"\x81\xf9\xef\xdc\x7a\x0f\x65\x41"
-			"\x2f\xa3\xd3\xf9\xc2\xcb\xc0\x4d"
-			"\x8f\xd3\x76\x96\xad\x49\x6d\x38"
-			"\x3d\x39\x0b\x6c\x80\xb7\x54\x69"
-			"\xf0\x2c\x90\x02\x29\x0d\x1c\x12"
-			"\xad\x55\xc3\x8b\x68\xd9\xcc\xb3"
-			"\xb2\x64\x33\x90\x5e\xca\x4b\xe2"
-			"\xfb\x75\xdc\x63\xf7\x9f\x82\x74"
-			"\xf0\xc9\xaa\x7f\xe9\x2a\x9b\x33"
-			"\xbc\x88\x00\x7f\xca\xb2\x1f\x14"
-			"\xdb\xc5\x8e\x7b\x11\x3c\x3e\x08"
-			"\xf3\x83\xe8\xe0\x94\x86\x2e\x92"
-			"\x78\x6b\x01\xc9\xc7\x83\xba\x21"
-			"\x6a\x25\x15\x33\x4e\x45\x08\xec"
-			"\x35\xdb\xe0\x6e\x31\x51\x79\xa9"
-			"\x42\x44\x65\xc1\xa0\xf1\xf9\x2a"
-			"\x70\xd5\xb6\xc6\xc1\x8c\x39\xfc"
-			"\x25\xa6\x55\xd9\xdd\x2d\x4c\xec"
-			"\x49\xc6\xeb\x0e\xa8\x25\x2a\x16"
-			"\x1b\x66\x84\xda\xe2\x92\xe5\xc0"
-			"\xc8\x53\x07\xaf\x80\x84\xec\xfd"
-			"\xcd\xd1\x6e\xcd\x6f\x6a\xf5\x36"
-			"\xc5\x15\xe5\x25\x7d\x77\xd1\x1a"
-			"\x93\x36\xa9\xcf\x7c\xa4\x54\x4a"
-			"\x06\x51\x48\x4e\xf6\x59\x87\xd2"
-			"\x04\x02\xef\xd3\x44\xde\x76\x31"
-			"\xb3\x34\x17\x1b\x9d\x66\x11\x9f"
-			"\x1e\xcc\x17\xe9\xc7\x3c\x1b\xe7"
-			"\xcb\x50\x08\xfc\xdc\x2b\x24\xdb"
-			"\x65\x83\xd0\x3b\xe3\x30\xea\x94"
-			"\x6c\xe7\xe8\x35\x32\xc7\xdb\x64"
-			"\xb4\x01\xab\x36\x2c\x77\x13\xaf"
-			"\xf8\x2b\x88\x3f\x54\x39\xc4\x44"
-			"\xfe\xef\x6f\x68\x34\xbe\x0f\x05"
-			"\x16\x6d\xf6\x0a\x30\xe7\xe3\xed"
-			"\xc4\xde\x3c\x1b\x13\xd8\xdb\xfe"
-			"\x41\x62\xe5\x28\xd4\x8d\xa3\xc7"
-			"\x93\x97\xc6\x48\x45\x1d\x9f\x83"
-			"\xdf\x4b\x40\x3e\x42\x25\x87\x80"
-			"\x4c\x7d\xa8\xd4\x98\x23\x95\x75"
-			"\x41\x8c\xda\x41\x9b\xd4\xa7\x06"
-			"\xb5\xf1\x71\x09\x53\xbe\xca\xbf"
-			"\x32\x03\xed\xf0\x50\x1c\x56\x39"
-			"\x5b\xa4\x75\x18\xf7\x9b\x58\xef"
-			"\x53\xfc\x2a\x38\x23\x15\x75\xcd"
-			"\x45\xe5\x5a\x82\x55\xba\x21\xfa"
-			"\xd4\xbd\xc6\x94\x7c\xc5\x80\x12"
-			"\xf7\x4b\x32\xc4\x9a\x82\xd8\x28"
-			"\x8f\xd9\xc2\x0f\x60\x03\xbe\x5e"
-			"\x21\xd6\x5f\x58\xbf\x5c\xb1\x32"
-			"\x82\x8d\xa9\xe5\xf2\x66\x1a\xc0"
-			"\xa0\xbc\x58\x2f\x71\xf5\x2f\xed"
-			"\xd1\x26\xb9\xd8\x49\x5a\x07\x19"
-			"\x01\x7c\x59\xb0\xf8\xa4\xb7\xd3"
-			"\x7b\x1a\x8c\x38\xf4\x50\xa4\x59"
-			"\xb0\xcc\x41\x0b\x88\x7f\xe5\x31"
-			"\xb3\x42\xba\xa2\x7e\xd4\x32\x71"
-			"\x45\x87\x48\xa9\xc2\xf2\x89\xb3"
-			"\xe4\xa7\x7e\x52\x15\x61\xfa\xfe"
-			"\xc9\xdd\x81\xeb\x13\xab\xab\xc3"
-			"\x98\x59\xd8\x16\x3d\x14\x7a\x1c"
-			"\x3c\x41\x9a\x16\x16\x9b\xd2\xd2"
-			"\x69\x3a\x29\x23\xac\x86\x32\xa5"
-			"\x48\x9c\x9e\xf3\x47\x77\x81\x70"
-			"\x24\xe8\x85\xd2\xf5\xb5\xfa\xff"
-			"\x59\x6a\xd3\x50\x59\x43\x59\xde"
-			"\xd9\xf1\x55\xa5\x0c\xc3\x1a\x1a"
-			"\x18\x34\x0d\x1a\x63\x33\xed\x10"
-			"\xe0\x1d\x2a\x18\xd2\xc0\x54\xa8"
-			"\xca\xb5\x9a\xd3\xdd\xca\x45\x84"
-			"\x50\xe7\x0f\xfe\xa4\x99\x5a\xbe"
-			"\x43\x2d\x9a\xcb\x92\x3f\x5a\x1d"
-			"\x85\xd8\xc9\xdf\x68\xc9\x12\x80"
-			"\x56\x0c\xdc\x00\xdc\x3a\x7d\x9d"
-			"\xa3\xa2\xe8\x4d\xbf\xf9\x70\xa0"
-			"\xa4\x13\x4f\x6b\xaf\x0a\x89\x7f"
-			"\xda\xf0\xbf\x9b\xc8\x1d\xe5\xf8"
-			"\x2e\x8b\x07\xb5\x73\x1b\xcc\xa2"
-			"\xa6\xad\x30\xbc\x78\x3c\x5b\x10"
-			"\xfa\x5e\x62\x2d\x9e\x64\xb3\x33"
-			"\xce\xf9\x1f\x86\xe7\x8b\xa2\xb8"
-			"\xe8\x99\x57\x8c\x11\xed\x66\xd9"
-			"\x3c\x72\xb9\xc3\xe6\x4e\x17\x3a"
-			"\x6a\xcb\x42\x24\x06\xed\x3e\x4e"
-			"\xa3\xe8\x6a\x94\xda\x0d\x4e\xd5"
-			"\x14\x19\xcf\xb6\x26\xd8\x2e\xcc"
-			"\x64\x76\x38\x49\x4d\xfe\x30\x6d"
-			"\xe4\xc8\x8c\x7b\xc4\xe0\x35\xba"
-			"\x22\x6e\x76\xe1\x1a\xf2\x53\xc3"
-			"\x28\xa2\x82\x1f\x61\x69\xad\xc1"
-			"\x7b\x28\x4b\x1e\x6c\x85\x95\x9b"
-			"\x51\xb5\x17\x7f\x12\x69\x8c\x24"
-			"\xd5\xc7\x5a\x5a\x11\x54\xff\x5a"
-			"\xf7\x16\xc3\x91\xa6\xf0\xdc\x0a"
-			"\xb6\xa7\x4a\x0d\x7a\x58\xfe\xa5"
-			"\xf5\xcb\x8f\x7b\x0e\xea\x57\xe7"
-			"\xbd\x79\xd6\x1c\x88\x23\x6c\xf2"
-			"\x4d\x29\x77\x53\x35\x6a\x00\x8d"
-			"\xcd\xa3\x58\xbe\x77\x99\x18\xf8"
-			"\xe6\xe1\x8f\xe9\x37\x8f\xe3\xe2"
-			"\x5a\x8a\x93\x25\xaf\xf3\x78\x80"
-			"\xbe\xa6\x1b\xc6\xac\x8b\x1c\x91"
-			"\x58\xe1\x9f\x89\x35\x9d\x1d\x21"
-			"\x29\x9f\xf4\x99\x02\x27\x0f\xa8"
-			"\x4f\x79\x94\x2b\x33\x2c\xda\xa2"
-			"\x26\x39\x83\x94\xef\x27\xd8\x53"
-			"\x8f\x66\x0d\xe4\x41\x7d\x34\xcd"
-			"\x43\x7c\x95\x0a\x53\xef\x66\xda"
-			"\x7e\x9b\xf3\x93\xaf\xd0\x73\x71"
-			"\xba\x40\x9b\x74\xf8\xd7\xd7\x41"
-			"\x6d\xaf\x72\x9c\x8d\x21\x87\x3c"
-			"\xfd\x0a\x90\xa9\x47\x96\x9e\xd3"
-			"\x88\xee\x73\xcf\x66\x2f\x52\x56"
-			"\x6d\xa9\x80\x4c\xe2\x6f\x62\x88"
-			"\x3f\x0e\x54\x17\x48\x80\x5d\xd3"
-			"\xc3\xda\x25\x3d\xa1\xc8\xcb\x9f"
-			"\x9b\x70\xb3\xa1\xeb\x04\x52\xa1"
-			"\xf2\x22\x0f\xfc\xc8\x18\xfa\xf9"
-			"\x85\x9c\xf1\xac\xeb\x0c\x02\x46"
-			"\x75\xd2\xf5\x2c\xe3\xd2\x59\x94"
-			"\x12\xf3\x3c\xfc\xd7\x92\xfa\x36"
-			"\xba\x61\x34\x38\x7c\xda\x48\x3e"
-			"\x08\xc9\x39\x23\x5e\x02\x2c\x1a"
-			"\x18\x7e\xb4\xd9\xfd\x9e\x40\x02"
-			"\xb1\x33\x37\x32\xe7\xde\xd6\xd0"
-			"\x7c\x58\x65\x4b\xf8\x34\x27\x9c"
-			"\x44\xb4\xbd\xe9\xe9\x4c\x78\x7d"
-			"\x4b\x9f\xce\xb1\xcd\x47\xa5\x37"
-			"\xe5\x6d\xbd\xb9\x43\x94\x0a\xd4"
-			"\xd6\xf9\x04\x5f\xb5\x66\x6c\x1a"
-			"\x35\x12\xe3\x36\x28\x27\x36\x58"
-			"\x01\x2b\x79\xe4\xba\x6d\x10\x7d"
-			"\x65\xdf\x84\x95\xf4\xd5\xb6\x8f"
-			"\x2b\x9f\x96\x00\x86\x60\xf0\x21"
-			"\x76\xa8\x6a\x8c\x28\x1c\xb3\x6b"
-			"\x97\xd7\xb6\x53\x2a\xcc\xab\x40"
-			"\x9d\x62\x79\x58\x52\xe6\x65\xb7"
-			"\xab\x55\x67\x9c\x89\x7c\x03\xb0"
-			"\x73\x59\xc5\x81\xf5\x18\x17\x5c"
-			"\x89\xf3\x78\x35\x44\x62\x78\x72"
-			"\xd0\x96\xeb\x31\xe7\x87\x77\x14"
-			"\x99\x51\xf2\x59\x26\x9e\xb5\xa6"
-			"\x45\xfe\x6e\xbd\x07\x4c\x94\x5a"
-			"\xa5\x7d\xfc\xf1\x2b\x77\xe2\xfe"
-			"\x17\xd4\x84\xa0\xac\xb5\xc7\xda"
-			"\xa9\x1a\xb6\xf3\x74\x11\xb4\x9d"
-			"\xfb\x79\x2e\x04\x2d\x50\x28\x83"
-			"\xbf\xc6\x52\xd3\x34\xd6\xe8\x7a"
-			"\xb6\xea\xe7\xa8\x6c\x15\x1e\x2c"
-			"\x57\xbc\x48\x4e\x5f\x5c\xb6\x92"
-			"\xd2\x49\x77\x81\x6d\x90\x70\xae"
-			"\x98\xa1\x03\x0d\x6b\xb9\x77\x14"
-			"\xf1\x4e\x23\xd3\xf8\x68\xbd\xc2"
-			"\xfe\x04\xb7\x5c\xc5\x17\x60\x8f"
-			"\x65\x54\xa4\x7a\x42\xdc\x18\x0d"
-			"\xb5\xcf\x0f\xd3\xc7\x91\x66\x1b"
-			"\x45\x42\x27\x75\x50\xe5\xee\xb8"
-			"\x7f\x33\x2c\xba\x4a\x92\x4d\x2c"
-			"\x3c\xe3\x0d\x80\x01\xba\x0d\x29"
-			"\xd8\x3c\xe9\x13\x16\x57\xe6\xea"
-			"\x94\x52\xe7\x00\x4d\x30\xb0\x0f"
-			"\x35\xb8\xb8\xa7\xb1\xb5\x3b\x44"
-			"\xe1\x2f\xfd\x88\xed\x43\xe7\x52"
-			"\x10\x93\xb3\x8a\x30\x6b\x0a\xf7"
-			"\x23\xc6\x50\x9d\x4a\xb0\xde\xc3"
-			"\xdc\x9b\x2f\x01\x56\x36\x09\xc5"
-			"\x2f\x6b\xfe\xf1\xd8\x27\x45\x03"
-			"\x30\x5e\x5c\x5b\xb4\x62\x0e\x1a"
-			"\xa9\x21\x2b\x92\x94\x87\x62\x57"
-			"\x4c\x10\x74\x1a\xf1\x0a\xc5\x84"
-			"\x3b\x9e\x72\x02\xd7\xcc\x09\x56"
-			"\xbd\x54\xc1\xf0\xc3\xe3\xb3\xf8"
-			"\xd2\x0d\x61\xcb\xef\xce\x0d\x05"
-			"\xb0\x98\xd9\x8e\x4f\xf9\xbc\x93"
-			"\xa6\xea\xc8\xcf\x10\x53\x4b\xf1"
-			"\xec\xfc\x89\xf9\x64\xb0\x22\xbf"
-			"\x9e\x55\x46\x9f\x7c\x50\x8e\x84"
-			"\x54\x20\x98\xd7\x6c\x40\x1e\xdb"
-			"\x69\x34\x78\x61\x24\x21\x9c\x8a"
-			"\xb3\x62\x31\x8b\x6e\xf5\x2a\x35"
-			"\x86\x13\xb1\x6c\x64\x2e\x41\xa5"
-			"\x05\xf2\x42\xba\xd2\x3a\x0d\x8e"
-			"\x8a\x59\x94\x3c\xcf\x36\x27\x82"
-			"\xc2\x45\xee\x58\xcd\x88\xb4\xec"
-			"\xde\xb2\x96\x0a\xaf\x38\x6f\x88"
-			"\xd7\xd8\xe1\xdf\xb9\x96\xa9\x0a"
-			"\xb1\x95\x28\x86\x20\xe9\x17\x49"
-			"\xa2\x29\x38\xaa\xa5\xe9\x6e\xf1"
-			"\x19\x27\xc0\xd5\x2a\x22\xc3\x0b"
-			"\xdb\x7c\x73\x10\xb9\xba\x89\x76"
-			"\x54\xae\x7d\x71\xb3\x93\xf6\x32"
-			"\xe6\x47\x43\x55\xac\xa0\x0d\xc2"
-			"\x93\x27\x4a\x8e\x0e\x74\x15\xc7"
-			"\x0b\x85\xd9\x0c\xa9\x30\x7a\x3e"
-			"\xea\x8f\x85\x6d\x3a\x12\x4f\x72"
-			"\x69\x58\x7a\x80\xbb\xb5\x97\xf3"
-			"\xcf\x70\xd2\x5d\xdd\x4d\x21\x79"
-			"\x54\x4d\xe4\x05\xe8\xbd\xc2\x62"
-			"\xb1\x3b\x77\x1c\xd6\x5c\xf3\xa0"
-			"\x79\x00\xa8\x6c\x29\xd9\x18\x24"
-			"\x36\xa2\x46\xc0\x96\x65\x7f\xbd"
-			"\x2a\xed\x36\x16\x0c\xaa\x9f\xf4"
-			"\xc5\xb4\xe2\x12\xed\x69\xed\x4f"
-			"\x26\x2c\x39\x52\x89\x98\xe7\x2c"
-			"\x99\xa4\x9e\xa3\x9b\x99\x46\x7a"
-			"\x3a\xdc\xa8\x59\xa3\xdb\xc3\x3b"
-			"\x95\x0d\x3b\x09\x6e\xee\x83\x5d"
-			"\x32\x4d\xed\xab\xfa\x98\x14\x4e"
-			"\xc3\x15\x45\x53\x61\xc4\x93\xbd"
-			"\x90\xf4\x99\x95\x4c\xe6\x76\x92"
-			"\x29\x90\x46\x30\x92\x69\x7d\x13"
-			"\xf2\xa5\xcd\x69\x49\x44\xb2\x0f"
-			"\x63\x40\x36\x5f\x09\xe2\x78\xf8"
-			"\x91\xe3\xe2\xfa\x10\xf7\xc8\x24"
-			"\xa8\x89\x32\x5c\x37\x25\x1d\xb2"
-			"\xea\x17\x8a\x0a\xa9\x64\xc3\x7c"
-			"\x3c\x7c\xbd\xc6\x79\x34\xe7\xe2"
-			"\x85\x8e\xbf\xf8\xde\x92\xa0\xae"
-			"\x20\xc4\xf6\xbb\x1f\x38\x19\x0e"
-			"\xe8\x79\x9c\xa1\x23\xe9\x54\x7e"
-			"\x37\x2f\xe2\x94\x32\xaf\xa0\x23"
-			"\x49\xe4\xc0\xb3\xac\x00\x8f\x36"
-			"\x05\xc4\xa6\x96\xec\x05\x98\x4f"
-			"\x96\x67\x57\x1f\x20\x86\x1b\x2d"
-			"\x69\xe4\x29\x93\x66\x5f\xaf\x6b"
-			"\x88\x26\x2c\x67\x02\x4b\x52\xd0"
-			"\x83\x7a\x43\x1f\xc0\x71\x15\x25"
-			"\x77\x65\x08\x60\x11\x76\x4c\x8d"
-			"\xed\xa9\x27\xc6\xb1\x2a\x2c\x6a"
-			"\x4a\x97\xf5\xc6\xb7\x70\x42\xd3"
-			"\x03\xd1\x24\x95\xec\x6d\xab\x38"
-			"\x72\xce\xe2\x8b\x33\xd7\x51\x09"
-			"\xdc\x45\xe0\x09\x96\x32\xf3\xc4"
-			"\x84\xdc\x73\x73\x2d\x1b\x11\x98"
-			"\xc5\x0e\x69\x28\x94\xc7\xb5\x4d"
-			"\xc8\x8a\xd0\xaa\x13\x2e\x18\x74"
-			"\xdd\xd1\x1e\xf3\x90\xe8\xfc\x9a"
-			"\x72\x4a\x0e\xd1\xe4\xfb\x0d\x96"
-			"\xd1\x0c\x79\x85\x1b\x1c\xfe\xe1"
-			"\x62\x8f\x7a\x73\x32\xab\xc8\x18"
-			"\x69\xe3\x34\x30\xdf\x13\xa6\xe5"
-			"\xe8\x0e\x67\x7f\x81\x11\xb4\x60"
-			"\xc7\xbd\x79\x65\x50\xdc\xc4\x5b"
-			"\xde\x39\xa4\x01\x72\x63\xf3\xd1"
-			"\x64\x4e\xdf\xfc\x27\x92\x37\x0d"
-			"\x57\xcd\x11\x4f\x11\x04\x8e\x1d"
-			"\x16\xf7\xcd\x92\x9a\x99\x30\x14"
-			"\xf1\x7c\x67\x1b\x1f\x41\x0b\xe8"
-			"\x32\xe8\xb8\xc1\x4f\x54\x86\x4f"
-			"\xe5\x79\x81\x73\xcd\x43\x59\x68"
-			"\x73\x02\x3b\x78\x21\x72\x43\x00"
-			"\x49\x17\xf7\x00\xaf\x68\x24\x53"
-			"\x05\x0a\xc3\x33\xe0\x33\x3f\x69"
-			"\xd2\x84\x2f\x0b\xed\xde\x04\xf4"
-			"\x11\x94\x13\x69\x51\x09\x28\xde"
-			"\x57\x5c\xef\xdc\x9a\x49\x1c\x17"
-			"\x97\xf3\x96\xc1\x7f\x5d\x2e\x7d"
-			"\x55\xb8\xb3\x02\x09\xb3\x1f\xe7"
-			"\xc9\x8d\xa3\x36\x34\x8a\x77\x13"
-			"\x30\x63\x4c\xa5\xcd\xc3\xe0\x7e"
-			"\x05\xa1\x7b\x0c\xcb\x74\x47\x31"
-			"\x62\x03\x43\xf1\x87\xb4\xb0\x85"
-			"\x87\x8e\x4b\x25\xc7\xcf\xae\x4b"
-			"\x36\x46\x3e\x62\xbc\x6f\xeb\x5f"
-			"\x73\xac\xe6\x07\xee\xc1\xa1\xd6"
-			"\xc4\xab\xc9\xd6\x89\x45\xe1\xf1"
-			"\x04\x4e\x1a\x6f\xbb\x4f\x3a\xa3"
-			"\xa0\xcb\xa3\x0a\xd8\x71\x35\x55"
-			"\xe4\xbc\x2e\x04\x06\xe6\xff\x5b"
-			"\x1c\xc0\x11\x7c\xc5\x17\xf3\x38"
-			"\xcf\xe9\xba\x0f\x0e\xef\x02\xc2"
-			"\x8d\xc6\xbc\x4b\x67\x20\x95\xd7"
-			"\x2c\x45\x5b\x86\x44\x8c\x6f\x2e"
-			"\x7e\x9f\x1c\x77\xba\x6b\x0e\xa3"
-			"\x69\xdc\xab\x24\x57\x60\x47\xc1"
-			"\xd1\xa5\x9d\x23\xe6\xb1\x37\xfe"
-			"\x93\xd2\x4c\x46\xf9\x0c\xc6\xfb"
-			"\xd6\x9d\x99\x69\xab\x7a\x07\x0c"
-			"\x65\xe7\xc4\x08\x96\xe2\xa5\x01"
-			"\x3f\x46\x07\x05\x7e\xe8\x9a\x90"
-			"\x50\xdc\xe9\x7a\xea\xa1\x39\x6e"
-			"\x66\xe4\x6f\xa5\x5f\xb2\xd9\x5b"
-			"\xf5\xdb\x2a\x32\xf0\x11\x6f\x7c"
-			"\x26\x10\x8f\x3d\x80\xe9\x58\xf7"
-			"\xe0\xa8\x57\xf8\xdb\x0e\xce\x99"
-			"\x63\x19\x3d\xd5\xec\x1b\x77\x69"
-			"\x98\xf6\xe4\x5f\x67\x17\x4b\x09"
-			"\x85\x62\x82\x70\x18\xe2\x9a\x78"
-			"\xe2\x62\xbd\xb4\xf1\x42\xc6\xfb"
-			"\x08\xd0\xbd\xeb\x4e\x09\xf2\xc8"
-			"\x1e\xdc\x3d\x32\x21\x56\x9c\x4f"
-			"\x35\xf3\x61\x06\x72\x84\xc4\x32"
-			"\xf2\xf1\xfa\x0b\x2f\xc3\xdb\x02"
-			"\x04\xc2\xde\x57\x64\x60\x8d\xcf"
-			"\xcb\x86\x5d\x97\x3e\xb1\x9c\x01"
-			"\xd6\x28\x8f\x99\xbc\x46\xeb\x05"
-			"\xaf\x7e\xb8\x21\x2a\x56\x85\x1c"
-			"\xb3\x71\xa0\xde\xca\x96\xf1\x78"
-			"\x49\xa2\x99\x81\x80\x5c\x01\xf5"
-			"\xa0\xa2\x56\x63\xe2\x70\x07\xa5"
-			"\x95\xd6\x85\xeb\x36\x9e\xa9\x51"
-			"\x66\x56\x5f\x1d\x02\x19\xe2\xf6"
-			"\x4f\x73\x38\x09\x75\x64\x48\xe0"
-			"\xf1\x7e\x0e\xe8\x9d\xf9\xed\x94"
-			"\xfe\x16\x26\x62\x49\x74\xf4\xb0"
-			"\xd4\xa9\x6c\xb0\xfd\x53\xe9\x81"
-			"\xe0\x7a\xbf\xcf\xb5\xc4\x01\x81"
-			"\x79\x99\x77\x01\x3b\xe9\xa2\xb6"
-			"\xe6\x6a\x8a\x9e\x56\x1c\x8d\x1e"
-			"\x8f\x06\x55\x2c\x6c\xdc\x92\x87"
-			"\x64\x3b\x4b\x19\xa1\x13\x64\x1d"
-			"\x4a\xe9\xc0\x00\xb8\x95\xef\x6b"
-			"\x1a\x86\x6d\x37\x52\x02\xc2\xe0"
-			"\xc8\xbb\x42\x0c\x02\x21\x4a\xc9"
-			"\xef\xa0\x54\xe4\x5e\x16\x53\x81"
-			"\x70\x62\x10\xaf\xde\xb8\xb5\xd3"
-			"\xe8\x5e\x6c\xc3\x8a\x3e\x18\x07"
-			"\xf2\x2f\x7d\xa7\xe1\x3d\x4e\xb4"
-			"\x26\xa7\xa3\x93\x86\xb2\x04\x1e"
-			"\x53\x5d\x86\xd6\xde\x65\xca\xe3"
-			"\x4e\xc1\xcf\xef\xc8\x70\x1b\x83"
-			"\x13\xdd\x18\x8b\x0d\x76\xd2\xf6"
-			"\x37\x7a\x93\x7a\x50\x11\x9f\x96"
-			"\x86\x25\xfd\xac\xdc\xbe\x18\x93"
-			"\x19\x6b\xec\x58\x4f\xb9\x75\xa7"
-			"\xdd\x3f\x2f\xec\xc8\x5a\x84\xab"
-			"\xd5\xe4\x8a\x07\xf6\x4d\x23\xd6"
-			"\x03\xfb\x03\x6a\xea\x66\xbf\xd4"
-			"\xb1\x34\xfb\x78\xe9\x55\xdc\x7c"
-			"\x3d\x9c\xe5\x9a\xac\xc3\x7a\x80"
-			"\x24\x6d\xa0\xef\x25\x7c\xb7\xea"
-			"\xce\x4d\x5f\x18\x60\xce\x87\x22"
-			"\x66\x2f\xd5\xdd\xdd\x02\x21\x75"
-			"\x82\xa0\x1f\x58\xc6\xd3\x62\xf7"
-			"\x32\xd8\xaf\x1e\x07\x77\x51\x96"
-			"\xd5\x6b\x1e\x7e\x80\x02\xe8\x67"
-			"\xea\x17\x0b\x10\xd2\x3f\x28\x25"
-			"\x4f\x05\x77\x02\x14\x69\xf0\x2c"
-			"\xbe\x0c\xf1\x74\x30\xd1\xb9\x9b"
-			"\xfc\x8c\xbb\x04\x16\xd9\xba\xc3"
-			"\xbc\x91\x8a\xc4\x30\xa4\xb0\x12"
-			"\x4c\x21\x87\xcb\xc9\x1d\x16\x96"
-			"\x07\x6f\x23\x54\xb9\x6f\x79\xe5"
-			"\x64\xc0\x64\xda\xb1\xae\xdd\x60"
-			"\x6c\x1a\x9d\xd3\x04\x8e\x45\xb0"
-			"\x92\x61\xd0\x48\x81\xed\x5e\x1d"
-			"\xa0\xc9\xa4\x33\xc7\x13\x51\x5d"
-			"\x7f\x83\x73\xb6\x70\x18\x65\x3e"
-			"\x2f\x0e\x7a\x12\x39\x98\xab\xd8"
-			"\x7e\x6f\xa3\xd1\xba\x56\xad\xbd"
-			"\xf0\x03\x01\x1c\x85\x35\x9f\xeb"
-			"\x19\x63\xa1\xaf\xfe\x2d\x35\x50"
-			"\x39\xa0\x65\x7c\x95\x7e\x6b\xfe"
-			"\xc1\xac\x07\x7c\x98\x4f\xbe\x57"
-			"\xa7\x22\xec\xe2\x7e\x29\x09\x53"
-			"\xe8\xbf\xb4\x7e\x3f\x8f\xfc\x14"
-			"\xce\x54\xf9\x18\x58\xb5\xff\x44"
-			"\x05\x9d\xce\x1b\xb6\x82\x23\xc8"
-			"\x2e\xbc\x69\xbb\x4a\x29\x0f\x65"
-			"\x94\xf0\x63\x06\x0e\xef\x8c\xbd"
-			"\xff\xfd\xb0\x21\x6e\x57\x05\x75"
-			"\xda\xd5\xc4\xeb\x8d\x32\xf7\x50"
-			"\xd3\x6f\x22\xed\x5f\x8e\xa2\x5b"
-			"\x80\x8c\xc8\x78\x40\x24\x4b\x89"
-			"\x30\xce\x7a\x97\x0e\xc4\xaf\xef"
-			"\x9b\xb4\xcd\x66\x74\x14\x04\x2b"
-			"\xf7\xce\x0b\x1c\x6e\xc2\x78\x8c"
-			"\xca\xc5\xd0\x1c\x95\x4a\x91\x2d"
-			"\xa7\x20\xeb\x86\x52\xb7\x67\xd8"
-			"\x0c\xd6\x04\x14\xde\x51\x74\x75"
-			"\xe7\x11\xb4\x87\xa3\x3d\x2d\xad"
-			"\x4f\xef\xa0\x0f\x70\x00\x6d\x13"
-			"\x19\x1d\x41\x50\xe9\xd8\xf0\x32"
-			"\x71\xbc\xd3\x11\xf2\xac\xbe\xaf"
-			"\x75\x46\x65\x4e\x07\x34\x37\xa3"
-			"\x89\xfe\x75\xd4\x70\x4c\xc6\x3f"
-			"\x69\x24\x0e\x38\x67\x43\x8c\xde"
-			"\x06\xb5\xb8\xe7\xc4\xf0\x41\x8f"
-			"\xf0\xbd\x2f\x0b\xb9\x18\xf8\xde"
-			"\x64\xb1\xdb\xee\x00\x50\x77\xe1"
-			"\xc7\xff\xa6\xfa\xdd\x70\xf4\xe3"
-			"\x93\xe9\x77\x35\x3d\x4b\x2f\x2b"
-			"\x6d\x55\xf0\xfc\x88\x54\x4e\x89"
-			"\xc1\x8a\x23\x31\x2d\x14\x2a\xb8"
-			"\x1b\x15\xdd\x9e\x6e\x7b\xda\x05"
-			"\x91\x7d\x62\x64\x96\x72\xde\xfc"
-			"\xc1\xec\xf0\x23\x51\x6f\xdb\x5b"
-			"\x1d\x08\x57\xce\x09\xb8\xf6\xcd"
-			"\x8d\x95\xf2\x20\xbf\x0f\x20\x57"
-			"\x98\x81\x84\x4f\x15\x5c\x76\xe7"
-			"\x3e\x0a\x3a\x6c\xc4\x8a\xbe\x78"
-			"\x74\x77\xc3\x09\x4b\x5d\x48\xe4"
-			"\xc8\xcb\x0b\xea\x17\x28\xcf\xcf"
-			"\x31\x32\x44\xa4\xe5\x0e\x1a\x98"
-			"\x94\xc4\xf0\xff\xae\x3e\x44\xe8"
-			"\xa5\xb3\xb5\x37\x2f\xe8\xaf\x6f"
-			"\x28\xc1\x37\x5f\x31\xd2\xb9\x33"
-			"\xb1\xb2\x52\x94\x75\x2c\x29\x59"
-			"\x06\xc2\x25\xe8\x71\x65\x4e\xed"
-			"\xc0\x9c\xb1\xbb\x25\xdc\x6c\xe7"
-			"\x4b\xa5\x7a\x54\x7a\x60\xff\x7a"
-			"\xe0\x50\x40\x96\x35\x63\xe4\x0b"
-			"\x76\xbd\xa4\x65\x00\x1b\x57\x88"
-			"\xae\xed\x39\x88\x42\x11\x3c\xed"
-			"\x85\x67\x7d\xb9\x68\x82\xe9\x43"
-			"\x3c\x47\x53\xfa\xe8\xf8\x9f\x1f"
-			"\x9f\xef\x0f\xf7\x30\xd9\x30\x0e"
-			"\xb9\x9f\x69\x18\x2f\x7e\xf8\xf8"
-			"\xf8\x8c\x0f\xd4\x02\x4d\xea\xcd"
-			"\x0a\x9c\x6f\x71\x6d\x5a\x4c\x60"
-			"\xce\x20\x56\x32\xc6\xc5\x99\x1f"
-			"\x09\xe6\x4e\x18\x1a\x15\x13\xa8"
-			"\x7d\xb1\x6b\xc0\xb2\x6d\xf8\x26"
-			"\x66\xf8\x3d\x18\x74\x70\x66\x7a"
-			"\x34\x17\xde\xba\x47\xf1\x06\x18"
-			"\xcb\xaf\xeb\x4a\x1e\x8f\xa7\x77"
-			"\xe0\x3b\x78\x62\x66\xc9\x10\xea"
-			"\x1f\xb7\x29\x0a\x45\xa1\x1d\x1e"
-			"\x1d\xe2\x65\x61\x50\x9c\xd7\x05"
-			"\xf2\x0b\x5b\x12\x61\x02\xc8\xe5"
-			"\x63\x4f\x20\x0c\x07\x17\x33\x5e"
-			"\x03\x9a\x53\x0f\x2e\x55\xfe\x50"
-			"\x43\x7d\xd0\xb6\x7e\x5a\xda\xae"
-			"\x58\xef\x15\xa9\x83\xd9\x46\xb1"
-			"\x42\xaa\xf5\x02\x6c\xce\x92\x06"
-			"\x1b\xdb\x66\x45\x91\x79\xc2\x2d"
-			"\xe6\x53\xd3\x14\xfd\xbb\x44\x63"
-			"\xc6\xd7\x3d\x7a\x0c\x75\x78\x9d"
-			"\x5c\xa6\x39\xb3\xe5\x63\xca\x8b"
-			"\xfe\xd3\xef\x60\x83\xf6\x8e\x70"
-			"\xb6\x67\xc7\x77\xed\x23\xef\x4c"
-			"\xf0\xed\x2d\x07\x59\x6f\xc1\x01"
-			"\x34\x37\x08\xab\xd9\x1f\x09\xb1"
-			"\xce\x5b\x17\xff\x74\xf8\x9c\xd5"
-			"\x2c\x56\x39\x79\x0f\x69\x44\x75"
-			"\x58\x27\x01\xc4\xbf\xa7\xa1\x1d"
-			"\x90\x17\x77\x86\x5a\x3f\xd9\xd1"
-			"\x0e\xa0\x10\xf8\xec\x1e\xa5\x7f"
-			"\x5e\x36\xd1\xe3\x04\x2c\x70\xf7"
-			"\x8e\xc0\x98\x2f\x6c\x94\x2b\x41"
-			"\xb7\x60\x00\xb7\x2e\xb8\x02\x8d"
-			"\xb8\xb0\xd3\x86\xba\x1d\xd7\x90"
-			"\xd6\xb6\xe1\xfc\xd7\xd8\x28\x06"
-			"\x63\x9b\xce\x61\x24\x79\xc0\x70"
-			"\x52\xd0\xb6\xd4\x28\x95\x24\x87"
-			"\x03\x1f\xb7\x9a\xda\xa3\xfb\x52"
-			"\x5b\x68\xe7\x4c\x8c\x24\xe1\x42"
-			"\xf7\xd5\xfd\xad\x06\x32\x9f\xba"
-			"\xc1\xfc\xdd\xc6\xfc\xfc\xb3\x38"
-			"\x74\x56\x58\x40\x02\x37\x52\x2c"
-			"\x55\xcc\xb3\x9e\x7a\xe9\xd4\x38"
-			"\x41\x5e\x0c\x35\xe2\x11\xd1\x13"
-			"\xf8\xb7\x8d\x72\x6b\x22\x2a\xb0"
-			"\xdb\x08\xba\x35\xb9\x3f\xc8\xd3"
-			"\x24\x90\xec\x58\xd2\x09\xc7\x2d"
-			"\xed\x38\x80\x36\x72\x43\x27\x49"
-			"\x4a\x80\x8a\xa2\xe8\xd3\xda\x30"
-			"\x7d\xb6\x82\x37\x86\x92\x86\x3e"
-			"\x08\xb2\x28\x5a\x55\x44\x24\x7d"
-			"\x40\x48\x8a\xb6\x89\x58\x08\xa0"
-			"\xd6\x6d\x3a\x17\xbf\xf6\x54\xa2"
-			"\xf5\xd3\x8c\x0f\x78\x12\x57\x8b"
-			"\xd5\xc2\xfd\x58\x5b\x7f\x38\xe3"
-			"\xcc\xb7\x7c\x48\xb3\x20\xe8\x81"
-			"\x14\x32\x45\x05\xe0\xdb\x9f\x75"
-			"\x85\xb4\x6a\xfc\x95\xe3\x54\x22"
-			"\x12\xee\x30\xfe\xd8\x30\xef\x34"
-			"\x50\xab\x46\x30\x98\x2f\xb7\xc0"
-			"\x15\xa2\x83\xb6\xf2\x06\x21\xa2"
-			"\xc3\x26\x37\x14\xd1\x4d\xb5\x10"
-			"\x52\x76\x4d\x6a\xee\xb5\x2b\x15"
-			"\xb7\xf9\x51\xe8\x2a\xaf\xc7\xfa"
-			"\x77\xaf\xb0\x05\x4d\xd1\x68\x8e"
-			"\x74\x05\x9f\x9d\x93\xa5\x3e\x7f"
-			"\x4e\x5f\x9d\xcb\x09\xc7\x83\xe3"
-			"\x02\x9d\x27\x1f\xef\x85\x05\x8d"
-			"\xec\x55\x88\x0f\x0d\x7c\x4c\xe8"
-			"\xa1\x75\xa0\xd8\x06\x47\x14\xef"
-			"\xaa\x61\xcf\x26\x15\xad\xd8\xa3"
-			"\xaa\x75\xf2\x78\x4a\x5a\x61\xdf"
-			"\x8b\xc7\x04\xbc\xb2\x32\xd2\x7e"
-			"\x42\xee\xb4\x2f\x51\xff\x7b\x2e"
-			"\xd3\x02\xe8\xdc\x5d\x0d\x50\xdc"
-			"\xae\xb7\x46\xf9\xa8\xe6\xd0\x16"
-			"\xcc\xe6\x2c\x81\xc7\xad\xe9\xf0"
-			"\x05\x72\x6d\x3d\x0a\x7a\xa9\x02"
-			"\xac\x82\x93\x6e\xb6\x1c\x28\xfc"
-			"\x44\x12\xfb\x73\x77\xd4\x13\x39"
-			"\x29\x88\x8a\xf3\x5c\xa6\x36\xa0"
-			"\x2a\xed\x7e\xb1\x1d\xd6\x4c\x6b"
-			"\x41\x01\x18\x5d\x5d\x07\x97\xa6"
-			"\x4b\xef\x31\x18\xea\xac\xb1\x84"
-			"\x21\xed\xda\x86",
-		.rlen = 4100,
-		.np	= 2,
-		.tap	= { 4064, 36 },
-	},
-};
-
-static const struct cipher_testvec aes_ctr_rfc3686_dec_tv_template[] = {
-	{ /* From RFC 3686 */
-		.key	= "\xae\x68\x52\xf8\x12\x10\x67\xcc"
-			  "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e"
-			  "\x00\x00\x00\x30",
-		.klen	= 20,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xe4\x09\x5d\x4f\xb7\xa7\xb3\x79"
-			  "\x2d\x61\x75\xa3\x26\x13\x11\xb8",
-		.ilen	= 16,
-		.result	= "Single block msg",
-		.rlen	= 16,
-	}, {
-		.key	= "\x7e\x24\x06\x78\x17\xfa\xe0\xd7"
-			  "\x43\xd6\xce\x1f\x32\x53\x91\x63"
-			  "\x00\x6c\xb6\xdb",
-		.klen	= 20,
-		.iv	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
-		.input	= "\x51\x04\xa1\x06\x16\x8a\x72\xd9"
-			  "\x79\x0d\x41\xee\x8e\xda\xd3\x88"
-			  "\xeb\x2e\x1e\xfc\x46\xda\x57\xc8"
-			  "\xfc\xe6\x30\xdf\x91\x41\xbe\x28",
-		.ilen	= 32,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.rlen	= 32,
-	}, {
-		.key	= "\x16\xaf\x5b\x14\x5f\xc9\xf5\x79"
-			  "\xc1\x75\xf9\x3e\x3b\xfb\x0e\xed"
-			  "\x86\x3d\x06\xcc\xfd\xb7\x85\x15"
-			  "\x00\x00\x00\x48",
-		.klen	= 28,
-		.iv	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
-		.input	= "\x4b\x55\x38\x4f\xe2\x59\xc9\xc8"
-			  "\x4e\x79\x35\xa0\x03\xcb\xe9\x28",
-		.ilen	= 16,
-		.result	= "Single block msg",
-		.rlen	= 16,
-	}, {
-		.key	= "\x7c\x5c\xb2\x40\x1b\x3d\xc3\x3c"
-			  "\x19\xe7\x34\x08\x19\xe0\xf6\x9c"
-			  "\x67\x8c\x3d\xb8\xe6\xf6\xa9\x1a"
-			  "\x00\x96\xb0\x3b",
-		.klen	= 28,
-		.iv	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
-		.input	= "\x45\x32\x43\xfc\x60\x9b\x23\x32"
-			  "\x7e\xdf\xaa\xfa\x71\x31\xcd\x9f"
-			  "\x84\x90\x70\x1c\x5a\xd4\xa7\x9c"
-			  "\xfc\x1f\xe0\xff\x42\xf4\xfb\x00",
-		.ilen	= 32,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.rlen	= 32,
-	}, {
-		.key	= "\x77\x6b\xef\xf2\x85\x1d\xb0\x6f"
-			  "\x4c\x8a\x05\x42\xc8\x69\x6f\x6c"
-			  "\x6a\x81\xaf\x1e\xec\x96\xb4\xd3"
-			  "\x7f\xc1\xd6\x89\xe6\xc1\xc1\x04"
-			  "\x00\x00\x00\x60",
-		.klen	= 36,
-		.iv	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
-		.input	= "\x14\x5a\xd0\x1d\xbf\x82\x4e\xc7"
-			  "\x56\x08\x63\xdc\x71\xe3\xe0\xc0",
-		.ilen	= 16,
-		.result	= "Single block msg",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf6\xd6\x6d\x6b\xd5\x2d\x59\xbb"
-			  "\x07\x96\x36\x58\x79\xef\xf8\x86"
-			  "\xc6\x6d\xd5\x1a\x5b\x6a\x99\x74"
-			  "\x4b\x50\x59\x0c\x87\xa2\x38\x84"
-			  "\x00\xfa\xac\x24",
-		.klen	= 36,
-		.iv	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
-		.input	= "\xf0\x5e\x23\x1b\x38\x94\x61\x2c"
-			  "\x49\xee\x00\x0b\x80\x4e\xb2\xa9"
-			  "\xb8\x30\x6b\x50\x8f\x83\x9d\x6a"
-			  "\x55\x30\x83\x1d\x93\x44\xaf\x1c",
-		.ilen	= 32,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.rlen	= 32,
-	},
-};
-
-static const struct cipher_testvec aes_ofb_enc_tv_template[] = {
-	 /* From NIST Special Publication 800-38A, Appendix F.5 */
-	{
-		.key	= "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
-			  "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
-		.klen	= 16,
-		.iv	= "\x00\x01\x02\x03\x04\x05\x06\x07\x08"
-			  "\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.input = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.ilen	= 64,
-		.result = "\x3b\x3f\xd9\x2e\xb7\x2d\xad\x20"
-			  "\x33\x34\x49\xf8\xe8\x3c\xfb\x4a"
-			  "\x77\x89\x50\x8d\x16\x91\x8f\x03\xf5"
-			  "\x3c\x52\xda\xc5\x4e\xd8\x25"
-			  "\x97\x40\x05\x1e\x9c\x5f\xec\xf6\x43"
-			  "\x44\xf7\xa8\x22\x60\xed\xcc"
-			  "\x30\x4c\x65\x28\xf6\x59\xc7\x78"
-			  "\x66\xa5\x10\xd9\xc1\xd6\xae\x5e",
-		.rlen	= 64,
-	}
-};
-
-static const struct cipher_testvec aes_ofb_dec_tv_template[] = {
-	 /* From NIST Special Publication 800-38A, Appendix F.5 */
-	{
-		.key	= "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
-			  "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
-		.klen	= 16,
-		.iv	= "\x00\x01\x02\x03\x04\x05\x06\x07\x08"
-			  "\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.input = "\x3b\x3f\xd9\x2e\xb7\x2d\xad\x20"
-			  "\x33\x34\x49\xf8\xe8\x3c\xfb\x4a"
-			  "\x77\x89\x50\x8d\x16\x91\x8f\x03\xf5"
-			  "\x3c\x52\xda\xc5\x4e\xd8\x25"
-			  "\x97\x40\x05\x1e\x9c\x5f\xec\xf6\x43"
-			  "\x44\xf7\xa8\x22\x60\xed\xcc"
-			  "\x30\x4c\x65\x28\xf6\x59\xc7\x78"
-			  "\x66\xa5\x10\xd9\xc1\xd6\xae\x5e",
-		.ilen	= 64,
-		.result = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.rlen	= 64,
-	}
-};
-
-static const struct aead_testvec aes_gcm_enc_tv_template[] = {
-	{ /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */
-		.key    = zeroed_string,
-		.klen	= 16,
-		.result	= "\x58\xe2\xfc\xce\xfa\x7e\x30\x61"
-			  "\x36\x7f\x1d\x57\xa4\xe7\x45\x5a",
-		.rlen	= 16,
-	}, {
-		.key    = zeroed_string,
-		.klen	= 16,
-		.input  = zeroed_string,
-		.ilen	= 16,
-		.result = "\x03\x88\xda\xce\x60\xb6\xa3\x92"
-			  "\xf3\x28\xc2\xb9\x71\xb2\xfe\x78"
-			  "\xab\x6e\x47\xd4\x2c\xec\x13\xbd"
-			  "\xf5\x3a\x67\xb2\x12\x57\xbd\xdf",
-		.rlen	= 32,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
-		.klen	= 16,
-		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
-			  "\xde\xca\xf8\x88",
-		.input	= "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
-			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
-			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
-			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
-			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
-			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
-			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
-			  "\xba\x63\x7b\x39\x1a\xaf\xd2\x55",
-		.ilen	= 64,
-		.result = "\x42\x83\x1e\xc2\x21\x77\x74\x24"
-			  "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c"
-			  "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0"
-			  "\x35\xc1\x7e\x23\x29\xac\xa1\x2e"
-			  "\x21\xd5\x14\xb2\x54\x66\x93\x1c"
-			  "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05"
-			  "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97"
-			  "\x3d\x58\xe0\x91\x47\x3f\x59\x85"
-			  "\x4d\x5c\x2a\xf3\x27\xcd\x64\xa6"
-			  "\x2c\xf3\x5a\xbd\x2b\xa6\xfa\xb4",
-		.rlen	= 80,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
-		.klen	= 16,
-		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
-			  "\xde\xca\xf8\x88",
-		.input	= "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
-			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
-			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
-			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
-			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
-			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
-			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
-			  "\xba\x63\x7b\x39",
-		.ilen	= 60,
-		.assoc	= "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
-			  "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
-			  "\xab\xad\xda\xd2",
-		.alen	= 20,
-		.result = "\x42\x83\x1e\xc2\x21\x77\x74\x24"
-			  "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c"
-			  "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0"
-			  "\x35\xc1\x7e\x23\x29\xac\xa1\x2e"
-			  "\x21\xd5\x14\xb2\x54\x66\x93\x1c"
-			  "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05"
-			  "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97"
-			  "\x3d\x58\xe0\x91"
-			  "\x5b\xc9\x4f\xbc\x32\x21\xa5\xdb"
-			  "\x94\xfa\xe9\x5a\xe7\x12\x1a\x47",
-		.rlen	= 76,
-	}, {
-		.key    = zeroed_string,
-		.klen	= 24,
-		.result	= "\xcd\x33\xb2\x8a\xc7\x73\xf7\x4b"
-			  "\xa0\x0e\xd1\xf3\x12\x57\x24\x35",
-		.rlen	= 16,
-	}, {
-		.key    = zeroed_string,
-		.klen	= 24,
-		.input  = zeroed_string,
-		.ilen	= 16,
-		.result = "\x98\xe7\x24\x7c\x07\xf0\xfe\x41"
-			  "\x1c\x26\x7e\x43\x84\xb0\xf6\x00"
-			  "\x2f\xf5\x8d\x80\x03\x39\x27\xab"
-			  "\x8e\xf4\xd4\x58\x75\x14\xf0\xfb",
-		.rlen	= 32,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c",
-		.klen	= 24,
-		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
-			  "\xde\xca\xf8\x88",
-		.input	= "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
-			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
-			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
-			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
-			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
-			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
-			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
-			  "\xba\x63\x7b\x39\x1a\xaf\xd2\x55",
-		.ilen	= 64,
-		.result = "\x39\x80\xca\x0b\x3c\x00\xe8\x41"
-			  "\xeb\x06\xfa\xc4\x87\x2a\x27\x57"
-			  "\x85\x9e\x1c\xea\xa6\xef\xd9\x84"
-			  "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c"
-			  "\x7d\x77\x3d\x00\xc1\x44\xc5\x25"
-			  "\xac\x61\x9d\x18\xc8\x4a\x3f\x47"
-			  "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9"
-			  "\xcc\xda\x27\x10\xac\xad\xe2\x56"
-			  "\x99\x24\xa7\xc8\x58\x73\x36\xbf"
-			  "\xb1\x18\x02\x4d\xb8\x67\x4a\x14",
-		.rlen	= 80,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c",
-		.klen	= 24,
-		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
-			  "\xde\xca\xf8\x88",
-		.input	= "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
-			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
-			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
-			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
-			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
-			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
-			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
-			  "\xba\x63\x7b\x39",
-		.ilen	= 60,
-		.assoc	= "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
-			  "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
-			  "\xab\xad\xda\xd2",
-		.alen	= 20,
-		.result = "\x39\x80\xca\x0b\x3c\x00\xe8\x41"
-			  "\xeb\x06\xfa\xc4\x87\x2a\x27\x57"
-			  "\x85\x9e\x1c\xea\xa6\xef\xd9\x84"
-			  "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c"
-			  "\x7d\x77\x3d\x00\xc1\x44\xc5\x25"
-			  "\xac\x61\x9d\x18\xc8\x4a\x3f\x47"
-			  "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9"
-			  "\xcc\xda\x27\x10"
-			  "\x25\x19\x49\x8e\x80\xf1\x47\x8f"
-			  "\x37\xba\x55\xbd\x6d\x27\x61\x8c",
-		.rlen	= 76,
-		.np	= 2,
-		.tap	= { 32, 28 },
-		.anp	= 2,
-		.atap	= { 8, 12 }
-	}, {
-		.key    = zeroed_string,
-		.klen	= 32,
-		.result	= "\x53\x0f\x8a\xfb\xc7\x45\x36\xb9"
-			  "\xa9\x63\xb4\xf1\xc4\xcb\x73\x8b",
-		.rlen	= 16,
-	}
-};
-
-static const struct aead_testvec aes_gcm_dec_tv_template[] = {
-	{ /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */
-		.key    = zeroed_string,
-		.klen	= 32,
-		.input	= "\xce\xa7\x40\x3d\x4d\x60\x6b\x6e"
-			  "\x07\x4e\xc5\xd3\xba\xf3\x9d\x18"
-			  "\xd0\xd1\xc8\xa7\x99\x99\x6b\xf0"
-			  "\x26\x5b\x98\xb5\xd4\x8a\xb9\x19",
-		.ilen	= 32,
-		.result  = zeroed_string,
-		.rlen	= 16,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
-		.klen	= 32,
-		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
-			  "\xde\xca\xf8\x88",
-		.input	= "\x52\x2d\xc1\xf0\x99\x56\x7d\x07"
-			  "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d"
-			  "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9"
-			  "\x75\x98\xa2\xbd\x25\x55\xd1\xaa"
-			  "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d"
-			  "\xa7\xb0\x8b\x10\x56\x82\x88\x38"
-			  "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a"
-			  "\xbc\xc9\xf6\x62\x89\x80\x15\xad"
-			  "\xb0\x94\xda\xc5\xd9\x34\x71\xbd"
-			  "\xec\x1a\x50\x22\x70\xe3\xcc\x6c",
-		.ilen	= 80,
-		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
-			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
-			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
-			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
-			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
-			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
-			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
-			  "\xba\x63\x7b\x39\x1a\xaf\xd2\x55",
-		.rlen	= 64,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
-		.klen	= 32,
-		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
-			  "\xde\xca\xf8\x88",
-		.input	= "\x52\x2d\xc1\xf0\x99\x56\x7d\x07"
-			  "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d"
-			  "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9"
-			  "\x75\x98\xa2\xbd\x25\x55\xd1\xaa"
-			  "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d"
-			  "\xa7\xb0\x8b\x10\x56\x82\x88\x38"
-			  "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a"
-			  "\xbc\xc9\xf6\x62"
-			  "\x76\xfc\x6e\xce\x0f\x4e\x17\x68"
-			  "\xcd\xdf\x88\x53\xbb\x2d\x55\x1b",
-		.ilen	= 76,
-		.assoc	= "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
-			  "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
-			  "\xab\xad\xda\xd2",
-		.alen	= 20,
-		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
-			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
-			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
-			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
-			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
-			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
-			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
-			  "\xba\x63\x7b\x39",
-		.rlen	= 60,
-		.np     = 2,
-		.tap    = { 48, 28 },
-		.anp	= 3,
-		.atap	= { 8, 8, 4 }
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
-		.klen	= 16,
-		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
-			  "\xde\xca\xf8\x88",
-		.input	= "\x42\x83\x1e\xc2\x21\x77\x74\x24"
-			  "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c"
-			  "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0"
-			  "\x35\xc1\x7e\x23\x29\xac\xa1\x2e"
-			  "\x21\xd5\x14\xb2\x54\x66\x93\x1c"
-			  "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05"
-			  "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97"
-			  "\x3d\x58\xe0\x91\x47\x3f\x59\x85"
-			  "\x4d\x5c\x2a\xf3\x27\xcd\x64\xa6"
-			  "\x2c\xf3\x5a\xbd\x2b\xa6\xfa\xb4",
-		.ilen	= 80,
-		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
-			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
-			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
-			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
-			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
-			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
-			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
-			  "\xba\x63\x7b\x39\x1a\xaf\xd2\x55",
-		.rlen	= 64,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
-		.klen	= 16,
-		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
-			  "\xde\xca\xf8\x88",
-		.input	= "\x42\x83\x1e\xc2\x21\x77\x74\x24"
-			  "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c"
-			  "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0"
-			  "\x35\xc1\x7e\x23\x29\xac\xa1\x2e"
-			  "\x21\xd5\x14\xb2\x54\x66\x93\x1c"
-			  "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05"
-			  "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97"
-			  "\x3d\x58\xe0\x91"
-			  "\x5b\xc9\x4f\xbc\x32\x21\xa5\xdb"
-			  "\x94\xfa\xe9\x5a\xe7\x12\x1a\x47",
-		.ilen	= 76,
-		.assoc	= "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
-			  "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
-			  "\xab\xad\xda\xd2",
-		.alen	= 20,
-		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
-			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
-			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
-			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
-			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
-			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
-			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
-			  "\xba\x63\x7b\x39",
-		.rlen	= 60,
-	}, {
-		.key    = zeroed_string,
-		.klen	= 24,
-		.input	= "\x98\xe7\x24\x7c\x07\xf0\xfe\x41"
-			  "\x1c\x26\x7e\x43\x84\xb0\xf6\x00"
-			  "\x2f\xf5\x8d\x80\x03\x39\x27\xab"
-			  "\x8e\xf4\xd4\x58\x75\x14\xf0\xfb",
-		.ilen	= 32,
-		.result  = zeroed_string,
-		.rlen	= 16,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c",
-		.klen	= 24,
-		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
-			  "\xde\xca\xf8\x88",
-		.input	= "\x39\x80\xca\x0b\x3c\x00\xe8\x41"
-			  "\xeb\x06\xfa\xc4\x87\x2a\x27\x57"
-			  "\x85\x9e\x1c\xea\xa6\xef\xd9\x84"
-			  "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c"
-			  "\x7d\x77\x3d\x00\xc1\x44\xc5\x25"
-			  "\xac\x61\x9d\x18\xc8\x4a\x3f\x47"
-			  "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9"
-			  "\xcc\xda\x27\x10\xac\xad\xe2\x56"
-			  "\x99\x24\xa7\xc8\x58\x73\x36\xbf"
-			  "\xb1\x18\x02\x4d\xb8\x67\x4a\x14",
-		.ilen	= 80,
-		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
-			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
-			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
-			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
-			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
-			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
-			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
-			  "\xba\x63\x7b\x39\x1a\xaf\xd2\x55",
-		.rlen	= 64,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c",
-		.klen	= 24,
-		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
-			  "\xde\xca\xf8\x88",
-		.input	= "\x39\x80\xca\x0b\x3c\x00\xe8\x41"
-			  "\xeb\x06\xfa\xc4\x87\x2a\x27\x57"
-			  "\x85\x9e\x1c\xea\xa6\xef\xd9\x84"
-			  "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c"
-			  "\x7d\x77\x3d\x00\xc1\x44\xc5\x25"
-			  "\xac\x61\x9d\x18\xc8\x4a\x3f\x47"
-			  "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9"
-			  "\xcc\xda\x27\x10"
-			  "\x25\x19\x49\x8e\x80\xf1\x47\x8f"
-			  "\x37\xba\x55\xbd\x6d\x27\x61\x8c",
-		.ilen	= 76,
-		.assoc	= "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
-			  "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
-			  "\xab\xad\xda\xd2",
-		.alen	= 20,
-		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
-			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
-			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
-			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
-			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
-			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
-			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
-			  "\xba\x63\x7b\x39",
-		.rlen	= 60,
-	}
-};
-
-static const struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = {
-	{ /* Generated using Crypto++ */
-		.key    = zeroed_string,
-		.klen	= 20,
-		.iv	= zeroed_string,
-		.input  = zeroed_string,
-		.ilen   = 16,
-		.assoc  = zeroed_string,
-		.alen   = 16,
-		.result	= "\x03\x88\xDA\xCE\x60\xB6\xA3\x92"
-			  "\xF3\x28\xC2\xB9\x71\xB2\xFE\x78"
-			  "\x97\xFE\x4C\x23\x37\x42\x01\xE0"
-			  "\x81\x9F\x8D\xC5\xD7\x41\xA0\x1B",
-		.rlen	= 32,
-	},{
-		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input  = zeroed_string,
-		.ilen   = 16,
-		.assoc  = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen   = 16,
-		.result	= "\xC0\x0D\x8B\x42\x0F\x8F\x34\x18"
-			  "\x88\xB1\xC5\xBC\xC5\xB6\xD6\x28"
-			  "\x6A\x9D\xDF\x11\x5E\xFE\x5E\x9D"
-			  "\x2F\x70\x44\x92\xF7\xF2\xE3\xEF",
-		.rlen	= 32,
-
-	}, {
-		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = zeroed_string,
-		.input  = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.ilen   = 16,
-		.assoc  = zeroed_string,
-		.alen   = 16,
-		.result	= "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
-			  "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
-			  "\x0B\x8F\x88\x69\x17\xE6\xB4\x3C"
-			  "\xB1\x68\xFD\x14\x52\x64\x61\xB2",
-		.rlen	= 32,
-	}, {
-		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = zeroed_string,
-		.input  = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.ilen   = 16,
-		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.alen   = 16,
-		.result	= "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
-			  "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
-			  "\x90\x92\xB7\xE3\x5F\xA3\x9A\x63"
-			  "\x7E\xD7\x1F\xD8\xD3\x7C\x4B\xF5",
-		.rlen	= 32,
-	}, {
-		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input  = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.ilen   = 16,
-		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen   = 16,
-		.result	= "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
-			  "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
-			  "\x64\x50\xF9\x32\x13\xFB\x74\x61"
-			  "\xF4\xED\x52\xD3\xC5\x10\x55\x3C",
-		.rlen	= 32,
-	}, {
-		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input  = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.ilen   = 64,
-		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen   = 16,
-		.result	= "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
-			  "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
-			  "\x98\x14\xA1\x42\x37\x80\xFD\x90"
-			  "\x68\x12\x01\xA8\x91\x89\xB9\x83"
-			  "\x5B\x11\x77\x12\x9B\xFF\x24\x89"
-			  "\x94\x5F\x18\x12\xBA\x27\x09\x39"
-			  "\x99\x96\x76\x42\x15\x1C\xCD\xCB"
-			  "\xDC\xD3\xDA\x65\x73\xAF\x80\xCD"
-			  "\xD2\xB6\xC2\x4A\x76\xC2\x92\x85"
-			  "\xBD\xCF\x62\x98\x58\x14\xE5\xBD",
-		.rlen	= 80,
-	}, {
-		.key    = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = "\x00\x00\x45\x67\x89\xab\xcd\xef",
-		.input  = "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff",
-		.ilen   = 192,
-		.assoc  = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
-			  "\xaa\xaa\xaa\xaa\x00\x00\x45\x67"
-			  "\x89\xab\xcd\xef",
-		.alen   = 20,
-		.result	= "\xC1\x76\x33\x85\xE2\x9B\x5F\xDE"
-			  "\xDE\x89\x3D\x42\xE7\xC9\x69\x8A"
-			  "\x44\x6D\xC3\x88\x46\x2E\xC2\x01"
-			  "\x5E\xF6\x0C\x39\xF0\xC4\xA5\x82"
-			  "\xCD\xE8\x31\xCC\x0A\x4C\xE4\x44"
-			  "\x41\xA9\x82\x6F\x22\xA1\x23\x1A"
-			  "\xA8\xE3\x16\xFD\x31\x5C\x27\x31"
-			  "\xF1\x7F\x01\x63\xA3\xAF\x70\xA1"
-			  "\xCF\x07\x57\x41\x67\xD0\xC4\x42"
-			  "\xDB\x18\xC6\x4C\x4C\xE0\x3D\x9F"
-			  "\x05\x07\xFB\x13\x7D\x4A\xCA\x5B"
-			  "\xF0\xBF\x64\x7E\x05\xB1\x72\xEE"
-			  "\x7C\x3B\xD4\xCD\x14\x03\xB2\x2C"
-			  "\xD3\xA9\xEE\xFA\x17\xFC\x9C\xDF"
-			  "\xC7\x75\x40\xFF\xAE\xAD\x1E\x59"
-			  "\x2F\x30\x24\xFB\xAD\x6B\x10\xFA"
-			  "\x6C\x9F\x5B\xE7\x25\xD5\xD0\x25"
-			  "\xAC\x4A\x4B\xDA\xFC\x7A\x85\x1B"
-			  "\x7E\x13\x06\x82\x08\x17\xA4\x35"
-			  "\xEC\xC5\x8D\x63\x96\x81\x0A\x8F"
-			  "\xA3\x05\x38\x95\x20\x1A\x47\x04"
-			  "\x6F\x6D\xDA\x8F\xEF\xC1\x76\x35"
-			  "\x6B\xC7\x4D\x0F\x94\x12\xCA\x3E"
-			  "\x2E\xD5\x03\x2E\x86\x7E\xAA\x3B"
-			  "\x37\x08\x1C\xCF\xBA\x5D\x71\x46"
-			  "\x80\x72\xB0\x4C\x82\x0D\x60\x3C",
-		.rlen	= 208,
-	}, { /* From draft-mcgrew-gcm-test-01 */
-		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
-			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
-			  "\x2E\x44\x3B\x68",
-		.klen	= 20,
-		.iv	= "\x49\x56\xED\x7E\x3B\x24\x4C\xFE",
-		.input	= "\x45\x00\x00\x48\x69\x9A\x00\x00"
-			  "\x80\x11\x4D\xB7\xC0\xA8\x01\x02"
-			  "\xC0\xA8\x01\x01\x0A\x9B\xF1\x56"
-			  "\x38\xD3\x01\x00\x00\x01\x00\x00"
-			  "\x00\x00\x00\x00\x04\x5F\x73\x69"
-			  "\x70\x04\x5F\x75\x64\x70\x03\x73"
-			  "\x69\x70\x09\x63\x79\x62\x65\x72"
-			  "\x63\x69\x74\x79\x02\x64\x6B\x00"
-			  "\x00\x21\x00\x01\x01\x02\x02\x01",
-		.ilen	= 72,
-		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
-			  "\x00\x00\x00\x00\x49\x56\xED\x7E"
-			  "\x3B\x24\x4C\xFE",
-		.alen	= 20,
-		.result	= "\xFE\xCF\x53\x7E\x72\x9D\x5B\x07"
-			  "\xDC\x30\xDF\x52\x8D\xD2\x2B\x76"
-			  "\x8D\x1B\x98\x73\x66\x96\xA6\xFD"
-			  "\x34\x85\x09\xFA\x13\xCE\xAC\x34"
-			  "\xCF\xA2\x43\x6F\x14\xA3\xF3\xCF"
-			  "\x65\x92\x5B\xF1\xF4\xA1\x3C\x5D"
-			  "\x15\xB2\x1E\x18\x84\xF5\xFF\x62"
-			  "\x47\xAE\xAB\xB7\x86\xB9\x3B\xCE"
-			  "\x61\xBC\x17\xD7\x68\xFD\x97\x32"
-			  "\x45\x90\x18\x14\x8F\x6C\xBE\x72"
-			  "\x2F\xD0\x47\x96\x56\x2D\xFD\xB4",
-		.rlen	= 88,
-	}, {
-		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
-			  "\xCA\xFE\xBA\xBE",
-		.klen	= 20,
-		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.input	= "\x45\x00\x00\x3E\x69\x8F\x00\x00"
-			  "\x80\x11\x4D\xCC\xC0\xA8\x01\x02"
-			  "\xC0\xA8\x01\x01\x0A\x98\x00\x35"
-			  "\x00\x2A\x23\x43\xB2\xD0\x01\x00"
-			  "\x00\x01\x00\x00\x00\x00\x00\x00"
-			  "\x03\x73\x69\x70\x09\x63\x79\x62"
-			  "\x65\x72\x63\x69\x74\x79\x02\x64"
-			  "\x6B\x00\x00\x01\x00\x01\x00\x01",
-		.ilen	= 64,
-		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
-			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.alen	= 16,
-		.result	= "\xDE\xB2\x2C\xD9\xB0\x7C\x72\xC1"
-			  "\x6E\x3A\x65\xBE\xEB\x8D\xF3\x04"
-			  "\xA5\xA5\x89\x7D\x33\xAE\x53\x0F"
-			  "\x1B\xA7\x6D\x5D\x11\x4D\x2A\x5C"
-			  "\x3D\xE8\x18\x27\xC1\x0E\x9A\x4F"
-			  "\x51\x33\x0D\x0E\xEC\x41\x66\x42"
-			  "\xCF\xBB\x85\xA5\xB4\x7E\x48\xA4"
-			  "\xEC\x3B\x9B\xA9\x5D\x91\x8B\xD1"
-			  "\x83\xB7\x0D\x3A\xA8\xBC\x6E\xE4"
-			  "\xC3\x09\xE9\xD8\x5A\x41\xAD\x4A",
-		.rlen	= 80,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x11\x22\x33\x44",
-		.klen	= 36,
-		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
-		.input	= "\x45\x00\x00\x30\x69\xA6\x40\x00"
-			  "\x80\x06\x26\x90\xC0\xA8\x01\x02"
-			  "\x93\x89\x15\x5E\x0A\x9E\x00\x8B"
-			  "\x2D\xC5\x7E\xE0\x00\x00\x00\x00"
-			  "\x70\x02\x40\x00\x20\xBF\x00\x00"
-			  "\x02\x04\x05\xB4\x01\x01\x04\x02"
-			  "\x01\x02\x02\x01",
-		.ilen	= 52,
-		.assoc	= "\x4A\x2C\xBF\xE3\x00\x00\x00\x02"
-			  "\x01\x02\x03\x04\x05\x06\x07\x08",
-		.alen	= 16,
-		.result	= "\xFF\x42\x5C\x9B\x72\x45\x99\xDF"
-			  "\x7A\x3B\xCD\x51\x01\x94\xE0\x0D"
-			  "\x6A\x78\x10\x7F\x1B\x0B\x1C\xBF"
-			  "\x06\xEF\xAE\x9D\x65\xA5\xD7\x63"
-			  "\x74\x8A\x63\x79\x85\x77\x1D\x34"
-			  "\x7F\x05\x45\x65\x9F\x14\xE9\x9D"
-			  "\xEF\x84\x2D\x8E\xB3\x35\xF4\xEE"
-			  "\xCF\xDB\xF8\x31\x82\x4B\x4C\x49"
-			  "\x15\x95\x6C\x96",
-		.rlen	= 68,
-	}, {
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x45\x00\x00\x3C\x99\xC5\x00\x00"
-			  "\x80\x01\xCB\x7A\x40\x67\x93\x18"
-			  "\x01\x01\x01\x01\x08\x00\x07\x5C"
-			  "\x02\x00\x44\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x75\x76\x77\x61\x62\x63\x64\x65"
-			  "\x66\x67\x68\x69\x01\x02\x02\x01",
-		.ilen	= 64,
-		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.alen	= 16,
-		.result	= "\x46\x88\xDA\xF2\xF9\x73\xA3\x92"
-			  "\x73\x29\x09\xC3\x31\xD5\x6D\x60"
-			  "\xF6\x94\xAB\xAA\x41\x4B\x5E\x7F"
-			  "\xF5\xFD\xCD\xFF\xF5\xE9\xA2\x84"
-			  "\x45\x64\x76\x49\x27\x19\xFF\xB6"
-			  "\x4D\xE7\xD9\xDC\xA1\xE1\xD8\x94"
-			  "\xBC\x3B\xD5\x78\x73\xED\x4D\x18"
-			  "\x1D\x19\xD4\xD5\xC8\xC1\x8A\xF3"
-			  "\xF8\x21\xD4\x96\xEE\xB0\x96\xE9"
-			  "\x8A\xD2\xB6\x9E\x47\x99\xC7\x1D",
-		.rlen	= 80,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E\x43",
-		.klen	= 20,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.input	= "\x45\x00\x00\x3C\x99\xC3\x00\x00"
-			  "\x80\x01\xCB\x7C\x40\x67\x93\x18"
-			  "\x01\x01\x01\x01\x08\x00\x08\x5C"
-			  "\x02\x00\x43\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x75\x76\x77\x61\x62\x63\x64\x65"
-			  "\x66\x67\x68\x69\x01\x02\x02\x01",
-		.ilen	= 64,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.result	= "\xFB\xA2\xCA\xA4\x85\x3C\xF9\xF0"
-			  "\xF2\x2C\xB1\x0D\x86\xDD\x83\xB0"
-			  "\xFE\xC7\x56\x91\xCF\x1A\x04\xB0"
-			  "\x0D\x11\x38\xEC\x9C\x35\x79\x17"
-			  "\x65\xAC\xBD\x87\x01\xAD\x79\x84"
-			  "\x5B\xF9\xFE\x3F\xBA\x48\x7B\xC9"
-			  "\x17\x55\xE6\x66\x2B\x4C\x8D\x0D"
-			  "\x1F\x5E\x22\x73\x95\x30\x32\x0A"
-			  "\xE0\xD7\x31\xCC\x97\x8E\xCA\xFA"
-			  "\xEA\xE8\x8F\x00\xE8\x0D\x6E\x48",
-		.rlen	= 80,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E\x43",
-		.klen	= 20,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.input	= "\x45\x00\x00\x1C\x42\xA2\x00\x00"
-			  "\x80\x01\x44\x1F\x40\x67\x93\xB6"
-			  "\xE0\x00\x00\x02\x0A\x00\xF5\xFF"
-			  "\x01\x02\x02\x01",
-		.ilen	= 28,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.result	= "\xFB\xA2\xCA\x84\x5E\x5D\xF9\xF0"
-			  "\xF2\x2C\x3E\x6E\x86\xDD\x83\x1E"
-			  "\x1F\xC6\x57\x92\xCD\x1A\xF9\x13"
-			  "\x0E\x13\x79\xED\x36\x9F\x07\x1F"
-			  "\x35\xE0\x34\xBE\x95\xF1\x12\xE4"
-			  "\xE7\xD0\x5D\x35",
-		.rlen	= 44,
-	}, {
-		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
-			  "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\xCA\xFE\xBA\xBE",
-		.klen	= 28,
-		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.input	= "\x45\x00\x00\x28\xA4\xAD\x40\x00"
-			  "\x40\x06\x78\x80\x0A\x01\x03\x8F"
-			  "\x0A\x01\x06\x12\x80\x23\x06\xB8"
-			  "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E"
-			  "\x50\x10\x16\xD0\x75\x68\x00\x01",
-		.ilen	= 40,
-		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
-			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.alen	= 16,
-		.result	= "\xA5\xB1\xF8\x06\x60\x29\xAE\xA4"
-			  "\x0E\x59\x8B\x81\x22\xDE\x02\x42"
-			  "\x09\x38\xB3\xAB\x33\xF8\x28\xE6"
-			  "\x87\xB8\x85\x8B\x5B\xFB\xDB\xD0"
-			  "\x31\x5B\x27\x45\x21\x44\xCC\x77"
-			  "\x95\x45\x7B\x96\x52\x03\x7F\x53"
-			  "\x18\x02\x7B\x5B\x4C\xD7\xA6\x36",
-		.rlen	= 56,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xDE\xCA\xF8\x88",
-		.klen	= 20,
-		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
-		.input	= "\x45\x00\x00\x49\x33\xBA\x00\x00"
-			  "\x7F\x11\x91\x06\xC3\xFB\x1D\x10"
-			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
-			  "\x00\x35\xDD\x7B\x80\x03\x02\xD5"
-			  "\x00\x00\x4E\x20\x00\x1E\x8C\x18"
-			  "\xD7\x5B\x81\xDC\x91\xBA\xA0\x47"
-			  "\x6B\x91\xB9\x24\xB2\x80\x38\x9D"
-			  "\x92\xC9\x63\xBA\xC0\x46\xEC\x95"
-			  "\x9B\x62\x66\xC0\x47\x22\xB1\x49"
-			  "\x23\x01\x01\x01",
-		.ilen	= 76,
-		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
-			  "\xCE\xFA\xCE\x74",
-		.alen	= 20,
-		.result	= "\x18\xA6\xFD\x42\xF7\x2C\xBF\x4A"
-			  "\xB2\xA2\xEA\x90\x1F\x73\xD8\x14"
-			  "\xE3\xE7\xF2\x43\xD9\x54\x12\xE1"
-			  "\xC3\x49\xC1\xD2\xFB\xEC\x16\x8F"
-			  "\x91\x90\xFE\xEB\xAF\x2C\xB0\x19"
-			  "\x84\xE6\x58\x63\x96\x5D\x74\x72"
-			  "\xB7\x9D\xA3\x45\xE0\xE7\x80\x19"
-			  "\x1F\x0D\x2F\x0E\x0F\x49\x6C\x22"
-			  "\x6F\x21\x27\xB2\x7D\xB3\x57\x24"
-			  "\xE7\x84\x5D\x68\x65\x1F\x57\xE6"
-			  "\x5F\x35\x4F\x75\xFF\x17\x01\x57"
-			  "\x69\x62\x34\x36",
-		.rlen	= 92,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x73\x61\x6C\x74",
-		.klen	= 36,
-		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
-		.input	= "\x45\x08\x00\x28\x73\x2C\x00\x00"
-			  "\x40\x06\xE9\xF9\x0A\x01\x06\x12"
-			  "\x0A\x01\x03\x8F\x06\xB8\x80\x23"
-			  "\xDD\x6B\xAF\xBE\xCB\x71\x26\x02"
-			  "\x50\x10\x1F\x64\x6D\x54\x00\x01",
-		.ilen	= 40,
-		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
-			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
-			  "\x69\x76\x65\x63",
-		.alen	= 20,
-		.result	= "\xF2\xD6\x9E\xCD\xBD\x5A\x0D\x5B"
-			  "\x8D\x5E\xF3\x8B\xAD\x4D\xA5\x8D"
-			  "\x1F\x27\x8F\xDE\x98\xEF\x67\x54"
-			  "\x9D\x52\x4A\x30\x18\xD9\xA5\x7F"
-			  "\xF4\xD3\xA3\x1C\xE6\x73\x11\x9E"
-			  "\x45\x16\x26\xC2\x41\x57\x71\xE3"
-			  "\xB7\xEE\xBC\xA6\x14\xC8\x9B\x35",
-		.rlen	= 56,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E\x43",
-		.klen	= 20,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.input	= "\x45\x00\x00\x49\x33\x3E\x00\x00"
-			  "\x7F\x11\x91\x82\xC3\xFB\x1D\x10"
-			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
-			  "\x00\x35\xCB\x45\x80\x03\x02\x5B"
-			  "\x00\x00\x01\xE0\x00\x1E\x8C\x18"
-			  "\xD6\x57\x59\xD5\x22\x84\xA0\x35"
-			  "\x2C\x71\x47\x5C\x88\x80\x39\x1C"
-			  "\x76\x4D\x6E\x5E\xE0\x49\x6B\x32"
-			  "\x5A\xE2\x70\xC0\x38\x99\x49\x39"
-			  "\x15\x01\x01\x01",
-		.ilen	= 76,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.result	= "\xFB\xA2\xCA\xD1\x2F\xC1\xF9\xF0"
-			  "\x0D\x3C\xEB\xF3\x05\x41\x0D\xB8"
-			  "\x3D\x77\x84\xB6\x07\x32\x3D\x22"
-			  "\x0F\x24\xB0\xA9\x7D\x54\x18\x28"
-			  "\x00\xCA\xDB\x0F\x68\xD9\x9E\xF0"
-			  "\xE0\xC0\xC8\x9A\xE9\xBE\xA8\x88"
-			  "\x4E\x52\xD6\x5B\xC1\xAF\xD0\x74"
-			  "\x0F\x74\x24\x44\x74\x7B\x5B\x39"
-			  "\xAB\x53\x31\x63\xAA\xD4\x55\x0E"
-			  "\xE5\x16\x09\x75\xCD\xB6\x08\xC5"
-			  "\x76\x91\x89\x60\x97\x63\xB8\xE1"
-			  "\x8C\xAA\x81\xE2",
-		.rlen	= 92,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x73\x61\x6C\x74",
-		.klen	= 36,
-		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
-		.input	= "\x63\x69\x73\x63\x6F\x01\x72\x75"
-			  "\x6C\x65\x73\x01\x74\x68\x65\x01"
-			  "\x6E\x65\x74\x77\x65\x01\x64\x65"
-			  "\x66\x69\x6E\x65\x01\x74\x68\x65"
-			  "\x74\x65\x63\x68\x6E\x6F\x6C\x6F"
-			  "\x67\x69\x65\x73\x01\x74\x68\x61"
-			  "\x74\x77\x69\x6C\x6C\x01\x64\x65"
-			  "\x66\x69\x6E\x65\x74\x6F\x6D\x6F"
-			  "\x72\x72\x6F\x77\x01\x02\x02\x01",
-		.ilen	= 72,
-		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
-			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
-			  "\x69\x76\x65\x63",
-		.alen	= 20,
-		.result	= "\xD4\xB7\xED\x86\xA1\x77\x7F\x2E"
-			  "\xA1\x3D\x69\x73\xD3\x24\xC6\x9E"
-			  "\x7B\x43\xF8\x26\xFB\x56\x83\x12"
-			  "\x26\x50\x8B\xEB\xD2\xDC\xEB\x18"
-			  "\xD0\xA6\xDF\x10\xE5\x48\x7D\xF0"
-			  "\x74\x11\x3E\x14\xC6\x41\x02\x4E"
-			  "\x3E\x67\x73\xD9\x1A\x62\xEE\x42"
-			  "\x9B\x04\x3A\x10\xE3\xEF\xE6\xB0"
-			  "\x12\xA4\x93\x63\x41\x23\x64\xF8"
-			  "\xC0\xCA\xC5\x87\xF2\x49\xE5\x6B"
-			  "\x11\xE2\x4F\x30\xE4\x4C\xCC\x76",
-		.rlen	= 88,
-	}, {
-		.key	= "\x7D\x77\x3D\x00\xC1\x44\xC5\x25"
-			  "\xAC\x61\x9D\x18\xC8\x4A\x3F\x47"
-			  "\xD9\x66\x42\x67",
-		.klen	= 20,
-		.iv	= "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
-		.input	= "\x01\x02\x02\x01",
-		.ilen	= 4,
-		.assoc	= "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF"
-			  "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
-		.alen	= 16,
-		.result	= "\x43\x7F\x86\x6B\xCB\x3F\x69\x9F"
-			  "\xE9\xB0\x82\x2B\xAC\x96\x1C\x45"
-			  "\x04\xBE\xF2\x70",
-		.rlen	= 20,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xDE\xCA\xF8\x88",
-		.klen	= 20,
-		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
-		.input	= "\x74\x6F\x01\x62\x65\x01\x6F\x72"
-			  "\x01\x6E\x6F\x74\x01\x74\x6F\x01"
-			  "\x62\x65\x00\x01",
-		.ilen	= 20,
-		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
-			  "\xCE\xFA\xCE\x74",
-		.alen	= 20,
-		.result	= "\x29\xC9\xFC\x69\xA1\x97\xD0\x38"
-			  "\xCC\xDD\x14\xE2\xDD\xFC\xAA\x05"
-			  "\x43\x33\x21\x64\x41\x25\x03\x52"
-			  "\x43\x03\xED\x3C\x6C\x5F\x28\x38"
-			  "\x43\xAF\x8C\x3E",
-		.rlen	= 36,
-	}, {
-		.key	= "\x6C\x65\x67\x61\x6C\x69\x7A\x65"
-			  "\x6D\x61\x72\x69\x6A\x75\x61\x6E"
-			  "\x61\x61\x6E\x64\x64\x6F\x69\x74"
-			  "\x62\x65\x66\x6F\x72\x65\x69\x61"
-			  "\x74\x75\x72\x6E",
-		.klen	= 36,
-		.iv	= "\x33\x30\x21\x69\x67\x65\x74\x6D",
-		.input	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
-			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
-			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01",
-		.ilen	= 52,
-		.assoc	= "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\x33\x30\x21\x69"
-			  "\x67\x65\x74\x6D",
-		.alen	= 20,
-		.result	= "\xF9\x7A\xB2\xAA\x35\x6D\x8E\xDC"
-			  "\xE1\x76\x44\xAC\x8C\x78\xE2\x5D"
-			  "\xD2\x4D\xED\xBB\x29\xEB\xF1\xB6"
-			  "\x4A\x27\x4B\x39\xB4\x9C\x3A\x86"
-			  "\x4C\xD3\xD7\x8C\xA4\xAE\x68\xA3"
-			  "\x2B\x42\x45\x8F\xB5\x7D\xBE\x82"
-			  "\x1D\xCC\x63\xB9\xD0\x93\x7B\xA2"
-			  "\x94\x5F\x66\x93\x68\x66\x1A\x32"
-			  "\x9F\xB4\xC0\x53",
-		.rlen	= 68,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E\x43",
-		.klen	= 20,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.input	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
-			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
-			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01",
-		.ilen	= 52,
-		.assoc	= "\x3F\x7E\xF6\x42\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.result	= "\xFB\xA2\xCA\xA8\xC6\xC5\xF9\xF0"
-			  "\xF2\x2C\xA5\x4A\x06\x12\x10\xAD"
-			  "\x3F\x6E\x57\x91\xCF\x1A\xCA\x21"
-			  "\x0D\x11\x7C\xEC\x9C\x35\x79\x17"
-			  "\x65\xAC\xBD\x87\x01\xAD\x79\x84"
-			  "\x5B\xF9\xFE\x3F\xBA\x48\x7B\xC9"
-			  "\x63\x21\x93\x06\x84\xEE\xCA\xDB"
-			  "\x56\x91\x25\x46\xE7\xA9\x5C\x97"
-			  "\x40\xD7\xCB\x05",
-		.rlen	= 68,
-	}, {
-		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
-			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
-			  "\x22\x43\x3C\x64",
-		.klen	= 20,
-		.iv	= "\x48\x55\xEC\x7D\x3A\x23\x4B\xFD",
-		.input	= "\x08\x00\xC6\xCD\x02\x00\x07\x00"
-			  "\x61\x62\x63\x64\x65\x66\x67\x68"
-			  "\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70"
-			  "\x71\x72\x73\x74\x01\x02\x02\x01",
-		.ilen	= 32,
-		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
-			  "\x00\x00\x00\x07\x48\x55\xEC\x7D"
-			  "\x3A\x23\x4B\xFD",
-		.alen	= 20,
-		.result	= "\x74\x75\x2E\x8A\xEB\x5D\x87\x3C"
-			  "\xD7\xC0\xF4\xAC\xC3\x6C\x4B\xFF"
-			  "\x84\xB7\xD7\xB9\x8F\x0C\xA8\xB6"
-			  "\xAC\xDA\x68\x94\xBC\x61\x90\x69"
-			  "\xEF\x9C\xBC\x28\xFE\x1B\x56\xA7"
-			  "\xC4\xE0\xD5\x8C\x86\xCD\x2B\xC0",
-		.rlen	= 48,
-	}
-};
-
-static const struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = {
-	{ /* Generated using Crypto++ */
-		.key    = zeroed_string,
-		.klen	= 20,
-		.iv     = zeroed_string,
-		.input	= "\x03\x88\xDA\xCE\x60\xB6\xA3\x92"
-			  "\xF3\x28\xC2\xB9\x71\xB2\xFE\x78"
-			  "\x97\xFE\x4C\x23\x37\x42\x01\xE0"
-			  "\x81\x9F\x8D\xC5\xD7\x41\xA0\x1B",
-		.ilen	= 32,
-		.assoc  = zeroed_string,
-		.alen   = 16,
-		.result = zeroed_string,
-		.rlen   = 16,
-
-	},{
-		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\xC0\x0D\x8B\x42\x0F\x8F\x34\x18"
-			  "\x88\xB1\xC5\xBC\xC5\xB6\xD6\x28"
-			  "\x6A\x9D\xDF\x11\x5E\xFE\x5E\x9D"
-			  "\x2F\x70\x44\x92\xF7\xF2\xE3\xEF",
-		.ilen	= 32,
-		.assoc  = "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen   = 16,
-		.result = zeroed_string,
-		.rlen   = 16,
-	}, {
-		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = zeroed_string,
-		.input	= "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
-			  "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
-			  "\x0B\x8F\x88\x69\x17\xE6\xB4\x3C"
-			  "\xB1\x68\xFD\x14\x52\x64\x61\xB2",
-		.ilen	= 32,
-		.assoc  = zeroed_string,
-		.alen   = 16,
-		.result = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.rlen   = 16,
-	}, {
-		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = zeroed_string,
-		.input	= "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
-			  "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
-			  "\x90\x92\xB7\xE3\x5F\xA3\x9A\x63"
-			  "\x7E\xD7\x1F\xD8\xD3\x7C\x4B\xF5",
-		.ilen	= 32,
-		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.alen   = 16,
-		.result = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.rlen   = 16,
-
-	}, {
-		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
-			  "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
-			  "\x64\x50\xF9\x32\x13\xFB\x74\x61"
-			  "\xF4\xED\x52\xD3\xC5\x10\x55\x3C",
-		.ilen	= 32,
-		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen   = 16,
-		.result = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.rlen   = 16,
-	}, {
-		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
-			  "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
-			  "\x98\x14\xA1\x42\x37\x80\xFD\x90"
-			  "\x68\x12\x01\xA8\x91\x89\xB9\x83"
-			  "\x5B\x11\x77\x12\x9B\xFF\x24\x89"
-			  "\x94\x5F\x18\x12\xBA\x27\x09\x39"
-			  "\x99\x96\x76\x42\x15\x1C\xCD\xCB"
-			  "\xDC\xD3\xDA\x65\x73\xAF\x80\xCD"
-			  "\xD2\xB6\xC2\x4A\x76\xC2\x92\x85"
-			  "\xBD\xCF\x62\x98\x58\x14\xE5\xBD",
-		.ilen	= 80,
-		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen   = 16,
-		.result = "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.rlen   = 64,
-	}, {
-		.key    = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv     = "\x00\x00\x45\x67\x89\xab\xcd\xef",
-		.input	= "\xC1\x76\x33\x85\xE2\x9B\x5F\xDE"
-			  "\xDE\x89\x3D\x42\xE7\xC9\x69\x8A"
-			  "\x44\x6D\xC3\x88\x46\x2E\xC2\x01"
-			  "\x5E\xF6\x0C\x39\xF0\xC4\xA5\x82"
-			  "\xCD\xE8\x31\xCC\x0A\x4C\xE4\x44"
-			  "\x41\xA9\x82\x6F\x22\xA1\x23\x1A"
-			  "\xA8\xE3\x16\xFD\x31\x5C\x27\x31"
-			  "\xF1\x7F\x01\x63\xA3\xAF\x70\xA1"
-			  "\xCF\x07\x57\x41\x67\xD0\xC4\x42"
-			  "\xDB\x18\xC6\x4C\x4C\xE0\x3D\x9F"
-			  "\x05\x07\xFB\x13\x7D\x4A\xCA\x5B"
-			  "\xF0\xBF\x64\x7E\x05\xB1\x72\xEE"
-			  "\x7C\x3B\xD4\xCD\x14\x03\xB2\x2C"
-			  "\xD3\xA9\xEE\xFA\x17\xFC\x9C\xDF"
-			  "\xC7\x75\x40\xFF\xAE\xAD\x1E\x59"
-			  "\x2F\x30\x24\xFB\xAD\x6B\x10\xFA"
-			  "\x6C\x9F\x5B\xE7\x25\xD5\xD0\x25"
-			  "\xAC\x4A\x4B\xDA\xFC\x7A\x85\x1B"
-			  "\x7E\x13\x06\x82\x08\x17\xA4\x35"
-			  "\xEC\xC5\x8D\x63\x96\x81\x0A\x8F"
-			  "\xA3\x05\x38\x95\x20\x1A\x47\x04"
-			  "\x6F\x6D\xDA\x8F\xEF\xC1\x76\x35"
-			  "\x6B\xC7\x4D\x0F\x94\x12\xCA\x3E"
-			  "\x2E\xD5\x03\x2E\x86\x7E\xAA\x3B"
-			  "\x37\x08\x1C\xCF\xBA\x5D\x71\x46"
-			  "\x80\x72\xB0\x4C\x82\x0D\x60\x3C",
-		.ilen	= 208,
-		.assoc  = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
-			  "\xaa\xaa\xaa\xaa\x00\x00\x45\x67"
-			  "\x89\xab\xcd\xef",
-		.alen   = 20,
-		.result = "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff",
-		.rlen   = 192,
-	}, {
-		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
-			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
-			  "\x2E\x44\x3B\x68",
-		.klen	= 20,
-		.iv	= "\x49\x56\xED\x7E\x3B\x24\x4C\xFE",
-		.result	= "\x45\x00\x00\x48\x69\x9A\x00\x00"
-			  "\x80\x11\x4D\xB7\xC0\xA8\x01\x02"
-			  "\xC0\xA8\x01\x01\x0A\x9B\xF1\x56"
-			  "\x38\xD3\x01\x00\x00\x01\x00\x00"
-			  "\x00\x00\x00\x00\x04\x5F\x73\x69"
-			  "\x70\x04\x5F\x75\x64\x70\x03\x73"
-			  "\x69\x70\x09\x63\x79\x62\x65\x72"
-			  "\x63\x69\x74\x79\x02\x64\x6B\x00"
-			  "\x00\x21\x00\x01\x01\x02\x02\x01",
-		.rlen	= 72,
-		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
-			  "\x00\x00\x00\x00\x49\x56\xED\x7E"
-			  "\x3B\x24\x4C\xFE",
-		.alen	= 20,
-		.input	= "\xFE\xCF\x53\x7E\x72\x9D\x5B\x07"
-			  "\xDC\x30\xDF\x52\x8D\xD2\x2B\x76"
-			  "\x8D\x1B\x98\x73\x66\x96\xA6\xFD"
-			  "\x34\x85\x09\xFA\x13\xCE\xAC\x34"
-			  "\xCF\xA2\x43\x6F\x14\xA3\xF3\xCF"
-			  "\x65\x92\x5B\xF1\xF4\xA1\x3C\x5D"
-			  "\x15\xB2\x1E\x18\x84\xF5\xFF\x62"
-			  "\x47\xAE\xAB\xB7\x86\xB9\x3B\xCE"
-			  "\x61\xBC\x17\xD7\x68\xFD\x97\x32"
-			  "\x45\x90\x18\x14\x8F\x6C\xBE\x72"
-			  "\x2F\xD0\x47\x96\x56\x2D\xFD\xB4",
-		.ilen	= 88,
-	}, {
-		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
-			  "\xCA\xFE\xBA\xBE",
-		.klen	= 20,
-		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.result	= "\x45\x00\x00\x3E\x69\x8F\x00\x00"
-			  "\x80\x11\x4D\xCC\xC0\xA8\x01\x02"
-			  "\xC0\xA8\x01\x01\x0A\x98\x00\x35"
-			  "\x00\x2A\x23\x43\xB2\xD0\x01\x00"
-			  "\x00\x01\x00\x00\x00\x00\x00\x00"
-			  "\x03\x73\x69\x70\x09\x63\x79\x62"
-			  "\x65\x72\x63\x69\x74\x79\x02\x64"
-			  "\x6B\x00\x00\x01\x00\x01\x00\x01",
-		.rlen	= 64,
-		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
-			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.alen	= 16,
-		.input	= "\xDE\xB2\x2C\xD9\xB0\x7C\x72\xC1"
-			  "\x6E\x3A\x65\xBE\xEB\x8D\xF3\x04"
-			  "\xA5\xA5\x89\x7D\x33\xAE\x53\x0F"
-			  "\x1B\xA7\x6D\x5D\x11\x4D\x2A\x5C"
-			  "\x3D\xE8\x18\x27\xC1\x0E\x9A\x4F"
-			  "\x51\x33\x0D\x0E\xEC\x41\x66\x42"
-			  "\xCF\xBB\x85\xA5\xB4\x7E\x48\xA4"
-			  "\xEC\x3B\x9B\xA9\x5D\x91\x8B\xD1"
-			  "\x83\xB7\x0D\x3A\xA8\xBC\x6E\xE4"
-			  "\xC3\x09\xE9\xD8\x5A\x41\xAD\x4A",
-		.ilen	= 80,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x11\x22\x33\x44",
-		.klen	= 36,
-		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
-		.result	= "\x45\x00\x00\x30\x69\xA6\x40\x00"
-			  "\x80\x06\x26\x90\xC0\xA8\x01\x02"
-			  "\x93\x89\x15\x5E\x0A\x9E\x00\x8B"
-			  "\x2D\xC5\x7E\xE0\x00\x00\x00\x00"
-			  "\x70\x02\x40\x00\x20\xBF\x00\x00"
-			  "\x02\x04\x05\xB4\x01\x01\x04\x02"
-			  "\x01\x02\x02\x01",
-		.rlen	= 52,
-		.assoc	= "\x4A\x2C\xBF\xE3\x00\x00\x00\x02"
-			  "\x01\x02\x03\x04\x05\x06\x07\x08",
-		.alen	= 16,
-		.input	= "\xFF\x42\x5C\x9B\x72\x45\x99\xDF"
-			  "\x7A\x3B\xCD\x51\x01\x94\xE0\x0D"
-			  "\x6A\x78\x10\x7F\x1B\x0B\x1C\xBF"
-			  "\x06\xEF\xAE\x9D\x65\xA5\xD7\x63"
-			  "\x74\x8A\x63\x79\x85\x77\x1D\x34"
-			  "\x7F\x05\x45\x65\x9F\x14\xE9\x9D"
-			  "\xEF\x84\x2D\x8E\xB3\x35\xF4\xEE"
-			  "\xCF\xDB\xF8\x31\x82\x4B\x4C\x49"
-			  "\x15\x95\x6C\x96",
-		.ilen	= 68,
-	}, {
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00",
-		.klen	= 20,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.result	= "\x45\x00\x00\x3C\x99\xC5\x00\x00"
-			  "\x80\x01\xCB\x7A\x40\x67\x93\x18"
-			  "\x01\x01\x01\x01\x08\x00\x07\x5C"
-			  "\x02\x00\x44\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x75\x76\x77\x61\x62\x63\x64\x65"
-			  "\x66\x67\x68\x69\x01\x02\x02\x01",
-		.rlen	= 64,
-		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.alen	= 16,
-		.input	= "\x46\x88\xDA\xF2\xF9\x73\xA3\x92"
-			  "\x73\x29\x09\xC3\x31\xD5\x6D\x60"
-			  "\xF6\x94\xAB\xAA\x41\x4B\x5E\x7F"
-			  "\xF5\xFD\xCD\xFF\xF5\xE9\xA2\x84"
-			  "\x45\x64\x76\x49\x27\x19\xFF\xB6"
-			  "\x4D\xE7\xD9\xDC\xA1\xE1\xD8\x94"
-			  "\xBC\x3B\xD5\x78\x73\xED\x4D\x18"
-			  "\x1D\x19\xD4\xD5\xC8\xC1\x8A\xF3"
-			  "\xF8\x21\xD4\x96\xEE\xB0\x96\xE9"
-			  "\x8A\xD2\xB6\x9E\x47\x99\xC7\x1D",
-		.ilen	= 80,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E\x43",
-		.klen	= 20,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.result	= "\x45\x00\x00\x3C\x99\xC3\x00\x00"
-			  "\x80\x01\xCB\x7C\x40\x67\x93\x18"
-			  "\x01\x01\x01\x01\x08\x00\x08\x5C"
-			  "\x02\x00\x43\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x75\x76\x77\x61\x62\x63\x64\x65"
-			  "\x66\x67\x68\x69\x01\x02\x02\x01",
-		.rlen	= 64,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.input	= "\xFB\xA2\xCA\xA4\x85\x3C\xF9\xF0"
-			  "\xF2\x2C\xB1\x0D\x86\xDD\x83\xB0"
-			  "\xFE\xC7\x56\x91\xCF\x1A\x04\xB0"
-			  "\x0D\x11\x38\xEC\x9C\x35\x79\x17"
-			  "\x65\xAC\xBD\x87\x01\xAD\x79\x84"
-			  "\x5B\xF9\xFE\x3F\xBA\x48\x7B\xC9"
-			  "\x17\x55\xE6\x66\x2B\x4C\x8D\x0D"
-			  "\x1F\x5E\x22\x73\x95\x30\x32\x0A"
-			  "\xE0\xD7\x31\xCC\x97\x8E\xCA\xFA"
-			  "\xEA\xE8\x8F\x00\xE8\x0D\x6E\x48",
-		.ilen	= 80,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E\x43",
-		.klen	= 20,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.result	= "\x45\x00\x00\x1C\x42\xA2\x00\x00"
-			  "\x80\x01\x44\x1F\x40\x67\x93\xB6"
-			  "\xE0\x00\x00\x02\x0A\x00\xF5\xFF"
-			  "\x01\x02\x02\x01",
-		.rlen	= 28,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.input	= "\xFB\xA2\xCA\x84\x5E\x5D\xF9\xF0"
-			  "\xF2\x2C\x3E\x6E\x86\xDD\x83\x1E"
-			  "\x1F\xC6\x57\x92\xCD\x1A\xF9\x13"
-			  "\x0E\x13\x79\xED\x36\x9F\x07\x1F"
-			  "\x35\xE0\x34\xBE\x95\xF1\x12\xE4"
-			  "\xE7\xD0\x5D\x35",
-		.ilen	= 44,
-	}, {
-		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
-			  "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\xCA\xFE\xBA\xBE",
-		.klen	= 28,
-		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.result	= "\x45\x00\x00\x28\xA4\xAD\x40\x00"
-			  "\x40\x06\x78\x80\x0A\x01\x03\x8F"
-			  "\x0A\x01\x06\x12\x80\x23\x06\xB8"
-			  "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E"
-			  "\x50\x10\x16\xD0\x75\x68\x00\x01",
-		.rlen	= 40,
-		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
-			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.alen	= 16,
-		.input	= "\xA5\xB1\xF8\x06\x60\x29\xAE\xA4"
-			  "\x0E\x59\x8B\x81\x22\xDE\x02\x42"
-			  "\x09\x38\xB3\xAB\x33\xF8\x28\xE6"
-			  "\x87\xB8\x85\x8B\x5B\xFB\xDB\xD0"
-			  "\x31\x5B\x27\x45\x21\x44\xCC\x77"
-			  "\x95\x45\x7B\x96\x52\x03\x7F\x53"
-			  "\x18\x02\x7B\x5B\x4C\xD7\xA6\x36",
-		.ilen	= 56,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xDE\xCA\xF8\x88",
-		.klen	= 20,
-		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
-		.result	= "\x45\x00\x00\x49\x33\xBA\x00\x00"
-			  "\x7F\x11\x91\x06\xC3\xFB\x1D\x10"
-			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
-			  "\x00\x35\xDD\x7B\x80\x03\x02\xD5"
-			  "\x00\x00\x4E\x20\x00\x1E\x8C\x18"
-			  "\xD7\x5B\x81\xDC\x91\xBA\xA0\x47"
-			  "\x6B\x91\xB9\x24\xB2\x80\x38\x9D"
-			  "\x92\xC9\x63\xBA\xC0\x46\xEC\x95"
-			  "\x9B\x62\x66\xC0\x47\x22\xB1\x49"
-			  "\x23\x01\x01\x01",
-		.rlen	= 76,
-		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
-			  "\xCE\xFA\xCE\x74",
-		.alen	= 20,
-		.input	= "\x18\xA6\xFD\x42\xF7\x2C\xBF\x4A"
-			  "\xB2\xA2\xEA\x90\x1F\x73\xD8\x14"
-			  "\xE3\xE7\xF2\x43\xD9\x54\x12\xE1"
-			  "\xC3\x49\xC1\xD2\xFB\xEC\x16\x8F"
-			  "\x91\x90\xFE\xEB\xAF\x2C\xB0\x19"
-			  "\x84\xE6\x58\x63\x96\x5D\x74\x72"
-			  "\xB7\x9D\xA3\x45\xE0\xE7\x80\x19"
-			  "\x1F\x0D\x2F\x0E\x0F\x49\x6C\x22"
-			  "\x6F\x21\x27\xB2\x7D\xB3\x57\x24"
-			  "\xE7\x84\x5D\x68\x65\x1F\x57\xE6"
-			  "\x5F\x35\x4F\x75\xFF\x17\x01\x57"
-			  "\x69\x62\x34\x36",
-		.ilen	= 92,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x73\x61\x6C\x74",
-		.klen	= 36,
-		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
-		.result	= "\x45\x08\x00\x28\x73\x2C\x00\x00"
-			  "\x40\x06\xE9\xF9\x0A\x01\x06\x12"
-			  "\x0A\x01\x03\x8F\x06\xB8\x80\x23"
-			  "\xDD\x6B\xAF\xBE\xCB\x71\x26\x02"
-			  "\x50\x10\x1F\x64\x6D\x54\x00\x01",
-		.rlen	= 40,
-		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
-			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
-			  "\x69\x76\x65\x63",
-		.alen	= 20,
-		.input	= "\xF2\xD6\x9E\xCD\xBD\x5A\x0D\x5B"
-			  "\x8D\x5E\xF3\x8B\xAD\x4D\xA5\x8D"
-			  "\x1F\x27\x8F\xDE\x98\xEF\x67\x54"
-			  "\x9D\x52\x4A\x30\x18\xD9\xA5\x7F"
-			  "\xF4\xD3\xA3\x1C\xE6\x73\x11\x9E"
-			  "\x45\x16\x26\xC2\x41\x57\x71\xE3"
-			  "\xB7\xEE\xBC\xA6\x14\xC8\x9B\x35",
-		.ilen	= 56,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E\x43",
-		.klen	= 20,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.result	= "\x45\x00\x00\x49\x33\x3E\x00\x00"
-			  "\x7F\x11\x91\x82\xC3\xFB\x1D\x10"
-			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
-			  "\x00\x35\xCB\x45\x80\x03\x02\x5B"
-			  "\x00\x00\x01\xE0\x00\x1E\x8C\x18"
-			  "\xD6\x57\x59\xD5\x22\x84\xA0\x35"
-			  "\x2C\x71\x47\x5C\x88\x80\x39\x1C"
-			  "\x76\x4D\x6E\x5E\xE0\x49\x6B\x32"
-			  "\x5A\xE2\x70\xC0\x38\x99\x49\x39"
-			  "\x15\x01\x01\x01",
-		.rlen	= 76,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.input	= "\xFB\xA2\xCA\xD1\x2F\xC1\xF9\xF0"
-			  "\x0D\x3C\xEB\xF3\x05\x41\x0D\xB8"
-			  "\x3D\x77\x84\xB6\x07\x32\x3D\x22"
-			  "\x0F\x24\xB0\xA9\x7D\x54\x18\x28"
-			  "\x00\xCA\xDB\x0F\x68\xD9\x9E\xF0"
-			  "\xE0\xC0\xC8\x9A\xE9\xBE\xA8\x88"
-			  "\x4E\x52\xD6\x5B\xC1\xAF\xD0\x74"
-			  "\x0F\x74\x24\x44\x74\x7B\x5B\x39"
-			  "\xAB\x53\x31\x63\xAA\xD4\x55\x0E"
-			  "\xE5\x16\x09\x75\xCD\xB6\x08\xC5"
-			  "\x76\x91\x89\x60\x97\x63\xB8\xE1"
-			  "\x8C\xAA\x81\xE2",
-		.ilen	= 92,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x73\x61\x6C\x74",
-		.klen	= 36,
-		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
-		.result	= "\x63\x69\x73\x63\x6F\x01\x72\x75"
-			  "\x6C\x65\x73\x01\x74\x68\x65\x01"
-			  "\x6E\x65\x74\x77\x65\x01\x64\x65"
-			  "\x66\x69\x6E\x65\x01\x74\x68\x65"
-			  "\x74\x65\x63\x68\x6E\x6F\x6C\x6F"
-			  "\x67\x69\x65\x73\x01\x74\x68\x61"
-			  "\x74\x77\x69\x6C\x6C\x01\x64\x65"
-			  "\x66\x69\x6E\x65\x74\x6F\x6D\x6F"
-			  "\x72\x72\x6F\x77\x01\x02\x02\x01",
-		.rlen	= 72,
-		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
-			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
-			  "\x69\x76\x65\x63",
-		.alen	= 20,
-		.input	= "\xD4\xB7\xED\x86\xA1\x77\x7F\x2E"
-			  "\xA1\x3D\x69\x73\xD3\x24\xC6\x9E"
-			  "\x7B\x43\xF8\x26\xFB\x56\x83\x12"
-			  "\x26\x50\x8B\xEB\xD2\xDC\xEB\x18"
-			  "\xD0\xA6\xDF\x10\xE5\x48\x7D\xF0"
-			  "\x74\x11\x3E\x14\xC6\x41\x02\x4E"
-			  "\x3E\x67\x73\xD9\x1A\x62\xEE\x42"
-			  "\x9B\x04\x3A\x10\xE3\xEF\xE6\xB0"
-			  "\x12\xA4\x93\x63\x41\x23\x64\xF8"
-			  "\xC0\xCA\xC5\x87\xF2\x49\xE5\x6B"
-			  "\x11\xE2\x4F\x30\xE4\x4C\xCC\x76",
-		.ilen	= 88,
-	}, {
-		.key	= "\x7D\x77\x3D\x00\xC1\x44\xC5\x25"
-			  "\xAC\x61\x9D\x18\xC8\x4A\x3F\x47"
-			  "\xD9\x66\x42\x67",
-		.klen	= 20,
-		.iv	= "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
-		.result	= "\x01\x02\x02\x01",
-		.rlen	= 4,
-		.assoc	= "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF"
-			  "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
-		.alen	= 16,
-		.input	= "\x43\x7F\x86\x6B\xCB\x3F\x69\x9F"
-			  "\xE9\xB0\x82\x2B\xAC\x96\x1C\x45"
-			  "\x04\xBE\xF2\x70",
-		.ilen	= 20,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xDE\xCA\xF8\x88",
-		.klen	= 20,
-		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
-		.result	= "\x74\x6F\x01\x62\x65\x01\x6F\x72"
-			  "\x01\x6E\x6F\x74\x01\x74\x6F\x01"
-			  "\x62\x65\x00\x01",
-		.rlen	= 20,
-		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
-			  "\xCE\xFA\xCE\x74",
-		.alen	= 20,
-		.input	= "\x29\xC9\xFC\x69\xA1\x97\xD0\x38"
-			  "\xCC\xDD\x14\xE2\xDD\xFC\xAA\x05"
-			  "\x43\x33\x21\x64\x41\x25\x03\x52"
-			  "\x43\x03\xED\x3C\x6C\x5F\x28\x38"
-			  "\x43\xAF\x8C\x3E",
-		.ilen	= 36,
-	}, {
-		.key	= "\x6C\x65\x67\x61\x6C\x69\x7A\x65"
-			  "\x6D\x61\x72\x69\x6A\x75\x61\x6E"
-			  "\x61\x61\x6E\x64\x64\x6F\x69\x74"
-			  "\x62\x65\x66\x6F\x72\x65\x69\x61"
-			  "\x74\x75\x72\x6E",
-		.klen	= 36,
-		.iv	= "\x33\x30\x21\x69\x67\x65\x74\x6D",
-		.result	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
-			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
-			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01",
-		.rlen	= 52,
-		.assoc	= "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\x33\x30\x21\x69"
-			  "\x67\x65\x74\x6D",
-		.alen	= 20,
-		.input	= "\xF9\x7A\xB2\xAA\x35\x6D\x8E\xDC"
-			  "\xE1\x76\x44\xAC\x8C\x78\xE2\x5D"
-			  "\xD2\x4D\xED\xBB\x29\xEB\xF1\xB6"
-			  "\x4A\x27\x4B\x39\xB4\x9C\x3A\x86"
-			  "\x4C\xD3\xD7\x8C\xA4\xAE\x68\xA3"
-			  "\x2B\x42\x45\x8F\xB5\x7D\xBE\x82"
-			  "\x1D\xCC\x63\xB9\xD0\x93\x7B\xA2"
-			  "\x94\x5F\x66\x93\x68\x66\x1A\x32"
-			  "\x9F\xB4\xC0\x53",
-		.ilen	= 68,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E\x43",
-		.klen	= 20,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.result	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
-			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
-			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01",
-		.rlen	= 52,
-		.assoc	= "\x3F\x7E\xF6\x42\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.input	= "\xFB\xA2\xCA\xA8\xC6\xC5\xF9\xF0"
-			  "\xF2\x2C\xA5\x4A\x06\x12\x10\xAD"
-			  "\x3F\x6E\x57\x91\xCF\x1A\xCA\x21"
-			  "\x0D\x11\x7C\xEC\x9C\x35\x79\x17"
-			  "\x65\xAC\xBD\x87\x01\xAD\x79\x84"
-			  "\x5B\xF9\xFE\x3F\xBA\x48\x7B\xC9"
-			  "\x63\x21\x93\x06\x84\xEE\xCA\xDB"
-			  "\x56\x91\x25\x46\xE7\xA9\x5C\x97"
-			  "\x40\xD7\xCB\x05",
-		.ilen	= 68,
-	}, {
-		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
-			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
-			  "\x22\x43\x3C\x64",
-		.klen	= 20,
-		.iv	= "\x48\x55\xEC\x7D\x3A\x23\x4B\xFD",
-		.result	= "\x08\x00\xC6\xCD\x02\x00\x07\x00"
-			  "\x61\x62\x63\x64\x65\x66\x67\x68"
-			  "\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70"
-			  "\x71\x72\x73\x74\x01\x02\x02\x01",
-		.rlen	= 32,
-		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
-			  "\x00\x00\x00\x07\x48\x55\xEC\x7D"
-			  "\x3A\x23\x4B\xFD",
-		.alen	= 20,
-		.input	= "\x74\x75\x2E\x8A\xEB\x5D\x87\x3C"
-			  "\xD7\xC0\xF4\xAC\xC3\x6C\x4B\xFF"
-			  "\x84\xB7\xD7\xB9\x8F\x0C\xA8\xB6"
-			  "\xAC\xDA\x68\x94\xBC\x61\x90\x69"
-			  "\xEF\x9C\xBC\x28\xFE\x1B\x56\xA7"
-			  "\xC4\xE0\xD5\x8C\x86\xCD\x2B\xC0",
-		.ilen	= 48,
-	}
-};
-
-static const struct aead_testvec aes_gcm_rfc4543_enc_tv_template[] = {
-	{ /* From draft-mcgrew-gcm-test-01 */
-		.key	= "\x4c\x80\xcd\xef\xbb\x5d\x10\xda"
-			  "\x90\x6a\xc7\x3c\x36\x13\xa6\x34"
-			  "\x22\x43\x3c\x64",
-		.klen	= 20,
-		.iv	= zeroed_string,
-		.assoc	= "\x00\x00\x43\x21\x00\x00\x00\x07"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.alen	= 16,
-		.input	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
-			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
-			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
-			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01",
-		.ilen	= 52,
-		.result	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
-			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
-			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
-			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01\xf2\xa9\xa8\x36"
-			  "\xe1\x55\x10\x6a\xa8\xdc\xd6\x18"
-			  "\xe4\x09\x9a\xaa",
-		.rlen	= 68,
-	}
-};
-
-static const struct aead_testvec aes_gcm_rfc4543_dec_tv_template[] = {
-	{ /* From draft-mcgrew-gcm-test-01 */
-		.key	= "\x4c\x80\xcd\xef\xbb\x5d\x10\xda"
-			  "\x90\x6a\xc7\x3c\x36\x13\xa6\x34"
-			  "\x22\x43\x3c\x64",
-		.klen	= 20,
-		.iv	= zeroed_string,
-		.assoc	= "\x00\x00\x43\x21\x00\x00\x00\x07"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.alen	= 16,
-		.input	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
-			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
-			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
-			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01\xf2\xa9\xa8\x36"
-			  "\xe1\x55\x10\x6a\xa8\xdc\xd6\x18"
-			  "\xe4\x09\x9a\xaa",
-		.ilen	= 68,
-		.result	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
-			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
-			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
-			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01",
-		.rlen	= 52,
-	}, { /* nearly same as previous, but should fail */
-		.key	= "\x4c\x80\xcd\xef\xbb\x5d\x10\xda"
-			  "\x90\x6a\xc7\x3c\x36\x13\xa6\x34"
-			  "\x22\x43\x3c\x64",
-		.klen	= 20,
-		.iv	= zeroed_string,
-		.assoc	= "\x00\x00\x43\x21\x00\x00\x00\x07"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.alen	= 16,
-		.input	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
-			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
-			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
-			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01\xf2\xa9\xa8\x36"
-			  "\xe1\x55\x10\x6a\xa8\xdc\xd6\x18"
-			  "\x00\x00\x00\x00",
-		.ilen	= 68,
-		.novrfy = 1,
-		.result	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
-			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
-			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
-			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01",
-		.rlen	= 52,
-	},
-};
-
-static const struct aead_testvec aes_ccm_enc_tv_template[] = {
-	{ /* From RFC 3610 */
-		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
-		.klen	= 16,
-		.iv	= "\x01\x00\x00\x00\x03\x02\x01\x00"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07",
-		.alen	= 8,
-		.input	= "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e",
-		.ilen	= 23,
-		.result	= "\x58\x8c\x97\x9a\x61\xc6\x63\xd2"
-			  "\xf0\x66\xd0\xc2\xc0\xf9\x89\x80"
-			  "\x6d\x5f\x6b\x61\xda\xc3\x84\x17"
-			  "\xe8\xd1\x2c\xfd\xf9\x26\xe0",
-		.rlen	= 31,
-	}, {
-		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
-		.klen	= 16,
-		.iv	= "\x01\x00\x00\x00\x07\x06\x05\x04"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b",
-		.alen	= 12,
-		.input	= "\x0c\x0d\x0e\x0f\x10\x11\x12\x13"
-			  "\x14\x15\x16\x17\x18\x19\x1a\x1b"
-			  "\x1c\x1d\x1e\x1f",
-		.ilen	= 20,
-		.result	= "\xdc\xf1\xfb\x7b\x5d\x9e\x23\xfb"
-			  "\x9d\x4e\x13\x12\x53\x65\x8a\xd8"
-			  "\x6e\xbd\xca\x3e\x51\xe8\x3f\x07"
-			  "\x7d\x9c\x2d\x93",
-		.rlen	= 28,
-	}, {
-		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
-		.klen	= 16,
-		.iv	= "\x01\x00\x00\x00\x0b\x0a\x09\x08"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07",
-		.alen	= 8,
-		.input	= "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20",
-		.ilen	= 25,
-		.result	= "\x82\x53\x1a\x60\xcc\x24\x94\x5a"
-			  "\x4b\x82\x79\x18\x1a\xb5\xc8\x4d"
-			  "\xf2\x1c\xe7\xf9\xb7\x3f\x42\xe1"
-			  "\x97\xea\x9c\x07\xe5\x6b\x5e\xb1"
-			  "\x7e\x5f\x4e",
-		.rlen	= 35,
-	}, {
-		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
-		.klen	= 16,
-		.iv	= "\x01\x00\x00\x00\x0c\x0b\x0a\x09"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b",
-		.alen	= 12,
-		.input	= "\x0c\x0d\x0e\x0f\x10\x11\x12\x13"
-			  "\x14\x15\x16\x17\x18\x19\x1a\x1b"
-			  "\x1c\x1d\x1e",
-		.ilen	= 19,
-		.result	= "\x07\x34\x25\x94\x15\x77\x85\x15"
-			  "\x2b\x07\x40\x98\x33\x0a\xbb\x14"
-			  "\x1b\x94\x7b\x56\x6a\xa9\x40\x6b"
-			  "\x4d\x99\x99\x88\xdd",
-		.rlen	= 29,
-	}, {
-		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
-			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
-		.klen	= 16,
-		.iv	= "\x01\x00\x33\x56\x8e\xf7\xb2\x63"
-			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
-		.assoc	= "\x63\x01\x8f\x76\xdc\x8a\x1b\xcb",
-		.alen	= 8,
-		.input	= "\x90\x20\xea\x6f\x91\xbd\xd8\x5a"
-			  "\xfa\x00\x39\xba\x4b\xaf\xf9\xbf"
-			  "\xb7\x9c\x70\x28\x94\x9c\xd0\xec",
-		.ilen	= 24,
-		.result	= "\x4c\xcb\x1e\x7c\xa9\x81\xbe\xfa"
-			  "\xa0\x72\x6c\x55\xd3\x78\x06\x12"
-			  "\x98\xc8\x5c\x92\x81\x4a\xbc\x33"
-			  "\xc5\x2e\xe8\x1d\x7d\x77\xc0\x8a",
-		.rlen	= 32,
-	}, {
-		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
-			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
-		.klen	= 16,
-		.iv	= "\x01\x00\xd5\x60\x91\x2d\x3f\x70"
-			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
-		.assoc	= "\xcd\x90\x44\xd2\xb7\x1f\xdb\x81"
-			  "\x20\xea\x60\xc0",
-		.alen	= 12,
-		.input	= "\x64\x35\xac\xba\xfb\x11\xa8\x2e"
-			  "\x2f\x07\x1d\x7c\xa4\xa5\xeb\xd9"
-			  "\x3a\x80\x3b\xa8\x7f",
-		.ilen	= 21,
-		.result	= "\x00\x97\x69\xec\xab\xdf\x48\x62"
-			  "\x55\x94\xc5\x92\x51\xe6\x03\x57"
-			  "\x22\x67\x5e\x04\xc8\x47\x09\x9e"
-			  "\x5a\xe0\x70\x45\x51",
-		.rlen	= 29,
-	}, {
-		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
-			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
-		.klen	= 16,
-		.iv	= "\x01\x00\x42\xff\xf8\xf1\x95\x1c"
-			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
-		.assoc	= "\xd8\x5b\xc7\xe6\x9f\x94\x4f\xb8",
-		.alen	= 8,
-		.input	= "\x8a\x19\xb9\x50\xbc\xf7\x1a\x01"
-			  "\x8e\x5e\x67\x01\xc9\x17\x87\x65"
-			  "\x98\x09\xd6\x7d\xbe\xdd\x18",
-		.ilen	= 23,
-		.result	= "\xbc\x21\x8d\xaa\x94\x74\x27\xb6"
-			  "\xdb\x38\x6a\x99\xac\x1a\xef\x23"
-			  "\xad\xe0\xb5\x29\x39\xcb\x6a\x63"
-			  "\x7c\xf9\xbe\xc2\x40\x88\x97\xc6"
-			  "\xba",
-		.rlen	= 33,
-	}, {
-		/* This is taken from FIPS CAVS. */
-		.key	= "\x83\xac\x54\x66\xc2\xeb\xe5\x05"
-			  "\x2e\x01\xd1\xfc\x5d\x82\x66\x2e",
-		.klen	= 16,
-		.iv	= "\x03\x96\xac\x59\x30\x07\xa1\xe2\xa2\xc7\x55\x24\0\0\0\0",
-		.alen	= 0,
-		.input	= "\x19\xc8\x81\xf6\xe9\x86\xff\x93"
-			  "\x0b\x78\x67\xe5\xbb\xb7\xfc\x6e"
-			  "\x83\x77\xb3\xa6\x0c\x8c\x9f\x9c"
-			  "\x35\x2e\xad\xe0\x62\xf9\x91\xa1",
-		.ilen	= 32,
-		.result	= "\xab\x6f\xe1\x69\x1d\x19\x99\xa8"
-			  "\x92\xa0\xc4\x6f\x7e\xe2\x8b\xb1"
-			  "\x70\xbb\x8c\xa6\x4c\x6e\x97\x8a"
-			  "\x57\x2b\xbe\x5d\x98\xa6\xb1\x32"
-			  "\xda\x24\xea\xd9\xa1\x39\x98\xfd"
-			  "\xa4\xbe\xd9\xf2\x1a\x6d\x22\xa8",
-		.rlen	= 48,
-	}, {
-		.key	= "\x1e\x2c\x7e\x01\x41\x9a\xef\xc0"
-			  "\x0d\x58\x96\x6e\x5c\xa2\x4b\xd3",
-		.klen	= 16,
-		.iv	= "\x03\x4f\xa3\x19\xd3\x01\x5a\xd8"
-			  "\x30\x60\x15\x56\x00\x00\x00\x00",
-		.assoc	= "\xda\xe6\x28\x9c\x45\x2d\xfd\x63"
-			  "\x5e\xda\x4c\xb6\xe6\xfc\xf9\xb7"
-			  "\x0c\x56\xcb\xe4\xe0\x05\x7a\xe1"
-			  "\x0a\x63\x09\x78\xbc\x2c\x55\xde",
-		.alen	= 32,
-		.input	= "\x87\xa3\x36\xfd\x96\xb3\x93\x78"
-			  "\xa9\x28\x63\xba\x12\xa3\x14\x85"
-			  "\x57\x1e\x06\xc9\x7b\x21\xef\x76"
-			  "\x7f\x38\x7e\x8e\x29\xa4\x3e\x7e",
-		.ilen	= 32,
-		.result	= "\x8a\x1e\x11\xf0\x02\x6b\xe2\x19"
-			  "\xfc\x70\xc4\x6d\x8e\xb7\x99\xab"
-			  "\xc5\x4b\xa2\xac\xd3\xf3\x48\xff"
-			  "\x3b\xb5\xce\x53\xef\xde\xbb\x02"
-			  "\xa9\x86\x15\x6c\x13\xfe\xda\x0a"
-			  "\x22\xb8\x29\x3d\xd8\x39\x9a\x23",
-		.rlen	= 48,
-	}, {
-		.key	= "\xf4\x6b\xc2\x75\x62\xfe\xb4\xe1"
-			  "\xa3\xf0\xff\xdd\x4e\x4b\x12\x75"
-			  "\x53\x14\x73\x66\x8d\x88\xf6\x80",
-		.klen	= 24,
-		.iv	= "\x03\xa0\x20\x35\x26\xf2\x21\x8d"
-			  "\x50\x20\xda\xe2\x00\x00\x00\x00",
-		.assoc	= "\x5b\x9e\x13\x67\x02\x5e\xef\xc1"
-			  "\x6c\xf9\xd7\x1e\x52\x8f\x7a\x47"
-			  "\xe9\xd4\xcf\x20\x14\x6e\xf0\x2d"
-			  "\xd8\x9e\x2b\x56\x10\x23\x56\xe7",
-		.alen	= 32,
-		.result	= "\x36\xea\x7a\x70\x08\xdc\x6a\xbc"
-			  "\xad\x0c\x7a\x63\xf6\x61\xfd\x9b",
-		.rlen	= 16,
-	}, {
-		.key	= "\x56\xdf\x5c\x8f\x26\x3f\x0e\x42"
-			  "\xef\x7a\xd3\xce\xfc\x84\x60\x62"
-			  "\xca\xb4\x40\xaf\x5f\xc9\xc9\x01",
-		.klen	= 24,
-		.iv	= "\x03\xd6\x3c\x8c\x86\x84\xb6\xcd"
-			  "\xef\x09\x2e\x94\x00\x00\x00\x00",
-		.assoc	= "\x02\x65\x78\x3c\xe9\x21\x30\x91"
-			  "\xb1\xb9\xda\x76\x9a\x78\x6d\x95"
-			  "\xf2\x88\x32\xa3\xf2\x50\xcb\x4c"
-			  "\xe3\x00\x73\x69\x84\x69\x87\x79",
-		.alen	= 32,
-		.input	= "\x9f\xd2\x02\x4b\x52\x49\x31\x3c"
-			  "\x43\x69\x3a\x2d\x8e\x70\xad\x7e"
-			  "\xe0\xe5\x46\x09\x80\x89\x13\xb2"
-			  "\x8c\x8b\xd9\x3f\x86\xfb\xb5\x6b",
-		.ilen	= 32,
-		.result	= "\x39\xdf\x7c\x3c\x5a\x29\xb9\x62"
-			  "\x5d\x51\xc2\x16\xd8\xbd\x06\x9f"
-			  "\x9b\x6a\x09\x70\xc1\x51\x83\xc2"
-			  "\x66\x88\x1d\x4f\x9a\xda\xe0\x1e"
-			  "\xc7\x79\x11\x58\xe5\x6b\x20\x40"
-			  "\x7a\xea\x46\x42\x8b\xe4\x6f\xe1",
-		.rlen	= 48,
-	}, {
-		.key	= "\xe0\x8d\x99\x71\x60\xd7\x97\x1a"
-			  "\xbd\x01\x99\xd5\x8a\xdf\x71\x3a"
-			  "\xd3\xdf\x24\x4b\x5e\x3d\x4b\x4e"
-			  "\x30\x7a\xb9\xd8\x53\x0a\x5e\x2b",
-		.klen	= 32,
-		.iv	= "\x03\x1e\x29\x91\xad\x8e\xc1\x53"
-			  "\x0a\xcf\x2d\xbe\x00\x00\x00\x00",
-		.assoc	= "\x19\xb6\x1f\x57\xc4\xf3\xf0\x8b"
-			  "\x78\x2b\x94\x02\x29\x0f\x42\x27"
-			  "\x6b\x75\xcb\x98\x34\x08\x7e\x79"
-			  "\xe4\x3e\x49\x0d\x84\x8b\x22\x87",
-		.alen	= 32,
-		.input	= "\xe1\xd9\xd8\x13\xeb\x3a\x75\x3f"
-			  "\x9d\xbd\x5f\x66\xbe\xdc\xbb\x66"
-			  "\xbf\x17\x99\x62\x4a\x39\x27\x1f"
-			  "\x1d\xdc\x24\xae\x19\x2f\x98\x4c",
-		.ilen	= 32,
-		.result	= "\x19\xb8\x61\x33\x45\x2b\x43\x96"
-			  "\x6f\x51\xd0\x20\x30\x7d\x9b\xc6"
-			  "\x26\x3d\xf8\xc9\x65\x16\xa8\x9f"
-			  "\xf0\x62\x17\x34\xf2\x1e\x8d\x75"
-			  "\x4e\x13\xcc\xc0\xc3\x2a\x54\x2d",
-		.rlen	= 40,
-	}, {
-		.key	= "\x7c\xc8\x18\x3b\x8d\x99\xe0\x7c"
-			  "\x45\x41\xb8\xbd\x5c\xa7\xc2\x32"
-			  "\x8a\xb8\x02\x59\xa4\xfe\xa9\x2c"
-			  "\x09\x75\x9a\x9b\x3c\x9b\x27\x39",
-		.klen	= 32,
-		.iv	= "\x03\xf9\xd9\x4e\x63\xb5\x3d\x9d"
-			  "\x43\xf6\x1e\x50\0\0\0\0",
-		.assoc	= "\x57\xf5\x6b\x8b\x57\x5c\x3d\x3b"
-			  "\x13\x02\x01\x0c\x83\x4c\x96\x35"
-			  "\x8e\xd6\x39\xcf\x7d\x14\x9b\x94"
-			  "\xb0\x39\x36\xe6\x8f\x57\xe0\x13",
-		.alen	= 32,
-		.input	= "\x3b\x6c\x29\x36\xb6\xef\x07\xa6"
-			  "\x83\x72\x07\x4f\xcf\xfa\x66\x89"
-			  "\x5f\xca\xb1\xba\xd5\x8f\x2c\x27"
-			  "\x30\xdb\x75\x09\x93\xd4\x65\xe4",
-		.ilen	= 32,
-		.result	= "\xb0\x88\x5a\x33\xaa\xe5\xc7\x1d"
-			  "\x85\x23\xc7\xc6\x2f\xf4\x1e\x3d"
-			  "\xcc\x63\x44\x25\x07\x78\x4f\x9e"
-			  "\x96\xb8\x88\xeb\xbc\x48\x1f\x06"
-			  "\x39\xaf\x39\xac\xd8\x4a\x80\x39"
-			  "\x7b\x72\x8a\xf7",
-		.rlen	= 44,
-	}, {
-		.key	= "\xab\xd0\xe9\x33\x07\x26\xe5\x83"
-			  "\x8c\x76\x95\xd4\xb6\xdc\xf3\x46"
-			  "\xf9\x8f\xad\xe3\x02\x13\x83\x77"
-			  "\x3f\xb0\xf1\xa1\xa1\x22\x0f\x2b",
-		.klen	= 32,
-		.iv	= "\x03\x24\xa7\x8b\x07\xcb\xcc\x0e"
-			  "\xe6\x33\xbf\xf5\x00\x00\x00\x00",
-		.assoc	= "\xd4\xdb\x30\x1d\x03\xfe\xfd\x5f"
-			  "\x87\xd4\x8c\xb6\xb6\xf1\x7a\x5d"
-			  "\xab\x90\x65\x8d\x8e\xca\x4d\x4f"
-			  "\x16\x0c\x40\x90\x4b\xc7\x36\x73",
-		.alen	= 32,
-		.input	= "\xf5\xc6\x7d\x48\xc1\xb7\xe6\x92"
-			  "\x97\x5a\xca\xc4\xa9\x6d\xf9\x3d"
-			  "\x6c\xde\xbc\xf1\x90\xea\x6a\xb2"
-			  "\x35\x86\x36\xaf\x5c\xfe\x4b\x3a",
-		.ilen	= 32,
-		.result	= "\x83\x6f\x40\x87\x72\xcf\xc1\x13"
-			  "\xef\xbb\x80\x21\x04\x6c\x58\x09"
-			  "\x07\x1b\xfc\xdf\xc0\x3f\x5b\xc7"
-			  "\xe0\x79\xa8\x6e\x71\x7c\x3f\xcf"
-			  "\x5c\xda\xb2\x33\xe5\x13\xe2\x0d"
-			  "\x74\xd1\xef\xb5\x0f\x3a\xb5\xf8",
-		.rlen	= 48,
-	}
-};
-
-static const struct aead_testvec aes_ccm_dec_tv_template[] = {
-	{ /* From RFC 3610 */
-		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
-		.klen	= 16,
-		.iv	= "\x01\x00\x00\x00\x03\x02\x01\x00"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07",
-		.alen	= 8,
-		.input	= "\x58\x8c\x97\x9a\x61\xc6\x63\xd2"
-			  "\xf0\x66\xd0\xc2\xc0\xf9\x89\x80"
-			  "\x6d\x5f\x6b\x61\xda\xc3\x84\x17"
-			  "\xe8\xd1\x2c\xfd\xf9\x26\xe0",
-		.ilen	= 31,
-		.result	= "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e",
-		.rlen	= 23,
-	}, {
-		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
-		.klen	= 16,
-		.iv	= "\x01\x00\x00\x00\x07\x06\x05\x04"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b",
-		.alen	= 12,
-		.input	= "\xdc\xf1\xfb\x7b\x5d\x9e\x23\xfb"
-			  "\x9d\x4e\x13\x12\x53\x65\x8a\xd8"
-			  "\x6e\xbd\xca\x3e\x51\xe8\x3f\x07"
-			  "\x7d\x9c\x2d\x93",
-		.ilen	= 28,
-		.result	= "\x0c\x0d\x0e\x0f\x10\x11\x12\x13"
-			  "\x14\x15\x16\x17\x18\x19\x1a\x1b"
-			  "\x1c\x1d\x1e\x1f",
-		.rlen	= 20,
-	}, {
-		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
-		.klen	= 16,
-		.iv	= "\x01\x00\x00\x00\x0b\x0a\x09\x08"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07",
-		.alen	= 8,
-		.input	= "\x82\x53\x1a\x60\xcc\x24\x94\x5a"
-			  "\x4b\x82\x79\x18\x1a\xb5\xc8\x4d"
-			  "\xf2\x1c\xe7\xf9\xb7\x3f\x42\xe1"
-			  "\x97\xea\x9c\x07\xe5\x6b\x5e\xb1"
-			  "\x7e\x5f\x4e",
-		.ilen	= 35,
-		.result	= "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20",
-		.rlen	= 25,
-	}, {
-		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
-		.klen	= 16,
-		.iv	= "\x01\x00\x00\x00\x0c\x0b\x0a\x09"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b",
-		.alen	= 12,
-		.input	= "\x07\x34\x25\x94\x15\x77\x85\x15"
-			  "\x2b\x07\x40\x98\x33\x0a\xbb\x14"
-			  "\x1b\x94\x7b\x56\x6a\xa9\x40\x6b"
-			  "\x4d\x99\x99\x88\xdd",
-		.ilen	= 29,
-		.result	= "\x0c\x0d\x0e\x0f\x10\x11\x12\x13"
-			  "\x14\x15\x16\x17\x18\x19\x1a\x1b"
-			  "\x1c\x1d\x1e",
-		.rlen	= 19,
-	}, {
-		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
-			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
-		.klen	= 16,
-		.iv	= "\x01\x00\x33\x56\x8e\xf7\xb2\x63"
-			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
-		.assoc	= "\x63\x01\x8f\x76\xdc\x8a\x1b\xcb",
-		.alen	= 8,
-		.input	= "\x4c\xcb\x1e\x7c\xa9\x81\xbe\xfa"
-			  "\xa0\x72\x6c\x55\xd3\x78\x06\x12"
-			  "\x98\xc8\x5c\x92\x81\x4a\xbc\x33"
-			  "\xc5\x2e\xe8\x1d\x7d\x77\xc0\x8a",
-		.ilen	= 32,
-		.result	= "\x90\x20\xea\x6f\x91\xbd\xd8\x5a"
-			  "\xfa\x00\x39\xba\x4b\xaf\xf9\xbf"
-			  "\xb7\x9c\x70\x28\x94\x9c\xd0\xec",
-		.rlen	= 24,
-	}, {
-		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
-			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
-		.klen	= 16,
-		.iv	= "\x01\x00\xd5\x60\x91\x2d\x3f\x70"
-			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
-		.assoc	= "\xcd\x90\x44\xd2\xb7\x1f\xdb\x81"
-			  "\x20\xea\x60\xc0",
-		.alen	= 12,
-		.input	= "\x00\x97\x69\xec\xab\xdf\x48\x62"
-			  "\x55\x94\xc5\x92\x51\xe6\x03\x57"
-			  "\x22\x67\x5e\x04\xc8\x47\x09\x9e"
-			  "\x5a\xe0\x70\x45\x51",
-		.ilen	= 29,
-		.result	= "\x64\x35\xac\xba\xfb\x11\xa8\x2e"
-			  "\x2f\x07\x1d\x7c\xa4\xa5\xeb\xd9"
-			  "\x3a\x80\x3b\xa8\x7f",
-		.rlen	= 21,
-	}, {
-		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
-			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
-		.klen	= 16,
-		.iv	= "\x01\x00\x42\xff\xf8\xf1\x95\x1c"
-			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
-		.assoc	= "\xd8\x5b\xc7\xe6\x9f\x94\x4f\xb8",
-		.alen	= 8,
-		.input	= "\xbc\x21\x8d\xaa\x94\x74\x27\xb6"
-			  "\xdb\x38\x6a\x99\xac\x1a\xef\x23"
-			  "\xad\xe0\xb5\x29\x39\xcb\x6a\x63"
-			  "\x7c\xf9\xbe\xc2\x40\x88\x97\xc6"
-			  "\xba",
-		.ilen	= 33,
-		.result	= "\x8a\x19\xb9\x50\xbc\xf7\x1a\x01"
-			  "\x8e\x5e\x67\x01\xc9\x17\x87\x65"
-			  "\x98\x09\xd6\x7d\xbe\xdd\x18",
-		.rlen	= 23,
-	}, {
-		/* This is taken from FIPS CAVS. */
-		.key	= "\xab\x2f\x8a\x74\xb7\x1c\xd2\xb1"
-			  "\xff\x80\x2e\x48\x7d\x82\xf8\xb9",
-		.klen	= 16,
-		.iv	= "\x03\xc6\xfb\x7d\x80\x0d\x13\xab"
-			  "\xd8\xa6\xb2\xd8\x00\x00\x00\x00",
-		.alen	= 0,
-		.input	= "\xd5\xe8\x93\x9f\xc7\x89\x2e\x2b",
-		.ilen	= 8,
-		.result	= "\x00",
-		.rlen	= 0,
-		.novrfy	= 1,
-	}, {
-		.key	= "\xab\x2f\x8a\x74\xb7\x1c\xd2\xb1"
-			  "\xff\x80\x2e\x48\x7d\x82\xf8\xb9",
-		.klen	= 16,
-		.iv	= "\x03\xaf\x94\x87\x78\x35\x82\x81"
-			  "\x7f\x88\x94\x68\x00\x00\x00\x00",
-		.alen	= 0,
-		.input	= "\x41\x3c\xb8\x87\x73\xcb\xf3\xf3",
-		.ilen	= 8,
-		.result	= "\x00",
-		.rlen	= 0,
-	}, {
-		.key	= "\x61\x0e\x8c\xae\xe3\x23\xb6\x38"
-			  "\x76\x1c\xf6\x3a\x67\xa3\x9c\xd8",
-		.klen	= 16,
-		.iv	= "\x03\xc6\xfb\x7d\x80\x0d\x13\xab"
-			  "\xd8\xa6\xb2\xd8\x00\x00\x00\x00",
-		.assoc	= "\xf3\x94\x87\x78\x35\x82\x81\x7f"
-			  "\x88\x94\x68\xb1\x78\x6b\x2b\xd6"
-			  "\x04\x1f\x4e\xed\x78\xd5\x33\x66"
-			  "\xd8\x94\x99\x91\x81\x54\x62\x57",
-		.alen	= 32,
-		.input	= "\xf0\x7c\x29\x02\xae\x1c\x2f\x55"
-			  "\xd0\xd1\x3d\x1a\xa3\x6d\xe4\x0a"
-			  "\x86\xb0\x87\x6b\x62\x33\x8c\x34"
-			  "\xce\xab\x57\xcc\x79\x0b\xe0\x6f"
-			  "\x5c\x3e\x48\x1f\x6c\x46\xf7\x51"
-			  "\x8b\x84\x83\x2a\xc1\x05\xb8\xc5",
-		.ilen	= 48,
-		.result	= "\x50\x82\x3e\x07\xe2\x1e\xb6\xfb"
-			  "\x33\xe4\x73\xce\xd2\xfb\x95\x79"
-			  "\xe8\xb4\xb5\x77\x11\x10\x62\x6f"
-			  "\x6a\x82\xd1\x13\xec\xf5\xd0\x48",
-		.rlen	= 32,
-		.novrfy	= 1,
-	}, {
-		.key	= "\x61\x0e\x8c\xae\xe3\x23\xb6\x38"
-			  "\x76\x1c\xf6\x3a\x67\xa3\x9c\xd8",
-		.klen	= 16,
-		.iv	= "\x03\x05\xe0\xc9\x0f\xed\x34\xea"
-			  "\x97\xd4\x3b\xdf\x00\x00\x00\x00",
-		.assoc	= "\x49\x5c\x50\x1f\x1d\x94\xcc\x81"
-			  "\xba\xb7\xb6\x03\xaf\xa5\xc1\xa1"
-			  "\xd8\x5c\x42\x68\xe0\x6c\xda\x89"
-			  "\x05\xac\x56\xac\x1b\x2a\xd3\x86",
-		.alen	= 32,
-		.input	= "\x39\xbe\x7d\x15\x62\x77\xf3\x3c"
-			  "\xad\x83\x52\x6d\x71\x03\x25\x1c"
-			  "\xed\x81\x3a\x9a\x16\x7d\x19\x80"
-			  "\x72\x04\x72\xd0\xf6\xff\x05\x0f"
-			  "\xb7\x14\x30\x00\x32\x9e\xa0\xa6"
-			  "\x9e\x5a\x18\xa1\xb8\xfe\xdb\xd3",
-		.ilen	= 48,
-		.result	= "\x75\x05\xbe\xc2\xd9\x1e\xde\x60"
-			  "\x47\x3d\x8c\x7d\xbd\xb5\xd9\xb7"
-			  "\xf2\xae\x61\x05\x8f\x82\x24\x3f"
-			  "\x9c\x67\x91\xe1\x38\x4f\xe4\x0c",
-		.rlen	= 32,
-	}, {
-		.key	= "\x39\xbb\xa7\xbe\x59\x97\x9e\x73"
-			  "\xa2\xbc\x6b\x98\xd7\x75\x7f\xe3"
-			  "\xa4\x48\x93\x39\x26\x71\x4a\xc6",
-		.klen	= 24,
-		.iv	= "\x03\xee\x49\x83\xe9\xa9\xff\xe9"
-			  "\x57\xba\xfd\x9e\x00\x00\x00\x00",
-		.assoc	= "\x44\xa6\x2c\x05\xe9\xe1\x43\xb1"
-			  "\x58\x7c\xf2\x5c\x6d\x39\x0a\x64"
-			  "\xa4\xf0\x13\x05\xd1\x77\x99\x67"
-			  "\x11\xc4\xc6\xdb\x00\x56\x36\x61",
-		.alen	= 32,
-		.input	= "\x71\x99\xfa\xf4\x44\x12\x68\x9b",
-		.ilen	= 8,
-		.result	= "\x00",
-		.rlen	= 0,
-	}, {
-		.key	= "\x58\x5d\xa0\x96\x65\x1a\x04\xd7"
-			  "\x96\xe5\xc5\x68\xaa\x95\x35\xe0"
-			  "\x29\xa0\xba\x9e\x48\x78\xd1\xba",
-		.klen	= 24,
-		.iv	= "\x03\xee\x49\x83\xe9\xa9\xff\xe9"
-			  "\x57\xba\xfd\x9e\x00\x00\x00\x00",
-		.assoc	= "\x44\xa6\x2c\x05\xe9\xe1\x43\xb1"
-			  "\x58\x7c\xf2\x5c\x6d\x39\x0a\x64"
-			  "\xa4\xf0\x13\x05\xd1\x77\x99\x67"
-			  "\x11\xc4\xc6\xdb\x00\x56\x36\x61",
-		.alen	= 32,
-		.input	= "\xfb\xe5\x5d\x34\xbe\xe5\xe8\xe7"
-			  "\x5a\xef\x2f\xbf\x1f\x7f\xd4\xb2"
-			  "\x66\xca\x61\x1e\x96\x7a\x61\xb3"
-			  "\x1c\x16\x45\x52\xba\x04\x9c\x9f"
-			  "\xb1\xd2\x40\xbc\x52\x7c\x6f\xb1",
-		.ilen	= 40,
-		.result	= "\x85\x34\x66\x42\xc8\x92\x0f\x36"
-			  "\x58\xe0\x6b\x91\x3c\x98\x5c\xbb"
-			  "\x0a\x85\xcc\x02\xad\x7a\x96\xe9"
-			  "\x65\x43\xa4\xc3\x0f\xdc\x55\x81",
-		.rlen	= 32,
-	}, {
-		.key	= "\x58\x5d\xa0\x96\x65\x1a\x04\xd7"
-			  "\x96\xe5\xc5\x68\xaa\x95\x35\xe0"
-			  "\x29\xa0\xba\x9e\x48\x78\xd1\xba",
-		.klen	= 24,
-		.iv	= "\x03\xd1\xfc\x57\x9c\xfe\xb8\x9c"
-			  "\xad\x71\xaa\x1f\x00\x00\x00\x00",
-		.assoc	= "\x86\x67\xa5\xa9\x14\x5f\x0d\xc6"
-			  "\xff\x14\xc7\x44\xbf\x6c\x3a\xc3"
-			  "\xff\xb6\x81\xbd\xe2\xd5\x06\xc7"
-			  "\x3c\xa1\x52\x13\x03\x8a\x23\x3a",
-		.alen	= 32,
-		.input	= "\x3f\x66\xb0\x9d\xe5\x4b\x38\x00"
-			  "\xc6\x0e\x6e\xe5\xd6\x98\xa6\x37"
-			  "\x8c\x26\x33\xc6\xb2\xa2\x17\xfa"
-			  "\x64\x19\xc0\x30\xd7\xfc\x14\x6b"
-			  "\xe3\x33\xc2\x04\xb0\x37\xbe\x3f"
-			  "\xa9\xb4\x2d\x68\x03\xa3\x44\xef",
-		.ilen	= 48,
-		.result	= "\x02\x87\x4d\x28\x80\x6e\xb2\xed"
-			  "\x99\x2a\xa8\xca\x04\x25\x45\x90"
-			  "\x1d\xdd\x5a\xd9\xe4\xdb\x9c\x9c"
-			  "\x49\xe9\x01\xfe\xa7\x80\x6d\x6b",
-		.rlen	= 32,
-		.novrfy	= 1,
-	}, {
-		.key	= "\xa4\x4b\x54\x29\x0a\xb8\x6d\x01"
-			  "\x5b\x80\x2a\xcf\x25\xc4\xb7\x5c"
-			  "\x20\x2c\xad\x30\xc2\x2b\x41\xfb"
-			  "\x0e\x85\xbc\x33\xad\x0f\x2b\xff",
-		.klen	= 32,
-		.iv	= "\x03\xee\x49\x83\xe9\xa9\xff\xe9"
-			  "\x57\xba\xfd\x9e\x00\x00\x00\x00",
-		.alen	= 0,
-		.input	= "\x1f\xb8\x8f\xa3\xdd\x54\x00\xf2",
-		.ilen	= 8,
-		.result	= "\x00",
-		.rlen	= 0,
-	}, {
-		.key	= "\x39\xbb\xa7\xbe\x59\x97\x9e\x73"
-			  "\xa2\xbc\x6b\x98\xd7\x75\x7f\xe3"
-			  "\xa4\x48\x93\x39\x26\x71\x4a\xc6"
-			  "\xae\x8f\x11\x4c\xc2\x9c\x4a\xbb",
-		.klen	= 32,
-		.iv	= "\x03\x85\x34\x66\x42\xc8\x92\x0f"
-			  "\x36\x58\xe0\x6b\x00\x00\x00\x00",
-		.alen	= 0,
-		.input	= "\x48\x01\x5e\x02\x24\x04\x66\x47"
-			  "\xa1\xea\x6f\xaf\xe8\xfc\xfb\xdd"
-			  "\xa5\xa9\x87\x8d\x84\xee\x2e\x77"
-			  "\xbb\x86\xb9\xf5\x5c\x6c\xff\xf6"
-			  "\x72\xc3\x8e\xf7\x70\xb1\xb2\x07"
-			  "\xbc\xa8\xa3\xbd\x83\x7c\x1d\x2a",
-		.ilen	= 48,
-		.result	= "\xdc\x56\xf2\x71\xb0\xb1\xa0\x6c"
-			  "\xf0\x97\x3a\xfb\x6d\xe7\x32\x99"
-			  "\x3e\xaf\x70\x5e\xb2\x4d\xea\x39"
-			  "\x89\xd4\x75\x7a\x63\xb1\xda\x93",
-		.rlen	= 32,
-		.novrfy	= 1,
-	}, {
-		.key	= "\x58\x5d\xa0\x96\x65\x1a\x04\xd7"
-			  "\x96\xe5\xc5\x68\xaa\x95\x35\xe0"
-			  "\x29\xa0\xba\x9e\x48\x78\xd1\xba"
-			  "\x0d\x1a\x53\x3b\xb5\xe3\xf8\x8b",
-		.klen	= 32,
-		.iv	= "\x03\xcf\x76\x3f\xd9\x95\x75\x8f"
-			  "\x44\x89\x40\x7b\x00\x00\x00\x00",
-		.assoc	= "\x8f\x86\x6c\x4d\x1d\xc5\x39\x88"
-			  "\xc8\xf3\x5c\x52\x10\x63\x6f\x2b"
-			  "\x8a\x2a\xc5\x6f\x30\x23\x58\x7b"
-			  "\xfb\x36\x03\x11\xb4\xd9\xf2\xfe",
-		.alen	= 32,
-		.input	= "\x48\x58\xd6\xf3\xad\x63\x58\xbf"
-			  "\xae\xc7\x5e\xae\x83\x8f\x7b\xe4"
-			  "\x78\x5c\x4c\x67\x71\x89\x94\xbf"
-			  "\x47\xf1\x63\x7e\x1c\x59\xbd\xc5"
-			  "\x7f\x44\x0a\x0c\x01\x18\x07\x92"
-			  "\xe1\xd3\x51\xce\x32\x6d\x0c\x5b",
-		.ilen	= 48,
-		.result	= "\xc2\x54\xc8\xde\x78\x87\x77\x40"
-			  "\x49\x71\xe4\xb7\xe7\xcb\x76\x61"
-			  "\x0a\x41\xb9\xe9\xc0\x76\x54\xab"
-			  "\x04\x49\x3b\x19\x93\x57\x25\x5d",
-		.rlen	= 32,
-	},
-};
-
-/*
- * rfc4309 refers to section 8 of rfc3610 for test vectors, but they all
- * use a 13-byte nonce, we only support an 11-byte nonce.  Worse,
- * they use AD lengths which are not valid ESP header lengths.
- *
- * These vectors are copied/generated from the ones for rfc4106 with
- * the key truncated by one byte..
- */
-static const struct aead_testvec aes_ccm_rfc4309_enc_tv_template[] = {
-	{ /* Generated using Crypto++ */
-		.key	= zeroed_string,
-		.klen	= 19,
-		.iv	= zeroed_string,
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.assoc	= zeroed_string,
-		.alen	= 16,
-		.result	= "\x2E\x9A\xCA\x6B\xDA\x54\xFC\x6F"
-			  "\x12\x50\xE8\xDE\x81\x3C\x63\x08"
-			  "\x1A\x22\xBA\x75\xEE\xD4\xD5\xB5"
-			  "\x27\x50\x01\xAC\x03\x33\x39\xFB",
-		.rlen	= 32,
-	},{
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen	= 16,
-		.result	= "\xCF\xB9\x99\x17\xC8\x86\x0E\x7F"
-			  "\x7E\x76\xF8\xE6\xF8\xCC\x1F\x17"
-			  "\x6A\xE0\x53\x9F\x4B\x73\x7E\xDA"
-			  "\x08\x09\x4E\xC4\x1E\xAD\xC6\xB0",
-		.rlen	= 32,
-
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= zeroed_string,
-		.input	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.ilen	= 16,
-		.assoc	= zeroed_string,
-		.alen	= 16,
-		.result	= "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6"
-			  "\x61\xF4\xF5\x41\x03\x4A\xE3\x86"
-			  "\xA1\xE2\xC2\x42\x2B\x81\x70\x40"
-			  "\xFD\x7F\x76\xD1\x03\x07\xBB\x0C",
-		.rlen	= 32,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= zeroed_string,
-		.input	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.ilen	= 16,
-		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.alen	= 16,
-		.result	= "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6"
-			  "\x61\xF4\xF5\x41\x03\x4A\xE3\x86"
-			  "\x5B\xC0\x73\xE0\x2B\x73\x68\xC9"
-			  "\x2D\x8C\x58\xC2\x90\x3D\xB0\x3E",
-		.rlen	= 32,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.ilen	= 16,
-		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen	= 16,
-		.result	= "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E"
-			  "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16"
-			  "\x43\x8E\x76\x57\x3B\xB4\x05\xE8"
-			  "\xA9\x9B\xBF\x25\xE0\x4F\xC0\xED",
-		.rlen	= 32,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.ilen	= 64,
-		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen	= 16,
-		.result	= "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E"
-			  "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16"
-			  "\x9C\xA4\x97\x83\x3F\x01\xA5\xF4"
-			  "\x43\x09\xE7\xB8\xE9\xD1\xD7\x02"
-			  "\x9B\xAB\x39\x18\xEB\x94\x34\x36"
-			  "\xE6\xC5\xC8\x9B\x00\x81\x9E\x49"
-			  "\x1D\x78\xE1\x48\xE3\xE9\xEA\x8E"
-			  "\x3A\x2B\x67\x5D\x35\x6A\x0F\xDB"
-			  "\x02\x73\xDD\xE7\x30\x4A\x30\x54"
-			  "\x1A\x9D\x09\xCA\xC8\x1C\x32\x5F",
-		.rlen	= 80,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= "\x00\x00\x45\x67\x89\xab\xcd\xef",
-		.input	= "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff",
-		.ilen	= 192,
-		.assoc	= "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
-			  "\xaa\xaa\xaa\xaa\x00\x00\x45\x67"
-			  "\x89\xab\xcd\xef",
-		.alen	= 20,
-		.result	= "\x64\x17\xDC\x24\x9D\x92\xBA\x5E"
-			  "\x7C\x64\x6D\x33\x46\x77\xAC\xB1"
-			  "\x5C\x9E\xE2\xC7\x27\x11\x3E\x95"
-			  "\x7D\xBE\x28\xC8\xC1\xCA\x5E\x8C"
-			  "\xB4\xE2\xDE\x9F\x53\x59\x26\xDB"
-			  "\x0C\xD4\xE4\x07\x9A\xE6\x3E\x01"
-			  "\x58\x0D\x3E\x3D\xD5\x21\xEB\x04"
-			  "\x06\x9D\x5F\xB9\x02\x49\x1A\x2B"
-			  "\xBA\xF0\x4E\x3B\x85\x50\x5B\x09"
-			  "\xFE\xEC\xFC\x54\xEC\x0C\xE2\x79"
-			  "\x8A\x2F\x5F\xD7\x05\x5D\xF1\x6D"
-			  "\x22\xEB\xD1\x09\x80\x3F\x5A\x70"
-			  "\xB2\xB9\xD3\x63\x99\xC2\x4D\x1B"
-			  "\x36\x12\x00\x89\xAA\x5D\x55\xDA"
-			  "\x1D\x5B\xD8\x3C\x5F\x09\xD2\xE6"
-			  "\x39\x41\x5C\xF0\xBE\x26\x4E\x5F"
-			  "\x2B\x50\x44\x52\xC2\x10\x7D\x38"
-			  "\x82\x64\x83\x0C\xAE\x49\xD0\xE5"
-			  "\x4F\xE5\x66\x4C\x58\x7A\xEE\x43"
-			  "\x3B\x51\xFE\xBA\x24\x8A\xFE\xDC"
-			  "\x19\x6D\x60\x66\x61\xF9\x9A\x3F"
-			  "\x75\xFC\x38\x53\x5B\xB5\xCD\x52"
-			  "\x4F\xE5\xE4\xC9\xFE\x10\xCB\x98"
-			  "\xF0\x06\x5B\x07\xAB\xBB\xF4\x0E"
-			  "\x2D\xC2\xDD\x5D\xDD\x22\x9A\xCC"
-			  "\x39\xAB\x63\xA5\x3D\x9C\x51\x8A",
-		.rlen	= 208,
-	}, { /* From draft-mcgrew-gcm-test-01 */
-		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
-			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
-			  "\x2E\x44\x3B",
-		.klen	= 19,
-		.iv	= "\x49\x56\xED\x7E\x3B\x24\x4C\xFE",
-		.input	= "\x45\x00\x00\x48\x69\x9A\x00\x00"
-			  "\x80\x11\x4D\xB7\xC0\xA8\x01\x02"
-			  "\xC0\xA8\x01\x01\x0A\x9B\xF1\x56"
-			  "\x38\xD3\x01\x00\x00\x01\x00\x00"
-			  "\x00\x00\x00\x00\x04\x5F\x73\x69"
-			  "\x70\x04\x5F\x75\x64\x70\x03\x73"
-			  "\x69\x70\x09\x63\x79\x62\x65\x72"
-			  "\x63\x69\x74\x79\x02\x64\x6B\x00"
-			  "\x00\x21\x00\x01\x01\x02\x02\x01",
-		.ilen	= 72,
-		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
-			  "\x00\x00\x00\x00\x49\x56\xED\x7E"
-			  "\x3B\x24\x4C\xFE",
-		.alen	= 20,
-		.result	= "\x89\xBA\x3E\xEF\xE6\xD6\xCF\xDB"
-			  "\x83\x60\xF5\xBA\x3A\x56\x79\xE6"
-			  "\x7E\x0C\x53\xCF\x9E\x87\xE0\x4E"
-			  "\x1A\x26\x01\x24\xC7\x2E\x3D\xBF"
-			  "\x29\x2C\x91\xC1\xB8\xA8\xCF\xE0"
-			  "\x39\xF8\x53\x6D\x31\x22\x2B\xBF"
-			  "\x98\x81\xFC\x34\xEE\x85\x36\xCD"
-			  "\x26\xDB\x6C\x7A\x0C\x77\x8A\x35"
-			  "\x18\x85\x54\xB2\xBC\xDD\x3F\x43"
-			  "\x61\x06\x8A\xDF\x86\x3F\xB4\xAC"
-			  "\x97\xDC\xBD\xFD\x92\x10\xC5\xFF",
-		.rlen	= 88,
-	}, {
-		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
-			  "\xCA\xFE\xBA",
-		.klen	= 19,
-		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.input	= "\x45\x00\x00\x3E\x69\x8F\x00\x00"
-			  "\x80\x11\x4D\xCC\xC0\xA8\x01\x02"
-			  "\xC0\xA8\x01\x01\x0A\x98\x00\x35"
-			  "\x00\x2A\x23\x43\xB2\xD0\x01\x00"
-			  "\x00\x01\x00\x00\x00\x00\x00\x00"
-			  "\x03\x73\x69\x70\x09\x63\x79\x62"
-			  "\x65\x72\x63\x69\x74\x79\x02\x64"
-			  "\x6B\x00\x00\x01\x00\x01\x00\x01",
-		.ilen	= 64,
-		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
-			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.alen	= 16,
-		.result	= "\x4B\xC2\x70\x60\x64\xD2\xF3\xC8"
-			  "\xE5\x26\x8A\xDE\xB8\x7E\x7D\x16"
-			  "\x56\xC7\xD2\x88\xBA\x8D\x58\xAF"
-			  "\xF5\x71\xB6\x37\x84\xA7\xB1\x99"
-			  "\x51\x5C\x0D\xA0\x27\xDE\xE7\x2D"
-			  "\xEF\x25\x88\x1F\x1D\x77\x11\xFF"
-			  "\xDB\xED\xEE\x56\x16\xC5\x5C\x9B"
-			  "\x00\x62\x1F\x68\x4E\x7C\xA0\x97"
-			  "\x10\x72\x7E\x53\x13\x3B\x68\xE4"
-			  "\x30\x99\x91\x79\x09\xEA\xFF\x6A",
-		.rlen	= 80,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x11\x22\x33",
-		.klen	= 35,
-		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
-		.input	= "\x45\x00\x00\x30\x69\xA6\x40\x00"
-			  "\x80\x06\x26\x90\xC0\xA8\x01\x02"
-			  "\x93\x89\x15\x5E\x0A\x9E\x00\x8B"
-			  "\x2D\xC5\x7E\xE0\x00\x00\x00\x00"
-			  "\x70\x02\x40\x00\x20\xBF\x00\x00"
-			  "\x02\x04\x05\xB4\x01\x01\x04\x02"
-			  "\x01\x02\x02\x01",
-		.ilen	= 52,
-		.assoc	= "\x4A\x2C\xBF\xE3\x00\x00\x00\x02"
-			  "\x01\x02\x03\x04\x05\x06\x07\x08",
-		.alen	= 16,
-		.result	= "\xD6\x31\x0D\x2B\x3D\x6F\xBD\x2F"
-			  "\x58\x41\x7E\xFF\x9A\x9E\x09\xB4"
-			  "\x1A\xF7\xF6\x42\x31\xCD\xBF\xAD"
-			  "\x27\x0E\x2C\xF2\xDB\x10\xDF\x55"
-			  "\x8F\x0D\xD7\xAC\x23\xBD\x42\x10"
-			  "\xD0\xB2\xAF\xD8\x37\xAC\x6B\x0B"
-			  "\x11\xD4\x0B\x12\xEC\xB4\xB1\x92"
-			  "\x23\xA6\x10\xB0\x26\xD6\xD9\x26"
-			  "\x5A\x48\x6A\x3E",
-		.rlen	= 68,
-	}, {
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x45\x00\x00\x3C\x99\xC5\x00\x00"
-			  "\x80\x01\xCB\x7A\x40\x67\x93\x18"
-			  "\x01\x01\x01\x01\x08\x00\x07\x5C"
-			  "\x02\x00\x44\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x75\x76\x77\x61\x62\x63\x64\x65"
-			  "\x66\x67\x68\x69\x01\x02\x02\x01",
-		.ilen	= 64,
-		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.alen	= 16,
-		.result	= "\x6B\x9A\xCA\x57\x43\x91\xFC\x6F"
-			  "\x92\x51\x23\xA4\xC1\x5B\xF0\x10"
-			  "\xF3\x13\xF4\xF8\xA1\x9A\xB4\xDC"
-			  "\x89\xC8\xF8\x42\x62\x95\xB7\xCB"
-			  "\xB8\xF5\x0F\x1B\x2E\x94\xA2\xA7"
-			  "\xBF\xFB\x8A\x92\x13\x63\xD1\x3C"
-			  "\x08\xF5\xE8\xA6\xAA\xF6\x34\xF9"
-			  "\x42\x05\xAF\xB3\xE7\x9A\xFC\xEE"
-			  "\x36\x25\xC1\x10\x12\x1C\xCA\x82"
-			  "\xEA\xE6\x63\x5A\x57\x28\xA9\x9A",
-		.rlen	= 80,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E",
-		.klen	= 19,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.input	= "\x45\x00\x00\x3C\x99\xC3\x00\x00"
-			  "\x80\x01\xCB\x7C\x40\x67\x93\x18"
-			  "\x01\x01\x01\x01\x08\x00\x08\x5C"
-			  "\x02\x00\x43\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x75\x76\x77\x61\x62\x63\x64\x65"
-			  "\x66\x67\x68\x69\x01\x02\x02\x01",
-		.ilen	= 64,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.result	= "\x6A\x6B\x45\x2B\x7C\x67\x52\xF6"
-			  "\x10\x60\x40\x62\x6B\x4F\x97\x8E"
-			  "\x0B\xB2\x22\x97\xCB\x21\xE0\x90"
-			  "\xA2\xE7\xD1\x41\x30\xE4\x4B\x1B"
-			  "\x79\x01\x58\x50\x01\x06\xE1\xE0"
-			  "\x2C\x83\x79\xD3\xDE\x46\x97\x1A"
-			  "\x30\xB8\xE5\xDF\xD7\x12\x56\x75"
-			  "\xD0\x95\xB7\xB8\x91\x42\xF7\xFD"
-			  "\x97\x57\xCA\xC1\x20\xD0\x86\xB9"
-			  "\x66\x9D\xB4\x2B\x96\x22\xAC\x67",
-		.rlen	= 80,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E",
-		.klen	= 19,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.input	= "\x45\x00\x00\x1C\x42\xA2\x00\x00"
-			  "\x80\x01\x44\x1F\x40\x67\x93\xB6"
-			  "\xE0\x00\x00\x02\x0A\x00\xF5\xFF"
-			  "\x01\x02\x02\x01",
-		.ilen	= 28,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.result	= "\x6A\x6B\x45\x0B\xA7\x06\x52\xF6"
-			  "\x10\x60\xCF\x01\x6B\x4F\x97\x20"
-			  "\xEA\xB3\x23\x94\xC9\x21\x1D\x33"
-			  "\xA1\xE5\x90\x40\x05\x37\x45\x70"
-			  "\xB5\xD6\x09\x0A\x23\x73\x33\xF9"
-			  "\x08\xB4\x22\xE4",
-		.rlen	= 44,
-	}, {
-		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
-			  "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\xCA\xFE\xBA",
-		.klen	= 27,
-		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.input	= "\x45\x00\x00\x28\xA4\xAD\x40\x00"
-			  "\x40\x06\x78\x80\x0A\x01\x03\x8F"
-			  "\x0A\x01\x06\x12\x80\x23\x06\xB8"
-			  "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E"
-			  "\x50\x10\x16\xD0\x75\x68\x00\x01",
-		.ilen	= 40,
-		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
-			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.alen	= 16,
-		.result	= "\x05\x22\x15\xD1\x52\x56\x85\x04"
-			  "\xA8\x5C\x5D\x6D\x7E\x6E\xF5\xFA"
-			  "\xEA\x16\x37\x50\xF3\xDF\x84\x3B"
-			  "\x2F\x32\x18\x57\x34\x2A\x8C\x23"
-			  "\x67\xDF\x6D\x35\x7B\x54\x0D\xFB"
-			  "\x34\xA5\x9F\x6C\x48\x30\x1E\x22"
-			  "\xFE\xB1\x22\x17\x17\x8A\xB9\x5B",
-		.rlen	= 56,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xDE\xCA\xF8",
-		.klen	= 19,
-		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
-		.input	= "\x45\x00\x00\x49\x33\xBA\x00\x00"
-			  "\x7F\x11\x91\x06\xC3\xFB\x1D\x10"
-			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
-			  "\x00\x35\xDD\x7B\x80\x03\x02\xD5"
-			  "\x00\x00\x4E\x20\x00\x1E\x8C\x18"
-			  "\xD7\x5B\x81\xDC\x91\xBA\xA0\x47"
-			  "\x6B\x91\xB9\x24\xB2\x80\x38\x9D"
-			  "\x92\xC9\x63\xBA\xC0\x46\xEC\x95"
-			  "\x9B\x62\x66\xC0\x47\x22\xB1\x49"
-			  "\x23\x01\x01\x01",
-		.ilen	= 76,
-		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
-			  "\xCE\xFA\xCE\x74",
-		.alen	= 20,
-		.result	= "\x92\xD0\x53\x79\x33\x38\xD5\xF3"
-			  "\x7D\xE4\x7A\x8E\x86\x03\xC9\x90"
-			  "\x96\x35\xAB\x9C\xFB\xE8\xA3\x76"
-			  "\xE9\xE9\xE2\xD1\x2E\x11\x0E\x00"
-			  "\xFA\xCE\xB5\x9E\x02\xA7\x7B\xEA"
-			  "\x71\x9A\x58\xFB\xA5\x8A\xE1\xB7"
-			  "\x9C\x39\x9D\xE3\xB5\x6E\x69\xE6"
-			  "\x63\xC9\xDB\x05\x69\x51\x12\xAD"
-			  "\x3E\x00\x32\x73\x86\xF2\xEE\xF5"
-			  "\x0F\xE8\x81\x7E\x84\xD3\xC0\x0D"
-			  "\x76\xD6\x55\xC6\xB4\xC2\x34\xC7"
-			  "\x12\x25\x0B\xF9",
-		.rlen	= 92,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x73\x61\x6C",
-		.klen	= 35,
-		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
-		.input	= "\x45\x08\x00\x28\x73\x2C\x00\x00"
-			  "\x40\x06\xE9\xF9\x0A\x01\x06\x12"
-			  "\x0A\x01\x03\x8F\x06\xB8\x80\x23"
-			  "\xDD\x6B\xAF\xBE\xCB\x71\x26\x02"
-			  "\x50\x10\x1F\x64\x6D\x54\x00\x01",
-		.ilen	= 40,
-		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
-			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
-			  "\x69\x76\x65\x63",
-		.alen	= 20,
-		.result	= "\xCC\x74\xB7\xD3\xB0\x38\x50\x42"
-			  "\x2C\x64\x87\x46\x1E\x34\x10\x05"
-			  "\x29\x6B\xBB\x36\xE9\x69\xAD\x92"
-			  "\x82\xA1\x10\x6A\xEB\x0F\xDC\x7D"
-			  "\x08\xBA\xF3\x91\xCA\xAA\x61\xDA"
-			  "\x62\xF4\x14\x61\x5C\x9D\xB5\xA7"
-			  "\xEE\xD7\xB9\x7E\x87\x99\x9B\x7D",
-		.rlen	= 56,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E",
-		.klen	= 19,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.input	= "\x45\x00\x00\x49\x33\x3E\x00\x00"
-			  "\x7F\x11\x91\x82\xC3\xFB\x1D\x10"
-			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
-			  "\x00\x35\xCB\x45\x80\x03\x02\x5B"
-			  "\x00\x00\x01\xE0\x00\x1E\x8C\x18"
-			  "\xD6\x57\x59\xD5\x22\x84\xA0\x35"
-			  "\x2C\x71\x47\x5C\x88\x80\x39\x1C"
-			  "\x76\x4D\x6E\x5E\xE0\x49\x6B\x32"
-			  "\x5A\xE2\x70\xC0\x38\x99\x49\x39"
-			  "\x15\x01\x01\x01",
-		.ilen	= 76,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.result	= "\x6A\x6B\x45\x5E\xD6\x9A\x52\xF6"
-			  "\xEF\x70\x1A\x9C\xE8\xD3\x19\x86"
-			  "\xC8\x02\xF0\xB0\x03\x09\xD9\x02"
-			  "\xA0\xD2\x59\x04\xD1\x85\x2A\x24"
-			  "\x1C\x67\x3E\xD8\x68\x72\x06\x94"
-			  "\x97\xBA\x4F\x76\x8D\xB0\x44\x5B"
-			  "\x69\xBF\xD5\xE2\x3D\xF1\x0B\x0C"
-			  "\xC0\xBF\xB1\x8F\x70\x09\x9E\xCE"
-			  "\xA5\xF2\x55\x58\x84\xFA\xF9\xB5"
-			  "\x23\xF4\x84\x40\x74\x14\x8A\x6B"
-			  "\xDB\xD7\x67\xED\xA4\x93\xF3\x47"
-			  "\xCC\xF7\x46\x6F",
-		.rlen	= 92,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x73\x61\x6C",
-		.klen	= 35,
-		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
-		.input	= "\x63\x69\x73\x63\x6F\x01\x72\x75"
-			  "\x6C\x65\x73\x01\x74\x68\x65\x01"
-			  "\x6E\x65\x74\x77\x65\x01\x64\x65"
-			  "\x66\x69\x6E\x65\x01\x74\x68\x65"
-			  "\x74\x65\x63\x68\x6E\x6F\x6C\x6F"
-			  "\x67\x69\x65\x73\x01\x74\x68\x61"
-			  "\x74\x77\x69\x6C\x6C\x01\x64\x65"
-			  "\x66\x69\x6E\x65\x74\x6F\x6D\x6F"
-			  "\x72\x72\x6F\x77\x01\x02\x02\x01",
-		.ilen	= 72,
-		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
-			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
-			  "\x69\x76\x65\x63",
-		.alen	= 20,
-		.result	= "\xEA\x15\xC4\x98\xAC\x15\x22\x37"
-			  "\x00\x07\x1D\xBE\x60\x5D\x73\x16"
-			  "\x4D\x0F\xCC\xCE\x8A\xD0\x49\xD4"
-			  "\x39\xA3\xD1\xB1\x21\x0A\x92\x1A"
-			  "\x2C\xCF\x8F\x9D\xC9\x91\x0D\xB4"
-			  "\x15\xFC\xBC\xA5\xC5\xBF\x54\xE5"
-			  "\x1C\xC7\x32\x41\x07\x7B\x2C\xB6"
-			  "\x5C\x23\x7C\x93\xEA\xEF\x23\x1C"
-			  "\x73\xF4\xE7\x12\x84\x4C\x37\x0A"
-			  "\x4A\x8F\x06\x37\x48\xF9\xF9\x05"
-			  "\x55\x13\x40\xC3\xD5\x55\x3A\x3D",
-		.rlen	= 88,
-	}, {
-		.key	= "\x7D\x77\x3D\x00\xC1\x44\xC5\x25"
-			  "\xAC\x61\x9D\x18\xC8\x4A\x3F\x47"
-			  "\xD9\x66\x42",
-		.klen	= 19,
-		.iv	= "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
-		.input	= "\x01\x02\x02\x01",
-		.ilen	= 4,
-		.assoc	= "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF"
-			  "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
-		.alen	= 16,
-		.result	= "\x4C\x72\x63\x30\x2F\xE6\x56\xDD"
-			  "\xD0\xD8\x60\x9D\x8B\xEF\x85\x90"
-			  "\xF7\x61\x24\x62",
-		.rlen	= 20,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xDE\xCA\xF8",
-		.klen	= 19,
-		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
-		.input	= "\x74\x6F\x01\x62\x65\x01\x6F\x72"
-			  "\x01\x6E\x6F\x74\x01\x74\x6F\x01"
-			  "\x62\x65\x00\x01",
-		.ilen	= 20,
-		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
-			  "\xCE\xFA\xCE\x74",
-		.alen	= 20,
-		.result	= "\xA3\xBF\x52\x52\x65\x83\xBA\x81"
-			  "\x03\x9B\x84\xFC\x44\x8C\xBB\x81"
-			  "\x36\xE1\x78\xBB\xA5\x49\x3A\xD0"
-			  "\xF0\x6B\x21\xAF\x98\xC0\x34\xDC"
-			  "\x17\x17\x65\xAD",
-		.rlen	= 36,
-	}, {
-		.key	= "\x6C\x65\x67\x61\x6C\x69\x7A\x65"
-			  "\x6D\x61\x72\x69\x6A\x75\x61\x6E"
-			  "\x61\x61\x6E\x64\x64\x6F\x69\x74"
-			  "\x62\x65\x66\x6F\x72\x65\x69\x61"
-			  "\x74\x75\x72",
-		.klen	= 35,
-		.iv	= "\x33\x30\x21\x69\x67\x65\x74\x6D",
-		.input	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
-			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
-			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01",
-		.ilen	= 52,
-		.assoc	= "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\x33\x30\x21\x69"
-			  "\x67\x65\x74\x6D",
-		.alen	= 20,
-		.result	= "\x96\xFD\x86\xF8\xD1\x98\xFF\x10"
-			  "\xAB\x8C\xDA\x8A\x5A\x08\x38\x1A"
-			  "\x48\x59\x80\x18\x1A\x18\x1A\x04"
-			  "\xC9\x0D\xE3\xE7\x0E\xA4\x0B\x75"
-			  "\x92\x9C\x52\x5C\x0B\xFB\xF8\xAF"
-			  "\x16\xC3\x35\xA8\xE7\xCE\x84\x04"
-			  "\xEB\x40\x6B\x7A\x8E\x75\xBB\x42"
-			  "\xE0\x63\x4B\x21\x44\xA2\x2B\x2B"
-			  "\x39\xDB\xC8\xDC",
-		.rlen	= 68,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E",
-		.klen	= 19,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.input	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
-			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
-			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01",
-		.ilen	= 52,
-		.assoc	= "\x3F\x7E\xF6\x42\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.result	= "\x6A\x6B\x45\x27\x3F\x9E\x52\xF6"
-			  "\x10\x60\x54\x25\xEB\x80\x04\x93"
-			  "\xCA\x1B\x23\x97\xCB\x21\x2E\x01"
-			  "\xA2\xE7\x95\x41\x30\xE4\x4B\x1B"
-			  "\x79\x01\x58\x50\x01\x06\xE1\xE0"
-			  "\x2C\x83\x79\xD3\xDE\x46\x97\x1A"
-			  "\x44\xCC\x90\xBF\x00\x94\x94\x92"
-			  "\x20\x17\x0C\x1B\x55\xDE\x7E\x68"
-			  "\xF4\x95\x5D\x4F",
-		.rlen	= 68,
-	}, {
-		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
-			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
-			  "\x22\x43\x3C",
-		.klen	= 19,
-		.iv	= "\x48\x55\xEC\x7D\x3A\x23\x4B\xFD",
-		.input	= "\x08\x00\xC6\xCD\x02\x00\x07\x00"
-			  "\x61\x62\x63\x64\x65\x66\x67\x68"
-			  "\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70"
-			  "\x71\x72\x73\x74\x01\x02\x02\x01",
-		.ilen	= 32,
-		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
-			  "\x00\x00\x00\x07\x48\x55\xEC\x7D"
-			  "\x3A\x23\x4B\xFD",
-		.alen	= 20,
-		.result	= "\x67\xE9\x28\xB3\x1C\xA4\x6D\x02"
-			  "\xF0\xB5\x37\xB6\x6B\x2F\xF5\x4F"
-			  "\xF8\xA3\x4C\x53\xB8\x12\x09\xBF"
-			  "\x58\x7D\xCF\x29\xA3\x41\x68\x6B"
-			  "\xCE\xE8\x79\x85\x3C\xB0\x3A\x8F"
-			  "\x16\xB0\xA1\x26\xC9\xBC\xBC\xA6",
-		.rlen	= 48,
-	}
-};
-
-static const struct aead_testvec aes_ccm_rfc4309_dec_tv_template[]	= {
-	{ /* Generated using Crypto++ */
-		.key	= zeroed_string,
-		.klen	= 19,
-		.iv	= zeroed_string,
-		.result	= zeroed_string,
-		.rlen	= 16,
-		.assoc	= zeroed_string,
-		.alen	= 16,
-		.input	= "\x2E\x9A\xCA\x6B\xDA\x54\xFC\x6F"
-			  "\x12\x50\xE8\xDE\x81\x3C\x63\x08"
-			  "\x1A\x22\xBA\x75\xEE\xD4\xD5\xB5"
-			  "\x27\x50\x01\xAC\x03\x33\x39\xFB",
-		.ilen	= 32,
-	},{
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.result	= zeroed_string,
-		.rlen	= 16,
-		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen	= 16,
-		.input	= "\xCF\xB9\x99\x17\xC8\x86\x0E\x7F"
-			  "\x7E\x76\xF8\xE6\xF8\xCC\x1F\x17"
-			  "\x6A\xE0\x53\x9F\x4B\x73\x7E\xDA"
-			  "\x08\x09\x4E\xC4\x1E\xAD\xC6\xB0",
-		.ilen	= 32,
-
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= zeroed_string,
-		.result	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.rlen	= 16,
-		.assoc	= zeroed_string,
-		.alen	= 16,
-		.input	= "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6"
-			  "\x61\xF4\xF5\x41\x03\x4A\xE3\x86"
-			  "\xA1\xE2\xC2\x42\x2B\x81\x70\x40"
-			  "\xFD\x7F\x76\xD1\x03\x07\xBB\x0C",
-		.ilen	= 32,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= zeroed_string,
-		.result	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.rlen	= 16,
-		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.alen	= 16,
-		.input	= "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6"
-			  "\x61\xF4\xF5\x41\x03\x4A\xE3\x86"
-			  "\x5B\xC0\x73\xE0\x2B\x73\x68\xC9"
-			  "\x2D\x8C\x58\xC2\x90\x3D\xB0\x3E",
-		.ilen	= 32,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.result	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.rlen	= 16,
-		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen	= 16,
-		.input	= "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E"
-			  "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16"
-			  "\x43\x8E\x76\x57\x3B\xB4\x05\xE8"
-			  "\xA9\x9B\xBF\x25\xE0\x4F\xC0\xED",
-		.ilen	= 32,
-	}, {
-		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
-			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.result	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.rlen	= 64,
-		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.alen	= 16,
-		.input	= "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E"
-			  "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16"
-			  "\x9C\xA4\x97\x83\x3F\x01\xA5\xF4"
-			  "\x43\x09\xE7\xB8\xE9\xD1\xD7\x02"
-			  "\x9B\xAB\x39\x18\xEB\x94\x34\x36"
-			  "\xE6\xC5\xC8\x9B\x00\x81\x9E\x49"
-			  "\x1D\x78\xE1\x48\xE3\xE9\xEA\x8E"
-			  "\x3A\x2B\x67\x5D\x35\x6A\x0F\xDB"
-			  "\x02\x73\xDD\xE7\x30\x4A\x30\x54"
-			  "\x1A\x9D\x09\xCA\xC8\x1C\x32\x5F",
-		.ilen	= 80,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= "\x00\x00\x45\x67\x89\xab\xcd\xef",
-		.result	= "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff"
-			  "\xff\xff\xff\xff\xff\xff\xff\xff",
-		.rlen	= 192,
-		.assoc	= "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
-			  "\xaa\xaa\xaa\xaa\x00\x00\x45\x67"
-			  "\x89\xab\xcd\xef",
-		.alen	= 20,
-		.input	= "\x64\x17\xDC\x24\x9D\x92\xBA\x5E"
-			  "\x7C\x64\x6D\x33\x46\x77\xAC\xB1"
-			  "\x5C\x9E\xE2\xC7\x27\x11\x3E\x95"
-			  "\x7D\xBE\x28\xC8\xC1\xCA\x5E\x8C"
-			  "\xB4\xE2\xDE\x9F\x53\x59\x26\xDB"
-			  "\x0C\xD4\xE4\x07\x9A\xE6\x3E\x01"
-			  "\x58\x0D\x3E\x3D\xD5\x21\xEB\x04"
-			  "\x06\x9D\x5F\xB9\x02\x49\x1A\x2B"
-			  "\xBA\xF0\x4E\x3B\x85\x50\x5B\x09"
-			  "\xFE\xEC\xFC\x54\xEC\x0C\xE2\x79"
-			  "\x8A\x2F\x5F\xD7\x05\x5D\xF1\x6D"
-			  "\x22\xEB\xD1\x09\x80\x3F\x5A\x70"
-			  "\xB2\xB9\xD3\x63\x99\xC2\x4D\x1B"
-			  "\x36\x12\x00\x89\xAA\x5D\x55\xDA"
-			  "\x1D\x5B\xD8\x3C\x5F\x09\xD2\xE6"
-			  "\x39\x41\x5C\xF0\xBE\x26\x4E\x5F"
-			  "\x2B\x50\x44\x52\xC2\x10\x7D\x38"
-			  "\x82\x64\x83\x0C\xAE\x49\xD0\xE5"
-			  "\x4F\xE5\x66\x4C\x58\x7A\xEE\x43"
-			  "\x3B\x51\xFE\xBA\x24\x8A\xFE\xDC"
-			  "\x19\x6D\x60\x66\x61\xF9\x9A\x3F"
-			  "\x75\xFC\x38\x53\x5B\xB5\xCD\x52"
-			  "\x4F\xE5\xE4\xC9\xFE\x10\xCB\x98"
-			  "\xF0\x06\x5B\x07\xAB\xBB\xF4\x0E"
-			  "\x2D\xC2\xDD\x5D\xDD\x22\x9A\xCC"
-			  "\x39\xAB\x63\xA5\x3D\x9C\x51\x8A",
-		.ilen	= 208,
-	}, { /* From draft-mcgrew-gcm-test-01 */
-		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
-			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
-			  "\x2E\x44\x3B",
-		.klen	= 19,
-		.iv	= "\x49\x56\xED\x7E\x3B\x24\x4C\xFE",
-		.result	= "\x45\x00\x00\x48\x69\x9A\x00\x00"
-			  "\x80\x11\x4D\xB7\xC0\xA8\x01\x02"
-			  "\xC0\xA8\x01\x01\x0A\x9B\xF1\x56"
-			  "\x38\xD3\x01\x00\x00\x01\x00\x00"
-			  "\x00\x00\x00\x00\x04\x5F\x73\x69"
-			  "\x70\x04\x5F\x75\x64\x70\x03\x73"
-			  "\x69\x70\x09\x63\x79\x62\x65\x72"
-			  "\x63\x69\x74\x79\x02\x64\x6B\x00"
-			  "\x00\x21\x00\x01\x01\x02\x02\x01",
-		.rlen	= 72,
-		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
-			  "\x00\x00\x00\x00\x49\x56\xED\x7E"
-			  "\x3B\x24\x4C\xFE",
-		.alen	= 20,
-		.input	= "\x89\xBA\x3E\xEF\xE6\xD6\xCF\xDB"
-			  "\x83\x60\xF5\xBA\x3A\x56\x79\xE6"
-			  "\x7E\x0C\x53\xCF\x9E\x87\xE0\x4E"
-			  "\x1A\x26\x01\x24\xC7\x2E\x3D\xBF"
-			  "\x29\x2C\x91\xC1\xB8\xA8\xCF\xE0"
-			  "\x39\xF8\x53\x6D\x31\x22\x2B\xBF"
-			  "\x98\x81\xFC\x34\xEE\x85\x36\xCD"
-			  "\x26\xDB\x6C\x7A\x0C\x77\x8A\x35"
-			  "\x18\x85\x54\xB2\xBC\xDD\x3F\x43"
-			  "\x61\x06\x8A\xDF\x86\x3F\xB4\xAC"
-			  "\x97\xDC\xBD\xFD\x92\x10\xC5\xFF",
-		.ilen	= 88,
-	}, {
-		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
-			  "\xCA\xFE\xBA",
-		.klen	= 19,
-		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.result	= "\x45\x00\x00\x3E\x69\x8F\x00\x00"
-			  "\x80\x11\x4D\xCC\xC0\xA8\x01\x02"
-			  "\xC0\xA8\x01\x01\x0A\x98\x00\x35"
-			  "\x00\x2A\x23\x43\xB2\xD0\x01\x00"
-			  "\x00\x01\x00\x00\x00\x00\x00\x00"
-			  "\x03\x73\x69\x70\x09\x63\x79\x62"
-			  "\x65\x72\x63\x69\x74\x79\x02\x64"
-			  "\x6B\x00\x00\x01\x00\x01\x00\x01",
-		.rlen	= 64,
-		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
-			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.alen	= 16,
-		.input	= "\x4B\xC2\x70\x60\x64\xD2\xF3\xC8"
-			  "\xE5\x26\x8A\xDE\xB8\x7E\x7D\x16"
-			  "\x56\xC7\xD2\x88\xBA\x8D\x58\xAF"
-			  "\xF5\x71\xB6\x37\x84\xA7\xB1\x99"
-			  "\x51\x5C\x0D\xA0\x27\xDE\xE7\x2D"
-			  "\xEF\x25\x88\x1F\x1D\x77\x11\xFF"
-			  "\xDB\xED\xEE\x56\x16\xC5\x5C\x9B"
-			  "\x00\x62\x1F\x68\x4E\x7C\xA0\x97"
-			  "\x10\x72\x7E\x53\x13\x3B\x68\xE4"
-			  "\x30\x99\x91\x79\x09\xEA\xFF\x6A",
-		.ilen	= 80,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x11\x22\x33",
-		.klen	= 35,
-		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
-		.result	= "\x45\x00\x00\x30\x69\xA6\x40\x00"
-			  "\x80\x06\x26\x90\xC0\xA8\x01\x02"
-			  "\x93\x89\x15\x5E\x0A\x9E\x00\x8B"
-			  "\x2D\xC5\x7E\xE0\x00\x00\x00\x00"
-			  "\x70\x02\x40\x00\x20\xBF\x00\x00"
-			  "\x02\x04\x05\xB4\x01\x01\x04\x02"
-			  "\x01\x02\x02\x01",
-		.rlen	= 52,
-		.assoc	= "\x4A\x2C\xBF\xE3\x00\x00\x00\x02"
-			  "\x01\x02\x03\x04\x05\x06\x07\x08",
-		.alen	= 16,
-		.input	= "\xD6\x31\x0D\x2B\x3D\x6F\xBD\x2F"
-			  "\x58\x41\x7E\xFF\x9A\x9E\x09\xB4"
-			  "\x1A\xF7\xF6\x42\x31\xCD\xBF\xAD"
-			  "\x27\x0E\x2C\xF2\xDB\x10\xDF\x55"
-			  "\x8F\x0D\xD7\xAC\x23\xBD\x42\x10"
-			  "\xD0\xB2\xAF\xD8\x37\xAC\x6B\x0B"
-			  "\x11\xD4\x0B\x12\xEC\xB4\xB1\x92"
-			  "\x23\xA6\x10\xB0\x26\xD6\xD9\x26"
-			  "\x5A\x48\x6A\x3E",
-		.ilen	= 68,
-	}, {
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00",
-		.klen	= 19,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.result	= "\x45\x00\x00\x3C\x99\xC5\x00\x00"
-			  "\x80\x01\xCB\x7A\x40\x67\x93\x18"
-			  "\x01\x01\x01\x01\x08\x00\x07\x5C"
-			  "\x02\x00\x44\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x75\x76\x77\x61\x62\x63\x64\x65"
-			  "\x66\x67\x68\x69\x01\x02\x02\x01",
-		.rlen	= 64,
-		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x01"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.alen	= 16,
-		.input	= "\x6B\x9A\xCA\x57\x43\x91\xFC\x6F"
-			  "\x92\x51\x23\xA4\xC1\x5B\xF0\x10"
-			  "\xF3\x13\xF4\xF8\xA1\x9A\xB4\xDC"
-			  "\x89\xC8\xF8\x42\x62\x95\xB7\xCB"
-			  "\xB8\xF5\x0F\x1B\x2E\x94\xA2\xA7"
-			  "\xBF\xFB\x8A\x92\x13\x63\xD1\x3C"
-			  "\x08\xF5\xE8\xA6\xAA\xF6\x34\xF9"
-			  "\x42\x05\xAF\xB3\xE7\x9A\xFC\xEE"
-			  "\x36\x25\xC1\x10\x12\x1C\xCA\x82"
-			  "\xEA\xE6\x63\x5A\x57\x28\xA9\x9A",
-		.ilen	= 80,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E",
-		.klen	= 19,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.result	= "\x45\x00\x00\x3C\x99\xC3\x00\x00"
-			  "\x80\x01\xCB\x7C\x40\x67\x93\x18"
-			  "\x01\x01\x01\x01\x08\x00\x08\x5C"
-			  "\x02\x00\x43\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x75\x76\x77\x61\x62\x63\x64\x65"
-			  "\x66\x67\x68\x69\x01\x02\x02\x01",
-		.rlen	= 64,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.input	= "\x6A\x6B\x45\x2B\x7C\x67\x52\xF6"
-			  "\x10\x60\x40\x62\x6B\x4F\x97\x8E"
-			  "\x0B\xB2\x22\x97\xCB\x21\xE0\x90"
-			  "\xA2\xE7\xD1\x41\x30\xE4\x4B\x1B"
-			  "\x79\x01\x58\x50\x01\x06\xE1\xE0"
-			  "\x2C\x83\x79\xD3\xDE\x46\x97\x1A"
-			  "\x30\xB8\xE5\xDF\xD7\x12\x56\x75"
-			  "\xD0\x95\xB7\xB8\x91\x42\xF7\xFD"
-			  "\x97\x57\xCA\xC1\x20\xD0\x86\xB9"
-			  "\x66\x9D\xB4\x2B\x96\x22\xAC\x67",
-		.ilen	= 80,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E",
-		.klen	= 19,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.result	= "\x45\x00\x00\x1C\x42\xA2\x00\x00"
-			  "\x80\x01\x44\x1F\x40\x67\x93\xB6"
-			  "\xE0\x00\x00\x02\x0A\x00\xF5\xFF"
-			  "\x01\x02\x02\x01",
-		.rlen	= 28,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.input	= "\x6A\x6B\x45\x0B\xA7\x06\x52\xF6"
-			  "\x10\x60\xCF\x01\x6B\x4F\x97\x20"
-			  "\xEA\xB3\x23\x94\xC9\x21\x1D\x33"
-			  "\xA1\xE5\x90\x40\x05\x37\x45\x70"
-			  "\xB5\xD6\x09\x0A\x23\x73\x33\xF9"
-			  "\x08\xB4\x22\xE4",
-		.ilen	= 44,
-	}, {
-		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
-			  "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
-			  "\xCA\xFE\xBA",
-		.klen	= 27,
-		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.result	= "\x45\x00\x00\x28\xA4\xAD\x40\x00"
-			  "\x40\x06\x78\x80\x0A\x01\x03\x8F"
-			  "\x0A\x01\x06\x12\x80\x23\x06\xB8"
-			  "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E"
-			  "\x50\x10\x16\xD0\x75\x68\x00\x01",
-		.rlen	= 40,
-		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
-			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
-		.alen	= 16,
-		.input	= "\x05\x22\x15\xD1\x52\x56\x85\x04"
-			  "\xA8\x5C\x5D\x6D\x7E\x6E\xF5\xFA"
-			  "\xEA\x16\x37\x50\xF3\xDF\x84\x3B"
-			  "\x2F\x32\x18\x57\x34\x2A\x8C\x23"
-			  "\x67\xDF\x6D\x35\x7B\x54\x0D\xFB"
-			  "\x34\xA5\x9F\x6C\x48\x30\x1E\x22"
-			  "\xFE\xB1\x22\x17\x17\x8A\xB9\x5B",
-		.ilen	= 56,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xDE\xCA\xF8",
-		.klen	= 19,
-		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
-		.result	= "\x45\x00\x00\x49\x33\xBA\x00\x00"
-			  "\x7F\x11\x91\x06\xC3\xFB\x1D\x10"
-			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
-			  "\x00\x35\xDD\x7B\x80\x03\x02\xD5"
-			  "\x00\x00\x4E\x20\x00\x1E\x8C\x18"
-			  "\xD7\x5B\x81\xDC\x91\xBA\xA0\x47"
-			  "\x6B\x91\xB9\x24\xB2\x80\x38\x9D"
-			  "\x92\xC9\x63\xBA\xC0\x46\xEC\x95"
-			  "\x9B\x62\x66\xC0\x47\x22\xB1\x49"
-			  "\x23\x01\x01\x01",
-		.rlen	= 76,
-		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
-			  "\xCE\xFA\xCE\x74",
-		.alen	= 20,
-		.input	= "\x92\xD0\x53\x79\x33\x38\xD5\xF3"
-			  "\x7D\xE4\x7A\x8E\x86\x03\xC9\x90"
-			  "\x96\x35\xAB\x9C\xFB\xE8\xA3\x76"
-			  "\xE9\xE9\xE2\xD1\x2E\x11\x0E\x00"
-			  "\xFA\xCE\xB5\x9E\x02\xA7\x7B\xEA"
-			  "\x71\x9A\x58\xFB\xA5\x8A\xE1\xB7"
-			  "\x9C\x39\x9D\xE3\xB5\x6E\x69\xE6"
-			  "\x63\xC9\xDB\x05\x69\x51\x12\xAD"
-			  "\x3E\x00\x32\x73\x86\xF2\xEE\xF5"
-			  "\x0F\xE8\x81\x7E\x84\xD3\xC0\x0D"
-			  "\x76\xD6\x55\xC6\xB4\xC2\x34\xC7"
-			  "\x12\x25\x0B\xF9",
-		.ilen	= 92,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x73\x61\x6C",
-		.klen	= 35,
-		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
-		.result	= "\x45\x08\x00\x28\x73\x2C\x00\x00"
-			  "\x40\x06\xE9\xF9\x0A\x01\x06\x12"
-			  "\x0A\x01\x03\x8F\x06\xB8\x80\x23"
-			  "\xDD\x6B\xAF\xBE\xCB\x71\x26\x02"
-			  "\x50\x10\x1F\x64\x6D\x54\x00\x01",
-		.rlen	= 40,
-		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
-			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
-			  "\x69\x76\x65\x63",
-		.alen	= 20,
-		.input	= "\xCC\x74\xB7\xD3\xB0\x38\x50\x42"
-			  "\x2C\x64\x87\x46\x1E\x34\x10\x05"
-			  "\x29\x6B\xBB\x36\xE9\x69\xAD\x92"
-			  "\x82\xA1\x10\x6A\xEB\x0F\xDC\x7D"
-			  "\x08\xBA\xF3\x91\xCA\xAA\x61\xDA"
-			  "\x62\xF4\x14\x61\x5C\x9D\xB5\xA7"
-			  "\xEE\xD7\xB9\x7E\x87\x99\x9B\x7D",
-		.ilen	= 56,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E",
-		.klen	= 19,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.result	= "\x45\x00\x00\x49\x33\x3E\x00\x00"
-			  "\x7F\x11\x91\x82\xC3\xFB\x1D\x10"
-			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
-			  "\x00\x35\xCB\x45\x80\x03\x02\x5B"
-			  "\x00\x00\x01\xE0\x00\x1E\x8C\x18"
-			  "\xD6\x57\x59\xD5\x22\x84\xA0\x35"
-			  "\x2C\x71\x47\x5C\x88\x80\x39\x1C"
-			  "\x76\x4D\x6E\x5E\xE0\x49\x6B\x32"
-			  "\x5A\xE2\x70\xC0\x38\x99\x49\x39"
-			  "\x15\x01\x01\x01",
-		.rlen	= 76,
-		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.input	= "\x6A\x6B\x45\x5E\xD6\x9A\x52\xF6"
-			  "\xEF\x70\x1A\x9C\xE8\xD3\x19\x86"
-			  "\xC8\x02\xF0\xB0\x03\x09\xD9\x02"
-			  "\xA0\xD2\x59\x04\xD1\x85\x2A\x24"
-			  "\x1C\x67\x3E\xD8\x68\x72\x06\x94"
-			  "\x97\xBA\x4F\x76\x8D\xB0\x44\x5B"
-			  "\x69\xBF\xD5\xE2\x3D\xF1\x0B\x0C"
-			  "\xC0\xBF\xB1\x8F\x70\x09\x9E\xCE"
-			  "\xA5\xF2\x55\x58\x84\xFA\xF9\xB5"
-			  "\x23\xF4\x84\x40\x74\x14\x8A\x6B"
-			  "\xDB\xD7\x67\xED\xA4\x93\xF3\x47"
-			  "\xCC\xF7\x46\x6F",
-		.ilen	= 92,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\x73\x61\x6C",
-		.klen	= 35,
-		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
-		.result	= "\x63\x69\x73\x63\x6F\x01\x72\x75"
-			  "\x6C\x65\x73\x01\x74\x68\x65\x01"
-			  "\x6E\x65\x74\x77\x65\x01\x64\x65"
-			  "\x66\x69\x6E\x65\x01\x74\x68\x65"
-			  "\x74\x65\x63\x68\x6E\x6F\x6C\x6F"
-			  "\x67\x69\x65\x73\x01\x74\x68\x61"
-			  "\x74\x77\x69\x6C\x6C\x01\x64\x65"
-			  "\x66\x69\x6E\x65\x74\x6F\x6D\x6F"
-			  "\x72\x72\x6F\x77\x01\x02\x02\x01",
-		.rlen	= 72,
-		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
-			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
-			  "\x69\x76\x65\x63",
-		.alen	= 20,
-		.input	= "\xEA\x15\xC4\x98\xAC\x15\x22\x37"
-			  "\x00\x07\x1D\xBE\x60\x5D\x73\x16"
-			  "\x4D\x0F\xCC\xCE\x8A\xD0\x49\xD4"
-			  "\x39\xA3\xD1\xB1\x21\x0A\x92\x1A"
-			  "\x2C\xCF\x8F\x9D\xC9\x91\x0D\xB4"
-			  "\x15\xFC\xBC\xA5\xC5\xBF\x54\xE5"
-			  "\x1C\xC7\x32\x41\x07\x7B\x2C\xB6"
-			  "\x5C\x23\x7C\x93\xEA\xEF\x23\x1C"
-			  "\x73\xF4\xE7\x12\x84\x4C\x37\x0A"
-			  "\x4A\x8F\x06\x37\x48\xF9\xF9\x05"
-			  "\x55\x13\x40\xC3\xD5\x55\x3A\x3D",
-		.ilen	= 88,
-	}, {
-		.key	= "\x7D\x77\x3D\x00\xC1\x44\xC5\x25"
-			  "\xAC\x61\x9D\x18\xC8\x4A\x3F\x47"
-			  "\xD9\x66\x42",
-		.klen	= 19,
-		.iv	= "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
-		.result	= "\x01\x02\x02\x01",
-		.rlen	= 4,
-		.assoc	= "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF"
-			  "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
-		.alen	= 16,
-		.input	= "\x4C\x72\x63\x30\x2F\xE6\x56\xDD"
-			  "\xD0\xD8\x60\x9D\x8B\xEF\x85\x90"
-			  "\xF7\x61\x24\x62",
-		.ilen	= 20,
-	}, {
-		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
-			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
-			  "\xDE\xCA\xF8",
-		.klen	= 19,
-		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
-		.result	= "\x74\x6F\x01\x62\x65\x01\x6F\x72"
-			  "\x01\x6E\x6F\x74\x01\x74\x6F\x01"
-			  "\x62\x65\x00\x01",
-		.rlen	= 20,
-		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
-			  "\xCE\xFA\xCE\x74",
-		.alen	= 20,
-		.input	= "\xA3\xBF\x52\x52\x65\x83\xBA\x81"
-			  "\x03\x9B\x84\xFC\x44\x8C\xBB\x81"
-			  "\x36\xE1\x78\xBB\xA5\x49\x3A\xD0"
-			  "\xF0\x6B\x21\xAF\x98\xC0\x34\xDC"
-			  "\x17\x17\x65\xAD",
-		.ilen	= 36,
-	}, {
-		.key	= "\x6C\x65\x67\x61\x6C\x69\x7A\x65"
-			  "\x6D\x61\x72\x69\x6A\x75\x61\x6E"
-			  "\x61\x61\x6E\x64\x64\x6F\x69\x74"
-			  "\x62\x65\x66\x6F\x72\x65\x69\x61"
-			  "\x74\x75\x72",
-		.klen	= 35,
-		.iv	= "\x33\x30\x21\x69\x67\x65\x74\x6D",
-		.result	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
-			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
-			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01",
-		.rlen	= 52,
-		.assoc	= "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\x33\x30\x21\x69"
-			  "\x67\x65\x74\x6D",
-		.alen	= 20,
-		.input	= "\x96\xFD\x86\xF8\xD1\x98\xFF\x10"
-			  "\xAB\x8C\xDA\x8A\x5A\x08\x38\x1A"
-			  "\x48\x59\x80\x18\x1A\x18\x1A\x04"
-			  "\xC9\x0D\xE3\xE7\x0E\xA4\x0B\x75"
-			  "\x92\x9C\x52\x5C\x0B\xFB\xF8\xAF"
-			  "\x16\xC3\x35\xA8\xE7\xCE\x84\x04"
-			  "\xEB\x40\x6B\x7A\x8E\x75\xBB\x42"
-			  "\xE0\x63\x4B\x21\x44\xA2\x2B\x2B"
-			  "\x39\xDB\xC8\xDC",
-		.ilen	= 68,
-	}, {
-		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
-			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
-			  "\x57\x69\x0E",
-		.klen	= 19,
-		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
-		.result	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
-			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
-			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
-			  "\x02\x00\x07\x00\x61\x62\x63\x64"
-			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
-			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
-			  "\x01\x02\x02\x01",
-		.rlen	= 52,
-		.assoc	= "\x3F\x7E\xF6\x42\x10\x10\x10\x10"
-			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
-			  "\xA2\xFC\xA1\xA3",
-		.alen	= 20,
-		.input	= "\x6A\x6B\x45\x27\x3F\x9E\x52\xF6"
-			  "\x10\x60\x54\x25\xEB\x80\x04\x93"
-			  "\xCA\x1B\x23\x97\xCB\x21\x2E\x01"
-			  "\xA2\xE7\x95\x41\x30\xE4\x4B\x1B"
-			  "\x79\x01\x58\x50\x01\x06\xE1\xE0"
-			  "\x2C\x83\x79\xD3\xDE\x46\x97\x1A"
-			  "\x44\xCC\x90\xBF\x00\x94\x94\x92"
-			  "\x20\x17\x0C\x1B\x55\xDE\x7E\x68"
-			  "\xF4\x95\x5D\x4F",
-		.ilen	= 68,
-	}, {
-		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
-			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
-			  "\x22\x43\x3C",
-		.klen	= 19,
-		.iv	= "\x48\x55\xEC\x7D\x3A\x23\x4B\xFD",
-		.result	= "\x08\x00\xC6\xCD\x02\x00\x07\x00"
-			  "\x61\x62\x63\x64\x65\x66\x67\x68"
-			  "\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70"
-			  "\x71\x72\x73\x74\x01\x02\x02\x01",
-		.rlen	= 32,
-		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
-			  "\x00\x00\x00\x07\x48\x55\xEC\x7D"
-			  "\x3A\x23\x4B\xFD",
-		.alen	= 20,
-		.input	= "\x67\xE9\x28\xB3\x1C\xA4\x6D\x02"
-			  "\xF0\xB5\x37\xB6\x6B\x2F\xF5\x4F"
-			  "\xF8\xA3\x4C\x53\xB8\x12\x09\xBF"
-			  "\x58\x7D\xCF\x29\xA3\x41\x68\x6B"
-			  "\xCE\xE8\x79\x85\x3C\xB0\x3A\x8F"
-			  "\x16\xB0\xA1\x26\xC9\xBC\xBC\xA6",
-		.ilen	= 48,
-	}
-};
-
-/*
- * ChaCha20-Poly1305 AEAD test vectors from RFC7539 2.8.2./A.5.
- */
-static const struct aead_testvec rfc7539_enc_tv_template[] = {
-	{
-		.key	= "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f",
-		.klen	= 32,
-		.iv	= "\x07\x00\x00\x00\x40\x41\x42\x43"
-			  "\x44\x45\x46\x47",
-		.assoc	= "\x50\x51\x52\x53\xc0\xc1\xc2\xc3"
-			  "\xc4\xc5\xc6\xc7",
-		.alen	= 12,
-		.input	= "\x4c\x61\x64\x69\x65\x73\x20\x61"
-			  "\x6e\x64\x20\x47\x65\x6e\x74\x6c"
-			  "\x65\x6d\x65\x6e\x20\x6f\x66\x20"
-			  "\x74\x68\x65\x20\x63\x6c\x61\x73"
-			  "\x73\x20\x6f\x66\x20\x27\x39\x39"
-			  "\x3a\x20\x49\x66\x20\x49\x20\x63"
-			  "\x6f\x75\x6c\x64\x20\x6f\x66\x66"
-			  "\x65\x72\x20\x79\x6f\x75\x20\x6f"
-			  "\x6e\x6c\x79\x20\x6f\x6e\x65\x20"
-			  "\x74\x69\x70\x20\x66\x6f\x72\x20"
-			  "\x74\x68\x65\x20\x66\x75\x74\x75"
-			  "\x72\x65\x2c\x20\x73\x75\x6e\x73"
-			  "\x63\x72\x65\x65\x6e\x20\x77\x6f"
-			  "\x75\x6c\x64\x20\x62\x65\x20\x69"
-			  "\x74\x2e",
-		.ilen	= 114,
-		.result	= "\xd3\x1a\x8d\x34\x64\x8e\x60\xdb"
-			  "\x7b\x86\xaf\xbc\x53\xef\x7e\xc2"
-			  "\xa4\xad\xed\x51\x29\x6e\x08\xfe"
-			  "\xa9\xe2\xb5\xa7\x36\xee\x62\xd6"
-			  "\x3d\xbe\xa4\x5e\x8c\xa9\x67\x12"
-			  "\x82\xfa\xfb\x69\xda\x92\x72\x8b"
-			  "\x1a\x71\xde\x0a\x9e\x06\x0b\x29"
-			  "\x05\xd6\xa5\xb6\x7e\xcd\x3b\x36"
-			  "\x92\xdd\xbd\x7f\x2d\x77\x8b\x8c"
-			  "\x98\x03\xae\xe3\x28\x09\x1b\x58"
-			  "\xfa\xb3\x24\xe4\xfa\xd6\x75\x94"
-			  "\x55\x85\x80\x8b\x48\x31\xd7\xbc"
-			  "\x3f\xf4\xde\xf0\x8e\x4b\x7a\x9d"
-			  "\xe5\x76\xd2\x65\x86\xce\xc6\x4b"
-			  "\x61\x16\x1a\xe1\x0b\x59\x4f\x09"
-			  "\xe2\x6a\x7e\x90\x2e\xcb\xd0\x60"
-			  "\x06\x91",
-		.rlen	= 130,
-	}, {
-		.key	= "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
-			  "\xf3\x33\x88\x86\x04\xf6\xb5\xf0"
-			  "\x47\x39\x17\xc1\x40\x2b\x80\x09"
-			  "\x9d\xca\x5c\xbc\x20\x70\x75\xc0",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x01\x02\x03\x04"
-			  "\x05\x06\x07\x08",
-		.assoc	= "\xf3\x33\x88\x86\x00\x00\x00\x00"
-			  "\x00\x00\x4e\x91",
-		.alen	= 12,
-		.input	= "\x49\x6e\x74\x65\x72\x6e\x65\x74"
-			  "\x2d\x44\x72\x61\x66\x74\x73\x20"
-			  "\x61\x72\x65\x20\x64\x72\x61\x66"
-			  "\x74\x20\x64\x6f\x63\x75\x6d\x65"
-			  "\x6e\x74\x73\x20\x76\x61\x6c\x69"
-			  "\x64\x20\x66\x6f\x72\x20\x61\x20"
-			  "\x6d\x61\x78\x69\x6d\x75\x6d\x20"
-			  "\x6f\x66\x20\x73\x69\x78\x20\x6d"
-			  "\x6f\x6e\x74\x68\x73\x20\x61\x6e"
-			  "\x64\x20\x6d\x61\x79\x20\x62\x65"
-			  "\x20\x75\x70\x64\x61\x74\x65\x64"
-			  "\x2c\x20\x72\x65\x70\x6c\x61\x63"
-			  "\x65\x64\x2c\x20\x6f\x72\x20\x6f"
-			  "\x62\x73\x6f\x6c\x65\x74\x65\x64"
-			  "\x20\x62\x79\x20\x6f\x74\x68\x65"
-			  "\x72\x20\x64\x6f\x63\x75\x6d\x65"
-			  "\x6e\x74\x73\x20\x61\x74\x20\x61"
-			  "\x6e\x79\x20\x74\x69\x6d\x65\x2e"
-			  "\x20\x49\x74\x20\x69\x73\x20\x69"
-			  "\x6e\x61\x70\x70\x72\x6f\x70\x72"
-			  "\x69\x61\x74\x65\x20\x74\x6f\x20"
-			  "\x75\x73\x65\x20\x49\x6e\x74\x65"
-			  "\x72\x6e\x65\x74\x2d\x44\x72\x61"
-			  "\x66\x74\x73\x20\x61\x73\x20\x72"
-			  "\x65\x66\x65\x72\x65\x6e\x63\x65"
-			  "\x20\x6d\x61\x74\x65\x72\x69\x61"
-			  "\x6c\x20\x6f\x72\x20\x74\x6f\x20"
-			  "\x63\x69\x74\x65\x20\x74\x68\x65"
-			  "\x6d\x20\x6f\x74\x68\x65\x72\x20"
-			  "\x74\x68\x61\x6e\x20\x61\x73\x20"
-			  "\x2f\xe2\x80\x9c\x77\x6f\x72\x6b"
-			  "\x20\x69\x6e\x20\x70\x72\x6f\x67"
-			  "\x72\x65\x73\x73\x2e\x2f\xe2\x80"
-			  "\x9d",
-		.ilen	= 265,
-		.result	= "\x64\xa0\x86\x15\x75\x86\x1a\xf4"
-			  "\x60\xf0\x62\xc7\x9b\xe6\x43\xbd"
-			  "\x5e\x80\x5c\xfd\x34\x5c\xf3\x89"
-			  "\xf1\x08\x67\x0a\xc7\x6c\x8c\xb2"
-			  "\x4c\x6c\xfc\x18\x75\x5d\x43\xee"
-			  "\xa0\x9e\xe9\x4e\x38\x2d\x26\xb0"
-			  "\xbd\xb7\xb7\x3c\x32\x1b\x01\x00"
-			  "\xd4\xf0\x3b\x7f\x35\x58\x94\xcf"
-			  "\x33\x2f\x83\x0e\x71\x0b\x97\xce"
-			  "\x98\xc8\xa8\x4a\xbd\x0b\x94\x81"
-			  "\x14\xad\x17\x6e\x00\x8d\x33\xbd"
-			  "\x60\xf9\x82\xb1\xff\x37\xc8\x55"
-			  "\x97\x97\xa0\x6e\xf4\xf0\xef\x61"
-			  "\xc1\x86\x32\x4e\x2b\x35\x06\x38"
-			  "\x36\x06\x90\x7b\x6a\x7c\x02\xb0"
-			  "\xf9\xf6\x15\x7b\x53\xc8\x67\xe4"
-			  "\xb9\x16\x6c\x76\x7b\x80\x4d\x46"
-			  "\xa5\x9b\x52\x16\xcd\xe7\xa4\xe9"
-			  "\x90\x40\xc5\xa4\x04\x33\x22\x5e"
-			  "\xe2\x82\xa1\xb0\xa0\x6c\x52\x3e"
-			  "\xaf\x45\x34\xd7\xf8\x3f\xa1\x15"
-			  "\x5b\x00\x47\x71\x8c\xbc\x54\x6a"
-			  "\x0d\x07\x2b\x04\xb3\x56\x4e\xea"
-			  "\x1b\x42\x22\x73\xf5\x48\x27\x1a"
-			  "\x0b\xb2\x31\x60\x53\xfa\x76\x99"
-			  "\x19\x55\xeb\xd6\x31\x59\x43\x4e"
-			  "\xce\xbb\x4e\x46\x6d\xae\x5a\x10"
-			  "\x73\xa6\x72\x76\x27\x09\x7a\x10"
-			  "\x49\xe6\x17\xd9\x1d\x36\x10\x94"
-			  "\xfa\x68\xf0\xff\x77\x98\x71\x30"
-			  "\x30\x5b\xea\xba\x2e\xda\x04\xdf"
-			  "\x99\x7b\x71\x4d\x6c\x6f\x2c\x29"
-			  "\xa6\xad\x5c\xb4\x02\x2b\x02\x70"
-			  "\x9b\xee\xad\x9d\x67\x89\x0c\xbb"
-			  "\x22\x39\x23\x36\xfe\xa1\x85\x1f"
-			  "\x38",
-		.rlen	= 281,
-	},
-};
-
-static const struct aead_testvec rfc7539_dec_tv_template[] = {
-	{
-		.key	= "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f",
-		.klen	= 32,
-		.iv	= "\x07\x00\x00\x00\x40\x41\x42\x43"
-			  "\x44\x45\x46\x47",
-		.assoc	= "\x50\x51\x52\x53\xc0\xc1\xc2\xc3"
-			  "\xc4\xc5\xc6\xc7",
-		.alen	= 12,
-		.input	= "\xd3\x1a\x8d\x34\x64\x8e\x60\xdb"
-			  "\x7b\x86\xaf\xbc\x53\xef\x7e\xc2"
-			  "\xa4\xad\xed\x51\x29\x6e\x08\xfe"
-			  "\xa9\xe2\xb5\xa7\x36\xee\x62\xd6"
-			  "\x3d\xbe\xa4\x5e\x8c\xa9\x67\x12"
-			  "\x82\xfa\xfb\x69\xda\x92\x72\x8b"
-			  "\x1a\x71\xde\x0a\x9e\x06\x0b\x29"
-			  "\x05\xd6\xa5\xb6\x7e\xcd\x3b\x36"
-			  "\x92\xdd\xbd\x7f\x2d\x77\x8b\x8c"
-			  "\x98\x03\xae\xe3\x28\x09\x1b\x58"
-			  "\xfa\xb3\x24\xe4\xfa\xd6\x75\x94"
-			  "\x55\x85\x80\x8b\x48\x31\xd7\xbc"
-			  "\x3f\xf4\xde\xf0\x8e\x4b\x7a\x9d"
-			  "\xe5\x76\xd2\x65\x86\xce\xc6\x4b"
-			  "\x61\x16\x1a\xe1\x0b\x59\x4f\x09"
-			  "\xe2\x6a\x7e\x90\x2e\xcb\xd0\x60"
-			  "\x06\x91",
-		.ilen	= 130,
-		.result	= "\x4c\x61\x64\x69\x65\x73\x20\x61"
-			  "\x6e\x64\x20\x47\x65\x6e\x74\x6c"
-			  "\x65\x6d\x65\x6e\x20\x6f\x66\x20"
-			  "\x74\x68\x65\x20\x63\x6c\x61\x73"
-			  "\x73\x20\x6f\x66\x20\x27\x39\x39"
-			  "\x3a\x20\x49\x66\x20\x49\x20\x63"
-			  "\x6f\x75\x6c\x64\x20\x6f\x66\x66"
-			  "\x65\x72\x20\x79\x6f\x75\x20\x6f"
-			  "\x6e\x6c\x79\x20\x6f\x6e\x65\x20"
-			  "\x74\x69\x70\x20\x66\x6f\x72\x20"
-			  "\x74\x68\x65\x20\x66\x75\x74\x75"
-			  "\x72\x65\x2c\x20\x73\x75\x6e\x73"
-			  "\x63\x72\x65\x65\x6e\x20\x77\x6f"
-			  "\x75\x6c\x64\x20\x62\x65\x20\x69"
-			  "\x74\x2e",
-		.rlen	= 114,
-	}, {
-		.key	= "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
-			  "\xf3\x33\x88\x86\x04\xf6\xb5\xf0"
-			  "\x47\x39\x17\xc1\x40\x2b\x80\x09"
-			  "\x9d\xca\x5c\xbc\x20\x70\x75\xc0",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x01\x02\x03\x04"
-			  "\x05\x06\x07\x08",
-		.assoc	= "\xf3\x33\x88\x86\x00\x00\x00\x00"
-			  "\x00\x00\x4e\x91",
-		.alen	= 12,
-		.input	= "\x64\xa0\x86\x15\x75\x86\x1a\xf4"
-			  "\x60\xf0\x62\xc7\x9b\xe6\x43\xbd"
-			  "\x5e\x80\x5c\xfd\x34\x5c\xf3\x89"
-			  "\xf1\x08\x67\x0a\xc7\x6c\x8c\xb2"
-			  "\x4c\x6c\xfc\x18\x75\x5d\x43\xee"
-			  "\xa0\x9e\xe9\x4e\x38\x2d\x26\xb0"
-			  "\xbd\xb7\xb7\x3c\x32\x1b\x01\x00"
-			  "\xd4\xf0\x3b\x7f\x35\x58\x94\xcf"
-			  "\x33\x2f\x83\x0e\x71\x0b\x97\xce"
-			  "\x98\xc8\xa8\x4a\xbd\x0b\x94\x81"
-			  "\x14\xad\x17\x6e\x00\x8d\x33\xbd"
-			  "\x60\xf9\x82\xb1\xff\x37\xc8\x55"
-			  "\x97\x97\xa0\x6e\xf4\xf0\xef\x61"
-			  "\xc1\x86\x32\x4e\x2b\x35\x06\x38"
-			  "\x36\x06\x90\x7b\x6a\x7c\x02\xb0"
-			  "\xf9\xf6\x15\x7b\x53\xc8\x67\xe4"
-			  "\xb9\x16\x6c\x76\x7b\x80\x4d\x46"
-			  "\xa5\x9b\x52\x16\xcd\xe7\xa4\xe9"
-			  "\x90\x40\xc5\xa4\x04\x33\x22\x5e"
-			  "\xe2\x82\xa1\xb0\xa0\x6c\x52\x3e"
-			  "\xaf\x45\x34\xd7\xf8\x3f\xa1\x15"
-			  "\x5b\x00\x47\x71\x8c\xbc\x54\x6a"
-			  "\x0d\x07\x2b\x04\xb3\x56\x4e\xea"
-			  "\x1b\x42\x22\x73\xf5\x48\x27\x1a"
-			  "\x0b\xb2\x31\x60\x53\xfa\x76\x99"
-			  "\x19\x55\xeb\xd6\x31\x59\x43\x4e"
-			  "\xce\xbb\x4e\x46\x6d\xae\x5a\x10"
-			  "\x73\xa6\x72\x76\x27\x09\x7a\x10"
-			  "\x49\xe6\x17\xd9\x1d\x36\x10\x94"
-			  "\xfa\x68\xf0\xff\x77\x98\x71\x30"
-			  "\x30\x5b\xea\xba\x2e\xda\x04\xdf"
-			  "\x99\x7b\x71\x4d\x6c\x6f\x2c\x29"
-			  "\xa6\xad\x5c\xb4\x02\x2b\x02\x70"
-			  "\x9b\xee\xad\x9d\x67\x89\x0c\xbb"
-			  "\x22\x39\x23\x36\xfe\xa1\x85\x1f"
-			  "\x38",
-		.ilen	= 281,
-		.result	= "\x49\x6e\x74\x65\x72\x6e\x65\x74"
-			  "\x2d\x44\x72\x61\x66\x74\x73\x20"
-			  "\x61\x72\x65\x20\x64\x72\x61\x66"
-			  "\x74\x20\x64\x6f\x63\x75\x6d\x65"
-			  "\x6e\x74\x73\x20\x76\x61\x6c\x69"
-			  "\x64\x20\x66\x6f\x72\x20\x61\x20"
-			  "\x6d\x61\x78\x69\x6d\x75\x6d\x20"
-			  "\x6f\x66\x20\x73\x69\x78\x20\x6d"
-			  "\x6f\x6e\x74\x68\x73\x20\x61\x6e"
-			  "\x64\x20\x6d\x61\x79\x20\x62\x65"
-			  "\x20\x75\x70\x64\x61\x74\x65\x64"
-			  "\x2c\x20\x72\x65\x70\x6c\x61\x63"
-			  "\x65\x64\x2c\x20\x6f\x72\x20\x6f"
-			  "\x62\x73\x6f\x6c\x65\x74\x65\x64"
-			  "\x20\x62\x79\x20\x6f\x74\x68\x65"
-			  "\x72\x20\x64\x6f\x63\x75\x6d\x65"
-			  "\x6e\x74\x73\x20\x61\x74\x20\x61"
-			  "\x6e\x79\x20\x74\x69\x6d\x65\x2e"
-			  "\x20\x49\x74\x20\x69\x73\x20\x69"
-			  "\x6e\x61\x70\x70\x72\x6f\x70\x72"
-			  "\x69\x61\x74\x65\x20\x74\x6f\x20"
-			  "\x75\x73\x65\x20\x49\x6e\x74\x65"
-			  "\x72\x6e\x65\x74\x2d\x44\x72\x61"
-			  "\x66\x74\x73\x20\x61\x73\x20\x72"
-			  "\x65\x66\x65\x72\x65\x6e\x63\x65"
-			  "\x20\x6d\x61\x74\x65\x72\x69\x61"
-			  "\x6c\x20\x6f\x72\x20\x74\x6f\x20"
-			  "\x63\x69\x74\x65\x20\x74\x68\x65"
-			  "\x6d\x20\x6f\x74\x68\x65\x72\x20"
-			  "\x74\x68\x61\x6e\x20\x61\x73\x20"
-			  "\x2f\xe2\x80\x9c\x77\x6f\x72\x6b"
-			  "\x20\x69\x6e\x20\x70\x72\x6f\x67"
-			  "\x72\x65\x73\x73\x2e\x2f\xe2\x80"
-			  "\x9d",
-		.rlen	= 265,
-	},
-};
-
-/*
- * draft-irtf-cfrg-chacha20-poly1305
- */
-static const struct aead_testvec rfc7539esp_enc_tv_template[] = {
-	{
-		.key	= "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
-			  "\xf3\x33\x88\x86\x04\xf6\xb5\xf0"
-			  "\x47\x39\x17\xc1\x40\x2b\x80\x09"
-			  "\x9d\xca\x5c\xbc\x20\x70\x75\xc0"
-			  "\x00\x00\x00\x00",
-		.klen	= 36,
-		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
-		.assoc	= "\xf3\x33\x88\x86\x00\x00\x00\x00"
-			  "\x00\x00\x4e\x91\x01\x02\x03\x04"
-			  "\x05\x06\x07\x08",
-		.alen	= 20,
-		.input	= "\x49\x6e\x74\x65\x72\x6e\x65\x74"
-			  "\x2d\x44\x72\x61\x66\x74\x73\x20"
-			  "\x61\x72\x65\x20\x64\x72\x61\x66"
-			  "\x74\x20\x64\x6f\x63\x75\x6d\x65"
-			  "\x6e\x74\x73\x20\x76\x61\x6c\x69"
-			  "\x64\x20\x66\x6f\x72\x20\x61\x20"
-			  "\x6d\x61\x78\x69\x6d\x75\x6d\x20"
-			  "\x6f\x66\x20\x73\x69\x78\x20\x6d"
-			  "\x6f\x6e\x74\x68\x73\x20\x61\x6e"
-			  "\x64\x20\x6d\x61\x79\x20\x62\x65"
-			  "\x20\x75\x70\x64\x61\x74\x65\x64"
-			  "\x2c\x20\x72\x65\x70\x6c\x61\x63"
-			  "\x65\x64\x2c\x20\x6f\x72\x20\x6f"
-			  "\x62\x73\x6f\x6c\x65\x74\x65\x64"
-			  "\x20\x62\x79\x20\x6f\x74\x68\x65"
-			  "\x72\x20\x64\x6f\x63\x75\x6d\x65"
-			  "\x6e\x74\x73\x20\x61\x74\x20\x61"
-			  "\x6e\x79\x20\x74\x69\x6d\x65\x2e"
-			  "\x20\x49\x74\x20\x69\x73\x20\x69"
-			  "\x6e\x61\x70\x70\x72\x6f\x70\x72"
-			  "\x69\x61\x74\x65\x20\x74\x6f\x20"
-			  "\x75\x73\x65\x20\x49\x6e\x74\x65"
-			  "\x72\x6e\x65\x74\x2d\x44\x72\x61"
-			  "\x66\x74\x73\x20\x61\x73\x20\x72"
-			  "\x65\x66\x65\x72\x65\x6e\x63\x65"
-			  "\x20\x6d\x61\x74\x65\x72\x69\x61"
-			  "\x6c\x20\x6f\x72\x20\x74\x6f\x20"
-			  "\x63\x69\x74\x65\x20\x74\x68\x65"
-			  "\x6d\x20\x6f\x74\x68\x65\x72\x20"
-			  "\x74\x68\x61\x6e\x20\x61\x73\x20"
-			  "\x2f\xe2\x80\x9c\x77\x6f\x72\x6b"
-			  "\x20\x69\x6e\x20\x70\x72\x6f\x67"
-			  "\x72\x65\x73\x73\x2e\x2f\xe2\x80"
-			  "\x9d",
-		.ilen	= 265,
-		.result	= "\x64\xa0\x86\x15\x75\x86\x1a\xf4"
-			  "\x60\xf0\x62\xc7\x9b\xe6\x43\xbd"
-			  "\x5e\x80\x5c\xfd\x34\x5c\xf3\x89"
-			  "\xf1\x08\x67\x0a\xc7\x6c\x8c\xb2"
-			  "\x4c\x6c\xfc\x18\x75\x5d\x43\xee"
-			  "\xa0\x9e\xe9\x4e\x38\x2d\x26\xb0"
-			  "\xbd\xb7\xb7\x3c\x32\x1b\x01\x00"
-			  "\xd4\xf0\x3b\x7f\x35\x58\x94\xcf"
-			  "\x33\x2f\x83\x0e\x71\x0b\x97\xce"
-			  "\x98\xc8\xa8\x4a\xbd\x0b\x94\x81"
-			  "\x14\xad\x17\x6e\x00\x8d\x33\xbd"
-			  "\x60\xf9\x82\xb1\xff\x37\xc8\x55"
-			  "\x97\x97\xa0\x6e\xf4\xf0\xef\x61"
-			  "\xc1\x86\x32\x4e\x2b\x35\x06\x38"
-			  "\x36\x06\x90\x7b\x6a\x7c\x02\xb0"
-			  "\xf9\xf6\x15\x7b\x53\xc8\x67\xe4"
-			  "\xb9\x16\x6c\x76\x7b\x80\x4d\x46"
-			  "\xa5\x9b\x52\x16\xcd\xe7\xa4\xe9"
-			  "\x90\x40\xc5\xa4\x04\x33\x22\x5e"
-			  "\xe2\x82\xa1\xb0\xa0\x6c\x52\x3e"
-			  "\xaf\x45\x34\xd7\xf8\x3f\xa1\x15"
-			  "\x5b\x00\x47\x71\x8c\xbc\x54\x6a"
-			  "\x0d\x07\x2b\x04\xb3\x56\x4e\xea"
-			  "\x1b\x42\x22\x73\xf5\x48\x27\x1a"
-			  "\x0b\xb2\x31\x60\x53\xfa\x76\x99"
-			  "\x19\x55\xeb\xd6\x31\x59\x43\x4e"
-			  "\xce\xbb\x4e\x46\x6d\xae\x5a\x10"
-			  "\x73\xa6\x72\x76\x27\x09\x7a\x10"
-			  "\x49\xe6\x17\xd9\x1d\x36\x10\x94"
-			  "\xfa\x68\xf0\xff\x77\x98\x71\x30"
-			  "\x30\x5b\xea\xba\x2e\xda\x04\xdf"
-			  "\x99\x7b\x71\x4d\x6c\x6f\x2c\x29"
-			  "\xa6\xad\x5c\xb4\x02\x2b\x02\x70"
-			  "\x9b\xee\xad\x9d\x67\x89\x0c\xbb"
-			  "\x22\x39\x23\x36\xfe\xa1\x85\x1f"
-			  "\x38",
-		.rlen	= 281,
-	},
-};
-
-static const struct aead_testvec rfc7539esp_dec_tv_template[] = {
-	{
-		.key	= "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
-			  "\xf3\x33\x88\x86\x04\xf6\xb5\xf0"
-			  "\x47\x39\x17\xc1\x40\x2b\x80\x09"
-			  "\x9d\xca\x5c\xbc\x20\x70\x75\xc0"
-			  "\x00\x00\x00\x00",
-		.klen	= 36,
-		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
-		.assoc	= "\xf3\x33\x88\x86\x00\x00\x00\x00"
-			  "\x00\x00\x4e\x91\x01\x02\x03\x04"
-			  "\x05\x06\x07\x08",
-		.alen	= 20,
-		.input	= "\x64\xa0\x86\x15\x75\x86\x1a\xf4"
-			  "\x60\xf0\x62\xc7\x9b\xe6\x43\xbd"
-			  "\x5e\x80\x5c\xfd\x34\x5c\xf3\x89"
-			  "\xf1\x08\x67\x0a\xc7\x6c\x8c\xb2"
-			  "\x4c\x6c\xfc\x18\x75\x5d\x43\xee"
-			  "\xa0\x9e\xe9\x4e\x38\x2d\x26\xb0"
-			  "\xbd\xb7\xb7\x3c\x32\x1b\x01\x00"
-			  "\xd4\xf0\x3b\x7f\x35\x58\x94\xcf"
-			  "\x33\x2f\x83\x0e\x71\x0b\x97\xce"
-			  "\x98\xc8\xa8\x4a\xbd\x0b\x94\x81"
-			  "\x14\xad\x17\x6e\x00\x8d\x33\xbd"
-			  "\x60\xf9\x82\xb1\xff\x37\xc8\x55"
-			  "\x97\x97\xa0\x6e\xf4\xf0\xef\x61"
-			  "\xc1\x86\x32\x4e\x2b\x35\x06\x38"
-			  "\x36\x06\x90\x7b\x6a\x7c\x02\xb0"
-			  "\xf9\xf6\x15\x7b\x53\xc8\x67\xe4"
-			  "\xb9\x16\x6c\x76\x7b\x80\x4d\x46"
-			  "\xa5\x9b\x52\x16\xcd\xe7\xa4\xe9"
-			  "\x90\x40\xc5\xa4\x04\x33\x22\x5e"
-			  "\xe2\x82\xa1\xb0\xa0\x6c\x52\x3e"
-			  "\xaf\x45\x34\xd7\xf8\x3f\xa1\x15"
-			  "\x5b\x00\x47\x71\x8c\xbc\x54\x6a"
-			  "\x0d\x07\x2b\x04\xb3\x56\x4e\xea"
-			  "\x1b\x42\x22\x73\xf5\x48\x27\x1a"
-			  "\x0b\xb2\x31\x60\x53\xfa\x76\x99"
-			  "\x19\x55\xeb\xd6\x31\x59\x43\x4e"
-			  "\xce\xbb\x4e\x46\x6d\xae\x5a\x10"
-			  "\x73\xa6\x72\x76\x27\x09\x7a\x10"
-			  "\x49\xe6\x17\xd9\x1d\x36\x10\x94"
-			  "\xfa\x68\xf0\xff\x77\x98\x71\x30"
-			  "\x30\x5b\xea\xba\x2e\xda\x04\xdf"
-			  "\x99\x7b\x71\x4d\x6c\x6f\x2c\x29"
-			  "\xa6\xad\x5c\xb4\x02\x2b\x02\x70"
-			  "\x9b\xee\xad\x9d\x67\x89\x0c\xbb"
-			  "\x22\x39\x23\x36\xfe\xa1\x85\x1f"
-			  "\x38",
-		.ilen	= 281,
-		.result	= "\x49\x6e\x74\x65\x72\x6e\x65\x74"
-			  "\x2d\x44\x72\x61\x66\x74\x73\x20"
-			  "\x61\x72\x65\x20\x64\x72\x61\x66"
-			  "\x74\x20\x64\x6f\x63\x75\x6d\x65"
-			  "\x6e\x74\x73\x20\x76\x61\x6c\x69"
-			  "\x64\x20\x66\x6f\x72\x20\x61\x20"
-			  "\x6d\x61\x78\x69\x6d\x75\x6d\x20"
-			  "\x6f\x66\x20\x73\x69\x78\x20\x6d"
-			  "\x6f\x6e\x74\x68\x73\x20\x61\x6e"
-			  "\x64\x20\x6d\x61\x79\x20\x62\x65"
-			  "\x20\x75\x70\x64\x61\x74\x65\x64"
-			  "\x2c\x20\x72\x65\x70\x6c\x61\x63"
-			  "\x65\x64\x2c\x20\x6f\x72\x20\x6f"
-			  "\x62\x73\x6f\x6c\x65\x74\x65\x64"
-			  "\x20\x62\x79\x20\x6f\x74\x68\x65"
-			  "\x72\x20\x64\x6f\x63\x75\x6d\x65"
-			  "\x6e\x74\x73\x20\x61\x74\x20\x61"
-			  "\x6e\x79\x20\x74\x69\x6d\x65\x2e"
-			  "\x20\x49\x74\x20\x69\x73\x20\x69"
-			  "\x6e\x61\x70\x70\x72\x6f\x70\x72"
-			  "\x69\x61\x74\x65\x20\x74\x6f\x20"
-			  "\x75\x73\x65\x20\x49\x6e\x74\x65"
-			  "\x72\x6e\x65\x74\x2d\x44\x72\x61"
-			  "\x66\x74\x73\x20\x61\x73\x20\x72"
-			  "\x65\x66\x65\x72\x65\x6e\x63\x65"
-			  "\x20\x6d\x61\x74\x65\x72\x69\x61"
-			  "\x6c\x20\x6f\x72\x20\x74\x6f\x20"
-			  "\x63\x69\x74\x65\x20\x74\x68\x65"
-			  "\x6d\x20\x6f\x74\x68\x65\x72\x20"
-			  "\x74\x68\x61\x6e\x20\x61\x73\x20"
-			  "\x2f\xe2\x80\x9c\x77\x6f\x72\x6b"
-			  "\x20\x69\x6e\x20\x70\x72\x6f\x67"
-			  "\x72\x65\x73\x73\x2e\x2f\xe2\x80"
-			  "\x9d",
-		.rlen	= 265,
-	},
-};
-
-/*
- * All key wrapping test vectors taken from
- * http://csrc.nist.gov/groups/STM/cavp/documents/mac/kwtestvectors.zip
- *
- * Note: as documented in keywrap.c, the ivout for encryption is the first
- * semiblock of the ciphertext from the test vector. For decryption, iv is
- * the first semiblock of the ciphertext.
- */
-static const struct cipher_testvec aes_kw_enc_tv_template[] = {
-	{
-		.key	= "\x75\x75\xda\x3a\x93\x60\x7c\xc2"
-			  "\xbf\xd8\xce\xc7\xaa\xdf\xd9\xa6",
-		.klen	= 16,
-		.input	= "\x42\x13\x6d\x3c\x38\x4a\x3e\xea"
-			  "\xc9\x5a\x06\x6f\xd2\x8f\xed\x3f",
-		.ilen	= 16,
-		.result	= "\xf6\x85\x94\x81\x6f\x64\xca\xa3"
-			  "\xf5\x6f\xab\xea\x25\x48\xf5\xfb",
-		.rlen	= 16,
-		.iv_out	= "\x03\x1f\x6b\xd7\xe6\x1e\x64\x3d",
-	},
-};
-
-static const struct cipher_testvec aes_kw_dec_tv_template[] = {
-	{
-		.key	= "\x80\xaa\x99\x73\x27\xa4\x80\x6b"
-			  "\x6a\x7a\x41\xa5\x2b\x86\xc3\x71"
-			  "\x03\x86\xf9\x32\x78\x6e\xf7\x96"
-			  "\x76\xfa\xfb\x90\xb8\x26\x3c\x5f",
-		.klen	= 32,
-		.input	= "\xd3\x3d\x3d\x97\x7b\xf0\xa9\x15"
-			  "\x59\xf9\x9c\x8a\xcd\x29\x3d\x43",
-		.ilen	= 16,
-		.result	= "\x0a\x25\x6b\xa7\x5c\xfa\x03\xaa"
-			  "\xa0\x2b\xa9\x42\x03\xf1\x5b\xaa",
-		.rlen	= 16,
-		.iv	= "\x42\x3c\x96\x0d\x8a\x2a\xc4\xc1",
-	},
-};
-
-/*
- * ANSI X9.31 Continuous Pseudo-Random Number Generator (AES mode)
- * test vectors, taken from Appendix B.2.9 and B.2.10:
- *     http://csrc.nist.gov/groups/STM/cavp/documents/rng/RNGVS.pdf
- * Only AES-128 is supported at this time.
- */
-static const struct cprng_testvec ansi_cprng_aes_tv_template[] = {
-	{
-		.key	= "\xf3\xb1\x66\x6d\x13\x60\x72\x42"
-			  "\xed\x06\x1c\xab\xb8\xd4\x62\x02",
-		.klen	= 16,
-		.dt	= "\xe6\xb3\xbe\x78\x2a\x23\xfa\x62"
-			  "\xd7\x1d\x4a\xfb\xb0\xe9\x22\xf9",
-		.dtlen	= 16,
-		.v	= "\x80\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.vlen	= 16,
-		.result	= "\x59\x53\x1e\xd1\x3b\xb0\xc0\x55"
-			  "\x84\x79\x66\x85\xc1\x2f\x76\x41",
-		.rlen	= 16,
-		.loops	= 1,
-	}, {
-		.key	= "\xf3\xb1\x66\x6d\x13\x60\x72\x42"
-			  "\xed\x06\x1c\xab\xb8\xd4\x62\x02",
-		.klen	= 16,
-		.dt	= "\xe6\xb3\xbe\x78\x2a\x23\xfa\x62"
-			  "\xd7\x1d\x4a\xfb\xb0\xe9\x22\xfa",
-		.dtlen	= 16,
-		.v	= "\xc0\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.vlen	= 16,
-		.result	= "\x7c\x22\x2c\xf4\xca\x8f\xa2\x4c"
-			  "\x1c\x9c\xb6\x41\xa9\xf3\x22\x0d",
-		.rlen	= 16,
-		.loops	= 1,
-	}, {
-		.key	= "\xf3\xb1\x66\x6d\x13\x60\x72\x42"
-			  "\xed\x06\x1c\xab\xb8\xd4\x62\x02",
-		.klen	= 16,
-		.dt	= "\xe6\xb3\xbe\x78\x2a\x23\xfa\x62"
-			  "\xd7\x1d\x4a\xfb\xb0\xe9\x22\xfb",
-		.dtlen	= 16,
-		.v	= "\xe0\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.vlen	= 16,
-		.result	= "\x8a\xaa\x00\x39\x66\x67\x5b\xe5"
-			  "\x29\x14\x28\x81\xa9\x4d\x4e\xc7",
-		.rlen	= 16,
-		.loops	= 1,
-	}, {
-		.key	= "\xf3\xb1\x66\x6d\x13\x60\x72\x42"
-			  "\xed\x06\x1c\xab\xb8\xd4\x62\x02",
-		.klen	= 16,
-		.dt	= "\xe6\xb3\xbe\x78\x2a\x23\xfa\x62"
-			  "\xd7\x1d\x4a\xfb\xb0\xe9\x22\xfc",
-		.dtlen	= 16,
-		.v	= "\xf0\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.vlen	= 16,
-		.result	= "\x88\xdd\xa4\x56\x30\x24\x23\xe5"
-			  "\xf6\x9d\xa5\x7e\x7b\x95\xc7\x3a",
-		.rlen	= 16,
-		.loops	= 1,
-	}, {
-		.key	= "\xf3\xb1\x66\x6d\x13\x60\x72\x42"
-			  "\xed\x06\x1c\xab\xb8\xd4\x62\x02",
-		.klen	= 16,
-		.dt	= "\xe6\xb3\xbe\x78\x2a\x23\xfa\x62"
-			  "\xd7\x1d\x4a\xfb\xb0\xe9\x22\xfd",
-		.dtlen	= 16,
-		.v	= "\xf8\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.vlen	= 16,
-		.result	= "\x05\x25\x92\x46\x61\x79\xd2\xcb"
-			  "\x78\xc4\x0b\x14\x0a\x5a\x9a\xc8",
-		.rlen	= 16,
-		.loops	= 1,
-	}, {	/* Monte Carlo Test */
-		.key	= "\x9f\x5b\x51\x20\x0b\xf3\x34\xb5"
-			  "\xd8\x2b\xe8\xc3\x72\x55\xc8\x48",
-		.klen	= 16,
-		.dt	= "\x63\x76\xbb\xe5\x29\x02\xba\x3b"
-			  "\x67\xc9\x25\xfa\x70\x1f\x11\xac",
-		.dtlen	= 16,
-		.v	= "\x57\x2c\x8e\x76\x87\x26\x47\x97"
-			  "\x7e\x74\xfb\xdd\xc4\x95\x01\xd1",
-		.vlen	= 16,
-		.result	= "\x48\xe9\xbd\x0d\x06\xee\x18\xfb"
-			  "\xe4\x57\x90\xd5\xc3\xfc\x9b\x73",
-		.rlen	= 16,
-		.loops	= 10000,
-	},
-};
-
-/*
- * SP800-90A DRBG Test vectors from
- * http://csrc.nist.gov/groups/STM/cavp/documents/drbg/drbgtestvectors.zip
- *
- * Test vectors for DRBG with prediction resistance. All types of DRBGs
- * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and
- * w/o personalization string, w/ and w/o additional input string).
- */
-static const struct drbg_testvec drbg_pr_sha256_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\x72\x88\x4c\xcd\x6c\x85\x57\x70\xf7\x0b\x8b\x86"
-			"\xc1\xeb\xd2\x4e\x36\x14\xab\x18\xc4\x9c\xc9\xcf"
-			"\x1a\xe8\xf7\x7b\x02\x49\x73\xd7\xf1\x42\x7d\xc6"
-			"\x3f\x29\x2d\xec\xd3\x66\x51\x3f\x1d\x8d\x5b\x4e",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\x38\x9c\x91\xfa\xc2\xa3\x46\x89\x56\x08\x3f\x62"
-			"\x73\xd5\x22\xa9\x29\x63\x3a\x1d\xe5\x5d\x5e\x4f"
-			"\x67\xb0\x67\x7a\x5e\x9e\x0c\x62",
-		.entprb = (unsigned char *)
-			"\xb2\x8f\x36\xb2\xf6\x8d\x39\x13\xfa\x6c\x66\xcf"
-			"\x62\x8a\x7e\x8c\x12\x33\x71\x9c\x69\xe4\xa5\xf0"
-			"\x8c\xee\xeb\x9c\xf5\x31\x98\x31",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x52\x7b\xa3\xad\x71\x77\xa4\x49\x42\x04\x61\xc7"
-			"\xf0\xaf\xa5\xfd\xd3\xb3\x0d\x6a\x61\xba\x35\x49"
-			"\xbb\xaa\xaf\xe4\x25\x7d\xb5\x48\xaf\x5c\x18\x3d"
-			"\x33\x8d\x9d\x45\xdf\x98\xd5\x94\xa8\xda\x92\xfe"
-			"\xc4\x3c\x94\x2a\xcf\x7f\x7b\xf2\xeb\x28\xa9\xf1"
-			"\xe0\x86\x30\xa8\xfe\xf2\x48\x90\x91\x0c\x75\xb5"
-			"\x3c\x00\xf0\x4d\x09\x4f\x40\xa7\xa2\x8c\x52\xdf"
-			"\x52\xef\x17\xbf\x3d\xd1\xa2\x31\xb4\xb8\xdc\xe6"
-			"\x5b\x0d\x1f\x78\x36\xb4\xe6\x4b\xa7\x11\x25\xd5"
-			"\x94\xc6\x97\x36\xab\xf0\xe5\x31\x28\x6a\xbb\xce"
-			"\x30\x81\xa6\x8f\x27\x14\xf8\x1c",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x5d\xf2\x14\xbc\xf6\xb5\x4e\x0b\xf0\x0d\x6f\x2d"
-			"\xe2\x01\x66\x7b\xd0\xa4\x73\xa4\x21\xdd\xb0\xc0"
-			"\x51\x79\x09\xf4\xea\xa9\x08\xfa\xa6\x67\xe0\xe1"
-			"\xd1\x88\xa8\xad\xee\x69\x74\xb3\x55\x06\x9b\xf6",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xef\x48\x06\xa2\xc2\x45\xf1\x44\xfa\x34\x2c\xeb"
-			"\x8d\x78\x3c\x09\x8f\x34\x72\x20\xf2\xe7\xfd\x13"
-			"\x76\x0a\xf6\xdc\x3c\xf5\xc0\x15",
-		.entprb = (unsigned char *)
-			"\x4b\xbe\xe5\x24\xed\x6a\x2d\x0c\xdb\x73\x5e\x09"
-			"\xf9\xad\x67\x7c\x51\x47\x8b\x6b\x30\x2a\xc6\xde"
-			"\x76\xaa\x55\x04\x8b\x0a\x72\x95",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x3b\x14\x71\x99\xa1\xda\xa0\x42\xe6\xc8\x85\x32"
-			"\x70\x20\x32\x53\x9a\xbe\xd1\x1e\x15\xef\xfb\x4c"
-			"\x25\x6e\x19\x3a\xf0\xb9\xcb\xde\xf0\x3b\xc6\x18"
-			"\x4d\x85\x5a\x9b\xf1\xe3\xc2\x23\x03\x93\x08\xdb"
-			"\xa7\x07\x4b\x33\x78\x40\x4d\xeb\x24\xf5\x6e\x81"
-			"\x4a\x1b\x6e\xa3\x94\x52\x43\xb0\xaf\x2e\x21\xf4"
-			"\x42\x46\x8e\x90\xed\x34\x21\x75\xea\xda\x67\xb6"
-			"\xe4\xf6\xff\xc6\x31\x6c\x9a\x5a\xdb\xb3\x97\x13"
-			"\x09\xd3\x20\x98\x33\x2d\x6d\xd7\xb5\x6a\xa8\xa9"
-			"\x9a\x5b\xd6\x87\x52\xa1\x89\x2b\x4b\x9c\x64\x60"
-			"\x50\x47\xa3\x63\x81\x16\xaf\x19",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\xbe\x13\xdb\x2a\xe9\xa8\xfe\x09\x97\xe1\xce\x5d"
-			"\xe8\xbb\xc0\x7c\x4f\xcb\x62\x19\x3f\x0f\xd2\xad"
-			"\xa9\xd0\x1d\x59\x02\xc4\xff\x70",
-		.addtlb = (unsigned char *)
-			"\x6f\x96\x13\xe2\xa7\xf5\x6c\xfe\xdf\x66\xe3\x31"
-			"\x63\x76\xbf\x20\x27\x06\x49\xf1\xf3\x01\x77\x41"
-			"\x9f\xeb\xe4\x38\xfe\x67\x00\xcd",
-		.addtllen = 32,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\xc6\x1c\xaf\x83\xa2\x56\x38\xf9\xb0\xbc\xd9\x85"
-			"\xf5\x2e\xc4\x46\x9c\xe1\xb9\x40\x98\x70\x10\x72"
-			"\xd7\x7d\x15\x85\xa1\x83\x5a\x97\xdf\xc8\xa8\xe8"
-			"\x03\x4c\xcb\x70\x35\x8b\x90\x94\x46\x8a\x6e\xa1",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xc9\x05\xa4\xcf\x28\x80\x4b\x93\x0f\x8b\xc6\xf9"
-			"\x09\x41\x58\x74\xe9\xec\x28\xc7\x53\x0a\x73\x60"
-			"\xba\x0a\xde\x57\x5b\x4b\x9f\x29",
-		.entprb = (unsigned char *)
-			"\x4f\x31\xd2\xeb\xac\xfa\xa8\xe2\x01\x7d\xf3\xbd"
-			"\x42\xbd\x20\xa0\x30\x65\x74\xd5\x5d\xd2\xad\xa4"
-			"\xa9\xeb\x1f\x4d\xf6\xfd\xb8\x26",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\xf6\x13\x05\xcb\x83\x60\x16\x42\x49\x1d\xc6\x25"
-			"\x3b\x8c\x31\xa3\xbe\x8b\xbd\x1c\xe2\xec\x1d\xde"
-			"\xbb\xbf\xa1\xac\xa8\x9f\x50\xce\x69\xce\xef\xd5"
-			"\xd6\xf2\xef\x6a\xf7\x81\x38\xdf\xbc\xa7\x5a\xb9"
-			"\xb2\x42\x65\xab\xe4\x86\x8d\x2d\x9d\x59\x99\x2c"
-			"\x5a\x0d\x71\x55\x98\xa4\x45\xc2\x8d\xdb\x05\x5e"
-			"\x50\x21\xf7\xcd\xe8\x98\x43\xce\x57\x74\x63\x4c"
-			"\xf3\xb1\xa5\x14\x1e\x9e\x01\xeb\x54\xd9\x56\xae"
-			"\xbd\xb6\x6f\x1a\x47\x6b\x3b\x44\xe4\xa2\xe9\x3c"
-			"\x6c\x83\x12\x30\xb8\x78\x7f\x8e\x54\x82\xd4\xfe"
-			"\x90\x35\x0d\x4c\x4d\x85\xe7\x13",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\xa5\xbf\xac\x4f\x71\xa1\xbb\x67\x94\xc6\x50\xc7"
-			"\x2a\x45\x9e\x10\xa8\xed\xf7\x52\x4f\xfe\x21\x90"
-			"\xa4\x1b\xe1\xe2\x53\xcc\x61\x47",
-		.perslen = 32,
-	}, {
-		.entropy = (unsigned char *)
-			"\xb6\xc1\x8d\xdf\x99\x54\xbe\x95\x10\x48\xd9\xf6"
-			"\xd7\x48\xa8\x73\x2d\x74\xde\x1e\xde\x57\x7e\xf4"
-			"\x7b\x7b\x64\xef\x88\x7a\xa8\x10\x4b\xe1\xc1\x87"
-			"\xbb\x0b\xe1\x39\x39\x50\xaf\x68\x9c\xa2\xbf\x5e",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xdc\x81\x0a\x01\x58\xa7\x2e\xce\xee\x48\x8c\x7c"
-			"\x77\x9e\x3c\xf1\x17\x24\x7a\xbb\xab\x9f\xca\x12"
-			"\x19\xaf\x97\x2d\x5f\xf9\xff\xfc",
-		.entprb = (unsigned char *)
-			"\xaf\xfc\x4f\x98\x8b\x93\x95\xc1\xb5\x8b\x7f\x73"
-			"\x6d\xa6\xbe\x6d\x33\xeb\x2c\x82\xb1\xaf\xc1\xb6"
-			"\xb6\x05\xe2\x44\xaa\xfd\xe7\xdb",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x51\x79\xde\x1c\x0f\x58\xf3\xf4\xc9\x57\x2e\x31"
-			"\xa7\x09\xa1\x53\x64\x63\xa2\xc5\x1d\x84\x88\x65"
-			"\x01\x1b\xc6\x16\x3c\x49\x5b\x42\x8e\x53\xf5\x18"
-			"\xad\x94\x12\x0d\x4f\x55\xcc\x45\x5c\x98\x0f\x42"
-			"\x28\x2f\x47\x11\xf9\xc4\x01\x97\x6b\xa0\x94\x50"
-			"\xa9\xd1\x5e\x06\x54\x3f\xdf\xbb\xc4\x98\xee\x8b"
-			"\xba\xa9\xfa\x49\xee\x1d\xdc\xfb\x50\xf6\x51\x9f"
-			"\x6c\x4a\x9a\x6f\x63\xa2\x7d\xad\xaf\x3a\x24\xa0"
-			"\xd9\x9f\x07\xeb\x15\xee\x26\xe0\xd5\x63\x39\xda"
-			"\x3c\x59\xd6\x33\x6c\x02\xe8\x05\x71\x46\x68\x44"
-			"\x63\x4a\x68\x72\xe9\xf5\x55\xfe",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x15\x20\x2f\xf6\x98\x28\x63\xa2\xc4\x4e\xbb\x6c"
-			"\xb2\x25\x92\x61\x79\xc9\x22\xc4\x61\x54\x96\xff"
-			"\x4a\x85\xca\x80\xfe\x0d\x1c\xd0",
-		.addtlb = (unsigned char *)
-			"\xde\x29\x8e\x03\x42\x61\xa3\x28\x5e\xc8\x80\xc2"
-			"\x6d\xbf\xad\x13\xe1\x8d\x2a\xc7\xe8\xc7\x18\x89"
-			"\x42\x58\x9e\xd6\xcc\xad\x7b\x1e",
-		.addtllen = 32,
-		.pers = (unsigned char *)
-			"\x84\xc3\x73\x9e\xce\xb3\xbc\x89\xf7\x62\xb3\xe1"
-			"\xd7\x48\x45\x8a\xa9\xcc\xe9\xed\xd5\x81\x84\x52"
-			"\x82\x4c\xdc\x19\xb8\xf8\x92\x5c",
-		.perslen = 32,
-	},
-};
-
-static const struct drbg_testvec drbg_pr_hmac_sha256_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\x99\x69\xe5\x4b\x47\x03\xff\x31\x78\x5b\x87\x9a"
-			"\x7e\x5c\x0e\xae\x0d\x3e\x30\x95\x59\xe9\xfe\x96"
-			"\xb0\x67\x6d\x49\xd5\x91\xea\x4d\x07\xd2\x0d\x46"
-			"\xd0\x64\x75\x7d\x30\x23\xca\xc2\x37\x61\x27\xab",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xc6\x0f\x29\x99\x10\x0f\x73\x8c\x10\xf7\x47\x92"
-			"\x67\x6a\x3f\xc4\xa2\x62\xd1\x37\x21\x79\x80\x46"
-			"\xe2\x9a\x29\x51\x81\x56\x9f\x54",
-		.entprb = (unsigned char *)
-			"\xc1\x1d\x45\x24\xc9\x07\x1b\xd3\x09\x60\x15\xfc"
-			"\xf7\xbc\x24\xa6\x07\xf2\x2f\xa0\x65\xc9\x37\x65"
-			"\x8a\x2a\x77\xa8\x69\x90\x89\xf4",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\xab\xc0\x15\x85\x60\x94\x80\x3a\x93\x8d\xff\xd2"
-			"\x0d\xa9\x48\x43\x87\x0e\xf9\x35\xb8\x2c\xfe\xc1"
-			"\x77\x06\xb8\xf5\x51\xb8\x38\x50\x44\x23\x5d\xd4"
-			"\x4b\x59\x9f\x94\xb3\x9b\xe7\x8d\xd4\x76\xe0\xcf"
-			"\x11\x30\x9c\x99\x5a\x73\x34\xe0\xa7\x8b\x37\xbc"
-			"\x95\x86\x23\x50\x86\xfa\x3b\x63\x7b\xa9\x1c\xf8"
-			"\xfb\x65\xef\xa2\x2a\x58\x9c\x13\x75\x31\xaa\x7b"
-			"\x2d\x4e\x26\x07\xaa\xc2\x72\x92\xb0\x1c\x69\x8e"
-			"\x6e\x01\xae\x67\x9e\xb8\x7c\x01\xa8\x9c\x74\x22"
-			"\xd4\x37\x2d\x6d\x75\x4a\xba\xbb\x4b\xf8\x96\xfc"
-			"\xb1\xcd\x09\xd6\x92\xd0\x28\x3f",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\xb9\x1f\xe9\xef\xdd\x9b\x7d\x20\xb6\xec\xe0\x2f"
-			"\xdb\x76\x24\xce\x41\xc8\x3a\x4a\x12\x7f\x3e\x2f"
-			"\xae\x05\x99\xea\xb5\x06\x71\x0d\x0c\x4c\xb4\x05"
-			"\x26\xc6\xbd\xf5\x7f\x2a\x3d\xf2\xb5\x49\x7b\xda",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xef\x67\x50\x9c\xa7\x7d\xdf\xb7\x2d\x81\x01\xa4"
-			"\x62\x81\x6a\x69\x5b\xb3\x37\x45\xa7\x34\x8e\x26"
-			"\x46\xd9\x26\xa2\x19\xd4\x94\x43",
-		.entprb = (unsigned char *)
-			"\x97\x75\x53\x53\xba\xb4\xa6\xb2\x91\x60\x71\x79"
-			"\xd1\x6b\x4a\x24\x9a\x34\x66\xcc\x33\xab\x07\x98"
-			"\x51\x78\x72\xb2\x79\xfd\x2c\xff",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x9c\xdc\x63\x8a\x19\x23\x22\x66\x0c\xc5\xb9\xd7"
-			"\xfb\x2a\xb0\x31\xe3\x8a\x36\xa8\x5a\xa8\x14\xda"
-			"\x1e\xa9\xcc\xfe\xb8\x26\x44\x83\x9f\xf6\xff\xaa"
-			"\xc8\x98\xb8\x30\x35\x3b\x3d\x36\xd2\x49\xd4\x40"
-			"\x62\x0a\x65\x10\x76\x55\xef\xc0\x95\x9c\xa7\xda"
-			"\x3f\xcf\xb7\x7b\xc6\xe1\x28\x52\xfc\x0c\xe2\x37"
-			"\x0d\x83\xa7\x51\x4b\x31\x47\x3c\xe1\x3c\xae\x70"
-			"\x01\xc8\xa3\xd3\xc2\xac\x77\x9c\xd1\x68\x77\x9b"
-			"\x58\x27\x3b\xa5\x0f\xc2\x7a\x8b\x04\x65\x62\xd5"
-			"\xe8\xd6\xfe\x2a\xaf\xd3\xd3\xfe\xbd\x18\xfb\xcd"
-			"\xcd\x66\xb5\x01\x69\x66\xa0\x3c",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x17\xc1\x56\xcb\xcc\x50\xd6\x03\x7d\x45\x76\xa3"
-			"\x75\x76\xc1\x4a\x66\x1b\x2e\xdf\xb0\x2e\x7d\x56"
-			"\x6d\x99\x3b\xc6\x58\xda\x03\xf6",
-		.addtlb = (unsigned char *)
-			"\x7c\x7b\x4a\x4b\x32\x5e\x6f\x67\x34\xf5\x21\x4c"
-			"\xf9\x96\xf9\xbf\x1c\x8c\x81\xd3\x9b\x60\x6a\x44"
-			"\xc6\x03\xa2\xfb\x13\x20\x19\xb7",
-		.addtllen = 32,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x13\x54\x96\xfc\x1b\x7d\x28\xf3\x18\xc9\xa7\x89"
-			"\xb6\xb3\xc8\x72\xac\x00\xd4\x59\x36\x25\x05\xaf"
-			"\xa5\xdb\x96\xcb\x3c\x58\x46\x87\xa5\xaa\xbf\x20"
-			"\x3b\xfe\x23\x0e\xd1\xc7\x41\x0f\x3f\xc9\xb3\x67",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xe2\xbd\xb7\x48\x08\x06\xf3\xe1\x93\x3c\xac\x79"
-			"\xa7\x2b\x11\xda\xe3\x2e\xe1\x91\xa5\x02\x19\x57"
-			"\x20\x28\xad\xf2\x60\xd7\xcd\x45",
-		.entprb = (unsigned char *)
-			"\x8b\xd4\x69\xfc\xff\x59\x95\x95\xc6\x51\xde\x71"
-			"\x68\x5f\xfc\xf9\x4a\xab\xec\x5a\xcb\xbe\xd3\x66"
-			"\x1f\xfa\x74\xd3\xac\xa6\x74\x60",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x1f\x9e\xaf\xe4\xd2\x46\xb7\x47\x41\x4c\x65\x99"
-			"\x01\xe9\x3b\xbb\x83\x0c\x0a\xb0\xc1\x3a\xe2\xb3"
-			"\x31\x4e\xeb\x93\x73\xee\x0b\x26\xc2\x63\xa5\x75"
-			"\x45\x99\xd4\x5c\x9f\xa1\xd4\x45\x87\x6b\x20\x61"
-			"\x40\xea\x78\xa5\x32\xdf\x9e\x66\x17\xaf\xb1\x88"
-			"\x9e\x2e\x23\xdd\xc1\xda\x13\x97\x88\xa5\xb6\x5e"
-			"\x90\x14\x4e\xef\x13\xab\x5c\xd9\x2c\x97\x9e\x7c"
-			"\xd7\xf8\xce\xea\x81\xf5\xcd\x71\x15\x49\x44\xce"
-			"\x83\xb6\x05\xfb\x7d\x30\xb5\x57\x2c\x31\x4f\xfc"
-			"\xfe\x80\xb6\xc0\x13\x0c\x5b\x9b\x2e\x8f\x3d\xfc"
-			"\xc2\xa3\x0c\x11\x1b\x80\x5f\xf3",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\x64\xb6\xfc\x60\xbc\x61\x76\x23\x6d\x3f\x4a\x0f"
-			"\xe1\xb4\xd5\x20\x9e\x70\xdd\x03\x53\x6d\xbf\xce"
-			"\xcd\x56\x80\xbc\xb8\x15\xc8\xaa",
-		.perslen = 32,
-	}, {
-		.entropy = (unsigned char *)
-			"\xc7\xcc\xbc\x67\x7e\x21\x66\x1e\x27\x2b\x63\xdd"
-			"\x3a\x78\xdc\xdf\x66\x6d\x3f\x24\xae\xcf\x37\x01"
-			"\xa9\x0d\x89\x8a\xa7\xdc\x81\x58\xae\xb2\x10\x15"
-			"\x7e\x18\x44\x6d\x13\xea\xdf\x37\x85\xfe\x81\xfb",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\x7b\xa1\x91\x5b\x3c\x04\xc4\x1b\x1d\x19\x2f\x1a"
-			"\x18\x81\x60\x3c\x6c\x62\x91\xb7\xe9\xf5\xcb\x96"
-			"\xbb\x81\x6a\xcc\xb5\xae\x55\xb6",
-		.entprb = (unsigned char *)
-			"\x99\x2c\xc7\x78\x7e\x3b\x88\x12\xef\xbe\xd3\xd2"
-			"\x7d\x2a\xa5\x86\xda\x8d\x58\x73\x4a\x0a\xb2\x2e"
-			"\xbb\x4c\x7e\xe3\x9a\xb6\x81\xc1",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x95\x6f\x95\xfc\x3b\xb7\xfe\x3e\xd0\x4e\x1a\x14"
-			"\x6c\x34\x7f\x7b\x1d\x0d\x63\x5e\x48\x9c\x69\xe6"
-			"\x46\x07\xd2\x87\xf3\x86\x52\x3d\x98\x27\x5e\xd7"
-			"\x54\xe7\x75\x50\x4f\xfb\x4d\xfd\xac\x2f\x4b\x77"
-			"\xcf\x9e\x8e\xcc\x16\xa2\x24\xcd\x53\xde\x3e\xc5"
-			"\x55\x5d\xd5\x26\x3f\x89\xdf\xca\x8b\x4e\x1e\xb6"
-			"\x88\x78\x63\x5c\xa2\x63\x98\x4e\x6f\x25\x59\xb1"
-			"\x5f\x2b\x23\xb0\x4b\xa5\x18\x5d\xc2\x15\x74\x40"
-			"\x59\x4c\xb4\x1e\xcf\x9a\x36\xfd\x43\xe2\x03\xb8"
-			"\x59\x91\x30\x89\x2a\xc8\x5a\x43\x23\x7c\x73\x72"
-			"\xda\x3f\xad\x2b\xba\x00\x6b\xd1",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x18\xe8\x17\xff\xef\x39\xc7\x41\x5c\x73\x03\x03"
-			"\xf6\x3d\xe8\x5f\xc8\xab\xe4\xab\x0f\xad\xe8\xd6"
-			"\x86\x88\x55\x28\xc1\x69\xdd\x76",
-		.addtlb = (unsigned char *)
-			"\xac\x07\xfc\xbe\x87\x0e\xd3\xea\x1f\x7e\xb8\xe7"
-			"\x9d\xec\xe8\xe7\xbc\xf3\x18\x25\x77\x35\x4a\xaa"
-			"\x00\x99\x2a\xdd\x0a\x00\x50\x82",
-		.addtllen = 32,
-		.pers = (unsigned char *)
-			"\xbc\x55\xab\x3c\xf6\x52\xb0\x11\x3d\x7b\x90\xb8"
-			"\x24\xc9\x26\x4e\x5a\x1e\x77\x0d\x3d\x58\x4a\xda"
-			"\xd1\x81\xe9\xf8\xeb\x30\x8f\x6f",
-		.perslen = 32,
-	},
-};
-
-static const struct drbg_testvec drbg_pr_ctr_aes128_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\xd1\x44\xc6\x61\x81\x6d\xca\x9d\x15\x28\x8a\x42"
-			"\x94\xd7\x28\x9c\x43\x77\x19\x29\x1a\x6d\xc3\xa2",
-		.entropylen = 24,
-		.entpra = (unsigned char *)
-			"\x96\xd8\x9e\x45\x32\xc9\xd2\x08\x7a\x6d\x97\x15"
-			"\xb4\xec\x80\xb1",
-		.entprb = (unsigned char *)
-			"\x8b\xb6\x72\xb5\x24\x0b\x98\x65\x95\x95\xe9\xc9"
-			"\x28\x07\xeb\xc2",
-		.entprlen = 16,
-		.expected = (unsigned char *)
-			"\x70\x19\xd0\x4c\x45\x78\xd6\x68\xa9\x9a\xaa\xfe"
-			"\xc1\xdf\x27\x9a\x1c\x0d\x0d\xf7\x24\x75\x46\xcc"
-			"\x77\x6b\xdf\x89\xc6\x94\xdc\x74\x50\x10\x70\x18"
-			"\x9b\xdc\x96\xb4\x89\x23\x40\x1a\xce\x09\x87\xce"
-			"\xd2\xf3\xd5\xe4\x51\x67\x74\x11\x5a\xcc\x8b\x3b"
-			"\x8a\xf1\x23\xa8",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x8e\x83\xe0\xeb\x37\xea\x3e\x53\x5e\x17\x6e\x77"
-			"\xbd\xb1\x53\x90\xfc\xdc\xc1\x3c\x9a\x88\x22\x94",
-		.entropylen = 24,
-		.entpra = (unsigned char *)
-			"\x6a\x85\xe7\x37\xc8\xf1\x04\x31\x98\x4f\xc8\x73"
-			"\x67\xd1\x08\xf8",
-		.entprb = (unsigned char *)
-			"\xd7\xa4\x68\xe2\x12\x74\xc3\xd9\xf1\xb7\x05\xbc"
-			"\xd4\xba\x04\x58",
-		.entprlen = 16,
-		.expected = (unsigned char *)
-			"\x78\xd6\xa6\x70\xff\xd1\x82\xf5\xa2\x88\x7f\x6d"
-			"\x3d\x8c\x39\xb1\xa8\xcb\x2c\x91\xab\x14\x7e\xbc"
-			"\x95\x45\x9f\x24\xb8\x20\xac\x21\x23\xdb\x72\xd7"
-			"\x12\x8d\x48\x95\xf3\x19\x0c\x43\xc6\x19\x45\xfc"
-			"\x8b\xac\x40\x29\x73\x00\x03\x45\x5e\x12\xff\x0c"
-			"\xc1\x02\x41\x82",
-		.expectedlen = 64,
-		.addtla = (unsigned char *)
-			"\xa2\xd9\x38\xcf\x8b\x29\x67\x5b\x65\x62\x6f\xe8"
-			"\xeb\xb3\x01\x76",
-		.addtlb = (unsigned char *)
-			"\x59\x63\x1e\x81\x8a\x14\xa8\xbb\xa1\xb8\x41\x25"
-			"\xd0\x7f\xcc\x43",
-		.addtllen = 16,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x04\xd9\x49\xa6\xdc\xe8\x6e\xbb\xf1\x08\x77\x2b"
-			"\x9e\x08\xca\x92\x65\x16\xda\x99\xa2\x59\xf3\xe8",
-		.entropylen = 24,
-		.entpra = (unsigned char *)
-			"\x38\x7e\x3f\x6b\x51\x70\x7b\x20\xec\x53\xd0\x66"
-			"\xc3\x0f\xe3\xb0",
-		.entprb = (unsigned char *)
-			"\xe0\x86\xa6\xaa\x5f\x72\x2f\xad\xf7\xef\x06\xb8"
-			"\xd6\x9c\x9d\xe8",
-		.entprlen = 16,
-		.expected = (unsigned char *)
-			"\xc9\x0a\xaf\x85\x89\x71\x44\x66\x4f\x25\x0b\x2b"
-			"\xde\xd8\xfa\xff\x52\x5a\x1b\x32\x5e\x41\x7a\x10"
-			"\x1f\xef\x1e\x62\x23\xe9\x20\x30\xc9\x0d\xad\x69"
-			"\xb4\x9c\x5b\xf4\x87\x42\xd5\xae\x5e\x5e\x43\xcc"
-			"\xd9\xfd\x0b\x93\x4a\xe3\xd4\x06\x37\x36\x0f\x3f"
-			"\x72\x82\x0c\xcf",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\xbf\xa4\x9a\x8f\x7b\xd8\xb1\x7a\x9d\xfa\x45\xed"
-			"\x21\x52\xb3\xad",
-		.perslen = 16,
-	}, {
-		.entropy = (unsigned char *)
-			"\x92\x89\x8f\x31\xfa\x1c\xff\x6d\x18\x2f\x26\x06"
-			"\x43\xdf\xf8\x18\xc2\xa4\xd9\x72\xc3\xb9\xb6\x97",
-		.entropylen = 24,
-		.entpra = (unsigned char *)
-			"\x20\x72\x8a\x06\xf8\x6f\x8d\xd4\x41\xe2\x72\xb7"
-			"\xc4\x2c\xe8\x10",
-		.entprb = (unsigned char *)
-			"\x3d\xb0\xf0\x94\xf3\x05\x50\x33\x17\x86\x3e\x22"
-			"\x08\xf7\xa5\x01",
-		.entprlen = 16,
-		.expected = (unsigned char *)
-			"\x5a\x35\x39\x87\x0f\x4d\x22\xa4\x09\x24\xee\x71"
-			"\xc9\x6f\xac\x72\x0a\xd6\xf0\x88\x82\xd0\x83\x28"
-			"\x73\xec\x3f\x93\xd8\xab\x45\x23\xf0\x7e\xac\x45"
-			"\x14\x5e\x93\x9f\xb1\xd6\x76\x43\x3d\xb6\xe8\x08"
-			"\x88\xf6\xda\x89\x08\x77\x42\xfe\x1a\xf4\x3f\xc4"
-			"\x23\xc5\x1f\x68",
-		.expectedlen = 64,
-		.addtla = (unsigned char *)
-			"\x1a\x40\xfa\xe3\xcc\x6c\x7c\xa0\xf8\xda\xba\x59"
-			"\x23\x6d\xad\x1d",
-		.addtlb = (unsigned char *)
-			"\x9f\x72\x76\x6c\xc7\x46\xe5\xed\x2e\x53\x20\x12"
-			"\xbc\x59\x31\x8c",
-		.addtllen = 16,
-		.pers = (unsigned char *)
-			"\xea\x65\xee\x60\x26\x4e\x7e\xb6\x0e\x82\x68\xc4"
-			"\x37\x3c\x5c\x0b",
-		.perslen = 16,
-	},
-};
-
-/*
- * SP800-90A DRBG Test vectors from
- * http://csrc.nist.gov/groups/STM/cavp/documents/drbg/drbgtestvectors.zip
- *
- * Test vectors for DRBG without prediction resistance. All types of DRBGs
- * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and
- * w/o personalization string, w/ and w/o additional input string).
- */
-static const struct drbg_testvec drbg_nopr_sha256_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\xa6\x5a\xd0\xf3\x45\xdb\x4e\x0e\xff\xe8\x75\xc3"
-			"\xa2\xe7\x1f\x42\xc7\x12\x9d\x62\x0f\xf5\xc1\x19"
-			"\xa9\xef\x55\xf0\x51\x85\xe0\xfb\x85\x81\xf9\x31"
-			"\x75\x17\x27\x6e\x06\xe9\x60\x7d\xdb\xcb\xcc\x2e",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xd3\xe1\x60\xc3\x5b\x99\xf3\x40\xb2\x62\x82\x64"
-			"\xd1\x75\x10\x60\xe0\x04\x5d\xa3\x83\xff\x57\xa5"
-			"\x7d\x73\xa6\x73\xd2\xb8\xd8\x0d\xaa\xf6\xa6\xc3"
-			"\x5a\x91\xbb\x45\x79\xd7\x3f\xd0\xc8\xfe\xd1\x11"
-			"\xb0\x39\x13\x06\x82\x8a\xdf\xed\x52\x8f\x01\x81"
-			"\x21\xb3\xfe\xbd\xc3\x43\xe7\x97\xb8\x7d\xbb\x63"
-			"\xdb\x13\x33\xde\xd9\xd1\xec\xe1\x77\xcf\xa6\xb7"
-			"\x1f\xe8\xab\x1d\xa4\x66\x24\xed\x64\x15\xe5\x1c"
-			"\xcd\xe2\xc7\xca\x86\xe2\x83\x99\x0e\xea\xeb\x91"
-			"\x12\x04\x15\x52\x8b\x22\x95\x91\x02\x81\xb0\x2d"
-			"\xd4\x31\xf4\xc9\xf7\x04\x27\xdf",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x73\xd3\xfb\xa3\x94\x5f\x2b\x5f\xb9\x8f\xf6\x9c"
-			"\x8a\x93\x17\xae\x19\xc3\x4c\xc3\xd6\xca\xa3\x2d"
-			"\x16\xfc\x42\xd2\x2d\xd5\x6f\x56\xcc\x1d\x30\xff"
-			"\x9e\x06\x3e\x09\xce\x58\xe6\x9a\x35\xb3\xa6\x56",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\x71\x7b\x93\x46\x1a\x40\xaa\x35\xa4\xaa\xc5\xe7"
-			"\x6d\x5b\x5b\x8a\xa0\xdf\x39\x7d\xae\x71\x58\x5b"
-			"\x3c\x7c\xb4\xf0\x89\xfa\x4a\x8c\xa9\x5c\x54\xc0"
-			"\x40\xdf\xbc\xce\x26\x81\x34\xf8\xba\x7d\x1c\xe8"
-			"\xad\x21\xe0\x74\xcf\x48\x84\x30\x1f\xa1\xd5\x4f"
-			"\x81\x42\x2f\xf4\xdb\x0b\x23\xf8\x73\x27\xb8\x1d"
-			"\x42\xf8\x44\x58\xd8\x5b\x29\x27\x0a\xf8\x69\x59"
-			"\xb5\x78\x44\xeb\x9e\xe0\x68\x6f\x42\x9a\xb0\x5b"
-			"\xe0\x4e\xcb\x6a\xaa\xe2\xd2\xd5\x33\x25\x3e\xe0"
-			"\x6c\xc7\x6a\x07\xa5\x03\x83\x9f\xe2\x8b\xd1\x1c"
-			"\x70\xa8\x07\x59\x97\xeb\xf6\xbe",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\xf4\xd5\x98\x3d\xa8\xfc\xfa\x37\xb7\x54\x67\x73"
-			"\xc7\xc3\xdd\x47\x34\x71\x02\x5d\xc1\xa0\xd3\x10"
-			"\xc1\x8b\xbd\xf5\x66\x34\x6f\xdd",
-		.addtlb = (unsigned char *)
-			"\xf7\x9e\x6a\x56\x0e\x73\xe9\xd9\x7a\xd1\x69\xe0"
-			"\x6f\x8c\x55\x1c\x44\xd1\xce\x6f\x28\xcc\xa4\x4d"
-			"\xa8\xc0\x85\xd1\x5a\x0c\x59\x40",
-		.addtllen = 32,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x2a\x85\xa9\x8b\xd0\xda\x83\xd6\xad\xab\x9f\xbb"
-			"\x54\x31\x15\x95\x1c\x4d\x49\x9f\x6a\x15\xf6\xe4"
-			"\x15\x50\x88\x06\x29\x0d\xed\x8d\xb9\x6f\x96\xe1"
-			"\x83\x9f\xf7\x88\xda\x84\xbf\x44\x28\xd9\x1d\xaa",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\x2d\x55\xde\xc9\xed\x05\x47\x07\x3d\x04\xfc\x28"
-			"\x0f\x92\xf0\x4d\xd8\x00\x32\x47\x0a\x1b\x1c\x4b"
-			"\xef\xd9\x97\xa1\x17\x67\xda\x26\x6c\xfe\x76\x46"
-			"\x6f\xbc\x6d\x82\x4e\x83\x8a\x98\x66\x6c\x01\xb6"
-			"\xe6\x64\xe0\x08\x10\x6f\xd3\x5d\x90\xe7\x0d\x72"
-			"\xa6\xa7\xe3\xbb\x98\x11\x12\x56\x23\xc2\x6d\xd1"
-			"\xc8\xa8\x7a\x39\xf3\x34\xe3\xb8\xf8\x66\x00\x77"
-			"\x7d\xcf\x3c\x3e\xfa\xc9\x0f\xaf\xe0\x24\xfa\xe9"
-			"\x84\xf9\x6a\x01\xf6\x35\xdb\x5c\xab\x2a\xef\x4e"
-			"\xac\xab\x55\xb8\x9b\xef\x98\x68\xaf\x51\xd8\x16"
-			"\xa5\x5e\xae\xf9\x1e\xd2\xdb\xe6",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\xa8\x80\xec\x98\x30\x98\x15\xd2\xc6\xc4\x68\xf1"
-			"\x3a\x1c\xbf\xce\x6a\x40\x14\xeb\x36\x99\x53\xda"
-			"\x57\x6b\xce\xa4\x1c\x66\x3d\xbc",
-		.perslen = 32,
-	}, {
-		.entropy = (unsigned char *)
-			"\x69\xed\x82\xa9\xc5\x7b\xbf\xe5\x1d\x2f\xcb\x7a"
-			"\xd3\x50\x7d\x96\xb4\xb9\x2b\x50\x77\x51\x27\x74"
-			"\x33\x74\xba\xf1\x30\xdf\x8e\xdf\x87\x1d\x87\xbc"
-			"\x96\xb2\xc3\xa7\xed\x60\x5e\x61\x4e\x51\x29\x1a",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xa5\x71\x24\x31\x11\xfe\x13\xe1\xa8\x24\x12\xfb"
-			"\x37\xa1\x27\xa5\xab\x77\xa1\x9f\xae\x8f\xaf\x13"
-			"\x93\xf7\x53\x85\x91\xb6\x1b\xab\xd4\x6b\xea\xb6"
-			"\xef\xda\x4c\x90\x6e\xef\x5f\xde\xe1\xc7\x10\x36"
-			"\xd5\x67\xbd\x14\xb6\x89\x21\x0c\xc9\x92\x65\x64"
-			"\xd0\xf3\x23\xe0\x7f\xd1\xe8\x75\xc2\x85\x06\xea"
-			"\xca\xc0\xcb\x79\x2d\x29\x82\xfc\xaa\x9a\xc6\x95"
-			"\x7e\xdc\x88\x65\xba\xec\x0e\x16\x87\xec\xa3\x9e"
-			"\xd8\x8c\x80\xab\x3a\x64\xe0\xcb\x0e\x45\x98\xdd"
-			"\x7c\x6c\x6c\x26\x11\x13\xc8\xce\xa9\x47\xa6\x06"
-			"\x57\xa2\x66\xbb\x2d\x7f\xf3\xc1",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x74\xd3\x6d\xda\xe8\xd6\x86\x5f\x63\x01\xfd\xf2"
-			"\x7d\x06\x29\x6d\x94\xd1\x66\xf0\xd2\x72\x67\x4e"
-			"\x77\xc5\x3d\x9e\x03\xe3\xa5\x78",
-		.addtlb = (unsigned char *)
-			"\xf6\xb6\x3d\xf0\x7c\x26\x04\xc5\x8b\xcd\x3e\x6a"
-			"\x9f\x9c\x3a\x2e\xdb\x47\x87\xe5\x8e\x00\x5e\x2b"
-			"\x74\x7f\xa6\xf6\x80\xcd\x9b\x21",
-		.addtllen = 32,
-		.pers = (unsigned char *)
-			"\x74\xa6\xe0\x08\xf9\x27\xee\x1d\x6e\x3c\x28\x20"
-			"\x87\xdd\xd7\x54\x31\x47\x78\x4b\xe5\x6d\xa3\x73"
-			"\xa9\x65\xb1\x10\xc1\xdc\x77\x7c",
-		.perslen = 32,
-	},
-};
-
-static const struct drbg_testvec drbg_nopr_hmac_sha256_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\xca\x85\x19\x11\x34\x93\x84\xbf\xfe\x89\xde\x1c"
-			"\xbd\xc4\x6e\x68\x31\xe4\x4d\x34\xa4\xfb\x93\x5e"
-			"\xe2\x85\xdd\x14\xb7\x1a\x74\x88\x65\x9b\xa9\x6c"
-			"\x60\x1d\xc6\x9f\xc9\x02\x94\x08\x05\xec\x0c\xa8",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xe5\x28\xe9\xab\xf2\xde\xce\x54\xd4\x7c\x7e\x75"
-			"\xe5\xfe\x30\x21\x49\xf8\x17\xea\x9f\xb4\xbe\xe6"
-			"\xf4\x19\x96\x97\xd0\x4d\x5b\x89\xd5\x4f\xbb\x97"
-			"\x8a\x15\xb5\xc4\x43\xc9\xec\x21\x03\x6d\x24\x60"
-			"\xb6\xf7\x3e\xba\xd0\xdc\x2a\xba\x6e\x62\x4a\xbf"
-			"\x07\x74\x5b\xc1\x07\x69\x4b\xb7\x54\x7b\xb0\x99"
-			"\x5f\x70\xde\x25\xd6\xb2\x9e\x2d\x30\x11\xbb\x19"
-			"\xd2\x76\x76\xc0\x71\x62\xc8\xb5\xcc\xde\x06\x68"
-			"\x96\x1d\xf8\x68\x03\x48\x2c\xb3\x7e\xd6\xd5\xc0"
-			"\xbb\x8d\x50\xcf\x1f\x50\xd4\x76\xaa\x04\x58\xbd"
-			"\xab\xa8\x06\xf4\x8b\xe9\xdc\xb8",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\xf9\x7a\x3c\xfd\x91\xfa\xa0\x46\xb9\xe6\x1b\x94"
-			"\x93\xd4\x36\xc4\x93\x1f\x60\x4b\x22\xf1\x08\x15"
-			"\x21\xb3\x41\x91\x51\xe8\xff\x06\x11\xf3\xa7\xd4"
-			"\x35\x95\x35\x7d\x58\x12\x0b\xd1\xe2\xdd\x8a\xed",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xc6\x87\x1c\xff\x08\x24\xfe\x55\xea\x76\x89\xa5"
-			"\x22\x29\x88\x67\x30\x45\x0e\x5d\x36\x2d\xa5\xbf"
-			"\x59\x0d\xcf\x9a\xcd\x67\xfe\xd4\xcb\x32\x10\x7d"
-			"\xf5\xd0\x39\x69\xa6\x6b\x1f\x64\x94\xfd\xf5\xd6"
-			"\x3d\x5b\x4d\x0d\x34\xea\x73\x99\xa0\x7d\x01\x16"
-			"\x12\x6d\x0d\x51\x8c\x7c\x55\xba\x46\xe1\x2f\x62"
-			"\xef\xc8\xfe\x28\xa5\x1c\x9d\x42\x8e\x6d\x37\x1d"
-			"\x73\x97\xab\x31\x9f\xc7\x3d\xed\x47\x22\xe5\xb4"
-			"\xf3\x00\x04\x03\x2a\x61\x28\xdf\x5e\x74\x97\xec"
-			"\xf8\x2c\xa7\xb0\xa5\x0e\x86\x7e\xf6\x72\x8a\x4f"
-			"\x50\x9a\x8c\x85\x90\x87\x03\x9c",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x51\x72\x89\xaf\xe4\x44\xa0\xfe\x5e\xd1\xa4\x1d"
-			"\xbb\xb5\xeb\x17\x15\x00\x79\xbd\xd3\x1e\x29\xcf"
-			"\x2f\xf3\x00\x34\xd8\x26\x8e\x3b",
-		.addtlb = (unsigned char *)
-			"\x88\x02\x8d\x29\xef\x80\xb4\xe6\xf0\xfe\x12\xf9"
-			"\x1d\x74\x49\xfe\x75\x06\x26\x82\xe8\x9c\x57\x14"
-			"\x40\xc0\xc9\xb5\x2c\x42\xa6\xe0",
-		.addtllen = 32,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x8d\xf0\x13\xb4\xd1\x03\x52\x30\x73\x91\x7d\xdf"
-			"\x6a\x86\x97\x93\x05\x9e\x99\x43\xfc\x86\x54\x54"
-			"\x9e\x7a\xb2\x2f\x7c\x29\xf1\x22\xda\x26\x25\xaf"
-			"\x2d\xdd\x4a\xbc\xce\x3c\xf4\xfa\x46\x59\xd8\x4e",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xb9\x1c\xba\x4c\xc8\x4f\xa2\x5d\xf8\x61\x0b\x81"
-			"\xb6\x41\x40\x27\x68\xa2\x09\x72\x34\x93\x2e\x37"
-			"\xd5\x90\xb1\x15\x4c\xbd\x23\xf9\x74\x52\xe3\x10"
-			"\xe2\x91\xc4\x51\x46\x14\x7f\x0d\xa2\xd8\x17\x61"
-			"\xfe\x90\xfb\xa6\x4f\x94\x41\x9c\x0f\x66\x2b\x28"
-			"\xc1\xed\x94\xda\x48\x7b\xb7\xe7\x3e\xec\x79\x8f"
-			"\xbc\xf9\x81\xb7\x91\xd1\xbe\x4f\x17\x7a\x89\x07"
-			"\xaa\x3c\x40\x16\x43\xa5\xb6\x2b\x87\xb8\x9d\x66"
-			"\xb3\xa6\x0e\x40\xd4\xa8\xe4\xe9\xd8\x2a\xf6\xd2"
-			"\x70\x0e\x6f\x53\x5c\xdb\x51\xf7\x5c\x32\x17\x29"
-			"\x10\x37\x41\x03\x0c\xcc\x3a\x56",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\xb5\x71\xe6\x6d\x7c\x33\x8b\xc0\x7b\x76\xad\x37"
-			"\x57\xbb\x2f\x94\x52\xbf\x7e\x07\x43\x7a\xe8\x58"
-			"\x1c\xe7\xbc\x7c\x3a\xc6\x51\xa9",
-		.perslen = 32,
-	}, {
-		.entropy = (unsigned char *)
-			"\xc2\xa5\x66\xa9\xa1\x81\x7b\x15\xc5\xc3\xb7\x78"
-			"\x17\x7a\xc8\x7c\x24\xe7\x97\xbe\x0a\x84\x5f\x11"
-			"\xc2\xfe\x39\x9d\xd3\x77\x32\xf2\xcb\x18\x94\xeb"
-			"\x2b\x97\xb3\xc5\x6e\x62\x83\x29\x51\x6f\x86\xec",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xb3\xa3\x69\x8d\x77\x76\x99\xa0\xdd\x9f\xa3\xf0"
-			"\xa9\xfa\x57\x83\x2d\x3c\xef\xac\x5d\xf2\x44\x37"
-			"\xc6\xd7\x3a\x0f\xe4\x10\x40\xf1\x72\x90\x38\xae"
-			"\xf1\xe9\x26\x35\x2e\xa5\x9d\xe1\x20\xbf\xb7\xb0"
-			"\x73\x18\x3a\x34\x10\x6e\xfe\xd6\x27\x8f\xf8\xad"
-			"\x84\x4b\xa0\x44\x81\x15\xdf\xdd\xf3\x31\x9a\x82"
-			"\xde\x6b\xb1\x1d\x80\xbd\x87\x1a\x9a\xcd\x35\xc7"
-			"\x36\x45\xe1\x27\x0f\xb9\xfe\x4f\xa8\x8e\xc0\xe4"
-			"\x65\x40\x9e\xa0\xcb\xa8\x09\xfe\x2f\x45\xe0\x49"
-			"\x43\xa2\xe3\x96\xbb\xb7\xdd\x2f\x4e\x07\x95\x30"
-			"\x35\x24\xcc\x9c\xc5\xea\x54\xa1",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x41\x3d\xd8\x3f\xe5\x68\x35\xab\xd4\x78\xcb\x96"
-			"\x93\xd6\x76\x35\x90\x1c\x40\x23\x9a\x26\x64\x62"
-			"\xd3\x13\x3b\x83\xe4\x9c\x82\x0b",
-		.addtlb = (unsigned char *)
-			"\xd5\xc4\xa7\x1f\x9d\x6d\x95\xa1\xbe\xdf\x0b\xd2"
-			"\x24\x7c\x27\x7d\x1f\x84\xa4\xe5\x7a\x4a\x88\x25"
-			"\xb8\x2a\x2d\x09\x7d\xe6\x3e\xf1",
-		.addtllen = 32,
-		.pers = (unsigned char *)
-			"\x13\xce\x4d\x8d\xd2\xdb\x97\x96\xf9\x41\x56\xc8"
-			"\xe8\xf0\x76\x9b\x0a\xa1\xc8\x2c\x13\x23\xb6\x15"
-			"\x36\x60\x3b\xca\x37\xc9\xee\x29",
-		.perslen = 32,
-	},
-};
-
-static const struct drbg_testvec drbg_nopr_ctr_aes192_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\xc3\x5c\x2f\xa2\xa8\x9d\x52\xa1\x1f\xa3\x2a\xa9"
-			"\x6c\x95\xb8\xf1\xc9\xa8\xf9\xcb\x24\x5a\x8b\x40"
-			"\xf3\xa6\xe5\xa7\xfb\xd9\xd3\xc6\x8e\x27\x7b\xa9"
-			"\xac\x9b\xbb\x00",
-		.entropylen = 40,
-		.expected = (unsigned char *)
-			"\x8c\x2e\x72\xab\xfd\x9b\xb8\x28\x4d\xb7\x9e\x17"
-			"\xa4\x3a\x31\x46\xcd\x76\x94\xe3\x52\x49\xfc\x33"
-			"\x83\x91\x4a\x71\x17\xf4\x13\x68\xe6\xd4\xf1\x48"
-			"\xff\x49\xbf\x29\x07\x6b\x50\x15\xc5\x9f\x45\x79"
-			"\x45\x66\x2e\x3d\x35\x03\x84\x3f\x4a\xa5\xa3\xdf"
-			"\x9a\x9d\xf1\x0d",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	},
-};
-
-static const struct drbg_testvec drbg_nopr_ctr_aes256_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\x36\x40\x19\x40\xfa\x8b\x1f\xba\x91\xa1\x66\x1f"
-			"\x21\x1d\x78\xa0\xb9\x38\x9a\x74\xe5\xbc\xcf\xec"
-			"\xe8\xd7\x66\xaf\x1a\x6d\x3b\x14\x49\x6f\x25\xb0"
-			"\xf1\x30\x1b\x4f\x50\x1b\xe3\x03\x80\xa1\x37\xeb",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\x58\x62\xeb\x38\xbd\x55\x8d\xd9\x78\xa6\x96\xe6"
-			"\xdf\x16\x47\x82\xdd\xd8\x87\xe7\xe9\xa6\xc9\xf3"
-			"\xf1\xfb\xaf\xb7\x89\x41\xb5\x35\xa6\x49\x12\xdf"
-			"\xd2\x24\xc6\xdc\x74\x54\xe5\x25\x0b\x3d\x97\x16"
-			"\x5e\x16\x26\x0c\x2f\xaf\x1c\xc7\x73\x5c\xb7\x5f"
-			"\xb4\xf0\x7e\x1d",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	},
-};
-
-static const struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\x87\xe1\xc5\x32\x99\x7f\x57\xa3\x5c\x28\x6d\xe8"
-			"\x64\xbf\xf2\x64\xa3\x9e\x98\xdb\x6c\x10\x78\x7f",
-		.entropylen = 24,
-		.expected = (unsigned char *)
-			"\x2c\x14\x7e\x24\x11\x9a\xd8\xd4\xb2\xed\x61\xc1"
-			"\x53\xd0\x50\xc9\x24\xff\x59\x75\x15\xf1\x17\x3a"
-			"\x3d\xf4\x4b\x2c\x84\x28\xef\x89\x0e\xb9\xde\xf3"
-			"\xe4\x78\x04\xb2\xfd\x9b\x35\x7f\xe1\x3f\x8a\x3e"
-			"\x10\xc8\x67\x0a\xf9\xdf\x2d\x6c\x96\xfb\xb2\xb8"
-			"\xcb\x2d\xd6\xb0",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x71\xbd\xce\x35\x42\x7d\x20\xbf\x58\xcf\x17\x74"
-			"\xce\x72\xd8\x33\x34\x50\x2d\x8f\x5b\x14\xc4\xdd",
-		.entropylen = 24,
-		.expected = (unsigned char *)
-			"\x97\x33\xe8\x20\x12\xe2\x7b\xa1\x46\x8f\xf2\x34"
-			"\xb3\xc9\xb6\x6b\x20\xb2\x4f\xee\x27\xd8\x0b\x21"
-			"\x8c\xff\x63\x73\x69\x29\xfb\xf3\x85\xcd\x88\x8e"
-			"\x43\x2c\x71\x8b\xa2\x55\xd2\x0f\x1d\x7f\xe3\xe1"
-			"\x2a\xa3\xe9\x2c\x25\x89\xc7\x14\x52\x99\x56\xcc"
-			"\xc3\xdf\xb3\x81",
-		.expectedlen = 64,
-		.addtla = (unsigned char *)
-			"\x66\xef\x42\xd6\x9a\x8c\x3d\x6d\x4a\x9e\x95\xa6"
-			"\x91\x4d\x81\x56",
-		.addtlb = (unsigned char *)
-			"\xe3\x18\x83\xd9\x4b\x5e\xc4\xcc\xaa\x61\x2f\xbb"
-			"\x4a\x55\xd1\xc6",
-		.addtllen = 16,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\xca\x4b\x1e\xfa\x75\xbd\x69\x36\x38\x73\xb8\xf9"
-			"\xdb\x4d\x35\x0e\x47\xbf\x6c\x37\x72\xfd\xf7\xa9",
-		.entropylen = 24,
-		.expected = (unsigned char *)
-			"\x59\xc3\x19\x79\x1b\xb1\xf3\x0e\xe9\x34\xae\x6e"
-			"\x8b\x1f\xad\x1f\x74\xca\x25\x45\x68\xb8\x7f\x75"
-			"\x12\xf8\xf2\xab\x4c\x23\x01\x03\x05\xe1\x70\xee"
-			"\x75\xd8\xcb\xeb\x23\x4c\x7a\x23\x6e\x12\x27\xdb"
-			"\x6f\x7a\xac\x3c\x44\xb7\x87\x4b\x65\x56\x74\x45"
-			"\x34\x30\x0c\x3d",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\xeb\xaa\x60\x2c\x4d\xbe\x33\xff\x1b\xef\xbf\x0a"
-			"\x0b\xc6\x97\x54",
-		.perslen = 16,
-	}, {
-		.entropy = (unsigned char *)
-			"\xc0\x70\x1f\x92\x50\x75\x8f\xcd\xf2\xbe\x73\x98"
-			"\x80\xdb\x66\xeb\x14\x68\xb4\xa5\x87\x9c\x2d\xa6",
-		.entropylen = 24,
-		.expected = (unsigned char *)
-			"\x97\xc0\xc0\xe5\xa0\xcc\xf2\x4f\x33\x63\x48\x8a"
-			"\xdb\x13\x0a\x35\x89\xbf\x80\x65\x62\xee\x13\x95"
-			"\x7c\x33\xd3\x7d\xf4\x07\x77\x7a\x2b\x65\x0b\x5f"
-			"\x45\x5c\x13\xf1\x90\x77\x7f\xc5\x04\x3f\xcc\x1a"
-			"\x38\xf8\xcd\x1b\xbb\xd5\x57\xd1\x4a\x4c\x2e\x8a"
-			"\x2b\x49\x1e\x5c",
-		.expectedlen = 64,
-		.addtla = (unsigned char *)
-			"\xf9\x01\xf8\x16\x7a\x1d\xff\xde\x8e\x3c\x83\xe2"
-			"\x44\x85\xe7\xfe",
-		.addtlb = (unsigned char *)
-			"\x17\x1c\x09\x38\xc2\x38\x9f\x97\x87\x60\x55\xb4"
-			"\x82\x16\x62\x7f",
-		.addtllen = 16,
-		.pers = (unsigned char *)
-			"\x80\x08\xae\xe8\xe9\x69\x40\xc5\x08\x73\xc7\x9f"
-			"\x8e\xcf\xe0\x02",
-		.perslen = 16,
-	},
-};
-
-/* Cast5 test vectors from RFC 2144 */
-static const struct cipher_testvec cast5_enc_tv_template[] = {
-	{
-		.key	= "\x01\x23\x45\x67\x12\x34\x56\x78"
-			  "\x23\x45\x67\x89\x34\x56\x78\x9a",
-		.klen	= 16,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.ilen	= 8,
-		.result	= "\x23\x8b\x4f\xe5\x84\x7e\x44\xb2",
-		.rlen	= 8,
-	}, {
-		.key	= "\x01\x23\x45\x67\x12\x34\x56\x78"
-			  "\x23\x45",
-		.klen	= 10,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.ilen	= 8,
-		.result	= "\xeb\x6a\x71\x1a\x2c\x02\x27\x1b",
-		.rlen	= 8,
-	}, {
-		.key	= "\x01\x23\x45\x67\x12",
-		.klen	= 5,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.ilen	= 8,
-		.result	= "\x7a\xc8\x16\xd1\x6e\x9b\x30\x2e",
-		.rlen	= 8,
-	}, { /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
-		.klen	= 16,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\x8D\xFC\x81\x9C\xCB\xAA\x5A\x1C"
-			  "\x7E\x95\xCF\x40\xAB\x4D\x6F\xEA"
-			  "\xD3\xD9\xB0\x9A\xB7\xC7\xE0\x2E"
-			  "\xD1\x39\x34\x92\x8F\xFA\x14\xF1"
-			  "\xD5\xD2\x7B\x59\x1F\x35\x28\xC2"
-			  "\x20\xD9\x42\x06\xC9\x0B\x10\x04"
-			  "\xF8\x79\xCD\x32\x86\x75\x4C\xB6"
-			  "\x7B\x1C\x52\xB1\x91\x64\x22\x4B"
-			  "\x13\xC7\xAE\x98\x0E\xB5\xCF\x6F"
-			  "\x3F\xF4\x43\x96\x73\x0D\xA2\x05"
-			  "\xDB\xFD\x28\x90\x2C\x56\xB9\x37"
-			  "\x5B\x69\x0C\xAD\x84\x67\xFF\x15"
-			  "\x4A\xD4\xA7\xD3\xDD\x99\x47\x3A"
-			  "\xED\x34\x35\x78\x6B\x91\xC9\x32"
-			  "\xE1\xBF\xBC\xB4\x04\x85\x6A\x39"
-			  "\xC0\xBA\x51\xD0\x0F\x4E\xD1\xE2"
-			  "\x1C\xFD\x0E\x05\x07\xF4\x10\xED"
-			  "\xA2\x17\xFF\xF5\x64\xC6\x1A\x22"
-			  "\xAD\x78\xE7\xD7\x11\xE9\x99\xB9"
-			  "\xAA\xEC\x6F\xF8\x3B\xBF\xCE\x77"
-			  "\x93\xE8\xAD\x1D\x50\x6C\xAE\xBC"
-			  "\xBA\x5C\x80\xD1\x91\x65\x51\x1B"
-			  "\xE8\x0A\xCD\x99\x96\x71\x3D\xB6"
-			  "\x78\x75\x37\x55\xC1\xF5\x90\x40"
-			  "\x34\xF4\x7E\xC8\xCC\x3A\x5F\x6E"
-			  "\x36\xA1\xA1\xC2\x3A\x72\x42\x8E"
-			  "\x0E\x37\x88\xE8\xCE\x83\xCB\xAD"
-			  "\xE0\x69\x77\x50\xC7\x0C\x99\xCA"
-			  "\x19\x5B\x30\x25\x9A\xEF\x9B\x0C"
-			  "\xEF\x8F\x74\x4C\xCF\x49\x4E\xB9"
-			  "\xC5\xAE\x9E\x2E\x78\x9A\xB9\x48"
-			  "\xD5\x81\xE4\x37\x1D\xBF\x27\xD9"
-			  "\xC5\xD6\x65\x43\x45\x8C\xBB\xB6"
-			  "\x55\xF4\x06\xBB\x49\x53\x8B\x1B"
-			  "\x07\xA9\x96\x69\x5B\xCB\x0F\xBC"
-			  "\x93\x85\x90\x0F\x0A\x68\x40\x2A"
-			  "\x95\xED\x2D\x88\xBF\x71\xD0\xBB"
-			  "\xEC\xB0\x77\x6C\x79\xFC\x3C\x05"
-			  "\x49\x3F\xB8\x24\xEF\x8E\x09\xA2"
-			  "\x1D\xEF\x92\x02\x96\xD4\x7F\xC8"
-			  "\x03\xB2\xCA\xDB\x17\x5C\x52\xCF"
-			  "\xDD\x70\x37\x63\xAA\xA5\x83\x20"
-			  "\x52\x02\xF6\xB9\xE7\x6E\x0A\xB6"
-			  "\x79\x03\xA0\xDA\xA3\x79\x21\xBD"
-			  "\xE3\x37\x3A\xC0\xF7\x2C\x32\xBE"
-			  "\x8B\xE8\xA6\x00\xC7\x32\xD5\x06"
-			  "\xBB\xE3\xAB\x06\x21\x82\xB8\x32"
-			  "\x31\x34\x2A\xA7\x1F\x64\x99\xBF"
-			  "\xFA\xDA\x3D\x75\xF7\x48\xD5\x48"
-			  "\x4B\x52\x7E\xF6\x7C\xAB\x67\x59"
-			  "\xC5\xDC\xA8\xC6\x63\x85\x4A\xDF"
-			  "\xF0\x40\x5F\xCF\xE3\x58\x52\x67"
-			  "\x7A\x24\x32\xC5\xEC\x9E\xA9\x6F"
-			  "\x58\x56\xDD\x94\x1F\x71\x8D\xF4"
-			  "\x6E\xFF\x2C\xA7\xA5\xD8\xBA\xAF"
-			  "\x1D\x8B\xA2\x46\xB5\xC4\x9F\x57"
-			  "\x8D\xD8\xB3\x3C\x02\x0D\xBB\x84"
-			  "\xC7\xBD\xB4\x9A\x6E\xBB\xB1\x37"
-			  "\x95\x79\xC4\xA7\xEA\x1D\xDC\x33"
-			  "\x5D\x0B\x3F\x03\x8F\x30\xF9\xAE"
-			  "\x4F\xFE\x24\x9C\x9A\x02\xE5\x57"
-			  "\xF5\xBC\x25\xD6\x02\x56\x57\x1C",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast5_dec_tv_template[] = {
-	{
-		.key	= "\x01\x23\x45\x67\x12\x34\x56\x78"
-			  "\x23\x45\x67\x89\x34\x56\x78\x9a",
-		.klen	= 16,
-		.input	= "\x23\x8b\x4f\xe5\x84\x7e\x44\xb2",
-		.ilen	= 8,
-		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.rlen	= 8,
-	}, {
-		.key	= "\x01\x23\x45\x67\x12\x34\x56\x78"
-			  "\x23\x45",
-		.klen	= 10,
-		.input	= "\xeb\x6a\x71\x1a\x2c\x02\x27\x1b",
-		.ilen	= 8,
-		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.rlen	= 8,
-	}, {
-		.key	= "\x01\x23\x45\x67\x12",
-		.klen	= 5,
-		.input	= "\x7a\xc8\x16\xd1\x6e\x9b\x30\x2e",
-		.ilen	= 8,
-		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.rlen	= 8,
-	}, { /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
-		.klen	= 16,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\x8D\xFC\x81\x9C\xCB\xAA\x5A\x1C"
-			  "\x7E\x95\xCF\x40\xAB\x4D\x6F\xEA"
-			  "\xD3\xD9\xB0\x9A\xB7\xC7\xE0\x2E"
-			  "\xD1\x39\x34\x92\x8F\xFA\x14\xF1"
-			  "\xD5\xD2\x7B\x59\x1F\x35\x28\xC2"
-			  "\x20\xD9\x42\x06\xC9\x0B\x10\x04"
-			  "\xF8\x79\xCD\x32\x86\x75\x4C\xB6"
-			  "\x7B\x1C\x52\xB1\x91\x64\x22\x4B"
-			  "\x13\xC7\xAE\x98\x0E\xB5\xCF\x6F"
-			  "\x3F\xF4\x43\x96\x73\x0D\xA2\x05"
-			  "\xDB\xFD\x28\x90\x2C\x56\xB9\x37"
-			  "\x5B\x69\x0C\xAD\x84\x67\xFF\x15"
-			  "\x4A\xD4\xA7\xD3\xDD\x99\x47\x3A"
-			  "\xED\x34\x35\x78\x6B\x91\xC9\x32"
-			  "\xE1\xBF\xBC\xB4\x04\x85\x6A\x39"
-			  "\xC0\xBA\x51\xD0\x0F\x4E\xD1\xE2"
-			  "\x1C\xFD\x0E\x05\x07\xF4\x10\xED"
-			  "\xA2\x17\xFF\xF5\x64\xC6\x1A\x22"
-			  "\xAD\x78\xE7\xD7\x11\xE9\x99\xB9"
-			  "\xAA\xEC\x6F\xF8\x3B\xBF\xCE\x77"
-			  "\x93\xE8\xAD\x1D\x50\x6C\xAE\xBC"
-			  "\xBA\x5C\x80\xD1\x91\x65\x51\x1B"
-			  "\xE8\x0A\xCD\x99\x96\x71\x3D\xB6"
-			  "\x78\x75\x37\x55\xC1\xF5\x90\x40"
-			  "\x34\xF4\x7E\xC8\xCC\x3A\x5F\x6E"
-			  "\x36\xA1\xA1\xC2\x3A\x72\x42\x8E"
-			  "\x0E\x37\x88\xE8\xCE\x83\xCB\xAD"
-			  "\xE0\x69\x77\x50\xC7\x0C\x99\xCA"
-			  "\x19\x5B\x30\x25\x9A\xEF\x9B\x0C"
-			  "\xEF\x8F\x74\x4C\xCF\x49\x4E\xB9"
-			  "\xC5\xAE\x9E\x2E\x78\x9A\xB9\x48"
-			  "\xD5\x81\xE4\x37\x1D\xBF\x27\xD9"
-			  "\xC5\xD6\x65\x43\x45\x8C\xBB\xB6"
-			  "\x55\xF4\x06\xBB\x49\x53\x8B\x1B"
-			  "\x07\xA9\x96\x69\x5B\xCB\x0F\xBC"
-			  "\x93\x85\x90\x0F\x0A\x68\x40\x2A"
-			  "\x95\xED\x2D\x88\xBF\x71\xD0\xBB"
-			  "\xEC\xB0\x77\x6C\x79\xFC\x3C\x05"
-			  "\x49\x3F\xB8\x24\xEF\x8E\x09\xA2"
-			  "\x1D\xEF\x92\x02\x96\xD4\x7F\xC8"
-			  "\x03\xB2\xCA\xDB\x17\x5C\x52\xCF"
-			  "\xDD\x70\x37\x63\xAA\xA5\x83\x20"
-			  "\x52\x02\xF6\xB9\xE7\x6E\x0A\xB6"
-			  "\x79\x03\xA0\xDA\xA3\x79\x21\xBD"
-			  "\xE3\x37\x3A\xC0\xF7\x2C\x32\xBE"
-			  "\x8B\xE8\xA6\x00\xC7\x32\xD5\x06"
-			  "\xBB\xE3\xAB\x06\x21\x82\xB8\x32"
-			  "\x31\x34\x2A\xA7\x1F\x64\x99\xBF"
-			  "\xFA\xDA\x3D\x75\xF7\x48\xD5\x48"
-			  "\x4B\x52\x7E\xF6\x7C\xAB\x67\x59"
-			  "\xC5\xDC\xA8\xC6\x63\x85\x4A\xDF"
-			  "\xF0\x40\x5F\xCF\xE3\x58\x52\x67"
-			  "\x7A\x24\x32\xC5\xEC\x9E\xA9\x6F"
-			  "\x58\x56\xDD\x94\x1F\x71\x8D\xF4"
-			  "\x6E\xFF\x2C\xA7\xA5\xD8\xBA\xAF"
-			  "\x1D\x8B\xA2\x46\xB5\xC4\x9F\x57"
-			  "\x8D\xD8\xB3\x3C\x02\x0D\xBB\x84"
-			  "\xC7\xBD\xB4\x9A\x6E\xBB\xB1\x37"
-			  "\x95\x79\xC4\xA7\xEA\x1D\xDC\x33"
-			  "\x5D\x0B\x3F\x03\x8F\x30\xF9\xAE"
-			  "\x4F\xFE\x24\x9C\x9A\x02\xE5\x57"
-			  "\xF5\xBC\x25\xD6\x02\x56\x57\x1C",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast5_cbc_enc_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
-		.klen	= 16,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\x05\x28\xCE\x61\x90\x80\xE1\x78"
-			  "\xB9\x2A\x97\x7C\xB0\x83\xD8\x1A"
-			  "\xDE\x58\x7F\xD7\xFD\x72\xB8\xFB"
-			  "\xDA\xF0\x6E\x77\x14\x47\x82\xBA"
-			  "\x29\x0E\x25\x6E\xB4\x39\xD9\x7F"
-			  "\x05\xA7\xA7\x3A\xC1\x5D\x9E\x39"
-			  "\xA7\xFB\x0D\x05\x00\xF3\x58\x67"
-			  "\x60\xEC\x73\x77\x46\x85\x9B\x6A"
-			  "\x08\x3E\xBE\x59\xFB\xE4\x96\x34"
-			  "\xB4\x05\x49\x1A\x97\x43\xAD\xA0"
-			  "\xA9\x1E\x6E\x74\xF1\x94\xEC\xA8"
-			  "\xB5\x8A\x20\xEA\x89\x6B\x19\xAA"
-			  "\xA7\xF1\x33\x67\x90\x23\x0D\xEE"
-			  "\x81\xD5\x78\x4F\xD3\x63\xEA\x46"
-			  "\xB5\xB2\x6E\xBB\xCA\x76\x06\x10"
-			  "\x96\x2A\x0A\xBA\xF9\x41\x5A\x1D"
-			  "\x36\x7C\x56\x14\x54\x83\xFA\xA1"
-			  "\x27\xDD\xBA\x8A\x90\x29\xD6\xA6"
-			  "\xFA\x48\x3E\x1E\x23\x6E\x98\xA8"
-			  "\xA7\xD9\x67\x92\x5C\x13\xB4\x71"
-			  "\xA8\xAA\x89\x4A\xA4\xB3\x49\x7C"
-			  "\x7D\x7F\xCE\x6F\x29\x2E\x7E\x37"
-			  "\xC8\x52\x60\xD9\xE7\xCA\x60\x98"
-			  "\xED\xCD\xE8\x60\x83\xAD\x34\x4D"
-			  "\x96\x4A\x99\x2B\xB7\x14\x75\x66"
-			  "\x6C\x2C\x1A\xBA\x4B\xBB\x49\x56"
-			  "\xE1\x86\xA2\x0E\xD0\xF0\x07\xD3"
-			  "\x18\x38\x09\x9C\x0E\x8B\x86\x07"
-			  "\x90\x12\x37\x49\x27\x98\x69\x18"
-			  "\xB0\xCC\xFB\xD3\xBD\x04\xA0\x85"
-			  "\x4B\x22\x97\x07\xB6\x97\xE9\x95"
-			  "\x0F\x88\x36\xA9\x44\x00\xC6\xE9"
-			  "\x27\x53\x5C\x5B\x1F\xD3\xE2\xEE"
-			  "\xD0\xCD\x63\x30\xA9\xC0\xDD\x49"
-			  "\xFE\x16\xA4\x07\x0D\xE2\x5D\x97"
-			  "\xDE\x89\xBA\x2E\xF3\xA9\x5E\xBE"
-			  "\x03\x55\x0E\x02\x41\x4A\x45\x06"
-			  "\xBE\xEA\x32\xF2\xDC\x91\x5C\x20"
-			  "\x94\x02\x30\xD2\xFC\x29\xFA\x8E"
-			  "\x34\xA0\x31\xB8\x34\xBA\xAE\x54"
-			  "\xB5\x88\x1F\xDC\x43\xDC\x22\x9F"
-			  "\xDC\xCE\xD3\xFA\xA4\xA8\xBC\x8A"
-			  "\xC7\x5A\x43\x21\xA5\xB1\xDB\xC3"
-			  "\x84\x3B\xB4\x9B\xB5\xA7\xF1\x0A"
-			  "\xB6\x37\x21\x19\x55\xC2\xBD\x99"
-			  "\x49\x24\xBB\x7C\xB3\x8E\xEF\xD2"
-			  "\x3A\xCF\xA0\x31\x28\x0E\x25\xA2"
-			  "\x11\xB4\x18\x17\x1A\x65\x92\x56"
-			  "\xE8\xE0\x52\x9C\x61\x18\x2A\xB1"
-			  "\x1A\x01\x22\x45\x17\x62\x52\x6C"
-			  "\x91\x44\xCF\x98\xC7\xC0\x79\x26"
-			  "\x32\x66\x6F\x23\x7F\x94\x36\x88"
-			  "\x3C\xC9\xD0\xB7\x45\x30\x31\x86"
-			  "\x3D\xC6\xA3\x98\x62\x84\x1A\x8B"
-			  "\x16\x88\xC7\xA3\xE9\x4F\xE0\x86"
-			  "\xA4\x93\xA8\x34\x5A\xCA\xDF\xCA"
-			  "\x46\x38\xD2\xF4\xE0\x2D\x1E\xC9"
-			  "\x7C\xEF\x53\xB7\x60\x72\x41\xBF"
-			  "\x29\x00\x87\x02\xAF\x44\x4C\xB7"
-			  "\x8C\xF5\x3F\x19\xF4\x80\x45\xA7"
-			  "\x15\x5F\xDB\xE9\xB1\x83\xD2\xE6"
-			  "\x1D\x18\x66\x44\x5B\x8F\x14\xEB",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast5_cbc_dec_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
-		.klen	= 16,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\x05\x28\xCE\x61\x90\x80\xE1\x78"
-			  "\xB9\x2A\x97\x7C\xB0\x83\xD8\x1A"
-			  "\xDE\x58\x7F\xD7\xFD\x72\xB8\xFB"
-			  "\xDA\xF0\x6E\x77\x14\x47\x82\xBA"
-			  "\x29\x0E\x25\x6E\xB4\x39\xD9\x7F"
-			  "\x05\xA7\xA7\x3A\xC1\x5D\x9E\x39"
-			  "\xA7\xFB\x0D\x05\x00\xF3\x58\x67"
-			  "\x60\xEC\x73\x77\x46\x85\x9B\x6A"
-			  "\x08\x3E\xBE\x59\xFB\xE4\x96\x34"
-			  "\xB4\x05\x49\x1A\x97\x43\xAD\xA0"
-			  "\xA9\x1E\x6E\x74\xF1\x94\xEC\xA8"
-			  "\xB5\x8A\x20\xEA\x89\x6B\x19\xAA"
-			  "\xA7\xF1\x33\x67\x90\x23\x0D\xEE"
-			  "\x81\xD5\x78\x4F\xD3\x63\xEA\x46"
-			  "\xB5\xB2\x6E\xBB\xCA\x76\x06\x10"
-			  "\x96\x2A\x0A\xBA\xF9\x41\x5A\x1D"
-			  "\x36\x7C\x56\x14\x54\x83\xFA\xA1"
-			  "\x27\xDD\xBA\x8A\x90\x29\xD6\xA6"
-			  "\xFA\x48\x3E\x1E\x23\x6E\x98\xA8"
-			  "\xA7\xD9\x67\x92\x5C\x13\xB4\x71"
-			  "\xA8\xAA\x89\x4A\xA4\xB3\x49\x7C"
-			  "\x7D\x7F\xCE\x6F\x29\x2E\x7E\x37"
-			  "\xC8\x52\x60\xD9\xE7\xCA\x60\x98"
-			  "\xED\xCD\xE8\x60\x83\xAD\x34\x4D"
-			  "\x96\x4A\x99\x2B\xB7\x14\x75\x66"
-			  "\x6C\x2C\x1A\xBA\x4B\xBB\x49\x56"
-			  "\xE1\x86\xA2\x0E\xD0\xF0\x07\xD3"
-			  "\x18\x38\x09\x9C\x0E\x8B\x86\x07"
-			  "\x90\x12\x37\x49\x27\x98\x69\x18"
-			  "\xB0\xCC\xFB\xD3\xBD\x04\xA0\x85"
-			  "\x4B\x22\x97\x07\xB6\x97\xE9\x95"
-			  "\x0F\x88\x36\xA9\x44\x00\xC6\xE9"
-			  "\x27\x53\x5C\x5B\x1F\xD3\xE2\xEE"
-			  "\xD0\xCD\x63\x30\xA9\xC0\xDD\x49"
-			  "\xFE\x16\xA4\x07\x0D\xE2\x5D\x97"
-			  "\xDE\x89\xBA\x2E\xF3\xA9\x5E\xBE"
-			  "\x03\x55\x0E\x02\x41\x4A\x45\x06"
-			  "\xBE\xEA\x32\xF2\xDC\x91\x5C\x20"
-			  "\x94\x02\x30\xD2\xFC\x29\xFA\x8E"
-			  "\x34\xA0\x31\xB8\x34\xBA\xAE\x54"
-			  "\xB5\x88\x1F\xDC\x43\xDC\x22\x9F"
-			  "\xDC\xCE\xD3\xFA\xA4\xA8\xBC\x8A"
-			  "\xC7\x5A\x43\x21\xA5\xB1\xDB\xC3"
-			  "\x84\x3B\xB4\x9B\xB5\xA7\xF1\x0A"
-			  "\xB6\x37\x21\x19\x55\xC2\xBD\x99"
-			  "\x49\x24\xBB\x7C\xB3\x8E\xEF\xD2"
-			  "\x3A\xCF\xA0\x31\x28\x0E\x25\xA2"
-			  "\x11\xB4\x18\x17\x1A\x65\x92\x56"
-			  "\xE8\xE0\x52\x9C\x61\x18\x2A\xB1"
-			  "\x1A\x01\x22\x45\x17\x62\x52\x6C"
-			  "\x91\x44\xCF\x98\xC7\xC0\x79\x26"
-			  "\x32\x66\x6F\x23\x7F\x94\x36\x88"
-			  "\x3C\xC9\xD0\xB7\x45\x30\x31\x86"
-			  "\x3D\xC6\xA3\x98\x62\x84\x1A\x8B"
-			  "\x16\x88\xC7\xA3\xE9\x4F\xE0\x86"
-			  "\xA4\x93\xA8\x34\x5A\xCA\xDF\xCA"
-			  "\x46\x38\xD2\xF4\xE0\x2D\x1E\xC9"
-			  "\x7C\xEF\x53\xB7\x60\x72\x41\xBF"
-			  "\x29\x00\x87\x02\xAF\x44\x4C\xB7"
-			  "\x8C\xF5\x3F\x19\xF4\x80\x45\xA7"
-			  "\x15\x5F\xDB\xE9\xB1\x83\xD2\xE6"
-			  "\x1D\x18\x66\x44\x5B\x8F\x14\xEB",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast5_ctr_enc_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
-		.klen	= 16,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A",
-		.ilen	= 17,
-		.result	= "\xFF\xC4\x2E\x82\x3D\xF8\xA8\x39"
-			  "\x7C\x52\xC4\xD3\xBB\x62\xC6\xA8"
-			  "\x0C",
-		.rlen	= 17,
-	}, { /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
-		.klen	= 16,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\xFF\xC4\x2E\x82\x3D\xF8\xA8\x39"
-			  "\x7C\x52\xC4\xD3\xBB\x62\xC6\xA8"
-			  "\x0C\x63\xA5\x55\xE3\xF8\x1C\x7F"
-			  "\xDC\x59\xF9\xA0\x52\xAD\x83\xDF"
-			  "\xD5\x3B\x53\x4A\xAA\x1F\x49\x44"
-			  "\xE8\x20\xCC\xF8\x97\xE6\xE0\x3C"
-			  "\x5A\xD2\x83\xEC\xEE\x25\x3F\xCF"
-			  "\x0D\xC2\x79\x80\x99\x6E\xFF\x7B"
-			  "\x64\xB0\x7B\x86\x29\x1D\x9F\x17"
-			  "\x10\xA5\xA5\xEB\x16\x55\x9E\xE3"
-			  "\x88\x18\x52\x56\x48\x58\xD1\x6B"
-			  "\xE8\x74\x6E\x48\xB0\x2E\x69\x63"
-			  "\x32\xAA\xAC\x26\x55\x45\x94\xDE"
-			  "\x30\x26\x26\xE6\x08\x82\x2F\x5F"
-			  "\xA7\x15\x94\x07\x75\x2D\xC6\x3A"
-			  "\x1B\xA0\x39\xFB\xBA\xB9\x06\x56"
-			  "\xF6\x9F\xF1\x2F\x9B\xF3\x89\x8B"
-			  "\x08\xC8\x9D\x5E\x6B\x95\x09\xC7"
-			  "\x98\xB7\x62\xA4\x1D\x25\xFA\xC5"
-			  "\x62\xC8\x5D\x6B\xB4\x85\x88\x7F"
-			  "\x3B\x29\xF9\xB4\x32\x62\x69\xBF"
-			  "\x32\xB8\xEB\xFD\x0E\x26\xAA\xA3"
-			  "\x44\x67\x90\x20\xAC\x41\xDF\x43"
-			  "\xC6\xC7\x19\x9F\x2C\x28\x74\xEB"
-			  "\x3E\x7F\x7A\x80\x5B\xE4\x08\x60"
-			  "\xC7\xC9\x71\x34\x44\xCE\x05\xFD"
-			  "\xA8\x91\xA8\x44\x5E\xD3\x89\x2C"
-			  "\xAE\x59\x0F\x07\x88\x79\x53\x26"
-			  "\xAF\xAC\xCB\x1D\x6F\x08\x25\x62"
-			  "\xD0\x82\x65\x66\xE4\x2A\x29\x1C"
-			  "\x9C\x64\x5F\x49\x9D\xF8\x62\xF9"
-			  "\xED\xC4\x13\x52\x75\xDC\xE4\xF9"
-			  "\x68\x0F\x8A\xCD\xA6\x8D\x75\xAA"
-			  "\x49\xA1\x86\x86\x37\x5C\x6B\x3D"
-			  "\x56\xE5\x6F\xBE\x27\xC0\x10\xF8"
-			  "\x3C\x4D\x17\x35\x14\xDC\x1C\xA0"
-			  "\x6E\xAE\xD1\x10\xDD\x83\x06\xC2"
-			  "\x23\xD3\xC7\x27\x15\x04\x2C\x27"
-			  "\xDD\x1F\x2E\x97\x09\x9C\x33\x7D"
-			  "\xAC\x50\x1B\x2E\xC9\x52\x0C\x14"
-			  "\x4B\x78\xC4\xDE\x07\x6A\x12\x02"
-			  "\x6E\xD7\x4B\x91\xB9\x88\x4D\x02"
-			  "\xC3\xB5\x04\xBC\xE0\x67\xCA\x18"
-			  "\x22\xA1\xAE\x9A\x21\xEF\xB2\x06"
-			  "\x35\xCD\xEC\x37\x70\x2D\xFC\x1E"
-			  "\xA8\x31\xE7\xFC\xE5\x8E\x88\x66"
-			  "\x16\xB5\xC8\x45\x21\x37\xBD\x24"
-			  "\xA9\xD5\x36\x12\x9F\x6E\x67\x80"
-			  "\x87\x54\xD5\xAF\x97\xE1\x15\xA7"
-			  "\x11\xF0\x63\x7B\xE1\x44\x14\x1C"
-			  "\x06\x32\x05\x8C\x6C\xDB\x9B\x36"
-			  "\x6A\x6B\xAD\x3A\x27\x55\x20\x4C"
-			  "\x76\x36\x43\xE8\x16\x60\xB5\xF3"
-			  "\xDF\x5A\xC6\xA5\x69\x78\x59\x51"
-			  "\x54\x68\x65\x06\x84\xDE\x3D\xAE"
-			  "\x38\x91\xBD\xCC\xA2\x8A\xEC\xE6"
-			  "\x9E\x83\xAE\x1E\x8E\x34\x5D\xDE"
-			  "\x91\xCE\x8F\xED\x40\xF7\xC8\x8B"
-			  "\x9A\x13\x4C\xAD\x89\x97\x9E\xD1"
-			  "\x91\x01\xD7\x21\x23\x28\x1E\xCC"
-			  "\x8C\x98\xDB\xDE\xFC\x72\x94\xAA"
-			  "\xC0\x0D\x96\xAA\x23\xF8\xFE\x13",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec cast5_ctr_dec_tv_template[] = {
-	{ /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
-		.klen	= 16,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\xFF\xC4\x2E\x82\x3D\xF8\xA8\x39"
-			  "\x7C\x52\xC4\xD3\xBB\x62\xC6\xA8"
-			  "\x0C",
-		.ilen	= 17,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A",
-		.rlen	= 17,
-	}, { /* Generated from TF test vectors */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
-		.klen	= 16,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
-		.input	= "\xFF\xC4\x2E\x82\x3D\xF8\xA8\x39"
-			  "\x7C\x52\xC4\xD3\xBB\x62\xC6\xA8"
-			  "\x0C\x63\xA5\x55\xE3\xF8\x1C\x7F"
-			  "\xDC\x59\xF9\xA0\x52\xAD\x83\xDF"
-			  "\xD5\x3B\x53\x4A\xAA\x1F\x49\x44"
-			  "\xE8\x20\xCC\xF8\x97\xE6\xE0\x3C"
-			  "\x5A\xD2\x83\xEC\xEE\x25\x3F\xCF"
-			  "\x0D\xC2\x79\x80\x99\x6E\xFF\x7B"
-			  "\x64\xB0\x7B\x86\x29\x1D\x9F\x17"
-			  "\x10\xA5\xA5\xEB\x16\x55\x9E\xE3"
-			  "\x88\x18\x52\x56\x48\x58\xD1\x6B"
-			  "\xE8\x74\x6E\x48\xB0\x2E\x69\x63"
-			  "\x32\xAA\xAC\x26\x55\x45\x94\xDE"
-			  "\x30\x26\x26\xE6\x08\x82\x2F\x5F"
-			  "\xA7\x15\x94\x07\x75\x2D\xC6\x3A"
-			  "\x1B\xA0\x39\xFB\xBA\xB9\x06\x56"
-			  "\xF6\x9F\xF1\x2F\x9B\xF3\x89\x8B"
-			  "\x08\xC8\x9D\x5E\x6B\x95\x09\xC7"
-			  "\x98\xB7\x62\xA4\x1D\x25\xFA\xC5"
-			  "\x62\xC8\x5D\x6B\xB4\x85\x88\x7F"
-			  "\x3B\x29\xF9\xB4\x32\x62\x69\xBF"
-			  "\x32\xB8\xEB\xFD\x0E\x26\xAA\xA3"
-			  "\x44\x67\x90\x20\xAC\x41\xDF\x43"
-			  "\xC6\xC7\x19\x9F\x2C\x28\x74\xEB"
-			  "\x3E\x7F\x7A\x80\x5B\xE4\x08\x60"
-			  "\xC7\xC9\x71\x34\x44\xCE\x05\xFD"
-			  "\xA8\x91\xA8\x44\x5E\xD3\x89\x2C"
-			  "\xAE\x59\x0F\x07\x88\x79\x53\x26"
-			  "\xAF\xAC\xCB\x1D\x6F\x08\x25\x62"
-			  "\xD0\x82\x65\x66\xE4\x2A\x29\x1C"
-			  "\x9C\x64\x5F\x49\x9D\xF8\x62\xF9"
-			  "\xED\xC4\x13\x52\x75\xDC\xE4\xF9"
-			  "\x68\x0F\x8A\xCD\xA6\x8D\x75\xAA"
-			  "\x49\xA1\x86\x86\x37\x5C\x6B\x3D"
-			  "\x56\xE5\x6F\xBE\x27\xC0\x10\xF8"
-			  "\x3C\x4D\x17\x35\x14\xDC\x1C\xA0"
-			  "\x6E\xAE\xD1\x10\xDD\x83\x06\xC2"
-			  "\x23\xD3\xC7\x27\x15\x04\x2C\x27"
-			  "\xDD\x1F\x2E\x97\x09\x9C\x33\x7D"
-			  "\xAC\x50\x1B\x2E\xC9\x52\x0C\x14"
-			  "\x4B\x78\xC4\xDE\x07\x6A\x12\x02"
-			  "\x6E\xD7\x4B\x91\xB9\x88\x4D\x02"
-			  "\xC3\xB5\x04\xBC\xE0\x67\xCA\x18"
-			  "\x22\xA1\xAE\x9A\x21\xEF\xB2\x06"
-			  "\x35\xCD\xEC\x37\x70\x2D\xFC\x1E"
-			  "\xA8\x31\xE7\xFC\xE5\x8E\x88\x66"
-			  "\x16\xB5\xC8\x45\x21\x37\xBD\x24"
-			  "\xA9\xD5\x36\x12\x9F\x6E\x67\x80"
-			  "\x87\x54\xD5\xAF\x97\xE1\x15\xA7"
-			  "\x11\xF0\x63\x7B\xE1\x44\x14\x1C"
-			  "\x06\x32\x05\x8C\x6C\xDB\x9B\x36"
-			  "\x6A\x6B\xAD\x3A\x27\x55\x20\x4C"
-			  "\x76\x36\x43\xE8\x16\x60\xB5\xF3"
-			  "\xDF\x5A\xC6\xA5\x69\x78\x59\x51"
-			  "\x54\x68\x65\x06\x84\xDE\x3D\xAE"
-			  "\x38\x91\xBD\xCC\xA2\x8A\xEC\xE6"
-			  "\x9E\x83\xAE\x1E\x8E\x34\x5D\xDE"
-			  "\x91\xCE\x8F\xED\x40\xF7\xC8\x8B"
-			  "\x9A\x13\x4C\xAD\x89\x97\x9E\xD1"
-			  "\x91\x01\xD7\x21\x23\x28\x1E\xCC"
-			  "\x8C\x98\xDB\xDE\xFC\x72\x94\xAA"
-			  "\xC0\x0D\x96\xAA\x23\xF8\xFE\x13",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 496 - 20, 4, 16 },
-	},
-};
-
-/*
- * ARC4 test vectors from OpenSSL
- */
-static const struct cipher_testvec arc4_enc_tv_template[] = {
-	{
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.ilen	= 8,
-		.result	= "\x75\xb7\x87\x80\x99\xe0\xc5\x96",
-		.rlen	= 8,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 8,
-		.result	= "\x74\x94\xc2\xe7\x10\x4b\x08\x79",
-		.rlen	= 8,
-	}, {
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 8,
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 8,
-		.result	= "\xde\x18\x89\x41\xa3\x37\x5d\x3a",
-		.rlen	= 8,
-	}, {
-		.key	= "\xef\x01\x23\x45",
-		.klen	= 4,
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00",
-		.ilen	= 20,
-		.result	= "\xd6\xa1\x41\xa7\xec\x3c\x38\xdf"
-			  "\xbd\x61\x5a\x11\x62\xe1\xc7\xba"
-			  "\x36\xb6\x78\x58",
-		.rlen	= 20,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.input	= "\x12\x34\x56\x78\x9A\xBC\xDE\xF0"
-			  "\x12\x34\x56\x78\x9A\xBC\xDE\xF0"
-			  "\x12\x34\x56\x78\x9A\xBC\xDE\xF0"
-			  "\x12\x34\x56\x78",
-		.ilen	= 28,
-		.result	= "\x66\xa0\x94\x9f\x8a\xf7\xd6\x89"
-			  "\x1f\x7f\x83\x2b\xa8\x33\xc0\x0c"
-			  "\x89\x2e\xbe\x30\x14\x3c\xe2\x87"
-			  "\x40\x01\x1e\xcf",
-		.rlen	= 28,
-	}, {
-		.key	= "\xef\x01\x23\x45",
-		.klen	= 4,
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00",
-		.ilen	= 10,
-		.result	= "\xd6\xa1\x41\xa7\xec\x3c\x38\xdf"
-			  "\xbd\x61",
-		.rlen	= 10,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
-			"\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 16,
-		.input	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF",
-		.ilen	= 8,
-		.result	= "\x69\x72\x36\x59\x1B\x52\x42\xB1",
-		.rlen	= 8,
-	},
-};
-
-static const struct cipher_testvec arc4_dec_tv_template[] = {
-	{
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.input	= "\x75\xb7\x87\x80\x99\xe0\xc5\x96",
-		.ilen	= 8,
-		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.rlen	= 8,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.input	= "\x74\x94\xc2\xe7\x10\x4b\x08\x79",
-		.ilen	= 8,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen	= 8,
-	}, {
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 8,
-		.input	= "\xde\x18\x89\x41\xa3\x37\x5d\x3a",
-		.ilen	= 8,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen	= 8,
-	}, {
-		.key	= "\xef\x01\x23\x45",
-		.klen	= 4,
-		.input	= "\xd6\xa1\x41\xa7\xec\x3c\x38\xdf"
-			  "\xbd\x61\x5a\x11\x62\xe1\xc7\xba"
-			  "\x36\xb6\x78\x58",
-		.ilen	= 20,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00",
-		.rlen	= 20,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
-		.klen	= 8,
-		.input	= "\x66\xa0\x94\x9f\x8a\xf7\xd6\x89"
-			  "\x1f\x7f\x83\x2b\xa8\x33\xc0\x0c"
-			  "\x89\x2e\xbe\x30\x14\x3c\xe2\x87"
-			  "\x40\x01\x1e\xcf",
-		.ilen	= 28,
-		.result	= "\x12\x34\x56\x78\x9A\xBC\xDE\xF0"
-			  "\x12\x34\x56\x78\x9A\xBC\xDE\xF0"
-			  "\x12\x34\x56\x78\x9A\xBC\xDE\xF0"
-			  "\x12\x34\x56\x78",
-		.rlen	= 28,
-	}, {
-		.key	= "\xef\x01\x23\x45",
-		.klen	= 4,
-		.input	= "\xd6\xa1\x41\xa7\xec\x3c\x38\xdf"
-			  "\xbd\x61",
-		.ilen	= 10,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00",
-		.rlen	= 10,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
-			"\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 16,
-		.input	= "\x69\x72\x36\x59\x1B\x52\x42\xB1",
-		.ilen	= 8,
-		.result	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF",
-		.rlen	= 8,
-	},
-};
-
-/*
- * TEA test vectors
- */
-static const struct cipher_testvec tea_enc_tv_template[] = {
-	{
-		.key    = zeroed_string,
-		.klen	= 16,
-		.input  = zeroed_string,
-		.ilen	= 8,
-		.result	= "\x0a\x3a\xea\x41\x40\xa9\xba\x94",
-		.rlen	= 8,
-	}, {
-		.key	= "\x2b\x02\x05\x68\x06\x14\x49\x76"
-			  "\x77\x5d\x0e\x26\x6c\x28\x78\x43",
-		.klen	= 16,
-		.input	= "\x74\x65\x73\x74\x20\x6d\x65\x2e",
-		.ilen	= 8,
-		.result	= "\x77\x5d\x2a\x6a\xf6\xce\x92\x09",
-		.rlen	= 8,
-	}, {
-		.key	= "\x09\x65\x43\x11\x66\x44\x39\x25"
-			  "\x51\x3a\x16\x10\x0a\x08\x12\x6e",
-		.klen	= 16,
-		.input	= "\x6c\x6f\x6e\x67\x65\x72\x5f\x74"
-			  "\x65\x73\x74\x5f\x76\x65\x63\x74",
-		.ilen	= 16,
-		.result	= "\xbe\x7a\xbb\x81\x95\x2d\x1f\x1e"
-			  "\xdd\x89\xa1\x25\x04\x21\xdf\x95",
-		.rlen	= 16,
-	}, {
-		.key	= "\x4d\x76\x32\x17\x05\x3f\x75\x2c"
-			  "\x5d\x04\x16\x36\x15\x72\x63\x2f",
-		.klen	= 16,
-		.input	= "\x54\x65\x61\x20\x69\x73\x20\x67"
-			  "\x6f\x6f\x64\x20\x66\x6f\x72\x20"
-			  "\x79\x6f\x75\x21\x21\x21\x20\x72"
-			  "\x65\x61\x6c\x6c\x79\x21\x21\x21",
-		.ilen	= 32,
-		.result	= "\xe0\x4d\x5d\x3c\xb7\x8c\x36\x47"
-			  "\x94\x18\x95\x91\xa9\xfc\x49\xf8"
-			  "\x44\xd1\x2d\xc2\x99\xb8\x08\x2a"
-			  "\x07\x89\x73\xc2\x45\x92\xc6\x90",
-		.rlen	= 32,
-	}
-};
-
-static const struct cipher_testvec tea_dec_tv_template[] = {
-	{
-		.key    = zeroed_string,
-		.klen	= 16,
-		.input	= "\x0a\x3a\xea\x41\x40\xa9\xba\x94",
-		.ilen	= 8,
-		.result = zeroed_string,
-		.rlen	= 8,
-	}, {
-		.key	= "\x2b\x02\x05\x68\x06\x14\x49\x76"
-			  "\x77\x5d\x0e\x26\x6c\x28\x78\x43",
-		.klen	= 16,
-		.input	= "\x77\x5d\x2a\x6a\xf6\xce\x92\x09",
-		.ilen	= 8,
-		.result	= "\x74\x65\x73\x74\x20\x6d\x65\x2e",
-		.rlen	= 8,
-	}, {
-		.key	= "\x09\x65\x43\x11\x66\x44\x39\x25"
-			  "\x51\x3a\x16\x10\x0a\x08\x12\x6e",
-		.klen	= 16,
-		.input	= "\xbe\x7a\xbb\x81\x95\x2d\x1f\x1e"
-			  "\xdd\x89\xa1\x25\x04\x21\xdf\x95",
-		.ilen   = 16,
-		.result	= "\x6c\x6f\x6e\x67\x65\x72\x5f\x74"
-			  "\x65\x73\x74\x5f\x76\x65\x63\x74",
-		.rlen	= 16,
-	}, {
-		.key	= "\x4d\x76\x32\x17\x05\x3f\x75\x2c"
-			  "\x5d\x04\x16\x36\x15\x72\x63\x2f",
-		.klen	= 16,
-		.input	= "\xe0\x4d\x5d\x3c\xb7\x8c\x36\x47"
-			  "\x94\x18\x95\x91\xa9\xfc\x49\xf8"
-			  "\x44\xd1\x2d\xc2\x99\xb8\x08\x2a"
-			  "\x07\x89\x73\xc2\x45\x92\xc6\x90",
-		.ilen	= 32,
-		.result	= "\x54\x65\x61\x20\x69\x73\x20\x67"
-			  "\x6f\x6f\x64\x20\x66\x6f\x72\x20"
-			  "\x79\x6f\x75\x21\x21\x21\x20\x72"
-			  "\x65\x61\x6c\x6c\x79\x21\x21\x21",
-		.rlen	= 32,
-	}
-};
-
-/*
- * XTEA test vectors
- */
-static const struct cipher_testvec xtea_enc_tv_template[] = {
-	{
-		.key    = zeroed_string,
-		.klen	= 16,
-		.input  = zeroed_string,
-		.ilen	= 8,
-		.result	= "\xd8\xd4\xe9\xde\xd9\x1e\x13\xf7",
-		.rlen	= 8,
-	}, {
-		.key	= "\x2b\x02\x05\x68\x06\x14\x49\x76"
-			  "\x77\x5d\x0e\x26\x6c\x28\x78\x43",
-		.klen	= 16,
-		.input	= "\x74\x65\x73\x74\x20\x6d\x65\x2e",
-		.ilen	= 8,
-		.result	= "\x94\xeb\xc8\x96\x84\x6a\x49\xa8",
-		.rlen	= 8,
-	}, {
-		.key	= "\x09\x65\x43\x11\x66\x44\x39\x25"
-			  "\x51\x3a\x16\x10\x0a\x08\x12\x6e",
-		.klen	= 16,
-		.input	= "\x6c\x6f\x6e\x67\x65\x72\x5f\x74"
-			  "\x65\x73\x74\x5f\x76\x65\x63\x74",
-		.ilen	= 16,
-		.result	= "\x3e\xce\xae\x22\x60\x56\xa8\x9d"
-			  "\x77\x4d\xd4\xb4\x87\x24\xe3\x9a",
-		.rlen	= 16,
-	}, {
-		.key	= "\x4d\x76\x32\x17\x05\x3f\x75\x2c"
-			  "\x5d\x04\x16\x36\x15\x72\x63\x2f",
-		.klen	= 16,
-		.input	= "\x54\x65\x61\x20\x69\x73\x20\x67"
-			  "\x6f\x6f\x64\x20\x66\x6f\x72\x20"
-			  "\x79\x6f\x75\x21\x21\x21\x20\x72"
-			  "\x65\x61\x6c\x6c\x79\x21\x21\x21",
-		.ilen	= 32,
-		.result	= "\x99\x81\x9f\x5d\x6f\x4b\x31\x3a"
-			  "\x86\xff\x6f\xd0\xe3\x87\x70\x07"
-			  "\x4d\xb8\xcf\xf3\x99\x50\xb3\xd4"
-			  "\x73\xa2\xfa\xc9\x16\x59\x5d\x81",
-		.rlen	= 32,
-	}
-};
-
-static const struct cipher_testvec xtea_dec_tv_template[] = {
-	{
-		.key    = zeroed_string,
-		.klen	= 16,
-		.input	= "\xd8\xd4\xe9\xde\xd9\x1e\x13\xf7",
-		.ilen	= 8,
-		.result = zeroed_string,
-		.rlen	= 8,
-	}, {
-		.key	= "\x2b\x02\x05\x68\x06\x14\x49\x76"
-			  "\x77\x5d\x0e\x26\x6c\x28\x78\x43",
-		.klen	= 16,
-		.input	= "\x94\xeb\xc8\x96\x84\x6a\x49\xa8",
-		.ilen	= 8,
-		.result	= "\x74\x65\x73\x74\x20\x6d\x65\x2e",
-		.rlen	= 8,
-	}, {
-		.key	= "\x09\x65\x43\x11\x66\x44\x39\x25"
-			  "\x51\x3a\x16\x10\x0a\x08\x12\x6e",
-		.klen	= 16,
-		.input	= "\x3e\xce\xae\x22\x60\x56\xa8\x9d"
-			  "\x77\x4d\xd4\xb4\x87\x24\xe3\x9a",
-		.ilen	= 16,
-		.result	= "\x6c\x6f\x6e\x67\x65\x72\x5f\x74"
-			  "\x65\x73\x74\x5f\x76\x65\x63\x74",
-		.rlen	= 16,
-	}, {
-		.key	= "\x4d\x76\x32\x17\x05\x3f\x75\x2c"
-			  "\x5d\x04\x16\x36\x15\x72\x63\x2f",
-		.klen	= 16,
-		.input	= "\x99\x81\x9f\x5d\x6f\x4b\x31\x3a"
-			  "\x86\xff\x6f\xd0\xe3\x87\x70\x07"
-			  "\x4d\xb8\xcf\xf3\x99\x50\xb3\xd4"
-			  "\x73\xa2\xfa\xc9\x16\x59\x5d\x81",
-		.ilen	= 32,
-		.result	= "\x54\x65\x61\x20\x69\x73\x20\x67"
-			  "\x6f\x6f\x64\x20\x66\x6f\x72\x20"
-			  "\x79\x6f\x75\x21\x21\x21\x20\x72"
-			  "\x65\x61\x6c\x6c\x79\x21\x21\x21",
-		.rlen	= 32,
-	}
-};
-
-/*
- * KHAZAD test vectors.
- */
-static const struct cipher_testvec khazad_enc_tv_template[] = {
-	{
-		.key	= "\x80\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 16,
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 8,
-		.result	= "\x49\xa4\xce\x32\xac\x19\x0e\x3f",
-		.rlen	= 8,
-	}, {
-		.key	= "\x38\x38\x38\x38\x38\x38\x38\x38"
-			  "\x38\x38\x38\x38\x38\x38\x38\x38",
-		.klen	= 16,
-		.input	= "\x38\x38\x38\x38\x38\x38\x38\x38",
-		.ilen	= 8,
-		.result	= "\x7e\x82\x12\xa1\xd9\x5b\xe4\xf9",
-		.rlen	= 8,
-	}, {
-		.key	= "\xa2\xa2\xa2\xa2\xa2\xa2\xa2\xa2"
-			"\xa2\xa2\xa2\xa2\xa2\xa2\xa2\xa2",
-		.klen	= 16,
-		.input	= "\xa2\xa2\xa2\xa2\xa2\xa2\xa2\xa2",
-		.ilen	= 8,
-		.result	= "\xaa\xbe\xc1\x95\xc5\x94\x1a\x9c",
-		.rlen	= 8,
-	}, {
-		.key	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f"
-			"\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
-		.klen	= 16,
-		.input	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
-		.ilen	= 8,
-		.result = "\x04\x74\xf5\x70\x50\x16\xd3\xb8",
-		.rlen	= 8,
-	}, {
-		.key	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f"
-			"\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
-		.klen	= 16,
-		.input	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f"
-			"\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
-		.ilen	= 16,
-		.result = "\x04\x74\xf5\x70\x50\x16\xd3\xb8"
-			"\x04\x74\xf5\x70\x50\x16\xd3\xb8",
-		.rlen	= 16,
-	},
-};
-
-static const struct cipher_testvec khazad_dec_tv_template[] = {
-	{
-		.key	= "\x80\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 16,
-		.input	= "\x49\xa4\xce\x32\xac\x19\x0e\x3f",
-		.ilen	= 8,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen	= 8,
-	}, {
-		.key	= "\x38\x38\x38\x38\x38\x38\x38\x38"
-			  "\x38\x38\x38\x38\x38\x38\x38\x38",
-		.klen	= 16,
-		.input	= "\x7e\x82\x12\xa1\xd9\x5b\xe4\xf9",
-		.ilen	= 8,
-		.result	= "\x38\x38\x38\x38\x38\x38\x38\x38",
-		.rlen	= 8,
-	}, {
-		.key	= "\xa2\xa2\xa2\xa2\xa2\xa2\xa2\xa2"
-			"\xa2\xa2\xa2\xa2\xa2\xa2\xa2\xa2",
-		.klen	= 16,
-		.input	= "\xaa\xbe\xc1\x95\xc5\x94\x1a\x9c",
-		.ilen	= 8,
-		.result	= "\xa2\xa2\xa2\xa2\xa2\xa2\xa2\xa2",
-		.rlen	= 8,
-	}, {
-		.key	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f"
-			"\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
-		.klen	= 16,
-		.input  = "\x04\x74\xf5\x70\x50\x16\xd3\xb8",
-		.ilen	= 8,
-		.result	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
-		.rlen	= 8,
-	}, {
-		.key	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f"
-			"\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
-		.klen	= 16,
-		.input  = "\x04\x74\xf5\x70\x50\x16\xd3\xb8"
-			"\x04\x74\xf5\x70\x50\x16\xd3\xb8",
-		.ilen	= 16,
-		.result	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f"
-			"\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
-		.rlen	= 16,
-	},
-};
-
-/*
- * Anubis test vectors.
- */
-
-static const struct cipher_testvec anubis_enc_tv_template[] = {
-	{
-		.key	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
-		.klen	= 16,
-		.input	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
-		.ilen	= 16,
-		.result	= "\x6d\xc5\xda\xa2\x26\x7d\x62\x6f"
-			  "\x08\xb7\x52\x8e\x6e\x6e\x86\x90",
-		.rlen	= 16,
-	}, {
-
-		.key	= "\x03\x03\x03\x03\x03\x03\x03\x03"
-			  "\x03\x03\x03\x03\x03\x03\x03\x03"
-			  "\x03\x03\x03\x03",
-		.klen	= 20,
-		.input	= "\x03\x03\x03\x03\x03\x03\x03\x03"
-			  "\x03\x03\x03\x03\x03\x03\x03\x03",
-		.ilen	= 16,
-		.result	= "\xdb\xf1\x42\xf4\xd1\x8a\xc7\x49"
-			  "\x87\x41\x6f\x82\x0a\x98\x64\xae",
-		.rlen	= 16,
-	}, {
-		.key	= "\x24\x24\x24\x24\x24\x24\x24\x24"
-			  "\x24\x24\x24\x24\x24\x24\x24\x24"
-			  "\x24\x24\x24\x24\x24\x24\x24\x24"
-			  "\x24\x24\x24\x24",
-		.klen	= 28,
-		.input	= "\x24\x24\x24\x24\x24\x24\x24\x24"
-			  "\x24\x24\x24\x24\x24\x24\x24\x24",
-		.ilen	= 16,
-		.result	= "\xfd\x1b\x4a\xe3\xbf\xf0\xad\x3d"
-			  "\x06\xd3\x61\x27\xfd\x13\x9e\xde",
-		.rlen	= 16,
-	}, {
-		.key	= "\x25\x25\x25\x25\x25\x25\x25\x25"
-			  "\x25\x25\x25\x25\x25\x25\x25\x25"
-			  "\x25\x25\x25\x25\x25\x25\x25\x25"
-			  "\x25\x25\x25\x25\x25\x25\x25\x25",
-		.klen	= 32,
-		.input	= "\x25\x25\x25\x25\x25\x25\x25\x25"
-			  "\x25\x25\x25\x25\x25\x25\x25\x25",
-		.ilen	= 16,
-		.result	= "\x1a\x91\xfb\x2b\xb7\x78\x6b\xc4"
-			"\x17\xd9\xff\x40\x3b\x0e\xe5\xfe",
-		.rlen	= 16,
-	}, {
-		.key	= "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35",
-		.klen	= 40,
-		.input	= "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35",
-		.ilen	= 16,
-		.result = "\xa5\x2c\x85\x6f\x9c\xba\xa0\x97"
-			  "\x9e\xc6\x84\x0f\x17\x21\x07\xee",
-		.rlen	= 16,
-	},
-};
-
-static const struct cipher_testvec anubis_dec_tv_template[] = {
-	{
-		.key	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
-		.klen	= 16,
-		.input	= "\x6d\xc5\xda\xa2\x26\x7d\x62\x6f"
-			  "\x08\xb7\x52\x8e\x6e\x6e\x86\x90",
-		.ilen	= 16,
-		.result	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
-		.rlen	= 16,
-	}, {
-
-		.key	= "\x03\x03\x03\x03\x03\x03\x03\x03"
-			  "\x03\x03\x03\x03\x03\x03\x03\x03"
-			  "\x03\x03\x03\x03",
-		.klen	= 20,
-		.input	= "\xdb\xf1\x42\xf4\xd1\x8a\xc7\x49"
-			  "\x87\x41\x6f\x82\x0a\x98\x64\xae",
-		.ilen	= 16,
-		.result	= "\x03\x03\x03\x03\x03\x03\x03\x03"
-			  "\x03\x03\x03\x03\x03\x03\x03\x03",
-		.rlen	= 16,
-	}, {
-		.key	= "\x24\x24\x24\x24\x24\x24\x24\x24"
-			  "\x24\x24\x24\x24\x24\x24\x24\x24"
-			  "\x24\x24\x24\x24\x24\x24\x24\x24"
-			  "\x24\x24\x24\x24",
-		.klen	= 28,
-		.input	= "\xfd\x1b\x4a\xe3\xbf\xf0\xad\x3d"
-			  "\x06\xd3\x61\x27\xfd\x13\x9e\xde",
-		.ilen	= 16,
-		.result	= "\x24\x24\x24\x24\x24\x24\x24\x24"
-			  "\x24\x24\x24\x24\x24\x24\x24\x24",
-		.rlen	= 16,
-	}, {
-		.key	= "\x25\x25\x25\x25\x25\x25\x25\x25"
-			  "\x25\x25\x25\x25\x25\x25\x25\x25"
-			  "\x25\x25\x25\x25\x25\x25\x25\x25"
-			  "\x25\x25\x25\x25\x25\x25\x25\x25",
-		.klen	= 32,
-		.input	= "\x1a\x91\xfb\x2b\xb7\x78\x6b\xc4"
-			"\x17\xd9\xff\x40\x3b\x0e\xe5\xfe",
-		.ilen	= 16,
-		.result	= "\x25\x25\x25\x25\x25\x25\x25\x25"
-			  "\x25\x25\x25\x25\x25\x25\x25\x25",
-		.rlen	= 16,
-	}, {
-		.key	= "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35",
-		.input = "\xa5\x2c\x85\x6f\x9c\xba\xa0\x97"
-			 "\x9e\xc6\x84\x0f\x17\x21\x07\xee",
-		.klen	= 40,
-		.ilen	= 16,
-		.result	= "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35",
-		.rlen	= 16,
-	},
-};
-
-static const struct cipher_testvec anubis_cbc_enc_tv_template[] = {
-	{
-		.key	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
-		.klen	= 16,
-		.input	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
-		.ilen	= 32,
-		.result	= "\x6d\xc5\xda\xa2\x26\x7d\x62\x6f"
-			  "\x08\xb7\x52\x8e\x6e\x6e\x86\x90"
-			  "\x86\xd8\xb5\x6f\x98\x5e\x8a\x66"
-			  "\x4f\x1f\x78\xa1\xbb\x37\xf1\xbe",
-		.rlen	= 32,
-	}, {
-		.key	= "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35",
-		.klen	= 40,
-		.input	= "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35",
-		.ilen	= 32,
-		.result = "\xa5\x2c\x85\x6f\x9c\xba\xa0\x97"
-			  "\x9e\xc6\x84\x0f\x17\x21\x07\xee"
-			  "\xa2\xbc\x06\x98\xc6\x4b\xda\x75"
-			  "\x2e\xaa\xbe\x58\xce\x01\x5b\xc7",
-		.rlen	= 32,
-	},
-};
-
-static const struct cipher_testvec anubis_cbc_dec_tv_template[] = {
-	{
-		.key	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
-		.klen	= 16,
-		.input	= "\x6d\xc5\xda\xa2\x26\x7d\x62\x6f"
-			  "\x08\xb7\x52\x8e\x6e\x6e\x86\x90"
-			  "\x86\xd8\xb5\x6f\x98\x5e\x8a\x66"
-			  "\x4f\x1f\x78\xa1\xbb\x37\xf1\xbe",
-		.ilen	= 32,
-		.result	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
-			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
-		.rlen	= 32,
-	}, {
-		.key	= "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35",
-		.klen	= 40,
-		.input = "\xa5\x2c\x85\x6f\x9c\xba\xa0\x97"
-			  "\x9e\xc6\x84\x0f\x17\x21\x07\xee"
-			  "\xa2\xbc\x06\x98\xc6\x4b\xda\x75"
-			  "\x2e\xaa\xbe\x58\xce\x01\x5b\xc7",
-		.ilen	= 32,
-		.result	= "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35"
-			  "\x35\x35\x35\x35\x35\x35\x35\x35",
-		.rlen	= 32,
-	},
-};
-
-/*
- * XETA test vectors
- */
-static const struct cipher_testvec xeta_enc_tv_template[] = {
-	{
-		.key    = zeroed_string,
-		.klen	= 16,
-		.input  = zeroed_string,
-		.ilen	= 8,
-		.result	= "\xaa\x22\x96\xe5\x6c\x61\xf3\x45",
-		.rlen	= 8,
-	}, {
-		.key	= "\x2b\x02\x05\x68\x06\x14\x49\x76"
-			  "\x77\x5d\x0e\x26\x6c\x28\x78\x43",
-		.klen	= 16,
-		.input	= "\x74\x65\x73\x74\x20\x6d\x65\x2e",
-		.ilen	= 8,
-		.result	= "\x82\x3e\xeb\x35\xdc\xdd\xd9\xc3",
-		.rlen	= 8,
-	}, {
-		.key	= "\x09\x65\x43\x11\x66\x44\x39\x25"
-			  "\x51\x3a\x16\x10\x0a\x08\x12\x6e",
-		.klen	= 16,
-		.input	= "\x6c\x6f\x6e\x67\x65\x72\x5f\x74"
-			  "\x65\x73\x74\x5f\x76\x65\x63\x74",
-		.ilen	= 16,
-		.result	= "\xe2\x04\xdb\xf2\x89\x85\x9e\xea"
-			  "\x61\x35\xaa\xed\xb5\xcb\x71\x2c",
-		.rlen	= 16,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ctext	= "\xa9\x78\xae\x1e\xea\xa2\x44\x4c"
+			  "\xa2\x7a\x64\x1f\xaf\x46\xc1\xe0"
+			  "\x6c\xb2\xf3\x92\x9a\xd6\x7d\x58"
+			  "\xb8\x2d\xb9\x5d\x58\x07\x66\x50"
+			  "\xea\x35\x35\x8c\xb2\x46\x61\x06"
+			  "\x5d\x65\xfc\x57\x8f\x69\x74\xab"
+			  "\x8a\x06\x69\xb5\x6c\xda\x66\xc7"
+			  "\x52\x90\xbb\x8e\x6d\x8b\xb5\xa2"
+			  "\x78\x1d\xc2\xa9\xc2\x73\x00\xc3"
+			  "\x32\x36\x7c\x97\x6b\x4e\x8a\x50"
+			  "\xe4\x91\x83\x96\x8f\xf4\x94\x1a"
+			  "\xa6\x27\xe1\x33\xcb\x91\xc6\x5f"
+			  "\x94\x75\xbc\xd7\x3e\x3e\x6f\x9e"
+			  "\xa9\x31\x80\x5e\xe5\xdb\xc8\x53"
+			  "\x01\x73\x68\x32\x25\x19\xfa\xfb"
+			  "\xe4\xcf\xb9\x3e\xa2\xa0\x8f\x31"
+			  "\xbf\x54\x06\x93\xa8\xb1\x0f\xb6"
+			  "\x7c\x3c\xde\x6f\x0f\xfb\x0c\x11"
+			  "\x39\x80\x39\x09\x97\x65\xf2\x83"
+			  "\xae\xe6\xa1\x6f\x47\xb8\x49\xde"
+			  "\x99\x36\x20\x7d\x97\x3b\xec\xfa"
+			  "\xb4\x33\x6e\x7a\xc7\x46\x84\x49"
+			  "\x91\xcd\xe1\x57\x0d\xed\x40\x08"
+			  "\x13\xf1\x4e\x3e\xa4\xa4\x5c\xe6"
+			  "\xd2\x0c\x20\x8f\x3e\xdf\x3f\x47"
+			  "\x9a\x2f\xde\x6d\x66\xc9\x99\x4a"
+			  "\x2d\x9e\x9d\x4b\x1a\x27\xa2\x12"
+			  "\x99\xf0\xf8\xb1\xb6\xf6\x57\xc3"
+			  "\xca\x1c\xa3\x8e\xed\x39\x28\xb5"
+			  "\x10\x1b\x4b\x08\x42\x00\x4a\xd3"
+			  "\xad\x5a\xc6\x8e\xc8\xbb\x95\xc4"
+			  "\x4b\xaa\xfe\xd5\x42\xa8\xa3\x6d"
+			  "\x3c\xf3\x34\x91\x2d\xb4\xdd\x20"
+			  "\x0c\x90\x6d\xa3\x9b\x66\x9d\x24"
+			  "\x02\xa6\xa9\x3f\x3f\x58\x5d\x47"
+			  "\x24\x65\x63\x7e\xbd\x8c\xe6\x52"
+			  "\x7d\xef\x33\x53\x63\xec\xaa\x0b"
+			  "\x64\x15\xa9\xa6\x1f\x10\x00\x38"
+			  "\x35\xa8\xe7\xbe\x23\x70\x22\xe0"
+			  "\xd3\xb9\xe6\xfd\xe6\xaa\x03\x50"
+			  "\xf3\x3c\x27\x36\x8b\xcc\xfe\x9c"
+			  "\x9c\xa3\xb3\xe7\x68\x9b\xa2\x71"
+			  "\xe0\x07\xd9\x1f\x68\x1f\xac\x5e"
+			  "\x7a\x74\x85\xa9\x6a\x90\xab\x2c"
+			  "\x38\x51\xbc\x1f\x43\x4a\x56\x1c"
+			  "\xf8\x47\x03\x4e\x67\xa8\x1f\x99"
+			  "\x04\x39\x73\x32\xb2\x86\x79\xe7"
+			  "\x14\x28\x70\xb8\xe2\x7d\x69\x85"
+			  "\xb6\x0f\xc5\xd0\xd0\x01\x5c\xe6"
+			  "\x09\x0f\x75\xf7\xb6\x81\xd2\x11"
+			  "\x20\x9c\xa1\xee\x11\x44\x79\xd0"
+			  "\xb2\x34\x77\xda\x10\x9a\x6f\x6f"
+			  "\xef\x7c\xd9\xdc\x35\xb7\x61\xdd"
+			  "\xf1\xa4\xc6\x1c\xbf\x05\x22\xac"
+			  "\xfe\x2f\x85\x00\x44\xdf\x33\x16"
+			  "\x35\xb6\xa3\xd3\x70\xdf\x69\x35"
+			  "\x6a\xc7\xb4\x99\x45\x27\xc8\x8e"
+			  "\x5a\x14\x30\xd0\x55\x3e\x4f\x64"
+			  "\x0d\x38\xe3\xdf\x8b\xa8\x93\x26"
+			  "\x75\xae\xf6\xb5\x23\x0b\x17\x31"
+			  "\xbf\x27\xb8\xb5\x94\x31\xa7\x8f"
+			  "\x43\xc4\x46\x24\x22\x4f\x8f\x7e"
+			  "\xe5\xf4\x6d\x1e\x0e\x18\x7a\xbb"
+			  "\xa6\x8f\xfb\x49\x49\xd8\x7e\x5a",
+		.len	= 512,
 	}, {
-		.key	= "\x4d\x76\x32\x17\x05\x3f\x75\x2c"
-			  "\x5d\x04\x16\x36\x15\x72\x63\x2f",
-		.klen	= 16,
-		.input	= "\x54\x65\x61\x20\x69\x73\x20\x67"
-			  "\x6f\x6f\x64\x20\x66\x6f\x72\x20"
-			  "\x79\x6f\x75\x21\x21\x21\x20\x72"
-			  "\x65\x61\x6c\x6c\x79\x21\x21\x21",
-		.ilen	= 32,
-		.result	= "\x0b\x03\xcd\x8a\xbe\x95\xfd\xb1"
-			  "\xc1\x44\x91\x0b\xa5\xc9\x1b\xb4"
-			  "\xa9\xda\x1e\x9e\xb1\x3e\x2a\x8f"
-			  "\xea\xa5\x6a\x85\xd1\xf4\xa8\xa5",
-		.rlen	= 32,
-	}
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x62\x49\x77\x57\x24\x70\x93\x69"
+			  "\x99\x59\x57\x49\x66\x96\x76\x27"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95"
+			  "\x02\x88\x41\x97\x16\x93\x99\x37"
+			  "\x51\x05\x82\x09\x74\x94\x45\x92",
+		.klen	= 64,
+		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ctext	= "\xd7\x4b\x93\x7d\x13\xa2\xa2\xe1"
+			  "\x35\x39\x71\x88\x76\x1e\xc9\xea"
+			  "\x86\xad\xf3\x14\x48\x3d\x5e\xe9"
+			  "\xe9\x2d\xb2\x56\x59\x35\x9d\xec"
+			  "\x84\xfa\x7e\x9d\x6d\x33\x36\x8f"
+			  "\xce\xf4\xa9\x21\x0b\x5f\x96\xec"
+			  "\xcb\xf9\x57\x68\x33\x88\x39\xbf"
+			  "\x2f\xbb\x59\x03\xbd\x66\x8b\x11"
+			  "\x11\x65\x51\x2e\xb8\x67\x05\xd1"
+			  "\x27\x11\x5c\xd4\xcc\x97\xc2\xb3"
+			  "\xa9\x55\xaf\x07\x56\xd1\xdc\xf5"
+			  "\x85\xdc\x46\xe6\xf0\x24\xeb\x93"
+			  "\x4d\xf0\x9b\xf5\x73\x1c\xda\x03"
+			  "\x22\xc8\x3a\x4f\xb4\x19\x91\x09"
+			  "\x54\x0b\xf6\xfe\x17\x3d\x1a\x53"
+			  "\x72\x60\x79\xcb\x0e\x32\x8a\x77"
+			  "\xd5\xed\xdb\x33\xd7\x62\x16\x69"
+			  "\x63\xe0\xab\xb5\xf6\x9c\x5f\x3d"
+			  "\x69\x35\x61\x86\xf8\x86\xb9\x89"
+			  "\x6e\x59\x35\xac\xf6\x6b\x33\xa0"
+			  "\xea\xef\x96\x62\xd8\xa9\xcf\x56"
+			  "\xbf\xdb\x8a\xfd\xa1\x82\x77\x73"
+			  "\x3d\x94\x4a\x49\x42\x6d\x08\x60"
+			  "\xa1\xea\xab\xb6\x88\x13\x94\xb8"
+			  "\x51\x98\xdb\x35\x85\xdf\xf6\xb9"
+			  "\x8f\xcd\xdf\x80\xd3\x40\x2d\x72"
+			  "\xb8\xb2\x6c\x02\x43\x35\x22\x2a"
+			  "\x31\xed\xcd\x16\x19\xdf\x62\x0f"
+			  "\x29\xcf\x87\x04\xec\x02\x4f\xe4"
+			  "\xa2\xed\x73\xc6\x69\xd3\x7e\x89"
+			  "\x0b\x76\x10\x7c\xd6\xf9\x6a\x25"
+			  "\xed\xcc\x60\x5d\x61\x20\xc1\x97"
+			  "\x56\x91\x57\x28\xbe\x71\x0d\xcd"
+			  "\xde\xc4\x9e\x55\x91\xbe\xd1\x28"
+			  "\x9b\x90\xeb\x73\xf3\x68\x51\xc6"
+			  "\xdf\x82\xcc\xd8\x1f\xce\x5b\x27"
+			  "\xc0\x60\x5e\x33\xd6\xa7\x20\xea"
+			  "\xb2\x54\xc7\x5d\x6a\x3b\x67\x47"
+			  "\xcf\xa0\xe3\xab\x86\xaf\xc1\x42"
+			  "\xe6\xb0\x23\x4a\xaf\x53\xdf\xa0"
+			  "\xad\x12\x32\x31\x03\xf7\x21\xbe"
+			  "\x2d\xd5\x82\x42\xb6\x4a\x3d\xcd"
+			  "\xd8\x81\x77\xa9\x49\x98\x6c\x09"
+			  "\xc5\xa3\x61\x12\x62\x85\x6b\xcd"
+			  "\xb3\xf4\x20\x0c\x41\xc4\x05\x37"
+			  "\x46\x5f\xeb\x71\x8b\xf1\xaf\x6e"
+			  "\xba\xf3\x50\x2e\xfe\xa8\x37\xeb"
+			  "\xe8\x8c\x4f\xa4\x0c\xf1\x31\xc8"
+			  "\x6e\x71\x4f\xa5\xd7\x97\x73\xe0"
+			  "\x93\x4a\x2f\xda\x7b\xe0\x20\x54"
+			  "\x1f\x8d\x85\x79\x0b\x7b\x5e\x75"
+			  "\xb9\x07\x67\xcc\xc8\xe7\x21\x15"
+			  "\xa7\xc8\x98\xff\x4b\x80\x1c\x12"
+			  "\xa8\x54\xe1\x38\x52\xe6\x74\x81"
+			  "\x97\x47\xa1\x41\x0e\xc0\x50\xe3"
+			  "\x55\x0e\xc3\xa7\x70\x77\xce\x07"
+			  "\xed\x8c\x88\xe6\xa1\x5b\x14\xec"
+			  "\xe6\xde\x06\x6d\x74\xc5\xd9\xfa"
+			  "\xe5\x2f\x5a\xff\xc8\x05\xee\x27"
+			  "\x35\x61\xbf\x0b\x19\x78\x9b\xd2"
+			  "\x04\xc7\x05\xb1\x79\xb4\xff\x5f"
+			  "\xf3\xea\x67\x52\x78\xc2\xce\x70"
+			  "\xa4\x05\x0b\xb2\xb3\xa8\x30\x97"
+			  "\x37\x30\xe1\x91\x8d\xb3\x2a\xff",
+		.len	= 512,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 512 - 20, 4, 16 },
+	},
 };
 
-static const struct cipher_testvec xeta_dec_tv_template[] = {
+/*
+ * Serpent test vectors.  These are backwards because Serpent writes
+ * octet sequences in right-to-left mode.
+ */
+static const struct cipher_testvec serpent_tv_template[] = {
 	{
-		.key    = zeroed_string,
-		.klen	= 16,
-		.input	= "\xaa\x22\x96\xe5\x6c\x61\xf3\x45",
-		.ilen	= 8,
-		.result = zeroed_string,
-		.rlen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.ctext	= "\x12\x07\xfc\xce\x9b\xd0\xd6\x47"
+			  "\x6a\xe9\x8f\xbe\xd1\x43\xa0\xe2",
+		.len	= 16,
 	}, {
-		.key	= "\x2b\x02\x05\x68\x06\x14\x49\x76"
-			  "\x77\x5d\x0e\x26\x6c\x28\x78\x43",
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
 		.klen	= 16,
-		.input	= "\x82\x3e\xeb\x35\xdc\xdd\xd9\xc3",
-		.ilen	= 8,
-		.result	= "\x74\x65\x73\x74\x20\x6d\x65\x2e",
-		.rlen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.ctext	= "\x4c\x7d\x8a\x32\x80\x72\xa2\x2c"
+			  "\x82\x3e\x4a\x1f\x3a\xcd\xa1\x6d",
+		.len	= 16,
 	}, {
-		.key	= "\x09\x65\x43\x11\x66\x44\x39\x25"
-			  "\x51\x3a\x16\x10\x0a\x08\x12\x6e",
-		.klen	= 16,
-		.input	= "\xe2\x04\xdb\xf2\x89\x85\x9e\xea"
-			  "\x61\x35\xaa\xed\xb5\xcb\x71\x2c",
-		.ilen	= 16,
-		.result	= "\x6c\x6f\x6e\x67\x65\x72\x5f\x74"
-			  "\x65\x73\x74\x5f\x76\x65\x63\x74",
-		.rlen	= 16,
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.klen	= 32,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.ctext	= "\xde\x26\x9f\xf8\x33\xe4\x32\xb8"
+			  "\x5b\x2e\x88\xd2\x70\x1c\xe7\x5c",
+		.len	= 16,
 	}, {
-		.key	= "\x4d\x76\x32\x17\x05\x3f\x75\x2c"
-			  "\x5d\x04\x16\x36\x15\x72\x63\x2f",
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80",
 		.klen	= 16,
-		.input	= "\x0b\x03\xcd\x8a\xbe\x95\xfd\xb1"
-			  "\xc1\x44\x91\x0b\xa5\xc9\x1b\xb4"
-			  "\xa9\xda\x1e\x9e\xb1\x3e\x2a\x8f"
-			  "\xea\xa5\x6a\x85\xd1\xf4\xa8\xa5",
-		.ilen	= 32,
-		.result	= "\x54\x65\x61\x20\x69\x73\x20\x67"
-			  "\x6f\x6f\x64\x20\x66\x6f\x72\x20"
-			  "\x79\x6f\x75\x21\x21\x21\x20\x72"
-			  "\x65\x61\x6c\x6c\x79\x21\x21\x21",
-		.rlen	= 32,
-	}
-};
-
-/*
- * FCrypt test vectors
- */
-static const struct cipher_testvec fcrypt_pcbc_enc_tv_template[] = {
-	{ /* http://www.openafs.org/pipermail/openafs-devel/2000-December/005320.html */
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 8,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 8,
-		.result	= "\x0E\x09\x00\xC7\x3E\xF7\xED\x41",
-		.rlen	= 8,
-	}, {
-		.key	= "\x11\x44\x77\xAA\xDD\x00\x33\x66",
-		.klen	= 8,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x12\x34\x56\x78\x9A\xBC\xDE\xF0",
-		.ilen	= 8,
-		.result	= "\xD8\xED\x78\x74\x77\xEC\x06\x80",
-		.rlen	= 8,
-	}, { /* From Arla */
-		.key	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
-		.klen	= 8,
-		.iv	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.input	= "The quick brown fox jumps over the lazy dogs.\0\0",
-		.ilen	= 48,
-		.result	= "\x00\xf0\x0e\x11\x75\xe6\x23\x82"
-			  "\xee\xac\x98\x62\x44\x51\xe4\x84"
-			  "\xc3\x59\xd8\xaa\x64\x60\xae\xf7"
-			  "\xd2\xd9\x13\x79\x72\xa3\x45\x03"
-			  "\x23\xb5\x62\xd7\x0c\xf5\x27\xd1"
-			  "\xf8\x91\x3c\xac\x44\x22\x92\xef",
-		.rlen	= 48,
-	}, {
-		.key	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.klen	= 8,
-		.iv	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
-		.input	= "The quick brown fox jumps over the lazy dogs.\0\0",
-		.ilen	= 48,
-		.result	= "\xca\x90\xf5\x9d\xcb\xd4\xd2\x3c"
-			  "\x01\x88\x7f\x3e\x31\x6e\x62\x9d"
-			  "\xd8\xe0\x57\xa3\x06\x3a\x42\x58"
-			  "\x2a\x28\xfe\x72\x52\x2f\xdd\xe0"
-			  "\x19\x89\x09\x1c\x2a\x8e\x8c\x94"
-			  "\xfc\xc7\x68\xe4\x88\xaa\xde\x0f",
-		.rlen	= 48,
-	}, { /* split-page version */
-		.key	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.klen	= 8,
-		.iv	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
-		.input	= "The quick brown fox jumps over the lazy dogs.\0\0",
-		.ilen	= 48,
-		.result	= "\xca\x90\xf5\x9d\xcb\xd4\xd2\x3c"
-			  "\x01\x88\x7f\x3e\x31\x6e\x62\x9d"
-			  "\xd8\xe0\x57\xa3\x06\x3a\x42\x58"
-			  "\x2a\x28\xfe\x72\x52\x2f\xdd\xe0"
-			  "\x19\x89\x09\x1c\x2a\x8e\x8c\x94"
-			  "\xfc\xc7\x68\xe4\x88\xaa\xde\x0f",
-		.rlen	= 48,
-		.np	= 2,
-		.tap	= { 20, 28 },
-	}
+		.ptext	= zeroed_string,
+		.ctext	= "\xdd\xd2\x6b\x98\xa5\xff\xd8\x2c"
+			  "\x05\x34\x5a\x9d\xad\xbf\xaf\x49",
+		.len	= 16,
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\xFB\xB0\x5D\xDE\xC0\xFE\xFC\xEB"
+			  "\xB1\x80\x10\x43\xDE\x62\x70\xBD"
+			  "\xFA\x8A\x93\xEA\x6B\xF7\xC5\xD7"
+			  "\x0C\xD1\xBB\x29\x25\x14\x4C\x22"
+			  "\x77\xA6\x38\x00\xDB\xB9\xE2\x07"
+			  "\xD1\xAC\x82\xBA\xEA\x67\xAA\x39"
+			  "\x99\x34\x89\x5B\x54\xE9\x12\x13"
+			  "\x3B\x04\xE5\x12\x42\xC5\x79\xAB"
+			  "\x0D\xC7\x3C\x58\x2D\xA3\x98\xF6"
+			  "\xE4\x61\x9E\x17\x0B\xCE\xE8\xAA"
+			  "\xB5\x6C\x1A\x3A\x67\x52\x81\x6A"
+			  "\x04\xFF\x8A\x1B\x96\xFE\xE6\x87"
+			  "\x3C\xD4\x39\x7D\x36\x9B\x03\xD5"
+			  "\xB6\xA0\x75\x3C\x83\xE6\x1C\x73"
+			  "\x9D\x74\x2B\x77\x53\x2D\xE5\xBD"
+			  "\x69\xDA\x7A\x01\xF5\x6A\x70\x39"
+			  "\x30\xD4\x2C\xF2\x8E\x06\x4B\x39"
+			  "\xB3\x12\x1D\xB3\x17\x46\xE6\xD6"
+			  "\xB6\x31\x36\x34\x38\x3C\x1D\x69"
+			  "\x9F\x47\x28\x9A\x1D\x96\x70\x54"
+			  "\x8E\x88\xCB\xE0\xF5\x6A\xAE\x0A"
+			  "\x3C\xD5\x93\x1C\x21\xC9\x14\x3A"
+			  "\x23\x9C\x9B\x79\xC7\x75\xC8\x39"
+			  "\xA6\xAC\x65\x9A\x99\x37\xAF\x6D"
+			  "\xBD\xB5\x32\xFD\xD8\x9C\x95\x7B"
+			  "\xC6\x6A\x80\x64\xEA\xEF\x6D\x3F"
+			  "\xA9\xFE\x5B\x16\xA3\xCF\x32\xC8"
+			  "\xEF\x50\x22\x20\x93\x30\xBE\xE2"
+			  "\x38\x05\x65\xAF\xBA\xB6\xE4\x72"
+			  "\xA9\xEE\x05\x42\x88\xBD\x9D\x49"
+			  "\xAD\x93\xCA\x4D\x45\x11\x43\x4D"
+			  "\xB8\xF5\x74\x2B\x48\xE7\x21\xE4"
+			  "\x4E\x3A\x4C\xDE\x65\x7A\x5A\xAD"
+			  "\x86\xE6\x23\xEC\x6B\xA7\x17\xE6"
+			  "\xF6\xA1\xAC\x29\xAE\xF9\x9B\x69"
+			  "\x73\x65\x65\x51\xD6\x0B\x4E\x8C"
+			  "\x17\x15\x9D\xB0\xCF\xB2\x42\x2B"
+			  "\x51\xC3\x03\xE8\xB7\x7D\x2D\x39"
+			  "\xE8\x10\x93\x16\xC8\x68\x4C\x60"
+			  "\x87\x70\x14\xD0\x01\x57\xCB\x42"
+			  "\x13\x59\xB1\x7F\x12\x4F\xBB\xC7"
+			  "\xBD\x2B\xD4\xA9\x12\x26\x4F\xDE"
+			  "\xFD\x72\xEC\xD7\x6F\x97\x14\x90"
+			  "\x0E\x37\x13\xE6\x67\x1D\xE5\xFE"
+			  "\x9E\x18\x3C\x8F\x3A\x3F\x59\x9B"
+			  "\x71\x80\x05\x35\x3F\x40\x0B\x21"
+			  "\x76\xE5\xEF\x42\x6C\xDB\x31\x05"
+			  "\x5F\x05\xCF\x14\xE3\xF0\x61\xA2"
+			  "\x49\x03\x5E\x77\x2E\x20\xBA\xA1"
+			  "\xAF\x46\x51\xC0\x2B\xC4\x64\x1E"
+			  "\x65\xCC\x51\x58\x0A\xDF\xF0\x5F"
+			  "\x75\x9F\x48\xCD\x81\xEC\xC3\xF6"
+			  "\xED\xC9\x4B\x7B\x4E\x26\x23\xE1"
+			  "\xBB\xE9\x83\x0B\xCF\xE4\xDE\x00"
+			  "\x48\xFF\xBF\x6C\xB4\x72\x16\xEF"
+			  "\xC7\x46\xEE\x48\x8C\xB8\xAF\x45"
+			  "\x91\x76\xE7\x6E\x65\x3D\x15\x86"
+			  "\x10\xF8\xDB\x66\x97\x7C\x43\x4D"
+			  "\x79\x12\x4E\xCE\x06\xD1\xD1\x6A"
+			  "\x34\xC1\xC9\xF2\x28\x4A\xCD\x02"
+			  "\x75\x55\x9B\xFF\x36\x73\xAB\x7C"
+			  "\xF4\x46\x2E\xEB\xAC\xF3\xD2\xB7",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
 };
 
-static const struct cipher_testvec fcrypt_pcbc_dec_tv_template[] = {
-	{ /* http://www.openafs.org/pipermail/openafs-devel/2000-December/005320.html */
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 8,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x0E\x09\x00\xC7\x3E\xF7\xED\x41",
-		.ilen	= 8,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen	= 8,
-	}, {
-		.key	= "\x11\x44\x77\xAA\xDD\x00\x33\x66",
-		.klen	= 8,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xD8\xED\x78\x74\x77\xEC\x06\x80",
-		.ilen	= 8,
-		.result	= "\x12\x34\x56\x78\x9A\xBC\xDE\xF0",
-		.rlen	= 8,
-	}, { /* From Arla */
-		.key	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
-		.klen	= 8,
-		.iv	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.input	= "\x00\xf0\x0e\x11\x75\xe6\x23\x82"
-			  "\xee\xac\x98\x62\x44\x51\xe4\x84"
-			  "\xc3\x59\xd8\xaa\x64\x60\xae\xf7"
-			  "\xd2\xd9\x13\x79\x72\xa3\x45\x03"
-			  "\x23\xb5\x62\xd7\x0c\xf5\x27\xd1"
-			  "\xf8\x91\x3c\xac\x44\x22\x92\xef",
-		.ilen	= 48,
-		.result	= "The quick brown fox jumps over the lazy dogs.\0\0",
-		.rlen	= 48,
-	}, {
-		.key	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.klen	= 8,
-		.iv	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
-		.input	= "\xca\x90\xf5\x9d\xcb\xd4\xd2\x3c"
-			  "\x01\x88\x7f\x3e\x31\x6e\x62\x9d"
-			  "\xd8\xe0\x57\xa3\x06\x3a\x42\x58"
-			  "\x2a\x28\xfe\x72\x52\x2f\xdd\xe0"
-			  "\x19\x89\x09\x1c\x2a\x8e\x8c\x94"
-			  "\xfc\xc7\x68\xe4\x88\xaa\xde\x0f",
-		.ilen	= 48,
-		.result	= "The quick brown fox jumps over the lazy dogs.\0\0",
-		.rlen	= 48,
-	}, { /* split-page version */
-		.key	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.klen	= 8,
-		.iv	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
-		.input	= "\xca\x90\xf5\x9d\xcb\xd4\xd2\x3c"
-			  "\x01\x88\x7f\x3e\x31\x6e\x62\x9d"
-			  "\xd8\xe0\x57\xa3\x06\x3a\x42\x58"
-			  "\x2a\x28\xfe\x72\x52\x2f\xdd\xe0"
-			  "\x19\x89\x09\x1c\x2a\x8e\x8c\x94"
-			  "\xfc\xc7\x68\xe4\x88\xaa\xde\x0f",
-		.ilen	= 48,
-		.result	= "The quick brown fox jumps over the lazy dogs.\0\0",
-		.rlen	= 48,
-		.np	= 2,
-		.tap	= { 20, 28 },
+static const struct cipher_testvec tnepres_tv_template[] = {
+	{ /* KeySize=0 */
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.ctext	= "\x41\xcc\x6b\x31\x59\x31\x45\x97"
+			  "\x6d\x6f\xbb\x38\x4b\x37\x21\x28",
+		.len	= 16,
+	},
+	{ /* KeySize=128, PT=0, I=1 */
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.key    = "\x80\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen   = 16,
+		.ctext	= "\x49\xaf\xbf\xad\x9d\x5a\x34\x05"
+			  "\x2c\xd8\xff\xa5\x98\x6b\xd2\xdd",
+		.len	= 16,
+	}, { /* KeySize=128 */
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.klen	= 16,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.ctext	= "\xea\xf4\xd7\xfc\xd8\x01\x34\x47"
+			  "\x81\x45\x0b\xfa\x0c\xd6\xad\x6e",
+		.len	= 16,
+	}, { /* KeySize=128, I=121 */
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80",
+		.klen	= 16,
+		.ptext	= zeroed_string,
+		.ctext	= "\x3d\xda\xbf\xc0\x06\xda\xab\x06"
+			  "\x46\x2a\xf4\xef\x81\x54\x4e\x26",
+		.len	= 16,
+	}, { /* KeySize=192, PT=0, I=1 */
+		.key	= "\x80\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 24,
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\xe7\x8e\x54\x02\xc7\x19\x55\x68"
+			  "\xac\x36\x78\xf7\xa3\xf6\x0c\x66",
+		.len	= 16,
+	}, { /* KeySize=256, PT=0, I=1 */
+		.key	= "\x80\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 32,
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\xab\xed\x96\xe7\x66\xbf\x28\xcb"
+			  "\xc0\xeb\xd2\x1a\x82\xef\x08\x19",
+		.len	= 16,
+	}, { /* KeySize=256, I=257 */
+		.key	= "\x1f\x1e\x1d\x1c\x1b\x1a\x19\x18"
+			  "\x17\x16\x15\x14\x13\x12\x11\x10"
+			  "\x0f\x0e\x0d\x0c\x0b\x0a\x09\x08"
+			  "\x07\x06\x05\x04\x03\x02\x01\x00",
+		.klen	= 32,
+		.ptext	= "\x0f\x0e\x0d\x0c\x0b\x0a\x09\x08"
+			  "\x07\x06\x05\x04\x03\x02\x01\x00",
+		.ctext	= "\x5c\xe7\x1c\x70\xd2\x88\x2e\x5b"
+			  "\xb8\x32\xe4\x33\xf8\x9f\x26\xde",
+		.len	= 16,
+	}, { /* KeySize=256 */
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.klen	= 32,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.ctext	= "\x64\xa9\x1a\x37\xed\x9f\xe7\x49"
+			  "\xa8\x4e\x76\xd6\xf5\x0d\x78\xee",
+		.len	= 16,
 	}
 };
 
-/*
- * CAMELLIA test vectors.
- */
-static const struct cipher_testvec camellia_enc_tv_template[] = {
-	{
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.klen	= 16,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.ilen	= 16,
-		.result	= "\x67\x67\x31\x38\x54\x96\x69\x73"
-			  "\x08\x57\x06\x56\x48\xea\xbe\x43",
-		.rlen	= 16,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
-			  "\x00\x11\x22\x33\x44\x55\x66\x77",
-		.klen	= 24,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.ilen	= 16,
-		.result	= "\xb4\x99\x34\x01\xb3\xe9\x96\xf8"
-			  "\x4e\xe5\xce\xe7\xd7\x9b\x09\xb9",
-		.rlen	= 16,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
-			  "\x00\x11\x22\x33\x44\x55\x66\x77"
-			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
+static const struct cipher_testvec serpent_cbc_tv_template[] = {
+	{ /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
-		.input	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.ilen	= 16,
-		.result	= "\x9a\xcc\x23\x7d\xff\x16\xd7\x6c"
-			  "\x20\xef\x7c\x91\x9e\x3a\x75\x09",
-		.rlen	= 16,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x3F\x85\x62\x3F\x1C\xF9\xD6\x1C"
-			  "\xF9\xD6\xB3\x90\x6D\x4A\x90\x6D"
-			  "\x4A\x27\x04\xE1\x27\x04\xE1\xBE"
-			  "\x9B\x78\xBE\x9B\x78\x55\x32\x0F",
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\x80\xCF\x11\x41\x1A\xB9\x4B\x9C"
+			  "\xFF\xB7\x6C\xEA\xF0\xAF\x77\x6E"
+			  "\x71\x75\x95\x9D\x4E\x1C\xCF\xAD"
+			  "\x81\x34\xE9\x8F\xAE\x5A\x91\x1C"
+			  "\x38\x63\x35\x7E\x79\x18\x0A\xE8"
+			  "\x67\x06\x76\xD5\xFF\x22\x2F\xDA"
+			  "\xB6\x2D\x57\x13\xB6\x3C\xBC\x97"
+			  "\xFE\x53\x75\x35\x97\x7F\x51\xEA"
+			  "\xDF\x5D\xE8\x9D\xCC\xD9\xAE\xE7"
+			  "\x62\x67\xFF\x04\xC2\x18\x22\x5F"
+			  "\x2E\x06\xC1\xE2\x26\xCD\xC6\x1E"
+			  "\xE5\x2C\x4E\x87\x23\xDD\xF0\x41"
+			  "\x08\xA5\xB4\x3E\x07\x1E\x0B\xBB"
+			  "\x72\x84\xF8\x0A\x3F\x38\x5E\x91"
+			  "\x15\x26\xE1\xDB\xA4\x3D\x74\xD2"
+			  "\x41\x1E\x3F\xA9\xC6\x7D\x2A\xAB"
+			  "\x27\xDF\x89\x1D\x86\x3E\xF7\x5A"
+			  "\xF6\xE3\x0F\xC7\x6B\x4C\x96\x7C"
+			  "\x2D\x12\xA5\x05\x92\xCB\xD7\x4A"
+			  "\x4D\x1E\x88\x21\xE1\x63\xB4\xFC"
+			  "\x4A\xF2\xCD\x35\xB9\xD7\x70\x97"
+			  "\x5A\x5E\x7E\x96\x52\x20\xDC\x25"
+			  "\xE9\x6B\x36\xB4\xE0\x98\x85\x2C"
+			  "\x3C\xD2\xF7\x78\x8A\x73\x26\x9B"
+			  "\xAF\x0B\x11\xE8\x4D\x67\x23\xE9"
+			  "\x77\xDF\x58\xF6\x6F\x9E\xA4\xC5"
+			  "\x10\xA1\x82\x0E\x80\xA0\x8F\x4B"
+			  "\xA1\xC0\x12\x54\x4E\xC9\x20\x92"
+			  "\x11\x00\x10\x4E\xB3\x7C\xCA\x63"
+			  "\xE5\x3F\xD3\x41\x37\xCD\x74\xB7"
+			  "\xA5\x7C\x61\xB8\x0B\x7A\x7F\x4D"
+			  "\xFE\x96\x7D\x1B\xBE\x60\x37\xB7"
+			  "\x81\x92\x66\x67\x15\x1E\x39\x98"
+			  "\x52\xC0\xF4\x69\xC0\x99\x4F\x5A"
+			  "\x2E\x32\xAD\x7C\x8B\xE9\xAD\x05"
+			  "\x55\xF9\x0A\x1F\x97\x5C\xFA\x2B"
+			  "\xF4\x99\x76\x3A\x6E\x4D\xE1\x4C"
+			  "\x14\x4E\x6F\x87\xEE\x1A\x85\xA3"
+			  "\x96\xC6\x66\x49\xDA\x0D\x71\xAC"
+			  "\x04\x05\x46\xD3\x90\x0F\x64\x64"
+			  "\x01\x66\x2C\x62\x5D\x34\xD1\xCB"
+			  "\x3A\x24\xCE\x95\xEF\xAE\x2C\x97"
+			  "\x0E\x0C\x1D\x36\x49\xEB\xE9\x3D"
+			  "\x62\xA6\x19\x28\x9E\x26\xB4\x3F"
+			  "\xD7\x55\x42\x3C\xCD\x72\x0A\xF0"
+			  "\x7D\xE9\x95\x45\x86\xED\xB1\xE0"
+			  "\x8D\xE9\xC5\x86\x13\x24\x28\x7D"
+			  "\x74\xEF\xCA\x50\x12\x7E\x64\x8F"
+			  "\x1B\xF5\x5B\xFE\xE2\xAC\xFA\xE7"
+			  "\xBD\x38\x8C\x11\x20\xEF\xB1\xAA"
+			  "\x7B\xE5\xE5\x78\xAD\x9D\x2D\xA2"
+			  "\x8E\xDD\x48\xB3\xEF\x18\x92\x7E"
+			  "\xE6\x75\x0D\x54\x64\x11\xA3\x3A"
+			  "\xDB\x97\x0F\xD3\xDF\x07\xD3\x7E"
+			  "\x1E\xD1\x87\xE4\x74\xBB\x46\xF4"
+			  "\xBA\x23\x2D\x8D\x29\x07\x12\xCF"
+			  "\x34\xCD\x72\x7F\x01\x30\xE7\xA0"
+			  "\xF8\xDD\xA8\x08\xF0\xBC\xB1\xA2"
+			  "\xCC\xE1\x6B\x5F\xBE\xEA\xF1\xE4"
+			  "\x02\xC4\xAF\xFA\xAD\x31\xF4\xBF"
+			  "\xFC\x66\xAA\x37\xF2\x37\x39\x6B"
+			  "\xBC\x08\x3A\xA2\x29\xB3\xDF\xD1",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec serpent_ctr_tv_template[] = {
+	{ /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -30146,393 +9136,237 @@ static const struct cipher_testvec camellia_enc_tv_template[] = {
 			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
 			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
 			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06"
-			  "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78"
-			  "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA"
-			  "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C"
-			  "\xF3\x67\xFE\x95\x09\xA0\x37\xCE"
-			  "\x42\xD9\x70\x07\x7B\x12\xA9\x1D"
-			  "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F"
-			  "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01"
-			  "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73"
-			  "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5"
-			  "\x59\xF0\x87\x1E\x92\x29\xC0\x34"
-			  "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6"
-			  "\x3D\xD4\x48\xDF\x76\x0D\x81\x18"
-			  "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A"
-			  "\x21\x95\x2C\xC3\x37\xCE\x65\xFC"
-			  "\x70\x07\x9E\x12\xA9\x40\xD7\x4B"
-			  "\xE2\x79\x10\x84\x1B\xB2\x26\xBD"
-			  "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F"
-			  "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1"
-			  "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13"
-			  "\x87\x1E\xB5\x29\xC0\x57\xEE\x62"
-			  "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4"
-			  "\x6B\x02\x76\x0D\xA4\x18\xAF\x46"
-			  "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8"
-			  "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07"
-			  "\x9E\x35\xCC\x40\xD7\x6E\x05\x79"
-			  "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB"
-			  "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D"
-			  "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF"
-			  "\x43\xDA\x71\x08\x7C\x13\xAA\x1E"
-			  "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90"
-			  "\x27\xBE\x32\xC9\x60\xF7\x6B\x02"
-			  "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74"
-			  "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6"
-			  "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35"
-			  "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7"
-			  "\x3E\xD5\x49\xE0\x77\x0E\x82\x19"
-			  "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B"
-			  "\x22\x96\x2D\xC4\x38\xCF\x66\xFD"
-			  "\x71\x08\x9F\x13\xAA\x41\xD8\x4C"
-			  "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE"
-			  "\x55\xEC\x60\xF7\x8E\x02\x99\x30"
-			  "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2"
-			  "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14"
-			  "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63"
-			  "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5"
-			  "\x6C\x03\x77\x0E\xA5\x19\xB0\x47"
-			  "\xDE\x52\xE9\x80\x17\x8B\x22\xB9"
-			  "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08"
-			  "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A"
-			  "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC"
-			  "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E"
-			  "\xF5\x69\x00\x97\x0B\xA2\x39\xD0"
-			  "\x44\xDB\x72\x09\x7D\x14\xAB\x1F"
-			  "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91"
-			  "\x28\xBF\x33\xCA\x61\xF8\x6C\x03"
-			  "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75"
-			  "\x0C\x80\x17\xAE\x22\xB9\x50\xE7"
-			  "\x5B\xF2\x89\x20\x94\x2B\xC2\x36"
-			  "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8"
-			  "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A"
-			  "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C"
-			  "\x00\x97\x2E\xC5\x39\xD0\x67\xFE"
-			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D",
-		.ilen	= 1008,
-		.result	= "\xED\xCD\xDB\xB8\x68\xCE\xBD\xEA"
-			  "\x9D\x9D\xCD\x9F\x4F\xFC\x4D\xB7"
-			  "\xA5\xFF\x6F\x43\x0F\xBA\x32\x04"
-			  "\xB3\xC2\xB9\x03\xAA\x91\x56\x29"
-			  "\x0D\xD0\xFD\xC4\x65\xA5\x69\xB9"
-			  "\xF1\xF6\xB1\xA5\xB2\x75\x4F\x8A"
-			  "\x8D\x7D\x1B\x9B\xC7\x68\x72\xF8"
-			  "\x01\x9B\x17\x0A\x29\xE7\x61\x28"
-			  "\x7F\xA7\x50\xCA\x20\x2C\x96\x3B"
-			  "\x6E\x5C\x5D\x3F\xB5\x7F\xF3\x2B"
-			  "\x04\xEF\x9D\xD4\xCE\x41\x28\x8E"
-			  "\x83\x54\xAE\x7C\x82\x46\x10\xC9"
-			  "\xC4\x8A\x1E\x1F\x4C\xA9\xFC\xEC"
-			  "\x3C\x8C\x30\xFC\x59\xD2\x54\xC4"
-			  "\x6F\x50\xC6\xCA\x8C\x14\x5B\x9C"
-			  "\x18\x56\x5B\xF8\x33\x0E\x4A\xDB"
-			  "\xEC\xB5\x6E\x5B\x31\xC4\x0E\x98"
-			  "\x9F\x32\xBA\xA2\x18\xCF\x55\x43"
-			  "\xFE\x80\x8F\x60\xCF\x05\x30\x9B"
-			  "\x70\x50\x1E\x9C\x08\x87\xE6\x20"
-			  "\xD2\xF3\x27\xF8\x2A\x8D\x12\xB2"
-			  "\xBC\x5F\xFE\x52\x52\xF6\x7F\xB6"
-			  "\xB8\x30\x86\x3B\x0F\x94\x1E\x79"
-			  "\x13\x94\x35\xA2\xB1\x35\x5B\x05"
-			  "\x2A\x98\x6B\x96\x4C\xB1\x20\xBE"
-			  "\xB6\x14\xC2\x06\xBF\xFD\x5F\x2A"
-			  "\xF5\x33\xC8\x19\x45\x14\x44\x5D"
-			  "\xFE\x94\x7B\xBB\x63\x13\x57\xC3"
-			  "\x2A\x8F\x6C\x11\x2A\x07\xA7\x6A"
-			  "\xBF\x20\xD3\x99\xC6\x00\x0B\xBF"
-			  "\x83\x46\x25\x3A\xB0\xF6\xC5\xC8"
-			  "\x00\xCA\xE5\x28\x4A\x7C\x95\x9C"
-			  "\x7B\x43\xAB\xF9\xE4\xF8\x74\xAB"
-			  "\xA7\xB8\x9C\x0F\x53\x7B\xB6\x74"
-			  "\x60\x64\x0D\x1C\x80\xD1\x20\x9E"
-			  "\xDC\x14\x27\x9B\xFC\xBD\x5C\x96"
-			  "\xD2\x51\xDC\x96\xEE\xE5\xEA\x2B"
-			  "\x02\x7C\xAA\x3C\xDC\x9D\x7B\x01"
-			  "\x20\xC3\xE1\x0B\xDD\xAB\xF3\x1E"
-			  "\x19\xA8\x84\x29\x5F\xCC\xC3\x5B"
-			  "\xE4\x33\x59\xDC\x12\xEB\x2B\x4D"
-			  "\x5B\x55\x23\xB7\x40\x31\xDE\xEE"
-			  "\x18\xC9\x3C\x4D\xBC\xED\xE0\x42"
-			  "\xAD\xDE\xA0\xA3\xC3\xFE\x44\xD3"
-			  "\xE1\x9A\xDA\xAB\x32\xFC\x1A\xBF"
-			  "\x63\xA9\xF0\x6A\x08\x46\xBD\x48"
-			  "\x83\x06\xAB\x82\x99\x01\x16\x1A"
-			  "\x03\x36\xC5\x59\x6B\xB8\x8C\x9F"
-			  "\xC6\x51\x3D\xE5\x7F\xBF\xAB\xBC"
-			  "\xC9\xA1\x88\x34\x5F\xA9\x7C\x3B"
-			  "\x9F\x1B\x98\x2B\x4F\xFB\x9B\xF0"
-			  "\xCD\xB6\x45\xB2\x29\x2E\x34\x23"
-			  "\xA9\x97\xC0\x22\x8C\x42\x9B\x5F"
-			  "\x40\xC8\xD7\x3D\x82\x9A\x6F\xAA"
-			  "\x74\x83\x29\x05\xE8\xC4\x4D\x01"
-			  "\xB5\xE5\x84\x3F\x7F\xD3\xE0\x99"
-			  "\xDA\xE7\x6F\x30\xFD\xAA\x92\x30"
-			  "\xA5\x46\x8B\xA2\xE6\x58\x62\x7C"
-			  "\x2C\x35\x1B\x38\x85\x7D\xE8\xF3"
-			  "\x87\x4F\xDA\xD8\x5F\xFC\xB6\x44"
-			  "\xD0\xE3\x9B\x8B\xBF\xD6\xB8\xC4"
-			  "\x73\xAE\x1D\x8B\x5B\x74\x8B\xCB"
-			  "\xA4\xAD\xCF\x5D\xD4\x58\xC9\xCD"
-			  "\xF7\x90\x68\xCF\xC9\x11\x52\x3E"
-			  "\xE8\xA1\xA3\x78\x8B\xD0\xAC\x0A"
-			  "\xD4\xC9\xA3\xA5\x55\x30\xC8\x3E"
-			  "\xED\x28\x39\xE9\x63\xED\x41\x70"
-			  "\x51\xE3\xC4\xA0\xFC\xD5\x43\xCB"
-			  "\x4D\x65\xC8\xFD\x3A\x91\x8F\x60"
-			  "\x8A\xA6\x6D\x9D\x3E\x01\x23\x4B"
-			  "\x50\x47\xC9\xDC\x9B\xDE\x37\xC5"
-			  "\xBF\x67\xB1\x6B\x78\x38\xD5\x7E"
-			  "\xB6\xFF\x67\x83\x3B\x6E\xBE\x23"
-			  "\x45\xFA\x1D\x69\x44\xFD\xC6\xB9"
-			  "\xD0\x4A\x92\xD1\xBE\xF6\x4A\xB7"
-			  "\xCA\xA8\xA2\x9E\x13\x87\x57\x92"
-			  "\x64\x7C\x85\x0B\xB3\x29\x37\xD8"
-			  "\xE6\xAA\xAF\xC4\x03\x67\xA3\xBF"
-			  "\x2E\x45\x83\xB6\xD8\x54\x00\x89"
-			  "\xF6\xBC\x3A\x7A\x88\x58\x51\xED"
-			  "\xF4\x4E\x01\xA5\xC3\x2E\xD9\x42"
-			  "\xBD\x6E\x0D\x0B\x21\xB0\x1A\xCC"
-			  "\xA4\xD3\x3F\xDC\x9B\x81\xD8\xF1"
-			  "\xEA\x7A\x6A\xB7\x07\xC9\x6D\x91"
-			  "\x6D\x3A\xF5\x5F\xA6\xFF\x87\x1E"
-			  "\x3F\xDD\xC0\x72\xEA\xAC\x08\x15"
-			  "\x21\xE6\xC6\xB6\x0D\xD8\x51\x86"
-			  "\x2A\x03\x73\xF7\x29\xD4\xC4\xE4"
-			  "\x7F\x95\x10\xF7\xAB\x3F\x92\x23"
-			  "\xD3\xCE\x9C\x2E\x46\x3B\x63\x43"
-			  "\xBB\xC2\x82\x7A\x83\xD5\x55\xE2"
-			  "\xE7\x9B\x2F\x92\xAF\xFD\x81\x56"
-			  "\x79\xFD\x3E\xF9\x46\xE0\x25\xD4"
-			  "\x38\xDE\xBC\x2C\xC4\x7A\x2A\x8F"
-			  "\x94\x4F\xD0\xAD\x9B\x37\x18\xD4"
-			  "\x0E\x4D\x0F\x02\x3A\xDC\x5A\xA2"
-			  "\x39\x25\x55\x20\x5A\xA6\x02\x9F"
-			  "\xE6\x77\x21\x77\xE5\x4B\x7B\x0B"
-			  "\x30\xF8\x5F\x33\x0F\x49\xCD\xFF"
-			  "\xF2\xE4\x35\xF9\xF0\x63\xC3\x7E"
-			  "\xF1\xA6\x73\xB4\xDF\xE7\xBB\x78"
-			  "\xFF\x21\xA9\xF3\xF3\xCF\x5D\xBA"
-			  "\xED\x87\x98\xAC\xFE\x48\x97\x6D"
-			  "\xA6\x7F\x69\x31\xB1\xC4\xFF\x14"
-			  "\xC6\x76\xD4\x10\xDD\xF6\x49\x2C"
-			  "\x9C\xC8\x6D\x76\xC0\x8F\x5F\x55"
-			  "\x2F\x3C\x8A\x30\xAA\xC3\x16\x55"
-			  "\xC6\xFC\x8D\x8B\xB9\xE5\x80\x6C"
-			  "\xC8\x7E\xBD\x65\x58\x36\xD5\xBC"
-			  "\xF0\x33\x52\x29\x70\xF9\x5C\xE9"
-			  "\xAC\x1F\xB5\x73\x56\x66\x54\xAF"
-			  "\x1B\x8F\x7D\xED\xAB\x03\xCE\xE3"
-			  "\xAE\x47\xB6\x69\x86\xE9\x01\x31"
-			  "\x83\x18\x3D\xF4\x74\x7B\xF9\x42"
-			  "\x4C\xFD\x75\x4A\x6D\xF0\x03\xA6"
-			  "\x2B\x20\x63\xDA\x49\x65\x5E\x8B"
-			  "\xC0\x19\xE3\x8D\xD9\xF3\xB0\x34"
-			  "\xD3\x52\xFC\x68\x00\x43\x1B\x37"
-			  "\x31\x93\x51\x1C\x63\x97\x70\xB0"
-			  "\x99\x78\x83\x13\xFD\xCF\x53\x81"
-			  "\x36\x46\xB5\x42\x52\x2F\x32\xEB"
-			  "\x4A\x3D\xF1\x8F\x1C\x54\x2E\xFC"
-			  "\x41\x75\x5A\x8C\x8E\x6F\xE7\x1A"
-			  "\xAE\xEF\x3E\x82\x12\x0B\x74\x72"
-			  "\xF8\xB2\xAA\x7A\xD6\xFF\xFA\x55"
-			  "\x33\x1A\xBB\xD3\xA2\x7E\x97\x66",
-		.rlen	= 1008,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 1008 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec camellia_dec_tv_template[] = {
-	{
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.klen	= 16,
-		.input	= "\x67\x67\x31\x38\x54\x96\x69\x73"
-			  "\x08\x57\x06\x56\x48\xea\xbe\x43",
-		.ilen	= 16,
-		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.rlen	= 16,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
-			  "\x00\x11\x22\x33\x44\x55\x66\x77",
-		.klen	= 24,
-		.input	= "\xb4\x99\x34\x01\xb3\xe9\x96\xf8"
-			  "\x4e\xe5\xce\xe7\xd7\x9b\x09\xb9",
-		.ilen	= 16,
-		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.rlen	= 16,
-	}, {
-		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
-			  "\x00\x11\x22\x33\x44\x55\x66\x77"
-			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
-		.klen	= 32,
-		.input	= "\x9a\xcc\x23\x7d\xff\x16\xd7\x6c"
-			  "\x20\xef\x7c\x91\x9e\x3a\x75\x09",
-		.ilen	= 16,
-		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
-			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
-		.rlen	= 16,
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\x84\x68\xEC\xF2\x1C\x88\x20\xCA"
+			  "\x37\x69\xE3\x3A\x22\x85\x48\x46"
+			  "\x70\xAA\x25\xB4\xCD\x8B\x04\x4E"
+			  "\x8D\x15\x2B\x98\xDF\x7B\x6D\xB9"
+			  "\xE0\x4A\x73\x00\x65\xB6\x1A\x0D"
+			  "\x5C\x60\xDF\x34\xDC\x60\x4C\xDF"
+			  "\xB5\x1F\x26\x8C\xDA\xC1\x11\xA8"
+			  "\x80\xFA\x37\x7A\x89\xAA\xAE\x7B"
+			  "\x92\x6E\xB9\xDC\xC9\x62\x4F\x88"
+			  "\x0A\x5D\x97\x2F\x6B\xAC\x03\x7C"
+			  "\x22\xF6\x55\x5A\xFA\x35\xA5\x17"
+			  "\xA1\x5C\x5E\x2B\x63\x2D\xB9\x91"
+			  "\x3E\x83\x26\x00\x4E\xD5\xBE\xCE"
+			  "\x79\xC4\x3D\xFC\x70\xA0\xAD\x96"
+			  "\xBA\x58\x2A\x1C\xDF\xC2\x3A\xA5"
+			  "\x7C\xB5\x12\x89\xED\xBF\xB6\x09"
+			  "\x13\x4F\x7D\x61\x3C\x5C\x27\xFC"
+			  "\x5D\xE1\x4F\xA1\xEA\xB3\xCA\xB9"
+			  "\xE6\xD0\x97\x81\xDE\xD1\xFB\x8A"
+			  "\x30\xDB\xA3\x5D\xEC\x25\x0B\x86"
+			  "\x71\xC8\xA7\x67\xE8\xBC\x7D\x4C"
+			  "\xAE\x82\xD3\x73\x31\x09\xCB\xB3"
+			  "\x4D\xD4\xC0\x8A\x2B\xFA\xA6\x55"
+			  "\x39\x0A\xBC\x6E\x75\xAB\xC2\xE2"
+			  "\x8A\xF2\x26\xCD\x63\x38\x35\xF7"
+			  "\xAE\x12\x83\xCD\x8A\x9E\x7E\x4C"
+			  "\xFE\x4D\xD7\xCE\x5C\x6E\x4C\xAF"
+			  "\xE3\xCD\x76\xA7\x87\xA1\x54\x7C"
+			  "\xEC\x32\xC7\x83\x2A\xFF\xF8\xEA"
+			  "\x87\xB2\x47\xA3\x9D\xC2\x9C\xA2"
+			  "\xB7\x2C\x7C\x1A\x24\xCB\x88\x61"
+			  "\xFF\xA7\x1A\x16\x01\xDD\x4B\xFC"
+			  "\x2E\xE0\x48\x67\x09\x42\xCC\x91"
+			  "\xBE\x20\x38\xC0\x5E\x3B\x95\x00"
+			  "\xA1\x96\x66\x0B\x8A\xE9\x9E\xF7"
+			  "\x6B\x34\x0A\x51\xC0\x3B\xEB\x71"
+			  "\x07\x97\x38\x4B\x5C\x56\x98\x67"
+			  "\x78\x9C\xD0\x0E\x2B\xB5\x67\x90"
+			  "\x75\xF8\xFE\x6D\x4E\x85\xCC\x0D"
+			  "\x18\x06\x15\x9D\x5A\x10\x13\x37"
+			  "\xA3\xD6\x68\xA2\xDF\x7E\xC7\x12"
+			  "\xC9\x0D\x4D\x91\xB0\x2A\x55\xFF"
+			  "\x6F\x73\x13\xDF\x28\xB5\x2A\x2C"
+			  "\xE4\xFC\x20\xD9\xF1\x7A\x82\xB1"
+			  "\xCB\x57\xB6\x3D\x8C\xF4\x8E\x27"
+			  "\x37\xDC\x35\xF3\x79\x01\x53\xA4"
+			  "\x7B\x37\xDE\x7C\x04\xAE\x50\xDB"
+			  "\x9B\x1E\x8C\x07\xA7\x52\x49\x50"
+			  "\x34\x25\x65\xDD\xA9\x8F\x7E\xBD"
+			  "\x7A\xC9\x36\xAE\xDE\x21\x48\x64"
+			  "\xC2\x02\xBA\xBE\x11\x1E\x3D\x9C"
+			  "\x98\x52\xCC\x04\xBD\x5E\x61\x26"
+			  "\x10\xD3\x21\xD9\x6E\x25\x98\x77"
+			  "\x8E\x98\x63\xF6\xF6\x52\xFB\x13"
+			  "\xAA\x30\xF2\xB9\xA4\x43\x53\x39"
+			  "\x1C\x97\x07\x7E\x6B\xFF\x3D\x43"
+			  "\xA6\x71\x6B\x66\x8F\x58\x3F\x71"
+			  "\x90\x47\x40\x92\xE6\x69\xD1\x96"
+			  "\x34\xB3\x3B\xE5\x43\xE4\xD5\x56"
+			  "\xB2\xE6\x7E\x86\x7A\x12\x17\x5B"
+			  "\x30\xF3\x9B\x0D\xFA\x57\xE4\x50"
+			  "\x40\x53\x77\x8C\x15\xF8\x8D\x13",
+		.len	= 496,
 	}, { /* Generated with Crypto++ */
-		.key	= "\x3F\x85\x62\x3F\x1C\xF9\xD6\x1C"
-			  "\xF9\xD6\xB3\x90\x6D\x4A\x90\x6D"
-			  "\x4A\x27\x04\xE1\x27\x04\xE1\xBE"
-			  "\x9B\x78\xBE\x9B\x78\x55\x32\x0F",
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
-		.input	= "\xED\xCD\xDB\xB8\x68\xCE\xBD\xEA"
-			  "\x9D\x9D\xCD\x9F\x4F\xFC\x4D\xB7"
-			  "\xA5\xFF\x6F\x43\x0F\xBA\x32\x04"
-			  "\xB3\xC2\xB9\x03\xAA\x91\x56\x29"
-			  "\x0D\xD0\xFD\xC4\x65\xA5\x69\xB9"
-			  "\xF1\xF6\xB1\xA5\xB2\x75\x4F\x8A"
-			  "\x8D\x7D\x1B\x9B\xC7\x68\x72\xF8"
-			  "\x01\x9B\x17\x0A\x29\xE7\x61\x28"
-			  "\x7F\xA7\x50\xCA\x20\x2C\x96\x3B"
-			  "\x6E\x5C\x5D\x3F\xB5\x7F\xF3\x2B"
-			  "\x04\xEF\x9D\xD4\xCE\x41\x28\x8E"
-			  "\x83\x54\xAE\x7C\x82\x46\x10\xC9"
-			  "\xC4\x8A\x1E\x1F\x4C\xA9\xFC\xEC"
-			  "\x3C\x8C\x30\xFC\x59\xD2\x54\xC4"
-			  "\x6F\x50\xC6\xCA\x8C\x14\x5B\x9C"
-			  "\x18\x56\x5B\xF8\x33\x0E\x4A\xDB"
-			  "\xEC\xB5\x6E\x5B\x31\xC4\x0E\x98"
-			  "\x9F\x32\xBA\xA2\x18\xCF\x55\x43"
-			  "\xFE\x80\x8F\x60\xCF\x05\x30\x9B"
-			  "\x70\x50\x1E\x9C\x08\x87\xE6\x20"
-			  "\xD2\xF3\x27\xF8\x2A\x8D\x12\xB2"
-			  "\xBC\x5F\xFE\x52\x52\xF6\x7F\xB6"
-			  "\xB8\x30\x86\x3B\x0F\x94\x1E\x79"
-			  "\x13\x94\x35\xA2\xB1\x35\x5B\x05"
-			  "\x2A\x98\x6B\x96\x4C\xB1\x20\xBE"
-			  "\xB6\x14\xC2\x06\xBF\xFD\x5F\x2A"
-			  "\xF5\x33\xC8\x19\x45\x14\x44\x5D"
-			  "\xFE\x94\x7B\xBB\x63\x13\x57\xC3"
-			  "\x2A\x8F\x6C\x11\x2A\x07\xA7\x6A"
-			  "\xBF\x20\xD3\x99\xC6\x00\x0B\xBF"
-			  "\x83\x46\x25\x3A\xB0\xF6\xC5\xC8"
-			  "\x00\xCA\xE5\x28\x4A\x7C\x95\x9C"
-			  "\x7B\x43\xAB\xF9\xE4\xF8\x74\xAB"
-			  "\xA7\xB8\x9C\x0F\x53\x7B\xB6\x74"
-			  "\x60\x64\x0D\x1C\x80\xD1\x20\x9E"
-			  "\xDC\x14\x27\x9B\xFC\xBD\x5C\x96"
-			  "\xD2\x51\xDC\x96\xEE\xE5\xEA\x2B"
-			  "\x02\x7C\xAA\x3C\xDC\x9D\x7B\x01"
-			  "\x20\xC3\xE1\x0B\xDD\xAB\xF3\x1E"
-			  "\x19\xA8\x84\x29\x5F\xCC\xC3\x5B"
-			  "\xE4\x33\x59\xDC\x12\xEB\x2B\x4D"
-			  "\x5B\x55\x23\xB7\x40\x31\xDE\xEE"
-			  "\x18\xC9\x3C\x4D\xBC\xED\xE0\x42"
-			  "\xAD\xDE\xA0\xA3\xC3\xFE\x44\xD3"
-			  "\xE1\x9A\xDA\xAB\x32\xFC\x1A\xBF"
-			  "\x63\xA9\xF0\x6A\x08\x46\xBD\x48"
-			  "\x83\x06\xAB\x82\x99\x01\x16\x1A"
-			  "\x03\x36\xC5\x59\x6B\xB8\x8C\x9F"
-			  "\xC6\x51\x3D\xE5\x7F\xBF\xAB\xBC"
-			  "\xC9\xA1\x88\x34\x5F\xA9\x7C\x3B"
-			  "\x9F\x1B\x98\x2B\x4F\xFB\x9B\xF0"
-			  "\xCD\xB6\x45\xB2\x29\x2E\x34\x23"
-			  "\xA9\x97\xC0\x22\x8C\x42\x9B\x5F"
-			  "\x40\xC8\xD7\x3D\x82\x9A\x6F\xAA"
-			  "\x74\x83\x29\x05\xE8\xC4\x4D\x01"
-			  "\xB5\xE5\x84\x3F\x7F\xD3\xE0\x99"
-			  "\xDA\xE7\x6F\x30\xFD\xAA\x92\x30"
-			  "\xA5\x46\x8B\xA2\xE6\x58\x62\x7C"
-			  "\x2C\x35\x1B\x38\x85\x7D\xE8\xF3"
-			  "\x87\x4F\xDA\xD8\x5F\xFC\xB6\x44"
-			  "\xD0\xE3\x9B\x8B\xBF\xD6\xB8\xC4"
-			  "\x73\xAE\x1D\x8B\x5B\x74\x8B\xCB"
-			  "\xA4\xAD\xCF\x5D\xD4\x58\xC9\xCD"
-			  "\xF7\x90\x68\xCF\xC9\x11\x52\x3E"
-			  "\xE8\xA1\xA3\x78\x8B\xD0\xAC\x0A"
-			  "\xD4\xC9\xA3\xA5\x55\x30\xC8\x3E"
-			  "\xED\x28\x39\xE9\x63\xED\x41\x70"
-			  "\x51\xE3\xC4\xA0\xFC\xD5\x43\xCB"
-			  "\x4D\x65\xC8\xFD\x3A\x91\x8F\x60"
-			  "\x8A\xA6\x6D\x9D\x3E\x01\x23\x4B"
-			  "\x50\x47\xC9\xDC\x9B\xDE\x37\xC5"
-			  "\xBF\x67\xB1\x6B\x78\x38\xD5\x7E"
-			  "\xB6\xFF\x67\x83\x3B\x6E\xBE\x23"
-			  "\x45\xFA\x1D\x69\x44\xFD\xC6\xB9"
-			  "\xD0\x4A\x92\xD1\xBE\xF6\x4A\xB7"
-			  "\xCA\xA8\xA2\x9E\x13\x87\x57\x92"
-			  "\x64\x7C\x85\x0B\xB3\x29\x37\xD8"
-			  "\xE6\xAA\xAF\xC4\x03\x67\xA3\xBF"
-			  "\x2E\x45\x83\xB6\xD8\x54\x00\x89"
-			  "\xF6\xBC\x3A\x7A\x88\x58\x51\xED"
-			  "\xF4\x4E\x01\xA5\xC3\x2E\xD9\x42"
-			  "\xBD\x6E\x0D\x0B\x21\xB0\x1A\xCC"
-			  "\xA4\xD3\x3F\xDC\x9B\x81\xD8\xF1"
-			  "\xEA\x7A\x6A\xB7\x07\xC9\x6D\x91"
-			  "\x6D\x3A\xF5\x5F\xA6\xFF\x87\x1E"
-			  "\x3F\xDD\xC0\x72\xEA\xAC\x08\x15"
-			  "\x21\xE6\xC6\xB6\x0D\xD8\x51\x86"
-			  "\x2A\x03\x73\xF7\x29\xD4\xC4\xE4"
-			  "\x7F\x95\x10\xF7\xAB\x3F\x92\x23"
-			  "\xD3\xCE\x9C\x2E\x46\x3B\x63\x43"
-			  "\xBB\xC2\x82\x7A\x83\xD5\x55\xE2"
-			  "\xE7\x9B\x2F\x92\xAF\xFD\x81\x56"
-			  "\x79\xFD\x3E\xF9\x46\xE0\x25\xD4"
-			  "\x38\xDE\xBC\x2C\xC4\x7A\x2A\x8F"
-			  "\x94\x4F\xD0\xAD\x9B\x37\x18\xD4"
-			  "\x0E\x4D\x0F\x02\x3A\xDC\x5A\xA2"
-			  "\x39\x25\x55\x20\x5A\xA6\x02\x9F"
-			  "\xE6\x77\x21\x77\xE5\x4B\x7B\x0B"
-			  "\x30\xF8\x5F\x33\x0F\x49\xCD\xFF"
-			  "\xF2\xE4\x35\xF9\xF0\x63\xC3\x7E"
-			  "\xF1\xA6\x73\xB4\xDF\xE7\xBB\x78"
-			  "\xFF\x21\xA9\xF3\xF3\xCF\x5D\xBA"
-			  "\xED\x87\x98\xAC\xFE\x48\x97\x6D"
-			  "\xA6\x7F\x69\x31\xB1\xC4\xFF\x14"
-			  "\xC6\x76\xD4\x10\xDD\xF6\x49\x2C"
-			  "\x9C\xC8\x6D\x76\xC0\x8F\x5F\x55"
-			  "\x2F\x3C\x8A\x30\xAA\xC3\x16\x55"
-			  "\xC6\xFC\x8D\x8B\xB9\xE5\x80\x6C"
-			  "\xC8\x7E\xBD\x65\x58\x36\xD5\xBC"
-			  "\xF0\x33\x52\x29\x70\xF9\x5C\xE9"
-			  "\xAC\x1F\xB5\x73\x56\x66\x54\xAF"
-			  "\x1B\x8F\x7D\xED\xAB\x03\xCE\xE3"
-			  "\xAE\x47\xB6\x69\x86\xE9\x01\x31"
-			  "\x83\x18\x3D\xF4\x74\x7B\xF9\x42"
-			  "\x4C\xFD\x75\x4A\x6D\xF0\x03\xA6"
-			  "\x2B\x20\x63\xDA\x49\x65\x5E\x8B"
-			  "\xC0\x19\xE3\x8D\xD9\xF3\xB0\x34"
-			  "\xD3\x52\xFC\x68\x00\x43\x1B\x37"
-			  "\x31\x93\x51\x1C\x63\x97\x70\xB0"
-			  "\x99\x78\x83\x13\xFD\xCF\x53\x81"
-			  "\x36\x46\xB5\x42\x52\x2F\x32\xEB"
-			  "\x4A\x3D\xF1\x8F\x1C\x54\x2E\xFC"
-			  "\x41\x75\x5A\x8C\x8E\x6F\xE7\x1A"
-			  "\xAE\xEF\x3E\x82\x12\x0B\x74\x72"
-			  "\xF8\xB2\xAA\x7A\xD6\xFF\xFA\x55"
-			  "\x33\x1A\xBB\xD3\xA2\x7E\x97\x66",
-		.ilen	= 1008,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
+			  "\x2B\xC2\x59",
+		.ctext	= "\x84\x68\xEC\xF2\x1C\x88\x20\xCA"
+			  "\x37\x69\xE3\x3A\x22\x85\x48\x46"
+			  "\x70\xAA\x25\xB4\xCD\x8B\x04\x4E"
+			  "\x8D\x15\x2B\x98\xDF\x7B\x6D\xB9"
+			  "\xE0\x4A\x73\x00\x65\xB6\x1A\x0D"
+			  "\x5C\x60\xDF\x34\xDC\x60\x4C\xDF"
+			  "\xB5\x1F\x26\x8C\xDA\xC1\x11\xA8"
+			  "\x80\xFA\x37\x7A\x89\xAA\xAE\x7B"
+			  "\x92\x6E\xB9\xDC\xC9\x62\x4F\x88"
+			  "\x0A\x5D\x97\x2F\x6B\xAC\x03\x7C"
+			  "\x22\xF6\x55\x5A\xFA\x35\xA5\x17"
+			  "\xA1\x5C\x5E\x2B\x63\x2D\xB9\x91"
+			  "\x3E\x83\x26\x00\x4E\xD5\xBE\xCE"
+			  "\x79\xC4\x3D\xFC\x70\xA0\xAD\x96"
+			  "\xBA\x58\x2A\x1C\xDF\xC2\x3A\xA5"
+			  "\x7C\xB5\x12\x89\xED\xBF\xB6\x09"
+			  "\x13\x4F\x7D\x61\x3C\x5C\x27\xFC"
+			  "\x5D\xE1\x4F\xA1\xEA\xB3\xCA\xB9"
+			  "\xE6\xD0\x97\x81\xDE\xD1\xFB\x8A"
+			  "\x30\xDB\xA3\x5D\xEC\x25\x0B\x86"
+			  "\x71\xC8\xA7\x67\xE8\xBC\x7D\x4C"
+			  "\xAE\x82\xD3\x73\x31\x09\xCB\xB3"
+			  "\x4D\xD4\xC0\x8A\x2B\xFA\xA6\x55"
+			  "\x39\x0A\xBC\x6E\x75\xAB\xC2\xE2"
+			  "\x8A\xF2\x26\xCD\x63\x38\x35\xF7"
+			  "\xAE\x12\x83\xCD\x8A\x9E\x7E\x4C"
+			  "\xFE\x4D\xD7\xCE\x5C\x6E\x4C\xAF"
+			  "\xE3\xCD\x76\xA7\x87\xA1\x54\x7C"
+			  "\xEC\x32\xC7\x83\x2A\xFF\xF8\xEA"
+			  "\x87\xB2\x47\xA3\x9D\xC2\x9C\xA2"
+			  "\xB7\x2C\x7C\x1A\x24\xCB\x88\x61"
+			  "\xFF\xA7\x1A\x16\x01\xDD\x4B\xFC"
+			  "\x2E\xE0\x48\x67\x09\x42\xCC\x91"
+			  "\xBE\x20\x38\xC0\x5E\x3B\x95\x00"
+			  "\xA1\x96\x66\x0B\x8A\xE9\x9E\xF7"
+			  "\x6B\x34\x0A\x51\xC0\x3B\xEB\x71"
+			  "\x07\x97\x38\x4B\x5C\x56\x98\x67"
+			  "\x78\x9C\xD0\x0E\x2B\xB5\x67\x90"
+			  "\x75\xF8\xFE\x6D\x4E\x85\xCC\x0D"
+			  "\x18\x06\x15\x9D\x5A\x10\x13\x37"
+			  "\xA3\xD6\x68\xA2\xDF\x7E\xC7\x12"
+			  "\xC9\x0D\x4D\x91\xB0\x2A\x55\xFF"
+			  "\x6F\x73\x13\xDF\x28\xB5\x2A\x2C"
+			  "\xE4\xFC\x20\xD9\xF1\x7A\x82\xB1"
+			  "\xCB\x57\xB6\x3D\x8C\xF4\x8E\x27"
+			  "\x37\xDC\x35\xF3\x79\x01\x53\xA4"
+			  "\x7B\x37\xDE\x7C\x04\xAE\x50\xDB"
+			  "\x9B\x1E\x8C\x07\xA7\x52\x49\x50"
+			  "\x34\x25\x65\xDD\xA9\x8F\x7E\xBD"
+			  "\x7A\xC9\x36\xAE\xDE\x21\x48\x64"
+			  "\xC2\x02\xBA\xBE\x11\x1E\x3D\x9C"
+			  "\x98\x52\xCC\x04\xBD\x5E\x61\x26"
+			  "\x10\xD3\x21\xD9\x6E\x25\x98\x77"
+			  "\x8E\x98\x63\xF6\xF6\x52\xFB\x13"
+			  "\xAA\x30\xF2\xB9\xA4\x43\x53\x39"
+			  "\x1C\x97\x07\x7E\x6B\xFF\x3D\x43"
+			  "\xA6\x71\x6B\x66\x8F\x58\x3F\x71"
+			  "\x90\x47\x40\x92\xE6\x69\xD1\x96"
+			  "\x34\xB3\x3B\xE5\x43\xE4\xD5\x56"
+			  "\xB2\xE6\x7E\x86\x7A\x12\x17\x5B"
+			  "\x30\xF3\x9B\x0D\xFA\x57\xE4\x50"
+			  "\x40\x53\x77\x8C\x15\xF8\x8D\x13"
+			  "\x38\xE2\xE5",
+		.len	= 499,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 499 - 16, 16 },
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -30593,107 +9427,1488 @@ static const struct cipher_testvec camellia_dec_tv_template[] = {
 			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
 			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06"
-			  "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78"
-			  "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA"
-			  "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C"
-			  "\xF3\x67\xFE\x95\x09\xA0\x37\xCE"
-			  "\x42\xD9\x70\x07\x7B\x12\xA9\x1D"
-			  "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F"
-			  "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01"
-			  "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73"
-			  "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5"
-			  "\x59\xF0\x87\x1E\x92\x29\xC0\x34"
-			  "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6"
-			  "\x3D\xD4\x48\xDF\x76\x0D\x81\x18"
-			  "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A"
-			  "\x21\x95\x2C\xC3\x37\xCE\x65\xFC"
-			  "\x70\x07\x9E\x12\xA9\x40\xD7\x4B"
-			  "\xE2\x79\x10\x84\x1B\xB2\x26\xBD"
-			  "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F"
-			  "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1"
-			  "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13"
-			  "\x87\x1E\xB5\x29\xC0\x57\xEE\x62"
-			  "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4"
-			  "\x6B\x02\x76\x0D\xA4\x18\xAF\x46"
-			  "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8"
-			  "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07"
-			  "\x9E\x35\xCC\x40\xD7\x6E\x05\x79"
-			  "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB"
-			  "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D"
-			  "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF"
-			  "\x43\xDA\x71\x08\x7C\x13\xAA\x1E"
-			  "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90"
-			  "\x27\xBE\x32\xC9\x60\xF7\x6B\x02"
-			  "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74"
-			  "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6"
-			  "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35"
-			  "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7"
-			  "\x3E\xD5\x49\xE0\x77\x0E\x82\x19"
-			  "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B"
-			  "\x22\x96\x2D\xC4\x38\xCF\x66\xFD"
-			  "\x71\x08\x9F\x13\xAA\x41\xD8\x4C"
-			  "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE"
-			  "\x55\xEC\x60\xF7\x8E\x02\x99\x30"
-			  "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2"
-			  "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14"
-			  "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63"
-			  "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5"
-			  "\x6C\x03\x77\x0E\xA5\x19\xB0\x47"
-			  "\xDE\x52\xE9\x80\x17\x8B\x22\xB9"
-			  "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08"
-			  "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A"
-			  "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC"
-			  "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E"
-			  "\xF5\x69\x00\x97\x0B\xA2\x39\xD0"
-			  "\x44\xDB\x72\x09\x7D\x14\xAB\x1F"
-			  "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91"
-			  "\x28\xBF\x33\xCA\x61\xF8\x6C\x03"
-			  "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75"
-			  "\x0C\x80\x17\xAE\x22\xB9\x50\xE7"
-			  "\x5B\xF2\x89\x20\x94\x2B\xC2\x36"
-			  "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8"
-			  "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A"
-			  "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C"
-			  "\x00\x97\x2E\xC5\x39\xD0\x67\xFE"
-			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D",
-		.rlen	= 1008,
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\x06\x9A\xF8\xB4\x53\x88\x62\xFC"
+			  "\x68\xB8\x2E\xDF\xC1\x05\x0F\x3D"
+			  "\xAF\x4D\x95\xAE\xC4\xE9\x1C\xDC"
+			  "\xF6\x2B\x8F\x90\x89\xF6\x7E\x1A"
+			  "\xA6\xB9\xE4\xF4\xFA\xCA\xE5\x7E"
+			  "\x71\x28\x06\x4F\xE8\x08\x39\xDA"
+			  "\xA5\x0E\xC8\xC0\xB8\x16\xE5\x69"
+			  "\xE5\xCA\xEC\x4F\x63\x2C\xC0\x9B"
+			  "\x9F\x3E\x39\x79\xF0\xCD\x64\x35"
+			  "\x4A\xD3\xC8\xA9\x31\xCD\x48\x5B"
+			  "\x92\x3D\x8F\x3F\x96\xBD\xB3\x18"
+			  "\x74\x2A\x5D\x29\x3F\x57\x8F\xE2"
+			  "\x67\x9A\xE0\xE5\xD4\x4A\xE2\x47"
+			  "\xBC\xF6\xEB\x14\xF3\x8C\x20\xC2"
+			  "\x7D\xE2\x43\x81\x86\x72\x2E\xB1"
+			  "\x39\xF6\x95\xE1\x1F\xCB\x76\x33"
+			  "\x5B\x7D\x23\x0F\x3A\x67\x2A\x2F"
+			  "\xB9\x37\x9D\xDD\x1F\x16\xA1\x3C"
+			  "\x70\xFE\x52\xAA\x93\x3C\xC4\x46"
+			  "\xB1\xE5\xFF\xDA\xAF\xE2\x84\xFE"
+			  "\x25\x92\xB2\x63\xBD\x49\x77\xB4"
+			  "\x22\xA4\x6A\xD5\x04\xE0\x45\x58"
+			  "\x1C\x34\x96\x7C\x03\x0C\x13\xA2"
+			  "\x05\x22\xE2\xCB\x5A\x35\x03\x09"
+			  "\x40\xD2\x82\x05\xCA\x58\x73\xF2"
+			  "\x29\x5E\x01\x47\x13\x32\x78\xBE"
+			  "\x06\xB0\x51\xDB\x6C\x31\xA0\x1C"
+			  "\x74\xBC\x8D\x25\xDF\xF8\x65\xD1"
+			  "\x38\x35\x11\x26\x4A\xB4\x06\x32"
+			  "\xFA\xD2\x07\x77\xB3\x74\x98\x80"
+			  "\x61\x59\xA8\x9F\xF3\x6F\x2A\xBF"
+			  "\xE6\xA5\x9A\xC4\x6B\xA6\x49\x6F"
+			  "\xBC\x47\xD9\xFB\xC6\xEF\x25\x65"
+			  "\x96\xAC\x9F\xE4\x81\x4B\xD8\xBA"
+			  "\xD6\x9B\xC9\x6D\x58\x40\x81\x02"
+			  "\x73\x44\x4E\x43\x6E\x37\xBB\x11"
+			  "\xE3\xF9\xB8\x2F\xEC\x76\x34\xEA"
+			  "\x90\xCD\xB7\x2E\x0E\x32\x71\xE8"
+			  "\xBB\x4E\x0B\x98\xA4\x17\x17\x5B"
+			  "\x07\xB5\x82\x3A\xC4\xE8\x42\x51"
+			  "\x5A\x4C\x4E\x7D\xBF\xC4\xC0\x4F"
+			  "\x68\xB8\xC6\x4A\x32\x6F\x0B\xD7"
+			  "\x85\xED\x6B\xFB\x72\xD2\xA5\x8F"
+			  "\xBF\xF9\xAC\x59\x50\xA8\x08\x70"
+			  "\xEC\xBD\x0A\xBF\xE5\x87\xA1\xC2"
+			  "\x92\x14\x78\xAF\xE8\xEA\x2E\xDD"
+			  "\xC1\x03\x9A\xAA\x89\x8B\x32\x46"
+			  "\x5B\x18\x27\xBA\x46\xAA\x64\xDE"
+			  "\xE3\xD5\xA3\xFC\x7B\x5B\x61\xDB"
+			  "\x7E\xDA\xEC\x30\x17\x19\xF8\x80"
+			  "\xB5\x5E\x27\xB5\x37\x3A\x1F\x28"
+			  "\x07\x73\xC3\x63\xCE\xFF\x8C\xFE"
+			  "\x81\x4E\xF8\x24\xF3\xB8\xC7\xE8"
+			  "\x16\x9A\xCC\x58\x2F\x88\x1C\x4B"
+			  "\xBB\x33\xA2\x73\xF0\x1C\x89\x0E"
+			  "\xDC\x34\x27\x89\x98\xCE\x1C\xA2"
+			  "\xD8\xB8\x90\xBE\xEC\x72\x28\x13"
+			  "\xAC\x7B\xF1\xD0\x7F\x7A\x28\x50"
+			  "\xB7\x99\x65\x8A\xC9\xC6\x21\x34"
+			  "\x7F\x67\x9D\xB7\x2C\xCC\xF5\x17"
+			  "\x2B\x89\xAC\xB0\xD7\x1E\x47\xB0"
+			  "\x61\xAF\xD4\x63\x6D\xB8\x2D\x20",
+		.len	= 496,
+	},
+};
+
+static const struct cipher_testvec serpent_lrw_tv_template[] = {
+	/* Generated from AES-LRW test vectors */
+	{
+		.key	= "\x45\x62\xac\x25\xf8\x28\x17\x6d"
+			  "\x4c\x26\x84\x14\xb5\x68\x01\x85"
+			  "\x25\x8e\x2a\x05\xe7\x3e\x9d\x03"
+			  "\xee\x5a\x83\x0c\xcc\x09\x4c\x87",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x6f\xbf\xd4\xa4\x5d\x71\x16\x79"
+			  "\x63\x9c\xa6\x8e\x40\xbe\x0d\x8a",
+		.len	= 16,
+	}, {
+		.key	= "\x59\x70\x47\x14\xf5\x57\x47\x8c"
+			  "\xd7\x79\xe8\x0f\x54\x88\x79\x44"
+			  "\x0d\x48\xf0\xb7\xb1\x5a\x53\xea"
+			  "\x1c\xaa\x6b\x29\xc2\xca\xfb\xaf",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x02",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\xfd\xb2\x66\x98\x80\x96\x55\xad"
+			  "\x08\x94\x54\x9c\x21\x7c\x69\xe3",
+		.len	= 16,
+	}, {
+		.key	= "\xd8\x2a\x91\x34\xb2\x6a\x56\x50"
+			  "\x30\xfe\x69\xe2\x37\x7f\x98\x47"
+			  "\xcd\xf9\x0b\x16\x0c\x64\x8f\xb6"
+			  "\xb0\x0d\x0d\x1b\xae\x85\x87\x1f",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x02\x00\x00\x00\x00",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x14\x5e\x3d\x70\xc0\x6e\x9c\x34"
+			  "\x5b\x5e\xcf\x0f\xe4\x8c\x21\x5c",
+		.len	= 16,
+	}, {
+		.key	= "\x0f\x6a\xef\xf8\xd3\xd2\xbb\x15"
+			  "\x25\x83\xf7\x3c\x1f\x01\x28\x74"
+			  "\xca\xc6\xbc\x35\x4d\x4a\x65\x54"
+			  "\x90\xae\x61\xcf\x7b\xae\xbd\xcc"
+			  "\xad\xe4\x94\xc5\x4a\x29\xae\x70",
+		.klen	= 40,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x25\x39\xaa\xa5\xf0\x65\xc8\xdc"
+			  "\x5d\x45\x95\x30\x8f\xff\x2f\x1b",
+		.len	= 16,
+	}, {
+		.key	= "\x8a\xd4\xee\x10\x2f\xbd\x81\xff"
+			  "\xf8\x86\xce\xac\x93\xc5\xad\xc6"
+			  "\xa0\x19\x07\xc0\x9d\xf7\xbb\xdd"
+			  "\x52\x13\xb2\xb7\xf0\xff\x11\xd8"
+			  "\xd6\x08\xd0\xcd\x2e\xb1\x17\x6f",
+		.klen	= 40,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x02\x00\x00\x00\x00",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x0c\x20\x20\x63\xd6\x8b\xfc\x8f"
+			  "\xc0\xe2\x17\xbb\xd2\x59\x6f\x26",
+		.len	= 16,
+	}, {
+		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
+			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
+			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
+			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
+			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
+			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\xc1\x35\x2e\x53\xf0\x96\x4d\x9c"
+			  "\x2e\x18\xe6\x99\xcd\xd3\x15\x68",
+		.len	= 16,
+	}, {
+		.key	= "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
+			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
+			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
+			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
+			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
+			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x02\x00\x00\x00\x00",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x86\x0a\xc6\xa9\x1a\x9f\xe7\xe6"
+			  "\x64\x3b\x33\xd6\xd5\x84\xd6\xdf",
+		.len	= 16,
+	}, {
+		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
+			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
+			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
+			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
+			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
+			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
+			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
+			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
+			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
+			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
+			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
+			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
+			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
+			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
+			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
+			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
+			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
+			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
+			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
+			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
+			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
+			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
+			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
+			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
+			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
+			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
+			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
+			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
+			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
+			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
+			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
+			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
+			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
+			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
+			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
+			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
+			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
+			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
+			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
+			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
+			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
+			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
+			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
+			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
+			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
+			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
+			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
+			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
+			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
+			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
+			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
+			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
+			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
+			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
+			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
+			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
+			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
+			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
+			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
+			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
+			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
+			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
+			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
+			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
+			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
+			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
+			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
+			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
+			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
+		.ctext	= "\xe3\x5a\x38\x0f\x4d\x92\x3a\x74"
+			  "\x15\xb1\x50\x8c\x9a\xd8\x99\x1d"
+			  "\x82\xec\xf1\x5f\x03\x6d\x02\x58"
+			  "\x90\x67\xfc\xdd\x8d\xe1\x38\x08"
+			  "\x7b\xc9\x9b\x4b\x04\x09\x50\x15"
+			  "\xce\xab\xda\x33\x30\x20\x12\xfa"
+			  "\x83\xc4\xa6\x9a\x2e\x7d\x90\xd9"
+			  "\xa6\xa6\x67\x43\xb4\xa7\xa8\x5c"
+			  "\xbb\x6a\x49\x2b\x8b\xf8\xd0\x22"
+			  "\xe5\x9e\xba\xe8\x8c\x67\xb8\x5b"
+			  "\x60\xbc\xf5\xa4\x95\x4e\x66\xe5"
+			  "\x6d\x8e\xa9\xf6\x65\x2e\x04\xf5"
+			  "\xba\xb5\xdb\x88\xc2\xf6\x7a\x4b"
+			  "\x89\x58\x7c\x9a\xae\x26\xe8\xb7"
+			  "\xb7\x28\xcc\xd6\xcc\xa5\x98\x4d"
+			  "\xb9\x91\xcb\xb4\xe4\x8b\x96\x47"
+			  "\x5f\x03\x8b\xdd\x94\xd1\xee\x12"
+			  "\xa7\x83\x80\xf2\xc1\x15\x74\x4f"
+			  "\x49\xf9\xb0\x7e\x6f\xdc\x73\x2f"
+			  "\xe2\xcf\xe0\x1b\x34\xa5\xa0\x52"
+			  "\xfb\x3c\x5d\x85\x91\xe6\x6d\x98"
+			  "\x04\xd6\xdd\x4c\x00\x64\xd9\x54"
+			  "\x5c\x3c\x08\x1d\x4c\x06\x9f\xb8"
+			  "\x1c\x4d\x8d\xdc\xa4\x3c\xb9\x3b"
+			  "\x9e\x85\xce\xc3\xa8\x4a\x0c\xd9"
+			  "\x04\xc3\x6f\x17\x66\xa9\x1f\x59"
+			  "\xd9\xe2\x19\x36\xa3\x88\xb8\x0b"
+			  "\x0f\x4a\x4d\xf8\xc8\x6f\xd5\x43"
+			  "\xeb\xa0\xab\x1f\x61\xc0\x06\xeb"
+			  "\x93\xb7\xb8\x6f\x0d\xbd\x07\x49"
+			  "\xb3\xac\x5d\xcf\x31\xa0\x27\x26"
+			  "\x21\xbe\x94\x2e\x19\xea\xf4\xee"
+			  "\xb5\x13\x89\xf7\x94\x0b\xef\x59"
+			  "\x44\xc5\x78\x8b\x3c\x3b\x71\x20"
+			  "\xf9\x35\x0c\x70\x74\xdc\x5b\xc2"
+			  "\xb4\x11\x0e\x2c\x61\xa1\x52\x46"
+			  "\x18\x11\x16\xc6\x86\x44\xa7\xaf"
+			  "\xd5\x0c\x7d\xa6\x9e\x25\x2d\x1b"
+			  "\x9a\x8f\x0f\xf8\x6a\x61\xa0\xea"
+			  "\x3f\x0e\x90\xd6\x8f\x83\x30\x64"
+			  "\xb5\x51\x2d\x08\x3c\xcd\x99\x36"
+			  "\x96\xd4\xb1\xb5\x48\x30\xca\x48"
+			  "\xf7\x11\xa8\xf5\x97\x8a\x6a\x6d"
+			  "\x12\x33\x2f\xc0\xe8\xda\xec\x8a"
+			  "\xe1\x88\x72\x63\xde\x20\xa3\xe1"
+			  "\x8e\xac\x84\x37\x35\xf5\xf7\x3f"
+			  "\x00\x02\x0e\xe4\xc1\x53\x68\x3f"
+			  "\xaa\xd5\xac\x52\x3d\x20\x2f\x4d"
+			  "\x7c\x83\xd0\xbd\xaa\x97\x35\x36"
+			  "\x98\x88\x59\x5d\xe7\x24\xe3\x90"
+			  "\x9d\x30\x47\xa7\xc3\x60\x35\xf4"
+			  "\xd5\xdb\x0e\x4d\x44\xc1\x81\x8b"
+			  "\xfd\xbd\xc3\x2b\xba\x68\xfe\x8d"
+			  "\x49\x5a\x3c\x8a\xa3\x01\xae\x25"
+			  "\x42\xab\xd2\x87\x1b\x35\xd6\xd2"
+			  "\xd7\x70\x1c\x1f\x72\xd1\xe1\x39"
+			  "\x1c\x58\xa2\xb4\xd0\x78\x55\x72"
+			  "\x76\x59\xea\xd9\xd7\x6e\x63\x8b"
+			  "\xcc\x9b\xa7\x74\x89\xfc\xa3\x68"
+			  "\x86\x28\xd1\xbb\x54\x8d\x66\xad"
+			  "\x2a\x92\xf9\x4e\x04\x3d\xae\xfd"
+			  "\x1b\x2b\x7f\xc3\x2f\x1a\x78\x0a"
+			  "\x5c\xc6\x84\xfe\x7c\xcb\x26\xfd"
+			  "\xd9\x51\x0f\xd7\x94\x2f\xc5\xa7",
+		.len	= 512,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 512 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec serpent_xts_tv_template[] = {
+	/* Generated from AES-XTS test vectors */
+	{
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\xe1\x08\xb8\x1d\x2c\xf5\x33\x64"
+			  "\xc8\x12\x04\xc7\xb3\x70\xe8\xc4"
+			  "\x6a\x31\xc5\xf3\x00\xca\xb9\x16"
+			  "\xde\xe2\x77\x66\xf7\xfe\x62\x08",
+		.len	= 32,
+	}, {
+		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
+			  "\x11\x11\x11\x11\x11\x11\x11\x11"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22",
+		.klen	= 32,
+		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44",
+		.ctext	= "\x1a\x0a\x09\x5f\xcd\x07\x07\x98"
+			  "\x41\x86\x12\xaf\xb3\xd7\x68\x13"
+			  "\xed\x81\xcd\x06\x87\x43\x1a\xbb"
+			  "\x13\x3d\xd6\x1e\x2b\xe1\x77\xbe",
+		.len	= 32,
+	}, {
+		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
+			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22",
+		.klen	= 32,
+		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44",
+		.ctext	= "\xf9\x9b\x28\xb8\x5c\xaf\x8c\x61"
+			  "\xb6\x1c\x81\x8f\x2c\x87\x60\x89"
+			  "\x0d\x8d\x7a\xe8\x60\x48\xcc\x86"
+			  "\xc1\x68\x45\xaa\x00\xe9\x24\xc5",
+		.len	= 32,
+	}, {
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ctext	= "\xfe\x47\x4a\xc8\x60\x7e\xb4\x8b"
+			  "\x0d\x10\xf4\xb0\x0d\xba\xf8\x53"
+			  "\x65\x6e\x38\x4b\xdb\xaa\xb1\x9e"
+			  "\x28\xca\xb0\x22\xb3\x85\x75\xf4"
+			  "\x00\x5c\x75\x14\x06\xd6\x25\x82"
+			  "\xe6\xcb\x08\xf7\x29\x90\x23\x8e"
+			  "\xa4\x68\x57\xe4\xf0\xd8\x32\xf3"
+			  "\x80\x51\x67\xb5\x0b\x85\x69\xe8"
+			  "\x19\xfe\xc4\xc7\x3e\xea\x90\xd3"
+			  "\x8f\xa3\xf2\x0a\xac\x17\x4b\xa0"
+			  "\x63\x5a\x16\x0f\xf0\xce\x66\x1f"
+			  "\x2c\x21\x07\xf1\xa4\x03\xa3\x44"
+			  "\x41\x61\x87\x5d\x6b\xb3\xef\xd4"
+			  "\xfc\xaa\x32\x7e\x55\x58\x04\x41"
+			  "\xc9\x07\x33\xc6\xa2\x68\xd6\x5a"
+			  "\x55\x79\x4b\x6f\xcf\x89\xb9\x19"
+			  "\xe5\x54\x13\x15\xb2\x1a\xfa\x15"
+			  "\xc2\xf0\x06\x59\xfa\xa0\x25\x05"
+			  "\x58\xfa\x43\x91\x16\x85\x40\xbb"
+			  "\x0d\x34\x4d\xc5\x1e\x20\xd5\x08"
+			  "\xcd\x22\x22\x41\x11\x9f\x6c\x7c"
+			  "\x8d\x57\xc9\xba\x57\xe8\x2c\xf7"
+			  "\xa0\x42\xa8\xde\xfc\xa3\xca\x98"
+			  "\x4b\x43\xb1\xce\x4b\xbf\x01\x67"
+			  "\x6e\x29\x60\xbd\x10\x14\x84\x82"
+			  "\x83\x82\x0c\x63\x73\x92\x02\x7c"
+			  "\x55\x37\x20\x80\x17\x51\xc8\xbc"
+			  "\x46\x02\xcb\x38\x07\x6d\xe2\x85"
+			  "\xaa\x29\xaf\x24\x58\x0d\xf0\x75"
+			  "\x08\x0a\xa5\x34\x25\x16\xf3\x74"
+			  "\xa7\x0b\x97\xbe\xc1\xa9\xdc\x29"
+			  "\x1a\x0a\x56\xc1\x1a\x91\x97\x8c"
+			  "\x0b\xc7\x16\xed\x5a\x22\xa6\x2e"
+			  "\x8c\x2b\x4f\x54\x76\x47\x53\x8e"
+			  "\xe8\x00\xec\x92\xb9\x55\xe6\xa2"
+			  "\xf3\xe2\x4f\x6a\x66\x60\xd0\x87"
+			  "\xe6\xd1\xcc\xe3\x6a\xc5\x2d\x21"
+			  "\xcc\x9d\x6a\xb6\x75\xaa\xe2\x19"
+			  "\x21\x9f\xa1\x5e\x4c\xfd\x72\xf9"
+			  "\x94\x4e\x63\xc7\xae\xfc\xed\x47"
+			  "\xe2\xfe\x7a\x63\x77\xfe\x97\x82"
+			  "\xb1\x10\x6e\x36\x1d\xe1\xc4\x80"
+			  "\xec\x69\x41\xec\xa7\x8a\xe0\x2f"
+			  "\xe3\x49\x26\xa2\x41\xb2\x08\x0f"
+			  "\x28\xb4\xa7\x39\xa1\x99\x2d\x1e"
+			  "\x43\x42\x35\xd0\xcf\xec\x77\x67"
+			  "\xb2\x3b\x9e\x1c\x35\xde\x4f\x5e"
+			  "\x73\x3f\x5d\x6f\x07\x4b\x2e\x50"
+			  "\xab\x6c\x6b\xff\xea\x00\x67\xaa"
+			  "\x0e\x82\x32\xdd\x3d\xb5\xe5\x76"
+			  "\x2b\x77\x3f\xbe\x12\x75\xfb\x92"
+			  "\xc6\x89\x67\x4d\xca\xf7\xd4\x50"
+			  "\xc0\x74\x47\xcc\xd9\x0a\xd4\xc6"
+			  "\x3b\x17\x2e\xe3\x35\xbb\x53\xb5"
+			  "\x86\xad\x51\xcc\xd5\x96\xb8\xdc"
+			  "\x03\x57\xe6\x98\x52\x2f\x61\x62"
+			  "\xc4\x5c\x9c\x36\x71\x07\xfb\x94"
+			  "\xe3\x02\xc4\x2b\x08\x75\xc7\x35"
+			  "\xfb\x2e\x88\x7b\xbb\x67\x00\xe1"
+			  "\xc9\xdd\x99\xb2\x13\x53\x1a\x4e"
+			  "\x76\x87\x19\x04\x1a\x2f\x38\x3e"
+			  "\xef\x91\x64\x1d\x18\x07\x4e\x31"
+			  "\x88\x21\x7c\xb0\xa5\x12\x4c\x3c"
+			  "\xb0\x20\xbd\xda\xdf\xf9\x7c\xdd",
+		.len	= 512,
+	}, {
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x62\x49\x77\x57\x24\x70\x93\x69"
+			  "\x99\x59\x57\x49\x66\x96\x76\x27"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95"
+			  "\x02\x88\x41\x97\x16\x93\x99\x37"
+			  "\x51\x05\x82\x09\x74\x94\x45\x92",
+		.klen	= 64,
+		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ctext	= "\x2b\xc9\xb4\x6b\x10\x94\xa9\x32"
+			  "\xaa\xb0\x20\xc6\x44\x3d\x74\x1f"
+			  "\x75\x01\xa7\xf6\xf5\xf7\x62\x1b"
+			  "\x80\x1b\x82\xcb\x01\x59\x91\x7f"
+			  "\x80\x3a\x98\xf0\xd2\xca\xc4\xc3"
+			  "\x34\xfd\xe6\x11\xf9\x33\x45\x12"
+			  "\x48\xc5\x8c\x25\xf1\xc5\xc5\x23"
+			  "\xd3\x44\xb4\x73\xd5\x04\xc0\xb7"
+			  "\xca\x2f\xf5\xcd\xc5\xb4\xdd\xb0"
+			  "\xf4\x60\xe8\xfb\xc6\x9c\xc5\x78"
+			  "\xcd\xec\x7d\xdc\x19\x9c\x72\x64"
+			  "\x63\x0b\x38\x2e\x76\xdd\x2d\x36"
+			  "\x49\xb0\x1d\xea\x78\x9e\x00\xca"
+			  "\x20\xcc\x1b\x1e\x98\x74\xab\xed"
+			  "\x79\xf7\xd0\x6c\xd8\x93\x80\x29"
+			  "\xac\xa5\x5e\x34\xa9\xab\xa0\x55"
+			  "\x9a\xea\xaa\x95\x4d\x7b\xfe\x46"
+			  "\x26\x8a\xfd\x88\xa2\xa8\xa6\xae"
+			  "\x25\x42\x17\xbf\x76\x8f\x1c\x3d"
+			  "\xec\x9a\xda\x64\x96\xb5\x61\xff"
+			  "\x99\xeb\x12\x96\x85\x82\x9d\xd5"
+			  "\x81\x85\x14\xa8\x59\xac\x8c\x94"
+			  "\xbb\x3b\x85\x2b\xdf\xb3\x0c\xba"
+			  "\x82\xc6\x4d\xca\x86\xea\x53\x28"
+			  "\x4c\xe0\x4e\x31\xe3\x73\x2f\x79"
+			  "\x9d\x42\xe1\x03\xe3\x8b\xc4\xff"
+			  "\x05\xca\x81\x7b\xda\xa2\xde\x63"
+			  "\x3a\x10\xbe\xc2\xac\x32\xc4\x05"
+			  "\x47\x7e\xef\x67\xe2\x5f\x5b\xae"
+			  "\xed\xf1\x70\x34\x16\x9a\x07\x7b"
+			  "\xf2\x25\x2b\xb0\xf8\x3c\x15\x9a"
+			  "\xa6\x59\x55\x5f\xc1\xf4\x1e\xcd"
+			  "\x93\x1f\x06\xba\xd4\x9a\x22\x69"
+			  "\xfa\x8e\x95\x0d\xf3\x23\x59\x2c"
+			  "\xfe\x00\xba\xf0\x0e\xbc\x6d\xd6"
+			  "\x62\xf0\x7a\x0e\x83\x3e\xdb\x32"
+			  "\xfd\x43\x7d\xda\x42\x51\x87\x43"
+			  "\x9d\xf9\xef\xf4\x30\x97\xf8\x09"
+			  "\x88\xfc\x3f\x93\x70\xc1\x4a\xec"
+			  "\x27\x5f\x11\xac\x71\xc7\x48\x46"
+			  "\x2f\xf9\xdf\x8d\x9f\xf7\x2e\x56"
+			  "\x0d\x4e\xb0\x32\x76\xce\x86\x81"
+			  "\xcd\xdf\xe4\x00\xbf\xfd\x5f\x24"
+			  "\xaf\xf7\x9a\xde\xff\x18\xac\x14"
+			  "\x90\xc5\x01\x39\x34\x0f\x24\xf3"
+			  "\x13\x2f\x5e\x4f\x30\x9a\x36\x40"
+			  "\xec\xea\xbc\xcd\x9e\x0e\x5b\x23"
+			  "\x50\x88\x97\x40\x69\xb1\x37\xf5"
+			  "\xc3\x15\xf9\x3f\xb7\x79\x64\xe8"
+			  "\x7b\x10\x20\xb9\x2b\x46\x83\x5b"
+			  "\xd8\x39\xfc\xe4\xfa\x88\x52\xf2"
+			  "\x72\xb0\x97\x4e\x89\xb3\x48\x00"
+			  "\xc1\x16\x73\x50\x77\xba\xa6\x65"
+			  "\x20\x2d\xb0\x02\x27\x89\xda\x99"
+			  "\x45\xfb\xe9\xd3\x1d\x39\x2f\xd6"
+			  "\x2a\xda\x09\x12\x11\xaf\xe6\x57"
+			  "\x01\x04\x8a\xff\x86\x8b\xac\xf8"
+			  "\xee\xe4\x1c\x98\x5b\xcf\x6b\x76"
+			  "\xa3\x0e\x33\x74\x40\x18\x39\x72"
+			  "\x66\x50\x31\xfd\x70\xdf\xe8\x51"
+			  "\x96\x21\x36\xb2\x9b\xfa\x85\xd1"
+			  "\x30\x05\xc8\x92\x98\x80\xff\x7a"
+			  "\xaf\x43\x0b\xc5\x20\x41\x92\x20"
+			  "\xd4\xa0\x91\x98\x11\x5f\x4d\xb1",
+		.len	= 512,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 512 - 20, 4, 16 },
+	},
+};
+
+/*
+ * SM4 test vector taken from the draft RFC
+ * https://tools.ietf.org/html/draft-crypto-sm4-00#ref-GBT.32907-2016
+ */
+
+static const struct cipher_testvec sm4_tv_template[] = {
+	{ /* SM4 Appendix A: Example Calculations. Example 1. */
+		.key	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
+			  "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
+		.klen	= 16,
+		.ptext	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
+			  "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
+		.ctext	= "\x68\x1E\xDF\x34\xD2\x06\x96\x5E"
+			  "\x86\xB3\xE9\x4F\x53\x6E\x42\x46",
+		.len	= 16,
+	}, { /*
+	      *  SM4 Appendix A: Example Calculations.
+	      *  Last 10 iterations of Example 2.
+	      */
+		.key    = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
+			  "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
+		.klen	= 16,
+		.ptext	= "\x99\x4a\xc3\xe7\xc3\x57\x89\x6a"
+			  "\x81\xfc\xa8\xe\x38\x3e\xef\x80"
+			  "\xb1\x98\xf2\xde\x3f\x4b\xae\xd1"
+			  "\xf0\xf1\x30\x4c\x1\x27\x5a\x8f"
+			  "\x45\xe1\x39\xb7\xae\xff\x1f\x27"
+			  "\xad\x57\x15\xab\x31\x5d\xc\xef"
+			  "\x8c\xc8\x80\xbd\x11\x98\xf3\x7b"
+			  "\xa2\xdd\x14\x20\xf9\xe8\xbb\x82"
+			  "\xf7\x32\xca\x4b\xa8\xf7\xb3\x4d"
+			  "\x27\xd1\xcd\xe6\xb6\x65\x5a\x23"
+			  "\xc2\xf3\x54\x84\x53\xe3\xb9\x20"
+			  "\xa5\x37\x0\xbe\xe7\x7b\x48\xfb"
+			  "\x21\x3d\x9e\x48\x1d\x9e\xf5\xbf"
+			  "\x77\xd5\xb4\x4a\x53\x71\x94\x7a"
+			  "\x88\xa6\x6e\x6\x93\xca\x43\xa5"
+			  "\xc4\xf6\xcd\x53\x4b\x7b\x8e\xfe"
+			  "\xb4\x28\x7c\x42\x29\x32\x5d\x88"
+			  "\xed\xce\x0\x19\xe\x16\x2\x6e"
+			  "\x87\xff\x2c\xac\xe8\xe7\xe9\xbf"
+			  "\x31\x51\xec\x47\xc3\x51\x83\xc1",
+		.ctext	= "\xb1\x98\xf2\xde\x3f\x4b\xae\xd1"
+			  "\xf0\xf1\x30\x4c\x1\x27\x5a\x8f"
+			  "\x45\xe1\x39\xb7\xae\xff\x1f\x27"
+			  "\xad\x57\x15\xab\x31\x5d\xc\xef"
+			  "\x8c\xc8\x80\xbd\x11\x98\xf3\x7b"
+			  "\xa2\xdd\x14\x20\xf9\xe8\xbb\x82"
+			  "\xf7\x32\xca\x4b\xa8\xf7\xb3\x4d"
+			  "\x27\xd1\xcd\xe6\xb6\x65\x5a\x23"
+			  "\xc2\xf3\x54\x84\x53\xe3\xb9\x20"
+			  "\xa5\x37\x0\xbe\xe7\x7b\x48\xfb"
+			  "\x21\x3d\x9e\x48\x1d\x9e\xf5\xbf"
+			  "\x77\xd5\xb4\x4a\x53\x71\x94\x7a"
+			  "\x88\xa6\x6e\x6\x93\xca\x43\xa5"
+			  "\xc4\xf6\xcd\x53\x4b\x7b\x8e\xfe"
+			  "\xb4\x28\x7c\x42\x29\x32\x5d\x88"
+			  "\xed\xce\x0\x19\xe\x16\x2\x6e"
+			  "\x87\xff\x2c\xac\xe8\xe7\xe9\xbf"
+			  "\x31\x51\xec\x47\xc3\x51\x83\xc1"
+			  "\x59\x52\x98\xc7\xc6\xfd\x27\x1f"
+			  "\x4\x2\xf8\x4\xc3\x3d\x3f\x66",
+		.len	= 160
+	}
+};
+
+/*
+ * Speck test vectors taken from the original paper:
+ * "The Simon and Speck Families of Lightweight Block Ciphers"
+ * https://eprint.iacr.org/2013/404.pdf
+ *
+ * Note that the paper does not make byte and word order clear.  But it was
+ * confirmed with the authors that the intended orders are little endian byte
+ * order and (y, x) word order.  Equivalently, the printed test vectors, when
+ * looking at only the bytes (ignoring the whitespace that divides them into
+ * words), are backwards: the left-most byte is actually the one with the
+ * highest memory address, while the right-most byte is actually the one with
+ * the lowest memory address.
+ */
+
+static const struct cipher_testvec speck128_tv_template[] = {
+	{ /* Speck128/128 */
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.klen	= 16,
+		.ptext	= "\x20\x6d\x61\x64\x65\x20\x69\x74"
+			  "\x20\x65\x71\x75\x69\x76\x61\x6c",
+		.ctext	= "\x18\x0d\x57\x5c\xdf\xfe\x60\x78"
+			  "\x65\x32\x78\x79\x51\x98\x5d\xa6",
+		.len	= 16,
+	}, { /* Speck128/192 */
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17",
+		.klen	= 24,
+		.ptext	= "\x65\x6e\x74\x20\x74\x6f\x20\x43"
+			  "\x68\x69\x65\x66\x20\x48\x61\x72",
+		.ctext	= "\x86\x18\x3c\xe0\x5d\x18\xbc\xf9"
+			  "\x66\x55\x13\x13\x3a\xcf\xe4\x1b",
+		.len	= 16,
+	}, { /* Speck128/256 */
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.klen	= 32,
+		.ptext	= "\x70\x6f\x6f\x6e\x65\x72\x2e\x20"
+			  "\x49\x6e\x20\x74\x68\x6f\x73\x65",
+		.ctext	= "\x43\x8f\x18\x9c\x8d\xb4\xee\x4e"
+			  "\x3e\xf5\xc0\x05\x04\x01\x09\x41",
+		.len	= 16,
+	},
+};
+
+/*
+ * Speck128-XTS test vectors, taken from the AES-XTS test vectors with the
+ * ciphertext recomputed with Speck128 as the cipher
+ */
+static const struct cipher_testvec speck128_xts_tv_template[] = {
+	{
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\xbe\xa0\xe7\x03\xd7\xfe\xab\x62"
+			  "\x3b\x99\x4a\x64\x74\x77\xac\xed"
+			  "\xd8\xf4\xa6\xcf\xae\xb9\x07\x42"
+			  "\x51\xd9\xb6\x1d\xe0\x5e\xbc\x54",
+		.len	= 32,
+	}, {
+		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
+			  "\x11\x11\x11\x11\x11\x11\x11\x11"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22",
+		.klen	= 32,
+		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44",
+		.ctext	= "\xfb\x53\x81\x75\x6f\x9f\x34\xad"
+			  "\x7e\x01\xed\x7b\xcc\xda\x4e\x4a"
+			  "\xd4\x84\xa4\x53\xd5\x88\x73\x1b"
+			  "\xfd\xcb\xae\x0d\xf3\x04\xee\xe6",
+		.len	= 32,
+	}, {
+		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
+			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22",
+		.klen	= 32,
+		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44",
+		.ctext	= "\x21\x52\x84\x15\xd1\xf7\x21\x55"
+			  "\xd9\x75\x4a\xd3\xc5\xdb\x9f\x7d"
+			  "\xda\x63\xb2\xf1\x82\xb0\x89\x59"
+			  "\x86\xd4\xaa\xaa\xdd\xff\x4f\x92",
+		.len	= 32,
+	}, {
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ctext	= "\x57\xb5\xf8\x71\x6e\x6d\xdd\x82"
+			  "\x53\xd0\xed\x2d\x30\xc1\x20\xef"
+			  "\x70\x67\x5e\xff\x09\x70\xbb\xc1"
+			  "\x3a\x7b\x48\x26\xd9\x0b\xf4\x48"
+			  "\xbe\xce\xb1\xc7\xb2\x67\xc4\xa7"
+			  "\x76\xf8\x36\x30\xb7\xb4\x9a\xd9"
+			  "\xf5\x9d\xd0\x7b\xc1\x06\x96\x44"
+			  "\x19\xc5\x58\x84\x63\xb9\x12\x68"
+			  "\x68\xc7\xaa\x18\x98\xf2\x1f\x5c"
+			  "\x39\xa6\xd8\x32\x2b\xc3\x51\xfd"
+			  "\x74\x79\x2e\xb4\x44\xd7\x69\xc4"
+			  "\xfc\x29\xe6\xed\x26\x1e\xa6\x9d"
+			  "\x1c\xbe\x00\x0e\x7f\x3a\xca\xfb"
+			  "\x6d\x13\x65\xa0\xf9\x31\x12\xe2"
+			  "\x26\xd1\xec\x2b\x0a\x8b\x59\x99"
+			  "\xa7\x49\xa0\x0e\x09\x33\x85\x50"
+			  "\xc3\x23\xca\x7a\xdd\x13\x45\x5f"
+			  "\xde\x4c\xa7\xcb\x00\x8a\x66\x6f"
+			  "\xa2\xb6\xb1\x2e\xe1\xa0\x18\xf6"
+			  "\xad\xf3\xbd\xeb\xc7\xef\x55\x4f"
+			  "\x79\x91\x8d\x36\x13\x7b\xd0\x4a"
+			  "\x6c\x39\xfb\x53\xb8\x6f\x02\x51"
+			  "\xa5\x20\xac\x24\x1c\x73\x59\x73"
+			  "\x58\x61\x3a\x87\x58\xb3\x20\x56"
+			  "\x39\x06\x2b\x4d\xd3\x20\x2b\x89"
+			  "\x3f\xa2\xf0\x96\xeb\x7f\xa4\xcd"
+			  "\x11\xae\xbd\xcb\x3a\xb4\xd9\x91"
+			  "\x09\x35\x71\x50\x65\xac\x92\xe3"
+			  "\x7b\x32\xc0\x7a\xdd\xd4\xc3\x92"
+			  "\x6f\xeb\x79\xde\x6f\xd3\x25\xc9"
+			  "\xcd\x63\xf5\x1e\x7a\x3b\x26\x9d"
+			  "\x77\x04\x80\xa9\xbf\x38\xb5\xbd"
+			  "\xb8\x05\x07\xbd\xfd\xab\x7b\xf8"
+			  "\x2a\x26\xcc\x49\x14\x6d\x55\x01"
+			  "\x06\x94\xd8\xb2\x2d\x53\x83\x1b"
+			  "\x8f\xd4\xdd\x57\x12\x7e\x18\xba"
+			  "\x8e\xe2\x4d\x80\xef\x7e\x6b\x9d"
+			  "\x24\xa9\x60\xa4\x97\x85\x86\x2a"
+			  "\x01\x00\x09\xf1\xcb\x4a\x24\x1c"
+			  "\xd8\xf6\xe6\x5b\xe7\x5d\xf2\xc4"
+			  "\x97\x1c\x10\xc6\x4d\x66\x4f\x98"
+			  "\x87\x30\xac\xd5\xea\x73\x49\x10"
+			  "\x80\xea\xe5\x5f\x4d\x5f\x03\x33"
+			  "\x66\x02\x35\x3d\x60\x06\x36\x4f"
+			  "\x14\x1c\xd8\x07\x1f\x78\xd0\xf8"
+			  "\x4f\x6c\x62\x7c\x15\xa5\x7c\x28"
+			  "\x7c\xcc\xeb\x1f\xd1\x07\x90\x93"
+			  "\x7e\xc2\xa8\x3a\x80\xc0\xf5\x30"
+			  "\xcc\x75\xcf\x16\x26\xa9\x26\x3b"
+			  "\xe7\x68\x2f\x15\x21\x5b\xe4\x00"
+			  "\xbd\x48\x50\xcd\x75\x70\xc4\x62"
+			  "\xbb\x41\xfb\x89\x4a\x88\x3b\x3b"
+			  "\x51\x66\x02\x69\x04\x97\x36\xd4"
+			  "\x75\xae\x0b\xa3\x42\xf8\xca\x79"
+			  "\x8f\x93\xe9\xcc\x38\xbd\xd6\xd2"
+			  "\xf9\x70\x4e\xc3\x6a\x8e\x25\xbd"
+			  "\xea\x15\x5a\xa0\x85\x7e\x81\x0d"
+			  "\x03\xe7\x05\x39\xf5\x05\x26\xee"
+			  "\xec\xaa\x1f\x3d\xc9\x98\x76\x01"
+			  "\x2c\xf4\xfc\xa3\x88\x77\x38\xc4"
+			  "\x50\x65\x50\x6d\x04\x1f\xdf\x5a"
+			  "\xaa\xf2\x01\xa9\xc1\x8d\xee\xca"
+			  "\x47\x26\xef\x39\xb8\xb4\xf2\xd1"
+			  "\xd6\xbb\x1b\x2a\xc1\x34\x14\xcf",
+		.len	= 512,
+	}, {
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x62\x49\x77\x57\x24\x70\x93\x69"
+			  "\x99\x59\x57\x49\x66\x96\x76\x27"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95"
+			  "\x02\x88\x41\x97\x16\x93\x99\x37"
+			  "\x51\x05\x82\x09\x74\x94\x45\x92",
+		.klen	= 64,
+		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ctext	= "\xc5\x85\x2a\x4b\x73\xe4\xf6\xf1"
+			  "\x7e\xf9\xf6\xe9\xa3\x73\x36\xcb"
+			  "\xaa\xb6\x22\xb0\x24\x6e\x3d\x73"
+			  "\x92\x99\xde\xd3\x76\xed\xcd\x63"
+			  "\x64\x3a\x22\x57\xc1\x43\x49\xd4"
+			  "\x79\x36\x31\x19\x62\xae\x10\x7e"
+			  "\x7d\xcf\x7a\xe2\x6b\xce\x27\xfa"
+			  "\xdc\x3d\xd9\x83\xd3\x42\x4c\xe0"
+			  "\x1b\xd6\x1d\x1a\x6f\xd2\x03\x00"
+			  "\xfc\x81\x99\x8a\x14\x62\xf5\x7e"
+			  "\x0d\xe7\x12\xe8\x17\x9d\x0b\xec"
+			  "\xe2\xf7\xc9\xa7\x63\xd1\x79\xb6"
+			  "\x62\x62\x37\xfe\x0a\x4c\x4a\x37"
+			  "\x70\xc7\x5e\x96\x5f\xbc\x8e\x9e"
+			  "\x85\x3c\x4f\x26\x64\x85\xbc\x68"
+			  "\xb0\xe0\x86\x5e\x26\x41\xce\x11"
+			  "\x50\xda\x97\x14\xe9\x9e\xc7\x6d"
+			  "\x3b\xdc\x43\xde\x2b\x27\x69\x7d"
+			  "\xfc\xb0\x28\xbd\x8f\xb1\xc6\x31"
+			  "\x14\x4d\xf0\x74\x37\xfd\x07\x25"
+			  "\x96\x55\xe5\xfc\x9e\x27\x2a\x74"
+			  "\x1b\x83\x4d\x15\x83\xac\x57\xa0"
+			  "\xac\xa5\xd0\x38\xef\x19\x56\x53"
+			  "\x25\x4b\xfc\xce\x04\x23\xe5\x6b"
+			  "\xf6\xc6\x6c\x32\x0b\xb3\x12\xc5"
+			  "\xed\x22\x34\x1c\x5d\xed\x17\x06"
+			  "\x36\xa3\xe6\x77\xb9\x97\x46\xb8"
+			  "\xe9\x3f\x7e\xc7\xbc\x13\x5c\xdc"
+			  "\x6e\x3f\x04\x5e\xd1\x59\xa5\x82"
+			  "\x35\x91\x3d\x1b\xe4\x97\x9f\x92"
+			  "\x1c\x5e\x5f\x6f\x41\xd4\x62\xa1"
+			  "\x8d\x39\xfc\x42\xfb\x38\x80\xb9"
+			  "\x0a\xe3\xcc\x6a\x93\xd9\x7a\xb1"
+			  "\xe9\x69\xaf\x0a\x6b\x75\x38\xa7"
+			  "\xa1\xbf\xf7\xda\x95\x93\x4b\x78"
+			  "\x19\xf5\x94\xf9\xd2\x00\x33\x37"
+			  "\xcf\xf5\x9e\x9c\xf3\xcc\xa6\xee"
+			  "\x42\xb2\x9e\x2c\x5f\x48\x23\x26"
+			  "\x15\x25\x17\x03\x3d\xfe\x2c\xfc"
+			  "\xeb\xba\xda\xe0\x00\x05\xb6\xa6"
+			  "\x07\xb3\xe8\x36\x5b\xec\x5b\xbf"
+			  "\xd6\x5b\x00\x74\xc6\x97\xf1\x6a"
+			  "\x49\xa1\xc3\xfa\x10\x52\xb9\x14"
+			  "\xad\xb7\x73\xf8\x78\x12\xc8\x59"
+			  "\x17\x80\x4c\x57\x39\xf1\x6d\x80"
+			  "\x25\x77\x0f\x5e\x7d\xf0\xaf\x21"
+			  "\xec\xce\xb7\xc8\x02\x8a\xed\x53"
+			  "\x2c\x25\x68\x2e\x1f\x85\x5e\x67"
+			  "\xd1\x07\x7a\x3a\x89\x08\xe0\x34"
+			  "\xdc\xdb\x26\xb4\x6b\x77\xfc\x40"
+			  "\x31\x15\x72\xa0\xf0\x73\xd9\x3b"
+			  "\xd5\xdb\xfe\xfc\x8f\xa9\x44\xa2"
+			  "\x09\x9f\xc6\x33\xe5\xe2\x88\xe8"
+			  "\xf3\xf0\x1a\xf4\xce\x12\x0f\xd6"
+			  "\xf7\x36\xe6\xa4\xf4\x7a\x10\x58"
+			  "\xcc\x1f\x48\x49\x65\x47\x75\xe9"
+			  "\x28\xe1\x65\x7b\xf2\xc4\xb5\x07"
+			  "\xf2\xec\x76\xd8\x8f\x09\xf3\x16"
+			  "\xa1\x51\x89\x3b\xeb\x96\x42\xac"
+			  "\x65\xe0\x67\x63\x29\xdc\xb4\x7d"
+			  "\xf2\x41\x51\x6a\xcb\xde\x3c\xfb"
+			  "\x66\x8d\x13\xca\xe0\x59\x2a\x00"
+			  "\xc9\x53\x4c\xe6\x9e\xe2\x73\xd5"
+			  "\x67\x19\xb2\xbd\x9a\x63\xd7\x5c",
+		.len	= 512,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 512 - 20, 4, 16 },
+	}
+};
+
+static const struct cipher_testvec speck64_tv_template[] = {
+	{ /* Speck64/96 */
+		.key	= "\x00\x01\x02\x03\x08\x09\x0a\x0b"
+			  "\x10\x11\x12\x13",
+		.klen	= 12,
+		.ptext	= "\x65\x61\x6e\x73\x20\x46\x61\x74",
+		.ctext	= "\x6c\x94\x75\x41\xec\x52\x79\x9f",
+		.len	= 8,
+	}, { /* Speck64/128 */
+		.key	= "\x00\x01\x02\x03\x08\x09\x0a\x0b"
+			  "\x10\x11\x12\x13\x18\x19\x1a\x1b",
+		.klen	= 16,
+		.ptext	= "\x2d\x43\x75\x74\x74\x65\x72\x3b",
+		.ctext	= "\x8b\x02\x4e\x45\x48\xa5\x6f\x8c",
+		.len	= 8,
+	},
+};
+
+/*
+ * Speck64-XTS test vectors, taken from the AES-XTS test vectors with the
+ * ciphertext recomputed with Speck64 as the cipher, and key lengths adjusted
+ */
+static const struct cipher_testvec speck64_xts_tv_template[] = {
+	{
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 24,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\x84\xaf\x54\x07\x19\xd4\x7c\xa6"
+			  "\xe4\xfe\xdf\xc4\x1f\x34\xc3\xc2"
+			  "\x80\xf5\x72\xe7\xcd\xf0\x99\x22"
+			  "\x35\xa7\x2f\x06\xef\xdc\x51\xaa",
+		.len	= 32,
+	}, {
+		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
+			  "\x11\x11\x11\x11\x11\x11\x11\x11"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22",
+		.klen	= 24,
+		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44",
+		.ctext	= "\x12\x56\x73\xcd\x15\x87\xa8\x59"
+			  "\xcf\x84\xae\xd9\x1c\x66\xd6\x9f"
+			  "\xb3\x12\x69\x7e\x36\xeb\x52\xff"
+			  "\x62\xdd\xba\x90\xb3\xe1\xee\x99",
+		.len	= 32,
+	}, {
+		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
+			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22",
+		.klen	= 24,
+		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44",
+		.ctext	= "\x15\x1b\xe4\x2c\xa2\x5a\x2d\x2c"
+			  "\x27\x36\xc0\xbf\x5d\xea\x36\x37"
+			  "\x2d\x1a\x88\xbc\x66\xb5\xd0\x0b"
+			  "\xa1\xbc\x19\xb2\x0f\x3b\x75\x34",
+		.len	= 32,
+	}, {
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93",
+		.klen	= 24,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ctext	= "\xaf\xa1\x81\xa6\x32\xbb\x15\x8e"
+			  "\xf8\x95\x2e\xd3\xe6\xee\x7e\x09"
+			  "\x0c\x1a\xf5\x02\x97\x8b\xe3\xb3"
+			  "\x11\xc7\x39\x96\xd0\x95\xf4\x56"
+			  "\xf4\xdd\x03\x38\x01\x44\x2c\xcf"
+			  "\x88\xae\x8e\x3c\xcd\xe7\xaa\x66"
+			  "\xfe\x3d\xc6\xfb\x01\x23\x51\x43"
+			  "\xd5\xd2\x13\x86\x94\x34\xe9\x62"
+			  "\xf9\x89\xe3\xd1\x7b\xbe\xf8\xef"
+			  "\x76\x35\x04\x3f\xdb\x23\x9d\x0b"
+			  "\x85\x42\xb9\x02\xd6\xcc\xdb\x96"
+			  "\xa7\x6b\x27\xb6\xd4\x45\x8f\x7d"
+			  "\xae\xd2\x04\xd5\xda\xc1\x7e\x24"
+			  "\x8c\x73\xbe\x48\x7e\xcf\x65\x28"
+			  "\x29\xe5\xbe\x54\x30\xcb\x46\x95"
+			  "\x4f\x2e\x8a\x36\xc8\x27\xc5\xbe"
+			  "\xd0\x1a\xaf\xab\x26\xcd\x9e\x69"
+			  "\xa1\x09\x95\x71\x26\xe9\xc4\xdf"
+			  "\xe6\x31\xc3\x46\xda\xaf\x0b\x41"
+			  "\x1f\xab\xb1\x8e\xd6\xfc\x0b\xb3"
+			  "\x82\xc0\x37\x27\xfc\x91\xa7\x05"
+			  "\xfb\xc5\xdc\x2b\x74\x96\x48\x43"
+			  "\x5d\x9c\x19\x0f\x60\x63\x3a\x1f"
+			  "\x6f\xf0\x03\xbe\x4d\xfd\xc8\x4a"
+			  "\xc6\xa4\x81\x6d\xc3\x12\x2a\x5c"
+			  "\x07\xff\xf3\x72\x74\x48\xb5\x40"
+			  "\x50\xb5\xdd\x90\x43\x31\x18\x15"
+			  "\x7b\xf2\xa6\xdb\x83\xc8\x4b\x4a"
+			  "\x29\x93\x90\x8b\xda\x07\xf0\x35"
+			  "\x6d\x90\x88\x09\x4e\x83\xf5\x5b"
+			  "\x94\x12\xbb\x33\x27\x1d\x3f\x23"
+			  "\x51\xa8\x7c\x07\xa2\xae\x77\xa6"
+			  "\x50\xfd\xcc\xc0\x4f\x80\x7a\x9f"
+			  "\x66\xdd\xcd\x75\x24\x8b\x33\xf7"
+			  "\x20\xdb\x83\x9b\x4f\x11\x63\x6e"
+			  "\xcf\x37\xef\xc9\x11\x01\x5c\x45"
+			  "\x32\x99\x7c\x3c\x9e\x42\x89\xe3"
+			  "\x70\x6d\x15\x9f\xb1\xe6\xb6\x05"
+			  "\xfe\x0c\xb9\x49\x2d\x90\x6d\xcc"
+			  "\x5d\x3f\xc1\xfe\x89\x0a\x2e\x2d"
+			  "\xa0\xa8\x89\x3b\x73\x39\xa5\x94"
+			  "\x4c\xa4\xa6\xbb\xa7\x14\x46\x89"
+			  "\x10\xff\xaf\xef\xca\xdd\x4f\x80"
+			  "\xb3\xdf\x3b\xab\xd4\xe5\x5a\xc7"
+			  "\x33\xca\x00\x8b\x8b\x3f\xea\xec"
+			  "\x68\x8a\xc2\x6d\xfd\xd4\x67\x0f"
+			  "\x22\x31\xe1\x0e\xfe\x5a\x04\xd5"
+			  "\x64\xa3\xf1\x1a\x76\x28\xcc\x35"
+			  "\x36\xa7\x0a\x74\xf7\x1c\x44\x9b"
+			  "\xc7\x1b\x53\x17\x02\xea\xd1\xad"
+			  "\x13\x51\x73\xc0\xa0\xb2\x05\x32"
+			  "\xa8\xa2\x37\x2e\xe1\x7a\x3a\x19"
+			  "\x26\xb4\x6c\x62\x5d\xb3\x1a\x1d"
+			  "\x59\xda\xee\x1a\x22\x18\xda\x0d"
+			  "\x88\x0f\x55\x8b\x72\x62\xfd\xc1"
+			  "\x69\x13\xcd\x0d\x5f\xc1\x09\x52"
+			  "\xee\xd6\xe3\x84\x4d\xee\xf6\x88"
+			  "\xaf\x83\xdc\x76\xf4\xc0\x93\x3f"
+			  "\x4a\x75\x2f\xb0\x0b\x3e\xc4\x54"
+			  "\x7d\x69\x8d\x00\x62\x77\x0d\x14"
+			  "\xbe\x7c\xa6\x7d\xc5\x24\x4f\xf3"
+			  "\x50\xf7\x5f\xf4\xc2\xca\x41\x97"
+			  "\x37\xbe\x75\x74\xcd\xf0\x75\x6e"
+			  "\x25\x23\x94\xbd\xda\x8d\xb0\xd4",
+		.len	= 512,
+	}, {
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x62\x49\x77\x57\x24\x70\x93\x69"
+			  "\x99\x59\x57\x49\x66\x96\x76\x27",
+		.klen	= 32,
+		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ctext	= "\x55\xed\x71\xd3\x02\x8e\x15\x3b"
+			  "\xc6\x71\x29\x2d\x3e\x89\x9f\x59"
+			  "\x68\x6a\xcc\x8a\x56\x97\xf3\x95"
+			  "\x4e\x51\x08\xda\x2a\xf8\x6f\x3c"
+			  "\x78\x16\xea\x80\xdb\x33\x75\x94"
+			  "\xf9\x29\xc4\x2b\x76\x75\x97\xc7"
+			  "\xf2\x98\x2c\xf9\xff\xc8\xd5\x2b"
+			  "\x18\xf1\xaf\xcf\x7c\xc5\x0b\xee"
+			  "\xad\x3c\x76\x7c\xe6\x27\xa2\x2a"
+			  "\xe4\x66\xe1\xab\xa2\x39\xfc\x7c"
+			  "\xf5\xec\x32\x74\xa3\xb8\x03\x88"
+			  "\x52\xfc\x2e\x56\x3f\xa1\xf0\x9f"
+			  "\x84\x5e\x46\xed\x20\x89\xb6\x44"
+			  "\x8d\xd0\xed\x54\x47\x16\xbe\x95"
+			  "\x8a\xb3\x6b\x72\xc4\x32\x52\x13"
+			  "\x1b\xb0\x82\xbe\xac\xf9\x70\xa6"
+			  "\x44\x18\xdd\x8c\x6e\xca\x6e\x45"
+			  "\x8f\x1e\x10\x07\x57\x25\x98\x7b"
+			  "\x17\x8c\x78\xdd\x80\xa7\xd9\xd8"
+			  "\x63\xaf\xb9\x67\x57\xfd\xbc\xdb"
+			  "\x44\xe9\xc5\x65\xd1\xc7\x3b\xff"
+			  "\x20\xa0\x80\x1a\xc3\x9a\xad\x5e"
+			  "\x5d\x3b\xd3\x07\xd9\xf5\xfd\x3d"
+			  "\x4a\x8b\xa8\xd2\x6e\x7a\x51\x65"
+			  "\x6c\x8e\x95\xe0\x45\xc9\x5f\x4a"
+			  "\x09\x3c\x3d\x71\x7f\x0c\x84\x2a"
+			  "\xc8\x48\x52\x1a\xc2\xd5\xd6\x78"
+			  "\x92\x1e\xa0\x90\x2e\xea\xf0\xf3"
+			  "\xdc\x0f\xb1\xaf\x0d\x9b\x06\x2e"
+			  "\x35\x10\x30\x82\x0d\xe7\xc5\x9b"
+			  "\xde\x44\x18\xbd\x9f\xd1\x45\xa9"
+			  "\x7b\x7a\x4a\xad\x35\x65\x27\xca"
+			  "\xb2\xc3\xd4\x9b\x71\x86\x70\xee"
+			  "\xf1\x89\x3b\x85\x4b\x5b\xaa\xaf"
+			  "\xfc\x42\xc8\x31\x59\xbe\x16\x60"
+			  "\x4f\xf9\xfa\x12\xea\xd0\xa7\x14"
+			  "\xf0\x7a\xf3\xd5\x8d\xbd\x81\xef"
+			  "\x52\x7f\x29\x51\x94\x20\x67\x3c"
+			  "\xd1\xaf\x77\x9f\x22\x5a\x4e\x63"
+			  "\xe7\xff\x73\x25\xd1\xdd\x96\x8a"
+			  "\x98\x52\x6d\xf3\xac\x3e\xf2\x18"
+			  "\x6d\xf6\x0a\x29\xa6\x34\x3d\xed"
+			  "\xe3\x27\x0d\x9d\x0a\x02\x44\x7e"
+			  "\x5a\x7e\x67\x0f\x0a\x9e\xd6\xad"
+			  "\x91\xe6\x4d\x81\x8c\x5c\x59\xaa"
+			  "\xfb\xeb\x56\x53\xd2\x7d\x4c\x81"
+			  "\x65\x53\x0f\x41\x11\xbd\x98\x99"
+			  "\xf9\xc6\xfa\x51\x2e\xa3\xdd\x8d"
+			  "\x84\x98\xf9\x34\xed\x33\x2a\x1f"
+			  "\x82\xed\xc1\x73\x98\xd3\x02\xdc"
+			  "\xe6\xc2\x33\x1d\xa2\xb4\xca\x76"
+			  "\x63\x51\x34\x9d\x96\x12\xae\xce"
+			  "\x83\xc9\x76\x5e\xa4\x1b\x53\x37"
+			  "\x17\xd5\xc0\x80\x1d\x62\xf8\x3d"
+			  "\x54\x27\x74\xbb\x10\x86\x57\x46"
+			  "\x68\xe1\xed\x14\xe7\x9d\xfc\x84"
+			  "\x47\xbc\xc2\xf8\x19\x4b\x99\xcf"
+			  "\x7a\xe9\xc4\xb8\x8c\x82\x72\x4d"
+			  "\x7b\x4f\x38\x55\x36\x71\x64\xc1"
+			  "\xfc\x5c\x75\x52\x33\x02\x18\xf8"
+			  "\x17\xe1\x2b\xc2\x43\x39\xbd\x76"
+			  "\x9b\x63\x76\x32\x2f\x19\x72\x10"
+			  "\x9f\x21\x0c\xf1\x66\x50\x7f\xa5"
+			  "\x0d\x1f\x46\xe0\xba\xd3\x2f\x3c",
+		.len	= 512,
 		.also_non_np = 1,
 		.np	= 3,
-		.tap	= { 1008 - 20, 4, 16 },
-	},
+		.tap	= { 512 - 20, 4, 16 },
+	}
 };
 
-static const struct cipher_testvec camellia_cbc_enc_tv_template[] = {
+/* Cast6 test vectors from RFC 2612 */
+static const struct cipher_testvec cast6_tv_template[] = {
 	{
-		.key    = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
-			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
-		.klen   = 16,
-		.iv	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.input	= "Single block msg",
-		.ilen   = 16,
-		.result = "\xea\x32\x12\x76\x3b\x50\x10\xe7"
-			  "\x18\xf6\xfd\x5d\xf6\x8f\x13\x51",
-		.rlen   = 16,
-	}, {
-		.key    = "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
-			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
-		.klen   = 16,
-		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.input  = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.ilen   = 32,
-		.result = "\xa5\xdf\x6e\x50\xda\x70\x6c\x01"
-			  "\x4a\xab\xf3\xf2\xd6\xfc\x6c\xfd"
-			  "\x19\xb4\x3e\x57\x1c\x02\x5e\xa0"
-			  "\x15\x78\xe0\x5e\xf2\xcb\x87\x16",
-		.rlen   = 32,
-	}, { /* Generated with Crypto++ */
+		.key	= "\x23\x42\xbb\x9e\xfa\x38\x54\x2c"
+			  "\x0a\xf7\x56\x47\xf2\x9f\x61\x5d",
+		.klen	= 16,
+		.ptext	= zeroed_string,
+		.ctext	= "\xc8\x42\xa0\x89\x72\xb4\x3d\x20"
+			  "\x83\x6c\x91\xd1\xb7\x53\x0f\x6b",
+		.len	= 16,
+	}, {
+		.key	= "\x23\x42\xbb\x9e\xfa\x38\x54\x2c"
+			  "\xbe\xd0\xac\x83\x94\x0a\xc2\x98"
+			  "\xba\xc7\x7a\x77\x17\x94\x28\x63",
+		.klen	= 24,
+		.ptext	= zeroed_string,
+		.ctext	= "\x1b\x38\x6c\x02\x10\xdc\xad\xcb"
+			  "\xdd\x0e\x41\xaa\x08\xa7\xa7\xe8",
+		.len	= 16,
+	}, {
+		.key	= "\x23\x42\xbb\x9e\xfa\x38\x54\x2c"
+			  "\xbe\xd0\xac\x83\x94\x0a\xc2\x98"
+			  "\x8d\x7c\x47\xce\x26\x49\x08\x46"
+			  "\x1c\xc1\xb5\x13\x7a\xe6\xb6\x04",
+		.klen	= 32,
+		.ptext	= zeroed_string,
+		.ctext	= "\x4f\x6a\x20\x38\x28\x68\x97\xb9"
+			  "\xc9\x87\x01\x36\x55\x33\x17\xfa",
+		.len	= 16,
+	}, { /* Generated from TF test vectors */
 		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
 			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
 			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
@@ -30701,7 +10916,7 @@ static const struct cipher_testvec camellia_cbc_enc_tv_template[] = {
 		.klen	= 32,
 		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
 			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -30711,696 +10926,16081 @@ static const struct cipher_testvec camellia_cbc_enc_tv_template[] = {
 			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
 			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
 			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06"
-			  "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78"
-			  "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA"
-			  "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C"
-			  "\xF3\x67\xFE\x95\x09\xA0\x37\xCE"
-			  "\x42\xD9\x70\x07\x7B\x12\xA9\x1D"
-			  "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F"
-			  "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01"
-			  "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73"
-			  "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5"
-			  "\x59\xF0\x87\x1E\x92\x29\xC0\x34"
-			  "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6"
-			  "\x3D\xD4\x48\xDF\x76\x0D\x81\x18"
-			  "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A"
-			  "\x21\x95\x2C\xC3\x37\xCE\x65\xFC"
-			  "\x70\x07\x9E\x12\xA9\x40\xD7\x4B"
-			  "\xE2\x79\x10\x84\x1B\xB2\x26\xBD"
-			  "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F"
-			  "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1"
-			  "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13"
-			  "\x87\x1E\xB5\x29\xC0\x57\xEE\x62"
-			  "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4"
-			  "\x6B\x02\x76\x0D\xA4\x18\xAF\x46"
-			  "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8"
-			  "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07"
-			  "\x9E\x35\xCC\x40\xD7\x6E\x05\x79"
-			  "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB"
-			  "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D"
-			  "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF"
-			  "\x43\xDA\x71\x08\x7C\x13\xAA\x1E"
-			  "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90"
-			  "\x27\xBE\x32\xC9\x60\xF7\x6B\x02"
-			  "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74"
-			  "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6"
-			  "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35"
-			  "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7"
-			  "\x3E\xD5\x49\xE0\x77\x0E\x82\x19"
-			  "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B"
-			  "\x22\x96\x2D\xC4\x38\xCF\x66\xFD"
-			  "\x71\x08\x9F\x13\xAA\x41\xD8\x4C"
-			  "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE"
-			  "\x55\xEC\x60\xF7\x8E\x02\x99\x30"
-			  "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2"
-			  "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14"
-			  "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63"
-			  "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5"
-			  "\x6C\x03\x77\x0E\xA5\x19\xB0\x47"
-			  "\xDE\x52\xE9\x80\x17\x8B\x22\xB9"
-			  "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08"
-			  "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A"
-			  "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC"
-			  "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E"
-			  "\xF5\x69\x00\x97\x0B\xA2\x39\xD0"
-			  "\x44\xDB\x72\x09\x7D\x14\xAB\x1F"
-			  "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91"
-			  "\x28\xBF\x33\xCA\x61\xF8\x6C\x03"
-			  "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75"
-			  "\x0C\x80\x17\xAE\x22\xB9\x50\xE7"
-			  "\x5B\xF2\x89\x20\x94\x2B\xC2\x36"
-			  "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8"
-			  "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A"
-			  "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C"
-			  "\x00\x97\x2E\xC5\x39\xD0\x67\xFE"
-			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D",
-		.ilen	= 1008,
-		.result	= "\xCD\x3E\x2A\x3B\x3E\x94\xC5\x77"
-			  "\xBA\xBB\x5B\xB1\xDE\x7B\xA4\x40"
-			  "\x88\x39\xE3\xFD\x94\x4B\x25\x58"
-			  "\xE1\x4B\xC4\x18\x7A\xFD\x17\x2B"
-			  "\xB9\xF9\xC2\x27\x6A\xB6\x31\x27"
-			  "\xA6\xAD\xEF\xE5\x5D\xE4\x02\x01"
-			  "\x56\x2E\x10\xC2\x2C\xFF\xC6\x83"
-			  "\xB5\xDC\x4F\x63\xAD\x0E\x63\x5E"
-			  "\x56\xC8\x18\x3D\x79\x86\x97\xEF"
-			  "\x57\x0E\x63\xA1\xC1\x41\x48\xB8"
-			  "\x98\xB7\x51\x6D\x18\xF6\x19\x82"
-			  "\x37\x49\x88\xA4\xEF\x91\x21\x47"
-			  "\x03\x28\xEA\x42\xF4\xFB\x7A\x58"
-			  "\x28\x90\x77\x46\xD8\xD2\x35\x16"
-			  "\x44\xA9\x9E\x49\x52\x2A\xE4\x16"
-			  "\x5D\xF7\x65\xEB\x0F\xC9\x29\xE6"
-			  "\xCF\x76\x91\x89\x8A\x94\x39\xFA"
-			  "\x6B\x5F\x63\x53\x74\x43\x91\xF5"
-			  "\x3F\xBC\x88\x53\xB2\x1A\x02\x3F"
-			  "\x9D\x32\x84\xEB\x56\x28\xD6\x06"
-			  "\xD5\xB2\x20\xA9\xFC\xC3\x76\x62"
-			  "\x32\xCC\x86\xC8\x36\x67\x5E\x7E"
-			  "\xA4\xAA\x15\x63\x6B\xA9\x86\xAF"
-			  "\x1A\x52\x82\x36\x5F\xF4\x3F\x7A"
-			  "\x9B\x78\x62\x3B\x02\x28\x60\xB3"
-			  "\xBA\x82\xB1\xDD\xC9\x60\x8F\x47"
-			  "\xF1\x6B\xFE\xE5\x39\x34\xA0\x28"
-			  "\xA4\xB3\xC9\x7E\xED\x28\x8D\x70"
-			  "\xB2\x1D\xFD\xC6\x00\xCF\x1A\x94"
-			  "\x28\xF8\xC1\x34\xB7\x58\xA5\x6C"
-			  "\x1A\x9D\xE4\xE4\xF6\xB9\xB4\xB0"
-			  "\x5D\x51\x54\x9A\x53\xA0\xF9\x32"
-			  "\xBD\x31\x54\x14\x7B\x33\xEE\x17"
-			  "\xD3\xC7\x1F\x48\xBF\x0B\x22\xA2"
-			  "\x7D\x0C\xDF\xD0\x2E\x98\xFA\xD2"
-			  "\xFA\xCF\x24\x1D\x99\x9B\xD0\x7E"
-			  "\xF4\x4F\x88\xFF\x45\x99\x4A\xF4"
-			  "\xF2\x0A\x5B\x3B\x21\xAB\x92\xAE"
-			  "\x40\x78\x91\x95\xC4\x2F\xA3\xE8"
-			  "\x18\xC7\x07\xA6\xC8\xC0\x66\x33"
-			  "\x35\xC0\xB4\xA0\xF8\xEE\x1E\xF3"
-			  "\x40\xF5\x40\x54\xF1\x84\x8C\xEA"
-			  "\x27\x38\x1F\xF8\x77\xC7\xDF\xD8"
-			  "\x1D\xE2\xD9\x59\x40\x4F\x59\xD4"
-			  "\xF8\x17\x99\x8D\x58\x2D\x72\x44"
-			  "\x9D\x1D\x91\x64\xD6\x3F\x0A\x82"
-			  "\xC7\x57\x3D\xEF\xD3\x41\xFA\xA7"
-			  "\x68\xA3\xB8\xA5\x93\x74\x2E\x85"
-			  "\x4C\x9D\x69\x59\xCE\x15\xAE\xBF"
-			  "\x9C\x8F\x14\x64\x5D\x7F\xCF\x0B"
-			  "\xCE\x43\x5D\x28\xC0\x2F\xFB\x18"
-			  "\x79\x9A\xFC\x43\x16\x7C\x6B\x7B"
-			  "\x38\xB8\x48\x36\x66\x4E\x20\x43"
-			  "\xBA\x76\x13\x9A\xC3\xF2\xEB\x52"
-			  "\xD7\xDC\xB2\x67\x63\x14\x25\xCD"
-			  "\xB1\x13\x4B\xDE\x8C\x59\x21\x84"
-			  "\x81\x8D\x97\x23\x45\x33\x7C\xF3"
-			  "\xC5\xBC\x79\x95\xAA\x84\x68\x31"
-			  "\x2D\x1A\x68\xFE\xEC\x92\x94\xDA"
-			  "\x94\x2A\x6F\xD6\xFE\xE5\x76\x97"
-			  "\xF4\x6E\xEE\xCB\x2B\x95\x4E\x36"
-			  "\x5F\x74\x8C\x86\x5B\x71\xD0\x20"
-			  "\x78\x1A\x7F\x18\x8C\xD9\xCD\xF5"
-			  "\x21\x41\x56\x72\x13\xE1\x86\x07"
-			  "\x07\x26\xF3\x4F\x7B\xEA\xB5\x18"
-			  "\xFE\x94\x2D\x9F\xE0\x72\x18\x65"
-			  "\xB2\xA5\x63\x48\xB4\x13\x22\xF7"
-			  "\x25\xF1\x80\xA8\x7F\x54\x86\x7B"
-			  "\x39\xAE\x95\x0C\x09\x32\x22\x2D"
-			  "\x4D\x73\x39\x0C\x09\x2C\x7C\x10"
-			  "\xD0\x4B\x53\xF6\x90\xC5\x99\x2F"
-			  "\x15\xE1\x7F\xC6\xC5\x7A\x52\x14"
-			  "\x65\xEE\x93\x54\xD0\x66\x15\x3C"
-			  "\x4C\x68\xFD\x64\x0F\xF9\x10\x39"
-			  "\x46\x7A\xDD\x97\x20\xEE\xC7\xD2"
-			  "\x98\x4A\xB6\xE6\xF5\xA8\x1F\x4F"
-			  "\xDB\xAB\x6D\xD5\x9B\x34\x16\x97"
-			  "\x2F\x64\xE5\x37\xEF\x0E\xA1\xE9"
-			  "\xBE\x31\x31\x96\x8B\x40\x18\x75"
-			  "\x11\x75\x14\x32\xA5\x2D\x1B\x6B"
-			  "\xDB\x59\xEB\xFA\x3D\x8E\x7C\xC4"
-			  "\xDE\x68\xC8\x9F\xC9\x99\xE3\xC6"
-			  "\x71\xB0\x12\x57\x89\x0D\xC0\x2B"
-			  "\x9F\x12\x6A\x04\x67\xF1\x95\x31"
-			  "\x59\xFD\x84\x95\x2C\x9C\x5B\xEC"
-			  "\x09\xB0\x43\x96\x4A\x64\x80\x40"
-			  "\xB9\x72\x19\xDD\x70\x42\xFA\xB1"
-			  "\x4A\x2C\x0C\x0A\x60\x6E\xE3\x7C"
-			  "\x37\x5A\xBE\xA4\x62\xCF\x29\xAB"
-			  "\x7F\x4D\xA6\xB3\xE2\xB6\x64\xC6"
-			  "\x33\x0B\xF3\xD5\x01\x38\x74\xA4"
-			  "\x67\x1E\x75\x68\xC3\xAD\x76\xE9"
-			  "\xE9\xBC\xF0\xEB\xD8\xFD\x31\x8A"
-			  "\x5F\xC9\x18\x94\x4B\x86\x66\xFC"
-			  "\xBD\x0B\x3D\xB3\x9F\xFA\x1F\xD9"
-			  "\x78\xC4\xE3\x24\x1C\x67\xA2\xF8"
-			  "\x43\xBC\x76\x75\xBF\x6C\x05\xB3"
-			  "\x32\xE8\x7C\x80\xDB\xC7\xB6\x61"
-			  "\x1A\x3E\x2B\xA7\x25\xED\x8F\xA0"
-			  "\x00\x4B\xF8\x90\xCA\xD8\xFB\x12"
-			  "\xAC\x1F\x18\xE9\xD2\x5E\xA2\x8E"
-			  "\xE4\x84\x6B\x9D\xEB\x1E\x6B\xA3"
-			  "\x7B\xDC\xCE\x15\x97\x27\xB2\x65"
-			  "\xBC\x0E\x47\xAB\x55\x13\x53\xAB"
-			  "\x0E\x34\x55\x02\x5F\x27\xC5\x89"
-			  "\xDF\xC5\x70\xC4\xDD\x76\x82\xEE"
-			  "\x68\xA6\x09\xB0\xE5\x5E\xF1\x0C"
-			  "\xE3\xF3\x09\x9B\xFE\x65\x4B\xB8"
-			  "\x30\xEC\xD5\x7C\x6A\xEC\x1D\xD2"
-			  "\x93\xB7\xA1\x1A\x02\xD4\xC0\xD6"
-			  "\x8D\x4D\x83\x9A\xED\x29\x4E\x14"
-			  "\x86\xD5\x3C\x1A\xD5\xB9\x0A\x6A"
-			  "\x72\x22\xD5\x92\x38\xF1\xA1\x86"
-			  "\xB2\x41\x51\xCA\x4E\xAB\x8F\xD3"
-			  "\x80\x56\xC3\xD7\x65\xE1\xB3\x86"
-			  "\xCB\xCE\x98\xA1\xD4\x59\x1C\x06"
-			  "\x01\xED\xF8\x29\x91\x19\x5C\x9A"
-			  "\xEE\x28\x1B\x48\xD7\x32\xEF\x9F"
-			  "\x6C\x2B\x66\x4E\x78\xD5\x8B\x72"
-			  "\x80\xE7\x29\xDC\x23\x55\x98\x54"
-			  "\xB1\xFF\x3E\x95\x56\xA8\x78\x78"
-			  "\xEF\xC4\xA5\x11\x2D\x2B\xD8\x93"
-			  "\x30\x6E\x7E\x51\xBB\x42\x5F\x03"
-			  "\x43\x94\x23\x7E\xEE\xF0\xA5\x79"
-			  "\x55\x01\xD4\x58\xB2\xF2\x85\x49"
-			  "\x70\xC5\xB9\x0B\x3B\x7A\x6E\x6C",
-		.rlen	= 1008,
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\xC3\x70\x22\x32\xF5\x80\xCB\x54"
+			  "\xFC\x30\xE0\xF6\xEB\x39\x57\xA6"
+			  "\xB6\xB9\xC5\xA4\x91\x55\x14\x97"
+			  "\xC1\x20\xFF\x6C\x5C\xF0\x67\xEA"
+			  "\x2F\xED\xD8\xC9\xFB\x38\x3F\xFE"
+			  "\x93\xBE\xDC\x00\xD3\x7F\xAD\x4C"
+			  "\x5A\x08\x92\xD1\x47\x0C\xFA\x6C"
+			  "\xD0\x6A\x99\x10\x72\xF8\x47\x62"
+			  "\x81\x42\xF8\xD8\xF5\xBB\x94\x08"
+			  "\xAA\x97\xA2\x8B\x69\xB3\xD2\x7E"
+			  "\xBC\xB5\x00\x0C\xE5\x44\x4B\x58"
+			  "\xE8\x63\xDC\xB3\xC4\xE5\x23\x12"
+			  "\x5A\x72\x85\x47\x8B\xEC\x9F\x26"
+			  "\x84\xB6\xED\x10\x33\x63\x9B\x5F"
+			  "\x4D\x53\xEE\x94\x45\x8B\x60\x58"
+			  "\x86\x20\xF9\x1E\x82\x08\x3E\x58"
+			  "\x60\x1B\x34\x19\x02\xBE\x4E\x09"
+			  "\xBB\x7C\x15\xCC\x60\x27\x55\x7A"
+			  "\x12\xB8\xD8\x08\x89\x3C\xA6\xF3"
+			  "\xF1\xDD\xA7\x07\xA3\x12\x85\x28"
+			  "\xE9\x57\xAC\x80\x0C\x5C\x0F\x3A"
+			  "\x5D\xC2\x91\xC7\x90\xE4\x8C\x43"
+			  "\x92\xE4\x7C\x26\x69\x4D\x83\x68"
+			  "\x14\x96\x42\x47\xBD\xA9\xE4\x8A"
+			  "\x33\x19\xEB\x54\x8E\x0D\x4B\x6E"
+			  "\x91\x51\xB5\x36\x08\xDE\x1C\x06"
+			  "\x03\xBD\xDE\x81\x26\xF7\x99\xC2"
+			  "\xBA\xF7\x6D\x87\x0D\xE4\xA6\xCF"
+			  "\xC1\xF5\x27\x05\xB8\x02\x57\x72"
+			  "\xE6\x42\x13\x0B\xC6\x47\x05\x74"
+			  "\x24\x15\xF7\x0D\xC2\x23\x9D\xB9"
+			  "\x3C\x77\x18\x93\xBA\xB4\xFC\x8C"
+			  "\x98\x82\x67\x67\xB4\xD7\xD3\x43"
+			  "\x23\x08\x02\xB7\x9B\x99\x05\xFB"
+			  "\xD3\xB5\x00\x0A\xA9\x9D\x66\xD6"
+			  "\x2E\x49\x58\xD0\xA8\x57\x29\x7F"
+			  "\x0A\x0E\x7D\xFC\x92\x83\xCC\x67"
+			  "\xA2\xB1\x70\x3A\x8F\x87\x4A\x8D"
+			  "\x17\xE2\x58\x2B\x88\x0D\x68\x62"
+			  "\xBF\x35\xD1\x6F\xC0\xF0\x18\x62"
+			  "\xB2\xC7\x2D\x58\xC7\x16\xDE\x08"
+			  "\xEB\x84\x1D\x25\xA7\x38\x94\x06"
+			  "\x93\x9D\xF8\xFE\x88\x71\xE7\x84"
+			  "\x2C\xA0\x38\xA3\x1D\x48\xCF\x29"
+			  "\x0B\xBC\xD8\x50\x99\x1A\x26\xFB"
+			  "\x8E\x75\x3D\x73\xEB\x6A\xED\x29"
+			  "\xE0\x8E\xED\xFC\xFE\x6F\xF6\xBA"
+			  "\x41\xE2\x10\x4C\x01\x8B\x69\x2B"
+			  "\x25\x3F\x4D\x70\x7B\x92\xD6\x3B"
+			  "\xAC\xF9\x77\x18\xD9\x6A\x30\xA6"
+			  "\x2E\xFA\x30\xFF\xC8\xD5\x1D\x06"
+			  "\x59\x28\x1D\x86\x43\x04\x5D\x3B"
+			  "\x99\x4C\x04\x5A\x21\x17\x8B\x76"
+			  "\x8F\x72\xCB\xA1\x9C\x29\x4C\xC3"
+			  "\x65\xA2\x58\x2A\xC5\x66\x24\xBF"
+			  "\xBA\xE6\x0C\xDD\x34\x24\x74\xC8"
+			  "\x84\x0A\x66\x2C\xBE\x8F\x32\xA9"
+			  "\xE7\xE4\xA1\xD7\xDA\xAB\x23\x1E"
+			  "\xEB\xEE\x6C\x94\x6F\x9C\x2E\xD1"
+			  "\x49\x2C\xF3\xD4\x90\xCC\x93\x4C"
+			  "\x84\x52\x6D\x68\xDE\xC6\x64\xB2"
+			  "\x11\x74\x93\x57\xB4\x7E\xC6\x00",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec cast6_cbc_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\xDF\x77\x68\x96\xC7\xBA\xF8\xE2"
+			  "\x0E\x24\x99\x1A\xAA\xF3\xC6\x9F"
+			  "\xA0\x73\xB3\x70\xC3\x68\x64\x70"
+			  "\xAD\x33\x02\xFB\x88\x74\xAA\x78"
+			  "\xC7\x47\x1A\x18\x61\x2D\xAC\x9F"
+			  "\x7E\x6F\xDF\x05\x13\x76\xA6\x72"
+			  "\xB7\x13\x09\x0F\x7D\x38\xDF\x25"
+			  "\x4E\xFD\x50\x45\xFA\x35\x6A\xC0"
+			  "\x57\x95\xE1\x21\x26\x10\x9A\x21"
+			  "\xA1\x8A\x51\x05\xD1\xB1\x78\x35"
+			  "\x98\xF5\xAE\xC0\xC1\x8B\x94\xFF"
+			  "\xD0\x69\x3F\x42\xC2\x01\xA7\x9B"
+			  "\x23\x16\x47\x72\x81\x13\x3A\x72"
+			  "\xEC\xD9\x40\x88\x00\x9C\xB0\xA8"
+			  "\x9C\xAC\xCE\x11\x73\x7B\x63\x3E"
+			  "\xA3\x63\x98\x7D\x35\xE4\xD9\x83"
+			  "\xE2\xD0\x52\x87\x0C\x1F\xB0\xB3"
+			  "\x41\x1A\x93\x8D\x76\x31\x9F\xF2"
+			  "\xFE\x09\xA3\x8F\x22\x6A\x3B\xB9"
+			  "\x6C\x9E\xE4\xA1\xA0\xC4\xE7\xA1"
+			  "\x21\x9C\x1A\xCA\x65\xDE\x44\x03"
+			  "\x99\xF2\xD2\x39\xE3\x3F\x0F\x37"
+			  "\x53\x50\x23\xA4\x81\x6E\xDA\xFB"
+			  "\xF8\x7B\x01\xD7\xB2\x32\x9C\xB8"
+			  "\xB1\x0E\x99\x17\xB5\x38\xF9\xD7"
+			  "\x86\x2D\x6E\x94\x5C\x99\x9D\xB3"
+			  "\xD3\x63\x4B\x2A\x7D\x44\x6A\xB2"
+			  "\xC1\x03\xE6\x5A\x37\xD8\x64\x18"
+			  "\xAA\x32\xCE\x29\xED\xC0\xA2\xCB"
+			  "\x8D\xAF\xCD\xBE\x8F\xB6\xEC\xB4"
+			  "\x89\x05\x81\x6E\x71\x4F\xC3\x28"
+			  "\x10\xC1\x62\xC4\x41\xE9\xD2\x39"
+			  "\xF3\x22\x39\x12\x2C\xC2\x95\x2D"
+			  "\xBF\x93\x58\x4B\x04\xD1\x8D\x57"
+			  "\xAE\xEB\x60\x03\x56\x35\xAD\x5A"
+			  "\xE9\xC3\xFF\x4E\x31\xE1\x37\xF8"
+			  "\x7D\xEE\x65\x8A\xB6\x88\x1A\x3E"
+			  "\x07\x09\x82\xBA\xF0\x80\x8A\xD0"
+			  "\xA0\x3F\x6A\xE9\x24\x87\x19\x65"
+			  "\x73\x3F\x12\x91\x47\x54\xBA\x39"
+			  "\x30\x5B\x1E\xE5\xC2\xF9\x3F\xEF"
+			  "\xD6\x75\xF9\xB8\x7C\x8B\x05\x76"
+			  "\xEE\xB7\x08\x25\x4B\xB6\x7B\x47"
+			  "\x72\xC0\x4C\xD4\xDA\xE0\x75\xF1"
+			  "\x7C\xE8\x94\x9E\x16\x6E\xB8\x12"
+			  "\xA1\xC1\x6E\x3B\x1C\x59\x41\x2D"
+			  "\x23\xFA\x7D\x77\xB8\x46\x75\xFE"
+			  "\x4F\x10\xD3\x09\x60\xA1\x36\x96"
+			  "\x5B\xC2\xDC\x6E\x84\x7D\x9B\x14"
+			  "\x80\x21\x83\x58\x3C\x76\xFD\x28"
+			  "\x1D\xF9\x93\x13\xD7\x0E\x62\x14"
+			  "\x5A\xC5\x4E\x08\xA5\x56\xA4\x3C"
+			  "\x68\x93\x44\x70\xDF\xCF\x4A\x51"
+			  "\x0B\x81\x29\x41\xE5\x62\x4D\x36"
+			  "\xB3\xEA\x94\xA6\xB9\xDD\x3F\x09"
+			  "\x62\x34\xA0\x6A\x7E\x7D\xF5\xF6"
+			  "\x01\x91\xB4\x27\xDA\x59\xD6\x17"
+			  "\x56\x4D\x82\x62\x37\xA3\x48\x01"
+			  "\x99\x91\x77\xB2\x08\x6B\x2C\x37"
+			  "\xC5\x5C\xAD\xB6\x07\xB6\x84\xF3"
+			  "\x4D\x59\x7D\xC5\x28\x69\xFA\x92"
+			  "\x22\x46\x89\x2D\x0F\x2B\x08\x24",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec cast6_ctr_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A",
+		.ctext	= "\x26\x0A\xF1\xE2\x3F\x8A\xEF\xA3"
+			  "\x53\x9A\x5E\x1B\x2A\x1A\xC6\x0A"
+			  "\x57",
+		.len	= 17,
+	}, { /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\x26\x0A\xF1\xE2\x3F\x8A\xEF\xA3"
+			  "\x53\x9A\x5E\x1B\x2A\x1A\xC6\x0A"
+			  "\x57\xA3\xEF\x47\x2A\xE8\x88\xA7"
+			  "\x3C\xD0\xEC\xB9\x94\x50\x7D\x56"
+			  "\xBC\xE1\xC1\xF5\xE1\xEE\x12\xF8"
+			  "\x4F\x03\x82\x3A\x93\x6B\x4C\xD3"
+			  "\xE3\xF3\xFA\xC2\x23\x55\x98\x20"
+			  "\x49\x76\x9B\x6B\xC1\x23\xBF\xE5"
+			  "\xD4\xC4\x2F\x61\xE1\x67\x2A\x30"
+			  "\x6F\x29\xCA\x54\xF8\x1B\xA6\x7D"
+			  "\x66\x45\xEE\xC8\x19\xBE\x50\xF0"
+			  "\x5F\x65\xF8\x1E\x4D\x07\x87\xD9"
+			  "\xD3\xD9\x1B\x09\x89\xFD\x42\xC5"
+			  "\xDB\xEB\x86\xF1\x67\x04\x0F\x5C"
+			  "\x81\xDF\x82\x12\xC7\x4C\x1B\x07"
+			  "\xDE\xE6\xFA\x29\x86\xD1\xB0\xBA"
+			  "\x3D\x6A\x69\x76\xEC\x0F\xB4\xE6"
+			  "\xCD\xA7\xF8\xA8\xB8\xE0\x33\xF5"
+			  "\x49\x61\x22\x52\x64\x8C\x46\x41"
+			  "\x1F\x48\x5F\x4F\xA2\x89\x36\x17"
+			  "\x20\xF8\x2F\x8F\x4B\xFA\xF2\xC0"
+			  "\x1E\x18\xA2\xF8\xB7\x6D\x98\xE3"
+			  "\x00\x14\x15\x59\xC1\x30\x64\xAF"
+			  "\xA8\x01\x38\xAB\xD4\x8B\xEC\x7C"
+			  "\x44\x9A\xC6\x2C\x2E\x2B\x2B\xF4"
+			  "\x02\x37\xC4\x69\xEF\x36\xC1\xF3"
+			  "\xA0\xFB\xFE\x29\xAD\x39\xCF\xD0"
+			  "\x51\x73\xA3\x22\x42\x41\xAB\xD2"
+			  "\x0F\x50\x14\xB9\x54\xD3\xD4\xFA"
+			  "\xBF\xC9\xBB\xCE\xC4\x1D\x2D\xAF"
+			  "\xC9\x3F\x07\x87\x42\x4B\x3A\x54"
+			  "\x34\x8E\x37\xA3\x03\x6F\x65\x66"
+			  "\xDB\x44\xC3\xE8\xD7\xDD\x7D\xDD"
+			  "\x61\xB4\x2B\x80\xA3\x98\x13\xF5"
+			  "\x5A\xD3\x34\x58\xC3\x6E\xF6\xB8"
+			  "\x0A\xC6\x50\x01\x8E\xD5\x6C\x7D"
+			  "\xFE\x16\xB6\xCF\xFC\x51\x40\xAE"
+			  "\xB3\x15\xAC\x90\x6F\x0B\x28\x3A"
+			  "\x60\x40\x38\x90\x20\x46\xC7\xB3"
+			  "\x0B\x12\x6D\x3B\x15\x14\xF9\xF4"
+			  "\x11\x41\x76\x6B\xB3\x60\x82\x3C"
+			  "\x84\xFB\x08\x2E\x92\x25\xCB\x79"
+			  "\x6F\x58\xC5\x94\x00\x00\x47\xB6"
+			  "\x9E\xDC\x0F\x29\x70\x46\x20\x76"
+			  "\x65\x75\x66\x5C\x00\x96\xB3\xE1"
+			  "\x0B\xA7\x11\x8B\x2E\x61\x4E\x45"
+			  "\x73\xFC\x91\xAB\x79\x41\x23\x14"
+			  "\x13\xB6\x72\x6C\x46\xB3\x03\x11"
+			  "\xE4\xF1\xEE\xC9\x7A\xCF\x96\x32"
+			  "\xB6\xF0\x8B\x97\xB4\xCF\x82\xB7"
+			  "\x15\x48\x44\x99\x09\xF6\xE0\xD7"
+			  "\xBC\xF1\x5B\x91\x4F\x30\x22\xA2"
+			  "\x45\xC4\x68\x55\xC2\xBE\xA7\xD2"
+			  "\x12\x53\x35\x9C\xF9\xE7\x35\x5D"
+			  "\x81\xE4\x86\x42\xC3\x58\xFB\xF0"
+			  "\x38\x9B\x8E\x5A\xEF\x83\x33\x0F"
+			  "\x00\x4E\x3F\x9F\xF5\x84\x62\xC4"
+			  "\x19\x35\x88\x22\x45\x59\x0E\x8F"
+			  "\xEC\x27\xDD\x4A\xA4\x1F\xBC\x41"
+			  "\x9B\x66\x8D\x32\xBA\x81\x34\x87"
+			  "\x0E\x74\x33\x30\x62\xB9\x89\xDF"
+			  "\xF9\xC5\xDD\x27\xB3\x39\xCB\xCB",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec cast6_lrw_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
+			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
+			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
+			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
+			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
+			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
+			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
+			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
+			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
+			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
+			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
+			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
+			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
+			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
+			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
+			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
+			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
+			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
+			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
+			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
+			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
+			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
+			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
+			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
+			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
+			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
+			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
+			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
+			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
+			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
+			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
+			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
+			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
+			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
+			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
+			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
+			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
+			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
+			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
+			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
+			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
+			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
+			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
+			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
+			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
+			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
+			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
+			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
+			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
+			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
+			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
+			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
+			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
+			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
+			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
+			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
+			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
+			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
+			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
+			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
+			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
+			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
+			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
+			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
+			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
+			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
+			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
+			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
+			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
+		.ctext	= "\x55\x25\x09\x8B\xB5\xD5\xF8\xBF"
+			  "\x37\x4A\xFE\x3C\x47\xD8\xE6\xEB"
+			  "\xCA\xA4\x9B\xB0\xAB\x6D\x64\xCA"
+			  "\x58\xB6\x73\xF0\xD7\x52\x34\xEF"
+			  "\xFB\x3E\x96\x81\xB7\x71\x34\xA4"
+			  "\x55\x20\xBE\x39\x5A\x2B\xF9\xD1"
+			  "\x65\x0B\xDA\xD3\x7E\xB3\xA6\xF7"
+			  "\x2E\x0B\x5A\x52\xDB\x39\x8C\x9B"
+			  "\x61\x17\x5F\xAF\xB6\x5A\xC8\x08"
+			  "\xA7\xB7\x2A\x11\x7C\x97\x38\x9D"
+			  "\x59\x0E\x66\x59\x5E\xD8\x8B\xCE"
+			  "\x70\xE0\xC3\x42\xB0\x8C\x0F\xBA"
+			  "\xB2\x0D\x81\xB6\xBE\x61\x1C\x2D"
+			  "\x7E\xEA\x91\x25\xAC\xEC\xF8\x28"
+			  "\x80\x1D\xF0\x30\xBA\x62\x77\x7D"
+			  "\xDB\x15\x69\xDF\xFA\x2A\x81\x64"
+			  "\x95\x5B\xA4\x7F\x3E\x4F\xE3\x30"
+			  "\xB0\x5C\xC2\x05\xF8\xF0\x29\xE7"
+			  "\x0A\xA0\x66\xB2\x5D\x0F\x39\x2B"
+			  "\xB4\xB3\x00\xA9\xD0\xAB\x63\x61"
+			  "\x5E\xDB\xFC\x11\x74\x25\x96\x65"
+			  "\xE8\xE2\x34\x57\x77\x15\x5E\x70"
+			  "\xFF\x10\x90\xC3\x64\xF0\x11\x0A"
+			  "\x63\x3A\xD3\x55\x92\x15\x4B\x0C"
+			  "\xC7\x08\x89\x17\x3B\x99\xAD\x63"
+			  "\xE7\x06\xDF\x52\xBC\x15\x64\x45"
+			  "\x9D\x7A\xFB\x69\xBC\x2D\x6E\xA9"
+			  "\x35\xD9\xD8\xF5\x0C\xC4\xA2\x23"
+			  "\x9C\x18\x8B\xA8\x8C\xFE\xF8\x0E"
+			  "\xBD\xAB\x60\x1A\x51\x17\x54\x27"
+			  "\xB6\xE8\xBE\x0F\xA9\xA5\x82\x19"
+			  "\x2F\x6F\x20\xA7\x47\xED\x74\x6C"
+			  "\x4E\xC1\xF8\x8C\x14\xF3\xBB\x1F"
+			  "\xED\x4D\x8F\x7C\x37\xEF\x19\xA1"
+			  "\x07\x16\xDE\x76\xCC\x5E\x94\x02"
+			  "\xFB\xBF\xE4\x81\x50\xCE\xFC\x0F"
+			  "\x9E\xCF\x3D\xF6\x67\x00\xBF\xA7"
+			  "\x6E\x21\x58\x36\x06\xDE\xB3\xD4"
+			  "\xA2\xFA\xD8\x4E\xE0\xB9\x7F\x23"
+			  "\x51\x21\x2B\x32\x68\xAA\xF8\xA8"
+			  "\x93\x08\xB5\x6D\xE6\x43\x2C\xB7"
+			  "\x31\xB2\x0F\xD0\xA2\x51\xC0\x25"
+			  "\x30\xC7\x10\x3F\x97\x27\x01\x8E"
+			  "\xFA\xD8\x4F\x78\xD8\x2E\x1D\xEB"
+			  "\xA1\x37\x52\x0F\x7B\x5E\x87\xA8"
+			  "\x22\xE2\xE6\x92\xA7\x5F\x11\x32"
+			  "\xCC\x93\x34\xFC\xD1\x7E\xAE\x54"
+			  "\xBC\x6A\x1B\x91\xD1\x2E\x21\xEC"
+			  "\x5D\xF1\xC4\xF1\x55\x20\xBF\xE5"
+			  "\x96\x3D\x69\x91\x20\x4E\xF2\x61"
+			  "\xDA\x77\xFE\xEE\xC3\x74\x57\x2A"
+			  "\x78\x39\xB0\xE0\xCF\x12\x56\xD6"
+			  "\x05\xDC\xF9\x19\x66\x44\x1D\xF9"
+			  "\x82\x37\xD4\xC2\x60\xB6\x31\xDF"
+			  "\x0C\xAF\xBC\x8B\x55\x9A\xC8\x2D"
+			  "\xAB\xA7\x88\x7B\x41\xE8\x29\xC9"
+			  "\x9B\x8D\xA7\x00\x86\x25\xB6\x14"
+			  "\xF5\x13\x73\xD7\x4B\x6B\x83\xF3"
+			  "\xAF\x96\x00\xE4\xB7\x3C\x65\xA6"
+			  "\x15\xB7\x94\x7D\x4E\x70\x4C\x75"
+			  "\xF3\xB4\x02\xA9\x17\x1C\x7A\x0A"
+			  "\xC0\xD5\x33\x11\x56\xDE\xDC\xF5"
+			  "\x8D\xD9\xCD\x3B\x22\x67\x18\xC7"
+			  "\xC4\xF5\x99\x61\xBC\xBB\x5B\x46",
+		.len	= 512,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 512 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec cast6_xts_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x62\x49\x77\x57\x24\x70\x93\x69"
+			  "\x99\x59\x57\x49\x66\x96\x76\x27"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95"
+			  "\x02\x88\x41\x97\x16\x93\x99\x37"
+			  "\x51\x05\x82\x09\x74\x94\x45\x92",
+		.klen	= 64,
+		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ctext	= "\xDE\x6F\x22\xA5\xE8\x39\xE8\x78"
+			  "\x88\x5A\x4F\x8D\x82\x76\x52\x6D"
+			  "\xB2\x41\x16\xF4\x2B\xA6\xEB\xF6"
+			  "\xE2\xC5\x62\x8D\x61\xA1\x01\xED"
+			  "\xD9\x38\x01\xC1\x43\x63\x4E\x88"
+			  "\xC9\x4B\x5A\x88\x80\xB7\x5C\x71"
+			  "\x47\xEE\x11\xD8\xB7\x2D\x5D\x13"
+			  "\x1A\xB1\x68\x5B\x61\xA7\xA9\x81"
+			  "\x8B\x83\xA1\x6A\xAA\x36\xD6\xB6"
+			  "\x60\x54\x09\x32\xFE\x6A\x76\x2E"
+			  "\x28\xFF\xD5\xD6\xDD\x1D\x45\x7D"
+			  "\xF0\x8B\xF3\x32\x4E\x6C\x12\xCB"
+			  "\xB8\x25\x70\xF8\x40\xBC\x90\x1B"
+			  "\x11\xC3\x59\xAF\xF0\x2F\x92\xDD"
+			  "\xD3\x3B\xCF\x60\xA1\x78\x94\x57"
+			  "\xAF\x76\xC1\x67\xA6\x3C\xCD\x98"
+			  "\xB1\xF7\x27\xB9\xA3\xBD\x10\xEA"
+			  "\xCD\x8B\xC2\xF2\x14\xF2\xB2\x67"
+			  "\x05\xDD\x1D\x58\x6E\x2F\x95\x08"
+			  "\x3A\xF8\x78\x76\x82\x56\xA7\xEC"
+			  "\x51\x4B\x85\x77\xC2\x4C\x4A\x34"
+			  "\x71\x38\x17\x91\x44\xE8\xFC\x65"
+			  "\x99\x0D\x52\x91\xEE\xF8\xEF\x27"
+			  "\x2A\x9E\x6E\x78\xC4\x26\x87\xF4"
+			  "\x8A\xF0\x2D\x04\xE8\x14\x92\x5D"
+			  "\x59\x22\x9B\x29\x5C\x18\xF0\xC3"
+			  "\x47\xF3\x76\xD8\xE4\xF3\x1B\xD1"
+			  "\x70\xA3\x0D\xB5\x70\x02\x1D\xA3"
+			  "\x91\x3B\x49\x73\x18\xAB\xD4\xC9"
+			  "\xC3\x1E\xEF\x1F\xFE\xD5\x59\x8A"
+			  "\xD7\xF6\xC9\x71\x67\x79\xD7\x0E"
+			  "\xBE\x1F\x8E\xEC\x55\x7E\x4F\x24"
+			  "\xE6\x87\xEA\xFE\x96\x25\x67\x8E"
+			  "\x93\x03\xFA\xFF\xCE\xAF\xB2\x3C"
+			  "\x6F\xEB\x57\xFB\xD3\x28\x87\xA9"
+			  "\xCE\xC2\xF5\x9C\xC6\x67\xB5\x97"
+			  "\x49\xF7\x04\xCB\xEF\x84\x98\x33"
+			  "\xAF\x38\xD3\x04\x1C\x24\x71\x38"
+			  "\xC7\x71\xDD\x43\x0D\x12\x4A\x18"
+			  "\xBA\xC4\xAF\xBA\xB2\x5B\xEB\x95"
+			  "\x02\x43\x5D\xCE\x19\xCC\xCD\x66"
+			  "\x91\x0B\x8C\x7F\x51\xC4\xBF\x3C"
+			  "\x8B\xF1\xCC\xAA\x29\xD7\x87\xCB"
+			  "\x3E\xC5\xF3\xC9\x75\xE8\xA3\x5B"
+			  "\x30\x45\xA9\xB7\xAF\x80\x64\x6F"
+			  "\x75\x4A\xA7\xC0\x6D\x19\x6B\xDE"
+			  "\x17\xDE\x6D\xEA\x87\x9F\x95\xAE"
+			  "\xF5\x3C\xEE\x54\xB8\x27\x84\xF8"
+			  "\x97\xA3\xE1\x6F\x38\x24\x34\x88"
+			  "\xCE\xBD\x32\x52\xE0\x00\x6C\x94"
+			  "\xC9\xD7\x5D\x37\x81\x33\x2E\x7F"
+			  "\x4F\x7E\x2E\x0D\x94\xBD\xEA\x59"
+			  "\x34\x39\xA8\x35\x12\xB7\xBC\xAC"
+			  "\xEA\x52\x9C\x78\x02\x6D\x92\x36"
+			  "\xFB\x59\x2B\xA4\xEA\x7B\x1B\x83"
+			  "\xE1\x4D\x5E\x2A\x7E\x92\xB1\x64"
+			  "\xDE\xE0\x27\x4B\x0A\x6F\x4C\xE3"
+			  "\xB0\xEB\x31\xE4\x69\x95\xAB\x35"
+			  "\x8B\x2C\xF5\x6B\x7F\xF1\xA2\x82"
+			  "\xF8\xD9\x47\x82\xA9\x82\x03\x91"
+			  "\x69\x1F\xBE\x4C\xE7\xC7\x34\x2F"
+			  "\x45\x72\x80\x17\x81\xBD\x9D\x62"
+			  "\xA1\xAC\xE8\xCF\xC6\x74\xCF\xDC"
+			  "\x22\x60\x4E\xE8\xA4\x5D\x85\xB9",
+		.len	= 512,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 512 - 20, 4, 16 },
+	},
+};
+
+/*
+ * AES test vectors.
+ */
+static const struct cipher_testvec aes_tv_template[] = {
+	{ /* From FIPS-197 */
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.klen	= 16,
+		.ptext	= "\x00\x11\x22\x33\x44\x55\x66\x77"
+			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
+		.ctext	= "\x69\xc4\xe0\xd8\x6a\x7b\x04\x30"
+			  "\xd8\xcd\xb7\x80\x70\xb4\xc5\x5a",
+		.len	= 16,
+	}, {
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17",
+		.klen	= 24,
+		.ptext	= "\x00\x11\x22\x33\x44\x55\x66\x77"
+			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
+		.ctext	= "\xdd\xa9\x7c\xa4\x86\x4c\xdf\xe0"
+			  "\x6e\xaf\x70\xa0\xec\x0d\x71\x91",
+		.len	= 16,
+	}, {
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.klen	= 32,
+		.ptext	= "\x00\x11\x22\x33\x44\x55\x66\x77"
+			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
+		.ctext	= "\x8e\xa2\xb7\xca\x51\x67\x45\xbf"
+			  "\xea\xfc\x49\x90\x4b\x49\x60\x89",
+		.len	= 16,
+	}, { /* Generated with Crypto++ */
+		.key	= "\xA6\xC9\x83\xA6\xC9\xEC\x0F\x32"
+			  "\x55\x0F\x32\x55\x78\x9B\xBE\x78"
+			  "\x9B\xBE\xE1\x04\x27\xE1\x04\x27"
+			  "\x4A\x6D\x90\x4A\x6D\x90\xB3\xD6",
+		.klen	= 32,
+		.ptext	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
+			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
+			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
+			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
+			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
+			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
+			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
+			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
+			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
+			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
+			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
+			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
+			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
+			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
+			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
+			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
+			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
+			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
+			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
+			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
+			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
+			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
+			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
+			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
+			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
+			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
+			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
+			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
+			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
+			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
+			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
+			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
+			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
+			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
+			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
+			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
+			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
+			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
+			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
+			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
+			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
+			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
+			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
+			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
+			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
+			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
+			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
+			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
+			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
+			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
+			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
+			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
+			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
+			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
+			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
+			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
+			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
+			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
+			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
+			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
+			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
+			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12",
+		.ctext	= "\x71\x73\xF7\xDB\x24\x93\x21\x6D"
+			  "\x61\x1E\xBB\x63\x42\x79\xDB\x64"
+			  "\x6F\x82\xC0\xCA\xA3\x9B\xFA\x0B"
+			  "\xD9\x08\xC7\x4A\x90\xAE\x8F\x5F"
+			  "\x5E\x06\xF0\x5F\x31\x51\x18\x37"
+			  "\x45\xD7\xCA\x3A\xFD\x6C\x3F\xE1"
+			  "\xDD\x8D\x22\x65\x2B\x00\x50\xCE"
+			  "\xBA\x28\x67\xD7\xCE\x0E\x0D\xEA"
+			  "\x78\x69\x7F\xAE\x8F\x8B\x69\x37"
+			  "\x75\xE0\xDC\x96\xE0\xB7\xF4\x09"
+			  "\xCB\x6D\xA2\xFB\xDA\xAF\x09\xF8"
+			  "\x81\x82\x27\xFA\x45\x9C\x29\xA4"
+			  "\x22\x8B\x78\x69\x5B\x46\xF9\x39"
+			  "\x1B\xCC\xF9\x1D\x09\xEB\xBC\x5C"
+			  "\x41\x72\x51\x97\x1D\x07\x49\xA0"
+			  "\x1B\x8E\x65\x4B\xB2\x6A\x12\x03"
+			  "\x6A\x60\x95\xAC\xBD\xAC\x1A\x64"
+			  "\xDE\x5A\xA5\xF0\x83\x2F\xCB\xCA"
+			  "\x22\x74\xA6\x6C\x9B\x73\xCE\x3F"
+			  "\xE1\x8B\x22\x17\x59\x0C\x47\x89"
+			  "\x33\xA1\xD6\x47\x03\x19\x4F\xA8"
+			  "\x67\x69\xF0\x5B\xF0\x20\xAD\x06"
+			  "\x27\x81\x92\xD8\xC5\xBA\x98\x12"
+			  "\xBE\x24\xB5\x2F\x75\x02\xC2\xAD"
+			  "\x12\x2F\x07\x32\xEE\x39\xAF\x64"
+			  "\x05\x8F\xB3\xD4\xEB\x1B\x46\x6E"
+			  "\xD9\x21\xF9\xC4\xB7\xC9\x45\x68"
+			  "\xB4\xA1\x74\x9F\x82\x47\xEB\xCC"
+			  "\xBD\x0A\x14\x95\x0F\x8B\xA8\x2F"
+			  "\x4B\x1B\xA7\xBF\x82\xA6\x43\x0C"
+			  "\xB9\x39\x4A\xA8\x10\x6F\x50\x7B"
+			  "\x25\xFB\x26\x81\xE0\x2F\xF0\x96"
+			  "\x8D\x8B\xAC\x92\x0F\xF6\xED\x64"
+			  "\x63\x29\x4C\x8E\x18\x13\xC5\xBF"
+			  "\xFC\xA0\xD9\xBF\x7C\x3A\x0E\x29"
+			  "\x6F\xD1\x6C\x6F\xA5\xDA\xBF\xB1"
+			  "\x30\xEA\x44\x2D\xC3\x8F\x16\xE1"
+			  "\x66\xFA\xA3\x21\x3E\xFC\x13\xCA"
+			  "\xF0\xF6\xF0\x59\xBD\x8F\x38\x50"
+			  "\x31\xCB\x69\x3F\x96\x15\xD6\xF5"
+			  "\xAE\xFF\xF6\xAA\x41\x85\x4C\x10"
+			  "\x58\xE3\xF9\x44\xE6\x28\xDA\x9A"
+			  "\xDC\x6A\x80\x34\x73\x97\x1B\xC5"
+			  "\xCA\x26\x16\x77\x0E\x60\xAB\x89"
+			  "\x0F\x04\x27\xBD\xCE\x3E\x71\xB4"
+			  "\xA0\xD7\x22\x7E\xDB\xEB\x24\x70"
+			  "\x42\x71\x51\x78\x70\xB3\xE0\x3D"
+			  "\x84\x8E\x8D\x7B\xD0\x6D\xEA\x92"
+			  "\x11\x08\x42\x4F\xE5\xAD\x26\x92"
+			  "\xD2\x00\xAE\xA8\xE3\x4B\x37\x47"
+			  "\x22\xC1\x95\xC1\x63\x7F\xCB\x03"
+			  "\xF3\xE3\xD7\x9D\x60\xC7\xBC\xEA"
+			  "\x35\xA2\xFD\x45\x52\x39\x13\x6F"
+			  "\xC1\x53\xF3\x53\xDF\x33\x84\xD7"
+			  "\xD2\xC8\x37\xB0\x75\xE3\x41\x46"
+			  "\xB3\xC7\x83\x2E\x8A\xBB\xA4\xE5"
+			  "\x7F\x3C\xFD\x8B\xEB\xEA\x63\xBD"
+			  "\xB7\x46\xE7\xBF\x09\x9C\x0D\x0F"
+			  "\x40\x86\x7F\x51\xE1\x11\x9C\xCB"
+			  "\x88\xE6\x68\x47\xE3\x2B\xC5\xFF"
+			  "\x09\x79\xA0\x43\x5C\x0D\x08\x58"
+			  "\x17\xBB\xC0\x6B\x62\x3F\x56\xE9",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec aes_cbc_tv_template[] = {
+	{ /* From RFC 3602 */
+		.key    = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
+			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
+		.klen   = 16,
+		.iv	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
+			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
+		.ptext	= "Single block msg",
+		.ctext	= "\xe3\x53\x77\x9c\x10\x79\xae\xb8"
+			  "\x27\x08\x94\x2d\xbe\x77\x18\x1a",
+		.len	= 16,
+		.also_non_np = 1,
+		.np	= 8,
+		.tap	= { 3, 2, 3, 2, 3, 1, 1, 1 },
+	}, {
+		.key    = "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
+			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
+		.klen   = 16,
+		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
+			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.ctext	= "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a"
+			  "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a"
+			  "\x75\x86\x60\x2d\x25\x3c\xff\xf9"
+			  "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1",
+		.len	= 32,
+	}, { /* From NIST SP800-38A */
+		.key	= "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
+			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
+			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
+		.klen	= 24,
+		.iv	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.ptext	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ctext	= "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d"
+			  "\x71\x78\x18\x3a\x9f\xa0\x71\xe8"
+			  "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4"
+			  "\xe5\xe7\x38\x76\x3f\x69\x14\x5a"
+			  "\x57\x1b\x24\x20\x12\xfb\x7a\xe0"
+			  "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0"
+			  "\x08\xb0\xe2\x79\x88\x59\x88\x81"
+			  "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd",
+		.len	= 64,
+	}, {
+		.key	= "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
+			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
+			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
+			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
+		.klen	= 32,
+		.iv	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.ptext	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ctext	= "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba"
+			  "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6"
+			  "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d"
+			  "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d"
+			  "\x39\xf2\x33\x69\xa9\xd9\xba\xcf"
+			  "\xa5\x30\xe2\x63\x04\x23\x14\x61"
+			  "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc"
+			  "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b",
+		.len	= 64,
+	}, { /* Generated with Crypto++ */
+		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55"
+			  "\x0F\x32\x55\x78\x9B\xBE\x78\x9B"
+			  "\xBE\xE1\x04\x27\xE1\x04\x27\x4A"
+			  "\x6D\x90\x4A\x6D\x90\xB3\xD6\xF9",
+		.klen	= 32,
+		.iv	= "\xE7\x82\x1D\xB8\x53\x11\xAC\x47"
+			  "\xE2\x7D\x18\xD6\x71\x0C\xA7\x42",
+		.ptext	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
+			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
+			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
+			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
+			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
+			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
+			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
+			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
+			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
+			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
+			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
+			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
+			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
+			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
+			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
+			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
+			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
+			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
+			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
+			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
+			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
+			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
+			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
+			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
+			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
+			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
+			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
+			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
+			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
+			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
+			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
+			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
+			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
+			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
+			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
+			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
+			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
+			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
+			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
+			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
+			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
+			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
+			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
+			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
+			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
+			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
+			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
+			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
+			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
+			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
+			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
+			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
+			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
+			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
+			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
+			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
+			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
+			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
+			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
+			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
+			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
+			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12",
+		.ctext	= "\xEA\x65\x8A\x19\xB0\x66\xC1\x3F"
+			  "\xCE\xF1\x97\x75\xC1\xFD\xB5\xAF"
+			  "\x52\x65\xF7\xFF\xBC\xD8\x2D\x9F"
+			  "\x2F\xB9\x26\x9B\x6F\x10\xB7\xB8"
+			  "\x26\xA1\x02\x46\xA2\xAD\xC6\xC0"
+			  "\x11\x15\xFF\x6D\x1E\x82\x04\xA6"
+			  "\xB1\x74\xD1\x08\x13\xFD\x90\x7C"
+			  "\xF5\xED\xD3\xDB\x5A\x0A\x0C\x2F"
+			  "\x0A\x70\xF1\x88\x07\xCF\x21\x26"
+			  "\x40\x40\x8A\xF5\x53\xF7\x24\x4F"
+			  "\x83\x38\x43\x5F\x08\x99\xEB\xE3"
+			  "\xDC\x02\x64\x67\x50\x6E\x15\xC3"
+			  "\x01\x1A\xA0\x81\x13\x65\xA6\x73"
+			  "\x71\xA6\x3B\x91\x83\x77\xBE\xFA"
+			  "\xDB\x71\x73\xA6\xC1\xAE\x43\xC3"
+			  "\x36\xCE\xD6\xEB\xF9\x30\x1C\x4F"
+			  "\x80\x38\x5E\x9C\x6E\xAB\x98\x2F"
+			  "\x53\xAF\xCF\xC8\x9A\xB8\x86\x43"
+			  "\x3E\x86\xE7\xA1\xF4\x2F\x30\x40"
+			  "\x03\xA8\x6C\x50\x42\x9F\x77\x59"
+			  "\x89\xA0\xC5\xEC\x9A\xB8\xDD\x99"
+			  "\x16\x24\x02\x07\x48\xAE\xF2\x31"
+			  "\x34\x0E\xC3\x85\xFE\x1C\x95\x99"
+			  "\x87\x58\x98\x8B\xE7\xC6\xC5\x70"
+			  "\x73\x81\x07\x7C\x56\x2F\xD8\x1B"
+			  "\xB7\xB9\x2B\xAB\xE3\x01\x87\x0F"
+			  "\xD8\xBB\xC0\x0D\xAC\x2C\x2F\x98"
+			  "\x3C\x0B\xA2\x99\x4A\x8C\xF7\x04"
+			  "\xE0\xE0\xCF\xD1\x81\x5B\xFE\xF5"
+			  "\x24\x04\xFD\xB8\xDF\x13\xD8\xCD"
+			  "\xF1\xE3\x3D\x98\x50\x02\x77\x9E"
+			  "\xBC\x22\xAB\xFA\xC2\x43\x1F\x66"
+			  "\x20\x02\x23\xDA\xDF\xA0\x89\xF6"
+			  "\xD8\xF3\x45\x24\x53\x6F\x16\x77"
+			  "\x02\x3E\x7B\x36\x5F\xA0\x3B\x78"
+			  "\x63\xA2\xBD\xB5\xA4\xCA\x1E\xD3"
+			  "\x57\xBC\x0B\x9F\x43\x51\x28\x4F"
+			  "\x07\x50\x6C\x68\x12\x07\xCF\xFA"
+			  "\x6B\x72\x0B\xEB\xF8\x88\x90\x2C"
+			  "\x7E\xF5\x91\xD1\x03\xD8\xD5\xBD"
+			  "\x22\x39\x7B\x16\x03\x01\x69\xAF"
+			  "\x3D\x38\x66\x28\x0C\xBE\x5B\xC5"
+			  "\x03\xB4\x2F\x51\x8A\x56\x17\x2B"
+			  "\x88\x42\x6D\x40\x68\x8F\xD0\x11"
+			  "\x19\xF9\x1F\x43\x79\x95\x31\xFA"
+			  "\x28\x7A\x3D\xF7\x66\xEB\xEF\xAC"
+			  "\x06\xB2\x01\xAD\xDB\x68\xDB\xEC"
+			  "\x8D\x53\x6E\x72\x68\xA3\xC7\x63"
+			  "\x43\x2B\x78\xE0\x04\x29\x8F\x72"
+			  "\xB2\x2C\xE6\x84\x03\x30\x6D\xCD"
+			  "\x26\x92\x37\xE1\x2F\xBB\x8B\x9D"
+			  "\xE4\x4C\xF6\x93\xBC\xD9\xAD\x44"
+			  "\x52\x65\xC7\xB0\x0E\x3F\x0E\x61"
+			  "\x56\x5D\x1C\x6D\xA7\x05\x2E\xBC"
+			  "\x58\x08\x15\xAB\x12\xAB\x17\x4A"
+			  "\x5E\x1C\xF2\xCD\xB8\xA2\xAE\xFB"
+			  "\x9B\x2E\x0E\x85\x34\x80\x0E\x3F"
+			  "\x4C\xB8\xDB\xCE\x1C\x90\xA1\x61"
+			  "\x6C\x69\x09\x35\x9E\xD4\xF4\xAD"
+			  "\xBC\x06\x41\xE3\x01\xB4\x4E\x0A"
+			  "\xE0\x1F\x91\xF8\x82\x96\x2D\x65"
+			  "\xA3\xAA\x13\xCC\x50\xFF\x7B\x02",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
+};
+
+static const struct aead_testvec hmac_md5_ecb_cipher_null_enc_tv_template[] = {
+	{ /* Input data from RFC 2410 Case 1 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x00"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen   = 8 + 16 + 0,
+		.iv     = "",
+		.input  = "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.ilen   = 8,
+		.result = "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\xaa\x42\xfe\x43\x8d\xea\xa3\x5a"
+			  "\xb9\x3d\x9f\xb1\xa3\x8e\x9b\xae",
+		.rlen   = 8 + 16,
+	}, { /* Input data from RFC 2410 Case 2 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x00"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen   = 8 + 16 + 0,
+		.iv     = "",
+		.input  = "Network Security People Have A Strange Sense Of Humor",
+		.ilen   = 53,
+		.result = "Network Security People Have A Strange Sense Of Humor"
+			  "\x73\xa5\x3e\x1c\x08\x0e\x8a\x8a"
+			  "\x8e\xb5\x5f\x90\x8e\xfe\x13\x23",
+		.rlen   = 53 + 16,
+	},
+};
+
+static const struct aead_testvec hmac_md5_ecb_cipher_null_dec_tv_template[] = {
+	{
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x00"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen   = 8 + 16 + 0,
+		.iv     = "",
+		.input  = "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\xaa\x42\xfe\x43\x8d\xea\xa3\x5a"
+			  "\xb9\x3d\x9f\xb1\xa3\x8e\x9b\xae",
+		.ilen   = 8 + 16,
+		.result = "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.rlen   = 8,
+	}, {
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x00"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen   = 8 + 16 + 0,
+		.iv     = "",
+		.input  = "Network Security People Have A Strange Sense Of Humor"
+			  "\x73\xa5\x3e\x1c\x08\x0e\x8a\x8a"
+			  "\x8e\xb5\x5f\x90\x8e\xfe\x13\x23",
+		.ilen   = 53 + 16,
+		.result = "Network Security People Have A Strange Sense Of Humor",
+		.rlen   = 53,
+	},
+};
+
+static const struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = {
+	{ /* RFC 3602 Case 1 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00"
+			  "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
+			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
+		.klen   = 8 + 20 + 16,
+		.iv     = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
+			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
+		.assoc	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
+			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
+		.alen	= 16,
+		.input  = "Single block msg",
+		.ilen   = 16,
+		.result = "\xe3\x53\x77\x9c\x10\x79\xae\xb8"
+			  "\x27\x08\x94\x2d\xbe\x77\x18\x1a"
+			  "\x1b\x13\xcb\xaf\x89\x5e\xe1\x2c"
+			  "\x13\xc5\x2e\xa3\xcc\xed\xdc\xb5"
+			  "\x03\x71\xa2\x06",
+		.rlen   = 16 + 20,
+	}, { /* RFC 3602 Case 2 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33"
+			  "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
+			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
+		.klen   = 8 + 20 + 16,
+		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
+			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
+		.assoc	= "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
+			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
+		.alen	= 16,
+		.input  = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.ilen   = 32,
+		.result = "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a"
+			  "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a"
+			  "\x75\x86\x60\x2d\x25\x3c\xff\xf9"
+			  "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1"
+			  "\xad\x9b\x4c\x5c\x85\xe1\xda\xae"
+			  "\xee\x81\x4e\xd7\xdb\x74\xcf\x58"
+			  "\x65\x39\xf8\xde",
+		.rlen   = 32 + 20,
+	}, { /* RFC 3602 Case 3 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"            /* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55"
+			  "\x6c\x3e\xa0\x47\x76\x30\xce\x21"
+			  "\xa2\xce\x33\x4a\xa7\x46\xc2\xcd",
+		.klen   = 8 + 20 + 16,
+		.iv     = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
+			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
+		.assoc	= "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
+			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
+		.alen	= 16,
+		.input  = "This is a 48-byte message (exactly 3 AES blocks)",
+		.ilen   = 48,
+		.result = "\xd0\xa0\x2b\x38\x36\x45\x17\x53"
+			  "\xd4\x93\x66\x5d\x33\xf0\xe8\x86"
+			  "\x2d\xea\x54\xcd\xb2\x93\xab\xc7"
+			  "\x50\x69\x39\x27\x67\x72\xf8\xd5"
+			  "\x02\x1c\x19\x21\x6b\xad\x52\x5c"
+			  "\x85\x79\x69\x5d\x83\xba\x26\x84"
+			  "\xc2\xec\x0c\xf8\x7f\x05\xba\xca"
+			  "\xff\xee\x4c\xd0\x93\xe6\x36\x7f"
+			  "\x8d\x62\xf2\x1e",
+		.rlen   = 48 + 20,
+	}, { /* RFC 3602 Case 4 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55"
+			  "\x56\xe4\x7a\x38\xc5\x59\x89\x74"
+			  "\xbc\x46\x90\x3d\xba\x29\x03\x49",
+		.klen   = 8 + 20 + 16,
+		.iv     = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
+			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
+		.assoc	= "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
+			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
+		.alen	= 16,
+		.input  = "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf",
+		.ilen   = 64,
+		.result = "\xc3\x0e\x32\xff\xed\xc0\x77\x4e"
+			  "\x6a\xff\x6a\xf0\x86\x9f\x71\xaa"
+			  "\x0f\x3a\xf0\x7a\x9a\x31\xa9\xc6"
+			  "\x84\xdb\x20\x7e\xb0\xef\x8e\x4e"
+			  "\x35\x90\x7a\xa6\x32\xc3\xff\xdf"
+			  "\x86\x8b\xb7\xb2\x9d\x3d\x46\xad"
+			  "\x83\xce\x9f\x9a\x10\x2e\xe9\x9d"
+			  "\x49\xa5\x3e\x87\xf4\xc3\xda\x55"
+			  "\x1c\x45\x57\xa9\x56\xcb\xa9\x2d"
+			  "\x18\xac\xf1\xc7\x5d\xd1\xcd\x0d"
+			  "\x1d\xbe\xc6\xe9",
+		.rlen   = 64 + 20,
+	}, { /* RFC 3602 Case 5 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"            /* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55"
+			  "\x90\xd3\x82\xb4\x10\xee\xba\x7a"
+			  "\xd9\x38\xc4\x6c\xec\x1a\x82\xbf",
+		.klen   = 8 + 20 + 16,
+		.iv     = "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
+			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
+			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
+		.alen   = 24,
+		.input  = "\x08\x00\x0e\xbd\xa7\x0a\x00\x00"
+			  "\x8e\x9c\x08\x3d\xb9\x5b\x07\x00"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x01\x02\x03\x04\x05\x06\x07\x08"
+			  "\x09\x0a\x0b\x0c\x0d\x0e\x0e\x01",
+		.ilen   = 80,
+		.result = "\xf6\x63\xc2\x5d\x32\x5c\x18\xc6"
+			  "\xa9\x45\x3e\x19\x4e\x12\x08\x49"
+			  "\xa4\x87\x0b\x66\xcc\x6b\x99\x65"
+			  "\x33\x00\x13\xb4\x89\x8d\xc8\x56"
+			  "\xa4\x69\x9e\x52\x3a\x55\xdb\x08"
+			  "\x0b\x59\xec\x3a\x8e\x4b\x7e\x52"
+			  "\x77\x5b\x07\xd1\xdb\x34\xed\x9c"
+			  "\x53\x8a\xb5\x0c\x55\x1b\x87\x4a"
+			  "\xa2\x69\xad\xd0\x47\xad\x2d\x59"
+			  "\x13\xac\x19\xb7\xcf\xba\xd4\xa6"
+			  "\x58\xc6\x84\x75\xe4\xe9\x6b\x0c"
+			  "\xe1\xc5\x0b\x73\x4d\x82\x55\xa8"
+			  "\x85\xe1\x59\xf7",
+		.rlen   = 80 + 20,
+       }, { /* NIST SP800-38A F.2.3 CBC-AES192.Encrypt */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"            /* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x18"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55"
+			  "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
+			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
+			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
+		.klen   = 8 + 20 + 24,
+		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.alen	= 16,
+		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ilen   = 64,
+		.result = "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d"
+			  "\x71\x78\x18\x3a\x9f\xa0\x71\xe8"
+			  "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4"
+			  "\xe5\xe7\x38\x76\x3f\x69\x14\x5a"
+			  "\x57\x1b\x24\x20\x12\xfb\x7a\xe0"
+			  "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0"
+			  "\x08\xb0\xe2\x79\x88\x59\x88\x81"
+			  "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd"
+			  "\x73\xe3\x19\x3f\x8b\xc9\xc6\xf4"
+			  "\x5a\xf1\x5b\xa8\x98\x07\xc5\x36"
+			  "\x47\x4c\xfc\x36",
+		.rlen   = 64 + 20,
+	}, { /* NIST SP800-38A F.2.5 CBC-AES256.Encrypt */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x20"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55"
+			  "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
+			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
+			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
+			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
+		.klen   = 8 + 20 + 32,
+		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.alen	= 16,
+		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ilen   = 64,
+		.result = "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba"
+			  "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6"
+			  "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d"
+			  "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d"
+			  "\x39\xf2\x33\x69\xa9\xd9\xba\xcf"
+			  "\xa5\x30\xe2\x63\x04\x23\x14\x61"
+			  "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc"
+			  "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b"
+			  "\xa3\xe8\x9b\x17\xe3\xf4\x7f\xde"
+			  "\x1b\x9f\xc6\x81\x26\x43\x4a\x87"
+			  "\x51\xee\xd6\x4e",
+		.rlen   = 64 + 20,
+	},
+};
+
+static const struct aead_testvec hmac_sha1_ecb_cipher_null_enc_tv_temp[] = {
+	{ /* Input data from RFC 2410 Case 1 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x00"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00",
+		.klen   = 8 + 20 + 0,
+		.iv     = "",
+		.input  = "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.ilen   = 8,
+		.result = "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\x40\xc3\x0a\xa1\xc9\xa0\x28\xab"
+			  "\x99\x5e\x19\x04\xd1\x72\xef\xb8"
+			  "\x8c\x5e\xe4\x08",
+		.rlen   = 8 + 20,
+	}, { /* Input data from RFC 2410 Case 2 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x00"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00",
+		.klen   = 8 + 20 + 0,
+		.iv     = "",
+		.input  = "Network Security People Have A Strange Sense Of Humor",
+		.ilen   = 53,
+		.result = "Network Security People Have A Strange Sense Of Humor"
+			  "\x75\x6f\x42\x1e\xf8\x50\x21\xd2"
+			  "\x65\x47\xee\x8e\x1a\xef\x16\xf6"
+			  "\x91\x56\xe4\xd6",
+		.rlen   = 53 + 20,
+	},
+};
+
+static const struct aead_testvec hmac_sha1_ecb_cipher_null_dec_tv_temp[] = {
+	{
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x00"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00",
+		.klen   = 8 + 20 + 0,
+		.iv     = "",
+		.input  = "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\x40\xc3\x0a\xa1\xc9\xa0\x28\xab"
+			  "\x99\x5e\x19\x04\xd1\x72\xef\xb8"
+			  "\x8c\x5e\xe4\x08",
+		.ilen   = 8 + 20,
+		.result = "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.rlen   = 8,
+	}, {
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x00"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00",
+		.klen   = 8 + 20 + 0,
+		.iv     = "",
+		.input  = "Network Security People Have A Strange Sense Of Humor"
+			  "\x75\x6f\x42\x1e\xf8\x50\x21\xd2"
+			  "\x65\x47\xee\x8e\x1a\xef\x16\xf6"
+			  "\x91\x56\xe4\xd6",
+		.ilen   = 53 + 20,
+		.result = "Network Security People Have A Strange Sense Of Humor",
+		.rlen   = 53,
+	},
+};
+
+static const struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = {
+	{ /* RFC 3602 Case 1 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
+			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
+		.klen   = 8 + 32 + 16,
+		.iv     = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
+			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
+		.assoc	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
+			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
+		.alen	= 16,
+		.input  = "Single block msg",
+		.ilen   = 16,
+		.result = "\xe3\x53\x77\x9c\x10\x79\xae\xb8"
+			  "\x27\x08\x94\x2d\xbe\x77\x18\x1a"
+			  "\xcc\xde\x2d\x6a\xae\xf1\x0b\xcc"
+			  "\x38\x06\x38\x51\xb4\xb8\xf3\x5b"
+			  "\x5c\x34\xa6\xa3\x6e\x0b\x05\xe5"
+			  "\x6a\x6d\x44\xaa\x26\xa8\x44\xa5",
+		.rlen   = 16 + 32,
+	}, { /* RFC 3602 Case 2 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
+			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
+		.klen   = 8 + 32 + 16,
+		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
+			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
+		.assoc	= "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
+			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
+		.alen	= 16,
+		.input  = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.ilen   = 32,
+		.result = "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a"
+			  "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a"
+			  "\x75\x86\x60\x2d\x25\x3c\xff\xf9"
+			  "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1"
+			  "\xf5\x33\x53\xf3\x68\x85\x2a\x99"
+			  "\x0e\x06\x58\x8f\xba\xf6\x06\xda"
+			  "\x49\x69\x0d\x5b\xd4\x36\x06\x62"
+			  "\x35\x5e\x54\x58\x53\x4d\xdf\xbf",
+		.rlen   = 32 + 32,
+	}, { /* RFC 3602 Case 3 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"            /* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x6c\x3e\xa0\x47\x76\x30\xce\x21"
+			  "\xa2\xce\x33\x4a\xa7\x46\xc2\xcd",
+		.klen   = 8 + 32 + 16,
+		.iv     = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
+			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
+		.assoc	= "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
+			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
+		.alen	= 16,
+		.input  = "This is a 48-byte message (exactly 3 AES blocks)",
+		.ilen   = 48,
+		.result = "\xd0\xa0\x2b\x38\x36\x45\x17\x53"
+			  "\xd4\x93\x66\x5d\x33\xf0\xe8\x86"
+			  "\x2d\xea\x54\xcd\xb2\x93\xab\xc7"
+			  "\x50\x69\x39\x27\x67\x72\xf8\xd5"
+			  "\x02\x1c\x19\x21\x6b\xad\x52\x5c"
+			  "\x85\x79\x69\x5d\x83\xba\x26\x84"
+			  "\x68\xb9\x3e\x90\x38\xa0\x88\x01"
+			  "\xe7\xc6\xce\x10\x31\x2f\x9b\x1d"
+			  "\x24\x78\xfb\xbe\x02\xe0\x4f\x40"
+			  "\x10\xbd\xaa\xc6\xa7\x79\xe0\x1a",
+		.rlen   = 48 + 32,
+	}, { /* RFC 3602 Case 4 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x56\xe4\x7a\x38\xc5\x59\x89\x74"
+			  "\xbc\x46\x90\x3d\xba\x29\x03\x49",
+		.klen   = 8 + 32 + 16,
+		.iv     = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
+			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
+		.assoc	= "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
+			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
+		.alen	= 16,
+		.input  = "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf",
+		.ilen   = 64,
+		.result = "\xc3\x0e\x32\xff\xed\xc0\x77\x4e"
+			  "\x6a\xff\x6a\xf0\x86\x9f\x71\xaa"
+			  "\x0f\x3a\xf0\x7a\x9a\x31\xa9\xc6"
+			  "\x84\xdb\x20\x7e\xb0\xef\x8e\x4e"
+			  "\x35\x90\x7a\xa6\x32\xc3\xff\xdf"
+			  "\x86\x8b\xb7\xb2\x9d\x3d\x46\xad"
+			  "\x83\xce\x9f\x9a\x10\x2e\xe9\x9d"
+			  "\x49\xa5\x3e\x87\xf4\xc3\xda\x55"
+			  "\x7a\x1b\xd4\x3c\xdb\x17\x95\xe2"
+			  "\xe0\x93\xec\xc9\x9f\xf7\xce\xd8"
+			  "\x3f\x54\xe2\x49\x39\xe3\x71\x25"
+			  "\x2b\x6c\xe9\x5d\xec\xec\x2b\x64",
+		.rlen   = 64 + 32,
+	}, { /* RFC 3602 Case 5 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"            /* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x90\xd3\x82\xb4\x10\xee\xba\x7a"
+			  "\xd9\x38\xc4\x6c\xec\x1a\x82\xbf",
+		.klen   = 8 + 32 + 16,
+		.iv     = "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
+			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
+			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
+		.alen   = 24,
+		.input  = "\x08\x00\x0e\xbd\xa7\x0a\x00\x00"
+			  "\x8e\x9c\x08\x3d\xb9\x5b\x07\x00"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x01\x02\x03\x04\x05\x06\x07\x08"
+			  "\x09\x0a\x0b\x0c\x0d\x0e\x0e\x01",
+		.ilen   = 80,
+		.result = "\xf6\x63\xc2\x5d\x32\x5c\x18\xc6"
+			  "\xa9\x45\x3e\x19\x4e\x12\x08\x49"
+			  "\xa4\x87\x0b\x66\xcc\x6b\x99\x65"
+			  "\x33\x00\x13\xb4\x89\x8d\xc8\x56"
+			  "\xa4\x69\x9e\x52\x3a\x55\xdb\x08"
+			  "\x0b\x59\xec\x3a\x8e\x4b\x7e\x52"
+			  "\x77\x5b\x07\xd1\xdb\x34\xed\x9c"
+			  "\x53\x8a\xb5\x0c\x55\x1b\x87\x4a"
+			  "\xa2\x69\xad\xd0\x47\xad\x2d\x59"
+			  "\x13\xac\x19\xb7\xcf\xba\xd4\xa6"
+			  "\xbb\xd4\x0f\xbe\xa3\x3b\x4c\xb8"
+			  "\x3a\xd2\xe1\x03\x86\xa5\x59\xb7"
+			  "\x73\xc3\x46\x20\x2c\xb1\xef\x68"
+			  "\xbb\x8a\x32\x7e\x12\x8c\x69\xcf",
+		.rlen   = 80 + 32,
+       }, { /* NIST SP800-38A F.2.3 CBC-AES192.Encrypt */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"            /* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x18"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
+			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
+			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
+		.klen   = 8 + 32 + 24,
+		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.alen   = 16,
+		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ilen   = 64,
+		.result = "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d"
+			  "\x71\x78\x18\x3a\x9f\xa0\x71\xe8"
+			  "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4"
+			  "\xe5\xe7\x38\x76\x3f\x69\x14\x5a"
+			  "\x57\x1b\x24\x20\x12\xfb\x7a\xe0"
+			  "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0"
+			  "\x08\xb0\xe2\x79\x88\x59\x88\x81"
+			  "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd"
+			  "\x2f\xee\x5f\xdb\x66\xfe\x79\x09"
+			  "\x61\x81\x31\xea\x5b\x3d\x8e\xfb"
+			  "\xca\x71\x85\x93\xf7\x85\x55\x8b"
+			  "\x7a\xe4\x94\xca\x8b\xba\x19\x33",
+		.rlen   = 64 + 32,
+	}, { /* NIST SP800-38A F.2.5 CBC-AES256.Encrypt */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x20"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
+			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
+			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
+			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
+		.klen   = 8 + 32 + 32,
+		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.alen   = 16,
+		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ilen   = 64,
+		.result = "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba"
+			  "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6"
+			  "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d"
+			  "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d"
+			  "\x39\xf2\x33\x69\xa9\xd9\xba\xcf"
+			  "\xa5\x30\xe2\x63\x04\x23\x14\x61"
+			  "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc"
+			  "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b"
+			  "\x24\x29\xed\xc2\x31\x49\xdb\xb1"
+			  "\x8f\x74\xbd\x17\x92\x03\xbe\x8f"
+			  "\xf3\x61\xde\x1c\xe9\xdb\xcd\xd0"
+			  "\xcc\xce\xe9\x85\x57\xcf\x6f\x5f",
+		.rlen   = 64 + 32,
+	},
+};
+
+static const struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = {
+	{ /* RFC 3602 Case 1 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
+			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
+		.klen   = 8 + 64 + 16,
+		.iv     = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
+			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
+		.assoc	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
+			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
+		.alen   = 16,
+		.input  = "Single block msg",
+		.ilen   = 16,
+		.result = "\xe3\x53\x77\x9c\x10\x79\xae\xb8"
+			  "\x27\x08\x94\x2d\xbe\x77\x18\x1a"
+			  "\x3f\xdc\xad\x90\x03\x63\x5e\x68"
+			  "\xc3\x13\xdd\xa4\x5c\x4d\x54\xa7"
+			  "\x19\x6e\x03\x75\x2b\xa1\x62\xce"
+			  "\xe0\xc6\x96\x75\xb2\x14\xca\x96"
+			  "\xec\xbd\x50\x08\x07\x64\x1a\x49"
+			  "\xe8\x9a\x7c\x06\x3d\xcb\xff\xb2"
+			  "\xfa\x20\x89\xdd\x9c\xac\x9e\x16"
+			  "\x18\x8a\xa0\x6d\x01\x6c\xa3\x3a",
+		.rlen   = 16 + 64,
+	}, { /* RFC 3602 Case 2 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
+			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
+		.klen   = 8 + 64 + 16,
+		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
+			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
+		.assoc	= "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
+			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
+		.alen   = 16,
+		.input  = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.ilen   = 32,
+		.result = "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a"
+			  "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a"
+			  "\x75\x86\x60\x2d\x25\x3c\xff\xf9"
+			  "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1"
+			  "\xda\xb2\x0c\xb2\x26\xc4\xd5\xef"
+			  "\x60\x38\xa4\x5e\x9a\x8c\x1b\x41"
+			  "\x03\x9f\xc4\x64\x7f\x01\x42\x9b"
+			  "\x0e\x1b\xea\xef\xbc\x88\x19\x5e"
+			  "\x31\x7e\xc2\x95\xfc\x09\x32\x0a"
+			  "\x46\x32\x7c\x41\x9c\x59\x3e\xe9"
+			  "\x8f\x9f\xd4\x31\xd6\x22\xbd\xf8"
+			  "\xf7\x0a\x94\xe5\xa9\xc3\xf6\x9d",
+		.rlen   = 32 + 64,
+	}, { /* RFC 3602 Case 3 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"            /* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
+			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
+			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
+			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
+			  "\x6c\x3e\xa0\x47\x76\x30\xce\x21"
+			  "\xa2\xce\x33\x4a\xa7\x46\xc2\xcd",
+		.klen   = 8 + 64 + 16,
+		.iv     = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
+			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
+		.assoc	= "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
+			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
+		.alen   = 16,
+		.input  = "This is a 48-byte message (exactly 3 AES blocks)",
+		.ilen   = 48,
+		.result = "\xd0\xa0\x2b\x38\x36\x45\x17\x53"
+			  "\xd4\x93\x66\x5d\x33\xf0\xe8\x86"
+			  "\x2d\xea\x54\xcd\xb2\x93\xab\xc7"
+			  "\x50\x69\x39\x27\x67\x72\xf8\xd5"
+			  "\x02\x1c\x19\x21\x6b\xad\x52\x5c"
+			  "\x85\x79\x69\x5d\x83\xba\x26\x84"
+			  "\x64\x19\x17\x5b\x57\xe0\x21\x0f"
+			  "\xca\xdb\xa1\x26\x38\x14\xa2\x69"
+			  "\xdb\x54\x67\x80\xc0\x54\xe0\xfd"
+			  "\x3e\x91\xe7\x91\x7f\x13\x38\x44"
+			  "\xb7\xb1\xd6\xc8\x7d\x48\x8d\x41"
+			  "\x08\xea\x29\x6c\x74\x67\x3f\xb0"
+			  "\xac\x7f\x5c\x1d\xf5\xee\x22\x66"
+			  "\x27\xa6\xb6\x13\xba\xba\xf0\xc2",
+		.rlen   = 48 + 64,
+	}, { /* RFC 3602 Case 4 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
+			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
+			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
+			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
+			  "\x56\xe4\x7a\x38\xc5\x59\x89\x74"
+			  "\xbc\x46\x90\x3d\xba\x29\x03\x49",
+		.klen   = 8 + 64 + 16,
+		.iv     = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
+			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
+		.assoc	= "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
+			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
+		.alen   = 16,
+		.input  = "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf",
+		.ilen   = 64,
+		.result = "\xc3\x0e\x32\xff\xed\xc0\x77\x4e"
+			  "\x6a\xff\x6a\xf0\x86\x9f\x71\xaa"
+			  "\x0f\x3a\xf0\x7a\x9a\x31\xa9\xc6"
+			  "\x84\xdb\x20\x7e\xb0\xef\x8e\x4e"
+			  "\x35\x90\x7a\xa6\x32\xc3\xff\xdf"
+			  "\x86\x8b\xb7\xb2\x9d\x3d\x46\xad"
+			  "\x83\xce\x9f\x9a\x10\x2e\xe9\x9d"
+			  "\x49\xa5\x3e\x87\xf4\xc3\xda\x55"
+			  "\x82\xcd\x42\x28\x21\x20\x15\xcc"
+			  "\xb7\xb2\x48\x40\xc7\x64\x41\x3a"
+			  "\x61\x32\x82\x85\xcf\x27\xed\xb4"
+			  "\xe4\x68\xa2\xf5\x79\x26\x27\xb2"
+			  "\x51\x67\x6a\xc4\xf0\x66\x55\x50"
+			  "\xbc\x6f\xed\xd5\x8d\xde\x23\x7c"
+			  "\x62\x98\x14\xd7\x2f\x37\x8d\xdf"
+			  "\xf4\x33\x80\xeb\x8e\xb4\xa4\xda",
+		.rlen   = 64 + 64,
+	}, { /* RFC 3602 Case 5 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"            /* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
+			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
+			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
+			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
+			  "\x90\xd3\x82\xb4\x10\xee\xba\x7a"
+			  "\xd9\x38\xc4\x6c\xec\x1a\x82\xbf",
+		.klen   = 8 + 64 + 16,
+		.iv     = "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
+			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\xe9\x6e\x8c\x08\xab\x46\x57\x63"
+			  "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93",
+		.alen   = 24,
+		.input  = "\x08\x00\x0e\xbd\xa7\x0a\x00\x00"
+			  "\x8e\x9c\x08\x3d\xb9\x5b\x07\x00"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x01\x02\x03\x04\x05\x06\x07\x08"
+			  "\x09\x0a\x0b\x0c\x0d\x0e\x0e\x01",
+		.ilen   = 80,
+		.result = "\xf6\x63\xc2\x5d\x32\x5c\x18\xc6"
+			  "\xa9\x45\x3e\x19\x4e\x12\x08\x49"
+			  "\xa4\x87\x0b\x66\xcc\x6b\x99\x65"
+			  "\x33\x00\x13\xb4\x89\x8d\xc8\x56"
+			  "\xa4\x69\x9e\x52\x3a\x55\xdb\x08"
+			  "\x0b\x59\xec\x3a\x8e\x4b\x7e\x52"
+			  "\x77\x5b\x07\xd1\xdb\x34\xed\x9c"
+			  "\x53\x8a\xb5\x0c\x55\x1b\x87\x4a"
+			  "\xa2\x69\xad\xd0\x47\xad\x2d\x59"
+			  "\x13\xac\x19\xb7\xcf\xba\xd4\xa6"
+			  "\x74\x84\x94\xe2\xd7\x7a\xf9\xbf"
+			  "\x00\x8a\xa2\xd5\xb7\xf3\x60\xcf"
+			  "\xa0\x47\xdf\x4e\x09\xf4\xb1\x7f"
+			  "\x14\xd9\x3d\x53\x8e\x12\xb3\x00"
+			  "\x4c\x0a\x4e\x32\x40\x43\x88\xce"
+			  "\x92\x26\xc1\x76\x20\x11\xeb\xba"
+			  "\x62\x4f\x9a\x62\x25\xc3\x75\x80"
+			  "\xb7\x0a\x17\xf5\xd7\x94\xb4\x14",
+		.rlen   = 80 + 64,
+       }, { /* NIST SP800-38A F.2.3 CBC-AES192.Encrypt */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"            /* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x18"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
+			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
+			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
+			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
+			  "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
+			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
+			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
+		.klen   = 8 + 64 + 24,
+		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.alen   = 16,
+		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ilen   = 64,
+		.result = "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d"
+			  "\x71\x78\x18\x3a\x9f\xa0\x71\xe8"
+			  "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4"
+			  "\xe5\xe7\x38\x76\x3f\x69\x14\x5a"
+			  "\x57\x1b\x24\x20\x12\xfb\x7a\xe0"
+			  "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0"
+			  "\x08\xb0\xe2\x79\x88\x59\x88\x81"
+			  "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd"
+			  "\x77\x4b\x69\x9d\x3a\x0d\xb4\x99"
+			  "\x8f\xc6\x8e\x0e\x72\x58\xe3\x56"
+			  "\xbb\x21\xd2\x7d\x93\x11\x17\x91"
+			  "\xc4\x83\xfd\x0a\xea\x71\xfe\x77"
+			  "\xae\x6f\x0a\xa5\xf0\xcf\xe1\x35"
+			  "\xba\x03\xd5\x32\xfa\x5f\x41\x58"
+			  "\x8d\x43\x98\xa7\x94\x16\x07\x02"
+			  "\x0f\xb6\x81\x50\x28\x95\x2e\x75",
+		.rlen   = 64 + 64,
+	}, { /* NIST SP800-38A F.2.5 CBC-AES256.Encrypt */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x20"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
+			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
+			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
+			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
+			  "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
+			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
+			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
+			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
+		.klen   = 8 + 64 + 32,
+		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.alen   = 16,
+		.input  = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ilen   = 64,
+		.result = "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba"
+			  "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6"
+			  "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d"
+			  "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d"
+			  "\x39\xf2\x33\x69\xa9\xd9\xba\xcf"
+			  "\xa5\x30\xe2\x63\x04\x23\x14\x61"
+			  "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc"
+			  "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b"
+			  "\xb2\x27\x69\x7f\x45\x64\x79\x2b"
+			  "\xb7\xb8\x4c\xd4\x75\x94\x68\x40"
+			  "\x2a\xea\x91\xc7\x3f\x7c\xed\x7b"
+			  "\x95\x2c\x9b\xa8\xf5\xe5\x52\x8d"
+			  "\x6b\xe1\xae\xf1\x74\xfa\x0d\x0c"
+			  "\xe3\x8d\x64\xc3\x8d\xff\x7c\x8c"
+			  "\xdb\xbf\xa0\xb4\x01\xa2\xa8\xa2"
+			  "\x2c\xb1\x62\x2c\x10\xca\xf1\x21",
+		.rlen   = 64 + 64,
+	},
+};
+
+static const struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = {
+	{ /*Generated with cryptopp*/
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+	.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x08"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+		  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55"
+			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24",
+		.klen	= 8 + 20 + 8,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.alen   = 16,
+		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.ilen	= 128,
+		.result	= "\x70\xd6\xde\x64\x87\x17\xf1\xe8"
+			  "\x54\x31\x85\x37\xed\x6b\x01\x8d"
+			  "\xe3\xcc\xe0\x1d\x5e\xf3\xfe\xf1"
+			  "\x41\xaa\x33\x91\xa7\x7d\x99\x88"
+			  "\x4d\x85\x6e\x2f\xa3\x69\xf5\x82"
+			  "\x3a\x6f\x25\xcb\x7d\x58\x1f\x9b"
+			  "\xaa\x9c\x11\xd5\x76\x67\xce\xde"
+			  "\x56\xd7\x5a\x80\x69\xea\x3a\x02"
+			  "\xf0\xc7\x7c\xe3\xcb\x40\xe5\x52"
+			  "\xd1\x10\x92\x78\x0b\x8e\x5b\xf1"
+			  "\xe3\x26\x1f\xe1\x15\x41\xc7\xba"
+			  "\x99\xdb\x08\x51\x1c\xd3\x01\xf4"
+			  "\x87\x47\x39\xb8\xd2\xdd\xbd\xfb"
+			  "\x66\x13\xdf\x1c\x01\x44\xf0\x7a"
+			  "\x1a\x6b\x13\xf5\xd5\x0b\xb8\xba"
+			  "\x53\xba\xe1\x76\xe3\x82\x07\x86"
+			  "\x95\x16\x20\x09\xf5\x95\x19\xfd"
+			  "\x3c\xc7\xe0\x42\xc0\x14\x69\xfa"
+			  "\x5c\x44\xa9\x37",
+			  .rlen	= 128 + 20,
+	},
+};
+
+static const struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = {
+	{ /*Generated with cryptopp*/
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x08"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+		  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24",
+		.klen	= 8 + 24 + 8,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.alen   = 16,
+		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.ilen	= 128,
+		.result	= "\x70\xd6\xde\x64\x87\x17\xf1\xe8"
+			  "\x54\x31\x85\x37\xed\x6b\x01\x8d"
+			  "\xe3\xcc\xe0\x1d\x5e\xf3\xfe\xf1"
+			  "\x41\xaa\x33\x91\xa7\x7d\x99\x88"
+			  "\x4d\x85\x6e\x2f\xa3\x69\xf5\x82"
+			  "\x3a\x6f\x25\xcb\x7d\x58\x1f\x9b"
+			  "\xaa\x9c\x11\xd5\x76\x67\xce\xde"
+			  "\x56\xd7\x5a\x80\x69\xea\x3a\x02"
+			  "\xf0\xc7\x7c\xe3\xcb\x40\xe5\x52"
+			  "\xd1\x10\x92\x78\x0b\x8e\x5b\xf1"
+			  "\xe3\x26\x1f\xe1\x15\x41\xc7\xba"
+			  "\x99\xdb\x08\x51\x1c\xd3\x01\xf4"
+		  "\x87\x47\x39\xb8\xd2\xdd\xbd\xfb"
+			  "\x66\x13\xdf\x1c\x01\x44\xf0\x7a"
+			  "\x1a\x6b\x13\xf5\xd5\x0b\xb8\xba"
+			  "\x53\xba\xe1\x76\xe3\x82\x07\x86"
+			  "\x9c\x2d\x7e\xee\x20\x34\x55\x0a"
+			  "\xce\xb5\x4e\x64\x53\xe7\xbf\x91"
+			  "\xab\xd4\xd9\xda\xc9\x12\xae\xf7",
+		.rlen	= 128 + 24,
+	},
+};
+
+static const struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = {
+	{ /*Generated with cryptopp*/
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x08"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24",
+		.klen	= 8 + 32 + 8,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.alen   = 16,
+		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.ilen	= 128,
+		.result	= "\x70\xd6\xde\x64\x87\x17\xf1\xe8"
+			  "\x54\x31\x85\x37\xed\x6b\x01\x8d"
+			  "\xe3\xcc\xe0\x1d\x5e\xf3\xfe\xf1"
+			  "\x41\xaa\x33\x91\xa7\x7d\x99\x88"
+			  "\x4d\x85\x6e\x2f\xa3\x69\xf5\x82"
+			  "\x3a\x6f\x25\xcb\x7d\x58\x1f\x9b"
+			  "\xaa\x9c\x11\xd5\x76\x67\xce\xde"
+			  "\x56\xd7\x5a\x80\x69\xea\x3a\x02"
+			  "\xf0\xc7\x7c\xe3\xcb\x40\xe5\x52"
+			  "\xd1\x10\x92\x78\x0b\x8e\x5b\xf1"
+			  "\xe3\x26\x1f\xe1\x15\x41\xc7\xba"
+		  "\x99\xdb\x08\x51\x1c\xd3\x01\xf4"
+			  "\x87\x47\x39\xb8\xd2\xdd\xbd\xfb"
+		  "\x66\x13\xdf\x1c\x01\x44\xf0\x7a"
+		  "\x1a\x6b\x13\xf5\xd5\x0b\xb8\xba"
+			  "\x53\xba\xe1\x76\xe3\x82\x07\x86"
+			  "\xc6\x58\xa1\x60\x70\x91\x39\x36"
+			  "\x50\xf6\x5d\xab\x4b\x51\x4e\x5e"
+			  "\xde\x63\xde\x76\x52\xde\x9f\xba"
+			  "\x90\xcf\x15\xf2\xbb\x6e\x84\x00",
+		.rlen	= 128 + 32,
+	},
+};
+
+static const struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = {
+	{ /*Generated with cryptopp*/
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x08"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
+			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
+			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24",
+		.klen	= 8 + 48 + 8,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.alen   = 16,
+		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.ilen	= 128,
+		.result	= "\x70\xd6\xde\x64\x87\x17\xf1\xe8"
+			  "\x54\x31\x85\x37\xed\x6b\x01\x8d"
+			  "\xe3\xcc\xe0\x1d\x5e\xf3\xfe\xf1"
+			  "\x41\xaa\x33\x91\xa7\x7d\x99\x88"
+			  "\x4d\x85\x6e\x2f\xa3\x69\xf5\x82"
+			  "\x3a\x6f\x25\xcb\x7d\x58\x1f\x9b"
+			  "\xaa\x9c\x11\xd5\x76\x67\xce\xde"
+			  "\x56\xd7\x5a\x80\x69\xea\x3a\x02"
+			  "\xf0\xc7\x7c\xe3\xcb\x40\xe5\x52"
+			  "\xd1\x10\x92\x78\x0b\x8e\x5b\xf1"
+			  "\xe3\x26\x1f\xe1\x15\x41\xc7\xba"
+			  "\x99\xdb\x08\x51\x1c\xd3\x01\xf4"
+			  "\x87\x47\x39\xb8\xd2\xdd\xbd\xfb"
+			  "\x66\x13\xdf\x1c\x01\x44\xf0\x7a"
+			  "\x1a\x6b\x13\xf5\xd5\x0b\xb8\xba"
+			  "\x53\xba\xe1\x76\xe3\x82\x07\x86"
+			  "\xa8\x8e\x9c\x74\x8c\x2b\x99\xa0"
+			  "\xc8\x8c\xef\x25\x07\x83\x11\x3a"
+			  "\x31\x8d\xbe\x3b\x6a\xd7\x96\xfe"
+			  "\x5e\x67\xb5\x74\xe7\xe7\x85\x61"
+			  "\x6a\x95\x26\x75\xcc\x53\x89\xf3"
+			  "\x74\xc9\x2a\x76\x20\xa2\x64\x62",
+		.rlen	= 128 + 48,
+	},
+};
+
+static const struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = {
+	{ /*Generated with cryptopp*/
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+		  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x08"	/* enc key length */
+		  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
+			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
+			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
+			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
+			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24",
+		.klen	= 8 + 64 + 8,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.alen   = 16,
+		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.ilen	= 128,
+		.result	= "\x70\xd6\xde\x64\x87\x17\xf1\xe8"
+			  "\x54\x31\x85\x37\xed\x6b\x01\x8d"
+			  "\xe3\xcc\xe0\x1d\x5e\xf3\xfe\xf1"
+			  "\x41\xaa\x33\x91\xa7\x7d\x99\x88"
+			  "\x4d\x85\x6e\x2f\xa3\x69\xf5\x82"
+			  "\x3a\x6f\x25\xcb\x7d\x58\x1f\x9b"
+			  "\xaa\x9c\x11\xd5\x76\x67\xce\xde"
+		  "\x56\xd7\x5a\x80\x69\xea\x3a\x02"
+			  "\xf0\xc7\x7c\xe3\xcb\x40\xe5\x52"
+		  "\xd1\x10\x92\x78\x0b\x8e\x5b\xf1"
+			  "\xe3\x26\x1f\xe1\x15\x41\xc7\xba"
+			  "\x99\xdb\x08\x51\x1c\xd3\x01\xf4"
+			  "\x87\x47\x39\xb8\xd2\xdd\xbd\xfb"
+			  "\x66\x13\xdf\x1c\x01\x44\xf0\x7a"
+		  "\x1a\x6b\x13\xf5\xd5\x0b\xb8\xba"
+			  "\x53\xba\xe1\x76\xe3\x82\x07\x86"
+			  "\xc6\x2c\x73\x88\xb0\x9d\x5f\x3e"
+			  "\x5b\x78\xca\x0e\xab\x8a\xa3\xbb"
+			  "\xd9\x1d\xc3\xe3\x05\xac\x76\xfb"
+			  "\x58\x83\xda\x67\xfb\x21\x24\xa2"
+			  "\xb1\xa7\xd7\x66\xa6\x8d\xa6\x93"
+			  "\x97\xe2\xe3\xb8\xaa\x48\x85\xee"
+			  "\x8c\xf6\x07\x95\x1f\xa6\x6c\x96"
+			  "\x99\xc7\x5c\x8d\xd8\xb5\x68\x7b",
+		.rlen	= 128 + 64,
+	},
+};
+
+static const struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = {
+	{ /*Generated with cryptopp*/
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x18"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55"
+		  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
+			  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
+			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
+		.klen	= 8 + 20 + 24,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.alen   = 16,
+		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+		  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.ilen	= 128,
+		.result	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
+			  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
+		  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
+		  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
+		  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
+			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
+			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
+			  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
+		  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
+			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
+			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
+			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
+			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
+			  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
+			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
+			  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19"
+			  "\x67\x6d\xb1\xf5\xb8\x10\xdc\xc6"
+			  "\x75\x86\x96\x6b\xb1\xc5\xe4\xcf"
+			  "\xd1\x60\x91\xb3",
+			  .rlen	= 128 + 20,
+	},
+};
+
+static const struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = {
+	{ /*Generated with cryptopp*/
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x18"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
+			  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
+			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
+		.klen	= 8 + 24 + 24,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.alen   = 16,
+		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+		  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.ilen	= 128,
+		.result	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
+		  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
+			  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
+			  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
+			  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
+			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
+			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
+		  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
+			  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
+			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
+			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
+			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
+			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
+		  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
+			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
+		  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19"
+			  "\x15\x24\x7f\x5a\x45\x4a\x66\xce"
+			  "\x2b\x0b\x93\x99\x2f\x9d\x0c\x6c"
+			  "\x56\x1f\xe1\xa6\x41\xb2\x4c\xd0",
+			  .rlen	= 128 + 24,
+	},
+};
+
+static const struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = {
+	{ /*Generated with cryptopp*/
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x18"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
+			  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
+			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
+		.klen	= 8 + 32 + 24,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.alen   = 16,
+		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.ilen	= 128,
+		.result	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
+			  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
+			  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
+			  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
+			  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
+			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
+			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
+			  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
+			  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
+			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
+			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
+			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
+			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
+			  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
+			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
+			  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19"
+			  "\x73\xb0\xea\x9f\xe8\x18\x80\xd6"
+			  "\x56\x38\x44\xc0\xdb\xe3\x4f\x71"
+			  "\xf7\xce\xd1\xd3\xf8\xbd\x3e\x4f"
+			  "\xca\x43\x95\xdf\x80\x61\x81\xa9",
+		.rlen	= 128 + 32,
+	},
+};
+
+static const struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = {
+	{ /*Generated with cryptopp*/
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x18"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
+			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
+			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
+			  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
+			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
+		.klen	= 8 + 48 + 24,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.alen   = 16,
+		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.ilen	= 128,
+		.result	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
+			  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
+			  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
+			  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
+			  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
+			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
+			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
+			  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
+			  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
+			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
+			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
+			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
+			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
+			  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
+			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
+			  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19"
+			  "\x6d\x77\xfc\x80\x9d\x8a\x9c\xb7"
+		  "\x70\xe7\x93\xbf\x73\xe6\x9f\x83"
+			  "\x99\x62\x23\xe6\x5b\xd0\xda\x18"
+			  "\xa4\x32\x8a\x0b\x46\xd7\xf0\x39"
+			  "\x36\x5d\x13\x2f\x86\x10\x78\xd6"
+			  "\xd6\xbe\x5c\xb9\x15\x89\xf9\x1b",
+		.rlen	= 128 + 48,
+	},
+};
+
+static const struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = {
+	{ /*Generated with cryptopp*/
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x18"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
+			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
+			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
+			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
+			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24"
+		  "\x44\x4D\x99\x5A\x12\xD6\x40\xC0"
+			  "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8",
+		.klen	= 8 + 64 + 24,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.alen   = 16,
+		.input	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+		  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.ilen	= 128,
+		.result	= "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4"
+			  "\x67\x17\x21\xc7\x6e\x8a\xd5\x49"
+			  "\x74\xb3\x49\x05\xc5\x1c\xd0\xed"
+			  "\x12\x56\x5c\x53\x96\xb6\x00\x7d"
+			  "\x90\x48\xfc\xf5\x8d\x29\x39\xcc"
+			  "\x8a\xd5\x35\x18\x36\x23\x4e\xd7"
+			  "\x76\xd1\xda\x0c\x94\x67\xbb\x04"
+			  "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea"
+			  "\x22\x64\x47\xaa\x8f\x75\x13\xbf"
+			  "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a"
+			  "\x71\x63\x2e\x89\x7b\x1e\x12\xca"
+			  "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a"
+			  "\xd6\xf9\x21\x31\x62\x44\x45\xa6"
+			  "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc"
+			  "\x9d\xde\xa5\x70\xe9\x42\x45\x8a"
+			  "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19"
+			  "\x41\xb5\x1f\xbb\xbd\x4e\xb8\x32"
+			  "\x22\x86\x4e\x57\x1b\x2a\xd8\x6e"
+			  "\xa9\xfb\xc8\xf3\xbf\x2d\xae\x2b"
+			  "\x3b\xbc\x41\xe8\x38\xbb\xf1\x60"
+			  "\x4c\x68\xa9\x4e\x8c\x73\xa7\xc0"
+			  "\x2a\x74\xd4\x65\x12\xcb\x55\xf2"
+			  "\xd5\x02\x6d\xe6\xaf\xc9\x2f\xf2"
+			  "\x57\xaa\x85\xf7\xf3\x6a\xcb\xdb",
+		.rlen	= 128 + 64,
+	},
+};
+
+static const struct cipher_testvec aes_lrw_tv_template[] = {
+	/* from http://grouper.ieee.org/groups/1619/email/pdf00017.pdf */
+	{ /* LRW-32-AES 1 */
+		.key    = "\x45\x62\xac\x25\xf8\x28\x17\x6d"
+			  "\x4c\x26\x84\x14\xb5\x68\x01\x85"
+			  "\x25\x8e\x2a\x05\xe7\x3e\x9d\x03"
+			  "\xee\x5a\x83\x0c\xcc\x09\x4c\x87",
+		.klen   = 32,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\xf1\xb2\x73\xcd\x65\xa3\xdf\x5f"
+			  "\xe9\x5d\x48\x92\x54\x63\x4e\xb8",
+		.len	= 16,
+	}, { /* LRW-32-AES 2 */
+		.key    = "\x59\x70\x47\x14\xf5\x57\x47\x8c"
+			  "\xd7\x79\xe8\x0f\x54\x88\x79\x44"
+			  "\x0d\x48\xf0\xb7\xb1\x5a\x53\xea"
+			  "\x1c\xaa\x6b\x29\xc2\xca\xfb\xaf",
+		.klen   = 32,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x02",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x00\xc8\x2b\xae\x95\xbb\xcd\xe5"
+			  "\x27\x4f\x07\x69\xb2\x60\xe1\x36",
+		.len	= 16,
+	}, { /* LRW-32-AES 3 */
+		.key    = "\xd8\x2a\x91\x34\xb2\x6a\x56\x50"
+			  "\x30\xfe\x69\xe2\x37\x7f\x98\x47"
+			  "\xcd\xf9\x0b\x16\x0c\x64\x8f\xb6"
+			  "\xb0\x0d\x0d\x1b\xae\x85\x87\x1f",
+		.klen   = 32,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x02\x00\x00\x00\x00",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x76\x32\x21\x83\xed\x8f\xf1\x82"
+			  "\xf9\x59\x62\x03\x69\x0e\x5e\x01",
+		.len	= 16,
+	}, { /* LRW-32-AES 4 */
+		.key    = "\x0f\x6a\xef\xf8\xd3\xd2\xbb\x15"
+			  "\x25\x83\xf7\x3c\x1f\x01\x28\x74"
+			  "\xca\xc6\xbc\x35\x4d\x4a\x65\x54"
+			  "\x90\xae\x61\xcf\x7b\xae\xbd\xcc"
+			  "\xad\xe4\x94\xc5\x4a\x29\xae\x70",
+		.klen   = 40,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x9c\x0f\x15\x2f\x55\xa2\xd8\xf0"
+			  "\xd6\x7b\x8f\x9e\x28\x22\xbc\x41",
+		.len	= 16,
+	}, { /* LRW-32-AES 5 */
+		.key    = "\x8a\xd4\xee\x10\x2f\xbd\x81\xff"
+			  "\xf8\x86\xce\xac\x93\xc5\xad\xc6"
+			  "\xa0\x19\x07\xc0\x9d\xf7\xbb\xdd"
+			  "\x52\x13\xb2\xb7\xf0\xff\x11\xd8"
+			  "\xd6\x08\xd0\xcd\x2e\xb1\x17\x6f",
+		.klen   = 40,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x02\x00\x00\x00\x00",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\xd4\x27\x6a\x7f\x14\x91\x3d\x65"
+			  "\xc8\x60\x48\x02\x87\xe3\x34\x06",
+		.len	= 16,
+	}, { /* LRW-32-AES 6 */
+		.key    = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
+			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
+			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
+			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
+			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
+			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
+		.klen   = 48,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\xbd\x06\xb8\xe1\xdb\x98\x89\x9e"
+			  "\xc4\x98\xe4\x91\xcf\x1c\x70\x2b",
+		.len	= 16,
+	}, { /* LRW-32-AES 7 */
+		.key    = "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
+			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
+			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
+			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
+			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
+			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
+		.klen   = 48,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x02\x00\x00\x00\x00",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x5b\x90\x8e\xc1\xab\xdd\x67\x5f"
+			  "\x3d\x69\x8a\x95\x53\xc8\x9c\xe5",
+		.len	= 16,
+	}, {
+/* http://www.mail-archive.com/stds-p1619@listserv.ieee.org/msg00173.html */
+		.key    = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
+			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
+			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
+			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
+			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
+			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
+		.klen   = 48,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
+			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
+			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
+			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
+			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
+			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
+			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
+			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
+			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
+			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
+			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
+			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
+			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
+			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
+			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
+			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
+			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
+			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
+			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
+			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
+			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
+			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
+			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
+			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
+			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
+			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
+			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
+			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
+			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
+			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
+			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
+			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
+			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
+			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
+			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
+			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
+			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
+			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
+			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
+			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
+			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
+			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
+			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
+			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
+			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
+			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
+			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
+			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
+			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
+			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
+			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
+			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
+			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
+			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
+			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
+			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
+			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
+			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
+			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
+			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
+			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
+			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
+			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
+			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
+		.ctext	= "\x1a\x1d\xa9\x30\xad\xf9\x2f\x9b"
+			  "\xb6\x1d\xae\xef\xf0\x2f\xf8\x5a"
+			  "\x39\x3c\xbf\x2a\xb2\x45\xb2\x23"
+			  "\x1b\x63\x3c\xcf\xaa\xbe\xcf\x4e"
+			  "\xfa\xe8\x29\xc2\x20\x68\x2b\x3c"
+			  "\x2e\x8b\xf7\x6e\x25\xbd\xe3\x3d"
+			  "\x66\x27\xd6\xaf\xd6\x64\x3e\xe3"
+			  "\xe8\x58\x46\x97\x39\x51\x07\xde"
+			  "\xcb\x37\xbc\xa9\xc0\x5f\x75\xc3"
+			  "\x0e\x84\x23\x1d\x16\xd4\x1c\x59"
+			  "\x9c\x1a\x02\x55\xab\x3a\x97\x1d"
+			  "\xdf\xdd\xc7\x06\x51\xd7\x70\xae"
+			  "\x23\xc6\x8c\xf5\x1e\xa0\xe5\x82"
+			  "\xb8\xb2\xbf\x04\xa0\x32\x8e\x68"
+			  "\xeb\xaf\x6e\x2d\x94\x22\x2f\xce"
+			  "\x4c\xb5\x59\xe2\xa2\x2f\xa0\x98"
+			  "\x1a\x97\xc6\xd4\xb5\x00\x59\xf2"
+			  "\x84\x14\x72\xb1\x9a\x6e\xa3\x7f"
+			  "\xea\x20\xe7\xcb\x65\x77\x3a\xdf"
+			  "\xc8\x97\x67\x15\xc2\x2a\x27\xcc"
+			  "\x18\x55\xa1\x24\x0b\x24\x24\xaf"
+			  "\x5b\xec\x68\xb8\xc8\xf5\xba\x63"
+			  "\xff\xed\x89\xce\xd5\x3d\x88\xf3"
+			  "\x25\xef\x05\x7c\x3a\xef\xeb\xd8"
+			  "\x7a\x32\x0d\xd1\x1e\x58\x59\x99"
+			  "\x90\x25\xb5\x26\xb0\xe3\x2b\x6c"
+			  "\x4c\xa9\x8b\x84\x4f\x5e\x01\x50"
+			  "\x41\x30\x58\xc5\x62\x74\x52\x1d"
+			  "\x45\x24\x6a\x42\x64\x4f\x97\x1c"
+			  "\xa8\x66\xb5\x6d\x79\xd4\x0d\x48"
+			  "\xc5\x5f\xf3\x90\x32\xdd\xdd\xe1"
+			  "\xe4\xa9\x9f\xfc\xc3\x52\x5a\x46"
+			  "\xe4\x81\x84\x95\x36\x59\x7a\x6b"
+			  "\xaa\xb3\x60\xad\xce\x9f\x9f\x28"
+			  "\xe0\x01\x75\x22\xc4\x4e\xa9\x62"
+			  "\x5c\x62\x0d\x00\xcb\x13\xe8\x43"
+			  "\x72\xd4\x2d\x53\x46\xb5\xd1\x16"
+			  "\x22\x18\xdf\x34\x33\xf5\xd6\x1c"
+			  "\xb8\x79\x78\x97\x94\xff\x72\x13"
+			  "\x4c\x27\xfc\xcb\xbf\x01\x53\xa6"
+			  "\xb4\x50\x6e\xde\xdf\xb5\x43\xa4"
+			  "\x59\xdf\x52\xf9\x7c\xe0\x11\x6f"
+			  "\x2d\x14\x8e\x24\x61\x2c\xe1\x17"
+			  "\xcc\xce\x51\x0c\x19\x8a\x82\x30"
+			  "\x94\xd5\x3d\x6a\x53\x06\x5e\xbd"
+			  "\xb7\xeb\xfa\xfd\x27\x51\xde\x85"
+			  "\x1e\x86\x53\x11\x53\x94\x00\xee"
+			  "\x2b\x8c\x08\x2a\xbf\xdd\xae\x11"
+			  "\xcb\x1e\xa2\x07\x9a\x80\xcf\x62"
+			  "\x9b\x09\xdc\x95\x3c\x96\x8e\xb1"
+			  "\x09\xbd\xe4\xeb\xdb\xca\x70\x7a"
+			  "\x9e\xfa\x31\x18\x45\x3c\x21\x33"
+			  "\xb0\xb3\x2b\xea\xf3\x71\x2d\xe1"
+			  "\x03\xad\x1b\x48\xd4\x67\x27\xf0"
+			  "\x62\xe4\x3d\xfb\x9b\x08\x76\xe7"
+			  "\xdd\x2b\x01\x39\x04\x5a\x58\x7a"
+			  "\xf7\x11\x90\xec\xbd\x51\x5c\x32"
+			  "\x6b\xd7\x35\x39\x02\x6b\xf2\xa6"
+			  "\xd0\x0d\x07\xe1\x06\xc4\x5b\x7d"
+			  "\xe4\x6a\xd7\xee\x15\x1f\x83\xb4"
+			  "\xa3\xa7\x5e\xc3\x90\xb7\xef\xd3"
+			  "\xb7\x4f\xf8\x92\x4c\xb7\x3c\x29"
+			  "\xcd\x7e\x2b\x5d\x43\xea\x42\xe7"
+			  "\x74\x3f\x7d\x58\x88\x75\xde\x3e",
+		.len	= 512,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 512 - 20, 4, 16 },
+	}
+};
+
+static const struct cipher_testvec aes_xts_tv_template[] = {
+	/* http://grouper.ieee.org/groups/1619/email/pdf00086.pdf */
+	{ /* XTS-AES 1 */
+		.key    = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen   = 32,
+		.fips_skip = 1,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\x91\x7c\xf6\x9e\xbd\x68\xb2\xec"
+			  "\x9b\x9f\xe9\xa3\xea\xdd\xa6\x92"
+			  "\xcd\x43\xd2\xf5\x95\x98\xed\x85"
+			  "\x8c\x02\xc2\x65\x2f\xbf\x92\x2e",
+		.len	= 32,
+	}, { /* XTS-AES 2 */
+		.key    = "\x11\x11\x11\x11\x11\x11\x11\x11"
+			  "\x11\x11\x11\x11\x11\x11\x11\x11"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22",
+		.klen   = 32,
+		.iv     = "\x33\x33\x33\x33\x33\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44",
+		.ctext	= "\xc4\x54\x18\x5e\x6a\x16\x93\x6e"
+			  "\x39\x33\x40\x38\xac\xef\x83\x8b"
+			  "\xfb\x18\x6f\xff\x74\x80\xad\xc4"
+			  "\x28\x93\x82\xec\xd6\xd3\x94\xf0",
+		.len	= 32,
+	}, { /* XTS-AES 3 */
+		.key    = "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
+			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22",
+		.klen   = 32,
+		.iv     = "\x33\x33\x33\x33\x33\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44",
+		.ctext	= "\xaf\x85\x33\x6b\x59\x7a\xfc\x1a"
+			  "\x90\x0b\x2e\xb2\x1e\xc9\x49\xd2"
+			  "\x92\xdf\x4c\x04\x7e\x0b\x21\x53"
+			  "\x21\x86\xa5\x97\x1a\x22\x7a\x89",
+		.len	= 32,
+	}, { /* XTS-AES 4 */
+		.key    = "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95",
+		.klen   = 32,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ctext	= "\x27\xa7\x47\x9b\xef\xa1\xd4\x76"
+			  "\x48\x9f\x30\x8c\xd4\xcf\xa6\xe2"
+			  "\xa9\x6e\x4b\xbe\x32\x08\xff\x25"
+			  "\x28\x7d\xd3\x81\x96\x16\xe8\x9c"
+			  "\xc7\x8c\xf7\xf5\xe5\x43\x44\x5f"
+			  "\x83\x33\xd8\xfa\x7f\x56\x00\x00"
+			  "\x05\x27\x9f\xa5\xd8\xb5\xe4\xad"
+			  "\x40\xe7\x36\xdd\xb4\xd3\x54\x12"
+			  "\x32\x80\x63\xfd\x2a\xab\x53\xe5"
+			  "\xea\x1e\x0a\x9f\x33\x25\x00\xa5"
+			  "\xdf\x94\x87\xd0\x7a\x5c\x92\xcc"
+			  "\x51\x2c\x88\x66\xc7\xe8\x60\xce"
+			  "\x93\xfd\xf1\x66\xa2\x49\x12\xb4"
+			  "\x22\x97\x61\x46\xae\x20\xce\x84"
+			  "\x6b\xb7\xdc\x9b\xa9\x4a\x76\x7a"
+			  "\xae\xf2\x0c\x0d\x61\xad\x02\x65"
+			  "\x5e\xa9\x2d\xc4\xc4\xe4\x1a\x89"
+			  "\x52\xc6\x51\xd3\x31\x74\xbe\x51"
+			  "\xa1\x0c\x42\x11\x10\xe6\xd8\x15"
+			  "\x88\xed\xe8\x21\x03\xa2\x52\xd8"
+			  "\xa7\x50\xe8\x76\x8d\xef\xff\xed"
+			  "\x91\x22\x81\x0a\xae\xb9\x9f\x91"
+			  "\x72\xaf\x82\xb6\x04\xdc\x4b\x8e"
+			  "\x51\xbc\xb0\x82\x35\xa6\xf4\x34"
+			  "\x13\x32\xe4\xca\x60\x48\x2a\x4b"
+			  "\xa1\xa0\x3b\x3e\x65\x00\x8f\xc5"
+			  "\xda\x76\xb7\x0b\xf1\x69\x0d\xb4"
+			  "\xea\xe2\x9c\x5f\x1b\xad\xd0\x3c"
+			  "\x5c\xcf\x2a\x55\xd7\x05\xdd\xcd"
+			  "\x86\xd4\x49\x51\x1c\xeb\x7e\xc3"
+			  "\x0b\xf1\x2b\x1f\xa3\x5b\x91\x3f"
+			  "\x9f\x74\x7a\x8a\xfd\x1b\x13\x0e"
+			  "\x94\xbf\xf9\x4e\xff\xd0\x1a\x91"
+			  "\x73\x5c\xa1\x72\x6a\xcd\x0b\x19"
+			  "\x7c\x4e\x5b\x03\x39\x36\x97\xe1"
+			  "\x26\x82\x6f\xb6\xbb\xde\x8e\xcc"
+			  "\x1e\x08\x29\x85\x16\xe2\xc9\xed"
+			  "\x03\xff\x3c\x1b\x78\x60\xf6\xde"
+			  "\x76\xd4\xce\xcd\x94\xc8\x11\x98"
+			  "\x55\xef\x52\x97\xca\x67\xe9\xf3"
+			  "\xe7\xff\x72\xb1\xe9\x97\x85\xca"
+			  "\x0a\x7e\x77\x20\xc5\xb3\x6d\xc6"
+			  "\xd7\x2c\xac\x95\x74\xc8\xcb\xbc"
+			  "\x2f\x80\x1e\x23\xe5\x6f\xd3\x44"
+			  "\xb0\x7f\x22\x15\x4b\xeb\xa0\xf0"
+			  "\x8c\xe8\x89\x1e\x64\x3e\xd9\x95"
+			  "\xc9\x4d\x9a\x69\xc9\xf1\xb5\xf4"
+			  "\x99\x02\x7a\x78\x57\x2a\xee\xbd"
+			  "\x74\xd2\x0c\xc3\x98\x81\xc2\x13"
+			  "\xee\x77\x0b\x10\x10\xe4\xbe\xa7"
+			  "\x18\x84\x69\x77\xae\x11\x9f\x7a"
+			  "\x02\x3a\xb5\x8c\xca\x0a\xd7\x52"
+			  "\xaf\xe6\x56\xbb\x3c\x17\x25\x6a"
+			  "\x9f\x6e\x9b\xf1\x9f\xdd\x5a\x38"
+			  "\xfc\x82\xbb\xe8\x72\xc5\x53\x9e"
+			  "\xdb\x60\x9e\xf4\xf7\x9c\x20\x3e"
+			  "\xbb\x14\x0f\x2e\x58\x3c\xb2\xad"
+			  "\x15\xb4\xaa\x5b\x65\x50\x16\xa8"
+			  "\x44\x92\x77\xdb\xd4\x77\xef\x2c"
+			  "\x8d\x6c\x01\x7d\xb7\x38\xb1\x8d"
+			  "\xeb\x4a\x42\x7d\x19\x23\xce\x3f"
+			  "\xf2\x62\x73\x57\x79\xa4\x18\xf2"
+			  "\x0a\x28\x2d\xf9\x20\x14\x7b\xea"
+			  "\xbe\x42\x1e\xe5\x31\x9d\x05\x68",
+		.len	= 512,
+	}, { /* XTS-AES 10, XTS-AES-256, data unit 512 bytes */
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x62\x49\x77\x57\x24\x70\x93\x69"
+			  "\x99\x59\x57\x49\x66\x96\x76\x27"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95"
+			  "\x02\x88\x41\x97\x16\x93\x99\x37"
+			  "\x51\x05\x82\x09\x74\x94\x45\x92",
+		.klen	= 64,
+		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ctext	= "\x1c\x3b\x3a\x10\x2f\x77\x03\x86"
+			  "\xe4\x83\x6c\x99\xe3\x70\xcf\x9b"
+			  "\xea\x00\x80\x3f\x5e\x48\x23\x57"
+			  "\xa4\xae\x12\xd4\x14\xa3\xe6\x3b"
+			  "\x5d\x31\xe2\x76\xf8\xfe\x4a\x8d"
+			  "\x66\xb3\x17\xf9\xac\x68\x3f\x44"
+			  "\x68\x0a\x86\xac\x35\xad\xfc\x33"
+			  "\x45\xbe\xfe\xcb\x4b\xb1\x88\xfd"
+			  "\x57\x76\x92\x6c\x49\xa3\x09\x5e"
+			  "\xb1\x08\xfd\x10\x98\xba\xec\x70"
+			  "\xaa\xa6\x69\x99\xa7\x2a\x82\xf2"
+			  "\x7d\x84\x8b\x21\xd4\xa7\x41\xb0"
+			  "\xc5\xcd\x4d\x5f\xff\x9d\xac\x89"
+			  "\xae\xba\x12\x29\x61\xd0\x3a\x75"
+			  "\x71\x23\xe9\x87\x0f\x8a\xcf\x10"
+			  "\x00\x02\x08\x87\x89\x14\x29\xca"
+			  "\x2a\x3e\x7a\x7d\x7d\xf7\xb1\x03"
+			  "\x55\x16\x5c\x8b\x9a\x6d\x0a\x7d"
+			  "\xe8\xb0\x62\xc4\x50\x0d\xc4\xcd"
+			  "\x12\x0c\x0f\x74\x18\xda\xe3\xd0"
+			  "\xb5\x78\x1c\x34\x80\x3f\xa7\x54"
+			  "\x21\xc7\x90\xdf\xe1\xde\x18\x34"
+			  "\xf2\x80\xd7\x66\x7b\x32\x7f\x6c"
+			  "\x8c\xd7\x55\x7e\x12\xac\x3a\x0f"
+			  "\x93\xec\x05\xc5\x2e\x04\x93\xef"
+			  "\x31\xa1\x2d\x3d\x92\x60\xf7\x9a"
+			  "\x28\x9d\x6a\x37\x9b\xc7\x0c\x50"
+			  "\x84\x14\x73\xd1\xa8\xcc\x81\xec"
+			  "\x58\x3e\x96\x45\xe0\x7b\x8d\x96"
+			  "\x70\x65\x5b\xa5\xbb\xcf\xec\xc6"
+			  "\xdc\x39\x66\x38\x0a\xd8\xfe\xcb"
+			  "\x17\xb6\xba\x02\x46\x9a\x02\x0a"
+			  "\x84\xe1\x8e\x8f\x84\x25\x20\x70"
+			  "\xc1\x3e\x9f\x1f\x28\x9b\xe5\x4f"
+			  "\xbc\x48\x14\x57\x77\x8f\x61\x60"
+			  "\x15\xe1\x32\x7a\x02\xb1\x40\xf1"
+			  "\x50\x5e\xb3\x09\x32\x6d\x68\x37"
+			  "\x8f\x83\x74\x59\x5c\x84\x9d\x84"
+			  "\xf4\xc3\x33\xec\x44\x23\x88\x51"
+			  "\x43\xcb\x47\xbd\x71\xc5\xed\xae"
+			  "\x9b\xe6\x9a\x2f\xfe\xce\xb1\xbe"
+			  "\xc9\xde\x24\x4f\xbe\x15\x99\x2b"
+			  "\x11\xb7\x7c\x04\x0f\x12\xbd\x8f"
+			  "\x6a\x97\x5a\x44\xa0\xf9\x0c\x29"
+			  "\xa9\xab\xc3\xd4\xd8\x93\x92\x72"
+			  "\x84\xc5\x87\x54\xcc\xe2\x94\x52"
+			  "\x9f\x86\x14\xdc\xd2\xab\xa9\x91"
+			  "\x92\x5f\xed\xc4\xae\x74\xff\xac"
+			  "\x6e\x33\x3b\x93\xeb\x4a\xff\x04"
+			  "\x79\xda\x9a\x41\x0e\x44\x50\xe0"
+			  "\xdd\x7a\xe4\xc6\xe2\x91\x09\x00"
+			  "\x57\x5d\xa4\x01\xfc\x07\x05\x9f"
+			  "\x64\x5e\x8b\x7e\x9b\xfd\xef\x33"
+			  "\x94\x30\x54\xff\x84\x01\x14\x93"
+			  "\xc2\x7b\x34\x29\xea\xed\xb4\xed"
+			  "\x53\x76\x44\x1a\x77\xed\x43\x85"
+			  "\x1a\xd7\x7f\x16\xf5\x41\xdf\xd2"
+			  "\x69\xd5\x0d\x6a\x5f\x14\xfb\x0a"
+			  "\xab\x1c\xbb\x4c\x15\x50\xbe\x97"
+			  "\xf7\xab\x40\x66\x19\x3c\x4c\xaa"
+			  "\x77\x3d\xad\x38\x01\x4b\xd2\x09"
+			  "\x2f\xa7\x55\xc8\x24\xbb\x5e\x54"
+			  "\xc4\xf3\x6f\xfd\xa9\xfc\xea\x70"
+			  "\xb9\xc6\xe6\x93\xe1\x48\xc1\x51",
+		.len	= 512,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 512 - 20, 4, 16 },
+	}
+};
+
+static const struct cipher_testvec aes_ctr_tv_template[] = {
+	{ /* From NIST Special Publication 800-38A, Appendix F.5 */
+		.key	= "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
+			  "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
+		.klen	= 16,
+		.iv	= "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ptext	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ctext	= "\x87\x4d\x61\x91\xb6\x20\xe3\x26"
+			  "\x1b\xef\x68\x64\x99\x0d\xb6\xce"
+			  "\x98\x06\xf6\x6b\x79\x70\xfd\xff"
+			  "\x86\x17\x18\x7b\xb9\xff\xfd\xff"
+			  "\x5a\xe4\xdf\x3e\xdb\xd5\xd3\x5e"
+			  "\x5b\x4f\x09\x02\x0d\xb0\x3e\xab"
+			  "\x1e\x03\x1d\xda\x2f\xbe\x03\xd1"
+			  "\x79\x21\x70\xa0\xf3\x00\x9c\xee",
+		.len	= 64,
+	}, {
+		.key	= "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
+			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
+			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
+		.klen	= 24,
+		.iv	= "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ptext	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ctext	= "\x1a\xbc\x93\x24\x17\x52\x1c\xa2"
+			  "\x4f\x2b\x04\x59\xfe\x7e\x6e\x0b"
+			  "\x09\x03\x39\xec\x0a\xa6\xfa\xef"
+			  "\xd5\xcc\xc2\xc6\xf4\xce\x8e\x94"
+			  "\x1e\x36\xb2\x6b\xd1\xeb\xc6\x70"
+			  "\xd1\xbd\x1d\x66\x56\x20\xab\xf7"
+			  "\x4f\x78\xa7\xf6\xd2\x98\x09\x58"
+			  "\x5a\x97\xda\xec\x58\xc6\xb0\x50",
+		.len	= 64,
+	}, {
+		.key	= "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
+			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
+			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
+			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
+		.klen	= 32,
+		.iv	= "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ptext	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ctext	= "\x60\x1e\xc3\x13\x77\x57\x89\xa5"
+			  "\xb7\xa7\xf5\x04\xbb\xf3\xd2\x28"
+			  "\xf4\x43\xe3\xca\x4d\x62\xb5\x9a"
+			  "\xca\x84\xe9\x90\xca\xca\xf5\xc5"
+			  "\x2b\x09\x30\xda\xa2\x3d\xe9\x4c"
+			  "\xe8\x70\x17\xba\x2d\x84\x98\x8d"
+			  "\xdf\xc9\xc5\x8d\xb6\x7a\xad\xa6"
+			  "\x13\xc2\xdd\x08\x45\x79\x41\xa6",
+		.len	= 64,
+	}, { /* Generated with Crypto++ */
+		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55"
+			  "\x0F\x32\x55\x78\x9B\xBE\x78\x9B"
+			  "\xBE\xE1\x04\x27\xE1\x04\x27\x4A"
+			  "\x6D\x90\x4A\x6D\x90\xB3\xD6\xF9",
+		.klen	= 32,
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.ptext	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
+			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
+			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
+			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
+			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
+			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
+			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
+			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
+			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
+			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
+			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
+			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
+			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
+			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
+			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
+			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
+			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
+			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
+			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
+			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
+			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
+			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
+			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
+			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
+			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
+			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
+			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
+			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
+			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
+			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
+			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
+			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
+			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
+			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
+			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
+			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
+			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
+			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
+			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
+			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
+			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
+			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
+			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
+			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
+			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
+			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
+			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
+			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
+			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
+			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
+			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
+			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
+			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
+			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
+			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
+			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
+			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
+			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
+			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
+			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
+			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
+			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12",
+		.ctext	= "\x04\xF3\xD3\x88\x17\xEF\xDC\xEF"
+			  "\x8B\x04\xF8\x3A\x66\x8D\x1A\x53"
+			  "\x57\x1F\x4B\x23\xE4\xA0\xAF\xF9"
+			  "\x69\x95\x35\x98\x8D\x4D\x8C\xC1"
+			  "\xF0\xB2\x7F\x80\xBB\x54\x28\xA2"
+			  "\x7A\x1B\x9F\x77\xEC\x0E\x6E\xDE"
+			  "\xF0\xEC\xB8\xE4\x20\x62\xEE\xDB"
+			  "\x5D\xF5\xDD\xE3\x54\xFC\xDD\xEB"
+			  "\x6A\xEE\x65\xA1\x21\xD6\xD7\x81"
+			  "\x47\x61\x12\x4D\xC2\x8C\xFA\x78"
+			  "\x1F\x28\x02\x01\xC3\xFC\x1F\xEC"
+			  "\x0F\x10\x4F\xB3\x12\x45\xC6\x3B"
+			  "\x7E\x08\xF9\x5A\xD0\x5D\x73\x2D"
+			  "\x58\xA4\xE5\xCB\x1C\xB4\xCE\x74"
+			  "\x32\x41\x1F\x31\x9C\x08\xA2\x5D"
+			  "\x67\xEB\x72\x1D\xF8\xE7\x70\x54"
+			  "\x34\x4B\x31\x69\x84\x66\x96\x44"
+			  "\x56\xCC\x1E\xD9\xE6\x13\x6A\xB9"
+			  "\x2D\x0A\x05\x45\x2D\x90\xCC\xDF"
+			  "\x16\x5C\x5F\x79\x34\x52\x54\xFE"
+			  "\xFE\xCD\xAD\x04\x2E\xAD\x86\x06"
+			  "\x1F\x37\xE8\x28\xBC\xD3\x8F\x5B"
+			  "\x92\x66\x87\x3B\x8A\x0A\x1A\xCC"
+			  "\x6E\xAB\x9F\x0B\xFA\x5C\xE6\xFD"
+			  "\x3C\x98\x08\x12\xEC\xAA\x9E\x11"
+			  "\xCA\xB2\x1F\xCE\x5E\x5B\xB2\x72"
+			  "\x9C\xCC\x5D\xC5\xE0\x32\xC0\x56"
+			  "\xD5\x45\x16\xD2\xAF\x13\x66\xF7"
+			  "\x8C\x67\xAC\x79\xB2\xAF\x56\x27"
+			  "\x3F\xCC\xFE\xCB\x1E\xC0\x75\xF1"
+			  "\xA7\xC9\xC3\x1D\x8E\xDD\xF9\xD4"
+			  "\x42\xC8\x21\x08\x16\xF7\x01\xD7"
+			  "\xAC\x8E\x3F\x1D\x56\xC1\x06\xE4"
+			  "\x9C\x62\xD6\xA5\x6A\x50\x44\xB3"
+			  "\x35\x1C\x82\xB9\x10\xF9\x42\xA1"
+			  "\xFC\x74\x9B\x44\x4F\x25\x02\xE3"
+			  "\x08\xF5\xD4\x32\x39\x08\x11\xE8"
+			  "\xD2\x6B\x50\x53\xD4\x08\xD1\x6B"
+			  "\x3A\x4A\x68\x7B\x7C\xCD\x46\x5E"
+			  "\x0D\x07\x19\xDB\x67\xD7\x98\x91"
+			  "\xD7\x17\x10\x9B\x7B\x8A\x9B\x33"
+			  "\xAE\xF3\x00\xA6\xD4\x15\xD9\xEA"
+			  "\x85\x99\x22\xE8\x91\x38\x70\x83"
+			  "\x93\x01\x24\x6C\xFA\x9A\xB9\x07"
+			  "\xEA\x8D\x3B\xD9\x2A\x43\x59\x16"
+			  "\x2F\x69\xEE\x84\x36\x44\x76\x98"
+			  "\xF3\x04\x2A\x7C\x74\x3D\x29\x2B"
+			  "\x0D\xAD\x8F\x44\x82\x9E\x57\x8D"
+			  "\xAC\xED\x18\x1F\x50\xA4\xF5\x98"
+			  "\x1F\xBD\x92\x91\x1B\x2D\xA6\xD6"
+			  "\xD2\xE3\x02\xAA\x92\x3B\xC6\xB3"
+			  "\x1B\x39\x72\xD5\x26\xCA\x04\xE0"
+			  "\xFC\x58\x78\xBB\xB1\x3F\xA1\x9C"
+			  "\x42\x24\x3E\x2E\x22\xBB\x4B\xBA"
+			  "\xF4\x52\x0A\xE6\xAE\x47\xB4\x7D"
+			  "\x1D\xA8\xBE\x81\x1A\x75\xDA\xAC"
+			  "\xA6\x25\x1E\xEF\x3A\xC0\x6C\x63"
+			  "\xEF\xDC\xC9\x79\x10\x26\xE8\x61"
+			  "\x29\xFC\xA4\x05\xDF\x7D\x5C\x63"
+			  "\x10\x09\x9B\x46\x9B\xF2\x2C\x2B"
+			  "\xFA\x3A\x05\x4C\xFA\xD1\xFF\xFE"
+			  "\xF1\x4C\xE5\xB2\x91\x64\x0C\x51",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	}, { /* Generated with Crypto++ */
+		.key	= "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55"
+			  "\x0F\x32\x55\x78\x9B\xBE\x78\x9B"
+			  "\xBE\xE1\x04\x27\xE1\x04\x27\x4A"
+			  "\x6D\x90\x4A\x6D\x90\xB3\xD6\xF9",
+		.klen	= 32,
+		.iv	= "\xE7\x82\x1D\xB8\x53\x11\xAC\x47"
+			  "\xE2\x7D\x18\xD6\x71\x0C\xA7\x42",
+		.ptext	= "\x50\xB9\x22\xAE\x17\x80\x0C\x75"
+			  "\xDE\x47\xD3\x3C\xA5\x0E\x9A\x03"
+			  "\x6C\xF8\x61\xCA\x33\xBF\x28\x91"
+			  "\x1D\x86\xEF\x58\xE4\x4D\xB6\x1F"
+			  "\xAB\x14\x7D\x09\x72\xDB\x44\xD0"
+			  "\x39\xA2\x0B\x97\x00\x69\xF5\x5E"
+			  "\xC7\x30\xBC\x25\x8E\x1A\x83\xEC"
+			  "\x55\xE1\x4A\xB3\x1C\xA8\x11\x7A"
+			  "\x06\x6F\xD8\x41\xCD\x36\x9F\x08"
+			  "\x94\xFD\x66\xF2\x5B\xC4\x2D\xB9"
+			  "\x22\x8B\x17\x80\xE9\x52\xDE\x47"
+			  "\xB0\x19\xA5\x0E\x77\x03\x6C\xD5"
+			  "\x3E\xCA\x33\x9C\x05\x91\xFA\x63"
+			  "\xEF\x58\xC1\x2A\xB6\x1F\x88\x14"
+			  "\x7D\xE6\x4F\xDB\x44\xAD\x16\xA2"
+			  "\x0B\x74\x00\x69\xD2\x3B\xC7\x30"
+			  "\x99\x02\x8E\xF7\x60\xEC\x55\xBE"
+			  "\x27\xB3\x1C\x85\x11\x7A\xE3\x4C"
+			  "\xD8\x41\xAA\x13\x9F\x08\x71\xFD"
+			  "\x66\xCF\x38\xC4\x2D\x96\x22\x8B"
+			  "\xF4\x5D\xE9\x52\xBB\x24\xB0\x19"
+			  "\x82\x0E\x77\xE0\x49\xD5\x3E\xA7"
+			  "\x10\x9C\x05\x6E\xFA\x63\xCC\x35"
+			  "\xC1\x2A\x93\x1F\x88\xF1\x5A\xE6"
+			  "\x4F\xB8\x21\xAD\x16\x7F\x0B\x74"
+			  "\xDD\x46\xD2\x3B\xA4\x0D\x99\x02"
+			  "\x6B\xF7\x60\xC9\x32\xBE\x27\x90"
+			  "\x1C\x85\xEE\x57\xE3\x4C\xB5\x1E"
+			  "\xAA\x13\x7C\x08\x71\xDA\x43\xCF"
+			  "\x38\xA1\x0A\x96\xFF\x68\xF4\x5D"
+			  "\xC6\x2F\xBB\x24\x8D\x19\x82\xEB"
+			  "\x54\xE0\x49\xB2\x1B\xA7\x10\x79"
+			  "\x05\x6E\xD7\x40\xCC\x35\x9E\x07"
+			  "\x93\xFC\x65\xF1\x5A\xC3\x2C\xB8"
+			  "\x21\x8A\x16\x7F\xE8\x51\xDD\x46"
+			  "\xAF\x18\xA4\x0D\x76\x02\x6B\xD4"
+			  "\x3D\xC9\x32\x9B\x04\x90\xF9\x62"
+			  "\xEE\x57\xC0\x29\xB5\x1E\x87\x13"
+			  "\x7C\xE5\x4E\xDA\x43\xAC\x15\xA1"
+			  "\x0A\x73\xFF\x68\xD1\x3A\xC6\x2F"
+			  "\x98\x01\x8D\xF6\x5F\xEB\x54\xBD"
+			  "\x26\xB2\x1B\x84\x10\x79\xE2\x4B"
+			  "\xD7\x40\xA9\x12\x9E\x07\x70\xFC"
+			  "\x65\xCE\x37\xC3\x2C\x95\x21\x8A"
+			  "\xF3\x5C\xE8\x51\xBA\x23\xAF\x18"
+			  "\x81\x0D\x76\xDF\x48\xD4\x3D\xA6"
+			  "\x0F\x9B\x04\x6D\xF9\x62\xCB\x34"
+			  "\xC0\x29\x92\x1E\x87\xF0\x59\xE5"
+			  "\x4E\xB7\x20\xAC\x15\x7E\x0A\x73"
+			  "\xDC\x45\xD1\x3A\xA3\x0C\x98\x01"
+			  "\x6A\xF6\x5F\xC8\x31\xBD\x26\x8F"
+			  "\x1B\x84\xED\x56\xE2\x4B\xB4\x1D"
+			  "\xA9\x12\x7B\x07\x70\xD9\x42\xCE"
+			  "\x37\xA0\x09\x95\xFE\x67\xF3\x5C"
+			  "\xC5\x2E\xBA\x23\x8C\x18\x81\xEA"
+			  "\x53\xDF\x48\xB1\x1A\xA6\x0F\x78"
+			  "\x04\x6D\xD6\x3F\xCB\x34\x9D\x06"
+			  "\x92\xFB\x64\xF0\x59\xC2\x2B\xB7"
+			  "\x20\x89\x15\x7E\xE7\x50\xDC\x45"
+			  "\xAE\x17\xA3\x0C\x75\x01\x6A\xD3"
+			  "\x3C\xC8\x31\x9A\x03\x8F\xF8\x61"
+			  "\xED\x56\xBF\x28\xB4\x1D\x86\x12"
+			  "\x7B\xE4\x4D",
+		.ctext	= "\xDA\x4E\x3F\xBC\xE8\xB6\x3A\xA2"
+			  "\xD5\x4D\x84\x4A\xA9\x0C\xE1\xA5"
+			  "\xB8\x73\xBC\xF9\xBB\x59\x2F\x44"
+			  "\x8B\xAB\x82\x6C\xB4\x32\x9A\xDE"
+			  "\x5A\x0B\xDB\x7A\x6B\xF2\x38\x9F"
+			  "\x06\xF7\xF7\xFF\xFF\xC0\x8A\x2E"
+			  "\x76\xEA\x06\x32\x23\xF3\x59\x2E"
+			  "\x75\xDE\x71\x86\x3C\x98\x23\x44"
+			  "\x5B\xF2\xFA\x6A\x00\xBB\xC1\xAD"
+			  "\x58\xBD\x3E\x6F\x2E\xB4\x19\x04"
+			  "\x70\x8B\x92\x55\x23\xE9\x6A\x3A"
+			  "\x78\x7A\x1B\x10\x85\x52\x9C\x12"
+			  "\xE4\x55\x81\x21\xCE\x53\xD0\x3B"
+			  "\x63\x77\x2C\x74\xD1\xF5\x60\xF3"
+			  "\xA1\xDE\x44\x3C\x8F\x4D\x2F\xDD"
+			  "\x8A\xFE\x3C\x42\x8E\xD3\xF2\x8E"
+			  "\xA8\x28\x69\x65\x31\xE1\x45\x83"
+			  "\xE4\x49\xC4\x9C\xA7\x28\xAA\x21"
+			  "\xCD\x5D\x0F\x15\xB7\x93\x07\x26"
+			  "\xB0\x65\x6D\x91\x90\x23\x7A\xC6"
+			  "\xDB\x68\xB0\xA1\x8E\xA4\x76\x4E"
+			  "\xC6\x91\x83\x20\x92\x4D\x63\x7A"
+			  "\x45\x18\x18\x74\x19\xAD\x71\x01"
+			  "\x6B\x23\xAD\x9D\x4E\xE4\x6E\x46"
+			  "\xC9\x73\x7A\xF9\x02\x95\xF4\x07"
+			  "\x0E\x7A\xA6\xC5\xAE\xFA\x15\x2C"
+			  "\x51\x71\xF1\xDC\x22\xB6\xAC\xD8"
+			  "\x19\x24\x44\xBC\x0C\xFB\x3C\x2D"
+			  "\xB1\x50\x47\x15\x0E\xDB\xB6\xD7"
+			  "\xE8\x61\xE5\x95\x52\x1E\x3E\x49"
+			  "\x70\xE9\x66\x04\x4C\xE1\xAF\xBD"
+			  "\xDD\x15\x3B\x20\x59\x24\xFF\xB0"
+			  "\x39\xAA\xE7\xBF\x23\xA3\x6E\xD5"
+			  "\x15\xF0\x61\x4F\xAE\x89\x10\x58"
+			  "\x5A\x33\x95\x52\x2A\xB5\x77\x9C"
+			  "\xA5\x43\x80\x40\x27\x2D\xAE\xD9"
+			  "\x3F\xE0\x80\x94\x78\x79\xCB\x7E"
+			  "\xAD\x12\x44\x4C\xEC\x27\xB0\xEE"
+			  "\x0B\x05\x2A\x82\x99\x58\xBB\x7A"
+			  "\x8D\x6D\x9D\x8E\xE2\x8E\xE7\x93"
+			  "\x2F\xB3\x09\x8D\x06\xD5\xEE\x70"
+			  "\x16\xAE\x35\xC5\x52\x0F\x46\x1F"
+			  "\x71\xF9\x5E\xF2\x67\xDC\x98\x2F"
+			  "\xA3\x23\xAA\xD5\xD0\x49\xF4\xA6"
+			  "\xF6\xB8\x32\xCD\xD6\x85\x73\x60"
+			  "\x59\x20\xE7\x55\x0E\x91\xE2\x0C"
+			  "\x3F\x1C\xEB\x3D\xDF\x52\x64\xF2"
+			  "\x7D\x8B\x5D\x63\x16\xB9\xB2\x5D"
+			  "\x5E\xAB\xB2\x97\xAB\x78\x44\xE7"
+			  "\xC6\x72\x20\xC5\x90\x9B\xDC\x5D"
+			  "\xB0\xEF\x44\xEF\x87\x31\x8D\xF4"
+			  "\xFB\x81\x5D\xF7\x96\x96\xD4\x50"
+			  "\x89\xA7\xF6\xB9\x67\x76\x40\x9E"
+			  "\x9D\x40\xD5\x2C\x30\xB8\x01\x8F"
+			  "\xE4\x7B\x71\x48\xA9\xA0\xA0\x1D"
+			  "\x87\x52\xA4\x91\xA9\xD7\xA9\x51"
+			  "\xD9\x59\xF7\xCC\x63\x22\xC1\x8D"
+			  "\x84\x7B\xD8\x22\x32\x5C\x6F\x1D"
+			  "\x6E\x9F\xFA\xDD\x49\x40\xDC\x37"
+			  "\x14\x8C\xE1\x80\x1B\xDD\x36\x2A"
+			  "\xD0\xE9\x54\x99\x5D\xBA\x3B\x11"
+			  "\xD8\xFE\xC9\x5B\x5C\x25\xE5\x76"
+			  "\xFB\xF2\x3F",
+		.len	= 499,
 		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 1008 - 20, 4, 16 },
+		.np	= 2,
+		.tap	= { 499 - 16, 16 },
+	},
+};
+
+static const struct cipher_testvec aes_ctr_rfc3686_tv_template[] = {
+	{ /* From RFC 3686 */
+		.key	= "\xae\x68\x52\xf8\x12\x10\x67\xcc"
+			  "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e"
+			  "\x00\x00\x00\x30",
+		.klen	= 20,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "Single block msg",
+		.ctext	= "\xe4\x09\x5d\x4f\xb7\xa7\xb3\x79"
+			  "\x2d\x61\x75\xa3\x26\x13\x11\xb8",
+		.len	= 16,
+	}, {
+		.key	= "\x7e\x24\x06\x78\x17\xfa\xe0\xd7"
+			  "\x43\xd6\xce\x1f\x32\x53\x91\x63"
+			  "\x00\x6c\xb6\xdb",
+		.klen	= 20,
+		.iv	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.ctext	= "\x51\x04\xa1\x06\x16\x8a\x72\xd9"
+			  "\x79\x0d\x41\xee\x8e\xda\xd3\x88"
+			  "\xeb\x2e\x1e\xfc\x46\xda\x57\xc8"
+			  "\xfc\xe6\x30\xdf\x91\x41\xbe\x28",
+		.len	= 32,
+	}, {
+		.key	= "\x16\xaf\x5b\x14\x5f\xc9\xf5\x79"
+			  "\xc1\x75\xf9\x3e\x3b\xfb\x0e\xed"
+			  "\x86\x3d\x06\xcc\xfd\xb7\x85\x15"
+			  "\x00\x00\x00\x48",
+		.klen	= 28,
+		.iv	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
+		.ptext	= "Single block msg",
+		.ctext	= "\x4b\x55\x38\x4f\xe2\x59\xc9\xc8"
+			  "\x4e\x79\x35\xa0\x03\xcb\xe9\x28",
+		.len	= 16,
+	}, {
+		.key	= "\x7c\x5c\xb2\x40\x1b\x3d\xc3\x3c"
+			  "\x19\xe7\x34\x08\x19\xe0\xf6\x9c"
+			  "\x67\x8c\x3d\xb8\xe6\xf6\xa9\x1a"
+			  "\x00\x96\xb0\x3b",
+		.klen	= 28,
+		.iv	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.ctext	= "\x45\x32\x43\xfc\x60\x9b\x23\x32"
+			  "\x7e\xdf\xaa\xfa\x71\x31\xcd\x9f"
+			  "\x84\x90\x70\x1c\x5a\xd4\xa7\x9c"
+			  "\xfc\x1f\xe0\xff\x42\xf4\xfb\x00",
+		.len	= 32,
+	}, {
+		.key	= "\x77\x6b\xef\xf2\x85\x1d\xb0\x6f"
+			  "\x4c\x8a\x05\x42\xc8\x69\x6f\x6c"
+			  "\x6a\x81\xaf\x1e\xec\x96\xb4\xd3"
+			  "\x7f\xc1\xd6\x89\xe6\xc1\xc1\x04"
+			  "\x00\x00\x00\x60",
+		.klen	= 36,
+		.iv	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
+		.ptext	= "Single block msg",
+		.ctext	= "\x14\x5a\xd0\x1d\xbf\x82\x4e\xc7"
+			  "\x56\x08\x63\xdc\x71\xe3\xe0\xc0",
+		.len	= 16,
+	}, {
+		.key	= "\xf6\xd6\x6d\x6b\xd5\x2d\x59\xbb"
+			  "\x07\x96\x36\x58\x79\xef\xf8\x86"
+			  "\xc6\x6d\xd5\x1a\x5b\x6a\x99\x74"
+			  "\x4b\x50\x59\x0c\x87\xa2\x38\x84"
+			  "\x00\xfa\xac\x24",
+		.klen	= 36,
+		.iv	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.ctext	= "\xf0\x5e\x23\x1b\x38\x94\x61\x2c"
+			  "\x49\xee\x00\x0b\x80\x4e\xb2\xa9"
+			  "\xb8\x30\x6b\x50\x8f\x83\x9d\x6a"
+			  "\x55\x30\x83\x1d\x93\x44\xaf\x1c",
+		.len	= 32,
+	}, {
+	// generated using Crypto++
+		.key = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			"\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			"\x10\x11\x12\x13\x14\x15\x16\x17"
+			"\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			"\x00\x00\x00\x00",
+		.klen = 32 + 4,
+		.iv = "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext =
+			"\x00\x01\x02\x03\x04\x05\x06\x07"
+			"\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			"\x10\x11\x12\x13\x14\x15\x16\x17"
+			"\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			"\x20\x21\x22\x23\x24\x25\x26\x27"
+			"\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			"\x30\x31\x32\x33\x34\x35\x36\x37"
+			"\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			"\x40\x41\x42\x43\x44\x45\x46\x47"
+			"\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			"\x50\x51\x52\x53\x54\x55\x56\x57"
+			"\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			"\x60\x61\x62\x63\x64\x65\x66\x67"
+			"\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			"\x70\x71\x72\x73\x74\x75\x76\x77"
+			"\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			"\x80\x81\x82\x83\x84\x85\x86\x87"
+			"\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			"\x90\x91\x92\x93\x94\x95\x96\x97"
+			"\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			"\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			"\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			"\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			"\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			"\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			"\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			"\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			"\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			"\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			"\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			"\x00\x03\x06\x09\x0c\x0f\x12\x15"
+			"\x18\x1b\x1e\x21\x24\x27\x2a\x2d"
+			"\x30\x33\x36\x39\x3c\x3f\x42\x45"
+			"\x48\x4b\x4e\x51\x54\x57\x5a\x5d"
+			"\x60\x63\x66\x69\x6c\x6f\x72\x75"
+			"\x78\x7b\x7e\x81\x84\x87\x8a\x8d"
+			"\x90\x93\x96\x99\x9c\x9f\xa2\xa5"
+			"\xa8\xab\xae\xb1\xb4\xb7\xba\xbd"
+			"\xc0\xc3\xc6\xc9\xcc\xcf\xd2\xd5"
+			"\xd8\xdb\xde\xe1\xe4\xe7\xea\xed"
+			"\xf0\xf3\xf6\xf9\xfc\xff\x02\x05"
+			"\x08\x0b\x0e\x11\x14\x17\x1a\x1d"
+			"\x20\x23\x26\x29\x2c\x2f\x32\x35"
+			"\x38\x3b\x3e\x41\x44\x47\x4a\x4d"
+			"\x50\x53\x56\x59\x5c\x5f\x62\x65"
+			"\x68\x6b\x6e\x71\x74\x77\x7a\x7d"
+			"\x80\x83\x86\x89\x8c\x8f\x92\x95"
+			"\x98\x9b\x9e\xa1\xa4\xa7\xaa\xad"
+			"\xb0\xb3\xb6\xb9\xbc\xbf\xc2\xc5"
+			"\xc8\xcb\xce\xd1\xd4\xd7\xda\xdd"
+			"\xe0\xe3\xe6\xe9\xec\xef\xf2\xf5"
+			"\xf8\xfb\xfe\x01\x04\x07\x0a\x0d"
+			"\x10\x13\x16\x19\x1c\x1f\x22\x25"
+			"\x28\x2b\x2e\x31\x34\x37\x3a\x3d"
+			"\x40\x43\x46\x49\x4c\x4f\x52\x55"
+			"\x58\x5b\x5e\x61\x64\x67\x6a\x6d"
+			"\x70\x73\x76\x79\x7c\x7f\x82\x85"
+			"\x88\x8b\x8e\x91\x94\x97\x9a\x9d"
+			"\xa0\xa3\xa6\xa9\xac\xaf\xb2\xb5"
+			"\xb8\xbb\xbe\xc1\xc4\xc7\xca\xcd"
+			"\xd0\xd3\xd6\xd9\xdc\xdf\xe2\xe5"
+			"\xe8\xeb\xee\xf1\xf4\xf7\xfa\xfd"
+			"\x00\x05\x0a\x0f\x14\x19\x1e\x23"
+			"\x28\x2d\x32\x37\x3c\x41\x46\x4b"
+			"\x50\x55\x5a\x5f\x64\x69\x6e\x73"
+			"\x78\x7d\x82\x87\x8c\x91\x96\x9b"
+			"\xa0\xa5\xaa\xaf\xb4\xb9\xbe\xc3"
+			"\xc8\xcd\xd2\xd7\xdc\xe1\xe6\xeb"
+			"\xf0\xf5\xfa\xff\x04\x09\x0e\x13"
+			"\x18\x1d\x22\x27\x2c\x31\x36\x3b"
+			"\x40\x45\x4a\x4f\x54\x59\x5e\x63"
+			"\x68\x6d\x72\x77\x7c\x81\x86\x8b"
+			"\x90\x95\x9a\x9f\xa4\xa9\xae\xb3"
+			"\xb8\xbd\xc2\xc7\xcc\xd1\xd6\xdb"
+			"\xe0\xe5\xea\xef\xf4\xf9\xfe\x03"
+			"\x08\x0d\x12\x17\x1c\x21\x26\x2b"
+			"\x30\x35\x3a\x3f\x44\x49\x4e\x53"
+			"\x58\x5d\x62\x67\x6c\x71\x76\x7b"
+			"\x80\x85\x8a\x8f\x94\x99\x9e\xa3"
+			"\xa8\xad\xb2\xb7\xbc\xc1\xc6\xcb"
+			"\xd0\xd5\xda\xdf\xe4\xe9\xee\xf3"
+			"\xf8\xfd\x02\x07\x0c\x11\x16\x1b"
+			"\x20\x25\x2a\x2f\x34\x39\x3e\x43"
+			"\x48\x4d\x52\x57\x5c\x61\x66\x6b"
+			"\x70\x75\x7a\x7f\x84\x89\x8e\x93"
+			"\x98\x9d\xa2\xa7\xac\xb1\xb6\xbb"
+			"\xc0\xc5\xca\xcf\xd4\xd9\xde\xe3"
+			"\xe8\xed\xf2\xf7\xfc\x01\x06\x0b"
+			"\x10\x15\x1a\x1f\x24\x29\x2e\x33"
+			"\x38\x3d\x42\x47\x4c\x51\x56\x5b"
+			"\x60\x65\x6a\x6f\x74\x79\x7e\x83"
+			"\x88\x8d\x92\x97\x9c\xa1\xa6\xab"
+			"\xb0\xb5\xba\xbf\xc4\xc9\xce\xd3"
+			"\xd8\xdd\xe2\xe7\xec\xf1\xf6\xfb"
+			"\x00\x07\x0e\x15\x1c\x23\x2a\x31"
+			"\x38\x3f\x46\x4d\x54\x5b\x62\x69"
+			"\x70\x77\x7e\x85\x8c\x93\x9a\xa1"
+			"\xa8\xaf\xb6\xbd\xc4\xcb\xd2\xd9"
+			"\xe0\xe7\xee\xf5\xfc\x03\x0a\x11"
+			"\x18\x1f\x26\x2d\x34\x3b\x42\x49"
+			"\x50\x57\x5e\x65\x6c\x73\x7a\x81"
+			"\x88\x8f\x96\x9d\xa4\xab\xb2\xb9"
+			"\xc0\xc7\xce\xd5\xdc\xe3\xea\xf1"
+			"\xf8\xff\x06\x0d\x14\x1b\x22\x29"
+			"\x30\x37\x3e\x45\x4c\x53\x5a\x61"
+			"\x68\x6f\x76\x7d\x84\x8b\x92\x99"
+			"\xa0\xa7\xae\xb5\xbc\xc3\xca\xd1"
+			"\xd8\xdf\xe6\xed\xf4\xfb\x02\x09"
+			"\x10\x17\x1e\x25\x2c\x33\x3a\x41"
+			"\x48\x4f\x56\x5d\x64\x6b\x72\x79"
+			"\x80\x87\x8e\x95\x9c\xa3\xaa\xb1"
+			"\xb8\xbf\xc6\xcd\xd4\xdb\xe2\xe9"
+			"\xf0\xf7\xfe\x05\x0c\x13\x1a\x21"
+			"\x28\x2f\x36\x3d\x44\x4b\x52\x59"
+			"\x60\x67\x6e\x75\x7c\x83\x8a\x91"
+			"\x98\x9f\xa6\xad\xb4\xbb\xc2\xc9"
+			"\xd0\xd7\xde\xe5\xec\xf3\xfa\x01"
+			"\x08\x0f\x16\x1d\x24\x2b\x32\x39"
+			"\x40\x47\x4e\x55\x5c\x63\x6a\x71"
+			"\x78\x7f\x86\x8d\x94\x9b\xa2\xa9"
+			"\xb0\xb7\xbe\xc5\xcc\xd3\xda\xe1"
+			"\xe8\xef\xf6\xfd\x04\x0b\x12\x19"
+			"\x20\x27\x2e\x35\x3c\x43\x4a\x51"
+			"\x58\x5f\x66\x6d\x74\x7b\x82\x89"
+			"\x90\x97\x9e\xa5\xac\xb3\xba\xc1"
+			"\xc8\xcf\xd6\xdd\xe4\xeb\xf2\xf9"
+			"\x00\x09\x12\x1b\x24\x2d\x36\x3f"
+			"\x48\x51\x5a\x63\x6c\x75\x7e\x87"
+			"\x90\x99\xa2\xab\xb4\xbd\xc6\xcf"
+			"\xd8\xe1\xea\xf3\xfc\x05\x0e\x17"
+			"\x20\x29\x32\x3b\x44\x4d\x56\x5f"
+			"\x68\x71\x7a\x83\x8c\x95\x9e\xa7"
+			"\xb0\xb9\xc2\xcb\xd4\xdd\xe6\xef"
+			"\xf8\x01\x0a\x13\x1c\x25\x2e\x37"
+			"\x40\x49\x52\x5b\x64\x6d\x76\x7f"
+			"\x88\x91\x9a\xa3\xac\xb5\xbe\xc7"
+			"\xd0\xd9\xe2\xeb\xf4\xfd\x06\x0f"
+			"\x18\x21\x2a\x33\x3c\x45\x4e\x57"
+			"\x60\x69\x72\x7b\x84\x8d\x96\x9f"
+			"\xa8\xb1\xba\xc3\xcc\xd5\xde\xe7"
+			"\xf0\xf9\x02\x0b\x14\x1d\x26\x2f"
+			"\x38\x41\x4a\x53\x5c\x65\x6e\x77"
+			"\x80\x89\x92\x9b\xa4\xad\xb6\xbf"
+			"\xc8\xd1\xda\xe3\xec\xf5\xfe\x07"
+			"\x10\x19\x22\x2b\x34\x3d\x46\x4f"
+			"\x58\x61\x6a\x73\x7c\x85\x8e\x97"
+			"\xa0\xa9\xb2\xbb\xc4\xcd\xd6\xdf"
+			"\xe8\xf1\xfa\x03\x0c\x15\x1e\x27"
+			"\x30\x39\x42\x4b\x54\x5d\x66\x6f"
+			"\x78\x81\x8a\x93\x9c\xa5\xae\xb7"
+			"\xc0\xc9\xd2\xdb\xe4\xed\xf6\xff"
+			"\x08\x11\x1a\x23\x2c\x35\x3e\x47"
+			"\x50\x59\x62\x6b\x74\x7d\x86\x8f"
+			"\x98\xa1\xaa\xb3\xbc\xc5\xce\xd7"
+			"\xe0\xe9\xf2\xfb\x04\x0d\x16\x1f"
+			"\x28\x31\x3a\x43\x4c\x55\x5e\x67"
+			"\x70\x79\x82\x8b\x94\x9d\xa6\xaf"
+			"\xb8\xc1\xca\xd3\xdc\xe5\xee\xf7"
+			"\x00\x0b\x16\x21\x2c\x37\x42\x4d"
+			"\x58\x63\x6e\x79\x84\x8f\x9a\xa5"
+			"\xb0\xbb\xc6\xd1\xdc\xe7\xf2\xfd"
+			"\x08\x13\x1e\x29\x34\x3f\x4a\x55"
+			"\x60\x6b\x76\x81\x8c\x97\xa2\xad"
+			"\xb8\xc3\xce\xd9\xe4\xef\xfa\x05"
+			"\x10\x1b\x26\x31\x3c\x47\x52\x5d"
+			"\x68\x73\x7e\x89\x94\x9f\xaa\xb5"
+			"\xc0\xcb\xd6\xe1\xec\xf7\x02\x0d"
+			"\x18\x23\x2e\x39\x44\x4f\x5a\x65"
+			"\x70\x7b\x86\x91\x9c\xa7\xb2\xbd"
+			"\xc8\xd3\xde\xe9\xf4\xff\x0a\x15"
+			"\x20\x2b\x36\x41\x4c\x57\x62\x6d"
+			"\x78\x83\x8e\x99\xa4\xaf\xba\xc5"
+			"\xd0\xdb\xe6\xf1\xfc\x07\x12\x1d"
+			"\x28\x33\x3e\x49\x54\x5f\x6a\x75"
+			"\x80\x8b\x96\xa1\xac\xb7\xc2\xcd"
+			"\xd8\xe3\xee\xf9\x04\x0f\x1a\x25"
+			"\x30\x3b\x46\x51\x5c\x67\x72\x7d"
+			"\x88\x93\x9e\xa9\xb4\xbf\xca\xd5"
+			"\xe0\xeb\xf6\x01\x0c\x17\x22\x2d"
+			"\x38\x43\x4e\x59\x64\x6f\x7a\x85"
+			"\x90\x9b\xa6\xb1\xbc\xc7\xd2\xdd"
+			"\xe8\xf3\xfe\x09\x14\x1f\x2a\x35"
+			"\x40\x4b\x56\x61\x6c\x77\x82\x8d"
+			"\x98\xa3\xae\xb9\xc4\xcf\xda\xe5"
+			"\xf0\xfb\x06\x11\x1c\x27\x32\x3d"
+			"\x48\x53\x5e\x69\x74\x7f\x8a\x95"
+			"\xa0\xab\xb6\xc1\xcc\xd7\xe2\xed"
+			"\xf8\x03\x0e\x19\x24\x2f\x3a\x45"
+			"\x50\x5b\x66\x71\x7c\x87\x92\x9d"
+			"\xa8\xb3\xbe\xc9\xd4\xdf\xea\xf5"
+			"\x00\x0d\x1a\x27\x34\x41\x4e\x5b"
+			"\x68\x75\x82\x8f\x9c\xa9\xb6\xc3"
+			"\xd0\xdd\xea\xf7\x04\x11\x1e\x2b"
+			"\x38\x45\x52\x5f\x6c\x79\x86\x93"
+			"\xa0\xad\xba\xc7\xd4\xe1\xee\xfb"
+			"\x08\x15\x22\x2f\x3c\x49\x56\x63"
+			"\x70\x7d\x8a\x97\xa4\xb1\xbe\xcb"
+			"\xd8\xe5\xf2\xff\x0c\x19\x26\x33"
+			"\x40\x4d\x5a\x67\x74\x81\x8e\x9b"
+			"\xa8\xb5\xc2\xcf\xdc\xe9\xf6\x03"
+			"\x10\x1d\x2a\x37\x44\x51\x5e\x6b"
+			"\x78\x85\x92\x9f\xac\xb9\xc6\xd3"
+			"\xe0\xed\xfa\x07\x14\x21\x2e\x3b"
+			"\x48\x55\x62\x6f\x7c\x89\x96\xa3"
+			"\xb0\xbd\xca\xd7\xe4\xf1\xfe\x0b"
+			"\x18\x25\x32\x3f\x4c\x59\x66\x73"
+			"\x80\x8d\x9a\xa7\xb4\xc1\xce\xdb"
+			"\xe8\xf5\x02\x0f\x1c\x29\x36\x43"
+			"\x50\x5d\x6a\x77\x84\x91\x9e\xab"
+			"\xb8\xc5\xd2\xdf\xec\xf9\x06\x13"
+			"\x20\x2d\x3a\x47\x54\x61\x6e\x7b"
+			"\x88\x95\xa2\xaf\xbc\xc9\xd6\xe3"
+			"\xf0\xfd\x0a\x17\x24\x31\x3e\x4b"
+			"\x58\x65\x72\x7f\x8c\x99\xa6\xb3"
+			"\xc0\xcd\xda\xe7\xf4\x01\x0e\x1b"
+			"\x28\x35\x42\x4f\x5c\x69\x76\x83"
+			"\x90\x9d\xaa\xb7\xc4\xd1\xde\xeb"
+			"\xf8\x05\x12\x1f\x2c\x39\x46\x53"
+			"\x60\x6d\x7a\x87\x94\xa1\xae\xbb"
+			"\xc8\xd5\xe2\xef\xfc\x09\x16\x23"
+			"\x30\x3d\x4a\x57\x64\x71\x7e\x8b"
+			"\x98\xa5\xb2\xbf\xcc\xd9\xe6\xf3"
+			"\x00\x0f\x1e\x2d\x3c\x4b\x5a\x69"
+			"\x78\x87\x96\xa5\xb4\xc3\xd2\xe1"
+			"\xf0\xff\x0e\x1d\x2c\x3b\x4a\x59"
+			"\x68\x77\x86\x95\xa4\xb3\xc2\xd1"
+			"\xe0\xef\xfe\x0d\x1c\x2b\x3a\x49"
+			"\x58\x67\x76\x85\x94\xa3\xb2\xc1"
+			"\xd0\xdf\xee\xfd\x0c\x1b\x2a\x39"
+			"\x48\x57\x66\x75\x84\x93\xa2\xb1"
+			"\xc0\xcf\xde\xed\xfc\x0b\x1a\x29"
+			"\x38\x47\x56\x65\x74\x83\x92\xa1"
+			"\xb0\xbf\xce\xdd\xec\xfb\x0a\x19"
+			"\x28\x37\x46\x55\x64\x73\x82\x91"
+			"\xa0\xaf\xbe\xcd\xdc\xeb\xfa\x09"
+			"\x18\x27\x36\x45\x54\x63\x72\x81"
+			"\x90\x9f\xae\xbd\xcc\xdb\xea\xf9"
+			"\x08\x17\x26\x35\x44\x53\x62\x71"
+			"\x80\x8f\x9e\xad\xbc\xcb\xda\xe9"
+			"\xf8\x07\x16\x25\x34\x43\x52\x61"
+			"\x70\x7f\x8e\x9d\xac\xbb\xca\xd9"
+			"\xe8\xf7\x06\x15\x24\x33\x42\x51"
+			"\x60\x6f\x7e\x8d\x9c\xab\xba\xc9"
+			"\xd8\xe7\xf6\x05\x14\x23\x32\x41"
+			"\x50\x5f\x6e\x7d\x8c\x9b\xaa\xb9"
+			"\xc8\xd7\xe6\xf5\x04\x13\x22\x31"
+			"\x40\x4f\x5e\x6d\x7c\x8b\x9a\xa9"
+			"\xb8\xc7\xd6\xe5\xf4\x03\x12\x21"
+			"\x30\x3f\x4e\x5d\x6c\x7b\x8a\x99"
+			"\xa8\xb7\xc6\xd5\xe4\xf3\x02\x11"
+			"\x20\x2f\x3e\x4d\x5c\x6b\x7a\x89"
+			"\x98\xa7\xb6\xc5\xd4\xe3\xf2\x01"
+			"\x10\x1f\x2e\x3d\x4c\x5b\x6a\x79"
+			"\x88\x97\xa6\xb5\xc4\xd3\xe2\xf1"
+			"\x00\x11\x22\x33\x44\x55\x66\x77"
+			"\x88\x99\xaa\xbb\xcc\xdd\xee\xff"
+			"\x10\x21\x32\x43\x54\x65\x76\x87"
+			"\x98\xa9\xba\xcb\xdc\xed\xfe\x0f"
+			"\x20\x31\x42\x53\x64\x75\x86\x97"
+			"\xa8\xb9\xca\xdb\xec\xfd\x0e\x1f"
+			"\x30\x41\x52\x63\x74\x85\x96\xa7"
+			"\xb8\xc9\xda\xeb\xfc\x0d\x1e\x2f"
+			"\x40\x51\x62\x73\x84\x95\xa6\xb7"
+			"\xc8\xd9\xea\xfb\x0c\x1d\x2e\x3f"
+			"\x50\x61\x72\x83\x94\xa5\xb6\xc7"
+			"\xd8\xe9\xfa\x0b\x1c\x2d\x3e\x4f"
+			"\x60\x71\x82\x93\xa4\xb5\xc6\xd7"
+			"\xe8\xf9\x0a\x1b\x2c\x3d\x4e\x5f"
+			"\x70\x81\x92\xa3\xb4\xc5\xd6\xe7"
+			"\xf8\x09\x1a\x2b\x3c\x4d\x5e\x6f"
+			"\x80\x91\xa2\xb3\xc4\xd5\xe6\xf7"
+			"\x08\x19\x2a\x3b\x4c\x5d\x6e\x7f"
+			"\x90\xa1\xb2\xc3\xd4\xe5\xf6\x07"
+			"\x18\x29\x3a\x4b\x5c\x6d\x7e\x8f"
+			"\xa0\xb1\xc2\xd3\xe4\xf5\x06\x17"
+			"\x28\x39\x4a\x5b\x6c\x7d\x8e\x9f"
+			"\xb0\xc1\xd2\xe3\xf4\x05\x16\x27"
+			"\x38\x49\x5a\x6b\x7c\x8d\x9e\xaf"
+			"\xc0\xd1\xe2\xf3\x04\x15\x26\x37"
+			"\x48\x59\x6a\x7b\x8c\x9d\xae\xbf"
+			"\xd0\xe1\xf2\x03\x14\x25\x36\x47"
+			"\x58\x69\x7a\x8b\x9c\xad\xbe\xcf"
+			"\xe0\xf1\x02\x13\x24\x35\x46\x57"
+			"\x68\x79\x8a\x9b\xac\xbd\xce\xdf"
+			"\xf0\x01\x12\x23\x34\x45\x56\x67"
+			"\x78\x89\x9a\xab\xbc\xcd\xde\xef"
+			"\x00\x13\x26\x39\x4c\x5f\x72\x85"
+			"\x98\xab\xbe\xd1\xe4\xf7\x0a\x1d"
+			"\x30\x43\x56\x69\x7c\x8f\xa2\xb5"
+			"\xc8\xdb\xee\x01\x14\x27\x3a\x4d"
+			"\x60\x73\x86\x99\xac\xbf\xd2\xe5"
+			"\xf8\x0b\x1e\x31\x44\x57\x6a\x7d"
+			"\x90\xa3\xb6\xc9\xdc\xef\x02\x15"
+			"\x28\x3b\x4e\x61\x74\x87\x9a\xad"
+			"\xc0\xd3\xe6\xf9\x0c\x1f\x32\x45"
+			"\x58\x6b\x7e\x91\xa4\xb7\xca\xdd"
+			"\xf0\x03\x16\x29\x3c\x4f\x62\x75"
+			"\x88\x9b\xae\xc1\xd4\xe7\xfa\x0d"
+			"\x20\x33\x46\x59\x6c\x7f\x92\xa5"
+			"\xb8\xcb\xde\xf1\x04\x17\x2a\x3d"
+			"\x50\x63\x76\x89\x9c\xaf\xc2\xd5"
+			"\xe8\xfb\x0e\x21\x34\x47\x5a\x6d"
+			"\x80\x93\xa6\xb9\xcc\xdf\xf2\x05"
+			"\x18\x2b\x3e\x51\x64\x77\x8a\x9d"
+			"\xb0\xc3\xd6\xe9\xfc\x0f\x22\x35"
+			"\x48\x5b\x6e\x81\x94\xa7\xba\xcd"
+			"\xe0\xf3\x06\x19\x2c\x3f\x52\x65"
+			"\x78\x8b\x9e\xb1\xc4\xd7\xea\xfd"
+			"\x10\x23\x36\x49\x5c\x6f\x82\x95"
+			"\xa8\xbb\xce\xe1\xf4\x07\x1a\x2d"
+			"\x40\x53\x66\x79\x8c\x9f\xb2\xc5"
+			"\xd8\xeb\xfe\x11\x24\x37\x4a\x5d"
+			"\x70\x83\x96\xa9\xbc\xcf\xe2\xf5"
+			"\x08\x1b\x2e\x41\x54\x67\x7a\x8d"
+			"\xa0\xb3\xc6\xd9\xec\xff\x12\x25"
+			"\x38\x4b\x5e\x71\x84\x97\xaa\xbd"
+			"\xd0\xe3\xf6\x09\x1c\x2f\x42\x55"
+			"\x68\x7b\x8e\xa1\xb4\xc7\xda\xed"
+			"\x00\x15\x2a\x3f\x54\x69\x7e\x93"
+			"\xa8\xbd\xd2\xe7\xfc\x11\x26\x3b"
+			"\x50\x65\x7a\x8f\xa4\xb9\xce\xe3"
+			"\xf8\x0d\x22\x37\x4c\x61\x76\x8b"
+			"\xa0\xb5\xca\xdf\xf4\x09\x1e\x33"
+			"\x48\x5d\x72\x87\x9c\xb1\xc6\xdb"
+			"\xf0\x05\x1a\x2f\x44\x59\x6e\x83"
+			"\x98\xad\xc2\xd7\xec\x01\x16\x2b"
+			"\x40\x55\x6a\x7f\x94\xa9\xbe\xd3"
+			"\xe8\xfd\x12\x27\x3c\x51\x66\x7b"
+			"\x90\xa5\xba\xcf\xe4\xf9\x0e\x23"
+			"\x38\x4d\x62\x77\x8c\xa1\xb6\xcb"
+			"\xe0\xf5\x0a\x1f\x34\x49\x5e\x73"
+			"\x88\x9d\xb2\xc7\xdc\xf1\x06\x1b"
+			"\x30\x45\x5a\x6f\x84\x99\xae\xc3"
+			"\xd8\xed\x02\x17\x2c\x41\x56\x6b"
+			"\x80\x95\xaa\xbf\xd4\xe9\xfe\x13"
+			"\x28\x3d\x52\x67\x7c\x91\xa6\xbb"
+			"\xd0\xe5\xfa\x0f\x24\x39\x4e\x63"
+			"\x78\x8d\xa2\xb7\xcc\xe1\xf6\x0b"
+			"\x20\x35\x4a\x5f\x74\x89\x9e\xb3"
+			"\xc8\xdd\xf2\x07\x1c\x31\x46\x5b"
+			"\x70\x85\x9a\xaf\xc4\xd9\xee\x03"
+			"\x18\x2d\x42\x57\x6c\x81\x96\xab"
+			"\xc0\xd5\xea\xff\x14\x29\x3e\x53"
+			"\x68\x7d\x92\xa7\xbc\xd1\xe6\xfb"
+			"\x10\x25\x3a\x4f\x64\x79\x8e\xa3"
+			"\xb8\xcd\xe2\xf7\x0c\x21\x36\x4b"
+			"\x60\x75\x8a\x9f\xb4\xc9\xde\xf3"
+			"\x08\x1d\x32\x47\x5c\x71\x86\x9b"
+			"\xb0\xc5\xda\xef\x04\x19\x2e\x43"
+			"\x58\x6d\x82\x97\xac\xc1\xd6\xeb"
+			"\x00\x17\x2e\x45\x5c\x73\x8a\xa1"
+			"\xb8\xcf\xe6\xfd\x14\x2b\x42\x59"
+			"\x70\x87\x9e\xb5\xcc\xe3\xfa\x11"
+			"\x28\x3f\x56\x6d\x84\x9b\xb2\xc9"
+			"\xe0\xf7\x0e\x25\x3c\x53\x6a\x81"
+			"\x98\xaf\xc6\xdd\xf4\x0b\x22\x39"
+			"\x50\x67\x7e\x95\xac\xc3\xda\xf1"
+			"\x08\x1f\x36\x4d\x64\x7b\x92\xa9"
+			"\xc0\xd7\xee\x05\x1c\x33\x4a\x61"
+			"\x78\x8f\xa6\xbd\xd4\xeb\x02\x19"
+			"\x30\x47\x5e\x75\x8c\xa3\xba\xd1"
+			"\xe8\xff\x16\x2d\x44\x5b\x72\x89"
+			"\xa0\xb7\xce\xe5\xfc\x13\x2a\x41"
+			"\x58\x6f\x86\x9d\xb4\xcb\xe2\xf9"
+			"\x10\x27\x3e\x55\x6c\x83\x9a\xb1"
+			"\xc8\xdf\xf6\x0d\x24\x3b\x52\x69"
+			"\x80\x97\xae\xc5\xdc\xf3\x0a\x21"
+			"\x38\x4f\x66\x7d\x94\xab\xc2\xd9"
+			"\xf0\x07\x1e\x35\x4c\x63\x7a\x91"
+			"\xa8\xbf\xd6\xed\x04\x1b\x32\x49"
+			"\x60\x77\x8e\xa5\xbc\xd3\xea\x01"
+			"\x18\x2f\x46\x5d\x74\x8b\xa2\xb9"
+			"\xd0\xe7\xfe\x15\x2c\x43\x5a\x71"
+			"\x88\x9f\xb6\xcd\xe4\xfb\x12\x29"
+			"\x40\x57\x6e\x85\x9c\xb3\xca\xe1"
+			"\xf8\x0f\x26\x3d\x54\x6b\x82\x99"
+			"\xb0\xc7\xde\xf5\x0c\x23\x3a\x51"
+			"\x68\x7f\x96\xad\xc4\xdb\xf2\x09"
+			"\x20\x37\x4e\x65\x7c\x93\xaa\xc1"
+			"\xd8\xef\x06\x1d\x34\x4b\x62\x79"
+			"\x90\xa7\xbe\xd5\xec\x03\x1a\x31"
+			"\x48\x5f\x76\x8d\xa4\xbb\xd2\xe9"
+			"\x00\x19\x32\x4b\x64\x7d\x96\xaf"
+			"\xc8\xe1\xfa\x13\x2c\x45\x5e\x77"
+			"\x90\xa9\xc2\xdb\xf4\x0d\x26\x3f"
+			"\x58\x71\x8a\xa3\xbc\xd5\xee\x07"
+			"\x20\x39\x52\x6b\x84\x9d\xb6\xcf"
+			"\xe8\x01\x1a\x33\x4c\x65\x7e\x97"
+			"\xb0\xc9\xe2\xfb\x14\x2d\x46\x5f"
+			"\x78\x91\xaa\xc3\xdc\xf5\x0e\x27"
+			"\x40\x59\x72\x8b\xa4\xbd\xd6\xef"
+			"\x08\x21\x3a\x53\x6c\x85\x9e\xb7"
+			"\xd0\xe9\x02\x1b\x34\x4d\x66\x7f"
+			"\x98\xb1\xca\xe3\xfc\x15\x2e\x47"
+			"\x60\x79\x92\xab\xc4\xdd\xf6\x0f"
+			"\x28\x41\x5a\x73\x8c\xa5\xbe\xd7"
+			"\xf0\x09\x22\x3b\x54\x6d\x86\x9f"
+			"\xb8\xd1\xea\x03\x1c\x35\x4e\x67"
+			"\x80\x99\xb2\xcb\xe4\xfd\x16\x2f"
+			"\x48\x61\x7a\x93\xac\xc5\xde\xf7"
+			"\x10\x29\x42\x5b\x74\x8d\xa6\xbf"
+			"\xd8\xf1\x0a\x23\x3c\x55\x6e\x87"
+			"\xa0\xb9\xd2\xeb\x04\x1d\x36\x4f"
+			"\x68\x81\x9a\xb3\xcc\xe5\xfe\x17"
+			"\x30\x49\x62\x7b\x94\xad\xc6\xdf"
+			"\xf8\x11\x2a\x43\x5c\x75\x8e\xa7"
+			"\xc0\xd9\xf2\x0b\x24\x3d\x56\x6f"
+			"\x88\xa1\xba\xd3\xec\x05\x1e\x37"
+			"\x50\x69\x82\x9b\xb4\xcd\xe6\xff"
+			"\x18\x31\x4a\x63\x7c\x95\xae\xc7"
+			"\xe0\xf9\x12\x2b\x44\x5d\x76\x8f"
+			"\xa8\xc1\xda\xf3\x0c\x25\x3e\x57"
+			"\x70\x89\xa2\xbb\xd4\xed\x06\x1f"
+			"\x38\x51\x6a\x83\x9c\xb5\xce\xe7"
+			"\x00\x1b\x36\x51\x6c\x87\xa2\xbd"
+			"\xd8\xf3\x0e\x29\x44\x5f\x7a\x95"
+			"\xb0\xcb\xe6\x01\x1c\x37\x52\x6d"
+			"\x88\xa3\xbe\xd9\xf4\x0f\x2a\x45"
+			"\x60\x7b\x96\xb1\xcc\xe7\x02\x1d"
+			"\x38\x53\x6e\x89\xa4\xbf\xda\xf5"
+			"\x10\x2b\x46\x61\x7c\x97\xb2\xcd"
+			"\xe8\x03\x1e\x39\x54\x6f\x8a\xa5"
+			"\xc0\xdb\xf6\x11\x2c\x47\x62\x7d"
+			"\x98\xb3\xce\xe9\x04\x1f\x3a\x55"
+			"\x70\x8b\xa6\xc1\xdc\xf7\x12\x2d"
+			"\x48\x63\x7e\x99\xb4\xcf\xea\x05"
+			"\x20\x3b\x56\x71\x8c\xa7\xc2\xdd"
+			"\xf8\x13\x2e\x49\x64\x7f\x9a\xb5"
+			"\xd0\xeb\x06\x21\x3c\x57\x72\x8d"
+			"\xa8\xc3\xde\xf9\x14\x2f\x4a\x65"
+			"\x80\x9b\xb6\xd1\xec\x07\x22\x3d"
+			"\x58\x73\x8e\xa9\xc4\xdf\xfa\x15"
+			"\x30\x4b\x66\x81\x9c\xb7\xd2\xed"
+			"\x08\x23\x3e\x59\x74\x8f\xaa\xc5"
+			"\xe0\xfb\x16\x31\x4c\x67\x82\x9d"
+			"\xb8\xd3\xee\x09\x24\x3f\x5a\x75"
+			"\x90\xab\xc6\xe1\xfc\x17\x32\x4d"
+			"\x68\x83\x9e\xb9\xd4\xef\x0a\x25"
+			"\x40\x5b\x76\x91\xac\xc7\xe2\xfd"
+			"\x18\x33\x4e\x69\x84\x9f\xba\xd5"
+			"\xf0\x0b\x26\x41\x5c\x77\x92\xad"
+			"\xc8\xe3\xfe\x19\x34\x4f\x6a\x85"
+			"\xa0\xbb\xd6\xf1\x0c\x27\x42\x5d"
+			"\x78\x93\xae\xc9\xe4\xff\x1a\x35"
+			"\x50\x6b\x86\xa1\xbc\xd7\xf2\x0d"
+			"\x28\x43\x5e\x79\x94\xaf\xca\xe5"
+			"\x00\x1d\x3a\x57\x74\x91\xae\xcb"
+			"\xe8\x05\x22\x3f\x5c\x79\x96\xb3"
+			"\xd0\xed\x0a\x27\x44\x61\x7e\x9b"
+			"\xb8\xd5\xf2\x0f\x2c\x49\x66\x83"
+			"\xa0\xbd\xda\xf7\x14\x31\x4e\x6b"
+			"\x88\xa5\xc2\xdf\xfc\x19\x36\x53"
+			"\x70\x8d\xaa\xc7\xe4\x01\x1e\x3b"
+			"\x58\x75\x92\xaf\xcc\xe9\x06\x23"
+			"\x40\x5d\x7a\x97\xb4\xd1\xee\x0b"
+			"\x28\x45\x62\x7f\x9c\xb9\xd6\xf3"
+			"\x10\x2d\x4a\x67\x84\xa1\xbe\xdb"
+			"\xf8\x15\x32\x4f\x6c\x89\xa6\xc3"
+			"\xe0\xfd\x1a\x37\x54\x71\x8e\xab"
+			"\xc8\xe5\x02\x1f\x3c\x59\x76\x93"
+			"\xb0\xcd\xea\x07\x24\x41\x5e\x7b"
+			"\x98\xb5\xd2\xef\x0c\x29\x46\x63"
+			"\x80\x9d\xba\xd7\xf4\x11\x2e\x4b"
+			"\x68\x85\xa2\xbf\xdc\xf9\x16\x33"
+			"\x50\x6d\x8a\xa7\xc4\xe1\xfe\x1b"
+			"\x38\x55\x72\x8f\xac\xc9\xe6\x03"
+			"\x20\x3d\x5a\x77\x94\xb1\xce\xeb"
+			"\x08\x25\x42\x5f\x7c\x99\xb6\xd3"
+			"\xf0\x0d\x2a\x47\x64\x81\x9e\xbb"
+			"\xd8\xf5\x12\x2f\x4c\x69\x86\xa3"
+			"\xc0\xdd\xfa\x17\x34\x51\x6e\x8b"
+			"\xa8\xc5\xe2\xff\x1c\x39\x56\x73"
+			"\x90\xad\xca\xe7\x04\x21\x3e\x5b"
+			"\x78\x95\xb2\xcf\xec\x09\x26\x43"
+			"\x60\x7d\x9a\xb7\xd4\xf1\x0e\x2b"
+			"\x48\x65\x82\x9f\xbc\xd9\xf6\x13"
+			"\x30\x4d\x6a\x87\xa4\xc1\xde\xfb"
+			"\x18\x35\x52\x6f\x8c\xa9\xc6\xe3"
+			"\x00\x1f\x3e\x5d\x7c\x9b\xba\xd9"
+			"\xf8\x17\x36\x55\x74\x93\xb2\xd1"
+			"\xf0\x0f\x2e\x4d\x6c\x8b\xaa\xc9"
+			"\xe8\x07\x26\x45\x64\x83\xa2\xc1"
+			"\xe0\xff\x1e\x3d\x5c\x7b\x9a\xb9"
+			"\xd8\xf7\x16\x35\x54\x73\x92\xb1"
+			"\xd0\xef\x0e\x2d\x4c\x6b\x8a\xa9"
+			"\xc8\xe7\x06\x25\x44\x63\x82\xa1"
+			"\xc0\xdf\xfe\x1d\x3c\x5b\x7a\x99"
+			"\xb8\xd7\xf6\x15\x34\x53\x72\x91"
+			"\xb0\xcf\xee\x0d\x2c\x4b\x6a\x89"
+			"\xa8\xc7\xe6\x05\x24\x43\x62\x81"
+			"\xa0\xbf\xde\xfd\x1c\x3b\x5a\x79"
+			"\x98\xb7\xd6\xf5\x14\x33\x52\x71"
+			"\x90\xaf\xce\xed\x0c\x2b\x4a\x69"
+			"\x88\xa7\xc6\xe5\x04\x23\x42\x61"
+			"\x80\x9f\xbe\xdd\xfc\x1b\x3a\x59"
+			"\x78\x97\xb6\xd5\xf4\x13\x32\x51"
+			"\x70\x8f\xae\xcd\xec\x0b\x2a\x49"
+			"\x68\x87\xa6\xc5\xe4\x03\x22\x41"
+			"\x60\x7f\x9e\xbd\xdc\xfb\x1a\x39"
+			"\x58\x77\x96\xb5\xd4\xf3\x12\x31"
+			"\x50\x6f\x8e\xad\xcc\xeb\x0a\x29"
+			"\x48\x67\x86\xa5\xc4\xe3\x02\x21"
+			"\x40\x5f\x7e\x9d\xbc\xdb\xfa\x19"
+			"\x38\x57\x76\x95\xb4\xd3\xf2\x11"
+			"\x30\x4f\x6e\x8d\xac\xcb\xea\x09"
+			"\x28\x47\x66\x85\xa4\xc3\xe2\x01"
+			"\x20\x3f\x5e\x7d\x9c\xbb\xda\xf9"
+			"\x18\x37\x56\x75\x94\xb3\xd2\xf1"
+			"\x10\x2f\x4e\x6d\x8c\xab\xca\xe9"
+			"\x08\x27\x46\x65\x84\xa3\xc2\xe1"
+			"\x00\x21\x42\x63",
+		.ctext =
+			"\xf0\x5c\x74\xad\x4e\xbc\x99\xe2"
+			"\xae\xff\x91\x3a\x44\xcf\x38\x32"
+			"\x1e\xad\xa7\xcd\xa1\x39\x95\xaa"
+			"\x10\xb1\xb3\x2e\x04\x31\x8f\x86"
+			"\xf2\x62\x74\x70\x0c\xa4\x46\x08"
+			"\xa8\xb7\x99\xa8\xe9\xd2\x73\x79"
+			"\x7e\x6e\xd4\x8f\x1e\xc7\x8e\x31"
+			"\x0b\xfa\x4b\xce\xfd\xf3\x57\x71"
+			"\xe9\x46\x03\xa5\x3d\x34\x00\xe2"
+			"\x18\xff\x75\x6d\x06\x2d\x00\xab"
+			"\xb9\x3e\x6c\x59\xc5\x84\x06\xb5"
+			"\x8b\xd0\x89\x9c\x4a\x79\x16\xc6"
+			"\x3d\x74\x54\xfa\x44\xcd\x23\x26"
+			"\x5c\xcf\x7e\x28\x92\x32\xbf\xdf"
+			"\xa7\x20\x3c\x74\x58\x2a\x9a\xde"
+			"\x61\x00\x1c\x4f\xff\x59\xc4\x22"
+			"\xac\x3c\xd0\xe8\x6c\xf9\x97\x1b"
+			"\x58\x9b\xad\x71\xe8\xa9\xb5\x0d"
+			"\xee\x2f\x04\x1f\x7f\xbc\x99\xee"
+			"\x84\xff\x42\x60\xdc\x3a\x18\xa5"
+			"\x81\xf9\xef\xdc\x7a\x0f\x65\x41"
+			"\x2f\xa3\xd3\xf9\xc2\xcb\xc0\x4d"
+			"\x8f\xd3\x76\x96\xad\x49\x6d\x38"
+			"\x3d\x39\x0b\x6c\x80\xb7\x54\x69"
+			"\xf0\x2c\x90\x02\x29\x0d\x1c\x12"
+			"\xad\x55\xc3\x8b\x68\xd9\xcc\xb3"
+			"\xb2\x64\x33\x90\x5e\xca\x4b\xe2"
+			"\xfb\x75\xdc\x63\xf7\x9f\x82\x74"
+			"\xf0\xc9\xaa\x7f\xe9\x2a\x9b\x33"
+			"\xbc\x88\x00\x7f\xca\xb2\x1f\x14"
+			"\xdb\xc5\x8e\x7b\x11\x3c\x3e\x08"
+			"\xf3\x83\xe8\xe0\x94\x86\x2e\x92"
+			"\x78\x6b\x01\xc9\xc7\x83\xba\x21"
+			"\x6a\x25\x15\x33\x4e\x45\x08\xec"
+			"\x35\xdb\xe0\x6e\x31\x51\x79\xa9"
+			"\x42\x44\x65\xc1\xa0\xf1\xf9\x2a"
+			"\x70\xd5\xb6\xc6\xc1\x8c\x39\xfc"
+			"\x25\xa6\x55\xd9\xdd\x2d\x4c\xec"
+			"\x49\xc6\xeb\x0e\xa8\x25\x2a\x16"
+			"\x1b\x66\x84\xda\xe2\x92\xe5\xc0"
+			"\xc8\x53\x07\xaf\x80\x84\xec\xfd"
+			"\xcd\xd1\x6e\xcd\x6f\x6a\xf5\x36"
+			"\xc5\x15\xe5\x25\x7d\x77\xd1\x1a"
+			"\x93\x36\xa9\xcf\x7c\xa4\x54\x4a"
+			"\x06\x51\x48\x4e\xf6\x59\x87\xd2"
+			"\x04\x02\xef\xd3\x44\xde\x76\x31"
+			"\xb3\x34\x17\x1b\x9d\x66\x11\x9f"
+			"\x1e\xcc\x17\xe9\xc7\x3c\x1b\xe7"
+			"\xcb\x50\x08\xfc\xdc\x2b\x24\xdb"
+			"\x65\x83\xd0\x3b\xe3\x30\xea\x94"
+			"\x6c\xe7\xe8\x35\x32\xc7\xdb\x64"
+			"\xb4\x01\xab\x36\x2c\x77\x13\xaf"
+			"\xf8\x2b\x88\x3f\x54\x39\xc4\x44"
+			"\xfe\xef\x6f\x68\x34\xbe\x0f\x05"
+			"\x16\x6d\xf6\x0a\x30\xe7\xe3\xed"
+			"\xc4\xde\x3c\x1b\x13\xd8\xdb\xfe"
+			"\x41\x62\xe5\x28\xd4\x8d\xa3\xc7"
+			"\x93\x97\xc6\x48\x45\x1d\x9f\x83"
+			"\xdf\x4b\x40\x3e\x42\x25\x87\x80"
+			"\x4c\x7d\xa8\xd4\x98\x23\x95\x75"
+			"\x41\x8c\xda\x41\x9b\xd4\xa7\x06"
+			"\xb5\xf1\x71\x09\x53\xbe\xca\xbf"
+			"\x32\x03\xed\xf0\x50\x1c\x56\x39"
+			"\x5b\xa4\x75\x18\xf7\x9b\x58\xef"
+			"\x53\xfc\x2a\x38\x23\x15\x75\xcd"
+			"\x45\xe5\x5a\x82\x55\xba\x21\xfa"
+			"\xd4\xbd\xc6\x94\x7c\xc5\x80\x12"
+			"\xf7\x4b\x32\xc4\x9a\x82\xd8\x28"
+			"\x8f\xd9\xc2\x0f\x60\x03\xbe\x5e"
+			"\x21\xd6\x5f\x58\xbf\x5c\xb1\x32"
+			"\x82\x8d\xa9\xe5\xf2\x66\x1a\xc0"
+			"\xa0\xbc\x58\x2f\x71\xf5\x2f\xed"
+			"\xd1\x26\xb9\xd8\x49\x5a\x07\x19"
+			"\x01\x7c\x59\xb0\xf8\xa4\xb7\xd3"
+			"\x7b\x1a\x8c\x38\xf4\x50\xa4\x59"
+			"\xb0\xcc\x41\x0b\x88\x7f\xe5\x31"
+			"\xb3\x42\xba\xa2\x7e\xd4\x32\x71"
+			"\x45\x87\x48\xa9\xc2\xf2\x89\xb3"
+			"\xe4\xa7\x7e\x52\x15\x61\xfa\xfe"
+			"\xc9\xdd\x81\xeb\x13\xab\xab\xc3"
+			"\x98\x59\xd8\x16\x3d\x14\x7a\x1c"
+			"\x3c\x41\x9a\x16\x16\x9b\xd2\xd2"
+			"\x69\x3a\x29\x23\xac\x86\x32\xa5"
+			"\x48\x9c\x9e\xf3\x47\x77\x81\x70"
+			"\x24\xe8\x85\xd2\xf5\xb5\xfa\xff"
+			"\x59\x6a\xd3\x50\x59\x43\x59\xde"
+			"\xd9\xf1\x55\xa5\x0c\xc3\x1a\x1a"
+			"\x18\x34\x0d\x1a\x63\x33\xed\x10"
+			"\xe0\x1d\x2a\x18\xd2\xc0\x54\xa8"
+			"\xca\xb5\x9a\xd3\xdd\xca\x45\x84"
+			"\x50\xe7\x0f\xfe\xa4\x99\x5a\xbe"
+			"\x43\x2d\x9a\xcb\x92\x3f\x5a\x1d"
+			"\x85\xd8\xc9\xdf\x68\xc9\x12\x80"
+			"\x56\x0c\xdc\x00\xdc\x3a\x7d\x9d"
+			"\xa3\xa2\xe8\x4d\xbf\xf9\x70\xa0"
+			"\xa4\x13\x4f\x6b\xaf\x0a\x89\x7f"
+			"\xda\xf0\xbf\x9b\xc8\x1d\xe5\xf8"
+			"\x2e\x8b\x07\xb5\x73\x1b\xcc\xa2"
+			"\xa6\xad\x30\xbc\x78\x3c\x5b\x10"
+			"\xfa\x5e\x62\x2d\x9e\x64\xb3\x33"
+			"\xce\xf9\x1f\x86\xe7\x8b\xa2\xb8"
+			"\xe8\x99\x57\x8c\x11\xed\x66\xd9"
+			"\x3c\x72\xb9\xc3\xe6\x4e\x17\x3a"
+			"\x6a\xcb\x42\x24\x06\xed\x3e\x4e"
+			"\xa3\xe8\x6a\x94\xda\x0d\x4e\xd5"
+			"\x14\x19\xcf\xb6\x26\xd8\x2e\xcc"
+			"\x64\x76\x38\x49\x4d\xfe\x30\x6d"
+			"\xe4\xc8\x8c\x7b\xc4\xe0\x35\xba"
+			"\x22\x6e\x76\xe1\x1a\xf2\x53\xc3"
+			"\x28\xa2\x82\x1f\x61\x69\xad\xc1"
+			"\x7b\x28\x4b\x1e\x6c\x85\x95\x9b"
+			"\x51\xb5\x17\x7f\x12\x69\x8c\x24"
+			"\xd5\xc7\x5a\x5a\x11\x54\xff\x5a"
+			"\xf7\x16\xc3\x91\xa6\xf0\xdc\x0a"
+			"\xb6\xa7\x4a\x0d\x7a\x58\xfe\xa5"
+			"\xf5\xcb\x8f\x7b\x0e\xea\x57\xe7"
+			"\xbd\x79\xd6\x1c\x88\x23\x6c\xf2"
+			"\x4d\x29\x77\x53\x35\x6a\x00\x8d"
+			"\xcd\xa3\x58\xbe\x77\x99\x18\xf8"
+			"\xe6\xe1\x8f\xe9\x37\x8f\xe3\xe2"
+			"\x5a\x8a\x93\x25\xaf\xf3\x78\x80"
+			"\xbe\xa6\x1b\xc6\xac\x8b\x1c\x91"
+			"\x58\xe1\x9f\x89\x35\x9d\x1d\x21"
+			"\x29\x9f\xf4\x99\x02\x27\x0f\xa8"
+			"\x4f\x79\x94\x2b\x33\x2c\xda\xa2"
+			"\x26\x39\x83\x94\xef\x27\xd8\x53"
+			"\x8f\x66\x0d\xe4\x41\x7d\x34\xcd"
+			"\x43\x7c\x95\x0a\x53\xef\x66\xda"
+			"\x7e\x9b\xf3\x93\xaf\xd0\x73\x71"
+			"\xba\x40\x9b\x74\xf8\xd7\xd7\x41"
+			"\x6d\xaf\x72\x9c\x8d\x21\x87\x3c"
+			"\xfd\x0a\x90\xa9\x47\x96\x9e\xd3"
+			"\x88\xee\x73\xcf\x66\x2f\x52\x56"
+			"\x6d\xa9\x80\x4c\xe2\x6f\x62\x88"
+			"\x3f\x0e\x54\x17\x48\x80\x5d\xd3"
+			"\xc3\xda\x25\x3d\xa1\xc8\xcb\x9f"
+			"\x9b\x70\xb3\xa1\xeb\x04\x52\xa1"
+			"\xf2\x22\x0f\xfc\xc8\x18\xfa\xf9"
+			"\x85\x9c\xf1\xac\xeb\x0c\x02\x46"
+			"\x75\xd2\xf5\x2c\xe3\xd2\x59\x94"
+			"\x12\xf3\x3c\xfc\xd7\x92\xfa\x36"
+			"\xba\x61\x34\x38\x7c\xda\x48\x3e"
+			"\x08\xc9\x39\x23\x5e\x02\x2c\x1a"
+			"\x18\x7e\xb4\xd9\xfd\x9e\x40\x02"
+			"\xb1\x33\x37\x32\xe7\xde\xd6\xd0"
+			"\x7c\x58\x65\x4b\xf8\x34\x27\x9c"
+			"\x44\xb4\xbd\xe9\xe9\x4c\x78\x7d"
+			"\x4b\x9f\xce\xb1\xcd\x47\xa5\x37"
+			"\xe5\x6d\xbd\xb9\x43\x94\x0a\xd4"
+			"\xd6\xf9\x04\x5f\xb5\x66\x6c\x1a"
+			"\x35\x12\xe3\x36\x28\x27\x36\x58"
+			"\x01\x2b\x79\xe4\xba\x6d\x10\x7d"
+			"\x65\xdf\x84\x95\xf4\xd5\xb6\x8f"
+			"\x2b\x9f\x96\x00\x86\x60\xf0\x21"
+			"\x76\xa8\x6a\x8c\x28\x1c\xb3\x6b"
+			"\x97\xd7\xb6\x53\x2a\xcc\xab\x40"
+			"\x9d\x62\x79\x58\x52\xe6\x65\xb7"
+			"\xab\x55\x67\x9c\x89\x7c\x03\xb0"
+			"\x73\x59\xc5\x81\xf5\x18\x17\x5c"
+			"\x89\xf3\x78\x35\x44\x62\x78\x72"
+			"\xd0\x96\xeb\x31\xe7\x87\x77\x14"
+			"\x99\x51\xf2\x59\x26\x9e\xb5\xa6"
+			"\x45\xfe\x6e\xbd\x07\x4c\x94\x5a"
+			"\xa5\x7d\xfc\xf1\x2b\x77\xe2\xfe"
+			"\x17\xd4\x84\xa0\xac\xb5\xc7\xda"
+			"\xa9\x1a\xb6\xf3\x74\x11\xb4\x9d"
+			"\xfb\x79\x2e\x04\x2d\x50\x28\x83"
+			"\xbf\xc6\x52\xd3\x34\xd6\xe8\x7a"
+			"\xb6\xea\xe7\xa8\x6c\x15\x1e\x2c"
+			"\x57\xbc\x48\x4e\x5f\x5c\xb6\x92"
+			"\xd2\x49\x77\x81\x6d\x90\x70\xae"
+			"\x98\xa1\x03\x0d\x6b\xb9\x77\x14"
+			"\xf1\x4e\x23\xd3\xf8\x68\xbd\xc2"
+			"\xfe\x04\xb7\x5c\xc5\x17\x60\x8f"
+			"\x65\x54\xa4\x7a\x42\xdc\x18\x0d"
+			"\xb5\xcf\x0f\xd3\xc7\x91\x66\x1b"
+			"\x45\x42\x27\x75\x50\xe5\xee\xb8"
+			"\x7f\x33\x2c\xba\x4a\x92\x4d\x2c"
+			"\x3c\xe3\x0d\x80\x01\xba\x0d\x29"
+			"\xd8\x3c\xe9\x13\x16\x57\xe6\xea"
+			"\x94\x52\xe7\x00\x4d\x30\xb0\x0f"
+			"\x35\xb8\xb8\xa7\xb1\xb5\x3b\x44"
+			"\xe1\x2f\xfd\x88\xed\x43\xe7\x52"
+			"\x10\x93\xb3\x8a\x30\x6b\x0a\xf7"
+			"\x23\xc6\x50\x9d\x4a\xb0\xde\xc3"
+			"\xdc\x9b\x2f\x01\x56\x36\x09\xc5"
+			"\x2f\x6b\xfe\xf1\xd8\x27\x45\x03"
+			"\x30\x5e\x5c\x5b\xb4\x62\x0e\x1a"
+			"\xa9\x21\x2b\x92\x94\x87\x62\x57"
+			"\x4c\x10\x74\x1a\xf1\x0a\xc5\x84"
+			"\x3b\x9e\x72\x02\xd7\xcc\x09\x56"
+			"\xbd\x54\xc1\xf0\xc3\xe3\xb3\xf8"
+			"\xd2\x0d\x61\xcb\xef\xce\x0d\x05"
+			"\xb0\x98\xd9\x8e\x4f\xf9\xbc\x93"
+			"\xa6\xea\xc8\xcf\x10\x53\x4b\xf1"
+			"\xec\xfc\x89\xf9\x64\xb0\x22\xbf"
+			"\x9e\x55\x46\x9f\x7c\x50\x8e\x84"
+			"\x54\x20\x98\xd7\x6c\x40\x1e\xdb"
+			"\x69\x34\x78\x61\x24\x21\x9c\x8a"
+			"\xb3\x62\x31\x8b\x6e\xf5\x2a\x35"
+			"\x86\x13\xb1\x6c\x64\x2e\x41\xa5"
+			"\x05\xf2\x42\xba\xd2\x3a\x0d\x8e"
+			"\x8a\x59\x94\x3c\xcf\x36\x27\x82"
+			"\xc2\x45\xee\x58\xcd\x88\xb4\xec"
+			"\xde\xb2\x96\x0a\xaf\x38\x6f\x88"
+			"\xd7\xd8\xe1\xdf\xb9\x96\xa9\x0a"
+			"\xb1\x95\x28\x86\x20\xe9\x17\x49"
+			"\xa2\x29\x38\xaa\xa5\xe9\x6e\xf1"
+			"\x19\x27\xc0\xd5\x2a\x22\xc3\x0b"
+			"\xdb\x7c\x73\x10\xb9\xba\x89\x76"
+			"\x54\xae\x7d\x71\xb3\x93\xf6\x32"
+			"\xe6\x47\x43\x55\xac\xa0\x0d\xc2"
+			"\x93\x27\x4a\x8e\x0e\x74\x15\xc7"
+			"\x0b\x85\xd9\x0c\xa9\x30\x7a\x3e"
+			"\xea\x8f\x85\x6d\x3a\x12\x4f\x72"
+			"\x69\x58\x7a\x80\xbb\xb5\x97\xf3"
+			"\xcf\x70\xd2\x5d\xdd\x4d\x21\x79"
+			"\x54\x4d\xe4\x05\xe8\xbd\xc2\x62"
+			"\xb1\x3b\x77\x1c\xd6\x5c\xf3\xa0"
+			"\x79\x00\xa8\x6c\x29\xd9\x18\x24"
+			"\x36\xa2\x46\xc0\x96\x65\x7f\xbd"
+			"\x2a\xed\x36\x16\x0c\xaa\x9f\xf4"
+			"\xc5\xb4\xe2\x12\xed\x69\xed\x4f"
+			"\x26\x2c\x39\x52\x89\x98\xe7\x2c"
+			"\x99\xa4\x9e\xa3\x9b\x99\x46\x7a"
+			"\x3a\xdc\xa8\x59\xa3\xdb\xc3\x3b"
+			"\x95\x0d\x3b\x09\x6e\xee\x83\x5d"
+			"\x32\x4d\xed\xab\xfa\x98\x14\x4e"
+			"\xc3\x15\x45\x53\x61\xc4\x93\xbd"
+			"\x90\xf4\x99\x95\x4c\xe6\x76\x92"
+			"\x29\x90\x46\x30\x92\x69\x7d\x13"
+			"\xf2\xa5\xcd\x69\x49\x44\xb2\x0f"
+			"\x63\x40\x36\x5f\x09\xe2\x78\xf8"
+			"\x91\xe3\xe2\xfa\x10\xf7\xc8\x24"
+			"\xa8\x89\x32\x5c\x37\x25\x1d\xb2"
+			"\xea\x17\x8a\x0a\xa9\x64\xc3\x7c"
+			"\x3c\x7c\xbd\xc6\x79\x34\xe7\xe2"
+			"\x85\x8e\xbf\xf8\xde\x92\xa0\xae"
+			"\x20\xc4\xf6\xbb\x1f\x38\x19\x0e"
+			"\xe8\x79\x9c\xa1\x23\xe9\x54\x7e"
+			"\x37\x2f\xe2\x94\x32\xaf\xa0\x23"
+			"\x49\xe4\xc0\xb3\xac\x00\x8f\x36"
+			"\x05\xc4\xa6\x96\xec\x05\x98\x4f"
+			"\x96\x67\x57\x1f\x20\x86\x1b\x2d"
+			"\x69\xe4\x29\x93\x66\x5f\xaf\x6b"
+			"\x88\x26\x2c\x67\x02\x4b\x52\xd0"
+			"\x83\x7a\x43\x1f\xc0\x71\x15\x25"
+			"\x77\x65\x08\x60\x11\x76\x4c\x8d"
+			"\xed\xa9\x27\xc6\xb1\x2a\x2c\x6a"
+			"\x4a\x97\xf5\xc6\xb7\x70\x42\xd3"
+			"\x03\xd1\x24\x95\xec\x6d\xab\x38"
+			"\x72\xce\xe2\x8b\x33\xd7\x51\x09"
+			"\xdc\x45\xe0\x09\x96\x32\xf3\xc4"
+			"\x84\xdc\x73\x73\x2d\x1b\x11\x98"
+			"\xc5\x0e\x69\x28\x94\xc7\xb5\x4d"
+			"\xc8\x8a\xd0\xaa\x13\x2e\x18\x74"
+			"\xdd\xd1\x1e\xf3\x90\xe8\xfc\x9a"
+			"\x72\x4a\x0e\xd1\xe4\xfb\x0d\x96"
+			"\xd1\x0c\x79\x85\x1b\x1c\xfe\xe1"
+			"\x62\x8f\x7a\x73\x32\xab\xc8\x18"
+			"\x69\xe3\x34\x30\xdf\x13\xa6\xe5"
+			"\xe8\x0e\x67\x7f\x81\x11\xb4\x60"
+			"\xc7\xbd\x79\x65\x50\xdc\xc4\x5b"
+			"\xde\x39\xa4\x01\x72\x63\xf3\xd1"
+			"\x64\x4e\xdf\xfc\x27\x92\x37\x0d"
+			"\x57\xcd\x11\x4f\x11\x04\x8e\x1d"
+			"\x16\xf7\xcd\x92\x9a\x99\x30\x14"
+			"\xf1\x7c\x67\x1b\x1f\x41\x0b\xe8"
+			"\x32\xe8\xb8\xc1\x4f\x54\x86\x4f"
+			"\xe5\x79\x81\x73\xcd\x43\x59\x68"
+			"\x73\x02\x3b\x78\x21\x72\x43\x00"
+			"\x49\x17\xf7\x00\xaf\x68\x24\x53"
+			"\x05\x0a\xc3\x33\xe0\x33\x3f\x69"
+			"\xd2\x84\x2f\x0b\xed\xde\x04\xf4"
+			"\x11\x94\x13\x69\x51\x09\x28\xde"
+			"\x57\x5c\xef\xdc\x9a\x49\x1c\x17"
+			"\x97\xf3\x96\xc1\x7f\x5d\x2e\x7d"
+			"\x55\xb8\xb3\x02\x09\xb3\x1f\xe7"
+			"\xc9\x8d\xa3\x36\x34\x8a\x77\x13"
+			"\x30\x63\x4c\xa5\xcd\xc3\xe0\x7e"
+			"\x05\xa1\x7b\x0c\xcb\x74\x47\x31"
+			"\x62\x03\x43\xf1\x87\xb4\xb0\x85"
+			"\x87\x8e\x4b\x25\xc7\xcf\xae\x4b"
+			"\x36\x46\x3e\x62\xbc\x6f\xeb\x5f"
+			"\x73\xac\xe6\x07\xee\xc1\xa1\xd6"
+			"\xc4\xab\xc9\xd6\x89\x45\xe1\xf1"
+			"\x04\x4e\x1a\x6f\xbb\x4f\x3a\xa3"
+			"\xa0\xcb\xa3\x0a\xd8\x71\x35\x55"
+			"\xe4\xbc\x2e\x04\x06\xe6\xff\x5b"
+			"\x1c\xc0\x11\x7c\xc5\x17\xf3\x38"
+			"\xcf\xe9\xba\x0f\x0e\xef\x02\xc2"
+			"\x8d\xc6\xbc\x4b\x67\x20\x95\xd7"
+			"\x2c\x45\x5b\x86\x44\x8c\x6f\x2e"
+			"\x7e\x9f\x1c\x77\xba\x6b\x0e\xa3"
+			"\x69\xdc\xab\x24\x57\x60\x47\xc1"
+			"\xd1\xa5\x9d\x23\xe6\xb1\x37\xfe"
+			"\x93\xd2\x4c\x46\xf9\x0c\xc6\xfb"
+			"\xd6\x9d\x99\x69\xab\x7a\x07\x0c"
+			"\x65\xe7\xc4\x08\x96\xe2\xa5\x01"
+			"\x3f\x46\x07\x05\x7e\xe8\x9a\x90"
+			"\x50\xdc\xe9\x7a\xea\xa1\x39\x6e"
+			"\x66\xe4\x6f\xa5\x5f\xb2\xd9\x5b"
+			"\xf5\xdb\x2a\x32\xf0\x11\x6f\x7c"
+			"\x26\x10\x8f\x3d\x80\xe9\x58\xf7"
+			"\xe0\xa8\x57\xf8\xdb\x0e\xce\x99"
+			"\x63\x19\x3d\xd5\xec\x1b\x77\x69"
+			"\x98\xf6\xe4\x5f\x67\x17\x4b\x09"
+			"\x85\x62\x82\x70\x18\xe2\x9a\x78"
+			"\xe2\x62\xbd\xb4\xf1\x42\xc6\xfb"
+			"\x08\xd0\xbd\xeb\x4e\x09\xf2\xc8"
+			"\x1e\xdc\x3d\x32\x21\x56\x9c\x4f"
+			"\x35\xf3\x61\x06\x72\x84\xc4\x32"
+			"\xf2\xf1\xfa\x0b\x2f\xc3\xdb\x02"
+			"\x04\xc2\xde\x57\x64\x60\x8d\xcf"
+			"\xcb\x86\x5d\x97\x3e\xb1\x9c\x01"
+			"\xd6\x28\x8f\x99\xbc\x46\xeb\x05"
+			"\xaf\x7e\xb8\x21\x2a\x56\x85\x1c"
+			"\xb3\x71\xa0\xde\xca\x96\xf1\x78"
+			"\x49\xa2\x99\x81\x80\x5c\x01\xf5"
+			"\xa0\xa2\x56\x63\xe2\x70\x07\xa5"
+			"\x95\xd6\x85\xeb\x36\x9e\xa9\x51"
+			"\x66\x56\x5f\x1d\x02\x19\xe2\xf6"
+			"\x4f\x73\x38\x09\x75\x64\x48\xe0"
+			"\xf1\x7e\x0e\xe8\x9d\xf9\xed\x94"
+			"\xfe\x16\x26\x62\x49\x74\xf4\xb0"
+			"\xd4\xa9\x6c\xb0\xfd\x53\xe9\x81"
+			"\xe0\x7a\xbf\xcf\xb5\xc4\x01\x81"
+			"\x79\x99\x77\x01\x3b\xe9\xa2\xb6"
+			"\xe6\x6a\x8a\x9e\x56\x1c\x8d\x1e"
+			"\x8f\x06\x55\x2c\x6c\xdc\x92\x87"
+			"\x64\x3b\x4b\x19\xa1\x13\x64\x1d"
+			"\x4a\xe9\xc0\x00\xb8\x95\xef\x6b"
+			"\x1a\x86\x6d\x37\x52\x02\xc2\xe0"
+			"\xc8\xbb\x42\x0c\x02\x21\x4a\xc9"
+			"\xef\xa0\x54\xe4\x5e\x16\x53\x81"
+			"\x70\x62\x10\xaf\xde\xb8\xb5\xd3"
+			"\xe8\x5e\x6c\xc3\x8a\x3e\x18\x07"
+			"\xf2\x2f\x7d\xa7\xe1\x3d\x4e\xb4"
+			"\x26\xa7\xa3\x93\x86\xb2\x04\x1e"
+			"\x53\x5d\x86\xd6\xde\x65\xca\xe3"
+			"\x4e\xc1\xcf\xef\xc8\x70\x1b\x83"
+			"\x13\xdd\x18\x8b\x0d\x76\xd2\xf6"
+			"\x37\x7a\x93\x7a\x50\x11\x9f\x96"
+			"\x86\x25\xfd\xac\xdc\xbe\x18\x93"
+			"\x19\x6b\xec\x58\x4f\xb9\x75\xa7"
+			"\xdd\x3f\x2f\xec\xc8\x5a\x84\xab"
+			"\xd5\xe4\x8a\x07\xf6\x4d\x23\xd6"
+			"\x03\xfb\x03\x6a\xea\x66\xbf\xd4"
+			"\xb1\x34\xfb\x78\xe9\x55\xdc\x7c"
+			"\x3d\x9c\xe5\x9a\xac\xc3\x7a\x80"
+			"\x24\x6d\xa0\xef\x25\x7c\xb7\xea"
+			"\xce\x4d\x5f\x18\x60\xce\x87\x22"
+			"\x66\x2f\xd5\xdd\xdd\x02\x21\x75"
+			"\x82\xa0\x1f\x58\xc6\xd3\x62\xf7"
+			"\x32\xd8\xaf\x1e\x07\x77\x51\x96"
+			"\xd5\x6b\x1e\x7e\x80\x02\xe8\x67"
+			"\xea\x17\x0b\x10\xd2\x3f\x28\x25"
+			"\x4f\x05\x77\x02\x14\x69\xf0\x2c"
+			"\xbe\x0c\xf1\x74\x30\xd1\xb9\x9b"
+			"\xfc\x8c\xbb\x04\x16\xd9\xba\xc3"
+			"\xbc\x91\x8a\xc4\x30\xa4\xb0\x12"
+			"\x4c\x21\x87\xcb\xc9\x1d\x16\x96"
+			"\x07\x6f\x23\x54\xb9\x6f\x79\xe5"
+			"\x64\xc0\x64\xda\xb1\xae\xdd\x60"
+			"\x6c\x1a\x9d\xd3\x04\x8e\x45\xb0"
+			"\x92\x61\xd0\x48\x81\xed\x5e\x1d"
+			"\xa0\xc9\xa4\x33\xc7\x13\x51\x5d"
+			"\x7f\x83\x73\xb6\x70\x18\x65\x3e"
+			"\x2f\x0e\x7a\x12\x39\x98\xab\xd8"
+			"\x7e\x6f\xa3\xd1\xba\x56\xad\xbd"
+			"\xf0\x03\x01\x1c\x85\x35\x9f\xeb"
+			"\x19\x63\xa1\xaf\xfe\x2d\x35\x50"
+			"\x39\xa0\x65\x7c\x95\x7e\x6b\xfe"
+			"\xc1\xac\x07\x7c\x98\x4f\xbe\x57"
+			"\xa7\x22\xec\xe2\x7e\x29\x09\x53"
+			"\xe8\xbf\xb4\x7e\x3f\x8f\xfc\x14"
+			"\xce\x54\xf9\x18\x58\xb5\xff\x44"
+			"\x05\x9d\xce\x1b\xb6\x82\x23\xc8"
+			"\x2e\xbc\x69\xbb\x4a\x29\x0f\x65"
+			"\x94\xf0\x63\x06\x0e\xef\x8c\xbd"
+			"\xff\xfd\xb0\x21\x6e\x57\x05\x75"
+			"\xda\xd5\xc4\xeb\x8d\x32\xf7\x50"
+			"\xd3\x6f\x22\xed\x5f\x8e\xa2\x5b"
+			"\x80\x8c\xc8\x78\x40\x24\x4b\x89"
+			"\x30\xce\x7a\x97\x0e\xc4\xaf\xef"
+			"\x9b\xb4\xcd\x66\x74\x14\x04\x2b"
+			"\xf7\xce\x0b\x1c\x6e\xc2\x78\x8c"
+			"\xca\xc5\xd0\x1c\x95\x4a\x91\x2d"
+			"\xa7\x20\xeb\x86\x52\xb7\x67\xd8"
+			"\x0c\xd6\x04\x14\xde\x51\x74\x75"
+			"\xe7\x11\xb4\x87\xa3\x3d\x2d\xad"
+			"\x4f\xef\xa0\x0f\x70\x00\x6d\x13"
+			"\x19\x1d\x41\x50\xe9\xd8\xf0\x32"
+			"\x71\xbc\xd3\x11\xf2\xac\xbe\xaf"
+			"\x75\x46\x65\x4e\x07\x34\x37\xa3"
+			"\x89\xfe\x75\xd4\x70\x4c\xc6\x3f"
+			"\x69\x24\x0e\x38\x67\x43\x8c\xde"
+			"\x06\xb5\xb8\xe7\xc4\xf0\x41\x8f"
+			"\xf0\xbd\x2f\x0b\xb9\x18\xf8\xde"
+			"\x64\xb1\xdb\xee\x00\x50\x77\xe1"
+			"\xc7\xff\xa6\xfa\xdd\x70\xf4\xe3"
+			"\x93\xe9\x77\x35\x3d\x4b\x2f\x2b"
+			"\x6d\x55\xf0\xfc\x88\x54\x4e\x89"
+			"\xc1\x8a\x23\x31\x2d\x14\x2a\xb8"
+			"\x1b\x15\xdd\x9e\x6e\x7b\xda\x05"
+			"\x91\x7d\x62\x64\x96\x72\xde\xfc"
+			"\xc1\xec\xf0\x23\x51\x6f\xdb\x5b"
+			"\x1d\x08\x57\xce\x09\xb8\xf6\xcd"
+			"\x8d\x95\xf2\x20\xbf\x0f\x20\x57"
+			"\x98\x81\x84\x4f\x15\x5c\x76\xe7"
+			"\x3e\x0a\x3a\x6c\xc4\x8a\xbe\x78"
+			"\x74\x77\xc3\x09\x4b\x5d\x48\xe4"
+			"\xc8\xcb\x0b\xea\x17\x28\xcf\xcf"
+			"\x31\x32\x44\xa4\xe5\x0e\x1a\x98"
+			"\x94\xc4\xf0\xff\xae\x3e\x44\xe8"
+			"\xa5\xb3\xb5\x37\x2f\xe8\xaf\x6f"
+			"\x28\xc1\x37\x5f\x31\xd2\xb9\x33"
+			"\xb1\xb2\x52\x94\x75\x2c\x29\x59"
+			"\x06\xc2\x25\xe8\x71\x65\x4e\xed"
+			"\xc0\x9c\xb1\xbb\x25\xdc\x6c\xe7"
+			"\x4b\xa5\x7a\x54\x7a\x60\xff\x7a"
+			"\xe0\x50\x40\x96\x35\x63\xe4\x0b"
+			"\x76\xbd\xa4\x65\x00\x1b\x57\x88"
+			"\xae\xed\x39\x88\x42\x11\x3c\xed"
+			"\x85\x67\x7d\xb9\x68\x82\xe9\x43"
+			"\x3c\x47\x53\xfa\xe8\xf8\x9f\x1f"
+			"\x9f\xef\x0f\xf7\x30\xd9\x30\x0e"
+			"\xb9\x9f\x69\x18\x2f\x7e\xf8\xf8"
+			"\xf8\x8c\x0f\xd4\x02\x4d\xea\xcd"
+			"\x0a\x9c\x6f\x71\x6d\x5a\x4c\x60"
+			"\xce\x20\x56\x32\xc6\xc5\x99\x1f"
+			"\x09\xe6\x4e\x18\x1a\x15\x13\xa8"
+			"\x7d\xb1\x6b\xc0\xb2\x6d\xf8\x26"
+			"\x66\xf8\x3d\x18\x74\x70\x66\x7a"
+			"\x34\x17\xde\xba\x47\xf1\x06\x18"
+			"\xcb\xaf\xeb\x4a\x1e\x8f\xa7\x77"
+			"\xe0\x3b\x78\x62\x66\xc9\x10\xea"
+			"\x1f\xb7\x29\x0a\x45\xa1\x1d\x1e"
+			"\x1d\xe2\x65\x61\x50\x9c\xd7\x05"
+			"\xf2\x0b\x5b\x12\x61\x02\xc8\xe5"
+			"\x63\x4f\x20\x0c\x07\x17\x33\x5e"
+			"\x03\x9a\x53\x0f\x2e\x55\xfe\x50"
+			"\x43\x7d\xd0\xb6\x7e\x5a\xda\xae"
+			"\x58\xef\x15\xa9\x83\xd9\x46\xb1"
+			"\x42\xaa\xf5\x02\x6c\xce\x92\x06"
+			"\x1b\xdb\x66\x45\x91\x79\xc2\x2d"
+			"\xe6\x53\xd3\x14\xfd\xbb\x44\x63"
+			"\xc6\xd7\x3d\x7a\x0c\x75\x78\x9d"
+			"\x5c\xa6\x39\xb3\xe5\x63\xca\x8b"
+			"\xfe\xd3\xef\x60\x83\xf6\x8e\x70"
+			"\xb6\x67\xc7\x77\xed\x23\xef\x4c"
+			"\xf0\xed\x2d\x07\x59\x6f\xc1\x01"
+			"\x34\x37\x08\xab\xd9\x1f\x09\xb1"
+			"\xce\x5b\x17\xff\x74\xf8\x9c\xd5"
+			"\x2c\x56\x39\x79\x0f\x69\x44\x75"
+			"\x58\x27\x01\xc4\xbf\xa7\xa1\x1d"
+			"\x90\x17\x77\x86\x5a\x3f\xd9\xd1"
+			"\x0e\xa0\x10\xf8\xec\x1e\xa5\x7f"
+			"\x5e\x36\xd1\xe3\x04\x2c\x70\xf7"
+			"\x8e\xc0\x98\x2f\x6c\x94\x2b\x41"
+			"\xb7\x60\x00\xb7\x2e\xb8\x02\x8d"
+			"\xb8\xb0\xd3\x86\xba\x1d\xd7\x90"
+			"\xd6\xb6\xe1\xfc\xd7\xd8\x28\x06"
+			"\x63\x9b\xce\x61\x24\x79\xc0\x70"
+			"\x52\xd0\xb6\xd4\x28\x95\x24\x87"
+			"\x03\x1f\xb7\x9a\xda\xa3\xfb\x52"
+			"\x5b\x68\xe7\x4c\x8c\x24\xe1\x42"
+			"\xf7\xd5\xfd\xad\x06\x32\x9f\xba"
+			"\xc1\xfc\xdd\xc6\xfc\xfc\xb3\x38"
+			"\x74\x56\x58\x40\x02\x37\x52\x2c"
+			"\x55\xcc\xb3\x9e\x7a\xe9\xd4\x38"
+			"\x41\x5e\x0c\x35\xe2\x11\xd1\x13"
+			"\xf8\xb7\x8d\x72\x6b\x22\x2a\xb0"
+			"\xdb\x08\xba\x35\xb9\x3f\xc8\xd3"
+			"\x24\x90\xec\x58\xd2\x09\xc7\x2d"
+			"\xed\x38\x80\x36\x72\x43\x27\x49"
+			"\x4a\x80\x8a\xa2\xe8\xd3\xda\x30"
+			"\x7d\xb6\x82\x37\x86\x92\x86\x3e"
+			"\x08\xb2\x28\x5a\x55\x44\x24\x7d"
+			"\x40\x48\x8a\xb6\x89\x58\x08\xa0"
+			"\xd6\x6d\x3a\x17\xbf\xf6\x54\xa2"
+			"\xf5\xd3\x8c\x0f\x78\x12\x57\x8b"
+			"\xd5\xc2\xfd\x58\x5b\x7f\x38\xe3"
+			"\xcc\xb7\x7c\x48\xb3\x20\xe8\x81"
+			"\x14\x32\x45\x05\xe0\xdb\x9f\x75"
+			"\x85\xb4\x6a\xfc\x95\xe3\x54\x22"
+			"\x12\xee\x30\xfe\xd8\x30\xef\x34"
+			"\x50\xab\x46\x30\x98\x2f\xb7\xc0"
+			"\x15\xa2\x83\xb6\xf2\x06\x21\xa2"
+			"\xc3\x26\x37\x14\xd1\x4d\xb5\x10"
+			"\x52\x76\x4d\x6a\xee\xb5\x2b\x15"
+			"\xb7\xf9\x51\xe8\x2a\xaf\xc7\xfa"
+			"\x77\xaf\xb0\x05\x4d\xd1\x68\x8e"
+			"\x74\x05\x9f\x9d\x93\xa5\x3e\x7f"
+			"\x4e\x5f\x9d\xcb\x09\xc7\x83\xe3"
+			"\x02\x9d\x27\x1f\xef\x85\x05\x8d"
+			"\xec\x55\x88\x0f\x0d\x7c\x4c\xe8"
+			"\xa1\x75\xa0\xd8\x06\x47\x14\xef"
+			"\xaa\x61\xcf\x26\x15\xad\xd8\xa3"
+			"\xaa\x75\xf2\x78\x4a\x5a\x61\xdf"
+			"\x8b\xc7\x04\xbc\xb2\x32\xd2\x7e"
+			"\x42\xee\xb4\x2f\x51\xff\x7b\x2e"
+			"\xd3\x02\xe8\xdc\x5d\x0d\x50\xdc"
+			"\xae\xb7\x46\xf9\xa8\xe6\xd0\x16"
+			"\xcc\xe6\x2c\x81\xc7\xad\xe9\xf0"
+			"\x05\x72\x6d\x3d\x0a\x7a\xa9\x02"
+			"\xac\x82\x93\x6e\xb6\x1c\x28\xfc"
+			"\x44\x12\xfb\x73\x77\xd4\x13\x39"
+			"\x29\x88\x8a\xf3\x5c\xa6\x36\xa0"
+			"\x2a\xed\x7e\xb1\x1d\xd6\x4c\x6b"
+			"\x41\x01\x18\x5d\x5d\x07\x97\xa6"
+			"\x4b\xef\x31\x18\xea\xac\xb1\x84"
+			"\x21\xed\xda\x86",
+		.len	= 4100,
+		.np	= 2,
+		.tap	= { 4064, 36 },
+	},
+};
+
+static const struct cipher_testvec aes_ofb_tv_template[] = {
+	 /* From NIST Special Publication 800-38A, Appendix F.5 */
+	{
+		.key	= "\x2b\x7e\x15\x16\x28\xae\xd2\xa6"
+			  "\xab\xf7\x15\x88\x09\xcf\x4f\x3c",
+		.klen	= 16,
+		.iv	= "\x00\x01\x02\x03\x04\x05\x06\x07\x08"
+			  "\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.ptext	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.ctext	= "\x3b\x3f\xd9\x2e\xb7\x2d\xad\x20"
+			  "\x33\x34\x49\xf8\xe8\x3c\xfb\x4a"
+			  "\x77\x89\x50\x8d\x16\x91\x8f\x03\xf5"
+			  "\x3c\x52\xda\xc5\x4e\xd8\x25"
+			  "\x97\x40\x05\x1e\x9c\x5f\xec\xf6\x43"
+			  "\x44\xf7\xa8\x22\x60\xed\xcc"
+			  "\x30\x4c\x65\x28\xf6\x59\xc7\x78"
+			  "\x66\xa5\x10\xd9\xc1\xd6\xae\x5e",
+		.len	= 64,
+	}
+};
+
+static const struct aead_testvec aes_gcm_enc_tv_template[] = {
+	{ /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */
+		.key    = zeroed_string,
+		.klen	= 16,
+		.result	= "\x58\xe2\xfc\xce\xfa\x7e\x30\x61"
+			  "\x36\x7f\x1d\x57\xa4\xe7\x45\x5a",
+		.rlen	= 16,
+	}, {
+		.key    = zeroed_string,
+		.klen	= 16,
+		.input  = zeroed_string,
+		.ilen	= 16,
+		.result = "\x03\x88\xda\xce\x60\xb6\xa3\x92"
+			  "\xf3\x28\xc2\xb9\x71\xb2\xfe\x78"
+			  "\xab\x6e\x47\xd4\x2c\xec\x13\xbd"
+			  "\xf5\x3a\x67\xb2\x12\x57\xbd\xdf",
+		.rlen	= 32,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
+		.klen	= 16,
+		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
+			  "\xde\xca\xf8\x88",
+		.input	= "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
+			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
+			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
+			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
+			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
+			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
+			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
+			  "\xba\x63\x7b\x39\x1a\xaf\xd2\x55",
+		.ilen	= 64,
+		.result = "\x42\x83\x1e\xc2\x21\x77\x74\x24"
+			  "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c"
+			  "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0"
+			  "\x35\xc1\x7e\x23\x29\xac\xa1\x2e"
+			  "\x21\xd5\x14\xb2\x54\x66\x93\x1c"
+			  "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05"
+			  "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97"
+			  "\x3d\x58\xe0\x91\x47\x3f\x59\x85"
+			  "\x4d\x5c\x2a\xf3\x27\xcd\x64\xa6"
+			  "\x2c\xf3\x5a\xbd\x2b\xa6\xfa\xb4",
+		.rlen	= 80,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
+		.klen	= 16,
+		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
+			  "\xde\xca\xf8\x88",
+		.input	= "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
+			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
+			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
+			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
+			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
+			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
+			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
+			  "\xba\x63\x7b\x39",
+		.ilen	= 60,
+		.assoc	= "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
+			  "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
+			  "\xab\xad\xda\xd2",
+		.alen	= 20,
+		.result = "\x42\x83\x1e\xc2\x21\x77\x74\x24"
+			  "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c"
+			  "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0"
+			  "\x35\xc1\x7e\x23\x29\xac\xa1\x2e"
+			  "\x21\xd5\x14\xb2\x54\x66\x93\x1c"
+			  "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05"
+			  "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97"
+			  "\x3d\x58\xe0\x91"
+			  "\x5b\xc9\x4f\xbc\x32\x21\xa5\xdb"
+			  "\x94\xfa\xe9\x5a\xe7\x12\x1a\x47",
+		.rlen	= 76,
+	}, {
+		.key    = zeroed_string,
+		.klen	= 24,
+		.result	= "\xcd\x33\xb2\x8a\xc7\x73\xf7\x4b"
+			  "\xa0\x0e\xd1\xf3\x12\x57\x24\x35",
+		.rlen	= 16,
+	}, {
+		.key    = zeroed_string,
+		.klen	= 24,
+		.input  = zeroed_string,
+		.ilen	= 16,
+		.result = "\x98\xe7\x24\x7c\x07\xf0\xfe\x41"
+			  "\x1c\x26\x7e\x43\x84\xb0\xf6\x00"
+			  "\x2f\xf5\x8d\x80\x03\x39\x27\xab"
+			  "\x8e\xf4\xd4\x58\x75\x14\xf0\xfb",
+		.rlen	= 32,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c",
+		.klen	= 24,
+		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
+			  "\xde\xca\xf8\x88",
+		.input	= "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
+			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
+			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
+			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
+			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
+			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
+			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
+			  "\xba\x63\x7b\x39\x1a\xaf\xd2\x55",
+		.ilen	= 64,
+		.result = "\x39\x80\xca\x0b\x3c\x00\xe8\x41"
+			  "\xeb\x06\xfa\xc4\x87\x2a\x27\x57"
+			  "\x85\x9e\x1c\xea\xa6\xef\xd9\x84"
+			  "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c"
+			  "\x7d\x77\x3d\x00\xc1\x44\xc5\x25"
+			  "\xac\x61\x9d\x18\xc8\x4a\x3f\x47"
+			  "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9"
+			  "\xcc\xda\x27\x10\xac\xad\xe2\x56"
+			  "\x99\x24\xa7\xc8\x58\x73\x36\xbf"
+			  "\xb1\x18\x02\x4d\xb8\x67\x4a\x14",
+		.rlen	= 80,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c",
+		.klen	= 24,
+		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
+			  "\xde\xca\xf8\x88",
+		.input	= "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
+			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
+			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
+			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
+			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
+			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
+			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
+			  "\xba\x63\x7b\x39",
+		.ilen	= 60,
+		.assoc	= "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
+			  "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
+			  "\xab\xad\xda\xd2",
+		.alen	= 20,
+		.result = "\x39\x80\xca\x0b\x3c\x00\xe8\x41"
+			  "\xeb\x06\xfa\xc4\x87\x2a\x27\x57"
+			  "\x85\x9e\x1c\xea\xa6\xef\xd9\x84"
+			  "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c"
+			  "\x7d\x77\x3d\x00\xc1\x44\xc5\x25"
+			  "\xac\x61\x9d\x18\xc8\x4a\x3f\x47"
+			  "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9"
+			  "\xcc\xda\x27\x10"
+			  "\x25\x19\x49\x8e\x80\xf1\x47\x8f"
+			  "\x37\xba\x55\xbd\x6d\x27\x61\x8c",
+		.rlen	= 76,
+		.np	= 2,
+		.tap	= { 32, 28 },
+		.anp	= 2,
+		.atap	= { 8, 12 }
+	}, {
+		.key    = zeroed_string,
+		.klen	= 32,
+		.result	= "\x53\x0f\x8a\xfb\xc7\x45\x36\xb9"
+			  "\xa9\x63\xb4\xf1\xc4\xcb\x73\x8b",
+		.rlen	= 16,
+	}
+};
+
+static const struct aead_testvec aes_gcm_dec_tv_template[] = {
+	{ /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */
+		.key    = zeroed_string,
+		.klen	= 32,
+		.input	= "\xce\xa7\x40\x3d\x4d\x60\x6b\x6e"
+			  "\x07\x4e\xc5\xd3\xba\xf3\x9d\x18"
+			  "\xd0\xd1\xc8\xa7\x99\x99\x6b\xf0"
+			  "\x26\x5b\x98\xb5\xd4\x8a\xb9\x19",
+		.ilen	= 32,
+		.result  = zeroed_string,
+		.rlen	= 16,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
+		.klen	= 32,
+		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
+			  "\xde\xca\xf8\x88",
+		.input	= "\x52\x2d\xc1\xf0\x99\x56\x7d\x07"
+			  "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d"
+			  "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9"
+			  "\x75\x98\xa2\xbd\x25\x55\xd1\xaa"
+			  "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d"
+			  "\xa7\xb0\x8b\x10\x56\x82\x88\x38"
+			  "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a"
+			  "\xbc\xc9\xf6\x62\x89\x80\x15\xad"
+			  "\xb0\x94\xda\xc5\xd9\x34\x71\xbd"
+			  "\xec\x1a\x50\x22\x70\xe3\xcc\x6c",
+		.ilen	= 80,
+		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
+			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
+			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
+			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
+			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
+			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
+			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
+			  "\xba\x63\x7b\x39\x1a\xaf\xd2\x55",
+		.rlen	= 64,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
+		.klen	= 32,
+		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
+			  "\xde\xca\xf8\x88",
+		.input	= "\x52\x2d\xc1\xf0\x99\x56\x7d\x07"
+			  "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d"
+			  "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9"
+			  "\x75\x98\xa2\xbd\x25\x55\xd1\xaa"
+			  "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d"
+			  "\xa7\xb0\x8b\x10\x56\x82\x88\x38"
+			  "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a"
+			  "\xbc\xc9\xf6\x62"
+			  "\x76\xfc\x6e\xce\x0f\x4e\x17\x68"
+			  "\xcd\xdf\x88\x53\xbb\x2d\x55\x1b",
+		.ilen	= 76,
+		.assoc	= "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
+			  "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
+			  "\xab\xad\xda\xd2",
+		.alen	= 20,
+		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
+			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
+			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
+			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
+			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
+			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
+			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
+			  "\xba\x63\x7b\x39",
+		.rlen	= 60,
+		.np     = 2,
+		.tap    = { 48, 28 },
+		.anp	= 3,
+		.atap	= { 8, 8, 4 }
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
+		.klen	= 16,
+		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
+			  "\xde\xca\xf8\x88",
+		.input	= "\x42\x83\x1e\xc2\x21\x77\x74\x24"
+			  "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c"
+			  "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0"
+			  "\x35\xc1\x7e\x23\x29\xac\xa1\x2e"
+			  "\x21\xd5\x14\xb2\x54\x66\x93\x1c"
+			  "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05"
+			  "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97"
+			  "\x3d\x58\xe0\x91\x47\x3f\x59\x85"
+			  "\x4d\x5c\x2a\xf3\x27\xcd\x64\xa6"
+			  "\x2c\xf3\x5a\xbd\x2b\xa6\xfa\xb4",
+		.ilen	= 80,
+		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
+			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
+			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
+			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
+			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
+			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
+			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
+			  "\xba\x63\x7b\x39\x1a\xaf\xd2\x55",
+		.rlen	= 64,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08",
+		.klen	= 16,
+		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
+			  "\xde\xca\xf8\x88",
+		.input	= "\x42\x83\x1e\xc2\x21\x77\x74\x24"
+			  "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c"
+			  "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0"
+			  "\x35\xc1\x7e\x23\x29\xac\xa1\x2e"
+			  "\x21\xd5\x14\xb2\x54\x66\x93\x1c"
+			  "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05"
+			  "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97"
+			  "\x3d\x58\xe0\x91"
+			  "\x5b\xc9\x4f\xbc\x32\x21\xa5\xdb"
+			  "\x94\xfa\xe9\x5a\xe7\x12\x1a\x47",
+		.ilen	= 76,
+		.assoc	= "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
+			  "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
+			  "\xab\xad\xda\xd2",
+		.alen	= 20,
+		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
+			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
+			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
+			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
+			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
+			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
+			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
+			  "\xba\x63\x7b\x39",
+		.rlen	= 60,
+	}, {
+		.key    = zeroed_string,
+		.klen	= 24,
+		.input	= "\x98\xe7\x24\x7c\x07\xf0\xfe\x41"
+			  "\x1c\x26\x7e\x43\x84\xb0\xf6\x00"
+			  "\x2f\xf5\x8d\x80\x03\x39\x27\xab"
+			  "\x8e\xf4\xd4\x58\x75\x14\xf0\xfb",
+		.ilen	= 32,
+		.result  = zeroed_string,
+		.rlen	= 16,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c",
+		.klen	= 24,
+		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
+			  "\xde\xca\xf8\x88",
+		.input	= "\x39\x80\xca\x0b\x3c\x00\xe8\x41"
+			  "\xeb\x06\xfa\xc4\x87\x2a\x27\x57"
+			  "\x85\x9e\x1c\xea\xa6\xef\xd9\x84"
+			  "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c"
+			  "\x7d\x77\x3d\x00\xc1\x44\xc5\x25"
+			  "\xac\x61\x9d\x18\xc8\x4a\x3f\x47"
+			  "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9"
+			  "\xcc\xda\x27\x10\xac\xad\xe2\x56"
+			  "\x99\x24\xa7\xc8\x58\x73\x36\xbf"
+			  "\xb1\x18\x02\x4d\xb8\x67\x4a\x14",
+		.ilen	= 80,
+		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
+			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
+			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
+			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
+			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
+			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
+			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
+			  "\xba\x63\x7b\x39\x1a\xaf\xd2\x55",
+		.rlen	= 64,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\xfe\xff\xe9\x92\x86\x65\x73\x1c",
+		.klen	= 24,
+		.iv	= "\xca\xfe\xba\xbe\xfa\xce\xdb\xad"
+			  "\xde\xca\xf8\x88",
+		.input	= "\x39\x80\xca\x0b\x3c\x00\xe8\x41"
+			  "\xeb\x06\xfa\xc4\x87\x2a\x27\x57"
+			  "\x85\x9e\x1c\xea\xa6\xef\xd9\x84"
+			  "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c"
+			  "\x7d\x77\x3d\x00\xc1\x44\xc5\x25"
+			  "\xac\x61\x9d\x18\xc8\x4a\x3f\x47"
+			  "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9"
+			  "\xcc\xda\x27\x10"
+			  "\x25\x19\x49\x8e\x80\xf1\x47\x8f"
+			  "\x37\xba\x55\xbd\x6d\x27\x61\x8c",
+		.ilen	= 76,
+		.assoc	= "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
+			  "\xfe\xed\xfa\xce\xde\xad\xbe\xef"
+			  "\xab\xad\xda\xd2",
+		.alen	= 20,
+		.result = "\xd9\x31\x32\x25\xf8\x84\x06\xe5"
+			  "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a"
+			  "\x86\xa7\xa9\x53\x15\x34\xf7\xda"
+			  "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72"
+			  "\x1c\x3c\x0c\x95\x95\x68\x09\x53"
+			  "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25"
+			  "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57"
+			  "\xba\x63\x7b\x39",
+		.rlen	= 60,
+	}
+};
+
+static const struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = {
+	{ /* Generated using Crypto++ */
+		.key    = zeroed_string,
+		.klen	= 20,
+		.iv	= zeroed_string,
+		.input  = zeroed_string,
+		.ilen   = 16,
+		.assoc  = zeroed_string,
+		.alen   = 16,
+		.result	= "\x03\x88\xDA\xCE\x60\xB6\xA3\x92"
+			  "\xF3\x28\xC2\xB9\x71\xB2\xFE\x78"
+			  "\x97\xFE\x4C\x23\x37\x42\x01\xE0"
+			  "\x81\x9F\x8D\xC5\xD7\x41\xA0\x1B",
+		.rlen	= 32,
+	},{
+		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.input  = zeroed_string,
+		.ilen   = 16,
+		.assoc  = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen   = 16,
+		.result	= "\xC0\x0D\x8B\x42\x0F\x8F\x34\x18"
+			  "\x88\xB1\xC5\xBC\xC5\xB6\xD6\x28"
+			  "\x6A\x9D\xDF\x11\x5E\xFE\x5E\x9D"
+			  "\x2F\x70\x44\x92\xF7\xF2\xE3\xEF",
+		.rlen	= 32,
+
+	}, {
+		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = zeroed_string,
+		.input  = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.ilen   = 16,
+		.assoc  = zeroed_string,
+		.alen   = 16,
+		.result	= "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
+			  "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
+			  "\x0B\x8F\x88\x69\x17\xE6\xB4\x3C"
+			  "\xB1\x68\xFD\x14\x52\x64\x61\xB2",
+		.rlen	= 32,
+	}, {
+		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = zeroed_string,
+		.input  = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.ilen   = 16,
+		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen   = 16,
+		.result	= "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
+			  "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
+			  "\x90\x92\xB7\xE3\x5F\xA3\x9A\x63"
+			  "\x7E\xD7\x1F\xD8\xD3\x7C\x4B\xF5",
+		.rlen	= 32,
+	}, {
+		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.input  = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.ilen   = 16,
+		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen   = 16,
+		.result	= "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
+			  "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
+			  "\x64\x50\xF9\x32\x13\xFB\x74\x61"
+			  "\xF4\xED\x52\xD3\xC5\x10\x55\x3C",
+		.rlen	= 32,
+	}, {
+		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.input  = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.ilen   = 64,
+		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen   = 16,
+		.result	= "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
+			  "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
+			  "\x98\x14\xA1\x42\x37\x80\xFD\x90"
+			  "\x68\x12\x01\xA8\x91\x89\xB9\x83"
+			  "\x5B\x11\x77\x12\x9B\xFF\x24\x89"
+			  "\x94\x5F\x18\x12\xBA\x27\x09\x39"
+			  "\x99\x96\x76\x42\x15\x1C\xCD\xCB"
+			  "\xDC\xD3\xDA\x65\x73\xAF\x80\xCD"
+			  "\xD2\xB6\xC2\x4A\x76\xC2\x92\x85"
+			  "\xBD\xCF\x62\x98\x58\x14\xE5\xBD",
+		.rlen	= 80,
+	}, {
+		.key    = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = "\x00\x00\x45\x67\x89\xab\xcd\xef",
+		.input  = "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff",
+		.ilen   = 192,
+		.assoc  = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
+			  "\xaa\xaa\xaa\xaa\x00\x00\x45\x67"
+			  "\x89\xab\xcd\xef",
+		.alen   = 20,
+		.result	= "\xC1\x76\x33\x85\xE2\x9B\x5F\xDE"
+			  "\xDE\x89\x3D\x42\xE7\xC9\x69\x8A"
+			  "\x44\x6D\xC3\x88\x46\x2E\xC2\x01"
+			  "\x5E\xF6\x0C\x39\xF0\xC4\xA5\x82"
+			  "\xCD\xE8\x31\xCC\x0A\x4C\xE4\x44"
+			  "\x41\xA9\x82\x6F\x22\xA1\x23\x1A"
+			  "\xA8\xE3\x16\xFD\x31\x5C\x27\x31"
+			  "\xF1\x7F\x01\x63\xA3\xAF\x70\xA1"
+			  "\xCF\x07\x57\x41\x67\xD0\xC4\x42"
+			  "\xDB\x18\xC6\x4C\x4C\xE0\x3D\x9F"
+			  "\x05\x07\xFB\x13\x7D\x4A\xCA\x5B"
+			  "\xF0\xBF\x64\x7E\x05\xB1\x72\xEE"
+			  "\x7C\x3B\xD4\xCD\x14\x03\xB2\x2C"
+			  "\xD3\xA9\xEE\xFA\x17\xFC\x9C\xDF"
+			  "\xC7\x75\x40\xFF\xAE\xAD\x1E\x59"
+			  "\x2F\x30\x24\xFB\xAD\x6B\x10\xFA"
+			  "\x6C\x9F\x5B\xE7\x25\xD5\xD0\x25"
+			  "\xAC\x4A\x4B\xDA\xFC\x7A\x85\x1B"
+			  "\x7E\x13\x06\x82\x08\x17\xA4\x35"
+			  "\xEC\xC5\x8D\x63\x96\x81\x0A\x8F"
+			  "\xA3\x05\x38\x95\x20\x1A\x47\x04"
+			  "\x6F\x6D\xDA\x8F\xEF\xC1\x76\x35"
+			  "\x6B\xC7\x4D\x0F\x94\x12\xCA\x3E"
+			  "\x2E\xD5\x03\x2E\x86\x7E\xAA\x3B"
+			  "\x37\x08\x1C\xCF\xBA\x5D\x71\x46"
+			  "\x80\x72\xB0\x4C\x82\x0D\x60\x3C",
+		.rlen	= 208,
+	}, { /* From draft-mcgrew-gcm-test-01 */
+		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
+			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
+			  "\x2E\x44\x3B\x68",
+		.klen	= 20,
+		.iv	= "\x49\x56\xED\x7E\x3B\x24\x4C\xFE",
+		.input	= "\x45\x00\x00\x48\x69\x9A\x00\x00"
+			  "\x80\x11\x4D\xB7\xC0\xA8\x01\x02"
+			  "\xC0\xA8\x01\x01\x0A\x9B\xF1\x56"
+			  "\x38\xD3\x01\x00\x00\x01\x00\x00"
+			  "\x00\x00\x00\x00\x04\x5F\x73\x69"
+			  "\x70\x04\x5F\x75\x64\x70\x03\x73"
+			  "\x69\x70\x09\x63\x79\x62\x65\x72"
+			  "\x63\x69\x74\x79\x02\x64\x6B\x00"
+			  "\x00\x21\x00\x01\x01\x02\x02\x01",
+		.ilen	= 72,
+		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
+			  "\x00\x00\x00\x00\x49\x56\xED\x7E"
+			  "\x3B\x24\x4C\xFE",
+		.alen	= 20,
+		.result	= "\xFE\xCF\x53\x7E\x72\x9D\x5B\x07"
+			  "\xDC\x30\xDF\x52\x8D\xD2\x2B\x76"
+			  "\x8D\x1B\x98\x73\x66\x96\xA6\xFD"
+			  "\x34\x85\x09\xFA\x13\xCE\xAC\x34"
+			  "\xCF\xA2\x43\x6F\x14\xA3\xF3\xCF"
+			  "\x65\x92\x5B\xF1\xF4\xA1\x3C\x5D"
+			  "\x15\xB2\x1E\x18\x84\xF5\xFF\x62"
+			  "\x47\xAE\xAB\xB7\x86\xB9\x3B\xCE"
+			  "\x61\xBC\x17\xD7\x68\xFD\x97\x32"
+			  "\x45\x90\x18\x14\x8F\x6C\xBE\x72"
+			  "\x2F\xD0\x47\x96\x56\x2D\xFD\xB4",
+		.rlen	= 88,
+	}, {
+		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
+			  "\xCA\xFE\xBA\xBE",
+		.klen	= 20,
+		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.input	= "\x45\x00\x00\x3E\x69\x8F\x00\x00"
+			  "\x80\x11\x4D\xCC\xC0\xA8\x01\x02"
+			  "\xC0\xA8\x01\x01\x0A\x98\x00\x35"
+			  "\x00\x2A\x23\x43\xB2\xD0\x01\x00"
+			  "\x00\x01\x00\x00\x00\x00\x00\x00"
+			  "\x03\x73\x69\x70\x09\x63\x79\x62"
+			  "\x65\x72\x63\x69\x74\x79\x02\x64"
+			  "\x6B\x00\x00\x01\x00\x01\x00\x01",
+		.ilen	= 64,
+		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
+			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.alen	= 16,
+		.result	= "\xDE\xB2\x2C\xD9\xB0\x7C\x72\xC1"
+			  "\x6E\x3A\x65\xBE\xEB\x8D\xF3\x04"
+			  "\xA5\xA5\x89\x7D\x33\xAE\x53\x0F"
+			  "\x1B\xA7\x6D\x5D\x11\x4D\x2A\x5C"
+			  "\x3D\xE8\x18\x27\xC1\x0E\x9A\x4F"
+			  "\x51\x33\x0D\x0E\xEC\x41\x66\x42"
+			  "\xCF\xBB\x85\xA5\xB4\x7E\x48\xA4"
+			  "\xEC\x3B\x9B\xA9\x5D\x91\x8B\xD1"
+			  "\x83\xB7\x0D\x3A\xA8\xBC\x6E\xE4"
+			  "\xC3\x09\xE9\xD8\x5A\x41\xAD\x4A",
+		.rlen	= 80,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x11\x22\x33\x44",
+		.klen	= 36,
+		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
+		.input	= "\x45\x00\x00\x30\x69\xA6\x40\x00"
+			  "\x80\x06\x26\x90\xC0\xA8\x01\x02"
+			  "\x93\x89\x15\x5E\x0A\x9E\x00\x8B"
+			  "\x2D\xC5\x7E\xE0\x00\x00\x00\x00"
+			  "\x70\x02\x40\x00\x20\xBF\x00\x00"
+			  "\x02\x04\x05\xB4\x01\x01\x04\x02"
+			  "\x01\x02\x02\x01",
+		.ilen	= 52,
+		.assoc	= "\x4A\x2C\xBF\xE3\x00\x00\x00\x02"
+			  "\x01\x02\x03\x04\x05\x06\x07\x08",
+		.alen	= 16,
+		.result	= "\xFF\x42\x5C\x9B\x72\x45\x99\xDF"
+			  "\x7A\x3B\xCD\x51\x01\x94\xE0\x0D"
+			  "\x6A\x78\x10\x7F\x1B\x0B\x1C\xBF"
+			  "\x06\xEF\xAE\x9D\x65\xA5\xD7\x63"
+			  "\x74\x8A\x63\x79\x85\x77\x1D\x34"
+			  "\x7F\x05\x45\x65\x9F\x14\xE9\x9D"
+			  "\xEF\x84\x2D\x8E\xB3\x35\xF4\xEE"
+			  "\xCF\xDB\xF8\x31\x82\x4B\x4C\x49"
+			  "\x15\x95\x6C\x96",
+		.rlen	= 68,
+	}, {
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.input	= "\x45\x00\x00\x3C\x99\xC5\x00\x00"
+			  "\x80\x01\xCB\x7A\x40\x67\x93\x18"
+			  "\x01\x01\x01\x01\x08\x00\x07\x5C"
+			  "\x02\x00\x44\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x75\x76\x77\x61\x62\x63\x64\x65"
+			  "\x66\x67\x68\x69\x01\x02\x02\x01",
+		.ilen	= 64,
+		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 16,
+		.result	= "\x46\x88\xDA\xF2\xF9\x73\xA3\x92"
+			  "\x73\x29\x09\xC3\x31\xD5\x6D\x60"
+			  "\xF6\x94\xAB\xAA\x41\x4B\x5E\x7F"
+			  "\xF5\xFD\xCD\xFF\xF5\xE9\xA2\x84"
+			  "\x45\x64\x76\x49\x27\x19\xFF\xB6"
+			  "\x4D\xE7\xD9\xDC\xA1\xE1\xD8\x94"
+			  "\xBC\x3B\xD5\x78\x73\xED\x4D\x18"
+			  "\x1D\x19\xD4\xD5\xC8\xC1\x8A\xF3"
+			  "\xF8\x21\xD4\x96\xEE\xB0\x96\xE9"
+			  "\x8A\xD2\xB6\x9E\x47\x99\xC7\x1D",
+		.rlen	= 80,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E\x43",
+		.klen	= 20,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.input	= "\x45\x00\x00\x3C\x99\xC3\x00\x00"
+			  "\x80\x01\xCB\x7C\x40\x67\x93\x18"
+			  "\x01\x01\x01\x01\x08\x00\x08\x5C"
+			  "\x02\x00\x43\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x75\x76\x77\x61\x62\x63\x64\x65"
+			  "\x66\x67\x68\x69\x01\x02\x02\x01",
+		.ilen	= 64,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.result	= "\xFB\xA2\xCA\xA4\x85\x3C\xF9\xF0"
+			  "\xF2\x2C\xB1\x0D\x86\xDD\x83\xB0"
+			  "\xFE\xC7\x56\x91\xCF\x1A\x04\xB0"
+			  "\x0D\x11\x38\xEC\x9C\x35\x79\x17"
+			  "\x65\xAC\xBD\x87\x01\xAD\x79\x84"
+			  "\x5B\xF9\xFE\x3F\xBA\x48\x7B\xC9"
+			  "\x17\x55\xE6\x66\x2B\x4C\x8D\x0D"
+			  "\x1F\x5E\x22\x73\x95\x30\x32\x0A"
+			  "\xE0\xD7\x31\xCC\x97\x8E\xCA\xFA"
+			  "\xEA\xE8\x8F\x00\xE8\x0D\x6E\x48",
+		.rlen	= 80,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E\x43",
+		.klen	= 20,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.input	= "\x45\x00\x00\x1C\x42\xA2\x00\x00"
+			  "\x80\x01\x44\x1F\x40\x67\x93\xB6"
+			  "\xE0\x00\x00\x02\x0A\x00\xF5\xFF"
+			  "\x01\x02\x02\x01",
+		.ilen	= 28,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.result	= "\xFB\xA2\xCA\x84\x5E\x5D\xF9\xF0"
+			  "\xF2\x2C\x3E\x6E\x86\xDD\x83\x1E"
+			  "\x1F\xC6\x57\x92\xCD\x1A\xF9\x13"
+			  "\x0E\x13\x79\xED\x36\x9F\x07\x1F"
+			  "\x35\xE0\x34\xBE\x95\xF1\x12\xE4"
+			  "\xE7\xD0\x5D\x35",
+		.rlen	= 44,
+	}, {
+		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
+			  "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\xCA\xFE\xBA\xBE",
+		.klen	= 28,
+		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.input	= "\x45\x00\x00\x28\xA4\xAD\x40\x00"
+			  "\x40\x06\x78\x80\x0A\x01\x03\x8F"
+			  "\x0A\x01\x06\x12\x80\x23\x06\xB8"
+			  "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E"
+			  "\x50\x10\x16\xD0\x75\x68\x00\x01",
+		.ilen	= 40,
+		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
+			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.alen	= 16,
+		.result	= "\xA5\xB1\xF8\x06\x60\x29\xAE\xA4"
+			  "\x0E\x59\x8B\x81\x22\xDE\x02\x42"
+			  "\x09\x38\xB3\xAB\x33\xF8\x28\xE6"
+			  "\x87\xB8\x85\x8B\x5B\xFB\xDB\xD0"
+			  "\x31\x5B\x27\x45\x21\x44\xCC\x77"
+			  "\x95\x45\x7B\x96\x52\x03\x7F\x53"
+			  "\x18\x02\x7B\x5B\x4C\xD7\xA6\x36",
+		.rlen	= 56,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xDE\xCA\xF8\x88",
+		.klen	= 20,
+		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
+		.input	= "\x45\x00\x00\x49\x33\xBA\x00\x00"
+			  "\x7F\x11\x91\x06\xC3\xFB\x1D\x10"
+			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
+			  "\x00\x35\xDD\x7B\x80\x03\x02\xD5"
+			  "\x00\x00\x4E\x20\x00\x1E\x8C\x18"
+			  "\xD7\x5B\x81\xDC\x91\xBA\xA0\x47"
+			  "\x6B\x91\xB9\x24\xB2\x80\x38\x9D"
+			  "\x92\xC9\x63\xBA\xC0\x46\xEC\x95"
+			  "\x9B\x62\x66\xC0\x47\x22\xB1\x49"
+			  "\x23\x01\x01\x01",
+		.ilen	= 76,
+		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
+			  "\xCE\xFA\xCE\x74",
+		.alen	= 20,
+		.result	= "\x18\xA6\xFD\x42\xF7\x2C\xBF\x4A"
+			  "\xB2\xA2\xEA\x90\x1F\x73\xD8\x14"
+			  "\xE3\xE7\xF2\x43\xD9\x54\x12\xE1"
+			  "\xC3\x49\xC1\xD2\xFB\xEC\x16\x8F"
+			  "\x91\x90\xFE\xEB\xAF\x2C\xB0\x19"
+			  "\x84\xE6\x58\x63\x96\x5D\x74\x72"
+			  "\xB7\x9D\xA3\x45\xE0\xE7\x80\x19"
+			  "\x1F\x0D\x2F\x0E\x0F\x49\x6C\x22"
+			  "\x6F\x21\x27\xB2\x7D\xB3\x57\x24"
+			  "\xE7\x84\x5D\x68\x65\x1F\x57\xE6"
+			  "\x5F\x35\x4F\x75\xFF\x17\x01\x57"
+			  "\x69\x62\x34\x36",
+		.rlen	= 92,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x73\x61\x6C\x74",
+		.klen	= 36,
+		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
+		.input	= "\x45\x08\x00\x28\x73\x2C\x00\x00"
+			  "\x40\x06\xE9\xF9\x0A\x01\x06\x12"
+			  "\x0A\x01\x03\x8F\x06\xB8\x80\x23"
+			  "\xDD\x6B\xAF\xBE\xCB\x71\x26\x02"
+			  "\x50\x10\x1F\x64\x6D\x54\x00\x01",
+		.ilen	= 40,
+		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
+			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
+			  "\x69\x76\x65\x63",
+		.alen	= 20,
+		.result	= "\xF2\xD6\x9E\xCD\xBD\x5A\x0D\x5B"
+			  "\x8D\x5E\xF3\x8B\xAD\x4D\xA5\x8D"
+			  "\x1F\x27\x8F\xDE\x98\xEF\x67\x54"
+			  "\x9D\x52\x4A\x30\x18\xD9\xA5\x7F"
+			  "\xF4\xD3\xA3\x1C\xE6\x73\x11\x9E"
+			  "\x45\x16\x26\xC2\x41\x57\x71\xE3"
+			  "\xB7\xEE\xBC\xA6\x14\xC8\x9B\x35",
+		.rlen	= 56,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E\x43",
+		.klen	= 20,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.input	= "\x45\x00\x00\x49\x33\x3E\x00\x00"
+			  "\x7F\x11\x91\x82\xC3\xFB\x1D\x10"
+			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
+			  "\x00\x35\xCB\x45\x80\x03\x02\x5B"
+			  "\x00\x00\x01\xE0\x00\x1E\x8C\x18"
+			  "\xD6\x57\x59\xD5\x22\x84\xA0\x35"
+			  "\x2C\x71\x47\x5C\x88\x80\x39\x1C"
+			  "\x76\x4D\x6E\x5E\xE0\x49\x6B\x32"
+			  "\x5A\xE2\x70\xC0\x38\x99\x49\x39"
+			  "\x15\x01\x01\x01",
+		.ilen	= 76,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.result	= "\xFB\xA2\xCA\xD1\x2F\xC1\xF9\xF0"
+			  "\x0D\x3C\xEB\xF3\x05\x41\x0D\xB8"
+			  "\x3D\x77\x84\xB6\x07\x32\x3D\x22"
+			  "\x0F\x24\xB0\xA9\x7D\x54\x18\x28"
+			  "\x00\xCA\xDB\x0F\x68\xD9\x9E\xF0"
+			  "\xE0\xC0\xC8\x9A\xE9\xBE\xA8\x88"
+			  "\x4E\x52\xD6\x5B\xC1\xAF\xD0\x74"
+			  "\x0F\x74\x24\x44\x74\x7B\x5B\x39"
+			  "\xAB\x53\x31\x63\xAA\xD4\x55\x0E"
+			  "\xE5\x16\x09\x75\xCD\xB6\x08\xC5"
+			  "\x76\x91\x89\x60\x97\x63\xB8\xE1"
+			  "\x8C\xAA\x81\xE2",
+		.rlen	= 92,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x73\x61\x6C\x74",
+		.klen	= 36,
+		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
+		.input	= "\x63\x69\x73\x63\x6F\x01\x72\x75"
+			  "\x6C\x65\x73\x01\x74\x68\x65\x01"
+			  "\x6E\x65\x74\x77\x65\x01\x64\x65"
+			  "\x66\x69\x6E\x65\x01\x74\x68\x65"
+			  "\x74\x65\x63\x68\x6E\x6F\x6C\x6F"
+			  "\x67\x69\x65\x73\x01\x74\x68\x61"
+			  "\x74\x77\x69\x6C\x6C\x01\x64\x65"
+			  "\x66\x69\x6E\x65\x74\x6F\x6D\x6F"
+			  "\x72\x72\x6F\x77\x01\x02\x02\x01",
+		.ilen	= 72,
+		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
+			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
+			  "\x69\x76\x65\x63",
+		.alen	= 20,
+		.result	= "\xD4\xB7\xED\x86\xA1\x77\x7F\x2E"
+			  "\xA1\x3D\x69\x73\xD3\x24\xC6\x9E"
+			  "\x7B\x43\xF8\x26\xFB\x56\x83\x12"
+			  "\x26\x50\x8B\xEB\xD2\xDC\xEB\x18"
+			  "\xD0\xA6\xDF\x10\xE5\x48\x7D\xF0"
+			  "\x74\x11\x3E\x14\xC6\x41\x02\x4E"
+			  "\x3E\x67\x73\xD9\x1A\x62\xEE\x42"
+			  "\x9B\x04\x3A\x10\xE3\xEF\xE6\xB0"
+			  "\x12\xA4\x93\x63\x41\x23\x64\xF8"
+			  "\xC0\xCA\xC5\x87\xF2\x49\xE5\x6B"
+			  "\x11\xE2\x4F\x30\xE4\x4C\xCC\x76",
+		.rlen	= 88,
+	}, {
+		.key	= "\x7D\x77\x3D\x00\xC1\x44\xC5\x25"
+			  "\xAC\x61\x9D\x18\xC8\x4A\x3F\x47"
+			  "\xD9\x66\x42\x67",
+		.klen	= 20,
+		.iv	= "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
+		.input	= "\x01\x02\x02\x01",
+		.ilen	= 4,
+		.assoc	= "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF"
+			  "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
+		.alen	= 16,
+		.result	= "\x43\x7F\x86\x6B\xCB\x3F\x69\x9F"
+			  "\xE9\xB0\x82\x2B\xAC\x96\x1C\x45"
+			  "\x04\xBE\xF2\x70",
+		.rlen	= 20,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xDE\xCA\xF8\x88",
+		.klen	= 20,
+		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
+		.input	= "\x74\x6F\x01\x62\x65\x01\x6F\x72"
+			  "\x01\x6E\x6F\x74\x01\x74\x6F\x01"
+			  "\x62\x65\x00\x01",
+		.ilen	= 20,
+		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
+			  "\xCE\xFA\xCE\x74",
+		.alen	= 20,
+		.result	= "\x29\xC9\xFC\x69\xA1\x97\xD0\x38"
+			  "\xCC\xDD\x14\xE2\xDD\xFC\xAA\x05"
+			  "\x43\x33\x21\x64\x41\x25\x03\x52"
+			  "\x43\x03\xED\x3C\x6C\x5F\x28\x38"
+			  "\x43\xAF\x8C\x3E",
+		.rlen	= 36,
+	}, {
+		.key	= "\x6C\x65\x67\x61\x6C\x69\x7A\x65"
+			  "\x6D\x61\x72\x69\x6A\x75\x61\x6E"
+			  "\x61\x61\x6E\x64\x64\x6F\x69\x74"
+			  "\x62\x65\x66\x6F\x72\x65\x69\x61"
+			  "\x74\x75\x72\x6E",
+		.klen	= 36,
+		.iv	= "\x33\x30\x21\x69\x67\x65\x74\x6D",
+		.input	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
+			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
+			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01",
+		.ilen	= 52,
+		.assoc	= "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\x33\x30\x21\x69"
+			  "\x67\x65\x74\x6D",
+		.alen	= 20,
+		.result	= "\xF9\x7A\xB2\xAA\x35\x6D\x8E\xDC"
+			  "\xE1\x76\x44\xAC\x8C\x78\xE2\x5D"
+			  "\xD2\x4D\xED\xBB\x29\xEB\xF1\xB6"
+			  "\x4A\x27\x4B\x39\xB4\x9C\x3A\x86"
+			  "\x4C\xD3\xD7\x8C\xA4\xAE\x68\xA3"
+			  "\x2B\x42\x45\x8F\xB5\x7D\xBE\x82"
+			  "\x1D\xCC\x63\xB9\xD0\x93\x7B\xA2"
+			  "\x94\x5F\x66\x93\x68\x66\x1A\x32"
+			  "\x9F\xB4\xC0\x53",
+		.rlen	= 68,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E\x43",
+		.klen	= 20,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.input	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
+			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
+			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01",
+		.ilen	= 52,
+		.assoc	= "\x3F\x7E\xF6\x42\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.result	= "\xFB\xA2\xCA\xA8\xC6\xC5\xF9\xF0"
+			  "\xF2\x2C\xA5\x4A\x06\x12\x10\xAD"
+			  "\x3F\x6E\x57\x91\xCF\x1A\xCA\x21"
+			  "\x0D\x11\x7C\xEC\x9C\x35\x79\x17"
+			  "\x65\xAC\xBD\x87\x01\xAD\x79\x84"
+			  "\x5B\xF9\xFE\x3F\xBA\x48\x7B\xC9"
+			  "\x63\x21\x93\x06\x84\xEE\xCA\xDB"
+			  "\x56\x91\x25\x46\xE7\xA9\x5C\x97"
+			  "\x40\xD7\xCB\x05",
+		.rlen	= 68,
+	}, {
+		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
+			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
+			  "\x22\x43\x3C\x64",
+		.klen	= 20,
+		.iv	= "\x48\x55\xEC\x7D\x3A\x23\x4B\xFD",
+		.input	= "\x08\x00\xC6\xCD\x02\x00\x07\x00"
+			  "\x61\x62\x63\x64\x65\x66\x67\x68"
+			  "\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70"
+			  "\x71\x72\x73\x74\x01\x02\x02\x01",
+		.ilen	= 32,
+		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
+			  "\x00\x00\x00\x07\x48\x55\xEC\x7D"
+			  "\x3A\x23\x4B\xFD",
+		.alen	= 20,
+		.result	= "\x74\x75\x2E\x8A\xEB\x5D\x87\x3C"
+			  "\xD7\xC0\xF4\xAC\xC3\x6C\x4B\xFF"
+			  "\x84\xB7\xD7\xB9\x8F\x0C\xA8\xB6"
+			  "\xAC\xDA\x68\x94\xBC\x61\x90\x69"
+			  "\xEF\x9C\xBC\x28\xFE\x1B\x56\xA7"
+			  "\xC4\xE0\xD5\x8C\x86\xCD\x2B\xC0",
+		.rlen	= 48,
+	}
+};
+
+static const struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = {
+	{ /* Generated using Crypto++ */
+		.key    = zeroed_string,
+		.klen	= 20,
+		.iv     = zeroed_string,
+		.input	= "\x03\x88\xDA\xCE\x60\xB6\xA3\x92"
+			  "\xF3\x28\xC2\xB9\x71\xB2\xFE\x78"
+			  "\x97\xFE\x4C\x23\x37\x42\x01\xE0"
+			  "\x81\x9F\x8D\xC5\xD7\x41\xA0\x1B",
+		.ilen	= 32,
+		.assoc  = zeroed_string,
+		.alen   = 16,
+		.result = zeroed_string,
+		.rlen   = 16,
+
+	},{
+		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.input	= "\xC0\x0D\x8B\x42\x0F\x8F\x34\x18"
+			  "\x88\xB1\xC5\xBC\xC5\xB6\xD6\x28"
+			  "\x6A\x9D\xDF\x11\x5E\xFE\x5E\x9D"
+			  "\x2F\x70\x44\x92\xF7\xF2\xE3\xEF",
+		.ilen	= 32,
+		.assoc  = "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen   = 16,
+		.result = zeroed_string,
+		.rlen   = 16,
+	}, {
+		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = zeroed_string,
+		.input	= "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
+			  "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
+			  "\x0B\x8F\x88\x69\x17\xE6\xB4\x3C"
+			  "\xB1\x68\xFD\x14\x52\x64\x61\xB2",
+		.ilen	= 32,
+		.assoc  = zeroed_string,
+		.alen   = 16,
+		.result = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.rlen   = 16,
+	}, {
+		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = zeroed_string,
+		.input	= "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
+			  "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
+			  "\x90\x92\xB7\xE3\x5F\xA3\x9A\x63"
+			  "\x7E\xD7\x1F\xD8\xD3\x7C\x4B\xF5",
+		.ilen	= 32,
+		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen   = 16,
+		.result = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.rlen   = 16,
+
+	}, {
+		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.input	= "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
+			  "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
+			  "\x64\x50\xF9\x32\x13\xFB\x74\x61"
+			  "\xF4\xED\x52\xD3\xC5\x10\x55\x3C",
+		.ilen	= 32,
+		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen   = 16,
+		.result = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.rlen   = 16,
+	}, {
+		.key    = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.input	= "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
+			  "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
+			  "\x98\x14\xA1\x42\x37\x80\xFD\x90"
+			  "\x68\x12\x01\xA8\x91\x89\xB9\x83"
+			  "\x5B\x11\x77\x12\x9B\xFF\x24\x89"
+			  "\x94\x5F\x18\x12\xBA\x27\x09\x39"
+			  "\x99\x96\x76\x42\x15\x1C\xCD\xCB"
+			  "\xDC\xD3\xDA\x65\x73\xAF\x80\xCD"
+			  "\xD2\xB6\xC2\x4A\x76\xC2\x92\x85"
+			  "\xBD\xCF\x62\x98\x58\x14\xE5\xBD",
+		.ilen	= 80,
+		.assoc  = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen   = 16,
+		.result = "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.rlen   = 64,
+	}, {
+		.key    = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv     = "\x00\x00\x45\x67\x89\xab\xcd\xef",
+		.input	= "\xC1\x76\x33\x85\xE2\x9B\x5F\xDE"
+			  "\xDE\x89\x3D\x42\xE7\xC9\x69\x8A"
+			  "\x44\x6D\xC3\x88\x46\x2E\xC2\x01"
+			  "\x5E\xF6\x0C\x39\xF0\xC4\xA5\x82"
+			  "\xCD\xE8\x31\xCC\x0A\x4C\xE4\x44"
+			  "\x41\xA9\x82\x6F\x22\xA1\x23\x1A"
+			  "\xA8\xE3\x16\xFD\x31\x5C\x27\x31"
+			  "\xF1\x7F\x01\x63\xA3\xAF\x70\xA1"
+			  "\xCF\x07\x57\x41\x67\xD0\xC4\x42"
+			  "\xDB\x18\xC6\x4C\x4C\xE0\x3D\x9F"
+			  "\x05\x07\xFB\x13\x7D\x4A\xCA\x5B"
+			  "\xF0\xBF\x64\x7E\x05\xB1\x72\xEE"
+			  "\x7C\x3B\xD4\xCD\x14\x03\xB2\x2C"
+			  "\xD3\xA9\xEE\xFA\x17\xFC\x9C\xDF"
+			  "\xC7\x75\x40\xFF\xAE\xAD\x1E\x59"
+			  "\x2F\x30\x24\xFB\xAD\x6B\x10\xFA"
+			  "\x6C\x9F\x5B\xE7\x25\xD5\xD0\x25"
+			  "\xAC\x4A\x4B\xDA\xFC\x7A\x85\x1B"
+			  "\x7E\x13\x06\x82\x08\x17\xA4\x35"
+			  "\xEC\xC5\x8D\x63\x96\x81\x0A\x8F"
+			  "\xA3\x05\x38\x95\x20\x1A\x47\x04"
+			  "\x6F\x6D\xDA\x8F\xEF\xC1\x76\x35"
+			  "\x6B\xC7\x4D\x0F\x94\x12\xCA\x3E"
+			  "\x2E\xD5\x03\x2E\x86\x7E\xAA\x3B"
+			  "\x37\x08\x1C\xCF\xBA\x5D\x71\x46"
+			  "\x80\x72\xB0\x4C\x82\x0D\x60\x3C",
+		.ilen	= 208,
+		.assoc  = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
+			  "\xaa\xaa\xaa\xaa\x00\x00\x45\x67"
+			  "\x89\xab\xcd\xef",
+		.alen   = 20,
+		.result = "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff",
+		.rlen   = 192,
+	}, {
+		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
+			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
+			  "\x2E\x44\x3B\x68",
+		.klen	= 20,
+		.iv	= "\x49\x56\xED\x7E\x3B\x24\x4C\xFE",
+		.result	= "\x45\x00\x00\x48\x69\x9A\x00\x00"
+			  "\x80\x11\x4D\xB7\xC0\xA8\x01\x02"
+			  "\xC0\xA8\x01\x01\x0A\x9B\xF1\x56"
+			  "\x38\xD3\x01\x00\x00\x01\x00\x00"
+			  "\x00\x00\x00\x00\x04\x5F\x73\x69"
+			  "\x70\x04\x5F\x75\x64\x70\x03\x73"
+			  "\x69\x70\x09\x63\x79\x62\x65\x72"
+			  "\x63\x69\x74\x79\x02\x64\x6B\x00"
+			  "\x00\x21\x00\x01\x01\x02\x02\x01",
+		.rlen	= 72,
+		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
+			  "\x00\x00\x00\x00\x49\x56\xED\x7E"
+			  "\x3B\x24\x4C\xFE",
+		.alen	= 20,
+		.input	= "\xFE\xCF\x53\x7E\x72\x9D\x5B\x07"
+			  "\xDC\x30\xDF\x52\x8D\xD2\x2B\x76"
+			  "\x8D\x1B\x98\x73\x66\x96\xA6\xFD"
+			  "\x34\x85\x09\xFA\x13\xCE\xAC\x34"
+			  "\xCF\xA2\x43\x6F\x14\xA3\xF3\xCF"
+			  "\x65\x92\x5B\xF1\xF4\xA1\x3C\x5D"
+			  "\x15\xB2\x1E\x18\x84\xF5\xFF\x62"
+			  "\x47\xAE\xAB\xB7\x86\xB9\x3B\xCE"
+			  "\x61\xBC\x17\xD7\x68\xFD\x97\x32"
+			  "\x45\x90\x18\x14\x8F\x6C\xBE\x72"
+			  "\x2F\xD0\x47\x96\x56\x2D\xFD\xB4",
+		.ilen	= 88,
+	}, {
+		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
+			  "\xCA\xFE\xBA\xBE",
+		.klen	= 20,
+		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.result	= "\x45\x00\x00\x3E\x69\x8F\x00\x00"
+			  "\x80\x11\x4D\xCC\xC0\xA8\x01\x02"
+			  "\xC0\xA8\x01\x01\x0A\x98\x00\x35"
+			  "\x00\x2A\x23\x43\xB2\xD0\x01\x00"
+			  "\x00\x01\x00\x00\x00\x00\x00\x00"
+			  "\x03\x73\x69\x70\x09\x63\x79\x62"
+			  "\x65\x72\x63\x69\x74\x79\x02\x64"
+			  "\x6B\x00\x00\x01\x00\x01\x00\x01",
+		.rlen	= 64,
+		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
+			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.alen	= 16,
+		.input	= "\xDE\xB2\x2C\xD9\xB0\x7C\x72\xC1"
+			  "\x6E\x3A\x65\xBE\xEB\x8D\xF3\x04"
+			  "\xA5\xA5\x89\x7D\x33\xAE\x53\x0F"
+			  "\x1B\xA7\x6D\x5D\x11\x4D\x2A\x5C"
+			  "\x3D\xE8\x18\x27\xC1\x0E\x9A\x4F"
+			  "\x51\x33\x0D\x0E\xEC\x41\x66\x42"
+			  "\xCF\xBB\x85\xA5\xB4\x7E\x48\xA4"
+			  "\xEC\x3B\x9B\xA9\x5D\x91\x8B\xD1"
+			  "\x83\xB7\x0D\x3A\xA8\xBC\x6E\xE4"
+			  "\xC3\x09\xE9\xD8\x5A\x41\xAD\x4A",
+		.ilen	= 80,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x11\x22\x33\x44",
+		.klen	= 36,
+		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
+		.result	= "\x45\x00\x00\x30\x69\xA6\x40\x00"
+			  "\x80\x06\x26\x90\xC0\xA8\x01\x02"
+			  "\x93\x89\x15\x5E\x0A\x9E\x00\x8B"
+			  "\x2D\xC5\x7E\xE0\x00\x00\x00\x00"
+			  "\x70\x02\x40\x00\x20\xBF\x00\x00"
+			  "\x02\x04\x05\xB4\x01\x01\x04\x02"
+			  "\x01\x02\x02\x01",
+		.rlen	= 52,
+		.assoc	= "\x4A\x2C\xBF\xE3\x00\x00\x00\x02"
+			  "\x01\x02\x03\x04\x05\x06\x07\x08",
+		.alen	= 16,
+		.input	= "\xFF\x42\x5C\x9B\x72\x45\x99\xDF"
+			  "\x7A\x3B\xCD\x51\x01\x94\xE0\x0D"
+			  "\x6A\x78\x10\x7F\x1B\x0B\x1C\xBF"
+			  "\x06\xEF\xAE\x9D\x65\xA5\xD7\x63"
+			  "\x74\x8A\x63\x79\x85\x77\x1D\x34"
+			  "\x7F\x05\x45\x65\x9F\x14\xE9\x9D"
+			  "\xEF\x84\x2D\x8E\xB3\x35\xF4\xEE"
+			  "\xCF\xDB\xF8\x31\x82\x4B\x4C\x49"
+			  "\x15\x95\x6C\x96",
+		.ilen	= 68,
+	}, {
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00",
+		.klen	= 20,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.result	= "\x45\x00\x00\x3C\x99\xC5\x00\x00"
+			  "\x80\x01\xCB\x7A\x40\x67\x93\x18"
+			  "\x01\x01\x01\x01\x08\x00\x07\x5C"
+			  "\x02\x00\x44\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x75\x76\x77\x61\x62\x63\x64\x65"
+			  "\x66\x67\x68\x69\x01\x02\x02\x01",
+		.rlen	= 64,
+		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 16,
+		.input	= "\x46\x88\xDA\xF2\xF9\x73\xA3\x92"
+			  "\x73\x29\x09\xC3\x31\xD5\x6D\x60"
+			  "\xF6\x94\xAB\xAA\x41\x4B\x5E\x7F"
+			  "\xF5\xFD\xCD\xFF\xF5\xE9\xA2\x84"
+			  "\x45\x64\x76\x49\x27\x19\xFF\xB6"
+			  "\x4D\xE7\xD9\xDC\xA1\xE1\xD8\x94"
+			  "\xBC\x3B\xD5\x78\x73\xED\x4D\x18"
+			  "\x1D\x19\xD4\xD5\xC8\xC1\x8A\xF3"
+			  "\xF8\x21\xD4\x96\xEE\xB0\x96\xE9"
+			  "\x8A\xD2\xB6\x9E\x47\x99\xC7\x1D",
+		.ilen	= 80,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E\x43",
+		.klen	= 20,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.result	= "\x45\x00\x00\x3C\x99\xC3\x00\x00"
+			  "\x80\x01\xCB\x7C\x40\x67\x93\x18"
+			  "\x01\x01\x01\x01\x08\x00\x08\x5C"
+			  "\x02\x00\x43\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x75\x76\x77\x61\x62\x63\x64\x65"
+			  "\x66\x67\x68\x69\x01\x02\x02\x01",
+		.rlen	= 64,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.input	= "\xFB\xA2\xCA\xA4\x85\x3C\xF9\xF0"
+			  "\xF2\x2C\xB1\x0D\x86\xDD\x83\xB0"
+			  "\xFE\xC7\x56\x91\xCF\x1A\x04\xB0"
+			  "\x0D\x11\x38\xEC\x9C\x35\x79\x17"
+			  "\x65\xAC\xBD\x87\x01\xAD\x79\x84"
+			  "\x5B\xF9\xFE\x3F\xBA\x48\x7B\xC9"
+			  "\x17\x55\xE6\x66\x2B\x4C\x8D\x0D"
+			  "\x1F\x5E\x22\x73\x95\x30\x32\x0A"
+			  "\xE0\xD7\x31\xCC\x97\x8E\xCA\xFA"
+			  "\xEA\xE8\x8F\x00\xE8\x0D\x6E\x48",
+		.ilen	= 80,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E\x43",
+		.klen	= 20,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.result	= "\x45\x00\x00\x1C\x42\xA2\x00\x00"
+			  "\x80\x01\x44\x1F\x40\x67\x93\xB6"
+			  "\xE0\x00\x00\x02\x0A\x00\xF5\xFF"
+			  "\x01\x02\x02\x01",
+		.rlen	= 28,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.input	= "\xFB\xA2\xCA\x84\x5E\x5D\xF9\xF0"
+			  "\xF2\x2C\x3E\x6E\x86\xDD\x83\x1E"
+			  "\x1F\xC6\x57\x92\xCD\x1A\xF9\x13"
+			  "\x0E\x13\x79\xED\x36\x9F\x07\x1F"
+			  "\x35\xE0\x34\xBE\x95\xF1\x12\xE4"
+			  "\xE7\xD0\x5D\x35",
+		.ilen	= 44,
+	}, {
+		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
+			  "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\xCA\xFE\xBA\xBE",
+		.klen	= 28,
+		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.result	= "\x45\x00\x00\x28\xA4\xAD\x40\x00"
+			  "\x40\x06\x78\x80\x0A\x01\x03\x8F"
+			  "\x0A\x01\x06\x12\x80\x23\x06\xB8"
+			  "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E"
+			  "\x50\x10\x16\xD0\x75\x68\x00\x01",
+		.rlen	= 40,
+		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
+			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.alen	= 16,
+		.input	= "\xA5\xB1\xF8\x06\x60\x29\xAE\xA4"
+			  "\x0E\x59\x8B\x81\x22\xDE\x02\x42"
+			  "\x09\x38\xB3\xAB\x33\xF8\x28\xE6"
+			  "\x87\xB8\x85\x8B\x5B\xFB\xDB\xD0"
+			  "\x31\x5B\x27\x45\x21\x44\xCC\x77"
+			  "\x95\x45\x7B\x96\x52\x03\x7F\x53"
+			  "\x18\x02\x7B\x5B\x4C\xD7\xA6\x36",
+		.ilen	= 56,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xDE\xCA\xF8\x88",
+		.klen	= 20,
+		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
+		.result	= "\x45\x00\x00\x49\x33\xBA\x00\x00"
+			  "\x7F\x11\x91\x06\xC3\xFB\x1D\x10"
+			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
+			  "\x00\x35\xDD\x7B\x80\x03\x02\xD5"
+			  "\x00\x00\x4E\x20\x00\x1E\x8C\x18"
+			  "\xD7\x5B\x81\xDC\x91\xBA\xA0\x47"
+			  "\x6B\x91\xB9\x24\xB2\x80\x38\x9D"
+			  "\x92\xC9\x63\xBA\xC0\x46\xEC\x95"
+			  "\x9B\x62\x66\xC0\x47\x22\xB1\x49"
+			  "\x23\x01\x01\x01",
+		.rlen	= 76,
+		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
+			  "\xCE\xFA\xCE\x74",
+		.alen	= 20,
+		.input	= "\x18\xA6\xFD\x42\xF7\x2C\xBF\x4A"
+			  "\xB2\xA2\xEA\x90\x1F\x73\xD8\x14"
+			  "\xE3\xE7\xF2\x43\xD9\x54\x12\xE1"
+			  "\xC3\x49\xC1\xD2\xFB\xEC\x16\x8F"
+			  "\x91\x90\xFE\xEB\xAF\x2C\xB0\x19"
+			  "\x84\xE6\x58\x63\x96\x5D\x74\x72"
+			  "\xB7\x9D\xA3\x45\xE0\xE7\x80\x19"
+			  "\x1F\x0D\x2F\x0E\x0F\x49\x6C\x22"
+			  "\x6F\x21\x27\xB2\x7D\xB3\x57\x24"
+			  "\xE7\x84\x5D\x68\x65\x1F\x57\xE6"
+			  "\x5F\x35\x4F\x75\xFF\x17\x01\x57"
+			  "\x69\x62\x34\x36",
+		.ilen	= 92,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x73\x61\x6C\x74",
+		.klen	= 36,
+		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
+		.result	= "\x45\x08\x00\x28\x73\x2C\x00\x00"
+			  "\x40\x06\xE9\xF9\x0A\x01\x06\x12"
+			  "\x0A\x01\x03\x8F\x06\xB8\x80\x23"
+			  "\xDD\x6B\xAF\xBE\xCB\x71\x26\x02"
+			  "\x50\x10\x1F\x64\x6D\x54\x00\x01",
+		.rlen	= 40,
+		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
+			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
+			  "\x69\x76\x65\x63",
+		.alen	= 20,
+		.input	= "\xF2\xD6\x9E\xCD\xBD\x5A\x0D\x5B"
+			  "\x8D\x5E\xF3\x8B\xAD\x4D\xA5\x8D"
+			  "\x1F\x27\x8F\xDE\x98\xEF\x67\x54"
+			  "\x9D\x52\x4A\x30\x18\xD9\xA5\x7F"
+			  "\xF4\xD3\xA3\x1C\xE6\x73\x11\x9E"
+			  "\x45\x16\x26\xC2\x41\x57\x71\xE3"
+			  "\xB7\xEE\xBC\xA6\x14\xC8\x9B\x35",
+		.ilen	= 56,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E\x43",
+		.klen	= 20,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.result	= "\x45\x00\x00\x49\x33\x3E\x00\x00"
+			  "\x7F\x11\x91\x82\xC3\xFB\x1D\x10"
+			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
+			  "\x00\x35\xCB\x45\x80\x03\x02\x5B"
+			  "\x00\x00\x01\xE0\x00\x1E\x8C\x18"
+			  "\xD6\x57\x59\xD5\x22\x84\xA0\x35"
+			  "\x2C\x71\x47\x5C\x88\x80\x39\x1C"
+			  "\x76\x4D\x6E\x5E\xE0\x49\x6B\x32"
+			  "\x5A\xE2\x70\xC0\x38\x99\x49\x39"
+			  "\x15\x01\x01\x01",
+		.rlen	= 76,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.input	= "\xFB\xA2\xCA\xD1\x2F\xC1\xF9\xF0"
+			  "\x0D\x3C\xEB\xF3\x05\x41\x0D\xB8"
+			  "\x3D\x77\x84\xB6\x07\x32\x3D\x22"
+			  "\x0F\x24\xB0\xA9\x7D\x54\x18\x28"
+			  "\x00\xCA\xDB\x0F\x68\xD9\x9E\xF0"
+			  "\xE0\xC0\xC8\x9A\xE9\xBE\xA8\x88"
+			  "\x4E\x52\xD6\x5B\xC1\xAF\xD0\x74"
+			  "\x0F\x74\x24\x44\x74\x7B\x5B\x39"
+			  "\xAB\x53\x31\x63\xAA\xD4\x55\x0E"
+			  "\xE5\x16\x09\x75\xCD\xB6\x08\xC5"
+			  "\x76\x91\x89\x60\x97\x63\xB8\xE1"
+			  "\x8C\xAA\x81\xE2",
+		.ilen	= 92,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x73\x61\x6C\x74",
+		.klen	= 36,
+		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
+		.result	= "\x63\x69\x73\x63\x6F\x01\x72\x75"
+			  "\x6C\x65\x73\x01\x74\x68\x65\x01"
+			  "\x6E\x65\x74\x77\x65\x01\x64\x65"
+			  "\x66\x69\x6E\x65\x01\x74\x68\x65"
+			  "\x74\x65\x63\x68\x6E\x6F\x6C\x6F"
+			  "\x67\x69\x65\x73\x01\x74\x68\x61"
+			  "\x74\x77\x69\x6C\x6C\x01\x64\x65"
+			  "\x66\x69\x6E\x65\x74\x6F\x6D\x6F"
+			  "\x72\x72\x6F\x77\x01\x02\x02\x01",
+		.rlen	= 72,
+		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
+			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
+			  "\x69\x76\x65\x63",
+		.alen	= 20,
+		.input	= "\xD4\xB7\xED\x86\xA1\x77\x7F\x2E"
+			  "\xA1\x3D\x69\x73\xD3\x24\xC6\x9E"
+			  "\x7B\x43\xF8\x26\xFB\x56\x83\x12"
+			  "\x26\x50\x8B\xEB\xD2\xDC\xEB\x18"
+			  "\xD0\xA6\xDF\x10\xE5\x48\x7D\xF0"
+			  "\x74\x11\x3E\x14\xC6\x41\x02\x4E"
+			  "\x3E\x67\x73\xD9\x1A\x62\xEE\x42"
+			  "\x9B\x04\x3A\x10\xE3\xEF\xE6\xB0"
+			  "\x12\xA4\x93\x63\x41\x23\x64\xF8"
+			  "\xC0\xCA\xC5\x87\xF2\x49\xE5\x6B"
+			  "\x11\xE2\x4F\x30\xE4\x4C\xCC\x76",
+		.ilen	= 88,
+	}, {
+		.key	= "\x7D\x77\x3D\x00\xC1\x44\xC5\x25"
+			  "\xAC\x61\x9D\x18\xC8\x4A\x3F\x47"
+			  "\xD9\x66\x42\x67",
+		.klen	= 20,
+		.iv	= "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
+		.result	= "\x01\x02\x02\x01",
+		.rlen	= 4,
+		.assoc	= "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF"
+			  "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
+		.alen	= 16,
+		.input	= "\x43\x7F\x86\x6B\xCB\x3F\x69\x9F"
+			  "\xE9\xB0\x82\x2B\xAC\x96\x1C\x45"
+			  "\x04\xBE\xF2\x70",
+		.ilen	= 20,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xDE\xCA\xF8\x88",
+		.klen	= 20,
+		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
+		.result	= "\x74\x6F\x01\x62\x65\x01\x6F\x72"
+			  "\x01\x6E\x6F\x74\x01\x74\x6F\x01"
+			  "\x62\x65\x00\x01",
+		.rlen	= 20,
+		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
+			  "\xCE\xFA\xCE\x74",
+		.alen	= 20,
+		.input	= "\x29\xC9\xFC\x69\xA1\x97\xD0\x38"
+			  "\xCC\xDD\x14\xE2\xDD\xFC\xAA\x05"
+			  "\x43\x33\x21\x64\x41\x25\x03\x52"
+			  "\x43\x03\xED\x3C\x6C\x5F\x28\x38"
+			  "\x43\xAF\x8C\x3E",
+		.ilen	= 36,
+	}, {
+		.key	= "\x6C\x65\x67\x61\x6C\x69\x7A\x65"
+			  "\x6D\x61\x72\x69\x6A\x75\x61\x6E"
+			  "\x61\x61\x6E\x64\x64\x6F\x69\x74"
+			  "\x62\x65\x66\x6F\x72\x65\x69\x61"
+			  "\x74\x75\x72\x6E",
+		.klen	= 36,
+		.iv	= "\x33\x30\x21\x69\x67\x65\x74\x6D",
+		.result	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
+			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
+			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01",
+		.rlen	= 52,
+		.assoc	= "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\x33\x30\x21\x69"
+			  "\x67\x65\x74\x6D",
+		.alen	= 20,
+		.input	= "\xF9\x7A\xB2\xAA\x35\x6D\x8E\xDC"
+			  "\xE1\x76\x44\xAC\x8C\x78\xE2\x5D"
+			  "\xD2\x4D\xED\xBB\x29\xEB\xF1\xB6"
+			  "\x4A\x27\x4B\x39\xB4\x9C\x3A\x86"
+			  "\x4C\xD3\xD7\x8C\xA4\xAE\x68\xA3"
+			  "\x2B\x42\x45\x8F\xB5\x7D\xBE\x82"
+			  "\x1D\xCC\x63\xB9\xD0\x93\x7B\xA2"
+			  "\x94\x5F\x66\x93\x68\x66\x1A\x32"
+			  "\x9F\xB4\xC0\x53",
+		.ilen	= 68,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E\x43",
+		.klen	= 20,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.result	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
+			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
+			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01",
+		.rlen	= 52,
+		.assoc	= "\x3F\x7E\xF6\x42\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.input	= "\xFB\xA2\xCA\xA8\xC6\xC5\xF9\xF0"
+			  "\xF2\x2C\xA5\x4A\x06\x12\x10\xAD"
+			  "\x3F\x6E\x57\x91\xCF\x1A\xCA\x21"
+			  "\x0D\x11\x7C\xEC\x9C\x35\x79\x17"
+			  "\x65\xAC\xBD\x87\x01\xAD\x79\x84"
+			  "\x5B\xF9\xFE\x3F\xBA\x48\x7B\xC9"
+			  "\x63\x21\x93\x06\x84\xEE\xCA\xDB"
+			  "\x56\x91\x25\x46\xE7\xA9\x5C\x97"
+			  "\x40\xD7\xCB\x05",
+		.ilen	= 68,
+	}, {
+		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
+			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
+			  "\x22\x43\x3C\x64",
+		.klen	= 20,
+		.iv	= "\x48\x55\xEC\x7D\x3A\x23\x4B\xFD",
+		.result	= "\x08\x00\xC6\xCD\x02\x00\x07\x00"
+			  "\x61\x62\x63\x64\x65\x66\x67\x68"
+			  "\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70"
+			  "\x71\x72\x73\x74\x01\x02\x02\x01",
+		.rlen	= 32,
+		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
+			  "\x00\x00\x00\x07\x48\x55\xEC\x7D"
+			  "\x3A\x23\x4B\xFD",
+		.alen	= 20,
+		.input	= "\x74\x75\x2E\x8A\xEB\x5D\x87\x3C"
+			  "\xD7\xC0\xF4\xAC\xC3\x6C\x4B\xFF"
+			  "\x84\xB7\xD7\xB9\x8F\x0C\xA8\xB6"
+			  "\xAC\xDA\x68\x94\xBC\x61\x90\x69"
+			  "\xEF\x9C\xBC\x28\xFE\x1B\x56\xA7"
+			  "\xC4\xE0\xD5\x8C\x86\xCD\x2B\xC0",
+		.ilen	= 48,
+	}
+};
+
+static const struct aead_testvec aes_gcm_rfc4543_enc_tv_template[] = {
+	{ /* From draft-mcgrew-gcm-test-01 */
+		.key	= "\x4c\x80\xcd\xef\xbb\x5d\x10\xda"
+			  "\x90\x6a\xc7\x3c\x36\x13\xa6\x34"
+			  "\x22\x43\x3c\x64",
+		.klen	= 20,
+		.iv	= zeroed_string,
+		.assoc	= "\x00\x00\x43\x21\x00\x00\x00\x07"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 16,
+		.input	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
+			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
+			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
+			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01",
+		.ilen	= 52,
+		.result	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
+			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
+			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
+			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01\xf2\xa9\xa8\x36"
+			  "\xe1\x55\x10\x6a\xa8\xdc\xd6\x18"
+			  "\xe4\x09\x9a\xaa",
+		.rlen	= 68,
+	}
+};
+
+static const struct aead_testvec aes_gcm_rfc4543_dec_tv_template[] = {
+	{ /* From draft-mcgrew-gcm-test-01 */
+		.key	= "\x4c\x80\xcd\xef\xbb\x5d\x10\xda"
+			  "\x90\x6a\xc7\x3c\x36\x13\xa6\x34"
+			  "\x22\x43\x3c\x64",
+		.klen	= 20,
+		.iv	= zeroed_string,
+		.assoc	= "\x00\x00\x43\x21\x00\x00\x00\x07"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 16,
+		.input	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
+			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
+			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
+			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01\xf2\xa9\xa8\x36"
+			  "\xe1\x55\x10\x6a\xa8\xdc\xd6\x18"
+			  "\xe4\x09\x9a\xaa",
+		.ilen	= 68,
+		.result	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
+			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
+			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
+			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01",
+		.rlen	= 52,
+	}, { /* nearly same as previous, but should fail */
+		.key	= "\x4c\x80\xcd\xef\xbb\x5d\x10\xda"
+			  "\x90\x6a\xc7\x3c\x36\x13\xa6\x34"
+			  "\x22\x43\x3c\x64",
+		.klen	= 20,
+		.iv	= zeroed_string,
+		.assoc	= "\x00\x00\x43\x21\x00\x00\x00\x07"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 16,
+		.input	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
+			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
+			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
+			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01\xf2\xa9\xa8\x36"
+			  "\xe1\x55\x10\x6a\xa8\xdc\xd6\x18"
+			  "\x00\x00\x00\x00",
+		.ilen	= 68,
+		.novrfy = 1,
+		.result	= "\x45\x00\x00\x30\xda\x3a\x00\x00"
+			  "\x80\x01\xdf\x3b\xc0\xa8\x00\x05"
+			  "\xc0\xa8\x00\x01\x08\x00\xc6\xcd"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6a\x6b\x6c"
+			  "\x6d\x6e\x6f\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01",
+		.rlen	= 52,
+	},
+};
+
+static const struct aead_testvec aes_ccm_enc_tv_template[] = {
+	{ /* From RFC 3610 */
+		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
+		.klen	= 16,
+		.iv	= "\x01\x00\x00\x00\x03\x02\x01\x00"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07",
+		.alen	= 8,
+		.input	= "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e",
+		.ilen	= 23,
+		.result	= "\x58\x8c\x97\x9a\x61\xc6\x63\xd2"
+			  "\xf0\x66\xd0\xc2\xc0\xf9\x89\x80"
+			  "\x6d\x5f\x6b\x61\xda\xc3\x84\x17"
+			  "\xe8\xd1\x2c\xfd\xf9\x26\xe0",
+		.rlen	= 31,
+	}, {
+		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
+		.klen	= 16,
+		.iv	= "\x01\x00\x00\x00\x07\x06\x05\x04"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b",
+		.alen	= 12,
+		.input	= "\x0c\x0d\x0e\x0f\x10\x11\x12\x13"
+			  "\x14\x15\x16\x17\x18\x19\x1a\x1b"
+			  "\x1c\x1d\x1e\x1f",
+		.ilen	= 20,
+		.result	= "\xdc\xf1\xfb\x7b\x5d\x9e\x23\xfb"
+			  "\x9d\x4e\x13\x12\x53\x65\x8a\xd8"
+			  "\x6e\xbd\xca\x3e\x51\xe8\x3f\x07"
+			  "\x7d\x9c\x2d\x93",
+		.rlen	= 28,
+	}, {
+		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
+		.klen	= 16,
+		.iv	= "\x01\x00\x00\x00\x0b\x0a\x09\x08"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07",
+		.alen	= 8,
+		.input	= "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20",
+		.ilen	= 25,
+		.result	= "\x82\x53\x1a\x60\xcc\x24\x94\x5a"
+			  "\x4b\x82\x79\x18\x1a\xb5\xc8\x4d"
+			  "\xf2\x1c\xe7\xf9\xb7\x3f\x42\xe1"
+			  "\x97\xea\x9c\x07\xe5\x6b\x5e\xb1"
+			  "\x7e\x5f\x4e",
+		.rlen	= 35,
+	}, {
+		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
+		.klen	= 16,
+		.iv	= "\x01\x00\x00\x00\x0c\x0b\x0a\x09"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b",
+		.alen	= 12,
+		.input	= "\x0c\x0d\x0e\x0f\x10\x11\x12\x13"
+			  "\x14\x15\x16\x17\x18\x19\x1a\x1b"
+			  "\x1c\x1d\x1e",
+		.ilen	= 19,
+		.result	= "\x07\x34\x25\x94\x15\x77\x85\x15"
+			  "\x2b\x07\x40\x98\x33\x0a\xbb\x14"
+			  "\x1b\x94\x7b\x56\x6a\xa9\x40\x6b"
+			  "\x4d\x99\x99\x88\xdd",
+		.rlen	= 29,
+	}, {
+		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
+			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
+		.klen	= 16,
+		.iv	= "\x01\x00\x33\x56\x8e\xf7\xb2\x63"
+			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
+		.assoc	= "\x63\x01\x8f\x76\xdc\x8a\x1b\xcb",
+		.alen	= 8,
+		.input	= "\x90\x20\xea\x6f\x91\xbd\xd8\x5a"
+			  "\xfa\x00\x39\xba\x4b\xaf\xf9\xbf"
+			  "\xb7\x9c\x70\x28\x94\x9c\xd0\xec",
+		.ilen	= 24,
+		.result	= "\x4c\xcb\x1e\x7c\xa9\x81\xbe\xfa"
+			  "\xa0\x72\x6c\x55\xd3\x78\x06\x12"
+			  "\x98\xc8\x5c\x92\x81\x4a\xbc\x33"
+			  "\xc5\x2e\xe8\x1d\x7d\x77\xc0\x8a",
+		.rlen	= 32,
+	}, {
+		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
+			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
+		.klen	= 16,
+		.iv	= "\x01\x00\xd5\x60\x91\x2d\x3f\x70"
+			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
+		.assoc	= "\xcd\x90\x44\xd2\xb7\x1f\xdb\x81"
+			  "\x20\xea\x60\xc0",
+		.alen	= 12,
+		.input	= "\x64\x35\xac\xba\xfb\x11\xa8\x2e"
+			  "\x2f\x07\x1d\x7c\xa4\xa5\xeb\xd9"
+			  "\x3a\x80\x3b\xa8\x7f",
+		.ilen	= 21,
+		.result	= "\x00\x97\x69\xec\xab\xdf\x48\x62"
+			  "\x55\x94\xc5\x92\x51\xe6\x03\x57"
+			  "\x22\x67\x5e\x04\xc8\x47\x09\x9e"
+			  "\x5a\xe0\x70\x45\x51",
+		.rlen	= 29,
+	}, {
+		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
+			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
+		.klen	= 16,
+		.iv	= "\x01\x00\x42\xff\xf8\xf1\x95\x1c"
+			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
+		.assoc	= "\xd8\x5b\xc7\xe6\x9f\x94\x4f\xb8",
+		.alen	= 8,
+		.input	= "\x8a\x19\xb9\x50\xbc\xf7\x1a\x01"
+			  "\x8e\x5e\x67\x01\xc9\x17\x87\x65"
+			  "\x98\x09\xd6\x7d\xbe\xdd\x18",
+		.ilen	= 23,
+		.result	= "\xbc\x21\x8d\xaa\x94\x74\x27\xb6"
+			  "\xdb\x38\x6a\x99\xac\x1a\xef\x23"
+			  "\xad\xe0\xb5\x29\x39\xcb\x6a\x63"
+			  "\x7c\xf9\xbe\xc2\x40\x88\x97\xc6"
+			  "\xba",
+		.rlen	= 33,
+	}, {
+		/* This is taken from FIPS CAVS. */
+		.key	= "\x83\xac\x54\x66\xc2\xeb\xe5\x05"
+			  "\x2e\x01\xd1\xfc\x5d\x82\x66\x2e",
+		.klen	= 16,
+		.iv	= "\x03\x96\xac\x59\x30\x07\xa1\xe2\xa2\xc7\x55\x24\0\0\0\0",
+		.alen	= 0,
+		.input	= "\x19\xc8\x81\xf6\xe9\x86\xff\x93"
+			  "\x0b\x78\x67\xe5\xbb\xb7\xfc\x6e"
+			  "\x83\x77\xb3\xa6\x0c\x8c\x9f\x9c"
+			  "\x35\x2e\xad\xe0\x62\xf9\x91\xa1",
+		.ilen	= 32,
+		.result	= "\xab\x6f\xe1\x69\x1d\x19\x99\xa8"
+			  "\x92\xa0\xc4\x6f\x7e\xe2\x8b\xb1"
+			  "\x70\xbb\x8c\xa6\x4c\x6e\x97\x8a"
+			  "\x57\x2b\xbe\x5d\x98\xa6\xb1\x32"
+			  "\xda\x24\xea\xd9\xa1\x39\x98\xfd"
+			  "\xa4\xbe\xd9\xf2\x1a\x6d\x22\xa8",
+		.rlen	= 48,
+	}, {
+		.key	= "\x1e\x2c\x7e\x01\x41\x9a\xef\xc0"
+			  "\x0d\x58\x96\x6e\x5c\xa2\x4b\xd3",
+		.klen	= 16,
+		.iv	= "\x03\x4f\xa3\x19\xd3\x01\x5a\xd8"
+			  "\x30\x60\x15\x56\x00\x00\x00\x00",
+		.assoc	= "\xda\xe6\x28\x9c\x45\x2d\xfd\x63"
+			  "\x5e\xda\x4c\xb6\xe6\xfc\xf9\xb7"
+			  "\x0c\x56\xcb\xe4\xe0\x05\x7a\xe1"
+			  "\x0a\x63\x09\x78\xbc\x2c\x55\xde",
+		.alen	= 32,
+		.input	= "\x87\xa3\x36\xfd\x96\xb3\x93\x78"
+			  "\xa9\x28\x63\xba\x12\xa3\x14\x85"
+			  "\x57\x1e\x06\xc9\x7b\x21\xef\x76"
+			  "\x7f\x38\x7e\x8e\x29\xa4\x3e\x7e",
+		.ilen	= 32,
+		.result	= "\x8a\x1e\x11\xf0\x02\x6b\xe2\x19"
+			  "\xfc\x70\xc4\x6d\x8e\xb7\x99\xab"
+			  "\xc5\x4b\xa2\xac\xd3\xf3\x48\xff"
+			  "\x3b\xb5\xce\x53\xef\xde\xbb\x02"
+			  "\xa9\x86\x15\x6c\x13\xfe\xda\x0a"
+			  "\x22\xb8\x29\x3d\xd8\x39\x9a\x23",
+		.rlen	= 48,
+	}, {
+		.key	= "\xf4\x6b\xc2\x75\x62\xfe\xb4\xe1"
+			  "\xa3\xf0\xff\xdd\x4e\x4b\x12\x75"
+			  "\x53\x14\x73\x66\x8d\x88\xf6\x80",
+		.klen	= 24,
+		.iv	= "\x03\xa0\x20\x35\x26\xf2\x21\x8d"
+			  "\x50\x20\xda\xe2\x00\x00\x00\x00",
+		.assoc	= "\x5b\x9e\x13\x67\x02\x5e\xef\xc1"
+			  "\x6c\xf9\xd7\x1e\x52\x8f\x7a\x47"
+			  "\xe9\xd4\xcf\x20\x14\x6e\xf0\x2d"
+			  "\xd8\x9e\x2b\x56\x10\x23\x56\xe7",
+		.alen	= 32,
+		.result	= "\x36\xea\x7a\x70\x08\xdc\x6a\xbc"
+			  "\xad\x0c\x7a\x63\xf6\x61\xfd\x9b",
+		.rlen	= 16,
+	}, {
+		.key	= "\x56\xdf\x5c\x8f\x26\x3f\x0e\x42"
+			  "\xef\x7a\xd3\xce\xfc\x84\x60\x62"
+			  "\xca\xb4\x40\xaf\x5f\xc9\xc9\x01",
+		.klen	= 24,
+		.iv	= "\x03\xd6\x3c\x8c\x86\x84\xb6\xcd"
+			  "\xef\x09\x2e\x94\x00\x00\x00\x00",
+		.assoc	= "\x02\x65\x78\x3c\xe9\x21\x30\x91"
+			  "\xb1\xb9\xda\x76\x9a\x78\x6d\x95"
+			  "\xf2\x88\x32\xa3\xf2\x50\xcb\x4c"
+			  "\xe3\x00\x73\x69\x84\x69\x87\x79",
+		.alen	= 32,
+		.input	= "\x9f\xd2\x02\x4b\x52\x49\x31\x3c"
+			  "\x43\x69\x3a\x2d\x8e\x70\xad\x7e"
+			  "\xe0\xe5\x46\x09\x80\x89\x13\xb2"
+			  "\x8c\x8b\xd9\x3f\x86\xfb\xb5\x6b",
+		.ilen	= 32,
+		.result	= "\x39\xdf\x7c\x3c\x5a\x29\xb9\x62"
+			  "\x5d\x51\xc2\x16\xd8\xbd\x06\x9f"
+			  "\x9b\x6a\x09\x70\xc1\x51\x83\xc2"
+			  "\x66\x88\x1d\x4f\x9a\xda\xe0\x1e"
+			  "\xc7\x79\x11\x58\xe5\x6b\x20\x40"
+			  "\x7a\xea\x46\x42\x8b\xe4\x6f\xe1",
+		.rlen	= 48,
+	}, {
+		.key	= "\xe0\x8d\x99\x71\x60\xd7\x97\x1a"
+			  "\xbd\x01\x99\xd5\x8a\xdf\x71\x3a"
+			  "\xd3\xdf\x24\x4b\x5e\x3d\x4b\x4e"
+			  "\x30\x7a\xb9\xd8\x53\x0a\x5e\x2b",
+		.klen	= 32,
+		.iv	= "\x03\x1e\x29\x91\xad\x8e\xc1\x53"
+			  "\x0a\xcf\x2d\xbe\x00\x00\x00\x00",
+		.assoc	= "\x19\xb6\x1f\x57\xc4\xf3\xf0\x8b"
+			  "\x78\x2b\x94\x02\x29\x0f\x42\x27"
+			  "\x6b\x75\xcb\x98\x34\x08\x7e\x79"
+			  "\xe4\x3e\x49\x0d\x84\x8b\x22\x87",
+		.alen	= 32,
+		.input	= "\xe1\xd9\xd8\x13\xeb\x3a\x75\x3f"
+			  "\x9d\xbd\x5f\x66\xbe\xdc\xbb\x66"
+			  "\xbf\x17\x99\x62\x4a\x39\x27\x1f"
+			  "\x1d\xdc\x24\xae\x19\x2f\x98\x4c",
+		.ilen	= 32,
+		.result	= "\x19\xb8\x61\x33\x45\x2b\x43\x96"
+			  "\x6f\x51\xd0\x20\x30\x7d\x9b\xc6"
+			  "\x26\x3d\xf8\xc9\x65\x16\xa8\x9f"
+			  "\xf0\x62\x17\x34\xf2\x1e\x8d\x75"
+			  "\x4e\x13\xcc\xc0\xc3\x2a\x54\x2d",
+		.rlen	= 40,
+	}, {
+		.key	= "\x7c\xc8\x18\x3b\x8d\x99\xe0\x7c"
+			  "\x45\x41\xb8\xbd\x5c\xa7\xc2\x32"
+			  "\x8a\xb8\x02\x59\xa4\xfe\xa9\x2c"
+			  "\x09\x75\x9a\x9b\x3c\x9b\x27\x39",
+		.klen	= 32,
+		.iv	= "\x03\xf9\xd9\x4e\x63\xb5\x3d\x9d"
+			  "\x43\xf6\x1e\x50\0\0\0\0",
+		.assoc	= "\x57\xf5\x6b\x8b\x57\x5c\x3d\x3b"
+			  "\x13\x02\x01\x0c\x83\x4c\x96\x35"
+			  "\x8e\xd6\x39\xcf\x7d\x14\x9b\x94"
+			  "\xb0\x39\x36\xe6\x8f\x57\xe0\x13",
+		.alen	= 32,
+		.input	= "\x3b\x6c\x29\x36\xb6\xef\x07\xa6"
+			  "\x83\x72\x07\x4f\xcf\xfa\x66\x89"
+			  "\x5f\xca\xb1\xba\xd5\x8f\x2c\x27"
+			  "\x30\xdb\x75\x09\x93\xd4\x65\xe4",
+		.ilen	= 32,
+		.result	= "\xb0\x88\x5a\x33\xaa\xe5\xc7\x1d"
+			  "\x85\x23\xc7\xc6\x2f\xf4\x1e\x3d"
+			  "\xcc\x63\x44\x25\x07\x78\x4f\x9e"
+			  "\x96\xb8\x88\xeb\xbc\x48\x1f\x06"
+			  "\x39\xaf\x39\xac\xd8\x4a\x80\x39"
+			  "\x7b\x72\x8a\xf7",
+		.rlen	= 44,
+	}, {
+		.key	= "\xab\xd0\xe9\x33\x07\x26\xe5\x83"
+			  "\x8c\x76\x95\xd4\xb6\xdc\xf3\x46"
+			  "\xf9\x8f\xad\xe3\x02\x13\x83\x77"
+			  "\x3f\xb0\xf1\xa1\xa1\x22\x0f\x2b",
+		.klen	= 32,
+		.iv	= "\x03\x24\xa7\x8b\x07\xcb\xcc\x0e"
+			  "\xe6\x33\xbf\xf5\x00\x00\x00\x00",
+		.assoc	= "\xd4\xdb\x30\x1d\x03\xfe\xfd\x5f"
+			  "\x87\xd4\x8c\xb6\xb6\xf1\x7a\x5d"
+			  "\xab\x90\x65\x8d\x8e\xca\x4d\x4f"
+			  "\x16\x0c\x40\x90\x4b\xc7\x36\x73",
+		.alen	= 32,
+		.input	= "\xf5\xc6\x7d\x48\xc1\xb7\xe6\x92"
+			  "\x97\x5a\xca\xc4\xa9\x6d\xf9\x3d"
+			  "\x6c\xde\xbc\xf1\x90\xea\x6a\xb2"
+			  "\x35\x86\x36\xaf\x5c\xfe\x4b\x3a",
+		.ilen	= 32,
+		.result	= "\x83\x6f\x40\x87\x72\xcf\xc1\x13"
+			  "\xef\xbb\x80\x21\x04\x6c\x58\x09"
+			  "\x07\x1b\xfc\xdf\xc0\x3f\x5b\xc7"
+			  "\xe0\x79\xa8\x6e\x71\x7c\x3f\xcf"
+			  "\x5c\xda\xb2\x33\xe5\x13\xe2\x0d"
+			  "\x74\xd1\xef\xb5\x0f\x3a\xb5\xf8",
+		.rlen	= 48,
+	}
+};
+
+static const struct aead_testvec aes_ccm_dec_tv_template[] = {
+	{ /* From RFC 3610 */
+		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
+		.klen	= 16,
+		.iv	= "\x01\x00\x00\x00\x03\x02\x01\x00"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07",
+		.alen	= 8,
+		.input	= "\x58\x8c\x97\x9a\x61\xc6\x63\xd2"
+			  "\xf0\x66\xd0\xc2\xc0\xf9\x89\x80"
+			  "\x6d\x5f\x6b\x61\xda\xc3\x84\x17"
+			  "\xe8\xd1\x2c\xfd\xf9\x26\xe0",
+		.ilen	= 31,
+		.result	= "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e",
+		.rlen	= 23,
+	}, {
+		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
+		.klen	= 16,
+		.iv	= "\x01\x00\x00\x00\x07\x06\x05\x04"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b",
+		.alen	= 12,
+		.input	= "\xdc\xf1\xfb\x7b\x5d\x9e\x23\xfb"
+			  "\x9d\x4e\x13\x12\x53\x65\x8a\xd8"
+			  "\x6e\xbd\xca\x3e\x51\xe8\x3f\x07"
+			  "\x7d\x9c\x2d\x93",
+		.ilen	= 28,
+		.result	= "\x0c\x0d\x0e\x0f\x10\x11\x12\x13"
+			  "\x14\x15\x16\x17\x18\x19\x1a\x1b"
+			  "\x1c\x1d\x1e\x1f",
+		.rlen	= 20,
+	}, {
+		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
+		.klen	= 16,
+		.iv	= "\x01\x00\x00\x00\x0b\x0a\x09\x08"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07",
+		.alen	= 8,
+		.input	= "\x82\x53\x1a\x60\xcc\x24\x94\x5a"
+			  "\x4b\x82\x79\x18\x1a\xb5\xc8\x4d"
+			  "\xf2\x1c\xe7\xf9\xb7\x3f\x42\xe1"
+			  "\x97\xea\x9c\x07\xe5\x6b\x5e\xb1"
+			  "\x7e\x5f\x4e",
+		.ilen	= 35,
+		.result	= "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20",
+		.rlen	= 25,
+	}, {
+		.key	= "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf",
+		.klen	= 16,
+		.iv	= "\x01\x00\x00\x00\x0c\x0b\x0a\x09"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\x00\x00",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b",
+		.alen	= 12,
+		.input	= "\x07\x34\x25\x94\x15\x77\x85\x15"
+			  "\x2b\x07\x40\x98\x33\x0a\xbb\x14"
+			  "\x1b\x94\x7b\x56\x6a\xa9\x40\x6b"
+			  "\x4d\x99\x99\x88\xdd",
+		.ilen	= 29,
+		.result	= "\x0c\x0d\x0e\x0f\x10\x11\x12\x13"
+			  "\x14\x15\x16\x17\x18\x19\x1a\x1b"
+			  "\x1c\x1d\x1e",
+		.rlen	= 19,
+	}, {
+		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
+			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
+		.klen	= 16,
+		.iv	= "\x01\x00\x33\x56\x8e\xf7\xb2\x63"
+			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
+		.assoc	= "\x63\x01\x8f\x76\xdc\x8a\x1b\xcb",
+		.alen	= 8,
+		.input	= "\x4c\xcb\x1e\x7c\xa9\x81\xbe\xfa"
+			  "\xa0\x72\x6c\x55\xd3\x78\x06\x12"
+			  "\x98\xc8\x5c\x92\x81\x4a\xbc\x33"
+			  "\xc5\x2e\xe8\x1d\x7d\x77\xc0\x8a",
+		.ilen	= 32,
+		.result	= "\x90\x20\xea\x6f\x91\xbd\xd8\x5a"
+			  "\xfa\x00\x39\xba\x4b\xaf\xf9\xbf"
+			  "\xb7\x9c\x70\x28\x94\x9c\xd0\xec",
+		.rlen	= 24,
+	}, {
+		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
+			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
+		.klen	= 16,
+		.iv	= "\x01\x00\xd5\x60\x91\x2d\x3f\x70"
+			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
+		.assoc	= "\xcd\x90\x44\xd2\xb7\x1f\xdb\x81"
+			  "\x20\xea\x60\xc0",
+		.alen	= 12,
+		.input	= "\x00\x97\x69\xec\xab\xdf\x48\x62"
+			  "\x55\x94\xc5\x92\x51\xe6\x03\x57"
+			  "\x22\x67\x5e\x04\xc8\x47\x09\x9e"
+			  "\x5a\xe0\x70\x45\x51",
+		.ilen	= 29,
+		.result	= "\x64\x35\xac\xba\xfb\x11\xa8\x2e"
+			  "\x2f\x07\x1d\x7c\xa4\xa5\xeb\xd9"
+			  "\x3a\x80\x3b\xa8\x7f",
+		.rlen	= 21,
+	}, {
+		.key	= "\xd7\x82\x8d\x13\xb2\xb0\xbd\xc3"
+			  "\x25\xa7\x62\x36\xdf\x93\xcc\x6b",
+		.klen	= 16,
+		.iv	= "\x01\x00\x42\xff\xf8\xf1\x95\x1c"
+			  "\x3c\x96\x96\x76\x6c\xfa\x00\x00",
+		.assoc	= "\xd8\x5b\xc7\xe6\x9f\x94\x4f\xb8",
+		.alen	= 8,
+		.input	= "\xbc\x21\x8d\xaa\x94\x74\x27\xb6"
+			  "\xdb\x38\x6a\x99\xac\x1a\xef\x23"
+			  "\xad\xe0\xb5\x29\x39\xcb\x6a\x63"
+			  "\x7c\xf9\xbe\xc2\x40\x88\x97\xc6"
+			  "\xba",
+		.ilen	= 33,
+		.result	= "\x8a\x19\xb9\x50\xbc\xf7\x1a\x01"
+			  "\x8e\x5e\x67\x01\xc9\x17\x87\x65"
+			  "\x98\x09\xd6\x7d\xbe\xdd\x18",
+		.rlen	= 23,
+	}, {
+		/* This is taken from FIPS CAVS. */
+		.key	= "\xab\x2f\x8a\x74\xb7\x1c\xd2\xb1"
+			  "\xff\x80\x2e\x48\x7d\x82\xf8\xb9",
+		.klen	= 16,
+		.iv	= "\x03\xc6\xfb\x7d\x80\x0d\x13\xab"
+			  "\xd8\xa6\xb2\xd8\x00\x00\x00\x00",
+		.alen	= 0,
+		.input	= "\xd5\xe8\x93\x9f\xc7\x89\x2e\x2b",
+		.ilen	= 8,
+		.result	= "\x00",
+		.rlen	= 0,
+		.novrfy	= 1,
+	}, {
+		.key	= "\xab\x2f\x8a\x74\xb7\x1c\xd2\xb1"
+			  "\xff\x80\x2e\x48\x7d\x82\xf8\xb9",
+		.klen	= 16,
+		.iv	= "\x03\xaf\x94\x87\x78\x35\x82\x81"
+			  "\x7f\x88\x94\x68\x00\x00\x00\x00",
+		.alen	= 0,
+		.input	= "\x41\x3c\xb8\x87\x73\xcb\xf3\xf3",
+		.ilen	= 8,
+		.result	= "\x00",
+		.rlen	= 0,
+	}, {
+		.key	= "\x61\x0e\x8c\xae\xe3\x23\xb6\x38"
+			  "\x76\x1c\xf6\x3a\x67\xa3\x9c\xd8",
+		.klen	= 16,
+		.iv	= "\x03\xc6\xfb\x7d\x80\x0d\x13\xab"
+			  "\xd8\xa6\xb2\xd8\x00\x00\x00\x00",
+		.assoc	= "\xf3\x94\x87\x78\x35\x82\x81\x7f"
+			  "\x88\x94\x68\xb1\x78\x6b\x2b\xd6"
+			  "\x04\x1f\x4e\xed\x78\xd5\x33\x66"
+			  "\xd8\x94\x99\x91\x81\x54\x62\x57",
+		.alen	= 32,
+		.input	= "\xf0\x7c\x29\x02\xae\x1c\x2f\x55"
+			  "\xd0\xd1\x3d\x1a\xa3\x6d\xe4\x0a"
+			  "\x86\xb0\x87\x6b\x62\x33\x8c\x34"
+			  "\xce\xab\x57\xcc\x79\x0b\xe0\x6f"
+			  "\x5c\x3e\x48\x1f\x6c\x46\xf7\x51"
+			  "\x8b\x84\x83\x2a\xc1\x05\xb8\xc5",
+		.ilen	= 48,
+		.result	= "\x50\x82\x3e\x07\xe2\x1e\xb6\xfb"
+			  "\x33\xe4\x73\xce\xd2\xfb\x95\x79"
+			  "\xe8\xb4\xb5\x77\x11\x10\x62\x6f"
+			  "\x6a\x82\xd1\x13\xec\xf5\xd0\x48",
+		.rlen	= 32,
+		.novrfy	= 1,
+	}, {
+		.key	= "\x61\x0e\x8c\xae\xe3\x23\xb6\x38"
+			  "\x76\x1c\xf6\x3a\x67\xa3\x9c\xd8",
+		.klen	= 16,
+		.iv	= "\x03\x05\xe0\xc9\x0f\xed\x34\xea"
+			  "\x97\xd4\x3b\xdf\x00\x00\x00\x00",
+		.assoc	= "\x49\x5c\x50\x1f\x1d\x94\xcc\x81"
+			  "\xba\xb7\xb6\x03\xaf\xa5\xc1\xa1"
+			  "\xd8\x5c\x42\x68\xe0\x6c\xda\x89"
+			  "\x05\xac\x56\xac\x1b\x2a\xd3\x86",
+		.alen	= 32,
+		.input	= "\x39\xbe\x7d\x15\x62\x77\xf3\x3c"
+			  "\xad\x83\x52\x6d\x71\x03\x25\x1c"
+			  "\xed\x81\x3a\x9a\x16\x7d\x19\x80"
+			  "\x72\x04\x72\xd0\xf6\xff\x05\x0f"
+			  "\xb7\x14\x30\x00\x32\x9e\xa0\xa6"
+			  "\x9e\x5a\x18\xa1\xb8\xfe\xdb\xd3",
+		.ilen	= 48,
+		.result	= "\x75\x05\xbe\xc2\xd9\x1e\xde\x60"
+			  "\x47\x3d\x8c\x7d\xbd\xb5\xd9\xb7"
+			  "\xf2\xae\x61\x05\x8f\x82\x24\x3f"
+			  "\x9c\x67\x91\xe1\x38\x4f\xe4\x0c",
+		.rlen	= 32,
+	}, {
+		.key	= "\x39\xbb\xa7\xbe\x59\x97\x9e\x73"
+			  "\xa2\xbc\x6b\x98\xd7\x75\x7f\xe3"
+			  "\xa4\x48\x93\x39\x26\x71\x4a\xc6",
+		.klen	= 24,
+		.iv	= "\x03\xee\x49\x83\xe9\xa9\xff\xe9"
+			  "\x57\xba\xfd\x9e\x00\x00\x00\x00",
+		.assoc	= "\x44\xa6\x2c\x05\xe9\xe1\x43\xb1"
+			  "\x58\x7c\xf2\x5c\x6d\x39\x0a\x64"
+			  "\xa4\xf0\x13\x05\xd1\x77\x99\x67"
+			  "\x11\xc4\xc6\xdb\x00\x56\x36\x61",
+		.alen	= 32,
+		.input	= "\x71\x99\xfa\xf4\x44\x12\x68\x9b",
+		.ilen	= 8,
+		.result	= "\x00",
+		.rlen	= 0,
+	}, {
+		.key	= "\x58\x5d\xa0\x96\x65\x1a\x04\xd7"
+			  "\x96\xe5\xc5\x68\xaa\x95\x35\xe0"
+			  "\x29\xa0\xba\x9e\x48\x78\xd1\xba",
+		.klen	= 24,
+		.iv	= "\x03\xee\x49\x83\xe9\xa9\xff\xe9"
+			  "\x57\xba\xfd\x9e\x00\x00\x00\x00",
+		.assoc	= "\x44\xa6\x2c\x05\xe9\xe1\x43\xb1"
+			  "\x58\x7c\xf2\x5c\x6d\x39\x0a\x64"
+			  "\xa4\xf0\x13\x05\xd1\x77\x99\x67"
+			  "\x11\xc4\xc6\xdb\x00\x56\x36\x61",
+		.alen	= 32,
+		.input	= "\xfb\xe5\x5d\x34\xbe\xe5\xe8\xe7"
+			  "\x5a\xef\x2f\xbf\x1f\x7f\xd4\xb2"
+			  "\x66\xca\x61\x1e\x96\x7a\x61\xb3"
+			  "\x1c\x16\x45\x52\xba\x04\x9c\x9f"
+			  "\xb1\xd2\x40\xbc\x52\x7c\x6f\xb1",
+		.ilen	= 40,
+		.result	= "\x85\x34\x66\x42\xc8\x92\x0f\x36"
+			  "\x58\xe0\x6b\x91\x3c\x98\x5c\xbb"
+			  "\x0a\x85\xcc\x02\xad\x7a\x96\xe9"
+			  "\x65\x43\xa4\xc3\x0f\xdc\x55\x81",
+		.rlen	= 32,
+	}, {
+		.key	= "\x58\x5d\xa0\x96\x65\x1a\x04\xd7"
+			  "\x96\xe5\xc5\x68\xaa\x95\x35\xe0"
+			  "\x29\xa0\xba\x9e\x48\x78\xd1\xba",
+		.klen	= 24,
+		.iv	= "\x03\xd1\xfc\x57\x9c\xfe\xb8\x9c"
+			  "\xad\x71\xaa\x1f\x00\x00\x00\x00",
+		.assoc	= "\x86\x67\xa5\xa9\x14\x5f\x0d\xc6"
+			  "\xff\x14\xc7\x44\xbf\x6c\x3a\xc3"
+			  "\xff\xb6\x81\xbd\xe2\xd5\x06\xc7"
+			  "\x3c\xa1\x52\x13\x03\x8a\x23\x3a",
+		.alen	= 32,
+		.input	= "\x3f\x66\xb0\x9d\xe5\x4b\x38\x00"
+			  "\xc6\x0e\x6e\xe5\xd6\x98\xa6\x37"
+			  "\x8c\x26\x33\xc6\xb2\xa2\x17\xfa"
+			  "\x64\x19\xc0\x30\xd7\xfc\x14\x6b"
+			  "\xe3\x33\xc2\x04\xb0\x37\xbe\x3f"
+			  "\xa9\xb4\x2d\x68\x03\xa3\x44\xef",
+		.ilen	= 48,
+		.result	= "\x02\x87\x4d\x28\x80\x6e\xb2\xed"
+			  "\x99\x2a\xa8\xca\x04\x25\x45\x90"
+			  "\x1d\xdd\x5a\xd9\xe4\xdb\x9c\x9c"
+			  "\x49\xe9\x01\xfe\xa7\x80\x6d\x6b",
+		.rlen	= 32,
+		.novrfy	= 1,
+	}, {
+		.key	= "\xa4\x4b\x54\x29\x0a\xb8\x6d\x01"
+			  "\x5b\x80\x2a\xcf\x25\xc4\xb7\x5c"
+			  "\x20\x2c\xad\x30\xc2\x2b\x41\xfb"
+			  "\x0e\x85\xbc\x33\xad\x0f\x2b\xff",
+		.klen	= 32,
+		.iv	= "\x03\xee\x49\x83\xe9\xa9\xff\xe9"
+			  "\x57\xba\xfd\x9e\x00\x00\x00\x00",
+		.alen	= 0,
+		.input	= "\x1f\xb8\x8f\xa3\xdd\x54\x00\xf2",
+		.ilen	= 8,
+		.result	= "\x00",
+		.rlen	= 0,
+	}, {
+		.key	= "\x39\xbb\xa7\xbe\x59\x97\x9e\x73"
+			  "\xa2\xbc\x6b\x98\xd7\x75\x7f\xe3"
+			  "\xa4\x48\x93\x39\x26\x71\x4a\xc6"
+			  "\xae\x8f\x11\x4c\xc2\x9c\x4a\xbb",
+		.klen	= 32,
+		.iv	= "\x03\x85\x34\x66\x42\xc8\x92\x0f"
+			  "\x36\x58\xe0\x6b\x00\x00\x00\x00",
+		.alen	= 0,
+		.input	= "\x48\x01\x5e\x02\x24\x04\x66\x47"
+			  "\xa1\xea\x6f\xaf\xe8\xfc\xfb\xdd"
+			  "\xa5\xa9\x87\x8d\x84\xee\x2e\x77"
+			  "\xbb\x86\xb9\xf5\x5c\x6c\xff\xf6"
+			  "\x72\xc3\x8e\xf7\x70\xb1\xb2\x07"
+			  "\xbc\xa8\xa3\xbd\x83\x7c\x1d\x2a",
+		.ilen	= 48,
+		.result	= "\xdc\x56\xf2\x71\xb0\xb1\xa0\x6c"
+			  "\xf0\x97\x3a\xfb\x6d\xe7\x32\x99"
+			  "\x3e\xaf\x70\x5e\xb2\x4d\xea\x39"
+			  "\x89\xd4\x75\x7a\x63\xb1\xda\x93",
+		.rlen	= 32,
+		.novrfy	= 1,
+	}, {
+		.key	= "\x58\x5d\xa0\x96\x65\x1a\x04\xd7"
+			  "\x96\xe5\xc5\x68\xaa\x95\x35\xe0"
+			  "\x29\xa0\xba\x9e\x48\x78\xd1\xba"
+			  "\x0d\x1a\x53\x3b\xb5\xe3\xf8\x8b",
+		.klen	= 32,
+		.iv	= "\x03\xcf\x76\x3f\xd9\x95\x75\x8f"
+			  "\x44\x89\x40\x7b\x00\x00\x00\x00",
+		.assoc	= "\x8f\x86\x6c\x4d\x1d\xc5\x39\x88"
+			  "\xc8\xf3\x5c\x52\x10\x63\x6f\x2b"
+			  "\x8a\x2a\xc5\x6f\x30\x23\x58\x7b"
+			  "\xfb\x36\x03\x11\xb4\xd9\xf2\xfe",
+		.alen	= 32,
+		.input	= "\x48\x58\xd6\xf3\xad\x63\x58\xbf"
+			  "\xae\xc7\x5e\xae\x83\x8f\x7b\xe4"
+			  "\x78\x5c\x4c\x67\x71\x89\x94\xbf"
+			  "\x47\xf1\x63\x7e\x1c\x59\xbd\xc5"
+			  "\x7f\x44\x0a\x0c\x01\x18\x07\x92"
+			  "\xe1\xd3\x51\xce\x32\x6d\x0c\x5b",
+		.ilen	= 48,
+		.result	= "\xc2\x54\xc8\xde\x78\x87\x77\x40"
+			  "\x49\x71\xe4\xb7\xe7\xcb\x76\x61"
+			  "\x0a\x41\xb9\xe9\xc0\x76\x54\xab"
+			  "\x04\x49\x3b\x19\x93\x57\x25\x5d",
+		.rlen	= 32,
+	},
+};
+
+/*
+ * rfc4309 refers to section 8 of rfc3610 for test vectors, but they all
+ * use a 13-byte nonce, we only support an 11-byte nonce.  Worse,
+ * they use AD lengths which are not valid ESP header lengths.
+ *
+ * These vectors are copied/generated from the ones for rfc4106 with
+ * the key truncated by one byte..
+ */
+static const struct aead_testvec aes_ccm_rfc4309_enc_tv_template[] = {
+	{ /* Generated using Crypto++ */
+		.key	= zeroed_string,
+		.klen	= 19,
+		.iv	= zeroed_string,
+		.input	= zeroed_string,
+		.ilen	= 16,
+		.assoc	= zeroed_string,
+		.alen	= 16,
+		.result	= "\x2E\x9A\xCA\x6B\xDA\x54\xFC\x6F"
+			  "\x12\x50\xE8\xDE\x81\x3C\x63\x08"
+			  "\x1A\x22\xBA\x75\xEE\xD4\xD5\xB5"
+			  "\x27\x50\x01\xAC\x03\x33\x39\xFB",
+		.rlen	= 32,
+	},{
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.input	= zeroed_string,
+		.ilen	= 16,
+		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen	= 16,
+		.result	= "\xCF\xB9\x99\x17\xC8\x86\x0E\x7F"
+			  "\x7E\x76\xF8\xE6\xF8\xCC\x1F\x17"
+			  "\x6A\xE0\x53\x9F\x4B\x73\x7E\xDA"
+			  "\x08\x09\x4E\xC4\x1E\xAD\xC6\xB0",
+		.rlen	= 32,
+
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= zeroed_string,
+		.input	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.ilen	= 16,
+		.assoc	= zeroed_string,
+		.alen	= 16,
+		.result	= "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6"
+			  "\x61\xF4\xF5\x41\x03\x4A\xE3\x86"
+			  "\xA1\xE2\xC2\x42\x2B\x81\x70\x40"
+			  "\xFD\x7F\x76\xD1\x03\x07\xBB\x0C",
+		.rlen	= 32,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= zeroed_string,
+		.input	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.ilen	= 16,
+		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 16,
+		.result	= "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6"
+			  "\x61\xF4\xF5\x41\x03\x4A\xE3\x86"
+			  "\x5B\xC0\x73\xE0\x2B\x73\x68\xC9"
+			  "\x2D\x8C\x58\xC2\x90\x3D\xB0\x3E",
+		.rlen	= 32,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.input	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.ilen	= 16,
+		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen	= 16,
+		.result	= "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E"
+			  "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16"
+			  "\x43\x8E\x76\x57\x3B\xB4\x05\xE8"
+			  "\xA9\x9B\xBF\x25\xE0\x4F\xC0\xED",
+		.rlen	= 32,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.input	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.ilen	= 64,
+		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen	= 16,
+		.result	= "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E"
+			  "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16"
+			  "\x9C\xA4\x97\x83\x3F\x01\xA5\xF4"
+			  "\x43\x09\xE7\xB8\xE9\xD1\xD7\x02"
+			  "\x9B\xAB\x39\x18\xEB\x94\x34\x36"
+			  "\xE6\xC5\xC8\x9B\x00\x81\x9E\x49"
+			  "\x1D\x78\xE1\x48\xE3\xE9\xEA\x8E"
+			  "\x3A\x2B\x67\x5D\x35\x6A\x0F\xDB"
+			  "\x02\x73\xDD\xE7\x30\x4A\x30\x54"
+			  "\x1A\x9D\x09\xCA\xC8\x1C\x32\x5F",
+		.rlen	= 80,
+	}, {
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= "\x00\x00\x45\x67\x89\xab\xcd\xef",
+		.input	= "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff",
+		.ilen	= 192,
+		.assoc	= "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
+			  "\xaa\xaa\xaa\xaa\x00\x00\x45\x67"
+			  "\x89\xab\xcd\xef",
+		.alen	= 20,
+		.result	= "\x64\x17\xDC\x24\x9D\x92\xBA\x5E"
+			  "\x7C\x64\x6D\x33\x46\x77\xAC\xB1"
+			  "\x5C\x9E\xE2\xC7\x27\x11\x3E\x95"
+			  "\x7D\xBE\x28\xC8\xC1\xCA\x5E\x8C"
+			  "\xB4\xE2\xDE\x9F\x53\x59\x26\xDB"
+			  "\x0C\xD4\xE4\x07\x9A\xE6\x3E\x01"
+			  "\x58\x0D\x3E\x3D\xD5\x21\xEB\x04"
+			  "\x06\x9D\x5F\xB9\x02\x49\x1A\x2B"
+			  "\xBA\xF0\x4E\x3B\x85\x50\x5B\x09"
+			  "\xFE\xEC\xFC\x54\xEC\x0C\xE2\x79"
+			  "\x8A\x2F\x5F\xD7\x05\x5D\xF1\x6D"
+			  "\x22\xEB\xD1\x09\x80\x3F\x5A\x70"
+			  "\xB2\xB9\xD3\x63\x99\xC2\x4D\x1B"
+			  "\x36\x12\x00\x89\xAA\x5D\x55\xDA"
+			  "\x1D\x5B\xD8\x3C\x5F\x09\xD2\xE6"
+			  "\x39\x41\x5C\xF0\xBE\x26\x4E\x5F"
+			  "\x2B\x50\x44\x52\xC2\x10\x7D\x38"
+			  "\x82\x64\x83\x0C\xAE\x49\xD0\xE5"
+			  "\x4F\xE5\x66\x4C\x58\x7A\xEE\x43"
+			  "\x3B\x51\xFE\xBA\x24\x8A\xFE\xDC"
+			  "\x19\x6D\x60\x66\x61\xF9\x9A\x3F"
+			  "\x75\xFC\x38\x53\x5B\xB5\xCD\x52"
+			  "\x4F\xE5\xE4\xC9\xFE\x10\xCB\x98"
+			  "\xF0\x06\x5B\x07\xAB\xBB\xF4\x0E"
+			  "\x2D\xC2\xDD\x5D\xDD\x22\x9A\xCC"
+			  "\x39\xAB\x63\xA5\x3D\x9C\x51\x8A",
+		.rlen	= 208,
+	}, { /* From draft-mcgrew-gcm-test-01 */
+		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
+			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
+			  "\x2E\x44\x3B",
+		.klen	= 19,
+		.iv	= "\x49\x56\xED\x7E\x3B\x24\x4C\xFE",
+		.input	= "\x45\x00\x00\x48\x69\x9A\x00\x00"
+			  "\x80\x11\x4D\xB7\xC0\xA8\x01\x02"
+			  "\xC0\xA8\x01\x01\x0A\x9B\xF1\x56"
+			  "\x38\xD3\x01\x00\x00\x01\x00\x00"
+			  "\x00\x00\x00\x00\x04\x5F\x73\x69"
+			  "\x70\x04\x5F\x75\x64\x70\x03\x73"
+			  "\x69\x70\x09\x63\x79\x62\x65\x72"
+			  "\x63\x69\x74\x79\x02\x64\x6B\x00"
+			  "\x00\x21\x00\x01\x01\x02\x02\x01",
+		.ilen	= 72,
+		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
+			  "\x00\x00\x00\x00\x49\x56\xED\x7E"
+			  "\x3B\x24\x4C\xFE",
+		.alen	= 20,
+		.result	= "\x89\xBA\x3E\xEF\xE6\xD6\xCF\xDB"
+			  "\x83\x60\xF5\xBA\x3A\x56\x79\xE6"
+			  "\x7E\x0C\x53\xCF\x9E\x87\xE0\x4E"
+			  "\x1A\x26\x01\x24\xC7\x2E\x3D\xBF"
+			  "\x29\x2C\x91\xC1\xB8\xA8\xCF\xE0"
+			  "\x39\xF8\x53\x6D\x31\x22\x2B\xBF"
+			  "\x98\x81\xFC\x34\xEE\x85\x36\xCD"
+			  "\x26\xDB\x6C\x7A\x0C\x77\x8A\x35"
+			  "\x18\x85\x54\xB2\xBC\xDD\x3F\x43"
+			  "\x61\x06\x8A\xDF\x86\x3F\xB4\xAC"
+			  "\x97\xDC\xBD\xFD\x92\x10\xC5\xFF",
+		.rlen	= 88,
+	}, {
+		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
+			  "\xCA\xFE\xBA",
+		.klen	= 19,
+		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.input	= "\x45\x00\x00\x3E\x69\x8F\x00\x00"
+			  "\x80\x11\x4D\xCC\xC0\xA8\x01\x02"
+			  "\xC0\xA8\x01\x01\x0A\x98\x00\x35"
+			  "\x00\x2A\x23\x43\xB2\xD0\x01\x00"
+			  "\x00\x01\x00\x00\x00\x00\x00\x00"
+			  "\x03\x73\x69\x70\x09\x63\x79\x62"
+			  "\x65\x72\x63\x69\x74\x79\x02\x64"
+			  "\x6B\x00\x00\x01\x00\x01\x00\x01",
+		.ilen	= 64,
+		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
+			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.alen	= 16,
+		.result	= "\x4B\xC2\x70\x60\x64\xD2\xF3\xC8"
+			  "\xE5\x26\x8A\xDE\xB8\x7E\x7D\x16"
+			  "\x56\xC7\xD2\x88\xBA\x8D\x58\xAF"
+			  "\xF5\x71\xB6\x37\x84\xA7\xB1\x99"
+			  "\x51\x5C\x0D\xA0\x27\xDE\xE7\x2D"
+			  "\xEF\x25\x88\x1F\x1D\x77\x11\xFF"
+			  "\xDB\xED\xEE\x56\x16\xC5\x5C\x9B"
+			  "\x00\x62\x1F\x68\x4E\x7C\xA0\x97"
+			  "\x10\x72\x7E\x53\x13\x3B\x68\xE4"
+			  "\x30\x99\x91\x79\x09\xEA\xFF\x6A",
+		.rlen	= 80,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x11\x22\x33",
+		.klen	= 35,
+		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
+		.input	= "\x45\x00\x00\x30\x69\xA6\x40\x00"
+			  "\x80\x06\x26\x90\xC0\xA8\x01\x02"
+			  "\x93\x89\x15\x5E\x0A\x9E\x00\x8B"
+			  "\x2D\xC5\x7E\xE0\x00\x00\x00\x00"
+			  "\x70\x02\x40\x00\x20\xBF\x00\x00"
+			  "\x02\x04\x05\xB4\x01\x01\x04\x02"
+			  "\x01\x02\x02\x01",
+		.ilen	= 52,
+		.assoc	= "\x4A\x2C\xBF\xE3\x00\x00\x00\x02"
+			  "\x01\x02\x03\x04\x05\x06\x07\x08",
+		.alen	= 16,
+		.result	= "\xD6\x31\x0D\x2B\x3D\x6F\xBD\x2F"
+			  "\x58\x41\x7E\xFF\x9A\x9E\x09\xB4"
+			  "\x1A\xF7\xF6\x42\x31\xCD\xBF\xAD"
+			  "\x27\x0E\x2C\xF2\xDB\x10\xDF\x55"
+			  "\x8F\x0D\xD7\xAC\x23\xBD\x42\x10"
+			  "\xD0\xB2\xAF\xD8\x37\xAC\x6B\x0B"
+			  "\x11\xD4\x0B\x12\xEC\xB4\xB1\x92"
+			  "\x23\xA6\x10\xB0\x26\xD6\xD9\x26"
+			  "\x5A\x48\x6A\x3E",
+		.rlen	= 68,
+	}, {
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.input	= "\x45\x00\x00\x3C\x99\xC5\x00\x00"
+			  "\x80\x01\xCB\x7A\x40\x67\x93\x18"
+			  "\x01\x01\x01\x01\x08\x00\x07\x5C"
+			  "\x02\x00\x44\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x75\x76\x77\x61\x62\x63\x64\x65"
+			  "\x66\x67\x68\x69\x01\x02\x02\x01",
+		.ilen	= 64,
+		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 16,
+		.result	= "\x6B\x9A\xCA\x57\x43\x91\xFC\x6F"
+			  "\x92\x51\x23\xA4\xC1\x5B\xF0\x10"
+			  "\xF3\x13\xF4\xF8\xA1\x9A\xB4\xDC"
+			  "\x89\xC8\xF8\x42\x62\x95\xB7\xCB"
+			  "\xB8\xF5\x0F\x1B\x2E\x94\xA2\xA7"
+			  "\xBF\xFB\x8A\x92\x13\x63\xD1\x3C"
+			  "\x08\xF5\xE8\xA6\xAA\xF6\x34\xF9"
+			  "\x42\x05\xAF\xB3\xE7\x9A\xFC\xEE"
+			  "\x36\x25\xC1\x10\x12\x1C\xCA\x82"
+			  "\xEA\xE6\x63\x5A\x57\x28\xA9\x9A",
+		.rlen	= 80,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E",
+		.klen	= 19,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.input	= "\x45\x00\x00\x3C\x99\xC3\x00\x00"
+			  "\x80\x01\xCB\x7C\x40\x67\x93\x18"
+			  "\x01\x01\x01\x01\x08\x00\x08\x5C"
+			  "\x02\x00\x43\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x75\x76\x77\x61\x62\x63\x64\x65"
+			  "\x66\x67\x68\x69\x01\x02\x02\x01",
+		.ilen	= 64,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.result	= "\x6A\x6B\x45\x2B\x7C\x67\x52\xF6"
+			  "\x10\x60\x40\x62\x6B\x4F\x97\x8E"
+			  "\x0B\xB2\x22\x97\xCB\x21\xE0\x90"
+			  "\xA2\xE7\xD1\x41\x30\xE4\x4B\x1B"
+			  "\x79\x01\x58\x50\x01\x06\xE1\xE0"
+			  "\x2C\x83\x79\xD3\xDE\x46\x97\x1A"
+			  "\x30\xB8\xE5\xDF\xD7\x12\x56\x75"
+			  "\xD0\x95\xB7\xB8\x91\x42\xF7\xFD"
+			  "\x97\x57\xCA\xC1\x20\xD0\x86\xB9"
+			  "\x66\x9D\xB4\x2B\x96\x22\xAC\x67",
+		.rlen	= 80,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E",
+		.klen	= 19,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.input	= "\x45\x00\x00\x1C\x42\xA2\x00\x00"
+			  "\x80\x01\x44\x1F\x40\x67\x93\xB6"
+			  "\xE0\x00\x00\x02\x0A\x00\xF5\xFF"
+			  "\x01\x02\x02\x01",
+		.ilen	= 28,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.result	= "\x6A\x6B\x45\x0B\xA7\x06\x52\xF6"
+			  "\x10\x60\xCF\x01\x6B\x4F\x97\x20"
+			  "\xEA\xB3\x23\x94\xC9\x21\x1D\x33"
+			  "\xA1\xE5\x90\x40\x05\x37\x45\x70"
+			  "\xB5\xD6\x09\x0A\x23\x73\x33\xF9"
+			  "\x08\xB4\x22\xE4",
+		.rlen	= 44,
+	}, {
+		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
+			  "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\xCA\xFE\xBA",
+		.klen	= 27,
+		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.input	= "\x45\x00\x00\x28\xA4\xAD\x40\x00"
+			  "\x40\x06\x78\x80\x0A\x01\x03\x8F"
+			  "\x0A\x01\x06\x12\x80\x23\x06\xB8"
+			  "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E"
+			  "\x50\x10\x16\xD0\x75\x68\x00\x01",
+		.ilen	= 40,
+		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
+			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.alen	= 16,
+		.result	= "\x05\x22\x15\xD1\x52\x56\x85\x04"
+			  "\xA8\x5C\x5D\x6D\x7E\x6E\xF5\xFA"
+			  "\xEA\x16\x37\x50\xF3\xDF\x84\x3B"
+			  "\x2F\x32\x18\x57\x34\x2A\x8C\x23"
+			  "\x67\xDF\x6D\x35\x7B\x54\x0D\xFB"
+			  "\x34\xA5\x9F\x6C\x48\x30\x1E\x22"
+			  "\xFE\xB1\x22\x17\x17\x8A\xB9\x5B",
+		.rlen	= 56,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xDE\xCA\xF8",
+		.klen	= 19,
+		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
+		.input	= "\x45\x00\x00\x49\x33\xBA\x00\x00"
+			  "\x7F\x11\x91\x06\xC3\xFB\x1D\x10"
+			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
+			  "\x00\x35\xDD\x7B\x80\x03\x02\xD5"
+			  "\x00\x00\x4E\x20\x00\x1E\x8C\x18"
+			  "\xD7\x5B\x81\xDC\x91\xBA\xA0\x47"
+			  "\x6B\x91\xB9\x24\xB2\x80\x38\x9D"
+			  "\x92\xC9\x63\xBA\xC0\x46\xEC\x95"
+			  "\x9B\x62\x66\xC0\x47\x22\xB1\x49"
+			  "\x23\x01\x01\x01",
+		.ilen	= 76,
+		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
+			  "\xCE\xFA\xCE\x74",
+		.alen	= 20,
+		.result	= "\x92\xD0\x53\x79\x33\x38\xD5\xF3"
+			  "\x7D\xE4\x7A\x8E\x86\x03\xC9\x90"
+			  "\x96\x35\xAB\x9C\xFB\xE8\xA3\x76"
+			  "\xE9\xE9\xE2\xD1\x2E\x11\x0E\x00"
+			  "\xFA\xCE\xB5\x9E\x02\xA7\x7B\xEA"
+			  "\x71\x9A\x58\xFB\xA5\x8A\xE1\xB7"
+			  "\x9C\x39\x9D\xE3\xB5\x6E\x69\xE6"
+			  "\x63\xC9\xDB\x05\x69\x51\x12\xAD"
+			  "\x3E\x00\x32\x73\x86\xF2\xEE\xF5"
+			  "\x0F\xE8\x81\x7E\x84\xD3\xC0\x0D"
+			  "\x76\xD6\x55\xC6\xB4\xC2\x34\xC7"
+			  "\x12\x25\x0B\xF9",
+		.rlen	= 92,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x73\x61\x6C",
+		.klen	= 35,
+		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
+		.input	= "\x45\x08\x00\x28\x73\x2C\x00\x00"
+			  "\x40\x06\xE9\xF9\x0A\x01\x06\x12"
+			  "\x0A\x01\x03\x8F\x06\xB8\x80\x23"
+			  "\xDD\x6B\xAF\xBE\xCB\x71\x26\x02"
+			  "\x50\x10\x1F\x64\x6D\x54\x00\x01",
+		.ilen	= 40,
+		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
+			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
+			  "\x69\x76\x65\x63",
+		.alen	= 20,
+		.result	= "\xCC\x74\xB7\xD3\xB0\x38\x50\x42"
+			  "\x2C\x64\x87\x46\x1E\x34\x10\x05"
+			  "\x29\x6B\xBB\x36\xE9\x69\xAD\x92"
+			  "\x82\xA1\x10\x6A\xEB\x0F\xDC\x7D"
+			  "\x08\xBA\xF3\x91\xCA\xAA\x61\xDA"
+			  "\x62\xF4\x14\x61\x5C\x9D\xB5\xA7"
+			  "\xEE\xD7\xB9\x7E\x87\x99\x9B\x7D",
+		.rlen	= 56,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E",
+		.klen	= 19,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.input	= "\x45\x00\x00\x49\x33\x3E\x00\x00"
+			  "\x7F\x11\x91\x82\xC3\xFB\x1D\x10"
+			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
+			  "\x00\x35\xCB\x45\x80\x03\x02\x5B"
+			  "\x00\x00\x01\xE0\x00\x1E\x8C\x18"
+			  "\xD6\x57\x59\xD5\x22\x84\xA0\x35"
+			  "\x2C\x71\x47\x5C\x88\x80\x39\x1C"
+			  "\x76\x4D\x6E\x5E\xE0\x49\x6B\x32"
+			  "\x5A\xE2\x70\xC0\x38\x99\x49\x39"
+			  "\x15\x01\x01\x01",
+		.ilen	= 76,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.result	= "\x6A\x6B\x45\x5E\xD6\x9A\x52\xF6"
+			  "\xEF\x70\x1A\x9C\xE8\xD3\x19\x86"
+			  "\xC8\x02\xF0\xB0\x03\x09\xD9\x02"
+			  "\xA0\xD2\x59\x04\xD1\x85\x2A\x24"
+			  "\x1C\x67\x3E\xD8\x68\x72\x06\x94"
+			  "\x97\xBA\x4F\x76\x8D\xB0\x44\x5B"
+			  "\x69\xBF\xD5\xE2\x3D\xF1\x0B\x0C"
+			  "\xC0\xBF\xB1\x8F\x70\x09\x9E\xCE"
+			  "\xA5\xF2\x55\x58\x84\xFA\xF9\xB5"
+			  "\x23\xF4\x84\x40\x74\x14\x8A\x6B"
+			  "\xDB\xD7\x67\xED\xA4\x93\xF3\x47"
+			  "\xCC\xF7\x46\x6F",
+		.rlen	= 92,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x73\x61\x6C",
+		.klen	= 35,
+		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
+		.input	= "\x63\x69\x73\x63\x6F\x01\x72\x75"
+			  "\x6C\x65\x73\x01\x74\x68\x65\x01"
+			  "\x6E\x65\x74\x77\x65\x01\x64\x65"
+			  "\x66\x69\x6E\x65\x01\x74\x68\x65"
+			  "\x74\x65\x63\x68\x6E\x6F\x6C\x6F"
+			  "\x67\x69\x65\x73\x01\x74\x68\x61"
+			  "\x74\x77\x69\x6C\x6C\x01\x64\x65"
+			  "\x66\x69\x6E\x65\x74\x6F\x6D\x6F"
+			  "\x72\x72\x6F\x77\x01\x02\x02\x01",
+		.ilen	= 72,
+		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
+			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
+			  "\x69\x76\x65\x63",
+		.alen	= 20,
+		.result	= "\xEA\x15\xC4\x98\xAC\x15\x22\x37"
+			  "\x00\x07\x1D\xBE\x60\x5D\x73\x16"
+			  "\x4D\x0F\xCC\xCE\x8A\xD0\x49\xD4"
+			  "\x39\xA3\xD1\xB1\x21\x0A\x92\x1A"
+			  "\x2C\xCF\x8F\x9D\xC9\x91\x0D\xB4"
+			  "\x15\xFC\xBC\xA5\xC5\xBF\x54\xE5"
+			  "\x1C\xC7\x32\x41\x07\x7B\x2C\xB6"
+			  "\x5C\x23\x7C\x93\xEA\xEF\x23\x1C"
+			  "\x73\xF4\xE7\x12\x84\x4C\x37\x0A"
+			  "\x4A\x8F\x06\x37\x48\xF9\xF9\x05"
+			  "\x55\x13\x40\xC3\xD5\x55\x3A\x3D",
+		.rlen	= 88,
+	}, {
+		.key	= "\x7D\x77\x3D\x00\xC1\x44\xC5\x25"
+			  "\xAC\x61\x9D\x18\xC8\x4A\x3F\x47"
+			  "\xD9\x66\x42",
+		.klen	= 19,
+		.iv	= "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
+		.input	= "\x01\x02\x02\x01",
+		.ilen	= 4,
+		.assoc	= "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF"
+			  "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
+		.alen	= 16,
+		.result	= "\x4C\x72\x63\x30\x2F\xE6\x56\xDD"
+			  "\xD0\xD8\x60\x9D\x8B\xEF\x85\x90"
+			  "\xF7\x61\x24\x62",
+		.rlen	= 20,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xDE\xCA\xF8",
+		.klen	= 19,
+		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
+		.input	= "\x74\x6F\x01\x62\x65\x01\x6F\x72"
+			  "\x01\x6E\x6F\x74\x01\x74\x6F\x01"
+			  "\x62\x65\x00\x01",
+		.ilen	= 20,
+		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
+			  "\xCE\xFA\xCE\x74",
+		.alen	= 20,
+		.result	= "\xA3\xBF\x52\x52\x65\x83\xBA\x81"
+			  "\x03\x9B\x84\xFC\x44\x8C\xBB\x81"
+			  "\x36\xE1\x78\xBB\xA5\x49\x3A\xD0"
+			  "\xF0\x6B\x21\xAF\x98\xC0\x34\xDC"
+			  "\x17\x17\x65\xAD",
+		.rlen	= 36,
+	}, {
+		.key	= "\x6C\x65\x67\x61\x6C\x69\x7A\x65"
+			  "\x6D\x61\x72\x69\x6A\x75\x61\x6E"
+			  "\x61\x61\x6E\x64\x64\x6F\x69\x74"
+			  "\x62\x65\x66\x6F\x72\x65\x69\x61"
+			  "\x74\x75\x72",
+		.klen	= 35,
+		.iv	= "\x33\x30\x21\x69\x67\x65\x74\x6D",
+		.input	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
+			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
+			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01",
+		.ilen	= 52,
+		.assoc	= "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\x33\x30\x21\x69"
+			  "\x67\x65\x74\x6D",
+		.alen	= 20,
+		.result	= "\x96\xFD\x86\xF8\xD1\x98\xFF\x10"
+			  "\xAB\x8C\xDA\x8A\x5A\x08\x38\x1A"
+			  "\x48\x59\x80\x18\x1A\x18\x1A\x04"
+			  "\xC9\x0D\xE3\xE7\x0E\xA4\x0B\x75"
+			  "\x92\x9C\x52\x5C\x0B\xFB\xF8\xAF"
+			  "\x16\xC3\x35\xA8\xE7\xCE\x84\x04"
+			  "\xEB\x40\x6B\x7A\x8E\x75\xBB\x42"
+			  "\xE0\x63\x4B\x21\x44\xA2\x2B\x2B"
+			  "\x39\xDB\xC8\xDC",
+		.rlen	= 68,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E",
+		.klen	= 19,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.input	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
+			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
+			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01",
+		.ilen	= 52,
+		.assoc	= "\x3F\x7E\xF6\x42\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.result	= "\x6A\x6B\x45\x27\x3F\x9E\x52\xF6"
+			  "\x10\x60\x54\x25\xEB\x80\x04\x93"
+			  "\xCA\x1B\x23\x97\xCB\x21\x2E\x01"
+			  "\xA2\xE7\x95\x41\x30\xE4\x4B\x1B"
+			  "\x79\x01\x58\x50\x01\x06\xE1\xE0"
+			  "\x2C\x83\x79\xD3\xDE\x46\x97\x1A"
+			  "\x44\xCC\x90\xBF\x00\x94\x94\x92"
+			  "\x20\x17\x0C\x1B\x55\xDE\x7E\x68"
+			  "\xF4\x95\x5D\x4F",
+		.rlen	= 68,
+	}, {
+		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
+			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
+			  "\x22\x43\x3C",
+		.klen	= 19,
+		.iv	= "\x48\x55\xEC\x7D\x3A\x23\x4B\xFD",
+		.input	= "\x08\x00\xC6\xCD\x02\x00\x07\x00"
+			  "\x61\x62\x63\x64\x65\x66\x67\x68"
+			  "\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70"
+			  "\x71\x72\x73\x74\x01\x02\x02\x01",
+		.ilen	= 32,
+		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
+			  "\x00\x00\x00\x07\x48\x55\xEC\x7D"
+			  "\x3A\x23\x4B\xFD",
+		.alen	= 20,
+		.result	= "\x67\xE9\x28\xB3\x1C\xA4\x6D\x02"
+			  "\xF0\xB5\x37\xB6\x6B\x2F\xF5\x4F"
+			  "\xF8\xA3\x4C\x53\xB8\x12\x09\xBF"
+			  "\x58\x7D\xCF\x29\xA3\x41\x68\x6B"
+			  "\xCE\xE8\x79\x85\x3C\xB0\x3A\x8F"
+			  "\x16\xB0\xA1\x26\xC9\xBC\xBC\xA6",
+		.rlen	= 48,
+	}
+};
+
+static const struct aead_testvec aes_ccm_rfc4309_dec_tv_template[]	= {
+	{ /* Generated using Crypto++ */
+		.key	= zeroed_string,
+		.klen	= 19,
+		.iv	= zeroed_string,
+		.result	= zeroed_string,
+		.rlen	= 16,
+		.assoc	= zeroed_string,
+		.alen	= 16,
+		.input	= "\x2E\x9A\xCA\x6B\xDA\x54\xFC\x6F"
+			  "\x12\x50\xE8\xDE\x81\x3C\x63\x08"
+			  "\x1A\x22\xBA\x75\xEE\xD4\xD5\xB5"
+			  "\x27\x50\x01\xAC\x03\x33\x39\xFB",
+		.ilen	= 32,
+	},{
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.result	= zeroed_string,
+		.rlen	= 16,
+		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen	= 16,
+		.input	= "\xCF\xB9\x99\x17\xC8\x86\x0E\x7F"
+			  "\x7E\x76\xF8\xE6\xF8\xCC\x1F\x17"
+			  "\x6A\xE0\x53\x9F\x4B\x73\x7E\xDA"
+			  "\x08\x09\x4E\xC4\x1E\xAD\xC6\xB0",
+		.ilen	= 32,
+
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= zeroed_string,
+		.result	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.rlen	= 16,
+		.assoc	= zeroed_string,
+		.alen	= 16,
+		.input	= "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6"
+			  "\x61\xF4\xF5\x41\x03\x4A\xE3\x86"
+			  "\xA1\xE2\xC2\x42\x2B\x81\x70\x40"
+			  "\xFD\x7F\x76\xD1\x03\x07\xBB\x0C",
+		.ilen	= 32,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= zeroed_string,
+		.result	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.rlen	= 16,
+		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 16,
+		.input	= "\x33\xDE\x73\xBC\xA6\xCE\x4E\xA6"
+			  "\x61\xF4\xF5\x41\x03\x4A\xE3\x86"
+			  "\x5B\xC0\x73\xE0\x2B\x73\x68\xC9"
+			  "\x2D\x8C\x58\xC2\x90\x3D\xB0\x3E",
+		.ilen	= 32,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.result	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.rlen	= 16,
+		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen	= 16,
+		.input	= "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E"
+			  "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16"
+			  "\x43\x8E\x76\x57\x3B\xB4\x05\xE8"
+			  "\xA9\x9B\xBF\x25\xE0\x4F\xC0\xED",
+		.ilen	= 32,
+	}, {
+		.key	= "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
+			  "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.result	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x01\x01\x01\x01\x01\x01\x01\x01",
+		.rlen	= 64,
+		.assoc	= "\x01\x01\x01\x01\x01\x01\x01\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.alen	= 16,
+		.input	= "\xCE\xB8\x98\x16\xC9\x87\x0F\x7E"
+			  "\x7F\x77\xF9\xE7\xF9\xCD\x1E\x16"
+			  "\x9C\xA4\x97\x83\x3F\x01\xA5\xF4"
+			  "\x43\x09\xE7\xB8\xE9\xD1\xD7\x02"
+			  "\x9B\xAB\x39\x18\xEB\x94\x34\x36"
+			  "\xE6\xC5\xC8\x9B\x00\x81\x9E\x49"
+			  "\x1D\x78\xE1\x48\xE3\xE9\xEA\x8E"
+			  "\x3A\x2B\x67\x5D\x35\x6A\x0F\xDB"
+			  "\x02\x73\xDD\xE7\x30\x4A\x30\x54"
+			  "\x1A\x9D\x09\xCA\xC8\x1C\x32\x5F",
+		.ilen	= 80,
+	}, {
+		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= "\x00\x00\x45\x67\x89\xab\xcd\xef",
+		.result	= "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff"
+			  "\xff\xff\xff\xff\xff\xff\xff\xff",
+		.rlen	= 192,
+		.assoc	= "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
+			  "\xaa\xaa\xaa\xaa\x00\x00\x45\x67"
+			  "\x89\xab\xcd\xef",
+		.alen	= 20,
+		.input	= "\x64\x17\xDC\x24\x9D\x92\xBA\x5E"
+			  "\x7C\x64\x6D\x33\x46\x77\xAC\xB1"
+			  "\x5C\x9E\xE2\xC7\x27\x11\x3E\x95"
+			  "\x7D\xBE\x28\xC8\xC1\xCA\x5E\x8C"
+			  "\xB4\xE2\xDE\x9F\x53\x59\x26\xDB"
+			  "\x0C\xD4\xE4\x07\x9A\xE6\x3E\x01"
+			  "\x58\x0D\x3E\x3D\xD5\x21\xEB\x04"
+			  "\x06\x9D\x5F\xB9\x02\x49\x1A\x2B"
+			  "\xBA\xF0\x4E\x3B\x85\x50\x5B\x09"
+			  "\xFE\xEC\xFC\x54\xEC\x0C\xE2\x79"
+			  "\x8A\x2F\x5F\xD7\x05\x5D\xF1\x6D"
+			  "\x22\xEB\xD1\x09\x80\x3F\x5A\x70"
+			  "\xB2\xB9\xD3\x63\x99\xC2\x4D\x1B"
+			  "\x36\x12\x00\x89\xAA\x5D\x55\xDA"
+			  "\x1D\x5B\xD8\x3C\x5F\x09\xD2\xE6"
+			  "\x39\x41\x5C\xF0\xBE\x26\x4E\x5F"
+			  "\x2B\x50\x44\x52\xC2\x10\x7D\x38"
+			  "\x82\x64\x83\x0C\xAE\x49\xD0\xE5"
+			  "\x4F\xE5\x66\x4C\x58\x7A\xEE\x43"
+			  "\x3B\x51\xFE\xBA\x24\x8A\xFE\xDC"
+			  "\x19\x6D\x60\x66\x61\xF9\x9A\x3F"
+			  "\x75\xFC\x38\x53\x5B\xB5\xCD\x52"
+			  "\x4F\xE5\xE4\xC9\xFE\x10\xCB\x98"
+			  "\xF0\x06\x5B\x07\xAB\xBB\xF4\x0E"
+			  "\x2D\xC2\xDD\x5D\xDD\x22\x9A\xCC"
+			  "\x39\xAB\x63\xA5\x3D\x9C\x51\x8A",
+		.ilen	= 208,
+	}, { /* From draft-mcgrew-gcm-test-01 */
+		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
+			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
+			  "\x2E\x44\x3B",
+		.klen	= 19,
+		.iv	= "\x49\x56\xED\x7E\x3B\x24\x4C\xFE",
+		.result	= "\x45\x00\x00\x48\x69\x9A\x00\x00"
+			  "\x80\x11\x4D\xB7\xC0\xA8\x01\x02"
+			  "\xC0\xA8\x01\x01\x0A\x9B\xF1\x56"
+			  "\x38\xD3\x01\x00\x00\x01\x00\x00"
+			  "\x00\x00\x00\x00\x04\x5F\x73\x69"
+			  "\x70\x04\x5F\x75\x64\x70\x03\x73"
+			  "\x69\x70\x09\x63\x79\x62\x65\x72"
+			  "\x63\x69\x74\x79\x02\x64\x6B\x00"
+			  "\x00\x21\x00\x01\x01\x02\x02\x01",
+		.rlen	= 72,
+		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
+			  "\x00\x00\x00\x00\x49\x56\xED\x7E"
+			  "\x3B\x24\x4C\xFE",
+		.alen	= 20,
+		.input	= "\x89\xBA\x3E\xEF\xE6\xD6\xCF\xDB"
+			  "\x83\x60\xF5\xBA\x3A\x56\x79\xE6"
+			  "\x7E\x0C\x53\xCF\x9E\x87\xE0\x4E"
+			  "\x1A\x26\x01\x24\xC7\x2E\x3D\xBF"
+			  "\x29\x2C\x91\xC1\xB8\xA8\xCF\xE0"
+			  "\x39\xF8\x53\x6D\x31\x22\x2B\xBF"
+			  "\x98\x81\xFC\x34\xEE\x85\x36\xCD"
+			  "\x26\xDB\x6C\x7A\x0C\x77\x8A\x35"
+			  "\x18\x85\x54\xB2\xBC\xDD\x3F\x43"
+			  "\x61\x06\x8A\xDF\x86\x3F\xB4\xAC"
+			  "\x97\xDC\xBD\xFD\x92\x10\xC5\xFF",
+		.ilen	= 88,
+	}, {
+		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
+			  "\xCA\xFE\xBA",
+		.klen	= 19,
+		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.result	= "\x45\x00\x00\x3E\x69\x8F\x00\x00"
+			  "\x80\x11\x4D\xCC\xC0\xA8\x01\x02"
+			  "\xC0\xA8\x01\x01\x0A\x98\x00\x35"
+			  "\x00\x2A\x23\x43\xB2\xD0\x01\x00"
+			  "\x00\x01\x00\x00\x00\x00\x00\x00"
+			  "\x03\x73\x69\x70\x09\x63\x79\x62"
+			  "\x65\x72\x63\x69\x74\x79\x02\x64"
+			  "\x6B\x00\x00\x01\x00\x01\x00\x01",
+		.rlen	= 64,
+		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
+			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.alen	= 16,
+		.input	= "\x4B\xC2\x70\x60\x64\xD2\xF3\xC8"
+			  "\xE5\x26\x8A\xDE\xB8\x7E\x7D\x16"
+			  "\x56\xC7\xD2\x88\xBA\x8D\x58\xAF"
+			  "\xF5\x71\xB6\x37\x84\xA7\xB1\x99"
+			  "\x51\x5C\x0D\xA0\x27\xDE\xE7\x2D"
+			  "\xEF\x25\x88\x1F\x1D\x77\x11\xFF"
+			  "\xDB\xED\xEE\x56\x16\xC5\x5C\x9B"
+			  "\x00\x62\x1F\x68\x4E\x7C\xA0\x97"
+			  "\x10\x72\x7E\x53\x13\x3B\x68\xE4"
+			  "\x30\x99\x91\x79\x09\xEA\xFF\x6A",
+		.ilen	= 80,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x11\x22\x33",
+		.klen	= 35,
+		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
+		.result	= "\x45\x00\x00\x30\x69\xA6\x40\x00"
+			  "\x80\x06\x26\x90\xC0\xA8\x01\x02"
+			  "\x93\x89\x15\x5E\x0A\x9E\x00\x8B"
+			  "\x2D\xC5\x7E\xE0\x00\x00\x00\x00"
+			  "\x70\x02\x40\x00\x20\xBF\x00\x00"
+			  "\x02\x04\x05\xB4\x01\x01\x04\x02"
+			  "\x01\x02\x02\x01",
+		.rlen	= 52,
+		.assoc	= "\x4A\x2C\xBF\xE3\x00\x00\x00\x02"
+			  "\x01\x02\x03\x04\x05\x06\x07\x08",
+		.alen	= 16,
+		.input	= "\xD6\x31\x0D\x2B\x3D\x6F\xBD\x2F"
+			  "\x58\x41\x7E\xFF\x9A\x9E\x09\xB4"
+			  "\x1A\xF7\xF6\x42\x31\xCD\xBF\xAD"
+			  "\x27\x0E\x2C\xF2\xDB\x10\xDF\x55"
+			  "\x8F\x0D\xD7\xAC\x23\xBD\x42\x10"
+			  "\xD0\xB2\xAF\xD8\x37\xAC\x6B\x0B"
+			  "\x11\xD4\x0B\x12\xEC\xB4\xB1\x92"
+			  "\x23\xA6\x10\xB0\x26\xD6\xD9\x26"
+			  "\x5A\x48\x6A\x3E",
+		.ilen	= 68,
+	}, {
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00",
+		.klen	= 19,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.result	= "\x45\x00\x00\x3C\x99\xC5\x00\x00"
+			  "\x80\x01\xCB\x7A\x40\x67\x93\x18"
+			  "\x01\x01\x01\x01\x08\x00\x07\x5C"
+			  "\x02\x00\x44\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x75\x76\x77\x61\x62\x63\x64\x65"
+			  "\x66\x67\x68\x69\x01\x02\x02\x01",
+		.rlen	= 64,
+		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x01"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 16,
+		.input	= "\x6B\x9A\xCA\x57\x43\x91\xFC\x6F"
+			  "\x92\x51\x23\xA4\xC1\x5B\xF0\x10"
+			  "\xF3\x13\xF4\xF8\xA1\x9A\xB4\xDC"
+			  "\x89\xC8\xF8\x42\x62\x95\xB7\xCB"
+			  "\xB8\xF5\x0F\x1B\x2E\x94\xA2\xA7"
+			  "\xBF\xFB\x8A\x92\x13\x63\xD1\x3C"
+			  "\x08\xF5\xE8\xA6\xAA\xF6\x34\xF9"
+			  "\x42\x05\xAF\xB3\xE7\x9A\xFC\xEE"
+			  "\x36\x25\xC1\x10\x12\x1C\xCA\x82"
+			  "\xEA\xE6\x63\x5A\x57\x28\xA9\x9A",
+		.ilen	= 80,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E",
+		.klen	= 19,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.result	= "\x45\x00\x00\x3C\x99\xC3\x00\x00"
+			  "\x80\x01\xCB\x7C\x40\x67\x93\x18"
+			  "\x01\x01\x01\x01\x08\x00\x08\x5C"
+			  "\x02\x00\x43\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x75\x76\x77\x61\x62\x63\x64\x65"
+			  "\x66\x67\x68\x69\x01\x02\x02\x01",
+		.rlen	= 64,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.input	= "\x6A\x6B\x45\x2B\x7C\x67\x52\xF6"
+			  "\x10\x60\x40\x62\x6B\x4F\x97\x8E"
+			  "\x0B\xB2\x22\x97\xCB\x21\xE0\x90"
+			  "\xA2\xE7\xD1\x41\x30\xE4\x4B\x1B"
+			  "\x79\x01\x58\x50\x01\x06\xE1\xE0"
+			  "\x2C\x83\x79\xD3\xDE\x46\x97\x1A"
+			  "\x30\xB8\xE5\xDF\xD7\x12\x56\x75"
+			  "\xD0\x95\xB7\xB8\x91\x42\xF7\xFD"
+			  "\x97\x57\xCA\xC1\x20\xD0\x86\xB9"
+			  "\x66\x9D\xB4\x2B\x96\x22\xAC\x67",
+		.ilen	= 80,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E",
+		.klen	= 19,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.result	= "\x45\x00\x00\x1C\x42\xA2\x00\x00"
+			  "\x80\x01\x44\x1F\x40\x67\x93\xB6"
+			  "\xE0\x00\x00\x02\x0A\x00\xF5\xFF"
+			  "\x01\x02\x02\x01",
+		.rlen	= 28,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.input	= "\x6A\x6B\x45\x0B\xA7\x06\x52\xF6"
+			  "\x10\x60\xCF\x01\x6B\x4F\x97\x20"
+			  "\xEA\xB3\x23\x94\xC9\x21\x1D\x33"
+			  "\xA1\xE5\x90\x40\x05\x37\x45\x70"
+			  "\xB5\xD6\x09\x0A\x23\x73\x33\xF9"
+			  "\x08\xB4\x22\xE4",
+		.ilen	= 44,
+	}, {
+		.key	= "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\x6D\x6A\x8F\x94\x67\x30\x83\x08"
+			  "\xFE\xFF\xE9\x92\x86\x65\x73\x1C"
+			  "\xCA\xFE\xBA",
+		.klen	= 27,
+		.iv	= "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.result	= "\x45\x00\x00\x28\xA4\xAD\x40\x00"
+			  "\x40\x06\x78\x80\x0A\x01\x03\x8F"
+			  "\x0A\x01\x06\x12\x80\x23\x06\xB8"
+			  "\xCB\x71\x26\x02\xDD\x6B\xB0\x3E"
+			  "\x50\x10\x16\xD0\x75\x68\x00\x01",
+		.rlen	= 40,
+		.assoc	= "\x00\x00\xA5\xF8\x00\x00\x00\x0A"
+			  "\xFA\xCE\xDB\xAD\xDE\xCA\xF8\x88",
+		.alen	= 16,
+		.input	= "\x05\x22\x15\xD1\x52\x56\x85\x04"
+			  "\xA8\x5C\x5D\x6D\x7E\x6E\xF5\xFA"
+			  "\xEA\x16\x37\x50\xF3\xDF\x84\x3B"
+			  "\x2F\x32\x18\x57\x34\x2A\x8C\x23"
+			  "\x67\xDF\x6D\x35\x7B\x54\x0D\xFB"
+			  "\x34\xA5\x9F\x6C\x48\x30\x1E\x22"
+			  "\xFE\xB1\x22\x17\x17\x8A\xB9\x5B",
+		.ilen	= 56,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xDE\xCA\xF8",
+		.klen	= 19,
+		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
+		.result	= "\x45\x00\x00\x49\x33\xBA\x00\x00"
+			  "\x7F\x11\x91\x06\xC3\xFB\x1D\x10"
+			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
+			  "\x00\x35\xDD\x7B\x80\x03\x02\xD5"
+			  "\x00\x00\x4E\x20\x00\x1E\x8C\x18"
+			  "\xD7\x5B\x81\xDC\x91\xBA\xA0\x47"
+			  "\x6B\x91\xB9\x24\xB2\x80\x38\x9D"
+			  "\x92\xC9\x63\xBA\xC0\x46\xEC\x95"
+			  "\x9B\x62\x66\xC0\x47\x22\xB1\x49"
+			  "\x23\x01\x01\x01",
+		.rlen	= 76,
+		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
+			  "\xCE\xFA\xCE\x74",
+		.alen	= 20,
+		.input	= "\x92\xD0\x53\x79\x33\x38\xD5\xF3"
+			  "\x7D\xE4\x7A\x8E\x86\x03\xC9\x90"
+			  "\x96\x35\xAB\x9C\xFB\xE8\xA3\x76"
+			  "\xE9\xE9\xE2\xD1\x2E\x11\x0E\x00"
+			  "\xFA\xCE\xB5\x9E\x02\xA7\x7B\xEA"
+			  "\x71\x9A\x58\xFB\xA5\x8A\xE1\xB7"
+			  "\x9C\x39\x9D\xE3\xB5\x6E\x69\xE6"
+			  "\x63\xC9\xDB\x05\x69\x51\x12\xAD"
+			  "\x3E\x00\x32\x73\x86\xF2\xEE\xF5"
+			  "\x0F\xE8\x81\x7E\x84\xD3\xC0\x0D"
+			  "\x76\xD6\x55\xC6\xB4\xC2\x34\xC7"
+			  "\x12\x25\x0B\xF9",
+		.ilen	= 92,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x73\x61\x6C",
+		.klen	= 35,
+		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
+		.result	= "\x45\x08\x00\x28\x73\x2C\x00\x00"
+			  "\x40\x06\xE9\xF9\x0A\x01\x06\x12"
+			  "\x0A\x01\x03\x8F\x06\xB8\x80\x23"
+			  "\xDD\x6B\xAF\xBE\xCB\x71\x26\x02"
+			  "\x50\x10\x1F\x64\x6D\x54\x00\x01",
+		.rlen	= 40,
+		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
+			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
+			  "\x69\x76\x65\x63",
+		.alen	= 20,
+		.input	= "\xCC\x74\xB7\xD3\xB0\x38\x50\x42"
+			  "\x2C\x64\x87\x46\x1E\x34\x10\x05"
+			  "\x29\x6B\xBB\x36\xE9\x69\xAD\x92"
+			  "\x82\xA1\x10\x6A\xEB\x0F\xDC\x7D"
+			  "\x08\xBA\xF3\x91\xCA\xAA\x61\xDA"
+			  "\x62\xF4\x14\x61\x5C\x9D\xB5\xA7"
+			  "\xEE\xD7\xB9\x7E\x87\x99\x9B\x7D",
+		.ilen	= 56,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E",
+		.klen	= 19,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.result	= "\x45\x00\x00\x49\x33\x3E\x00\x00"
+			  "\x7F\x11\x91\x82\xC3\xFB\x1D\x10"
+			  "\xC2\xB1\xD3\x26\xC0\x28\x31\xCE"
+			  "\x00\x35\xCB\x45\x80\x03\x02\x5B"
+			  "\x00\x00\x01\xE0\x00\x1E\x8C\x18"
+			  "\xD6\x57\x59\xD5\x22\x84\xA0\x35"
+			  "\x2C\x71\x47\x5C\x88\x80\x39\x1C"
+			  "\x76\x4D\x6E\x5E\xE0\x49\x6B\x32"
+			  "\x5A\xE2\x70\xC0\x38\x99\x49\x39"
+			  "\x15\x01\x01\x01",
+		.rlen	= 76,
+		.assoc	= "\x42\xF6\x7E\x3F\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.input	= "\x6A\x6B\x45\x5E\xD6\x9A\x52\xF6"
+			  "\xEF\x70\x1A\x9C\xE8\xD3\x19\x86"
+			  "\xC8\x02\xF0\xB0\x03\x09\xD9\x02"
+			  "\xA0\xD2\x59\x04\xD1\x85\x2A\x24"
+			  "\x1C\x67\x3E\xD8\x68\x72\x06\x94"
+			  "\x97\xBA\x4F\x76\x8D\xB0\x44\x5B"
+			  "\x69\xBF\xD5\xE2\x3D\xF1\x0B\x0C"
+			  "\xC0\xBF\xB1\x8F\x70\x09\x9E\xCE"
+			  "\xA5\xF2\x55\x58\x84\xFA\xF9\xB5"
+			  "\x23\xF4\x84\x40\x74\x14\x8A\x6B"
+			  "\xDB\xD7\x67\xED\xA4\x93\xF3\x47"
+			  "\xCC\xF7\x46\x6F",
+		.ilen	= 92,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\x73\x61\x6C",
+		.klen	= 35,
+		.iv	= "\x61\x6E\x64\x01\x69\x76\x65\x63",
+		.result	= "\x63\x69\x73\x63\x6F\x01\x72\x75"
+			  "\x6C\x65\x73\x01\x74\x68\x65\x01"
+			  "\x6E\x65\x74\x77\x65\x01\x64\x65"
+			  "\x66\x69\x6E\x65\x01\x74\x68\x65"
+			  "\x74\x65\x63\x68\x6E\x6F\x6C\x6F"
+			  "\x67\x69\x65\x73\x01\x74\x68\x61"
+			  "\x74\x77\x69\x6C\x6C\x01\x64\x65"
+			  "\x66\x69\x6E\x65\x74\x6F\x6D\x6F"
+			  "\x72\x72\x6F\x77\x01\x02\x02\x01",
+		.rlen	= 72,
+		.assoc	= "\x17\x40\x5E\x67\x15\x6F\x31\x26"
+			  "\xDD\x0D\xB9\x9B\x61\x6E\x64\x01"
+			  "\x69\x76\x65\x63",
+		.alen	= 20,
+		.input	= "\xEA\x15\xC4\x98\xAC\x15\x22\x37"
+			  "\x00\x07\x1D\xBE\x60\x5D\x73\x16"
+			  "\x4D\x0F\xCC\xCE\x8A\xD0\x49\xD4"
+			  "\x39\xA3\xD1\xB1\x21\x0A\x92\x1A"
+			  "\x2C\xCF\x8F\x9D\xC9\x91\x0D\xB4"
+			  "\x15\xFC\xBC\xA5\xC5\xBF\x54\xE5"
+			  "\x1C\xC7\x32\x41\x07\x7B\x2C\xB6"
+			  "\x5C\x23\x7C\x93\xEA\xEF\x23\x1C"
+			  "\x73\xF4\xE7\x12\x84\x4C\x37\x0A"
+			  "\x4A\x8F\x06\x37\x48\xF9\xF9\x05"
+			  "\x55\x13\x40\xC3\xD5\x55\x3A\x3D",
+		.ilen	= 88,
+	}, {
+		.key	= "\x7D\x77\x3D\x00\xC1\x44\xC5\x25"
+			  "\xAC\x61\x9D\x18\xC8\x4A\x3F\x47"
+			  "\xD9\x66\x42",
+		.klen	= 19,
+		.iv	= "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
+		.result	= "\x01\x02\x02\x01",
+		.rlen	= 4,
+		.assoc	= "\x33\x54\x67\xAE\xFF\xFF\xFF\xFF"
+			  "\x43\x45\x7E\x91\x82\x44\x3B\xC6",
+		.alen	= 16,
+		.input	= "\x4C\x72\x63\x30\x2F\xE6\x56\xDD"
+			  "\xD0\xD8\x60\x9D\x8B\xEF\x85\x90"
+			  "\xF7\x61\x24\x62",
+		.ilen	= 20,
+	}, {
+		.key	= "\xAB\xBC\xCD\xDE\xF0\x01\x12\x23"
+			  "\x34\x45\x56\x67\x78\x89\x9A\xAB"
+			  "\xDE\xCA\xF8",
+		.klen	= 19,
+		.iv	= "\xCA\xFE\xDE\xBA\xCE\xFA\xCE\x74",
+		.result	= "\x74\x6F\x01\x62\x65\x01\x6F\x72"
+			  "\x01\x6E\x6F\x74\x01\x74\x6F\x01"
+			  "\x62\x65\x00\x01",
+		.rlen	= 20,
+		.assoc	= "\x00\x00\x01\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x01\xCA\xFE\xDE\xBA"
+			  "\xCE\xFA\xCE\x74",
+		.alen	= 20,
+		.input	= "\xA3\xBF\x52\x52\x65\x83\xBA\x81"
+			  "\x03\x9B\x84\xFC\x44\x8C\xBB\x81"
+			  "\x36\xE1\x78\xBB\xA5\x49\x3A\xD0"
+			  "\xF0\x6B\x21\xAF\x98\xC0\x34\xDC"
+			  "\x17\x17\x65\xAD",
+		.ilen	= 36,
+	}, {
+		.key	= "\x6C\x65\x67\x61\x6C\x69\x7A\x65"
+			  "\x6D\x61\x72\x69\x6A\x75\x61\x6E"
+			  "\x61\x61\x6E\x64\x64\x6F\x69\x74"
+			  "\x62\x65\x66\x6F\x72\x65\x69\x61"
+			  "\x74\x75\x72",
+		.klen	= 35,
+		.iv	= "\x33\x30\x21\x69\x67\x65\x74\x6D",
+		.result	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
+			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
+			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01",
+		.rlen	= 52,
+		.assoc	= "\x79\x6B\x69\x63\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\x33\x30\x21\x69"
+			  "\x67\x65\x74\x6D",
+		.alen	= 20,
+		.input	= "\x96\xFD\x86\xF8\xD1\x98\xFF\x10"
+			  "\xAB\x8C\xDA\x8A\x5A\x08\x38\x1A"
+			  "\x48\x59\x80\x18\x1A\x18\x1A\x04"
+			  "\xC9\x0D\xE3\xE7\x0E\xA4\x0B\x75"
+			  "\x92\x9C\x52\x5C\x0B\xFB\xF8\xAF"
+			  "\x16\xC3\x35\xA8\xE7\xCE\x84\x04"
+			  "\xEB\x40\x6B\x7A\x8E\x75\xBB\x42"
+			  "\xE0\x63\x4B\x21\x44\xA2\x2B\x2B"
+			  "\x39\xDB\xC8\xDC",
+		.ilen	= 68,
+	}, {
+		.key	= "\x3D\xE0\x98\x74\xB3\x88\xE6\x49"
+			  "\x19\x88\xD0\xC3\x60\x7E\xAE\x1F"
+			  "\x57\x69\x0E",
+		.klen	= 19,
+		.iv	= "\x4E\x28\x00\x00\xA2\xFC\xA1\xA3",
+		.result	= "\x45\x00\x00\x30\xDA\x3A\x00\x00"
+			  "\x80\x01\xDF\x3B\xC0\xA8\x00\x05"
+			  "\xC0\xA8\x00\x01\x08\x00\xC6\xCD"
+			  "\x02\x00\x07\x00\x61\x62\x63\x64"
+			  "\x65\x66\x67\x68\x69\x6A\x6B\x6C"
+			  "\x6D\x6E\x6F\x70\x71\x72\x73\x74"
+			  "\x01\x02\x02\x01",
+		.rlen	= 52,
+		.assoc	= "\x3F\x7E\xF6\x42\x10\x10\x10\x10"
+			  "\x10\x10\x10\x10\x4E\x28\x00\x00"
+			  "\xA2\xFC\xA1\xA3",
+		.alen	= 20,
+		.input	= "\x6A\x6B\x45\x27\x3F\x9E\x52\xF6"
+			  "\x10\x60\x54\x25\xEB\x80\x04\x93"
+			  "\xCA\x1B\x23\x97\xCB\x21\x2E\x01"
+			  "\xA2\xE7\x95\x41\x30\xE4\x4B\x1B"
+			  "\x79\x01\x58\x50\x01\x06\xE1\xE0"
+			  "\x2C\x83\x79\xD3\xDE\x46\x97\x1A"
+			  "\x44\xCC\x90\xBF\x00\x94\x94\x92"
+			  "\x20\x17\x0C\x1B\x55\xDE\x7E\x68"
+			  "\xF4\x95\x5D\x4F",
+		.ilen	= 68,
+	}, {
+		.key	= "\x4C\x80\xCD\xEF\xBB\x5D\x10\xDA"
+			  "\x90\x6A\xC7\x3C\x36\x13\xA6\x34"
+			  "\x22\x43\x3C",
+		.klen	= 19,
+		.iv	= "\x48\x55\xEC\x7D\x3A\x23\x4B\xFD",
+		.result	= "\x08\x00\xC6\xCD\x02\x00\x07\x00"
+			  "\x61\x62\x63\x64\x65\x66\x67\x68"
+			  "\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70"
+			  "\x71\x72\x73\x74\x01\x02\x02\x01",
+		.rlen	= 32,
+		.assoc	= "\x00\x00\x43\x21\x87\x65\x43\x21"
+			  "\x00\x00\x00\x07\x48\x55\xEC\x7D"
+			  "\x3A\x23\x4B\xFD",
+		.alen	= 20,
+		.input	= "\x67\xE9\x28\xB3\x1C\xA4\x6D\x02"
+			  "\xF0\xB5\x37\xB6\x6B\x2F\xF5\x4F"
+			  "\xF8\xA3\x4C\x53\xB8\x12\x09\xBF"
+			  "\x58\x7D\xCF\x29\xA3\x41\x68\x6B"
+			  "\xCE\xE8\x79\x85\x3C\xB0\x3A\x8F"
+			  "\x16\xB0\xA1\x26\xC9\xBC\xBC\xA6",
+		.ilen	= 48,
+	}
+};
+
+/*
+ * ChaCha20-Poly1305 AEAD test vectors from RFC7539 2.8.2./A.5.
+ */
+static const struct aead_testvec rfc7539_enc_tv_template[] = {
+	{
+		.key	= "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f",
+		.klen	= 32,
+		.iv	= "\x07\x00\x00\x00\x40\x41\x42\x43"
+			  "\x44\x45\x46\x47",
+		.assoc	= "\x50\x51\x52\x53\xc0\xc1\xc2\xc3"
+			  "\xc4\xc5\xc6\xc7",
+		.alen	= 12,
+		.input	= "\x4c\x61\x64\x69\x65\x73\x20\x61"
+			  "\x6e\x64\x20\x47\x65\x6e\x74\x6c"
+			  "\x65\x6d\x65\x6e\x20\x6f\x66\x20"
+			  "\x74\x68\x65\x20\x63\x6c\x61\x73"
+			  "\x73\x20\x6f\x66\x20\x27\x39\x39"
+			  "\x3a\x20\x49\x66\x20\x49\x20\x63"
+			  "\x6f\x75\x6c\x64\x20\x6f\x66\x66"
+			  "\x65\x72\x20\x79\x6f\x75\x20\x6f"
+			  "\x6e\x6c\x79\x20\x6f\x6e\x65\x20"
+			  "\x74\x69\x70\x20\x66\x6f\x72\x20"
+			  "\x74\x68\x65\x20\x66\x75\x74\x75"
+			  "\x72\x65\x2c\x20\x73\x75\x6e\x73"
+			  "\x63\x72\x65\x65\x6e\x20\x77\x6f"
+			  "\x75\x6c\x64\x20\x62\x65\x20\x69"
+			  "\x74\x2e",
+		.ilen	= 114,
+		.result	= "\xd3\x1a\x8d\x34\x64\x8e\x60\xdb"
+			  "\x7b\x86\xaf\xbc\x53\xef\x7e\xc2"
+			  "\xa4\xad\xed\x51\x29\x6e\x08\xfe"
+			  "\xa9\xe2\xb5\xa7\x36\xee\x62\xd6"
+			  "\x3d\xbe\xa4\x5e\x8c\xa9\x67\x12"
+			  "\x82\xfa\xfb\x69\xda\x92\x72\x8b"
+			  "\x1a\x71\xde\x0a\x9e\x06\x0b\x29"
+			  "\x05\xd6\xa5\xb6\x7e\xcd\x3b\x36"
+			  "\x92\xdd\xbd\x7f\x2d\x77\x8b\x8c"
+			  "\x98\x03\xae\xe3\x28\x09\x1b\x58"
+			  "\xfa\xb3\x24\xe4\xfa\xd6\x75\x94"
+			  "\x55\x85\x80\x8b\x48\x31\xd7\xbc"
+			  "\x3f\xf4\xde\xf0\x8e\x4b\x7a\x9d"
+			  "\xe5\x76\xd2\x65\x86\xce\xc6\x4b"
+			  "\x61\x16\x1a\xe1\x0b\x59\x4f\x09"
+			  "\xe2\x6a\x7e\x90\x2e\xcb\xd0\x60"
+			  "\x06\x91",
+		.rlen	= 130,
+	}, {
+		.key	= "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
+			  "\xf3\x33\x88\x86\x04\xf6\xb5\xf0"
+			  "\x47\x39\x17\xc1\x40\x2b\x80\x09"
+			  "\x9d\xca\x5c\xbc\x20\x70\x75\xc0",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x01\x02\x03\x04"
+			  "\x05\x06\x07\x08",
+		.assoc	= "\xf3\x33\x88\x86\x00\x00\x00\x00"
+			  "\x00\x00\x4e\x91",
+		.alen	= 12,
+		.input	= "\x49\x6e\x74\x65\x72\x6e\x65\x74"
+			  "\x2d\x44\x72\x61\x66\x74\x73\x20"
+			  "\x61\x72\x65\x20\x64\x72\x61\x66"
+			  "\x74\x20\x64\x6f\x63\x75\x6d\x65"
+			  "\x6e\x74\x73\x20\x76\x61\x6c\x69"
+			  "\x64\x20\x66\x6f\x72\x20\x61\x20"
+			  "\x6d\x61\x78\x69\x6d\x75\x6d\x20"
+			  "\x6f\x66\x20\x73\x69\x78\x20\x6d"
+			  "\x6f\x6e\x74\x68\x73\x20\x61\x6e"
+			  "\x64\x20\x6d\x61\x79\x20\x62\x65"
+			  "\x20\x75\x70\x64\x61\x74\x65\x64"
+			  "\x2c\x20\x72\x65\x70\x6c\x61\x63"
+			  "\x65\x64\x2c\x20\x6f\x72\x20\x6f"
+			  "\x62\x73\x6f\x6c\x65\x74\x65\x64"
+			  "\x20\x62\x79\x20\x6f\x74\x68\x65"
+			  "\x72\x20\x64\x6f\x63\x75\x6d\x65"
+			  "\x6e\x74\x73\x20\x61\x74\x20\x61"
+			  "\x6e\x79\x20\x74\x69\x6d\x65\x2e"
+			  "\x20\x49\x74\x20\x69\x73\x20\x69"
+			  "\x6e\x61\x70\x70\x72\x6f\x70\x72"
+			  "\x69\x61\x74\x65\x20\x74\x6f\x20"
+			  "\x75\x73\x65\x20\x49\x6e\x74\x65"
+			  "\x72\x6e\x65\x74\x2d\x44\x72\x61"
+			  "\x66\x74\x73\x20\x61\x73\x20\x72"
+			  "\x65\x66\x65\x72\x65\x6e\x63\x65"
+			  "\x20\x6d\x61\x74\x65\x72\x69\x61"
+			  "\x6c\x20\x6f\x72\x20\x74\x6f\x20"
+			  "\x63\x69\x74\x65\x20\x74\x68\x65"
+			  "\x6d\x20\x6f\x74\x68\x65\x72\x20"
+			  "\x74\x68\x61\x6e\x20\x61\x73\x20"
+			  "\x2f\xe2\x80\x9c\x77\x6f\x72\x6b"
+			  "\x20\x69\x6e\x20\x70\x72\x6f\x67"
+			  "\x72\x65\x73\x73\x2e\x2f\xe2\x80"
+			  "\x9d",
+		.ilen	= 265,
+		.result	= "\x64\xa0\x86\x15\x75\x86\x1a\xf4"
+			  "\x60\xf0\x62\xc7\x9b\xe6\x43\xbd"
+			  "\x5e\x80\x5c\xfd\x34\x5c\xf3\x89"
+			  "\xf1\x08\x67\x0a\xc7\x6c\x8c\xb2"
+			  "\x4c\x6c\xfc\x18\x75\x5d\x43\xee"
+			  "\xa0\x9e\xe9\x4e\x38\x2d\x26\xb0"
+			  "\xbd\xb7\xb7\x3c\x32\x1b\x01\x00"
+			  "\xd4\xf0\x3b\x7f\x35\x58\x94\xcf"
+			  "\x33\x2f\x83\x0e\x71\x0b\x97\xce"
+			  "\x98\xc8\xa8\x4a\xbd\x0b\x94\x81"
+			  "\x14\xad\x17\x6e\x00\x8d\x33\xbd"
+			  "\x60\xf9\x82\xb1\xff\x37\xc8\x55"
+			  "\x97\x97\xa0\x6e\xf4\xf0\xef\x61"
+			  "\xc1\x86\x32\x4e\x2b\x35\x06\x38"
+			  "\x36\x06\x90\x7b\x6a\x7c\x02\xb0"
+			  "\xf9\xf6\x15\x7b\x53\xc8\x67\xe4"
+			  "\xb9\x16\x6c\x76\x7b\x80\x4d\x46"
+			  "\xa5\x9b\x52\x16\xcd\xe7\xa4\xe9"
+			  "\x90\x40\xc5\xa4\x04\x33\x22\x5e"
+			  "\xe2\x82\xa1\xb0\xa0\x6c\x52\x3e"
+			  "\xaf\x45\x34\xd7\xf8\x3f\xa1\x15"
+			  "\x5b\x00\x47\x71\x8c\xbc\x54\x6a"
+			  "\x0d\x07\x2b\x04\xb3\x56\x4e\xea"
+			  "\x1b\x42\x22\x73\xf5\x48\x27\x1a"
+			  "\x0b\xb2\x31\x60\x53\xfa\x76\x99"
+			  "\x19\x55\xeb\xd6\x31\x59\x43\x4e"
+			  "\xce\xbb\x4e\x46\x6d\xae\x5a\x10"
+			  "\x73\xa6\x72\x76\x27\x09\x7a\x10"
+			  "\x49\xe6\x17\xd9\x1d\x36\x10\x94"
+			  "\xfa\x68\xf0\xff\x77\x98\x71\x30"
+			  "\x30\x5b\xea\xba\x2e\xda\x04\xdf"
+			  "\x99\x7b\x71\x4d\x6c\x6f\x2c\x29"
+			  "\xa6\xad\x5c\xb4\x02\x2b\x02\x70"
+			  "\x9b\xee\xad\x9d\x67\x89\x0c\xbb"
+			  "\x22\x39\x23\x36\xfe\xa1\x85\x1f"
+			  "\x38",
+		.rlen	= 281,
+	},
+};
+
+static const struct aead_testvec rfc7539_dec_tv_template[] = {
+	{
+		.key	= "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f",
+		.klen	= 32,
+		.iv	= "\x07\x00\x00\x00\x40\x41\x42\x43"
+			  "\x44\x45\x46\x47",
+		.assoc	= "\x50\x51\x52\x53\xc0\xc1\xc2\xc3"
+			  "\xc4\xc5\xc6\xc7",
+		.alen	= 12,
+		.input	= "\xd3\x1a\x8d\x34\x64\x8e\x60\xdb"
+			  "\x7b\x86\xaf\xbc\x53\xef\x7e\xc2"
+			  "\xa4\xad\xed\x51\x29\x6e\x08\xfe"
+			  "\xa9\xe2\xb5\xa7\x36\xee\x62\xd6"
+			  "\x3d\xbe\xa4\x5e\x8c\xa9\x67\x12"
+			  "\x82\xfa\xfb\x69\xda\x92\x72\x8b"
+			  "\x1a\x71\xde\x0a\x9e\x06\x0b\x29"
+			  "\x05\xd6\xa5\xb6\x7e\xcd\x3b\x36"
+			  "\x92\xdd\xbd\x7f\x2d\x77\x8b\x8c"
+			  "\x98\x03\xae\xe3\x28\x09\x1b\x58"
+			  "\xfa\xb3\x24\xe4\xfa\xd6\x75\x94"
+			  "\x55\x85\x80\x8b\x48\x31\xd7\xbc"
+			  "\x3f\xf4\xde\xf0\x8e\x4b\x7a\x9d"
+			  "\xe5\x76\xd2\x65\x86\xce\xc6\x4b"
+			  "\x61\x16\x1a\xe1\x0b\x59\x4f\x09"
+			  "\xe2\x6a\x7e\x90\x2e\xcb\xd0\x60"
+			  "\x06\x91",
+		.ilen	= 130,
+		.result	= "\x4c\x61\x64\x69\x65\x73\x20\x61"
+			  "\x6e\x64\x20\x47\x65\x6e\x74\x6c"
+			  "\x65\x6d\x65\x6e\x20\x6f\x66\x20"
+			  "\x74\x68\x65\x20\x63\x6c\x61\x73"
+			  "\x73\x20\x6f\x66\x20\x27\x39\x39"
+			  "\x3a\x20\x49\x66\x20\x49\x20\x63"
+			  "\x6f\x75\x6c\x64\x20\x6f\x66\x66"
+			  "\x65\x72\x20\x79\x6f\x75\x20\x6f"
+			  "\x6e\x6c\x79\x20\x6f\x6e\x65\x20"
+			  "\x74\x69\x70\x20\x66\x6f\x72\x20"
+			  "\x74\x68\x65\x20\x66\x75\x74\x75"
+			  "\x72\x65\x2c\x20\x73\x75\x6e\x73"
+			  "\x63\x72\x65\x65\x6e\x20\x77\x6f"
+			  "\x75\x6c\x64\x20\x62\x65\x20\x69"
+			  "\x74\x2e",
+		.rlen	= 114,
+	}, {
+		.key	= "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
+			  "\xf3\x33\x88\x86\x04\xf6\xb5\xf0"
+			  "\x47\x39\x17\xc1\x40\x2b\x80\x09"
+			  "\x9d\xca\x5c\xbc\x20\x70\x75\xc0",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x01\x02\x03\x04"
+			  "\x05\x06\x07\x08",
+		.assoc	= "\xf3\x33\x88\x86\x00\x00\x00\x00"
+			  "\x00\x00\x4e\x91",
+		.alen	= 12,
+		.input	= "\x64\xa0\x86\x15\x75\x86\x1a\xf4"
+			  "\x60\xf0\x62\xc7\x9b\xe6\x43\xbd"
+			  "\x5e\x80\x5c\xfd\x34\x5c\xf3\x89"
+			  "\xf1\x08\x67\x0a\xc7\x6c\x8c\xb2"
+			  "\x4c\x6c\xfc\x18\x75\x5d\x43\xee"
+			  "\xa0\x9e\xe9\x4e\x38\x2d\x26\xb0"
+			  "\xbd\xb7\xb7\x3c\x32\x1b\x01\x00"
+			  "\xd4\xf0\x3b\x7f\x35\x58\x94\xcf"
+			  "\x33\x2f\x83\x0e\x71\x0b\x97\xce"
+			  "\x98\xc8\xa8\x4a\xbd\x0b\x94\x81"
+			  "\x14\xad\x17\x6e\x00\x8d\x33\xbd"
+			  "\x60\xf9\x82\xb1\xff\x37\xc8\x55"
+			  "\x97\x97\xa0\x6e\xf4\xf0\xef\x61"
+			  "\xc1\x86\x32\x4e\x2b\x35\x06\x38"
+			  "\x36\x06\x90\x7b\x6a\x7c\x02\xb0"
+			  "\xf9\xf6\x15\x7b\x53\xc8\x67\xe4"
+			  "\xb9\x16\x6c\x76\x7b\x80\x4d\x46"
+			  "\xa5\x9b\x52\x16\xcd\xe7\xa4\xe9"
+			  "\x90\x40\xc5\xa4\x04\x33\x22\x5e"
+			  "\xe2\x82\xa1\xb0\xa0\x6c\x52\x3e"
+			  "\xaf\x45\x34\xd7\xf8\x3f\xa1\x15"
+			  "\x5b\x00\x47\x71\x8c\xbc\x54\x6a"
+			  "\x0d\x07\x2b\x04\xb3\x56\x4e\xea"
+			  "\x1b\x42\x22\x73\xf5\x48\x27\x1a"
+			  "\x0b\xb2\x31\x60\x53\xfa\x76\x99"
+			  "\x19\x55\xeb\xd6\x31\x59\x43\x4e"
+			  "\xce\xbb\x4e\x46\x6d\xae\x5a\x10"
+			  "\x73\xa6\x72\x76\x27\x09\x7a\x10"
+			  "\x49\xe6\x17\xd9\x1d\x36\x10\x94"
+			  "\xfa\x68\xf0\xff\x77\x98\x71\x30"
+			  "\x30\x5b\xea\xba\x2e\xda\x04\xdf"
+			  "\x99\x7b\x71\x4d\x6c\x6f\x2c\x29"
+			  "\xa6\xad\x5c\xb4\x02\x2b\x02\x70"
+			  "\x9b\xee\xad\x9d\x67\x89\x0c\xbb"
+			  "\x22\x39\x23\x36\xfe\xa1\x85\x1f"
+			  "\x38",
+		.ilen	= 281,
+		.result	= "\x49\x6e\x74\x65\x72\x6e\x65\x74"
+			  "\x2d\x44\x72\x61\x66\x74\x73\x20"
+			  "\x61\x72\x65\x20\x64\x72\x61\x66"
+			  "\x74\x20\x64\x6f\x63\x75\x6d\x65"
+			  "\x6e\x74\x73\x20\x76\x61\x6c\x69"
+			  "\x64\x20\x66\x6f\x72\x20\x61\x20"
+			  "\x6d\x61\x78\x69\x6d\x75\x6d\x20"
+			  "\x6f\x66\x20\x73\x69\x78\x20\x6d"
+			  "\x6f\x6e\x74\x68\x73\x20\x61\x6e"
+			  "\x64\x20\x6d\x61\x79\x20\x62\x65"
+			  "\x20\x75\x70\x64\x61\x74\x65\x64"
+			  "\x2c\x20\x72\x65\x70\x6c\x61\x63"
+			  "\x65\x64\x2c\x20\x6f\x72\x20\x6f"
+			  "\x62\x73\x6f\x6c\x65\x74\x65\x64"
+			  "\x20\x62\x79\x20\x6f\x74\x68\x65"
+			  "\x72\x20\x64\x6f\x63\x75\x6d\x65"
+			  "\x6e\x74\x73\x20\x61\x74\x20\x61"
+			  "\x6e\x79\x20\x74\x69\x6d\x65\x2e"
+			  "\x20\x49\x74\x20\x69\x73\x20\x69"
+			  "\x6e\x61\x70\x70\x72\x6f\x70\x72"
+			  "\x69\x61\x74\x65\x20\x74\x6f\x20"
+			  "\x75\x73\x65\x20\x49\x6e\x74\x65"
+			  "\x72\x6e\x65\x74\x2d\x44\x72\x61"
+			  "\x66\x74\x73\x20\x61\x73\x20\x72"
+			  "\x65\x66\x65\x72\x65\x6e\x63\x65"
+			  "\x20\x6d\x61\x74\x65\x72\x69\x61"
+			  "\x6c\x20\x6f\x72\x20\x74\x6f\x20"
+			  "\x63\x69\x74\x65\x20\x74\x68\x65"
+			  "\x6d\x20\x6f\x74\x68\x65\x72\x20"
+			  "\x74\x68\x61\x6e\x20\x61\x73\x20"
+			  "\x2f\xe2\x80\x9c\x77\x6f\x72\x6b"
+			  "\x20\x69\x6e\x20\x70\x72\x6f\x67"
+			  "\x72\x65\x73\x73\x2e\x2f\xe2\x80"
+			  "\x9d",
+		.rlen	= 265,
+	},
+};
+
+/*
+ * draft-irtf-cfrg-chacha20-poly1305
+ */
+static const struct aead_testvec rfc7539esp_enc_tv_template[] = {
+	{
+		.key	= "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
+			  "\xf3\x33\x88\x86\x04\xf6\xb5\xf0"
+			  "\x47\x39\x17\xc1\x40\x2b\x80\x09"
+			  "\x9d\xca\x5c\xbc\x20\x70\x75\xc0"
+			  "\x00\x00\x00\x00",
+		.klen	= 36,
+		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
+		.assoc	= "\xf3\x33\x88\x86\x00\x00\x00\x00"
+			  "\x00\x00\x4e\x91\x01\x02\x03\x04"
+			  "\x05\x06\x07\x08",
+		.alen	= 20,
+		.input	= "\x49\x6e\x74\x65\x72\x6e\x65\x74"
+			  "\x2d\x44\x72\x61\x66\x74\x73\x20"
+			  "\x61\x72\x65\x20\x64\x72\x61\x66"
+			  "\x74\x20\x64\x6f\x63\x75\x6d\x65"
+			  "\x6e\x74\x73\x20\x76\x61\x6c\x69"
+			  "\x64\x20\x66\x6f\x72\x20\x61\x20"
+			  "\x6d\x61\x78\x69\x6d\x75\x6d\x20"
+			  "\x6f\x66\x20\x73\x69\x78\x20\x6d"
+			  "\x6f\x6e\x74\x68\x73\x20\x61\x6e"
+			  "\x64\x20\x6d\x61\x79\x20\x62\x65"
+			  "\x20\x75\x70\x64\x61\x74\x65\x64"
+			  "\x2c\x20\x72\x65\x70\x6c\x61\x63"
+			  "\x65\x64\x2c\x20\x6f\x72\x20\x6f"
+			  "\x62\x73\x6f\x6c\x65\x74\x65\x64"
+			  "\x20\x62\x79\x20\x6f\x74\x68\x65"
+			  "\x72\x20\x64\x6f\x63\x75\x6d\x65"
+			  "\x6e\x74\x73\x20\x61\x74\x20\x61"
+			  "\x6e\x79\x20\x74\x69\x6d\x65\x2e"
+			  "\x20\x49\x74\x20\x69\x73\x20\x69"
+			  "\x6e\x61\x70\x70\x72\x6f\x70\x72"
+			  "\x69\x61\x74\x65\x20\x74\x6f\x20"
+			  "\x75\x73\x65\x20\x49\x6e\x74\x65"
+			  "\x72\x6e\x65\x74\x2d\x44\x72\x61"
+			  "\x66\x74\x73\x20\x61\x73\x20\x72"
+			  "\x65\x66\x65\x72\x65\x6e\x63\x65"
+			  "\x20\x6d\x61\x74\x65\x72\x69\x61"
+			  "\x6c\x20\x6f\x72\x20\x74\x6f\x20"
+			  "\x63\x69\x74\x65\x20\x74\x68\x65"
+			  "\x6d\x20\x6f\x74\x68\x65\x72\x20"
+			  "\x74\x68\x61\x6e\x20\x61\x73\x20"
+			  "\x2f\xe2\x80\x9c\x77\x6f\x72\x6b"
+			  "\x20\x69\x6e\x20\x70\x72\x6f\x67"
+			  "\x72\x65\x73\x73\x2e\x2f\xe2\x80"
+			  "\x9d",
+		.ilen	= 265,
+		.result	= "\x64\xa0\x86\x15\x75\x86\x1a\xf4"
+			  "\x60\xf0\x62\xc7\x9b\xe6\x43\xbd"
+			  "\x5e\x80\x5c\xfd\x34\x5c\xf3\x89"
+			  "\xf1\x08\x67\x0a\xc7\x6c\x8c\xb2"
+			  "\x4c\x6c\xfc\x18\x75\x5d\x43\xee"
+			  "\xa0\x9e\xe9\x4e\x38\x2d\x26\xb0"
+			  "\xbd\xb7\xb7\x3c\x32\x1b\x01\x00"
+			  "\xd4\xf0\x3b\x7f\x35\x58\x94\xcf"
+			  "\x33\x2f\x83\x0e\x71\x0b\x97\xce"
+			  "\x98\xc8\xa8\x4a\xbd\x0b\x94\x81"
+			  "\x14\xad\x17\x6e\x00\x8d\x33\xbd"
+			  "\x60\xf9\x82\xb1\xff\x37\xc8\x55"
+			  "\x97\x97\xa0\x6e\xf4\xf0\xef\x61"
+			  "\xc1\x86\x32\x4e\x2b\x35\x06\x38"
+			  "\x36\x06\x90\x7b\x6a\x7c\x02\xb0"
+			  "\xf9\xf6\x15\x7b\x53\xc8\x67\xe4"
+			  "\xb9\x16\x6c\x76\x7b\x80\x4d\x46"
+			  "\xa5\x9b\x52\x16\xcd\xe7\xa4\xe9"
+			  "\x90\x40\xc5\xa4\x04\x33\x22\x5e"
+			  "\xe2\x82\xa1\xb0\xa0\x6c\x52\x3e"
+			  "\xaf\x45\x34\xd7\xf8\x3f\xa1\x15"
+			  "\x5b\x00\x47\x71\x8c\xbc\x54\x6a"
+			  "\x0d\x07\x2b\x04\xb3\x56\x4e\xea"
+			  "\x1b\x42\x22\x73\xf5\x48\x27\x1a"
+			  "\x0b\xb2\x31\x60\x53\xfa\x76\x99"
+			  "\x19\x55\xeb\xd6\x31\x59\x43\x4e"
+			  "\xce\xbb\x4e\x46\x6d\xae\x5a\x10"
+			  "\x73\xa6\x72\x76\x27\x09\x7a\x10"
+			  "\x49\xe6\x17\xd9\x1d\x36\x10\x94"
+			  "\xfa\x68\xf0\xff\x77\x98\x71\x30"
+			  "\x30\x5b\xea\xba\x2e\xda\x04\xdf"
+			  "\x99\x7b\x71\x4d\x6c\x6f\x2c\x29"
+			  "\xa6\xad\x5c\xb4\x02\x2b\x02\x70"
+			  "\x9b\xee\xad\x9d\x67\x89\x0c\xbb"
+			  "\x22\x39\x23\x36\xfe\xa1\x85\x1f"
+			  "\x38",
+		.rlen	= 281,
+	},
+};
+
+static const struct aead_testvec rfc7539esp_dec_tv_template[] = {
+	{
+		.key	= "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
+			  "\xf3\x33\x88\x86\x04\xf6\xb5\xf0"
+			  "\x47\x39\x17\xc1\x40\x2b\x80\x09"
+			  "\x9d\xca\x5c\xbc\x20\x70\x75\xc0"
+			  "\x00\x00\x00\x00",
+		.klen	= 36,
+		.iv	= "\x01\x02\x03\x04\x05\x06\x07\x08",
+		.assoc	= "\xf3\x33\x88\x86\x00\x00\x00\x00"
+			  "\x00\x00\x4e\x91\x01\x02\x03\x04"
+			  "\x05\x06\x07\x08",
+		.alen	= 20,
+		.input	= "\x64\xa0\x86\x15\x75\x86\x1a\xf4"
+			  "\x60\xf0\x62\xc7\x9b\xe6\x43\xbd"
+			  "\x5e\x80\x5c\xfd\x34\x5c\xf3\x89"
+			  "\xf1\x08\x67\x0a\xc7\x6c\x8c\xb2"
+			  "\x4c\x6c\xfc\x18\x75\x5d\x43\xee"
+			  "\xa0\x9e\xe9\x4e\x38\x2d\x26\xb0"
+			  "\xbd\xb7\xb7\x3c\x32\x1b\x01\x00"
+			  "\xd4\xf0\x3b\x7f\x35\x58\x94\xcf"
+			  "\x33\x2f\x83\x0e\x71\x0b\x97\xce"
+			  "\x98\xc8\xa8\x4a\xbd\x0b\x94\x81"
+			  "\x14\xad\x17\x6e\x00\x8d\x33\xbd"
+			  "\x60\xf9\x82\xb1\xff\x37\xc8\x55"
+			  "\x97\x97\xa0\x6e\xf4\xf0\xef\x61"
+			  "\xc1\x86\x32\x4e\x2b\x35\x06\x38"
+			  "\x36\x06\x90\x7b\x6a\x7c\x02\xb0"
+			  "\xf9\xf6\x15\x7b\x53\xc8\x67\xe4"
+			  "\xb9\x16\x6c\x76\x7b\x80\x4d\x46"
+			  "\xa5\x9b\x52\x16\xcd\xe7\xa4\xe9"
+			  "\x90\x40\xc5\xa4\x04\x33\x22\x5e"
+			  "\xe2\x82\xa1\xb0\xa0\x6c\x52\x3e"
+			  "\xaf\x45\x34\xd7\xf8\x3f\xa1\x15"
+			  "\x5b\x00\x47\x71\x8c\xbc\x54\x6a"
+			  "\x0d\x07\x2b\x04\xb3\x56\x4e\xea"
+			  "\x1b\x42\x22\x73\xf5\x48\x27\x1a"
+			  "\x0b\xb2\x31\x60\x53\xfa\x76\x99"
+			  "\x19\x55\xeb\xd6\x31\x59\x43\x4e"
+			  "\xce\xbb\x4e\x46\x6d\xae\x5a\x10"
+			  "\x73\xa6\x72\x76\x27\x09\x7a\x10"
+			  "\x49\xe6\x17\xd9\x1d\x36\x10\x94"
+			  "\xfa\x68\xf0\xff\x77\x98\x71\x30"
+			  "\x30\x5b\xea\xba\x2e\xda\x04\xdf"
+			  "\x99\x7b\x71\x4d\x6c\x6f\x2c\x29"
+			  "\xa6\xad\x5c\xb4\x02\x2b\x02\x70"
+			  "\x9b\xee\xad\x9d\x67\x89\x0c\xbb"
+			  "\x22\x39\x23\x36\xfe\xa1\x85\x1f"
+			  "\x38",
+		.ilen	= 281,
+		.result	= "\x49\x6e\x74\x65\x72\x6e\x65\x74"
+			  "\x2d\x44\x72\x61\x66\x74\x73\x20"
+			  "\x61\x72\x65\x20\x64\x72\x61\x66"
+			  "\x74\x20\x64\x6f\x63\x75\x6d\x65"
+			  "\x6e\x74\x73\x20\x76\x61\x6c\x69"
+			  "\x64\x20\x66\x6f\x72\x20\x61\x20"
+			  "\x6d\x61\x78\x69\x6d\x75\x6d\x20"
+			  "\x6f\x66\x20\x73\x69\x78\x20\x6d"
+			  "\x6f\x6e\x74\x68\x73\x20\x61\x6e"
+			  "\x64\x20\x6d\x61\x79\x20\x62\x65"
+			  "\x20\x75\x70\x64\x61\x74\x65\x64"
+			  "\x2c\x20\x72\x65\x70\x6c\x61\x63"
+			  "\x65\x64\x2c\x20\x6f\x72\x20\x6f"
+			  "\x62\x73\x6f\x6c\x65\x74\x65\x64"
+			  "\x20\x62\x79\x20\x6f\x74\x68\x65"
+			  "\x72\x20\x64\x6f\x63\x75\x6d\x65"
+			  "\x6e\x74\x73\x20\x61\x74\x20\x61"
+			  "\x6e\x79\x20\x74\x69\x6d\x65\x2e"
+			  "\x20\x49\x74\x20\x69\x73\x20\x69"
+			  "\x6e\x61\x70\x70\x72\x6f\x70\x72"
+			  "\x69\x61\x74\x65\x20\x74\x6f\x20"
+			  "\x75\x73\x65\x20\x49\x6e\x74\x65"
+			  "\x72\x6e\x65\x74\x2d\x44\x72\x61"
+			  "\x66\x74\x73\x20\x61\x73\x20\x72"
+			  "\x65\x66\x65\x72\x65\x6e\x63\x65"
+			  "\x20\x6d\x61\x74\x65\x72\x69\x61"
+			  "\x6c\x20\x6f\x72\x20\x74\x6f\x20"
+			  "\x63\x69\x74\x65\x20\x74\x68\x65"
+			  "\x6d\x20\x6f\x74\x68\x65\x72\x20"
+			  "\x74\x68\x61\x6e\x20\x61\x73\x20"
+			  "\x2f\xe2\x80\x9c\x77\x6f\x72\x6b"
+			  "\x20\x69\x6e\x20\x70\x72\x6f\x67"
+			  "\x72\x65\x73\x73\x2e\x2f\xe2\x80"
+			  "\x9d",
+		.rlen	= 265,
+	},
+};
+
+static const struct aead_testvec aegis128_enc_tv_template[] = {
+	{
+		.key	= "\x0f\xc9\x8e\x67\x44\x9e\xaa\x86"
+			  "\x20\x36\x2c\x24\xfe\xc9\x30\x81",
+		.klen	= 16,
+		.iv	= "\x1e\x92\x1c\xcf\x88\x3d\x54\x0d"
+			  "\x40\x6d\x59\x48\xfc\x92\x61\x03",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x07\xa5\x11\xf2\x9d\x40\xb8\x6d"
+			  "\xda\xb8\x12\x34\x4c\x53\xd9\x72",
+		.rlen	= 16,
+	}, {
+		.key	= "\x4b\xed\xc8\x07\x54\x1a\x52\xa2"
+			  "\xa1\x10\xde\xb5\xf8\xed\xf3\x87",
+		.klen	= 16,
+		.iv	= "\x5a\xb7\x56\x6e\x98\xb9\xfd\x29"
+			  "\xc1\x47\x0b\xda\xf6\xb6\x23\x09",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x79",
+		.ilen	= 1,
+		.result	= "\x9e\x78\x52\xae\xcb\x9e\xe4\xd3"
+			  "\x9a\xd7\x5d\xd7\xaa\x9a\xe9\x5a"
+			  "\xcc",
+		.rlen	= 17,
+	}, {
+		.key	= "\x88\x12\x01\xa6\x64\x96\xfb\xbe"
+			  "\x22\xea\x90\x47\xf2\x11\xb5\x8e",
+		.klen	= 16,
+		.iv	= "\x97\xdb\x90\x0e\xa8\x35\xa5\x45"
+			  "\x42\x21\xbd\x6b\xf0\xda\xe6\x0f",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xb5\x6e\xad\xdd\x30\x72\xfa\x53"
+			  "\x82\x8e\x16\xb4\xed\x6d\x47",
+		.ilen	= 15,
+		.result	= "\xc3\x80\x83\x04\x5f\xaa\x61\xc7"
+			  "\xca\xdd\x6f\xac\x85\x08\xb5\x35"
+			  "\x2b\xc2\x3e\x0b\x1b\x39\x37\x2b"
+			  "\x7a\x21\x16\xb3\xe6\x67\x66",
+		.rlen	= 31,
+	}, {
+		.key	= "\xc4\x37\x3b\x45\x74\x11\xa4\xda"
+			  "\xa2\xc5\x42\xd8\xec\x36\x78\x94",
+		.klen	= 16,
+		.iv	= "\xd3\x00\xc9\xad\xb8\xb0\x4e\x61"
+			  "\xc3\xfb\x6f\xfd\xea\xff\xa9\x15",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xf2\x92\xe6\x7d\x40\xee\xa3\x6f"
+			  "\x03\x68\xc8\x45\xe7\x91\x0a\x18",
+		.ilen	= 16,
+		.result	= "\x23\x25\x30\xe5\x6a\xb6\x36\x7d"
+			  "\x38\xfd\x3a\xd2\xc2\x58\xa9\x11"
+			  "\x1e\xa8\x30\x9c\x16\xa4\xdb\x65"
+			  "\x51\x10\x16\x27\x70\x9b\x64\x29",
+		.rlen	= 32,
+	}, {
+		.key	= "\x01\x5c\x75\xe5\x84\x8d\x4d\xf6"
+			  "\x23\x9f\xf4\x6a\xe6\x5a\x3b\x9a",
+		.klen	= 16,
+		.iv	= "\x10\x25\x03\x4c\xc8\x2c\xf7\x7d"
+			  "\x44\xd5\x21\x8e\xe4\x23\x6b\x1c",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x2e\xb7\x20\x1c\x50\x6a\x4b\x8b"
+			  "\x84\x42\x7a\xd7\xe1\xb5\xcd\x1f"
+			  "\xd3",
+		.ilen	= 17,
+		.result	= "\x2a\x8d\x56\x91\xc6\xf3\x56\xa5"
+			  "\x1f\xf0\x89\x2e\x13\xad\xe6\xf6"
+			  "\x46\x80\xb1\x0e\x18\x30\x40\x97"
+			  "\x03\xdf\x64\x3c\xbe\x93\x9e\xc9"
+			  "\x3b",
+		.rlen	= 33,
+	}, {
+		.key	= "\x3d\x80\xae\x84\x94\x09\xf6\x12"
+			  "\xa4\x79\xa6\xfb\xe0\x7f\xfd\xa0",
+		.klen	= 16,
+		.iv	= "\x4c\x49\x3d\xec\xd8\xa8\xa0\x98"
+			  "\xc5\xb0\xd3\x1f\xde\x48\x2e\x22",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x6b\xdc\x5a\xbb\x60\xe5\xf4\xa6"
+			  "\x05\x1d\x2c\x68\xdb\xda\x8f\x25"
+			  "\xfe\x8d\x45\x19\x1e\xc0\x0b\x99"
+			  "\x88\x11\x39\x12\x1c\x3a\xbb",
+		.ilen	= 31,
+		.result	= "\x4e\xf6\xfa\x13\xde\x43\x63\x4c"
+			  "\xe2\x04\x3e\xe4\x85\x14\xb6\x3f"
+			  "\xb1\x8f\x4c\xdb\x41\xa2\x14\x99"
+			  "\xf5\x53\x0f\x73\x86\x7e\x97\xa1"
+			  "\x4b\x56\x5b\x94\xce\xcd\x74\xcd"
+			  "\x75\xc4\x53\x01\x89\x45\x59",
+		.rlen	= 47,
+	}, {
+		.key	= "\x7a\xa5\xe8\x23\xa4\x84\x9e\x2d"
+			  "\x25\x53\x58\x8c\xda\xa3\xc0\xa6",
+		.klen	= 16,
+		.iv	= "\x89\x6e\x77\x8b\xe8\x23\x49\xb4"
+			  "\x45\x8a\x85\xb1\xd8\x6c\xf1\x28",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xa7\x00\x93\x5b\x70\x61\x9d\xc2"
+			  "\x86\xf7\xde\xfa\xd5\xfe\x52\x2b"
+			  "\x28\x50\x51\x9d\x24\x60\x8d\xb3"
+			  "\x49\x3e\x17\xea\xf6\x99\x5a\xdd",
+		.ilen	= 32,
+		.result	= "\xa4\x9a\xb7\xfd\xa0\xd4\xd6\x47"
+			  "\x95\xf4\x58\x38\x14\x83\x27\x01"
+			  "\x4c\xed\x32\x2c\xf7\xd6\x31\xf7"
+			  "\x38\x1b\x2c\xc9\xb6\x31\xce\xaa"
+			  "\xa5\x3c\x1a\x18\x5c\xce\xb9\xdf"
+			  "\x51\x52\x77\xf2\x5e\x85\x80\x41",
+		.rlen	= 48,
+	}, {
+		.key	= "\xb6\xca\x22\xc3\xb4\x00\x47\x49"
+			  "\xa6\x2d\x0a\x1e\xd4\xc7\x83\xad",
+		.klen	= 16,
+		.iv	= "\xc5\x93\xb0\x2a\xf8\x9f\xf1\xd0"
+			  "\xc6\x64\x37\x42\xd2\x90\xb3\x2e",
+		.assoc	= "\xd5",
+		.alen	= 1,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xfb\xd4\x83\x71\x9e\x63\xad\x60"
+			  "\xb9\xf9\xeb\x34\x52\x49\xcf\xb7",
+		.rlen	= 16,
+	}, {
+		.key	= "\xf3\xee\x5c\x62\xc4\x7c\xf0\x65"
+			  "\x27\x08\xbd\xaf\xce\xec\x45\xb3",
+		.klen	= 16,
+		.iv	= "\x02\xb8\xea\xca\x09\x1b\x9a\xec"
+			  "\x47\x3e\xe9\xd4\xcc\xb5\x76\x34",
+		.assoc	= "\x11\x81\x78\x32\x4d\xb9\x44\x73"
+			  "\x68\x75\x16\xf8\xcb\x7e\xa7",
+		.alen	= 15,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x0c\xaf\x2e\x96\xf6\x97\x08\x71"
+			  "\x7d\x3a\x84\xc4\x44\x57\x77\x7e",
+		.rlen	= 16,
+	}, {
+		.key	= "\x2f\x13\x95\x01\xd5\xf7\x99\x81"
+			  "\xa8\xe2\x6f\x41\xc8\x10\x08\xb9",
+		.klen	= 16,
+		.iv	= "\x3f\xdc\x24\x69\x19\x96\x43\x08"
+			  "\xc8\x18\x9b\x65\xc6\xd9\x39\x3b",
+		.assoc	= "\x4e\xa5\xb2\xd1\x5d\x35\xed\x8f"
+			  "\xe8\x4f\xc8\x89\xc5\xa2\x69\xbc",
+		.alen	= 16,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xc7\x87\x09\x3b\xc7\x19\x74\x22"
+			  "\x22\xa5\x67\x10\xb2\x36\xb3\x45",
+		.rlen	= 16,
+	}, {
+		.key	= "\x6c\x38\xcf\xa1\xe5\x73\x41\x9d"
+			  "\x29\xbc\x21\xd2\xc2\x35\xcb\xbf",
+		.klen	= 16,
+		.iv	= "\x7b\x01\x5d\x08\x29\x12\xec\x24"
+			  "\x49\xf3\x4d\xf7\xc0\xfe\xfb\x41",
+		.assoc	= "\x8a\xca\xec\x70\x6d\xb1\x96\xab"
+			  "\x69\x29\x7a\x1b\xbf\xc7\x2c\xc2"
+			  "\x07",
+		.alen	= 17,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x02\xc6\x3b\x46\x65\xb2\xef\x91"
+			  "\x31\xf0\x45\x48\x8a\x2a\xed\xe4",
+		.rlen	= 16,
+	}, {
+		.key	= "\xa8\x5c\x09\x40\xf5\xef\xea\xb8"
+			  "\xaa\x96\xd3\x64\xbc\x59\x8d\xc6",
+		.klen	= 16,
+		.iv	= "\xb8\x26\x97\xa8\x39\x8e\x94\x3f"
+			  "\xca\xcd\xff\x88\xba\x22\xbe\x47",
+		.assoc	= "\xc7\xef\x26\x10\x7d\x2c\x3f\xc6"
+			  "\xea\x03\x2c\xac\xb9\xeb\xef\xc9"
+			  "\x31\x6b\x08\x12\xfc\xd8\x37\x2d"
+			  "\xe0\x17\x3a\x2e\x83\x5c\x8f",
+		.alen	= 31,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x20\x85\xa8\xd0\x91\x48\x85\xf3"
+			  "\x5a\x16\xc0\x57\x68\x47\xdd\xcb",
+		.rlen	= 16,
+	}, {
+		.key	= "\xe5\x81\x42\xdf\x05\x6a\x93\xd4"
+			  "\x2b\x70\x85\xf5\xb6\x7d\x50\xcc",
+		.klen	= 16,
+		.iv	= "\xf4\x4a\xd1\x47\x49\x09\x3d\x5b"
+			  "\x4b\xa7\xb1\x19\xb4\x46\x81\x4d",
+		.assoc	= "\x03\x14\x5f\xaf\x8d\xa8\xe7\xe2"
+			  "\x6b\xde\xde\x3e\xb3\x10\xb1\xcf"
+			  "\x5c\x2d\x14\x96\x01\x78\xb9\x47"
+			  "\xa1\x44\x19\x06\x5d\xbb\x2e\x2f",
+		.alen	= 32,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x6a\xf8\x8d\x9c\x42\x75\x35\x79"
+			  "\xc1\x96\xbd\x31\x6e\x69\x1b\x50",
+		.rlen	= 16,
+	}, {
+		.key	= "\x22\xa6\x7c\x7f\x15\xe6\x3c\xf0"
+			  "\xac\x4b\x37\x86\xb0\xa2\x13\xd2",
+		.klen	= 16,
+		.iv	= "\x31\x6f\x0b\xe6\x59\x85\xe6\x77"
+			  "\xcc\x81\x63\xab\xae\x6b\x43\x54",
+		.assoc	= "\x40",
+		.alen	= 1,
+		.input	= "\x4f",
+		.ilen	= 1,
+		.result	= "\x01\x24\xb1\xba\xf6\xd3\xdf\x83"
+			  "\x70\x45\xe3\x2a\x9d\x5c\x63\x98"
+			  "\x39",
+		.rlen	= 17,
+	}, {
+		.key	= "\x5e\xcb\xb6\x1e\x25\x62\xe4\x0c"
+			  "\x2d\x25\xe9\x18\xaa\xc6\xd5\xd8",
+		.klen	= 16,
+		.iv	= "\x6d\x94\x44\x86\x69\x00\x8f\x93"
+			  "\x4d\x5b\x15\x3c\xa8\x8f\x06\x5a",
+		.assoc	= "\x7c\x5d\xd3\xee\xad\x9f\x39\x1a"
+			  "\x6d\x92\x42\x61\xa7\x58\x37",
+		.alen	= 15,
+		.input	= "\x8b\x26\x61\x55\xf1\x3e\xe3\xa1"
+			  "\x8d\xc8\x6e\x85\xa5\x21\x67",
+		.ilen	= 15,
+		.result	= "\x18\x78\xc2\x6e\xe1\xf7\xe6\x8a"
+			  "\xca\x0e\x62\x00\xa8\x21\xb5\x21"
+			  "\x3d\x36\xdb\xf7\xcc\x31\x94\x9c"
+			  "\x98\xbd\x71\x7a\xef\xa4\xfa",
+		.rlen	= 31,
+	}, {
+		.key	= "\x9b\xef\xf0\xbd\x35\xdd\x8d\x28"
+			  "\xad\xff\x9b\xa9\xa4\xeb\x98\xdf",
+		.klen	= 16,
+		.iv	= "\xaa\xb8\x7e\x25\x79\x7c\x37\xaf"
+			  "\xce\x36\xc7\xce\xa2\xb4\xc9\x60",
+		.assoc	= "\xb9\x82\x0c\x8d\xbd\x1b\xe2\x36"
+			  "\xee\x6c\xf4\xf2\xa1\x7d\xf9\xe2",
+		.alen	= 16,
+		.input	= "\xc8\x4b\x9b\xf5\x01\xba\x8c\xbd"
+			  "\x0e\xa3\x21\x16\x9f\x46\x2a\x63",
+		.ilen	= 16,
+		.result	= "\xea\xd1\x81\x75\xb4\x13\x1d\x86"
+			  "\xd4\x17\x26\xe5\xd6\x89\x39\x04"
+			  "\xa9\x6c\xca\xac\x40\x73\xb2\x4c"
+			  "\x9c\xb9\x0e\x79\x4c\x40\x65\xc6",
+		.rlen	= 32,
+	}, {
+		.key	= "\xd7\x14\x29\x5d\x45\x59\x36\x44"
+			  "\x2e\xd9\x4d\x3b\x9e\x0f\x5b\xe5",
+		.klen	= 16,
+		.iv	= "\xe6\xdd\xb8\xc4\x89\xf8\xe0\xca"
+			  "\x4f\x10\x7a\x5f\x9c\xd8\x8b\x66",
+		.assoc	= "\xf5\xa6\x46\x2c\xce\x97\x8a\x51"
+			  "\x6f\x46\xa6\x83\x9b\xa1\xbc\xe8"
+			  "\x05",
+		.alen	= 17,
+		.input	= "\x05\x70\xd5\x94\x12\x36\x35\xd8"
+			  "\x8f\x7d\xd3\xa8\x99\x6a\xed\x69"
+			  "\xd0",
+		.ilen	= 17,
+		.result	= "\xf4\xb2\x84\xd1\x81\xfa\x98\x1c"
+			  "\x38\x2d\x69\x90\x1c\x71\x38\x98"
+			  "\x9f\xe1\x19\x3b\x63\x91\xaf\x6e"
+			  "\x4b\x07\x2c\xac\x53\xc5\xd5\xfe"
+			  "\x93",
+		.rlen	= 33,
+	}, {
+		.key	= "\x14\x39\x63\xfc\x56\xd5\xdf\x5f"
+			  "\xaf\xb3\xff\xcc\x98\x33\x1d\xeb",
+		.klen	= 16,
+		.iv	= "\x23\x02\xf1\x64\x9a\x73\x89\xe6"
+			  "\xd0\xea\x2c\xf1\x96\xfc\x4e\x6d",
+		.assoc	= "\x32\xcb\x80\xcc\xde\x12\x33\x6d"
+			  "\xf0\x20\x58\x15\x95\xc6\x7f\xee"
+			  "\x2f\xf9\x4e\x2c\x1b\x98\x43\xc7"
+			  "\x68\x28\x73\x40\x9f\x96\x4a",
+		.alen	= 31,
+		.input	= "\x41\x94\x0e\x33\x22\xb1\xdd\xf4"
+			  "\x10\x57\x85\x39\x93\x8f\xaf\x70"
+			  "\xfa\xa9\xd0\x4d\x5c\x40\x23\xcd"
+			  "\x98\x34\xab\x37\x56\xae\x32",
+		.ilen	= 31,
+		.result	= "\xa0\xe7\x0a\x60\xe7\xb8\x8a\xdb"
+			  "\x94\xd3\x93\xf2\x41\x86\x16\xdd"
+			  "\x4c\xe8\xe7\xe0\x62\x48\x89\x40"
+			  "\xc0\x49\x9b\x63\x32\xec\x8b\xdb"
+			  "\xdc\xa6\xea\x2c\xc2\x7f\xf5\x04"
+			  "\xcb\xe5\x47\xbb\xa7\xd1\x9d",
+		.rlen	= 47,
+	}, {
+		.key	= "\x50\x5d\x9d\x9b\x66\x50\x88\x7b"
+			  "\x30\x8e\xb1\x5e\x92\x58\xe0\xf1",
+		.klen	= 16,
+		.iv	= "\x5f\x27\x2b\x03\xaa\xef\x32\x02"
+			  "\x50\xc4\xde\x82\x90\x21\x11\x73",
+		.assoc	= "\x6e\xf0\xba\x6b\xee\x8e\xdc\x89"
+			  "\x71\xfb\x0a\xa6\x8f\xea\x41\xf4"
+			  "\x5a\xbb\x59\xb0\x20\x38\xc5\xe0"
+			  "\x29\x56\x52\x19\x79\xf5\xe9\x37",
+		.alen	= 32,
+		.input	= "\x7e\xb9\x48\xd3\x32\x2d\x86\x10"
+			  "\x91\x31\x37\xcb\x8d\xb3\x72\x76"
+			  "\x24\x6b\xdc\xd1\x61\xe0\xa5\xe7"
+			  "\x5a\x61\x8a\x0f\x30\x0d\xd1\xec",
+		.ilen	= 32,
+		.result	= "\x62\xdc\x2d\x68\x2d\x71\xbb\x33"
+			  "\x13\xdf\xc0\x46\xf6\x61\x94\xa7"
+			  "\x60\xd3\xd4\xca\xd9\xbe\x82\xf3"
+			  "\xf1\x5b\xa0\xfa\x15\xba\xda\xea"
+			  "\x87\x68\x47\x08\x5d\xdd\x83\xb0"
+			  "\x60\xf4\x93\x20\xdf\x34\x8f\xea",
+		.rlen	= 48,
+	}, {
+		.key	= "\x8d\x82\xd6\x3b\x76\xcc\x30\x97"
+			  "\xb1\x68\x63\xef\x8c\x7c\xa3\xf7",
+		.klen	= 16,
+		.iv	= "\x9c\x4b\x65\xa2\xba\x6b\xdb\x1e"
+			  "\xd1\x9e\x90\x13\x8a\x45\xd3\x79",
+		.assoc	= "\xab\x14\xf3\x0a\xfe\x0a\x85\xa5"
+			  "\xf2\xd5\xbc\x38\x89\x0e\x04\xfb"
+			  "\x84\x7d\x65\x34\x25\xd8\x47\xfa"
+			  "\xeb\x83\x31\xf1\x54\x54\x89\x0d"
+			  "\x9d",
+		.alen	= 33,
+		.input	= "\xba\xde\x82\x72\x42\xa9\x2f\x2c"
+			  "\x12\x0b\xe9\x5c\x87\xd7\x35\x7c"
+			  "\x4f\x2e\xe8\x55\x66\x80\x27\x00"
+			  "\x1b\x8f\x68\xe7\x0a\x6c\x71\xc3"
+			  "\x21\x78\x55\x9d\x9c\x65\x7b\xcd"
+			  "\x0a\x34\x97\xff\x47\x37\xb0\x2a"
+			  "\x80\x0d\x19\x98\x33\xa9\x7a\xe3"
+			  "\x2e\x4c\xc6\xf3\x8c\x88\x42\x01"
+			  "\xbd",
+		.ilen	= 65,
+		.result	= "\x84\xc5\x21\xab\xe1\xeb\xbb\x6d"
+			  "\xaa\x2a\xaf\xeb\x3b\x3b\x69\xe7"
+			  "\x2c\x47\xef\x9d\xb7\x53\x36\xb7"
+			  "\xb6\xf5\xe5\xa8\xc9\x9e\x02\xd7"
+			  "\x83\x88\xc2\xbd\x2f\xf9\x10\xc0"
+			  "\xf5\xa1\x6e\xd3\x97\x64\x82\xa3"
+			  "\xfb\xda\x2c\xb1\x94\xa1\x58\x32"
+			  "\xe8\xd4\x39\xfc\x9e\x26\xf9\xf1"
+			  "\x61\xe6\xae\x07\xf2\xe0\xa7\x44"
+			  "\x96\x28\x3b\xee\x6b\xc6\x16\x31"
+			  "\x3f",
+		.rlen	= 81,
+	}, {
+		.key	= "\xc9\xa7\x10\xda\x86\x48\xd9\xb3"
+			  "\x32\x42\x15\x80\x85\xa1\x65\xfe",
+		.klen	= 16,
+		.iv	= "\xd8\x70\x9f\x42\xca\xe6\x83\x3a"
+			  "\x52\x79\x42\xa5\x84\x6a\x96\x7f",
+		.assoc	= "\xe8\x39\x2d\xaa\x0e\x85\x2d\xc1"
+			  "\x72\xaf\x6e\xc9\x82\x33\xc7\x01"
+			  "\xaf\x40\x70\xb8\x2a\x78\xc9\x14"
+			  "\xac\xb1\x10\xca\x2e\xb3\x28\xe4"
+			  "\xac\xfa\x58\x7f\xe5\x73\x09\x8c"
+			  "\x1d\x40\x87\x8c\xd9\x75\xc0\x55"
+			  "\xa2\xda\x07\xd1\xc2\xa9\xd1\xbb"
+			  "\x09\x4f\x77\x62\x88\x2d\xf2\x68"
+			  "\x54",
+		.alen	= 65,
+		.input	= "\xf7\x02\xbb\x11\x52\x24\xd8\x48"
+			  "\x93\xe6\x9b\xee\x81\xfc\xf7\x82"
+			  "\x79\xf0\xf3\xd9\x6c\x20\xa9\x1a"
+			  "\xdc\xbc\x47\xc0\xe4\xcb\x10\x99"
+			  "\x2f",
+		.ilen	= 33,
+		.result	= "\x8f\x23\x47\xfb\xf2\xac\x23\x83"
+			  "\x77\x09\xac\x74\xef\xd2\x56\xae"
+			  "\x20\x7b\x7b\xca\x45\x8e\xc8\xc2"
+			  "\x50\xbd\xc7\x44\x1c\x54\x98\xd8"
+			  "\x1f\xd0\x9a\x79\xaa\xf9\xe1\xb3"
+			  "\xb4\x98\x5a\x9b\xe4\x4d\xbf\x4e"
+			  "\x39",
+		.rlen	= 49,
+	}, {
+		.key	= "\x06\xcc\x4a\x79\x96\xc3\x82\xcf"
+			  "\xb3\x1c\xc7\x12\x7f\xc5\x28\x04",
+		.klen	= 16,
+		.iv	= "\x15\x95\xd8\xe1\xda\x62\x2c\x56"
+			  "\xd3\x53\xf4\x36\x7e\x8e\x59\x85",
+		.assoc	= "\x24\x5e\x67\x49\x1e\x01\xd6\xdd"
+			  "\xf3\x89\x20\x5b\x7c\x57\x89\x07",
+		.alen	= 16,
+		.input	= "\x33\x27\xf5\xb1\x62\xa0\x80\x63"
+			  "\x14\xc0\x4d\x7f\x7b\x20\xba\x89",
+		.ilen	= 16,
+		.result	= "\x42\xc3\x58\xfb\x29\xe2\x4a\x56"
+			  "\xf1\xf5\xe1\x51\x55\x4b\x0a\x45"
+			  "\x46\xb5\x8d\xac\xb6\x34\xd8\x8b"
+			  "\xde\x20\x59\x77\xc1\x74\x90",
+		.rlen	= 31,
+	}, {
+		.key	= "\x42\xf0\x84\x19\xa6\x3f\x2b\xea"
+			  "\x34\xf6\x79\xa3\x79\xe9\xeb\x0a",
+		.klen	= 16,
+		.iv	= "\x51\xb9\x12\x80\xea\xde\xd5\x71"
+			  "\x54\x2d\xa6\xc8\x78\xb2\x1b\x8c",
+		.assoc	= "\x61\x83\xa0\xe8\x2e\x7d\x7f\xf8"
+			  "\x74\x63\xd2\xec\x76\x7c\x4c\x0d",
+		.alen	= 16,
+		.input	= "\x70\x4c\x2f\x50\x72\x1c\x29\x7f"
+			  "\x95\x9a\xff\x10\x75\x45\x7d\x8f",
+		.ilen	= 16,
+		.result	= "\xb2\xfb\xf6\x97\x69\x7a\xe9\xec"
+			  "\xe2\x94\xa1\x8b\xa0\x2b\x60\x72"
+			  "\x1d\x04\xdd\x6a\xef\x46\x8f\x68"
+			  "\xe9\xe0\x17\x45\x70\x12",
+		.rlen	= 30,
+	}, {
+		.key	= "\x7f\x15\xbd\xb8\xb6\xba\xd3\x06"
+			  "\xb5\xd1\x2b\x35\x73\x0e\xad\x10",
+		.klen	= 16,
+		.iv	= "\x8e\xde\x4c\x20\xfa\x59\x7e\x8d"
+			  "\xd5\x07\x58\x59\x72\xd7\xde\x92",
+		.assoc	= "\x9d\xa7\xda\x88\x3e\xf8\x28\x14"
+			  "\xf5\x3e\x85\x7d\x70\xa0\x0f\x13",
+		.alen	= 16,
+		.input	= "\xac\x70\x69\xef\x82\x97\xd2\x9b"
+			  "\x15\x74\xb1\xa2\x6f\x69\x3f\x95",
+		.ilen	= 16,
+		.result	= "\x47\xda\x54\x42\x51\x72\xc4\x8b"
+			  "\xf5\x57\x0f\x2f\x49\x0e\x11\x3b"
+			  "\x78\x93\xec\xfc\xf4\xff\xe1\x2d",
+		.rlen	= 24,
+	},
+};
+
+/*
+ * AEGIS-128 test vectors - generated via reference implementation from
+ * SUPERCOP (https://bench.cr.yp.to/supercop.html):
+ *
+ *   https://bench.cr.yp.to/supercop/supercop-20170228.tar.xz
+ *   (see crypto_aead/aegis128/)
+ */
+static const struct aead_testvec aegis128_dec_tv_template[] = {
+	{
+		.key	= "\x0f\xc9\x8e\x67\x44\x9e\xaa\x86"
+			  "\x20\x36\x2c\x24\xfe\xc9\x30\x81",
+		.klen	= 16,
+		.iv	= "\x1e\x92\x1c\xcf\x88\x3d\x54\x0d"
+			  "\x40\x6d\x59\x48\xfc\x92\x61\x03",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x07\xa5\x11\xf2\x9d\x40\xb8\x6d"
+			  "\xda\xb8\x12\x34\x4c\x53\xd9\x72",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x4b\xed\xc8\x07\x54\x1a\x52\xa2"
+			  "\xa1\x10\xde\xb5\xf8\xed\xf3\x87",
+		.klen	= 16,
+		.iv	= "\x5a\xb7\x56\x6e\x98\xb9\xfd\x29"
+			  "\xc1\x47\x0b\xda\xf6\xb6\x23\x09",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x9e\x78\x52\xae\xcb\x9e\xe4\xd3"
+			  "\x9a\xd7\x5d\xd7\xaa\x9a\xe9\x5a"
+			  "\xcc",
+		.ilen	= 17,
+		.result	= "\x79",
+		.rlen	= 1,
+	}, {
+		.key	= "\x88\x12\x01\xa6\x64\x96\xfb\xbe"
+			  "\x22\xea\x90\x47\xf2\x11\xb5\x8e",
+		.klen	= 16,
+		.iv	= "\x97\xdb\x90\x0e\xa8\x35\xa5\x45"
+			  "\x42\x21\xbd\x6b\xf0\xda\xe6\x0f",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xc3\x80\x83\x04\x5f\xaa\x61\xc7"
+			  "\xca\xdd\x6f\xac\x85\x08\xb5\x35"
+			  "\x2b\xc2\x3e\x0b\x1b\x39\x37\x2b"
+			  "\x7a\x21\x16\xb3\xe6\x67\x66",
+		.ilen	= 31,
+		.result	= "\xb5\x6e\xad\xdd\x30\x72\xfa\x53"
+			  "\x82\x8e\x16\xb4\xed\x6d\x47",
+		.rlen	= 15,
+	}, {
+		.key	= "\xc4\x37\x3b\x45\x74\x11\xa4\xda"
+			  "\xa2\xc5\x42\xd8\xec\x36\x78\x94",
+		.klen	= 16,
+		.iv	= "\xd3\x00\xc9\xad\xb8\xb0\x4e\x61"
+			  "\xc3\xfb\x6f\xfd\xea\xff\xa9\x15",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x23\x25\x30\xe5\x6a\xb6\x36\x7d"
+			  "\x38\xfd\x3a\xd2\xc2\x58\xa9\x11"
+			  "\x1e\xa8\x30\x9c\x16\xa4\xdb\x65"
+			  "\x51\x10\x16\x27\x70\x9b\x64\x29",
+		.ilen	= 32,
+		.result	= "\xf2\x92\xe6\x7d\x40\xee\xa3\x6f"
+			  "\x03\x68\xc8\x45\xe7\x91\x0a\x18",
+		.rlen	= 16,
+	}, {
+		.key	= "\x01\x5c\x75\xe5\x84\x8d\x4d\xf6"
+			  "\x23\x9f\xf4\x6a\xe6\x5a\x3b\x9a",
+		.klen	= 16,
+		.iv	= "\x10\x25\x03\x4c\xc8\x2c\xf7\x7d"
+			  "\x44\xd5\x21\x8e\xe4\x23\x6b\x1c",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x2a\x8d\x56\x91\xc6\xf3\x56\xa5"
+			  "\x1f\xf0\x89\x2e\x13\xad\xe6\xf6"
+			  "\x46\x80\xb1\x0e\x18\x30\x40\x97"
+			  "\x03\xdf\x64\x3c\xbe\x93\x9e\xc9"
+			  "\x3b",
+		.ilen	= 33,
+		.result	= "\x2e\xb7\x20\x1c\x50\x6a\x4b\x8b"
+			  "\x84\x42\x7a\xd7\xe1\xb5\xcd\x1f"
+			  "\xd3",
+		.rlen	= 17,
+	}, {
+		.key	= "\x3d\x80\xae\x84\x94\x09\xf6\x12"
+			  "\xa4\x79\xa6\xfb\xe0\x7f\xfd\xa0",
+		.klen	= 16,
+		.iv	= "\x4c\x49\x3d\xec\xd8\xa8\xa0\x98"
+			  "\xc5\xb0\xd3\x1f\xde\x48\x2e\x22",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x4e\xf6\xfa\x13\xde\x43\x63\x4c"
+			  "\xe2\x04\x3e\xe4\x85\x14\xb6\x3f"
+			  "\xb1\x8f\x4c\xdb\x41\xa2\x14\x99"
+			  "\xf5\x53\x0f\x73\x86\x7e\x97\xa1"
+			  "\x4b\x56\x5b\x94\xce\xcd\x74\xcd"
+			  "\x75\xc4\x53\x01\x89\x45\x59",
+		.ilen	= 47,
+		.result	= "\x6b\xdc\x5a\xbb\x60\xe5\xf4\xa6"
+			  "\x05\x1d\x2c\x68\xdb\xda\x8f\x25"
+			  "\xfe\x8d\x45\x19\x1e\xc0\x0b\x99"
+			  "\x88\x11\x39\x12\x1c\x3a\xbb",
+		.rlen	= 31,
+	}, {
+		.key	= "\x7a\xa5\xe8\x23\xa4\x84\x9e\x2d"
+			  "\x25\x53\x58\x8c\xda\xa3\xc0\xa6",
+		.klen	= 16,
+		.iv	= "\x89\x6e\x77\x8b\xe8\x23\x49\xb4"
+			  "\x45\x8a\x85\xb1\xd8\x6c\xf1\x28",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xa4\x9a\xb7\xfd\xa0\xd4\xd6\x47"
+			  "\x95\xf4\x58\x38\x14\x83\x27\x01"
+			  "\x4c\xed\x32\x2c\xf7\xd6\x31\xf7"
+			  "\x38\x1b\x2c\xc9\xb6\x31\xce\xaa"
+			  "\xa5\x3c\x1a\x18\x5c\xce\xb9\xdf"
+			  "\x51\x52\x77\xf2\x5e\x85\x80\x41",
+		.ilen	= 48,
+		.result	= "\xa7\x00\x93\x5b\x70\x61\x9d\xc2"
+			  "\x86\xf7\xde\xfa\xd5\xfe\x52\x2b"
+			  "\x28\x50\x51\x9d\x24\x60\x8d\xb3"
+			  "\x49\x3e\x17\xea\xf6\x99\x5a\xdd",
+		.rlen	= 32,
+	}, {
+		.key	= "\xb6\xca\x22\xc3\xb4\x00\x47\x49"
+			  "\xa6\x2d\x0a\x1e\xd4\xc7\x83\xad",
+		.klen	= 16,
+		.iv	= "\xc5\x93\xb0\x2a\xf8\x9f\xf1\xd0"
+			  "\xc6\x64\x37\x42\xd2\x90\xb3\x2e",
+		.assoc	= "\xd5",
+		.alen	= 1,
+		.input	= "\xfb\xd4\x83\x71\x9e\x63\xad\x60"
+			  "\xb9\xf9\xeb\x34\x52\x49\xcf\xb7",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xf3\xee\x5c\x62\xc4\x7c\xf0\x65"
+			  "\x27\x08\xbd\xaf\xce\xec\x45\xb3",
+		.klen	= 16,
+		.iv	= "\x02\xb8\xea\xca\x09\x1b\x9a\xec"
+			  "\x47\x3e\xe9\xd4\xcc\xb5\x76\x34",
+		.assoc	= "\x11\x81\x78\x32\x4d\xb9\x44\x73"
+			  "\x68\x75\x16\xf8\xcb\x7e\xa7",
+		.alen	= 15,
+		.input	= "\x0c\xaf\x2e\x96\xf6\x97\x08\x71"
+			  "\x7d\x3a\x84\xc4\x44\x57\x77\x7e",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x2f\x13\x95\x01\xd5\xf7\x99\x81"
+			  "\xa8\xe2\x6f\x41\xc8\x10\x08\xb9",
+		.klen	= 16,
+		.iv	= "\x3f\xdc\x24\x69\x19\x96\x43\x08"
+			  "\xc8\x18\x9b\x65\xc6\xd9\x39\x3b",
+		.assoc	= "\x4e\xa5\xb2\xd1\x5d\x35\xed\x8f"
+			  "\xe8\x4f\xc8\x89\xc5\xa2\x69\xbc",
+		.alen	= 16,
+		.input	= "\xc7\x87\x09\x3b\xc7\x19\x74\x22"
+			  "\x22\xa5\x67\x10\xb2\x36\xb3\x45",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x6c\x38\xcf\xa1\xe5\x73\x41\x9d"
+			  "\x29\xbc\x21\xd2\xc2\x35\xcb\xbf",
+		.klen	= 16,
+		.iv	= "\x7b\x01\x5d\x08\x29\x12\xec\x24"
+			  "\x49\xf3\x4d\xf7\xc0\xfe\xfb\x41",
+		.assoc	= "\x8a\xca\xec\x70\x6d\xb1\x96\xab"
+			  "\x69\x29\x7a\x1b\xbf\xc7\x2c\xc2"
+			  "\x07",
+		.alen	= 17,
+		.input	= "\x02\xc6\x3b\x46\x65\xb2\xef\x91"
+			  "\x31\xf0\x45\x48\x8a\x2a\xed\xe4",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xa8\x5c\x09\x40\xf5\xef\xea\xb8"
+			  "\xaa\x96\xd3\x64\xbc\x59\x8d\xc6",
+		.klen	= 16,
+		.iv	= "\xb8\x26\x97\xa8\x39\x8e\x94\x3f"
+			  "\xca\xcd\xff\x88\xba\x22\xbe\x47",
+		.assoc	= "\xc7\xef\x26\x10\x7d\x2c\x3f\xc6"
+			  "\xea\x03\x2c\xac\xb9\xeb\xef\xc9"
+			  "\x31\x6b\x08\x12\xfc\xd8\x37\x2d"
+			  "\xe0\x17\x3a\x2e\x83\x5c\x8f",
+		.alen	= 31,
+		.input	= "\x20\x85\xa8\xd0\x91\x48\x85\xf3"
+			  "\x5a\x16\xc0\x57\x68\x47\xdd\xcb",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xe5\x81\x42\xdf\x05\x6a\x93\xd4"
+			  "\x2b\x70\x85\xf5\xb6\x7d\x50\xcc",
+		.klen	= 16,
+		.iv	= "\xf4\x4a\xd1\x47\x49\x09\x3d\x5b"
+			  "\x4b\xa7\xb1\x19\xb4\x46\x81\x4d",
+		.assoc	= "\x03\x14\x5f\xaf\x8d\xa8\xe7\xe2"
+			  "\x6b\xde\xde\x3e\xb3\x10\xb1\xcf"
+			  "\x5c\x2d\x14\x96\x01\x78\xb9\x47"
+			  "\xa1\x44\x19\x06\x5d\xbb\x2e\x2f",
+		.alen	= 32,
+		.input	= "\x6a\xf8\x8d\x9c\x42\x75\x35\x79"
+			  "\xc1\x96\xbd\x31\x6e\x69\x1b\x50",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x22\xa6\x7c\x7f\x15\xe6\x3c\xf0"
+			  "\xac\x4b\x37\x86\xb0\xa2\x13\xd2",
+		.klen	= 16,
+		.iv	= "\x31\x6f\x0b\xe6\x59\x85\xe6\x77"
+			  "\xcc\x81\x63\xab\xae\x6b\x43\x54",
+		.assoc	= "\x40",
+		.alen	= 1,
+		.input	= "\x01\x24\xb1\xba\xf6\xd3\xdf\x83"
+			  "\x70\x45\xe3\x2a\x9d\x5c\x63\x98"
+			  "\x39",
+		.ilen	= 17,
+		.result	= "\x4f",
+		.rlen	= 1,
+	}, {
+		.key	= "\x5e\xcb\xb6\x1e\x25\x62\xe4\x0c"
+			  "\x2d\x25\xe9\x18\xaa\xc6\xd5\xd8",
+		.klen	= 16,
+		.iv	= "\x6d\x94\x44\x86\x69\x00\x8f\x93"
+			  "\x4d\x5b\x15\x3c\xa8\x8f\x06\x5a",
+		.assoc	= "\x7c\x5d\xd3\xee\xad\x9f\x39\x1a"
+			  "\x6d\x92\x42\x61\xa7\x58\x37",
+		.alen	= 15,
+		.input	= "\x18\x78\xc2\x6e\xe1\xf7\xe6\x8a"
+			  "\xca\x0e\x62\x00\xa8\x21\xb5\x21"
+			  "\x3d\x36\xdb\xf7\xcc\x31\x94\x9c"
+			  "\x98\xbd\x71\x7a\xef\xa4\xfa",
+		.ilen	= 31,
+		.result	= "\x8b\x26\x61\x55\xf1\x3e\xe3\xa1"
+			  "\x8d\xc8\x6e\x85\xa5\x21\x67",
+		.rlen	= 15,
+	}, {
+		.key	= "\x9b\xef\xf0\xbd\x35\xdd\x8d\x28"
+			  "\xad\xff\x9b\xa9\xa4\xeb\x98\xdf",
+		.klen	= 16,
+		.iv	= "\xaa\xb8\x7e\x25\x79\x7c\x37\xaf"
+			  "\xce\x36\xc7\xce\xa2\xb4\xc9\x60",
+		.assoc	= "\xb9\x82\x0c\x8d\xbd\x1b\xe2\x36"
+			  "\xee\x6c\xf4\xf2\xa1\x7d\xf9\xe2",
+		.alen	= 16,
+		.input	= "\xea\xd1\x81\x75\xb4\x13\x1d\x86"
+			  "\xd4\x17\x26\xe5\xd6\x89\x39\x04"
+			  "\xa9\x6c\xca\xac\x40\x73\xb2\x4c"
+			  "\x9c\xb9\x0e\x79\x4c\x40\x65\xc6",
+		.ilen	= 32,
+		.result	= "\xc8\x4b\x9b\xf5\x01\xba\x8c\xbd"
+			  "\x0e\xa3\x21\x16\x9f\x46\x2a\x63",
+		.rlen	= 16,
+	}, {
+		.key	= "\xd7\x14\x29\x5d\x45\x59\x36\x44"
+			  "\x2e\xd9\x4d\x3b\x9e\x0f\x5b\xe5",
+		.klen	= 16,
+		.iv	= "\xe6\xdd\xb8\xc4\x89\xf8\xe0\xca"
+			  "\x4f\x10\x7a\x5f\x9c\xd8\x8b\x66",
+		.assoc	= "\xf5\xa6\x46\x2c\xce\x97\x8a\x51"
+			  "\x6f\x46\xa6\x83\x9b\xa1\xbc\xe8"
+			  "\x05",
+		.alen	= 17,
+		.input	= "\xf4\xb2\x84\xd1\x81\xfa\x98\x1c"
+			  "\x38\x2d\x69\x90\x1c\x71\x38\x98"
+			  "\x9f\xe1\x19\x3b\x63\x91\xaf\x6e"
+			  "\x4b\x07\x2c\xac\x53\xc5\xd5\xfe"
+			  "\x93",
+		.ilen	= 33,
+		.result	= "\x05\x70\xd5\x94\x12\x36\x35\xd8"
+			  "\x8f\x7d\xd3\xa8\x99\x6a\xed\x69"
+			  "\xd0",
+		.rlen	= 17,
+	}, {
+		.key	= "\x14\x39\x63\xfc\x56\xd5\xdf\x5f"
+			  "\xaf\xb3\xff\xcc\x98\x33\x1d\xeb",
+		.klen	= 16,
+		.iv	= "\x23\x02\xf1\x64\x9a\x73\x89\xe6"
+			  "\xd0\xea\x2c\xf1\x96\xfc\x4e\x6d",
+		.assoc	= "\x32\xcb\x80\xcc\xde\x12\x33\x6d"
+			  "\xf0\x20\x58\x15\x95\xc6\x7f\xee"
+			  "\x2f\xf9\x4e\x2c\x1b\x98\x43\xc7"
+			  "\x68\x28\x73\x40\x9f\x96\x4a",
+		.alen	= 31,
+		.input	= "\xa0\xe7\x0a\x60\xe7\xb8\x8a\xdb"
+			  "\x94\xd3\x93\xf2\x41\x86\x16\xdd"
+			  "\x4c\xe8\xe7\xe0\x62\x48\x89\x40"
+			  "\xc0\x49\x9b\x63\x32\xec\x8b\xdb"
+			  "\xdc\xa6\xea\x2c\xc2\x7f\xf5\x04"
+			  "\xcb\xe5\x47\xbb\xa7\xd1\x9d",
+		.ilen	= 47,
+		.result	= "\x41\x94\x0e\x33\x22\xb1\xdd\xf4"
+			  "\x10\x57\x85\x39\x93\x8f\xaf\x70"
+			  "\xfa\xa9\xd0\x4d\x5c\x40\x23\xcd"
+			  "\x98\x34\xab\x37\x56\xae\x32",
+		.rlen	= 31,
+	}, {
+		.key	= "\x50\x5d\x9d\x9b\x66\x50\x88\x7b"
+			  "\x30\x8e\xb1\x5e\x92\x58\xe0\xf1",
+		.klen	= 16,
+		.iv	= "\x5f\x27\x2b\x03\xaa\xef\x32\x02"
+			  "\x50\xc4\xde\x82\x90\x21\x11\x73",
+		.assoc	= "\x6e\xf0\xba\x6b\xee\x8e\xdc\x89"
+			  "\x71\xfb\x0a\xa6\x8f\xea\x41\xf4"
+			  "\x5a\xbb\x59\xb0\x20\x38\xc5\xe0"
+			  "\x29\x56\x52\x19\x79\xf5\xe9\x37",
+		.alen	= 32,
+		.input	= "\x62\xdc\x2d\x68\x2d\x71\xbb\x33"
+			  "\x13\xdf\xc0\x46\xf6\x61\x94\xa7"
+			  "\x60\xd3\xd4\xca\xd9\xbe\x82\xf3"
+			  "\xf1\x5b\xa0\xfa\x15\xba\xda\xea"
+			  "\x87\x68\x47\x08\x5d\xdd\x83\xb0"
+			  "\x60\xf4\x93\x20\xdf\x34\x8f\xea",
+		.ilen	= 48,
+		.result	= "\x7e\xb9\x48\xd3\x32\x2d\x86\x10"
+			  "\x91\x31\x37\xcb\x8d\xb3\x72\x76"
+			  "\x24\x6b\xdc\xd1\x61\xe0\xa5\xe7"
+			  "\x5a\x61\x8a\x0f\x30\x0d\xd1\xec",
+		.rlen	= 32,
+	}, {
+		.key	= "\x8d\x82\xd6\x3b\x76\xcc\x30\x97"
+			  "\xb1\x68\x63\xef\x8c\x7c\xa3\xf7",
+		.klen	= 16,
+		.iv	= "\x9c\x4b\x65\xa2\xba\x6b\xdb\x1e"
+			  "\xd1\x9e\x90\x13\x8a\x45\xd3\x79",
+		.assoc	= "\xab\x14\xf3\x0a\xfe\x0a\x85\xa5"
+			  "\xf2\xd5\xbc\x38\x89\x0e\x04\xfb"
+			  "\x84\x7d\x65\x34\x25\xd8\x47\xfa"
+			  "\xeb\x83\x31\xf1\x54\x54\x89\x0d"
+			  "\x9d",
+		.alen	= 33,
+		.input	= "\x84\xc5\x21\xab\xe1\xeb\xbb\x6d"
+			  "\xaa\x2a\xaf\xeb\x3b\x3b\x69\xe7"
+			  "\x2c\x47\xef\x9d\xb7\x53\x36\xb7"
+			  "\xb6\xf5\xe5\xa8\xc9\x9e\x02\xd7"
+			  "\x83\x88\xc2\xbd\x2f\xf9\x10\xc0"
+			  "\xf5\xa1\x6e\xd3\x97\x64\x82\xa3"
+			  "\xfb\xda\x2c\xb1\x94\xa1\x58\x32"
+			  "\xe8\xd4\x39\xfc\x9e\x26\xf9\xf1"
+			  "\x61\xe6\xae\x07\xf2\xe0\xa7\x44"
+			  "\x96\x28\x3b\xee\x6b\xc6\x16\x31"
+			  "\x3f",
+		.ilen	= 81,
+		.result	= "\xba\xde\x82\x72\x42\xa9\x2f\x2c"
+			  "\x12\x0b\xe9\x5c\x87\xd7\x35\x7c"
+			  "\x4f\x2e\xe8\x55\x66\x80\x27\x00"
+			  "\x1b\x8f\x68\xe7\x0a\x6c\x71\xc3"
+			  "\x21\x78\x55\x9d\x9c\x65\x7b\xcd"
+			  "\x0a\x34\x97\xff\x47\x37\xb0\x2a"
+			  "\x80\x0d\x19\x98\x33\xa9\x7a\xe3"
+			  "\x2e\x4c\xc6\xf3\x8c\x88\x42\x01"
+			  "\xbd",
+		.rlen	= 65,
+	}, {
+		.key	= "\xc9\xa7\x10\xda\x86\x48\xd9\xb3"
+			  "\x32\x42\x15\x80\x85\xa1\x65\xfe",
+		.klen	= 16,
+		.iv	= "\xd8\x70\x9f\x42\xca\xe6\x83\x3a"
+			  "\x52\x79\x42\xa5\x84\x6a\x96\x7f",
+		.assoc	= "\xe8\x39\x2d\xaa\x0e\x85\x2d\xc1"
+			  "\x72\xaf\x6e\xc9\x82\x33\xc7\x01"
+			  "\xaf\x40\x70\xb8\x2a\x78\xc9\x14"
+			  "\xac\xb1\x10\xca\x2e\xb3\x28\xe4"
+			  "\xac\xfa\x58\x7f\xe5\x73\x09\x8c"
+			  "\x1d\x40\x87\x8c\xd9\x75\xc0\x55"
+			  "\xa2\xda\x07\xd1\xc2\xa9\xd1\xbb"
+			  "\x09\x4f\x77\x62\x88\x2d\xf2\x68"
+			  "\x54",
+		.alen	= 65,
+		.input	= "\x8f\x23\x47\xfb\xf2\xac\x23\x83"
+			  "\x77\x09\xac\x74\xef\xd2\x56\xae"
+			  "\x20\x7b\x7b\xca\x45\x8e\xc8\xc2"
+			  "\x50\xbd\xc7\x44\x1c\x54\x98\xd8"
+			  "\x1f\xd0\x9a\x79\xaa\xf9\xe1\xb3"
+			  "\xb4\x98\x5a\x9b\xe4\x4d\xbf\x4e"
+			  "\x39",
+		.ilen	= 49,
+		.result	= "\xf7\x02\xbb\x11\x52\x24\xd8\x48"
+			  "\x93\xe6\x9b\xee\x81\xfc\xf7\x82"
+			  "\x79\xf0\xf3\xd9\x6c\x20\xa9\x1a"
+			  "\xdc\xbc\x47\xc0\xe4\xcb\x10\x99"
+			  "\x2f",
+		.rlen	= 33,
+	}, {
+		.key	= "\x06\xcc\x4a\x79\x96\xc3\x82\xcf"
+			  "\xb3\x1c\xc7\x12\x7f\xc5\x28\x04",
+		.klen	= 16,
+		.iv	= "\x15\x95\xd8\xe1\xda\x62\x2c\x56"
+			  "\xd3\x53\xf4\x36\x7e\x8e\x59\x85",
+		.assoc	= "\x24\x5e\x67\x49\x1e\x01\xd6\xdd"
+			  "\xf3\x89\x20\x5b\x7c\x57\x89\x07",
+		.alen	= 16,
+		.input	= "\x42\xc3\x58\xfb\x29\xe2\x4a\x56"
+			  "\xf1\xf5\xe1\x51\x55\x4b\x0a\x45"
+			  "\x46\xb5\x8d\xac\xb6\x34\xd8\x8b"
+			  "\xde\x20\x59\x77\xc1\x74\x90",
+		.ilen	= 31,
+		.result	= "\x33\x27\xf5\xb1\x62\xa0\x80\x63"
+			  "\x14\xc0\x4d\x7f\x7b\x20\xba\x89",
+		.rlen	= 16,
+	}, {
+		.key	= "\x42\xf0\x84\x19\xa6\x3f\x2b\xea"
+			  "\x34\xf6\x79\xa3\x79\xe9\xeb\x0a",
+		.klen	= 16,
+		.iv	= "\x51\xb9\x12\x80\xea\xde\xd5\x71"
+			  "\x54\x2d\xa6\xc8\x78\xb2\x1b\x8c",
+		.assoc	= "\x61\x83\xa0\xe8\x2e\x7d\x7f\xf8"
+			  "\x74\x63\xd2\xec\x76\x7c\x4c\x0d",
+		.alen	= 16,
+		.input	= "\xb2\xfb\xf6\x97\x69\x7a\xe9\xec"
+			  "\xe2\x94\xa1\x8b\xa0\x2b\x60\x72"
+			  "\x1d\x04\xdd\x6a\xef\x46\x8f\x68"
+			  "\xe9\xe0\x17\x45\x70\x12",
+		.ilen	= 30,
+		.result	= "\x70\x4c\x2f\x50\x72\x1c\x29\x7f"
+			  "\x95\x9a\xff\x10\x75\x45\x7d\x8f",
+		.rlen	= 16,
+	}, {
+		.key	= "\x7f\x15\xbd\xb8\xb6\xba\xd3\x06"
+			  "\xb5\xd1\x2b\x35\x73\x0e\xad\x10",
+		.klen	= 16,
+		.iv	= "\x8e\xde\x4c\x20\xfa\x59\x7e\x8d"
+			  "\xd5\x07\x58\x59\x72\xd7\xde\x92",
+		.assoc	= "\x9d\xa7\xda\x88\x3e\xf8\x28\x14"
+			  "\xf5\x3e\x85\x7d\x70\xa0\x0f\x13",
+		.alen	= 16,
+		.input	= "\x47\xda\x54\x42\x51\x72\xc4\x8b"
+			  "\xf5\x57\x0f\x2f\x49\x0e\x11\x3b"
+			  "\x78\x93\xec\xfc\xf4\xff\xe1\x2d",
+		.ilen	= 24,
+		.result	= "\xac\x70\x69\xef\x82\x97\xd2\x9b"
+			  "\x15\x74\xb1\xa2\x6f\x69\x3f\x95",
+		.rlen	= 16,
+	},
+};
+
+/*
+ * AEGIS-128L test vectors - generated via reference implementation from
+ * SUPERCOP (https://bench.cr.yp.to/supercop.html):
+ *
+ *   https://bench.cr.yp.to/supercop/supercop-20170228.tar.xz
+ *   (see crypto_aead/aegis128l/)
+ */
+static const struct aead_testvec aegis128l_enc_tv_template[] = {
+	{
+		.key	= "\x0f\xc9\x8e\x67\x44\x9e\xaa\x86"
+			  "\x20\x36\x2c\x24\xfe\xc9\x30\x81",
+		.klen	= 16,
+		.iv	= "\x1e\x92\x1c\xcf\x88\x3d\x54\x0d"
+			  "\x40\x6d\x59\x48\xfc\x92\x61\x03",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x30\x4f\xf3\xe9\xb1\xfa\x81\xa6"
+			  "\x20\x72\x78\xdd\x93\xc8\x57\xef",
+		.rlen	= 16,
+	}, {
+		.key	= "\x4b\xed\xc8\x07\x54\x1a\x52\xa2"
+			  "\xa1\x10\xde\xb5\xf8\xed\xf3\x87",
+		.klen	= 16,
+		.iv	= "\x5a\xb7\x56\x6e\x98\xb9\xfd\x29"
+			  "\xc1\x47\x0b\xda\xf6\xb6\x23\x09",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x79",
+		.ilen	= 1,
+		.result	= "\xa9\x24\xa0\xb6\x2d\xdd\x29\xdb"
+			  "\x40\xb3\x71\xc5\x22\x58\x31\x77"
+			  "\x6d",
+		.rlen	= 17,
+	}, {
+		.key	= "\x88\x12\x01\xa6\x64\x96\xfb\xbe"
+			  "\x22\xea\x90\x47\xf2\x11\xb5\x8e",
+		.klen	= 16,
+		.iv	= "\x97\xdb\x90\x0e\xa8\x35\xa5\x45"
+			  "\x42\x21\xbd\x6b\xf0\xda\xe6\x0f",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xb5\x6e\xad\xdd\x30\x72\xfa\x53"
+			  "\x82\x8e\x16\xb4\xed\x6d\x47",
+		.ilen	= 15,
+		.result	= "\xbb\x0a\x53\xc4\xaa\x7e\xa4\x03"
+			  "\x2b\xee\x62\x99\x7b\x98\x13\x1f"
+			  "\xe0\x76\x4c\x2e\x53\x99\x4f\xbe"
+			  "\xe1\xa8\x04\x7f\xe1\x71\xbe",
+		.rlen	= 31,
+	}, {
+		.key	= "\xc4\x37\x3b\x45\x74\x11\xa4\xda"
+			  "\xa2\xc5\x42\xd8\xec\x36\x78\x94",
+		.klen	= 16,
+		.iv	= "\xd3\x00\xc9\xad\xb8\xb0\x4e\x61"
+			  "\xc3\xfb\x6f\xfd\xea\xff\xa9\x15",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xf2\x92\xe6\x7d\x40\xee\xa3\x6f"
+			  "\x03\x68\xc8\x45\xe7\x91\x0a\x18",
+		.ilen	= 16,
+		.result	= "\x66\xdf\x6e\x71\xc0\x6e\xa4\x4c"
+			  "\x9d\xb7\x8c\x9a\xdb\x1f\xd2\x2e"
+			  "\x23\xb6\xa4\xfb\xd3\x86\xdd\xbb"
+			  "\xde\x54\x9b\xf5\x92\x8b\x93\xc5",
+		.rlen	= 32,
+	}, {
+		.key	= "\x01\x5c\x75\xe5\x84\x8d\x4d\xf6"
+			  "\x23\x9f\xf4\x6a\xe6\x5a\x3b\x9a",
+		.klen	= 16,
+		.iv	= "\x10\x25\x03\x4c\xc8\x2c\xf7\x7d"
+			  "\x44\xd5\x21\x8e\xe4\x23\x6b\x1c",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x2e\xb7\x20\x1c\x50\x6a\x4b\x8b"
+			  "\x84\x42\x7a\xd7\xe1\xb5\xcd\x1f"
+			  "\xd3",
+		.ilen	= 17,
+		.result	= "\x4f\xc3\x69\xb6\xd3\xa4\x64\x8b"
+			  "\x71\xc3\x8a\x91\x22\x4f\x1b\xd2"
+			  "\x33\x6d\x86\xbc\xf8\x2f\x06\xf9"
+			  "\x82\x64\xc7\x72\x00\x30\xfc\xf0"
+			  "\xf8",
+		.rlen	= 33,
+	}, {
+		.key	= "\x3d\x80\xae\x84\x94\x09\xf6\x12"
+			  "\xa4\x79\xa6\xfb\xe0\x7f\xfd\xa0",
+		.klen	= 16,
+		.iv	= "\x4c\x49\x3d\xec\xd8\xa8\xa0\x98"
+			  "\xc5\xb0\xd3\x1f\xde\x48\x2e\x22",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x6b\xdc\x5a\xbb\x60\xe5\xf4\xa6"
+			  "\x05\x1d\x2c\x68\xdb\xda\x8f\x25"
+			  "\xfe\x8d\x45\x19\x1e\xc0\x0b\x99"
+			  "\x88\x11\x39\x12\x1c\x3a\xbb",
+		.ilen	= 31,
+		.result	= "\xe3\x93\x15\xae\x5f\x9d\x3c\xb5"
+			  "\xd6\x9d\xee\xee\xcf\xaa\xaf\xe1"
+			  "\x45\x10\x96\xe0\xbf\x55\x0f\x4c"
+			  "\x1a\xfd\xf4\xda\x4e\x10\xde\xc9"
+			  "\x0e\x6f\xc7\x3c\x49\x94\x41\xfc"
+			  "\x59\x28\x88\x3c\x79\x10\x6b",
+		.rlen	= 47,
+	}, {
+		.key	= "\x7a\xa5\xe8\x23\xa4\x84\x9e\x2d"
+			  "\x25\x53\x58\x8c\xda\xa3\xc0\xa6",
+		.klen	= 16,
+		.iv	= "\x89\x6e\x77\x8b\xe8\x23\x49\xb4"
+			  "\x45\x8a\x85\xb1\xd8\x6c\xf1\x28",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xa7\x00\x93\x5b\x70\x61\x9d\xc2"
+			  "\x86\xf7\xde\xfa\xd5\xfe\x52\x2b"
+			  "\x28\x50\x51\x9d\x24\x60\x8d\xb3"
+			  "\x49\x3e\x17\xea\xf6\x99\x5a\xdd",
+		.ilen	= 32,
+		.result	= "\x1c\x8e\x22\x34\xfd\xab\xe6\x0d"
+			  "\x1c\x9f\x06\x54\x8b\x0b\xb4\x40"
+			  "\xde\x11\x59\x3e\xfd\x74\xf6\x42"
+			  "\x97\x17\xf7\x24\xb6\x7e\xc4\xc6"
+			  "\x06\xa3\x94\xda\x3d\x7f\x55\x0a"
+			  "\x92\x07\x2f\xa6\xf3\x6b\x2c\xfc",
+		.rlen	= 48,
+	}, {
+		.key	= "\xb6\xca\x22\xc3\xb4\x00\x47\x49"
+			  "\xa6\x2d\x0a\x1e\xd4\xc7\x83\xad",
+		.klen	= 16,
+		.iv	= "\xc5\x93\xb0\x2a\xf8\x9f\xf1\xd0"
+			  "\xc6\x64\x37\x42\xd2\x90\xb3\x2e",
+		.assoc	= "\xd5",
+		.alen	= 1,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xa0\x2a\xb4\x9a\x91\x00\x15\xb8"
+			  "\x0f\x9a\x15\x60\x0e\x9b\x13\x8f",
+		.rlen	= 16,
+	}, {
+		.key	= "\xf3\xee\x5c\x62\xc4\x7c\xf0\x65"
+			  "\x27\x08\xbd\xaf\xce\xec\x45\xb3",
+		.klen	= 16,
+		.iv	= "\x02\xb8\xea\xca\x09\x1b\x9a\xec"
+			  "\x47\x3e\xe9\xd4\xcc\xb5\x76\x34",
+		.assoc	= "\x11\x81\x78\x32\x4d\xb9\x44\x73"
+			  "\x68\x75\x16\xf8\xcb\x7e\xa7",
+		.alen	= 15,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x4c\x26\xad\x9c\x14\xfd\x9c\x8c"
+			  "\x84\xfb\x26\xfb\xd5\xca\x62\x39",
+		.rlen	= 16,
+	}, {
+		.key	= "\x2f\x13\x95\x01\xd5\xf7\x99\x81"
+			  "\xa8\xe2\x6f\x41\xc8\x10\x08\xb9",
+		.klen	= 16,
+		.iv	= "\x3f\xdc\x24\x69\x19\x96\x43\x08"
+			  "\xc8\x18\x9b\x65\xc6\xd9\x39\x3b",
+		.assoc	= "\x4e\xa5\xb2\xd1\x5d\x35\xed\x8f"
+			  "\xe8\x4f\xc8\x89\xc5\xa2\x69\xbc",
+		.alen	= 16,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x45\x85\x0e\x0f\xf4\xae\x96\xa1"
+			  "\x99\x4d\x6d\xb4\x67\x32\xb0\x3a",
+		.rlen	= 16,
+	}, {
+		.key	= "\x6c\x38\xcf\xa1\xe5\x73\x41\x9d"
+			  "\x29\xbc\x21\xd2\xc2\x35\xcb\xbf",
+		.klen	= 16,
+		.iv	= "\x7b\x01\x5d\x08\x29\x12\xec\x24"
+			  "\x49\xf3\x4d\xf7\xc0\xfe\xfb\x41",
+		.assoc	= "\x8a\xca\xec\x70\x6d\xb1\x96\xab"
+			  "\x69\x29\x7a\x1b\xbf\xc7\x2c\xc2"
+			  "\x07",
+		.alen	= 17,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x33\xb1\x42\x97\x8e\x16\x7b\x63"
+			  "\x06\xba\x5b\xcb\xae\x6d\x8b\x56",
+		.rlen	= 16,
+	}, {
+		.key	= "\xa8\x5c\x09\x40\xf5\xef\xea\xb8"
+			  "\xaa\x96\xd3\x64\xbc\x59\x8d\xc6",
+		.klen	= 16,
+		.iv	= "\xb8\x26\x97\xa8\x39\x8e\x94\x3f"
+			  "\xca\xcd\xff\x88\xba\x22\xbe\x47",
+		.assoc	= "\xc7\xef\x26\x10\x7d\x2c\x3f\xc6"
+			  "\xea\x03\x2c\xac\xb9\xeb\xef\xc9"
+			  "\x31\x6b\x08\x12\xfc\xd8\x37\x2d"
+			  "\xe0\x17\x3a\x2e\x83\x5c\x8f",
+		.alen	= 31,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xda\x44\x08\x8c\x2a\xa5\x07\x35"
+			  "\x0b\x54\x4e\x6d\xe3\xfd\xc4\x5f",
+		.rlen	= 16,
+	}, {
+		.key	= "\xe5\x81\x42\xdf\x05\x6a\x93\xd4"
+			  "\x2b\x70\x85\xf5\xb6\x7d\x50\xcc",
+		.klen	= 16,
+		.iv	= "\xf4\x4a\xd1\x47\x49\x09\x3d\x5b"
+			  "\x4b\xa7\xb1\x19\xb4\x46\x81\x4d",
+		.assoc	= "\x03\x14\x5f\xaf\x8d\xa8\xe7\xe2"
+			  "\x6b\xde\xde\x3e\xb3\x10\xb1\xcf"
+			  "\x5c\x2d\x14\x96\x01\x78\xb9\x47"
+			  "\xa1\x44\x19\x06\x5d\xbb\x2e\x2f",
+		.alen	= 32,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x1b\xb1\xf1\xa8\x9e\xc2\xb2\x88"
+			  "\x40\x7f\x7b\x19\x7a\x52\x8c\xf0",
+		.rlen	= 16,
+	}, {
+		.key	= "\x22\xa6\x7c\x7f\x15\xe6\x3c\xf0"
+			  "\xac\x4b\x37\x86\xb0\xa2\x13\xd2",
+		.klen	= 16,
+		.iv	= "\x31\x6f\x0b\xe6\x59\x85\xe6\x77"
+			  "\xcc\x81\x63\xab\xae\x6b\x43\x54",
+		.assoc	= "\x40",
+		.alen	= 1,
+		.input	= "\x4f",
+		.ilen	= 1,
+		.result	= "\x6e\xc8\xfb\x15\x9d\x98\x49\xc9"
+			  "\xa0\x98\x09\x85\xbe\x56\x8e\x79"
+			  "\xf4",
+		.rlen	= 17,
+	}, {
+		.key	= "\x5e\xcb\xb6\x1e\x25\x62\xe4\x0c"
+			  "\x2d\x25\xe9\x18\xaa\xc6\xd5\xd8",
+		.klen	= 16,
+		.iv	= "\x6d\x94\x44\x86\x69\x00\x8f\x93"
+			  "\x4d\x5b\x15\x3c\xa8\x8f\x06\x5a",
+		.assoc	= "\x7c\x5d\xd3\xee\xad\x9f\x39\x1a"
+			  "\x6d\x92\x42\x61\xa7\x58\x37",
+		.alen	= 15,
+		.input	= "\x8b\x26\x61\x55\xf1\x3e\xe3\xa1"
+			  "\x8d\xc8\x6e\x85\xa5\x21\x67",
+		.ilen	= 15,
+		.result	= "\x99\x2e\x84\x50\x64\x5c\xab\x29"
+			  "\x20\xba\xb9\x2f\x62\x3a\xce\x2a"
+			  "\x75\x25\x3b\xe3\x40\xe0\x1d\xfc"
+			  "\x20\x63\x0b\x49\x7e\x97\x08",
+		.rlen	= 31,
+	}, {
+		.key	= "\x9b\xef\xf0\xbd\x35\xdd\x8d\x28"
+			  "\xad\xff\x9b\xa9\xa4\xeb\x98\xdf",
+		.klen	= 16,
+		.iv	= "\xaa\xb8\x7e\x25\x79\x7c\x37\xaf"
+			  "\xce\x36\xc7\xce\xa2\xb4\xc9\x60",
+		.assoc	= "\xb9\x82\x0c\x8d\xbd\x1b\xe2\x36"
+			  "\xee\x6c\xf4\xf2\xa1\x7d\xf9\xe2",
+		.alen	= 16,
+		.input	= "\xc8\x4b\x9b\xf5\x01\xba\x8c\xbd"
+			  "\x0e\xa3\x21\x16\x9f\x46\x2a\x63",
+		.ilen	= 16,
+		.result	= "\xd9\x8e\xfd\x50\x8f\x02\x9f\xee"
+			  "\x78\x08\x12\xec\x09\xaf\x53\x14"
+			  "\x90\x3e\x3d\x76\xad\x71\x21\x08"
+			  "\x77\xe5\x4b\x15\xc2\xe6\xbc\xdb",
+		.rlen	= 32,
+	}, {
+		.key	= "\xd7\x14\x29\x5d\x45\x59\x36\x44"
+			  "\x2e\xd9\x4d\x3b\x9e\x0f\x5b\xe5",
+		.klen	= 16,
+		.iv	= "\xe6\xdd\xb8\xc4\x89\xf8\xe0\xca"
+			  "\x4f\x10\x7a\x5f\x9c\xd8\x8b\x66",
+		.assoc	= "\xf5\xa6\x46\x2c\xce\x97\x8a\x51"
+			  "\x6f\x46\xa6\x83\x9b\xa1\xbc\xe8"
+			  "\x05",
+		.alen	= 17,
+		.input	= "\x05\x70\xd5\x94\x12\x36\x35\xd8"
+			  "\x8f\x7d\xd3\xa8\x99\x6a\xed\x69"
+			  "\xd0",
+		.ilen	= 17,
+		.result	= "\xf3\xe7\x95\x86\xcf\x34\x95\x96"
+			  "\x17\xfe\x1b\xae\x1b\x31\xf2\x1a"
+			  "\xbd\xbc\xc9\x4e\x11\x29\x09\x5c"
+			  "\x05\xd3\xb4\x2e\x4a\x74\x59\x49"
+			  "\x7d",
+		.rlen	= 33,
+	}, {
+		.key	= "\x14\x39\x63\xfc\x56\xd5\xdf\x5f"
+			  "\xaf\xb3\xff\xcc\x98\x33\x1d\xeb",
+		.klen	= 16,
+		.iv	= "\x23\x02\xf1\x64\x9a\x73\x89\xe6"
+			  "\xd0\xea\x2c\xf1\x96\xfc\x4e\x6d",
+		.assoc	= "\x32\xcb\x80\xcc\xde\x12\x33\x6d"
+			  "\xf0\x20\x58\x15\x95\xc6\x7f\xee"
+			  "\x2f\xf9\x4e\x2c\x1b\x98\x43\xc7"
+			  "\x68\x28\x73\x40\x9f\x96\x4a",
+		.alen	= 31,
+		.input	= "\x41\x94\x0e\x33\x22\xb1\xdd\xf4"
+			  "\x10\x57\x85\x39\x93\x8f\xaf\x70"
+			  "\xfa\xa9\xd0\x4d\x5c\x40\x23\xcd"
+			  "\x98\x34\xab\x37\x56\xae\x32",
+		.ilen	= 31,
+		.result	= "\x06\x96\xb2\xbf\x63\xf4\x1e\x24"
+			  "\x0d\x19\x15\x61\x65\x3b\x06\x26"
+			  "\x71\xe8\x7e\x16\xdb\x96\x01\x01"
+			  "\x52\xcd\x49\x5b\x07\x33\x4e\xe7"
+			  "\xaa\x91\xf5\xd5\xc6\xfe\x41\xb5"
+			  "\xed\x90\xce\xb9\xcd\xcc\xa1",
+		.rlen	= 47,
+	}, {
+		.key	= "\x50\x5d\x9d\x9b\x66\x50\x88\x7b"
+			  "\x30\x8e\xb1\x5e\x92\x58\xe0\xf1",
+		.klen	= 16,
+		.iv	= "\x5f\x27\x2b\x03\xaa\xef\x32\x02"
+			  "\x50\xc4\xde\x82\x90\x21\x11\x73",
+		.assoc	= "\x6e\xf0\xba\x6b\xee\x8e\xdc\x89"
+			  "\x71\xfb\x0a\xa6\x8f\xea\x41\xf4"
+			  "\x5a\xbb\x59\xb0\x20\x38\xc5\xe0"
+			  "\x29\x56\x52\x19\x79\xf5\xe9\x37",
+		.alen	= 32,
+		.input	= "\x7e\xb9\x48\xd3\x32\x2d\x86\x10"
+			  "\x91\x31\x37\xcb\x8d\xb3\x72\x76"
+			  "\x24\x6b\xdc\xd1\x61\xe0\xa5\xe7"
+			  "\x5a\x61\x8a\x0f\x30\x0d\xd1\xec",
+		.ilen	= 32,
+		.result	= "\xf9\xd7\xee\x17\xfd\x24\xcd\xf1"
+			  "\xbc\x0f\x35\x97\x97\x0c\x4b\x18"
+			  "\xce\x58\xc8\x3b\xd4\x85\x93\x79"
+			  "\xcc\x9c\xea\xc1\x73\x13\x0b\x4c"
+			  "\xcc\x6f\x28\xf8\xa4\x4e\xb8\x56"
+			  "\x64\x4e\x47\xce\xb2\xb4\x92\xb4",
+		.rlen	= 48,
+	}, {
+		.key	= "\x8d\x82\xd6\x3b\x76\xcc\x30\x97"
+			  "\xb1\x68\x63\xef\x8c\x7c\xa3\xf7",
+		.klen	= 16,
+		.iv	= "\x9c\x4b\x65\xa2\xba\x6b\xdb\x1e"
+			  "\xd1\x9e\x90\x13\x8a\x45\xd3\x79",
+		.assoc	= "\xab\x14\xf3\x0a\xfe\x0a\x85\xa5"
+			  "\xf2\xd5\xbc\x38\x89\x0e\x04\xfb"
+			  "\x84\x7d\x65\x34\x25\xd8\x47\xfa"
+			  "\xeb\x83\x31\xf1\x54\x54\x89\x0d"
+			  "\x9d",
+		.alen	= 33,
+		.input	= "\xba\xde\x82\x72\x42\xa9\x2f\x2c"
+			  "\x12\x0b\xe9\x5c\x87\xd7\x35\x7c"
+			  "\x4f\x2e\xe8\x55\x66\x80\x27\x00"
+			  "\x1b\x8f\x68\xe7\x0a\x6c\x71\xc3"
+			  "\x21\x78\x55\x9d\x9c\x65\x7b\xcd"
+			  "\x0a\x34\x97\xff\x47\x37\xb0\x2a"
+			  "\x80\x0d\x19\x98\x33\xa9\x7a\xe3"
+			  "\x2e\x4c\xc6\xf3\x8c\x88\x42\x01"
+			  "\xbd",
+		.ilen	= 65,
+		.result	= "\x58\xfa\x3a\x3d\xd9\x88\x63\xe8"
+			  "\xc5\x78\x50\x8b\x4a\xc9\xdf\x7f"
+			  "\x4b\xfa\xc8\x2e\x67\x43\xf3\x63"
+			  "\x42\x8e\x99\x5a\x9c\x0b\x84\x77"
+			  "\xbc\x46\x76\x48\x82\xc7\x57\x96"
+			  "\xe1\x65\xd1\xed\x1d\xdd\x80\x24"
+			  "\xa6\x4d\xa9\xf1\x53\x8b\x5e\x0e"
+			  "\x26\xb9\xcc\x37\xe5\x43\xe1\x5a"
+			  "\x8a\xd6\x8c\x5a\xe4\x95\xd1\x8d"
+			  "\xf7\x33\x64\xc1\xd3\xf2\xfc\x35"
+			  "\x01",
+		.rlen	= 81,
+	}, {
+		.key	= "\xc9\xa7\x10\xda\x86\x48\xd9\xb3"
+			  "\x32\x42\x15\x80\x85\xa1\x65\xfe",
+		.klen	= 16,
+		.iv	= "\xd8\x70\x9f\x42\xca\xe6\x83\x3a"
+			  "\x52\x79\x42\xa5\x84\x6a\x96\x7f",
+		.assoc	= "\xe8\x39\x2d\xaa\x0e\x85\x2d\xc1"
+			  "\x72\xaf\x6e\xc9\x82\x33\xc7\x01"
+			  "\xaf\x40\x70\xb8\x2a\x78\xc9\x14"
+			  "\xac\xb1\x10\xca\x2e\xb3\x28\xe4"
+			  "\xac\xfa\x58\x7f\xe5\x73\x09\x8c"
+			  "\x1d\x40\x87\x8c\xd9\x75\xc0\x55"
+			  "\xa2\xda\x07\xd1\xc2\xa9\xd1\xbb"
+			  "\x09\x4f\x77\x62\x88\x2d\xf2\x68"
+			  "\x54",
+		.alen	= 65,
+		.input	= "\xf7\x02\xbb\x11\x52\x24\xd8\x48"
+			  "\x93\xe6\x9b\xee\x81\xfc\xf7\x82"
+			  "\x79\xf0\xf3\xd9\x6c\x20\xa9\x1a"
+			  "\xdc\xbc\x47\xc0\xe4\xcb\x10\x99"
+			  "\x2f",
+		.ilen	= 33,
+		.result	= "\x4c\xa9\xac\x71\xed\x10\xa6\x24"
+			  "\xb7\xa7\xdf\x8b\xf5\xc2\x41\xcb"
+			  "\x05\xc9\xd6\x97\xb6\x10\x7f\x17"
+			  "\xc2\xc0\x93\xcf\xe0\x94\xfd\x99"
+			  "\xf2\x62\x25\x28\x01\x23\x6f\x8b"
+			  "\x04\x52\xbc\xb0\x3e\x66\x52\x90"
+			  "\x9f",
+		.rlen	= 49,
+	}, {
+		.key	= "\x06\xcc\x4a\x79\x96\xc3\x82\xcf"
+			  "\xb3\x1c\xc7\x12\x7f\xc5\x28\x04",
+		.klen	= 16,
+		.iv	= "\x15\x95\xd8\xe1\xda\x62\x2c\x56"
+			  "\xd3\x53\xf4\x36\x7e\x8e\x59\x85",
+		.assoc	= "\x24\x5e\x67\x49\x1e\x01\xd6\xdd"
+			  "\xf3\x89\x20\x5b\x7c\x57\x89\x07",
+		.alen	= 16,
+		.input	= "\x33\x27\xf5\xb1\x62\xa0\x80\x63"
+			  "\x14\xc0\x4d\x7f\x7b\x20\xba\x89",
+		.ilen	= 16,
+		.result	= "\x6d\xed\x04\x7a\x2f\x0c\x30\xa5"
+			  "\x96\xe6\x97\xe4\x10\xeb\x40\x95"
+			  "\xc5\x9a\xdf\x31\xd5\xa5\xa6\xec"
+			  "\x05\xa8\x31\x50\x11\x19\x44",
+		.rlen	= 31,
+	}, {
+		.key	= "\x42\xf0\x84\x19\xa6\x3f\x2b\xea"
+			  "\x34\xf6\x79\xa3\x79\xe9\xeb\x0a",
+		.klen	= 16,
+		.iv	= "\x51\xb9\x12\x80\xea\xde\xd5\x71"
+			  "\x54\x2d\xa6\xc8\x78\xb2\x1b\x8c",
+		.assoc	= "\x61\x83\xa0\xe8\x2e\x7d\x7f\xf8"
+			  "\x74\x63\xd2\xec\x76\x7c\x4c\x0d",
+		.alen	= 16,
+		.input	= "\x70\x4c\x2f\x50\x72\x1c\x29\x7f"
+			  "\x95\x9a\xff\x10\x75\x45\x7d\x8f",
+		.ilen	= 16,
+		.result	= "\x30\x95\x7d\xea\xdc\x62\xc0\x88"
+			  "\xa1\xe3\x8d\x8c\xac\x04\x10\xa7"
+			  "\xfa\xfa\x07\xbd\xa0\xf0\x36\xeb"
+			  "\x21\x93\x2e\x31\x84\x83",
+		.rlen	= 30,
+	}, {
+		.key	= "\x7f\x15\xbd\xb8\xb6\xba\xd3\x06"
+			  "\xb5\xd1\x2b\x35\x73\x0e\xad\x10",
+		.klen	= 16,
+		.iv	= "\x8e\xde\x4c\x20\xfa\x59\x7e\x8d"
+			  "\xd5\x07\x58\x59\x72\xd7\xde\x92",
+		.assoc	= "\x9d\xa7\xda\x88\x3e\xf8\x28\x14"
+			  "\xf5\x3e\x85\x7d\x70\xa0\x0f\x13",
+		.alen	= 16,
+		.input	= "\xac\x70\x69\xef\x82\x97\xd2\x9b"
+			  "\x15\x74\xb1\xa2\x6f\x69\x3f\x95",
+		.ilen	= 16,
+		.result	= "\x93\xcd\xee\xd4\xcb\x9d\x8d\x16"
+			  "\x63\x0d\x43\xd5\x49\xca\xa8\x85"
+			  "\x49\xc0\xae\x13\xbc\x26\x1d\x4b",
+		.rlen	= 24,
 	},
 };
 
-static const struct cipher_testvec camellia_cbc_dec_tv_template[] = {
+static const struct aead_testvec aegis128l_dec_tv_template[] = {
 	{
-		.key    = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
-			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
-		.klen   = 16,
-		.iv	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.input	= "\xea\x32\x12\x76\x3b\x50\x10\xe7"
-			  "\x18\xf6\xfd\x5d\xf6\x8f\x13\x51",
-		.ilen   = 16,
-		.result = "Single block msg",
-		.rlen   = 16,
+		.key	= "\x0f\xc9\x8e\x67\x44\x9e\xaa\x86"
+			  "\x20\x36\x2c\x24\xfe\xc9\x30\x81",
+		.klen	= 16,
+		.iv	= "\x1e\x92\x1c\xcf\x88\x3d\x54\x0d"
+			  "\x40\x6d\x59\x48\xfc\x92\x61\x03",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x30\x4f\xf3\xe9\xb1\xfa\x81\xa6"
+			  "\x20\x72\x78\xdd\x93\xc8\x57\xef",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
 	}, {
-		.key    = "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
-			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
-		.klen   = 16,
-		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.input = "\xa5\xdf\x6e\x50\xda\x70\x6c\x01"
-			  "\x4a\xab\xf3\xf2\xd6\xfc\x6c\xfd"
-			  "\x19\xb4\x3e\x57\x1c\x02\x5e\xa0"
-			  "\x15\x78\xe0\x5e\xf2\xcb\x87\x16",
-		.ilen   = 32,
-		.result = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
-		.rlen   = 32,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.key	= "\x4b\xed\xc8\x07\x54\x1a\x52\xa2"
+			  "\xa1\x10\xde\xb5\xf8\xed\xf3\x87",
+		.klen	= 16,
+		.iv	= "\x5a\xb7\x56\x6e\x98\xb9\xfd\x29"
+			  "\xc1\x47\x0b\xda\xf6\xb6\x23\x09",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xa9\x24\xa0\xb6\x2d\xdd\x29\xdb"
+			  "\x40\xb3\x71\xc5\x22\x58\x31\x77"
+			  "\x6d",
+		.ilen	= 17,
+		.result	= "\x79",
+		.rlen	= 1,
+	}, {
+		.key	= "\x88\x12\x01\xa6\x64\x96\xfb\xbe"
+			  "\x22\xea\x90\x47\xf2\x11\xb5\x8e",
+		.klen	= 16,
+		.iv	= "\x97\xdb\x90\x0e\xa8\x35\xa5\x45"
+			  "\x42\x21\xbd\x6b\xf0\xda\xe6\x0f",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xbb\x0a\x53\xc4\xaa\x7e\xa4\x03"
+			  "\x2b\xee\x62\x99\x7b\x98\x13\x1f"
+			  "\xe0\x76\x4c\x2e\x53\x99\x4f\xbe"
+			  "\xe1\xa8\x04\x7f\xe1\x71\xbe",
+		.ilen	= 31,
+		.result	= "\xb5\x6e\xad\xdd\x30\x72\xfa\x53"
+			  "\x82\x8e\x16\xb4\xed\x6d\x47",
+		.rlen	= 15,
+	}, {
+		.key	= "\xc4\x37\x3b\x45\x74\x11\xa4\xda"
+			  "\xa2\xc5\x42\xd8\xec\x36\x78\x94",
+		.klen	= 16,
+		.iv	= "\xd3\x00\xc9\xad\xb8\xb0\x4e\x61"
+			  "\xc3\xfb\x6f\xfd\xea\xff\xa9\x15",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x66\xdf\x6e\x71\xc0\x6e\xa4\x4c"
+			  "\x9d\xb7\x8c\x9a\xdb\x1f\xd2\x2e"
+			  "\x23\xb6\xa4\xfb\xd3\x86\xdd\xbb"
+			  "\xde\x54\x9b\xf5\x92\x8b\x93\xc5",
+		.ilen	= 32,
+		.result	= "\xf2\x92\xe6\x7d\x40\xee\xa3\x6f"
+			  "\x03\x68\xc8\x45\xe7\x91\x0a\x18",
+		.rlen	= 16,
+	}, {
+		.key	= "\x01\x5c\x75\xe5\x84\x8d\x4d\xf6"
+			  "\x23\x9f\xf4\x6a\xe6\x5a\x3b\x9a",
+		.klen	= 16,
+		.iv	= "\x10\x25\x03\x4c\xc8\x2c\xf7\x7d"
+			  "\x44\xd5\x21\x8e\xe4\x23\x6b\x1c",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x4f\xc3\x69\xb6\xd3\xa4\x64\x8b"
+			  "\x71\xc3\x8a\x91\x22\x4f\x1b\xd2"
+			  "\x33\x6d\x86\xbc\xf8\x2f\x06\xf9"
+			  "\x82\x64\xc7\x72\x00\x30\xfc\xf0"
+			  "\xf8",
+		.ilen	= 33,
+		.result	= "\x2e\xb7\x20\x1c\x50\x6a\x4b\x8b"
+			  "\x84\x42\x7a\xd7\xe1\xb5\xcd\x1f"
+			  "\xd3",
+		.rlen	= 17,
+	}, {
+		.key	= "\x3d\x80\xae\x84\x94\x09\xf6\x12"
+			  "\xa4\x79\xa6\xfb\xe0\x7f\xfd\xa0",
+		.klen	= 16,
+		.iv	= "\x4c\x49\x3d\xec\xd8\xa8\xa0\x98"
+			  "\xc5\xb0\xd3\x1f\xde\x48\x2e\x22",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xe3\x93\x15\xae\x5f\x9d\x3c\xb5"
+			  "\xd6\x9d\xee\xee\xcf\xaa\xaf\xe1"
+			  "\x45\x10\x96\xe0\xbf\x55\x0f\x4c"
+			  "\x1a\xfd\xf4\xda\x4e\x10\xde\xc9"
+			  "\x0e\x6f\xc7\x3c\x49\x94\x41\xfc"
+			  "\x59\x28\x88\x3c\x79\x10\x6b",
+		.ilen	= 47,
+		.result	= "\x6b\xdc\x5a\xbb\x60\xe5\xf4\xa6"
+			  "\x05\x1d\x2c\x68\xdb\xda\x8f\x25"
+			  "\xfe\x8d\x45\x19\x1e\xc0\x0b\x99"
+			  "\x88\x11\x39\x12\x1c\x3a\xbb",
+		.rlen	= 31,
+	}, {
+		.key	= "\x7a\xa5\xe8\x23\xa4\x84\x9e\x2d"
+			  "\x25\x53\x58\x8c\xda\xa3\xc0\xa6",
+		.klen	= 16,
+		.iv	= "\x89\x6e\x77\x8b\xe8\x23\x49\xb4"
+			  "\x45\x8a\x85\xb1\xd8\x6c\xf1\x28",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x1c\x8e\x22\x34\xfd\xab\xe6\x0d"
+			  "\x1c\x9f\x06\x54\x8b\x0b\xb4\x40"
+			  "\xde\x11\x59\x3e\xfd\x74\xf6\x42"
+			  "\x97\x17\xf7\x24\xb6\x7e\xc4\xc6"
+			  "\x06\xa3\x94\xda\x3d\x7f\x55\x0a"
+			  "\x92\x07\x2f\xa6\xf3\x6b\x2c\xfc",
+		.ilen	= 48,
+		.result	= "\xa7\x00\x93\x5b\x70\x61\x9d\xc2"
+			  "\x86\xf7\xde\xfa\xd5\xfe\x52\x2b"
+			  "\x28\x50\x51\x9d\x24\x60\x8d\xb3"
+			  "\x49\x3e\x17\xea\xf6\x99\x5a\xdd",
+		.rlen	= 32,
+	}, {
+		.key	= "\xb6\xca\x22\xc3\xb4\x00\x47\x49"
+			  "\xa6\x2d\x0a\x1e\xd4\xc7\x83\xad",
+		.klen	= 16,
+		.iv	= "\xc5\x93\xb0\x2a\xf8\x9f\xf1\xd0"
+			  "\xc6\x64\x37\x42\xd2\x90\xb3\x2e",
+		.assoc	= "\xd5",
+		.alen	= 1,
+		.input	= "\xa0\x2a\xb4\x9a\x91\x00\x15\xb8"
+			  "\x0f\x9a\x15\x60\x0e\x9b\x13\x8f",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xf3\xee\x5c\x62\xc4\x7c\xf0\x65"
+			  "\x27\x08\xbd\xaf\xce\xec\x45\xb3",
+		.klen	= 16,
+		.iv	= "\x02\xb8\xea\xca\x09\x1b\x9a\xec"
+			  "\x47\x3e\xe9\xd4\xcc\xb5\x76\x34",
+		.assoc	= "\x11\x81\x78\x32\x4d\xb9\x44\x73"
+			  "\x68\x75\x16\xf8\xcb\x7e\xa7",
+		.alen	= 15,
+		.input	= "\x4c\x26\xad\x9c\x14\xfd\x9c\x8c"
+			  "\x84\xfb\x26\xfb\xd5\xca\x62\x39",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x2f\x13\x95\x01\xd5\xf7\x99\x81"
+			  "\xa8\xe2\x6f\x41\xc8\x10\x08\xb9",
+		.klen	= 16,
+		.iv	= "\x3f\xdc\x24\x69\x19\x96\x43\x08"
+			  "\xc8\x18\x9b\x65\xc6\xd9\x39\x3b",
+		.assoc	= "\x4e\xa5\xb2\xd1\x5d\x35\xed\x8f"
+			  "\xe8\x4f\xc8\x89\xc5\xa2\x69\xbc",
+		.alen	= 16,
+		.input	= "\x45\x85\x0e\x0f\xf4\xae\x96\xa1"
+			  "\x99\x4d\x6d\xb4\x67\x32\xb0\x3a",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x6c\x38\xcf\xa1\xe5\x73\x41\x9d"
+			  "\x29\xbc\x21\xd2\xc2\x35\xcb\xbf",
+		.klen	= 16,
+		.iv	= "\x7b\x01\x5d\x08\x29\x12\xec\x24"
+			  "\x49\xf3\x4d\xf7\xc0\xfe\xfb\x41",
+		.assoc	= "\x8a\xca\xec\x70\x6d\xb1\x96\xab"
+			  "\x69\x29\x7a\x1b\xbf\xc7\x2c\xc2"
+			  "\x07",
+		.alen	= 17,
+		.input	= "\x33\xb1\x42\x97\x8e\x16\x7b\x63"
+			  "\x06\xba\x5b\xcb\xae\x6d\x8b\x56",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xa8\x5c\x09\x40\xf5\xef\xea\xb8"
+			  "\xaa\x96\xd3\x64\xbc\x59\x8d\xc6",
+		.klen	= 16,
+		.iv	= "\xb8\x26\x97\xa8\x39\x8e\x94\x3f"
+			  "\xca\xcd\xff\x88\xba\x22\xbe\x47",
+		.assoc	= "\xc7\xef\x26\x10\x7d\x2c\x3f\xc6"
+			  "\xea\x03\x2c\xac\xb9\xeb\xef\xc9"
+			  "\x31\x6b\x08\x12\xfc\xd8\x37\x2d"
+			  "\xe0\x17\x3a\x2e\x83\x5c\x8f",
+		.alen	= 31,
+		.input	= "\xda\x44\x08\x8c\x2a\xa5\x07\x35"
+			  "\x0b\x54\x4e\x6d\xe3\xfd\xc4\x5f",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xe5\x81\x42\xdf\x05\x6a\x93\xd4"
+			  "\x2b\x70\x85\xf5\xb6\x7d\x50\xcc",
+		.klen	= 16,
+		.iv	= "\xf4\x4a\xd1\x47\x49\x09\x3d\x5b"
+			  "\x4b\xa7\xb1\x19\xb4\x46\x81\x4d",
+		.assoc	= "\x03\x14\x5f\xaf\x8d\xa8\xe7\xe2"
+			  "\x6b\xde\xde\x3e\xb3\x10\xb1\xcf"
+			  "\x5c\x2d\x14\x96\x01\x78\xb9\x47"
+			  "\xa1\x44\x19\x06\x5d\xbb\x2e\x2f",
+		.alen	= 32,
+		.input	= "\x1b\xb1\xf1\xa8\x9e\xc2\xb2\x88"
+			  "\x40\x7f\x7b\x19\x7a\x52\x8c\xf0",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x22\xa6\x7c\x7f\x15\xe6\x3c\xf0"
+			  "\xac\x4b\x37\x86\xb0\xa2\x13\xd2",
+		.klen	= 16,
+		.iv	= "\x31\x6f\x0b\xe6\x59\x85\xe6\x77"
+			  "\xcc\x81\x63\xab\xae\x6b\x43\x54",
+		.assoc	= "\x40",
+		.alen	= 1,
+		.input	= "\x6e\xc8\xfb\x15\x9d\x98\x49\xc9"
+			  "\xa0\x98\x09\x85\xbe\x56\x8e\x79"
+			  "\xf4",
+		.ilen	= 17,
+		.result	= "\x4f",
+		.rlen	= 1,
+	}, {
+		.key	= "\x5e\xcb\xb6\x1e\x25\x62\xe4\x0c"
+			  "\x2d\x25\xe9\x18\xaa\xc6\xd5\xd8",
+		.klen	= 16,
+		.iv	= "\x6d\x94\x44\x86\x69\x00\x8f\x93"
+			  "\x4d\x5b\x15\x3c\xa8\x8f\x06\x5a",
+		.assoc	= "\x7c\x5d\xd3\xee\xad\x9f\x39\x1a"
+			  "\x6d\x92\x42\x61\xa7\x58\x37",
+		.alen	= 15,
+		.input	= "\x99\x2e\x84\x50\x64\x5c\xab\x29"
+			  "\x20\xba\xb9\x2f\x62\x3a\xce\x2a"
+			  "\x75\x25\x3b\xe3\x40\xe0\x1d\xfc"
+			  "\x20\x63\x0b\x49\x7e\x97\x08",
+		.ilen	= 31,
+		.result	= "\x8b\x26\x61\x55\xf1\x3e\xe3\xa1"
+			  "\x8d\xc8\x6e\x85\xa5\x21\x67",
+		.rlen	= 15,
+	}, {
+		.key	= "\x9b\xef\xf0\xbd\x35\xdd\x8d\x28"
+			  "\xad\xff\x9b\xa9\xa4\xeb\x98\xdf",
+		.klen	= 16,
+		.iv	= "\xaa\xb8\x7e\x25\x79\x7c\x37\xaf"
+			  "\xce\x36\xc7\xce\xa2\xb4\xc9\x60",
+		.assoc	= "\xb9\x82\x0c\x8d\xbd\x1b\xe2\x36"
+			  "\xee\x6c\xf4\xf2\xa1\x7d\xf9\xe2",
+		.alen	= 16,
+		.input	= "\xd9\x8e\xfd\x50\x8f\x02\x9f\xee"
+			  "\x78\x08\x12\xec\x09\xaf\x53\x14"
+			  "\x90\x3e\x3d\x76\xad\x71\x21\x08"
+			  "\x77\xe5\x4b\x15\xc2\xe6\xbc\xdb",
+		.ilen	= 32,
+		.result	= "\xc8\x4b\x9b\xf5\x01\xba\x8c\xbd"
+			  "\x0e\xa3\x21\x16\x9f\x46\x2a\x63",
+		.rlen	= 16,
+	}, {
+		.key	= "\xd7\x14\x29\x5d\x45\x59\x36\x44"
+			  "\x2e\xd9\x4d\x3b\x9e\x0f\x5b\xe5",
+		.klen	= 16,
+		.iv	= "\xe6\xdd\xb8\xc4\x89\xf8\xe0\xca"
+			  "\x4f\x10\x7a\x5f\x9c\xd8\x8b\x66",
+		.assoc	= "\xf5\xa6\x46\x2c\xce\x97\x8a\x51"
+			  "\x6f\x46\xa6\x83\x9b\xa1\xbc\xe8"
+			  "\x05",
+		.alen	= 17,
+		.input	= "\xf3\xe7\x95\x86\xcf\x34\x95\x96"
+			  "\x17\xfe\x1b\xae\x1b\x31\xf2\x1a"
+			  "\xbd\xbc\xc9\x4e\x11\x29\x09\x5c"
+			  "\x05\xd3\xb4\x2e\x4a\x74\x59\x49"
+			  "\x7d",
+		.ilen	= 33,
+		.result	= "\x05\x70\xd5\x94\x12\x36\x35\xd8"
+			  "\x8f\x7d\xd3\xa8\x99\x6a\xed\x69"
+			  "\xd0",
+		.rlen	= 17,
+	}, {
+		.key	= "\x14\x39\x63\xfc\x56\xd5\xdf\x5f"
+			  "\xaf\xb3\xff\xcc\x98\x33\x1d\xeb",
+		.klen	= 16,
+		.iv	= "\x23\x02\xf1\x64\x9a\x73\x89\xe6"
+			  "\xd0\xea\x2c\xf1\x96\xfc\x4e\x6d",
+		.assoc	= "\x32\xcb\x80\xcc\xde\x12\x33\x6d"
+			  "\xf0\x20\x58\x15\x95\xc6\x7f\xee"
+			  "\x2f\xf9\x4e\x2c\x1b\x98\x43\xc7"
+			  "\x68\x28\x73\x40\x9f\x96\x4a",
+		.alen	= 31,
+		.input	= "\x06\x96\xb2\xbf\x63\xf4\x1e\x24"
+			  "\x0d\x19\x15\x61\x65\x3b\x06\x26"
+			  "\x71\xe8\x7e\x16\xdb\x96\x01\x01"
+			  "\x52\xcd\x49\x5b\x07\x33\x4e\xe7"
+			  "\xaa\x91\xf5\xd5\xc6\xfe\x41\xb5"
+			  "\xed\x90\xce\xb9\xcd\xcc\xa1",
+		.ilen	= 47,
+		.result	= "\x41\x94\x0e\x33\x22\xb1\xdd\xf4"
+			  "\x10\x57\x85\x39\x93\x8f\xaf\x70"
+			  "\xfa\xa9\xd0\x4d\x5c\x40\x23\xcd"
+			  "\x98\x34\xab\x37\x56\xae\x32",
+		.rlen	= 31,
+	}, {
+		.key	= "\x50\x5d\x9d\x9b\x66\x50\x88\x7b"
+			  "\x30\x8e\xb1\x5e\x92\x58\xe0\xf1",
+		.klen	= 16,
+		.iv	= "\x5f\x27\x2b\x03\xaa\xef\x32\x02"
+			  "\x50\xc4\xde\x82\x90\x21\x11\x73",
+		.assoc	= "\x6e\xf0\xba\x6b\xee\x8e\xdc\x89"
+			  "\x71\xfb\x0a\xa6\x8f\xea\x41\xf4"
+			  "\x5a\xbb\x59\xb0\x20\x38\xc5\xe0"
+			  "\x29\x56\x52\x19\x79\xf5\xe9\x37",
+		.alen	= 32,
+		.input	= "\xf9\xd7\xee\x17\xfd\x24\xcd\xf1"
+			  "\xbc\x0f\x35\x97\x97\x0c\x4b\x18"
+			  "\xce\x58\xc8\x3b\xd4\x85\x93\x79"
+			  "\xcc\x9c\xea\xc1\x73\x13\x0b\x4c"
+			  "\xcc\x6f\x28\xf8\xa4\x4e\xb8\x56"
+			  "\x64\x4e\x47\xce\xb2\xb4\x92\xb4",
+		.ilen	= 48,
+		.result	= "\x7e\xb9\x48\xd3\x32\x2d\x86\x10"
+			  "\x91\x31\x37\xcb\x8d\xb3\x72\x76"
+			  "\x24\x6b\xdc\xd1\x61\xe0\xa5\xe7"
+			  "\x5a\x61\x8a\x0f\x30\x0d\xd1\xec",
+		.rlen	= 32,
+	}, {
+		.key	= "\x8d\x82\xd6\x3b\x76\xcc\x30\x97"
+			  "\xb1\x68\x63\xef\x8c\x7c\xa3\xf7",
+		.klen	= 16,
+		.iv	= "\x9c\x4b\x65\xa2\xba\x6b\xdb\x1e"
+			  "\xd1\x9e\x90\x13\x8a\x45\xd3\x79",
+		.assoc	= "\xab\x14\xf3\x0a\xfe\x0a\x85\xa5"
+			  "\xf2\xd5\xbc\x38\x89\x0e\x04\xfb"
+			  "\x84\x7d\x65\x34\x25\xd8\x47\xfa"
+			  "\xeb\x83\x31\xf1\x54\x54\x89\x0d"
+			  "\x9d",
+		.alen	= 33,
+		.input	= "\x58\xfa\x3a\x3d\xd9\x88\x63\xe8"
+			  "\xc5\x78\x50\x8b\x4a\xc9\xdf\x7f"
+			  "\x4b\xfa\xc8\x2e\x67\x43\xf3\x63"
+			  "\x42\x8e\x99\x5a\x9c\x0b\x84\x77"
+			  "\xbc\x46\x76\x48\x82\xc7\x57\x96"
+			  "\xe1\x65\xd1\xed\x1d\xdd\x80\x24"
+			  "\xa6\x4d\xa9\xf1\x53\x8b\x5e\x0e"
+			  "\x26\xb9\xcc\x37\xe5\x43\xe1\x5a"
+			  "\x8a\xd6\x8c\x5a\xe4\x95\xd1\x8d"
+			  "\xf7\x33\x64\xc1\xd3\xf2\xfc\x35"
+			  "\x01",
+		.ilen	= 81,
+		.result	= "\xba\xde\x82\x72\x42\xa9\x2f\x2c"
+			  "\x12\x0b\xe9\x5c\x87\xd7\x35\x7c"
+			  "\x4f\x2e\xe8\x55\x66\x80\x27\x00"
+			  "\x1b\x8f\x68\xe7\x0a\x6c\x71\xc3"
+			  "\x21\x78\x55\x9d\x9c\x65\x7b\xcd"
+			  "\x0a\x34\x97\xff\x47\x37\xb0\x2a"
+			  "\x80\x0d\x19\x98\x33\xa9\x7a\xe3"
+			  "\x2e\x4c\xc6\xf3\x8c\x88\x42\x01"
+			  "\xbd",
+		.rlen	= 65,
+	}, {
+		.key	= "\xc9\xa7\x10\xda\x86\x48\xd9\xb3"
+			  "\x32\x42\x15\x80\x85\xa1\x65\xfe",
+		.klen	= 16,
+		.iv	= "\xd8\x70\x9f\x42\xca\xe6\x83\x3a"
+			  "\x52\x79\x42\xa5\x84\x6a\x96\x7f",
+		.assoc	= "\xe8\x39\x2d\xaa\x0e\x85\x2d\xc1"
+			  "\x72\xaf\x6e\xc9\x82\x33\xc7\x01"
+			  "\xaf\x40\x70\xb8\x2a\x78\xc9\x14"
+			  "\xac\xb1\x10\xca\x2e\xb3\x28\xe4"
+			  "\xac\xfa\x58\x7f\xe5\x73\x09\x8c"
+			  "\x1d\x40\x87\x8c\xd9\x75\xc0\x55"
+			  "\xa2\xda\x07\xd1\xc2\xa9\xd1\xbb"
+			  "\x09\x4f\x77\x62\x88\x2d\xf2\x68"
+			  "\x54",
+		.alen	= 65,
+		.input	= "\x4c\xa9\xac\x71\xed\x10\xa6\x24"
+			  "\xb7\xa7\xdf\x8b\xf5\xc2\x41\xcb"
+			  "\x05\xc9\xd6\x97\xb6\x10\x7f\x17"
+			  "\xc2\xc0\x93\xcf\xe0\x94\xfd\x99"
+			  "\xf2\x62\x25\x28\x01\x23\x6f\x8b"
+			  "\x04\x52\xbc\xb0\x3e\x66\x52\x90"
+			  "\x9f",
+		.ilen	= 49,
+		.result	= "\xf7\x02\xbb\x11\x52\x24\xd8\x48"
+			  "\x93\xe6\x9b\xee\x81\xfc\xf7\x82"
+			  "\x79\xf0\xf3\xd9\x6c\x20\xa9\x1a"
+			  "\xdc\xbc\x47\xc0\xe4\xcb\x10\x99"
+			  "\x2f",
+		.rlen	= 33,
+	}, {
+		.key	= "\x06\xcc\x4a\x79\x96\xc3\x82\xcf"
+			  "\xb3\x1c\xc7\x12\x7f\xc5\x28\x04",
+		.klen	= 16,
+		.iv	= "\x15\x95\xd8\xe1\xda\x62\x2c\x56"
+			  "\xd3\x53\xf4\x36\x7e\x8e\x59\x85",
+		.assoc	= "\x24\x5e\x67\x49\x1e\x01\xd6\xdd"
+			  "\xf3\x89\x20\x5b\x7c\x57\x89\x07",
+		.alen	= 16,
+		.input	= "\x6d\xed\x04\x7a\x2f\x0c\x30\xa5"
+			  "\x96\xe6\x97\xe4\x10\xeb\x40\x95"
+			  "\xc5\x9a\xdf\x31\xd5\xa5\xa6\xec"
+			  "\x05\xa8\x31\x50\x11\x19\x44",
+		.ilen	= 31,
+		.result	= "\x33\x27\xf5\xb1\x62\xa0\x80\x63"
+			  "\x14\xc0\x4d\x7f\x7b\x20\xba\x89",
+		.rlen	= 16,
+	}, {
+		.key	= "\x42\xf0\x84\x19\xa6\x3f\x2b\xea"
+			  "\x34\xf6\x79\xa3\x79\xe9\xeb\x0a",
+		.klen	= 16,
+		.iv	= "\x51\xb9\x12\x80\xea\xde\xd5\x71"
+			  "\x54\x2d\xa6\xc8\x78\xb2\x1b\x8c",
+		.assoc	= "\x61\x83\xa0\xe8\x2e\x7d\x7f\xf8"
+			  "\x74\x63\xd2\xec\x76\x7c\x4c\x0d",
+		.alen	= 16,
+		.input	= "\x30\x95\x7d\xea\xdc\x62\xc0\x88"
+			  "\xa1\xe3\x8d\x8c\xac\x04\x10\xa7"
+			  "\xfa\xfa\x07\xbd\xa0\xf0\x36\xeb"
+			  "\x21\x93\x2e\x31\x84\x83",
+		.ilen	= 30,
+		.result	= "\x70\x4c\x2f\x50\x72\x1c\x29\x7f"
+			  "\x95\x9a\xff\x10\x75\x45\x7d\x8f",
+		.rlen	= 16,
+	}, {
+		.key	= "\x7f\x15\xbd\xb8\xb6\xba\xd3\x06"
+			  "\xb5\xd1\x2b\x35\x73\x0e\xad\x10",
+		.klen	= 16,
+		.iv	= "\x8e\xde\x4c\x20\xfa\x59\x7e\x8d"
+			  "\xd5\x07\x58\x59\x72\xd7\xde\x92",
+		.assoc	= "\x9d\xa7\xda\x88\x3e\xf8\x28\x14"
+			  "\xf5\x3e\x85\x7d\x70\xa0\x0f\x13",
+		.alen	= 16,
+		.input	= "\x93\xcd\xee\xd4\xcb\x9d\x8d\x16"
+			  "\x63\x0d\x43\xd5\x49\xca\xa8\x85"
+			  "\x49\xc0\xae\x13\xbc\x26\x1d\x4b",
+		.ilen	= 24,
+		.result	= "\xac\x70\x69\xef\x82\x97\xd2\x9b"
+			  "\x15\x74\xb1\xa2\x6f\x69\x3f\x95",
+		.rlen	= 16,
+	},
+};
+
+/*
+ * AEGIS-256 test vectors - generated via reference implementation from
+ * SUPERCOP (https://bench.cr.yp.to/supercop.html):
+ *
+ *   https://bench.cr.yp.to/supercop/supercop-20170228.tar.xz
+ *   (see crypto_aead/aegis256/)
+ */
+static const struct aead_testvec aegis256_enc_tv_template[] = {
+	{
+		.key	= "\x0f\xc9\x8e\x67\x44\x9e\xaa\x86"
+			  "\x20\x36\x2c\x24\xfe\xc9\x30\x81"
+			  "\xca\xb0\x82\x21\x41\xa8\xe0\x06"
+			  "\x30\x0b\x37\xf6\xb6\x17\xe7\xb5",
+		.klen	= 32,
+		.iv	= "\x1e\x92\x1c\xcf\x88\x3d\x54\x0d"
+			  "\x40\x6d\x59\x48\xfc\x92\x61\x03"
+			  "\x95\x61\x05\x42\x82\x50\xc0\x0c"
+			  "\x60\x16\x6f\xec\x6d\x2f\xcf\x6b",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xd5\x65\x3a\xa9\x03\x51\xd7\xaa"
+			  "\xfa\x4b\xd8\xa2\x41\x9b\xc1\xb2",
+		.rlen	= 16,
+	}, {
+		.key	= "\x4b\xed\xc8\x07\x54\x1a\x52\xa2"
+			  "\xa1\x10\xde\xb5\xf8\xed\xf3\x87"
+			  "\xf4\x72\x8e\xa5\x46\x48\x62\x20"
+			  "\xf1\x38\x16\xce\x90\x76\x87\x8c",
+		.klen	= 32,
+		.iv	= "\x5a\xb7\x56\x6e\x98\xb9\xfd\x29"
+			  "\xc1\x47\x0b\xda\xf6\xb6\x23\x09"
+			  "\xbf\x23\x11\xc6\x87\xf0\x42\x26"
+			  "\x22\x44\x4e\xc4\x47\x8e\x6e\x41",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x79",
+		.ilen	= 1,
+		.result	= "\x84\xa2\x8f\xad\xdb\x8d\x2c\x16"
+			  "\x9e\x89\xd9\x06\xa6\xa8\x14\x29"
+			  "\x8b",
+		.rlen	= 17,
+	}, {
+		.key	= "\x88\x12\x01\xa6\x64\x96\xfb\xbe"
+			  "\x22\xea\x90\x47\xf2\x11\xb5\x8e"
+			  "\x1f\x35\x9a\x29\x4b\xe8\xe4\x39"
+			  "\xb3\x66\xf5\xa6\x6a\xd5\x26\x62",
+		.klen	= 32,
+		.iv	= "\x97\xdb\x90\x0e\xa8\x35\xa5\x45"
+			  "\x42\x21\xbd\x6b\xf0\xda\xe6\x0f"
+			  "\xe9\xe5\x1d\x4a\x8c\x90\xc4\x40"
+			  "\xe3\x71\x2d\x9c\x21\xed\x0e\x18",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xb5\x6e\xad\xdd\x30\x72\xfa\x53"
+			  "\x82\x8e\x16\xb4\xed\x6d\x47",
+		.ilen	= 15,
+		.result	= "\x09\x94\x1f\xa6\x13\xc3\x74\x75"
+			  "\x17\xad\x8a\x0e\xd8\x66\x9a\x28"
+			  "\xd7\x30\x66\x09\x2a\xdc\xfa\x2a"
+			  "\x9f\x3b\xd7\xdd\x66\xd1\x2b",
+		.rlen	= 31,
+	}, {
+		.key	= "\xc4\x37\x3b\x45\x74\x11\xa4\xda"
+			  "\xa2\xc5\x42\xd8\xec\x36\x78\x94"
+			  "\x49\xf7\xa5\xad\x50\x88\x66\x53"
+			  "\x74\x94\xd4\x7f\x44\x34\xc5\x39",
+		.klen	= 32,
+		.iv	= "\xd3\x00\xc9\xad\xb8\xb0\x4e\x61"
+			  "\xc3\xfb\x6f\xfd\xea\xff\xa9\x15"
+			  "\x14\xa8\x28\xce\x92\x30\x46\x59"
+			  "\xa4\x9f\x0b\x75\xfb\x4c\xad\xee",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xf2\x92\xe6\x7d\x40\xee\xa3\x6f"
+			  "\x03\x68\xc8\x45\xe7\x91\x0a\x18",
+		.ilen	= 16,
+		.result	= "\x8a\x46\xa2\x22\x8c\x03\xab\x6f"
+			  "\x54\x63\x4e\x7f\xc9\x8e\xfa\x70"
+			  "\x7b\xe5\x8d\x78\xbc\xe9\xb6\xa1"
+			  "\x29\x17\xc8\x3b\x52\xa4\x98\x72",
+		.rlen	= 32,
+	}, {
+		.key	= "\x01\x5c\x75\xe5\x84\x8d\x4d\xf6"
+			  "\x23\x9f\xf4\x6a\xe6\x5a\x3b\x9a"
+			  "\x74\xb9\xb1\x32\x55\x28\xe8\x6d"
+			  "\x35\xc1\xb3\x57\x1f\x93\x64\x0f",
+		.klen	= 32,
+		.iv	= "\x10\x25\x03\x4c\xc8\x2c\xf7\x7d"
+			  "\x44\xd5\x21\x8e\xe4\x23\x6b\x1c"
+			  "\x3e\x6a\x34\x53\x97\xd0\xc8\x73"
+			  "\x66\xcd\xea\x4d\xd5\xab\x4c\xc5",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x2e\xb7\x20\x1c\x50\x6a\x4b\x8b"
+			  "\x84\x42\x7a\xd7\xe1\xb5\xcd\x1f"
+			  "\xd3",
+		.ilen	= 17,
+		.result	= "\x71\x6b\x37\x0b\x02\x61\x28\x12"
+			  "\x83\xab\x66\x90\x84\xc7\xd1\xc5"
+			  "\xb2\x7a\xb4\x7b\xb4\xfe\x02\xb2"
+			  "\xc0\x00\x39\x13\xb5\x51\x68\x44"
+			  "\xad",
+		.rlen	= 33,
+	}, {
+		.key	= "\x3d\x80\xae\x84\x94\x09\xf6\x12"
+			  "\xa4\x79\xa6\xfb\xe0\x7f\xfd\xa0"
+			  "\x9e\x7c\xbc\xb6\x5b\xc8\x6a\x86"
+			  "\xf7\xef\x91\x30\xf9\xf2\x04\xe6",
+		.klen	= 32,
+		.iv	= "\x4c\x49\x3d\xec\xd8\xa8\xa0\x98"
+			  "\xc5\xb0\xd3\x1f\xde\x48\x2e\x22"
+			  "\x69\x2c\x3f\xd7\x9c\x70\x4a\x8d"
+			  "\x27\xfa\xc9\x26\xaf\x0a\xeb\x9c",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x6b\xdc\x5a\xbb\x60\xe5\xf4\xa6"
+			  "\x05\x1d\x2c\x68\xdb\xda\x8f\x25"
+			  "\xfe\x8d\x45\x19\x1e\xc0\x0b\x99"
+			  "\x88\x11\x39\x12\x1c\x3a\xbb",
+		.ilen	= 31,
+		.result	= "\xaf\xa4\x34\x0d\x59\xe6\x1c\x2f"
+			  "\x06\x3b\x52\x18\x49\x75\x1b\xf0"
+			  "\x53\x09\x72\x7b\x45\x79\xe0\xbe"
+			  "\x89\x85\x23\x15\xb8\x79\x07\x4c"
+			  "\x53\x7a\x15\x37\x0a\xee\xb7\xfb"
+			  "\xc4\x1f\x12\x27\xcf\x77\x90",
+		.rlen	= 47,
+	}, {
+		.key	= "\x7a\xa5\xe8\x23\xa4\x84\x9e\x2d"
+			  "\x25\x53\x58\x8c\xda\xa3\xc0\xa6"
+			  "\xc8\x3e\xc8\x3a\x60\x68\xec\xa0"
+			  "\xb8\x1c\x70\x08\xd3\x51\xa3\xbd",
+		.klen	= 32,
+		.iv	= "\x89\x6e\x77\x8b\xe8\x23\x49\xb4"
+			  "\x45\x8a\x85\xb1\xd8\x6c\xf1\x28"
+			  "\x93\xef\x4b\x5b\xa1\x10\xcc\xa6"
+			  "\xe8\x28\xa8\xfe\x89\x69\x8b\x72",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xa7\x00\x93\x5b\x70\x61\x9d\xc2"
+			  "\x86\xf7\xde\xfa\xd5\xfe\x52\x2b"
+			  "\x28\x50\x51\x9d\x24\x60\x8d\xb3"
+			  "\x49\x3e\x17\xea\xf6\x99\x5a\xdd",
+		.ilen	= 32,
+		.result	= "\xe2\xc9\x0b\x33\x31\x02\xb3\xb4"
+			  "\x33\xfe\xeb\xa8\xb7\x9b\xb2\xd7"
+			  "\xeb\x0f\x05\x2b\xba\xb3\xca\xef"
+			  "\xf6\xd1\xb6\xc0\xb9\x9b\x85\xc5"
+			  "\xbf\x7a\x3e\xcc\x31\x76\x09\x80"
+			  "\x32\x5d\xbb\xe8\x38\x0e\x77\xd3",
+		.rlen	= 48,
+	}, {
+		.key	= "\xb6\xca\x22\xc3\xb4\x00\x47\x49"
+			  "\xa6\x2d\x0a\x1e\xd4\xc7\x83\xad"
+			  "\xf3\x00\xd4\xbf\x65\x08\x6e\xb9"
+			  "\x7a\x4a\x4f\xe0\xad\xb0\x42\x93",
+		.klen	= 32,
+		.iv	= "\xc5\x93\xb0\x2a\xf8\x9f\xf1\xd0"
+			  "\xc6\x64\x37\x42\xd2\x90\xb3\x2e"
+			  "\xbd\xb1\x57\xe0\xa6\xb0\x4e\xc0"
+			  "\xaa\x55\x87\xd6\x63\xc8\x2a\x49",
+		.assoc	= "\xd5",
+		.alen	= 1,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x96\x43\x30\xca\x6c\x4f\xd7\x12"
+			  "\xba\xd9\xb3\x18\x86\xdf\xc3\x52",
+		.rlen	= 16,
+	}, {
+		.key	= "\xf3\xee\x5c\x62\xc4\x7c\xf0\x65"
+			  "\x27\x08\xbd\xaf\xce\xec\x45\xb3"
+			  "\x1d\xc3\xdf\x43\x6a\xa8\xf0\xd3"
+			  "\x3b\x77\x2e\xb9\x87\x0f\xe1\x6a",
+		.klen	= 32,
+		.iv	= "\x02\xb8\xea\xca\x09\x1b\x9a\xec"
+			  "\x47\x3e\xe9\xd4\xcc\xb5\x76\x34"
+			  "\xe8\x73\x62\x64\xab\x50\xd0\xda"
+			  "\x6b\x83\x66\xaf\x3e\x27\xc9\x1f",
+		.assoc	= "\x11\x81\x78\x32\x4d\xb9\x44\x73"
+			  "\x68\x75\x16\xf8\xcb\x7e\xa7",
+		.alen	= 15,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x2f\xab\x45\xe2\xa7\x46\xc5\x83"
+			  "\x11\x9f\xb0\x74\xee\xc7\x03\xdd",
+		.rlen	= 16,
+	}, {
+		.key	= "\x2f\x13\x95\x01\xd5\xf7\x99\x81"
+			  "\xa8\xe2\x6f\x41\xc8\x10\x08\xb9"
+			  "\x47\x85\xeb\xc7\x6f\x48\x72\xed"
+			  "\xfc\xa5\x0d\x91\x61\x6e\x81\x40",
+		.klen	= 32,
+		.iv	= "\x3f\xdc\x24\x69\x19\x96\x43\x08"
+			  "\xc8\x18\x9b\x65\xc6\xd9\x39\x3b"
+			  "\x12\x35\x6e\xe8\xb0\xf0\x52\xf3"
+			  "\x2d\xb0\x45\x87\x18\x86\x68\xf6",
+		.assoc	= "\x4e\xa5\xb2\xd1\x5d\x35\xed\x8f"
+			  "\xe8\x4f\xc8\x89\xc5\xa2\x69\xbc",
+		.alen	= 16,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x16\x44\x73\x33\x5d\xf2\xb9\x04"
+			  "\x6b\x79\x98\xef\xdb\xd5\xc5\xf1",
+		.rlen	= 16,
+	}, {
+		.key	= "\x6c\x38\xcf\xa1\xe5\x73\x41\x9d"
+			  "\x29\xbc\x21\xd2\xc2\x35\xcb\xbf"
+			  "\x72\x47\xf6\x4b\x74\xe8\xf4\x06"
+			  "\xbe\xd3\xec\x6a\x3b\xcd\x20\x17",
+		.klen	= 32,
+		.iv	= "\x7b\x01\x5d\x08\x29\x12\xec\x24"
+			  "\x49\xf3\x4d\xf7\xc0\xfe\xfb\x41"
+			  "\x3c\xf8\x79\x6c\xb6\x90\xd4\x0d"
+			  "\xee\xde\x23\x60\xf2\xe5\x08\xcc",
+		.assoc	= "\x8a\xca\xec\x70\x6d\xb1\x96\xab"
+			  "\x69\x29\x7a\x1b\xbf\xc7\x2c\xc2"
+			  "\x07",
+		.alen	= 17,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xa4\x9b\xb8\x47\xc0\xed\x7a\x45"
+			  "\x98\x54\x8c\xed\x3d\x17\xf0\xdd",
+		.rlen	= 16,
+	}, {
+		.key	= "\xa8\x5c\x09\x40\xf5\xef\xea\xb8"
+			  "\xaa\x96\xd3\x64\xbc\x59\x8d\xc6"
+			  "\x9c\x0a\x02\xd0\x79\x88\x76\x20"
+			  "\x7f\x00\xca\x42\x15\x2c\xbf\xed",
+		.klen	= 32,
+		.iv	= "\xb8\x26\x97\xa8\x39\x8e\x94\x3f"
+			  "\xca\xcd\xff\x88\xba\x22\xbe\x47"
+			  "\x67\xba\x85\xf1\xbb\x30\x56\x26"
+			  "\xaf\x0b\x02\x38\xcc\x44\xa7\xa3",
+		.assoc	= "\xc7\xef\x26\x10\x7d\x2c\x3f\xc6"
+			  "\xea\x03\x2c\xac\xb9\xeb\xef\xc9"
+			  "\x31\x6b\x08\x12\xfc\xd8\x37\x2d"
+			  "\xe0\x17\x3a\x2e\x83\x5c\x8f",
+		.alen	= 31,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x20\x24\xe2\x33\x5c\x60\xc9\xf0"
+			  "\xa4\x96\x2f\x0d\x53\xc2\xf8\xfc",
+		.rlen	= 16,
+	}, {
+		.key	= "\xe5\x81\x42\xdf\x05\x6a\x93\xd4"
+			  "\x2b\x70\x85\xf5\xb6\x7d\x50\xcc"
+			  "\xc6\xcc\x0e\x54\x7f\x28\xf8\x3a"
+			  "\x40\x2e\xa9\x1a\xf0\x8b\x5e\xc4",
+		.klen	= 32,
+		.iv	= "\xf4\x4a\xd1\x47\x49\x09\x3d\x5b"
+			  "\x4b\xa7\xb1\x19\xb4\x46\x81\x4d"
+			  "\x91\x7c\x91\x75\xc0\xd0\xd8\x40"
+			  "\x71\x39\xe1\x10\xa6\xa3\x46\x7a",
+		.assoc	= "\x03\x14\x5f\xaf\x8d\xa8\xe7\xe2"
+			  "\x6b\xde\xde\x3e\xb3\x10\xb1\xcf"
+			  "\x5c\x2d\x14\x96\x01\x78\xb9\x47"
+			  "\xa1\x44\x19\x06\x5d\xbb\x2e\x2f",
+		.alen	= 32,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x6f\x4a\xb9\xe0\xff\x51\xa3\xf1"
+			  "\xd2\x64\x3e\x66\x6a\xb2\x03\xc0",
+		.rlen	= 16,
+	}, {
+		.key	= "\x22\xa6\x7c\x7f\x15\xe6\x3c\xf0"
+			  "\xac\x4b\x37\x86\xb0\xa2\x13\xd2"
+			  "\xf1\x8e\x19\xd8\x84\xc8\x7a\x53"
+			  "\x02\x5b\x88\xf3\xca\xea\xfe\x9b",
+		.klen	= 32,
+		.iv	= "\x31\x6f\x0b\xe6\x59\x85\xe6\x77"
+			  "\xcc\x81\x63\xab\xae\x6b\x43\x54"
+			  "\xbb\x3f\x9c\xf9\xc5\x70\x5a\x5a"
+			  "\x32\x67\xc0\xe9\x80\x02\xe5\x50",
+		.assoc	= "\x40",
+		.alen	= 1,
+		.input	= "\x4f",
+		.ilen	= 1,
+		.result	= "\x2c\xfb\xad\x7e\xbe\xa0\x9a\x5b"
+			  "\x7a\x3f\x81\xf7\xfc\x1b\x79\x83"
+			  "\xc7",
+		.rlen	= 17,
+	}, {
+		.key	= "\x5e\xcb\xb6\x1e\x25\x62\xe4\x0c"
+			  "\x2d\x25\xe9\x18\xaa\xc6\xd5\xd8"
+			  "\x1b\x50\x25\x5d\x89\x68\xfc\x6d"
+			  "\xc3\x89\x67\xcb\xa4\x49\x9d\x71",
+		.klen	= 32,
+		.iv	= "\x6d\x94\x44\x86\x69\x00\x8f\x93"
+			  "\x4d\x5b\x15\x3c\xa8\x8f\x06\x5a"
+			  "\xe6\x01\xa8\x7e\xca\x10\xdc\x73"
+			  "\xf4\x94\x9f\xc1\x5a\x61\x85\x27",
+		.assoc	= "\x7c\x5d\xd3\xee\xad\x9f\x39\x1a"
+			  "\x6d\x92\x42\x61\xa7\x58\x37",
+		.alen	= 15,
+		.input	= "\x8b\x26\x61\x55\xf1\x3e\xe3\xa1"
+			  "\x8d\xc8\x6e\x85\xa5\x21\x67",
+		.ilen	= 15,
+		.result	= "\x1f\x7f\xca\x3c\x2b\xe7\x27\xba"
+			  "\x7e\x98\x83\x02\x34\x23\xf7\x94"
+			  "\xde\x35\xe6\x1d\x14\x18\xe5\x38"
+			  "\x14\x80\x6a\xa7\x1b\xae\x1d",
+		.rlen	= 31,
+	}, {
+		.key	= "\x9b\xef\xf0\xbd\x35\xdd\x8d\x28"
+			  "\xad\xff\x9b\xa9\xa4\xeb\x98\xdf"
+			  "\x46\x13\x31\xe1\x8e\x08\x7e\x87"
+			  "\x85\xb6\x46\xa3\x7e\xa8\x3c\x48",
+		.klen	= 32,
+		.iv	= "\xaa\xb8\x7e\x25\x79\x7c\x37\xaf"
+			  "\xce\x36\xc7\xce\xa2\xb4\xc9\x60"
+			  "\x10\xc3\xb3\x02\xcf\xb0\x5e\x8d"
+			  "\xb5\xc2\x7e\x9a\x35\xc0\x24\xfd",
+		.assoc	= "\xb9\x82\x0c\x8d\xbd\x1b\xe2\x36"
+			  "\xee\x6c\xf4\xf2\xa1\x7d\xf9\xe2",
+		.alen	= 16,
+		.input	= "\xc8\x4b\x9b\xf5\x01\xba\x8c\xbd"
+			  "\x0e\xa3\x21\x16\x9f\x46\x2a\x63",
+		.ilen	= 16,
+		.result	= "\x05\x86\x9e\xd7\x2b\xa3\x97\x01"
+			  "\xbe\x28\x98\x10\x6f\xe9\x61\x32"
+			  "\x96\xbb\xb1\x2e\x8f\x0c\x44\xb9"
+			  "\x46\x2d\x55\xe3\x42\x67\xf2\xaf",
+		.rlen	= 32,
+	}, {
+		.key	= "\xd7\x14\x29\x5d\x45\x59\x36\x44"
+			  "\x2e\xd9\x4d\x3b\x9e\x0f\x5b\xe5"
+			  "\x70\xd5\x3c\x65\x93\xa8\x00\xa0"
+			  "\x46\xe4\x25\x7c\x58\x08\xdb\x1e",
+		.klen	= 32,
+		.iv	= "\xe6\xdd\xb8\xc4\x89\xf8\xe0\xca"
+			  "\x4f\x10\x7a\x5f\x9c\xd8\x8b\x66"
+			  "\x3b\x86\xbf\x86\xd4\x50\xe0\xa7"
+			  "\x76\xef\x5c\x72\x0f\x1f\xc3\xd4",
+		.assoc	= "\xf5\xa6\x46\x2c\xce\x97\x8a\x51"
+			  "\x6f\x46\xa6\x83\x9b\xa1\xbc\xe8"
+			  "\x05",
+		.alen	= 17,
+		.input	= "\x05\x70\xd5\x94\x12\x36\x35\xd8"
+			  "\x8f\x7d\xd3\xa8\x99\x6a\xed\x69"
+			  "\xd0",
+		.ilen	= 17,
+		.result	= "\x9c\xe0\x06\x7b\x86\xcf\x2e\xd8"
+			  "\x45\x65\x1b\x72\x9b\xaa\xa3\x1e"
+			  "\x87\x9d\x26\xdf\xff\x81\x11\xd2"
+			  "\x47\x41\xb9\x24\xc1\x8a\xa3\x8b"
+			  "\x55",
+		.rlen	= 33,
+	}, {
+		.key	= "\x14\x39\x63\xfc\x56\xd5\xdf\x5f"
+			  "\xaf\xb3\xff\xcc\x98\x33\x1d\xeb"
+			  "\x9a\x97\x48\xe9\x98\x48\x82\xba"
+			  "\x07\x11\x04\x54\x32\x67\x7b\xf5",
+		.klen	= 32,
+		.iv	= "\x23\x02\xf1\x64\x9a\x73\x89\xe6"
+			  "\xd0\xea\x2c\xf1\x96\xfc\x4e\x6d"
+			  "\x65\x48\xcb\x0a\xda\xf0\x62\xc0"
+			  "\x38\x1d\x3b\x4a\xe9\x7e\x62\xaa",
+		.assoc	= "\x32\xcb\x80\xcc\xde\x12\x33\x6d"
+			  "\xf0\x20\x58\x15\x95\xc6\x7f\xee"
+			  "\x2f\xf9\x4e\x2c\x1b\x98\x43\xc7"
+			  "\x68\x28\x73\x40\x9f\x96\x4a",
+		.alen	= 31,
+		.input	= "\x41\x94\x0e\x33\x22\xb1\xdd\xf4"
+			  "\x10\x57\x85\x39\x93\x8f\xaf\x70"
+			  "\xfa\xa9\xd0\x4d\x5c\x40\x23\xcd"
+			  "\x98\x34\xab\x37\x56\xae\x32",
+		.ilen	= 31,
+		.result	= "\xa0\xc8\xde\x83\x0d\xc3\x4e\xd5"
+			  "\x69\x7f\x7a\xdd\x8c\x46\xda\xba"
+			  "\x0a\x5c\x0e\x7f\xac\xee\x02\xd2"
+			  "\xe5\x4b\x0a\xba\xb8\xa4\x7b\x66"
+			  "\xde\xae\xdb\xc2\xc0\x0b\xf7\x2b"
+			  "\xdf\xb8\xea\xd8\xa9\x38\xed",
+		.rlen	= 47,
+	}, {
+		.key	= "\x50\x5d\x9d\x9b\x66\x50\x88\x7b"
+			  "\x30\x8e\xb1\x5e\x92\x58\xe0\xf1"
+			  "\xc5\x5a\x53\x6e\x9d\xe8\x04\xd4"
+			  "\xc9\x3f\xe2\x2d\x0c\xc6\x1a\xcb",
+		.klen	= 32,
+		.iv	= "\x5f\x27\x2b\x03\xaa\xef\x32\x02"
+			  "\x50\xc4\xde\x82\x90\x21\x11\x73"
+			  "\x8f\x0a\xd6\x8f\xdf\x90\xe4\xda"
+			  "\xf9\x4a\x1a\x23\xc3\xdd\x02\x81",
+		.assoc	= "\x6e\xf0\xba\x6b\xee\x8e\xdc\x89"
+			  "\x71\xfb\x0a\xa6\x8f\xea\x41\xf4"
+			  "\x5a\xbb\x59\xb0\x20\x38\xc5\xe0"
+			  "\x29\x56\x52\x19\x79\xf5\xe9\x37",
+		.alen	= 32,
+		.input	= "\x7e\xb9\x48\xd3\x32\x2d\x86\x10"
+			  "\x91\x31\x37\xcb\x8d\xb3\x72\x76"
+			  "\x24\x6b\xdc\xd1\x61\xe0\xa5\xe7"
+			  "\x5a\x61\x8a\x0f\x30\x0d\xd1\xec",
+		.ilen	= 32,
+		.result	= "\xd3\x68\x14\x70\x3c\x01\x43\x86"
+			  "\x02\xab\xbe\x75\xaa\xe7\xf5\x53"
+			  "\x5c\x05\xbd\x9b\x19\xbb\x2a\x61"
+			  "\x8f\x69\x05\x75\x8e\xca\x60\x0c"
+			  "\x5b\xa2\x48\x61\x32\x74\x11\x2b"
+			  "\xf6\xcf\x06\x78\x6f\x78\x1a\x4a",
+		.rlen	= 48,
+	}, {
+		.key	= "\x8d\x82\xd6\x3b\x76\xcc\x30\x97"
+			  "\xb1\x68\x63\xef\x8c\x7c\xa3\xf7"
+			  "\xef\x1c\x5f\xf2\xa3\x88\x86\xed"
+			  "\x8a\x6d\xc1\x05\xe7\x25\xb9\xa2",
+		.klen	= 32,
+		.iv	= "\x9c\x4b\x65\xa2\xba\x6b\xdb\x1e"
+			  "\xd1\x9e\x90\x13\x8a\x45\xd3\x79"
+			  "\xba\xcd\xe2\x13\xe4\x30\x66\xf4"
+			  "\xba\x78\xf9\xfb\x9d\x3c\xa1\x58",
+		.assoc	= "\xab\x14\xf3\x0a\xfe\x0a\x85\xa5"
+			  "\xf2\xd5\xbc\x38\x89\x0e\x04\xfb"
+			  "\x84\x7d\x65\x34\x25\xd8\x47\xfa"
+			  "\xeb\x83\x31\xf1\x54\x54\x89\x0d"
+			  "\x9d",
+		.alen	= 33,
+		.input	= "\xba\xde\x82\x72\x42\xa9\x2f\x2c"
+			  "\x12\x0b\xe9\x5c\x87\xd7\x35\x7c"
+			  "\x4f\x2e\xe8\x55\x66\x80\x27\x00"
+			  "\x1b\x8f\x68\xe7\x0a\x6c\x71\xc3"
+			  "\x21\x78\x55\x9d\x9c\x65\x7b\xcd"
+			  "\x0a\x34\x97\xff\x47\x37\xb0\x2a"
+			  "\x80\x0d\x19\x98\x33\xa9\x7a\xe3"
+			  "\x2e\x4c\xc6\xf3\x8c\x88\x42\x01"
+			  "\xbd",
+		.ilen	= 65,
+		.result	= "\x07\x0a\x35\xb0\x82\x03\x5a\xd2"
+			  "\x15\x3a\x6c\x72\x83\x9b\xb1\x75"
+			  "\xea\xf2\xfc\xff\xc6\xf1\x13\xa4"
+			  "\x1a\x93\x33\x79\x97\x82\x81\xc0"
+			  "\x96\xc2\x00\xab\x39\xae\xa1\x62"
+			  "\x53\xa3\x86\xc9\x07\x8c\xaf\x22"
+			  "\x47\x31\x29\xca\x4a\x95\xf5\xd5"
+			  "\x20\x63\x5a\x54\x80\x2c\x4a\x63"
+			  "\xfb\x18\x73\x31\x4f\x08\x21\x5d"
+			  "\x20\xe9\xc3\x7e\xea\x25\x77\x3a"
+			  "\x65",
+		.rlen	= 81,
+	}, {
+		.key	= "\xc9\xa7\x10\xda\x86\x48\xd9\xb3"
+			  "\x32\x42\x15\x80\x85\xa1\x65\xfe"
+			  "\x19\xde\x6b\x76\xa8\x28\x08\x07"
+			  "\x4b\x9a\xa0\xdd\xc1\x84\x58\x79",
 		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\xCD\x3E\x2A\x3B\x3E\x94\xC5\x77"
-			  "\xBA\xBB\x5B\xB1\xDE\x7B\xA4\x40"
-			  "\x88\x39\xE3\xFD\x94\x4B\x25\x58"
-			  "\xE1\x4B\xC4\x18\x7A\xFD\x17\x2B"
-			  "\xB9\xF9\xC2\x27\x6A\xB6\x31\x27"
-			  "\xA6\xAD\xEF\xE5\x5D\xE4\x02\x01"
-			  "\x56\x2E\x10\xC2\x2C\xFF\xC6\x83"
-			  "\xB5\xDC\x4F\x63\xAD\x0E\x63\x5E"
-			  "\x56\xC8\x18\x3D\x79\x86\x97\xEF"
-			  "\x57\x0E\x63\xA1\xC1\x41\x48\xB8"
-			  "\x98\xB7\x51\x6D\x18\xF6\x19\x82"
-			  "\x37\x49\x88\xA4\xEF\x91\x21\x47"
-			  "\x03\x28\xEA\x42\xF4\xFB\x7A\x58"
-			  "\x28\x90\x77\x46\xD8\xD2\x35\x16"
-			  "\x44\xA9\x9E\x49\x52\x2A\xE4\x16"
-			  "\x5D\xF7\x65\xEB\x0F\xC9\x29\xE6"
-			  "\xCF\x76\x91\x89\x8A\x94\x39\xFA"
-			  "\x6B\x5F\x63\x53\x74\x43\x91\xF5"
-			  "\x3F\xBC\x88\x53\xB2\x1A\x02\x3F"
-			  "\x9D\x32\x84\xEB\x56\x28\xD6\x06"
-			  "\xD5\xB2\x20\xA9\xFC\xC3\x76\x62"
-			  "\x32\xCC\x86\xC8\x36\x67\x5E\x7E"
-			  "\xA4\xAA\x15\x63\x6B\xA9\x86\xAF"
-			  "\x1A\x52\x82\x36\x5F\xF4\x3F\x7A"
-			  "\x9B\x78\x62\x3B\x02\x28\x60\xB3"
-			  "\xBA\x82\xB1\xDD\xC9\x60\x8F\x47"
-			  "\xF1\x6B\xFE\xE5\x39\x34\xA0\x28"
-			  "\xA4\xB3\xC9\x7E\xED\x28\x8D\x70"
-			  "\xB2\x1D\xFD\xC6\x00\xCF\x1A\x94"
-			  "\x28\xF8\xC1\x34\xB7\x58\xA5\x6C"
-			  "\x1A\x9D\xE4\xE4\xF6\xB9\xB4\xB0"
-			  "\x5D\x51\x54\x9A\x53\xA0\xF9\x32"
-			  "\xBD\x31\x54\x14\x7B\x33\xEE\x17"
-			  "\xD3\xC7\x1F\x48\xBF\x0B\x22\xA2"
-			  "\x7D\x0C\xDF\xD0\x2E\x98\xFA\xD2"
-			  "\xFA\xCF\x24\x1D\x99\x9B\xD0\x7E"
-			  "\xF4\x4F\x88\xFF\x45\x99\x4A\xF4"
-			  "\xF2\x0A\x5B\x3B\x21\xAB\x92\xAE"
-			  "\x40\x78\x91\x95\xC4\x2F\xA3\xE8"
-			  "\x18\xC7\x07\xA6\xC8\xC0\x66\x33"
-			  "\x35\xC0\xB4\xA0\xF8\xEE\x1E\xF3"
-			  "\x40\xF5\x40\x54\xF1\x84\x8C\xEA"
-			  "\x27\x38\x1F\xF8\x77\xC7\xDF\xD8"
-			  "\x1D\xE2\xD9\x59\x40\x4F\x59\xD4"
-			  "\xF8\x17\x99\x8D\x58\x2D\x72\x44"
-			  "\x9D\x1D\x91\x64\xD6\x3F\x0A\x82"
-			  "\xC7\x57\x3D\xEF\xD3\x41\xFA\xA7"
-			  "\x68\xA3\xB8\xA5\x93\x74\x2E\x85"
-			  "\x4C\x9D\x69\x59\xCE\x15\xAE\xBF"
-			  "\x9C\x8F\x14\x64\x5D\x7F\xCF\x0B"
-			  "\xCE\x43\x5D\x28\xC0\x2F\xFB\x18"
-			  "\x79\x9A\xFC\x43\x16\x7C\x6B\x7B"
-			  "\x38\xB8\x48\x36\x66\x4E\x20\x43"
-			  "\xBA\x76\x13\x9A\xC3\xF2\xEB\x52"
-			  "\xD7\xDC\xB2\x67\x63\x14\x25\xCD"
-			  "\xB1\x13\x4B\xDE\x8C\x59\x21\x84"
-			  "\x81\x8D\x97\x23\x45\x33\x7C\xF3"
-			  "\xC5\xBC\x79\x95\xAA\x84\x68\x31"
-			  "\x2D\x1A\x68\xFE\xEC\x92\x94\xDA"
-			  "\x94\x2A\x6F\xD6\xFE\xE5\x76\x97"
-			  "\xF4\x6E\xEE\xCB\x2B\x95\x4E\x36"
-			  "\x5F\x74\x8C\x86\x5B\x71\xD0\x20"
-			  "\x78\x1A\x7F\x18\x8C\xD9\xCD\xF5"
-			  "\x21\x41\x56\x72\x13\xE1\x86\x07"
-			  "\x07\x26\xF3\x4F\x7B\xEA\xB5\x18"
-			  "\xFE\x94\x2D\x9F\xE0\x72\x18\x65"
-			  "\xB2\xA5\x63\x48\xB4\x13\x22\xF7"
-			  "\x25\xF1\x80\xA8\x7F\x54\x86\x7B"
-			  "\x39\xAE\x95\x0C\x09\x32\x22\x2D"
-			  "\x4D\x73\x39\x0C\x09\x2C\x7C\x10"
-			  "\xD0\x4B\x53\xF6\x90\xC5\x99\x2F"
-			  "\x15\xE1\x7F\xC6\xC5\x7A\x52\x14"
-			  "\x65\xEE\x93\x54\xD0\x66\x15\x3C"
-			  "\x4C\x68\xFD\x64\x0F\xF9\x10\x39"
-			  "\x46\x7A\xDD\x97\x20\xEE\xC7\xD2"
-			  "\x98\x4A\xB6\xE6\xF5\xA8\x1F\x4F"
-			  "\xDB\xAB\x6D\xD5\x9B\x34\x16\x97"
-			  "\x2F\x64\xE5\x37\xEF\x0E\xA1\xE9"
-			  "\xBE\x31\x31\x96\x8B\x40\x18\x75"
-			  "\x11\x75\x14\x32\xA5\x2D\x1B\x6B"
-			  "\xDB\x59\xEB\xFA\x3D\x8E\x7C\xC4"
-			  "\xDE\x68\xC8\x9F\xC9\x99\xE3\xC6"
-			  "\x71\xB0\x12\x57\x89\x0D\xC0\x2B"
-			  "\x9F\x12\x6A\x04\x67\xF1\x95\x31"
-			  "\x59\xFD\x84\x95\x2C\x9C\x5B\xEC"
-			  "\x09\xB0\x43\x96\x4A\x64\x80\x40"
-			  "\xB9\x72\x19\xDD\x70\x42\xFA\xB1"
-			  "\x4A\x2C\x0C\x0A\x60\x6E\xE3\x7C"
-			  "\x37\x5A\xBE\xA4\x62\xCF\x29\xAB"
-			  "\x7F\x4D\xA6\xB3\xE2\xB6\x64\xC6"
-			  "\x33\x0B\xF3\xD5\x01\x38\x74\xA4"
-			  "\x67\x1E\x75\x68\xC3\xAD\x76\xE9"
-			  "\xE9\xBC\xF0\xEB\xD8\xFD\x31\x8A"
-			  "\x5F\xC9\x18\x94\x4B\x86\x66\xFC"
-			  "\xBD\x0B\x3D\xB3\x9F\xFA\x1F\xD9"
-			  "\x78\xC4\xE3\x24\x1C\x67\xA2\xF8"
-			  "\x43\xBC\x76\x75\xBF\x6C\x05\xB3"
-			  "\x32\xE8\x7C\x80\xDB\xC7\xB6\x61"
-			  "\x1A\x3E\x2B\xA7\x25\xED\x8F\xA0"
-			  "\x00\x4B\xF8\x90\xCA\xD8\xFB\x12"
-			  "\xAC\x1F\x18\xE9\xD2\x5E\xA2\x8E"
-			  "\xE4\x84\x6B\x9D\xEB\x1E\x6B\xA3"
-			  "\x7B\xDC\xCE\x15\x97\x27\xB2\x65"
-			  "\xBC\x0E\x47\xAB\x55\x13\x53\xAB"
-			  "\x0E\x34\x55\x02\x5F\x27\xC5\x89"
-			  "\xDF\xC5\x70\xC4\xDD\x76\x82\xEE"
-			  "\x68\xA6\x09\xB0\xE5\x5E\xF1\x0C"
-			  "\xE3\xF3\x09\x9B\xFE\x65\x4B\xB8"
-			  "\x30\xEC\xD5\x7C\x6A\xEC\x1D\xD2"
-			  "\x93\xB7\xA1\x1A\x02\xD4\xC0\xD6"
-			  "\x8D\x4D\x83\x9A\xED\x29\x4E\x14"
-			  "\x86\xD5\x3C\x1A\xD5\xB9\x0A\x6A"
-			  "\x72\x22\xD5\x92\x38\xF1\xA1\x86"
-			  "\xB2\x41\x51\xCA\x4E\xAB\x8F\xD3"
-			  "\x80\x56\xC3\xD7\x65\xE1\xB3\x86"
-			  "\xCB\xCE\x98\xA1\xD4\x59\x1C\x06"
-			  "\x01\xED\xF8\x29\x91\x19\x5C\x9A"
-			  "\xEE\x28\x1B\x48\xD7\x32\xEF\x9F"
-			  "\x6C\x2B\x66\x4E\x78\xD5\x8B\x72"
-			  "\x80\xE7\x29\xDC\x23\x55\x98\x54"
-			  "\xB1\xFF\x3E\x95\x56\xA8\x78\x78"
-			  "\xEF\xC4\xA5\x11\x2D\x2B\xD8\x93"
-			  "\x30\x6E\x7E\x51\xBB\x42\x5F\x03"
-			  "\x43\x94\x23\x7E\xEE\xF0\xA5\x79"
-			  "\x55\x01\xD4\x58\xB2\xF2\x85\x49"
-			  "\x70\xC5\xB9\x0B\x3B\x7A\x6E\x6C",
-		.ilen	= 1008,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06"
-			  "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78"
-			  "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA"
-			  "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C"
-			  "\xF3\x67\xFE\x95\x09\xA0\x37\xCE"
-			  "\x42\xD9\x70\x07\x7B\x12\xA9\x1D"
-			  "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F"
-			  "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01"
-			  "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73"
-			  "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5"
-			  "\x59\xF0\x87\x1E\x92\x29\xC0\x34"
-			  "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6"
-			  "\x3D\xD4\x48\xDF\x76\x0D\x81\x18"
-			  "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A"
-			  "\x21\x95\x2C\xC3\x37\xCE\x65\xFC"
-			  "\x70\x07\x9E\x12\xA9\x40\xD7\x4B"
-			  "\xE2\x79\x10\x84\x1B\xB2\x26\xBD"
-			  "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F"
-			  "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1"
-			  "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13"
-			  "\x87\x1E\xB5\x29\xC0\x57\xEE\x62"
-			  "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4"
-			  "\x6B\x02\x76\x0D\xA4\x18\xAF\x46"
-			  "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8"
-			  "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07"
-			  "\x9E\x35\xCC\x40\xD7\x6E\x05\x79"
-			  "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB"
-			  "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D"
-			  "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF"
-			  "\x43\xDA\x71\x08\x7C\x13\xAA\x1E"
-			  "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90"
-			  "\x27\xBE\x32\xC9\x60\xF7\x6B\x02"
-			  "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74"
-			  "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6"
-			  "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35"
-			  "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7"
-			  "\x3E\xD5\x49\xE0\x77\x0E\x82\x19"
-			  "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B"
-			  "\x22\x96\x2D\xC4\x38\xCF\x66\xFD"
-			  "\x71\x08\x9F\x13\xAA\x41\xD8\x4C"
-			  "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE"
-			  "\x55\xEC\x60\xF7\x8E\x02\x99\x30"
-			  "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2"
-			  "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14"
-			  "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63"
-			  "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5"
-			  "\x6C\x03\x77\x0E\xA5\x19\xB0\x47"
-			  "\xDE\x52\xE9\x80\x17\x8B\x22\xB9"
-			  "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08"
-			  "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A"
-			  "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC"
-			  "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E"
-			  "\xF5\x69\x00\x97\x0B\xA2\x39\xD0"
-			  "\x44\xDB\x72\x09\x7D\x14\xAB\x1F"
-			  "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91"
-			  "\x28\xBF\x33\xCA\x61\xF8\x6C\x03"
-			  "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75"
-			  "\x0C\x80\x17\xAE\x22\xB9\x50\xE7"
-			  "\x5B\xF2\x89\x20\x94\x2B\xC2\x36"
-			  "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8"
-			  "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A"
-			  "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C"
-			  "\x00\x97\x2E\xC5\x39\xD0\x67\xFE"
-			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D",
-		.rlen	= 1008,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 1008 - 20, 4, 16 },
+		.iv	= "\xd8\x70\x9f\x42\xca\xe6\x83\x3a"
+			  "\x52\x79\x42\xa5\x84\x6a\x96\x7f"
+			  "\xe4\x8f\xed\x97\xe9\xd0\xe8\x0d"
+			  "\x7c\xa6\xd8\xd4\x77\x9b\x40\x2e",
+		.assoc	= "\xe8\x39\x2d\xaa\x0e\x85\x2d\xc1"
+			  "\x72\xaf\x6e\xc9\x82\x33\xc7\x01"
+			  "\xaf\x40\x70\xb8\x2a\x78\xc9\x14"
+			  "\xac\xb1\x10\xca\x2e\xb3\x28\xe4"
+			  "\xac\xfa\x58\x7f\xe5\x73\x09\x8c"
+			  "\x1d\x40\x87\x8c\xd9\x75\xc0\x55"
+			  "\xa2\xda\x07\xd1\xc2\xa9\xd1\xbb"
+			  "\x09\x4f\x77\x62\x88\x2d\xf2\x68"
+			  "\x54",
+		.alen	= 65,
+		.input	= "\xf7\x02\xbb\x11\x52\x24\xd8\x48"
+			  "\x93\xe6\x9b\xee\x81\xfc\xf7\x82"
+			  "\x79\xf0\xf3\xd9\x6c\x20\xa9\x1a"
+			  "\xdc\xbc\x47\xc0\xe4\xcb\x10\x99"
+			  "\x2f",
+		.ilen	= 33,
+		.result	= "\x33\xc1\xda\xfa\x15\x21\x07\x8e"
+			  "\x93\x68\xea\x64\x7b\x3d\x4b\x6b"
+			  "\x71\x5e\x5e\x6b\x92\xaa\x65\xc2"
+			  "\x7a\x2a\xc1\xa9\x0a\xa1\x24\x81"
+			  "\x26\x3a\x5a\x09\xe8\xce\x73\x72"
+			  "\xde\x7b\x58\x9e\x85\xb9\xa4\x28"
+			  "\xda",
+		.rlen	= 49,
+	}, {
+		.key	= "\x06\xcc\x4a\x79\x96\xc3\x82\xcf"
+			  "\xb3\x1c\xc7\x12\x7f\xc5\x28\x04"
+			  "\x44\xa1\x76\xfb\xad\xc8\x8a\x21"
+			  "\x0d\xc8\x7f\xb6\x9b\xe3\xf8\x4f",
+		.klen	= 32,
+		.iv	= "\x15\x95\xd8\xe1\xda\x62\x2c\x56"
+			  "\xd3\x53\xf4\x36\x7e\x8e\x59\x85"
+			  "\x0e\x51\xf9\x1c\xee\x70\x6a\x27"
+			  "\x3d\xd3\xb7\xac\x51\xfa\xdf\x05",
+		.assoc	= "\x24\x5e\x67\x49\x1e\x01\xd6\xdd"
+			  "\xf3\x89\x20\x5b\x7c\x57\x89\x07",
+		.alen	= 16,
+		.input	= "\x33\x27\xf5\xb1\x62\xa0\x80\x63"
+			  "\x14\xc0\x4d\x7f\x7b\x20\xba\x89",
+		.ilen	= 16,
+		.result	= "\x3e\xf8\x86\x3d\x39\xf8\x96\x02"
+			  "\x0f\xdf\xc9\x6e\x37\x1e\x57\x99"
+			  "\x07\x2a\x1a\xac\xd1\xda\xfd\x3b"
+			  "\xc7\xff\xbd\xbc\x85\x09\x0b",
+		.rlen	= 31,
+	}, {
+		.key	= "\x42\xf0\x84\x19\xa6\x3f\x2b\xea"
+			  "\x34\xf6\x79\xa3\x79\xe9\xeb\x0a"
+			  "\x6e\x63\x82\x7f\xb2\x68\x0c\x3a"
+			  "\xce\xf5\x5e\x8e\x75\x42\x97\x26",
+		.klen	= 32,
+		.iv	= "\x51\xb9\x12\x80\xea\xde\xd5\x71"
+			  "\x54\x2d\xa6\xc8\x78\xb2\x1b\x8c"
+			  "\x39\x14\x05\xa0\xf3\x10\xec\x41"
+			  "\xff\x01\x95\x84\x2b\x59\x7f\xdb",
+		.assoc	= "\x61\x83\xa0\xe8\x2e\x7d\x7f\xf8"
+			  "\x74\x63\xd2\xec\x76\x7c\x4c\x0d",
+		.alen	= 16,
+		.input	= "\x70\x4c\x2f\x50\x72\x1c\x29\x7f"
+			  "\x95\x9a\xff\x10\x75\x45\x7d\x8f",
+		.ilen	= 16,
+		.result	= "\x2f\xc4\xd8\x0d\xa6\x07\xef\x2e"
+			  "\x6c\xd9\x84\x63\x70\x97\x61\x37"
+			  "\x08\x2f\x16\x90\x9e\x62\x30\x0d"
+			  "\x62\xd5\xc8\xf0\x46\x1a",
+		.rlen	= 30,
+	}, {
+		.key	= "\x7f\x15\xbd\xb8\xb6\xba\xd3\x06"
+			  "\xb5\xd1\x2b\x35\x73\x0e\xad\x10"
+			  "\x98\x25\x8d\x03\xb7\x08\x8e\x54"
+			  "\x90\x23\x3d\x67\x4f\xa1\x36\xfc",
+		.klen	= 32,
+		.iv	= "\x8e\xde\x4c\x20\xfa\x59\x7e\x8d"
+			  "\xd5\x07\x58\x59\x72\xd7\xde\x92"
+			  "\x63\xd6\x10\x24\xf8\xb0\x6e\x5a"
+			  "\xc0\x2e\x74\x5d\x06\xb8\x1e\xb2",
+		.assoc	= "\x9d\xa7\xda\x88\x3e\xf8\x28\x14"
+			  "\xf5\x3e\x85\x7d\x70\xa0\x0f\x13",
+		.alen	= 16,
+		.input	= "\xac\x70\x69\xef\x82\x97\xd2\x9b"
+			  "\x15\x74\xb1\xa2\x6f\x69\x3f\x95",
+		.ilen	= 16,
+		.result	= "\xce\xf3\x17\x87\x49\xc2\x00\x46"
+			  "\xc6\x12\x5c\x8f\x81\x38\xaa\x55"
+			  "\xf8\x67\x75\xf1\x75\xe3\x2a\x24",
+		.rlen	= 24,
+	},
+};
+
+static const struct aead_testvec aegis256_dec_tv_template[] = {
+	{
+		.key	= "\x0f\xc9\x8e\x67\x44\x9e\xaa\x86"
+			  "\x20\x36\x2c\x24\xfe\xc9\x30\x81"
+			  "\xca\xb0\x82\x21\x41\xa8\xe0\x06"
+			  "\x30\x0b\x37\xf6\xb6\x17\xe7\xb5",
+		.klen	= 32,
+		.iv	= "\x1e\x92\x1c\xcf\x88\x3d\x54\x0d"
+			  "\x40\x6d\x59\x48\xfc\x92\x61\x03"
+			  "\x95\x61\x05\x42\x82\x50\xc0\x0c"
+			  "\x60\x16\x6f\xec\x6d\x2f\xcf\x6b",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xd5\x65\x3a\xa9\x03\x51\xd7\xaa"
+			  "\xfa\x4b\xd8\xa2\x41\x9b\xc1\xb2",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x4b\xed\xc8\x07\x54\x1a\x52\xa2"
+			  "\xa1\x10\xde\xb5\xf8\xed\xf3\x87"
+			  "\xf4\x72\x8e\xa5\x46\x48\x62\x20"
+			  "\xf1\x38\x16\xce\x90\x76\x87\x8c",
+		.klen	= 32,
+		.iv	= "\x5a\xb7\x56\x6e\x98\xb9\xfd\x29"
+			  "\xc1\x47\x0b\xda\xf6\xb6\x23\x09"
+			  "\xbf\x23\x11\xc6\x87\xf0\x42\x26"
+			  "\x22\x44\x4e\xc4\x47\x8e\x6e\x41",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x84\xa2\x8f\xad\xdb\x8d\x2c\x16"
+			  "\x9e\x89\xd9\x06\xa6\xa8\x14\x29"
+			  "\x8b",
+		.ilen	= 17,
+		.result	= "\x79",
+		.rlen	= 1,
+	}, {
+		.key	= "\x88\x12\x01\xa6\x64\x96\xfb\xbe"
+			  "\x22\xea\x90\x47\xf2\x11\xb5\x8e"
+			  "\x1f\x35\x9a\x29\x4b\xe8\xe4\x39"
+			  "\xb3\x66\xf5\xa6\x6a\xd5\x26\x62",
+		.klen	= 32,
+		.iv	= "\x97\xdb\x90\x0e\xa8\x35\xa5\x45"
+			  "\x42\x21\xbd\x6b\xf0\xda\xe6\x0f"
+			  "\xe9\xe5\x1d\x4a\x8c\x90\xc4\x40"
+			  "\xe3\x71\x2d\x9c\x21\xed\x0e\x18",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x09\x94\x1f\xa6\x13\xc3\x74\x75"
+			  "\x17\xad\x8a\x0e\xd8\x66\x9a\x28"
+			  "\xd7\x30\x66\x09\x2a\xdc\xfa\x2a"
+			  "\x9f\x3b\xd7\xdd\x66\xd1\x2b",
+		.ilen	= 31,
+		.result	= "\xb5\x6e\xad\xdd\x30\x72\xfa\x53"
+			  "\x82\x8e\x16\xb4\xed\x6d\x47",
+		.rlen	= 15,
+	}, {
+		.key	= "\xc4\x37\x3b\x45\x74\x11\xa4\xda"
+			  "\xa2\xc5\x42\xd8\xec\x36\x78\x94"
+			  "\x49\xf7\xa5\xad\x50\x88\x66\x53"
+			  "\x74\x94\xd4\x7f\x44\x34\xc5\x39",
+		.klen	= 32,
+		.iv	= "\xd3\x00\xc9\xad\xb8\xb0\x4e\x61"
+			  "\xc3\xfb\x6f\xfd\xea\xff\xa9\x15"
+			  "\x14\xa8\x28\xce\x92\x30\x46\x59"
+			  "\xa4\x9f\x0b\x75\xfb\x4c\xad\xee",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x8a\x46\xa2\x22\x8c\x03\xab\x6f"
+			  "\x54\x63\x4e\x7f\xc9\x8e\xfa\x70"
+			  "\x7b\xe5\x8d\x78\xbc\xe9\xb6\xa1"
+			  "\x29\x17\xc8\x3b\x52\xa4\x98\x72",
+		.ilen	= 32,
+		.result	= "\xf2\x92\xe6\x7d\x40\xee\xa3\x6f"
+			  "\x03\x68\xc8\x45\xe7\x91\x0a\x18",
+		.rlen	= 16,
+	}, {
+		.key	= "\x01\x5c\x75\xe5\x84\x8d\x4d\xf6"
+			  "\x23\x9f\xf4\x6a\xe6\x5a\x3b\x9a"
+			  "\x74\xb9\xb1\x32\x55\x28\xe8\x6d"
+			  "\x35\xc1\xb3\x57\x1f\x93\x64\x0f",
+		.klen	= 32,
+		.iv	= "\x10\x25\x03\x4c\xc8\x2c\xf7\x7d"
+			  "\x44\xd5\x21\x8e\xe4\x23\x6b\x1c"
+			  "\x3e\x6a\x34\x53\x97\xd0\xc8\x73"
+			  "\x66\xcd\xea\x4d\xd5\xab\x4c\xc5",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x71\x6b\x37\x0b\x02\x61\x28\x12"
+			  "\x83\xab\x66\x90\x84\xc7\xd1\xc5"
+			  "\xb2\x7a\xb4\x7b\xb4\xfe\x02\xb2"
+			  "\xc0\x00\x39\x13\xb5\x51\x68\x44"
+			  "\xad",
+		.ilen	= 33,
+		.result	= "\x2e\xb7\x20\x1c\x50\x6a\x4b\x8b"
+			  "\x84\x42\x7a\xd7\xe1\xb5\xcd\x1f"
+			  "\xd3",
+		.rlen	= 17,
+	}, {
+		.key	= "\x3d\x80\xae\x84\x94\x09\xf6\x12"
+			  "\xa4\x79\xa6\xfb\xe0\x7f\xfd\xa0"
+			  "\x9e\x7c\xbc\xb6\x5b\xc8\x6a\x86"
+			  "\xf7\xef\x91\x30\xf9\xf2\x04\xe6",
+		.klen	= 32,
+		.iv	= "\x4c\x49\x3d\xec\xd8\xa8\xa0\x98"
+			  "\xc5\xb0\xd3\x1f\xde\x48\x2e\x22"
+			  "\x69\x2c\x3f\xd7\x9c\x70\x4a\x8d"
+			  "\x27\xfa\xc9\x26\xaf\x0a\xeb\x9c",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xaf\xa4\x34\x0d\x59\xe6\x1c\x2f"
+			  "\x06\x3b\x52\x18\x49\x75\x1b\xf0"
+			  "\x53\x09\x72\x7b\x45\x79\xe0\xbe"
+			  "\x89\x85\x23\x15\xb8\x79\x07\x4c"
+			  "\x53\x7a\x15\x37\x0a\xee\xb7\xfb"
+			  "\xc4\x1f\x12\x27\xcf\x77\x90",
+		.ilen	= 47,
+		.result	= "\x6b\xdc\x5a\xbb\x60\xe5\xf4\xa6"
+			  "\x05\x1d\x2c\x68\xdb\xda\x8f\x25"
+			  "\xfe\x8d\x45\x19\x1e\xc0\x0b\x99"
+			  "\x88\x11\x39\x12\x1c\x3a\xbb",
+		.rlen	= 31,
+	}, {
+		.key	= "\x7a\xa5\xe8\x23\xa4\x84\x9e\x2d"
+			  "\x25\x53\x58\x8c\xda\xa3\xc0\xa6"
+			  "\xc8\x3e\xc8\x3a\x60\x68\xec\xa0"
+			  "\xb8\x1c\x70\x08\xd3\x51\xa3\xbd",
+		.klen	= 32,
+		.iv	= "\x89\x6e\x77\x8b\xe8\x23\x49\xb4"
+			  "\x45\x8a\x85\xb1\xd8\x6c\xf1\x28"
+			  "\x93\xef\x4b\x5b\xa1\x10\xcc\xa6"
+			  "\xe8\x28\xa8\xfe\x89\x69\x8b\x72",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xe2\xc9\x0b\x33\x31\x02\xb3\xb4"
+			  "\x33\xfe\xeb\xa8\xb7\x9b\xb2\xd7"
+			  "\xeb\x0f\x05\x2b\xba\xb3\xca\xef"
+			  "\xf6\xd1\xb6\xc0\xb9\x9b\x85\xc5"
+			  "\xbf\x7a\x3e\xcc\x31\x76\x09\x80"
+			  "\x32\x5d\xbb\xe8\x38\x0e\x77\xd3",
+		.ilen	= 48,
+		.result	= "\xa7\x00\x93\x5b\x70\x61\x9d\xc2"
+			  "\x86\xf7\xde\xfa\xd5\xfe\x52\x2b"
+			  "\x28\x50\x51\x9d\x24\x60\x8d\xb3"
+			  "\x49\x3e\x17\xea\xf6\x99\x5a\xdd",
+		.rlen	= 32,
+	}, {
+		.key	= "\xb6\xca\x22\xc3\xb4\x00\x47\x49"
+			  "\xa6\x2d\x0a\x1e\xd4\xc7\x83\xad"
+			  "\xf3\x00\xd4\xbf\x65\x08\x6e\xb9"
+			  "\x7a\x4a\x4f\xe0\xad\xb0\x42\x93",
+		.klen	= 32,
+		.iv	= "\xc5\x93\xb0\x2a\xf8\x9f\xf1\xd0"
+			  "\xc6\x64\x37\x42\xd2\x90\xb3\x2e"
+			  "\xbd\xb1\x57\xe0\xa6\xb0\x4e\xc0"
+			  "\xaa\x55\x87\xd6\x63\xc8\x2a\x49",
+		.assoc	= "\xd5",
+		.alen	= 1,
+		.input	= "\x96\x43\x30\xca\x6c\x4f\xd7\x12"
+			  "\xba\xd9\xb3\x18\x86\xdf\xc3\x52",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xf3\xee\x5c\x62\xc4\x7c\xf0\x65"
+			  "\x27\x08\xbd\xaf\xce\xec\x45\xb3"
+			  "\x1d\xc3\xdf\x43\x6a\xa8\xf0\xd3"
+			  "\x3b\x77\x2e\xb9\x87\x0f\xe1\x6a",
+		.klen	= 32,
+		.iv	= "\x02\xb8\xea\xca\x09\x1b\x9a\xec"
+			  "\x47\x3e\xe9\xd4\xcc\xb5\x76\x34"
+			  "\xe8\x73\x62\x64\xab\x50\xd0\xda"
+			  "\x6b\x83\x66\xaf\x3e\x27\xc9\x1f",
+		.assoc	= "\x11\x81\x78\x32\x4d\xb9\x44\x73"
+			  "\x68\x75\x16\xf8\xcb\x7e\xa7",
+		.alen	= 15,
+		.input	= "\x2f\xab\x45\xe2\xa7\x46\xc5\x83"
+			  "\x11\x9f\xb0\x74\xee\xc7\x03\xdd",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x2f\x13\x95\x01\xd5\xf7\x99\x81"
+			  "\xa8\xe2\x6f\x41\xc8\x10\x08\xb9"
+			  "\x47\x85\xeb\xc7\x6f\x48\x72\xed"
+			  "\xfc\xa5\x0d\x91\x61\x6e\x81\x40",
+		.klen	= 32,
+		.iv	= "\x3f\xdc\x24\x69\x19\x96\x43\x08"
+			  "\xc8\x18\x9b\x65\xc6\xd9\x39\x3b"
+			  "\x12\x35\x6e\xe8\xb0\xf0\x52\xf3"
+			  "\x2d\xb0\x45\x87\x18\x86\x68\xf6",
+		.assoc	= "\x4e\xa5\xb2\xd1\x5d\x35\xed\x8f"
+			  "\xe8\x4f\xc8\x89\xc5\xa2\x69\xbc",
+		.alen	= 16,
+		.input	= "\x16\x44\x73\x33\x5d\xf2\xb9\x04"
+			  "\x6b\x79\x98\xef\xdb\xd5\xc5\xf1",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x6c\x38\xcf\xa1\xe5\x73\x41\x9d"
+			  "\x29\xbc\x21\xd2\xc2\x35\xcb\xbf"
+			  "\x72\x47\xf6\x4b\x74\xe8\xf4\x06"
+			  "\xbe\xd3\xec\x6a\x3b\xcd\x20\x17",
+		.klen	= 32,
+		.iv	= "\x7b\x01\x5d\x08\x29\x12\xec\x24"
+			  "\x49\xf3\x4d\xf7\xc0\xfe\xfb\x41"
+			  "\x3c\xf8\x79\x6c\xb6\x90\xd4\x0d"
+			  "\xee\xde\x23\x60\xf2\xe5\x08\xcc",
+		.assoc	= "\x8a\xca\xec\x70\x6d\xb1\x96\xab"
+			  "\x69\x29\x7a\x1b\xbf\xc7\x2c\xc2"
+			  "\x07",
+		.alen	= 17,
+		.input	= "\xa4\x9b\xb8\x47\xc0\xed\x7a\x45"
+			  "\x98\x54\x8c\xed\x3d\x17\xf0\xdd",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xa8\x5c\x09\x40\xf5\xef\xea\xb8"
+			  "\xaa\x96\xd3\x64\xbc\x59\x8d\xc6"
+			  "\x9c\x0a\x02\xd0\x79\x88\x76\x20"
+			  "\x7f\x00\xca\x42\x15\x2c\xbf\xed",
+		.klen	= 32,
+		.iv	= "\xb8\x26\x97\xa8\x39\x8e\x94\x3f"
+			  "\xca\xcd\xff\x88\xba\x22\xbe\x47"
+			  "\x67\xba\x85\xf1\xbb\x30\x56\x26"
+			  "\xaf\x0b\x02\x38\xcc\x44\xa7\xa3",
+		.assoc	= "\xc7\xef\x26\x10\x7d\x2c\x3f\xc6"
+			  "\xea\x03\x2c\xac\xb9\xeb\xef\xc9"
+			  "\x31\x6b\x08\x12\xfc\xd8\x37\x2d"
+			  "\xe0\x17\x3a\x2e\x83\x5c\x8f",
+		.alen	= 31,
+		.input	= "\x20\x24\xe2\x33\x5c\x60\xc9\xf0"
+			  "\xa4\x96\x2f\x0d\x53\xc2\xf8\xfc",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xe5\x81\x42\xdf\x05\x6a\x93\xd4"
+			  "\x2b\x70\x85\xf5\xb6\x7d\x50\xcc"
+			  "\xc6\xcc\x0e\x54\x7f\x28\xf8\x3a"
+			  "\x40\x2e\xa9\x1a\xf0\x8b\x5e\xc4",
+		.klen	= 32,
+		.iv	= "\xf4\x4a\xd1\x47\x49\x09\x3d\x5b"
+			  "\x4b\xa7\xb1\x19\xb4\x46\x81\x4d"
+			  "\x91\x7c\x91\x75\xc0\xd0\xd8\x40"
+			  "\x71\x39\xe1\x10\xa6\xa3\x46\x7a",
+		.assoc	= "\x03\x14\x5f\xaf\x8d\xa8\xe7\xe2"
+			  "\x6b\xde\xde\x3e\xb3\x10\xb1\xcf"
+			  "\x5c\x2d\x14\x96\x01\x78\xb9\x47"
+			  "\xa1\x44\x19\x06\x5d\xbb\x2e\x2f",
+		.alen	= 32,
+		.input	= "\x6f\x4a\xb9\xe0\xff\x51\xa3\xf1"
+			  "\xd2\x64\x3e\x66\x6a\xb2\x03\xc0",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x22\xa6\x7c\x7f\x15\xe6\x3c\xf0"
+			  "\xac\x4b\x37\x86\xb0\xa2\x13\xd2"
+			  "\xf1\x8e\x19\xd8\x84\xc8\x7a\x53"
+			  "\x02\x5b\x88\xf3\xca\xea\xfe\x9b",
+		.klen	= 32,
+		.iv	= "\x31\x6f\x0b\xe6\x59\x85\xe6\x77"
+			  "\xcc\x81\x63\xab\xae\x6b\x43\x54"
+			  "\xbb\x3f\x9c\xf9\xc5\x70\x5a\x5a"
+			  "\x32\x67\xc0\xe9\x80\x02\xe5\x50",
+		.assoc	= "\x40",
+		.alen	= 1,
+		.input	= "\x2c\xfb\xad\x7e\xbe\xa0\x9a\x5b"
+			  "\x7a\x3f\x81\xf7\xfc\x1b\x79\x83"
+			  "\xc7",
+		.ilen	= 17,
+		.result	= "\x4f",
+		.rlen	= 1,
+	}, {
+		.key	= "\x5e\xcb\xb6\x1e\x25\x62\xe4\x0c"
+			  "\x2d\x25\xe9\x18\xaa\xc6\xd5\xd8"
+			  "\x1b\x50\x25\x5d\x89\x68\xfc\x6d"
+			  "\xc3\x89\x67\xcb\xa4\x49\x9d\x71",
+		.klen	= 32,
+		.iv	= "\x6d\x94\x44\x86\x69\x00\x8f\x93"
+			  "\x4d\x5b\x15\x3c\xa8\x8f\x06\x5a"
+			  "\xe6\x01\xa8\x7e\xca\x10\xdc\x73"
+			  "\xf4\x94\x9f\xc1\x5a\x61\x85\x27",
+		.assoc	= "\x7c\x5d\xd3\xee\xad\x9f\x39\x1a"
+			  "\x6d\x92\x42\x61\xa7\x58\x37",
+		.alen	= 15,
+		.input	= "\x1f\x7f\xca\x3c\x2b\xe7\x27\xba"
+			  "\x7e\x98\x83\x02\x34\x23\xf7\x94"
+			  "\xde\x35\xe6\x1d\x14\x18\xe5\x38"
+			  "\x14\x80\x6a\xa7\x1b\xae\x1d",
+		.ilen	= 31,
+		.result	= "\x8b\x26\x61\x55\xf1\x3e\xe3\xa1"
+			  "\x8d\xc8\x6e\x85\xa5\x21\x67",
+		.rlen	= 15,
+	}, {
+		.key	= "\x9b\xef\xf0\xbd\x35\xdd\x8d\x28"
+			  "\xad\xff\x9b\xa9\xa4\xeb\x98\xdf"
+			  "\x46\x13\x31\xe1\x8e\x08\x7e\x87"
+			  "\x85\xb6\x46\xa3\x7e\xa8\x3c\x48",
+		.klen	= 32,
+		.iv	= "\xaa\xb8\x7e\x25\x79\x7c\x37\xaf"
+			  "\xce\x36\xc7\xce\xa2\xb4\xc9\x60"
+			  "\x10\xc3\xb3\x02\xcf\xb0\x5e\x8d"
+			  "\xb5\xc2\x7e\x9a\x35\xc0\x24\xfd",
+		.assoc	= "\xb9\x82\x0c\x8d\xbd\x1b\xe2\x36"
+			  "\xee\x6c\xf4\xf2\xa1\x7d\xf9\xe2",
+		.alen	= 16,
+		.input	= "\x05\x86\x9e\xd7\x2b\xa3\x97\x01"
+			  "\xbe\x28\x98\x10\x6f\xe9\x61\x32"
+			  "\x96\xbb\xb1\x2e\x8f\x0c\x44\xb9"
+			  "\x46\x2d\x55\xe3\x42\x67\xf2\xaf",
+		.ilen	= 32,
+		.result	= "\xc8\x4b\x9b\xf5\x01\xba\x8c\xbd"
+			  "\x0e\xa3\x21\x16\x9f\x46\x2a\x63",
+		.rlen	= 16,
+	}, {
+		.key	= "\xd7\x14\x29\x5d\x45\x59\x36\x44"
+			  "\x2e\xd9\x4d\x3b\x9e\x0f\x5b\xe5"
+			  "\x70\xd5\x3c\x65\x93\xa8\x00\xa0"
+			  "\x46\xe4\x25\x7c\x58\x08\xdb\x1e",
+		.klen	= 32,
+		.iv	= "\xe6\xdd\xb8\xc4\x89\xf8\xe0\xca"
+			  "\x4f\x10\x7a\x5f\x9c\xd8\x8b\x66"
+			  "\x3b\x86\xbf\x86\xd4\x50\xe0\xa7"
+			  "\x76\xef\x5c\x72\x0f\x1f\xc3\xd4",
+		.assoc	= "\xf5\xa6\x46\x2c\xce\x97\x8a\x51"
+			  "\x6f\x46\xa6\x83\x9b\xa1\xbc\xe8"
+			  "\x05",
+		.alen	= 17,
+		.input	= "\x9c\xe0\x06\x7b\x86\xcf\x2e\xd8"
+			  "\x45\x65\x1b\x72\x9b\xaa\xa3\x1e"
+			  "\x87\x9d\x26\xdf\xff\x81\x11\xd2"
+			  "\x47\x41\xb9\x24\xc1\x8a\xa3\x8b"
+			  "\x55",
+		.ilen	= 33,
+		.result	= "\x05\x70\xd5\x94\x12\x36\x35\xd8"
+			  "\x8f\x7d\xd3\xa8\x99\x6a\xed\x69"
+			  "\xd0",
+		.rlen	= 17,
+	}, {
+		.key	= "\x14\x39\x63\xfc\x56\xd5\xdf\x5f"
+			  "\xaf\xb3\xff\xcc\x98\x33\x1d\xeb"
+			  "\x9a\x97\x48\xe9\x98\x48\x82\xba"
+			  "\x07\x11\x04\x54\x32\x67\x7b\xf5",
+		.klen	= 32,
+		.iv	= "\x23\x02\xf1\x64\x9a\x73\x89\xe6"
+			  "\xd0\xea\x2c\xf1\x96\xfc\x4e\x6d"
+			  "\x65\x48\xcb\x0a\xda\xf0\x62\xc0"
+			  "\x38\x1d\x3b\x4a\xe9\x7e\x62\xaa",
+		.assoc	= "\x32\xcb\x80\xcc\xde\x12\x33\x6d"
+			  "\xf0\x20\x58\x15\x95\xc6\x7f\xee"
+			  "\x2f\xf9\x4e\x2c\x1b\x98\x43\xc7"
+			  "\x68\x28\x73\x40\x9f\x96\x4a",
+		.alen	= 31,
+		.input	= "\xa0\xc8\xde\x83\x0d\xc3\x4e\xd5"
+			  "\x69\x7f\x7a\xdd\x8c\x46\xda\xba"
+			  "\x0a\x5c\x0e\x7f\xac\xee\x02\xd2"
+			  "\xe5\x4b\x0a\xba\xb8\xa4\x7b\x66"
+			  "\xde\xae\xdb\xc2\xc0\x0b\xf7\x2b"
+			  "\xdf\xb8\xea\xd8\xa9\x38\xed",
+		.ilen	= 47,
+		.result	= "\x41\x94\x0e\x33\x22\xb1\xdd\xf4"
+			  "\x10\x57\x85\x39\x93\x8f\xaf\x70"
+			  "\xfa\xa9\xd0\x4d\x5c\x40\x23\xcd"
+			  "\x98\x34\xab\x37\x56\xae\x32",
+		.rlen	= 31,
+	}, {
+		.key	= "\x50\x5d\x9d\x9b\x66\x50\x88\x7b"
+			  "\x30\x8e\xb1\x5e\x92\x58\xe0\xf1"
+			  "\xc5\x5a\x53\x6e\x9d\xe8\x04\xd4"
+			  "\xc9\x3f\xe2\x2d\x0c\xc6\x1a\xcb",
+		.klen	= 32,
+		.iv	= "\x5f\x27\x2b\x03\xaa\xef\x32\x02"
+			  "\x50\xc4\xde\x82\x90\x21\x11\x73"
+			  "\x8f\x0a\xd6\x8f\xdf\x90\xe4\xda"
+			  "\xf9\x4a\x1a\x23\xc3\xdd\x02\x81",
+		.assoc	= "\x6e\xf0\xba\x6b\xee\x8e\xdc\x89"
+			  "\x71\xfb\x0a\xa6\x8f\xea\x41\xf4"
+			  "\x5a\xbb\x59\xb0\x20\x38\xc5\xe0"
+			  "\x29\x56\x52\x19\x79\xf5\xe9\x37",
+		.alen	= 32,
+		.input	= "\xd3\x68\x14\x70\x3c\x01\x43\x86"
+			  "\x02\xab\xbe\x75\xaa\xe7\xf5\x53"
+			  "\x5c\x05\xbd\x9b\x19\xbb\x2a\x61"
+			  "\x8f\x69\x05\x75\x8e\xca\x60\x0c"
+			  "\x5b\xa2\x48\x61\x32\x74\x11\x2b"
+			  "\xf6\xcf\x06\x78\x6f\x78\x1a\x4a",
+		.ilen	= 48,
+		.result	= "\x7e\xb9\x48\xd3\x32\x2d\x86\x10"
+			  "\x91\x31\x37\xcb\x8d\xb3\x72\x76"
+			  "\x24\x6b\xdc\xd1\x61\xe0\xa5\xe7"
+			  "\x5a\x61\x8a\x0f\x30\x0d\xd1\xec",
+		.rlen	= 32,
+	}, {
+		.key	= "\x8d\x82\xd6\x3b\x76\xcc\x30\x97"
+			  "\xb1\x68\x63\xef\x8c\x7c\xa3\xf7"
+			  "\xef\x1c\x5f\xf2\xa3\x88\x86\xed"
+			  "\x8a\x6d\xc1\x05\xe7\x25\xb9\xa2",
+		.klen	= 32,
+		.iv	= "\x9c\x4b\x65\xa2\xba\x6b\xdb\x1e"
+			  "\xd1\x9e\x90\x13\x8a\x45\xd3\x79"
+			  "\xba\xcd\xe2\x13\xe4\x30\x66\xf4"
+			  "\xba\x78\xf9\xfb\x9d\x3c\xa1\x58",
+		.assoc	= "\xab\x14\xf3\x0a\xfe\x0a\x85\xa5"
+			  "\xf2\xd5\xbc\x38\x89\x0e\x04\xfb"
+			  "\x84\x7d\x65\x34\x25\xd8\x47\xfa"
+			  "\xeb\x83\x31\xf1\x54\x54\x89\x0d"
+			  "\x9d",
+		.alen	= 33,
+		.input	= "\x07\x0a\x35\xb0\x82\x03\x5a\xd2"
+			  "\x15\x3a\x6c\x72\x83\x9b\xb1\x75"
+			  "\xea\xf2\xfc\xff\xc6\xf1\x13\xa4"
+			  "\x1a\x93\x33\x79\x97\x82\x81\xc0"
+			  "\x96\xc2\x00\xab\x39\xae\xa1\x62"
+			  "\x53\xa3\x86\xc9\x07\x8c\xaf\x22"
+			  "\x47\x31\x29\xca\x4a\x95\xf5\xd5"
+			  "\x20\x63\x5a\x54\x80\x2c\x4a\x63"
+			  "\xfb\x18\x73\x31\x4f\x08\x21\x5d"
+			  "\x20\xe9\xc3\x7e\xea\x25\x77\x3a"
+			  "\x65",
+		.ilen	= 81,
+		.result	= "\xba\xde\x82\x72\x42\xa9\x2f\x2c"
+			  "\x12\x0b\xe9\x5c\x87\xd7\x35\x7c"
+			  "\x4f\x2e\xe8\x55\x66\x80\x27\x00"
+			  "\x1b\x8f\x68\xe7\x0a\x6c\x71\xc3"
+			  "\x21\x78\x55\x9d\x9c\x65\x7b\xcd"
+			  "\x0a\x34\x97\xff\x47\x37\xb0\x2a"
+			  "\x80\x0d\x19\x98\x33\xa9\x7a\xe3"
+			  "\x2e\x4c\xc6\xf3\x8c\x88\x42\x01"
+			  "\xbd",
+		.rlen	= 65,
+	}, {
+		.key	= "\xc9\xa7\x10\xda\x86\x48\xd9\xb3"
+			  "\x32\x42\x15\x80\x85\xa1\x65\xfe"
+			  "\x19\xde\x6b\x76\xa8\x28\x08\x07"
+			  "\x4b\x9a\xa0\xdd\xc1\x84\x58\x79",
+		.klen	= 32,
+		.iv	= "\xd8\x70\x9f\x42\xca\xe6\x83\x3a"
+			  "\x52\x79\x42\xa5\x84\x6a\x96\x7f"
+			  "\xe4\x8f\xed\x97\xe9\xd0\xe8\x0d"
+			  "\x7c\xa6\xd8\xd4\x77\x9b\x40\x2e",
+		.assoc	= "\xe8\x39\x2d\xaa\x0e\x85\x2d\xc1"
+			  "\x72\xaf\x6e\xc9\x82\x33\xc7\x01"
+			  "\xaf\x40\x70\xb8\x2a\x78\xc9\x14"
+			  "\xac\xb1\x10\xca\x2e\xb3\x28\xe4"
+			  "\xac\xfa\x58\x7f\xe5\x73\x09\x8c"
+			  "\x1d\x40\x87\x8c\xd9\x75\xc0\x55"
+			  "\xa2\xda\x07\xd1\xc2\xa9\xd1\xbb"
+			  "\x09\x4f\x77\x62\x88\x2d\xf2\x68"
+			  "\x54",
+		.alen	= 65,
+		.input	= "\x33\xc1\xda\xfa\x15\x21\x07\x8e"
+			  "\x93\x68\xea\x64\x7b\x3d\x4b\x6b"
+			  "\x71\x5e\x5e\x6b\x92\xaa\x65\xc2"
+			  "\x7a\x2a\xc1\xa9\x0a\xa1\x24\x81"
+			  "\x26\x3a\x5a\x09\xe8\xce\x73\x72"
+			  "\xde\x7b\x58\x9e\x85\xb9\xa4\x28"
+			  "\xda",
+		.ilen	= 49,
+		.result	= "\xf7\x02\xbb\x11\x52\x24\xd8\x48"
+			  "\x93\xe6\x9b\xee\x81\xfc\xf7\x82"
+			  "\x79\xf0\xf3\xd9\x6c\x20\xa9\x1a"
+			  "\xdc\xbc\x47\xc0\xe4\xcb\x10\x99"
+			  "\x2f",
+		.rlen	= 33,
+	}, {
+		.key	= "\x06\xcc\x4a\x79\x96\xc3\x82\xcf"
+			  "\xb3\x1c\xc7\x12\x7f\xc5\x28\x04"
+			  "\x44\xa1\x76\xfb\xad\xc8\x8a\x21"
+			  "\x0d\xc8\x7f\xb6\x9b\xe3\xf8\x4f",
+		.klen	= 32,
+		.iv	= "\x15\x95\xd8\xe1\xda\x62\x2c\x56"
+			  "\xd3\x53\xf4\x36\x7e\x8e\x59\x85"
+			  "\x0e\x51\xf9\x1c\xee\x70\x6a\x27"
+			  "\x3d\xd3\xb7\xac\x51\xfa\xdf\x05",
+		.assoc	= "\x24\x5e\x67\x49\x1e\x01\xd6\xdd"
+			  "\xf3\x89\x20\x5b\x7c\x57\x89\x07",
+		.alen	= 16,
+		.input	= "\x3e\xf8\x86\x3d\x39\xf8\x96\x02"
+			  "\x0f\xdf\xc9\x6e\x37\x1e\x57\x99"
+			  "\x07\x2a\x1a\xac\xd1\xda\xfd\x3b"
+			  "\xc7\xff\xbd\xbc\x85\x09\x0b",
+		.ilen	= 31,
+		.result	= "\x33\x27\xf5\xb1\x62\xa0\x80\x63"
+			  "\x14\xc0\x4d\x7f\x7b\x20\xba\x89",
+		.rlen	= 16,
+	}, {
+		.key	= "\x42\xf0\x84\x19\xa6\x3f\x2b\xea"
+			  "\x34\xf6\x79\xa3\x79\xe9\xeb\x0a"
+			  "\x6e\x63\x82\x7f\xb2\x68\x0c\x3a"
+			  "\xce\xf5\x5e\x8e\x75\x42\x97\x26",
+		.klen	= 32,
+		.iv	= "\x51\xb9\x12\x80\xea\xde\xd5\x71"
+			  "\x54\x2d\xa6\xc8\x78\xb2\x1b\x8c"
+			  "\x39\x14\x05\xa0\xf3\x10\xec\x41"
+			  "\xff\x01\x95\x84\x2b\x59\x7f\xdb",
+		.assoc	= "\x61\x83\xa0\xe8\x2e\x7d\x7f\xf8"
+			  "\x74\x63\xd2\xec\x76\x7c\x4c\x0d",
+		.alen	= 16,
+		.input	= "\x2f\xc4\xd8\x0d\xa6\x07\xef\x2e"
+			  "\x6c\xd9\x84\x63\x70\x97\x61\x37"
+			  "\x08\x2f\x16\x90\x9e\x62\x30\x0d"
+			  "\x62\xd5\xc8\xf0\x46\x1a",
+		.ilen	= 30,
+		.result	= "\x70\x4c\x2f\x50\x72\x1c\x29\x7f"
+			  "\x95\x9a\xff\x10\x75\x45\x7d\x8f",
+		.rlen	= 16,
+	}, {
+		.key	= "\x7f\x15\xbd\xb8\xb6\xba\xd3\x06"
+			  "\xb5\xd1\x2b\x35\x73\x0e\xad\x10"
+			  "\x98\x25\x8d\x03\xb7\x08\x8e\x54"
+			  "\x90\x23\x3d\x67\x4f\xa1\x36\xfc",
+		.klen	= 32,
+		.iv	= "\x8e\xde\x4c\x20\xfa\x59\x7e\x8d"
+			  "\xd5\x07\x58\x59\x72\xd7\xde\x92"
+			  "\x63\xd6\x10\x24\xf8\xb0\x6e\x5a"
+			  "\xc0\x2e\x74\x5d\x06\xb8\x1e\xb2",
+		.assoc	= "\x9d\xa7\xda\x88\x3e\xf8\x28\x14"
+			  "\xf5\x3e\x85\x7d\x70\xa0\x0f\x13",
+		.alen	= 16,
+		.input	= "\xce\xf3\x17\x87\x49\xc2\x00\x46"
+			  "\xc6\x12\x5c\x8f\x81\x38\xaa\x55"
+			  "\xf8\x67\x75\xf1\x75\xe3\x2a\x24",
+		.ilen	= 24,
+		.result	= "\xac\x70\x69\xef\x82\x97\xd2\x9b"
+			  "\x15\x74\xb1\xa2\x6f\x69\x3f\x95",
+		.rlen	= 16,
+	},
+};
+
+/*
+ * MORUS-640 test vectors - generated via reference implementation from
+ * SUPERCOP (https://bench.cr.yp.to/supercop.html):
+ *
+ *   https://bench.cr.yp.to/supercop/supercop-20170228.tar.xz
+ *   (see crypto_aead/morus640128v2/)
+ */
+static const struct aead_testvec morus640_enc_tv_template[] = {
+	{
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 16,
+		.iv	= "\x0f\xc9\x8e\x67\x44\x9e\xaa\x86"
+			  "\x20\x36\x2c\x24\xfe\xc9\x30\x81",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x89\x62\x7d\xf3\x07\x9d\x52\x05"
+			  "\x53\xc3\x04\x60\x93\xb4\x37\x9a",
+		.rlen	= 16,
+	}, {
+		.key	= "\x3c\x24\x39\x9f\x10\x7b\xa8\x1b"
+			  "\x80\xda\xb2\x91\xf9\x24\xc2\x06",
+		.klen	= 16,
+		.iv	= "\x4b\xed\xc8\x07\x54\x1a\x52\xa2"
+			  "\xa1\x10\xde\xb5\xf8\xed\xf3\x87",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x69",
+		.ilen	= 1,
+		.result	= "\xa8\x8d\xe4\x90\xb5\x50\x8f\x78"
+			  "\xb6\x10\x9a\x59\x5f\x61\x37\x70"
+			  "\x09",
+		.rlen	= 17,
+	}, {
+		.key	= "\x79\x49\x73\x3e\x20\xf7\x51\x37"
+			  "\x01\xb4\x64\x22\xf3\x48\x85\x0c",
+		.klen	= 16,
+		.iv	= "\x88\x12\x01\xa6\x64\x96\xfb\xbe"
+			  "\x22\xea\x90\x47\xf2\x11\xb5\x8e",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xa6\xa4\x1e\x76\xec\xd4\x50\xcc"
+			  "\x62\x58\xe9\x8f\xef\xa4\x17",
+		.ilen	= 15,
+		.result	= "\x76\xdd\xb9\x05\x3d\xce\x61\x38"
+			  "\xf3\xef\xf7\xe5\xd7\xfd\x70\xa5"
+			  "\xcf\x9d\x64\xb8\x0a\x9f\xfd\x8b"
+			  "\xd4\x6e\xfe\xd9\xc8\x63\x4b",
+		.rlen	= 31,
+	}, {
+		.key	= "\xb5\x6e\xad\xdd\x30\x72\xfa\x53"
+			  "\x82\x8e\x16\xb4\xed\x6d\x47\x12",
+		.klen	= 16,
+		.iv	= "\xc4\x37\x3b\x45\x74\x11\xa4\xda"
+			  "\xa2\xc5\x42\xd8\xec\x36\x78\x94",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xe2\xc9\x58\x15\xfc\x4f\xf8\xe8"
+			  "\xe3\x32\x9b\x21\xe9\xc8\xd9\x97",
+		.ilen	= 16,
+		.result	= "\xdc\x72\xe8\x14\xfb\x63\xad\x72"
+			  "\x1f\x57\x9a\x1f\x88\x81\xdb\xd6"
+			  "\xc1\x91\x9d\xb9\x25\xc4\x99\x4c"
+			  "\x97\xcd\x8a\x0c\x9d\x68\x00\x1c",
+		.rlen	= 32,
+	}, {
+		.key	= "\xf2\x92\xe6\x7d\x40\xee\xa3\x6f"
+			  "\x03\x68\xc8\x45\xe7\x91\x0a\x18",
+		.klen	= 16,
+		.iv	= "\x01\x5c\x75\xe5\x84\x8d\x4d\xf6"
+			  "\x23\x9f\xf4\x6a\xe6\x5a\x3b\x9a",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x1f\xee\x92\xb4\x0c\xcb\xa1\x04"
+			  "\x64\x0c\x4d\xb2\xe3\xec\x9c\x9d"
+			  "\x09",
+		.ilen	= 17,
+		.result	= "\x6b\x4f\x3b\x90\x9a\xa2\xb3\x82"
+			  "\x0a\xb8\x55\xee\xeb\x73\x4d\x7f"
+			  "\x54\x11\x3a\x8a\x31\xa3\xb5\xf2"
+			  "\xcd\x49\xdb\xf3\xee\x26\xbd\xa2"
+			  "\x0d",
+		.rlen	= 33,
+	}, {
+		.key	= "\x2e\xb7\x20\x1c\x50\x6a\x4b\x8b"
+			  "\x84\x42\x7a\xd7\xe1\xb5\xcd\x1f",
+		.klen	= 16,
+		.iv	= "\x3d\x80\xae\x84\x94\x09\xf6\x12"
+			  "\xa4\x79\xa6\xfb\xe0\x7f\xfd\xa0",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x5c\x13\xcb\x54\x1c\x47\x4a\x1f"
+			  "\xe5\xe6\xff\x44\xdd\x11\x5f\xa3"
+			  "\x33\xdd\xc2\xf8\xdd\x18\x2b\x93"
+			  "\x57\x05\x01\x1c\x66\x22\xd3",
+		.ilen	= 31,
+		.result	= "\x59\xd1\x0f\x6b\xee\x27\x84\x92"
+			  "\xb7\xa9\xb5\xdd\x02\xa4\x12\xa5"
+			  "\x50\x32\xb4\x9a\x2e\x35\x83\x55"
+			  "\x36\x12\x12\xed\xa3\x31\xc5\x30"
+			  "\xa7\xe2\x4a\x6d\x05\x59\x43\x91"
+			  "\x75\xfa\x6c\x17\xc6\x73\xca",
+		.rlen	= 47,
+	}, {
+		.key	= "\x6b\xdc\x5a\xbb\x60\xe5\xf4\xa6"
+			  "\x05\x1d\x2c\x68\xdb\xda\x8f\x25",
+		.klen	= 16,
+		.iv	= "\x7a\xa5\xe8\x23\xa4\x84\x9e\x2d"
+			  "\x25\x53\x58\x8c\xda\xa3\xc0\xa6",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x98\x37\x05\xf3\x2c\xc2\xf3\x3b"
+			  "\x66\xc0\xb1\xd5\xd7\x35\x21\xaa"
+			  "\x5d\x9f\xce\x7c\xe2\xb8\xad\xad"
+			  "\x19\x33\xe0\xf4\x40\x81\x72\x28",
+		.ilen	= 32,
+		.result	= "\xdb\x49\x68\x0f\x91\x5b\x21\xb1"
+			  "\xcf\x50\xb2\x4c\x32\xe1\xa6\x69"
+			  "\xc0\xfb\x44\x1f\xa0\x9a\xeb\x39"
+			  "\x1b\xde\x68\x38\xcc\x27\x52\xc5"
+			  "\xf6\x3e\x74\xea\x66\x5b\x5f\x0c"
+			  "\x65\x9e\x58\xe6\x52\xa2\xfe\x59",
+		.rlen	= 48,
+	}, {
+		.key	= "\xa7\x00\x93\x5b\x70\x61\x9d\xc2"
+			  "\x86\xf7\xde\xfa\xd5\xfe\x52\x2b",
+		.klen	= 16,
+		.iv	= "\xb6\xca\x22\xc3\xb4\x00\x47\x49"
+			  "\xa6\x2d\x0a\x1e\xd4\xc7\x83\xad",
+		.assoc	= "\xc5",
+		.alen	= 1,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x56\xe7\x24\x52\xdd\x95\x60\x5b"
+			  "\x09\x48\x39\x69\x9c\xb3\x62\x46",
+		.rlen	= 16,
+	}, {
+		.key	= "\xe4\x25\xcd\xfa\x80\xdd\x46\xde"
+			  "\x07\xd1\x90\x8b\xcf\x23\x15\x31",
+		.klen	= 16,
+		.iv	= "\xf3\xee\x5c\x62\xc4\x7c\xf0\x65"
+			  "\x27\x08\xbd\xaf\xce\xec\x45\xb3",
+		.assoc	= "\x02\xb8\xea\xca\x09\x1b\x9a\xec"
+			  "\x47\x3e\xe9\xd4\xcc\xb5\x76",
+		.alen	= 15,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xdd\xfa\x6c\x1f\x5d\x86\x87\x01"
+			  "\x13\xe5\x73\x46\x46\xf2\x5c\xe1",
+		.rlen	= 16,
+	}, {
+		.key	= "\x20\x4a\x07\x99\x91\x58\xee\xfa"
+			  "\x88\xab\x42\x1c\xc9\x47\xd7\x38",
+		.klen	= 16,
+		.iv	= "\x2f\x13\x95\x01\xd5\xf7\x99\x81"
+			  "\xa8\xe2\x6f\x41\xc8\x10\x08\xb9",
+		.assoc	= "\x3f\xdc\x24\x69\x19\x96\x43\x08"
+			  "\xc8\x18\x9b\x65\xc6\xd9\x39\x3b",
+		.alen	= 16,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xa6\x1b\xb9\xd7\x5e\x3c\xcf\xac"
+			  "\xa9\x21\x45\x0b\x16\x52\xf7\xe1",
+		.rlen	= 16,
+	}, {
+		.key	= "\x5d\x6f\x41\x39\xa1\xd4\x97\x16"
+			  "\x09\x85\xf4\xae\xc3\x6b\x9a\x3e",
+		.klen	= 16,
+		.iv	= "\x6c\x38\xcf\xa1\xe5\x73\x41\x9d"
+			  "\x29\xbc\x21\xd2\xc2\x35\xcb\xbf",
+		.assoc	= "\x7b\x01\x5d\x08\x29\x12\xec\x24"
+			  "\x49\xf3\x4d\xf7\xc0\xfe\xfb\x41"
+			  "\x3c",
+		.alen	= 17,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x15\xff\xde\x3b\x34\xfc\xf6\xf9"
+			  "\xbb\xa8\x62\xad\x0a\xf5\x48\x60",
+		.rlen	= 16,
+	}, {
+		.key	= "\x99\x93\x7a\xd8\xb1\x50\x40\x31"
+			  "\x8a\x60\xa6\x3f\xbd\x90\x5d\x44",
+		.klen	= 16,
+		.iv	= "\xa8\x5c\x09\x40\xf5\xef\xea\xb8"
+			  "\xaa\x96\xd3\x64\xbc\x59\x8d\xc6",
+		.assoc	= "\xb8\x26\x97\xa8\x39\x8e\x94\x3f"
+			  "\xca\xcd\xff\x88\xba\x22\xbe\x47"
+			  "\x67\xba\x85\xf1\xbb\x30\x56\x26"
+			  "\xaf\x0b\x02\x38\xcc\x44\xa7",
+		.alen	= 31,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xd2\x9d\xf8\x3b\xd7\x84\xe9\x2d"
+			  "\x4b\xef\x75\x16\x0a\x99\xae\x6b",
+		.rlen	= 16,
+	}, {
+		.key	= "\xd6\xb8\xb4\x77\xc1\xcb\xe9\x4d"
+			  "\x0a\x3a\x58\xd1\xb7\xb4\x1f\x4a",
+		.klen	= 16,
+		.iv	= "\xe5\x81\x42\xdf\x05\x6a\x93\xd4"
+			  "\x2b\x70\x85\xf5\xb6\x7d\x50\xcc",
+		.assoc	= "\xf4\x4a\xd1\x47\x49\x09\x3d\x5b"
+			  "\x4b\xa7\xb1\x19\xb4\x46\x81\x4d"
+			  "\x91\x7c\x91\x75\xc0\xd0\xd8\x40"
+			  "\x71\x39\xe1\x10\xa6\xa3\x46\x7a",
+		.alen	= 32,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xe4\x8d\xa7\xa7\x45\xc1\x31\x4f"
+			  "\xce\xfb\xaf\xd6\xc2\xe6\xee\xc0",
+		.rlen	= 16,
+	}, {
+		.key	= "\x12\xdd\xee\x17\xd1\x47\x92\x69"
+			  "\x8b\x14\x0a\x62\xb1\xd9\xe2\x50",
+		.klen	= 16,
+		.iv	= "\x22\xa6\x7c\x7f\x15\xe6\x3c\xf0"
+			  "\xac\x4b\x37\x86\xb0\xa2\x13\xd2",
+		.assoc	= "\x31",
+		.alen	= 1,
+		.input	= "\x40",
+		.ilen	= 1,
+		.result	= "\xe2\x67\x38\x4f\xb9\xad\x7d\x38"
+			  "\x01\xfe\x84\x14\x85\xf8\xd1\xe3"
+			  "\x22",
+		.rlen	= 17,
+	}, {
+		.key	= "\x4f\x01\x27\xb6\xe1\xc3\x3a\x85"
+			  "\x0c\xee\xbc\xf4\xab\xfd\xa5\x57",
+		.klen	= 16,
+		.iv	= "\x5e\xcb\xb6\x1e\x25\x62\xe4\x0c"
+			  "\x2d\x25\xe9\x18\xaa\xc6\xd5\xd8",
+		.assoc	= "\x6d\x94\x44\x86\x69\x00\x8f\x93"
+			  "\x4d\x5b\x15\x3c\xa8\x8f\x06",
+		.alen	= 15,
+		.input	= "\x7c\x5d\xd3\xee\xad\x9f\x39\x1a"
+			  "\x6d\x92\x42\x61\xa7\x58\x37",
+		.ilen	= 15,
+		.result	= "\x77\x32\x61\xeb\xb4\x33\x29\x92"
+			  "\x29\x95\xc5\x8e\x85\x76\xab\xfc"
+			  "\x07\x95\xa7\x44\x74\xf7\x22\xff"
+			  "\xd8\xd8\x36\x3d\x8a\x7f\x9e",
+		.rlen	= 31,
+	}, {
+		.key	= "\x8b\x26\x61\x55\xf1\x3e\xe3\xa1"
+			  "\x8d\xc8\x6e\x85\xa5\x21\x67\x5d",
+		.klen	= 16,
+		.iv	= "\x9b\xef\xf0\xbd\x35\xdd\x8d\x28"
+			  "\xad\xff\x9b\xa9\xa4\xeb\x98\xdf",
+		.assoc	= "\xaa\xb8\x7e\x25\x79\x7c\x37\xaf"
+			  "\xce\x36\xc7\xce\xa2\xb4\xc9\x60",
+		.alen	= 16,
+		.input	= "\xb9\x82\x0c\x8d\xbd\x1b\xe2\x36"
+			  "\xee\x6c\xf4\xf2\xa1\x7d\xf9\xe2",
+		.ilen	= 16,
+		.result	= "\xd8\xfd\x44\x45\xf6\x42\x12\x38"
+			  "\xf2\x0b\xea\x4f\x9e\x11\x61\x07"
+			  "\x48\x67\x98\x18\x9b\xd0\x0c\x59"
+			  "\x67\xa4\x11\xb3\x2b\xd6\xc1\x70",
+		.rlen	= 32,
+	}, {
+		.key	= "\xc8\x4b\x9b\xf5\x01\xba\x8c\xbd"
+			  "\x0e\xa3\x21\x16\x9f\x46\x2a\x63",
+		.klen	= 16,
+		.iv	= "\xd7\x14\x29\x5d\x45\x59\x36\x44"
+			  "\x2e\xd9\x4d\x3b\x9e\x0f\x5b\xe5",
+		.assoc	= "\xe6\xdd\xb8\xc4\x89\xf8\xe0\xca"
+			  "\x4f\x10\x7a\x5f\x9c\xd8\x8b\x66"
+			  "\x3b",
+		.alen	= 17,
+		.input	= "\xf5\xa6\x46\x2c\xce\x97\x8a\x51"
+			  "\x6f\x46\xa6\x83\x9b\xa1\xbc\xe8"
+			  "\x05",
+		.ilen	= 17,
+		.result	= "\xb1\xab\x53\x4e\xc7\x40\x16\xb6"
+			  "\x71\x3a\x00\x9f\x41\x88\xb0\xb2"
+			  "\x71\x83\x85\x5f\xc8\x79\x0a\x99"
+			  "\x99\xdc\x89\x1c\x88\xd2\x3e\xf9"
+			  "\x83",
+		.rlen	= 33,
+	}, {
+		.key	= "\x05\x70\xd5\x94\x12\x36\x35\xd8"
+			  "\x8f\x7d\xd3\xa8\x99\x6a\xed\x69",
+		.klen	= 16,
+		.iv	= "\x14\x39\x63\xfc\x56\xd5\xdf\x5f"
+			  "\xaf\xb3\xff\xcc\x98\x33\x1d\xeb",
+		.assoc	= "\x23\x02\xf1\x64\x9a\x73\x89\xe6"
+			  "\xd0\xea\x2c\xf1\x96\xfc\x4e\x6d"
+			  "\x65\x48\xcb\x0a\xda\xf0\x62\xc0"
+			  "\x38\x1d\x3b\x4a\xe9\x7e\x62",
+		.alen	= 31,
+		.input	= "\x32\xcb\x80\xcc\xde\x12\x33\x6d"
+			  "\xf0\x20\x58\x15\x95\xc6\x7f\xee"
+			  "\x2f\xf9\x4e\x2c\x1b\x98\x43\xc7"
+			  "\x68\x28\x73\x40\x9f\x96\x4a",
+		.ilen	= 31,
+		.result	= "\x29\xc4\xf0\x03\xc1\x86\xdf\x06"
+			  "\x5c\x7b\xef\x64\x87\x00\xd1\x37"
+			  "\xa7\x08\xbc\x7f\x8f\x41\x54\xd0"
+			  "\x3e\xf1\xc3\xa2\x96\x84\xdd\x2a"
+			  "\x2d\x21\x30\xf9\x02\xdb\x06\x0c"
+			  "\xf1\x5a\x66\x69\xe0\xca\x83",
+		.rlen	= 47,
+	}, {
+		.key	= "\x41\x94\x0e\x33\x22\xb1\xdd\xf4"
+			  "\x10\x57\x85\x39\x93\x8f\xaf\x70",
+		.klen	= 16,
+		.iv	= "\x50\x5d\x9d\x9b\x66\x50\x88\x7b"
+			  "\x30\x8e\xb1\x5e\x92\x58\xe0\xf1",
+		.assoc	= "\x5f\x27\x2b\x03\xaa\xef\x32\x02"
+			  "\x50\xc4\xde\x82\x90\x21\x11\x73"
+			  "\x8f\x0a\xd6\x8f\xdf\x90\xe4\xda"
+			  "\xf9\x4a\x1a\x23\xc3\xdd\x02\x81",
+		.alen	= 32,
+		.input	= "\x6e\xf0\xba\x6b\xee\x8e\xdc\x89"
+			  "\x71\xfb\x0a\xa6\x8f\xea\x41\xf4"
+			  "\x5a\xbb\x59\xb0\x20\x38\xc5\xe0"
+			  "\x29\x56\x52\x19\x79\xf5\xe9\x37",
+		.ilen	= 32,
+		.result	= "\xe2\x2e\x44\xdf\xd3\x60\x6d\xb2"
+			  "\x70\x57\x37\xc5\xc2\x4f\x8d\x14"
+			  "\xc6\xbf\x8b\xec\xf5\x62\x67\xf2"
+			  "\x2f\xa1\xe6\xd6\xa7\xb1\x8c\x54"
+			  "\xe5\x6b\x49\xf9\x6e\x90\xc3\xaa"
+			  "\x7a\x00\x2e\x4d\x7f\x31\x2e\x81",
+		.rlen	= 48,
+	}, {
+		.key	= "\x7e\xb9\x48\xd3\x32\x2d\x86\x10"
+			  "\x91\x31\x37\xcb\x8d\xb3\x72\x76",
+		.klen	= 16,
+		.iv	= "\x8d\x82\xd6\x3b\x76\xcc\x30\x97"
+			  "\xb1\x68\x63\xef\x8c\x7c\xa3\xf7",
+		.assoc	= "\x9c\x4b\x65\xa2\xba\x6b\xdb\x1e"
+			  "\xd1\x9e\x90\x13\x8a\x45\xd3\x79"
+			  "\xba\xcd\xe2\x13\xe4\x30\x66\xf4"
+			  "\xba\x78\xf9\xfb\x9d\x3c\xa1\x58"
+			  "\x1a",
+		.alen	= 33,
+		.input	= "\xab\x14\xf3\x0a\xfe\x0a\x85\xa5"
+			  "\xf2\xd5\xbc\x38\x89\x0e\x04\xfb"
+			  "\x84\x7d\x65\x34\x25\xd8\x47\xfa"
+			  "\xeb\x83\x31\xf1\x54\x54\x89\x0d"
+			  "\x9d\x4d\x54\x51\x84\x61\xf6\x8e"
+			  "\x03\x31\xf2\x25\x16\xcc\xaa\xc6"
+			  "\x75\x73\x20\x30\x59\x54\xb2\xf0"
+			  "\x3a\x4b\xe0\x23\x8e\xa6\x08\x35"
+			  "\x8a",
+		.ilen	= 65,
+		.result	= "\xc7\xca\x26\x61\x57\xee\xa2\xb9"
+			  "\xb1\x37\xde\x95\x06\x90\x11\x08"
+			  "\x4d\x30\x9f\x24\xc0\x56\xb7\xe1"
+			  "\x0b\x9f\xd2\x57\xe9\xd2\xb1\x76"
+			  "\x56\x9a\xb4\x58\xc5\x08\xfc\xb5"
+			  "\xf2\x31\x9b\xc9\xcd\xb3\x64\xdb"
+			  "\x6f\x50\xbf\xf4\x73\x9d\xfb\x6b"
+			  "\xef\x35\x25\x48\xed\xcf\x29\xa8"
+			  "\xac\xc3\xb9\xcb\x61\x8f\x73\x92"
+			  "\x2c\x7a\x6f\xda\xf9\x09\x6f\xe1"
+			  "\xc4",
+		.rlen	= 81,
+	}, {
+		.key	= "\xba\xde\x82\x72\x42\xa9\x2f\x2c"
+			  "\x12\x0b\xe9\x5c\x87\xd7\x35\x7c",
+		.klen	= 16,
+		.iv	= "\xc9\xa7\x10\xda\x86\x48\xd9\xb3"
+			  "\x32\x42\x15\x80\x85\xa1\x65\xfe",
+		.assoc	= "\xd8\x70\x9f\x42\xca\xe6\x83\x3a"
+			  "\x52\x79\x42\xa5\x84\x6a\x96\x7f"
+			  "\xe4\x8f\xed\x97\xe9\xd0\xe8\x0d"
+			  "\x7c\xa6\xd8\xd4\x77\x9b\x40\x2e"
+			  "\x28\xce\x57\x34\xcd\x6e\x84\x4c"
+			  "\x17\x3c\xe1\xb2\xa8\x0b\xbb\xf1"
+			  "\x96\x41\x0d\x69\xe8\x54\x0a\xc8"
+			  "\x15\x4e\x91\x92\x89\x4b\xb7\x9b"
+			  "\x21",
+		.alen	= 65,
+		.input	= "\xe8\x39\x2d\xaa\x0e\x85\x2d\xc1"
+			  "\x72\xaf\x6e\xc9\x82\x33\xc7\x01"
+			  "\xaf\x40\x70\xb8\x2a\x78\xc9\x14"
+			  "\xac\xb1\x10\xca\x2e\xb3\x28\xe4"
+			  "\xac",
+		.ilen	= 33,
+		.result	= "\x57\xcd\x3d\x46\xc5\xf9\x68\x3b"
+			  "\x2c\x0f\xb4\x7e\x7b\x64\x3e\x40"
+			  "\xf3\x78\x63\x34\x89\x79\x39\x6b"
+			  "\x61\x64\x4a\x9a\xfa\x70\xa4\xd3"
+			  "\x54\x0b\xea\x05\xa6\x95\x64\xed"
+			  "\x3d\x69\xa2\x0c\x27\x56\x2f\x34"
+			  "\x66",
+		.rlen	= 49,
+	}, {
+		.key	= "\xf7\x02\xbb\x11\x52\x24\xd8\x48"
+			  "\x93\xe6\x9b\xee\x81\xfc\xf7\x82",
+		.klen	= 16,
+		.iv	= "\x06\xcc\x4a\x79\x96\xc3\x82\xcf"
+			  "\xb3\x1c\xc7\x12\x7f\xc5\x28\x04",
+		.assoc	= "\x15\x95\xd8\xe1\xda\x62\x2c\x56"
+			  "\xd3\x53\xf4\x36\x7e\x8e\x59\x85",
+		.alen	= 16,
+		.input	= "\x24\x5e\x67\x49\x1e\x01\xd6\xdd"
+			  "\xf3\x89\x20\x5b\x7c\x57\x89\x07",
+		.ilen	= 16,
+		.result	= "\xfc\x85\x06\x28\x8f\xe8\x23\x1f"
+			  "\x33\x98\x87\xde\x08\xb6\xb6\xae"
+			  "\x3e\xa4\xf8\x19\xf1\x92\x60\x39"
+			  "\xb9\x6b\x3f\xdf\xc8\xcb\x30",
+		.rlen	= 31,
+	}, {
+		.key	= "\x33\x27\xf5\xb1\x62\xa0\x80\x63"
+			  "\x14\xc0\x4d\x7f\x7b\x20\xba\x89",
+		.klen	= 16,
+		.iv	= "\x42\xf0\x84\x19\xa6\x3f\x2b\xea"
+			  "\x34\xf6\x79\xa3\x79\xe9\xeb\x0a",
+		.assoc	= "\x51\xb9\x12\x80\xea\xde\xd5\x71"
+			  "\x54\x2d\xa6\xc8\x78\xb2\x1b\x8c",
+		.alen	= 16,
+		.input	= "\x61\x83\xa0\xe8\x2e\x7d\x7f\xf8"
+			  "\x74\x63\xd2\xec\x76\x7c\x4c\x0d",
+		.ilen	= 16,
+		.result	= "\x74\x7d\x70\x07\xe9\xba\x01\xee"
+			  "\x6c\xc6\x6f\x50\x25\x33\xbe\x50"
+			  "\x17\xb8\x17\x62\xed\x80\xa2\xf5"
+			  "\x03\xde\x85\x71\x5d\x34",
+		.rlen	= 30,
+	}, {
+		.key	= "\x70\x4c\x2f\x50\x72\x1c\x29\x7f"
+			  "\x95\x9a\xff\x10\x75\x45\x7d\x8f",
+		.klen	= 16,
+		.iv	= "\x7f\x15\xbd\xb8\xb6\xba\xd3\x06"
+			  "\xb5\xd1\x2b\x35\x73\x0e\xad\x10",
+		.assoc	= "\x8e\xde\x4c\x20\xfa\x59\x7e\x8d"
+			  "\xd5\x07\x58\x59\x72\xd7\xde\x92",
+		.alen	= 16,
+		.input	= "\x9d\xa7\xda\x88\x3e\xf8\x28\x14"
+			  "\xf5\x3e\x85\x7d\x70\xa0\x0f\x13",
+		.ilen	= 16,
+		.result	= "\xf4\xb3\x85\xf9\xac\xde\xb1\x38"
+			  "\x29\xfd\x6c\x7c\x49\xe5\x1d\xaf"
+			  "\xba\xea\xd4\xfa\x3f\x11\x33\x98",
+		.rlen	= 24,
+	}, {
+		.key	= "\xac\x70\x69\xef\x82\x97\xd2\x9b"
+			  "\x15\x74\xb1\xa2\x6f\x69\x3f\x95",
+		.klen	= 16,
+		.iv	= "\xbb\x3a\xf7\x57\xc6\x36\x7c\x22"
+			  "\x36\xab\xde\xc6\x6d\x32\x70\x17",
+		.assoc	= "\xcb\x03\x85\xbf\x0a\xd5\x26\xa9"
+			  "\x56\xe1\x0a\xeb\x6c\xfb\xa1\x98",
+		.alen	= 16,
+		.input	= "\xda\xcc\x14\x27\x4e\x74\xd1\x30"
+			  "\x76\x18\x37\x0f\x6a\xc4\xd1\x1a",
+		.ilen	= 16,
+		.result	= "\xe6\x5c\x49\x4f\x78\xf3\x62\x86"
+			  "\xe1\xb7\xa5\xc3\x32\x88\x3c\x8c"
+			  "\x6e",
+		.rlen	= 17,
+	},
+};
+
+static const struct aead_testvec morus640_dec_tv_template[] = {
+	{
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 16,
+		.iv	= "\x0f\xc9\x8e\x67\x44\x9e\xaa\x86"
+			  "\x20\x36\x2c\x24\xfe\xc9\x30\x81",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x89\x62\x7d\xf3\x07\x9d\x52\x05"
+			  "\x53\xc3\x04\x60\x93\xb4\x37\x9a",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x3c\x24\x39\x9f\x10\x7b\xa8\x1b"
+			  "\x80\xda\xb2\x91\xf9\x24\xc2\x06",
+		.klen	= 16,
+		.iv	= "\x4b\xed\xc8\x07\x54\x1a\x52\xa2"
+			  "\xa1\x10\xde\xb5\xf8\xed\xf3\x87",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xa8\x8d\xe4\x90\xb5\x50\x8f\x78"
+			  "\xb6\x10\x9a\x59\x5f\x61\x37\x70"
+			  "\x09",
+		.ilen	= 17,
+		.result	= "\x69",
+		.rlen	= 1,
+	}, {
+		.key	= "\x79\x49\x73\x3e\x20\xf7\x51\x37"
+			  "\x01\xb4\x64\x22\xf3\x48\x85\x0c",
+		.klen	= 16,
+		.iv	= "\x88\x12\x01\xa6\x64\x96\xfb\xbe"
+			  "\x22\xea\x90\x47\xf2\x11\xb5\x8e",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x76\xdd\xb9\x05\x3d\xce\x61\x38"
+			  "\xf3\xef\xf7\xe5\xd7\xfd\x70\xa5"
+			  "\xcf\x9d\x64\xb8\x0a\x9f\xfd\x8b"
+			  "\xd4\x6e\xfe\xd9\xc8\x63\x4b",
+		.ilen	= 31,
+		.result	= "\xa6\xa4\x1e\x76\xec\xd4\x50\xcc"
+			  "\x62\x58\xe9\x8f\xef\xa4\x17",
+		.rlen	= 15,
+	}, {
+		.key	= "\xb5\x6e\xad\xdd\x30\x72\xfa\x53"
+			  "\x82\x8e\x16\xb4\xed\x6d\x47\x12",
+		.klen	= 16,
+		.iv	= "\xc4\x37\x3b\x45\x74\x11\xa4\xda"
+			  "\xa2\xc5\x42\xd8\xec\x36\x78\x94",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xdc\x72\xe8\x14\xfb\x63\xad\x72"
+			  "\x1f\x57\x9a\x1f\x88\x81\xdb\xd6"
+			  "\xc1\x91\x9d\xb9\x25\xc4\x99\x4c"
+			  "\x97\xcd\x8a\x0c\x9d\x68\x00\x1c",
+		.ilen	= 32,
+		.result	= "\xe2\xc9\x58\x15\xfc\x4f\xf8\xe8"
+			  "\xe3\x32\x9b\x21\xe9\xc8\xd9\x97",
+		.rlen	= 16,
+	}, {
+		.key	= "\xf2\x92\xe6\x7d\x40\xee\xa3\x6f"
+			  "\x03\x68\xc8\x45\xe7\x91\x0a\x18",
+		.klen	= 16,
+		.iv	= "\x01\x5c\x75\xe5\x84\x8d\x4d\xf6"
+			  "\x23\x9f\xf4\x6a\xe6\x5a\x3b\x9a",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x6b\x4f\x3b\x90\x9a\xa2\xb3\x82"
+			  "\x0a\xb8\x55\xee\xeb\x73\x4d\x7f"
+			  "\x54\x11\x3a\x8a\x31\xa3\xb5\xf2"
+			  "\xcd\x49\xdb\xf3\xee\x26\xbd\xa2"
+			  "\x0d",
+		.ilen	= 33,
+		.result	= "\x1f\xee\x92\xb4\x0c\xcb\xa1\x04"
+			  "\x64\x0c\x4d\xb2\xe3\xec\x9c\x9d"
+			  "\x09",
+		.rlen	= 17,
+	}, {
+		.key	= "\x2e\xb7\x20\x1c\x50\x6a\x4b\x8b"
+			  "\x84\x42\x7a\xd7\xe1\xb5\xcd\x1f",
+		.klen	= 16,
+		.iv	= "\x3d\x80\xae\x84\x94\x09\xf6\x12"
+			  "\xa4\x79\xa6\xfb\xe0\x7f\xfd\xa0",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x59\xd1\x0f\x6b\xee\x27\x84\x92"
+			  "\xb7\xa9\xb5\xdd\x02\xa4\x12\xa5"
+			  "\x50\x32\xb4\x9a\x2e\x35\x83\x55"
+			  "\x36\x12\x12\xed\xa3\x31\xc5\x30"
+			  "\xa7\xe2\x4a\x6d\x05\x59\x43\x91"
+			  "\x75\xfa\x6c\x17\xc6\x73\xca",
+		.ilen	= 47,
+		.result	= "\x5c\x13\xcb\x54\x1c\x47\x4a\x1f"
+			  "\xe5\xe6\xff\x44\xdd\x11\x5f\xa3"
+			  "\x33\xdd\xc2\xf8\xdd\x18\x2b\x93"
+			  "\x57\x05\x01\x1c\x66\x22\xd3",
+		.rlen	= 31,
+	}, {
+		.key	= "\x6b\xdc\x5a\xbb\x60\xe5\xf4\xa6"
+			  "\x05\x1d\x2c\x68\xdb\xda\x8f\x25",
+		.klen	= 16,
+		.iv	= "\x7a\xa5\xe8\x23\xa4\x84\x9e\x2d"
+			  "\x25\x53\x58\x8c\xda\xa3\xc0\xa6",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xdb\x49\x68\x0f\x91\x5b\x21\xb1"
+			  "\xcf\x50\xb2\x4c\x32\xe1\xa6\x69"
+			  "\xc0\xfb\x44\x1f\xa0\x9a\xeb\x39"
+			  "\x1b\xde\x68\x38\xcc\x27\x52\xc5"
+			  "\xf6\x3e\x74\xea\x66\x5b\x5f\x0c"
+			  "\x65\x9e\x58\xe6\x52\xa2\xfe\x59",
+		.ilen	= 48,
+		.result	= "\x98\x37\x05\xf3\x2c\xc2\xf3\x3b"
+			  "\x66\xc0\xb1\xd5\xd7\x35\x21\xaa"
+			  "\x5d\x9f\xce\x7c\xe2\xb8\xad\xad"
+			  "\x19\x33\xe0\xf4\x40\x81\x72\x28",
+		.rlen	= 32,
+	}, {
+		.key	= "\xa7\x00\x93\x5b\x70\x61\x9d\xc2"
+			  "\x86\xf7\xde\xfa\xd5\xfe\x52\x2b",
+		.klen	= 16,
+		.iv	= "\xb6\xca\x22\xc3\xb4\x00\x47\x49"
+			  "\xa6\x2d\x0a\x1e\xd4\xc7\x83\xad",
+		.assoc	= "\xc5",
+		.alen	= 1,
+		.input	= "\x56\xe7\x24\x52\xdd\x95\x60\x5b"
+			  "\x09\x48\x39\x69\x9c\xb3\x62\x46",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xe4\x25\xcd\xfa\x80\xdd\x46\xde"
+			  "\x07\xd1\x90\x8b\xcf\x23\x15\x31",
+		.klen	= 16,
+		.iv	= "\xf3\xee\x5c\x62\xc4\x7c\xf0\x65"
+			  "\x27\x08\xbd\xaf\xce\xec\x45\xb3",
+		.assoc	= "\x02\xb8\xea\xca\x09\x1b\x9a\xec"
+			  "\x47\x3e\xe9\xd4\xcc\xb5\x76",
+		.alen	= 15,
+		.input	= "\xdd\xfa\x6c\x1f\x5d\x86\x87\x01"
+			  "\x13\xe5\x73\x46\x46\xf2\x5c\xe1",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x20\x4a\x07\x99\x91\x58\xee\xfa"
+			  "\x88\xab\x42\x1c\xc9\x47\xd7\x38",
+		.klen	= 16,
+		.iv	= "\x2f\x13\x95\x01\xd5\xf7\x99\x81"
+			  "\xa8\xe2\x6f\x41\xc8\x10\x08\xb9",
+		.assoc	= "\x3f\xdc\x24\x69\x19\x96\x43\x08"
+			  "\xc8\x18\x9b\x65\xc6\xd9\x39\x3b",
+		.alen	= 16,
+		.input	= "\xa6\x1b\xb9\xd7\x5e\x3c\xcf\xac"
+			  "\xa9\x21\x45\x0b\x16\x52\xf7\xe1",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x5d\x6f\x41\x39\xa1\xd4\x97\x16"
+			  "\x09\x85\xf4\xae\xc3\x6b\x9a\x3e",
+		.klen	= 16,
+		.iv	= "\x6c\x38\xcf\xa1\xe5\x73\x41\x9d"
+			  "\x29\xbc\x21\xd2\xc2\x35\xcb\xbf",
+		.assoc	= "\x7b\x01\x5d\x08\x29\x12\xec\x24"
+			  "\x49\xf3\x4d\xf7\xc0\xfe\xfb\x41"
+			  "\x3c",
+		.alen	= 17,
+		.input	= "\x15\xff\xde\x3b\x34\xfc\xf6\xf9"
+			  "\xbb\xa8\x62\xad\x0a\xf5\x48\x60",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x99\x93\x7a\xd8\xb1\x50\x40\x31"
+			  "\x8a\x60\xa6\x3f\xbd\x90\x5d\x44",
+		.klen	= 16,
+		.iv	= "\xa8\x5c\x09\x40\xf5\xef\xea\xb8"
+			  "\xaa\x96\xd3\x64\xbc\x59\x8d\xc6",
+		.assoc	= "\xb8\x26\x97\xa8\x39\x8e\x94\x3f"
+			  "\xca\xcd\xff\x88\xba\x22\xbe\x47"
+			  "\x67\xba\x85\xf1\xbb\x30\x56\x26"
+			  "\xaf\x0b\x02\x38\xcc\x44\xa7",
+		.alen	= 31,
+		.input	= "\xd2\x9d\xf8\x3b\xd7\x84\xe9\x2d"
+			  "\x4b\xef\x75\x16\x0a\x99\xae\x6b",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xd6\xb8\xb4\x77\xc1\xcb\xe9\x4d"
+			  "\x0a\x3a\x58\xd1\xb7\xb4\x1f\x4a",
+		.klen	= 16,
+		.iv	= "\xe5\x81\x42\xdf\x05\x6a\x93\xd4"
+			  "\x2b\x70\x85\xf5\xb6\x7d\x50\xcc",
+		.assoc	= "\xf4\x4a\xd1\x47\x49\x09\x3d\x5b"
+			  "\x4b\xa7\xb1\x19\xb4\x46\x81\x4d"
+			  "\x91\x7c\x91\x75\xc0\xd0\xd8\x40"
+			  "\x71\x39\xe1\x10\xa6\xa3\x46\x7a",
+		.alen	= 32,
+		.input	= "\xe4\x8d\xa7\xa7\x45\xc1\x31\x4f"
+			  "\xce\xfb\xaf\xd6\xc2\xe6\xee\xc0",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x12\xdd\xee\x17\xd1\x47\x92\x69"
+			  "\x8b\x14\x0a\x62\xb1\xd9\xe2\x50",
+		.klen	= 16,
+		.iv	= "\x22\xa6\x7c\x7f\x15\xe6\x3c\xf0"
+			  "\xac\x4b\x37\x86\xb0\xa2\x13\xd2",
+		.assoc	= "\x31",
+		.alen	= 1,
+		.input	= "\xe2\x67\x38\x4f\xb9\xad\x7d\x38"
+			  "\x01\xfe\x84\x14\x85\xf8\xd1\xe3"
+			  "\x22",
+		.ilen	= 17,
+		.result	= "\x40",
+		.rlen	= 1,
+	}, {
+		.key	= "\x4f\x01\x27\xb6\xe1\xc3\x3a\x85"
+			  "\x0c\xee\xbc\xf4\xab\xfd\xa5\x57",
+		.klen	= 16,
+		.iv	= "\x5e\xcb\xb6\x1e\x25\x62\xe4\x0c"
+			  "\x2d\x25\xe9\x18\xaa\xc6\xd5\xd8",
+		.assoc	= "\x6d\x94\x44\x86\x69\x00\x8f\x93"
+			  "\x4d\x5b\x15\x3c\xa8\x8f\x06",
+		.alen	= 15,
+		.input	= "\x77\x32\x61\xeb\xb4\x33\x29\x92"
+			  "\x29\x95\xc5\x8e\x85\x76\xab\xfc"
+			  "\x07\x95\xa7\x44\x74\xf7\x22\xff"
+			  "\xd8\xd8\x36\x3d\x8a\x7f\x9e",
+		.ilen	= 31,
+		.result	= "\x7c\x5d\xd3\xee\xad\x9f\x39\x1a"
+			  "\x6d\x92\x42\x61\xa7\x58\x37",
+		.rlen	= 15,
+	}, {
+		.key	= "\x8b\x26\x61\x55\xf1\x3e\xe3\xa1"
+			  "\x8d\xc8\x6e\x85\xa5\x21\x67\x5d",
+		.klen	= 16,
+		.iv	= "\x9b\xef\xf0\xbd\x35\xdd\x8d\x28"
+			  "\xad\xff\x9b\xa9\xa4\xeb\x98\xdf",
+		.assoc	= "\xaa\xb8\x7e\x25\x79\x7c\x37\xaf"
+			  "\xce\x36\xc7\xce\xa2\xb4\xc9\x60",
+		.alen	= 16,
+		.input	= "\xd8\xfd\x44\x45\xf6\x42\x12\x38"
+			  "\xf2\x0b\xea\x4f\x9e\x11\x61\x07"
+			  "\x48\x67\x98\x18\x9b\xd0\x0c\x59"
+			  "\x67\xa4\x11\xb3\x2b\xd6\xc1\x70",
+		.ilen	= 32,
+		.result	= "\xb9\x82\x0c\x8d\xbd\x1b\xe2\x36"
+			  "\xee\x6c\xf4\xf2\xa1\x7d\xf9\xe2",
+		.rlen	= 16,
+	}, {
+		.key	= "\xc8\x4b\x9b\xf5\x01\xba\x8c\xbd"
+			  "\x0e\xa3\x21\x16\x9f\x46\x2a\x63",
+		.klen	= 16,
+		.iv	= "\xd7\x14\x29\x5d\x45\x59\x36\x44"
+			  "\x2e\xd9\x4d\x3b\x9e\x0f\x5b\xe5",
+		.assoc	= "\xe6\xdd\xb8\xc4\x89\xf8\xe0\xca"
+			  "\x4f\x10\x7a\x5f\x9c\xd8\x8b\x66"
+			  "\x3b",
+		.alen	= 17,
+		.input	= "\xb1\xab\x53\x4e\xc7\x40\x16\xb6"
+			  "\x71\x3a\x00\x9f\x41\x88\xb0\xb2"
+			  "\x71\x83\x85\x5f\xc8\x79\x0a\x99"
+			  "\x99\xdc\x89\x1c\x88\xd2\x3e\xf9"
+			  "\x83",
+		.ilen	= 33,
+		.result	= "\xf5\xa6\x46\x2c\xce\x97\x8a\x51"
+			  "\x6f\x46\xa6\x83\x9b\xa1\xbc\xe8"
+			  "\x05",
+		.rlen	= 17,
+	}, {
+		.key	= "\x05\x70\xd5\x94\x12\x36\x35\xd8"
+			  "\x8f\x7d\xd3\xa8\x99\x6a\xed\x69",
+		.klen	= 16,
+		.iv	= "\x14\x39\x63\xfc\x56\xd5\xdf\x5f"
+			  "\xaf\xb3\xff\xcc\x98\x33\x1d\xeb",
+		.assoc	= "\x23\x02\xf1\x64\x9a\x73\x89\xe6"
+			  "\xd0\xea\x2c\xf1\x96\xfc\x4e\x6d"
+			  "\x65\x48\xcb\x0a\xda\xf0\x62\xc0"
+			  "\x38\x1d\x3b\x4a\xe9\x7e\x62",
+		.alen	= 31,
+		.input	= "\x29\xc4\xf0\x03\xc1\x86\xdf\x06"
+			  "\x5c\x7b\xef\x64\x87\x00\xd1\x37"
+			  "\xa7\x08\xbc\x7f\x8f\x41\x54\xd0"
+			  "\x3e\xf1\xc3\xa2\x96\x84\xdd\x2a"
+			  "\x2d\x21\x30\xf9\x02\xdb\x06\x0c"
+			  "\xf1\x5a\x66\x69\xe0\xca\x83",
+		.ilen	= 47,
+		.result	= "\x32\xcb\x80\xcc\xde\x12\x33\x6d"
+			  "\xf0\x20\x58\x15\x95\xc6\x7f\xee"
+			  "\x2f\xf9\x4e\x2c\x1b\x98\x43\xc7"
+			  "\x68\x28\x73\x40\x9f\x96\x4a",
+		.rlen	= 31,
+	}, {
+		.key	= "\x41\x94\x0e\x33\x22\xb1\xdd\xf4"
+			  "\x10\x57\x85\x39\x93\x8f\xaf\x70",
+		.klen	= 16,
+		.iv	= "\x50\x5d\x9d\x9b\x66\x50\x88\x7b"
+			  "\x30\x8e\xb1\x5e\x92\x58\xe0\xf1",
+		.assoc	= "\x5f\x27\x2b\x03\xaa\xef\x32\x02"
+			  "\x50\xc4\xde\x82\x90\x21\x11\x73"
+			  "\x8f\x0a\xd6\x8f\xdf\x90\xe4\xda"
+			  "\xf9\x4a\x1a\x23\xc3\xdd\x02\x81",
+		.alen	= 32,
+		.input	= "\xe2\x2e\x44\xdf\xd3\x60\x6d\xb2"
+			  "\x70\x57\x37\xc5\xc2\x4f\x8d\x14"
+			  "\xc6\xbf\x8b\xec\xf5\x62\x67\xf2"
+			  "\x2f\xa1\xe6\xd6\xa7\xb1\x8c\x54"
+			  "\xe5\x6b\x49\xf9\x6e\x90\xc3\xaa"
+			  "\x7a\x00\x2e\x4d\x7f\x31\x2e\x81",
+		.ilen	= 48,
+		.result	= "\x6e\xf0\xba\x6b\xee\x8e\xdc\x89"
+			  "\x71\xfb\x0a\xa6\x8f\xea\x41\xf4"
+			  "\x5a\xbb\x59\xb0\x20\x38\xc5\xe0"
+			  "\x29\x56\x52\x19\x79\xf5\xe9\x37",
+		.rlen	= 32,
+	}, {
+		.key	= "\x7e\xb9\x48\xd3\x32\x2d\x86\x10"
+			  "\x91\x31\x37\xcb\x8d\xb3\x72\x76",
+		.klen	= 16,
+		.iv	= "\x8d\x82\xd6\x3b\x76\xcc\x30\x97"
+			  "\xb1\x68\x63\xef\x8c\x7c\xa3\xf7",
+		.assoc	= "\x9c\x4b\x65\xa2\xba\x6b\xdb\x1e"
+			  "\xd1\x9e\x90\x13\x8a\x45\xd3\x79"
+			  "\xba\xcd\xe2\x13\xe4\x30\x66\xf4"
+			  "\xba\x78\xf9\xfb\x9d\x3c\xa1\x58"
+			  "\x1a",
+		.alen	= 33,
+		.input	= "\xc7\xca\x26\x61\x57\xee\xa2\xb9"
+			  "\xb1\x37\xde\x95\x06\x90\x11\x08"
+			  "\x4d\x30\x9f\x24\xc0\x56\xb7\xe1"
+			  "\x0b\x9f\xd2\x57\xe9\xd2\xb1\x76"
+			  "\x56\x9a\xb4\x58\xc5\x08\xfc\xb5"
+			  "\xf2\x31\x9b\xc9\xcd\xb3\x64\xdb"
+			  "\x6f\x50\xbf\xf4\x73\x9d\xfb\x6b"
+			  "\xef\x35\x25\x48\xed\xcf\x29\xa8"
+			  "\xac\xc3\xb9\xcb\x61\x8f\x73\x92"
+			  "\x2c\x7a\x6f\xda\xf9\x09\x6f\xe1"
+			  "\xc4",
+		.ilen	= 81,
+		.result	= "\xab\x14\xf3\x0a\xfe\x0a\x85\xa5"
+			  "\xf2\xd5\xbc\x38\x89\x0e\x04\xfb"
+			  "\x84\x7d\x65\x34\x25\xd8\x47\xfa"
+			  "\xeb\x83\x31\xf1\x54\x54\x89\x0d"
+			  "\x9d\x4d\x54\x51\x84\x61\xf6\x8e"
+			  "\x03\x31\xf2\x25\x16\xcc\xaa\xc6"
+			  "\x75\x73\x20\x30\x59\x54\xb2\xf0"
+			  "\x3a\x4b\xe0\x23\x8e\xa6\x08\x35"
+			  "\x8a",
+		.rlen	= 65,
+	}, {
+		.key	= "\xba\xde\x82\x72\x42\xa9\x2f\x2c"
+			  "\x12\x0b\xe9\x5c\x87\xd7\x35\x7c",
+		.klen	= 16,
+		.iv	= "\xc9\xa7\x10\xda\x86\x48\xd9\xb3"
+			  "\x32\x42\x15\x80\x85\xa1\x65\xfe",
+		.assoc	= "\xd8\x70\x9f\x42\xca\xe6\x83\x3a"
+			  "\x52\x79\x42\xa5\x84\x6a\x96\x7f"
+			  "\xe4\x8f\xed\x97\xe9\xd0\xe8\x0d"
+			  "\x7c\xa6\xd8\xd4\x77\x9b\x40\x2e"
+			  "\x28\xce\x57\x34\xcd\x6e\x84\x4c"
+			  "\x17\x3c\xe1\xb2\xa8\x0b\xbb\xf1"
+			  "\x96\x41\x0d\x69\xe8\x54\x0a\xc8"
+			  "\x15\x4e\x91\x92\x89\x4b\xb7\x9b"
+			  "\x21",
+		.alen	= 65,
+		.input	= "\x57\xcd\x3d\x46\xc5\xf9\x68\x3b"
+			  "\x2c\x0f\xb4\x7e\x7b\x64\x3e\x40"
+			  "\xf3\x78\x63\x34\x89\x79\x39\x6b"
+			  "\x61\x64\x4a\x9a\xfa\x70\xa4\xd3"
+			  "\x54\x0b\xea\x05\xa6\x95\x64\xed"
+			  "\x3d\x69\xa2\x0c\x27\x56\x2f\x34"
+			  "\x66",
+		.ilen	= 49,
+		.result	= "\xe8\x39\x2d\xaa\x0e\x85\x2d\xc1"
+			  "\x72\xaf\x6e\xc9\x82\x33\xc7\x01"
+			  "\xaf\x40\x70\xb8\x2a\x78\xc9\x14"
+			  "\xac\xb1\x10\xca\x2e\xb3\x28\xe4"
+			  "\xac",
+		.rlen	= 33,
+	}, {
+		.key	= "\xf7\x02\xbb\x11\x52\x24\xd8\x48"
+			  "\x93\xe6\x9b\xee\x81\xfc\xf7\x82",
+		.klen	= 16,
+		.iv	= "\x06\xcc\x4a\x79\x96\xc3\x82\xcf"
+			  "\xb3\x1c\xc7\x12\x7f\xc5\x28\x04",
+		.assoc	= "\x15\x95\xd8\xe1\xda\x62\x2c\x56"
+			  "\xd3\x53\xf4\x36\x7e\x8e\x59\x85",
+		.alen	= 16,
+		.input	= "\xfc\x85\x06\x28\x8f\xe8\x23\x1f"
+			  "\x33\x98\x87\xde\x08\xb6\xb6\xae"
+			  "\x3e\xa4\xf8\x19\xf1\x92\x60\x39"
+			  "\xb9\x6b\x3f\xdf\xc8\xcb\x30",
+		.ilen	= 31,
+		.result	= "\x24\x5e\x67\x49\x1e\x01\xd6\xdd"
+			  "\xf3\x89\x20\x5b\x7c\x57\x89\x07",
+		.rlen	= 16,
+	}, {
+		.key	= "\x33\x27\xf5\xb1\x62\xa0\x80\x63"
+			  "\x14\xc0\x4d\x7f\x7b\x20\xba\x89",
+		.klen	= 16,
+		.iv	= "\x42\xf0\x84\x19\xa6\x3f\x2b\xea"
+			  "\x34\xf6\x79\xa3\x79\xe9\xeb\x0a",
+		.assoc	= "\x51\xb9\x12\x80\xea\xde\xd5\x71"
+			  "\x54\x2d\xa6\xc8\x78\xb2\x1b\x8c",
+		.alen	= 16,
+		.input	= "\x74\x7d\x70\x07\xe9\xba\x01\xee"
+			  "\x6c\xc6\x6f\x50\x25\x33\xbe\x50"
+			  "\x17\xb8\x17\x62\xed\x80\xa2\xf5"
+			  "\x03\xde\x85\x71\x5d\x34",
+		.ilen	= 30,
+		.result	= "\x61\x83\xa0\xe8\x2e\x7d\x7f\xf8"
+			  "\x74\x63\xd2\xec\x76\x7c\x4c\x0d",
+		.rlen	= 16,
+	}, {
+		.key	= "\x70\x4c\x2f\x50\x72\x1c\x29\x7f"
+			  "\x95\x9a\xff\x10\x75\x45\x7d\x8f",
+		.klen	= 16,
+		.iv	= "\x7f\x15\xbd\xb8\xb6\xba\xd3\x06"
+			  "\xb5\xd1\x2b\x35\x73\x0e\xad\x10",
+		.assoc	= "\x8e\xde\x4c\x20\xfa\x59\x7e\x8d"
+			  "\xd5\x07\x58\x59\x72\xd7\xde\x92",
+		.alen	= 16,
+		.input	= "\xf4\xb3\x85\xf9\xac\xde\xb1\x38"
+			  "\x29\xfd\x6c\x7c\x49\xe5\x1d\xaf"
+			  "\xba\xea\xd4\xfa\x3f\x11\x33\x98",
+		.ilen	= 24,
+		.result	= "\x9d\xa7\xda\x88\x3e\xf8\x28\x14"
+			  "\xf5\x3e\x85\x7d\x70\xa0\x0f\x13",
+		.rlen	= 16,
+	}, {
+		.key	= "\xac\x70\x69\xef\x82\x97\xd2\x9b"
+			  "\x15\x74\xb1\xa2\x6f\x69\x3f\x95",
+		.klen	= 16,
+		.iv	= "\xbb\x3a\xf7\x57\xc6\x36\x7c\x22"
+			  "\x36\xab\xde\xc6\x6d\x32\x70\x17",
+		.assoc	= "\xcb\x03\x85\xbf\x0a\xd5\x26\xa9"
+			  "\x56\xe1\x0a\xeb\x6c\xfb\xa1\x98",
+		.alen	= 16,
+		.input	= "\xe6\x5c\x49\x4f\x78\xf3\x62\x86"
+			  "\xe1\xb7\xa5\xc3\x32\x88\x3c\x8c"
+			  "\x6e",
+		.ilen	= 17,
+		.result	= "\xda\xcc\x14\x27\x4e\x74\xd1\x30"
+			  "\x76\x18\x37\x0f\x6a\xc4\xd1\x1a",
+		.rlen	= 16,
+	},
+};
+
+/*
+ * MORUS-1280 test vectors - generated via reference implementation from
+ * SUPERCOP (https://bench.cr.yp.to/supercop.html):
+ *
+ *   https://bench.cr.yp.to/supercop/supercop-20170228.tar.xz
+ *   (see crypto_aead/morus1280128v2/ and crypto_aead/morus1280256v2/ )
+ */
+static const struct aead_testvec morus1280_enc_tv_template[] = {
+	{
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 16,
+		.iv	= "\x0f\xc9\x8e\x67\x44\x9e\xaa\x86"
+			  "\x20\x36\x2c\x24\xfe\xc9\x30\x81",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x91\x85\x0f\xf5\x52\x9e\xce\xce"
+			  "\x65\x99\xc7\xbf\xd3\x76\xe8\x98",
+		.rlen	= 16,
+	}, {
+		.key	= "\x3c\x24\x39\x9f\x10\x7b\xa8\x1b"
+			  "\x80\xda\xb2\x91\xf9\x24\xc2\x06",
+		.klen	= 16,
+		.iv	= "\x4b\xed\xc8\x07\x54\x1a\x52\xa2"
+			  "\xa1\x10\xde\xb5\xf8\xed\xf3\x87",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x69",
+		.ilen	= 1,
+		.result	= "\x88\xc3\x4c\xf0\x2f\x43\x76\x13"
+			  "\x96\xda\x76\x34\x33\x4e\xd5\x39"
+			  "\x73",
+		.rlen	= 17,
+	}, {
+		.key	= "\x79\x49\x73\x3e\x20\xf7\x51\x37"
+			  "\x01\xb4\x64\x22\xf3\x48\x85\x0c",
+		.klen	= 16,
+		.iv	= "\x88\x12\x01\xa6\x64\x96\xfb\xbe"
+			  "\x22\xea\x90\x47\xf2\x11\xb5\x8e",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xa6\xa4\x1e\x76\xec\xd4\x50\xcc"
+			  "\x62\x58\xe9\x8f\xef\xa4\x17\x91"
+			  "\xb4\x96\x9f\x6b\xce\x38\xa5\x46"
+			  "\x13\x7d\x64\x93\xd7\x05\xf5",
+		.ilen	= 31,
+		.result	= "\x3e\x5c\x3b\x58\x3b\x7d\x2a\x22"
+			  "\x75\x0b\x24\xa6\x0e\xc3\xde\x52"
+			  "\x97\x0b\x64\xd4\xce\x90\x52\xf7"
+			  "\xef\xdb\x6a\x38\xd2\xa8\xa1\x0d"
+			  "\xe0\x61\x33\x24\xc6\x4d\x51\xbc"
+			  "\xa4\x21\x74\xcf\x19\x16\x59",
+		.rlen	= 47,
+	}, {
+		.key	= "\xb5\x6e\xad\xdd\x30\x72\xfa\x53"
+			  "\x82\x8e\x16\xb4\xed\x6d\x47\x12",
+		.klen	= 16,
+		.iv	= "\xc4\x37\x3b\x45\x74\x11\xa4\xda"
+			  "\xa2\xc5\x42\xd8\xec\x36\x78\x94",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xe2\xc9\x58\x15\xfc\x4f\xf8\xe8"
+			  "\xe3\x32\x9b\x21\xe9\xc8\xd9\x97"
+			  "\xde\x58\xab\xf0\xd3\xd8\x27\x60"
+			  "\xd5\xaa\x43\x6b\xb1\x64\x95\xa4",
+		.ilen	= 32,
+		.result	= "\x30\x82\x9c\x2b\x67\xcb\xf9\x1f"
+			  "\xde\x9f\x77\xb2\xda\x92\x61\x5c"
+			  "\x09\x0b\x2d\x9a\x26\xaa\x1c\x06"
+			  "\xab\x74\xb7\x2b\x95\x5f\x9f\xa1"
+			  "\x9a\xff\x50\xa0\xa2\xff\xc5\xad"
+			  "\x21\x8e\x84\x5c\x12\x61\xb2\xae",
+		.rlen	= 48,
+	}, {
+		.key	= "\xf2\x92\xe6\x7d\x40\xee\xa3\x6f"
+			  "\x03\x68\xc8\x45\xe7\x91\x0a\x18",
+		.klen	= 16,
+		.iv	= "\x01\x5c\x75\xe5\x84\x8d\x4d\xf6"
+			  "\x23\x9f\xf4\x6a\xe6\x5a\x3b\x9a",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x1f\xee\x92\xb4\x0c\xcb\xa1\x04"
+			  "\x64\x0c\x4d\xb2\xe3\xec\x9c\x9d"
+			  "\x09\x1a\xb7\x74\xd8\x78\xa9\x79"
+			  "\x96\xd8\x22\x43\x8c\xc3\x34\x7b"
+			  "\xc4",
+		.ilen	= 33,
+		.result	= "\x67\x5d\x8e\x45\xc8\x39\xf5\x17"
+			  "\xc1\x1d\x2a\xdd\x88\x67\xda\x1f"
+			  "\x6d\xe8\x37\x28\x5a\xc1\x5e\x9f"
+			  "\xa6\xec\xc6\x92\x05\x4b\xc0\xa3"
+			  "\x63\xef\x88\xa4\x9b\x0a\x5c\xed"
+			  "\x2b\x6a\xac\x63\x52\xaa\x10\x94"
+			  "\xd0",
+		.rlen	= 49,
+	}, {
+		.key	= "\x2e\xb7\x20\x1c\x50\x6a\x4b\x8b"
+			  "\x84\x42\x7a\xd7\xe1\xb5\xcd\x1f",
+		.klen	= 16,
+		.iv	= "\x3d\x80\xae\x84\x94\x09\xf6\x12"
+			  "\xa4\x79\xa6\xfb\xe0\x7f\xfd\xa0",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x5c\x13\xcb\x54\x1c\x47\x4a\x1f"
+			  "\xe5\xe6\xff\x44\xdd\x11\x5f\xa3"
+			  "\x33\xdd\xc2\xf8\xdd\x18\x2b\x93"
+			  "\x57\x05\x01\x1c\x66\x22\xd3\x51"
+			  "\xd3\xdf\x18\xc9\x30\x66\xed\xb1"
+			  "\x96\x58\xd5\x8c\x64\x8c\x7c\xf5"
+			  "\x01\xd0\x74\x5f\x9b\xaa\xf6\xd1"
+			  "\xe6\x16\xa2\xac\xde\x47\x40",
+		.ilen	= 63,
+		.result	= "\x7d\x61\x1a\x35\x20\xcc\x07\x88"
+			  "\x03\x98\x87\xcf\xc0\x6e\x4d\x19"
+			  "\xe3\xd4\x0b\xfb\x29\x8f\x49\x1a"
+			  "\x3a\x06\x77\xce\x71\x2c\xcd\xdd"
+			  "\xed\xf6\xc9\xbe\xa6\x3b\xb8\xfc"
+			  "\x6c\xbe\x77\xed\x74\x0e\x20\x85"
+			  "\xd0\x65\xde\x24\x6f\xe3\x25\xc5"
+			  "\xdf\x5b\x0f\xbd\x8a\x88\x78\xc9"
+			  "\xe5\x81\x37\xde\x84\x7a\xf6\x84"
+			  "\x99\x7a\x72\x9c\x54\x31\xa1",
+		.rlen	= 79,
+	}, {
+		.key	= "\x6b\xdc\x5a\xbb\x60\xe5\xf4\xa6"
+			  "\x05\x1d\x2c\x68\xdb\xda\x8f\x25",
+		.klen	= 16,
+		.iv	= "\x7a\xa5\xe8\x23\xa4\x84\x9e\x2d"
+			  "\x25\x53\x58\x8c\xda\xa3\xc0\xa6",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x98\x37\x05\xf3\x2c\xc2\xf3\x3b"
+			  "\x66\xc0\xb1\xd5\xd7\x35\x21\xaa"
+			  "\x5d\x9f\xce\x7c\xe2\xb8\xad\xad"
+			  "\x19\x33\xe0\xf4\x40\x81\x72\x28"
+			  "\xe1\x8b\x1c\xf8\x91\x78\xff\xaf"
+			  "\xb0\x68\x69\xf2\x27\x35\x91\x84"
+			  "\x2e\x37\x5b\x00\x04\xff\x16\x9c"
+			  "\xb5\x19\x39\xeb\xd9\xcd\x29\x9a",
+		.ilen	= 64,
+		.result	= "\x05\xc5\xb1\xf9\x1b\xb9\xab\x2c"
+			  "\xa5\x07\x12\xa7\x12\x39\x60\x66"
+			  "\x30\x81\x4a\x03\x78\x28\x45\x52"
+			  "\xd2\x2b\x24\xfd\x8b\xa5\xb7\x66"
+			  "\x6f\x45\xd7\x3b\x67\x6f\x51\xb9"
+			  "\xc0\x3d\x6c\xca\x1e\xae\xff\xb6"
+			  "\x79\xa9\xe4\x82\x5d\x4c\x2d\xdf"
+			  "\xeb\x71\x40\xc9\x2c\x40\x45\x6d"
+			  "\x73\x77\x01\xf3\x4f\xf3\x9d\x2a"
+			  "\x5d\x57\xa8\xa1\x18\xa2\xad\xcb",
+		.rlen	= 80,
+	}, {
+		.key	= "\xa7\x00\x93\x5b\x70\x61\x9d\xc2"
+			  "\x86\xf7\xde\xfa\xd5\xfe\x52\x2b",
+		.klen	= 16,
+		.iv	= "\xb6\xca\x22\xc3\xb4\x00\x47\x49"
+			  "\xa6\x2d\x0a\x1e\xd4\xc7\x83\xad",
+		.assoc	= "\xc5",
+		.alen	= 1,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x4d\xbf\x11\xac\x7f\x97\x0b\x2e"
+			  "\x89\x3b\x9d\x0f\x83\x1c\x08\xc3",
+		.rlen	= 16,
+	}, {
+		.key	= "\xe4\x25\xcd\xfa\x80\xdd\x46\xde"
+			  "\x07\xd1\x90\x8b\xcf\x23\x15\x31",
+		.klen	= 16,
+		.iv	= "\xf3\xee\x5c\x62\xc4\x7c\xf0\x65"
+			  "\x27\x08\xbd\xaf\xce\xec\x45\xb3",
+		.assoc	= "\x02\xb8\xea\xca\x09\x1b\x9a\xec"
+			  "\x47\x3e\xe9\xd4\xcc\xb5\x76\x34"
+			  "\xe8\x73\x62\x64\xab\x50\xd0\xda"
+			  "\x6b\x83\x66\xaf\x3e\x27\xc9",
+		.alen	= 31,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x5b\xc0\x8d\x54\xe4\xec\xbe\x38"
+			  "\x03\x12\xf9\xcc\x9e\x46\x42\x92",
+		.rlen	= 16,
+	}, {
+		.key	= "\x20\x4a\x07\x99\x91\x58\xee\xfa"
+			  "\x88\xab\x42\x1c\xc9\x47\xd7\x38",
+		.klen	= 16,
+		.iv	= "\x2f\x13\x95\x01\xd5\xf7\x99\x81"
+			  "\xa8\xe2\x6f\x41\xc8\x10\x08\xb9",
+		.assoc	= "\x3f\xdc\x24\x69\x19\x96\x43\x08"
+			  "\xc8\x18\x9b\x65\xc6\xd9\x39\x3b"
+			  "\x12\x35\x6e\xe8\xb0\xf0\x52\xf3"
+			  "\x2d\xb0\x45\x87\x18\x86\x68\xf6",
+		.alen	= 32,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x48\xc5\xc3\x4c\x40\x2e\x2f\xc2"
+			  "\x6d\x65\xe0\x67\x9c\x1d\xa0\xf0",
+		.rlen	= 16,
+	}, {
+		.key	= "\x5d\x6f\x41\x39\xa1\xd4\x97\x16"
+			  "\x09\x85\xf4\xae\xc3\x6b\x9a\x3e",
+		.klen	= 16,
+		.iv	= "\x6c\x38\xcf\xa1\xe5\x73\x41\x9d"
+			  "\x29\xbc\x21\xd2\xc2\x35\xcb\xbf",
+		.assoc	= "\x7b\x01\x5d\x08\x29\x12\xec\x24"
+			  "\x49\xf3\x4d\xf7\xc0\xfe\xfb\x41"
+			  "\x3c\xf8\x79\x6c\xb6\x90\xd4\x0d"
+			  "\xee\xde\x23\x60\xf2\xe5\x08\xcc"
+			  "\x97",
+		.alen	= 33,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x28\x64\x78\x51\x55\xd8\x56\x4a"
+			  "\x58\x3e\xf7\xbe\xee\x21\xfe\x94",
+		.rlen	= 16,
+	}, {
+		.key	= "\x99\x93\x7a\xd8\xb1\x50\x40\x31"
+			  "\x8a\x60\xa6\x3f\xbd\x90\x5d\x44",
+		.klen	= 16,
+		.iv	= "\xa8\x5c\x09\x40\xf5\xef\xea\xb8"
+			  "\xaa\x96\xd3\x64\xbc\x59\x8d\xc6",
+		.assoc	= "\xb8\x26\x97\xa8\x39\x8e\x94\x3f"
+			  "\xca\xcd\xff\x88\xba\x22\xbe\x47"
+			  "\x67\xba\x85\xf1\xbb\x30\x56\x26"
+			  "\xaf\x0b\x02\x38\xcc\x44\xa7\xa3"
+			  "\xa6\xbf\x31\x93\x60\xcd\xda\x63"
+			  "\x2c\xb1\xaa\x19\xc8\x19\xf8\xeb"
+			  "\x03\xa1\xe8\xbe\x37\x54\xec\xa2"
+			  "\xcd\x2c\x45\x58\xbd\x8e\x80",
+		.alen	= 63,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xb3\xa6\x00\x4e\x09\x20\xac\x21"
+			  "\x77\x72\x69\x76\x2d\x36\xe5\xc8",
+		.rlen	= 16,
+	}, {
+		.key	= "\xd6\xb8\xb4\x77\xc1\xcb\xe9\x4d"
+			  "\x0a\x3a\x58\xd1\xb7\xb4\x1f\x4a",
+		.klen	= 16,
+		.iv	= "\xe5\x81\x42\xdf\x05\x6a\x93\xd4"
+			  "\x2b\x70\x85\xf5\xb6\x7d\x50\xcc",
+		.assoc	= "\xf4\x4a\xd1\x47\x49\x09\x3d\x5b"
+			  "\x4b\xa7\xb1\x19\xb4\x46\x81\x4d"
+			  "\x91\x7c\x91\x75\xc0\xd0\xd8\x40"
+			  "\x71\x39\xe1\x10\xa6\xa3\x46\x7a"
+			  "\xb4\x6b\x35\xc2\xc1\xdf\xed\x60"
+			  "\x46\xc1\x3e\x7f\x8c\xc2\x0e\x7a"
+			  "\x30\x08\xd0\x5f\xa0\xaa\x0c\x6d"
+			  "\x9c\x2f\xdb\x97\xb8\x15\x69\x01",
+		.alen	= 64,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x65\x33\x7b\xa1\x63\xf4\x20\xdd"
+			  "\xe4\xb9\x4a\xaa\x9a\x21\xaa\x14",
+		.rlen	= 16,
+	}, {
+		.key	= "\x12\xdd\xee\x17\xd1\x47\x92\x69"
+			  "\x8b\x14\x0a\x62\xb1\xd9\xe2\x50",
+		.klen	= 16,
+		.iv	= "\x22\xa6\x7c\x7f\x15\xe6\x3c\xf0"
+			  "\xac\x4b\x37\x86\xb0\xa2\x13\xd2",
+		.assoc	= "\x31",
+		.alen	= 1,
+		.input	= "\x40",
+		.ilen	= 1,
+		.result	= "\x1d\x47\x17\x34\x86\xf5\x54\x1a"
+			  "\x6d\x28\xb8\x5d\x6c\xcf\xa0\xb9"
+			  "\xbf",
+		.rlen	= 17,
+	}, {
+		.key	= "\x4f\x01\x27\xb6\xe1\xc3\x3a\x85"
+			  "\x0c\xee\xbc\xf4\xab\xfd\xa5\x57",
+		.klen	= 16,
+		.iv	= "\x5e\xcb\xb6\x1e\x25\x62\xe4\x0c"
+			  "\x2d\x25\xe9\x18\xaa\xc6\xd5\xd8",
+		.assoc	= "\x6d\x94\x44\x86\x69\x00\x8f\x93"
+			  "\x4d\x5b\x15\x3c\xa8\x8f\x06\x5a"
+			  "\xe6\x01\xa8\x7e\xca\x10\xdc\x73"
+			  "\xf4\x94\x9f\xc1\x5a\x61\x85",
+		.alen	= 31,
+		.input	= "\x7c\x5d\xd3\xee\xad\x9f\x39\x1a"
+			  "\x6d\x92\x42\x61\xa7\x58\x37\xdb"
+			  "\xb0\xb2\x2b\x9f\x0b\xb8\xbd\x7a"
+			  "\x24\xa0\xd6\xb7\x11\x79\x6c",
+		.ilen	= 31,
+		.result	= "\x78\x90\x52\xae\x0f\xf7\x2e\xef"
+			  "\x63\x09\x08\x58\xb5\x56\xbd\x72"
+			  "\x6e\x42\xcf\x27\x04\x7c\xdb\x92"
+			  "\x18\xe9\xa4\x33\x90\xba\x62\xb5"
+			  "\x70\xd3\x88\x9b\x4f\x05\xa7\x51"
+			  "\x85\x87\x17\x09\x42\xed\x4e",
+		.rlen	= 47,
+	}, {
+		.key	= "\x8b\x26\x61\x55\xf1\x3e\xe3\xa1"
+			  "\x8d\xc8\x6e\x85\xa5\x21\x67\x5d",
+		.klen	= 16,
+		.iv	= "\x9b\xef\xf0\xbd\x35\xdd\x8d\x28"
+			  "\xad\xff\x9b\xa9\xa4\xeb\x98\xdf",
+		.assoc	= "\xaa\xb8\x7e\x25\x79\x7c\x37\xaf"
+			  "\xce\x36\xc7\xce\xa2\xb4\xc9\x60"
+			  "\x10\xc3\xb3\x02\xcf\xb0\x5e\x8d"
+			  "\xb5\xc2\x7e\x9a\x35\xc0\x24\xfd",
+		.alen	= 32,
+		.input	= "\xb9\x82\x0c\x8d\xbd\x1b\xe2\x36"
+			  "\xee\x6c\xf4\xf2\xa1\x7d\xf9\xe2"
+			  "\xdb\x74\x36\x23\x11\x58\x3f\x93"
+			  "\xe5\xcd\xb5\x90\xeb\xd8\x0c\xb3",
+		.ilen	= 32,
+		.result	= "\x1d\x2c\x57\xe0\x50\x38\x3d\x41"
+			  "\x2e\x71\xc8\x3b\x92\x43\x58\xaf"
+			  "\x5a\xfb\xad\x8f\xd9\xd5\x8a\x5e"
+			  "\xdb\xf3\xcd\x3a\x2b\xe1\x2c\x1a"
+			  "\xb0\xed\xe3\x0c\x6e\xf9\xf2\xd6"
+			  "\x90\xe6\xb1\x0e\xa5\x8a\xac\xb7",
+		.rlen	= 48,
+	}, {
+		.key	= "\xc8\x4b\x9b\xf5\x01\xba\x8c\xbd"
+			  "\x0e\xa3\x21\x16\x9f\x46\x2a\x63",
+		.klen	= 16,
+		.iv	= "\xd7\x14\x29\x5d\x45\x59\x36\x44"
+			  "\x2e\xd9\x4d\x3b\x9e\x0f\x5b\xe5",
+		.assoc	= "\xe6\xdd\xb8\xc4\x89\xf8\xe0\xca"
+			  "\x4f\x10\x7a\x5f\x9c\xd8\x8b\x66"
+			  "\x3b\x86\xbf\x86\xd4\x50\xe0\xa7"
+			  "\x76\xef\x5c\x72\x0f\x1f\xc3\xd4"
+			  "\xee",
+		.alen	= 33,
+		.input	= "\xf5\xa6\x46\x2c\xce\x97\x8a\x51"
+			  "\x6f\x46\xa6\x83\x9b\xa1\xbc\xe8"
+			  "\x05\x36\x42\xa7\x16\xf8\xc1\xad"
+			  "\xa7\xfb\x94\x68\xc5\x37\xab\x8a"
+			  "\x72",
+		.ilen	= 33,
+		.result	= "\x59\x10\x84\x1c\x83\x4c\x8b\xfc"
+			  "\xfd\x2e\x4b\x46\x84\xff\x78\x4e"
+			  "\x50\xda\x5c\xb9\x61\x1d\xf5\xb9"
+			  "\xfe\xbb\x7f\xae\x8c\xc1\x24\xbd"
+			  "\x8c\x6f\x1f\x9b\xce\xc6\xc1\x37"
+			  "\x08\x06\x5a\xe5\x96\x10\x95\xc2"
+			  "\x5e",
+		.rlen	= 49,
+	}, {
+		.key	= "\x05\x70\xd5\x94\x12\x36\x35\xd8"
+			  "\x8f\x7d\xd3\xa8\x99\x6a\xed\x69",
+		.klen	= 16,
+		.iv	= "\x14\x39\x63\xfc\x56\xd5\xdf\x5f"
+			  "\xaf\xb3\xff\xcc\x98\x33\x1d\xeb",
+		.assoc	= "\x23\x02\xf1\x64\x9a\x73\x89\xe6"
+			  "\xd0\xea\x2c\xf1\x96\xfc\x4e\x6d"
+			  "\x65\x48\xcb\x0a\xda\xf0\x62\xc0"
+			  "\x38\x1d\x3b\x4a\xe9\x7e\x62\xaa"
+			  "\xfd\xc9\x4a\xa9\xa9\x39\x4b\x54"
+			  "\xc8\x0e\x24\x7f\x5e\x10\x7a\x45"
+			  "\x10\x0b\x56\x85\xad\x54\xaa\x66"
+			  "\xa8\x43\xcd\xd4\x9b\xb7\xfa",
+		.alen	= 63,
+		.input	= "\x32\xcb\x80\xcc\xde\x12\x33\x6d"
+			  "\xf0\x20\x58\x15\x95\xc6\x7f\xee"
+			  "\x2f\xf9\x4e\x2c\x1b\x98\x43\xc7"
+			  "\x68\x28\x73\x40\x9f\x96\x4a\x60"
+			  "\x80\xf4\x4b\xf4\xc1\x3d\xd0\x93"
+			  "\xcf\x12\xc9\x59\x8f\x7a\x7f\xa8"
+			  "\x1b\xa5\x50\xed\x87\xa9\x72\x59"
+			  "\x9c\x44\xb2\xa4\x99\x98\x34",
+		.ilen	= 63,
+		.result	= "\x9a\x12\xbc\xdf\x72\xa8\x56\x22"
+			  "\x49\x2d\x07\x92\xfc\x3d\x6d\x5f"
+			  "\xef\x36\x19\xae\x91\xfa\xd6\x63"
+			  "\x46\xea\x8a\x39\x14\x21\xa6\x37"
+			  "\x18\xfc\x97\x3e\x16\xa5\x4d\x39"
+			  "\x45\x2e\x69\xcc\x9c\x5f\xdf\x6d"
+			  "\x5e\xa2\xbf\xac\x83\x32\x72\x52"
+			  "\x58\x58\x23\x40\xfd\xa5\xc2\xe6"
+			  "\xe9\x5a\x50\x98\x00\x58\xc9\x86"
+			  "\x4f\x20\x37\xdb\x7b\x22\xa3",
+		.rlen	= 79,
+	}, {
+		.key	= "\x41\x94\x0e\x33\x22\xb1\xdd\xf4"
+			  "\x10\x57\x85\x39\x93\x8f\xaf\x70",
+		.klen	= 16,
+		.iv	= "\x50\x5d\x9d\x9b\x66\x50\x88\x7b"
+			  "\x30\x8e\xb1\x5e\x92\x58\xe0\xf1",
+		.assoc	= "\x5f\x27\x2b\x03\xaa\xef\x32\x02"
+			  "\x50\xc4\xde\x82\x90\x21\x11\x73"
+			  "\x8f\x0a\xd6\x8f\xdf\x90\xe4\xda"
+			  "\xf9\x4a\x1a\x23\xc3\xdd\x02\x81"
+			  "\x0b\x76\x4f\xd7\x0a\x4b\x5e\x51"
+			  "\xe3\x1d\xb9\xe5\x21\xb9\x8f\xd4"
+			  "\x3d\x72\x3e\x26\x16\xa9\xca\x32"
+			  "\x77\x47\x63\x14\x95\x3d\xe4\x34",
+		.alen	= 64,
+		.input	= "\x6e\xf0\xba\x6b\xee\x8e\xdc\x89"
+			  "\x71\xfb\x0a\xa6\x8f\xea\x41\xf4"
+			  "\x5a\xbb\x59\xb0\x20\x38\xc5\xe0"
+			  "\x29\x56\x52\x19\x79\xf5\xe9\x37"
+			  "\x8f\xa1\x50\x23\x22\x4f\xe3\x91"
+			  "\xe9\x21\x5e\xbf\x52\x23\x95\x37"
+			  "\x48\x0c\x38\x8f\xf0\xff\x92\x24"
+			  "\x6b\x47\x49\xe3\x94\x1f\x1e\x01",
+		.ilen	= 64,
+		.result	= "\xe6\xeb\x92\x5a\x5b\xf0\x2d\xbb"
+			  "\x23\xec\x35\xe3\xae\xc9\xfb\x0b"
+			  "\x90\x14\x46\xeb\xa8\x8d\xb0\x9b"
+			  "\x39\xda\x8b\x48\xec\xb2\x00\x4e"
+			  "\x80\x6f\x46\x4f\x9b\x1e\xbb\x35"
+			  "\xea\x5a\xbc\xa2\x36\xa5\x89\x45"
+			  "\xc2\xd6\xd7\x15\x0b\xf6\x6c\x56"
+			  "\xec\x99\x7d\x61\xb3\x15\x93\xed"
+			  "\x83\x1e\xd9\x48\x84\x0b\x37\xfe"
+			  "\x95\x74\x44\xd5\x54\xa6\x27\x06",
+		.rlen	= 80,
+	}, {
+		.key	= "\x7e\xb9\x48\xd3\x32\x2d\x86\x10"
+			  "\x91\x31\x37\xcb\x8d\xb3\x72\x76",
+		.klen	= 16,
+		.iv	= "\x8d\x82\xd6\x3b\x76\xcc\x30\x97"
+			  "\xb1\x68\x63\xef\x8c\x7c\xa3\xf7",
+		.assoc	= "\x9c\x4b\x65\xa2\xba\x6b\xdb\x1e"
+			  "\xd1\x9e\x90\x13\x8a\x45\xd3\x79"
+			  "\xba\xcd\xe2\x13\xe4\x30\x66\xf4"
+			  "\xba\x78\xf9\xfb\x9d\x3c\xa1\x58"
+			  "\x1a\x22\x53\x05\x6b\x5c\x71\x4f"
+			  "\xfd\x2d\x4d\x4c\xe5\x62\xa5\x63"
+			  "\x6a\xda\x26\xc8\x7f\xff\xea\xfd"
+			  "\x46\x4a\xfa\x53\x8f\xc4\xcd\x68"
+			  "\x58",
+		.alen	= 65,
+		.input	= "\xab\x14\xf3\x0a\xfe\x0a\x85\xa5"
+			  "\xf2\xd5\xbc\x38\x89\x0e\x04\xfb"
+			  "\x84\x7d\x65\x34\x25\xd8\x47\xfa"
+			  "\xeb\x83\x31\xf1\x54\x54\x89\x0d"
+			  "\x9d\x4d\x54\x51\x84\x61\xf6\x8e"
+			  "\x03\x31\xf2\x25\x16\xcc\xaa\xc6"
+			  "\x75\x73\x20\x30\x59\x54\xb2\xf0"
+			  "\x3a\x4b\xe0\x23\x8e\xa6\x08\x35"
+			  "\x8a\xdf\x27\xa0\xe4\x60\x99\xae"
+			  "\x8e\x43\xd9\x39\x7b\x10\x40\x67"
+			  "\x5c\x7e\xc9\x70\x63\x34\xca\x59"
+			  "\xfe\x86\xbc\xb7\x9c\x39\xf3\x6d"
+			  "\x6a\x41\x64\x6f\x16\x7f\x65\x7e"
+			  "\x89\x84\x68\xeb\xb0\x51\xbe\x55"
+			  "\x33\x16\x59\x6c\x3b\xef\x88\xad"
+			  "\x2f\xab\xbc\x25\x76\x87\x41\x2f"
+			  "\x36",
+		.ilen	= 129,
+		.result	= "\x89\x24\x27\x86\xdc\xd7\x6b\xd9"
+			  "\xd1\xcd\xdc\x16\xdd\x2c\xc1\xfb"
+			  "\x52\xb5\xb3\xab\x50\x99\x3f\xa0"
+			  "\x38\xa4\x74\xa5\x04\x15\x63\x05"
+			  "\x8f\x54\x81\x06\x5a\x6b\xa4\x63"
+			  "\x6d\xa7\x21\xcb\xff\x42\x30\x8e"
+			  "\x3b\xd1\xca\x3f\x4b\x1a\xb8\xc3"
+			  "\x42\x01\xe6\xbc\x75\x15\x87\xee"
+			  "\xc9\x8e\x65\x01\xd9\xd8\xb5\x9f"
+			  "\x48\x86\xa6\x5f\x2c\xc7\xb5\xb0"
+			  "\xed\x5d\x14\x7c\x3f\x40\xb1\x0b"
+			  "\x72\xef\x94\x8d\x7a\x85\x56\xe5"
+			  "\x56\x08\x15\x56\xba\xaf\xbd\xf0"
+			  "\x20\xef\xa0\xf6\xa9\xad\xa2\xc9"
+			  "\x1c\x3b\x28\x51\x7e\x77\xb2\x18"
+			  "\x4f\x61\x64\x37\x22\x36\x6d\x78"
+			  "\xed\xed\x35\xe8\x83\xa5\xec\x25"
+			  "\x6b\xff\x5f\x1a\x09\x96\x3d\xdc"
+			  "\x20",
+		.rlen	= 145,
+	}, {
+		.key	= "\xba\xde\x82\x72\x42\xa9\x2f\x2c"
+			  "\x12\x0b\xe9\x5c\x87\xd7\x35\x7c",
+		.klen	= 16,
+		.iv	= "\xc9\xa7\x10\xda\x86\x48\xd9\xb3"
+			  "\x32\x42\x15\x80\x85\xa1\x65\xfe",
+		.assoc	= "\xd8\x70\x9f\x42\xca\xe6\x83\x3a"
+			  "\x52\x79\x42\xa5\x84\x6a\x96\x7f"
+			  "\xe4\x8f\xed\x97\xe9\xd0\xe8\x0d"
+			  "\x7c\xa6\xd8\xd4\x77\x9b\x40\x2e"
+			  "\x28\xce\x57\x34\xcd\x6e\x84\x4c"
+			  "\x17\x3c\xe1\xb2\xa8\x0b\xbb\xf1"
+			  "\x96\x41\x0d\x69\xe8\x54\x0a\xc8"
+			  "\x15\x4e\x91\x92\x89\x4b\xb7\x9b"
+			  "\x21\xf7\x42\x89\xac\x12\x2a\x54"
+			  "\x69\xee\x18\xc7\x8d\xed\xe8\xfd"
+			  "\xbb\x04\x28\xe6\x8a\x3c\x98\xc1"
+			  "\x04\x2d\xa9\xa1\x24\x83\xff\xe9"
+			  "\x55\x7a\xf0\xd1\xf6\x63\x05\xe1"
+			  "\xd9\x1e\x75\x72\xc1\x9f\xae\x32"
+			  "\xe1\x6b\xcd\x9e\x61\x19\x23\x86"
+			  "\xd9\xd2\xaf\x8e\xd5\xd3\xa8\xa9"
+			  "\x51",
+		.alen	= 129,
+		.input	= "\xe8\x39\x2d\xaa\x0e\x85\x2d\xc1"
+			  "\x72\xaf\x6e\xc9\x82\x33\xc7\x01"
+			  "\xaf\x40\x70\xb8\x2a\x78\xc9\x14"
+			  "\xac\xb1\x10\xca\x2e\xb3\x28\xe4"
+			  "\xac\xfa\x58\x7f\xe5\x73\x09\x8c"
+			  "\x1d\x40\x87\x8c\xd9\x75\xc0\x55"
+			  "\xa2\xda\x07\xd1\xc2\xa9\xd1\xbb"
+			  "\x09\x4f\x77\x62\x88\x2d\xf2\x68"
+			  "\x54",
+		.ilen	= 65,
+		.result	= "\x36\x78\xb9\x22\xde\x62\x35\x55"
+			  "\x1a\x7a\xf5\x45\xbc\xd7\x15\x82"
+			  "\x01\xe9\x5a\x07\xea\x46\xaf\x91"
+			  "\xcb\x73\xa5\xee\xe1\xb4\xbf\xc2"
+			  "\xdb\xd2\x9d\x59\xde\xfc\x83\x00"
+			  "\xf5\x46\xac\x97\xd5\x57\xa9\xb9"
+			  "\x1f\x8c\xe8\xca\x68\x8b\x91\x0c"
+			  "\x01\xbe\x0a\xaf\x7c\xf6\x67\xa4"
+			  "\xbf\xbc\x88\x3f\x5d\xd1\xf9\x19"
+			  "\x0f\x9d\xb2\xaf\xb9\x6e\x17\xdf"
+			  "\xa2",
+		.rlen	= 81,
+	}, {
+		.key	= "\xf7\x02\xbb\x11\x52\x24\xd8\x48"
+			  "\x93\xe6\x9b\xee\x81\xfc\xf7\x82",
+		.klen	= 16,
+		.iv	= "\x06\xcc\x4a\x79\x96\xc3\x82\xcf"
+			  "\xb3\x1c\xc7\x12\x7f\xc5\x28\x04",
+		.assoc	= "\x15\x95\xd8\xe1\xda\x62\x2c\x56"
+			  "\xd3\x53\xf4\x36\x7e\x8e\x59\x85"
+			  "\x0e\x51\xf9\x1c\xee\x70\x6a\x27"
+			  "\x3d\xd3\xb7\xac\x51\xfa\xdf\x05",
+		.alen	= 32,
+		.input	= "\x24\x5e\x67\x49\x1e\x01\xd6\xdd"
+			  "\xf3\x89\x20\x5b\x7c\x57\x89\x07"
+			  "\xd9\x02\x7c\x3d\x2f\x18\x4b\x2d"
+			  "\x6e\xde\xee\xa2\x08\x12\xc7\xba",
+		.ilen	= 32,
+		.result	= "\x08\x1b\x95\x0e\x41\x95\x02\x4b"
+			  "\x9c\xbb\xa8\xd0\x7c\xd3\x44\x6e"
+			  "\x89\x14\x33\x70\x0a\xbc\xea\x39"
+			  "\x88\xaa\x2b\xd5\x73\x11\x55\xf5"
+			  "\x33\x33\x9c\xd7\x42\x34\x49\x8e"
+			  "\x2f\x03\x30\x05\x47\xaf\x34",
+		.rlen	= 47,
+	}, {
+		.key	= "\x33\x27\xf5\xb1\x62\xa0\x80\x63"
+			  "\x14\xc0\x4d\x7f\x7b\x20\xba\x89",
+		.klen	= 16,
+		.iv	= "\x42\xf0\x84\x19\xa6\x3f\x2b\xea"
+			  "\x34\xf6\x79\xa3\x79\xe9\xeb\x0a",
+		.assoc	= "\x51\xb9\x12\x80\xea\xde\xd5\x71"
+			  "\x54\x2d\xa6\xc8\x78\xb2\x1b\x8c"
+			  "\x39\x14\x05\xa0\xf3\x10\xec\x41"
+			  "\xff\x01\x95\x84\x2b\x59\x7f\xdb",
+		.alen	= 32,
+		.input	= "\x61\x83\xa0\xe8\x2e\x7d\x7f\xf8"
+			  "\x74\x63\xd2\xec\x76\x7c\x4c\x0d"
+			  "\x03\xc4\x88\xc1\x35\xb8\xcd\x47"
+			  "\x2f\x0c\xcd\x7a\xe2\x71\x66\x91",
+		.ilen	= 32,
+		.result	= "\x97\xca\xf4\xe0\x8d\x89\xbf\x68"
+			  "\x0c\x60\xb9\x27\xdf\xaa\x41\xc6"
+			  "\x25\xd8\xf7\x1f\x10\x15\x48\x61"
+			  "\x4c\x95\x00\xdf\x51\x9b\x7f\xe6"
+			  "\x24\x40\x9e\xbe\x3b\xeb\x1b\x98"
+			  "\xb9\x9c\xe5\xef\xf2\x05",
+		.rlen	= 46,
+	}, {
+		.key	= "\x70\x4c\x2f\x50\x72\x1c\x29\x7f"
+			  "\x95\x9a\xff\x10\x75\x45\x7d\x8f",
+		.klen	= 16,
+		.iv	= "\x7f\x15\xbd\xb8\xb6\xba\xd3\x06"
+			  "\xb5\xd1\x2b\x35\x73\x0e\xad\x10",
+		.assoc	= "\x8e\xde\x4c\x20\xfa\x59\x7e\x8d"
+			  "\xd5\x07\x58\x59\x72\xd7\xde\x92"
+			  "\x63\xd6\x10\x24\xf8\xb0\x6e\x5a"
+			  "\xc0\x2e\x74\x5d\x06\xb8\x1e\xb2",
+		.alen	= 32,
+		.input	= "\x9d\xa7\xda\x88\x3e\xf8\x28\x14"
+			  "\xf5\x3e\x85\x7d\x70\xa0\x0f\x13"
+			  "\x2e\x86\x93\x45\x3a\x58\x4f\x61"
+			  "\xf0\x3a\xac\x53\xbc\xd0\x06\x68",
+		.ilen	= 32,
+		.result	= "\x63\x4c\x2a\x8e\xb4\x6b\x63\x0d"
+			  "\xb5\xec\x9b\x4e\x12\x23\xa3\xcf"
+			  "\x1a\x5a\x70\x15\x5a\x10\x40\x51"
+			  "\xca\x47\x4c\x9d\xc9\x97\xf4\x77"
+			  "\xdb\xc8\x10\x2d\xdc\x65\x20\x3f",
+		.rlen	= 40,
+	}, {
+		.key	= "\xac\x70\x69\xef\x82\x97\xd2\x9b"
+			  "\x15\x74\xb1\xa2\x6f\x69\x3f\x95",
+		.klen	= 16,
+		.iv	= "\xbb\x3a\xf7\x57\xc6\x36\x7c\x22"
+			  "\x36\xab\xde\xc6\x6d\x32\x70\x17",
+		.assoc	= "\xcb\x03\x85\xbf\x0a\xd5\x26\xa9"
+			  "\x56\xe1\x0a\xeb\x6c\xfb\xa1\x98"
+			  "\x8d\x98\x1c\xa8\xfe\x50\xf0\x74"
+			  "\x81\x5c\x53\x35\xe0\x17\xbd\x88",
+		.alen	= 32,
+		.input	= "\xda\xcc\x14\x27\x4e\x74\xd1\x30"
+			  "\x76\x18\x37\x0f\x6a\xc4\xd1\x1a"
+			  "\x58\x49\x9f\xc9\x3f\xf8\xd1\x7a"
+			  "\xb2\x67\x8b\x2b\x96\x2f\xa5\x3e",
+		.ilen	= 32,
+		.result	= "\xf1\x62\x44\xc7\x5f\x19\xca\x43"
+			  "\x47\x2c\xaf\x68\x82\xbd\x51\xef"
+			  "\x3d\x65\xd8\x45\x2d\x06\x07\x78"
+			  "\x08\x2e\xb3\x23\xcd\x81\x12\x55"
+			  "\x1a",
+		.rlen	= 33,
+	}, {
+		.key	= "\xe9\x95\xa2\x8f\x93\x13\x7b\xb7"
+			  "\x96\x4e\x63\x33\x69\x8d\x02\x9b"
+			  "\x23\xf9\x22\xeb\x80\xa0\xb1\x81"
+			  "\xe2\x73\xc3\x21\x4d\x47\x8d\xf4",
+		.klen	= 32,
+		.iv	= "\xf8\x5e\x31\xf7\xd7\xb2\x25\x3e"
+			  "\xb7\x85\x90\x58\x67\x57\x33\x1d",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xdf\x2f\x83\xc0\x45\x4a\x2c\xcf"
+			  "\xb9\xd2\x41\xf6\x80\xa1\x52\x70",
+		.rlen	= 16,
+	}, {
+		.key	= "\x25\xba\xdc\x2e\xa3\x8f\x24\xd3"
+			  "\x17\x29\x15\xc5\x63\xb2\xc5\xa1"
+			  "\x4d\xbc\x2d\x6f\x85\x40\x33\x9a"
+			  "\xa3\xa0\xa1\xfa\x27\xa6\x2c\xca",
+		.klen	= 32,
+		.iv	= "\x34\x83\x6a\x96\xe7\x2d\xce\x5a"
+			  "\x38\x5f\x42\xe9\x61\x7b\xf5\x23",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x53",
+		.ilen	= 1,
+		.result	= "\x01\xd8\x55\x3c\xc0\x5a\x4b\xc7"
+			  "\x01\xf4\x08\xe3\x0d\xf7\xf0\x78"
+			  "\x53",
+		.rlen	= 17,
+	}, {
+		.key	= "\x62\xdf\x16\xcd\xb3\x0a\xcc\xef"
+			  "\x98\x03\xc7\x56\x5d\xd6\x87\xa8"
+			  "\x77\x7e\x39\xf3\x8a\xe0\xb5\xb4"
+			  "\x65\xce\x80\xd2\x01\x05\xcb\xa1",
+		.klen	= 32,
+		.iv	= "\x71\xa8\xa4\x35\xf7\xa9\x76\x75"
+			  "\xb8\x39\xf4\x7a\x5b\x9f\xb8\x29",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x8f\x3a\xc1\x05\x7f\xe7\xcb\x83"
+			  "\xf9\xa6\x4d\xc3\x58\x31\x19\x2c"
+			  "\xd7\x90\xc2\x56\x4e\xd8\x57\xc7"
+			  "\xf6\xf0\x27\xb4\x25\x4c\x83",
+		.ilen	= 31,
+		.result	= "\xc2\x4b\x41\x0f\x2d\xb9\x62\x07"
+			  "\xff\x8e\x74\xf8\xa1\xa6\xd5\x37"
+			  "\xa5\x64\x31\x5c\xca\x73\x9b\x43"
+			  "\xe6\x70\x63\x46\x95\xcb\xf7\xb5"
+			  "\x20\x8c\x75\x7a\x2a\x17\x2f\xa9"
+			  "\xb8\x4d\x11\x42\xd1\xf8\xf1",
+		.rlen	= 47,
+	}, {
+		.key	= "\x9e\x03\x4f\x6d\xc3\x86\x75\x0a"
+			  "\x19\xdd\x79\xe8\x57\xfb\x4a\xae"
+			  "\xa2\x40\x45\x77\x90\x80\x37\xce"
+			  "\x26\xfb\x5f\xaa\xdb\x64\x6b\x77",
+		.klen	= 32,
+		.iv	= "\xae\xcc\xde\xd5\x07\x25\x1f\x91"
+			  "\x39\x14\xa6\x0c\x55\xc4\x7b\x30",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xcc\x5f\xfb\xa4\x8f\x63\x74\x9f"
+			  "\x7a\x81\xff\x55\x52\x56\xdc\x33"
+			  "\x01\x52\xcd\xdb\x53\x78\xd9\xe1"
+			  "\xb7\x1d\x06\x8d\xff\xab\x22\x98",
+		.ilen	= 32,
+		.result	= "\xbb\x01\x7c\xd1\x2c\x33\x7b\x37"
+			  "\x0a\xee\xc4\x30\x19\xd7\x3a\x6f"
+			  "\xf8\x2b\x67\xf5\x3b\x84\x87\x2a"
+			  "\xfb\x07\x7a\x82\xb5\xe4\x85\x26"
+			  "\x1e\xa8\xe5\x04\x54\xce\xe5\x5f"
+			  "\xb5\x3f\xc1\xd5\x7f\xbd\xd2\xa6",
+		.rlen	= 48,
+	}, {
+		.key	= "\xdb\x28\x89\x0c\xd3\x01\x1e\x26"
+			  "\x9a\xb7\x2b\x79\x51\x1f\x0d\xb4"
+			  "\xcc\x03\x50\xfc\x95\x20\xb9\xe7"
+			  "\xe8\x29\x3e\x83\xb5\xc3\x0a\x4e",
+		.klen	= 32,
+		.iv	= "\xea\xf1\x18\x74\x17\xa0\xc8\xad"
+			  "\xba\xee\x58\x9d\x4f\xe8\x3d\x36",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x08\x84\x34\x44\x9f\xde\x1c\xbb"
+			  "\xfb\x5b\xb1\xe6\x4c\x7a\x9f\x39"
+			  "\x2c\x14\xd9\x5f\x59\x18\x5b\xfb"
+			  "\x79\x4b\xe5\x65\xd9\x0a\xc1\x6f"
+			  "\x2e",
+		.ilen	= 33,
+		.result	= "\xc2\xf4\x40\x55\xf9\x59\xff\x73"
+			  "\x08\xf5\x98\x92\x0c\x7b\x35\x9a"
+			  "\xa8\xf4\x42\x7e\x6f\x93\xca\x22"
+			  "\x23\x06\x1e\xf8\x89\x22\xf4\x46"
+			  "\x7c\x7c\x67\x75\xab\xe5\x75\xaa"
+			  "\x15\xd7\x83\x19\xfd\x31\x59\x5b"
+			  "\x32",
+		.rlen	= 49,
+	}, {
+		.key	= "\x17\x4d\xc3\xab\xe3\x7d\xc7\x42"
+			  "\x1b\x91\xdd\x0a\x4b\x43\xcf\xba"
+			  "\xf6\xc5\x5c\x80\x9a\xc0\x3b\x01"
+			  "\xa9\x56\x1d\x5b\x8f\x22\xa9\x25",
+		.klen	= 32,
+		.iv	= "\x27\x16\x51\x13\x27\x1c\x71\xc9"
+			  "\x3b\xc8\x0a\x2f\x49\x0c\x00\x3c",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x45\xa8\x6e\xe3\xaf\x5a\xc5\xd7"
+			  "\x7c\x35\x63\x77\x46\x9f\x61\x3f"
+			  "\x56\xd7\xe4\xe3\x5e\xb8\xdc\x14"
+			  "\x3a\x79\xc4\x3e\xb3\x69\x61\x46"
+			  "\x3c\xb6\x83\x4e\xb4\x26\xc7\x73"
+			  "\x22\xda\x52\x8b\x7d\x11\x98\xea"
+			  "\x62\xe1\x14\x1e\xdc\xfe\x0f\xad"
+			  "\x20\x76\x5a\xdc\x4e\x71\x13",
+		.ilen	= 63,
+		.result	= "\xc9\x82\x3b\x4b\x87\x84\xa5\xdb"
+			  "\xa0\x8c\xd3\x3e\x7f\x8d\xe8\x28"
+			  "\x2a\xdc\xfa\x01\x84\x87\x9a\x70"
+			  "\x81\x75\x37\x0a\xd2\x75\xa9\xb6"
+			  "\x21\x72\xee\x7e\x65\x95\xe5\xcc"
+			  "\x01\xb7\x39\xa6\x51\x15\xca\xff"
+			  "\x61\xdc\x97\x38\xcc\xf4\xca\xc7"
+			  "\x83\x9b\x05\x11\x72\x60\xf0\xb4"
+			  "\x7e\x06\xab\x0a\xc0\xbb\x59\x23"
+			  "\xaa\x2d\xfc\x4e\x35\x05\x59",
+		.rlen	= 79,
+	}, {
+		.key	= "\x54\x71\xfd\x4b\xf3\xf9\x6f\x5e"
+			  "\x9c\x6c\x8f\x9c\x45\x68\x92\xc1"
+			  "\x21\x87\x67\x04\x9f\x60\xbd\x1b"
+			  "\x6a\x84\xfc\x34\x6a\x81\x48\xfb",
+		.klen	= 32,
+		.iv	= "\x63\x3b\x8b\xb3\x37\x98\x1a\xe5"
+			  "\xbc\xa2\xbc\xc0\x43\x31\xc2\x42",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x81\xcd\xa8\x82\xbf\xd6\x6e\xf3"
+			  "\xfd\x0f\x15\x09\x40\xc3\x24\x45"
+			  "\x81\x99\xf0\x67\x63\x58\x5e\x2e"
+			  "\xfb\xa6\xa3\x16\x8d\xc8\x00\x1c"
+			  "\x4b\x62\x87\x7c\x15\x38\xda\x70"
+			  "\x3d\xea\xe7\xf2\x40\xba\xae\x79"
+			  "\x8f\x48\xfc\xbf\x45\x53\x2e\x78"
+			  "\xef\x79\xf0\x1b\x49\xf7\xfd\x9c",
+		.ilen	= 64,
+		.result	= "\x11\x7c\x7d\xef\xce\x29\x95\xec"
+			  "\x7e\x9f\x42\xa6\x26\x07\xa1\x75"
+			  "\x2f\x4e\x09\x9a\xf6\x6b\xc2\xfa"
+			  "\x0d\xd0\x17\xdc\x25\x1e\x9b\xdc"
+			  "\x5f\x8c\x1c\x60\x15\x4f\x9b\x20"
+			  "\x7b\xff\xcd\x82\x60\x84\xf4\xa5"
+			  "\x20\x9a\x05\x19\x5b\x02\x0a\x72"
+			  "\x43\x11\x26\x58\xcf\xc5\x41\xcf"
+			  "\x13\xcc\xde\x32\x92\xfa\x86\xf2"
+			  "\xaf\x16\xe8\x8f\xca\xb6\xfd\x54",
+		.rlen	= 80,
+	}, {
+		.key	= "\x90\x96\x36\xea\x03\x74\x18\x7a"
+			  "\x1d\x46\x42\x2d\x3f\x8c\x54\xc7"
+			  "\x4b\x4a\x73\x89\xa4\x00\x3f\x34"
+			  "\x2c\xb1\xdb\x0c\x44\xe0\xe8\xd2",
+		.klen	= 32,
+		.iv	= "\xa0\x5f\xc5\x52\x47\x13\xc2\x01"
+			  "\x3d\x7c\x6e\x52\x3d\x55\x85\x48",
+		.assoc	= "\xaf",
+		.alen	= 1,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x9b\xc5\x3b\x20\x0a\x88\x56\xbe"
+			  "\x69\xdf\xc4\xc4\x02\x46\x3a\xf0",
+		.rlen	= 16,
+	}, {
+		.key	= "\xcd\xbb\x70\x89\x13\xf0\xc1\x95"
+			  "\x9e\x20\xf4\xbf\x39\xb1\x17\xcd"
+			  "\x76\x0c\x7f\x0d\xa9\xa0\xc1\x4e"
+			  "\xed\xdf\xb9\xe4\x1e\x3f\x87\xa8",
+		.klen	= 32,
+		.iv	= "\xdc\x84\xfe\xf1\x58\x8f\x6b\x1c"
+			  "\xbe\x57\x20\xe3\x37\x7a\x48\x4f",
+		.assoc	= "\xeb\x4d\x8d\x59\x9c\x2e\x15\xa3"
+			  "\xde\x8d\x4d\x07\x36\x43\x78\xd0"
+			  "\x0b\x6d\x84\x4f\x2c\xf0\x82\x5b"
+			  "\x4e\xf6\x29\xd1\x8b\x6f\x56",
+		.alen	= 31,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xe0\x6d\xa1\x07\x98\x2f\x40\x2d"
+			  "\x2e\x9a\xd6\x61\x43\xc0\x74\x69",
+		.rlen	= 16,
+	}, {
+		.key	= "\x0a\xe0\xaa\x29\x24\x6c\x6a\xb1"
+			  "\x1f\xfa\xa6\x50\x33\xd5\xda\xd3"
+			  "\xa0\xce\x8a\x91\xae\x40\x43\x68"
+			  "\xae\x0d\x98\xbd\xf8\x9e\x26\x7f",
+		.klen	= 32,
+		.iv	= "\x19\xa9\x38\x91\x68\x0b\x14\x38"
+			  "\x3f\x31\xd2\x74\x31\x9e\x0a\x55",
+		.assoc	= "\x28\x72\xc7\xf8\xac\xaa\xbe\xbf"
+			  "\x5f\x67\xff\x99\x30\x67\x3b\xd6"
+			  "\x35\x2f\x90\xd3\x31\x90\x04\x74"
+			  "\x0f\x23\x08\xa9\x65\xce\xf6\xea",
+		.alen	= 32,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\xb9\x57\x13\x3e\x82\x31\x61\x65"
+			  "\x0d\x7f\x6c\x96\x93\x5c\x50\xe2",
+		.rlen	= 16,
+	}, {
+		.key	= "\x46\x04\xe3\xc8\x34\xe7\x12\xcd"
+			  "\xa0\xd4\x58\xe2\x2d\xf9\x9c\xda"
+			  "\xca\x91\x96\x15\xb4\xe0\xc5\x81"
+			  "\x70\x3a\x77\x95\xd2\xfd\xc5\x55",
+		.klen	= 32,
+		.iv	= "\x55\xcd\x72\x30\x78\x86\xbd\x54"
+			  "\xc0\x0b\x84\x06\x2b\xc2\xcd\x5b",
+		.assoc	= "\x64\x97\x00\x98\xbc\x25\x67\xdb"
+			  "\xe0\x41\xb1\x2a\x2a\x8c\xfe\xdd"
+			  "\x5f\xf2\x9c\x58\x36\x30\x86\x8e"
+			  "\xd1\x51\xe6\x81\x3f\x2d\x95\xc1"
+			  "\x01",
+		.alen	= 33,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x81\x96\x34\xde\xbb\x36\xdd\x3e"
+			  "\x4e\x5e\xcb\x44\x21\xb8\x3f\xf1",
+		.rlen	= 16,
+	}, {
+		.key	= "\x83\x29\x1d\x67\x44\x63\xbb\xe9"
+			  "\x20\xaf\x0a\x73\x27\x1e\x5f\xe0"
+			  "\xf5\x53\xa1\x9a\xb9\x80\x47\x9b"
+			  "\x31\x68\x56\x6e\xac\x5c\x65\x2c",
+		.klen	= 32,
+		.iv	= "\x92\xf2\xac\xcf\x88\x02\x65\x70"
+			  "\x41\xe5\x36\x97\x25\xe7\x90\x61",
+		.assoc	= "\xa1\xbb\x3a\x37\xcc\xa1\x10\xf7"
+			  "\x61\x1c\x63\xbc\x24\xb0\xc0\xe3"
+			  "\x8a\xb4\xa7\xdc\x3b\xd0\x08\xa8"
+			  "\x92\x7f\xc5\x5a\x19\x8c\x34\x97"
+			  "\x0f\x95\x9b\x18\xe4\x8d\xb4\x24"
+			  "\xb9\x33\x28\x18\xe1\x9d\x14\xe0"
+			  "\x64\xb2\x89\x7d\x78\xa8\x05\x7e"
+			  "\x07\x8c\xfc\x88\x2d\xb8\x53",
+		.alen	= 63,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x2e\x99\xb6\x79\x57\x56\x80\x36"
+			  "\x8e\xc4\x1c\x12\x7d\x71\x36\x0c",
+		.rlen	= 16,
+	}, {
+		.key	= "\xbf\x4e\x57\x07\x54\xdf\x64\x05"
+			  "\xa1\x89\xbc\x04\x21\x42\x22\xe6"
+			  "\x1f\x15\xad\x1e\xbe\x20\xc9\xb4"
+			  "\xf3\x95\x35\x46\x86\xbb\x04\x03",
+		.klen	= 32,
+		.iv	= "\xce\x17\xe5\x6f\x98\x7e\x0e\x8c"
+			  "\xc2\xbf\xe8\x29\x1f\x0b\x52\x68",
+		.assoc	= "\xdd\xe0\x74\xd6\xdc\x1d\xb8\x13"
+			  "\xe2\xf6\x15\x4d\x1e\xd4\x83\xe9"
+			  "\xb4\x76\xb3\x60\x40\x70\x8a\xc1"
+			  "\x53\xac\xa4\x32\xf3\xeb\xd3\x6e"
+			  "\x1e\x42\xa0\x46\x45\x9f\xc7\x22"
+			  "\xd3\x43\xbc\x7e\xa5\x47\x2a\x6f"
+			  "\x91\x19\x70\x1e\xe1\xfe\x25\x49"
+			  "\xd6\x8f\x93\xc7\x28\x3f\x3d\x03",
+		.alen	= 64,
+		.input	= "",
+		.ilen	= 0,
+		.result	= "\x7b\x25\x3d\x47\xd4\xa7\x08\xce"
+			  "\x3b\x89\x40\x36\xba\x6d\x0e\xa2",
+		.rlen	= 16,
+	}, {
+		.key	= "\xfc\x72\x90\xa6\x64\x5a\x0d\x21"
+			  "\x22\x63\x6e\x96\x1b\x67\xe4\xec"
+			  "\x49\xd7\xb9\xa2\xc3\xc0\x4b\xce"
+			  "\xb4\xc3\x14\x1e\x61\x1a\xa3\xd9",
+		.klen	= 32,
+		.iv	= "\x0b\x3c\x1f\x0e\xa8\xf9\xb7\xa7"
+			  "\x42\x9a\x9a\xba\x19\x30\x15\x6e",
+		.assoc	= "\x1a",
+		.alen	= 1,
+		.input	= "\x29",
+		.ilen	= 1,
+		.result	= "\xe6\x09\x6f\x95\x9a\x18\xc8\xf6"
+			  "\x17\x75\x81\x16\xdf\x26\xff\x67"
+			  "\x92",
+		.rlen	= 17,
+	}, {
+		.key	= "\x38\x97\xca\x45\x74\xd6\xb6\x3c"
+			  "\xa3\x3d\x20\x27\x15\x8b\xa7\xf2"
+			  "\x74\x9a\xc4\x27\xc8\x60\xcd\xe8"
+			  "\x75\xf0\xf2\xf7\x3b\x79\x42\xb0",
+		.klen	= 32,
+		.iv	= "\x47\x60\x59\xad\xb8\x75\x60\xc3"
+			  "\xc3\x74\x4c\x4c\x13\x54\xd8\x74",
+		.assoc	= "\x56\x29\xe7\x15\xfc\x14\x0a\x4a"
+			  "\xe4\xaa\x79\x70\x12\x1d\x08\xf6"
+			  "\x09\xfb\xca\x69\x4b\xb0\x8e\xf5"
+			  "\xd6\x07\x62\xe3\xa8\xa9\x12",
+		.alen	= 31,
+		.input	= "\x66\xf3\x75\x7d\x40\xb3\xb4\xd1"
+			  "\x04\xe1\xa6\x94\x10\xe6\x39\x77"
+			  "\xd3\xac\x4d\x8a\x8c\x58\x6e\xfb"
+			  "\x06\x13\x9a\xd9\x5e\xc0\xfa",
+		.ilen	= 31,
+		.result	= "\x82\xc0\x56\xf0\xd7\xc4\xc9\xfd"
+			  "\x3c\xd1\x2a\xd4\x15\x86\x9d\xda"
+			  "\xea\x6c\x6f\xa1\x33\xb0\x7a\x01"
+			  "\x57\xe7\xf3\x7b\x73\xe7\x54\x10"
+			  "\xc6\x91\xe2\xc6\xa0\x69\xe7\xe6"
+			  "\x76\xc3\xf5\x3a\x76\xfd\x4a",
+		.rlen	= 47,
+	}, {
+		.key	= "\x75\xbc\x04\xe5\x84\x52\x5e\x58"
+			  "\x24\x17\xd2\xb9\x0e\xaf\x6a\xf9"
+			  "\x9e\x5c\xd0\xab\xcd\x00\x4f\x01"
+			  "\x37\x1e\xd1\xcf\x15\xd8\xe2\x86",
+		.klen	= 32,
+		.iv	= "\x84\x85\x92\x4d\xc8\xf1\x08\xdf"
+			  "\x44\x4e\xff\xdd\x0d\x78\x9a\x7a",
+		.assoc	= "\x93\x4e\x21\xb4\x0c\x90\xb3\x66"
+			  "\x65\x84\x2b\x01\x0b\x42\xcb\xfc"
+			  "\x33\xbd\xd6\xed\x50\x50\x10\x0e"
+			  "\x97\x35\x41\xbb\x82\x08\xb1\xf2",
+		.alen	= 32,
+		.input	= "\xa2\x17\xaf\x1c\x50\x2e\x5d\xed"
+			  "\x85\xbb\x58\x26\x0a\x0b\xfc\x7d"
+			  "\xfe\x6e\x59\x0e\x91\xf8\xf0\x15"
+			  "\xc8\x40\x78\xb1\x38\x1f\x99\xa7",
+		.ilen	= 32,
+		.result	= "\x01\x47\x8e\x6c\xf6\x64\x89\x3a"
+			  "\x71\xce\xe4\xaa\x45\x70\xe6\x84"
+			  "\x62\x48\x08\x64\x86\x6a\xdf\xec"
+			  "\xb4\xa0\xfb\x34\x03\x0c\x19\xf4"
+			  "\x2b\x7b\x36\x73\xec\x54\xa9\x1e"
+			  "\x30\x85\xdb\xe4\xac\xe9\x2c\xca",
+		.rlen	= 48,
+	}, {
+		.key	= "\xb1\xe1\x3e\x84\x94\xcd\x07\x74"
+			  "\xa5\xf2\x84\x4a\x08\xd4\x2c\xff"
+			  "\xc8\x1e\xdb\x2f\xd2\xa0\xd1\x1b"
+			  "\xf8\x4c\xb0\xa8\xef\x37\x81\x5d",
+		.klen	= 32,
+		.iv	= "\xc0\xaa\xcc\xec\xd8\x6c\xb1\xfb"
+			  "\xc5\x28\xb1\x6e\x07\x9d\x5d\x81",
+		.assoc	= "\xd0\x73\x5a\x54\x1d\x0b\x5b\x82"
+			  "\xe5\x5f\xdd\x93\x05\x66\x8e\x02"
+			  "\x5e\x80\xe1\x71\x55\xf0\x92\x28"
+			  "\x59\x62\x20\x94\x5c\x67\x50\xc8"
+			  "\x58",
+		.alen	= 33,
+		.input	= "\xdf\x3c\xe9\xbc\x61\xaa\x06\x09"
+			  "\x06\x95\x0a\xb7\x04\x2f\xbe\x84"
+			  "\x28\x30\x64\x92\x96\x98\x72\x2e"
+			  "\x89\x6e\x57\x8a\x13\x7e\x38\x7e"
+			  "\xdb",
+		.ilen	= 33,
+		.result	= "\x85\xe0\xf8\x0f\x8e\x49\xe3\x60"
+			  "\xcb\x4a\x54\x94\xcf\xf5\x7e\x34"
+			  "\xe9\xf8\x80\x65\x53\xd0\x72\x70"
+			  "\x4f\x7d\x9d\xd1\x15\x6f\xb9\x2c"
+			  "\xfa\xe8\xdd\xac\x2e\xe1\x3f\x67"
+			  "\x63\x0f\x1a\x59\xb7\x89\xdb\xf4"
+			  "\xc3",
+		.rlen	= 49,
+	}, {
+		.key	= "\xee\x05\x77\x23\xa5\x49\xb0\x90"
+			  "\x26\xcc\x36\xdc\x02\xf8\xef\x05"
+			  "\xf3\xe1\xe7\xb3\xd8\x40\x53\x35"
+			  "\xb9\x79\x8f\x80\xc9\x96\x20\x33",
+		.klen	= 32,
+		.iv	= "\xfd\xce\x06\x8b\xe9\xe8\x5a\x17"
+			  "\x46\x02\x63\x00\x01\xc1\x20\x87",
+		.assoc	= "\x0c\x98\x94\xf3\x2d\x87\x04\x9e"
+			  "\x66\x39\x8f\x24\xff\x8a\x50\x08"
+			  "\x88\x42\xed\xf6\x5a\x90\x14\x42"
+			  "\x1a\x90\xfe\x6c\x36\xc6\xf0\x9f"
+			  "\x66\xa0\xb5\x2d\x2c\xf8\x25\x15"
+			  "\x55\x90\xa2\x7e\x77\x94\x96\x3a"
+			  "\x71\x1c\xf7\x44\xee\xa8\xc3\x42"
+			  "\xe2\xa3\x84\x04\x0b\xe1\xce",
+		.alen	= 63,
+		.input	= "\x1b\x61\x23\x5b\x71\x26\xae\x25"
+			  "\x87\x6f\xbc\x49\xfe\x53\x81\x8a"
+			  "\x53\xf2\x70\x17\x9b\x38\xf4\x48"
+			  "\x4b\x9b\x36\x62\xed\xdd\xd8\x54"
+			  "\xea\xcb\xb6\x79\x45\xfc\xaa\x54"
+			  "\x5c\x94\x47\x58\xa7\xff\x9c\x9e"
+			  "\x7c\xb6\xf1\xac\xc8\xfd\x8b\x35"
+			  "\xd5\xa4\x6a\xd4\x09\xc2\x08",
+		.ilen	= 63,
+		.result	= "\x00\xe5\x5b\x87\x5c\x20\x22\x8a"
+			  "\xda\x1f\xd3\xff\xbb\xb2\xb0\xf8"
+			  "\xef\xe9\xeb\x9e\x7c\x80\xf4\x2b"
+			  "\x59\xc0\x79\xbc\x17\xa0\x15\x01"
+			  "\xf5\x72\xfb\x5a\xe7\xaf\x07\xe3"
+			  "\x1b\x49\x21\x34\x23\x63\x55\x5e"
+			  "\xee\x4f\x34\x17\xfa\xfe\xa5\x0c"
+			  "\xed\x0b\x23\xea\x9b\xda\x57\x2f"
+			  "\xf6\xa9\xae\x0d\x4e\x40\x96\x45"
+			  "\x7f\xfa\xf0\xbf\xc4\x98\x78",
+		.rlen	= 79,
+	}, {
+		.key	= "\x2a\x2a\xb1\xc3\xb5\xc5\x59\xac"
+			  "\xa7\xa6\xe8\x6d\xfc\x1d\xb2\x0b"
+			  "\x1d\xa3\xf3\x38\xdd\xe0\xd5\x4e"
+			  "\x7b\xa7\x6e\x58\xa3\xf5\xbf\x0a",
+		.klen	= 32,
+		.iv	= "\x39\xf3\x3f\x2b\xf9\x64\x03\x33"
+			  "\xc7\xdd\x15\x91\xfb\xe6\xe2\x8d",
+		.assoc	= "\x49\xbc\xce\x92\x3d\x02\xad\xba"
+			  "\xe7\x13\x41\xb6\xf9\xaf\x13\x0f"
+			  "\xb2\x04\xf8\x7a\x5f\x30\x96\x5b"
+			  "\xdc\xbd\xdd\x44\x10\x25\x8f\x75"
+			  "\x75\x4d\xb9\x5b\x8e\x0a\x38\x13"
+			  "\x6f\x9f\x36\xe4\x3a\x3e\xac\xc9"
+			  "\x9d\x83\xde\xe5\x57\xfd\xe3\x0e"
+			  "\xb1\xa7\x1b\x44\x05\x67\xb7\x37",
+		.alen	= 64,
+		.input	= "\x58\x85\x5c\xfa\x81\xa1\x57\x40"
+			  "\x08\x4a\x6e\xda\xf8\x78\x44\x90"
+			  "\x7d\xb5\x7b\x9b\xa1\xd8\x76\x62"
+			  "\x0c\xc9\x15\x3b\xc7\x3c\x77\x2b"
+			  "\xf8\x78\xba\xa7\xa6\x0e\xbd\x52"
+			  "\x76\xa3\xdc\xbe\x6b\xa8\xb1\x2d"
+			  "\xa9\x1d\xd8\x4e\x31\x53\xab\x00"
+			  "\xa5\xa7\x01\x13\x04\x49\xf2\x04",
+		.ilen	= 64,
+		.result	= "\x28\xdd\xb9\x4a\x12\xc7\x0a\xe1"
+			  "\x58\x06\x1a\x9b\x8c\x67\xdf\xeb"
+			  "\x35\x35\x60\x9d\x06\x40\x65\xc1"
+			  "\x93\xe8\xb3\x82\x50\x29\xdd\xb5"
+			  "\x2b\xcb\xde\x18\x78\x6b\x42\xbe"
+			  "\x6d\x24\xd0\xb2\x7d\xd7\x08\x8f"
+			  "\x4a\x18\x98\xad\x8c\xf2\x97\xb4"
+			  "\xf4\x77\xe4\xbf\x41\x3b\xc4\x06"
+			  "\xce\x9e\x34\x81\xf0\x89\x11\x13"
+			  "\x02\x65\xa1\x7c\xdf\x07\x33\x06",
+		.rlen	= 80,
+	}, {
+		.key	= "\x67\x4f\xeb\x62\xc5\x40\x01\xc7"
+			  "\x28\x80\x9a\xfe\xf6\x41\x74\x12"
+			  "\x48\x65\xfe\xbc\xe2\x80\x57\x68"
+			  "\x3c\xd4\x4d\x31\x7d\x54\x5f\xe1",
+		.klen	= 32,
+		.iv	= "\x76\x18\x79\xca\x09\xdf\xac\x4e"
+			  "\x48\xb7\xc7\x23\xf5\x0a\xa5\x93",
+		.assoc	= "\x85\xe1\x08\x32\x4d\x7e\x56\xd5"
+			  "\x68\xed\xf3\x47\xf3\xd3\xd6\x15"
+			  "\xdd\xc7\x04\xfe\x64\xd0\x18\x75"
+			  "\x9d\xeb\xbc\x1d\xea\x84\x2e\x4c"
+			  "\x83\xf9\xbe\x8a\xef\x1c\x4b\x10"
+			  "\x89\xaf\xcb\x4b\xfe\xe7\xc1\x58"
+			  "\xca\xea\xc6\x87\xc0\x53\x03\xd9"
+			  "\x80\xaa\xb2\x83\xff\xee\xa1\x6a"
+			  "\x04",
+		.alen	= 65,
+		.input	= "\x94\xaa\x96\x9a\x91\x1d\x00\x5c"
+			  "\x88\x24\x20\x6b\xf2\x9c\x06\x96"
+			  "\xa7\x77\x87\x1f\xa6\x78\xf8\x7b"
+			  "\xcd\xf6\xf4\x13\xa1\x9b\x16\x02"
+			  "\x07\x24\xbf\xd5\x08\x20\xd0\x4f"
+			  "\x90\xb3\x70\x24\x2f\x51\xc7\xbb"
+			  "\xd6\x84\xc0\xef\x9a\xa8\xca\xcc"
+			  "\x74\xab\x97\x53\xfe\xd0\xdb\x37"
+			  "\x37\x6a\x0e\x9f\x3f\xa3\x2a\xe3"
+			  "\x1b\x34\x6d\x51\x72\x2b\x17\xe7"
+			  "\x4d\xaa\x2c\x18\xda\xa3\x33\x89"
+			  "\x2a\x9f\xf4\xd2\xed\x76\x3d\x3f"
+			  "\x3c\x15\x9d\x8e\x4f\x3c\x27\xb0"
+			  "\x42\x3f\x2f\x8a\xd4\xc2\x10\xb2"
+			  "\x27\x7f\xe3\x34\x80\x02\x49\x4b"
+			  "\x07\x68\x22\x2a\x88\x25\x53\xb2"
+			  "\x2f",
+		.ilen	= 129,
+		.result	= "\x85\x39\x69\x35\xfb\xf9\xb0\xa6"
+			  "\x85\x43\x88\xd0\xd7\x78\x60\x19"
+			  "\x3e\x1f\xb1\xa4\xd6\xc5\x96\xec"
+			  "\xf7\x84\x85\xc7\x27\x0f\x74\x57"
+			  "\x28\x9e\xdd\x90\x3c\x43\x12\xc5"
+			  "\x51\x3d\x39\x8f\xa5\xf4\xe0\x0b"
+			  "\x57\x04\xf1\x6d\xfe\x9b\x84\x27"
+			  "\xe8\xeb\x4d\xda\x02\x0a\xc5\x49"
+			  "\x1a\x55\x5e\x50\x56\x4d\x94\xda"
+			  "\x20\xf8\x12\x54\x50\xb3\x11\xda"
+			  "\xed\x44\x27\x67\xd5\xd1\x8b\x4b"
+			  "\x38\x67\x56\x65\x59\xda\xe6\x97"
+			  "\x81\xae\x2f\x92\x3b\xae\x22\x1c"
+			  "\x91\x59\x38\x18\x00\xe8\xba\x92"
+			  "\x04\x19\x56\xdf\xb0\x82\xeb\x6f"
+			  "\x2e\xdb\x54\x3c\x4b\xbb\x60\x90"
+			  "\x4c\x50\x10\x62\xba\x7a\xb1\x68"
+			  "\x37\xd7\x87\x4e\xe4\x66\x09\x1f"
+			  "\xa5",
+		.rlen	= 145,
+	}, {
+		.key	= "\xa3\x73\x24\x01\xd5\xbc\xaa\xe3"
+			  "\xa9\x5a\x4c\x90\xf0\x65\x37\x18"
+			  "\x72\x28\x0a\x40\xe7\x20\xd9\x82"
+			  "\xfe\x02\x2b\x09\x57\xb3\xfe\xb7",
+		.klen	= 32,
+		.iv	= "\xb3\x3d\xb3\x69\x19\x5b\x54\x6a"
+			  "\xc9\x91\x79\xb4\xef\x2e\x68\x99",
+		.assoc	= "\xc2\x06\x41\xd1\x5d\xfa\xff\xf1"
+			  "\xe9\xc7\xa5\xd9\xed\xf8\x98\x1b"
+			  "\x07\x89\x10\x82\x6a\x70\x9a\x8f"
+			  "\x5e\x19\x9b\xf5\xc5\xe3\xcd\x22"
+			  "\x92\xa5\xc2\xb8\x51\x2e\x5e\x0e"
+			  "\xa4\xbe\x5f\xb1\xc1\x90\xd7\xe7"
+			  "\xf7\x52\xae\x28\x29\xa8\x22\xa4"
+			  "\x4f\xae\x48\xc2\xfa\x75\x8b\x9e"
+			  "\xce\x83\x2a\x88\x07\x55\xbb\x89"
+			  "\xf6\xdf\xac\xdf\x83\x08\xbf\x7d"
+			  "\xac\x30\x8b\x8e\x02\xac\x00\xf1"
+			  "\x30\x46\xe1\xbc\x75\xbf\x49\xbb"
+			  "\x26\x4e\x29\xf0\x2f\x21\xc6\x13"
+			  "\x92\xd9\x3d\x11\xe4\x10\x00\x8e"
+			  "\xd4\xd4\x58\x65\xa6\x2b\xe3\x25"
+			  "\xb1\x8f\x15\x93\xe7\x71\xb9\x2c"
+			  "\x4b",
+		.alen	= 129,
+		.input	= "\xd1\xcf\xd0\x39\xa1\x99\xa9\x78"
+			  "\x09\xfe\xd2\xfd\xec\xc1\xc9\x9d"
+			  "\xd2\x39\x93\xa3\xab\x18\x7a\x95"
+			  "\x8f\x24\xd3\xeb\x7b\xfa\xb5\xd8"
+			  "\x15\xd1\xc3\x04\x69\x32\xe3\x4d"
+			  "\xaa\xc2\x04\x8b\xf2\xfa\xdc\x4a"
+			  "\x02\xeb\xa8\x90\x03\xfd\xea\x97"
+			  "\x43\xaf\x2e\x92\xf8\x57\xc5\x6a"
+			  "\x00",
+		.ilen	= 65,
+		.result	= "\x7d\xde\x53\x22\xe4\x23\x3b\x30"
+			  "\x78\xde\x35\x90\x7a\xd9\x0b\x93"
+			  "\xf6\x0e\x0b\xed\x40\xee\x10\x9c"
+			  "\x96\x3a\xd3\x34\xb2\xd0\x67\xcf"
+			  "\x63\x7f\x2d\x0c\xcf\x96\xec\x64"
+			  "\x1a\x87\xcc\x7d\x2c\x5e\x81\x4b"
+			  "\xd2\x8f\x4c\x7c\x00\xb1\xb4\xe0"
+			  "\x87\x4d\xb1\xbc\xd8\x78\x2c\x17"
+			  "\xf2\x3b\xd8\x28\x40\xe2\x76\xf6"
+			  "\x20\x13\x83\x46\xaf\xff\xe3\x0f"
+			  "\x72",
+		.rlen	= 81,
+	}, {
+		.key	= "\xe0\x98\x5e\xa1\xe5\x38\x53\xff"
+			  "\x2a\x35\xfe\x21\xea\x8a\xfa\x1e"
+			  "\x9c\xea\x15\xc5\xec\xc0\x5b\x9b"
+			  "\xbf\x2f\x0a\xe1\x32\x12\x9d\x8e",
+		.klen	= 32,
+		.iv	= "\xef\x61\xed\x08\x29\xd7\xfd\x86"
+			  "\x4a\x6b\x2b\x46\xe9\x53\x2a\xa0",
+		.assoc	= "\xfe\x2a\x7b\x70\x6d\x75\xa7\x0d"
+			  "\x6a\xa2\x57\x6a\xe7\x1c\x5b\x21"
+			  "\x31\x4b\x1b\x07\x6f\x10\x1c\xa8"
+			  "\x20\x46\x7a\xce\x9f\x42\x6d\xf9",
+		.alen	= 32,
+		.input	= "\x0d\xf4\x09\xd8\xb1\x14\x51\x94"
+			  "\x8a\xd8\x84\x8e\xe6\xe5\x8c\xa3"
+			  "\xfc\xfc\x9e\x28\xb0\xb8\xfc\xaf"
+			  "\x50\x52\xb1\xc4\x55\x59\x55\xaf",
+		.ilen	= 32,
+		.result	= "\x5a\xcd\x8c\x57\xf2\x6a\xb6\xbe"
+			  "\x53\xc7\xaa\x9a\x60\x74\x9c\xc4"
+			  "\xa2\xc2\xd0\x6d\xe1\x03\x63\xdc"
+			  "\xbb\x51\x7e\x9c\x89\x73\xde\x4e"
+			  "\x24\xf8\x52\x7c\x15\x41\x0e\xba"
+			  "\x69\x0e\x36\x5f\x2f\x22\x8c",
+		.rlen	= 47,
+	}, {
+		.key	= "\x1c\xbd\x98\x40\xf5\xb3\xfc\x1b"
+			  "\xaa\x0f\xb0\xb3\xe4\xae\xbc\x24"
+			  "\xc7\xac\x21\x49\xf1\x60\xdd\xb5"
+			  "\x80\x5d\xe9\xba\x0c\x71\x3c\x64",
+		.klen	= 32,
+		.iv	= "\x2c\x86\x26\xa8\x39\x52\xa6\xa2"
+			  "\xcb\x45\xdd\xd7\xe3\x77\xed\xa6",
+		.assoc	= "\x3b\x4f\xb5\x10\x7d\xf1\x50\x29"
+			  "\xeb\x7c\x0a\xfb\xe1\x40\x1e\x27"
+			  "\x5c\x0d\x27\x8b\x74\xb0\x9e\xc2"
+			  "\xe1\x74\x59\xa6\x79\xa1\x0c\xd0",
+		.alen	= 32,
+		.input	= "\x4a\x18\x43\x77\xc1\x90\xfa\xb0"
+			  "\x0b\xb2\x36\x20\xe0\x09\x4e\xa9"
+			  "\x26\xbe\xaa\xac\xb5\x58\x7e\xc8"
+			  "\x11\x7f\x90\x9c\x2f\xb8\xf4\x85",
+		.ilen	= 32,
+		.result	= "\x47\xd6\xce\x78\xd6\xbf\x4a\x51"
+			  "\xb8\xda\x92\x3c\xfd\xda\xac\x8e"
+			  "\x8d\x88\xd7\x4d\x90\xe5\xeb\xa1"
+			  "\xab\xd6\x7c\x76\xad\xea\x7d\x76"
+			  "\x53\xee\xb0\xcd\xd0\x02\xbb\x70"
+			  "\x5b\x6f\x7b\xe2\x8c\xe8",
+		.rlen	= 46,
+	}, {
+		.key	= "\x59\xe1\xd2\xdf\x05\x2f\xa4\x37"
+			  "\x2b\xe9\x63\x44\xde\xd3\x7f\x2b"
+			  "\xf1\x6f\x2d\xcd\xf6\x00\x5f\xcf"
+			  "\x42\x8a\xc8\x92\xe6\xd0\xdc\x3b",
+		.klen	= 32,
+		.iv	= "\x68\xab\x60\x47\x49\xce\x4f\xbe"
+			  "\x4c\x20\x8f\x68\xdd\x9c\xb0\xac",
+		.assoc	= "\x77\x74\xee\xaf\x8d\x6d\xf9\x45"
+			  "\x6c\x56\xbc\x8d\xdb\x65\xe0\x2e"
+			  "\x86\xd0\x32\x0f\x79\x50\x20\xdb"
+			  "\xa2\xa1\x37\x7e\x53\x00\xab\xa6",
+		.alen	= 32,
+		.input	= "\x86\x3d\x7d\x17\xd1\x0c\xa3\xcc"
+			  "\x8c\x8d\xe8\xb1\xda\x2e\x11\xaf"
+			  "\x51\x80\xb5\x30\xba\xf8\x00\xe2"
+			  "\xd3\xad\x6f\x75\x09\x18\x93\x5c",
+		.ilen	= 32,
+		.result	= "\x9f\xa9\x2b\xa4\x8f\x00\x05\x2b"
+			  "\xe7\x68\x81\x51\xbb\xfb\xdf\x60"
+			  "\xbb\xac\xe8\xc1\xdc\x68\xae\x68"
+			  "\x3a\xcd\x7a\x06\x49\xfe\x80\x11"
+			  "\xe6\x61\x99\xe2\xdd\xbe\x2c\xbf",
+		.rlen	= 40,
+	}, {
+		.key	= "\x96\x06\x0b\x7f\x15\xab\x4d\x53"
+			  "\xac\xc3\x15\xd6\xd8\xf7\x42\x31"
+			  "\x1b\x31\x38\x51\xfc\xa0\xe1\xe8"
+			  "\x03\xb8\xa7\x6b\xc0\x2f\x7b\x11",
+		.klen	= 32,
+		.iv	= "\xa5\xcf\x9a\xe6\x59\x4a\xf7\xd9"
+			  "\xcd\xfa\x41\xfa\xd7\xc0\x72\xb2",
+		.assoc	= "\xb4\x99\x28\x4e\x9d\xe8\xa2\x60"
+			  "\xed\x30\x6e\x1e\xd5\x89\xa3\x34"
+			  "\xb1\x92\x3e\x93\x7e\xf0\xa2\xf5"
+			  "\x64\xcf\x16\x57\x2d\x5f\x4a\x7d",
+		.alen	= 32,
+		.input	= "\xc3\x62\xb7\xb6\xe2\x87\x4c\xe7"
+			  "\x0d\x67\x9a\x43\xd4\x52\xd4\xb5"
+			  "\x7b\x43\xc1\xb5\xbf\x98\x82\xfc"
+			  "\x94\xda\x4e\x4d\xe4\x77\x32\x32",
+		.ilen	= 32,
+		.result	= "\xe2\x34\xfa\x25\xfd\xfb\x89\x5e"
+			  "\x5b\x4e\x0b\x15\x6e\x39\xfb\x0c"
+			  "\x73\xc7\xd9\x6b\xbe\xce\x9b\x70"
+			  "\xc7\x4f\x96\x16\x03\xfc\xea\xfb"
+			  "\x56",
+		.rlen	= 33,
+	},
+};
+
+static const struct aead_testvec morus1280_dec_tv_template[] = {
+	{
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 16,
+		.iv	= "\x0f\xc9\x8e\x67\x44\x9e\xaa\x86"
+			  "\x20\x36\x2c\x24\xfe\xc9\x30\x81",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x91\x85\x0f\xf5\x52\x9e\xce\xce"
+			  "\x65\x99\xc7\xbf\xd3\x76\xe8\x98",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x3c\x24\x39\x9f\x10\x7b\xa8\x1b"
+			  "\x80\xda\xb2\x91\xf9\x24\xc2\x06",
+		.klen	= 16,
+		.iv	= "\x4b\xed\xc8\x07\x54\x1a\x52\xa2"
+			  "\xa1\x10\xde\xb5\xf8\xed\xf3\x87",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x88\xc3\x4c\xf0\x2f\x43\x76\x13"
+			  "\x96\xda\x76\x34\x33\x4e\xd5\x39"
+			  "\x73",
+		.ilen	= 17,
+		.result	= "\x69",
+		.rlen	= 1,
+	}, {
+		.key	= "\x79\x49\x73\x3e\x20\xf7\x51\x37"
+			  "\x01\xb4\x64\x22\xf3\x48\x85\x0c",
+		.klen	= 16,
+		.iv	= "\x88\x12\x01\xa6\x64\x96\xfb\xbe"
+			  "\x22\xea\x90\x47\xf2\x11\xb5\x8e",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x3e\x5c\x3b\x58\x3b\x7d\x2a\x22"
+			  "\x75\x0b\x24\xa6\x0e\xc3\xde\x52"
+			  "\x97\x0b\x64\xd4\xce\x90\x52\xf7"
+			  "\xef\xdb\x6a\x38\xd2\xa8\xa1\x0d"
+			  "\xe0\x61\x33\x24\xc6\x4d\x51\xbc"
+			  "\xa4\x21\x74\xcf\x19\x16\x59",
+		.ilen	= 47,
+		.result	= "\xa6\xa4\x1e\x76\xec\xd4\x50\xcc"
+			  "\x62\x58\xe9\x8f\xef\xa4\x17\x91"
+			  "\xb4\x96\x9f\x6b\xce\x38\xa5\x46"
+			  "\x13\x7d\x64\x93\xd7\x05\xf5",
+		.rlen	= 31,
+	}, {
+		.key	= "\xb5\x6e\xad\xdd\x30\x72\xfa\x53"
+			  "\x82\x8e\x16\xb4\xed\x6d\x47\x12",
+		.klen	= 16,
+		.iv	= "\xc4\x37\x3b\x45\x74\x11\xa4\xda"
+			  "\xa2\xc5\x42\xd8\xec\x36\x78\x94",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x30\x82\x9c\x2b\x67\xcb\xf9\x1f"
+			  "\xde\x9f\x77\xb2\xda\x92\x61\x5c"
+			  "\x09\x0b\x2d\x9a\x26\xaa\x1c\x06"
+			  "\xab\x74\xb7\x2b\x95\x5f\x9f\xa1"
+			  "\x9a\xff\x50\xa0\xa2\xff\xc5\xad"
+			  "\x21\x8e\x84\x5c\x12\x61\xb2\xae",
+		.ilen	= 48,
+		.result	= "\xe2\xc9\x58\x15\xfc\x4f\xf8\xe8"
+			  "\xe3\x32\x9b\x21\xe9\xc8\xd9\x97"
+			  "\xde\x58\xab\xf0\xd3\xd8\x27\x60"
+			  "\xd5\xaa\x43\x6b\xb1\x64\x95\xa4",
+		.rlen	= 32,
+	}, {
+		.key	= "\xf2\x92\xe6\x7d\x40\xee\xa3\x6f"
+			  "\x03\x68\xc8\x45\xe7\x91\x0a\x18",
+		.klen	= 16,
+		.iv	= "\x01\x5c\x75\xe5\x84\x8d\x4d\xf6"
+			  "\x23\x9f\xf4\x6a\xe6\x5a\x3b\x9a",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x67\x5d\x8e\x45\xc8\x39\xf5\x17"
+			  "\xc1\x1d\x2a\xdd\x88\x67\xda\x1f"
+			  "\x6d\xe8\x37\x28\x5a\xc1\x5e\x9f"
+			  "\xa6\xec\xc6\x92\x05\x4b\xc0\xa3"
+			  "\x63\xef\x88\xa4\x9b\x0a\x5c\xed"
+			  "\x2b\x6a\xac\x63\x52\xaa\x10\x94"
+			  "\xd0",
+		.ilen	= 49,
+		.result	= "\x1f\xee\x92\xb4\x0c\xcb\xa1\x04"
+			  "\x64\x0c\x4d\xb2\xe3\xec\x9c\x9d"
+			  "\x09\x1a\xb7\x74\xd8\x78\xa9\x79"
+			  "\x96\xd8\x22\x43\x8c\xc3\x34\x7b"
+			  "\xc4",
+		.rlen	= 33,
+	}, {
+		.key	= "\x2e\xb7\x20\x1c\x50\x6a\x4b\x8b"
+			  "\x84\x42\x7a\xd7\xe1\xb5\xcd\x1f",
+		.klen	= 16,
+		.iv	= "\x3d\x80\xae\x84\x94\x09\xf6\x12"
+			  "\xa4\x79\xa6\xfb\xe0\x7f\xfd\xa0",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x7d\x61\x1a\x35\x20\xcc\x07\x88"
+			  "\x03\x98\x87\xcf\xc0\x6e\x4d\x19"
+			  "\xe3\xd4\x0b\xfb\x29\x8f\x49\x1a"
+			  "\x3a\x06\x77\xce\x71\x2c\xcd\xdd"
+			  "\xed\xf6\xc9\xbe\xa6\x3b\xb8\xfc"
+			  "\x6c\xbe\x77\xed\x74\x0e\x20\x85"
+			  "\xd0\x65\xde\x24\x6f\xe3\x25\xc5"
+			  "\xdf\x5b\x0f\xbd\x8a\x88\x78\xc9"
+			  "\xe5\x81\x37\xde\x84\x7a\xf6\x84"
+			  "\x99\x7a\x72\x9c\x54\x31\xa1",
+		.ilen	= 79,
+		.result	= "\x5c\x13\xcb\x54\x1c\x47\x4a\x1f"
+			  "\xe5\xe6\xff\x44\xdd\x11\x5f\xa3"
+			  "\x33\xdd\xc2\xf8\xdd\x18\x2b\x93"
+			  "\x57\x05\x01\x1c\x66\x22\xd3\x51"
+			  "\xd3\xdf\x18\xc9\x30\x66\xed\xb1"
+			  "\x96\x58\xd5\x8c\x64\x8c\x7c\xf5"
+			  "\x01\xd0\x74\x5f\x9b\xaa\xf6\xd1"
+			  "\xe6\x16\xa2\xac\xde\x47\x40",
+		.rlen	= 63,
+	}, {
+		.key	= "\x6b\xdc\x5a\xbb\x60\xe5\xf4\xa6"
+			  "\x05\x1d\x2c\x68\xdb\xda\x8f\x25",
+		.klen	= 16,
+		.iv	= "\x7a\xa5\xe8\x23\xa4\x84\x9e\x2d"
+			  "\x25\x53\x58\x8c\xda\xa3\xc0\xa6",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x05\xc5\xb1\xf9\x1b\xb9\xab\x2c"
+			  "\xa5\x07\x12\xa7\x12\x39\x60\x66"
+			  "\x30\x81\x4a\x03\x78\x28\x45\x52"
+			  "\xd2\x2b\x24\xfd\x8b\xa5\xb7\x66"
+			  "\x6f\x45\xd7\x3b\x67\x6f\x51\xb9"
+			  "\xc0\x3d\x6c\xca\x1e\xae\xff\xb6"
+			  "\x79\xa9\xe4\x82\x5d\x4c\x2d\xdf"
+			  "\xeb\x71\x40\xc9\x2c\x40\x45\x6d"
+			  "\x73\x77\x01\xf3\x4f\xf3\x9d\x2a"
+			  "\x5d\x57\xa8\xa1\x18\xa2\xad\xcb",
+		.ilen	= 80,
+		.result	= "\x98\x37\x05\xf3\x2c\xc2\xf3\x3b"
+			  "\x66\xc0\xb1\xd5\xd7\x35\x21\xaa"
+			  "\x5d\x9f\xce\x7c\xe2\xb8\xad\xad"
+			  "\x19\x33\xe0\xf4\x40\x81\x72\x28"
+			  "\xe1\x8b\x1c\xf8\x91\x78\xff\xaf"
+			  "\xb0\x68\x69\xf2\x27\x35\x91\x84"
+			  "\x2e\x37\x5b\x00\x04\xff\x16\x9c"
+			  "\xb5\x19\x39\xeb\xd9\xcd\x29\x9a",
+		.rlen	= 64,
+	}, {
+		.key	= "\xa7\x00\x93\x5b\x70\x61\x9d\xc2"
+			  "\x86\xf7\xde\xfa\xd5\xfe\x52\x2b",
+		.klen	= 16,
+		.iv	= "\xb6\xca\x22\xc3\xb4\x00\x47\x49"
+			  "\xa6\x2d\x0a\x1e\xd4\xc7\x83\xad",
+		.assoc	= "\xc5",
+		.alen	= 1,
+		.input	= "\x4d\xbf\x11\xac\x7f\x97\x0b\x2e"
+			  "\x89\x3b\x9d\x0f\x83\x1c\x08\xc3",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xe4\x25\xcd\xfa\x80\xdd\x46\xde"
+			  "\x07\xd1\x90\x8b\xcf\x23\x15\x31",
+		.klen	= 16,
+		.iv	= "\xf3\xee\x5c\x62\xc4\x7c\xf0\x65"
+			  "\x27\x08\xbd\xaf\xce\xec\x45\xb3",
+		.assoc	= "\x02\xb8\xea\xca\x09\x1b\x9a\xec"
+			  "\x47\x3e\xe9\xd4\xcc\xb5\x76\x34"
+			  "\xe8\x73\x62\x64\xab\x50\xd0\xda"
+			  "\x6b\x83\x66\xaf\x3e\x27\xc9",
+		.alen	= 31,
+		.input	= "\x5b\xc0\x8d\x54\xe4\xec\xbe\x38"
+			  "\x03\x12\xf9\xcc\x9e\x46\x42\x92",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x20\x4a\x07\x99\x91\x58\xee\xfa"
+			  "\x88\xab\x42\x1c\xc9\x47\xd7\x38",
+		.klen	= 16,
+		.iv	= "\x2f\x13\x95\x01\xd5\xf7\x99\x81"
+			  "\xa8\xe2\x6f\x41\xc8\x10\x08\xb9",
+		.assoc	= "\x3f\xdc\x24\x69\x19\x96\x43\x08"
+			  "\xc8\x18\x9b\x65\xc6\xd9\x39\x3b"
+			  "\x12\x35\x6e\xe8\xb0\xf0\x52\xf3"
+			  "\x2d\xb0\x45\x87\x18\x86\x68\xf6",
+		.alen	= 32,
+		.input	= "\x48\xc5\xc3\x4c\x40\x2e\x2f\xc2"
+			  "\x6d\x65\xe0\x67\x9c\x1d\xa0\xf0",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x5d\x6f\x41\x39\xa1\xd4\x97\x16"
+			  "\x09\x85\xf4\xae\xc3\x6b\x9a\x3e",
+		.klen	= 16,
+		.iv	= "\x6c\x38\xcf\xa1\xe5\x73\x41\x9d"
+			  "\x29\xbc\x21\xd2\xc2\x35\xcb\xbf",
+		.assoc	= "\x7b\x01\x5d\x08\x29\x12\xec\x24"
+			  "\x49\xf3\x4d\xf7\xc0\xfe\xfb\x41"
+			  "\x3c\xf8\x79\x6c\xb6\x90\xd4\x0d"
+			  "\xee\xde\x23\x60\xf2\xe5\x08\xcc"
+			  "\x97",
+		.alen	= 33,
+		.input	= "\x28\x64\x78\x51\x55\xd8\x56\x4a"
+			  "\x58\x3e\xf7\xbe\xee\x21\xfe\x94",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x99\x93\x7a\xd8\xb1\x50\x40\x31"
+			  "\x8a\x60\xa6\x3f\xbd\x90\x5d\x44",
+		.klen	= 16,
+		.iv	= "\xa8\x5c\x09\x40\xf5\xef\xea\xb8"
+			  "\xaa\x96\xd3\x64\xbc\x59\x8d\xc6",
+		.assoc	= "\xb8\x26\x97\xa8\x39\x8e\x94\x3f"
+			  "\xca\xcd\xff\x88\xba\x22\xbe\x47"
+			  "\x67\xba\x85\xf1\xbb\x30\x56\x26"
+			  "\xaf\x0b\x02\x38\xcc\x44\xa7\xa3"
+			  "\xa6\xbf\x31\x93\x60\xcd\xda\x63"
+			  "\x2c\xb1\xaa\x19\xc8\x19\xf8\xeb"
+			  "\x03\xa1\xe8\xbe\x37\x54\xec\xa2"
+			  "\xcd\x2c\x45\x58\xbd\x8e\x80",
+		.alen	= 63,
+		.input	= "\xb3\xa6\x00\x4e\x09\x20\xac\x21"
+			  "\x77\x72\x69\x76\x2d\x36\xe5\xc8",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xd6\xb8\xb4\x77\xc1\xcb\xe9\x4d"
+			  "\x0a\x3a\x58\xd1\xb7\xb4\x1f\x4a",
+		.klen	= 16,
+		.iv	= "\xe5\x81\x42\xdf\x05\x6a\x93\xd4"
+			  "\x2b\x70\x85\xf5\xb6\x7d\x50\xcc",
+		.assoc	= "\xf4\x4a\xd1\x47\x49\x09\x3d\x5b"
+			  "\x4b\xa7\xb1\x19\xb4\x46\x81\x4d"
+			  "\x91\x7c\x91\x75\xc0\xd0\xd8\x40"
+			  "\x71\x39\xe1\x10\xa6\xa3\x46\x7a"
+			  "\xb4\x6b\x35\xc2\xc1\xdf\xed\x60"
+			  "\x46\xc1\x3e\x7f\x8c\xc2\x0e\x7a"
+			  "\x30\x08\xd0\x5f\xa0\xaa\x0c\x6d"
+			  "\x9c\x2f\xdb\x97\xb8\x15\x69\x01",
+		.alen	= 64,
+		.input	= "\x65\x33\x7b\xa1\x63\xf4\x20\xdd"
+			  "\xe4\xb9\x4a\xaa\x9a\x21\xaa\x14",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x12\xdd\xee\x17\xd1\x47\x92\x69"
+			  "\x8b\x14\x0a\x62\xb1\xd9\xe2\x50",
+		.klen	= 16,
+		.iv	= "\x22\xa6\x7c\x7f\x15\xe6\x3c\xf0"
+			  "\xac\x4b\x37\x86\xb0\xa2\x13\xd2",
+		.assoc	= "\x31",
+		.alen	= 1,
+		.input	= "\x1d\x47\x17\x34\x86\xf5\x54\x1a"
+			  "\x6d\x28\xb8\x5d\x6c\xcf\xa0\xb9"
+			  "\xbf",
+		.ilen	= 17,
+		.result	= "\x40",
+		.rlen	= 1,
+	}, {
+		.key	= "\x4f\x01\x27\xb6\xe1\xc3\x3a\x85"
+			  "\x0c\xee\xbc\xf4\xab\xfd\xa5\x57",
+		.klen	= 16,
+		.iv	= "\x5e\xcb\xb6\x1e\x25\x62\xe4\x0c"
+			  "\x2d\x25\xe9\x18\xaa\xc6\xd5\xd8",
+		.assoc	= "\x6d\x94\x44\x86\x69\x00\x8f\x93"
+			  "\x4d\x5b\x15\x3c\xa8\x8f\x06\x5a"
+			  "\xe6\x01\xa8\x7e\xca\x10\xdc\x73"
+			  "\xf4\x94\x9f\xc1\x5a\x61\x85",
+		.alen	= 31,
+		.input	= "\x78\x90\x52\xae\x0f\xf7\x2e\xef"
+			  "\x63\x09\x08\x58\xb5\x56\xbd\x72"
+			  "\x6e\x42\xcf\x27\x04\x7c\xdb\x92"
+			  "\x18\xe9\xa4\x33\x90\xba\x62\xb5"
+			  "\x70\xd3\x88\x9b\x4f\x05\xa7\x51"
+			  "\x85\x87\x17\x09\x42\xed\x4e",
+		.ilen	= 47,
+		.result	= "\x7c\x5d\xd3\xee\xad\x9f\x39\x1a"
+			  "\x6d\x92\x42\x61\xa7\x58\x37\xdb"
+			  "\xb0\xb2\x2b\x9f\x0b\xb8\xbd\x7a"
+			  "\x24\xa0\xd6\xb7\x11\x79\x6c",
+		.rlen	= 31,
+	}, {
+		.key	= "\x8b\x26\x61\x55\xf1\x3e\xe3\xa1"
+			  "\x8d\xc8\x6e\x85\xa5\x21\x67\x5d",
+		.klen	= 16,
+		.iv	= "\x9b\xef\xf0\xbd\x35\xdd\x8d\x28"
+			  "\xad\xff\x9b\xa9\xa4\xeb\x98\xdf",
+		.assoc	= "\xaa\xb8\x7e\x25\x79\x7c\x37\xaf"
+			  "\xce\x36\xc7\xce\xa2\xb4\xc9\x60"
+			  "\x10\xc3\xb3\x02\xcf\xb0\x5e\x8d"
+			  "\xb5\xc2\x7e\x9a\x35\xc0\x24\xfd",
+		.alen	= 32,
+		.input	= "\x1d\x2c\x57\xe0\x50\x38\x3d\x41"
+			  "\x2e\x71\xc8\x3b\x92\x43\x58\xaf"
+			  "\x5a\xfb\xad\x8f\xd9\xd5\x8a\x5e"
+			  "\xdb\xf3\xcd\x3a\x2b\xe1\x2c\x1a"
+			  "\xb0\xed\xe3\x0c\x6e\xf9\xf2\xd6"
+			  "\x90\xe6\xb1\x0e\xa5\x8a\xac\xb7",
+		.ilen	= 48,
+		.result	= "\xb9\x82\x0c\x8d\xbd\x1b\xe2\x36"
+			  "\xee\x6c\xf4\xf2\xa1\x7d\xf9\xe2"
+			  "\xdb\x74\x36\x23\x11\x58\x3f\x93"
+			  "\xe5\xcd\xb5\x90\xeb\xd8\x0c\xb3",
+		.rlen	= 32,
+	}, {
+		.key	= "\xc8\x4b\x9b\xf5\x01\xba\x8c\xbd"
+			  "\x0e\xa3\x21\x16\x9f\x46\x2a\x63",
+		.klen	= 16,
+		.iv	= "\xd7\x14\x29\x5d\x45\x59\x36\x44"
+			  "\x2e\xd9\x4d\x3b\x9e\x0f\x5b\xe5",
+		.assoc	= "\xe6\xdd\xb8\xc4\x89\xf8\xe0\xca"
+			  "\x4f\x10\x7a\x5f\x9c\xd8\x8b\x66"
+			  "\x3b\x86\xbf\x86\xd4\x50\xe0\xa7"
+			  "\x76\xef\x5c\x72\x0f\x1f\xc3\xd4"
+			  "\xee",
+		.alen	= 33,
+		.input	= "\x59\x10\x84\x1c\x83\x4c\x8b\xfc"
+			  "\xfd\x2e\x4b\x46\x84\xff\x78\x4e"
+			  "\x50\xda\x5c\xb9\x61\x1d\xf5\xb9"
+			  "\xfe\xbb\x7f\xae\x8c\xc1\x24\xbd"
+			  "\x8c\x6f\x1f\x9b\xce\xc6\xc1\x37"
+			  "\x08\x06\x5a\xe5\x96\x10\x95\xc2"
+			  "\x5e",
+		.ilen	= 49,
+		.result	= "\xf5\xa6\x46\x2c\xce\x97\x8a\x51"
+			  "\x6f\x46\xa6\x83\x9b\xa1\xbc\xe8"
+			  "\x05\x36\x42\xa7\x16\xf8\xc1\xad"
+			  "\xa7\xfb\x94\x68\xc5\x37\xab\x8a"
+			  "\x72",
+		.rlen	= 33,
+	}, {
+		.key	= "\x05\x70\xd5\x94\x12\x36\x35\xd8"
+			  "\x8f\x7d\xd3\xa8\x99\x6a\xed\x69",
+		.klen	= 16,
+		.iv	= "\x14\x39\x63\xfc\x56\xd5\xdf\x5f"
+			  "\xaf\xb3\xff\xcc\x98\x33\x1d\xeb",
+		.assoc	= "\x23\x02\xf1\x64\x9a\x73\x89\xe6"
+			  "\xd0\xea\x2c\xf1\x96\xfc\x4e\x6d"
+			  "\x65\x48\xcb\x0a\xda\xf0\x62\xc0"
+			  "\x38\x1d\x3b\x4a\xe9\x7e\x62\xaa"
+			  "\xfd\xc9\x4a\xa9\xa9\x39\x4b\x54"
+			  "\xc8\x0e\x24\x7f\x5e\x10\x7a\x45"
+			  "\x10\x0b\x56\x85\xad\x54\xaa\x66"
+			  "\xa8\x43\xcd\xd4\x9b\xb7\xfa",
+		.alen	= 63,
+		.input	= "\x9a\x12\xbc\xdf\x72\xa8\x56\x22"
+			  "\x49\x2d\x07\x92\xfc\x3d\x6d\x5f"
+			  "\xef\x36\x19\xae\x91\xfa\xd6\x63"
+			  "\x46\xea\x8a\x39\x14\x21\xa6\x37"
+			  "\x18\xfc\x97\x3e\x16\xa5\x4d\x39"
+			  "\x45\x2e\x69\xcc\x9c\x5f\xdf\x6d"
+			  "\x5e\xa2\xbf\xac\x83\x32\x72\x52"
+			  "\x58\x58\x23\x40\xfd\xa5\xc2\xe6"
+			  "\xe9\x5a\x50\x98\x00\x58\xc9\x86"
+			  "\x4f\x20\x37\xdb\x7b\x22\xa3",
+		.ilen	= 79,
+		.result	= "\x32\xcb\x80\xcc\xde\x12\x33\x6d"
+			  "\xf0\x20\x58\x15\x95\xc6\x7f\xee"
+			  "\x2f\xf9\x4e\x2c\x1b\x98\x43\xc7"
+			  "\x68\x28\x73\x40\x9f\x96\x4a\x60"
+			  "\x80\xf4\x4b\xf4\xc1\x3d\xd0\x93"
+			  "\xcf\x12\xc9\x59\x8f\x7a\x7f\xa8"
+			  "\x1b\xa5\x50\xed\x87\xa9\x72\x59"
+			  "\x9c\x44\xb2\xa4\x99\x98\x34",
+		.rlen	= 63,
+	}, {
+		.key	= "\x41\x94\x0e\x33\x22\xb1\xdd\xf4"
+			  "\x10\x57\x85\x39\x93\x8f\xaf\x70",
+		.klen	= 16,
+		.iv	= "\x50\x5d\x9d\x9b\x66\x50\x88\x7b"
+			  "\x30\x8e\xb1\x5e\x92\x58\xe0\xf1",
+		.assoc	= "\x5f\x27\x2b\x03\xaa\xef\x32\x02"
+			  "\x50\xc4\xde\x82\x90\x21\x11\x73"
+			  "\x8f\x0a\xd6\x8f\xdf\x90\xe4\xda"
+			  "\xf9\x4a\x1a\x23\xc3\xdd\x02\x81"
+			  "\x0b\x76\x4f\xd7\x0a\x4b\x5e\x51"
+			  "\xe3\x1d\xb9\xe5\x21\xb9\x8f\xd4"
+			  "\x3d\x72\x3e\x26\x16\xa9\xca\x32"
+			  "\x77\x47\x63\x14\x95\x3d\xe4\x34",
+		.alen	= 64,
+		.input	= "\xe6\xeb\x92\x5a\x5b\xf0\x2d\xbb"
+			  "\x23\xec\x35\xe3\xae\xc9\xfb\x0b"
+			  "\x90\x14\x46\xeb\xa8\x8d\xb0\x9b"
+			  "\x39\xda\x8b\x48\xec\xb2\x00\x4e"
+			  "\x80\x6f\x46\x4f\x9b\x1e\xbb\x35"
+			  "\xea\x5a\xbc\xa2\x36\xa5\x89\x45"
+			  "\xc2\xd6\xd7\x15\x0b\xf6\x6c\x56"
+			  "\xec\x99\x7d\x61\xb3\x15\x93\xed"
+			  "\x83\x1e\xd9\x48\x84\x0b\x37\xfe"
+			  "\x95\x74\x44\xd5\x54\xa6\x27\x06",
+		.ilen	= 80,
+		.result	= "\x6e\xf0\xba\x6b\xee\x8e\xdc\x89"
+			  "\x71\xfb\x0a\xa6\x8f\xea\x41\xf4"
+			  "\x5a\xbb\x59\xb0\x20\x38\xc5\xe0"
+			  "\x29\x56\x52\x19\x79\xf5\xe9\x37"
+			  "\x8f\xa1\x50\x23\x22\x4f\xe3\x91"
+			  "\xe9\x21\x5e\xbf\x52\x23\x95\x37"
+			  "\x48\x0c\x38\x8f\xf0\xff\x92\x24"
+			  "\x6b\x47\x49\xe3\x94\x1f\x1e\x01",
+		.rlen	= 64,
+	}, {
+		.key	= "\x7e\xb9\x48\xd3\x32\x2d\x86\x10"
+			  "\x91\x31\x37\xcb\x8d\xb3\x72\x76",
+		.klen	= 16,
+		.iv	= "\x8d\x82\xd6\x3b\x76\xcc\x30\x97"
+			  "\xb1\x68\x63\xef\x8c\x7c\xa3\xf7",
+		.assoc	= "\x9c\x4b\x65\xa2\xba\x6b\xdb\x1e"
+			  "\xd1\x9e\x90\x13\x8a\x45\xd3\x79"
+			  "\xba\xcd\xe2\x13\xe4\x30\x66\xf4"
+			  "\xba\x78\xf9\xfb\x9d\x3c\xa1\x58"
+			  "\x1a\x22\x53\x05\x6b\x5c\x71\x4f"
+			  "\xfd\x2d\x4d\x4c\xe5\x62\xa5\x63"
+			  "\x6a\xda\x26\xc8\x7f\xff\xea\xfd"
+			  "\x46\x4a\xfa\x53\x8f\xc4\xcd\x68"
+			  "\x58",
+		.alen	= 65,
+		.input	= "\x89\x24\x27\x86\xdc\xd7\x6b\xd9"
+			  "\xd1\xcd\xdc\x16\xdd\x2c\xc1\xfb"
+			  "\x52\xb5\xb3\xab\x50\x99\x3f\xa0"
+			  "\x38\xa4\x74\xa5\x04\x15\x63\x05"
+			  "\x8f\x54\x81\x06\x5a\x6b\xa4\x63"
+			  "\x6d\xa7\x21\xcb\xff\x42\x30\x8e"
+			  "\x3b\xd1\xca\x3f\x4b\x1a\xb8\xc3"
+			  "\x42\x01\xe6\xbc\x75\x15\x87\xee"
+			  "\xc9\x8e\x65\x01\xd9\xd8\xb5\x9f"
+			  "\x48\x86\xa6\x5f\x2c\xc7\xb5\xb0"
+			  "\xed\x5d\x14\x7c\x3f\x40\xb1\x0b"
+			  "\x72\xef\x94\x8d\x7a\x85\x56\xe5"
+			  "\x56\x08\x15\x56\xba\xaf\xbd\xf0"
+			  "\x20\xef\xa0\xf6\xa9\xad\xa2\xc9"
+			  "\x1c\x3b\x28\x51\x7e\x77\xb2\x18"
+			  "\x4f\x61\x64\x37\x22\x36\x6d\x78"
+			  "\xed\xed\x35\xe8\x83\xa5\xec\x25"
+			  "\x6b\xff\x5f\x1a\x09\x96\x3d\xdc"
+			  "\x20",
+		.ilen	= 145,
+		.result	= "\xab\x14\xf3\x0a\xfe\x0a\x85\xa5"
+			  "\xf2\xd5\xbc\x38\x89\x0e\x04\xfb"
+			  "\x84\x7d\x65\x34\x25\xd8\x47\xfa"
+			  "\xeb\x83\x31\xf1\x54\x54\x89\x0d"
+			  "\x9d\x4d\x54\x51\x84\x61\xf6\x8e"
+			  "\x03\x31\xf2\x25\x16\xcc\xaa\xc6"
+			  "\x75\x73\x20\x30\x59\x54\xb2\xf0"
+			  "\x3a\x4b\xe0\x23\x8e\xa6\x08\x35"
+			  "\x8a\xdf\x27\xa0\xe4\x60\x99\xae"
+			  "\x8e\x43\xd9\x39\x7b\x10\x40\x67"
+			  "\x5c\x7e\xc9\x70\x63\x34\xca\x59"
+			  "\xfe\x86\xbc\xb7\x9c\x39\xf3\x6d"
+			  "\x6a\x41\x64\x6f\x16\x7f\x65\x7e"
+			  "\x89\x84\x68\xeb\xb0\x51\xbe\x55"
+			  "\x33\x16\x59\x6c\x3b\xef\x88\xad"
+			  "\x2f\xab\xbc\x25\x76\x87\x41\x2f"
+			  "\x36",
+		.rlen	= 129,
+	}, {
+		.key	= "\xba\xde\x82\x72\x42\xa9\x2f\x2c"
+			  "\x12\x0b\xe9\x5c\x87\xd7\x35\x7c",
+		.klen	= 16,
+		.iv	= "\xc9\xa7\x10\xda\x86\x48\xd9\xb3"
+			  "\x32\x42\x15\x80\x85\xa1\x65\xfe",
+		.assoc	= "\xd8\x70\x9f\x42\xca\xe6\x83\x3a"
+			  "\x52\x79\x42\xa5\x84\x6a\x96\x7f"
+			  "\xe4\x8f\xed\x97\xe9\xd0\xe8\x0d"
+			  "\x7c\xa6\xd8\xd4\x77\x9b\x40\x2e"
+			  "\x28\xce\x57\x34\xcd\x6e\x84\x4c"
+			  "\x17\x3c\xe1\xb2\xa8\x0b\xbb\xf1"
+			  "\x96\x41\x0d\x69\xe8\x54\x0a\xc8"
+			  "\x15\x4e\x91\x92\x89\x4b\xb7\x9b"
+			  "\x21\xf7\x42\x89\xac\x12\x2a\x54"
+			  "\x69\xee\x18\xc7\x8d\xed\xe8\xfd"
+			  "\xbb\x04\x28\xe6\x8a\x3c\x98\xc1"
+			  "\x04\x2d\xa9\xa1\x24\x83\xff\xe9"
+			  "\x55\x7a\xf0\xd1\xf6\x63\x05\xe1"
+			  "\xd9\x1e\x75\x72\xc1\x9f\xae\x32"
+			  "\xe1\x6b\xcd\x9e\x61\x19\x23\x86"
+			  "\xd9\xd2\xaf\x8e\xd5\xd3\xa8\xa9"
+			  "\x51",
+		.alen	= 129,
+		.input	= "\x36\x78\xb9\x22\xde\x62\x35\x55"
+			  "\x1a\x7a\xf5\x45\xbc\xd7\x15\x82"
+			  "\x01\xe9\x5a\x07\xea\x46\xaf\x91"
+			  "\xcb\x73\xa5\xee\xe1\xb4\xbf\xc2"
+			  "\xdb\xd2\x9d\x59\xde\xfc\x83\x00"
+			  "\xf5\x46\xac\x97\xd5\x57\xa9\xb9"
+			  "\x1f\x8c\xe8\xca\x68\x8b\x91\x0c"
+			  "\x01\xbe\x0a\xaf\x7c\xf6\x67\xa4"
+			  "\xbf\xbc\x88\x3f\x5d\xd1\xf9\x19"
+			  "\x0f\x9d\xb2\xaf\xb9\x6e\x17\xdf"
+			  "\xa2",
+		.ilen	= 81,
+		.result	= "\xe8\x39\x2d\xaa\x0e\x85\x2d\xc1"
+			  "\x72\xaf\x6e\xc9\x82\x33\xc7\x01"
+			  "\xaf\x40\x70\xb8\x2a\x78\xc9\x14"
+			  "\xac\xb1\x10\xca\x2e\xb3\x28\xe4"
+			  "\xac\xfa\x58\x7f\xe5\x73\x09\x8c"
+			  "\x1d\x40\x87\x8c\xd9\x75\xc0\x55"
+			  "\xa2\xda\x07\xd1\xc2\xa9\xd1\xbb"
+			  "\x09\x4f\x77\x62\x88\x2d\xf2\x68"
+			  "\x54",
+		.rlen	= 65,
+	}, {
+		.key	= "\xf7\x02\xbb\x11\x52\x24\xd8\x48"
+			  "\x93\xe6\x9b\xee\x81\xfc\xf7\x82",
+		.klen	= 16,
+		.iv	= "\x06\xcc\x4a\x79\x96\xc3\x82\xcf"
+			  "\xb3\x1c\xc7\x12\x7f\xc5\x28\x04",
+		.assoc	= "\x15\x95\xd8\xe1\xda\x62\x2c\x56"
+			  "\xd3\x53\xf4\x36\x7e\x8e\x59\x85"
+			  "\x0e\x51\xf9\x1c\xee\x70\x6a\x27"
+			  "\x3d\xd3\xb7\xac\x51\xfa\xdf\x05",
+		.alen	= 32,
+		.input	= "\x08\x1b\x95\x0e\x41\x95\x02\x4b"
+			  "\x9c\xbb\xa8\xd0\x7c\xd3\x44\x6e"
+			  "\x89\x14\x33\x70\x0a\xbc\xea\x39"
+			  "\x88\xaa\x2b\xd5\x73\x11\x55\xf5"
+			  "\x33\x33\x9c\xd7\x42\x34\x49\x8e"
+			  "\x2f\x03\x30\x05\x47\xaf\x34",
+		.ilen	= 47,
+		.result	= "\x24\x5e\x67\x49\x1e\x01\xd6\xdd"
+			  "\xf3\x89\x20\x5b\x7c\x57\x89\x07"
+			  "\xd9\x02\x7c\x3d\x2f\x18\x4b\x2d"
+			  "\x6e\xde\xee\xa2\x08\x12\xc7\xba",
+		.rlen	= 32,
+	}, {
+		.key	= "\x33\x27\xf5\xb1\x62\xa0\x80\x63"
+			  "\x14\xc0\x4d\x7f\x7b\x20\xba\x89",
+		.klen	= 16,
+		.iv	= "\x42\xf0\x84\x19\xa6\x3f\x2b\xea"
+			  "\x34\xf6\x79\xa3\x79\xe9\xeb\x0a",
+		.assoc	= "\x51\xb9\x12\x80\xea\xde\xd5\x71"
+			  "\x54\x2d\xa6\xc8\x78\xb2\x1b\x8c"
+			  "\x39\x14\x05\xa0\xf3\x10\xec\x41"
+			  "\xff\x01\x95\x84\x2b\x59\x7f\xdb",
+		.alen	= 32,
+		.input	= "\x97\xca\xf4\xe0\x8d\x89\xbf\x68"
+			  "\x0c\x60\xb9\x27\xdf\xaa\x41\xc6"
+			  "\x25\xd8\xf7\x1f\x10\x15\x48\x61"
+			  "\x4c\x95\x00\xdf\x51\x9b\x7f\xe6"
+			  "\x24\x40\x9e\xbe\x3b\xeb\x1b\x98"
+			  "\xb9\x9c\xe5\xef\xf2\x05",
+		.ilen	= 46,
+		.result	= "\x61\x83\xa0\xe8\x2e\x7d\x7f\xf8"
+			  "\x74\x63\xd2\xec\x76\x7c\x4c\x0d"
+			  "\x03\xc4\x88\xc1\x35\xb8\xcd\x47"
+			  "\x2f\x0c\xcd\x7a\xe2\x71\x66\x91",
+		.rlen	= 32,
+	}, {
+		.key	= "\x70\x4c\x2f\x50\x72\x1c\x29\x7f"
+			  "\x95\x9a\xff\x10\x75\x45\x7d\x8f",
+		.klen	= 16,
+		.iv	= "\x7f\x15\xbd\xb8\xb6\xba\xd3\x06"
+			  "\xb5\xd1\x2b\x35\x73\x0e\xad\x10",
+		.assoc	= "\x8e\xde\x4c\x20\xfa\x59\x7e\x8d"
+			  "\xd5\x07\x58\x59\x72\xd7\xde\x92"
+			  "\x63\xd6\x10\x24\xf8\xb0\x6e\x5a"
+			  "\xc0\x2e\x74\x5d\x06\xb8\x1e\xb2",
+		.alen	= 32,
+		.input	= "\x63\x4c\x2a\x8e\xb4\x6b\x63\x0d"
+			  "\xb5\xec\x9b\x4e\x12\x23\xa3\xcf"
+			  "\x1a\x5a\x70\x15\x5a\x10\x40\x51"
+			  "\xca\x47\x4c\x9d\xc9\x97\xf4\x77"
+			  "\xdb\xc8\x10\x2d\xdc\x65\x20\x3f",
+		.ilen	= 40,
+		.result	= "\x9d\xa7\xda\x88\x3e\xf8\x28\x14"
+			  "\xf5\x3e\x85\x7d\x70\xa0\x0f\x13"
+			  "\x2e\x86\x93\x45\x3a\x58\x4f\x61"
+			  "\xf0\x3a\xac\x53\xbc\xd0\x06\x68",
+		.rlen	= 32,
+	}, {
+		.key	= "\xac\x70\x69\xef\x82\x97\xd2\x9b"
+			  "\x15\x74\xb1\xa2\x6f\x69\x3f\x95",
+		.klen	= 16,
+		.iv	= "\xbb\x3a\xf7\x57\xc6\x36\x7c\x22"
+			  "\x36\xab\xde\xc6\x6d\x32\x70\x17",
+		.assoc	= "\xcb\x03\x85\xbf\x0a\xd5\x26\xa9"
+			  "\x56\xe1\x0a\xeb\x6c\xfb\xa1\x98"
+			  "\x8d\x98\x1c\xa8\xfe\x50\xf0\x74"
+			  "\x81\x5c\x53\x35\xe0\x17\xbd\x88",
+		.alen	= 32,
+		.input	= "\xf1\x62\x44\xc7\x5f\x19\xca\x43"
+			  "\x47\x2c\xaf\x68\x82\xbd\x51\xef"
+			  "\x3d\x65\xd8\x45\x2d\x06\x07\x78"
+			  "\x08\x2e\xb3\x23\xcd\x81\x12\x55"
+			  "\x1a",
+		.ilen	= 33,
+		.result	= "\xda\xcc\x14\x27\x4e\x74\xd1\x30"
+			  "\x76\x18\x37\x0f\x6a\xc4\xd1\x1a"
+			  "\x58\x49\x9f\xc9\x3f\xf8\xd1\x7a"
+			  "\xb2\x67\x8b\x2b\x96\x2f\xa5\x3e",
+		.rlen	= 32,
+	}, {
+		.key	= "\xe9\x95\xa2\x8f\x93\x13\x7b\xb7"
+			  "\x96\x4e\x63\x33\x69\x8d\x02\x9b"
+			  "\x23\xf9\x22\xeb\x80\xa0\xb1\x81"
+			  "\xe2\x73\xc3\x21\x4d\x47\x8d\xf4",
+		.klen	= 32,
+		.iv	= "\xf8\x5e\x31\xf7\xd7\xb2\x25\x3e"
+			  "\xb7\x85\x90\x58\x67\x57\x33\x1d",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xdf\x2f\x83\xc0\x45\x4a\x2c\xcf"
+			  "\xb9\xd2\x41\xf6\x80\xa1\x52\x70",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x25\xba\xdc\x2e\xa3\x8f\x24\xd3"
+			  "\x17\x29\x15\xc5\x63\xb2\xc5\xa1"
+			  "\x4d\xbc\x2d\x6f\x85\x40\x33\x9a"
+			  "\xa3\xa0\xa1\xfa\x27\xa6\x2c\xca",
+		.klen	= 32,
+		.iv	= "\x34\x83\x6a\x96\xe7\x2d\xce\x5a"
+			  "\x38\x5f\x42\xe9\x61\x7b\xf5\x23",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x01\xd8\x55\x3c\xc0\x5a\x4b\xc7"
+			  "\x01\xf4\x08\xe3\x0d\xf7\xf0\x78"
+			  "\x53",
+		.ilen	= 17,
+		.result	= "\x53",
+		.rlen	= 1,
+	}, {
+		.key	= "\x62\xdf\x16\xcd\xb3\x0a\xcc\xef"
+			  "\x98\x03\xc7\x56\x5d\xd6\x87\xa8"
+			  "\x77\x7e\x39\xf3\x8a\xe0\xb5\xb4"
+			  "\x65\xce\x80\xd2\x01\x05\xcb\xa1",
+		.klen	= 32,
+		.iv	= "\x71\xa8\xa4\x35\xf7\xa9\x76\x75"
+			  "\xb8\x39\xf4\x7a\x5b\x9f\xb8\x29",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xc2\x4b\x41\x0f\x2d\xb9\x62\x07"
+			  "\xff\x8e\x74\xf8\xa1\xa6\xd5\x37"
+			  "\xa5\x64\x31\x5c\xca\x73\x9b\x43"
+			  "\xe6\x70\x63\x46\x95\xcb\xf7\xb5"
+			  "\x20\x8c\x75\x7a\x2a\x17\x2f\xa9"
+			  "\xb8\x4d\x11\x42\xd1\xf8\xf1",
+		.ilen	= 47,
+		.result	= "\x8f\x3a\xc1\x05\x7f\xe7\xcb\x83"
+			  "\xf9\xa6\x4d\xc3\x58\x31\x19\x2c"
+			  "\xd7\x90\xc2\x56\x4e\xd8\x57\xc7"
+			  "\xf6\xf0\x27\xb4\x25\x4c\x83",
+		.rlen	= 31,
+	}, {
+		.key	= "\x9e\x03\x4f\x6d\xc3\x86\x75\x0a"
+			  "\x19\xdd\x79\xe8\x57\xfb\x4a\xae"
+			  "\xa2\x40\x45\x77\x90\x80\x37\xce"
+			  "\x26\xfb\x5f\xaa\xdb\x64\x6b\x77",
+		.klen	= 32,
+		.iv	= "\xae\xcc\xde\xd5\x07\x25\x1f\x91"
+			  "\x39\x14\xa6\x0c\x55\xc4\x7b\x30",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xbb\x01\x7c\xd1\x2c\x33\x7b\x37"
+			  "\x0a\xee\xc4\x30\x19\xd7\x3a\x6f"
+			  "\xf8\x2b\x67\xf5\x3b\x84\x87\x2a"
+			  "\xfb\x07\x7a\x82\xb5\xe4\x85\x26"
+			  "\x1e\xa8\xe5\x04\x54\xce\xe5\x5f"
+			  "\xb5\x3f\xc1\xd5\x7f\xbd\xd2\xa6",
+		.ilen	= 48,
+		.result	= "\xcc\x5f\xfb\xa4\x8f\x63\x74\x9f"
+			  "\x7a\x81\xff\x55\x52\x56\xdc\x33"
+			  "\x01\x52\xcd\xdb\x53\x78\xd9\xe1"
+			  "\xb7\x1d\x06\x8d\xff\xab\x22\x98",
+		.rlen	= 32,
+	}, {
+		.key	= "\xdb\x28\x89\x0c\xd3\x01\x1e\x26"
+			  "\x9a\xb7\x2b\x79\x51\x1f\x0d\xb4"
+			  "\xcc\x03\x50\xfc\x95\x20\xb9\xe7"
+			  "\xe8\x29\x3e\x83\xb5\xc3\x0a\x4e",
+		.klen	= 32,
+		.iv	= "\xea\xf1\x18\x74\x17\xa0\xc8\xad"
+			  "\xba\xee\x58\x9d\x4f\xe8\x3d\x36",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xc2\xf4\x40\x55\xf9\x59\xff\x73"
+			  "\x08\xf5\x98\x92\x0c\x7b\x35\x9a"
+			  "\xa8\xf4\x42\x7e\x6f\x93\xca\x22"
+			  "\x23\x06\x1e\xf8\x89\x22\xf4\x46"
+			  "\x7c\x7c\x67\x75\xab\xe5\x75\xaa"
+			  "\x15\xd7\x83\x19\xfd\x31\x59\x5b"
+			  "\x32",
+		.ilen	= 49,
+		.result	= "\x08\x84\x34\x44\x9f\xde\x1c\xbb"
+			  "\xfb\x5b\xb1\xe6\x4c\x7a\x9f\x39"
+			  "\x2c\x14\xd9\x5f\x59\x18\x5b\xfb"
+			  "\x79\x4b\xe5\x65\xd9\x0a\xc1\x6f"
+			  "\x2e",
+		.rlen	= 33,
+	}, {
+		.key	= "\x17\x4d\xc3\xab\xe3\x7d\xc7\x42"
+			  "\x1b\x91\xdd\x0a\x4b\x43\xcf\xba"
+			  "\xf6\xc5\x5c\x80\x9a\xc0\x3b\x01"
+			  "\xa9\x56\x1d\x5b\x8f\x22\xa9\x25",
+		.klen	= 32,
+		.iv	= "\x27\x16\x51\x13\x27\x1c\x71\xc9"
+			  "\x3b\xc8\x0a\x2f\x49\x0c\x00\x3c",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\xc9\x82\x3b\x4b\x87\x84\xa5\xdb"
+			  "\xa0\x8c\xd3\x3e\x7f\x8d\xe8\x28"
+			  "\x2a\xdc\xfa\x01\x84\x87\x9a\x70"
+			  "\x81\x75\x37\x0a\xd2\x75\xa9\xb6"
+			  "\x21\x72\xee\x7e\x65\x95\xe5\xcc"
+			  "\x01\xb7\x39\xa6\x51\x15\xca\xff"
+			  "\x61\xdc\x97\x38\xcc\xf4\xca\xc7"
+			  "\x83\x9b\x05\x11\x72\x60\xf0\xb4"
+			  "\x7e\x06\xab\x0a\xc0\xbb\x59\x23"
+			  "\xaa\x2d\xfc\x4e\x35\x05\x59",
+		.ilen	= 79,
+		.result	= "\x45\xa8\x6e\xe3\xaf\x5a\xc5\xd7"
+			  "\x7c\x35\x63\x77\x46\x9f\x61\x3f"
+			  "\x56\xd7\xe4\xe3\x5e\xb8\xdc\x14"
+			  "\x3a\x79\xc4\x3e\xb3\x69\x61\x46"
+			  "\x3c\xb6\x83\x4e\xb4\x26\xc7\x73"
+			  "\x22\xda\x52\x8b\x7d\x11\x98\xea"
+			  "\x62\xe1\x14\x1e\xdc\xfe\x0f\xad"
+			  "\x20\x76\x5a\xdc\x4e\x71\x13",
+		.rlen	= 63,
+	}, {
+		.key	= "\x54\x71\xfd\x4b\xf3\xf9\x6f\x5e"
+			  "\x9c\x6c\x8f\x9c\x45\x68\x92\xc1"
+			  "\x21\x87\x67\x04\x9f\x60\xbd\x1b"
+			  "\x6a\x84\xfc\x34\x6a\x81\x48\xfb",
+		.klen	= 32,
+		.iv	= "\x63\x3b\x8b\xb3\x37\x98\x1a\xe5"
+			  "\xbc\xa2\xbc\xc0\x43\x31\xc2\x42",
+		.assoc	= "",
+		.alen	= 0,
+		.input	= "\x11\x7c\x7d\xef\xce\x29\x95\xec"
+			  "\x7e\x9f\x42\xa6\x26\x07\xa1\x75"
+			  "\x2f\x4e\x09\x9a\xf6\x6b\xc2\xfa"
+			  "\x0d\xd0\x17\xdc\x25\x1e\x9b\xdc"
+			  "\x5f\x8c\x1c\x60\x15\x4f\x9b\x20"
+			  "\x7b\xff\xcd\x82\x60\x84\xf4\xa5"
+			  "\x20\x9a\x05\x19\x5b\x02\x0a\x72"
+			  "\x43\x11\x26\x58\xcf\xc5\x41\xcf"
+			  "\x13\xcc\xde\x32\x92\xfa\x86\xf2"
+			  "\xaf\x16\xe8\x8f\xca\xb6\xfd\x54",
+		.ilen	= 80,
+		.result	= "\x81\xcd\xa8\x82\xbf\xd6\x6e\xf3"
+			  "\xfd\x0f\x15\x09\x40\xc3\x24\x45"
+			  "\x81\x99\xf0\x67\x63\x58\x5e\x2e"
+			  "\xfb\xa6\xa3\x16\x8d\xc8\x00\x1c"
+			  "\x4b\x62\x87\x7c\x15\x38\xda\x70"
+			  "\x3d\xea\xe7\xf2\x40\xba\xae\x79"
+			  "\x8f\x48\xfc\xbf\x45\x53\x2e\x78"
+			  "\xef\x79\xf0\x1b\x49\xf7\xfd\x9c",
+		.rlen	= 64,
+	}, {
+		.key	= "\x90\x96\x36\xea\x03\x74\x18\x7a"
+			  "\x1d\x46\x42\x2d\x3f\x8c\x54\xc7"
+			  "\x4b\x4a\x73\x89\xa4\x00\x3f\x34"
+			  "\x2c\xb1\xdb\x0c\x44\xe0\xe8\xd2",
+		.klen	= 32,
+		.iv	= "\xa0\x5f\xc5\x52\x47\x13\xc2\x01"
+			  "\x3d\x7c\x6e\x52\x3d\x55\x85\x48",
+		.assoc	= "\xaf",
+		.alen	= 1,
+		.input	= "\x9b\xc5\x3b\x20\x0a\x88\x56\xbe"
+			  "\x69\xdf\xc4\xc4\x02\x46\x3a\xf0",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xcd\xbb\x70\x89\x13\xf0\xc1\x95"
+			  "\x9e\x20\xf4\xbf\x39\xb1\x17\xcd"
+			  "\x76\x0c\x7f\x0d\xa9\xa0\xc1\x4e"
+			  "\xed\xdf\xb9\xe4\x1e\x3f\x87\xa8",
+		.klen	= 32,
+		.iv	= "\xdc\x84\xfe\xf1\x58\x8f\x6b\x1c"
+			  "\xbe\x57\x20\xe3\x37\x7a\x48\x4f",
+		.assoc	= "\xeb\x4d\x8d\x59\x9c\x2e\x15\xa3"
+			  "\xde\x8d\x4d\x07\x36\x43\x78\xd0"
+			  "\x0b\x6d\x84\x4f\x2c\xf0\x82\x5b"
+			  "\x4e\xf6\x29\xd1\x8b\x6f\x56",
+		.alen	= 31,
+		.input	= "\xe0\x6d\xa1\x07\x98\x2f\x40\x2d"
+			  "\x2e\x9a\xd6\x61\x43\xc0\x74\x69",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x0a\xe0\xaa\x29\x24\x6c\x6a\xb1"
+			  "\x1f\xfa\xa6\x50\x33\xd5\xda\xd3"
+			  "\xa0\xce\x8a\x91\xae\x40\x43\x68"
+			  "\xae\x0d\x98\xbd\xf8\x9e\x26\x7f",
+		.klen	= 32,
+		.iv	= "\x19\xa9\x38\x91\x68\x0b\x14\x38"
+			  "\x3f\x31\xd2\x74\x31\x9e\x0a\x55",
+		.assoc	= "\x28\x72\xc7\xf8\xac\xaa\xbe\xbf"
+			  "\x5f\x67\xff\x99\x30\x67\x3b\xd6"
+			  "\x35\x2f\x90\xd3\x31\x90\x04\x74"
+			  "\x0f\x23\x08\xa9\x65\xce\xf6\xea",
+		.alen	= 32,
+		.input	= "\xb9\x57\x13\x3e\x82\x31\x61\x65"
+			  "\x0d\x7f\x6c\x96\x93\x5c\x50\xe2",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x46\x04\xe3\xc8\x34\xe7\x12\xcd"
+			  "\xa0\xd4\x58\xe2\x2d\xf9\x9c\xda"
+			  "\xca\x91\x96\x15\xb4\xe0\xc5\x81"
+			  "\x70\x3a\x77\x95\xd2\xfd\xc5\x55",
+		.klen	= 32,
+		.iv	= "\x55\xcd\x72\x30\x78\x86\xbd\x54"
+			  "\xc0\x0b\x84\x06\x2b\xc2\xcd\x5b",
+		.assoc	= "\x64\x97\x00\x98\xbc\x25\x67\xdb"
+			  "\xe0\x41\xb1\x2a\x2a\x8c\xfe\xdd"
+			  "\x5f\xf2\x9c\x58\x36\x30\x86\x8e"
+			  "\xd1\x51\xe6\x81\x3f\x2d\x95\xc1"
+			  "\x01",
+		.alen	= 33,
+		.input	= "\x81\x96\x34\xde\xbb\x36\xdd\x3e"
+			  "\x4e\x5e\xcb\x44\x21\xb8\x3f\xf1",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\x83\x29\x1d\x67\x44\x63\xbb\xe9"
+			  "\x20\xaf\x0a\x73\x27\x1e\x5f\xe0"
+			  "\xf5\x53\xa1\x9a\xb9\x80\x47\x9b"
+			  "\x31\x68\x56\x6e\xac\x5c\x65\x2c",
+		.klen	= 32,
+		.iv	= "\x92\xf2\xac\xcf\x88\x02\x65\x70"
+			  "\x41\xe5\x36\x97\x25\xe7\x90\x61",
+		.assoc	= "\xa1\xbb\x3a\x37\xcc\xa1\x10\xf7"
+			  "\x61\x1c\x63\xbc\x24\xb0\xc0\xe3"
+			  "\x8a\xb4\xa7\xdc\x3b\xd0\x08\xa8"
+			  "\x92\x7f\xc5\x5a\x19\x8c\x34\x97"
+			  "\x0f\x95\x9b\x18\xe4\x8d\xb4\x24"
+			  "\xb9\x33\x28\x18\xe1\x9d\x14\xe0"
+			  "\x64\xb2\x89\x7d\x78\xa8\x05\x7e"
+			  "\x07\x8c\xfc\x88\x2d\xb8\x53",
+		.alen	= 63,
+		.input	= "\x2e\x99\xb6\x79\x57\x56\x80\x36"
+			  "\x8e\xc4\x1c\x12\x7d\x71\x36\x0c",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xbf\x4e\x57\x07\x54\xdf\x64\x05"
+			  "\xa1\x89\xbc\x04\x21\x42\x22\xe6"
+			  "\x1f\x15\xad\x1e\xbe\x20\xc9\xb4"
+			  "\xf3\x95\x35\x46\x86\xbb\x04\x03",
+		.klen	= 32,
+		.iv	= "\xce\x17\xe5\x6f\x98\x7e\x0e\x8c"
+			  "\xc2\xbf\xe8\x29\x1f\x0b\x52\x68",
+		.assoc	= "\xdd\xe0\x74\xd6\xdc\x1d\xb8\x13"
+			  "\xe2\xf6\x15\x4d\x1e\xd4\x83\xe9"
+			  "\xb4\x76\xb3\x60\x40\x70\x8a\xc1"
+			  "\x53\xac\xa4\x32\xf3\xeb\xd3\x6e"
+			  "\x1e\x42\xa0\x46\x45\x9f\xc7\x22"
+			  "\xd3\x43\xbc\x7e\xa5\x47\x2a\x6f"
+			  "\x91\x19\x70\x1e\xe1\xfe\x25\x49"
+			  "\xd6\x8f\x93\xc7\x28\x3f\x3d\x03",
+		.alen	= 64,
+		.input	= "\x7b\x25\x3d\x47\xd4\xa7\x08\xce"
+			  "\x3b\x89\x40\x36\xba\x6d\x0e\xa2",
+		.ilen	= 16,
+		.result	= "",
+		.rlen	= 0,
+	}, {
+		.key	= "\xfc\x72\x90\xa6\x64\x5a\x0d\x21"
+			  "\x22\x63\x6e\x96\x1b\x67\xe4\xec"
+			  "\x49\xd7\xb9\xa2\xc3\xc0\x4b\xce"
+			  "\xb4\xc3\x14\x1e\x61\x1a\xa3\xd9",
+		.klen	= 32,
+		.iv	= "\x0b\x3c\x1f\x0e\xa8\xf9\xb7\xa7"
+			  "\x42\x9a\x9a\xba\x19\x30\x15\x6e",
+		.assoc	= "\x1a",
+		.alen	= 1,
+		.input	= "\xe6\x09\x6f\x95\x9a\x18\xc8\xf6"
+			  "\x17\x75\x81\x16\xdf\x26\xff\x67"
+			  "\x92",
+		.ilen	= 17,
+		.result	= "\x29",
+		.rlen	= 1,
+	}, {
+		.key	= "\x38\x97\xca\x45\x74\xd6\xb6\x3c"
+			  "\xa3\x3d\x20\x27\x15\x8b\xa7\xf2"
+			  "\x74\x9a\xc4\x27\xc8\x60\xcd\xe8"
+			  "\x75\xf0\xf2\xf7\x3b\x79\x42\xb0",
+		.klen	= 32,
+		.iv	= "\x47\x60\x59\xad\xb8\x75\x60\xc3"
+			  "\xc3\x74\x4c\x4c\x13\x54\xd8\x74",
+		.assoc	= "\x56\x29\xe7\x15\xfc\x14\x0a\x4a"
+			  "\xe4\xaa\x79\x70\x12\x1d\x08\xf6"
+			  "\x09\xfb\xca\x69\x4b\xb0\x8e\xf5"
+			  "\xd6\x07\x62\xe3\xa8\xa9\x12",
+		.alen	= 31,
+		.input	= "\x82\xc0\x56\xf0\xd7\xc4\xc9\xfd"
+			  "\x3c\xd1\x2a\xd4\x15\x86\x9d\xda"
+			  "\xea\x6c\x6f\xa1\x33\xb0\x7a\x01"
+			  "\x57\xe7\xf3\x7b\x73\xe7\x54\x10"
+			  "\xc6\x91\xe2\xc6\xa0\x69\xe7\xe6"
+			  "\x76\xc3\xf5\x3a\x76\xfd\x4a",
+		.ilen	= 47,
+		.result	= "\x66\xf3\x75\x7d\x40\xb3\xb4\xd1"
+			  "\x04\xe1\xa6\x94\x10\xe6\x39\x77"
+			  "\xd3\xac\x4d\x8a\x8c\x58\x6e\xfb"
+			  "\x06\x13\x9a\xd9\x5e\xc0\xfa",
+		.rlen	= 31,
+	}, {
+		.key	= "\x75\xbc\x04\xe5\x84\x52\x5e\x58"
+			  "\x24\x17\xd2\xb9\x0e\xaf\x6a\xf9"
+			  "\x9e\x5c\xd0\xab\xcd\x00\x4f\x01"
+			  "\x37\x1e\xd1\xcf\x15\xd8\xe2\x86",
+		.klen	= 32,
+		.iv	= "\x84\x85\x92\x4d\xc8\xf1\x08\xdf"
+			  "\x44\x4e\xff\xdd\x0d\x78\x9a\x7a",
+		.assoc	= "\x93\x4e\x21\xb4\x0c\x90\xb3\x66"
+			  "\x65\x84\x2b\x01\x0b\x42\xcb\xfc"
+			  "\x33\xbd\xd6\xed\x50\x50\x10\x0e"
+			  "\x97\x35\x41\xbb\x82\x08\xb1\xf2",
+		.alen	= 32,
+		.input	= "\x01\x47\x8e\x6c\xf6\x64\x89\x3a"
+			  "\x71\xce\xe4\xaa\x45\x70\xe6\x84"
+			  "\x62\x48\x08\x64\x86\x6a\xdf\xec"
+			  "\xb4\xa0\xfb\x34\x03\x0c\x19\xf4"
+			  "\x2b\x7b\x36\x73\xec\x54\xa9\x1e"
+			  "\x30\x85\xdb\xe4\xac\xe9\x2c\xca",
+		.ilen	= 48,
+		.result	= "\xa2\x17\xaf\x1c\x50\x2e\x5d\xed"
+			  "\x85\xbb\x58\x26\x0a\x0b\xfc\x7d"
+			  "\xfe\x6e\x59\x0e\x91\xf8\xf0\x15"
+			  "\xc8\x40\x78\xb1\x38\x1f\x99\xa7",
+		.rlen	= 32,
+	}, {
+		.key	= "\xb1\xe1\x3e\x84\x94\xcd\x07\x74"
+			  "\xa5\xf2\x84\x4a\x08\xd4\x2c\xff"
+			  "\xc8\x1e\xdb\x2f\xd2\xa0\xd1\x1b"
+			  "\xf8\x4c\xb0\xa8\xef\x37\x81\x5d",
+		.klen	= 32,
+		.iv	= "\xc0\xaa\xcc\xec\xd8\x6c\xb1\xfb"
+			  "\xc5\x28\xb1\x6e\x07\x9d\x5d\x81",
+		.assoc	= "\xd0\x73\x5a\x54\x1d\x0b\x5b\x82"
+			  "\xe5\x5f\xdd\x93\x05\x66\x8e\x02"
+			  "\x5e\x80\xe1\x71\x55\xf0\x92\x28"
+			  "\x59\x62\x20\x94\x5c\x67\x50\xc8"
+			  "\x58",
+		.alen	= 33,
+		.input	= "\x85\xe0\xf8\x0f\x8e\x49\xe3\x60"
+			  "\xcb\x4a\x54\x94\xcf\xf5\x7e\x34"
+			  "\xe9\xf8\x80\x65\x53\xd0\x72\x70"
+			  "\x4f\x7d\x9d\xd1\x15\x6f\xb9\x2c"
+			  "\xfa\xe8\xdd\xac\x2e\xe1\x3f\x67"
+			  "\x63\x0f\x1a\x59\xb7\x89\xdb\xf4"
+			  "\xc3",
+		.ilen	= 49,
+		.result	= "\xdf\x3c\xe9\xbc\x61\xaa\x06\x09"
+			  "\x06\x95\x0a\xb7\x04\x2f\xbe\x84"
+			  "\x28\x30\x64\x92\x96\x98\x72\x2e"
+			  "\x89\x6e\x57\x8a\x13\x7e\x38\x7e"
+			  "\xdb",
+		.rlen	= 33,
+	}, {
+		.key	= "\xee\x05\x77\x23\xa5\x49\xb0\x90"
+			  "\x26\xcc\x36\xdc\x02\xf8\xef\x05"
+			  "\xf3\xe1\xe7\xb3\xd8\x40\x53\x35"
+			  "\xb9\x79\x8f\x80\xc9\x96\x20\x33",
+		.klen	= 32,
+		.iv	= "\xfd\xce\x06\x8b\xe9\xe8\x5a\x17"
+			  "\x46\x02\x63\x00\x01\xc1\x20\x87",
+		.assoc	= "\x0c\x98\x94\xf3\x2d\x87\x04\x9e"
+			  "\x66\x39\x8f\x24\xff\x8a\x50\x08"
+			  "\x88\x42\xed\xf6\x5a\x90\x14\x42"
+			  "\x1a\x90\xfe\x6c\x36\xc6\xf0\x9f"
+			  "\x66\xa0\xb5\x2d\x2c\xf8\x25\x15"
+			  "\x55\x90\xa2\x7e\x77\x94\x96\x3a"
+			  "\x71\x1c\xf7\x44\xee\xa8\xc3\x42"
+			  "\xe2\xa3\x84\x04\x0b\xe1\xce",
+		.alen	= 63,
+		.input	= "\x00\xe5\x5b\x87\x5c\x20\x22\x8a"
+			  "\xda\x1f\xd3\xff\xbb\xb2\xb0\xf8"
+			  "\xef\xe9\xeb\x9e\x7c\x80\xf4\x2b"
+			  "\x59\xc0\x79\xbc\x17\xa0\x15\x01"
+			  "\xf5\x72\xfb\x5a\xe7\xaf\x07\xe3"
+			  "\x1b\x49\x21\x34\x23\x63\x55\x5e"
+			  "\xee\x4f\x34\x17\xfa\xfe\xa5\x0c"
+			  "\xed\x0b\x23\xea\x9b\xda\x57\x2f"
+			  "\xf6\xa9\xae\x0d\x4e\x40\x96\x45"
+			  "\x7f\xfa\xf0\xbf\xc4\x98\x78",
+		.ilen	= 79,
+		.result	= "\x1b\x61\x23\x5b\x71\x26\xae\x25"
+			  "\x87\x6f\xbc\x49\xfe\x53\x81\x8a"
+			  "\x53\xf2\x70\x17\x9b\x38\xf4\x48"
+			  "\x4b\x9b\x36\x62\xed\xdd\xd8\x54"
+			  "\xea\xcb\xb6\x79\x45\xfc\xaa\x54"
+			  "\x5c\x94\x47\x58\xa7\xff\x9c\x9e"
+			  "\x7c\xb6\xf1\xac\xc8\xfd\x8b\x35"
+			  "\xd5\xa4\x6a\xd4\x09\xc2\x08",
+		.rlen	= 63,
+	}, {
+		.key	= "\x2a\x2a\xb1\xc3\xb5\xc5\x59\xac"
+			  "\xa7\xa6\xe8\x6d\xfc\x1d\xb2\x0b"
+			  "\x1d\xa3\xf3\x38\xdd\xe0\xd5\x4e"
+			  "\x7b\xa7\x6e\x58\xa3\xf5\xbf\x0a",
+		.klen	= 32,
+		.iv	= "\x39\xf3\x3f\x2b\xf9\x64\x03\x33"
+			  "\xc7\xdd\x15\x91\xfb\xe6\xe2\x8d",
+		.assoc	= "\x49\xbc\xce\x92\x3d\x02\xad\xba"
+			  "\xe7\x13\x41\xb6\xf9\xaf\x13\x0f"
+			  "\xb2\x04\xf8\x7a\x5f\x30\x96\x5b"
+			  "\xdc\xbd\xdd\x44\x10\x25\x8f\x75"
+			  "\x75\x4d\xb9\x5b\x8e\x0a\x38\x13"
+			  "\x6f\x9f\x36\xe4\x3a\x3e\xac\xc9"
+			  "\x9d\x83\xde\xe5\x57\xfd\xe3\x0e"
+			  "\xb1\xa7\x1b\x44\x05\x67\xb7\x37",
+		.alen	= 64,
+		.input	= "\x28\xdd\xb9\x4a\x12\xc7\x0a\xe1"
+			  "\x58\x06\x1a\x9b\x8c\x67\xdf\xeb"
+			  "\x35\x35\x60\x9d\x06\x40\x65\xc1"
+			  "\x93\xe8\xb3\x82\x50\x29\xdd\xb5"
+			  "\x2b\xcb\xde\x18\x78\x6b\x42\xbe"
+			  "\x6d\x24\xd0\xb2\x7d\xd7\x08\x8f"
+			  "\x4a\x18\x98\xad\x8c\xf2\x97\xb4"
+			  "\xf4\x77\xe4\xbf\x41\x3b\xc4\x06"
+			  "\xce\x9e\x34\x81\xf0\x89\x11\x13"
+			  "\x02\x65\xa1\x7c\xdf\x07\x33\x06",
+		.ilen	= 80,
+		.result	= "\x58\x85\x5c\xfa\x81\xa1\x57\x40"
+			  "\x08\x4a\x6e\xda\xf8\x78\x44\x90"
+			  "\x7d\xb5\x7b\x9b\xa1\xd8\x76\x62"
+			  "\x0c\xc9\x15\x3b\xc7\x3c\x77\x2b"
+			  "\xf8\x78\xba\xa7\xa6\x0e\xbd\x52"
+			  "\x76\xa3\xdc\xbe\x6b\xa8\xb1\x2d"
+			  "\xa9\x1d\xd8\x4e\x31\x53\xab\x00"
+			  "\xa5\xa7\x01\x13\x04\x49\xf2\x04",
+		.rlen	= 64,
+	}, {
+		.key	= "\x67\x4f\xeb\x62\xc5\x40\x01\xc7"
+			  "\x28\x80\x9a\xfe\xf6\x41\x74\x12"
+			  "\x48\x65\xfe\xbc\xe2\x80\x57\x68"
+			  "\x3c\xd4\x4d\x31\x7d\x54\x5f\xe1",
+		.klen	= 32,
+		.iv	= "\x76\x18\x79\xca\x09\xdf\xac\x4e"
+			  "\x48\xb7\xc7\x23\xf5\x0a\xa5\x93",
+		.assoc	= "\x85\xe1\x08\x32\x4d\x7e\x56\xd5"
+			  "\x68\xed\xf3\x47\xf3\xd3\xd6\x15"
+			  "\xdd\xc7\x04\xfe\x64\xd0\x18\x75"
+			  "\x9d\xeb\xbc\x1d\xea\x84\x2e\x4c"
+			  "\x83\xf9\xbe\x8a\xef\x1c\x4b\x10"
+			  "\x89\xaf\xcb\x4b\xfe\xe7\xc1\x58"
+			  "\xca\xea\xc6\x87\xc0\x53\x03\xd9"
+			  "\x80\xaa\xb2\x83\xff\xee\xa1\x6a"
+			  "\x04",
+		.alen	= 65,
+		.input	= "\x85\x39\x69\x35\xfb\xf9\xb0\xa6"
+			  "\x85\x43\x88\xd0\xd7\x78\x60\x19"
+			  "\x3e\x1f\xb1\xa4\xd6\xc5\x96\xec"
+			  "\xf7\x84\x85\xc7\x27\x0f\x74\x57"
+			  "\x28\x9e\xdd\x90\x3c\x43\x12\xc5"
+			  "\x51\x3d\x39\x8f\xa5\xf4\xe0\x0b"
+			  "\x57\x04\xf1\x6d\xfe\x9b\x84\x27"
+			  "\xe8\xeb\x4d\xda\x02\x0a\xc5\x49"
+			  "\x1a\x55\x5e\x50\x56\x4d\x94\xda"
+			  "\x20\xf8\x12\x54\x50\xb3\x11\xda"
+			  "\xed\x44\x27\x67\xd5\xd1\x8b\x4b"
+			  "\x38\x67\x56\x65\x59\xda\xe6\x97"
+			  "\x81\xae\x2f\x92\x3b\xae\x22\x1c"
+			  "\x91\x59\x38\x18\x00\xe8\xba\x92"
+			  "\x04\x19\x56\xdf\xb0\x82\xeb\x6f"
+			  "\x2e\xdb\x54\x3c\x4b\xbb\x60\x90"
+			  "\x4c\x50\x10\x62\xba\x7a\xb1\x68"
+			  "\x37\xd7\x87\x4e\xe4\x66\x09\x1f"
+			  "\xa5",
+		.ilen	= 145,
+		.result	= "\x94\xaa\x96\x9a\x91\x1d\x00\x5c"
+			  "\x88\x24\x20\x6b\xf2\x9c\x06\x96"
+			  "\xa7\x77\x87\x1f\xa6\x78\xf8\x7b"
+			  "\xcd\xf6\xf4\x13\xa1\x9b\x16\x02"
+			  "\x07\x24\xbf\xd5\x08\x20\xd0\x4f"
+			  "\x90\xb3\x70\x24\x2f\x51\xc7\xbb"
+			  "\xd6\x84\xc0\xef\x9a\xa8\xca\xcc"
+			  "\x74\xab\x97\x53\xfe\xd0\xdb\x37"
+			  "\x37\x6a\x0e\x9f\x3f\xa3\x2a\xe3"
+			  "\x1b\x34\x6d\x51\x72\x2b\x17\xe7"
+			  "\x4d\xaa\x2c\x18\xda\xa3\x33\x89"
+			  "\x2a\x9f\xf4\xd2\xed\x76\x3d\x3f"
+			  "\x3c\x15\x9d\x8e\x4f\x3c\x27\xb0"
+			  "\x42\x3f\x2f\x8a\xd4\xc2\x10\xb2"
+			  "\x27\x7f\xe3\x34\x80\x02\x49\x4b"
+			  "\x07\x68\x22\x2a\x88\x25\x53\xb2"
+			  "\x2f",
+		.rlen	= 129,
+	}, {
+		.key	= "\xa3\x73\x24\x01\xd5\xbc\xaa\xe3"
+			  "\xa9\x5a\x4c\x90\xf0\x65\x37\x18"
+			  "\x72\x28\x0a\x40\xe7\x20\xd9\x82"
+			  "\xfe\x02\x2b\x09\x57\xb3\xfe\xb7",
+		.klen	= 32,
+		.iv	= "\xb3\x3d\xb3\x69\x19\x5b\x54\x6a"
+			  "\xc9\x91\x79\xb4\xef\x2e\x68\x99",
+		.assoc	= "\xc2\x06\x41\xd1\x5d\xfa\xff\xf1"
+			  "\xe9\xc7\xa5\xd9\xed\xf8\x98\x1b"
+			  "\x07\x89\x10\x82\x6a\x70\x9a\x8f"
+			  "\x5e\x19\x9b\xf5\xc5\xe3\xcd\x22"
+			  "\x92\xa5\xc2\xb8\x51\x2e\x5e\x0e"
+			  "\xa4\xbe\x5f\xb1\xc1\x90\xd7\xe7"
+			  "\xf7\x52\xae\x28\x29\xa8\x22\xa4"
+			  "\x4f\xae\x48\xc2\xfa\x75\x8b\x9e"
+			  "\xce\x83\x2a\x88\x07\x55\xbb\x89"
+			  "\xf6\xdf\xac\xdf\x83\x08\xbf\x7d"
+			  "\xac\x30\x8b\x8e\x02\xac\x00\xf1"
+			  "\x30\x46\xe1\xbc\x75\xbf\x49\xbb"
+			  "\x26\x4e\x29\xf0\x2f\x21\xc6\x13"
+			  "\x92\xd9\x3d\x11\xe4\x10\x00\x8e"
+			  "\xd4\xd4\x58\x65\xa6\x2b\xe3\x25"
+			  "\xb1\x8f\x15\x93\xe7\x71\xb9\x2c"
+			  "\x4b",
+		.alen	= 129,
+		.input	= "\x7d\xde\x53\x22\xe4\x23\x3b\x30"
+			  "\x78\xde\x35\x90\x7a\xd9\x0b\x93"
+			  "\xf6\x0e\x0b\xed\x40\xee\x10\x9c"
+			  "\x96\x3a\xd3\x34\xb2\xd0\x67\xcf"
+			  "\x63\x7f\x2d\x0c\xcf\x96\xec\x64"
+			  "\x1a\x87\xcc\x7d\x2c\x5e\x81\x4b"
+			  "\xd2\x8f\x4c\x7c\x00\xb1\xb4\xe0"
+			  "\x87\x4d\xb1\xbc\xd8\x78\x2c\x17"
+			  "\xf2\x3b\xd8\x28\x40\xe2\x76\xf6"
+			  "\x20\x13\x83\x46\xaf\xff\xe3\x0f"
+			  "\x72",
+		.ilen	= 81,
+		.result	= "\xd1\xcf\xd0\x39\xa1\x99\xa9\x78"
+			  "\x09\xfe\xd2\xfd\xec\xc1\xc9\x9d"
+			  "\xd2\x39\x93\xa3\xab\x18\x7a\x95"
+			  "\x8f\x24\xd3\xeb\x7b\xfa\xb5\xd8"
+			  "\x15\xd1\xc3\x04\x69\x32\xe3\x4d"
+			  "\xaa\xc2\x04\x8b\xf2\xfa\xdc\x4a"
+			  "\x02\xeb\xa8\x90\x03\xfd\xea\x97"
+			  "\x43\xaf\x2e\x92\xf8\x57\xc5\x6a"
+			  "\x00",
+		.rlen	= 65,
+	}, {
+		.key	= "\xe0\x98\x5e\xa1\xe5\x38\x53\xff"
+			  "\x2a\x35\xfe\x21\xea\x8a\xfa\x1e"
+			  "\x9c\xea\x15\xc5\xec\xc0\x5b\x9b"
+			  "\xbf\x2f\x0a\xe1\x32\x12\x9d\x8e",
+		.klen	= 32,
+		.iv	= "\xef\x61\xed\x08\x29\xd7\xfd\x86"
+			  "\x4a\x6b\x2b\x46\xe9\x53\x2a\xa0",
+		.assoc	= "\xfe\x2a\x7b\x70\x6d\x75\xa7\x0d"
+			  "\x6a\xa2\x57\x6a\xe7\x1c\x5b\x21"
+			  "\x31\x4b\x1b\x07\x6f\x10\x1c\xa8"
+			  "\x20\x46\x7a\xce\x9f\x42\x6d\xf9",
+		.alen	= 32,
+		.input	= "\x5a\xcd\x8c\x57\xf2\x6a\xb6\xbe"
+			  "\x53\xc7\xaa\x9a\x60\x74\x9c\xc4"
+			  "\xa2\xc2\xd0\x6d\xe1\x03\x63\xdc"
+			  "\xbb\x51\x7e\x9c\x89\x73\xde\x4e"
+			  "\x24\xf8\x52\x7c\x15\x41\x0e\xba"
+			  "\x69\x0e\x36\x5f\x2f\x22\x8c",
+		.ilen	= 47,
+		.result	= "\x0d\xf4\x09\xd8\xb1\x14\x51\x94"
+			  "\x8a\xd8\x84\x8e\xe6\xe5\x8c\xa3"
+			  "\xfc\xfc\x9e\x28\xb0\xb8\xfc\xaf"
+			  "\x50\x52\xb1\xc4\x55\x59\x55\xaf",
+		.rlen	= 32,
+	}, {
+		.key	= "\x1c\xbd\x98\x40\xf5\xb3\xfc\x1b"
+			  "\xaa\x0f\xb0\xb3\xe4\xae\xbc\x24"
+			  "\xc7\xac\x21\x49\xf1\x60\xdd\xb5"
+			  "\x80\x5d\xe9\xba\x0c\x71\x3c\x64",
+		.klen	= 32,
+		.iv	= "\x2c\x86\x26\xa8\x39\x52\xa6\xa2"
+			  "\xcb\x45\xdd\xd7\xe3\x77\xed\xa6",
+		.assoc	= "\x3b\x4f\xb5\x10\x7d\xf1\x50\x29"
+			  "\xeb\x7c\x0a\xfb\xe1\x40\x1e\x27"
+			  "\x5c\x0d\x27\x8b\x74\xb0\x9e\xc2"
+			  "\xe1\x74\x59\xa6\x79\xa1\x0c\xd0",
+		.alen	= 32,
+		.input	= "\x47\xd6\xce\x78\xd6\xbf\x4a\x51"
+			  "\xb8\xda\x92\x3c\xfd\xda\xac\x8e"
+			  "\x8d\x88\xd7\x4d\x90\xe5\xeb\xa1"
+			  "\xab\xd6\x7c\x76\xad\xea\x7d\x76"
+			  "\x53\xee\xb0\xcd\xd0\x02\xbb\x70"
+			  "\x5b\x6f\x7b\xe2\x8c\xe8",
+		.ilen	= 46,
+		.result	= "\x4a\x18\x43\x77\xc1\x90\xfa\xb0"
+			  "\x0b\xb2\x36\x20\xe0\x09\x4e\xa9"
+			  "\x26\xbe\xaa\xac\xb5\x58\x7e\xc8"
+			  "\x11\x7f\x90\x9c\x2f\xb8\xf4\x85",
+		.rlen	= 32,
+	}, {
+		.key	= "\x59\xe1\xd2\xdf\x05\x2f\xa4\x37"
+			  "\x2b\xe9\x63\x44\xde\xd3\x7f\x2b"
+			  "\xf1\x6f\x2d\xcd\xf6\x00\x5f\xcf"
+			  "\x42\x8a\xc8\x92\xe6\xd0\xdc\x3b",
+		.klen	= 32,
+		.iv	= "\x68\xab\x60\x47\x49\xce\x4f\xbe"
+			  "\x4c\x20\x8f\x68\xdd\x9c\xb0\xac",
+		.assoc	= "\x77\x74\xee\xaf\x8d\x6d\xf9\x45"
+			  "\x6c\x56\xbc\x8d\xdb\x65\xe0\x2e"
+			  "\x86\xd0\x32\x0f\x79\x50\x20\xdb"
+			  "\xa2\xa1\x37\x7e\x53\x00\xab\xa6",
+		.alen	= 32,
+		.input	= "\x9f\xa9\x2b\xa4\x8f\x00\x05\x2b"
+			  "\xe7\x68\x81\x51\xbb\xfb\xdf\x60"
+			  "\xbb\xac\xe8\xc1\xdc\x68\xae\x68"
+			  "\x3a\xcd\x7a\x06\x49\xfe\x80\x11"
+			  "\xe6\x61\x99\xe2\xdd\xbe\x2c\xbf",
+		.ilen	= 40,
+		.result	= "\x86\x3d\x7d\x17\xd1\x0c\xa3\xcc"
+			  "\x8c\x8d\xe8\xb1\xda\x2e\x11\xaf"
+			  "\x51\x80\xb5\x30\xba\xf8\x00\xe2"
+			  "\xd3\xad\x6f\x75\x09\x18\x93\x5c",
+		.rlen	= 32,
+	}, {
+		.key	= "\x96\x06\x0b\x7f\x15\xab\x4d\x53"
+			  "\xac\xc3\x15\xd6\xd8\xf7\x42\x31"
+			  "\x1b\x31\x38\x51\xfc\xa0\xe1\xe8"
+			  "\x03\xb8\xa7\x6b\xc0\x2f\x7b\x11",
+		.klen	= 32,
+		.iv	= "\xa5\xcf\x9a\xe6\x59\x4a\xf7\xd9"
+			  "\xcd\xfa\x41\xfa\xd7\xc0\x72\xb2",
+		.assoc	= "\xb4\x99\x28\x4e\x9d\xe8\xa2\x60"
+			  "\xed\x30\x6e\x1e\xd5\x89\xa3\x34"
+			  "\xb1\x92\x3e\x93\x7e\xf0\xa2\xf5"
+			  "\x64\xcf\x16\x57\x2d\x5f\x4a\x7d",
+		.alen	= 32,
+		.input	= "\xe2\x34\xfa\x25\xfd\xfb\x89\x5e"
+			  "\x5b\x4e\x0b\x15\x6e\x39\xfb\x0c"
+			  "\x73\xc7\xd9\x6b\xbe\xce\x9b\x70"
+			  "\xc7\x4f\x96\x16\x03\xfc\xea\xfb"
+			  "\x56",
+		.ilen	= 33,
+		.result	= "\xc3\x62\xb7\xb6\xe2\x87\x4c\xe7"
+			  "\x0d\x67\x9a\x43\xd4\x52\xd4\xb5"
+			  "\x7b\x43\xc1\xb5\xbf\x98\x82\xfc"
+			  "\x94\xda\x4e\x4d\xe4\x77\x32\x32",
+		.rlen	= 32,
+	},
+};
+
+/*
+ * All key wrapping test vectors taken from
+ * http://csrc.nist.gov/groups/STM/cavp/documents/mac/kwtestvectors.zip
+ *
+ * Note: as documented in keywrap.c, the ivout for encryption is the first
+ * semiblock of the ciphertext from the test vector. For decryption, iv is
+ * the first semiblock of the ciphertext.
+ */
+static const struct cipher_testvec aes_kw_tv_template[] = {
+	{
+		.key	= "\x75\x75\xda\x3a\x93\x60\x7c\xc2"
+			  "\xbf\xd8\xce\xc7\xaa\xdf\xd9\xa6",
+		.klen	= 16,
+		.ptext	= "\x42\x13\x6d\x3c\x38\x4a\x3e\xea"
+			  "\xc9\x5a\x06\x6f\xd2\x8f\xed\x3f",
+		.ctext	= "\xf6\x85\x94\x81\x6f\x64\xca\xa3"
+			  "\xf5\x6f\xab\xea\x25\x48\xf5\xfb",
+		.len	= 16,
+		.iv	= "\x03\x1f\x6b\xd7\xe6\x1e\x64\x3d",
+		.generates_iv = true,
+	}, {
+		.key	= "\x80\xaa\x99\x73\x27\xa4\x80\x6b"
+			  "\x6a\x7a\x41\xa5\x2b\x86\xc3\x71"
+			  "\x03\x86\xf9\x32\x78\x6e\xf7\x96"
+			  "\x76\xfa\xfb\x90\xb8\x26\x3c\x5f",
+		.klen	= 32,
+		.ptext	= "\x0a\x25\x6b\xa7\x5c\xfa\x03\xaa"
+			  "\xa0\x2b\xa9\x42\x03\xf1\x5b\xaa",
+		.ctext	= "\xd3\x3d\x3d\x97\x7b\xf0\xa9\x15"
+			  "\x59\xf9\x9c\x8a\xcd\x29\x3d\x43",
+		.len	= 16,
+		.iv	= "\x42\x3c\x96\x0d\x8a\x2a\xc4\xc1",
+		.generates_iv = true,
+	},
+};
+
+/*
+ * ANSI X9.31 Continuous Pseudo-Random Number Generator (AES mode)
+ * test vectors, taken from Appendix B.2.9 and B.2.10:
+ *     http://csrc.nist.gov/groups/STM/cavp/documents/rng/RNGVS.pdf
+ * Only AES-128 is supported at this time.
+ */
+static const struct cprng_testvec ansi_cprng_aes_tv_template[] = {
+	{
+		.key	= "\xf3\xb1\x66\x6d\x13\x60\x72\x42"
+			  "\xed\x06\x1c\xab\xb8\xd4\x62\x02",
+		.klen	= 16,
+		.dt	= "\xe6\xb3\xbe\x78\x2a\x23\xfa\x62"
+			  "\xd7\x1d\x4a\xfb\xb0\xe9\x22\xf9",
+		.dtlen	= 16,
+		.v	= "\x80\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.vlen	= 16,
+		.result	= "\x59\x53\x1e\xd1\x3b\xb0\xc0\x55"
+			  "\x84\x79\x66\x85\xc1\x2f\x76\x41",
+		.rlen	= 16,
+		.loops	= 1,
+	}, {
+		.key	= "\xf3\xb1\x66\x6d\x13\x60\x72\x42"
+			  "\xed\x06\x1c\xab\xb8\xd4\x62\x02",
+		.klen	= 16,
+		.dt	= "\xe6\xb3\xbe\x78\x2a\x23\xfa\x62"
+			  "\xd7\x1d\x4a\xfb\xb0\xe9\x22\xfa",
+		.dtlen	= 16,
+		.v	= "\xc0\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.vlen	= 16,
+		.result	= "\x7c\x22\x2c\xf4\xca\x8f\xa2\x4c"
+			  "\x1c\x9c\xb6\x41\xa9\xf3\x22\x0d",
+		.rlen	= 16,
+		.loops	= 1,
+	}, {
+		.key	= "\xf3\xb1\x66\x6d\x13\x60\x72\x42"
+			  "\xed\x06\x1c\xab\xb8\xd4\x62\x02",
+		.klen	= 16,
+		.dt	= "\xe6\xb3\xbe\x78\x2a\x23\xfa\x62"
+			  "\xd7\x1d\x4a\xfb\xb0\xe9\x22\xfb",
+		.dtlen	= 16,
+		.v	= "\xe0\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.vlen	= 16,
+		.result	= "\x8a\xaa\x00\x39\x66\x67\x5b\xe5"
+			  "\x29\x14\x28\x81\xa9\x4d\x4e\xc7",
+		.rlen	= 16,
+		.loops	= 1,
+	}, {
+		.key	= "\xf3\xb1\x66\x6d\x13\x60\x72\x42"
+			  "\xed\x06\x1c\xab\xb8\xd4\x62\x02",
+		.klen	= 16,
+		.dt	= "\xe6\xb3\xbe\x78\x2a\x23\xfa\x62"
+			  "\xd7\x1d\x4a\xfb\xb0\xe9\x22\xfc",
+		.dtlen	= 16,
+		.v	= "\xf0\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.vlen	= 16,
+		.result	= "\x88\xdd\xa4\x56\x30\x24\x23\xe5"
+			  "\xf6\x9d\xa5\x7e\x7b\x95\xc7\x3a",
+		.rlen	= 16,
+		.loops	= 1,
+	}, {
+		.key	= "\xf3\xb1\x66\x6d\x13\x60\x72\x42"
+			  "\xed\x06\x1c\xab\xb8\xd4\x62\x02",
+		.klen	= 16,
+		.dt	= "\xe6\xb3\xbe\x78\x2a\x23\xfa\x62"
+			  "\xd7\x1d\x4a\xfb\xb0\xe9\x22\xfd",
+		.dtlen	= 16,
+		.v	= "\xf8\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.vlen	= 16,
+		.result	= "\x05\x25\x92\x46\x61\x79\xd2\xcb"
+			  "\x78\xc4\x0b\x14\x0a\x5a\x9a\xc8",
+		.rlen	= 16,
+		.loops	= 1,
+	}, {	/* Monte Carlo Test */
+		.key	= "\x9f\x5b\x51\x20\x0b\xf3\x34\xb5"
+			  "\xd8\x2b\xe8\xc3\x72\x55\xc8\x48",
+		.klen	= 16,
+		.dt	= "\x63\x76\xbb\xe5\x29\x02\xba\x3b"
+			  "\x67\xc9\x25\xfa\x70\x1f\x11\xac",
+		.dtlen	= 16,
+		.v	= "\x57\x2c\x8e\x76\x87\x26\x47\x97"
+			  "\x7e\x74\xfb\xdd\xc4\x95\x01\xd1",
+		.vlen	= 16,
+		.result	= "\x48\xe9\xbd\x0d\x06\xee\x18\xfb"
+			  "\xe4\x57\x90\xd5\xc3\xfc\x9b\x73",
+		.rlen	= 16,
+		.loops	= 10000,
+	},
+};
+
+/*
+ * SP800-90A DRBG Test vectors from
+ * http://csrc.nist.gov/groups/STM/cavp/documents/drbg/drbgtestvectors.zip
+ *
+ * Test vectors for DRBG with prediction resistance. All types of DRBGs
+ * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and
+ * w/o personalization string, w/ and w/o additional input string).
+ */
+static const struct drbg_testvec drbg_pr_sha256_tv_template[] = {
+	{
+		.entropy = (unsigned char *)
+			"\x72\x88\x4c\xcd\x6c\x85\x57\x70\xf7\x0b\x8b\x86"
+			"\xc1\xeb\xd2\x4e\x36\x14\xab\x18\xc4\x9c\xc9\xcf"
+			"\x1a\xe8\xf7\x7b\x02\x49\x73\xd7\xf1\x42\x7d\xc6"
+			"\x3f\x29\x2d\xec\xd3\x66\x51\x3f\x1d\x8d\x5b\x4e",
+		.entropylen = 48,
+		.entpra = (unsigned char *)
+			"\x38\x9c\x91\xfa\xc2\xa3\x46\x89\x56\x08\x3f\x62"
+			"\x73\xd5\x22\xa9\x29\x63\x3a\x1d\xe5\x5d\x5e\x4f"
+			"\x67\xb0\x67\x7a\x5e\x9e\x0c\x62",
+		.entprb = (unsigned char *)
+			"\xb2\x8f\x36\xb2\xf6\x8d\x39\x13\xfa\x6c\x66\xcf"
+			"\x62\x8a\x7e\x8c\x12\x33\x71\x9c\x69\xe4\xa5\xf0"
+			"\x8c\xee\xeb\x9c\xf5\x31\x98\x31",
+		.entprlen = 32,
+		.expected = (unsigned char *)
+			"\x52\x7b\xa3\xad\x71\x77\xa4\x49\x42\x04\x61\xc7"
+			"\xf0\xaf\xa5\xfd\xd3\xb3\x0d\x6a\x61\xba\x35\x49"
+			"\xbb\xaa\xaf\xe4\x25\x7d\xb5\x48\xaf\x5c\x18\x3d"
+			"\x33\x8d\x9d\x45\xdf\x98\xd5\x94\xa8\xda\x92\xfe"
+			"\xc4\x3c\x94\x2a\xcf\x7f\x7b\xf2\xeb\x28\xa9\xf1"
+			"\xe0\x86\x30\xa8\xfe\xf2\x48\x90\x91\x0c\x75\xb5"
+			"\x3c\x00\xf0\x4d\x09\x4f\x40\xa7\xa2\x8c\x52\xdf"
+			"\x52\xef\x17\xbf\x3d\xd1\xa2\x31\xb4\xb8\xdc\xe6"
+			"\x5b\x0d\x1f\x78\x36\xb4\xe6\x4b\xa7\x11\x25\xd5"
+			"\x94\xc6\x97\x36\xab\xf0\xe5\x31\x28\x6a\xbb\xce"
+			"\x30\x81\xa6\x8f\x27\x14\xf8\x1c",
+		.expectedlen = 128,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\x5d\xf2\x14\xbc\xf6\xb5\x4e\x0b\xf0\x0d\x6f\x2d"
+			"\xe2\x01\x66\x7b\xd0\xa4\x73\xa4\x21\xdd\xb0\xc0"
+			"\x51\x79\x09\xf4\xea\xa9\x08\xfa\xa6\x67\xe0\xe1"
+			"\xd1\x88\xa8\xad\xee\x69\x74\xb3\x55\x06\x9b\xf6",
+		.entropylen = 48,
+		.entpra = (unsigned char *)
+			"\xef\x48\x06\xa2\xc2\x45\xf1\x44\xfa\x34\x2c\xeb"
+			"\x8d\x78\x3c\x09\x8f\x34\x72\x20\xf2\xe7\xfd\x13"
+			"\x76\x0a\xf6\xdc\x3c\xf5\xc0\x15",
+		.entprb = (unsigned char *)
+			"\x4b\xbe\xe5\x24\xed\x6a\x2d\x0c\xdb\x73\x5e\x09"
+			"\xf9\xad\x67\x7c\x51\x47\x8b\x6b\x30\x2a\xc6\xde"
+			"\x76\xaa\x55\x04\x8b\x0a\x72\x95",
+		.entprlen = 32,
+		.expected = (unsigned char *)
+			"\x3b\x14\x71\x99\xa1\xda\xa0\x42\xe6\xc8\x85\x32"
+			"\x70\x20\x32\x53\x9a\xbe\xd1\x1e\x15\xef\xfb\x4c"
+			"\x25\x6e\x19\x3a\xf0\xb9\xcb\xde\xf0\x3b\xc6\x18"
+			"\x4d\x85\x5a\x9b\xf1\xe3\xc2\x23\x03\x93\x08\xdb"
+			"\xa7\x07\x4b\x33\x78\x40\x4d\xeb\x24\xf5\x6e\x81"
+			"\x4a\x1b\x6e\xa3\x94\x52\x43\xb0\xaf\x2e\x21\xf4"
+			"\x42\x46\x8e\x90\xed\x34\x21\x75\xea\xda\x67\xb6"
+			"\xe4\xf6\xff\xc6\x31\x6c\x9a\x5a\xdb\xb3\x97\x13"
+			"\x09\xd3\x20\x98\x33\x2d\x6d\xd7\xb5\x6a\xa8\xa9"
+			"\x9a\x5b\xd6\x87\x52\xa1\x89\x2b\x4b\x9c\x64\x60"
+			"\x50\x47\xa3\x63\x81\x16\xaf\x19",
+		.expectedlen = 128,
+		.addtla = (unsigned char *)
+			"\xbe\x13\xdb\x2a\xe9\xa8\xfe\x09\x97\xe1\xce\x5d"
+			"\xe8\xbb\xc0\x7c\x4f\xcb\x62\x19\x3f\x0f\xd2\xad"
+			"\xa9\xd0\x1d\x59\x02\xc4\xff\x70",
+		.addtlb = (unsigned char *)
+			"\x6f\x96\x13\xe2\xa7\xf5\x6c\xfe\xdf\x66\xe3\x31"
+			"\x63\x76\xbf\x20\x27\x06\x49\xf1\xf3\x01\x77\x41"
+			"\x9f\xeb\xe4\x38\xfe\x67\x00\xcd",
+		.addtllen = 32,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\xc6\x1c\xaf\x83\xa2\x56\x38\xf9\xb0\xbc\xd9\x85"
+			"\xf5\x2e\xc4\x46\x9c\xe1\xb9\x40\x98\x70\x10\x72"
+			"\xd7\x7d\x15\x85\xa1\x83\x5a\x97\xdf\xc8\xa8\xe8"
+			"\x03\x4c\xcb\x70\x35\x8b\x90\x94\x46\x8a\x6e\xa1",
+		.entropylen = 48,
+		.entpra = (unsigned char *)
+			"\xc9\x05\xa4\xcf\x28\x80\x4b\x93\x0f\x8b\xc6\xf9"
+			"\x09\x41\x58\x74\xe9\xec\x28\xc7\x53\x0a\x73\x60"
+			"\xba\x0a\xde\x57\x5b\x4b\x9f\x29",
+		.entprb = (unsigned char *)
+			"\x4f\x31\xd2\xeb\xac\xfa\xa8\xe2\x01\x7d\xf3\xbd"
+			"\x42\xbd\x20\xa0\x30\x65\x74\xd5\x5d\xd2\xad\xa4"
+			"\xa9\xeb\x1f\x4d\xf6\xfd\xb8\x26",
+		.entprlen = 32,
+		.expected = (unsigned char *)
+			"\xf6\x13\x05\xcb\x83\x60\x16\x42\x49\x1d\xc6\x25"
+			"\x3b\x8c\x31\xa3\xbe\x8b\xbd\x1c\xe2\xec\x1d\xde"
+			"\xbb\xbf\xa1\xac\xa8\x9f\x50\xce\x69\xce\xef\xd5"
+			"\xd6\xf2\xef\x6a\xf7\x81\x38\xdf\xbc\xa7\x5a\xb9"
+			"\xb2\x42\x65\xab\xe4\x86\x8d\x2d\x9d\x59\x99\x2c"
+			"\x5a\x0d\x71\x55\x98\xa4\x45\xc2\x8d\xdb\x05\x5e"
+			"\x50\x21\xf7\xcd\xe8\x98\x43\xce\x57\x74\x63\x4c"
+			"\xf3\xb1\xa5\x14\x1e\x9e\x01\xeb\x54\xd9\x56\xae"
+			"\xbd\xb6\x6f\x1a\x47\x6b\x3b\x44\xe4\xa2\xe9\x3c"
+			"\x6c\x83\x12\x30\xb8\x78\x7f\x8e\x54\x82\xd4\xfe"
+			"\x90\x35\x0d\x4c\x4d\x85\xe7\x13",
+		.expectedlen = 128,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = (unsigned char *)
+			"\xa5\xbf\xac\x4f\x71\xa1\xbb\x67\x94\xc6\x50\xc7"
+			"\x2a\x45\x9e\x10\xa8\xed\xf7\x52\x4f\xfe\x21\x90"
+			"\xa4\x1b\xe1\xe2\x53\xcc\x61\x47",
+		.perslen = 32,
+	}, {
+		.entropy = (unsigned char *)
+			"\xb6\xc1\x8d\xdf\x99\x54\xbe\x95\x10\x48\xd9\xf6"
+			"\xd7\x48\xa8\x73\x2d\x74\xde\x1e\xde\x57\x7e\xf4"
+			"\x7b\x7b\x64\xef\x88\x7a\xa8\x10\x4b\xe1\xc1\x87"
+			"\xbb\x0b\xe1\x39\x39\x50\xaf\x68\x9c\xa2\xbf\x5e",
+		.entropylen = 48,
+		.entpra = (unsigned char *)
+			"\xdc\x81\x0a\x01\x58\xa7\x2e\xce\xee\x48\x8c\x7c"
+			"\x77\x9e\x3c\xf1\x17\x24\x7a\xbb\xab\x9f\xca\x12"
+			"\x19\xaf\x97\x2d\x5f\xf9\xff\xfc",
+		.entprb = (unsigned char *)
+			"\xaf\xfc\x4f\x98\x8b\x93\x95\xc1\xb5\x8b\x7f\x73"
+			"\x6d\xa6\xbe\x6d\x33\xeb\x2c\x82\xb1\xaf\xc1\xb6"
+			"\xb6\x05\xe2\x44\xaa\xfd\xe7\xdb",
+		.entprlen = 32,
+		.expected = (unsigned char *)
+			"\x51\x79\xde\x1c\x0f\x58\xf3\xf4\xc9\x57\x2e\x31"
+			"\xa7\x09\xa1\x53\x64\x63\xa2\xc5\x1d\x84\x88\x65"
+			"\x01\x1b\xc6\x16\x3c\x49\x5b\x42\x8e\x53\xf5\x18"
+			"\xad\x94\x12\x0d\x4f\x55\xcc\x45\x5c\x98\x0f\x42"
+			"\x28\x2f\x47\x11\xf9\xc4\x01\x97\x6b\xa0\x94\x50"
+			"\xa9\xd1\x5e\x06\x54\x3f\xdf\xbb\xc4\x98\xee\x8b"
+			"\xba\xa9\xfa\x49\xee\x1d\xdc\xfb\x50\xf6\x51\x9f"
+			"\x6c\x4a\x9a\x6f\x63\xa2\x7d\xad\xaf\x3a\x24\xa0"
+			"\xd9\x9f\x07\xeb\x15\xee\x26\xe0\xd5\x63\x39\xda"
+			"\x3c\x59\xd6\x33\x6c\x02\xe8\x05\x71\x46\x68\x44"
+			"\x63\x4a\x68\x72\xe9\xf5\x55\xfe",
+		.expectedlen = 128,
+		.addtla = (unsigned char *)
+			"\x15\x20\x2f\xf6\x98\x28\x63\xa2\xc4\x4e\xbb\x6c"
+			"\xb2\x25\x92\x61\x79\xc9\x22\xc4\x61\x54\x96\xff"
+			"\x4a\x85\xca\x80\xfe\x0d\x1c\xd0",
+		.addtlb = (unsigned char *)
+			"\xde\x29\x8e\x03\x42\x61\xa3\x28\x5e\xc8\x80\xc2"
+			"\x6d\xbf\xad\x13\xe1\x8d\x2a\xc7\xe8\xc7\x18\x89"
+			"\x42\x58\x9e\xd6\xcc\xad\x7b\x1e",
+		.addtllen = 32,
+		.pers = (unsigned char *)
+			"\x84\xc3\x73\x9e\xce\xb3\xbc\x89\xf7\x62\xb3\xe1"
+			"\xd7\x48\x45\x8a\xa9\xcc\xe9\xed\xd5\x81\x84\x52"
+			"\x82\x4c\xdc\x19\xb8\xf8\x92\x5c",
+		.perslen = 32,
+	},
+};
+
+static const struct drbg_testvec drbg_pr_hmac_sha256_tv_template[] = {
+	{
+		.entropy = (unsigned char *)
+			"\x99\x69\xe5\x4b\x47\x03\xff\x31\x78\x5b\x87\x9a"
+			"\x7e\x5c\x0e\xae\x0d\x3e\x30\x95\x59\xe9\xfe\x96"
+			"\xb0\x67\x6d\x49\xd5\x91\xea\x4d\x07\xd2\x0d\x46"
+			"\xd0\x64\x75\x7d\x30\x23\xca\xc2\x37\x61\x27\xab",
+		.entropylen = 48,
+		.entpra = (unsigned char *)
+			"\xc6\x0f\x29\x99\x10\x0f\x73\x8c\x10\xf7\x47\x92"
+			"\x67\x6a\x3f\xc4\xa2\x62\xd1\x37\x21\x79\x80\x46"
+			"\xe2\x9a\x29\x51\x81\x56\x9f\x54",
+		.entprb = (unsigned char *)
+			"\xc1\x1d\x45\x24\xc9\x07\x1b\xd3\x09\x60\x15\xfc"
+			"\xf7\xbc\x24\xa6\x07\xf2\x2f\xa0\x65\xc9\x37\x65"
+			"\x8a\x2a\x77\xa8\x69\x90\x89\xf4",
+		.entprlen = 32,
+		.expected = (unsigned char *)
+			"\xab\xc0\x15\x85\x60\x94\x80\x3a\x93\x8d\xff\xd2"
+			"\x0d\xa9\x48\x43\x87\x0e\xf9\x35\xb8\x2c\xfe\xc1"
+			"\x77\x06\xb8\xf5\x51\xb8\x38\x50\x44\x23\x5d\xd4"
+			"\x4b\x59\x9f\x94\xb3\x9b\xe7\x8d\xd4\x76\xe0\xcf"
+			"\x11\x30\x9c\x99\x5a\x73\x34\xe0\xa7\x8b\x37\xbc"
+			"\x95\x86\x23\x50\x86\xfa\x3b\x63\x7b\xa9\x1c\xf8"
+			"\xfb\x65\xef\xa2\x2a\x58\x9c\x13\x75\x31\xaa\x7b"
+			"\x2d\x4e\x26\x07\xaa\xc2\x72\x92\xb0\x1c\x69\x8e"
+			"\x6e\x01\xae\x67\x9e\xb8\x7c\x01\xa8\x9c\x74\x22"
+			"\xd4\x37\x2d\x6d\x75\x4a\xba\xbb\x4b\xf8\x96\xfc"
+			"\xb1\xcd\x09\xd6\x92\xd0\x28\x3f",
+		.expectedlen = 128,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\xb9\x1f\xe9\xef\xdd\x9b\x7d\x20\xb6\xec\xe0\x2f"
+			"\xdb\x76\x24\xce\x41\xc8\x3a\x4a\x12\x7f\x3e\x2f"
+			"\xae\x05\x99\xea\xb5\x06\x71\x0d\x0c\x4c\xb4\x05"
+			"\x26\xc6\xbd\xf5\x7f\x2a\x3d\xf2\xb5\x49\x7b\xda",
+		.entropylen = 48,
+		.entpra = (unsigned char *)
+			"\xef\x67\x50\x9c\xa7\x7d\xdf\xb7\x2d\x81\x01\xa4"
+			"\x62\x81\x6a\x69\x5b\xb3\x37\x45\xa7\x34\x8e\x26"
+			"\x46\xd9\x26\xa2\x19\xd4\x94\x43",
+		.entprb = (unsigned char *)
+			"\x97\x75\x53\x53\xba\xb4\xa6\xb2\x91\x60\x71\x79"
+			"\xd1\x6b\x4a\x24\x9a\x34\x66\xcc\x33\xab\x07\x98"
+			"\x51\x78\x72\xb2\x79\xfd\x2c\xff",
+		.entprlen = 32,
+		.expected = (unsigned char *)
+			"\x9c\xdc\x63\x8a\x19\x23\x22\x66\x0c\xc5\xb9\xd7"
+			"\xfb\x2a\xb0\x31\xe3\x8a\x36\xa8\x5a\xa8\x14\xda"
+			"\x1e\xa9\xcc\xfe\xb8\x26\x44\x83\x9f\xf6\xff\xaa"
+			"\xc8\x98\xb8\x30\x35\x3b\x3d\x36\xd2\x49\xd4\x40"
+			"\x62\x0a\x65\x10\x76\x55\xef\xc0\x95\x9c\xa7\xda"
+			"\x3f\xcf\xb7\x7b\xc6\xe1\x28\x52\xfc\x0c\xe2\x37"
+			"\x0d\x83\xa7\x51\x4b\x31\x47\x3c\xe1\x3c\xae\x70"
+			"\x01\xc8\xa3\xd3\xc2\xac\x77\x9c\xd1\x68\x77\x9b"
+			"\x58\x27\x3b\xa5\x0f\xc2\x7a\x8b\x04\x65\x62\xd5"
+			"\xe8\xd6\xfe\x2a\xaf\xd3\xd3\xfe\xbd\x18\xfb\xcd"
+			"\xcd\x66\xb5\x01\x69\x66\xa0\x3c",
+		.expectedlen = 128,
+		.addtla = (unsigned char *)
+			"\x17\xc1\x56\xcb\xcc\x50\xd6\x03\x7d\x45\x76\xa3"
+			"\x75\x76\xc1\x4a\x66\x1b\x2e\xdf\xb0\x2e\x7d\x56"
+			"\x6d\x99\x3b\xc6\x58\xda\x03\xf6",
+		.addtlb = (unsigned char *)
+			"\x7c\x7b\x4a\x4b\x32\x5e\x6f\x67\x34\xf5\x21\x4c"
+			"\xf9\x96\xf9\xbf\x1c\x8c\x81\xd3\x9b\x60\x6a\x44"
+			"\xc6\x03\xa2\xfb\x13\x20\x19\xb7",
+		.addtllen = 32,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\x13\x54\x96\xfc\x1b\x7d\x28\xf3\x18\xc9\xa7\x89"
+			"\xb6\xb3\xc8\x72\xac\x00\xd4\x59\x36\x25\x05\xaf"
+			"\xa5\xdb\x96\xcb\x3c\x58\x46\x87\xa5\xaa\xbf\x20"
+			"\x3b\xfe\x23\x0e\xd1\xc7\x41\x0f\x3f\xc9\xb3\x67",
+		.entropylen = 48,
+		.entpra = (unsigned char *)
+			"\xe2\xbd\xb7\x48\x08\x06\xf3\xe1\x93\x3c\xac\x79"
+			"\xa7\x2b\x11\xda\xe3\x2e\xe1\x91\xa5\x02\x19\x57"
+			"\x20\x28\xad\xf2\x60\xd7\xcd\x45",
+		.entprb = (unsigned char *)
+			"\x8b\xd4\x69\xfc\xff\x59\x95\x95\xc6\x51\xde\x71"
+			"\x68\x5f\xfc\xf9\x4a\xab\xec\x5a\xcb\xbe\xd3\x66"
+			"\x1f\xfa\x74\xd3\xac\xa6\x74\x60",
+		.entprlen = 32,
+		.expected = (unsigned char *)
+			"\x1f\x9e\xaf\xe4\xd2\x46\xb7\x47\x41\x4c\x65\x99"
+			"\x01\xe9\x3b\xbb\x83\x0c\x0a\xb0\xc1\x3a\xe2\xb3"
+			"\x31\x4e\xeb\x93\x73\xee\x0b\x26\xc2\x63\xa5\x75"
+			"\x45\x99\xd4\x5c\x9f\xa1\xd4\x45\x87\x6b\x20\x61"
+			"\x40\xea\x78\xa5\x32\xdf\x9e\x66\x17\xaf\xb1\x88"
+			"\x9e\x2e\x23\xdd\xc1\xda\x13\x97\x88\xa5\xb6\x5e"
+			"\x90\x14\x4e\xef\x13\xab\x5c\xd9\x2c\x97\x9e\x7c"
+			"\xd7\xf8\xce\xea\x81\xf5\xcd\x71\x15\x49\x44\xce"
+			"\x83\xb6\x05\xfb\x7d\x30\xb5\x57\x2c\x31\x4f\xfc"
+			"\xfe\x80\xb6\xc0\x13\x0c\x5b\x9b\x2e\x8f\x3d\xfc"
+			"\xc2\xa3\x0c\x11\x1b\x80\x5f\xf3",
+		.expectedlen = 128,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = (unsigned char *)
+			"\x64\xb6\xfc\x60\xbc\x61\x76\x23\x6d\x3f\x4a\x0f"
+			"\xe1\xb4\xd5\x20\x9e\x70\xdd\x03\x53\x6d\xbf\xce"
+			"\xcd\x56\x80\xbc\xb8\x15\xc8\xaa",
+		.perslen = 32,
+	}, {
+		.entropy = (unsigned char *)
+			"\xc7\xcc\xbc\x67\x7e\x21\x66\x1e\x27\x2b\x63\xdd"
+			"\x3a\x78\xdc\xdf\x66\x6d\x3f\x24\xae\xcf\x37\x01"
+			"\xa9\x0d\x89\x8a\xa7\xdc\x81\x58\xae\xb2\x10\x15"
+			"\x7e\x18\x44\x6d\x13\xea\xdf\x37\x85\xfe\x81\xfb",
+		.entropylen = 48,
+		.entpra = (unsigned char *)
+			"\x7b\xa1\x91\x5b\x3c\x04\xc4\x1b\x1d\x19\x2f\x1a"
+			"\x18\x81\x60\x3c\x6c\x62\x91\xb7\xe9\xf5\xcb\x96"
+			"\xbb\x81\x6a\xcc\xb5\xae\x55\xb6",
+		.entprb = (unsigned char *)
+			"\x99\x2c\xc7\x78\x7e\x3b\x88\x12\xef\xbe\xd3\xd2"
+			"\x7d\x2a\xa5\x86\xda\x8d\x58\x73\x4a\x0a\xb2\x2e"
+			"\xbb\x4c\x7e\xe3\x9a\xb6\x81\xc1",
+		.entprlen = 32,
+		.expected = (unsigned char *)
+			"\x95\x6f\x95\xfc\x3b\xb7\xfe\x3e\xd0\x4e\x1a\x14"
+			"\x6c\x34\x7f\x7b\x1d\x0d\x63\x5e\x48\x9c\x69\xe6"
+			"\x46\x07\xd2\x87\xf3\x86\x52\x3d\x98\x27\x5e\xd7"
+			"\x54\xe7\x75\x50\x4f\xfb\x4d\xfd\xac\x2f\x4b\x77"
+			"\xcf\x9e\x8e\xcc\x16\xa2\x24\xcd\x53\xde\x3e\xc5"
+			"\x55\x5d\xd5\x26\x3f\x89\xdf\xca\x8b\x4e\x1e\xb6"
+			"\x88\x78\x63\x5c\xa2\x63\x98\x4e\x6f\x25\x59\xb1"
+			"\x5f\x2b\x23\xb0\x4b\xa5\x18\x5d\xc2\x15\x74\x40"
+			"\x59\x4c\xb4\x1e\xcf\x9a\x36\xfd\x43\xe2\x03\xb8"
+			"\x59\x91\x30\x89\x2a\xc8\x5a\x43\x23\x7c\x73\x72"
+			"\xda\x3f\xad\x2b\xba\x00\x6b\xd1",
+		.expectedlen = 128,
+		.addtla = (unsigned char *)
+			"\x18\xe8\x17\xff\xef\x39\xc7\x41\x5c\x73\x03\x03"
+			"\xf6\x3d\xe8\x5f\xc8\xab\xe4\xab\x0f\xad\xe8\xd6"
+			"\x86\x88\x55\x28\xc1\x69\xdd\x76",
+		.addtlb = (unsigned char *)
+			"\xac\x07\xfc\xbe\x87\x0e\xd3\xea\x1f\x7e\xb8\xe7"
+			"\x9d\xec\xe8\xe7\xbc\xf3\x18\x25\x77\x35\x4a\xaa"
+			"\x00\x99\x2a\xdd\x0a\x00\x50\x82",
+		.addtllen = 32,
+		.pers = (unsigned char *)
+			"\xbc\x55\xab\x3c\xf6\x52\xb0\x11\x3d\x7b\x90\xb8"
+			"\x24\xc9\x26\x4e\x5a\x1e\x77\x0d\x3d\x58\x4a\xda"
+			"\xd1\x81\xe9\xf8\xeb\x30\x8f\x6f",
+		.perslen = 32,
+	},
+};
+
+static const struct drbg_testvec drbg_pr_ctr_aes128_tv_template[] = {
+	{
+		.entropy = (unsigned char *)
+			"\xd1\x44\xc6\x61\x81\x6d\xca\x9d\x15\x28\x8a\x42"
+			"\x94\xd7\x28\x9c\x43\x77\x19\x29\x1a\x6d\xc3\xa2",
+		.entropylen = 24,
+		.entpra = (unsigned char *)
+			"\x96\xd8\x9e\x45\x32\xc9\xd2\x08\x7a\x6d\x97\x15"
+			"\xb4\xec\x80\xb1",
+		.entprb = (unsigned char *)
+			"\x8b\xb6\x72\xb5\x24\x0b\x98\x65\x95\x95\xe9\xc9"
+			"\x28\x07\xeb\xc2",
+		.entprlen = 16,
+		.expected = (unsigned char *)
+			"\x70\x19\xd0\x4c\x45\x78\xd6\x68\xa9\x9a\xaa\xfe"
+			"\xc1\xdf\x27\x9a\x1c\x0d\x0d\xf7\x24\x75\x46\xcc"
+			"\x77\x6b\xdf\x89\xc6\x94\xdc\x74\x50\x10\x70\x18"
+			"\x9b\xdc\x96\xb4\x89\x23\x40\x1a\xce\x09\x87\xce"
+			"\xd2\xf3\xd5\xe4\x51\x67\x74\x11\x5a\xcc\x8b\x3b"
+			"\x8a\xf1\x23\xa8",
+		.expectedlen = 64,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\x8e\x83\xe0\xeb\x37\xea\x3e\x53\x5e\x17\x6e\x77"
+			"\xbd\xb1\x53\x90\xfc\xdc\xc1\x3c\x9a\x88\x22\x94",
+		.entropylen = 24,
+		.entpra = (unsigned char *)
+			"\x6a\x85\xe7\x37\xc8\xf1\x04\x31\x98\x4f\xc8\x73"
+			"\x67\xd1\x08\xf8",
+		.entprb = (unsigned char *)
+			"\xd7\xa4\x68\xe2\x12\x74\xc3\xd9\xf1\xb7\x05\xbc"
+			"\xd4\xba\x04\x58",
+		.entprlen = 16,
+		.expected = (unsigned char *)
+			"\x78\xd6\xa6\x70\xff\xd1\x82\xf5\xa2\x88\x7f\x6d"
+			"\x3d\x8c\x39\xb1\xa8\xcb\x2c\x91\xab\x14\x7e\xbc"
+			"\x95\x45\x9f\x24\xb8\x20\xac\x21\x23\xdb\x72\xd7"
+			"\x12\x8d\x48\x95\xf3\x19\x0c\x43\xc6\x19\x45\xfc"
+			"\x8b\xac\x40\x29\x73\x00\x03\x45\x5e\x12\xff\x0c"
+			"\xc1\x02\x41\x82",
+		.expectedlen = 64,
+		.addtla = (unsigned char *)
+			"\xa2\xd9\x38\xcf\x8b\x29\x67\x5b\x65\x62\x6f\xe8"
+			"\xeb\xb3\x01\x76",
+		.addtlb = (unsigned char *)
+			"\x59\x63\x1e\x81\x8a\x14\xa8\xbb\xa1\xb8\x41\x25"
+			"\xd0\x7f\xcc\x43",
+		.addtllen = 16,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\x04\xd9\x49\xa6\xdc\xe8\x6e\xbb\xf1\x08\x77\x2b"
+			"\x9e\x08\xca\x92\x65\x16\xda\x99\xa2\x59\xf3\xe8",
+		.entropylen = 24,
+		.entpra = (unsigned char *)
+			"\x38\x7e\x3f\x6b\x51\x70\x7b\x20\xec\x53\xd0\x66"
+			"\xc3\x0f\xe3\xb0",
+		.entprb = (unsigned char *)
+			"\xe0\x86\xa6\xaa\x5f\x72\x2f\xad\xf7\xef\x06\xb8"
+			"\xd6\x9c\x9d\xe8",
+		.entprlen = 16,
+		.expected = (unsigned char *)
+			"\xc9\x0a\xaf\x85\x89\x71\x44\x66\x4f\x25\x0b\x2b"
+			"\xde\xd8\xfa\xff\x52\x5a\x1b\x32\x5e\x41\x7a\x10"
+			"\x1f\xef\x1e\x62\x23\xe9\x20\x30\xc9\x0d\xad\x69"
+			"\xb4\x9c\x5b\xf4\x87\x42\xd5\xae\x5e\x5e\x43\xcc"
+			"\xd9\xfd\x0b\x93\x4a\xe3\xd4\x06\x37\x36\x0f\x3f"
+			"\x72\x82\x0c\xcf",
+		.expectedlen = 64,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = (unsigned char *)
+			"\xbf\xa4\x9a\x8f\x7b\xd8\xb1\x7a\x9d\xfa\x45\xed"
+			"\x21\x52\xb3\xad",
+		.perslen = 16,
+	}, {
+		.entropy = (unsigned char *)
+			"\x92\x89\x8f\x31\xfa\x1c\xff\x6d\x18\x2f\x26\x06"
+			"\x43\xdf\xf8\x18\xc2\xa4\xd9\x72\xc3\xb9\xb6\x97",
+		.entropylen = 24,
+		.entpra = (unsigned char *)
+			"\x20\x72\x8a\x06\xf8\x6f\x8d\xd4\x41\xe2\x72\xb7"
+			"\xc4\x2c\xe8\x10",
+		.entprb = (unsigned char *)
+			"\x3d\xb0\xf0\x94\xf3\x05\x50\x33\x17\x86\x3e\x22"
+			"\x08\xf7\xa5\x01",
+		.entprlen = 16,
+		.expected = (unsigned char *)
+			"\x5a\x35\x39\x87\x0f\x4d\x22\xa4\x09\x24\xee\x71"
+			"\xc9\x6f\xac\x72\x0a\xd6\xf0\x88\x82\xd0\x83\x28"
+			"\x73\xec\x3f\x93\xd8\xab\x45\x23\xf0\x7e\xac\x45"
+			"\x14\x5e\x93\x9f\xb1\xd6\x76\x43\x3d\xb6\xe8\x08"
+			"\x88\xf6\xda\x89\x08\x77\x42\xfe\x1a\xf4\x3f\xc4"
+			"\x23\xc5\x1f\x68",
+		.expectedlen = 64,
+		.addtla = (unsigned char *)
+			"\x1a\x40\xfa\xe3\xcc\x6c\x7c\xa0\xf8\xda\xba\x59"
+			"\x23\x6d\xad\x1d",
+		.addtlb = (unsigned char *)
+			"\x9f\x72\x76\x6c\xc7\x46\xe5\xed\x2e\x53\x20\x12"
+			"\xbc\x59\x31\x8c",
+		.addtllen = 16,
+		.pers = (unsigned char *)
+			"\xea\x65\xee\x60\x26\x4e\x7e\xb6\x0e\x82\x68\xc4"
+			"\x37\x3c\x5c\x0b",
+		.perslen = 16,
+	},
+};
+
+/*
+ * SP800-90A DRBG Test vectors from
+ * http://csrc.nist.gov/groups/STM/cavp/documents/drbg/drbgtestvectors.zip
+ *
+ * Test vectors for DRBG without prediction resistance. All types of DRBGs
+ * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and
+ * w/o personalization string, w/ and w/o additional input string).
+ */
+static const struct drbg_testvec drbg_nopr_sha256_tv_template[] = {
+	{
+		.entropy = (unsigned char *)
+			"\xa6\x5a\xd0\xf3\x45\xdb\x4e\x0e\xff\xe8\x75\xc3"
+			"\xa2\xe7\x1f\x42\xc7\x12\x9d\x62\x0f\xf5\xc1\x19"
+			"\xa9\xef\x55\xf0\x51\x85\xe0\xfb\x85\x81\xf9\x31"
+			"\x75\x17\x27\x6e\x06\xe9\x60\x7d\xdb\xcb\xcc\x2e",
+		.entropylen = 48,
+		.expected = (unsigned char *)
+			"\xd3\xe1\x60\xc3\x5b\x99\xf3\x40\xb2\x62\x82\x64"
+			"\xd1\x75\x10\x60\xe0\x04\x5d\xa3\x83\xff\x57\xa5"
+			"\x7d\x73\xa6\x73\xd2\xb8\xd8\x0d\xaa\xf6\xa6\xc3"
+			"\x5a\x91\xbb\x45\x79\xd7\x3f\xd0\xc8\xfe\xd1\x11"
+			"\xb0\x39\x13\x06\x82\x8a\xdf\xed\x52\x8f\x01\x81"
+			"\x21\xb3\xfe\xbd\xc3\x43\xe7\x97\xb8\x7d\xbb\x63"
+			"\xdb\x13\x33\xde\xd9\xd1\xec\xe1\x77\xcf\xa6\xb7"
+			"\x1f\xe8\xab\x1d\xa4\x66\x24\xed\x64\x15\xe5\x1c"
+			"\xcd\xe2\xc7\xca\x86\xe2\x83\x99\x0e\xea\xeb\x91"
+			"\x12\x04\x15\x52\x8b\x22\x95\x91\x02\x81\xb0\x2d"
+			"\xd4\x31\xf4\xc9\xf7\x04\x27\xdf",
+		.expectedlen = 128,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\x73\xd3\xfb\xa3\x94\x5f\x2b\x5f\xb9\x8f\xf6\x9c"
+			"\x8a\x93\x17\xae\x19\xc3\x4c\xc3\xd6\xca\xa3\x2d"
+			"\x16\xfc\x42\xd2\x2d\xd5\x6f\x56\xcc\x1d\x30\xff"
+			"\x9e\x06\x3e\x09\xce\x58\xe6\x9a\x35\xb3\xa6\x56",
+		.entropylen = 48,
+		.expected = (unsigned char *)
+			"\x71\x7b\x93\x46\x1a\x40\xaa\x35\xa4\xaa\xc5\xe7"
+			"\x6d\x5b\x5b\x8a\xa0\xdf\x39\x7d\xae\x71\x58\x5b"
+			"\x3c\x7c\xb4\xf0\x89\xfa\x4a\x8c\xa9\x5c\x54\xc0"
+			"\x40\xdf\xbc\xce\x26\x81\x34\xf8\xba\x7d\x1c\xe8"
+			"\xad\x21\xe0\x74\xcf\x48\x84\x30\x1f\xa1\xd5\x4f"
+			"\x81\x42\x2f\xf4\xdb\x0b\x23\xf8\x73\x27\xb8\x1d"
+			"\x42\xf8\x44\x58\xd8\x5b\x29\x27\x0a\xf8\x69\x59"
+			"\xb5\x78\x44\xeb\x9e\xe0\x68\x6f\x42\x9a\xb0\x5b"
+			"\xe0\x4e\xcb\x6a\xaa\xe2\xd2\xd5\x33\x25\x3e\xe0"
+			"\x6c\xc7\x6a\x07\xa5\x03\x83\x9f\xe2\x8b\xd1\x1c"
+			"\x70\xa8\x07\x59\x97\xeb\xf6\xbe",
+		.expectedlen = 128,
+		.addtla = (unsigned char *)
+			"\xf4\xd5\x98\x3d\xa8\xfc\xfa\x37\xb7\x54\x67\x73"
+			"\xc7\xc3\xdd\x47\x34\x71\x02\x5d\xc1\xa0\xd3\x10"
+			"\xc1\x8b\xbd\xf5\x66\x34\x6f\xdd",
+		.addtlb = (unsigned char *)
+			"\xf7\x9e\x6a\x56\x0e\x73\xe9\xd9\x7a\xd1\x69\xe0"
+			"\x6f\x8c\x55\x1c\x44\xd1\xce\x6f\x28\xcc\xa4\x4d"
+			"\xa8\xc0\x85\xd1\x5a\x0c\x59\x40",
+		.addtllen = 32,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\x2a\x85\xa9\x8b\xd0\xda\x83\xd6\xad\xab\x9f\xbb"
+			"\x54\x31\x15\x95\x1c\x4d\x49\x9f\x6a\x15\xf6\xe4"
+			"\x15\x50\x88\x06\x29\x0d\xed\x8d\xb9\x6f\x96\xe1"
+			"\x83\x9f\xf7\x88\xda\x84\xbf\x44\x28\xd9\x1d\xaa",
+		.entropylen = 48,
+		.expected = (unsigned char *)
+			"\x2d\x55\xde\xc9\xed\x05\x47\x07\x3d\x04\xfc\x28"
+			"\x0f\x92\xf0\x4d\xd8\x00\x32\x47\x0a\x1b\x1c\x4b"
+			"\xef\xd9\x97\xa1\x17\x67\xda\x26\x6c\xfe\x76\x46"
+			"\x6f\xbc\x6d\x82\x4e\x83\x8a\x98\x66\x6c\x01\xb6"
+			"\xe6\x64\xe0\x08\x10\x6f\xd3\x5d\x90\xe7\x0d\x72"
+			"\xa6\xa7\xe3\xbb\x98\x11\x12\x56\x23\xc2\x6d\xd1"
+			"\xc8\xa8\x7a\x39\xf3\x34\xe3\xb8\xf8\x66\x00\x77"
+			"\x7d\xcf\x3c\x3e\xfa\xc9\x0f\xaf\xe0\x24\xfa\xe9"
+			"\x84\xf9\x6a\x01\xf6\x35\xdb\x5c\xab\x2a\xef\x4e"
+			"\xac\xab\x55\xb8\x9b\xef\x98\x68\xaf\x51\xd8\x16"
+			"\xa5\x5e\xae\xf9\x1e\xd2\xdb\xe6",
+		.expectedlen = 128,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = (unsigned char *)
+			"\xa8\x80\xec\x98\x30\x98\x15\xd2\xc6\xc4\x68\xf1"
+			"\x3a\x1c\xbf\xce\x6a\x40\x14\xeb\x36\x99\x53\xda"
+			"\x57\x6b\xce\xa4\x1c\x66\x3d\xbc",
+		.perslen = 32,
+	}, {
+		.entropy = (unsigned char *)
+			"\x69\xed\x82\xa9\xc5\x7b\xbf\xe5\x1d\x2f\xcb\x7a"
+			"\xd3\x50\x7d\x96\xb4\xb9\x2b\x50\x77\x51\x27\x74"
+			"\x33\x74\xba\xf1\x30\xdf\x8e\xdf\x87\x1d\x87\xbc"
+			"\x96\xb2\xc3\xa7\xed\x60\x5e\x61\x4e\x51\x29\x1a",
+		.entropylen = 48,
+		.expected = (unsigned char *)
+			"\xa5\x71\x24\x31\x11\xfe\x13\xe1\xa8\x24\x12\xfb"
+			"\x37\xa1\x27\xa5\xab\x77\xa1\x9f\xae\x8f\xaf\x13"
+			"\x93\xf7\x53\x85\x91\xb6\x1b\xab\xd4\x6b\xea\xb6"
+			"\xef\xda\x4c\x90\x6e\xef\x5f\xde\xe1\xc7\x10\x36"
+			"\xd5\x67\xbd\x14\xb6\x89\x21\x0c\xc9\x92\x65\x64"
+			"\xd0\xf3\x23\xe0\x7f\xd1\xe8\x75\xc2\x85\x06\xea"
+			"\xca\xc0\xcb\x79\x2d\x29\x82\xfc\xaa\x9a\xc6\x95"
+			"\x7e\xdc\x88\x65\xba\xec\x0e\x16\x87\xec\xa3\x9e"
+			"\xd8\x8c\x80\xab\x3a\x64\xe0\xcb\x0e\x45\x98\xdd"
+			"\x7c\x6c\x6c\x26\x11\x13\xc8\xce\xa9\x47\xa6\x06"
+			"\x57\xa2\x66\xbb\x2d\x7f\xf3\xc1",
+		.expectedlen = 128,
+		.addtla = (unsigned char *)
+			"\x74\xd3\x6d\xda\xe8\xd6\x86\x5f\x63\x01\xfd\xf2"
+			"\x7d\x06\x29\x6d\x94\xd1\x66\xf0\xd2\x72\x67\x4e"
+			"\x77\xc5\x3d\x9e\x03\xe3\xa5\x78",
+		.addtlb = (unsigned char *)
+			"\xf6\xb6\x3d\xf0\x7c\x26\x04\xc5\x8b\xcd\x3e\x6a"
+			"\x9f\x9c\x3a\x2e\xdb\x47\x87\xe5\x8e\x00\x5e\x2b"
+			"\x74\x7f\xa6\xf6\x80\xcd\x9b\x21",
+		.addtllen = 32,
+		.pers = (unsigned char *)
+			"\x74\xa6\xe0\x08\xf9\x27\xee\x1d\x6e\x3c\x28\x20"
+			"\x87\xdd\xd7\x54\x31\x47\x78\x4b\xe5\x6d\xa3\x73"
+			"\xa9\x65\xb1\x10\xc1\xdc\x77\x7c",
+		.perslen = 32,
+	},
+};
+
+static const struct drbg_testvec drbg_nopr_hmac_sha256_tv_template[] = {
+	{
+		.entropy = (unsigned char *)
+			"\xca\x85\x19\x11\x34\x93\x84\xbf\xfe\x89\xde\x1c"
+			"\xbd\xc4\x6e\x68\x31\xe4\x4d\x34\xa4\xfb\x93\x5e"
+			"\xe2\x85\xdd\x14\xb7\x1a\x74\x88\x65\x9b\xa9\x6c"
+			"\x60\x1d\xc6\x9f\xc9\x02\x94\x08\x05\xec\x0c\xa8",
+		.entropylen = 48,
+		.expected = (unsigned char *)
+			"\xe5\x28\xe9\xab\xf2\xde\xce\x54\xd4\x7c\x7e\x75"
+			"\xe5\xfe\x30\x21\x49\xf8\x17\xea\x9f\xb4\xbe\xe6"
+			"\xf4\x19\x96\x97\xd0\x4d\x5b\x89\xd5\x4f\xbb\x97"
+			"\x8a\x15\xb5\xc4\x43\xc9\xec\x21\x03\x6d\x24\x60"
+			"\xb6\xf7\x3e\xba\xd0\xdc\x2a\xba\x6e\x62\x4a\xbf"
+			"\x07\x74\x5b\xc1\x07\x69\x4b\xb7\x54\x7b\xb0\x99"
+			"\x5f\x70\xde\x25\xd6\xb2\x9e\x2d\x30\x11\xbb\x19"
+			"\xd2\x76\x76\xc0\x71\x62\xc8\xb5\xcc\xde\x06\x68"
+			"\x96\x1d\xf8\x68\x03\x48\x2c\xb3\x7e\xd6\xd5\xc0"
+			"\xbb\x8d\x50\xcf\x1f\x50\xd4\x76\xaa\x04\x58\xbd"
+			"\xab\xa8\x06\xf4\x8b\xe9\xdc\xb8",
+		.expectedlen = 128,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\xf9\x7a\x3c\xfd\x91\xfa\xa0\x46\xb9\xe6\x1b\x94"
+			"\x93\xd4\x36\xc4\x93\x1f\x60\x4b\x22\xf1\x08\x15"
+			"\x21\xb3\x41\x91\x51\xe8\xff\x06\x11\xf3\xa7\xd4"
+			"\x35\x95\x35\x7d\x58\x12\x0b\xd1\xe2\xdd\x8a\xed",
+		.entropylen = 48,
+		.expected = (unsigned char *)
+			"\xc6\x87\x1c\xff\x08\x24\xfe\x55\xea\x76\x89\xa5"
+			"\x22\x29\x88\x67\x30\x45\x0e\x5d\x36\x2d\xa5\xbf"
+			"\x59\x0d\xcf\x9a\xcd\x67\xfe\xd4\xcb\x32\x10\x7d"
+			"\xf5\xd0\x39\x69\xa6\x6b\x1f\x64\x94\xfd\xf5\xd6"
+			"\x3d\x5b\x4d\x0d\x34\xea\x73\x99\xa0\x7d\x01\x16"
+			"\x12\x6d\x0d\x51\x8c\x7c\x55\xba\x46\xe1\x2f\x62"
+			"\xef\xc8\xfe\x28\xa5\x1c\x9d\x42\x8e\x6d\x37\x1d"
+			"\x73\x97\xab\x31\x9f\xc7\x3d\xed\x47\x22\xe5\xb4"
+			"\xf3\x00\x04\x03\x2a\x61\x28\xdf\x5e\x74\x97\xec"
+			"\xf8\x2c\xa7\xb0\xa5\x0e\x86\x7e\xf6\x72\x8a\x4f"
+			"\x50\x9a\x8c\x85\x90\x87\x03\x9c",
+		.expectedlen = 128,
+		.addtla = (unsigned char *)
+			"\x51\x72\x89\xaf\xe4\x44\xa0\xfe\x5e\xd1\xa4\x1d"
+			"\xbb\xb5\xeb\x17\x15\x00\x79\xbd\xd3\x1e\x29\xcf"
+			"\x2f\xf3\x00\x34\xd8\x26\x8e\x3b",
+		.addtlb = (unsigned char *)
+			"\x88\x02\x8d\x29\xef\x80\xb4\xe6\xf0\xfe\x12\xf9"
+			"\x1d\x74\x49\xfe\x75\x06\x26\x82\xe8\x9c\x57\x14"
+			"\x40\xc0\xc9\xb5\x2c\x42\xa6\xe0",
+		.addtllen = 32,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\x8d\xf0\x13\xb4\xd1\x03\x52\x30\x73\x91\x7d\xdf"
+			"\x6a\x86\x97\x93\x05\x9e\x99\x43\xfc\x86\x54\x54"
+			"\x9e\x7a\xb2\x2f\x7c\x29\xf1\x22\xda\x26\x25\xaf"
+			"\x2d\xdd\x4a\xbc\xce\x3c\xf4\xfa\x46\x59\xd8\x4e",
+		.entropylen = 48,
+		.expected = (unsigned char *)
+			"\xb9\x1c\xba\x4c\xc8\x4f\xa2\x5d\xf8\x61\x0b\x81"
+			"\xb6\x41\x40\x27\x68\xa2\x09\x72\x34\x93\x2e\x37"
+			"\xd5\x90\xb1\x15\x4c\xbd\x23\xf9\x74\x52\xe3\x10"
+			"\xe2\x91\xc4\x51\x46\x14\x7f\x0d\xa2\xd8\x17\x61"
+			"\xfe\x90\xfb\xa6\x4f\x94\x41\x9c\x0f\x66\x2b\x28"
+			"\xc1\xed\x94\xda\x48\x7b\xb7\xe7\x3e\xec\x79\x8f"
+			"\xbc\xf9\x81\xb7\x91\xd1\xbe\x4f\x17\x7a\x89\x07"
+			"\xaa\x3c\x40\x16\x43\xa5\xb6\x2b\x87\xb8\x9d\x66"
+			"\xb3\xa6\x0e\x40\xd4\xa8\xe4\xe9\xd8\x2a\xf6\xd2"
+			"\x70\x0e\x6f\x53\x5c\xdb\x51\xf7\x5c\x32\x17\x29"
+			"\x10\x37\x41\x03\x0c\xcc\x3a\x56",
+		.expectedlen = 128,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = (unsigned char *)
+			"\xb5\x71\xe6\x6d\x7c\x33\x8b\xc0\x7b\x76\xad\x37"
+			"\x57\xbb\x2f\x94\x52\xbf\x7e\x07\x43\x7a\xe8\x58"
+			"\x1c\xe7\xbc\x7c\x3a\xc6\x51\xa9",
+		.perslen = 32,
+	}, {
+		.entropy = (unsigned char *)
+			"\xc2\xa5\x66\xa9\xa1\x81\x7b\x15\xc5\xc3\xb7\x78"
+			"\x17\x7a\xc8\x7c\x24\xe7\x97\xbe\x0a\x84\x5f\x11"
+			"\xc2\xfe\x39\x9d\xd3\x77\x32\xf2\xcb\x18\x94\xeb"
+			"\x2b\x97\xb3\xc5\x6e\x62\x83\x29\x51\x6f\x86\xec",
+		.entropylen = 48,
+		.expected = (unsigned char *)
+			"\xb3\xa3\x69\x8d\x77\x76\x99\xa0\xdd\x9f\xa3\xf0"
+			"\xa9\xfa\x57\x83\x2d\x3c\xef\xac\x5d\xf2\x44\x37"
+			"\xc6\xd7\x3a\x0f\xe4\x10\x40\xf1\x72\x90\x38\xae"
+			"\xf1\xe9\x26\x35\x2e\xa5\x9d\xe1\x20\xbf\xb7\xb0"
+			"\x73\x18\x3a\x34\x10\x6e\xfe\xd6\x27\x8f\xf8\xad"
+			"\x84\x4b\xa0\x44\x81\x15\xdf\xdd\xf3\x31\x9a\x82"
+			"\xde\x6b\xb1\x1d\x80\xbd\x87\x1a\x9a\xcd\x35\xc7"
+			"\x36\x45\xe1\x27\x0f\xb9\xfe\x4f\xa8\x8e\xc0\xe4"
+			"\x65\x40\x9e\xa0\xcb\xa8\x09\xfe\x2f\x45\xe0\x49"
+			"\x43\xa2\xe3\x96\xbb\xb7\xdd\x2f\x4e\x07\x95\x30"
+			"\x35\x24\xcc\x9c\xc5\xea\x54\xa1",
+		.expectedlen = 128,
+		.addtla = (unsigned char *)
+			"\x41\x3d\xd8\x3f\xe5\x68\x35\xab\xd4\x78\xcb\x96"
+			"\x93\xd6\x76\x35\x90\x1c\x40\x23\x9a\x26\x64\x62"
+			"\xd3\x13\x3b\x83\xe4\x9c\x82\x0b",
+		.addtlb = (unsigned char *)
+			"\xd5\xc4\xa7\x1f\x9d\x6d\x95\xa1\xbe\xdf\x0b\xd2"
+			"\x24\x7c\x27\x7d\x1f\x84\xa4\xe5\x7a\x4a\x88\x25"
+			"\xb8\x2a\x2d\x09\x7d\xe6\x3e\xf1",
+		.addtllen = 32,
+		.pers = (unsigned char *)
+			"\x13\xce\x4d\x8d\xd2\xdb\x97\x96\xf9\x41\x56\xc8"
+			"\xe8\xf0\x76\x9b\x0a\xa1\xc8\x2c\x13\x23\xb6\x15"
+			"\x36\x60\x3b\xca\x37\xc9\xee\x29",
+		.perslen = 32,
+	},
+};
+
+static const struct drbg_testvec drbg_nopr_ctr_aes192_tv_template[] = {
+	{
+		.entropy = (unsigned char *)
+			"\xc3\x5c\x2f\xa2\xa8\x9d\x52\xa1\x1f\xa3\x2a\xa9"
+			"\x6c\x95\xb8\xf1\xc9\xa8\xf9\xcb\x24\x5a\x8b\x40"
+			"\xf3\xa6\xe5\xa7\xfb\xd9\xd3\xc6\x8e\x27\x7b\xa9"
+			"\xac\x9b\xbb\x00",
+		.entropylen = 40,
+		.expected = (unsigned char *)
+			"\x8c\x2e\x72\xab\xfd\x9b\xb8\x28\x4d\xb7\x9e\x17"
+			"\xa4\x3a\x31\x46\xcd\x76\x94\xe3\x52\x49\xfc\x33"
+			"\x83\x91\x4a\x71\x17\xf4\x13\x68\xe6\xd4\xf1\x48"
+			"\xff\x49\xbf\x29\x07\x6b\x50\x15\xc5\x9f\x45\x79"
+			"\x45\x66\x2e\x3d\x35\x03\x84\x3f\x4a\xa5\xa3\xdf"
+			"\x9a\x9d\xf1\x0d",
+		.expectedlen = 64,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = NULL,
+		.perslen = 0,
 	},
 };
 
-static const struct cipher_testvec camellia_ctr_enc_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
-			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
-			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
-			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
-			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
-			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
-			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
-			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
-			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
-			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
-			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
-			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
-			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
-			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
-			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
-			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
-			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
-			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
-			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
-			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
-			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
-			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
-			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
-			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
-			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
-			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
-			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
-			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
-			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
-			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
-			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
-			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
-			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
-			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
-			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
-			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
-			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
-			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
-			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
-			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
-			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
-			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
-			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
-			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
-			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
-			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
-			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
-			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
-			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
-			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
-			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
-			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
-			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
-			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
-			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
-			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
-			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
-			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
-			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
-			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
-			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.ilen	= 496,
-		.result	= "\xF3\x06\x3A\x84\xCD\xBA\x8E\x11"
-			  "\xB7\x74\x6F\x5C\x97\xFB\x36\xFE"
-			  "\xDE\x71\x58\xD4\x15\xD1\xC1\xA4"
-			  "\xC9\x28\x74\xA6\x6B\xC7\x95\xA6"
-			  "\x6C\x77\xF7\x2F\xDF\xC7\xBB\x85"
-			  "\x60\xFC\xE8\x94\xE8\xB5\x09\x2C"
-			  "\x1E\x43\xEF\x6C\xE9\x98\xC5\xA0"
-			  "\x7B\x13\xE5\x7F\xF8\x49\x9A\x8C"
-			  "\xE6\x7B\x08\xC3\x32\x66\x55\x4E"
-			  "\xA5\x44\x1D\x2C\x18\xC7\x29\x1F"
-			  "\x61\x28\x4A\xE3\xCD\xE5\x47\xB2"
-			  "\x82\x2F\x66\x83\x91\x51\xAE\xD7"
-			  "\x1C\x91\x3C\x57\xE3\x1D\x5A\xC9"
-			  "\xFD\xC5\x58\x58\xEF\xCC\x33\xC9"
-			  "\x0F\xEA\x26\x32\xD1\x15\x19\x2D"
-			  "\x25\xB4\x7F\xB0\xDF\xFB\x88\x60"
-			  "\x4E\x4D\x06\x7D\xCC\x1F\xED\x3B"
-			  "\x68\x84\xD5\xB3\x1B\xE7\xB9\xA1"
-			  "\x68\x8B\x2C\x1A\x44\xDA\x63\xD3"
-			  "\x29\xE9\x59\x32\x1F\x30\x1C\x43"
-			  "\xEA\x3A\xA3\x6B\x54\x3C\xAA\x11"
-			  "\xAD\x38\x20\xC9\xB9\x8A\x64\x66"
-			  "\x5A\x07\x49\xDF\xA1\x9C\xF9\x76"
-			  "\x36\x65\xB6\x81\x8F\x76\x09\xE5"
-			  "\xEB\xD1\x29\xA4\xE4\xF4\x4C\xCD"
-			  "\xAF\xFC\xB9\x16\xD9\xC3\x73\x6A"
-			  "\x33\x12\xF8\x7E\xBC\xCC\x7D\x80"
-			  "\xBF\x3C\x25\x06\x13\x84\xFA\x35"
-			  "\xF7\x40\xFA\xA1\x44\x13\x70\xD8"
-			  "\x01\xF9\x85\x15\x63\xEC\x7D\xB9"
-			  "\x02\xD8\xBA\x41\x6C\x92\x68\x66"
-			  "\x95\xDD\xD6\x42\xE7\xBB\xE1\xFD"
-			  "\x28\x3E\x94\xB6\xBD\xA7\xBF\x47"
-			  "\x58\x8D\xFF\x19\x30\x75\x0D\x48"
-			  "\x94\xE9\xA6\xCD\xB3\x8E\x1E\xCD"
-			  "\x59\xBC\x1A\xAC\x3C\x4F\xA9\xEB"
-			  "\xF4\xA7\xE4\x75\x4A\x18\x40\xC9"
-			  "\x1E\xEC\x06\x9C\x28\x4B\xF7\x2B"
-			  "\xE2\xEF\xD6\x42\x2E\xBB\xFC\x0A"
-			  "\x79\xA2\x99\x28\x93\x1B\x00\x57"
-			  "\x35\x1E\x1A\x93\x90\xA4\x68\x95"
-			  "\x5E\x57\x40\xD5\xA9\xAA\x19\x48"
-			  "\xEC\xFF\x76\x77\xDC\x78\x89\x76"
-			  "\xE5\x3B\x00\xEC\x58\x4D\xD1\xE3"
-			  "\xC8\x6C\x2C\x45\x5E\x5F\xD9\x4E"
-			  "\x71\xA5\x36\x6D\x03\xF1\xC7\xD5"
-			  "\xF3\x63\xC0\xD8\xCB\x2B\xF1\xA8"
-			  "\xB9\x2B\xE6\x0B\xB9\x65\x78\xA0"
-			  "\xC4\x46\xE6\x9B\x8B\x43\x2D\xAB"
-			  "\x70\xA6\xE0\x59\x1E\xAC\x9D\xE0"
-			  "\x76\x44\x45\xF3\x24\x11\x57\x98"
-			  "\x9A\x86\xB4\x12\x80\x28\x86\x20"
-			  "\x23\x9D\x2D\xE9\x38\x32\xB1\xE1"
-			  "\xCF\x0A\x23\x73\x7D\xC5\x80\x3D"
-			  "\x9F\x6D\xA0\xD0\xEE\x93\x8A\x79"
-			  "\x3A\xDD\x1D\xBB\x9E\x26\x5D\x01"
-			  "\x44\xD0\xD4\x4E\xC3\xF1\xE4\x38"
-			  "\x09\x62\x0A\x1A\x4E\xD2\x63\x0F"
-			  "\x6E\x3E\xD2\xA4\x3A\xF4\xF3\xFF"
-			  "\x7E\x42\xEC\xB6\x6F\x4D\x6B\x48"
-			  "\xE6\xA6\x50\x80\x78\x9E\xF1\xB0"
-			  "\x4D\xB2\x0D\x3D\xFC\x40\x25\x4D",
-		.rlen	= 496,
-	}, { /* Generated with Crypto++ */
+static const struct drbg_testvec drbg_nopr_ctr_aes256_tv_template[] = {
+	{
+		.entropy = (unsigned char *)
+			"\x36\x40\x19\x40\xfa\x8b\x1f\xba\x91\xa1\x66\x1f"
+			"\x21\x1d\x78\xa0\xb9\x38\x9a\x74\xe5\xbc\xcf\xec"
+			"\xe8\xd7\x66\xaf\x1a\x6d\x3b\x14\x49\x6f\x25\xb0"
+			"\xf1\x30\x1b\x4f\x50\x1b\xe3\x03\x80\xa1\x37\xeb",
+		.entropylen = 48,
+		.expected = (unsigned char *)
+			"\x58\x62\xeb\x38\xbd\x55\x8d\xd9\x78\xa6\x96\xe6"
+			"\xdf\x16\x47\x82\xdd\xd8\x87\xe7\xe9\xa6\xc9\xf3"
+			"\xf1\xfb\xaf\xb7\x89\x41\xb5\x35\xa6\x49\x12\xdf"
+			"\xd2\x24\xc6\xdc\x74\x54\xe5\x25\x0b\x3d\x97\x16"
+			"\x5e\x16\x26\x0c\x2f\xaf\x1c\xc7\x73\x5c\xb7\x5f"
+			"\xb4\xf0\x7e\x1d",
+		.expectedlen = 64,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = NULL,
+		.perslen = 0,
+	},
+};
+
+static const struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = {
+	{
+		.entropy = (unsigned char *)
+			"\x87\xe1\xc5\x32\x99\x7f\x57\xa3\x5c\x28\x6d\xe8"
+			"\x64\xbf\xf2\x64\xa3\x9e\x98\xdb\x6c\x10\x78\x7f",
+		.entropylen = 24,
+		.expected = (unsigned char *)
+			"\x2c\x14\x7e\x24\x11\x9a\xd8\xd4\xb2\xed\x61\xc1"
+			"\x53\xd0\x50\xc9\x24\xff\x59\x75\x15\xf1\x17\x3a"
+			"\x3d\xf4\x4b\x2c\x84\x28\xef\x89\x0e\xb9\xde\xf3"
+			"\xe4\x78\x04\xb2\xfd\x9b\x35\x7f\xe1\x3f\x8a\x3e"
+			"\x10\xc8\x67\x0a\xf9\xdf\x2d\x6c\x96\xfb\xb2\xb8"
+			"\xcb\x2d\xd6\xb0",
+		.expectedlen = 64,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\x71\xbd\xce\x35\x42\x7d\x20\xbf\x58\xcf\x17\x74"
+			"\xce\x72\xd8\x33\x34\x50\x2d\x8f\x5b\x14\xc4\xdd",
+		.entropylen = 24,
+		.expected = (unsigned char *)
+			"\x97\x33\xe8\x20\x12\xe2\x7b\xa1\x46\x8f\xf2\x34"
+			"\xb3\xc9\xb6\x6b\x20\xb2\x4f\xee\x27\xd8\x0b\x21"
+			"\x8c\xff\x63\x73\x69\x29\xfb\xf3\x85\xcd\x88\x8e"
+			"\x43\x2c\x71\x8b\xa2\x55\xd2\x0f\x1d\x7f\xe3\xe1"
+			"\x2a\xa3\xe9\x2c\x25\x89\xc7\x14\x52\x99\x56\xcc"
+			"\xc3\xdf\xb3\x81",
+		.expectedlen = 64,
+		.addtla = (unsigned char *)
+			"\x66\xef\x42\xd6\x9a\x8c\x3d\x6d\x4a\x9e\x95\xa6"
+			"\x91\x4d\x81\x56",
+		.addtlb = (unsigned char *)
+			"\xe3\x18\x83\xd9\x4b\x5e\xc4\xcc\xaa\x61\x2f\xbb"
+			"\x4a\x55\xd1\xc6",
+		.addtllen = 16,
+		.pers = NULL,
+		.perslen = 0,
+	}, {
+		.entropy = (unsigned char *)
+			"\xca\x4b\x1e\xfa\x75\xbd\x69\x36\x38\x73\xb8\xf9"
+			"\xdb\x4d\x35\x0e\x47\xbf\x6c\x37\x72\xfd\xf7\xa9",
+		.entropylen = 24,
+		.expected = (unsigned char *)
+			"\x59\xc3\x19\x79\x1b\xb1\xf3\x0e\xe9\x34\xae\x6e"
+			"\x8b\x1f\xad\x1f\x74\xca\x25\x45\x68\xb8\x7f\x75"
+			"\x12\xf8\xf2\xab\x4c\x23\x01\x03\x05\xe1\x70\xee"
+			"\x75\xd8\xcb\xeb\x23\x4c\x7a\x23\x6e\x12\x27\xdb"
+			"\x6f\x7a\xac\x3c\x44\xb7\x87\x4b\x65\x56\x74\x45"
+			"\x34\x30\x0c\x3d",
+		.expectedlen = 64,
+		.addtla = NULL,
+		.addtlb = NULL,
+		.addtllen = 0,
+		.pers = (unsigned char *)
+			"\xeb\xaa\x60\x2c\x4d\xbe\x33\xff\x1b\xef\xbf\x0a"
+			"\x0b\xc6\x97\x54",
+		.perslen = 16,
+	}, {
+		.entropy = (unsigned char *)
+			"\xc0\x70\x1f\x92\x50\x75\x8f\xcd\xf2\xbe\x73\x98"
+			"\x80\xdb\x66\xeb\x14\x68\xb4\xa5\x87\x9c\x2d\xa6",
+		.entropylen = 24,
+		.expected = (unsigned char *)
+			"\x97\xc0\xc0\xe5\xa0\xcc\xf2\x4f\x33\x63\x48\x8a"
+			"\xdb\x13\x0a\x35\x89\xbf\x80\x65\x62\xee\x13\x95"
+			"\x7c\x33\xd3\x7d\xf4\x07\x77\x7a\x2b\x65\x0b\x5f"
+			"\x45\x5c\x13\xf1\x90\x77\x7f\xc5\x04\x3f\xcc\x1a"
+			"\x38\xf8\xcd\x1b\xbb\xd5\x57\xd1\x4a\x4c\x2e\x8a"
+			"\x2b\x49\x1e\x5c",
+		.expectedlen = 64,
+		.addtla = (unsigned char *)
+			"\xf9\x01\xf8\x16\x7a\x1d\xff\xde\x8e\x3c\x83\xe2"
+			"\x44\x85\xe7\xfe",
+		.addtlb = (unsigned char *)
+			"\x17\x1c\x09\x38\xc2\x38\x9f\x97\x87\x60\x55\xb4"
+			"\x82\x16\x62\x7f",
+		.addtllen = 16,
+		.pers = (unsigned char *)
+			"\x80\x08\xae\xe8\xe9\x69\x40\xc5\x08\x73\xc7\x9f"
+			"\x8e\xcf\xe0\x02",
+		.perslen = 16,
+	},
+};
+
+/* Cast5 test vectors from RFC 2144 */
+static const struct cipher_testvec cast5_tv_template[] = {
+	{
+		.key	= "\x01\x23\x45\x67\x12\x34\x56\x78"
+			  "\x23\x45\x67\x89\x34\x56\x78\x9a",
+		.klen	= 16,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.ctext	= "\x23\x8b\x4f\xe5\x84\x7e\x44\xb2",
+		.len	= 8,
+	}, {
+		.key	= "\x01\x23\x45\x67\x12\x34\x56\x78"
+			  "\x23\x45",
+		.klen	= 10,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.ctext	= "\xeb\x6a\x71\x1a\x2c\x02\x27\x1b",
+		.len	= 8,
+	}, {
+		.key	= "\x01\x23\x45\x67\x12",
+		.klen	= 5,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.ctext	= "\x7a\xc8\x16\xd1\x6e\x9b\x30\x2e",
+		.len	= 8,
+	}, { /* Generated from TF test vectors */
 		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -31461,213 +27061,83 @@ static const struct cipher_testvec camellia_ctr_enc_tv_template[] = {
 			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
 			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06"
-			  "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78"
-			  "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA"
-			  "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C"
-			  "\xF3\x67\xFE\x95\x09\xA0\x37\xCE"
-			  "\x42\xD9\x70\x07\x7B\x12\xA9\x1D"
-			  "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F"
-			  "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01"
-			  "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73"
-			  "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5"
-			  "\x59\xF0\x87\x1E\x92\x29\xC0\x34"
-			  "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6"
-			  "\x3D\xD4\x48\xDF\x76\x0D\x81\x18"
-			  "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A"
-			  "\x21\x95\x2C\xC3\x37\xCE\x65\xFC"
-			  "\x70\x07\x9E\x12\xA9\x40\xD7\x4B"
-			  "\xE2\x79\x10\x84\x1B\xB2\x26\xBD"
-			  "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F"
-			  "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1"
-			  "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13"
-			  "\x87\x1E\xB5\x29\xC0\x57\xEE\x62"
-			  "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4"
-			  "\x6B\x02\x76\x0D\xA4\x18\xAF\x46"
-			  "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8"
-			  "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07"
-			  "\x9E\x35\xCC\x40\xD7\x6E\x05\x79"
-			  "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB"
-			  "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D"
-			  "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF"
-			  "\x43\xDA\x71\x08\x7C\x13\xAA\x1E"
-			  "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90"
-			  "\x27\xBE\x32\xC9\x60\xF7\x6B\x02"
-			  "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74"
-			  "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6"
-			  "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35"
-			  "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7"
-			  "\x3E\xD5\x49\xE0\x77\x0E\x82\x19"
-			  "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B"
-			  "\x22\x96\x2D\xC4\x38\xCF\x66\xFD"
-			  "\x71\x08\x9F\x13\xAA\x41\xD8\x4C"
-			  "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE"
-			  "\x55\xEC\x60\xF7\x8E\x02\x99\x30"
-			  "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2"
-			  "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14"
-			  "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63"
-			  "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5"
-			  "\x6C\x03\x77\x0E\xA5\x19\xB0\x47"
-			  "\xDE\x52\xE9\x80\x17\x8B\x22\xB9"
-			  "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08"
-			  "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A"
-			  "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC"
-			  "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E"
-			  "\xF5\x69\x00\x97\x0B\xA2\x39\xD0"
-			  "\x44\xDB\x72\x09\x7D\x14\xAB\x1F"
-			  "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91"
-			  "\x28\xBF\x33\xCA\x61\xF8\x6C\x03"
-			  "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75"
-			  "\x0C\x80\x17\xAE\x22\xB9\x50\xE7"
-			  "\x5B\xF2\x89\x20\x94\x2B\xC2\x36"
-			  "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8"
-			  "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A"
-			  "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C"
-			  "\x00\x97\x2E\xC5\x39\xD0\x67\xFE"
-			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D"
-			  "\xE4\x7B\x12",
-		.ilen	= 1011,
-		.result	= "\xF3\x06\x3A\x84\xCD\xBA\x8E\x11"
-			  "\xB7\x74\x6F\x5C\x97\xFB\x36\xFE"
-			  "\xDE\x71\x58\xD4\x15\xD1\xC1\xA4"
-			  "\xC9\x28\x74\xA6\x6B\xC7\x95\xA6"
-			  "\x6C\x77\xF7\x2F\xDF\xC7\xBB\x85"
-			  "\x60\xFC\xE8\x94\xE8\xB5\x09\x2C"
-			  "\x1E\x43\xEF\x6C\xE9\x98\xC5\xA0"
-			  "\x7B\x13\xE5\x7F\xF8\x49\x9A\x8C"
-			  "\xE6\x7B\x08\xC3\x32\x66\x55\x4E"
-			  "\xA5\x44\x1D\x2C\x18\xC7\x29\x1F"
-			  "\x61\x28\x4A\xE3\xCD\xE5\x47\xB2"
-			  "\x82\x2F\x66\x83\x91\x51\xAE\xD7"
-			  "\x1C\x91\x3C\x57\xE3\x1D\x5A\xC9"
-			  "\xFD\xC5\x58\x58\xEF\xCC\x33\xC9"
-			  "\x0F\xEA\x26\x32\xD1\x15\x19\x2D"
-			  "\x25\xB4\x7F\xB0\xDF\xFB\x88\x60"
-			  "\x4E\x4D\x06\x7D\xCC\x1F\xED\x3B"
-			  "\x68\x84\xD5\xB3\x1B\xE7\xB9\xA1"
-			  "\x68\x8B\x2C\x1A\x44\xDA\x63\xD3"
-			  "\x29\xE9\x59\x32\x1F\x30\x1C\x43"
-			  "\xEA\x3A\xA3\x6B\x54\x3C\xAA\x11"
-			  "\xAD\x38\x20\xC9\xB9\x8A\x64\x66"
-			  "\x5A\x07\x49\xDF\xA1\x9C\xF9\x76"
-			  "\x36\x65\xB6\x81\x8F\x76\x09\xE5"
-			  "\xEB\xD1\x29\xA4\xE4\xF4\x4C\xCD"
-			  "\xAF\xFC\xB9\x16\xD9\xC3\x73\x6A"
-			  "\x33\x12\xF8\x7E\xBC\xCC\x7D\x80"
-			  "\xBF\x3C\x25\x06\x13\x84\xFA\x35"
-			  "\xF7\x40\xFA\xA1\x44\x13\x70\xD8"
-			  "\x01\xF9\x85\x15\x63\xEC\x7D\xB9"
-			  "\x02\xD8\xBA\x41\x6C\x92\x68\x66"
-			  "\x95\xDD\xD6\x42\xE7\xBB\xE1\xFD"
-			  "\x28\x3E\x94\xB6\xBD\xA7\xBF\x47"
-			  "\x58\x8D\xFF\x19\x30\x75\x0D\x48"
-			  "\x94\xE9\xA6\xCD\xB3\x8E\x1E\xCD"
-			  "\x59\xBC\x1A\xAC\x3C\x4F\xA9\xEB"
-			  "\xF4\xA7\xE4\x75\x4A\x18\x40\xC9"
-			  "\x1E\xEC\x06\x9C\x28\x4B\xF7\x2B"
-			  "\xE2\xEF\xD6\x42\x2E\xBB\xFC\x0A"
-			  "\x79\xA2\x99\x28\x93\x1B\x00\x57"
-			  "\x35\x1E\x1A\x93\x90\xA4\x68\x95"
-			  "\x5E\x57\x40\xD5\xA9\xAA\x19\x48"
-			  "\xEC\xFF\x76\x77\xDC\x78\x89\x76"
-			  "\xE5\x3B\x00\xEC\x58\x4D\xD1\xE3"
-			  "\xC8\x6C\x2C\x45\x5E\x5F\xD9\x4E"
-			  "\x71\xA5\x36\x6D\x03\xF1\xC7\xD5"
-			  "\xF3\x63\xC0\xD8\xCB\x2B\xF1\xA8"
-			  "\xB9\x2B\xE6\x0B\xB9\x65\x78\xA0"
-			  "\xC4\x46\xE6\x9B\x8B\x43\x2D\xAB"
-			  "\x70\xA6\xE0\x59\x1E\xAC\x9D\xE0"
-			  "\x76\x44\x45\xF3\x24\x11\x57\x98"
-			  "\x9A\x86\xB4\x12\x80\x28\x86\x20"
-			  "\x23\x9D\x2D\xE9\x38\x32\xB1\xE1"
-			  "\xCF\x0A\x23\x73\x7D\xC5\x80\x3D"
-			  "\x9F\x6D\xA0\xD0\xEE\x93\x8A\x79"
-			  "\x3A\xDD\x1D\xBB\x9E\x26\x5D\x01"
-			  "\x44\xD0\xD4\x4E\xC3\xF1\xE4\x38"
-			  "\x09\x62\x0A\x1A\x4E\xD2\x63\x0F"
-			  "\x6E\x3E\xD2\xA4\x3A\xF4\xF3\xFF"
-			  "\x7E\x42\xEC\xB6\x6F\x4D\x6B\x48"
-			  "\xE6\xA6\x50\x80\x78\x9E\xF1\xB0"
-			  "\x4D\xB2\x0D\x3D\xFC\x40\x25\x4D"
-			  "\x93\x11\x1C\xE9\xD2\x9F\x6E\x90"
-			  "\xE5\x41\x4A\xE2\x3C\x45\x29\x35"
-			  "\xEC\xD6\x47\x50\xCB\x7B\xA2\x32"
-			  "\xF7\x8B\x62\xF1\xE3\x9A\xFE\xC7"
-			  "\x1D\x8C\x02\x72\x68\x09\xE9\xB6"
-			  "\x4A\x80\xE6\xB1\x56\xDF\x90\xD4"
-			  "\x93\x74\xA4\xCE\x20\x23\xBF\x48"
-			  "\xA5\xDE\x1B\xFA\x40\x69\x31\x98"
-			  "\x62\x6E\xA5\xC7\xBF\x0C\x62\xE5"
-			  "\x6D\xE1\x93\xF1\x83\x10\x1C\xCA"
-			  "\xF6\x5C\x19\xF8\x90\x78\xCB\xE4"
-			  "\x0B\x3A\xB5\xF8\x43\x86\xD3\x3F"
-			  "\xBA\x83\x34\x3C\x42\xCC\x7D\x28"
-			  "\x29\x63\x4F\xD8\x02\x17\xC5\x07"
-			  "\x2C\xA4\xAC\x79\xCB\xC3\xA9\x09"
-			  "\x81\x45\x18\xED\xE4\xCB\x42\x3B"
-			  "\x87\x2D\x23\xDC\xC5\xBA\x45\xBD"
-			  "\x92\xE5\x02\x97\x96\xCE\xAD\xEC"
-			  "\xBA\xD8\x76\xF8\xCA\xC1\x31\xEC"
-			  "\x1E\x4F\x3F\x83\xF8\x33\xE8\x6E"
-			  "\xCC\xF8\x5F\xDD\x65\x50\x99\x69"
-			  "\xAF\x48\xCE\xA5\xBA\xB6\x14\x9F"
-			  "\x05\x93\xB2\xE6\x59\xC8\x28\xFE"
-			  "\x8F\x37\xF9\x64\xB9\xA5\x56\x8F"
-			  "\xF1\x1B\x90\xEF\xAE\xEB\xFC\x09"
-			  "\x11\x7A\xF2\x19\x0A\x0A\x9A\x3C"
-			  "\xE2\x5E\x29\xFA\x31\x9B\xC1\x74"
-			  "\x1E\x10\x3E\x07\xA9\x31\x6D\xF8"
-			  "\x81\xF5\xD5\x8A\x04\x23\x51\xAC"
-			  "\xA2\xE2\x63\xFD\x27\x1F\x79\x5B"
-			  "\x1F\xE8\xDA\x11\x49\x4D\x1C\xBA"
-			  "\x54\xCC\x0F\xBA\x92\x69\xE5\xCB"
-			  "\x41\x1A\x67\xA6\x40\x82\x70\x8C"
-			  "\x19\x79\x08\xA4\x51\x20\x7D\xC9"
-			  "\x12\x27\xAE\x20\x0D\x2C\xA1\x6D"
-			  "\xF4\x55\xD4\xE7\xE6\xD4\x28\x08"
-			  "\x00\x70\x12\x56\x56\x50\xAD\x14"
-			  "\x5C\x3E\xA2\xD1\x36\x3F\x36\x48"
-			  "\xED\xB1\x57\x3E\x5D\x15\xF6\x1E"
-			  "\x53\xE9\xA4\x3E\xED\x7D\xCF\x7D"
-			  "\x29\xAF\xF3\x1E\x51\xA8\x9F\x85"
-			  "\x8B\xF0\xBB\xCE\xCC\x39\xC3\x64"
-			  "\x4B\xF2\xAD\x70\x19\xD4\x44\x8F"
-			  "\x91\x76\xE8\x15\x66\x34\x9F\xF6"
-			  "\x0F\x15\xA4\xA8\x24\xF8\x58\xB1"
-			  "\x38\x46\x47\xC7\x9B\xCA\xE9\x42"
-			  "\x44\xAA\xE6\xB5\x9C\x91\xA4\xD3"
-			  "\x16\xA0\xED\x42\xBE\xB5\x06\x19"
-			  "\xBE\x67\xE8\xBC\x22\x32\xA4\x1E"
-			  "\x93\xEB\xBE\xE9\xE1\x93\xE5\x31"
-			  "\x3A\xA2\x75\xDF\xE3\x6B\xE7\xCC"
-			  "\xB4\x70\x20\xE0\x6D\x82\x7C\xC8"
-			  "\x94\x5C\x5E\x37\x18\xAD\xED\x8B"
-			  "\x44\x86\xCA\x5E\x07\xB7\x70\x8D"
-			  "\x40\x48\x19\x73\x7C\x78\x64\x0B"
-			  "\xDB\x01\xCA\xAE\x63\x19\xE9\xD1"
-			  "\x6B\x2C\x84\x10\x45\x42\x2E\xC3"
-			  "\xDF\x7F\xAA\xE8\x87\x1B\x63\x46"
-			  "\x74\x28\x9D\x05\x30\x20\x62\x41"
-			  "\xC0\x9F\x2C\x36\x2B\x78\xD7\x26"
-			  "\xDF\x58\x51\xED\xFA\xDC\x87\x79"
-			  "\xBF\x8C\xBF\xC4\x0F\xE5\x05\xDA"
-			  "\x45\xE3\x35\x0D\x69\x91\x54\x1C"
-			  "\xE7\x2C\x49\x08\x8B\x72\xFA\x5C"
-			  "\xF1\x6B\xD9",
-		.rlen	= 1011,
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\x8D\xFC\x81\x9C\xCB\xAA\x5A\x1C"
+			  "\x7E\x95\xCF\x40\xAB\x4D\x6F\xEA"
+			  "\xD3\xD9\xB0\x9A\xB7\xC7\xE0\x2E"
+			  "\xD1\x39\x34\x92\x8F\xFA\x14\xF1"
+			  "\xD5\xD2\x7B\x59\x1F\x35\x28\xC2"
+			  "\x20\xD9\x42\x06\xC9\x0B\x10\x04"
+			  "\xF8\x79\xCD\x32\x86\x75\x4C\xB6"
+			  "\x7B\x1C\x52\xB1\x91\x64\x22\x4B"
+			  "\x13\xC7\xAE\x98\x0E\xB5\xCF\x6F"
+			  "\x3F\xF4\x43\x96\x73\x0D\xA2\x05"
+			  "\xDB\xFD\x28\x90\x2C\x56\xB9\x37"
+			  "\x5B\x69\x0C\xAD\x84\x67\xFF\x15"
+			  "\x4A\xD4\xA7\xD3\xDD\x99\x47\x3A"
+			  "\xED\x34\x35\x78\x6B\x91\xC9\x32"
+			  "\xE1\xBF\xBC\xB4\x04\x85\x6A\x39"
+			  "\xC0\xBA\x51\xD0\x0F\x4E\xD1\xE2"
+			  "\x1C\xFD\x0E\x05\x07\xF4\x10\xED"
+			  "\xA2\x17\xFF\xF5\x64\xC6\x1A\x22"
+			  "\xAD\x78\xE7\xD7\x11\xE9\x99\xB9"
+			  "\xAA\xEC\x6F\xF8\x3B\xBF\xCE\x77"
+			  "\x93\xE8\xAD\x1D\x50\x6C\xAE\xBC"
+			  "\xBA\x5C\x80\xD1\x91\x65\x51\x1B"
+			  "\xE8\x0A\xCD\x99\x96\x71\x3D\xB6"
+			  "\x78\x75\x37\x55\xC1\xF5\x90\x40"
+			  "\x34\xF4\x7E\xC8\xCC\x3A\x5F\x6E"
+			  "\x36\xA1\xA1\xC2\x3A\x72\x42\x8E"
+			  "\x0E\x37\x88\xE8\xCE\x83\xCB\xAD"
+			  "\xE0\x69\x77\x50\xC7\x0C\x99\xCA"
+			  "\x19\x5B\x30\x25\x9A\xEF\x9B\x0C"
+			  "\xEF\x8F\x74\x4C\xCF\x49\x4E\xB9"
+			  "\xC5\xAE\x9E\x2E\x78\x9A\xB9\x48"
+			  "\xD5\x81\xE4\x37\x1D\xBF\x27\xD9"
+			  "\xC5\xD6\x65\x43\x45\x8C\xBB\xB6"
+			  "\x55\xF4\x06\xBB\x49\x53\x8B\x1B"
+			  "\x07\xA9\x96\x69\x5B\xCB\x0F\xBC"
+			  "\x93\x85\x90\x0F\x0A\x68\x40\x2A"
+			  "\x95\xED\x2D\x88\xBF\x71\xD0\xBB"
+			  "\xEC\xB0\x77\x6C\x79\xFC\x3C\x05"
+			  "\x49\x3F\xB8\x24\xEF\x8E\x09\xA2"
+			  "\x1D\xEF\x92\x02\x96\xD4\x7F\xC8"
+			  "\x03\xB2\xCA\xDB\x17\x5C\x52\xCF"
+			  "\xDD\x70\x37\x63\xAA\xA5\x83\x20"
+			  "\x52\x02\xF6\xB9\xE7\x6E\x0A\xB6"
+			  "\x79\x03\xA0\xDA\xA3\x79\x21\xBD"
+			  "\xE3\x37\x3A\xC0\xF7\x2C\x32\xBE"
+			  "\x8B\xE8\xA6\x00\xC7\x32\xD5\x06"
+			  "\xBB\xE3\xAB\x06\x21\x82\xB8\x32"
+			  "\x31\x34\x2A\xA7\x1F\x64\x99\xBF"
+			  "\xFA\xDA\x3D\x75\xF7\x48\xD5\x48"
+			  "\x4B\x52\x7E\xF6\x7C\xAB\x67\x59"
+			  "\xC5\xDC\xA8\xC6\x63\x85\x4A\xDF"
+			  "\xF0\x40\x5F\xCF\xE3\x58\x52\x67"
+			  "\x7A\x24\x32\xC5\xEC\x9E\xA9\x6F"
+			  "\x58\x56\xDD\x94\x1F\x71\x8D\xF4"
+			  "\x6E\xFF\x2C\xA7\xA5\xD8\xBA\xAF"
+			  "\x1D\x8B\xA2\x46\xB5\xC4\x9F\x57"
+			  "\x8D\xD8\xB3\x3C\x02\x0D\xBB\x84"
+			  "\xC7\xBD\xB4\x9A\x6E\xBB\xB1\x37"
+			  "\x95\x79\xC4\xA7\xEA\x1D\xDC\x33"
+			  "\x5D\x0B\x3F\x03\x8F\x30\xF9\xAE"
+			  "\x4F\xFE\x24\x9C\x9A\x02\xE5\x57"
+			  "\xF5\xBC\x25\xD6\x02\x56\x57\x1C",
+		.len	= 496,
 		.also_non_np = 1,
-		.np	= 2,
-		.tap	= { 1011 - 16, 16 },
-	}, { /* Generated with Crypto++ */
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec cast5_cbc_tv_template[] = {
+	{ /* Generated from TF test vectors */
 		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -31728,275 +27198,652 @@ static const struct cipher_testvec camellia_ctr_enc_tv_template[] = {
 			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
 			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
-			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06"
-			  "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78"
-			  "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA"
-			  "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C"
-			  "\xF3\x67\xFE\x95\x09\xA0\x37\xCE"
-			  "\x42\xD9\x70\x07\x7B\x12\xA9\x1D"
-			  "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F"
-			  "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01"
-			  "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73"
-			  "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5"
-			  "\x59\xF0\x87\x1E\x92\x29\xC0\x34"
-			  "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6"
-			  "\x3D\xD4\x48\xDF\x76\x0D\x81\x18"
-			  "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A"
-			  "\x21\x95\x2C\xC3\x37\xCE\x65\xFC"
-			  "\x70\x07\x9E\x12\xA9\x40\xD7\x4B"
-			  "\xE2\x79\x10\x84\x1B\xB2\x26\xBD"
-			  "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F"
-			  "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1"
-			  "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13"
-			  "\x87\x1E\xB5\x29\xC0\x57\xEE\x62"
-			  "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4"
-			  "\x6B\x02\x76\x0D\xA4\x18\xAF\x46"
-			  "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8"
-			  "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07"
-			  "\x9E\x35\xCC\x40\xD7\x6E\x05\x79"
-			  "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB"
-			  "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D"
-			  "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF"
-			  "\x43\xDA\x71\x08\x7C\x13\xAA\x1E"
-			  "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90"
-			  "\x27\xBE\x32\xC9\x60\xF7\x6B\x02"
-			  "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74"
-			  "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6"
-			  "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35"
-			  "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7"
-			  "\x3E\xD5\x49\xE0\x77\x0E\x82\x19"
-			  "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B"
-			  "\x22\x96\x2D\xC4\x38\xCF\x66\xFD"
-			  "\x71\x08\x9F\x13\xAA\x41\xD8\x4C"
-			  "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE"
-			  "\x55\xEC\x60\xF7\x8E\x02\x99\x30"
-			  "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2"
-			  "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14"
-			  "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63"
-			  "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5"
-			  "\x6C\x03\x77\x0E\xA5\x19\xB0\x47"
-			  "\xDE\x52\xE9\x80\x17\x8B\x22\xB9"
-			  "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08"
-			  "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A"
-			  "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC"
-			  "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E"
-			  "\xF5\x69\x00\x97\x0B\xA2\x39\xD0"
-			  "\x44\xDB\x72\x09\x7D\x14\xAB\x1F"
-			  "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91"
-			  "\x28\xBF\x33\xCA\x61\xF8\x6C\x03"
-			  "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75"
-			  "\x0C\x80\x17\xAE\x22\xB9\x50\xE7"
-			  "\x5B\xF2\x89\x20\x94\x2B\xC2\x36"
-			  "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8"
-			  "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A"
-			  "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C"
-			  "\x00\x97\x2E\xC5\x39\xD0\x67\xFE"
-			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D",
-		.ilen	= 1008,
-		.result	= "\x85\x79\x6C\x8B\x2B\x6D\x14\xF9"
-			  "\xA6\x83\xB6\x80\x5B\x3A\xF3\x7E"
-			  "\x30\x29\xEB\x1F\xDC\x19\x5F\xEB"
-			  "\xF7\xC4\x27\x04\x51\x87\xD7\x6F"
-			  "\xB8\x4E\x07\xFB\xAC\x3B\x08\xB4"
-			  "\x4D\xCB\xE8\xE1\x71\x7D\x4F\x48"
-			  "\xCD\x81\x64\xA5\xC4\x07\x1A\x9A"
-			  "\x4B\x62\x90\x0E\xC8\xB3\x2B\x6B"
-			  "\x8F\x9C\x6E\x72\x4B\xBA\xEF\x07"
-			  "\x2C\x56\x07\x5E\x37\x30\x60\xA9"
-			  "\xE3\xEF\xD6\x69\xE1\xA1\x77\x64"
-			  "\x93\x75\x7A\xB7\x7A\x3B\xE9\x43"
-			  "\x23\x35\x95\x91\x80\x8A\xC7\xCF"
-			  "\xC3\xD5\xBF\xE7\xFE\x4C\x06\x6B"
-			  "\x05\x19\x48\xE2\x62\xBA\x4F\xF2"
-			  "\xFB\xEE\xE4\xCB\x79\x9D\xA3\x10"
-			  "\x1D\x29\x8C\x1D\x7A\x88\x5A\xDD"
-			  "\x4E\xB6\x18\xAA\xCD\xE6\x33\x96"
-			  "\xD9\x0F\x90\x5A\x78\x76\x4D\x77"
-			  "\x3C\x20\x89\x3B\xA3\xF9\x07\xFD"
-			  "\xE4\xE8\x20\x2D\x15\x0A\x63\x49"
-			  "\xF5\x4F\x89\xD8\xDE\xA1\x28\x78"
-			  "\x28\x07\x09\x1B\x03\x94\x1D\x4B"
-			  "\x82\x28\x1E\x1D\x95\xBA\xAC\x85"
-			  "\x71\x6E\x3C\x18\x4B\x77\x74\x79"
-			  "\xBF\x67\x0A\x53\x3C\x94\xD9\x60"
-			  "\xE9\x6D\x40\x34\xA0\x2A\x53\x5D"
-			  "\x27\xD5\x47\xF9\xC3\x4B\x27\x29"
-			  "\xE4\x76\x9C\x3F\xA7\x1C\x87\xFC"
-			  "\x6E\x0F\xCF\x9B\x60\xF0\xF0\x8B"
-			  "\x70\x1C\x84\x81\x72\x4D\xB4\x98"
-			  "\x23\x62\xE7\x6A\x2B\xFC\xA5\xB2"
-			  "\xFF\xF5\x71\x07\xCD\x90\x23\x13"
-			  "\x19\xD7\x79\x36\x6C\x9D\x55\x8B"
-			  "\x93\x78\x86\x05\x69\x46\xD0\xC5"
-			  "\x39\x09\xEB\x79\xEF\xFA\x9F\xAE"
-			  "\xF3\xD5\x44\xC3\xFD\x86\xD2\x7C"
-			  "\x83\x4B\xD8\x75\x9C\x18\x04\x7B"
-			  "\x73\xAD\x72\xA4\xF6\xAB\xCF\x4B"
-			  "\xCC\x01\x45\x90\xA6\x43\x05\x0C"
-			  "\x6C\x4F\x62\x77\x57\x97\x9F\xEE"
-			  "\x75\xA7\x3C\x38\xD1\x0F\x3D\x0E"
-			  "\x2C\x43\x98\xFB\x13\x65\x73\xE4"
-			  "\x3C\x1E\xD6\x90\x08\xF7\xE0\x99"
-			  "\x3B\xF1\x9D\x6C\x48\xA9\x0E\x32"
-			  "\x17\xC2\xCC\x20\xA1\x19\x26\xAA"
-			  "\xE0\x75\x2F\xFB\x54\x66\x0A\xDF"
-			  "\xB5\xF2\x1F\xC1\x34\x3C\x30\x56"
-			  "\xE8\xDC\xF7\x92\x6B\xBF\x17\x24"
-			  "\xEC\x94\xB5\x3B\xD6\xCE\xA2\x54"
-			  "\x10\x7F\x50\xDE\x69\x77\xD5\x37"
-			  "\xFE\x9C\x10\x83\xC5\xEB\xC9\x53"
-			  "\xB7\xF3\xC4\x20\xAF\x0A\x7E\x57"
-			  "\x3A\xE6\x75\xFE\x89\x00\x6E\x48"
-			  "\xFB\x99\x17\x2C\xF6\x64\x40\x95"
-			  "\x5E\xDC\x7A\xA6\x70\xC7\xF4\xDD"
-			  "\x52\x05\x24\x34\xF9\x0E\xC8\x64"
-			  "\x6D\xE2\xD8\x80\x53\x31\x4C\xFE"
-			  "\xB4\x3A\x5F\x19\xCF\x42\x1B\x22"
-			  "\x0B\x2D\x7B\xF1\xC5\x43\xF7\x5E"
-			  "\x12\xA8\x01\x64\x16\x0B\x26\x5A"
-			  "\x0C\x95\x0F\x40\xC5\x5A\x06\x7C"
-			  "\xCF\xF5\xD5\xB7\x7A\x34\x23\xB6"
-			  "\xAA\x9E\xA8\x98\xA2\xF8\x3D\xD3"
-			  "\x3F\x23\x69\x63\x56\x96\x45\xD6"
-			  "\x74\x23\x1D\x5C\x63\xCC\xD8\x78"
-			  "\x16\xE2\x9C\xD2\x80\x02\xF2\x28"
-			  "\x69\x2F\xC4\xA8\x15\x15\x24\x3B"
-			  "\xCB\xF0\x14\xE4\x62\xC8\xF3\xD1"
-			  "\x03\x58\x1B\x33\x77\x74\x1F\xB4"
-			  "\x07\x86\xF2\x21\xB7\x41\xAE\xBF"
-			  "\x25\xC2\xFF\x51\xEF\xEA\xCE\xC4"
-			  "\x5F\xD9\xB8\x18\x6A\xF0\x0F\x0D"
-			  "\xF8\x04\xBB\x6D\x62\x33\x87\x26"
-			  "\x4F\x2F\x14\x6E\xDC\xDB\x66\x09"
-			  "\x2A\xEF\x7D\x84\x10\xAC\x82\x5E"
-			  "\xD2\xE4\xAD\x74\x7A\x6D\xCC\x3A"
-			  "\x7B\x62\xD8\xD6\x07\x2D\xF7\xDF"
-			  "\x9B\xB3\x82\xCF\x9C\x1D\x76\x5C"
-			  "\xAC\x7B\xD4\x9B\x45\xA1\x64\x11"
-			  "\x66\xF1\xA7\x0B\xF9\xDD\x00\xDD"
-			  "\xA4\x45\x3D\x3E\x03\xC9\x2E\xCB"
-			  "\xC3\x14\x84\x72\xFD\x41\xDC\xBD"
-			  "\x75\xBE\xA8\xE5\x16\x48\x64\x39"
-			  "\xCA\xF3\xE6\xDC\x25\x24\xF1\x6D"
-			  "\xB2\x8D\xC5\x38\x54\xD3\x5D\x6D"
-			  "\x0B\x29\x10\x15\x0E\x13\x3B\xAC"
-			  "\x7E\xCC\x9E\x3E\x18\x48\xA6\x02"
-			  "\xEF\x03\xB2\x2E\xE3\xD2\x70\x21"
-			  "\xB4\x19\x26\xBE\x3A\x3D\x05\xE0"
-			  "\xF8\x09\xAF\xE4\x31\x26\x92\x2F"
-			  "\x8F\x55\xAC\xED\x0B\xB2\xA5\x34"
-			  "\xBE\x50\xB1\x02\x22\x96\xE3\x40"
-			  "\x7B\x70\x50\x6E\x3B\xD5\xE5\xA0"
-			  "\x8E\xA2\xAD\x14\x60\x5C\x7A\x2B"
-			  "\x3D\x1B\x7F\xC1\xC0\x2C\x56\x36"
-			  "\xD2\x0A\x32\x06\x97\x34\xB9\xF4"
-			  "\x6F\x9F\x7E\x80\xD0\x9D\xF7\x6A"
-			  "\x21\xC1\xA2\x6A\xB1\x96\x5B\x4D"
-			  "\x7A\x15\x6C\xC4\x4E\xB8\xE0\x9E"
-			  "\x6C\x50\xF3\x9C\xC9\xB5\x23\xB7"
-			  "\xF1\xD4\x29\x4A\x23\xC4\xAD\x1E"
-			  "\x2C\x07\xD2\x43\x5F\x57\x93\xCA"
-			  "\x85\xF9\x9F\xAD\x4C\xF1\xE4\xB1"
-			  "\x1A\x8E\x28\xA4\xB6\x52\x77\x7E"
-			  "\x68\xC6\x47\xB9\x76\xCC\x65\x5F"
-			  "\x0B\xF9\x67\x93\xD8\x0E\x9A\x37"
-			  "\x5F\x41\xED\x64\x6C\xAD\x5F\xED"
-			  "\x3F\x8D\xFB\x8E\x1E\xA0\xE4\x1F"
-			  "\xC2\xC7\xED\x18\x43\xE1\x20\x86"
-			  "\x5D\xBC\x30\x70\x22\xA1\xDC\x53"
-			  "\x10\x3A\x8D\x47\x82\xCD\x7F\x59"
-			  "\x03\x2D\x6D\xF5\xE7\x79\xD4\x07"
-			  "\x68\x2A\xA5\x42\x19\x4D\xAF\xF5"
-			  "\xED\x47\x83\xBC\x5F\x62\x84\xDA"
-			  "\xDA\x41\xFF\xB0\x1D\x64\xA3\xC8"
-			  "\xBD\x4E\xE0\xB8\x7F\xEE\x55\x0A"
-			  "\x4E\x61\xB2\x51\xF6\x9C\x95\xF6"
-			  "\x92\xBB\xF6\xC5\xF0\x09\x86\xDE"
-			  "\x37\x9E\x29\xF9\x2A\x18\x73\x0D"
-			  "\xDC\x7E\x6B\x7B\x1B\x43\x8C\xEA"
-			  "\x13\xC8\x1A\x47\x0A\x2D\x6D\x56"
-			  "\xCD\xD2\xE7\x53\x1A\xAB\x1C\x3C"
-			  "\xC5\x9B\x03\x70\x29\x2A\x49\x09"
-			  "\x67\xA1\xEA\xD6\x3A\x5B\xBF\x71"
-			  "\x1D\x48\x64\x6C\xFB\xC0\x9E\x36",
-		.rlen	= 1008,
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\x05\x28\xCE\x61\x90\x80\xE1\x78"
+			  "\xB9\x2A\x97\x7C\xB0\x83\xD8\x1A"
+			  "\xDE\x58\x7F\xD7\xFD\x72\xB8\xFB"
+			  "\xDA\xF0\x6E\x77\x14\x47\x82\xBA"
+			  "\x29\x0E\x25\x6E\xB4\x39\xD9\x7F"
+			  "\x05\xA7\xA7\x3A\xC1\x5D\x9E\x39"
+			  "\xA7\xFB\x0D\x05\x00\xF3\x58\x67"
+			  "\x60\xEC\x73\x77\x46\x85\x9B\x6A"
+			  "\x08\x3E\xBE\x59\xFB\xE4\x96\x34"
+			  "\xB4\x05\x49\x1A\x97\x43\xAD\xA0"
+			  "\xA9\x1E\x6E\x74\xF1\x94\xEC\xA8"
+			  "\xB5\x8A\x20\xEA\x89\x6B\x19\xAA"
+			  "\xA7\xF1\x33\x67\x90\x23\x0D\xEE"
+			  "\x81\xD5\x78\x4F\xD3\x63\xEA\x46"
+			  "\xB5\xB2\x6E\xBB\xCA\x76\x06\x10"
+			  "\x96\x2A\x0A\xBA\xF9\x41\x5A\x1D"
+			  "\x36\x7C\x56\x14\x54\x83\xFA\xA1"
+			  "\x27\xDD\xBA\x8A\x90\x29\xD6\xA6"
+			  "\xFA\x48\x3E\x1E\x23\x6E\x98\xA8"
+			  "\xA7\xD9\x67\x92\x5C\x13\xB4\x71"
+			  "\xA8\xAA\x89\x4A\xA4\xB3\x49\x7C"
+			  "\x7D\x7F\xCE\x6F\x29\x2E\x7E\x37"
+			  "\xC8\x52\x60\xD9\xE7\xCA\x60\x98"
+			  "\xED\xCD\xE8\x60\x83\xAD\x34\x4D"
+			  "\x96\x4A\x99\x2B\xB7\x14\x75\x66"
+			  "\x6C\x2C\x1A\xBA\x4B\xBB\x49\x56"
+			  "\xE1\x86\xA2\x0E\xD0\xF0\x07\xD3"
+			  "\x18\x38\x09\x9C\x0E\x8B\x86\x07"
+			  "\x90\x12\x37\x49\x27\x98\x69\x18"
+			  "\xB0\xCC\xFB\xD3\xBD\x04\xA0\x85"
+			  "\x4B\x22\x97\x07\xB6\x97\xE9\x95"
+			  "\x0F\x88\x36\xA9\x44\x00\xC6\xE9"
+			  "\x27\x53\x5C\x5B\x1F\xD3\xE2\xEE"
+			  "\xD0\xCD\x63\x30\xA9\xC0\xDD\x49"
+			  "\xFE\x16\xA4\x07\x0D\xE2\x5D\x97"
+			  "\xDE\x89\xBA\x2E\xF3\xA9\x5E\xBE"
+			  "\x03\x55\x0E\x02\x41\x4A\x45\x06"
+			  "\xBE\xEA\x32\xF2\xDC\x91\x5C\x20"
+			  "\x94\x02\x30\xD2\xFC\x29\xFA\x8E"
+			  "\x34\xA0\x31\xB8\x34\xBA\xAE\x54"
+			  "\xB5\x88\x1F\xDC\x43\xDC\x22\x9F"
+			  "\xDC\xCE\xD3\xFA\xA4\xA8\xBC\x8A"
+			  "\xC7\x5A\x43\x21\xA5\xB1\xDB\xC3"
+			  "\x84\x3B\xB4\x9B\xB5\xA7\xF1\x0A"
+			  "\xB6\x37\x21\x19\x55\xC2\xBD\x99"
+			  "\x49\x24\xBB\x7C\xB3\x8E\xEF\xD2"
+			  "\x3A\xCF\xA0\x31\x28\x0E\x25\xA2"
+			  "\x11\xB4\x18\x17\x1A\x65\x92\x56"
+			  "\xE8\xE0\x52\x9C\x61\x18\x2A\xB1"
+			  "\x1A\x01\x22\x45\x17\x62\x52\x6C"
+			  "\x91\x44\xCF\x98\xC7\xC0\x79\x26"
+			  "\x32\x66\x6F\x23\x7F\x94\x36\x88"
+			  "\x3C\xC9\xD0\xB7\x45\x30\x31\x86"
+			  "\x3D\xC6\xA3\x98\x62\x84\x1A\x8B"
+			  "\x16\x88\xC7\xA3\xE9\x4F\xE0\x86"
+			  "\xA4\x93\xA8\x34\x5A\xCA\xDF\xCA"
+			  "\x46\x38\xD2\xF4\xE0\x2D\x1E\xC9"
+			  "\x7C\xEF\x53\xB7\x60\x72\x41\xBF"
+			  "\x29\x00\x87\x02\xAF\x44\x4C\xB7"
+			  "\x8C\xF5\x3F\x19\xF4\x80\x45\xA7"
+			  "\x15\x5F\xDB\xE9\xB1\x83\xD2\xE6"
+			  "\x1D\x18\x66\x44\x5B\x8F\x14\xEB",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec cast5_ctr_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A",
+		.ctext	= "\xFF\xC4\x2E\x82\x3D\xF8\xA8\x39"
+			  "\x7C\x52\xC4\xD3\xBB\x62\xC6\xA8"
+			  "\x0C",
+		.len	= 17,
+	}, { /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\xFF\xC4\x2E\x82\x3D\xF8\xA8\x39"
+			  "\x7C\x52\xC4\xD3\xBB\x62\xC6\xA8"
+			  "\x0C\x63\xA5\x55\xE3\xF8\x1C\x7F"
+			  "\xDC\x59\xF9\xA0\x52\xAD\x83\xDF"
+			  "\xD5\x3B\x53\x4A\xAA\x1F\x49\x44"
+			  "\xE8\x20\xCC\xF8\x97\xE6\xE0\x3C"
+			  "\x5A\xD2\x83\xEC\xEE\x25\x3F\xCF"
+			  "\x0D\xC2\x79\x80\x99\x6E\xFF\x7B"
+			  "\x64\xB0\x7B\x86\x29\x1D\x9F\x17"
+			  "\x10\xA5\xA5\xEB\x16\x55\x9E\xE3"
+			  "\x88\x18\x52\x56\x48\x58\xD1\x6B"
+			  "\xE8\x74\x6E\x48\xB0\x2E\x69\x63"
+			  "\x32\xAA\xAC\x26\x55\x45\x94\xDE"
+			  "\x30\x26\x26\xE6\x08\x82\x2F\x5F"
+			  "\xA7\x15\x94\x07\x75\x2D\xC6\x3A"
+			  "\x1B\xA0\x39\xFB\xBA\xB9\x06\x56"
+			  "\xF6\x9F\xF1\x2F\x9B\xF3\x89\x8B"
+			  "\x08\xC8\x9D\x5E\x6B\x95\x09\xC7"
+			  "\x98\xB7\x62\xA4\x1D\x25\xFA\xC5"
+			  "\x62\xC8\x5D\x6B\xB4\x85\x88\x7F"
+			  "\x3B\x29\xF9\xB4\x32\x62\x69\xBF"
+			  "\x32\xB8\xEB\xFD\x0E\x26\xAA\xA3"
+			  "\x44\x67\x90\x20\xAC\x41\xDF\x43"
+			  "\xC6\xC7\x19\x9F\x2C\x28\x74\xEB"
+			  "\x3E\x7F\x7A\x80\x5B\xE4\x08\x60"
+			  "\xC7\xC9\x71\x34\x44\xCE\x05\xFD"
+			  "\xA8\x91\xA8\x44\x5E\xD3\x89\x2C"
+			  "\xAE\x59\x0F\x07\x88\x79\x53\x26"
+			  "\xAF\xAC\xCB\x1D\x6F\x08\x25\x62"
+			  "\xD0\x82\x65\x66\xE4\x2A\x29\x1C"
+			  "\x9C\x64\x5F\x49\x9D\xF8\x62\xF9"
+			  "\xED\xC4\x13\x52\x75\xDC\xE4\xF9"
+			  "\x68\x0F\x8A\xCD\xA6\x8D\x75\xAA"
+			  "\x49\xA1\x86\x86\x37\x5C\x6B\x3D"
+			  "\x56\xE5\x6F\xBE\x27\xC0\x10\xF8"
+			  "\x3C\x4D\x17\x35\x14\xDC\x1C\xA0"
+			  "\x6E\xAE\xD1\x10\xDD\x83\x06\xC2"
+			  "\x23\xD3\xC7\x27\x15\x04\x2C\x27"
+			  "\xDD\x1F\x2E\x97\x09\x9C\x33\x7D"
+			  "\xAC\x50\x1B\x2E\xC9\x52\x0C\x14"
+			  "\x4B\x78\xC4\xDE\x07\x6A\x12\x02"
+			  "\x6E\xD7\x4B\x91\xB9\x88\x4D\x02"
+			  "\xC3\xB5\x04\xBC\xE0\x67\xCA\x18"
+			  "\x22\xA1\xAE\x9A\x21\xEF\xB2\x06"
+			  "\x35\xCD\xEC\x37\x70\x2D\xFC\x1E"
+			  "\xA8\x31\xE7\xFC\xE5\x8E\x88\x66"
+			  "\x16\xB5\xC8\x45\x21\x37\xBD\x24"
+			  "\xA9\xD5\x36\x12\x9F\x6E\x67\x80"
+			  "\x87\x54\xD5\xAF\x97\xE1\x15\xA7"
+			  "\x11\xF0\x63\x7B\xE1\x44\x14\x1C"
+			  "\x06\x32\x05\x8C\x6C\xDB\x9B\x36"
+			  "\x6A\x6B\xAD\x3A\x27\x55\x20\x4C"
+			  "\x76\x36\x43\xE8\x16\x60\xB5\xF3"
+			  "\xDF\x5A\xC6\xA5\x69\x78\x59\x51"
+			  "\x54\x68\x65\x06\x84\xDE\x3D\xAE"
+			  "\x38\x91\xBD\xCC\xA2\x8A\xEC\xE6"
+			  "\x9E\x83\xAE\x1E\x8E\x34\x5D\xDE"
+			  "\x91\xCE\x8F\xED\x40\xF7\xC8\x8B"
+			  "\x9A\x13\x4C\xAD\x89\x97\x9E\xD1"
+			  "\x91\x01\xD7\x21\x23\x28\x1E\xCC"
+			  "\x8C\x98\xDB\xDE\xFC\x72\x94\xAA"
+			  "\xC0\x0D\x96\xAA\x23\xF8\xFE\x13",
+		.len	= 496,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 496 - 20, 4, 16 },
+	},
+};
+
+/*
+ * ARC4 test vectors from OpenSSL
+ */
+static const struct cipher_testvec arc4_tv_template[] = {
+	{
+		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.klen	= 8,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.ctext	= "\x75\xb7\x87\x80\x99\xe0\xc5\x96",
+		.len	= 8,
+	}, {
+		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.klen	= 8,
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\x74\x94\xc2\xe7\x10\x4b\x08\x79",
+		.len	= 8,
+	}, {
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 8,
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\xde\x18\x89\x41\xa3\x37\x5d\x3a",
+		.len	= 8,
+	}, {
+		.key	= "\xef\x01\x23\x45",
+		.klen	= 4,
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00",
+		.ctext	= "\xd6\xa1\x41\xa7\xec\x3c\x38\xdf"
+			  "\xbd\x61\x5a\x11\x62\xe1\xc7\xba"
+			  "\x36\xb6\x78\x58",
+		.len	= 20,
+	}, {
+		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
+		.klen	= 8,
+		.ptext	= "\x12\x34\x56\x78\x9A\xBC\xDE\xF0"
+			  "\x12\x34\x56\x78\x9A\xBC\xDE\xF0"
+			  "\x12\x34\x56\x78\x9A\xBC\xDE\xF0"
+			  "\x12\x34\x56\x78",
+		.ctext	= "\x66\xa0\x94\x9f\x8a\xf7\xd6\x89"
+			  "\x1f\x7f\x83\x2b\xa8\x33\xc0\x0c"
+			  "\x89\x2e\xbe\x30\x14\x3c\xe2\x87"
+			  "\x40\x01\x1e\xcf",
+		.len	= 28,
+	}, {
+		.key	= "\xef\x01\x23\x45",
+		.klen	= 4,
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00",
+		.ctext	= "\xd6\xa1\x41\xa7\xec\x3c\x38\xdf"
+			  "\xbd\x61",
+		.len	= 10,
+	}, {
+		.key	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
+			"\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 16,
+		.ptext	= "\x01\x23\x45\x67\x89\xAB\xCD\xEF",
+		.ctext	= "\x69\x72\x36\x59\x1B\x52\x42\xB1",
+		.len	= 8,
+	},
+};
+
+/*
+ * TEA test vectors
+ */
+static const struct cipher_testvec tea_tv_template[] = {
+	{
+		.key    = zeroed_string,
+		.klen	= 16,
+		.ptext	= zeroed_string,
+		.ctext	= "\x0a\x3a\xea\x41\x40\xa9\xba\x94",
+		.len	= 8,
+	}, {
+		.key	= "\x2b\x02\x05\x68\x06\x14\x49\x76"
+			  "\x77\x5d\x0e\x26\x6c\x28\x78\x43",
+		.klen	= 16,
+		.ptext	= "\x74\x65\x73\x74\x20\x6d\x65\x2e",
+		.ctext	= "\x77\x5d\x2a\x6a\xf6\xce\x92\x09",
+		.len	= 8,
+	}, {
+		.key	= "\x09\x65\x43\x11\x66\x44\x39\x25"
+			  "\x51\x3a\x16\x10\x0a\x08\x12\x6e",
+		.klen	= 16,
+		.ptext	= "\x6c\x6f\x6e\x67\x65\x72\x5f\x74"
+			  "\x65\x73\x74\x5f\x76\x65\x63\x74",
+		.ctext	= "\xbe\x7a\xbb\x81\x95\x2d\x1f\x1e"
+			  "\xdd\x89\xa1\x25\x04\x21\xdf\x95",
+		.len	= 16,
+	}, {
+		.key	= "\x4d\x76\x32\x17\x05\x3f\x75\x2c"
+			  "\x5d\x04\x16\x36\x15\x72\x63\x2f",
+		.klen	= 16,
+		.ptext	= "\x54\x65\x61\x20\x69\x73\x20\x67"
+			  "\x6f\x6f\x64\x20\x66\x6f\x72\x20"
+			  "\x79\x6f\x75\x21\x21\x21\x20\x72"
+			  "\x65\x61\x6c\x6c\x79\x21\x21\x21",
+		.ctext	= "\xe0\x4d\x5d\x3c\xb7\x8c\x36\x47"
+			  "\x94\x18\x95\x91\xa9\xfc\x49\xf8"
+			  "\x44\xd1\x2d\xc2\x99\xb8\x08\x2a"
+			  "\x07\x89\x73\xc2\x45\x92\xc6\x90",
+		.len	= 32,
+	}
+};
+
+/*
+ * XTEA test vectors
+ */
+static const struct cipher_testvec xtea_tv_template[] = {
+	{
+		.key    = zeroed_string,
+		.klen	= 16,
+		.ptext	= zeroed_string,
+		.ctext	= "\xd8\xd4\xe9\xde\xd9\x1e\x13\xf7",
+		.len	= 8,
+	}, {
+		.key	= "\x2b\x02\x05\x68\x06\x14\x49\x76"
+			  "\x77\x5d\x0e\x26\x6c\x28\x78\x43",
+		.klen	= 16,
+		.ptext	= "\x74\x65\x73\x74\x20\x6d\x65\x2e",
+		.ctext	= "\x94\xeb\xc8\x96\x84\x6a\x49\xa8",
+		.len	= 8,
+	}, {
+		.key	= "\x09\x65\x43\x11\x66\x44\x39\x25"
+			  "\x51\x3a\x16\x10\x0a\x08\x12\x6e",
+		.klen	= 16,
+		.ptext	= "\x6c\x6f\x6e\x67\x65\x72\x5f\x74"
+			  "\x65\x73\x74\x5f\x76\x65\x63\x74",
+		.ctext	= "\x3e\xce\xae\x22\x60\x56\xa8\x9d"
+			  "\x77\x4d\xd4\xb4\x87\x24\xe3\x9a",
+		.len	= 16,
+	}, {
+		.key	= "\x4d\x76\x32\x17\x05\x3f\x75\x2c"
+			  "\x5d\x04\x16\x36\x15\x72\x63\x2f",
+		.klen	= 16,
+		.ptext	= "\x54\x65\x61\x20\x69\x73\x20\x67"
+			  "\x6f\x6f\x64\x20\x66\x6f\x72\x20"
+			  "\x79\x6f\x75\x21\x21\x21\x20\x72"
+			  "\x65\x61\x6c\x6c\x79\x21\x21\x21",
+		.ctext	= "\x99\x81\x9f\x5d\x6f\x4b\x31\x3a"
+			  "\x86\xff\x6f\xd0\xe3\x87\x70\x07"
+			  "\x4d\xb8\xcf\xf3\x99\x50\xb3\xd4"
+			  "\x73\xa2\xfa\xc9\x16\x59\x5d\x81",
+		.len	= 32,
+	}
+};
+
+/*
+ * KHAZAD test vectors.
+ */
+static const struct cipher_testvec khazad_tv_template[] = {
+	{
+		.key	= "\x80\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 16,
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\x49\xa4\xce\x32\xac\x19\x0e\x3f",
+		.len	= 8,
+	}, {
+		.key	= "\x38\x38\x38\x38\x38\x38\x38\x38"
+			  "\x38\x38\x38\x38\x38\x38\x38\x38",
+		.klen	= 16,
+		.ptext	= "\x38\x38\x38\x38\x38\x38\x38\x38",
+		.ctext	= "\x7e\x82\x12\xa1\xd9\x5b\xe4\xf9",
+		.len	= 8,
+	}, {
+		.key	= "\xa2\xa2\xa2\xa2\xa2\xa2\xa2\xa2"
+			"\xa2\xa2\xa2\xa2\xa2\xa2\xa2\xa2",
+		.klen	= 16,
+		.ptext	= "\xa2\xa2\xa2\xa2\xa2\xa2\xa2\xa2",
+		.ctext	= "\xaa\xbe\xc1\x95\xc5\x94\x1a\x9c",
+		.len	= 8,
+	}, {
+		.key	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f"
+			"\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
+		.klen	= 16,
+		.ptext	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
+		.ctext	= "\x04\x74\xf5\x70\x50\x16\xd3\xb8",
+		.len	= 8,
+	}, {
+		.key	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f"
+			"\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
+		.klen	= 16,
+		.ptext	= "\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f"
+			"\x2f\x2f\x2f\x2f\x2f\x2f\x2f\x2f",
+		.ctext	= "\x04\x74\xf5\x70\x50\x16\xd3\xb8"
+			"\x04\x74\xf5\x70\x50\x16\xd3\xb8",
+		.len	= 16,
 	},
 };
 
-static const struct cipher_testvec camellia_ctr_dec_tv_template[] = {
-	{ /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+/*
+ * Anubis test vectors.
+ */
+
+static const struct cipher_testvec anubis_tv_template[] = {
+	{
+		.key	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
+			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
+		.klen	= 16,
+		.ptext	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
+			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
+		.ctext	= "\x6d\xc5\xda\xa2\x26\x7d\x62\x6f"
+			  "\x08\xb7\x52\x8e\x6e\x6e\x86\x90",
+		.len	= 16,
+	}, {
+
+		.key	= "\x03\x03\x03\x03\x03\x03\x03\x03"
+			  "\x03\x03\x03\x03\x03\x03\x03\x03"
+			  "\x03\x03\x03\x03",
+		.klen	= 20,
+		.ptext	= "\x03\x03\x03\x03\x03\x03\x03\x03"
+			  "\x03\x03\x03\x03\x03\x03\x03\x03",
+		.ctext	= "\xdb\xf1\x42\xf4\xd1\x8a\xc7\x49"
+			  "\x87\x41\x6f\x82\x0a\x98\x64\xae",
+		.len	= 16,
+	}, {
+		.key	= "\x24\x24\x24\x24\x24\x24\x24\x24"
+			  "\x24\x24\x24\x24\x24\x24\x24\x24"
+			  "\x24\x24\x24\x24\x24\x24\x24\x24"
+			  "\x24\x24\x24\x24",
+		.klen	= 28,
+		.ptext	= "\x24\x24\x24\x24\x24\x24\x24\x24"
+			  "\x24\x24\x24\x24\x24\x24\x24\x24",
+		.ctext	= "\xfd\x1b\x4a\xe3\xbf\xf0\xad\x3d"
+			  "\x06\xd3\x61\x27\xfd\x13\x9e\xde",
+		.len	= 16,
+	}, {
+		.key	= "\x25\x25\x25\x25\x25\x25\x25\x25"
+			  "\x25\x25\x25\x25\x25\x25\x25\x25"
+			  "\x25\x25\x25\x25\x25\x25\x25\x25"
+			  "\x25\x25\x25\x25\x25\x25\x25\x25",
 		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\xF3\x06\x3A\x84\xCD\xBA\x8E\x11"
-			  "\xB7\x74\x6F\x5C\x97\xFB\x36\xFE"
-			  "\xDE\x71\x58\xD4\x15\xD1\xC1\xA4"
-			  "\xC9\x28\x74\xA6\x6B\xC7\x95\xA6"
-			  "\x6C\x77\xF7\x2F\xDF\xC7\xBB\x85"
-			  "\x60\xFC\xE8\x94\xE8\xB5\x09\x2C"
-			  "\x1E\x43\xEF\x6C\xE9\x98\xC5\xA0"
-			  "\x7B\x13\xE5\x7F\xF8\x49\x9A\x8C"
-			  "\xE6\x7B\x08\xC3\x32\x66\x55\x4E"
-			  "\xA5\x44\x1D\x2C\x18\xC7\x29\x1F"
-			  "\x61\x28\x4A\xE3\xCD\xE5\x47\xB2"
-			  "\x82\x2F\x66\x83\x91\x51\xAE\xD7"
-			  "\x1C\x91\x3C\x57\xE3\x1D\x5A\xC9"
-			  "\xFD\xC5\x58\x58\xEF\xCC\x33\xC9"
-			  "\x0F\xEA\x26\x32\xD1\x15\x19\x2D"
-			  "\x25\xB4\x7F\xB0\xDF\xFB\x88\x60"
-			  "\x4E\x4D\x06\x7D\xCC\x1F\xED\x3B"
-			  "\x68\x84\xD5\xB3\x1B\xE7\xB9\xA1"
-			  "\x68\x8B\x2C\x1A\x44\xDA\x63\xD3"
-			  "\x29\xE9\x59\x32\x1F\x30\x1C\x43"
-			  "\xEA\x3A\xA3\x6B\x54\x3C\xAA\x11"
-			  "\xAD\x38\x20\xC9\xB9\x8A\x64\x66"
-			  "\x5A\x07\x49\xDF\xA1\x9C\xF9\x76"
-			  "\x36\x65\xB6\x81\x8F\x76\x09\xE5"
-			  "\xEB\xD1\x29\xA4\xE4\xF4\x4C\xCD"
-			  "\xAF\xFC\xB9\x16\xD9\xC3\x73\x6A"
-			  "\x33\x12\xF8\x7E\xBC\xCC\x7D\x80"
-			  "\xBF\x3C\x25\x06\x13\x84\xFA\x35"
-			  "\xF7\x40\xFA\xA1\x44\x13\x70\xD8"
-			  "\x01\xF9\x85\x15\x63\xEC\x7D\xB9"
-			  "\x02\xD8\xBA\x41\x6C\x92\x68\x66"
-			  "\x95\xDD\xD6\x42\xE7\xBB\xE1\xFD"
-			  "\x28\x3E\x94\xB6\xBD\xA7\xBF\x47"
-			  "\x58\x8D\xFF\x19\x30\x75\x0D\x48"
-			  "\x94\xE9\xA6\xCD\xB3\x8E\x1E\xCD"
-			  "\x59\xBC\x1A\xAC\x3C\x4F\xA9\xEB"
-			  "\xF4\xA7\xE4\x75\x4A\x18\x40\xC9"
-			  "\x1E\xEC\x06\x9C\x28\x4B\xF7\x2B"
-			  "\xE2\xEF\xD6\x42\x2E\xBB\xFC\x0A"
-			  "\x79\xA2\x99\x28\x93\x1B\x00\x57"
-			  "\x35\x1E\x1A\x93\x90\xA4\x68\x95"
-			  "\x5E\x57\x40\xD5\xA9\xAA\x19\x48"
-			  "\xEC\xFF\x76\x77\xDC\x78\x89\x76"
-			  "\xE5\x3B\x00\xEC\x58\x4D\xD1\xE3"
-			  "\xC8\x6C\x2C\x45\x5E\x5F\xD9\x4E"
-			  "\x71\xA5\x36\x6D\x03\xF1\xC7\xD5"
-			  "\xF3\x63\xC0\xD8\xCB\x2B\xF1\xA8"
-			  "\xB9\x2B\xE6\x0B\xB9\x65\x78\xA0"
-			  "\xC4\x46\xE6\x9B\x8B\x43\x2D\xAB"
-			  "\x70\xA6\xE0\x59\x1E\xAC\x9D\xE0"
-			  "\x76\x44\x45\xF3\x24\x11\x57\x98"
-			  "\x9A\x86\xB4\x12\x80\x28\x86\x20"
-			  "\x23\x9D\x2D\xE9\x38\x32\xB1\xE1"
-			  "\xCF\x0A\x23\x73\x7D\xC5\x80\x3D"
-			  "\x9F\x6D\xA0\xD0\xEE\x93\x8A\x79"
-			  "\x3A\xDD\x1D\xBB\x9E\x26\x5D\x01"
-			  "\x44\xD0\xD4\x4E\xC3\xF1\xE4\x38"
-			  "\x09\x62\x0A\x1A\x4E\xD2\x63\x0F"
-			  "\x6E\x3E\xD2\xA4\x3A\xF4\xF3\xFF"
-			  "\x7E\x42\xEC\xB6\x6F\x4D\x6B\x48"
-			  "\xE6\xA6\x50\x80\x78\x9E\xF1\xB0"
-			  "\x4D\xB2\x0D\x3D\xFC\x40\x25\x4D",
-		.ilen	= 496,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.ptext	= "\x25\x25\x25\x25\x25\x25\x25\x25"
+			  "\x25\x25\x25\x25\x25\x25\x25\x25",
+		.ctext	= "\x1a\x91\xfb\x2b\xb7\x78\x6b\xc4"
+			"\x17\xd9\xff\x40\x3b\x0e\xe5\xfe",
+		.len	= 16,
+	}, {
+		.key	= "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35",
+		.klen	= 40,
+		.ptext	= "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35",
+		.ctext	= "\xa5\x2c\x85\x6f\x9c\xba\xa0\x97"
+			  "\x9e\xc6\x84\x0f\x17\x21\x07\xee",
+		.len	= 16,
+	},
+};
+
+static const struct cipher_testvec anubis_cbc_tv_template[] = {
+	{
+		.key	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
+			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
+		.klen	= 16,
+		.ptext	= "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
+			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
+			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
+			  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe",
+		.ctext	= "\x6d\xc5\xda\xa2\x26\x7d\x62\x6f"
+			  "\x08\xb7\x52\x8e\x6e\x6e\x86\x90"
+			  "\x86\xd8\xb5\x6f\x98\x5e\x8a\x66"
+			  "\x4f\x1f\x78\xa1\xbb\x37\xf1\xbe",
+		.len	= 32,
+	}, {
+		.key	= "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35",
+		.klen	= 40,
+		.ptext	= "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35"
+			  "\x35\x35\x35\x35\x35\x35\x35\x35",
+		.ctext	= "\xa5\x2c\x85\x6f\x9c\xba\xa0\x97"
+			  "\x9e\xc6\x84\x0f\x17\x21\x07\xee"
+			  "\xa2\xbc\x06\x98\xc6\x4b\xda\x75"
+			  "\x2e\xaa\xbe\x58\xce\x01\x5b\xc7",
+		.len	= 32,
+	},
+};
+
+/*
+ * XETA test vectors
+ */
+static const struct cipher_testvec xeta_tv_template[] = {
+	{
+		.key    = zeroed_string,
+		.klen	= 16,
+		.ptext	= zeroed_string,
+		.ctext	= "\xaa\x22\x96\xe5\x6c\x61\xf3\x45",
+		.len	= 8,
+	}, {
+		.key	= "\x2b\x02\x05\x68\x06\x14\x49\x76"
+			  "\x77\x5d\x0e\x26\x6c\x28\x78\x43",
+		.klen	= 16,
+		.ptext	= "\x74\x65\x73\x74\x20\x6d\x65\x2e",
+		.ctext	= "\x82\x3e\xeb\x35\xdc\xdd\xd9\xc3",
+		.len	= 8,
+	}, {
+		.key	= "\x09\x65\x43\x11\x66\x44\x39\x25"
+			  "\x51\x3a\x16\x10\x0a\x08\x12\x6e",
+		.klen	= 16,
+		.ptext	= "\x6c\x6f\x6e\x67\x65\x72\x5f\x74"
+			  "\x65\x73\x74\x5f\x76\x65\x63\x74",
+		.ctext	= "\xe2\x04\xdb\xf2\x89\x85\x9e\xea"
+			  "\x61\x35\xaa\xed\xb5\xcb\x71\x2c",
+		.len	= 16,
+	}, {
+		.key	= "\x4d\x76\x32\x17\x05\x3f\x75\x2c"
+			  "\x5d\x04\x16\x36\x15\x72\x63\x2f",
+		.klen	= 16,
+		.ptext	= "\x54\x65\x61\x20\x69\x73\x20\x67"
+			  "\x6f\x6f\x64\x20\x66\x6f\x72\x20"
+			  "\x79\x6f\x75\x21\x21\x21\x20\x72"
+			  "\x65\x61\x6c\x6c\x79\x21\x21\x21",
+		.ctext	= "\x0b\x03\xcd\x8a\xbe\x95\xfd\xb1"
+			  "\xc1\x44\x91\x0b\xa5\xc9\x1b\xb4"
+			  "\xa9\xda\x1e\x9e\xb1\x3e\x2a\x8f"
+			  "\xea\xa5\x6a\x85\xd1\xf4\xa8\xa5",
+		.len	= 32,
+	}
+};
+
+/*
+ * FCrypt test vectors
+ */
+static const struct cipher_testvec fcrypt_pcbc_tv_template[] = {
+	{ /* http://www.openafs.org/pipermail/openafs-devel/2000-December/005320.html */
+		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.klen	= 8,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ctext	= "\x0E\x09\x00\xC7\x3E\xF7\xED\x41",
+		.len	= 8,
+	}, {
+		.key	= "\x11\x44\x77\xAA\xDD\x00\x33\x66",
+		.klen	= 8,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x12\x34\x56\x78\x9A\xBC\xDE\xF0",
+		.ctext	= "\xD8\xED\x78\x74\x77\xEC\x06\x80",
+		.len	= 8,
+	}, { /* From Arla */
+		.key	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
+		.klen	= 8,
+		.iv	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.ptext	= "The quick brown fox jumps over the lazy dogs.\0\0",
+		.ctext	= "\x00\xf0\x0e\x11\x75\xe6\x23\x82"
+			  "\xee\xac\x98\x62\x44\x51\xe4\x84"
+			  "\xc3\x59\xd8\xaa\x64\x60\xae\xf7"
+			  "\xd2\xd9\x13\x79\x72\xa3\x45\x03"
+			  "\x23\xb5\x62\xd7\x0c\xf5\x27\xd1"
+			  "\xf8\x91\x3c\xac\x44\x22\x92\xef",
+		.len	= 48,
+	}, {
+		.key	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.klen	= 8,
+		.iv	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
+		.ptext	= "The quick brown fox jumps over the lazy dogs.\0\0",
+		.ctext	= "\xca\x90\xf5\x9d\xcb\xd4\xd2\x3c"
+			  "\x01\x88\x7f\x3e\x31\x6e\x62\x9d"
+			  "\xd8\xe0\x57\xa3\x06\x3a\x42\x58"
+			  "\x2a\x28\xfe\x72\x52\x2f\xdd\xe0"
+			  "\x19\x89\x09\x1c\x2a\x8e\x8c\x94"
+			  "\xfc\xc7\x68\xe4\x88\xaa\xde\x0f",
+		.len	= 48,
+	}, { /* split-page version */
+		.key	= "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.klen	= 8,
+		.iv	= "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87",
+		.ptext	= "The quick brown fox jumps over the lazy dogs.\0\0",
+		.ctext	= "\xca\x90\xf5\x9d\xcb\xd4\xd2\x3c"
+			  "\x01\x88\x7f\x3e\x31\x6e\x62\x9d"
+			  "\xd8\xe0\x57\xa3\x06\x3a\x42\x58"
+			  "\x2a\x28\xfe\x72\x52\x2f\xdd\xe0"
+			  "\x19\x89\x09\x1c\x2a\x8e\x8c\x94"
+			  "\xfc\xc7\x68\xe4\x88\xaa\xde\x0f",
+		.len	= 48,
+		.np	= 2,
+		.tap	= { 20, 28 },
+	}
+};
+
+/*
+ * CAMELLIA test vectors.
+ */
+static const struct cipher_testvec camellia_tv_template[] = {
+	{
+		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.klen	= 16,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.ctext	= "\x67\x67\x31\x38\x54\x96\x69\x73"
+			  "\x08\x57\x06\x56\x48\xea\xbe\x43",
+		.len	= 16,
+	}, {
+		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
+			  "\x00\x11\x22\x33\x44\x55\x66\x77",
+		.klen	= 24,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.ctext	= "\xb4\x99\x34\x01\xb3\xe9\x96\xf8"
+			  "\x4e\xe5\xce\xe7\xd7\x9b\x09\xb9",
+		.len	= 16,
+	}, {
+		.key	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\xfe\xdc\xba\x98\x76\x54\x32\x10"
+			  "\x00\x11\x22\x33\x44\x55\x66\x77"
+			  "\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
+		.klen	= 32,
+		.ptext	= "\x01\x23\x45\x67\x89\xab\xcd\xef"
+			  "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+		.ctext	= "\x9a\xcc\x23\x7d\xff\x16\xd7\x6c"
+			  "\x20\xef\x7c\x91\x9e\x3a\x75\x09",
+		.len	= 16,
+	}, { /* Generated with Crypto++ */
+		.key	= "\x3F\x85\x62\x3F\x1C\xF9\xD6\x1C"
+			  "\xF9\xD6\xB3\x90\x6D\x4A\x90\x6D"
+			  "\x4A\x27\x04\xE1\x27\x04\xE1\xBE"
+			  "\x9B\x78\xBE\x9B\x78\x55\x32\x0F",
+		.klen	= 32,
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -32057,145 +27904,239 @@ static const struct cipher_testvec camellia_ctr_dec_tv_template[] = {
 			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
 			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
-			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
-		.rlen	= 496,
-	}, { /* Generated with Crypto++ */
-		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
-			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
-			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
-			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
-		.klen	= 32,
-		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
-			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
-		.input	= "\xF3\x06\x3A\x84\xCD\xBA\x8E\x11"
-			  "\xB7\x74\x6F\x5C\x97\xFB\x36\xFE"
-			  "\xDE\x71\x58\xD4\x15\xD1\xC1\xA4"
-			  "\xC9\x28\x74\xA6\x6B\xC7\x95\xA6"
-			  "\x6C\x77\xF7\x2F\xDF\xC7\xBB\x85"
-			  "\x60\xFC\xE8\x94\xE8\xB5\x09\x2C"
-			  "\x1E\x43\xEF\x6C\xE9\x98\xC5\xA0"
-			  "\x7B\x13\xE5\x7F\xF8\x49\x9A\x8C"
-			  "\xE6\x7B\x08\xC3\x32\x66\x55\x4E"
-			  "\xA5\x44\x1D\x2C\x18\xC7\x29\x1F"
-			  "\x61\x28\x4A\xE3\xCD\xE5\x47\xB2"
-			  "\x82\x2F\x66\x83\x91\x51\xAE\xD7"
-			  "\x1C\x91\x3C\x57\xE3\x1D\x5A\xC9"
-			  "\xFD\xC5\x58\x58\xEF\xCC\x33\xC9"
-			  "\x0F\xEA\x26\x32\xD1\x15\x19\x2D"
-			  "\x25\xB4\x7F\xB0\xDF\xFB\x88\x60"
-			  "\x4E\x4D\x06\x7D\xCC\x1F\xED\x3B"
-			  "\x68\x84\xD5\xB3\x1B\xE7\xB9\xA1"
-			  "\x68\x8B\x2C\x1A\x44\xDA\x63\xD3"
-			  "\x29\xE9\x59\x32\x1F\x30\x1C\x43"
-			  "\xEA\x3A\xA3\x6B\x54\x3C\xAA\x11"
-			  "\xAD\x38\x20\xC9\xB9\x8A\x64\x66"
-			  "\x5A\x07\x49\xDF\xA1\x9C\xF9\x76"
-			  "\x36\x65\xB6\x81\x8F\x76\x09\xE5"
-			  "\xEB\xD1\x29\xA4\xE4\xF4\x4C\xCD"
-			  "\xAF\xFC\xB9\x16\xD9\xC3\x73\x6A"
-			  "\x33\x12\xF8\x7E\xBC\xCC\x7D\x80"
-			  "\xBF\x3C\x25\x06\x13\x84\xFA\x35"
-			  "\xF7\x40\xFA\xA1\x44\x13\x70\xD8"
-			  "\x01\xF9\x85\x15\x63\xEC\x7D\xB9"
-			  "\x02\xD8\xBA\x41\x6C\x92\x68\x66"
-			  "\x95\xDD\xD6\x42\xE7\xBB\xE1\xFD"
-			  "\x28\x3E\x94\xB6\xBD\xA7\xBF\x47"
-			  "\x58\x8D\xFF\x19\x30\x75\x0D\x48"
-			  "\x94\xE9\xA6\xCD\xB3\x8E\x1E\xCD"
-			  "\x59\xBC\x1A\xAC\x3C\x4F\xA9\xEB"
-			  "\xF4\xA7\xE4\x75\x4A\x18\x40\xC9"
-			  "\x1E\xEC\x06\x9C\x28\x4B\xF7\x2B"
-			  "\xE2\xEF\xD6\x42\x2E\xBB\xFC\x0A"
-			  "\x79\xA2\x99\x28\x93\x1B\x00\x57"
-			  "\x35\x1E\x1A\x93\x90\xA4\x68\x95"
-			  "\x5E\x57\x40\xD5\xA9\xAA\x19\x48"
-			  "\xEC\xFF\x76\x77\xDC\x78\x89\x76"
-			  "\xE5\x3B\x00\xEC\x58\x4D\xD1\xE3"
-			  "\xC8\x6C\x2C\x45\x5E\x5F\xD9\x4E"
-			  "\x71\xA5\x36\x6D\x03\xF1\xC7\xD5"
-			  "\xF3\x63\xC0\xD8\xCB\x2B\xF1\xA8"
-			  "\xB9\x2B\xE6\x0B\xB9\x65\x78\xA0"
-			  "\xC4\x46\xE6\x9B\x8B\x43\x2D\xAB"
-			  "\x70\xA6\xE0\x59\x1E\xAC\x9D\xE0"
-			  "\x76\x44\x45\xF3\x24\x11\x57\x98"
-			  "\x9A\x86\xB4\x12\x80\x28\x86\x20"
-			  "\x23\x9D\x2D\xE9\x38\x32\xB1\xE1"
-			  "\xCF\x0A\x23\x73\x7D\xC5\x80\x3D"
-			  "\x9F\x6D\xA0\xD0\xEE\x93\x8A\x79"
-			  "\x3A\xDD\x1D\xBB\x9E\x26\x5D\x01"
-			  "\x44\xD0\xD4\x4E\xC3\xF1\xE4\x38"
-			  "\x09\x62\x0A\x1A\x4E\xD2\x63\x0F"
-			  "\x6E\x3E\xD2\xA4\x3A\xF4\xF3\xFF"
-			  "\x7E\x42\xEC\xB6\x6F\x4D\x6B\x48"
-			  "\xE6\xA6\x50\x80\x78\x9E\xF1\xB0"
-			  "\x4D\xB2\x0D\x3D\xFC\x40\x25\x4D"
-			  "\x93\x11\x1C\xE9\xD2\x9F\x6E\x90"
-			  "\xE5\x41\x4A\xE2\x3C\x45\x29\x35"
-			  "\xEC\xD6\x47\x50\xCB\x7B\xA2\x32"
-			  "\xF7\x8B\x62\xF1\xE3\x9A\xFE\xC7"
-			  "\x1D\x8C\x02\x72\x68\x09\xE9\xB6"
-			  "\x4A\x80\xE6\xB1\x56\xDF\x90\xD4"
-			  "\x93\x74\xA4\xCE\x20\x23\xBF\x48"
-			  "\xA5\xDE\x1B\xFA\x40\x69\x31\x98"
-			  "\x62\x6E\xA5\xC7\xBF\x0C\x62\xE5"
-			  "\x6D\xE1\x93\xF1\x83\x10\x1C\xCA"
-			  "\xF6\x5C\x19\xF8\x90\x78\xCB\xE4"
-			  "\x0B\x3A\xB5\xF8\x43\x86\xD3\x3F"
-			  "\xBA\x83\x34\x3C\x42\xCC\x7D\x28"
-			  "\x29\x63\x4F\xD8\x02\x17\xC5\x07"
-			  "\x2C\xA4\xAC\x79\xCB\xC3\xA9\x09"
-			  "\x81\x45\x18\xED\xE4\xCB\x42\x3B"
-			  "\x87\x2D\x23\xDC\xC5\xBA\x45\xBD"
-			  "\x92\xE5\x02\x97\x96\xCE\xAD\xEC"
-			  "\xBA\xD8\x76\xF8\xCA\xC1\x31\xEC"
-			  "\x1E\x4F\x3F\x83\xF8\x33\xE8\x6E"
-			  "\xCC\xF8\x5F\xDD\x65\x50\x99\x69"
-			  "\xAF\x48\xCE\xA5\xBA\xB6\x14\x9F"
-			  "\x05\x93\xB2\xE6\x59\xC8\x28\xFE"
-			  "\x8F\x37\xF9\x64\xB9\xA5\x56\x8F"
-			  "\xF1\x1B\x90\xEF\xAE\xEB\xFC\x09"
-			  "\x11\x7A\xF2\x19\x0A\x0A\x9A\x3C"
-			  "\xE2\x5E\x29\xFA\x31\x9B\xC1\x74"
-			  "\x1E\x10\x3E\x07\xA9\x31\x6D\xF8"
-			  "\x81\xF5\xD5\x8A\x04\x23\x51\xAC"
-			  "\xA2\xE2\x63\xFD\x27\x1F\x79\x5B"
-			  "\x1F\xE8\xDA\x11\x49\x4D\x1C\xBA"
-			  "\x54\xCC\x0F\xBA\x92\x69\xE5\xCB"
-			  "\x41\x1A\x67\xA6\x40\x82\x70\x8C"
-			  "\x19\x79\x08\xA4\x51\x20\x7D\xC9"
-			  "\x12\x27\xAE\x20\x0D\x2C\xA1\x6D"
-			  "\xF4\x55\xD4\xE7\xE6\xD4\x28\x08"
-			  "\x00\x70\x12\x56\x56\x50\xAD\x14"
-			  "\x5C\x3E\xA2\xD1\x36\x3F\x36\x48"
-			  "\xED\xB1\x57\x3E\x5D\x15\xF6\x1E"
-			  "\x53\xE9\xA4\x3E\xED\x7D\xCF\x7D"
-			  "\x29\xAF\xF3\x1E\x51\xA8\x9F\x85"
-			  "\x8B\xF0\xBB\xCE\xCC\x39\xC3\x64"
-			  "\x4B\xF2\xAD\x70\x19\xD4\x44\x8F"
-			  "\x91\x76\xE8\x15\x66\x34\x9F\xF6"
-			  "\x0F\x15\xA4\xA8\x24\xF8\x58\xB1"
-			  "\x38\x46\x47\xC7\x9B\xCA\xE9\x42"
-			  "\x44\xAA\xE6\xB5\x9C\x91\xA4\xD3"
-			  "\x16\xA0\xED\x42\xBE\xB5\x06\x19"
-			  "\xBE\x67\xE8\xBC\x22\x32\xA4\x1E"
-			  "\x93\xEB\xBE\xE9\xE1\x93\xE5\x31"
-			  "\x3A\xA2\x75\xDF\xE3\x6B\xE7\xCC"
-			  "\xB4\x70\x20\xE0\x6D\x82\x7C\xC8"
-			  "\x94\x5C\x5E\x37\x18\xAD\xED\x8B"
-			  "\x44\x86\xCA\x5E\x07\xB7\x70\x8D"
-			  "\x40\x48\x19\x73\x7C\x78\x64\x0B"
-			  "\xDB\x01\xCA\xAE\x63\x19\xE9\xD1"
-			  "\x6B\x2C\x84\x10\x45\x42\x2E\xC3"
-			  "\xDF\x7F\xAA\xE8\x87\x1B\x63\x46"
-			  "\x74\x28\x9D\x05\x30\x20\x62\x41"
-			  "\xC0\x9F\x2C\x36\x2B\x78\xD7\x26"
-			  "\xDF\x58\x51\xED\xFA\xDC\x87\x79"
-			  "\xBF\x8C\xBF\xC4\x0F\xE5\x05\xDA"
-			  "\x45\xE3\x35\x0D\x69\x91\x54\x1C"
-			  "\xE7\x2C\x49\x08\x8B\x72\xFA\x5C"
-			  "\xF1\x6B\xD9",
-		.ilen	= 1011,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
+			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06"
+			  "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78"
+			  "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA"
+			  "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C"
+			  "\xF3\x67\xFE\x95\x09\xA0\x37\xCE"
+			  "\x42\xD9\x70\x07\x7B\x12\xA9\x1D"
+			  "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F"
+			  "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01"
+			  "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73"
+			  "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5"
+			  "\x59\xF0\x87\x1E\x92\x29\xC0\x34"
+			  "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6"
+			  "\x3D\xD4\x48\xDF\x76\x0D\x81\x18"
+			  "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A"
+			  "\x21\x95\x2C\xC3\x37\xCE\x65\xFC"
+			  "\x70\x07\x9E\x12\xA9\x40\xD7\x4B"
+			  "\xE2\x79\x10\x84\x1B\xB2\x26\xBD"
+			  "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F"
+			  "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1"
+			  "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13"
+			  "\x87\x1E\xB5\x29\xC0\x57\xEE\x62"
+			  "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4"
+			  "\x6B\x02\x76\x0D\xA4\x18\xAF\x46"
+			  "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8"
+			  "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07"
+			  "\x9E\x35\xCC\x40\xD7\x6E\x05\x79"
+			  "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB"
+			  "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D"
+			  "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF"
+			  "\x43\xDA\x71\x08\x7C\x13\xAA\x1E"
+			  "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90"
+			  "\x27\xBE\x32\xC9\x60\xF7\x6B\x02"
+			  "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74"
+			  "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6"
+			  "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35"
+			  "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7"
+			  "\x3E\xD5\x49\xE0\x77\x0E\x82\x19"
+			  "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B"
+			  "\x22\x96\x2D\xC4\x38\xCF\x66\xFD"
+			  "\x71\x08\x9F\x13\xAA\x41\xD8\x4C"
+			  "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE"
+			  "\x55\xEC\x60\xF7\x8E\x02\x99\x30"
+			  "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2"
+			  "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14"
+			  "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63"
+			  "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5"
+			  "\x6C\x03\x77\x0E\xA5\x19\xB0\x47"
+			  "\xDE\x52\xE9\x80\x17\x8B\x22\xB9"
+			  "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08"
+			  "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A"
+			  "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC"
+			  "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E"
+			  "\xF5\x69\x00\x97\x0B\xA2\x39\xD0"
+			  "\x44\xDB\x72\x09\x7D\x14\xAB\x1F"
+			  "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91"
+			  "\x28\xBF\x33\xCA\x61\xF8\x6C\x03"
+			  "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75"
+			  "\x0C\x80\x17\xAE\x22\xB9\x50\xE7"
+			  "\x5B\xF2\x89\x20\x94\x2B\xC2\x36"
+			  "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8"
+			  "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A"
+			  "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C"
+			  "\x00\x97\x2E\xC5\x39\xD0\x67\xFE"
+			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D",
+		.ctext	= "\xED\xCD\xDB\xB8\x68\xCE\xBD\xEA"
+			  "\x9D\x9D\xCD\x9F\x4F\xFC\x4D\xB7"
+			  "\xA5\xFF\x6F\x43\x0F\xBA\x32\x04"
+			  "\xB3\xC2\xB9\x03\xAA\x91\x56\x29"
+			  "\x0D\xD0\xFD\xC4\x65\xA5\x69\xB9"
+			  "\xF1\xF6\xB1\xA5\xB2\x75\x4F\x8A"
+			  "\x8D\x7D\x1B\x9B\xC7\x68\x72\xF8"
+			  "\x01\x9B\x17\x0A\x29\xE7\x61\x28"
+			  "\x7F\xA7\x50\xCA\x20\x2C\x96\x3B"
+			  "\x6E\x5C\x5D\x3F\xB5\x7F\xF3\x2B"
+			  "\x04\xEF\x9D\xD4\xCE\x41\x28\x8E"
+			  "\x83\x54\xAE\x7C\x82\x46\x10\xC9"
+			  "\xC4\x8A\x1E\x1F\x4C\xA9\xFC\xEC"
+			  "\x3C\x8C\x30\xFC\x59\xD2\x54\xC4"
+			  "\x6F\x50\xC6\xCA\x8C\x14\x5B\x9C"
+			  "\x18\x56\x5B\xF8\x33\x0E\x4A\xDB"
+			  "\xEC\xB5\x6E\x5B\x31\xC4\x0E\x98"
+			  "\x9F\x32\xBA\xA2\x18\xCF\x55\x43"
+			  "\xFE\x80\x8F\x60\xCF\x05\x30\x9B"
+			  "\x70\x50\x1E\x9C\x08\x87\xE6\x20"
+			  "\xD2\xF3\x27\xF8\x2A\x8D\x12\xB2"
+			  "\xBC\x5F\xFE\x52\x52\xF6\x7F\xB6"
+			  "\xB8\x30\x86\x3B\x0F\x94\x1E\x79"
+			  "\x13\x94\x35\xA2\xB1\x35\x5B\x05"
+			  "\x2A\x98\x6B\x96\x4C\xB1\x20\xBE"
+			  "\xB6\x14\xC2\x06\xBF\xFD\x5F\x2A"
+			  "\xF5\x33\xC8\x19\x45\x14\x44\x5D"
+			  "\xFE\x94\x7B\xBB\x63\x13\x57\xC3"
+			  "\x2A\x8F\x6C\x11\x2A\x07\xA7\x6A"
+			  "\xBF\x20\xD3\x99\xC6\x00\x0B\xBF"
+			  "\x83\x46\x25\x3A\xB0\xF6\xC5\xC8"
+			  "\x00\xCA\xE5\x28\x4A\x7C\x95\x9C"
+			  "\x7B\x43\xAB\xF9\xE4\xF8\x74\xAB"
+			  "\xA7\xB8\x9C\x0F\x53\x7B\xB6\x74"
+			  "\x60\x64\x0D\x1C\x80\xD1\x20\x9E"
+			  "\xDC\x14\x27\x9B\xFC\xBD\x5C\x96"
+			  "\xD2\x51\xDC\x96\xEE\xE5\xEA\x2B"
+			  "\x02\x7C\xAA\x3C\xDC\x9D\x7B\x01"
+			  "\x20\xC3\xE1\x0B\xDD\xAB\xF3\x1E"
+			  "\x19\xA8\x84\x29\x5F\xCC\xC3\x5B"
+			  "\xE4\x33\x59\xDC\x12\xEB\x2B\x4D"
+			  "\x5B\x55\x23\xB7\x40\x31\xDE\xEE"
+			  "\x18\xC9\x3C\x4D\xBC\xED\xE0\x42"
+			  "\xAD\xDE\xA0\xA3\xC3\xFE\x44\xD3"
+			  "\xE1\x9A\xDA\xAB\x32\xFC\x1A\xBF"
+			  "\x63\xA9\xF0\x6A\x08\x46\xBD\x48"
+			  "\x83\x06\xAB\x82\x99\x01\x16\x1A"
+			  "\x03\x36\xC5\x59\x6B\xB8\x8C\x9F"
+			  "\xC6\x51\x3D\xE5\x7F\xBF\xAB\xBC"
+			  "\xC9\xA1\x88\x34\x5F\xA9\x7C\x3B"
+			  "\x9F\x1B\x98\x2B\x4F\xFB\x9B\xF0"
+			  "\xCD\xB6\x45\xB2\x29\x2E\x34\x23"
+			  "\xA9\x97\xC0\x22\x8C\x42\x9B\x5F"
+			  "\x40\xC8\xD7\x3D\x82\x9A\x6F\xAA"
+			  "\x74\x83\x29\x05\xE8\xC4\x4D\x01"
+			  "\xB5\xE5\x84\x3F\x7F\xD3\xE0\x99"
+			  "\xDA\xE7\x6F\x30\xFD\xAA\x92\x30"
+			  "\xA5\x46\x8B\xA2\xE6\x58\x62\x7C"
+			  "\x2C\x35\x1B\x38\x85\x7D\xE8\xF3"
+			  "\x87\x4F\xDA\xD8\x5F\xFC\xB6\x44"
+			  "\xD0\xE3\x9B\x8B\xBF\xD6\xB8\xC4"
+			  "\x73\xAE\x1D\x8B\x5B\x74\x8B\xCB"
+			  "\xA4\xAD\xCF\x5D\xD4\x58\xC9\xCD"
+			  "\xF7\x90\x68\xCF\xC9\x11\x52\x3E"
+			  "\xE8\xA1\xA3\x78\x8B\xD0\xAC\x0A"
+			  "\xD4\xC9\xA3\xA5\x55\x30\xC8\x3E"
+			  "\xED\x28\x39\xE9\x63\xED\x41\x70"
+			  "\x51\xE3\xC4\xA0\xFC\xD5\x43\xCB"
+			  "\x4D\x65\xC8\xFD\x3A\x91\x8F\x60"
+			  "\x8A\xA6\x6D\x9D\x3E\x01\x23\x4B"
+			  "\x50\x47\xC9\xDC\x9B\xDE\x37\xC5"
+			  "\xBF\x67\xB1\x6B\x78\x38\xD5\x7E"
+			  "\xB6\xFF\x67\x83\x3B\x6E\xBE\x23"
+			  "\x45\xFA\x1D\x69\x44\xFD\xC6\xB9"
+			  "\xD0\x4A\x92\xD1\xBE\xF6\x4A\xB7"
+			  "\xCA\xA8\xA2\x9E\x13\x87\x57\x92"
+			  "\x64\x7C\x85\x0B\xB3\x29\x37\xD8"
+			  "\xE6\xAA\xAF\xC4\x03\x67\xA3\xBF"
+			  "\x2E\x45\x83\xB6\xD8\x54\x00\x89"
+			  "\xF6\xBC\x3A\x7A\x88\x58\x51\xED"
+			  "\xF4\x4E\x01\xA5\xC3\x2E\xD9\x42"
+			  "\xBD\x6E\x0D\x0B\x21\xB0\x1A\xCC"
+			  "\xA4\xD3\x3F\xDC\x9B\x81\xD8\xF1"
+			  "\xEA\x7A\x6A\xB7\x07\xC9\x6D\x91"
+			  "\x6D\x3A\xF5\x5F\xA6\xFF\x87\x1E"
+			  "\x3F\xDD\xC0\x72\xEA\xAC\x08\x15"
+			  "\x21\xE6\xC6\xB6\x0D\xD8\x51\x86"
+			  "\x2A\x03\x73\xF7\x29\xD4\xC4\xE4"
+			  "\x7F\x95\x10\xF7\xAB\x3F\x92\x23"
+			  "\xD3\xCE\x9C\x2E\x46\x3B\x63\x43"
+			  "\xBB\xC2\x82\x7A\x83\xD5\x55\xE2"
+			  "\xE7\x9B\x2F\x92\xAF\xFD\x81\x56"
+			  "\x79\xFD\x3E\xF9\x46\xE0\x25\xD4"
+			  "\x38\xDE\xBC\x2C\xC4\x7A\x2A\x8F"
+			  "\x94\x4F\xD0\xAD\x9B\x37\x18\xD4"
+			  "\x0E\x4D\x0F\x02\x3A\xDC\x5A\xA2"
+			  "\x39\x25\x55\x20\x5A\xA6\x02\x9F"
+			  "\xE6\x77\x21\x77\xE5\x4B\x7B\x0B"
+			  "\x30\xF8\x5F\x33\x0F\x49\xCD\xFF"
+			  "\xF2\xE4\x35\xF9\xF0\x63\xC3\x7E"
+			  "\xF1\xA6\x73\xB4\xDF\xE7\xBB\x78"
+			  "\xFF\x21\xA9\xF3\xF3\xCF\x5D\xBA"
+			  "\xED\x87\x98\xAC\xFE\x48\x97\x6D"
+			  "\xA6\x7F\x69\x31\xB1\xC4\xFF\x14"
+			  "\xC6\x76\xD4\x10\xDD\xF6\x49\x2C"
+			  "\x9C\xC8\x6D\x76\xC0\x8F\x5F\x55"
+			  "\x2F\x3C\x8A\x30\xAA\xC3\x16\x55"
+			  "\xC6\xFC\x8D\x8B\xB9\xE5\x80\x6C"
+			  "\xC8\x7E\xBD\x65\x58\x36\xD5\xBC"
+			  "\xF0\x33\x52\x29\x70\xF9\x5C\xE9"
+			  "\xAC\x1F\xB5\x73\x56\x66\x54\xAF"
+			  "\x1B\x8F\x7D\xED\xAB\x03\xCE\xE3"
+			  "\xAE\x47\xB6\x69\x86\xE9\x01\x31"
+			  "\x83\x18\x3D\xF4\x74\x7B\xF9\x42"
+			  "\x4C\xFD\x75\x4A\x6D\xF0\x03\xA6"
+			  "\x2B\x20\x63\xDA\x49\x65\x5E\x8B"
+			  "\xC0\x19\xE3\x8D\xD9\xF3\xB0\x34"
+			  "\xD3\x52\xFC\x68\x00\x43\x1B\x37"
+			  "\x31\x93\x51\x1C\x63\x97\x70\xB0"
+			  "\x99\x78\x83\x13\xFD\xCF\x53\x81"
+			  "\x36\x46\xB5\x42\x52\x2F\x32\xEB"
+			  "\x4A\x3D\xF1\x8F\x1C\x54\x2E\xFC"
+			  "\x41\x75\x5A\x8C\x8E\x6F\xE7\x1A"
+			  "\xAE\xEF\x3E\x82\x12\x0B\x74\x72"
+			  "\xF8\xB2\xAA\x7A\xD6\xFF\xFA\x55"
+			  "\x33\x1A\xBB\xD3\xA2\x7E\x97\x66",
+		.len	= 1008,
+		.also_non_np = 1,
+		.np	= 3,
+		.tap	= { 1008 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec camellia_cbc_tv_template[] = {
+	{
+		.key    = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
+			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
+		.klen   = 16,
+		.iv	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
+			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
+		.ptext	= "Single block msg",
+		.ctext	= "\xea\x32\x12\x76\x3b\x50\x10\xe7"
+			  "\x18\xf6\xfd\x5d\xf6\x8f\x13\x51",
+		.len	= 16,
+	}, {
+		.key    = "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
+			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
+		.klen   = 16,
+		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
+			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.ctext	= "\xa5\xdf\x6e\x50\xda\x70\x6c\x01"
+			  "\x4a\xab\xf3\xf2\xd6\xfc\x6c\xfd"
+			  "\x19\xb4\x3e\x57\x1c\x02\x5e\xa0"
+			  "\x15\x78\xe0\x5e\xf2\xcb\x87\x16",
+		.len	= 32,
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -32320,148 +28261,549 @@ static const struct cipher_testvec camellia_ctr_dec_tv_template[] = {
 			  "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A"
 			  "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C"
 			  "\x00\x97\x2E\xC5\x39\xD0\x67\xFE"
-			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D"
-			  "\xE4\x7B\x12",
-		.rlen	= 1011,
+			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D",
+		.ctext	= "\xCD\x3E\x2A\x3B\x3E\x94\xC5\x77"
+			  "\xBA\xBB\x5B\xB1\xDE\x7B\xA4\x40"
+			  "\x88\x39\xE3\xFD\x94\x4B\x25\x58"
+			  "\xE1\x4B\xC4\x18\x7A\xFD\x17\x2B"
+			  "\xB9\xF9\xC2\x27\x6A\xB6\x31\x27"
+			  "\xA6\xAD\xEF\xE5\x5D\xE4\x02\x01"
+			  "\x56\x2E\x10\xC2\x2C\xFF\xC6\x83"
+			  "\xB5\xDC\x4F\x63\xAD\x0E\x63\x5E"
+			  "\x56\xC8\x18\x3D\x79\x86\x97\xEF"
+			  "\x57\x0E\x63\xA1\xC1\x41\x48\xB8"
+			  "\x98\xB7\x51\x6D\x18\xF6\x19\x82"
+			  "\x37\x49\x88\xA4\xEF\x91\x21\x47"
+			  "\x03\x28\xEA\x42\xF4\xFB\x7A\x58"
+			  "\x28\x90\x77\x46\xD8\xD2\x35\x16"
+			  "\x44\xA9\x9E\x49\x52\x2A\xE4\x16"
+			  "\x5D\xF7\x65\xEB\x0F\xC9\x29\xE6"
+			  "\xCF\x76\x91\x89\x8A\x94\x39\xFA"
+			  "\x6B\x5F\x63\x53\x74\x43\x91\xF5"
+			  "\x3F\xBC\x88\x53\xB2\x1A\x02\x3F"
+			  "\x9D\x32\x84\xEB\x56\x28\xD6\x06"
+			  "\xD5\xB2\x20\xA9\xFC\xC3\x76\x62"
+			  "\x32\xCC\x86\xC8\x36\x67\x5E\x7E"
+			  "\xA4\xAA\x15\x63\x6B\xA9\x86\xAF"
+			  "\x1A\x52\x82\x36\x5F\xF4\x3F\x7A"
+			  "\x9B\x78\x62\x3B\x02\x28\x60\xB3"
+			  "\xBA\x82\xB1\xDD\xC9\x60\x8F\x47"
+			  "\xF1\x6B\xFE\xE5\x39\x34\xA0\x28"
+			  "\xA4\xB3\xC9\x7E\xED\x28\x8D\x70"
+			  "\xB2\x1D\xFD\xC6\x00\xCF\x1A\x94"
+			  "\x28\xF8\xC1\x34\xB7\x58\xA5\x6C"
+			  "\x1A\x9D\xE4\xE4\xF6\xB9\xB4\xB0"
+			  "\x5D\x51\x54\x9A\x53\xA0\xF9\x32"
+			  "\xBD\x31\x54\x14\x7B\x33\xEE\x17"
+			  "\xD3\xC7\x1F\x48\xBF\x0B\x22\xA2"
+			  "\x7D\x0C\xDF\xD0\x2E\x98\xFA\xD2"
+			  "\xFA\xCF\x24\x1D\x99\x9B\xD0\x7E"
+			  "\xF4\x4F\x88\xFF\x45\x99\x4A\xF4"
+			  "\xF2\x0A\x5B\x3B\x21\xAB\x92\xAE"
+			  "\x40\x78\x91\x95\xC4\x2F\xA3\xE8"
+			  "\x18\xC7\x07\xA6\xC8\xC0\x66\x33"
+			  "\x35\xC0\xB4\xA0\xF8\xEE\x1E\xF3"
+			  "\x40\xF5\x40\x54\xF1\x84\x8C\xEA"
+			  "\x27\x38\x1F\xF8\x77\xC7\xDF\xD8"
+			  "\x1D\xE2\xD9\x59\x40\x4F\x59\xD4"
+			  "\xF8\x17\x99\x8D\x58\x2D\x72\x44"
+			  "\x9D\x1D\x91\x64\xD6\x3F\x0A\x82"
+			  "\xC7\x57\x3D\xEF\xD3\x41\xFA\xA7"
+			  "\x68\xA3\xB8\xA5\x93\x74\x2E\x85"
+			  "\x4C\x9D\x69\x59\xCE\x15\xAE\xBF"
+			  "\x9C\x8F\x14\x64\x5D\x7F\xCF\x0B"
+			  "\xCE\x43\x5D\x28\xC0\x2F\xFB\x18"
+			  "\x79\x9A\xFC\x43\x16\x7C\x6B\x7B"
+			  "\x38\xB8\x48\x36\x66\x4E\x20\x43"
+			  "\xBA\x76\x13\x9A\xC3\xF2\xEB\x52"
+			  "\xD7\xDC\xB2\x67\x63\x14\x25\xCD"
+			  "\xB1\x13\x4B\xDE\x8C\x59\x21\x84"
+			  "\x81\x8D\x97\x23\x45\x33\x7C\xF3"
+			  "\xC5\xBC\x79\x95\xAA\x84\x68\x31"
+			  "\x2D\x1A\x68\xFE\xEC\x92\x94\xDA"
+			  "\x94\x2A\x6F\xD6\xFE\xE5\x76\x97"
+			  "\xF4\x6E\xEE\xCB\x2B\x95\x4E\x36"
+			  "\x5F\x74\x8C\x86\x5B\x71\xD0\x20"
+			  "\x78\x1A\x7F\x18\x8C\xD9\xCD\xF5"
+			  "\x21\x41\x56\x72\x13\xE1\x86\x07"
+			  "\x07\x26\xF3\x4F\x7B\xEA\xB5\x18"
+			  "\xFE\x94\x2D\x9F\xE0\x72\x18\x65"
+			  "\xB2\xA5\x63\x48\xB4\x13\x22\xF7"
+			  "\x25\xF1\x80\xA8\x7F\x54\x86\x7B"
+			  "\x39\xAE\x95\x0C\x09\x32\x22\x2D"
+			  "\x4D\x73\x39\x0C\x09\x2C\x7C\x10"
+			  "\xD0\x4B\x53\xF6\x90\xC5\x99\x2F"
+			  "\x15\xE1\x7F\xC6\xC5\x7A\x52\x14"
+			  "\x65\xEE\x93\x54\xD0\x66\x15\x3C"
+			  "\x4C\x68\xFD\x64\x0F\xF9\x10\x39"
+			  "\x46\x7A\xDD\x97\x20\xEE\xC7\xD2"
+			  "\x98\x4A\xB6\xE6\xF5\xA8\x1F\x4F"
+			  "\xDB\xAB\x6D\xD5\x9B\x34\x16\x97"
+			  "\x2F\x64\xE5\x37\xEF\x0E\xA1\xE9"
+			  "\xBE\x31\x31\x96\x8B\x40\x18\x75"
+			  "\x11\x75\x14\x32\xA5\x2D\x1B\x6B"
+			  "\xDB\x59\xEB\xFA\x3D\x8E\x7C\xC4"
+			  "\xDE\x68\xC8\x9F\xC9\x99\xE3\xC6"
+			  "\x71\xB0\x12\x57\x89\x0D\xC0\x2B"
+			  "\x9F\x12\x6A\x04\x67\xF1\x95\x31"
+			  "\x59\xFD\x84\x95\x2C\x9C\x5B\xEC"
+			  "\x09\xB0\x43\x96\x4A\x64\x80\x40"
+			  "\xB9\x72\x19\xDD\x70\x42\xFA\xB1"
+			  "\x4A\x2C\x0C\x0A\x60\x6E\xE3\x7C"
+			  "\x37\x5A\xBE\xA4\x62\xCF\x29\xAB"
+			  "\x7F\x4D\xA6\xB3\xE2\xB6\x64\xC6"
+			  "\x33\x0B\xF3\xD5\x01\x38\x74\xA4"
+			  "\x67\x1E\x75\x68\xC3\xAD\x76\xE9"
+			  "\xE9\xBC\xF0\xEB\xD8\xFD\x31\x8A"
+			  "\x5F\xC9\x18\x94\x4B\x86\x66\xFC"
+			  "\xBD\x0B\x3D\xB3\x9F\xFA\x1F\xD9"
+			  "\x78\xC4\xE3\x24\x1C\x67\xA2\xF8"
+			  "\x43\xBC\x76\x75\xBF\x6C\x05\xB3"
+			  "\x32\xE8\x7C\x80\xDB\xC7\xB6\x61"
+			  "\x1A\x3E\x2B\xA7\x25\xED\x8F\xA0"
+			  "\x00\x4B\xF8\x90\xCA\xD8\xFB\x12"
+			  "\xAC\x1F\x18\xE9\xD2\x5E\xA2\x8E"
+			  "\xE4\x84\x6B\x9D\xEB\x1E\x6B\xA3"
+			  "\x7B\xDC\xCE\x15\x97\x27\xB2\x65"
+			  "\xBC\x0E\x47\xAB\x55\x13\x53\xAB"
+			  "\x0E\x34\x55\x02\x5F\x27\xC5\x89"
+			  "\xDF\xC5\x70\xC4\xDD\x76\x82\xEE"
+			  "\x68\xA6\x09\xB0\xE5\x5E\xF1\x0C"
+			  "\xE3\xF3\x09\x9B\xFE\x65\x4B\xB8"
+			  "\x30\xEC\xD5\x7C\x6A\xEC\x1D\xD2"
+			  "\x93\xB7\xA1\x1A\x02\xD4\xC0\xD6"
+			  "\x8D\x4D\x83\x9A\xED\x29\x4E\x14"
+			  "\x86\xD5\x3C\x1A\xD5\xB9\x0A\x6A"
+			  "\x72\x22\xD5\x92\x38\xF1\xA1\x86"
+			  "\xB2\x41\x51\xCA\x4E\xAB\x8F\xD3"
+			  "\x80\x56\xC3\xD7\x65\xE1\xB3\x86"
+			  "\xCB\xCE\x98\xA1\xD4\x59\x1C\x06"
+			  "\x01\xED\xF8\x29\x91\x19\x5C\x9A"
+			  "\xEE\x28\x1B\x48\xD7\x32\xEF\x9F"
+			  "\x6C\x2B\x66\x4E\x78\xD5\x8B\x72"
+			  "\x80\xE7\x29\xDC\x23\x55\x98\x54"
+			  "\xB1\xFF\x3E\x95\x56\xA8\x78\x78"
+			  "\xEF\xC4\xA5\x11\x2D\x2B\xD8\x93"
+			  "\x30\x6E\x7E\x51\xBB\x42\x5F\x03"
+			  "\x43\x94\x23\x7E\xEE\xF0\xA5\x79"
+			  "\x55\x01\xD4\x58\xB2\xF2\x85\x49"
+			  "\x70\xC5\xB9\x0B\x3B\x7A\x6E\x6C",
+		.len	= 1008,
 		.also_non_np = 1,
-		.np	= 2,
-		.tap	= { 1011 - 16, 16 },
-	}, { /* Generated with Crypto++ */
+		.np	= 3,
+		.tap	= { 1008 - 20, 4, 16 },
+	},
+};
+
+static const struct cipher_testvec camellia_ctr_tv_template[] = {
+	{ /* Generated with Crypto++ */
 		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
 			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
 			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
 			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
-		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
-			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
-		.input	= "\x85\x79\x6C\x8B\x2B\x6D\x14\xF9"
-			  "\xA6\x83\xB6\x80\x5B\x3A\xF3\x7E"
-			  "\x30\x29\xEB\x1F\xDC\x19\x5F\xEB"
-			  "\xF7\xC4\x27\x04\x51\x87\xD7\x6F"
-			  "\xB8\x4E\x07\xFB\xAC\x3B\x08\xB4"
-			  "\x4D\xCB\xE8\xE1\x71\x7D\x4F\x48"
-			  "\xCD\x81\x64\xA5\xC4\x07\x1A\x9A"
-			  "\x4B\x62\x90\x0E\xC8\xB3\x2B\x6B"
-			  "\x8F\x9C\x6E\x72\x4B\xBA\xEF\x07"
-			  "\x2C\x56\x07\x5E\x37\x30\x60\xA9"
-			  "\xE3\xEF\xD6\x69\xE1\xA1\x77\x64"
-			  "\x93\x75\x7A\xB7\x7A\x3B\xE9\x43"
-			  "\x23\x35\x95\x91\x80\x8A\xC7\xCF"
-			  "\xC3\xD5\xBF\xE7\xFE\x4C\x06\x6B"
-			  "\x05\x19\x48\xE2\x62\xBA\x4F\xF2"
-			  "\xFB\xEE\xE4\xCB\x79\x9D\xA3\x10"
-			  "\x1D\x29\x8C\x1D\x7A\x88\x5A\xDD"
-			  "\x4E\xB6\x18\xAA\xCD\xE6\x33\x96"
-			  "\xD9\x0F\x90\x5A\x78\x76\x4D\x77"
-			  "\x3C\x20\x89\x3B\xA3\xF9\x07\xFD"
-			  "\xE4\xE8\x20\x2D\x15\x0A\x63\x49"
-			  "\xF5\x4F\x89\xD8\xDE\xA1\x28\x78"
-			  "\x28\x07\x09\x1B\x03\x94\x1D\x4B"
-			  "\x82\x28\x1E\x1D\x95\xBA\xAC\x85"
-			  "\x71\x6E\x3C\x18\x4B\x77\x74\x79"
-			  "\xBF\x67\x0A\x53\x3C\x94\xD9\x60"
-			  "\xE9\x6D\x40\x34\xA0\x2A\x53\x5D"
-			  "\x27\xD5\x47\xF9\xC3\x4B\x27\x29"
-			  "\xE4\x76\x9C\x3F\xA7\x1C\x87\xFC"
-			  "\x6E\x0F\xCF\x9B\x60\xF0\xF0\x8B"
-			  "\x70\x1C\x84\x81\x72\x4D\xB4\x98"
-			  "\x23\x62\xE7\x6A\x2B\xFC\xA5\xB2"
-			  "\xFF\xF5\x71\x07\xCD\x90\x23\x13"
-			  "\x19\xD7\x79\x36\x6C\x9D\x55\x8B"
-			  "\x93\x78\x86\x05\x69\x46\xD0\xC5"
-			  "\x39\x09\xEB\x79\xEF\xFA\x9F\xAE"
-			  "\xF3\xD5\x44\xC3\xFD\x86\xD2\x7C"
-			  "\x83\x4B\xD8\x75\x9C\x18\x04\x7B"
-			  "\x73\xAD\x72\xA4\xF6\xAB\xCF\x4B"
-			  "\xCC\x01\x45\x90\xA6\x43\x05\x0C"
-			  "\x6C\x4F\x62\x77\x57\x97\x9F\xEE"
-			  "\x75\xA7\x3C\x38\xD1\x0F\x3D\x0E"
-			  "\x2C\x43\x98\xFB\x13\x65\x73\xE4"
-			  "\x3C\x1E\xD6\x90\x08\xF7\xE0\x99"
-			  "\x3B\xF1\x9D\x6C\x48\xA9\x0E\x32"
-			  "\x17\xC2\xCC\x20\xA1\x19\x26\xAA"
-			  "\xE0\x75\x2F\xFB\x54\x66\x0A\xDF"
-			  "\xB5\xF2\x1F\xC1\x34\x3C\x30\x56"
-			  "\xE8\xDC\xF7\x92\x6B\xBF\x17\x24"
-			  "\xEC\x94\xB5\x3B\xD6\xCE\xA2\x54"
-			  "\x10\x7F\x50\xDE\x69\x77\xD5\x37"
-			  "\xFE\x9C\x10\x83\xC5\xEB\xC9\x53"
-			  "\xB7\xF3\xC4\x20\xAF\x0A\x7E\x57"
-			  "\x3A\xE6\x75\xFE\x89\x00\x6E\x48"
-			  "\xFB\x99\x17\x2C\xF6\x64\x40\x95"
-			  "\x5E\xDC\x7A\xA6\x70\xC7\xF4\xDD"
-			  "\x52\x05\x24\x34\xF9\x0E\xC8\x64"
-			  "\x6D\xE2\xD8\x80\x53\x31\x4C\xFE"
-			  "\xB4\x3A\x5F\x19\xCF\x42\x1B\x22"
-			  "\x0B\x2D\x7B\xF1\xC5\x43\xF7\x5E"
-			  "\x12\xA8\x01\x64\x16\x0B\x26\x5A"
-			  "\x0C\x95\x0F\x40\xC5\x5A\x06\x7C"
-			  "\xCF\xF5\xD5\xB7\x7A\x34\x23\xB6"
-			  "\xAA\x9E\xA8\x98\xA2\xF8\x3D\xD3"
-			  "\x3F\x23\x69\x63\x56\x96\x45\xD6"
-			  "\x74\x23\x1D\x5C\x63\xCC\xD8\x78"
-			  "\x16\xE2\x9C\xD2\x80\x02\xF2\x28"
-			  "\x69\x2F\xC4\xA8\x15\x15\x24\x3B"
-			  "\xCB\xF0\x14\xE4\x62\xC8\xF3\xD1"
-			  "\x03\x58\x1B\x33\x77\x74\x1F\xB4"
-			  "\x07\x86\xF2\x21\xB7\x41\xAE\xBF"
-			  "\x25\xC2\xFF\x51\xEF\xEA\xCE\xC4"
-			  "\x5F\xD9\xB8\x18\x6A\xF0\x0F\x0D"
-			  "\xF8\x04\xBB\x6D\x62\x33\x87\x26"
-			  "\x4F\x2F\x14\x6E\xDC\xDB\x66\x09"
-			  "\x2A\xEF\x7D\x84\x10\xAC\x82\x5E"
-			  "\xD2\xE4\xAD\x74\x7A\x6D\xCC\x3A"
-			  "\x7B\x62\xD8\xD6\x07\x2D\xF7\xDF"
-			  "\x9B\xB3\x82\xCF\x9C\x1D\x76\x5C"
-			  "\xAC\x7B\xD4\x9B\x45\xA1\x64\x11"
-			  "\x66\xF1\xA7\x0B\xF9\xDD\x00\xDD"
-			  "\xA4\x45\x3D\x3E\x03\xC9\x2E\xCB"
-			  "\xC3\x14\x84\x72\xFD\x41\xDC\xBD"
-			  "\x75\xBE\xA8\xE5\x16\x48\x64\x39"
-			  "\xCA\xF3\xE6\xDC\x25\x24\xF1\x6D"
-			  "\xB2\x8D\xC5\x38\x54\xD3\x5D\x6D"
-			  "\x0B\x29\x10\x15\x0E\x13\x3B\xAC"
-			  "\x7E\xCC\x9E\x3E\x18\x48\xA6\x02"
-			  "\xEF\x03\xB2\x2E\xE3\xD2\x70\x21"
-			  "\xB4\x19\x26\xBE\x3A\x3D\x05\xE0"
-			  "\xF8\x09\xAF\xE4\x31\x26\x92\x2F"
-			  "\x8F\x55\xAC\xED\x0B\xB2\xA5\x34"
-			  "\xBE\x50\xB1\x02\x22\x96\xE3\x40"
-			  "\x7B\x70\x50\x6E\x3B\xD5\xE5\xA0"
-			  "\x8E\xA2\xAD\x14\x60\x5C\x7A\x2B"
-			  "\x3D\x1B\x7F\xC1\xC0\x2C\x56\x36"
-			  "\xD2\x0A\x32\x06\x97\x34\xB9\xF4"
-			  "\x6F\x9F\x7E\x80\xD0\x9D\xF7\x6A"
-			  "\x21\xC1\xA2\x6A\xB1\x96\x5B\x4D"
-			  "\x7A\x15\x6C\xC4\x4E\xB8\xE0\x9E"
-			  "\x6C\x50\xF3\x9C\xC9\xB5\x23\xB7"
-			  "\xF1\xD4\x29\x4A\x23\xC4\xAD\x1E"
-			  "\x2C\x07\xD2\x43\x5F\x57\x93\xCA"
-			  "\x85\xF9\x9F\xAD\x4C\xF1\xE4\xB1"
-			  "\x1A\x8E\x28\xA4\xB6\x52\x77\x7E"
-			  "\x68\xC6\x47\xB9\x76\xCC\x65\x5F"
-			  "\x0B\xF9\x67\x93\xD8\x0E\x9A\x37"
-			  "\x5F\x41\xED\x64\x6C\xAD\x5F\xED"
-			  "\x3F\x8D\xFB\x8E\x1E\xA0\xE4\x1F"
-			  "\xC2\xC7\xED\x18\x43\xE1\x20\x86"
-			  "\x5D\xBC\x30\x70\x22\xA1\xDC\x53"
-			  "\x10\x3A\x8D\x47\x82\xCD\x7F\x59"
-			  "\x03\x2D\x6D\xF5\xE7\x79\xD4\x07"
-			  "\x68\x2A\xA5\x42\x19\x4D\xAF\xF5"
-			  "\xED\x47\x83\xBC\x5F\x62\x84\xDA"
-			  "\xDA\x41\xFF\xB0\x1D\x64\xA3\xC8"
-			  "\xBD\x4E\xE0\xB8\x7F\xEE\x55\x0A"
-			  "\x4E\x61\xB2\x51\xF6\x9C\x95\xF6"
-			  "\x92\xBB\xF6\xC5\xF0\x09\x86\xDE"
-			  "\x37\x9E\x29\xF9\x2A\x18\x73\x0D"
-			  "\xDC\x7E\x6B\x7B\x1B\x43\x8C\xEA"
-			  "\x13\xC8\x1A\x47\x0A\x2D\x6D\x56"
-			  "\xCD\xD2\xE7\x53\x1A\xAB\x1C\x3C"
-			  "\xC5\x9B\x03\x70\x29\x2A\x49\x09"
-			  "\x67\xA1\xEA\xD6\x3A\x5B\xBF\x71"
-			  "\x1D\x48\x64\x6C\xFB\xC0\x9E\x36",
-		.ilen	= 1008,
-		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ctext	= "\xF3\x06\x3A\x84\xCD\xBA\x8E\x11"
+			  "\xB7\x74\x6F\x5C\x97\xFB\x36\xFE"
+			  "\xDE\x71\x58\xD4\x15\xD1\xC1\xA4"
+			  "\xC9\x28\x74\xA6\x6B\xC7\x95\xA6"
+			  "\x6C\x77\xF7\x2F\xDF\xC7\xBB\x85"
+			  "\x60\xFC\xE8\x94\xE8\xB5\x09\x2C"
+			  "\x1E\x43\xEF\x6C\xE9\x98\xC5\xA0"
+			  "\x7B\x13\xE5\x7F\xF8\x49\x9A\x8C"
+			  "\xE6\x7B\x08\xC3\x32\x66\x55\x4E"
+			  "\xA5\x44\x1D\x2C\x18\xC7\x29\x1F"
+			  "\x61\x28\x4A\xE3\xCD\xE5\x47\xB2"
+			  "\x82\x2F\x66\x83\x91\x51\xAE\xD7"
+			  "\x1C\x91\x3C\x57\xE3\x1D\x5A\xC9"
+			  "\xFD\xC5\x58\x58\xEF\xCC\x33\xC9"
+			  "\x0F\xEA\x26\x32\xD1\x15\x19\x2D"
+			  "\x25\xB4\x7F\xB0\xDF\xFB\x88\x60"
+			  "\x4E\x4D\x06\x7D\xCC\x1F\xED\x3B"
+			  "\x68\x84\xD5\xB3\x1B\xE7\xB9\xA1"
+			  "\x68\x8B\x2C\x1A\x44\xDA\x63\xD3"
+			  "\x29\xE9\x59\x32\x1F\x30\x1C\x43"
+			  "\xEA\x3A\xA3\x6B\x54\x3C\xAA\x11"
+			  "\xAD\x38\x20\xC9\xB9\x8A\x64\x66"
+			  "\x5A\x07\x49\xDF\xA1\x9C\xF9\x76"
+			  "\x36\x65\xB6\x81\x8F\x76\x09\xE5"
+			  "\xEB\xD1\x29\xA4\xE4\xF4\x4C\xCD"
+			  "\xAF\xFC\xB9\x16\xD9\xC3\x73\x6A"
+			  "\x33\x12\xF8\x7E\xBC\xCC\x7D\x80"
+			  "\xBF\x3C\x25\x06\x13\x84\xFA\x35"
+			  "\xF7\x40\xFA\xA1\x44\x13\x70\xD8"
+			  "\x01\xF9\x85\x15\x63\xEC\x7D\xB9"
+			  "\x02\xD8\xBA\x41\x6C\x92\x68\x66"
+			  "\x95\xDD\xD6\x42\xE7\xBB\xE1\xFD"
+			  "\x28\x3E\x94\xB6\xBD\xA7\xBF\x47"
+			  "\x58\x8D\xFF\x19\x30\x75\x0D\x48"
+			  "\x94\xE9\xA6\xCD\xB3\x8E\x1E\xCD"
+			  "\x59\xBC\x1A\xAC\x3C\x4F\xA9\xEB"
+			  "\xF4\xA7\xE4\x75\x4A\x18\x40\xC9"
+			  "\x1E\xEC\x06\x9C\x28\x4B\xF7\x2B"
+			  "\xE2\xEF\xD6\x42\x2E\xBB\xFC\x0A"
+			  "\x79\xA2\x99\x28\x93\x1B\x00\x57"
+			  "\x35\x1E\x1A\x93\x90\xA4\x68\x95"
+			  "\x5E\x57\x40\xD5\xA9\xAA\x19\x48"
+			  "\xEC\xFF\x76\x77\xDC\x78\x89\x76"
+			  "\xE5\x3B\x00\xEC\x58\x4D\xD1\xE3"
+			  "\xC8\x6C\x2C\x45\x5E\x5F\xD9\x4E"
+			  "\x71\xA5\x36\x6D\x03\xF1\xC7\xD5"
+			  "\xF3\x63\xC0\xD8\xCB\x2B\xF1\xA8"
+			  "\xB9\x2B\xE6\x0B\xB9\x65\x78\xA0"
+			  "\xC4\x46\xE6\x9B\x8B\x43\x2D\xAB"
+			  "\x70\xA6\xE0\x59\x1E\xAC\x9D\xE0"
+			  "\x76\x44\x45\xF3\x24\x11\x57\x98"
+			  "\x9A\x86\xB4\x12\x80\x28\x86\x20"
+			  "\x23\x9D\x2D\xE9\x38\x32\xB1\xE1"
+			  "\xCF\x0A\x23\x73\x7D\xC5\x80\x3D"
+			  "\x9F\x6D\xA0\xD0\xEE\x93\x8A\x79"
+			  "\x3A\xDD\x1D\xBB\x9E\x26\x5D\x01"
+			  "\x44\xD0\xD4\x4E\xC3\xF1\xE4\x38"
+			  "\x09\x62\x0A\x1A\x4E\xD2\x63\x0F"
+			  "\x6E\x3E\xD2\xA4\x3A\xF4\xF3\xFF"
+			  "\x7E\x42\xEC\xB6\x6F\x4D\x6B\x48"
+			  "\xE6\xA6\x50\x80\x78\x9E\xF1\xB0"
+			  "\x4D\xB2\x0D\x3D\xFC\x40\x25\x4D",
+		.len	= 496,
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
+			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06"
+			  "\x9D\x34\xCB\x3F\xD6\x6D\x04\x78"
+			  "\x0F\xA6\x1A\xB1\x48\xDF\x53\xEA"
+			  "\x81\x18\x8C\x23\xBA\x2E\xC5\x5C"
+			  "\xF3\x67\xFE\x95\x09\xA0\x37\xCE"
+			  "\x42\xD9\x70\x07\x7B\x12\xA9\x1D"
+			  "\xB4\x4B\xE2\x56\xED\x84\x1B\x8F"
+			  "\x26\xBD\x31\xC8\x5F\xF6\x6A\x01"
+			  "\x98\x0C\xA3\x3A\xD1\x45\xDC\x73"
+			  "\x0A\x7E\x15\xAC\x20\xB7\x4E\xE5"
+			  "\x59\xF0\x87\x1E\x92\x29\xC0\x34"
+			  "\xCB\x62\xF9\x6D\x04\x9B\x0F\xA6"
+			  "\x3D\xD4\x48\xDF\x76\x0D\x81\x18"
+			  "\xAF\x23\xBA\x51\xE8\x5C\xF3\x8A"
+			  "\x21\x95\x2C\xC3\x37\xCE\x65\xFC"
+			  "\x70\x07\x9E\x12\xA9\x40\xD7\x4B"
+			  "\xE2\x79\x10\x84\x1B\xB2\x26\xBD"
+			  "\x54\xEB\x5F\xF6\x8D\x01\x98\x2F"
+			  "\xC6\x3A\xD1\x68\xFF\x73\x0A\xA1"
+			  "\x15\xAC\x43\xDA\x4E\xE5\x7C\x13"
+			  "\x87\x1E\xB5\x29\xC0\x57\xEE\x62"
+			  "\xF9\x90\x04\x9B\x32\xC9\x3D\xD4"
+			  "\x6B\x02\x76\x0D\xA4\x18\xAF\x46"
+			  "\xDD\x51\xE8\x7F\x16\x8A\x21\xB8"
+			  "\x2C\xC3\x5A\xF1\x65\xFC\x93\x07"
+			  "\x9E\x35\xCC\x40\xD7\x6E\x05\x79"
+			  "\x10\xA7\x1B\xB2\x49\xE0\x54\xEB"
+			  "\x82\x19\x8D\x24\xBB\x2F\xC6\x5D"
+			  "\xF4\x68\xFF\x96\x0A\xA1\x38\xCF"
+			  "\x43\xDA\x71\x08\x7C\x13\xAA\x1E"
+			  "\xB5\x4C\xE3\x57\xEE\x85\x1C\x90"
+			  "\x27\xBE\x32\xC9\x60\xF7\x6B\x02"
+			  "\x99\x0D\xA4\x3B\xD2\x46\xDD\x74"
+			  "\x0B\x7F\x16\xAD\x21\xB8\x4F\xE6"
+			  "\x5A\xF1\x88\x1F\x93\x2A\xC1\x35"
+			  "\xCC\x63\xFA\x6E\x05\x9C\x10\xA7"
+			  "\x3E\xD5\x49\xE0\x77\x0E\x82\x19"
+			  "\xB0\x24\xBB\x52\xE9\x5D\xF4\x8B"
+			  "\x22\x96\x2D\xC4\x38\xCF\x66\xFD"
+			  "\x71\x08\x9F\x13\xAA\x41\xD8\x4C"
+			  "\xE3\x7A\x11\x85\x1C\xB3\x27\xBE"
+			  "\x55\xEC\x60\xF7\x8E\x02\x99\x30"
+			  "\xC7\x3B\xD2\x69\x00\x74\x0B\xA2"
+			  "\x16\xAD\x44\xDB\x4F\xE6\x7D\x14"
+			  "\x88\x1F\xB6\x2A\xC1\x58\xEF\x63"
+			  "\xFA\x91\x05\x9C\x33\xCA\x3E\xD5"
+			  "\x6C\x03\x77\x0E\xA5\x19\xB0\x47"
+			  "\xDE\x52\xE9\x80\x17\x8B\x22\xB9"
+			  "\x2D\xC4\x5B\xF2\x66\xFD\x94\x08"
+			  "\x9F\x36\xCD\x41\xD8\x6F\x06\x7A"
+			  "\x11\xA8\x1C\xB3\x4A\xE1\x55\xEC"
+			  "\x83\x1A\x8E\x25\xBC\x30\xC7\x5E"
+			  "\xF5\x69\x00\x97\x0B\xA2\x39\xD0"
+			  "\x44\xDB\x72\x09\x7D\x14\xAB\x1F"
+			  "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91"
+			  "\x28\xBF\x33\xCA\x61\xF8\x6C\x03"
+			  "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75"
+			  "\x0C\x80\x17\xAE\x22\xB9\x50\xE7"
+			  "\x5B\xF2\x89\x20\x94\x2B\xC2\x36"
+			  "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8"
+			  "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A"
+			  "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C"
+			  "\x00\x97\x2E\xC5\x39\xD0\x67\xFE"
+			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D"
+			  "\xE4\x7B\x12",
+		.ctext	= "\xF3\x06\x3A\x84\xCD\xBA\x8E\x11"
+			  "\xB7\x74\x6F\x5C\x97\xFB\x36\xFE"
+			  "\xDE\x71\x58\xD4\x15\xD1\xC1\xA4"
+			  "\xC9\x28\x74\xA6\x6B\xC7\x95\xA6"
+			  "\x6C\x77\xF7\x2F\xDF\xC7\xBB\x85"
+			  "\x60\xFC\xE8\x94\xE8\xB5\x09\x2C"
+			  "\x1E\x43\xEF\x6C\xE9\x98\xC5\xA0"
+			  "\x7B\x13\xE5\x7F\xF8\x49\x9A\x8C"
+			  "\xE6\x7B\x08\xC3\x32\x66\x55\x4E"
+			  "\xA5\x44\x1D\x2C\x18\xC7\x29\x1F"
+			  "\x61\x28\x4A\xE3\xCD\xE5\x47\xB2"
+			  "\x82\x2F\x66\x83\x91\x51\xAE\xD7"
+			  "\x1C\x91\x3C\x57\xE3\x1D\x5A\xC9"
+			  "\xFD\xC5\x58\x58\xEF\xCC\x33\xC9"
+			  "\x0F\xEA\x26\x32\xD1\x15\x19\x2D"
+			  "\x25\xB4\x7F\xB0\xDF\xFB\x88\x60"
+			  "\x4E\x4D\x06\x7D\xCC\x1F\xED\x3B"
+			  "\x68\x84\xD5\xB3\x1B\xE7\xB9\xA1"
+			  "\x68\x8B\x2C\x1A\x44\xDA\x63\xD3"
+			  "\x29\xE9\x59\x32\x1F\x30\x1C\x43"
+			  "\xEA\x3A\xA3\x6B\x54\x3C\xAA\x11"
+			  "\xAD\x38\x20\xC9\xB9\x8A\x64\x66"
+			  "\x5A\x07\x49\xDF\xA1\x9C\xF9\x76"
+			  "\x36\x65\xB6\x81\x8F\x76\x09\xE5"
+			  "\xEB\xD1\x29\xA4\xE4\xF4\x4C\xCD"
+			  "\xAF\xFC\xB9\x16\xD9\xC3\x73\x6A"
+			  "\x33\x12\xF8\x7E\xBC\xCC\x7D\x80"
+			  "\xBF\x3C\x25\x06\x13\x84\xFA\x35"
+			  "\xF7\x40\xFA\xA1\x44\x13\x70\xD8"
+			  "\x01\xF9\x85\x15\x63\xEC\x7D\xB9"
+			  "\x02\xD8\xBA\x41\x6C\x92\x68\x66"
+			  "\x95\xDD\xD6\x42\xE7\xBB\xE1\xFD"
+			  "\x28\x3E\x94\xB6\xBD\xA7\xBF\x47"
+			  "\x58\x8D\xFF\x19\x30\x75\x0D\x48"
+			  "\x94\xE9\xA6\xCD\xB3\x8E\x1E\xCD"
+			  "\x59\xBC\x1A\xAC\x3C\x4F\xA9\xEB"
+			  "\xF4\xA7\xE4\x75\x4A\x18\x40\xC9"
+			  "\x1E\xEC\x06\x9C\x28\x4B\xF7\x2B"
+			  "\xE2\xEF\xD6\x42\x2E\xBB\xFC\x0A"
+			  "\x79\xA2\x99\x28\x93\x1B\x00\x57"
+			  "\x35\x1E\x1A\x93\x90\xA4\x68\x95"
+			  "\x5E\x57\x40\xD5\xA9\xAA\x19\x48"
+			  "\xEC\xFF\x76\x77\xDC\x78\x89\x76"
+			  "\xE5\x3B\x00\xEC\x58\x4D\xD1\xE3"
+			  "\xC8\x6C\x2C\x45\x5E\x5F\xD9\x4E"
+			  "\x71\xA5\x36\x6D\x03\xF1\xC7\xD5"
+			  "\xF3\x63\xC0\xD8\xCB\x2B\xF1\xA8"
+			  "\xB9\x2B\xE6\x0B\xB9\x65\x78\xA0"
+			  "\xC4\x46\xE6\x9B\x8B\x43\x2D\xAB"
+			  "\x70\xA6\xE0\x59\x1E\xAC\x9D\xE0"
+			  "\x76\x44\x45\xF3\x24\x11\x57\x98"
+			  "\x9A\x86\xB4\x12\x80\x28\x86\x20"
+			  "\x23\x9D\x2D\xE9\x38\x32\xB1\xE1"
+			  "\xCF\x0A\x23\x73\x7D\xC5\x80\x3D"
+			  "\x9F\x6D\xA0\xD0\xEE\x93\x8A\x79"
+			  "\x3A\xDD\x1D\xBB\x9E\x26\x5D\x01"
+			  "\x44\xD0\xD4\x4E\xC3\xF1\xE4\x38"
+			  "\x09\x62\x0A\x1A\x4E\xD2\x63\x0F"
+			  "\x6E\x3E\xD2\xA4\x3A\xF4\xF3\xFF"
+			  "\x7E\x42\xEC\xB6\x6F\x4D\x6B\x48"
+			  "\xE6\xA6\x50\x80\x78\x9E\xF1\xB0"
+			  "\x4D\xB2\x0D\x3D\xFC\x40\x25\x4D"
+			  "\x93\x11\x1C\xE9\xD2\x9F\x6E\x90"
+			  "\xE5\x41\x4A\xE2\x3C\x45\x29\x35"
+			  "\xEC\xD6\x47\x50\xCB\x7B\xA2\x32"
+			  "\xF7\x8B\x62\xF1\xE3\x9A\xFE\xC7"
+			  "\x1D\x8C\x02\x72\x68\x09\xE9\xB6"
+			  "\x4A\x80\xE6\xB1\x56\xDF\x90\xD4"
+			  "\x93\x74\xA4\xCE\x20\x23\xBF\x48"
+			  "\xA5\xDE\x1B\xFA\x40\x69\x31\x98"
+			  "\x62\x6E\xA5\xC7\xBF\x0C\x62\xE5"
+			  "\x6D\xE1\x93\xF1\x83\x10\x1C\xCA"
+			  "\xF6\x5C\x19\xF8\x90\x78\xCB\xE4"
+			  "\x0B\x3A\xB5\xF8\x43\x86\xD3\x3F"
+			  "\xBA\x83\x34\x3C\x42\xCC\x7D\x28"
+			  "\x29\x63\x4F\xD8\x02\x17\xC5\x07"
+			  "\x2C\xA4\xAC\x79\xCB\xC3\xA9\x09"
+			  "\x81\x45\x18\xED\xE4\xCB\x42\x3B"
+			  "\x87\x2D\x23\xDC\xC5\xBA\x45\xBD"
+			  "\x92\xE5\x02\x97\x96\xCE\xAD\xEC"
+			  "\xBA\xD8\x76\xF8\xCA\xC1\x31\xEC"
+			  "\x1E\x4F\x3F\x83\xF8\x33\xE8\x6E"
+			  "\xCC\xF8\x5F\xDD\x65\x50\x99\x69"
+			  "\xAF\x48\xCE\xA5\xBA\xB6\x14\x9F"
+			  "\x05\x93\xB2\xE6\x59\xC8\x28\xFE"
+			  "\x8F\x37\xF9\x64\xB9\xA5\x56\x8F"
+			  "\xF1\x1B\x90\xEF\xAE\xEB\xFC\x09"
+			  "\x11\x7A\xF2\x19\x0A\x0A\x9A\x3C"
+			  "\xE2\x5E\x29\xFA\x31\x9B\xC1\x74"
+			  "\x1E\x10\x3E\x07\xA9\x31\x6D\xF8"
+			  "\x81\xF5\xD5\x8A\x04\x23\x51\xAC"
+			  "\xA2\xE2\x63\xFD\x27\x1F\x79\x5B"
+			  "\x1F\xE8\xDA\x11\x49\x4D\x1C\xBA"
+			  "\x54\xCC\x0F\xBA\x92\x69\xE5\xCB"
+			  "\x41\x1A\x67\xA6\x40\x82\x70\x8C"
+			  "\x19\x79\x08\xA4\x51\x20\x7D\xC9"
+			  "\x12\x27\xAE\x20\x0D\x2C\xA1\x6D"
+			  "\xF4\x55\xD4\xE7\xE6\xD4\x28\x08"
+			  "\x00\x70\x12\x56\x56\x50\xAD\x14"
+			  "\x5C\x3E\xA2\xD1\x36\x3F\x36\x48"
+			  "\xED\xB1\x57\x3E\x5D\x15\xF6\x1E"
+			  "\x53\xE9\xA4\x3E\xED\x7D\xCF\x7D"
+			  "\x29\xAF\xF3\x1E\x51\xA8\x9F\x85"
+			  "\x8B\xF0\xBB\xCE\xCC\x39\xC3\x64"
+			  "\x4B\xF2\xAD\x70\x19\xD4\x44\x8F"
+			  "\x91\x76\xE8\x15\x66\x34\x9F\xF6"
+			  "\x0F\x15\xA4\xA8\x24\xF8\x58\xB1"
+			  "\x38\x46\x47\xC7\x9B\xCA\xE9\x42"
+			  "\x44\xAA\xE6\xB5\x9C\x91\xA4\xD3"
+			  "\x16\xA0\xED\x42\xBE\xB5\x06\x19"
+			  "\xBE\x67\xE8\xBC\x22\x32\xA4\x1E"
+			  "\x93\xEB\xBE\xE9\xE1\x93\xE5\x31"
+			  "\x3A\xA2\x75\xDF\xE3\x6B\xE7\xCC"
+			  "\xB4\x70\x20\xE0\x6D\x82\x7C\xC8"
+			  "\x94\x5C\x5E\x37\x18\xAD\xED\x8B"
+			  "\x44\x86\xCA\x5E\x07\xB7\x70\x8D"
+			  "\x40\x48\x19\x73\x7C\x78\x64\x0B"
+			  "\xDB\x01\xCA\xAE\x63\x19\xE9\xD1"
+			  "\x6B\x2C\x84\x10\x45\x42\x2E\xC3"
+			  "\xDF\x7F\xAA\xE8\x87\x1B\x63\x46"
+			  "\x74\x28\x9D\x05\x30\x20\x62\x41"
+			  "\xC0\x9F\x2C\x36\x2B\x78\xD7\x26"
+			  "\xDF\x58\x51\xED\xFA\xDC\x87\x79"
+			  "\xBF\x8C\xBF\xC4\x0F\xE5\x05\xDA"
+			  "\x45\xE3\x35\x0D\x69\x91\x54\x1C"
+			  "\xE7\x2C\x49\x08\x8B\x72\xFA\x5C"
+			  "\xF1\x6B\xD9",
+		.len	= 1011,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 1011 - 16, 16 },
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.ptext	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
 			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
 			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
@@ -32579,273 +28921,146 @@ static const struct cipher_testvec camellia_ctr_dec_tv_template[] = {
 			  "\x44\xDB\x72\x09\x7D\x14\xAB\x1F"
 			  "\xB6\x4D\xE4\x58\xEF\x86\x1D\x91"
 			  "\x28\xBF\x33\xCA\x61\xF8\x6C\x03"
-			  "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75"
-			  "\x0C\x80\x17\xAE\x22\xB9\x50\xE7"
-			  "\x5B\xF2\x89\x20\x94\x2B\xC2\x36"
-			  "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8"
-			  "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A"
-			  "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C"
-			  "\x00\x97\x2E\xC5\x39\xD0\x67\xFE"
-			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D",
-		.rlen	= 1008,
-	},
-};
-
-static const struct cipher_testvec camellia_lrw_enc_tv_template[] = {
-	/* Generated from AES-LRW test vectors */
-	{
-		.key	= "\x45\x62\xac\x25\xf8\x28\x17\x6d"
-			  "\x4c\x26\x84\x14\xb5\x68\x01\x85"
-			  "\x25\x8e\x2a\x05\xe7\x3e\x9d\x03"
-			  "\xee\x5a\x83\x0c\xcc\x09\x4c\x87",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x92\x68\x19\xd7\xb7\x5b\x0a\x31"
-			  "\x97\xcc\x72\xbe\x99\x17\xeb\x3e",
-		.rlen	= 16,
-	}, {
-		.key	= "\x59\x70\x47\x14\xf5\x57\x47\x8c"
-			  "\xd7\x79\xe8\x0f\x54\x88\x79\x44"
-			  "\x0d\x48\xf0\xb7\xb1\x5a\x53\xea"
-			  "\x1c\xaa\x6b\x29\xc2\xca\xfb\xaf",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x02",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x73\x09\xb7\x50\xb6\x77\x30\x50"
-			  "\x5c\x8a\x9c\x26\x77\x9d\xfc\x4a",
-		.rlen	= 16,
-	}, {
-		.key	= "\xd8\x2a\x91\x34\xb2\x6a\x56\x50"
-			  "\x30\xfe\x69\xe2\x37\x7f\x98\x47"
-			  "\xcd\xf9\x0b\x16\x0c\x64\x8f\xb6"
-			  "\xb0\x0d\x0d\x1b\xae\x85\x87\x1f",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x90\xae\x83\xe0\x22\xb9\x60\x91"
-			  "\xfa\xa9\xb7\x98\xe3\xed\x87\x01",
-		.rlen	= 16,
-	}, {
-		.key	= "\x0f\x6a\xef\xf8\xd3\xd2\xbb\x15"
-			  "\x25\x83\xf7\x3c\x1f\x01\x28\x74"
-			  "\xca\xc6\xbc\x35\x4d\x4a\x65\x54"
-			  "\x90\xae\x61\xcf\x7b\xae\xbd\xcc"
-			  "\xad\xe4\x94\xc5\x4a\x29\xae\x70",
-		.klen	= 40,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x99\xe9\x6e\xd4\xc9\x21\xa5\xf0"
-			  "\xd8\x83\xef\xd9\x07\x16\x5f\x35",
-		.rlen	= 16,
-	}, {
-		.key	= "\x8a\xd4\xee\x10\x2f\xbd\x81\xff"
-			  "\xf8\x86\xce\xac\x93\xc5\xad\xc6"
-			  "\xa0\x19\x07\xc0\x9d\xf7\xbb\xdd"
-			  "\x52\x13\xb2\xb7\xf0\xff\x11\xd8"
-			  "\xd6\x08\xd0\xcd\x2e\xb1\x17\x6f",
-		.klen	= 40,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x42\x88\xf4\xcb\x21\x11\x6d\x8e"
-			  "\xde\x1a\xf2\x29\xf1\x4a\xe0\x15",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x40\xaa\x34\x86\x4a\x8f\x78\xb9"
-			  "\xdb\xdb\x0f\x3d\x48\x70\xbe\x8d",
-		.rlen	= 16,
-	}, {
-		.key	= "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
-			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
-			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
-			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
-			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
-			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.ilen	= 16,
-		.result	= "\x04\xab\x28\x37\x31\x7a\x26\xab"
-			  "\xa1\x70\x1b\x9c\xe7\xdd\x83\xff",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
-			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
-			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
-			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
-			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
-			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
-			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
-			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
-			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
-			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
-			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
-			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
-			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
-			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
-			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
-			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
-			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
-			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
-			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
-			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
-			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
-			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
-			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
-			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
-			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
-			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
-			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
-			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
-			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
-			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
-			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
-			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
-			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
-			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
-			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
-			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
-			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
-			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
-			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
-			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
-			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
-			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
-			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
-			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
-			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
-			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
-			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
-			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
-			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
-			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
-			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
-			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
-			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
-			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
-			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
-			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
-			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
-			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
-			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
-			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
-			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
-			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
-			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
-			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
-		.ilen	= 512,
-		.result	= "\x90\x69\x8e\xf2\x14\x86\x59\xf9"
-			  "\xec\xe7\xfa\x3f\x48\x9d\x7f\x96"
-			  "\x67\x76\xac\x2c\xd2\x63\x18\x93"
-			  "\x13\xf8\xf1\xf6\x71\x77\xb3\xee"
-			  "\x93\xb2\xcc\xf3\x26\xc1\x16\x4f"
-			  "\xd4\xe8\x43\xc1\x68\xa3\x3e\x06"
-			  "\x38\x51\xff\xa8\xb9\xa4\xeb\xb1"
-			  "\x62\xdd\x78\x81\xea\x1d\xef\x04"
-			  "\x1d\x07\xc1\x67\xc8\xd6\x77\xa1"
-			  "\x84\x95\xf4\x9a\xd9\xbc\x2d\xe2"
-			  "\xf6\x80\xfc\x91\x2a\xbc\x42\xa0"
-			  "\x40\x41\x69\xaa\x71\xc0\x37\xec"
-			  "\x39\xf3\xf2\xec\x82\xc3\x88\x79"
-			  "\xbc\xc3\xaa\xb7\xcf\x6a\x72\x80"
-			  "\x4c\xf4\x84\x8f\x13\x9e\x94\x5c"
-			  "\xe5\xb2\x91\xbb\x92\x51\x4d\xf1"
-			  "\xd6\x0d\x71\x6b\x7a\xc2\x2f\x12"
-			  "\x6f\x75\xc7\x80\x99\x50\x84\xcf"
-			  "\xa8\xeb\xd6\xe1\x1c\x59\x81\x7e"
-			  "\xb9\xb3\xde\x7a\x93\x14\x12\xa2"
-			  "\xf7\x43\xb3\x9d\x1a\x87\x65\x91"
-			  "\x42\x08\x40\x82\x06\x1c\x2d\x55"
-			  "\x6e\x48\xd5\x74\x07\x6e\x9d\x80"
-			  "\xeb\xb4\x97\xa1\x36\xdf\xfa\x74"
-			  "\x79\x7f\x5a\x75\xe7\x71\xc8\x8c"
-			  "\x7e\xf8\x3a\x77\xcd\x32\x05\xf9"
-			  "\x3d\xd4\xe9\xa2\xbb\xc4\x8b\x83"
-			  "\x42\x5c\x82\xfa\xe9\x4b\x96\x3b"
-			  "\x7f\x89\x8b\xf9\xf1\x87\xda\xf0"
-			  "\x87\xef\x13\x5d\xf0\xe2\xc5\xc1"
-			  "\xed\x14\xa9\x57\x19\x63\x40\x04"
-			  "\x24\xeb\x6e\x19\xd1\x3d\x70\x78"
-			  "\xeb\xda\x55\x70\x2c\x4f\x41\x5b"
-			  "\x56\x9f\x1a\xd3\xac\xf1\xc0\xc3"
-			  "\x21\xec\xd7\xd2\x55\x32\x7c\x2e"
-			  "\x3c\x48\x8e\xb4\x85\x35\x47\xfe"
-			  "\xe2\x88\x79\x98\x6a\xc9\x8d\xff"
-			  "\xe9\x89\x6e\xb8\xe2\x97\x00\xbd"
-			  "\xa4\x8f\xba\xd0\x8c\xcb\x79\x99"
-			  "\xb3\xb2\xb2\x7a\xc3\xb7\xef\x75"
-			  "\x23\x52\x76\xc3\x50\x6e\x66\xf8"
-			  "\xa2\xe2\xce\xba\x40\x21\x3f\xc9"
-			  "\x0a\x32\x7f\xf7\x08\x8c\x66\xcf"
-			  "\xd3\xdf\x57\x59\x83\xb8\xe1\x85"
-			  "\xd6\x8f\xfb\x48\x1f\x3a\xc4\x2f"
-			  "\xb4\x2d\x58\xab\xd8\x7f\x5e\x3a"
-			  "\xbc\x62\x3e\xe2\x6a\x52\x0d\x76"
-			  "\x2f\x1c\x1a\x30\xed\x95\x2a\x44"
-			  "\x35\xa5\x83\x04\x84\x01\x99\x56"
-			  "\xb7\xe3\x10\x96\xfa\xdc\x19\xdd"
-			  "\xe2\x7f\xcb\xa0\x49\x1b\xff\x4c"
-			  "\x73\xf6\xbb\x94\x00\xe8\xa9\x3d"
-			  "\xe2\x20\xe9\x3f\xfa\x07\x5d\x77"
-			  "\x06\xd5\x4f\x4d\x02\xb8\x40\x1b"
-			  "\x30\xed\x1a\x50\x19\xef\xc4\x2c"
-			  "\x02\xd9\xc5\xd3\x11\x33\x37\xe5"
-			  "\x2b\xa3\x95\xa6\xee\xd8\x74\x1d"
-			  "\x68\xa0\xeb\xbf\xdd\x5e\x99\x96"
-			  "\x91\xc3\x94\x24\xa5\x12\xa2\x37"
-			  "\xb3\xac\xcf\x2a\xfd\x55\x34\xfe"
-			  "\x79\x92\x3e\xe6\x1b\x49\x57\x5d"
-			  "\x93\x6c\x01\xf7\xcc\x4e\x20\xd1"
-			  "\xb2\x1a\xd8\x4c\xbd\x1d\x10\xe9"
-			  "\x5a\xa8\x92\x7f\xba\xe6\x0c\x95",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
+			  "\x9A\x0E\xA5\x3C\xD3\x47\xDE\x75"
+			  "\x0C\x80\x17\xAE\x22\xB9\x50\xE7"
+			  "\x5B\xF2\x89\x20\x94\x2B\xC2\x36"
+			  "\xCD\x64\xFB\x6F\x06\x9D\x11\xA8"
+			  "\x3F\xD6\x4A\xE1\x78\x0F\x83\x1A"
+			  "\xB1\x25\xBC\x53\xEA\x5E\xF5\x8C"
+			  "\x00\x97\x2E\xC5\x39\xD0\x67\xFE"
+			  "\x72\x09\xA0\x14\xAB\x42\xD9\x4D",
+		.ctext	= "\x85\x79\x6C\x8B\x2B\x6D\x14\xF9"
+			  "\xA6\x83\xB6\x80\x5B\x3A\xF3\x7E"
+			  "\x30\x29\xEB\x1F\xDC\x19\x5F\xEB"
+			  "\xF7\xC4\x27\x04\x51\x87\xD7\x6F"
+			  "\xB8\x4E\x07\xFB\xAC\x3B\x08\xB4"
+			  "\x4D\xCB\xE8\xE1\x71\x7D\x4F\x48"
+			  "\xCD\x81\x64\xA5\xC4\x07\x1A\x9A"
+			  "\x4B\x62\x90\x0E\xC8\xB3\x2B\x6B"
+			  "\x8F\x9C\x6E\x72\x4B\xBA\xEF\x07"
+			  "\x2C\x56\x07\x5E\x37\x30\x60\xA9"
+			  "\xE3\xEF\xD6\x69\xE1\xA1\x77\x64"
+			  "\x93\x75\x7A\xB7\x7A\x3B\xE9\x43"
+			  "\x23\x35\x95\x91\x80\x8A\xC7\xCF"
+			  "\xC3\xD5\xBF\xE7\xFE\x4C\x06\x6B"
+			  "\x05\x19\x48\xE2\x62\xBA\x4F\xF2"
+			  "\xFB\xEE\xE4\xCB\x79\x9D\xA3\x10"
+			  "\x1D\x29\x8C\x1D\x7A\x88\x5A\xDD"
+			  "\x4E\xB6\x18\xAA\xCD\xE6\x33\x96"
+			  "\xD9\x0F\x90\x5A\x78\x76\x4D\x77"
+			  "\x3C\x20\x89\x3B\xA3\xF9\x07\xFD"
+			  "\xE4\xE8\x20\x2D\x15\x0A\x63\x49"
+			  "\xF5\x4F\x89\xD8\xDE\xA1\x28\x78"
+			  "\x28\x07\x09\x1B\x03\x94\x1D\x4B"
+			  "\x82\x28\x1E\x1D\x95\xBA\xAC\x85"
+			  "\x71\x6E\x3C\x18\x4B\x77\x74\x79"
+			  "\xBF\x67\x0A\x53\x3C\x94\xD9\x60"
+			  "\xE9\x6D\x40\x34\xA0\x2A\x53\x5D"
+			  "\x27\xD5\x47\xF9\xC3\x4B\x27\x29"
+			  "\xE4\x76\x9C\x3F\xA7\x1C\x87\xFC"
+			  "\x6E\x0F\xCF\x9B\x60\xF0\xF0\x8B"
+			  "\x70\x1C\x84\x81\x72\x4D\xB4\x98"
+			  "\x23\x62\xE7\x6A\x2B\xFC\xA5\xB2"
+			  "\xFF\xF5\x71\x07\xCD\x90\x23\x13"
+			  "\x19\xD7\x79\x36\x6C\x9D\x55\x8B"
+			  "\x93\x78\x86\x05\x69\x46\xD0\xC5"
+			  "\x39\x09\xEB\x79\xEF\xFA\x9F\xAE"
+			  "\xF3\xD5\x44\xC3\xFD\x86\xD2\x7C"
+			  "\x83\x4B\xD8\x75\x9C\x18\x04\x7B"
+			  "\x73\xAD\x72\xA4\xF6\xAB\xCF\x4B"
+			  "\xCC\x01\x45\x90\xA6\x43\x05\x0C"
+			  "\x6C\x4F\x62\x77\x57\x97\x9F\xEE"
+			  "\x75\xA7\x3C\x38\xD1\x0F\x3D\x0E"
+			  "\x2C\x43\x98\xFB\x13\x65\x73\xE4"
+			  "\x3C\x1E\xD6\x90\x08\xF7\xE0\x99"
+			  "\x3B\xF1\x9D\x6C\x48\xA9\x0E\x32"
+			  "\x17\xC2\xCC\x20\xA1\x19\x26\xAA"
+			  "\xE0\x75\x2F\xFB\x54\x66\x0A\xDF"
+			  "\xB5\xF2\x1F\xC1\x34\x3C\x30\x56"
+			  "\xE8\xDC\xF7\x92\x6B\xBF\x17\x24"
+			  "\xEC\x94\xB5\x3B\xD6\xCE\xA2\x54"
+			  "\x10\x7F\x50\xDE\x69\x77\xD5\x37"
+			  "\xFE\x9C\x10\x83\xC5\xEB\xC9\x53"
+			  "\xB7\xF3\xC4\x20\xAF\x0A\x7E\x57"
+			  "\x3A\xE6\x75\xFE\x89\x00\x6E\x48"
+			  "\xFB\x99\x17\x2C\xF6\x64\x40\x95"
+			  "\x5E\xDC\x7A\xA6\x70\xC7\xF4\xDD"
+			  "\x52\x05\x24\x34\xF9\x0E\xC8\x64"
+			  "\x6D\xE2\xD8\x80\x53\x31\x4C\xFE"
+			  "\xB4\x3A\x5F\x19\xCF\x42\x1B\x22"
+			  "\x0B\x2D\x7B\xF1\xC5\x43\xF7\x5E"
+			  "\x12\xA8\x01\x64\x16\x0B\x26\x5A"
+			  "\x0C\x95\x0F\x40\xC5\x5A\x06\x7C"
+			  "\xCF\xF5\xD5\xB7\x7A\x34\x23\xB6"
+			  "\xAA\x9E\xA8\x98\xA2\xF8\x3D\xD3"
+			  "\x3F\x23\x69\x63\x56\x96\x45\xD6"
+			  "\x74\x23\x1D\x5C\x63\xCC\xD8\x78"
+			  "\x16\xE2\x9C\xD2\x80\x02\xF2\x28"
+			  "\x69\x2F\xC4\xA8\x15\x15\x24\x3B"
+			  "\xCB\xF0\x14\xE4\x62\xC8\xF3\xD1"
+			  "\x03\x58\x1B\x33\x77\x74\x1F\xB4"
+			  "\x07\x86\xF2\x21\xB7\x41\xAE\xBF"
+			  "\x25\xC2\xFF\x51\xEF\xEA\xCE\xC4"
+			  "\x5F\xD9\xB8\x18\x6A\xF0\x0F\x0D"
+			  "\xF8\x04\xBB\x6D\x62\x33\x87\x26"
+			  "\x4F\x2F\x14\x6E\xDC\xDB\x66\x09"
+			  "\x2A\xEF\x7D\x84\x10\xAC\x82\x5E"
+			  "\xD2\xE4\xAD\x74\x7A\x6D\xCC\x3A"
+			  "\x7B\x62\xD8\xD6\x07\x2D\xF7\xDF"
+			  "\x9B\xB3\x82\xCF\x9C\x1D\x76\x5C"
+			  "\xAC\x7B\xD4\x9B\x45\xA1\x64\x11"
+			  "\x66\xF1\xA7\x0B\xF9\xDD\x00\xDD"
+			  "\xA4\x45\x3D\x3E\x03\xC9\x2E\xCB"
+			  "\xC3\x14\x84\x72\xFD\x41\xDC\xBD"
+			  "\x75\xBE\xA8\xE5\x16\x48\x64\x39"
+			  "\xCA\xF3\xE6\xDC\x25\x24\xF1\x6D"
+			  "\xB2\x8D\xC5\x38\x54\xD3\x5D\x6D"
+			  "\x0B\x29\x10\x15\x0E\x13\x3B\xAC"
+			  "\x7E\xCC\x9E\x3E\x18\x48\xA6\x02"
+			  "\xEF\x03\xB2\x2E\xE3\xD2\x70\x21"
+			  "\xB4\x19\x26\xBE\x3A\x3D\x05\xE0"
+			  "\xF8\x09\xAF\xE4\x31\x26\x92\x2F"
+			  "\x8F\x55\xAC\xED\x0B\xB2\xA5\x34"
+			  "\xBE\x50\xB1\x02\x22\x96\xE3\x40"
+			  "\x7B\x70\x50\x6E\x3B\xD5\xE5\xA0"
+			  "\x8E\xA2\xAD\x14\x60\x5C\x7A\x2B"
+			  "\x3D\x1B\x7F\xC1\xC0\x2C\x56\x36"
+			  "\xD2\x0A\x32\x06\x97\x34\xB9\xF4"
+			  "\x6F\x9F\x7E\x80\xD0\x9D\xF7\x6A"
+			  "\x21\xC1\xA2\x6A\xB1\x96\x5B\x4D"
+			  "\x7A\x15\x6C\xC4\x4E\xB8\xE0\x9E"
+			  "\x6C\x50\xF3\x9C\xC9\xB5\x23\xB7"
+			  "\xF1\xD4\x29\x4A\x23\xC4\xAD\x1E"
+			  "\x2C\x07\xD2\x43\x5F\x57\x93\xCA"
+			  "\x85\xF9\x9F\xAD\x4C\xF1\xE4\xB1"
+			  "\x1A\x8E\x28\xA4\xB6\x52\x77\x7E"
+			  "\x68\xC6\x47\xB9\x76\xCC\x65\x5F"
+			  "\x0B\xF9\x67\x93\xD8\x0E\x9A\x37"
+			  "\x5F\x41\xED\x64\x6C\xAD\x5F\xED"
+			  "\x3F\x8D\xFB\x8E\x1E\xA0\xE4\x1F"
+			  "\xC2\xC7\xED\x18\x43\xE1\x20\x86"
+			  "\x5D\xBC\x30\x70\x22\xA1\xDC\x53"
+			  "\x10\x3A\x8D\x47\x82\xCD\x7F\x59"
+			  "\x03\x2D\x6D\xF5\xE7\x79\xD4\x07"
+			  "\x68\x2A\xA5\x42\x19\x4D\xAF\xF5"
+			  "\xED\x47\x83\xBC\x5F\x62\x84\xDA"
+			  "\xDA\x41\xFF\xB0\x1D\x64\xA3\xC8"
+			  "\xBD\x4E\xE0\xB8\x7F\xEE\x55\x0A"
+			  "\x4E\x61\xB2\x51\xF6\x9C\x95\xF6"
+			  "\x92\xBB\xF6\xC5\xF0\x09\x86\xDE"
+			  "\x37\x9E\x29\xF9\x2A\x18\x73\x0D"
+			  "\xDC\x7E\x6B\x7B\x1B\x43\x8C\xEA"
+			  "\x13\xC8\x1A\x47\x0A\x2D\x6D\x56"
+			  "\xCD\xD2\xE7\x53\x1A\xAB\x1C\x3C"
+			  "\xC5\x9B\x03\x70\x29\x2A\x49\x09"
+			  "\x67\xA1\xEA\xD6\x3A\x5B\xBF\x71"
+			  "\x1D\x48\x64\x6C\xFB\xC0\x9E\x36",
+		.len	= 1008,
 	},
 };
 
-static const struct cipher_testvec camellia_lrw_dec_tv_template[] = {
+static const struct cipher_testvec camellia_lrw_tv_template[] = {
 	/* Generated from AES-LRW test vectors */
-	/* same as enc vectors with input and result reversed */
 	{
 		.key	= "\x45\x62\xac\x25\xf8\x28\x17\x6d"
 			  "\x4c\x26\x84\x14\xb5\x68\x01\x85"
@@ -32854,12 +29069,11 @@ static const struct cipher_testvec camellia_lrw_dec_tv_template[] = {
 		.klen	= 32,
 		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x92\x68\x19\xd7\xb7\x5b\x0a\x31"
-			  "\x97\xcc\x72\xbe\x99\x17\xeb\x3e",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
 			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
+		.ctext	= "\x92\x68\x19\xd7\xb7\x5b\x0a\x31"
+			  "\x97\xcc\x72\xbe\x99\x17\xeb\x3e",
+		.len	= 16,
 	}, {
 		.key	= "\x59\x70\x47\x14\xf5\x57\x47\x8c"
 			  "\xd7\x79\xe8\x0f\x54\x88\x79\x44"
@@ -32868,12 +29082,11 @@ static const struct cipher_testvec camellia_lrw_dec_tv_template[] = {
 		.klen	= 32,
 		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x02",
-		.input	= "\x73\x09\xb7\x50\xb6\x77\x30\x50"
-			  "\x5c\x8a\x9c\x26\x77\x9d\xfc\x4a",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
 			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
+		.ctext	= "\x73\x09\xb7\x50\xb6\x77\x30\x50"
+			  "\x5c\x8a\x9c\x26\x77\x9d\xfc\x4a",
+		.len	= 16,
 	}, {
 		.key	= "\xd8\x2a\x91\x34\xb2\x6a\x56\x50"
 			  "\x30\xfe\x69\xe2\x37\x7f\x98\x47"
@@ -32882,12 +29095,11 @@ static const struct cipher_testvec camellia_lrw_dec_tv_template[] = {
 		.klen	= 32,
 		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x90\xae\x83\xe0\x22\xb9\x60\x91"
-			  "\xfa\xa9\xb7\x98\xe3\xed\x87\x01",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
 			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
+		.ctext	= "\x90\xae\x83\xe0\x22\xb9\x60\x91"
+			  "\xfa\xa9\xb7\x98\xe3\xed\x87\x01",
+		.len	= 16,
 	}, {
 		.key	= "\x0f\x6a\xef\xf8\xd3\xd2\xbb\x15"
 			  "\x25\x83\xf7\x3c\x1f\x01\x28\x74"
@@ -32897,12 +29109,11 @@ static const struct cipher_testvec camellia_lrw_dec_tv_template[] = {
 		.klen	= 40,
 		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x99\xe9\x6e\xd4\xc9\x21\xa5\xf0"
-			  "\xd8\x83\xef\xd9\x07\x16\x5f\x35",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
 			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
+		.ctext	= "\x99\xe9\x6e\xd4\xc9\x21\xa5\xf0"
+			  "\xd8\x83\xef\xd9\x07\x16\x5f\x35",
+		.len	= 16,
 	}, {
 		.key	= "\x8a\xd4\xee\x10\x2f\xbd\x81\xff"
 			  "\xf8\x86\xce\xac\x93\xc5\xad\xc6"
@@ -32910,122 +29121,54 @@ static const struct cipher_testvec camellia_lrw_dec_tv_template[] = {
 			  "\x52\x13\xb2\xb7\xf0\xff\x11\xd8"
 			  "\xd6\x08\xd0\xcd\x2e\xb1\x17\x6f",
 		.klen	= 40,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x42\x88\xf4\xcb\x21\x11\x6d\x8e"
-			  "\xde\x1a\xf2\x29\xf1\x4a\xe0\x15",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x40\xaa\x34\x86\x4a\x8f\x78\xb9"
-			  "\xdb\xdb\x0f\x3d\x48\x70\xbe\x8d",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
-			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
-			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
-			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
-			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
-			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x02\x00\x00\x00\x00",
-		.input	= "\x04\xab\x28\x37\x31\x7a\x26\xab"
-			  "\xa1\x70\x1b\x9c\xe7\xdd\x83\xff",
-		.ilen	= 16,
-		.result	= "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x41\x42\x43\x44\x45\x46",
-		.rlen	= 16,
-	}, {
-		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
-			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
-			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
-			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
-			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
-			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
-		.klen	= 48,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x90\x69\x8e\xf2\x14\x86\x59\xf9"
-			  "\xec\xe7\xfa\x3f\x48\x9d\x7f\x96"
-			  "\x67\x76\xac\x2c\xd2\x63\x18\x93"
-			  "\x13\xf8\xf1\xf6\x71\x77\xb3\xee"
-			  "\x93\xb2\xcc\xf3\x26\xc1\x16\x4f"
-			  "\xd4\xe8\x43\xc1\x68\xa3\x3e\x06"
-			  "\x38\x51\xff\xa8\xb9\xa4\xeb\xb1"
-			  "\x62\xdd\x78\x81\xea\x1d\xef\x04"
-			  "\x1d\x07\xc1\x67\xc8\xd6\x77\xa1"
-			  "\x84\x95\xf4\x9a\xd9\xbc\x2d\xe2"
-			  "\xf6\x80\xfc\x91\x2a\xbc\x42\xa0"
-			  "\x40\x41\x69\xaa\x71\xc0\x37\xec"
-			  "\x39\xf3\xf2\xec\x82\xc3\x88\x79"
-			  "\xbc\xc3\xaa\xb7\xcf\x6a\x72\x80"
-			  "\x4c\xf4\x84\x8f\x13\x9e\x94\x5c"
-			  "\xe5\xb2\x91\xbb\x92\x51\x4d\xf1"
-			  "\xd6\x0d\x71\x6b\x7a\xc2\x2f\x12"
-			  "\x6f\x75\xc7\x80\x99\x50\x84\xcf"
-			  "\xa8\xeb\xd6\xe1\x1c\x59\x81\x7e"
-			  "\xb9\xb3\xde\x7a\x93\x14\x12\xa2"
-			  "\xf7\x43\xb3\x9d\x1a\x87\x65\x91"
-			  "\x42\x08\x40\x82\x06\x1c\x2d\x55"
-			  "\x6e\x48\xd5\x74\x07\x6e\x9d\x80"
-			  "\xeb\xb4\x97\xa1\x36\xdf\xfa\x74"
-			  "\x79\x7f\x5a\x75\xe7\x71\xc8\x8c"
-			  "\x7e\xf8\x3a\x77\xcd\x32\x05\xf9"
-			  "\x3d\xd4\xe9\xa2\xbb\xc4\x8b\x83"
-			  "\x42\x5c\x82\xfa\xe9\x4b\x96\x3b"
-			  "\x7f\x89\x8b\xf9\xf1\x87\xda\xf0"
-			  "\x87\xef\x13\x5d\xf0\xe2\xc5\xc1"
-			  "\xed\x14\xa9\x57\x19\x63\x40\x04"
-			  "\x24\xeb\x6e\x19\xd1\x3d\x70\x78"
-			  "\xeb\xda\x55\x70\x2c\x4f\x41\x5b"
-			  "\x56\x9f\x1a\xd3\xac\xf1\xc0\xc3"
-			  "\x21\xec\xd7\xd2\x55\x32\x7c\x2e"
-			  "\x3c\x48\x8e\xb4\x85\x35\x47\xfe"
-			  "\xe2\x88\x79\x98\x6a\xc9\x8d\xff"
-			  "\xe9\x89\x6e\xb8\xe2\x97\x00\xbd"
-			  "\xa4\x8f\xba\xd0\x8c\xcb\x79\x99"
-			  "\xb3\xb2\xb2\x7a\xc3\xb7\xef\x75"
-			  "\x23\x52\x76\xc3\x50\x6e\x66\xf8"
-			  "\xa2\xe2\xce\xba\x40\x21\x3f\xc9"
-			  "\x0a\x32\x7f\xf7\x08\x8c\x66\xcf"
-			  "\xd3\xdf\x57\x59\x83\xb8\xe1\x85"
-			  "\xd6\x8f\xfb\x48\x1f\x3a\xc4\x2f"
-			  "\xb4\x2d\x58\xab\xd8\x7f\x5e\x3a"
-			  "\xbc\x62\x3e\xe2\x6a\x52\x0d\x76"
-			  "\x2f\x1c\x1a\x30\xed\x95\x2a\x44"
-			  "\x35\xa5\x83\x04\x84\x01\x99\x56"
-			  "\xb7\xe3\x10\x96\xfa\xdc\x19\xdd"
-			  "\xe2\x7f\xcb\xa0\x49\x1b\xff\x4c"
-			  "\x73\xf6\xbb\x94\x00\xe8\xa9\x3d"
-			  "\xe2\x20\xe9\x3f\xfa\x07\x5d\x77"
-			  "\x06\xd5\x4f\x4d\x02\xb8\x40\x1b"
-			  "\x30\xed\x1a\x50\x19\xef\xc4\x2c"
-			  "\x02\xd9\xc5\xd3\x11\x33\x37\xe5"
-			  "\x2b\xa3\x95\xa6\xee\xd8\x74\x1d"
-			  "\x68\xa0\xeb\xbf\xdd\x5e\x99\x96"
-			  "\x91\xc3\x94\x24\xa5\x12\xa2\x37"
-			  "\xb3\xac\xcf\x2a\xfd\x55\x34\xfe"
-			  "\x79\x92\x3e\xe6\x1b\x49\x57\x5d"
-			  "\x93\x6c\x01\xf7\xcc\x4e\x20\xd1"
-			  "\xb2\x1a\xd8\x4c\xbd\x1d\x10\xe9"
-			  "\x5a\xa8\x92\x7f\xba\xe6\x0c\x95",
-		.ilen	= 512,
-		.result	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x02\x00\x00\x00\x00",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x42\x88\xf4\xcb\x21\x11\x6d\x8e"
+			  "\xde\x1a\xf2\x29\xf1\x4a\xe0\x15",
+		.len	= 16,
+	}, {
+		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
+			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
+			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
+			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
+			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
+			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x40\xaa\x34\x86\x4a\x8f\x78\xb9"
+			  "\xdb\xdb\x0f\x3d\x48\x70\xbe\x8d",
+		.len	= 16,
+	}, {
+		.key	= "\xfb\x76\x15\xb2\x3d\x80\x89\x1d"
+			  "\xd4\x70\x98\x0b\xc7\x95\x84\xc8"
+			  "\xb2\xfb\x64\xce\x60\x97\x87\x8d"
+			  "\x17\xfc\xe4\x5a\x49\xe8\x30\xb7"
+			  "\x6e\x78\x17\xe7\x2d\x5e\x12\xd4"
+			  "\x60\x64\x04\x7a\xf1\x2f\x9e\x0c",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x02\x00\x00\x00\x00",
+		.ptext	= "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x41\x42\x43\x44\x45\x46",
+		.ctext	= "\x04\xab\x28\x37\x31\x7a\x26\xab"
+			  "\xa1\x70\x1b\x9c\xe7\xdd\x83\xff",
+		.len	= 16,
+	}, {
+		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
+			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
+			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
+			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
+			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
+			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.ptext	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
 			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
 			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
 			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
@@ -33089,14 +29232,78 @@ static const struct cipher_testvec camellia_lrw_dec_tv_template[] = {
 			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
 			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
 			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
-		.rlen	= 512,
+		.ctext	= "\x90\x69\x8e\xf2\x14\x86\x59\xf9"
+			  "\xec\xe7\xfa\x3f\x48\x9d\x7f\x96"
+			  "\x67\x76\xac\x2c\xd2\x63\x18\x93"
+			  "\x13\xf8\xf1\xf6\x71\x77\xb3\xee"
+			  "\x93\xb2\xcc\xf3\x26\xc1\x16\x4f"
+			  "\xd4\xe8\x43\xc1\x68\xa3\x3e\x06"
+			  "\x38\x51\xff\xa8\xb9\xa4\xeb\xb1"
+			  "\x62\xdd\x78\x81\xea\x1d\xef\x04"
+			  "\x1d\x07\xc1\x67\xc8\xd6\x77\xa1"
+			  "\x84\x95\xf4\x9a\xd9\xbc\x2d\xe2"
+			  "\xf6\x80\xfc\x91\x2a\xbc\x42\xa0"
+			  "\x40\x41\x69\xaa\x71\xc0\x37\xec"
+			  "\x39\xf3\xf2\xec\x82\xc3\x88\x79"
+			  "\xbc\xc3\xaa\xb7\xcf\x6a\x72\x80"
+			  "\x4c\xf4\x84\x8f\x13\x9e\x94\x5c"
+			  "\xe5\xb2\x91\xbb\x92\x51\x4d\xf1"
+			  "\xd6\x0d\x71\x6b\x7a\xc2\x2f\x12"
+			  "\x6f\x75\xc7\x80\x99\x50\x84\xcf"
+			  "\xa8\xeb\xd6\xe1\x1c\x59\x81\x7e"
+			  "\xb9\xb3\xde\x7a\x93\x14\x12\xa2"
+			  "\xf7\x43\xb3\x9d\x1a\x87\x65\x91"
+			  "\x42\x08\x40\x82\x06\x1c\x2d\x55"
+			  "\x6e\x48\xd5\x74\x07\x6e\x9d\x80"
+			  "\xeb\xb4\x97\xa1\x36\xdf\xfa\x74"
+			  "\x79\x7f\x5a\x75\xe7\x71\xc8\x8c"
+			  "\x7e\xf8\x3a\x77\xcd\x32\x05\xf9"
+			  "\x3d\xd4\xe9\xa2\xbb\xc4\x8b\x83"
+			  "\x42\x5c\x82\xfa\xe9\x4b\x96\x3b"
+			  "\x7f\x89\x8b\xf9\xf1\x87\xda\xf0"
+			  "\x87\xef\x13\x5d\xf0\xe2\xc5\xc1"
+			  "\xed\x14\xa9\x57\x19\x63\x40\x04"
+			  "\x24\xeb\x6e\x19\xd1\x3d\x70\x78"
+			  "\xeb\xda\x55\x70\x2c\x4f\x41\x5b"
+			  "\x56\x9f\x1a\xd3\xac\xf1\xc0\xc3"
+			  "\x21\xec\xd7\xd2\x55\x32\x7c\x2e"
+			  "\x3c\x48\x8e\xb4\x85\x35\x47\xfe"
+			  "\xe2\x88\x79\x98\x6a\xc9\x8d\xff"
+			  "\xe9\x89\x6e\xb8\xe2\x97\x00\xbd"
+			  "\xa4\x8f\xba\xd0\x8c\xcb\x79\x99"
+			  "\xb3\xb2\xb2\x7a\xc3\xb7\xef\x75"
+			  "\x23\x52\x76\xc3\x50\x6e\x66\xf8"
+			  "\xa2\xe2\xce\xba\x40\x21\x3f\xc9"
+			  "\x0a\x32\x7f\xf7\x08\x8c\x66\xcf"
+			  "\xd3\xdf\x57\x59\x83\xb8\xe1\x85"
+			  "\xd6\x8f\xfb\x48\x1f\x3a\xc4\x2f"
+			  "\xb4\x2d\x58\xab\xd8\x7f\x5e\x3a"
+			  "\xbc\x62\x3e\xe2\x6a\x52\x0d\x76"
+			  "\x2f\x1c\x1a\x30\xed\x95\x2a\x44"
+			  "\x35\xa5\x83\x04\x84\x01\x99\x56"
+			  "\xb7\xe3\x10\x96\xfa\xdc\x19\xdd"
+			  "\xe2\x7f\xcb\xa0\x49\x1b\xff\x4c"
+			  "\x73\xf6\xbb\x94\x00\xe8\xa9\x3d"
+			  "\xe2\x20\xe9\x3f\xfa\x07\x5d\x77"
+			  "\x06\xd5\x4f\x4d\x02\xb8\x40\x1b"
+			  "\x30\xed\x1a\x50\x19\xef\xc4\x2c"
+			  "\x02\xd9\xc5\xd3\x11\x33\x37\xe5"
+			  "\x2b\xa3\x95\xa6\xee\xd8\x74\x1d"
+			  "\x68\xa0\xeb\xbf\xdd\x5e\x99\x96"
+			  "\x91\xc3\x94\x24\xa5\x12\xa2\x37"
+			  "\xb3\xac\xcf\x2a\xfd\x55\x34\xfe"
+			  "\x79\x92\x3e\xe6\x1b\x49\x57\x5d"
+			  "\x93\x6c\x01\xf7\xcc\x4e\x20\xd1"
+			  "\xb2\x1a\xd8\x4c\xbd\x1d\x10\xe9"
+			  "\x5a\xa8\x92\x7f\xba\xe6\x0c\x95",
+		.len	= 512,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 512 - 20, 4, 16 },
 	},
 };
 
-static const struct cipher_testvec camellia_xts_enc_tv_template[] = {
+static const struct cipher_testvec camellia_xts_tv_template[] = {
 	/* Generated from AES-XTS test vectors */
 	{
 		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
@@ -33106,16 +29313,15 @@ static const struct cipher_testvec camellia_xts_enc_tv_template[] = {
 		.klen	= 32,
 		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 32,
-		.result	= "\x06\xcb\xa5\xf1\x04\x63\xb2\x41"
+		.ctext	= "\x06\xcb\xa5\xf1\x04\x63\xb2\x41"
 			  "\xdc\xca\xfa\x09\xba\x74\xb9\x05"
 			  "\x78\xba\xa4\xf8\x67\x4d\x7e\xad"
 			  "\x20\x18\xf5\x0c\x41\x16\x2a\x61",
-		.rlen	= 32,
+		.len	= 32,
 	}, {
 		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
 			  "\x11\x11\x11\x11\x11\x11\x11\x11"
@@ -33124,185 +29330,41 @@ static const struct cipher_testvec camellia_xts_enc_tv_template[] = {
 		.klen	= 32,
 		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
 			  "\x44\x44\x44\x44\x44\x44\x44\x44"
 			  "\x44\x44\x44\x44\x44\x44\x44\x44"
 			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen	= 32,
-		.result	= "\xc2\xb9\xdc\x44\x1d\xdf\xf2\x86"
+		.ctext	= "\xc2\xb9\xdc\x44\x1d\xdf\xf2\x86"
 			  "\x8d\x35\x42\x0a\xa5\x5e\x3d\x4f"
 			  "\xb5\x37\x06\xff\xbd\xd4\x91\x70"
-			  "\x80\x1f\xb2\x39\x10\x89\x44\xf5",
-		.rlen	= 32,
-	}, {
-		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.ilen	= 32,
-		.result	= "\x52\x1f\x9d\xf5\x5a\x58\x5a\x7e"
-			  "\x9f\xd0\x8e\x02\x9c\x9a\x6a\xa7"
-			  "\xb4\x3b\xce\xe7\x17\xaa\x89\x6a"
-			  "\x35\x3c\x6b\xb5\x61\x1c\x79\x38",
-		.rlen	= 32,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\xc7\xf9\x0a\xaa\xcb\xb5\x8f\x33"
-			  "\x60\xc3\xe9\x47\x90\xb7\x50\x57"
-			  "\xa3\xad\x81\x2f\xf5\x22\x96\x02"
-			  "\xaa\x7f\xea\xac\x29\x78\xca\x2a"
-			  "\x7c\xcd\x31\x1a\x3c\x40\x0a\x73"
-			  "\x09\x66\xad\x72\x0e\x4d\x5d\x77"
-			  "\xbc\xb8\x76\x80\x37\x59\xa9\x01"
-			  "\x9e\xfb\xdb\x6c\x93\xef\xb6\x8d"
-			  "\x1e\xc1\x94\xa8\xd4\xb5\xb0\x01"
-			  "\xd5\x01\x97\x28\xcd\x7a\x1f\xe8"
-			  "\x08\xda\x76\x00\x65\xcf\x7b\x31"
-			  "\xc6\xfa\xf2\x3b\x00\xa7\x6a\x9e"
-			  "\x6c\x43\x80\x87\xe0\xbb\x4e\xe5"
-			  "\xdc\x8a\xdf\xc3\x1d\x1b\x41\x04"
-			  "\xfb\x54\xdd\x29\x27\xc2\x65\x17"
-			  "\x36\x88\xb0\x85\x8d\x73\x7e\x4b"
-			  "\x1d\x16\x8a\x52\xbc\xa6\xbc\xa4"
-			  "\x8c\xd1\x04\x16\xbf\x8c\x01\x0f"
-			  "\x7e\x6b\x59\x15\x29\xd1\x9b\xd3"
-			  "\x6c\xee\xac\xdc\x45\x58\xca\x5b"
-			  "\x70\x0e\x6a\x12\x86\x82\x79\x9f"
-			  "\x16\xd4\x9d\x67\xcd\x70\x65\x26"
-			  "\x21\x72\x1e\xa1\x94\x8a\x83\x0c"
-			  "\x92\x42\x58\x5e\xa2\xc5\x31\xf3"
-			  "\x7b\xd1\x31\xd4\x15\x80\x31\x61"
-			  "\x5c\x53\x10\xdd\xea\xc8\x83\x5c"
-			  "\x7d\xa7\x05\x66\xcc\x1e\xbb\x05"
-			  "\x47\xae\xb4\x0f\x84\xd8\xf6\xb5"
-			  "\xa1\xc6\x52\x00\x52\xe8\xdc\xd9"
-			  "\x16\x31\xb2\x47\x91\x67\xaa\x28"
-			  "\x2c\x29\x85\xa3\xf7\xf2\x24\x93"
-			  "\x23\x80\x1f\xa8\x1b\x82\x8d\xdc"
-			  "\x9f\x0b\xcd\xb4\x3c\x20\xbc\xec"
-			  "\x4f\xc7\xee\xf8\xfd\xd9\xfb\x7e"
-			  "\x3f\x0d\x23\xfa\x3f\xa7\xcc\x66"
-			  "\x1c\xfe\xa6\x86\xf6\xf7\x85\xc7"
-			  "\x43\xc1\xd4\xfc\xe4\x79\xc9\x1d"
-			  "\xf8\x89\xcd\x20\x27\x84\x5d\x5c"
-			  "\x8e\x4f\x1f\xeb\x08\x21\x4f\xa3"
-			  "\xe0\x7e\x0b\x9c\xe7\x42\xcf\xb7"
-			  "\x3f\x43\xcc\x86\x71\x34\x6a\xd9"
-			  "\x5e\xec\x8f\x36\xc9\x0a\x03\xfe"
-			  "\x18\x41\xdc\x9e\x2e\x75\x20\x3e"
-			  "\xcc\x77\xe0\x8f\xe8\x43\x37\x4c"
-			  "\xed\x1a\x5a\xb3\xfa\x43\xc9\x71"
-			  "\x9f\xc5\xce\xcf\xff\xe7\x77\x1e"
-			  "\x35\x93\xde\x6b\xc0\x6a\x7e\xa9"
-			  "\x34\xb8\x27\x74\x08\xda\xf2\x4a"
-			  "\x23\x5b\x9f\x55\x3a\x57\x82\x52"
-			  "\xea\x6d\xc3\xc7\xf2\xc8\xb5\xdc"
-			  "\xc5\xb9\xbb\xaa\xf2\x29\x9f\x49"
-			  "\x7a\xef\xfe\xdc\x9f\xc9\x28\xe2"
-			  "\x96\x0b\x35\x84\x05\x0d\xd6\x2a"
-			  "\xea\x5a\xbf\x69\xde\xee\x4f\x8f"
-			  "\x84\xb9\xcf\xa7\x57\xea\xe0\xe8"
-			  "\x96\xef\x0f\x0e\xec\xc7\xa6\x74"
-			  "\xb1\xfe\x7a\x6d\x11\xdd\x0e\x15"
-			  "\x4a\x1e\x73\x7f\x55\xea\xf6\xe1"
-			  "\x5b\xb6\x71\xda\xb0\x0c\xba\x26"
-			  "\x5c\x48\x38\x6d\x1c\x32\xb2\x7d"
-			  "\x05\x87\xc2\x1e\x7e\x2d\xd4\x33"
-			  "\xcc\x06\xdb\xe7\x82\x29\x63\xd1"
-			  "\x52\x84\x4f\xee\x27\xe8\x02\xd4"
-			  "\x34\x3c\x69\xc2\xbd\x20\xe6\x7a",
-		.rlen	= 512,
+			  "\x80\x1f\xb2\x39\x10\x89\x44\xf5",
+		.len	= 32,
+	}, {
+		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
+			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22"
+			  "\x22\x22\x22\x22\x22\x22\x22\x22",
+		.klen	= 32,
+		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44"
+			  "\x44\x44\x44\x44\x44\x44\x44\x44",
+		.ctext	= "\x52\x1f\x9d\xf5\x5a\x58\x5a\x7e"
+			  "\x9f\xd0\x8e\x02\x9c\x9a\x6a\xa7"
+			  "\xb4\x3b\xce\xe7\x17\xaa\x89\x6a"
+			  "\x35\x3c\x6b\xb5\x61\x1c\x79\x38",
+		.len	= 32,
 	}, {
 		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
 			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
 			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95",
+		.klen	= 32,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
 			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
 			  "\x10\x11\x12\x13\x14\x15\x16\x17"
 			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
@@ -33366,144 +29428,7 @@ static const struct cipher_testvec camellia_xts_enc_tv_template[] = {
 			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
 			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
 			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.ilen	= 512,
-		.result	= "\x49\xcd\xb8\xbf\x2f\x73\x37\x28"
-			  "\x9a\x7f\x6e\x57\x55\xb8\x07\x88"
-			  "\x4a\x0d\x8b\x55\x60\xed\xb6\x7b"
-			  "\xf1\x74\xac\x96\x05\x7b\x32\xca"
-			  "\xd1\x4e\xf1\x58\x29\x16\x24\x6c"
-			  "\xf2\xb3\xe4\x88\x84\xac\x4d\xee"
-			  "\x97\x07\x82\xf0\x07\x12\x38\x0a"
-			  "\x67\x62\xaf\xfd\x85\x9f\x0a\x55"
-			  "\xa5\x20\xc5\x60\xe4\x68\x53\xa4"
-			  "\x0e\x2e\x65\xe3\xe4\x0c\x30\x7c"
-			  "\x1c\x01\x4f\x55\xa9\x13\xeb\x25"
-			  "\x21\x87\xbc\xd3\xe7\x67\x4f\x38"
-			  "\xa8\x14\x25\x71\xe9\x2e\x4c\x21"
-			  "\x41\x82\x0c\x45\x39\x35\xa8\x75"
-			  "\x03\x29\x01\x84\x8c\xab\x48\xbe"
-			  "\x11\x56\x22\x67\xb7\x67\x1a\x09"
-			  "\xa1\x72\x25\x41\x3c\x39\x65\x80"
-			  "\x7d\x2f\xf8\x2c\x73\x04\x58\x9d"
-			  "\xdd\x16\x8b\x63\x70\x4e\xc5\x17"
-			  "\x21\xe0\x84\x51\x4b\x6f\x05\x52"
-			  "\xe3\x63\x34\xfa\xa4\xaf\x33\x20"
-			  "\xc1\xae\x32\xc4\xb8\x2b\xdb\x76"
-			  "\xd9\x02\x31\x2f\xa3\xc6\xd0\x7b"
-			  "\xaf\x1b\x84\xe3\x9b\xbf\xa6\xe0"
-			  "\xb8\x8a\x13\x88\x71\xf4\x11\xa5"
-			  "\xe9\xa9\x10\x33\xe0\xbe\x49\x89"
-			  "\x41\x22\xf5\x9d\x80\x3e\x3b\x76"
-			  "\x01\x16\x50\x6e\x7c\x6a\x81\xe9"
-			  "\x13\x2c\xde\xb2\x5f\x79\xba\xb2"
-			  "\xb1\x75\xae\xd2\x07\x98\x4b\x69"
-			  "\xae\x7d\x5b\x90\xc2\x6c\xe6\x98"
-			  "\xd3\x4c\xa1\xa3\x9c\xc9\x33\x6a"
-			  "\x0d\x23\xb1\x79\x25\x13\x4b\xe5"
-			  "\xaf\x93\x20\x5c\x7f\x06\x7a\x34"
-			  "\x0b\x78\xe3\x67\x26\xe0\xad\x95"
-			  "\xc5\x4e\x26\x22\xcf\x73\x77\x62"
-			  "\x3e\x10\xd7\x90\x4b\x52\x1c\xc9"
-			  "\xef\x38\x52\x18\x0e\x29\x7e\xef"
-			  "\x34\xfe\x31\x95\xc5\xbc\xa8\xe2"
-			  "\xa8\x4e\x9f\xea\xa6\xf0\xfe\x5d"
-			  "\xc5\x39\x86\xed\x2f\x6d\xa0\xfe"
-			  "\x96\xcd\x41\x10\x78\x4e\x0c\xc9"
-			  "\xc3\x6d\x0f\xb7\xe8\xe0\x62\xab"
-			  "\x8b\xf1\x21\x89\xa1\x12\xaa\xfa"
-			  "\x9d\x70\xbe\x4c\xa8\x98\x89\x01"
-			  "\xb9\xe2\x61\xde\x0c\x4a\x0b\xaa"
-			  "\x89\xf5\x14\x79\x18\x8f\x3b\x0d"
-			  "\x21\x17\xf8\x59\x15\x24\x64\x22"
-			  "\x57\x48\x80\xd5\x3d\x92\x30\x07"
-			  "\xd9\xa1\x4a\x23\x16\x43\x48\x0e"
-			  "\x2b\x2d\x1b\x87\xef\x7e\xbd\xfa"
-			  "\x49\xbc\x7e\x68\x6e\xa8\x46\x95"
-			  "\xad\x5e\xfe\x0a\xa8\xd3\x1a\x5d"
-			  "\x6b\x84\xf3\x00\xba\x52\x05\x02"
-			  "\xe3\x96\x4e\xb6\x79\x3f\x43\xd3"
-			  "\x4d\x3f\xd6\xab\x0a\xc4\x75\x2d"
-			  "\xd1\x08\xc3\x6a\xc8\x37\x29\xa0"
-			  "\xcc\x9a\x05\xdd\x5c\xe1\xff\x66"
-			  "\xf2\x7a\x1d\xf2\xaf\xa9\x48\x89"
-			  "\xf5\x21\x0f\x02\x48\x83\x74\xbf"
-			  "\x2e\xe6\x93\x7b\xa0\xf4\xb1\x2b"
-			  "\xb1\x02\x0a\x5c\x79\x19\x3b\x75"
-			  "\xb7\x16\xd8\x12\x5c\xcd\x7d\x4e"
-			  "\xd5\xc6\x99\xcc\x4e\x6c\x94\x95",
-		.rlen	= 512,
-		.also_non_np = 1,
-		.np	= 3,
-		.tap	= { 512 - 20, 4, 16 },
-	},
-};
-
-static const struct cipher_testvec camellia_xts_dec_tv_template[] = {
-	/* Generated from AES-XTS test vectors */
-	/* same as enc vectors with input and result reversed */
-	{
-		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x06\xcb\xa5\xf1\x04\x63\xb2\x41"
-			  "\xdc\xca\xfa\x09\xba\x74\xb9\x05"
-			  "\x78\xba\xa4\xf8\x67\x4d\x7e\xad"
-			  "\x20\x18\xf5\x0c\x41\x16\x2a\x61",
-		.ilen	= 32,
-		.result	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.rlen	= 32,
-	}, {
-		.key	= "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x11\x11\x11\x11\x11\x11\x11\x11"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xc2\xb9\xdc\x44\x1d\xdf\xf2\x86"
-			  "\x8d\x35\x42\x0a\xa5\x5e\x3d\x4f"
-			  "\xb5\x37\x06\xff\xbd\xd4\x91\x70"
-			  "\x80\x1f\xb2\x39\x10\x89\x44\xf5",
-		.ilen	= 32,
-		.result	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen	= 32,
-	}, {
-		.key	= "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8"
-			  "\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22"
-			  "\x22\x22\x22\x22\x22\x22\x22\x22",
-		.klen	= 32,
-		.iv	= "\x33\x33\x33\x33\x33\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x52\x1f\x9d\xf5\x5a\x58\x5a\x7e"
-			  "\x9f\xd0\x8e\x02\x9c\x9a\x6a\xa7"
-			  "\xb4\x3b\xce\xe7\x17\xaa\x89\x6a"
-			  "\x35\x3c\x6b\xb5\x61\x1c\x79\x38",
-		.ilen	= 32,
-		.result	= "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44"
-			  "\x44\x44\x44\x44\x44\x44\x44\x44",
-		.rlen	= 32,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95",
-		.klen	= 32,
-		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\xc7\xf9\x0a\xaa\xcb\xb5\x8f\x33"
+		.ctext	= "\xc7\xf9\x0a\xaa\xcb\xb5\x8f\x33"
 			  "\x60\xc3\xe9\x47\x90\xb7\x50\x57"
 			  "\xa3\xad\x81\x2f\xf5\x22\x96\x02"
 			  "\xaa\x7f\xea\xac\x29\x78\xca\x2a"
@@ -33567,8 +29492,20 @@ static const struct cipher_testvec camellia_xts_dec_tv_template[] = {
 			  "\xcc\x06\xdb\xe7\x82\x29\x63\xd1"
 			  "\x52\x84\x4f\xee\x27\xe8\x02\xd4"
 			  "\x34\x3c\x69\xc2\xbd\x20\xe6\x7a",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+		.len	= 512,
+	}, {
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x62\x49\x77\x57\x24\x70\x93\x69"
+			  "\x99\x59\x57\x49\x66\x96\x76\x27"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95"
+			  "\x02\x88\x41\x97\x16\x93\x99\x37"
+			  "\x51\x05\x82\x09\x74\x94\x45\x92",
+		.klen	= 64,
+		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
 			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
 			  "\x10\x11\x12\x13\x14\x15\x16\x17"
 			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
@@ -33632,20 +29569,7 @@ static const struct cipher_testvec camellia_xts_dec_tv_template[] = {
 			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
 			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
 			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
-	}, {
-		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
-			  "\x23\x53\x60\x28\x74\x71\x35\x26"
-			  "\x62\x49\x77\x57\x24\x70\x93\x69"
-			  "\x99\x59\x57\x49\x66\x96\x76\x27"
-			  "\x31\x41\x59\x26\x53\x58\x97\x93"
-			  "\x23\x84\x62\x64\x33\x83\x27\x95"
-			  "\x02\x88\x41\x97\x16\x93\x99\x37"
-			  "\x51\x05\x82\x09\x74\x94\x45\x92",
-		.klen	= 64,
-		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x49\xcd\xb8\xbf\x2f\x73\x37\x28"
+		.ctext	= "\x49\xcd\xb8\xbf\x2f\x73\x37\x28"
 			  "\x9a\x7f\x6e\x57\x55\xb8\x07\x88"
 			  "\x4a\x0d\x8b\x55\x60\xed\xb6\x7b"
 			  "\xf1\x74\xac\x96\x05\x7b\x32\xca"
@@ -33691,90 +29615,25 @@ static const struct cipher_testvec camellia_xts_dec_tv_template[] = {
 			  "\x8b\xf1\x21\x89\xa1\x12\xaa\xfa"
 			  "\x9d\x70\xbe\x4c\xa8\x98\x89\x01"
 			  "\xb9\xe2\x61\xde\x0c\x4a\x0b\xaa"
-			  "\x89\xf5\x14\x79\x18\x8f\x3b\x0d"
-			  "\x21\x17\xf8\x59\x15\x24\x64\x22"
-			  "\x57\x48\x80\xd5\x3d\x92\x30\x07"
-			  "\xd9\xa1\x4a\x23\x16\x43\x48\x0e"
-			  "\x2b\x2d\x1b\x87\xef\x7e\xbd\xfa"
-			  "\x49\xbc\x7e\x68\x6e\xa8\x46\x95"
-			  "\xad\x5e\xfe\x0a\xa8\xd3\x1a\x5d"
-			  "\x6b\x84\xf3\x00\xba\x52\x05\x02"
-			  "\xe3\x96\x4e\xb6\x79\x3f\x43\xd3"
-			  "\x4d\x3f\xd6\xab\x0a\xc4\x75\x2d"
-			  "\xd1\x08\xc3\x6a\xc8\x37\x29\xa0"
-			  "\xcc\x9a\x05\xdd\x5c\xe1\xff\x66"
-			  "\xf2\x7a\x1d\xf2\xaf\xa9\x48\x89"
-			  "\xf5\x21\x0f\x02\x48\x83\x74\xbf"
-			  "\x2e\xe6\x93\x7b\xa0\xf4\xb1\x2b"
-			  "\xb1\x02\x0a\x5c\x79\x19\x3b\x75"
-			  "\xb7\x16\xd8\x12\x5c\xcd\x7d\x4e"
-			  "\xd5\xc6\x99\xcc\x4e\x6c\x94\x95",
-		.ilen	= 512,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
-			  "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-			  "\x10\x11\x12\x13\x14\x15\x16\x17"
-			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-			  "\x20\x21\x22\x23\x24\x25\x26\x27"
-			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
-			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\x40\x41\x42\x43\x44\x45\x46\x47"
-			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
-			  "\x50\x51\x52\x53\x54\x55\x56\x57"
-			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
-			  "\x60\x61\x62\x63\x64\x65\x66\x67"
-			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
-			  "\x70\x71\x72\x73\x74\x75\x76\x77"
-			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
-			  "\x80\x81\x82\x83\x84\x85\x86\x87"
-			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
-			  "\x90\x91\x92\x93\x94\x95\x96\x97"
-			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
-			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
-			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
-			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
-			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
-			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
-			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
-			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
-		.rlen	= 512,
+			  "\x89\xf5\x14\x79\x18\x8f\x3b\x0d"
+			  "\x21\x17\xf8\x59\x15\x24\x64\x22"
+			  "\x57\x48\x80\xd5\x3d\x92\x30\x07"
+			  "\xd9\xa1\x4a\x23\x16\x43\x48\x0e"
+			  "\x2b\x2d\x1b\x87\xef\x7e\xbd\xfa"
+			  "\x49\xbc\x7e\x68\x6e\xa8\x46\x95"
+			  "\xad\x5e\xfe\x0a\xa8\xd3\x1a\x5d"
+			  "\x6b\x84\xf3\x00\xba\x52\x05\x02"
+			  "\xe3\x96\x4e\xb6\x79\x3f\x43\xd3"
+			  "\x4d\x3f\xd6\xab\x0a\xc4\x75\x2d"
+			  "\xd1\x08\xc3\x6a\xc8\x37\x29\xa0"
+			  "\xcc\x9a\x05\xdd\x5c\xe1\xff\x66"
+			  "\xf2\x7a\x1d\xf2\xaf\xa9\x48\x89"
+			  "\xf5\x21\x0f\x02\x48\x83\x74\xbf"
+			  "\x2e\xe6\x93\x7b\xa0\xf4\xb1\x2b"
+			  "\xb1\x02\x0a\x5c\x79\x19\x3b\x75"
+			  "\xb7\x16\xd8\x12\x5c\xcd\x7d\x4e"
+			  "\xd5\xc6\x99\xcc\x4e\x6c\x94\x95",
+		.len	= 512,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 512 - 20, 4, 16 },
@@ -33784,91 +29643,45 @@ static const struct cipher_testvec camellia_xts_dec_tv_template[] = {
 /*
  * SEED test vectors
  */
-static const struct cipher_testvec seed_enc_tv_template[] = {
+static const struct cipher_testvec seed_tv_template[] = {
 	{
 		.key    = zeroed_string,
 		.klen	= 16,
-		.input  = "\x00\x01\x02\x03\x04\x05\x06\x07"
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
 			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.ilen	= 16,
-		.result	= "\x5e\xba\xc6\xe0\x05\x4e\x16\x68"
+		.ctext	= "\x5e\xba\xc6\xe0\x05\x4e\x16\x68"
 			  "\x19\xaf\xf1\xcc\x6d\x34\x6c\xdb",
-		.rlen	= 16,
+		.len	= 16,
 	}, {
 		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
 			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
 		.klen	= 16,
-		.input	= zeroed_string,
-		.ilen	= 16,
-		.result	= "\xc1\x1f\x22\xf2\x01\x40\x50\x50"
+		.ptext	= zeroed_string,
+		.ctext	= "\xc1\x1f\x22\xf2\x01\x40\x50\x50"
 			  "\x84\x48\x35\x97\xe4\x37\x0f\x43",
-		.rlen	= 16,
+		.len	= 16,
 	}, {
 		.key	= "\x47\x06\x48\x08\x51\xe6\x1b\xe8"
 			  "\x5d\x74\xbf\xb3\xfd\x95\x61\x85",
 		.klen	= 16,
-		.input	= "\x83\xa2\xf8\xa2\x88\x64\x1f\xb9"
+		.ptext	= "\x83\xa2\xf8\xa2\x88\x64\x1f\xb9"
 			  "\xa4\xe9\xa5\xcc\x2f\x13\x1c\x7d",
-		.ilen	= 16,
-		.result	= "\xee\x54\xd1\x3e\xbc\xae\x70\x6d"
+		.ctext	= "\xee\x54\xd1\x3e\xbc\xae\x70\x6d"
 			  "\x22\x6b\xc3\x14\x2c\xd4\x0d\x4a",
-		.rlen	= 16,
+		.len	= 16,
 	}, {
 		.key	= "\x28\xdb\xc3\xbc\x49\xff\xd8\x7d"
 			  "\xcf\xa5\x09\xb1\x1d\x42\x2b\xe7",
 		.klen	= 16,
-		.input	= "\xb4\x1e\x6b\xe2\xeb\xa8\x4a\x14"
+		.ptext	= "\xb4\x1e\x6b\xe2\xeb\xa8\x4a\x14"
 			  "\x8e\x2e\xed\x84\x59\x3c\x5e\xc7",
-		.ilen	= 16,
-		.result	= "\x9b\x9b\x7b\xfc\xd1\x81\x3c\xb9"
-			  "\x5d\x0b\x36\x18\xf4\x0f\x51\x22",
-		.rlen	= 16,
-	}
-};
-
-static const struct cipher_testvec seed_dec_tv_template[] = {
-	{
-		.key    = zeroed_string,
-		.klen	= 16,
-		.input	= "\x5e\xba\xc6\xe0\x05\x4e\x16\x68"
-			  "\x19\xaf\xf1\xcc\x6d\x34\x6c\xdb",
-		.ilen	= 16,
-		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.rlen	= 16,
-	}, {
-		.key	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.klen	= 16,
-		.input	= "\xc1\x1f\x22\xf2\x01\x40\x50\x50"
-			  "\x84\x48\x35\x97\xe4\x37\x0f\x43",
-		.ilen	= 16,
-		.result	= zeroed_string,
-		.rlen	= 16,
-	}, {
-		.key	= "\x47\x06\x48\x08\x51\xe6\x1b\xe8"
-			  "\x5d\x74\xbf\xb3\xfd\x95\x61\x85",
-		.klen	= 16,
-		.input	= "\xee\x54\xd1\x3e\xbc\xae\x70\x6d"
-			  "\x22\x6b\xc3\x14\x2c\xd4\x0d\x4a",
-		.ilen	= 16,
-		.result	= "\x83\xa2\xf8\xa2\x88\x64\x1f\xb9"
-			  "\xa4\xe9\xa5\xcc\x2f\x13\x1c\x7d",
-		.rlen	= 16,
-	}, {
-		.key	= "\x28\xdb\xc3\xbc\x49\xff\xd8\x7d"
-			  "\xcf\xa5\x09\xb1\x1d\x42\x2b\xe7",
-		.klen	= 16,
-		.input	= "\x9b\x9b\x7b\xfc\xd1\x81\x3c\xb9"
+		.ctext	= "\x9b\x9b\x7b\xfc\xd1\x81\x3c\xb9"
 			  "\x5d\x0b\x36\x18\xf4\x0f\x51\x22",
-		.ilen	= 16,
-		.result	= "\xb4\x1e\x6b\xe2\xeb\xa8\x4a\x14"
-			  "\x8e\x2e\xed\x84\x59\x3c\x5e\xc7",
-		.rlen	= 16,
+		.len	= 16,
 	}
 };
 
-static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
+static const struct cipher_testvec salsa20_stream_tv_template[] = {
 	/*
 	* Testvectors from verified.test-vectors submitted to ECRYPT.
 	* They are truncated to size 39, 64, 111, 129 to test a variety
@@ -33879,24 +29692,23 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 			"\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F",
 		.klen	= 16,
 		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 39,
-		.result	= "\x2D\xD5\xC3\xF7\xBA\x2B\x20\xF7"
+		.ctext	= "\x2D\xD5\xC3\xF7\xBA\x2B\x20\xF7"
 			 "\x68\x02\x41\x0C\x68\x86\x88\x89"
 			 "\x5A\xD8\xC1\xBD\x4E\xA6\xC9\xB1"
 			 "\x40\xFB\x9B\x90\xE2\x10\x49\xBF"
 			 "\x58\x3F\x52\x79\x70\xEB\xC1",
-		.rlen	= 39,
+		.len	= 39,
 	}, { /* Set 5, vector 0 */
 		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00",
 		.klen	= 16,
 		.iv     = "\x80\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
@@ -33904,8 +29716,7 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 64,
-		.result	= "\xB6\x6C\x1E\x44\x46\xDD\x95\x57"
+		.ctext	= "\xB6\x6C\x1E\x44\x46\xDD\x95\x57"
 			 "\xE5\x78\xE2\x23\xB0\xB7\x68\x01"
 			 "\x7B\x23\xB2\x67\xBB\x02\x34\xAE"
 			 "\x46\x26\xBF\x44\x3F\x21\x97\x76"
@@ -33913,7 +29724,7 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 			 "\xCD\x0D\xE9\xA9\x53\x8F\x4A\x09"
 			 "\xCA\x9A\xC0\x73\x2E\x30\xBC\xF9"
 			 "\x8E\x4F\x13\xE4\xB9\xE2\x01\xD9",
-		.rlen	= 64,
+		.len	= 64,
 	}, { /* Set 3, vector 27 */
 		.key	= "\x1B\x1C\x1D\x1E\x1F\x20\x21\x22"
 			"\x23\x24\x25\x26\x27\x28\x29\x2A"
@@ -33921,7 +29732,7 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 			"\x33\x34\x35\x36\x37\x38\x39\x3A",
 		.klen	= 32,
 		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
@@ -33935,8 +29746,7 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 111,
-		.result	= "\xAE\x39\x50\x8E\xAC\x9A\xEC\xE7"
+		.ctext	= "\xAE\x39\x50\x8E\xAC\x9A\xEC\xE7"
 			 "\xBF\x97\xBB\x20\xB9\xDE\xE4\x1F"
 			 "\x87\xD9\x47\xF8\x28\x91\x35\x98"
 			 "\xDB\x72\xCC\x23\x29\x48\x56\x5E"
@@ -33950,7 +29760,7 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 			 "\xB1\x41\x3F\x19\x2F\xC4\x3B\xC6"
 			 "\x95\x46\x45\x54\xE9\x75\x03\x08"
 			 "\x44\xAF\xE5\x8A\x81\x12\x09",
-		.rlen	= 111,
+		.len	= 111,
 	}, { /* Set 5, vector 27 */
 		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
@@ -33958,7 +29768,7 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 			"\x00\x00\x00\x00\x00\x00\x00\x00",
 		.klen	= 32,
 		.iv     = "\x00\x00\x00\x10\x00\x00\x00\x00",
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
@@ -33975,8 +29785,7 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00",
-		.ilen	= 129,
-		.result	= "\xD2\xDB\x1A\x5C\xF1\xC1\xAC\xDB"
+		.ctext	= "\xD2\xDB\x1A\x5C\xF1\xC1\xAC\xDB"
 			 "\xE8\x1A\x7A\x43\x40\xEF\x53\x43"
 			 "\x5E\x7F\x4B\x1A\x50\x52\x3F\x8D"
 			 "\x28\x3D\xCF\x85\x1D\x69\x6E\x60"
@@ -33993,7 +29802,7 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 			 "\x2E\x40\x48\x75\xE9\xE2\x21\x45"
 			 "\x0B\xC9\xB6\xB5\x66\xBC\x9A\x59"
 			 "\x5A",
-		.rlen	= 129,
+		.len	= 129,
 	}, { /* large test vector generated using Crypto++ */
 		.key =  "\x00\x01\x02\x03\x04\x05\x06\x07"
 			"\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
@@ -34002,7 +29811,7 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 		.klen = 32,
 		.iv =	"\x00\x00\x00\x00\x00\x00\x00\x00"
 			"\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input =
+		.ptext =
 			"\x00\x01\x02\x03\x04\x05\x06\x07"
 			"\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
 			"\x10\x11\x12\x13\x14\x15\x16\x17"
@@ -34516,8 +30325,7 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 			"\x10\x2f\x4e\x6d\x8c\xab\xca\xe9"
 			"\x08\x27\x46\x65\x84\xa3\xc2\xe1"
 			"\x00\x21\x42\x63",
-		.ilen = 4100,
-		.result =
+		.ctext =
 			"\xb5\x81\xf5\x64\x18\x73\xe3\xf0"
 			"\x4c\x13\xf2\x77\x18\x60\x65\x5e"
 			"\x29\x01\xce\x98\x55\x53\xf9\x0c"
@@ -35031,13 +30839,13 @@ static const struct cipher_testvec salsa20_stream_enc_tv_template[] = {
 			"\xfc\x3f\x09\x7a\x0b\xdc\xc5\x1b"
 			"\x87\x13\xc6\x5b\x59\x8d\xf2\xc8"
 			"\xaf\xdf\x11\x95",
-		.rlen	= 4100,
+		.len	= 4100,
 		.np	= 2,
 		.tap	= { 4064, 36 },
 	},
 };
 
-static const struct cipher_testvec chacha20_enc_tv_template[] = {
+static const struct cipher_testvec chacha20_tv_template[] = {
 	{ /* RFC7539 A.2. Test Vector #1 */
 		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
@@ -35046,7 +30854,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 		.klen	= 32,
 		.iv     = "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.input	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+		.ptext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
@@ -35054,8 +30862,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.ilen	= 64,
-		.result	= "\x76\xb8\xe0\xad\xa0\xf1\x3d\x90"
+		.ctext	= "\x76\xb8\xe0\xad\xa0\xf1\x3d\x90"
 			  "\x40\x5d\x6a\xe5\x53\x86\xbd\x28"
 			  "\xbd\xd2\x19\xb8\xa0\x8d\xed\x1a"
 			  "\xa8\x36\xef\xcc\x8b\x77\x0d\xc7"
@@ -35063,7 +30870,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 			  "\x77\x24\xe0\x3f\xb8\xd8\x4a\x37"
 			  "\x6a\x43\xb8\xf4\x15\x18\xa1\x1c"
 			  "\xc3\x87\xb6\x69\xb2\xee\x65\x86",
-		.rlen	= 64,
+		.len	= 64,
 	}, { /* RFC7539 A.2. Test Vector #2 */
 		.key	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
@@ -35072,7 +30879,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 		.klen	= 32,
 		.iv     = "\x01\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x02",
-		.input	= "\x41\x6e\x79\x20\x73\x75\x62\x6d"
+		.ptext	= "\x41\x6e\x79\x20\x73\x75\x62\x6d"
 			  "\x69\x73\x73\x69\x6f\x6e\x20\x74"
 			  "\x6f\x20\x74\x68\x65\x20\x49\x45"
 			  "\x54\x46\x20\x69\x6e\x74\x65\x6e"
@@ -35119,8 +30926,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 			  "\x20\x77\x68\x69\x63\x68\x20\x61"
 			  "\x72\x65\x20\x61\x64\x64\x72\x65"
 			  "\x73\x73\x65\x64\x20\x74\x6f",
-		.ilen	= 375,
-		.result	= "\xa3\xfb\xf0\x7d\xf3\xfa\x2f\xde"
+		.ctext	= "\xa3\xfb\xf0\x7d\xf3\xfa\x2f\xde"
 			  "\x4f\x37\x6c\xa2\x3e\x82\x73\x70"
 			  "\x41\x60\x5d\x9f\x4f\x4f\x57\xbd"
 			  "\x8c\xff\x2c\x1d\x4b\x79\x55\xec"
@@ -35167,7 +30973,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 			  "\x7a\xc6\x1d\xd2\x9c\x6f\x21\xba"
 			  "\x5b\x86\x2f\x37\x30\xe3\x7c\xfd"
 			  "\xc4\xfd\x80\x6c\x22\xf2\x21",
-		.rlen	= 375,
+		.len	= 375,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 375 - 20, 4, 16 },
@@ -35180,7 +30986,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 		.klen	= 32,
 		.iv     = "\x2a\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x02",
-		.input	= "\x27\x54\x77\x61\x73\x20\x62\x72"
+		.ptext	= "\x27\x54\x77\x61\x73\x20\x62\x72"
 			  "\x69\x6c\x6c\x69\x67\x2c\x20\x61"
 			  "\x6e\x64\x20\x74\x68\x65\x20\x73"
 			  "\x6c\x69\x74\x68\x79\x20\x74\x6f"
@@ -35196,8 +31002,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 			  "\x68\x65\x20\x6d\x6f\x6d\x65\x20"
 			  "\x72\x61\x74\x68\x73\x20\x6f\x75"
 			  "\x74\x67\x72\x61\x62\x65\x2e",
-		.ilen	= 127,
-		.result	= "\x62\xe6\x34\x7f\x95\xed\x87\xa4"
+		.ctext	= "\x62\xe6\x34\x7f\x95\xed\x87\xa4"
 			  "\x5f\xfa\xe7\x42\x6f\x27\xa1\xdf"
 			  "\x5f\xb6\x91\x10\x04\x4c\x0d\x73"
 			  "\x11\x8e\xff\xa9\x5b\x01\xe5\xcf"
@@ -35213,7 +31018,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 			  "\x50\xd6\x15\x4b\x6d\xa7\x31\xb1"
 			  "\x87\xb5\x8d\xfd\x72\x8a\xfa\x36"
 			  "\x75\x7a\x79\x7a\xc1\x88\xd1",
-		.rlen	= 127,
+		.len	= 127,
 	}, { /* Self-made test vector for long data */
 		.key	= "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
 			  "\xf3\x33\x88\x86\x04\xf6\xb5\xf0"
@@ -35222,7 +31027,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 		.klen	= 32,
 		.iv     = "\x1c\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x01",
-		.input	= "\x49\xee\xe0\xdc\x24\x90\x40\xcd"
+		.ptext	= "\x49\xee\xe0\xdc\x24\x90\x40\xcd"
 			  "\xc5\x40\x8f\x47\x05\xbc\xdd\x81"
 			  "\x47\xc6\x8d\xe6\xb1\x8f\xd7\xcb"
 			  "\x09\x0e\x6e\x22\x48\x1f\xbf\xb8"
@@ -35383,8 +31188,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 			  "\x08\x7c\xbc\x66\x8a\xb0\xb6\x9f"
 			  "\x92\xd6\x41\x7c\x5b\x2a\x00\x79"
 			  "\x72",
-		.ilen	= 1281,
-		.result	= "\x45\xe8\xe0\xb6\x9c\xca\xfd\x87"
+		.ctext	= "\x45\xe8\xe0\xb6\x9c\xca\xfd\x87"
 			  "\xe8\x1d\x37\x96\x8a\xe3\x40\x35"
 			  "\xcf\x5e\x3a\x46\x3d\xfb\xd0\x69"
 			  "\xde\xaf\x7a\xd5\x0d\xe9\x52\xec"
@@ -35545,7 +31349,7 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 			  "\x23\x45\x89\x42\xa0\x30\xeb\xbf"
 			  "\xa1\xed\xad\xd5\x76\xfa\x24\x8f"
 			  "\x98",
-		.rlen	= 1281,
+		.len	= 1281,
 		.also_non_np = 1,
 		.np	= 3,
 		.tap	= { 1200, 1, 80 },
@@ -35555,132 +31359,28 @@ static const struct cipher_testvec chacha20_enc_tv_template[] = {
 /*
  * CTS (Cipher Text Stealing) mode tests
  */
-static const struct cipher_testvec cts_mode_enc_tv_template[] = {
-	{ /* from rfc3962 */
-		.klen	= 16,
-		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
-			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.ilen	= 17,
-		.input  = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
-			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
-			  "\x20",
-		.rlen	= 17,
-		.result	= "\xc6\x35\x35\x68\xf2\xbf\x8c\xb4"
-			  "\xd8\xa5\x80\x36\x2d\xa7\xff\x7f"
-			  "\x97",
-	}, {
-		.klen	= 16,
-		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
-			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.ilen   = 31,
-		.input  = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
-			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
-			  "\x20\x47\x65\x6e\x65\x72\x61\x6c"
-			  "\x20\x47\x61\x75\x27\x73\x20",
-		.rlen   = 31,
-		.result = "\xfc\x00\x78\x3e\x0e\xfd\xb2\xc1"
-			  "\xd4\x45\xd4\xc8\xef\xf7\xed\x22"
-			  "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
-			  "\xc0\x7b\x25\xe2\x5e\xcf\xe5",
-	}, {
-		.klen	= 16,
-		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
-			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.ilen   = 32,
-		.input  = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
-			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
-			  "\x20\x47\x65\x6e\x65\x72\x61\x6c"
-			  "\x20\x47\x61\x75\x27\x73\x20\x43",
-		.rlen   = 32,
-		.result = "\x39\x31\x25\x23\xa7\x86\x62\xd5"
-			  "\xbe\x7f\xcb\xcc\x98\xeb\xf5\xa8"
-			  "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
-			  "\xc0\x7b\x25\xe2\x5e\xcf\xe5\x84",
-	}, {
-		.klen	= 16,
-		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
-			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.ilen   = 47,
-		.input  = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
-			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
-			  "\x20\x47\x65\x6e\x65\x72\x61\x6c"
-			  "\x20\x47\x61\x75\x27\x73\x20\x43"
-			  "\x68\x69\x63\x6b\x65\x6e\x2c\x20"
-			  "\x70\x6c\x65\x61\x73\x65\x2c",
-		.rlen   = 47,
-		.result = "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
-			  "\xc0\x7b\x25\xe2\x5e\xcf\xe5\x84"
-			  "\xb3\xff\xfd\x94\x0c\x16\xa1\x8c"
-			  "\x1b\x55\x49\xd2\xf8\x38\x02\x9e"
-			  "\x39\x31\x25\x23\xa7\x86\x62\xd5"
-			  "\xbe\x7f\xcb\xcc\x98\xeb\xf5",
-	}, {
-		.klen	= 16,
-		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
-			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.ilen   = 48,
-		.input  = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
-			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
-			  "\x20\x47\x65\x6e\x65\x72\x61\x6c"
-			  "\x20\x47\x61\x75\x27\x73\x20\x43"
-			  "\x68\x69\x63\x6b\x65\x6e\x2c\x20"
-			  "\x70\x6c\x65\x61\x73\x65\x2c\x20",
-		.rlen   = 48,
-		.result = "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
-			  "\xc0\x7b\x25\xe2\x5e\xcf\xe5\x84"
-			  "\x9d\xad\x8b\xbb\x96\xc4\xcd\xc0"
-			  "\x3b\xc1\x03\xe1\xa1\x94\xbb\xd8"
-			  "\x39\x31\x25\x23\xa7\x86\x62\xd5"
-			  "\xbe\x7f\xcb\xcc\x98\xeb\xf5\xa8",
-	}, {
-		.klen	= 16,
-		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
-			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.ilen   = 64,
-		.input  = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
-			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
-			  "\x20\x47\x65\x6e\x65\x72\x61\x6c"
-			  "\x20\x47\x61\x75\x27\x73\x20\x43"
-			  "\x68\x69\x63\x6b\x65\x6e\x2c\x20"
-			  "\x70\x6c\x65\x61\x73\x65\x2c\x20"
-			  "\x61\x6e\x64\x20\x77\x6f\x6e\x74"
-			  "\x6f\x6e\x20\x73\x6f\x75\x70\x2e",
-		.rlen   = 64,
-		.result = "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
-			  "\xc0\x7b\x25\xe2\x5e\xcf\xe5\x84"
-			  "\x39\x31\x25\x23\xa7\x86\x62\xd5"
-			  "\xbe\x7f\xcb\xcc\x98\xeb\xf5\xa8"
-			  "\x48\x07\xef\xe8\x36\xee\x89\xa5"
-			  "\x26\x73\x0d\xbc\x2f\x7b\xc8\x40"
-			  "\x9d\xad\x8b\xbb\x96\xc4\xcd\xc0"
-			  "\x3b\xc1\x03\xe1\xa1\x94\xbb\xd8",
-	}
-};
-
-static const struct cipher_testvec cts_mode_dec_tv_template[] = {
+static const struct cipher_testvec cts_mode_tv_template[] = {
 	{ /* from rfc3962 */
 		.klen	= 16,
 		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
 			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.rlen	= 17,
-		.result = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
+		.ptext	= "\x49\x20\x77\x6f\x75\x6c\x64\x20"
 			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
 			  "\x20",
-		.ilen	= 17,
-		.input	= "\xc6\x35\x35\x68\xf2\xbf\x8c\xb4"
+		.len	= 17,
+		.ctext	= "\xc6\x35\x35\x68\xf2\xbf\x8c\xb4"
 			  "\xd8\xa5\x80\x36\x2d\xa7\xff\x7f"
 			  "\x97",
 	}, {
 		.klen	= 16,
 		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
 			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.rlen   = 31,
-		.result = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
+		.ptext	= "\x49\x20\x77\x6f\x75\x6c\x64\x20"
 			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
 			  "\x20\x47\x65\x6e\x65\x72\x61\x6c"
 			  "\x20\x47\x61\x75\x27\x73\x20",
-		.ilen   = 31,
-		.input  = "\xfc\x00\x78\x3e\x0e\xfd\xb2\xc1"
+		.len	= 31,
+		.ctext	= "\xfc\x00\x78\x3e\x0e\xfd\xb2\xc1"
 			  "\xd4\x45\xd4\xc8\xef\xf7\xed\x22"
 			  "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
 			  "\xc0\x7b\x25\xe2\x5e\xcf\xe5",
@@ -35688,13 +31388,12 @@ static const struct cipher_testvec cts_mode_dec_tv_template[] = {
 		.klen	= 16,
 		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
 			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.rlen   = 32,
-		.result = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
+		.ptext	= "\x49\x20\x77\x6f\x75\x6c\x64\x20"
 			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
 			  "\x20\x47\x65\x6e\x65\x72\x61\x6c"
 			  "\x20\x47\x61\x75\x27\x73\x20\x43",
-		.ilen   = 32,
-		.input  = "\x39\x31\x25\x23\xa7\x86\x62\xd5"
+		.len	= 32,
+		.ctext	= "\x39\x31\x25\x23\xa7\x86\x62\xd5"
 			  "\xbe\x7f\xcb\xcc\x98\xeb\xf5\xa8"
 			  "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
 			  "\xc0\x7b\x25\xe2\x5e\xcf\xe5\x84",
@@ -35702,15 +31401,14 @@ static const struct cipher_testvec cts_mode_dec_tv_template[] = {
 		.klen	= 16,
 		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
 			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.rlen   = 47,
-		.result = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
+		.ptext	= "\x49\x20\x77\x6f\x75\x6c\x64\x20"
 			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
 			  "\x20\x47\x65\x6e\x65\x72\x61\x6c"
 			  "\x20\x47\x61\x75\x27\x73\x20\x43"
 			  "\x68\x69\x63\x6b\x65\x6e\x2c\x20"
 			  "\x70\x6c\x65\x61\x73\x65\x2c",
-		.ilen   = 47,
-		.input  = "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
+		.len	= 47,
+		.ctext	= "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
 			  "\xc0\x7b\x25\xe2\x5e\xcf\xe5\x84"
 			  "\xb3\xff\xfd\x94\x0c\x16\xa1\x8c"
 			  "\x1b\x55\x49\xd2\xf8\x38\x02\x9e"
@@ -35720,15 +31418,14 @@ static const struct cipher_testvec cts_mode_dec_tv_template[] = {
 		.klen	= 16,
 		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
 			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.rlen   = 48,
-		.result = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
+		.ptext	= "\x49\x20\x77\x6f\x75\x6c\x64\x20"
 			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
 			  "\x20\x47\x65\x6e\x65\x72\x61\x6c"
 			  "\x20\x47\x61\x75\x27\x73\x20\x43"
 			  "\x68\x69\x63\x6b\x65\x6e\x2c\x20"
 			  "\x70\x6c\x65\x61\x73\x65\x2c\x20",
-		.ilen   = 48,
-		.input  = "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
+		.len	= 48,
+		.ctext	= "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
 			  "\xc0\x7b\x25\xe2\x5e\xcf\xe5\x84"
 			  "\x9d\xad\x8b\xbb\x96\xc4\xcd\xc0"
 			  "\x3b\xc1\x03\xe1\xa1\x94\xbb\xd8"
@@ -35738,8 +31435,7 @@ static const struct cipher_testvec cts_mode_dec_tv_template[] = {
 		.klen	= 16,
 		.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
 			  "\x74\x65\x72\x69\x79\x61\x6b\x69",
-		.rlen   = 64,
-		.result = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
+		.ptext	= "\x49\x20\x77\x6f\x75\x6c\x64\x20"
 			  "\x6c\x69\x6b\x65\x20\x74\x68\x65"
 			  "\x20\x47\x65\x6e\x65\x72\x61\x6c"
 			  "\x20\x47\x61\x75\x27\x73\x20\x43"
@@ -35747,8 +31443,8 @@ static const struct cipher_testvec cts_mode_dec_tv_template[] = {
 			  "\x70\x6c\x65\x61\x73\x65\x2c\x20"
 			  "\x61\x6e\x64\x20\x77\x6f\x6e\x74"
 			  "\x6f\x6e\x20\x73\x6f\x75\x70\x2e",
-		.ilen   = 64,
-		.input  = "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
+		.len	= 64,
+		.ctext	= "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
 			  "\xc0\x7b\x25\xe2\x5e\xcf\xe5\x84"
 			  "\x39\x31\x25\x23\xa7\x86\x62\xd5"
 			  "\xbe\x7f\xcb\xcc\x98\xeb\xf5\xa8"
@@ -36057,6 +31753,15 @@ static const struct hash_testvec michael_mic_tv_template[] = {
  * CRC32 test vectors
  */
 static const struct hash_testvec crc32_tv_template[] = {
+	{
+		.psize = 0,
+		.digest = "\x00\x00\x00\x00",
+	},
+	{
+		.plaintext = "abcdefg",
+		.psize = 7,
+		.digest = "\xd8\xb5\x46\xac",
+	},
 	{
 		.key = "\x87\xa9\xcb\xed",
 		.ksize = 4,
@@ -36493,6 +32198,11 @@ static const struct hash_testvec crc32c_tv_template[] = {
 		.psize = 0,
 		.digest = "\x00\x00\x00\x00",
 	},
+	{
+		.plaintext = "abcdefg",
+		.psize = 7,
+		.digest = "\x41\xf4\x27\xe6",
+	},
 	{
 		.key = "\x87\xa9\xcb\xed",
 		.ksize = 4,
@@ -36921,94 +32631,6 @@ static const struct hash_testvec crc32c_tv_template[] = {
 	}
 };
 
-/*
- * Blakcifn CRC test vectors
- */
-static const struct hash_testvec bfin_crc_tv_template[] = {
-	{
-		.psize = 0,
-		.digest = "\x00\x00\x00\x00",
-	},
-	{
-		.key = "\x87\xa9\xcb\xed",
-		.ksize = 4,
-		.psize = 0,
-		.digest = "\x87\xa9\xcb\xed",
-	},
-	{
-		.key = "\xff\xff\xff\xff",
-		.ksize = 4,
-		.plaintext = "\x01\x02\x03\x04\x05\x06\x07\x08"
-			     "\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10"
-			     "\x11\x12\x13\x14\x15\x16\x17\x18"
-			     "\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20"
-			     "\x21\x22\x23\x24\x25\x26\x27\x28",
-		.psize = 40,
-		.digest = "\x84\x0c\x8d\xa2",
-	},
-	{
-		.key = "\xff\xff\xff\xff",
-		.ksize = 4,
-		.plaintext = "\x01\x02\x03\x04\x05\x06\x07\x08"
-			     "\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10"
-			     "\x11\x12\x13\x14\x15\x16\x17\x18"
-			     "\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20"
-			     "\x21\x22\x23\x24\x25\x26",
-		.psize = 38,
-		.digest = "\x8c\x58\xec\xb7",
-	},
-	{
-		.key = "\xff\xff\xff\xff",
-		.ksize = 4,
-		.plaintext = "\x01\x02\x03\x04\x05\x06\x07\x08"
-			     "\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10"
-			     "\x11\x12\x13\x14\x15\x16\x17\x18"
-			     "\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20"
-			     "\x21\x22\x23\x24\x25\x26\x27",
-		.psize = 39,
-		.digest = "\xdc\x50\x28\x7b",
-	},
-	{
-		.key = "\xff\xff\xff\xff",
-		.ksize = 4,
-		.plaintext = "\x01\x02\x03\x04\x05\x06\x07\x08"
-			     "\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10"
-			     "\x11\x12\x13\x14\x15\x16\x17\x18"
-			     "\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20"
-			     "\x21\x22\x23\x24\x25\x26\x27\x28"
-			     "\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30"
-			     "\x31\x32\x33\x34\x35\x36\x37\x38"
-			     "\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40"
-			     "\x41\x42\x43\x44\x45\x46\x47\x48"
-			     "\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50"
-			     "\x51\x52\x53\x54\x55\x56\x57\x58"
-			     "\x59\x5a\x5b\x5c\x5d\x5e\x5f\x60"
-			     "\x61\x62\x63\x64\x65\x66\x67\x68"
-			     "\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70"
-			     "\x71\x72\x73\x74\x75\x76\x77\x78"
-			     "\x79\x7a\x7b\x7c\x7d\x7e\x7f\x80"
-			     "\x81\x82\x83\x84\x85\x86\x87\x88"
-			     "\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90"
-			     "\x91\x92\x93\x94\x95\x96\x97\x98"
-			     "\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0"
-			     "\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8"
-			     "\xa9\xaa\xab\xac\xad\xae\xaf\xb0"
-			     "\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8"
-			     "\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0"
-			     "\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8"
-			     "\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0"
-			     "\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8"
-			     "\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0"
-			     "\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8"
-			     "\xe9\xea\xeb\xec\xed\xee\xef\xf0",
-		.psize = 240,
-		.digest = "\x10\x19\x4a\x5c",
-		.np = 2,
-		.tap = { 31, 209 }
-	},
-
-};
-
 static const struct comp_testvec lz4_comp_tv_template[] = {
 	{
 		.inlen	= 255,
@@ -37131,4 +32753,75 @@ static const struct comp_testvec lz4hc_decomp_tv_template[] = {
 	},
 };
 
+static const struct comp_testvec zstd_comp_tv_template[] = {
+	{
+		.inlen	= 68,
+		.outlen	= 39,
+		.input	= "The algorithm is zstd. "
+			  "The algorithm is zstd. "
+			  "The algorithm is zstd.",
+		.output	= "\x28\xb5\x2f\xfd\x00\x50\xf5\x00\x00\xb8\x54\x68\x65"
+			  "\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x20\x69\x73"
+			  "\x20\x7a\x73\x74\x64\x2e\x20\x01\x00\x55\x73\x36\x01"
+			  ,
+	},
+	{
+		.inlen	= 244,
+		.outlen	= 151,
+		.input	= "zstd, short for Zstandard, is a fast lossless "
+			  "compression algorithm, targeting real-time "
+			  "compression scenarios at zlib-level and better "
+			  "compression ratios. The zstd compression library "
+			  "provides in-memory compression and decompression "
+			  "functions.",
+		.output	= "\x28\xb5\x2f\xfd\x00\x50\x75\x04\x00\x42\x4b\x1e\x17"
+			  "\x90\x81\x31\x00\xf2\x2f\xe4\x36\xc9\xef\x92\x88\x32"
+			  "\xc9\xf2\x24\x94\xd8\x68\x9a\x0f\x00\x0c\xc4\x31\x6f"
+			  "\x0d\x0c\x38\xac\x5c\x48\x03\xcd\x63\x67\xc0\xf3\xad"
+			  "\x4e\x90\xaa\x78\xa0\xa4\xc5\x99\xda\x2f\xb6\x24\x60"
+			  "\xe2\x79\x4b\xaa\xb6\x6b\x85\x0b\xc9\xc6\x04\x66\x86"
+			  "\xe2\xcc\xe2\x25\x3f\x4f\x09\xcd\xb8\x9d\xdb\xc1\x90"
+			  "\xa9\x11\xbc\x35\x44\x69\x2d\x9c\x64\x4f\x13\x31\x64"
+			  "\xcc\xfb\x4d\x95\x93\x86\x7f\x33\x7f\x1a\xef\xe9\x30"
+			  "\xf9\x67\xa1\x94\x0a\x69\x0f\x60\xcd\xc3\xab\x99\xdc"
+			  "\x42\xed\x97\x05\x00\x33\xc3\x15\x95\x3a\x06\xa0\x0e"
+			  "\x20\xa9\x0e\x82\xb9\x43\x45\x01",
+	},
+};
+
+static const struct comp_testvec zstd_decomp_tv_template[] = {
+	{
+		.inlen	= 43,
+		.outlen	= 68,
+		.input	= "\x28\xb5\x2f\xfd\x04\x50\xf5\x00\x00\xb8\x54\x68\x65"
+			  "\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x20\x69\x73"
+			  "\x20\x7a\x73\x74\x64\x2e\x20\x01\x00\x55\x73\x36\x01"
+			  "\x6b\xf4\x13\x35",
+		.output	= "The algorithm is zstd. "
+			  "The algorithm is zstd. "
+			  "The algorithm is zstd.",
+	},
+	{
+		.inlen	= 155,
+		.outlen	= 244,
+		.input	= "\x28\xb5\x2f\xfd\x04\x50\x75\x04\x00\x42\x4b\x1e\x17"
+			  "\x90\x81\x31\x00\xf2\x2f\xe4\x36\xc9\xef\x92\x88\x32"
+			  "\xc9\xf2\x24\x94\xd8\x68\x9a\x0f\x00\x0c\xc4\x31\x6f"
+			  "\x0d\x0c\x38\xac\x5c\x48\x03\xcd\x63\x67\xc0\xf3\xad"
+			  "\x4e\x90\xaa\x78\xa0\xa4\xc5\x99\xda\x2f\xb6\x24\x60"
+			  "\xe2\x79\x4b\xaa\xb6\x6b\x85\x0b\xc9\xc6\x04\x66\x86"
+			  "\xe2\xcc\xe2\x25\x3f\x4f\x09\xcd\xb8\x9d\xdb\xc1\x90"
+			  "\xa9\x11\xbc\x35\x44\x69\x2d\x9c\x64\x4f\x13\x31\x64"
+			  "\xcc\xfb\x4d\x95\x93\x86\x7f\x33\x7f\x1a\xef\xe9\x30"
+			  "\xf9\x67\xa1\x94\x0a\x69\x0f\x60\xcd\xc3\xab\x99\xdc"
+			  "\x42\xed\x97\x05\x00\x33\xc3\x15\x95\x3a\x06\xa0\x0e"
+			  "\x20\xa9\x0e\x82\xb9\x43\x45\x01\xaa\x6d\xda\x0d",
+		.output	= "zstd, short for Zstandard, is a fast lossless "
+			  "compression algorithm, targeting real-time "
+			  "compression scenarios at zlib-level and better "
+			  "compression ratios. The zstd compression library "
+			  "provides in-memory compression and decompression "
+			  "functions.",
+	},
+};
 #endif	/* _CRYPTO_TESTMGR_H */
diff --git a/crypto/zstd.c b/crypto/zstd.c
new file mode 100644
index 000000000000..9a76b3ed8b8b
--- /dev/null
+++ b/crypto/zstd.c
@@ -0,0 +1,265 @@
+/*
+ * Cryptographic API.
+ *
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+#include <linux/crypto.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/net.h>
+#include <linux/vmalloc.h>
+#include <linux/zstd.h>
+#include <crypto/internal/scompress.h>
+
+
+#define ZSTD_DEF_LEVEL	3
+
+struct zstd_ctx {
+	ZSTD_CCtx *cctx;
+	ZSTD_DCtx *dctx;
+	void *cwksp;
+	void *dwksp;
+};
+
+static ZSTD_parameters zstd_params(void)
+{
+	return ZSTD_getParams(ZSTD_DEF_LEVEL, 0, 0);
+}
+
+static int zstd_comp_init(struct zstd_ctx *ctx)
+{
+	int ret = 0;
+	const ZSTD_parameters params = zstd_params();
+	const size_t wksp_size = ZSTD_CCtxWorkspaceBound(params.cParams);
+
+	ctx->cwksp = vzalloc(wksp_size);
+	if (!ctx->cwksp) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ctx->cctx = ZSTD_initCCtx(ctx->cwksp, wksp_size);
+	if (!ctx->cctx) {
+		ret = -EINVAL;
+		goto out_free;
+	}
+out:
+	return ret;
+out_free:
+	vfree(ctx->cwksp);
+	goto out;
+}
+
+static int zstd_decomp_init(struct zstd_ctx *ctx)
+{
+	int ret = 0;
+	const size_t wksp_size = ZSTD_DCtxWorkspaceBound();
+
+	ctx->dwksp = vzalloc(wksp_size);
+	if (!ctx->dwksp) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ctx->dctx = ZSTD_initDCtx(ctx->dwksp, wksp_size);
+	if (!ctx->dctx) {
+		ret = -EINVAL;
+		goto out_free;
+	}
+out:
+	return ret;
+out_free:
+	vfree(ctx->dwksp);
+	goto out;
+}
+
+static void zstd_comp_exit(struct zstd_ctx *ctx)
+{
+	vfree(ctx->cwksp);
+	ctx->cwksp = NULL;
+	ctx->cctx = NULL;
+}
+
+static void zstd_decomp_exit(struct zstd_ctx *ctx)
+{
+	vfree(ctx->dwksp);
+	ctx->dwksp = NULL;
+	ctx->dctx = NULL;
+}
+
+static int __zstd_init(void *ctx)
+{
+	int ret;
+
+	ret = zstd_comp_init(ctx);
+	if (ret)
+		return ret;
+	ret = zstd_decomp_init(ctx);
+	if (ret)
+		zstd_comp_exit(ctx);
+	return ret;
+}
+
+static void *zstd_alloc_ctx(struct crypto_scomp *tfm)
+{
+	int ret;
+	struct zstd_ctx *ctx;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	ret = __zstd_init(ctx);
+	if (ret) {
+		kfree(ctx);
+		return ERR_PTR(ret);
+	}
+
+	return ctx;
+}
+
+static int zstd_init(struct crypto_tfm *tfm)
+{
+	struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	return __zstd_init(ctx);
+}
+
+static void __zstd_exit(void *ctx)
+{
+	zstd_comp_exit(ctx);
+	zstd_decomp_exit(ctx);
+}
+
+static void zstd_free_ctx(struct crypto_scomp *tfm, void *ctx)
+{
+	__zstd_exit(ctx);
+	kzfree(ctx);
+}
+
+static void zstd_exit(struct crypto_tfm *tfm)
+{
+	struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	__zstd_exit(ctx);
+}
+
+static int __zstd_compress(const u8 *src, unsigned int slen,
+			   u8 *dst, unsigned int *dlen, void *ctx)
+{
+	size_t out_len;
+	struct zstd_ctx *zctx = ctx;
+	const ZSTD_parameters params = zstd_params();
+
+	out_len = ZSTD_compressCCtx(zctx->cctx, dst, *dlen, src, slen, params);
+	if (ZSTD_isError(out_len))
+		return -EINVAL;
+	*dlen = out_len;
+	return 0;
+}
+
+static int zstd_compress(struct crypto_tfm *tfm, const u8 *src,
+			 unsigned int slen, u8 *dst, unsigned int *dlen)
+{
+	struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	return __zstd_compress(src, slen, dst, dlen, ctx);
+}
+
+static int zstd_scompress(struct crypto_scomp *tfm, const u8 *src,
+			  unsigned int slen, u8 *dst, unsigned int *dlen,
+			  void *ctx)
+{
+	return __zstd_compress(src, slen, dst, dlen, ctx);
+}
+
+static int __zstd_decompress(const u8 *src, unsigned int slen,
+			     u8 *dst, unsigned int *dlen, void *ctx)
+{
+	size_t out_len;
+	struct zstd_ctx *zctx = ctx;
+
+	out_len = ZSTD_decompressDCtx(zctx->dctx, dst, *dlen, src, slen);
+	if (ZSTD_isError(out_len))
+		return -EINVAL;
+	*dlen = out_len;
+	return 0;
+}
+
+static int zstd_decompress(struct crypto_tfm *tfm, const u8 *src,
+			   unsigned int slen, u8 *dst, unsigned int *dlen)
+{
+	struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	return __zstd_decompress(src, slen, dst, dlen, ctx);
+}
+
+static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src,
+			    unsigned int slen, u8 *dst, unsigned int *dlen,
+			    void *ctx)
+{
+	return __zstd_decompress(src, slen, dst, dlen, ctx);
+}
+
+static struct crypto_alg alg = {
+	.cra_name		= "zstd",
+	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
+	.cra_ctxsize		= sizeof(struct zstd_ctx),
+	.cra_module		= THIS_MODULE,
+	.cra_init		= zstd_init,
+	.cra_exit		= zstd_exit,
+	.cra_u			= { .compress = {
+	.coa_compress		= zstd_compress,
+	.coa_decompress		= zstd_decompress } }
+};
+
+static struct scomp_alg scomp = {
+	.alloc_ctx		= zstd_alloc_ctx,
+	.free_ctx		= zstd_free_ctx,
+	.compress		= zstd_scompress,
+	.decompress		= zstd_sdecompress,
+	.base			= {
+		.cra_name	= "zstd",
+		.cra_driver_name = "zstd-scomp",
+		.cra_module	 = THIS_MODULE,
+	}
+};
+
+static int __init zstd_mod_init(void)
+{
+	int ret;
+
+	ret = crypto_register_alg(&alg);
+	if (ret)
+		return ret;
+
+	ret = crypto_register_scomp(&scomp);
+	if (ret)
+		crypto_unregister_alg(&alg);
+
+	return ret;
+}
+
+static void __exit zstd_mod_fini(void)
+{
+	crypto_unregister_alg(&alg);
+	crypto_unregister_scomp(&scomp);
+}
+
+module_init(zstd_mod_init);
+module_exit(zstd_mod_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Zstd Compression Algorithm");
+MODULE_ALIAS_CRYPTO("zstd");
diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig
index 516d7b36d6fb..b533eeb6139d 100644
--- a/drivers/acpi/Kconfig
+++ b/drivers/acpi/Kconfig
@@ -547,6 +547,9 @@ config ACPI_CONFIGFS
 
 if ARM64
 source "drivers/acpi/arm64/Kconfig"
+
+config ACPI_PPTT
+	bool
 endif
 
 config TPS68470_PMIC_OPREGION
diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile
index 48e202752754..6d59aa109a91 100644
--- a/drivers/acpi/Makefile
+++ b/drivers/acpi/Makefile
@@ -88,6 +88,7 @@ obj-$(CONFIG_ACPI_BGRT)		+= bgrt.o
 obj-$(CONFIG_ACPI_CPPC_LIB)	+= cppc_acpi.o
 obj-$(CONFIG_ACPI_SPCR_TABLE)	+= spcr.o
 obj-$(CONFIG_ACPI_DEBUGGER_USER) += acpi_dbg.o
+obj-$(CONFIG_ACPI_PPTT) 	+= pptt.o
 
 # processor has its own "processor." module_param namespace
 processor-y			:= processor_driver.o
diff --git a/drivers/acpi/ac.c b/drivers/acpi/ac.c
index 2d8de2f8c1ed..cdd3136829f1 100644
--- a/drivers/acpi/ac.c
+++ b/drivers/acpi/ac.c
@@ -82,11 +82,11 @@ static SIMPLE_DEV_PM_OPS(acpi_ac_pm, NULL, acpi_ac_resume);
 #ifdef CONFIG_ACPI_PROCFS_POWER
 extern struct proc_dir_entry *acpi_lock_ac_dir(void);
 extern void *acpi_unlock_ac_dir(struct proc_dir_entry *acpi_ac_dir);
-static int acpi_ac_open_fs(struct inode *inode, struct file *file);
 #endif
 
 
 static int ac_sleep_before_get_state_ms;
+static int ac_check_pmic = 1;
 
 static struct acpi_driver acpi_ac_driver = {
 	.name = "ac",
@@ -111,16 +111,6 @@ struct acpi_ac {
 
 #define to_acpi_ac(x) power_supply_get_drvdata(x)
 
-#ifdef CONFIG_ACPI_PROCFS_POWER
-static const struct file_operations acpi_ac_fops = {
-	.owner = THIS_MODULE,
-	.open = acpi_ac_open_fs,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-#endif
-
 /* --------------------------------------------------------------------------
                                AC Adapter Management
    -------------------------------------------------------------------------- */
@@ -209,11 +199,6 @@ static int acpi_ac_seq_show(struct seq_file *seq, void *offset)
 	return 0;
 }
 
-static int acpi_ac_open_fs(struct inode *inode, struct file *file)
-{
-	return single_open(file, acpi_ac_seq_show, PDE_DATA(inode));
-}
-
 static int acpi_ac_add_fs(struct acpi_ac *ac)
 {
 	struct proc_dir_entry *entry = NULL;
@@ -228,9 +213,8 @@ static int acpi_ac_add_fs(struct acpi_ac *ac)
 	}
 
 	/* 'state' [R] */
-	entry = proc_create_data(ACPI_AC_FILE_STATE,
-				 S_IRUGO, acpi_device_dir(ac->device),
-				 &acpi_ac_fops, ac);
+	entry = proc_create_single_data(ACPI_AC_FILE_STATE, S_IRUGO,
+			acpi_device_dir(ac->device), acpi_ac_seq_show, ac);
 	if (!entry)
 		return -ENODEV;
 	return 0;
@@ -310,21 +294,43 @@ static int acpi_ac_battery_notify(struct notifier_block *nb,
 	return NOTIFY_OK;
 }
 
-static int thinkpad_e530_quirk(const struct dmi_system_id *d)
+static int __init thinkpad_e530_quirk(const struct dmi_system_id *d)
 {
 	ac_sleep_before_get_state_ms = 1000;
 	return 0;
 }
 
-static const struct dmi_system_id ac_dmi_table[] = {
+static int __init ac_do_not_check_pmic_quirk(const struct dmi_system_id *d)
+{
+	ac_check_pmic = 0;
+	return 0;
+}
+
+static const struct dmi_system_id ac_dmi_table[]  __initconst = {
 	{
+	/* Thinkpad e530 */
 	.callback = thinkpad_e530_quirk,
-	.ident = "thinkpad e530",
 	.matches = {
 		DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
 		DMI_MATCH(DMI_PRODUCT_NAME, "32597CG"),
 		},
 	},
+	{
+		/* ECS EF20EA */
+		.callback = ac_do_not_check_pmic_quirk,
+		.matches = {
+			DMI_MATCH(DMI_PRODUCT_NAME, "EF20EA"),
+		},
+	},
+	{
+		/* Lenovo Ideapad Miix 320 */
+		.callback = ac_do_not_check_pmic_quirk,
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "80XF"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "Lenovo MIIX 320-10ICR"),
+		},
+	},
 	{},
 };
 
@@ -384,7 +390,6 @@ static int acpi_ac_add(struct acpi_device *device)
 		kfree(ac);
 	}
 
-	dmi_check_system(ac_dmi_table);
 	return result;
 }
 
@@ -442,13 +447,17 @@ static int __init acpi_ac_init(void)
 	if (acpi_disabled)
 		return -ENODEV;
 
-	for (i = 0; i < ARRAY_SIZE(acpi_ac_blacklist); i++)
-		if (acpi_dev_present(acpi_ac_blacklist[i].hid, "1",
-				     acpi_ac_blacklist[i].hrv)) {
-			pr_info(PREFIX "AC: found native %s PMIC, not loading\n",
-				acpi_ac_blacklist[i].hid);
-			return -ENODEV;
-		}
+	dmi_check_system(ac_dmi_table);
+
+	if (ac_check_pmic) {
+		for (i = 0; i < ARRAY_SIZE(acpi_ac_blacklist); i++)
+			if (acpi_dev_present(acpi_ac_blacklist[i].hid, "1",
+					     acpi_ac_blacklist[i].hrv)) {
+				pr_info(PREFIX "AC: found native %s PMIC, not loading\n",
+					acpi_ac_blacklist[i].hid);
+				return -ENODEV;
+			}
+	}
 
 #ifdef CONFIG_ACPI_PROCFS_POWER
 	acpi_ac_dir = acpi_lock_ac_dir();
diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c
index d553b0087947..2664452fa112 100644
--- a/drivers/acpi/acpi_apd.c
+++ b/drivers/acpi/acpi_apd.c
@@ -11,6 +11,7 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/platform_data/clk-st.h>
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
 #include <linux/clkdev.h>
@@ -72,6 +73,47 @@ static int acpi_apd_setup(struct apd_private_data *pdata)
 }
 
 #ifdef CONFIG_X86_AMD_PLATFORM_DEVICE
+
+static int misc_check_res(struct acpi_resource *ares, void *data)
+{
+	struct resource res;
+
+	return !acpi_dev_resource_memory(ares, &res);
+}
+
+static int st_misc_setup(struct apd_private_data *pdata)
+{
+	struct acpi_device *adev = pdata->adev;
+	struct platform_device *clkdev;
+	struct st_clk_data *clk_data;
+	struct resource_entry *rentry;
+	struct list_head resource_list;
+	int ret;
+
+	clk_data = devm_kzalloc(&adev->dev, sizeof(*clk_data), GFP_KERNEL);
+	if (!clk_data)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&resource_list);
+	ret = acpi_dev_get_resources(adev, &resource_list, misc_check_res,
+				     NULL);
+	if (ret < 0)
+		return -ENOENT;
+
+	list_for_each_entry(rentry, &resource_list, node) {
+		clk_data->base = devm_ioremap(&adev->dev, rentry->res->start,
+					      resource_size(rentry->res));
+		break;
+	}
+
+	acpi_dev_free_resource_list(&resource_list);
+
+	clkdev = platform_device_register_data(&adev->dev, "clk-st",
+					       PLATFORM_DEVID_NONE, clk_data,
+					       sizeof(*clk_data));
+	return PTR_ERR_OR_ZERO(clkdev);
+}
+
 static const struct apd_device_desc cz_i2c_desc = {
 	.setup = acpi_apd_setup,
 	.fixed_clk_rate = 133000000,
@@ -94,6 +136,10 @@ static const struct apd_device_desc cz_uart_desc = {
 	.fixed_clk_rate = 48000000,
 	.properties = uart_properties,
 };
+
+static const struct apd_device_desc st_misc_desc = {
+	.setup = st_misc_setup,
+};
 #endif
 
 #ifdef CONFIG_ARM64
@@ -179,6 +225,7 @@ static const struct acpi_device_id acpi_apd_device_ids[] = {
 	{ "AMD0020", APD_ADDR(cz_uart_desc) },
 	{ "AMDI0020", APD_ADDR(cz_uart_desc) },
 	{ "AMD0030", },
+	{ "AMD0040", APD_ADDR(st_misc_desc)},
 #endif
 #ifdef CONFIG_ARM64
 	{ "APMC0D0F", APD_ADDR(xgene_i2c_desc) },
diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c
index 2bcffec8dbf0..cb6ac5c65c2e 100644
--- a/drivers/acpi/acpi_lpss.c
+++ b/drivers/acpi/acpi_lpss.c
@@ -69,6 +69,10 @@ ACPI_MODULE_NAME("acpi_lpss");
 #define LPSS_SAVE_CTX			BIT(4)
 #define LPSS_NO_D3_DELAY		BIT(5)
 
+/* Crystal Cove PMIC shares same ACPI ID between different platforms */
+#define BYT_CRC_HRV			2
+#define CHT_CRC_HRV			3
+
 struct lpss_private_data;
 
 struct lpss_device_desc {
@@ -162,7 +166,7 @@ static void byt_pwm_setup(struct lpss_private_data *pdata)
 	if (!adev->pnp.unique_id || strcmp(adev->pnp.unique_id, "1"))
 		return;
 
-	if (!acpi_dev_present("INT33FD", NULL, -1))
+	if (!acpi_dev_present("INT33FD", NULL, BYT_CRC_HRV))
 		pwm_add_table(byt_pwm_lookup, ARRAY_SIZE(byt_pwm_lookup));
 }
 
diff --git a/drivers/acpi/acpi_watchdog.c b/drivers/acpi/acpi_watchdog.c
index 4bde16fb97d8..95600309ce42 100644
--- a/drivers/acpi/acpi_watchdog.c
+++ b/drivers/acpi/acpi_watchdog.c
@@ -12,35 +12,51 @@
 #define pr_fmt(fmt) "ACPI: watchdog: " fmt
 
 #include <linux/acpi.h>
-#include <linux/dmi.h>
 #include <linux/ioport.h>
 #include <linux/platform_device.h>
 
 #include "internal.h"
 
-static const struct dmi_system_id acpi_watchdog_skip[] = {
-	{
-		/*
-		 * On Lenovo Z50-70 there are two issues with the WDAT
-		 * table. First some of the instructions use RTC SRAM
-		 * to store persistent information. This does not work well
-		 * with Linux RTC driver. Second, more important thing is
-		 * that the instructions do not actually reset the system.
-		 *
-		 * On this particular system iTCO_wdt seems to work just
-		 * fine so we prefer that over WDAT for now.
-		 *
-		 * See also https://bugzilla.kernel.org/show_bug.cgi?id=199033.
-		 */
-		.ident = "Lenovo Z50-70",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "20354"),
-			DMI_MATCH(DMI_PRODUCT_VERSION, "Lenovo Z50-70"),
-		},
-	},
-	{}
-};
+#ifdef CONFIG_RTC_MC146818_LIB
+#include <linux/mc146818rtc.h>
+
+/*
+ * There are several systems where the WDAT table is accessing RTC SRAM to
+ * store persistent information. This does not work well with the Linux RTC
+ * driver so on those systems we skip WDAT driver and prefer iTCO_wdt
+ * instead.
+ *
+ * See also https://bugzilla.kernel.org/show_bug.cgi?id=199033.
+ */
+static bool acpi_watchdog_uses_rtc(const struct acpi_table_wdat *wdat)
+{
+	const struct acpi_wdat_entry *entries;
+	int i;
+
+	entries = (struct acpi_wdat_entry *)(wdat + 1);
+	for (i = 0; i < wdat->entries; i++) {
+		const struct acpi_generic_address *gas;
+
+		gas = &entries[i].register_region;
+		if (gas->space_id == ACPI_ADR_SPACE_SYSTEM_IO) {
+			switch (gas->address) {
+			case RTC_PORT(0):
+			case RTC_PORT(1):
+			case RTC_PORT(2):
+			case RTC_PORT(3):
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+#else
+static bool acpi_watchdog_uses_rtc(const struct acpi_table_wdat *wdat)
+{
+	return false;
+}
+#endif
 
 static const struct acpi_table_wdat *acpi_watchdog_get_wdat(void)
 {
@@ -50,9 +66,6 @@ static const struct acpi_table_wdat *acpi_watchdog_get_wdat(void)
 	if (acpi_disabled)
 		return NULL;
 
-	if (dmi_check_system(acpi_watchdog_skip))
-		return NULL;
-
 	status = acpi_get_table(ACPI_SIG_WDAT, 0,
 				(struct acpi_table_header **)&wdat);
 	if (ACPI_FAILURE(status)) {
@@ -60,6 +73,11 @@ static const struct acpi_table_wdat *acpi_watchdog_get_wdat(void)
 		return NULL;
 	}
 
+	if (acpi_watchdog_uses_rtc(wdat)) {
+		pr_info("Skipping WDAT on this system because it uses RTC SRAM\n");
+		return NULL;
+	}
+
 	return wdat;
 }
 
diff --git a/drivers/acpi/acpica/acapps.h b/drivers/acpi/acpica/acapps.h
index a2a85122fafe..5a9c2febc0fb 100644
--- a/drivers/acpi/acpica/acapps.h
+++ b/drivers/acpi/acpica/acapps.h
@@ -143,6 +143,8 @@ acpi_status
 fl_split_input_pathname(char *input_path,
 			char **out_directory_path, char **out_filename);
 
+char *fl_get_file_basename(char *file_pathname);
+
 char *ad_generate_filename(char *prefix, char *table_id);
 
 void
diff --git a/drivers/acpi/acpica/acglobal.h b/drivers/acpi/acpica/acglobal.h
index 0bc550072a21..1e6204518496 100644
--- a/drivers/acpi/acpica/acglobal.h
+++ b/drivers/acpi/acpica/acglobal.h
@@ -82,7 +82,7 @@ ACPI_GLOBAL(u8, acpi_gbl_global_lock_pending);
  * interrupt level
  */
 ACPI_GLOBAL(acpi_spinlock, acpi_gbl_gpe_lock);	/* For GPE data structs and registers */
-ACPI_GLOBAL(acpi_spinlock, acpi_gbl_hardware_lock);	/* For ACPI H/W except GPE registers */
+ACPI_GLOBAL(acpi_raw_spinlock, acpi_gbl_hardware_lock);	/* For ACPI H/W except GPE registers */
 ACPI_GLOBAL(acpi_spinlock, acpi_gbl_reference_count_lock);
 
 /* Mutex for _OSI support */
diff --git a/drivers/acpi/acpica/dbnames.c b/drivers/acpi/acpica/dbnames.c
index 170802c62179..dc94de91033e 100644
--- a/drivers/acpi/acpica/dbnames.c
+++ b/drivers/acpi/acpica/dbnames.c
@@ -189,9 +189,15 @@ void acpi_db_dump_namespace(char *start_arg, char *depth_arg)
 	}
 
 	acpi_db_set_output_destination(ACPI_DB_DUPLICATE_OUTPUT);
-	acpi_os_printf("ACPI Namespace (from %4.4s (%p) subtree):\n",
-		       ((struct acpi_namespace_node *)subtree_entry)->name.
-		       ascii, subtree_entry);
+
+	if (((struct acpi_namespace_node *)subtree_entry)->parent) {
+		acpi_os_printf("ACPI Namespace (from %4.4s (%p) subtree):\n",
+			       ((struct acpi_namespace_node *)subtree_entry)->
+			       name.ascii, subtree_entry);
+	} else {
+		acpi_os_printf("ACPI Namespace (from %s):\n",
+			       ACPI_NAMESPACE_ROOT);
+	}
 
 	/* Display the subtree */
 
diff --git a/drivers/acpi/acpica/dbtest.c b/drivers/acpi/acpica/dbtest.c
index 3892680a5258..8a5462439a97 100644
--- a/drivers/acpi/acpica/dbtest.c
+++ b/drivers/acpi/acpica/dbtest.c
@@ -30,6 +30,8 @@ acpi_db_test_buffer_type(struct acpi_namespace_node *node, u32 bit_length);
 static acpi_status
 acpi_db_test_string_type(struct acpi_namespace_node *node, u32 byte_length);
 
+static acpi_status acpi_db_test_package_type(struct acpi_namespace_node *node);
+
 static acpi_status
 acpi_db_read_from_object(struct acpi_namespace_node *node,
 			 acpi_object_type expected_type,
@@ -273,6 +275,11 @@ acpi_db_test_one_object(acpi_handle obj_handle,
 		bit_length = byte_length * 8;
 		break;
 
+	case ACPI_TYPE_PACKAGE:
+
+		local_type = ACPI_TYPE_PACKAGE;
+		break;
+
 	case ACPI_TYPE_FIELD_UNIT:
 	case ACPI_TYPE_BUFFER_FIELD:
 	case ACPI_TYPE_LOCAL_REGION_FIELD:
@@ -305,6 +312,7 @@ acpi_db_test_one_object(acpi_handle obj_handle,
 
 	acpi_os_printf("%14s: %4.4s",
 		       acpi_ut_get_type_name(node->type), node->name.ascii);
+
 	if (!obj_desc) {
 		acpi_os_printf(" Ignoring, no attached object\n");
 		return (AE_OK);
@@ -322,14 +330,13 @@ acpi_db_test_one_object(acpi_handle obj_handle,
 		case ACPI_ADR_SPACE_SYSTEM_MEMORY:
 		case ACPI_ADR_SPACE_SYSTEM_IO:
 		case ACPI_ADR_SPACE_PCI_CONFIG:
-		case ACPI_ADR_SPACE_EC:
 
 			break;
 
 		default:
 
 			acpi_os_printf
-			    ("    %s space is not supported [%4.4s]\n",
+			    ("    %s space is not supported in this command [%4.4s]\n",
 			     acpi_ut_get_region_name(region_obj->region.
 						     space_id),
 			     region_obj->region.node->name.ascii);
@@ -359,6 +366,11 @@ acpi_db_test_one_object(acpi_handle obj_handle,
 		status = acpi_db_test_buffer_type(node, bit_length);
 		break;
 
+	case ACPI_TYPE_PACKAGE:
+
+		status = acpi_db_test_package_type(node);
+		break;
+
 	default:
 
 		acpi_os_printf(" Ignoring, type not implemented (%2.2X)",
@@ -366,6 +378,13 @@ acpi_db_test_one_object(acpi_handle obj_handle,
 		break;
 	}
 
+	/* Exit on error, but don't abort the namespace walk */
+
+	if (ACPI_FAILURE(status)) {
+		status = AE_OK;
+		goto exit;
+	}
+
 	switch (node->type) {
 	case ACPI_TYPE_LOCAL_REGION_FIELD:
 
@@ -373,12 +392,14 @@ acpi_db_test_one_object(acpi_handle obj_handle,
 		acpi_os_printf(" (%s)",
 			       acpi_ut_get_region_name(region_obj->region.
 						       space_id));
+
 		break;
 
 	default:
 		break;
 	}
 
+exit:
 	acpi_os_printf("\n");
 	return (status);
 }
@@ -431,7 +452,6 @@ acpi_db_test_integer_type(struct acpi_namespace_node *node, u32 bit_length)
 	if (temp1->integer.value == value_to_write) {
 		value_to_write = 0;
 	}
-
 	/* Write a new value */
 
 	write_value.type = ACPI_TYPE_INTEGER;
@@ -706,6 +726,35 @@ acpi_db_test_string_type(struct acpi_namespace_node *node, u32 byte_length)
 	return (status);
 }
 
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_db_test_package_type
+ *
+ * PARAMETERS:  node                - Parent NS node for the object
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Test read for a Package object.
+ *
+ ******************************************************************************/
+
+static acpi_status acpi_db_test_package_type(struct acpi_namespace_node *node)
+{
+	union acpi_object *temp1 = NULL;
+	acpi_status status;
+
+	/* Read the original value */
+
+	status = acpi_db_read_from_object(node, ACPI_TYPE_PACKAGE, &temp1);
+	if (ACPI_FAILURE(status)) {
+		return (status);
+	}
+
+	acpi_os_printf(" %8.8X Elements", temp1->package.count);
+	acpi_os_free(temp1);
+	return (status);
+}
+
 /*******************************************************************************
  *
  * FUNCTION:    acpi_db_read_from_object
@@ -746,8 +795,8 @@ acpi_db_read_from_object(struct acpi_namespace_node *node,
 	acpi_gbl_method_executing = TRUE;
 	status = acpi_evaluate_object(read_handle, NULL,
 				      &param_objects, &return_obj);
-	acpi_gbl_method_executing = FALSE;
 
+	acpi_gbl_method_executing = FALSE;
 	if (ACPI_FAILURE(status)) {
 		acpi_os_printf("Could not read from object, %s",
 			       acpi_format_exception(status));
@@ -760,6 +809,7 @@ acpi_db_read_from_object(struct acpi_namespace_node *node,
 	case ACPI_TYPE_INTEGER:
 	case ACPI_TYPE_BUFFER:
 	case ACPI_TYPE_STRING:
+	case ACPI_TYPE_PACKAGE:
 		/*
 		 * Did we receive the type we wanted? Most important for the
 		 * Integer/Buffer case (when a field is larger than an Integer,
@@ -771,6 +821,7 @@ acpi_db_read_from_object(struct acpi_namespace_node *node,
 			     acpi_ut_get_type_name(expected_type),
 			     acpi_ut_get_type_name(ret_value->type));
 
+			acpi_os_free(return_obj.pointer);
 			return (AE_TYPE);
 		}
 
diff --git a/drivers/acpi/acpica/dswscope.c b/drivers/acpi/acpica/dswscope.c
index d1422f984f6e..7592176a8fa2 100644
--- a/drivers/acpi/acpica/dswscope.c
+++ b/drivers/acpi/acpica/dswscope.c
@@ -115,7 +115,7 @@ acpi_ds_scope_stack_push(struct acpi_namespace_node *node,
 				      acpi_ut_get_type_name(old_scope_info->
 							    common.value)));
 	} else {
-		ACPI_DEBUG_PRINT_RAW((ACPI_DB_EXEC, "[\\___] (%s)", "ROOT"));
+		ACPI_DEBUG_PRINT_RAW((ACPI_DB_EXEC, ACPI_NAMESPACE_ROOT));
 	}
 
 	ACPI_DEBUG_PRINT_RAW((ACPI_DB_EXEC,
@@ -166,14 +166,14 @@ acpi_status acpi_ds_scope_stack_pop(struct acpi_walk_state *walk_state)
 
 	new_scope_info = walk_state->scope_info;
 	if (new_scope_info) {
-		ACPI_DEBUG_PRINT_RAW((ACPI_DB_EXEC,
-				      "[%4.4s] (%s)\n",
+		ACPI_DEBUG_PRINT_RAW((ACPI_DB_EXEC, "[%4.4s] (%s)\n",
 				      acpi_ut_get_node_name(new_scope_info->
 							    scope.node),
 				      acpi_ut_get_type_name(new_scope_info->
 							    common.value)));
 	} else {
-		ACPI_DEBUG_PRINT_RAW((ACPI_DB_EXEC, "[\\___] (ROOT)\n"));
+		ACPI_DEBUG_PRINT_RAW((ACPI_DB_EXEC, "%s\n",
+				      ACPI_NAMESPACE_ROOT));
 	}
 
 	acpi_ut_delete_generic_state(scope_info);
diff --git a/drivers/acpi/acpica/evgpe.c b/drivers/acpi/acpica/evgpe.c
index abbd59063906..e10fec99a182 100644
--- a/drivers/acpi/acpica/evgpe.c
+++ b/drivers/acpi/acpica/evgpe.c
@@ -634,6 +634,12 @@ acpi_ev_detect_gpe(struct acpi_namespace_node *gpe_device,
 
 	flags = acpi_os_acquire_lock(acpi_gbl_gpe_lock);
 
+	if (!gpe_event_info) {
+		gpe_event_info = acpi_ev_get_gpe_event_info(gpe_device, gpe_number);
+		if (!gpe_event_info)
+			goto error_exit;
+	}
+
 	/* Get the info block for the entire GPE register */
 
 	gpe_register_info = gpe_event_info->register_info;
diff --git a/drivers/acpi/acpica/evxfgpe.c b/drivers/acpi/acpica/evxfgpe.c
index c80e3bdf4805..b2d5f66cc1b0 100644
--- a/drivers/acpi/acpica/evxfgpe.c
+++ b/drivers/acpi/acpica/evxfgpe.c
@@ -637,6 +637,28 @@ acpi_get_gpe_status(acpi_handle gpe_device,
 
 ACPI_EXPORT_SYMBOL(acpi_get_gpe_status)
 
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_gispatch_gpe
+ *
+ * PARAMETERS:  gpe_device          - Parent GPE Device. NULL for GPE0/GPE1
+ *              gpe_number          - GPE level within the GPE block
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Detect and dispatch a General Purpose Event to either a function
+ *              (e.g. EC) or method (e.g. _Lxx/_Exx) handler.
+ *
+ ******************************************************************************/
+void acpi_dispatch_gpe(acpi_handle gpe_device, u32 gpe_number)
+{
+	ACPI_FUNCTION_TRACE(acpi_dispatch_gpe);
+
+	acpi_ev_detect_gpe(gpe_device, NULL, gpe_number);
+}
+
+ACPI_EXPORT_SYMBOL(acpi_dispatch_gpe)
+
 /*******************************************************************************
  *
  * FUNCTION:    acpi_finish_gpe
diff --git a/drivers/acpi/acpica/hwregs.c b/drivers/acpi/acpica/hwregs.c
index 27a86ad55b58..3de794bcf8fa 100644
--- a/drivers/acpi/acpica/hwregs.c
+++ b/drivers/acpi/acpica/hwregs.c
@@ -390,14 +390,14 @@ acpi_status acpi_hw_clear_acpi_status(void)
 			  ACPI_BITMASK_ALL_FIXED_STATUS,
 			  ACPI_FORMAT_UINT64(acpi_gbl_xpm1a_status.address)));
 
-	lock_flags = acpi_os_acquire_lock(acpi_gbl_hardware_lock);
+	lock_flags = acpi_os_acquire_raw_lock(acpi_gbl_hardware_lock);
 
 	/* Clear the fixed events in PM1 A/B */
 
 	status = acpi_hw_register_write(ACPI_REGISTER_PM1_STATUS,
 					ACPI_BITMASK_ALL_FIXED_STATUS);
 
-	acpi_os_release_lock(acpi_gbl_hardware_lock, lock_flags);
+	acpi_os_release_raw_lock(acpi_gbl_hardware_lock, lock_flags);
 
 	if (ACPI_FAILURE(status)) {
 		goto exit;
diff --git a/drivers/acpi/acpica/hwxface.c b/drivers/acpi/acpica/hwxface.c
index 5d1396870bd0..6e39a771a56e 100644
--- a/drivers/acpi/acpica/hwxface.c
+++ b/drivers/acpi/acpica/hwxface.c
@@ -227,7 +227,7 @@ acpi_status acpi_write_bit_register(u32 register_id, u32 value)
 		return_ACPI_STATUS(AE_BAD_PARAMETER);
 	}
 
-	lock_flags = acpi_os_acquire_lock(acpi_gbl_hardware_lock);
+	lock_flags = acpi_os_acquire_raw_lock(acpi_gbl_hardware_lock);
 
 	/*
 	 * At this point, we know that the parent register is one of the
@@ -288,7 +288,7 @@ acpi_status acpi_write_bit_register(u32 register_id, u32 value)
 
 unlock_and_exit:
 
-	acpi_os_release_lock(acpi_gbl_hardware_lock, lock_flags);
+	acpi_os_release_raw_lock(acpi_gbl_hardware_lock, lock_flags);
 	return_ACPI_STATUS(status);
 }
 
diff --git a/drivers/acpi/acpica/rsdump.c b/drivers/acpi/acpica/rsdump.c
index b12a0b1cd9ce..6601e71b45e3 100644
--- a/drivers/acpi/acpica/rsdump.c
+++ b/drivers/acpi/acpica/rsdump.c
@@ -539,7 +539,7 @@ static void acpi_rs_out_title(const char *title)
 
 static void acpi_rs_dump_byte_list(u16 length, u8 * data)
 {
-	u8 i;
+	u16 i;
 
 	for (i = 0; i < length; i++) {
 		acpi_os_printf("%25s%2.2X : %2.2X\n", "Byte", i, data[i]);
diff --git a/drivers/acpi/acpica/tbinstal.c b/drivers/acpi/acpica/tbinstal.c
index c5085b7ae8c9..5f8e7b561c90 100644
--- a/drivers/acpi/acpica/tbinstal.c
+++ b/drivers/acpi/acpica/tbinstal.c
@@ -88,7 +88,7 @@ acpi_tb_install_table_with_override(struct acpi_table_desc *new_table_desc,
  * DESCRIPTION: This function is called to verify and install an ACPI table.
  *              When this function is called by "Load" or "LoadTable" opcodes,
  *              or by acpi_load_table() API, the "Reload" parameter is set.
- *              After sucessfully returning from this function, table is
+ *              After successfully returning from this function, table is
  *              "INSTALLED" but not "VALIDATED".
  *
  ******************************************************************************/
diff --git a/drivers/acpi/acpica/utbuffer.c b/drivers/acpi/acpica/utbuffer.c
index 148aeb84e561..fffa6f5ae59e 100644
--- a/drivers/acpi/acpica/utbuffer.c
+++ b/drivers/acpi/acpica/utbuffer.c
@@ -53,7 +53,7 @@ void acpi_ut_dump_buffer(u8 *buffer, u32 count, u32 display, u32 base_offset)
 
 		/* Print current offset */
 
-		acpi_os_printf("%6.4X: ", (base_offset + i));
+		acpi_os_printf("%8.4X: ", (base_offset + i));
 
 		/* Print 16 hex chars */
 
@@ -219,7 +219,7 @@ acpi_ut_dump_buffer_to_file(ACPI_FILE file,
 
 		/* Print current offset */
 
-		fprintf(file, "%6.4X: ", (base_offset + i));
+		fprintf(file, "%8.4X: ", (base_offset + i));
 
 		/* Print 16 hex chars */
 
diff --git a/drivers/acpi/acpica/utmutex.c b/drivers/acpi/acpica/utmutex.c
index d2d93e388f40..2e465e6a0ab6 100644
--- a/drivers/acpi/acpica/utmutex.c
+++ b/drivers/acpi/acpica/utmutex.c
@@ -52,7 +52,7 @@ acpi_status acpi_ut_mutex_initialize(void)
 		return_ACPI_STATUS (status);
 	}
 
-	status = acpi_os_create_lock (&acpi_gbl_hardware_lock);
+	status = acpi_os_create_raw_lock(&acpi_gbl_hardware_lock);
 	if (ACPI_FAILURE (status)) {
 		return_ACPI_STATUS (status);
 	}
@@ -109,7 +109,7 @@ void acpi_ut_mutex_terminate(void)
 	/* Delete the spinlocks */
 
 	acpi_os_delete_lock(acpi_gbl_gpe_lock);
-	acpi_os_delete_lock(acpi_gbl_hardware_lock);
+	acpi_os_delete_raw_lock(acpi_gbl_hardware_lock);
 	acpi_os_delete_lock(acpi_gbl_reference_count_lock);
 
 	/* Delete the reader/writer lock */
diff --git a/drivers/acpi/acpica/utobject.c b/drivers/acpi/acpica/utobject.c
index 5b78fe08d7d7..ae6d8cc18cec 100644
--- a/drivers/acpi/acpica/utobject.c
+++ b/drivers/acpi/acpica/utobject.c
@@ -8,6 +8,7 @@
  *****************************************************************************/
 
 #include <acpi/acpi.h>
+#include <linux/kmemleak.h>
 #include "accommon.h"
 #include "acnamesp.h"
 
@@ -70,6 +71,7 @@ union acpi_operand_object *acpi_ut_create_internal_object_dbg(const char
 	if (!object) {
 		return_PTR(NULL);
 	}
+	kmemleak_not_leak(object);
 
 	switch (type) {
 	case ACPI_TYPE_REGION:
diff --git a/drivers/acpi/acpica/utprint.c b/drivers/acpi/acpica/utprint.c
index 35ffd8d51c65..a98c334c3bb7 100644
--- a/drivers/acpi/acpica/utprint.c
+++ b/drivers/acpi/acpica/utprint.c
@@ -470,6 +470,7 @@ int vsnprintf(char *string, acpi_size size, const char *format, va_list args)
 		case 'X':
 
 			type |= ACPI_FORMAT_UPPER;
+			/* FALLTHROUGH */
 
 		case 'x':
 
diff --git a/drivers/acpi/acpica/utstring.c b/drivers/acpi/acpica/utstring.c
index bd57a77bbcb2..5bef0b059406 100644
--- a/drivers/acpi/acpica/utstring.c
+++ b/drivers/acpi/acpica/utstring.c
@@ -141,7 +141,7 @@ void acpi_ut_repair_name(char *name)
 	 * Special case for the root node. This can happen if we get an
 	 * error during the execution of module-level code.
 	 */
-	if (ACPI_COMPARE_NAME(name, "\\___")) {
+	if (ACPI_COMPARE_NAME(name, ACPI_ROOT_PATHNAME)) {
 		return;
 	}
 
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index 1efefe919555..02c6fd9caff7 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -481,7 +481,7 @@ static void ghes_do_proc(struct ghes *ghes,
 		if (guid_equal(sec_type, &CPER_SEC_PLATFORM_MEM)) {
 			struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata);
 
-			ghes_edac_report_mem_error(ghes, sev, mem_err);
+			ghes_edac_report_mem_error(sev, mem_err);
 
 			arch_apei_report_mem_error(sev, mem_err);
 			ghes_handle_memory_failure(gdata, sev);
@@ -1087,10 +1087,6 @@ static int ghes_probe(struct platform_device *ghes_dev)
 		goto err;
 	}
 
-	rc = ghes_edac_register(ghes, &ghes_dev->dev);
-	if (rc < 0)
-		goto err;
-
 	switch (generic->notify.type) {
 	case ACPI_HEST_NOTIFY_POLLED:
 		timer_setup(&ghes->timer, ghes_poll_func, TIMER_DEFERRABLE);
@@ -1102,14 +1098,14 @@ static int ghes_probe(struct platform_device *ghes_dev)
 		if (rc) {
 			pr_err(GHES_PFX "Failed to map GSI to IRQ for generic hardware error source: %d\n",
 			       generic->header.source_id);
-			goto err_edac_unreg;
+			goto err;
 		}
 		rc = request_irq(ghes->irq, ghes_irq_func, IRQF_SHARED,
 				 "GHES IRQ", ghes);
 		if (rc) {
 			pr_err(GHES_PFX "Failed to register IRQ for generic hardware error source: %d\n",
 			       generic->header.source_id);
-			goto err_edac_unreg;
+			goto err;
 		}
 		break;
 
@@ -1132,14 +1128,16 @@ static int ghes_probe(struct platform_device *ghes_dev)
 	default:
 		BUG();
 	}
+
 	platform_set_drvdata(ghes_dev, ghes);
 
+	ghes_edac_register(ghes, &ghes_dev->dev);
+
 	/* Handle any pending errors right away */
 	ghes_proc(ghes);
 
 	return 0;
-err_edac_unreg:
-	ghes_edac_unregister(ghes);
+
 err:
 	if (ghes) {
 		ghes_fini(ghes);
diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c
index bdb24d636d9a..b0113a5802a3 100644
--- a/drivers/acpi/battery.c
+++ b/drivers/acpi/battery.c
@@ -74,6 +74,8 @@ static async_cookie_t async_cookie;
 static bool battery_driver_registered;
 static int battery_bix_broken_package;
 static int battery_notification_delay_ms;
+static int battery_ac_is_broken;
+static int battery_check_pmic = 1;
 static unsigned int cache_time = 1000;
 module_param(cache_time, uint, 0644);
 MODULE_PARM_DESC(cache_time, "cache time in milliseconds");
@@ -81,14 +83,6 @@ MODULE_PARM_DESC(cache_time, "cache time in milliseconds");
 #ifdef CONFIG_ACPI_PROCFS_POWER
 extern struct proc_dir_entry *acpi_lock_battery_dir(void);
 extern void *acpi_unlock_battery_dir(struct proc_dir_entry *acpi_battery_dir);
-
-enum acpi_battery_files {
-	info_tag = 0,
-	state_tag,
-	alarm_tag,
-	ACPI_BATTERY_NUMFILES,
-};
-
 #endif
 
 static const struct acpi_device_id battery_device_ids[] = {
@@ -215,6 +209,20 @@ static bool acpi_battery_is_degraded(struct acpi_battery *battery)
 		battery->full_charge_capacity < battery->design_capacity;
 }
 
+static int acpi_battery_handle_discharging(struct acpi_battery *battery)
+{
+	/*
+	 * Some devices wrongly report discharging if the battery's charge level
+	 * was above the device's start charging threshold atm the AC adapter
+	 * was plugged in and the device thus did not start a new charge cycle.
+	 */
+	if ((battery_ac_is_broken || power_supply_is_system_supplied()) &&
+	    battery->rate_now == 0)
+		return POWER_SUPPLY_STATUS_NOT_CHARGING;
+
+	return POWER_SUPPLY_STATUS_DISCHARGING;
+}
+
 static int acpi_battery_get_property(struct power_supply *psy,
 				     enum power_supply_property psp,
 				     union power_supply_propval *val)
@@ -230,7 +238,7 @@ static int acpi_battery_get_property(struct power_supply *psy,
 	switch (psp) {
 	case POWER_SUPPLY_PROP_STATUS:
 		if (battery->state & ACPI_BATTERY_STATE_DISCHARGING)
-			val->intval = POWER_SUPPLY_STATUS_DISCHARGING;
+			val->intval = acpi_battery_handle_discharging(battery);
 		else if (battery->state & ACPI_BATTERY_STATE_CHARGING)
 			val->intval = POWER_SUPPLY_STATUS_CHARGING;
 		else if (acpi_battery_is_charged(battery))
@@ -985,9 +993,10 @@ static const char *acpi_battery_units(const struct acpi_battery *battery)
 		"mA" : "mW";
 }
 
-static int acpi_battery_print_info(struct seq_file *seq, int result)
+static int acpi_battery_info_proc_show(struct seq_file *seq, void *offset)
 {
 	struct acpi_battery *battery = seq->private;
+	int result = acpi_battery_update(battery, false);
 
 	if (result)
 		goto end;
@@ -1041,9 +1050,10 @@ static int acpi_battery_print_info(struct seq_file *seq, int result)
 	return result;
 }
 
-static int acpi_battery_print_state(struct seq_file *seq, int result)
+static int acpi_battery_state_proc_show(struct seq_file *seq, void *offset)
 {
 	struct acpi_battery *battery = seq->private;
+	int result = acpi_battery_update(battery, false);
 
 	if (result)
 		goto end;
@@ -1088,9 +1098,10 @@ static int acpi_battery_print_state(struct seq_file *seq, int result)
 	return result;
 }
 
-static int acpi_battery_print_alarm(struct seq_file *seq, int result)
+static int acpi_battery_alarm_proc_show(struct seq_file *seq, void *offset)
 {
 	struct acpi_battery *battery = seq->private;
+	int result = acpi_battery_update(battery, false);
 
 	if (result)
 		goto end;
@@ -1142,82 +1153,22 @@ static ssize_t acpi_battery_write_alarm(struct file *file,
 	return result;
 }
 
-typedef int(*print_func)(struct seq_file *seq, int result);
-
-static print_func acpi_print_funcs[ACPI_BATTERY_NUMFILES] = {
-	acpi_battery_print_info,
-	acpi_battery_print_state,
-	acpi_battery_print_alarm,
-};
-
-static int acpi_battery_read(int fid, struct seq_file *seq)
+static int acpi_battery_alarm_proc_open(struct inode *inode, struct file *file)
 {
-	struct acpi_battery *battery = seq->private;
-	int result = acpi_battery_update(battery, false);
-	return acpi_print_funcs[fid](seq, result);
-}
-
-#define DECLARE_FILE_FUNCTIONS(_name) \
-static int acpi_battery_read_##_name(struct seq_file *seq, void *offset) \
-{ \
-	return acpi_battery_read(_name##_tag, seq); \
-} \
-static int acpi_battery_##_name##_open_fs(struct inode *inode, struct file *file) \
-{ \
-	return single_open(file, acpi_battery_read_##_name, PDE_DATA(inode)); \
+	return single_open(file, acpi_battery_alarm_proc_show, PDE_DATA(inode));
 }
 
-DECLARE_FILE_FUNCTIONS(info);
-DECLARE_FILE_FUNCTIONS(state);
-DECLARE_FILE_FUNCTIONS(alarm);
-
-#undef DECLARE_FILE_FUNCTIONS
-
-#define FILE_DESCRIPTION_RO(_name) \
-	{ \
-	.name = __stringify(_name), \
-	.mode = S_IRUGO, \
-	.ops = { \
-		.open = acpi_battery_##_name##_open_fs, \
-		.read = seq_read, \
-		.llseek = seq_lseek, \
-		.release = single_release, \
-		.owner = THIS_MODULE, \
-		}, \
-	}
-
-#define FILE_DESCRIPTION_RW(_name) \
-	{ \
-	.name = __stringify(_name), \
-	.mode = S_IFREG | S_IRUGO | S_IWUSR, \
-	.ops = { \
-		.open = acpi_battery_##_name##_open_fs, \
-		.read = seq_read, \
-		.llseek = seq_lseek, \
-		.write = acpi_battery_write_##_name, \
-		.release = single_release, \
-		.owner = THIS_MODULE, \
-		}, \
-	}
-
-static const struct battery_file {
-	struct file_operations ops;
-	umode_t mode;
-	const char *name;
-} acpi_battery_file[] = {
-	FILE_DESCRIPTION_RO(info),
-	FILE_DESCRIPTION_RO(state),
-	FILE_DESCRIPTION_RW(alarm),
+static const struct file_operations acpi_battery_alarm_fops = {
+	.owner		= THIS_MODULE,
+	.open		= acpi_battery_alarm_proc_open,
+	.read		= seq_read,
+	.write		= acpi_battery_write_alarm,
+	.llseek		= seq_lseek,
+	.release	= single_release,
 };
 
-#undef FILE_DESCRIPTION_RO
-#undef FILE_DESCRIPTION_RW
-
 static int acpi_battery_add_fs(struct acpi_device *device)
 {
-	struct proc_dir_entry *entry = NULL;
-	int i;
-
 	printk(KERN_WARNING PREFIX "Deprecated procfs I/F for battery is loaded,"
 			" please retry with CONFIG_ACPI_PROCFS_POWER cleared\n");
 	if (!acpi_device_dir(device)) {
@@ -1227,28 +1178,24 @@ static int acpi_battery_add_fs(struct acpi_device *device)
 			return -ENODEV;
 	}
 
-	for (i = 0; i < ACPI_BATTERY_NUMFILES; ++i) {
-		entry = proc_create_data(acpi_battery_file[i].name,
-					 acpi_battery_file[i].mode,
-					 acpi_device_dir(device),
-					 &acpi_battery_file[i].ops,
-					 acpi_driver_data(device));
-		if (!entry)
-			return -ENODEV;
-	}
+	if (!proc_create_single_data("info", S_IRUGO, acpi_device_dir(device),
+			acpi_battery_info_proc_show, acpi_driver_data(device)))
+		return -ENODEV;
+	if (!proc_create_single_data("state", S_IRUGO, acpi_device_dir(device),
+			acpi_battery_state_proc_show, acpi_driver_data(device)))
+		return -ENODEV;
+	if (!proc_create_data("alarm", S_IFREG | S_IRUGO | S_IWUSR,
+			acpi_device_dir(device), &acpi_battery_alarm_fops,
+			acpi_driver_data(device)))
+		return -ENODEV;
 	return 0;
 }
 
 static void acpi_battery_remove_fs(struct acpi_device *device)
 {
-	int i;
 	if (!acpi_device_dir(device))
 		return;
-	for (i = 0; i < ACPI_BATTERY_NUMFILES; ++i)
-		remove_proc_entry(acpi_battery_file[i].name,
-				  acpi_device_dir(device));
-
-	remove_proc_entry(acpi_device_bid(device), acpi_battery_dir);
+	remove_proc_subtree(acpi_device_bid(device), acpi_battery_dir);
 	acpi_device_dir(device) = NULL;
 }
 
@@ -1332,23 +1279,64 @@ battery_notification_delay_quirk(const struct dmi_system_id *d)
 	return 0;
 }
 
+static int __init
+battery_ac_is_broken_quirk(const struct dmi_system_id *d)
+{
+	battery_ac_is_broken = 1;
+	return 0;
+}
+
+static int __init
+battery_do_not_check_pmic_quirk(const struct dmi_system_id *d)
+{
+	battery_check_pmic = 0;
+	return 0;
+}
+
 static const struct dmi_system_id bat_dmi_table[] __initconst = {
 	{
+		/* NEC LZ750/LS */
 		.callback = battery_bix_broken_package_quirk,
-		.ident = "NEC LZ750/LS",
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "NEC"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "PC-LZ750LS"),
 		},
 	},
 	{
+		/* Acer Aspire V5-573G */
 		.callback = battery_notification_delay_quirk,
-		.ident = "Acer Aspire V5-573G",
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire V5-573G"),
 		},
 	},
+	{
+		/* Point of View mobii wintab p800w */
+		.callback = battery_ac_is_broken_quirk,
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
+			DMI_MATCH(DMI_BOARD_NAME, "Aptio CRB"),
+			DMI_MATCH(DMI_BIOS_VERSION, "3BAIR1013"),
+			/* Above matches are too generic, add bios-date match */
+			DMI_MATCH(DMI_BIOS_DATE, "08/22/2014"),
+		},
+	},
+	{
+		/* ECS EF20EA */
+		.callback = battery_do_not_check_pmic_quirk,
+		.matches = {
+			DMI_MATCH(DMI_PRODUCT_NAME, "EF20EA"),
+		},
+	},
+	{
+		/* Lenovo Ideapad Miix 320 */
+		.callback = battery_do_not_check_pmic_quirk,
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "80XF"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "Lenovo MIIX 320-10ICR"),
+		},
+	},
 	{},
 };
 
@@ -1488,16 +1476,18 @@ static void __init acpi_battery_init_async(void *unused, async_cookie_t cookie)
 	unsigned int i;
 	int result;
 
-	for (i = 0; i < ARRAY_SIZE(acpi_battery_blacklist); i++)
-		if (acpi_dev_present(acpi_battery_blacklist[i], "1", -1)) {
-			pr_info(PREFIX ACPI_BATTERY_DEVICE_NAME
-				": found native %s PMIC, not loading\n",
-				acpi_battery_blacklist[i]);
-			return;
-		}
-
 	dmi_check_system(bat_dmi_table);
 
+	if (battery_check_pmic) {
+		for (i = 0; i < ARRAY_SIZE(acpi_battery_blacklist); i++)
+			if (acpi_dev_present(acpi_battery_blacklist[i], "1", -1)) {
+				pr_info(PREFIX ACPI_BATTERY_DEVICE_NAME
+					": found native %s PMIC, not loading\n",
+					acpi_battery_blacklist[i]);
+				return;
+			}
+	}
+
 #ifdef CONFIG_ACPI_PROCFS_POWER
 	acpi_battery_dir = acpi_lock_battery_dir();
 	if (!acpi_battery_dir)
diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c
index f1cc4f9d31cd..2345a5ee2dbb 100644
--- a/drivers/acpi/button.c
+++ b/drivers/acpi/button.c
@@ -263,19 +263,6 @@ static int acpi_button_state_seq_show(struct seq_file *seq, void *offset)
 	return 0;
 }
 
-static int acpi_button_state_open_fs(struct inode *inode, struct file *file)
-{
-	return single_open(file, acpi_button_state_seq_show, PDE_DATA(inode));
-}
-
-static const struct file_operations acpi_button_state_fops = {
-	.owner = THIS_MODULE,
-	.open = acpi_button_state_open_fs,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-
 static int acpi_button_add_fs(struct acpi_device *device)
 {
 	struct acpi_button *button = acpi_driver_data(device);
@@ -311,9 +298,9 @@ static int acpi_button_add_fs(struct acpi_device *device)
 	}
 
 	/* create /proc/acpi/button/lid/LID/state */
-	entry = proc_create_data(ACPI_BUTTON_FILE_STATE,
-				 S_IRUGO, acpi_device_dir(device),
-				 &acpi_button_state_fops, device);
+	entry = proc_create_single_data(ACPI_BUTTON_FILE_STATE, S_IRUGO,
+			acpi_device_dir(device), acpi_button_state_seq_show,
+			device);
 	if (!entry) {
 		ret = -ENODEV;
 		goto remove_dev_dir;
diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
index 735c74a4cbdb..d9ce4b162e2c 100644
--- a/drivers/acpi/cppc_acpi.c
+++ b/drivers/acpi/cppc_acpi.c
@@ -39,6 +39,7 @@
 
 #include <linux/cpufreq.h>
 #include <linux/delay.h>
+#include <linux/iopoll.h>
 #include <linux/ktime.h>
 #include <linux/rwsem.h>
 #include <linux/wait.h>
@@ -49,7 +50,7 @@ struct cppc_pcc_data {
 	struct mbox_chan *pcc_channel;
 	void __iomem *pcc_comm_addr;
 	bool pcc_channel_acquired;
-	ktime_t deadline;
+	unsigned int deadline_us;
 	unsigned int pcc_mpar, pcc_mrtt, pcc_nominal;
 
 	bool pending_pcc_write_cmd;	/* Any pending/batched PCC write cmds? */
@@ -156,6 +157,9 @@ show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, highest_perf);
 show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_perf);
 show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, nominal_perf);
 show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_nonlinear_perf);
+show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_freq);
+show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, nominal_freq);
+
 show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, reference_perf);
 show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, wraparound_time);
 
@@ -183,6 +187,8 @@ static struct attribute *cppc_attrs[] = {
 	&lowest_perf.attr,
 	&lowest_nonlinear_perf.attr,
 	&nominal_perf.attr,
+	&nominal_freq.attr,
+	&lowest_freq.attr,
 	NULL
 };
 
@@ -193,42 +199,31 @@ static struct kobj_type cppc_ktype = {
 
 static int check_pcc_chan(int pcc_ss_id, bool chk_err_bit)
 {
-	int ret = -EIO, status = 0;
+	int ret, status;
 	struct cppc_pcc_data *pcc_ss_data = pcc_data[pcc_ss_id];
 	struct acpi_pcct_shared_memory __iomem *generic_comm_base =
 		pcc_ss_data->pcc_comm_addr;
-	ktime_t next_deadline = ktime_add(ktime_get(),
-					  pcc_ss_data->deadline);
 
 	if (!pcc_ss_data->platform_owns_pcc)
 		return 0;
 
-	/* Retry in case the remote processor was too slow to catch up. */
-	while (!ktime_after(ktime_get(), next_deadline)) {
-		/*
-		 * Per spec, prior to boot the PCC space wil be initialized by
-		 * platform and should have set the command completion bit when
-		 * PCC can be used by OSPM
-		 */
-		status = readw_relaxed(&generic_comm_base->status);
-		if (status & PCC_CMD_COMPLETE_MASK) {
-			ret = 0;
-			if (chk_err_bit && (status & PCC_ERROR_MASK))
-				ret = -EIO;
-			break;
-		}
-		/*
-		 * Reducing the bus traffic in case this loop takes longer than
-		 * a few retries.
-		 */
-		udelay(3);
-	}
+	/*
+	 * Poll PCC status register every 3us(delay_us) for maximum of
+	 * deadline_us(timeout_us) until PCC command complete bit is set(cond)
+	 */
+	ret = readw_relaxed_poll_timeout(&generic_comm_base->status, status,
+					status & PCC_CMD_COMPLETE_MASK, 3,
+					pcc_ss_data->deadline_us);
 
-	if (likely(!ret))
+	if (likely(!ret)) {
 		pcc_ss_data->platform_owns_pcc = false;
-	else
-		pr_err("PCC check channel failed for ss: %d. Status=%x\n",
-		       pcc_ss_id, status);
+		if (chk_err_bit && (status & PCC_ERROR_MASK))
+			ret = -EIO;
+	}
+
+	if (unlikely(ret))
+		pr_err("PCC check channel failed for ss: %d. ret=%d\n",
+		       pcc_ss_id, ret);
 
 	return ret;
 }
@@ -580,7 +575,7 @@ static int register_pcc_channel(int pcc_ss_idx)
 		 * So add an arbitrary amount of wait on top of Nominal.
 		 */
 		usecs_lat = NUM_RETRIES * cppc_ss->latency;
-		pcc_data[pcc_ss_idx]->deadline = ns_to_ktime(usecs_lat * NSEC_PER_USEC);
+		pcc_data[pcc_ss_idx]->deadline_us = usecs_lat;
 		pcc_data[pcc_ss_idx]->pcc_mrtt = cppc_ss->min_turnaround_time;
 		pcc_data[pcc_ss_idx]->pcc_mpar = cppc_ss->max_access_rate;
 		pcc_data[pcc_ss_idx]->pcc_nominal = cppc_ss->latency;
@@ -613,7 +608,6 @@ bool __weak cpc_ffh_supported(void)
 	return false;
 }
 
-
 /**
  * pcc_data_alloc() - Allocate the pcc_data memory for pcc subspace
  *
@@ -641,6 +635,34 @@ int pcc_data_alloc(int pcc_ss_id)
 
 	return 0;
 }
+
+/* Check if CPPC revision + num_ent combination is supported */
+static bool is_cppc_supported(int revision, int num_ent)
+{
+	int expected_num_ent;
+
+	switch (revision) {
+	case CPPC_V2_REV:
+		expected_num_ent = CPPC_V2_NUM_ENT;
+		break;
+	case CPPC_V3_REV:
+		expected_num_ent = CPPC_V3_NUM_ENT;
+		break;
+	default:
+		pr_debug("Firmware exports unsupported CPPC revision: %d\n",
+			revision);
+		return false;
+	}
+
+	if (expected_num_ent != num_ent) {
+		pr_debug("Firmware exports %d entries. Expected: %d for CPPC rev:%d\n",
+			num_ent, expected_num_ent, revision);
+		return false;
+	}
+
+	return true;
+}
+
 /*
  * An example CPC table looks like the following.
  *
@@ -731,14 +753,6 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 				cpc_obj->type);
 		goto out_free;
 	}
-
-	/* Only support CPPCv2. Bail otherwise. */
-	if (num_ent != CPPC_NUM_ENT) {
-		pr_debug("Firmware exports %d entries. Expected: %d\n",
-				num_ent, CPPC_NUM_ENT);
-		goto out_free;
-	}
-
 	cpc_ptr->num_entries = num_ent;
 
 	/* Second entry should be revision. */
@@ -750,12 +764,10 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 				cpc_obj->type);
 		goto out_free;
 	}
+	cpc_ptr->version = cpc_rev;
 
-	if (cpc_rev != CPPC_REV) {
-		pr_debug("Firmware exports revision:%d. Expected:%d\n",
-				cpc_rev, CPPC_REV);
+	if (!is_cppc_supported(cpc_rev, num_ent))
 		goto out_free;
-	}
 
 	/* Iterate through remaining entries in _CPC */
 	for (i = 2; i < num_ent; i++) {
@@ -808,6 +820,18 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 		}
 	}
 	per_cpu(cpu_pcc_subspace_idx, pr->id) = pcc_subspace_id;
+
+	/*
+	 * Initialize the remaining cpc_regs as unsupported.
+	 * Example: In case FW exposes CPPC v2, the below loop will initialize
+	 * LOWEST_FREQ and NOMINAL_FREQ regs as unsupported
+	 */
+	for (i = num_ent - 2; i < MAX_CPC_REG_ENT; i++) {
+		cpc_ptr->cpc_regs[i].type = ACPI_TYPE_INTEGER;
+		cpc_ptr->cpc_regs[i].cpc_entry.int_value = 0;
+	}
+
+
 	/* Store CPU Logical ID */
 	cpc_ptr->cpu_id = pr->id;
 
@@ -1037,26 +1061,34 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 {
 	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum);
 	struct cpc_register_resource *highest_reg, *lowest_reg,
-		*lowest_non_linear_reg, *nominal_reg;
-	u64 high, low, nom, min_nonlinear;
+		*lowest_non_linear_reg, *nominal_reg,
+		*low_freq_reg = NULL, *nom_freq_reg = NULL;
+	u64 high, low, nom, min_nonlinear, low_f = 0, nom_f = 0;
 	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum);
-	struct cppc_pcc_data *pcc_ss_data;
+	struct cppc_pcc_data *pcc_ss_data = NULL;
 	int ret = 0, regs_in_pcc = 0;
 
-	if (!cpc_desc || pcc_ss_id < 0) {
+	if (!cpc_desc) {
 		pr_debug("No CPC descriptor for CPU:%d\n", cpunum);
 		return -ENODEV;
 	}
 
-	pcc_ss_data = pcc_data[pcc_ss_id];
 	highest_reg = &cpc_desc->cpc_regs[HIGHEST_PERF];
 	lowest_reg = &cpc_desc->cpc_regs[LOWEST_PERF];
 	lowest_non_linear_reg = &cpc_desc->cpc_regs[LOW_NON_LINEAR_PERF];
 	nominal_reg = &cpc_desc->cpc_regs[NOMINAL_PERF];
+	low_freq_reg = &cpc_desc->cpc_regs[LOWEST_FREQ];
+	nom_freq_reg = &cpc_desc->cpc_regs[NOMINAL_FREQ];
 
 	/* Are any of the regs PCC ?*/
 	if (CPC_IN_PCC(highest_reg) || CPC_IN_PCC(lowest_reg) ||
-		CPC_IN_PCC(lowest_non_linear_reg) || CPC_IN_PCC(nominal_reg)) {
+		CPC_IN_PCC(lowest_non_linear_reg) || CPC_IN_PCC(nominal_reg) ||
+		CPC_IN_PCC(low_freq_reg) || CPC_IN_PCC(nom_freq_reg)) {
+		if (pcc_ss_id < 0) {
+			pr_debug("Invalid pcc_ss_id\n");
+			return -ENODEV;
+		}
+		pcc_ss_data = pcc_data[pcc_ss_id];
 		regs_in_pcc = 1;
 		down_write(&pcc_ss_data->pcc_lock);
 		/* Ring doorbell once to update PCC subspace */
@@ -1081,6 +1113,17 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 	if (!high || !low || !nom || !min_nonlinear)
 		ret = -EFAULT;
 
+	/* Read optional lowest and nominal frequencies if present */
+	if (CPC_SUPPORTED(low_freq_reg))
+		cpc_read(cpunum, low_freq_reg, &low_f);
+
+	if (CPC_SUPPORTED(nom_freq_reg))
+		cpc_read(cpunum, nom_freq_reg, &nom_f);
+
+	perf_caps->lowest_freq = low_f;
+	perf_caps->nominal_freq = nom_f;
+
+
 out_err:
 	if (regs_in_pcc)
 		up_write(&pcc_ss_data->pcc_lock);
@@ -1101,16 +1144,15 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs)
 	struct cpc_register_resource *delivered_reg, *reference_reg,
 		*ref_perf_reg, *ctr_wrap_reg;
 	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum);
-	struct cppc_pcc_data *pcc_ss_data;
+	struct cppc_pcc_data *pcc_ss_data = NULL;
 	u64 delivered, reference, ref_perf, ctr_wrap_time;
 	int ret = 0, regs_in_pcc = 0;
 
-	if (!cpc_desc || pcc_ss_id < 0) {
+	if (!cpc_desc) {
 		pr_debug("No CPC descriptor for CPU:%d\n", cpunum);
 		return -ENODEV;
 	}
 
-	pcc_ss_data = pcc_data[pcc_ss_id];
 	delivered_reg = &cpc_desc->cpc_regs[DELIVERED_CTR];
 	reference_reg = &cpc_desc->cpc_regs[REFERENCE_CTR];
 	ref_perf_reg = &cpc_desc->cpc_regs[REFERENCE_PERF];
@@ -1126,6 +1168,11 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs)
 	/* Are any of the regs PCC ?*/
 	if (CPC_IN_PCC(delivered_reg) || CPC_IN_PCC(reference_reg) ||
 		CPC_IN_PCC(ctr_wrap_reg) || CPC_IN_PCC(ref_perf_reg)) {
+		if (pcc_ss_id < 0) {
+			pr_debug("Invalid pcc_ss_id\n");
+			return -ENODEV;
+		}
+		pcc_ss_data = pcc_data[pcc_ss_id];
 		down_write(&pcc_ss_data->pcc_lock);
 		regs_in_pcc = 1;
 		/* Ring doorbell once to update PCC subspace */
@@ -1176,15 +1223,14 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu);
 	struct cpc_register_resource *desired_reg;
 	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu);
-	struct cppc_pcc_data *pcc_ss_data;
+	struct cppc_pcc_data *pcc_ss_data = NULL;
 	int ret = 0;
 
-	if (!cpc_desc || pcc_ss_id < 0) {
+	if (!cpc_desc) {
 		pr_debug("No CPC descriptor for CPU:%d\n", cpu);
 		return -ENODEV;
 	}
 
-	pcc_ss_data = pcc_data[pcc_ss_id];
 	desired_reg = &cpc_desc->cpc_regs[DESIRED_PERF];
 
 	/*
@@ -1195,6 +1241,11 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 	 * achieve that goal here
 	 */
 	if (CPC_IN_PCC(desired_reg)) {
+		if (pcc_ss_id < 0) {
+			pr_debug("Invalid pcc_ss_id\n");
+			return -ENODEV;
+		}
+		pcc_ss_data = pcc_data[pcc_ss_id];
 		down_read(&pcc_ss_data->pcc_lock); /* BEGIN Phase-I */
 		if (pcc_ss_data->platform_owns_pcc) {
 			ret = check_pcc_chan(pcc_ss_id, false);
diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c
index 3d96e4da2d98..a7c2673ffd36 100644
--- a/drivers/acpi/device_pm.c
+++ b/drivers/acpi/device_pm.c
@@ -1257,10 +1257,7 @@ int acpi_dev_pm_attach(struct device *dev, bool power_on)
 	struct acpi_device *adev = ACPI_COMPANION(dev);
 
 	if (!adev)
-		return -ENODEV;
-
-	if (dev->pm_domain)
-		return -EEXIST;
+		return 0;
 
 	/*
 	 * Only attach the power domain to the first device if the
@@ -1268,7 +1265,7 @@ int acpi_dev_pm_attach(struct device *dev, bool power_on)
 	 * management twice.
 	 */
 	if (!acpi_device_is_first_physical_node(adev, dev))
-		return -EBUSY;
+		return 0;
 
 	acpi_add_pm_notifier(adev, dev, acpi_pm_notify_work_func);
 	dev_pm_domain_set(dev, &acpi_general_pm_domain);
@@ -1278,7 +1275,7 @@ int acpi_dev_pm_attach(struct device *dev, bool power_on)
 	}
 
 	dev->pm_domain->detach = acpi_dev_pm_detach;
-	return 0;
+	return 1;
 }
 EXPORT_SYMBOL_GPL(acpi_dev_pm_attach);
 #endif /* CONFIG_PM */
diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c
index 30a572956557..bb94cf0731fe 100644
--- a/drivers/acpi/ec.c
+++ b/drivers/acpi/ec.c
@@ -1034,6 +1034,12 @@ void acpi_ec_unblock_transactions(void)
 		acpi_ec_start(first_ec, true);
 }
 
+void acpi_ec_dispatch_gpe(void)
+{
+	if (first_ec)
+		acpi_dispatch_gpe(NULL, first_ec->gpe);
+}
+
 /* --------------------------------------------------------------------------
                                 Event Management
    -------------------------------------------------------------------------- */
diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h
index 1d0a501bc7f0..530a3f675490 100644
--- a/drivers/acpi/internal.h
+++ b/drivers/acpi/internal.h
@@ -188,6 +188,7 @@ int acpi_ec_ecdt_probe(void);
 int acpi_ec_dsdt_probe(void);
 void acpi_ec_block_transactions(void);
 void acpi_ec_unblock_transactions(void);
+void acpi_ec_dispatch_gpe(void);
 int acpi_ec_add_query_handler(struct acpi_ec *ec, u8 query_bit,
 			      acpi_handle handle, acpi_ec_query_func func,
 			      void *data);
diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c
index 0da18bde6a16..7433035ded95 100644
--- a/drivers/acpi/pci_root.c
+++ b/drivers/acpi/pci_root.c
@@ -153,6 +153,7 @@ static struct pci_osc_bit_struct pci_osc_control_bit[] = {
 	{ OSC_PCI_EXPRESS_PME_CONTROL, "PME" },
 	{ OSC_PCI_EXPRESS_AER_CONTROL, "AER" },
 	{ OSC_PCI_EXPRESS_CAPABILITY_CONTROL, "PCIeCapability" },
+	{ OSC_PCI_EXPRESS_LTR_CONTROL, "LTR" },
 };
 
 static void decode_osc_bits(struct acpi_pci_root *root, char *msg, u32 word,
@@ -472,9 +473,17 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm)
 	}
 
 	control = OSC_PCI_EXPRESS_CAPABILITY_CONTROL
-		| OSC_PCI_EXPRESS_NATIVE_HP_CONTROL
 		| OSC_PCI_EXPRESS_PME_CONTROL;
 
+	if (IS_ENABLED(CONFIG_PCIEASPM))
+		control |= OSC_PCI_EXPRESS_LTR_CONTROL;
+
+	if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE))
+		control |= OSC_PCI_EXPRESS_NATIVE_HP_CONTROL;
+
+	if (IS_ENABLED(CONFIG_HOTPLUG_PCI_SHPC))
+		control |= OSC_PCI_SHPC_NATIVE_HP_CONTROL;
+
 	if (pci_aer_available()) {
 		if (aer_acpi_firmware_first())
 			dev_info(&device->dev,
@@ -900,11 +909,15 @@ struct pci_bus *acpi_pci_root_create(struct acpi_pci_root *root,
 
 	host_bridge = to_pci_host_bridge(bus->bridge);
 	if (!(root->osc_control_set & OSC_PCI_EXPRESS_NATIVE_HP_CONTROL))
-		host_bridge->native_hotplug = 0;
+		host_bridge->native_pcie_hotplug = 0;
+	if (!(root->osc_control_set & OSC_PCI_SHPC_NATIVE_HP_CONTROL))
+		host_bridge->native_shpc_hotplug = 0;
 	if (!(root->osc_control_set & OSC_PCI_EXPRESS_AER_CONTROL))
 		host_bridge->native_aer = 0;
 	if (!(root->osc_control_set & OSC_PCI_EXPRESS_PME_CONTROL))
 		host_bridge->native_pme = 0;
+	if (!(root->osc_control_set & OSC_PCI_EXPRESS_LTR_CONTROL))
+		host_bridge->native_ltr = 0;
 
 	pci_scan_child_bus(bus);
 	pci_set_host_bridge_release(host_bridge, acpi_pci_root_release_info,
diff --git a/drivers/acpi/pptt.c b/drivers/acpi/pptt.c
new file mode 100644
index 000000000000..e5ea1974d1e3
--- /dev/null
+++ b/drivers/acpi/pptt.c
@@ -0,0 +1,655 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * pptt.c - parsing of Processor Properties Topology Table (PPTT)
+ *
+ * Copyright (C) 2018, ARM
+ *
+ * This file implements parsing of the Processor Properties Topology Table
+ * which is optionally used to describe the processor and cache topology.
+ * Due to the relative pointers used throughout the table, this doesn't
+ * leverage the existing subtable parsing in the kernel.
+ *
+ * The PPTT structure is an inverted tree, with each node potentially
+ * holding one or two inverted tree data structures describing
+ * the caches available at that level. Each cache structure optionally
+ * contains properties describing the cache at a given level which can be
+ * used to override hardware probed values.
+ */
+#define pr_fmt(fmt) "ACPI PPTT: " fmt
+
+#include <linux/acpi.h>
+#include <linux/cacheinfo.h>
+#include <acpi/processor.h>
+
+static struct acpi_subtable_header *fetch_pptt_subtable(struct acpi_table_header *table_hdr,
+							u32 pptt_ref)
+{
+	struct acpi_subtable_header *entry;
+
+	/* there isn't a subtable at reference 0 */
+	if (pptt_ref < sizeof(struct acpi_subtable_header))
+		return NULL;
+
+	if (pptt_ref + sizeof(struct acpi_subtable_header) > table_hdr->length)
+		return NULL;
+
+	entry = ACPI_ADD_PTR(struct acpi_subtable_header, table_hdr, pptt_ref);
+
+	if (entry->length == 0)
+		return NULL;
+
+	if (pptt_ref + entry->length > table_hdr->length)
+		return NULL;
+
+	return entry;
+}
+
+static struct acpi_pptt_processor *fetch_pptt_node(struct acpi_table_header *table_hdr,
+						   u32 pptt_ref)
+{
+	return (struct acpi_pptt_processor *)fetch_pptt_subtable(table_hdr, pptt_ref);
+}
+
+static struct acpi_pptt_cache *fetch_pptt_cache(struct acpi_table_header *table_hdr,
+						u32 pptt_ref)
+{
+	return (struct acpi_pptt_cache *)fetch_pptt_subtable(table_hdr, pptt_ref);
+}
+
+static struct acpi_subtable_header *acpi_get_pptt_resource(struct acpi_table_header *table_hdr,
+							   struct acpi_pptt_processor *node,
+							   int resource)
+{
+	u32 *ref;
+
+	if (resource >= node->number_of_priv_resources)
+		return NULL;
+
+	ref = ACPI_ADD_PTR(u32, node, sizeof(struct acpi_pptt_processor));
+	ref += resource;
+
+	return fetch_pptt_subtable(table_hdr, *ref);
+}
+
+static inline bool acpi_pptt_match_type(int table_type, int type)
+{
+	return ((table_type & ACPI_PPTT_MASK_CACHE_TYPE) == type ||
+		table_type & ACPI_PPTT_CACHE_TYPE_UNIFIED & type);
+}
+
+/**
+ * acpi_pptt_walk_cache() - Attempt to find the requested acpi_pptt_cache
+ * @table_hdr: Pointer to the head of the PPTT table
+ * @local_level: passed res reflects this cache level
+ * @res: cache resource in the PPTT we want to walk
+ * @found: returns a pointer to the requested level if found
+ * @level: the requested cache level
+ * @type: the requested cache type
+ *
+ * Attempt to find a given cache level, while counting the max number
+ * of cache levels for the cache node.
+ *
+ * Given a pptt resource, verify that it is a cache node, then walk
+ * down each level of caches, counting how many levels are found
+ * as well as checking the cache type (icache, dcache, unified). If a
+ * level & type match, then we set found, and continue the search.
+ * Once the entire cache branch has been walked return its max
+ * depth.
+ *
+ * Return: The cache structure and the level we terminated with.
+ */
+static int acpi_pptt_walk_cache(struct acpi_table_header *table_hdr,
+				int local_level,
+				struct acpi_subtable_header *res,
+				struct acpi_pptt_cache **found,
+				int level, int type)
+{
+	struct acpi_pptt_cache *cache;
+
+	if (res->type != ACPI_PPTT_TYPE_CACHE)
+		return 0;
+
+	cache = (struct acpi_pptt_cache *) res;
+	while (cache) {
+		local_level++;
+
+		if (local_level == level &&
+		    cache->flags & ACPI_PPTT_CACHE_TYPE_VALID &&
+		    acpi_pptt_match_type(cache->attributes, type)) {
+			if (*found != NULL && cache != *found)
+				pr_warn("Found duplicate cache level/type unable to determine uniqueness\n");
+
+			pr_debug("Found cache @ level %d\n", level);
+			*found = cache;
+			/*
+			 * continue looking at this node's resource list
+			 * to verify that we don't find a duplicate
+			 * cache node.
+			 */
+		}
+		cache = fetch_pptt_cache(table_hdr, cache->next_level_of_cache);
+	}
+	return local_level;
+}
+
+static struct acpi_pptt_cache *acpi_find_cache_level(struct acpi_table_header *table_hdr,
+						     struct acpi_pptt_processor *cpu_node,
+						     int *starting_level, int level,
+						     int type)
+{
+	struct acpi_subtable_header *res;
+	int number_of_levels = *starting_level;
+	int resource = 0;
+	struct acpi_pptt_cache *ret = NULL;
+	int local_level;
+
+	/* walk down from processor node */
+	while ((res = acpi_get_pptt_resource(table_hdr, cpu_node, resource))) {
+		resource++;
+
+		local_level = acpi_pptt_walk_cache(table_hdr, *starting_level,
+						   res, &ret, level, type);
+		/*
+		 * we are looking for the max depth. Since its potentially
+		 * possible for a given node to have resources with differing
+		 * depths verify that the depth we have found is the largest.
+		 */
+		if (number_of_levels < local_level)
+			number_of_levels = local_level;
+	}
+	if (number_of_levels > *starting_level)
+		*starting_level = number_of_levels;
+
+	return ret;
+}
+
+/**
+ * acpi_count_levels() - Given a PPTT table, and a cpu node, count the caches
+ * @table_hdr: Pointer to the head of the PPTT table
+ * @cpu_node: processor node we wish to count caches for
+ *
+ * Given a processor node containing a processing unit, walk into it and count
+ * how many levels exist solely for it, and then walk up each level until we hit
+ * the root node (ignore the package level because it may be possible to have
+ * caches that exist across packages). Count the number of cache levels that
+ * exist at each level on the way up.
+ *
+ * Return: Total number of levels found.
+ */
+static int acpi_count_levels(struct acpi_table_header *table_hdr,
+			     struct acpi_pptt_processor *cpu_node)
+{
+	int total_levels = 0;
+
+	do {
+		acpi_find_cache_level(table_hdr, cpu_node, &total_levels, 0, 0);
+		cpu_node = fetch_pptt_node(table_hdr, cpu_node->parent);
+	} while (cpu_node);
+
+	return total_levels;
+}
+
+/**
+ * acpi_pptt_leaf_node() - Given a processor node, determine if its a leaf
+ * @table_hdr: Pointer to the head of the PPTT table
+ * @node: passed node is checked to see if its a leaf
+ *
+ * Determine if the *node parameter is a leaf node by iterating the
+ * PPTT table, looking for nodes which reference it.
+ *
+ * Return: 0 if we find a node referencing the passed node (or table error),
+ * or 1 if we don't.
+ */
+static int acpi_pptt_leaf_node(struct acpi_table_header *table_hdr,
+			       struct acpi_pptt_processor *node)
+{
+	struct acpi_subtable_header *entry;
+	unsigned long table_end;
+	u32 node_entry;
+	struct acpi_pptt_processor *cpu_node;
+	u32 proc_sz;
+
+	table_end = (unsigned long)table_hdr + table_hdr->length;
+	node_entry = ACPI_PTR_DIFF(node, table_hdr);
+	entry = ACPI_ADD_PTR(struct acpi_subtable_header, table_hdr,
+			     sizeof(struct acpi_table_pptt));
+	proc_sz = sizeof(struct acpi_pptt_processor *);
+
+	while ((unsigned long)entry + proc_sz < table_end) {
+		cpu_node = (struct acpi_pptt_processor *)entry;
+		if (entry->type == ACPI_PPTT_TYPE_PROCESSOR &&
+		    cpu_node->parent == node_entry)
+			return 0;
+		if (entry->length == 0)
+			return 0;
+		entry = ACPI_ADD_PTR(struct acpi_subtable_header, entry,
+				     entry->length);
+
+	}
+	return 1;
+}
+
+/**
+ * acpi_find_processor_node() - Given a PPTT table find the requested processor
+ * @table_hdr:  Pointer to the head of the PPTT table
+ * @acpi_cpu_id: cpu we are searching for
+ *
+ * Find the subtable entry describing the provided processor.
+ * This is done by iterating the PPTT table looking for processor nodes
+ * which have an acpi_processor_id that matches the acpi_cpu_id parameter
+ * passed into the function. If we find a node that matches this criteria
+ * we verify that its a leaf node in the topology rather than depending
+ * on the valid flag, which doesn't need to be set for leaf nodes.
+ *
+ * Return: NULL, or the processors acpi_pptt_processor*
+ */
+static struct acpi_pptt_processor *acpi_find_processor_node(struct acpi_table_header *table_hdr,
+							    u32 acpi_cpu_id)
+{
+	struct acpi_subtable_header *entry;
+	unsigned long table_end;
+	struct acpi_pptt_processor *cpu_node;
+	u32 proc_sz;
+
+	table_end = (unsigned long)table_hdr + table_hdr->length;
+	entry = ACPI_ADD_PTR(struct acpi_subtable_header, table_hdr,
+			     sizeof(struct acpi_table_pptt));
+	proc_sz = sizeof(struct acpi_pptt_processor *);
+
+	/* find the processor structure associated with this cpuid */
+	while ((unsigned long)entry + proc_sz < table_end) {
+		cpu_node = (struct acpi_pptt_processor *)entry;
+
+		if (entry->length == 0) {
+			pr_warn("Invalid zero length subtable\n");
+			break;
+		}
+		if (entry->type == ACPI_PPTT_TYPE_PROCESSOR &&
+		    acpi_cpu_id == cpu_node->acpi_processor_id &&
+		     acpi_pptt_leaf_node(table_hdr, cpu_node)) {
+			return (struct acpi_pptt_processor *)entry;
+		}
+
+		entry = ACPI_ADD_PTR(struct acpi_subtable_header, entry,
+				     entry->length);
+	}
+
+	return NULL;
+}
+
+static int acpi_find_cache_levels(struct acpi_table_header *table_hdr,
+				  u32 acpi_cpu_id)
+{
+	int number_of_levels = 0;
+	struct acpi_pptt_processor *cpu;
+
+	cpu = acpi_find_processor_node(table_hdr, acpi_cpu_id);
+	if (cpu)
+		number_of_levels = acpi_count_levels(table_hdr, cpu);
+
+	return number_of_levels;
+}
+
+static u8 acpi_cache_type(enum cache_type type)
+{
+	switch (type) {
+	case CACHE_TYPE_DATA:
+		pr_debug("Looking for data cache\n");
+		return ACPI_PPTT_CACHE_TYPE_DATA;
+	case CACHE_TYPE_INST:
+		pr_debug("Looking for instruction cache\n");
+		return ACPI_PPTT_CACHE_TYPE_INSTR;
+	default:
+	case CACHE_TYPE_UNIFIED:
+		pr_debug("Looking for unified cache\n");
+		/*
+		 * It is important that ACPI_PPTT_CACHE_TYPE_UNIFIED
+		 * contains the bit pattern that will match both
+		 * ACPI unified bit patterns because we use it later
+		 * to match both cases.
+		 */
+		return ACPI_PPTT_CACHE_TYPE_UNIFIED;
+	}
+}
+
+static struct acpi_pptt_cache *acpi_find_cache_node(struct acpi_table_header *table_hdr,
+						    u32 acpi_cpu_id,
+						    enum cache_type type,
+						    unsigned int level,
+						    struct acpi_pptt_processor **node)
+{
+	int total_levels = 0;
+	struct acpi_pptt_cache *found = NULL;
+	struct acpi_pptt_processor *cpu_node;
+	u8 acpi_type = acpi_cache_type(type);
+
+	pr_debug("Looking for CPU %d's level %d cache type %d\n",
+		 acpi_cpu_id, level, acpi_type);
+
+	cpu_node = acpi_find_processor_node(table_hdr, acpi_cpu_id);
+
+	while (cpu_node && !found) {
+		found = acpi_find_cache_level(table_hdr, cpu_node,
+					      &total_levels, level, acpi_type);
+		*node = cpu_node;
+		cpu_node = fetch_pptt_node(table_hdr, cpu_node->parent);
+	}
+
+	return found;
+}
+
+/* total number of attributes checked by the properties code */
+#define PPTT_CHECKED_ATTRIBUTES 4
+
+/**
+ * update_cache_properties() - Update cacheinfo for the given processor
+ * @this_leaf: Kernel cache info structure being updated
+ * @found_cache: The PPTT node describing this cache instance
+ * @cpu_node: A unique reference to describe this cache instance
+ *
+ * The ACPI spec implies that the fields in the cache structures are used to
+ * extend and correct the information probed from the hardware. Lets only
+ * set fields that we determine are VALID.
+ *
+ * Return: nothing. Side effect of updating the global cacheinfo
+ */
+static void update_cache_properties(struct cacheinfo *this_leaf,
+				    struct acpi_pptt_cache *found_cache,
+				    struct acpi_pptt_processor *cpu_node)
+{
+	int valid_flags = 0;
+
+	this_leaf->fw_token = cpu_node;
+	if (found_cache->flags & ACPI_PPTT_SIZE_PROPERTY_VALID) {
+		this_leaf->size = found_cache->size;
+		valid_flags++;
+	}
+	if (found_cache->flags & ACPI_PPTT_LINE_SIZE_VALID) {
+		this_leaf->coherency_line_size = found_cache->line_size;
+		valid_flags++;
+	}
+	if (found_cache->flags & ACPI_PPTT_NUMBER_OF_SETS_VALID) {
+		this_leaf->number_of_sets = found_cache->number_of_sets;
+		valid_flags++;
+	}
+	if (found_cache->flags & ACPI_PPTT_ASSOCIATIVITY_VALID) {
+		this_leaf->ways_of_associativity = found_cache->associativity;
+		valid_flags++;
+	}
+	if (found_cache->flags & ACPI_PPTT_WRITE_POLICY_VALID) {
+		switch (found_cache->attributes & ACPI_PPTT_MASK_WRITE_POLICY) {
+		case ACPI_PPTT_CACHE_POLICY_WT:
+			this_leaf->attributes = CACHE_WRITE_THROUGH;
+			break;
+		case ACPI_PPTT_CACHE_POLICY_WB:
+			this_leaf->attributes = CACHE_WRITE_BACK;
+			break;
+		}
+	}
+	if (found_cache->flags & ACPI_PPTT_ALLOCATION_TYPE_VALID) {
+		switch (found_cache->attributes & ACPI_PPTT_MASK_ALLOCATION_TYPE) {
+		case ACPI_PPTT_CACHE_READ_ALLOCATE:
+			this_leaf->attributes |= CACHE_READ_ALLOCATE;
+			break;
+		case ACPI_PPTT_CACHE_WRITE_ALLOCATE:
+			this_leaf->attributes |= CACHE_WRITE_ALLOCATE;
+			break;
+		case ACPI_PPTT_CACHE_RW_ALLOCATE:
+		case ACPI_PPTT_CACHE_RW_ALLOCATE_ALT:
+			this_leaf->attributes |=
+				CACHE_READ_ALLOCATE | CACHE_WRITE_ALLOCATE;
+			break;
+		}
+	}
+	/*
+	 * If the above flags are valid, and the cache type is NOCACHE
+	 * update the cache type as well.
+	 */
+	if (this_leaf->type == CACHE_TYPE_NOCACHE &&
+	    valid_flags == PPTT_CHECKED_ATTRIBUTES)
+		this_leaf->type = CACHE_TYPE_UNIFIED;
+}
+
+static void cache_setup_acpi_cpu(struct acpi_table_header *table,
+				 unsigned int cpu)
+{
+	struct acpi_pptt_cache *found_cache;
+	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
+	u32 acpi_cpu_id = get_acpi_id_for_cpu(cpu);
+	struct cacheinfo *this_leaf;
+	unsigned int index = 0;
+	struct acpi_pptt_processor *cpu_node = NULL;
+
+	while (index < get_cpu_cacheinfo(cpu)->num_leaves) {
+		this_leaf = this_cpu_ci->info_list + index;
+		found_cache = acpi_find_cache_node(table, acpi_cpu_id,
+						   this_leaf->type,
+						   this_leaf->level,
+						   &cpu_node);
+		pr_debug("found = %p %p\n", found_cache, cpu_node);
+		if (found_cache)
+			update_cache_properties(this_leaf,
+						found_cache,
+						cpu_node);
+
+		index++;
+	}
+}
+
+/* Passing level values greater than this will result in search termination */
+#define PPTT_ABORT_PACKAGE 0xFF
+
+static struct acpi_pptt_processor *acpi_find_processor_package_id(struct acpi_table_header *table_hdr,
+								  struct acpi_pptt_processor *cpu,
+								  int level, int flag)
+{
+	struct acpi_pptt_processor *prev_node;
+
+	while (cpu && level) {
+		if (cpu->flags & flag)
+			break;
+		pr_debug("level %d\n", level);
+		prev_node = fetch_pptt_node(table_hdr, cpu->parent);
+		if (prev_node == NULL)
+			break;
+		cpu = prev_node;
+		level--;
+	}
+	return cpu;
+}
+
+/**
+ * topology_get_acpi_cpu_tag() - Find a unique topology value for a feature
+ * @table: Pointer to the head of the PPTT table
+ * @cpu: Kernel logical cpu number
+ * @level: A level that terminates the search
+ * @flag: A flag which terminates the search
+ *
+ * Get a unique value given a cpu, and a topology level, that can be
+ * matched to determine which cpus share common topological features
+ * at that level.
+ *
+ * Return: Unique value, or -ENOENT if unable to locate cpu
+ */
+static int topology_get_acpi_cpu_tag(struct acpi_table_header *table,
+				     unsigned int cpu, int level, int flag)
+{
+	struct acpi_pptt_processor *cpu_node;
+	u32 acpi_cpu_id = get_acpi_id_for_cpu(cpu);
+
+	cpu_node = acpi_find_processor_node(table, acpi_cpu_id);
+	if (cpu_node) {
+		cpu_node = acpi_find_processor_package_id(table, cpu_node,
+							  level, flag);
+		/* Only the first level has a guaranteed id */
+		if (level == 0)
+			return cpu_node->acpi_processor_id;
+		return ACPI_PTR_DIFF(cpu_node, table);
+	}
+	pr_warn_once("PPTT table found, but unable to locate core %d (%d)\n",
+		    cpu, acpi_cpu_id);
+	return -ENOENT;
+}
+
+static int find_acpi_cpu_topology_tag(unsigned int cpu, int level, int flag)
+{
+	struct acpi_table_header *table;
+	acpi_status status;
+	int retval;
+
+	status = acpi_get_table(ACPI_SIG_PPTT, 0, &table);
+	if (ACPI_FAILURE(status)) {
+		pr_warn_once("No PPTT table found, cpu topology may be inaccurate\n");
+		return -ENOENT;
+	}
+	retval = topology_get_acpi_cpu_tag(table, cpu, level, flag);
+	pr_debug("Topology Setup ACPI cpu %d, level %d ret = %d\n",
+		 cpu, level, retval);
+	acpi_put_table(table);
+
+	return retval;
+}
+
+/**
+ * acpi_find_last_cache_level() - Determines the number of cache levels for a PE
+ * @cpu: Kernel logical cpu number
+ *
+ * Given a logical cpu number, returns the number of levels of cache represented
+ * in the PPTT. Errors caused by lack of a PPTT table, or otherwise, return 0
+ * indicating we didn't find any cache levels.
+ *
+ * Return: Cache levels visible to this core.
+ */
+int acpi_find_last_cache_level(unsigned int cpu)
+{
+	u32 acpi_cpu_id;
+	struct acpi_table_header *table;
+	int number_of_levels = 0;
+	acpi_status status;
+
+	pr_debug("Cache Setup find last level cpu=%d\n", cpu);
+
+	acpi_cpu_id = get_acpi_id_for_cpu(cpu);
+	status = acpi_get_table(ACPI_SIG_PPTT, 0, &table);
+	if (ACPI_FAILURE(status)) {
+		pr_warn_once("No PPTT table found, cache topology may be inaccurate\n");
+	} else {
+		number_of_levels = acpi_find_cache_levels(table, acpi_cpu_id);
+		acpi_put_table(table);
+	}
+	pr_debug("Cache Setup find last level level=%d\n", number_of_levels);
+
+	return number_of_levels;
+}
+
+/**
+ * cache_setup_acpi() - Override CPU cache topology with data from the PPTT
+ * @cpu: Kernel logical cpu number
+ *
+ * Updates the global cache info provided by cpu_get_cacheinfo()
+ * when there are valid properties in the acpi_pptt_cache nodes. A
+ * successful parse may not result in any updates if none of the
+ * cache levels have any valid flags set.  Futher, a unique value is
+ * associated with each known CPU cache entry. This unique value
+ * can be used to determine whether caches are shared between cpus.
+ *
+ * Return: -ENOENT on failure to find table, or 0 on success
+ */
+int cache_setup_acpi(unsigned int cpu)
+{
+	struct acpi_table_header *table;
+	acpi_status status;
+
+	pr_debug("Cache Setup ACPI cpu %d\n", cpu);
+
+	status = acpi_get_table(ACPI_SIG_PPTT, 0, &table);
+	if (ACPI_FAILURE(status)) {
+		pr_warn_once("No PPTT table found, cache topology may be inaccurate\n");
+		return -ENOENT;
+	}
+
+	cache_setup_acpi_cpu(table, cpu);
+	acpi_put_table(table);
+
+	return status;
+}
+
+/**
+ * find_acpi_cpu_topology() - Determine a unique topology value for a given cpu
+ * @cpu: Kernel logical cpu number
+ * @level: The topological level for which we would like a unique ID
+ *
+ * Determine a topology unique ID for each thread/core/cluster/mc_grouping
+ * /socket/etc. This ID can then be used to group peers, which will have
+ * matching ids.
+ *
+ * The search terminates when either the requested level is found or
+ * we reach a root node. Levels beyond the termination point will return the
+ * same unique ID. The unique id for level 0 is the acpi processor id. All
+ * other levels beyond this use a generated value to uniquely identify
+ * a topological feature.
+ *
+ * Return: -ENOENT if the PPTT doesn't exist, or the cpu cannot be found.
+ * Otherwise returns a value which represents a unique topological feature.
+ */
+int find_acpi_cpu_topology(unsigned int cpu, int level)
+{
+	return find_acpi_cpu_topology_tag(cpu, level, 0);
+}
+
+/**
+ * find_acpi_cpu_cache_topology() - Determine a unique cache topology value
+ * @cpu: Kernel logical cpu number
+ * @level: The cache level for which we would like a unique ID
+ *
+ * Determine a unique ID for each unified cache in the system
+ *
+ * Return: -ENOENT if the PPTT doesn't exist, or the cpu cannot be found.
+ * Otherwise returns a value which represents a unique topological feature.
+ */
+int find_acpi_cpu_cache_topology(unsigned int cpu, int level)
+{
+	struct acpi_table_header *table;
+	struct acpi_pptt_cache *found_cache;
+	acpi_status status;
+	u32 acpi_cpu_id = get_acpi_id_for_cpu(cpu);
+	struct acpi_pptt_processor *cpu_node = NULL;
+	int ret = -1;
+
+	status = acpi_get_table(ACPI_SIG_PPTT, 0, &table);
+	if (ACPI_FAILURE(status)) {
+		pr_warn_once("No PPTT table found, topology may be inaccurate\n");
+		return -ENOENT;
+	}
+
+	found_cache = acpi_find_cache_node(table, acpi_cpu_id,
+					   CACHE_TYPE_UNIFIED,
+					   level,
+					   &cpu_node);
+	if (found_cache)
+		ret = ACPI_PTR_DIFF(cpu_node, table);
+
+	acpi_put_table(table);
+
+	return ret;
+}
+
+
+/**
+ * find_acpi_cpu_topology_package() - Determine a unique cpu package value
+ * @cpu: Kernel logical cpu number
+ *
+ * Determine a topology unique package ID for the given cpu.
+ * This ID can then be used to group peers, which will have matching ids.
+ *
+ * The search terminates when either a level is found with the PHYSICAL_PACKAGE
+ * flag set or we reach a root node.
+ *
+ * Return: -ENOENT if the PPTT doesn't exist, or the cpu cannot be found.
+ * Otherwise returns a value which represents the package for this cpu.
+ */
+int find_acpi_cpu_topology_package(unsigned int cpu)
+{
+	return find_acpi_cpu_topology_tag(cpu, PPTT_ABORT_PACKAGE,
+					  ACPI_PPTT_PHYSICAL_PACKAGE);
+}
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index abb559cd28d7..8ae28dc0b784 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -33,6 +33,7 @@
 #include <linux/cpuidle.h>
 #include <linux/cpu.h>
 #include <acpi/processor.h>
+#include <linux/kvm_para.h>
 
 /*
  * Include the apic definitions for x86 to have the APIC timer related defines
@@ -665,7 +666,8 @@ static void __cpuidle acpi_idle_do_entry(struct acpi_processor_cx *cx)
 		/* Dummy wait op - must do something useless after P_LVL2 read
 		   because chipsets cannot guarantee that STPCLK# signal
 		   gets asserted in time to freeze execution properly. */
-		inl(acpi_gbl_FADT.xpm_timer_block.address);
+		if (!kvm_para_available())
+			inl(acpi_gbl_FADT.xpm_timer_block.address);
 	}
 }
 
@@ -687,7 +689,8 @@ static int acpi_idle_play_dead(struct cpuidle_device *dev, int index)
 		else if (cx->entry_method == ACPI_CSTATE_SYSTEMIO) {
 			inb(cx->address);
 			/* See comment in acpi_idle_do_entry() */
-			inl(acpi_gbl_FADT.xpm_timer_block.address);
+			if (!kvm_para_available())
+				inl(acpi_gbl_FADT.xpm_timer_block.address);
 		} else
 			return -ENODEV;
 	}
diff --git a/drivers/acpi/reboot.c b/drivers/acpi/reboot.c
index 71769fd687b2..6fa9c2a4cfe9 100644
--- a/drivers/acpi/reboot.c
+++ b/drivers/acpi/reboot.c
@@ -8,8 +8,8 @@ void acpi_reboot(void)
 {
 	struct acpi_generic_address *rr;
 	struct pci_bus *bus0;
-	u8 reset_value;
 	unsigned int devfn;
+	u8 reset_value;
 
 	if (acpi_disabled)
 		return;
@@ -40,7 +40,7 @@ void acpi_reboot(void)
 		/* Form PCI device/function pair. */
 		devfn = PCI_DEVFN((rr->address >> 32) & 0xffff,
 				  (rr->address >> 16) & 0xffff);
-		printk(KERN_DEBUG "Resetting with ACPI PCI RESET_REG.");
+		printk(KERN_DEBUG "Resetting with ACPI PCI RESET_REG.\n");
 		/* Write the value that resets us. */
 		pci_bus_write_config_byte(bus0, devfn,
 				(rr->address & 0xffff), reset_value);
diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c
index 974e58457697..5d0486f1cfcd 100644
--- a/drivers/acpi/sleep.c
+++ b/drivers/acpi/sleep.c
@@ -989,6 +989,13 @@ static void acpi_s2idle_wake(void)
 	    !irqd_is_wakeup_armed(irq_get_irq_data(acpi_sci_irq))) {
 		pm_system_cancel_wakeup();
 		s2idle_wakeup = true;
+		/*
+		 * On some platforms with the LPS0 _DSM device noirq resume
+		 * takes too much time for EC wakeup events to survive, so look
+		 * for them now.
+		 */
+		if (lps0_device_handle)
+			acpi_ec_dispatch_gpe();
 	}
 }
 
diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c
index 849c4fb19b03..a3d012b08fc5 100644
--- a/drivers/acpi/tables.c
+++ b/drivers/acpi/tables.c
@@ -222,7 +222,7 @@ void acpi_table_print_madt_entry(struct acpi_subtable_header *header)
  * acpi_parse_entries_array - for each proc_num find a suitable subtable
  *
  * @id: table id (for debugging purposes)
- * @table_size: single entry size
+ * @table_size: size of the root table
  * @table_header: where does the table start?
  * @proc: array of acpi_subtable_proc struct containing entry id
  *        and associated handler with it
@@ -233,6 +233,11 @@ void acpi_table_print_madt_entry(struct acpi_subtable_header *header)
  * on it. Assumption is that there's only single handler for particular
  * entry id.
  *
+ * The table_size is not the size of the complete ACPI table (the length
+ * field in the header struct), but only the size of the root table; i.e.,
+ * the offset from the very first byte of the complete ACPI table, to the
+ * first byte of the very first subtable.
+ *
  * On success returns sum of all matching entries for all proc handlers.
  * Otherwise, -ENODEV or -EINVAL is returned.
  */
@@ -400,7 +405,7 @@ int __init acpi_table_parse(char *id, acpi_tbl_table_handler handler)
 		return -ENODEV;
 }
 
-/* 
+/*
  * The BIOS is supposed to supply a single APIC/MADT,
  * but some report two.  Provide a knob to use either.
  * (don't you wish instance 0 and 1 were not the same?)
@@ -457,7 +462,7 @@ static const char * const table_sigs[] = {
 	ACPI_SIG_UEFI, ACPI_SIG_WAET, ACPI_SIG_WDAT, ACPI_SIG_WDDT,
 	ACPI_SIG_WDRT, ACPI_SIG_DSDT, ACPI_SIG_FADT, ACPI_SIG_PSDT,
 	ACPI_SIG_RSDT, ACPI_SIG_XSDT, ACPI_SIG_SSDT, ACPI_SIG_IORT,
-	ACPI_SIG_NFIT, ACPI_SIG_HMAT, NULL };
+	ACPI_SIG_NFIT, ACPI_SIG_HMAT, ACPI_SIG_PPTT, NULL };
 
 #define ACPI_HEADER_SIZE sizeof(struct acpi_table_header)
 
diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c
index 4a3ac31c07d0..41b706403ef7 100644
--- a/drivers/amba/bus.c
+++ b/drivers/amba/bus.c
@@ -20,6 +20,7 @@
 #include <linux/sizes.h>
 #include <linux/limits.h>
 #include <linux/clk/clk-conf.h>
+#include <linux/platform_device.h>
 
 #include <asm/irq.h>
 
@@ -101,8 +102,8 @@ static ssize_t driver_override_store(struct device *_dev,
 	if (strlen(driver_override)) {
 		dev->driver_override = driver_override;
 	} else {
-	       kfree(driver_override);
-	       dev->driver_override = NULL;
+		kfree(driver_override);
+		dev->driver_override = NULL;
 	}
 	device_unlock(_dev);
 
@@ -193,15 +194,18 @@ static const struct dev_pm_ops amba_pm = {
 /*
  * Primecells are part of the Advanced Microcontroller Bus Architecture,
  * so we call the bus "amba".
+ * DMA configuration for platform and AMBA bus is same. So here we reuse
+ * platform's DMA config routine.
  */
 struct bus_type amba_bustype = {
 	.name		= "amba",
 	.dev_groups	= amba_dev_groups,
 	.match		= amba_match,
 	.uevent		= amba_uevent,
+	.dma_configure	= platform_dma_configure,
 	.pm		= &amba_pm,
-	.force_dma	= true,
 };
+EXPORT_SYMBOL_GPL(amba_bustype);
 
 static int __init amba_init(void)
 {
@@ -248,7 +252,7 @@ static int amba_probe(struct device *dev)
 			break;
 
 		ret = dev_pm_domain_attach(dev, true);
-		if (ret == -EPROBE_DEFER)
+		if (ret)
 			break;
 
 		ret = amba_get_enable_pclk(pcdev);
@@ -375,7 +379,7 @@ static int amba_device_try_add(struct amba_device *dev, struct resource *parent)
 	}
 
 	ret = dev_pm_domain_attach(&dev->dev, true);
-	if (ret == -EPROBE_DEFER) {
+	if (ret) {
 		iounmap(tmp);
 		goto err_release;
 	}
diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig
index 7dce3795b887..ee4880bfdcdc 100644
--- a/drivers/android/Kconfig
+++ b/drivers/android/Kconfig
@@ -10,7 +10,7 @@ if ANDROID
 
 config ANDROID_BINDER_IPC
 	bool "Android Binder IPC Driver"
-	depends on MMU
+	depends on MMU && !M68K
 	default n
 	---help---
 	  Binder is used in Android for both communication between processes,
@@ -32,19 +32,6 @@ config ANDROID_BINDER_DEVICES
 	  created. Each binder device has its own context manager, and is
 	  therefore logically separated from the other devices.
 
-config ANDROID_BINDER_IPC_32BIT
-	bool "Use old (Android 4.4 and earlier) 32-bit binder API"
-	depends on !64BIT && ANDROID_BINDER_IPC
-	default y
-	---help---
-	  The Binder API has been changed to support both 32 and 64bit
-	  applications in a mixed environment.
-
-	  Enable this to support an old 32-bit Android user-space (v4.4 and
-	  earlier).
-
-	  Note that enabling this will break newer Android user-space.
-
 config ANDROID_BINDER_IPC_SELFTEST
 	bool "Android Binder IPC Driver Selftest"
 	depends on ANDROID_BINDER_IPC
diff --git a/drivers/android/binder.c b/drivers/android/binder.c
index e578eee31589..95283f3bb51c 100644
--- a/drivers/android/binder.c
+++ b/drivers/android/binder.c
@@ -72,10 +72,6 @@
 #include <linux/security.h>
 #include <linux/spinlock.h>
 
-#ifdef CONFIG_ANDROID_BINDER_IPC_32BIT
-#define BINDER_IPC_32BIT 1
-#endif
-
 #include <uapi/linux/android/binder.h>
 #include "binder_alloc.h"
 #include "binder_trace.h"
@@ -2058,8 +2054,8 @@ static size_t binder_validate_object(struct binder_buffer *buffer, u64 offset)
 	struct binder_object_header *hdr;
 	size_t object_size = 0;
 
-	if (offset > buffer->data_size - sizeof(*hdr) ||
-	    buffer->data_size < sizeof(*hdr) ||
+	if (buffer->data_size < sizeof(*hdr) ||
+	    offset > buffer->data_size - sizeof(*hdr) ||
 	    !IS_ALIGNED(offset, sizeof(u32)))
 		return 0;
 
@@ -3925,10 +3921,11 @@ static int binder_thread_read(struct binder_proc *proc,
 			binder_inner_proc_unlock(proc);
 			if (put_user(e->cmd, (uint32_t __user *)ptr))
 				return -EFAULT;
+			cmd = e->cmd;
 			e->cmd = BR_OK;
 			ptr += sizeof(uint32_t);
 
-			binder_stat_br(proc, thread, e->cmd);
+			binder_stat_br(proc, thread, cmd);
 		} break;
 		case BINDER_WORK_TRANSACTION_COMPLETE: {
 			binder_inner_proc_unlock(proc);
@@ -4696,7 +4693,7 @@ static void binder_vma_close(struct vm_area_struct *vma)
 	binder_defer_work(proc, BINDER_DEFERRED_PUT_FILES);
 }
 
-static int binder_vm_fault(struct vm_fault *vmf)
+static vm_fault_t binder_vm_fault(struct vm_fault *vmf)
 {
 	return VM_FAULT_SIGBUS;
 }
@@ -4730,7 +4727,9 @@ static int binder_mmap(struct file *filp, struct vm_area_struct *vma)
 		failure_string = "bad vm_flags";
 		goto err_bad_arg;
 	}
-	vma->vm_flags = (vma->vm_flags | VM_DONTCOPY) & ~VM_MAYWRITE;
+	vma->vm_flags |= VM_DONTCOPY | VM_MIXEDMAP;
+	vma->vm_flags &= ~VM_MAYWRITE;
+
 	vma->vm_ops = &binder_vm_ops;
 	vma->vm_private_data = proc;
 
diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c
index 5a426c877dfb..4f382d51def1 100644
--- a/drivers/android/binder_alloc.c
+++ b/drivers/android/binder_alloc.c
@@ -219,7 +219,7 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate,
 		mm = alloc->vma_vm_mm;
 
 	if (mm) {
-		down_write(&mm->mmap_sem);
+		down_read(&mm->mmap_sem);
 		vma = alloc->vma;
 	}
 
@@ -288,7 +288,7 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate,
 		/* vm_insert_page does not seem to increment the refcount */
 	}
 	if (mm) {
-		up_write(&mm->mmap_sem);
+		up_read(&mm->mmap_sem);
 		mmput(mm);
 	}
 	return 0;
@@ -321,7 +321,7 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate,
 	}
 err_no_vma:
 	if (mm) {
-		up_write(&mm->mmap_sem);
+		up_read(&mm->mmap_sem);
 		mmput(mm);
 	}
 	return vma ? -ENOMEM : -ESRCH;
diff --git a/drivers/ata/acard-ahci.c b/drivers/ata/acard-ahci.c
index 940ddbc59aa7..583e366be7e2 100644
--- a/drivers/ata/acard-ahci.c
+++ b/drivers/ata/acard-ahci.c
@@ -271,7 +271,7 @@ static void acard_ahci_qc_prep(struct ata_queued_cmd *qc)
 	 * Fill in command table information.  First, the header,
 	 * a SATA Register - Host to Device command FIS.
 	 */
-	cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;
+	cmd_tbl = pp->cmd_tbl + qc->hw_tag * AHCI_CMD_TBL_SZ;
 
 	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
 	if (is_atapi) {
@@ -294,7 +294,7 @@ static void acard_ahci_qc_prep(struct ata_queued_cmd *qc)
 	if (is_atapi)
 		opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
 
-	ahci_fill_cmd_slot(pp, qc->tag, opts);
+	ahci_fill_cmd_slot(pp, qc->hw_tag, opts);
 }
 
 static bool acard_ahci_qc_fill_rtf(struct ata_queued_cmd *qc)
diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h
index 824bd399f02e..1609ebab4e23 100644
--- a/drivers/ata/ahci.h
+++ b/drivers/ata/ahci.h
@@ -390,7 +390,7 @@ extern struct device_attribute *ahci_sdev_attrs[];
  */
 #define AHCI_SHT(drv_name)						\
 	ATA_NCQ_SHT(drv_name),						\
-	.can_queue		= AHCI_MAX_CMDS - 1,			\
+	.can_queue		= AHCI_MAX_CMDS,			\
 	.sg_tablesize		= AHCI_MAX_SG,				\
 	.dma_boundary		= AHCI_DMA_BOUNDARY,			\
 	.shost_attrs		= ahci_shost_attrs,			\
diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c
index e5d90977caec..965842a08743 100644
--- a/drivers/ata/libahci.c
+++ b/drivers/ata/libahci.c
@@ -1649,7 +1649,7 @@ static void ahci_qc_prep(struct ata_queued_cmd *qc)
 	 * Fill in command table information.  First, the header,
 	 * a SATA Register - Host to Device command FIS.
 	 */
-	cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;
+	cmd_tbl = pp->cmd_tbl + qc->hw_tag * AHCI_CMD_TBL_SZ;
 
 	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
 	if (is_atapi) {
@@ -1670,7 +1670,7 @@ static void ahci_qc_prep(struct ata_queued_cmd *qc)
 	if (is_atapi)
 		opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
 
-	ahci_fill_cmd_slot(pp, qc->tag, opts);
+	ahci_fill_cmd_slot(pp, qc->hw_tag, opts);
 }
 
 static void ahci_fbs_dec_intr(struct ata_port *ap)
@@ -2006,7 +2006,7 @@ unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
 	pp->active_link = qc->dev->link;
 
 	if (ata_is_ncq(qc->tf.protocol))
-		writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
+		writel(1 << qc->hw_tag, port_mmio + PORT_SCR_ACT);
 
 	if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) {
 		u32 fbs = readl(port_mmio + PORT_FBS);
@@ -2016,7 +2016,7 @@ unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
 		pp->fbs_last_dev = qc->dev->link->pmp;
 	}
 
-	writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
+	writel(1 << qc->hw_tag, port_mmio + PORT_CMD_ISSUE);
 
 	ahci_sw_activity(qc->dev->link);
 
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index 346b163f6e89..cdcd55cb2a9a 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -759,7 +759,7 @@ int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev,
 	tf->flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE;
 	tf->flags |= tf_flags;
 
-	if (ata_ncq_enabled(dev) && likely(tag != ATA_TAG_INTERNAL)) {
+	if (ata_ncq_enabled(dev) && !ata_tag_internal(tag)) {
 		/* yay, NCQ */
 		if (!lba_48_ok(block, n_block))
 			return -ERANGE;
@@ -1570,8 +1570,9 @@ unsigned ata_exec_internal_sg(struct ata_device *dev,
 	u8 command = tf->command;
 	int auto_timeout = 0;
 	struct ata_queued_cmd *qc;
-	unsigned int tag, preempted_tag;
-	u32 preempted_sactive, preempted_qc_active;
+	unsigned int preempted_tag;
+	u32 preempted_sactive;
+	u64 preempted_qc_active;
 	int preempted_nr_active_links;
 	DECLARE_COMPLETION_ONSTACK(wait);
 	unsigned long flags;
@@ -1587,20 +1588,10 @@ unsigned ata_exec_internal_sg(struct ata_device *dev,
 	}
 
 	/* initialize internal qc */
+	qc = __ata_qc_from_tag(ap, ATA_TAG_INTERNAL);
 
-	/* XXX: Tag 0 is used for drivers with legacy EH as some
-	 * drivers choke if any other tag is given.  This breaks
-	 * ata_tag_internal() test for those drivers.  Don't use new
-	 * EH stuff without converting to it.
-	 */
-	if (ap->ops->error_handler)
-		tag = ATA_TAG_INTERNAL;
-	else
-		tag = 0;
-
-	qc = __ata_qc_from_tag(ap, tag);
-
-	qc->tag = tag;
+	qc->tag = ATA_TAG_INTERNAL;
+	qc->hw_tag = 0;
 	qc->scsicmd = NULL;
 	qc->ap = ap;
 	qc->dev = dev;
@@ -2295,7 +2286,7 @@ static int ata_dev_config_ncq(struct ata_device *dev,
 		return 0;
 	}
 	if (ap->flags & ATA_FLAG_NCQ) {
-		hdepth = min(ap->scsi_host->can_queue, ATA_MAX_QUEUE - 1);
+		hdepth = min(ap->scsi_host->can_queue, ATA_MAX_QUEUE);
 		dev->flags |= ATA_DFLAG_NCQ;
 	}
 
@@ -3573,9 +3564,11 @@ static int ata_dev_set_mode(struct ata_device *dev)
 	DPRINTK("xfer_shift=%u, xfer_mode=0x%x\n",
 		dev->xfer_shift, (int)dev->xfer_mode);
 
-	ata_dev_info(dev, "configured for %s%s\n",
-		     ata_mode_string(ata_xfer_mode2mask(dev->xfer_mode)),
-		     dev_err_whine);
+	if (!(ehc->i.flags & ATA_EHI_QUIET) ||
+	    ehc->i.flags & ATA_EHI_DID_HARDRESET)
+		ata_dev_info(dev, "configured for %s%s\n",
+			     ata_mode_string(ata_xfer_mode2mask(dev->xfer_mode)),
+			     dev_err_whine);
 
 	return 0;
 
@@ -5136,7 +5129,7 @@ struct ata_queued_cmd *ata_qc_new_init(struct ata_device *dev, int tag)
 	}
 
 	qc = __ata_qc_from_tag(ap, tag);
-	qc->tag = tag;
+	qc->tag = qc->hw_tag = tag;
 	qc->scsicmd = NULL;
 	qc->ap = ap;
 	qc->dev = dev;
@@ -5166,7 +5159,7 @@ void ata_qc_free(struct ata_queued_cmd *qc)
 
 	qc->flags = 0;
 	tag = qc->tag;
-	if (likely(ata_tag_valid(tag))) {
+	if (ata_tag_valid(tag)) {
 		qc->tag = ATA_TAG_POISON;
 		if (ap->flags & ATA_FLAG_SAS_HOST)
 			ata_sas_free_tag(tag, ap);
@@ -5188,7 +5181,7 @@ void __ata_qc_complete(struct ata_queued_cmd *qc)
 
 	/* command should be marked inactive atomically with qc completion */
 	if (ata_is_ncq(qc->tf.protocol)) {
-		link->sactive &= ~(1 << qc->tag);
+		link->sactive &= ~(1 << qc->hw_tag);
 		if (!link->sactive)
 			ap->nr_active_links--;
 	} else {
@@ -5206,7 +5199,7 @@ void __ata_qc_complete(struct ata_queued_cmd *qc)
 	 * is called. (when rc != 0 and atapi request sense is needed)
 	 */
 	qc->flags &= ~ATA_QCFLAG_ACTIVE;
-	ap->qc_active &= ~(1 << qc->tag);
+	ap->qc_active &= ~(1ULL << qc->tag);
 
 	/* call completion callback */
 	qc->complete_fn(qc);
@@ -5363,29 +5356,29 @@ void ata_qc_complete(struct ata_queued_cmd *qc)
  *	RETURNS:
  *	Number of completed commands on success, -errno otherwise.
  */
-int ata_qc_complete_multiple(struct ata_port *ap, u32 qc_active)
+int ata_qc_complete_multiple(struct ata_port *ap, u64 qc_active)
 {
 	int nr_done = 0;
-	u32 done_mask;
+	u64 done_mask;
 
 	done_mask = ap->qc_active ^ qc_active;
 
 	if (unlikely(done_mask & qc_active)) {
-		ata_port_err(ap, "illegal qc_active transition (%08x->%08x)\n",
+		ata_port_err(ap, "illegal qc_active transition (%08llx->%08llx)\n",
 			     ap->qc_active, qc_active);
 		return -EINVAL;
 	}
 
 	while (done_mask) {
 		struct ata_queued_cmd *qc;
-		unsigned int tag = __ffs(done_mask);
+		unsigned int tag = __ffs64(done_mask);
 
 		qc = ata_qc_from_tag(ap, tag);
 		if (qc) {
 			ata_qc_complete(qc);
 			nr_done++;
 		}
-		done_mask &= ~(1 << tag);
+		done_mask &= ~(1ULL << tag);
 	}
 
 	return nr_done;
@@ -5416,11 +5409,11 @@ void ata_qc_issue(struct ata_queued_cmd *qc)
 	WARN_ON_ONCE(ap->ops->error_handler && ata_tag_valid(link->active_tag));
 
 	if (ata_is_ncq(prot)) {
-		WARN_ON_ONCE(link->sactive & (1 << qc->tag));
+		WARN_ON_ONCE(link->sactive & (1 << qc->hw_tag));
 
 		if (!link->sactive)
 			ap->nr_active_links++;
-		link->sactive |= 1 << qc->tag;
+		link->sactive |= 1 << qc->hw_tag;
 	} else {
 		WARN_ON_ONCE(link->sactive);
 
@@ -5429,7 +5422,7 @@ void ata_qc_issue(struct ata_queued_cmd *qc)
 	}
 
 	qc->flags |= ATA_QCFLAG_ACTIVE;
-	ap->qc_active |= 1 << qc->tag;
+	ap->qc_active |= 1ULL << qc->tag;
 
 	/*
 	 * We guarantee to LLDs that they will have at least one
@@ -6428,7 +6421,7 @@ void ata_host_init(struct ata_host *host, struct device *dev,
 {
 	spin_lock_init(&host->lock);
 	mutex_init(&host->eh_mutex);
-	host->n_tags = ATA_MAX_QUEUE - 1;
+	host->n_tags = ATA_MAX_QUEUE;
 	host->dev = dev;
 	host->ops = ops;
 }
@@ -6510,7 +6503,7 @@ int ata_host_register(struct ata_host *host, struct scsi_host_template *sht)
 {
 	int i, rc;
 
-	host->n_tags = clamp(sht->can_queue, 1, ATA_MAX_QUEUE - 1);
+	host->n_tags = clamp(sht->can_queue, 1, ATA_MAX_QUEUE);
 
 	/* host must have been started */
 	if (!(host->flags & ATA_HOST_STARTED)) {
diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index 513b260bcff1..d5412145d76d 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -500,57 +500,6 @@ void ata_eh_release(struct ata_port *ap)
 	mutex_unlock(&ap->host->eh_mutex);
 }
 
-/**
- *	ata_scsi_timed_out - SCSI layer time out callback
- *	@cmd: timed out SCSI command
- *
- *	Handles SCSI layer timeout.  We race with normal completion of
- *	the qc for @cmd.  If the qc is already gone, we lose and let
- *	the scsi command finish (EH_HANDLED).  Otherwise, the qc has
- *	timed out and EH should be invoked.  Prevent ata_qc_complete()
- *	from finishing it by setting EH_SCHEDULED and return
- *	EH_NOT_HANDLED.
- *
- *	TODO: kill this function once old EH is gone.
- *
- *	LOCKING:
- *	Called from timer context
- *
- *	RETURNS:
- *	EH_HANDLED or EH_NOT_HANDLED
- */
-enum blk_eh_timer_return ata_scsi_timed_out(struct scsi_cmnd *cmd)
-{
-	struct Scsi_Host *host = cmd->device->host;
-	struct ata_port *ap = ata_shost_to_port(host);
-	unsigned long flags;
-	struct ata_queued_cmd *qc;
-	enum blk_eh_timer_return ret;
-
-	DPRINTK("ENTER\n");
-
-	if (ap->ops->error_handler) {
-		ret = BLK_EH_NOT_HANDLED;
-		goto out;
-	}
-
-	ret = BLK_EH_HANDLED;
-	spin_lock_irqsave(ap->lock, flags);
-	qc = ata_qc_from_tag(ap, ap->link.active_tag);
-	if (qc) {
-		WARN_ON(qc->scsicmd != cmd);
-		qc->flags |= ATA_QCFLAG_EH_SCHEDULED;
-		qc->err_mask |= AC_ERR_TIMEOUT;
-		ret = BLK_EH_NOT_HANDLED;
-	}
-	spin_unlock_irqrestore(ap->lock, flags);
-
- out:
-	DPRINTK("EXIT, ret=%d\n", ret);
-	return ret;
-}
-EXPORT_SYMBOL(ata_scsi_timed_out);
-
 static void ata_eh_unload(struct ata_port *ap)
 {
 	struct ata_link *link;
@@ -873,9 +822,12 @@ static int ata_eh_nr_in_flight(struct ata_port *ap)
 	int nr = 0;
 
 	/* count only non-internal commands */
-	for (tag = 0; tag < ATA_MAX_QUEUE - 1; tag++)
+	for (tag = 0; tag < ATA_MAX_QUEUE; tag++) {
+		if (ata_tag_internal(tag))
+			continue;
 		if (ata_qc_from_tag(ap, tag))
 			nr++;
+	}
 
 	return nr;
 }
@@ -900,7 +852,7 @@ void ata_eh_fastdrain_timerfn(struct timer_list *t)
 		/* No progress during the last interval, tag all
 		 * in-flight qcs as timed out and freeze the port.
 		 */
-		for (tag = 0; tag < ATA_MAX_QUEUE - 1; tag++) {
+		for (tag = 0; tag < ATA_MAX_QUEUE; tag++) {
 			struct ata_queued_cmd *qc = ata_qc_from_tag(ap, tag);
 			if (qc)
 				qc->err_mask |= AC_ERR_TIMEOUT;
@@ -1054,7 +1006,8 @@ static int ata_do_link_abort(struct ata_port *ap, struct ata_link *link)
 	/* we're gonna abort all commands, no need for fast drain */
 	ata_eh_set_pending(ap, 0);
 
-	for (tag = 0; tag < ATA_MAX_QUEUE; tag++) {
+	/* include internal tag in iteration */
+	for (tag = 0; tag <= ATA_MAX_QUEUE; tag++) {
 		struct ata_queued_cmd *qc = ata_qc_from_tag(ap, tag);
 
 		if (qc && (!link || qc->dev->link == link)) {
@@ -1483,6 +1436,10 @@ static const char *ata_err_string(unsigned int err_mask)
 		return "invalid argument";
 	if (err_mask & AC_ERR_DEV)
 		return "device error";
+	if (err_mask & AC_ERR_NCQ)
+		return "NCQ error";
+	if (err_mask & AC_ERR_NODEV_HINT)
+		return "Polling detection error";
 	return "unknown error";
 }
 
@@ -1866,10 +1823,10 @@ static unsigned int ata_eh_analyze_tf(struct ata_queued_cmd *qc,
 	if (qc->flags & ATA_QCFLAG_SENSE_VALID) {
 		int ret = scsi_check_sense(qc->scsicmd);
 		/*
-		 * SUCCESS here means that the sense code could
+		 * SUCCESS here means that the sense code could be
 		 * evaluated and should be passed to the upper layers
 		 * for correct evaluation.
-		 * FAILED means the sense code could not interpreted
+		 * FAILED means the sense code could not be interpreted
 		 * and the device would need to be reset.
 		 * NEEDS_RETRY and ADD_TO_MLQUEUE means that the
 		 * command would need to be retried.
@@ -2149,6 +2106,21 @@ static inline int ata_eh_worth_retry(struct ata_queued_cmd *qc)
 	return qc->err_mask != AC_ERR_DEV;  /* retry if not dev error */
 }
 
+/**
+ *      ata_eh_quiet - check if we need to be quiet about a command error
+ *      @qc: qc to check
+ *
+ *      Look at the qc flags anbd its scsi command request flags to determine
+ *      if we need to be quiet about the command failure.
+ */
+static inline bool ata_eh_quiet(struct ata_queued_cmd *qc)
+{
+	if (qc->scsicmd &&
+	    qc->scsicmd->request->rq_flags & RQF_QUIET)
+		qc->flags |= ATA_QCFLAG_QUIET;
+	return qc->flags & ATA_QCFLAG_QUIET;
+}
+
 /**
  *	ata_eh_link_autopsy - analyze error and determine recovery action
  *	@link: host link to perform autopsy on
@@ -2166,7 +2138,7 @@ static void ata_eh_link_autopsy(struct ata_link *link)
 	struct ata_eh_context *ehc = &link->eh_context;
 	struct ata_device *dev;
 	unsigned int all_err_mask = 0, eflags = 0;
-	int tag;
+	int tag, nr_failed = 0, nr_quiet = 0;
 	u32 serror;
 	int rc;
 
@@ -2218,12 +2190,16 @@ static void ata_eh_link_autopsy(struct ata_link *link)
 		if (qc->err_mask & ~AC_ERR_OTHER)
 			qc->err_mask &= ~AC_ERR_OTHER;
 
-		/* SENSE_VALID trumps dev/unknown error and revalidation */
+		/*
+		 * SENSE_VALID trumps dev/unknown error and revalidation. Upper
+		 * layers will determine whether the command is worth retrying
+		 * based on the sense data and device class/type. Otherwise,
+		 * determine directly if the command is worth retrying using its
+		 * error mask and flags.
+		 */
 		if (qc->flags & ATA_QCFLAG_SENSE_VALID)
 			qc->err_mask &= ~(AC_ERR_DEV | AC_ERR_OTHER);
-
-		/* determine whether the command is worth retrying */
-		if (ata_eh_worth_retry(qc))
+		else if (ata_eh_worth_retry(qc))
 			qc->flags |= ATA_QCFLAG_RETRY;
 
 		/* accumulate error info */
@@ -2232,8 +2208,17 @@ static void ata_eh_link_autopsy(struct ata_link *link)
 		if (qc->flags & ATA_QCFLAG_IO)
 			eflags |= ATA_EFLAG_IS_IO;
 		trace_ata_eh_link_autopsy_qc(qc);
+
+		/* Count quiet errors */
+		if (ata_eh_quiet(qc))
+			nr_quiet++;
+		nr_failed++;
 	}
 
+	/* If all failed commands requested silence, then be quiet */
+	if (nr_quiet == nr_failed)
+		ehc->i.flags |= ATA_EHI_QUIET;
+
 	/* enforce default EH actions */
 	if (ap->pflags & ATA_PFLAG_FROZEN ||
 	    all_err_mask & (AC_ERR_HSM | AC_ERR_TIMEOUT))
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index 89a9d4a2efc8..6a91d04351d9 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -872,6 +872,9 @@ static struct ata_queued_cmd *ata_scsi_qc_new(struct ata_device *dev,
 
 		qc->sg = scsi_sglist(cmd);
 		qc->n_elem = scsi_sg_count(cmd);
+
+		if (cmd->request->rq_flags & RQF_QUIET)
+			qc->flags |= ATA_QCFLAG_QUIET;
 	} else {
 		cmd->result = (DID_OK << 16) | (QUEUE_FULL << 1);
 		cmd->scsi_done(cmd);
@@ -1316,7 +1319,7 @@ static int ata_scsi_dev_config(struct scsi_device *sdev,
 		int depth;
 
 		depth = min(sdev->host->can_queue, ata_id_queue_depth(dev->id));
-		depth = min(ATA_MAX_QUEUE - 1, depth);
+		depth = min(ATA_MAX_QUEUE, depth);
 		scsi_change_queue_depth(sdev, depth);
 	}
 
@@ -1429,7 +1432,7 @@ int __ata_change_queue_depth(struct ata_port *ap, struct scsi_device *sdev,
 	/* limit and apply queue depth */
 	queue_depth = min(queue_depth, sdev->host->can_queue);
 	queue_depth = min(queue_depth, ata_id_queue_depth(dev->id));
-	queue_depth = min(queue_depth, ATA_MAX_QUEUE - 1);
+	queue_depth = min(queue_depth, ATA_MAX_QUEUE);
 
 	if (sdev->queue_depth == queue_depth)
 		return -EINVAL;
@@ -1895,7 +1898,7 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc)
 	qc->nbytes = n_block * scmd->device->sector_size;
 
 	rc = ata_build_rw_tf(&qc->tf, qc->dev, block, n_block, tf_flags,
-			     qc->tag, class);
+			     qc->hw_tag, class);
 
 	if (likely(rc == 0))
 		return 0;
@@ -3233,7 +3236,7 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc)
 
 	/* For NCQ commands copy the tag value */
 	if (ata_is_ncq(tf->protocol))
-		tf->nsect = qc->tag << 3;
+		tf->nsect = qc->hw_tag << 3;
 
 	/* enforce correct master/slave bit */
 	tf->device = dev->devno ?
@@ -3513,7 +3516,7 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc)
 		tf->protocol = ATA_PROT_NCQ;
 		tf->command = ATA_CMD_FPDMA_SEND;
 		tf->hob_nsect = ATA_SUBCMD_FPDMA_SEND_DSM & 0x1f;
-		tf->nsect = qc->tag << 3;
+		tf->nsect = qc->hw_tag << 3;
 		tf->hob_feature = (size / 512) >> 8;
 		tf->feature = size / 512;
 
@@ -3733,7 +3736,7 @@ static unsigned int ata_scsi_zbc_in_xlat(struct ata_queued_cmd *qc)
 		tf->protocol = ATA_PROT_NCQ;
 		tf->command = ATA_CMD_FPDMA_RECV;
 		tf->hob_nsect = ATA_SUBCMD_FPDMA_RECV_ZAC_MGMT_IN & 0x1f;
-		tf->nsect = qc->tag << 3;
+		tf->nsect = qc->hw_tag << 3;
 		tf->feature = sect & 0xff;
 		tf->hob_feature = (sect >> 8) & 0xff;
 		tf->auxiliary = ATA_SUBCMD_ZAC_MGMT_IN_REPORT_ZONES | (options << 8);
@@ -3812,7 +3815,7 @@ static unsigned int ata_scsi_zbc_out_xlat(struct ata_queued_cmd *qc)
 		tf->protocol = ATA_PROT_NCQ_NODATA;
 		tf->command = ATA_CMD_NCQ_NON_DATA;
 		tf->feature = ATA_SUBCMD_NCQ_NON_DATA_ZAC_MGMT_OUT;
-		tf->nsect = qc->tag << 3;
+		tf->nsect = qc->hw_tag << 3;
 		tf->auxiliary = sa | ((u16)all << 8);
 	} else {
 		tf->protocol = ATA_PROT_NODATA;
@@ -5051,6 +5054,18 @@ int ata_sas_port_init(struct ata_port *ap)
 }
 EXPORT_SYMBOL_GPL(ata_sas_port_init);
 
+int ata_sas_tport_add(struct device *parent, struct ata_port *ap)
+{
+	return ata_tport_add(parent, ap);
+}
+EXPORT_SYMBOL_GPL(ata_sas_tport_add);
+
+void ata_sas_tport_delete(struct ata_port *ap)
+{
+	ata_tport_delete(ap);
+}
+EXPORT_SYMBOL_GPL(ata_sas_tport_delete);
+
 /**
  *	ata_sas_port_destroy - Destroy a SATA port allocated by ata_sas_port_alloc
  *	@ap: SATA port to destroy
@@ -5117,7 +5132,7 @@ int ata_sas_allocate_tag(struct ata_port *ap)
 		tag = tag < max_queue ? tag : 0;
 
 		/* the last tag is reserved for internal command. */
-		if (tag == ATA_TAG_INTERNAL)
+		if (ata_tag_internal(tag))
 			continue;
 
 		if (!test_and_set_bit(tag, &ap->sas_tag_allocated)) {
diff --git a/drivers/ata/libata-zpodd.c b/drivers/ata/libata-zpodd.c
index de4ddd0e8550..b3ed8f9953a8 100644
--- a/drivers/ata/libata-zpodd.c
+++ b/drivers/ata/libata-zpodd.c
@@ -35,7 +35,7 @@ struct zpodd {
 static int eject_tray(struct ata_device *dev)
 {
 	struct ata_taskfile tf;
-	static const char cdb[] = {  GPCMD_START_STOP_UNIT,
+	static const char cdb[ATAPI_CDB_LEN] = {  GPCMD_START_STOP_UNIT,
 		0, 0, 0,
 		0x02,     /* LoEj */
 		0, 0, 0, 0, 0, 0, 0,
diff --git a/drivers/ata/pata_hpt37x.c b/drivers/ata/pata_hpt37x.c
index 3ba843f5cdc0..ef8aaeb0c575 100644
--- a/drivers/ata/pata_hpt37x.c
+++ b/drivers/ata/pata_hpt37x.c
@@ -224,17 +224,14 @@ static int hpt_dma_blacklisted(const struct ata_device *dev, char *modestr,
 			       const char * const list[])
 {
 	unsigned char model_num[ATA_ID_PROD_LEN + 1];
-	int i = 0;
+	int i;
 
 	ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
 
-	while (list[i] != NULL) {
-		if (!strcmp(list[i], model_num)) {
-			pr_warn("%s is not supported for %s\n",
-				modestr, list[i]);
-			return 1;
-		}
-		i++;
+	i = match_string(list, -1, model_num);
+	if (i >= 0) {
+		pr_warn("%s is not supported for %s\n", modestr, list[i]);
+		return 1;
 	}
 	return 0;
 }
diff --git a/drivers/ata/sata_dwc_460ex.c b/drivers/ata/sata_dwc_460ex.c
index ce3d6674ef80..6f142aa54f5f 100644
--- a/drivers/ata/sata_dwc_460ex.c
+++ b/drivers/ata/sata_dwc_460ex.c
@@ -761,7 +761,7 @@ static void sata_dwc_dma_xfer_complete(struct ata_port *ap, u32 check_status)
 	if (tag > 0) {
 		dev_info(ap->dev,
 			 "%s tag=%u cmd=0x%02x dma dir=%s proto=%s dmacr=0x%08x\n",
-			 __func__, qc->tag, qc->tf.command,
+			 __func__, qc->hw_tag, qc->tf.command,
 			 get_dma_dir_descript(qc->dma_dir),
 			 get_prot_descript(qc->tf.protocol),
 			 sata_dwc_readl(&hsdev->sata_dwc_regs->dmacr));
@@ -789,7 +789,7 @@ static int sata_dwc_qc_complete(struct ata_port *ap, struct ata_queued_cmd *qc,
 {
 	u8 status = 0;
 	u32 mask = 0x0;
-	u8 tag = qc->tag;
+	u8 tag = qc->hw_tag;
 	struct sata_dwc_device *hsdev = HSDEV_FROM_AP(ap);
 	struct sata_dwc_device_port *hsdevp = HSDEVP_FROM_AP(ap);
 	hsdev->sactive_queued = 0;
@@ -997,7 +997,7 @@ static void sata_dwc_bmdma_setup_by_tag(struct ata_queued_cmd *qc, u8 tag)
 
 static void sata_dwc_bmdma_setup(struct ata_queued_cmd *qc)
 {
-	u8 tag = qc->tag;
+	u8 tag = qc->hw_tag;
 
 	if (ata_is_ncq(qc->tf.protocol)) {
 		dev_dbg(qc->ap->dev, "%s: ap->link.sactive=0x%08x tag=%d\n",
@@ -1059,7 +1059,7 @@ static void sata_dwc_bmdma_start_by_tag(struct ata_queued_cmd *qc, u8 tag)
 
 static void sata_dwc_bmdma_start(struct ata_queued_cmd *qc)
 {
-	u8 tag = qc->tag;
+	u8 tag = qc->hw_tag;
 
 	if (ata_is_ncq(qc->tf.protocol)) {
 		dev_dbg(qc->ap->dev, "%s: ap->link.sactive=0x%08x tag=%d\n",
@@ -1074,17 +1074,17 @@ static void sata_dwc_bmdma_start(struct ata_queued_cmd *qc)
 static unsigned int sata_dwc_qc_issue(struct ata_queued_cmd *qc)
 {
 	u32 sactive;
-	u8 tag = qc->tag;
+	u8 tag = qc->hw_tag;
 	struct ata_port *ap = qc->ap;
 	struct sata_dwc_device_port *hsdevp = HSDEVP_FROM_AP(ap);
 
 #ifdef DEBUG_NCQ
-	if (qc->tag > 0 || ap->link.sactive > 1)
+	if (qc->hw_tag > 0 || ap->link.sactive > 1)
 		dev_info(ap->dev,
 			 "%s ap id=%d cmd(0x%02x)=%s qc tag=%d prot=%s ap active_tag=0x%08x ap sactive=0x%08x\n",
 			 __func__, ap->print_id, qc->tf.command,
 			 ata_get_cmd_descript(qc->tf.command),
-			 qc->tag, get_prot_descript(qc->tf.protocol),
+			 qc->hw_tag, get_prot_descript(qc->tf.protocol),
 			 ap->link.active_tag, ap->link.sactive);
 #endif
 
diff --git a/drivers/ata/sata_fsl.c b/drivers/ata/sata_fsl.c
index 95bf3abda6f6..b8d9cfc60374 100644
--- a/drivers/ata/sata_fsl.c
+++ b/drivers/ata/sata_fsl.c
@@ -519,7 +519,7 @@ static void sata_fsl_qc_prep(struct ata_queued_cmd *qc)
 	struct sata_fsl_port_priv *pp = ap->private_data;
 	struct sata_fsl_host_priv *host_priv = ap->host->private_data;
 	void __iomem *hcr_base = host_priv->hcr_base;
-	unsigned int tag = sata_fsl_tag(qc->tag, hcr_base);
+	unsigned int tag = sata_fsl_tag(qc->hw_tag, hcr_base);
 	struct command_desc *cd;
 	u32 desc_info = CMD_DESC_RES | CMD_DESC_SNOOP_ENABLE;
 	u32 num_prde = 0;
@@ -566,7 +566,7 @@ static unsigned int sata_fsl_qc_issue(struct ata_queued_cmd *qc)
 	struct ata_port *ap = qc->ap;
 	struct sata_fsl_host_priv *host_priv = ap->host->private_data;
 	void __iomem *hcr_base = host_priv->hcr_base;
-	unsigned int tag = sata_fsl_tag(qc->tag, hcr_base);
+	unsigned int tag = sata_fsl_tag(qc->hw_tag, hcr_base);
 
 	VPRINTK("xx_qc_issue called,CQ=0x%x,CA=0x%x,CE=0x%x,CC=0x%x\n",
 		ioread32(CQ + hcr_base),
@@ -595,7 +595,7 @@ static bool sata_fsl_qc_fill_rtf(struct ata_queued_cmd *qc)
 	struct sata_fsl_port_priv *pp = qc->ap->private_data;
 	struct sata_fsl_host_priv *host_priv = qc->ap->host->private_data;
 	void __iomem *hcr_base = host_priv->hcr_base;
-	unsigned int tag = sata_fsl_tag(qc->tag, hcr_base);
+	unsigned int tag = sata_fsl_tag(qc->hw_tag, hcr_base);
 	struct command_desc *cd;
 
 	cd = pp->cmdentry + tag;
@@ -1266,7 +1266,7 @@ static void sata_fsl_host_intr(struct ata_port *ap)
 	}
 
 	VPRINTK("Status of all queues :\n");
-	VPRINTK("done_mask/CC = 0x%x, CA = 0x%x, CE=0x%x,CQ=0x%x,apqa=0x%x\n",
+	VPRINTK("done_mask/CC = 0x%x, CA = 0x%x, CE=0x%x,CQ=0x%x,apqa=0x%llx\n",
 		done_mask,
 		ioread32(hcr_base + CA),
 		ioread32(hcr_base + CE),
@@ -1293,7 +1293,7 @@ static void sata_fsl_host_intr(struct ata_port *ap)
 		ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask);
 		return;
 
-	} else if ((ap->qc_active & (1 << ATA_TAG_INTERNAL))) {
+	} else if ((ap->qc_active & (1ULL << ATA_TAG_INTERNAL))) {
 		iowrite32(1, hcr_base + CC);
 		qc = ata_qc_from_tag(ap, ATA_TAG_INTERNAL);
 
diff --git a/drivers/ata/sata_mv.c b/drivers/ata/sata_mv.c
index 42d4589b43d4..cddf96f6e431 100644
--- a/drivers/ata/sata_mv.c
+++ b/drivers/ata/sata_mv.c
@@ -1802,7 +1802,7 @@ static void mv_fill_sg(struct ata_queued_cmd *qc)
 	struct mv_sg *mv_sg, *last_sg = NULL;
 	unsigned int si;
 
-	mv_sg = pp->sg_tbl[qc->tag];
+	mv_sg = pp->sg_tbl[qc->hw_tag];
 	for_each_sg(qc->sg, sg, qc->n_elem, si) {
 		dma_addr_t addr = sg_dma_address(sg);
 		u32 sg_len = sg_dma_len(sg);
@@ -1903,9 +1903,9 @@ static void mv_bmdma_setup(struct ata_queued_cmd *qc)
 	writel(0, port_mmio + BMDMA_CMD);
 
 	/* load PRD table addr. */
-	writel((pp->sg_tbl_dma[qc->tag] >> 16) >> 16,
+	writel((pp->sg_tbl_dma[qc->hw_tag] >> 16) >> 16,
 		port_mmio + BMDMA_PRD_HIGH);
-	writelfl(pp->sg_tbl_dma[qc->tag],
+	writelfl(pp->sg_tbl_dma[qc->hw_tag],
 		port_mmio + BMDMA_PRD_LOW);
 
 	/* issue r/w command */
@@ -2071,17 +2071,17 @@ static void mv_qc_prep(struct ata_queued_cmd *qc)
 	 */
 	if (!(tf->flags & ATA_TFLAG_WRITE))
 		flags |= CRQB_FLAG_READ;
-	WARN_ON(MV_MAX_Q_DEPTH <= qc->tag);
-	flags |= qc->tag << CRQB_TAG_SHIFT;
+	WARN_ON(MV_MAX_Q_DEPTH <= qc->hw_tag);
+	flags |= qc->hw_tag << CRQB_TAG_SHIFT;
 	flags |= (qc->dev->link->pmp & 0xf) << CRQB_PMP_SHIFT;
 
 	/* get current queue index from software */
 	in_index = pp->req_idx;
 
 	pp->crqb[in_index].sg_addr =
-		cpu_to_le32(pp->sg_tbl_dma[qc->tag] & 0xffffffff);
+		cpu_to_le32(pp->sg_tbl_dma[qc->hw_tag] & 0xffffffff);
 	pp->crqb[in_index].sg_addr_hi =
-		cpu_to_le32((pp->sg_tbl_dma[qc->tag] >> 16) >> 16);
+		cpu_to_le32((pp->sg_tbl_dma[qc->hw_tag] >> 16) >> 16);
 	pp->crqb[in_index].ctrl_flags = cpu_to_le16(flags);
 
 	cw = &pp->crqb[in_index].ata_cmd[0];
@@ -2164,17 +2164,17 @@ static void mv_qc_prep_iie(struct ata_queued_cmd *qc)
 	if (!(tf->flags & ATA_TFLAG_WRITE))
 		flags |= CRQB_FLAG_READ;
 
-	WARN_ON(MV_MAX_Q_DEPTH <= qc->tag);
-	flags |= qc->tag << CRQB_TAG_SHIFT;
-	flags |= qc->tag << CRQB_HOSTQ_SHIFT;
+	WARN_ON(MV_MAX_Q_DEPTH <= qc->hw_tag);
+	flags |= qc->hw_tag << CRQB_TAG_SHIFT;
+	flags |= qc->hw_tag << CRQB_HOSTQ_SHIFT;
 	flags |= (qc->dev->link->pmp & 0xf) << CRQB_PMP_SHIFT;
 
 	/* get current queue index from software */
 	in_index = pp->req_idx;
 
 	crqb = (struct mv_crqb_iie *) &pp->crqb[in_index];
-	crqb->addr = cpu_to_le32(pp->sg_tbl_dma[qc->tag] & 0xffffffff);
-	crqb->addr_hi = cpu_to_le32((pp->sg_tbl_dma[qc->tag] >> 16) >> 16);
+	crqb->addr = cpu_to_le32(pp->sg_tbl_dma[qc->hw_tag] & 0xffffffff);
+	crqb->addr_hi = cpu_to_le32((pp->sg_tbl_dma[qc->hw_tag] >> 16) >> 16);
 	crqb->flags = cpu_to_le32(flags);
 
 	crqb->ata_cmd[0] = cpu_to_le32(
@@ -2539,7 +2539,7 @@ static int mv_handle_fbs_ncq_dev_err(struct ata_port *ap)
 	failed_links = hweight16(new_map);
 
 	ata_port_info(ap,
-		      "%s: pmp_map=%04x qc_map=%04x failed_links=%d nr_active_links=%d\n",
+		      "%s: pmp_map=%04x qc_map=%04llx failed_links=%d nr_active_links=%d\n",
 		      __func__, pp->delayed_eh_pmp_map,
 		      ap->qc_active, failed_links,
 		      ap->nr_active_links);
diff --git a/drivers/ata/sata_nv.c b/drivers/ata/sata_nv.c
index 8c683ddd0f58..10ae11aa1926 100644
--- a/drivers/ata/sata_nv.c
+++ b/drivers/ata/sata_nv.c
@@ -400,7 +400,7 @@ static struct scsi_host_template nv_adma_sht = {
 
 static struct scsi_host_template nv_swncq_sht = {
 	ATA_NCQ_SHT(DRV_NAME),
-	.can_queue		= ATA_MAX_QUEUE,
+	.can_queue		= ATA_MAX_QUEUE - 1,
 	.sg_tablesize		= LIBATA_MAX_PRD,
 	.dma_boundary		= ATA_DMA_BOUNDARY,
 	.slave_configure	= nv_swncq_slave_config,
@@ -740,32 +740,16 @@ static int nv_adma_slave_config(struct scsi_device *sdev)
 	sdev1 = ap->host->ports[1]->link.device[0].sdev;
 	if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
 	    (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
-		/** We have to set the DMA mask to 32-bit if either port is in
-		    ATAPI mode, since they are on the same PCI device which is
-		    used for DMA mapping. If we set the mask we also need to set
-		    the bounce limit on both ports to ensure that the block
-		    layer doesn't feed addresses that cause DMA mapping to
-		    choke. If either SCSI device is not allocated yet, it's OK
-		    since that port will discover its correct setting when it
-		    does get allocated.
-		    Note: Setting 32-bit mask should not fail. */
-		if (sdev0)
-			blk_queue_bounce_limit(sdev0->request_queue,
-					       ATA_DMA_MASK);
-		if (sdev1)
-			blk_queue_bounce_limit(sdev1->request_queue,
-					       ATA_DMA_MASK);
-
-		dma_set_mask(&pdev->dev, ATA_DMA_MASK);
+		/*
+		 * We have to set the DMA mask to 32-bit if either port is in
+		 * ATAPI mode, since they are on the same PCI device which is
+		 * used for DMA mapping.  If either SCSI device is not allocated
+		 * yet, it's OK since that port will discover its correct
+		 * setting when it does get allocated.
+		 */
+		rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
 	} else {
-		/** This shouldn't fail as it was set to this value before */
-		dma_set_mask(&pdev->dev, pp->adma_dma_mask);
-		if (sdev0)
-			blk_queue_bounce_limit(sdev0->request_queue,
-					       pp->adma_dma_mask);
-		if (sdev1)
-			blk_queue_bounce_limit(sdev1->request_queue,
-					       pp->adma_dma_mask);
+		rc = dma_set_mask(&pdev->dev, pp->adma_dma_mask);
 	}
 
 	blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
@@ -1131,12 +1115,11 @@ static int nv_adma_port_start(struct ata_port *ap)
 
 	VPRINTK("ENTER\n");
 
-	/* Ensure DMA mask is set to 32-bit before allocating legacy PRD and
-	   pad buffers */
-	rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
-	if (rc)
-		return rc;
-	rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
+	/*
+	 * Ensure DMA mask is set to 32-bit before allocating legacy PRD and
+	 * pad buffers.
+	 */
+	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 	if (rc)
 		return rc;
 
@@ -1156,13 +1139,16 @@ static int nv_adma_port_start(struct ata_port *ap)
 	pp->notifier_clear_block = pp->gen_block +
 	       NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
 
-	/* Now that the legacy PRD and padding buffer are allocated we can
-	   safely raise the DMA mask to allocate the CPB/APRD table.
-	   These are allowed to fail since we store the value that ends up
-	   being used to set as the bounce limit in slave_config later if
-	   needed. */
-	dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
-	dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
+	/*
+	 * Now that the legacy PRD and padding buffer are allocated we can
+	 * try to raise the DMA mask to allocate the CPB/APRD table.
+	 */
+	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (rc) {
+		rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+		if (rc)
+			return rc;
+	}
 	pp->adma_dma_mask = *dev->dma_mask;
 
 	mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
@@ -1356,11 +1342,11 @@ static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
 
 	for_each_sg(qc->sg, sg, qc->n_elem, si) {
 		aprd = (si < 5) ? &cpb->aprd[si] :
-			       &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];
+			&pp->aprd[NV_ADMA_SGTBL_LEN * qc->hw_tag + (si-5)];
 		nv_adma_fill_aprd(qc, sg, si, aprd);
 	}
 	if (si > 5)
-		cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
+		cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->hw_tag)));
 	else
 		cpb->next_aprd = cpu_to_le64(0);
 }
@@ -1385,7 +1371,7 @@ static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
 static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
 {
 	struct nv_adma_port_priv *pp = qc->ap->private_data;
-	struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
+	struct nv_adma_cpb *cpb = &pp->cpb[qc->hw_tag];
 	u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
 		       NV_CPB_CTL_IEN;
 
@@ -1403,7 +1389,7 @@ static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
 	wmb();
 
 	cpb->len		= 3;
-	cpb->tag		= qc->tag;
+	cpb->tag		= qc->hw_tag;
 	cpb->next_cpb_idx	= 0;
 
 	/* turn on NCQ flags for NCQ commands */
@@ -1466,9 +1452,9 @@ static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
 		pp->last_issue_ncq = curr_ncq;
 	}
 
-	writew(qc->tag, mmio + NV_ADMA_APPEND);
+	writew(qc->hw_tag, mmio + NV_ADMA_APPEND);
 
-	DPRINTK("Issued tag %u\n", qc->tag);
+	DPRINTK("Issued tag %u\n", qc->hw_tag);
 
 	return 0;
 }
@@ -1730,8 +1716,8 @@ static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
 
 	/* queue is full */
 	WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
-	dq->defer_bits |= (1 << qc->tag);
-	dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
+	dq->defer_bits |= (1 << qc->hw_tag);
+	dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->hw_tag;
 }
 
 static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
@@ -1796,7 +1782,7 @@ static void nv_swncq_ncq_stop(struct ata_port *ap)
 	u32 sactive;
 	u32 done_mask;
 
-	ata_port_err(ap, "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
+	ata_port_err(ap, "EH in SWNCQ mode,QC:qc_active 0x%llX sactive 0x%X\n",
 		     ap->qc_active, ap->link.sactive);
 	ata_port_err(ap,
 		"SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n  "
@@ -2010,7 +1996,7 @@ static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
 	struct ata_bmdma_prd *prd;
 	unsigned int si, idx;
 
-	prd = pp->prd + ATA_MAX_PRD * qc->tag;
+	prd = pp->prd + ATA_MAX_PRD * qc->hw_tag;
 
 	idx = 0;
 	for_each_sg(qc->sg, sg, qc->n_elem, si) {
@@ -2048,16 +2034,16 @@ static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
 
 	DPRINTK("Enter\n");
 
-	writel((1 << qc->tag), pp->sactive_block);
-	pp->last_issue_tag = qc->tag;
-	pp->dhfis_bits &= ~(1 << qc->tag);
-	pp->dmafis_bits &= ~(1 << qc->tag);
-	pp->qc_active |= (0x1 << qc->tag);
+	writel((1 << qc->hw_tag), pp->sactive_block);
+	pp->last_issue_tag = qc->hw_tag;
+	pp->dhfis_bits &= ~(1 << qc->hw_tag);
+	pp->dmafis_bits &= ~(1 << qc->hw_tag);
+	pp->qc_active |= (0x1 << qc->hw_tag);
 
 	ap->ops->sff_tf_load(ap, &qc->tf);	 /* load tf registers */
 	ap->ops->sff_exec_command(ap, &qc->tf);
 
-	DPRINTK("Issued tag %u\n", qc->tag);
+	DPRINTK("Issued tag %u\n", qc->hw_tag);
 
 	return 0;
 }
@@ -2207,7 +2193,7 @@ static void nv_swncq_dmafis(struct ata_port *ap)
 	rw = qc->tf.flags & ATA_TFLAG_WRITE;
 
 	/* load PRD table addr. */
-	iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
+	iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->hw_tag,
 		  ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
 
 	/* specify data direction, triple-check start bit is clear */
diff --git a/drivers/ata/sata_sil24.c b/drivers/ata/sata_sil24.c
index 010ca101d412..319f517137cd 100644
--- a/drivers/ata/sata_sil24.c
+++ b/drivers/ata/sata_sil24.c
@@ -849,7 +849,7 @@ static void sil24_qc_prep(struct ata_queued_cmd *qc)
 	struct sil24_sge *sge;
 	u16 ctrl = 0;
 
-	cb = &pp->cmd_block[sil24_tag(qc->tag)];
+	cb = &pp->cmd_block[sil24_tag(qc->hw_tag)];
 
 	if (!ata_is_atapi(qc->tf.protocol)) {
 		prb = &cb->ata.prb;
@@ -891,7 +891,7 @@ static unsigned int sil24_qc_issue(struct ata_queued_cmd *qc)
 	struct ata_port *ap = qc->ap;
 	struct sil24_port_priv *pp = ap->private_data;
 	void __iomem *port = sil24_port_base(ap);
-	unsigned int tag = sil24_tag(qc->tag);
+	unsigned int tag = sil24_tag(qc->hw_tag);
 	dma_addr_t paddr;
 	void __iomem *activate;
 
@@ -911,7 +911,7 @@ static unsigned int sil24_qc_issue(struct ata_queued_cmd *qc)
 
 static bool sil24_qc_fill_rtf(struct ata_queued_cmd *qc)
 {
-	sil24_read_tf(qc->ap, qc->tag, &qc->result_tf);
+	sil24_read_tf(qc->ap, qc->hw_tag, &qc->result_tf);
 	return true;
 }
 
diff --git a/drivers/atm/zatm.c b/drivers/atm/zatm.c
index 9c9a22958717..a8d2eb0ceb8d 100644
--- a/drivers/atm/zatm.c
+++ b/drivers/atm/zatm.c
@@ -1151,8 +1151,8 @@ static void eprom_get_byte(struct zatm_dev *zatm_dev, unsigned char *byte,
 }
 
 
-static unsigned char eprom_try_esi(struct atm_dev *dev, unsigned short cmd,
-				   int offset, int swap)
+static int eprom_try_esi(struct atm_dev *dev, unsigned short cmd, int offset,
+			 int swap)
 {
 	unsigned char buf[ZEPROM_SIZE];
 	struct zatm_dev *zatm_dev;
diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
index 29b0eb452b3a..3e63a900b330 100644
--- a/drivers/base/Kconfig
+++ b/drivers/base/Kconfig
@@ -70,81 +70,25 @@ config STANDALONE
 	  If unsure, say Y.
 
 config PREVENT_FIRMWARE_BUILD
-	bool "Prevent firmware from being built"
+	bool "Disable drivers features which enable custom firmware building"
 	default y
 	help
-	  Say yes to avoid building firmware. Firmware is usually shipped
-	  with the driver and only when updating the firmware should a
-	  rebuild be made.
-	  If unsure, say Y here.
-
-config FW_LOADER
-	tristate "Userspace firmware loading support" if EXPERT
-	default y
-	---help---
-	  This option is provided for the case where none of the in-tree modules
-	  require userspace firmware loading support, but a module built
-	  out-of-tree does.
-
-config EXTRA_FIRMWARE
-	string "External firmware blobs to build into the kernel binary"
-	depends on FW_LOADER
-	help
-	  Various drivers in the kernel source tree may require firmware,
-	  which is generally available in your distribution's linux-firmware
-	  package.
-
-	  The linux-firmware package should install firmware into
-	  /lib/firmware/ on your system, so they can be loaded by userspace
-	  helpers on request.
-
-	  This option allows firmware to be built into the kernel for the case
-	  where the user either cannot or doesn't want to provide it from
-	  userspace at runtime (for example, when the firmware in question is
-	  required for accessing the boot device, and the user doesn't want to
-	  use an initrd).
-
-	  This option is a string and takes the (space-separated) names of the
-	  firmware files -- the same names that appear in MODULE_FIRMWARE()
-	  and request_firmware() in the source. These files should exist under
-	  the directory specified by the EXTRA_FIRMWARE_DIR option, which is
-	  /lib/firmware by default.
-
-	  For example, you might set CONFIG_EXTRA_FIRMWARE="usb8388.bin", copy
-	  the usb8388.bin file into /lib/firmware, and build the kernel. Then
-	  any request_firmware("usb8388.bin") will be satisfied internally
-	  without needing to call out to userspace.
-
-	  WARNING: If you include additional firmware files into your binary
-	  kernel image that are not available under the terms of the GPL,
-	  then it may be a violation of the GPL to distribute the resulting
-	  image since it combines both GPL and non-GPL work. You should
-	  consult a lawyer of your own before distributing such an image.
-
-config EXTRA_FIRMWARE_DIR
-	string "Firmware blobs root directory"
-	depends on EXTRA_FIRMWARE != ""
-	default "/lib/firmware"
-	help
-	  This option controls the directory in which the kernel build system
-	  looks for the firmware files listed in the EXTRA_FIRMWARE option.
-
-config FW_LOADER_USER_HELPER
-	bool
-
-config FW_LOADER_USER_HELPER_FALLBACK
-	bool "Fallback user-helper invocation for firmware loading"
-	depends on FW_LOADER
-	select FW_LOADER_USER_HELPER
-	help
-	  This option enables / disables the invocation of user-helper
-	  (e.g. udev) for loading firmware files as a fallback after the
-	  direct file loading in kernel fails.  The user-mode helper is
-	  no longer required unless you have a special firmware file that
-	  resides in a non-standard path. Moreover, the udev support has
-	  been deprecated upstream.
-
-	  If you are unsure about this, say N here.
+	  Say yes to disable driver features which enable building a custom
+	  driver firmware at kernel build time. These drivers do not use the
+	  kernel firmware API to load firmware (CONFIG_FW_LOADER), instead they
+	  use their own custom loading mechanism. The required firmware is
+	  usually shipped with the driver, building the driver firmware
+	  should only be needed if you have an updated firmware source.
+
+	  Firmware should not be being built as part of kernel, these days
+	  you should always prevent this and say Y here. There are only two
+	  old drivers which enable building of its firmware at kernel build
+	  time:
+
+	    o CONFIG_WANXL through CONFIG_WANXL_BUILD_FIRMWARE
+	    o CONFIG_SCSI_AIC79XX through CONFIG_AIC79XX_BUILD_FIRMWARE
+
+source "drivers/base/firmware_loader/Kconfig"
 
 config WANT_DEV_COREDUMP
 	bool
diff --git a/drivers/base/base.h b/drivers/base/base.h
index d800de650fa5..a75c3025fb78 100644
--- a/drivers/base/base.h
+++ b/drivers/base/base.h
@@ -161,3 +161,6 @@ extern void device_links_driver_cleanup(struct device *dev);
 extern void device_links_no_driver(struct device *dev);
 extern bool device_links_busy(struct device *dev);
 extern void device_links_unbind_consumers(struct device *dev);
+
+/* device pm support */
+void device_pm_move_to_tail(struct device *dev);
diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c
index edf726267282..2880e2ab01f5 100644
--- a/drivers/base/cacheinfo.c
+++ b/drivers/base/cacheinfo.c
@@ -32,50 +32,10 @@ struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu)
 }
 
 #ifdef CONFIG_OF
-static int cache_setup_of_node(unsigned int cpu)
-{
-	struct device_node *np;
-	struct cacheinfo *this_leaf;
-	struct device *cpu_dev = get_cpu_device(cpu);
-	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
-	unsigned int index = 0;
-
-	/* skip if of_node is already populated */
-	if (this_cpu_ci->info_list->of_node)
-		return 0;
-
-	if (!cpu_dev) {
-		pr_err("No cpu device for CPU %d\n", cpu);
-		return -ENODEV;
-	}
-	np = cpu_dev->of_node;
-	if (!np) {
-		pr_err("Failed to find cpu%d device node\n", cpu);
-		return -ENOENT;
-	}
-
-	while (index < cache_leaves(cpu)) {
-		this_leaf = this_cpu_ci->info_list + index;
-		if (this_leaf->level != 1)
-			np = of_find_next_cache_node(np);
-		else
-			np = of_node_get(np);/* cpu node itself */
-		if (!np)
-			break;
-		this_leaf->of_node = np;
-		index++;
-	}
-
-	if (index != cache_leaves(cpu)) /* not all OF nodes populated */
-		return -ENOENT;
-
-	return 0;
-}
-
 static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
 					   struct cacheinfo *sib_leaf)
 {
-	return sib_leaf->of_node == this_leaf->of_node;
+	return sib_leaf->fw_token == this_leaf->fw_token;
 }
 
 /* OF properties to query for a given cache type */
@@ -111,7 +71,7 @@ static inline int get_cacheinfo_idx(enum cache_type type)
 	return type;
 }
 
-static void cache_size(struct cacheinfo *this_leaf)
+static void cache_size(struct cacheinfo *this_leaf, struct device_node *np)
 {
 	const char *propname;
 	const __be32 *cache_size;
@@ -120,13 +80,14 @@ static void cache_size(struct cacheinfo *this_leaf)
 	ct_idx = get_cacheinfo_idx(this_leaf->type);
 	propname = cache_type_info[ct_idx].size_prop;
 
-	cache_size = of_get_property(this_leaf->of_node, propname, NULL);
+	cache_size = of_get_property(np, propname, NULL);
 	if (cache_size)
 		this_leaf->size = of_read_number(cache_size, 1);
 }
 
 /* not cache_line_size() because that's a macro in include/linux/cache.h */
-static void cache_get_line_size(struct cacheinfo *this_leaf)
+static void cache_get_line_size(struct cacheinfo *this_leaf,
+				struct device_node *np)
 {
 	const __be32 *line_size;
 	int i, lim, ct_idx;
@@ -138,7 +99,7 @@ static void cache_get_line_size(struct cacheinfo *this_leaf)
 		const char *propname;
 
 		propname = cache_type_info[ct_idx].line_size_props[i];
-		line_size = of_get_property(this_leaf->of_node, propname, NULL);
+		line_size = of_get_property(np, propname, NULL);
 		if (line_size)
 			break;
 	}
@@ -147,7 +108,7 @@ static void cache_get_line_size(struct cacheinfo *this_leaf)
 		this_leaf->coherency_line_size = of_read_number(line_size, 1);
 }
 
-static void cache_nr_sets(struct cacheinfo *this_leaf)
+static void cache_nr_sets(struct cacheinfo *this_leaf, struct device_node *np)
 {
 	const char *propname;
 	const __be32 *nr_sets;
@@ -156,7 +117,7 @@ static void cache_nr_sets(struct cacheinfo *this_leaf)
 	ct_idx = get_cacheinfo_idx(this_leaf->type);
 	propname = cache_type_info[ct_idx].nr_sets_prop;
 
-	nr_sets = of_get_property(this_leaf->of_node, propname, NULL);
+	nr_sets = of_get_property(np, propname, NULL);
 	if (nr_sets)
 		this_leaf->number_of_sets = of_read_number(nr_sets, 1);
 }
@@ -175,41 +136,77 @@ static void cache_associativity(struct cacheinfo *this_leaf)
 		this_leaf->ways_of_associativity = (size / nr_sets) / line_size;
 }
 
-static bool cache_node_is_unified(struct cacheinfo *this_leaf)
+static bool cache_node_is_unified(struct cacheinfo *this_leaf,
+				  struct device_node *np)
 {
-	return of_property_read_bool(this_leaf->of_node, "cache-unified");
+	return of_property_read_bool(np, "cache-unified");
 }
 
-static void cache_of_override_properties(unsigned int cpu)
+static void cache_of_set_props(struct cacheinfo *this_leaf,
+			       struct device_node *np)
 {
-	int index;
+	/*
+	 * init_cache_level must setup the cache level correctly
+	 * overriding the architecturally specified levels, so
+	 * if type is NONE at this stage, it should be unified
+	 */
+	if (this_leaf->type == CACHE_TYPE_NOCACHE &&
+	    cache_node_is_unified(this_leaf, np))
+		this_leaf->type = CACHE_TYPE_UNIFIED;
+	cache_size(this_leaf, np);
+	cache_get_line_size(this_leaf, np);
+	cache_nr_sets(this_leaf, np);
+	cache_associativity(this_leaf);
+}
+
+static int cache_setup_of_node(unsigned int cpu)
+{
+	struct device_node *np;
 	struct cacheinfo *this_leaf;
+	struct device *cpu_dev = get_cpu_device(cpu);
 	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
+	unsigned int index = 0;
 
-	for (index = 0; index < cache_leaves(cpu); index++) {
+	/* skip if fw_token is already populated */
+	if (this_cpu_ci->info_list->fw_token) {
+		return 0;
+	}
+
+	if (!cpu_dev) {
+		pr_err("No cpu device for CPU %d\n", cpu);
+		return -ENODEV;
+	}
+	np = cpu_dev->of_node;
+	if (!np) {
+		pr_err("Failed to find cpu%d device node\n", cpu);
+		return -ENOENT;
+	}
+
+	while (index < cache_leaves(cpu)) {
 		this_leaf = this_cpu_ci->info_list + index;
-		/*
-		 * init_cache_level must setup the cache level correctly
-		 * overriding the architecturally specified levels, so
-		 * if type is NONE at this stage, it should be unified
-		 */
-		if (this_leaf->type == CACHE_TYPE_NOCACHE &&
-		    cache_node_is_unified(this_leaf))
-			this_leaf->type = CACHE_TYPE_UNIFIED;
-		cache_size(this_leaf);
-		cache_get_line_size(this_leaf);
-		cache_nr_sets(this_leaf);
-		cache_associativity(this_leaf);
+		if (this_leaf->level != 1)
+			np = of_find_next_cache_node(np);
+		else
+			np = of_node_get(np);/* cpu node itself */
+		if (!np)
+			break;
+		cache_of_set_props(this_leaf, np);
+		this_leaf->fw_token = np;
+		index++;
 	}
+
+	if (index != cache_leaves(cpu)) /* not all OF nodes populated */
+		return -ENOENT;
+
+	return 0;
 }
 #else
-static void cache_of_override_properties(unsigned int cpu) { }
 static inline int cache_setup_of_node(unsigned int cpu) { return 0; }
 static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
 					   struct cacheinfo *sib_leaf)
 {
 	/*
-	 * For non-DT systems, assume unique level 1 cache, system-wide
+	 * For non-DT/ACPI systems, assume unique level 1 caches, system-wide
 	 * shared caches for all other levels. This will be used only if
 	 * arch specific code has not populated shared_cpu_map
 	 */
@@ -217,6 +214,11 @@ static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
 }
 #endif
 
+int __weak cache_setup_acpi(unsigned int cpu)
+{
+	return -ENOTSUPP;
+}
+
 static int cache_shared_cpu_map_setup(unsigned int cpu)
 {
 	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
@@ -230,8 +232,8 @@ static int cache_shared_cpu_map_setup(unsigned int cpu)
 	if (of_have_populated_dt())
 		ret = cache_setup_of_node(cpu);
 	else if (!acpi_disabled)
-		/* No cache property/hierarchy support yet in ACPI */
-		ret = -ENOTSUPP;
+		ret = cache_setup_acpi(cpu);
+
 	if (ret)
 		return ret;
 
@@ -282,16 +284,11 @@ static void cache_shared_cpu_map_remove(unsigned int cpu)
 			cpumask_clear_cpu(cpu, &sib_leaf->shared_cpu_map);
 			cpumask_clear_cpu(sibling, &this_leaf->shared_cpu_map);
 		}
-		of_node_put(this_leaf->of_node);
+		if (of_have_populated_dt())
+			of_node_put(this_leaf->fw_token);
 	}
 }
 
-static void cache_override_properties(unsigned int cpu)
-{
-	if (of_have_populated_dt())
-		return cache_of_override_properties(cpu);
-}
-
 static void free_cache_attributes(unsigned int cpu)
 {
 	if (!per_cpu_cacheinfo(cpu))
@@ -325,12 +322,17 @@ static int detect_cache_attributes(unsigned int cpu)
 	if (per_cpu_cacheinfo(cpu) == NULL)
 		return -ENOMEM;
 
+	/*
+	 * populate_cache_leaves() may completely setup the cache leaves and
+	 * shared_cpu_map or it may leave it partially setup.
+	 */
 	ret = populate_cache_leaves(cpu);
 	if (ret)
 		goto free_ci;
 	/*
-	 * For systems using DT for cache hierarchy, of_node and shared_cpu_map
-	 * will be set up here only if they are not populated already
+	 * For systems using DT for cache hierarchy, fw_token
+	 * and shared_cpu_map will be set up here only if they are
+	 * not populated already
 	 */
 	ret = cache_shared_cpu_map_setup(cpu);
 	if (ret) {
@@ -338,7 +340,6 @@ static int detect_cache_attributes(unsigned int cpu)
 		goto free_ci;
 	}
 
-	cache_override_properties(cpu);
 	return 0;
 
 free_ci:
diff --git a/drivers/base/core.c b/drivers/base/core.c
index b610816eb887..36622b52e419 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -144,6 +144,26 @@ static int device_reorder_to_tail(struct device *dev, void *not_used)
 	return 0;
 }
 
+/**
+ * device_pm_move_to_tail - Move set of devices to the end of device lists
+ * @dev: Device to move
+ *
+ * This is a device_reorder_to_tail() wrapper taking the requisite locks.
+ *
+ * It moves the @dev along with all of its children and all of its consumers
+ * to the ends of the device_kset and dpm_list, recursively.
+ */
+void device_pm_move_to_tail(struct device *dev)
+{
+	int idx;
+
+	idx = device_links_read_lock();
+	device_pm_lock();
+	device_reorder_to_tail(dev, NULL);
+	device_pm_unlock();
+	device_links_read_unlock(idx);
+}
+
 /**
  * device_link_add - Create a link between two devices.
  * @consumer: Consumer end of the link.
@@ -1467,7 +1487,7 @@ class_dir_create_and_add(struct class *class, struct kobject *parent_kobj)
 
 	dir = kzalloc(sizeof(*dir), GFP_KERNEL);
 	if (!dir)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	dir->class = class;
 	kobject_init(&dir->kobj, &class_dir_ktype);
@@ -1477,7 +1497,7 @@ class_dir_create_and_add(struct class *class, struct kobject *parent_kobj)
 	retval = kobject_add(&dir->kobj, parent_kobj, "%s", class->name);
 	if (retval < 0) {
 		kobject_put(&dir->kobj);
-		return NULL;
+		return ERR_PTR(retval);
 	}
 	return &dir->kobj;
 }
@@ -1784,6 +1804,10 @@ int device_add(struct device *dev)
 
 	parent = get_device(dev->parent);
 	kobj = get_device_parent(dev, parent);
+	if (IS_ERR(kobj)) {
+		error = PTR_ERR(kobj);
+		goto parent_error;
+	}
 	if (kobj)
 		dev->kobj.parent = kobj;
 
@@ -1882,6 +1906,7 @@ int device_add(struct device *dev)
 	kobject_del(&dev->kobj);
  Error:
 	cleanup_glue_dir(dev, glue_dir);
+parent_error:
 	put_device(parent);
 name_error:
 	kfree(dev->p);
@@ -2406,7 +2431,7 @@ static void device_create_release(struct device *dev)
 	kfree(dev);
 }
 
-static struct device *
+static __printf(6, 0) struct device *
 device_create_groups_vargs(struct class *class, struct device *parent,
 			   dev_t devt, void *drvdata,
 			   const struct attribute_group **groups,
@@ -2684,7 +2709,7 @@ static int device_move_class_links(struct device *dev,
 /**
  * device_move - moves a device to a new parent
  * @dev: the pointer to the struct device to be moved
- * @new_parent: the new parent of the device (can by NULL)
+ * @new_parent: the new parent of the device (can be NULL)
  * @dpm_order: how to reorder the dpm_list
  */
 int device_move(struct device *dev, struct device *new_parent,
@@ -2701,6 +2726,11 @@ int device_move(struct device *dev, struct device *new_parent,
 	device_pm_lock();
 	new_parent = get_device(new_parent);
 	new_parent_kobj = get_device_parent(dev, new_parent);
+	if (IS_ERR(new_parent_kobj)) {
+		error = PTR_ERR(new_parent_kobj);
+		put_device(new_parent);
+		goto out;
+	}
 
 	pr_debug("device: '%s': %s: moving to '%s'\n", dev_name(dev),
 		 __func__, new_parent ? dev_name(new_parent) : "<NULL>");
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index c9f54089429b..a41c91bfac0e 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -122,9 +122,7 @@ static void deferred_probe_work_func(struct work_struct *work)
 		 * the list is a good order for suspend but deferred
 		 * probe makes that very unsafe.
 		 */
-		device_pm_lock();
-		device_pm_move_last(dev);
-		device_pm_unlock();
+		device_pm_move_to_tail(dev);
 
 		dev_dbg(dev, "Retrying from deferred list\n");
 		if (initcall_debug && !initcalls_done)
@@ -582,7 +580,7 @@ int driver_probe_device(struct device_driver *drv, struct device *dev)
 	pr_debug("bus: '%s': %s: matched device %s with driver %s\n",
 		 drv->bus->name, __func__, dev_name(dev), drv->name);
 
-	pm_runtime_get_suppliers(dev);
+	pm_runtime_resume_suppliers(dev);
 	if (dev->parent)
 		pm_runtime_get_sync(dev->parent);
 
@@ -593,7 +591,6 @@ int driver_probe_device(struct device_driver *drv, struct device *dev)
 	if (dev->parent)
 		pm_runtime_put(dev->parent);
 
-	pm_runtime_put_suppliers(dev);
 	return ret;
 }
 
diff --git a/drivers/base/dma-mapping.c b/drivers/base/dma-mapping.c
index d82566d6e237..f831a582209c 100644
--- a/drivers/base/dma-mapping.c
+++ b/drivers/base/dma-mapping.c
@@ -329,36 +329,13 @@ void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags)
 #endif
 
 /*
- * Common configuration to enable DMA API use for a device
+ * enables DMA API use for a device
  */
-#include <linux/pci.h>
-
 int dma_configure(struct device *dev)
 {
-	struct device *bridge = NULL, *dma_dev = dev;
-	enum dev_dma_attr attr;
-	int ret = 0;
-
-	if (dev_is_pci(dev)) {
-		bridge = pci_get_host_bridge_device(to_pci_dev(dev));
-		dma_dev = bridge;
-		if (IS_ENABLED(CONFIG_OF) && dma_dev->parent &&
-		    dma_dev->parent->of_node)
-			dma_dev = dma_dev->parent;
-	}
-
-	if (dma_dev->of_node) {
-		ret = of_dma_configure(dev, dma_dev->of_node);
-	} else if (has_acpi_companion(dma_dev)) {
-		attr = acpi_get_dma_attr(to_acpi_device_node(dma_dev->fwnode));
-		if (attr != DEV_DMA_NOT_SUPPORTED)
-			ret = acpi_dma_configure(dev, attr);
-	}
-
-	if (bridge)
-		pci_put_host_bridge_device(bridge);
-
-	return ret;
+	if (dev->bus->dma_configure)
+		return dev->bus->dma_configure(dev);
+	return 0;
 }
 
 void dma_deconfigure(struct device *dev)
diff --git a/drivers/base/driver.c b/drivers/base/driver.c
index ba912558a510..857c8f1b876e 100644
--- a/drivers/base/driver.c
+++ b/drivers/base/driver.c
@@ -148,7 +148,11 @@ int driver_register(struct device_driver *drv)
 	int ret;
 	struct device_driver *other;
 
-	BUG_ON(!drv->bus->p);
+	if (!drv->bus->p) {
+		pr_err("Driver '%s' was unable to register with bus_type '%s' because the bus was not initialized.\n",
+			   drv->name, drv->bus->name);
+		return -EINVAL;
+	}
 
 	if ((drv->bus->probe && drv->probe) ||
 	    (drv->bus->remove && drv->remove) ||
diff --git a/drivers/base/firmware_loader/Kconfig b/drivers/base/firmware_loader/Kconfig
new file mode 100644
index 000000000000..eb15d976a9ea
--- /dev/null
+++ b/drivers/base/firmware_loader/Kconfig
@@ -0,0 +1,154 @@
+menu "Firmware loader"
+
+config FW_LOADER
+	tristate "Firmware loading facility" if EXPERT
+	default y
+	help
+	  This enables the firmware loading facility in the kernel. The kernel
+	  will first look for built-in firmware, if it has any. Next, it will
+	  look for the requested firmware in a series of filesystem paths:
+
+		o firmware_class path module parameter or kernel boot param
+		o /lib/firmware/updates/UTS_RELEASE
+		o /lib/firmware/updates
+		o /lib/firmware/UTS_RELEASE
+		o /lib/firmware
+
+	  Enabling this feature only increases your kernel image by about
+	  828 bytes, enable this option unless you are certain you don't
+	  need firmware.
+
+	  You typically want this built-in (=y) but you can also enable this
+	  as a module, in which case the firmware_class module will be built.
+	  You also want to be sure to enable this built-in if you are going to
+	  enable built-in firmware (CONFIG_EXTRA_FIRMWARE).
+
+if FW_LOADER
+
+config EXTRA_FIRMWARE
+	string "Build named firmware blobs into the kernel binary"
+	help
+	  Device drivers which require firmware can typically deal with
+	  having the kernel load firmware from the various supported
+	  /lib/firmware/ paths. This option enables you to build into the
+	  kernel firmware files. Built-in firmware searches are preceded
+	  over firmware lookups using your filesystem over the supported
+	  /lib/firmware paths documented on CONFIG_FW_LOADER.
+
+	  This may be useful for testing or if the firmware is required early on
+	  in boot and cannot rely on the firmware being placed in an initrd or
+	  initramfs.
+
+	  This option is a string and takes the (space-separated) names of the
+	  firmware files -- the same names that appear in MODULE_FIRMWARE()
+	  and request_firmware() in the source. These files should exist under
+	  the directory specified by the EXTRA_FIRMWARE_DIR option, which is
+	  /lib/firmware by default.
+
+	  For example, you might set CONFIG_EXTRA_FIRMWARE="usb8388.bin", copy
+	  the usb8388.bin file into /lib/firmware, and build the kernel. Then
+	  any request_firmware("usb8388.bin") will be satisfied internally
+	  inside the kernel without ever looking at your filesystem at runtime.
+
+	  WARNING: If you include additional firmware files into your binary
+	  kernel image that are not available under the terms of the GPL,
+	  then it may be a violation of the GPL to distribute the resulting
+	  image since it combines both GPL and non-GPL work. You should
+	  consult a lawyer of your own before distributing such an image.
+
+config EXTRA_FIRMWARE_DIR
+	string "Firmware blobs root directory"
+	depends on EXTRA_FIRMWARE != ""
+	default "/lib/firmware"
+	help
+	  This option controls the directory in which the kernel build system
+	  looks for the firmware files listed in the EXTRA_FIRMWARE option.
+
+config FW_LOADER_USER_HELPER
+	bool "Enable the firmware sysfs fallback mechanism"
+	help
+	  This option enables a sysfs loading facility to enable firmware
+	  loading to the kernel through userspace as a fallback mechanism
+	  if and only if the kernel's direct filesystem lookup for the
+	  firmware failed using the different /lib/firmware/ paths, or the
+	  path specified in the firmware_class path module parameter, or the
+	  firmware_class path kernel boot parameter if the firmware_class is
+	  built-in. For details on how to work with the sysfs fallback mechanism
+	  refer to Documentation/driver-api/firmware/fallback-mechanisms.rst.
+
+	  The direct filesystem lookup for firmware is always used first now.
+
+	  If the kernel's direct filesystem lookup for firmware fails to find
+	  the requested firmware a sysfs fallback loading facility is made
+	  available and userspace is informed about this through uevents.
+	  The uevent can be suppressed if the driver explicitly requested it,
+	  this is known as the driver using the custom fallback mechanism.
+	  If the custom fallback mechanism is used userspace must always
+	  acknowledge failure to find firmware as the timeout for the fallback
+	  mechanism is disabled, and failed requests will linger forever.
+
+	  This used to be the default firmware loading facility, and udev used
+	  to listen for uvents to load firmware for the kernel. The firmware
+	  loading facility functionality in udev has been removed, as such it
+	  can no longer be relied upon as a fallback mechanism. Linux no longer
+	  relies on or uses a fallback mechanism in userspace. If you need to
+	  rely on one refer to the permissively licensed firmwared:
+
+	  https://github.com/teg/firmwared
+
+	  Since this was the default firmware loading facility at one point,
+	  old userspace may exist which relies upon it, and as such this
+	  mechanism can never be removed from the kernel.
+
+	  You should only enable this functionality if you are certain you
+	  require a fallback mechanism and have a userspace mechanism ready to
+	  load firmware in case it is not found. One main reason for this may
+	  be if you have drivers which require firmware built-in and for
+	  whatever reason cannot place the required firmware in initramfs.
+	  Another reason kernels may have this feature enabled is to support a
+	  driver which explicitly relies on this fallback mechanism. Only two
+	  drivers need this today:
+
+	    o CONFIG_LEDS_LP55XX_COMMON
+	    o CONFIG_DELL_RBU
+
+	  Outside of supporting the above drivers, another reason for needing
+	  this may be that your firmware resides outside of the paths the kernel
+	  looks for and cannot possibly be specified using the firmware_class
+	  path module parameter or kernel firmware_class path boot parameter
+	  if firmware_class is built-in.
+
+	  A modern use case may be to temporarily mount a custom partition
+	  during provisioning which is only accessible to userspace, and then
+	  to use it to look for and fetch the required firmware. Such type of
+	  driver functionality may not even ever be desirable upstream by
+	  vendors, and as such is only required to be supported as an interface
+	  for provisioning. Since udev's firmware loading facility has been
+	  removed you can use firmwared or a fork of it to customize how you
+	  want to load firmware based on uevents issued.
+
+	  Enabling this option will increase your kernel image size by about
+	  13436 bytes.
+
+	  If you are unsure about this, say N here, unless you are Linux
+	  distribution and need to support the above two drivers, or you are
+	  certain you need to support some really custom firmware loading
+	  facility in userspace.
+
+config FW_LOADER_USER_HELPER_FALLBACK
+	bool "Force the firmware sysfs fallback mechanism when possible"
+	depends on FW_LOADER_USER_HELPER
+	help
+	  Enabling this option forces a sysfs userspace fallback mechanism
+	  to be used for all firmware requests which explicitly do not disable a
+	  a fallback mechanism. Firmware calls which do prohibit a fallback
+	  mechanism is request_firmware_direct(). This option is kept for
+          backward compatibility purposes given this precise mechanism can also
+	  be enabled by setting the proc sysctl value to true:
+
+	       /proc/sys/kernel/firmware_config/force_sysfs_fallback
+
+	  If you are unsure about this, say N here.
+
+endif # FW_LOADER
+endmenu
diff --git a/drivers/base/firmware_loader/fallback.c b/drivers/base/firmware_loader/fallback.c
index 358354148dec..b676a99c469c 100644
--- a/drivers/base/firmware_loader/fallback.c
+++ b/drivers/base/firmware_loader/fallback.c
@@ -125,7 +125,7 @@ static ssize_t timeout_show(struct class *class, struct class_attribute *attr,
 }
 
 /**
- * firmware_timeout_store - set number of seconds to wait for firmware
+ * firmware_timeout_store() - set number of seconds to wait for firmware
  * @class: device class pointer
  * @attr: device attribute pointer
  * @buf: buffer to scan for timeout value
@@ -239,7 +239,7 @@ static int map_fw_priv_pages(struct fw_priv *fw_priv)
 }
 
 /**
- * firmware_loading_store - set value in the 'loading' control file
+ * firmware_loading_store() - set value in the 'loading' control file
  * @dev: device pointer
  * @attr: device attribute pointer
  * @buf: buffer to scan for loading control value
@@ -431,7 +431,7 @@ static int fw_realloc_pages(struct fw_sysfs *fw_sysfs, int min_size)
 }
 
 /**
- * firmware_data_write - write method for firmware
+ * firmware_data_write() - write method for firmware
  * @filp: open sysfs file
  * @kobj: kobject for the device
  * @bin_attr: bin_attr structure
@@ -512,7 +512,7 @@ static const struct attribute_group *fw_dev_attr_groups[] = {
 
 static struct fw_sysfs *
 fw_create_instance(struct firmware *firmware, const char *fw_name,
-		   struct device *device, unsigned int opt_flags)
+		   struct device *device, enum fw_opt opt_flags)
 {
 	struct fw_sysfs *fw_sysfs;
 	struct device *f_dev;
@@ -537,7 +537,7 @@ fw_create_instance(struct firmware *firmware, const char *fw_name,
 }
 
 /**
- * fw_load_sysfs_fallback - load a firmware via the sysfs fallback mechanism
+ * fw_load_sysfs_fallback() - load a firmware via the sysfs fallback mechanism
  * @fw_sysfs: firmware sysfs information for the firmware to load
  * @opt_flags: flags of options, FW_OPT_*
  * @timeout: timeout to wait for the load
@@ -545,7 +545,7 @@ fw_create_instance(struct firmware *firmware, const char *fw_name,
  * In charge of constructing a sysfs fallback interface for firmware loading.
  **/
 static int fw_load_sysfs_fallback(struct fw_sysfs *fw_sysfs,
-				  unsigned int opt_flags, long timeout)
+				  enum fw_opt opt_flags, long timeout)
 {
 	int retval = 0;
 	struct device *f_dev = &fw_sysfs->dev;
@@ -599,7 +599,7 @@ static int fw_load_sysfs_fallback(struct fw_sysfs *fw_sysfs,
 
 static int fw_load_from_user_helper(struct firmware *firmware,
 				    const char *name, struct device *device,
-				    unsigned int opt_flags)
+				    enum fw_opt opt_flags)
 {
 	struct fw_sysfs *fw_sysfs;
 	long timeout;
@@ -640,7 +640,7 @@ static int fw_load_from_user_helper(struct firmware *firmware,
 	return ret;
 }
 
-static bool fw_force_sysfs_fallback(unsigned int opt_flags)
+static bool fw_force_sysfs_fallback(enum fw_opt opt_flags)
 {
 	if (fw_fallback_config.force_sysfs_fallback)
 		return true;
@@ -649,7 +649,7 @@ static bool fw_force_sysfs_fallback(unsigned int opt_flags)
 	return true;
 }
 
-static bool fw_run_sysfs_fallback(unsigned int opt_flags)
+static bool fw_run_sysfs_fallback(enum fw_opt opt_flags)
 {
 	if (fw_fallback_config.ignore_sysfs_fallback) {
 		pr_info_once("Ignoring firmware sysfs fallback due to sysctl knob\n");
@@ -662,14 +662,39 @@ static bool fw_run_sysfs_fallback(unsigned int opt_flags)
 	return fw_force_sysfs_fallback(opt_flags);
 }
 
-int fw_sysfs_fallback(struct firmware *fw, const char *name,
-		      struct device *device,
-		      unsigned int opt_flags,
-		      int ret)
+/**
+ * firmware_fallback_sysfs() - use the fallback mechanism to find firmware
+ * @fw: pointer to firmware image
+ * @name: name of firmware file to look for
+ * @device: device for which firmware is being loaded
+ * @opt_flags: options to control firmware loading behaviour
+ * @ret: return value from direct lookup which triggered the fallback mechanism
+ *
+ * This function is called if direct lookup for the firmware failed, it enables
+ * a fallback mechanism through userspace by exposing a sysfs loading
+ * interface. Userspace is in charge of loading the firmware through the syfs
+ * loading interface. This syfs fallback mechanism may be disabled completely
+ * on a system by setting the proc sysctl value ignore_sysfs_fallback to true.
+ * If this false we check if the internal API caller set the @FW_OPT_NOFALLBACK
+ * flag, if so it would also disable the fallback mechanism. A system may want
+ * to enfoce the sysfs fallback mechanism at all times, it can do this by
+ * setting ignore_sysfs_fallback to false and force_sysfs_fallback to true.
+ * Enabling force_sysfs_fallback is functionally equivalent to build a kernel
+ * with CONFIG_FW_LOADER_USER_HELPER_FALLBACK.
+ **/
+int firmware_fallback_sysfs(struct firmware *fw, const char *name,
+			    struct device *device,
+			    enum fw_opt opt_flags,
+			    int ret)
 {
 	if (!fw_run_sysfs_fallback(opt_flags))
 		return ret;
 
-	dev_warn(device, "Falling back to user helper\n");
+	if (!(opt_flags & FW_OPT_NO_WARN))
+		dev_warn(device, "Falling back to syfs fallback for: %s\n",
+				 name);
+	else
+		dev_dbg(device, "Falling back to sysfs fallback for: %s\n",
+				name);
 	return fw_load_from_user_helper(fw, name, device, opt_flags);
 }
diff --git a/drivers/base/firmware_loader/fallback.h b/drivers/base/firmware_loader/fallback.h
index f8255670a663..21063503e4ea 100644
--- a/drivers/base/firmware_loader/fallback.h
+++ b/drivers/base/firmware_loader/fallback.h
@@ -5,6 +5,8 @@
 #include <linux/firmware.h>
 #include <linux/device.h>
 
+#include "firmware.h"
+
 /**
  * struct firmware_fallback_config - firmware fallback configuration settings
  *
@@ -29,10 +31,10 @@ struct firmware_fallback_config {
 };
 
 #ifdef CONFIG_FW_LOADER_USER_HELPER
-int fw_sysfs_fallback(struct firmware *fw, const char *name,
-		      struct device *device,
-		      unsigned int opt_flags,
-		      int ret);
+int firmware_fallback_sysfs(struct firmware *fw, const char *name,
+			    struct device *device,
+			    enum fw_opt opt_flags,
+			    int ret);
 void kill_pending_fw_fallback_reqs(bool only_kill_custom);
 
 void fw_fallback_set_cache_timeout(void);
@@ -41,10 +43,10 @@ void fw_fallback_set_default_timeout(void);
 int register_sysfs_loader(void);
 void unregister_sysfs_loader(void);
 #else /* CONFIG_FW_LOADER_USER_HELPER */
-static inline int fw_sysfs_fallback(struct firmware *fw, const char *name,
-				    struct device *device,
-				    unsigned int opt_flags,
-				    int ret)
+static inline int firmware_fallback_sysfs(struct firmware *fw, const char *name,
+					  struct device *device,
+					  enum fw_opt opt_flags,
+					  int ret)
 {
 	/* Keep carrying over the same error */
 	return ret;
diff --git a/drivers/base/firmware_loader/firmware.h b/drivers/base/firmware_loader/firmware.h
index 64acbb1a392c..4c1395f8e7ed 100644
--- a/drivers/base/firmware_loader/firmware.h
+++ b/drivers/base/firmware_loader/firmware.h
@@ -2,6 +2,7 @@
 #ifndef __FIRMWARE_LOADER_H
 #define __FIRMWARE_LOADER_H
 
+#include <linux/bitops.h>
 #include <linux/firmware.h>
 #include <linux/types.h>
 #include <linux/kref.h>
@@ -10,13 +11,33 @@
 
 #include <generated/utsrelease.h>
 
-/* firmware behavior options */
-#define FW_OPT_UEVENT			(1U << 0)
-#define FW_OPT_NOWAIT			(1U << 1)
-#define FW_OPT_USERHELPER		(1U << 2)
-#define FW_OPT_NO_WARN			(1U << 3)
-#define FW_OPT_NOCACHE			(1U << 4)
-#define FW_OPT_NOFALLBACK		(1U << 5)
+/**
+ * enum fw_opt - options to control firmware loading behaviour
+ *
+ * @FW_OPT_UEVENT: Enables the fallback mechanism to send a kobject uevent
+ *	when the firmware is not found. Userspace is in charge to load the
+ *	firmware using the sysfs loading facility.
+ * @FW_OPT_NOWAIT: Used to describe the firmware request is asynchronous.
+ * @FW_OPT_USERHELPER: Enable the fallback mechanism, in case the direct
+ *	filesystem lookup fails at finding the firmware.  For details refer to
+ *	firmware_fallback_sysfs().
+ * @FW_OPT_NO_WARN: Quiet, avoid printing warning messages.
+ * @FW_OPT_NOCACHE: Disables firmware caching. Firmware caching is used to
+ *	cache the firmware upon suspend, so that upon resume races against the
+ *	firmware file lookup on storage is avoided. Used for calls where the
+ *	file may be too big, or where the driver takes charge of its own
+ *	firmware caching mechanism.
+ * @FW_OPT_NOFALLBACK: Disable the fallback mechanism. Takes precedence over
+ *	&FW_OPT_UEVENT and &FW_OPT_USERHELPER.
+ */
+enum fw_opt {
+	FW_OPT_UEVENT =         BIT(0),
+	FW_OPT_NOWAIT =         BIT(1),
+	FW_OPT_USERHELPER =     BIT(2),
+	FW_OPT_NO_WARN =        BIT(3),
+	FW_OPT_NOCACHE =        BIT(4),
+	FW_OPT_NOFALLBACK =     BIT(5),
+};
 
 enum fw_status {
 	FW_STATUS_UNKNOWN,
@@ -110,6 +131,6 @@ static inline void fw_state_done(struct fw_priv *fw_priv)
 }
 
 int assign_fw(struct firmware *fw, struct device *device,
-	      unsigned int opt_flags);
+	      enum fw_opt opt_flags);
 
 #endif /* __FIRMWARE_LOADER_H */
diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c
index eb34089e4299..0943e7065e0e 100644
--- a/drivers/base/firmware_loader/main.c
+++ b/drivers/base/firmware_loader/main.c
@@ -443,7 +443,7 @@ static int fw_add_devm_name(struct device *dev, const char *name)
 #endif
 
 int assign_fw(struct firmware *fw, struct device *device,
-	      unsigned int opt_flags)
+	      enum fw_opt opt_flags)
 {
 	struct fw_priv *fw_priv = fw->priv;
 	int ret;
@@ -558,7 +558,7 @@ static void fw_abort_batch_reqs(struct firmware *fw)
 static int
 _request_firmware(const struct firmware **firmware_p, const char *name,
 		  struct device *device, void *buf, size_t size,
-		  unsigned int opt_flags)
+		  enum fw_opt opt_flags)
 {
 	struct firmware *fw = NULL;
 	int ret;
@@ -581,7 +581,7 @@ _request_firmware(const struct firmware **firmware_p, const char *name,
 			dev_warn(device,
 				 "Direct firmware load for %s failed with error %d\n",
 				 name, ret);
-		ret = fw_sysfs_fallback(fw, name, device, opt_flags, ret);
+		ret = firmware_fallback_sysfs(fw, name, device, opt_flags, ret);
 	} else
 		ret = assign_fw(fw, device, opt_flags);
 
@@ -597,7 +597,7 @@ _request_firmware(const struct firmware **firmware_p, const char *name,
 }
 
 /**
- * request_firmware: - send firmware request and wait for it
+ * request_firmware() - send firmware request and wait for it
  * @firmware_p: pointer to firmware image
  * @name: name of firmware file
  * @device: device for which firmware is being loaded
@@ -632,7 +632,34 @@ request_firmware(const struct firmware **firmware_p, const char *name,
 EXPORT_SYMBOL(request_firmware);
 
 /**
- * request_firmware_direct: - load firmware directly without usermode helper
+ * firmware_request_nowarn() - request for an optional fw module
+ * @firmware: pointer to firmware image
+ * @name: name of firmware file
+ * @device: device for which firmware is being loaded
+ *
+ * This function is similar in behaviour to request_firmware(), except
+ * it doesn't produce warning messages when the file is not found.
+ * The sysfs fallback mechanism is enabled if direct filesystem lookup fails,
+ * however, however failures to find the firmware file with it are still
+ * suppressed. It is therefore up to the driver to check for the return value
+ * of this call and to decide when to inform the users of errors.
+ **/
+int firmware_request_nowarn(const struct firmware **firmware, const char *name,
+			    struct device *device)
+{
+	int ret;
+
+	/* Need to pin this module until return */
+	__module_get(THIS_MODULE);
+	ret = _request_firmware(firmware, name, device, NULL, 0,
+				FW_OPT_UEVENT | FW_OPT_NO_WARN);
+	module_put(THIS_MODULE);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(firmware_request_nowarn);
+
+/**
+ * request_firmware_direct() - load firmware directly without usermode helper
  * @firmware_p: pointer to firmware image
  * @name: name of firmware file
  * @device: device for which firmware is being loaded
@@ -657,7 +684,7 @@ int request_firmware_direct(const struct firmware **firmware_p,
 EXPORT_SYMBOL_GPL(request_firmware_direct);
 
 /**
- * firmware_request_cache: - cache firmware for suspend so resume can use it
+ * firmware_request_cache() - cache firmware for suspend so resume can use it
  * @name: name of firmware file
  * @device: device for which firmware should be cached for
  *
@@ -681,7 +708,7 @@ int firmware_request_cache(struct device *device, const char *name)
 EXPORT_SYMBOL_GPL(firmware_request_cache);
 
 /**
- * request_firmware_into_buf - load firmware into a previously allocated buffer
+ * request_firmware_into_buf() - load firmware into a previously allocated buffer
  * @firmware_p: pointer to firmware image
  * @name: name of firmware file
  * @device: device for which firmware is being loaded and DMA region allocated
@@ -713,7 +740,7 @@ request_firmware_into_buf(const struct firmware **firmware_p, const char *name,
 EXPORT_SYMBOL(request_firmware_into_buf);
 
 /**
- * release_firmware: - release the resource associated with a firmware image
+ * release_firmware() - release the resource associated with a firmware image
  * @fw: firmware resource to release
  **/
 void release_firmware(const struct firmware *fw)
@@ -734,7 +761,7 @@ struct firmware_work {
 	struct device *device;
 	void *context;
 	void (*cont)(const struct firmware *fw, void *context);
-	unsigned int opt_flags;
+	enum fw_opt opt_flags;
 };
 
 static void request_firmware_work_func(struct work_struct *work)
@@ -755,7 +782,7 @@ static void request_firmware_work_func(struct work_struct *work)
 }
 
 /**
- * request_firmware_nowait - asynchronous version of request_firmware
+ * request_firmware_nowait() - asynchronous version of request_firmware
  * @module: module requesting the firmware
  * @uevent: sends uevent to copy the firmware image if this flag
  *	is non-zero else the firmware copy must be done manually.
@@ -824,7 +851,7 @@ EXPORT_SYMBOL(request_firmware_nowait);
 static ASYNC_DOMAIN_EXCLUSIVE(fw_cache_domain);
 
 /**
- * cache_firmware - cache one firmware image in kernel memory space
+ * cache_firmware() - cache one firmware image in kernel memory space
  * @fw_name: the firmware image name
  *
  * Cache firmware in kernel memory so that drivers can use it when
@@ -866,7 +893,7 @@ static struct fw_priv *lookup_fw_priv(const char *fw_name)
 }
 
 /**
- * uncache_firmware - remove one cached firmware image
+ * uncache_firmware() - remove one cached firmware image
  * @fw_name: the firmware image name
  *
  * Uncache one firmware image which has been cached successfully
@@ -1042,7 +1069,7 @@ static void __device_uncache_fw_images(void)
 }
 
 /**
- * device_cache_fw_images - cache devices' firmware
+ * device_cache_fw_images() - cache devices' firmware
  *
  * If one device called request_firmware or its nowait version
  * successfully before, the firmware names are recored into the
@@ -1075,7 +1102,7 @@ static void device_cache_fw_images(void)
 }
 
 /**
- * device_uncache_fw_images - uncache devices' firmware
+ * device_uncache_fw_images() - uncache devices' firmware
  *
  * uncache all firmwares which have been cached successfully
  * by device_uncache_fw_images earlier
@@ -1092,7 +1119,7 @@ static void device_uncache_fw_images_work(struct work_struct *work)
 }
 
 /**
- * device_uncache_fw_images_delay - uncache devices firmwares
+ * device_uncache_fw_images_delay() - uncache devices firmwares
  * @delay: number of milliseconds to delay uncache device firmwares
  *
  * uncache all devices's firmwares which has been cached successfully
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index bffe8616bd55..f5e560188a18 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -649,13 +649,19 @@ static const struct attribute_group *memory_memblk_attr_groups[] = {
 static
 int register_memory(struct memory_block *memory)
 {
+	int ret;
+
 	memory->dev.bus = &memory_subsys;
 	memory->dev.id = memory->start_section_nr / sections_per_block;
 	memory->dev.release = memory_block_release;
 	memory->dev.groups = memory_memblk_attr_groups;
 	memory->dev.offline = memory->state == MEM_OFFLINE;
 
-	return device_register(&memory->dev);
+	ret = device_register(&memory->dev);
+	if (ret)
+		put_device(&memory->dev);
+
+	return ret;
 }
 
 static int init_memory_block(struct memory_block **memory,
diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c
index 8e22073aeeed..60d6cc618f1c 100644
--- a/drivers/base/platform-msi.c
+++ b/drivers/base/platform-msi.c
@@ -101,6 +101,9 @@ static void platform_msi_update_chip_ops(struct msi_domain_info *info)
 		chip->irq_set_affinity = msi_domain_set_affinity;
 	if (!chip->irq_write_msi_msg)
 		chip->irq_write_msi_msg = platform_msi_write_msg;
+	if (WARN_ON((info->flags & MSI_FLAG_LEVEL_CAPABLE) &&
+		    !(chip->flags & IRQCHIP_SUPPORTS_LEVEL_MSI)))
+		info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;
 }
 
 static void platform_msi_free_descs(struct device *dev, int base, int nvec)
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index 8075ddc70a17..dff82a3c2caa 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -572,17 +572,16 @@ static int platform_drv_probe(struct device *_dev)
 		return ret;
 
 	ret = dev_pm_domain_attach(_dev, true);
-	if (ret != -EPROBE_DEFER) {
-		if (drv->probe) {
-			ret = drv->probe(dev);
-			if (ret)
-				dev_pm_domain_detach(_dev, true);
-		} else {
-			/* don't fail if just dev_pm_domain_attach failed */
-			ret = 0;
-		}
+	if (ret)
+		goto out;
+
+	if (drv->probe) {
+		ret = drv->probe(dev);
+		if (ret)
+			dev_pm_domain_detach(_dev, true);
 	}
 
+out:
 	if (drv->prevent_deferred_probe && ret == -EPROBE_DEFER) {
 		dev_warn(_dev, "probe deferral not supported\n");
 		ret = -ENXIO;
@@ -1130,6 +1129,22 @@ int platform_pm_restore(struct device *dev)
 
 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 
+int platform_dma_configure(struct device *dev)
+{
+	enum dev_dma_attr attr;
+	int ret = 0;
+
+	if (dev->of_node) {
+		ret = of_dma_configure(dev, dev->of_node, true);
+	} else if (has_acpi_companion(dev)) {
+		attr = acpi_get_dma_attr(to_acpi_device_node(dev->fwnode));
+		if (attr != DEV_DMA_NOT_SUPPORTED)
+			ret = acpi_dma_configure(dev, attr);
+	}
+
+	return ret;
+}
+
 static const struct dev_pm_ops platform_dev_pm_ops = {
 	.runtime_suspend = pm_generic_runtime_suspend,
 	.runtime_resume = pm_generic_runtime_resume,
@@ -1141,8 +1156,8 @@ struct bus_type platform_bus_type = {
 	.dev_groups	= platform_dev_groups,
 	.match		= platform_match,
 	.uevent		= platform_uevent,
+	.dma_configure	= platform_dma_configure,
 	.pm		= &platform_dev_pm_ops,
-	.force_dma	= true,
 };
 EXPORT_SYMBOL_GPL(platform_bus_type);
 
diff --git a/drivers/base/power/common.c b/drivers/base/power/common.c
index f6a9ad52cbbf..7ae62b6355b8 100644
--- a/drivers/base/power/common.c
+++ b/drivers/base/power/common.c
@@ -98,17 +98,21 @@ EXPORT_SYMBOL_GPL(dev_pm_put_subsys_data);
  * Callers must ensure proper synchronization of this function with power
  * management callbacks.
  *
- * Returns 0 on successfully attached PM domain or negative error code.
+ * Returns 0 on successfully attached PM domain, or when it is found that the
+ * device doesn't need a PM domain, else a negative error code.
  */
 int dev_pm_domain_attach(struct device *dev, bool power_on)
 {
 	int ret;
 
+	if (dev->pm_domain)
+		return 0;
+
 	ret = acpi_dev_pm_attach(dev, power_on);
-	if (ret)
+	if (!ret)
 		ret = genpd_dev_pm_attach(dev);
 
-	return ret;
+	return ret < 0 ? ret : 0;
 }
 EXPORT_SYMBOL_GPL(dev_pm_domain_attach);
 
diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
index 1ea0e2502e8e..6f403d6fccb2 100644
--- a/drivers/base/power/domain.c
+++ b/drivers/base/power/domain.c
@@ -10,6 +10,7 @@
 #include <linux/kernel.h>
 #include <linux/io.h>
 #include <linux/platform_device.h>
+#include <linux/pm_opp.h>
 #include <linux/pm_runtime.h>
 #include <linux/pm_domain.h>
 #include <linux/pm_qos.h>
@@ -1315,7 +1316,6 @@ EXPORT_SYMBOL_GPL(pm_genpd_syscore_poweron);
 #endif /* CONFIG_PM_SLEEP */
 
 static struct generic_pm_domain_data *genpd_alloc_dev_data(struct device *dev,
-					struct generic_pm_domain *genpd,
 					struct gpd_timing_data *td)
 {
 	struct generic_pm_domain_data *gpd_data;
@@ -1377,24 +1377,19 @@ static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
 			    struct gpd_timing_data *td)
 {
 	struct generic_pm_domain_data *gpd_data;
-	int ret = 0;
+	int ret;
 
 	dev_dbg(dev, "%s()\n", __func__);
 
 	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev))
 		return -EINVAL;
 
-	gpd_data = genpd_alloc_dev_data(dev, genpd, td);
+	gpd_data = genpd_alloc_dev_data(dev, td);
 	if (IS_ERR(gpd_data))
 		return PTR_ERR(gpd_data);
 
 	genpd_lock(genpd);
 
-	if (genpd->prepared_count > 0) {
-		ret = -EAGAIN;
-		goto out;
-	}
-
 	ret = genpd->attach_dev ? genpd->attach_dev(genpd, dev) : 0;
 	if (ret)
 		goto out;
@@ -1418,23 +1413,21 @@ static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
 }
 
 /**
- * __pm_genpd_add_device - Add a device to an I/O PM domain.
+ * pm_genpd_add_device - Add a device to an I/O PM domain.
  * @genpd: PM domain to add the device to.
  * @dev: Device to be added.
- * @td: Set of PM QoS timing parameters to attach to the device.
  */
-int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
-			  struct gpd_timing_data *td)
+int pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev)
 {
 	int ret;
 
 	mutex_lock(&gpd_list_lock);
-	ret = genpd_add_device(genpd, dev, td);
+	ret = genpd_add_device(genpd, dev, NULL);
 	mutex_unlock(&gpd_list_lock);
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(__pm_genpd_add_device);
+EXPORT_SYMBOL_GPL(pm_genpd_add_device);
 
 static int genpd_remove_device(struct generic_pm_domain *genpd,
 			       struct device *dev)
@@ -1481,13 +1474,13 @@ static int genpd_remove_device(struct generic_pm_domain *genpd,
 
 /**
  * pm_genpd_remove_device - Remove a device from an I/O PM domain.
- * @genpd: PM domain to remove the device from.
  * @dev: Device to be removed.
  */
-int pm_genpd_remove_device(struct generic_pm_domain *genpd,
-			   struct device *dev)
+int pm_genpd_remove_device(struct device *dev)
 {
-	if (!genpd || genpd != genpd_lookup_dev(dev))
+	struct generic_pm_domain *genpd = genpd_lookup_dev(dev);
+
+	if (!genpd)
 		return -EINVAL;
 
 	return genpd_remove_device(genpd, dev);
@@ -1696,6 +1689,9 @@ int pm_genpd_init(struct generic_pm_domain *genpd,
 			return ret;
 	}
 
+	device_initialize(&genpd->dev);
+	dev_set_name(&genpd->dev, "%s", genpd->name);
+
 	mutex_lock(&gpd_list_lock);
 	list_add(&genpd->gpd_list_node, &gpd_list);
 	mutex_unlock(&gpd_list_lock);
@@ -1892,14 +1888,33 @@ int of_genpd_add_provider_simple(struct device_node *np,
 
 	mutex_lock(&gpd_list_lock);
 
-	if (genpd_present(genpd)) {
-		ret = genpd_add_provider(np, genpd_xlate_simple, genpd);
-		if (!ret) {
-			genpd->provider = &np->fwnode;
-			genpd->has_provider = true;
+	if (!genpd_present(genpd))
+		goto unlock;
+
+	genpd->dev.of_node = np;
+
+	/* Parse genpd OPP table */
+	if (genpd->set_performance_state) {
+		ret = dev_pm_opp_of_add_table(&genpd->dev);
+		if (ret) {
+			dev_err(&genpd->dev, "Failed to add OPP table: %d\n",
+				ret);
+			goto unlock;
 		}
 	}
 
+	ret = genpd_add_provider(np, genpd_xlate_simple, genpd);
+	if (ret) {
+		if (genpd->set_performance_state)
+			dev_pm_opp_of_remove_table(&genpd->dev);
+
+		goto unlock;
+	}
+
+	genpd->provider = &np->fwnode;
+	genpd->has_provider = true;
+
+unlock:
 	mutex_unlock(&gpd_list_lock);
 
 	return ret;
@@ -1914,6 +1929,7 @@ EXPORT_SYMBOL_GPL(of_genpd_add_provider_simple);
 int of_genpd_add_provider_onecell(struct device_node *np,
 				  struct genpd_onecell_data *data)
 {
+	struct generic_pm_domain *genpd;
 	unsigned int i;
 	int ret = -EINVAL;
 
@@ -1926,13 +1942,27 @@ int of_genpd_add_provider_onecell(struct device_node *np,
 		data->xlate = genpd_xlate_onecell;
 
 	for (i = 0; i < data->num_domains; i++) {
-		if (!data->domains[i])
+		genpd = data->domains[i];
+
+		if (!genpd)
 			continue;
-		if (!genpd_present(data->domains[i]))
+		if (!genpd_present(genpd))
 			goto error;
 
-		data->domains[i]->provider = &np->fwnode;
-		data->domains[i]->has_provider = true;
+		genpd->dev.of_node = np;
+
+		/* Parse genpd OPP table */
+		if (genpd->set_performance_state) {
+			ret = dev_pm_opp_of_add_table_indexed(&genpd->dev, i);
+			if (ret) {
+				dev_err(&genpd->dev, "Failed to add OPP table for index %d: %d\n",
+					i, ret);
+				goto error;
+			}
+		}
+
+		genpd->provider = &np->fwnode;
+		genpd->has_provider = true;
 	}
 
 	ret = genpd_add_provider(np, data->xlate, data);
@@ -1945,10 +1975,16 @@ int of_genpd_add_provider_onecell(struct device_node *np,
 
 error:
 	while (i--) {
-		if (!data->domains[i])
+		genpd = data->domains[i];
+
+		if (!genpd)
 			continue;
-		data->domains[i]->provider = NULL;
-		data->domains[i]->has_provider = false;
+
+		genpd->provider = NULL;
+		genpd->has_provider = false;
+
+		if (genpd->set_performance_state)
+			dev_pm_opp_of_remove_table(&genpd->dev);
 	}
 
 	mutex_unlock(&gpd_list_lock);
@@ -1975,10 +2011,17 @@ void of_genpd_del_provider(struct device_node *np)
 			 * provider, set the 'has_provider' to false
 			 * so that the PM domain can be safely removed.
 			 */
-			list_for_each_entry(gpd, &gpd_list, gpd_list_node)
-				if (gpd->provider == &np->fwnode)
+			list_for_each_entry(gpd, &gpd_list, gpd_list_node) {
+				if (gpd->provider == &np->fwnode) {
 					gpd->has_provider = false;
 
+					if (!gpd->set_performance_state)
+						continue;
+
+					dev_pm_opp_of_remove_table(&gpd->dev);
+				}
+			}
+
 			list_del(&cp->link);
 			of_node_put(cp->node);
 			kfree(cp);
@@ -2185,31 +2228,25 @@ static void genpd_dev_pm_sync(struct device *dev)
  * Parse device's OF node to find a PM domain specifier. If such is found,
  * attaches the device to retrieved pm_domain ops.
  *
- * Both generic and legacy Samsung-specific DT bindings are supported to keep
- * backwards compatibility with existing DTBs.
- *
- * Returns 0 on successfully attached PM domain or negative error code. Note
- * that if a power-domain exists for the device, but it cannot be found or
- * turned on, then return -EPROBE_DEFER to ensure that the device is not
- * probed and to re-try again later.
+ * Returns 1 on successfully attached PM domain, 0 when the device don't need a
+ * PM domain or a negative error code in case of failures. Note that if a
+ * power-domain exists for the device, but it cannot be found or turned on,
+ * then return -EPROBE_DEFER to ensure that the device is not probed and to
+ * re-try again later.
  */
 int genpd_dev_pm_attach(struct device *dev)
 {
 	struct of_phandle_args pd_args;
 	struct generic_pm_domain *pd;
-	unsigned int i;
 	int ret;
 
 	if (!dev->of_node)
-		return -ENODEV;
-
-	if (dev->pm_domain)
-		return -EEXIST;
+		return 0;
 
 	ret = of_parse_phandle_with_args(dev->of_node, "power-domains",
 					"#power-domain-cells", 0, &pd_args);
 	if (ret < 0)
-		return ret;
+		return 0;
 
 	mutex_lock(&gpd_list_lock);
 	pd = genpd_get_from_provider(&pd_args);
@@ -2223,21 +2260,14 @@ int genpd_dev_pm_attach(struct device *dev)
 
 	dev_dbg(dev, "adding to PM domain %s\n", pd->name);
 
-	for (i = 1; i < GENPD_RETRY_MAX_MS; i <<= 1) {
-		ret = genpd_add_device(pd, dev, NULL);
-		if (ret != -EAGAIN)
-			break;
-
-		mdelay(i);
-		cond_resched();
-	}
+	ret = genpd_add_device(pd, dev, NULL);
 	mutex_unlock(&gpd_list_lock);
 
 	if (ret < 0) {
 		if (ret != -EPROBE_DEFER)
 			dev_err(dev, "failed to add to PM domain %s: %d",
 				pd->name, ret);
-		goto out;
+		return ret;
 	}
 
 	dev->pm_domain->detach = genpd_dev_pm_detach;
@@ -2246,8 +2276,11 @@ int genpd_dev_pm_attach(struct device *dev)
 	genpd_lock(pd);
 	ret = genpd_power_on(pd, 0);
 	genpd_unlock(pd);
-out:
-	return ret ? -EPROBE_DEFER : 0;
+
+	if (ret)
+		genpd_remove_device(pd, dev);
+
+	return ret ? -EPROBE_DEFER : 1;
 }
 EXPORT_SYMBOL_GPL(genpd_dev_pm_attach);
 
@@ -2361,6 +2394,55 @@ int of_genpd_parse_idle_states(struct device_node *dn,
 }
 EXPORT_SYMBOL_GPL(of_genpd_parse_idle_states);
 
+/**
+ * of_genpd_opp_to_performance_state- Gets performance state of device's
+ * power domain corresponding to a DT node's "required-opps" property.
+ *
+ * @dev: Device for which the performance-state needs to be found.
+ * @opp_node: DT node where the "required-opps" property is present. This can be
+ *	the device node itself (if it doesn't have an OPP table) or a node
+ *	within the OPP table of a device (if device has an OPP table).
+ * @state: Pointer to return performance state.
+ *
+ * Returns performance state corresponding to the "required-opps" property of
+ * a DT node. This calls platform specific genpd->opp_to_performance_state()
+ * callback to translate power domain OPP to performance state.
+ *
+ * Returns performance state on success and 0 on failure.
+ */
+unsigned int of_genpd_opp_to_performance_state(struct device *dev,
+					       struct device_node *opp_node)
+{
+	struct generic_pm_domain *genpd;
+	struct dev_pm_opp *opp;
+	int state = 0;
+
+	genpd = dev_to_genpd(dev);
+	if (IS_ERR(genpd))
+		return 0;
+
+	if (unlikely(!genpd->set_performance_state))
+		return 0;
+
+	genpd_lock(genpd);
+
+	opp = of_dev_pm_opp_find_required_opp(&genpd->dev, opp_node);
+	if (IS_ERR(opp)) {
+		dev_err(dev, "Failed to find required OPP: %ld\n",
+			PTR_ERR(opp));
+		goto unlock;
+	}
+
+	state = genpd->opp_to_performance_state(genpd, opp);
+	dev_pm_opp_put(opp);
+
+unlock:
+	genpd_unlock(genpd);
+
+	return state;
+}
+EXPORT_SYMBOL_GPL(of_genpd_opp_to_performance_state);
+
 #endif /* CONFIG_PM_GENERIC_DOMAINS_OF */
 
 
@@ -2628,6 +2710,19 @@ static int genpd_devices_show(struct seq_file *s, void *data)
 	return ret;
 }
 
+static int genpd_perf_state_show(struct seq_file *s, void *data)
+{
+	struct generic_pm_domain *genpd = s->private;
+
+	if (genpd_lock_interruptible(genpd))
+		return -ERESTARTSYS;
+
+	seq_printf(s, "%u\n", genpd->performance_state);
+
+	genpd_unlock(genpd);
+	return 0;
+}
+
 #define define_genpd_open_function(name) \
 static int genpd_##name##_open(struct inode *inode, struct file *file) \
 { \
@@ -2641,6 +2736,7 @@ define_genpd_open_function(idle_states);
 define_genpd_open_function(active_time);
 define_genpd_open_function(total_idle_time);
 define_genpd_open_function(devices);
+define_genpd_open_function(perf_state);
 
 #define define_genpd_debugfs_fops(name) \
 static const struct file_operations genpd_##name##_fops = { \
@@ -2657,6 +2753,7 @@ define_genpd_debugfs_fops(idle_states);
 define_genpd_debugfs_fops(active_time);
 define_genpd_debugfs_fops(total_idle_time);
 define_genpd_debugfs_fops(devices);
+define_genpd_debugfs_fops(perf_state);
 
 static int __init genpd_debug_init(void)
 {
@@ -2690,6 +2787,9 @@ static int __init genpd_debug_init(void)
 				d, genpd, &genpd_total_idle_time_fops);
 		debugfs_create_file("devices", 0444,
 				d, genpd, &genpd_devices_fops);
+		if (genpd->set_performance_state)
+			debugfs_create_file("perf_state", 0444,
+					    d, genpd, &genpd_perf_state_fops);
 	}
 
 	return 0;
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index e5e067091572..3f68e2919dc5 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -192,34 +192,31 @@ void device_pm_move_last(struct device *dev)
 	list_move_tail(&dev->power.entry, &dpm_list);
 }
 
-static ktime_t initcall_debug_start(struct device *dev)
+static ktime_t initcall_debug_start(struct device *dev, void *cb)
 {
-	ktime_t calltime = 0;
-
-	if (pm_print_times_enabled) {
-		pr_info("calling  %s+ @ %i, parent: %s\n",
-			dev_name(dev), task_pid_nr(current),
-			dev->parent ? dev_name(dev->parent) : "none");
-		calltime = ktime_get();
-	}
+	if (!pm_print_times_enabled)
+		return 0;
 
-	return calltime;
+	dev_info(dev, "calling %pF @ %i, parent: %s\n", cb,
+		 task_pid_nr(current),
+		 dev->parent ? dev_name(dev->parent) : "none");
+	return ktime_get();
 }
 
 static void initcall_debug_report(struct device *dev, ktime_t calltime,
-				  int error, pm_message_t state,
-				  const char *info)
+				  void *cb, int error)
 {
 	ktime_t rettime;
 	s64 nsecs;
 
+	if (!pm_print_times_enabled)
+		return;
+
 	rettime = ktime_get();
 	nsecs = (s64) ktime_to_ns(ktime_sub(rettime, calltime));
 
-	if (pm_print_times_enabled) {
-		pr_info("call %s+ returned %d after %Ld usecs\n", dev_name(dev),
-			error, (unsigned long long)nsecs >> 10);
-	}
+	dev_info(dev, "%pF returned %d after %Ld usecs\n", cb, error,
+		 (unsigned long long)nsecs >> 10);
 }
 
 /**
@@ -446,7 +443,7 @@ static int dpm_run_callback(pm_callback_t cb, struct device *dev,
 	if (!cb)
 		return 0;
 
-	calltime = initcall_debug_start(dev);
+	calltime = initcall_debug_start(dev, cb);
 
 	pm_dev_dbg(dev, state, info);
 	trace_device_pm_callback_start(dev, info, state.event);
@@ -454,7 +451,7 @@ static int dpm_run_callback(pm_callback_t cb, struct device *dev,
 	trace_device_pm_callback_end(dev, error);
 	suspend_report_result(cb, error);
 
-	initcall_debug_report(dev, calltime, error, state, info);
+	initcall_debug_report(dev, calltime, cb, error);
 
 	return error;
 }
@@ -1664,14 +1661,14 @@ static int legacy_suspend(struct device *dev, pm_message_t state,
 	int error;
 	ktime_t calltime;
 
-	calltime = initcall_debug_start(dev);
+	calltime = initcall_debug_start(dev, cb);
 
 	trace_device_pm_callback_start(dev, info, state.event);
 	error = cb(dev, state);
 	trace_device_pm_callback_end(dev, error);
 	suspend_report_result(cb, error);
 
-	initcall_debug_report(dev, calltime, error, state, info);
+	initcall_debug_report(dev, calltime, cb, error);
 
 	return error;
 }
diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h
index 86e67e70b509..c511def48b48 100644
--- a/drivers/base/power/power.h
+++ b/drivers/base/power/power.h
@@ -56,14 +56,6 @@ static inline void device_wakeup_detach_irq(struct device *dev)
 {
 }
 
-static inline void device_wakeup_arm_wake_irqs(void)
-{
-}
-
-static inline void device_wakeup_disarm_wake_irqs(void)
-{
-}
-
 #endif /* CONFIG_PM_SLEEP */
 
 /*
@@ -95,28 +87,6 @@ static inline void pm_runtime_remove(struct device *dev) {}
 
 static inline int dpm_sysfs_add(struct device *dev) { return 0; }
 static inline void dpm_sysfs_remove(struct device *dev) {}
-static inline void rpm_sysfs_remove(struct device *dev) {}
-static inline int wakeup_sysfs_add(struct device *dev) { return 0; }
-static inline void wakeup_sysfs_remove(struct device *dev) {}
-static inline int pm_qos_sysfs_add(struct device *dev) { return 0; }
-static inline void pm_qos_sysfs_remove(struct device *dev) {}
-
-static inline void dev_pm_arm_wake_irq(struct wake_irq *wirq)
-{
-}
-
-static inline void dev_pm_disarm_wake_irq(struct wake_irq *wirq)
-{
-}
-
-static inline void dev_pm_enable_wake_irq_check(struct device *dev,
-						bool can_change_status)
-{
-}
-
-static inline void dev_pm_disable_wake_irq_check(struct device *dev)
-{
-}
 
 #endif
 
diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
index 8bef3cb2424d..c6030f100c08 100644
--- a/drivers/base/power/runtime.c
+++ b/drivers/base/power/runtime.c
@@ -1563,37 +1563,16 @@ void pm_runtime_clean_up_links(struct device *dev)
 }
 
 /**
- * pm_runtime_get_suppliers - Resume and reference-count supplier devices.
+ * pm_runtime_resume_suppliers - Resume supplier devices.
  * @dev: Consumer device.
  */
-void pm_runtime_get_suppliers(struct device *dev)
+void pm_runtime_resume_suppliers(struct device *dev)
 {
-	struct device_link *link;
-	int idx;
-
-	idx = device_links_read_lock();
-
-	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
-		if (link->flags & DL_FLAG_PM_RUNTIME)
-			pm_runtime_get_sync(link->supplier);
-
-	device_links_read_unlock(idx);
-}
-
-/**
- * pm_runtime_put_suppliers - Drop references to supplier devices.
- * @dev: Consumer device.
- */
-void pm_runtime_put_suppliers(struct device *dev)
-{
-	struct device_link *link;
 	int idx;
 
 	idx = device_links_read_lock();
 
-	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
-		if (link->flags & DL_FLAG_PM_RUNTIME)
-			pm_runtime_put(link->supplier);
+	rpm_get_suppliers(dev);
 
 	device_links_read_unlock(idx);
 }
@@ -1607,6 +1586,8 @@ void pm_runtime_new_link(struct device *dev)
 
 void pm_runtime_drop_link(struct device *dev)
 {
+	rpm_put_suppliers(dev);
+
 	spin_lock_irq(&dev->power.lock);
 	WARN_ON(dev->power.links_count == 0);
 	dev->power.links_count--;
diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c
index ea01621ed769..5fa1898755a3 100644
--- a/drivers/base/power/wakeup.c
+++ b/drivers/base/power/wakeup.c
@@ -57,7 +57,7 @@ static void split_counters(unsigned int *cnt, unsigned int *inpr)
 /* A preserved old value of the events counter. */
 static unsigned int saved_count;
 
-static DEFINE_SPINLOCK(events_lock);
+static DEFINE_RAW_SPINLOCK(events_lock);
 
 static void pm_wakeup_timer_fn(struct timer_list *t);
 
@@ -183,11 +183,10 @@ void wakeup_source_add(struct wakeup_source *ws)
 	spin_lock_init(&ws->lock);
 	timer_setup(&ws->timer, pm_wakeup_timer_fn, 0);
 	ws->active = false;
-	ws->last_time = ktime_get();
 
-	spin_lock_irqsave(&events_lock, flags);
+	raw_spin_lock_irqsave(&events_lock, flags);
 	list_add_rcu(&ws->entry, &wakeup_sources);
-	spin_unlock_irqrestore(&events_lock, flags);
+	raw_spin_unlock_irqrestore(&events_lock, flags);
 }
 EXPORT_SYMBOL_GPL(wakeup_source_add);
 
@@ -202,9 +201,9 @@ void wakeup_source_remove(struct wakeup_source *ws)
 	if (WARN_ON(!ws))
 		return;
 
-	spin_lock_irqsave(&events_lock, flags);
+	raw_spin_lock_irqsave(&events_lock, flags);
 	list_del_rcu(&ws->entry);
-	spin_unlock_irqrestore(&events_lock, flags);
+	raw_spin_unlock_irqrestore(&events_lock, flags);
 	synchronize_srcu(&wakeup_srcu);
 }
 EXPORT_SYMBOL_GPL(wakeup_source_remove);
@@ -843,7 +842,7 @@ bool pm_wakeup_pending(void)
 	unsigned long flags;
 	bool ret = false;
 
-	spin_lock_irqsave(&events_lock, flags);
+	raw_spin_lock_irqsave(&events_lock, flags);
 	if (events_check_enabled) {
 		unsigned int cnt, inpr;
 
@@ -851,10 +850,10 @@ bool pm_wakeup_pending(void)
 		ret = (cnt != saved_count || inpr > 0);
 		events_check_enabled = !ret;
 	}
-	spin_unlock_irqrestore(&events_lock, flags);
+	raw_spin_unlock_irqrestore(&events_lock, flags);
 
 	if (ret) {
-		pr_info("PM: Wakeup pending, aborting suspend\n");
+		pr_debug("PM: Wakeup pending, aborting suspend\n");
 		pm_print_active_wakeup_sources();
 	}
 
@@ -940,13 +939,13 @@ bool pm_save_wakeup_count(unsigned int count)
 	unsigned long flags;
 
 	events_check_enabled = false;
-	spin_lock_irqsave(&events_lock, flags);
+	raw_spin_lock_irqsave(&events_lock, flags);
 	split_counters(&cnt, &inpr);
 	if (cnt == count && inpr == 0) {
 		saved_count = count;
 		events_check_enabled = true;
 	}
-	spin_unlock_irqrestore(&events_lock, flags);
+	raw_spin_unlock_irqrestore(&events_lock, flags);
 	return events_check_enabled;
 }
 
@@ -1029,32 +1028,75 @@ static int print_wakeup_source_stats(struct seq_file *m,
 	return 0;
 }
 
-/**
- * wakeup_sources_stats_show - Print wakeup sources statistics information.
- * @m: seq_file to print the statistics into.
- */
-static int wakeup_sources_stats_show(struct seq_file *m, void *unused)
+static void *wakeup_sources_stats_seq_start(struct seq_file *m,
+					loff_t *pos)
 {
 	struct wakeup_source *ws;
-	int srcuidx;
+	loff_t n = *pos;
+	int *srcuidx = m->private;
 
-	seq_puts(m, "name\t\tactive_count\tevent_count\twakeup_count\t"
-		"expire_count\tactive_since\ttotal_time\tmax_time\t"
-		"last_change\tprevent_suspend_time\n");
+	if (n == 0) {
+		seq_puts(m, "name\t\tactive_count\tevent_count\twakeup_count\t"
+			"expire_count\tactive_since\ttotal_time\tmax_time\t"
+			"last_change\tprevent_suspend_time\n");
+	}
 
-	srcuidx = srcu_read_lock(&wakeup_srcu);
-	list_for_each_entry_rcu(ws, &wakeup_sources, entry)
-		print_wakeup_source_stats(m, ws);
-	srcu_read_unlock(&wakeup_srcu, srcuidx);
+	*srcuidx = srcu_read_lock(&wakeup_srcu);
+	list_for_each_entry_rcu(ws, &wakeup_sources, entry) {
+		if (n-- <= 0)
+			return ws;
+	}
+
+	return NULL;
+}
+
+static void *wakeup_sources_stats_seq_next(struct seq_file *m,
+					void *v, loff_t *pos)
+{
+	struct wakeup_source *ws = v;
+	struct wakeup_source *next_ws = NULL;
 
-	print_wakeup_source_stats(m, &deleted_ws);
+	++(*pos);
+
+	list_for_each_entry_continue_rcu(ws, &wakeup_sources, entry) {
+		next_ws = ws;
+		break;
+	}
+
+	return next_ws;
+}
+
+static void wakeup_sources_stats_seq_stop(struct seq_file *m, void *v)
+{
+	int *srcuidx = m->private;
+
+	srcu_read_unlock(&wakeup_srcu, *srcuidx);
+}
+
+/**
+ * wakeup_sources_stats_seq_show - Print wakeup sources statistics information.
+ * @m: seq_file to print the statistics into.
+ * @v: wakeup_source of each iteration
+ */
+static int wakeup_sources_stats_seq_show(struct seq_file *m, void *v)
+{
+	struct wakeup_source *ws = v;
+
+	print_wakeup_source_stats(m, ws);
 
 	return 0;
 }
 
+static const struct seq_operations wakeup_sources_stats_seq_ops = {
+	.start = wakeup_sources_stats_seq_start,
+	.next  = wakeup_sources_stats_seq_next,
+	.stop  = wakeup_sources_stats_seq_stop,
+	.show  = wakeup_sources_stats_seq_show,
+};
+
 static int wakeup_sources_stats_open(struct inode *inode, struct file *file)
 {
-	return single_open(file, wakeup_sources_stats_show, NULL);
+	return seq_open_private(file, &wakeup_sources_stats_seq_ops, sizeof(int));
 }
 
 static const struct file_operations wakeup_sources_stats_fops = {
@@ -1062,7 +1104,7 @@ static const struct file_operations wakeup_sources_stats_fops = {
 	.open = wakeup_sources_stats_open,
 	.read = seq_read,
 	.llseek = seq_lseek,
-	.release = single_release,
+	.release = seq_release_private,
 };
 
 static int __init wakeup_sources_debugfs_init(void)
diff --git a/drivers/base/property.c b/drivers/base/property.c
index 8f205f6461ed..240ab5230ff6 100644
--- a/drivers/base/property.c
+++ b/drivers/base/property.c
@@ -56,6 +56,72 @@ pset_prop_get(const struct property_set *pset, const char *name)
 	return NULL;
 }
 
+static const void *property_get_pointer(const struct property_entry *prop)
+{
+	switch (prop->type) {
+	case DEV_PROP_U8:
+		if (prop->is_array)
+			return prop->pointer.u8_data;
+		return &prop->value.u8_data;
+	case DEV_PROP_U16:
+		if (prop->is_array)
+			return prop->pointer.u16_data;
+		return &prop->value.u16_data;
+	case DEV_PROP_U32:
+		if (prop->is_array)
+			return prop->pointer.u32_data;
+		return &prop->value.u32_data;
+	case DEV_PROP_U64:
+		if (prop->is_array)
+			return prop->pointer.u64_data;
+		return &prop->value.u64_data;
+	case DEV_PROP_STRING:
+		if (prop->is_array)
+			return prop->pointer.str;
+		return &prop->value.str;
+	default:
+		return NULL;
+	}
+}
+
+static void property_set_pointer(struct property_entry *prop, const void *pointer)
+{
+	switch (prop->type) {
+	case DEV_PROP_U8:
+		if (prop->is_array)
+			prop->pointer.u8_data = pointer;
+		else
+			prop->value.u8_data = *((u8 *)pointer);
+		break;
+	case DEV_PROP_U16:
+		if (prop->is_array)
+			prop->pointer.u16_data = pointer;
+		else
+			prop->value.u16_data = *((u16 *)pointer);
+		break;
+	case DEV_PROP_U32:
+		if (prop->is_array)
+			prop->pointer.u32_data = pointer;
+		else
+			prop->value.u32_data = *((u32 *)pointer);
+		break;
+	case DEV_PROP_U64:
+		if (prop->is_array)
+			prop->pointer.u64_data = pointer;
+		else
+			prop->value.u64_data = *((u64 *)pointer);
+		break;
+	case DEV_PROP_STRING:
+		if (prop->is_array)
+			prop->pointer.str = pointer;
+		else
+			prop->value.str = pointer;
+		break;
+	default:
+		break;
+	}
+}
+
 static const void *pset_prop_find(const struct property_set *pset,
 				  const char *propname, size_t length)
 {
@@ -65,10 +131,7 @@ static const void *pset_prop_find(const struct property_set *pset,
 	prop = pset_prop_get(pset, propname);
 	if (!prop)
 		return ERR_PTR(-EINVAL);
-	if (prop->is_array)
-		pointer = prop->pointer.raw_data;
-	else
-		pointer = &prop->value.raw_data;
+	pointer = property_get_pointer(prop);
 	if (!pointer)
 		return ERR_PTR(-ENODATA);
 	if (length > prop->length)
@@ -698,16 +761,17 @@ EXPORT_SYMBOL_GPL(fwnode_property_get_reference_args);
 
 static void property_entry_free_data(const struct property_entry *p)
 {
+	const void *pointer = property_get_pointer(p);
 	size_t i, nval;
 
 	if (p->is_array) {
-		if (p->is_string && p->pointer.str) {
+		if (p->type == DEV_PROP_STRING && p->pointer.str) {
 			nval = p->length / sizeof(const char *);
 			for (i = 0; i < nval; i++)
 				kfree(p->pointer.str[i]);
 		}
-		kfree(p->pointer.raw_data);
-	} else if (p->is_string) {
+		kfree(pointer);
+	} else if (p->type == DEV_PROP_STRING) {
 		kfree(p->value.str);
 	}
 	kfree(p->name);
@@ -716,7 +780,7 @@ static void property_entry_free_data(const struct property_entry *p)
 static int property_copy_string_array(struct property_entry *dst,
 				      const struct property_entry *src)
 {
-	char **d;
+	const char **d;
 	size_t nval = src->length / sizeof(*d);
 	int i;
 
@@ -734,40 +798,44 @@ static int property_copy_string_array(struct property_entry *dst,
 		}
 	}
 
-	dst->pointer.raw_data = d;
+	dst->pointer.str = d;
 	return 0;
 }
 
 static int property_entry_copy_data(struct property_entry *dst,
 				    const struct property_entry *src)
 {
+	const void *pointer = property_get_pointer(src);
+	const void *new;
 	int error;
 
 	if (src->is_array) {
 		if (!src->length)
 			return -ENODATA;
 
-		if (src->is_string) {
+		if (src->type == DEV_PROP_STRING) {
 			error = property_copy_string_array(dst, src);
 			if (error)
 				return error;
+			new = dst->pointer.str;
 		} else {
-			dst->pointer.raw_data = kmemdup(src->pointer.raw_data,
-							src->length, GFP_KERNEL);
-			if (!dst->pointer.raw_data)
+			new = kmemdup(pointer, src->length, GFP_KERNEL);
+			if (!new)
 				return -ENOMEM;
 		}
-	} else if (src->is_string) {
-		dst->value.str = kstrdup(src->value.str, GFP_KERNEL);
-		if (!dst->value.str && src->value.str)
+	} else if (src->type == DEV_PROP_STRING) {
+		new = kstrdup(src->value.str, GFP_KERNEL);
+		if (!new && src->value.str)
 			return -ENOMEM;
 	} else {
-		dst->value.raw_data = src->value.raw_data;
+		new = pointer;
 	}
 
 	dst->length = src->length;
 	dst->is_array = src->is_array;
-	dst->is_string = src->is_string;
+	dst->type = src->type;
+
+	property_set_pointer(dst, new);
 
 	dst->name = kstrdup(src->name, GFP_KERNEL);
 	if (!dst->name)
diff --git a/drivers/base/regmap/regmap-mmio.c b/drivers/base/regmap/regmap-mmio.c
index 5cadfd3394d8..8741fb5f8f54 100644
--- a/drivers/base/regmap/regmap-mmio.c
+++ b/drivers/base/regmap/regmap-mmio.c
@@ -206,7 +206,8 @@ static void regmap_mmio_free_context(void *context)
 
 	if (!IS_ERR(ctx->clk)) {
 		clk_unprepare(ctx->clk);
-		clk_put(ctx->clk);
+		if (!ctx->attached_clk)
+			clk_put(ctx->clk);
 	}
 	kfree(context);
 }
diff --git a/drivers/base/regmap/regmap-slimbus.c b/drivers/base/regmap/regmap-slimbus.c
index c90bee81d954..91d501eda8a9 100644
--- a/drivers/base/regmap/regmap-slimbus.c
+++ b/drivers/base/regmap/regmap-slimbus.c
@@ -41,7 +41,7 @@ static struct regmap_bus regmap_slimbus_bus = {
 static const struct regmap_bus *regmap_get_slimbus(struct slim_device *slim,
 					const struct regmap_config *config)
 {
-	if (config->val_bits == 8 && config->reg_bits == 8)
+	if (config->val_bits == 8 && config->reg_bits == 16)
 		return &regmap_slimbus_bus;
 
 	return ERR_PTR(-ENOTSUPP);
diff --git a/drivers/bcma/main.c b/drivers/bcma/main.c
index e6986c7608f1..fc1f4acdd189 100644
--- a/drivers/bcma/main.c
+++ b/drivers/bcma/main.c
@@ -207,7 +207,7 @@ static void bcma_of_fill_device(struct device *parent,
 
 	core->irq = bcma_of_get_irq(parent, core, 0);
 
-	of_dma_configure(&core->dev, node);
+	of_dma_configure(&core->dev, node, false);
 }
 
 unsigned int bcma_core_irq(struct bcma_device *core, int num)
diff --git a/drivers/block/DAC960.c b/drivers/block/DAC960.c
index f781eff7d23e..6ca77d6047d6 100644
--- a/drivers/block/DAC960.c
+++ b/drivers/block/DAC960.c
@@ -1179,7 +1179,6 @@ static bool DAC960_V1_EnableMemoryMailboxInterface(DAC960_Controller_T
 
   if (pci_set_dma_mask(Controller->PCIDevice, DMA_BIT_MASK(32)))
 	return DAC960_Failure(Controller, "DMA mask out of range");
-  Controller->BounceBufferLimit = DMA_BIT_MASK(32);
 
   if ((hw_type == DAC960_PD_Controller) || (hw_type == DAC960_P_Controller)) {
     CommandMailboxesSize =  0;
@@ -1380,11 +1379,8 @@ static bool DAC960_V2_EnableMemoryMailboxInterface(DAC960_Controller_T
   dma_addr_t	CommandMailboxDMA;
   DAC960_V2_CommandStatus_T CommandStatus;
 
-	if (!pci_set_dma_mask(Controller->PCIDevice, DMA_BIT_MASK(64)))
-		Controller->BounceBufferLimit = DMA_BIT_MASK(64);
-	else if (!pci_set_dma_mask(Controller->PCIDevice, DMA_BIT_MASK(32)))
-		Controller->BounceBufferLimit = DMA_BIT_MASK(32);
-	else
+	if (pci_set_dma_mask(Controller->PCIDevice, DMA_BIT_MASK(64)) &&
+	    pci_set_dma_mask(Controller->PCIDevice, DMA_BIT_MASK(32)))
 		return DAC960_Failure(Controller, "DMA mask out of range");
 
   /* This is a temporary dma mapping, used only in the scope of this function */
@@ -2540,7 +2536,6 @@ static bool DAC960_RegisterBlockDevice(DAC960_Controller_T *Controller)
 		continue;
   	}
   	Controller->RequestQueue[n] = RequestQueue;
-  	blk_queue_bounce_limit(RequestQueue, Controller->BounceBufferLimit);
   	RequestQueue->queuedata = Controller;
 	blk_queue_max_segments(RequestQueue, Controller->DriverScatterGatherLimit);
 	blk_queue_max_hw_sectors(RequestQueue, Controller->MaxBlocksPerCommand);
@@ -6451,19 +6446,6 @@ static int dac960_proc_show(struct seq_file *m, void *v)
   return 0;
 }
 
-static int dac960_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, dac960_proc_show, NULL);
-}
-
-static const struct file_operations dac960_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= dac960_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int dac960_initial_status_proc_show(struct seq_file *m, void *v)
 {
 	DAC960_Controller_T *Controller = (DAC960_Controller_T *)m->private;
@@ -6471,19 +6453,6 @@ static int dac960_initial_status_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int dac960_initial_status_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, dac960_initial_status_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations dac960_initial_status_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= dac960_initial_status_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int dac960_current_status_proc_show(struct seq_file *m, void *v)
 {
   DAC960_Controller_T *Controller = (DAC960_Controller_T *) m->private;
@@ -6517,19 +6486,6 @@ static int dac960_current_status_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int dac960_current_status_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, dac960_current_status_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations dac960_current_status_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= dac960_current_status_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int dac960_user_command_proc_show(struct seq_file *m, void *v)
 {
 	DAC960_Controller_T *Controller = (DAC960_Controller_T *)m->private;
@@ -6584,17 +6540,19 @@ static void DAC960_CreateProcEntries(DAC960_Controller_T *Controller)
 
 	if (DAC960_ProcDirectoryEntry == NULL) {
 		DAC960_ProcDirectoryEntry = proc_mkdir("rd", NULL);
-		proc_create("status", 0, DAC960_ProcDirectoryEntry,
-			    &dac960_proc_fops);
+		proc_create_single("status", 0, DAC960_ProcDirectoryEntry,
+				dac960_proc_show);
 	}
 
 	snprintf(Controller->ControllerName, sizeof(Controller->ControllerName),
 		 "c%d", Controller->ControllerNumber);
 	ControllerProcEntry = proc_mkdir(Controller->ControllerName,
 					 DAC960_ProcDirectoryEntry);
-	proc_create_data("initial_status", 0, ControllerProcEntry, &dac960_initial_status_proc_fops, Controller);
-	proc_create_data("current_status", 0, ControllerProcEntry, &dac960_current_status_proc_fops, Controller);
-	proc_create_data("user_command", S_IWUSR | S_IRUSR, ControllerProcEntry, &dac960_user_command_proc_fops, Controller);
+	proc_create_single_data("initial_status", 0, ControllerProcEntry,
+			dac960_initial_status_proc_show, Controller);
+	proc_create_single_data("current_status", 0, ControllerProcEntry,
+			dac960_current_status_proc_show, Controller);
+	proc_create_data("user_command", 0600, ControllerProcEntry, &dac960_user_command_proc_fops, Controller);
 	Controller->ControllerProcEntry = ControllerProcEntry;
 }
 
diff --git a/drivers/block/DAC960.h b/drivers/block/DAC960.h
index 21aff470d268..1439e651928b 100644
--- a/drivers/block/DAC960.h
+++ b/drivers/block/DAC960.h
@@ -2295,7 +2295,6 @@ typedef struct DAC960_Controller
   unsigned short MaxBlocksPerCommand;
   unsigned short ControllerScatterGatherLimit;
   unsigned short DriverScatterGatherLimit;
-  u64		BounceBufferLimit;
   unsigned int CombinedStatusBufferLength;
   unsigned int InitialStatusLength;
   unsigned int CurrentStatusLength;
diff --git a/drivers/block/aoe/aoeblk.c b/drivers/block/aoe/aoeblk.c
index 6797e6c23c8a..429ebb84b592 100644
--- a/drivers/block/aoe/aoeblk.c
+++ b/drivers/block/aoe/aoeblk.c
@@ -159,14 +159,14 @@ static int aoe_debugfs_open(struct inode *inode, struct file *file)
 	return single_open(file, aoedisk_debugfs_show, inode->i_private);
 }
 
-static DEVICE_ATTR(state, S_IRUGO, aoedisk_show_state, NULL);
-static DEVICE_ATTR(mac, S_IRUGO, aoedisk_show_mac, NULL);
-static DEVICE_ATTR(netif, S_IRUGO, aoedisk_show_netif, NULL);
+static DEVICE_ATTR(state, 0444, aoedisk_show_state, NULL);
+static DEVICE_ATTR(mac, 0444, aoedisk_show_mac, NULL);
+static DEVICE_ATTR(netif, 0444, aoedisk_show_netif, NULL);
 static struct device_attribute dev_attr_firmware_version = {
-	.attr = { .name = "firmware-version", .mode = S_IRUGO },
+	.attr = { .name = "firmware-version", .mode = 0444 },
 	.show = aoedisk_show_fwver,
 };
-static DEVICE_ATTR(payload, S_IRUGO, aoedisk_show_payload, NULL);
+static DEVICE_ATTR(payload, 0444, aoedisk_show_payload, NULL);
 
 static struct attribute *aoe_attrs[] = {
 	&dev_attr_state.attr,
@@ -388,7 +388,6 @@ aoeblk_gdalloc(void *vp)
 			d->aoemajor, d->aoeminor);
 		goto err_mempool;
 	}
-	blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH);
 
 	spin_lock_irqsave(&d->lock, flags);
 	WARN_ON(!(d->flags & DEVFL_GD_NOW));
diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c
index 540bb60cd071..096882e54095 100644
--- a/drivers/block/aoe/aoecmd.c
+++ b/drivers/block/aoe/aoecmd.c
@@ -1032,8 +1032,9 @@ bvcpy(struct sk_buff *skb, struct bio *bio, struct bvec_iter iter, long cnt)
 	iter.bi_size = cnt;
 
 	__bio_for_each_segment(bv, bio, iter, iter) {
-		char *p = page_address(bv.bv_page) + bv.bv_offset;
+		char *p = kmap_atomic(bv.bv_page) + bv.bv_offset;
 		skb_copy_bits(skb, soff, p, bv.bv_len);
+		kunmap_atomic(p);
 		soff += bv.bv_len;
 	}
 }
diff --git a/drivers/block/brd.c b/drivers/block/brd.c
index 66cb0f857f64..bb976598ee43 100644
--- a/drivers/block/brd.c
+++ b/drivers/block/brd.c
@@ -331,15 +331,15 @@ static const struct block_device_operations brd_fops = {
  * And now the modules code and kernel interface.
  */
 static int rd_nr = CONFIG_BLK_DEV_RAM_COUNT;
-module_param(rd_nr, int, S_IRUGO);
+module_param(rd_nr, int, 0444);
 MODULE_PARM_DESC(rd_nr, "Maximum number of brd devices");
 
 unsigned long rd_size = CONFIG_BLK_DEV_RAM_SIZE;
-module_param(rd_size, ulong, S_IRUGO);
+module_param(rd_size, ulong, 0444);
 MODULE_PARM_DESC(rd_size, "Size of each RAM disk in kbytes.");
 
 static int max_part = 1;
-module_param(max_part, int, S_IRUGO);
+module_param(max_part, int, 0444);
 MODULE_PARM_DESC(max_part, "Num Minors to reserve between devices");
 
 MODULE_LICENSE("GPL");
@@ -402,6 +402,10 @@ static struct brd_device *brd_alloc(int i)
 	set_capacity(disk, rd_size * 2);
 	disk->queue->backing_dev_info->capabilities |= BDI_CAP_SYNCHRONOUS_IO;
 
+	/* Tell the block layer that this is not a rotational device */
+	blk_queue_flag_set(QUEUE_FLAG_NONROT, disk->queue);
+	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, disk->queue);
+
 	return brd;
 
 out_free_queue:
diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c
index 9f4e6f502b84..11a85b740327 100644
--- a/drivers/block/drbd/drbd_bitmap.c
+++ b/drivers/block/drbd/drbd_bitmap.c
@@ -977,7 +977,7 @@ static void drbd_bm_endio(struct bio *bio)
 	bm_page_unlock_io(device, idx);
 
 	if (ctx->flags & BM_AIO_COPY_PAGES)
-		mempool_free(bio->bi_io_vec[0].bv_page, drbd_md_io_page_pool);
+		mempool_free(bio->bi_io_vec[0].bv_page, &drbd_md_io_page_pool);
 
 	bio_put(bio);
 
@@ -1014,7 +1014,8 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho
 	bm_set_page_unchanged(b->bm_pages[page_nr]);
 
 	if (ctx->flags & BM_AIO_COPY_PAGES) {
-		page = mempool_alloc(drbd_md_io_page_pool, __GFP_HIGHMEM|__GFP_RECLAIM);
+		page = mempool_alloc(&drbd_md_io_page_pool,
+				GFP_NOIO | __GFP_HIGHMEM);
 		copy_highpage(page, b->bm_pages[page_nr]);
 		bm_store_page_idx(page, page_nr);
 	} else
diff --git a/drivers/block/drbd/drbd_debugfs.c b/drivers/block/drbd/drbd_debugfs.c
index ab21976a87b2..5d5e8d6a8a56 100644
--- a/drivers/block/drbd/drbd_debugfs.c
+++ b/drivers/block/drbd/drbd_debugfs.c
@@ -481,9 +481,9 @@ void drbd_debugfs_resource_add(struct drbd_resource *resource)
 		goto fail;
 	resource->debugfs_res_connections = dentry;
 
-	dentry = debugfs_create_file("in_flight_summary", S_IRUSR|S_IRGRP,
-			resource->debugfs_res, resource,
-			&in_flight_summary_fops);
+	dentry = debugfs_create_file("in_flight_summary", 0440,
+				     resource->debugfs_res, resource,
+				     &in_flight_summary_fops);
 	if (IS_ERR_OR_NULL(dentry))
 		goto fail;
 	resource->debugfs_res_in_flight_summary = dentry;
@@ -645,16 +645,16 @@ void drbd_debugfs_connection_add(struct drbd_connection *connection)
 		goto fail;
 	connection->debugfs_conn = dentry;
 
-	dentry = debugfs_create_file("callback_history", S_IRUSR|S_IRGRP,
-			connection->debugfs_conn, connection,
-			&connection_callback_history_fops);
+	dentry = debugfs_create_file("callback_history", 0440,
+				     connection->debugfs_conn, connection,
+				     &connection_callback_history_fops);
 	if (IS_ERR_OR_NULL(dentry))
 		goto fail;
 	connection->debugfs_conn_callback_history = dentry;
 
-	dentry = debugfs_create_file("oldest_requests", S_IRUSR|S_IRGRP,
-			connection->debugfs_conn, connection,
-			&connection_oldest_requests_fops);
+	dentry = debugfs_create_file("oldest_requests", 0440,
+				     connection->debugfs_conn, connection,
+				     &connection_oldest_requests_fops);
 	if (IS_ERR_OR_NULL(dentry))
 		goto fail;
 	connection->debugfs_conn_oldest_requests = dentry;
@@ -824,7 +824,7 @@ void drbd_debugfs_device_add(struct drbd_device *device)
 	device->debugfs_minor = dentry;
 
 #define DCF(name)	do {					\
-	dentry = debugfs_create_file(#name, S_IRUSR|S_IRGRP,	\
+	dentry = debugfs_create_file(#name, 0440,	\
 			device->debugfs_vol, device,		\
 			&device_ ## name ## _fops);		\
 	if (IS_ERR_OR_NULL(dentry))				\
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index 06ecee1b528e..bc4ed2ed40a2 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -1405,8 +1405,8 @@ extern struct kmem_cache *drbd_request_cache;
 extern struct kmem_cache *drbd_ee_cache;	/* peer requests */
 extern struct kmem_cache *drbd_bm_ext_cache;	/* bitmap extents */
 extern struct kmem_cache *drbd_al_ext_cache;	/* activity log extents */
-extern mempool_t *drbd_request_mempool;
-extern mempool_t *drbd_ee_mempool;
+extern mempool_t drbd_request_mempool;
+extern mempool_t drbd_ee_mempool;
 
 /* drbd's page pool, used to buffer data received from the peer,
  * or data requested by the peer.
@@ -1432,16 +1432,16 @@ extern wait_queue_head_t drbd_pp_wait;
  * 128 should be plenty, currently we probably can get away with as few as 1.
  */
 #define DRBD_MIN_POOL_PAGES	128
-extern mempool_t *drbd_md_io_page_pool;
+extern mempool_t drbd_md_io_page_pool;
 
 /* We also need to make sure we get a bio
  * when we need it for housekeeping purposes */
-extern struct bio_set *drbd_md_io_bio_set;
+extern struct bio_set drbd_md_io_bio_set;
 /* to allocate from that set */
 extern struct bio *bio_alloc_drbd(gfp_t gfp_mask);
 
 /* And a bio_set for cloning */
-extern struct bio_set *drbd_io_bio_set;
+extern struct bio_set drbd_io_bio_set;
 
 extern struct mutex resources_mutex;
 
@@ -1643,7 +1643,7 @@ void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backin
 
 /* drbd_proc.c */
 extern struct proc_dir_entry *drbd_proc;
-extern const struct file_operations drbd_proc_fops;
+int drbd_seq_show(struct seq_file *seq, void *v);
 
 /* drbd_actlog.c */
 extern bool drbd_al_begin_io_prepare(struct drbd_device *device, struct drbd_interval *i);
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 185f1ef00a7c..e8c1c085fadd 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -124,11 +124,11 @@ struct kmem_cache *drbd_request_cache;
 struct kmem_cache *drbd_ee_cache;	/* peer requests */
 struct kmem_cache *drbd_bm_ext_cache;	/* bitmap extents */
 struct kmem_cache *drbd_al_ext_cache;	/* activity log extents */
-mempool_t *drbd_request_mempool;
-mempool_t *drbd_ee_mempool;
-mempool_t *drbd_md_io_page_pool;
-struct bio_set *drbd_md_io_bio_set;
-struct bio_set *drbd_io_bio_set;
+mempool_t drbd_request_mempool;
+mempool_t drbd_ee_mempool;
+mempool_t drbd_md_io_page_pool;
+struct bio_set drbd_md_io_bio_set;
+struct bio_set drbd_io_bio_set;
 
 /* I do not use a standard mempool, because:
    1) I want to hand out the pre-allocated objects first.
@@ -153,10 +153,10 @@ struct bio *bio_alloc_drbd(gfp_t gfp_mask)
 {
 	struct bio *bio;
 
-	if (!drbd_md_io_bio_set)
+	if (!bioset_initialized(&drbd_md_io_bio_set))
 		return bio_alloc(gfp_mask, 1);
 
-	bio = bio_alloc_bioset(gfp_mask, 1, drbd_md_io_bio_set);
+	bio = bio_alloc_bioset(gfp_mask, 1, &drbd_md_io_bio_set);
 	if (!bio)
 		return NULL;
 	return bio;
@@ -2097,16 +2097,11 @@ static void drbd_destroy_mempools(void)
 
 	/* D_ASSERT(device, atomic_read(&drbd_pp_vacant)==0); */
 
-	if (drbd_io_bio_set)
-		bioset_free(drbd_io_bio_set);
-	if (drbd_md_io_bio_set)
-		bioset_free(drbd_md_io_bio_set);
-	if (drbd_md_io_page_pool)
-		mempool_destroy(drbd_md_io_page_pool);
-	if (drbd_ee_mempool)
-		mempool_destroy(drbd_ee_mempool);
-	if (drbd_request_mempool)
-		mempool_destroy(drbd_request_mempool);
+	bioset_exit(&drbd_io_bio_set);
+	bioset_exit(&drbd_md_io_bio_set);
+	mempool_exit(&drbd_md_io_page_pool);
+	mempool_exit(&drbd_ee_mempool);
+	mempool_exit(&drbd_request_mempool);
 	if (drbd_ee_cache)
 		kmem_cache_destroy(drbd_ee_cache);
 	if (drbd_request_cache)
@@ -2116,11 +2111,6 @@ static void drbd_destroy_mempools(void)
 	if (drbd_al_ext_cache)
 		kmem_cache_destroy(drbd_al_ext_cache);
 
-	drbd_io_bio_set      = NULL;
-	drbd_md_io_bio_set   = NULL;
-	drbd_md_io_page_pool = NULL;
-	drbd_ee_mempool      = NULL;
-	drbd_request_mempool = NULL;
 	drbd_ee_cache        = NULL;
 	drbd_request_cache   = NULL;
 	drbd_bm_ext_cache    = NULL;
@@ -2133,18 +2123,7 @@ static int drbd_create_mempools(void)
 {
 	struct page *page;
 	const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * drbd_minor_count;
-	int i;
-
-	/* prepare our caches and mempools */
-	drbd_request_mempool = NULL;
-	drbd_ee_cache        = NULL;
-	drbd_request_cache   = NULL;
-	drbd_bm_ext_cache    = NULL;
-	drbd_al_ext_cache    = NULL;
-	drbd_pp_pool         = NULL;
-	drbd_md_io_page_pool = NULL;
-	drbd_md_io_bio_set   = NULL;
-	drbd_io_bio_set      = NULL;
+	int i, ret;
 
 	/* caches */
 	drbd_request_cache = kmem_cache_create(
@@ -2168,26 +2147,26 @@ static int drbd_create_mempools(void)
 		goto Enomem;
 
 	/* mempools */
-	drbd_io_bio_set = bioset_create(BIO_POOL_SIZE, 0, 0);
-	if (drbd_io_bio_set == NULL)
+	ret = bioset_init(&drbd_io_bio_set, BIO_POOL_SIZE, 0, 0);
+	if (ret)
 		goto Enomem;
 
-	drbd_md_io_bio_set = bioset_create(DRBD_MIN_POOL_PAGES, 0,
-					   BIOSET_NEED_BVECS);
-	if (drbd_md_io_bio_set == NULL)
+	ret = bioset_init(&drbd_md_io_bio_set, DRBD_MIN_POOL_PAGES, 0,
+			  BIOSET_NEED_BVECS);
+	if (ret)
 		goto Enomem;
 
-	drbd_md_io_page_pool = mempool_create_page_pool(DRBD_MIN_POOL_PAGES, 0);
-	if (drbd_md_io_page_pool == NULL)
+	ret = mempool_init_page_pool(&drbd_md_io_page_pool, DRBD_MIN_POOL_PAGES, 0);
+	if (ret)
 		goto Enomem;
 
-	drbd_request_mempool = mempool_create_slab_pool(number,
-		drbd_request_cache);
-	if (drbd_request_mempool == NULL)
+	ret = mempool_init_slab_pool(&drbd_request_mempool, number,
+				     drbd_request_cache);
+	if (ret)
 		goto Enomem;
 
-	drbd_ee_mempool = mempool_create_slab_pool(number, drbd_ee_cache);
-	if (drbd_ee_mempool == NULL)
+	ret = mempool_init_slab_pool(&drbd_ee_mempool, number, drbd_ee_cache);
+	if (ret)
 		goto Enomem;
 
 	/* drbd's page pool */
@@ -3010,7 +2989,8 @@ static int __init drbd_init(void)
 		goto fail;
 
 	err = -ENOMEM;
-	drbd_proc = proc_create_data("drbd", S_IFREG | S_IRUGO , NULL, &drbd_proc_fops, NULL);
+	drbd_proc = proc_create_single("drbd", S_IFREG | 0444 , NULL,
+			drbd_seq_show);
 	if (!drbd_proc)	{
 		pr_err("unable to register proc file\n");
 		goto fail;
diff --git a/drivers/block/drbd/drbd_proc.c b/drivers/block/drbd/drbd_proc.c
index 582caeb0de86..74ef29247bb5 100644
--- a/drivers/block/drbd/drbd_proc.c
+++ b/drivers/block/drbd/drbd_proc.c
@@ -33,18 +33,7 @@
 #include <linux/drbd.h>
 #include "drbd_int.h"
 
-static int drbd_proc_open(struct inode *inode, struct file *file);
-static int drbd_proc_release(struct inode *inode, struct file *file);
-
-
 struct proc_dir_entry *drbd_proc;
-const struct file_operations drbd_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= drbd_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= drbd_proc_release,
-};
 
 static void seq_printf_with_thousands_grouping(struct seq_file *seq, long v)
 {
@@ -235,7 +224,7 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se
 	}
 }
 
-static int drbd_seq_show(struct seq_file *seq, void *v)
+int drbd_seq_show(struct seq_file *seq, void *v)
 {
 	int i, prev_i = -1;
 	const char *sn;
@@ -345,24 +334,3 @@ static int drbd_seq_show(struct seq_file *seq, void *v)
 
 	return 0;
 }
-
-static int drbd_proc_open(struct inode *inode, struct file *file)
-{
-	int err;
-
-	if (try_module_get(THIS_MODULE)) {
-		err = single_open(file, drbd_seq_show, NULL);
-		if (err)
-			module_put(THIS_MODULE);
-		return err;
-	}
-	return -ENODEV;
-}
-
-static int drbd_proc_release(struct inode *inode, struct file *file)
-{
-	module_put(THIS_MODULE);
-	return single_release(inode, file);
-}
-
-/* PROC FS stuff end */
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index c72dee0ef083..be9450f5ad1c 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -378,7 +378,7 @@ drbd_alloc_peer_req(struct drbd_peer_device *peer_device, u64 id, sector_t secto
 	if (drbd_insert_fault(device, DRBD_FAULT_AL_EE))
 		return NULL;
 
-	peer_req = mempool_alloc(drbd_ee_mempool, gfp_mask & ~__GFP_HIGHMEM);
+	peer_req = mempool_alloc(&drbd_ee_mempool, gfp_mask & ~__GFP_HIGHMEM);
 	if (!peer_req) {
 		if (!(gfp_mask & __GFP_NOWARN))
 			drbd_err(device, "%s: allocation failed\n", __func__);
@@ -409,7 +409,7 @@ drbd_alloc_peer_req(struct drbd_peer_device *peer_device, u64 id, sector_t secto
 	return peer_req;
 
  fail:
-	mempool_free(peer_req, drbd_ee_mempool);
+	mempool_free(peer_req, &drbd_ee_mempool);
 	return NULL;
 }
 
@@ -426,7 +426,7 @@ void __drbd_free_peer_req(struct drbd_device *device, struct drbd_peer_request *
 		peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO;
 		drbd_al_complete_io(device, &peer_req->i);
 	}
-	mempool_free(peer_req, drbd_ee_mempool);
+	mempool_free(peer_req, &drbd_ee_mempool);
 }
 
 int drbd_free_peer_reqs(struct drbd_device *device, struct list_head *list)
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index a500e738d929..a47e4987ee46 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -55,7 +55,7 @@ static struct drbd_request *drbd_req_new(struct drbd_device *device, struct bio
 {
 	struct drbd_request *req;
 
-	req = mempool_alloc(drbd_request_mempool, GFP_NOIO);
+	req = mempool_alloc(&drbd_request_mempool, GFP_NOIO);
 	if (!req)
 		return NULL;
 	memset(req, 0, sizeof(*req));
@@ -184,7 +184,7 @@ void drbd_req_destroy(struct kref *kref)
 		}
 	}
 
-	mempool_free(req, drbd_request_mempool);
+	mempool_free(req, &drbd_request_mempool);
 }
 
 static void wake_all_senders(struct drbd_connection *connection)
diff --git a/drivers/block/drbd/drbd_req.h b/drivers/block/drbd/drbd_req.h
index cb97b3b30962..94c654020f0f 100644
--- a/drivers/block/drbd/drbd_req.h
+++ b/drivers/block/drbd/drbd_req.h
@@ -269,7 +269,7 @@ enum drbd_req_state_bits {
 static inline void drbd_req_make_private_bio(struct drbd_request *req, struct bio *bio_src)
 {
 	struct bio *bio;
-	bio = bio_clone_fast(bio_src, GFP_NOIO, drbd_io_bio_set);
+	bio = bio_clone_fast(bio_src, GFP_NOIO, &drbd_io_bio_set);
 
 	req->private_bio = bio;
 
diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c
index 8ec7235fc93b..8871b5044d9e 100644
--- a/drivers/block/floppy.c
+++ b/drivers/block/floppy.c
@@ -4450,7 +4450,7 @@ static ssize_t floppy_cmos_show(struct device *dev,
 	return sprintf(buf, "%X\n", UDP->cmos);
 }
 
-static DEVICE_ATTR(cmos, S_IRUGO, floppy_cmos_show, NULL);
+static DEVICE_ATTR(cmos, 0444, floppy_cmos_show, NULL);
 
 static struct attribute *floppy_dev_attrs[] = {
 	&dev_attr_cmos.attr,
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index 55cf554bc914..4838b0dbaad3 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -732,7 +732,7 @@ static ssize_t loop_attr_do_show_##_name(struct device *d,		\
 	return loop_attr_show(d, b, loop_attr_##_name##_show);		\
 }									\
 static struct device_attribute loop_attr_##_name =			\
-	__ATTR(_name, S_IRUGO, loop_attr_do_show_##_name, NULL);
+	__ATTR(_name, 0444, loop_attr_do_show_##_name, NULL);
 
 static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
 {
@@ -809,16 +809,17 @@ static struct attribute_group loop_attribute_group = {
 	.attrs= loop_attrs,
 };
 
-static int loop_sysfs_init(struct loop_device *lo)
+static void loop_sysfs_init(struct loop_device *lo)
 {
-	return sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
-				  &loop_attribute_group);
+	lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
+						&loop_attribute_group);
 }
 
 static void loop_sysfs_exit(struct loop_device *lo)
 {
-	sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
-			   &loop_attribute_group);
+	if (lo->sysfs_inited)
+		sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
+				   &loop_attribute_group);
 }
 
 static void loop_config_discard(struct loop_device *lo)
@@ -1677,9 +1678,9 @@ static const struct block_device_operations lo_fops = {
  * And now the modules code and kernel interface.
  */
 static int max_loop;
-module_param(max_loop, int, S_IRUGO);
+module_param(max_loop, int, 0444);
 MODULE_PARM_DESC(max_loop, "Maximum number of loop devices");
-module_param(max_part, int, S_IRUGO);
+module_param(max_part, int, 0444);
 MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR);
diff --git a/drivers/block/loop.h b/drivers/block/loop.h
index b78de9879f4f..4d42c7af7de7 100644
--- a/drivers/block/loop.h
+++ b/drivers/block/loop.h
@@ -58,6 +58,7 @@ struct loop_device {
 	struct kthread_worker	worker;
 	struct task_struct	*worker_task;
 	bool			use_dio;
+	bool			sysfs_inited;
 
 	struct request_queue	*lo_queue;
 	struct blk_mq_tag_set	tag_set;
diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c
index 769c551e3d71..c73626decb46 100644
--- a/drivers/block/mtip32xx/mtip32xx.c
+++ b/drivers/block/mtip32xx/mtip32xx.c
@@ -2285,7 +2285,7 @@ static ssize_t mtip_hw_show_status(struct device *dev,
 	return size;
 }
 
-static DEVICE_ATTR(status, S_IRUGO, mtip_hw_show_status, NULL);
+static DEVICE_ATTR(status, 0444, mtip_hw_show_status, NULL);
 
 /* debugsfs entries */
 
@@ -2566,10 +2566,9 @@ static int mtip_hw_debugfs_init(struct driver_data *dd)
 		return -1;
 	}
 
-	debugfs_create_file("flags", S_IRUGO, dd->dfs_node, dd,
-							&mtip_flags_fops);
-	debugfs_create_file("registers", S_IRUGO, dd->dfs_node, dd,
-							&mtip_regs_fops);
+	debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops);
+	debugfs_create_file("registers", 0444, dd->dfs_node, dd,
+			    &mtip_regs_fops);
 
 	return 0;
 }
@@ -2726,15 +2725,11 @@ static void mtip_softirq_done_fn(struct request *rq)
 	blk_mq_end_request(rq, cmd->status);
 }
 
-static void mtip_abort_cmd(struct request *req, void *data,
-							bool reserved)
+static void mtip_abort_cmd(struct request *req, void *data, bool reserved)
 {
 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
 	struct driver_data *dd = data;
 
-	if (!blk_mq_request_started(req))
-		return;
-
 	dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag);
 
 	clear_bit(req->tag, dd->port->cmds_to_issue);
@@ -2742,14 +2737,10 @@ static void mtip_abort_cmd(struct request *req, void *data,
 	mtip_softirq_done_fn(req);
 }
 
-static void mtip_queue_cmd(struct request *req, void *data,
-							bool reserved)
+static void mtip_queue_cmd(struct request *req, void *data, bool reserved)
 {
 	struct driver_data *dd = data;
 
-	if (!blk_mq_request_started(req))
-		return;
-
 	set_bit(req->tag, dd->port->cmds_to_issue);
 	blk_abort_request(req);
 }
@@ -3720,7 +3711,8 @@ static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req,
 		struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
 
 		cmd->status = BLK_STS_TIMEOUT;
-		return BLK_EH_HANDLED;
+		blk_mq_complete_request(req);
+		return BLK_EH_DONE;
 	}
 
 	if (test_bit(req->tag, dd->port->cmds_to_issue))
@@ -3862,7 +3854,6 @@ static int mtip_block_initialize(struct driver_data *dd)
 	blk_queue_max_hw_sectors(dd->queue, 0xffff);
 	blk_queue_max_segment_size(dd->queue, 0x400000);
 	blk_queue_io_min(dd->queue, 4096);
-	blk_queue_bounce_limit(dd->queue, dd->pdev->dma_mask);
 
 	/* Signal trim support */
 	if (dd->trim_supp == true) {
@@ -4273,7 +4264,7 @@ static int mtip_pci_probe(struct pci_dev *pdev,
 	if (!dd->isr_workq) {
 		dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance);
 		rv = -ENOMEM;
-		goto block_initialize_err;
+		goto setmask_err;
 	}
 
 	memset(cpu_list, 0, sizeof(cpu_list));
@@ -4614,7 +4605,7 @@ static int __init mtip_init(void)
 	}
 	if (dfs_parent) {
 		dfs_device_status = debugfs_create_file("device_status",
-					S_IRUGO, dfs_parent, NULL,
+					0444, dfs_parent, NULL,
 					&mtip_device_status_fops);
 		if (IS_ERR_OR_NULL(dfs_device_status)) {
 			pr_err("Error creating device_status node\n");
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index afbc202ca6fd..3ed1ef8ee528 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -166,16 +166,19 @@ static ssize_t pid_show(struct device *dev,
 }
 
 static const struct device_attribute pid_attr = {
-	.attr = { .name = "pid", .mode = S_IRUGO},
+	.attr = { .name = "pid", .mode = 0444},
 	.show = pid_show,
 };
 
 static void nbd_dev_remove(struct nbd_device *nbd)
 {
 	struct gendisk *disk = nbd->disk;
+	struct request_queue *q;
+
 	if (disk) {
+		q = disk->queue;
 		del_gendisk(disk);
-		blk_cleanup_queue(disk->queue);
+		blk_cleanup_queue(q);
 		blk_mq_free_tag_set(&nbd->tag_set);
 		disk->private_data = NULL;
 		put_disk(disk);
@@ -213,7 +216,15 @@ static void nbd_mark_nsock_dead(struct nbd_device *nbd, struct nbd_sock *nsock,
 	}
 	if (!nsock->dead) {
 		kernel_sock_shutdown(nsock->sock, SHUT_RDWR);
-		atomic_dec(&nbd->config->live_connections);
+		if (atomic_dec_return(&nbd->config->live_connections) == 0) {
+			if (test_and_clear_bit(NBD_DISCONNECT_REQUESTED,
+					       &nbd->config->runtime_flags)) {
+				set_bit(NBD_DISCONNECTED,
+					&nbd->config->runtime_flags);
+				dev_info(nbd_to_dev(nbd),
+					"Disconnected due to user request.\n");
+			}
+		}
 	}
 	nsock->dead = true;
 	nsock->pending = NULL;
@@ -231,9 +242,22 @@ static void nbd_size_clear(struct nbd_device *nbd)
 static void nbd_size_update(struct nbd_device *nbd)
 {
 	struct nbd_config *config = nbd->config;
+	struct block_device *bdev = bdget_disk(nbd->disk, 0);
+
+	if (config->flags & NBD_FLAG_SEND_TRIM) {
+		nbd->disk->queue->limits.discard_granularity = config->blksize;
+		blk_queue_max_discard_sectors(nbd->disk->queue, UINT_MAX);
+	}
 	blk_queue_logical_block_size(nbd->disk->queue, config->blksize);
 	blk_queue_physical_block_size(nbd->disk->queue, config->blksize);
 	set_capacity(nbd->disk, config->bytesize >> 9);
+	if (bdev) {
+		if (bdev->bd_disk)
+			bd_set_size(bdev, config->bytesize);
+		else
+			bdev->bd_invalidated = 1;
+		bdput(bdev);
+	}
 	kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);
 }
 
@@ -243,6 +267,8 @@ static void nbd_size_set(struct nbd_device *nbd, loff_t blocksize,
 	struct nbd_config *config = nbd->config;
 	config->blksize = blocksize;
 	config->bytesize = blocksize * nr_blocks;
+	if (nbd->task_recv != NULL)
+		nbd_size_update(nbd);
 }
 
 static void nbd_complete_rq(struct request *req)
@@ -286,13 +312,15 @@ static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req,
 
 	if (!refcount_inc_not_zero(&nbd->config_refs)) {
 		cmd->status = BLK_STS_TIMEOUT;
-		return BLK_EH_HANDLED;
+		goto done;
 	}
 	config = nbd->config;
 
 	if (config->num_connections > 1) {
 		dev_err_ratelimited(nbd_to_dev(nbd),
-				    "Connection timed out, retrying\n");
+				    "Connection timed out, retrying (%d/%d alive)\n",
+				    atomic_read(&config->live_connections),
+				    config->num_connections);
 		/*
 		 * Hooray we have more connections, requeue this IO, the submit
 		 * path will put it on a real connection.
@@ -314,7 +342,7 @@ static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req,
 			}
 			blk_mq_requeue_request(req, true);
 			nbd_config_put(nbd);
-			return BLK_EH_NOT_HANDLED;
+			return BLK_EH_DONE;
 		}
 	} else {
 		dev_err_ratelimited(nbd_to_dev(nbd),
@@ -324,8 +352,9 @@ static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req,
 	cmd->status = BLK_STS_IOERR;
 	sock_shutdown(nbd);
 	nbd_config_put(nbd);
-
-	return BLK_EH_HANDLED;
+done:
+	blk_mq_complete_request(req);
+	return BLK_EH_DONE;
 }
 
 /*
@@ -647,11 +676,8 @@ static void recv_work(struct work_struct *work)
 
 static void nbd_clear_req(struct request *req, void *data, bool reserved)
 {
-	struct nbd_cmd *cmd;
+	struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req);
 
-	if (!blk_mq_request_started(req))
-		return;
-	cmd = blk_mq_rq_to_pdu(req);
 	cmd->status = BLK_STS_IOERR;
 	blk_mq_complete_request(req);
 }
@@ -714,10 +740,9 @@ static int wait_for_reconnect(struct nbd_device *nbd)
 		return 0;
 	if (test_bit(NBD_DISCONNECTED, &config->runtime_flags))
 		return 0;
-	wait_event_timeout(config->conn_wait,
-			   atomic_read(&config->live_connections),
-			   config->dead_conn_timeout);
-	return atomic_read(&config->live_connections);
+	return wait_event_timeout(config->conn_wait,
+				  atomic_read(&config->live_connections) > 0,
+				  config->dead_conn_timeout) > 0;
 }
 
 static int nbd_handle_cmd(struct nbd_cmd *cmd, int index)
@@ -950,10 +975,6 @@ static void nbd_bdev_reset(struct block_device *bdev)
 	if (bdev->bd_openers > 1)
 		return;
 	bd_set_size(bdev, 0);
-	if (max_part > 0) {
-		blkdev_reread_part(bdev);
-		bdev->bd_invalidated = 1;
-	}
 }
 
 static void nbd_parse_flags(struct nbd_device *nbd)
@@ -1040,6 +1061,8 @@ static void nbd_config_put(struct nbd_device *nbd)
 		nbd->config = NULL;
 
 		nbd->tag_set.timeout = 0;
+		nbd->disk->queue->limits.discard_granularity = 0;
+		blk_queue_max_discard_sectors(nbd->disk->queue, UINT_MAX);
 		blk_queue_flag_clear(QUEUE_FLAG_DISCARD, nbd->disk->queue);
 
 		mutex_unlock(&nbd->config_lock);
@@ -1109,7 +1132,6 @@ static int nbd_start_device_ioctl(struct nbd_device *nbd, struct block_device *b
 	if (ret)
 		return ret;
 
-	bd_set_size(bdev, config->bytesize);
 	if (max_part)
 		bdev->bd_invalidated = 1;
 	mutex_unlock(&nbd->config_lock);
@@ -1118,7 +1140,7 @@ static int nbd_start_device_ioctl(struct nbd_device *nbd, struct block_device *b
 	if (ret)
 		sock_shutdown(nbd);
 	mutex_lock(&nbd->config_lock);
-	bd_set_size(bdev, 0);
+	nbd_bdev_reset(bdev);
 	/* user requested, ignore socket errors */
 	if (test_bit(NBD_DISCONNECT_REQUESTED, &config->runtime_flags))
 		ret = 0;
@@ -1269,6 +1291,9 @@ static int nbd_open(struct block_device *bdev, fmode_t mode)
 		refcount_set(&nbd->config_refs, 1);
 		refcount_inc(&nbd->refs);
 		mutex_unlock(&nbd->config_lock);
+		bdev->bd_invalidated = 1;
+	} else if (nbd_disconnected(nbd->config)) {
+		bdev->bd_invalidated = 1;
 	}
 out:
 	mutex_unlock(&nbd_index_mutex);
@@ -1490,8 +1515,8 @@ static int nbd_dev_add(int index)
 	 */
 	blk_queue_flag_set(QUEUE_FLAG_NONROT, disk->queue);
 	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, disk->queue);
-	disk->queue->limits.discard_granularity = 512;
-	blk_queue_max_discard_sectors(disk->queue, UINT_MAX);
+	disk->queue->limits.discard_granularity = 0;
+	blk_queue_max_discard_sectors(disk->queue, 0);
 	blk_queue_max_segment_size(disk->queue, UINT_MAX);
 	blk_queue_max_segments(disk->queue, USHRT_MAX);
 	blk_queue_max_hw_sectors(disk->queue, 65536);
@@ -1755,6 +1780,7 @@ static int nbd_genl_disconnect(struct sk_buff *skb, struct genl_info *info)
 	}
 	mutex_lock(&nbd->config_lock);
 	nbd_disconnect(nbd);
+	nbd_clear_sock(nbd);
 	mutex_unlock(&nbd->config_lock);
 	if (test_and_clear_bit(NBD_HAS_CONFIG_REF,
 			       &nbd->config->runtime_flags))
@@ -2093,7 +2119,8 @@ static int __init nbd_init(void)
 	if (nbds_max > 1UL << (MINORBITS - part_shift))
 		return -EINVAL;
 	recv_workqueue = alloc_workqueue("knbd-recv",
-					 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
+					 WQ_MEM_RECLAIM | WQ_HIGHPRI |
+					 WQ_UNBOUND, 0);
 	if (!recv_workqueue)
 		return -ENOMEM;
 
diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c
index a76553293a31..2bdadd7f1454 100644
--- a/drivers/block/null_blk.c
+++ b/drivers/block/null_blk.c
@@ -157,23 +157,23 @@ enum {
 };
 
 static int g_no_sched;
-module_param_named(no_sched, g_no_sched, int, S_IRUGO);
+module_param_named(no_sched, g_no_sched, int, 0444);
 MODULE_PARM_DESC(no_sched, "No io scheduler");
 
 static int g_submit_queues = 1;
-module_param_named(submit_queues, g_submit_queues, int, S_IRUGO);
+module_param_named(submit_queues, g_submit_queues, int, 0444);
 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
 
 static int g_home_node = NUMA_NO_NODE;
-module_param_named(home_node, g_home_node, int, S_IRUGO);
+module_param_named(home_node, g_home_node, int, 0444);
 MODULE_PARM_DESC(home_node, "Home node for the device");
 
 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
 static char g_timeout_str[80];
-module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), S_IRUGO);
+module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444);
 
 static char g_requeue_str[80];
-module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), S_IRUGO);
+module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444);
 #endif
 
 static int g_queue_mode = NULL_Q_MQ;
@@ -203,27 +203,27 @@ static const struct kernel_param_ops null_queue_mode_param_ops = {
 	.get	= param_get_int,
 };
 
-device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, S_IRUGO);
+device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444);
 MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
 
 static int g_gb = 250;
-module_param_named(gb, g_gb, int, S_IRUGO);
+module_param_named(gb, g_gb, int, 0444);
 MODULE_PARM_DESC(gb, "Size in GB");
 
 static int g_bs = 512;
-module_param_named(bs, g_bs, int, S_IRUGO);
+module_param_named(bs, g_bs, int, 0444);
 MODULE_PARM_DESC(bs, "Block size (in bytes)");
 
 static int nr_devices = 1;
-module_param(nr_devices, int, S_IRUGO);
+module_param(nr_devices, int, 0444);
 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
 
 static bool g_blocking;
-module_param_named(blocking, g_blocking, bool, S_IRUGO);
+module_param_named(blocking, g_blocking, bool, 0444);
 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
 
 static bool shared_tags;
-module_param(shared_tags, bool, S_IRUGO);
+module_param(shared_tags, bool, 0444);
 MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
 
 static int g_irqmode = NULL_IRQ_SOFTIRQ;
@@ -239,19 +239,19 @@ static const struct kernel_param_ops null_irqmode_param_ops = {
 	.get	= param_get_int,
 };
 
-device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, S_IRUGO);
+device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444);
 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
 
 static unsigned long g_completion_nsec = 10000;
-module_param_named(completion_nsec, g_completion_nsec, ulong, S_IRUGO);
+module_param_named(completion_nsec, g_completion_nsec, ulong, 0444);
 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
 
 static int g_hw_queue_depth = 64;
-module_param_named(hw_queue_depth, g_hw_queue_depth, int, S_IRUGO);
+module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444);
 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
 
 static bool g_use_per_node_hctx;
-module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, S_IRUGO);
+module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444);
 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
 
 static struct nullb_device *null_alloc_dev(void);
@@ -1365,7 +1365,8 @@ static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio)
 static enum blk_eh_timer_return null_rq_timed_out_fn(struct request *rq)
 {
 	pr_info("null: rq %p timed out\n", rq);
-	return BLK_EH_HANDLED;
+	blk_mq_complete_request(rq);
+	return BLK_EH_DONE;
 }
 
 static int null_rq_prep_fn(struct request_queue *q, struct request *req)
@@ -1427,7 +1428,8 @@ static void null_request_fn(struct request_queue *q)
 static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res)
 {
 	pr_info("null: rq %p timed out\n", rq);
-	return BLK_EH_HANDLED;
+	blk_mq_complete_request(rq);
+	return BLK_EH_DONE;
 }
 
 static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c
index 27a44b97393a..8961b190e256 100644
--- a/drivers/block/paride/pd.c
+++ b/drivers/block/paride/pd.c
@@ -740,7 +740,7 @@ static int pd_special_command(struct pd_unit *disk,
 {
 	struct request *rq;
 
-	rq = blk_get_request(disk->gd->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	rq = blk_get_request(disk->gd->queue, REQ_OP_DRV_IN, 0);
 	if (IS_ERR(rq))
 		return PTR_ERR(rq);
 
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index c61d20c9f3f8..b3f83cd96f33 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -97,8 +97,8 @@ static int pktdev_major;
 static int write_congestion_on  = PKT_WRITE_CONGESTION_ON;
 static int write_congestion_off = PKT_WRITE_CONGESTION_OFF;
 static struct mutex ctl_mutex;	/* Serialize open/close/setup/teardown */
-static mempool_t *psd_pool;
-static struct bio_set *pkt_bio_set;
+static mempool_t psd_pool;
+static struct bio_set pkt_bio_set;
 
 static struct class	*class_pktcdvd = NULL;    /* /sys/class/pktcdvd */
 static struct dentry	*pkt_debugfs_root = NULL; /* /sys/kernel/debug/pktcdvd */
@@ -478,8 +478,8 @@ static void pkt_debugfs_dev_new(struct pktcdvd_device *pd)
 	if (!pd->dfs_d_root)
 		return;
 
-	pd->dfs_f_info = debugfs_create_file("info", S_IRUGO,
-				pd->dfs_d_root, pd, &debug_fops);
+	pd->dfs_f_info = debugfs_create_file("info", 0444,
+					     pd->dfs_d_root, pd, &debug_fops);
 }
 
 static void pkt_debugfs_dev_remove(struct pktcdvd_device *pd)
@@ -631,7 +631,7 @@ static inline struct pkt_rb_node *pkt_rbtree_next(struct pkt_rb_node *node)
 static void pkt_rbtree_erase(struct pktcdvd_device *pd, struct pkt_rb_node *node)
 {
 	rb_erase(&node->rb_node, &pd->bio_queue);
-	mempool_free(node, pd->rb_pool);
+	mempool_free(node, &pd->rb_pool);
 	pd->bio_queue_size--;
 	BUG_ON(pd->bio_queue_size < 0);
 }
@@ -704,13 +704,13 @@ static int pkt_generic_packet(struct pktcdvd_device *pd, struct packet_command *
 	int ret = 0;
 
 	rq = blk_get_request(q, (cgc->data_direction == CGC_DATA_WRITE) ?
-			     REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, __GFP_RECLAIM);
+			     REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, 0);
 	if (IS_ERR(rq))
 		return PTR_ERR(rq);
 
 	if (cgc->buflen) {
 		ret = blk_rq_map_kern(q, rq, cgc->buffer, cgc->buflen,
-				      __GFP_RECLAIM);
+				      GFP_NOIO);
 		if (ret)
 			goto out;
 	}
@@ -1285,7 +1285,7 @@ static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt)
 	 * Fill-in bvec with data from orig_bios.
 	 */
 	spin_lock(&pkt->lock);
-	bio_copy_data(pkt->w_bio, pkt->orig_bios.head);
+	bio_list_copy_data(pkt->w_bio, pkt->orig_bios.head);
 
 	pkt_set_state(pkt, PACKET_WRITE_WAIT_STATE);
 	spin_unlock(&pkt->lock);
@@ -2303,14 +2303,14 @@ static void pkt_end_io_read_cloned(struct bio *bio)
 	psd->bio->bi_status = bio->bi_status;
 	bio_put(bio);
 	bio_endio(psd->bio);
-	mempool_free(psd, psd_pool);
+	mempool_free(psd, &psd_pool);
 	pkt_bio_finished(pd);
 }
 
 static void pkt_make_request_read(struct pktcdvd_device *pd, struct bio *bio)
 {
-	struct bio *cloned_bio = bio_clone_fast(bio, GFP_NOIO, pkt_bio_set);
-	struct packet_stacked_data *psd = mempool_alloc(psd_pool, GFP_NOIO);
+	struct bio *cloned_bio = bio_clone_fast(bio, GFP_NOIO, &pkt_bio_set);
+	struct packet_stacked_data *psd = mempool_alloc(&psd_pool, GFP_NOIO);
 
 	psd->pd = pd;
 	psd->bio = bio;
@@ -2381,7 +2381,7 @@ static void pkt_make_request_write(struct request_queue *q, struct bio *bio)
 	/*
 	 * No matching packet found. Store the bio in the work queue.
 	 */
-	node = mempool_alloc(pd->rb_pool, GFP_NOIO);
+	node = mempool_alloc(&pd->rb_pool, GFP_NOIO);
 	node->bio = bio;
 	spin_lock(&pd->lock);
 	BUG_ON(pd->bio_queue_size < 0);
@@ -2451,7 +2451,7 @@ static blk_qc_t pkt_make_request(struct request_queue *q, struct bio *bio)
 
 			split = bio_split(bio, last_zone -
 					  bio->bi_iter.bi_sector,
-					  GFP_NOIO, pkt_bio_set);
+					  GFP_NOIO, &pkt_bio_set);
 			bio_chain(split, bio);
 		} else {
 			split = bio;
@@ -2538,18 +2538,6 @@ static int pkt_seq_show(struct seq_file *m, void *p)
 	return 0;
 }
 
-static int pkt_seq_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, pkt_seq_show, PDE_DATA(inode));
-}
-
-static const struct file_operations pkt_proc_fops = {
-	.open	= pkt_seq_open,
-	.read	= seq_read,
-	.llseek	= seq_lseek,
-	.release = single_release
-};
-
 static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev)
 {
 	int i;
@@ -2604,7 +2592,7 @@ static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev)
 		goto out_mem;
 	}
 
-	proc_create_data(pd->name, 0, pkt_proc, &pkt_proc_fops, pd);
+	proc_create_single_data(pd->name, 0, pkt_proc, pkt_seq_show, pd);
 	pkt_dbg(1, pd, "writer mapped to %s\n", bdevname(bdev, b));
 	return 0;
 
@@ -2707,9 +2695,9 @@ static int pkt_setup_dev(dev_t dev, dev_t* pkt_dev)
 	if (!pd)
 		goto out_mutex;
 
-	pd->rb_pool = mempool_create_kmalloc_pool(PKT_RB_POOL_SIZE,
-						  sizeof(struct pkt_rb_node));
-	if (!pd->rb_pool)
+	ret = mempool_init_kmalloc_pool(&pd->rb_pool, PKT_RB_POOL_SIZE,
+					sizeof(struct pkt_rb_node));
+	if (ret)
 		goto out_mem;
 
 	INIT_LIST_HEAD(&pd->cdrw.pkt_free_list);
@@ -2766,7 +2754,7 @@ static int pkt_setup_dev(dev_t dev, dev_t* pkt_dev)
 out_mem2:
 	put_disk(disk);
 out_mem:
-	mempool_destroy(pd->rb_pool);
+	mempool_exit(&pd->rb_pool);
 	kfree(pd);
 out_mutex:
 	mutex_unlock(&ctl_mutex);
@@ -2817,7 +2805,7 @@ static int pkt_remove_dev(dev_t pkt_dev)
 	blk_cleanup_queue(pd->disk->queue);
 	put_disk(pd->disk);
 
-	mempool_destroy(pd->rb_pool);
+	mempool_exit(&pd->rb_pool);
 	kfree(pd);
 
 	/* This is safe: open() is still holding a reference. */
@@ -2914,14 +2902,14 @@ static int __init pkt_init(void)
 
 	mutex_init(&ctl_mutex);
 
-	psd_pool = mempool_create_kmalloc_pool(PSD_POOL_SIZE,
-					sizeof(struct packet_stacked_data));
-	if (!psd_pool)
-		return -ENOMEM;
-	pkt_bio_set = bioset_create(BIO_POOL_SIZE, 0, 0);
-	if (!pkt_bio_set) {
-		mempool_destroy(psd_pool);
-		return -ENOMEM;
+	ret = mempool_init_kmalloc_pool(&psd_pool, PSD_POOL_SIZE,
+				    sizeof(struct packet_stacked_data));
+	if (ret)
+		return ret;
+	ret = bioset_init(&pkt_bio_set, BIO_POOL_SIZE, 0, 0);
+	if (ret) {
+		mempool_exit(&psd_pool);
+		return ret;
 	}
 
 	ret = register_blkdev(pktdev_major, DRIVER_NAME);
@@ -2954,8 +2942,8 @@ static int __init pkt_init(void)
 out:
 	unregister_blkdev(pktdev_major, DRIVER_NAME);
 out2:
-	mempool_destroy(psd_pool);
-	bioset_free(pkt_bio_set);
+	mempool_exit(&psd_pool);
+	bioset_exit(&pkt_bio_set);
 	return ret;
 }
 
@@ -2968,8 +2956,8 @@ static void __exit pkt_exit(void)
 	pkt_sysfs_cleanup();
 
 	unregister_blkdev(pktdev_major, DRIVER_NAME);
-	mempool_destroy(psd_pool);
-	bioset_free(pkt_bio_set);
+	mempool_exit(&psd_pool);
+	bioset_exit(&pkt_bio_set);
 }
 
 MODULE_DESCRIPTION("Packet writing layer for CD/DVD drives");
diff --git a/drivers/block/ps3disk.c b/drivers/block/ps3disk.c
index 075662f2cf46..afe1508d82c6 100644
--- a/drivers/block/ps3disk.c
+++ b/drivers/block/ps3disk.c
@@ -465,8 +465,6 @@ static int ps3disk_probe(struct ps3_system_bus_device *_dev)
 	priv->queue = queue;
 	queue->queuedata = dev;
 
-	blk_queue_bounce_limit(queue, BLK_BOUNCE_HIGH);
-
 	blk_queue_max_hw_sectors(queue, dev->bounce_size >> 9);
 	blk_queue_segment_boundary(queue, -1UL);
 	blk_queue_dma_alignment(queue, dev->blk_size-1);
diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c
index 6a55959cbf78..8fa4533a1249 100644
--- a/drivers/block/ps3vram.c
+++ b/drivers/block/ps3vram.c
@@ -521,26 +521,13 @@ static int ps3vram_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int ps3vram_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ps3vram_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations ps3vram_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= ps3vram_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static void ps3vram_proc_init(struct ps3_system_bus_device *dev)
 {
 	struct ps3vram_priv *priv = ps3_system_bus_get_drvdata(dev);
 	struct proc_dir_entry *pde;
 
-	pde = proc_create_data(DEVICE_NAME, 0444, NULL, &ps3vram_proc_fops,
-			       priv);
+	pde = proc_create_single_data(DEVICE_NAME, 0444, NULL,
+			ps3vram_proc_show, priv);
 	if (!pde)
 		dev_warn(&dev->core, "failed to create /proc entry\n");
 }
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index 33b36fea1d73..af354047ac4b 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -424,7 +424,7 @@ static struct workqueue_struct *rbd_wq;
  * single-major requires >= 0.75 version of userspace rbd utility.
  */
 static bool single_major = true;
-module_param(single_major, bool, S_IRUGO);
+module_param(single_major, bool, 0444);
 MODULE_PARM_DESC(single_major, "Use a single major number for all rbd devices (default: true)");
 
 static ssize_t rbd_add(struct bus_type *bus, const char *buf,
@@ -468,11 +468,11 @@ static ssize_t rbd_supported_features_show(struct bus_type *bus, char *buf)
 	return sprintf(buf, "0x%llx\n", RBD_FEATURES_SUPPORTED);
 }
 
-static BUS_ATTR(add, S_IWUSR, NULL, rbd_add);
-static BUS_ATTR(remove, S_IWUSR, NULL, rbd_remove);
-static BUS_ATTR(add_single_major, S_IWUSR, NULL, rbd_add_single_major);
-static BUS_ATTR(remove_single_major, S_IWUSR, NULL, rbd_remove_single_major);
-static BUS_ATTR(supported_features, S_IRUGO, rbd_supported_features_show, NULL);
+static BUS_ATTR(add, 0200, NULL, rbd_add);
+static BUS_ATTR(remove, 0200, NULL, rbd_remove);
+static BUS_ATTR(add_single_major, 0200, NULL, rbd_add_single_major);
+static BUS_ATTR(remove_single_major, 0200, NULL, rbd_remove_single_major);
+static BUS_ATTR(supported_features, 0444, rbd_supported_features_show, NULL);
 
 static struct attribute *rbd_bus_attrs[] = {
 	&bus_attr_add.attr,
@@ -4204,22 +4204,22 @@ static ssize_t rbd_image_refresh(struct device *dev,
 	return size;
 }
 
-static DEVICE_ATTR(size, S_IRUGO, rbd_size_show, NULL);
-static DEVICE_ATTR(features, S_IRUGO, rbd_features_show, NULL);
-static DEVICE_ATTR(major, S_IRUGO, rbd_major_show, NULL);
-static DEVICE_ATTR(minor, S_IRUGO, rbd_minor_show, NULL);
-static DEVICE_ATTR(client_addr, S_IRUGO, rbd_client_addr_show, NULL);
-static DEVICE_ATTR(client_id, S_IRUGO, rbd_client_id_show, NULL);
-static DEVICE_ATTR(cluster_fsid, S_IRUGO, rbd_cluster_fsid_show, NULL);
-static DEVICE_ATTR(config_info, S_IRUSR, rbd_config_info_show, NULL);
-static DEVICE_ATTR(pool, S_IRUGO, rbd_pool_show, NULL);
-static DEVICE_ATTR(pool_id, S_IRUGO, rbd_pool_id_show, NULL);
-static DEVICE_ATTR(name, S_IRUGO, rbd_name_show, NULL);
-static DEVICE_ATTR(image_id, S_IRUGO, rbd_image_id_show, NULL);
-static DEVICE_ATTR(refresh, S_IWUSR, NULL, rbd_image_refresh);
-static DEVICE_ATTR(current_snap, S_IRUGO, rbd_snap_show, NULL);
-static DEVICE_ATTR(snap_id, S_IRUGO, rbd_snap_id_show, NULL);
-static DEVICE_ATTR(parent, S_IRUGO, rbd_parent_show, NULL);
+static DEVICE_ATTR(size, 0444, rbd_size_show, NULL);
+static DEVICE_ATTR(features, 0444, rbd_features_show, NULL);
+static DEVICE_ATTR(major, 0444, rbd_major_show, NULL);
+static DEVICE_ATTR(minor, 0444, rbd_minor_show, NULL);
+static DEVICE_ATTR(client_addr, 0444, rbd_client_addr_show, NULL);
+static DEVICE_ATTR(client_id, 0444, rbd_client_id_show, NULL);
+static DEVICE_ATTR(cluster_fsid, 0444, rbd_cluster_fsid_show, NULL);
+static DEVICE_ATTR(config_info, 0400, rbd_config_info_show, NULL);
+static DEVICE_ATTR(pool, 0444, rbd_pool_show, NULL);
+static DEVICE_ATTR(pool_id, 0444, rbd_pool_id_show, NULL);
+static DEVICE_ATTR(name, 0444, rbd_name_show, NULL);
+static DEVICE_ATTR(image_id, 0444, rbd_image_id_show, NULL);
+static DEVICE_ATTR(refresh, 0200, NULL, rbd_image_refresh);
+static DEVICE_ATTR(current_snap, 0444, rbd_snap_show, NULL);
+static DEVICE_ATTR(snap_id, 0444, rbd_snap_id_show, NULL);
+static DEVICE_ATTR(parent, 0444, rbd_parent_show, NULL);
 
 static struct attribute *rbd_attrs[] = {
 	&dev_attr_size.attr,
diff --git a/drivers/block/rsxx/core.c b/drivers/block/rsxx/core.c
index 34997df132e2..09537bee387f 100644
--- a/drivers/block/rsxx/core.c
+++ b/drivers/block/rsxx/core.c
@@ -247,19 +247,19 @@ static void rsxx_debugfs_dev_new(struct rsxx_cardinfo *card)
 	if (IS_ERR_OR_NULL(card->debugfs_dir))
 		goto failed_debugfs_dir;
 
-	debugfs_stats = debugfs_create_file("stats", S_IRUGO,
+	debugfs_stats = debugfs_create_file("stats", 0444,
 					    card->debugfs_dir, card,
 					    &debugfs_stats_fops);
 	if (IS_ERR_OR_NULL(debugfs_stats))
 		goto failed_debugfs_stats;
 
-	debugfs_pci_regs = debugfs_create_file("pci_regs", S_IRUGO,
+	debugfs_pci_regs = debugfs_create_file("pci_regs", 0444,
 					       card->debugfs_dir, card,
 					       &debugfs_pci_regs_fops);
 	if (IS_ERR_OR_NULL(debugfs_pci_regs))
 		goto failed_debugfs_pci_regs;
 
-	debugfs_cram = debugfs_create_file("cram", S_IRUGO | S_IWUSR,
+	debugfs_cram = debugfs_create_file("cram", 0644,
 					   card->debugfs_dir, card,
 					   &debugfs_cram_fops);
 	if (IS_ERR_OR_NULL(debugfs_cram))
diff --git a/drivers/block/sx8.c b/drivers/block/sx8.c
index 08586dc14e85..4d90e5eba2f5 100644
--- a/drivers/block/sx8.c
+++ b/drivers/block/sx8.c
@@ -567,7 +567,7 @@ static struct carm_request *carm_get_special(struct carm_host *host)
 	if (!crq)
 		return NULL;
 
-	rq = blk_get_request(host->oob_q, REQ_OP_DRV_OUT, GFP_KERNEL);
+	rq = blk_get_request(host->oob_q, REQ_OP_DRV_OUT, 0);
 	if (IS_ERR(rq)) {
 		spin_lock_irqsave(&host->lock, flags);
 		carm_put_request(host, crq);
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 4a07593c2efd..23752dc99b00 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -298,7 +298,7 @@ static int virtblk_get_id(struct gendisk *disk, char *id_str)
 	struct request *req;
 	int err;
 
-	req = blk_get_request(q, REQ_OP_DRV_IN, GFP_KERNEL);
+	req = blk_get_request(q, REQ_OP_DRV_IN, 0);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 
@@ -371,7 +371,7 @@ static ssize_t virtblk_serial_show(struct device *dev,
 	return err;
 }
 
-static DEVICE_ATTR(serial, S_IRUGO, virtblk_serial_show, NULL);
+static DEVICE_ATTR(serial, 0444, virtblk_serial_show, NULL);
 
 /* The queue's logical block size must be set before calling this */
 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize)
@@ -576,10 +576,10 @@ virtblk_cache_type_show(struct device *dev, struct device_attribute *attr,
 }
 
 static const struct device_attribute dev_attr_cache_type_ro =
-	__ATTR(cache_type, S_IRUGO,
+	__ATTR(cache_type, 0444,
 	       virtblk_cache_type_show, NULL);
 static const struct device_attribute dev_attr_cache_type_rw =
-	__ATTR(cache_type, S_IRUGO|S_IWUSR,
+	__ATTR(cache_type, 0644,
 	       virtblk_cache_type_show, virtblk_cache_type_store);
 
 static int virtblk_init_request(struct blk_mq_tag_set *set, struct request *rq,
diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c
index 987d665e82de..b55b245e8052 100644
--- a/drivers/block/xen-blkback/blkback.c
+++ b/drivers/block/xen-blkback/blkback.c
@@ -98,7 +98,7 @@ MODULE_PARM_DESC(max_queues,
  * backend, 4KB page granularity is used.
  */
 unsigned int xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
-module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, S_IRUGO);
+module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, 0444);
 MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring");
 /*
  * The LRU mechanism to clean the lists of persistent grants needs to
diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c
index 21c1be1eb226..66412eededda 100644
--- a/drivers/block/xen-blkback/xenbus.c
+++ b/drivers/block/xen-blkback/xenbus.c
@@ -367,7 +367,7 @@ int __init xen_blkif_interface_init(void)
 out:									\
 		return sprintf(buf, format, result);			\
 	}								\
-	static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
+	static DEVICE_ATTR(name, 0444, show_##name, NULL)
 
 VBD_SHOW_ALLRING(oo_req,  "%llu\n");
 VBD_SHOW_ALLRING(rd_req,  "%llu\n");
@@ -403,7 +403,7 @@ static const struct attribute_group xen_vbdstat_group = {
 									\
 		return sprintf(buf, format, ##args);			\
 	}								\
-	static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
+	static DEVICE_ATTR(name, 0444, show_##name, NULL)
 
 VBD_SHOW(physical_device, "%x:%x\n", be->major, be->minor);
 VBD_SHOW(mode, "%s\n", be->mode);
diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c
index 2a8e7813bd1a..ae00a82f350b 100644
--- a/drivers/block/xen-blkfront.c
+++ b/drivers/block/xen-blkfront.c
@@ -129,13 +129,12 @@ static const struct block_device_operations xlvbd_block_fops;
  */
 
 static unsigned int xen_blkif_max_segments = 32;
-module_param_named(max_indirect_segments, xen_blkif_max_segments, uint,
-		   S_IRUGO);
+module_param_named(max_indirect_segments, xen_blkif_max_segments, uint, 0444);
 MODULE_PARM_DESC(max_indirect_segments,
 		 "Maximum amount of segments in indirect requests (default is 32)");
 
 static unsigned int xen_blkif_max_queues = 4;
-module_param_named(max_queues, xen_blkif_max_queues, uint, S_IRUGO);
+module_param_named(max_queues, xen_blkif_max_queues, uint, 0444);
 MODULE_PARM_DESC(max_queues, "Maximum number of hardware queues/rings used per virtual disk");
 
 /*
@@ -143,7 +142,7 @@ MODULE_PARM_DESC(max_queues, "Maximum number of hardware queues/rings used per v
  * backend, 4KB page granularity is used.
  */
 static unsigned int xen_blkif_max_ring_order;
-module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, S_IRUGO);
+module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, 0444);
 MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring");
 
 #define BLK_RING_SIZE(info)	\
diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig
index ac3a31d433b2..635235759a0a 100644
--- a/drivers/block/zram/Kconfig
+++ b/drivers/block/zram/Kconfig
@@ -13,7 +13,7 @@ config ZRAM
 	  It has several use cases, for example: /tmp storage, use as swap
 	  disks and maybe many more.
 
-	  See zram.txt for more information.
+	  See Documentation/blockdev/zram.txt for more information.
 
 config ZRAM_WRITEBACK
        bool "Write back incompressible page to backing device"
@@ -25,4 +25,14 @@ config ZRAM_WRITEBACK
 	 For this feature, admin should set up backing device via
 	 /sys/block/zramX/backing_dev.
 
-	 See zram.txt for more infomration.
+	 See Documentation/blockdev/zram.txt for more information.
+
+config ZRAM_MEMORY_TRACKING
+	bool "Track zRam block status"
+	depends on ZRAM && DEBUG_FS
+	help
+	  With this feature, admin can track the state of allocated blocks
+	  of zRAM. Admin could see the information via
+	  /sys/kernel/debug/zram/zramX/block_state.
+
+	  See Documentation/blockdev/zram.txt for more information.
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index 0f3fadd71230..da51293e7c03 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -31,6 +31,7 @@
 #include <linux/err.h>
 #include <linux/idr.h>
 #include <linux/sysfs.h>
+#include <linux/debugfs.h>
 #include <linux/cpuhotplug.h>
 
 #include "zram_drv.h"
@@ -52,11 +53,28 @@ static size_t huge_class_size;
 
 static void zram_free_page(struct zram *zram, size_t index);
 
+static void zram_slot_lock(struct zram *zram, u32 index)
+{
+	bit_spin_lock(ZRAM_LOCK, &zram->table[index].value);
+}
+
+static void zram_slot_unlock(struct zram *zram, u32 index)
+{
+	bit_spin_unlock(ZRAM_LOCK, &zram->table[index].value);
+}
+
 static inline bool init_done(struct zram *zram)
 {
 	return zram->disksize;
 }
 
+static inline bool zram_allocated(struct zram *zram, u32 index)
+{
+
+	return (zram->table[index].value >> (ZRAM_FLAG_SHIFT + 1)) ||
+					zram->table[index].handle;
+}
+
 static inline struct zram *dev_to_zram(struct device *dev)
 {
 	return (struct zram *)dev_to_disk(dev)->private_data;
@@ -73,7 +91,7 @@ static void zram_set_handle(struct zram *zram, u32 index, unsigned long handle)
 }
 
 /* flag operations require table entry bit_spin_lock() being held */
-static int zram_test_flag(struct zram *zram, u32 index,
+static bool zram_test_flag(struct zram *zram, u32 index,
 			enum zram_pageflags flag)
 {
 	return zram->table[index].value & BIT(flag);
@@ -600,6 +618,114 @@ static int read_from_bdev(struct zram *zram, struct bio_vec *bvec,
 static void zram_wb_clear(struct zram *zram, u32 index) {}
 #endif
 
+#ifdef CONFIG_ZRAM_MEMORY_TRACKING
+
+static struct dentry *zram_debugfs_root;
+
+static void zram_debugfs_create(void)
+{
+	zram_debugfs_root = debugfs_create_dir("zram", NULL);
+}
+
+static void zram_debugfs_destroy(void)
+{
+	debugfs_remove_recursive(zram_debugfs_root);
+}
+
+static void zram_accessed(struct zram *zram, u32 index)
+{
+	zram->table[index].ac_time = ktime_get_boottime();
+}
+
+static void zram_reset_access(struct zram *zram, u32 index)
+{
+	zram->table[index].ac_time = 0;
+}
+
+static ssize_t read_block_state(struct file *file, char __user *buf,
+				size_t count, loff_t *ppos)
+{
+	char *kbuf;
+	ssize_t index, written = 0;
+	struct zram *zram = file->private_data;
+	unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
+	struct timespec64 ts;
+
+	kbuf = kvmalloc(count, GFP_KERNEL);
+	if (!kbuf)
+		return -ENOMEM;
+
+	down_read(&zram->init_lock);
+	if (!init_done(zram)) {
+		up_read(&zram->init_lock);
+		kvfree(kbuf);
+		return -EINVAL;
+	}
+
+	for (index = *ppos; index < nr_pages; index++) {
+		int copied;
+
+		zram_slot_lock(zram, index);
+		if (!zram_allocated(zram, index))
+			goto next;
+
+		ts = ktime_to_timespec64(zram->table[index].ac_time);
+		copied = snprintf(kbuf + written, count,
+			"%12zd %12lld.%06lu %c%c%c\n",
+			index, (s64)ts.tv_sec,
+			ts.tv_nsec / NSEC_PER_USEC,
+			zram_test_flag(zram, index, ZRAM_SAME) ? 's' : '.',
+			zram_test_flag(zram, index, ZRAM_WB) ? 'w' : '.',
+			zram_test_flag(zram, index, ZRAM_HUGE) ? 'h' : '.');
+
+		if (count < copied) {
+			zram_slot_unlock(zram, index);
+			break;
+		}
+		written += copied;
+		count -= copied;
+next:
+		zram_slot_unlock(zram, index);
+		*ppos += 1;
+	}
+
+	up_read(&zram->init_lock);
+	if (copy_to_user(buf, kbuf, written))
+		written = -EFAULT;
+	kvfree(kbuf);
+
+	return written;
+}
+
+static const struct file_operations proc_zram_block_state_op = {
+	.open = simple_open,
+	.read = read_block_state,
+	.llseek = default_llseek,
+};
+
+static void zram_debugfs_register(struct zram *zram)
+{
+	if (!zram_debugfs_root)
+		return;
+
+	zram->debugfs_dir = debugfs_create_dir(zram->disk->disk_name,
+						zram_debugfs_root);
+	debugfs_create_file("block_state", 0400, zram->debugfs_dir,
+				zram, &proc_zram_block_state_op);
+}
+
+static void zram_debugfs_unregister(struct zram *zram)
+{
+	debugfs_remove_recursive(zram->debugfs_dir);
+}
+#else
+static void zram_debugfs_create(void) {};
+static void zram_debugfs_destroy(void) {};
+static void zram_accessed(struct zram *zram, u32 index) {};
+static void zram_reset_access(struct zram *zram, u32 index) {};
+static void zram_debugfs_register(struct zram *zram) {};
+static void zram_debugfs_unregister(struct zram *zram) {};
+#endif
 
 /*
  * We switched to per-cpu streams and this attr is not needed anymore.
@@ -719,14 +845,15 @@ static ssize_t mm_stat_show(struct device *dev,
 	max_used = atomic_long_read(&zram->stats.max_used_pages);
 
 	ret = scnprintf(buf, PAGE_SIZE,
-			"%8llu %8llu %8llu %8lu %8ld %8llu %8lu\n",
+			"%8llu %8llu %8llu %8lu %8ld %8llu %8lu %8llu\n",
 			orig_size << PAGE_SHIFT,
 			(u64)atomic64_read(&zram->stats.compr_data_size),
 			mem_used << PAGE_SHIFT,
 			zram->limit_pages << PAGE_SHIFT,
 			max_used << PAGE_SHIFT,
 			(u64)atomic64_read(&zram->stats.same_pages),
-			pool_stats.pages_compacted);
+			pool_stats.pages_compacted,
+			(u64)atomic64_read(&zram->stats.huge_pages));
 	up_read(&zram->init_lock);
 
 	return ret;
@@ -753,16 +880,6 @@ static DEVICE_ATTR_RO(io_stat);
 static DEVICE_ATTR_RO(mm_stat);
 static DEVICE_ATTR_RO(debug_stat);
 
-static void zram_slot_lock(struct zram *zram, u32 index)
-{
-	bit_spin_lock(ZRAM_ACCESS, &zram->table[index].value);
-}
-
-static void zram_slot_unlock(struct zram *zram, u32 index)
-{
-	bit_spin_unlock(ZRAM_ACCESS, &zram->table[index].value);
-}
-
 static void zram_meta_free(struct zram *zram, u64 disksize)
 {
 	size_t num_pages = disksize >> PAGE_SHIFT;
@@ -805,6 +922,13 @@ static void zram_free_page(struct zram *zram, size_t index)
 {
 	unsigned long handle;
 
+	zram_reset_access(zram, index);
+
+	if (zram_test_flag(zram, index, ZRAM_HUGE)) {
+		zram_clear_flag(zram, index, ZRAM_HUGE);
+		atomic64_dec(&zram->stats.huge_pages);
+	}
+
 	if (zram_wb_enabled(zram) && zram_test_flag(zram, index, ZRAM_WB)) {
 		zram_wb_clear(zram, index);
 		atomic64_dec(&zram->stats.pages_stored);
@@ -973,6 +1097,7 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
 	}
 
 	if (unlikely(comp_len >= huge_class_size)) {
+		comp_len = PAGE_SIZE;
 		if (zram_wb_enabled(zram) && allow_wb) {
 			zcomp_stream_put(zram->comp);
 			ret = write_to_bdev(zram, bvec, index, bio, &element);
@@ -984,7 +1109,6 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
 			allow_wb = false;
 			goto compress_again;
 		}
-		comp_len = PAGE_SIZE;
 	}
 
 	/*
@@ -1046,6 +1170,11 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
 	zram_slot_lock(zram, index);
 	zram_free_page(zram, index);
 
+	if (comp_len == PAGE_SIZE) {
+		zram_set_flag(zram, index, ZRAM_HUGE);
+		atomic64_inc(&zram->stats.huge_pages);
+	}
+
 	if (flags) {
 		zram_set_flag(zram, index, flags);
 		zram_set_element(zram, index, element);
@@ -1166,6 +1295,10 @@ static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
 
 	generic_end_io_acct(q, rw_acct, &zram->disk->part0, start_time);
 
+	zram_slot_lock(zram, index);
+	zram_accessed(zram, index);
+	zram_slot_unlock(zram, index);
+
 	if (unlikely(ret < 0)) {
 		if (!is_write)
 			atomic64_inc(&zram->stats.failed_reads);
@@ -1577,6 +1710,7 @@ static int zram_add(void)
 	}
 	strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
 
+	zram_debugfs_register(zram);
 	pr_info("Added device: %s\n", zram->disk->disk_name);
 	return device_id;
 
@@ -1610,6 +1744,7 @@ static int zram_remove(struct zram *zram)
 	zram->claim = true;
 	mutex_unlock(&bdev->bd_mutex);
 
+	zram_debugfs_unregister(zram);
 	/*
 	 * Remove sysfs first, so no one will perform a disksize
 	 * store while we destroy the devices. This also helps during
@@ -1712,6 +1847,7 @@ static void destroy_devices(void)
 {
 	class_unregister(&zram_control_class);
 	idr_for_each(&zram_index_idr, &zram_remove_cb, NULL);
+	zram_debugfs_destroy();
 	idr_destroy(&zram_index_idr);
 	unregister_blkdev(zram_major, "zram");
 	cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE);
@@ -1733,6 +1869,7 @@ static int __init zram_init(void)
 		return ret;
 	}
 
+	zram_debugfs_create();
 	zram_major = register_blkdev(0, "zram");
 	if (zram_major <= 0) {
 		pr_err("Unable to get major number\n");
diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h
index 008861220723..72c8584b6dff 100644
--- a/drivers/block/zram/zram_drv.h
+++ b/drivers/block/zram/zram_drv.h
@@ -43,10 +43,11 @@
 
 /* Flags for zram pages (table[page_no].value) */
 enum zram_pageflags {
-	/* Page consists the same element */
-	ZRAM_SAME = ZRAM_FLAG_SHIFT,
-	ZRAM_ACCESS,	/* page is now accessed */
+	/* zram slot is locked */
+	ZRAM_LOCK = ZRAM_FLAG_SHIFT,
+	ZRAM_SAME,	/* Page consists the same element */
 	ZRAM_WB,	/* page is stored on backing_device */
+	ZRAM_HUGE,	/* Incompressible page */
 
 	__NR_ZRAM_PAGEFLAGS,
 };
@@ -60,6 +61,9 @@ struct zram_table_entry {
 		unsigned long element;
 	};
 	unsigned long value;
+#ifdef CONFIG_ZRAM_MEMORY_TRACKING
+	ktime_t ac_time;
+#endif
 };
 
 struct zram_stats {
@@ -71,6 +75,7 @@ struct zram_stats {
 	atomic64_t invalid_io;	/* non-page-aligned I/O requests */
 	atomic64_t notify_free;	/* no. of swap slot free notifications */
 	atomic64_t same_pages;		/* no. of same element filled pages */
+	atomic64_t huge_pages;		/* no. of huge pages */
 	atomic64_t pages_stored;	/* no. of pages currently stored */
 	atomic_long_t max_used_pages;	/* no. of maximum pages stored */
 	atomic64_t writestall;		/* no. of write slow paths */
@@ -107,5 +112,8 @@ struct zram {
 	unsigned long nr_pages;
 	spinlock_t bitmap_lock;
 #endif
+#ifdef CONFIG_ZRAM_MEMORY_TRACKING
+	struct dentry *debugfs_dir;
+#endif
 };
 #endif
diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig
index 010f5f579e68..f3c643a0473c 100644
--- a/drivers/bluetooth/Kconfig
+++ b/drivers/bluetooth/Kconfig
@@ -197,6 +197,7 @@ config BT_HCIUART_BCM
 config BT_HCIUART_QCA
 	bool "Qualcomm Atheros protocol support"
 	depends on BT_HCIUART
+	depends on BT_HCIUART_SERDEV
 	select BT_HCIUART_H4
 	select BT_QCA
 	help
diff --git a/drivers/bluetooth/btbcm.c b/drivers/bluetooth/btbcm.c
index 6659f113042c..99cde1f9467d 100644
--- a/drivers/bluetooth/btbcm.c
+++ b/drivers/bluetooth/btbcm.c
@@ -315,10 +315,12 @@ static int btbcm_read_info(struct hci_dev *hdev)
 	return 0;
 }
 
-static const struct {
+struct bcm_subver_table {
 	u16 subver;
 	const char *name;
-} bcm_uart_subver_table[] = {
+};
+
+static const struct bcm_subver_table bcm_uart_subver_table[] = {
 	{ 0x4103, "BCM4330B1"	},	/* 002.001.003 */
 	{ 0x410e, "BCM43341B0"	},	/* 002.001.014 */
 	{ 0x4406, "BCM4324B3"	},	/* 002.004.006 */
@@ -330,12 +332,28 @@ static const struct {
 	{ }
 };
 
-int btbcm_initialize(struct hci_dev *hdev, char *fw_name, size_t len)
+static const struct bcm_subver_table bcm_usb_subver_table[] = {
+	{ 0x210b, "BCM43142A0"	},	/* 001.001.011 */
+	{ 0x2112, "BCM4314A0"	},	/* 001.001.018 */
+	{ 0x2118, "BCM20702A0"	},	/* 001.001.024 */
+	{ 0x2126, "BCM4335A0"	},	/* 001.001.038 */
+	{ 0x220e, "BCM20702A1"	},	/* 001.002.014 */
+	{ 0x230f, "BCM4354A2"	},	/* 001.003.015 */
+	{ 0x4106, "BCM4335B0"	},	/* 002.001.006 */
+	{ 0x410e, "BCM20702B0"	},	/* 002.001.014 */
+	{ 0x6109, "BCM4335C0"	},	/* 003.001.009 */
+	{ 0x610c, "BCM4354"	},	/* 003.001.012 */
+	{ }
+};
+
+int btbcm_initialize(struct hci_dev *hdev, char *fw_name, size_t len,
+		     bool reinit)
 {
-	u16 subver, rev;
-	const char *hw_name = NULL;
+	u16 subver, rev, pid, vid;
+	const char *hw_name = "BCM";
 	struct sk_buff *skb;
 	struct hci_rp_read_local_version *ver;
+	const struct bcm_subver_table *bcm_subver_table;
 	int i, err;
 
 	/* Reset */
@@ -354,30 +372,44 @@ int btbcm_initialize(struct hci_dev *hdev, char *fw_name, size_t len)
 	kfree_skb(skb);
 
 	/* Read controller information */
-	err = btbcm_read_info(hdev);
-	if (err)
-		return err;
+	if (!reinit) {
+		err = btbcm_read_info(hdev);
+		if (err)
+			return err;
+	}
 
-	switch ((rev & 0xf000) >> 12) {
-	case 0:
-	case 1:
-	case 2:
-	case 3:
-		for (i = 0; bcm_uart_subver_table[i].name; i++) {
-			if (subver == bcm_uart_subver_table[i].subver) {
-				hw_name = bcm_uart_subver_table[i].name;
-				break;
-			}
+	/* Upper nibble of rev should be between 0 and 3? */
+	if (((rev & 0xf000) >> 12) > 3)
+		return 0;
+
+	bcm_subver_table = (hdev->bus == HCI_USB) ? bcm_usb_subver_table :
+						    bcm_uart_subver_table;
+
+	for (i = 0; bcm_subver_table[i].name; i++) {
+		if (subver == bcm_subver_table[i].subver) {
+			hw_name = bcm_subver_table[i].name;
+			break;
 		}
+	}
 
-		snprintf(fw_name, len, "brcm/%s.hcd", hw_name ? : "BCM");
-		break;
-	default:
-		return 0;
+	if (hdev->bus == HCI_USB) {
+		/* Read USB Product Info */
+		skb = btbcm_read_usb_product(hdev);
+		if (IS_ERR(skb))
+			return PTR_ERR(skb);
+
+		vid = get_unaligned_le16(skb->data + 1);
+		pid = get_unaligned_le16(skb->data + 3);
+		kfree_skb(skb);
+
+		snprintf(fw_name, len, "brcm/%s-%4.4x-%4.4x.hcd",
+			 hw_name, vid, pid);
+	} else {
+		snprintf(fw_name, len, "brcm/%s.hcd", hw_name);
 	}
 
 	bt_dev_info(hdev, "%s (%3.3u.%3.3u.%3.3u) build %4.4u",
-		    hw_name ? : "BCM", (subver & 0xe000) >> 13,
+		    hw_name, (subver & 0xe000) >> 13,
 		    (subver & 0x1f00) >> 8, (subver & 0x00ff), rev & 0x0fff);
 
 	return 0;
@@ -386,30 +418,14 @@ EXPORT_SYMBOL_GPL(btbcm_initialize);
 
 int btbcm_finalize(struct hci_dev *hdev)
 {
-	struct sk_buff *skb;
-	struct hci_rp_read_local_version *ver;
-	u16 subver, rev;
+	char fw_name[64];
 	int err;
 
-	/* Reset */
-	err = btbcm_reset(hdev);
+	/* Re-initialize */
+	err = btbcm_initialize(hdev, fw_name, sizeof(fw_name), true);
 	if (err)
 		return err;
 
-	/* Read Local Version Info */
-	skb = btbcm_read_local_version(hdev);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
-
-	ver = (struct hci_rp_read_local_version *)skb->data;
-	rev = le16_to_cpu(ver->hci_rev);
-	subver = le16_to_cpu(ver->lmp_subver);
-	kfree_skb(skb);
-
-	bt_dev_info(hdev, "BCM (%3.3u.%3.3u.%3.3u) build %4.4u",
-		    (subver & 0xe000) >> 13, (subver & 0x1f00) >> 8,
-		    (subver & 0x00ff), rev & 0x0fff);
-
 	btbcm_check_bdaddr(hdev);
 
 	set_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks);
@@ -418,95 +434,18 @@ int btbcm_finalize(struct hci_dev *hdev)
 }
 EXPORT_SYMBOL_GPL(btbcm_finalize);
 
-static const struct {
-	u16 subver;
-	const char *name;
-} bcm_usb_subver_table[] = {
-	{ 0x210b, "BCM43142A0"	},	/* 001.001.011 */
-	{ 0x2112, "BCM4314A0"	},	/* 001.001.018 */
-	{ 0x2118, "BCM20702A0"	},	/* 001.001.024 */
-	{ 0x2126, "BCM4335A0"	},	/* 001.001.038 */
-	{ 0x220e, "BCM20702A1"	},	/* 001.002.014 */
-	{ 0x230f, "BCM4354A2"	},	/* 001.003.015 */
-	{ 0x4106, "BCM4335B0"	},	/* 002.001.006 */
-	{ 0x410e, "BCM20702B0"	},	/* 002.001.014 */
-	{ 0x6109, "BCM4335C0"	},	/* 003.001.009 */
-	{ 0x610c, "BCM4354"	},	/* 003.001.012 */
-	{ }
-};
-
 int btbcm_setup_patchram(struct hci_dev *hdev)
 {
 	char fw_name[64];
 	const struct firmware *fw;
-	u16 subver, rev, pid, vid;
-	const char *hw_name = NULL;
 	struct sk_buff *skb;
-	struct hci_rp_read_local_version *ver;
-	int i, err;
-
-	/* Reset */
-	err = btbcm_reset(hdev);
-	if (err)
-		return err;
-
-	/* Read Local Version Info */
-	skb = btbcm_read_local_version(hdev);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
-
-	ver = (struct hci_rp_read_local_version *)skb->data;
-	rev = le16_to_cpu(ver->hci_rev);
-	subver = le16_to_cpu(ver->lmp_subver);
-	kfree_skb(skb);
+	int err;
 
-	/* Read controller information */
-	err = btbcm_read_info(hdev);
+	/* Initialize */
+	err = btbcm_initialize(hdev, fw_name, sizeof(fw_name), false);
 	if (err)
 		return err;
 
-	switch ((rev & 0xf000) >> 12) {
-	case 0:
-	case 3:
-		for (i = 0; bcm_uart_subver_table[i].name; i++) {
-			if (subver == bcm_uart_subver_table[i].subver) {
-				hw_name = bcm_uart_subver_table[i].name;
-				break;
-			}
-		}
-
-		snprintf(fw_name, sizeof(fw_name), "brcm/%s.hcd",
-			 hw_name ? : "BCM");
-		break;
-	case 1:
-	case 2:
-		/* Read USB Product Info */
-		skb = btbcm_read_usb_product(hdev);
-		if (IS_ERR(skb))
-			return PTR_ERR(skb);
-
-		vid = get_unaligned_le16(skb->data + 1);
-		pid = get_unaligned_le16(skb->data + 3);
-		kfree_skb(skb);
-
-		for (i = 0; bcm_usb_subver_table[i].name; i++) {
-			if (subver == bcm_usb_subver_table[i].subver) {
-				hw_name = bcm_usb_subver_table[i].name;
-				break;
-			}
-		}
-
-		snprintf(fw_name, sizeof(fw_name), "brcm/%s-%4.4x-%4.4x.hcd",
-			 hw_name ? : "BCM", vid, pid);
-		break;
-	default:
-		return 0;
-	}
-
-	bt_dev_info(hdev, "%s (%3.3u.%3.3u.%3.3u) build %4.4u",
-		    hw_name ? : "BCM", (subver & 0xe000) >> 13,
-		    (subver & 0x1f00) >> 8, (subver & 0x00ff), rev & 0x0fff);
-
 	err = request_firmware(&fw, fw_name, &hdev->dev);
 	if (err < 0) {
 		bt_dev_info(hdev, "BCM: Patch %s not found", fw_name);
@@ -517,25 +456,11 @@ int btbcm_setup_patchram(struct hci_dev *hdev)
 
 	release_firmware(fw);
 
-	/* Reset */
-	err = btbcm_reset(hdev);
+	/* Re-initialize */
+	err = btbcm_initialize(hdev, fw_name, sizeof(fw_name), true);
 	if (err)
 		return err;
 
-	/* Read Local Version Info */
-	skb = btbcm_read_local_version(hdev);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
-
-	ver = (struct hci_rp_read_local_version *)skb->data;
-	rev = le16_to_cpu(ver->hci_rev);
-	subver = le16_to_cpu(ver->lmp_subver);
-	kfree_skb(skb);
-
-	bt_dev_info(hdev, "%s (%3.3u.%3.3u.%3.3u) build %4.4u",
-		    hw_name ? : "BCM", (subver & 0xe000) >> 13,
-		    (subver & 0x1f00) >> 8, (subver & 0x00ff), rev & 0x0fff);
-
 	/* Read Local Name */
 	skb = btbcm_read_local_name(hdev);
 	if (IS_ERR(skb))
diff --git a/drivers/bluetooth/btbcm.h b/drivers/bluetooth/btbcm.h
index cfe6ad4cc621..5346515c880c 100644
--- a/drivers/bluetooth/btbcm.h
+++ b/drivers/bluetooth/btbcm.h
@@ -73,7 +73,8 @@ int btbcm_patchram(struct hci_dev *hdev, const struct firmware *fw);
 int btbcm_setup_patchram(struct hci_dev *hdev);
 int btbcm_setup_apple(struct hci_dev *hdev);
 
-int btbcm_initialize(struct hci_dev *hdev, char *fw_name, size_t len);
+int btbcm_initialize(struct hci_dev *hdev, char *fw_name, size_t len,
+		     bool reinit);
 int btbcm_finalize(struct hci_dev *hdev);
 
 #else
@@ -104,7 +105,7 @@ static inline int btbcm_setup_apple(struct hci_dev *hdev)
 }
 
 static inline int btbcm_initialize(struct hci_dev *hdev, char *fw_name,
-				   size_t len)
+				   size_t len, bool reinit)
 {
 	return 0;
 }
diff --git a/drivers/bluetooth/btmrvl_debugfs.c b/drivers/bluetooth/btmrvl_debugfs.c
index 1828ed8cae7a..c4867576be00 100644
--- a/drivers/bluetooth/btmrvl_debugfs.c
+++ b/drivers/bluetooth/btmrvl_debugfs.c
@@ -35,15 +35,9 @@ static ssize_t btmrvl_hscfgcmd_write(struct file *file,
 			const char __user *ubuf, size_t count, loff_t *ppos)
 {
 	struct btmrvl_private *priv = file->private_data;
-	char buf[16];
 	long result, ret;
 
-	memset(buf, 0, sizeof(buf));
-
-	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
-		return -EFAULT;
-
-	ret = kstrtol(buf, 10, &result);
+	ret = kstrtol_from_user(ubuf, count, 10, &result);
 	if (ret)
 		return ret;
 
@@ -81,15 +75,9 @@ static ssize_t btmrvl_pscmd_write(struct file *file, const char __user *ubuf,
 						size_t count, loff_t *ppos)
 {
 	struct btmrvl_private *priv = file->private_data;
-	char buf[16];
 	long result, ret;
 
-	memset(buf, 0, sizeof(buf));
-
-	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
-		return -EFAULT;
-
-	ret = kstrtol(buf, 10, &result);
+	ret = kstrtol_from_user(ubuf, count, 10, &result);
 	if (ret)
 		return ret;
 
@@ -127,15 +115,9 @@ static ssize_t btmrvl_hscmd_write(struct file *file, const char __user *ubuf,
 						size_t count, loff_t *ppos)
 {
 	struct btmrvl_private *priv = file->private_data;
-	char buf[16];
 	long result, ret;
 
-	memset(buf, 0, sizeof(buf));
-
-	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
-		return -EFAULT;
-
-	ret = kstrtol(buf, 10, &result);
+	ret = kstrtol_from_user(ubuf, count, 10, &result);
 	if (ret)
 		return ret;
 
@@ -167,35 +149,6 @@ static const struct file_operations btmrvl_hscmd_fops = {
 	.llseek = default_llseek,
 };
 
-static ssize_t btmrvl_fwdump_write(struct file *file, const char __user *ubuf,
-				   size_t count, loff_t *ppos)
-{
-	struct btmrvl_private *priv = file->private_data;
-	char buf[16];
-	bool result;
-
-	memset(buf, 0, sizeof(buf));
-
-	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
-		return -EFAULT;
-
-	if (strtobool(buf, &result))
-		return -EINVAL;
-
-	if (!result)
-		return -EINVAL;
-
-	btmrvl_firmware_dump(priv);
-
-	return count;
-}
-
-static const struct file_operations btmrvl_fwdump_fops = {
-	.write	= btmrvl_fwdump_write,
-	.open	= simple_open,
-	.llseek = default_llseek,
-};
-
 void btmrvl_debugfs_init(struct hci_dev *hdev)
 {
 	struct btmrvl_private *priv = hci_get_drvdata(hdev);
@@ -226,8 +179,6 @@ void btmrvl_debugfs_init(struct hci_dev *hdev)
 			    priv, &btmrvl_hscmd_fops);
 	debugfs_create_file("hscfgcmd", 0644, dbg->config_dir,
 			    priv, &btmrvl_hscfgcmd_fops);
-	debugfs_create_file("fw_dump", 0200, dbg->config_dir,
-			    priv, &btmrvl_fwdump_fops);
 
 	dbg->status_dir = debugfs_create_dir("status", hdev->debugfs);
 	debugfs_create_u8("curpsmode", 0444, dbg->status_dir,
diff --git a/drivers/bluetooth/btmrvl_drv.h b/drivers/bluetooth/btmrvl_drv.h
index fc3caf4541ba..f0454541e5fd 100644
--- a/drivers/bluetooth/btmrvl_drv.h
+++ b/drivers/bluetooth/btmrvl_drv.h
@@ -110,7 +110,6 @@ struct btmrvl_private {
 				u8 *payload, u16 nb);
 	int (*hw_wakeup_firmware)(struct btmrvl_private *priv);
 	int (*hw_process_int_status)(struct btmrvl_private *priv);
-	void (*firmware_dump)(struct btmrvl_private *priv);
 	spinlock_t driver_lock;		/* spinlock used by driver */
 #ifdef CONFIG_DEBUG_FS
 	void *debugfs_data;
@@ -183,7 +182,6 @@ int btmrvl_send_hscfg_cmd(struct btmrvl_private *priv);
 int btmrvl_enable_ps(struct btmrvl_private *priv);
 int btmrvl_prepare_command(struct btmrvl_private *priv);
 int btmrvl_enable_hs(struct btmrvl_private *priv);
-void btmrvl_firmware_dump(struct btmrvl_private *priv);
 
 #ifdef CONFIG_DEBUG_FS
 void btmrvl_debugfs_init(struct hci_dev *hdev);
diff --git a/drivers/bluetooth/btmrvl_main.c b/drivers/bluetooth/btmrvl_main.c
index f6c694a1b9b0..708ad21683eb 100644
--- a/drivers/bluetooth/btmrvl_main.c
+++ b/drivers/bluetooth/btmrvl_main.c
@@ -358,12 +358,6 @@ int btmrvl_prepare_command(struct btmrvl_private *priv)
 	return ret;
 }
 
-void btmrvl_firmware_dump(struct btmrvl_private *priv)
-{
-	if (priv->firmware_dump)
-		priv->firmware_dump(priv);
-}
-
 static int btmrvl_tx_pkt(struct btmrvl_private *priv, struct sk_buff *skb)
 {
 	int ret = 0;
diff --git a/drivers/bluetooth/btmrvl_sdio.c b/drivers/bluetooth/btmrvl_sdio.c
index 6f99b9f3d57f..888bac49a87b 100644
--- a/drivers/bluetooth/btmrvl_sdio.c
+++ b/drivers/bluetooth/btmrvl_sdio.c
@@ -1311,9 +1311,11 @@ rdwr_status btmrvl_sdio_rdwr_firmware(struct btmrvl_private *priv,
 }
 
 /* This function dump sdio register and memory data */
-static void btmrvl_sdio_dump_firmware(struct btmrvl_private *priv)
+static void btmrvl_sdio_coredump(struct device *dev)
 {
-	struct btmrvl_sdio_card *card = priv->btmrvl_dev.card;
+	struct sdio_func *func = dev_to_sdio_func(dev);
+	struct btmrvl_sdio_card *card;
+	struct btmrvl_private *priv;
 	int ret = 0;
 	unsigned int reg, reg_start, reg_end;
 	enum rdwr_status stat;
@@ -1321,6 +1323,9 @@ static void btmrvl_sdio_dump_firmware(struct btmrvl_private *priv)
 	u8 dump_num = 0, idx, i, read_reg, doneflag = 0;
 	u32 memory_size, fw_dump_len = 0;
 
+	card = sdio_get_drvdata(func);
+	priv = card->priv;
+
 	/* dump sdio register first */
 	btmrvl_sdio_dump_regs(priv);
 
@@ -1547,7 +1552,6 @@ static int btmrvl_sdio_probe(struct sdio_func *func,
 	priv->hw_host_to_card = btmrvl_sdio_host_to_card;
 	priv->hw_wakeup_firmware = btmrvl_sdio_wakeup_fw;
 	priv->hw_process_int_status = btmrvl_sdio_process_int_status;
-	priv->firmware_dump = btmrvl_sdio_dump_firmware;
 
 	if (btmrvl_register_hdev(priv)) {
 		BT_ERR("Register hdev failed!");
@@ -1717,6 +1721,7 @@ static struct sdio_driver bt_mrvl_sdio = {
 	.remove		= btmrvl_sdio_remove,
 	.drv = {
 		.owner = THIS_MODULE,
+		.coredump = btmrvl_sdio_coredump,
 		.pm = &btmrvl_sdio_pm_ops,
 	}
 };
diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c
index 2793d4180d2f..8219816c54a0 100644
--- a/drivers/bluetooth/btqca.c
+++ b/drivers/bluetooth/btqca.c
@@ -127,28 +127,41 @@ static void rome_tlv_check_data(struct rome_config *config,
 	BT_DBG("TLV Type\t\t : 0x%x", type_len & 0x000000ff);
 	BT_DBG("Length\t\t : %d bytes", length);
 
+	config->dnld_mode = ROME_SKIP_EVT_NONE;
+
 	switch (config->type) {
 	case TLV_TYPE_PATCH:
 		tlv_patch = (struct tlv_type_patch *)tlv->data;
-		BT_DBG("Total Length\t\t : %d bytes",
+
+		/* For Rome version 1.1 to 3.1, all segment commands
+		 * are acked by a vendor specific event (VSE).
+		 * For Rome >= 3.2, the download mode field indicates
+		 * if VSE is skipped by the controller.
+		 * In case VSE is skipped, only the last segment is acked.
+		 */
+		config->dnld_mode = tlv_patch->download_mode;
+
+		BT_DBG("Total Length           : %d bytes",
 		       le32_to_cpu(tlv_patch->total_size));
-		BT_DBG("Patch Data Length\t : %d bytes",
+		BT_DBG("Patch Data Length      : %d bytes",
 		       le32_to_cpu(tlv_patch->data_length));
 		BT_DBG("Signing Format Version : 0x%x",
 		       tlv_patch->format_version);
-		BT_DBG("Signature Algorithm\t : 0x%x",
+		BT_DBG("Signature Algorithm    : 0x%x",
 		       tlv_patch->signature);
-		BT_DBG("Reserved\t\t : 0x%x",
-		       le16_to_cpu(tlv_patch->reserved1));
-		BT_DBG("Product ID\t\t : 0x%04x",
+		BT_DBG("Download mode          : 0x%x",
+		       tlv_patch->download_mode);
+		BT_DBG("Reserved               : 0x%x",
+		       tlv_patch->reserved1);
+		BT_DBG("Product ID             : 0x%04x",
 		       le16_to_cpu(tlv_patch->product_id));
-		BT_DBG("Rom Build Version\t : 0x%04x",
+		BT_DBG("Rom Build Version      : 0x%04x",
 		       le16_to_cpu(tlv_patch->rom_build));
-		BT_DBG("Patch Version\t\t : 0x%04x",
+		BT_DBG("Patch Version          : 0x%04x",
 		       le16_to_cpu(tlv_patch->patch_version));
-		BT_DBG("Reserved\t\t : 0x%x",
+		BT_DBG("Reserved               : 0x%x",
 		       le16_to_cpu(tlv_patch->reserved2));
-		BT_DBG("Patch Entry Address\t : 0x%x",
+		BT_DBG("Patch Entry Address    : 0x%x",
 		       le32_to_cpu(tlv_patch->entry));
 		break;
 
@@ -194,8 +207,8 @@ static void rome_tlv_check_data(struct rome_config *config,
 	}
 }
 
-static int rome_tlv_send_segment(struct hci_dev *hdev, int idx, int seg_size,
-				 const u8 *data)
+static int rome_tlv_send_segment(struct hci_dev *hdev, int seg_size,
+				 const u8 *data, enum rome_tlv_dnld_mode mode)
 {
 	struct sk_buff *skb;
 	struct edl_event_hdr *edl;
@@ -203,12 +216,14 @@ static int rome_tlv_send_segment(struct hci_dev *hdev, int idx, int seg_size,
 	u8 cmd[MAX_SIZE_PER_TLV_SEGMENT + 2];
 	int err = 0;
 
-	BT_DBG("%s: Download segment #%d size %d", hdev->name, idx, seg_size);
-
 	cmd[0] = EDL_PATCH_TLV_REQ_CMD;
 	cmd[1] = seg_size;
 	memcpy(cmd + 2, data, seg_size);
 
+	if (mode == ROME_SKIP_EVT_VSE_CC || mode == ROME_SKIP_EVT_VSE)
+		return __hci_cmd_send(hdev, EDL_PATCH_CMD_OPCODE, seg_size + 2,
+				      cmd);
+
 	skb = __hci_cmd_sync_ev(hdev, EDL_PATCH_CMD_OPCODE, seg_size + 2, cmd,
 				HCI_VENDOR_PKT, HCI_INIT_TIMEOUT);
 	if (IS_ERR(skb)) {
@@ -245,47 +260,12 @@ static int rome_tlv_send_segment(struct hci_dev *hdev, int idx, int seg_size,
 	return err;
 }
 
-static int rome_tlv_download_request(struct hci_dev *hdev,
-				     const struct firmware *fw)
-{
-	const u8 *buffer, *data;
-	int total_segment, remain_size;
-	int ret, i;
-
-	if (!fw || !fw->data)
-		return -EINVAL;
-
-	total_segment = fw->size / MAX_SIZE_PER_TLV_SEGMENT;
-	remain_size = fw->size % MAX_SIZE_PER_TLV_SEGMENT;
-
-	BT_DBG("%s: Total segment num %d remain size %d total size %zu",
-	       hdev->name, total_segment, remain_size, fw->size);
-
-	data = fw->data;
-	for (i = 0; i < total_segment; i++) {
-		buffer = data + i * MAX_SIZE_PER_TLV_SEGMENT;
-		ret = rome_tlv_send_segment(hdev, i, MAX_SIZE_PER_TLV_SEGMENT,
-					    buffer);
-		if (ret < 0)
-			return -EIO;
-	}
-
-	if (remain_size) {
-		buffer = data + total_segment * MAX_SIZE_PER_TLV_SEGMENT;
-		ret = rome_tlv_send_segment(hdev, total_segment, remain_size,
-					    buffer);
-		if (ret < 0)
-			return -EIO;
-	}
-
-	return 0;
-}
-
 static int rome_download_firmware(struct hci_dev *hdev,
 				  struct rome_config *config)
 {
 	const struct firmware *fw;
-	int ret;
+	const u8 *segment;
+	int ret, remain, i = 0;
 
 	bt_dev_info(hdev, "ROME Downloading %s", config->fwname);
 
@@ -298,10 +278,24 @@ static int rome_download_firmware(struct hci_dev *hdev,
 
 	rome_tlv_check_data(config, fw);
 
-	ret = rome_tlv_download_request(hdev, fw);
-	if (ret) {
-		BT_ERR("%s: Failed to download file: %s (%d)", hdev->name,
-		       config->fwname, ret);
+	segment = fw->data;
+	remain = fw->size;
+	while (remain > 0) {
+		int segsize = min(MAX_SIZE_PER_TLV_SEGMENT, remain);
+
+		bt_dev_dbg(hdev, "Send segment %d, size %d", i++, segsize);
+
+		remain -= segsize;
+		/* The last segment is always acked regardless download mode */
+		if (!remain || segsize < MAX_SIZE_PER_TLV_SEGMENT)
+			config->dnld_mode = ROME_SKIP_EVT_NONE;
+
+		ret = rome_tlv_send_segment(hdev, segsize, segment,
+					    config->dnld_mode);
+		if (ret)
+			break;
+
+		segment += segsize;
 	}
 
 	release_firmware(fw);
diff --git a/drivers/bluetooth/btqca.h b/drivers/bluetooth/btqca.h
index 65e994b96c47..13d77fd873b6 100644
--- a/drivers/bluetooth/btqca.h
+++ b/drivers/bluetooth/btqca.h
@@ -61,6 +61,13 @@ enum qca_bardrate {
 	QCA_BAUDRATE_RESERVED
 };
 
+enum rome_tlv_dnld_mode {
+	ROME_SKIP_EVT_NONE,
+	ROME_SKIP_EVT_VSE,
+	ROME_SKIP_EVT_CC,
+	ROME_SKIP_EVT_VSE_CC
+};
+
 enum rome_tlv_type {
 	TLV_TYPE_PATCH = 1,
 	TLV_TYPE_NVM
@@ -70,6 +77,7 @@ struct rome_config {
 	u8 type;
 	char fwname[64];
 	uint8_t user_baud_rate;
+	enum rome_tlv_dnld_mode dnld_mode;
 };
 
 struct edl_event_hdr {
@@ -94,7 +102,8 @@ struct tlv_type_patch {
 	__le32 data_length;
 	__u8   format_version;
 	__u8   signature;
-	__le16 reserved1;
+	__u8   download_mode;
+	__u8   reserved1;
 	__le16 product_id;
 	__le16 rom_build;
 	__le16 patch_version;
diff --git a/drivers/bluetooth/btqcomsmd.c b/drivers/bluetooth/btqcomsmd.c
index 2c9a5fc9137d..7df3eed1ef5e 100644
--- a/drivers/bluetooth/btqcomsmd.c
+++ b/drivers/bluetooth/btqcomsmd.c
@@ -65,6 +65,7 @@ static int btqcomsmd_cmd_callback(struct rpmsg_device *rpdev, void *data,
 {
 	struct btqcomsmd *btq = priv;
 
+	btq->hdev->stat.byte_rx += count;
 	return btqcomsmd_recv(btq->hdev, HCI_EVENT_PKT, data, count);
 }
 
@@ -76,12 +77,21 @@ static int btqcomsmd_send(struct hci_dev *hdev, struct sk_buff *skb)
 	switch (hci_skb_pkt_type(skb)) {
 	case HCI_ACLDATA_PKT:
 		ret = rpmsg_send(btq->acl_channel, skb->data, skb->len);
+		if (ret) {
+			hdev->stat.err_tx++;
+			break;
+		}
 		hdev->stat.acl_tx++;
 		hdev->stat.byte_tx += skb->len;
 		break;
 	case HCI_COMMAND_PKT:
 		ret = rpmsg_send(btq->cmd_channel, skb->data, skb->len);
+		if (ret) {
+			hdev->stat.err_tx++;
+			break;
+		}
 		hdev->stat.cmd_tx++;
+		hdev->stat.byte_tx += skb->len;
 		break;
 	default:
 		ret = -EILSEQ;
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index b937cc1e2c07..f73a27ea28cc 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -276,6 +276,8 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x04ca, 0x3011), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x04ca, 0x3015), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x04ca, 0x3016), .driver_info = BTUSB_QCA_ROME },
+	{ USB_DEVICE(0x04ca, 0x301a), .driver_info = BTUSB_QCA_ROME },
+	{ USB_DEVICE(0x13d3, 0x3496), .driver_info = BTUSB_QCA_ROME },
 
 	/* Broadcom BCM2035 */
 	{ USB_DEVICE(0x0a5c, 0x2009), .driver_info = BTUSB_BCM92035 },
@@ -371,6 +373,9 @@ static const struct usb_device_id blacklist_table[] = {
 	/* Additional Realtek 8723BU Bluetooth devices */
 	{ USB_DEVICE(0x7392, 0xa611), .driver_info = BTUSB_REALTEK },
 
+	/* Additional Realtek 8723DE Bluetooth devices */
+	{ USB_DEVICE(0x2ff8, 0xb011), .driver_info = BTUSB_REALTEK },
+
 	/* Additional Realtek 8821AE Bluetooth devices */
 	{ USB_DEVICE(0x0b05, 0x17dc), .driver_info = BTUSB_REALTEK },
 	{ USB_DEVICE(0x13d3, 0x3414), .driver_info = BTUSB_REALTEK },
@@ -379,6 +384,7 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x13d3, 0x3462), .driver_info = BTUSB_REALTEK },
 
 	/* Additional Realtek 8822BE Bluetooth devices */
+	{ USB_DEVICE(0x13d3, 0x3526), .driver_info = BTUSB_REALTEK },
 	{ USB_DEVICE(0x0b05, 0x185c), .driver_info = BTUSB_REALTEK },
 
 	/* Silicon Wave based devices */
@@ -406,6 +412,13 @@ static const struct dmi_system_id btusb_needs_reset_resume_table[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "XPS 13 9360"),
 		},
 	},
+	{
+		/* Dell Inspiron 5565 (QCA ROME device 0cf3:e009) */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 5565"),
+		},
+	},
 	{}
 };
 
@@ -2497,11 +2510,9 @@ static const struct qca_device_info qca_devices_table[] = {
 	{ 0x00000302, 28, 4, 18 }, /* Rome 3.2 */
 };
 
-static int btusb_qca_send_vendor_req(struct hci_dev *hdev, u8 request,
+static int btusb_qca_send_vendor_req(struct usb_device *udev, u8 request,
 				     void *data, u16 size)
 {
-	struct btusb_data *btdata = hci_get_drvdata(hdev);
-	struct usb_device *udev = btdata->udev;
 	int pipe, err;
 	u8 *buf;
 
@@ -2516,7 +2527,7 @@ static int btusb_qca_send_vendor_req(struct hci_dev *hdev, u8 request,
 	err = usb_control_msg(udev, pipe, request, USB_TYPE_VENDOR | USB_DIR_IN,
 			      0, 0, buf, size, USB_CTRL_SET_TIMEOUT);
 	if (err < 0) {
-		bt_dev_err(hdev, "Failed to access otp area (%d)", err);
+		dev_err(&udev->dev, "Failed to access otp area (%d)", err);
 		goto done;
 	}
 
@@ -2666,20 +2677,38 @@ static int btusb_setup_qca_load_nvm(struct hci_dev *hdev,
 	return err;
 }
 
+/* identify the ROM version and check whether patches are needed */
+static bool btusb_qca_need_patch(struct usb_device *udev)
+{
+	struct qca_version ver;
+
+	if (btusb_qca_send_vendor_req(udev, QCA_GET_TARGET_VERSION, &ver,
+				      sizeof(ver)) < 0)
+		return false;
+	/* only low ROM versions need patches */
+	return !(le32_to_cpu(ver.rom_version) & ~0xffffU);
+}
+
 static int btusb_setup_qca(struct hci_dev *hdev)
 {
+	struct btusb_data *btdata = hci_get_drvdata(hdev);
+	struct usb_device *udev = btdata->udev;
 	const struct qca_device_info *info = NULL;
 	struct qca_version ver;
 	u32 ver_rom;
 	u8 status;
 	int i, err;
 
-	err = btusb_qca_send_vendor_req(hdev, QCA_GET_TARGET_VERSION, &ver,
+	err = btusb_qca_send_vendor_req(udev, QCA_GET_TARGET_VERSION, &ver,
 					sizeof(ver));
 	if (err < 0)
 		return err;
 
 	ver_rom = le32_to_cpu(ver.rom_version);
+	/* Don't care about high ROM versions */
+	if (ver_rom & ~0xffffU)
+		return 0;
+
 	for (i = 0; i < ARRAY_SIZE(qca_devices_table); i++) {
 		if (ver_rom == qca_devices_table[i].rom_version)
 			info = &qca_devices_table[i];
@@ -2689,7 +2718,7 @@ static int btusb_setup_qca(struct hci_dev *hdev)
 		return -ENODEV;
 	}
 
-	err = btusb_qca_send_vendor_req(hdev, QCA_CHECK_STATUS, &status,
+	err = btusb_qca_send_vendor_req(udev, QCA_CHECK_STATUS, &status,
 					sizeof(status));
 	if (err < 0)
 		return err;
@@ -2903,7 +2932,8 @@ static int btusb_probe(struct usb_interface *intf,
 		/* Old firmware would otherwise let ath3k driver load
 		 * patch and sysconfig files
 		 */
-		if (le16_to_cpu(udev->descriptor.bcdDevice) <= 0x0001)
+		if (le16_to_cpu(udev->descriptor.bcdDevice) <= 0x0001 &&
+		    !btusb_qca_need_patch(udev))
 			return -ENODEV;
 	}
 
@@ -3065,6 +3095,7 @@ static int btusb_probe(struct usb_interface *intf,
 	}
 
 	if (id->driver_info & BTUSB_ATH3012) {
+		data->setup_on_usb = btusb_setup_qca;
 		hdev->set_bdaddr = btusb_set_bdaddr_ath3012;
 		set_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, &hdev->quirks);
 		set_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks);
diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c
index 441f5e1deb11..ddbd8c6a0ceb 100644
--- a/drivers/bluetooth/hci_bcm.c
+++ b/drivers/bluetooth/hci_bcm.c
@@ -380,10 +380,6 @@ static int bcm_open(struct hci_uart *hu)
 	mutex_lock(&bcm_device_lock);
 
 	if (hu->serdev) {
-		err = serdev_device_open(hu->serdev);
-		if (err)
-			goto err_free;
-
 		bcm->dev = serdev_device_get_drvdata(hu->serdev);
 		goto out;
 	}
@@ -420,13 +416,10 @@ static int bcm_open(struct hci_uart *hu)
 	return 0;
 
 err_unset_hu:
-	if (hu->serdev)
-		serdev_device_close(hu->serdev);
 #ifdef CONFIG_PM
-	else
+	if (!hu->serdev)
 		bcm->dev->hu = NULL;
 #endif
-err_free:
 	mutex_unlock(&bcm_device_lock);
 	hu->priv = NULL;
 	kfree(bcm);
@@ -445,7 +438,6 @@ static int bcm_close(struct hci_uart *hu)
 	mutex_lock(&bcm_device_lock);
 
 	if (hu->serdev) {
-		serdev_device_close(hu->serdev);
 		bdev = serdev_device_get_drvdata(hu->serdev);
 	} else if (bcm_device_exists(bcm->dev)) {
 		bdev = bcm->dev;
@@ -501,7 +493,7 @@ static int bcm_setup(struct hci_uart *hu)
 	hu->hdev->set_diag = bcm_set_diag;
 	hu->hdev->set_bdaddr = btbcm_set_bdaddr;
 
-	err = btbcm_initialize(hu->hdev, fw_name, sizeof(fw_name));
+	err = btbcm_initialize(hu->hdev, fw_name, sizeof(fw_name), false);
 	if (err)
 		return err;
 
@@ -794,19 +786,21 @@ static const struct acpi_gpio_mapping acpi_bcm_int_first_gpios[] = {
 	{ },
 };
 
-#ifdef CONFIG_ACPI
-/* IRQ polarity of some chipsets are not defined correctly in ACPI table. */
-static const struct dmi_system_id bcm_active_low_irq_dmi_table[] = {
-	{	/* Handle ThinkPad 8 tablets with BCM2E55 chipset ACPI ID */
-		.ident = "Lenovo ThinkPad 8",
+/* Some firmware reports an IRQ which does not work (wrong pin in fw table?) */
+static const struct dmi_system_id bcm_broken_irq_dmi_table[] = {
+	{
+		.ident = "Meegopad T08",
 		.matches = {
-			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
-			DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "ThinkPad 8"),
+			DMI_EXACT_MATCH(DMI_BOARD_VENDOR,
+					"To be filled by OEM."),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "T3 MRD"),
+			DMI_EXACT_MATCH(DMI_BOARD_VERSION, "V1.1"),
 		},
 	},
 	{ }
 };
 
+#ifdef CONFIG_ACPI
 static int bcm_resource(struct acpi_resource *ares, void *data)
 {
 	struct bcm_device *dev = data;
@@ -904,6 +898,8 @@ static int bcm_gpio_set_shutdown(struct bcm_device *dev, bool powered)
 
 static int bcm_get_resources(struct bcm_device *dev)
 {
+	const struct dmi_system_id *dmi_id;
+
 	dev->name = dev_name(dev->dev);
 
 	if (x86_apple_machine && !bcm_apple_get_resources(dev))
@@ -936,6 +932,13 @@ static int bcm_get_resources(struct bcm_device *dev)
 		dev->irq = gpiod_to_irq(gpio);
 	}
 
+	dmi_id = dmi_first_match(bcm_broken_irq_dmi_table);
+	if (dmi_id) {
+		dev_info(dev->dev, "%s: Has a broken IRQ config, disabling IRQ support / runtime-pm\n",
+			 dmi_id->ident);
+		dev->irq = 0;
+	}
+
 	dev_dbg(dev->dev, "BCM irq: %d\n", dev->irq);
 	return 0;
 }
@@ -944,7 +947,6 @@ static int bcm_get_resources(struct bcm_device *dev)
 static int bcm_acpi_probe(struct bcm_device *dev)
 {
 	LIST_HEAD(resources);
-	const struct dmi_system_id *dmi_id;
 	const struct acpi_gpio_mapping *gpio_mapping = acpi_bcm_int_last_gpios;
 	struct resource_entry *entry;
 	int ret;
@@ -991,13 +993,6 @@ static int bcm_acpi_probe(struct bcm_device *dev)
 		dev->irq_active_low = irq_polarity;
 		dev_warn(dev->dev, "Overwriting IRQ polarity to active %s by module-param\n",
 			 dev->irq_active_low ? "low" : "high");
-	} else {
-		dmi_id = dmi_first_match(bcm_active_low_irq_dmi_table);
-		if (dmi_id) {
-			dev_warn(dev->dev, "%s: Overwriting IRQ polarity to active low",
-				 dmi_id->ident);
-			dev->irq_active_low = true;
-		}
 	}
 
 	return 0;
diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c
index b6a71705b7d6..963bb0309e25 100644
--- a/drivers/bluetooth/hci_ldisc.c
+++ b/drivers/bluetooth/hci_ldisc.c
@@ -195,7 +195,7 @@ static void hci_uart_write_work(struct work_struct *work)
 	clear_bit(HCI_UART_SENDING, &hu->tx_state);
 }
 
-static void hci_uart_init_work(struct work_struct *work)
+void hci_uart_init_work(struct work_struct *work)
 {
 	struct hci_uart *hu = container_of(work, struct hci_uart, init_ready);
 	int err;
@@ -229,15 +229,6 @@ int hci_uart_init_ready(struct hci_uart *hu)
 }
 
 /* ------- Interface to HCI layer ------ */
-/* Initialize device */
-static int hci_uart_open(struct hci_dev *hdev)
-{
-	BT_DBG("%s %p", hdev->name, hdev);
-
-	/* Nothing to do for UART driver */
-	return 0;
-}
-
 /* Reset device */
 static int hci_uart_flush(struct hci_dev *hdev)
 {
@@ -264,6 +255,17 @@ static int hci_uart_flush(struct hci_dev *hdev)
 	return 0;
 }
 
+/* Initialize device */
+static int hci_uart_open(struct hci_dev *hdev)
+{
+	BT_DBG("%s %p", hdev->name, hdev);
+
+	/* Undo clearing this from hci_uart_close() */
+	hdev->flush = hci_uart_flush;
+
+	return 0;
+}
+
 /* Close device */
 static int hci_uart_close(struct hci_dev *hdev)
 {
@@ -447,6 +449,8 @@ static int hci_uart_setup(struct hci_dev *hdev)
 		btbcm_check_bdaddr(hdev);
 		break;
 #endif
+	default:
+		break;
 	}
 
 done:
diff --git a/drivers/bluetooth/hci_ll.c b/drivers/bluetooth/hci_ll.c
index 27e414b4e3a2..3e767f245ed5 100644
--- a/drivers/bluetooth/hci_ll.c
+++ b/drivers/bluetooth/hci_ll.c
@@ -141,7 +141,6 @@ static int ll_open(struct hci_uart *hu)
 
 	if (hu->serdev) {
 		struct ll_device *lldev = serdev_device_get_drvdata(hu->serdev);
-		serdev_device_open(hu->serdev);
 		if (!IS_ERR(lldev->ext_clk))
 			clk_prepare_enable(lldev->ext_clk);
 	}
@@ -179,8 +178,6 @@ static int ll_close(struct hci_uart *hu)
 		gpiod_set_value_cansleep(lldev->enable_gpio, 0);
 
 		clk_disable_unprepare(lldev->ext_clk);
-
-		serdev_device_close(hu->serdev);
 	}
 
 	hu->priv = NULL;
diff --git a/drivers/bluetooth/hci_nokia.c b/drivers/bluetooth/hci_nokia.c
index 3539fd03f47e..14d159e2042d 100644
--- a/drivers/bluetooth/hci_nokia.c
+++ b/drivers/bluetooth/hci_nokia.c
@@ -477,8 +477,6 @@ static int nokia_open(struct hci_uart *hu)
 
 	dev_dbg(dev, "protocol open");
 
-	serdev_device_open(hu->serdev);
-
 	pm_runtime_enable(dev);
 
 	return 0;
@@ -513,7 +511,6 @@ static int nokia_close(struct hci_uart *hu)
 	gpiod_set_value(btdev->wakeup_bt, 0);
 
 	pm_runtime_disable(&btdev->serdev->dev);
-	serdev_device_close(btdev->serdev);
 
 	return 0;
 }
diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
index 05ec530b8a3a..51790dd02afb 100644
--- a/drivers/bluetooth/hci_qca.c
+++ b/drivers/bluetooth/hci_qca.c
@@ -29,7 +29,12 @@
  */
 
 #include <linux/kernel.h>
+#include <linux/clk.h>
 #include <linux/debugfs.h>
+#include <linux/gpio/consumer.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/serdev.h>
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>
@@ -50,6 +55,9 @@
 #define IBS_TX_IDLE_TIMEOUT_MS		2000
 #define BAUDRATE_SETTLE_TIMEOUT_MS	300
 
+/* susclk rate */
+#define SUSCLK_RATE_32KHZ	32768
+
 /* HCI_IBS transmit side sleep protocol states */
 enum tx_ibs_states {
 	HCI_IBS_TX_ASLEEP,
@@ -111,6 +119,12 @@ struct qca_data {
 	u64 votes_off;
 };
 
+struct qca_serdev {
+	struct hci_uart	 serdev_hu;
+	struct gpio_desc *bt_en;
+	struct clk	 *susclk;
+};
+
 static void __serial_clock_on(struct tty_struct *tty)
 {
 	/* TODO: Some chipset requires to enable UART clock on client
@@ -386,6 +400,7 @@ static void hci_ibs_wake_retrans_timeout(struct timer_list *t)
 /* Initialize protocol */
 static int qca_open(struct hci_uart *hu)
 {
+	struct qca_serdev *qcadev;
 	struct qca_data *qca;
 
 	BT_DBG("hu %p qca_open", hu);
@@ -444,6 +459,13 @@ static int qca_open(struct hci_uart *hu)
 	timer_setup(&qca->tx_idle_timer, hci_ibs_tx_idle_timeout, 0);
 	qca->tx_idle_delay = IBS_TX_IDLE_TIMEOUT_MS;
 
+	if (hu->serdev) {
+		serdev_device_open(hu->serdev);
+
+		qcadev = serdev_device_get_drvdata(hu->serdev);
+		gpiod_set_value_cansleep(qcadev->bt_en, 1);
+	}
+
 	BT_DBG("HCI_UART_QCA open, tx_idle_delay=%u, wake_retrans=%u",
 	       qca->tx_idle_delay, qca->wake_retrans);
 
@@ -512,6 +534,7 @@ static int qca_flush(struct hci_uart *hu)
 /* Close protocol */
 static int qca_close(struct hci_uart *hu)
 {
+	struct qca_serdev *qcadev;
 	struct qca_data *qca = hu->priv;
 
 	BT_DBG("hu %p qca close", hu);
@@ -525,6 +548,13 @@ static int qca_close(struct hci_uart *hu)
 	destroy_workqueue(qca->workqueue);
 	qca->hu = NULL;
 
+	if (hu->serdev) {
+		serdev_device_close(hu->serdev);
+
+		qcadev = serdev_device_get_drvdata(hu->serdev);
+		gpiod_set_value_cansleep(qcadev->bt_en, 0);
+	}
+
 	kfree_skb(qca->rx_skb);
 
 	hu->priv = NULL;
@@ -880,11 +910,19 @@ static int qca_set_baudrate(struct hci_dev *hdev, uint8_t baudrate)
 	 */
 	set_current_state(TASK_UNINTERRUPTIBLE);
 	schedule_timeout(msecs_to_jiffies(BAUDRATE_SETTLE_TIMEOUT_MS));
-	set_current_state(TASK_INTERRUPTIBLE);
+	set_current_state(TASK_RUNNING);
 
 	return 0;
 }
 
+static inline void host_set_baudrate(struct hci_uart *hu, unsigned int speed)
+{
+	if (hu->serdev)
+		serdev_device_set_baudrate(hu->serdev, speed);
+	else
+		hci_uart_set_baudrate(hu, speed);
+}
+
 static int qca_setup(struct hci_uart *hu)
 {
 	struct hci_dev *hdev = hu->hdev;
@@ -905,7 +943,7 @@ static int qca_setup(struct hci_uart *hu)
 		speed = hu->proto->init_speed;
 
 	if (speed)
-		hci_uart_set_baudrate(hu, speed);
+		host_set_baudrate(hu, speed);
 
 	/* Setup user speed if needed */
 	speed = 0;
@@ -924,7 +962,7 @@ static int qca_setup(struct hci_uart *hu)
 				   ret);
 			return ret;
 		}
-		hci_uart_set_baudrate(hu, speed);
+		host_set_baudrate(hu, speed);
 	}
 
 	/* Setup patch / NVM configurations */
@@ -935,6 +973,12 @@ static int qca_setup(struct hci_uart *hu)
 	} else if (ret == -ENOENT) {
 		/* No patch/nvm-config found, run with original fw/config */
 		ret = 0;
+	} else if (ret == -EAGAIN) {
+		/*
+		 * Userspace firmware loader will return -EAGAIN in case no
+		 * patch/nvm-config is found, so run with original fw/config.
+		 */
+		ret = 0;
 	}
 
 	/* Setup bdaddr */
@@ -958,12 +1002,80 @@ static struct hci_uart_proto qca_proto = {
 	.dequeue	= qca_dequeue,
 };
 
+static int qca_serdev_probe(struct serdev_device *serdev)
+{
+	struct qca_serdev *qcadev;
+	int err;
+
+	qcadev = devm_kzalloc(&serdev->dev, sizeof(*qcadev), GFP_KERNEL);
+	if (!qcadev)
+		return -ENOMEM;
+
+	qcadev->serdev_hu.serdev = serdev;
+	serdev_device_set_drvdata(serdev, qcadev);
+
+	qcadev->bt_en = devm_gpiod_get(&serdev->dev, "enable",
+				       GPIOD_OUT_LOW);
+	if (IS_ERR(qcadev->bt_en)) {
+		dev_err(&serdev->dev, "failed to acquire enable gpio\n");
+		return PTR_ERR(qcadev->bt_en);
+	}
+
+	qcadev->susclk = devm_clk_get(&serdev->dev, NULL);
+	if (IS_ERR(qcadev->susclk)) {
+		dev_err(&serdev->dev, "failed to acquire clk\n");
+		return PTR_ERR(qcadev->susclk);
+	}
+
+	err = clk_set_rate(qcadev->susclk, SUSCLK_RATE_32KHZ);
+	if (err)
+		return err;
+
+	err = clk_prepare_enable(qcadev->susclk);
+	if (err)
+		return err;
+
+	err = hci_uart_register_device(&qcadev->serdev_hu, &qca_proto);
+	if (err)
+		clk_disable_unprepare(qcadev->susclk);
+
+	return err;
+}
+
+static void qca_serdev_remove(struct serdev_device *serdev)
+{
+	struct qca_serdev *qcadev = serdev_device_get_drvdata(serdev);
+
+	hci_uart_unregister_device(&qcadev->serdev_hu);
+
+	clk_disable_unprepare(qcadev->susclk);
+}
+
+static const struct of_device_id qca_bluetooth_of_match[] = {
+	{ .compatible = "qcom,qca6174-bt" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, qca_bluetooth_of_match);
+
+static struct serdev_device_driver qca_serdev_driver = {
+	.probe = qca_serdev_probe,
+	.remove = qca_serdev_remove,
+	.driver = {
+		.name = "hci_uart_qca",
+		.of_match_table = qca_bluetooth_of_match,
+	},
+};
+
 int __init qca_init(void)
 {
+	serdev_device_driver_register(&qca_serdev_driver);
+
 	return hci_uart_register_proto(&qca_proto);
 }
 
 int __exit qca_deinit(void)
 {
+	serdev_device_driver_unregister(&qca_serdev_driver);
+
 	return hci_uart_unregister_proto(&qca_proto);
 }
diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c
index e0e6461b9200..aa2543b3c286 100644
--- a/drivers/bluetooth/hci_serdev.c
+++ b/drivers/bluetooth/hci_serdev.c
@@ -101,14 +101,6 @@ static void hci_uart_write_work(struct work_struct *work)
 
 /* ------- Interface to HCI layer ------ */
 
-/* Initialize device */
-static int hci_uart_open(struct hci_dev *hdev)
-{
-	BT_DBG("%s %p", hdev->name, hdev);
-
-	return 0;
-}
-
 /* Reset device */
 static int hci_uart_flush(struct hci_dev *hdev)
 {
@@ -129,6 +121,17 @@ static int hci_uart_flush(struct hci_dev *hdev)
 	return 0;
 }
 
+/* Initialize device */
+static int hci_uart_open(struct hci_dev *hdev)
+{
+	BT_DBG("%s %p", hdev->name, hdev);
+
+	/* Undo clearing this from hci_uart_close() */
+	hdev->flush = hci_uart_flush;
+
+	return 0;
+}
+
 /* Close device */
 static int hci_uart_close(struct hci_dev *hdev)
 {
@@ -204,9 +207,8 @@ static int hci_uart_setup(struct hci_dev *hdev)
 		return 0;
 	}
 
-	if (skb->len != sizeof(*ver)) {
+	if (skb->len != sizeof(*ver))
 		bt_dev_err(hdev, "Event length mismatch for version info");
-	}
 
 	kfree_skb(skb);
 	return 0;
@@ -282,10 +284,14 @@ int hci_uart_register_device(struct hci_uart *hu,
 
 	serdev_device_set_client_ops(hu->serdev, &hci_serdev_client_ops);
 
-	err = p->open(hu);
+	err = serdev_device_open(hu->serdev);
 	if (err)
 		return err;
 
+	err = p->open(hu);
+	if (err)
+		goto err_open;
+
 	hu->proto = p;
 	set_bit(HCI_UART_PROTO_READY, &hu->flags);
 
@@ -302,6 +308,7 @@ int hci_uart_register_device(struct hci_uart *hu,
 	hdev->bus = HCI_UART;
 	hci_set_drvdata(hdev, hu);
 
+	INIT_WORK(&hu->init_ready, hci_uart_init_work);
 	INIT_WORK(&hu->write_work, hci_uart_write_work);
 	percpu_init_rwsem(&hu->proto_lock);
 
@@ -351,6 +358,8 @@ int hci_uart_register_device(struct hci_uart *hu,
 err_alloc:
 	clear_bit(HCI_UART_PROTO_READY, &hu->flags);
 	p->close(hu);
+err_open:
+	serdev_device_close(hu->serdev);
 	return err;
 }
 EXPORT_SYMBOL_GPL(hci_uart_register_device);
@@ -365,5 +374,6 @@ void hci_uart_unregister_device(struct hci_uart *hu)
 	cancel_work_sync(&hu->write_work);
 
 	hu->proto->close(hu);
+	serdev_device_close(hu->serdev);
 }
 EXPORT_SYMBOL_GPL(hci_uart_unregister_device);
diff --git a/drivers/bluetooth/hci_uart.h b/drivers/bluetooth/hci_uart.h
index 66e8c68e4607..00cab2fd7a1b 100644
--- a/drivers/bluetooth/hci_uart.h
+++ b/drivers/bluetooth/hci_uart.h
@@ -116,6 +116,7 @@ void hci_uart_unregister_device(struct hci_uart *hu);
 
 int hci_uart_tx_wakeup(struct hci_uart *hu);
 int hci_uart_init_ready(struct hci_uart *hu);
+void hci_uart_init_work(struct work_struct *work);
 void hci_uart_set_baudrate(struct hci_uart *hu, unsigned int speed);
 void hci_uart_set_flow_control(struct hci_uart *hu, bool enable);
 void hci_uart_set_speeds(struct hci_uart *hu, unsigned int init_speed,
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 6dc177bf4c42..d1c0b60e9326 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -33,7 +33,6 @@ config HISILICON_LPC
 	bool "Support for ISA I/O space on HiSilicon Hip06/7"
 	depends on ARM64 && (ARCH_HISI || COMPILE_TEST)
 	select INDIRECT_PIO
-	select MFD_CORE if ACPI
 	help
 	  Driver to enable I/O access to devices attached to the Low Pin
 	  Count bus on the HiSilicon Hip06/7 SoC.
diff --git a/drivers/bus/arm-cci.c b/drivers/bus/arm-cci.c
index 443e4c3fd357..b8184a903583 100644
--- a/drivers/bus/arm-cci.c
+++ b/drivers/bus/arm-cci.c
@@ -371,8 +371,6 @@ asmlinkage void __naked cci_enable_port_for_self(void)
 	[sizeof_struct_cpu_port] "i" (sizeof(struct cpu_port)),
 	[sizeof_struct_ace_port] "i" (sizeof(struct cci_ace_port)),
 	[offsetof_port_phys] "i" (offsetof(struct cci_ace_port, phys)) );
-
-	unreachable();
 }
 
 /**
diff --git a/drivers/bus/fsl-mc/fsl-mc-msi.c b/drivers/bus/fsl-mc/fsl-mc-msi.c
index ec35e255b496..8b9c66d7c4ff 100644
--- a/drivers/bus/fsl-mc/fsl-mc-msi.c
+++ b/drivers/bus/fsl-mc/fsl-mc-msi.c
@@ -163,6 +163,8 @@ struct irq_domain *fsl_mc_msi_create_irq_domain(struct fwnode_handle *fwnode,
 {
 	struct irq_domain *domain;
 
+	if (WARN_ON((info->flags & MSI_FLAG_LEVEL_CAPABLE)))
+		info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;
 	if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
 		fsl_mc_msi_update_dom_ops(info);
 	if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
diff --git a/drivers/bus/hisi_lpc.c b/drivers/bus/hisi_lpc.c
index 2d4611e4c339..d5f85455fa62 100644
--- a/drivers/bus/hisi_lpc.c
+++ b/drivers/bus/hisi_lpc.c
@@ -11,12 +11,12 @@
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/logic_pio.h>
-#include <linux/mfd/core.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
 #include <linux/pci.h>
+#include <linux/serial_8250.h>
 #include <linux/slab.h>
 
 #define DRV_NAME "hisi-lpc"
@@ -341,15 +341,6 @@ static const struct logic_pio_host_ops hisi_lpc_ops = {
 };
 
 #ifdef CONFIG_ACPI
-#define MFD_CHILD_NAME_PREFIX DRV_NAME"-"
-#define MFD_CHILD_NAME_LEN (ACPI_ID_LEN + sizeof(MFD_CHILD_NAME_PREFIX) - 1)
-
-struct hisi_lpc_mfd_cell {
-	struct mfd_cell_acpi_match acpi_match;
-	char name[MFD_CHILD_NAME_LEN];
-	char pnpid[ACPI_ID_LEN];
-};
-
 static int hisi_lpc_acpi_xlat_io_res(struct acpi_device *adev,
 				     struct acpi_device *host,
 				     struct resource *res)
@@ -368,7 +359,7 @@ static int hisi_lpc_acpi_xlat_io_res(struct acpi_device *adev,
 }
 
 /*
- * hisi_lpc_acpi_set_io_res - set the resources for a child's MFD
+ * hisi_lpc_acpi_set_io_res - set the resources for a child
  * @child: the device node to be updated the I/O resource
  * @hostdev: the device node associated with host controller
  * @res: double pointer to be set to the address of translated resources
@@ -452,78 +443,122 @@ static int hisi_lpc_acpi_set_io_res(struct device *child,
 	return 0;
 }
 
+static int hisi_lpc_acpi_remove_subdev(struct device *dev, void *unused)
+{
+	platform_device_unregister(to_platform_device(dev));
+	return 0;
+}
+
+struct hisi_lpc_acpi_cell {
+	const char *hid;
+	const char *name;
+	void *pdata;
+	size_t pdata_size;
+};
+
 /*
  * hisi_lpc_acpi_probe - probe children for ACPI FW
  * @hostdev: LPC host device pointer
  *
  * Returns 0 when successful, and a negative value for failure.
  *
- * Scan all child devices and create a per-device MFD with
- * logical PIO translated IO resources.
+ * Create a platform device per child, fixing up the resources
+ * from bus addresses to Logical PIO addresses.
+ *
  */
 static int hisi_lpc_acpi_probe(struct device *hostdev)
 {
 	struct acpi_device *adev = ACPI_COMPANION(hostdev);
-	struct hisi_lpc_mfd_cell *hisi_lpc_mfd_cells;
-	struct mfd_cell *mfd_cells;
 	struct acpi_device *child;
-	int size, ret, count = 0, cell_num = 0;
-
-	list_for_each_entry(child, &adev->children, node)
-		cell_num++;
-
-	/* allocate the mfd cell and companion ACPI info, one per child */
-	size = sizeof(*mfd_cells) + sizeof(*hisi_lpc_mfd_cells);
-	mfd_cells = devm_kcalloc(hostdev, cell_num, size, GFP_KERNEL);
-	if (!mfd_cells)
-		return -ENOMEM;
+	int ret;
 
-	hisi_lpc_mfd_cells = (struct hisi_lpc_mfd_cell *)&mfd_cells[cell_num];
 	/* Only consider the children of the host */
 	list_for_each_entry(child, &adev->children, node) {
-		struct mfd_cell *mfd_cell = &mfd_cells[count];
-		struct hisi_lpc_mfd_cell *hisi_lpc_mfd_cell =
-					&hisi_lpc_mfd_cells[count];
-		struct mfd_cell_acpi_match *acpi_match =
-					&hisi_lpc_mfd_cell->acpi_match;
-		char *name = hisi_lpc_mfd_cell[count].name;
-		char *pnpid = hisi_lpc_mfd_cell[count].pnpid;
-		struct mfd_cell_acpi_match match = {
-			.pnpid = pnpid,
+		const char *hid = acpi_device_hid(child);
+		const struct hisi_lpc_acpi_cell *cell;
+		struct platform_device *pdev;
+		const struct resource *res;
+		bool found = false;
+		int num_res;
+
+		ret = hisi_lpc_acpi_set_io_res(&child->dev, &adev->dev, &res,
+					       &num_res);
+		if (ret) {
+			dev_warn(hostdev, "set resource fail (%d)\n", ret);
+			goto fail;
+		}
+
+		cell = (struct hisi_lpc_acpi_cell []){
+			/* ipmi */
+			{
+				.hid = "IPI0001",
+				.name = "hisi-lpc-ipmi",
+			},
+			/* 8250-compatible uart */
+			{
+				.hid = "HISI1031",
+				.name = "serial8250",
+				.pdata = (struct plat_serial8250_port []) {
+					{
+						.iobase = res->start,
+						.uartclk = 1843200,
+						.iotype = UPIO_PORT,
+						.flags = UPF_BOOT_AUTOCONF,
+					},
+					{}
+				},
+				.pdata_size = 2 *
+					sizeof(struct plat_serial8250_port),
+			},
+			{}
 		};
 
-		/*
-		 * For any instances of this host controller (Hip06 and Hip07
-		 * are the only chipsets), we would not have multiple slaves
-		 * with the same HID. And in any system we would have just one
-		 * controller active. So don't worrry about MFD name clashes.
-		 */
-		snprintf(name, MFD_CHILD_NAME_LEN, MFD_CHILD_NAME_PREFIX"%s",
-			 acpi_device_hid(child));
-		snprintf(pnpid, ACPI_ID_LEN, "%s", acpi_device_hid(child));
-
-		memcpy(acpi_match, &match, sizeof(*acpi_match));
-		mfd_cell->name = name;
-		mfd_cell->acpi_match = acpi_match;
-
-		ret = hisi_lpc_acpi_set_io_res(&child->dev, &adev->dev,
-					       &mfd_cell->resources,
-					       &mfd_cell->num_resources);
-		if (ret) {
-			dev_warn(&child->dev, "set resource fail (%d)\n", ret);
-			return ret;
+		for (; cell && cell->name; cell++) {
+			if (!strcmp(cell->hid, hid)) {
+				found = true;
+				break;
+			}
 		}
-		count++;
-	}
 
-	ret = mfd_add_devices(hostdev, PLATFORM_DEVID_NONE,
-			      mfd_cells, cell_num, NULL, 0, NULL);
-	if (ret) {
-		dev_err(hostdev, "failed to add mfd cells (%d)\n", ret);
-		return ret;
+		if (!found) {
+			dev_warn(hostdev,
+				 "could not find cell for child device (%s)\n",
+				 hid);
+			ret = -ENODEV;
+			goto fail;
+		}
+
+		pdev = platform_device_alloc(cell->name, PLATFORM_DEVID_AUTO);
+		if (!pdev) {
+			ret = -ENOMEM;
+			goto fail;
+		}
+
+		pdev->dev.parent = hostdev;
+		ACPI_COMPANION_SET(&pdev->dev, child);
+
+		ret = platform_device_add_resources(pdev, res, num_res);
+		if (ret)
+			goto fail;
+
+		ret = platform_device_add_data(pdev, cell->pdata,
+					       cell->pdata_size);
+		if (ret)
+			goto fail;
+
+		ret = platform_device_add(pdev);
+		if (ret)
+			goto fail;
+
+		acpi_device_set_enumerated(child);
 	}
 
 	return 0;
+
+fail:
+	device_for_each_child(hostdev, NULL,
+			      hisi_lpc_acpi_remove_subdev);
+	return ret;
 }
 
 static const struct acpi_device_id hisi_lpc_acpi_match[] = {
diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c
index 7cd2fd04b212..1cc29629d238 100644
--- a/drivers/bus/ti-sysc.c
+++ b/drivers/bus/ti-sysc.c
@@ -19,6 +19,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
+#include <linux/reset.h>
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
 #include <linux/slab.h>
@@ -32,10 +33,18 @@ static const char * const reg_names[] = { "rev", "sysc", "syss", };
 enum sysc_clocks {
 	SYSC_FCK,
 	SYSC_ICK,
+	SYSC_OPTFCK0,
+	SYSC_OPTFCK1,
+	SYSC_OPTFCK2,
+	SYSC_OPTFCK3,
+	SYSC_OPTFCK4,
+	SYSC_OPTFCK5,
+	SYSC_OPTFCK6,
+	SYSC_OPTFCK7,
 	SYSC_MAX_CLOCKS,
 };
 
-static const char * const clock_names[] = { "fck", "ick", };
+static const char * const clock_names[SYSC_ICK + 1] = { "fck", "ick", };
 
 #define SYSC_IDLEMODE_MASK		3
 #define SYSC_CLOCKACTIVITY_MASK		3
@@ -48,6 +57,8 @@ static const char * const clock_names[] = { "fck", "ick", };
  * @module_va: virtual address of the interconnect target module
  * @offsets: register offsets from module base
  * @clocks: clocks used by the interconnect target module
+ * @clock_roles: clock role names for the found clocks
+ * @nr_clocks: number of clocks used by the interconnect target module
  * @legacy_mode: configured for legacy mode if set
  * @cap: interconnect target module capabilities
  * @cfg: interconnect target module configuration
@@ -61,7 +72,10 @@ struct sysc {
 	u32 module_size;
 	void __iomem *module_va;
 	int offsets[SYSC_MAX_REGS];
-	struct clk *clocks[SYSC_MAX_CLOCKS];
+	struct clk **clocks;
+	const char **clock_roles;
+	int nr_clocks;
+	struct reset_control *rsts;
 	const char *legacy_mode;
 	const struct sysc_capabilities *cap;
 	struct sysc_config cfg;
@@ -88,6 +102,11 @@ static u32 sysc_read(struct sysc *ddata, int offset)
 	return readl_relaxed(ddata->module_va + offset);
 }
 
+static bool sysc_opt_clks_needed(struct sysc *ddata)
+{
+	return !!(ddata->cfg.quirks & SYSC_QUIRK_OPT_CLKS_NEEDED);
+}
+
 static u32 sysc_read_revision(struct sysc *ddata)
 {
 	int offset = ddata->offsets[SYSC_REVISION];
@@ -98,21 +117,28 @@ static u32 sysc_read_revision(struct sysc *ddata)
 	return sysc_read(ddata, offset);
 }
 
-static int sysc_get_one_clock(struct sysc *ddata,
-			      enum sysc_clocks index)
+static int sysc_get_one_clock(struct sysc *ddata, const char *name)
 {
-	const char *name;
-	int error;
+	int error, i, index = -ENODEV;
+
+	if (!strncmp(clock_names[SYSC_FCK], name, 3))
+		index = SYSC_FCK;
+	else if (!strncmp(clock_names[SYSC_ICK], name, 3))
+		index = SYSC_ICK;
+
+	if (index < 0) {
+		for (i = SYSC_OPTFCK0; i < SYSC_MAX_CLOCKS; i++) {
+			if (!ddata->clocks[i]) {
+				index = i;
+				break;
+			}
+		}
+	}
 
-	switch (index) {
-	case SYSC_FCK:
-		break;
-	case SYSC_ICK:
-		break;
-	default:
-		return -EINVAL;
+	if (index < 0) {
+		dev_err(ddata->dev, "clock %s not added\n", name);
+		return index;
 	}
-	name = clock_names[index];
 
 	ddata->clocks[index] = devm_clk_get(ddata->dev, name);
 	if (IS_ERR(ddata->clocks[index])) {
@@ -138,10 +164,50 @@ static int sysc_get_one_clock(struct sysc *ddata,
 
 static int sysc_get_clocks(struct sysc *ddata)
 {
-	int i, error;
+	struct device_node *np = ddata->dev->of_node;
+	struct property *prop;
+	const char *name;
+	int nr_fck = 0, nr_ick = 0, i, error = 0;
 
-	for (i = 0; i < SYSC_MAX_CLOCKS; i++) {
-		error = sysc_get_one_clock(ddata, i);
+	ddata->clock_roles = devm_kzalloc(ddata->dev,
+					  sizeof(*ddata->clock_roles) *
+					  SYSC_MAX_CLOCKS,
+					  GFP_KERNEL);
+	if (!ddata->clock_roles)
+		return -ENOMEM;
+
+	of_property_for_each_string(np, "clock-names", prop, name) {
+		if (!strncmp(clock_names[SYSC_FCK], name, 3))
+			nr_fck++;
+		if (!strncmp(clock_names[SYSC_ICK], name, 3))
+			nr_ick++;
+		ddata->clock_roles[ddata->nr_clocks] = name;
+		ddata->nr_clocks++;
+	}
+
+	if (ddata->nr_clocks < 1)
+		return 0;
+
+	if (ddata->nr_clocks > SYSC_MAX_CLOCKS) {
+		dev_err(ddata->dev, "too many clocks for %pOF\n", np);
+
+		return -EINVAL;
+	}
+
+	if (nr_fck > 1 || nr_ick > 1) {
+		dev_err(ddata->dev, "max one fck and ick for %pOF\n", np);
+
+		return -EINVAL;
+	}
+
+	ddata->clocks = devm_kzalloc(ddata->dev,
+				     sizeof(*ddata->clocks) * ddata->nr_clocks,
+				     GFP_KERNEL);
+	if (!ddata->clocks)
+		return -ENOMEM;
+
+	for (i = 0; i < ddata->nr_clocks; i++) {
+		error = sysc_get_one_clock(ddata, ddata->clock_roles[i]);
 		if (error && error != -ENOENT)
 			return error;
 	}
@@ -149,6 +215,42 @@ static int sysc_get_clocks(struct sysc *ddata)
 	return 0;
 }
 
+/**
+ * sysc_init_resets - reset module on init
+ * @ddata: device driver data
+ *
+ * A module can have both OCP softreset control and external rstctrl.
+ * If more complicated rstctrl resets are needed, please handle these
+ * directly from the child device driver and map only the module reset
+ * for the parent interconnect target module device.
+ *
+ * Automatic reset of the module on init can be skipped with the
+ * "ti,no-reset-on-init" device tree property.
+ */
+static int sysc_init_resets(struct sysc *ddata)
+{
+	int error;
+
+	ddata->rsts =
+		devm_reset_control_array_get_optional_exclusive(ddata->dev);
+	if (IS_ERR(ddata->rsts))
+		return PTR_ERR(ddata->rsts);
+
+	if (ddata->cfg.quirks & SYSC_QUIRK_NO_RESET_ON_INIT)
+		goto deassert;
+
+	error = reset_control_assert(ddata->rsts);
+	if (error)
+		return error;
+
+deassert:
+	error = reset_control_deassert(ddata->rsts);
+	if (error)
+		return error;
+
+	return 0;
+}
+
 /**
  * sysc_parse_and_check_child_range - parses module IO region from ranges
  * @ddata: device driver data
@@ -533,9 +635,13 @@ static int __maybe_unused sysc_runtime_suspend(struct device *dev)
 		goto idled;
 	}
 
-	for (i = 0; i < SYSC_MAX_CLOCKS; i++) {
+	for (i = 0; i < ddata->nr_clocks; i++) {
 		if (IS_ERR_OR_NULL(ddata->clocks[i]))
 			continue;
+
+		if (i >= SYSC_OPTFCK0 && !sysc_opt_clks_needed(ddata))
+			break;
+
 		clk_disable(ddata->clocks[i]);
 	}
 
@@ -572,9 +678,13 @@ static int __maybe_unused sysc_runtime_resume(struct device *dev)
 		goto awake;
 	}
 
-	for (i = 0; i < SYSC_MAX_CLOCKS; i++) {
+	for (i = 0; i < ddata->nr_clocks; i++) {
 		if (IS_ERR_OR_NULL(ddata->clocks[i]))
 			continue;
+
+		if (i >= SYSC_OPTFCK0 && !sysc_opt_clks_needed(ddata))
+			break;
+
 		error = clk_enable(ddata->clocks[i]);
 		if (error)
 			return error;
@@ -590,23 +700,103 @@ static int __maybe_unused sysc_runtime_resume(struct device *dev)
 static int sysc_suspend(struct device *dev)
 {
 	struct sysc *ddata;
+	int error;
 
 	ddata = dev_get_drvdata(dev);
 
+	if (ddata->cfg.quirks & (SYSC_QUIRK_RESOURCE_PROVIDER |
+				 SYSC_QUIRK_LEGACY_IDLE))
+		return 0;
+
 	if (!ddata->enabled)
 		return 0;
 
+	dev_dbg(ddata->dev, "%s %s\n", __func__,
+		ddata->name ? ddata->name : "");
+
+	error = pm_runtime_put_sync_suspend(dev);
+	if (error < 0) {
+		dev_warn(ddata->dev, "%s not idle %i %s\n",
+			 __func__, error,
+			 ddata->name ? ddata->name : "");
+
+		return 0;
+	}
+
 	ddata->needs_resume = true;
 
-	return sysc_runtime_suspend(dev);
+	return 0;
 }
 
 static int sysc_resume(struct device *dev)
+{
+	struct sysc *ddata;
+	int error;
+
+	ddata = dev_get_drvdata(dev);
+
+	if (ddata->cfg.quirks & (SYSC_QUIRK_RESOURCE_PROVIDER |
+				 SYSC_QUIRK_LEGACY_IDLE))
+		return 0;
+
+	if (ddata->needs_resume) {
+		dev_dbg(ddata->dev, "%s %s\n", __func__,
+			ddata->name ? ddata->name : "");
+
+		error = pm_runtime_get_sync(dev);
+		if (error < 0) {
+			dev_err(ddata->dev, "%s  error %i %s\n",
+				__func__, error,
+				 ddata->name ? ddata->name : "");
+
+			return error;
+		}
+
+		ddata->needs_resume = false;
+	}
+
+	return 0;
+}
+
+static int sysc_noirq_suspend(struct device *dev)
 {
 	struct sysc *ddata;
 
 	ddata = dev_get_drvdata(dev);
+
+	if (ddata->cfg.quirks & SYSC_QUIRK_LEGACY_IDLE)
+		return 0;
+
+	if (!(ddata->cfg.quirks & SYSC_QUIRK_RESOURCE_PROVIDER))
+		return 0;
+
+	if (!ddata->enabled)
+		return 0;
+
+	dev_dbg(ddata->dev, "%s %s\n", __func__,
+		ddata->name ? ddata->name : "");
+
+	ddata->needs_resume = true;
+
+	return sysc_runtime_suspend(dev);
+}
+
+static int sysc_noirq_resume(struct device *dev)
+{
+	struct sysc *ddata;
+
+	ddata = dev_get_drvdata(dev);
+
+	if (ddata->cfg.quirks & SYSC_QUIRK_LEGACY_IDLE)
+		return 0;
+
+	if (!(ddata->cfg.quirks & SYSC_QUIRK_RESOURCE_PROVIDER))
+		return 0;
+
 	if (ddata->needs_resume) {
+		dev_dbg(ddata->dev, "%s %s\n", __func__,
+			ddata->name ? ddata->name : "");
+
 		ddata->needs_resume = false;
 
 		return sysc_runtime_resume(dev);
@@ -618,6 +808,7 @@ static int sysc_resume(struct device *dev)
 
 static const struct dev_pm_ops sysc_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(sysc_suspend, sysc_resume)
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(sysc_noirq_suspend, sysc_noirq_resume)
 	SET_RUNTIME_PM_OPS(sysc_runtime_suspend,
 			   sysc_runtime_resume,
 			   NULL)
@@ -649,9 +840,29 @@ struct sysc_revision_quirk {
 	}
 
 static const struct sysc_revision_quirk sysc_revision_quirks[] = {
+	/* These need to use noirq_suspend */
+	SYSC_QUIRK("control", 0, 0, 0x10, -1, 0x40000900, 0xffffffff,
+		   SYSC_QUIRK_RESOURCE_PROVIDER),
+	SYSC_QUIRK("i2c", 0, 0, 0x10, 0x90, 0x5040000a, 0xffffffff,
+		   SYSC_QUIRK_RESOURCE_PROVIDER),
+	SYSC_QUIRK("mcspi", 0, 0, 0x10, -1, 0x40300a0b, 0xffffffff,
+		   SYSC_QUIRK_RESOURCE_PROVIDER),
+	SYSC_QUIRK("prcm", 0, 0, -1, -1, 0x40000100, 0xffffffff,
+		   SYSC_QUIRK_RESOURCE_PROVIDER),
+	SYSC_QUIRK("ocp2scp", 0, 0, 0x10, 0x14, 0x50060005, 0xffffffff,
+		   SYSC_QUIRK_RESOURCE_PROVIDER),
+	SYSC_QUIRK("padconf", 0, 0, 0x10, -1, 0x4fff0800, 0xffffffff,
+		   SYSC_QUIRK_RESOURCE_PROVIDER),
+	SYSC_QUIRK("scm", 0, 0, 0x10, -1, 0x40000900, 0xffffffff,
+		   SYSC_QUIRK_RESOURCE_PROVIDER),
+	SYSC_QUIRK("scrm", 0, 0, -1, -1, 0x00000010, 0xffffffff,
+		   SYSC_QUIRK_RESOURCE_PROVIDER),
+	SYSC_QUIRK("sdma", 0, 0, 0x2c, 0x28, 0x00010900, 0xffffffff,
+		   SYSC_QUIRK_RESOURCE_PROVIDER),
+
 	/* These drivers need to be fixed to not use pm_runtime_irq_safe() */
 	SYSC_QUIRK("gpio", 0, 0, 0x10, 0x114, 0x50600801, 0xffffffff,
-		   SYSC_QUIRK_LEGACY_IDLE),
+		   SYSC_QUIRK_LEGACY_IDLE | SYSC_QUIRK_OPT_CLKS_IN_RESET),
 	SYSC_QUIRK("mmu", 0, 0, 0x10, 0x14, 0x00000020, 0xffffffff,
 		   SYSC_QUIRK_LEGACY_IDLE),
 	SYSC_QUIRK("mmu", 0, 0, 0x10, 0x14, 0x00000030, 0xffffffff,
@@ -664,8 +875,40 @@ static const struct sysc_revision_quirk sysc_revision_quirks[] = {
 		   SYSC_QUIRK_LEGACY_IDLE),
 	SYSC_QUIRK("timer", 0, 0, 0x10, 0x14, 0x00000015, 0xffffffff,
 		   SYSC_QUIRK_LEGACY_IDLE),
+	/* Some timers on omap4 and later */
+	SYSC_QUIRK("timer", 0, 0, 0x10, -1, 0x4fff1301, 0xffffffff,
+		   SYSC_QUIRK_LEGACY_IDLE),
 	SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x00000052, 0xffffffff,
 		   SYSC_QUIRK_LEGACY_IDLE),
+	/* Uarts on omap4 and later */
+	SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x50411e03, 0xffffffff,
+		   SYSC_QUIRK_LEGACY_IDLE),
+
+	/* These devices don't yet suspend properly without legacy setting */
+	SYSC_QUIRK("sdio", 0, 0, 0x10, -1, 0x40202301, 0xffffffff,
+		   SYSC_QUIRK_LEGACY_IDLE),
+	SYSC_QUIRK("wdt", 0, 0, 0x10, 0x14, 0x502a0500, 0xffffffff,
+		   SYSC_QUIRK_LEGACY_IDLE),
+	SYSC_QUIRK("wdt", 0, 0, 0x10, 0x14, 0x502a0d00, 0xffffffff,
+		   SYSC_QUIRK_LEGACY_IDLE),
+
+#ifdef DEBUG
+	SYSC_QUIRK("aess", 0, 0, 0x10, -1, 0x40000000, 0xffffffff, 0),
+	SYSC_QUIRK("gpu", 0, 0x1fc00, 0x1fc10, -1, 0, 0, 0),
+	SYSC_QUIRK("hdq1w", 0, 0, 0x14, 0x18, 0x00000006, 0xffffffff, 0),
+	SYSC_QUIRK("hsi", 0, 0, 0x10, 0x14, 0x50043101, 0xffffffff, 0),
+	SYSC_QUIRK("iss", 0, 0, 0x10, -1, 0x40000101, 0xffffffff, 0),
+	SYSC_QUIRK("mcasp", 0, 0, 0x4, -1, 0x44306302, 0xffffffff, 0),
+	SYSC_QUIRK("mcbsp", 0, -1, 0x8c, -1, 0, 0, 0),
+	SYSC_QUIRK("mailbox", 0, 0, 0x10, -1, 0x00000400, 0xffffffff, 0),
+	SYSC_QUIRK("slimbus", 0, 0, 0x10, -1, 0x40000902, 0xffffffff, 0),
+	SYSC_QUIRK("slimbus", 0, 0, 0x10, -1, 0x40002903, 0xffffffff, 0),
+	SYSC_QUIRK("spinlock", 0, 0, 0x10, -1, 0x50020000, 0xffffffff, 0),
+	SYSC_QUIRK("usbhstll", 0, 0, 0x10, 0x14, 0x00000004, 0xffffffff, 0),
+	SYSC_QUIRK("usb_host_hs", 0, 0, 0x10, 0x14, 0x50700100, 0xffffffff, 0),
+	SYSC_QUIRK("usb_otg_hs", 0, 0x400, 0x404, 0x408, 0x00000050,
+		   0xffffffff, 0),
+#endif
 };
 
 static void sysc_init_revision_quirks(struct sysc *ddata)
@@ -716,6 +959,7 @@ static int sysc_init_module(struct sysc *ddata)
 
 		return 0;
 	}
+
 	ddata->revision = sysc_read_revision(ddata);
 	pm_runtime_put_sync(ddata->dev);
 
@@ -811,29 +1055,58 @@ static int sysc_init_syss_mask(struct sysc *ddata)
 }
 
 /*
- * Many child device drivers need to have fck available to get the clock
- * rate for device internal configuration.
+ * Many child device drivers need to have fck and opt clocks available
+ * to get the clock rate for device internal configuration etc.
  */
-static int sysc_child_add_fck(struct sysc *ddata,
-			      struct device *child)
+static int sysc_child_add_named_clock(struct sysc *ddata,
+				      struct device *child,
+				      const char *name)
 {
-	struct clk *fck;
+	struct clk *clk;
 	struct clk_lookup *l;
-	const char *name = clock_names[SYSC_FCK];
+	int error = 0;
 
-	if (IS_ERR_OR_NULL(ddata->clocks[SYSC_FCK]))
+	if (!name)
 		return 0;
 
-	fck = clk_get(child, name);
-	if (!IS_ERR(fck)) {
-		clk_put(fck);
+	clk = clk_get(child, name);
+	if (!IS_ERR(clk)) {
+		clk_put(clk);
 
 		return -EEXIST;
 	}
 
-	l = clkdev_create(ddata->clocks[SYSC_FCK], name, dev_name(child));
+	clk = clk_get(ddata->dev, name);
+	if (IS_ERR(clk))
+		return -ENODEV;
+
+	l = clkdev_create(clk, name, dev_name(child));
+	if (!l)
+		error = -ENOMEM;
+
+	clk_put(clk);
+
+	return error;
+}
+
+static int sysc_child_add_clocks(struct sysc *ddata,
+				 struct device *child)
+{
+	int i, error;
+
+	for (i = 0; i < ddata->nr_clocks; i++) {
+		error = sysc_child_add_named_clock(ddata,
+						   child,
+						   ddata->clock_roles[i]);
+		if (error && error != -EEXIST) {
+			dev_err(ddata->dev, "could not add child clock %s: %i\n",
+				ddata->clock_roles[i], error);
+
+			return error;
+		}
+	}
 
-	return l ? 0 : -ENODEV;
+	return 0;
 }
 
 static struct device_type sysc_device_type = {
@@ -891,18 +1164,33 @@ static int sysc_child_suspend_noirq(struct device *dev)
 
 	ddata = sysc_child_to_parent(dev);
 
+	dev_dbg(ddata->dev, "%s %s\n", __func__,
+		ddata->name ? ddata->name : "");
+
 	error = pm_generic_suspend_noirq(dev);
-	if (error)
+	if (error) {
+		dev_err(dev, "%s error at %i: %i\n",
+			__func__, __LINE__, error);
+
 		return error;
+	}
 
 	if (!pm_runtime_status_suspended(dev)) {
 		error = pm_generic_runtime_suspend(dev);
-		if (error)
+		if (error) {
+			dev_err(dev, "%s error at %i: %i\n",
+				__func__, __LINE__, error);
+
 			return error;
+		}
 
 		error = sysc_runtime_suspend(ddata->dev);
-		if (error)
+		if (error) {
+			dev_err(dev, "%s error at %i: %i\n",
+				__func__, __LINE__, error);
+
 			return error;
+		}
 
 		ddata->child_needs_resume = true;
 	}
@@ -917,6 +1205,9 @@ static int sysc_child_resume_noirq(struct device *dev)
 
 	ddata = sysc_child_to_parent(dev);
 
+	dev_dbg(ddata->dev, "%s %s\n", __func__,
+		ddata->name ? ddata->name : "");
+
 	if (ddata->child_needs_resume) {
 		ddata->child_needs_resume = false;
 
@@ -983,10 +1274,9 @@ static int sysc_notifier_call(struct notifier_block *nb,
 
 	switch (event) {
 	case BUS_NOTIFY_ADD_DEVICE:
-		error = sysc_child_add_fck(ddata, dev);
-		if (error && error != -EEXIST)
-			dev_warn(ddata->dev, "could not add %s fck: %i\n",
-				 dev_name(dev), error);
+		error = sysc_child_add_clocks(ddata, dev);
+		if (error)
+			return error;
 		sysc_legacy_idle_quirk(ddata, dev);
 		break;
 	default:
@@ -1314,6 +1604,11 @@ static void ti_sysc_idle(struct work_struct *work)
 		pm_runtime_put_sync(ddata->dev);
 }
 
+static const struct of_device_id sysc_match_table[] = {
+	{ .compatible = "simple-bus", },
+	{ /* sentinel */ },
+};
+
 static int sysc_probe(struct platform_device *pdev)
 {
 	struct ti_sysc_platform_data *pdata = dev_get_platdata(&pdev->dev);
@@ -1359,8 +1654,11 @@ static int sysc_probe(struct platform_device *pdev)
 	if (error)
 		goto unprepare;
 
-	pm_runtime_enable(ddata->dev);
+	error = sysc_init_resets(ddata);
+	if (error)
+		return error;
 
+	pm_runtime_enable(ddata->dev);
 	error = sysc_init_module(ddata);
 	if (error)
 		goto unprepare;
@@ -1375,8 +1673,8 @@ static int sysc_probe(struct platform_device *pdev)
 	sysc_show_registers(ddata);
 
 	ddata->dev->type = &sysc_device_type;
-	error = of_platform_populate(ddata->dev->of_node,
-				     NULL, pdata ? pdata->auxdata : NULL,
+	error = of_platform_populate(ddata->dev->of_node, sysc_match_table,
+				     pdata ? pdata->auxdata : NULL,
 				     ddata->dev);
 	if (error)
 		goto err;
@@ -1391,6 +1689,9 @@ static int sysc_probe(struct platform_device *pdev)
 		pm_runtime_put(&pdev->dev);
 	}
 
+	if (!of_get_available_child_count(ddata->dev->of_node))
+		reset_control_assert(ddata->rsts);
+
 	return 0;
 
 err:
@@ -1420,6 +1721,7 @@ static int sysc_remove(struct platform_device *pdev)
 
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
+	reset_control_assert(ddata->rsts);
 
 unprepare:
 	sysc_unprepare(ddata);
diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c
index bfc566d3f31a..9adc8c3eb0fa 100644
--- a/drivers/cdrom/cdrom.c
+++ b/drivers/cdrom/cdrom.c
@@ -2192,7 +2192,7 @@ static int cdrom_read_cdda_bpc(struct cdrom_device_info *cdi, __u8 __user *ubuf,
 
 		len = nr * CD_FRAMESIZE_RAW;
 
-		rq = blk_get_request(q, REQ_OP_SCSI_IN, GFP_KERNEL);
+		rq = blk_get_request(q, REQ_OP_SCSI_IN, 0);
 		if (IS_ERR(rq)) {
 			ret = PTR_ERR(rq);
 			break;
diff --git a/drivers/char/apm-emulation.c b/drivers/char/apm-emulation.c
index a5e2f9e557ea..53436c03dbce 100644
--- a/drivers/char/apm-emulation.c
+++ b/drivers/char/apm-emulation.c
@@ -461,19 +461,6 @@ static int proc_apm_show(struct seq_file *m, void *v)
 
 	return 0;
 }
-
-static int proc_apm_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_apm_show, NULL);
-}
-
-static const struct file_operations apm_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= proc_apm_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
 #endif
 
 static int kapmd(void *arg)
@@ -657,7 +644,7 @@ static int __init apm_init(void)
 	wake_up_process(kapmd_tsk);
 
 #ifdef CONFIG_PROC_FS
-	proc_create("apm", 0, NULL, &apm_proc_fops);
+	proc_create_single("apm", 0, NULL, proc_apm_show);
 #endif
 
 	ret = misc_register(&apm_device);
diff --git a/drivers/char/ds1620.c b/drivers/char/ds1620.c
index eb53cbadb68f..a5ecf6dae02e 100644
--- a/drivers/char/ds1620.c
+++ b/drivers/char/ds1620.c
@@ -345,18 +345,6 @@ static int ds1620_proc_therm_show(struct seq_file *m, void *v)
 		   fan_state[netwinder_get_fan()]);
 	return 0;
 }
-
-static int ds1620_proc_therm_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ds1620_proc_therm_show, NULL);
-}
-
-static const struct file_operations ds1620_proc_therm_fops = {
-	.open		= ds1620_proc_therm_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
 #endif
 
 static const struct file_operations ds1620_fops = {
@@ -404,7 +392,7 @@ static int __init ds1620_init(void)
 		return ret;
 
 #ifdef THERM_USE_PROC
-	if (!proc_create("therm", 0, NULL, &ds1620_proc_therm_fops))
+	if (!proc_create_single("therm", 0, NULL, ds1620_proc_therm_show))
 		printk(KERN_ERR "therm: unable to register /proc/therm\n");
 #endif
 
diff --git a/drivers/char/efirtc.c b/drivers/char/efirtc.c
index dc62568b7dde..d9aab643997e 100644
--- a/drivers/char/efirtc.c
+++ b/drivers/char/efirtc.c
@@ -358,19 +358,6 @@ static int efi_rtc_proc_show(struct seq_file *m, void *v)
 
 	return 0;
 }
-
-static int efi_rtc_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, efi_rtc_proc_show, NULL);
-}
-
-static const struct file_operations efi_rtc_proc_fops = {
-	.open		= efi_rtc_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int __init 
 efi_rtc_init(void)
 {
@@ -386,7 +373,7 @@ efi_rtc_init(void)
 		return ret;
 	}
 
-	dir = proc_create("driver/efirtc", 0, NULL, &efi_rtc_proc_fops);
+	dir = proc_create_single("driver/efirtc", 0, NULL, efi_rtc_proc_show);
 	if (dir == NULL) {
 		printk(KERN_ERR "efirtc: can't create /proc/driver/efirtc.\n");
 		misc_deregister(&efi_rtc_dev);
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index d53541e96bee..c34b257d852d 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -347,6 +347,7 @@ config HW_RANDOM_STM32
 	tristate "STMicroelectronics STM32 random number generator"
 	depends on HW_RANDOM && (ARCH_STM32 || COMPILE_TEST)
 	depends on HAS_IOMEM
+	default HW_RANDOM
 	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on STM32 microcontrollers.
diff --git a/drivers/char/hw_random/n2-drv.c b/drivers/char/hw_random/n2-drv.c
index 92dd4e925315..f8411515fe1c 100644
--- a/drivers/char/hw_random/n2-drv.c
+++ b/drivers/char/hw_random/n2-drv.c
@@ -435,7 +435,7 @@ static int n2rng_data_read(struct hwrng *rng, u32 *data)
 			*data = np->test_data & 0xffffffff;
 			len = 4;
 		} else {
-			dev_err(&np->op->dev, "RNG error, restesting\n");
+			dev_err(&np->op->dev, "RNG error, retesting\n");
 			np->flags &= ~N2RNG_FLAG_READY;
 			if (!(np->flags & N2RNG_FLAG_SHUTDOWN))
 				schedule_delayed_work(&np->work, 0);
diff --git a/drivers/char/hw_random/stm32-rng.c b/drivers/char/hw_random/stm32-rng.c
index 0d2328da3b76..042860d97b15 100644
--- a/drivers/char/hw_random/stm32-rng.c
+++ b/drivers/char/hw_random/stm32-rng.c
@@ -187,8 +187,13 @@ static int stm32_rng_runtime_resume(struct device *dev)
 }
 #endif
 
-static UNIVERSAL_DEV_PM_OPS(stm32_rng_pm_ops, stm32_rng_runtime_suspend,
-			    stm32_rng_runtime_resume, NULL);
+static const struct dev_pm_ops stm32_rng_pm_ops = {
+	SET_RUNTIME_PM_OPS(stm32_rng_runtime_suspend,
+			   stm32_rng_runtime_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+};
+
 
 static const struct of_device_id stm32_rng_match[] = {
 	{
diff --git a/drivers/char/hw_random/via-rng.c b/drivers/char/hw_random/via-rng.c
index 6e9df558325b..ffe9b0c6c647 100644
--- a/drivers/char/hw_random/via-rng.c
+++ b/drivers/char/hw_random/via-rng.c
@@ -135,7 +135,7 @@ static int via_rng_init(struct hwrng *rng)
 	 * is always enabled if CPUID rng_en is set.  There is no
 	 * RNG configuration like it used to be the case in this
 	 * register */
-	if ((c->x86 == 6) && (c->x86_model >= 0x0f)) {
+	if (((c->x86 == 6) && (c->x86_model >= 0x0f))  || (c->x86 > 6)){
 		if (!boot_cpu_has(X86_FEATURE_XSTORE_EN)) {
 			pr_err(PFX "can't enable hardware RNG "
 				"if XSTORE is not enabled\n");
diff --git a/drivers/char/ipmi/Kconfig b/drivers/char/ipmi/Kconfig
index 3bda116c8aa0..c108441882cc 100644
--- a/drivers/char/ipmi/Kconfig
+++ b/drivers/char/ipmi/Kconfig
@@ -22,14 +22,6 @@ config IPMI_DMI_DECODE
 
 if IPMI_HANDLER
 
-config IPMI_PROC_INTERFACE
-       bool 'Provide an interface for IPMI stats in /proc (deprecated)'
-       depends on PROC_FS
-       default y
-       help
-         Do not use this any more, use sysfs for this info.  It will be
-	 removed in future kernel versions.
-
 config IPMI_PANIC_EVENT
        bool 'Generate a panic event to all BMCs on a panic'
        help
@@ -111,6 +103,21 @@ config ASPEED_KCS_IPMI_BMC
 	  The driver implements the BMC side of the KCS contorller, it
 	  provides the access of KCS IO space for BMC side.
 
+config NPCM7XX_KCS_IPMI_BMC
+	depends on ARCH_NPCM7XX || COMPILE_TEST
+	select IPMI_KCS_BMC
+	select REGMAP_MMIO
+	tristate "NPCM7xx KCS IPMI BMC driver"
+	help
+	  Provides a driver for the KCS (Keyboard Controller Style) IPMI
+	  interface found on Nuvoton NPCM7xx SOCs.
+
+	  The driver implements the BMC side of the KCS contorller, it
+	  provides the access of KCS IO space for BMC side.
+
+	  This support is also available as a module.  If so, the module
+	  will be called kcs_bmc_npcm7xx.
+
 config ASPEED_BT_IPMI_BMC
 	depends on ARCH_ASPEED || COMPILE_TEST
        depends on REGMAP && REGMAP_MMIO && MFD_SYSCON
diff --git a/drivers/char/ipmi/Makefile b/drivers/char/ipmi/Makefile
index 21e9e872d973..7a3baf301a8f 100644
--- a/drivers/char/ipmi/Makefile
+++ b/drivers/char/ipmi/Makefile
@@ -24,3 +24,4 @@ obj-$(CONFIG_IPMI_POWEROFF) += ipmi_poweroff.o
 obj-$(CONFIG_IPMI_KCS_BMC) += kcs_bmc.o
 obj-$(CONFIG_ASPEED_BT_IPMI_BMC) += bt-bmc.o
 obj-$(CONFIG_ASPEED_KCS_IPMI_BMC) += kcs_bmc_aspeed.o
+obj-$(CONFIG_NPCM7XX_KCS_IPMI_BMC) += kcs_bmc_npcm7xx.o
diff --git a/drivers/char/ipmi/ipmi_bt_sm.c b/drivers/char/ipmi/ipmi_bt_sm.c
index fd4ea8d87d4b..a3397664f800 100644
--- a/drivers/char/ipmi/ipmi_bt_sm.c
+++ b/drivers/char/ipmi/ipmi_bt_sm.c
@@ -504,11 +504,12 @@ static enum si_sm_result bt_event(struct si_sm_data *bt, long time)
 		if (status & BT_H_BUSY)		/* clear a leftover H_BUSY */
 			BT_CONTROL(BT_H_BUSY);
 
+		bt->timeout = bt->BT_CAP_req2rsp;
+
 		/* Read BT capabilities if it hasn't been done yet */
 		if (!bt->BT_CAP_outreqs)
 			BT_STATE_CHANGE(BT_STATE_CAPABILITIES_BEGIN,
 					SI_SM_CALL_WITHOUT_DELAY);
-		bt->timeout = bt->BT_CAP_req2rsp;
 		BT_SI_SM_RETURN(SI_SM_IDLE);
 
 	case BT_STATE_XACTION_START:
diff --git a/drivers/char/ipmi/ipmi_devintf.c b/drivers/char/ipmi/ipmi_devintf.c
index 8ecfd47806fa..1a486aec99b6 100644
--- a/drivers/char/ipmi/ipmi_devintf.c
+++ b/drivers/char/ipmi/ipmi_devintf.c
@@ -26,7 +26,7 @@
 
 struct ipmi_file_private
 {
-	ipmi_user_t          user;
+	struct ipmi_user     *user;
 	spinlock_t           recv_msg_lock;
 	struct list_head     recv_msgs;
 	struct file          *file;
@@ -37,7 +37,6 @@ struct ipmi_file_private
 	unsigned int         default_retry_time_ms;
 };
 
-static DEFINE_MUTEX(ipmi_mutex);
 static void file_receive_handler(struct ipmi_recv_msg *msg,
 				 void                 *handler_data)
 {
@@ -45,17 +44,15 @@ static void file_receive_handler(struct ipmi_recv_msg *msg,
 	int                      was_empty;
 	unsigned long            flags;
 
-	spin_lock_irqsave(&(priv->recv_msg_lock), flags);
-
-	was_empty = list_empty(&(priv->recv_msgs));
-	list_add_tail(&(msg->link), &(priv->recv_msgs));
+	spin_lock_irqsave(&priv->recv_msg_lock, flags);
+	was_empty = list_empty(&priv->recv_msgs);
+	list_add_tail(&msg->link, &priv->recv_msgs);
+	spin_unlock_irqrestore(&priv->recv_msg_lock, flags);
 
 	if (was_empty) {
 		wake_up_interruptible(&priv->wait);
 		kill_fasync(&priv->fasync_queue, SIGIO, POLL_IN);
 	}
-
-	spin_unlock_irqrestore(&(priv->recv_msg_lock), flags);
 }
 
 static __poll_t ipmi_poll(struct file *file, poll_table *wait)
@@ -68,7 +65,7 @@ static __poll_t ipmi_poll(struct file *file, poll_table *wait)
 
 	spin_lock_irqsave(&priv->recv_msg_lock, flags);
 
-	if (!list_empty(&(priv->recv_msgs)))
+	if (!list_empty(&priv->recv_msgs))
 		mask |= (EPOLLIN | EPOLLRDNORM);
 
 	spin_unlock_irqrestore(&priv->recv_msg_lock, flags);
@@ -79,13 +76,8 @@ static __poll_t ipmi_poll(struct file *file, poll_table *wait)
 static int ipmi_fasync(int fd, struct file *file, int on)
 {
 	struct ipmi_file_private *priv = file->private_data;
-	int                      result;
 
-	mutex_lock(&ipmi_mutex); /* could race against open() otherwise */
-	result = fasync_helper(fd, file, on, &priv->fasync_queue);
-	mutex_unlock(&ipmi_mutex);
-
-	return (result);
+	return fasync_helper(fd, file, on, &priv->fasync_queue);
 }
 
 static const struct ipmi_user_hndl ipmi_hndlrs =
@@ -99,18 +91,16 @@ static int ipmi_open(struct inode *inode, struct file *file)
 	int                      rv;
 	struct ipmi_file_private *priv;
 
-
 	priv = kmalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	mutex_lock(&ipmi_mutex);
 	priv->file = file;
 
 	rv = ipmi_create_user(if_num,
 			      &ipmi_hndlrs,
 			      priv,
-			      &(priv->user));
+			      &priv->user);
 	if (rv) {
 		kfree(priv);
 		goto out;
@@ -118,8 +108,8 @@ static int ipmi_open(struct inode *inode, struct file *file)
 
 	file->private_data = priv;
 
-	spin_lock_init(&(priv->recv_msg_lock));
-	INIT_LIST_HEAD(&(priv->recv_msgs));
+	spin_lock_init(&priv->recv_msg_lock);
+	INIT_LIST_HEAD(&priv->recv_msgs);
 	init_waitqueue_head(&priv->wait);
 	priv->fasync_queue = NULL;
 	mutex_init(&priv->recv_mutex);
@@ -129,7 +119,6 @@ static int ipmi_open(struct inode *inode, struct file *file)
 	priv->default_retry_time_ms = 0;
 
 out:
-	mutex_unlock(&ipmi_mutex);
 	return rv;
 }
 
@@ -137,7 +126,7 @@ static int ipmi_release(struct inode *inode, struct file *file)
 {
 	struct ipmi_file_private *priv = file->private_data;
 	int                      rv;
-	struct  ipmi_recv_msg *msg, *next;
+	struct ipmi_recv_msg *msg, *next;
 
 	rv = ipmi_destroy_user(priv->user);
 	if (rv)
@@ -146,13 +135,12 @@ static int ipmi_release(struct inode *inode, struct file *file)
 	list_for_each_entry_safe(msg, next, &priv->recv_msgs, link)
 		ipmi_free_recv_msg(msg);
 
-
 	kfree(priv);
 
 	return 0;
 }
 
-static int handle_send_req(ipmi_user_t     user,
+static int handle_send_req(struct ipmi_user *user,
 			   struct ipmi_req *req,
 			   int             retries,
 			   unsigned int    retry_time_ms)
@@ -189,8 +177,7 @@ static int handle_send_req(ipmi_user_t     user,
 
 		if (copy_from_user(msg.data,
 				   req->msg.data,
-				   req->msg.data_len))
-		{
+				   req->msg.data_len)) {
 			rv = -EFAULT;
 			goto out;
 		}
@@ -233,25 +220,24 @@ static int handle_recv(struct ipmi_file_private *priv,
 	mutex_lock(&priv->recv_mutex);
 
 	/* Grab the message off the list. */
-	spin_lock_irqsave(&(priv->recv_msg_lock), flags);
+	spin_lock_irqsave(&priv->recv_msg_lock, flags);
 	if (list_empty(&(priv->recv_msgs))) {
-		spin_unlock_irqrestore(&(priv->recv_msg_lock), flags);
+		spin_unlock_irqrestore(&priv->recv_msg_lock, flags);
 		rv = -EAGAIN;
 		goto recv_err;
 	}
 	entry = priv->recv_msgs.next;
 	msg = list_entry(entry, struct ipmi_recv_msg, link);
 	list_del(entry);
-	spin_unlock_irqrestore(&(priv->recv_msg_lock), flags);
+	spin_unlock_irqrestore(&priv->recv_msg_lock, flags);
 
 	addr_len = ipmi_addr_length(msg->addr.addr_type);
-	if (rsp->addr_len < addr_len)
-	{
+	if (rsp->addr_len < addr_len) {
 		rv = -EINVAL;
 		goto recv_putback_on_err;
 	}
 
-	if (copy_to_user(rsp->addr, &(msg->addr), addr_len)) {
+	if (copy_to_user(rsp->addr, &msg->addr, addr_len)) {
 		rv = -EFAULT;
 		goto recv_putback_on_err;
 	}
@@ -273,8 +259,7 @@ static int handle_recv(struct ipmi_file_private *priv,
 
 		if (copy_to_user(rsp->msg.data,
 				 msg->msg.data,
-				 msg->msg.data_len))
-		{
+				 msg->msg.data_len)) {
 			rv = -EFAULT;
 			goto recv_putback_on_err;
 		}
@@ -294,9 +279,9 @@ static int handle_recv(struct ipmi_file_private *priv,
 recv_putback_on_err:
 	/* If we got an error, put the message back onto
 	   the head of the queue. */
-	spin_lock_irqsave(&(priv->recv_msg_lock), flags);
-	list_add(entry, &(priv->recv_msgs));
-	spin_unlock_irqrestore(&(priv->recv_msg_lock), flags);
+	spin_lock_irqsave(&priv->recv_msg_lock, flags);
+	list_add(entry, &priv->recv_msgs);
+	spin_unlock_irqrestore(&priv->recv_msg_lock, flags);
 recv_err:
 	mutex_unlock(&priv->recv_mutex);
 	return rv;
@@ -307,9 +292,9 @@ static int copyout_recv(struct ipmi_recv *rsp, void __user *to)
 	return copy_to_user(to, rsp, sizeof(struct ipmi_recv)) ? -EFAULT : 0;
 }
 
-static int ipmi_ioctl(struct file   *file,
-		      unsigned int  cmd,
-		      unsigned long data)
+static long ipmi_ioctl(struct file   *file,
+		       unsigned int  cmd,
+		       unsigned long data)
 {
 	int                      rv = -EINVAL;
 	struct ipmi_file_private *priv = file->private_data;
@@ -320,16 +305,20 @@ static int ipmi_ioctl(struct file   *file,
 	case IPMICTL_SEND_COMMAND:
 	{
 		struct ipmi_req req;
+		int retries;
+		unsigned int retry_time_ms;
 
 		if (copy_from_user(&req, arg, sizeof(req))) {
 			rv = -EFAULT;
 			break;
 		}
 
-		rv = handle_send_req(priv->user,
-				     &req,
-				     priv->default_retries,
-				     priv->default_retry_time_ms);
+		mutex_lock(&priv->recv_mutex);
+		retries = priv->default_retries;
+		retry_time_ms = priv->default_retry_time_ms;
+		mutex_unlock(&priv->recv_mutex);
+
+		rv = handle_send_req(priv->user, &req, retries, retry_time_ms);
 		break;
 	}
 
@@ -569,8 +558,10 @@ static int ipmi_ioctl(struct file   *file,
 			break;
 		}
 
+		mutex_lock(&priv->recv_mutex);
 		priv->default_retries = parms.retries;
 		priv->default_retry_time_ms = parms.retry_time_ms;
+		mutex_unlock(&priv->recv_mutex);
 		rv = 0;
 		break;
 	}
@@ -579,8 +570,10 @@ static int ipmi_ioctl(struct file   *file,
 	{
 		struct ipmi_timing_parms parms;
 
+		mutex_lock(&priv->recv_mutex);
 		parms.retries = priv->default_retries;
 		parms.retry_time_ms = priv->default_retry_time_ms;
+		mutex_unlock(&priv->recv_mutex);
 
 		if (copy_to_user(arg, &parms, sizeof(parms))) {
 			rv = -EFAULT;
@@ -615,30 +608,16 @@ static int ipmi_ioctl(struct file   *file,
 		rv = ipmi_set_maintenance_mode(priv->user, mode);
 		break;
 	}
+
+	default:
+		rv = -ENOTTY;
+		break;
 	}
   
 	return rv;
 }
 
-/*
- * Note: it doesn't make sense to take the BKL here but
- *       not in compat_ipmi_ioctl. -arnd
- */
-static long ipmi_unlocked_ioctl(struct file   *file,
-			        unsigned int  cmd,
-			        unsigned long data)
-{
-	int ret;
-
-	mutex_lock(&ipmi_mutex);
-	ret = ipmi_ioctl(file, cmd, data);
-	mutex_unlock(&ipmi_mutex);
-
-	return ret;
-}
-
 #ifdef CONFIG_COMPAT
-
 /*
  * The following code contains code for supporting 32-bit compatible
  * ioctls on 64-bit kernels.  This allows running 32-bit apps on the
@@ -749,15 +728,21 @@ static long compat_ipmi_ioctl(struct file *filep, unsigned int cmd,
 	{
 		struct ipmi_req	rp;
 		struct compat_ipmi_req r32;
+		int retries;
+		unsigned int retry_time_ms;
 
 		if (copy_from_user(&r32, compat_ptr(arg), sizeof(r32)))
 			return -EFAULT;
 
 		get_compat_ipmi_req(&rp, &r32);
 
+		mutex_lock(&priv->recv_mutex);
+		retries = priv->default_retries;
+		retry_time_ms = priv->default_retry_time_ms;
+		mutex_unlock(&priv->recv_mutex);
+
 		return handle_send_req(priv->user, &rp,
-				priv->default_retries,
-				priv->default_retry_time_ms);
+				       retries, retry_time_ms);
 	}
 	case COMPAT_IPMICTL_SEND_COMMAND_SETTIME:
 	{
@@ -791,25 +776,13 @@ static long compat_ipmi_ioctl(struct file *filep, unsigned int cmd,
 		return ipmi_ioctl(filep, cmd, arg);
 	}
 }
-
-static long unlocked_compat_ipmi_ioctl(struct file *filep, unsigned int cmd,
-				       unsigned long arg)
-{
-	int ret;
-
-	mutex_lock(&ipmi_mutex);
-	ret = compat_ipmi_ioctl(filep, cmd, arg);
-	mutex_unlock(&ipmi_mutex);
-
-	return ret;
-}
 #endif
 
 static const struct file_operations ipmi_fops = {
 	.owner		= THIS_MODULE,
-	.unlocked_ioctl	= ipmi_unlocked_ioctl,
+	.unlocked_ioctl	= ipmi_ioctl,
 #ifdef CONFIG_COMPAT
-	.compat_ioctl   = unlocked_compat_ipmi_ioctl,
+	.compat_ioctl   = compat_ipmi_ioctl,
 #endif
 	.open		= ipmi_open,
 	.release	= ipmi_release,
diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
index 361148938801..51832b8a2c62 100644
--- a/drivers/char/ipmi/ipmi_msghandler.c
+++ b/drivers/char/ipmi/ipmi_msghandler.c
@@ -37,11 +37,30 @@
 static struct ipmi_recv_msg *ipmi_alloc_recv_msg(void);
 static int ipmi_init_msghandler(void);
 static void smi_recv_tasklet(unsigned long);
-static void handle_new_recv_msgs(ipmi_smi_t intf);
-static void need_waiter(ipmi_smi_t intf);
-static int handle_one_recv_msg(ipmi_smi_t          intf,
+static void handle_new_recv_msgs(struct ipmi_smi *intf);
+static void need_waiter(struct ipmi_smi *intf);
+static int handle_one_recv_msg(struct ipmi_smi *intf,
 			       struct ipmi_smi_msg *msg);
 
+#ifdef DEBUG
+static void ipmi_debug_msg(const char *title, unsigned char *data,
+			   unsigned int len)
+{
+	int i, pos;
+	char buf[100];
+
+	pos = snprintf(buf, sizeof(buf), "%s: ", title);
+	for (i = 0; i < len; i++)
+		pos += snprintf(buf + pos, sizeof(buf) - pos,
+				" %2.2x", data[i]);
+	pr_debug("%s\n", buf);
+}
+#else
+static void ipmi_debug_msg(const char *title, unsigned char *data,
+			   unsigned int len)
+{ }
+#endif
+
 static int initialized;
 
 enum ipmi_panic_event_op {
@@ -112,14 +131,13 @@ module_param_cb(panic_op, &panic_op_ops, NULL, 0600);
 MODULE_PARM_DESC(panic_op, "Sets if the IPMI driver will attempt to store panic information in the event log in the event of a panic.  Set to 'none' for no, 'event' for a single event, or 'string' for a generic event and the panic string in IPMI OEM events.");
 
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-static struct proc_dir_entry *proc_ipmi_root;
-#endif /* CONFIG_IPMI_PROC_INTERFACE */
+#define MAX_EVENTS_IN_QUEUE	25
 
 /* Remain in auto-maintenance mode for this amount of time (in ms). */
-#define IPMI_MAINTENANCE_MODE_TIMEOUT 30000
-
-#define MAX_EVENTS_IN_QUEUE	25
+static unsigned long maintenance_mode_timeout_ms = 30000;
+module_param(maintenance_mode_timeout_ms, ulong, 0644);
+MODULE_PARM_DESC(maintenance_mode_timeout_ms,
+		 "The time (milliseconds) after the last maintenance message that the connection stays in maintenance mode.");
 
 /*
  * Don't let a message sit in a queue forever, always time it with at lest
@@ -127,6 +145,31 @@ static struct proc_dir_entry *proc_ipmi_root;
  */
 #define MAX_MSG_TIMEOUT		60000
 
+/*
+ * Timeout times below are in milliseconds, and are done off a 1
+ * second timer.  So setting the value to 1000 would mean anything
+ * between 0 and 1000ms.  So really the only reasonable minimum
+ * setting it 2000ms, which is between 1 and 2 seconds.
+ */
+
+/* The default timeout for message retries. */
+static unsigned long default_retry_ms = 2000;
+module_param(default_retry_ms, ulong, 0644);
+MODULE_PARM_DESC(default_retry_ms,
+		 "The time (milliseconds) between retry sends");
+
+/* The default timeout for maintenance mode message retries. */
+static unsigned long default_maintenance_retry_ms = 3000;
+module_param(default_maintenance_retry_ms, ulong, 0644);
+MODULE_PARM_DESC(default_maintenance_retry_ms,
+		 "The time (milliseconds) between retry sends in maintenance mode");
+
+/* The default maximum number of retries */
+static unsigned int default_max_retries = 4;
+module_param(default_max_retries, uint, 0644);
+MODULE_PARM_DESC(default_max_retries,
+		 "The time (milliseconds) between retry sends in maintenance mode");
+
 /* Call every ~1000 ms. */
 #define IPMI_TIMEOUT_TIME	1000
 
@@ -150,8 +193,12 @@ static struct proc_dir_entry *proc_ipmi_root;
 struct ipmi_user {
 	struct list_head link;
 
-	/* Set to false when the user is destroyed. */
-	bool valid;
+	/*
+	 * Set to NULL when the user is destroyed, a pointer to myself
+	 * so srcu_dereference can be used on it.
+	 */
+	struct ipmi_user *self;
+	struct srcu_struct release_barrier;
 
 	struct kref refcount;
 
@@ -160,16 +207,33 @@ struct ipmi_user {
 	void             *handler_data;
 
 	/* The interface this user is bound to. */
-	ipmi_smi_t intf;
+	struct ipmi_smi *intf;
 
 	/* Does this interface receive IPMI events? */
 	bool gets_events;
 };
 
+static struct ipmi_user *acquire_ipmi_user(struct ipmi_user *user, int *index)
+	__acquires(user->release_barrier)
+{
+	struct ipmi_user *ruser;
+
+	*index = srcu_read_lock(&user->release_barrier);
+	ruser = srcu_dereference(user->self, &user->release_barrier);
+	if (!ruser)
+		srcu_read_unlock(&user->release_barrier, *index);
+	return ruser;
+}
+
+static void release_ipmi_user(struct ipmi_user *user, int index)
+{
+	srcu_read_unlock(&user->release_barrier, index);
+}
+
 struct cmd_rcvr {
 	struct list_head link;
 
-	ipmi_user_t   user;
+	struct ipmi_user *user;
 	unsigned char netfn;
 	unsigned char cmd;
 	unsigned int  chans;
@@ -247,13 +311,6 @@ struct ipmi_my_addrinfo {
 	unsigned char lun;
 };
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-struct ipmi_proc_entry {
-	char                   *name;
-	struct ipmi_proc_entry *next;
-};
-#endif
-
 /*
  * Note that the product id, manufacturer id, guid, and device id are
  * immutable in this structure, so dyn_mutex is not required for
@@ -275,7 +332,7 @@ struct bmc_device {
 };
 #define to_bmc_device(x) container_of((x), struct bmc_device, pdev.dev)
 
-static int bmc_get_device_id(ipmi_smi_t intf, struct bmc_device *bmc,
+static int bmc_get_device_id(struct ipmi_smi *intf, struct bmc_device *bmc,
 			     struct ipmi_device_id *id,
 			     bool *guid_set, guid_t *guid);
 
@@ -397,10 +454,11 @@ struct ipmi_smi {
 	struct list_head link;
 
 	/*
-	 * The list of upper layers that are using me.  seq_lock
-	 * protects this.
+	 * The list of upper layers that are using me.  seq_lock write
+	 * protects this.  Read protection is with srcu.
 	 */
 	struct list_head users;
+	struct srcu_struct users_srcu;
 
 	/* Used for wake ups at startup. */
 	wait_queue_head_t waitq;
@@ -420,24 +478,9 @@ struct ipmi_smi {
 	bool in_bmc_register;  /* Handle recursive situations.  Yuck. */
 	struct work_struct bmc_reg_work;
 
-	/*
-	 * This is the lower-layer's sender routine.  Note that you
-	 * must either be holding the ipmi_interfaces_mutex or be in
-	 * an umpreemptible region to use this.  You must fetch the
-	 * value into a local variable and make sure it is not NULL.
-	 */
 	const struct ipmi_smi_handlers *handlers;
 	void                     *send_info;
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-	/* A list of proc entries for this interface. */
-	struct mutex           proc_entry_lock;
-	struct ipmi_proc_entry *proc_entries;
-
-	struct proc_dir_entry *proc_dir;
-	char                  proc_dir_name[10];
-#endif
-
 	/* Driver-model device for the system interface. */
 	struct device          *si_dev;
 
@@ -502,6 +545,13 @@ struct ipmi_smi {
 	int auto_maintenance_timeout;
 	spinlock_t maintenance_mode_lock; /* Used in a timer... */
 
+	/*
+	 * If we are doing maintenance on something on IPMB, extend
+	 * the timeout time to avoid timeouts writing firmware and
+	 * such.
+	 */
+	int ipmb_maintenance_mode_timeout;
+
 	/*
 	 * A cheap hack, if this is non-null and a message to an
 	 * interface comes in with a NULL user, call this routine with
@@ -510,7 +560,8 @@ struct ipmi_smi {
 	 *
 	 * Protected by bmc_reg_mutex.
 	 */
-	void (*null_user_handler)(ipmi_smi_t intf, struct ipmi_recv_msg *msg);
+	void (*null_user_handler)(struct ipmi_smi *intf,
+				  struct ipmi_recv_msg *msg);
 
 	/*
 	 * When we are scanning the channels for an SMI, this will
@@ -536,12 +587,12 @@ struct ipmi_smi {
 };
 #define to_si_intf_from_dev(device) container_of(device, struct ipmi_smi, dev)
 
-static void __get_guid(ipmi_smi_t intf);
-static void __ipmi_bmc_unregister(ipmi_smi_t intf);
-static int __ipmi_bmc_register(ipmi_smi_t intf,
+static void __get_guid(struct ipmi_smi *intf);
+static void __ipmi_bmc_unregister(struct ipmi_smi *intf);
+static int __ipmi_bmc_register(struct ipmi_smi *intf,
 			       struct ipmi_device_id *id,
 			       bool guid_set, guid_t *guid, int intf_num);
-static int __scan_channels(ipmi_smi_t intf, struct ipmi_device_id *id);
+static int __scan_channels(struct ipmi_smi *intf, struct ipmi_device_id *id);
 
 
 /**
@@ -560,6 +611,7 @@ static DEFINE_MUTEX(ipmidriver_mutex);
 
 static LIST_HEAD(ipmi_interfaces);
 static DEFINE_MUTEX(ipmi_interfaces_mutex);
+DEFINE_STATIC_SRCU(ipmi_interfaces_srcu);
 
 /*
  * List of watchers that want to know when smi's are added and deleted.
@@ -620,7 +672,7 @@ static void free_smi_msg_list(struct list_head *q)
 	}
 }
 
-static void clean_up_interface_data(ipmi_smi_t intf)
+static void clean_up_interface_data(struct ipmi_smi *intf)
 {
 	int              i;
 	struct cmd_rcvr  *rcvr, *rcvr2;
@@ -652,7 +704,7 @@ static void clean_up_interface_data(ipmi_smi_t intf)
 
 static void intf_free(struct kref *ref)
 {
-	ipmi_smi_t intf = container_of(ref, struct ipmi_smi, refcount);
+	struct ipmi_smi *intf = container_of(ref, struct ipmi_smi, refcount);
 
 	clean_up_interface_data(intf);
 	kfree(intf);
@@ -660,65 +712,39 @@ static void intf_free(struct kref *ref)
 
 struct watcher_entry {
 	int              intf_num;
-	ipmi_smi_t       intf;
+	struct ipmi_smi  *intf;
 	struct list_head link;
 };
 
 int ipmi_smi_watcher_register(struct ipmi_smi_watcher *watcher)
 {
-	ipmi_smi_t intf;
-	LIST_HEAD(to_deliver);
-	struct watcher_entry *e, *e2;
+	struct ipmi_smi *intf;
+	int index;
 
 	mutex_lock(&smi_watchers_mutex);
 
-	mutex_lock(&ipmi_interfaces_mutex);
-
-	/* Build a list of things to deliver. */
-	list_for_each_entry(intf, &ipmi_interfaces, link) {
-		if (intf->intf_num == -1)
-			continue;
-		e = kmalloc(sizeof(*e), GFP_KERNEL);
-		if (!e)
-			goto out_err;
-		kref_get(&intf->refcount);
-		e->intf = intf;
-		e->intf_num = intf->intf_num;
-		list_add_tail(&e->link, &to_deliver);
-	}
-
-	/* We will succeed, so add it to the list. */
 	list_add(&watcher->link, &smi_watchers);
 
-	mutex_unlock(&ipmi_interfaces_mutex);
+	index = srcu_read_lock(&ipmi_interfaces_srcu);
+	list_for_each_entry_rcu(intf, &ipmi_interfaces, link) {
+		int intf_num = READ_ONCE(intf->intf_num);
 
-	list_for_each_entry_safe(e, e2, &to_deliver, link) {
-		list_del(&e->link);
-		watcher->new_smi(e->intf_num, e->intf->si_dev);
-		kref_put(&e->intf->refcount, intf_free);
-		kfree(e);
+		if (intf_num == -1)
+			continue;
+		watcher->new_smi(intf_num, intf->si_dev);
 	}
+	srcu_read_unlock(&ipmi_interfaces_srcu, index);
 
 	mutex_unlock(&smi_watchers_mutex);
 
 	return 0;
-
- out_err:
-	mutex_unlock(&ipmi_interfaces_mutex);
-	mutex_unlock(&smi_watchers_mutex);
-	list_for_each_entry_safe(e, e2, &to_deliver, link) {
-		list_del(&e->link);
-		kref_put(&e->intf->refcount, intf_free);
-		kfree(e);
-	}
-	return -ENOMEM;
 }
 EXPORT_SYMBOL(ipmi_smi_watcher_register);
 
 int ipmi_smi_watcher_unregister(struct ipmi_smi_watcher *watcher)
 {
 	mutex_lock(&smi_watchers_mutex);
-	list_del(&(watcher->link));
+	list_del(&watcher->link);
 	mutex_unlock(&smi_watchers_mutex);
 	return 0;
 }
@@ -732,12 +758,14 @@ call_smi_watchers(int i, struct device *dev)
 {
 	struct ipmi_smi_watcher *w;
 
+	mutex_lock(&smi_watchers_mutex);
 	list_for_each_entry(w, &smi_watchers, link) {
 		if (try_module_get(w->owner)) {
 			w->new_smi(i, dev);
 			module_put(w->owner);
 		}
 	}
+	mutex_unlock(&smi_watchers_mutex);
 }
 
 static int
@@ -831,18 +859,17 @@ unsigned int ipmi_addr_length(int addr_type)
 }
 EXPORT_SYMBOL(ipmi_addr_length);
 
-static void deliver_response(struct ipmi_recv_msg *msg)
+static int deliver_response(struct ipmi_smi *intf, struct ipmi_recv_msg *msg)
 {
-	if (!msg->user) {
-		ipmi_smi_t    intf = msg->user_msg_data;
+	int rv = 0;
 
+	if (!msg->user) {
 		/* Special handling for NULL users. */
 		if (intf->null_user_handler) {
 			intf->null_user_handler(intf, msg);
-			ipmi_inc_stat(intf, handled_local_responses);
 		} else {
 			/* No handler, so give up. */
-			ipmi_inc_stat(intf, unhandled_local_responses);
+			rv = -EINVAL;
 		}
 		ipmi_free_recv_msg(msg);
 	} else if (!oops_in_progress) {
@@ -851,21 +878,40 @@ static void deliver_response(struct ipmi_recv_msg *msg)
 		 * receive handler doesn't much meaning and has a deadlock
 		 * risk.  At this moment, simply skip it in that case.
 		 */
+		int index;
+		struct ipmi_user *user = acquire_ipmi_user(msg->user, &index);
 
-		ipmi_user_t user = msg->user;
-		user->handler->ipmi_recv_hndl(msg, user->handler_data);
+		if (user) {
+			user->handler->ipmi_recv_hndl(msg, user->handler_data);
+			release_ipmi_user(msg->user, index);
+		} else {
+			/* User went away, give up. */
+			ipmi_free_recv_msg(msg);
+			rv = -EINVAL;
+		}
 	}
+
+	return rv;
 }
 
-static void
-deliver_err_response(struct ipmi_recv_msg *msg, int err)
+static void deliver_local_response(struct ipmi_smi *intf,
+				   struct ipmi_recv_msg *msg)
+{
+	if (deliver_response(intf, msg))
+		ipmi_inc_stat(intf, unhandled_local_responses);
+	else
+		ipmi_inc_stat(intf, handled_local_responses);
+}
+
+static void deliver_err_response(struct ipmi_smi *intf,
+				 struct ipmi_recv_msg *msg, int err)
 {
 	msg->recv_type = IPMI_RESPONSE_RECV_TYPE;
 	msg->msg_data[0] = err;
 	msg->msg.netfn |= 1; /* Convert to a response. */
 	msg->msg.data_len = 1;
 	msg->msg.data = msg->msg_data;
-	deliver_response(msg);
+	deliver_local_response(intf, msg);
 }
 
 /*
@@ -873,7 +919,7 @@ deliver_err_response(struct ipmi_recv_msg *msg, int err)
  * message with the given timeout to the sequence table.  This must be
  * called with the interface's seq_lock held.
  */
-static int intf_next_seq(ipmi_smi_t           intf,
+static int intf_next_seq(struct ipmi_smi      *intf,
 			 struct ipmi_recv_msg *recv_msg,
 			 unsigned long        timeout,
 			 int                  retries,
@@ -884,6 +930,11 @@ static int intf_next_seq(ipmi_smi_t           intf,
 	int          rv = 0;
 	unsigned int i;
 
+	if (timeout == 0)
+		timeout = default_retry_ms;
+	if (retries < 0)
+		retries = default_max_retries;
+
 	for (i = intf->curr_seq; (i+1)%IPMI_IPMB_NUM_SEQ != intf->curr_seq;
 					i = (i+1)%IPMI_IPMB_NUM_SEQ) {
 		if (!intf->seq_table[i].inuse)
@@ -921,7 +972,7 @@ static int intf_next_seq(ipmi_smi_t           intf,
  * guard against message coming in after their timeout and the
  * sequence number being reused).
  */
-static int intf_find_seq(ipmi_smi_t           intf,
+static int intf_find_seq(struct ipmi_smi      *intf,
 			 unsigned char        seq,
 			 short                channel,
 			 unsigned char        cmd,
@@ -935,26 +986,26 @@ static int intf_find_seq(ipmi_smi_t           intf,
 	if (seq >= IPMI_IPMB_NUM_SEQ)
 		return -EINVAL;
 
-	spin_lock_irqsave(&(intf->seq_lock), flags);
+	spin_lock_irqsave(&intf->seq_lock, flags);
 	if (intf->seq_table[seq].inuse) {
 		struct ipmi_recv_msg *msg = intf->seq_table[seq].recv_msg;
 
 		if ((msg->addr.channel == channel) && (msg->msg.cmd == cmd)
 				&& (msg->msg.netfn == netfn)
-				&& (ipmi_addr_equal(addr, &(msg->addr)))) {
+				&& (ipmi_addr_equal(addr, &msg->addr))) {
 			*recv_msg = msg;
 			intf->seq_table[seq].inuse = 0;
 			rv = 0;
 		}
 	}
-	spin_unlock_irqrestore(&(intf->seq_lock), flags);
+	spin_unlock_irqrestore(&intf->seq_lock, flags);
 
 	return rv;
 }
 
 
 /* Start the timer for a specific sequence table entry. */
-static int intf_start_seq_timer(ipmi_smi_t intf,
+static int intf_start_seq_timer(struct ipmi_smi *intf,
 				long       msgid)
 {
 	int           rv = -ENODEV;
@@ -965,24 +1016,24 @@ static int intf_start_seq_timer(ipmi_smi_t intf,
 
 	GET_SEQ_FROM_MSGID(msgid, seq, seqid);
 
-	spin_lock_irqsave(&(intf->seq_lock), flags);
+	spin_lock_irqsave(&intf->seq_lock, flags);
 	/*
 	 * We do this verification because the user can be deleted
 	 * while a message is outstanding.
 	 */
 	if ((intf->seq_table[seq].inuse)
 				&& (intf->seq_table[seq].seqid == seqid)) {
-		struct seq_table *ent = &(intf->seq_table[seq]);
+		struct seq_table *ent = &intf->seq_table[seq];
 		ent->timeout = ent->orig_timeout;
 		rv = 0;
 	}
-	spin_unlock_irqrestore(&(intf->seq_lock), flags);
+	spin_unlock_irqrestore(&intf->seq_lock, flags);
 
 	return rv;
 }
 
 /* Got an error for the send message for a specific sequence number. */
-static int intf_err_seq(ipmi_smi_t   intf,
+static int intf_err_seq(struct ipmi_smi *intf,
 			long         msgid,
 			unsigned int err)
 {
@@ -995,23 +1046,23 @@ static int intf_err_seq(ipmi_smi_t   intf,
 
 	GET_SEQ_FROM_MSGID(msgid, seq, seqid);
 
-	spin_lock_irqsave(&(intf->seq_lock), flags);
+	spin_lock_irqsave(&intf->seq_lock, flags);
 	/*
 	 * We do this verification because the user can be deleted
 	 * while a message is outstanding.
 	 */
 	if ((intf->seq_table[seq].inuse)
 				&& (intf->seq_table[seq].seqid == seqid)) {
-		struct seq_table *ent = &(intf->seq_table[seq]);
+		struct seq_table *ent = &intf->seq_table[seq];
 
 		ent->inuse = 0;
 		msg = ent->recv_msg;
 		rv = 0;
 	}
-	spin_unlock_irqrestore(&(intf->seq_lock), flags);
+	spin_unlock_irqrestore(&intf->seq_lock, flags);
 
 	if (msg)
-		deliver_err_response(msg, err);
+		deliver_err_response(intf, msg, err);
 
 	return rv;
 }
@@ -1020,12 +1071,12 @@ static int intf_err_seq(ipmi_smi_t   intf,
 int ipmi_create_user(unsigned int          if_num,
 		     const struct ipmi_user_hndl *handler,
 		     void                  *handler_data,
-		     ipmi_user_t           *user)
+		     struct ipmi_user      **user)
 {
 	unsigned long flags;
-	ipmi_user_t   new_user;
-	int           rv = 0;
-	ipmi_smi_t    intf;
+	struct ipmi_user *new_user;
+	int           rv = 0, index;
+	struct ipmi_smi *intf;
 
 	/*
 	 * There is no module usecount here, because it's not
@@ -1059,7 +1110,7 @@ int ipmi_create_user(unsigned int          if_num,
 	if (!new_user)
 		return -ENOMEM;
 
-	mutex_lock(&ipmi_interfaces_mutex);
+	index = srcu_read_lock(&ipmi_interfaces_srcu);
 	list_for_each_entry_rcu(intf, &ipmi_interfaces, link) {
 		if (intf->intf_num == if_num)
 			goto found;
@@ -1069,6 +1120,10 @@ int ipmi_create_user(unsigned int          if_num,
 	goto out_kfree;
 
  found:
+	rv = init_srcu_struct(&new_user->release_barrier);
+	if (rv)
+		goto out_kfree;
+
 	/* Note that each existing user holds a refcount to the interface. */
 	kref_get(&intf->refcount);
 
@@ -1078,26 +1133,7 @@ int ipmi_create_user(unsigned int          if_num,
 	new_user->intf = intf;
 	new_user->gets_events = false;
 
-	if (!try_module_get(intf->handlers->owner)) {
-		rv = -ENODEV;
-		goto out_kref;
-	}
-
-	if (intf->handlers->inc_usecount) {
-		rv = intf->handlers->inc_usecount(intf->send_info);
-		if (rv) {
-			module_put(intf->handlers->owner);
-			goto out_kref;
-		}
-	}
-
-	/*
-	 * Hold the lock so intf->handlers is guaranteed to be good
-	 * until now
-	 */
-	mutex_unlock(&ipmi_interfaces_mutex);
-
-	new_user->valid = true;
+	rcu_assign_pointer(new_user->self, new_user);
 	spin_lock_irqsave(&intf->seq_lock, flags);
 	list_add_rcu(&new_user->link, &intf->users);
 	spin_unlock_irqrestore(&intf->seq_lock, flags);
@@ -1106,13 +1142,12 @@ int ipmi_create_user(unsigned int          if_num,
 		if (atomic_inc_return(&intf->event_waiters) == 1)
 			need_waiter(intf);
 	}
+	srcu_read_unlock(&ipmi_interfaces_srcu, index);
 	*user = new_user;
 	return 0;
 
-out_kref:
-	kref_put(&intf->refcount, intf_free);
 out_kfree:
-	mutex_unlock(&ipmi_interfaces_mutex);
+	srcu_read_unlock(&ipmi_interfaces_srcu, index);
 	kfree(new_user);
 	return rv;
 }
@@ -1120,26 +1155,25 @@ EXPORT_SYMBOL(ipmi_create_user);
 
 int ipmi_get_smi_info(int if_num, struct ipmi_smi_info *data)
 {
-	int           rv = 0;
-	ipmi_smi_t    intf;
-	const struct ipmi_smi_handlers *handlers;
+	int rv, index;
+	struct ipmi_smi *intf;
 
-	mutex_lock(&ipmi_interfaces_mutex);
+	index = srcu_read_lock(&ipmi_interfaces_srcu);
 	list_for_each_entry_rcu(intf, &ipmi_interfaces, link) {
 		if (intf->intf_num == if_num)
 			goto found;
 	}
+	srcu_read_unlock(&ipmi_interfaces_srcu, index);
+
 	/* Not found, return an error */
-	rv = -EINVAL;
-	mutex_unlock(&ipmi_interfaces_mutex);
-	return rv;
+	return -EINVAL;
 
 found:
-	handlers = intf->handlers;
-	rv = -ENOSYS;
-	if (handlers->get_smi_info)
-		rv = handlers->get_smi_info(intf->send_info, data);
-	mutex_unlock(&ipmi_interfaces_mutex);
+	if (!intf->handlers->get_smi_info)
+		rv = -ENOTTY;
+	else
+		rv = intf->handlers->get_smi_info(intf->send_info, data);
+	srcu_read_unlock(&ipmi_interfaces_srcu, index);
 
 	return rv;
 }
@@ -1147,19 +1181,34 @@ EXPORT_SYMBOL(ipmi_get_smi_info);
 
 static void free_user(struct kref *ref)
 {
-	ipmi_user_t user = container_of(ref, struct ipmi_user, refcount);
+	struct ipmi_user *user = container_of(ref, struct ipmi_user, refcount);
 	kfree(user);
 }
 
-int ipmi_destroy_user(ipmi_user_t user)
+static void _ipmi_destroy_user(struct ipmi_user *user)
 {
-	ipmi_smi_t       intf = user->intf;
+	struct ipmi_smi  *intf = user->intf;
 	int              i;
 	unsigned long    flags;
 	struct cmd_rcvr  *rcvr;
 	struct cmd_rcvr  *rcvrs = NULL;
 
-	user->valid = false;
+	if (!acquire_ipmi_user(user, &i)) {
+		/*
+		 * The user has already been cleaned up, just make sure
+		 * nothing is using it and return.
+		 */
+		synchronize_srcu(&user->release_barrier);
+		return;
+	}
+
+	rcu_assign_pointer(user->self, NULL);
+	release_ipmi_user(user, i);
+
+	synchronize_srcu(&user->release_barrier);
+
+	if (user->handler->shutdown)
+		user->handler->shutdown(user->handler_data);
 
 	if (user->handler->ipmi_watchdog_pretimeout)
 		atomic_dec(&intf->event_waiters);
@@ -1184,7 +1233,7 @@ int ipmi_destroy_user(ipmi_user_t user)
 	 * Remove the user from the command receiver's table.  First
 	 * we build a list of everything (not using the standard link,
 	 * since other things may be using it till we do
-	 * synchronize_rcu()) then free everything in that list.
+	 * synchronize_srcu()) then free everything in that list.
 	 */
 	mutex_lock(&intf->cmd_rcvrs_mutex);
 	list_for_each_entry_rcu(rcvr, &intf->cmd_rcvrs, link) {
@@ -1202,109 +1251,156 @@ int ipmi_destroy_user(ipmi_user_t user)
 		kfree(rcvr);
 	}
 
-	mutex_lock(&ipmi_interfaces_mutex);
-	if (intf->handlers) {
-		module_put(intf->handlers->owner);
-		if (intf->handlers->dec_usecount)
-			intf->handlers->dec_usecount(intf->send_info);
-	}
-	mutex_unlock(&ipmi_interfaces_mutex);
-
 	kref_put(&intf->refcount, intf_free);
+}
 
+int ipmi_destroy_user(struct ipmi_user *user)
+{
+	_ipmi_destroy_user(user);
+
+	cleanup_srcu_struct(&user->release_barrier);
 	kref_put(&user->refcount, free_user);
 
 	return 0;
 }
 EXPORT_SYMBOL(ipmi_destroy_user);
 
-int ipmi_get_version(ipmi_user_t   user,
+int ipmi_get_version(struct ipmi_user *user,
 		     unsigned char *major,
 		     unsigned char *minor)
 {
 	struct ipmi_device_id id;
-	int rv;
+	int rv, index;
 
-	rv = bmc_get_device_id(user->intf, NULL, &id, NULL, NULL);
-	if (rv)
-		return rv;
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
 
-	*major = ipmi_version_major(&id);
-	*minor = ipmi_version_minor(&id);
+	rv = bmc_get_device_id(user->intf, NULL, &id, NULL, NULL);
+	if (!rv) {
+		*major = ipmi_version_major(&id);
+		*minor = ipmi_version_minor(&id);
+	}
+	release_ipmi_user(user, index);
 
-	return 0;
+	return rv;
 }
 EXPORT_SYMBOL(ipmi_get_version);
 
-int ipmi_set_my_address(ipmi_user_t   user,
+int ipmi_set_my_address(struct ipmi_user *user,
 			unsigned int  channel,
 			unsigned char address)
 {
+	int index, rv = 0;
+
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
+
 	if (channel >= IPMI_MAX_CHANNELS)
-		return -EINVAL;
-	user->intf->addrinfo[channel].address = address;
-	return 0;
+		rv = -EINVAL;
+	else
+		user->intf->addrinfo[channel].address = address;
+	release_ipmi_user(user, index);
+
+	return rv;
 }
 EXPORT_SYMBOL(ipmi_set_my_address);
 
-int ipmi_get_my_address(ipmi_user_t   user,
+int ipmi_get_my_address(struct ipmi_user *user,
 			unsigned int  channel,
 			unsigned char *address)
 {
+	int index, rv = 0;
+
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
+
 	if (channel >= IPMI_MAX_CHANNELS)
-		return -EINVAL;
-	*address = user->intf->addrinfo[channel].address;
-	return 0;
+		rv = -EINVAL;
+	else
+		*address = user->intf->addrinfo[channel].address;
+	release_ipmi_user(user, index);
+
+	return rv;
 }
 EXPORT_SYMBOL(ipmi_get_my_address);
 
-int ipmi_set_my_LUN(ipmi_user_t   user,
+int ipmi_set_my_LUN(struct ipmi_user *user,
 		    unsigned int  channel,
 		    unsigned char LUN)
 {
+	int index, rv = 0;
+
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
+
 	if (channel >= IPMI_MAX_CHANNELS)
-		return -EINVAL;
-	user->intf->addrinfo[channel].lun = LUN & 0x3;
+		rv = -EINVAL;
+	else
+		user->intf->addrinfo[channel].lun = LUN & 0x3;
+	release_ipmi_user(user, index);
+
 	return 0;
 }
 EXPORT_SYMBOL(ipmi_set_my_LUN);
 
-int ipmi_get_my_LUN(ipmi_user_t   user,
+int ipmi_get_my_LUN(struct ipmi_user *user,
 		    unsigned int  channel,
 		    unsigned char *address)
 {
+	int index, rv = 0;
+
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
+
 	if (channel >= IPMI_MAX_CHANNELS)
-		return -EINVAL;
-	*address = user->intf->addrinfo[channel].lun;
-	return 0;
+		rv = -EINVAL;
+	else
+		*address = user->intf->addrinfo[channel].lun;
+	release_ipmi_user(user, index);
+
+	return rv;
 }
 EXPORT_SYMBOL(ipmi_get_my_LUN);
 
-int ipmi_get_maintenance_mode(ipmi_user_t user)
+int ipmi_get_maintenance_mode(struct ipmi_user *user)
 {
-	int           mode;
+	int mode, index;
 	unsigned long flags;
 
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
+
 	spin_lock_irqsave(&user->intf->maintenance_mode_lock, flags);
 	mode = user->intf->maintenance_mode;
 	spin_unlock_irqrestore(&user->intf->maintenance_mode_lock, flags);
+	release_ipmi_user(user, index);
 
 	return mode;
 }
 EXPORT_SYMBOL(ipmi_get_maintenance_mode);
 
-static void maintenance_mode_update(ipmi_smi_t intf)
+static void maintenance_mode_update(struct ipmi_smi *intf)
 {
 	if (intf->handlers->set_maintenance_mode)
 		intf->handlers->set_maintenance_mode(
 			intf->send_info, intf->maintenance_mode_enable);
 }
 
-int ipmi_set_maintenance_mode(ipmi_user_t user, int mode)
+int ipmi_set_maintenance_mode(struct ipmi_user *user, int mode)
 {
-	int           rv = 0;
+	int rv = 0, index;
 	unsigned long flags;
-	ipmi_smi_t    intf = user->intf;
+	struct ipmi_smi *intf = user->intf;
+
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
 
 	spin_lock_irqsave(&intf->maintenance_mode_lock, flags);
 	if (intf->maintenance_mode != mode) {
@@ -1332,17 +1428,23 @@ int ipmi_set_maintenance_mode(ipmi_user_t user, int mode)
 	}
  out_unlock:
 	spin_unlock_irqrestore(&intf->maintenance_mode_lock, flags);
+	release_ipmi_user(user, index);
 
 	return rv;
 }
 EXPORT_SYMBOL(ipmi_set_maintenance_mode);
 
-int ipmi_set_gets_events(ipmi_user_t user, bool val)
+int ipmi_set_gets_events(struct ipmi_user *user, bool val)
 {
 	unsigned long        flags;
-	ipmi_smi_t           intf = user->intf;
+	struct ipmi_smi      *intf = user->intf;
 	struct ipmi_recv_msg *msg, *msg2;
 	struct list_head     msgs;
+	int index;
+
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
 
 	INIT_LIST_HEAD(&msgs);
 
@@ -1383,7 +1485,7 @@ int ipmi_set_gets_events(ipmi_user_t user, bool val)
 		list_for_each_entry_safe(msg, msg2, &msgs, link) {
 			msg->user = user;
 			kref_get(&user->refcount);
-			deliver_response(msg);
+			deliver_local_response(intf, msg);
 		}
 
 		spin_lock_irqsave(&intf->events_lock, flags);
@@ -1392,12 +1494,13 @@ int ipmi_set_gets_events(ipmi_user_t user, bool val)
 
  out:
 	spin_unlock_irqrestore(&intf->events_lock, flags);
+	release_ipmi_user(user, index);
 
 	return 0;
 }
 EXPORT_SYMBOL(ipmi_set_gets_events);
 
-static struct cmd_rcvr *find_cmd_rcvr(ipmi_smi_t    intf,
+static struct cmd_rcvr *find_cmd_rcvr(struct ipmi_smi *intf,
 				      unsigned char netfn,
 				      unsigned char cmd,
 				      unsigned char chan)
@@ -1412,7 +1515,7 @@ static struct cmd_rcvr *find_cmd_rcvr(ipmi_smi_t    intf,
 	return NULL;
 }
 
-static int is_cmd_rcvr_exclusive(ipmi_smi_t    intf,
+static int is_cmd_rcvr_exclusive(struct ipmi_smi *intf,
 				 unsigned char netfn,
 				 unsigned char cmd,
 				 unsigned int  chans)
@@ -1427,19 +1530,24 @@ static int is_cmd_rcvr_exclusive(ipmi_smi_t    intf,
 	return 1;
 }
 
-int ipmi_register_for_cmd(ipmi_user_t   user,
+int ipmi_register_for_cmd(struct ipmi_user *user,
 			  unsigned char netfn,
 			  unsigned char cmd,
 			  unsigned int  chans)
 {
-	ipmi_smi_t      intf = user->intf;
+	struct ipmi_smi *intf = user->intf;
 	struct cmd_rcvr *rcvr;
-	int             rv = 0;
+	int rv = 0, index;
 
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
 
 	rcvr = kmalloc(sizeof(*rcvr), GFP_KERNEL);
-	if (!rcvr)
-		return -ENOMEM;
+	if (!rcvr) {
+		rv = -ENOMEM;
+		goto out_release;
+	}
 	rcvr->cmd = cmd;
 	rcvr->netfn = netfn;
 	rcvr->chans = chans;
@@ -1457,24 +1565,30 @@ int ipmi_register_for_cmd(ipmi_user_t   user,
 
 	list_add_rcu(&rcvr->link, &intf->cmd_rcvrs);
 
- out_unlock:
+out_unlock:
 	mutex_unlock(&intf->cmd_rcvrs_mutex);
 	if (rv)
 		kfree(rcvr);
+out_release:
+	release_ipmi_user(user, index);
 
 	return rv;
 }
 EXPORT_SYMBOL(ipmi_register_for_cmd);
 
-int ipmi_unregister_for_cmd(ipmi_user_t   user,
+int ipmi_unregister_for_cmd(struct ipmi_user *user,
 			    unsigned char netfn,
 			    unsigned char cmd,
 			    unsigned int  chans)
 {
-	ipmi_smi_t      intf = user->intf;
+	struct ipmi_smi *intf = user->intf;
 	struct cmd_rcvr *rcvr;
 	struct cmd_rcvr *rcvrs = NULL;
-	int i, rv = -ENOENT;
+	int i, rv = -ENOENT, index;
+
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
 
 	mutex_lock(&intf->cmd_rcvrs_mutex);
 	for (i = 0; i < IPMI_NUM_CHANNELS; i++) {
@@ -1495,12 +1609,14 @@ int ipmi_unregister_for_cmd(ipmi_user_t   user,
 	}
 	mutex_unlock(&intf->cmd_rcvrs_mutex);
 	synchronize_rcu();
+	release_ipmi_user(user, index);
 	while (rcvrs) {
 		atomic_dec(&intf->event_waiters);
 		rcvr = rcvrs;
 		rcvrs = rcvr->next;
 		kfree(rcvr);
 	}
+
 	return rv;
 }
 EXPORT_SYMBOL(ipmi_unregister_for_cmd);
@@ -1535,21 +1651,19 @@ static inline void format_ipmb_msg(struct ipmi_smi_msg   *smi_msg,
 		smi_msg->data[3] = 0;
 	smi_msg->data[i+3] = ipmb_addr->slave_addr;
 	smi_msg->data[i+4] = (msg->netfn << 2) | (ipmb_addr->lun & 0x3);
-	smi_msg->data[i+5] = ipmb_checksum(&(smi_msg->data[i+3]), 2);
+	smi_msg->data[i+5] = ipmb_checksum(&smi_msg->data[i + 3], 2);
 	smi_msg->data[i+6] = source_address;
 	smi_msg->data[i+7] = (ipmb_seq << 2) | source_lun;
 	smi_msg->data[i+8] = msg->cmd;
 
 	/* Now tack on the data to the message. */
 	if (msg->data_len > 0)
-		memcpy(&(smi_msg->data[i+9]), msg->data,
-		       msg->data_len);
+		memcpy(&smi_msg->data[i + 9], msg->data, msg->data_len);
 	smi_msg->data_size = msg->data_len + 9;
 
 	/* Now calculate the checksum and tack it on. */
 	smi_msg->data[i+smi_msg->data_size]
-		= ipmb_checksum(&(smi_msg->data[i+6]),
-				smi_msg->data_size-6);
+		= ipmb_checksum(&smi_msg->data[i + 6], smi_msg->data_size - 6);
 
 	/*
 	 * Add on the checksum size and the offset from the
@@ -1574,21 +1688,19 @@ static inline void format_lan_msg(struct ipmi_smi_msg   *smi_msg,
 	smi_msg->data[3] = lan_addr->session_handle;
 	smi_msg->data[4] = lan_addr->remote_SWID;
 	smi_msg->data[5] = (msg->netfn << 2) | (lan_addr->lun & 0x3);
-	smi_msg->data[6] = ipmb_checksum(&(smi_msg->data[4]), 2);
+	smi_msg->data[6] = ipmb_checksum(&smi_msg->data[4], 2);
 	smi_msg->data[7] = lan_addr->local_SWID;
 	smi_msg->data[8] = (ipmb_seq << 2) | source_lun;
 	smi_msg->data[9] = msg->cmd;
 
 	/* Now tack on the data to the message. */
 	if (msg->data_len > 0)
-		memcpy(&(smi_msg->data[10]), msg->data,
-		       msg->data_len);
+		memcpy(&smi_msg->data[10], msg->data, msg->data_len);
 	smi_msg->data_size = msg->data_len + 10;
 
 	/* Now calculate the checksum and tack it on. */
 	smi_msg->data[smi_msg->data_size]
-		= ipmb_checksum(&(smi_msg->data[7]),
-				smi_msg->data_size-7);
+		= ipmb_checksum(&smi_msg->data[7], smi_msg->data_size - 7);
 
 	/*
 	 * Add on the checksum size and the offset from the
@@ -1599,7 +1711,7 @@ static inline void format_lan_msg(struct ipmi_smi_msg   *smi_msg,
 	smi_msg->msgid = msgid;
 }
 
-static struct ipmi_smi_msg *smi_add_send_msg(ipmi_smi_t intf,
+static struct ipmi_smi_msg *smi_add_send_msg(struct ipmi_smi *intf,
 					     struct ipmi_smi_msg *smi_msg,
 					     int priority)
 {
@@ -1617,7 +1729,8 @@ static struct ipmi_smi_msg *smi_add_send_msg(ipmi_smi_t intf,
 }
 
 
-static void smi_send(ipmi_smi_t intf, const struct ipmi_smi_handlers *handlers,
+static void smi_send(struct ipmi_smi *intf,
+		     const struct ipmi_smi_handlers *handlers,
 		     struct ipmi_smi_msg *smi_msg, int priority)
 {
 	int run_to_completion = intf->run_to_completion;
@@ -1636,405 +1749,435 @@ static void smi_send(ipmi_smi_t intf, const struct ipmi_smi_handlers *handlers,
 		handlers->sender(intf->send_info, smi_msg);
 }
 
-/*
- * Separate from ipmi_request so that the user does not have to be
- * supplied in certain circumstances (mainly at panic time).  If
- * messages are supplied, they will be freed, even if an error
- * occurs.
- */
-static int i_ipmi_request(ipmi_user_t          user,
-			  ipmi_smi_t           intf,
-			  struct ipmi_addr     *addr,
-			  long                 msgid,
-			  struct kernel_ipmi_msg *msg,
-			  void                 *user_msg_data,
-			  void                 *supplied_smi,
-			  struct ipmi_recv_msg *supplied_recv,
-			  int                  priority,
-			  unsigned char        source_address,
-			  unsigned char        source_lun,
-			  int                  retries,
-			  unsigned int         retry_time_ms)
+static bool is_maintenance_mode_cmd(struct kernel_ipmi_msg *msg)
 {
-	int                      rv = 0;
-	struct ipmi_smi_msg      *smi_msg;
-	struct ipmi_recv_msg     *recv_msg;
-	unsigned long            flags;
+	return (((msg->netfn == IPMI_NETFN_APP_REQUEST)
+		 && ((msg->cmd == IPMI_COLD_RESET_CMD)
+		     || (msg->cmd == IPMI_WARM_RESET_CMD)))
+		|| (msg->netfn == IPMI_NETFN_FIRMWARE_REQUEST));
+}
 
+static int i_ipmi_req_sysintf(struct ipmi_smi        *intf,
+			      struct ipmi_addr       *addr,
+			      long                   msgid,
+			      struct kernel_ipmi_msg *msg,
+			      struct ipmi_smi_msg    *smi_msg,
+			      struct ipmi_recv_msg   *recv_msg,
+			      int                    retries,
+			      unsigned int           retry_time_ms)
+{
+	struct ipmi_system_interface_addr *smi_addr;
 
-	if (supplied_recv)
-		recv_msg = supplied_recv;
-	else {
-		recv_msg = ipmi_alloc_recv_msg();
-		if (recv_msg == NULL)
-			return -ENOMEM;
-	}
-	recv_msg->user_msg_data = user_msg_data;
+	if (msg->netfn & 1)
+		/* Responses are not allowed to the SMI. */
+		return -EINVAL;
 
-	if (supplied_smi)
-		smi_msg = (struct ipmi_smi_msg *) supplied_smi;
-	else {
-		smi_msg = ipmi_alloc_smi_msg();
-		if (smi_msg == NULL) {
-			ipmi_free_recv_msg(recv_msg);
-			return -ENOMEM;
-		}
+	smi_addr = (struct ipmi_system_interface_addr *) addr;
+	if (smi_addr->lun > 3) {
+		ipmi_inc_stat(intf, sent_invalid_commands);
+		return -EINVAL;
 	}
 
-	rcu_read_lock();
-	if (intf->in_shutdown) {
-		rv = -ENODEV;
-		goto out_err;
-	}
+	memcpy(&recv_msg->addr, smi_addr, sizeof(*smi_addr));
 
-	recv_msg->user = user;
-	if (user)
-		kref_get(&user->refcount);
-	recv_msg->msgid = msgid;
-	/*
-	 * Store the message to send in the receive message so timeout
-	 * responses can get the proper response data.
-	 */
-	recv_msg->msg = *msg;
+	if ((msg->netfn == IPMI_NETFN_APP_REQUEST)
+	    && ((msg->cmd == IPMI_SEND_MSG_CMD)
+		|| (msg->cmd == IPMI_GET_MSG_CMD)
+		|| (msg->cmd == IPMI_READ_EVENT_MSG_BUFFER_CMD))) {
+		/*
+		 * We don't let the user do these, since we manage
+		 * the sequence numbers.
+		 */
+		ipmi_inc_stat(intf, sent_invalid_commands);
+		return -EINVAL;
+	}
 
-	if (addr->addr_type == IPMI_SYSTEM_INTERFACE_ADDR_TYPE) {
-		struct ipmi_system_interface_addr *smi_addr;
+	if (is_maintenance_mode_cmd(msg)) {
+		unsigned long flags;
 
-		if (msg->netfn & 1) {
-			/* Responses are not allowed to the SMI. */
-			rv = -EINVAL;
-			goto out_err;
+		spin_lock_irqsave(&intf->maintenance_mode_lock, flags);
+		intf->auto_maintenance_timeout
+			= maintenance_mode_timeout_ms;
+		if (!intf->maintenance_mode
+		    && !intf->maintenance_mode_enable) {
+			intf->maintenance_mode_enable = true;
+			maintenance_mode_update(intf);
 		}
+		spin_unlock_irqrestore(&intf->maintenance_mode_lock,
+				       flags);
+	}
 
-		smi_addr = (struct ipmi_system_interface_addr *) addr;
-		if (smi_addr->lun > 3) {
-			ipmi_inc_stat(intf, sent_invalid_commands);
-			rv = -EINVAL;
-			goto out_err;
-		}
+	if (msg->data_len + 2 > IPMI_MAX_MSG_LENGTH) {
+		ipmi_inc_stat(intf, sent_invalid_commands);
+		return -EMSGSIZE;
+	}
 
-		memcpy(&recv_msg->addr, smi_addr, sizeof(*smi_addr));
+	smi_msg->data[0] = (msg->netfn << 2) | (smi_addr->lun & 0x3);
+	smi_msg->data[1] = msg->cmd;
+	smi_msg->msgid = msgid;
+	smi_msg->user_data = recv_msg;
+	if (msg->data_len > 0)
+		memcpy(&smi_msg->data[2], msg->data, msg->data_len);
+	smi_msg->data_size = msg->data_len + 2;
+	ipmi_inc_stat(intf, sent_local_commands);
 
-		if ((msg->netfn == IPMI_NETFN_APP_REQUEST)
-		    && ((msg->cmd == IPMI_SEND_MSG_CMD)
-			|| (msg->cmd == IPMI_GET_MSG_CMD)
-			|| (msg->cmd == IPMI_READ_EVENT_MSG_BUFFER_CMD))) {
-			/*
-			 * We don't let the user do these, since we manage
-			 * the sequence numbers.
-			 */
-			ipmi_inc_stat(intf, sent_invalid_commands);
-			rv = -EINVAL;
-			goto out_err;
-		}
+	return 0;
+}
 
-		if (((msg->netfn == IPMI_NETFN_APP_REQUEST)
-		      && ((msg->cmd == IPMI_COLD_RESET_CMD)
-			  || (msg->cmd == IPMI_WARM_RESET_CMD)))
-		     || (msg->netfn == IPMI_NETFN_FIRMWARE_REQUEST)) {
-			spin_lock_irqsave(&intf->maintenance_mode_lock, flags);
-			intf->auto_maintenance_timeout
-				= IPMI_MAINTENANCE_MODE_TIMEOUT;
-			if (!intf->maintenance_mode
-			    && !intf->maintenance_mode_enable) {
-				intf->maintenance_mode_enable = true;
-				maintenance_mode_update(intf);
-			}
-			spin_unlock_irqrestore(&intf->maintenance_mode_lock,
-					       flags);
-		}
+static int i_ipmi_req_ipmb(struct ipmi_smi        *intf,
+			   struct ipmi_addr       *addr,
+			   long                   msgid,
+			   struct kernel_ipmi_msg *msg,
+			   struct ipmi_smi_msg    *smi_msg,
+			   struct ipmi_recv_msg   *recv_msg,
+			   unsigned char          source_address,
+			   unsigned char          source_lun,
+			   int                    retries,
+			   unsigned int           retry_time_ms)
+{
+	struct ipmi_ipmb_addr *ipmb_addr;
+	unsigned char ipmb_seq;
+	long seqid;
+	int broadcast = 0;
+	struct ipmi_channel *chans;
+	int rv = 0;
 
-		if ((msg->data_len + 2) > IPMI_MAX_MSG_LENGTH) {
-			ipmi_inc_stat(intf, sent_invalid_commands);
-			rv = -EMSGSIZE;
-			goto out_err;
-		}
+	if (addr->channel >= IPMI_MAX_CHANNELS) {
+		ipmi_inc_stat(intf, sent_invalid_commands);
+		return -EINVAL;
+	}
 
-		smi_msg->data[0] = (msg->netfn << 2) | (smi_addr->lun & 0x3);
-		smi_msg->data[1] = msg->cmd;
-		smi_msg->msgid = msgid;
-		smi_msg->user_data = recv_msg;
-		if (msg->data_len > 0)
-			memcpy(&(smi_msg->data[2]), msg->data, msg->data_len);
-		smi_msg->data_size = msg->data_len + 2;
-		ipmi_inc_stat(intf, sent_local_commands);
-	} else if (is_ipmb_addr(addr) || is_ipmb_bcast_addr(addr)) {
-		struct ipmi_ipmb_addr *ipmb_addr;
-		unsigned char         ipmb_seq;
-		long                  seqid;
-		int                   broadcast = 0;
-		struct ipmi_channel   *chans;
+	chans = READ_ONCE(intf->channel_list)->c;
 
-		if (addr->channel >= IPMI_MAX_CHANNELS) {
-			ipmi_inc_stat(intf, sent_invalid_commands);
-			rv = -EINVAL;
-			goto out_err;
-		}
+	if (chans[addr->channel].medium != IPMI_CHANNEL_MEDIUM_IPMB) {
+		ipmi_inc_stat(intf, sent_invalid_commands);
+		return -EINVAL;
+	}
 
-		chans = READ_ONCE(intf->channel_list)->c;
+	if (addr->addr_type == IPMI_IPMB_BROADCAST_ADDR_TYPE) {
+		/*
+		 * Broadcasts add a zero at the beginning of the
+		 * message, but otherwise is the same as an IPMB
+		 * address.
+		 */
+		addr->addr_type = IPMI_IPMB_ADDR_TYPE;
+		broadcast = 1;
+		retries = 0; /* Don't retry broadcasts. */
+	}
 
-		if (chans[addr->channel].medium != IPMI_CHANNEL_MEDIUM_IPMB) {
-			ipmi_inc_stat(intf, sent_invalid_commands);
-			rv = -EINVAL;
-			goto out_err;
-		}
+	/*
+	 * 9 for the header and 1 for the checksum, plus
+	 * possibly one for the broadcast.
+	 */
+	if ((msg->data_len + 10 + broadcast) > IPMI_MAX_MSG_LENGTH) {
+		ipmi_inc_stat(intf, sent_invalid_commands);
+		return -EMSGSIZE;
+	}
 
-		if (retries < 0) {
-		    if (addr->addr_type == IPMI_IPMB_BROADCAST_ADDR_TYPE)
-			retries = 0; /* Don't retry broadcasts. */
-		    else
-			retries = 4;
-		}
-		if (addr->addr_type == IPMI_IPMB_BROADCAST_ADDR_TYPE) {
-		    /*
-		     * Broadcasts add a zero at the beginning of the
-		     * message, but otherwise is the same as an IPMB
-		     * address.
-		     */
-		    addr->addr_type = IPMI_IPMB_ADDR_TYPE;
-		    broadcast = 1;
-		}
+	ipmb_addr = (struct ipmi_ipmb_addr *) addr;
+	if (ipmb_addr->lun > 3) {
+		ipmi_inc_stat(intf, sent_invalid_commands);
+		return -EINVAL;
+	}
 
+	memcpy(&recv_msg->addr, ipmb_addr, sizeof(*ipmb_addr));
 
-		/* Default to 1 second retries. */
-		if (retry_time_ms == 0)
-		    retry_time_ms = 1000;
+	if (recv_msg->msg.netfn & 0x1) {
+		/*
+		 * It's a response, so use the user's sequence
+		 * from msgid.
+		 */
+		ipmi_inc_stat(intf, sent_ipmb_responses);
+		format_ipmb_msg(smi_msg, msg, ipmb_addr, msgid,
+				msgid, broadcast,
+				source_address, source_lun);
 
 		/*
-		 * 9 for the header and 1 for the checksum, plus
-		 * possibly one for the broadcast.
+		 * Save the receive message so we can use it
+		 * to deliver the response.
 		 */
-		if ((msg->data_len + 10 + broadcast) > IPMI_MAX_MSG_LENGTH) {
-			ipmi_inc_stat(intf, sent_invalid_commands);
-			rv = -EMSGSIZE;
-			goto out_err;
-		}
+		smi_msg->user_data = recv_msg;
+	} else {
+		/* It's a command, so get a sequence for it. */
+		unsigned long flags;
 
-		ipmb_addr = (struct ipmi_ipmb_addr *) addr;
-		if (ipmb_addr->lun > 3) {
-			ipmi_inc_stat(intf, sent_invalid_commands);
-			rv = -EINVAL;
-			goto out_err;
-		}
+		spin_lock_irqsave(&intf->seq_lock, flags);
 
-		memcpy(&recv_msg->addr, ipmb_addr, sizeof(*ipmb_addr));
+		if (is_maintenance_mode_cmd(msg))
+			intf->ipmb_maintenance_mode_timeout =
+				maintenance_mode_timeout_ms;
 
-		if (recv_msg->msg.netfn & 0x1) {
-			/*
-			 * It's a response, so use the user's sequence
-			 * from msgid.
-			 */
-			ipmi_inc_stat(intf, sent_ipmb_responses);
-			format_ipmb_msg(smi_msg, msg, ipmb_addr, msgid,
-					msgid, broadcast,
-					source_address, source_lun);
+		if (intf->ipmb_maintenance_mode_timeout && retry_time_ms == 0)
+			/* Different default in maintenance mode */
+			retry_time_ms = default_maintenance_retry_ms;
 
+		/*
+		 * Create a sequence number with a 1 second
+		 * timeout and 4 retries.
+		 */
+		rv = intf_next_seq(intf,
+				   recv_msg,
+				   retry_time_ms,
+				   retries,
+				   broadcast,
+				   &ipmb_seq,
+				   &seqid);
+		if (rv)
 			/*
-			 * Save the receive message so we can use it
-			 * to deliver the response.
+			 * We have used up all the sequence numbers,
+			 * probably, so abort.
 			 */
-			smi_msg->user_data = recv_msg;
-		} else {
-			/* It's a command, so get a sequence for it. */
+			goto out_err;
 
-			spin_lock_irqsave(&(intf->seq_lock), flags);
+		ipmi_inc_stat(intf, sent_ipmb_commands);
 
-			/*
-			 * Create a sequence number with a 1 second
-			 * timeout and 4 retries.
-			 */
-			rv = intf_next_seq(intf,
-					   recv_msg,
-					   retry_time_ms,
-					   retries,
-					   broadcast,
-					   &ipmb_seq,
-					   &seqid);
-			if (rv) {
-				/*
-				 * We have used up all the sequence numbers,
-				 * probably, so abort.
-				 */
-				spin_unlock_irqrestore(&(intf->seq_lock),
-						       flags);
-				goto out_err;
-			}
+		/*
+		 * Store the sequence number in the message,
+		 * so that when the send message response
+		 * comes back we can start the timer.
+		 */
+		format_ipmb_msg(smi_msg, msg, ipmb_addr,
+				STORE_SEQ_IN_MSGID(ipmb_seq, seqid),
+				ipmb_seq, broadcast,
+				source_address, source_lun);
 
-			ipmi_inc_stat(intf, sent_ipmb_commands);
+		/*
+		 * Copy the message into the recv message data, so we
+		 * can retransmit it later if necessary.
+		 */
+		memcpy(recv_msg->msg_data, smi_msg->data,
+		       smi_msg->data_size);
+		recv_msg->msg.data = recv_msg->msg_data;
+		recv_msg->msg.data_len = smi_msg->data_size;
 
-			/*
-			 * Store the sequence number in the message,
-			 * so that when the send message response
-			 * comes back we can start the timer.
-			 */
-			format_ipmb_msg(smi_msg, msg, ipmb_addr,
-					STORE_SEQ_IN_MSGID(ipmb_seq, seqid),
-					ipmb_seq, broadcast,
-					source_address, source_lun);
+		/*
+		 * We don't unlock until here, because we need
+		 * to copy the completed message into the
+		 * recv_msg before we release the lock.
+		 * Otherwise, race conditions may bite us.  I
+		 * know that's pretty paranoid, but I prefer
+		 * to be correct.
+		 */
+out_err:
+		spin_unlock_irqrestore(&intf->seq_lock, flags);
+	}
 
-			/*
-			 * Copy the message into the recv message data, so we
-			 * can retransmit it later if necessary.
-			 */
-			memcpy(recv_msg->msg_data, smi_msg->data,
-			       smi_msg->data_size);
-			recv_msg->msg.data = recv_msg->msg_data;
-			recv_msg->msg.data_len = smi_msg->data_size;
+	return rv;
+}
 
-			/*
-			 * We don't unlock until here, because we need
-			 * to copy the completed message into the
-			 * recv_msg before we release the lock.
-			 * Otherwise, race conditions may bite us.  I
-			 * know that's pretty paranoid, but I prefer
-			 * to be correct.
-			 */
-			spin_unlock_irqrestore(&(intf->seq_lock), flags);
-		}
-	} else if (is_lan_addr(addr)) {
-		struct ipmi_lan_addr  *lan_addr;
-		unsigned char         ipmb_seq;
-		long                  seqid;
-		struct ipmi_channel   *chans;
+static int i_ipmi_req_lan(struct ipmi_smi        *intf,
+			  struct ipmi_addr       *addr,
+			  long                   msgid,
+			  struct kernel_ipmi_msg *msg,
+			  struct ipmi_smi_msg    *smi_msg,
+			  struct ipmi_recv_msg   *recv_msg,
+			  unsigned char          source_lun,
+			  int                    retries,
+			  unsigned int           retry_time_ms)
+{
+	struct ipmi_lan_addr  *lan_addr;
+	unsigned char ipmb_seq;
+	long seqid;
+	struct ipmi_channel *chans;
+	int rv = 0;
 
-		if (addr->channel >= IPMI_MAX_CHANNELS) {
-			ipmi_inc_stat(intf, sent_invalid_commands);
-			rv = -EINVAL;
-			goto out_err;
-		}
+	if (addr->channel >= IPMI_MAX_CHANNELS) {
+		ipmi_inc_stat(intf, sent_invalid_commands);
+		return -EINVAL;
+	}
 
-		chans = READ_ONCE(intf->channel_list)->c;
+	chans = READ_ONCE(intf->channel_list)->c;
 
-		if ((chans[addr->channel].medium
+	if ((chans[addr->channel].medium
 				!= IPMI_CHANNEL_MEDIUM_8023LAN)
-		    && (chans[addr->channel].medium
-				!= IPMI_CHANNEL_MEDIUM_ASYNC)) {
-			ipmi_inc_stat(intf, sent_invalid_commands);
-			rv = -EINVAL;
-			goto out_err;
-		}
+			&& (chans[addr->channel].medium
+			    != IPMI_CHANNEL_MEDIUM_ASYNC)) {
+		ipmi_inc_stat(intf, sent_invalid_commands);
+		return -EINVAL;
+	}
 
-		retries = 4;
+	/* 11 for the header and 1 for the checksum. */
+	if ((msg->data_len + 12) > IPMI_MAX_MSG_LENGTH) {
+		ipmi_inc_stat(intf, sent_invalid_commands);
+		return -EMSGSIZE;
+	}
 
-		/* Default to 1 second retries. */
-		if (retry_time_ms == 0)
-		    retry_time_ms = 1000;
+	lan_addr = (struct ipmi_lan_addr *) addr;
+	if (lan_addr->lun > 3) {
+		ipmi_inc_stat(intf, sent_invalid_commands);
+		return -EINVAL;
+	}
 
-		/* 11 for the header and 1 for the checksum. */
-		if ((msg->data_len + 12) > IPMI_MAX_MSG_LENGTH) {
-			ipmi_inc_stat(intf, sent_invalid_commands);
-			rv = -EMSGSIZE;
-			goto out_err;
-		}
+	memcpy(&recv_msg->addr, lan_addr, sizeof(*lan_addr));
 
-		lan_addr = (struct ipmi_lan_addr *) addr;
-		if (lan_addr->lun > 3) {
-			ipmi_inc_stat(intf, sent_invalid_commands);
-			rv = -EINVAL;
-			goto out_err;
-		}
+	if (recv_msg->msg.netfn & 0x1) {
+		/*
+		 * It's a response, so use the user's sequence
+		 * from msgid.
+		 */
+		ipmi_inc_stat(intf, sent_lan_responses);
+		format_lan_msg(smi_msg, msg, lan_addr, msgid,
+			       msgid, source_lun);
 
-		memcpy(&recv_msg->addr, lan_addr, sizeof(*lan_addr));
+		/*
+		 * Save the receive message so we can use it
+		 * to deliver the response.
+		 */
+		smi_msg->user_data = recv_msg;
+	} else {
+		/* It's a command, so get a sequence for it. */
+		unsigned long flags;
 
-		if (recv_msg->msg.netfn & 0x1) {
-			/*
-			 * It's a response, so use the user's sequence
-			 * from msgid.
-			 */
-			ipmi_inc_stat(intf, sent_lan_responses);
-			format_lan_msg(smi_msg, msg, lan_addr, msgid,
-				       msgid, source_lun);
+		spin_lock_irqsave(&intf->seq_lock, flags);
 
+		/*
+		 * Create a sequence number with a 1 second
+		 * timeout and 4 retries.
+		 */
+		rv = intf_next_seq(intf,
+				   recv_msg,
+				   retry_time_ms,
+				   retries,
+				   0,
+				   &ipmb_seq,
+				   &seqid);
+		if (rv)
 			/*
-			 * Save the receive message so we can use it
-			 * to deliver the response.
+			 * We have used up all the sequence numbers,
+			 * probably, so abort.
 			 */
-			smi_msg->user_data = recv_msg;
-		} else {
-			/* It's a command, so get a sequence for it. */
+			goto out_err;
 
-			spin_lock_irqsave(&(intf->seq_lock), flags);
+		ipmi_inc_stat(intf, sent_lan_commands);
 
-			/*
-			 * Create a sequence number with a 1 second
-			 * timeout and 4 retries.
-			 */
-			rv = intf_next_seq(intf,
-					   recv_msg,
-					   retry_time_ms,
-					   retries,
-					   0,
-					   &ipmb_seq,
-					   &seqid);
-			if (rv) {
-				/*
-				 * We have used up all the sequence numbers,
-				 * probably, so abort.
-				 */
-				spin_unlock_irqrestore(&(intf->seq_lock),
-						       flags);
-				goto out_err;
-			}
+		/*
+		 * Store the sequence number in the message,
+		 * so that when the send message response
+		 * comes back we can start the timer.
+		 */
+		format_lan_msg(smi_msg, msg, lan_addr,
+			       STORE_SEQ_IN_MSGID(ipmb_seq, seqid),
+			       ipmb_seq, source_lun);
+
+		/*
+		 * Copy the message into the recv message data, so we
+		 * can retransmit it later if necessary.
+		 */
+		memcpy(recv_msg->msg_data, smi_msg->data,
+		       smi_msg->data_size);
+		recv_msg->msg.data = recv_msg->msg_data;
+		recv_msg->msg.data_len = smi_msg->data_size;
+
+		/*
+		 * We don't unlock until here, because we need
+		 * to copy the completed message into the
+		 * recv_msg before we release the lock.
+		 * Otherwise, race conditions may bite us.  I
+		 * know that's pretty paranoid, but I prefer
+		 * to be correct.
+		 */
+out_err:
+		spin_unlock_irqrestore(&intf->seq_lock, flags);
+	}
 
-			ipmi_inc_stat(intf, sent_lan_commands);
+	return rv;
+}
+
+/*
+ * Separate from ipmi_request so that the user does not have to be
+ * supplied in certain circumstances (mainly at panic time).  If
+ * messages are supplied, they will be freed, even if an error
+ * occurs.
+ */
+static int i_ipmi_request(struct ipmi_user     *user,
+			  struct ipmi_smi      *intf,
+			  struct ipmi_addr     *addr,
+			  long                 msgid,
+			  struct kernel_ipmi_msg *msg,
+			  void                 *user_msg_data,
+			  void                 *supplied_smi,
+			  struct ipmi_recv_msg *supplied_recv,
+			  int                  priority,
+			  unsigned char        source_address,
+			  unsigned char        source_lun,
+			  int                  retries,
+			  unsigned int         retry_time_ms)
+{
+	struct ipmi_smi_msg *smi_msg;
+	struct ipmi_recv_msg *recv_msg;
+	int rv = 0;
+
+	if (supplied_recv)
+		recv_msg = supplied_recv;
+	else {
+		recv_msg = ipmi_alloc_recv_msg();
+		if (recv_msg == NULL) {
+			rv = -ENOMEM;
+			goto out;
+		}
+	}
+	recv_msg->user_msg_data = user_msg_data;
+
+	if (supplied_smi)
+		smi_msg = (struct ipmi_smi_msg *) supplied_smi;
+	else {
+		smi_msg = ipmi_alloc_smi_msg();
+		if (smi_msg == NULL) {
+			ipmi_free_recv_msg(recv_msg);
+			rv = -ENOMEM;
+			goto out;
+		}
+	}
 
-			/*
-			 * Store the sequence number in the message,
-			 * so that when the send message response
-			 * comes back we can start the timer.
-			 */
-			format_lan_msg(smi_msg, msg, lan_addr,
-				       STORE_SEQ_IN_MSGID(ipmb_seq, seqid),
-				       ipmb_seq, source_lun);
+	rcu_read_lock();
+	if (intf->in_shutdown) {
+		rv = -ENODEV;
+		goto out_err;
+	}
 
-			/*
-			 * Copy the message into the recv message data, so we
-			 * can retransmit it later if necessary.
-			 */
-			memcpy(recv_msg->msg_data, smi_msg->data,
-			       smi_msg->data_size);
-			recv_msg->msg.data = recv_msg->msg_data;
-			recv_msg->msg.data_len = smi_msg->data_size;
+	recv_msg->user = user;
+	if (user)
+		/* The put happens when the message is freed. */
+		kref_get(&user->refcount);
+	recv_msg->msgid = msgid;
+	/*
+	 * Store the message to send in the receive message so timeout
+	 * responses can get the proper response data.
+	 */
+	recv_msg->msg = *msg;
 
-			/*
-			 * We don't unlock until here, because we need
-			 * to copy the completed message into the
-			 * recv_msg before we release the lock.
-			 * Otherwise, race conditions may bite us.  I
-			 * know that's pretty paranoid, but I prefer
-			 * to be correct.
-			 */
-			spin_unlock_irqrestore(&(intf->seq_lock), flags);
-		}
+	if (addr->addr_type == IPMI_SYSTEM_INTERFACE_ADDR_TYPE) {
+		rv = i_ipmi_req_sysintf(intf, addr, msgid, msg, smi_msg,
+					recv_msg, retries, retry_time_ms);
+	} else if (is_ipmb_addr(addr) || is_ipmb_bcast_addr(addr)) {
+		rv = i_ipmi_req_ipmb(intf, addr, msgid, msg, smi_msg, recv_msg,
+				     source_address, source_lun,
+				     retries, retry_time_ms);
+	} else if (is_lan_addr(addr)) {
+		rv = i_ipmi_req_lan(intf, addr, msgid, msg, smi_msg, recv_msg,
+				    source_lun, retries, retry_time_ms);
 	} else {
 	    /* Unknown address type. */
 		ipmi_inc_stat(intf, sent_invalid_commands);
 		rv = -EINVAL;
-		goto out_err;
 	}
 
-#ifdef DEBUG_MSGING
-	{
-		int m;
-		for (m = 0; m < smi_msg->data_size; m++)
-			printk(" %2.2x", smi_msg->data[m]);
-		printk("\n");
-	}
-#endif
+	if (rv) {
+out_err:
+		ipmi_free_smi_msg(smi_msg);
+		ipmi_free_recv_msg(recv_msg);
+	} else {
+		ipmi_debug_msg("Send", smi_msg->data, smi_msg->data_size);
 
-	smi_send(intf, intf->handlers, smi_msg, priority);
+		smi_send(intf, intf->handlers, smi_msg, priority);
+	}
 	rcu_read_unlock();
 
-	return 0;
-
- out_err:
-	rcu_read_unlock();
-	ipmi_free_smi_msg(smi_msg);
-	ipmi_free_recv_msg(recv_msg);
+out:
 	return rv;
 }
 
-static int check_addr(ipmi_smi_t       intf,
+static int check_addr(struct ipmi_smi  *intf,
 		      struct ipmi_addr *addr,
 		      unsigned char    *saddr,
 		      unsigned char    *lun)
@@ -2046,7 +2189,7 @@ static int check_addr(ipmi_smi_t       intf,
 	return 0;
 }
 
-int ipmi_request_settime(ipmi_user_t      user,
+int ipmi_request_settime(struct ipmi_user *user,
 			 struct ipmi_addr *addr,
 			 long             msgid,
 			 struct kernel_ipmi_msg  *msg,
@@ -2056,29 +2199,36 @@ int ipmi_request_settime(ipmi_user_t      user,
 			 unsigned int     retry_time_ms)
 {
 	unsigned char saddr = 0, lun = 0;
-	int           rv;
+	int rv, index;
 
 	if (!user)
 		return -EINVAL;
+
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
+
 	rv = check_addr(user->intf, addr, &saddr, &lun);
-	if (rv)
-		return rv;
-	return i_ipmi_request(user,
-			      user->intf,
-			      addr,
-			      msgid,
-			      msg,
-			      user_msg_data,
-			      NULL, NULL,
-			      priority,
-			      saddr,
-			      lun,
-			      retries,
-			      retry_time_ms);
+	if (!rv)
+		rv = i_ipmi_request(user,
+				    user->intf,
+				    addr,
+				    msgid,
+				    msg,
+				    user_msg_data,
+				    NULL, NULL,
+				    priority,
+				    saddr,
+				    lun,
+				    retries,
+				    retry_time_ms);
+
+	release_ipmi_user(user, index);
+	return rv;
 }
 EXPORT_SYMBOL(ipmi_request_settime);
 
-int ipmi_request_supply_msgs(ipmi_user_t          user,
+int ipmi_request_supply_msgs(struct ipmi_user     *user,
 			     struct ipmi_addr     *addr,
 			     long                 msgid,
 			     struct kernel_ipmi_msg *msg,
@@ -2088,29 +2238,37 @@ int ipmi_request_supply_msgs(ipmi_user_t          user,
 			     int                  priority)
 {
 	unsigned char saddr = 0, lun = 0;
-	int           rv;
+	int rv, index;
 
 	if (!user)
 		return -EINVAL;
+
+	user = acquire_ipmi_user(user, &index);
+	if (!user)
+		return -ENODEV;
+
 	rv = check_addr(user->intf, addr, &saddr, &lun);
-	if (rv)
-		return rv;
-	return i_ipmi_request(user,
-			      user->intf,
-			      addr,
-			      msgid,
-			      msg,
-			      user_msg_data,
-			      supplied_smi,
-			      supplied_recv,
-			      priority,
-			      saddr,
-			      lun,
-			      -1, 0);
+	if (!rv)
+		rv = i_ipmi_request(user,
+				    user->intf,
+				    addr,
+				    msgid,
+				    msg,
+				    user_msg_data,
+				    supplied_smi,
+				    supplied_recv,
+				    priority,
+				    saddr,
+				    lun,
+				    -1, 0);
+
+	release_ipmi_user(user, index);
+	return rv;
 }
 EXPORT_SYMBOL(ipmi_request_supply_msgs);
 
-static void bmc_device_id_handler(ipmi_smi_t intf, struct ipmi_recv_msg *msg)
+static void bmc_device_id_handler(struct ipmi_smi *intf,
+				  struct ipmi_recv_msg *msg)
 {
 	int rv;
 
@@ -2142,7 +2300,7 @@ static void bmc_device_id_handler(ipmi_smi_t intf, struct ipmi_recv_msg *msg)
 }
 
 static int
-send_get_device_id_cmd(ipmi_smi_t intf)
+send_get_device_id_cmd(struct ipmi_smi *intf)
 {
 	struct ipmi_system_interface_addr si;
 	struct kernel_ipmi_msg msg;
@@ -2170,7 +2328,7 @@ send_get_device_id_cmd(ipmi_smi_t intf)
 			      -1, 0);
 }
 
-static int __get_device_id(ipmi_smi_t intf, struct bmc_device *bmc)
+static int __get_device_id(struct ipmi_smi *intf, struct bmc_device *bmc)
 {
 	int rv;
 
@@ -2204,7 +2362,7 @@ static int __get_device_id(ipmi_smi_t intf, struct bmc_device *bmc)
  * Except for the first time this is called (in ipmi_register_smi()),
  * this will always return good data;
  */
-static int __bmc_get_device_id(ipmi_smi_t intf, struct bmc_device *bmc,
+static int __bmc_get_device_id(struct ipmi_smi *intf, struct bmc_device *bmc,
 			       struct ipmi_device_id *id,
 			       bool *guid_set, guid_t *guid, int intf_num)
 {
@@ -2337,223 +2495,13 @@ static int __bmc_get_device_id(ipmi_smi_t intf, struct bmc_device *bmc,
 	return rv;
 }
 
-static int bmc_get_device_id(ipmi_smi_t intf, struct bmc_device *bmc,
+static int bmc_get_device_id(struct ipmi_smi *intf, struct bmc_device *bmc,
 			     struct ipmi_device_id *id,
 			     bool *guid_set, guid_t *guid)
 {
 	return __bmc_get_device_id(intf, bmc, id, guid_set, guid, -1);
 }
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-static int smi_ipmb_proc_show(struct seq_file *m, void *v)
-{
-	ipmi_smi_t intf = m->private;
-	int        i;
-
-	seq_printf(m, "%x", intf->addrinfo[0].address);
-	for (i = 1; i < IPMI_MAX_CHANNELS; i++)
-		seq_printf(m, " %x", intf->addrinfo[i].address);
-	seq_putc(m, '\n');
-
-	return 0;
-}
-
-static int smi_ipmb_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, smi_ipmb_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations smi_ipmb_proc_ops = {
-	.open		= smi_ipmb_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-static int smi_version_proc_show(struct seq_file *m, void *v)
-{
-	ipmi_smi_t intf = m->private;
-	struct ipmi_device_id id;
-	int rv;
-
-	rv = bmc_get_device_id(intf, NULL, &id, NULL, NULL);
-	if (rv)
-		return rv;
-
-	seq_printf(m, "%u.%u\n",
-		   ipmi_version_major(&id),
-		   ipmi_version_minor(&id));
-
-	return 0;
-}
-
-static int smi_version_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, smi_version_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations smi_version_proc_ops = {
-	.open		= smi_version_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-static int smi_stats_proc_show(struct seq_file *m, void *v)
-{
-	ipmi_smi_t intf = m->private;
-
-	seq_printf(m, "sent_invalid_commands:       %u\n",
-		       ipmi_get_stat(intf, sent_invalid_commands));
-	seq_printf(m, "sent_local_commands:         %u\n",
-		       ipmi_get_stat(intf, sent_local_commands));
-	seq_printf(m, "handled_local_responses:     %u\n",
-		       ipmi_get_stat(intf, handled_local_responses));
-	seq_printf(m, "unhandled_local_responses:   %u\n",
-		       ipmi_get_stat(intf, unhandled_local_responses));
-	seq_printf(m, "sent_ipmb_commands:          %u\n",
-		       ipmi_get_stat(intf, sent_ipmb_commands));
-	seq_printf(m, "sent_ipmb_command_errs:      %u\n",
-		       ipmi_get_stat(intf, sent_ipmb_command_errs));
-	seq_printf(m, "retransmitted_ipmb_commands: %u\n",
-		       ipmi_get_stat(intf, retransmitted_ipmb_commands));
-	seq_printf(m, "timed_out_ipmb_commands:     %u\n",
-		       ipmi_get_stat(intf, timed_out_ipmb_commands));
-	seq_printf(m, "timed_out_ipmb_broadcasts:   %u\n",
-		       ipmi_get_stat(intf, timed_out_ipmb_broadcasts));
-	seq_printf(m, "sent_ipmb_responses:         %u\n",
-		       ipmi_get_stat(intf, sent_ipmb_responses));
-	seq_printf(m, "handled_ipmb_responses:      %u\n",
-		       ipmi_get_stat(intf, handled_ipmb_responses));
-	seq_printf(m, "invalid_ipmb_responses:      %u\n",
-		       ipmi_get_stat(intf, invalid_ipmb_responses));
-	seq_printf(m, "unhandled_ipmb_responses:    %u\n",
-		       ipmi_get_stat(intf, unhandled_ipmb_responses));
-	seq_printf(m, "sent_lan_commands:           %u\n",
-		       ipmi_get_stat(intf, sent_lan_commands));
-	seq_printf(m, "sent_lan_command_errs:       %u\n",
-		       ipmi_get_stat(intf, sent_lan_command_errs));
-	seq_printf(m, "retransmitted_lan_commands:  %u\n",
-		       ipmi_get_stat(intf, retransmitted_lan_commands));
-	seq_printf(m, "timed_out_lan_commands:      %u\n",
-		       ipmi_get_stat(intf, timed_out_lan_commands));
-	seq_printf(m, "sent_lan_responses:          %u\n",
-		       ipmi_get_stat(intf, sent_lan_responses));
-	seq_printf(m, "handled_lan_responses:       %u\n",
-		       ipmi_get_stat(intf, handled_lan_responses));
-	seq_printf(m, "invalid_lan_responses:       %u\n",
-		       ipmi_get_stat(intf, invalid_lan_responses));
-	seq_printf(m, "unhandled_lan_responses:     %u\n",
-		       ipmi_get_stat(intf, unhandled_lan_responses));
-	seq_printf(m, "handled_commands:            %u\n",
-		       ipmi_get_stat(intf, handled_commands));
-	seq_printf(m, "invalid_commands:            %u\n",
-		       ipmi_get_stat(intf, invalid_commands));
-	seq_printf(m, "unhandled_commands:          %u\n",
-		       ipmi_get_stat(intf, unhandled_commands));
-	seq_printf(m, "invalid_events:              %u\n",
-		       ipmi_get_stat(intf, invalid_events));
-	seq_printf(m, "events:                      %u\n",
-		       ipmi_get_stat(intf, events));
-	seq_printf(m, "failed rexmit LAN msgs:      %u\n",
-		       ipmi_get_stat(intf, dropped_rexmit_lan_commands));
-	seq_printf(m, "failed rexmit IPMB msgs:     %u\n",
-		       ipmi_get_stat(intf, dropped_rexmit_ipmb_commands));
-	return 0;
-}
-
-static int smi_stats_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, smi_stats_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations smi_stats_proc_ops = {
-	.open		= smi_stats_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-int ipmi_smi_add_proc_entry(ipmi_smi_t smi, char *name,
-			    const struct file_operations *proc_ops,
-			    void *data)
-{
-	int                    rv = 0;
-	struct proc_dir_entry  *file;
-	struct ipmi_proc_entry *entry;
-
-	/* Create a list element. */
-	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
-	if (!entry)
-		return -ENOMEM;
-	entry->name = kstrdup(name, GFP_KERNEL);
-	if (!entry->name) {
-		kfree(entry);
-		return -ENOMEM;
-	}
-
-	file = proc_create_data(name, 0, smi->proc_dir, proc_ops, data);
-	if (!file) {
-		kfree(entry->name);
-		kfree(entry);
-		rv = -ENOMEM;
-	} else {
-		mutex_lock(&smi->proc_entry_lock);
-		/* Stick it on the list. */
-		entry->next = smi->proc_entries;
-		smi->proc_entries = entry;
-		mutex_unlock(&smi->proc_entry_lock);
-	}
-
-	return rv;
-}
-EXPORT_SYMBOL(ipmi_smi_add_proc_entry);
-
-static int add_proc_entries(ipmi_smi_t smi, int num)
-{
-	int rv = 0;
-
-	sprintf(smi->proc_dir_name, "%d", num);
-	smi->proc_dir = proc_mkdir(smi->proc_dir_name, proc_ipmi_root);
-	if (!smi->proc_dir)
-		rv = -ENOMEM;
-
-	if (rv == 0)
-		rv = ipmi_smi_add_proc_entry(smi, "stats",
-					     &smi_stats_proc_ops,
-					     smi);
-
-	if (rv == 0)
-		rv = ipmi_smi_add_proc_entry(smi, "ipmb",
-					     &smi_ipmb_proc_ops,
-					     smi);
-
-	if (rv == 0)
-		rv = ipmi_smi_add_proc_entry(smi, "version",
-					     &smi_version_proc_ops,
-					     smi);
-
-	return rv;
-}
-
-static void remove_proc_entries(ipmi_smi_t smi)
-{
-	struct ipmi_proc_entry *entry;
-
-	mutex_lock(&smi->proc_entry_lock);
-	while (smi->proc_entries) {
-		entry = smi->proc_entries;
-		smi->proc_entries = entry->next;
-
-		remove_proc_entry(entry->name, smi->proc_dir);
-		kfree(entry->name);
-		kfree(entry);
-	}
-	mutex_unlock(&smi->proc_entry_lock);
-	remove_proc_entry(smi->proc_dir_name, proc_ipmi_root);
-}
-#endif /* CONFIG_IPMI_PROC_INTERFACE */
-
 static ssize_t device_id_show(struct device *dev,
 			      struct device_attribute *attr,
 			      char *buf)
@@ -2885,7 +2833,7 @@ cleanup_bmc_device(struct kref *ref)
 /*
  * Must be called with intf->bmc_reg_mutex held.
  */
-static void __ipmi_bmc_unregister(ipmi_smi_t intf)
+static void __ipmi_bmc_unregister(struct ipmi_smi *intf)
 {
 	struct bmc_device *bmc = intf->bmc;
 
@@ -2905,7 +2853,7 @@ static void __ipmi_bmc_unregister(ipmi_smi_t intf)
 	intf->bmc_registered = false;
 }
 
-static void ipmi_bmc_unregister(ipmi_smi_t intf)
+static void ipmi_bmc_unregister(struct ipmi_smi *intf)
 {
 	mutex_lock(&intf->bmc_reg_mutex);
 	__ipmi_bmc_unregister(intf);
@@ -2915,7 +2863,7 @@ static void ipmi_bmc_unregister(ipmi_smi_t intf)
 /*
  * Must be called with intf->bmc_reg_mutex held.
  */
-static int __ipmi_bmc_register(ipmi_smi_t intf,
+static int __ipmi_bmc_register(struct ipmi_smi *intf,
 			       struct ipmi_device_id *id,
 			       bool guid_set, guid_t *guid, int intf_num)
 {
@@ -3077,7 +3025,7 @@ static int __ipmi_bmc_register(ipmi_smi_t intf,
 }
 
 static int
-send_guid_cmd(ipmi_smi_t intf, int chan)
+send_guid_cmd(struct ipmi_smi *intf, int chan)
 {
 	struct kernel_ipmi_msg            msg;
 	struct ipmi_system_interface_addr si;
@@ -3104,7 +3052,7 @@ send_guid_cmd(ipmi_smi_t intf, int chan)
 			      -1, 0);
 }
 
-static void guid_handler(ipmi_smi_t intf, struct ipmi_recv_msg *msg)
+static void guid_handler(struct ipmi_smi *intf, struct ipmi_recv_msg *msg)
 {
 	struct bmc_device *bmc = intf->bmc;
 
@@ -3139,7 +3087,7 @@ static void guid_handler(ipmi_smi_t intf, struct ipmi_recv_msg *msg)
 	wake_up(&intf->waitq);
 }
 
-static void __get_guid(ipmi_smi_t intf)
+static void __get_guid(struct ipmi_smi *intf)
 {
 	int rv;
 	struct bmc_device *bmc = intf->bmc;
@@ -3160,7 +3108,7 @@ static void __get_guid(ipmi_smi_t intf)
 }
 
 static int
-send_channel_info_cmd(ipmi_smi_t intf, int chan)
+send_channel_info_cmd(struct ipmi_smi *intf, int chan)
 {
 	struct kernel_ipmi_msg            msg;
 	unsigned char                     data[1];
@@ -3190,7 +3138,7 @@ send_channel_info_cmd(ipmi_smi_t intf, int chan)
 }
 
 static void
-channel_handler(ipmi_smi_t intf, struct ipmi_recv_msg *msg)
+channel_handler(struct ipmi_smi *intf, struct ipmi_recv_msg *msg)
 {
 	int rv = 0;
 	int ch;
@@ -3262,7 +3210,7 @@ channel_handler(ipmi_smi_t intf, struct ipmi_recv_msg *msg)
 /*
  * Must be holding intf->bmc_reg_mutex to call this.
  */
-static int __scan_channels(ipmi_smi_t intf, struct ipmi_device_id *id)
+static int __scan_channels(struct ipmi_smi *intf, struct ipmi_device_id *id)
 {
 	int rv;
 
@@ -3306,7 +3254,7 @@ static int __scan_channels(ipmi_smi_t intf, struct ipmi_device_id *id)
 	return 0;
 }
 
-static void ipmi_poll(ipmi_smi_t intf)
+static void ipmi_poll(struct ipmi_smi *intf)
 {
 	if (intf->handlers->poll)
 		intf->handlers->poll(intf->send_info);
@@ -3314,7 +3262,7 @@ static void ipmi_poll(ipmi_smi_t intf)
 	handle_new_recv_msgs(intf);
 }
 
-void ipmi_poll_interface(ipmi_user_t user)
+void ipmi_poll_interface(struct ipmi_user *user)
 {
 	ipmi_poll(user->intf);
 }
@@ -3322,7 +3270,8 @@ EXPORT_SYMBOL(ipmi_poll_interface);
 
 static void redo_bmc_reg(struct work_struct *work)
 {
-	ipmi_smi_t intf = container_of(work, struct ipmi_smi, bmc_reg_work);
+	struct ipmi_smi *intf = container_of(work, struct ipmi_smi,
+					     bmc_reg_work);
 
 	if (!intf->in_shutdown)
 		bmc_get_device_id(intf, NULL, NULL, NULL, NULL);
@@ -3337,8 +3286,7 @@ int ipmi_register_smi(const struct ipmi_smi_handlers *handlers,
 {
 	int              i, j;
 	int              rv;
-	ipmi_smi_t       intf;
-	ipmi_smi_t       tintf;
+	struct ipmi_smi *intf, *tintf;
 	struct list_head *link;
 	struct ipmi_device_id id;
 
@@ -3362,6 +3310,13 @@ int ipmi_register_smi(const struct ipmi_smi_handlers *handlers,
 	if (!intf)
 		return -ENOMEM;
 
+	rv = init_srcu_struct(&intf->users_srcu);
+	if (rv) {
+		kfree(intf);
+		return rv;
+	}
+
+
 	intf->bmc = &intf->tmp_bmc;
 	INIT_LIST_HEAD(&intf->bmc->intfs);
 	mutex_init(&intf->bmc->dyn_mutex);
@@ -3386,9 +3341,6 @@ int ipmi_register_smi(const struct ipmi_smi_handlers *handlers,
 		intf->seq_table[j].seqid = 0;
 	}
 	intf->curr_seq = 0;
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-	mutex_init(&intf->proc_entry_lock);
-#endif
 	spin_lock_init(&intf->waiting_rcv_msgs_lock);
 	INIT_LIST_HEAD(&intf->waiting_rcv_msgs);
 	tasklet_init(&intf->recv_tasklet,
@@ -3410,11 +3362,6 @@ int ipmi_register_smi(const struct ipmi_smi_handlers *handlers,
 	for (i = 0; i < IPMI_NUM_STATS; i++)
 		atomic_set(&intf->stats[i], 0);
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-	intf->proc_dir = NULL;
-#endif
-
-	mutex_lock(&smi_watchers_mutex);
 	mutex_lock(&ipmi_interfaces_mutex);
 	/* Look for a hole in the numbers. */
 	i = 0;
@@ -3445,25 +3392,14 @@ int ipmi_register_smi(const struct ipmi_smi_handlers *handlers,
 	mutex_lock(&intf->bmc_reg_mutex);
 	rv = __scan_channels(intf, &id);
 	mutex_unlock(&intf->bmc_reg_mutex);
-	if (rv)
-		goto out;
-
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-	rv = add_proc_entries(intf, i);
-#endif
 
  out:
 	if (rv) {
 		ipmi_bmc_unregister(intf);
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-		if (intf->proc_dir)
-			remove_proc_entries(intf);
-#endif
-		intf->handlers = NULL;
 		list_del_rcu(&intf->link);
 		mutex_unlock(&ipmi_interfaces_mutex);
-		mutex_unlock(&smi_watchers_mutex);
-		synchronize_rcu();
+		synchronize_srcu(&ipmi_interfaces_srcu);
+		cleanup_srcu_struct(&intf->users_srcu);
 		kref_put(&intf->refcount, intf_free);
 	} else {
 		/*
@@ -3474,16 +3410,16 @@ int ipmi_register_smi(const struct ipmi_smi_handlers *handlers,
 		smp_wmb();
 		intf->intf_num = i;
 		mutex_unlock(&ipmi_interfaces_mutex);
+
 		/* After this point the interface is legal to use. */
 		call_smi_watchers(i, intf->si_dev);
-		mutex_unlock(&smi_watchers_mutex);
 	}
 
 	return rv;
 }
 EXPORT_SYMBOL(ipmi_register_smi);
 
-static void deliver_smi_err_response(ipmi_smi_t intf,
+static void deliver_smi_err_response(struct ipmi_smi *intf,
 				     struct ipmi_smi_msg *msg,
 				     unsigned char err)
 {
@@ -3495,7 +3431,7 @@ static void deliver_smi_err_response(ipmi_smi_t intf,
 	handle_one_recv_msg(intf, msg);
 }
 
-static void cleanup_smi_msgs(ipmi_smi_t intf)
+static void cleanup_smi_msgs(struct ipmi_smi *intf)
 {
 	int              i;
 	struct seq_table *ent;
@@ -3528,60 +3464,58 @@ static void cleanup_smi_msgs(ipmi_smi_t intf)
 	}
 
 	for (i = 0; i < IPMI_IPMB_NUM_SEQ; i++) {
-		ent = &(intf->seq_table[i]);
+		ent = &intf->seq_table[i];
 		if (!ent->inuse)
 			continue;
-		deliver_err_response(ent->recv_msg, IPMI_ERR_UNSPECIFIED);
+		deliver_err_response(intf, ent->recv_msg, IPMI_ERR_UNSPECIFIED);
 	}
 }
 
-int ipmi_unregister_smi(ipmi_smi_t intf)
+void ipmi_unregister_smi(struct ipmi_smi *intf)
 {
 	struct ipmi_smi_watcher *w;
-	int intf_num = intf->intf_num;
-	ipmi_user_t user;
+	int intf_num = intf->intf_num, index;
 
-	mutex_lock(&smi_watchers_mutex);
 	mutex_lock(&ipmi_interfaces_mutex);
 	intf->intf_num = -1;
 	intf->in_shutdown = true;
 	list_del_rcu(&intf->link);
 	mutex_unlock(&ipmi_interfaces_mutex);
-	synchronize_rcu();
-
-	cleanup_smi_msgs(intf);
-
-	/* Clean up the effects of users on the lower-level software. */
-	mutex_lock(&ipmi_interfaces_mutex);
-	rcu_read_lock();
-	list_for_each_entry_rcu(user, &intf->users, link) {
-		module_put(intf->handlers->owner);
-		if (intf->handlers->dec_usecount)
-			intf->handlers->dec_usecount(intf->send_info);
-	}
-	rcu_read_unlock();
-	intf->handlers = NULL;
-	mutex_unlock(&ipmi_interfaces_mutex);
+	synchronize_srcu(&ipmi_interfaces_srcu);
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-	remove_proc_entries(intf);
-#endif
-	ipmi_bmc_unregister(intf);
+	/* At this point no users can be added to the interface. */
 
 	/*
 	 * Call all the watcher interfaces to tell them that
-	 * an interface is gone.
+	 * an interface is going away.
 	 */
+	mutex_lock(&smi_watchers_mutex);
 	list_for_each_entry(w, &smi_watchers, link)
 		w->smi_gone(intf_num);
 	mutex_unlock(&smi_watchers_mutex);
 
+	index = srcu_read_lock(&intf->users_srcu);
+	while (!list_empty(&intf->users)) {
+		struct ipmi_user *user =
+			container_of(list_next_rcu(&intf->users),
+				     struct ipmi_user, link);
+
+		_ipmi_destroy_user(user);
+	}
+	srcu_read_unlock(&intf->users_srcu, index);
+
+	intf->handlers->shutdown(intf->send_info);
+
+	cleanup_smi_msgs(intf);
+
+	ipmi_bmc_unregister(intf);
+
+	cleanup_srcu_struct(&intf->users_srcu);
 	kref_put(&intf->refcount, intf_free);
-	return 0;
 }
 EXPORT_SYMBOL(ipmi_unregister_smi);
 
-static int handle_ipmb_get_msg_rsp(ipmi_smi_t          intf,
+static int handle_ipmb_get_msg_rsp(struct ipmi_smi *intf,
 				   struct ipmi_smi_msg *msg)
 {
 	struct ipmi_ipmb_addr ipmb_addr;
@@ -3616,7 +3550,7 @@ static int handle_ipmb_get_msg_rsp(ipmi_smi_t          intf,
 			  msg->rsp[3] & 0x0f,
 			  msg->rsp[8],
 			  (msg->rsp[4] >> 2) & (~1),
-			  (struct ipmi_addr *) &(ipmb_addr),
+			  (struct ipmi_addr *) &ipmb_addr,
 			  &recv_msg)) {
 		/*
 		 * We were unable to find the sequence number,
@@ -3626,9 +3560,7 @@ static int handle_ipmb_get_msg_rsp(ipmi_smi_t          intf,
 		return 0;
 	}
 
-	memcpy(recv_msg->msg_data,
-	       &(msg->rsp[9]),
-	       msg->rsp_size - 9);
+	memcpy(recv_msg->msg_data, &msg->rsp[9], msg->rsp_size - 9);
 	/*
 	 * The other fields matched, so no need to set them, except
 	 * for netfn, which needs to be the response that was
@@ -3638,13 +3570,15 @@ static int handle_ipmb_get_msg_rsp(ipmi_smi_t          intf,
 	recv_msg->msg.data = recv_msg->msg_data;
 	recv_msg->msg.data_len = msg->rsp_size - 10;
 	recv_msg->recv_type = IPMI_RESPONSE_RECV_TYPE;
-	ipmi_inc_stat(intf, handled_ipmb_responses);
-	deliver_response(recv_msg);
+	if (deliver_response(intf, recv_msg))
+		ipmi_inc_stat(intf, unhandled_ipmb_responses);
+	else
+		ipmi_inc_stat(intf, handled_ipmb_responses);
 
 	return 0;
 }
 
-static int handle_ipmb_get_msg_cmd(ipmi_smi_t          intf,
+static int handle_ipmb_get_msg_cmd(struct ipmi_smi *intf,
 				   struct ipmi_smi_msg *msg)
 {
 	struct cmd_rcvr          *rcvr;
@@ -3652,7 +3586,7 @@ static int handle_ipmb_get_msg_cmd(ipmi_smi_t          intf,
 	unsigned char            netfn;
 	unsigned char            cmd;
 	unsigned char            chan;
-	ipmi_user_t              user = NULL;
+	struct ipmi_user         *user = NULL;
 	struct ipmi_ipmb_addr    *ipmb_addr;
 	struct ipmi_recv_msg     *recv_msg;
 
@@ -3689,24 +3623,17 @@ static int handle_ipmb_get_msg_cmd(ipmi_smi_t          intf,
 		msg->data[2] = msg->rsp[3];
 		msg->data[3] = msg->rsp[6];
 		msg->data[4] = ((netfn + 1) << 2) | (msg->rsp[7] & 0x3);
-		msg->data[5] = ipmb_checksum(&(msg->data[3]), 2);
+		msg->data[5] = ipmb_checksum(&msg->data[3], 2);
 		msg->data[6] = intf->addrinfo[msg->rsp[3] & 0xf].address;
 		/* rqseq/lun */
 		msg->data[7] = (msg->rsp[7] & 0xfc) | (msg->rsp[4] & 0x3);
 		msg->data[8] = msg->rsp[8]; /* cmd */
 		msg->data[9] = IPMI_INVALID_CMD_COMPLETION_CODE;
-		msg->data[10] = ipmb_checksum(&(msg->data[6]), 4);
+		msg->data[10] = ipmb_checksum(&msg->data[6], 4);
 		msg->data_size = 11;
 
-#ifdef DEBUG_MSGING
-	{
-		int m;
-		printk("Invalid command:");
-		for (m = 0; m < msg->data_size; m++)
-			printk(" %2.2x", msg->data[m]);
-		printk("\n");
-	}
-#endif
+		ipmi_debug_msg("Invalid command:", msg->data, msg->data_size);
+
 		rcu_read_lock();
 		if (!intf->in_shutdown) {
 			smi_send(intf, intf->handlers, msg, 0);
@@ -3719,9 +3646,6 @@ static int handle_ipmb_get_msg_cmd(ipmi_smi_t          intf,
 		}
 		rcu_read_unlock();
 	} else {
-		/* Deliver the message to the user. */
-		ipmi_inc_stat(intf, handled_commands);
-
 		recv_msg = ipmi_alloc_recv_msg();
 		if (!recv_msg) {
 			/*
@@ -3755,17 +3679,19 @@ static int handle_ipmb_get_msg_cmd(ipmi_smi_t          intf,
 			 * at the end also needs to be removed.
 			 */
 			recv_msg->msg.data_len = msg->rsp_size - 10;
-			memcpy(recv_msg->msg_data,
-			       &(msg->rsp[9]),
+			memcpy(recv_msg->msg_data, &msg->rsp[9],
 			       msg->rsp_size - 10);
-			deliver_response(recv_msg);
+			if (deliver_response(intf, recv_msg))
+				ipmi_inc_stat(intf, unhandled_commands);
+			else
+				ipmi_inc_stat(intf, handled_commands);
 		}
 	}
 
 	return rv;
 }
 
-static int handle_lan_get_msg_rsp(ipmi_smi_t          intf,
+static int handle_lan_get_msg_rsp(struct ipmi_smi *intf,
 				  struct ipmi_smi_msg *msg)
 {
 	struct ipmi_lan_addr  lan_addr;
@@ -3804,7 +3730,7 @@ static int handle_lan_get_msg_rsp(ipmi_smi_t          intf,
 			  msg->rsp[3] & 0x0f,
 			  msg->rsp[10],
 			  (msg->rsp[6] >> 2) & (~1),
-			  (struct ipmi_addr *) &(lan_addr),
+			  (struct ipmi_addr *) &lan_addr,
 			  &recv_msg)) {
 		/*
 		 * We were unable to find the sequence number,
@@ -3814,9 +3740,7 @@ static int handle_lan_get_msg_rsp(ipmi_smi_t          intf,
 		return 0;
 	}
 
-	memcpy(recv_msg->msg_data,
-	       &(msg->rsp[11]),
-	       msg->rsp_size - 11);
+	memcpy(recv_msg->msg_data, &msg->rsp[11], msg->rsp_size - 11);
 	/*
 	 * The other fields matched, so no need to set them, except
 	 * for netfn, which needs to be the response that was
@@ -3826,13 +3750,15 @@ static int handle_lan_get_msg_rsp(ipmi_smi_t          intf,
 	recv_msg->msg.data = recv_msg->msg_data;
 	recv_msg->msg.data_len = msg->rsp_size - 12;
 	recv_msg->recv_type = IPMI_RESPONSE_RECV_TYPE;
-	ipmi_inc_stat(intf, handled_lan_responses);
-	deliver_response(recv_msg);
+	if (deliver_response(intf, recv_msg))
+		ipmi_inc_stat(intf, unhandled_lan_responses);
+	else
+		ipmi_inc_stat(intf, handled_lan_responses);
 
 	return 0;
 }
 
-static int handle_lan_get_msg_cmd(ipmi_smi_t          intf,
+static int handle_lan_get_msg_cmd(struct ipmi_smi *intf,
 				  struct ipmi_smi_msg *msg)
 {
 	struct cmd_rcvr          *rcvr;
@@ -3840,7 +3766,7 @@ static int handle_lan_get_msg_cmd(ipmi_smi_t          intf,
 	unsigned char            netfn;
 	unsigned char            cmd;
 	unsigned char            chan;
-	ipmi_user_t              user = NULL;
+	struct ipmi_user         *user = NULL;
 	struct ipmi_lan_addr     *lan_addr;
 	struct ipmi_recv_msg     *recv_msg;
 
@@ -3878,9 +3804,6 @@ static int handle_lan_get_msg_cmd(ipmi_smi_t          intf,
 		 */
 		rv = 0;
 	} else {
-		/* Deliver the message to the user. */
-		ipmi_inc_stat(intf, handled_commands);
-
 		recv_msg = ipmi_alloc_recv_msg();
 		if (!recv_msg) {
 			/*
@@ -3916,10 +3839,12 @@ static int handle_lan_get_msg_cmd(ipmi_smi_t          intf,
 			 * at the end also needs to be removed.
 			 */
 			recv_msg->msg.data_len = msg->rsp_size - 12;
-			memcpy(recv_msg->msg_data,
-			       &(msg->rsp[11]),
+			memcpy(recv_msg->msg_data, &msg->rsp[11],
 			       msg->rsp_size - 12);
-			deliver_response(recv_msg);
+			if (deliver_response(intf, recv_msg))
+				ipmi_inc_stat(intf, unhandled_commands);
+			else
+				ipmi_inc_stat(intf, handled_commands);
 		}
 	}
 
@@ -3932,7 +3857,7 @@ static int handle_lan_get_msg_cmd(ipmi_smi_t          intf,
  * the OEM.  See IPMI 2.0 specification, Chapter 6 and
  * Chapter 22, sections 22.6 and 22.24 for more details.
  */
-static int handle_oem_get_msg_cmd(ipmi_smi_t          intf,
+static int handle_oem_get_msg_cmd(struct ipmi_smi *intf,
 				  struct ipmi_smi_msg *msg)
 {
 	struct cmd_rcvr       *rcvr;
@@ -3940,7 +3865,7 @@ static int handle_oem_get_msg_cmd(ipmi_smi_t          intf,
 	unsigned char         netfn;
 	unsigned char         cmd;
 	unsigned char         chan;
-	ipmi_user_t           user = NULL;
+	struct ipmi_user *user = NULL;
 	struct ipmi_system_interface_addr *smi_addr;
 	struct ipmi_recv_msg  *recv_msg;
 
@@ -3987,9 +3912,6 @@ static int handle_oem_get_msg_cmd(ipmi_smi_t          intf,
 
 		rv = 0;
 	} else {
-		/* Deliver the message to the user. */
-		ipmi_inc_stat(intf, handled_commands);
-
 		recv_msg = ipmi_alloc_recv_msg();
 		if (!recv_msg) {
 			/*
@@ -4007,7 +3929,7 @@ static int handle_oem_get_msg_cmd(ipmi_smi_t          intf,
 			 * requirements
 			 */
 			smi_addr = ((struct ipmi_system_interface_addr *)
-				    &(recv_msg->addr));
+				    &recv_msg->addr);
 			smi_addr->addr_type = IPMI_SYSTEM_INTERFACE_ADDR_TYPE;
 			smi_addr->channel = IPMI_BMC_CHANNEL;
 			smi_addr->lun = msg->rsp[0] & 3;
@@ -4024,10 +3946,12 @@ static int handle_oem_get_msg_cmd(ipmi_smi_t          intf,
 			 * the Channel Byte in the "GET MESSAGE" command
 			 */
 			recv_msg->msg.data_len = msg->rsp_size - 4;
-			memcpy(recv_msg->msg_data,
-			       &(msg->rsp[4]),
+			memcpy(recv_msg->msg_data, &msg->rsp[4],
 			       msg->rsp_size - 4);
-			deliver_response(recv_msg);
+			if (deliver_response(intf, recv_msg))
+				ipmi_inc_stat(intf, unhandled_commands);
+			else
+				ipmi_inc_stat(intf, handled_commands);
 		}
 	}
 
@@ -4040,26 +3964,25 @@ static void copy_event_into_recv_msg(struct ipmi_recv_msg *recv_msg,
 	struct ipmi_system_interface_addr *smi_addr;
 
 	recv_msg->msgid = 0;
-	smi_addr = (struct ipmi_system_interface_addr *) &(recv_msg->addr);
+	smi_addr = (struct ipmi_system_interface_addr *) &recv_msg->addr;
 	smi_addr->addr_type = IPMI_SYSTEM_INTERFACE_ADDR_TYPE;
 	smi_addr->channel = IPMI_BMC_CHANNEL;
 	smi_addr->lun = msg->rsp[0] & 3;
 	recv_msg->recv_type = IPMI_ASYNC_EVENT_RECV_TYPE;
 	recv_msg->msg.netfn = msg->rsp[0] >> 2;
 	recv_msg->msg.cmd = msg->rsp[1];
-	memcpy(recv_msg->msg_data, &(msg->rsp[3]), msg->rsp_size - 3);
+	memcpy(recv_msg->msg_data, &msg->rsp[3], msg->rsp_size - 3);
 	recv_msg->msg.data = recv_msg->msg_data;
 	recv_msg->msg.data_len = msg->rsp_size - 3;
 }
 
-static int handle_read_event_rsp(ipmi_smi_t          intf,
+static int handle_read_event_rsp(struct ipmi_smi *intf,
 				 struct ipmi_smi_msg *msg)
 {
 	struct ipmi_recv_msg *recv_msg, *recv_msg2;
 	struct list_head     msgs;
-	ipmi_user_t          user;
-	int                  rv = 0;
-	int                  deliver_count = 0;
+	struct ipmi_user     *user;
+	int rv = 0, deliver_count = 0, index;
 	unsigned long        flags;
 
 	if (msg->rsp_size < 19) {
@@ -4083,7 +4006,7 @@ static int handle_read_event_rsp(ipmi_smi_t          intf,
 	 * Allocate and fill in one message for every user that is
 	 * getting events.
 	 */
-	rcu_read_lock();
+	index = srcu_read_lock(&intf->users_srcu);
 	list_for_each_entry_rcu(user, &intf->users, link) {
 		if (!user->gets_events)
 			continue;
@@ -4110,15 +4033,15 @@ static int handle_read_event_rsp(ipmi_smi_t          intf,
 		copy_event_into_recv_msg(recv_msg, msg);
 		recv_msg->user = user;
 		kref_get(&user->refcount);
-		list_add_tail(&(recv_msg->link), &msgs);
+		list_add_tail(&recv_msg->link, &msgs);
 	}
-	rcu_read_unlock();
+	srcu_read_unlock(&intf->users_srcu, index);
 
 	if (deliver_count) {
 		/* Now deliver all the messages. */
 		list_for_each_entry_safe(recv_msg, recv_msg2, &msgs, link) {
 			list_del(&recv_msg->link);
-			deliver_response(recv_msg);
+			deliver_local_response(intf, recv_msg);
 		}
 	} else if (intf->waiting_events_count < MAX_EVENTS_IN_QUEUE) {
 		/*
@@ -4137,7 +4060,7 @@ static int handle_read_event_rsp(ipmi_smi_t          intf,
 		}
 
 		copy_event_into_recv_msg(recv_msg, msg);
-		list_add_tail(&(recv_msg->link), &(intf->waiting_events));
+		list_add_tail(&recv_msg->link, &intf->waiting_events);
 		intf->waiting_events_count++;
 	} else if (!intf->event_msg_printed) {
 		/*
@@ -4150,16 +4073,16 @@ static int handle_read_event_rsp(ipmi_smi_t          intf,
 	}
 
  out:
-	spin_unlock_irqrestore(&(intf->events_lock), flags);
+	spin_unlock_irqrestore(&intf->events_lock, flags);
 
 	return rv;
 }
 
-static int handle_bmc_rsp(ipmi_smi_t          intf,
+static int handle_bmc_rsp(struct ipmi_smi *intf,
 			  struct ipmi_smi_msg *msg)
 {
 	struct ipmi_recv_msg *recv_msg;
-	struct ipmi_user     *user;
+	struct ipmi_system_interface_addr *smi_addr;
 
 	recv_msg = (struct ipmi_recv_msg *) msg->user_data;
 	if (recv_msg == NULL) {
@@ -4168,32 +4091,19 @@ static int handle_bmc_rsp(ipmi_smi_t          intf,
 		return 0;
 	}
 
-	user = recv_msg->user;
-	/* Make sure the user still exists. */
-	if (user && !user->valid) {
-		/* The user for the message went away, so give up. */
-		ipmi_inc_stat(intf, unhandled_local_responses);
-		ipmi_free_recv_msg(recv_msg);
-	} else {
-		struct ipmi_system_interface_addr *smi_addr;
-
-		ipmi_inc_stat(intf, handled_local_responses);
-		recv_msg->recv_type = IPMI_RESPONSE_RECV_TYPE;
-		recv_msg->msgid = msg->msgid;
-		smi_addr = ((struct ipmi_system_interface_addr *)
-			    &(recv_msg->addr));
-		smi_addr->addr_type = IPMI_SYSTEM_INTERFACE_ADDR_TYPE;
-		smi_addr->channel = IPMI_BMC_CHANNEL;
-		smi_addr->lun = msg->rsp[0] & 3;
-		recv_msg->msg.netfn = msg->rsp[0] >> 2;
-		recv_msg->msg.cmd = msg->rsp[1];
-		memcpy(recv_msg->msg_data,
-		       &(msg->rsp[2]),
-		       msg->rsp_size - 2);
-		recv_msg->msg.data = recv_msg->msg_data;
-		recv_msg->msg.data_len = msg->rsp_size - 2;
-		deliver_response(recv_msg);
-	}
+	recv_msg->recv_type = IPMI_RESPONSE_RECV_TYPE;
+	recv_msg->msgid = msg->msgid;
+	smi_addr = ((struct ipmi_system_interface_addr *)
+		    &recv_msg->addr);
+	smi_addr->addr_type = IPMI_SYSTEM_INTERFACE_ADDR_TYPE;
+	smi_addr->channel = IPMI_BMC_CHANNEL;
+	smi_addr->lun = msg->rsp[0] & 3;
+	recv_msg->msg.netfn = msg->rsp[0] >> 2;
+	recv_msg->msg.cmd = msg->rsp[1];
+	memcpy(recv_msg->msg_data, &msg->rsp[2], msg->rsp_size - 2);
+	recv_msg->msg.data = recv_msg->msg_data;
+	recv_msg->msg.data_len = msg->rsp_size - 2;
+	deliver_local_response(intf, recv_msg);
 
 	return 0;
 }
@@ -4203,19 +4113,13 @@ static int handle_bmc_rsp(ipmi_smi_t          intf,
  * 0 if the message should be freed, or -1 if the message should not
  * be freed or requeued.
  */
-static int handle_one_recv_msg(ipmi_smi_t          intf,
+static int handle_one_recv_msg(struct ipmi_smi *intf,
 			       struct ipmi_smi_msg *msg)
 {
 	int requeue;
 	int chan;
 
-#ifdef DEBUG_MSGING
-	int m;
-	printk("Recv:");
-	for (m = 0; m < msg->rsp_size; m++)
-		printk(" %2.2x", msg->rsp[m]);
-	printk("\n");
-#endif
+	ipmi_debug_msg("Recv:", msg->rsp, msg->rsp_size);
 	if (msg->rsp_size < 2) {
 		/* Message is too small to be correct. */
 		dev_warn(intf->si_dev,
@@ -4252,7 +4156,7 @@ static int handle_one_recv_msg(ipmi_smi_t          intf,
 		 * It's a response to a response we sent.  For this we
 		 * deliver a send message response to the user.
 		 */
-		struct ipmi_recv_msg     *recv_msg = msg->user_data;
+		struct ipmi_recv_msg *recv_msg = msg->user_data;
 
 		requeue = 0;
 		if (msg->rsp_size < 2)
@@ -4267,15 +4171,11 @@ static int handle_one_recv_msg(ipmi_smi_t          intf,
 		if (!recv_msg)
 			goto out;
 
-		/* Make sure the user still exists. */
-		if (!recv_msg->user || !recv_msg->user->valid)
-			goto out;
-
 		recv_msg->recv_type = IPMI_RESPONSE_RESPONSE_TYPE;
 		recv_msg->msg.data = recv_msg->msg_data;
 		recv_msg->msg.data_len = 1;
 		recv_msg->msg_data[0] = msg->rsp[2];
-		deliver_response(recv_msg);
+		deliver_local_response(intf, recv_msg);
 	} else if ((msg->rsp[0] == ((IPMI_NETFN_APP_REQUEST|1) << 2))
 		   && (msg->rsp[1] == IPMI_GET_MSG_CMD)) {
 		struct ipmi_channel   *chans;
@@ -4367,7 +4267,7 @@ static int handle_one_recv_msg(ipmi_smi_t          intf,
 /*
  * If there are messages in the queue or pretimeouts, handle them.
  */
-static void handle_new_recv_msgs(ipmi_smi_t intf)
+static void handle_new_recv_msgs(struct ipmi_smi *intf)
 {
 	struct ipmi_smi_msg  *smi_msg;
 	unsigned long        flags = 0;
@@ -4412,22 +4312,23 @@ static void handle_new_recv_msgs(ipmi_smi_t intf)
 	 * deliver pretimeouts to all the users.
 	 */
 	if (atomic_add_unless(&intf->watchdog_pretimeouts_to_deliver, -1, 0)) {
-		ipmi_user_t user;
+		struct ipmi_user *user;
+		int index;
 
-		rcu_read_lock();
+		index = srcu_read_lock(&intf->users_srcu);
 		list_for_each_entry_rcu(user, &intf->users, link) {
 			if (user->handler->ipmi_watchdog_pretimeout)
 				user->handler->ipmi_watchdog_pretimeout(
 					user->handler_data);
 		}
-		rcu_read_unlock();
+		srcu_read_unlock(&intf->users_srcu, index);
 	}
 }
 
 static void smi_recv_tasklet(unsigned long val)
 {
 	unsigned long flags = 0; /* keep us warning-free. */
-	ipmi_smi_t intf = (ipmi_smi_t) val;
+	struct ipmi_smi *intf = (struct ipmi_smi *) val;
 	int run_to_completion = intf->run_to_completion;
 	struct ipmi_smi_msg *newmsg = NULL;
 
@@ -4469,7 +4370,7 @@ static void smi_recv_tasklet(unsigned long val)
 }
 
 /* Handle a new message from the lower layer. */
-void ipmi_smi_msg_received(ipmi_smi_t          intf,
+void ipmi_smi_msg_received(struct ipmi_smi *intf,
 			   struct ipmi_smi_msg *msg)
 {
 	unsigned long flags = 0; /* keep us warning-free. */
@@ -4550,7 +4451,7 @@ void ipmi_smi_msg_received(ipmi_smi_t          intf,
 }
 EXPORT_SYMBOL(ipmi_smi_msg_received);
 
-void ipmi_smi_watchdog_pretimeout(ipmi_smi_t intf)
+void ipmi_smi_watchdog_pretimeout(struct ipmi_smi *intf)
 {
 	if (intf->in_shutdown)
 		return;
@@ -4561,7 +4462,7 @@ void ipmi_smi_watchdog_pretimeout(ipmi_smi_t intf)
 EXPORT_SYMBOL(ipmi_smi_watchdog_pretimeout);
 
 static struct ipmi_smi_msg *
-smi_from_recv_msg(ipmi_smi_t intf, struct ipmi_recv_msg *recv_msg,
+smi_from_recv_msg(struct ipmi_smi *intf, struct ipmi_recv_msg *recv_msg,
 		  unsigned char seq, long seqid)
 {
 	struct ipmi_smi_msg *smi_msg = ipmi_alloc_smi_msg();
@@ -4576,26 +4477,18 @@ smi_from_recv_msg(ipmi_smi_t intf, struct ipmi_recv_msg *recv_msg,
 	smi_msg->data_size = recv_msg->msg.data_len;
 	smi_msg->msgid = STORE_SEQ_IN_MSGID(seq, seqid);
 
-#ifdef DEBUG_MSGING
-	{
-		int m;
-		printk("Resend: ");
-		for (m = 0; m < smi_msg->data_size; m++)
-			printk(" %2.2x", smi_msg->data[m]);
-		printk("\n");
-	}
-#endif
+	ipmi_debug_msg("Resend: ", smi_msg->data, smi_msg->data_size);
+
 	return smi_msg;
 }
 
-static void check_msg_timeout(ipmi_smi_t intf, struct seq_table *ent,
+static void check_msg_timeout(struct ipmi_smi *intf, struct seq_table *ent,
 			      struct list_head *timeouts,
 			      unsigned long timeout_period,
 			      int slot, unsigned long *flags,
 			      unsigned int *waiting_msgs)
 {
-	struct ipmi_recv_msg     *msg;
-	const struct ipmi_smi_handlers *handlers;
+	struct ipmi_recv_msg *msg;
 
 	if (intf->in_shutdown)
 		return;
@@ -4653,8 +4546,7 @@ static void check_msg_timeout(ipmi_smi_t intf, struct seq_table *ent,
 		 * only for messages to the local MC, which don't get
 		 * resent.
 		 */
-		handlers = intf->handlers;
-		if (handlers) {
+		if (intf->handlers) {
 			if (is_lan_addr(&ent->recv_msg->addr))
 				ipmi_inc_stat(intf,
 					      retransmitted_lan_commands);
@@ -4662,7 +4554,7 @@ static void check_msg_timeout(ipmi_smi_t intf, struct seq_table *ent,
 				ipmi_inc_stat(intf,
 					      retransmitted_ipmb_commands);
 
-			smi_send(intf, handlers, smi_msg, 0);
+			smi_send(intf, intf->handlers, smi_msg, 0);
 		} else
 			ipmi_free_smi_msg(smi_msg);
 
@@ -4670,7 +4562,7 @@ static void check_msg_timeout(ipmi_smi_t intf, struct seq_table *ent,
 	}
 }
 
-static unsigned int ipmi_timeout_handler(ipmi_smi_t intf,
+static unsigned int ipmi_timeout_handler(struct ipmi_smi *intf,
 					 unsigned long timeout_period)
 {
 	struct list_head     timeouts;
@@ -4694,14 +4586,20 @@ static unsigned int ipmi_timeout_handler(ipmi_smi_t intf,
 	 */
 	INIT_LIST_HEAD(&timeouts);
 	spin_lock_irqsave(&intf->seq_lock, flags);
+	if (intf->ipmb_maintenance_mode_timeout) {
+		if (intf->ipmb_maintenance_mode_timeout <= timeout_period)
+			intf->ipmb_maintenance_mode_timeout = 0;
+		else
+			intf->ipmb_maintenance_mode_timeout -= timeout_period;
+	}
 	for (i = 0; i < IPMI_IPMB_NUM_SEQ; i++)
-		check_msg_timeout(intf, &(intf->seq_table[i]),
+		check_msg_timeout(intf, &intf->seq_table[i],
 				  &timeouts, timeout_period, i,
 				  &flags, &waiting_msgs);
 	spin_unlock_irqrestore(&intf->seq_lock, flags);
 
 	list_for_each_entry_safe(msg, msg2, &timeouts, link)
-		deliver_err_response(msg, IPMI_TIMEOUT_COMPLETION_CODE);
+		deliver_err_response(intf, msg, IPMI_TIMEOUT_COMPLETION_CODE);
 
 	/*
 	 * Maintenance mode handling.  Check the timeout
@@ -4731,7 +4629,7 @@ static unsigned int ipmi_timeout_handler(ipmi_smi_t intf,
 	return waiting_msgs;
 }
 
-static void ipmi_request_event(ipmi_smi_t intf)
+static void ipmi_request_event(struct ipmi_smi *intf)
 {
 	/* No event requests when in maintenance mode. */
 	if (intf->maintenance_mode_enable)
@@ -4747,13 +4645,13 @@ static atomic_t stop_operation;
 
 static void ipmi_timeout(struct timer_list *unused)
 {
-	ipmi_smi_t intf;
-	int nt = 0;
+	struct ipmi_smi *intf;
+	int nt = 0, index;
 
 	if (atomic_read(&stop_operation))
 		return;
 
-	rcu_read_lock();
+	index = srcu_read_lock(&ipmi_interfaces_srcu);
 	list_for_each_entry_rcu(intf, &ipmi_interfaces, link) {
 		int lnt = 0;
 
@@ -4776,13 +4674,13 @@ static void ipmi_timeout(struct timer_list *unused)
 
 		nt += lnt;
 	}
-	rcu_read_unlock();
+	srcu_read_unlock(&ipmi_interfaces_srcu, index);
 
 	if (nt)
 		mod_timer(&ipmi_timer, jiffies + IPMI_TIMEOUT_JIFFIES);
 }
 
-static void need_waiter(ipmi_smi_t intf)
+static void need_waiter(struct ipmi_smi *intf)
 {
 	/* Racy, but worst case we start the timer twice. */
 	if (!timer_pending(&ipmi_timer))
@@ -4853,8 +4751,8 @@ static void dummy_recv_done_handler(struct ipmi_recv_msg *msg)
 /*
  * Inside a panic, send a message and wait for a response.
  */
-static void ipmi_panic_request_and_wait(ipmi_smi_t           intf,
-					struct ipmi_addr     *addr,
+static void ipmi_panic_request_and_wait(struct ipmi_smi *intf,
+					struct ipmi_addr *addr,
 					struct kernel_ipmi_msg *msg)
 {
 	struct ipmi_smi_msg  smi_msg;
@@ -4885,7 +4783,8 @@ static void ipmi_panic_request_and_wait(ipmi_smi_t           intf,
 		ipmi_poll(intf);
 }
 
-static void event_receiver_fetcher(ipmi_smi_t intf, struct ipmi_recv_msg *msg)
+static void event_receiver_fetcher(struct ipmi_smi *intf,
+				   struct ipmi_recv_msg *msg)
 {
 	if ((msg->addr.addr_type == IPMI_SYSTEM_INTERFACE_ADDR_TYPE)
 	    && (msg->msg.netfn == IPMI_NETFN_SENSOR_EVENT_RESPONSE)
@@ -4897,7 +4796,7 @@ static void event_receiver_fetcher(ipmi_smi_t intf, struct ipmi_recv_msg *msg)
 	}
 }
 
-static void device_id_fetcher(ipmi_smi_t intf, struct ipmi_recv_msg *msg)
+static void device_id_fetcher(struct ipmi_smi *intf, struct ipmi_recv_msg *msg)
 {
 	if ((msg->addr.addr_type == IPMI_SYSTEM_INTERFACE_ADDR_TYPE)
 	    && (msg->msg.netfn == IPMI_NETFN_APP_RESPONSE)
@@ -4912,13 +4811,15 @@ static void device_id_fetcher(ipmi_smi_t intf, struct ipmi_recv_msg *msg)
 	}
 }
 
-static void send_panic_events(char *str)
+static void send_panic_events(struct ipmi_smi *intf, char *str)
 {
-	struct kernel_ipmi_msg            msg;
-	ipmi_smi_t                        intf;
-	unsigned char                     data[16];
+	struct kernel_ipmi_msg msg;
+	unsigned char data[16];
 	struct ipmi_system_interface_addr *si;
-	struct ipmi_addr                  addr;
+	struct ipmi_addr addr;
+	char *p = str;
+	struct ipmi_ipmb_addr *ipmb;
+	int j;
 
 	if (ipmi_send_panic_event == IPMI_SEND_PANIC_EVENT_NONE)
 		return;
@@ -4949,15 +4850,8 @@ static void send_panic_events(char *str)
 		data[7] = str[2];
 	}
 
-	/* For every registered interface, send the event. */
-	list_for_each_entry_rcu(intf, &ipmi_interfaces, link) {
-		if (!intf->handlers || !intf->handlers->poll)
-			/* Interface is not ready or can't run at panic time. */
-			continue;
-
-		/* Send the event announcing the panic. */
-		ipmi_panic_request_and_wait(intf, &addr, &msg);
-	}
+	/* Send the event announcing the panic. */
+	ipmi_panic_request_and_wait(intf, &addr, &msg);
 
 	/*
 	 * On every interface, dump a bunch of OEM event holding the
@@ -4966,111 +4860,100 @@ static void send_panic_events(char *str)
 	if (ipmi_send_panic_event != IPMI_SEND_PANIC_EVENT_STRING || !str)
 		return;
 
-	/* For every registered interface, send the event. */
-	list_for_each_entry_rcu(intf, &ipmi_interfaces, link) {
-		char                  *p = str;
-		struct ipmi_ipmb_addr *ipmb;
-		int                   j;
-
-		if (intf->intf_num == -1)
-			/* Interface was not ready yet. */
-			continue;
+	/*
+	 * intf_num is used as an marker to tell if the
+	 * interface is valid.  Thus we need a read barrier to
+	 * make sure data fetched before checking intf_num
+	 * won't be used.
+	 */
+	smp_rmb();
 
-		/*
-		 * intf_num is used as an marker to tell if the
-		 * interface is valid.  Thus we need a read barrier to
-		 * make sure data fetched before checking intf_num
-		 * won't be used.
-		 */
-		smp_rmb();
+	/*
+	 * First job here is to figure out where to send the
+	 * OEM events.  There's no way in IPMI to send OEM
+	 * events using an event send command, so we have to
+	 * find the SEL to put them in and stick them in
+	 * there.
+	 */
 
-		/*
-		 * First job here is to figure out where to send the
-		 * OEM events.  There's no way in IPMI to send OEM
-		 * events using an event send command, so we have to
-		 * find the SEL to put them in and stick them in
-		 * there.
-		 */
+	/* Get capabilities from the get device id. */
+	intf->local_sel_device = 0;
+	intf->local_event_generator = 0;
+	intf->event_receiver = 0;
 
-		/* Get capabilities from the get device id. */
-		intf->local_sel_device = 0;
-		intf->local_event_generator = 0;
-		intf->event_receiver = 0;
+	/* Request the device info from the local MC. */
+	msg.netfn = IPMI_NETFN_APP_REQUEST;
+	msg.cmd = IPMI_GET_DEVICE_ID_CMD;
+	msg.data = NULL;
+	msg.data_len = 0;
+	intf->null_user_handler = device_id_fetcher;
+	ipmi_panic_request_and_wait(intf, &addr, &msg);
 
-		/* Request the device info from the local MC. */
-		msg.netfn = IPMI_NETFN_APP_REQUEST;
-		msg.cmd = IPMI_GET_DEVICE_ID_CMD;
+	if (intf->local_event_generator) {
+		/* Request the event receiver from the local MC. */
+		msg.netfn = IPMI_NETFN_SENSOR_EVENT_REQUEST;
+		msg.cmd = IPMI_GET_EVENT_RECEIVER_CMD;
 		msg.data = NULL;
 		msg.data_len = 0;
-		intf->null_user_handler = device_id_fetcher;
+		intf->null_user_handler = event_receiver_fetcher;
 		ipmi_panic_request_and_wait(intf, &addr, &msg);
+	}
+	intf->null_user_handler = NULL;
 
-		if (intf->local_event_generator) {
-			/* Request the event receiver from the local MC. */
-			msg.netfn = IPMI_NETFN_SENSOR_EVENT_REQUEST;
-			msg.cmd = IPMI_GET_EVENT_RECEIVER_CMD;
-			msg.data = NULL;
-			msg.data_len = 0;
-			intf->null_user_handler = event_receiver_fetcher;
-			ipmi_panic_request_and_wait(intf, &addr, &msg);
-		}
-		intf->null_user_handler = NULL;
+	/*
+	 * Validate the event receiver.  The low bit must not
+	 * be 1 (it must be a valid IPMB address), it cannot
+	 * be zero, and it must not be my address.
+	 */
+	if (((intf->event_receiver & 1) == 0)
+	    && (intf->event_receiver != 0)
+	    && (intf->event_receiver != intf->addrinfo[0].address)) {
+		/*
+		 * The event receiver is valid, send an IPMB
+		 * message.
+		 */
+		ipmb = (struct ipmi_ipmb_addr *) &addr;
+		ipmb->addr_type = IPMI_IPMB_ADDR_TYPE;
+		ipmb->channel = 0; /* FIXME - is this right? */
+		ipmb->lun = intf->event_receiver_lun;
+		ipmb->slave_addr = intf->event_receiver;
+	} else if (intf->local_sel_device) {
+		/*
+		 * The event receiver was not valid (or was
+		 * me), but I am an SEL device, just dump it
+		 * in my SEL.
+		 */
+		si = (struct ipmi_system_interface_addr *) &addr;
+		si->addr_type = IPMI_SYSTEM_INTERFACE_ADDR_TYPE;
+		si->channel = IPMI_BMC_CHANNEL;
+		si->lun = 0;
+	} else
+		return; /* No where to send the event. */
 
+	msg.netfn = IPMI_NETFN_STORAGE_REQUEST; /* Storage. */
+	msg.cmd = IPMI_ADD_SEL_ENTRY_CMD;
+	msg.data = data;
+	msg.data_len = 16;
+
+	j = 0;
+	while (*p) {
+		int size = strlen(p);
+
+		if (size > 11)
+			size = 11;
+		data[0] = 0;
+		data[1] = 0;
+		data[2] = 0xf0; /* OEM event without timestamp. */
+		data[3] = intf->addrinfo[0].address;
+		data[4] = j++; /* sequence # */
 		/*
-		 * Validate the event receiver.  The low bit must not
-		 * be 1 (it must be a valid IPMB address), it cannot
-		 * be zero, and it must not be my address.
+		 * Always give 11 bytes, so strncpy will fill
+		 * it with zeroes for me.
 		 */
-		if (((intf->event_receiver & 1) == 0)
-		    && (intf->event_receiver != 0)
-		    && (intf->event_receiver != intf->addrinfo[0].address)) {
-			/*
-			 * The event receiver is valid, send an IPMB
-			 * message.
-			 */
-			ipmb = (struct ipmi_ipmb_addr *) &addr;
-			ipmb->addr_type = IPMI_IPMB_ADDR_TYPE;
-			ipmb->channel = 0; /* FIXME - is this right? */
-			ipmb->lun = intf->event_receiver_lun;
-			ipmb->slave_addr = intf->event_receiver;
-		} else if (intf->local_sel_device) {
-			/*
-			 * The event receiver was not valid (or was
-			 * me), but I am an SEL device, just dump it
-			 * in my SEL.
-			 */
-			si = (struct ipmi_system_interface_addr *) &addr;
-			si->addr_type = IPMI_SYSTEM_INTERFACE_ADDR_TYPE;
-			si->channel = IPMI_BMC_CHANNEL;
-			si->lun = 0;
-		} else
-			continue; /* No where to send the event. */
-
-		msg.netfn = IPMI_NETFN_STORAGE_REQUEST; /* Storage. */
-		msg.cmd = IPMI_ADD_SEL_ENTRY_CMD;
-		msg.data = data;
-		msg.data_len = 16;
-
-		j = 0;
-		while (*p) {
-			int size = strlen(p);
-
-			if (size > 11)
-				size = 11;
-			data[0] = 0;
-			data[1] = 0;
-			data[2] = 0xf0; /* OEM event without timestamp. */
-			data[3] = intf->addrinfo[0].address;
-			data[4] = j++; /* sequence # */
-			/*
-			 * Always give 11 bytes, so strncpy will fill
-			 * it with zeroes for me.
-			 */
-			strncpy(data+5, p, 11);
-			p += size;
+		strncpy(data+5, p, 11);
+		p += size;
 
-			ipmi_panic_request_and_wait(intf, &addr, &msg);
-		}
+		ipmi_panic_request_and_wait(intf, &addr, &msg);
 	}
 }
 
@@ -5080,7 +4963,8 @@ static int panic_event(struct notifier_block *this,
 		       unsigned long         event,
 		       void                  *ptr)
 {
-	ipmi_smi_t intf;
+	struct ipmi_smi *intf;
+	struct ipmi_user *user;
 
 	if (has_panicked)
 		return NOTIFY_DONE;
@@ -5088,10 +4972,13 @@ static int panic_event(struct notifier_block *this,
 
 	/* For every registered interface, set it to run to completion. */
 	list_for_each_entry_rcu(intf, &ipmi_interfaces, link) {
-		if (!intf->handlers)
+		if (!intf->handlers || intf->intf_num == -1)
 			/* Interface is not ready. */
 			continue;
 
+		if (!intf->handlers->poll)
+			continue;
+
 		/*
 		 * If we were interrupted while locking xmit_msgs_lock or
 		 * waiting_rcv_msgs_lock, the corresponding list may be
@@ -5113,9 +5000,15 @@ static int panic_event(struct notifier_block *this,
 		if (intf->handlers->set_run_to_completion)
 			intf->handlers->set_run_to_completion(intf->send_info,
 							      1);
-	}
 
-	send_panic_events(ptr);
+		list_for_each_entry_rcu(user, &intf->users, link) {
+			if (user->handler->ipmi_panic_handler)
+				user->handler->ipmi_panic_handler(
+					user->handler_data);
+		}
+
+		send_panic_events(intf, ptr);
+	}
 
 	return NOTIFY_DONE;
 }
@@ -5141,16 +5034,6 @@ static int ipmi_init_msghandler(void)
 
 	pr_info("ipmi message handler version " IPMI_DRIVER_VERSION "\n");
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-	proc_ipmi_root = proc_mkdir("ipmi", NULL);
-	if (!proc_ipmi_root) {
-	    pr_err(PFX "Unable to create IPMI proc dir");
-	    driver_unregister(&ipmidriver.driver);
-	    return -ENOMEM;
-	}
-
-#endif /* CONFIG_IPMI_PROC_INTERFACE */
-
 	timer_setup(&ipmi_timer, ipmi_timeout, 0);
 	mod_timer(&ipmi_timer, jiffies + IPMI_TIMEOUT_JIFFIES);
 
@@ -5189,10 +5072,6 @@ static void __exit cleanup_ipmi(void)
 	atomic_inc(&stop_operation);
 	del_timer_sync(&ipmi_timer);
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-	proc_remove(proc_ipmi_root);
-#endif /* CONFIG_IPMI_PROC_INTERFACE */
-
 	driver_unregister(&ipmidriver.driver);
 
 	initialized = 0;
diff --git a/drivers/char/ipmi/ipmi_poweroff.c b/drivers/char/ipmi/ipmi_poweroff.c
index 7996337852f2..f6e19410dc57 100644
--- a/drivers/char/ipmi/ipmi_poweroff.c
+++ b/drivers/char/ipmi/ipmi_poweroff.c
@@ -39,9 +39,9 @@ static int ifnum_to_use = -1;
 
 /* Our local state. */
 static int ready;
-static ipmi_user_t ipmi_user;
+static struct ipmi_user *ipmi_user;
 static int ipmi_ifnum;
-static void (*specific_poweroff_func)(ipmi_user_t user);
+static void (*specific_poweroff_func)(struct ipmi_user *user);
 
 /* Holds the old poweroff function so we can restore it on removal. */
 static void (*old_poweroff_func)(void);
@@ -118,7 +118,7 @@ static const struct ipmi_user_hndl ipmi_poweroff_handler = {
 };
 
 
-static int ipmi_request_wait_for_response(ipmi_user_t            user,
+static int ipmi_request_wait_for_response(struct ipmi_user       *user,
 					  struct ipmi_addr       *addr,
 					  struct kernel_ipmi_msg *send_msg)
 {
@@ -138,7 +138,7 @@ static int ipmi_request_wait_for_response(ipmi_user_t            user,
 }
 
 /* Wait for message to complete, spinning. */
-static int ipmi_request_in_rc_mode(ipmi_user_t            user,
+static int ipmi_request_in_rc_mode(struct ipmi_user       *user,
 				   struct ipmi_addr       *addr,
 				   struct kernel_ipmi_msg *send_msg)
 {
@@ -178,9 +178,9 @@ static int ipmi_request_in_rc_mode(ipmi_user_t            user,
 #define IPMI_MOTOROLA_MANUFACTURER_ID		0x0000A1
 #define IPMI_MOTOROLA_PPS_IPMC_PRODUCT_ID	0x0051
 
-static void (*atca_oem_poweroff_hook)(ipmi_user_t user);
+static void (*atca_oem_poweroff_hook)(struct ipmi_user *user);
 
-static void pps_poweroff_atca(ipmi_user_t user)
+static void pps_poweroff_atca(struct ipmi_user *user)
 {
 	struct ipmi_system_interface_addr smi_addr;
 	struct kernel_ipmi_msg            send_msg;
@@ -208,7 +208,7 @@ static void pps_poweroff_atca(ipmi_user_t user)
 	return;
 }
 
-static int ipmi_atca_detect(ipmi_user_t user)
+static int ipmi_atca_detect(struct ipmi_user *user)
 {
 	struct ipmi_system_interface_addr smi_addr;
 	struct kernel_ipmi_msg            send_msg;
@@ -245,7 +245,7 @@ static int ipmi_atca_detect(ipmi_user_t user)
 	return !rv;
 }
 
-static void ipmi_poweroff_atca(ipmi_user_t user)
+static void ipmi_poweroff_atca(struct ipmi_user *user)
 {
 	struct ipmi_system_interface_addr smi_addr;
 	struct kernel_ipmi_msg            send_msg;
@@ -309,13 +309,13 @@ static void ipmi_poweroff_atca(ipmi_user_t user)
 #define IPMI_CPI1_PRODUCT_ID		0x000157
 #define IPMI_CPI1_MANUFACTURER_ID	0x0108
 
-static int ipmi_cpi1_detect(ipmi_user_t user)
+static int ipmi_cpi1_detect(struct ipmi_user *user)
 {
 	return ((mfg_id == IPMI_CPI1_MANUFACTURER_ID)
 		&& (prod_id == IPMI_CPI1_PRODUCT_ID));
 }
 
-static void ipmi_poweroff_cpi1(ipmi_user_t user)
+static void ipmi_poweroff_cpi1(struct ipmi_user *user)
 {
 	struct ipmi_system_interface_addr smi_addr;
 	struct ipmi_ipmb_addr             ipmb_addr;
@@ -424,7 +424,7 @@ static void ipmi_poweroff_cpi1(ipmi_user_t user)
  */
 
 #define DELL_IANA_MFR_ID {0xA2, 0x02, 0x00}
-static int ipmi_dell_chassis_detect(ipmi_user_t user)
+static int ipmi_dell_chassis_detect(struct ipmi_user *user)
 {
 	const char ipmi_version_major = ipmi_version & 0xF;
 	const char ipmi_version_minor = (ipmi_version >> 4) & 0xF;
@@ -445,7 +445,7 @@ static int ipmi_dell_chassis_detect(ipmi_user_t user)
 
 #define HP_IANA_MFR_ID 0x0b
 #define HP_BMC_PROD_ID 0x8201
-static int ipmi_hp_chassis_detect(ipmi_user_t user)
+static int ipmi_hp_chassis_detect(struct ipmi_user *user)
 {
 	if (mfg_id == HP_IANA_MFR_ID
 		&& prod_id == HP_BMC_PROD_ID
@@ -461,13 +461,13 @@ static int ipmi_hp_chassis_detect(ipmi_user_t user)
 #define IPMI_NETFN_CHASSIS_REQUEST	0
 #define IPMI_CHASSIS_CONTROL_CMD	0x02
 
-static int ipmi_chassis_detect(ipmi_user_t user)
+static int ipmi_chassis_detect(struct ipmi_user *user)
 {
 	/* Chassis support, use it. */
 	return (capabilities & 0x80);
 }
 
-static void ipmi_poweroff_chassis(ipmi_user_t user)
+static void ipmi_poweroff_chassis(struct ipmi_user *user)
 {
 	struct ipmi_system_interface_addr smi_addr;
 	struct kernel_ipmi_msg            send_msg;
@@ -517,8 +517,8 @@ static void ipmi_poweroff_chassis(ipmi_user_t user)
 /* Table of possible power off functions. */
 struct poweroff_function {
 	char *platform_type;
-	int  (*detect)(ipmi_user_t user);
-	void (*poweroff_func)(ipmi_user_t user);
+	int  (*detect)(struct ipmi_user *user);
+	void (*poweroff_func)(struct ipmi_user *user);
 };
 
 static struct poweroff_function poweroff_functions[] = {
diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c
index ff870aa91cfe..ad353be871bf 100644
--- a/drivers/char/ipmi/ipmi_si_intf.c
+++ b/drivers/char/ipmi/ipmi_si_intf.c
@@ -122,8 +122,8 @@ enum si_stat_indexes {
 };
 
 struct smi_info {
-	int                    intf_num;
-	ipmi_smi_t             intf;
+	int                    si_num;
+	struct ipmi_smi        *intf;
 	struct si_sm_data      *si_sm;
 	const struct si_sm_handlers *handlers;
 	spinlock_t             si_lock;
@@ -261,7 +261,6 @@ static int num_max_busy_us;
 static bool unload_when_empty = true;
 
 static int try_smi_init(struct smi_info *smi);
-static void shutdown_one_si(struct smi_info *smi_info);
 static void cleanup_one_si(struct smi_info *smi_info);
 static void cleanup_ipmi_si(void);
 
@@ -287,10 +286,7 @@ static void deliver_recv_msg(struct smi_info *smi_info,
 			     struct ipmi_smi_msg *msg)
 {
 	/* Deliver the message to the upper layer. */
-	if (smi_info->intf)
-		ipmi_smi_msg_received(smi_info->intf, msg);
-	else
-		ipmi_free_smi_msg(msg);
+	ipmi_smi_msg_received(smi_info->intf, msg);
 }
 
 static void return_hosed_msg(struct smi_info *smi_info, int cCode)
@@ -471,8 +467,7 @@ static void handle_flags(struct smi_info *smi_info)
 
 		start_clear_flags(smi_info);
 		smi_info->msg_flags &= ~WDT_PRE_TIMEOUT_INT;
-		if (smi_info->intf)
-			ipmi_smi_watchdog_pretimeout(smi_info->intf);
+		ipmi_smi_watchdog_pretimeout(smi_info->intf);
 	} else if (smi_info->msg_flags & RECEIVE_MSG_AVAIL) {
 		/* Messages available. */
 		smi_info->curr_msg = alloc_msg_handle_irq(smi_info);
@@ -798,8 +793,7 @@ static enum si_sm_result smi_event_handler(struct smi_info *smi_info,
 	 * We prefer handling attn over new messages.  But don't do
 	 * this if there is not yet an upper layer to handle anything.
 	 */
-	if (likely(smi_info->intf) &&
-	    (si_sm_result == SI_SM_ATTN || smi_info->got_attn)) {
+	if (si_sm_result == SI_SM_ATTN || smi_info->got_attn) {
 		unsigned char msg[2];
 
 		if (smi_info->si_state != SI_NORMAL) {
@@ -962,8 +956,8 @@ static inline int ipmi_thread_busy_wait(enum si_sm_result smi_result,
 {
 	unsigned int max_busy_us = 0;
 
-	if (smi_info->intf_num < num_max_busy_us)
-		max_busy_us = kipmid_max_busy_us[smi_info->intf_num];
+	if (smi_info->si_num < num_max_busy_us)
+		max_busy_us = kipmid_max_busy_us[smi_info->si_num];
 	if (max_busy_us == 0 || smi_result != SI_SM_CALL_WITH_DELAY)
 		ipmi_si_set_not_busy(busy_until);
 	else if (!ipmi_si_is_busy(busy_until)) {
@@ -1143,8 +1137,8 @@ irqreturn_t ipmi_si_irq_handler(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static int smi_start_processing(void       *send_info,
-				ipmi_smi_t intf)
+static int smi_start_processing(void            *send_info,
+				struct ipmi_smi *intf)
 {
 	struct smi_info *new_smi = send_info;
 	int             enable = 0;
@@ -1165,8 +1159,8 @@ static int smi_start_processing(void       *send_info,
 	/*
 	 * Check if the user forcefully enabled the daemon.
 	 */
-	if (new_smi->intf_num < num_force_kipmid)
-		enable = force_kipmid[new_smi->intf_num];
+	if (new_smi->si_num < num_force_kipmid)
+		enable = force_kipmid[new_smi->si_num];
 	/*
 	 * The BT interface is efficient enough to not need a thread,
 	 * and there is no need for a thread if we have interrupts.
@@ -1176,7 +1170,7 @@ static int smi_start_processing(void       *send_info,
 
 	if (enable) {
 		new_smi->thread = kthread_run(ipmi_thread, new_smi,
-					      "kipmi%d", new_smi->intf_num);
+					      "kipmi%d", new_smi->si_num);
 		if (IS_ERR(new_smi->thread)) {
 			dev_notice(new_smi->io.dev, "Could not start"
 				   " kernel thread due to error %ld, only using"
@@ -1209,9 +1203,11 @@ static void set_maintenance_mode(void *send_info, bool enable)
 		atomic_set(&smi_info->req_events, 0);
 }
 
+static void shutdown_smi(void *send_info);
 static const struct ipmi_smi_handlers handlers = {
 	.owner                  = THIS_MODULE,
 	.start_processing       = smi_start_processing,
+	.shutdown               = shutdown_smi,
 	.get_smi_info		= get_smi_info,
 	.sender			= sender,
 	.request_events		= request_events,
@@ -1592,102 +1588,6 @@ static int try_enable_event_buffer(struct smi_info *smi_info)
 	return rv;
 }
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-static int smi_type_proc_show(struct seq_file *m, void *v)
-{
-	struct smi_info *smi = m->private;
-
-	seq_printf(m, "%s\n", si_to_str[smi->io.si_type]);
-
-	return 0;
-}
-
-static int smi_type_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, smi_type_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations smi_type_proc_ops = {
-	.open		= smi_type_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-static int smi_si_stats_proc_show(struct seq_file *m, void *v)
-{
-	struct smi_info *smi = m->private;
-
-	seq_printf(m, "interrupts_enabled:    %d\n",
-		       smi->io.irq && !smi->interrupt_disabled);
-	seq_printf(m, "short_timeouts:        %u\n",
-		       smi_get_stat(smi, short_timeouts));
-	seq_printf(m, "long_timeouts:         %u\n",
-		       smi_get_stat(smi, long_timeouts));
-	seq_printf(m, "idles:                 %u\n",
-		       smi_get_stat(smi, idles));
-	seq_printf(m, "interrupts:            %u\n",
-		       smi_get_stat(smi, interrupts));
-	seq_printf(m, "attentions:            %u\n",
-		       smi_get_stat(smi, attentions));
-	seq_printf(m, "flag_fetches:          %u\n",
-		       smi_get_stat(smi, flag_fetches));
-	seq_printf(m, "hosed_count:           %u\n",
-		       smi_get_stat(smi, hosed_count));
-	seq_printf(m, "complete_transactions: %u\n",
-		       smi_get_stat(smi, complete_transactions));
-	seq_printf(m, "events:                %u\n",
-		       smi_get_stat(smi, events));
-	seq_printf(m, "watchdog_pretimeouts:  %u\n",
-		       smi_get_stat(smi, watchdog_pretimeouts));
-	seq_printf(m, "incoming_messages:     %u\n",
-		       smi_get_stat(smi, incoming_messages));
-	return 0;
-}
-
-static int smi_si_stats_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, smi_si_stats_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations smi_si_stats_proc_ops = {
-	.open		= smi_si_stats_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-static int smi_params_proc_show(struct seq_file *m, void *v)
-{
-	struct smi_info *smi = m->private;
-
-	seq_printf(m,
-		   "%s,%s,0x%lx,rsp=%d,rsi=%d,rsh=%d,irq=%d,ipmb=%d\n",
-		   si_to_str[smi->io.si_type],
-		   addr_space_to_str[smi->io.addr_type],
-		   smi->io.addr_data,
-		   smi->io.regspacing,
-		   smi->io.regsize,
-		   smi->io.regshift,
-		   smi->io.irq,
-		   smi->io.slave_addr);
-
-	return 0;
-}
-
-static int smi_params_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, smi_params_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations smi_params_proc_ops = {
-	.open		= smi_params_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-#endif
-
 #define IPMI_SI_ATTR(name) \
 static ssize_t ipmi_##name##_show(struct device *dev,			\
 				  struct device_attribute *attr,	\
@@ -2006,14 +1906,8 @@ int ipmi_si_add_smi(struct si_sm_io *io)
 
 	list_add_tail(&new_smi->link, &smi_infos);
 
-	if (initialized) {
+	if (initialized)
 		rv = try_smi_init(new_smi);
-		if (rv) {
-			cleanup_one_si(new_smi);
-			mutex_unlock(&smi_infos_lock);
-			return rv;
-		}
-	}
 out_err:
 	mutex_unlock(&smi_infos_lock);
 	return rv;
@@ -2056,19 +1950,19 @@ static int try_smi_init(struct smi_info *new_smi)
 		goto out_err;
 	}
 
-	new_smi->intf_num = smi_num;
+	new_smi->si_num = smi_num;
 
 	/* Do this early so it's available for logs. */
 	if (!new_smi->io.dev) {
 		init_name = kasprintf(GFP_KERNEL, "ipmi_si.%d",
-				      new_smi->intf_num);
+				      new_smi->si_num);
 
 		/*
 		 * If we don't already have a device from something
 		 * else (like PCI), then register a new one.
 		 */
 		new_smi->pdev = platform_device_alloc("ipmi_si",
-						      new_smi->intf_num);
+						      new_smi->si_num);
 		if (!new_smi->pdev) {
 			pr_err(PFX "Unable to allocate platform device\n");
 			rv = -ENOMEM;
@@ -2182,35 +2076,6 @@ static int try_smi_init(struct smi_info *new_smi)
 		goto out_err;
 	}
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-	rv = ipmi_smi_add_proc_entry(new_smi->intf, "type",
-				     &smi_type_proc_ops,
-				     new_smi);
-	if (rv) {
-		dev_err(new_smi->io.dev,
-			"Unable to create proc entry: %d\n", rv);
-		goto out_err;
-	}
-
-	rv = ipmi_smi_add_proc_entry(new_smi->intf, "si_stats",
-				     &smi_si_stats_proc_ops,
-				     new_smi);
-	if (rv) {
-		dev_err(new_smi->io.dev,
-			"Unable to create proc entry: %d\n", rv);
-		goto out_err;
-	}
-
-	rv = ipmi_smi_add_proc_entry(new_smi->intf, "params",
-				     &smi_params_proc_ops,
-				     new_smi);
-	if (rv) {
-		dev_err(new_smi->io.dev,
-			"Unable to create proc entry: %d\n", rv);
-		goto out_err;
-	}
-#endif
-
 	/* Don't increment till we know we have succeeded. */
 	smi_num++;
 
@@ -2223,7 +2088,8 @@ static int try_smi_init(struct smi_info *new_smi)
 	return 0;
 
 out_err:
-	shutdown_one_si(new_smi);
+	ipmi_unregister_smi(new_smi->intf);
+	new_smi->intf = NULL;
 
 	kfree(init_name);
 
@@ -2301,20 +2167,9 @@ static int init_ipmi_si(void)
 }
 module_init(init_ipmi_si);
 
-static void shutdown_one_si(struct smi_info *smi_info)
+static void shutdown_smi(void *send_info)
 {
-	int           rv = 0;
-
-	if (smi_info->intf) {
-		ipmi_smi_t intf = smi_info->intf;
-
-		smi_info->intf = NULL;
-		rv = ipmi_unregister_smi(intf);
-		if (rv) {
-			pr_err(PFX "Unable to unregister device: errno=%d\n",
-			       rv);
-		}
-	}
+	struct smi_info *smi_info = send_info;
 
 	if (smi_info->dev_group_added) {
 		device_remove_group(smi_info->io.dev, &ipmi_si_dev_attr_group);
@@ -2372,6 +2227,10 @@ static void shutdown_one_si(struct smi_info *smi_info)
 	smi_info->si_sm = NULL;
 }
 
+/*
+ * Must be called with smi_infos_lock held, to serialize the
+ * smi_info->intf check.
+ */
 static void cleanup_one_si(struct smi_info *smi_info)
 {
 	if (!smi_info)
@@ -2379,7 +2238,10 @@ static void cleanup_one_si(struct smi_info *smi_info)
 
 	list_del(&smi_info->link);
 
-	shutdown_one_si(smi_info);
+	if (smi_info->intf) {
+		ipmi_unregister_smi(smi_info->intf);
+		smi_info->intf = NULL;
+	}
 
 	if (smi_info->pdev) {
 		if (smi_info->pdev_registered)
diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
index 35a82f4bfd78..22f634eb09fd 100644
--- a/drivers/char/ipmi/ipmi_ssif.c
+++ b/drivers/char/ipmi/ipmi_ssif.c
@@ -193,8 +193,7 @@ typedef void (*ssif_i2c_done)(struct ssif_info *ssif_info, int result,
 			     unsigned char *data, unsigned int len);
 
 struct ssif_info {
-	ipmi_smi_t          intf;
-	int                 intf_num;
+	struct ipmi_smi     *intf;
 	spinlock_t	    lock;
 	struct ipmi_smi_msg *waiting_msg;
 	struct ipmi_smi_msg *curr_msg;
@@ -290,8 +289,6 @@ struct ssif_info {
 
 static bool initialized;
 
-static atomic_t next_intf = ATOMIC_INIT(0);
-
 static void return_hosed_msg(struct ssif_info *ssif_info,
 			     struct ipmi_smi_msg *msg);
 static void start_next_msg(struct ssif_info *ssif_info, unsigned long *flags);
@@ -315,17 +312,13 @@ static void ipmi_ssif_unlock_cond(struct ssif_info *ssif_info,
 static void deliver_recv_msg(struct ssif_info *ssif_info,
 			     struct ipmi_smi_msg *msg)
 {
-	ipmi_smi_t    intf = ssif_info->intf;
-
-	if (!intf) {
-		ipmi_free_smi_msg(msg);
-	} else if (msg->rsp_size < 0) {
+	if (msg->rsp_size < 0) {
 		return_hosed_msg(ssif_info, msg);
 		pr_err(PFX
 		       "Malformed message in deliver_recv_msg: rsp_size = %d\n",
 		       msg->rsp_size);
 	} else {
-		ipmi_smi_msg_received(intf, msg);
+		ipmi_smi_msg_received(ssif_info->intf, msg);
 	}
 }
 
@@ -452,12 +445,10 @@ static void start_recv_msg_fetch(struct ssif_info *ssif_info,
 static void handle_flags(struct ssif_info *ssif_info, unsigned long *flags)
 {
 	if (ssif_info->msg_flags & WDT_PRE_TIMEOUT_INT) {
-		ipmi_smi_t intf = ssif_info->intf;
 		/* Watchdog pre-timeout */
 		ssif_inc_stat(ssif_info, watchdog_pretimeouts);
 		start_clear_flags(ssif_info, flags);
-		if (intf)
-			ipmi_smi_watchdog_pretimeout(intf);
+		ipmi_smi_watchdog_pretimeout(ssif_info->intf);
 	} else if (ssif_info->msg_flags & RECEIVE_MSG_AVAIL)
 		/* Messages available. */
 		start_recv_msg_fetch(ssif_info, flags);
@@ -1094,27 +1085,8 @@ static void request_events(void *send_info)
 	}
 }
 
-static int inc_usecount(void *send_info)
-{
-	struct ssif_info *ssif_info = send_info;
-
-	if (!i2c_get_adapter(i2c_adapter_id(ssif_info->client->adapter)))
-		return -ENODEV;
-
-	i2c_use_client(ssif_info->client);
-	return 0;
-}
-
-static void dec_usecount(void *send_info)
-{
-	struct ssif_info *ssif_info = send_info;
-
-	i2c_release_client(ssif_info->client);
-	i2c_put_adapter(ssif_info->client->adapter);
-}
-
-static int ssif_start_processing(void *send_info,
-				 ipmi_smi_t intf)
+static int ssif_start_processing(void            *send_info,
+				 struct ipmi_smi *intf)
 {
 	struct ssif_info *ssif_info = send_info;
 
@@ -1225,25 +1197,9 @@ static const struct attribute_group ipmi_ssif_dev_attr_group = {
 	.attrs		= ipmi_ssif_dev_attrs,
 };
 
-static int ssif_remove(struct i2c_client *client)
+static void shutdown_ssif(void *send_info)
 {
-	struct ssif_info *ssif_info = i2c_get_clientdata(client);
-	struct ssif_addr_info *addr_info;
-	int rv;
-
-	if (!ssif_info)
-		return 0;
-
-	/*
-	 * After this point, we won't deliver anything asychronously
-	 * to the message handler.  We can unregister ourself.
-	 */
-	rv = ipmi_unregister_smi(ssif_info->intf);
-	if (rv) {
-		pr_err(PFX "Unable to unregister device: errno=%d\n", rv);
-		return rv;
-	}
-	ssif_info->intf = NULL;
+	struct ssif_info *ssif_info = send_info;
 
 	device_remove_group(&ssif_info->client->dev, &ipmi_ssif_dev_attr_group);
 	dev_set_drvdata(&ssif_info->client->dev, NULL);
@@ -1259,6 +1215,30 @@ static int ssif_remove(struct i2c_client *client)
 		kthread_stop(ssif_info->thread);
 	}
 
+	/*
+	 * No message can be outstanding now, we have removed the
+	 * upper layer and it permitted us to do so.
+	 */
+	kfree(ssif_info);
+}
+
+static int ssif_remove(struct i2c_client *client)
+{
+	struct ssif_info *ssif_info = i2c_get_clientdata(client);
+	struct ipmi_smi *intf;
+	struct ssif_addr_info *addr_info;
+
+	if (!ssif_info)
+		return 0;
+
+	/*
+	 * After this point, we won't deliver anything asychronously
+	 * to the message handler.  We can unregister ourself.
+	 */
+	intf = ssif_info->intf;
+	ssif_info->intf = NULL;
+	ipmi_unregister_smi(intf);
+
 	list_for_each_entry(addr_info, &ssif_infos, link) {
 		if (addr_info->client == client) {
 			addr_info->client = NULL;
@@ -1266,11 +1246,6 @@ static int ssif_remove(struct i2c_client *client)
 		}
 	}
 
-	/*
-	 * No message can be outstanding now, we have removed the
-	 * upper layer and it permitted us to do so.
-	 */
-	kfree(ssif_info);
 	return 0;
 }
 
@@ -1341,72 +1316,6 @@ static int ssif_detect(struct i2c_client *client, struct i2c_board_info *info)
 	return rv;
 }
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-static int smi_type_proc_show(struct seq_file *m, void *v)
-{
-	seq_puts(m, "ssif\n");
-
-	return 0;
-}
-
-static int smi_type_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, smi_type_proc_show, inode->i_private);
-}
-
-static const struct file_operations smi_type_proc_ops = {
-	.open		= smi_type_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-static int smi_stats_proc_show(struct seq_file *m, void *v)
-{
-	struct ssif_info *ssif_info = m->private;
-
-	seq_printf(m, "sent_messages:          %u\n",
-		   ssif_get_stat(ssif_info, sent_messages));
-	seq_printf(m, "sent_messages_parts:    %u\n",
-		   ssif_get_stat(ssif_info, sent_messages_parts));
-	seq_printf(m, "send_retries:           %u\n",
-		   ssif_get_stat(ssif_info, send_retries));
-	seq_printf(m, "send_errors:            %u\n",
-		   ssif_get_stat(ssif_info, send_errors));
-	seq_printf(m, "received_messages:      %u\n",
-		   ssif_get_stat(ssif_info, received_messages));
-	seq_printf(m, "received_message_parts: %u\n",
-		   ssif_get_stat(ssif_info, received_message_parts));
-	seq_printf(m, "receive_retries:        %u\n",
-		   ssif_get_stat(ssif_info, receive_retries));
-	seq_printf(m, "receive_errors:         %u\n",
-		   ssif_get_stat(ssif_info, receive_errors));
-	seq_printf(m, "flag_fetches:           %u\n",
-		   ssif_get_stat(ssif_info, flag_fetches));
-	seq_printf(m, "hosed:                  %u\n",
-		   ssif_get_stat(ssif_info, hosed));
-	seq_printf(m, "events:                 %u\n",
-		   ssif_get_stat(ssif_info, events));
-	seq_printf(m, "watchdog_pretimeouts:   %u\n",
-		   ssif_get_stat(ssif_info, watchdog_pretimeouts));
-	seq_printf(m, "alerts:                 %u\n",
-		   ssif_get_stat(ssif_info, alerts));
-	return 0;
-}
-
-static int smi_stats_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, smi_stats_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations smi_stats_proc_ops = {
-	.open		= smi_stats_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-#endif
-
 static int strcmp_nospace(char *s1, char *s2)
 {
 	while (*s1 && *s2) {
@@ -1678,8 +1587,6 @@ static int ssif_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	}
 
  found:
-	ssif_info->intf_num = atomic_inc_return(&next_intf);
-
 	if (ssif_dbg_probe) {
 		pr_info("ssif_probe: i2c_probe found device at i2c address %x\n",
 			client->addr);
@@ -1697,11 +1604,10 @@ static int ssif_probe(struct i2c_client *client, const struct i2c_device_id *id)
 
 	ssif_info->handlers.owner = THIS_MODULE;
 	ssif_info->handlers.start_processing = ssif_start_processing;
+	ssif_info->handlers.shutdown = shutdown_ssif;
 	ssif_info->handlers.get_smi_info = get_smi_info;
 	ssif_info->handlers.sender = sender;
 	ssif_info->handlers.request_events = request_events;
-	ssif_info->handlers.inc_usecount = inc_usecount;
-	ssif_info->handlers.dec_usecount = dec_usecount;
 
 	{
 		unsigned int thread_num;
@@ -1740,24 +1646,6 @@ static int ssif_probe(struct i2c_client *client, const struct i2c_device_id *id)
 		goto out_remove_attr;
 	}
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-	rv = ipmi_smi_add_proc_entry(ssif_info->intf, "type",
-				     &smi_type_proc_ops,
-				     ssif_info);
-	if (rv) {
-		pr_err(PFX "Unable to create proc entry: %d\n", rv);
-		goto out_err_unreg;
-	}
-
-	rv = ipmi_smi_add_proc_entry(ssif_info->intf, "ssif_stats",
-				     &smi_stats_proc_ops,
-				     ssif_info);
-	if (rv) {
-		pr_err(PFX "Unable to create proc entry: %d\n", rv);
-		goto out_err_unreg;
-	}
-#endif
-
  out:
 	if (rv) {
 		/*
@@ -1775,11 +1663,6 @@ static int ssif_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	kfree(resp);
 	return rv;
 
-#ifdef CONFIG_IPMI_PROC_INTERFACE
-out_err_unreg:
-	ipmi_unregister_smi(ssif_info->intf);
-#endif
-
 out_remove_attr:
 	device_remove_group(&ssif_info->client->dev, &ipmi_ssif_dev_attr_group);
 	dev_set_drvdata(&ssif_info->client->dev, NULL);
diff --git a/drivers/char/ipmi/ipmi_watchdog.c b/drivers/char/ipmi/ipmi_watchdog.c
index 22bc287eac2d..ca1c5c5109f0 100644
--- a/drivers/char/ipmi/ipmi_watchdog.c
+++ b/drivers/char/ipmi/ipmi_watchdog.c
@@ -125,7 +125,7 @@
 static DEFINE_MUTEX(ipmi_watchdog_mutex);
 static bool nowayout = WATCHDOG_NOWAYOUT;
 
-static ipmi_user_t watchdog_user;
+static struct ipmi_user *watchdog_user;
 static int watchdog_ifnum;
 
 /* Default the timeout to 10 seconds. */
@@ -153,7 +153,7 @@ static DEFINE_SPINLOCK(ipmi_read_lock);
 static char data_to_read;
 static DECLARE_WAIT_QUEUE_HEAD(read_q);
 static struct fasync_struct *fasync_q;
-static char pretimeout_since_last_heartbeat;
+static atomic_t pretimeout_since_last_heartbeat;
 static char expect_close;
 
 static int ifnum_to_use = -1;
@@ -303,9 +303,6 @@ MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
 /* Default state of the timer. */
 static unsigned char ipmi_watchdog_state = WDOG_TIMEOUT_NONE;
 
-/* If shutting down via IPMI, we ignore the heartbeat. */
-static int ipmi_ignore_heartbeat;
-
 /* Is someone using the watchdog?  Only one user is allowed. */
 static unsigned long ipmi_wdog_open;
 
@@ -329,35 +326,33 @@ static int testing_nmi;
 static int nmi_handler_registered;
 #endif
 
-static int ipmi_heartbeat(void);
+static int __ipmi_heartbeat(void);
 
 /*
- * We use a mutex to make sure that only one thing can send a set
- * timeout at one time, because we only have one copy of the data.
- * The mutex is claimed when the set_timeout is sent and freed
- * when both messages are free.
+ * We use a mutex to make sure that only one thing can send a set a
+ * message at one time.  The mutex is claimed when a message is sent
+ * and freed when both the send and receive messages are free.
  */
-static atomic_t set_timeout_tofree = ATOMIC_INIT(0);
-static DEFINE_MUTEX(set_timeout_lock);
-static DECLARE_COMPLETION(set_timeout_wait);
-static void set_timeout_free_smi(struct ipmi_smi_msg *msg)
+static atomic_t msg_tofree = ATOMIC_INIT(0);
+static DECLARE_COMPLETION(msg_wait);
+static void msg_free_smi(struct ipmi_smi_msg *msg)
 {
-    if (atomic_dec_and_test(&set_timeout_tofree))
-	    complete(&set_timeout_wait);
+	if (atomic_dec_and_test(&msg_tofree))
+		complete(&msg_wait);
 }
-static void set_timeout_free_recv(struct ipmi_recv_msg *msg)
+static void msg_free_recv(struct ipmi_recv_msg *msg)
 {
-    if (atomic_dec_and_test(&set_timeout_tofree))
-	    complete(&set_timeout_wait);
+	if (atomic_dec_and_test(&msg_tofree))
+		complete(&msg_wait);
 }
-static struct ipmi_smi_msg set_timeout_smi_msg = {
-	.done = set_timeout_free_smi
+static struct ipmi_smi_msg smi_msg = {
+	.done = msg_free_smi
 };
-static struct ipmi_recv_msg set_timeout_recv_msg = {
-	.done = set_timeout_free_recv
+static struct ipmi_recv_msg recv_msg = {
+	.done = msg_free_recv
 };
 
-static int i_ipmi_set_timeout(struct ipmi_smi_msg  *smi_msg,
+static int __ipmi_set_timeout(struct ipmi_smi_msg  *smi_msg,
 			      struct ipmi_recv_msg *recv_msg,
 			      int                  *send_heartbeat_now)
 {
@@ -368,9 +363,6 @@ static int i_ipmi_set_timeout(struct ipmi_smi_msg  *smi_msg,
 	int                               hbnow = 0;
 
 
-	/* These can be cleared as we are setting the timeout. */
-	pretimeout_since_last_heartbeat = 0;
-
 	data[0] = 0;
 	WDOG_SET_TIMER_USE(data[0], WDOG_TIMER_USE_SMS_OS);
 
@@ -414,46 +406,48 @@ static int i_ipmi_set_timeout(struct ipmi_smi_msg  *smi_msg,
 				      smi_msg,
 				      recv_msg,
 				      1);
-	if (rv) {
-		printk(KERN_WARNING PFX "set timeout error: %d\n",
-		       rv);
-	}
-
-	if (send_heartbeat_now)
-	    *send_heartbeat_now = hbnow;
+	if (rv)
+		pr_warn(PFX "set timeout error: %d\n", rv);
+	else if (send_heartbeat_now)
+		*send_heartbeat_now = hbnow;
 
 	return rv;
 }
 
-static int ipmi_set_timeout(int do_heartbeat)
+static int _ipmi_set_timeout(int do_heartbeat)
 {
 	int send_heartbeat_now;
 	int rv;
 
+	if (!watchdog_user)
+		return -ENODEV;
 
-	/* We can only send one of these at a time. */
-	mutex_lock(&set_timeout_lock);
-
-	atomic_set(&set_timeout_tofree, 2);
+	atomic_set(&msg_tofree, 2);
 
-	rv = i_ipmi_set_timeout(&set_timeout_smi_msg,
-				&set_timeout_recv_msg,
+	rv = __ipmi_set_timeout(&smi_msg,
+				&recv_msg,
 				&send_heartbeat_now);
-	if (rv) {
-		mutex_unlock(&set_timeout_lock);
-		goto out;
-	}
-
-	wait_for_completion(&set_timeout_wait);
+	if (rv)
+		return rv;
 
-	mutex_unlock(&set_timeout_lock);
+	wait_for_completion(&msg_wait);
 
 	if ((do_heartbeat == IPMI_SET_TIMEOUT_FORCE_HB)
-	    || ((send_heartbeat_now)
-		&& (do_heartbeat == IPMI_SET_TIMEOUT_HB_IF_NECESSARY)))
-		rv = ipmi_heartbeat();
+		|| ((send_heartbeat_now)
+		    && (do_heartbeat == IPMI_SET_TIMEOUT_HB_IF_NECESSARY)))
+		rv = __ipmi_heartbeat();
+
+	return rv;
+}
+
+static int ipmi_set_timeout(int do_heartbeat)
+{
+	int rv;
+
+	mutex_lock(&ipmi_watchdog_mutex);
+	rv = _ipmi_set_timeout(do_heartbeat);
+	mutex_unlock(&ipmi_watchdog_mutex);
 
-out:
 	return rv;
 }
 
@@ -531,13 +525,12 @@ static void panic_halt_ipmi_set_timeout(void)
 	while (atomic_read(&panic_done_count) != 0)
 		ipmi_poll_interface(watchdog_user);
 	atomic_add(1, &panic_done_count);
-	rv = i_ipmi_set_timeout(&panic_halt_smi_msg,
+	rv = __ipmi_set_timeout(&panic_halt_smi_msg,
 				&panic_halt_recv_msg,
 				&send_heartbeat_now);
 	if (rv) {
 		atomic_sub(1, &panic_done_count);
-		printk(KERN_WARNING PFX
-		       "Unable to extend the watchdog timeout.");
+		pr_warn(PFX "Unable to extend the watchdog timeout.");
 	} else {
 		if (send_heartbeat_now)
 			panic_halt_ipmi_heartbeat();
@@ -546,69 +539,22 @@ static void panic_halt_ipmi_set_timeout(void)
 		ipmi_poll_interface(watchdog_user);
 }
 
-/*
- * We use a mutex to make sure that only one thing can send a
- * heartbeat at one time, because we only have one copy of the data.
- * The semaphore is claimed when the set_timeout is sent and freed
- * when both messages are free.
- */
-static atomic_t heartbeat_tofree = ATOMIC_INIT(0);
-static DEFINE_MUTEX(heartbeat_lock);
-static DECLARE_COMPLETION(heartbeat_wait);
-static void heartbeat_free_smi(struct ipmi_smi_msg *msg)
-{
-    if (atomic_dec_and_test(&heartbeat_tofree))
-	    complete(&heartbeat_wait);
-}
-static void heartbeat_free_recv(struct ipmi_recv_msg *msg)
-{
-    if (atomic_dec_and_test(&heartbeat_tofree))
-	    complete(&heartbeat_wait);
-}
-static struct ipmi_smi_msg heartbeat_smi_msg = {
-	.done = heartbeat_free_smi
-};
-static struct ipmi_recv_msg heartbeat_recv_msg = {
-	.done = heartbeat_free_recv
-};
-
-static int ipmi_heartbeat(void)
+static int __ipmi_heartbeat(void)
 {
-	struct kernel_ipmi_msg            msg;
-	int                               rv;
+	struct kernel_ipmi_msg msg;
+	int rv;
 	struct ipmi_system_interface_addr addr;
-	int				  timeout_retries = 0;
-
-	if (ipmi_ignore_heartbeat)
-		return 0;
-
-	if (ipmi_start_timer_on_heartbeat) {
-		ipmi_start_timer_on_heartbeat = 0;
-		ipmi_watchdog_state = action_val;
-		return ipmi_set_timeout(IPMI_SET_TIMEOUT_FORCE_HB);
-	} else if (pretimeout_since_last_heartbeat) {
-		/*
-		 * A pretimeout occurred, make sure we set the timeout.
-		 * We don't want to set the action, though, we want to
-		 * leave that alone (thus it can't be combined with the
-		 * above operation.
-		 */
-		return ipmi_set_timeout(IPMI_SET_TIMEOUT_HB_IF_NECESSARY);
-	}
-
-	mutex_lock(&heartbeat_lock);
+	int timeout_retries = 0;
 
 restart:
-	atomic_set(&heartbeat_tofree, 2);
-
 	/*
 	 * Don't reset the timer if we have the timer turned off, that
 	 * re-enables the watchdog.
 	 */
-	if (ipmi_watchdog_state == WDOG_TIMEOUT_NONE) {
-		mutex_unlock(&heartbeat_lock);
+	if (ipmi_watchdog_state == WDOG_TIMEOUT_NONE)
 		return 0;
-	}
+
+	atomic_set(&msg_tofree, 2);
 
 	addr.addr_type = IPMI_SYSTEM_INTERFACE_ADDR_TYPE;
 	addr.channel = IPMI_BMC_CHANNEL;
@@ -623,26 +569,23 @@ static int ipmi_heartbeat(void)
 				      0,
 				      &msg,
 				      NULL,
-				      &heartbeat_smi_msg,
-				      &heartbeat_recv_msg,
+				      &smi_msg,
+				      &recv_msg,
 				      1);
 	if (rv) {
-		mutex_unlock(&heartbeat_lock);
-		printk(KERN_WARNING PFX "heartbeat failure: %d\n",
-		       rv);
+		pr_warn(PFX "heartbeat send failure: %d\n", rv);
 		return rv;
 	}
 
 	/* Wait for the heartbeat to be sent. */
-	wait_for_completion(&heartbeat_wait);
+	wait_for_completion(&msg_wait);
 
-	if (heartbeat_recv_msg.msg.data[0] == IPMI_WDOG_TIMER_NOT_INIT_RESP)  {
+	if (recv_msg.msg.data[0] == IPMI_WDOG_TIMER_NOT_INIT_RESP)  {
 		timeout_retries++;
 		if (timeout_retries > 3) {
-			printk(KERN_ERR PFX ": Unable to restore the IPMI"
-			       " watchdog's settings, giving up.\n");
+			pr_err(PFX ": Unable to restore the IPMI watchdog's settings, giving up.\n");
 			rv = -EIO;
-			goto out_unlock;
+			goto out;
 		}
 
 		/*
@@ -651,18 +594,17 @@ static int ipmi_heartbeat(void)
 		 * to restore the timer's info.  Note that we still hold
 		 * the heartbeat lock, to keep a heartbeat from happening
 		 * in this process, so must say no heartbeat to avoid a
-		 * deadlock on this mutex.
+		 * deadlock on this mutex
 		 */
-		rv = ipmi_set_timeout(IPMI_SET_TIMEOUT_NO_HB);
+		rv = _ipmi_set_timeout(IPMI_SET_TIMEOUT_NO_HB);
 		if (rv) {
-			printk(KERN_ERR PFX ": Unable to send the command to"
-			       " set the watchdog's settings, giving up.\n");
-			goto out_unlock;
+			pr_err(PFX ": Unable to send the command to set the watchdog's settings, giving up.\n");
+			goto out;
 		}
 
-		/* We might need a new heartbeat, so do it now */
+		/* Might need a heartbeat send, go ahead and do it. */
 		goto restart;
-	} else if (heartbeat_recv_msg.msg.data[0] != 0) {
+	} else if (recv_msg.msg.data[0] != 0) {
 		/*
 		 * Got an error in the heartbeat response.  It was already
 		 * reported in ipmi_wdog_msg_handler, but we should return
@@ -671,8 +613,43 @@ static int ipmi_heartbeat(void)
 		rv = -EINVAL;
 	}
 
-out_unlock:
-	mutex_unlock(&heartbeat_lock);
+out:
+	return rv;
+}
+
+static int _ipmi_heartbeat(void)
+{
+	int rv;
+
+	if (!watchdog_user)
+		return -ENODEV;
+
+	if (ipmi_start_timer_on_heartbeat) {
+		ipmi_start_timer_on_heartbeat = 0;
+		ipmi_watchdog_state = action_val;
+		rv = _ipmi_set_timeout(IPMI_SET_TIMEOUT_FORCE_HB);
+	} else if (atomic_cmpxchg(&pretimeout_since_last_heartbeat, 1, 0)) {
+		/*
+		 * A pretimeout occurred, make sure we set the timeout.
+		 * We don't want to set the action, though, we want to
+		 * leave that alone (thus it can't be combined with the
+		 * above operation.
+		 */
+		rv = _ipmi_set_timeout(IPMI_SET_TIMEOUT_HB_IF_NECESSARY);
+	} else {
+		rv = __ipmi_heartbeat();
+	}
+
+	return rv;
+}
+
+static int ipmi_heartbeat(void)
+{
+	int rv;
+
+	mutex_lock(&ipmi_watchdog_mutex);
+	rv = _ipmi_heartbeat();
+	mutex_unlock(&ipmi_watchdog_mutex);
 
 	return rv;
 }
@@ -700,7 +677,7 @@ static int ipmi_ioctl(struct file *file,
 		if (i)
 			return -EFAULT;
 		timeout = val;
-		return ipmi_set_timeout(IPMI_SET_TIMEOUT_HB_IF_NECESSARY);
+		return _ipmi_set_timeout(IPMI_SET_TIMEOUT_HB_IF_NECESSARY);
 
 	case WDIOC_GETTIMEOUT:
 		i = copy_to_user(argp, &timeout, sizeof(timeout));
@@ -713,7 +690,7 @@ static int ipmi_ioctl(struct file *file,
 		if (i)
 			return -EFAULT;
 		pretimeout = val;
-		return ipmi_set_timeout(IPMI_SET_TIMEOUT_HB_IF_NECESSARY);
+		return _ipmi_set_timeout(IPMI_SET_TIMEOUT_HB_IF_NECESSARY);
 
 	case WDIOC_GETPRETIMEOUT:
 		i = copy_to_user(argp, &pretimeout, sizeof(pretimeout));
@@ -722,7 +699,7 @@ static int ipmi_ioctl(struct file *file,
 		return 0;
 
 	case WDIOC_KEEPALIVE:
-		return ipmi_heartbeat();
+		return _ipmi_heartbeat();
 
 	case WDIOC_SETOPTIONS:
 		i = copy_from_user(&val, argp, sizeof(int));
@@ -730,13 +707,13 @@ static int ipmi_ioctl(struct file *file,
 			return -EFAULT;
 		if (val & WDIOS_DISABLECARD) {
 			ipmi_watchdog_state = WDOG_TIMEOUT_NONE;
-			ipmi_set_timeout(IPMI_SET_TIMEOUT_NO_HB);
+			_ipmi_set_timeout(IPMI_SET_TIMEOUT_NO_HB);
 			ipmi_start_timer_on_heartbeat = 0;
 		}
 
 		if (val & WDIOS_ENABLECARD) {
 			ipmi_watchdog_state = action_val;
-			ipmi_set_timeout(IPMI_SET_TIMEOUT_FORCE_HB);
+			_ipmi_set_timeout(IPMI_SET_TIMEOUT_FORCE_HB);
 		}
 		return 0;
 
@@ -810,7 +787,7 @@ static ssize_t ipmi_read(struct file *file,
 	 * Reading returns if the pretimeout has gone off, and it only does
 	 * it once per pretimeout.
 	 */
-	spin_lock(&ipmi_read_lock);
+	spin_lock_irq(&ipmi_read_lock);
 	if (!data_to_read) {
 		if (file->f_flags & O_NONBLOCK) {
 			rv = -EAGAIN;
@@ -821,9 +798,9 @@ static ssize_t ipmi_read(struct file *file,
 		add_wait_queue(&read_q, &wait);
 		while (!data_to_read) {
 			set_current_state(TASK_INTERRUPTIBLE);
-			spin_unlock(&ipmi_read_lock);
+			spin_unlock_irq(&ipmi_read_lock);
 			schedule();
-			spin_lock(&ipmi_read_lock);
+			spin_lock_irq(&ipmi_read_lock);
 		}
 		remove_wait_queue(&read_q, &wait);
 
@@ -835,7 +812,7 @@ static ssize_t ipmi_read(struct file *file,
 	data_to_read = 0;
 
  out:
-	spin_unlock(&ipmi_read_lock);
+	spin_unlock_irq(&ipmi_read_lock);
 
 	if (rv == 0) {
 		if (copy_to_user(buf, &data_to_read, 1))
@@ -873,10 +850,10 @@ static __poll_t ipmi_poll(struct file *file, poll_table *wait)
 
 	poll_wait(file, &read_q, wait);
 
-	spin_lock(&ipmi_read_lock);
+	spin_lock_irq(&ipmi_read_lock);
 	if (data_to_read)
 		mask |= (EPOLLIN | EPOLLRDNORM);
-	spin_unlock(&ipmi_read_lock);
+	spin_unlock_irq(&ipmi_read_lock);
 
 	return mask;
 }
@@ -894,11 +871,13 @@ static int ipmi_close(struct inode *ino, struct file *filep)
 {
 	if (iminor(ino) == WATCHDOG_MINOR) {
 		if (expect_close == 42) {
+			mutex_lock(&ipmi_watchdog_mutex);
 			ipmi_watchdog_state = WDOG_TIMEOUT_NONE;
-			ipmi_set_timeout(IPMI_SET_TIMEOUT_NO_HB);
+			_ipmi_set_timeout(IPMI_SET_TIMEOUT_NO_HB);
+			mutex_unlock(&ipmi_watchdog_mutex);
 		} else {
-			printk(KERN_CRIT PFX
-			       "Unexpected close, not stopping watchdog!\n");
+			pr_crit(PFX
+				"Unexpected close, not stopping watchdog!\n");
 			ipmi_heartbeat();
 		}
 		clear_bit(0, &ipmi_wdog_open);
@@ -932,11 +911,9 @@ static void ipmi_wdog_msg_handler(struct ipmi_recv_msg *msg,
 {
 	if (msg->msg.cmd == IPMI_WDOG_RESET_TIMER &&
 			msg->msg.data[0] == IPMI_WDOG_TIMER_NOT_INIT_RESP)
-		printk(KERN_INFO PFX "response: The IPMI controller appears"
-		       " to have been reset, will attempt to reinitialize"
-		       " the watchdog timer\n");
+		pr_info(PFX "response: The IPMI controller appears to have been reset, will attempt to reinitialize the watchdog timer\n");
 	else if (msg->msg.data[0] != 0)
-		printk(KERN_ERR PFX "response: Error %x on cmd %x\n",
+		pr_err(PFX "response: Error %x on cmd %x\n",
 		       msg->msg.data[0],
 		       msg->msg.cmd);
 
@@ -950,12 +927,13 @@ static void ipmi_wdog_pretimeout_handler(void *handler_data)
 			if (atomic_inc_and_test(&preop_panic_excl))
 				panic("Watchdog pre-timeout");
 		} else if (preop_val == WDOG_PREOP_GIVE_DATA) {
-			spin_lock(&ipmi_read_lock);
+			unsigned long flags;
+
+			spin_lock_irqsave(&ipmi_read_lock, flags);
 			data_to_read = 1;
 			wake_up_interruptible(&read_q);
 			kill_fasync(&fasync_q, SIGIO, POLL_IN);
-
-			spin_unlock(&ipmi_read_lock);
+			spin_unlock_irqrestore(&ipmi_read_lock, flags);
 		}
 	}
 
@@ -963,12 +941,34 @@ static void ipmi_wdog_pretimeout_handler(void *handler_data)
 	 * On some machines, the heartbeat will give an error and not
 	 * work unless we re-enable the timer.  So do so.
 	 */
-	pretimeout_since_last_heartbeat = 1;
+	atomic_set(&pretimeout_since_last_heartbeat, 1);
+}
+
+static void ipmi_wdog_panic_handler(void *user_data)
+{
+	static int panic_event_handled;
+
+	/*
+	 * On a panic, if we have a panic timeout, make sure to extend
+	 * the watchdog timer to a reasonable value to complete the
+	 * panic, if the watchdog timer is running.  Plus the
+	 * pretimeout is meaningless at panic time.
+	 */
+	if (watchdog_user && !panic_event_handled &&
+	    ipmi_watchdog_state != WDOG_TIMEOUT_NONE) {
+		/* Make sure we do this only once. */
+		panic_event_handled = 1;
+
+		timeout = panic_wdt_timeout;
+		pretimeout = 0;
+		panic_halt_ipmi_set_timeout();
+	}
 }
 
 static const struct ipmi_user_hndl ipmi_hndlrs = {
 	.ipmi_recv_hndl           = ipmi_wdog_msg_handler,
-	.ipmi_watchdog_pretimeout = ipmi_wdog_pretimeout_handler
+	.ipmi_watchdog_pretimeout = ipmi_wdog_pretimeout_handler,
+	.ipmi_panic_handler       = ipmi_wdog_panic_handler
 };
 
 static void ipmi_register_watchdog(int ipmi_intf)
@@ -985,7 +985,7 @@ static void ipmi_register_watchdog(int ipmi_intf)
 
 	rv = ipmi_create_user(ipmi_intf, &ipmi_hndlrs, NULL, &watchdog_user);
 	if (rv < 0) {
-		printk(KERN_CRIT PFX "Unable to register with ipmi\n");
+		pr_crit(PFX "Unable to register with ipmi\n");
 		goto out;
 	}
 
@@ -1002,7 +1002,7 @@ static void ipmi_register_watchdog(int ipmi_intf)
 	if (rv < 0) {
 		ipmi_destroy_user(watchdog_user);
 		watchdog_user = NULL;
-		printk(KERN_CRIT PFX "Unable to register misc device\n");
+		pr_crit(PFX "Unable to register misc device\n");
 	}
 
 #ifdef HAVE_DIE_NMI
@@ -1024,9 +1024,8 @@ static void ipmi_register_watchdog(int ipmi_intf)
 
 		rv = ipmi_set_timeout(IPMI_SET_TIMEOUT_FORCE_HB);
 		if (rv) {
-			printk(KERN_WARNING PFX "Error starting timer to"
-			       " test NMI: 0x%x.  The NMI pretimeout will"
-			       " likely not work\n", rv);
+			pr_warn(PFX "Error starting timer to test NMI: 0x%x.  The NMI pretimeout will likely not work\n",
+				rv);
 			rv = 0;
 			goto out_restore;
 		}
@@ -1034,9 +1033,7 @@ static void ipmi_register_watchdog(int ipmi_intf)
 		msleep(1500);
 
 		if (testing_nmi != 2) {
-			printk(KERN_WARNING PFX "IPMI NMI didn't seem to"
-			       " occur.  The NMI pretimeout will"
-			       " likely not work\n");
+			pr_warn(PFX "IPMI NMI didn't seem to occur.  The NMI pretimeout will likely not work\n");
 		}
  out_restore:
 		testing_nmi = 0;
@@ -1052,7 +1049,7 @@ static void ipmi_register_watchdog(int ipmi_intf)
 		start_now = 0; /* Disable this function after first startup. */
 		ipmi_watchdog_state = action_val;
 		ipmi_set_timeout(IPMI_SET_TIMEOUT_FORCE_HB);
-		printk(KERN_INFO PFX "Starting now!\n");
+		pr_info(PFX "Starting now!\n");
 	} else {
 		/* Stop the timer now. */
 		ipmi_watchdog_state = WDOG_TIMEOUT_NONE;
@@ -1063,34 +1060,38 @@ static void ipmi_register_watchdog(int ipmi_intf)
 static void ipmi_unregister_watchdog(int ipmi_intf)
 {
 	int rv;
+	struct ipmi_user *loc_user = watchdog_user;
 
-	if (!watchdog_user)
-		goto out;
+	if (!loc_user)
+		return;
 
 	if (watchdog_ifnum != ipmi_intf)
-		goto out;
+		return;
 
 	/* Make sure no one can call us any more. */
 	misc_deregister(&ipmi_wdog_miscdev);
 
+	watchdog_user = NULL;
+
 	/*
 	 * Wait to make sure the message makes it out.  The lower layer has
 	 * pointers to our buffers, we want to make sure they are done before
 	 * we release our memory.
 	 */
-	while (atomic_read(&set_timeout_tofree))
-		schedule_timeout_uninterruptible(1);
+	while (atomic_read(&msg_tofree))
+		msg_free_smi(NULL);
+
+	mutex_lock(&ipmi_watchdog_mutex);
 
 	/* Disconnect from IPMI. */
-	rv = ipmi_destroy_user(watchdog_user);
-	if (rv) {
-		printk(KERN_WARNING PFX "error unlinking from IPMI: %d\n",
-		       rv);
-	}
-	watchdog_user = NULL;
+	rv = ipmi_destroy_user(loc_user);
+	if (rv)
+		pr_warn(PFX "error unlinking from IPMI: %d\n",  rv);
 
- out:
-	return;
+	/* If it comes back, restart it properly. */
+	ipmi_start_timer_on_heartbeat = 1;
+
+	mutex_unlock(&ipmi_watchdog_mutex);
 }
 
 #ifdef HAVE_DIE_NMI
@@ -1124,7 +1125,7 @@ ipmi_nmi(unsigned int val, struct pt_regs *regs)
 		/* On some machines, the heartbeat will give
 		   an error and not work unless we re-enable
 		   the timer.   So do so. */
-		pretimeout_since_last_heartbeat = 1;
+		atomic_set(&pretimeout_since_last_heartbeat, 1);
 		if (atomic_inc_and_test(&preop_panic_excl))
 			nmi_panic(regs, PFX "pre-timeout");
 	}
@@ -1167,36 +1168,6 @@ static struct notifier_block wdog_reboot_notifier = {
 	.priority	= 0
 };
 
-static int wdog_panic_handler(struct notifier_block *this,
-			      unsigned long         event,
-			      void                  *unused)
-{
-	static int panic_event_handled;
-
-	/* On a panic, if we have a panic timeout, make sure to extend
-	   the watchdog timer to a reasonable value to complete the
-	   panic, if the watchdog timer is running.  Plus the
-	   pretimeout is meaningless at panic time. */
-	if (watchdog_user && !panic_event_handled &&
-	    ipmi_watchdog_state != WDOG_TIMEOUT_NONE) {
-		/* Make sure we do this only once. */
-		panic_event_handled = 1;
-
-		timeout = panic_wdt_timeout;
-		pretimeout = 0;
-		panic_halt_ipmi_set_timeout();
-	}
-
-	return NOTIFY_OK;
-}
-
-static struct notifier_block wdog_panic_notifier = {
-	.notifier_call	= wdog_panic_handler,
-	.next		= NULL,
-	.priority	= 150	/* priority: INT_MAX >= x >= 0 */
-};
-
-
 static void ipmi_new_smi(int if_num, struct device *device)
 {
 	ipmi_register_watchdog(if_num);
@@ -1288,9 +1259,7 @@ static void check_parms(void)
 	if (preaction_val == WDOG_PRETIMEOUT_NMI) {
 		do_nmi = 1;
 		if (preop_val == WDOG_PREOP_GIVE_DATA) {
-			printk(KERN_WARNING PFX "Pretimeout op is to give data"
-			       " but NMI pretimeout is enabled, setting"
-			       " pretimeout op to none\n");
+			pr_warn(PFX "Pretimeout op is to give data but NMI pretimeout is enabled, setting pretimeout op to none\n");
 			preop_op("preop_none", NULL);
 			do_nmi = 0;
 		}
@@ -1299,8 +1268,7 @@ static void check_parms(void)
 		rv = register_nmi_handler(NMI_UNKNOWN, ipmi_nmi, 0,
 						"ipmi");
 		if (rv) {
-			printk(KERN_WARNING PFX
-			       "Can't register nmi handler\n");
+			pr_warn(PFX "Can't register nmi handler\n");
 			return;
 		} else
 			nmi_handler_registered = 1;
@@ -1317,27 +1285,24 @@ static int __init ipmi_wdog_init(void)
 
 	if (action_op(action, NULL)) {
 		action_op("reset", NULL);
-		printk(KERN_INFO PFX "Unknown action '%s', defaulting to"
-		       " reset\n", action);
+		pr_info(PFX "Unknown action '%s', defaulting to reset\n",
+			action);
 	}
 
 	if (preaction_op(preaction, NULL)) {
 		preaction_op("pre_none", NULL);
-		printk(KERN_INFO PFX "Unknown preaction '%s', defaulting to"
-		       " none\n", preaction);
+		pr_info(PFX "Unknown preaction '%s', defaulting to none\n",
+			preaction);
 	}
 
 	if (preop_op(preop, NULL)) {
 		preop_op("preop_none", NULL);
-		printk(KERN_INFO PFX "Unknown preop '%s', defaulting to"
-		       " none\n", preop);
+		pr_info(PFX "Unknown preop '%s', defaulting to none\n", preop);
 	}
 
 	check_parms();
 
 	register_reboot_notifier(&wdog_reboot_notifier);
-	atomic_notifier_chain_register(&panic_notifier_list,
-			&wdog_panic_notifier);
 
 	rv = ipmi_smi_watcher_register(&smi_watcher);
 	if (rv) {
@@ -1345,14 +1310,12 @@ static int __init ipmi_wdog_init(void)
 		if (nmi_handler_registered)
 			unregister_nmi_handler(NMI_UNKNOWN, "ipmi");
 #endif
-		atomic_notifier_chain_unregister(&panic_notifier_list,
-						 &wdog_panic_notifier);
 		unregister_reboot_notifier(&wdog_reboot_notifier);
-		printk(KERN_WARNING PFX "can't register smi watcher\n");
+		pr_warn(PFX "can't register smi watcher\n");
 		return rv;
 	}
 
-	printk(KERN_INFO PFX "driver initialized\n");
+	pr_info(PFX "driver initialized\n");
 
 	return 0;
 }
@@ -1367,8 +1330,6 @@ static void __exit ipmi_wdog_exit(void)
 		unregister_nmi_handler(NMI_UNKNOWN, "ipmi");
 #endif
 
-	atomic_notifier_chain_unregister(&panic_notifier_list,
-					 &wdog_panic_notifier);
 	unregister_reboot_notifier(&wdog_reboot_notifier);
 }
 module_exit(ipmi_wdog_exit);
diff --git a/drivers/char/ipmi/kcs_bmc_npcm7xx.c b/drivers/char/ipmi/kcs_bmc_npcm7xx.c
new file mode 100644
index 000000000000..722f7391fe1f
--- /dev/null
+++ b/drivers/char/ipmi/kcs_bmc_npcm7xx.c
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018, Nuvoton Corporation.
+ * Copyright (c) 2018, Intel Corporation.
+ */
+
+#define pr_fmt(fmt) "nuvoton-kcs-bmc: " fmt
+
+#include <linux/atomic.h>
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#include "kcs_bmc.h"
+
+#define DEVICE_NAME	"npcm-kcs-bmc"
+#define KCS_CHANNEL_MAX	3
+
+#define KCS1ST		0x0C
+#define KCS2ST		0x1E
+#define KCS3ST		0x30
+
+#define KCS1DO		0x0E
+#define KCS2DO		0x20
+#define KCS3DO		0x32
+
+#define KCS1DI		0x10
+#define KCS2DI		0x22
+#define KCS3DI		0x34
+
+#define KCS1CTL		0x18
+#define KCS2CTL		0x2A
+#define KCS3CTL		0x3C
+#define    KCS_CTL_IBFIE	BIT(0)
+
+#define KCS1IE		0x1C
+#define KCS2IE		0x2E
+#define KCS3IE		0x40
+#define    KCS_IE_IRQE          BIT(0)
+#define    KCS_IE_HIRQE         BIT(3)
+
+/*
+ * 7.2.4 Core KCS Registers
+ * Registers in this module are 8 bits. An 8-bit register must be accessed
+ * by an 8-bit read or write.
+ *
+ * sts: KCS Channel n Status Register (KCSnST).
+ * dob: KCS Channel n Data Out Buffer Register (KCSnDO).
+ * dib: KCS Channel n Data In Buffer Register (KCSnDI).
+ * ctl: KCS Channel n Control Register (KCSnCTL).
+ * ie : KCS Channel n  Interrupt Enable Register (KCSnIE).
+ */
+struct npcm7xx_kcs_reg {
+	u32 sts;
+	u32 dob;
+	u32 dib;
+	u32 ctl;
+	u32 ie;
+};
+
+struct npcm7xx_kcs_bmc {
+	struct regmap *map;
+
+	const struct npcm7xx_kcs_reg *reg;
+};
+
+static const struct npcm7xx_kcs_reg npcm7xx_kcs_reg_tbl[KCS_CHANNEL_MAX] = {
+	{ .sts = KCS1ST, .dob = KCS1DO, .dib = KCS1DI, .ctl = KCS1CTL, .ie = KCS1IE },
+	{ .sts = KCS2ST, .dob = KCS2DO, .dib = KCS2DI, .ctl = KCS2CTL, .ie = KCS2IE },
+	{ .sts = KCS3ST, .dob = KCS3DO, .dib = KCS3DI, .ctl = KCS3CTL, .ie = KCS3IE },
+};
+
+static u8 npcm7xx_kcs_inb(struct kcs_bmc *kcs_bmc, u32 reg)
+{
+	struct npcm7xx_kcs_bmc *priv = kcs_bmc_priv(kcs_bmc);
+	u32 val = 0;
+	int rc;
+
+	rc = regmap_read(priv->map, reg, &val);
+	WARN(rc != 0, "regmap_read() failed: %d\n", rc);
+
+	return rc == 0 ? (u8)val : 0;
+}
+
+static void npcm7xx_kcs_outb(struct kcs_bmc *kcs_bmc, u32 reg, u8 data)
+{
+	struct npcm7xx_kcs_bmc *priv = kcs_bmc_priv(kcs_bmc);
+	int rc;
+
+	rc = regmap_write(priv->map, reg, data);
+	WARN(rc != 0, "regmap_write() failed: %d\n", rc);
+}
+
+static void npcm7xx_kcs_enable_channel(struct kcs_bmc *kcs_bmc, bool enable)
+{
+	struct npcm7xx_kcs_bmc *priv = kcs_bmc_priv(kcs_bmc);
+
+	regmap_update_bits(priv->map, priv->reg->ctl, KCS_CTL_IBFIE,
+			   enable ? KCS_CTL_IBFIE : 0);
+
+	regmap_update_bits(priv->map, priv->reg->ie, KCS_IE_IRQE | KCS_IE_HIRQE,
+			   enable ? KCS_IE_IRQE | KCS_IE_HIRQE : 0);
+}
+
+static irqreturn_t npcm7xx_kcs_irq(int irq, void *arg)
+{
+	struct kcs_bmc *kcs_bmc = arg;
+
+	if (!kcs_bmc_handle_event(kcs_bmc))
+		return IRQ_HANDLED;
+
+	return IRQ_NONE;
+}
+
+static int npcm7xx_kcs_config_irq(struct kcs_bmc *kcs_bmc,
+				  struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	int irq;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	return devm_request_irq(dev, irq, npcm7xx_kcs_irq, IRQF_SHARED,
+				dev_name(dev), kcs_bmc);
+}
+
+static int npcm7xx_kcs_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct npcm7xx_kcs_bmc *priv;
+	struct kcs_bmc *kcs_bmc;
+	u32 chan;
+	int rc;
+
+	rc = of_property_read_u32(dev->of_node, "kcs_chan", &chan);
+	if (rc != 0 || chan == 0 || chan > KCS_CHANNEL_MAX) {
+		dev_err(dev, "no valid 'kcs_chan' configured\n");
+		return -ENODEV;
+	}
+
+	kcs_bmc = kcs_bmc_alloc(dev, sizeof(*priv), chan);
+	if (!kcs_bmc)
+		return -ENOMEM;
+
+	priv = kcs_bmc_priv(kcs_bmc);
+	priv->map = syscon_node_to_regmap(dev->parent->of_node);
+	if (IS_ERR(priv->map)) {
+		dev_err(dev, "Couldn't get regmap\n");
+		return -ENODEV;
+	}
+	priv->reg = &npcm7xx_kcs_reg_tbl[chan - 1];
+
+	kcs_bmc->ioreg.idr = priv->reg->dib;
+	kcs_bmc->ioreg.odr = priv->reg->dob;
+	kcs_bmc->ioreg.str = priv->reg->sts;
+	kcs_bmc->io_inputb = npcm7xx_kcs_inb;
+	kcs_bmc->io_outputb = npcm7xx_kcs_outb;
+
+	dev_set_drvdata(dev, kcs_bmc);
+
+	npcm7xx_kcs_enable_channel(kcs_bmc, true);
+	rc = npcm7xx_kcs_config_irq(kcs_bmc, pdev);
+	if (rc)
+		return rc;
+
+	rc = misc_register(&kcs_bmc->miscdev);
+	if (rc) {
+		dev_err(dev, "Unable to register device\n");
+		return rc;
+	}
+
+	pr_info("channel=%u idr=0x%x odr=0x%x str=0x%x\n",
+		chan,
+		kcs_bmc->ioreg.idr, kcs_bmc->ioreg.odr, kcs_bmc->ioreg.str);
+
+	return 0;
+}
+
+static int npcm7xx_kcs_remove(struct platform_device *pdev)
+{
+	struct kcs_bmc *kcs_bmc = dev_get_drvdata(&pdev->dev);
+
+	misc_deregister(&kcs_bmc->miscdev);
+
+	return 0;
+}
+
+static const struct of_device_id npcm_kcs_bmc_match[] = {
+	{ .compatible = "nuvoton,npcm750-kcs-bmc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, npcm_kcs_bmc_match);
+
+static struct platform_driver npcm_kcs_bmc_driver = {
+	.driver = {
+		.name		= DEVICE_NAME,
+		.of_match_table	= npcm_kcs_bmc_match,
+	},
+	.probe	= npcm7xx_kcs_probe,
+	.remove	= npcm7xx_kcs_remove,
+};
+module_platform_driver(npcm_kcs_bmc_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Avi Fishman <avifishman70@gmail.com>");
+MODULE_AUTHOR("Haiyue Wang <haiyue.wang@linux.intel.com>");
+MODULE_DESCRIPTION("NPCM7xx device interface to the KCS BMC device");
diff --git a/drivers/char/misc.c b/drivers/char/misc.c
index 1bb9e7cc82e3..53cfe574d8d4 100644
--- a/drivers/char/misc.c
+++ b/drivers/char/misc.c
@@ -95,19 +95,6 @@ static const struct seq_operations misc_seq_ops = {
 	.stop  = misc_seq_stop,
 	.show  = misc_seq_show,
 };
-
-static int misc_seq_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &misc_seq_ops);
-}
-
-static const struct file_operations misc_proc_fops = {
-	.owner	 = THIS_MODULE,
-	.open    = misc_seq_open,
-	.read    = seq_read,
-	.llseek  = seq_lseek,
-	.release = seq_release,
-};
 #endif
 
 static int misc_open(struct inode *inode, struct file *file)
@@ -282,7 +269,7 @@ static int __init misc_init(void)
 	int err;
 	struct proc_dir_entry *ret;
 
-	ret = proc_create("misc", 0, NULL, &misc_proc_fops);
+	ret = proc_create_seq("misc", 0, NULL, &misc_seq_ops);
 	misc_class = class_create(THIS_MODULE, "misc");
 	err = PTR_ERR(misc_class);
 	if (IS_ERR(misc_class))
diff --git a/drivers/char/mspec.c b/drivers/char/mspec.c
index 7b75669d3670..058876b55b09 100644
--- a/drivers/char/mspec.c
+++ b/drivers/char/mspec.c
@@ -191,7 +191,7 @@ mspec_close(struct vm_area_struct *vma)
  *
  * Creates a mspec page and maps it to user space.
  */
-static int
+static vm_fault_t
 mspec_fault(struct vm_fault *vmf)
 {
 	unsigned long paddr, maddr;
@@ -223,14 +223,7 @@ mspec_fault(struct vm_fault *vmf)
 
 	pfn = paddr >> PAGE_SHIFT;
 
-	/*
-	 * vm_insert_pfn can fail with -EBUSY, but in that case it will
-	 * be because another thread has installed the pte first, so it
-	 * is no problem.
-	 */
-	vm_insert_pfn(vmf->vma, vmf->address, pfn);
-
-	return VM_FAULT_NOPAGE;
+	return vmf_insert_pfn(vmf->vma, vmf->address, pfn);
 }
 
 static const struct vm_operations_struct mspec_vm_ops = {
diff --git a/drivers/char/nvram.c b/drivers/char/nvram.c
index 678fa97e41fb..25264d65e716 100644
--- a/drivers/char/nvram.c
+++ b/drivers/char/nvram.c
@@ -389,22 +389,9 @@ static int nvram_proc_read(struct seq_file *seq, void *offset)
 	return 0;
 }
 
-static int nvram_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, nvram_proc_read, NULL);
-}
-
-static const struct file_operations nvram_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= nvram_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int nvram_add_proc_fs(void)
 {
-	if (!proc_create("driver/nvram", 0, NULL, &nvram_proc_fops))
+	if (!proc_create_single("driver/nvram", 0, NULL, nvram_proc_read))
 		return -ENOMEM;
 	return 0;
 }
diff --git a/drivers/char/pcmcia/cm4040_cs.c b/drivers/char/pcmcia/cm4040_cs.c
index f80965407d3c..d5e43606339c 100644
--- a/drivers/char/pcmcia/cm4040_cs.c
+++ b/drivers/char/pcmcia/cm4040_cs.c
@@ -505,7 +505,7 @@ static void cm4040_reader_release(struct pcmcia_device *link)
 
 	DEBUGP(3, dev, "-> cm4040_reader_release\n");
 	while (link->open) {
-		DEBUGP(3, dev, KERN_INFO MODULE_NAME ": delaying release "
+		DEBUGP(3, dev, MODULE_NAME ": delaying release "
 		       "until process has terminated\n");
  		wait_event(dev->devq, (link->open == 0));
 	}
diff --git a/drivers/char/pcmcia/synclink_cs.c b/drivers/char/pcmcia/synclink_cs.c
index aa502e9fb7fa..66b04194aa9f 100644
--- a/drivers/char/pcmcia/synclink_cs.c
+++ b/drivers/char/pcmcia/synclink_cs.c
@@ -2616,19 +2616,6 @@ static int mgslpc_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int mgslpc_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, mgslpc_proc_show, NULL);
-}
-
-static const struct file_operations mgslpc_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= mgslpc_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int rx_alloc_buffers(MGSLPC_INFO *info)
 {
 	/* each buffer has header and data */
@@ -2815,7 +2802,7 @@ static const struct tty_operations mgslpc_ops = {
 	.tiocmget = tiocmget,
 	.tiocmset = tiocmset,
 	.get_icount = mgslpc_get_icount,
-	.proc_fops = &mgslpc_proc_fops,
+	.proc_show = mgslpc_proc_show,
 };
 
 static int __init synclink_cs_init(void)
diff --git a/drivers/char/random.c b/drivers/char/random.c
index cd888d4ee605..a8fb0020ba5c 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -402,8 +402,7 @@ static struct poolinfo {
 /*
  * Static global variables
  */
-static DECLARE_WAIT_QUEUE_HEAD(random_read_wait);
-static DECLARE_WAIT_QUEUE_HEAD(random_write_wait);
+static DECLARE_WAIT_QUEUE_HEAD(random_wait);
 static struct fasync_struct *fasync;
 
 static DEFINE_SPINLOCK(random_ready_list_lock);
@@ -722,8 +721,8 @@ static void credit_entropy_bits(struct entropy_store *r, int nbits)
 
 		/* should we wake readers? */
 		if (entropy_bits >= random_read_wakeup_bits &&
-		    wq_has_sleeper(&random_read_wait)) {
-			wake_up_interruptible(&random_read_wait);
+		    wq_has_sleeper(&random_wait)) {
+			wake_up_interruptible_poll(&random_wait, POLLIN);
 			kill_fasync(&fasync, SIGIO, POLL_IN);
 		}
 		/* If the input pool is getting full, send some
@@ -1397,7 +1396,7 @@ static size_t account(struct entropy_store *r, size_t nbytes, int min,
 	trace_debit_entropy(r->name, 8 * ibytes);
 	if (ibytes &&
 	    (r->entropy_count >> ENTROPY_SHIFT) < random_write_wakeup_bits) {
-		wake_up_interruptible(&random_write_wait);
+		wake_up_interruptible_poll(&random_wait, POLLOUT);
 		kill_fasync(&fasync, SIGIO, POLL_OUT);
 	}
 
@@ -1839,7 +1838,7 @@ _random_read(int nonblock, char __user *buf, size_t nbytes)
 		if (nonblock)
 			return -EAGAIN;
 
-		wait_event_interruptible(random_read_wait,
+		wait_event_interruptible(random_wait,
 			ENTROPY_BITS(&input_pool) >=
 			random_read_wakeup_bits);
 		if (signal_pending(current))
@@ -1876,14 +1875,17 @@ urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
 	return ret;
 }
 
+static struct wait_queue_head *
+random_get_poll_head(struct file *file, __poll_t events)
+{
+	return &random_wait;
+}
+
 static __poll_t
-random_poll(struct file *file, poll_table * wait)
+random_poll_mask(struct file *file, __poll_t events)
 {
-	__poll_t mask;
+	__poll_t mask = 0;
 
-	poll_wait(file, &random_read_wait, wait);
-	poll_wait(file, &random_write_wait, wait);
-	mask = 0;
 	if (ENTROPY_BITS(&input_pool) >= random_read_wakeup_bits)
 		mask |= EPOLLIN | EPOLLRDNORM;
 	if (ENTROPY_BITS(&input_pool) < random_write_wakeup_bits)
@@ -1990,7 +1992,8 @@ static int random_fasync(int fd, struct file *filp, int on)
 const struct file_operations random_fops = {
 	.read  = random_read,
 	.write = random_write,
-	.poll  = random_poll,
+	.get_poll_head  = random_get_poll_head,
+	.poll_mask  = random_poll_mask,
 	.unlocked_ioctl = random_ioctl,
 	.fasync = random_fasync,
 	.llseek = noop_llseek,
@@ -2323,7 +2326,7 @@ void add_hwgenerator_randomness(const char *buffer, size_t count,
 	 * We'll be woken up again once below random_write_wakeup_thresh,
 	 * or when the calling thread is about to terminate.
 	 */
-	wait_event_interruptible(random_write_wait, kthread_should_stop() ||
+	wait_event_interruptible(random_wait, kthread_should_stop() ||
 			ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits);
 	mix_pool_bytes(poolp, buffer, count);
 	credit_entropy_bits(poolp, entropy);
diff --git a/drivers/char/rtc.c b/drivers/char/rtc.c
index 57dc546628b5..94fedeeec035 100644
--- a/drivers/char/rtc.c
+++ b/drivers/char/rtc.c
@@ -171,7 +171,7 @@ static void mask_rtc_irq_bit(unsigned char bit)
 #endif
 
 #ifdef CONFIG_PROC_FS
-static int rtc_proc_open(struct inode *inode, struct file *file);
+static int rtc_proc_show(struct seq_file *seq, void *v);
 #endif
 
 /*
@@ -832,16 +832,6 @@ static struct miscdevice rtc_dev = {
 	.fops		= &rtc_fops,
 };
 
-#ifdef CONFIG_PROC_FS
-static const struct file_operations rtc_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= rtc_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-#endif
-
 static resource_size_t rtc_size;
 
 static struct resource * __init rtc_request_region(resource_size_t size)
@@ -982,7 +972,7 @@ static int __init rtc_init(void)
 	}
 
 #ifdef CONFIG_PROC_FS
-	ent = proc_create("driver/rtc", 0, NULL, &rtc_proc_fops);
+	ent = proc_create_single("driver/rtc", 0, NULL, rtc_proc_show);
 	if (!ent)
 		printk(KERN_WARNING "rtc: Failed to register with procfs.\n");
 #endif
@@ -1201,11 +1191,6 @@ static int rtc_proc_show(struct seq_file *seq, void *v)
 #undef YN
 #undef NY
 }
-
-static int rtc_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, rtc_proc_show, NULL);
-}
 #endif
 
 static void rtc_get_rtc_time(struct rtc_time *rtc_tm)
diff --git a/drivers/char/toshiba.c b/drivers/char/toshiba.c
index 5488516da8ea..802376fe851a 100644
--- a/drivers/char/toshiba.c
+++ b/drivers/char/toshiba.c
@@ -326,19 +326,6 @@ static int proc_toshiba_show(struct seq_file *m, void *v)
 		key);
 	return 0;
 }
-
-static int proc_toshiba_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_toshiba_show, NULL);
-}
-
-static const struct file_operations proc_toshiba_fops = {
-	.owner		= THIS_MODULE,
-	.open		= proc_toshiba_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
 #endif
 
 
@@ -524,7 +511,7 @@ static int __init toshiba_init(void)
 	{
 		struct proc_dir_entry *pde;
 
-		pde = proc_create("toshiba", 0, NULL, &proc_toshiba_fops);
+		pde = proc_create_single("toshiba", 0, NULL, proc_toshiba_show);
 		if (!pde) {
 			misc_deregister(&tosh_device);
 			return -ENOMEM;
diff --git a/drivers/char/tpm/Makefile b/drivers/char/tpm/Makefile
index acd758381c58..4e9c33ca1f8f 100644
--- a/drivers/char/tpm/Makefile
+++ b/drivers/char/tpm/Makefile
@@ -4,11 +4,11 @@
 #
 obj-$(CONFIG_TCG_TPM) += tpm.o
 tpm-y := tpm-interface.o tpm-dev.o tpm-sysfs.o tpm-chip.o tpm2-cmd.o \
-	 tpm-dev-common.o tpmrm-dev.o tpm1_eventlog.o tpm2_eventlog.o \
-         tpm2-space.o
-tpm-$(CONFIG_ACPI) += tpm_ppi.o tpm_eventlog_acpi.o
-tpm-$(CONFIG_EFI) += tpm_eventlog_efi.o
-tpm-$(CONFIG_OF) += tpm_eventlog_of.o
+	 tpm-dev-common.o tpmrm-dev.o eventlog/common.o eventlog/tpm1.o \
+	 eventlog/tpm2.o tpm2-space.o
+tpm-$(CONFIG_ACPI) += tpm_ppi.o eventlog/acpi.o
+tpm-$(CONFIG_EFI) += eventlog/efi.o
+tpm-$(CONFIG_OF) += eventlog/of.o
 obj-$(CONFIG_TCG_TIS_CORE) += tpm_tis_core.o
 obj-$(CONFIG_TCG_TIS) += tpm_tis.o
 obj-$(CONFIG_TCG_TIS_SPI) += tpm_tis_spi.o
diff --git a/drivers/char/tpm/eventlog/acpi.c b/drivers/char/tpm/eventlog/acpi.c
new file mode 100644
index 000000000000..7c53b1973b62
--- /dev/null
+++ b/drivers/char/tpm/eventlog/acpi.c
@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2005 IBM Corporation
+ *
+ * Authors:
+ *	Seiji Munetoh <munetoh@jp.ibm.com>
+ *	Stefan Berger <stefanb@us.ibm.com>
+ *	Reiner Sailer <sailer@watson.ibm.com>
+ *	Kylene Hall <kjhall@us.ibm.com>
+ *	Nayna Jain <nayna@linux.vnet.ibm.com>
+ *
+ * Maintained by: <tpmdd-devel@lists.sourceforge.net>
+ *
+ * Access to the event log extended by the TCG BIOS of PC platform
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <linux/seq_file.h>
+#include <linux/fs.h>
+#include <linux/security.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/acpi.h>
+#include <linux/tpm_eventlog.h>
+
+#include "../tpm.h"
+#include "common.h"
+
+struct acpi_tcpa {
+	struct acpi_table_header hdr;
+	u16 platform_class;
+	union {
+		struct client_hdr {
+			u32 log_max_len __packed;
+			u64 log_start_addr __packed;
+		} client;
+		struct server_hdr {
+			u16 reserved;
+			u64 log_max_len __packed;
+			u64 log_start_addr __packed;
+		} server;
+	};
+};
+
+/* read binary bios log */
+int tpm_read_log_acpi(struct tpm_chip *chip)
+{
+	struct acpi_tcpa *buff;
+	acpi_status status;
+	void __iomem *virt;
+	u64 len, start;
+	struct tpm_bios_log *log;
+
+	if (chip->flags & TPM_CHIP_FLAG_TPM2)
+		return -ENODEV;
+
+	log = &chip->log;
+
+	/* Unfortuntely ACPI does not associate the event log with a specific
+	 * TPM, like PPI. Thus all ACPI TPMs will read the same log.
+	 */
+	if (!chip->acpi_dev_handle)
+		return -ENODEV;
+
+	/* Find TCPA entry in RSDT (ACPI_LOGICAL_ADDRESSING) */
+	status = acpi_get_table(ACPI_SIG_TCPA, 1,
+				(struct acpi_table_header **)&buff);
+
+	if (ACPI_FAILURE(status))
+		return -ENODEV;
+
+	switch(buff->platform_class) {
+	case BIOS_SERVER:
+		len = buff->server.log_max_len;
+		start = buff->server.log_start_addr;
+		break;
+	case BIOS_CLIENT:
+	default:
+		len = buff->client.log_max_len;
+		start = buff->client.log_start_addr;
+		break;
+	}
+	if (!len) {
+		dev_warn(&chip->dev, "%s: TCPA log area empty\n", __func__);
+		return -EIO;
+	}
+
+	/* malloc EventLog space */
+	log->bios_event_log = kmalloc(len, GFP_KERNEL);
+	if (!log->bios_event_log)
+		return -ENOMEM;
+
+	log->bios_event_log_end = log->bios_event_log + len;
+
+	virt = acpi_os_map_iomem(start, len);
+	if (!virt)
+		goto err;
+
+	memcpy_fromio(log->bios_event_log, virt, len);
+
+	acpi_os_unmap_iomem(virt, len);
+	return EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2;
+
+err:
+	kfree(log->bios_event_log);
+	log->bios_event_log = NULL;
+	return -EIO;
+
+}
diff --git a/drivers/char/tpm/eventlog/common.c b/drivers/char/tpm/eventlog/common.c
new file mode 100644
index 000000000000..5a8720df2b51
--- /dev/null
+++ b/drivers/char/tpm/eventlog/common.c
@@ -0,0 +1,195 @@
+/*
+ * Copyright (C) 2005, 2012 IBM Corporation
+ *
+ * Authors:
+ *	Kent Yoder <key@linux.vnet.ibm.com>
+ *	Seiji Munetoh <munetoh@jp.ibm.com>
+ *	Stefan Berger <stefanb@us.ibm.com>
+ *	Reiner Sailer <sailer@watson.ibm.com>
+ *	Kylene Hall <kjhall@us.ibm.com>
+ *	Nayna Jain <nayna@linux.vnet.ibm.com>
+ *
+ * Access to the event log created by a system's firmware / BIOS
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <linux/seq_file.h>
+#include <linux/fs.h>
+#include <linux/security.h>
+#include <linux/module.h>
+#include <linux/tpm_eventlog.h>
+
+#include "../tpm.h"
+#include "common.h"
+
+static int tpm_bios_measurements_open(struct inode *inode,
+					    struct file *file)
+{
+	int err;
+	struct seq_file *seq;
+	struct tpm_chip_seqops *chip_seqops;
+	const struct seq_operations *seqops;
+	struct tpm_chip *chip;
+
+	inode_lock(inode);
+	if (!inode->i_private) {
+		inode_unlock(inode);
+		return -ENODEV;
+	}
+	chip_seqops = (struct tpm_chip_seqops *)inode->i_private;
+	seqops = chip_seqops->seqops;
+	chip = chip_seqops->chip;
+	get_device(&chip->dev);
+	inode_unlock(inode);
+
+	/* now register seq file */
+	err = seq_open(file, seqops);
+	if (!err) {
+		seq = file->private_data;
+		seq->private = chip;
+	}
+
+	return err;
+}
+
+static int tpm_bios_measurements_release(struct inode *inode,
+					 struct file *file)
+{
+	struct seq_file *seq = (struct seq_file *)file->private_data;
+	struct tpm_chip *chip = (struct tpm_chip *)seq->private;
+
+	put_device(&chip->dev);
+
+	return seq_release(inode, file);
+}
+
+static const struct file_operations tpm_bios_measurements_ops = {
+	.owner = THIS_MODULE,
+	.open = tpm_bios_measurements_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = tpm_bios_measurements_release,
+};
+
+static int tpm_read_log(struct tpm_chip *chip)
+{
+	int rc;
+
+	if (chip->log.bios_event_log != NULL) {
+		dev_dbg(&chip->dev,
+			"%s: ERROR - event log already initialized\n",
+			__func__);
+		return -EFAULT;
+	}
+
+	rc = tpm_read_log_acpi(chip);
+	if (rc != -ENODEV)
+		return rc;
+
+	rc = tpm_read_log_efi(chip);
+	if (rc != -ENODEV)
+		return rc;
+
+	return tpm_read_log_of(chip);
+}
+
+/*
+ * tpm_bios_log_setup() - Read the event log from the firmware
+ * @chip: TPM chip to use.
+ *
+ * If an event log is found then the securityfs files are setup to
+ * export it to userspace, otherwise nothing is done.
+ *
+ * Returns -ENODEV if the firmware has no event log or securityfs is not
+ * supported.
+ */
+int tpm_bios_log_setup(struct tpm_chip *chip)
+{
+	const char *name = dev_name(&chip->dev);
+	unsigned int cnt;
+	int log_version;
+	int rc = 0;
+
+	rc = tpm_read_log(chip);
+	if (rc < 0)
+		return rc;
+	log_version = rc;
+
+	cnt = 0;
+	chip->bios_dir[cnt] = securityfs_create_dir(name, NULL);
+	/* NOTE: securityfs_create_dir can return ENODEV if securityfs is
+	 * compiled out. The caller should ignore the ENODEV return code.
+	 */
+	if (IS_ERR(chip->bios_dir[cnt]))
+		goto err;
+	cnt++;
+
+	chip->bin_log_seqops.chip = chip;
+	if (log_version == EFI_TCG2_EVENT_LOG_FORMAT_TCG_2)
+		chip->bin_log_seqops.seqops =
+			&tpm2_binary_b_measurements_seqops;
+	else
+		chip->bin_log_seqops.seqops =
+			&tpm1_binary_b_measurements_seqops;
+
+
+	chip->bios_dir[cnt] =
+	    securityfs_create_file("binary_bios_measurements",
+				   0440, chip->bios_dir[0],
+				   (void *)&chip->bin_log_seqops,
+				   &tpm_bios_measurements_ops);
+	if (IS_ERR(chip->bios_dir[cnt]))
+		goto err;
+	cnt++;
+
+	if (!(chip->flags & TPM_CHIP_FLAG_TPM2)) {
+
+		chip->ascii_log_seqops.chip = chip;
+		chip->ascii_log_seqops.seqops =
+			&tpm1_ascii_b_measurements_seqops;
+
+		chip->bios_dir[cnt] =
+			securityfs_create_file("ascii_bios_measurements",
+					       0440, chip->bios_dir[0],
+					       (void *)&chip->ascii_log_seqops,
+					       &tpm_bios_measurements_ops);
+		if (IS_ERR(chip->bios_dir[cnt]))
+			goto err;
+		cnt++;
+	}
+
+	return 0;
+
+err:
+	rc = PTR_ERR(chip->bios_dir[cnt]);
+	chip->bios_dir[cnt] = NULL;
+	tpm_bios_log_teardown(chip);
+	return rc;
+}
+
+void tpm_bios_log_teardown(struct tpm_chip *chip)
+{
+	int i;
+	struct inode *inode;
+
+	/* securityfs_remove currently doesn't take care of handling sync
+	 * between removal and opening of pseudo files. To handle this, a
+	 * workaround is added by making i_private = NULL here during removal
+	 * and to check it during open(), both within inode_lock()/unlock().
+	 * This design ensures that open() either safely gets kref or fails.
+	 */
+	for (i = (TPM_NUM_EVENT_LOG_FILES - 1); i >= 0; i--) {
+		if (chip->bios_dir[i]) {
+			inode = d_inode(chip->bios_dir[i]);
+			inode_lock(inode);
+			inode->i_private = NULL;
+			inode_unlock(inode);
+			securityfs_remove(chip->bios_dir[i]);
+		}
+	}
+}
diff --git a/drivers/char/tpm/eventlog/common.h b/drivers/char/tpm/eventlog/common.h
new file mode 100644
index 000000000000..47ff8136ceb5
--- /dev/null
+++ b/drivers/char/tpm/eventlog/common.h
@@ -0,0 +1,35 @@
+#ifndef __TPM_EVENTLOG_COMMON_H__
+#define __TPM_EVENTLOG_COMMON_H__
+
+#include "../tpm.h"
+
+extern const struct seq_operations tpm1_ascii_b_measurements_seqops;
+extern const struct seq_operations tpm1_binary_b_measurements_seqops;
+extern const struct seq_operations tpm2_binary_b_measurements_seqops;
+
+#if defined(CONFIG_ACPI)
+int tpm_read_log_acpi(struct tpm_chip *chip);
+#else
+static inline int tpm_read_log_acpi(struct tpm_chip *chip)
+{
+	return -ENODEV;
+}
+#endif
+#if defined(CONFIG_OF)
+int tpm_read_log_of(struct tpm_chip *chip);
+#else
+static inline int tpm_read_log_of(struct tpm_chip *chip)
+{
+	return -ENODEV;
+}
+#endif
+#if defined(CONFIG_EFI)
+int tpm_read_log_efi(struct tpm_chip *chip);
+#else
+static inline int tpm_read_log_efi(struct tpm_chip *chip)
+{
+	return -ENODEV;
+}
+#endif
+
+#endif
diff --git a/drivers/char/tpm/eventlog/efi.c b/drivers/char/tpm/eventlog/efi.c
new file mode 100644
index 000000000000..3e673ab22cb4
--- /dev/null
+++ b/drivers/char/tpm/eventlog/efi.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2017 Google
+ *
+ * Authors:
+ *      Thiebaud Weksteen <tweek@google.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <linux/efi.h>
+#include <linux/tpm_eventlog.h>
+
+#include "../tpm.h"
+#include "common.h"
+
+/* read binary bios log from EFI configuration table */
+int tpm_read_log_efi(struct tpm_chip *chip)
+{
+
+	struct linux_efi_tpm_eventlog *log_tbl;
+	struct tpm_bios_log *log;
+	u32 log_size;
+	u8 tpm_log_version;
+
+	if (!(chip->flags & TPM_CHIP_FLAG_TPM2))
+		return -ENODEV;
+
+	if (efi.tpm_log == EFI_INVALID_TABLE_ADDR)
+		return -ENODEV;
+
+	log = &chip->log;
+
+	log_tbl = memremap(efi.tpm_log, sizeof(*log_tbl), MEMREMAP_WB);
+	if (!log_tbl) {
+		pr_err("Could not map UEFI TPM log table !\n");
+		return -ENOMEM;
+	}
+
+	log_size = log_tbl->size;
+	memunmap(log_tbl);
+
+	log_tbl = memremap(efi.tpm_log, sizeof(*log_tbl) + log_size,
+			   MEMREMAP_WB);
+	if (!log_tbl) {
+		pr_err("Could not map UEFI TPM log table payload!\n");
+		return -ENOMEM;
+	}
+
+	/* malloc EventLog space */
+	log->bios_event_log = kmemdup(log_tbl->log, log_size, GFP_KERNEL);
+	if (!log->bios_event_log)
+		goto err_memunmap;
+	log->bios_event_log_end = log->bios_event_log + log_size;
+
+	tpm_log_version = log_tbl->version;
+	memunmap(log_tbl);
+	return tpm_log_version;
+
+err_memunmap:
+	memunmap(log_tbl);
+	return -ENOMEM;
+}
diff --git a/drivers/char/tpm/eventlog/of.c b/drivers/char/tpm/eventlog/of.c
new file mode 100644
index 000000000000..bba5fba6cb3b
--- /dev/null
+++ b/drivers/char/tpm/eventlog/of.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2012 IBM Corporation
+ *
+ * Author: Ashley Lai <ashleydlai@gmail.com>
+ *         Nayna Jain <nayna@linux.vnet.ibm.com>
+ *
+ * Maintained by: <tpmdd-devel@lists.sourceforge.net>
+ *
+ * Read the event log created by the firmware on PPC64
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <linux/slab.h>
+#include <linux/of.h>
+#include <linux/tpm_eventlog.h>
+
+#include "../tpm.h"
+#include "common.h"
+
+int tpm_read_log_of(struct tpm_chip *chip)
+{
+	struct device_node *np;
+	const u32 *sizep;
+	const u64 *basep;
+	struct tpm_bios_log *log;
+	u32 size;
+	u64 base;
+
+	log = &chip->log;
+	if (chip->dev.parent && chip->dev.parent->of_node)
+		np = chip->dev.parent->of_node;
+	else
+		return -ENODEV;
+
+	if (of_property_read_bool(np, "powered-while-suspended"))
+		chip->flags |= TPM_CHIP_FLAG_ALWAYS_POWERED;
+
+	sizep = of_get_property(np, "linux,sml-size", NULL);
+	basep = of_get_property(np, "linux,sml-base", NULL);
+	if (sizep == NULL && basep == NULL)
+		return -ENODEV;
+	if (sizep == NULL || basep == NULL)
+		return -EIO;
+
+	/*
+	 * For both vtpm/tpm, firmware has log addr and log size in big
+	 * endian format. But in case of vtpm, there is a method called
+	 * sml-handover which is run during kernel init even before
+	 * device tree is setup. This sml-handover function takes care
+	 * of endianness and writes to sml-base and sml-size in little
+	 * endian format. For this reason, vtpm doesn't need conversion
+	 * but physical tpm needs the conversion.
+	 */
+	if (of_property_match_string(np, "compatible", "IBM,vtpm") < 0) {
+		size = be32_to_cpup((__force __be32 *)sizep);
+		base = be64_to_cpup((__force __be64 *)basep);
+	} else {
+		size = *sizep;
+		base = *basep;
+	}
+
+	if (size == 0) {
+		dev_warn(&chip->dev, "%s: Event log area empty\n", __func__);
+		return -EIO;
+	}
+
+	log->bios_event_log = kmemdup(__va(base), size, GFP_KERNEL);
+	if (!log->bios_event_log)
+		return -ENOMEM;
+
+	log->bios_event_log_end = log->bios_event_log + size;
+
+	if (chip->flags & TPM_CHIP_FLAG_TPM2)
+		return EFI_TCG2_EVENT_LOG_FORMAT_TCG_2;
+	return EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2;
+}
diff --git a/drivers/char/tpm/eventlog/tpm1.c b/drivers/char/tpm/eventlog/tpm1.c
new file mode 100644
index 000000000000..58c84784ba25
--- /dev/null
+++ b/drivers/char/tpm/eventlog/tpm1.c
@@ -0,0 +1,310 @@
+/*
+ * Copyright (C) 2005, 2012 IBM Corporation
+ *
+ * Authors:
+ *	Kent Yoder <key@linux.vnet.ibm.com>
+ *	Seiji Munetoh <munetoh@jp.ibm.com>
+ *	Stefan Berger <stefanb@us.ibm.com>
+ *	Reiner Sailer <sailer@watson.ibm.com>
+ *	Kylene Hall <kjhall@us.ibm.com>
+ *	Nayna Jain <nayna@linux.vnet.ibm.com>
+ *
+ * Maintained by: <tpmdd-devel@lists.sourceforge.net>
+ *
+ * Access to the event log created by a system's firmware / BIOS
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <linux/seq_file.h>
+#include <linux/efi.h>
+#include <linux/fs.h>
+#include <linux/security.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/tpm_eventlog.h>
+
+#include "../tpm.h"
+#include "common.h"
+
+
+static const char* tcpa_event_type_strings[] = {
+	"PREBOOT",
+	"POST CODE",
+	"",
+	"NO ACTION",
+	"SEPARATOR",
+	"ACTION",
+	"EVENT TAG",
+	"S-CRTM Contents",
+	"S-CRTM Version",
+	"CPU Microcode",
+	"Platform Config Flags",
+	"Table of Devices",
+	"Compact Hash",
+	"IPL",
+	"IPL Partition Data",
+	"Non-Host Code",
+	"Non-Host Config",
+	"Non-Host Info"
+};
+
+static const char* tcpa_pc_event_id_strings[] = {
+	"",
+	"SMBIOS",
+	"BIS Certificate",
+	"POST BIOS ",
+	"ESCD ",
+	"CMOS",
+	"NVRAM",
+	"Option ROM",
+	"Option ROM config",
+	"",
+	"Option ROM microcode ",
+	"S-CRTM Version",
+	"S-CRTM Contents ",
+	"POST Contents ",
+	"Table of Devices",
+};
+
+/* returns pointer to start of pos. entry of tcg log */
+static void *tpm1_bios_measurements_start(struct seq_file *m, loff_t *pos)
+{
+	loff_t i;
+	struct tpm_chip *chip = m->private;
+	struct tpm_bios_log *log = &chip->log;
+	void *addr = log->bios_event_log;
+	void *limit = log->bios_event_log_end;
+	struct tcpa_event *event;
+	u32 converted_event_size;
+	u32 converted_event_type;
+
+
+	/* read over *pos measurements */
+	for (i = 0; i < *pos; i++) {
+		event = addr;
+
+		converted_event_size =
+		    do_endian_conversion(event->event_size);
+		converted_event_type =
+		    do_endian_conversion(event->event_type);
+
+		if ((addr + sizeof(struct tcpa_event)) < limit) {
+			if ((converted_event_type == 0) &&
+			    (converted_event_size == 0))
+				return NULL;
+			addr += (sizeof(struct tcpa_event) +
+				 converted_event_size);
+		}
+	}
+
+	/* now check if current entry is valid */
+	if ((addr + sizeof(struct tcpa_event)) >= limit)
+		return NULL;
+
+	event = addr;
+
+	converted_event_size = do_endian_conversion(event->event_size);
+	converted_event_type = do_endian_conversion(event->event_type);
+
+	if (((converted_event_type == 0) && (converted_event_size == 0))
+	    || ((addr + sizeof(struct tcpa_event) + converted_event_size)
+		>= limit))
+		return NULL;
+
+	return addr;
+}
+
+static void *tpm1_bios_measurements_next(struct seq_file *m, void *v,
+					loff_t *pos)
+{
+	struct tcpa_event *event = v;
+	struct tpm_chip *chip = m->private;
+	struct tpm_bios_log *log = &chip->log;
+	void *limit = log->bios_event_log_end;
+	u32 converted_event_size;
+	u32 converted_event_type;
+
+	converted_event_size = do_endian_conversion(event->event_size);
+
+	v += sizeof(struct tcpa_event) + converted_event_size;
+
+	/* now check if current entry is valid */
+	if ((v + sizeof(struct tcpa_event)) >= limit)
+		return NULL;
+
+	event = v;
+
+	converted_event_size = do_endian_conversion(event->event_size);
+	converted_event_type = do_endian_conversion(event->event_type);
+
+	if (((converted_event_type == 0) && (converted_event_size == 0)) ||
+	    ((v + sizeof(struct tcpa_event) + converted_event_size) >= limit))
+		return NULL;
+
+	(*pos)++;
+	return v;
+}
+
+static void tpm1_bios_measurements_stop(struct seq_file *m, void *v)
+{
+}
+
+static int get_event_name(char *dest, struct tcpa_event *event,
+			unsigned char * event_entry)
+{
+	const char *name = "";
+	/* 41 so there is room for 40 data and 1 nul */
+	char data[41] = "";
+	int i, n_len = 0, d_len = 0;
+	struct tcpa_pc_event *pc_event;
+
+	switch (do_endian_conversion(event->event_type)) {
+	case PREBOOT:
+	case POST_CODE:
+	case UNUSED:
+	case NO_ACTION:
+	case SCRTM_CONTENTS:
+	case SCRTM_VERSION:
+	case CPU_MICROCODE:
+	case PLATFORM_CONFIG_FLAGS:
+	case TABLE_OF_DEVICES:
+	case COMPACT_HASH:
+	case IPL:
+	case IPL_PARTITION_DATA:
+	case NONHOST_CODE:
+	case NONHOST_CONFIG:
+	case NONHOST_INFO:
+		name = tcpa_event_type_strings[do_endian_conversion
+						(event->event_type)];
+		n_len = strlen(name);
+		break;
+	case SEPARATOR:
+	case ACTION:
+		if (MAX_TEXT_EVENT >
+		    do_endian_conversion(event->event_size)) {
+			name = event_entry;
+			n_len = do_endian_conversion(event->event_size);
+		}
+		break;
+	case EVENT_TAG:
+		pc_event = (struct tcpa_pc_event *)event_entry;
+
+		/* ToDo Row data -> Base64 */
+
+		switch (do_endian_conversion(pc_event->event_id)) {
+		case SMBIOS:
+		case BIS_CERT:
+		case CMOS:
+		case NVRAM:
+		case OPTION_ROM_EXEC:
+		case OPTION_ROM_CONFIG:
+		case S_CRTM_VERSION:
+			name = tcpa_pc_event_id_strings[do_endian_conversion
+							(pc_event->event_id)];
+			n_len = strlen(name);
+			break;
+		/* hash data */
+		case POST_BIOS_ROM:
+		case ESCD:
+		case OPTION_ROM_MICROCODE:
+		case S_CRTM_CONTENTS:
+		case POST_CONTENTS:
+			name = tcpa_pc_event_id_strings[do_endian_conversion
+							(pc_event->event_id)];
+			n_len = strlen(name);
+			for (i = 0; i < 20; i++)
+				d_len += sprintf(&data[2*i], "%02x",
+						pc_event->event_data[i]);
+			break;
+		default:
+			break;
+		}
+	default:
+		break;
+	}
+
+	return snprintf(dest, MAX_TEXT_EVENT, "[%.*s%.*s]",
+			n_len, name, d_len, data);
+
+}
+
+static int tpm1_binary_bios_measurements_show(struct seq_file *m, void *v)
+{
+	struct tcpa_event *event = v;
+	struct tcpa_event temp_event;
+	char *temp_ptr;
+	int i;
+
+	memcpy(&temp_event, event, sizeof(struct tcpa_event));
+
+	/* convert raw integers for endianness */
+	temp_event.pcr_index = do_endian_conversion(event->pcr_index);
+	temp_event.event_type = do_endian_conversion(event->event_type);
+	temp_event.event_size = do_endian_conversion(event->event_size);
+
+	temp_ptr = (char *) &temp_event;
+
+	for (i = 0; i < (sizeof(struct tcpa_event) - 1) ; i++)
+		seq_putc(m, temp_ptr[i]);
+
+	temp_ptr = (char *) v;
+
+	for (i = (sizeof(struct tcpa_event) - 1);
+	     i < (sizeof(struct tcpa_event) + temp_event.event_size); i++)
+		seq_putc(m, temp_ptr[i]);
+
+	return 0;
+
+}
+
+static int tpm1_ascii_bios_measurements_show(struct seq_file *m, void *v)
+{
+	int len = 0;
+	char *eventname;
+	struct tcpa_event *event = v;
+	unsigned char *event_entry =
+	    (unsigned char *)(v + sizeof(struct tcpa_event));
+
+	eventname = kmalloc(MAX_TEXT_EVENT, GFP_KERNEL);
+	if (!eventname) {
+		printk(KERN_ERR "%s: ERROR - No Memory for event name\n ",
+		       __func__);
+		return -EFAULT;
+	}
+
+	/* 1st: PCR */
+	seq_printf(m, "%2d ", do_endian_conversion(event->pcr_index));
+
+	/* 2nd: SHA1 */
+	seq_printf(m, "%20phN", event->pcr_value);
+
+	/* 3rd: event type identifier */
+	seq_printf(m, " %02x", do_endian_conversion(event->event_type));
+
+	len += get_event_name(eventname, event, event_entry);
+
+	/* 4th: eventname <= max + \'0' delimiter */
+	seq_printf(m, " %s\n", eventname);
+
+	kfree(eventname);
+	return 0;
+}
+
+const struct seq_operations tpm1_ascii_b_measurements_seqops = {
+	.start = tpm1_bios_measurements_start,
+	.next = tpm1_bios_measurements_next,
+	.stop = tpm1_bios_measurements_stop,
+	.show = tpm1_ascii_bios_measurements_show,
+};
+
+const struct seq_operations tpm1_binary_b_measurements_seqops = {
+	.start = tpm1_bios_measurements_start,
+	.next = tpm1_bios_measurements_next,
+	.stop = tpm1_bios_measurements_stop,
+	.show = tpm1_binary_bios_measurements_show,
+};
diff --git a/drivers/char/tpm/eventlog/tpm2.c b/drivers/char/tpm/eventlog/tpm2.c
new file mode 100644
index 000000000000..1b8fa9de2cac
--- /dev/null
+++ b/drivers/char/tpm/eventlog/tpm2.c
@@ -0,0 +1,210 @@
+/*
+ * Copyright (C) 2016 IBM Corporation
+ *
+ * Authors:
+ *      Nayna Jain <nayna@linux.vnet.ibm.com>
+ *
+ * Access to TPM 2.0 event log as written by Firmware.
+ * It assumes that writer of event log has followed TCG Specification
+ * for Family "2.0" and written the event data in little endian.
+ * With that, it doesn't need any endian conversion for structure
+ * content.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include <linux/seq_file.h>
+#include <linux/fs.h>
+#include <linux/security.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/tpm_eventlog.h>
+
+#include "../tpm.h"
+#include "common.h"
+
+/*
+ * calc_tpm2_event_size() - calculate the event size, where event
+ * is an entry in the TPM 2.0 event log. The event is of type Crypto
+ * Agile Log Entry Format as defined in TCG EFI Protocol Specification
+ * Family "2.0".
+
+ * @event: event whose size is to be calculated.
+ * @event_header: the first event in the event log.
+ *
+ * Returns size of the event. If it is an invalid event, returns 0.
+ */
+static int calc_tpm2_event_size(struct tcg_pcr_event2 *event,
+				struct tcg_pcr_event *event_header)
+{
+	struct tcg_efi_specid_event *efispecid;
+	struct tcg_event_field *event_field;
+	void *marker;
+	void *marker_start;
+	u32 halg_size;
+	size_t size;
+	u16 halg;
+	int i;
+	int j;
+
+	marker = event;
+	marker_start = marker;
+	marker = marker + sizeof(event->pcr_idx) + sizeof(event->event_type)
+		+ sizeof(event->count);
+
+	efispecid = (struct tcg_efi_specid_event *)event_header->event;
+
+	/* Check if event is malformed. */
+	if (event->count > efispecid->num_algs)
+		return 0;
+
+	for (i = 0; i < event->count; i++) {
+		halg_size = sizeof(event->digests[i].alg_id);
+		memcpy(&halg, marker, halg_size);
+		marker = marker + halg_size;
+		for (j = 0; j < efispecid->num_algs; j++) {
+			if (halg == efispecid->digest_sizes[j].alg_id) {
+				marker +=
+					efispecid->digest_sizes[j].digest_size;
+				break;
+			}
+		}
+		/* Algorithm without known length. Such event is unparseable. */
+		if (j == efispecid->num_algs)
+			return 0;
+	}
+
+	event_field = (struct tcg_event_field *)marker;
+	marker = marker + sizeof(event_field->event_size)
+		+ event_field->event_size;
+	size = marker - marker_start;
+
+	if ((event->event_type == 0) && (event_field->event_size == 0))
+		return 0;
+
+	return size;
+}
+
+static void *tpm2_bios_measurements_start(struct seq_file *m, loff_t *pos)
+{
+	struct tpm_chip *chip = m->private;
+	struct tpm_bios_log *log = &chip->log;
+	void *addr = log->bios_event_log;
+	void *limit = log->bios_event_log_end;
+	struct tcg_pcr_event *event_header;
+	struct tcg_pcr_event2 *event;
+	size_t size;
+	int i;
+
+	event_header = addr;
+	size = sizeof(struct tcg_pcr_event) - sizeof(event_header->event)
+		+ event_header->event_size;
+
+	if (*pos == 0) {
+		if (addr + size < limit) {
+			if ((event_header->event_type == 0) &&
+			    (event_header->event_size == 0))
+				return NULL;
+			return SEQ_START_TOKEN;
+		}
+	}
+
+	if (*pos > 0) {
+		addr += size;
+		event = addr;
+		size = calc_tpm2_event_size(event, event_header);
+		if ((addr + size >=  limit) || (size == 0))
+			return NULL;
+	}
+
+	for (i = 0; i < (*pos - 1); i++) {
+		event = addr;
+		size = calc_tpm2_event_size(event, event_header);
+
+		if ((addr + size >= limit) || (size == 0))
+			return NULL;
+		addr += size;
+	}
+
+	return addr;
+}
+
+static void *tpm2_bios_measurements_next(struct seq_file *m, void *v,
+					 loff_t *pos)
+{
+	struct tcg_pcr_event *event_header;
+	struct tcg_pcr_event2 *event;
+	struct tpm_chip *chip = m->private;
+	struct tpm_bios_log *log = &chip->log;
+	void *limit = log->bios_event_log_end;
+	size_t event_size;
+	void *marker;
+
+	event_header = log->bios_event_log;
+
+	if (v == SEQ_START_TOKEN) {
+		event_size = sizeof(struct tcg_pcr_event) -
+			sizeof(event_header->event) + event_header->event_size;
+		marker = event_header;
+	} else {
+		event = v;
+		event_size = calc_tpm2_event_size(event, event_header);
+		if (event_size == 0)
+			return NULL;
+		marker = event;
+	}
+
+	marker = marker + event_size;
+	if (marker >= limit)
+		return NULL;
+	v = marker;
+	event = v;
+
+	event_size = calc_tpm2_event_size(event, event_header);
+	if (((v + event_size) >= limit) || (event_size == 0))
+		return NULL;
+
+	(*pos)++;
+	return v;
+}
+
+static void tpm2_bios_measurements_stop(struct seq_file *m, void *v)
+{
+}
+
+static int tpm2_binary_bios_measurements_show(struct seq_file *m, void *v)
+{
+	struct tpm_chip *chip = m->private;
+	struct tpm_bios_log *log = &chip->log;
+	struct tcg_pcr_event *event_header = log->bios_event_log;
+	struct tcg_pcr_event2 *event = v;
+	void *temp_ptr;
+	size_t size;
+
+	if (v == SEQ_START_TOKEN) {
+		size = sizeof(struct tcg_pcr_event) -
+			sizeof(event_header->event) + event_header->event_size;
+
+		temp_ptr = event_header;
+
+		if (size > 0)
+			seq_write(m, temp_ptr, size);
+	} else {
+		size = calc_tpm2_event_size(event, event_header);
+		temp_ptr = event;
+		if (size > 0)
+			seq_write(m, temp_ptr, size);
+	}
+
+	return 0;
+}
+
+const struct seq_operations tpm2_binary_b_measurements_seqops = {
+	.start = tpm2_bios_measurements_start,
+	.next = tpm2_bios_measurements_next,
+	.stop = tpm2_bios_measurements_stop,
+	.show = tpm2_binary_bios_measurements_show,
+};
diff --git a/drivers/char/tpm/st33zp24/spi.c b/drivers/char/tpm/st33zp24/spi.c
index 0fc4f20b5f83..d7909ab287a8 100644
--- a/drivers/char/tpm/st33zp24/spi.c
+++ b/drivers/char/tpm/st33zp24/spi.c
@@ -40,7 +40,7 @@
 #define ST33ZP24_OK					0x5A
 #define ST33ZP24_UNDEFINED_ERR				0x80
 #define ST33ZP24_BADLOCALITY				0x81
-#define ST33ZP24_TISREGISTER_UKNOWN			0x82
+#define ST33ZP24_TISREGISTER_UNKNOWN			0x82
 #define ST33ZP24_LOCALITY_NOT_ACTIVATED			0x83
 #define ST33ZP24_HASH_END_BEFORE_HASH_START		0x84
 #define ST33ZP24_BAD_COMMAND_ORDER			0x85
@@ -84,7 +84,7 @@ static int st33zp24_status_to_errno(u8 code)
 		return 0;
 	case ST33ZP24_UNDEFINED_ERR:
 	case ST33ZP24_BADLOCALITY:
-	case ST33ZP24_TISREGISTER_UKNOWN:
+	case ST33ZP24_TISREGISTER_UNKNOWN:
 	case ST33ZP24_LOCALITY_NOT_ACTIVATED:
 	case ST33ZP24_HASH_END_BEFORE_HASH_START:
 	case ST33ZP24_BAD_COMMAND_ORDER:
diff --git a/drivers/char/tpm/st33zp24/st33zp24.c b/drivers/char/tpm/st33zp24/st33zp24.c
index f95b9c75175b..abd675bec88c 100644
--- a/drivers/char/tpm/st33zp24/st33zp24.c
+++ b/drivers/char/tpm/st33zp24/st33zp24.c
@@ -373,8 +373,6 @@ static int st33zp24_send(struct tpm_chip *chip, unsigned char *buf,
 	int ret;
 	u8 data;
 
-	if (!chip)
-		return -EBUSY;
 	if (len < TPM_HEADER_SIZE)
 		return -EBUSY;
 
diff --git a/drivers/char/tpm/tpm-dev-common.c b/drivers/char/tpm/tpm-dev-common.c
index 230b99288024..e4a04b2d3c32 100644
--- a/drivers/char/tpm/tpm-dev-common.c
+++ b/drivers/char/tpm/tpm-dev-common.c
@@ -37,7 +37,7 @@ static void timeout_work(struct work_struct *work)
 	struct file_priv *priv = container_of(work, struct file_priv, work);
 
 	mutex_lock(&priv->buffer_mutex);
-	atomic_set(&priv->data_pending, 0);
+	priv->data_pending = 0;
 	memset(priv->data_buffer, 0, sizeof(priv->data_buffer));
 	mutex_unlock(&priv->buffer_mutex);
 }
@@ -46,7 +46,6 @@ void tpm_common_open(struct file *file, struct tpm_chip *chip,
 		     struct file_priv *priv)
 {
 	priv->chip = chip;
-	atomic_set(&priv->data_pending, 0);
 	mutex_init(&priv->buffer_mutex);
 	timer_setup(&priv->user_read_timer, user_reader_timeout, 0);
 	INIT_WORK(&priv->work, timeout_work);
@@ -58,29 +57,24 @@ ssize_t tpm_common_read(struct file *file, char __user *buf,
 			size_t size, loff_t *off)
 {
 	struct file_priv *priv = file->private_data;
-	ssize_t ret_size;
-	ssize_t orig_ret_size;
+	ssize_t ret_size = 0;
 	int rc;
 
 	del_singleshot_timer_sync(&priv->user_read_timer);
 	flush_work(&priv->work);
-	ret_size = atomic_read(&priv->data_pending);
-	if (ret_size > 0) {	/* relay data */
-		orig_ret_size = ret_size;
-		if (size < ret_size)
-			ret_size = size;
+	mutex_lock(&priv->buffer_mutex);
 
-		mutex_lock(&priv->buffer_mutex);
+	if (priv->data_pending) {
+		ret_size = min_t(ssize_t, size, priv->data_pending);
 		rc = copy_to_user(buf, priv->data_buffer, ret_size);
-		memset(priv->data_buffer, 0, orig_ret_size);
+		memset(priv->data_buffer, 0, priv->data_pending);
 		if (rc)
 			ret_size = -EFAULT;
 
-		mutex_unlock(&priv->buffer_mutex);
+		priv->data_pending = 0;
 	}
 
-	atomic_set(&priv->data_pending, 0);
-
+	mutex_unlock(&priv->buffer_mutex);
 	return ret_size;
 }
 
@@ -91,17 +85,19 @@ ssize_t tpm_common_write(struct file *file, const char __user *buf,
 	size_t in_size = size;
 	ssize_t out_size;
 
+	if (in_size > TPM_BUFSIZE)
+		return -E2BIG;
+
+	mutex_lock(&priv->buffer_mutex);
+
 	/* Cannot perform a write until the read has cleared either via
 	 * tpm_read or a user_read_timer timeout. This also prevents split
 	 * buffered writes from blocking here.
 	 */
-	if (atomic_read(&priv->data_pending) != 0)
+	if (priv->data_pending != 0) {
+		mutex_unlock(&priv->buffer_mutex);
 		return -EBUSY;
-
-	if (in_size > TPM_BUFSIZE)
-		return -E2BIG;
-
-	mutex_lock(&priv->buffer_mutex);
+	}
 
 	if (copy_from_user
 	    (priv->data_buffer, (void __user *) buf, in_size)) {
@@ -132,7 +128,7 @@ ssize_t tpm_common_write(struct file *file, const char __user *buf,
 		return out_size;
 	}
 
-	atomic_set(&priv->data_pending, out_size);
+	priv->data_pending = out_size;
 	mutex_unlock(&priv->buffer_mutex);
 
 	/* Set a timeout by which the reader must come claim the result */
@@ -149,5 +145,5 @@ void tpm_common_release(struct file *file, struct file_priv *priv)
 	del_singleshot_timer_sync(&priv->user_read_timer);
 	flush_work(&priv->work);
 	file->private_data = NULL;
-	atomic_set(&priv->data_pending, 0);
+	priv->data_pending = 0;
 }
diff --git a/drivers/char/tpm/tpm-dev.h b/drivers/char/tpm/tpm-dev.h
index ba3b6f9dacf7..b24cfb4d3ee1 100644
--- a/drivers/char/tpm/tpm-dev.h
+++ b/drivers/char/tpm/tpm-dev.h
@@ -8,7 +8,7 @@ struct file_priv {
 	struct tpm_chip *chip;
 
 	/* Data passed to and from the tpm via the read/write calls */
-	atomic_t data_pending;
+	size_t data_pending;
 	struct mutex buffer_mutex;
 
 	struct timer_list user_read_timer;      /* user needs to claim result */
diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c
index c43a9e28995e..e32f6e85dc6d 100644
--- a/drivers/char/tpm/tpm-interface.c
+++ b/drivers/char/tpm/tpm-interface.c
@@ -489,7 +489,7 @@ static ssize_t tpm_try_transmit(struct tpm_chip *chip,
 			goto out;
 		}
 
-		tpm_msleep(TPM_TIMEOUT);
+		tpm_msleep(TPM_TIMEOUT_POLL);
 		rmb();
 	} while (time_before(jiffies, stop));
 
@@ -587,7 +587,7 @@ ssize_t tpm_transmit(struct tpm_chip *chip, struct tpm_space *space,
 		 */
 		if (rc == TPM2_RC_TESTING && cc == TPM2_CC_SELF_TEST)
 			break;
-		delay_msec *= 2;
+
 		if (delay_msec > TPM2_DURATION_LONG) {
 			if (rc == TPM2_RC_RETRY)
 				dev_err(&chip->dev, "in retry loop\n");
@@ -597,6 +597,7 @@ ssize_t tpm_transmit(struct tpm_chip *chip, struct tpm_space *space,
 			break;
 		}
 		tpm_msleep(delay_msec);
+		delay_msec *= 2;
 		memcpy(buf, save, save_size);
 	}
 	return ret;
diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h
index 7f2d0f489e9c..4426649e431c 100644
--- a/drivers/char/tpm/tpm.h
+++ b/drivers/char/tpm/tpm.h
@@ -53,7 +53,10 @@ enum tpm_const {
 enum tpm_timeout {
 	TPM_TIMEOUT = 5,	/* msecs */
 	TPM_TIMEOUT_RETRY = 100, /* msecs */
-	TPM_TIMEOUT_RANGE_US = 300	/* usecs */
+	TPM_TIMEOUT_RANGE_US = 300,	/* usecs */
+	TPM_TIMEOUT_POLL = 1,	/* msecs */
+	TPM_TIMEOUT_USECS_MIN = 100,      /* usecs */
+	TPM_TIMEOUT_USECS_MAX = 500      /* usecs */
 };
 
 /* TPM addresses */
@@ -590,33 +593,6 @@ int tpm2_prepare_space(struct tpm_chip *chip, struct tpm_space *space, u32 cc,
 int tpm2_commit_space(struct tpm_chip *chip, struct tpm_space *space,
 		      u32 cc, u8 *buf, size_t *bufsiz);
 
-extern const struct seq_operations tpm2_binary_b_measurements_seqops;
-
-#if defined(CONFIG_ACPI)
-int tpm_read_log_acpi(struct tpm_chip *chip);
-#else
-static inline int tpm_read_log_acpi(struct tpm_chip *chip)
-{
-	return -ENODEV;
-}
-#endif
-#if defined(CONFIG_OF)
-int tpm_read_log_of(struct tpm_chip *chip);
-#else
-static inline int tpm_read_log_of(struct tpm_chip *chip)
-{
-	return -ENODEV;
-}
-#endif
-#if defined(CONFIG_EFI)
-int tpm_read_log_efi(struct tpm_chip *chip);
-#else
-static inline int tpm_read_log_efi(struct tpm_chip *chip)
-{
-	return -ENODEV;
-}
-#endif
-
 int tpm_bios_log_setup(struct tpm_chip *chip);
 void tpm_bios_log_teardown(struct tpm_chip *chip);
 #endif
diff --git a/drivers/char/tpm/tpm1_eventlog.c b/drivers/char/tpm/tpm1_eventlog.c
deleted file mode 100644
index add798bd69d0..000000000000
--- a/drivers/char/tpm/tpm1_eventlog.c
+++ /dev/null
@@ -1,476 +0,0 @@
-/*
- * Copyright (C) 2005, 2012 IBM Corporation
- *
- * Authors:
- *	Kent Yoder <key@linux.vnet.ibm.com>
- *	Seiji Munetoh <munetoh@jp.ibm.com>
- *	Stefan Berger <stefanb@us.ibm.com>
- *	Reiner Sailer <sailer@watson.ibm.com>
- *	Kylene Hall <kjhall@us.ibm.com>
- *	Nayna Jain <nayna@linux.vnet.ibm.com>
- *
- * Maintained by: <tpmdd-devel@lists.sourceforge.net>
- *
- * Access to the event log created by a system's firmware / BIOS
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- *
- */
-
-#include <linux/seq_file.h>
-#include <linux/efi.h>
-#include <linux/fs.h>
-#include <linux/security.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/tpm_eventlog.h>
-
-#include "tpm.h"
-
-
-static const char* tcpa_event_type_strings[] = {
-	"PREBOOT",
-	"POST CODE",
-	"",
-	"NO ACTION",
-	"SEPARATOR",
-	"ACTION",
-	"EVENT TAG",
-	"S-CRTM Contents",
-	"S-CRTM Version",
-	"CPU Microcode",
-	"Platform Config Flags",
-	"Table of Devices",
-	"Compact Hash",
-	"IPL",
-	"IPL Partition Data",
-	"Non-Host Code",
-	"Non-Host Config",
-	"Non-Host Info"
-};
-
-static const char* tcpa_pc_event_id_strings[] = {
-	"",
-	"SMBIOS",
-	"BIS Certificate",
-	"POST BIOS ",
-	"ESCD ",
-	"CMOS",
-	"NVRAM",
-	"Option ROM",
-	"Option ROM config",
-	"",
-	"Option ROM microcode ",
-	"S-CRTM Version",
-	"S-CRTM Contents ",
-	"POST Contents ",
-	"Table of Devices",
-};
-
-/* returns pointer to start of pos. entry of tcg log */
-static void *tpm_bios_measurements_start(struct seq_file *m, loff_t *pos)
-{
-	loff_t i;
-	struct tpm_chip *chip = m->private;
-	struct tpm_bios_log *log = &chip->log;
-	void *addr = log->bios_event_log;
-	void *limit = log->bios_event_log_end;
-	struct tcpa_event *event;
-	u32 converted_event_size;
-	u32 converted_event_type;
-
-
-	/* read over *pos measurements */
-	for (i = 0; i < *pos; i++) {
-		event = addr;
-
-		converted_event_size =
-		    do_endian_conversion(event->event_size);
-		converted_event_type =
-		    do_endian_conversion(event->event_type);
-
-		if ((addr + sizeof(struct tcpa_event)) < limit) {
-			if ((converted_event_type == 0) &&
-			    (converted_event_size == 0))
-				return NULL;
-			addr += (sizeof(struct tcpa_event) +
-				 converted_event_size);
-		}
-	}
-
-	/* now check if current entry is valid */
-	if ((addr + sizeof(struct tcpa_event)) >= limit)
-		return NULL;
-
-	event = addr;
-
-	converted_event_size = do_endian_conversion(event->event_size);
-	converted_event_type = do_endian_conversion(event->event_type);
-
-	if (((converted_event_type == 0) && (converted_event_size == 0))
-	    || ((addr + sizeof(struct tcpa_event) + converted_event_size)
-		>= limit))
-		return NULL;
-
-	return addr;
-}
-
-static void *tpm_bios_measurements_next(struct seq_file *m, void *v,
-					loff_t *pos)
-{
-	struct tcpa_event *event = v;
-	struct tpm_chip *chip = m->private;
-	struct tpm_bios_log *log = &chip->log;
-	void *limit = log->bios_event_log_end;
-	u32 converted_event_size;
-	u32 converted_event_type;
-
-	converted_event_size = do_endian_conversion(event->event_size);
-
-	v += sizeof(struct tcpa_event) + converted_event_size;
-
-	/* now check if current entry is valid */
-	if ((v + sizeof(struct tcpa_event)) >= limit)
-		return NULL;
-
-	event = v;
-
-	converted_event_size = do_endian_conversion(event->event_size);
-	converted_event_type = do_endian_conversion(event->event_type);
-
-	if (((converted_event_type == 0) && (converted_event_size == 0)) ||
-	    ((v + sizeof(struct tcpa_event) + converted_event_size) >= limit))
-		return NULL;
-
-	(*pos)++;
-	return v;
-}
-
-static void tpm_bios_measurements_stop(struct seq_file *m, void *v)
-{
-}
-
-static int get_event_name(char *dest, struct tcpa_event *event,
-			unsigned char * event_entry)
-{
-	const char *name = "";
-	/* 41 so there is room for 40 data and 1 nul */
-	char data[41] = "";
-	int i, n_len = 0, d_len = 0;
-	struct tcpa_pc_event *pc_event;
-
-	switch (do_endian_conversion(event->event_type)) {
-	case PREBOOT:
-	case POST_CODE:
-	case UNUSED:
-	case NO_ACTION:
-	case SCRTM_CONTENTS:
-	case SCRTM_VERSION:
-	case CPU_MICROCODE:
-	case PLATFORM_CONFIG_FLAGS:
-	case TABLE_OF_DEVICES:
-	case COMPACT_HASH:
-	case IPL:
-	case IPL_PARTITION_DATA:
-	case NONHOST_CODE:
-	case NONHOST_CONFIG:
-	case NONHOST_INFO:
-		name = tcpa_event_type_strings[do_endian_conversion
-						(event->event_type)];
-		n_len = strlen(name);
-		break;
-	case SEPARATOR:
-	case ACTION:
-		if (MAX_TEXT_EVENT >
-		    do_endian_conversion(event->event_size)) {
-			name = event_entry;
-			n_len = do_endian_conversion(event->event_size);
-		}
-		break;
-	case EVENT_TAG:
-		pc_event = (struct tcpa_pc_event *)event_entry;
-
-		/* ToDo Row data -> Base64 */
-
-		switch (do_endian_conversion(pc_event->event_id)) {
-		case SMBIOS:
-		case BIS_CERT:
-		case CMOS:
-		case NVRAM:
-		case OPTION_ROM_EXEC:
-		case OPTION_ROM_CONFIG:
-		case S_CRTM_VERSION:
-			name = tcpa_pc_event_id_strings[do_endian_conversion
-							(pc_event->event_id)];
-			n_len = strlen(name);
-			break;
-		/* hash data */
-		case POST_BIOS_ROM:
-		case ESCD:
-		case OPTION_ROM_MICROCODE:
-		case S_CRTM_CONTENTS:
-		case POST_CONTENTS:
-			name = tcpa_pc_event_id_strings[do_endian_conversion
-							(pc_event->event_id)];
-			n_len = strlen(name);
-			for (i = 0; i < 20; i++)
-				d_len += sprintf(&data[2*i], "%02x",
-						pc_event->event_data[i]);
-			break;
-		default:
-			break;
-		}
-	default:
-		break;
-	}
-
-	return snprintf(dest, MAX_TEXT_EVENT, "[%.*s%.*s]",
-			n_len, name, d_len, data);
-
-}
-
-static int tpm_binary_bios_measurements_show(struct seq_file *m, void *v)
-{
-	struct tcpa_event *event = v;
-	struct tcpa_event temp_event;
-	char *temp_ptr;
-	int i;
-
-	memcpy(&temp_event, event, sizeof(struct tcpa_event));
-
-	/* convert raw integers for endianness */
-	temp_event.pcr_index = do_endian_conversion(event->pcr_index);
-	temp_event.event_type = do_endian_conversion(event->event_type);
-	temp_event.event_size = do_endian_conversion(event->event_size);
-
-	temp_ptr = (char *) &temp_event;
-
-	for (i = 0; i < (sizeof(struct tcpa_event) - 1) ; i++)
-		seq_putc(m, temp_ptr[i]);
-
-	temp_ptr = (char *) v;
-
-	for (i = (sizeof(struct tcpa_event) - 1);
-	     i < (sizeof(struct tcpa_event) + temp_event.event_size); i++)
-		seq_putc(m, temp_ptr[i]);
-
-	return 0;
-
-}
-
-static int tpm_bios_measurements_release(struct inode *inode,
-					 struct file *file)
-{
-	struct seq_file *seq = (struct seq_file *)file->private_data;
-	struct tpm_chip *chip = (struct tpm_chip *)seq->private;
-
-	put_device(&chip->dev);
-
-	return seq_release(inode, file);
-}
-
-static int tpm_ascii_bios_measurements_show(struct seq_file *m, void *v)
-{
-	int len = 0;
-	char *eventname;
-	struct tcpa_event *event = v;
-	unsigned char *event_entry =
-	    (unsigned char *)(v + sizeof(struct tcpa_event));
-
-	eventname = kmalloc(MAX_TEXT_EVENT, GFP_KERNEL);
-	if (!eventname) {
-		printk(KERN_ERR "%s: ERROR - No Memory for event name\n ",
-		       __func__);
-		return -EFAULT;
-	}
-
-	/* 1st: PCR */
-	seq_printf(m, "%2d ", do_endian_conversion(event->pcr_index));
-
-	/* 2nd: SHA1 */
-	seq_printf(m, "%20phN", event->pcr_value);
-
-	/* 3rd: event type identifier */
-	seq_printf(m, " %02x", do_endian_conversion(event->event_type));
-
-	len += get_event_name(eventname, event, event_entry);
-
-	/* 4th: eventname <= max + \'0' delimiter */
-	seq_printf(m, " %s\n", eventname);
-
-	kfree(eventname);
-	return 0;
-}
-
-static const struct seq_operations tpm_ascii_b_measurements_seqops = {
-	.start = tpm_bios_measurements_start,
-	.next = tpm_bios_measurements_next,
-	.stop = tpm_bios_measurements_stop,
-	.show = tpm_ascii_bios_measurements_show,
-};
-
-static const struct seq_operations tpm_binary_b_measurements_seqops = {
-	.start = tpm_bios_measurements_start,
-	.next = tpm_bios_measurements_next,
-	.stop = tpm_bios_measurements_stop,
-	.show = tpm_binary_bios_measurements_show,
-};
-
-static int tpm_bios_measurements_open(struct inode *inode,
-					    struct file *file)
-{
-	int err;
-	struct seq_file *seq;
-	struct tpm_chip_seqops *chip_seqops;
-	const struct seq_operations *seqops;
-	struct tpm_chip *chip;
-
-	inode_lock(inode);
-	if (!inode->i_private) {
-		inode_unlock(inode);
-		return -ENODEV;
-	}
-	chip_seqops = (struct tpm_chip_seqops *)inode->i_private;
-	seqops = chip_seqops->seqops;
-	chip = chip_seqops->chip;
-	get_device(&chip->dev);
-	inode_unlock(inode);
-
-	/* now register seq file */
-	err = seq_open(file, seqops);
-	if (!err) {
-		seq = file->private_data;
-		seq->private = chip;
-	}
-
-	return err;
-}
-
-static const struct file_operations tpm_bios_measurements_ops = {
-	.owner = THIS_MODULE,
-	.open = tpm_bios_measurements_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = tpm_bios_measurements_release,
-};
-
-static int tpm_read_log(struct tpm_chip *chip)
-{
-	int rc;
-
-	if (chip->log.bios_event_log != NULL) {
-		dev_dbg(&chip->dev,
-			"%s: ERROR - event log already initialized\n",
-			__func__);
-		return -EFAULT;
-	}
-
-	rc = tpm_read_log_acpi(chip);
-	if (rc != -ENODEV)
-		return rc;
-
-	rc = tpm_read_log_efi(chip);
-	if (rc != -ENODEV)
-		return rc;
-
-	return tpm_read_log_of(chip);
-}
-
-/*
- * tpm_bios_log_setup() - Read the event log from the firmware
- * @chip: TPM chip to use.
- *
- * If an event log is found then the securityfs files are setup to
- * export it to userspace, otherwise nothing is done.
- *
- * Returns -ENODEV if the firmware has no event log or securityfs is not
- * supported.
- */
-int tpm_bios_log_setup(struct tpm_chip *chip)
-{
-	const char *name = dev_name(&chip->dev);
-	unsigned int cnt;
-	int log_version;
-	int rc = 0;
-
-	rc = tpm_read_log(chip);
-	if (rc < 0)
-		return rc;
-	log_version = rc;
-
-	cnt = 0;
-	chip->bios_dir[cnt] = securityfs_create_dir(name, NULL);
-	/* NOTE: securityfs_create_dir can return ENODEV if securityfs is
-	 * compiled out. The caller should ignore the ENODEV return code.
-	 */
-	if (IS_ERR(chip->bios_dir[cnt]))
-		goto err;
-	cnt++;
-
-	chip->bin_log_seqops.chip = chip;
-	if (log_version == EFI_TCG2_EVENT_LOG_FORMAT_TCG_2)
-		chip->bin_log_seqops.seqops =
-			&tpm2_binary_b_measurements_seqops;
-	else
-		chip->bin_log_seqops.seqops =
-			&tpm_binary_b_measurements_seqops;
-
-
-	chip->bios_dir[cnt] =
-	    securityfs_create_file("binary_bios_measurements",
-				   0440, chip->bios_dir[0],
-				   (void *)&chip->bin_log_seqops,
-				   &tpm_bios_measurements_ops);
-	if (IS_ERR(chip->bios_dir[cnt]))
-		goto err;
-	cnt++;
-
-	if (!(chip->flags & TPM_CHIP_FLAG_TPM2)) {
-
-		chip->ascii_log_seqops.chip = chip;
-		chip->ascii_log_seqops.seqops =
-			&tpm_ascii_b_measurements_seqops;
-
-		chip->bios_dir[cnt] =
-			securityfs_create_file("ascii_bios_measurements",
-					       0440, chip->bios_dir[0],
-					       (void *)&chip->ascii_log_seqops,
-					       &tpm_bios_measurements_ops);
-		if (IS_ERR(chip->bios_dir[cnt]))
-			goto err;
-		cnt++;
-	}
-
-	return 0;
-
-err:
-	rc = PTR_ERR(chip->bios_dir[cnt]);
-	chip->bios_dir[cnt] = NULL;
-	tpm_bios_log_teardown(chip);
-	return rc;
-}
-
-void tpm_bios_log_teardown(struct tpm_chip *chip)
-{
-	int i;
-	struct inode *inode;
-
-	/* securityfs_remove currently doesn't take care of handling sync
-	 * between removal and opening of pseudo files. To handle this, a
-	 * workaround is added by making i_private = NULL here during removal
-	 * and to check it during open(), both within inode_lock()/unlock().
-	 * This design ensures that open() either safely gets kref or fails.
-	 */
-	for (i = (TPM_NUM_EVENT_LOG_FILES - 1); i >= 0; i--) {
-		if (chip->bios_dir[i]) {
-			inode = d_inode(chip->bios_dir[i]);
-			inode_lock(inode);
-			inode->i_private = NULL;
-			inode_unlock(inode);
-			securityfs_remove(chip->bios_dir[i]);
-		}
-	}
-}
diff --git a/drivers/char/tpm/tpm2-space.c b/drivers/char/tpm/tpm2-space.c
index 4e4014eabdb9..6122d3276f72 100644
--- a/drivers/char/tpm/tpm2-space.c
+++ b/drivers/char/tpm/tpm2-space.c
@@ -102,8 +102,9 @@ static int tpm2_load_context(struct tpm_chip *chip, u8 *buf,
 		 * TPM_RC_REFERENCE_H0 means the session has been
 		 * flushed outside the space
 		 */
-		rc = -ENOENT;
+		*handle = 0;
 		tpm_buf_destroy(&tbuf);
+		return -ENOENT;
 	} else if (rc > 0) {
 		dev_warn(&chip->dev, "%s: failed with a TPM error 0x%04X\n",
 			 __func__, rc);
diff --git a/drivers/char/tpm/tpm2_eventlog.c b/drivers/char/tpm/tpm2_eventlog.c
deleted file mode 100644
index 1ce4411292ba..000000000000
--- a/drivers/char/tpm/tpm2_eventlog.c
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Copyright (C) 2016 IBM Corporation
- *
- * Authors:
- *      Nayna Jain <nayna@linux.vnet.ibm.com>
- *
- * Access to TPM 2.0 event log as written by Firmware.
- * It assumes that writer of event log has followed TCG Specification
- * for Family "2.0" and written the event data in little endian.
- * With that, it doesn't need any endian conversion for structure
- * content.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/seq_file.h>
-#include <linux/fs.h>
-#include <linux/security.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/tpm_eventlog.h>
-
-#include "tpm.h"
-
-/*
- * calc_tpm2_event_size() - calculate the event size, where event
- * is an entry in the TPM 2.0 event log. The event is of type Crypto
- * Agile Log Entry Format as defined in TCG EFI Protocol Specification
- * Family "2.0".
-
- * @event: event whose size is to be calculated.
- * @event_header: the first event in the event log.
- *
- * Returns size of the event. If it is an invalid event, returns 0.
- */
-static int calc_tpm2_event_size(struct tcg_pcr_event2 *event,
-				struct tcg_pcr_event *event_header)
-{
-	struct tcg_efi_specid_event *efispecid;
-	struct tcg_event_field *event_field;
-	void *marker;
-	void *marker_start;
-	u32 halg_size;
-	size_t size;
-	u16 halg;
-	int i;
-	int j;
-
-	marker = event;
-	marker_start = marker;
-	marker = marker + sizeof(event->pcr_idx) + sizeof(event->event_type)
-		+ sizeof(event->count);
-
-	efispecid = (struct tcg_efi_specid_event *)event_header->event;
-
-	/* Check if event is malformed. */
-	if (event->count > efispecid->num_algs)
-		return 0;
-
-	for (i = 0; i < event->count; i++) {
-		halg_size = sizeof(event->digests[i].alg_id);
-		memcpy(&halg, marker, halg_size);
-		marker = marker + halg_size;
-		for (j = 0; j < efispecid->num_algs; j++) {
-			if (halg == efispecid->digest_sizes[j].alg_id) {
-				marker +=
-					efispecid->digest_sizes[j].digest_size;
-				break;
-			}
-		}
-		/* Algorithm without known length. Such event is unparseable. */
-		if (j == efispecid->num_algs)
-			return 0;
-	}
-
-	event_field = (struct tcg_event_field *)marker;
-	marker = marker + sizeof(event_field->event_size)
-		+ event_field->event_size;
-	size = marker - marker_start;
-
-	if ((event->event_type == 0) && (event_field->event_size == 0))
-		return 0;
-
-	return size;
-}
-
-static void *tpm2_bios_measurements_start(struct seq_file *m, loff_t *pos)
-{
-	struct tpm_chip *chip = m->private;
-	struct tpm_bios_log *log = &chip->log;
-	void *addr = log->bios_event_log;
-	void *limit = log->bios_event_log_end;
-	struct tcg_pcr_event *event_header;
-	struct tcg_pcr_event2 *event;
-	size_t size;
-	int i;
-
-	event_header = addr;
-	size = sizeof(struct tcg_pcr_event) - sizeof(event_header->event)
-		+ event_header->event_size;
-
-	if (*pos == 0) {
-		if (addr + size < limit) {
-			if ((event_header->event_type == 0) &&
-			    (event_header->event_size == 0))
-				return NULL;
-			return SEQ_START_TOKEN;
-		}
-	}
-
-	if (*pos > 0) {
-		addr += size;
-		event = addr;
-		size = calc_tpm2_event_size(event, event_header);
-		if ((addr + size >=  limit) || (size == 0))
-			return NULL;
-	}
-
-	for (i = 0; i < (*pos - 1); i++) {
-		event = addr;
-		size = calc_tpm2_event_size(event, event_header);
-
-		if ((addr + size >= limit) || (size == 0))
-			return NULL;
-		addr += size;
-	}
-
-	return addr;
-}
-
-static void *tpm2_bios_measurements_next(struct seq_file *m, void *v,
-					 loff_t *pos)
-{
-	struct tcg_pcr_event *event_header;
-	struct tcg_pcr_event2 *event;
-	struct tpm_chip *chip = m->private;
-	struct tpm_bios_log *log = &chip->log;
-	void *limit = log->bios_event_log_end;
-	size_t event_size;
-	void *marker;
-
-	event_header = log->bios_event_log;
-
-	if (v == SEQ_START_TOKEN) {
-		event_size = sizeof(struct tcg_pcr_event) -
-			sizeof(event_header->event) + event_header->event_size;
-		marker = event_header;
-	} else {
-		event = v;
-		event_size = calc_tpm2_event_size(event, event_header);
-		if (event_size == 0)
-			return NULL;
-		marker = event;
-	}
-
-	marker = marker + event_size;
-	if (marker >= limit)
-		return NULL;
-	v = marker;
-	event = v;
-
-	event_size = calc_tpm2_event_size(event, event_header);
-	if (((v + event_size) >= limit) || (event_size == 0))
-		return NULL;
-
-	(*pos)++;
-	return v;
-}
-
-static void tpm2_bios_measurements_stop(struct seq_file *m, void *v)
-{
-}
-
-static int tpm2_binary_bios_measurements_show(struct seq_file *m, void *v)
-{
-	struct tpm_chip *chip = m->private;
-	struct tpm_bios_log *log = &chip->log;
-	struct tcg_pcr_event *event_header = log->bios_event_log;
-	struct tcg_pcr_event2 *event = v;
-	void *temp_ptr;
-	size_t size;
-
-	if (v == SEQ_START_TOKEN) {
-		size = sizeof(struct tcg_pcr_event) -
-			sizeof(event_header->event) + event_header->event_size;
-
-		temp_ptr = event_header;
-
-		if (size > 0)
-			seq_write(m, temp_ptr, size);
-	} else {
-		size = calc_tpm2_event_size(event, event_header);
-		temp_ptr = event;
-		if (size > 0)
-			seq_write(m, temp_ptr, size);
-	}
-
-	return 0;
-}
-
-const struct seq_operations tpm2_binary_b_measurements_seqops = {
-	.start = tpm2_bios_measurements_start,
-	.next = tpm2_bios_measurements_next,
-	.stop = tpm2_bios_measurements_stop,
-	.show = tpm2_binary_bios_measurements_show,
-};
diff --git a/drivers/char/tpm/tpm_crb.c b/drivers/char/tpm/tpm_crb.c
index 7f78482cd157..34fbc6cb097b 100644
--- a/drivers/char/tpm/tpm_crb.c
+++ b/drivers/char/tpm/tpm_crb.c
@@ -511,8 +511,10 @@ static int crb_map_io(struct acpi_device *device, struct crb_priv *priv,
 
 	priv->regs_t = crb_map_res(dev, priv, &io_res, buf->control_address,
 				   sizeof(struct crb_regs_tail));
-	if (IS_ERR(priv->regs_t))
-		return PTR_ERR(priv->regs_t);
+	if (IS_ERR(priv->regs_t)) {
+		ret = PTR_ERR(priv->regs_t);
+		goto out_relinquish_locality;
+	}
 
 	/*
 	 * PTT HW bug w/a: wake up the device to access
@@ -520,7 +522,7 @@ static int crb_map_io(struct acpi_device *device, struct crb_priv *priv,
 	 */
 	ret = crb_cmd_ready(dev, priv);
 	if (ret)
-		return ret;
+		goto out_relinquish_locality;
 
 	pa_high = ioread32(&priv->regs_t->ctrl_cmd_pa_high);
 	pa_low  = ioread32(&priv->regs_t->ctrl_cmd_pa_low);
@@ -565,6 +567,8 @@ static int crb_map_io(struct acpi_device *device, struct crb_priv *priv,
 
 	crb_go_idle(dev, priv);
 
+out_relinquish_locality:
+
 	__crb_relinquish_locality(dev, priv, 0);
 
 	return ret;
diff --git a/drivers/char/tpm/tpm_eventlog_acpi.c b/drivers/char/tpm/tpm_eventlog_acpi.c
deleted file mode 100644
index 66f19e93c216..000000000000
--- a/drivers/char/tpm/tpm_eventlog_acpi.c
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Copyright (C) 2005 IBM Corporation
- *
- * Authors:
- *	Seiji Munetoh <munetoh@jp.ibm.com>
- *	Stefan Berger <stefanb@us.ibm.com>
- *	Reiner Sailer <sailer@watson.ibm.com>
- *	Kylene Hall <kjhall@us.ibm.com>
- *	Nayna Jain <nayna@linux.vnet.ibm.com>
- *
- * Maintained by: <tpmdd-devel@lists.sourceforge.net>
- *
- * Access to the event log extended by the TCG BIOS of PC platform
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- *
- */
-
-#include <linux/seq_file.h>
-#include <linux/fs.h>
-#include <linux/security.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/acpi.h>
-#include <linux/tpm_eventlog.h>
-
-#include "tpm.h"
-
-struct acpi_tcpa {
-	struct acpi_table_header hdr;
-	u16 platform_class;
-	union {
-		struct client_hdr {
-			u32 log_max_len __packed;
-			u64 log_start_addr __packed;
-		} client;
-		struct server_hdr {
-			u16 reserved;
-			u64 log_max_len __packed;
-			u64 log_start_addr __packed;
-		} server;
-	};
-};
-
-/* read binary bios log */
-int tpm_read_log_acpi(struct tpm_chip *chip)
-{
-	struct acpi_tcpa *buff;
-	acpi_status status;
-	void __iomem *virt;
-	u64 len, start;
-	struct tpm_bios_log *log;
-
-	if (chip->flags & TPM_CHIP_FLAG_TPM2)
-		return -ENODEV;
-
-	log = &chip->log;
-
-	/* Unfortuntely ACPI does not associate the event log with a specific
-	 * TPM, like PPI. Thus all ACPI TPMs will read the same log.
-	 */
-	if (!chip->acpi_dev_handle)
-		return -ENODEV;
-
-	/* Find TCPA entry in RSDT (ACPI_LOGICAL_ADDRESSING) */
-	status = acpi_get_table(ACPI_SIG_TCPA, 1,
-				(struct acpi_table_header **)&buff);
-
-	if (ACPI_FAILURE(status))
-		return -ENODEV;
-
-	switch(buff->platform_class) {
-	case BIOS_SERVER:
-		len = buff->server.log_max_len;
-		start = buff->server.log_start_addr;
-		break;
-	case BIOS_CLIENT:
-	default:
-		len = buff->client.log_max_len;
-		start = buff->client.log_start_addr;
-		break;
-	}
-	if (!len) {
-		dev_warn(&chip->dev, "%s: TCPA log area empty\n", __func__);
-		return -EIO;
-	}
-
-	/* malloc EventLog space */
-	log->bios_event_log = kmalloc(len, GFP_KERNEL);
-	if (!log->bios_event_log)
-		return -ENOMEM;
-
-	log->bios_event_log_end = log->bios_event_log + len;
-
-	virt = acpi_os_map_iomem(start, len);
-	if (!virt)
-		goto err;
-
-	memcpy_fromio(log->bios_event_log, virt, len);
-
-	acpi_os_unmap_iomem(virt, len);
-	return EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2;
-
-err:
-	kfree(log->bios_event_log);
-	log->bios_event_log = NULL;
-	return -EIO;
-
-}
diff --git a/drivers/char/tpm/tpm_eventlog_efi.c b/drivers/char/tpm/tpm_eventlog_efi.c
deleted file mode 100644
index e3f9ffd341d2..000000000000
--- a/drivers/char/tpm/tpm_eventlog_efi.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (C) 2017 Google
- *
- * Authors:
- *      Thiebaud Weksteen <tweek@google.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- *
- */
-
-#include <linux/efi.h>
-#include <linux/tpm_eventlog.h>
-
-#include "tpm.h"
-
-/* read binary bios log from EFI configuration table */
-int tpm_read_log_efi(struct tpm_chip *chip)
-{
-
-	struct linux_efi_tpm_eventlog *log_tbl;
-	struct tpm_bios_log *log;
-	u32 log_size;
-	u8 tpm_log_version;
-
-	if (!(chip->flags & TPM_CHIP_FLAG_TPM2))
-		return -ENODEV;
-
-	if (efi.tpm_log == EFI_INVALID_TABLE_ADDR)
-		return -ENODEV;
-
-	log = &chip->log;
-
-	log_tbl = memremap(efi.tpm_log, sizeof(*log_tbl), MEMREMAP_WB);
-	if (!log_tbl) {
-		pr_err("Could not map UEFI TPM log table !\n");
-		return -ENOMEM;
-	}
-
-	log_size = log_tbl->size;
-	memunmap(log_tbl);
-
-	log_tbl = memremap(efi.tpm_log, sizeof(*log_tbl) + log_size,
-			   MEMREMAP_WB);
-	if (!log_tbl) {
-		pr_err("Could not map UEFI TPM log table payload!\n");
-		return -ENOMEM;
-	}
-
-	/* malloc EventLog space */
-	log->bios_event_log = kmalloc(log_size, GFP_KERNEL);
-	if (!log->bios_event_log)
-		goto err_memunmap;
-	memcpy(log->bios_event_log, log_tbl->log, log_size);
-	log->bios_event_log_end = log->bios_event_log + log_size;
-
-	tpm_log_version = log_tbl->version;
-	memunmap(log_tbl);
-	return tpm_log_version;
-
-err_memunmap:
-	memunmap(log_tbl);
-	return -ENOMEM;
-}
diff --git a/drivers/char/tpm/tpm_eventlog_of.c b/drivers/char/tpm/tpm_eventlog_of.c
deleted file mode 100644
index 96fd5646f866..000000000000
--- a/drivers/char/tpm/tpm_eventlog_of.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright 2012 IBM Corporation
- *
- * Author: Ashley Lai <ashleydlai@gmail.com>
- *         Nayna Jain <nayna@linux.vnet.ibm.com>
- *
- * Maintained by: <tpmdd-devel@lists.sourceforge.net>
- *
- * Read the event log created by the firmware on PPC64
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- *
- */
-
-#include <linux/slab.h>
-#include <linux/of.h>
-#include <linux/tpm_eventlog.h>
-
-#include "tpm.h"
-
-int tpm_read_log_of(struct tpm_chip *chip)
-{
-	struct device_node *np;
-	const u32 *sizep;
-	const u64 *basep;
-	struct tpm_bios_log *log;
-	u32 size;
-	u64 base;
-
-	log = &chip->log;
-	if (chip->dev.parent && chip->dev.parent->of_node)
-		np = chip->dev.parent->of_node;
-	else
-		return -ENODEV;
-
-	if (of_property_read_bool(np, "powered-while-suspended"))
-		chip->flags |= TPM_CHIP_FLAG_ALWAYS_POWERED;
-
-	sizep = of_get_property(np, "linux,sml-size", NULL);
-	basep = of_get_property(np, "linux,sml-base", NULL);
-	if (sizep == NULL && basep == NULL)
-		return -ENODEV;
-	if (sizep == NULL || basep == NULL)
-		return -EIO;
-
-	/*
-	 * For both vtpm/tpm, firmware has log addr and log size in big
-	 * endian format. But in case of vtpm, there is a method called
-	 * sml-handover which is run during kernel init even before
-	 * device tree is setup. This sml-handover function takes care
-	 * of endianness and writes to sml-base and sml-size in little
-	 * endian format. For this reason, vtpm doesn't need conversion
-	 * but physical tpm needs the conversion.
-	 */
-	if (of_property_match_string(np, "compatible", "IBM,vtpm") < 0) {
-		size = be32_to_cpup(sizep);
-		base = be64_to_cpup(basep);
-	} else {
-		size = *sizep;
-		base = *basep;
-	}
-
-	if (size == 0) {
-		dev_warn(&chip->dev, "%s: Event log area empty\n", __func__);
-		return -EIO;
-	}
-
-	log->bios_event_log = kmalloc(size, GFP_KERNEL);
-	if (!log->bios_event_log)
-		return -ENOMEM;
-
-	log->bios_event_log_end = log->bios_event_log + size;
-
-	memcpy(log->bios_event_log, __va(base), size);
-
-	if (chip->flags & TPM_CHIP_FLAG_TPM2)
-		return EFI_TCG2_EVENT_LOG_FORMAT_TCG_2;
-	return EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2;
-}
diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c
index 5a1f47b43947..8b46aaa9e049 100644
--- a/drivers/char/tpm/tpm_tis_core.c
+++ b/drivers/char/tpm/tpm_tis_core.c
@@ -31,12 +31,6 @@
 #include "tpm.h"
 #include "tpm_tis_core.h"
 
-/* This is a polling delay to check for status and burstcount.
- * As per ddwg input, expectation is that status check and burstcount
- * check should return within few usecs.
- */
-#define TPM_POLL_SLEEP	1  /* msec */
-
 static void tpm_tis_clkrun_enable(struct tpm_chip *chip, bool value);
 
 static bool wait_for_tpm_stat_cond(struct tpm_chip *chip, u8 mask,
@@ -90,7 +84,8 @@ static int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask,
 		}
 	} else {
 		do {
-			tpm_msleep(TPM_POLL_SLEEP);
+			usleep_range(TPM_TIMEOUT_USECS_MIN,
+				     TPM_TIMEOUT_USECS_MAX);
 			status = chip->ops->status(chip);
 			if ((status & mask) == mask)
 				return 0;
@@ -143,13 +138,58 @@ static bool check_locality(struct tpm_chip *chip, int l)
 	return false;
 }
 
+static bool locality_inactive(struct tpm_chip *chip, int l)
+{
+	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
+	int rc;
+	u8 access;
+
+	rc = tpm_tis_read8(priv, TPM_ACCESS(l), &access);
+	if (rc < 0)
+		return false;
+
+	if ((access & (TPM_ACCESS_VALID | TPM_ACCESS_ACTIVE_LOCALITY))
+	    == TPM_ACCESS_VALID)
+		return true;
+
+	return false;
+}
+
 static int release_locality(struct tpm_chip *chip, int l)
 {
 	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
+	unsigned long stop, timeout;
+	long rc;
 
 	tpm_tis_write8(priv, TPM_ACCESS(l), TPM_ACCESS_ACTIVE_LOCALITY);
 
-	return 0;
+	stop = jiffies + chip->timeout_a;
+
+	if (chip->flags & TPM_CHIP_FLAG_IRQ) {
+again:
+		timeout = stop - jiffies;
+		if ((long)timeout <= 0)
+			return -1;
+
+		rc = wait_event_interruptible_timeout(priv->int_queue,
+						      (locality_inactive(chip, l)),
+						      timeout);
+
+		if (rc > 0)
+			return 0;
+
+		if (rc == -ERESTARTSYS && freezing(current)) {
+			clear_thread_flag(TIF_SIGPENDING);
+			goto again;
+		}
+	} else {
+		do {
+			if (locality_inactive(chip, l))
+				return 0;
+			tpm_msleep(TPM_TIMEOUT);
+		} while (time_before(jiffies, stop));
+	}
+	return -1;
 }
 
 static int request_locality(struct tpm_chip *chip, int l)
@@ -234,7 +274,7 @@ static int get_burstcount(struct tpm_chip *chip)
 		burstcnt = (value >> 8) & 0xFFFF;
 		if (burstcnt)
 			return burstcnt;
-		tpm_msleep(TPM_POLL_SLEEP);
+		usleep_range(TPM_TIMEOUT_USECS_MIN, TPM_TIMEOUT_USECS_MAX);
 	} while (time_before(jiffies, stop));
 	return -EBUSY;
 }
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index 34968a381d0f..721572a8c429 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -277,6 +277,7 @@ config COMMON_CLK_STM32H7
 	---help---
 	  Support for stm32h7 SoC family clocks
 
+source "drivers/clk/actions/Kconfig"
 source "drivers/clk/bcm/Kconfig"
 source "drivers/clk/hisilicon/Kconfig"
 source "drivers/clk/imgtec/Kconfig"
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index de6d06ac790b..0bb25dd009d1 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -33,6 +33,7 @@ obj-$(CONFIG_CLK_HSDK)			+= clk-hsdk-pll.o
 obj-$(CONFIG_COMMON_CLK_MAX77686)	+= clk-max77686.o
 obj-$(CONFIG_ARCH_MOXART)		+= clk-moxart.o
 obj-$(CONFIG_ARCH_NOMADIK)		+= clk-nomadik.o
+obj-$(CONFIG_ARCH_NPCM7XX)	    	+= clk-npcm7xx.o
 obj-$(CONFIG_ARCH_NSPIRE)		+= clk-nspire.o
 obj-$(CONFIG_COMMON_CLK_OXNAS)		+= clk-oxnas.o
 obj-$(CONFIG_COMMON_CLK_PALMAS)		+= clk-palmas.o
@@ -59,6 +60,7 @@ obj-$(CONFIG_COMMON_CLK_WM831X)		+= clk-wm831x.o
 obj-$(CONFIG_COMMON_CLK_XGENE)		+= clk-xgene.o
 
 # please keep this section sorted lexicographically by directory path name
+obj-y					+= actions/
 obj-$(CONFIG_COMMON_CLK_AT91)		+= at91/
 obj-$(CONFIG_ARCH_ARTPEC)		+= axis/
 obj-$(CONFIG_ARC_PLAT_AXS10X)		+= axs10x/
@@ -94,7 +96,7 @@ obj-$(CONFIG_ARCH_SPRD)			+= sprd/
 obj-$(CONFIG_ARCH_STI)			+= st/
 obj-$(CONFIG_ARCH_STRATIX10)		+= socfpga/
 obj-$(CONFIG_ARCH_SUNXI)		+= sunxi/
-obj-$(CONFIG_ARCH_SUNXI)		+= sunxi-ng/
+obj-$(CONFIG_SUNXI_CCU)			+= sunxi-ng/
 obj-$(CONFIG_ARCH_TEGRA)		+= tegra/
 obj-y					+= ti/
 obj-$(CONFIG_CLK_UNIPHIER)		+= uniphier/
diff --git a/drivers/clk/actions/Kconfig b/drivers/clk/actions/Kconfig
new file mode 100644
index 000000000000..8854adb37847
--- /dev/null
+++ b/drivers/clk/actions/Kconfig
@@ -0,0 +1,14 @@
+config CLK_ACTIONS
+	bool "Clock driver for Actions Semi SoCs"
+	depends on ARCH_ACTIONS || COMPILE_TEST
+	default ARCH_ACTIONS
+
+if CLK_ACTIONS
+
+# SoC Drivers
+
+config CLK_OWL_S900
+	bool "Support for the Actions Semi OWL S900 clocks"
+	depends on (ARM64 && ARCH_ACTIONS) || COMPILE_TEST
+	default ARM64 && ARCH_ACTIONS
+endif
diff --git a/drivers/clk/actions/Makefile b/drivers/clk/actions/Makefile
new file mode 100644
index 000000000000..76e431434d10
--- /dev/null
+++ b/drivers/clk/actions/Makefile
@@ -0,0 +1,12 @@
+obj-$(CONFIG_CLK_ACTIONS)	+= clk-owl.o
+
+clk-owl-y			+= owl-common.o
+clk-owl-y			+= owl-gate.o
+clk-owl-y			+= owl-mux.o
+clk-owl-y			+= owl-divider.o
+clk-owl-y			+= owl-factor.o
+clk-owl-y			+= owl-composite.o
+clk-owl-y			+= owl-pll.o
+
+# SoC support
+obj-$(CONFIG_CLK_OWL_S900)	+= owl-s900.o
diff --git a/drivers/clk/actions/owl-common.c b/drivers/clk/actions/owl-common.c
new file mode 100644
index 000000000000..61c1071b5180
--- /dev/null
+++ b/drivers/clk/actions/owl-common.c
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL common clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include "owl-common.h"
+
+static const struct regmap_config owl_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0x00cc,
+	.fast_io	= true,
+};
+
+static void owl_clk_set_regmap(const struct owl_clk_desc *desc,
+			 struct regmap *regmap)
+{
+	int i;
+	struct owl_clk_common *clks;
+
+	for (i = 0; i < desc->num_clks; i++) {
+		clks = desc->clks[i];
+		if (!clks)
+			continue;
+
+		clks->regmap = regmap;
+	}
+}
+
+int owl_clk_regmap_init(struct platform_device *pdev,
+			 const struct owl_clk_desc *desc)
+{
+	void __iomem *base;
+	struct regmap *regmap;
+	struct resource *res;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	regmap = devm_regmap_init_mmio(&pdev->dev, base, &owl_regmap_config);
+	if (IS_ERR(regmap)) {
+		pr_err("failed to init regmap\n");
+		return PTR_ERR(regmap);
+	}
+
+	owl_clk_set_regmap(desc, regmap);
+
+	return 0;
+}
+
+int owl_clk_probe(struct device *dev, struct clk_hw_onecell_data *hw_clks)
+{
+	int i, ret;
+	struct clk_hw *hw;
+
+	for (i = 0; i < hw_clks->num; i++) {
+
+		hw = hw_clks->hws[i];
+
+		if (IS_ERR_OR_NULL(hw))
+			continue;
+
+		ret = devm_clk_hw_register(dev, hw);
+		if (ret) {
+			dev_err(dev, "Couldn't register clock %d - %s\n",
+				i, hw->init->name);
+			return ret;
+		}
+	}
+
+	ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, hw_clks);
+	if (ret)
+		dev_err(dev, "Failed to add clock provider\n");
+
+	return ret;
+}
diff --git a/drivers/clk/actions/owl-common.h b/drivers/clk/actions/owl-common.h
new file mode 100644
index 000000000000..4fd726ec54a6
--- /dev/null
+++ b/drivers/clk/actions/owl-common.h
@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL common clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#ifndef _OWL_COMMON_H_
+#define _OWL_COMMON_H_
+
+#include <linux/clk-provider.h>
+#include <linux/of_platform.h>
+#include <linux/regmap.h>
+
+struct device_node;
+
+struct owl_clk_common {
+	struct regmap			*regmap;
+	struct clk_hw			hw;
+};
+
+struct owl_clk_desc {
+	struct owl_clk_common		**clks;
+	unsigned long			num_clks;
+	struct clk_hw_onecell_data	*hw_clks;
+};
+
+static inline struct owl_clk_common *
+	hw_to_owl_clk_common(const struct clk_hw *hw)
+{
+	return container_of(hw, struct owl_clk_common, hw);
+}
+
+int owl_clk_regmap_init(struct platform_device *pdev,
+			 const struct owl_clk_desc *desc);
+int owl_clk_probe(struct device *dev, struct clk_hw_onecell_data *hw_clks);
+
+#endif /* _OWL_COMMON_H_ */
diff --git a/drivers/clk/actions/owl-composite.c b/drivers/clk/actions/owl-composite.c
new file mode 100644
index 000000000000..101706e0c66f
--- /dev/null
+++ b/drivers/clk/actions/owl-composite.c
@@ -0,0 +1,199 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL composite clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+
+#include "owl-composite.h"
+
+static u8 owl_comp_get_parent(struct clk_hw *hw)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+
+	return owl_mux_helper_get_parent(&comp->common, &comp->mux_hw);
+}
+
+static int owl_comp_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+
+	return owl_mux_helper_set_parent(&comp->common, &comp->mux_hw, index);
+}
+
+static void owl_comp_disable(struct clk_hw *hw)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+	struct owl_clk_common *common = &comp->common;
+
+	owl_gate_set(common, &comp->gate_hw, false);
+}
+
+static int owl_comp_enable(struct clk_hw *hw)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+	struct owl_clk_common *common = &comp->common;
+
+	owl_gate_set(common, &comp->gate_hw, true);
+
+	return 0;
+}
+
+static int owl_comp_is_enabled(struct clk_hw *hw)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+	struct owl_clk_common *common = &comp->common;
+
+	return owl_gate_clk_is_enabled(common, &comp->gate_hw);
+}
+
+static long owl_comp_div_round_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long *parent_rate)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+
+	return owl_divider_helper_round_rate(&comp->common, &comp->rate.div_hw,
+					rate, parent_rate);
+}
+
+static unsigned long owl_comp_div_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+
+	return owl_divider_helper_recalc_rate(&comp->common, &comp->rate.div_hw,
+					parent_rate);
+}
+
+static int owl_comp_div_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+
+	return owl_divider_helper_set_rate(&comp->common, &comp->rate.div_hw,
+					rate, parent_rate);
+}
+
+static long owl_comp_fact_round_rate(struct clk_hw *hw, unsigned long rate,
+			unsigned long *parent_rate)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+
+	return owl_factor_helper_round_rate(&comp->common,
+					&comp->rate.factor_hw,
+					rate, parent_rate);
+}
+
+static unsigned long owl_comp_fact_recalc_rate(struct clk_hw *hw,
+			unsigned long parent_rate)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+
+	return owl_factor_helper_recalc_rate(&comp->common,
+					&comp->rate.factor_hw,
+					parent_rate);
+}
+
+static int owl_comp_fact_set_rate(struct clk_hw *hw, unsigned long rate,
+			unsigned long parent_rate)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+
+	return owl_factor_helper_set_rate(&comp->common,
+					&comp->rate.factor_hw,
+					rate, parent_rate);
+}
+
+static long owl_comp_fix_fact_round_rate(struct clk_hw *hw, unsigned long rate,
+			unsigned long *parent_rate)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+	struct clk_fixed_factor *fix_fact_hw = &comp->rate.fix_fact_hw;
+
+	return comp->fix_fact_ops->round_rate(&fix_fact_hw->hw, rate, parent_rate);
+}
+
+static unsigned long owl_comp_fix_fact_recalc_rate(struct clk_hw *hw,
+			unsigned long parent_rate)
+{
+	struct owl_composite *comp = hw_to_owl_comp(hw);
+	struct clk_fixed_factor *fix_fact_hw = &comp->rate.fix_fact_hw;
+
+	return comp->fix_fact_ops->recalc_rate(&fix_fact_hw->hw, parent_rate);
+
+}
+
+static int owl_comp_fix_fact_set_rate(struct clk_hw *hw, unsigned long rate,
+			unsigned long parent_rate)
+{
+	/*
+	 * We must report success but we can do so unconditionally because
+	 * owl_comp_fix_fact_round_rate returns values that ensure this call is
+	 * a nop.
+	 */
+
+	return 0;
+}
+
+const struct clk_ops owl_comp_div_ops = {
+	/* mux_ops */
+	.get_parent	= owl_comp_get_parent,
+	.set_parent	= owl_comp_set_parent,
+
+	/* gate_ops */
+	.disable	= owl_comp_disable,
+	.enable		= owl_comp_enable,
+	.is_enabled	= owl_comp_is_enabled,
+
+	/* div_ops */
+	.round_rate	= owl_comp_div_round_rate,
+	.recalc_rate	= owl_comp_div_recalc_rate,
+	.set_rate	= owl_comp_div_set_rate,
+};
+
+
+const struct clk_ops owl_comp_fact_ops = {
+	/* mux_ops */
+	.get_parent	= owl_comp_get_parent,
+	.set_parent	= owl_comp_set_parent,
+
+	/* gate_ops */
+	.disable	= owl_comp_disable,
+	.enable		= owl_comp_enable,
+	.is_enabled	= owl_comp_is_enabled,
+
+	/* fact_ops */
+	.round_rate	= owl_comp_fact_round_rate,
+	.recalc_rate	= owl_comp_fact_recalc_rate,
+	.set_rate	= owl_comp_fact_set_rate,
+};
+
+const struct clk_ops owl_comp_fix_fact_ops = {
+	/* gate_ops */
+	.disable	= owl_comp_disable,
+	.enable		= owl_comp_enable,
+	.is_enabled	= owl_comp_is_enabled,
+
+	/* fix_fact_ops */
+	.round_rate	= owl_comp_fix_fact_round_rate,
+	.recalc_rate	= owl_comp_fix_fact_recalc_rate,
+	.set_rate	= owl_comp_fix_fact_set_rate,
+};
+
+
+const struct clk_ops owl_comp_pass_ops = {
+	/* mux_ops */
+	.get_parent	= owl_comp_get_parent,
+	.set_parent	= owl_comp_set_parent,
+
+	/* gate_ops */
+	.disable	= owl_comp_disable,
+	.enable		= owl_comp_enable,
+	.is_enabled	= owl_comp_is_enabled,
+};
diff --git a/drivers/clk/actions/owl-composite.h b/drivers/clk/actions/owl-composite.h
new file mode 100644
index 000000000000..b410ed5bf308
--- /dev/null
+++ b/drivers/clk/actions/owl-composite.h
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL composite clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#ifndef _OWL_COMPOSITE_H_
+#define _OWL_COMPOSITE_H_
+
+#include "owl-common.h"
+#include "owl-mux.h"
+#include "owl-gate.h"
+#include "owl-factor.h"
+#include "owl-fixed-factor.h"
+#include "owl-divider.h"
+
+union owl_rate {
+	struct owl_divider_hw	div_hw;
+	struct owl_factor_hw	factor_hw;
+	struct clk_fixed_factor	fix_fact_hw;
+};
+
+struct owl_composite {
+	struct owl_mux_hw	mux_hw;
+	struct owl_gate_hw	gate_hw;
+	union owl_rate		rate;
+
+	const struct clk_ops	*fix_fact_ops;
+
+	struct owl_clk_common	common;
+};
+
+#define OWL_COMP_DIV(_struct, _name, _parent,				\
+		     _mux, _gate, _div, _flags)				\
+	struct owl_composite _struct = {				\
+		.mux_hw		= _mux,					\
+		.gate_hw	= _gate,				\
+		.rate.div_hw	= _div,					\
+		.common = {						\
+			.regmap		= NULL,				\
+			.hw.init	= CLK_HW_INIT_PARENTS(_name,	\
+						     _parent,		\
+						      &owl_comp_div_ops,\
+						     _flags),		\
+		},							\
+	}
+
+#define OWL_COMP_DIV_FIXED(_struct, _name, _parent,			\
+		     _gate, _div, _flags)				\
+	struct owl_composite _struct = {				\
+		.gate_hw	= _gate,				\
+		.rate.div_hw	= _div,					\
+		.common = {						\
+			.regmap		= NULL,				\
+			.hw.init	= CLK_HW_INIT(_name,		\
+						     _parent,		\
+						      &owl_comp_div_ops,\
+						     _flags),		\
+		},							\
+	}
+
+#define OWL_COMP_FACTOR(_struct, _name, _parent,			\
+			_mux, _gate, _factor, _flags)			\
+	struct owl_composite _struct = {				\
+		.mux_hw		= _mux,					\
+		.gate_hw	= _gate,				\
+		.rate.factor_hw	= _factor,				\
+		.common = {						\
+			.regmap		= NULL,				\
+			.hw.init	= CLK_HW_INIT_PARENTS(_name,	\
+						     _parent,		\
+						     &owl_comp_fact_ops,\
+						     _flags),		\
+		},							\
+	}
+
+#define OWL_COMP_FIXED_FACTOR(_struct, _name, _parent,			\
+			_gate, _mul, _div, _flags)			\
+	struct owl_composite _struct = {				\
+		.gate_hw		= _gate,			\
+		.rate.fix_fact_hw.mult	= _mul,				\
+		.rate.fix_fact_hw.div	= _div,				\
+		.fix_fact_ops		= &clk_fixed_factor_ops,	\
+		.common = {						\
+			.regmap		= NULL,				\
+			.hw.init	= CLK_HW_INIT(_name,		\
+						 _parent,		\
+						 &owl_comp_fix_fact_ops,\
+						 _flags),		\
+		},							\
+	}
+
+#define OWL_COMP_PASS(_struct, _name, _parent,				\
+		      _mux, _gate, _flags)				\
+	struct owl_composite _struct = {				\
+		.mux_hw		= _mux,					\
+		.gate_hw	= _gate,				\
+		.common = {						\
+			.regmap		= NULL,				\
+			.hw.init	= CLK_HW_INIT_PARENTS(_name,	\
+						     _parent,		\
+						     &owl_comp_pass_ops,\
+						     _flags),		\
+		},							\
+	}
+
+static inline struct owl_composite *hw_to_owl_comp(const struct clk_hw *hw)
+{
+	struct owl_clk_common *common = hw_to_owl_clk_common(hw);
+
+	return container_of(common, struct owl_composite, common);
+}
+
+extern const struct clk_ops owl_comp_div_ops;
+extern const struct clk_ops owl_comp_fact_ops;
+extern const struct clk_ops owl_comp_fix_fact_ops;
+extern const struct clk_ops owl_comp_pass_ops;
+extern const struct clk_ops clk_fixed_factor_ops;
+
+#endif /* _OWL_COMPOSITE_H_ */
diff --git a/drivers/clk/actions/owl-divider.c b/drivers/clk/actions/owl-divider.c
new file mode 100644
index 000000000000..cddac00fe324
--- /dev/null
+++ b/drivers/clk/actions/owl-divider.c
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL divider clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+
+#include "owl-divider.h"
+
+long owl_divider_helper_round_rate(struct owl_clk_common *common,
+				const struct owl_divider_hw *div_hw,
+				unsigned long rate,
+				unsigned long *parent_rate)
+{
+	return divider_round_rate(&common->hw, rate, parent_rate,
+				  div_hw->table, div_hw->width,
+				  div_hw->div_flags);
+}
+
+static long owl_divider_round_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long *parent_rate)
+{
+	struct owl_divider *div = hw_to_owl_divider(hw);
+
+	return owl_divider_helper_round_rate(&div->common, &div->div_hw,
+					     rate, parent_rate);
+}
+
+unsigned long owl_divider_helper_recalc_rate(struct owl_clk_common *common,
+					 const struct owl_divider_hw *div_hw,
+					 unsigned long parent_rate)
+{
+	unsigned long val;
+	unsigned int reg;
+
+	regmap_read(common->regmap, div_hw->reg, &reg);
+	val = reg >> div_hw->shift;
+	val &= (1 << div_hw->width) - 1;
+
+	return divider_recalc_rate(&common->hw, parent_rate,
+				   val, div_hw->table,
+				   div_hw->div_flags,
+				   div_hw->width);
+}
+
+static unsigned long owl_divider_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	struct owl_divider *div = hw_to_owl_divider(hw);
+
+	return owl_divider_helper_recalc_rate(&div->common,
+					      &div->div_hw, parent_rate);
+}
+
+int owl_divider_helper_set_rate(const struct owl_clk_common *common,
+				const struct owl_divider_hw *div_hw,
+				unsigned long rate,
+				unsigned long parent_rate)
+{
+	unsigned long val;
+	unsigned int reg;
+
+	val = divider_get_val(rate, parent_rate, div_hw->table,
+			      div_hw->width, 0);
+
+	regmap_read(common->regmap, div_hw->reg, &reg);
+	reg &= ~GENMASK(div_hw->width + div_hw->shift - 1, div_hw->shift);
+
+	regmap_write(common->regmap, div_hw->reg,
+			  reg | (val << div_hw->shift));
+
+	return 0;
+}
+
+static int owl_divider_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct owl_divider *div = hw_to_owl_divider(hw);
+
+	return owl_divider_helper_set_rate(&div->common, &div->div_hw,
+					rate, parent_rate);
+}
+
+const struct clk_ops owl_divider_ops = {
+	.recalc_rate = owl_divider_recalc_rate,
+	.round_rate = owl_divider_round_rate,
+	.set_rate = owl_divider_set_rate,
+};
diff --git a/drivers/clk/actions/owl-divider.h b/drivers/clk/actions/owl-divider.h
new file mode 100644
index 000000000000..92d3e3d23967
--- /dev/null
+++ b/drivers/clk/actions/owl-divider.h
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL divider clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#ifndef _OWL_DIVIDER_H_
+#define _OWL_DIVIDER_H_
+
+#include "owl-common.h"
+
+struct owl_divider_hw {
+	u32			reg;
+	u8			shift;
+	u8			width;
+	u8			div_flags;
+	struct clk_div_table	*table;
+};
+
+struct owl_divider {
+	struct owl_divider_hw	div_hw;
+	struct owl_clk_common	common;
+};
+
+#define OWL_DIVIDER_HW(_reg, _shift, _width, _div_flags, _table)	\
+	{								\
+		.reg		= _reg,					\
+		.shift		= _shift,				\
+		.width		= _width,				\
+		.div_flags	= _div_flags,				\
+		.table		= _table,				\
+	}
+
+#define OWL_DIVIDER(_struct, _name, _parent, _reg,			\
+		    _shift, _width, _table, _div_flags, _flags)		\
+	struct owl_divider _struct = {					\
+		.div_hw	= OWL_DIVIDER_HW(_reg, _shift, _width,		\
+					 _div_flags, _table),		\
+		.common = {						\
+			.regmap		= NULL,				\
+			.hw.init	= CLK_HW_INIT(_name,		\
+						      _parent,		\
+						      &owl_divider_ops,	\
+						      _flags),		\
+		},							\
+	}
+
+static inline struct owl_divider *hw_to_owl_divider(const struct clk_hw *hw)
+{
+	struct owl_clk_common *common = hw_to_owl_clk_common(hw);
+
+	return container_of(common, struct owl_divider, common);
+}
+
+long owl_divider_helper_round_rate(struct owl_clk_common *common,
+				const struct owl_divider_hw *div_hw,
+				unsigned long rate,
+				unsigned long *parent_rate);
+
+unsigned long owl_divider_helper_recalc_rate(struct owl_clk_common *common,
+					 const struct owl_divider_hw *div_hw,
+					 unsigned long parent_rate);
+
+int owl_divider_helper_set_rate(const struct owl_clk_common *common,
+				const struct owl_divider_hw *div_hw,
+				unsigned long rate,
+				unsigned long parent_rate);
+
+extern const struct clk_ops owl_divider_ops;
+
+#endif /* _OWL_DIVIDER_H_ */
diff --git a/drivers/clk/actions/owl-factor.c b/drivers/clk/actions/owl-factor.c
new file mode 100644
index 000000000000..317d4a9e112e
--- /dev/null
+++ b/drivers/clk/actions/owl-factor.c
@@ -0,0 +1,222 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL factor clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#include "owl-factor.h"
+
+static unsigned int _get_table_maxval(const struct clk_factor_table *table)
+{
+	unsigned int maxval = 0;
+	const struct clk_factor_table *clkt;
+
+	for (clkt = table; clkt->div; clkt++)
+		if (clkt->val > maxval)
+			maxval = clkt->val;
+	return maxval;
+}
+
+static int _get_table_div_mul(const struct clk_factor_table *table,
+			unsigned int val, unsigned int *mul, unsigned int *div)
+{
+	const struct clk_factor_table *clkt;
+
+	for (clkt = table; clkt->div; clkt++) {
+		if (clkt->val == val) {
+			*mul = clkt->mul;
+			*div = clkt->div;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static unsigned int _get_table_val(const struct clk_factor_table *table,
+			unsigned long rate, unsigned long parent_rate)
+{
+	const struct clk_factor_table *clkt;
+	int val = -1;
+	u64 calc_rate;
+
+	for (clkt = table; clkt->div; clkt++) {
+		calc_rate = parent_rate * clkt->mul;
+		do_div(calc_rate, clkt->div);
+
+		if ((unsigned long)calc_rate <= rate) {
+			val = clkt->val;
+			break;
+		}
+	}
+
+	if (val == -1)
+		val = _get_table_maxval(table);
+
+	return val;
+}
+
+static int clk_val_best(struct clk_hw *hw, unsigned long rate,
+			unsigned long *best_parent_rate)
+{
+	struct owl_factor *factor = hw_to_owl_factor(hw);
+	struct owl_factor_hw *factor_hw = &factor->factor_hw;
+	const struct clk_factor_table *clkt = factor_hw->table;
+	unsigned long parent_rate, try_parent_rate, best = 0, cur_rate;
+	unsigned long parent_rate_saved = *best_parent_rate;
+	int bestval = 0;
+
+	if (!rate)
+		rate = 1;
+
+	if (!(clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT)) {
+		parent_rate = *best_parent_rate;
+		bestval = _get_table_val(clkt, rate, parent_rate);
+		return bestval;
+	}
+
+	for (clkt = factor_hw->table; clkt->div; clkt++) {
+		try_parent_rate = rate * clkt->div / clkt->mul;
+
+		if (try_parent_rate == parent_rate_saved) {
+			pr_debug("%s: [%d %d %d] found try_parent_rate %ld\n",
+				__func__, clkt->val, clkt->mul, clkt->div,
+				try_parent_rate);
+			/*
+			 * It's the most ideal case if the requested rate can be
+			 * divided from parent clock without any need to change
+			 * parent rate, so return the divider immediately.
+			 */
+			*best_parent_rate = parent_rate_saved;
+			return clkt->val;
+		}
+
+		parent_rate = clk_hw_round_rate(clk_hw_get_parent(hw),
+				try_parent_rate);
+		cur_rate = DIV_ROUND_UP(parent_rate, clkt->div) * clkt->mul;
+		if (cur_rate <= rate && cur_rate > best) {
+			bestval = clkt->val;
+			best = cur_rate;
+			*best_parent_rate = parent_rate;
+		}
+	}
+
+	if (!bestval) {
+		bestval = _get_table_maxval(clkt);
+		*best_parent_rate = clk_hw_round_rate(
+				clk_hw_get_parent(hw), 1);
+	}
+
+	return bestval;
+}
+
+long owl_factor_helper_round_rate(struct owl_clk_common *common,
+				const struct owl_factor_hw *factor_hw,
+				unsigned long rate,
+				unsigned long *parent_rate)
+{
+	const struct clk_factor_table *clkt = factor_hw->table;
+	unsigned int val, mul = 0, div = 1;
+
+	val = clk_val_best(&common->hw, rate, parent_rate);
+	_get_table_div_mul(clkt, val, &mul, &div);
+
+	return *parent_rate * mul / div;
+}
+
+static long owl_factor_round_rate(struct clk_hw *hw, unsigned long rate,
+			unsigned long *parent_rate)
+{
+	struct owl_factor *factor = hw_to_owl_factor(hw);
+	struct owl_factor_hw *factor_hw = &factor->factor_hw;
+
+	return owl_factor_helper_round_rate(&factor->common, factor_hw,
+					rate, parent_rate);
+}
+
+unsigned long owl_factor_helper_recalc_rate(struct owl_clk_common *common,
+					 const struct owl_factor_hw *factor_hw,
+					 unsigned long parent_rate)
+{
+	const struct clk_factor_table *clkt = factor_hw->table;
+	unsigned long long int rate;
+	u32 reg, val, mul, div;
+
+	div = 0;
+	mul = 0;
+
+	regmap_read(common->regmap, factor_hw->reg, &reg);
+
+	val = reg >> factor_hw->shift;
+	val &= div_mask(factor_hw);
+
+	_get_table_div_mul(clkt, val, &mul, &div);
+	if (!div) {
+		WARN(!(factor_hw->fct_flags & CLK_DIVIDER_ALLOW_ZERO),
+			"%s: Zero divisor and CLK_DIVIDER_ALLOW_ZERO not set\n",
+			__clk_get_name(common->hw.clk));
+		return parent_rate;
+	}
+
+	rate = (unsigned long long int)parent_rate * mul;
+	do_div(rate, div);
+
+	return rate;
+}
+
+static unsigned long owl_factor_recalc_rate(struct clk_hw *hw,
+			unsigned long parent_rate)
+{
+	struct owl_factor *factor = hw_to_owl_factor(hw);
+	struct owl_factor_hw *factor_hw = &factor->factor_hw;
+	struct owl_clk_common *common = &factor->common;
+
+	return owl_factor_helper_recalc_rate(common, factor_hw, parent_rate);
+}
+
+int owl_factor_helper_set_rate(const struct owl_clk_common *common,
+				const struct owl_factor_hw *factor_hw,
+				unsigned long rate,
+				unsigned long parent_rate)
+{
+	u32 val, reg;
+
+	val = _get_table_val(factor_hw->table, rate, parent_rate);
+
+	if (val > div_mask(factor_hw))
+		val = div_mask(factor_hw);
+
+	regmap_read(common->regmap, factor_hw->reg, &reg);
+
+	reg &= ~(div_mask(factor_hw) << factor_hw->shift);
+	reg |= val << factor_hw->shift;
+
+	regmap_write(common->regmap, factor_hw->reg, reg);
+
+	return 0;
+}
+
+static int owl_factor_set_rate(struct clk_hw *hw, unsigned long rate,
+			       unsigned long parent_rate)
+{
+	struct owl_factor *factor = hw_to_owl_factor(hw);
+	struct owl_factor_hw *factor_hw = &factor->factor_hw;
+	struct owl_clk_common *common = &factor->common;
+
+	return owl_factor_helper_set_rate(common, factor_hw,
+					rate, parent_rate);
+}
+
+const struct clk_ops owl_factor_ops = {
+	.round_rate	= owl_factor_round_rate,
+	.recalc_rate	= owl_factor_recalc_rate,
+	.set_rate	= owl_factor_set_rate,
+};
diff --git a/drivers/clk/actions/owl-factor.h b/drivers/clk/actions/owl-factor.h
new file mode 100644
index 000000000000..f1a7ffe896e1
--- /dev/null
+++ b/drivers/clk/actions/owl-factor.h
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL factor clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#ifndef _OWL_FACTOR_H_
+#define _OWL_FACTOR_H_
+
+#include "owl-common.h"
+
+struct clk_factor_table {
+	unsigned int		val;
+	unsigned int		mul;
+	unsigned int		div;
+};
+
+struct owl_factor_hw {
+	u32			reg;
+	u8			shift;
+	u8			width;
+	u8			fct_flags;
+	struct clk_factor_table	*table;
+};
+
+struct owl_factor {
+	struct owl_factor_hw	factor_hw;
+	struct owl_clk_common	common;
+};
+
+#define OWL_FACTOR_HW(_reg, _shift, _width, _fct_flags, _table)		\
+	{								\
+		.reg		= _reg,					\
+		.shift		= _shift,				\
+		.width		= _width,				\
+		.fct_flags	= _fct_flags,				\
+		.table		= _table,				\
+	}
+
+#define OWL_FACTOR(_struct, _name, _parent, _reg,			\
+		   _shift, _width, _table, _fct_flags, _flags)		\
+	struct owl_factor _struct = {					\
+		.factor_hw = OWL_FACTOR_HW(_reg, _shift,		\
+					   _width, _fct_flags, _table),	\
+		.common = {						\
+			.regmap		= NULL,				\
+			.hw.init	= CLK_HW_INIT(_name,		\
+						      _parent,		\
+						      &owl_factor_ops,	\
+						      _flags),		\
+		},							\
+	}
+
+#define div_mask(d) ((1 << ((d)->width)) - 1)
+
+static inline struct owl_factor *hw_to_owl_factor(const struct clk_hw *hw)
+{
+	struct owl_clk_common *common = hw_to_owl_clk_common(hw);
+
+	return container_of(common, struct owl_factor, common);
+}
+
+long owl_factor_helper_round_rate(struct owl_clk_common *common,
+				const struct owl_factor_hw *factor_hw,
+				unsigned long rate,
+				unsigned long *parent_rate);
+
+unsigned long owl_factor_helper_recalc_rate(struct owl_clk_common *common,
+					 const struct owl_factor_hw *factor_hw,
+					 unsigned long parent_rate);
+
+int owl_factor_helper_set_rate(const struct owl_clk_common *common,
+				const struct owl_factor_hw *factor_hw,
+				unsigned long rate,
+				unsigned long parent_rate);
+
+extern const struct clk_ops owl_factor_ops;
+
+#endif /* _OWL_FACTOR_H_ */
diff --git a/drivers/clk/actions/owl-fixed-factor.h b/drivers/clk/actions/owl-fixed-factor.h
new file mode 100644
index 000000000000..cc9fe36c0964
--- /dev/null
+++ b/drivers/clk/actions/owl-fixed-factor.h
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL fixed factor clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#ifndef _OWL_FIXED_FACTOR_H_
+#define _OWL_FIXED_FACTOR_H_
+
+#include "owl-common.h"
+
+#define OWL_FIX_FACT(_struct, _name, _parent, _mul, _div, _flags)	\
+	struct clk_fixed_factor _struct = {				\
+		.mult		= _mul,					\
+		.div		= _div,					\
+		.hw.init	= CLK_HW_INIT(_name,			\
+					      _parent,			\
+					      &clk_fixed_factor_ops,	\
+					      _flags),			\
+	}
+
+extern const struct clk_ops clk_fixed_factor_ops;
+
+#endif /* _OWL_FIXED_FACTOR_H_ */
diff --git a/drivers/clk/actions/owl-gate.c b/drivers/clk/actions/owl-gate.c
new file mode 100644
index 000000000000..f11500ba46a7
--- /dev/null
+++ b/drivers/clk/actions/owl-gate.c
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL gate clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+
+#include "owl-gate.h"
+
+void owl_gate_set(const struct owl_clk_common *common,
+		 const struct owl_gate_hw *gate_hw, bool enable)
+{
+	int set = gate_hw->gate_flags & CLK_GATE_SET_TO_DISABLE ? 1 : 0;
+	u32 reg;
+
+	set ^= enable;
+
+	regmap_read(common->regmap, gate_hw->reg, &reg);
+
+	if (set)
+		reg |= BIT(gate_hw->bit_idx);
+	else
+		reg &= ~BIT(gate_hw->bit_idx);
+
+	regmap_write(common->regmap, gate_hw->reg, reg);
+}
+
+static void owl_gate_disable(struct clk_hw *hw)
+{
+	struct owl_gate *gate = hw_to_owl_gate(hw);
+	struct owl_clk_common *common = &gate->common;
+
+	owl_gate_set(common, &gate->gate_hw, false);
+}
+
+static int owl_gate_enable(struct clk_hw *hw)
+{
+	struct owl_gate *gate = hw_to_owl_gate(hw);
+	struct owl_clk_common *common = &gate->common;
+
+	owl_gate_set(common, &gate->gate_hw, true);
+
+	return 0;
+}
+
+int owl_gate_clk_is_enabled(const struct owl_clk_common *common,
+		   const struct owl_gate_hw *gate_hw)
+{
+	u32 reg;
+
+	regmap_read(common->regmap, gate_hw->reg, &reg);
+
+	if (gate_hw->gate_flags & CLK_GATE_SET_TO_DISABLE)
+		reg ^= BIT(gate_hw->bit_idx);
+
+	return !!(reg & BIT(gate_hw->bit_idx));
+}
+
+static int owl_gate_is_enabled(struct clk_hw *hw)
+{
+	struct owl_gate *gate = hw_to_owl_gate(hw);
+	struct owl_clk_common *common = &gate->common;
+
+	return owl_gate_clk_is_enabled(common, &gate->gate_hw);
+}
+
+const struct clk_ops owl_gate_ops = {
+	.disable	= owl_gate_disable,
+	.enable		= owl_gate_enable,
+	.is_enabled	= owl_gate_is_enabled,
+};
diff --git a/drivers/clk/actions/owl-gate.h b/drivers/clk/actions/owl-gate.h
new file mode 100644
index 000000000000..c2d61ceebce2
--- /dev/null
+++ b/drivers/clk/actions/owl-gate.h
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL gate clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#ifndef _OWL_GATE_H_
+#define _OWL_GATE_H_
+
+#include "owl-common.h"
+
+struct owl_gate_hw {
+	u32			reg;
+	u8			bit_idx;
+	u8			gate_flags;
+};
+
+struct owl_gate {
+	struct owl_gate_hw	gate_hw;
+	struct owl_clk_common	common;
+};
+
+#define OWL_GATE_HW(_reg, _bit_idx, _gate_flags)	\
+	{						\
+		.reg		= _reg,			\
+		.bit_idx	= _bit_idx,		\
+		.gate_flags	= _gate_flags,		\
+	}
+
+#define OWL_GATE(_struct, _name, _parent, _reg,				\
+		_bit_idx, _gate_flags, _flags)				\
+	struct owl_gate _struct = {					\
+		.gate_hw = OWL_GATE_HW(_reg, _bit_idx, _gate_flags),	\
+		.common = {						\
+			.regmap		= NULL,				\
+			.hw.init	= CLK_HW_INIT(_name,		\
+						      _parent,		\
+						      &owl_gate_ops,	\
+						      _flags),		\
+		}							\
+	}								\
+
+#define OWL_GATE_NO_PARENT(_struct, _name, _reg,			\
+		_bit_idx, _gate_flags, _flags)				\
+	struct owl_gate _struct = {					\
+		.gate_hw = OWL_GATE_HW(_reg, _bit_idx, _gate_flags),	\
+		.common = {						\
+			.regmap		= NULL,				\
+			.hw.init	= CLK_HW_INIT_NO_PARENT(_name,	\
+						      &owl_gate_ops,	\
+						      _flags),		\
+		},							\
+	}								\
+
+static inline struct owl_gate *hw_to_owl_gate(const struct clk_hw *hw)
+{
+	struct owl_clk_common *common = hw_to_owl_clk_common(hw);
+
+	return container_of(common, struct owl_gate, common);
+}
+
+void owl_gate_set(const struct owl_clk_common *common,
+		 const struct owl_gate_hw *gate_hw, bool enable);
+int owl_gate_clk_is_enabled(const struct owl_clk_common *common,
+		   const struct owl_gate_hw *gate_hw);
+
+extern const struct clk_ops owl_gate_ops;
+
+#endif /* _OWL_GATE_H_ */
diff --git a/drivers/clk/actions/owl-mux.c b/drivers/clk/actions/owl-mux.c
new file mode 100644
index 000000000000..f9c6cf2540e4
--- /dev/null
+++ b/drivers/clk/actions/owl-mux.c
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL mux clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+
+#include "owl-mux.h"
+
+u8 owl_mux_helper_get_parent(const struct owl_clk_common *common,
+			     const struct owl_mux_hw *mux_hw)
+{
+	u32 reg;
+	u8 parent;
+
+	regmap_read(common->regmap, mux_hw->reg, &reg);
+	parent = reg >> mux_hw->shift;
+	parent &= BIT(mux_hw->width) - 1;
+
+	return parent;
+}
+
+static u8 owl_mux_get_parent(struct clk_hw *hw)
+{
+	struct owl_mux *mux = hw_to_owl_mux(hw);
+
+	return owl_mux_helper_get_parent(&mux->common, &mux->mux_hw);
+}
+
+int owl_mux_helper_set_parent(const struct owl_clk_common *common,
+			      struct owl_mux_hw *mux_hw, u8 index)
+{
+	u32 reg;
+
+	regmap_read(common->regmap, mux_hw->reg, &reg);
+	reg &= ~GENMASK(mux_hw->width + mux_hw->shift - 1, mux_hw->shift);
+	regmap_write(common->regmap, mux_hw->reg,
+			reg | (index << mux_hw->shift));
+
+	return 0;
+}
+
+static int owl_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct owl_mux *mux = hw_to_owl_mux(hw);
+
+	return owl_mux_helper_set_parent(&mux->common, &mux->mux_hw, index);
+}
+
+const struct clk_ops owl_mux_ops = {
+	.get_parent = owl_mux_get_parent,
+	.set_parent = owl_mux_set_parent,
+	.determine_rate = __clk_mux_determine_rate,
+};
diff --git a/drivers/clk/actions/owl-mux.h b/drivers/clk/actions/owl-mux.h
new file mode 100644
index 000000000000..834284c8c3ae
--- /dev/null
+++ b/drivers/clk/actions/owl-mux.h
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL mux clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#ifndef _OWL_MUX_H_
+#define _OWL_MUX_H_
+
+#include "owl-common.h"
+
+struct owl_mux_hw {
+	u32			reg;
+	u8			shift;
+	u8			width;
+};
+
+struct owl_mux {
+	struct owl_mux_hw	mux_hw;
+	struct owl_clk_common	common;
+};
+
+#define OWL_MUX_HW(_reg, _shift, _width)		\
+	{						\
+		.reg	= _reg,				\
+		.shift	= _shift,			\
+		.width	= _width,			\
+	}
+
+#define OWL_MUX(_struct, _name, _parents, _reg,				\
+		_shift, _width, _flags)					\
+	struct owl_mux _struct = {					\
+		.mux_hw	= OWL_MUX_HW(_reg, _shift, _width),		\
+		.common = {						\
+			.regmap = NULL,					\
+			.hw.init = CLK_HW_INIT_PARENTS(_name,		\
+						       _parents,	\
+						       &owl_mux_ops,	\
+						       _flags),		\
+		},							\
+	}
+
+static inline struct owl_mux *hw_to_owl_mux(const struct clk_hw *hw)
+{
+	struct owl_clk_common *common = hw_to_owl_clk_common(hw);
+
+	return container_of(common, struct owl_mux, common);
+}
+
+u8 owl_mux_helper_get_parent(const struct owl_clk_common *common,
+			     const struct owl_mux_hw *mux_hw);
+int owl_mux_helper_set_parent(const struct owl_clk_common *common,
+			      struct owl_mux_hw *mux_hw, u8 index);
+
+extern const struct clk_ops owl_mux_ops;
+
+#endif /* _OWL_MUX_H_ */
diff --git a/drivers/clk/actions/owl-pll.c b/drivers/clk/actions/owl-pll.c
new file mode 100644
index 000000000000..058e06d7099f
--- /dev/null
+++ b/drivers/clk/actions/owl-pll.c
@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL pll clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#include <linux/clk-provider.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+
+#include "owl-pll.h"
+
+static u32 owl_pll_calculate_mul(struct owl_pll_hw *pll_hw, unsigned long rate)
+{
+	u32 mul;
+
+	mul = DIV_ROUND_CLOSEST(rate, pll_hw->bfreq);
+	if (mul < pll_hw->min_mul)
+		mul = pll_hw->min_mul;
+	else if (mul > pll_hw->max_mul)
+		mul = pll_hw->max_mul;
+
+	return mul &= mul_mask(pll_hw);
+}
+
+static unsigned long _get_table_rate(const struct clk_pll_table *table,
+		unsigned int val)
+{
+	const struct clk_pll_table *clkt;
+
+	for (clkt = table; clkt->rate; clkt++)
+		if (clkt->val == val)
+			return clkt->rate;
+
+	return 0;
+}
+
+static const struct clk_pll_table *_get_pll_table(
+		const struct clk_pll_table *table, unsigned long rate)
+{
+	const struct clk_pll_table *clkt;
+
+	for (clkt = table; clkt->rate; clkt++) {
+		if (clkt->rate == rate) {
+			table = clkt;
+			break;
+		} else if (clkt->rate < rate)
+			table = clkt;
+	}
+
+	return table;
+}
+
+static long owl_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+		unsigned long *parent_rate)
+{
+	struct owl_pll *pll = hw_to_owl_pll(hw);
+	struct owl_pll_hw *pll_hw = &pll->pll_hw;
+	const struct clk_pll_table *clkt;
+	u32 mul;
+
+	if (pll_hw->table) {
+		clkt = _get_pll_table(pll_hw->table, rate);
+		return clkt->rate;
+	}
+
+	/* fixed frequency */
+	if (pll_hw->width == 0)
+		return pll_hw->bfreq;
+
+	mul = owl_pll_calculate_mul(pll_hw, rate);
+
+	return pll_hw->bfreq * mul;
+}
+
+static unsigned long owl_pll_recalc_rate(struct clk_hw *hw,
+		unsigned long parent_rate)
+{
+	struct owl_pll *pll = hw_to_owl_pll(hw);
+	struct owl_pll_hw *pll_hw = &pll->pll_hw;
+	const struct owl_clk_common *common = &pll->common;
+	u32 val;
+
+	if (pll_hw->table) {
+		regmap_read(common->regmap, pll_hw->reg, &val);
+
+		val = val >> pll_hw->shift;
+		val &= mul_mask(pll_hw);
+
+		return _get_table_rate(pll_hw->table, val);
+	}
+
+	/* fixed frequency */
+	if (pll_hw->width == 0)
+		return pll_hw->bfreq;
+
+	regmap_read(common->regmap, pll_hw->reg, &val);
+
+	val = val >> pll_hw->shift;
+	val &= mul_mask(pll_hw);
+
+	return pll_hw->bfreq * val;
+}
+
+static int owl_pll_is_enabled(struct clk_hw *hw)
+{
+	struct owl_pll *pll = hw_to_owl_pll(hw);
+	struct owl_pll_hw *pll_hw = &pll->pll_hw;
+	const struct owl_clk_common *common = &pll->common;
+	u32 reg;
+
+	regmap_read(common->regmap, pll_hw->reg, &reg);
+
+	return !!(reg & BIT(pll_hw->bit_idx));
+}
+
+static void owl_pll_set(const struct owl_clk_common *common,
+		       const struct owl_pll_hw *pll_hw, bool enable)
+{
+	u32 reg;
+
+	regmap_read(common->regmap, pll_hw->reg, &reg);
+
+	if (enable)
+		reg |= BIT(pll_hw->bit_idx);
+	else
+		reg &= ~BIT(pll_hw->bit_idx);
+
+	regmap_write(common->regmap, pll_hw->reg, reg);
+}
+
+static int owl_pll_enable(struct clk_hw *hw)
+{
+	struct owl_pll *pll = hw_to_owl_pll(hw);
+	const struct owl_clk_common *common = &pll->common;
+
+	owl_pll_set(common, &pll->pll_hw, true);
+
+	return 0;
+}
+
+static void owl_pll_disable(struct clk_hw *hw)
+{
+	struct owl_pll *pll = hw_to_owl_pll(hw);
+	const struct owl_clk_common *common = &pll->common;
+
+	owl_pll_set(common, &pll->pll_hw, false);
+}
+
+static int owl_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+		unsigned long parent_rate)
+{
+	struct owl_pll *pll = hw_to_owl_pll(hw);
+	struct owl_pll_hw *pll_hw = &pll->pll_hw;
+	const struct owl_clk_common *common = &pll->common;
+	const struct clk_pll_table *clkt;
+	u32 val, reg;
+
+	/* fixed frequency */
+	if (pll_hw->width == 0)
+		return 0;
+
+	if (pll_hw->table) {
+		clkt = _get_pll_table(pll_hw->table, rate);
+		val = clkt->val;
+	} else {
+		val = owl_pll_calculate_mul(pll_hw, rate);
+	}
+
+	regmap_read(common->regmap, pll_hw->reg, &reg);
+
+	reg &= ~mul_mask(pll_hw);
+	reg |= val << pll_hw->shift;
+
+	regmap_write(common->regmap, pll_hw->reg, reg);
+
+	udelay(PLL_STABILITY_WAIT_US);
+
+	return 0;
+}
+
+const struct clk_ops owl_pll_ops = {
+	.enable = owl_pll_enable,
+	.disable = owl_pll_disable,
+	.is_enabled = owl_pll_is_enabled,
+	.round_rate = owl_pll_round_rate,
+	.recalc_rate = owl_pll_recalc_rate,
+	.set_rate = owl_pll_set_rate,
+};
diff --git a/drivers/clk/actions/owl-pll.h b/drivers/clk/actions/owl-pll.h
new file mode 100644
index 000000000000..0aae30abd5dc
--- /dev/null
+++ b/drivers/clk/actions/owl-pll.h
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL pll clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#ifndef _OWL_PLL_H_
+#define _OWL_PLL_H_
+
+#include "owl-common.h"
+
+/* last entry should have rate = 0 */
+struct clk_pll_table {
+	unsigned int		val;
+	unsigned long		rate;
+};
+
+struct owl_pll_hw {
+	u32			reg;
+	u32			bfreq;
+	u8			bit_idx;
+	u8			shift;
+	u8			width;
+	u8			min_mul;
+	u8			max_mul;
+	const struct clk_pll_table *table;
+};
+
+struct owl_pll {
+	struct owl_pll_hw	pll_hw;
+	struct owl_clk_common	common;
+};
+
+#define OWL_PLL_HW(_reg, _bfreq, _bit_idx, _shift,			\
+		   _width, _min_mul, _max_mul, _table)			\
+	{								\
+		.reg		= _reg,					\
+		.bfreq		= _bfreq,				\
+		.bit_idx	= _bit_idx,				\
+		.shift		= _shift,				\
+		.width		= _width,				\
+		.min_mul	= _min_mul,				\
+		.max_mul	= _max_mul,				\
+		.table		= _table,				\
+	}
+
+#define OWL_PLL(_struct, _name, _parent, _reg, _bfreq, _bit_idx,	\
+		_shift, _width, _min_mul, _max_mul, _table, _flags)	\
+	struct owl_pll _struct = {					\
+		.pll_hw	= OWL_PLL_HW(_reg, _bfreq, _bit_idx, _shift,	\
+				     _width, _min_mul,			\
+				     _max_mul, _table),			\
+		.common = {						\
+			.regmap = NULL,					\
+			.hw.init = CLK_HW_INIT(_name,			\
+					       _parent,			\
+					       &owl_pll_ops,		\
+					       _flags),			\
+		},							\
+	}
+
+#define OWL_PLL_NO_PARENT(_struct, _name, _reg, _bfreq, _bit_idx,	\
+		_shift, _width, _min_mul, _max_mul, _table, _flags)	\
+	struct owl_pll _struct = {					\
+		.pll_hw	= OWL_PLL_HW(_reg, _bfreq, _bit_idx, _shift,	\
+				     _width, _min_mul,			\
+				     _max_mul, _table),			\
+		.common = {						\
+			.regmap = NULL,					\
+			.hw.init = CLK_HW_INIT_NO_PARENT(_name,		\
+					       &owl_pll_ops,		\
+					       _flags),			\
+		},							\
+	}
+
+#define mul_mask(m)		((1 << ((m)->width)) - 1)
+#define PLL_STABILITY_WAIT_US	(50)
+
+static inline struct owl_pll *hw_to_owl_pll(const struct clk_hw *hw)
+{
+	struct owl_clk_common *common = hw_to_owl_clk_common(hw);
+
+	return container_of(common, struct owl_pll, common);
+}
+
+extern const struct clk_ops owl_pll_ops;
+
+#endif /* _OWL_PLL_H_ */
diff --git a/drivers/clk/actions/owl-s900.c b/drivers/clk/actions/owl-s900.c
new file mode 100644
index 000000000000..7f60ed6afe63
--- /dev/null
+++ b/drivers/clk/actions/owl-s900.c
@@ -0,0 +1,721 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// OWL S900 SoC clock driver
+//
+// Copyright (c) 2014 Actions Semi Inc.
+// Author: David Liu <liuwei@actions-semi.com>
+//
+// Copyright (c) 2018 Linaro Ltd.
+// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "owl-common.h"
+#include "owl-composite.h"
+#include "owl-divider.h"
+#include "owl-factor.h"
+#include "owl-fixed-factor.h"
+#include "owl-gate.h"
+#include "owl-mux.h"
+#include "owl-pll.h"
+
+#include <dt-bindings/clock/actions,s900-cmu.h>
+
+#define CMU_COREPLL		(0x0000)
+#define CMU_DEVPLL		(0x0004)
+#define CMU_DDRPLL		(0x0008)
+#define CMU_NANDPLL		(0x000C)
+#define CMU_DISPLAYPLL		(0x0010)
+#define CMU_AUDIOPLL		(0x0014)
+#define CMU_TVOUTPLL		(0x0018)
+#define CMU_BUSCLK		(0x001C)
+#define CMU_SENSORCLK		(0x0020)
+#define CMU_LCDCLK		(0x0024)
+#define CMU_DSICLK		(0x0028)
+#define CMU_CSICLK		(0x002C)
+#define CMU_DECLK		(0x0030)
+#define CMU_BISPCLK		(0x0034)
+#define CMU_IMXCLK		(0x0038)
+#define CMU_HDECLK		(0x003C)
+#define CMU_VDECLK		(0x0040)
+#define CMU_VCECLK		(0x0044)
+#define CMU_NANDCCLK		(0x004C)
+#define CMU_SD0CLK		(0x0050)
+#define CMU_SD1CLK		(0x0054)
+#define CMU_SD2CLK		(0x0058)
+#define CMU_UART0CLK		(0x005C)
+#define CMU_UART1CLK		(0x0060)
+#define CMU_UART2CLK		(0x0064)
+#define CMU_PWM0CLK		(0x0070)
+#define CMU_PWM1CLK		(0x0074)
+#define CMU_PWM2CLK		(0x0078)
+#define CMU_PWM3CLK		(0x007C)
+#define CMU_USBPLL		(0x0080)
+#define CMU_ASSISTPLL		(0x0084)
+#define CMU_EDPCLK		(0x0088)
+#define CMU_GPU3DCLK		(0x0090)
+#define CMU_CORECTL		(0x009C)
+#define CMU_DEVCLKEN0		(0x00A0)
+#define CMU_DEVCLKEN1		(0x00A4)
+#define CMU_DEVRST0		(0x00A8)
+#define CMU_DEVRST1		(0x00AC)
+#define CMU_UART3CLK		(0x00B0)
+#define CMU_UART4CLK		(0x00B4)
+#define CMU_UART5CLK		(0x00B8)
+#define CMU_UART6CLK		(0x00BC)
+#define CMU_TLSCLK		(0x00C0)
+#define CMU_SD3CLK		(0x00C4)
+#define CMU_PWM4CLK		(0x00C8)
+#define CMU_PWM5CLK		(0x00CC)
+
+static struct clk_pll_table clk_audio_pll_table[] = {
+	{ 0, 45158400 }, { 1, 49152000 },
+	{ 0, 0 },
+};
+
+static struct clk_pll_table clk_edp_pll_table[] = {
+	{ 0, 810000000 }, { 1, 135000000 }, { 2, 270000000 },
+	{ 0, 0 },
+};
+
+/* pll clocks */
+static OWL_PLL_NO_PARENT(core_pll_clk, "core_pll_clk", CMU_COREPLL, 24000000, 9, 0, 8, 5, 107, NULL, CLK_IGNORE_UNUSED);
+static OWL_PLL_NO_PARENT(dev_pll_clk, "dev_pll_clk", CMU_DEVPLL, 6000000, 8, 0, 8, 20, 180, NULL, CLK_IGNORE_UNUSED);
+static OWL_PLL_NO_PARENT(ddr_pll_clk, "ddr_pll_clk", CMU_DDRPLL, 24000000, 8, 0, 8, 5, 45, NULL, CLK_IGNORE_UNUSED);
+static OWL_PLL_NO_PARENT(nand_pll_clk, "nand_pll_clk", CMU_NANDPLL, 6000000, 8, 0, 8, 4, 100, NULL, CLK_IGNORE_UNUSED);
+static OWL_PLL_NO_PARENT(display_pll_clk, "display_pll_clk", CMU_DISPLAYPLL, 6000000, 8, 0, 8, 20, 180, NULL, CLK_IGNORE_UNUSED);
+static OWL_PLL_NO_PARENT(assist_pll_clk, "assist_pll_clk", CMU_ASSISTPLL, 500000000, 0, 0, 0, 0, 0, NULL, CLK_IGNORE_UNUSED);
+static OWL_PLL_NO_PARENT(audio_pll_clk, "audio_pll_clk", CMU_AUDIOPLL, 0, 4, 0, 1, 0, 0, clk_audio_pll_table, CLK_IGNORE_UNUSED);
+static OWL_PLL(edp_pll_clk, "edp_pll_clk", "edp24M_clk", CMU_EDPCLK, 0, 9, 0, 2, 0, 0, clk_edp_pll_table, CLK_IGNORE_UNUSED);
+
+static const char *cpu_clk_mux_p[] = { "losc", "hosc", "core_pll_clk", };
+static const char *dev_clk_p[] = { "hosc", "dev_pll_clk", };
+static const char *noc_clk_mux_p[] = { "dev_clk", "assist_pll_clk", };
+static const char *dmm_clk_mux_p[] = { "dev_clk", "nand_pll_clk", "assist_pll_clk", "ddr_clk_src", };
+static const char *bisp_clk_mux_p[] = { "assist_pll_clk", "dev_clk", };
+static const char *csi_clk_mux_p[] = { "display_pll_clk", "dev_clk", };
+static const char *de_clk_mux_p[] = { "assist_pll_clk", "dev_clk", };
+static const char *gpu_clk_mux_p[] = { "dev_clk", "display_pll_clk", "ddr_clk_src", };
+static const char *hde_clk_mux_p[] = { "dev_clk", "display_pll_clk", "ddr_clk_src", };
+static const char *imx_clk_mux_p[] = { "assist_pll_clk", "dev_clk", };
+static const char *lcd_clk_mux_p[] = { "display_pll_clk", "nand_pll_clk", };
+static const char *nand_clk_mux_p[] = { "dev_clk", "nand_pll_clk", };
+static const char *sd_clk_mux_p[] = { "dev_clk", "nand_pll_clk", };
+static const char *sensor_clk_mux_p[] = { "hosc", "bisp_clk", };
+static const char *uart_clk_mux_p[] = { "hosc", "dev_pll_clk", };
+static const char *vce_clk_mux_p[] = { "dev_clk", "display_pll_clk", "assist_pll_clk", "ddr_clk_src", };
+static const char *i2s_clk_mux_p[] = { "audio_pll_clk", };
+static const char *edp_clk_mux_p[] = { "assist_pll_clk", "display_pll_clk", };
+
+/* mux clocks */
+static OWL_MUX(cpu_clk, "cpu_clk", cpu_clk_mux_p, CMU_BUSCLK, 0, 2, CLK_SET_RATE_PARENT);
+static OWL_MUX(dev_clk, "dev_clk", dev_clk_p, CMU_DEVPLL, 12, 1, CLK_SET_RATE_PARENT);
+static OWL_MUX(noc_clk_mux, "noc_clk_mux", noc_clk_mux_p, CMU_BUSCLK, 7, 1, CLK_SET_RATE_PARENT);
+
+static struct clk_div_table nand_div_table[] = {
+	{ 0, 1 }, { 1, 2 }, { 2, 4 }, { 3, 6 },
+	{ 4, 8 }, { 5, 10 }, { 6, 12 }, { 7, 14 },
+	{ 8, 16 }, { 9, 18 }, { 10, 20 }, { 11, 22 },
+	{ 12, 24 }, { 13, 26 }, { 14, 28 }, { 15, 30 },
+	{ 0, 0 },
+};
+
+static struct clk_div_table apb_div_table[] = {
+	{ 1, 2 }, { 2, 3 }, { 3, 4 },
+	{ 0, 0 },
+};
+
+static struct clk_div_table eth_mac_div_table[] = {
+	{ 0, 2 }, { 1, 4 },
+	{ 0, 0 },
+};
+
+static struct clk_div_table rmii_ref_div_table[] = {
+	{ 0, 4 },	  { 1, 10 },
+	{ 0, 0 },
+};
+
+static struct clk_div_table usb3_mac_div_table[] = {
+	{ 1, 2 }, { 2, 3 }, { 3, 4 },
+	{ 0, 8 },
+};
+
+static struct clk_div_table i2s_div_table[] = {
+	{ 0, 1 }, { 1, 2 }, { 2, 3 }, { 3, 4 },
+	{ 4, 6 }, { 5, 8 }, { 6, 12 }, { 7, 16 },
+	{ 8, 24 },
+	{ 0, 0 },
+};
+
+static struct clk_div_table hdmia_div_table[] = {
+	{ 0, 1 }, { 1, 2 }, { 2, 3 }, { 3, 4 },
+	{ 4, 6 }, { 5, 8 }, { 6, 12 }, { 7, 16 },
+	{ 8, 24 },
+	{ 0, 0 },
+};
+
+/* divider clocks */
+static OWL_DIVIDER(noc_clk_div, "noc_clk_div", "noc_clk", CMU_BUSCLK, 19, 1, NULL, 0, 0);
+static OWL_DIVIDER(ahb_clk, "ahb_clk", "noc_clk_div", CMU_BUSCLK, 4, 1, NULL, 0, 0);
+static OWL_DIVIDER(apb_clk, "apb_clk", "ahb_clk", CMU_BUSCLK, 8, 2, apb_div_table, 0, 0);
+static OWL_DIVIDER(usb3_mac_clk, "usb3_mac_clk", "assist_pll_clk", CMU_ASSISTPLL, 12, 2, usb3_mac_div_table, 0, 0);
+static OWL_DIVIDER(rmii_ref_clk, "rmii_ref_clk", "assist_pll_clk", CMU_ASSISTPLL, 8, 1, rmii_ref_div_table, 0, 0);
+
+static struct clk_factor_table sd_factor_table[] = {
+	/* bit0 ~ 4 */
+	{ 0, 1, 1 }, { 1, 1, 2 }, { 2, 1, 3 }, { 3, 1, 4 },
+	{ 4, 1, 5 }, { 5, 1, 6 }, { 6, 1, 7 }, { 7, 1, 8 },
+	{ 8, 1, 9 }, { 9, 1, 10 }, { 10, 1, 11 }, { 11, 1, 12 },
+	{ 12, 1, 13 }, { 13, 1, 14 }, { 14, 1, 15 }, { 15, 1, 16 },
+	{ 16, 1, 17 }, { 17, 1, 18 }, { 18, 1, 19 }, { 19, 1, 20 },
+	{ 20, 1, 21 }, { 21, 1, 22 }, { 22, 1, 23 }, { 23, 1, 24 },
+	{ 24, 1, 25 }, { 25, 1, 26 }, { 26, 1, 27 }, { 27, 1, 28 },
+	{ 28, 1, 29 }, { 29, 1, 30 }, { 30, 1, 31 }, { 31, 1, 32 },
+
+	/* bit8: /128 */
+	{ 256, 1, 1 * 128 }, { 257, 1, 2 * 128 }, { 258, 1, 3 * 128 }, { 259, 1, 4 * 128 },
+	{ 260, 1, 5 * 128 }, { 261, 1, 6 * 128 }, { 262, 1, 7 * 128 }, { 263, 1, 8 * 128 },
+	{ 264, 1, 9 * 128 }, { 265, 1, 10 * 128 }, { 266, 1, 11 * 128 }, { 267, 1, 12 * 128 },
+	{ 268, 1, 13 * 128 }, { 269, 1, 14 * 128 }, { 270, 1, 15 * 128 }, { 271, 1, 16 * 128 },
+	{ 272, 1, 17 * 128 }, { 273, 1, 18 * 128 }, { 274, 1, 19 * 128 }, { 275, 1, 20 * 128 },
+	{ 276, 1, 21 * 128 }, { 277, 1, 22 * 128 }, { 278, 1, 23 * 128 }, { 279, 1, 24 * 128 },
+	{ 280, 1, 25 * 128 }, { 281, 1, 26 * 128 }, { 282, 1, 27 * 128 }, { 283, 1, 28 * 128 },
+	{ 284, 1, 29 * 128 }, { 285, 1, 30 * 128 }, { 286, 1, 31 * 128 }, { 287, 1, 32 * 128 },
+
+	{ 0, 0 },
+};
+
+static struct clk_factor_table dmm_factor_table[] = {
+	{ 0, 1, 1 }, { 1, 2, 3 }, { 2, 1, 2 }, { 3, 1, 3 },
+	{ 4, 1, 4 },
+	{ 0, 0, 0 },
+};
+
+static struct clk_factor_table noc_factor_table[] = {
+	{ 0, 1, 1 },   { 1, 2, 3 }, { 2, 1, 2 }, { 3, 1, 3 }, { 4, 1, 4 },
+	{ 0, 0, 0 },
+};
+
+static struct clk_factor_table bisp_factor_table[] = {
+	{ 0, 1, 1 }, { 1, 2, 3 }, { 2, 1, 2 }, { 3, 2, 5 },
+	{ 4, 1, 3 }, { 5, 1, 4 }, { 6, 1, 6 }, { 7, 1, 8 },
+	{ 0, 0, 0 },
+};
+
+/* factor clocks */
+static OWL_FACTOR(noc_clk, "noc_clk", "noc_clk_mux", CMU_BUSCLK, 16, 3, noc_factor_table, 0, 0);
+static OWL_FACTOR(de_clk1, "de_clk1", "de_clk", CMU_DECLK, 0, 3, bisp_factor_table, 0, 0);
+static OWL_FACTOR(de_clk2, "de_clk2", "de_clk", CMU_DECLK, 4, 3, bisp_factor_table, 0, 0);
+static OWL_FACTOR(de_clk3, "de_clk3", "de_clk", CMU_DECLK, 8, 3, bisp_factor_table, 0, 0);
+
+/* gate clocks */
+static OWL_GATE(gpio_clk, "gpio_clk", "apb_clk", CMU_DEVCLKEN0, 18, 0, 0);
+static OWL_GATE_NO_PARENT(gpu_clk, "gpu_clk", CMU_DEVCLKEN0, 30, 0, 0);
+static OWL_GATE(dmac_clk, "dmac_clk", "noc_clk_div", CMU_DEVCLKEN0, 1, 0, 0);
+static OWL_GATE(timer_clk, "timer_clk", "hosc", CMU_DEVCLKEN1, 27, 0, 0);
+static OWL_GATE_NO_PARENT(dsi_clk, "dsi_clk", CMU_DEVCLKEN0, 12, 0, 0);
+static OWL_GATE(ddr0_clk, "ddr0_clk", "ddr_pll_clk", CMU_DEVCLKEN0, 31, 0, CLK_IGNORE_UNUSED);
+static OWL_GATE(ddr1_clk, "ddr1_clk", "ddr_pll_clk", CMU_DEVCLKEN0, 29, 0, CLK_IGNORE_UNUSED);
+static OWL_GATE_NO_PARENT(usb3_480mpll0_clk, "usb3_480mpll0_clk", CMU_USBPLL, 3, 0, 0);
+static OWL_GATE_NO_PARENT(usb3_480mphy0_clk, "usb3_480mphy0_clk", CMU_USBPLL, 2, 0, 0);
+static OWL_GATE_NO_PARENT(usb3_5gphy_clk, "usb3_5gphy_clk", CMU_USBPLL, 1, 0, 0);
+static OWL_GATE_NO_PARENT(usb3_cce_clk, "usb3_cce_clk", CMU_USBPLL, 0, 0, 0);
+static OWL_GATE(edp24M_clk, "edp24M_clk", "diff24M", CMU_EDPCLK, 8, 0, 0);
+static OWL_GATE(edp_link_clk, "edp_link_clk", "edp_pll_clk", CMU_DEVCLKEN0, 10, 0, 0);
+static OWL_GATE_NO_PARENT(usbh0_pllen_clk, "usbh0_pllen_clk", CMU_USBPLL, 12, 0, 0);
+static OWL_GATE_NO_PARENT(usbh0_phy_clk, "usbh0_phy_clk", CMU_USBPLL, 10, 0, 0);
+static OWL_GATE_NO_PARENT(usbh0_cce_clk, "usbh0_cce_clk", CMU_USBPLL, 8, 0, 0);
+static OWL_GATE_NO_PARENT(usbh1_pllen_clk, "usbh1_pllen_clk", CMU_USBPLL, 13, 0, 0);
+static OWL_GATE_NO_PARENT(usbh1_phy_clk, "usbh1_phy_clk", CMU_USBPLL, 11, 0, 0);
+static OWL_GATE_NO_PARENT(usbh1_cce_clk, "usbh1_cce_clk", CMU_USBPLL, 9, 0, 0);
+static OWL_GATE(spi0_clk, "spi0_clk", "ahb_clk", CMU_DEVCLKEN1, 10, 0, CLK_IGNORE_UNUSED);
+static OWL_GATE(spi1_clk, "spi1_clk", "ahb_clk", CMU_DEVCLKEN1, 11, 0, CLK_IGNORE_UNUSED);
+static OWL_GATE(spi2_clk, "spi2_clk", "ahb_clk", CMU_DEVCLKEN1, 12, 0, CLK_IGNORE_UNUSED);
+static OWL_GATE(spi3_clk, "spi3_clk", "ahb_clk", CMU_DEVCLKEN1, 13, 0, CLK_IGNORE_UNUSED);
+
+/* composite clocks */
+static OWL_COMP_FACTOR(bisp_clk, "bisp_clk", bisp_clk_mux_p,
+			OWL_MUX_HW(CMU_BISPCLK, 4, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 14, 0),
+			OWL_FACTOR_HW(CMU_BISPCLK, 0, 3, 0, bisp_factor_table),
+			0);
+
+static OWL_COMP_DIV(csi0_clk, "csi0_clk", csi_clk_mux_p,
+			OWL_MUX_HW(CMU_CSICLK, 4, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 13, 0),
+			OWL_DIVIDER_HW(CMU_CSICLK, 0, 4, 0, NULL),
+			0);
+
+static OWL_COMP_DIV(csi1_clk, "csi1_clk", csi_clk_mux_p,
+			OWL_MUX_HW(CMU_CSICLK, 20, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 15, 0),
+			OWL_DIVIDER_HW(CMU_CSICLK, 16, 4, 0, NULL),
+			0);
+
+static OWL_COMP_PASS(de_clk, "de_clk", de_clk_mux_p,
+			OWL_MUX_HW(CMU_DECLK, 12, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 8, 0),
+			0);
+
+static OWL_COMP_FACTOR(dmm_clk, "dmm_clk", dmm_clk_mux_p,
+			OWL_MUX_HW(CMU_BUSCLK, 10, 2),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 19, 0),
+			OWL_FACTOR_HW(CMU_BUSCLK, 12, 3, 0, dmm_factor_table),
+			CLK_IGNORE_UNUSED);
+
+static OWL_COMP_FACTOR(edp_clk, "edp_clk", edp_clk_mux_p,
+			OWL_MUX_HW(CMU_EDPCLK, 19, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 10, 0),
+			OWL_FACTOR_HW(CMU_EDPCLK, 16, 3, 0, bisp_factor_table),
+			0);
+
+static OWL_COMP_DIV_FIXED(eth_mac_clk, "eth_mac_clk", "assist_pll_clk",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 22, 0),
+			OWL_DIVIDER_HW(CMU_ASSISTPLL, 10, 1, 0, eth_mac_div_table),
+			0);
+
+static OWL_COMP_FACTOR(gpu_core_clk, "gpu_core_clk", gpu_clk_mux_p,
+			OWL_MUX_HW(CMU_GPU3DCLK, 4, 2),
+			OWL_GATE_HW(CMU_GPU3DCLK, 15, 0),
+			OWL_FACTOR_HW(CMU_GPU3DCLK, 0, 3, 0, bisp_factor_table),
+			0);
+
+static OWL_COMP_FACTOR(gpu_mem_clk, "gpu_mem_clk", gpu_clk_mux_p,
+			OWL_MUX_HW(CMU_GPU3DCLK, 20, 2),
+			OWL_GATE_HW(CMU_GPU3DCLK, 14, 0),
+			OWL_FACTOR_HW(CMU_GPU3DCLK, 16, 3, 0, bisp_factor_table),
+			0);
+
+static OWL_COMP_FACTOR(gpu_sys_clk, "gpu_sys_clk", gpu_clk_mux_p,
+			OWL_MUX_HW(CMU_GPU3DCLK, 28, 2),
+			OWL_GATE_HW(CMU_GPU3DCLK, 13, 0),
+			OWL_FACTOR_HW(CMU_GPU3DCLK, 24, 3, 0, bisp_factor_table),
+			0);
+
+static OWL_COMP_FACTOR(hde_clk, "hde_clk", hde_clk_mux_p,
+			OWL_MUX_HW(CMU_HDECLK, 4, 2),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 27, 0),
+			OWL_FACTOR_HW(CMU_HDECLK, 0, 3, 0, bisp_factor_table),
+			0);
+
+static OWL_COMP_DIV(hdmia_clk, "hdmia_clk", i2s_clk_mux_p,
+			OWL_MUX_HW(CMU_AUDIOPLL, 24, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 22, 0),
+			OWL_DIVIDER_HW(CMU_AUDIOPLL, 24, 4, 0, hdmia_div_table),
+			0);
+
+static OWL_COMP_FIXED_FACTOR(i2c0_clk, "i2c0_clk", "assist_pll_clk",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 14, 0),
+			1, 5, 0);
+
+static OWL_COMP_FIXED_FACTOR(i2c1_clk, "i2c1_clk", "assist_pll_clk",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 15, 0),
+			1, 5, 0);
+
+static OWL_COMP_FIXED_FACTOR(i2c2_clk, "i2c2_clk", "assist_pll_clk",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 30, 0),
+			1, 5, 0);
+
+static OWL_COMP_FIXED_FACTOR(i2c3_clk, "i2c3_clk", "assist_pll_clk",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 31, 0),
+			1, 5, 0);
+
+static OWL_COMP_FIXED_FACTOR(i2c4_clk, "i2c4_clk", "assist_pll_clk",
+			OWL_GATE_HW(CMU_DEVCLKEN0, 17, 0),
+			1, 5, 0);
+
+static OWL_COMP_FIXED_FACTOR(i2c5_clk, "i2c5_clk", "assist_pll_clk",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 1, 0),
+			1, 5, 0);
+
+static OWL_COMP_DIV(i2srx_clk, "i2srx_clk", i2s_clk_mux_p,
+			OWL_MUX_HW(CMU_AUDIOPLL, 24, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 21, 0),
+			OWL_DIVIDER_HW(CMU_AUDIOPLL, 20, 4, 0, i2s_div_table),
+			0);
+
+static OWL_COMP_DIV(i2stx_clk, "i2stx_clk", i2s_clk_mux_p,
+			OWL_MUX_HW(CMU_AUDIOPLL, 24, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 20, 0),
+			OWL_DIVIDER_HW(CMU_AUDIOPLL, 16, 4, 0, i2s_div_table),
+			0);
+
+static OWL_COMP_FACTOR(imx_clk, "imx_clk", imx_clk_mux_p,
+			OWL_MUX_HW(CMU_IMXCLK, 4, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN1, 17, 0),
+			OWL_FACTOR_HW(CMU_IMXCLK, 0, 3, 0, bisp_factor_table),
+			0);
+
+static OWL_COMP_DIV(lcd_clk, "lcd_clk", lcd_clk_mux_p,
+			OWL_MUX_HW(CMU_LCDCLK, 12, 2),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 9, 0),
+			OWL_DIVIDER_HW(CMU_LCDCLK, 0, 5, 0, NULL),
+			0);
+
+static OWL_COMP_DIV(nand0_clk, "nand0_clk", nand_clk_mux_p,
+			OWL_MUX_HW(CMU_NANDCCLK, 8, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 4, 0),
+			OWL_DIVIDER_HW(CMU_NANDCCLK, 0, 4, 0, nand_div_table),
+			CLK_SET_RATE_PARENT);
+
+static OWL_COMP_DIV(nand1_clk, "nand1_clk", nand_clk_mux_p,
+			OWL_MUX_HW(CMU_NANDCCLK, 24, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 11, 0),
+			OWL_DIVIDER_HW(CMU_NANDCCLK, 16, 4, 0, nand_div_table),
+			CLK_SET_RATE_PARENT);
+
+static OWL_COMP_DIV_FIXED(pwm0_clk, "pwm0_clk", "hosc",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 23, 0),
+			OWL_DIVIDER_HW(CMU_PWM0CLK, 0, 6, 0, NULL),
+			0);
+
+static OWL_COMP_DIV_FIXED(pwm1_clk, "pwm1_clk", "hosc",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 24, 0),
+			OWL_DIVIDER_HW(CMU_PWM1CLK, 0, 6, 0, NULL),
+			0);
+/*
+ * pwm2 may be for backlight, do not gate it
+ * even it is "unused", because it may be
+ * enabled at boot stage, and in kernel, driver
+ * has no effective method to know the real status,
+ * so, the best way is keeping it as what it was.
+ */
+static OWL_COMP_DIV_FIXED(pwm2_clk, "pwm2_clk", "hosc",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 25, 0),
+			OWL_DIVIDER_HW(CMU_PWM2CLK, 0, 6, 0, NULL),
+			CLK_IGNORE_UNUSED);
+
+static OWL_COMP_DIV_FIXED(pwm3_clk, "pwm3_clk", "hosc",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 26, 0),
+			OWL_DIVIDER_HW(CMU_PWM3CLK, 0, 6, 0, NULL),
+			0);
+
+static OWL_COMP_DIV_FIXED(pwm4_clk, "pwm4_clk", "hosc",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 4, 0),
+			OWL_DIVIDER_HW(CMU_PWM4CLK, 0, 6, 0, NULL),
+			0);
+
+static OWL_COMP_DIV_FIXED(pwm5_clk, "pwm5_clk", "hosc",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 5, 0),
+			OWL_DIVIDER_HW(CMU_PWM5CLK, 0, 6, 0, NULL),
+			0);
+
+static OWL_COMP_FACTOR(sd0_clk, "sd0_clk", sd_clk_mux_p,
+			OWL_MUX_HW(CMU_SD0CLK, 9, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 5, 0),
+			OWL_FACTOR_HW(CMU_SD0CLK, 0, 9, 0, sd_factor_table),
+			0);
+
+static OWL_COMP_FACTOR(sd1_clk, "sd1_clk", sd_clk_mux_p,
+			OWL_MUX_HW(CMU_SD1CLK, 9, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 6, 0),
+			OWL_FACTOR_HW(CMU_SD1CLK, 0, 9, 0, sd_factor_table),
+			0);
+
+static OWL_COMP_FACTOR(sd2_clk, "sd2_clk", sd_clk_mux_p,
+			OWL_MUX_HW(CMU_SD2CLK, 9, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 7, 0),
+			OWL_FACTOR_HW(CMU_SD2CLK, 0, 9, 0, sd_factor_table),
+			0);
+
+static OWL_COMP_FACTOR(sd3_clk, "sd3_clk", sd_clk_mux_p,
+			OWL_MUX_HW(CMU_SD3CLK, 9, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 16, 0),
+			OWL_FACTOR_HW(CMU_SD3CLK, 0, 9, 0, sd_factor_table),
+			0);
+
+static OWL_COMP_DIV(sensor_clk, "sensor_clk", sensor_clk_mux_p,
+			OWL_MUX_HW(CMU_SENSORCLK, 4, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 14, 0),
+			OWL_DIVIDER_HW(CMU_SENSORCLK, 0, 4, 0, NULL),
+			0);
+
+static OWL_COMP_DIV_FIXED(speed_sensor_clk, "speed_sensor_clk",
+			"hosc",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 0, 0),
+			OWL_DIVIDER_HW(CMU_TLSCLK, 0, 4, CLK_DIVIDER_POWER_OF_TWO, NULL),
+			0);
+
+static OWL_COMP_DIV_FIXED(thermal_sensor_clk, "thermal_sensor_clk",
+			"hosc",
+			OWL_GATE_HW(CMU_DEVCLKEN1, 2, 0),
+			OWL_DIVIDER_HW(CMU_TLSCLK, 8, 4, CLK_DIVIDER_POWER_OF_TWO, NULL),
+			0);
+
+static OWL_COMP_DIV(uart0_clk, "uart0_clk", uart_clk_mux_p,
+			OWL_MUX_HW(CMU_UART0CLK, 16, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN1, 6, 0),
+			OWL_DIVIDER_HW(CMU_UART0CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL),
+			CLK_IGNORE_UNUSED);
+
+static OWL_COMP_DIV(uart1_clk, "uart1_clk", uart_clk_mux_p,
+			OWL_MUX_HW(CMU_UART1CLK, 16, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN1, 7, 0),
+			OWL_DIVIDER_HW(CMU_UART1CLK, 1, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL),
+			CLK_IGNORE_UNUSED);
+
+static OWL_COMP_DIV(uart2_clk, "uart2_clk", uart_clk_mux_p,
+			OWL_MUX_HW(CMU_UART2CLK, 16, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN1, 8, 0),
+			OWL_DIVIDER_HW(CMU_UART2CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL),
+			CLK_IGNORE_UNUSED);
+
+static OWL_COMP_DIV(uart3_clk, "uart3_clk", uart_clk_mux_p,
+			OWL_MUX_HW(CMU_UART3CLK, 16, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN1, 19, 0),
+			OWL_DIVIDER_HW(CMU_UART3CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL),
+			CLK_IGNORE_UNUSED);
+
+static OWL_COMP_DIV(uart4_clk, "uart4_clk", uart_clk_mux_p,
+			OWL_MUX_HW(CMU_UART4CLK, 16, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN1, 20, 0),
+			OWL_DIVIDER_HW(CMU_UART4CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL),
+			CLK_IGNORE_UNUSED);
+
+static OWL_COMP_DIV(uart5_clk, "uart5_clk", uart_clk_mux_p,
+			OWL_MUX_HW(CMU_UART5CLK, 16, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN1, 21, 0),
+			OWL_DIVIDER_HW(CMU_UART5CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL),
+			CLK_IGNORE_UNUSED);
+
+static OWL_COMP_DIV(uart6_clk, "uart6_clk", uart_clk_mux_p,
+			OWL_MUX_HW(CMU_UART6CLK, 16, 1),
+			OWL_GATE_HW(CMU_DEVCLKEN1, 18, 0),
+			OWL_DIVIDER_HW(CMU_UART6CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL),
+			CLK_IGNORE_UNUSED);
+
+static OWL_COMP_FACTOR(vce_clk, "vce_clk", vce_clk_mux_p,
+			OWL_MUX_HW(CMU_VCECLK, 4, 2),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 26, 0),
+			OWL_FACTOR_HW(CMU_VCECLK, 0, 3, 0, bisp_factor_table),
+			0);
+
+static OWL_COMP_FACTOR(vde_clk, "vde_clk", hde_clk_mux_p,
+			OWL_MUX_HW(CMU_VDECLK, 4, 2),
+			OWL_GATE_HW(CMU_DEVCLKEN0, 25, 0),
+			OWL_FACTOR_HW(CMU_VDECLK, 0, 3, 0, bisp_factor_table),
+			0);
+
+static struct owl_clk_common *s900_clks[] = {
+	&core_pll_clk.common,
+	&dev_pll_clk.common,
+	&ddr_pll_clk.common,
+	&nand_pll_clk.common,
+	&display_pll_clk.common,
+	&assist_pll_clk.common,
+	&audio_pll_clk.common,
+	&edp_pll_clk.common,
+	&cpu_clk.common,
+	&dev_clk.common,
+	&noc_clk_mux.common,
+	&noc_clk_div.common,
+	&ahb_clk.common,
+	&apb_clk.common,
+	&usb3_mac_clk.common,
+	&rmii_ref_clk.common,
+	&noc_clk.common,
+	&de_clk1.common,
+	&de_clk2.common,
+	&de_clk3.common,
+	&gpio_clk.common,
+	&gpu_clk.common,
+	&dmac_clk.common,
+	&timer_clk.common,
+	&dsi_clk.common,
+	&ddr0_clk.common,
+	&ddr1_clk.common,
+	&usb3_480mpll0_clk.common,
+	&usb3_480mphy0_clk.common,
+	&usb3_5gphy_clk.common,
+	&usb3_cce_clk.common,
+	&edp24M_clk.common,
+	&edp_link_clk.common,
+	&usbh0_pllen_clk.common,
+	&usbh0_phy_clk.common,
+	&usbh0_cce_clk.common,
+	&usbh1_pllen_clk.common,
+	&usbh1_phy_clk.common,
+	&usbh1_cce_clk.common,
+	&i2c0_clk.common,
+	&i2c1_clk.common,
+	&i2c2_clk.common,
+	&i2c3_clk.common,
+	&i2c4_clk.common,
+	&i2c5_clk.common,
+	&spi0_clk.common,
+	&spi1_clk.common,
+	&spi2_clk.common,
+	&spi3_clk.common,
+	&bisp_clk.common,
+	&csi0_clk.common,
+	&csi1_clk.common,
+	&de_clk.common,
+	&dmm_clk.common,
+	&edp_clk.common,
+	&eth_mac_clk.common,
+	&gpu_core_clk.common,
+	&gpu_mem_clk.common,
+	&gpu_sys_clk.common,
+	&hde_clk.common,
+	&hdmia_clk.common,
+	&i2srx_clk.common,
+	&i2stx_clk.common,
+	&imx_clk.common,
+	&lcd_clk.common,
+	&nand0_clk.common,
+	&nand1_clk.common,
+	&pwm0_clk.common,
+	&pwm1_clk.common,
+	&pwm2_clk.common,
+	&pwm3_clk.common,
+	&pwm4_clk.common,
+	&pwm5_clk.common,
+	&sd0_clk.common,
+	&sd1_clk.common,
+	&sd2_clk.common,
+	&sd3_clk.common,
+	&sensor_clk.common,
+	&speed_sensor_clk.common,
+	&thermal_sensor_clk.common,
+	&uart0_clk.common,
+	&uart1_clk.common,
+	&uart2_clk.common,
+	&uart3_clk.common,
+	&uart4_clk.common,
+	&uart5_clk.common,
+	&uart6_clk.common,
+	&vce_clk.common,
+	&vde_clk.common,
+};
+
+static struct clk_hw_onecell_data s900_hw_clks = {
+	.hws	= {
+		[CLK_CORE_PLL]		= &core_pll_clk.common.hw,
+		[CLK_DEV_PLL]		= &dev_pll_clk.common.hw,
+		[CLK_DDR_PLL]		= &ddr_pll_clk.common.hw,
+		[CLK_NAND_PLL]		= &nand_pll_clk.common.hw,
+		[CLK_DISPLAY_PLL]	= &display_pll_clk.common.hw,
+		[CLK_ASSIST_PLL]	= &assist_pll_clk.common.hw,
+		[CLK_AUDIO_PLL]		= &audio_pll_clk.common.hw,
+		[CLK_EDP_PLL]		= &edp_pll_clk.common.hw,
+		[CLK_CPU]		= &cpu_clk.common.hw,
+		[CLK_DEV]		= &dev_clk.common.hw,
+		[CLK_NOC_MUX]		= &noc_clk_mux.common.hw,
+		[CLK_NOC_DIV]		= &noc_clk_div.common.hw,
+		[CLK_AHB]		= &ahb_clk.common.hw,
+		[CLK_APB]		= &apb_clk.common.hw,
+		[CLK_USB3_MAC]		= &usb3_mac_clk.common.hw,
+		[CLK_RMII_REF]		= &rmii_ref_clk.common.hw,
+		[CLK_NOC]		= &noc_clk.common.hw,
+		[CLK_DE1]		= &de_clk1.common.hw,
+		[CLK_DE2]		= &de_clk2.common.hw,
+		[CLK_DE3]		= &de_clk3.common.hw,
+		[CLK_GPIO]		= &gpio_clk.common.hw,
+		[CLK_GPU]		= &gpu_clk.common.hw,
+		[CLK_DMAC]		= &dmac_clk.common.hw,
+		[CLK_TIMER]		= &timer_clk.common.hw,
+		[CLK_DSI]		= &dsi_clk.common.hw,
+		[CLK_DDR0]		= &ddr0_clk.common.hw,
+		[CLK_DDR1]		= &ddr1_clk.common.hw,
+		[CLK_USB3_480MPLL0]	= &usb3_480mpll0_clk.common.hw,
+		[CLK_USB3_480MPHY0]	= &usb3_480mphy0_clk.common.hw,
+		[CLK_USB3_5GPHY]	= &usb3_5gphy_clk.common.hw,
+		[CLK_USB3_CCE]		= &usb3_cce_clk.common.hw,
+		[CLK_24M_EDP]		= &edp24M_clk.common.hw,
+		[CLK_EDP_LINK]		= &edp_link_clk.common.hw,
+		[CLK_USB2H0_PLLEN]	= &usbh0_pllen_clk.common.hw,
+		[CLK_USB2H0_PHY]	= &usbh0_phy_clk.common.hw,
+		[CLK_USB2H0_CCE]	= &usbh0_cce_clk.common.hw,
+		[CLK_USB2H1_PLLEN]	= &usbh1_pllen_clk.common.hw,
+		[CLK_USB2H1_PHY]	= &usbh1_phy_clk.common.hw,
+		[CLK_USB2H1_CCE]	= &usbh1_cce_clk.common.hw,
+		[CLK_I2C0]		= &i2c0_clk.common.hw,
+		[CLK_I2C1]		= &i2c1_clk.common.hw,
+		[CLK_I2C2]		= &i2c2_clk.common.hw,
+		[CLK_I2C3]		= &i2c3_clk.common.hw,
+		[CLK_I2C4]		= &i2c4_clk.common.hw,
+		[CLK_I2C5]		= &i2c5_clk.common.hw,
+		[CLK_SPI0]		= &spi0_clk.common.hw,
+		[CLK_SPI1]		= &spi1_clk.common.hw,
+		[CLK_SPI2]		= &spi2_clk.common.hw,
+		[CLK_SPI3]		= &spi3_clk.common.hw,
+		[CLK_BISP]		= &bisp_clk.common.hw,
+		[CLK_CSI0]		= &csi0_clk.common.hw,
+		[CLK_CSI1]		= &csi1_clk.common.hw,
+		[CLK_DE0]		= &de_clk.common.hw,
+		[CLK_DMM]		= &dmm_clk.common.hw,
+		[CLK_EDP]		= &edp_clk.common.hw,
+		[CLK_ETH_MAC]		= &eth_mac_clk.common.hw,
+		[CLK_GPU_CORE]		= &gpu_core_clk.common.hw,
+		[CLK_GPU_MEM]		= &gpu_mem_clk.common.hw,
+		[CLK_GPU_SYS]		= &gpu_sys_clk.common.hw,
+		[CLK_HDE]		= &hde_clk.common.hw,
+		[CLK_HDMI_AUDIO]	= &hdmia_clk.common.hw,
+		[CLK_I2SRX]		= &i2srx_clk.common.hw,
+		[CLK_I2STX]		= &i2stx_clk.common.hw,
+		[CLK_IMX]		= &imx_clk.common.hw,
+		[CLK_LCD]		= &lcd_clk.common.hw,
+		[CLK_NAND0]		= &nand0_clk.common.hw,
+		[CLK_NAND1]		= &nand1_clk.common.hw,
+		[CLK_PWM0]		= &pwm0_clk.common.hw,
+		[CLK_PWM1]		= &pwm1_clk.common.hw,
+		[CLK_PWM2]		= &pwm2_clk.common.hw,
+		[CLK_PWM3]		= &pwm3_clk.common.hw,
+		[CLK_PWM4]		= &pwm4_clk.common.hw,
+		[CLK_PWM5]		= &pwm5_clk.common.hw,
+		[CLK_SD0]		= &sd0_clk.common.hw,
+		[CLK_SD1]		= &sd1_clk.common.hw,
+		[CLK_SD2]		= &sd2_clk.common.hw,
+		[CLK_SD3]		= &sd3_clk.common.hw,
+		[CLK_SENSOR]		= &sensor_clk.common.hw,
+		[CLK_SPEED_SENSOR]	= &speed_sensor_clk.common.hw,
+		[CLK_THERMAL_SENSOR]	= &thermal_sensor_clk.common.hw,
+		[CLK_UART0]		= &uart0_clk.common.hw,
+		[CLK_UART1]		= &uart1_clk.common.hw,
+		[CLK_UART2]		= &uart2_clk.common.hw,
+		[CLK_UART3]		= &uart3_clk.common.hw,
+		[CLK_UART4]		= &uart4_clk.common.hw,
+		[CLK_UART5]		= &uart5_clk.common.hw,
+		[CLK_UART6]		= &uart6_clk.common.hw,
+		[CLK_VCE]		= &vce_clk.common.hw,
+		[CLK_VDE]		= &vde_clk.common.hw,
+	},
+	.num	= CLK_NR_CLKS,
+};
+
+static const struct owl_clk_desc s900_clk_desc = {
+	.clks	    = s900_clks,
+	.num_clks   = ARRAY_SIZE(s900_clks),
+
+	.hw_clks    = &s900_hw_clks,
+};
+
+static int s900_clk_probe(struct platform_device *pdev)
+{
+	const struct owl_clk_desc *desc;
+
+	desc = &s900_clk_desc;
+	owl_clk_regmap_init(pdev, desc);
+
+	return owl_clk_probe(&pdev->dev, desc->hw_clks);
+}
+
+static const struct of_device_id s900_clk_of_match[] = {
+	{ .compatible = "actions,s900-cmu", },
+	{ /* sentinel */ }
+};
+
+static struct platform_driver s900_clk_driver = {
+	.probe = s900_clk_probe,
+	.driver = {
+		.name = "s900-cmu",
+		.of_match_table = s900_clk_of_match,
+	},
+};
+
+static int __init s900_clk_init(void)
+{
+	return platform_driver_register(&s900_clk_driver);
+}
+core_initcall(s900_clk_init);
diff --git a/drivers/clk/at91/clk-pll.c b/drivers/clk/at91/clk-pll.c
index 7d3223fc7161..72b6091eb7b9 100644
--- a/drivers/clk/at91/clk-pll.c
+++ b/drivers/clk/at91/clk-pll.c
@@ -132,19 +132,8 @@ static unsigned long clk_pll_recalc_rate(struct clk_hw *hw,
 					 unsigned long parent_rate)
 {
 	struct clk_pll *pll = to_clk_pll(hw);
-	unsigned int pllr;
-	u16 mul;
-	u8 div;
-
-	regmap_read(pll->regmap, PLL_REG(pll->id), &pllr);
-
-	div = PLL_DIV(pllr);
-	mul = PLL_MUL(pllr, pll->layout);
-
-	if (!div || !mul)
-		return 0;
 
-	return (parent_rate / div) * (mul + 1);
+	return (parent_rate / pll->div) * (pll->mul + 1);
 }
 
 static long clk_pll_get_best_div_mul(struct clk_pll *pll, unsigned long rate,
diff --git a/drivers/clk/clk-aspeed.c b/drivers/clk/clk-aspeed.c
index 5eb50c31e455..eb5fb7f88ccd 100644
--- a/drivers/clk/clk-aspeed.c
+++ b/drivers/clk/clk-aspeed.c
@@ -16,6 +16,8 @@
 
 #define ASPEED_NUM_CLKS		35
 
+#define ASPEED_RESET2_OFFSET	32
+
 #define ASPEED_RESET_CTRL	0x04
 #define ASPEED_CLK_SELECTION	0x08
 #define ASPEED_CLK_STOP_CTRL	0x0c
@@ -30,6 +32,7 @@
 #define  CLKIN_25MHZ_EN		BIT(23)
 #define  AST2400_CLK_SOURCE_SEL	BIT(18)
 #define ASPEED_CLK_SELECTION_2	0xd8
+#define ASPEED_RESET_CTRL2	0xd4
 
 /* Globally visible clocks */
 static DEFINE_SPINLOCK(aspeed_clk_lock);
@@ -88,7 +91,7 @@ static const struct aspeed_gate_data aspeed_gates[] = {
 	[ASPEED_CLK_GATE_GCLK] =	{  1,  7, "gclk-gate",		NULL,	0 }, /* 2D engine */
 	[ASPEED_CLK_GATE_MCLK] =	{  2, -1, "mclk-gate",		"mpll",	CLK_IS_CRITICAL }, /* SDRAM */
 	[ASPEED_CLK_GATE_VCLK] =	{  3,  6, "vclk-gate",		NULL,	0 }, /* Video Capture */
-	[ASPEED_CLK_GATE_BCLK] =	{  4, 10, "bclk-gate",		"bclk",	0 }, /* PCIe/PCI */
+	[ASPEED_CLK_GATE_BCLK] =	{  4,  8, "bclk-gate",		"bclk",	0 }, /* PCIe/PCI */
 	[ASPEED_CLK_GATE_DCLK] =	{  5, -1, "dclk-gate",		NULL,	0 }, /* DAC */
 	[ASPEED_CLK_GATE_REFCLK] =	{  6, -1, "refclk-gate",	"clkin", CLK_IS_CRITICAL },
 	[ASPEED_CLK_GATE_USBPORT2CLK] =	{  7,  3, "usb-port2-gate",	NULL,	0 }, /* USB2.0 Host port 2 */
@@ -291,47 +294,72 @@ struct aspeed_reset {
 #define to_aspeed_reset(p) container_of((p), struct aspeed_reset, rcdev)
 
 static const u8 aspeed_resets[] = {
+	/* SCU04 resets */
 	[ASPEED_RESET_XDMA]	= 25,
 	[ASPEED_RESET_MCTP]	= 24,
 	[ASPEED_RESET_ADC]	= 23,
 	[ASPEED_RESET_JTAG_MASTER] = 22,
 	[ASPEED_RESET_MIC]	= 18,
 	[ASPEED_RESET_PWM]	=  9,
-	[ASPEED_RESET_PCIVGA]	=  8,
+	[ASPEED_RESET_PECI]	= 10,
 	[ASPEED_RESET_I2C]	=  2,
 	[ASPEED_RESET_AHB]	=  1,
+
+	/*
+	 * SCUD4 resets start at an offset to separate them from
+	 * the SCU04 resets.
+	 */
+	[ASPEED_RESET_CRT1]	= ASPEED_RESET2_OFFSET + 5,
 };
 
 static int aspeed_reset_deassert(struct reset_controller_dev *rcdev,
 				 unsigned long id)
 {
 	struct aspeed_reset *ar = to_aspeed_reset(rcdev);
-	u32 rst = BIT(aspeed_resets[id]);
+	u32 reg = ASPEED_RESET_CTRL;
+	u32 bit = aspeed_resets[id];
+
+	if (bit >= ASPEED_RESET2_OFFSET) {
+		bit -= ASPEED_RESET2_OFFSET;
+		reg = ASPEED_RESET_CTRL2;
+	}
 
-	return regmap_update_bits(ar->map, ASPEED_RESET_CTRL, rst, 0);
+	return regmap_update_bits(ar->map, reg, BIT(bit), 0);
 }
 
 static int aspeed_reset_assert(struct reset_controller_dev *rcdev,
 			       unsigned long id)
 {
 	struct aspeed_reset *ar = to_aspeed_reset(rcdev);
-	u32 rst = BIT(aspeed_resets[id]);
+	u32 reg = ASPEED_RESET_CTRL;
+	u32 bit = aspeed_resets[id];
 
-	return regmap_update_bits(ar->map, ASPEED_RESET_CTRL, rst, rst);
+	if (bit >= ASPEED_RESET2_OFFSET) {
+		bit -= ASPEED_RESET2_OFFSET;
+		reg = ASPEED_RESET_CTRL2;
+	}
+
+	return regmap_update_bits(ar->map, reg, BIT(bit), BIT(bit));
 }
 
 static int aspeed_reset_status(struct reset_controller_dev *rcdev,
 			       unsigned long id)
 {
 	struct aspeed_reset *ar = to_aspeed_reset(rcdev);
-	u32 val, rst = BIT(aspeed_resets[id]);
-	int ret;
+	u32 reg = ASPEED_RESET_CTRL;
+	u32 bit = aspeed_resets[id];
+	int ret, val;
+
+	if (bit >= ASPEED_RESET2_OFFSET) {
+		bit -= ASPEED_RESET2_OFFSET;
+		reg = ASPEED_RESET_CTRL2;
+	}
 
-	ret = regmap_read(ar->map, ASPEED_RESET_CTRL, &val);
+	ret = regmap_read(ar->map, reg, &val);
 	if (ret)
 		return ret;
 
-	return !!(val & rst);
+	return !!(val & BIT(bit));
 }
 
 static const struct reset_control_ops aspeed_reset_ops = {
diff --git a/drivers/clk/clk-bulk.c b/drivers/clk/clk-bulk.c
index 4c10456f8a32..6904ed6da504 100644
--- a/drivers/clk/clk-bulk.c
+++ b/drivers/clk/clk-bulk.c
@@ -42,8 +42,9 @@ int __must_check clk_bulk_get(struct device *dev, int num_clks,
 		clks[i].clk = clk_get(dev, clks[i].id);
 		if (IS_ERR(clks[i].clk)) {
 			ret = PTR_ERR(clks[i].clk);
-			dev_err(dev, "Failed to get clk '%s': %d\n",
-				clks[i].id, ret);
+			if (ret != -EPROBE_DEFER)
+				dev_err(dev, "Failed to get clk '%s': %d\n",
+					clks[i].id, ret);
 			clks[i].clk = NULL;
 			goto err;
 		}
diff --git a/drivers/clk/clk-npcm7xx.c b/drivers/clk/clk-npcm7xx.c
new file mode 100644
index 000000000000..740af90a9508
--- /dev/null
+++ b/drivers/clk/clk-npcm7xx.c
@@ -0,0 +1,656 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Nuvoton NPCM7xx Clock Generator
+ * All the clocks are initialized by the bootloader, so this driver allow only
+ * reading of current settings directly from the hardware.
+ *
+ * Copyright (C) 2018 Nuvoton Technologies tali.perry@nuvoton.com
+ */
+
+#include <linux/module.h>
+#include <linux/clk-provider.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+#include <linux/err.h>
+#include <linux/bitfield.h>
+
+#include <dt-bindings/clock/nuvoton,npcm7xx-clock.h>
+
+struct npcm7xx_clk_pll {
+	struct clk_hw	hw;
+	void __iomem	*pllcon;
+	u8		flags;
+};
+
+#define to_npcm7xx_clk_pll(_hw) container_of(_hw, struct npcm7xx_clk_pll, hw)
+
+#define PLLCON_LOKI	BIT(31)
+#define PLLCON_LOKS	BIT(30)
+#define PLLCON_FBDV	GENMASK(27, 16)
+#define PLLCON_OTDV2	GENMASK(15, 13)
+#define PLLCON_PWDEN	BIT(12)
+#define PLLCON_OTDV1	GENMASK(10, 8)
+#define PLLCON_INDV	GENMASK(5, 0)
+
+static unsigned long npcm7xx_clk_pll_recalc_rate(struct clk_hw *hw,
+						 unsigned long parent_rate)
+{
+	struct npcm7xx_clk_pll *pll = to_npcm7xx_clk_pll(hw);
+	unsigned long fbdv, indv, otdv1, otdv2;
+	unsigned int val;
+	u64 ret;
+
+	if (parent_rate == 0) {
+		pr_err("%s: parent rate is zero", __func__);
+		return 0;
+	}
+
+	val = readl_relaxed(pll->pllcon);
+
+	indv = FIELD_GET(PLLCON_INDV, val);
+	fbdv = FIELD_GET(PLLCON_FBDV, val);
+	otdv1 = FIELD_GET(PLLCON_OTDV1, val);
+	otdv2 = FIELD_GET(PLLCON_OTDV2, val);
+
+	ret = (u64)parent_rate * fbdv;
+	do_div(ret, indv * otdv1 * otdv2);
+
+	return ret;
+}
+
+static const struct clk_ops npcm7xx_clk_pll_ops = {
+	.recalc_rate = npcm7xx_clk_pll_recalc_rate,
+};
+
+static struct clk_hw *
+npcm7xx_clk_register_pll(void __iomem *pllcon, const char *name,
+			 const char *parent_name, unsigned long flags)
+{
+	struct npcm7xx_clk_pll *pll;
+	struct clk_init_data init;
+	struct clk_hw *hw;
+	int ret;
+
+	pll = kzalloc(sizeof(*pll), GFP_KERNEL);
+	if (!pll)
+		return ERR_PTR(-ENOMEM);
+
+	pr_debug("%s reg, name=%s, p=%s\n", __func__, name, parent_name);
+
+	init.name = name;
+	init.ops = &npcm7xx_clk_pll_ops;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+	init.flags = flags;
+
+	pll->pllcon = pllcon;
+	pll->hw.init = &init;
+
+	hw = &pll->hw;
+
+	ret = clk_hw_register(NULL, hw);
+	if (ret) {
+		kfree(pll);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+}
+
+#define NPCM7XX_CLKEN1          (0x00)
+#define NPCM7XX_CLKEN2          (0x28)
+#define NPCM7XX_CLKEN3          (0x30)
+#define NPCM7XX_CLKSEL          (0x04)
+#define NPCM7XX_CLKDIV1         (0x08)
+#define NPCM7XX_CLKDIV2         (0x2C)
+#define NPCM7XX_CLKDIV3         (0x58)
+#define NPCM7XX_PLLCON0         (0x0C)
+#define NPCM7XX_PLLCON1         (0x10)
+#define NPCM7XX_PLLCON2         (0x54)
+#define NPCM7XX_SWRSTR          (0x14)
+#define NPCM7XX_IRQWAKECON      (0x18)
+#define NPCM7XX_IRQWAKEFLAG     (0x1C)
+#define NPCM7XX_IPSRST1         (0x20)
+#define NPCM7XX_IPSRST2         (0x24)
+#define NPCM7XX_IPSRST3         (0x34)
+#define NPCM7XX_WD0RCR          (0x38)
+#define NPCM7XX_WD1RCR          (0x3C)
+#define NPCM7XX_WD2RCR          (0x40)
+#define NPCM7XX_SWRSTC1         (0x44)
+#define NPCM7XX_SWRSTC2         (0x48)
+#define NPCM7XX_SWRSTC3         (0x4C)
+#define NPCM7XX_SWRSTC4         (0x50)
+#define NPCM7XX_CORSTC          (0x5C)
+#define NPCM7XX_PLLCONG         (0x60)
+#define NPCM7XX_AHBCKFI         (0x64)
+#define NPCM7XX_SECCNT          (0x68)
+#define NPCM7XX_CNTR25M         (0x6C)
+
+struct npcm7xx_clk_gate_data {
+	u32 reg;
+	u8 bit_idx;
+	const char *name;
+	const char *parent_name;
+	unsigned long flags;
+	/*
+	 * If this clock is exported via DT, set onecell_idx to constant
+	 * defined in include/dt-bindings/clock/nuvoton, NPCM7XX-clock.h for
+	 * this specific clock.  Otherwise, set to -1.
+	 */
+	int onecell_idx;
+};
+
+struct npcm7xx_clk_mux_data {
+	u8 shift;
+	u8 mask;
+	u32 *table;
+	const char *name;
+	const char * const *parent_names;
+	u8 num_parents;
+	unsigned long flags;
+	/*
+	 * If this clock is exported via DT, set onecell_idx to constant
+	 * defined in include/dt-bindings/clock/nuvoton, NPCM7XX-clock.h for
+	 * this specific clock.  Otherwise, set to -1.
+	 */
+	int onecell_idx;
+
+};
+
+struct npcm7xx_clk_div_fixed_data {
+	u8 mult;
+	u8 div;
+	const char *name;
+	const char *parent_name;
+	u8 clk_divider_flags;
+	/*
+	 * If this clock is exported via DT, set onecell_idx to constant
+	 * defined in include/dt-bindings/clock/nuvoton, NPCM7XX-clock.h for
+	 * this specific clock.  Otherwise, set to -1.
+	 */
+	int onecell_idx;
+};
+
+
+struct npcm7xx_clk_div_data {
+	u32 reg;
+	u8 shift;
+	u8 width;
+	const char *name;
+	const char *parent_name;
+	u8 clk_divider_flags;
+	unsigned long flags;
+	/*
+	 * If this clock is exported via DT, set onecell_idx to constant
+	 * defined in include/dt-bindings/clock/nuvoton, NPCM7XX-clock.h for
+	 * this specific clock.  Otherwise, set to -1.
+	 */
+	int onecell_idx;
+};
+
+struct npcm7xx_clk_pll_data {
+	u32 reg;
+	const char *name;
+	const char *parent_name;
+	unsigned long flags;
+	/*
+	 * If this clock is exported via DT, set onecell_idx to constant
+	 * defined in include/dt-bindings/clock/nuvoton, NPCM7XX-clock.h for
+	 * this specific clock.  Otherwise, set to -1.
+	 */
+	int onecell_idx;
+};
+
+/*
+ * Single copy of strings used to refer to clocks within this driver indexed by
+ * above enum.
+ */
+#define NPCM7XX_CLK_S_REFCLK      "refclk"
+#define NPCM7XX_CLK_S_SYSBYPCK    "sysbypck"
+#define NPCM7XX_CLK_S_MCBYPCK     "mcbypck"
+#define NPCM7XX_CLK_S_GFXBYPCK    "gfxbypck"
+#define NPCM7XX_CLK_S_PLL0        "pll0"
+#define NPCM7XX_CLK_S_PLL1        "pll1"
+#define NPCM7XX_CLK_S_PLL1_DIV2   "pll1_div2"
+#define NPCM7XX_CLK_S_PLL2        "pll2"
+#define NPCM7XX_CLK_S_PLL_GFX     "pll_gfx"
+#define NPCM7XX_CLK_S_PLL2_DIV2   "pll2_div2"
+#define NPCM7XX_CLK_S_PIX_MUX     "gfx_pixel"
+#define NPCM7XX_CLK_S_GPRFSEL_MUX "gprfsel_mux"
+#define NPCM7XX_CLK_S_MC_MUX      "mc_phy"
+#define NPCM7XX_CLK_S_CPU_MUX     "cpu"  /*AKA system clock.*/
+#define NPCM7XX_CLK_S_MC          "mc"
+#define NPCM7XX_CLK_S_AXI         "axi"  /*AKA CLK2*/
+#define NPCM7XX_CLK_S_AHB         "ahb"  /*AKA CLK4*/
+#define NPCM7XX_CLK_S_CLKOUT_MUX  "clkout_mux"
+#define NPCM7XX_CLK_S_UART_MUX    "uart_mux"
+#define NPCM7XX_CLK_S_TIM_MUX     "timer_mux"
+#define NPCM7XX_CLK_S_SD_MUX      "sd_mux"
+#define NPCM7XX_CLK_S_GFXM_MUX    "gfxm_mux"
+#define NPCM7XX_CLK_S_SU_MUX      "serial_usb_mux"
+#define NPCM7XX_CLK_S_DVC_MUX     "dvc_mux"
+#define NPCM7XX_CLK_S_GFX_MUX     "gfx_mux"
+#define NPCM7XX_CLK_S_GFX_PIXEL   "gfx_pixel"
+#define NPCM7XX_CLK_S_SPI0        "spi0"
+#define NPCM7XX_CLK_S_SPI3        "spi3"
+#define NPCM7XX_CLK_S_SPIX        "spix"
+#define NPCM7XX_CLK_S_APB1        "apb1"
+#define NPCM7XX_CLK_S_APB2        "apb2"
+#define NPCM7XX_CLK_S_APB3        "apb3"
+#define NPCM7XX_CLK_S_APB4        "apb4"
+#define NPCM7XX_CLK_S_APB5        "apb5"
+#define NPCM7XX_CLK_S_TOCK        "tock"
+#define NPCM7XX_CLK_S_CLKOUT      "clkout"
+#define NPCM7XX_CLK_S_UART        "uart"
+#define NPCM7XX_CLK_S_TIMER       "timer"
+#define NPCM7XX_CLK_S_MMC         "mmc"
+#define NPCM7XX_CLK_S_SDHC        "sdhc"
+#define NPCM7XX_CLK_S_ADC         "adc"
+#define NPCM7XX_CLK_S_GFX         "gfx0_gfx1_mem"
+#define NPCM7XX_CLK_S_USBIF       "serial_usbif"
+#define NPCM7XX_CLK_S_USB_HOST    "usb_host"
+#define NPCM7XX_CLK_S_USB_BRIDGE  "usb_bridge"
+#define NPCM7XX_CLK_S_PCI         "pci"
+
+static u32 pll_mux_table[] = {0, 1, 2, 3};
+static const char * const pll_mux_parents[] __initconst = {
+	NPCM7XX_CLK_S_PLL0,
+	NPCM7XX_CLK_S_PLL1_DIV2,
+	NPCM7XX_CLK_S_REFCLK,
+	NPCM7XX_CLK_S_PLL2_DIV2,
+};
+
+static u32 cpuck_mux_table[] = {0, 1, 2, 3};
+static const char * const cpuck_mux_parents[] __initconst = {
+	NPCM7XX_CLK_S_PLL0,
+	NPCM7XX_CLK_S_PLL1_DIV2,
+	NPCM7XX_CLK_S_REFCLK,
+	NPCM7XX_CLK_S_SYSBYPCK,
+};
+
+static u32 pixcksel_mux_table[] = {0, 2};
+static const char * const pixcksel_mux_parents[] __initconst = {
+	NPCM7XX_CLK_S_PLL_GFX,
+	NPCM7XX_CLK_S_REFCLK,
+};
+
+static u32 sucksel_mux_table[] = {2, 3};
+static const char * const sucksel_mux_parents[] __initconst = {
+	NPCM7XX_CLK_S_REFCLK,
+	NPCM7XX_CLK_S_PLL2_DIV2,
+};
+
+static u32 mccksel_mux_table[] = {0, 2, 3};
+static const char * const mccksel_mux_parents[] __initconst = {
+	NPCM7XX_CLK_S_PLL1_DIV2,
+	NPCM7XX_CLK_S_REFCLK,
+	NPCM7XX_CLK_S_MCBYPCK,
+};
+
+static u32 clkoutsel_mux_table[] = {0, 1, 2, 3, 4};
+static const char * const clkoutsel_mux_parents[] __initconst = {
+	NPCM7XX_CLK_S_PLL0,
+	NPCM7XX_CLK_S_PLL1_DIV2,
+	NPCM7XX_CLK_S_REFCLK,
+	NPCM7XX_CLK_S_PLL_GFX, // divided by 2
+	NPCM7XX_CLK_S_PLL2_DIV2,
+};
+
+static u32 gfxmsel_mux_table[] = {2, 3};
+static const char * const gfxmsel_mux_parents[] __initconst = {
+	NPCM7XX_CLK_S_REFCLK,
+	NPCM7XX_CLK_S_PLL2_DIV2,
+};
+
+static u32 dvcssel_mux_table[] = {2, 3};
+static const char * const dvcssel_mux_parents[] __initconst = {
+	NPCM7XX_CLK_S_REFCLK,
+	NPCM7XX_CLK_S_PLL2,
+};
+
+static const struct npcm7xx_clk_pll_data npcm7xx_plls[] __initconst = {
+	{NPCM7XX_PLLCON0, NPCM7XX_CLK_S_PLL0, NPCM7XX_CLK_S_REFCLK, 0, -1},
+
+	{NPCM7XX_PLLCON1, NPCM7XX_CLK_S_PLL1,
+	NPCM7XX_CLK_S_REFCLK, 0, -1},
+
+	{NPCM7XX_PLLCON2, NPCM7XX_CLK_S_PLL2,
+	NPCM7XX_CLK_S_REFCLK, 0, -1},
+
+	{NPCM7XX_PLLCONG, NPCM7XX_CLK_S_PLL_GFX,
+	NPCM7XX_CLK_S_REFCLK, 0, -1},
+};
+
+static const struct npcm7xx_clk_mux_data npcm7xx_muxes[] __initconst = {
+	{0, GENMASK(1, 0), cpuck_mux_table, NPCM7XX_CLK_S_CPU_MUX,
+	cpuck_mux_parents, ARRAY_SIZE(cpuck_mux_parents), CLK_IS_CRITICAL,
+	NPCM7XX_CLK_CPU},
+
+	{4, GENMASK(1, 0), pixcksel_mux_table, NPCM7XX_CLK_S_PIX_MUX,
+	pixcksel_mux_parents, ARRAY_SIZE(pixcksel_mux_parents), 0,
+	NPCM7XX_CLK_GFX_PIXEL},
+
+	{6, GENMASK(1, 0), pll_mux_table, NPCM7XX_CLK_S_SD_MUX,
+	pll_mux_parents, ARRAY_SIZE(pll_mux_parents), 0, -1},
+
+	{8, GENMASK(1, 0), pll_mux_table, NPCM7XX_CLK_S_UART_MUX,
+	pll_mux_parents, ARRAY_SIZE(pll_mux_parents), 0, -1},
+
+	{10, GENMASK(1, 0), sucksel_mux_table, NPCM7XX_CLK_S_SU_MUX,
+	sucksel_mux_parents, ARRAY_SIZE(sucksel_mux_parents), 0, -1},
+
+	{12, GENMASK(1, 0), mccksel_mux_table, NPCM7XX_CLK_S_MC_MUX,
+	mccksel_mux_parents, ARRAY_SIZE(mccksel_mux_parents), 0, -1},
+
+	{14, GENMASK(1, 0), pll_mux_table, NPCM7XX_CLK_S_TIM_MUX,
+	pll_mux_parents, ARRAY_SIZE(pll_mux_parents), 0, -1},
+
+	{16, GENMASK(1, 0), pll_mux_table, NPCM7XX_CLK_S_GFX_MUX,
+	pll_mux_parents, ARRAY_SIZE(pll_mux_parents), 0, -1},
+
+	{18, GENMASK(2, 0), clkoutsel_mux_table, NPCM7XX_CLK_S_CLKOUT_MUX,
+	clkoutsel_mux_parents, ARRAY_SIZE(clkoutsel_mux_parents), 0, -1},
+
+	{21, GENMASK(1, 0), gfxmsel_mux_table, NPCM7XX_CLK_S_GFXM_MUX,
+	gfxmsel_mux_parents, ARRAY_SIZE(gfxmsel_mux_parents), 0, -1},
+
+	{23, GENMASK(1, 0), dvcssel_mux_table, NPCM7XX_CLK_S_DVC_MUX,
+	dvcssel_mux_parents, ARRAY_SIZE(dvcssel_mux_parents), 0, -1},
+};
+
+/* fixed ratio dividers (no register): */
+static const struct npcm7xx_clk_div_fixed_data npcm7xx_divs_fx[] __initconst = {
+	{ 1, 2, NPCM7XX_CLK_S_MC, NPCM7XX_CLK_S_MC_MUX, 0, NPCM7XX_CLK_MC},
+	{ 1, 2, NPCM7XX_CLK_S_PLL1_DIV2, NPCM7XX_CLK_S_PLL1, 0, -1},
+	{ 1, 2, NPCM7XX_CLK_S_PLL2_DIV2, NPCM7XX_CLK_S_PLL2, 0, -1},
+};
+
+/* configurable dividers: */
+static const struct npcm7xx_clk_div_data npcm7xx_divs[] __initconst = {
+	{NPCM7XX_CLKDIV1, 28, 3, NPCM7XX_CLK_S_ADC,
+	NPCM7XX_CLK_S_TIMER, CLK_DIVIDER_POWER_OF_TWO, 0, NPCM7XX_CLK_ADC},
+	/*30-28 ADCCKDIV*/
+	{NPCM7XX_CLKDIV1, 26, 2, NPCM7XX_CLK_S_AHB,
+	NPCM7XX_CLK_S_AXI, 0, CLK_IS_CRITICAL, NPCM7XX_CLK_AHB},
+	/*27-26 CLK4DIV*/
+	{NPCM7XX_CLKDIV1, 21, 5, NPCM7XX_CLK_S_TIMER,
+	NPCM7XX_CLK_S_TIM_MUX, 0, 0, NPCM7XX_CLK_TIMER},
+	/*25-21 TIMCKDIV*/
+	{NPCM7XX_CLKDIV1, 16, 5, NPCM7XX_CLK_S_UART,
+	NPCM7XX_CLK_S_UART_MUX, 0, 0, NPCM7XX_CLK_UART},
+	/*20-16 UARTDIV*/
+	{NPCM7XX_CLKDIV1, 11, 5, NPCM7XX_CLK_S_MMC,
+	NPCM7XX_CLK_S_SD_MUX, 0, 0, NPCM7XX_CLK_MMC},
+	/*15-11 MMCCKDIV*/
+	{NPCM7XX_CLKDIV1, 6, 5, NPCM7XX_CLK_S_SPI3,
+	NPCM7XX_CLK_S_AHB, 0, 0, NPCM7XX_CLK_SPI3},
+	/*10-6 AHB3CKDIV*/
+	{NPCM7XX_CLKDIV1, 2, 4, NPCM7XX_CLK_S_PCI,
+	NPCM7XX_CLK_S_GFX_MUX, 0, 0, NPCM7XX_CLK_PCI},
+	/*5-2 PCICKDIV*/
+	{NPCM7XX_CLKDIV1, 0, 1, NPCM7XX_CLK_S_AXI,
+	NPCM7XX_CLK_S_CPU_MUX, CLK_DIVIDER_POWER_OF_TWO, CLK_IS_CRITICAL,
+	NPCM7XX_CLK_AXI},/*0 CLK2DIV*/
+
+	{NPCM7XX_CLKDIV2, 30, 2, NPCM7XX_CLK_S_APB4,
+	NPCM7XX_CLK_S_AHB, CLK_DIVIDER_POWER_OF_TWO, 0, NPCM7XX_CLK_APB4},
+	/*31-30 APB4CKDIV*/
+	{NPCM7XX_CLKDIV2, 28, 2, NPCM7XX_CLK_S_APB3,
+	NPCM7XX_CLK_S_AHB, CLK_DIVIDER_POWER_OF_TWO, 0, NPCM7XX_CLK_APB3},
+	/*29-28 APB3CKDIV*/
+	{NPCM7XX_CLKDIV2, 26, 2, NPCM7XX_CLK_S_APB2,
+	NPCM7XX_CLK_S_AHB, CLK_DIVIDER_POWER_OF_TWO, 0, NPCM7XX_CLK_APB2},
+	/*27-26 APB2CKDIV*/
+	{NPCM7XX_CLKDIV2, 24, 2, NPCM7XX_CLK_S_APB1,
+	NPCM7XX_CLK_S_AHB, CLK_DIVIDER_POWER_OF_TWO, 0, NPCM7XX_CLK_APB1},
+	/*25-24 APB1CKDIV*/
+	{NPCM7XX_CLKDIV2, 22, 2, NPCM7XX_CLK_S_APB5,
+	NPCM7XX_CLK_S_AHB, CLK_DIVIDER_POWER_OF_TWO, 0, NPCM7XX_CLK_APB5},
+	/*23-22 APB5CKDIV*/
+	{NPCM7XX_CLKDIV2, 16, 5, NPCM7XX_CLK_S_CLKOUT,
+	NPCM7XX_CLK_S_CLKOUT_MUX, 0, 0, NPCM7XX_CLK_CLKOUT},
+	/*20-16 CLKOUTDIV*/
+	{NPCM7XX_CLKDIV2, 13, 3, NPCM7XX_CLK_S_GFX,
+	NPCM7XX_CLK_S_GFX_MUX, 0, 0, NPCM7XX_CLK_GFX},
+	/*15-13 GFXCKDIV*/
+	{NPCM7XX_CLKDIV2, 8, 5, NPCM7XX_CLK_S_USB_BRIDGE,
+	NPCM7XX_CLK_S_SU_MUX, 0, 0, NPCM7XX_CLK_SU},
+	/*12-8 SUCKDIV*/
+	{NPCM7XX_CLKDIV2, 4, 4, NPCM7XX_CLK_S_USB_HOST,
+	NPCM7XX_CLK_S_SU_MUX, 0, 0, NPCM7XX_CLK_SU48},
+	/*7-4 SU48CKDIV*/
+	{NPCM7XX_CLKDIV2, 0, 4, NPCM7XX_CLK_S_SDHC,
+	NPCM7XX_CLK_S_SD_MUX, 0, 0, NPCM7XX_CLK_SDHC}
+	,/*3-0 SD1CKDIV*/
+
+	{NPCM7XX_CLKDIV3, 6, 5, NPCM7XX_CLK_S_SPI0,
+	NPCM7XX_CLK_S_AHB, 0, 0, NPCM7XX_CLK_SPI0},
+	/*10-6 SPI0CKDV*/
+	{NPCM7XX_CLKDIV3, 1, 5, NPCM7XX_CLK_S_SPIX,
+	NPCM7XX_CLK_S_AHB, 0, 0, NPCM7XX_CLK_SPIX},
+	/*5-1 SPIXCKDV*/
+
+};
+
+static const struct npcm7xx_clk_gate_data npcm7xx_gates[] __initconst = {
+	{NPCM7XX_CLKEN1, 31, "smb1-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN1, 30, "smb0-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN1, 29, "smb7-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN1, 28, "smb6-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN1, 27, "adc-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN1, 26, "wdt-gate", NPCM7XX_CLK_S_TIMER, 0},
+	{NPCM7XX_CLKEN1, 25, "usbdev3-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN1, 24, "usbdev6-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN1, 23, "usbdev5-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN1, 22, "usbdev4-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN1, 21, "emc2-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN1, 20, "timer5_9-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN1, 19, "timer0_4-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN1, 18, "pwmm0-gate", NPCM7XX_CLK_S_APB3, 0},
+	{NPCM7XX_CLKEN1, 17, "huart-gate", NPCM7XX_CLK_S_UART, 0},
+	{NPCM7XX_CLKEN1, 16, "smb5-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN1, 15, "smb4-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN1, 14, "smb3-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN1, 13, "smb2-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN1, 12, "mc-gate", NPCM7XX_CLK_S_MC, 0},
+	{NPCM7XX_CLKEN1, 11, "uart01-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN1, 10, "aes-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN1, 9, "peci-gate", NPCM7XX_CLK_S_APB3, 0},
+	{NPCM7XX_CLKEN1, 8, "usbdev2-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN1, 7, "uart23-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN1, 6, "emc1-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN1, 5, "usbdev1-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN1, 4, "shm-gate", NPCM7XX_CLK_S_AHB, 0},
+	/* bit 3 is reserved */
+	{NPCM7XX_CLKEN1, 2, "kcs-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN1, 1, "spi3-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN1, 0, "spi0-gate", NPCM7XX_CLK_S_AHB, 0},
+
+	{NPCM7XX_CLKEN2, 31, "cp-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN2, 30, "tock-gate", NPCM7XX_CLK_S_TOCK, 0},
+	/* bit 29 is reserved */
+	{NPCM7XX_CLKEN2, 28, "gmac1-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN2, 27, "usbif-gate", NPCM7XX_CLK_S_USBIF, 0},
+	{NPCM7XX_CLKEN2, 26, "usbhost-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN2, 25, "gmac2-gate", NPCM7XX_CLK_S_AHB, 0},
+	/* bit 24 is reserved */
+	{NPCM7XX_CLKEN2, 23, "pspi2-gate", NPCM7XX_CLK_S_APB5, 0},
+	{NPCM7XX_CLKEN2, 22, "pspi1-gate", NPCM7XX_CLK_S_APB5, 0},
+	{NPCM7XX_CLKEN2, 21, "3des-gate", NPCM7XX_CLK_S_AHB, 0},
+	/* bit 20 is reserved */
+	{NPCM7XX_CLKEN2, 19, "siox2-gate", NPCM7XX_CLK_S_APB3, 0},
+	{NPCM7XX_CLKEN2, 18, "siox1-gate", NPCM7XX_CLK_S_APB3, 0},
+	/* bit 17 is reserved */
+	{NPCM7XX_CLKEN2, 16, "fuse-gate", NPCM7XX_CLK_S_APB4, 0},
+	/*  bit 15 is reserved */
+	{NPCM7XX_CLKEN2, 14, "vcd-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN2, 13, "ece-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN2, 12, "vdma-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN2, 11, "ahbpcibrg-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN2, 10, "gfxsys-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN2, 9, "sdhc-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN2, 8, "mmc-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN2, 7, "mft7-gate", NPCM7XX_CLK_S_APB4, 0},
+	{NPCM7XX_CLKEN2, 6, "mft6-gate", NPCM7XX_CLK_S_APB4, 0},
+	{NPCM7XX_CLKEN2, 5, "mft5-gate", NPCM7XX_CLK_S_APB4, 0},
+	{NPCM7XX_CLKEN2, 4, "mft4-gate", NPCM7XX_CLK_S_APB4, 0},
+	{NPCM7XX_CLKEN2, 3, "mft3-gate", NPCM7XX_CLK_S_APB4, 0},
+	{NPCM7XX_CLKEN2, 2, "mft2-gate", NPCM7XX_CLK_S_APB4, 0},
+	{NPCM7XX_CLKEN2, 1, "mft1-gate", NPCM7XX_CLK_S_APB4, 0},
+	{NPCM7XX_CLKEN2, 0, "mft0-gate", NPCM7XX_CLK_S_APB4, 0},
+
+	{NPCM7XX_CLKEN3, 31, "gpiom7-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN3, 30, "gpiom6-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN3, 29, "gpiom5-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN3, 28, "gpiom4-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN3, 27, "gpiom3-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN3, 26, "gpiom2-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN3, 25, "gpiom1-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN3, 24, "gpiom0-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN3, 23, "espi-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN3, 22, "smb11-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN3, 21, "smb10-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN3, 20, "smb9-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN3, 19, "smb8-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN3, 18, "smb15-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN3, 17, "rng-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN3, 16, "timer10_14-gate", NPCM7XX_CLK_S_APB1, 0},
+	{NPCM7XX_CLKEN3, 15, "pcirc-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN3, 14, "sececc-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN3, 13, "sha-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN3, 12, "smb14-gate", NPCM7XX_CLK_S_APB2, 0},
+	/* bit 11 is reserved */
+	/* bit 10 is reserved */
+	{NPCM7XX_CLKEN3, 9, "pcimbx-gate", NPCM7XX_CLK_S_AHB, 0},
+	/* bit 8 is reserved */
+	{NPCM7XX_CLKEN3, 7, "usbdev9-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN3, 6, "usbdev8-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN3, 5, "usbdev7-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN3, 4, "usbdev0-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN3, 3, "smb13-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN3, 2, "spix-gate", NPCM7XX_CLK_S_AHB, 0},
+	{NPCM7XX_CLKEN3, 1, "smb12-gate", NPCM7XX_CLK_S_APB2, 0},
+	{NPCM7XX_CLKEN3, 0, "pwmm1-gate", NPCM7XX_CLK_S_APB3, 0},
+};
+
+static DEFINE_SPINLOCK(npcm7xx_clk_lock);
+
+static void __init npcm7xx_clk_init(struct device_node *clk_np)
+{
+	struct clk_hw_onecell_data *npcm7xx_clk_data;
+	void __iomem *clk_base;
+	struct resource res;
+	struct clk_hw *hw;
+	int ret;
+	int i;
+
+	ret = of_address_to_resource(clk_np, 0, &res);
+	if (ret) {
+		pr_err("%s: failed to get resource, ret %d\n", clk_np->name,
+			ret);
+		return;
+	}
+
+	clk_base = ioremap(res.start, resource_size(&res));
+	if (!clk_base)
+		goto npcm7xx_init_error;
+
+	npcm7xx_clk_data = kzalloc(sizeof(*npcm7xx_clk_data->hws) *
+		NPCM7XX_NUM_CLOCKS + sizeof(npcm7xx_clk_data), GFP_KERNEL);
+	if (!npcm7xx_clk_data)
+		goto npcm7xx_init_np_err;
+
+	npcm7xx_clk_data->num = NPCM7XX_NUM_CLOCKS;
+
+	for (i = 0; i < NPCM7XX_NUM_CLOCKS; i++)
+		npcm7xx_clk_data->hws[i] = ERR_PTR(-EPROBE_DEFER);
+
+	/* Register plls */
+	for (i = 0; i < ARRAY_SIZE(npcm7xx_plls); i++) {
+		const struct npcm7xx_clk_pll_data *pll_data = &npcm7xx_plls[i];
+
+		hw = npcm7xx_clk_register_pll(clk_base + pll_data->reg,
+			pll_data->name, pll_data->parent_name, pll_data->flags);
+		if (IS_ERR(hw)) {
+			pr_err("npcm7xx_clk: Can't register pll\n");
+			goto npcm7xx_init_fail;
+		}
+
+		if (pll_data->onecell_idx >= 0)
+			npcm7xx_clk_data->hws[pll_data->onecell_idx] = hw;
+	}
+
+	/* Register fixed dividers */
+	hw = clk_hw_register_fixed_factor(NULL, NPCM7XX_CLK_S_PLL1_DIV2,
+			NPCM7XX_CLK_S_PLL1, 0, 1, 2);
+	if (IS_ERR(hw)) {
+		pr_err("npcm7xx_clk: Can't register fixed div\n");
+		goto npcm7xx_init_fail;
+	}
+
+	hw = clk_hw_register_fixed_factor(NULL, NPCM7XX_CLK_S_PLL2_DIV2,
+			NPCM7XX_CLK_S_PLL2, 0, 1, 2);
+	if (IS_ERR(hw)) {
+		pr_err("npcm7xx_clk: Can't register div2\n");
+		goto npcm7xx_init_fail;
+	}
+
+	/* Register muxes */
+	for (i = 0; i < ARRAY_SIZE(npcm7xx_muxes); i++) {
+		const struct npcm7xx_clk_mux_data *mux_data = &npcm7xx_muxes[i];
+
+		hw = clk_hw_register_mux_table(NULL,
+			mux_data->name,
+			mux_data->parent_names, mux_data->num_parents,
+			mux_data->flags, clk_base + NPCM7XX_CLKSEL,
+			mux_data->shift, mux_data->mask, 0,
+			mux_data->table, &npcm7xx_clk_lock);
+
+		if (IS_ERR(hw)) {
+			pr_err("npcm7xx_clk: Can't register mux\n");
+			goto npcm7xx_init_fail;
+		}
+
+		if (mux_data->onecell_idx >= 0)
+			npcm7xx_clk_data->hws[mux_data->onecell_idx] = hw;
+	}
+
+	/* Register clock dividers specified in npcm7xx_divs */
+	for (i = 0; i < ARRAY_SIZE(npcm7xx_divs); i++) {
+		const struct npcm7xx_clk_div_data *div_data = &npcm7xx_divs[i];
+
+		hw = clk_hw_register_divider(NULL, div_data->name,
+				div_data->parent_name,
+				div_data->flags,
+				clk_base + div_data->reg,
+				div_data->shift, div_data->width,
+				div_data->clk_divider_flags, &npcm7xx_clk_lock);
+		if (IS_ERR(hw)) {
+			pr_err("npcm7xx_clk: Can't register div table\n");
+			goto npcm7xx_init_fail;
+		}
+
+		if (div_data->onecell_idx >= 0)
+			npcm7xx_clk_data->hws[div_data->onecell_idx] = hw;
+	}
+
+	ret = of_clk_add_hw_provider(clk_np, of_clk_hw_onecell_get,
+					npcm7xx_clk_data);
+	if (ret)
+		pr_err("failed to add DT provider: %d\n", ret);
+
+	of_node_put(clk_np);
+
+	return;
+
+npcm7xx_init_fail:
+	kfree(npcm7xx_clk_data->hws);
+npcm7xx_init_np_err:
+	iounmap(clk_base);
+npcm7xx_init_error:
+	of_node_put(clk_np);
+}
+CLK_OF_DECLARE(npcm7xx_clk_init, "nuvoton,npcm750-clk", npcm7xx_clk_init);
diff --git a/drivers/clk/clk-stm32mp1.c b/drivers/clk/clk-stm32mp1.c
index edd3cf451401..e68cb478f21f 100644
--- a/drivers/clk/clk-stm32mp1.c
+++ b/drivers/clk/clk-stm32mp1.c
@@ -579,14 +579,9 @@ clk_stm32_register_gate_ops(struct device *dev,
 			    spinlock_t *lock)
 {
 	struct clk_init_data init = { NULL };
-	struct clk_gate *gate;
 	struct clk_hw *hw;
 	int ret;
 
-	gate = kzalloc(sizeof(*gate), GFP_KERNEL);
-	if (!gate)
-		return ERR_PTR(-ENOMEM);
-
 	init.name = name;
 	init.parent_names = &parent_name;
 	init.num_parents = 1;
@@ -604,10 +599,8 @@ clk_stm32_register_gate_ops(struct device *dev,
 	hw->init = &init;
 
 	ret = clk_hw_register(dev, hw);
-	if (ret) {
-		kfree(gate);
+	if (ret)
 		hw = ERR_PTR(ret);
-	}
 
 	return hw;
 }
@@ -1988,7 +1981,8 @@ static const struct clock_config stm32mp1_clock_cfg[] = {
 		  _DIV(RCC_MCO2CFGR, 4, 4, 0, NULL)),
 
 	/* Debug clocks */
-	GATE(CK_DBG, "ck_sys_dbg", "ck_axi", 0, RCC_DBGCFGR, 8, 0),
+	GATE(CK_DBG, "ck_sys_dbg", "ck_axi", CLK_IGNORE_UNUSED,
+	     RCC_DBGCFGR, 8, 0),
 
 	COMPOSITE(CK_TRACE, "ck_trace", ck_trace_src, CLK_OPS_PARENT_ENABLE,
 		  _GATE(RCC_DBGCFGR, 9, 0),
diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index 7af555f0e60c..9ae92aae4cbf 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -549,7 +549,8 @@ static void clk_core_rate_unprotect(struct clk_core *core)
 	if (!core)
 		return;
 
-	if (WARN_ON(core->protect_count == 0))
+	if (WARN(core->protect_count == 0,
+	    "%s already unprotected\n", core->name))
 		return;
 
 	if (--core->protect_count > 0)
@@ -682,16 +683,18 @@ static void clk_core_unprepare(struct clk_core *core)
 	if (!core)
 		return;
 
-	if (WARN_ON(core->prepare_count == 0))
+	if (WARN(core->prepare_count == 0,
+	    "%s already unprepared\n", core->name))
 		return;
 
-	if (WARN_ON(core->prepare_count == 1 && core->flags & CLK_IS_CRITICAL))
+	if (WARN(core->prepare_count == 1 && core->flags & CLK_IS_CRITICAL,
+	    "Unpreparing critical %s\n", core->name))
 		return;
 
 	if (--core->prepare_count > 0)
 		return;
 
-	WARN_ON(core->enable_count > 0);
+	WARN(core->enable_count > 0, "Unpreparing enabled %s\n", core->name);
 
 	trace_clk_unprepare(core);
 
@@ -809,10 +812,11 @@ static void clk_core_disable(struct clk_core *core)
 	if (!core)
 		return;
 
-	if (WARN_ON(core->enable_count == 0))
+	if (WARN(core->enable_count == 0, "%s already disabled\n", core->name))
 		return;
 
-	if (WARN_ON(core->enable_count == 1 && core->flags & CLK_IS_CRITICAL))
+	if (WARN(core->enable_count == 1 && core->flags & CLK_IS_CRITICAL,
+	    "Disabling critical %s\n", core->name))
 		return;
 
 	if (--core->enable_count > 0)
@@ -867,7 +871,8 @@ static int clk_core_enable(struct clk_core *core)
 	if (!core)
 		return 0;
 
-	if (WARN_ON(core->prepare_count == 0))
+	if (WARN(core->prepare_count == 0,
+	    "Enabling unprepared %s\n", core->name))
 		return -ESHUTDOWN;
 
 	if (core->enable_count == 0) {
@@ -3907,7 +3912,7 @@ int of_clk_parent_fill(struct device_node *np, const char **parents,
 EXPORT_SYMBOL_GPL(of_clk_parent_fill);
 
 struct clock_provider {
-	of_clk_init_cb_t clk_init_cb;
+	void (*clk_init_cb)(struct device_node *);
 	struct device_node *np;
 	struct list_head node;
 };
diff --git a/drivers/clk/davinci/pll-da830.c b/drivers/clk/davinci/pll-da830.c
index 929a3d3a9adb..0a0d06fb25fd 100644
--- a/drivers/clk/davinci/pll-da830.c
+++ b/drivers/clk/davinci/pll-da830.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/clkdev.h>
+#include <linux/clk/davinci.h>
 #include <linux/bitops.h>
 #include <linux/init.h>
 #include <linux/types.h>
@@ -36,11 +37,11 @@ SYSCLK(5, pll0_sysclk5, pll0_pllen, 5, 0);
 SYSCLK(6, pll0_sysclk6, pll0_pllen, 5, SYSCLK_FIXED_DIV);
 SYSCLK(7, pll0_sysclk7, pll0_pllen, 5, 0);
 
-int da830_pll_init(struct device *dev, void __iomem *base)
+int da830_pll_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
 	struct clk *clk;
 
-	davinci_pll_clk_register(dev, &da830_pll_info, "ref_clk", base);
+	davinci_pll_clk_register(dev, &da830_pll_info, "ref_clk", base, cfgchip);
 
 	clk = davinci_pll_sysclk_register(dev, &pll0_sysclk2, base);
 	clk_register_clkdev(clk, "pll0_sysclk2", "da830-psc0");
diff --git a/drivers/clk/davinci/pll-da850.c b/drivers/clk/davinci/pll-da850.c
index 2a038b7908cc..0f7198191ea2 100644
--- a/drivers/clk/davinci/pll-da850.c
+++ b/drivers/clk/davinci/pll-da850.c
@@ -7,10 +7,14 @@
 
 #include <linux/bitops.h>
 #include <linux/clk-provider.h>
+#include <linux/clk/davinci.h>
 #include <linux/clkdev.h>
+#include <linux/device.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/mfd/da8xx-cfgchip.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_address.h>
 #include <linux/of.h>
 #include <linux/types.h>
 
@@ -81,11 +85,11 @@ static const struct davinci_pll_obsclk_info da850_pll0_obsclk_info = {
 	.ocsrc_mask = GENMASK(4, 0),
 };
 
-int da850_pll0_init(struct device *dev, void __iomem *base)
+int da850_pll0_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
 	struct clk *clk;
 
-	davinci_pll_clk_register(dev, &da850_pll0_info, "ref_clk", base);
+	davinci_pll_clk_register(dev, &da850_pll0_info, "ref_clk", base, cfgchip);
 
 	clk = davinci_pll_sysclk_register(dev, &pll0_sysclk1, base);
 	clk_register_clkdev(clk, "pll0_sysclk1", "da850-psc0");
@@ -134,11 +138,22 @@ static const struct davinci_pll_sysclk_info *da850_pll0_sysclk_info[] = {
 	NULL
 };
 
-int of_da850_pll0_init(struct device *dev, void __iomem *base)
+void of_da850_pll0_init(struct device_node *node)
 {
-	return of_davinci_pll_init(dev, &da850_pll0_info,
-				   &da850_pll0_obsclk_info,
-				   da850_pll0_sysclk_info, 7, base);
+	void __iomem *base;
+	struct regmap *cfgchip;
+
+	base = of_iomap(node, 0);
+	if (!base) {
+		pr_err("%s: ioremap failed\n", __func__);
+		return;
+	}
+
+	cfgchip = syscon_regmap_lookup_by_compatible("ti,da830-cfgchip");
+
+	of_davinci_pll_init(NULL, node, &da850_pll0_info,
+			    &da850_pll0_obsclk_info,
+			    da850_pll0_sysclk_info, 7, base, cfgchip);
 }
 
 static const struct davinci_pll_clk_info da850_pll1_info = {
@@ -179,11 +194,11 @@ static const struct davinci_pll_obsclk_info da850_pll1_obsclk_info = {
 	.ocsrc_mask = GENMASK(4, 0),
 };
 
-int da850_pll1_init(struct device *dev, void __iomem *base)
+int da850_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
 	struct clk *clk;
 
-	davinci_pll_clk_register(dev, &da850_pll1_info, "oscin", base);
+	davinci_pll_clk_register(dev, &da850_pll1_info, "oscin", base, cfgchip);
 
 	davinci_pll_sysclk_register(dev, &pll1_sysclk1, base);
 
@@ -204,9 +219,9 @@ static const struct davinci_pll_sysclk_info *da850_pll1_sysclk_info[] = {
 	NULL
 };
 
-int of_da850_pll1_init(struct device *dev, void __iomem *base)
+int of_da850_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
-	return of_davinci_pll_init(dev, &da850_pll1_info,
+	return of_davinci_pll_init(dev, dev->of_node, &da850_pll1_info,
 				   &da850_pll1_obsclk_info,
-				   da850_pll1_sysclk_info, 3, base);
+				   da850_pll1_sysclk_info, 3, base, cfgchip);
 }
diff --git a/drivers/clk/davinci/pll-dm355.c b/drivers/clk/davinci/pll-dm355.c
index 5345f8286c50..505aed80be9a 100644
--- a/drivers/clk/davinci/pll-dm355.c
+++ b/drivers/clk/davinci/pll-dm355.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/bitops.h>
+#include <linux/clk/davinci.h>
 #include <linux/clkdev.h>
 #include <linux/init.h>
 #include <linux/types.h>
@@ -22,16 +23,16 @@ static const struct davinci_pll_clk_info dm355_pll1_info = {
 		 PLL_POSTDIV_ALWAYS_ENABLED | PLL_POSTDIV_FIXED_DIV,
 };
 
-SYSCLK(1, pll1_sysclk1, pll1, 5, SYSCLK_FIXED_DIV | SYSCLK_ALWAYS_ENABLED);
-SYSCLK(2, pll1_sysclk2, pll1, 5, SYSCLK_FIXED_DIV | SYSCLK_ALWAYS_ENABLED);
-SYSCLK(3, pll1_sysclk3, pll1, 5, SYSCLK_ALWAYS_ENABLED);
-SYSCLK(4, pll1_sysclk4, pll1, 5, SYSCLK_ALWAYS_ENABLED);
+SYSCLK(1, pll1_sysclk1, pll1_pllen, 5, SYSCLK_FIXED_DIV | SYSCLK_ALWAYS_ENABLED);
+SYSCLK(2, pll1_sysclk2, pll1_pllen, 5, SYSCLK_FIXED_DIV | SYSCLK_ALWAYS_ENABLED);
+SYSCLK(3, pll1_sysclk3, pll1_pllen, 5, SYSCLK_ALWAYS_ENABLED);
+SYSCLK(4, pll1_sysclk4, pll1_pllen, 5, SYSCLK_ALWAYS_ENABLED);
 
-int dm355_pll1_init(struct device *dev, void __iomem *base)
+int dm355_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
 	struct clk *clk;
 
-	davinci_pll_clk_register(dev, &dm355_pll1_info, "ref_clk", base);
+	davinci_pll_clk_register(dev, &dm355_pll1_info, "ref_clk", base, cfgchip);
 
 	clk = davinci_pll_sysclk_register(dev, &pll1_sysclk1, base);
 	clk_register_clkdev(clk, "pll1_sysclk1", "dm355-psc");
@@ -62,17 +63,14 @@ static const struct davinci_pll_clk_info dm355_pll2_info = {
 		 PLL_POSTDIV_ALWAYS_ENABLED | PLL_POSTDIV_FIXED_DIV,
 };
 
-SYSCLK(1, pll2_sysclk1, pll2, 5, SYSCLK_FIXED_DIV);
-SYSCLK(2, pll2_sysclk2, pll2, 5, SYSCLK_FIXED_DIV | SYSCLK_ALWAYS_ENABLED);
+SYSCLK(1, pll2_sysclk1, pll2_pllen, 5, SYSCLK_FIXED_DIV | SYSCLK_ALWAYS_ENABLED);
 
-int dm355_pll2_init(struct device *dev, void __iomem *base)
+int dm355_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
-	davinci_pll_clk_register(dev, &dm355_pll2_info, "oscin", base);
+	davinci_pll_clk_register(dev, &dm355_pll2_info, "oscin", base, cfgchip);
 
 	davinci_pll_sysclk_register(dev, &pll2_sysclk1, base);
 
-	davinci_pll_sysclk_register(dev, &pll2_sysclk2, base);
-
 	davinci_pll_sysclkbp_clk_register(dev, "pll2_sysclkbp", base);
 
 	return 0;
diff --git a/drivers/clk/davinci/pll-dm365.c b/drivers/clk/davinci/pll-dm365.c
index 5f8d9f42d0f3..2d29712753a3 100644
--- a/drivers/clk/davinci/pll-dm365.c
+++ b/drivers/clk/davinci/pll-dm365.c
@@ -7,6 +7,7 @@
 
 #include <linux/bitops.h>
 #include <linux/clkdev.h>
+#include <linux/clk/davinci.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/types.h>
@@ -56,11 +57,11 @@ static const struct davinci_pll_obsclk_info dm365_pll1_obsclk_info = {
 	.ocsrc_mask = BIT(4),
 };
 
-int dm365_pll1_init(struct device *dev, void __iomem *base)
+int dm365_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
 	struct clk *clk;
 
-	davinci_pll_clk_register(dev, &dm365_pll1_info, "ref_clk", base);
+	davinci_pll_clk_register(dev, &dm365_pll1_info, "ref_clk", base, cfgchip);
 
 	clk = davinci_pll_sysclk_register(dev, &pll1_sysclk1, base);
 	clk_register_clkdev(clk, "pll1_sysclk1", "dm365-psc");
@@ -119,11 +120,11 @@ static const struct davinci_pll_obsclk_info dm365_pll2_obsclk_info = {
 	.ocsrc_mask = BIT(4),
 };
 
-int dm365_pll2_init(struct device *dev, void __iomem *base)
+int dm365_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
 	struct clk *clk;
 
-	davinci_pll_clk_register(dev, &dm365_pll2_info, "oscin", base);
+	davinci_pll_clk_register(dev, &dm365_pll2_info, "oscin", base, cfgchip);
 
 	davinci_pll_sysclk_register(dev, &pll2_sysclk1, base);
 
diff --git a/drivers/clk/davinci/pll-dm644x.c b/drivers/clk/davinci/pll-dm644x.c
index 69bf785377cf..7650fadfaac8 100644
--- a/drivers/clk/davinci/pll-dm644x.c
+++ b/drivers/clk/davinci/pll-dm644x.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/bitops.h>
+#include <linux/clk/davinci.h>
 #include <linux/clkdev.h>
 #include <linux/init.h>
 #include <linux/types.h>
@@ -27,11 +28,11 @@ SYSCLK(2, pll1_sysclk2, pll1_pllen, 4, SYSCLK_FIXED_DIV);
 SYSCLK(3, pll1_sysclk3, pll1_pllen, 4, SYSCLK_FIXED_DIV);
 SYSCLK(5, pll1_sysclk5, pll1_pllen, 4, SYSCLK_FIXED_DIV);
 
-int dm644x_pll1_init(struct device *dev, void __iomem *base)
+int dm644x_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
 	struct clk *clk;
 
-	davinci_pll_clk_register(dev, &dm644x_pll1_info, "ref_clk", base);
+	davinci_pll_clk_register(dev, &dm644x_pll1_info, "ref_clk", base, cfgchip);
 
 	clk = davinci_pll_sysclk_register(dev, &pll1_sysclk1, base);
 	clk_register_clkdev(clk, "pll1_sysclk1", "dm644x-psc");
@@ -66,9 +67,9 @@ static const struct davinci_pll_clk_info dm644x_pll2_info = {
 SYSCLK(1, pll2_sysclk1, pll2_pllen, 4, 0);
 SYSCLK(2, pll2_sysclk2, pll2_pllen, 4, 0);
 
-int dm644x_pll2_init(struct device *dev, void __iomem *base)
+int dm644x_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
-	davinci_pll_clk_register(dev, &dm644x_pll2_info, "oscin", base);
+	davinci_pll_clk_register(dev, &dm644x_pll2_info, "oscin", base, cfgchip);
 
 	davinci_pll_sysclk_register(dev, &pll2_sysclk1, base);
 
diff --git a/drivers/clk/davinci/pll-dm646x.c b/drivers/clk/davinci/pll-dm646x.c
index a61cc3256418..26982970df0e 100644
--- a/drivers/clk/davinci/pll-dm646x.c
+++ b/drivers/clk/davinci/pll-dm646x.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/clk/davinci.h>
 #include <linux/clkdev.h>
 #include <linux/init.h>
 #include <linux/types.h>
@@ -29,11 +30,11 @@ SYSCLK(6, pll1_sysclk6, pll1_pllen, 4, 0);
 SYSCLK(8, pll1_sysclk8, pll1_pllen, 4, 0);
 SYSCLK(9, pll1_sysclk9, pll1_pllen, 4, 0);
 
-int dm646x_pll1_init(struct device *dev, void __iomem *base)
+int dm646x_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
 	struct clk *clk;
 
-	davinci_pll_clk_register(dev, &dm646x_pll1_info, "ref_clk", base);
+	davinci_pll_clk_register(dev, &dm646x_pll1_info, "ref_clk", base, cfgchip);
 
 	clk = davinci_pll_sysclk_register(dev, &pll1_sysclk1, base);
 	clk_register_clkdev(clk, "pll1_sysclk1", "dm646x-psc");
@@ -72,11 +73,11 @@ static const struct davinci_pll_clk_info dm646x_pll2_info = {
 	.flags = 0,
 };
 
-SYSCLK(1, pll2_sysclk1, pll2_pllen, 4, 0);
+SYSCLK(1, pll2_sysclk1, pll2_pllen, 4, SYSCLK_ALWAYS_ENABLED);
 
-int dm646x_pll2_init(struct device *dev, void __iomem *base)
+int dm646x_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip)
 {
-	davinci_pll_clk_register(dev, &dm646x_pll2_info, "oscin", base);
+	davinci_pll_clk_register(dev, &dm646x_pll2_info, "oscin", base, cfgchip);
 
 	davinci_pll_sysclk_register(dev, &pll2_sysclk1, base);
 
diff --git a/drivers/clk/davinci/pll.c b/drivers/clk/davinci/pll.c
index 23a24c944f1d..65abd371692d 100644
--- a/drivers/clk/davinci/pll.c
+++ b/drivers/clk/davinci/pll.c
@@ -11,6 +11,7 @@
 
 #include <linux/clk-provider.h>
 #include <linux/clk.h>
+#include <linux/clk/davinci.h>
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/io.h>
@@ -223,6 +224,7 @@ static const struct clk_ops dm365_pll_ops = {
 
 /**
  * davinci_pll_div_register - common *DIV clock implementation
+ * @dev: The PLL platform device or NULL
  * @name: the clock name
  * @parent_name: the parent clock name
  * @reg: the *DIV register
@@ -240,17 +242,21 @@ static struct clk *davinci_pll_div_register(struct device *dev,
 	const struct clk_ops *divider_ops = &clk_divider_ops;
 	struct clk_gate *gate;
 	struct clk_divider *divider;
+	struct clk *clk;
+	int ret;
 
-	gate = devm_kzalloc(dev, sizeof(*gate), GFP_KERNEL);
+	gate = kzalloc(sizeof(*gate), GFP_KERNEL);
 	if (!gate)
 		return ERR_PTR(-ENOMEM);
 
 	gate->reg = reg;
 	gate->bit_idx = DIV_ENABLE_SHIFT;
 
-	divider = devm_kzalloc(dev, sizeof(*divider), GFP_KERNEL);
-	if (!divider)
-		return ERR_PTR(-ENOMEM);
+	divider = kzalloc(sizeof(*divider), GFP_KERNEL);
+	if (!divider) {
+		ret = -ENOMEM;
+		goto err_free_gate;
+	}
 
 	divider->reg = reg;
 	divider->shift = DIV_RATIO_SHIFT;
@@ -261,9 +267,22 @@ static struct clk *davinci_pll_div_register(struct device *dev,
 		divider_ops = &clk_divider_ro_ops;
 	}
 
-	return clk_register_composite(dev, name, parent_names, num_parents,
-				      NULL, NULL, &divider->hw, divider_ops,
-				      &gate->hw, &clk_gate_ops, flags);
+	clk = clk_register_composite(dev, name, parent_names, num_parents,
+				     NULL, NULL, &divider->hw, divider_ops,
+				     &gate->hw, &clk_gate_ops, flags);
+	if (IS_ERR(clk)) {
+		ret = PTR_ERR(clk);
+		goto err_free_divider;
+	}
+
+	return clk;
+
+err_free_divider:
+	kfree(divider);
+err_free_gate:
+	kfree(gate);
+
+	return ERR_PTR(ret);
 }
 
 struct davinci_pllen_clk {
@@ -321,36 +340,17 @@ static int davinci_pllen_rate_change(struct notifier_block *nb,
 	return NOTIFY_OK;
 }
 
-static struct davinci_pll_platform_data *davinci_pll_get_pdata(struct device *dev)
-{
-	struct davinci_pll_platform_data *pdata = dev_get_platdata(dev);
-
-	/*
-	 * Platform data is optional, so allocate a new struct if one was not
-	 * provided. For device tree, this will always be the case.
-	 */
-	if (!pdata)
-		pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
-	if (!pdata)
-		return NULL;
-
-	/* for device tree, we need to fill in the struct */
-	if (dev->of_node)
-		pdata->cfgchip =
-			syscon_regmap_lookup_by_compatible("ti,da830-cfgchip");
-
-	return pdata;
-}
-
 static struct notifier_block davinci_pllen_notifier = {
 	.notifier_call = davinci_pllen_rate_change,
 };
 
 /**
  * davinci_pll_clk_register - Register a PLL clock
+ * @dev: The PLL platform device or NULL
  * @info: The device-specific clock info
  * @parent_name: The parent clock name
  * @base: The PLL's memory region
+ * @cfgchip: CFGCHIP syscon regmap for info->unlock_reg or NULL
  *
  * This creates a series of clocks that represent the PLL.
  *
@@ -366,9 +366,9 @@ static struct notifier_block davinci_pllen_notifier = {
 struct clk *davinci_pll_clk_register(struct device *dev,
 				     const struct davinci_pll_clk_info *info,
 				     const char *parent_name,
-				     void __iomem *base)
+				     void __iomem *base,
+				     struct regmap *cfgchip)
 {
-	struct davinci_pll_platform_data *pdata;
 	char prediv_name[MAX_NAME_SIZE];
 	char pllout_name[MAX_NAME_SIZE];
 	char postdiv_name[MAX_NAME_SIZE];
@@ -376,11 +376,12 @@ struct clk *davinci_pll_clk_register(struct device *dev,
 	struct clk_init_data init;
 	struct davinci_pll_clk *pllout;
 	struct davinci_pllen_clk *pllen;
-	struct clk *pllout_clk, *clk;
-
-	pdata = davinci_pll_get_pdata(dev);
-	if (!pdata)
-		return ERR_PTR(-ENOMEM);
+	struct clk *oscin_clk = NULL;
+	struct clk *prediv_clk = NULL;
+	struct clk *pllout_clk;
+	struct clk *postdiv_clk = NULL;
+	struct clk *pllen_clk;
+	int ret;
 
 	if (info->flags & PLL_HAS_CLKMODE) {
 		/*
@@ -392,10 +393,10 @@ struct clk *davinci_pll_clk_register(struct device *dev,
 		 * a number of different things. In this driver we use it to
 		 * mean the signal after the PLLCTL[CLKMODE] switch.
 		 */
-		clk = clk_register_fixed_factor(dev, OSCIN_CLK_NAME,
-						parent_name, 0, 1, 1);
-		if (IS_ERR(clk))
-			return clk;
+		oscin_clk = clk_register_fixed_factor(dev, OSCIN_CLK_NAME,
+						      parent_name, 0, 1, 1);
+		if (IS_ERR(oscin_clk))
+			return oscin_clk;
 
 		parent_name = OSCIN_CLK_NAME;
 	}
@@ -411,30 +412,34 @@ struct clk *davinci_pll_clk_register(struct device *dev,
 
 		/* Some? DM355 chips don't correctly report the PREDIV value */
 		if (info->flags & PLL_PREDIV_FIXED8)
-			clk = clk_register_fixed_factor(dev, prediv_name,
-						parent_name, flags, 1, 8);
+			prediv_clk = clk_register_fixed_factor(dev, prediv_name,
+							parent_name, flags, 1, 8);
 		else
-			clk = davinci_pll_div_register(dev, prediv_name,
+			prediv_clk = davinci_pll_div_register(dev, prediv_name,
 				parent_name, base + PREDIV, fixed, flags);
-		if (IS_ERR(clk))
-			return clk;
+		if (IS_ERR(prediv_clk)) {
+			ret = PTR_ERR(prediv_clk);
+			goto err_unregister_oscin;
+		}
 
 		parent_name = prediv_name;
 	}
 
 	/* Unlock writing to PLL registers */
 	if (info->unlock_reg) {
-		if (IS_ERR_OR_NULL(pdata->cfgchip))
+		if (IS_ERR_OR_NULL(cfgchip))
 			dev_warn(dev, "Failed to get CFGCHIP (%ld)\n",
-				 PTR_ERR(pdata->cfgchip));
+				 PTR_ERR(cfgchip));
 		else
-			regmap_write_bits(pdata->cfgchip, info->unlock_reg,
+			regmap_write_bits(cfgchip, info->unlock_reg,
 					  info->unlock_mask, 0);
 	}
 
-	pllout = devm_kzalloc(dev, sizeof(*pllout), GFP_KERNEL);
-	if (!pllout)
-		return ERR_PTR(-ENOMEM);
+	pllout = kzalloc(sizeof(*pllout), GFP_KERNEL);
+	if (!pllout) {
+		ret = -ENOMEM;
+		goto err_unregister_prediv;
+	}
 
 	snprintf(pllout_name, MAX_NAME_SIZE, "%s_pllout", info->name);
 
@@ -456,9 +461,11 @@ struct clk *davinci_pll_clk_register(struct device *dev,
 	pllout->pllm_min = info->pllm_min;
 	pllout->pllm_max = info->pllm_max;
 
-	pllout_clk = devm_clk_register(dev, &pllout->hw);
-	if (IS_ERR(pllout_clk))
-		return pllout_clk;
+	pllout_clk = clk_register(dev, &pllout->hw);
+	if (IS_ERR(pllout_clk)) {
+		ret = PTR_ERR(pllout_clk);
+		goto err_free_pllout;
+	}
 
 	clk_hw_set_rate_range(&pllout->hw, info->pllout_min_rate,
 			      info->pllout_max_rate);
@@ -474,17 +481,21 @@ struct clk *davinci_pll_clk_register(struct device *dev,
 		if (info->flags & PLL_POSTDIV_ALWAYS_ENABLED)
 			flags |= CLK_IS_CRITICAL;
 
-		clk = davinci_pll_div_register(dev, postdiv_name, parent_name,
-					       base + POSTDIV, fixed, flags);
-		if (IS_ERR(clk))
-			return clk;
+		postdiv_clk = davinci_pll_div_register(dev, postdiv_name,
+				parent_name, base + POSTDIV, fixed, flags);
+		if (IS_ERR(postdiv_clk)) {
+			ret = PTR_ERR(postdiv_clk);
+			goto err_unregister_pllout;
+		}
 
 		parent_name = postdiv_name;
 	}
 
-	pllen = devm_kzalloc(dev, sizeof(*pllout), GFP_KERNEL);
-	if (!pllen)
-		return ERR_PTR(-ENOMEM);
+	pllen = kzalloc(sizeof(*pllout), GFP_KERNEL);
+	if (!pllen) {
+		ret = -ENOMEM;
+		goto err_unregister_postdiv;
+	}
 
 	snprintf(pllen_name, MAX_NAME_SIZE, "%s_pllen", info->name);
 
@@ -497,17 +508,35 @@ struct clk *davinci_pll_clk_register(struct device *dev,
 	pllen->hw.init = &init;
 	pllen->base = base;
 
-	clk = devm_clk_register(dev, &pllen->hw);
-	if (IS_ERR(clk))
-		return clk;
+	pllen_clk = clk_register(dev, &pllen->hw);
+	if (IS_ERR(pllen_clk)) {
+		ret = PTR_ERR(pllen_clk);
+		goto err_free_pllen;
+	}
 
-	clk_notifier_register(clk, &davinci_pllen_notifier);
+	clk_notifier_register(pllen_clk, &davinci_pllen_notifier);
 
 	return pllout_clk;
+
+err_free_pllen:
+	kfree(pllen);
+err_unregister_postdiv:
+	clk_unregister(postdiv_clk);
+err_unregister_pllout:
+	clk_unregister(pllout_clk);
+err_free_pllout:
+	kfree(pllout);
+err_unregister_prediv:
+	clk_unregister(prediv_clk);
+err_unregister_oscin:
+	clk_unregister(oscin_clk);
+
+	return ERR_PTR(ret);
 }
 
 /**
  * davinci_pll_auxclk_register - Register bypass clock (AUXCLK)
+ * @dev: The PLL platform device or NULL
  * @name: The clock name
  * @base: The PLL memory region
  */
@@ -521,6 +550,7 @@ struct clk *davinci_pll_auxclk_register(struct device *dev,
 
 /**
  * davinci_pll_sysclkbp_clk_register - Register bypass divider clock (SYSCLKBP)
+ * @dev: The PLL platform device or NULL
  * @name: The clock name
  * @base: The PLL memory region
  */
@@ -535,6 +565,7 @@ struct clk *davinci_pll_sysclkbp_clk_register(struct device *dev,
 
 /**
  * davinci_pll_obsclk_register - Register oscillator divider clock (OBSCLK)
+ * @dev: The PLL platform device or NULL
  * @info: The clock info
  * @base: The PLL memory region
  */
@@ -546,9 +577,11 @@ davinci_pll_obsclk_register(struct device *dev,
 	struct clk_mux *mux;
 	struct clk_gate *gate;
 	struct clk_divider *divider;
+	struct clk *clk;
 	u32 oscdiv;
+	int ret;
 
-	mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL);
+	mux = kzalloc(sizeof(*mux), GFP_KERNEL);
 	if (!mux)
 		return ERR_PTR(-ENOMEM);
 
@@ -556,16 +589,20 @@ davinci_pll_obsclk_register(struct device *dev,
 	mux->table = info->table;
 	mux->mask = info->ocsrc_mask;
 
-	gate = devm_kzalloc(dev, sizeof(*gate), GFP_KERNEL);
-	if (!gate)
-		return ERR_PTR(-ENOMEM);
+	gate = kzalloc(sizeof(*gate), GFP_KERNEL);
+	if (!gate) {
+		ret = -ENOMEM;
+		goto err_free_mux;
+	}
 
 	gate->reg = base + CKEN;
 	gate->bit_idx = CKEN_OBSCLK_SHIFT;
 
-	divider = devm_kzalloc(dev, sizeof(*divider), GFP_KERNEL);
-	if (!divider)
-		return ERR_PTR(-ENOMEM);
+	divider = kzalloc(sizeof(*divider), GFP_KERNEL);
+	if (!divider) {
+		ret = -ENOMEM;
+		goto err_free_gate;
+	}
 
 	divider->reg = base + OSCDIV;
 	divider->shift = DIV_RATIO_SHIFT;
@@ -576,11 +613,27 @@ davinci_pll_obsclk_register(struct device *dev,
 	oscdiv |= BIT(DIV_ENABLE_SHIFT);
 	writel(oscdiv, base + OSCDIV);
 
-	return clk_register_composite(dev, info->name, info->parent_names,
-				      info->num_parents,
-				      &mux->hw, &clk_mux_ops,
-				      &divider->hw, &clk_divider_ops,
-				      &gate->hw, &clk_gate_ops, 0);
+	clk = clk_register_composite(dev, info->name, info->parent_names,
+				     info->num_parents,
+				     &mux->hw, &clk_mux_ops,
+				     &divider->hw, &clk_divider_ops,
+				     &gate->hw, &clk_gate_ops, 0);
+
+	if (IS_ERR(clk)) {
+		ret = PTR_ERR(clk);
+		goto err_free_divider;
+	}
+
+	return clk;
+
+err_free_divider:
+	kfree(divider);
+err_free_gate:
+	kfree(gate);
+err_free_mux:
+	kfree(mux);
+
+	return ERR_PTR(ret);
 }
 
 /* The PLL SYSCLKn clocks have a mechanism for synchronizing rate changes. */
@@ -616,6 +669,7 @@ static struct notifier_block davinci_pll_sysclk_notifier = {
 
 /**
  * davinci_pll_sysclk_register - Register divider clocks (SYSCLKn)
+ * @dev: The PLL platform device or NULL
  * @info: The clock info
  * @base: The PLL memory region
  */
@@ -630,6 +684,7 @@ davinci_pll_sysclk_register(struct device *dev,
 	struct clk *clk;
 	u32 reg;
 	u32 flags = 0;
+	int ret;
 
 	/* PLLDIVn registers are not entirely consecutive */
 	if (info->id < 4)
@@ -637,16 +692,18 @@ davinci_pll_sysclk_register(struct device *dev,
 	else
 		reg = PLLDIV4 + 4 * (info->id - 4);
 
-	gate = devm_kzalloc(dev, sizeof(*gate), GFP_KERNEL);
+	gate = kzalloc(sizeof(*gate), GFP_KERNEL);
 	if (!gate)
 		return ERR_PTR(-ENOMEM);
 
 	gate->reg = base + reg;
 	gate->bit_idx = DIV_ENABLE_SHIFT;
 
-	divider = devm_kzalloc(dev, sizeof(*divider), GFP_KERNEL);
-	if (!divider)
-		return ERR_PTR(-ENOMEM);
+	divider = kzalloc(sizeof(*divider), GFP_KERNEL);
+	if (!divider) {
+		ret = -ENOMEM;
+		goto err_free_gate;
+	}
 
 	divider->reg = base + reg;
 	divider->shift = DIV_RATIO_SHIFT;
@@ -668,22 +725,31 @@ davinci_pll_sysclk_register(struct device *dev,
 	clk = clk_register_composite(dev, info->name, &info->parent_name, 1,
 				     NULL, NULL, &divider->hw, divider_ops,
 				     &gate->hw, &clk_gate_ops, flags);
-	if (IS_ERR(clk))
-		return clk;
+	if (IS_ERR(clk)) {
+		ret = PTR_ERR(clk);
+		goto err_free_divider;
+	}
 
 	clk_notifier_register(clk, &davinci_pll_sysclk_notifier);
 
 	return clk;
+
+err_free_divider:
+	kfree(divider);
+err_free_gate:
+	kfree(gate);
+
+	return ERR_PTR(ret);
 }
 
-int of_davinci_pll_init(struct device *dev,
+int of_davinci_pll_init(struct device *dev, struct device_node *node,
 			const struct davinci_pll_clk_info *info,
 			const struct davinci_pll_obsclk_info *obsclk_info,
 			const struct davinci_pll_sysclk_info **div_info,
 			u8 max_sysclk_id,
-			void __iomem *base)
+			void __iomem *base,
+			struct regmap *cfgchip)
 {
-	struct device_node *node = dev->of_node;
 	struct device_node *child;
 	const char *parent_name;
 	struct clk *clk;
@@ -693,7 +759,7 @@ int of_davinci_pll_init(struct device *dev,
 	else
 		parent_name = OSCIN_CLK_NAME;
 
-	clk = davinci_pll_clk_register(dev, info, parent_name, base);
+	clk = davinci_pll_clk_register(dev, info, parent_name, base, cfgchip);
 	if (IS_ERR(clk)) {
 		dev_err(dev, "failed to register %s\n", info->name);
 		return PTR_ERR(clk);
@@ -711,13 +777,15 @@ int of_davinci_pll_init(struct device *dev,
 		int n_clks =  max_sysclk_id + 1;
 		int i;
 
-		clk_data = devm_kzalloc(dev, sizeof(*clk_data), GFP_KERNEL);
+		clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
 		if (!clk_data)
 			return -ENOMEM;
 
-		clks = devm_kmalloc_array(dev, n_clks, sizeof(*clks), GFP_KERNEL);
-		if (!clks)
+		clks = kmalloc_array(n_clks, sizeof(*clks), GFP_KERNEL);
+		if (!clks) {
+			kfree(clk_data);
 			return -ENOMEM;
+		}
 
 		clk_data->clks = clks;
 		clk_data->clk_num = n_clks;
@@ -770,32 +838,73 @@ int of_davinci_pll_init(struct device *dev,
 	return 0;
 }
 
+static struct davinci_pll_platform_data *davinci_pll_get_pdata(struct device *dev)
+{
+	struct davinci_pll_platform_data *pdata = dev_get_platdata(dev);
+
+	/*
+	 * Platform data is optional, so allocate a new struct if one was not
+	 * provided. For device tree, this will always be the case.
+	 */
+	if (!pdata)
+		pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
+	if (!pdata)
+		return NULL;
+
+	/* for device tree, we need to fill in the struct */
+	if (dev->of_node)
+		pdata->cfgchip =
+			syscon_regmap_lookup_by_compatible("ti,da830-cfgchip");
+
+	return pdata;
+}
+
+/* needed in early boot for clocksource/clockevent */
+#ifdef CONFIG_ARCH_DAVINCI_DA850
+CLK_OF_DECLARE(da850_pll0, "ti,da850-pll0", of_da850_pll0_init);
+#endif
+
 static const struct of_device_id davinci_pll_of_match[] = {
-	{ .compatible = "ti,da850-pll0", .data = of_da850_pll0_init },
+#ifdef CONFIG_ARCH_DAVINCI_DA850
 	{ .compatible = "ti,da850-pll1", .data = of_da850_pll1_init },
+#endif
 	{ }
 };
 
 static const struct platform_device_id davinci_pll_id_table[] = {
+#ifdef CONFIG_ARCH_DAVINCI_DA830
 	{ .name = "da830-pll",   .driver_data = (kernel_ulong_t)da830_pll_init   },
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DA850
 	{ .name = "da850-pll0",  .driver_data = (kernel_ulong_t)da850_pll0_init  },
 	{ .name = "da850-pll1",  .driver_data = (kernel_ulong_t)da850_pll1_init  },
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM355
 	{ .name = "dm355-pll1",  .driver_data = (kernel_ulong_t)dm355_pll1_init  },
 	{ .name = "dm355-pll2",  .driver_data = (kernel_ulong_t)dm355_pll2_init  },
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM365
 	{ .name = "dm365-pll1",  .driver_data = (kernel_ulong_t)dm365_pll1_init  },
 	{ .name = "dm365-pll2",  .driver_data = (kernel_ulong_t)dm365_pll2_init  },
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM644x
 	{ .name = "dm644x-pll1", .driver_data = (kernel_ulong_t)dm644x_pll1_init },
 	{ .name = "dm644x-pll2", .driver_data = (kernel_ulong_t)dm644x_pll2_init },
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM646x
 	{ .name = "dm646x-pll1", .driver_data = (kernel_ulong_t)dm646x_pll1_init },
 	{ .name = "dm646x-pll2", .driver_data = (kernel_ulong_t)dm646x_pll2_init },
+#endif
 	{ }
 };
 
-typedef int (*davinci_pll_init)(struct device *dev, void __iomem *base);
+typedef int (*davinci_pll_init)(struct device *dev, void __iomem *base,
+				struct regmap *cfgchip);
 
 static int davinci_pll_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	struct davinci_pll_platform_data *pdata;
 	const struct of_device_id *of_id;
 	davinci_pll_init pll_init = NULL;
 	struct resource *res;
@@ -812,12 +921,18 @@ static int davinci_pll_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
+	pdata = davinci_pll_get_pdata(dev);
+	if (!pdata) {
+		dev_err(dev, "missing platform data\n");
+		return -EINVAL;
+	}
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	base = devm_ioremap_resource(dev, res);
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
-	return pll_init(dev, base);
+	return pll_init(dev, base, pdata->cfgchip);
 }
 
 static struct platform_driver davinci_pll_driver = {
diff --git a/drivers/clk/davinci/pll.h b/drivers/clk/davinci/pll.h
index b1b6fb23f972..7cc354dd29e2 100644
--- a/drivers/clk/davinci/pll.h
+++ b/drivers/clk/davinci/pll.h
@@ -11,6 +11,7 @@
 #include <linux/bitops.h>
 #include <linux/clk-provider.h>
 #include <linux/of.h>
+#include <linux/regmap.h>
 #include <linux/types.h>
 
 #define PLL_HAS_CLKMODE			BIT(0) /* PLL has PLLCTL[CLKMODE] */
@@ -94,7 +95,8 @@ struct davinci_pll_obsclk_info {
 struct clk *davinci_pll_clk_register(struct device *dev,
 				     const struct davinci_pll_clk_info *info,
 				     const char *parent_name,
-				     void __iomem *base);
+				     void __iomem *base,
+				     struct regmap *cfgchip);
 struct clk *davinci_pll_auxclk_register(struct device *dev,
 					const char *name,
 					void __iomem *base);
@@ -110,32 +112,29 @@ davinci_pll_sysclk_register(struct device *dev,
 			    const struct davinci_pll_sysclk_info *info,
 			    void __iomem *base);
 
-int of_davinci_pll_init(struct device *dev,
+int of_davinci_pll_init(struct device *dev, struct device_node *node,
 			const struct davinci_pll_clk_info *info,
 			const struct davinci_pll_obsclk_info *obsclk_info,
 			const struct davinci_pll_sysclk_info **div_info,
 			u8 max_sysclk_id,
-			void __iomem *base);
+			void __iomem *base,
+			struct regmap *cfgchip);
 
 /* Platform-specific callbacks */
 
-int da830_pll_init(struct device *dev, void __iomem *base);
-
-int da850_pll0_init(struct device *dev, void __iomem *base);
-int da850_pll1_init(struct device *dev, void __iomem *base);
-int of_da850_pll0_init(struct device *dev, void __iomem *base);
-int of_da850_pll1_init(struct device *dev, void __iomem *base);
-
-int dm355_pll1_init(struct device *dev, void __iomem *base);
-int dm355_pll2_init(struct device *dev, void __iomem *base);
-
-int dm365_pll1_init(struct device *dev, void __iomem *base);
-int dm365_pll2_init(struct device *dev, void __iomem *base);
-
-int dm644x_pll1_init(struct device *dev, void __iomem *base);
-int dm644x_pll2_init(struct device *dev, void __iomem *base);
-
-int dm646x_pll1_init(struct device *dev, void __iomem *base);
-int dm646x_pll2_init(struct device *dev, void __iomem *base);
+#ifdef CONFIG_ARCH_DAVINCI_DA850
+int da850_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
+void of_da850_pll0_init(struct device_node *node);
+int of_da850_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM355
+int dm355_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM644x
+int dm644x_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM646x
+int dm646x_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip);
+#endif
 
 #endif /* __CLK_DAVINCI_PLL_H___ */
diff --git a/drivers/clk/davinci/psc-da830.c b/drivers/clk/davinci/psc-da830.c
index f61abf5632ff..081b039fcb02 100644
--- a/drivers/clk/davinci/psc-da830.c
+++ b/drivers/clk/davinci/psc-da830.c
@@ -55,7 +55,8 @@ const struct davinci_psc_init_data da830_psc0_init_data = {
 	.psc_init		= &da830_psc0_init,
 };
 
-LPSC_CLKDEV2(usb0_clkdev,	NULL,	"musb-da8xx",
+LPSC_CLKDEV3(usb0_clkdev,	"fck",	"da830-usb-phy-clks",
+				NULL,	"musb-da8xx",
 				NULL,	"cppi41-dmaengine");
 LPSC_CLKDEV1(usb1_clkdev,	NULL,	"ohci-da8xx");
 /* REVISIT: gpio-davinci.c should be modified to drop con_id */
diff --git a/drivers/clk/davinci/psc-dm355.c b/drivers/clk/davinci/psc-dm355.c
index 6995ecea2677..ddd250107c4e 100644
--- a/drivers/clk/davinci/psc-dm355.c
+++ b/drivers/clk/davinci/psc-dm355.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/clk/davinci.h>
 #include <linux/clk.h>
 #include <linux/clkdev.h>
 #include <linux/init.h>
@@ -41,14 +42,14 @@ static const struct davinci_lpsc_clk_info dm355_psc_info[] = {
 	LPSC(5,  0, timer3,      pll1_auxclk,  NULL,               0),
 	LPSC(6,  0, spi1,        pll1_sysclk2, spi1_clkdev,        0),
 	LPSC(7,  0, mmcsd1,      pll1_sysclk2, mmcsd1_clkdev,      0),
-	LPSC(8,  0, asp1,        pll1_sysclk2, NULL,               0),
+	LPSC(8,  0, asp1,        pll1_sysclk2, mcbsp1_clkdev,      0),
 	LPSC(9,  0, usb,         pll1_sysclk2, usb_clkdev,         0),
 	LPSC(10, 0, pwm3,        pll1_auxclk,  NULL,               0),
 	LPSC(11, 0, spi2,        pll1_sysclk2, spi2_clkdev,        0),
 	LPSC(12, 0, rto,         pll1_auxclk,  NULL,               0),
 	LPSC(14, 0, aemif,       pll1_sysclk2, aemif_clkdev,       0),
 	LPSC(15, 0, mmcsd0,      pll1_sysclk2, mmcsd0_clkdev,      0),
-	LPSC(17, 0, asp0,        pll1_sysclk2, NULL,               0),
+	LPSC(17, 0, asp0,        pll1_sysclk2, mcbsp0_clkdev,      0),
 	LPSC(18, 0, i2c,         pll1_auxclk,  i2c_clkdev,         0),
 	LPSC(19, 0, uart0,       pll1_auxclk,  uart0_clkdev,       0),
 	LPSC(20, 0, uart1,       pll1_auxclk,  uart1_clkdev,       0),
@@ -68,7 +69,7 @@ static const struct davinci_lpsc_clk_info dm355_psc_info[] = {
 	{ }
 };
 
-static int dm355_psc_init(struct device *dev, void __iomem *base)
+int dm355_psc_init(struct device *dev, void __iomem *base)
 {
 	return davinci_psc_register_clocks(dev, dm355_psc_info, 42, base);
 }
diff --git a/drivers/clk/davinci/psc-dm365.c b/drivers/clk/davinci/psc-dm365.c
index 3ad915f37376..8c73086cc676 100644
--- a/drivers/clk/davinci/psc-dm365.c
+++ b/drivers/clk/davinci/psc-dm365.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/clk/davinci.h>
 #include <linux/clk.h>
 #include <linux/clkdev.h>
 #include <linux/init.h>
@@ -65,15 +66,28 @@ static const struct davinci_lpsc_clk_info dm365_psc_info[] = {
 	LPSC(31, 0, arm,         pll2_sysclk2, NULL,               LPSC_ALWAYS_ENABLED),
 	LPSC(38, 0, spi3,        pll1_sysclk4, spi3_clkdev,        0),
 	LPSC(39, 0, spi4,        pll1_auxclk,  spi4_clkdev,        0),
-	LPSC(40, 0, emac,        pll2_sysclk4, emac_clkdev,        0),
-	LPSC(44, 1, voice_codec, pll1_sysclk3, voice_codec_clkdev, 0),
-	LPSC(46, 1, vpss_dac,    pll1_sysclk3, vpss_dac_clkdev,    0),
+	LPSC(40, 0, emac,        pll1_sysclk4, emac_clkdev,        0),
+	/*
+	 * The TRM (ARM Subsystem User's Guide) shows two clocks input into
+	 * voice codec module (PLL2 SYSCLK4 with a DIV2 and PLL1 SYSCLK4). Its
+	 * not fully clear from documentation which clock should be considered
+	 * as parent for PSC. The clock chosen here is to maintain
+	 * compatibility with existing code in arch/arm/mach-davinci/dm365.c
+	 */
+	LPSC(44, 0, voice_codec, pll2_sysclk4, voice_codec_clkdev, 0),
+	/*
+	 * Its not fully clear from TRM (ARM Subsystem User's Guide) as to what
+	 * the parent of VPSS DAC LPSC should actually be. PLL1 SYSCLK3 feeds
+	 * into HDVICP and MJCP. The clock chosen here is to remain compatible
+	 * with code existing in arch/arm/mach-davinci/dm365.c
+	 */
+	LPSC(46, 0, vpss_dac,    pll1_sysclk3, vpss_dac_clkdev,    0),
 	LPSC(47, 0, vpss_master, pll1_sysclk5, vpss_master_clkdev, 0),
 	LPSC(50, 0, mjcp,        pll1_sysclk3, NULL,               0),
 	{ }
 };
 
-static int dm365_psc_init(struct device *dev, void __iomem *base)
+int dm365_psc_init(struct device *dev, void __iomem *base)
 {
 	return davinci_psc_register_clocks(dev, dm365_psc_info, 52, base);
 }
diff --git a/drivers/clk/davinci/psc-dm644x.c b/drivers/clk/davinci/psc-dm644x.c
index c22367baa46f..fc0230e3a3d6 100644
--- a/drivers/clk/davinci/psc-dm644x.c
+++ b/drivers/clk/davinci/psc-dm644x.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/clk/davinci.h>
 #include <linux/clk.h>
 #include <linux/clkdev.h>
 #include <linux/init.h>
@@ -63,7 +64,7 @@ static const struct davinci_lpsc_clk_info dm644x_psc_info[] = {
 	{ }
 };
 
-static int dm644x_psc_init(struct device *dev, void __iomem *base)
+int dm644x_psc_init(struct device *dev, void __iomem *base)
 {
 	return davinci_psc_register_clocks(dev, dm644x_psc_info, 41, base);
 }
diff --git a/drivers/clk/davinci/psc-dm646x.c b/drivers/clk/davinci/psc-dm646x.c
index 468ef86ea40b..c3f82ed70a80 100644
--- a/drivers/clk/davinci/psc-dm646x.c
+++ b/drivers/clk/davinci/psc-dm646x.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/clk/davinci.h>
 #include <linux/clk.h>
 #include <linux/clkdev.h>
 #include <linux/init.h>
@@ -58,7 +59,7 @@ static const struct davinci_lpsc_clk_info dm646x_psc_info[] = {
 	{ }
 };
 
-static int dm646x_psc_init(struct device *dev, void __iomem *base)
+int dm646x_psc_init(struct device *dev, void __iomem *base)
 {
 	return davinci_psc_register_clocks(dev, dm646x_psc_info, 46, base);
 }
diff --git a/drivers/clk/davinci/psc.c b/drivers/clk/davinci/psc.c
index ce170e600f09..fffbed5e263b 100644
--- a/drivers/clk/davinci/psc.c
+++ b/drivers/clk/davinci/psc.c
@@ -15,6 +15,7 @@
 
 #include <linux/clk-provider.h>
 #include <linux/clk.h>
+#include <linux/clk/davinci.h>
 #include <linux/clkdev.h>
 #include <linux/err.h>
 #include <linux/of_address.h>
@@ -63,7 +64,7 @@ struct davinci_psc_data {
 
 /**
  * struct davinci_lpsc_clk - LPSC clock structure
- * @dev: the device that provides this LPSC
+ * @dev: the device that provides this LPSC or NULL
  * @hw: clk_hw for the LPSC
  * @pm_domain: power domain for the LPSC
  * @genpd_clk: clock reference owned by @pm_domain
@@ -221,6 +222,7 @@ static void davinci_psc_genpd_detach_dev(struct generic_pm_domain *pm_domain,
 
 /**
  * davinci_lpsc_clk_register - register LPSC clock
+ * @dev: the clocks's device or NULL
  * @name: name of this clock
  * @parent_name: name of clock's parent
  * @regmap: PSC MMIO region
@@ -238,7 +240,7 @@ davinci_lpsc_clk_register(struct device *dev, const char *name,
 	int ret;
 	bool is_on;
 
-	lpsc = devm_kzalloc(dev, sizeof(*lpsc), GFP_KERNEL);
+	lpsc = kzalloc(sizeof(*lpsc), GFP_KERNEL);
 	if (!lpsc)
 		return ERR_PTR(-ENOMEM);
 
@@ -261,9 +263,15 @@ davinci_lpsc_clk_register(struct device *dev, const char *name,
 	lpsc->pd = pd;
 	lpsc->flags = flags;
 
-	ret = devm_clk_hw_register(dev, &lpsc->hw);
-	if (ret < 0)
+	ret = clk_hw_register(dev, &lpsc->hw);
+	if (ret < 0) {
+		kfree(lpsc);
 		return ERR_PTR(ret);
+	}
+
+	/* for now, genpd is only registered when using device-tree */
+	if (!dev || !dev->of_node)
+		return lpsc;
 
 	/* genpd attach needs a way to look up this clock */
 	ret = clk_hw_register_clkdev(&lpsc->hw, name, best_dev_name(dev));
@@ -378,13 +386,15 @@ __davinci_psc_register_clocks(struct device *dev,
 	struct regmap *regmap;
 	int i, ret;
 
-	psc = devm_kzalloc(dev, sizeof(*psc), GFP_KERNEL);
+	psc = kzalloc(sizeof(*psc), GFP_KERNEL);
 	if (!psc)
 		return ERR_PTR(-ENOMEM);
 
-	clks = devm_kmalloc_array(dev, num_clks, sizeof(*clks), GFP_KERNEL);
-	if (!clks)
-		return ERR_PTR(-ENOMEM);
+	clks = kmalloc_array(num_clks, sizeof(*clks), GFP_KERNEL);
+	if (!clks) {
+		ret = -ENOMEM;
+		goto err_free_psc;
+	}
 
 	psc->clk_data.clks = clks;
 	psc->clk_data.clk_num = num_clks;
@@ -396,16 +406,20 @@ __davinci_psc_register_clocks(struct device *dev,
 	for (i = 0; i < num_clks; i++)
 		clks[i] = ERR_PTR(-ENOENT);
 
-	pm_domains = devm_kcalloc(dev, num_clks, sizeof(*pm_domains), GFP_KERNEL);
-	if (!pm_domains)
-		return ERR_PTR(-ENOMEM);
+	pm_domains = kcalloc(num_clks, sizeof(*pm_domains), GFP_KERNEL);
+	if (!pm_domains) {
+		ret = -ENOMEM;
+		goto err_free_clks;
+	}
 
 	psc->pm_data.domains = pm_domains;
 	psc->pm_data.num_domains = num_clks;
 
-	regmap = devm_regmap_init_mmio(dev, base, &davinci_psc_regmap_config);
-	if (IS_ERR(regmap))
-		return ERR_CAST(regmap);
+	regmap = regmap_init_mmio(dev, base, &davinci_psc_regmap_config);
+	if (IS_ERR(regmap)) {
+		ret = PTR_ERR(regmap);
+		goto err_free_pm_domains;
+	}
 
 	for (; info->name; info++) {
 		struct davinci_lpsc_clk *lpsc;
@@ -423,6 +437,13 @@ __davinci_psc_register_clocks(struct device *dev,
 		pm_domains[info->md] = &lpsc->pm_domain;
 	}
 
+	/*
+	 * for now, a reset controller is only registered when there is a device
+	 * to associate it with.
+	 */
+	if (!dev)
+		return psc;
+
 	psc->rcdev.ops = &davinci_psc_reset_ops;
 	psc->rcdev.owner = THIS_MODULE;
 	psc->rcdev.dev = dev;
@@ -436,6 +457,15 @@ __davinci_psc_register_clocks(struct device *dev,
 		dev_warn(dev, "Failed to register reset controller (%d)\n", ret);
 
 	return psc;
+
+err_free_pm_domains:
+	kfree(pm_domains);
+err_free_clks:
+	kfree(clks);
+err_free_psc:
+	kfree(psc);
+
+	return ERR_PTR(ret);
 }
 
 int davinci_psc_register_clocks(struct device *dev,
@@ -483,20 +513,34 @@ int of_davinci_psc_clk_init(struct device *dev,
 }
 
 static const struct of_device_id davinci_psc_of_match[] = {
+#ifdef CONFIG_ARCH_DAVINCI_DA850
 	{ .compatible = "ti,da850-psc0", .data = &of_da850_psc0_init_data },
 	{ .compatible = "ti,da850-psc1", .data = &of_da850_psc1_init_data },
+#endif
 	{ }
 };
 
 static const struct platform_device_id davinci_psc_id_table[] = {
+#ifdef CONFIG_ARCH_DAVINCI_DA830
 	{ .name = "da830-psc0", .driver_data = (kernel_ulong_t)&da830_psc0_init_data },
 	{ .name = "da830-psc1", .driver_data = (kernel_ulong_t)&da830_psc1_init_data },
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DA850
 	{ .name = "da850-psc0", .driver_data = (kernel_ulong_t)&da850_psc0_init_data },
 	{ .name = "da850-psc1", .driver_data = (kernel_ulong_t)&da850_psc1_init_data },
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM355
 	{ .name = "dm355-psc",  .driver_data = (kernel_ulong_t)&dm355_psc_init_data  },
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM365
 	{ .name = "dm365-psc",  .driver_data = (kernel_ulong_t)&dm365_psc_init_data  },
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM644x
 	{ .name = "dm644x-psc", .driver_data = (kernel_ulong_t)&dm644x_psc_init_data },
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM646x
 	{ .name = "dm646x-psc", .driver_data = (kernel_ulong_t)&dm646x_psc_init_data },
+#endif
 	{ }
 };
 
diff --git a/drivers/clk/davinci/psc.h b/drivers/clk/davinci/psc.h
index c2a7df6413fe..6a42529d31a9 100644
--- a/drivers/clk/davinci/psc.h
+++ b/drivers/clk/davinci/psc.h
@@ -94,15 +94,27 @@ struct davinci_psc_init_data {
 	int (*psc_init)(struct device *dev, void __iomem *base);
 };
 
+#ifdef CONFIG_ARCH_DAVINCI_DA830
 extern const struct davinci_psc_init_data da830_psc0_init_data;
 extern const struct davinci_psc_init_data da830_psc1_init_data;
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DA850
 extern const struct davinci_psc_init_data da850_psc0_init_data;
 extern const struct davinci_psc_init_data da850_psc1_init_data;
 extern const struct davinci_psc_init_data of_da850_psc0_init_data;
 extern const struct davinci_psc_init_data of_da850_psc1_init_data;
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM355
 extern const struct davinci_psc_init_data dm355_psc_init_data;
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM356
 extern const struct davinci_psc_init_data dm365_psc_init_data;
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM644x
 extern const struct davinci_psc_init_data dm644x_psc_init_data;
+#endif
+#ifdef CONFIG_ARCH_DAVINCI_DM646x
 extern const struct davinci_psc_init_data dm646x_psc_init_data;
+#endif
 
 #endif /* __CLK_DAVINCI_PSC_H__ */
diff --git a/drivers/clk/hisilicon/crg-hi3798cv200.c b/drivers/clk/hisilicon/crg-hi3798cv200.c
index 743eec131528..4fe0b2a9baf1 100644
--- a/drivers/clk/hisilicon/crg-hi3798cv200.c
+++ b/drivers/clk/hisilicon/crg-hi3798cv200.c
@@ -186,6 +186,23 @@ static const struct hisi_gate_clock hi3798cv200_gate_clks[] = {
 		CLK_SET_RATE_PARENT, 0xbc, 0, 0 },
 	{ HISTB_USB2_PHY2_REF_CLK, "clk_u2_phy2_ref", "24m",
 		CLK_SET_RATE_PARENT, 0xbc, 2, 0 },
+	/* USB3 */
+	{ HISTB_USB3_BUS_CLK, "clk_u3_bus", NULL,
+		CLK_SET_RATE_PARENT, 0xb0, 0, 0 },
+	{ HISTB_USB3_UTMI_CLK, "clk_u3_utmi", NULL,
+		CLK_SET_RATE_PARENT, 0xb0, 4, 0 },
+	{ HISTB_USB3_PIPE_CLK, "clk_u3_pipe", NULL,
+		CLK_SET_RATE_PARENT, 0xb0, 3, 0 },
+	{ HISTB_USB3_SUSPEND_CLK, "clk_u3_suspend", NULL,
+		CLK_SET_RATE_PARENT, 0xb0, 2, 0 },
+	{ HISTB_USB3_BUS_CLK1, "clk_u3_bus1", NULL,
+		CLK_SET_RATE_PARENT, 0xb0, 16, 0 },
+	{ HISTB_USB3_UTMI_CLK1, "clk_u3_utmi1", NULL,
+		CLK_SET_RATE_PARENT, 0xb0, 20, 0 },
+	{ HISTB_USB3_PIPE_CLK1, "clk_u3_pipe1", NULL,
+		CLK_SET_RATE_PARENT, 0xb0, 19, 0 },
+	{ HISTB_USB3_SUSPEND_CLK1, "clk_u3_suspend1", NULL,
+		CLK_SET_RATE_PARENT, 0xb0, 18, 0 },
 };
 
 static struct hisi_clock_data *hi3798cv200_clk_register(
diff --git a/drivers/clk/imx/clk-imx6sx.c b/drivers/clk/imx/clk-imx6sx.c
index bc3f9ebf2d9e..0178ee26a53a 100644
--- a/drivers/clk/imx/clk-imx6sx.c
+++ b/drivers/clk/imx/clk-imx6sx.c
@@ -80,7 +80,7 @@ static const char *lvds_sels[]	= {
 	"arm", "pll1_sys", "dummy", "dummy", "dummy", "dummy", "dummy", "pll5_video_div",
 	"dummy", "dummy", "pcie_ref_125m", "dummy", "usbphy1", "usbphy2",
 };
-static const char *pll_bypass_src_sels[] = { "osc", "lvds1_in", };
+static const char *pll_bypass_src_sels[] = { "osc", "lvds1_in", "lvds2_in", "dummy", };
 static const char *pll1_bypass_sels[] = { "pll1", "pll1_bypass_src", };
 static const char *pll2_bypass_sels[] = { "pll2", "pll2_bypass_src", };
 static const char *pll3_bypass_sels[] = { "pll3", "pll3_bypass_src", };
@@ -158,8 +158,9 @@ static void __init imx6sx_clocks_init(struct device_node *ccm_node)
 	clks[IMX6SX_CLK_IPP_DI0] = of_clk_get_by_name(ccm_node, "ipp_di0");
 	clks[IMX6SX_CLK_IPP_DI1] = of_clk_get_by_name(ccm_node, "ipp_di1");
 
-	/* Clock source from external clock via CLK1 PAD */
-	clks[IMX6SX_CLK_ANACLK1] = imx_obtain_fixed_clock("anaclk1", 0);
+	/* Clock source from external clock via CLK1/2 PAD */
+	clks[IMX6SX_CLK_ANACLK1] = of_clk_get_by_name(ccm_node, "anaclk1");
+	clks[IMX6SX_CLK_ANACLK2] = of_clk_get_by_name(ccm_node, "anaclk2");
 
 	np = of_find_compatible_node(NULL, NULL, "fsl,imx6sx-anatop");
 	base = of_iomap(np, 0);
@@ -228,7 +229,9 @@ static void __init imx6sx_clocks_init(struct device_node *ccm_node)
 	clks[IMX6SX_CLK_PCIE_REF_125M] = imx_clk_gate("pcie_ref_125m", "pcie_ref", base + 0xe0, 19);
 
 	clks[IMX6SX_CLK_LVDS1_OUT] = imx_clk_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x160, 10, BIT(12));
+	clks[IMX6SX_CLK_LVDS2_OUT] = imx_clk_gate_exclusive("lvds2_out", "lvds2_sel", base + 0x160, 11, BIT(13));
 	clks[IMX6SX_CLK_LVDS1_IN]  = imx_clk_gate_exclusive("lvds1_in",  "anaclk1",   base + 0x160, 12, BIT(10));
+	clks[IMX6SX_CLK_LVDS2_IN]  = imx_clk_gate_exclusive("lvds2_in",  "anaclk2",   base + 0x160, 13, BIT(11));
 
 	clks[IMX6SX_CLK_ENET_REF] = clk_register_divider_table(NULL, "enet_ref", "pll6_enet", 0,
 			base + 0xe0, 0, 2, 0, clk_enet_ref_table,
@@ -270,6 +273,7 @@ static void __init imx6sx_clocks_init(struct device_node *ccm_node)
 
 	/*                                                name                reg           shift   width   parent_names       num_parents */
 	clks[IMX6SX_CLK_LVDS1_SEL]          = imx_clk_mux("lvds1_sel",        base + 0x160, 0,      5,      lvds_sels,         ARRAY_SIZE(lvds_sels));
+	clks[IMX6SX_CLK_LVDS2_SEL]          = imx_clk_mux("lvds2_sel",        base + 0x160, 5,      5,      lvds_sels,         ARRAY_SIZE(lvds_sels));
 
 	np = ccm_node;
 	base = of_iomap(np, 0);
diff --git a/drivers/clk/imx/clk-imx6ul.c b/drivers/clk/imx/clk-imx6ul.c
index 12320118f8de..ba563ba50b40 100644
--- a/drivers/clk/imx/clk-imx6ul.c
+++ b/drivers/clk/imx/clk-imx6ul.c
@@ -40,7 +40,7 @@ static const char *axi_alt_sels[] = { "pll2_pfd2_396m", "pll3_pfd1_540m", };
 static const char *axi_sels[] = {"periph", "axi_alt_sel", };
 static const char *periph_pre_sels[] = { "pll2_bus", "pll2_pfd2_396m", "pll2_pfd0_352m", "pll2_198m", };
 static const char *periph2_pre_sels[] = { "pll2_bus", "pll2_pfd2_396m", "pll2_pfd0_352m", "pll4_audio_div", };
-static const char *periph_clk2_sels[] = { "pll3_usb_otg", "osc", "osc", };
+static const char *periph_clk2_sels[] = { "pll3_usb_otg", "osc", "pll2_bypass_src", };
 static const char *periph2_clk2_sels[] = { "pll3_usb_otg", "osc", };
 static const char *periph_sels[] = { "periph_pre", "periph_clk2", };
 static const char *periph2_sels[] = { "periph2_pre", "periph2_clk2", };
@@ -68,6 +68,13 @@ static const char *sim_sels[] = { "sim_podf", "ipp_di0", "ipp_di1", "ldb_di0", "
 static const char *epdc_pre_sels[] = { "pll2_bus", "pll3_usb_otg", "pll5_video_div", "pll2_pfd0_352m", "pll2_pfd2_396m", "pll3_pfd2_508m", };
 static const char *esai_sels[] = { "pll4_audio_div", "pll3_pfd2_508m", "pll5_video_div", "pll3_usb_otg", };
 static const char *epdc_sels[] = { "epdc_podf", "ipp_di0", "ipp_di1", "ldb_di0", "ldb_di1", };
+static const char *cko1_sels[] = { "dummy", "dummy", "dummy", "dummy", "dummy", "axi", "enfc", "dummy", "dummy",
+				   "dummy", "lcdif_pix", "ahb", "ipg", "ipg_per", "ckil", "pll4_audio_div", };
+static const char *cko2_sels[] = { "dummy", "dummy", "dummy", "usdhc1", "dummy", "dummy", "ecspi_root", "dummy",
+				   "dummy", "dummy", "dummy", "dummy", "dummy", "dummy", "osc", "dummy",
+				   "dummy", "usdhc2", "sai1", "sai2", "sai3", "dummy", "dummy", "can_root",
+				   "dummy", "dummy", "dummy", "dummy", "uart_serial", "spdif", "dummy", "dummy", };
+static const char *cko_sels[] = { "cko1", "cko2", };
 
 static struct clk *clks[IMX6UL_CLK_END];
 static struct clk_onecell_data clk_data;
@@ -273,6 +280,10 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	clks[IMX6UL_CLK_LDB_DI0_DIV_SEL]  = imx_clk_mux("ldb_di0", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels));
 	clks[IMX6UL_CLK_LDB_DI1_DIV_SEL]  = imx_clk_mux("ldb_di1", base + 0x20, 11, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels));
 
+	clks[IMX6UL_CLK_CKO1_SEL]	  = imx_clk_mux("cko1_sel", base + 0x60, 0,  4, cko1_sels, ARRAY_SIZE(cko1_sels));
+	clks[IMX6UL_CLK_CKO2_SEL]	  = imx_clk_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels));
+	clks[IMX6UL_CLK_CKO]		  = imx_clk_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels));
+
 	clks[IMX6UL_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7);
 	clks[IMX6UL_CLK_LDB_DI0_DIV_7]	 = imx_clk_fixed_factor("ldb_di0_div_7",   "ldb_di0_sel", 1, 7);
 	clks[IMX6UL_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "qspi1_sel", 2, 7);
@@ -316,6 +327,9 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	clks[IMX6UL_CLK_LCDIF_PRED]	= imx_clk_divider("lcdif_pred",	   "lcdif_pre_sel",	base + 0x38, 12, 3);
 	clks[IMX6UL_CLK_CSI_PODF]       = imx_clk_divider("csi_podf",      "csi_sel",           base + 0x3c, 11, 3);
 
+	clks[IMX6UL_CLK_CKO1_PODF]	= imx_clk_divider("cko1_podf",     "cko1_sel",          base + 0x60, 4,  3);
+	clks[IMX6UL_CLK_CKO2_PODF]	= imx_clk_divider("cko2_podf",     "cko2_sel",          base + 0x60, 21, 3);
+
 	clks[IMX6UL_CLK_ARM]		= imx_clk_busy_divider("arm",	    "pll1_sw",	base +	0x10, 0,  3,  base + 0x48, 16);
 	clks[IMX6UL_CLK_MMDC_PODF]	= imx_clk_busy_divider("mmdc_podf", "periph2",	base +  0x14, 3,  3,  base + 0x48, 2);
 	clks[IMX6UL_CLK_AXI_PODF]	= imx_clk_busy_divider("axi_podf",  "axi_sel",	base +  0x14, 16, 3,  base + 0x48, 0);
@@ -445,6 +459,10 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	clks[IMX6UL_CLK_PWM6]		= imx_clk_gate2("pwm6",		"perclk",	 base +	0x80,	28);
 	clks[IMX6UL_CLK_PWM7]		= imx_clk_gate2("pwm7",		"perclk",	 base + 0x80,	30);
 
+	/* CCOSR */
+	clks[IMX6UL_CLK_CKO1]		= imx_clk_gate("cko1",		"cko1_podf",	 base + 0x60,	7);
+	clks[IMX6UL_CLK_CKO2]		= imx_clk_gate("cko2",		"cko2_podf",	 base + 0x60,	24);
+
 	/* mask handshake of mmdc */
 	writel_relaxed(BM_CCM_CCDR_MMDC_CH0_MASK, base + CCDR);
 
diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig
index 92afe5989e97..3dd1dab92223 100644
--- a/drivers/clk/mediatek/Kconfig
+++ b/drivers/clk/mediatek/Kconfig
@@ -60,6 +60,12 @@ config COMMON_CLK_MT2701_AUDSYS
 	---help---
 	  This driver supports Mediatek MT2701 audsys clocks.
 
+config COMMON_CLK_MT2701_G3DSYS
+	bool "Clock driver for MediaTek MT2701 g3dsys"
+	depends on COMMON_CLK_MT2701
+	---help---
+	  This driver supports MediaTek MT2701 g3dsys clocks.
+
 config COMMON_CLK_MT2712
 	bool "Clock driver for MediaTek MT2712"
 	depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST
diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile
index b80eff2abb31..844b55d2770d 100644
--- a/drivers/clk/mediatek/Makefile
+++ b/drivers/clk/mediatek/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_COMMON_CLK_MT2701) += clk-mt2701.o
 obj-$(CONFIG_COMMON_CLK_MT2701_AUDSYS) += clk-mt2701-aud.o
 obj-$(CONFIG_COMMON_CLK_MT2701_BDPSYS) += clk-mt2701-bdp.o
 obj-$(CONFIG_COMMON_CLK_MT2701_ETHSYS) += clk-mt2701-eth.o
+obj-$(CONFIG_COMMON_CLK_MT2701_G3DSYS) += clk-mt2701-g3d.o
 obj-$(CONFIG_COMMON_CLK_MT2701_HIFSYS) += clk-mt2701-hif.o
 obj-$(CONFIG_COMMON_CLK_MT2701_IMGSYS) += clk-mt2701-img.o
 obj-$(CONFIG_COMMON_CLK_MT2701_MMSYS) += clk-mt2701-mm.o
diff --git a/drivers/clk/mediatek/clk-mt2701-g3d.c b/drivers/clk/mediatek/clk-mt2701-g3d.c
new file mode 100644
index 000000000000..1328c112a38f
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt2701-g3d.c
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Sean Wang <sean.wang@mediatek.com>
+ *
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt2701-clk.h>
+
+#define GATE_G3D(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &g3d_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+static const struct mtk_gate_regs g3d_cg_regs = {
+	.sta_ofs = 0x0,
+	.set_ofs = 0x4,
+	.clr_ofs = 0x8,
+};
+
+static const struct mtk_gate g3d_clks[] = {
+	GATE_G3D(CLK_G3DSYS_CORE, "g3d_core", "mfg_sel", 0),
+};
+
+static int clk_mt2701_g3dsys_init(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+	int r;
+
+	clk_data = mtk_alloc_clk_data(CLK_G3DSYS_NR);
+
+	mtk_clk_register_gates(node, g3d_clks, ARRAY_SIZE(g3d_clks),
+			       clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r)
+		dev_err(&pdev->dev,
+			"could not register clock provider: %s: %d\n",
+			pdev->name, r);
+
+	mtk_register_reset_controller(node, 1, 0xc);
+
+	return r;
+}
+
+static const struct of_device_id of_match_clk_mt2701_g3d[] = {
+	{
+		.compatible = "mediatek,mt2701-g3dsys",
+		.data = clk_mt2701_g3dsys_init,
+	}, {
+		/* sentinel */
+	}
+};
+
+static int clk_mt2701_g3d_probe(struct platform_device *pdev)
+{
+	int (*clk_init)(struct platform_device *);
+	int r;
+
+	clk_init = of_device_get_match_data(&pdev->dev);
+	if (!clk_init)
+		return -EINVAL;
+
+	r = clk_init(pdev);
+	if (r)
+		dev_err(&pdev->dev,
+			"could not register clock provider: %s: %d\n",
+			pdev->name, r);
+
+	return r;
+}
+
+static struct platform_driver clk_mt2701_g3d_drv = {
+	.probe = clk_mt2701_g3d_probe,
+	.driver = {
+		.name = "clk-mt2701-g3d",
+		.of_match_table = of_match_clk_mt2701_g3d,
+	},
+};
+
+builtin_platform_driver(clk_mt2701_g3d_drv);
diff --git a/drivers/clk/mediatek/clk-mt2701.c b/drivers/clk/mediatek/clk-mt2701.c
index deca7527f92f..4dda8988b2f0 100644
--- a/drivers/clk/mediatek/clk-mt2701.c
+++ b/drivers/clk/mediatek/clk-mt2701.c
@@ -46,8 +46,6 @@ static const struct mtk_fixed_clk top_fixed_clks[] = {
 		340 * MHZ),
 	FIXED_CLK(CLK_TOP_HDMI_0_PLL340M, "hdmi_0_pll340m", "clk26m",
 		340 * MHZ),
-	FIXED_CLK(CLK_TOP_HDMITX_CLKDIG_CTS, "hdmitx_dig_cts", "clk26m",
-		300 * MHZ),
 	FIXED_CLK(CLK_TOP_HADDS2_FB, "hadds2_fbclk", "clk26m",
 		27 * MHZ),
 	FIXED_CLK(CLK_TOP_WBG_DIG_416M, "wbg_dig_ck_416m", "clk26m",
@@ -977,6 +975,10 @@ static const struct mtk_pll_data apmixed_plls[] = {
 				21, 0x2d0, 4, 0x0, 0x2d4, 0),
 };
 
+static const struct mtk_fixed_factor apmixed_fixed_divs[] = {
+	FACTOR(CLK_APMIXED_HDMI_REF, "hdmi_ref", "tvdpll", 1, 1),
+};
+
 static int mtk_apmixedsys_init(struct platform_device *pdev)
 {
 	struct clk_onecell_data *clk_data;
@@ -988,6 +990,8 @@ static int mtk_apmixedsys_init(struct platform_device *pdev)
 
 	mtk_clk_register_plls(node, apmixed_plls, ARRAY_SIZE(apmixed_plls),
 								clk_data);
+	mtk_clk_register_factors(apmixed_fixed_divs, ARRAY_SIZE(apmixed_fixed_divs),
+								clk_data);
 
 	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
 }
diff --git a/drivers/clk/meson/Kconfig b/drivers/clk/meson/Kconfig
index d5cbec522aec..815659eebea3 100644
--- a/drivers/clk/meson/Kconfig
+++ b/drivers/clk/meson/Kconfig
@@ -3,6 +3,12 @@ config COMMON_CLK_AMLOGIC
 	depends on OF
 	depends on ARCH_MESON || COMPILE_TEST
 
+config COMMON_CLK_MESON_AO
+	bool
+	depends on OF
+	depends on ARCH_MESON || COMPILE_TEST
+	select COMMON_CLK_REGMAP_MESON
+
 config COMMON_CLK_REGMAP_MESON
 	bool
 	select REGMAP
@@ -21,6 +27,7 @@ config COMMON_CLK_GXBB
 	bool
 	depends on COMMON_CLK_AMLOGIC
 	select RESET_CONTROLLER
+	select COMMON_CLK_MESON_AO
 	select COMMON_CLK_REGMAP_MESON
 	select MFD_SYSCON
 	help
@@ -31,6 +38,7 @@ config COMMON_CLK_AXG
 	bool
 	depends on COMMON_CLK_AMLOGIC
 	select RESET_CONTROLLER
+	select COMMON_CLK_MESON_AO
 	select COMMON_CLK_REGMAP_MESON
 	select MFD_SYSCON
 	help
diff --git a/drivers/clk/meson/Makefile b/drivers/clk/meson/Makefile
index ffee82e60b7a..d0d13aeb369a 100644
--- a/drivers/clk/meson/Makefile
+++ b/drivers/clk/meson/Makefile
@@ -3,7 +3,8 @@
 #
 
 obj-$(CONFIG_COMMON_CLK_AMLOGIC) += clk-pll.o clk-mpll.o clk-audio-divider.o
+obj-$(CONFIG_COMMON_CLK_MESON_AO) += meson-aoclk.o
 obj-$(CONFIG_COMMON_CLK_MESON8B) += meson8b.o
 obj-$(CONFIG_COMMON_CLK_GXBB)	 += gxbb.o gxbb-aoclk.o gxbb-aoclk-32k.o
-obj-$(CONFIG_COMMON_CLK_AXG)	 += axg.o
+obj-$(CONFIG_COMMON_CLK_AXG)	 += axg.o axg-aoclk.o
 obj-$(CONFIG_COMMON_CLK_REGMAP_MESON)	+= clk-regmap.o
diff --git a/drivers/clk/meson/axg-aoclk.c b/drivers/clk/meson/axg-aoclk.c
new file mode 100644
index 000000000000..29e088542387
--- /dev/null
+++ b/drivers/clk/meson/axg-aoclk.c
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Amlogic Meson-AXG Clock Controller Driver
+ *
+ * Copyright (c) 2016 Baylibre SAS.
+ * Author: Michael Turquette <mturquette@baylibre.com>
+ *
+ * Copyright (c) 2018 Amlogic, inc.
+ * Author: Qiufang Dai <qiufang.dai@amlogic.com>
+ */
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+#include <linux/reset-controller.h>
+#include <linux/mfd/syscon.h>
+#include "clk-regmap.h"
+#include "meson-aoclk.h"
+#include "axg-aoclk.h"
+
+#define AXG_AO_GATE(_name, _bit)					\
+static struct clk_regmap axg_aoclk_##_name = {				\
+	.data = &(struct clk_regmap_gate_data) {			\
+		.offset = (AO_RTI_GEN_CNTL_REG0),			\
+		.bit_idx = (_bit),					\
+	},								\
+	.hw.init = &(struct clk_init_data) {				\
+		.name =  "axg_ao_" #_name,				\
+		.ops = &clk_regmap_gate_ops,				\
+		.parent_names = (const char *[]){ "clk81" },		\
+		.num_parents = 1,					\
+		.flags = CLK_IGNORE_UNUSED,				\
+	},								\
+}
+
+AXG_AO_GATE(remote, 0);
+AXG_AO_GATE(i2c_master, 1);
+AXG_AO_GATE(i2c_slave, 2);
+AXG_AO_GATE(uart1, 3);
+AXG_AO_GATE(uart2, 5);
+AXG_AO_GATE(ir_blaster, 6);
+AXG_AO_GATE(saradc, 7);
+
+static struct clk_regmap axg_aoclk_clk81 = {
+	.data = &(struct clk_regmap_mux_data) {
+		.offset = AO_RTI_PWR_CNTL_REG0,
+		.mask = 0x1,
+		.shift = 8,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "axg_ao_clk81",
+		.ops = &clk_regmap_mux_ro_ops,
+		.parent_names = (const char *[]){ "clk81", "ao_alt_xtal"},
+		.num_parents = 2,
+	},
+};
+
+static struct clk_regmap axg_aoclk_saradc_mux = {
+	.data = &(struct clk_regmap_mux_data) {
+		.offset = AO_SAR_CLK,
+		.mask = 0x3,
+		.shift = 9,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "axg_ao_saradc_mux",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = (const char *[]){ "xtal", "axg_ao_clk81" },
+		.num_parents = 2,
+	},
+};
+
+static struct clk_regmap axg_aoclk_saradc_div = {
+	.data = &(struct clk_regmap_div_data) {
+		.offset = AO_SAR_CLK,
+		.shift = 0,
+		.width = 8,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "axg_ao_saradc_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "axg_ao_saradc_mux" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap axg_aoclk_saradc_gate = {
+	.data = &(struct clk_regmap_gate_data) {
+		.offset = AO_SAR_CLK,
+		.bit_idx = 8,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "axg_ao_saradc_gate",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "axg_ao_saradc_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const unsigned int axg_aoclk_reset[] = {
+	[RESET_AO_REMOTE]	= 16,
+	[RESET_AO_I2C_MASTER]	= 18,
+	[RESET_AO_I2C_SLAVE]	= 19,
+	[RESET_AO_UART1]	= 17,
+	[RESET_AO_UART2]	= 22,
+	[RESET_AO_IR_BLASTER]	= 23,
+};
+
+static struct clk_regmap *axg_aoclk_regmap[] = {
+	[CLKID_AO_REMOTE]	= &axg_aoclk_remote,
+	[CLKID_AO_I2C_MASTER]	= &axg_aoclk_i2c_master,
+	[CLKID_AO_I2C_SLAVE]	= &axg_aoclk_i2c_slave,
+	[CLKID_AO_UART1]	= &axg_aoclk_uart1,
+	[CLKID_AO_UART2]	= &axg_aoclk_uart2,
+	[CLKID_AO_IR_BLASTER]	= &axg_aoclk_ir_blaster,
+	[CLKID_AO_SAR_ADC]	= &axg_aoclk_saradc,
+	[CLKID_AO_CLK81]	= &axg_aoclk_clk81,
+	[CLKID_AO_SAR_ADC_SEL]	= &axg_aoclk_saradc_mux,
+	[CLKID_AO_SAR_ADC_DIV]	= &axg_aoclk_saradc_div,
+	[CLKID_AO_SAR_ADC_CLK]	= &axg_aoclk_saradc_gate,
+};
+
+static const struct clk_hw_onecell_data axg_aoclk_onecell_data = {
+	.hws = {
+		[CLKID_AO_REMOTE]	= &axg_aoclk_remote.hw,
+		[CLKID_AO_I2C_MASTER]	= &axg_aoclk_i2c_master.hw,
+		[CLKID_AO_I2C_SLAVE]	= &axg_aoclk_i2c_slave.hw,
+		[CLKID_AO_UART1]	= &axg_aoclk_uart1.hw,
+		[CLKID_AO_UART2]	= &axg_aoclk_uart2.hw,
+		[CLKID_AO_IR_BLASTER]	= &axg_aoclk_ir_blaster.hw,
+		[CLKID_AO_SAR_ADC]	= &axg_aoclk_saradc.hw,
+		[CLKID_AO_CLK81]	= &axg_aoclk_clk81.hw,
+		[CLKID_AO_SAR_ADC_SEL]	= &axg_aoclk_saradc_mux.hw,
+		[CLKID_AO_SAR_ADC_DIV]	= &axg_aoclk_saradc_div.hw,
+		[CLKID_AO_SAR_ADC_CLK]	= &axg_aoclk_saradc_gate.hw,
+	},
+	.num = NR_CLKS,
+};
+
+static const struct meson_aoclk_data axg_aoclkc_data = {
+	.reset_reg	= AO_RTI_GEN_CNTL_REG0,
+	.num_reset	= ARRAY_SIZE(axg_aoclk_reset),
+	.reset		= axg_aoclk_reset,
+	.num_clks	= ARRAY_SIZE(axg_aoclk_regmap),
+	.clks		= axg_aoclk_regmap,
+	.hw_data	= &axg_aoclk_onecell_data,
+};
+
+static const struct of_device_id axg_aoclkc_match_table[] = {
+	{
+		.compatible	= "amlogic,meson-axg-aoclkc",
+		.data		= &axg_aoclkc_data,
+	},
+	{ }
+};
+
+static struct platform_driver axg_aoclkc_driver = {
+	.probe		= meson_aoclkc_probe,
+	.driver		= {
+		.name	= "axg-aoclkc",
+		.of_match_table = axg_aoclkc_match_table,
+	},
+};
+
+builtin_platform_driver(axg_aoclkc_driver);
diff --git a/drivers/clk/meson/axg-aoclk.h b/drivers/clk/meson/axg-aoclk.h
new file mode 100644
index 000000000000..91384d8dd844
--- /dev/null
+++ b/drivers/clk/meson/axg-aoclk.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
+/*
+ * Copyright (c) 2017 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * Copyright (c) 2018 Amlogic, inc.
+ * Author: Qiufang Dai <qiufang.dai@amlogic.com>
+ */
+
+#ifndef __AXG_AOCLKC_H
+#define __AXG_AOCLKC_H
+
+#define NR_CLKS	11
+/* AO Configuration Clock registers offsets
+ * Register offsets from the data sheet must be multiplied by 4.
+ */
+#define AO_RTI_PWR_CNTL_REG1	0x0C
+#define AO_RTI_PWR_CNTL_REG0	0x10
+#define AO_RTI_GEN_CNTL_REG0	0x40
+#define AO_OSCIN_CNTL		0x58
+#define AO_CRT_CLK_CNTL1	0x68
+#define AO_SAR_CLK		0x90
+#define AO_RTC_ALT_CLK_CNTL0	0x94
+#define AO_RTC_ALT_CLK_CNTL1	0x98
+
+#include <dt-bindings/clock/axg-aoclkc.h>
+#include <dt-bindings/reset/axg-aoclkc.h>
+
+#endif /* __AXG_AOCLKC_H */
diff --git a/drivers/clk/meson/gxbb-aoclk.c b/drivers/clk/meson/gxbb-aoclk.c
index 9ec23ae9a219..408e3e2fca18 100644
--- a/drivers/clk/meson/gxbb-aoclk.c
+++ b/drivers/clk/meson/gxbb-aoclk.c
@@ -52,39 +52,12 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-#include <linux/clk-provider.h>
-#include <linux/of_address.h>
 #include <linux/platform_device.h>
-#include <linux/reset-controller.h>
 #include <linux/mfd/syscon.h>
-#include <linux/regmap.h>
-#include <linux/init.h>
-#include <linux/delay.h>
-#include <dt-bindings/clock/gxbb-aoclkc.h>
-#include <dt-bindings/reset/gxbb-aoclkc.h>
 #include "clk-regmap.h"
+#include "meson-aoclk.h"
 #include "gxbb-aoclk.h"
 
-struct gxbb_aoclk_reset_controller {
-	struct reset_controller_dev reset;
-	unsigned int *data;
-	struct regmap *regmap;
-};
-
-static int gxbb_aoclk_do_reset(struct reset_controller_dev *rcdev,
-			       unsigned long id)
-{
-	struct gxbb_aoclk_reset_controller *reset =
-		container_of(rcdev, struct gxbb_aoclk_reset_controller, reset);
-
-	return regmap_write(reset->regmap, AO_RTI_GEN_CNTL_REG0,
-			    BIT(reset->data[id]));
-}
-
-static const struct reset_control_ops gxbb_aoclk_reset_ops = {
-	.reset = gxbb_aoclk_do_reset,
-};
-
 #define GXBB_AO_GATE(_name, _bit)					\
 static struct clk_regmap _name##_ao = {					\
 	.data = &(struct clk_regmap_gate_data) {			\
@@ -96,7 +69,7 @@ static struct clk_regmap _name##_ao = {					\
 		.ops = &clk_regmap_gate_ops,				\
 		.parent_names = (const char *[]){ "clk81" },		\
 		.num_parents = 1,					\
-		.flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED),	\
+		.flags = CLK_IGNORE_UNUSED,				\
 	},								\
 }
 
@@ -117,7 +90,7 @@ static struct aoclk_cec_32k cec_32k_ao = {
 	},
 };
 
-static unsigned int gxbb_aoclk_reset[] = {
+static const unsigned int gxbb_aoclk_reset[] = {
 	[RESET_AO_REMOTE] = 16,
 	[RESET_AO_I2C_MASTER] = 18,
 	[RESET_AO_I2C_SLAVE] = 19,
@@ -135,7 +108,7 @@ static struct clk_regmap *gxbb_aoclk_gate[] = {
 	[CLKID_AO_IR_BLASTER] = &ir_blaster_ao,
 };
 
-static struct clk_hw_onecell_data gxbb_aoclk_onecell_data = {
+static const struct clk_hw_onecell_data gxbb_aoclk_onecell_data = {
 	.hws = {
 		[CLKID_AO_REMOTE] = &remote_ao.hw,
 		[CLKID_AO_I2C_MASTER] = &i2c_master_ao.hw,
@@ -145,58 +118,55 @@ static struct clk_hw_onecell_data gxbb_aoclk_onecell_data = {
 		[CLKID_AO_IR_BLASTER] = &ir_blaster_ao.hw,
 		[CLKID_AO_CEC_32K] = &cec_32k_ao.hw,
 	},
-	.num = 7,
+	.num = NR_CLKS,
 };
 
-static int gxbb_aoclkc_probe(struct platform_device *pdev)
+static int gxbb_register_cec_ao_32k(struct platform_device *pdev)
 {
-	struct gxbb_aoclk_reset_controller *rstc;
 	struct device *dev = &pdev->dev;
 	struct regmap *regmap;
-	int ret, clkid;
-
-	rstc = devm_kzalloc(dev, sizeof(*rstc), GFP_KERNEL);
-	if (!rstc)
-		return -ENOMEM;
+	int ret;
 
 	regmap = syscon_node_to_regmap(of_get_parent(dev->of_node));
 	if (IS_ERR(regmap)) {
 		dev_err(dev, "failed to get regmap\n");
-		return -ENODEV;
-	}
-
-	/* Reset Controller */
-	rstc->regmap = regmap;
-	rstc->data = gxbb_aoclk_reset;
-	rstc->reset.ops = &gxbb_aoclk_reset_ops;
-	rstc->reset.nr_resets = ARRAY_SIZE(gxbb_aoclk_reset);
-	rstc->reset.of_node = dev->of_node;
-	ret = devm_reset_controller_register(dev, &rstc->reset);
-
-	/*
-	 * Populate regmap and register all clks
-	 */
-	for (clkid = 0; clkid < ARRAY_SIZE(gxbb_aoclk_gate); clkid++) {
-		gxbb_aoclk_gate[clkid]->map = regmap;
-
-		ret = devm_clk_hw_register(dev,
-					   gxbb_aoclk_onecell_data.hws[clkid]);
-		if (ret)
-			return ret;
+		return PTR_ERR(regmap);
 	}
 
 	/* Specific clocks */
 	cec_32k_ao.regmap = regmap;
 	ret = devm_clk_hw_register(dev, &cec_32k_ao.hw);
+	if (ret) {
+		dev_err(&pdev->dev, "clk cec_32k_ao register failed.\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct meson_aoclk_data gxbb_aoclkc_data = {
+	.reset_reg	= AO_RTI_GEN_CNTL_REG0,
+	.num_reset	= ARRAY_SIZE(gxbb_aoclk_reset),
+	.reset		= gxbb_aoclk_reset,
+	.num_clks	= ARRAY_SIZE(gxbb_aoclk_gate),
+	.clks		= gxbb_aoclk_gate,
+	.hw_data	= &gxbb_aoclk_onecell_data,
+};
+
+static int gxbb_aoclkc_probe(struct platform_device *pdev)
+{
+	int ret = gxbb_register_cec_ao_32k(pdev);
 	if (ret)
 		return ret;
 
-	return of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get,
-			&gxbb_aoclk_onecell_data);
+	return meson_aoclkc_probe(pdev);
 }
 
 static const struct of_device_id gxbb_aoclkc_match_table[] = {
-	{ .compatible = "amlogic,meson-gx-aoclkc" },
+	{
+		.compatible	= "amlogic,meson-gx-aoclkc",
+		.data		= &gxbb_aoclkc_data,
+	},
 	{ }
 };
 
diff --git a/drivers/clk/meson/gxbb-aoclk.h b/drivers/clk/meson/gxbb-aoclk.h
index badc4c22b4ee..6e7fa4134a00 100644
--- a/drivers/clk/meson/gxbb-aoclk.h
+++ b/drivers/clk/meson/gxbb-aoclk.h
@@ -8,6 +8,8 @@
 #ifndef __GXBB_AOCLKC_H
 #define __GXBB_AOCLKC_H
 
+#define NR_CLKS	7
+
 /* AO Configuration Clock registers offsets */
 #define AO_RTI_PWR_CNTL_REG1	0x0c
 #define AO_RTI_PWR_CNTL_REG0	0x10
@@ -26,4 +28,7 @@ struct aoclk_cec_32k {
 
 extern const struct clk_ops meson_aoclk_cec_32k_ops;
 
+#include <dt-bindings/clock/gxbb-aoclkc.h>
+#include <dt-bindings/reset/gxbb-aoclkc.h>
+
 #endif /* __GXBB_AOCLKC_H */
diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c
index b1e4d9557610..a893b1edda22 100644
--- a/drivers/clk/meson/gxbb.c
+++ b/drivers/clk/meson/gxbb.c
@@ -1543,6 +1543,102 @@ static struct clk_regmap gxbb_vapb = {
 	},
 };
 
+/* VDEC clocks */
+
+static const char * const gxbb_vdec_parent_names[] = {
+	"fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7"
+};
+
+static struct clk_regmap gxbb_vdec_1_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.mask = 0x3,
+		.shift = 9,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_1_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = gxbb_vdec_parent_names,
+		.num_parents = ARRAY_SIZE(gxbb_vdec_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap gxbb_vdec_1_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.shift = 0,
+		.width = 7,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_1_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vdec_1_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap gxbb_vdec_1 = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.bit_idx = 8,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vdec_1",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vdec_1_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap gxbb_vdec_hevc_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.mask = 0x3,
+		.shift = 25,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_hevc_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = gxbb_vdec_parent_names,
+		.num_parents = ARRAY_SIZE(gxbb_vdec_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap gxbb_vdec_hevc_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.shift = 16,
+		.width = 7,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_hevc_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vdec_hevc_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap gxbb_vdec_hevc = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.bit_idx = 24,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vdec_hevc",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vdec_hevc_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
 /* Everything Else (EE) domain gates */
 static MESON_GATE(gxbb_ddr, HHI_GCLK_MPEG0, 0);
 static MESON_GATE(gxbb_dos, HHI_GCLK_MPEG0, 1);
@@ -1786,6 +1882,12 @@ static struct clk_hw_onecell_data gxbb_hw_onecell_data = {
 		[CLKID_FCLK_DIV4_DIV]	    = &gxbb_fclk_div4_div.hw,
 		[CLKID_FCLK_DIV5_DIV]	    = &gxbb_fclk_div5_div.hw,
 		[CLKID_FCLK_DIV7_DIV]	    = &gxbb_fclk_div7_div.hw,
+		[CLKID_VDEC_1_SEL]	    = &gxbb_vdec_1_sel.hw,
+		[CLKID_VDEC_1_DIV]	    = &gxbb_vdec_1_div.hw,
+		[CLKID_VDEC_1]		    = &gxbb_vdec_1.hw,
+		[CLKID_VDEC_HEVC_SEL]	    = &gxbb_vdec_hevc_sel.hw,
+		[CLKID_VDEC_HEVC_DIV]	    = &gxbb_vdec_hevc_div.hw,
+		[CLKID_VDEC_HEVC]	    = &gxbb_vdec_hevc.hw,
 		[NR_CLKS]		    = NULL,
 	},
 	.num = NR_CLKS,
@@ -1942,6 +2044,12 @@ static struct clk_hw_onecell_data gxl_hw_onecell_data = {
 		[CLKID_FCLK_DIV4_DIV]	    = &gxbb_fclk_div4_div.hw,
 		[CLKID_FCLK_DIV5_DIV]	    = &gxbb_fclk_div5_div.hw,
 		[CLKID_FCLK_DIV7_DIV]	    = &gxbb_fclk_div7_div.hw,
+		[CLKID_VDEC_1_SEL]	    = &gxbb_vdec_1_sel.hw,
+		[CLKID_VDEC_1_DIV]	    = &gxbb_vdec_1_div.hw,
+		[CLKID_VDEC_1]		    = &gxbb_vdec_1.hw,
+		[CLKID_VDEC_HEVC_SEL]	    = &gxbb_vdec_hevc_sel.hw,
+		[CLKID_VDEC_HEVC_DIV]	    = &gxbb_vdec_hevc_div.hw,
+		[CLKID_VDEC_HEVC]	    = &gxbb_vdec_hevc.hw,
 		[NR_CLKS]		    = NULL,
 	},
 	.num = NR_CLKS,
@@ -2100,6 +2208,12 @@ static struct clk_regmap *const gx_clk_regmaps[] = {
 	&gxbb_fclk_div4,
 	&gxbb_fclk_div5,
 	&gxbb_fclk_div7,
+	&gxbb_vdec_1_sel,
+	&gxbb_vdec_1_div,
+	&gxbb_vdec_1,
+	&gxbb_vdec_hevc_sel,
+	&gxbb_vdec_hevc_div,
+	&gxbb_vdec_hevc,
 };
 
 struct clkc_data {
diff --git a/drivers/clk/meson/gxbb.h b/drivers/clk/meson/gxbb.h
index 9febf3f03739..b3ef54f65a9d 100644
--- a/drivers/clk/meson/gxbb.h
+++ b/drivers/clk/meson/gxbb.h
@@ -204,8 +204,12 @@
 #define CLKID_FCLK_DIV4_DIV	  148
 #define CLKID_FCLK_DIV5_DIV	  149
 #define CLKID_FCLK_DIV7_DIV	  150
+#define CLKID_VDEC_1_SEL	  151
+#define CLKID_VDEC_1_DIV	  152
+#define CLKID_VDEC_HEVC_SEL	  154
+#define CLKID_VDEC_HEVC_DIV	  155
 
-#define NR_CLKS			  151
+#define NR_CLKS			  157
 
 /* include the CLKIDs that have been made part of the DT binding */
 #include <dt-bindings/clock/gxbb-clkc.h>
diff --git a/drivers/clk/meson/meson-aoclk.c b/drivers/clk/meson/meson-aoclk.c
new file mode 100644
index 000000000000..f965845917e3
--- /dev/null
+++ b/drivers/clk/meson/meson-aoclk.c
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Amlogic Meson-AXG Clock Controller Driver
+ *
+ * Copyright (c) 2016 BayLibre, SAS.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * Copyright (c) 2018 Amlogic, inc.
+ * Author: Qiufang Dai <qiufang.dai@amlogic.com>
+ * Author: Yixun Lan <yixun.lan@amlogic.com>
+ */
+
+#include <linux/platform_device.h>
+#include <linux/reset-controller.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_device.h>
+#include "clk-regmap.h"
+#include "meson-aoclk.h"
+
+static int meson_aoclk_do_reset(struct reset_controller_dev *rcdev,
+			       unsigned long id)
+{
+	struct meson_aoclk_reset_controller *rstc =
+		container_of(rcdev, struct meson_aoclk_reset_controller, reset);
+
+	return regmap_write(rstc->regmap, rstc->data->reset_reg,
+			    BIT(rstc->data->reset[id]));
+}
+
+static const struct reset_control_ops meson_aoclk_reset_ops = {
+	.reset = meson_aoclk_do_reset,
+};
+
+int meson_aoclkc_probe(struct platform_device *pdev)
+{
+	struct meson_aoclk_reset_controller *rstc;
+	struct meson_aoclk_data *data;
+	struct device *dev = &pdev->dev;
+	struct regmap *regmap;
+	int ret, clkid;
+
+	data = (struct meson_aoclk_data *) of_device_get_match_data(dev);
+	if (!data)
+		return -ENODEV;
+
+	rstc = devm_kzalloc(dev, sizeof(*rstc), GFP_KERNEL);
+	if (!rstc)
+		return -ENOMEM;
+
+	regmap = syscon_node_to_regmap(of_get_parent(dev->of_node));
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "failed to get regmap\n");
+		return PTR_ERR(regmap);
+	}
+
+	/* Reset Controller */
+	rstc->data = data;
+	rstc->regmap = regmap;
+	rstc->reset.ops = &meson_aoclk_reset_ops;
+	rstc->reset.nr_resets = data->num_reset,
+	rstc->reset.of_node = dev->of_node;
+	ret = devm_reset_controller_register(dev, &rstc->reset);
+	if (ret) {
+		dev_err(dev, "failed to register reset controller\n");
+		return ret;
+	}
+
+	/*
+	 * Populate regmap and register all clks
+	 */
+	for (clkid = 0; clkid < data->num_clks; clkid++) {
+		data->clks[clkid]->map = regmap;
+
+		ret = devm_clk_hw_register(dev, data->hw_data->hws[clkid]);
+		if (ret)
+			return ret;
+	}
+
+	return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get,
+		(void *) data->hw_data);
+}
diff --git a/drivers/clk/meson/meson-aoclk.h b/drivers/clk/meson/meson-aoclk.h
new file mode 100644
index 000000000000..ab2819e88922
--- /dev/null
+++ b/drivers/clk/meson/meson-aoclk.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
+/*
+ * Copyright (c) 2017 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * Copyright (c) 2018 Amlogic, inc.
+ * Author: Qiufang Dai <qiufang.dai@amlogic.com>
+ * Author: Yixun Lan <yixun.lan@amlogic.com>
+ */
+
+#ifndef __MESON_AOCLK_H__
+#define __MESON_AOCLK_H__
+
+#include <linux/platform_device.h>
+#include <linux/reset-controller.h>
+#include "clk-regmap.h"
+
+struct meson_aoclk_data {
+	const unsigned int			reset_reg;
+	const int				num_reset;
+	const unsigned int			*reset;
+	int					num_clks;
+	struct clk_regmap			**clks;
+	const struct clk_hw_onecell_data	*hw_data;
+};
+
+struct meson_aoclk_reset_controller {
+	struct reset_controller_dev		reset;
+	const struct meson_aoclk_data		*data;
+	struct regmap				*regmap;
+};
+
+int meson_aoclkc_probe(struct platform_device *pdev);
+#endif
diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c
index d0524ec71aad..40068c1b5e80 100644
--- a/drivers/clk/meson/meson8b.c
+++ b/drivers/clk/meson/meson8b.c
@@ -640,6 +640,54 @@ static struct clk_regmap meson8b_cpu_clk = {
 	},
 };
 
+static struct clk_regmap meson8b_nand_clk_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_NAND_CLK_CNTL,
+		.mask = 0x7,
+		.shift = 9,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "nand_clk_sel",
+		.ops = &clk_regmap_mux_ops,
+		/* FIXME all other parents are unknown: */
+		.parent_names = (const char *[]){ "fclk_div4", "fclk_div3",
+			"fclk_div5", "fclk_div7", "xtal" },
+		.num_parents = 5,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_nand_clk_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset =  HHI_NAND_CLK_CNTL,
+		.shift = 0,
+		.width = 7,
+		.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "nand_clk_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "nand_clk_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_nand_clk_gate = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_NAND_CLK_CNTL,
+		.bit_idx = 8,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "nand_clk_gate",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "nand_clk_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
 /* Everything Else (EE) domain gates */
 
 static MESON_GATE(meson8b_ddr, HHI_GCLK_MPEG0, 0);
@@ -835,6 +883,9 @@ static struct clk_hw_onecell_data meson8b_hw_onecell_data = {
 		[CLKID_FCLK_DIV4_DIV]	    = &meson8b_fclk_div4_div.hw,
 		[CLKID_FCLK_DIV5_DIV]	    = &meson8b_fclk_div5_div.hw,
 		[CLKID_FCLK_DIV7_DIV]	    = &meson8b_fclk_div7_div.hw,
+		[CLKID_NAND_SEL]	    = &meson8b_nand_clk_sel.hw,
+		[CLKID_NAND_DIV]	    = &meson8b_nand_clk_div.hw,
+		[CLKID_NAND_CLK]	    = &meson8b_nand_clk_gate.hw,
 		[CLK_NR_CLKS]		    = NULL,
 	},
 	.num = CLK_NR_CLKS,
@@ -940,6 +991,9 @@ static struct clk_regmap *const meson8b_clk_regmaps[] = {
 	&meson8b_fclk_div4,
 	&meson8b_fclk_div5,
 	&meson8b_fclk_div7,
+	&meson8b_nand_clk_sel,
+	&meson8b_nand_clk_div,
+	&meson8b_nand_clk_gate,
 };
 
 static const struct meson8b_clk_reset_line {
diff --git a/drivers/clk/meson/meson8b.h b/drivers/clk/meson/meson8b.h
index 6e414bd36981..a687dc60b6d0 100644
--- a/drivers/clk/meson/meson8b.h
+++ b/drivers/clk/meson/meson8b.h
@@ -40,6 +40,7 @@
 #define HHI_VID_CLK_CNTL		0x17c /* 0x5f offset in data sheet */
 #define HHI_VID_DIVIDER_CNTL		0x198 /* 0x66 offset in data sheet */
 #define HHI_SYS_CPU_CLK_CNTL0		0x19c /* 0x67 offset in data sheet */
+#define HHI_NAND_CLK_CNTL		0x25c /* 0x97 offset in data sheet */
 #define HHI_MPLL_CNTL			0x280 /* 0xa0 offset in data sheet */
 #define HHI_SYS_PLL_CNTL		0x300 /* 0xc0 offset in data sheet */
 #define HHI_VID_PLL_CNTL		0x320 /* 0xc8 offset in data sheet */
@@ -83,8 +84,10 @@
 #define CLKID_FCLK_DIV4_DIV	107
 #define CLKID_FCLK_DIV5_DIV	108
 #define CLKID_FCLK_DIV7_DIV	109
+#define CLKID_NAND_SEL		110
+#define CLKID_NAND_DIV		111
 
-#define CLK_NR_CLKS		110
+#define CLK_NR_CLKS		113
 
 /*
  * include the CLKID and RESETID that have
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index fbf4532f94b8..aec69d243d1c 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -218,6 +218,23 @@ config MSM_MMCC_8996
 	  Say Y if you want to support multimedia devices such as display,
 	  graphics, video encode/decode, camera, etc.
 
+config MSM_GCC_8998
+	tristate "MSM8998 Global Clock Controller"
+	depends on COMMON_CLK_QCOM
+	help
+	  Support for the global clock controller on msm8998 devices.
+	  Say Y if you want to use peripheral devices such as UART, SPI,
+	  i2c, USB, UFS, SD/eMMC, PCIe, etc.
+
+config SDM_GCC_845
+	tristate "SDM845 Global Clock Controller"
+	select QCOM_GDSC
+	depends on COMMON_CLK_QCOM
+	help
+	  Support for the global clock controller on msm8998 devices.
+	  Say Y if you want to use peripheral devices such as UART, SPI,
+	  I2C, USB, UFS, SDDC, PCIe, etc.
+
 config SPMI_PMIC_CLKDIV
 	tristate "SPMI PMIC clkdiv Support"
 	depends on (COMMON_CLK_QCOM && SPMI) || COMPILE_TEST
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index 230332cf317e..bf8fb25de900 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -30,6 +30,7 @@ obj-$(CONFIG_MSM_GCC_8974) += gcc-msm8974.o
 obj-$(CONFIG_MSM_GCC_8994) += gcc-msm8994.o
 obj-$(CONFIG_MSM_GCC_8996) += gcc-msm8996.o
 obj-$(CONFIG_MSM_LCC_8960) += lcc-msm8960.o
+obj-$(CONFIG_MSM_GCC_8998) += gcc-msm8998.o
 obj-$(CONFIG_MSM_MMCC_8960) += mmcc-msm8960.o
 obj-$(CONFIG_MSM_MMCC_8974) += mmcc-msm8974.o
 obj-$(CONFIG_MSM_MMCC_8996) += mmcc-msm8996.o
@@ -37,4 +38,5 @@ obj-$(CONFIG_QCOM_A53PLL) += a53-pll.o
 obj-$(CONFIG_QCOM_CLK_APCS_MSM8916) += apcs-msm8916.o
 obj-$(CONFIG_QCOM_CLK_RPM) += clk-rpm.o
 obj-$(CONFIG_QCOM_CLK_SMD_RPM) += clk-smd-rpm.o
+obj-$(CONFIG_SDM_GCC_845) += gcc-sdm845.o
 obj-$(CONFIG_SPMI_PMIC_CLKDIV) += clk-spmi-pmic-div.o
diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c
index 6d04cd96482a..9722b701fbdb 100644
--- a/drivers/clk/qcom/clk-alpha-pll.c
+++ b/drivers/clk/qcom/clk-alpha-pll.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2015, 2018, The Linux Foundation. All rights reserved.
  *
  * This software is licensed under the terms of the GNU General Public
  * License version 2, as published by the Free Software Foundation, and
@@ -58,6 +58,8 @@
 #define PLL_TEST_CTL(p)		((p)->offset + (p)->regs[PLL_OFF_TEST_CTL])
 #define PLL_TEST_CTL_U(p)	((p)->offset + (p)->regs[PLL_OFF_TEST_CTL_U])
 #define PLL_STATUS(p)		((p)->offset + (p)->regs[PLL_OFF_STATUS])
+#define PLL_OPMODE(p)		((p)->offset + (p)->regs[PLL_OFF_OPMODE])
+#define PLL_FRAC(p)		((p)->offset + (p)->regs[PLL_OFF_FRAC])
 
 const u8 clk_alpha_pll_regs[][PLL_OFF_MAX_REGS] = {
 	[CLK_ALPHA_PLL_TYPE_DEFAULT] =  {
@@ -90,6 +92,18 @@ const u8 clk_alpha_pll_regs[][PLL_OFF_MAX_REGS] = {
 		[PLL_OFF_TEST_CTL] = 0x1c,
 		[PLL_OFF_STATUS] = 0x24,
 	},
+	[CLK_ALPHA_PLL_TYPE_FABIA] =  {
+		[PLL_OFF_L_VAL] = 0x04,
+		[PLL_OFF_USER_CTL] = 0x0c,
+		[PLL_OFF_USER_CTL_U] = 0x10,
+		[PLL_OFF_CONFIG_CTL] = 0x14,
+		[PLL_OFF_CONFIG_CTL_U] = 0x18,
+		[PLL_OFF_TEST_CTL] = 0x1c,
+		[PLL_OFF_TEST_CTL_U] = 0x20,
+		[PLL_OFF_STATUS] = 0x24,
+		[PLL_OFF_OPMODE] = 0x2c,
+		[PLL_OFF_FRAC] = 0x38,
+	},
 };
 EXPORT_SYMBOL_GPL(clk_alpha_pll_regs);
 
@@ -108,6 +122,12 @@ EXPORT_SYMBOL_GPL(clk_alpha_pll_regs);
 #define PLL_HUAYRA_N_MASK		0xff
 #define PLL_HUAYRA_ALPHA_WIDTH		16
 
+#define FABIA_OPMODE_STANDBY	0x0
+#define FABIA_OPMODE_RUN	0x1
+
+#define FABIA_PLL_OUT_MASK	0x7
+#define FABIA_PLL_RATE_MARGIN	500
+
 #define pll_alpha_width(p)					\
 		((PLL_ALPHA_VAL_U(p) - PLL_ALPHA_VAL(p) == 4) ?	\
 				 ALPHA_REG_BITWIDTH : ALPHA_REG_16BIT_WIDTH)
@@ -441,16 +461,12 @@ clk_alpha_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
 	return alpha_pll_calc_rate(prate, l, a, alpha_width);
 }
 
-static int clk_alpha_pll_update_latch(struct clk_alpha_pll *pll,
-				      int (*is_enabled)(struct clk_hw *))
+
+static int __clk_alpha_pll_update_latch(struct clk_alpha_pll *pll)
 {
 	int ret;
 	u32 mode;
 
-	if (!is_enabled(&pll->clkr.hw) ||
-	    !(pll->flags & SUPPORTS_DYNAMIC_UPDATE))
-		return 0;
-
 	regmap_read(pll->clkr.regmap, PLL_MODE(pll), &mode);
 
 	/* Latch the input to the PLL */
@@ -489,6 +505,16 @@ static int clk_alpha_pll_update_latch(struct clk_alpha_pll *pll,
 	return 0;
 }
 
+static int clk_alpha_pll_update_latch(struct clk_alpha_pll *pll,
+				      int (*is_enabled)(struct clk_hw *))
+{
+	if (!is_enabled(&pll->clkr.hw) ||
+	    !(pll->flags & SUPPORTS_DYNAMIC_UPDATE))
+		return 0;
+
+	return __clk_alpha_pll_update_latch(pll);
+}
+
 static int __clk_alpha_pll_set_rate(struct clk_hw *hw, unsigned long rate,
 				    unsigned long prate,
 				    int (*is_enabled)(struct clk_hw *))
@@ -832,3 +858,264 @@ const struct clk_ops clk_alpha_pll_postdiv_ro_ops = {
 	.recalc_rate = clk_alpha_pll_postdiv_recalc_rate,
 };
 EXPORT_SYMBOL_GPL(clk_alpha_pll_postdiv_ro_ops);
+
+void clk_fabia_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap,
+			     const struct alpha_pll_config *config)
+{
+	u32 val, mask;
+
+	if (config->l)
+		regmap_write(regmap, PLL_L_VAL(pll), config->l);
+
+	if (config->alpha)
+		regmap_write(regmap, PLL_FRAC(pll), config->alpha);
+
+	if (config->config_ctl_val)
+		regmap_write(regmap, PLL_CONFIG_CTL(pll),
+						config->config_ctl_val);
+
+	if (config->post_div_mask) {
+		mask = config->post_div_mask;
+		val = config->post_div_val;
+		regmap_update_bits(regmap, PLL_USER_CTL(pll), mask, val);
+	}
+
+	regmap_update_bits(regmap, PLL_MODE(pll), PLL_UPDATE_BYPASS,
+							PLL_UPDATE_BYPASS);
+
+	regmap_update_bits(regmap, PLL_MODE(pll), PLL_RESET_N, PLL_RESET_N);
+}
+
+static int alpha_pll_fabia_enable(struct clk_hw *hw)
+{
+	int ret;
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 val, opmode_val;
+	struct regmap *regmap = pll->clkr.regmap;
+
+	ret = regmap_read(regmap, PLL_MODE(pll), &val);
+	if (ret)
+		return ret;
+
+	/* If in FSM mode, just vote for it */
+	if (val & PLL_VOTE_FSM_ENA) {
+		ret = clk_enable_regmap(hw);
+		if (ret)
+			return ret;
+		return wait_for_pll_enable_active(pll);
+	}
+
+	ret = regmap_read(regmap, PLL_OPMODE(pll), &opmode_val);
+	if (ret)
+		return ret;
+
+	/* Skip If PLL is already running */
+	if ((opmode_val & FABIA_OPMODE_RUN) && (val & PLL_OUTCTRL))
+		return 0;
+
+	ret = regmap_update_bits(regmap, PLL_MODE(pll), PLL_OUTCTRL, 0);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(regmap, PLL_OPMODE(pll), FABIA_OPMODE_STANDBY);
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(regmap, PLL_MODE(pll), PLL_RESET_N,
+				 PLL_RESET_N);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(regmap, PLL_OPMODE(pll), FABIA_OPMODE_RUN);
+	if (ret)
+		return ret;
+
+	ret = wait_for_pll_enable_lock(pll);
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(regmap, PLL_USER_CTL(pll),
+				 FABIA_PLL_OUT_MASK, FABIA_PLL_OUT_MASK);
+	if (ret)
+		return ret;
+
+	return regmap_update_bits(regmap, PLL_MODE(pll), PLL_OUTCTRL,
+				 PLL_OUTCTRL);
+}
+
+static void alpha_pll_fabia_disable(struct clk_hw *hw)
+{
+	int ret;
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 val;
+	struct regmap *regmap = pll->clkr.regmap;
+
+	ret = regmap_read(regmap, PLL_MODE(pll), &val);
+	if (ret)
+		return;
+
+	/* If in FSM mode, just unvote it */
+	if (val & PLL_FSM_ENA) {
+		clk_disable_regmap(hw);
+		return;
+	}
+
+	ret = regmap_update_bits(regmap, PLL_MODE(pll), PLL_OUTCTRL, 0);
+	if (ret)
+		return;
+
+	/* Disable main outputs */
+	ret = regmap_update_bits(regmap, PLL_USER_CTL(pll), FABIA_PLL_OUT_MASK,
+				 0);
+	if (ret)
+		return;
+
+	/* Place the PLL in STANDBY */
+	regmap_write(regmap, PLL_OPMODE(pll), FABIA_OPMODE_STANDBY);
+}
+
+static unsigned long alpha_pll_fabia_recalc_rate(struct clk_hw *hw,
+						unsigned long parent_rate)
+{
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 l, frac, alpha_width = pll_alpha_width(pll);
+
+	regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l);
+	regmap_read(pll->clkr.regmap, PLL_FRAC(pll), &frac);
+
+	return alpha_pll_calc_rate(parent_rate, l, frac, alpha_width);
+}
+
+static int alpha_pll_fabia_set_rate(struct clk_hw *hw, unsigned long rate,
+						unsigned long prate)
+{
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 val, l, alpha_width = pll_alpha_width(pll);
+	u64 a;
+	unsigned long rrate;
+	int ret = 0;
+
+	ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val);
+	if (ret)
+		return ret;
+
+	rrate = alpha_pll_round_rate(rate, prate, &l, &a, alpha_width);
+
+	/*
+	 * Due to limited number of bits for fractional rate programming, the
+	 * rounded up rate could be marginally higher than the requested rate.
+	 */
+	if (rrate > (rate + FABIA_PLL_RATE_MARGIN) || rrate < rate) {
+		pr_err("Call set rate on the PLL with rounded rates!\n");
+		return -EINVAL;
+	}
+
+	regmap_write(pll->clkr.regmap, PLL_L_VAL(pll), l);
+	regmap_write(pll->clkr.regmap, PLL_FRAC(pll), a);
+
+	return __clk_alpha_pll_update_latch(pll);
+}
+
+const struct clk_ops clk_alpha_pll_fabia_ops = {
+	.enable = alpha_pll_fabia_enable,
+	.disable = alpha_pll_fabia_disable,
+	.is_enabled = clk_alpha_pll_is_enabled,
+	.set_rate = alpha_pll_fabia_set_rate,
+	.recalc_rate = alpha_pll_fabia_recalc_rate,
+	.round_rate = clk_alpha_pll_round_rate,
+};
+EXPORT_SYMBOL_GPL(clk_alpha_pll_fabia_ops);
+
+const struct clk_ops clk_alpha_pll_fixed_fabia_ops = {
+	.enable = alpha_pll_fabia_enable,
+	.disable = alpha_pll_fabia_disable,
+	.is_enabled = clk_alpha_pll_is_enabled,
+	.recalc_rate = alpha_pll_fabia_recalc_rate,
+	.round_rate = clk_alpha_pll_round_rate,
+};
+EXPORT_SYMBOL_GPL(clk_alpha_pll_fixed_fabia_ops);
+
+static unsigned long clk_alpha_pll_postdiv_fabia_recalc_rate(struct clk_hw *hw,
+					unsigned long parent_rate)
+{
+	struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw);
+	u32 i, div = 1, val;
+	int ret;
+
+	if (!pll->post_div_table) {
+		pr_err("Missing the post_div_table for the PLL\n");
+		return -EINVAL;
+	}
+
+	ret = regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &val);
+	if (ret)
+		return ret;
+
+	val >>= pll->post_div_shift;
+	val &= BIT(pll->width) - 1;
+
+	for (i = 0; i < pll->num_post_div; i++) {
+		if (pll->post_div_table[i].val == val) {
+			div = pll->post_div_table[i].div;
+			break;
+		}
+	}
+
+	return (parent_rate / div);
+}
+
+static long clk_alpha_pll_postdiv_fabia_round_rate(struct clk_hw *hw,
+				unsigned long rate, unsigned long *prate)
+{
+	struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw);
+
+	if (!pll->post_div_table) {
+		pr_err("Missing the post_div_table for the PLL\n");
+		return -EINVAL;
+	}
+
+	return divider_round_rate(hw, rate, prate, pll->post_div_table,
+				pll->width, CLK_DIVIDER_ROUND_CLOSEST);
+}
+
+static int clk_alpha_pll_postdiv_fabia_set_rate(struct clk_hw *hw,
+				unsigned long rate, unsigned long parent_rate)
+{
+	struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw);
+	int i, val = 0, div, ret;
+
+	/*
+	 * If the PLL is in FSM mode, then treat set_rate callback as a
+	 * no-operation.
+	 */
+	ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val);
+	if (ret)
+		return ret;
+
+	if (val & PLL_VOTE_FSM_ENA)
+		return 0;
+
+	if (!pll->post_div_table) {
+		pr_err("Missing the post_div_table for the PLL\n");
+		return -EINVAL;
+	}
+
+	div = DIV_ROUND_UP_ULL((u64)parent_rate, rate);
+	for (i = 0; i < pll->num_post_div; i++) {
+		if (pll->post_div_table[i].div == div) {
+			val = pll->post_div_table[i].val;
+			break;
+		}
+	}
+
+	return regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll),
+				(BIT(pll->width) - 1) << pll->post_div_shift,
+				val << pll->post_div_shift);
+}
+
+const struct clk_ops clk_alpha_pll_postdiv_fabia_ops = {
+	.recalc_rate = clk_alpha_pll_postdiv_fabia_recalc_rate,
+	.round_rate = clk_alpha_pll_postdiv_fabia_round_rate,
+	.set_rate = clk_alpha_pll_postdiv_fabia_set_rate,
+};
+EXPORT_SYMBOL_GPL(clk_alpha_pll_postdiv_fabia_ops);
diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h
index 7593e8a56cf2..f981b486c468 100644
--- a/drivers/clk/qcom/clk-alpha-pll.h
+++ b/drivers/clk/qcom/clk-alpha-pll.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2015, 2018, The Linux Foundation. All rights reserved.
  *
  * This software is licensed under the terms of the GNU General Public
  * License version 2, as published by the Free Software Foundation, and
@@ -22,6 +22,7 @@ enum {
 	CLK_ALPHA_PLL_TYPE_DEFAULT,
 	CLK_ALPHA_PLL_TYPE_HUAYRA,
 	CLK_ALPHA_PLL_TYPE_BRAMMO,
+	CLK_ALPHA_PLL_TYPE_FABIA,
 	CLK_ALPHA_PLL_TYPE_MAX,
 };
 
@@ -36,6 +37,8 @@ enum {
 	PLL_OFF_TEST_CTL,
 	PLL_OFF_TEST_CTL_U,
 	PLL_OFF_STATUS,
+	PLL_OFF_OPMODE,
+	PLL_OFF_FRAC,
 	PLL_OFF_MAX_REGS
 };
 
@@ -73,6 +76,10 @@ struct clk_alpha_pll {
  * @offset: base address of registers
  * @regs: alpha pll register map (see @clk_alpha_pll_regs)
  * @width: width of post-divider
+ * @post_div_shift: shift to differentiate between odd & even post-divider
+ * @post_div_table: table with PLL odd and even post-divider settings
+ * @num_post_div: Number of PLL post-divider settings
+ *
  * @clkr: regmap clock handle
  */
 struct clk_alpha_pll_postdiv {
@@ -81,6 +88,9 @@ struct clk_alpha_pll_postdiv {
 	const u8 *regs;
 
 	struct clk_regmap clkr;
+	int post_div_shift;
+	const struct clk_div_table *post_div_table;
+	size_t num_post_div;
 };
 
 struct alpha_pll_config {
@@ -109,7 +119,13 @@ extern const struct clk_ops clk_alpha_pll_postdiv_ops;
 extern const struct clk_ops clk_alpha_pll_huayra_ops;
 extern const struct clk_ops clk_alpha_pll_postdiv_ro_ops;
 
+extern const struct clk_ops clk_alpha_pll_fabia_ops;
+extern const struct clk_ops clk_alpha_pll_fixed_fabia_ops;
+extern const struct clk_ops clk_alpha_pll_postdiv_fabia_ops;
+
 void clk_alpha_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap,
 			     const struct alpha_pll_config *config);
+void clk_fabia_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap,
+				const struct alpha_pll_config *config);
 
 #endif
diff --git a/drivers/clk/qcom/clk-branch.c b/drivers/clk/qcom/clk-branch.c
index 26f7af315066..c58c5538b1b6 100644
--- a/drivers/clk/qcom/clk-branch.c
+++ b/drivers/clk/qcom/clk-branch.c
@@ -77,8 +77,11 @@ static int clk_branch_wait(const struct clk_branch *br, bool enabling,
 	bool voted = br->halt_check & BRANCH_VOTED;
 	const char *name = clk_hw_get_name(&br->clkr.hw);
 
-	/* Skip checking halt bit if the clock is in hardware gated mode */
-	if (clk_branch_in_hwcg_mode(br))
+	/*
+	 * Skip checking halt bit if we're explicitly ignoring the bit or the
+	 * clock is in hardware gated mode
+	 */
+	if (br->halt_check == BRANCH_HALT_SKIP || clk_branch_in_hwcg_mode(br))
 		return 0;
 
 	if (br->halt_check == BRANCH_HALT_DELAY || (!enabling && voted)) {
diff --git a/drivers/clk/qcom/clk-branch.h b/drivers/clk/qcom/clk-branch.h
index 284df3f3c55f..1702efb1c511 100644
--- a/drivers/clk/qcom/clk-branch.h
+++ b/drivers/clk/qcom/clk-branch.h
@@ -42,6 +42,7 @@ struct clk_branch {
 #define BRANCH_HALT_ENABLE		1 /* pol: 0 = halt */
 #define BRANCH_HALT_ENABLE_VOTED	(BRANCH_HALT_ENABLE | BRANCH_VOTED)
 #define BRANCH_HALT_DELAY		2 /* No bit to check; just delay */
+#define BRANCH_HALT_SKIP		3 /* Don't check halt bit */
 
 	struct clk_regmap clkr;
 };
diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h
index 2a7489a84e69..b209a2fe86b9 100644
--- a/drivers/clk/qcom/clk-rcg.h
+++ b/drivers/clk/qcom/clk-rcg.h
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2013, The Linux Foundation. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2013, 2018, The Linux Foundation. All rights reserved. */
 
 #ifndef __QCOM_CLK_RCG_H__
 #define __QCOM_CLK_RCG_H__
@@ -144,6 +134,7 @@ extern const struct clk_ops clk_dyn_rcg_ops;
  * @cmd_rcgr: corresponds to *_CMD_RCGR
  * @mnd_width: number of bits in m/n/d values
  * @hid_width: number of bits in half integer divider
+ * @safe_src_index: safe src index value
  * @parent_map: map from software's parent index to hardware's src_sel field
  * @freq_tbl: frequency table
  * @clkr: regmap clock handle
@@ -153,6 +144,7 @@ struct clk_rcg2 {
 	u32			cmd_rcgr;
 	u8			mnd_width;
 	u8			hid_width;
+	u8			safe_src_index;
 	const struct parent_map	*parent_map;
 	const struct freq_tbl	*freq_tbl;
 	struct clk_regmap	clkr;
@@ -167,5 +159,6 @@ extern const struct clk_ops clk_byte_ops;
 extern const struct clk_ops clk_byte2_ops;
 extern const struct clk_ops clk_pixel_ops;
 extern const struct clk_ops clk_gfx3d_ops;
+extern const struct clk_ops clk_rcg2_shared_ops;
 
 #endif
diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c
index bbeaf9c09dbb..52208d4165f4 100644
--- a/drivers/clk/qcom/clk-rcg2.c
+++ b/drivers/clk/qcom/clk-rcg2.c
@@ -1,14 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2013, The Linux Foundation. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+ * Copyright (c) 2013, 2018, The Linux Foundation. All rights reserved.
  */
 
 #include <linux/kernel.h>
@@ -42,6 +34,7 @@
 #define CFG_MODE_SHIFT		12
 #define CFG_MODE_MASK		(0x3 << CFG_MODE_SHIFT)
 #define CFG_MODE_DUAL_EDGE	(0x2 << CFG_MODE_SHIFT)
+#define CFG_HW_CLK_CTRL_MASK	BIT(20)
 
 #define M_REG			0x8
 #define N_REG			0xc
@@ -211,6 +204,7 @@ static int _freq_tbl_determine_rate(struct clk_hw *hw, const struct freq_tbl *f,
 	clk_flags = clk_hw_get_flags(hw);
 	p = clk_hw_get_parent_by_index(hw, index);
 	if (clk_flags & CLK_SET_RATE_PARENT) {
+		rate = f->freq;
 		if (f->pre_div) {
 			rate /= 2;
 			rate *= f->pre_div + 1;
@@ -248,7 +242,7 @@ static int clk_rcg2_determine_floor_rate(struct clk_hw *hw,
 	return _freq_tbl_determine_rate(hw, rcg->freq_tbl, req, FLOOR);
 }
 
-static int clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f)
+static int __clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f)
 {
 	u32 cfg, mask;
 	struct clk_hw *hw = &rcg->clkr.hw;
@@ -276,13 +270,21 @@ static int clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f)
 	}
 
 	mask = BIT(rcg->hid_width) - 1;
-	mask |= CFG_SRC_SEL_MASK | CFG_MODE_MASK;
+	mask |= CFG_SRC_SEL_MASK | CFG_MODE_MASK | CFG_HW_CLK_CTRL_MASK;
 	cfg = f->pre_div << CFG_SRC_DIV_SHIFT;
 	cfg |= rcg->parent_map[index].cfg << CFG_SRC_SEL_SHIFT;
 	if (rcg->mnd_width && f->n && (f->m != f->n))
 		cfg |= CFG_MODE_DUAL_EDGE;
-	ret = regmap_update_bits(rcg->clkr.regmap,
-			rcg->cmd_rcgr + CFG_REG, mask, cfg);
+
+	return regmap_update_bits(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG,
+					mask, cfg);
+}
+
+static int clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f)
+{
+	int ret;
+
+	ret = __clk_rcg2_configure(rcg, f);
 	if (ret)
 		return ret;
 
@@ -789,3 +791,141 @@ const struct clk_ops clk_gfx3d_ops = {
 	.determine_rate = clk_gfx3d_determine_rate,
 };
 EXPORT_SYMBOL_GPL(clk_gfx3d_ops);
+
+static int clk_rcg2_set_force_enable(struct clk_hw *hw)
+{
+	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
+	const char *name = clk_hw_get_name(hw);
+	int ret, count;
+
+	ret = regmap_update_bits(rcg->clkr.regmap, rcg->cmd_rcgr + CMD_REG,
+				 CMD_ROOT_EN, CMD_ROOT_EN);
+	if (ret)
+		return ret;
+
+	/* wait for RCG to turn ON */
+	for (count = 500; count > 0; count--) {
+		if (clk_rcg2_is_enabled(hw))
+			return 0;
+
+		udelay(1);
+	}
+
+	pr_err("%s: RCG did not turn on\n", name);
+	return -ETIMEDOUT;
+}
+
+static int clk_rcg2_clear_force_enable(struct clk_hw *hw)
+{
+	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
+
+	return regmap_update_bits(rcg->clkr.regmap, rcg->cmd_rcgr + CMD_REG,
+					CMD_ROOT_EN, 0);
+}
+
+static int
+clk_rcg2_shared_force_enable_clear(struct clk_hw *hw, const struct freq_tbl *f)
+{
+	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
+	int ret;
+
+	ret = clk_rcg2_set_force_enable(hw);
+	if (ret)
+		return ret;
+
+	ret = clk_rcg2_configure(rcg, f);
+	if (ret)
+		return ret;
+
+	return clk_rcg2_clear_force_enable(hw);
+}
+
+static int clk_rcg2_shared_set_rate(struct clk_hw *hw, unsigned long rate,
+				    unsigned long parent_rate)
+{
+	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
+	const struct freq_tbl *f;
+
+	f = qcom_find_freq(rcg->freq_tbl, rate);
+	if (!f)
+		return -EINVAL;
+
+	/*
+	 * In case clock is disabled, update the CFG, M, N and D registers
+	 * and don't hit the update bit of CMD register.
+	 */
+	if (!__clk_is_enabled(hw->clk))
+		return __clk_rcg2_configure(rcg, f);
+
+	return clk_rcg2_shared_force_enable_clear(hw, f);
+}
+
+static int clk_rcg2_shared_set_rate_and_parent(struct clk_hw *hw,
+		unsigned long rate, unsigned long parent_rate, u8 index)
+{
+	return clk_rcg2_shared_set_rate(hw, rate, parent_rate);
+}
+
+static int clk_rcg2_shared_enable(struct clk_hw *hw)
+{
+	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
+	int ret;
+
+	/*
+	 * Set the update bit because required configuration has already
+	 * been written in clk_rcg2_shared_set_rate()
+	 */
+	ret = clk_rcg2_set_force_enable(hw);
+	if (ret)
+		return ret;
+
+	ret = update_config(rcg);
+	if (ret)
+		return ret;
+
+	return clk_rcg2_clear_force_enable(hw);
+}
+
+static void clk_rcg2_shared_disable(struct clk_hw *hw)
+{
+	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
+	u32 cfg;
+
+	/*
+	 * Store current configuration as switching to safe source would clear
+	 * the SRC and DIV of CFG register
+	 */
+	regmap_read(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG, &cfg);
+
+	/*
+	 * Park the RCG at a safe configuration - sourced off of safe source.
+	 * Force enable and disable the RCG while configuring it to safeguard
+	 * against any update signal coming from the downstream clock.
+	 * The current parent is still prepared and enabled at this point, and
+	 * the safe source is always on while application processor subsystem
+	 * is online. Therefore, the RCG can safely switch its parent.
+	 */
+	clk_rcg2_set_force_enable(hw);
+
+	regmap_write(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG,
+		     rcg->safe_src_index << CFG_SRC_SEL_SHIFT);
+
+	update_config(rcg);
+
+	clk_rcg2_clear_force_enable(hw);
+
+	/* Write back the stored configuration corresponding to current rate */
+	regmap_write(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG, cfg);
+}
+
+const struct clk_ops clk_rcg2_shared_ops = {
+	.enable = clk_rcg2_shared_enable,
+	.disable = clk_rcg2_shared_disable,
+	.get_parent = clk_rcg2_get_parent,
+	.set_parent = clk_rcg2_set_parent,
+	.recalc_rate = clk_rcg2_recalc_rate,
+	.determine_rate = clk_rcg2_determine_rate,
+	.set_rate = clk_rcg2_shared_set_rate,
+	.set_rate_and_parent = clk_rcg2_shared_set_rate_and_parent,
+};
+EXPORT_SYMBOL_GPL(clk_rcg2_shared_ops);
diff --git a/drivers/clk/qcom/gcc-msm8998.c b/drivers/clk/qcom/gcc-msm8998.c
new file mode 100644
index 000000000000..78d87f5c7098
--- /dev/null
+++ b/drivers/clk/qcom/gcc-msm8998.c
@@ -0,0 +1,2834 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/kernel.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+
+#include <dt-bindings/clock/qcom,gcc-msm8998.h>
+
+#include "common.h"
+#include "clk-regmap.h"
+#include "clk-alpha-pll.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-branch.h"
+#include "reset.h"
+#include "gdsc.h"
+
+#define F(f, s, h, m, n) { (f), (s), (2 * (h) - 1), (m), (n) }
+
+enum {
+	P_AUD_REF_CLK,
+	P_CORE_BI_PLL_TEST_SE,
+	P_GPLL0_OUT_MAIN,
+	P_GPLL4_OUT_MAIN,
+	P_PLL0_EARLY_DIV_CLK_SRC,
+	P_SLEEP_CLK,
+	P_XO,
+};
+
+static const struct parent_map gcc_parent_map_0[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_PLL0_EARLY_DIV_CLK_SRC, 6 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_0[] = {
+	"xo",
+	"gpll0_out_main",
+	"gpll0_out_main",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_1[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_1[] = {
+	"xo",
+	"gpll0_out_main",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_2[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_SLEEP_CLK, 5 },
+	{ P_PLL0_EARLY_DIV_CLK_SRC, 6 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_2[] = {
+	"xo",
+	"gpll0_out_main",
+	"core_pi_sleep_clk",
+	"gpll0_out_main",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_3[] = {
+	{ P_XO, 0 },
+	{ P_SLEEP_CLK, 5 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_3[] = {
+	"xo",
+	"core_pi_sleep_clk",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_4[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL4_OUT_MAIN, 5 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_4[] = {
+	"xo",
+	"gpll0_out_main",
+	"gpll4_out_main",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_5[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_AUD_REF_CLK, 2 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_5[] = {
+	"xo",
+	"gpll0_out_main",
+	"aud_ref_clk",
+	"core_bi_pll_test_se",
+};
+
+static struct pll_vco fabia_vco[] = {
+	{ 250000000, 2000000000, 0 },
+	{ 125000000, 1000000000, 1 },
+};
+
+static struct clk_alpha_pll gpll0 = {
+	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.vco_table = fabia_vco,
+	.num_vco = ARRAY_SIZE(fabia_vco),
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpll0",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		}
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll0_out_even = {
+	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll0_out_even",
+		.parent_names = (const char *[]){ "gpll0" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll0_out_main = {
+	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll0_out_main",
+		.parent_names = (const char *[]){ "gpll0" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll0_out_odd = {
+	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll0_out_odd",
+		.parent_names = (const char *[]){ "gpll0" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll0_out_test = {
+	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll0_out_test",
+		.parent_names = (const char *[]){ "gpll0" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll1 = {
+	.offset = 0x1000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.vco_table = fabia_vco,
+	.num_vco = ARRAY_SIZE(fabia_vco),
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(1),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpll1",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		}
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll1_out_even = {
+	.offset = 0x1000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll1_out_even",
+		.parent_names = (const char *[]){ "gpll1" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll1_out_main = {
+	.offset = 0x1000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll1_out_main",
+		.parent_names = (const char *[]){ "gpll1" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll1_out_odd = {
+	.offset = 0x1000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll1_out_odd",
+		.parent_names = (const char *[]){ "gpll1" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll1_out_test = {
+	.offset = 0x1000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll1_out_test",
+		.parent_names = (const char *[]){ "gpll1" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll2 = {
+	.offset = 0x2000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.vco_table = fabia_vco,
+	.num_vco = ARRAY_SIZE(fabia_vco),
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(2),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpll2",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		}
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll2_out_even = {
+	.offset = 0x2000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll2_out_even",
+		.parent_names = (const char *[]){ "gpll2" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll2_out_main = {
+	.offset = 0x2000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll2_out_main",
+		.parent_names = (const char *[]){ "gpll2" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll2_out_odd = {
+	.offset = 0x2000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll2_out_odd",
+		.parent_names = (const char *[]){ "gpll2" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll2_out_test = {
+	.offset = 0x2000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll2_out_test",
+		.parent_names = (const char *[]){ "gpll2" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll3 = {
+	.offset = 0x3000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.vco_table = fabia_vco,
+	.num_vco = ARRAY_SIZE(fabia_vco),
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(3),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpll3",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		}
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll3_out_even = {
+	.offset = 0x3000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll3_out_even",
+		.parent_names = (const char *[]){ "gpll3" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll3_out_main = {
+	.offset = 0x3000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll3_out_main",
+		.parent_names = (const char *[]){ "gpll3" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll3_out_odd = {
+	.offset = 0x3000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll3_out_odd",
+		.parent_names = (const char *[]){ "gpll3" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll3_out_test = {
+	.offset = 0x3000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll3_out_test",
+		.parent_names = (const char *[]){ "gpll3" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll4 = {
+	.offset = 0x77000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.vco_table = fabia_vco,
+	.num_vco = ARRAY_SIZE(fabia_vco),
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpll4",
+			.parent_names = (const char *[]){ "xo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		}
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll4_out_even = {
+	.offset = 0x77000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll4_out_even",
+		.parent_names = (const char *[]){ "gpll4" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll4_out_main = {
+	.offset = 0x77000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll4_out_main",
+		.parent_names = (const char *[]){ "gpll4" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll4_out_odd = {
+	.offset = 0x77000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll4_out_odd",
+		.parent_names = (const char *[]){ "gpll4" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll4_out_test = {
+	.offset = 0x77000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll4_out_test",
+		.parent_names = (const char *[]){ "gpll4" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_blsp1_qup1_i2c_apps_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x19020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup1_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_blsp1_qup1_spi_apps_clk_src[] = {
+	F(960000, P_XO, 10, 1, 2),
+	F(4800000, P_XO, 4, 0, 0),
+	F(9600000, P_XO, 2, 0, 0),
+	F(15000000, P_GPLL0_OUT_MAIN, 10, 1, 4),
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_GPLL0_OUT_MAIN, 12, 1, 2),
+	F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_qup1_spi_apps_clk_src = {
+	.cmd_rcgr = 0x1900c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup1_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup2_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x1b020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup2_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup2_spi_apps_clk_src = {
+	.cmd_rcgr = 0x1b00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup2_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup3_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x1d020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup3_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup3_spi_apps_clk_src = {
+	.cmd_rcgr = 0x1d00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup3_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup4_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x1f020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup4_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup4_spi_apps_clk_src = {
+	.cmd_rcgr = 0x1f00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup4_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup5_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x21020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup5_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup5_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2100c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup5_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup6_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x23020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup6_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup6_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2300c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup6_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_blsp1_uart1_apps_clk_src[] = {
+	F(3686400, P_GPLL0_OUT_MAIN, 1, 96, 15625),
+	F(7372800, P_GPLL0_OUT_MAIN, 1, 192, 15625),
+	F(14745600, P_GPLL0_OUT_MAIN, 1, 384, 15625),
+	F(16000000, P_GPLL0_OUT_MAIN, 5, 2, 15),
+	F(19200000, P_XO, 1, 0, 0),
+	F(24000000, P_GPLL0_OUT_MAIN, 5, 1, 5),
+	F(32000000, P_GPLL0_OUT_MAIN, 1, 4, 75),
+	F(40000000, P_GPLL0_OUT_MAIN, 15, 0, 0),
+	F(46400000, P_GPLL0_OUT_MAIN, 1, 29, 375),
+	F(48000000, P_GPLL0_OUT_MAIN, 12.5, 0, 0),
+	F(51200000, P_GPLL0_OUT_MAIN, 1, 32, 375),
+	F(56000000, P_GPLL0_OUT_MAIN, 1, 7, 75),
+	F(58982400, P_GPLL0_OUT_MAIN, 1, 1536, 15625),
+	F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0),
+	F(63157895, P_GPLL0_OUT_MAIN, 9.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_uart1_apps_clk_src = {
+	.cmd_rcgr = 0x1a00c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart1_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart2_apps_clk_src = {
+	.cmd_rcgr = 0x1c00c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart2_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart3_apps_clk_src = {
+	.cmd_rcgr = 0x1e00c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart3_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup1_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x26020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup1_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup1_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2600c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup1_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup2_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x28020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup2_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup2_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2800c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup2_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup3_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x2a020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup3_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup3_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2a00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup3_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup4_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x2c020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup4_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup4_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2c00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup4_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup5_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x2e020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup5_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup5_spi_apps_clk_src = {
+	.cmd_rcgr = 0x2e00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup5_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup6_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x30020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_blsp1_qup1_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup6_i2c_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup6_spi_apps_clk_src = {
+	.cmd_rcgr = 0x3000c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_qup1_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup6_spi_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart1_apps_clk_src = {
+	.cmd_rcgr = 0x2700c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_uart1_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart2_apps_clk_src = {
+	.cmd_rcgr = 0x2900c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_uart2_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart3_apps_clk_src = {
+	.cmd_rcgr = 0x2b00c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_blsp1_uart1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_uart3_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gp1_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gp1_clk_src = {
+	.cmd_rcgr = 0x64004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gp1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp1_clk_src",
+		.parent_names = gcc_parent_names_2,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gp2_clk_src = {
+	.cmd_rcgr = 0x65004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gp1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp2_clk_src",
+		.parent_names = gcc_parent_names_2,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gp3_clk_src = {
+	.cmd_rcgr = 0x66004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gp1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp3_clk_src",
+		.parent_names = gcc_parent_names_2,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_hmss_ahb_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(37500000, P_GPLL0_OUT_MAIN, 16, 0, 0),
+	F(75000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 hmss_ahb_clk_src = {
+	.cmd_rcgr = 0x48014,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_hmss_ahb_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "hmss_ahb_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_hmss_rbcpr_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 hmss_rbcpr_clk_src = {
+	.cmd_rcgr = 0x48044,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_hmss_rbcpr_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "hmss_rbcpr_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_pcie_aux_clk_src[] = {
+	F(1010526, P_XO, 1, 1, 19),
+	{ }
+};
+
+static struct clk_rcg2 pcie_aux_clk_src = {
+	.cmd_rcgr = 0x6c000,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_pcie_aux_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pcie_aux_clk_src",
+		.parent_names = gcc_parent_names_3,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_pdm2_clk_src[] = {
+	F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 pdm2_clk_src = {
+	.cmd_rcgr = 0x33010,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_pdm2_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pdm2_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_sdcc2_apps_clk_src[] = {
+	F(144000, P_XO, 16, 3, 25),
+	F(400000, P_XO, 12, 1, 4),
+	F(20000000, P_GPLL0_OUT_MAIN, 15, 1, 2),
+	F(25000000, P_GPLL0_OUT_MAIN, 12, 1, 2),
+	F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc2_apps_clk_src = {
+	.cmd_rcgr = 0x14010,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_sdcc2_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc2_apps_clk_src",
+		.parent_names = gcc_parent_names_4,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_sdcc4_apps_clk_src[] = {
+	F(144000, P_XO, 16, 3, 25),
+	F(400000, P_XO, 12, 1, 4),
+	F(20000000, P_GPLL0_OUT_MAIN, 15, 1, 2),
+	F(25000000, P_GPLL0_OUT_MAIN, 12, 1, 2),
+	F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc4_apps_clk_src = {
+	.cmd_rcgr = 0x16010,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_sdcc4_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc4_apps_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_tsif_ref_clk_src[] = {
+	F(105495, P_XO, 1, 1, 182),
+	{ }
+};
+
+static struct clk_rcg2 tsif_ref_clk_src = {
+	.cmd_rcgr = 0x36010,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_tsif_ref_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "tsif_ref_clk_src",
+		.parent_names = gcc_parent_names_5,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_ufs_axi_clk_src[] = {
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 ufs_axi_clk_src = {
+	.cmd_rcgr = 0x75018,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_ufs_axi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ufs_axi_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_usb30_master_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(120000000, P_GPLL0_OUT_MAIN, 5, 0, 0),
+	F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb30_master_clk_src = {
+	.cmd_rcgr = 0xf014,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_usb30_master_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb30_master_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 usb30_mock_utmi_clk_src = {
+	.cmd_rcgr = 0xf028,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_hmss_rbcpr_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb30_mock_utmi_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_usb3_phy_aux_clk_src[] = {
+	F(1200000, P_XO, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb3_phy_aux_clk_src = {
+	.cmd_rcgr = 0x5000c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_usb3_phy_aux_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb3_phy_aux_clk_src",
+		.parent_names = gcc_parent_names_3,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_aggre1_noc_xo_clk = {
+	.halt_reg = 0x8202c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8202c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre1_noc_xo_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre1_ufs_axi_clk = {
+	.halt_reg = 0x82028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x82028,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre1_ufs_axi_clk",
+			.parent_names = (const char *[]){
+				"ufs_axi_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre1_usb3_axi_clk = {
+	.halt_reg = 0x82024,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x82024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre1_usb3_axi_clk",
+			.parent_names = (const char *[]){
+				"usb30_master_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_apss_qdss_tsctr_div2_clk = {
+	.halt_reg = 0x48090,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x48090,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_apss_qdss_tsctr_div2_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_apss_qdss_tsctr_div8_clk = {
+	.halt_reg = 0x48094,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x48094,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_apss_qdss_tsctr_div8_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_bimc_hmss_axi_clk = {
+	.halt_reg = 0x48004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(22),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_bimc_hmss_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_bimc_mss_q6_axi_clk = {
+	.halt_reg = 0x4401c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4401c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_bimc_mss_q6_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_ahb_clk = {
+	.halt_reg = 0x17004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(17),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = {
+	.halt_reg = 0x19008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x19008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup1_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup1_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = {
+	.halt_reg = 0x19004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x19004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup1_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup1_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = {
+	.halt_reg = 0x1b008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1b008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup2_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup2_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = {
+	.halt_reg = 0x1b004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1b004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup2_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup2_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = {
+	.halt_reg = 0x1d008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1d008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup3_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup3_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = {
+	.halt_reg = 0x1d004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1d004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup3_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup3_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = {
+	.halt_reg = 0x1f008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1f008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup4_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup4_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = {
+	.halt_reg = 0x1f004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1f004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup4_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup4_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = {
+	.halt_reg = 0x21008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup5_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup5_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = {
+	.halt_reg = 0x21004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup5_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup5_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = {
+	.halt_reg = 0x23008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x23008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup6_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup6_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = {
+	.halt_reg = 0x23004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x23004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup6_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup6_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_sleep_clk = {
+	.halt_reg = 0x17008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(16),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart1_apps_clk = {
+	.halt_reg = 0x1a004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart1_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_uart1_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart2_apps_clk = {
+	.halt_reg = 0x1c004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1c004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart2_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_uart2_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart3_apps_clk = {
+	.halt_reg = 0x1e004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1e004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart3_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_uart3_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_ahb_clk = {
+	.halt_reg = 0x25004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(15),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup1_i2c_apps_clk = {
+	.halt_reg = 0x26008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x26008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup1_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup1_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup1_spi_apps_clk = {
+	.halt_reg = 0x26004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x26004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup1_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup1_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup2_i2c_apps_clk = {
+	.halt_reg = 0x28008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup2_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup2_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup2_spi_apps_clk = {
+	.halt_reg = 0x28004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup2_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup2_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup3_i2c_apps_clk = {
+	.halt_reg = 0x2a008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2a008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup3_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup3_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup3_spi_apps_clk = {
+	.halt_reg = 0x2a004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup3_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup3_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup4_i2c_apps_clk = {
+	.halt_reg = 0x2c008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2c008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup4_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup4_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup4_spi_apps_clk = {
+	.halt_reg = 0x2c004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2c004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup4_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup4_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup5_i2c_apps_clk = {
+	.halt_reg = 0x2e008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup5_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup5_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup5_spi_apps_clk = {
+	.halt_reg = 0x2e004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup5_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup5_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup6_i2c_apps_clk = {
+	.halt_reg = 0x30008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x30008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup6_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup6_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup6_spi_apps_clk = {
+	.halt_reg = 0x30004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x30004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup6_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_qup6_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_sleep_clk = {
+	.halt_reg = 0x25008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(14),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart1_apps_clk = {
+	.halt_reg = 0x27004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x27004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_uart1_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_uart1_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart2_apps_clk = {
+	.halt_reg = 0x29004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_uart2_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_uart2_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart3_apps_clk = {
+	.halt_reg = 0x2b004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2b004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_uart3_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp2_uart3_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cfg_noc_usb3_axi_clk = {
+	.halt_reg = 0x5018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_cfg_noc_usb3_axi_clk",
+			.parent_names = (const char *[]){
+				"usb30_master_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x64000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x64000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp1_clk",
+			.parent_names = (const char *[]){
+				"gp1_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x65000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x65000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp2_clk",
+			.parent_names = (const char *[]){
+				"gp2_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x66000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x66000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp3_clk",
+			.parent_names = (const char *[]){
+				"gp3_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_bimc_gfx_clk = {
+	.halt_reg = 0x71010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x71010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_bimc_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_bimc_gfx_src_clk = {
+	.halt_reg = 0x7100c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x7100c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_bimc_gfx_src_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_cfg_ahb_clk = {
+	.halt_reg = 0x71004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x71004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_snoc_dvm_gfx_clk = {
+	.halt_reg = 0x71018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x71018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_snoc_dvm_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_hmss_ahb_clk = {
+	.halt_reg = 0x48000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(21),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_hmss_ahb_clk",
+			.parent_names = (const char *[]){
+				"hmss_ahb_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_hmss_at_clk = {
+	.halt_reg = 0x48010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x48010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_hmss_at_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_hmss_dvm_bus_clk = {
+	.halt_reg = 0x4808c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4808c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_hmss_dvm_bus_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_hmss_rbcpr_clk = {
+	.halt_reg = 0x48008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x48008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_hmss_rbcpr_clk",
+			.parent_names = (const char *[]){
+				"hmss_rbcpr_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_hmss_trig_clk = {
+	.halt_reg = 0x4800c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4800c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_hmss_trig_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_lpass_at_clk = {
+	.halt_reg = 0x47020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x47020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_lpass_at_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_lpass_trig_clk = {
+	.halt_reg = 0x4701c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4701c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_lpass_trig_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mmss_noc_cfg_ahb_clk = {
+	.halt_reg = 0x9004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x9004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mmss_noc_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mmss_qm_ahb_clk = {
+	.halt_reg = 0x9030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x9030,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mmss_qm_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mmss_qm_core_clk = {
+	.halt_reg = 0x900c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x900c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mmss_qm_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mmss_sys_noc_axi_clk = {
+	.halt_reg = 0x9000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x9000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mmss_sys_noc_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_at_clk = {
+	.halt_reg = 0x8a00c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8a00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_at_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_aux_clk = {
+	.halt_reg = 0x6b014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6b014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_aux_clk",
+			.parent_names = (const char *[]){
+				"pcie_aux_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_cfg_ahb_clk = {
+	.halt_reg = 0x6b010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6b010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_mstr_axi_clk = {
+	.halt_reg = 0x6b00c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6b00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_mstr_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_pipe_clk = {
+	.halt_reg = 0x6b018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6b018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_pipe_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_slv_axi_clk = {
+	.halt_reg = 0x6b008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6b008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_slv_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_phy_aux_clk = {
+	.halt_reg = 0x6f004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6f004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_phy_aux_clk",
+			.parent_names = (const char *[]){
+				"pcie_aux_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm2_clk = {
+	.halt_reg = 0x3300c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3300c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm2_clk",
+			.parent_names = (const char *[]){
+				"pdm2_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_ahb_clk = {
+	.halt_reg = 0x33004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x33004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_xo4_clk = {
+	.halt_reg = 0x33008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x33008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm_xo4_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_prng_ahb_clk = {
+	.halt_reg = 0x34004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(13),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_prng_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_ahb_clk = {
+	.halt_reg = 0x14008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x14008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_apps_clk = {
+	.halt_reg = 0x14004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x14004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_apps_clk",
+			.parent_names = (const char *[]){
+				"sdcc2_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_ahb_clk = {
+	.halt_reg = 0x16008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x16008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc4_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_apps_clk = {
+	.halt_reg = 0x16004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x16004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc4_apps_clk",
+			.parent_names = (const char *[]){
+				"sdcc4_apps_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tsif_ahb_clk = {
+	.halt_reg = 0x36004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x36004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_tsif_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tsif_inactivity_timers_clk = {
+	.halt_reg = 0x3600c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_tsif_inactivity_timers_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tsif_ref_clk = {
+	.halt_reg = 0x36008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x36008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_tsif_ref_clk",
+			.parent_names = (const char *[]){
+				"tsif_ref_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_ahb_clk = {
+	.halt_reg = 0x7500c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x7500c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_axi_clk = {
+	.halt_reg = 0x75008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x75008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_axi_clk",
+			.parent_names = (const char *[]){
+				"ufs_axi_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_ice_core_clk = {
+	.halt_reg = 0x7600c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x7600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_ice_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_aux_clk = {
+	.halt_reg = 0x76040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x76040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_aux_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_rx_symbol_0_clk = {
+	.halt_reg = 0x75014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x75014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_rx_symbol_0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_rx_symbol_1_clk = {
+	.halt_reg = 0x7605c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x7605c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_rx_symbol_1_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_tx_symbol_0_clk = {
+	.halt_reg = 0x75010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x75010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_tx_symbol_0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_unipro_core_clk = {
+	.halt_reg = 0x76008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x76008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_unipro_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_master_clk = {
+	.halt_reg = 0xf008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xf008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_master_clk",
+			.parent_names = (const char *[]){
+				"usb30_master_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_mock_utmi_clk = {
+	.halt_reg = 0xf010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xf010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_mock_utmi_clk",
+			.parent_names = (const char *[]){
+				"usb30_mock_utmi_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_sleep_clk = {
+	.halt_reg = 0xf00c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xf00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_phy_aux_clk = {
+	.halt_reg = 0x50000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x50000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_phy_aux_clk",
+			.parent_names = (const char *[]){
+				"usb3_phy_aux_clk_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_phy_pipe_clk = {
+	.halt_reg = 0x50004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x50004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_phy_pipe_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_phy_cfg_ahb2phy_clk = {
+	.halt_reg = 0x6a004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_phy_cfg_ahb2phy_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc pcie_0_gdsc = {
+	.gdscr = 0x6b004,
+	.gds_hw_ctrl = 0x0,
+	.pd = {
+		.name = "pcie_0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = VOTABLE,
+};
+
+static struct gdsc ufs_gdsc = {
+	.gdscr = 0x75004,
+	.gds_hw_ctrl = 0x0,
+	.pd = {
+		.name = "ufs_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = VOTABLE,
+};
+
+static struct gdsc usb_30_gdsc = {
+	.gdscr = 0xf004,
+	.gds_hw_ctrl = 0x0,
+	.pd = {
+		.name = "usb_30_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = VOTABLE,
+};
+
+static struct clk_regmap *gcc_msm8998_clocks[] = {
+	[BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP1_SPI_APPS_CLK_SRC] = &blsp1_qup1_spi_apps_clk_src.clkr,
+	[BLSP1_QUP2_I2C_APPS_CLK_SRC] = &blsp1_qup2_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP2_SPI_APPS_CLK_SRC] = &blsp1_qup2_spi_apps_clk_src.clkr,
+	[BLSP1_QUP3_I2C_APPS_CLK_SRC] = &blsp1_qup3_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP3_SPI_APPS_CLK_SRC] = &blsp1_qup3_spi_apps_clk_src.clkr,
+	[BLSP1_QUP4_I2C_APPS_CLK_SRC] = &blsp1_qup4_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP4_SPI_APPS_CLK_SRC] = &blsp1_qup4_spi_apps_clk_src.clkr,
+	[BLSP1_QUP5_I2C_APPS_CLK_SRC] = &blsp1_qup5_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP5_SPI_APPS_CLK_SRC] = &blsp1_qup5_spi_apps_clk_src.clkr,
+	[BLSP1_QUP6_I2C_APPS_CLK_SRC] = &blsp1_qup6_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP6_SPI_APPS_CLK_SRC] = &blsp1_qup6_spi_apps_clk_src.clkr,
+	[BLSP1_UART1_APPS_CLK_SRC] = &blsp1_uart1_apps_clk_src.clkr,
+	[BLSP1_UART2_APPS_CLK_SRC] = &blsp1_uart2_apps_clk_src.clkr,
+	[BLSP1_UART3_APPS_CLK_SRC] = &blsp1_uart3_apps_clk_src.clkr,
+	[BLSP2_QUP1_I2C_APPS_CLK_SRC] = &blsp2_qup1_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP1_SPI_APPS_CLK_SRC] = &blsp2_qup1_spi_apps_clk_src.clkr,
+	[BLSP2_QUP2_I2C_APPS_CLK_SRC] = &blsp2_qup2_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP2_SPI_APPS_CLK_SRC] = &blsp2_qup2_spi_apps_clk_src.clkr,
+	[BLSP2_QUP3_I2C_APPS_CLK_SRC] = &blsp2_qup3_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP3_SPI_APPS_CLK_SRC] = &blsp2_qup3_spi_apps_clk_src.clkr,
+	[BLSP2_QUP4_I2C_APPS_CLK_SRC] = &blsp2_qup4_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP4_SPI_APPS_CLK_SRC] = &blsp2_qup4_spi_apps_clk_src.clkr,
+	[BLSP2_QUP5_I2C_APPS_CLK_SRC] = &blsp2_qup5_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP5_SPI_APPS_CLK_SRC] = &blsp2_qup5_spi_apps_clk_src.clkr,
+	[BLSP2_QUP6_I2C_APPS_CLK_SRC] = &blsp2_qup6_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP6_SPI_APPS_CLK_SRC] = &blsp2_qup6_spi_apps_clk_src.clkr,
+	[BLSP2_UART1_APPS_CLK_SRC] = &blsp2_uart1_apps_clk_src.clkr,
+	[BLSP2_UART2_APPS_CLK_SRC] = &blsp2_uart2_apps_clk_src.clkr,
+	[BLSP2_UART3_APPS_CLK_SRC] = &blsp2_uart3_apps_clk_src.clkr,
+	[GCC_AGGRE1_NOC_XO_CLK] = &gcc_aggre1_noc_xo_clk.clkr,
+	[GCC_AGGRE1_UFS_AXI_CLK] = &gcc_aggre1_ufs_axi_clk.clkr,
+	[GCC_AGGRE1_USB3_AXI_CLK] = &gcc_aggre1_usb3_axi_clk.clkr,
+	[GCC_APSS_QDSS_TSCTR_DIV2_CLK] = &gcc_apss_qdss_tsctr_div2_clk.clkr,
+	[GCC_APSS_QDSS_TSCTR_DIV8_CLK] = &gcc_apss_qdss_tsctr_div8_clk.clkr,
+	[GCC_BIMC_HMSS_AXI_CLK] = &gcc_bimc_hmss_axi_clk.clkr,
+	[GCC_BIMC_MSS_Q6_AXI_CLK] = &gcc_bimc_mss_q6_axi_clk.clkr,
+	[GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr,
+	[GCC_BLSP1_QUP1_I2C_APPS_CLK] = &gcc_blsp1_qup1_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP1_SPI_APPS_CLK] = &gcc_blsp1_qup1_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP2_I2C_APPS_CLK] = &gcc_blsp1_qup2_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP2_SPI_APPS_CLK] = &gcc_blsp1_qup2_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP3_I2C_APPS_CLK] = &gcc_blsp1_qup3_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP3_SPI_APPS_CLK] = &gcc_blsp1_qup3_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP4_I2C_APPS_CLK] = &gcc_blsp1_qup4_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP4_SPI_APPS_CLK] = &gcc_blsp1_qup4_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP5_I2C_APPS_CLK] = &gcc_blsp1_qup5_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP5_SPI_APPS_CLK] = &gcc_blsp1_qup5_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP6_I2C_APPS_CLK] = &gcc_blsp1_qup6_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP6_SPI_APPS_CLK] = &gcc_blsp1_qup6_spi_apps_clk.clkr,
+	[GCC_BLSP1_SLEEP_CLK] = &gcc_blsp1_sleep_clk.clkr,
+	[GCC_BLSP1_UART1_APPS_CLK] = &gcc_blsp1_uart1_apps_clk.clkr,
+	[GCC_BLSP1_UART2_APPS_CLK] = &gcc_blsp1_uart2_apps_clk.clkr,
+	[GCC_BLSP1_UART3_APPS_CLK] = &gcc_blsp1_uart3_apps_clk.clkr,
+	[GCC_BLSP2_AHB_CLK] = &gcc_blsp2_ahb_clk.clkr,
+	[GCC_BLSP2_QUP1_I2C_APPS_CLK] = &gcc_blsp2_qup1_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP1_SPI_APPS_CLK] = &gcc_blsp2_qup1_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP2_I2C_APPS_CLK] = &gcc_blsp2_qup2_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP2_SPI_APPS_CLK] = &gcc_blsp2_qup2_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP3_I2C_APPS_CLK] = &gcc_blsp2_qup3_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP3_SPI_APPS_CLK] = &gcc_blsp2_qup3_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP4_I2C_APPS_CLK] = &gcc_blsp2_qup4_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP4_SPI_APPS_CLK] = &gcc_blsp2_qup4_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP5_I2C_APPS_CLK] = &gcc_blsp2_qup5_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP5_SPI_APPS_CLK] = &gcc_blsp2_qup5_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP6_I2C_APPS_CLK] = &gcc_blsp2_qup6_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP6_SPI_APPS_CLK] = &gcc_blsp2_qup6_spi_apps_clk.clkr,
+	[GCC_BLSP2_SLEEP_CLK] = &gcc_blsp2_sleep_clk.clkr,
+	[GCC_BLSP2_UART1_APPS_CLK] = &gcc_blsp2_uart1_apps_clk.clkr,
+	[GCC_BLSP2_UART2_APPS_CLK] = &gcc_blsp2_uart2_apps_clk.clkr,
+	[GCC_BLSP2_UART3_APPS_CLK] = &gcc_blsp2_uart3_apps_clk.clkr,
+	[GCC_CFG_NOC_USB3_AXI_CLK] = &gcc_cfg_noc_usb3_axi_clk.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
+	[GCC_GPU_BIMC_GFX_CLK] = &gcc_gpu_bimc_gfx_clk.clkr,
+	[GCC_GPU_BIMC_GFX_SRC_CLK] = &gcc_gpu_bimc_gfx_src_clk.clkr,
+	[GCC_GPU_CFG_AHB_CLK] = &gcc_gpu_cfg_ahb_clk.clkr,
+	[GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr,
+	[GCC_HMSS_AHB_CLK] = &gcc_hmss_ahb_clk.clkr,
+	[GCC_HMSS_AT_CLK] = &gcc_hmss_at_clk.clkr,
+	[GCC_HMSS_DVM_BUS_CLK] = &gcc_hmss_dvm_bus_clk.clkr,
+	[GCC_HMSS_RBCPR_CLK] = &gcc_hmss_rbcpr_clk.clkr,
+	[GCC_HMSS_TRIG_CLK] = &gcc_hmss_trig_clk.clkr,
+	[GCC_LPASS_AT_CLK] = &gcc_lpass_at_clk.clkr,
+	[GCC_LPASS_TRIG_CLK] = &gcc_lpass_trig_clk.clkr,
+	[GCC_MMSS_NOC_CFG_AHB_CLK] = &gcc_mmss_noc_cfg_ahb_clk.clkr,
+	[GCC_MMSS_QM_AHB_CLK] = &gcc_mmss_qm_ahb_clk.clkr,
+	[GCC_MMSS_QM_CORE_CLK] = &gcc_mmss_qm_core_clk.clkr,
+	[GCC_MMSS_SYS_NOC_AXI_CLK] = &gcc_mmss_sys_noc_axi_clk.clkr,
+	[GCC_MSS_AT_CLK] = &gcc_mss_at_clk.clkr,
+	[GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr,
+	[GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr,
+	[GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr,
+	[GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr,
+	[GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr,
+	[GCC_PCIE_PHY_AUX_CLK] = &gcc_pcie_phy_aux_clk.clkr,
+	[GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr,
+	[GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr,
+	[GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr,
+	[GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr,
+	[GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr,
+	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
+	[GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr,
+	[GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr,
+	[GCC_TSIF_AHB_CLK] = &gcc_tsif_ahb_clk.clkr,
+	[GCC_TSIF_INACTIVITY_TIMERS_CLK] = &gcc_tsif_inactivity_timers_clk.clkr,
+	[GCC_TSIF_REF_CLK] = &gcc_tsif_ref_clk.clkr,
+	[GCC_UFS_AHB_CLK] = &gcc_ufs_ahb_clk.clkr,
+	[GCC_UFS_AXI_CLK] = &gcc_ufs_axi_clk.clkr,
+	[GCC_UFS_ICE_CORE_CLK] = &gcc_ufs_ice_core_clk.clkr,
+	[GCC_UFS_PHY_AUX_CLK] = &gcc_ufs_phy_aux_clk.clkr,
+	[GCC_UFS_RX_SYMBOL_0_CLK] = &gcc_ufs_rx_symbol_0_clk.clkr,
+	[GCC_UFS_RX_SYMBOL_1_CLK] = &gcc_ufs_rx_symbol_1_clk.clkr,
+	[GCC_UFS_TX_SYMBOL_0_CLK] = &gcc_ufs_tx_symbol_0_clk.clkr,
+	[GCC_UFS_UNIPRO_CORE_CLK] = &gcc_ufs_unipro_core_clk.clkr,
+	[GCC_USB30_MASTER_CLK] = &gcc_usb30_master_clk.clkr,
+	[GCC_USB30_MOCK_UTMI_CLK] = &gcc_usb30_mock_utmi_clk.clkr,
+	[GCC_USB30_SLEEP_CLK] = &gcc_usb30_sleep_clk.clkr,
+	[GCC_USB3_PHY_AUX_CLK] = &gcc_usb3_phy_aux_clk.clkr,
+	[GCC_USB3_PHY_PIPE_CLK] = &gcc_usb3_phy_pipe_clk.clkr,
+	[GCC_USB_PHY_CFG_AHB2PHY_CLK] = &gcc_usb_phy_cfg_ahb2phy_clk.clkr,
+	[GP1_CLK_SRC] = &gp1_clk_src.clkr,
+	[GP2_CLK_SRC] = &gp2_clk_src.clkr,
+	[GP3_CLK_SRC] = &gp3_clk_src.clkr,
+	[GPLL0] = &gpll0.clkr,
+	[GPLL0_OUT_EVEN] = &gpll0_out_even.clkr,
+	[GPLL0_OUT_MAIN] = &gpll0_out_main.clkr,
+	[GPLL0_OUT_ODD] = &gpll0_out_odd.clkr,
+	[GPLL0_OUT_TEST] = &gpll0_out_test.clkr,
+	[GPLL1] = &gpll1.clkr,
+	[GPLL1_OUT_EVEN] = &gpll1_out_even.clkr,
+	[GPLL1_OUT_MAIN] = &gpll1_out_main.clkr,
+	[GPLL1_OUT_ODD] = &gpll1_out_odd.clkr,
+	[GPLL1_OUT_TEST] = &gpll1_out_test.clkr,
+	[GPLL2] = &gpll2.clkr,
+	[GPLL2_OUT_EVEN] = &gpll2_out_even.clkr,
+	[GPLL2_OUT_MAIN] = &gpll2_out_main.clkr,
+	[GPLL2_OUT_ODD] = &gpll2_out_odd.clkr,
+	[GPLL2_OUT_TEST] = &gpll2_out_test.clkr,
+	[GPLL3] = &gpll3.clkr,
+	[GPLL3_OUT_EVEN] = &gpll3_out_even.clkr,
+	[GPLL3_OUT_MAIN] = &gpll3_out_main.clkr,
+	[GPLL3_OUT_ODD] = &gpll3_out_odd.clkr,
+	[GPLL3_OUT_TEST] = &gpll3_out_test.clkr,
+	[GPLL4] = &gpll4.clkr,
+	[GPLL4_OUT_EVEN] = &gpll4_out_even.clkr,
+	[GPLL4_OUT_MAIN] = &gpll4_out_main.clkr,
+	[GPLL4_OUT_ODD] = &gpll4_out_odd.clkr,
+	[GPLL4_OUT_TEST] = &gpll4_out_test.clkr,
+	[HMSS_AHB_CLK_SRC] = &hmss_ahb_clk_src.clkr,
+	[HMSS_RBCPR_CLK_SRC] = &hmss_rbcpr_clk_src.clkr,
+	[PCIE_AUX_CLK_SRC] = &pcie_aux_clk_src.clkr,
+	[PDM2_CLK_SRC] = &pdm2_clk_src.clkr,
+	[SDCC2_APPS_CLK_SRC] = &sdcc2_apps_clk_src.clkr,
+	[SDCC4_APPS_CLK_SRC] = &sdcc4_apps_clk_src.clkr,
+	[TSIF_REF_CLK_SRC] = &tsif_ref_clk_src.clkr,
+	[UFS_AXI_CLK_SRC] = &ufs_axi_clk_src.clkr,
+	[USB30_MASTER_CLK_SRC] = &usb30_master_clk_src.clkr,
+	[USB30_MOCK_UTMI_CLK_SRC] = &usb30_mock_utmi_clk_src.clkr,
+	[USB3_PHY_AUX_CLK_SRC] = &usb3_phy_aux_clk_src.clkr,
+};
+
+static struct gdsc *gcc_msm8998_gdscs[] = {
+	[PCIE_0_GDSC] = &pcie_0_gdsc,
+	[UFS_GDSC] = &ufs_gdsc,
+	[USB_30_GDSC] = &usb_30_gdsc,
+};
+
+static const struct qcom_reset_map gcc_msm8998_resets[] = {
+	[GCC_BLSP1_QUP1_BCR] = { 0x102400 },
+	[GCC_BLSP1_QUP2_BCR] = { 0x110592 },
+	[GCC_BLSP1_QUP3_BCR] = { 0x118784 },
+	[GCC_BLSP1_QUP4_BCR] = { 0x126976 },
+	[GCC_BLSP1_QUP5_BCR] = { 0x135168 },
+	[GCC_BLSP1_QUP6_BCR] = { 0x143360 },
+	[GCC_BLSP2_QUP1_BCR] = { 0x155648 },
+	[GCC_BLSP2_QUP2_BCR] = { 0x163840 },
+	[GCC_BLSP2_QUP3_BCR] = { 0x172032 },
+	[GCC_BLSP2_QUP4_BCR] = { 0x180224 },
+	[GCC_BLSP2_QUP5_BCR] = { 0x188416 },
+	[GCC_BLSP2_QUP6_BCR] = { 0x196608 },
+	[GCC_PCIE_0_BCR] = { 0x438272 },
+	[GCC_PDM_BCR] = { 0x208896 },
+	[GCC_SDCC2_BCR] = { 0x81920 },
+	[GCC_SDCC4_BCR] = { 0x90112 },
+	[GCC_TSIF_BCR] = { 0x221184 },
+	[GCC_UFS_BCR] = { 0x479232 },
+	[GCC_USB_30_BCR] = { 0x61440 },
+};
+
+static const struct regmap_config gcc_msm8998_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0x8f000,
+	.fast_io	= true,
+};
+
+static const struct qcom_cc_desc gcc_msm8998_desc = {
+	.config = &gcc_msm8998_regmap_config,
+	.clks = gcc_msm8998_clocks,
+	.num_clks = ARRAY_SIZE(gcc_msm8998_clocks),
+	.resets = gcc_msm8998_resets,
+	.num_resets = ARRAY_SIZE(gcc_msm8998_resets),
+	.gdscs = gcc_msm8998_gdscs,
+	.num_gdscs = ARRAY_SIZE(gcc_msm8998_gdscs),
+};
+
+static int gcc_msm8998_probe(struct platform_device *pdev)
+{
+	struct regmap *regmap;
+	int ret;
+
+	regmap = qcom_cc_map(pdev, &gcc_msm8998_desc);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	/*
+	 * Set the HMSS_AHB_CLK_SLEEP_ENA bit to allow the hmss_ahb_clk to be
+	 * turned off by hardware during certain apps low power modes.
+	 */
+	ret = regmap_update_bits(regmap, 0x52008, BIT(21), BIT(21));
+	if (ret)
+		return ret;
+
+	return qcom_cc_really_probe(pdev, &gcc_msm8998_desc, regmap);
+}
+
+static const struct of_device_id gcc_msm8998_match_table[] = {
+	{ .compatible = "qcom,gcc-msm8998" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_msm8998_match_table);
+
+static struct platform_driver gcc_msm8998_driver = {
+	.probe		= gcc_msm8998_probe,
+	.driver		= {
+		.name	= "gcc-msm8998",
+		.of_match_table = gcc_msm8998_match_table,
+	},
+};
+
+static int __init gcc_msm8998_init(void)
+{
+	return platform_driver_register(&gcc_msm8998_driver);
+}
+core_initcall(gcc_msm8998_init);
+
+static void __exit gcc_msm8998_exit(void)
+{
+	platform_driver_unregister(&gcc_msm8998_driver);
+}
+module_exit(gcc_msm8998_exit);
+
+MODULE_DESCRIPTION("QCOM GCC msm8998 Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:gcc-msm8998");
diff --git a/drivers/clk/qcom/gcc-sdm845.c b/drivers/clk/qcom/gcc-sdm845.c
new file mode 100644
index 000000000000..e78e6f5b99fc
--- /dev/null
+++ b/drivers/clk/qcom/gcc-sdm845.c
@@ -0,0 +1,3465 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/kernel.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+
+#include <dt-bindings/clock/qcom,gcc-sdm845.h>
+
+#include "common.h"
+#include "clk-regmap.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-branch.h"
+#include "clk-alpha-pll.h"
+#include "gdsc.h"
+#include "reset.h"
+
+#define F(f, s, h, m, n) { (f), (s), (2 * (h) - 1), (m), (n) }
+
+enum {
+	P_BI_TCXO,
+	P_AUD_REF_CLK,
+	P_CORE_BI_PLL_TEST_SE,
+	P_GPLL0_OUT_EVEN,
+	P_GPLL0_OUT_MAIN,
+	P_GPLL4_OUT_MAIN,
+	P_SLEEP_CLK,
+};
+
+static const struct parent_map gcc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL0_OUT_EVEN, 6 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_0[] = {
+	"bi_tcxo",
+	"gpll0",
+	"gpll0_out_even",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_SLEEP_CLK, 5 },
+	{ P_GPLL0_OUT_EVEN, 6 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_1[] = {
+	"bi_tcxo",
+	"gpll0",
+	"core_pi_sleep_clk",
+	"gpll0_out_even",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_SLEEP_CLK, 5 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_2[] = {
+	"bi_tcxo",
+	"core_pi_sleep_clk",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_3[] = {
+	"bi_tcxo",
+	"gpll0",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_4[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_4[] = {
+	"bi_tcxo",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_5[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL4_OUT_MAIN, 5 },
+	{ P_GPLL0_OUT_EVEN, 6 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_5[] = {
+	"bi_tcxo",
+	"gpll0",
+	"gpll4",
+	"gpll0_out_even",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_6[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_AUD_REF_CLK, 2 },
+	{ P_GPLL0_OUT_EVEN, 6 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_6[] = {
+	"bi_tcxo",
+	"gpll0",
+	"aud_ref_clk",
+	"gpll0_out_even",
+	"core_bi_pll_test_se",
+};
+
+static const char * const gcc_parent_names_7[] = {
+	"bi_tcxo",
+	"gpll0",
+	"gpll0_out_even",
+	"core_bi_pll_test_se",
+};
+
+static const char * const gcc_parent_names_8[] = {
+	"bi_tcxo",
+	"gpll0",
+	"core_bi_pll_test_se",
+};
+
+static const struct parent_map gcc_parent_map_10[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL4_OUT_MAIN, 5 },
+	{ P_GPLL0_OUT_EVEN, 6 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_10[] = {
+	"bi_tcxo",
+	"gpll0",
+	"gpll4",
+	"gpll0_out_even",
+	"core_bi_pll_test_se",
+};
+
+static struct clk_alpha_pll gpll0 = {
+	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA],
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpll0",
+			.parent_names = (const char *[]){ "bi_tcxo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_fabia_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gpll4 = {
+	.offset = 0x76000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA],
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpll4",
+			.parent_names = (const char *[]){ "bi_tcxo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_fabia_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_fabia_even[] = {
+	{ 0x0, 1 },
+	{ 0x1, 2 },
+	{ 0x3, 4 },
+	{ 0x7, 8 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpll0_out_even = {
+	.offset = 0x0,
+	.post_div_shift = 8,
+	.post_div_table = post_div_table_fabia_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_fabia_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll0_out_even",
+		.parent_names = (const char *[]){ "gpll0" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_fabia_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_cpuss_ahb_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_cpuss_ahb_clk_src = {
+	.cmd_rcgr = 0x48014,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_cpuss_ahb_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_cpuss_ahb_clk_src",
+		.parent_names = gcc_parent_names_7,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_cpuss_rbcpr_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_cpuss_rbcpr_clk_src = {
+	.cmd_rcgr = 0x4815c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_cpuss_rbcpr_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_cpuss_rbcpr_clk_src",
+		.parent_names = gcc_parent_names_8,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_gp1_clk_src = {
+	.cmd_rcgr = 0x64004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_gp1_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp2_clk_src = {
+	.cmd_rcgr = 0x65004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_gp2_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp3_clk_src = {
+	.cmd_rcgr = 0x66004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_gp3_clk_src",
+		.parent_names = gcc_parent_names_1,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = {
+	F(9600000, P_BI_TCXO, 2, 0, 0),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_0_aux_clk_src = {
+	.cmd_rcgr = 0x6b028,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_pcie_0_aux_clk_src",
+		.parent_names = gcc_parent_names_2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie_1_aux_clk_src = {
+	.cmd_rcgr = 0x8d028,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_pcie_1_aux_clk_src",
+		.parent_names = gcc_parent_names_2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_phy_refgen_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_phy_refgen_clk_src = {
+	.cmd_rcgr = 0x6f014,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_phy_refgen_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_pcie_phy_refgen_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = {
+	F(9600000, P_BI_TCXO, 2, 0, 0),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pdm2_clk_src = {
+	.cmd_rcgr = 0x33010,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pdm2_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_pdm2_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = {
+	F(7372800, P_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(64000000, P_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(80000000, P_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(102400000, P_GPLL0_OUT_EVEN, 1, 128, 375),
+	F(112000000, P_GPLL0_OUT_EVEN, 1, 28, 75),
+	F(117964800, P_GPLL0_OUT_EVEN, 1, 6144, 15625),
+	F(120000000, P_GPLL0_OUT_EVEN, 2.5, 0, 0),
+	F(128000000, P_GPLL0_OUT_MAIN, 1, 16, 75),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = {
+	.cmd_rcgr = 0x17034,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap0_s0_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = {
+	.cmd_rcgr = 0x17164,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap0_s1_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = {
+	.cmd_rcgr = 0x17294,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap0_s2_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = {
+	.cmd_rcgr = 0x173c4,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap0_s3_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = {
+	.cmd_rcgr = 0x174f4,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap0_s4_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = {
+	.cmd_rcgr = 0x17624,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap0_s5_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s6_clk_src = {
+	.cmd_rcgr = 0x17754,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap0_s6_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s7_clk_src = {
+	.cmd_rcgr = 0x17884,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap0_s7_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = {
+	.cmd_rcgr = 0x18018,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap1_s0_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = {
+	.cmd_rcgr = 0x18148,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap1_s1_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s2_clk_src = {
+	.cmd_rcgr = 0x18278,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap1_s2_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = {
+	.cmd_rcgr = 0x183a8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap1_s3_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = {
+	.cmd_rcgr = 0x184d8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap1_s4_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = {
+	.cmd_rcgr = 0x18608,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap1_s5_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s6_clk_src = {
+	.cmd_rcgr = 0x18738,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap1_s6_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s7_clk_src = {
+	.cmd_rcgr = 0x18868,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_qupv3_wrap1_s7_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = {
+	F(400000, P_BI_TCXO, 12, 1, 4),
+	F(9600000, P_BI_TCXO, 2, 0, 0),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(201500000, P_GPLL4_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
+	.cmd_rcgr = 0x1400c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_10,
+	.freq_tbl = ftbl_gcc_sdcc2_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_sdcc2_apps_clk_src",
+		.parent_names = gcc_parent_names_10,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = {
+	F(400000, P_BI_TCXO, 12, 1, 4),
+	F(9600000, P_BI_TCXO, 2, 0, 0),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(25000000, P_GPLL0_OUT_MAIN, 12, 1, 2),
+	F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc4_apps_clk_src = {
+	.cmd_rcgr = 0x1600c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_sdcc4_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_sdcc4_apps_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_tsif_ref_clk_src[] = {
+	F(105495, P_BI_TCXO, 2, 1, 91),
+	{ }
+};
+
+static struct clk_rcg2 gcc_tsif_ref_clk_src = {
+	.cmd_rcgr = 0x36010,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_6,
+	.freq_tbl = ftbl_gcc_tsif_ref_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_tsif_ref_clk_src",
+		.parent_names = gcc_parent_names_6,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_card_axi_clk_src[] = {
+	F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_card_axi_clk_src = {
+	.cmd_rcgr = 0x7501c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_ufs_card_axi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_ufs_card_axi_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_card_ice_core_clk_src[] = {
+	F(37500000, P_GPLL0_OUT_EVEN, 8, 0, 0),
+	F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_card_ice_core_clk_src = {
+	.cmd_rcgr = 0x7505c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_ufs_card_ice_core_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_ufs_card_phy_aux_clk_src = {
+	.cmd_rcgr = 0x75090,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_cpuss_rbcpr_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_ufs_card_phy_aux_clk_src",
+		.parent_names = gcc_parent_names_4,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_card_unipro_core_clk_src[] = {
+	F(37500000, P_GPLL0_OUT_EVEN, 8, 0, 0),
+	F(75000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_card_unipro_core_clk_src = {
+	.cmd_rcgr = 0x75074,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_ufs_card_unipro_core_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_ufs_card_unipro_core_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = {
+	F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = {
+	.cmd_rcgr = 0x7701c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_ufs_phy_axi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_ufs_phy_axi_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = {
+	.cmd_rcgr = 0x7705c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_ufs_phy_ice_core_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = {
+	.cmd_rcgr = 0x77090,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_ufs_phy_phy_aux_clk_src",
+		.parent_names = gcc_parent_names_4,
+		.num_parents = 2,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = {
+	.cmd_rcgr = 0x77074,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_ufs_card_unipro_core_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_ufs_phy_unipro_core_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = {
+	F(33333333, P_GPLL0_OUT_EVEN, 9, 0, 0),
+	F(66666667, P_GPLL0_OUT_EVEN, 4.5, 0, 0),
+	F(133333333, P_GPLL0_OUT_MAIN, 4.5, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_usb30_prim_master_clk_src = {
+	.cmd_rcgr = 0xf018,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_usb30_prim_master_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_usb30_prim_master_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb30_prim_mock_utmi_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(20000000, P_GPLL0_OUT_EVEN, 15, 0, 0),
+	F(40000000, P_GPLL0_OUT_EVEN, 7.5, 0, 0),
+	F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = {
+	.cmd_rcgr = 0xf030,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_usb30_prim_mock_utmi_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb30_sec_master_clk_src = {
+	.cmd_rcgr = 0x10018,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_usb30_prim_master_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_usb30_sec_master_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb30_sec_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x10030,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_usb30_sec_mock_utmi_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = {
+	.cmd_rcgr = 0xf05c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_cpuss_rbcpr_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_usb3_prim_phy_aux_clk_src",
+		.parent_names = gcc_parent_names_2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb3_sec_phy_aux_clk_src = {
+	.cmd_rcgr = 0x1005c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_cpuss_rbcpr_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_usb3_sec_phy_aux_clk_src",
+		.parent_names = gcc_parent_names_2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_vs_ctrl_clk_src = {
+	.cmd_rcgr = 0x7a030,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_cpuss_rbcpr_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_vs_ctrl_clk_src",
+		.parent_names = gcc_parent_names_3,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_vsensor_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0),
+	F(600000000, P_GPLL0_OUT_MAIN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_vsensor_clk_src = {
+	.cmd_rcgr = 0x7a018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_vsensor_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_vsensor_clk_src",
+		.parent_names = gcc_parent_names_8,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_aggre_noc_pcie_tbu_clk = {
+	.halt_reg = 0x90014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x90014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre_noc_pcie_tbu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre_ufs_card_axi_clk = {
+	.halt_reg = 0x82028,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x82028,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x82028,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre_ufs_card_axi_clk",
+			.parent_names = (const char *[]){
+				"gcc_ufs_card_axi_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre_ufs_phy_axi_clk = {
+	.halt_reg = 0x82024,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x82024,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x82024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre_ufs_phy_axi_clk",
+			.parent_names = (const char *[]){
+				"gcc_ufs_phy_axi_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre_usb3_prim_axi_clk = {
+	.halt_reg = 0x8201c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8201c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre_usb3_prim_axi_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb30_prim_master_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre_usb3_sec_axi_clk = {
+	.halt_reg = 0x82020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x82020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_aggre_usb3_sec_axi_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb30_sec_master_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_apc_vs_clk = {
+	.halt_reg = 0x7a050,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x7a050,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_apc_vs_clk",
+			.parent_names = (const char *[]){
+				"gcc_vsensor_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_boot_rom_ahb_clk = {
+	.halt_reg = 0x38004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x38004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(10),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_boot_rom_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camera_ahb_clk = {
+	.halt_reg = 0xb008,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0xb008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xb008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camera_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camera_axi_clk = {
+	.halt_reg = 0xb020,
+	.halt_check = BRANCH_VOTED,
+	.clkr = {
+		.enable_reg = 0xb020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camera_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camera_xo_clk = {
+	.halt_reg = 0xb02c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xb02c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camera_xo_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ce1_ahb_clk = {
+	.halt_reg = 0x4100c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x4100c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(3),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ce1_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ce1_axi_clk = {
+	.halt_reg = 0x41008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ce1_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ce1_clk = {
+	.halt_reg = 0x41004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(5),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ce1_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = {
+	.halt_reg = 0x502c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x502c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_cfg_noc_usb3_prim_axi_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb30_prim_master_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cfg_noc_usb3_sec_axi_clk = {
+	.halt_reg = 0x5030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5030,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_cfg_noc_usb3_sec_axi_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb30_sec_master_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cpuss_ahb_clk = {
+	.halt_reg = 0x48000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(21),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_cpuss_ahb_clk",
+			.parent_names = (const char *[]){
+				"gcc_cpuss_ahb_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT | CLK_IS_CRITICAL,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cpuss_rbcpr_clk = {
+	.halt_reg = 0x48008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x48008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_cpuss_rbcpr_clk",
+			.parent_names = (const char *[]){
+				"gcc_cpuss_rbcpr_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ddrss_gpu_axi_clk = {
+	.halt_reg = 0x44038,
+	.halt_check = BRANCH_VOTED,
+	.clkr = {
+		.enable_reg = 0x44038,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ddrss_gpu_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_disp_ahb_clk = {
+	.halt_reg = 0xb00c,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0xb00c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xb00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_disp_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_disp_axi_clk = {
+	.halt_reg = 0xb024,
+	.halt_check = BRANCH_VOTED,
+	.clkr = {
+		.enable_reg = 0xb024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_disp_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_disp_gpll0_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(18),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_disp_gpll0_clk_src",
+			.parent_names = (const char *[]){
+				"gpll0",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_disp_gpll0_div_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(19),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_disp_gpll0_div_clk_src",
+			.parent_names = (const char *[]){
+				"gpll0_out_even",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_disp_xo_clk = {
+	.halt_reg = 0xb030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xb030,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_disp_xo_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x64000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x64000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp1_clk",
+			.parent_names = (const char *[]){
+				"gcc_gp1_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x65000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x65000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp2_clk",
+			.parent_names = (const char *[]){
+				"gcc_gp2_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x66000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x66000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp3_clk",
+			.parent_names = (const char *[]){
+				"gcc_gp3_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_cfg_ahb_clk = {
+	.halt_reg = 0x71004,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x71004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x71004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gpll0_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(15),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_gpll0_clk_src",
+			.parent_names = (const char *[]){
+				"gpll0",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gpll0_div_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(16),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_gpll0_div_clk_src",
+			.parent_names = (const char *[]){
+				"gpll0_out_even",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_iref_clk = {
+	.halt_reg = 0x8c010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8c010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_iref_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_memnoc_gfx_clk = {
+	.halt_reg = 0x7100c,
+	.halt_check = BRANCH_VOTED,
+	.clkr = {
+		.enable_reg = 0x7100c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_memnoc_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_snoc_dvm_gfx_clk = {
+	.halt_reg = 0x71018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x71018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_snoc_dvm_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_vs_clk = {
+	.halt_reg = 0x7a04c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x7a04c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_vs_clk",
+			.parent_names = (const char *[]){
+				"gcc_vsensor_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_axis2_clk = {
+	.halt_reg = 0x8a008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8a008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_axis2_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_cfg_ahb_clk = {
+	.halt_reg = 0x8a000,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x8a000,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x8a000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_gpll0_div_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(17),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_gpll0_div_clk_src",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_mfab_axis_clk = {
+	.halt_reg = 0x8a004,
+	.halt_check = BRANCH_VOTED,
+	.hwcg_reg = 0x8a004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x8a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_mfab_axis_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_q6_memnoc_axi_clk = {
+	.halt_reg = 0x8a154,
+	.halt_check = BRANCH_VOTED,
+	.clkr = {
+		.enable_reg = 0x8a154,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_q6_memnoc_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_snoc_axi_clk = {
+	.halt_reg = 0x8a150,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8a150,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_snoc_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_vs_clk = {
+	.halt_reg = 0x7a048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x7a048,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_vs_clk",
+			.parent_names = (const char *[]){
+				"gcc_vsensor_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_aux_clk = {
+	.halt_reg = 0x6b01c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(3),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_aux_clk",
+			.parent_names = (const char *[]){
+				"gcc_pcie_0_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_cfg_ahb_clk = {
+	.halt_reg = 0x6b018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(2),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_clkref_clk = {
+	.halt_reg = 0x8c00c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8c00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_clkref_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_mstr_axi_clk = {
+	.halt_reg = 0x6b014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(1),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_mstr_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_pipe_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_pipe_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_slv_axi_clk = {
+	.halt_reg = 0x6b010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_slv_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = {
+	.halt_reg = 0x6b00c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(5),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_0_slv_q2a_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_aux_clk = {
+	.halt_reg = 0x8d01c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(29),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_aux_clk",
+			.parent_names = (const char *[]){
+				"gcc_pcie_1_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_cfg_ahb_clk = {
+	.halt_reg = 0x8d018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x8d018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(28),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_clkref_clk = {
+	.halt_reg = 0x8c02c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8c02c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_clkref_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_mstr_axi_clk = {
+	.halt_reg = 0x8d014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(27),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_mstr_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_pipe_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(30),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_pipe_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_slv_axi_clk = {
+	.halt_reg = 0x8d010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x8d010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(26),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_slv_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_slv_q2a_axi_clk = {
+	.halt_reg = 0x8d00c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(25),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_1_slv_q2a_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_phy_aux_clk = {
+	.halt_reg = 0x6f004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6f004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_phy_aux_clk",
+			.parent_names = (const char *[]){
+				"gcc_pcie_0_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_phy_refgen_clk = {
+	.halt_reg = 0x6f02c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6f02c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie_phy_refgen_clk",
+			.parent_names = (const char *[]){
+				"gcc_pcie_phy_refgen_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm2_clk = {
+	.halt_reg = 0x3300c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3300c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm2_clk",
+			.parent_names = (const char *[]){
+				"gcc_pdm2_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_ahb_clk = {
+	.halt_reg = 0x33004,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x33004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x33004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_xo4_clk = {
+	.halt_reg = 0x33008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x33008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm_xo4_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_prng_ahb_clk = {
+	.halt_reg = 0x34004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x34004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(13),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_prng_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_camera_ahb_clk = {
+	.halt_reg = 0xb014,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0xb014,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xb014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qmip_camera_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_disp_ahb_clk = {
+	.halt_reg = 0xb018,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0xb018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xb018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qmip_disp_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_video_ahb_clk = {
+	.halt_reg = 0xb010,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0xb010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xb010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qmip_video_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s0_clk = {
+	.halt_reg = 0x17030,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(10),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap0_s0_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap0_s0_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s1_clk = {
+	.halt_reg = 0x17160,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap0_s1_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap0_s1_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s2_clk = {
+	.halt_reg = 0x17290,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(12),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap0_s2_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap0_s2_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s3_clk = {
+	.halt_reg = 0x173c0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(13),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap0_s3_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap0_s3_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s4_clk = {
+	.halt_reg = 0x174f0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(14),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap0_s4_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap0_s4_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s5_clk = {
+	.halt_reg = 0x17620,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(15),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap0_s5_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap0_s5_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s6_clk = {
+	.halt_reg = 0x17750,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(16),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap0_s6_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap0_s6_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s7_clk = {
+	.halt_reg = 0x17880,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(17),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap0_s7_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap0_s7_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s0_clk = {
+	.halt_reg = 0x18014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(22),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap1_s0_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap1_s0_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s1_clk = {
+	.halt_reg = 0x18144,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(23),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap1_s1_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap1_s1_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s2_clk = {
+	.halt_reg = 0x18274,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(24),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap1_s2_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap1_s2_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s3_clk = {
+	.halt_reg = 0x183a4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(25),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap1_s3_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap1_s3_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s4_clk = {
+	.halt_reg = 0x184d4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(26),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap1_s4_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap1_s4_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s5_clk = {
+	.halt_reg = 0x18604,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(27),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap1_s5_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap1_s5_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s6_clk = {
+	.halt_reg = 0x18734,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(28),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap1_s6_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap1_s6_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s7_clk = {
+	.halt_reg = 0x18864,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(29),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap1_s7_clk",
+			.parent_names = (const char *[]){
+				"gcc_qupv3_wrap1_s7_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_0_m_ahb_clk = {
+	.halt_reg = 0x17004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(6),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap_0_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_0_s_ahb_clk = {
+	.halt_reg = 0x17008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x17008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(7),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap_0_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_1_m_ahb_clk = {
+	.halt_reg = 0x1800c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(20),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap_1_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = {
+	.halt_reg = 0x18010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x18010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x5200c,
+		.enable_mask = BIT(21),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qupv3_wrap_1_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_ahb_clk = {
+	.halt_reg = 0x14008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x14008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_apps_clk = {
+	.halt_reg = 0x14004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x14004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_apps_clk",
+			.parent_names = (const char *[]){
+				"gcc_sdcc2_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_ahb_clk = {
+	.halt_reg = 0x16008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x16008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc4_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_apps_clk = {
+	.halt_reg = 0x16004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x16004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc4_apps_clk",
+			.parent_names = (const char *[]){
+				"gcc_sdcc4_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sys_noc_cpuss_ahb_clk = {
+	.halt_reg = 0x414c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sys_noc_cpuss_ahb_clk",
+			.parent_names = (const char *[]){
+				"gcc_cpuss_ahb_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT | CLK_IS_CRITICAL,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tsif_ahb_clk = {
+	.halt_reg = 0x36004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x36004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_tsif_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tsif_inactivity_timers_clk = {
+	.halt_reg = 0x3600c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_tsif_inactivity_timers_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tsif_ref_clk = {
+	.halt_reg = 0x36008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x36008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_tsif_ref_clk",
+			.parent_names = (const char *[]){
+				"gcc_tsif_ref_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_card_ahb_clk = {
+	.halt_reg = 0x75010,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x75010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x75010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_card_axi_clk = {
+	.halt_reg = 0x7500c,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x7500c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7500c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_axi_clk",
+			.parent_names = (const char *[]){
+				"gcc_ufs_card_axi_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_card_clkref_clk = {
+	.halt_reg = 0x8c004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8c004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_clkref_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_card_ice_core_clk = {
+	.halt_reg = 0x75058,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x75058,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x75058,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_ice_core_clk",
+			.parent_names = (const char *[]){
+				"gcc_ufs_card_ice_core_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_card_phy_aux_clk = {
+	.halt_reg = 0x7508c,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x7508c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7508c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_phy_aux_clk",
+			.parent_names = (const char *[]){
+				"gcc_ufs_card_phy_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_card_rx_symbol_0_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x75018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_rx_symbol_0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_card_rx_symbol_1_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x750a8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_rx_symbol_1_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_card_tx_symbol_0_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x75014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_tx_symbol_0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_card_unipro_core_clk = {
+	.halt_reg = 0x75054,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x75054,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x75054,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_unipro_core_clk",
+			.parent_names = (const char *[]){
+				"gcc_ufs_card_unipro_core_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_mem_clkref_clk = {
+	.halt_reg = 0x8c000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8c000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_mem_clkref_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_ahb_clk = {
+	.halt_reg = 0x77010,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x77010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x77010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_axi_clk = {
+	.halt_reg = 0x7700c,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x7700c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7700c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_axi_clk",
+			.parent_names = (const char *[]){
+				"gcc_ufs_phy_axi_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_ice_core_clk = {
+	.halt_reg = 0x77058,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x77058,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x77058,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_ice_core_clk",
+			.parent_names = (const char *[]){
+				"gcc_ufs_phy_ice_core_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_phy_aux_clk = {
+	.halt_reg = 0x7708c,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x7708c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7708c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_phy_aux_clk",
+			.parent_names = (const char *[]){
+				"gcc_ufs_phy_phy_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x77018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_rx_symbol_0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x770a8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_rx_symbol_1_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x77014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_tx_symbol_0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_unipro_core_clk = {
+	.halt_reg = 0x77054,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x77054,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x77054,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_unipro_core_clk",
+			.parent_names = (const char *[]){
+				"gcc_ufs_phy_unipro_core_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_master_clk = {
+	.halt_reg = 0xf00c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xf00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_prim_master_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb30_prim_master_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_mock_utmi_clk = {
+	.halt_reg = 0xf014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xf014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_prim_mock_utmi_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb30_prim_mock_utmi_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_sleep_clk = {
+	.halt_reg = 0xf010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xf010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_prim_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_sec_master_clk = {
+	.halt_reg = 0x1000c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1000c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_sec_master_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb30_sec_master_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_sec_mock_utmi_clk = {
+	.halt_reg = 0x10014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_sec_mock_utmi_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb30_sec_mock_utmi_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_sec_sleep_clk = {
+	.halt_reg = 0x10010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb30_sec_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_clkref_clk = {
+	.halt_reg = 0x8c008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8c008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_prim_clkref_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_aux_clk = {
+	.halt_reg = 0xf04c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xf04c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_prim_phy_aux_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb3_prim_phy_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = {
+	.halt_reg = 0xf050,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xf050,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_prim_phy_com_aux_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb3_prim_phy_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_pipe_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0xf054,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_prim_phy_pipe_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_sec_clkref_clk = {
+	.halt_reg = 0x8c028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8c028,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_sec_clkref_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_sec_phy_aux_clk = {
+	.halt_reg = 0x1004c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1004c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_sec_phy_aux_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb3_sec_phy_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_sec_phy_com_aux_clk = {
+	.halt_reg = 0x10050,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10050,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_sec_phy_com_aux_clk",
+			.parent_names = (const char *[]){
+				"gcc_usb3_sec_phy_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_sec_phy_pipe_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x10054,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb3_sec_phy_pipe_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_phy_cfg_ahb2phy_clk = {
+	.halt_reg = 0x6a004,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x6a004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x6a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_phy_cfg_ahb2phy_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_vdda_vs_clk = {
+	.halt_reg = 0x7a00c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x7a00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_vdda_vs_clk",
+			.parent_names = (const char *[]){
+				"gcc_vsensor_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_vddcx_vs_clk = {
+	.halt_reg = 0x7a004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x7a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_vddcx_vs_clk",
+			.parent_names = (const char *[]){
+				"gcc_vsensor_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_vddmx_vs_clk = {
+	.halt_reg = 0x7a008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x7a008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_vddmx_vs_clk",
+			.parent_names = (const char *[]){
+				"gcc_vsensor_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_ahb_clk = {
+	.halt_reg = 0xb004,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0xb004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xb004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_video_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_axi_clk = {
+	.halt_reg = 0xb01c,
+	.halt_check = BRANCH_VOTED,
+	.clkr = {
+		.enable_reg = 0xb01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_video_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_xo_clk = {
+	.halt_reg = 0xb028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xb028,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_video_xo_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_vs_ctrl_ahb_clk = {
+	.halt_reg = 0x7a014,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x7a014,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7a014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_vs_ctrl_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_vs_ctrl_clk = {
+	.halt_reg = 0x7a010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x7a010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_vs_ctrl_clk",
+			.parent_names = (const char *[]){
+				"gcc_vs_ctrl_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc pcie_0_gdsc = {
+	.gdscr = 0x6b004,
+	.pd = {
+		.name = "pcie_0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR,
+};
+
+static struct gdsc pcie_1_gdsc = {
+	.gdscr = 0x8d004,
+	.pd = {
+		.name = "pcie_1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR,
+};
+
+static struct gdsc ufs_card_gdsc = {
+	.gdscr = 0x75004,
+	.pd = {
+		.name = "ufs_card_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR,
+};
+
+static struct gdsc ufs_phy_gdsc = {
+	.gdscr = 0x77004,
+	.pd = {
+		.name = "ufs_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR,
+};
+
+static struct gdsc usb30_prim_gdsc = {
+	.gdscr = 0xf004,
+	.pd = {
+		.name = "usb30_prim_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR,
+};
+
+static struct gdsc usb30_sec_gdsc = {
+	.gdscr = 0x10004,
+	.pd = {
+		.name = "usb30_sec_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR,
+};
+
+static struct gdsc hlos1_vote_aggre_noc_mmu_audio_tbu_gdsc = {
+	.gdscr = 0x7d030,
+	.pd = {
+		.name = "hlos1_vote_aggre_noc_mmu_audio_tbu_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc hlos1_vote_aggre_noc_mmu_pcie_tbu_gdsc = {
+	.gdscr = 0x7d03c,
+	.pd = {
+		.name = "hlos1_vote_aggre_noc_mmu_pcie_tbu_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc hlos1_vote_aggre_noc_mmu_tbu1_gdsc = {
+	.gdscr = 0x7d034,
+	.pd = {
+		.name = "hlos1_vote_aggre_noc_mmu_tbu1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc hlos1_vote_aggre_noc_mmu_tbu2_gdsc = {
+	.gdscr = 0x7d038,
+	.pd = {
+		.name = "hlos1_vote_aggre_noc_mmu_tbu2_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc = {
+	.gdscr = 0x7d040,
+	.pd = {
+		.name = "hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc = {
+	.gdscr = 0x7d048,
+	.pd = {
+		.name = "hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc hlos1_vote_mmnoc_mmu_tbu_sf_gdsc = {
+	.gdscr = 0x7d044,
+	.pd = {
+		.name = "hlos1_vote_mmnoc_mmu_tbu_sf_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct clk_regmap *gcc_sdm845_clocks[] = {
+	[GCC_AGGRE_NOC_PCIE_TBU_CLK] = &gcc_aggre_noc_pcie_tbu_clk.clkr,
+	[GCC_AGGRE_UFS_CARD_AXI_CLK] = &gcc_aggre_ufs_card_axi_clk.clkr,
+	[GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr,
+	[GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr,
+	[GCC_AGGRE_USB3_SEC_AXI_CLK] = &gcc_aggre_usb3_sec_axi_clk.clkr,
+	[GCC_APC_VS_CLK] = &gcc_apc_vs_clk.clkr,
+	[GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr,
+	[GCC_CAMERA_AHB_CLK] = &gcc_camera_ahb_clk.clkr,
+	[GCC_CAMERA_AXI_CLK] = &gcc_camera_axi_clk.clkr,
+	[GCC_CAMERA_XO_CLK] = &gcc_camera_xo_clk.clkr,
+	[GCC_CE1_AHB_CLK] = &gcc_ce1_ahb_clk.clkr,
+	[GCC_CE1_AXI_CLK] = &gcc_ce1_axi_clk.clkr,
+	[GCC_CE1_CLK] = &gcc_ce1_clk.clkr,
+	[GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr,
+	[GCC_CFG_NOC_USB3_SEC_AXI_CLK] = &gcc_cfg_noc_usb3_sec_axi_clk.clkr,
+	[GCC_CPUSS_AHB_CLK] = &gcc_cpuss_ahb_clk.clkr,
+	[GCC_CPUSS_AHB_CLK_SRC] = &gcc_cpuss_ahb_clk_src.clkr,
+	[GCC_CPUSS_RBCPR_CLK] = &gcc_cpuss_rbcpr_clk.clkr,
+	[GCC_CPUSS_RBCPR_CLK_SRC] = &gcc_cpuss_rbcpr_clk_src.clkr,
+	[GCC_DDRSS_GPU_AXI_CLK] = &gcc_ddrss_gpu_axi_clk.clkr,
+	[GCC_DISP_AHB_CLK] = &gcc_disp_ahb_clk.clkr,
+	[GCC_DISP_AXI_CLK] = &gcc_disp_axi_clk.clkr,
+	[GCC_DISP_GPLL0_CLK_SRC] = &gcc_disp_gpll0_clk_src.clkr,
+	[GCC_DISP_GPLL0_DIV_CLK_SRC] = &gcc_disp_gpll0_div_clk_src.clkr,
+	[GCC_DISP_XO_CLK] = &gcc_disp_xo_clk.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
+	[GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr,
+	[GCC_GPU_CFG_AHB_CLK] = &gcc_gpu_cfg_ahb_clk.clkr,
+	[GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr,
+	[GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr,
+	[GCC_GPU_IREF_CLK] = &gcc_gpu_iref_clk.clkr,
+	[GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr,
+	[GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr,
+	[GCC_GPU_VS_CLK] = &gcc_gpu_vs_clk.clkr,
+	[GCC_MSS_AXIS2_CLK] = &gcc_mss_axis2_clk.clkr,
+	[GCC_MSS_CFG_AHB_CLK] = &gcc_mss_cfg_ahb_clk.clkr,
+	[GCC_MSS_GPLL0_DIV_CLK_SRC] = &gcc_mss_gpll0_div_clk_src.clkr,
+	[GCC_MSS_MFAB_AXIS_CLK] = &gcc_mss_mfab_axis_clk.clkr,
+	[GCC_MSS_Q6_MEMNOC_AXI_CLK] = &gcc_mss_q6_memnoc_axi_clk.clkr,
+	[GCC_MSS_SNOC_AXI_CLK] = &gcc_mss_snoc_axi_clk.clkr,
+	[GCC_MSS_VS_CLK] = &gcc_mss_vs_clk.clkr,
+	[GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr,
+	[GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr,
+	[GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr,
+	[GCC_PCIE_0_CLKREF_CLK] = &gcc_pcie_0_clkref_clk.clkr,
+	[GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr,
+	[GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr,
+	[GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr,
+	[GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr,
+	[GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr,
+	[GCC_PCIE_1_AUX_CLK_SRC] = &gcc_pcie_1_aux_clk_src.clkr,
+	[GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr,
+	[GCC_PCIE_1_CLKREF_CLK] = &gcc_pcie_1_clkref_clk.clkr,
+	[GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr,
+	[GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr,
+	[GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr,
+	[GCC_PCIE_1_SLV_Q2A_AXI_CLK] = &gcc_pcie_1_slv_q2a_axi_clk.clkr,
+	[GCC_PCIE_PHY_AUX_CLK] = &gcc_pcie_phy_aux_clk.clkr,
+	[GCC_PCIE_PHY_REFGEN_CLK] = &gcc_pcie_phy_refgen_clk.clkr,
+	[GCC_PCIE_PHY_REFGEN_CLK_SRC] = &gcc_pcie_phy_refgen_clk_src.clkr,
+	[GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr,
+	[GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr,
+	[GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr,
+	[GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr,
+	[GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr,
+	[GCC_QMIP_CAMERA_AHB_CLK] = &gcc_qmip_camera_ahb_clk.clkr,
+	[GCC_QMIP_DISP_AHB_CLK] = &gcc_qmip_disp_ahb_clk.clkr,
+	[GCC_QMIP_VIDEO_AHB_CLK] = &gcc_qmip_video_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr,
+	[GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr,
+	[GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr,
+	[GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr,
+	[GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr,
+	[GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S5_CLK] = &gcc_qupv3_wrap0_s5_clk.clkr,
+	[GCC_QUPV3_WRAP0_S5_CLK_SRC] = &gcc_qupv3_wrap0_s5_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S6_CLK] = &gcc_qupv3_wrap0_s6_clk.clkr,
+	[GCC_QUPV3_WRAP0_S6_CLK_SRC] = &gcc_qupv3_wrap0_s6_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S7_CLK] = &gcc_qupv3_wrap0_s7_clk.clkr,
+	[GCC_QUPV3_WRAP0_S7_CLK_SRC] = &gcc_qupv3_wrap0_s7_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr,
+	[GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr,
+	[GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr,
+	[GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr,
+	[GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr,
+	[GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr,
+	[GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S6_CLK] = &gcc_qupv3_wrap1_s6_clk.clkr,
+	[GCC_QUPV3_WRAP1_S6_CLK_SRC] = &gcc_qupv3_wrap1_s6_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S7_CLK] = &gcc_qupv3_wrap1_s7_clk.clkr,
+	[GCC_QUPV3_WRAP1_S7_CLK_SRC] = &gcc_qupv3_wrap1_s7_clk_src.clkr,
+	[GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_1_M_AHB_CLK] = &gcc_qupv3_wrap_1_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr,
+	[GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr,
+	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
+	[GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr,
+	[GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr,
+	[GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr,
+	[GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr,
+	[GCC_SYS_NOC_CPUSS_AHB_CLK] = &gcc_sys_noc_cpuss_ahb_clk.clkr,
+	[GCC_TSIF_AHB_CLK] = &gcc_tsif_ahb_clk.clkr,
+	[GCC_TSIF_INACTIVITY_TIMERS_CLK] =
+					&gcc_tsif_inactivity_timers_clk.clkr,
+	[GCC_TSIF_REF_CLK] = &gcc_tsif_ref_clk.clkr,
+	[GCC_TSIF_REF_CLK_SRC] = &gcc_tsif_ref_clk_src.clkr,
+	[GCC_UFS_CARD_AHB_CLK] = &gcc_ufs_card_ahb_clk.clkr,
+	[GCC_UFS_CARD_AXI_CLK] = &gcc_ufs_card_axi_clk.clkr,
+	[GCC_UFS_CARD_AXI_CLK_SRC] = &gcc_ufs_card_axi_clk_src.clkr,
+	[GCC_UFS_CARD_CLKREF_CLK] = &gcc_ufs_card_clkref_clk.clkr,
+	[GCC_UFS_CARD_ICE_CORE_CLK] = &gcc_ufs_card_ice_core_clk.clkr,
+	[GCC_UFS_CARD_ICE_CORE_CLK_SRC] = &gcc_ufs_card_ice_core_clk_src.clkr,
+	[GCC_UFS_CARD_PHY_AUX_CLK] = &gcc_ufs_card_phy_aux_clk.clkr,
+	[GCC_UFS_CARD_PHY_AUX_CLK_SRC] = &gcc_ufs_card_phy_aux_clk_src.clkr,
+	[GCC_UFS_CARD_RX_SYMBOL_0_CLK] = &gcc_ufs_card_rx_symbol_0_clk.clkr,
+	[GCC_UFS_CARD_RX_SYMBOL_1_CLK] = &gcc_ufs_card_rx_symbol_1_clk.clkr,
+	[GCC_UFS_CARD_TX_SYMBOL_0_CLK] = &gcc_ufs_card_tx_symbol_0_clk.clkr,
+	[GCC_UFS_CARD_UNIPRO_CORE_CLK] = &gcc_ufs_card_unipro_core_clk.clkr,
+	[GCC_UFS_CARD_UNIPRO_CORE_CLK_SRC] =
+					&gcc_ufs_card_unipro_core_clk_src.clkr,
+	[GCC_UFS_MEM_CLKREF_CLK] = &gcc_ufs_mem_clkref_clk.clkr,
+	[GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr,
+	[GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr,
+	[GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr,
+	[GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr,
+	[GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr,
+	[GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr,
+	[GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr,
+	[GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr,
+	[GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr,
+	[GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] =
+					&gcc_ufs_phy_unipro_core_clk_src.clkr,
+	[GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr,
+	[GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] =
+					&gcc_usb30_prim_mock_utmi_clk_src.clkr,
+	[GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr,
+	[GCC_USB30_SEC_MASTER_CLK] = &gcc_usb30_sec_master_clk.clkr,
+	[GCC_USB30_SEC_MASTER_CLK_SRC] = &gcc_usb30_sec_master_clk_src.clkr,
+	[GCC_USB30_SEC_MOCK_UTMI_CLK] = &gcc_usb30_sec_mock_utmi_clk.clkr,
+	[GCC_USB30_SEC_MOCK_UTMI_CLK_SRC] =
+					&gcc_usb30_sec_mock_utmi_clk_src.clkr,
+	[GCC_USB30_SEC_SLEEP_CLK] = &gcc_usb30_sec_sleep_clk.clkr,
+	[GCC_USB3_PRIM_CLKREF_CLK] = &gcc_usb3_prim_clkref_clk.clkr,
+	[GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr,
+	[GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr,
+	[GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr,
+	[GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr,
+	[GCC_USB3_SEC_CLKREF_CLK] = &gcc_usb3_sec_clkref_clk.clkr,
+	[GCC_USB3_SEC_PHY_AUX_CLK] = &gcc_usb3_sec_phy_aux_clk.clkr,
+	[GCC_USB3_SEC_PHY_AUX_CLK_SRC] = &gcc_usb3_sec_phy_aux_clk_src.clkr,
+	[GCC_USB3_SEC_PHY_COM_AUX_CLK] = &gcc_usb3_sec_phy_com_aux_clk.clkr,
+	[GCC_USB3_SEC_PHY_PIPE_CLK] = &gcc_usb3_sec_phy_pipe_clk.clkr,
+	[GCC_USB_PHY_CFG_AHB2PHY_CLK] = &gcc_usb_phy_cfg_ahb2phy_clk.clkr,
+	[GCC_VDDA_VS_CLK] = &gcc_vdda_vs_clk.clkr,
+	[GCC_VDDCX_VS_CLK] = &gcc_vddcx_vs_clk.clkr,
+	[GCC_VDDMX_VS_CLK] = &gcc_vddmx_vs_clk.clkr,
+	[GCC_VIDEO_AHB_CLK] = &gcc_video_ahb_clk.clkr,
+	[GCC_VIDEO_AXI_CLK] = &gcc_video_axi_clk.clkr,
+	[GCC_VIDEO_XO_CLK] = &gcc_video_xo_clk.clkr,
+	[GCC_VS_CTRL_AHB_CLK] = &gcc_vs_ctrl_ahb_clk.clkr,
+	[GCC_VS_CTRL_CLK] = &gcc_vs_ctrl_clk.clkr,
+	[GCC_VS_CTRL_CLK_SRC] = &gcc_vs_ctrl_clk_src.clkr,
+	[GCC_VSENSOR_CLK_SRC] = &gcc_vsensor_clk_src.clkr,
+	[GPLL0] = &gpll0.clkr,
+	[GPLL0_OUT_EVEN] = &gpll0_out_even.clkr,
+	[GPLL4] = &gpll4.clkr,
+};
+
+static const struct qcom_reset_map gcc_sdm845_resets[] = {
+	[GCC_MMSS_BCR] = { 0xb000 },
+	[GCC_PCIE_0_BCR] = { 0x6b000 },
+	[GCC_PCIE_1_BCR] = { 0x8d000 },
+	[GCC_PCIE_PHY_BCR] = { 0x6f000 },
+	[GCC_PDM_BCR] = { 0x33000 },
+	[GCC_PRNG_BCR] = { 0x34000 },
+	[GCC_QUPV3_WRAPPER_0_BCR] = { 0x17000 },
+	[GCC_QUPV3_WRAPPER_1_BCR] = { 0x18000 },
+	[GCC_QUSB2PHY_PRIM_BCR] = { 0x12000 },
+	[GCC_QUSB2PHY_SEC_BCR] = { 0x12004 },
+	[GCC_SDCC2_BCR] = { 0x14000 },
+	[GCC_SDCC4_BCR] = { 0x16000 },
+	[GCC_TSIF_BCR] = { 0x36000 },
+	[GCC_UFS_CARD_BCR] = { 0x75000 },
+	[GCC_UFS_PHY_BCR] = { 0x77000 },
+	[GCC_USB30_PRIM_BCR] = { 0xf000 },
+	[GCC_USB30_SEC_BCR] = { 0x10000 },
+	[GCC_USB3_PHY_PRIM_BCR] = { 0x50000 },
+	[GCC_USB3PHY_PHY_PRIM_BCR] = { 0x50004 },
+	[GCC_USB3_DP_PHY_PRIM_BCR] = { 0x50008 },
+	[GCC_USB3_PHY_SEC_BCR] = { 0x5000c },
+	[GCC_USB3PHY_PHY_SEC_BCR] = { 0x50010 },
+	[GCC_USB3_DP_PHY_SEC_BCR] = { 0x50014 },
+	[GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x6a000 },
+	[GCC_PCIE_0_PHY_BCR] = { 0x6c01c },
+	[GCC_PCIE_1_PHY_BCR] = { 0x8e01c },
+};
+
+static struct gdsc *gcc_sdm845_gdscs[] = {
+	[PCIE_0_GDSC] = &pcie_0_gdsc,
+	[PCIE_1_GDSC] = &pcie_1_gdsc,
+	[UFS_CARD_GDSC] = &ufs_card_gdsc,
+	[UFS_PHY_GDSC] = &ufs_phy_gdsc,
+	[USB30_PRIM_GDSC] = &usb30_prim_gdsc,
+	[USB30_SEC_GDSC] = &usb30_sec_gdsc,
+	[HLOS1_VOTE_AGGRE_NOC_MMU_AUDIO_TBU_GDSC] =
+			&hlos1_vote_aggre_noc_mmu_audio_tbu_gdsc,
+	[HLOS1_VOTE_AGGRE_NOC_MMU_PCIE_TBU_GDSC] =
+			&hlos1_vote_aggre_noc_mmu_pcie_tbu_gdsc,
+	[HLOS1_VOTE_AGGRE_NOC_MMU_TBU1_GDSC] =
+			&hlos1_vote_aggre_noc_mmu_tbu1_gdsc,
+	[HLOS1_VOTE_AGGRE_NOC_MMU_TBU2_GDSC] =
+			&hlos1_vote_aggre_noc_mmu_tbu2_gdsc,
+	[HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC] =
+			&hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc,
+	[HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC] =
+			&hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc,
+	[HLOS1_VOTE_MMNOC_MMU_TBU_SF_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_sf_gdsc,
+};
+
+static const struct regmap_config gcc_sdm845_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0x182090,
+	.fast_io	= true,
+};
+
+static const struct qcom_cc_desc gcc_sdm845_desc = {
+	.config = &gcc_sdm845_regmap_config,
+	.clks = gcc_sdm845_clocks,
+	.num_clks = ARRAY_SIZE(gcc_sdm845_clocks),
+	.resets = gcc_sdm845_resets,
+	.num_resets = ARRAY_SIZE(gcc_sdm845_resets),
+	.gdscs = gcc_sdm845_gdscs,
+	.num_gdscs = ARRAY_SIZE(gcc_sdm845_gdscs),
+};
+
+static const struct of_device_id gcc_sdm845_match_table[] = {
+	{ .compatible = "qcom,gcc-sdm845" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_sdm845_match_table);
+
+static int gcc_sdm845_probe(struct platform_device *pdev)
+{
+	struct regmap *regmap;
+
+	regmap = qcom_cc_map(pdev, &gcc_sdm845_desc);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	/* Disable the GPLL0 active input to MMSS and GPU via MISC registers */
+	regmap_update_bits(regmap, 0x09ffc, 0x3, 0x3);
+	regmap_update_bits(regmap, 0x71028, 0x3, 0x3);
+
+	/* Enable CPUSS clocks */
+	regmap_update_bits(regmap, 0x48190, BIT(0), 0x1);
+	regmap_update_bits(regmap, 0x52004, BIT(22), 0x1);
+
+	return qcom_cc_really_probe(pdev, &gcc_sdm845_desc, regmap);
+}
+
+static struct platform_driver gcc_sdm845_driver = {
+	.probe		= gcc_sdm845_probe,
+	.driver		= {
+		.name	= "gcc-sdm845",
+		.of_match_table = gcc_sdm845_match_table,
+	},
+};
+
+static int __init gcc_sdm845_init(void)
+{
+	return platform_driver_register(&gcc_sdm845_driver);
+}
+subsys_initcall(gcc_sdm845_init);
+
+static void __exit gcc_sdm845_exit(void)
+{
+	platform_driver_unregister(&gcc_sdm845_driver);
+}
+module_exit(gcc_sdm845_exit);
+
+MODULE_DESCRIPTION("QTI GCC SDM845 Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:gcc-sdm845");
diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c
index a4f3580587b7..4696e241db89 100644
--- a/drivers/clk/qcom/gdsc.c
+++ b/drivers/clk/qcom/gdsc.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2015, 2017-2018, The Linux Foundation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 and
@@ -31,6 +31,12 @@
 #define HW_CONTROL_MASK		BIT(1)
 #define SW_COLLAPSE_MASK	BIT(0)
 #define GMEM_CLAMP_IO_MASK	BIT(0)
+#define GMEM_RESET_MASK		BIT(4)
+
+/* CFG_GDSCR */
+#define GDSC_POWER_UP_COMPLETE		BIT(16)
+#define GDSC_POWER_DOWN_COMPLETE	BIT(15)
+#define CFG_GDSCR_OFFSET		0x4
 
 /* Wait 2^n CXO cycles between all states. Here, n=2 (4 cycles). */
 #define EN_REST_WAIT_VAL	(0x2 << 20)
@@ -40,20 +46,50 @@
 #define RETAIN_MEM		BIT(14)
 #define RETAIN_PERIPH		BIT(13)
 
-#define TIMEOUT_US		100
+#define TIMEOUT_US		500
 
 #define domain_to_gdsc(domain) container_of(domain, struct gdsc, pd)
 
-static int gdsc_is_enabled(struct gdsc *sc, unsigned int reg)
+enum gdsc_status {
+	GDSC_OFF,
+	GDSC_ON
+};
+
+/* Returns 1 if GDSC status is status, 0 if not, and < 0 on error */
+static int gdsc_check_status(struct gdsc *sc, enum gdsc_status status)
 {
+	unsigned int reg;
 	u32 val;
 	int ret;
 
+	if (sc->flags & POLL_CFG_GDSCR)
+		reg = sc->gdscr + CFG_GDSCR_OFFSET;
+	else if (sc->gds_hw_ctrl)
+		reg = sc->gds_hw_ctrl;
+	else
+		reg = sc->gdscr;
+
 	ret = regmap_read(sc->regmap, reg, &val);
 	if (ret)
 		return ret;
 
-	return !!(val & PWR_ON_MASK);
+	if (sc->flags & POLL_CFG_GDSCR) {
+		switch (status) {
+		case GDSC_ON:
+			return !!(val & GDSC_POWER_UP_COMPLETE);
+		case GDSC_OFF:
+			return !!(val & GDSC_POWER_DOWN_COMPLETE);
+		}
+	}
+
+	switch (status) {
+	case GDSC_ON:
+		return !!(val & PWR_ON_MASK);
+	case GDSC_OFF:
+		return !(val & PWR_ON_MASK);
+	}
+
+	return -EINVAL;
 }
 
 static int gdsc_hwctrl(struct gdsc *sc, bool en)
@@ -63,34 +99,33 @@ static int gdsc_hwctrl(struct gdsc *sc, bool en)
 	return regmap_update_bits(sc->regmap, sc->gdscr, HW_CONTROL_MASK, val);
 }
 
-static int gdsc_poll_status(struct gdsc *sc, unsigned int reg, bool en)
+static int gdsc_poll_status(struct gdsc *sc, enum gdsc_status status)
 {
 	ktime_t start;
 
 	start = ktime_get();
 	do {
-		if (gdsc_is_enabled(sc, reg) == en)
+		if (gdsc_check_status(sc, status))
 			return 0;
 	} while (ktime_us_delta(ktime_get(), start) < TIMEOUT_US);
 
-	if (gdsc_is_enabled(sc, reg) == en)
+	if (gdsc_check_status(sc, status))
 		return 0;
 
 	return -ETIMEDOUT;
 }
 
-static int gdsc_toggle_logic(struct gdsc *sc, bool en)
+static int gdsc_toggle_logic(struct gdsc *sc, enum gdsc_status status)
 {
 	int ret;
-	u32 val = en ? 0 : SW_COLLAPSE_MASK;
-	unsigned int status_reg = sc->gdscr;
+	u32 val = (status == GDSC_ON) ? 0 : SW_COLLAPSE_MASK;
 
 	ret = regmap_update_bits(sc->regmap, sc->gdscr, SW_COLLAPSE_MASK, val);
 	if (ret)
 		return ret;
 
 	/* If disabling votable gdscs, don't poll on status */
-	if ((sc->flags & VOTABLE) && !en) {
+	if ((sc->flags & VOTABLE) && status == GDSC_OFF) {
 		/*
 		 * Add a short delay here to ensure that an enable
 		 * right after it was disabled does not put it in an
@@ -101,7 +136,6 @@ static int gdsc_toggle_logic(struct gdsc *sc, bool en)
 	}
 
 	if (sc->gds_hw_ctrl) {
-		status_reg = sc->gds_hw_ctrl;
 		/*
 		 * The gds hw controller asserts/de-asserts the status bit soon
 		 * after it receives a power on/off request from a master.
@@ -115,7 +149,7 @@ static int gdsc_toggle_logic(struct gdsc *sc, bool en)
 		udelay(1);
 	}
 
-	return gdsc_poll_status(sc, status_reg, en);
+	return gdsc_poll_status(sc, status);
 }
 
 static inline int gdsc_deassert_reset(struct gdsc *sc)
@@ -166,6 +200,14 @@ static inline void gdsc_assert_clamp_io(struct gdsc *sc)
 			   GMEM_CLAMP_IO_MASK, 1);
 }
 
+static inline void gdsc_assert_reset_aon(struct gdsc *sc)
+{
+	regmap_update_bits(sc->regmap, sc->clamp_io_ctrl,
+			   GMEM_RESET_MASK, 1);
+	udelay(1);
+	regmap_update_bits(sc->regmap, sc->clamp_io_ctrl,
+			   GMEM_RESET_MASK, 0);
+}
 static int gdsc_enable(struct generic_pm_domain *domain)
 {
 	struct gdsc *sc = domain_to_gdsc(domain);
@@ -174,10 +216,19 @@ static int gdsc_enable(struct generic_pm_domain *domain)
 	if (sc->pwrsts == PWRSTS_ON)
 		return gdsc_deassert_reset(sc);
 
-	if (sc->flags & CLAMP_IO)
+	if (sc->flags & SW_RESET) {
+		gdsc_assert_reset(sc);
+		udelay(1);
+		gdsc_deassert_reset(sc);
+	}
+
+	if (sc->flags & CLAMP_IO) {
+		if (sc->flags & AON_RESET)
+			gdsc_assert_reset_aon(sc);
 		gdsc_deassert_clamp_io(sc);
+	}
 
-	ret = gdsc_toggle_logic(sc, true);
+	ret = gdsc_toggle_logic(sc, GDSC_ON);
 	if (ret)
 		return ret;
 
@@ -222,8 +273,6 @@ static int gdsc_disable(struct generic_pm_domain *domain)
 
 	/* Turn off HW trigger mode if supported */
 	if (sc->flags & HW_CTRL) {
-		unsigned int reg;
-
 		ret = gdsc_hwctrl(sc, false);
 		if (ret < 0)
 			return ret;
@@ -235,8 +284,7 @@ static int gdsc_disable(struct generic_pm_domain *domain)
 		 */
 		udelay(1);
 
-		reg = sc->gds_hw_ctrl ? sc->gds_hw_ctrl : sc->gdscr;
-		ret = gdsc_poll_status(sc, reg, true);
+		ret = gdsc_poll_status(sc, GDSC_ON);
 		if (ret)
 			return ret;
 	}
@@ -244,7 +292,7 @@ static int gdsc_disable(struct generic_pm_domain *domain)
 	if (sc->pwrsts & PWRSTS_OFF)
 		gdsc_clear_mem_on(sc);
 
-	ret = gdsc_toggle_logic(sc, false);
+	ret = gdsc_toggle_logic(sc, GDSC_OFF);
 	if (ret)
 		return ret;
 
@@ -258,7 +306,6 @@ static int gdsc_init(struct gdsc *sc)
 {
 	u32 mask, val;
 	int on, ret;
-	unsigned int reg;
 
 	/*
 	 * Disable HW trigger: collapse/restore occur based on registers writes.
@@ -274,13 +321,12 @@ static int gdsc_init(struct gdsc *sc)
 
 	/* Force gdsc ON if only ON state is supported */
 	if (sc->pwrsts == PWRSTS_ON) {
-		ret = gdsc_toggle_logic(sc, true);
+		ret = gdsc_toggle_logic(sc, GDSC_ON);
 		if (ret)
 			return ret;
 	}
 
-	reg = sc->gds_hw_ctrl ? sc->gds_hw_ctrl : sc->gdscr;
-	on = gdsc_is_enabled(sc, reg);
+	on = gdsc_check_status(sc, GDSC_ON);
 	if (on < 0)
 		return on;
 
diff --git a/drivers/clk/qcom/gdsc.h b/drivers/clk/qcom/gdsc.h
index 39648348e5ec..b0cbb87dd02b 100644
--- a/drivers/clk/qcom/gdsc.h
+++ b/drivers/clk/qcom/gdsc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2015, 2017-2018, The Linux Foundation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 and
@@ -53,6 +53,9 @@ struct gdsc {
 #define VOTABLE		BIT(0)
 #define CLAMP_IO	BIT(1)
 #define HW_CTRL		BIT(2)
+#define SW_RESET	BIT(3)
+#define AON_RESET	BIT(4)
+#define POLL_CFG_GDSCR	BIT(5)
 	struct reset_controller_dev	*rcdev;
 	unsigned int			*resets;
 	unsigned int			reset_count;
diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig
index ef76c861ec84..f9ba71311727 100644
--- a/drivers/clk/renesas/Kconfig
+++ b/drivers/clk/renesas/Kconfig
@@ -7,6 +7,7 @@ config CLK_RENESAS
 	select CLK_R8A7740 if ARCH_R8A7740
 	select CLK_R8A7743 if ARCH_R8A7743
 	select CLK_R8A7745 if ARCH_R8A7745
+	select CLK_R8A77470 if ARCH_R8A77470
 	select CLK_R8A7778 if ARCH_R8A7778
 	select CLK_R8A7779 if ARCH_R8A7779
 	select CLK_R8A7790 if ARCH_R8A7790
@@ -18,6 +19,7 @@ config CLK_RENESAS
 	select CLK_R8A77965 if ARCH_R8A77965
 	select CLK_R8A77970 if ARCH_R8A77970
 	select CLK_R8A77980 if ARCH_R8A77980
+	select CLK_R8A77990 if ARCH_R8A77990
 	select CLK_R8A77995 if ARCH_R8A77995
 	select CLK_SH73A0 if ARCH_SH73A0
 
@@ -60,6 +62,10 @@ config CLK_R8A7745
 	bool "RZ/G1E clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN2_CPG
 
+config CLK_R8A77470
+	bool "RZ/G1C clock support" if COMPILE_TEST
+	select CLK_RCAR_GEN2_CPG
+
 config CLK_R8A7778
 	bool "R-Car M1A clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSTP
@@ -111,6 +117,10 @@ config CLK_R8A77980
 	bool "R-Car V3H clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN3_CPG
 
+config CLK_R8A77990
+	bool "R-Car E3 clock support" if COMPILE_TEST
+	select CLK_RCAR_GEN3_CPG
+
 config CLK_R8A77995
 	bool "R-Car D3 clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN3_CPG
diff --git a/drivers/clk/renesas/Makefile b/drivers/clk/renesas/Makefile
index 6c0f19636e3e..fe5bac9215e5 100644
--- a/drivers/clk/renesas/Makefile
+++ b/drivers/clk/renesas/Makefile
@@ -6,6 +6,7 @@ obj-$(CONFIG_CLK_R8A73A4)		+= clk-r8a73a4.o
 obj-$(CONFIG_CLK_R8A7740)		+= clk-r8a7740.o
 obj-$(CONFIG_CLK_R8A7743)		+= r8a7743-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A7745)		+= r8a7745-cpg-mssr.o
+obj-$(CONFIG_CLK_R8A77470)		+= r8a77470-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A7778)		+= clk-r8a7778.o
 obj-$(CONFIG_CLK_R8A7779)		+= clk-r8a7779.o
 obj-$(CONFIG_CLK_R8A7790)		+= r8a7790-cpg-mssr.o
@@ -17,6 +18,7 @@ obj-$(CONFIG_CLK_R8A7796)		+= r8a7796-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A77965)		+= r8a77965-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A77970)		+= r8a77970-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A77980)		+= r8a77980-cpg-mssr.o
+obj-$(CONFIG_CLK_R8A77990)		+= r8a77990-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A77995)		+= r8a77995-cpg-mssr.o
 obj-$(CONFIG_CLK_SH73A0)		+= clk-sh73a0.o
 
diff --git a/drivers/clk/renesas/r8a7743-cpg-mssr.c b/drivers/clk/renesas/r8a7743-cpg-mssr.c
index d3c8b1e2969f..011c170ec3f9 100644
--- a/drivers/clk/renesas/r8a7743-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7743-cpg-mssr.c
@@ -52,7 +52,6 @@ static const struct cpg_core_clk r8a7743_core_clks[] __initconst = {
 
 	/* Core Clock Outputs */
 	DEF_BASE("z",    R8A7743_CLK_Z,    CLK_TYPE_GEN2_Z,	CLK_PLL0),
-	DEF_BASE("lb",   R8A7743_CLK_LB,   CLK_TYPE_GEN2_LB,	CLK_PLL1),
 	DEF_BASE("sdh",  R8A7743_CLK_SDH,  CLK_TYPE_GEN2_SDH,	CLK_PLL1),
 	DEF_BASE("sd0",  R8A7743_CLK_SD0,  CLK_TYPE_GEN2_SD0,	CLK_PLL1),
 	DEF_BASE("qspi", R8A7743_CLK_QSPI, CLK_TYPE_GEN2_QSPI,	CLK_PLL1_DIV2),
@@ -63,6 +62,7 @@ static const struct cpg_core_clk r8a7743_core_clks[] __initconst = {
 	DEF_FIXED("zs",    R8A7743_CLK_ZS,	CLK_PLL1,	    6, 1),
 	DEF_FIXED("hp",    R8A7743_CLK_HP,	CLK_PLL1,	   12, 1),
 	DEF_FIXED("b",     R8A7743_CLK_B,	CLK_PLL1,	   12, 1),
+	DEF_FIXED("lb",    R8A7743_CLK_LB,	CLK_PLL1,	   24, 1),
 	DEF_FIXED("p",     R8A7743_CLK_P,	CLK_PLL1,	   24, 1),
 	DEF_FIXED("cl",    R8A7743_CLK_CL,	CLK_PLL1,	   48, 1),
 	DEF_FIXED("m2",    R8A7743_CLK_M2,	CLK_PLL1,	    8, 1),
diff --git a/drivers/clk/renesas/r8a7745-cpg-mssr.c b/drivers/clk/renesas/r8a7745-cpg-mssr.c
index 87f5a3619e4f..4b0a9243b748 100644
--- a/drivers/clk/renesas/r8a7745-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7745-cpg-mssr.c
@@ -51,7 +51,6 @@ static const struct cpg_core_clk r8a7745_core_clks[] __initconst = {
 	DEF_FIXED(".pll1_div2",	CLK_PLL1_DIV2, CLK_PLL1, 2, 1),
 
 	/* Core Clock Outputs */
-	DEF_BASE("lb",   R8A7745_CLK_LB,   CLK_TYPE_GEN2_LB,	CLK_PLL1),
 	DEF_BASE("sdh",  R8A7745_CLK_SDH,  CLK_TYPE_GEN2_SDH,	CLK_PLL1),
 	DEF_BASE("sd0",  R8A7745_CLK_SD0,  CLK_TYPE_GEN2_SD0,	CLK_PLL1),
 	DEF_BASE("qspi", R8A7745_CLK_QSPI, CLK_TYPE_GEN2_QSPI,	CLK_PLL1_DIV2),
@@ -63,6 +62,7 @@ static const struct cpg_core_clk r8a7745_core_clks[] __initconst = {
 	DEF_FIXED("zs",    R8A7745_CLK_ZS,	CLK_PLL1,	    6, 1),
 	DEF_FIXED("hp",    R8A7745_CLK_HP,	CLK_PLL1,	   12, 1),
 	DEF_FIXED("b",     R8A7745_CLK_B,	CLK_PLL1,	   12, 1),
+	DEF_FIXED("lb",    R8A7745_CLK_LB,	CLK_PLL1,	   24, 1),
 	DEF_FIXED("p",     R8A7745_CLK_P,	CLK_PLL1,	   24, 1),
 	DEF_FIXED("cl",    R8A7745_CLK_CL,	CLK_PLL1,	   48, 1),
 	DEF_FIXED("cp",    R8A7745_CLK_CP,	CLK_PLL1,	   48, 1),
diff --git a/drivers/clk/renesas/r8a77470-cpg-mssr.c b/drivers/clk/renesas/r8a77470-cpg-mssr.c
new file mode 100644
index 000000000000..ab0fb10b6bf0
--- /dev/null
+++ b/drivers/clk/renesas/r8a77470-cpg-mssr.c
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * r8a77470 Clock Pulse Generator / Module Standby and Software Reset
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ */
+
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/soc/renesas/rcar-rst.h>
+
+#include <dt-bindings/clock/r8a77470-cpg-mssr.h>
+
+#include "renesas-cpg-mssr.h"
+#include "rcar-gen2-cpg.h"
+
+enum clk_ids {
+	/* Core Clock Outputs exported to DT */
+	LAST_DT_CORE_CLK = R8A77470_CLK_OSC,
+
+	/* External Input Clocks */
+	CLK_EXTAL,
+	CLK_USB_EXTAL,
+
+	/* Internal Core Clocks */
+	CLK_MAIN,
+	CLK_PLL0,
+	CLK_PLL1,
+	CLK_PLL3,
+	CLK_PLL1_DIV2,
+
+	/* Module Clocks */
+	MOD_CLK_BASE
+};
+
+static const struct cpg_core_clk r8a77470_core_clks[] __initconst = {
+	/* External Clock Inputs */
+	DEF_INPUT("extal",	CLK_EXTAL),
+	DEF_INPUT("usb_extal",	CLK_USB_EXTAL),
+
+	/* Internal Core Clocks */
+	DEF_BASE(".main",	CLK_MAIN, CLK_TYPE_GEN2_MAIN, CLK_EXTAL),
+	DEF_BASE(".pll0",	CLK_PLL0, CLK_TYPE_GEN2_PLL0, CLK_MAIN),
+	DEF_BASE(".pll1",	CLK_PLL1, CLK_TYPE_GEN2_PLL1, CLK_MAIN),
+	DEF_BASE(".pll3",	CLK_PLL3, CLK_TYPE_GEN2_PLL3, CLK_MAIN),
+
+	DEF_FIXED(".pll1_div2",	CLK_PLL1_DIV2, CLK_PLL1, 2, 1),
+
+	/* Core Clock Outputs */
+	DEF_BASE("sdh",  R8A77470_CLK_SDH,  CLK_TYPE_GEN2_SDH,	CLK_PLL1),
+	DEF_BASE("sd0",  R8A77470_CLK_SD0,  CLK_TYPE_GEN2_SD0,	CLK_PLL1),
+	DEF_BASE("sd1",  R8A77470_CLK_SD1,  CLK_TYPE_GEN2_SD1,	CLK_PLL1),
+	DEF_BASE("qspi", R8A77470_CLK_QSPI, CLK_TYPE_GEN2_QSPI,	CLK_PLL1_DIV2),
+	DEF_BASE("rcan", R8A77470_CLK_RCAN, CLK_TYPE_GEN2_RCAN,	CLK_USB_EXTAL),
+
+	DEF_FIXED("z2",    R8A77470_CLK_Z2,	CLK_PLL0,	    1, 1),
+	DEF_FIXED("zx",    R8A77470_CLK_ZX,	CLK_PLL1,	    3, 1),
+	DEF_FIXED("zs",    R8A77470_CLK_ZS,	CLK_PLL1,	    6, 1),
+	DEF_FIXED("hp",    R8A77470_CLK_HP,	CLK_PLL1,	   12, 1),
+	DEF_FIXED("b",     R8A77470_CLK_B,	CLK_PLL1,	   12, 1),
+	DEF_FIXED("lb",	   R8A77470_CLK_LB,     CLK_PLL1,	   24, 1),
+	DEF_FIXED("p",     R8A77470_CLK_P,	CLK_PLL1,	   24, 1),
+	DEF_FIXED("cl",    R8A77470_CLK_CL,	CLK_PLL1,	   48, 1),
+	DEF_FIXED("cp",    R8A77470_CLK_CP,	CLK_PLL1,	   48, 1),
+	DEF_FIXED("m2",    R8A77470_CLK_M2,	CLK_PLL1,	    8, 1),
+	DEF_FIXED("zb3",   R8A77470_CLK_ZB3,	CLK_PLL3,	    4, 1),
+	DEF_FIXED("mp",    R8A77470_CLK_MP,	CLK_PLL1_DIV2,	   15, 1),
+	DEF_FIXED("cpex",  R8A77470_CLK_CPEX,	CLK_EXTAL,	    2, 1),
+	DEF_FIXED("r",     R8A77470_CLK_R,	CLK_PLL1,	49152, 1),
+	DEF_FIXED("osc",   R8A77470_CLK_OSC,	CLK_PLL1,	12288, 1),
+
+	DEF_DIV6P1("sd2",  R8A77470_CLK_SD2,	CLK_PLL1_DIV2,	0x078),
+};
+
+static const struct mssr_mod_clk r8a77470_mod_clks[] __initconst = {
+	DEF_MOD("msiof0",		   0,	R8A77470_CLK_MP),
+	DEF_MOD("vcp0",			 101,	R8A77470_CLK_ZS),
+	DEF_MOD("vpc0",			 103,	R8A77470_CLK_ZS),
+	DEF_MOD("tmu1",			 111,	R8A77470_CLK_P),
+	DEF_MOD("3dg",			 112,	R8A77470_CLK_ZS),
+	DEF_MOD("2d-dmac",		 115,	R8A77470_CLK_ZS),
+	DEF_MOD("fdp1-0",		 119,	R8A77470_CLK_ZS),
+	DEF_MOD("tmu3",			 121,	R8A77470_CLK_P),
+	DEF_MOD("tmu2",			 122,	R8A77470_CLK_P),
+	DEF_MOD("cmt0",			 124,	R8A77470_CLK_R),
+	DEF_MOD("vsp1du0",		 128,	R8A77470_CLK_ZS),
+	DEF_MOD("vsp1-sy",		 131,	R8A77470_CLK_ZS),
+	DEF_MOD("msiof2",		 205,	R8A77470_CLK_MP),
+	DEF_MOD("msiof1",		 208,	R8A77470_CLK_MP),
+	DEF_MOD("sys-dmac1",		 218,	R8A77470_CLK_ZS),
+	DEF_MOD("sys-dmac0",		 219,	R8A77470_CLK_ZS),
+	DEF_MOD("sdhi2",		 312,	R8A77470_CLK_SD2),
+	DEF_MOD("sdhi1",		 313,	R8A77470_CLK_SD1),
+	DEF_MOD("sdhi0",		 314,	R8A77470_CLK_SD0),
+	DEF_MOD("usbhs-dmac0-ch1",	 326,	R8A77470_CLK_HP),
+	DEF_MOD("usbhs-dmac1-ch1",	 327,	R8A77470_CLK_HP),
+	DEF_MOD("cmt1",			 329,	R8A77470_CLK_R),
+	DEF_MOD("usbhs-dmac0-ch0",	 330,	R8A77470_CLK_HP),
+	DEF_MOD("usbhs-dmac1-ch0",	 331,	R8A77470_CLK_HP),
+	DEF_MOD("rwdt",			 402,	R8A77470_CLK_R),
+	DEF_MOD("irqc",			 407,	R8A77470_CLK_CP),
+	DEF_MOD("intc-sys",		 408,	R8A77470_CLK_ZS),
+	DEF_MOD("audio-dmac0",		 502,	R8A77470_CLK_HP),
+	DEF_MOD("pwm",			 523,	R8A77470_CLK_P),
+	DEF_MOD("usb-ehci-0",		 703,	R8A77470_CLK_MP),
+	DEF_MOD("usbhs-0",		 704,	R8A77470_CLK_HP),
+	DEF_MOD("usb-ehci-1",		 705,	R8A77470_CLK_MP),
+	DEF_MOD("usbhs-1",		 706,	R8A77470_CLK_HP),
+	DEF_MOD("hscif2",		 713,	R8A77470_CLK_ZS),
+	DEF_MOD("scif5",		 714,	R8A77470_CLK_P),
+	DEF_MOD("scif4",		 715,	R8A77470_CLK_P),
+	DEF_MOD("hscif1",		 716,	R8A77470_CLK_ZS),
+	DEF_MOD("hscif0",		 717,	R8A77470_CLK_ZS),
+	DEF_MOD("scif3",		 718,	R8A77470_CLK_P),
+	DEF_MOD("scif2",		 719,	R8A77470_CLK_P),
+	DEF_MOD("scif1",		 720,	R8A77470_CLK_P),
+	DEF_MOD("scif0",		 721,	R8A77470_CLK_P),
+	DEF_MOD("du1",			 723,	R8A77470_CLK_ZX),
+	DEF_MOD("du0",			 724,	R8A77470_CLK_ZX),
+	DEF_MOD("ipmmu-sgx",		 800,	R8A77470_CLK_ZX),
+	DEF_MOD("etheravb",		 812,	R8A77470_CLK_HP),
+	DEF_MOD("ether",		 813,	R8A77470_CLK_P),
+	DEF_MOD("gpio5",		 907,	R8A77470_CLK_CP),
+	DEF_MOD("gpio4",		 908,	R8A77470_CLK_CP),
+	DEF_MOD("gpio3",		 909,	R8A77470_CLK_CP),
+	DEF_MOD("gpio2",		 910,	R8A77470_CLK_CP),
+	DEF_MOD("gpio1",		 911,	R8A77470_CLK_CP),
+	DEF_MOD("gpio0",		 912,	R8A77470_CLK_CP),
+	DEF_MOD("can1",			 915,	R8A77470_CLK_P),
+	DEF_MOD("can0",			 916,	R8A77470_CLK_P),
+	DEF_MOD("qspi_mod-1",		 917,	R8A77470_CLK_QSPI),
+	DEF_MOD("qspi_mod-0",		 918,	R8A77470_CLK_QSPI),
+	DEF_MOD("i2c4",			 927,	R8A77470_CLK_HP),
+	DEF_MOD("i2c3",			 928,	R8A77470_CLK_HP),
+	DEF_MOD("i2c2",			 929,	R8A77470_CLK_HP),
+	DEF_MOD("i2c1",			 930,	R8A77470_CLK_HP),
+	DEF_MOD("i2c0",			 931,	R8A77470_CLK_HP),
+	DEF_MOD("ssi-all",		1005,	R8A77470_CLK_P),
+	DEF_MOD("ssi9",			1006,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi8",			1007,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi7",			1008,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi6",			1009,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi5",			1010,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi4",			1011,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi3",			1012,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi2",			1013,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi1",			1014,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi0",			1015,	MOD_CLK_ID(1005)),
+	DEF_MOD("scu-all",		1017,	R8A77470_CLK_P),
+	DEF_MOD("scu-dvc1",		1018,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-dvc0",		1019,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu1-mix1",	1020,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu0-mix0",	1021,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src6",		1025,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src5",		1026,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src4",		1027,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src3",		1028,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src2",		1029,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src1",		1030,	MOD_CLK_ID(1017)),
+};
+
+static const unsigned int r8a77470_crit_mod_clks[] __initconst = {
+	MOD_CLK_ID(402),	/* RWDT */
+	MOD_CLK_ID(408),	/* INTC-SYS (GIC) */
+};
+
+/*
+ * CPG Clock Data
+ */
+
+/*
+ *    MD	EXTAL		PLL0	PLL1	PLL3
+ * 14 13	(MHz)		*1	*2
+ *---------------------------------------------------
+ * 0  0		20		x80	x78	x50
+ * 0  1		26		x60	x60	x56
+ * 1  0		Prohibitted setting
+ * 1  1		30		x52	x52	x50
+ *
+ * *1 :	Table 7.4 indicates VCO output (PLL0 = VCO)
+ * *2 :	Table 7.4 indicates VCO output (PLL1 = VCO)
+ */
+#define CPG_PLL_CONFIG_INDEX(md)	((((md) & BIT(14)) >> 13) | \
+					 (((md) & BIT(13)) >> 13))
+
+static const struct rcar_gen2_cpg_pll_config cpg_pll_configs[4] __initconst = {
+	/* EXTAL div	PLL1 mult x2	PLL3 mult */
+	{ 1,		156,		50,	},
+	{ 1,		120,		56,	},
+	{ /* Invalid*/				},
+	{ 1,		104,		50,	},
+};
+
+static int __init r8a77470_cpg_mssr_init(struct device *dev)
+{
+	const struct rcar_gen2_cpg_pll_config *cpg_pll_config;
+	u32 cpg_mode;
+	int error;
+
+	error = rcar_rst_read_mode_pins(&cpg_mode);
+	if (error)
+		return error;
+
+	cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)];
+
+	return rcar_gen2_cpg_init(cpg_pll_config, 2, cpg_mode);
+}
+
+const struct cpg_mssr_info r8a77470_cpg_mssr_info __initconst = {
+	/* Core Clocks */
+	.core_clks = r8a77470_core_clks,
+	.num_core_clks = ARRAY_SIZE(r8a77470_core_clks),
+	.last_dt_core_clk = LAST_DT_CORE_CLK,
+	.num_total_core_clks = MOD_CLK_BASE,
+
+	/* Module Clocks */
+	.mod_clks = r8a77470_mod_clks,
+	.num_mod_clks = ARRAY_SIZE(r8a77470_mod_clks),
+	.num_hw_mod_clks = 12 * 32,
+
+	/* Critical Module Clocks */
+	.crit_mod_clks = r8a77470_crit_mod_clks,
+	.num_crit_mod_clks = ARRAY_SIZE(r8a77470_crit_mod_clks),
+
+	/* Callbacks */
+	.init = r8a77470_cpg_mssr_init,
+	.cpg_clk_register = rcar_gen2_cpg_clk_register,
+};
diff --git a/drivers/clk/renesas/r8a7791-cpg-mssr.c b/drivers/clk/renesas/r8a7791-cpg-mssr.c
index 820b220b09cc..1b91f03b7598 100644
--- a/drivers/clk/renesas/r8a7791-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7791-cpg-mssr.c
@@ -57,7 +57,6 @@ static struct cpg_core_clk r8a7791_core_clks[] __initdata = {
 
 	/* Core Clock Outputs */
 	DEF_BASE("z",    R8A7791_CLK_Z,    CLK_TYPE_GEN2_Z,    CLK_PLL0),
-	DEF_BASE("lb",   R8A7791_CLK_LB,   CLK_TYPE_GEN2_LB,   CLK_PLL1),
 	DEF_BASE("adsp", R8A7791_CLK_ADSP, CLK_TYPE_GEN2_ADSP, CLK_PLL1),
 	DEF_BASE("sdh",  R8A7791_CLK_SDH,  CLK_TYPE_GEN2_SDH,  CLK_PLL1),
 	DEF_BASE("sd0",  R8A7791_CLK_SD0,  CLK_TYPE_GEN2_SD0,  CLK_PLL1),
@@ -70,6 +69,7 @@ static struct cpg_core_clk r8a7791_core_clks[] __initdata = {
 	DEF_FIXED("hp",     R8A7791_CLK_HP,    CLK_PLL1,         12, 1),
 	DEF_FIXED("i",      R8A7791_CLK_I,     CLK_PLL1,          2, 1),
 	DEF_FIXED("b",      R8A7791_CLK_B,     CLK_PLL1,         12, 1),
+	DEF_FIXED("lb",     R8A7791_CLK_LB,    CLK_PLL1,         24, 1),
 	DEF_FIXED("p",      R8A7791_CLK_P,     CLK_PLL1,         24, 1),
 	DEF_FIXED("cl",     R8A7791_CLK_CL,    CLK_PLL1,         48, 1),
 	DEF_FIXED("m2",     R8A7791_CLK_M2,    CLK_PLL1,          8, 1),
diff --git a/drivers/clk/renesas/r8a7792-cpg-mssr.c b/drivers/clk/renesas/r8a7792-cpg-mssr.c
index 609a54080496..493e07859f5f 100644
--- a/drivers/clk/renesas/r8a7792-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7792-cpg-mssr.c
@@ -53,7 +53,6 @@ static const struct cpg_core_clk r8a7792_core_clks[] __initconst = {
 	DEF_FIXED(".pll1_div2", CLK_PLL1_DIV2, CLK_PLL1, 2, 1),
 
 	/* Core Clock Outputs */
-	DEF_BASE("lb",   R8A7792_CLK_LB,   CLK_TYPE_GEN2_LB,   CLK_PLL1),
 	DEF_BASE("qspi", R8A7792_CLK_QSPI, CLK_TYPE_GEN2_QSPI, CLK_PLL1_DIV2),
 
 	DEF_FIXED("z",      R8A7792_CLK_Z,     CLK_PLL0,          1, 1),
@@ -63,6 +62,7 @@ static const struct cpg_core_clk r8a7792_core_clks[] __initconst = {
 	DEF_FIXED("hp",     R8A7792_CLK_HP,    CLK_PLL1,         12, 1),
 	DEF_FIXED("i",      R8A7792_CLK_I,     CLK_PLL1,          3, 1),
 	DEF_FIXED("b",      R8A7792_CLK_B,     CLK_PLL1,         12, 1),
+	DEF_FIXED("lb",     R8A7792_CLK_LB,    CLK_PLL1,         24, 1),
 	DEF_FIXED("p",      R8A7792_CLK_P,     CLK_PLL1,         24, 1),
 	DEF_FIXED("cl",     R8A7792_CLK_CL,    CLK_PLL1,         48, 1),
 	DEF_FIXED("m2",     R8A7792_CLK_M2,    CLK_PLL1,          8, 1),
diff --git a/drivers/clk/renesas/r8a7794-cpg-mssr.c b/drivers/clk/renesas/r8a7794-cpg-mssr.c
index 2a40bbeaeeaf..088f4b79fdfc 100644
--- a/drivers/clk/renesas/r8a7794-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7794-cpg-mssr.c
@@ -55,7 +55,6 @@ static const struct cpg_core_clk r8a7794_core_clks[] __initconst = {
 	DEF_FIXED(".pll1_div2", CLK_PLL1_DIV2, CLK_PLL1, 2, 1),
 
 	/* Core Clock Outputs */
-	DEF_BASE("lb",   R8A7794_CLK_LB,   CLK_TYPE_GEN2_LB,   CLK_PLL1),
 	DEF_BASE("adsp", R8A7794_CLK_ADSP, CLK_TYPE_GEN2_ADSP, CLK_PLL1),
 	DEF_BASE("sdh",  R8A7794_CLK_SDH,  CLK_TYPE_GEN2_SDH,  CLK_PLL1),
 	DEF_BASE("sd0",  R8A7794_CLK_SD0,  CLK_TYPE_GEN2_SD0,  CLK_PLL1),
@@ -69,6 +68,7 @@ static const struct cpg_core_clk r8a7794_core_clks[] __initconst = {
 	DEF_FIXED("hp",     R8A7794_CLK_HP,    CLK_PLL1,         12, 1),
 	DEF_FIXED("i",      R8A7794_CLK_I,     CLK_PLL1,          2, 1),
 	DEF_FIXED("b",      R8A7794_CLK_B,     CLK_PLL1,         12, 1),
+	DEF_FIXED("lb",     R8A7794_CLK_LB,    CLK_PLL1,         24, 1),
 	DEF_FIXED("p",      R8A7794_CLK_P,     CLK_PLL1,         24, 1),
 	DEF_FIXED("cl",     R8A7794_CLK_CL,    CLK_PLL1,         48, 1),
 	DEF_FIXED("cp",     R8A7794_CLK_CP,    CLK_PLL1,         48, 1),
diff --git a/drivers/clk/renesas/r8a77965-cpg-mssr.c b/drivers/clk/renesas/r8a77965-cpg-mssr.c
index b1acfb60351c..8fae5e9c4a77 100644
--- a/drivers/clk/renesas/r8a77965-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a77965-cpg-mssr.c
@@ -116,6 +116,10 @@ static const struct mssr_mod_clk r8a77965_mod_clks[] __initconst = {
 	DEF_MOD("scif3",		204,	R8A77965_CLK_S3D4),
 	DEF_MOD("scif1",		206,	R8A77965_CLK_S3D4),
 	DEF_MOD("scif0",		207,	R8A77965_CLK_S3D4),
+	DEF_MOD("msiof3",		208,	R8A77965_CLK_MSO),
+	DEF_MOD("msiof2",		209,	R8A77965_CLK_MSO),
+	DEF_MOD("msiof1",		210,	R8A77965_CLK_MSO),
+	DEF_MOD("msiof0",		211,	R8A77965_CLK_MSO),
 	DEF_MOD("sys-dmac2",		217,	R8A77965_CLK_S0D3),
 	DEF_MOD("sys-dmac1",		218,	R8A77965_CLK_S0D3),
 	DEF_MOD("sys-dmac0",		219,	R8A77965_CLK_S0D3),
diff --git a/drivers/clk/renesas/r8a77980-cpg-mssr.c b/drivers/clk/renesas/r8a77980-cpg-mssr.c
index 7aaae73a321a..d7ebd9ec0059 100644
--- a/drivers/clk/renesas/r8a77980-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a77980-cpg-mssr.c
@@ -116,7 +116,7 @@ static const struct mssr_mod_clk r8a77980_mod_clks[] __initconst = {
 	DEF_MOD("sys-dmac1",		 218,	R8A77980_CLK_S0D3),
 	DEF_MOD("tpu0",			 304,	R8A77980_CLK_S3D4),
 	DEF_MOD("sdif",			 314,	R8A77980_CLK_SD0),
-	DEF_MOD("pciec0",		 319,	R8A77980_CLK_S3D1),
+	DEF_MOD("pciec0",		 319,	R8A77980_CLK_S2D2),
 	DEF_MOD("intc-ex",		 407,	R8A77980_CLK_CP),
 	DEF_MOD("intc-ap",		 408,	R8A77980_CLK_S0D3),
 	DEF_MOD("hscif3",		 517,	R8A77980_CLK_S3D1),
diff --git a/drivers/clk/renesas/r8a77990-cpg-mssr.c b/drivers/clk/renesas/r8a77990-cpg-mssr.c
new file mode 100644
index 000000000000..9e14f1486fbb
--- /dev/null
+++ b/drivers/clk/renesas/r8a77990-cpg-mssr.c
@@ -0,0 +1,289 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * r8a77990 Clock Pulse Generator / Module Standby and Software Reset
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ *
+ * Based on r8a7795-cpg-mssr.c
+ *
+ * Copyright (C) 2015 Glider bvba
+ * Copyright (C) 2015 Renesas Electronics Corp.
+ */
+
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/soc/renesas/rcar-rst.h>
+
+#include <dt-bindings/clock/r8a77990-cpg-mssr.h>
+
+#include "renesas-cpg-mssr.h"
+#include "rcar-gen3-cpg.h"
+
+enum clk_ids {
+	/* Core Clock Outputs exported to DT */
+	LAST_DT_CORE_CLK = R8A77990_CLK_CPEX,
+
+	/* External Input Clocks */
+	CLK_EXTAL,
+
+	/* Internal Core Clocks */
+	CLK_MAIN,
+	CLK_PLL0,
+	CLK_PLL1,
+	CLK_PLL3,
+	CLK_PLL0D4,
+	CLK_PLL0D6,
+	CLK_PLL0D8,
+	CLK_PLL0D20,
+	CLK_PLL0D24,
+	CLK_PLL1D2,
+	CLK_PE,
+	CLK_S0,
+	CLK_S1,
+	CLK_S2,
+	CLK_S3,
+	CLK_SDSRC,
+
+	/* Module Clocks */
+	MOD_CLK_BASE
+};
+
+static const struct cpg_core_clk r8a77990_core_clks[] __initconst = {
+	/* External Clock Inputs */
+	DEF_INPUT("extal",     CLK_EXTAL),
+
+	/* Internal Core Clocks */
+	DEF_BASE(".main",      CLK_MAIN, CLK_TYPE_GEN3_MAIN,       CLK_EXTAL),
+	DEF_BASE(".pll1",      CLK_PLL1, CLK_TYPE_GEN3_PLL1,       CLK_MAIN),
+	DEF_BASE(".pll3",      CLK_PLL3, CLK_TYPE_GEN3_PLL3,       CLK_MAIN),
+
+	DEF_FIXED(".pll0",     CLK_PLL0,           CLK_MAIN,	   1, 100),
+	DEF_FIXED(".pll0d4",   CLK_PLL0D4,         CLK_PLL0,       4, 1),
+	DEF_FIXED(".pll0d6",   CLK_PLL0D6,         CLK_PLL0,       6, 1),
+	DEF_FIXED(".pll0d8",   CLK_PLL0D8,         CLK_PLL0,       8, 1),
+	DEF_FIXED(".pll0d20",  CLK_PLL0D20,        CLK_PLL0,      20, 1),
+	DEF_FIXED(".pll0d24",  CLK_PLL0D24,        CLK_PLL0,      24, 1),
+	DEF_FIXED(".pll1d2",   CLK_PLL1D2,         CLK_PLL1,       2, 1),
+	DEF_FIXED(".pe",       CLK_PE,             CLK_PLL0D20,    1, 1),
+	DEF_FIXED(".s0",       CLK_S0,             CLK_PLL1,       2, 1),
+	DEF_FIXED(".s1",       CLK_S1,             CLK_PLL1,       3, 1),
+	DEF_FIXED(".s2",       CLK_S2,             CLK_PLL1,       4, 1),
+	DEF_FIXED(".s3",       CLK_S3,             CLK_PLL1,       6, 1),
+	DEF_FIXED(".sdsrc",    CLK_SDSRC,          CLK_PLL1,       2, 1),
+
+	/* Core Clock Outputs */
+	DEF_FIXED("za2",       R8A77990_CLK_ZA2,   CLK_PLL0D24,    1, 1),
+	DEF_FIXED("za8",       R8A77990_CLK_ZA8,   CLK_PLL0D8,     1, 1),
+	DEF_FIXED("ztr",       R8A77990_CLK_ZTR,   CLK_PLL1,       6, 1),
+	DEF_FIXED("zt",        R8A77990_CLK_ZT,    CLK_PLL1,       4, 1),
+	DEF_FIXED("zx",        R8A77990_CLK_ZX,    CLK_PLL1,       3, 1),
+	DEF_FIXED("s0d1",      R8A77990_CLK_S0D1,  CLK_S0,         1, 1),
+	DEF_FIXED("s0d3",      R8A77990_CLK_S0D3,  CLK_S0,         3, 1),
+	DEF_FIXED("s0d6",      R8A77990_CLK_S0D6,  CLK_S0,         6, 1),
+	DEF_FIXED("s0d12",     R8A77990_CLK_S0D12, CLK_S0,        12, 1),
+	DEF_FIXED("s0d24",     R8A77990_CLK_S0D24, CLK_S0,        24, 1),
+	DEF_FIXED("s1d1",      R8A77990_CLK_S1D1,  CLK_S1,         1, 1),
+	DEF_FIXED("s1d2",      R8A77990_CLK_S1D2,  CLK_S1,         2, 1),
+	DEF_FIXED("s1d4",      R8A77990_CLK_S1D4,  CLK_S1,         4, 1),
+	DEF_FIXED("s2d1",      R8A77990_CLK_S2D1,  CLK_S2,         1, 1),
+	DEF_FIXED("s2d2",      R8A77990_CLK_S2D2,  CLK_S2,         2, 1),
+	DEF_FIXED("s2d4",      R8A77990_CLK_S2D4,  CLK_S2,         4, 1),
+	DEF_FIXED("s3d1",      R8A77990_CLK_S3D1,  CLK_S3,         1, 1),
+	DEF_FIXED("s3d2",      R8A77990_CLK_S3D2,  CLK_S3,         2, 1),
+	DEF_FIXED("s3d4",      R8A77990_CLK_S3D4,  CLK_S3,         4, 1),
+
+	DEF_GEN3_SD("sd0",     R8A77990_CLK_SD0,   CLK_SDSRC,	  0x0074),
+	DEF_GEN3_SD("sd1",     R8A77990_CLK_SD1,   CLK_SDSRC,	  0x0078),
+	DEF_GEN3_SD("sd3",     R8A77990_CLK_SD3,   CLK_SDSRC,	  0x026c),
+
+	DEF_FIXED("cl",        R8A77990_CLK_CL,    CLK_PLL1,      48, 1),
+	DEF_FIXED("cp",        R8A77990_CLK_CP,    CLK_EXTAL,      2, 1),
+	DEF_FIXED("cpex",      R8A77990_CLK_CPEX,  CLK_EXTAL,      4, 1),
+	DEF_FIXED("osc",       R8A77990_CLK_OSC,   CLK_EXTAL,    384, 1),
+	DEF_FIXED("r",         R8A77990_CLK_R,     CLK_EXTAL,   1536, 1),
+
+	DEF_GEN3_PE("s0d6c",   R8A77990_CLK_S0D6C, CLK_S0, 6, CLK_PE, 2),
+	DEF_GEN3_PE("s3d1c",   R8A77990_CLK_S3D1C, CLK_S3, 1, CLK_PE, 1),
+	DEF_GEN3_PE("s3d2c",   R8A77990_CLK_S3D2C, CLK_S3, 2, CLK_PE, 2),
+	DEF_GEN3_PE("s3d4c",   R8A77990_CLK_S3D4C, CLK_S3, 4, CLK_PE, 4),
+
+	DEF_DIV6P1("canfd",    R8A77990_CLK_CANFD, CLK_PLL0D6, 0x244),
+	DEF_DIV6P1("csi0",     R8A77990_CLK_CSI0,  CLK_PLL1D2, 0x00c),
+	DEF_DIV6P1("mso",      R8A77990_CLK_MSO,   CLK_PLL1D2, 0x014),
+};
+
+static const struct mssr_mod_clk r8a77990_mod_clks[] __initconst = {
+	DEF_MOD("scif5",		 202,	R8A77990_CLK_S3D4C),
+	DEF_MOD("scif4",		 203,	R8A77990_CLK_S3D4C),
+	DEF_MOD("scif3",		 204,	R8A77990_CLK_S3D4C),
+	DEF_MOD("scif1",		 206,	R8A77990_CLK_S3D4C),
+	DEF_MOD("scif0",		 207,	R8A77990_CLK_S3D4C),
+	DEF_MOD("msiof3",		 208,	R8A77990_CLK_MSO),
+	DEF_MOD("msiof2",		 209,	R8A77990_CLK_MSO),
+	DEF_MOD("msiof1",		 210,	R8A77990_CLK_MSO),
+	DEF_MOD("msiof0",		 211,	R8A77990_CLK_MSO),
+	DEF_MOD("sys-dmac2",		 217,	R8A77990_CLK_S3D1),
+	DEF_MOD("sys-dmac1",		 218,	R8A77990_CLK_S3D1),
+	DEF_MOD("sys-dmac0",		 219,	R8A77990_CLK_S3D1),
+
+	DEF_MOD("cmt3",			 300,	R8A77990_CLK_R),
+	DEF_MOD("cmt2",			 301,	R8A77990_CLK_R),
+	DEF_MOD("cmt1",			 302,	R8A77990_CLK_R),
+	DEF_MOD("cmt0",			 303,	R8A77990_CLK_R),
+	DEF_MOD("scif2",		 310,	R8A77990_CLK_S3D4C),
+	DEF_MOD("sdif3",		 311,	R8A77990_CLK_SD3),
+	DEF_MOD("sdif1",		 313,	R8A77990_CLK_SD1),
+	DEF_MOD("sdif0",		 314,	R8A77990_CLK_SD0),
+	DEF_MOD("pcie0",		 319,	R8A77990_CLK_S3D1),
+	DEF_MOD("usb3-if0",		 328,	R8A77990_CLK_S3D1),
+	DEF_MOD("usb-dmac0",		 330,	R8A77990_CLK_S3D1),
+	DEF_MOD("usb-dmac1",		 331,	R8A77990_CLK_S3D1),
+
+	DEF_MOD("rwdt",			 402,	R8A77990_CLK_R),
+	DEF_MOD("intc-ex",		 407,	R8A77990_CLK_CP),
+	DEF_MOD("intc-ap",		 408,	R8A77990_CLK_S0D3),
+
+	DEF_MOD("audmac0",		 502,	R8A77990_CLK_S3D4),
+	DEF_MOD("drif7",		 508,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif6",		 509,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif5",		 510,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif4",		 511,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif3",		 512,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif2",		 513,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif1",		 514,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif0",		 515,	R8A77990_CLK_S3D2),
+	DEF_MOD("hscif4",		 516,	R8A77990_CLK_S3D1C),
+	DEF_MOD("hscif3",		 517,	R8A77990_CLK_S3D1C),
+	DEF_MOD("hscif2",		 518,	R8A77990_CLK_S3D1C),
+	DEF_MOD("hscif1",		 519,	R8A77990_CLK_S3D1C),
+	DEF_MOD("hscif0",		 520,	R8A77990_CLK_S3D1C),
+	DEF_MOD("thermal",		 522,	R8A77990_CLK_CP),
+	DEF_MOD("pwm",			 523,	R8A77990_CLK_S3D4C),
+
+	DEF_MOD("fcpvd1",		 602,	R8A77990_CLK_S1D2),
+	DEF_MOD("fcpvd0",		 603,	R8A77990_CLK_S1D2),
+	DEF_MOD("fcpvb0",		 607,	R8A77990_CLK_S0D1),
+	DEF_MOD("fcpvi0",		 611,	R8A77990_CLK_S0D1),
+	DEF_MOD("fcpf0",		 615,	R8A77990_CLK_S0D1),
+	DEF_MOD("fcpcs",		 619,	R8A77990_CLK_S0D1),
+	DEF_MOD("vspd1",		 622,	R8A77990_CLK_S1D2),
+	DEF_MOD("vspd0",		 623,	R8A77990_CLK_S1D2),
+	DEF_MOD("vspb",			 626,	R8A77990_CLK_S0D1),
+	DEF_MOD("vspi0",		 631,	R8A77990_CLK_S0D1),
+
+	DEF_MOD("ehci0",		 703,	R8A77990_CLK_S3D4),
+	DEF_MOD("hsusb",		 704,	R8A77990_CLK_S3D4),
+	DEF_MOD("csi40",		 716,	R8A77990_CLK_CSI0),
+	DEF_MOD("du1",			 723,	R8A77990_CLK_S2D1),
+	DEF_MOD("du0",			 724,	R8A77990_CLK_S2D1),
+	DEF_MOD("lvds",			 727,	R8A77990_CLK_S2D1),
+
+	DEF_MOD("vin5",			 806,	R8A77990_CLK_S1D2),
+	DEF_MOD("vin4",			 807,	R8A77990_CLK_S1D2),
+	DEF_MOD("etheravb",		 812,	R8A77990_CLK_S3D2),
+
+	DEF_MOD("gpio6",		 906,	R8A77990_CLK_S3D4),
+	DEF_MOD("gpio5",		 907,	R8A77990_CLK_S3D4),
+	DEF_MOD("gpio4",		 908,	R8A77990_CLK_S3D4),
+	DEF_MOD("gpio3",		 909,	R8A77990_CLK_S3D4),
+	DEF_MOD("gpio2",		 910,	R8A77990_CLK_S3D4),
+	DEF_MOD("gpio1",		 911,	R8A77990_CLK_S3D4),
+	DEF_MOD("gpio0",		 912,	R8A77990_CLK_S3D4),
+	DEF_MOD("can-fd",		 914,	R8A77990_CLK_S3D2),
+	DEF_MOD("can-if1",		 915,	R8A77990_CLK_S3D4),
+	DEF_MOD("can-if0",		 916,	R8A77990_CLK_S3D4),
+	DEF_MOD("i2c6",			 918,	R8A77990_CLK_S3D2),
+	DEF_MOD("i2c5",			 919,	R8A77990_CLK_S3D2),
+	DEF_MOD("i2c-dvfs",		 926,	R8A77990_CLK_CP),
+	DEF_MOD("i2c4",			 927,	R8A77990_CLK_S3D2),
+	DEF_MOD("i2c3",			 928,	R8A77990_CLK_S3D2),
+	DEF_MOD("i2c2",			 929,	R8A77990_CLK_S3D2),
+	DEF_MOD("i2c1",			 930,	R8A77990_CLK_S3D2),
+	DEF_MOD("i2c0",			 931,	R8A77990_CLK_S3D2),
+
+	DEF_MOD("ssi-all",		1005,	R8A77990_CLK_S3D4),
+	DEF_MOD("ssi9",			1006,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi8",			1007,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi7",			1008,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi6",			1009,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi5",			1010,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi4",			1011,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi3",			1012,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi2",			1013,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi1",			1014,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi0",			1015,	MOD_CLK_ID(1005)),
+	DEF_MOD("scu-all",		1017,	R8A77990_CLK_S3D4),
+	DEF_MOD("scu-dvc1",		1018,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-dvc0",		1019,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu1-mix1",	1020,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu0-mix0",	1021,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src9",		1022,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src8",		1023,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src7",		1024,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src6",		1025,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src5",		1026,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src4",		1027,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src3",		1028,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src2",		1029,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src1",		1030,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src0",		1031,	MOD_CLK_ID(1017)),
+};
+
+static const unsigned int r8a77990_crit_mod_clks[] __initconst = {
+	MOD_CLK_ID(408),	/* INTC-AP (GIC) */
+};
+
+/*
+ * CPG Clock Data
+ */
+
+/*
+ * MD19		EXTAL (MHz)	PLL0		PLL1		PLL3
+ *--------------------------------------------------------------------
+ * 0		48 x 1		x100/4		x100/3		x100/3
+ * 1		48 x 1		x100/4		x100/3		 x58/3
+ */
+#define CPG_PLL_CONFIG_INDEX(md)	(((md) & BIT(19)) >> 19)
+
+static const struct rcar_gen3_cpg_pll_config cpg_pll_configs[2] __initconst = {
+	/* EXTAL div	PLL1 mult/div	PLL3 mult/div */
+	{ 1,		100,	3,	100,	3,	},
+	{ 1,		100,	3,	 58,	3,	},
+};
+
+static int __init r8a77990_cpg_mssr_init(struct device *dev)
+{
+	const struct rcar_gen3_cpg_pll_config *cpg_pll_config;
+	u32 cpg_mode;
+	int error;
+
+	error = rcar_rst_read_mode_pins(&cpg_mode);
+	if (error)
+		return error;
+
+	cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)];
+
+	return rcar_gen3_cpg_init(cpg_pll_config, 0, cpg_mode);
+}
+
+const struct cpg_mssr_info r8a77990_cpg_mssr_info __initconst = {
+	/* Core Clocks */
+	.core_clks = r8a77990_core_clks,
+	.num_core_clks = ARRAY_SIZE(r8a77990_core_clks),
+	.last_dt_core_clk = LAST_DT_CORE_CLK,
+	.num_total_core_clks = MOD_CLK_BASE,
+
+	/* Module Clocks */
+	.mod_clks = r8a77990_mod_clks,
+	.num_mod_clks = ARRAY_SIZE(r8a77990_mod_clks),
+	.num_hw_mod_clks = 12 * 32,
+
+	/* Critical Module Clocks */
+	.crit_mod_clks = r8a77990_crit_mod_clks,
+	.num_crit_mod_clks = ARRAY_SIZE(r8a77990_crit_mod_clks),
+
+	/* Callbacks */
+	.init = r8a77990_cpg_mssr_init,
+	.cpg_clk_register = rcar_gen3_cpg_clk_register,
+};
diff --git a/drivers/clk/renesas/rcar-gen2-cpg.c b/drivers/clk/renesas/rcar-gen2-cpg.c
index feb14579a71b..daf88bc2cdae 100644
--- a/drivers/clk/renesas/rcar-gen2-cpg.c
+++ b/drivers/clk/renesas/rcar-gen2-cpg.c
@@ -16,6 +16,7 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/slab.h>
+#include <linux/sys_soc.h>
 
 #include "renesas-cpg-mssr.h"
 #include "rcar-gen2-cpg.h"
@@ -260,6 +261,17 @@ static const struct clk_div_table cpg_sd01_div_table[] = {
 static const struct rcar_gen2_cpg_pll_config *cpg_pll_config __initdata;
 static unsigned int cpg_pll0_div __initdata;
 static u32 cpg_mode __initdata;
+static u32 cpg_quirks __initdata;
+
+#define SD_SKIP_FIRST	BIT(0)		/* Skip first clock in SD table */
+
+static const struct soc_device_attribute cpg_quirks_match[] __initconst = {
+	{
+		.soc_id = "r8a77470",
+		.data = (void *)SD_SKIP_FIRST,
+	},
+	{ /* sentinel */ }
+};
 
 struct clk * __init rcar_gen2_cpg_clk_register(struct device *dev,
 	const struct cpg_core_clk *core, const struct cpg_mssr_info *info,
@@ -327,11 +339,17 @@ struct clk * __init rcar_gen2_cpg_clk_register(struct device *dev,
 
 	case CLK_TYPE_GEN2_SD0:
 		table = cpg_sd01_div_table;
+		if (cpg_quirks & SD_SKIP_FIRST)
+			table++;
+
 		shift = 4;
 		break;
 
 	case CLK_TYPE_GEN2_SD1:
 		table = cpg_sd01_div_table;
+		if (cpg_quirks & SD_SKIP_FIRST)
+			table++;
+
 		shift = 0;
 		break;
 
@@ -360,9 +378,15 @@ struct clk * __init rcar_gen2_cpg_clk_register(struct device *dev,
 int __init rcar_gen2_cpg_init(const struct rcar_gen2_cpg_pll_config *config,
 			      unsigned int pll0_div, u32 mode)
 {
+	const struct soc_device_attribute *attr;
+
 	cpg_pll_config = config;
 	cpg_pll0_div = pll0_div;
 	cpg_mode = mode;
+	attr = soc_device_match(cpg_quirks_match);
+	if (attr)
+		cpg_quirks = (uintptr_t)attr->data;
+	pr_debug("%s: mode = 0x%x quirks = 0x%x\n", __func__, mode, cpg_quirks);
 
 	spin_lock_init(&cpg_lock);
 
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c
index 4e88e980fb76..49e510691eee 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.c
+++ b/drivers/clk/renesas/renesas-cpg-mssr.c
@@ -652,6 +652,12 @@ static const struct of_device_id cpg_mssr_match[] = {
 		.data = &r8a7745_cpg_mssr_info,
 	},
 #endif
+#ifdef CONFIG_CLK_R8A77470
+	{
+		.compatible = "renesas,r8a77470-cpg-mssr",
+		.data = &r8a77470_cpg_mssr_info,
+	},
+#endif
 #ifdef CONFIG_CLK_R8A7790
 	{
 		.compatible = "renesas,r8a7790-cpg-mssr",
@@ -711,6 +717,12 @@ static const struct of_device_id cpg_mssr_match[] = {
 		.data = &r8a77980_cpg_mssr_info,
 	},
 #endif
+#ifdef CONFIG_CLK_R8A77990
+	{
+		.compatible = "renesas,r8a77990-cpg-mssr",
+		.data = &r8a77990_cpg_mssr_info,
+	},
+#endif
 #ifdef CONFIG_CLK_R8A77995
 	{
 		.compatible = "renesas,r8a77995-cpg-mssr",
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.h b/drivers/clk/renesas/renesas-cpg-mssr.h
index 97ccb093c10f..642f720b9b05 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.h
+++ b/drivers/clk/renesas/renesas-cpg-mssr.h
@@ -133,6 +133,7 @@ struct cpg_mssr_info {
 
 extern const struct cpg_mssr_info r8a7743_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7745_cpg_mssr_info;
+extern const struct cpg_mssr_info r8a77470_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7790_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7791_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7792_cpg_mssr_info;
@@ -142,6 +143,7 @@ extern const struct cpg_mssr_info r8a7796_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a77965_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a77970_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a77980_cpg_mssr_info;
+extern const struct cpg_mssr_info r8a77990_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a77995_cpg_mssr_info;
 
 
diff --git a/drivers/clk/rockchip/Makefile b/drivers/clk/rockchip/Makefile
index 59b8d320960a..98e7b9429b83 100644
--- a/drivers/clk/rockchip/Makefile
+++ b/drivers/clk/rockchip/Makefile
@@ -3,7 +3,6 @@
 # Rockchip Clock specific Makefile
 #
 
-obj-y	+= clk-rockchip.o
 obj-y	+= clk.o
 obj-y	+= clk-pll.o
 obj-y	+= clk-cpu.o
diff --git a/drivers/clk/rockchip/clk-rockchip.c b/drivers/clk/rockchip/clk-rockchip.c
deleted file mode 100644
index 2c9bb81144c9..000000000000
--- a/drivers/clk/rockchip/clk-rockchip.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright (c) 2013 MundoReader S.L.
- * Author: Heiko Stuebner <heiko@sntech.de>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/clk-provider.h>
-#include <linux/clkdev.h>
-#include <linux/of.h>
-#include <linux/of_address.h>
-
-static DEFINE_SPINLOCK(clk_lock);
-
-/*
- * Gate clocks
- */
-
-static void __init rk2928_gate_clk_init(struct device_node *node)
-{
-	struct clk_onecell_data *clk_data;
-	const char *clk_parent;
-	const char *clk_name;
-	void __iomem *reg;
-	void __iomem *reg_idx;
-	int flags;
-	int qty;
-	int reg_bit;
-	int clkflags = CLK_SET_RATE_PARENT;
-	int i;
-
-	qty = of_property_count_strings(node, "clock-output-names");
-	if (qty < 0) {
-		pr_err("%s: error in clock-output-names %d\n", __func__, qty);
-		return;
-	}
-
-	if (qty == 0) {
-		pr_info("%s: nothing to do\n", __func__);
-		return;
-	}
-
-	reg = of_iomap(node, 0);
-	if (!reg)
-		return;
-
-	clk_data = kzalloc(sizeof(struct clk_onecell_data), GFP_KERNEL);
-	if (!clk_data) {
-		iounmap(reg);
-		return;
-	}
-
-	clk_data->clks = kzalloc(qty * sizeof(struct clk *), GFP_KERNEL);
-	if (!clk_data->clks) {
-		kfree(clk_data);
-		iounmap(reg);
-		return;
-	}
-
-	flags = CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE;
-
-	for (i = 0; i < qty; i++) {
-		of_property_read_string_index(node, "clock-output-names",
-					      i, &clk_name);
-
-		/* ignore empty slots */
-		if (!strcmp("reserved", clk_name))
-			continue;
-
-		clk_parent = of_clk_get_parent_name(node, i);
-
-		/* keep all gates untouched for now */
-		clkflags |= CLK_IGNORE_UNUSED;
-
-		reg_idx = reg + (4 * (i / 16));
-		reg_bit = (i % 16);
-
-		clk_data->clks[i] = clk_register_gate(NULL, clk_name,
-						      clk_parent, clkflags,
-						      reg_idx, reg_bit,
-						      flags,
-						      &clk_lock);
-		WARN_ON(IS_ERR(clk_data->clks[i]));
-	}
-
-	clk_data->clk_num = qty;
-
-	of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
-}
-CLK_OF_DECLARE(rk2928_gate, "rockchip,rk2928-gate-clk", rk2928_gate_clk_init);
diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c
index 3cd8ad59e0b7..326b3fa44f5d 100644
--- a/drivers/clk/rockchip/clk.c
+++ b/drivers/clk/rockchip/clk.c
@@ -274,18 +274,10 @@ static struct clk *rockchip_clk_register_frac_branch(
 		struct clk_mux *frac_mux = &frac->mux;
 		struct clk_init_data init;
 		struct clk *mux_clk;
-		int i, ret;
-
-		frac->mux_frac_idx = -1;
-		for (i = 0; i < child->num_parents; i++) {
-			if (!strcmp(name, child->parent_names[i])) {
-				pr_debug("%s: found fractional parent in mux at pos %d\n",
-					 __func__, i);
-				frac->mux_frac_idx = i;
-				break;
-			}
-		}
+		int ret;
 
+		frac->mux_frac_idx = match_string(child->parent_names,
+						  child->num_parents, name);
 		frac->mux_ops = &clk_mux_ops;
 		frac->clk_nb.notifier_call = rockchip_clk_frac_notifier_cb;
 
@@ -312,6 +304,8 @@ static struct clk *rockchip_clk_register_frac_branch(
 
 		/* notifier on the fraction divider to catch rate changes */
 		if (frac->mux_frac_idx >= 0) {
+			pr_debug("%s: found fractional parent in mux at pos %d\n",
+				 __func__, frac->mux_frac_idx);
 			ret = clk_notifier_register(clk, &frac->clk_nb);
 			if (ret)
 				pr_err("%s: failed to register clock notifier for %s\n",
diff --git a/drivers/clk/samsung/clk-s3c2410-dclk.c b/drivers/clk/samsung/clk-s3c2410-dclk.c
index 077df3e539a7..f41d89cef0f1 100644
--- a/drivers/clk/samsung/clk-s3c2410-dclk.c
+++ b/drivers/clk/samsung/clk-s3c2410-dclk.c
@@ -219,8 +219,7 @@ static int s3c24xx_dclk1_div_notify(struct notifier_block *nb,
 #ifdef CONFIG_PM_SLEEP
 static int s3c24xx_dclk_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct s3c24xx_dclk *s3c24xx_dclk = platform_get_drvdata(pdev);
+	struct s3c24xx_dclk *s3c24xx_dclk = dev_get_drvdata(dev);
 
 	s3c24xx_dclk->reg_save = readl_relaxed(s3c24xx_dclk->base);
 	return 0;
@@ -228,8 +227,7 @@ static int s3c24xx_dclk_suspend(struct device *dev)
 
 static int s3c24xx_dclk_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct s3c24xx_dclk *s3c24xx_dclk = platform_get_drvdata(pdev);
+	struct s3c24xx_dclk *s3c24xx_dclk = dev_get_drvdata(dev);
 
 	writel_relaxed(s3c24xx_dclk->reg_save, s3c24xx_dclk->base);
 	return 0;
diff --git a/drivers/clk/socfpga/clk-s10.c b/drivers/clk/socfpga/clk-s10.c
index 3a11c382a663..72714633e39c 100644
--- a/drivers/clk/socfpga/clk-s10.c
+++ b/drivers/clk/socfpga/clk-s10.c
@@ -260,46 +260,45 @@ static int s10_clk_register_pll(const struct stratix10_pll_clock *clks,
 	return 0;
 }
 
-static struct stratix10_clock_data *__socfpga_s10_clk_init(struct device_node *np,
+static struct stratix10_clock_data *__socfpga_s10_clk_init(struct platform_device *pdev,
 						    int nr_clks)
 {
+	struct device_node *np = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
 	struct stratix10_clock_data *clk_data;
 	struct clk **clk_table;
+	struct resource *res;
 	void __iomem *base;
 
-	base = of_iomap(np, 0);
-	if (!base) {
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base)) {
 		pr_err("%s: failed to map clock registers\n", __func__);
-		goto err;
+		return ERR_CAST(base);
 	}
 
-	clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
+	clk_data = devm_kzalloc(dev, sizeof(*clk_data), GFP_KERNEL);
 	if (!clk_data)
-		goto err;
+		return ERR_PTR(-ENOMEM);
 
 	clk_data->base = base;
-	clk_table = kcalloc(nr_clks, sizeof(*clk_table), GFP_KERNEL);
+	clk_table = devm_kcalloc(dev, nr_clks, sizeof(*clk_table), GFP_KERNEL);
 	if (!clk_table)
-		goto err_data;
+		return ERR_PTR(-ENOMEM);
 
 	clk_data->clk_data.clks = clk_table;
 	clk_data->clk_data.clk_num = nr_clks;
 	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data->clk_data);
 	return clk_data;
-
-err_data:
-	kfree(clk_data);
-err:
-	return NULL;
 }
 
-static int s10_clkmgr_init(struct device_node *np)
+static int s10_clkmgr_init(struct platform_device *pdev)
 {
 	struct stratix10_clock_data *clk_data;
 
-	clk_data = __socfpga_s10_clk_init(np, STRATIX10_NUM_CLKS);
-	if (!clk_data)
-		return -ENOMEM;
+	clk_data = __socfpga_s10_clk_init(pdev, STRATIX10_NUM_CLKS);
+	if (IS_ERR(clk_data))
+		return PTR_ERR(clk_data);
 
 	s10_clk_register_pll(s10_pll_clks, ARRAY_SIZE(s10_pll_clks), clk_data);
 
@@ -317,11 +316,7 @@ static int s10_clkmgr_init(struct device_node *np)
 
 static int s10_clkmgr_probe(struct platform_device *pdev)
 {
-	struct device_node *np = pdev->dev.of_node;
-
-	s10_clkmgr_init(np);
-
-	return 0;
+	return	s10_clkmgr_init(pdev);
 }
 
 static const struct of_device_id stratix10_clkmgr_match_table[] = {
@@ -334,6 +329,7 @@ static struct platform_driver stratix10_clkmgr_driver = {
 	.probe		= s10_clkmgr_probe,
 	.driver		= {
 		.name	= "stratix10-clkmgr",
+		.suppress_bind_attrs = true,
 		.of_match_table = stratix10_clkmgr_match_table,
 	},
 };
diff --git a/drivers/clk/spear/spear6xx_clock.c b/drivers/clk/spear/spear6xx_clock.c
index f911d9f77763..47810be7f15c 100644
--- a/drivers/clk/spear/spear6xx_clock.c
+++ b/drivers/clk/spear/spear6xx_clock.c
@@ -147,7 +147,7 @@ void __init spear6xx_clk_init(void __iomem *misc_base)
 
 	clk = clk_register_fixed_factor(NULL, "wdt_clk", "osc_30m_clk", 0, 1,
 			1);
-	clk_register_clkdev(clk, NULL, "wdt");
+	clk_register_clkdev(clk, NULL, "fc880000.wdt");
 
 	/* clock derived from pll1 clk */
 	clk = clk_register_fixed_factor(NULL, "cpu_clk", "pll1_clk",
diff --git a/drivers/clk/sunxi-ng/Kconfig b/drivers/clk/sunxi-ng/Kconfig
index 79dfd296c3d1..826674d090fd 100644
--- a/drivers/clk/sunxi-ng/Kconfig
+++ b/drivers/clk/sunxi-ng/Kconfig
@@ -16,6 +16,11 @@ config SUN50I_H6_CCU
 	default ARM64 && ARCH_SUNXI
 	depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST
 
+config SUN50I_H6_R_CCU
+	bool "Support for the Allwinner H6 PRCM CCU"
+	default ARM64 && ARCH_SUNXI
+	depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST
+
 config SUN4I_A10_CCU
 	bool "Support for the Allwinner A10/A20 CCU"
 	default MACH_SUN4I
diff --git a/drivers/clk/sunxi-ng/Makefile b/drivers/clk/sunxi-ng/Makefile
index 128a40ee5c5e..49454700f2e5 100644
--- a/drivers/clk/sunxi-ng/Makefile
+++ b/drivers/clk/sunxi-ng/Makefile
@@ -1,28 +1,29 @@
 # SPDX-License-Identifier: GPL-2.0
 # Common objects
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_common.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_mmc_timing.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_reset.o
+obj-y				+= ccu_common.o
+obj-y				+= ccu_mmc_timing.o
+obj-y				+= ccu_reset.o
 
 # Base clock types
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_div.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_frac.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_gate.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_mux.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_mult.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_phase.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_sdm.o
+obj-y				+= ccu_div.o
+obj-y				+= ccu_frac.o
+obj-y				+= ccu_gate.o
+obj-y				+= ccu_mux.o
+obj-y				+= ccu_mult.o
+obj-y				+= ccu_phase.o
+obj-y				+= ccu_sdm.o
 
 # Multi-factor clocks
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_nk.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_nkm.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_nkmp.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_nm.o
-lib-$(CONFIG_SUNXI_CCU)		+= ccu_mp.o
+obj-y				+= ccu_nk.o
+obj-y				+= ccu_nkm.o
+obj-y				+= ccu_nkmp.o
+obj-y				+= ccu_nm.o
+obj-y				+= ccu_mp.o
 
 # SoC support
 obj-$(CONFIG_SUN50I_A64_CCU)	+= ccu-sun50i-a64.o
 obj-$(CONFIG_SUN50I_H6_CCU)	+= ccu-sun50i-h6.o
+obj-$(CONFIG_SUN50I_H6_R_CCU)	+= ccu-sun50i-h6-r.o
 obj-$(CONFIG_SUN4I_A10_CCU)	+= ccu-sun4i-a10.o
 obj-$(CONFIG_SUN5I_CCU)		+= ccu-sun5i.o
 obj-$(CONFIG_SUN6I_A31_CCU)	+= ccu-sun6i-a31.o
@@ -37,12 +38,3 @@ obj-$(CONFIG_SUN8I_R40_CCU)	+= ccu-sun8i-r40.o
 obj-$(CONFIG_SUN9I_A80_CCU)	+= ccu-sun9i-a80.o
 obj-$(CONFIG_SUN9I_A80_CCU)	+= ccu-sun9i-a80-de.o
 obj-$(CONFIG_SUN9I_A80_CCU)	+= ccu-sun9i-a80-usb.o
-
-# The lib-y file goals is supposed to work only in arch/*/lib or lib/. In our
-# case, we want to use that goal, but even though lib.a will be properly
-# generated, it will not be linked in, eventually resulting in a linker error
-# for missing symbols.
-#
-# We can work around that by explicitly adding lib.a to the obj-y goal. This is
-# an undocumented behaviour, but works well for now.
-obj-$(CONFIG_SUNXI_CCU)		+= lib.a
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c
new file mode 100644
index 000000000000..27554eaf6929
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c
@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2017 Icenowy Zheng <icenowy@aosc.xyz>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+
+#include "ccu_common.h"
+#include "ccu_reset.h"
+
+#include "ccu_div.h"
+#include "ccu_gate.h"
+#include "ccu_mp.h"
+#include "ccu_nm.h"
+
+#include "ccu-sun50i-h6-r.h"
+
+/*
+ * Information about AR100 and AHB/APB clocks in R_CCU are gathered from
+ * clock definitions in the BSP source code.
+ */
+
+static const char * const ar100_r_apb2_parents[] = { "osc24M", "osc32k",
+					     "pll-periph0", "iosc" };
+static const struct ccu_mux_var_prediv ar100_r_apb2_predivs[] = {
+	{ .index = 2, .shift = 0, .width = 5 },
+};
+
+static struct ccu_div ar100_clk = {
+	.div		= _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO),
+
+	.mux		= {
+		.shift	= 24,
+		.width	= 2,
+
+		.var_predivs	= ar100_r_apb2_predivs,
+		.n_var_predivs	= ARRAY_SIZE(ar100_r_apb2_predivs),
+	},
+
+	.common		= {
+		.reg		= 0x000,
+		.features	= CCU_FEATURE_VARIABLE_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("ar100",
+						      ar100_r_apb2_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static CLK_FIXED_FACTOR(r_ahb_clk, "r-ahb", "ar100", 1, 1, 0);
+
+static struct ccu_div r_apb1_clk = {
+	.div		= _SUNXI_CCU_DIV(0, 2),
+
+	.common		= {
+		.reg		= 0x00c,
+		.hw.init	= CLK_HW_INIT("r-apb1",
+					      "r-ahb",
+					      &ccu_div_ops,
+					      0),
+	},
+};
+
+static struct ccu_div r_apb2_clk = {
+	.div		= _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO),
+
+	.mux		= {
+		.shift	= 24,
+		.width	= 2,
+
+		.var_predivs	= ar100_r_apb2_predivs,
+		.n_var_predivs	= ARRAY_SIZE(ar100_r_apb2_predivs),
+	},
+
+	.common		= {
+		.reg		= 0x010,
+		.features	= CCU_FEATURE_VARIABLE_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("r-apb2",
+						      ar100_r_apb2_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+/*
+ * Information about the gate/resets are gathered from the clock header file
+ * in the BSP source code, although most of them are unused. The existence
+ * of the hardware block is verified with "3.1 Memory Mapping" chapter in
+ * "Allwinner H6 V200 User Manual V1.1"; and the parent APB buses are verified
+ * with "3.3.2.1 System Bus Tree" chapter inthe same document.
+ */
+static SUNXI_CCU_GATE(r_apb1_timer_clk,	"r-apb1-timer",	"r-apb1",
+		      0x11c, BIT(0), 0);
+static SUNXI_CCU_GATE(r_apb1_twd_clk,	"r-apb1-twd",	"r-apb1",
+		      0x12c, BIT(0), 0);
+static SUNXI_CCU_GATE(r_apb1_pwm_clk,	"r-apb1-pwm",	"r-apb1",
+		      0x13c, BIT(0), 0);
+static SUNXI_CCU_GATE(r_apb2_uart_clk,	"r-apb2-uart",	"r-apb2",
+		      0x18c, BIT(0), 0);
+static SUNXI_CCU_GATE(r_apb2_i2c_clk,	"r-apb2-i2c",	"r-apb2",
+		      0x19c, BIT(0), 0);
+static SUNXI_CCU_GATE(r_apb1_ir_clk,	"r-apb1-ir",	"r-apb1",
+		      0x1cc, BIT(0), 0);
+static SUNXI_CCU_GATE(r_apb1_w1_clk,	"r-apb1-w1",	"r-apb1",
+		      0x1cc, BIT(0), 0);
+
+/* Information of IR(RX) mod clock is gathered from BSP source code */
+static const char * const r_mod0_default_parents[] = { "osc32k", "osc24M" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(ir_clk, "ir",
+				  r_mod0_default_parents, 0x1c0,
+				  0, 5,		/* M */
+				  8, 2,		/* P */
+				  24, 1,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+/*
+ * BSP didn't use the 1-wire function at all now, and the information about
+ * this mod clock is guessed from the IR mod clock above. The existence of
+ * this mod clock is proven by BSP clock header, and the dividers are verified
+ * by contents in the 1-wire related chapter of the User Manual.
+ */
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(w1_clk, "w1",
+				  r_mod0_default_parents, 0x1e0,
+				  0, 5,		/* M */
+				  8, 2,		/* P */
+				  24, 1,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static struct ccu_common *sun50i_h6_r_ccu_clks[] = {
+	&ar100_clk.common,
+	&r_apb1_clk.common,
+	&r_apb2_clk.common,
+	&r_apb1_timer_clk.common,
+	&r_apb1_twd_clk.common,
+	&r_apb1_pwm_clk.common,
+	&r_apb2_uart_clk.common,
+	&r_apb2_i2c_clk.common,
+	&r_apb1_ir_clk.common,
+	&r_apb1_w1_clk.common,
+	&ir_clk.common,
+	&w1_clk.common,
+};
+
+static struct clk_hw_onecell_data sun50i_h6_r_hw_clks = {
+	.hws	= {
+		[CLK_AR100]		= &ar100_clk.common.hw,
+		[CLK_R_AHB]		= &r_ahb_clk.hw,
+		[CLK_R_APB1]		= &r_apb1_clk.common.hw,
+		[CLK_R_APB2]		= &r_apb2_clk.common.hw,
+		[CLK_R_APB1_TIMER]	= &r_apb1_timer_clk.common.hw,
+		[CLK_R_APB1_TWD]	= &r_apb1_twd_clk.common.hw,
+		[CLK_R_APB1_PWM]	= &r_apb1_pwm_clk.common.hw,
+		[CLK_R_APB2_UART]	= &r_apb2_uart_clk.common.hw,
+		[CLK_R_APB2_I2C]	= &r_apb2_i2c_clk.common.hw,
+		[CLK_R_APB1_IR]		= &r_apb1_ir_clk.common.hw,
+		[CLK_R_APB1_W1]		= &r_apb1_w1_clk.common.hw,
+		[CLK_IR]		= &ir_clk.common.hw,
+		[CLK_W1]		= &w1_clk.common.hw,
+	},
+	.num	= CLK_NUMBER,
+};
+
+static struct ccu_reset_map sun50i_h6_r_ccu_resets[] = {
+	[RST_R_APB1_TIMER]	=  { 0x11c, BIT(16) },
+	[RST_R_APB1_TWD]	=  { 0x12c, BIT(16) },
+	[RST_R_APB1_PWM]	=  { 0x13c, BIT(16) },
+	[RST_R_APB2_UART]	=  { 0x18c, BIT(16) },
+	[RST_R_APB2_I2C]	=  { 0x19c, BIT(16) },
+	[RST_R_APB1_IR]		=  { 0x1cc, BIT(16) },
+	[RST_R_APB1_W1]		=  { 0x1ec, BIT(16) },
+};
+
+static const struct sunxi_ccu_desc sun50i_h6_r_ccu_desc = {
+	.ccu_clks	= sun50i_h6_r_ccu_clks,
+	.num_ccu_clks	= ARRAY_SIZE(sun50i_h6_r_ccu_clks),
+
+	.hw_clks	= &sun50i_h6_r_hw_clks,
+
+	.resets		= sun50i_h6_r_ccu_resets,
+	.num_resets	= ARRAY_SIZE(sun50i_h6_r_ccu_resets),
+};
+
+static void __init sunxi_r_ccu_init(struct device_node *node,
+				    const struct sunxi_ccu_desc *desc)
+{
+	void __iomem *reg;
+
+	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
+	if (IS_ERR(reg)) {
+		pr_err("%pOF: Could not map the clock registers\n", node);
+		return;
+	}
+
+	sunxi_ccu_probe(node, reg, desc);
+}
+
+static void __init sun50i_h6_r_ccu_setup(struct device_node *node)
+{
+	sunxi_r_ccu_init(node, &sun50i_h6_r_ccu_desc);
+}
+CLK_OF_DECLARE(sun50i_h6_r_ccu, "allwinner,sun50i-h6-r-ccu",
+	       sun50i_h6_r_ccu_setup);
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h
new file mode 100644
index 000000000000..782117dc0b28
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2017 Icenowy Zheng <icenowy@aosc.xyz>
+ */
+
+#ifndef _CCU_SUN50I_H6_R_H
+#define _CCU_SUN50I_H6_R_H
+
+#include <dt-bindings/clock/sun50i-h6-r-ccu.h>
+#include <dt-bindings/reset/sun50i-h6-r-ccu.h>
+
+/* AHB/APB bus clocks are not exported except APB1 for R_PIO */
+#define CLK_R_AHB	1
+
+#define CLK_R_APB2	3
+
+#define CLK_NUMBER	(CLK_W1 + 1)
+
+#endif /* _CCU_SUN50I_H6_R_H */
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c
index 933f2e68f42a..65ba6455feb7 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c
@@ -12,7 +12,8 @@
  */
 
 #include <linux/clk-provider.h>
-#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
 
 #include "ccu_common.h"
 #include "ccu_reset.h"
@@ -1250,17 +1251,45 @@ static struct ccu_mux_nb sun8i_r40_cpu_nb = {
 	.bypass_index	= 1, /* index of 24 MHz oscillator */
 };
 
-static void __init sun8i_r40_ccu_setup(struct device_node *node)
+/*
+ * Add a regmap for the GMAC driver (dwmac-sun8i) to access the
+ * GMAC configuration register.
+ * Only this register is allowed to be written, in order to
+ * prevent overriding critical clock configuration.
+ */
+
+#define SUN8I_R40_GMAC_CFG_REG 0x164
+static bool sun8i_r40_ccu_regmap_accessible_reg(struct device *dev,
+						unsigned int reg)
+{
+	if (reg == SUN8I_R40_GMAC_CFG_REG)
+		return true;
+	return false;
+}
+
+static struct regmap_config sun8i_r40_ccu_regmap_config = {
+	.reg_bits	= 32,
+	.val_bits	= 32,
+	.reg_stride	= 4,
+	.max_register	= 0x320, /* PLL_LOCK_CTRL_REG */
+
+	/* other devices have no business accessing other registers */
+	.readable_reg	= sun8i_r40_ccu_regmap_accessible_reg,
+	.writeable_reg	= sun8i_r40_ccu_regmap_accessible_reg,
+};
+
+static int sun8i_r40_ccu_probe(struct platform_device *pdev)
 {
+	struct resource *res;
+	struct regmap *regmap;
 	void __iomem *reg;
 	u32 val;
+	int ret;
 
-	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
-	if (IS_ERR(reg)) {
-		pr_err("%s: Could not map the clock registers\n",
-		       of_node_full_name(node));
-		return;
-	}
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	reg = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(reg))
+		return PTR_ERR(reg);
 
 	/* Force the PLL-Audio-1x divider to 4 */
 	val = readl(reg + SUN8I_R40_PLL_AUDIO_REG);
@@ -1277,7 +1306,14 @@ static void __init sun8i_r40_ccu_setup(struct device_node *node)
 	val &= ~GENMASK(25, 20);
 	writel(val, reg + SUN8I_R40_USB_CLK_REG);
 
-	sunxi_ccu_probe(node, reg, &sun8i_r40_ccu_desc);
+	regmap = devm_regmap_init_mmio(&pdev->dev, reg,
+				       &sun8i_r40_ccu_regmap_config);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	ret = sunxi_ccu_probe(pdev->dev.of_node, reg, &sun8i_r40_ccu_desc);
+	if (ret)
+		return ret;
 
 	/* Gate then ungate PLL CPU after any rate changes */
 	ccu_pll_notifier_register(&sun8i_r40_pll_cpu_nb);
@@ -1285,6 +1321,20 @@ static void __init sun8i_r40_ccu_setup(struct device_node *node)
 	/* Reparent CPU during PLL CPU rate changes */
 	ccu_mux_notifier_register(pll_cpu_clk.common.hw.clk,
 				  &sun8i_r40_cpu_nb);
+
+	return 0;
 }
-CLK_OF_DECLARE(sun8i_r40_ccu, "allwinner,sun8i-r40-ccu",
-	       sun8i_r40_ccu_setup);
+
+static const struct of_device_id sun8i_r40_ccu_ids[] = {
+	{ .compatible = "allwinner,sun8i-r40-ccu" },
+	{ }
+};
+
+static struct platform_driver sun8i_r40_ccu_driver = {
+	.probe	= sun8i_r40_ccu_probe,
+	.driver	= {
+		.name	= "sun8i-r40-ccu",
+		.of_match_table	= sun8i_r40_ccu_ids,
+	},
+};
+builtin_platform_driver(sun8i_r40_ccu_driver);
diff --git a/drivers/clk/tegra/clk-tegra114.c b/drivers/clk/tegra/clk-tegra114.c
index 5d5a22d529f5..1824f014202b 100644
--- a/drivers/clk/tegra/clk-tegra114.c
+++ b/drivers/clk/tegra/clk-tegra114.c
@@ -1367,7 +1367,7 @@ static void __init tegra114_clock_init(struct device_node *np)
 	tegra_super_clk_gen4_init(clk_base, pmc_base, tegra114_clks,
 					&pll_x_params);
 
-	tegra_add_of_provider(np);
+	tegra_add_of_provider(np, of_clk_src_onecell_get);
 	tegra_register_devclks(devclks, ARRAY_SIZE(devclks));
 
 	tegra_clk_apply_init_table = tegra114_clock_apply_init_table;
diff --git a/drivers/clk/tegra/clk-tegra124.c b/drivers/clk/tegra/clk-tegra124.c
index 50088e976611..0c69c7970950 100644
--- a/drivers/clk/tegra/clk-tegra124.c
+++ b/drivers/clk/tegra/clk-tegra124.c
@@ -1479,7 +1479,7 @@ static void __init tegra124_132_clock_init_post(struct device_node *np)
 				  &pll_x_params);
 	tegra_init_special_resets(1, tegra124_reset_assert,
 				  tegra124_reset_deassert);
-	tegra_add_of_provider(np);
+	tegra_add_of_provider(np, of_clk_src_onecell_get);
 
 	clks[TEGRA124_CLK_EMC] = tegra_clk_register_emc(clk_base, np,
 							&emc_lock);
diff --git a/drivers/clk/tegra/clk-tegra20.c b/drivers/clk/tegra/clk-tegra20.c
index 0ee56dd04cec..cc857d4d4a86 100644
--- a/drivers/clk/tegra/clk-tegra20.c
+++ b/drivers/clk/tegra/clk-tegra20.c
@@ -26,6 +26,8 @@
 #include "clk.h"
 #include "clk-id.h"
 
+#define MISC_CLK_ENB 0x48
+
 #define OSC_CTRL 0x50
 #define OSC_CTRL_OSC_FREQ_MASK (3<<30)
 #define OSC_CTRL_OSC_FREQ_13MHZ (0<<30)
@@ -831,15 +833,25 @@ static void __init tegra20_periph_clk_init(void)
 				    periph_clk_enb_refcnt);
 	clks[TEGRA20_CLK_PEX] = clk;
 
+	/* dev1 OSC divider */
+	clk_register_divider(NULL, "dev1_osc_div", "clk_m",
+			     0, clk_base + MISC_CLK_ENB, 22, 2,
+			     CLK_DIVIDER_POWER_OF_TWO | CLK_DIVIDER_READ_ONLY,
+			     NULL);
+
+	/* dev2 OSC divider */
+	clk_register_divider(NULL, "dev2_osc_div", "clk_m",
+			     0, clk_base + MISC_CLK_ENB, 20, 2,
+			     CLK_DIVIDER_POWER_OF_TWO | CLK_DIVIDER_READ_ONLY,
+			     NULL);
+
 	/* cdev1 */
-	clk = clk_register_fixed_rate(NULL, "cdev1_fixed", NULL, 0, 26000000);
-	clk = tegra_clk_register_periph_gate("cdev1", "cdev1_fixed", 0,
+	clk = tegra_clk_register_periph_gate("cdev1", "cdev1_mux", 0,
 				    clk_base, 0, 94, periph_clk_enb_refcnt);
 	clks[TEGRA20_CLK_CDEV1] = clk;
 
 	/* cdev2 */
-	clk = clk_register_fixed_rate(NULL, "cdev2_fixed", NULL, 0, 26000000);
-	clk = tegra_clk_register_periph_gate("cdev2", "cdev2_fixed", 0,
+	clk = tegra_clk_register_periph_gate("cdev2", "cdev2_mux", 0,
 				    clk_base, 0, 93, periph_clk_enb_refcnt);
 	clks[TEGRA20_CLK_CDEV2] = clk;
 
@@ -1077,6 +1089,36 @@ static const struct of_device_id pmc_match[] __initconst = {
 	{ },
 };
 
+static struct clk *tegra20_clk_src_onecell_get(struct of_phandle_args *clkspec,
+					       void *data)
+{
+	struct clk_hw *parent_hw;
+	struct clk_hw *hw;
+	struct clk *clk;
+
+	clk = of_clk_src_onecell_get(clkspec, data);
+	if (IS_ERR(clk))
+		return clk;
+
+	/*
+	 * Tegra20 CDEV1 and CDEV2 clocks are a bit special case, their parent
+	 * clock is created by the pinctrl driver. It is possible for clk user
+	 * to request these clocks before pinctrl driver got probed and hence
+	 * user will get an orphaned clock. That might be undesirable because
+	 * user may expect parent clock to be enabled by the child.
+	 */
+	if (clkspec->args[0] == TEGRA20_CLK_CDEV1 ||
+	    clkspec->args[0] == TEGRA20_CLK_CDEV2) {
+		hw = __clk_get_hw(clk);
+
+		parent_hw = clk_hw_get_parent(hw);
+		if (!parent_hw)
+			return ERR_PTR(-EPROBE_DEFER);
+	}
+
+	return clk;
+}
+
 static void __init tegra20_clock_init(struct device_node *np)
 {
 	struct device_node *node;
@@ -1115,7 +1157,7 @@ static void __init tegra20_clock_init(struct device_node *np)
 
 	tegra_init_dup_clks(tegra_clk_duplicates, clks, TEGRA20_CLK_CLK_MAX);
 
-	tegra_add_of_provider(np);
+	tegra_add_of_provider(np, tegra20_clk_src_onecell_get);
 	tegra_register_devclks(devclks, ARRAY_SIZE(devclks));
 
 	tegra_clk_apply_init_table = tegra20_clock_apply_init_table;
diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c
index 9fb5d51ccce4..5435d01c636a 100644
--- a/drivers/clk/tegra/clk-tegra210.c
+++ b/drivers/clk/tegra/clk-tegra210.c
@@ -3567,7 +3567,7 @@ static void __init tegra210_clock_init(struct device_node *np)
 	tegra_init_special_resets(2, tegra210_reset_assert,
 				  tegra210_reset_deassert);
 
-	tegra_add_of_provider(np);
+	tegra_add_of_provider(np, of_clk_src_onecell_get);
 	tegra_register_devclks(devclks, ARRAY_SIZE(devclks));
 
 	tegra210_mbist_clk_init();
diff --git a/drivers/clk/tegra/clk-tegra30.c b/drivers/clk/tegra/clk-tegra30.c
index b316dfb6f6c7..acfe661b2ae7 100644
--- a/drivers/clk/tegra/clk-tegra30.c
+++ b/drivers/clk/tegra/clk-tegra30.c
@@ -1349,7 +1349,7 @@ static void __init tegra30_clock_init(struct device_node *np)
 
 	tegra_init_dup_clks(tegra_clk_duplicates, clks, TEGRA30_CLK_CLK_MAX);
 
-	tegra_add_of_provider(np);
+	tegra_add_of_provider(np, of_clk_src_onecell_get);
 	tegra_register_devclks(devclks, ARRAY_SIZE(devclks));
 
 	tegra_clk_apply_init_table = tegra30_clock_apply_init_table;
diff --git a/drivers/clk/tegra/clk.c b/drivers/clk/tegra/clk.c
index ba923f0d5953..593d76a114f9 100644
--- a/drivers/clk/tegra/clk.c
+++ b/drivers/clk/tegra/clk.c
@@ -298,7 +298,8 @@ static struct reset_controller_dev rst_ctlr = {
 	.of_reset_n_cells = 1,
 };
 
-void __init tegra_add_of_provider(struct device_node *np)
+void __init tegra_add_of_provider(struct device_node *np,
+				  void *clk_src_onecell_get)
 {
 	int i;
 
@@ -314,7 +315,7 @@ void __init tegra_add_of_provider(struct device_node *np)
 
 	clk_data.clks = clks;
 	clk_data.clk_num = clk_num;
-	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+	of_clk_add_provider(np, clk_src_onecell_get, &clk_data);
 
 	rst_ctlr.of_node = np;
 	rst_ctlr.nr_resets = periph_banks * 32 + num_special_reset;
diff --git a/drivers/clk/tegra/clk.h b/drivers/clk/tegra/clk.h
index ba7e20e6a82b..e1f88463b600 100644
--- a/drivers/clk/tegra/clk.h
+++ b/drivers/clk/tegra/clk.h
@@ -763,7 +763,7 @@ struct clk **tegra_clk_init(void __iomem *clk_base, int num, int periph_banks);
 
 struct clk **tegra_lookup_dt_id(int clk_id, struct tegra_clk *tegra_clk);
 
-void tegra_add_of_provider(struct device_node *np);
+void tegra_add_of_provider(struct device_node *np, void *clk_src_onecell_get);
 void tegra_register_devclks(struct tegra_devclk *dev_clks, int num);
 
 void tegra_audio_clk_init(void __iomem *clk_base,
diff --git a/drivers/clk/uniphier/clk-uniphier-sys.c b/drivers/clk/uniphier/clk-uniphier-sys.c
index ebc78ab2df05..4f5ff9fa11fd 100644
--- a/drivers/clk/uniphier/clk-uniphier-sys.c
+++ b/drivers/clk/uniphier/clk-uniphier-sys.c
@@ -51,6 +51,9 @@
 #define UNIPHIER_LD11_SYS_CLK_STDMAC(idx)				\
 	UNIPHIER_CLK_GATE("stdmac", (idx), NULL, 0x210c, 8)
 
+#define UNIPHIER_LD11_SYS_CLK_HSC(idx)					\
+	UNIPHIER_CLK_GATE("hsc", (idx), NULL, 0x210c, 9)
+
 #define UNIPHIER_PRO4_SYS_CLK_GIO(idx)					\
 	UNIPHIER_CLK_GATE("gio", (idx), NULL, 0x2104, 6)
 
@@ -182,6 +185,7 @@ const struct uniphier_clk_data uniphier_ld11_sys_clk_data[] = {
 	/* Index 5 reserved for eMMC PHY */
 	UNIPHIER_LD11_SYS_CLK_ETHER(6),
 	UNIPHIER_LD11_SYS_CLK_STDMAC(8),			/* HSC, MIO */
+	UNIPHIER_LD11_SYS_CLK_HSC(9),
 	UNIPHIER_CLK_FACTOR("usb2", -1, "ref", 24, 25),
 	UNIPHIER_LD11_SYS_CLK_AIO(40),
 	UNIPHIER_LD11_SYS_CLK_EVEA(41),
@@ -215,6 +219,7 @@ const struct uniphier_clk_data uniphier_ld20_sys_clk_data[] = {
 	UNIPHIER_LD20_SYS_CLK_SD,
 	UNIPHIER_LD11_SYS_CLK_ETHER(6),
 	UNIPHIER_LD11_SYS_CLK_STDMAC(8),			/* HSC */
+	UNIPHIER_LD11_SYS_CLK_HSC(9),
 	/* GIO is always clock-enabled: no function for 0x210c bit5 */
 	/*
 	 * clock for USB Link is enabled by the logic "OR" of bit 14 and bit 15.
diff --git a/drivers/clk/x86/Makefile b/drivers/clk/x86/Makefile
index 1367afb03858..00303bc05415 100644
--- a/drivers/clk/x86/Makefile
+++ b/drivers/clk/x86/Makefile
@@ -1,3 +1,4 @@
+obj-$(CONFIG_PMC_ATOM)		+= clk-pmc-atom.o
+obj-$(CONFIG_X86_AMD_PLATFORM_DEVICE)	+= clk-st.o
 clk-x86-lpss-objs		:= clk-lpt.o
 obj-$(CONFIG_X86_INTEL_LPSS)	+= clk-x86-lpss.o
-obj-$(CONFIG_PMC_ATOM)		+= clk-pmc-atom.o
diff --git a/drivers/clk/x86/clk-st.c b/drivers/clk/x86/clk-st.c
new file mode 100644
index 000000000000..fb62f3938008
--- /dev/null
+++ b/drivers/clk/x86/clk-st.c
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: MIT
+/*
+ * clock framework for AMD Stoney based clocks
+ *
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ */
+
+#include <linux/clk.h>
+#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
+#include <linux/platform_data/clk-st.h>
+#include <linux/platform_device.h>
+
+/* Clock Driving Strength 2 register */
+#define CLKDRVSTR2	0x28
+/* Clock Control 1 register */
+#define MISCCLKCNTL1	0x40
+/* Auxiliary clock1 enable bit */
+#define OSCCLKENB	2
+/* 25Mhz auxiliary output clock freq bit */
+#define OSCOUT1CLK25MHZ	16
+
+#define ST_CLK_48M	0
+#define ST_CLK_25M	1
+#define ST_CLK_MUX	2
+#define ST_CLK_GATE	3
+#define ST_MAX_CLKS	4
+
+static const char * const clk_oscout1_parents[] = { "clk48MHz", "clk25MHz" };
+static struct clk_hw *hws[ST_MAX_CLKS];
+
+static int st_clk_probe(struct platform_device *pdev)
+{
+	struct st_clk_data *st_data;
+
+	st_data = dev_get_platdata(&pdev->dev);
+	if (!st_data || !st_data->base)
+		return -EINVAL;
+
+	hws[ST_CLK_48M] = clk_hw_register_fixed_rate(NULL, "clk48MHz", NULL, 0,
+						     48000000);
+	hws[ST_CLK_25M] = clk_hw_register_fixed_rate(NULL, "clk25MHz", NULL, 0,
+						     25000000);
+
+	hws[ST_CLK_MUX] = clk_hw_register_mux(NULL, "oscout1_mux",
+		clk_oscout1_parents, ARRAY_SIZE(clk_oscout1_parents),
+		0, st_data->base + CLKDRVSTR2, OSCOUT1CLK25MHZ, 3, 0, NULL);
+
+	clk_set_parent(hws[ST_CLK_MUX]->clk, hws[ST_CLK_25M]->clk);
+
+	hws[ST_CLK_GATE] = clk_hw_register_gate(NULL, "oscout1", "oscout1_mux",
+		0, st_data->base + MISCCLKCNTL1, OSCCLKENB,
+		CLK_GATE_SET_TO_DISABLE, NULL);
+
+	clk_hw_register_clkdev(hws[ST_CLK_GATE], "oscout1", NULL);
+
+	return 0;
+}
+
+static int st_clk_remove(struct platform_device *pdev)
+{
+	int i;
+
+	for (i = 0; i < ST_MAX_CLKS; i++)
+		clk_hw_unregister(hws[i]);
+	return 0;
+}
+
+static struct platform_driver st_clk_driver = {
+	.driver = {
+		.name = "clk-st",
+		.suppress_bind_attrs = true,
+	},
+	.probe = st_clk_probe,
+	.remove = st_clk_remove,
+};
+builtin_platform_driver(st_clk_driver);
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index 8e8a09755d10..dec0dd88ec15 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -450,8 +450,10 @@ config MTK_TIMER
 	  Support for Mediatek timer driver.
 
 config SPRD_TIMER
-	bool "Spreadtrum timer driver" if COMPILE_TEST
+	bool "Spreadtrum timer driver" if EXPERT
 	depends on HAS_IOMEM
+	depends on (ARCH_SPRD || COMPILE_TEST)
+	default ARCH_SPRD
 	select TIMER_OF
 	help
 	  Enables support for the Spreadtrum timer driver.
diff --git a/drivers/clocksource/arc_timer.c b/drivers/clocksource/arc_timer.c
index 471b428d8034..20da9b1d7f7d 100644
--- a/drivers/clocksource/arc_timer.c
+++ b/drivers/clocksource/arc_timer.c
@@ -61,6 +61,20 @@ static u64 arc_read_gfrc(struct clocksource *cs)
 	unsigned long flags;
 	u32 l, h;
 
+	/*
+	 * From a programming model pov, there seems to be just one instance of
+	 * MCIP_CMD/MCIP_READBACK however micro-architecturally there's
+	 * an instance PER ARC CORE (not per cluster), and there are dedicated
+	 * hardware decode logic (per core) inside ARConnect to handle
+	 * simultaneous read/write accesses from cores via those two registers.
+	 * So several concurrent commands to ARConnect are OK if they are
+	 * trying to access two different sub-components (like GFRC,
+	 * inter-core interrupt, etc...). HW also supports simultaneously
+	 * accessing GFRC by multiple cores.
+	 * That's why it is safe to disable hard interrupts on the local CPU
+	 * before access to GFRC instead of taking global MCIP spinlock
+	 * defined in arch/arc/kernel/mcip.c
+	 */
 	local_irq_save(flags);
 
 	__mcip_cmd(CMD_GFRC_READ_LO, 0);
diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c
index 986b6796b631..54f8a331b53a 100644
--- a/drivers/clocksource/mips-gic-timer.c
+++ b/drivers/clocksource/mips-gic-timer.c
@@ -5,6 +5,9 @@
  *
  * Copyright (C) 2012 MIPS Technologies, Inc.  All rights reserved.
  */
+
+#define pr_fmt(fmt) "mips-gic-timer: " fmt
+
 #include <linux/clk.h>
 #include <linux/clockchips.h>
 #include <linux/cpu.h>
@@ -136,8 +139,7 @@ static int gic_clockevent_init(void)
 
 	ret = setup_percpu_irq(gic_timer_irq, &gic_compare_irqaction);
 	if (ret < 0) {
-		pr_err("GIC timer IRQ %d setup failed: %d\n",
-		       gic_timer_irq, ret);
+		pr_err("IRQ %d setup failed (%d)\n", gic_timer_irq, ret);
 		return ret;
 	}
 
@@ -176,7 +178,7 @@ static int __init __gic_clocksource_init(void)
 
 	ret = clocksource_register_hz(&gic_clocksource, gic_frequency);
 	if (ret < 0)
-		pr_warn("GIC: Unable to register clocksource\n");
+		pr_warn("Unable to register clocksource\n");
 
 	return ret;
 }
@@ -188,7 +190,7 @@ static int __init gic_clocksource_of_init(struct device_node *node)
 
 	if (!mips_gic_present() || !node->parent ||
 	    !of_device_is_compatible(node->parent, "mti,gic")) {
-		pr_warn("No DT definition for the mips gic driver\n");
+		pr_warn("No DT definition\n");
 		return -ENXIO;
 	}
 
@@ -196,7 +198,7 @@ static int __init gic_clocksource_of_init(struct device_node *node)
 	if (!IS_ERR(clk)) {
 		ret = clk_prepare_enable(clk);
 		if (ret < 0) {
-			pr_err("GIC failed to enable clock\n");
+			pr_err("Failed to enable clock\n");
 			clk_put(clk);
 			return ret;
 		}
@@ -204,12 +206,12 @@ static int __init gic_clocksource_of_init(struct device_node *node)
 		gic_frequency = clk_get_rate(clk);
 	} else if (of_property_read_u32(node, "clock-frequency",
 					&gic_frequency)) {
-		pr_err("GIC frequency not specified.\n");
+		pr_err("Frequency not specified\n");
 		return -EINVAL;
 	}
 	gic_timer_irq = irq_of_parse_and_map(node, 0);
 	if (!gic_timer_irq) {
-		pr_err("GIC timer IRQ not specified.\n");
+		pr_err("IRQ not specified\n");
 		return -EINVAL;
 	}
 
@@ -220,7 +222,7 @@ static int __init gic_clocksource_of_init(struct device_node *node)
 	ret = gic_clockevent_init();
 	if (!ret && !IS_ERR(clk)) {
 		if (clk_notifier_register(clk, &gic_clk_nb) < 0)
-			pr_warn("GIC: Unable to register clock notifier\n");
+			pr_warn("Unable to register clock notifier\n");
 	}
 
 	/* And finally start the counter */
diff --git a/drivers/clocksource/timer-sp804.c b/drivers/clocksource/timer-sp804.c
index 3ac9dec9a038..e01222ea888f 100644
--- a/drivers/clocksource/timer-sp804.c
+++ b/drivers/clocksource/timer-sp804.c
@@ -27,6 +27,7 @@
 #include <linux/io.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/of_clk.h>
 #include <linux/of_irq.h>
 #include <linux/sched_clock.h>
 
@@ -245,7 +246,7 @@ static int __init sp804_of_init(struct device_node *np)
 		clk1 = NULL;
 
 	/* Get the 2nd clock if the timer has 3 timer clocks */
-	if (of_count_phandle_with_args(np, "clocks", "#clock-cells") == 3) {
+	if (of_clk_get_parent_count(np) == 3) {
 		clk2 = of_clk_get(np, 1);
 		if (IS_ERR(clk2)) {
 			pr_err("sp804: %s clock not found: %d\n", np->name,
diff --git a/drivers/connector/cn_proc.c b/drivers/connector/cn_proc.c
index a782ce87715c..ed5e42461094 100644
--- a/drivers/connector/cn_proc.c
+++ b/drivers/connector/cn_proc.c
@@ -262,6 +262,8 @@ void proc_coredump_connector(struct task_struct *task)
 	ev->what = PROC_EVENT_COREDUMP;
 	ev->event_data.coredump.process_pid = task->pid;
 	ev->event_data.coredump.process_tgid = task->tgid;
+	ev->event_data.coredump.parent_pid = task->real_parent->pid;
+	ev->event_data.coredump.parent_tgid = task->real_parent->tgid;
 
 	memcpy(&msg->id, &cn_proc_event_id, sizeof(msg->id));
 	msg->ack = 0; /* not used */
@@ -288,6 +290,8 @@ void proc_exit_connector(struct task_struct *task)
 	ev->event_data.exit.process_tgid = task->tgid;
 	ev->event_data.exit.exit_code = task->exit_code;
 	ev->event_data.exit.exit_signal = task->exit_signal;
+	ev->event_data.exit.parent_pid = task->real_parent->pid;
+	ev->event_data.exit.parent_tgid = task->real_parent->tgid;
 
 	memcpy(&msg->id, &cn_proc_event_id, sizeof(msg->id));
 	msg->ack = 0; /* not used */
diff --git a/drivers/connector/connector.c b/drivers/connector/connector.c
index 8615594bd065..e718b8c69a56 100644
--- a/drivers/connector/connector.c
+++ b/drivers/connector/connector.c
@@ -260,19 +260,6 @@ static int cn_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int cn_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, cn_proc_show, NULL);
-}
-
-static const struct file_operations cn_file_ops = {
-	.owner   = THIS_MODULE,
-	.open    = cn_proc_open,
-	.read    = seq_read,
-	.llseek  = seq_lseek,
-	.release = single_release
-};
-
 static struct cn_dev cdev = {
 	.input   = cn_rx_skb,
 };
@@ -297,7 +284,7 @@ static int cn_init(void)
 
 	cn_already_initialized = 1;
 
-	proc_create("connector", S_IRUGO, init_net.proc_net, &cn_file_ops);
+	proc_create_single("connector", S_IRUGO, init_net.proc_net, cn_proc_show);
 
 	return 0;
 }
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index 96b35b8b3606..c7ce928fbf1f 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -124,6 +124,17 @@ config ARM_OMAP2PLUS_CPUFREQ
 	depends on ARCH_OMAP2PLUS
 	default ARCH_OMAP2PLUS
 
+config ARM_QCOM_CPUFREQ_KRYO
+	bool "Qualcomm Kryo based CPUFreq"
+	depends on ARM64
+	depends on QCOM_QFPROM
+	depends on QCOM_SMEM
+	select PM_OPP
+	help
+	  This adds the CPUFreq driver for Qualcomm Kryo SoC based boards.
+
+	  If in doubt, say N.
+
 config ARM_S3C_CPUFREQ
 	bool
 	help
@@ -264,7 +275,7 @@ config ARM_TANGO_CPUFREQ
 	default y
 
 config ARM_TEGRA20_CPUFREQ
-	bool "Tegra20 CPUFreq support"
+	tristate "Tegra20 CPUFreq support"
 	depends on ARCH_TEGRA
 	default y
 	help
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 8d24ade3bd02..fb4a2ecac43b 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -65,6 +65,7 @@ obj-$(CONFIG_MACH_MVEBU_V7)		+= mvebu-cpufreq.o
 obj-$(CONFIG_ARM_OMAP2PLUS_CPUFREQ)	+= omap-cpufreq.o
 obj-$(CONFIG_ARM_PXA2xx_CPUFREQ)	+= pxa2xx-cpufreq.o
 obj-$(CONFIG_PXA3xx)			+= pxa3xx-cpufreq.o
+obj-$(CONFIG_ARM_QCOM_CPUFREQ_KRYO)	+= qcom-cpufreq-kryo.o
 obj-$(CONFIG_ARM_S3C2410_CPUFREQ)	+= s3c2410-cpufreq.o
 obj-$(CONFIG_ARM_S3C2412_CPUFREQ)	+= s3c2412-cpufreq.o
 obj-$(CONFIG_ARM_S3C2416_CPUFREQ)	+= s3c2416-cpufreq.o
diff --git a/drivers/cpufreq/armada-37xx-cpufreq.c b/drivers/cpufreq/armada-37xx-cpufreq.c
index 72a2975499db..739da90ff3f6 100644
--- a/drivers/cpufreq/armada-37xx-cpufreq.c
+++ b/drivers/cpufreq/armada-37xx-cpufreq.c
@@ -23,6 +23,8 @@
 #include <linux/regmap.h>
 #include <linux/slab.h>
 
+#include "cpufreq-dt.h"
+
 /* Power management in North Bridge register set */
 #define ARMADA_37XX_NB_L0L1	0x18
 #define ARMADA_37XX_NB_L2L3	0x1C
@@ -56,6 +58,16 @@
  */
 #define LOAD_LEVEL_NR	4
 
+struct armada37xx_cpufreq_state {
+	struct regmap *regmap;
+	u32 nb_l0l1;
+	u32 nb_l2l3;
+	u32 nb_dyn_mod;
+	u32 nb_cpu_load;
+};
+
+static struct armada37xx_cpufreq_state *armada37xx_cpufreq_state;
+
 struct armada_37xx_dvfs {
 	u32 cpu_freq_max;
 	u8 divider[LOAD_LEVEL_NR];
@@ -136,7 +148,7 @@ static void __init armada37xx_cpufreq_dvfs_setup(struct regmap *base,
 	clk_set_parent(clk, parent);
 }
 
-static void __init armada37xx_cpufreq_disable_dvfs(struct regmap *base)
+static void armada37xx_cpufreq_disable_dvfs(struct regmap *base)
 {
 	unsigned int reg = ARMADA_37XX_NB_DYN_MOD,
 		mask = ARMADA_37XX_NB_DFS_EN;
@@ -162,10 +174,47 @@ static void __init armada37xx_cpufreq_enable_dvfs(struct regmap *base)
 	regmap_update_bits(base, reg, mask, mask);
 }
 
+static int armada37xx_cpufreq_suspend(struct cpufreq_policy *policy)
+{
+	struct armada37xx_cpufreq_state *state = armada37xx_cpufreq_state;
+
+	regmap_read(state->regmap, ARMADA_37XX_NB_L0L1, &state->nb_l0l1);
+	regmap_read(state->regmap, ARMADA_37XX_NB_L2L3, &state->nb_l2l3);
+	regmap_read(state->regmap, ARMADA_37XX_NB_CPU_LOAD,
+		    &state->nb_cpu_load);
+	regmap_read(state->regmap, ARMADA_37XX_NB_DYN_MOD, &state->nb_dyn_mod);
+
+	return 0;
+}
+
+static int armada37xx_cpufreq_resume(struct cpufreq_policy *policy)
+{
+	struct armada37xx_cpufreq_state *state = armada37xx_cpufreq_state;
+
+	/* Ensure DVFS is disabled otherwise the following registers are RO */
+	armada37xx_cpufreq_disable_dvfs(state->regmap);
+
+	regmap_write(state->regmap, ARMADA_37XX_NB_L0L1, state->nb_l0l1);
+	regmap_write(state->regmap, ARMADA_37XX_NB_L2L3, state->nb_l2l3);
+	regmap_write(state->regmap, ARMADA_37XX_NB_CPU_LOAD,
+		     state->nb_cpu_load);
+
+	/*
+	 * NB_DYN_MOD register is the one that actually enable back DVFS if it
+	 * was enabled before the suspend operation. This must be done last
+	 * otherwise other registers are not writable.
+	 */
+	regmap_write(state->regmap, ARMADA_37XX_NB_DYN_MOD, state->nb_dyn_mod);
+
+	return 0;
+}
+
 static int __init armada37xx_cpufreq_driver_init(void)
 {
+	struct cpufreq_dt_platform_data pdata;
 	struct armada_37xx_dvfs *dvfs;
 	struct platform_device *pdev;
+	unsigned long freq;
 	unsigned int cur_frequency;
 	struct regmap *nb_pm_base;
 	struct device *cpu_dev;
@@ -207,33 +256,58 @@ static int __init armada37xx_cpufreq_driver_init(void)
 	}
 
 	dvfs = armada_37xx_cpu_freq_info_get(cur_frequency);
-	if (!dvfs)
+	if (!dvfs) {
+		clk_put(clk);
 		return -EINVAL;
+	}
+
+	armada37xx_cpufreq_state = kmalloc(sizeof(*armada37xx_cpufreq_state),
+					   GFP_KERNEL);
+	if (!armada37xx_cpufreq_state) {
+		clk_put(clk);
+		return -ENOMEM;
+	}
+
+	armada37xx_cpufreq_state->regmap = nb_pm_base;
 
 	armada37xx_cpufreq_dvfs_setup(nb_pm_base, clk, dvfs->divider);
 	clk_put(clk);
 
 	for (load_lvl = ARMADA_37XX_DVFS_LOAD_0; load_lvl < LOAD_LEVEL_NR;
 	     load_lvl++) {
-		unsigned long freq = cur_frequency / dvfs->divider[load_lvl];
+		freq = cur_frequency / dvfs->divider[load_lvl];
 
 		ret = dev_pm_opp_add(cpu_dev, freq, 0);
-		if (ret) {
-			/* clean-up the already added opp before leaving */
-			while (load_lvl-- > ARMADA_37XX_DVFS_LOAD_0) {
-				freq = cur_frequency / dvfs->divider[load_lvl];
-				dev_pm_opp_remove(cpu_dev, freq);
-			}
-			return ret;
-		}
+		if (ret)
+			goto remove_opp;
 	}
 
 	/* Now that everything is setup, enable the DVFS at hardware level */
 	armada37xx_cpufreq_enable_dvfs(nb_pm_base);
 
-	pdev = platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
+	pdata.suspend = armada37xx_cpufreq_suspend;
+	pdata.resume = armada37xx_cpufreq_resume;
+
+	pdev = platform_device_register_data(NULL, "cpufreq-dt", -1, &pdata,
+					     sizeof(pdata));
+	ret = PTR_ERR_OR_ZERO(pdev);
+	if (ret)
+		goto disable_dvfs;
+
+	return 0;
+
+disable_dvfs:
+	armada37xx_cpufreq_disable_dvfs(nb_pm_base);
+remove_opp:
+	/* clean-up the already added opp before leaving */
+	while (load_lvl-- > ARMADA_37XX_DVFS_LOAD_0) {
+		freq = cur_frequency / dvfs->divider[load_lvl];
+		dev_pm_opp_remove(cpu_dev, freq);
+	}
+
+	kfree(armada37xx_cpufreq_state);
 
-	return PTR_ERR_OR_ZERO(pdev);
+	return ret;
 }
 /* late_initcall, to guarantee the driver is loaded after A37xx clock driver */
 late_initcall(armada37xx_cpufreq_driver_init);
diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index b15115a48775..3464580ac3ca 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -42,9 +42,6 @@
  */
 static struct cppc_cpudata **all_cpu_data;
 
-/* Capture the max KHz from DMI */
-static u64 cppc_dmi_max_khz;
-
 /* Callback function used to retrieve the max frequency from DMI */
 static void cppc_find_dmi_mhz(const struct dmi_header *dm, void *private)
 {
@@ -75,6 +72,64 @@ static u64 cppc_get_dmi_max_khz(void)
 	return (1000 * mhz);
 }
 
+/*
+ * If CPPC lowest_freq and nominal_freq registers are exposed then we can
+ * use them to convert perf to freq and vice versa
+ *
+ * If the perf/freq point lies between Nominal and Lowest, we can treat
+ * (Low perf, Low freq) and (Nom Perf, Nom freq) as 2D co-ordinates of a line
+ * and extrapolate the rest
+ * For perf/freq > Nominal, we use the ratio perf:freq at Nominal for conversion
+ */
+static unsigned int cppc_cpufreq_perf_to_khz(struct cppc_cpudata *cpu,
+					unsigned int perf)
+{
+	static u64 max_khz;
+	struct cppc_perf_caps *caps = &cpu->perf_caps;
+	u64 mul, div;
+
+	if (caps->lowest_freq && caps->nominal_freq) {
+		if (perf >= caps->nominal_perf) {
+			mul = caps->nominal_freq;
+			div = caps->nominal_perf;
+		} else {
+			mul = caps->nominal_freq - caps->lowest_freq;
+			div = caps->nominal_perf - caps->lowest_perf;
+		}
+	} else {
+		if (!max_khz)
+			max_khz = cppc_get_dmi_max_khz();
+		mul = max_khz;
+		div = cpu->perf_caps.highest_perf;
+	}
+	return (u64)perf * mul / div;
+}
+
+static unsigned int cppc_cpufreq_khz_to_perf(struct cppc_cpudata *cpu,
+					unsigned int freq)
+{
+	static u64 max_khz;
+	struct cppc_perf_caps *caps = &cpu->perf_caps;
+	u64  mul, div;
+
+	if (caps->lowest_freq && caps->nominal_freq) {
+		if (freq >= caps->nominal_freq) {
+			mul = caps->nominal_perf;
+			div = caps->nominal_freq;
+		} else {
+			mul = caps->lowest_perf;
+			div = caps->lowest_freq;
+		}
+	} else {
+		if (!max_khz)
+			max_khz = cppc_get_dmi_max_khz();
+		mul = cpu->perf_caps.highest_perf;
+		div = max_khz;
+	}
+
+	return (u64)freq * mul / div;
+}
+
 static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,
 		unsigned int target_freq,
 		unsigned int relation)
@@ -86,7 +141,7 @@ static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,
 
 	cpu = all_cpu_data[policy->cpu];
 
-	desired_perf = (u64)target_freq * cpu->perf_caps.highest_perf / cppc_dmi_max_khz;
+	desired_perf = cppc_cpufreq_khz_to_perf(cpu, target_freq);
 	/* Return if it is exactly the same perf */
 	if (desired_perf == cpu->perf_ctrls.desired_perf)
 		return ret;
@@ -186,24 +241,24 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
 		return ret;
 	}
 
-	cppc_dmi_max_khz = cppc_get_dmi_max_khz();
+	/* Convert the lowest and nominal freq from MHz to KHz */
+	cpu->perf_caps.lowest_freq *= 1000;
+	cpu->perf_caps.nominal_freq *= 1000;
 
 	/*
 	 * Set min to lowest nonlinear perf to avoid any efficiency penalty (see
 	 * Section 8.4.7.1.1.5 of ACPI 6.1 spec)
 	 */
-	policy->min = cpu->perf_caps.lowest_nonlinear_perf * cppc_dmi_max_khz /
-		cpu->perf_caps.highest_perf;
-	policy->max = cppc_dmi_max_khz;
+	policy->min = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.lowest_nonlinear_perf);
+	policy->max = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.highest_perf);
 
 	/*
 	 * Set cpuinfo.min_freq to Lowest to make the full range of performance
 	 * available if userspace wants to use any perf between lowest & lowest
 	 * nonlinear perf
 	 */
-	policy->cpuinfo.min_freq = cpu->perf_caps.lowest_perf * cppc_dmi_max_khz /
-		cpu->perf_caps.highest_perf;
-	policy->cpuinfo.max_freq = cppc_dmi_max_khz;
+	policy->cpuinfo.min_freq = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.lowest_perf);
+	policy->cpuinfo.max_freq = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.highest_perf);
 
 	policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu_num);
 	policy->shared_type = cpu->shared_type;
@@ -229,7 +284,8 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
 	cpu->cur_policy = policy;
 
 	/* Set policy->cur to max now. The governors will adjust later. */
-	policy->cur = cppc_dmi_max_khz;
+	policy->cur = cppc_cpufreq_perf_to_khz(cpu,
+					cpu->perf_caps.highest_perf);
 	cpu->perf_ctrls.desired_perf = cpu->perf_caps.highest_perf;
 
 	ret = cppc_set_perf(cpu_num, &cpu->perf_ctrls);
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index 3b585e4bfac5..fe14c57de6ca 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -66,8 +66,6 @@ static const struct of_device_id whitelist[] __initconst = {
 	{ .compatible = "renesas,r8a7792", },
 	{ .compatible = "renesas,r8a7793", },
 	{ .compatible = "renesas,r8a7794", },
-	{ .compatible = "renesas,r8a7795", },
-	{ .compatible = "renesas,r8a7796", },
 	{ .compatible = "renesas,sh73a0", },
 
 	{ .compatible = "rockchip,rk2928", },
@@ -118,6 +116,9 @@ static const struct of_device_id blacklist[] __initconst = {
 
 	{ .compatible = "nvidia,tegra124", },
 
+	{ .compatible = "qcom,apq8096", },
+	{ .compatible = "qcom,msm8996", },
+
 	{ .compatible = "st,stih407", },
 	{ .compatible = "st,stih410", },
 
diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c
index 190ea0dccb79..0a9ebf00be46 100644
--- a/drivers/cpufreq/cpufreq-dt.c
+++ b/drivers/cpufreq/cpufreq-dt.c
@@ -346,8 +346,14 @@ static int dt_cpufreq_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	if (data && data->have_governor_per_policy)
-		dt_cpufreq_driver.flags |= CPUFREQ_HAVE_GOVERNOR_PER_POLICY;
+	if (data) {
+		if (data->have_governor_per_policy)
+			dt_cpufreq_driver.flags |= CPUFREQ_HAVE_GOVERNOR_PER_POLICY;
+
+		dt_cpufreq_driver.resume = data->resume;
+		if (data->suspend)
+			dt_cpufreq_driver.suspend = data->suspend;
+	}
 
 	ret = cpufreq_register_driver(&dt_cpufreq_driver);
 	if (ret)
diff --git a/drivers/cpufreq/cpufreq-dt.h b/drivers/cpufreq/cpufreq-dt.h
index 54d774e46c43..d5aeea13433e 100644
--- a/drivers/cpufreq/cpufreq-dt.h
+++ b/drivers/cpufreq/cpufreq-dt.h
@@ -12,8 +12,13 @@
 
 #include <linux/types.h>
 
+struct cpufreq_policy;
+
 struct cpufreq_dt_platform_data {
 	bool have_governor_per_policy;
+
+	int (*suspend)(struct cpufreq_policy *policy);
+	int (*resume)(struct cpufreq_policy *policy);
 };
 
 #endif /* __CPUFREQ_DT_H__ */
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 075d18f6ba7a..b0dfd3222013 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -89,16 +89,7 @@ static void cpufreq_governor_limits(struct cpufreq_policy *policy);
  * The mutex locks both lists.
  */
 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
-static struct srcu_notifier_head cpufreq_transition_notifier_list;
-
-static bool init_cpufreq_transition_notifier_list_called;
-static int __init init_cpufreq_transition_notifier_list(void)
-{
-	srcu_init_notifier_head(&cpufreq_transition_notifier_list);
-	init_cpufreq_transition_notifier_list_called = true;
-	return 0;
-}
-pure_initcall(init_cpufreq_transition_notifier_list);
+SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
 
 static int off __read_mostly;
 static int cpufreq_disabled(void)
@@ -300,8 +291,19 @@ static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 #endif
 }
 
-static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
-		struct cpufreq_freqs *freqs, unsigned int state)
+/**
+ * cpufreq_notify_transition - Notify frequency transition and adjust_jiffies.
+ * @policy: cpufreq policy to enable fast frequency switching for.
+ * @freqs: contain details of the frequency update.
+ * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
+ *
+ * This function calls the transition notifiers and the "adjust_jiffies"
+ * function. It is called twice on all CPU frequency changes that have
+ * external effects.
+ */
+static void cpufreq_notify_transition(struct cpufreq_policy *policy,
+				      struct cpufreq_freqs *freqs,
+				      unsigned int state)
 {
 	BUG_ON(irqs_disabled());
 
@@ -313,54 +315,44 @@ static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 		 state, freqs->new);
 
 	switch (state) {
-
 	case CPUFREQ_PRECHANGE:
-		/* detect if the driver reported a value as "old frequency"
+		/*
+		 * Detect if the driver reported a value as "old frequency"
 		 * which is not equal to what the cpufreq core thinks is
 		 * "old frequency".
 		 */
 		if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
-			if ((policy) && (policy->cpu == freqs->cpu) &&
-			    (policy->cur) && (policy->cur != freqs->old)) {
+			if (policy->cur && (policy->cur != freqs->old)) {
 				pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 					 freqs->old, policy->cur);
 				freqs->old = policy->cur;
 			}
 		}
-		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
-				CPUFREQ_PRECHANGE, freqs);
+
+		for_each_cpu(freqs->cpu, policy->cpus) {
+			srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
+						 CPUFREQ_PRECHANGE, freqs);
+		}
+
 		adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 		break;
 
 	case CPUFREQ_POSTCHANGE:
 		adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
-		pr_debug("FREQ: %lu - CPU: %lu\n",
-			 (unsigned long)freqs->new, (unsigned long)freqs->cpu);
-		trace_cpu_frequency(freqs->new, freqs->cpu);
+		pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
+			 cpumask_pr_args(policy->cpus));
+
+		for_each_cpu(freqs->cpu, policy->cpus) {
+			trace_cpu_frequency(freqs->new, freqs->cpu);
+			srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
+						 CPUFREQ_POSTCHANGE, freqs);
+		}
+
 		cpufreq_stats_record_transition(policy, freqs->new);
-		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
-				CPUFREQ_POSTCHANGE, freqs);
-		if (likely(policy) && likely(policy->cpu == freqs->cpu))
-			policy->cur = freqs->new;
-		break;
+		policy->cur = freqs->new;
 	}
 }
 
-/**
- * cpufreq_notify_transition - call notifier chain and adjust_jiffies
- * on frequency transition.
- *
- * This function calls the transition notifiers and the "adjust_jiffies"
- * function. It is called twice on all CPU frequency changes that have
- * external effects.
- */
-static void cpufreq_notify_transition(struct cpufreq_policy *policy,
-		struct cpufreq_freqs *freqs, unsigned int state)
-{
-	for_each_cpu(freqs->cpu, policy->cpus)
-		__cpufreq_notify_transition(policy, freqs, state);
-}
-
 /* Do post notifications when there are chances that transition has failed */
 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 		struct cpufreq_freqs *freqs, int transition_failed)
@@ -696,6 +688,8 @@ static ssize_t store_##file_name					\
 	struct cpufreq_policy new_policy;				\
 									\
 	memcpy(&new_policy, policy, sizeof(*policy));			\
+	new_policy.min = policy->user_policy.min;			\
+	new_policy.max = policy->user_policy.max;			\
 									\
 	ret = sscanf(buf, "%u", &new_policy.object);			\
 	if (ret != 1)							\
@@ -1764,8 +1758,6 @@ int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
 	if (cpufreq_disabled())
 		return -EINVAL;
 
-	WARN_ON(!init_cpufreq_transition_notifier_list_called);
-
 	switch (list) {
 	case CPUFREQ_TRANSITION_NOTIFIER:
 		mutex_lock(&cpufreq_fast_switch_lock);
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index ca38229b045a..871bf9cf55cf 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -278,7 +278,7 @@ static void dbs_update_util_handler(struct update_util_data *data, u64 time,
 	struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;
 	u64 delta_ns, lst;
 
-	if (!cpufreq_can_do_remote_dvfs(policy_dbs->policy))
+	if (!cpufreq_this_cpu_can_update(policy_dbs->policy))
 		return;
 
 	/*
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 17e566afbb41..08960a55eb27 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -1939,13 +1939,51 @@ static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy)
 	return 0;
 }
 
+/* Use of trace in passive mode:
+ *
+ * In passive mode the trace core_busy field (also known as the
+ * performance field, and lablelled as such on the graphs; also known as
+ * core_avg_perf) is not needed and so is re-assigned to indicate if the
+ * driver call was via the normal or fast switch path. Various graphs
+ * output from the intel_pstate_tracer.py utility that include core_busy
+ * (or performance or core_avg_perf) have a fixed y-axis from 0 to 100%,
+ * so we use 10 to indicate the the normal path through the driver, and
+ * 90 to indicate the fast switch path through the driver.
+ * The scaled_busy field is not used, and is set to 0.
+ */
+
+#define	INTEL_PSTATE_TRACE_TARGET 10
+#define	INTEL_PSTATE_TRACE_FAST_SWITCH 90
+
+static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, int old_pstate)
+{
+	struct sample *sample;
+
+	if (!trace_pstate_sample_enabled())
+		return;
+
+	if (!intel_pstate_sample(cpu, ktime_get()))
+		return;
+
+	sample = &cpu->sample;
+	trace_pstate_sample(trace_type,
+		0,
+		old_pstate,
+		cpu->pstate.current_pstate,
+		sample->mperf,
+		sample->aperf,
+		sample->tsc,
+		get_avg_frequency(cpu),
+		fp_toint(cpu->iowait_boost * 100));
+}
+
 static int intel_cpufreq_target(struct cpufreq_policy *policy,
 				unsigned int target_freq,
 				unsigned int relation)
 {
 	struct cpudata *cpu = all_cpu_data[policy->cpu];
 	struct cpufreq_freqs freqs;
-	int target_pstate;
+	int target_pstate, old_pstate;
 
 	update_turbo_state();
 
@@ -1965,12 +2003,14 @@ static int intel_cpufreq_target(struct cpufreq_policy *policy,
 		break;
 	}
 	target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
+	old_pstate = cpu->pstate.current_pstate;
 	if (target_pstate != cpu->pstate.current_pstate) {
 		cpu->pstate.current_pstate = target_pstate;
 		wrmsrl_on_cpu(policy->cpu, MSR_IA32_PERF_CTL,
 			      pstate_funcs.get_val(cpu, target_pstate));
 	}
 	freqs.new = target_pstate * cpu->pstate.scaling;
+	intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_TARGET, old_pstate);
 	cpufreq_freq_transition_end(policy, &freqs, false);
 
 	return 0;
@@ -1980,13 +2020,15 @@ static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
 					      unsigned int target_freq)
 {
 	struct cpudata *cpu = all_cpu_data[policy->cpu];
-	int target_pstate;
+	int target_pstate, old_pstate;
 
 	update_turbo_state();
 
 	target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling);
 	target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
+	old_pstate = cpu->pstate.current_pstate;
 	intel_pstate_update_pstate(cpu, target_pstate);
+	intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_FAST_SWITCH, old_pstate);
 	return target_pstate * cpu->pstate.scaling;
 }
 
diff --git a/drivers/cpufreq/qcom-cpufreq-kryo.c b/drivers/cpufreq/qcom-cpufreq-kryo.c
new file mode 100644
index 000000000000..d049fe4b80c4
--- /dev/null
+++ b/drivers/cpufreq/qcom-cpufreq-kryo.c
@@ -0,0 +1,212 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ */
+
+/*
+ * In Certain QCOM SoCs like apq8096 and msm8996 that have KRYO processors,
+ * the CPU frequency subset and voltage value of each OPP varies
+ * based on the silicon variant in use. Qualcomm Process Voltage Scaling Tables
+ * defines the voltage and frequency value based on the msm-id in SMEM
+ * and speedbin blown in the efuse combination.
+ * The qcom-cpufreq-kryo driver reads the msm-id and efuse value from the SoC
+ * to provide the OPP framework with required information.
+ * This is used to determine the voltage and frequency value for each OPP of
+ * operating-points-v2 table when it is parsed by the OPP framework.
+ */
+
+#include <linux/cpu.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+#include <linux/soc/qcom/smem.h>
+
+#define MSM_ID_SMEM	137
+
+enum _msm_id {
+	MSM8996V3 = 0xF6ul,
+	APQ8096V3 = 0x123ul,
+	MSM8996SG = 0x131ul,
+	APQ8096SG = 0x138ul,
+};
+
+enum _msm8996_version {
+	MSM8996_V3,
+	MSM8996_SG,
+	NUM_OF_MSM8996_VERSIONS,
+};
+
+static enum _msm8996_version __init qcom_cpufreq_kryo_get_msm_id(void)
+{
+	size_t len;
+	u32 *msm_id;
+	enum _msm8996_version version;
+
+	msm_id = qcom_smem_get(QCOM_SMEM_HOST_ANY, MSM_ID_SMEM, &len);
+	if (IS_ERR(msm_id))
+		return NUM_OF_MSM8996_VERSIONS;
+
+	/* The first 4 bytes are format, next to them is the actual msm-id */
+	msm_id++;
+
+	switch ((enum _msm_id)*msm_id) {
+	case MSM8996V3:
+	case APQ8096V3:
+		version = MSM8996_V3;
+		break;
+	case MSM8996SG:
+	case APQ8096SG:
+		version = MSM8996_SG;
+		break;
+	default:
+		version = NUM_OF_MSM8996_VERSIONS;
+	}
+
+	return version;
+}
+
+static int qcom_cpufreq_kryo_probe(struct platform_device *pdev)
+{
+	struct opp_table *opp_tables[NR_CPUS] = {0};
+	struct platform_device *cpufreq_dt_pdev;
+	enum _msm8996_version msm8996_version;
+	struct nvmem_cell *speedbin_nvmem;
+	struct device_node *np;
+	struct device *cpu_dev;
+	unsigned cpu;
+	u8 *speedbin;
+	u32 versions;
+	size_t len;
+	int ret;
+
+	cpu_dev = get_cpu_device(0);
+	if (NULL == cpu_dev)
+		ret = -ENODEV;
+
+	msm8996_version = qcom_cpufreq_kryo_get_msm_id();
+	if (NUM_OF_MSM8996_VERSIONS == msm8996_version) {
+		dev_err(cpu_dev, "Not Snapdragon 820/821!");
+		return -ENODEV;
+	}
+
+	np = dev_pm_opp_of_get_opp_desc_node(cpu_dev);
+	if (IS_ERR(np))
+		return PTR_ERR(np);
+
+	ret = of_device_is_compatible(np, "operating-points-v2-kryo-cpu");
+	if (!ret) {
+		of_node_put(np);
+		return -ENOENT;
+	}
+
+	speedbin_nvmem = of_nvmem_cell_get(np, NULL);
+	of_node_put(np);
+	if (IS_ERR(speedbin_nvmem)) {
+		dev_err(cpu_dev, "Could not get nvmem cell: %ld\n",
+			PTR_ERR(speedbin_nvmem));
+		return PTR_ERR(speedbin_nvmem);
+	}
+
+	speedbin = nvmem_cell_read(speedbin_nvmem, &len);
+	nvmem_cell_put(speedbin_nvmem);
+
+	switch (msm8996_version) {
+	case MSM8996_V3:
+		versions = 1 << (unsigned int)(*speedbin);
+		break;
+	case MSM8996_SG:
+		versions = 1 << ((unsigned int)(*speedbin) + 4);
+		break;
+	default:
+		BUG();
+		break;
+	}
+
+	for_each_possible_cpu(cpu) {
+		cpu_dev = get_cpu_device(cpu);
+		if (NULL == cpu_dev) {
+			ret = -ENODEV;
+			goto free_opp;
+		}
+
+		opp_tables[cpu] = dev_pm_opp_set_supported_hw(cpu_dev,
+							      &versions, 1);
+		if (IS_ERR(opp_tables[cpu])) {
+			ret = PTR_ERR(opp_tables[cpu]);
+			dev_err(cpu_dev, "Failed to set supported hardware\n");
+			goto free_opp;
+		}
+	}
+
+	cpufreq_dt_pdev = platform_device_register_simple("cpufreq-dt", -1,
+							  NULL, 0);
+	if (!IS_ERR(cpufreq_dt_pdev))
+		return 0;
+
+	ret = PTR_ERR(cpufreq_dt_pdev);
+	dev_err(cpu_dev, "Failed to register platform device\n");
+
+free_opp:
+	for_each_possible_cpu(cpu) {
+		if (IS_ERR_OR_NULL(opp_tables[cpu]))
+			break;
+		dev_pm_opp_put_supported_hw(opp_tables[cpu]);
+	}
+
+	return ret;
+}
+
+static struct platform_driver qcom_cpufreq_kryo_driver = {
+	.probe = qcom_cpufreq_kryo_probe,
+	.driver = {
+		.name = "qcom-cpufreq-kryo",
+	},
+};
+
+static const struct of_device_id qcom_cpufreq_kryo_match_list[] __initconst = {
+	{ .compatible = "qcom,apq8096", },
+	{ .compatible = "qcom,msm8996", },
+};
+
+/*
+ * Since the driver depends on smem and nvmem drivers, which may
+ * return EPROBE_DEFER, all the real activity is done in the probe,
+ * which may be defered as well. The init here is only registering
+ * the driver and the platform device.
+ */
+static int __init qcom_cpufreq_kryo_init(void)
+{
+	struct device_node *np = of_find_node_by_path("/");
+	const struct of_device_id *match;
+	int ret;
+
+	if (!np)
+		return -ENODEV;
+
+	match = of_match_node(qcom_cpufreq_kryo_match_list, np);
+	of_node_put(np);
+	if (!match)
+		return -ENODEV;
+
+	ret = platform_driver_register(&qcom_cpufreq_kryo_driver);
+	if (unlikely(ret < 0))
+		return ret;
+
+	ret = PTR_ERR_OR_ZERO(platform_device_register_simple(
+		"qcom-cpufreq-kryo", -1, NULL, 0));
+	if (0 == ret)
+		return 0;
+
+	platform_driver_unregister(&qcom_cpufreq_kryo_driver);
+	return ret;
+}
+module_init(qcom_cpufreq_kryo_init);
+
+MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Kryo CPUfreq driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/s3c2440-cpufreq.c b/drivers/cpufreq/s3c2440-cpufreq.c
index d0d75b65ddd6..d2f67b7a20dd 100644
--- a/drivers/cpufreq/s3c2440-cpufreq.c
+++ b/drivers/cpufreq/s3c2440-cpufreq.c
@@ -143,7 +143,7 @@ static void s3c2440_cpufreq_setdivs(struct s3c_cpufreq_config *cfg)
 {
 	unsigned long clkdiv, camdiv;
 
-	s3c_freq_dbg("%s: divsiors: h=%d, p=%d\n", __func__,
+	s3c_freq_dbg("%s: divisors: h=%d, p=%d\n", __func__,
 		     cfg->divs.h_divisor, cfg->divs.p_divisor);
 
 	clkdiv = __raw_readl(S3C2410_CLKDIVN);
diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c
index b4dbc77459b6..50b1551ba894 100644
--- a/drivers/cpufreq/scmi-cpufreq.c
+++ b/drivers/cpufreq/scmi-cpufreq.c
@@ -117,7 +117,7 @@ static int scmi_cpufreq_init(struct cpufreq_policy *policy)
 		return -ENODEV;
 	}
 
-	ret = handle->perf_ops->add_opps_to_device(handle, cpu_dev);
+	ret = handle->perf_ops->device_opps_add(handle, cpu_dev);
 	if (ret) {
 		dev_warn(cpu_dev, "failed to add opps to the device\n");
 		return ret;
@@ -164,7 +164,7 @@ static int scmi_cpufreq_init(struct cpufreq_policy *policy)
 	/* SCMI allows DVFS request for any domain from any CPU */
 	policy->dvfs_possible_from_any_cpu = true;
 
-	latency = handle->perf_ops->get_transition_latency(handle, cpu_dev);
+	latency = handle->perf_ops->transition_latency_get(handle, cpu_dev);
 	if (!latency)
 		latency = CPUFREQ_ETERNAL;
 
diff --git a/drivers/cpufreq/speedstep-lib.c b/drivers/cpufreq/speedstep-lib.c
index e3a9962ee410..cabb6f48eb77 100644
--- a/drivers/cpufreq/speedstep-lib.c
+++ b/drivers/cpufreq/speedstep-lib.c
@@ -252,7 +252,7 @@ EXPORT_SYMBOL_GPL(speedstep_get_frequency);
  *********************************************************************/
 
 /* Keep in sync with the x86_cpu_id tables in the different modules */
-unsigned int speedstep_detect_processor(void)
+enum speedstep_processor speedstep_detect_processor(void)
 {
 	struct cpuinfo_x86 *c = &cpu_data(0);
 	u32 ebx, msr_lo, msr_hi;
diff --git a/drivers/cpufreq/tegra20-cpufreq.c b/drivers/cpufreq/tegra20-cpufreq.c
index 2bd62845e9d5..05f57dcd5215 100644
--- a/drivers/cpufreq/tegra20-cpufreq.c
+++ b/drivers/cpufreq/tegra20-cpufreq.c
@@ -16,16 +16,13 @@
  *
  */
 
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/sched.h>
+#include <linux/clk.h>
 #include <linux/cpufreq.h>
-#include <linux/delay.h>
-#include <linux/init.h>
 #include <linux/err.h>
-#include <linux/clk.h>
-#include <linux/io.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
 
 static struct cpufreq_frequency_table freq_table[] = {
 	{ .frequency = 216000 },
@@ -39,25 +36,27 @@ static struct cpufreq_frequency_table freq_table[] = {
 	{ .frequency = CPUFREQ_TABLE_END },
 };
 
-#define NUM_CPUS	2
-
-static struct clk *cpu_clk;
-static struct clk *pll_x_clk;
-static struct clk *pll_p_clk;
-static struct clk *emc_clk;
-static bool pll_x_prepared;
+struct tegra20_cpufreq {
+	struct device *dev;
+	struct cpufreq_driver driver;
+	struct clk *cpu_clk;
+	struct clk *pll_x_clk;
+	struct clk *pll_p_clk;
+	bool pll_x_prepared;
+};
 
 static unsigned int tegra_get_intermediate(struct cpufreq_policy *policy,
 					   unsigned int index)
 {
-	unsigned int ifreq = clk_get_rate(pll_p_clk) / 1000;
+	struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data();
+	unsigned int ifreq = clk_get_rate(cpufreq->pll_p_clk) / 1000;
 
 	/*
 	 * Don't switch to intermediate freq if:
 	 * - we are already at it, i.e. policy->cur == ifreq
 	 * - index corresponds to ifreq
 	 */
-	if ((freq_table[index].frequency == ifreq) || (policy->cur == ifreq))
+	if (freq_table[index].frequency == ifreq || policy->cur == ifreq)
 		return 0;
 
 	return ifreq;
@@ -66,6 +65,7 @@ static unsigned int tegra_get_intermediate(struct cpufreq_policy *policy,
 static int tegra_target_intermediate(struct cpufreq_policy *policy,
 				     unsigned int index)
 {
+	struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data();
 	int ret;
 
 	/*
@@ -78,47 +78,37 @@ static int tegra_target_intermediate(struct cpufreq_policy *policy,
 	 * Also, we wouldn't be using pll_x anymore and must not take extra
 	 * reference to it, as it can be disabled now to save some power.
 	 */
-	clk_prepare_enable(pll_x_clk);
+	clk_prepare_enable(cpufreq->pll_x_clk);
 
-	ret = clk_set_parent(cpu_clk, pll_p_clk);
+	ret = clk_set_parent(cpufreq->cpu_clk, cpufreq->pll_p_clk);
 	if (ret)
-		clk_disable_unprepare(pll_x_clk);
+		clk_disable_unprepare(cpufreq->pll_x_clk);
 	else
-		pll_x_prepared = true;
+		cpufreq->pll_x_prepared = true;
 
 	return ret;
 }
 
 static int tegra_target(struct cpufreq_policy *policy, unsigned int index)
 {
+	struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data();
 	unsigned long rate = freq_table[index].frequency;
-	unsigned int ifreq = clk_get_rate(pll_p_clk) / 1000;
-	int ret = 0;
-
-	/*
-	 * Vote on memory bus frequency based on cpu frequency
-	 * This sets the minimum frequency, display or avp may request higher
-	 */
-	if (rate >= 816000)
-		clk_set_rate(emc_clk, 600000000); /* cpu 816 MHz, emc max */
-	else if (rate >= 456000)
-		clk_set_rate(emc_clk, 300000000); /* cpu 456 MHz, emc 150Mhz */
-	else
-		clk_set_rate(emc_clk, 100000000);  /* emc 50Mhz */
+	unsigned int ifreq = clk_get_rate(cpufreq->pll_p_clk) / 1000;
+	int ret;
 
 	/*
 	 * target freq == pll_p, don't need to take extra reference to pll_x_clk
 	 * as it isn't used anymore.
 	 */
 	if (rate == ifreq)
-		return clk_set_parent(cpu_clk, pll_p_clk);
+		return clk_set_parent(cpufreq->cpu_clk, cpufreq->pll_p_clk);
 
-	ret = clk_set_rate(pll_x_clk, rate * 1000);
+	ret = clk_set_rate(cpufreq->pll_x_clk, rate * 1000);
 	/* Restore to earlier frequency on error, i.e. pll_x */
 	if (ret)
-		pr_err("Failed to change pll_x to %lu\n", rate);
+		dev_err(cpufreq->dev, "Failed to change pll_x to %lu\n", rate);
 
-	ret = clk_set_parent(cpu_clk, pll_x_clk);
+	ret = clk_set_parent(cpufreq->cpu_clk, cpufreq->pll_x_clk);
 	/* This shouldn't fail while changing or restoring */
 	WARN_ON(ret);
 
@@ -126,9 +116,9 @@ static int tegra_target(struct cpufreq_policy *policy, unsigned int index)
 	 * Drop count to pll_x clock only if we switched to intermediate freq
 	 * earlier while transitioning to a target frequency.
 	 */
-	if (pll_x_prepared) {
-		clk_disable_unprepare(pll_x_clk);
-		pll_x_prepared = false;
+	if (cpufreq->pll_x_prepared) {
+		clk_disable_unprepare(cpufreq->pll_x_clk);
+		cpufreq->pll_x_prepared = false;
 	}
 
 	return ret;
@@ -136,81 +126,111 @@ static int tegra_target(struct cpufreq_policy *policy, unsigned int index)
 
 static int tegra_cpu_init(struct cpufreq_policy *policy)
 {
+	struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data();
 	int ret;
 
-	if (policy->cpu >= NUM_CPUS)
-		return -EINVAL;
-
-	clk_prepare_enable(emc_clk);
-	clk_prepare_enable(cpu_clk);
+	clk_prepare_enable(cpufreq->cpu_clk);
 
 	/* FIXME: what's the actual transition time? */
 	ret = cpufreq_generic_init(policy, freq_table, 300 * 1000);
 	if (ret) {
-		clk_disable_unprepare(cpu_clk);
-		clk_disable_unprepare(emc_clk);
+		clk_disable_unprepare(cpufreq->cpu_clk);
 		return ret;
 	}
 
-	policy->clk = cpu_clk;
+	policy->clk = cpufreq->cpu_clk;
 	policy->suspend_freq = freq_table[0].frequency;
 	return 0;
 }
 
 static int tegra_cpu_exit(struct cpufreq_policy *policy)
 {
-	clk_disable_unprepare(cpu_clk);
-	clk_disable_unprepare(emc_clk);
+	struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data();
+
+	clk_disable_unprepare(cpufreq->cpu_clk);
 	return 0;
 }
 
-static struct cpufreq_driver tegra_cpufreq_driver = {
-	.flags			= CPUFREQ_NEED_INITIAL_FREQ_CHECK,
-	.verify			= cpufreq_generic_frequency_table_verify,
-	.get_intermediate	= tegra_get_intermediate,
-	.target_intermediate	= tegra_target_intermediate,
-	.target_index		= tegra_target,
-	.get			= cpufreq_generic_get,
-	.init			= tegra_cpu_init,
-	.exit			= tegra_cpu_exit,
-	.name			= "tegra",
-	.attr			= cpufreq_generic_attr,
-	.suspend		= cpufreq_generic_suspend,
-};
-
-static int __init tegra_cpufreq_init(void)
+static int tegra20_cpufreq_probe(struct platform_device *pdev)
 {
-	cpu_clk = clk_get_sys(NULL, "cclk");
-	if (IS_ERR(cpu_clk))
-		return PTR_ERR(cpu_clk);
-
-	pll_x_clk = clk_get_sys(NULL, "pll_x");
-	if (IS_ERR(pll_x_clk))
-		return PTR_ERR(pll_x_clk);
-
-	pll_p_clk = clk_get_sys(NULL, "pll_p");
-	if (IS_ERR(pll_p_clk))
-		return PTR_ERR(pll_p_clk);
-
-	emc_clk = clk_get_sys("cpu", "emc");
-	if (IS_ERR(emc_clk)) {
-		clk_put(cpu_clk);
-		return PTR_ERR(emc_clk);
+	struct tegra20_cpufreq *cpufreq;
+	int err;
+
+	cpufreq = devm_kzalloc(&pdev->dev, sizeof(*cpufreq), GFP_KERNEL);
+	if (!cpufreq)
+		return -ENOMEM;
+
+	cpufreq->cpu_clk = clk_get_sys(NULL, "cclk");
+	if (IS_ERR(cpufreq->cpu_clk))
+		return PTR_ERR(cpufreq->cpu_clk);
+
+	cpufreq->pll_x_clk = clk_get_sys(NULL, "pll_x");
+	if (IS_ERR(cpufreq->pll_x_clk)) {
+		err = PTR_ERR(cpufreq->pll_x_clk);
+		goto put_cpu;
+	}
+
+	cpufreq->pll_p_clk = clk_get_sys(NULL, "pll_p");
+	if (IS_ERR(cpufreq->pll_p_clk)) {
+		err = PTR_ERR(cpufreq->pll_p_clk);
+		goto put_pll_x;
 	}
 
-	return cpufreq_register_driver(&tegra_cpufreq_driver);
+	cpufreq->dev = &pdev->dev;
+	cpufreq->driver.get = cpufreq_generic_get;
+	cpufreq->driver.attr = cpufreq_generic_attr;
+	cpufreq->driver.init = tegra_cpu_init;
+	cpufreq->driver.exit = tegra_cpu_exit;
+	cpufreq->driver.flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK;
+	cpufreq->driver.verify = cpufreq_generic_frequency_table_verify;
+	cpufreq->driver.suspend = cpufreq_generic_suspend;
+	cpufreq->driver.driver_data = cpufreq;
+	cpufreq->driver.target_index = tegra_target;
+	cpufreq->driver.get_intermediate = tegra_get_intermediate;
+	cpufreq->driver.target_intermediate = tegra_target_intermediate;
+	snprintf(cpufreq->driver.name, CPUFREQ_NAME_LEN, "tegra");
+
+	err = cpufreq_register_driver(&cpufreq->driver);
+	if (err)
+		goto put_pll_p;
+
+	platform_set_drvdata(pdev, cpufreq);
+
+	return 0;
+
+put_pll_p:
+	clk_put(cpufreq->pll_p_clk);
+put_pll_x:
+	clk_put(cpufreq->pll_x_clk);
+put_cpu:
+	clk_put(cpufreq->cpu_clk);
+
+	return err;
 }
 
-static void __exit tegra_cpufreq_exit(void)
+static int tegra20_cpufreq_remove(struct platform_device *pdev)
 {
-        cpufreq_unregister_driver(&tegra_cpufreq_driver);
-	clk_put(emc_clk);
-	clk_put(cpu_clk);
+	struct tegra20_cpufreq *cpufreq = platform_get_drvdata(pdev);
+
+	cpufreq_unregister_driver(&cpufreq->driver);
+
+	clk_put(cpufreq->pll_p_clk);
+	clk_put(cpufreq->pll_x_clk);
+	clk_put(cpufreq->cpu_clk);
+
+	return 0;
 }
 
+static struct platform_driver tegra20_cpufreq_driver = {
+	.probe		= tegra20_cpufreq_probe,
+	.remove		= tegra20_cpufreq_remove,
+	.driver		= {
+		.name	= "tegra20-cpufreq",
+	},
+};
+module_platform_driver(tegra20_cpufreq_driver);
 
+MODULE_ALIAS("platform:tegra20-cpufreq");
 MODULE_AUTHOR("Colin Cross <ccross@android.com>");
-MODULE_DESCRIPTION("cpufreq driver for Nvidia Tegra2");
+MODULE_DESCRIPTION("NVIDIA Tegra20 cpufreq driver");
 MODULE_LICENSE("GPL");
-module_init(tegra_cpufreq_init);
-module_exit(tegra_cpufreq_exit);
diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c
index 5d359aff3cc5..9fed1b829292 100644
--- a/drivers/cpuidle/governor.c
+++ b/drivers/cpuidle/governor.c
@@ -8,8 +8,10 @@
  * This code is licenced under the GPL.
  */
 
-#include <linux/mutex.h>
+#include <linux/cpu.h>
 #include <linux/cpuidle.h>
+#include <linux/mutex.h>
+#include <linux/pm_qos.h>
 
 #include "cpuidle.h"
 
@@ -93,3 +95,16 @@ int cpuidle_register_governor(struct cpuidle_governor *gov)
 
 	return ret;
 }
+
+/**
+ * cpuidle_governor_latency_req - Compute a latency constraint for CPU
+ * @cpu: Target CPU
+ */
+int cpuidle_governor_latency_req(unsigned int cpu)
+{
+	int global_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY);
+	struct device *device = get_cpu_device(cpu);
+	int device_req = dev_pm_qos_raw_read_value(device);
+
+	return device_req < global_req ? device_req : global_req;
+}
diff --git a/drivers/cpuidle/governors/ladder.c b/drivers/cpuidle/governors/ladder.c
index b24883f85c99..704880a6612a 100644
--- a/drivers/cpuidle/governors/ladder.c
+++ b/drivers/cpuidle/governors/ladder.c
@@ -14,10 +14,8 @@
 
 #include <linux/kernel.h>
 #include <linux/cpuidle.h>
-#include <linux/pm_qos.h>
 #include <linux/jiffies.h>
 #include <linux/tick.h>
-#include <linux/cpu.h>
 
 #include <asm/io.h>
 #include <linux/uaccess.h>
@@ -69,16 +67,10 @@ static int ladder_select_state(struct cpuidle_driver *drv,
 			       struct cpuidle_device *dev, bool *dummy)
 {
 	struct ladder_device *ldev = this_cpu_ptr(&ladder_devices);
-	struct device *device = get_cpu_device(dev->cpu);
 	struct ladder_device_state *last_state;
 	int last_residency, last_idx = ldev->last_state_idx;
 	int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0;
-	int latency_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY);
-	int resume_latency = dev_pm_qos_raw_read_value(device);
-
-	if (resume_latency < latency_req &&
-	    resume_latency != PM_QOS_RESUME_LATENCY_NO_CONSTRAINT)
-		latency_req = resume_latency;
+	int latency_req = cpuidle_governor_latency_req(dev->cpu);
 
 	/* Special case when user has set very strict latency requirement */
 	if (unlikely(latency_req == 0)) {
diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 1bfe03ceb236..1aef60d160eb 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -12,7 +12,6 @@
 
 #include <linux/kernel.h>
 #include <linux/cpuidle.h>
-#include <linux/pm_qos.h>
 #include <linux/time.h>
 #include <linux/ktime.h>
 #include <linux/hrtimer.h>
@@ -21,7 +20,6 @@
 #include <linux/sched/loadavg.h>
 #include <linux/sched/stat.h>
 #include <linux/math64.h>
-#include <linux/cpu.h>
 
 /*
  * Please note when changing the tuning values:
@@ -286,15 +284,13 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 		       bool *stop_tick)
 {
 	struct menu_device *data = this_cpu_ptr(&menu_devices);
-	struct device *device = get_cpu_device(dev->cpu);
-	int latency_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY);
+	int latency_req = cpuidle_governor_latency_req(dev->cpu);
 	int i;
 	int first_idx;
 	int idx;
 	unsigned int interactivity_req;
 	unsigned int expected_interval;
 	unsigned long nr_iowaiters, cpu_load;
-	int resume_latency = dev_pm_qos_raw_read_value(device);
 	ktime_t delta_next;
 
 	if (data->needs_update) {
@@ -302,10 +298,6 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 		data->needs_update = 0;
 	}
 
-	if (resume_latency < latency_req &&
-	    resume_latency != PM_QOS_RESUME_LATENCY_NO_CONSTRAINT)
-		latency_req = resume_latency;
-
 	/* Special case when user has set very strict latency requirement */
 	if (unlikely(latency_req == 0)) {
 		*stop_tick = false;
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index d1ea1a07cecb..43cccf6aff61 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -302,6 +302,7 @@ config CRYPTO_DEV_PPC4XX
 	select CRYPTO_AEAD
 	select CRYPTO_AES
 	select CRYPTO_CCM
+	select CRYPTO_CTR
 	select CRYPTO_GCM
 	select CRYPTO_BLKCIPHER
 	help
@@ -419,7 +420,7 @@ config CRYPTO_DEV_EXYNOS_RNG
 config CRYPTO_DEV_S5P
 	tristate "Support for Samsung S5PV210/Exynos crypto accelerator"
 	depends on ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST
-	depends on HAS_IOMEM && HAS_DMA
+	depends on HAS_IOMEM
 	select CRYPTO_AES
 	select CRYPTO_BLKCIPHER
 	help
@@ -466,7 +467,6 @@ endif # if CRYPTO_DEV_UX500
 
 config CRYPTO_DEV_ATMEL_AUTHENC
 	tristate "Support for Atmel IPSEC/SSL hw accelerator"
-	depends on HAS_DMA
 	depends on ARCH_AT91 || COMPILE_TEST
 	select CRYPTO_AUTHENC
 	select CRYPTO_DEV_ATMEL_AES
@@ -479,7 +479,6 @@ config CRYPTO_DEV_ATMEL_AUTHENC
 
 config CRYPTO_DEV_ATMEL_AES
 	tristate "Support for Atmel AES hw accelerator"
-	depends on HAS_DMA
 	depends on ARCH_AT91 || COMPILE_TEST
 	select CRYPTO_AES
 	select CRYPTO_AEAD
@@ -494,7 +493,6 @@ config CRYPTO_DEV_ATMEL_AES
 
 config CRYPTO_DEV_ATMEL_TDES
 	tristate "Support for Atmel DES/TDES hw accelerator"
-	depends on HAS_DMA
 	depends on ARCH_AT91 || COMPILE_TEST
 	select CRYPTO_DES
 	select CRYPTO_BLKCIPHER
@@ -508,7 +506,6 @@ config CRYPTO_DEV_ATMEL_TDES
 
 config CRYPTO_DEV_ATMEL_SHA
 	tristate "Support for Atmel SHA hw accelerator"
-	depends on HAS_DMA
 	depends on ARCH_AT91 || COMPILE_TEST
 	select CRYPTO_HASH
 	help
@@ -574,7 +571,8 @@ config CRYPTO_DEV_CAVIUM_ZIP
 
 config CRYPTO_DEV_QCE
 	tristate "Qualcomm crypto engine accelerator"
-	depends on (ARCH_QCOM || COMPILE_TEST) && HAS_DMA && HAS_IOMEM
+	depends on ARCH_QCOM || COMPILE_TEST
+	depends on HAS_IOMEM
 	select CRYPTO_AES
 	select CRYPTO_DES
 	select CRYPTO_ECB
@@ -598,7 +596,6 @@ source "drivers/crypto/vmx/Kconfig"
 config CRYPTO_DEV_IMGTEC_HASH
 	tristate "Imagination Technologies hardware hash accelerator"
 	depends on MIPS || COMPILE_TEST
-	depends on HAS_DMA
 	select CRYPTO_MD5
 	select CRYPTO_SHA1
 	select CRYPTO_SHA256
@@ -650,7 +647,6 @@ config CRYPTO_DEV_ROCKCHIP
 
 config CRYPTO_DEV_MEDIATEK
 	tristate "MediaTek's EIP97 Cryptographic Engine driver"
-	depends on HAS_DMA
 	depends on (ARM && ARCH_MEDIATEK) || COMPILE_TEST
 	select CRYPTO_AES
 	select CRYPTO_AEAD
@@ -688,9 +684,10 @@ source "drivers/crypto/stm32/Kconfig"
 
 config CRYPTO_DEV_SAFEXCEL
 	tristate "Inside Secure's SafeXcel cryptographic engine driver"
-	depends on HAS_DMA && OF
+	depends on OF
 	depends on (ARM64 && ARCH_MVEBU) || (COMPILE_TEST && 64BIT)
 	select CRYPTO_AES
+	select CRYPTO_AUTHENC
 	select CRYPTO_BLKCIPHER
 	select CRYPTO_HASH
 	select CRYPTO_HMAC
@@ -706,7 +703,6 @@ config CRYPTO_DEV_SAFEXCEL
 config CRYPTO_DEV_ARTPEC6
 	tristate "Support for Axis ARTPEC-6/7 hardware crypto acceleration."
 	depends on ARM && (ARCH_ARTPEC || COMPILE_TEST)
-	depends on HAS_DMA
 	depends on OF
 	select CRYPTO_AEAD
 	select CRYPTO_AES
diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c
index ea83d0bff0e9..f5c07498ea4f 100644
--- a/drivers/crypto/amcc/crypto4xx_alg.c
+++ b/drivers/crypto/amcc/crypto4xx_alg.c
@@ -31,6 +31,7 @@
 #include <crypto/gcm.h>
 #include <crypto/sha.h>
 #include <crypto/ctr.h>
+#include <crypto/skcipher.h>
 #include "crypto4xx_reg_def.h"
 #include "crypto4xx_core.h"
 #include "crypto4xx_sa.h"
@@ -74,51 +75,57 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm,
 	sa->sa_command_1.bf.copy_hdr = cp_hdr;
 }
 
-int crypto4xx_encrypt(struct ablkcipher_request *req)
+static inline int crypto4xx_crypt(struct skcipher_request *req,
+				  const unsigned int ivlen, bool decrypt)
 {
-	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
-	unsigned int ivlen = crypto_ablkcipher_ivsize(
-		crypto_ablkcipher_reqtfm(req));
-	__le32 iv[ivlen];
+	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
+	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
+	__le32 iv[AES_IV_SIZE];
 
 	if (ivlen)
-		crypto4xx_memcpy_to_le32(iv, req->info, ivlen);
+		crypto4xx_memcpy_to_le32(iv, req->iv, ivlen);
 
 	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
-		req->nbytes, iv, ivlen, ctx->sa_out, ctx->sa_len, 0);
+		req->cryptlen, iv, ivlen, decrypt ? ctx->sa_in : ctx->sa_out,
+		ctx->sa_len, 0, NULL);
 }
 
-int crypto4xx_decrypt(struct ablkcipher_request *req)
+int crypto4xx_encrypt_noiv(struct skcipher_request *req)
 {
-	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
-	unsigned int ivlen = crypto_ablkcipher_ivsize(
-		crypto_ablkcipher_reqtfm(req));
-	__le32 iv[ivlen];
+	return crypto4xx_crypt(req, 0, false);
+}
 
-	if (ivlen)
-		crypto4xx_memcpy_to_le32(iv, req->info, ivlen);
+int crypto4xx_encrypt_iv(struct skcipher_request *req)
+{
+	return crypto4xx_crypt(req, AES_IV_SIZE, false);
+}
 
-	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
-		req->nbytes, iv, ivlen, ctx->sa_in, ctx->sa_len, 0);
+int crypto4xx_decrypt_noiv(struct skcipher_request *req)
+{
+	return crypto4xx_crypt(req, 0, true);
+}
+
+int crypto4xx_decrypt_iv(struct skcipher_request *req)
+{
+	return crypto4xx_crypt(req, AES_IV_SIZE, true);
 }
 
 /**
  * AES Functions
  */
-static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher,
+static int crypto4xx_setkey_aes(struct crypto_skcipher *cipher,
 				const u8 *key,
 				unsigned int keylen,
 				unsigned char cm,
 				u8 fb)
 {
-	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
-	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
 	struct dynamic_sa_ctl *sa;
 	int    rc;
 
 	if (keylen != AES_KEYSIZE_256 &&
 		keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_128) {
-		crypto_ablkcipher_set_flags(cipher,
+		crypto_skcipher_set_flags(cipher,
 				CRYPTO_TFM_RES_BAD_KEY_LEN);
 		return -EINVAL;
 	}
@@ -134,7 +141,8 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher,
 	/* Setup SA */
 	sa = ctx->sa_in;
 
-	set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV,
+	set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, (cm == CRYPTO_MODE_CBC ?
+				 SA_SAVE_IV : SA_NOT_SAVE_IV),
 				 SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE,
 				 SA_NO_HEADER_PROC, SA_HASH_ALG_NULL,
 				 SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO,
@@ -158,39 +166,38 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher,
 	return 0;
 }
 
-int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher,
+int crypto4xx_setkey_aes_cbc(struct crypto_skcipher *cipher,
 			     const u8 *key, unsigned int keylen)
 {
 	return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CBC,
 				    CRYPTO_FEEDBACK_MODE_NO_FB);
 }
 
-int crypto4xx_setkey_aes_cfb(struct crypto_ablkcipher *cipher,
+int crypto4xx_setkey_aes_cfb(struct crypto_skcipher *cipher,
 			     const u8 *key, unsigned int keylen)
 {
 	return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CFB,
 				    CRYPTO_FEEDBACK_MODE_128BIT_CFB);
 }
 
-int crypto4xx_setkey_aes_ecb(struct crypto_ablkcipher *cipher,
+int crypto4xx_setkey_aes_ecb(struct crypto_skcipher *cipher,
 			     const u8 *key, unsigned int keylen)
 {
 	return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_ECB,
 				    CRYPTO_FEEDBACK_MODE_NO_FB);
 }
 
-int crypto4xx_setkey_aes_ofb(struct crypto_ablkcipher *cipher,
+int crypto4xx_setkey_aes_ofb(struct crypto_skcipher *cipher,
 			     const u8 *key, unsigned int keylen)
 {
 	return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_OFB,
 				    CRYPTO_FEEDBACK_MODE_64BIT_OFB);
 }
 
-int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher,
+int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher,
 			     const u8 *key, unsigned int keylen)
 {
-	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
-	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
 	int rc;
 
 	rc = crypto4xx_setkey_aes(cipher, key, keylen - CTR_RFC3686_NONCE_SIZE,
@@ -204,35 +211,117 @@ int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher,
 	return 0;
 }
 
-int crypto4xx_rfc3686_encrypt(struct ablkcipher_request *req)
+int crypto4xx_rfc3686_encrypt(struct skcipher_request *req)
 {
-	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
+	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
 	__le32 iv[AES_IV_SIZE / 4] = {
 		ctx->iv_nonce,
-		cpu_to_le32p((u32 *) req->info),
-		cpu_to_le32p((u32 *) (req->info + 4)),
+		cpu_to_le32p((u32 *) req->iv),
+		cpu_to_le32p((u32 *) (req->iv + 4)),
 		cpu_to_le32(1) };
 
 	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
-				  req->nbytes, iv, AES_IV_SIZE,
-				  ctx->sa_out, ctx->sa_len, 0);
+				  req->cryptlen, iv, AES_IV_SIZE,
+				  ctx->sa_out, ctx->sa_len, 0, NULL);
 }
 
-int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req)
+int crypto4xx_rfc3686_decrypt(struct skcipher_request *req)
 {
-	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
+	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
 	__le32 iv[AES_IV_SIZE / 4] = {
 		ctx->iv_nonce,
-		cpu_to_le32p((u32 *) req->info),
-		cpu_to_le32p((u32 *) (req->info + 4)),
+		cpu_to_le32p((u32 *) req->iv),
+		cpu_to_le32p((u32 *) (req->iv + 4)),
 		cpu_to_le32(1) };
 
 	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
-				  req->nbytes, iv, AES_IV_SIZE,
-				  ctx->sa_out, ctx->sa_len, 0);
+				  req->cryptlen, iv, AES_IV_SIZE,
+				  ctx->sa_out, ctx->sa_len, 0, NULL);
+}
+
+static int
+crypto4xx_ctr_crypt(struct skcipher_request *req, bool encrypt)
+{
+	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
+	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
+	size_t iv_len = crypto_skcipher_ivsize(cipher);
+	unsigned int counter = be32_to_cpup((__be32 *)(req->iv + iv_len - 4));
+	unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) /
+			AES_BLOCK_SIZE;
+
+	/*
+	 * The hardware uses only the last 32-bits as the counter while the
+	 * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that
+	 * the whole IV is a counter.  So fallback if the counter is going to
+	 * overlow.
+	 */
+	if (counter + nblks < counter) {
+		struct skcipher_request *subreq = skcipher_request_ctx(req);
+		int ret;
+
+		skcipher_request_set_tfm(subreq, ctx->sw_cipher.cipher);
+		skcipher_request_set_callback(subreq, req->base.flags,
+			NULL, NULL);
+		skcipher_request_set_crypt(subreq, req->src, req->dst,
+			req->cryptlen, req->iv);
+		ret = encrypt ? crypto_skcipher_encrypt(subreq)
+			: crypto_skcipher_decrypt(subreq);
+		skcipher_request_zero(subreq);
+		return ret;
+	}
+
+	return encrypt ? crypto4xx_encrypt_iv(req)
+		       : crypto4xx_decrypt_iv(req);
+}
+
+static int crypto4xx_sk_setup_fallback(struct crypto4xx_ctx *ctx,
+				       struct crypto_skcipher *cipher,
+				       const u8 *key,
+				       unsigned int keylen)
+{
+	int rc;
+
+	crypto_skcipher_clear_flags(ctx->sw_cipher.cipher,
+				    CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_set_flags(ctx->sw_cipher.cipher,
+		crypto_skcipher_get_flags(cipher) & CRYPTO_TFM_REQ_MASK);
+	rc = crypto_skcipher_setkey(ctx->sw_cipher.cipher, key, keylen);
+	crypto_skcipher_clear_flags(cipher, CRYPTO_TFM_RES_MASK);
+	crypto_skcipher_set_flags(cipher,
+		crypto_skcipher_get_flags(ctx->sw_cipher.cipher) &
+			CRYPTO_TFM_RES_MASK);
+
+	return rc;
+}
+
+int crypto4xx_setkey_aes_ctr(struct crypto_skcipher *cipher,
+			     const u8 *key, unsigned int keylen)
+{
+	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
+	int rc;
+
+	rc = crypto4xx_sk_setup_fallback(ctx, cipher, key, keylen);
+	if (rc)
+		return rc;
+
+	return crypto4xx_setkey_aes(cipher, key, keylen,
+		CRYPTO_MODE_CTR, CRYPTO_FEEDBACK_MODE_NO_FB);
+}
+
+int crypto4xx_encrypt_ctr(struct skcipher_request *req)
+{
+	return crypto4xx_ctr_crypt(req, true);
+}
+
+int crypto4xx_decrypt_ctr(struct skcipher_request *req)
+{
+	return crypto4xx_ctr_crypt(req, false);
 }
 
 static inline bool crypto4xx_aead_need_fallback(struct aead_request *req,
+						unsigned int len,
 						bool is_ccm, bool decrypt)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
@@ -242,14 +331,14 @@ static inline bool crypto4xx_aead_need_fallback(struct aead_request *req,
 		return true;
 
 	/*
-	 * hardware does not handle cases where cryptlen
-	 * is less than a block
+	 * hardware does not handle cases where plaintext
+	 * is less than a block.
 	 */
-	if (req->cryptlen < AES_BLOCK_SIZE)
+	if (len < AES_BLOCK_SIZE)
 		return true;
 
-	/* assoc len needs to be a multiple of 4 */
-	if (req->assoclen & 0x3)
+	/* assoc len needs to be a multiple of 4 and <= 1020 */
+	if (req->assoclen & 0x3 || req->assoclen > 1020)
 		return true;
 
 	/* CCM supports only counter field length of 2 and 4 bytes */
@@ -262,13 +351,7 @@ static inline bool crypto4xx_aead_need_fallback(struct aead_request *req,
 static int crypto4xx_aead_fallback(struct aead_request *req,
 	struct crypto4xx_ctx *ctx, bool do_decrypt)
 {
-	char aead_req_data[sizeof(struct aead_request) +
-			   crypto_aead_reqsize(ctx->sw_cipher.aead)]
-		__aligned(__alignof__(struct aead_request));
-
-	struct aead_request *subreq = (void *) aead_req_data;
-
-	memset(subreq, 0, sizeof(aead_req_data));
+	struct aead_request *subreq = aead_request_ctx(req);
 
 	aead_request_set_tfm(subreq, ctx->sw_cipher.aead);
 	aead_request_set_callback(subreq, req->base.flags,
@@ -280,10 +363,10 @@ static int crypto4xx_aead_fallback(struct aead_request *req,
 			    crypto_aead_encrypt(subreq);
 }
 
-static int crypto4xx_setup_fallback(struct crypto4xx_ctx *ctx,
-				    struct crypto_aead *cipher,
-				    const u8 *key,
-				    unsigned int keylen)
+static int crypto4xx_aead_setup_fallback(struct crypto4xx_ctx *ctx,
+					 struct crypto_aead *cipher,
+					 const u8 *key,
+					 unsigned int keylen)
 {
 	int rc;
 
@@ -311,7 +394,7 @@ int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, const u8 *key,
 	struct dynamic_sa_ctl *sa;
 	int rc = 0;
 
-	rc = crypto4xx_setup_fallback(ctx, cipher, key, keylen);
+	rc = crypto4xx_aead_setup_fallback(ctx, cipher, key, keylen);
 	if (rc)
 		return rc;
 
@@ -366,19 +449,20 @@ int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, const u8 *key,
 static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt)
 {
 	struct crypto4xx_ctx *ctx  = crypto_tfm_ctx(req->base.tfm);
+	struct crypto4xx_aead_reqctx *rctx = aead_request_ctx(req);
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
-	unsigned int len = req->cryptlen;
 	__le32 iv[16];
-	u32 tmp_sa[ctx->sa_len * 4];
+	u32 tmp_sa[SA_AES128_CCM_LEN + 4];
 	struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *)tmp_sa;
-
-	if (crypto4xx_aead_need_fallback(req, true, decrypt))
-		return crypto4xx_aead_fallback(req, ctx, decrypt);
+	unsigned int len = req->cryptlen;
 
 	if (decrypt)
 		len -= crypto_aead_authsize(aead);
 
-	memcpy(tmp_sa, decrypt ? ctx->sa_in : ctx->sa_out, sizeof(tmp_sa));
+	if (crypto4xx_aead_need_fallback(req, len, true, decrypt))
+		return crypto4xx_aead_fallback(req, ctx, decrypt);
+
+	memcpy(tmp_sa, decrypt ? ctx->sa_in : ctx->sa_out, ctx->sa_len * 4);
 	sa->sa_command_0.bf.digest_len = crypto_aead_authsize(aead) >> 2;
 
 	if (req->iv[0] == 1) {
@@ -391,7 +475,7 @@ static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt)
 
 	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
 				  len, iv, sizeof(iv),
-				  sa, ctx->sa_len, req->assoclen);
+				  sa, ctx->sa_len, req->assoclen, rctx->dst);
 }
 
 int crypto4xx_encrypt_aes_ccm(struct aead_request *req)
@@ -470,7 +554,7 @@ int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher,
 		return -EINVAL;
 	}
 
-	rc = crypto4xx_setup_fallback(ctx, cipher, key, keylen);
+	rc = crypto4xx_aead_setup_fallback(ctx, cipher, key, keylen);
 	if (rc)
 		return rc;
 
@@ -523,22 +607,23 @@ static inline int crypto4xx_crypt_aes_gcm(struct aead_request *req,
 					  bool decrypt)
 {
 	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
-	unsigned int len = req->cryptlen;
+	struct crypto4xx_aead_reqctx *rctx = aead_request_ctx(req);
 	__le32 iv[4];
+	unsigned int len = req->cryptlen;
+
+	if (decrypt)
+		len -= crypto_aead_authsize(crypto_aead_reqtfm(req));
 
-	if (crypto4xx_aead_need_fallback(req, false, decrypt))
+	if (crypto4xx_aead_need_fallback(req, len, false, decrypt))
 		return crypto4xx_aead_fallback(req, ctx, decrypt);
 
 	crypto4xx_memcpy_to_le32(iv, req->iv, GCM_AES_IV_SIZE);
 	iv[3] = cpu_to_le32(1);
 
-	if (decrypt)
-		len -= crypto_aead_authsize(crypto_aead_reqtfm(req));
-
 	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
 				  len, iv, sizeof(iv),
 				  decrypt ? ctx->sa_in : ctx->sa_out,
-				  ctx->sa_len, req->assoclen);
+				  ctx->sa_len, req->assoclen, rctx->dst);
 }
 
 int crypto4xx_encrypt_aes_gcm(struct aead_request *req)
@@ -623,7 +708,7 @@ int crypto4xx_hash_update(struct ahash_request *req)
 
 	return crypto4xx_build_pd(&req->base, ctx, req->src, &dst,
 				  req->nbytes, NULL, 0, ctx->sa_in,
-				  ctx->sa_len, 0);
+				  ctx->sa_len, 0, NULL);
 }
 
 int crypto4xx_hash_final(struct ahash_request *req)
@@ -642,7 +727,7 @@ int crypto4xx_hash_digest(struct ahash_request *req)
 
 	return crypto4xx_build_pd(&req->base, ctx, req->src, &dst,
 				  req->nbytes, NULL, 0, ctx->sa_in,
-				  ctx->sa_len, 0);
+				  ctx->sa_len, 0, NULL);
 }
 
 /**
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index 76f459ad2821..9cb234c72549 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -41,6 +41,7 @@
 #include <crypto/gcm.h>
 #include <crypto/sha.h>
 #include <crypto/scatterwalk.h>
+#include <crypto/skcipher.h>
 #include <crypto/internal/aead.h>
 #include <crypto/internal/skcipher.h>
 #include "crypto4xx_reg_def.h"
@@ -526,31 +527,38 @@ static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev,
 	}
 }
 
-static void crypto4xx_ablkcipher_done(struct crypto4xx_device *dev,
+static void crypto4xx_cipher_done(struct crypto4xx_device *dev,
 				     struct pd_uinfo *pd_uinfo,
 				     struct ce_pd *pd)
 {
-	struct crypto4xx_ctx *ctx;
-	struct ablkcipher_request *ablk_req;
+	struct skcipher_request *req;
 	struct scatterlist *dst;
 	dma_addr_t addr;
 
-	ablk_req = ablkcipher_request_cast(pd_uinfo->async_req);
-	ctx  = crypto_tfm_ctx(ablk_req->base.tfm);
+	req = skcipher_request_cast(pd_uinfo->async_req);
 
 	if (pd_uinfo->using_sd) {
-		crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, ablk_req->nbytes,
-					  ablk_req->dst);
+		crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo,
+					  req->cryptlen, req->dst);
 	} else {
 		dst = pd_uinfo->dest_va;
 		addr = dma_map_page(dev->core_dev->device, sg_page(dst),
 				    dst->offset, dst->length, DMA_FROM_DEVICE);
 	}
+
+	if (pd_uinfo->sa_va->sa_command_0.bf.save_iv == SA_SAVE_IV) {
+		struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+
+		crypto4xx_memcpy_from_le32((u32 *)req->iv,
+			pd_uinfo->sr_va->save_iv,
+			crypto_skcipher_ivsize(skcipher));
+	}
+
 	crypto4xx_ret_sg_desc(dev, pd_uinfo);
 
 	if (pd_uinfo->state & PD_ENTRY_BUSY)
-		ablkcipher_request_complete(ablk_req, -EINPROGRESS);
-	ablkcipher_request_complete(ablk_req, 0);
+		skcipher_request_complete(req, -EINPROGRESS);
+	skcipher_request_complete(req, 0);
 }
 
 static void crypto4xx_ahash_done(struct crypto4xx_device *dev,
@@ -580,7 +588,7 @@ static void crypto4xx_aead_done(struct crypto4xx_device *dev,
 	struct scatterlist *dst = pd_uinfo->dest_va;
 	size_t cp_len = crypto_aead_authsize(
 		crypto_aead_reqtfm(aead_req));
-	u32 icv[cp_len];
+	u32 icv[AES_BLOCK_SIZE];
 	int err = 0;
 
 	if (pd_uinfo->using_sd) {
@@ -595,7 +603,7 @@ static void crypto4xx_aead_done(struct crypto4xx_device *dev,
 	if (pd_uinfo->sa_va->sa_command_0.bf.dir == DIR_OUTBOUND) {
 		/* append icv at the end */
 		crypto4xx_memcpy_from_le32(icv, pd_uinfo->sr_va->save_digest,
-					   cp_len);
+					   sizeof(icv));
 
 		scatterwalk_map_and_copy(icv, dst, aead_req->cryptlen,
 					 cp_len, 1);
@@ -605,7 +613,7 @@ static void crypto4xx_aead_done(struct crypto4xx_device *dev,
 			aead_req->assoclen + aead_req->cryptlen -
 			cp_len, cp_len, 0);
 
-		crypto4xx_memcpy_from_le32(icv, icv, cp_len);
+		crypto4xx_memcpy_from_le32(icv, icv, sizeof(icv));
 
 		if (crypto_memneq(icv, pd_uinfo->sr_va->save_digest, cp_len))
 			err = -EBADMSG;
@@ -641,8 +649,8 @@ static void crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx)
 	struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx];
 
 	switch (crypto_tfm_alg_type(pd_uinfo->async_req->tfm)) {
-	case CRYPTO_ALG_TYPE_ABLKCIPHER:
-		crypto4xx_ablkcipher_done(dev, pd_uinfo, pd);
+	case CRYPTO_ALG_TYPE_SKCIPHER:
+		crypto4xx_cipher_done(dev, pd_uinfo, pd);
 		break;
 	case CRYPTO_ALG_TYPE_AEAD:
 		crypto4xx_aead_done(dev, pd_uinfo, pd);
@@ -687,9 +695,9 @@ int crypto4xx_build_pd(struct crypto_async_request *req,
 		       const __le32 *iv, const u32 iv_len,
 		       const struct dynamic_sa_ctl *req_sa,
 		       const unsigned int sa_len,
-		       const unsigned int assoclen)
+		       const unsigned int assoclen,
+		       struct scatterlist *_dst)
 {
-	struct scatterlist _dst[2];
 	struct crypto4xx_device *dev = ctx->dev;
 	struct dynamic_sa_ctl *sa;
 	struct ce_gd *gd;
@@ -936,15 +944,27 @@ static void crypto4xx_ctx_init(struct crypto4xx_alg *amcc_alg,
 	ctx->sa_len = 0;
 }
 
-static int crypto4xx_ablk_init(struct crypto_tfm *tfm)
+static int crypto4xx_sk_init(struct crypto_skcipher *sk)
 {
-	struct crypto_alg *alg = tfm->__crt_alg;
+	struct skcipher_alg *alg = crypto_skcipher_alg(sk);
 	struct crypto4xx_alg *amcc_alg;
-	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto4xx_ctx *ctx =  crypto_skcipher_ctx(sk);
+
+	if (alg->base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
+		ctx->sw_cipher.cipher =
+			crypto_alloc_skcipher(alg->base.cra_name, 0,
+					      CRYPTO_ALG_NEED_FALLBACK |
+					      CRYPTO_ALG_ASYNC);
+		if (IS_ERR(ctx->sw_cipher.cipher))
+			return PTR_ERR(ctx->sw_cipher.cipher);
+
+		crypto_skcipher_set_reqsize(sk,
+			sizeof(struct skcipher_request) + 32 +
+			crypto_skcipher_reqsize(ctx->sw_cipher.cipher));
+	}
 
 	amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.cipher);
 	crypto4xx_ctx_init(amcc_alg, ctx);
-	tfm->crt_ablkcipher.reqsize = sizeof(struct crypto4xx_ctx);
 	return 0;
 }
 
@@ -953,9 +973,13 @@ static void crypto4xx_common_exit(struct crypto4xx_ctx *ctx)
 	crypto4xx_free_sa(ctx);
 }
 
-static void crypto4xx_ablk_exit(struct crypto_tfm *tfm)
+static void crypto4xx_sk_exit(struct crypto_skcipher *sk)
 {
-	crypto4xx_common_exit(crypto_tfm_ctx(tfm));
+	struct crypto4xx_ctx *ctx =  crypto_skcipher_ctx(sk);
+
+	crypto4xx_common_exit(ctx);
+	if (ctx->sw_cipher.cipher)
+		crypto_free_skcipher(ctx->sw_cipher.cipher);
 }
 
 static int crypto4xx_aead_init(struct crypto_aead *tfm)
@@ -972,9 +996,9 @@ static int crypto4xx_aead_init(struct crypto_aead *tfm)
 
 	amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.aead);
 	crypto4xx_ctx_init(amcc_alg, ctx);
-	crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
-				max(sizeof(struct crypto4xx_ctx), 32 +
-				crypto_aead_reqsize(ctx->sw_cipher.aead)));
+	crypto_aead_set_reqsize(tfm, max(sizeof(struct aead_request) + 32 +
+				crypto_aead_reqsize(ctx->sw_cipher.aead),
+				sizeof(struct crypto4xx_aead_reqctx)));
 	return 0;
 }
 
@@ -1012,7 +1036,7 @@ static int crypto4xx_register_alg(struct crypto4xx_device *sec_dev,
 			break;
 
 		default:
-			rc = crypto_register_alg(&alg->alg.u.cipher);
+			rc = crypto_register_skcipher(&alg->alg.u.cipher);
 			break;
 		}
 
@@ -1041,7 +1065,7 @@ static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev)
 			break;
 
 		default:
-			crypto_unregister_alg(&alg->alg.u.cipher);
+			crypto_unregister_skcipher(&alg->alg.u.cipher);
 		}
 		kfree(alg);
 	}
@@ -1103,126 +1127,131 @@ static irqreturn_t crypto4xx_ce_interrupt_handler_revb(int irq, void *data)
  */
 static struct crypto4xx_alg_common crypto4xx_alg[] = {
 	/* Crypto AES modes */
-	{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = {
-		.cra_name 	= "cbc(aes)",
-		.cra_driver_name = "cbc-aes-ppc4xx",
-		.cra_priority 	= CRYPTO4XX_CRYPTO_PRIORITY,
-		.cra_flags	= CRYPTO_ALG_TYPE_ABLKCIPHER |
-				  CRYPTO_ALG_ASYNC |
-				  CRYPTO_ALG_KERN_DRIVER_ONLY,
-		.cra_blocksize 	= AES_BLOCK_SIZE,
-		.cra_ctxsize 	= sizeof(struct crypto4xx_ctx),
-		.cra_type 	= &crypto_ablkcipher_type,
-		.cra_init	= crypto4xx_ablk_init,
-		.cra_exit	= crypto4xx_ablk_exit,
-		.cra_module 	= THIS_MODULE,
-		.cra_u 		= {
-			.ablkcipher = {
-				.min_keysize 	= AES_MIN_KEY_SIZE,
-				.max_keysize 	= AES_MAX_KEY_SIZE,
-				.ivsize		= AES_IV_SIZE,
-				.setkey 	= crypto4xx_setkey_aes_cbc,
-				.encrypt 	= crypto4xx_encrypt,
-				.decrypt 	= crypto4xx_decrypt,
-			}
-		}
-	}},
-	{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = {
-		.cra_name	= "cfb(aes)",
-		.cra_driver_name = "cfb-aes-ppc4xx",
-		.cra_priority	= CRYPTO4XX_CRYPTO_PRIORITY,
-		.cra_flags	= CRYPTO_ALG_TYPE_ABLKCIPHER |
-				  CRYPTO_ALG_ASYNC |
-				  CRYPTO_ALG_KERN_DRIVER_ONLY,
-		.cra_blocksize	= AES_BLOCK_SIZE,
-		.cra_ctxsize	= sizeof(struct crypto4xx_ctx),
-		.cra_type	= &crypto_ablkcipher_type,
-		.cra_init	= crypto4xx_ablk_init,
-		.cra_exit	= crypto4xx_ablk_exit,
-		.cra_module	= THIS_MODULE,
-		.cra_u		= {
-			.ablkcipher = {
-				.min_keysize	= AES_MIN_KEY_SIZE,
-				.max_keysize	= AES_MAX_KEY_SIZE,
-				.ivsize		= AES_IV_SIZE,
-				.setkey		= crypto4xx_setkey_aes_cfb,
-				.encrypt	= crypto4xx_encrypt,
-				.decrypt	= crypto4xx_decrypt,
-			}
-		}
+	{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = {
+		.base = {
+			.cra_name = "cbc(aes)",
+			.cra_driver_name = "cbc-aes-ppc4xx",
+			.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
+			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+				CRYPTO_ALG_ASYNC |
+				CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct crypto4xx_ctx),
+			.cra_module = THIS_MODULE,
+		},
+		.min_keysize = AES_MIN_KEY_SIZE,
+		.max_keysize = AES_MAX_KEY_SIZE,
+		.ivsize	= AES_IV_SIZE,
+		.setkey = crypto4xx_setkey_aes_cbc,
+		.encrypt = crypto4xx_encrypt_iv,
+		.decrypt = crypto4xx_decrypt_iv,
+		.init = crypto4xx_sk_init,
+		.exit = crypto4xx_sk_exit,
 	} },
-	{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = {
-		.cra_name	= "rfc3686(ctr(aes))",
-		.cra_driver_name = "rfc3686-ctr-aes-ppc4xx",
-		.cra_priority	= CRYPTO4XX_CRYPTO_PRIORITY,
-		.cra_flags	= CRYPTO_ALG_TYPE_ABLKCIPHER |
-				  CRYPTO_ALG_ASYNC |
-				  CRYPTO_ALG_KERN_DRIVER_ONLY,
-		.cra_blocksize	= AES_BLOCK_SIZE,
-		.cra_ctxsize	= sizeof(struct crypto4xx_ctx),
-		.cra_type	= &crypto_ablkcipher_type,
-		.cra_init	= crypto4xx_ablk_init,
-		.cra_exit	= crypto4xx_ablk_exit,
-		.cra_module	= THIS_MODULE,
-		.cra_u		= {
-			.ablkcipher = {
-				.min_keysize	= AES_MIN_KEY_SIZE +
-						  CTR_RFC3686_NONCE_SIZE,
-				.max_keysize	= AES_MAX_KEY_SIZE +
-						  CTR_RFC3686_NONCE_SIZE,
-				.ivsize		= CTR_RFC3686_IV_SIZE,
-				.setkey		= crypto4xx_setkey_rfc3686,
-				.encrypt	= crypto4xx_rfc3686_encrypt,
-				.decrypt	= crypto4xx_rfc3686_decrypt,
-			}
-		}
+	{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = {
+		.base = {
+			.cra_name = "cfb(aes)",
+			.cra_driver_name = "cfb-aes-ppc4xx",
+			.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
+			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+				CRYPTO_ALG_ASYNC |
+				CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct crypto4xx_ctx),
+			.cra_module = THIS_MODULE,
+		},
+		.min_keysize = AES_MIN_KEY_SIZE,
+		.max_keysize = AES_MAX_KEY_SIZE,
+		.ivsize	= AES_IV_SIZE,
+		.setkey	= crypto4xx_setkey_aes_cfb,
+		.encrypt = crypto4xx_encrypt_iv,
+		.decrypt = crypto4xx_decrypt_iv,
+		.init = crypto4xx_sk_init,
+		.exit = crypto4xx_sk_exit,
 	} },
-	{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = {
-		.cra_name	= "ecb(aes)",
-		.cra_driver_name = "ecb-aes-ppc4xx",
-		.cra_priority	= CRYPTO4XX_CRYPTO_PRIORITY,
-		.cra_flags	= CRYPTO_ALG_TYPE_ABLKCIPHER |
-				  CRYPTO_ALG_ASYNC |
-				  CRYPTO_ALG_KERN_DRIVER_ONLY,
-		.cra_blocksize	= AES_BLOCK_SIZE,
-		.cra_ctxsize	= sizeof(struct crypto4xx_ctx),
-		.cra_type	= &crypto_ablkcipher_type,
-		.cra_init	= crypto4xx_ablk_init,
-		.cra_exit	= crypto4xx_ablk_exit,
-		.cra_module	= THIS_MODULE,
-		.cra_u		= {
-			.ablkcipher = {
-				.min_keysize	= AES_MIN_KEY_SIZE,
-				.max_keysize	= AES_MAX_KEY_SIZE,
-				.setkey		= crypto4xx_setkey_aes_ecb,
-				.encrypt	= crypto4xx_encrypt,
-				.decrypt	= crypto4xx_decrypt,
-			}
-		}
+	{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = {
+		.base = {
+			.cra_name = "ctr(aes)",
+			.cra_driver_name = "ctr-aes-ppc4xx",
+			.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
+			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+				CRYPTO_ALG_NEED_FALLBACK |
+				CRYPTO_ALG_ASYNC |
+				CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct crypto4xx_ctx),
+			.cra_module = THIS_MODULE,
+		},
+		.min_keysize = AES_MIN_KEY_SIZE,
+		.max_keysize = AES_MAX_KEY_SIZE,
+		.ivsize	= AES_IV_SIZE,
+		.setkey	= crypto4xx_setkey_aes_ctr,
+		.encrypt = crypto4xx_encrypt_ctr,
+		.decrypt = crypto4xx_decrypt_ctr,
+		.init = crypto4xx_sk_init,
+		.exit = crypto4xx_sk_exit,
 	} },
-	{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = {
-		.cra_name	= "ofb(aes)",
-		.cra_driver_name = "ofb-aes-ppc4xx",
-		.cra_priority	= CRYPTO4XX_CRYPTO_PRIORITY,
-		.cra_flags	= CRYPTO_ALG_TYPE_ABLKCIPHER |
-				  CRYPTO_ALG_ASYNC |
-				  CRYPTO_ALG_KERN_DRIVER_ONLY,
-		.cra_blocksize	= AES_BLOCK_SIZE,
-		.cra_ctxsize	= sizeof(struct crypto4xx_ctx),
-		.cra_type	= &crypto_ablkcipher_type,
-		.cra_init	= crypto4xx_ablk_init,
-		.cra_exit	= crypto4xx_ablk_exit,
-		.cra_module	= THIS_MODULE,
-		.cra_u		= {
-			.ablkcipher = {
-				.min_keysize	= AES_MIN_KEY_SIZE,
-				.max_keysize	= AES_MAX_KEY_SIZE,
-				.ivsize		= AES_IV_SIZE,
-				.setkey		= crypto4xx_setkey_aes_ofb,
-				.encrypt	= crypto4xx_encrypt,
-				.decrypt	= crypto4xx_decrypt,
-			}
-		}
+	{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = {
+		.base = {
+			.cra_name = "rfc3686(ctr(aes))",
+			.cra_driver_name = "rfc3686-ctr-aes-ppc4xx",
+			.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
+			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+				CRYPTO_ALG_ASYNC |
+				CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct crypto4xx_ctx),
+			.cra_module = THIS_MODULE,
+		},
+		.min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+		.max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+		.ivsize	= CTR_RFC3686_IV_SIZE,
+		.setkey = crypto4xx_setkey_rfc3686,
+		.encrypt = crypto4xx_rfc3686_encrypt,
+		.decrypt = crypto4xx_rfc3686_decrypt,
+		.init = crypto4xx_sk_init,
+		.exit = crypto4xx_sk_exit,
+	} },
+	{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = {
+		.base = {
+			.cra_name = "ecb(aes)",
+			.cra_driver_name = "ecb-aes-ppc4xx",
+			.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
+			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+				CRYPTO_ALG_ASYNC |
+				CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct crypto4xx_ctx),
+			.cra_module = THIS_MODULE,
+		},
+		.min_keysize = AES_MIN_KEY_SIZE,
+		.max_keysize = AES_MAX_KEY_SIZE,
+		.setkey	= crypto4xx_setkey_aes_ecb,
+		.encrypt = crypto4xx_encrypt_noiv,
+		.decrypt = crypto4xx_decrypt_noiv,
+		.init = crypto4xx_sk_init,
+		.exit = crypto4xx_sk_exit,
+	} },
+	{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = {
+		.base = {
+			.cra_name = "ofb(aes)",
+			.cra_driver_name = "ofb-aes-ppc4xx",
+			.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
+			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+				CRYPTO_ALG_ASYNC |
+				CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct crypto4xx_ctx),
+			.cra_module = THIS_MODULE,
+		},
+		.min_keysize = AES_MIN_KEY_SIZE,
+		.max_keysize = AES_MAX_KEY_SIZE,
+		.ivsize	= AES_IV_SIZE,
+		.setkey	= crypto4xx_setkey_aes_ofb,
+		.encrypt = crypto4xx_encrypt_iv,
+		.decrypt = crypto4xx_decrypt_iv,
+		.init = crypto4xx_sk_init,
+		.exit = crypto4xx_sk_exit,
 	} },
 
 	/* AEAD */
diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h
index 23b726da6534..e2ca56722f07 100644
--- a/drivers/crypto/amcc/crypto4xx_core.h
+++ b/drivers/crypto/amcc/crypto4xx_core.h
@@ -25,6 +25,7 @@
 #include <linux/ratelimit.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
 #include "crypto4xx_reg_def.h"
 #include "crypto4xx_sa.h"
 
@@ -127,14 +128,19 @@ struct crypto4xx_ctx {
 	__le32 iv_nonce;
 	u32 sa_len;
 	union {
+		struct crypto_skcipher *cipher;
 		struct crypto_aead *aead;
 	} sw_cipher;
 };
 
+struct crypto4xx_aead_reqctx {
+	struct scatterlist dst[2];
+};
+
 struct crypto4xx_alg_common {
 	u32 type;
 	union {
-		struct crypto_alg cipher;
+		struct skcipher_alg cipher;
 		struct ahash_alg hash;
 		struct aead_alg aead;
 	} u;
@@ -157,21 +163,28 @@ int crypto4xx_build_pd(struct crypto_async_request *req,
 		       const __le32 *iv, const u32 iv_len,
 		       const struct dynamic_sa_ctl *sa,
 		       const unsigned int sa_len,
-		       const unsigned int assoclen);
-int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher,
+		       const unsigned int assoclen,
+		       struct scatterlist *dst_tmp);
+int crypto4xx_setkey_aes_cbc(struct crypto_skcipher *cipher,
+			     const u8 *key, unsigned int keylen);
+int crypto4xx_setkey_aes_cfb(struct crypto_skcipher *cipher,
 			     const u8 *key, unsigned int keylen);
-int crypto4xx_setkey_aes_cfb(struct crypto_ablkcipher *cipher,
+int crypto4xx_setkey_aes_ctr(struct crypto_skcipher *cipher,
 			     const u8 *key, unsigned int keylen);
-int crypto4xx_setkey_aes_ecb(struct crypto_ablkcipher *cipher,
+int crypto4xx_setkey_aes_ecb(struct crypto_skcipher *cipher,
 			     const u8 *key, unsigned int keylen);
-int crypto4xx_setkey_aes_ofb(struct crypto_ablkcipher *cipher,
+int crypto4xx_setkey_aes_ofb(struct crypto_skcipher *cipher,
 			     const u8 *key, unsigned int keylen);
-int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher,
+int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher,
 			     const u8 *key, unsigned int keylen);
-int crypto4xx_encrypt(struct ablkcipher_request *req);
-int crypto4xx_decrypt(struct ablkcipher_request *req);
-int crypto4xx_rfc3686_encrypt(struct ablkcipher_request *req);
-int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req);
+int crypto4xx_encrypt_ctr(struct skcipher_request *req);
+int crypto4xx_decrypt_ctr(struct skcipher_request *req);
+int crypto4xx_encrypt_iv(struct skcipher_request *req);
+int crypto4xx_decrypt_iv(struct skcipher_request *req);
+int crypto4xx_encrypt_noiv(struct skcipher_request *req);
+int crypto4xx_decrypt_noiv(struct skcipher_request *req);
+int crypto4xx_rfc3686_encrypt(struct skcipher_request *req);
+int crypto4xx_rfc3686_decrypt(struct skcipher_request *req);
 int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm);
 int crypto4xx_hash_digest(struct ahash_request *req);
 int crypto4xx_hash_final(struct ahash_request *req);
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 7207a535942d..d67667970f7e 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -769,15 +769,18 @@ struct aead_edesc {
  * @src_nents: number of segments in input s/w scatterlist
  * @dst_nents: number of segments in output s/w scatterlist
  * @iv_dma: dma address of iv for checking continuity and link table
+ * @iv_dir: DMA mapping direction for IV
  * @sec4_sg_bytes: length of dma mapped sec4_sg space
  * @sec4_sg_dma: bus physical mapped address of h/w link table
  * @sec4_sg: pointer to h/w link table
  * @hw_desc: the h/w job descriptor followed by any referenced link tables
+ *	     and IV
  */
 struct ablkcipher_edesc {
 	int src_nents;
 	int dst_nents;
 	dma_addr_t iv_dma;
+	enum dma_data_direction iv_dir;
 	int sec4_sg_bytes;
 	dma_addr_t sec4_sg_dma;
 	struct sec4_sg_entry *sec4_sg;
@@ -787,7 +790,8 @@ struct ablkcipher_edesc {
 static void caam_unmap(struct device *dev, struct scatterlist *src,
 		       struct scatterlist *dst, int src_nents,
 		       int dst_nents,
-		       dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
+		       dma_addr_t iv_dma, int ivsize,
+		       enum dma_data_direction iv_dir, dma_addr_t sec4_sg_dma,
 		       int sec4_sg_bytes)
 {
 	if (dst != src) {
@@ -799,7 +803,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
 	}
 
 	if (iv_dma)
-		dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
+		dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
 	if (sec4_sg_bytes)
 		dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
 				 DMA_TO_DEVICE);
@@ -810,7 +814,7 @@ static void aead_unmap(struct device *dev,
 		       struct aead_request *req)
 {
 	caam_unmap(dev, req->src, req->dst,
-		   edesc->src_nents, edesc->dst_nents, 0, 0,
+		   edesc->src_nents, edesc->dst_nents, 0, 0, DMA_NONE,
 		   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 }
 
@@ -823,7 +827,7 @@ static void ablkcipher_unmap(struct device *dev,
 
 	caam_unmap(dev, req->src, req->dst,
 		   edesc->src_nents, edesc->dst_nents,
-		   edesc->iv_dma, ivsize,
+		   edesc->iv_dma, ivsize, edesc->iv_dir,
 		   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 }
 
@@ -912,6 +916,18 @@ static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 	scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize,
 				 ivsize, 0);
 
+	/* In case initial IV was generated, copy it in GIVCIPHER request */
+	if (edesc->iv_dir == DMA_FROM_DEVICE) {
+		u8 *iv;
+		struct skcipher_givcrypt_request *greq;
+
+		greq = container_of(req, struct skcipher_givcrypt_request,
+				    creq);
+		iv = (u8 *)edesc->hw_desc + desc_bytes(edesc->hw_desc) +
+		     edesc->sec4_sg_bytes;
+		memcpy(greq->giv, iv, ivsize);
+	}
+
 	kfree(edesc);
 
 	ablkcipher_request_complete(req, err);
@@ -922,10 +938,10 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 {
 	struct ablkcipher_request *req = context;
 	struct ablkcipher_edesc *edesc;
+#ifdef DEBUG
 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 
-#ifdef DEBUG
 	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 #endif
 
@@ -943,14 +959,6 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 		     edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
 
 	ablkcipher_unmap(jrdev, edesc, req);
-
-	/*
-	 * The crypto API expects us to set the IV (req->info) to the last
-	 * ciphertext block.
-	 */
-	scatterwalk_map_and_copy(req->info, req->src, req->nbytes - ivsize,
-				 ivsize, 0);
-
 	kfree(edesc);
 
 	ablkcipher_request_complete(req, err);
@@ -1099,15 +1107,14 @@ static void init_authenc_job(struct aead_request *req,
  */
 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
 				struct ablkcipher_edesc *edesc,
-				struct ablkcipher_request *req,
-				bool iv_contig)
+				struct ablkcipher_request *req)
 {
 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 	u32 *desc = edesc->hw_desc;
-	u32 out_options = 0, in_options;
-	dma_addr_t dst_dma, src_dma;
-	int len, sec4_sg_index = 0;
+	u32 out_options = 0;
+	dma_addr_t dst_dma;
+	int len;
 
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
@@ -1123,30 +1130,18 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
 	len = desc_len(sh_desc);
 	init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
 
-	if (iv_contig) {
-		src_dma = edesc->iv_dma;
-		in_options = 0;
-	} else {
-		src_dma = edesc->sec4_sg_dma;
-		sec4_sg_index += edesc->src_nents + 1;
-		in_options = LDST_SGF;
-	}
-	append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
+	append_seq_in_ptr(desc, edesc->sec4_sg_dma, req->nbytes + ivsize,
+			  LDST_SGF);
 
 	if (likely(req->src == req->dst)) {
-		if (edesc->src_nents == 1 && iv_contig) {
-			dst_dma = sg_dma_address(req->src);
-		} else {
-			dst_dma = edesc->sec4_sg_dma +
-				sizeof(struct sec4_sg_entry);
-			out_options = LDST_SGF;
-		}
+		dst_dma = edesc->sec4_sg_dma + sizeof(struct sec4_sg_entry);
+		out_options = LDST_SGF;
 	} else {
 		if (edesc->dst_nents == 1) {
 			dst_dma = sg_dma_address(req->dst);
 		} else {
-			dst_dma = edesc->sec4_sg_dma +
-				sec4_sg_index * sizeof(struct sec4_sg_entry);
+			dst_dma = edesc->sec4_sg_dma + (edesc->src_nents + 1) *
+				  sizeof(struct sec4_sg_entry);
 			out_options = LDST_SGF;
 		}
 	}
@@ -1158,13 +1153,12 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
  */
 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
 				    struct ablkcipher_edesc *edesc,
-				    struct ablkcipher_request *req,
-				    bool iv_contig)
+				    struct ablkcipher_request *req)
 {
 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 	u32 *desc = edesc->hw_desc;
-	u32 out_options, in_options;
+	u32 in_options;
 	dma_addr_t dst_dma, src_dma;
 	int len, sec4_sg_index = 0;
 
@@ -1190,15 +1184,9 @@ static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
 	}
 	append_seq_in_ptr(desc, src_dma, req->nbytes, in_options);
 
-	if (iv_contig) {
-		dst_dma = edesc->iv_dma;
-		out_options = 0;
-	} else {
-		dst_dma = edesc->sec4_sg_dma +
-			  sec4_sg_index * sizeof(struct sec4_sg_entry);
-		out_options = LDST_SGF;
-	}
-	append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options);
+	dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
+		  sizeof(struct sec4_sg_entry);
+	append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, LDST_SGF);
 }
 
 /*
@@ -1287,7 +1275,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 			GFP_DMA | flags);
 	if (!edesc) {
 		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
-			   0, 0, 0);
+			   0, DMA_NONE, 0, 0);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -1491,8 +1479,7 @@ static int aead_decrypt(struct aead_request *req)
  * allocate and map the ablkcipher extended descriptor for ablkcipher
  */
 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
-						       *req, int desc_bytes,
-						       bool *iv_contig_out)
+						       *req, int desc_bytes)
 {
 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
@@ -1501,8 +1488,8 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 		       GFP_KERNEL : GFP_ATOMIC;
 	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
 	struct ablkcipher_edesc *edesc;
-	dma_addr_t iv_dma = 0;
-	bool in_contig;
+	dma_addr_t iv_dma;
+	u8 *iv;
 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 	int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
 
@@ -1546,33 +1533,20 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 		}
 	}
 
-	iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
-	if (dma_mapping_error(jrdev, iv_dma)) {
-		dev_err(jrdev, "unable to map IV\n");
-		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
-			   0, 0, 0);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	if (mapped_src_nents == 1 &&
-	    iv_dma + ivsize == sg_dma_address(req->src)) {
-		in_contig = true;
-		sec4_sg_ents = 0;
-	} else {
-		in_contig = false;
-		sec4_sg_ents = 1 + mapped_src_nents;
-	}
+	sec4_sg_ents = 1 + mapped_src_nents;
 	dst_sg_idx = sec4_sg_ents;
 	sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
 	sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
 
-	/* allocate space for base edesc and hw desc commands, link tables */
-	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
+	/*
+	 * allocate space for base edesc and hw desc commands, link tables, IV
+	 */
+	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize,
 			GFP_DMA | flags);
 	if (!edesc) {
 		dev_err(jrdev, "could not allocate extended descriptor\n");
-		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
-			   iv_dma, ivsize, 0, 0);
+		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, DMA_NONE, 0, 0);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -1581,13 +1555,24 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 	edesc->sec4_sg_bytes = sec4_sg_bytes;
 	edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
 			 desc_bytes;
+	edesc->iv_dir = DMA_TO_DEVICE;
 
-	if (!in_contig) {
-		dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
-		sg_to_sec4_sg_last(req->src, mapped_src_nents,
-				   edesc->sec4_sg + 1, 0);
+	/* Make sure IV is located in a DMAable area */
+	iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes;
+	memcpy(iv, req->info, ivsize);
+
+	iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_TO_DEVICE);
+	if (dma_mapping_error(jrdev, iv_dma)) {
+		dev_err(jrdev, "unable to map IV\n");
+		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, DMA_NONE, 0, 0);
+		kfree(edesc);
+		return ERR_PTR(-ENOMEM);
 	}
 
+	dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
+	sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg + 1, 0);
+
 	if (mapped_dst_nents > 1) {
 		sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
 				   edesc->sec4_sg + dst_sg_idx, 0);
@@ -1598,7 +1583,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 		dev_err(jrdev, "unable to map S/G table\n");
 		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
-			   iv_dma, ivsize, 0, 0);
+			   iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 		kfree(edesc);
 		return ERR_PTR(-ENOMEM);
 	}
@@ -1611,7 +1596,6 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 		       sec4_sg_bytes, 1);
 #endif
 
-	*iv_contig_out = in_contig;
 	return edesc;
 }
 
@@ -1621,19 +1605,16 @@ static int ablkcipher_encrypt(struct ablkcipher_request *req)
 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
 	struct device *jrdev = ctx->jrdev;
-	bool iv_contig;
 	u32 *desc;
 	int ret = 0;
 
 	/* allocate extended descriptor */
-	edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
-				       CAAM_CMD_SZ, &iv_contig);
+	edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
 	if (IS_ERR(edesc))
 		return PTR_ERR(edesc);
 
 	/* Create and submit job descriptor*/
-	init_ablkcipher_job(ctx->sh_desc_enc,
-		ctx->sh_desc_enc_dma, edesc, req, iv_contig);
+	init_ablkcipher_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req);
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
@@ -1657,20 +1638,25 @@ static int ablkcipher_decrypt(struct ablkcipher_request *req)
 	struct ablkcipher_edesc *edesc;
 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 	struct device *jrdev = ctx->jrdev;
-	bool iv_contig;
 	u32 *desc;
 	int ret = 0;
 
 	/* allocate extended descriptor */
-	edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
-				       CAAM_CMD_SZ, &iv_contig);
+	edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
 	if (IS_ERR(edesc))
 		return PTR_ERR(edesc);
 
+	/*
+	 * The crypto API expects us to set the IV (req->info) to the last
+	 * ciphertext block.
+	 */
+	scatterwalk_map_and_copy(req->info, req->src, req->nbytes - ivsize,
+				 ivsize, 0);
+
 	/* Create and submit job descriptor*/
-	init_ablkcipher_job(ctx->sh_desc_dec,
-		ctx->sh_desc_dec_dma, edesc, req, iv_contig);
+	init_ablkcipher_job(ctx->sh_desc_dec, ctx->sh_desc_dec_dma, edesc, req);
 	desc = edesc->hw_desc;
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
@@ -1695,8 +1681,7 @@ static int ablkcipher_decrypt(struct ablkcipher_request *req)
  */
 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 				struct skcipher_givcrypt_request *greq,
-				int desc_bytes,
-				bool *iv_contig_out)
+				int desc_bytes)
 {
 	struct ablkcipher_request *req = &greq->creq;
 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
@@ -1706,8 +1691,8 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 		       GFP_KERNEL : GFP_ATOMIC;
 	int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents;
 	struct ablkcipher_edesc *edesc;
-	dma_addr_t iv_dma = 0;
-	bool out_contig;
+	dma_addr_t iv_dma;
+	u8 *iv;
 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 	int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
 
@@ -1752,36 +1737,20 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 		}
 	}
 
-	/*
-	 * Check if iv can be contiguous with source and destination.
-	 * If so, include it. If not, create scatterlist.
-	 */
-	iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
-	if (dma_mapping_error(jrdev, iv_dma)) {
-		dev_err(jrdev, "unable to map IV\n");
-		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
-			   0, 0, 0);
-		return ERR_PTR(-ENOMEM);
-	}
-
 	sec4_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0;
 	dst_sg_idx = sec4_sg_ents;
-	if (mapped_dst_nents == 1 &&
-	    iv_dma + ivsize == sg_dma_address(req->dst)) {
-		out_contig = true;
-	} else {
-		out_contig = false;
-		sec4_sg_ents += 1 + mapped_dst_nents;
-	}
+	sec4_sg_ents += 1 + mapped_dst_nents;
 
-	/* allocate space for base edesc and hw desc commands, link tables */
+	/*
+	 * allocate space for base edesc and hw desc commands, link tables, IV
+	 */
 	sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
-	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
+	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize,
 			GFP_DMA | flags);
 	if (!edesc) {
 		dev_err(jrdev, "could not allocate extended descriptor\n");
-		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
-			   iv_dma, ivsize, 0, 0);
+		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, DMA_NONE, 0, 0);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -1790,24 +1759,33 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 	edesc->sec4_sg_bytes = sec4_sg_bytes;
 	edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
 			 desc_bytes;
+	edesc->iv_dir = DMA_FROM_DEVICE;
+
+	/* Make sure IV is located in a DMAable area */
+	iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes;
+	iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_FROM_DEVICE);
+	if (dma_mapping_error(jrdev, iv_dma)) {
+		dev_err(jrdev, "unable to map IV\n");
+		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, DMA_NONE, 0, 0);
+		kfree(edesc);
+		return ERR_PTR(-ENOMEM);
+	}
 
 	if (mapped_src_nents > 1)
 		sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg,
 				   0);
 
-	if (!out_contig) {
-		dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx,
-				   iv_dma, ivsize, 0);
-		sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
-				   edesc->sec4_sg + dst_sg_idx + 1, 0);
-	}
+	dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx, iv_dma, ivsize, 0);
+	sg_to_sec4_sg_last(req->dst, mapped_dst_nents, edesc->sec4_sg +
+			   dst_sg_idx + 1, 0);
 
 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 					    sec4_sg_bytes, DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 		dev_err(jrdev, "unable to map S/G table\n");
 		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
-			   iv_dma, ivsize, 0, 0);
+			   iv_dma, ivsize, DMA_FROM_DEVICE, 0, 0);
 		kfree(edesc);
 		return ERR_PTR(-ENOMEM);
 	}
@@ -1820,7 +1798,6 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 		       sec4_sg_bytes, 1);
 #endif
 
-	*iv_contig_out = out_contig;
 	return edesc;
 }
 
@@ -1831,19 +1808,17 @@ static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
 	struct device *jrdev = ctx->jrdev;
-	bool iv_contig = false;
 	u32 *desc;
 	int ret = 0;
 
 	/* allocate extended descriptor */
-	edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN *
-				       CAAM_CMD_SZ, &iv_contig);
+	edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
 	if (IS_ERR(edesc))
 		return PTR_ERR(edesc);
 
 	/* Create and submit job descriptor*/
 	init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma,
-				edesc, req, iv_contig);
+				edesc, req);
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR,
 		       "ablkcipher jobdesc@" __stringify(__LINE__) ": ",
diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c
index 8ae7a1be7dfd..a408edd84f34 100644
--- a/drivers/crypto/caam/caamalg_desc.c
+++ b/drivers/crypto/caam/caamalg_desc.c
@@ -1093,7 +1093,7 @@ void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
 				    (0x6 << MOVE_LEN_SHIFT));
 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
-				     (0x8 << MOVE_LEN_SHIFT));
+				     (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
 
 	/* Will read assoclen + cryptlen bytes */
 	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
@@ -1178,7 +1178,7 @@ void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
 				    (0x6 << MOVE_LEN_SHIFT));
 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
-				     (0x8 << MOVE_LEN_SHIFT));
+				     (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
 
 	/* Will read assoclen + cryptlen bytes */
 	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index cacda0831390..6e61cc93c2b0 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -728,7 +728,7 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
  * @assoclen: associated data length, in CAAM endianness
  * @assoclen_dma: bus physical mapped address of req->assoclen
  * @drv_req: driver-specific request structure
- * @sgt: the h/w link table
+ * @sgt: the h/w link table, followed by IV
  */
 struct aead_edesc {
 	int src_nents;
@@ -739,9 +739,6 @@ struct aead_edesc {
 	unsigned int assoclen;
 	dma_addr_t assoclen_dma;
 	struct caam_drv_req drv_req;
-#define CAAM_QI_MAX_AEAD_SG						\
-	((CAAM_QI_MEMCACHE_SIZE - offsetof(struct aead_edesc, sgt)) /	\
-	 sizeof(struct qm_sg_entry))
 	struct qm_sg_entry sgt[0];
 };
 
@@ -753,7 +750,7 @@ struct aead_edesc {
  * @qm_sg_bytes: length of dma mapped h/w link table
  * @qm_sg_dma: bus physical mapped address of h/w link table
  * @drv_req: driver-specific request structure
- * @sgt: the h/w link table
+ * @sgt: the h/w link table, followed by IV
  */
 struct ablkcipher_edesc {
 	int src_nents;
@@ -762,9 +759,6 @@ struct ablkcipher_edesc {
 	int qm_sg_bytes;
 	dma_addr_t qm_sg_dma;
 	struct caam_drv_req drv_req;
-#define CAAM_QI_MAX_ABLKCIPHER_SG					    \
-	((CAAM_QI_MEMCACHE_SIZE - offsetof(struct ablkcipher_edesc, sgt)) / \
-	 sizeof(struct qm_sg_entry))
 	struct qm_sg_entry sgt[0];
 };
 
@@ -986,17 +980,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 		}
 	}
 
-	if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) {
+	if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
 		ivsize = crypto_aead_ivsize(aead);
-		iv_dma = dma_map_single(qidev, req->iv, ivsize, DMA_TO_DEVICE);
-		if (dma_mapping_error(qidev, iv_dma)) {
-			dev_err(qidev, "unable to map IV\n");
-			caam_unmap(qidev, req->src, req->dst, src_nents,
-				   dst_nents, 0, 0, op_type, 0, 0);
-			qi_cache_free(edesc);
-			return ERR_PTR(-ENOMEM);
-		}
-	}
 
 	/*
 	 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
@@ -1004,16 +989,33 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 	 */
 	qm_sg_ents = 1 + !!ivsize + mapped_src_nents +
 		     (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
-	if (unlikely(qm_sg_ents > CAAM_QI_MAX_AEAD_SG)) {
-		dev_err(qidev, "Insufficient S/G entries: %d > %zu\n",
-			qm_sg_ents, CAAM_QI_MAX_AEAD_SG);
-		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-			   iv_dma, ivsize, op_type, 0, 0);
+	sg_table = &edesc->sgt[0];
+	qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
+	if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
+		     CAAM_QI_MEMCACHE_SIZE)) {
+		dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
+			qm_sg_ents, ivsize);
+		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, 0, 0, 0);
 		qi_cache_free(edesc);
 		return ERR_PTR(-ENOMEM);
 	}
-	sg_table = &edesc->sgt[0];
-	qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
+
+	if (ivsize) {
+		u8 *iv = (u8 *)(sg_table + qm_sg_ents);
+
+		/* Make sure IV is located in a DMAable area */
+		memcpy(iv, req->iv, ivsize);
+
+		iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
+		if (dma_mapping_error(qidev, iv_dma)) {
+			dev_err(qidev, "unable to map IV\n");
+			caam_unmap(qidev, req->src, req->dst, src_nents,
+				   dst_nents, 0, 0, 0, 0, 0);
+			qi_cache_free(edesc);
+			return ERR_PTR(-ENOMEM);
+		}
+	}
 
 	edesc->src_nents = src_nents;
 	edesc->dst_nents = dst_nents;
@@ -1166,15 +1168,27 @@ static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status)
 #endif
 
 	ablkcipher_unmap(qidev, edesc, req);
-	qi_cache_free(edesc);
+
+	/* In case initial IV was generated, copy it in GIVCIPHER request */
+	if (edesc->drv_req.drv_ctx->op_type == GIVENCRYPT) {
+		u8 *iv;
+		struct skcipher_givcrypt_request *greq;
+
+		greq = container_of(req, struct skcipher_givcrypt_request,
+				    creq);
+		iv = (u8 *)edesc->sgt + edesc->qm_sg_bytes;
+		memcpy(greq->giv, iv, ivsize);
+	}
 
 	/*
 	 * The crypto API expects us to set the IV (req->info) to the last
 	 * ciphertext block. This is used e.g. by the CTS mode.
 	 */
-	scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize,
-				 ivsize, 0);
+	if (edesc->drv_req.drv_ctx->op_type != DECRYPT)
+		scatterwalk_map_and_copy(req->info, req->dst, req->nbytes -
+					 ivsize, ivsize, 0);
 
+	qi_cache_free(edesc);
 	ablkcipher_request_complete(req, status);
 }
 
@@ -1189,9 +1203,9 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
 	struct ablkcipher_edesc *edesc;
 	dma_addr_t iv_dma;
-	bool in_contig;
+	u8 *iv;
 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
-	int dst_sg_idx, qm_sg_ents;
+	int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
 	struct qm_sg_entry *sg_table, *fd_sgt;
 	struct caam_drv_ctx *drv_ctx;
 	enum optype op_type = encrypt ? ENCRYPT : DECRYPT;
@@ -1238,55 +1252,53 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 		}
 	}
 
-	iv_dma = dma_map_single(qidev, req->info, ivsize, DMA_TO_DEVICE);
-	if (dma_mapping_error(qidev, iv_dma)) {
-		dev_err(qidev, "unable to map IV\n");
-		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-			   0, 0, 0, 0);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	if (mapped_src_nents == 1 &&
-	    iv_dma + ivsize == sg_dma_address(req->src)) {
-		in_contig = true;
-		qm_sg_ents = 0;
-	} else {
-		in_contig = false;
-		qm_sg_ents = 1 + mapped_src_nents;
-	}
+	qm_sg_ents = 1 + mapped_src_nents;
 	dst_sg_idx = qm_sg_ents;
 
 	qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
-	if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) {
-		dev_err(qidev, "Insufficient S/G entries: %d > %zu\n",
-			qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG);
-		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-			   iv_dma, ivsize, op_type, 0, 0);
+	qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
+	if (unlikely(offsetof(struct ablkcipher_edesc, sgt) + qm_sg_bytes +
+		     ivsize > CAAM_QI_MEMCACHE_SIZE)) {
+		dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
+			qm_sg_ents, ivsize);
+		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, 0, 0, 0);
 		return ERR_PTR(-ENOMEM);
 	}
 
-	/* allocate space for base edesc and link tables */
+	/* allocate space for base edesc, link tables and IV */
 	edesc = qi_cache_alloc(GFP_DMA | flags);
 	if (unlikely(!edesc)) {
 		dev_err(qidev, "could not allocate extended descriptor\n");
-		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-			   iv_dma, ivsize, op_type, 0, 0);
+		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, 0, 0, 0);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	/* Make sure IV is located in a DMAable area */
+	sg_table = &edesc->sgt[0];
+	iv = (u8 *)(sg_table + qm_sg_ents);
+	memcpy(iv, req->info, ivsize);
+
+	iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
+	if (dma_mapping_error(qidev, iv_dma)) {
+		dev_err(qidev, "unable to map IV\n");
+		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, 0, 0, 0);
+		qi_cache_free(edesc);
 		return ERR_PTR(-ENOMEM);
 	}
 
 	edesc->src_nents = src_nents;
 	edesc->dst_nents = dst_nents;
 	edesc->iv_dma = iv_dma;
-	sg_table = &edesc->sgt[0];
-	edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
+	edesc->qm_sg_bytes = qm_sg_bytes;
 	edesc->drv_req.app_ctx = req;
 	edesc->drv_req.cbk = ablkcipher_done;
 	edesc->drv_req.drv_ctx = drv_ctx;
 
-	if (!in_contig) {
-		dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
-		sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0);
-	}
+	dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
+	sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0);
 
 	if (mapped_dst_nents > 1)
 		sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
@@ -1304,20 +1316,12 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 
 	fd_sgt = &edesc->drv_req.fd_sgt[0];
 
-	if (!in_contig)
-		dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,
-					  ivsize + req->nbytes, 0);
-	else
-		dma_to_qm_sg_one_last(&fd_sgt[1], iv_dma, ivsize + req->nbytes,
-				      0);
+	dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,
+				  ivsize + req->nbytes, 0);
 
 	if (req->src == req->dst) {
-		if (!in_contig)
-			dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +
-					     sizeof(*sg_table), req->nbytes, 0);
-		else
-			dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src),
-					 req->nbytes, 0);
+		dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +
+				     sizeof(*sg_table), req->nbytes, 0);
 	} else if (mapped_dst_nents > 1) {
 		dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
 				     sizeof(*sg_table), req->nbytes, 0);
@@ -1341,10 +1345,10 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 	int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents;
 	struct ablkcipher_edesc *edesc;
 	dma_addr_t iv_dma;
-	bool out_contig;
+	u8 *iv;
 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 	struct qm_sg_entry *sg_table, *fd_sgt;
-	int dst_sg_idx, qm_sg_ents;
+	int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
 	struct caam_drv_ctx *drv_ctx;
 
 	drv_ctx = get_drv_ctx(ctx, GIVENCRYPT);
@@ -1392,46 +1396,45 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 		mapped_dst_nents = src_nents;
 	}
 
-	iv_dma = dma_map_single(qidev, creq->giv, ivsize, DMA_FROM_DEVICE);
-	if (dma_mapping_error(qidev, iv_dma)) {
-		dev_err(qidev, "unable to map IV\n");
-		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-			   0, 0, 0, 0);
-		return ERR_PTR(-ENOMEM);
-	}
-
 	qm_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0;
 	dst_sg_idx = qm_sg_ents;
-	if (mapped_dst_nents == 1 &&
-	    iv_dma + ivsize == sg_dma_address(req->dst)) {
-		out_contig = true;
-	} else {
-		out_contig = false;
-		qm_sg_ents += 1 + mapped_dst_nents;
-	}
 
-	if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) {
-		dev_err(qidev, "Insufficient S/G entries: %d > %zu\n",
-			qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG);
-		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-			   iv_dma, ivsize, GIVENCRYPT, 0, 0);
+	qm_sg_ents += 1 + mapped_dst_nents;
+	qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
+	if (unlikely(offsetof(struct ablkcipher_edesc, sgt) + qm_sg_bytes +
+		     ivsize > CAAM_QI_MEMCACHE_SIZE)) {
+		dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
+			qm_sg_ents, ivsize);
+		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, 0, 0, 0);
 		return ERR_PTR(-ENOMEM);
 	}
 
-	/* allocate space for base edesc and link tables */
+	/* allocate space for base edesc, link tables and IV */
 	edesc = qi_cache_alloc(GFP_DMA | flags);
 	if (!edesc) {
 		dev_err(qidev, "could not allocate extended descriptor\n");
-		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-			   iv_dma, ivsize, GIVENCRYPT, 0, 0);
+		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, 0, 0, 0);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	/* Make sure IV is located in a DMAable area */
+	sg_table = &edesc->sgt[0];
+	iv = (u8 *)(sg_table + qm_sg_ents);
+	iv_dma = dma_map_single(qidev, iv, ivsize, DMA_FROM_DEVICE);
+	if (dma_mapping_error(qidev, iv_dma)) {
+		dev_err(qidev, "unable to map IV\n");
+		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, 0, 0, 0);
+		qi_cache_free(edesc);
 		return ERR_PTR(-ENOMEM);
 	}
 
 	edesc->src_nents = src_nents;
 	edesc->dst_nents = dst_nents;
 	edesc->iv_dma = iv_dma;
-	sg_table = &edesc->sgt[0];
-	edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
+	edesc->qm_sg_bytes = qm_sg_bytes;
 	edesc->drv_req.app_ctx = req;
 	edesc->drv_req.cbk = ablkcipher_done;
 	edesc->drv_req.drv_ctx = drv_ctx;
@@ -1439,11 +1442,9 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 	if (mapped_src_nents > 1)
 		sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table, 0);
 
-	if (!out_contig) {
-		dma_to_qm_sg_one(sg_table + dst_sg_idx, iv_dma, ivsize, 0);
-		sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
-				 dst_sg_idx + 1, 0);
-	}
+	dma_to_qm_sg_one(sg_table + dst_sg_idx, iv_dma, ivsize, 0);
+	sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + dst_sg_idx + 1,
+			 0);
 
 	edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes,
 					  DMA_TO_DEVICE);
@@ -1464,13 +1465,8 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 		dma_to_qm_sg_one(&fd_sgt[1], sg_dma_address(req->src),
 				 req->nbytes, 0);
 
-	if (!out_contig)
-		dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
-				     sizeof(*sg_table), ivsize + req->nbytes,
-				     0);
-	else
-		dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst),
-				 ivsize + req->nbytes, 0);
+	dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
+			     sizeof(*sg_table), ivsize + req->nbytes, 0);
 
 	return edesc;
 }
@@ -1480,6 +1476,7 @@ static inline int ablkcipher_crypt(struct ablkcipher_request *req, bool encrypt)
 	struct ablkcipher_edesc *edesc;
 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 	int ret;
 
 	if (unlikely(caam_congested))
@@ -1490,6 +1487,14 @@ static inline int ablkcipher_crypt(struct ablkcipher_request *req, bool encrypt)
 	if (IS_ERR(edesc))
 		return PTR_ERR(edesc);
 
+	/*
+	 * The crypto API expects us to set the IV (req->info) to the last
+	 * ciphertext block.
+	 */
+	if (!encrypt)
+		scatterwalk_map_and_copy(req->info, req->src, req->nbytes -
+					 ivsize, ivsize, 0);
+
 	ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
 	if (!ret) {
 		ret = -EINPROGRESS;
diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c
index 7a897209f181..578ea63a3109 100644
--- a/drivers/crypto/caam/caampkc.c
+++ b/drivers/crypto/caam/caampkc.c
@@ -66,7 +66,7 @@ static void rsa_priv_f2_unmap(struct device *dev, struct rsa_edesc *edesc,
 	struct caam_rsa_key *key = &ctx->key;
 	struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2;
 	size_t p_sz = key->p_sz;
-	size_t q_sz = key->p_sz;
+	size_t q_sz = key->q_sz;
 
 	dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE);
 	dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE);
@@ -83,7 +83,7 @@ static void rsa_priv_f3_unmap(struct device *dev, struct rsa_edesc *edesc,
 	struct caam_rsa_key *key = &ctx->key;
 	struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3;
 	size_t p_sz = key->p_sz;
-	size_t q_sz = key->p_sz;
+	size_t q_sz = key->q_sz;
 
 	dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE);
 	dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE);
@@ -166,18 +166,71 @@ static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err,
 	akcipher_request_complete(req, err);
 }
 
+static int caam_rsa_count_leading_zeros(struct scatterlist *sgl,
+					unsigned int nbytes,
+					unsigned int flags)
+{
+	struct sg_mapping_iter miter;
+	int lzeros, ents;
+	unsigned int len;
+	unsigned int tbytes = nbytes;
+	const u8 *buff;
+
+	ents = sg_nents_for_len(sgl, nbytes);
+	if (ents < 0)
+		return ents;
+
+	sg_miter_start(&miter, sgl, ents, SG_MITER_FROM_SG | flags);
+
+	lzeros = 0;
+	len = 0;
+	while (nbytes > 0) {
+		while (len && !*buff) {
+			lzeros++;
+			len--;
+			buff++;
+		}
+
+		if (len && *buff)
+			break;
+
+		sg_miter_next(&miter);
+		buff = miter.addr;
+		len = miter.length;
+
+		nbytes -= lzeros;
+		lzeros = 0;
+	}
+
+	miter.consumed = lzeros;
+	sg_miter_stop(&miter);
+	nbytes -= lzeros;
+
+	return tbytes - nbytes;
+}
+
 static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
 					 size_t desclen)
 {
 	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
 	struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
 	struct device *dev = ctx->dev;
+	struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
 	struct rsa_edesc *edesc;
 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 		       GFP_KERNEL : GFP_ATOMIC;
+	int sg_flags = (flags == GFP_ATOMIC) ? SG_MITER_ATOMIC : 0;
 	int sgc;
 	int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
 	int src_nents, dst_nents;
+	int lzeros;
+
+	lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len, sg_flags);
+	if (lzeros < 0)
+		return ERR_PTR(lzeros);
+
+	req->src_len -= lzeros;
+	req->src = scatterwalk_ffwd(req_ctx->src, req->src, lzeros);
 
 	src_nents = sg_nents_for_len(req->src, req->src_len);
 	dst_nents = sg_nents_for_len(req->dst, req->dst_len);
@@ -344,7 +397,7 @@ static int set_rsa_priv_f2_pdb(struct akcipher_request *req,
 	struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2;
 	int sec4_sg_index = 0;
 	size_t p_sz = key->p_sz;
-	size_t q_sz = key->p_sz;
+	size_t q_sz = key->q_sz;
 
 	pdb->d_dma = dma_map_single(dev, key->d, key->d_sz, DMA_TO_DEVICE);
 	if (dma_mapping_error(dev, pdb->d_dma)) {
@@ -419,7 +472,7 @@ static int set_rsa_priv_f3_pdb(struct akcipher_request *req,
 	struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3;
 	int sec4_sg_index = 0;
 	size_t p_sz = key->p_sz;
-	size_t q_sz = key->p_sz;
+	size_t q_sz = key->q_sz;
 
 	pdb->p_dma = dma_map_single(dev, key->p, p_sz, DMA_TO_DEVICE);
 	if (dma_mapping_error(dev, pdb->p_dma)) {
@@ -730,19 +783,12 @@ static u8 *caam_read_rsa_crt(const u8 *ptr, size_t nbytes, size_t dstlen)
  */
 static inline u8 *caam_read_raw_data(const u8 *buf, size_t *nbytes)
 {
-	u8 *val;
 
 	caam_rsa_drop_leading_zeros(&buf, nbytes);
 	if (!*nbytes)
 		return NULL;
 
-	val = kzalloc(*nbytes, GFP_DMA | GFP_KERNEL);
-	if (!val)
-		return NULL;
-
-	memcpy(val, buf, *nbytes);
-
-	return val;
+	return kmemdup(buf, *nbytes, GFP_DMA | GFP_KERNEL);
 }
 
 static int caam_rsa_check_key_length(unsigned int len)
@@ -953,6 +999,7 @@ static struct akcipher_alg caam_rsa = {
 	.max_size = caam_rsa_max_size,
 	.init = caam_rsa_init_tfm,
 	.exit = caam_rsa_exit_tfm,
+	.reqsize = sizeof(struct caam_rsa_req_ctx),
 	.base = {
 		.cra_name = "rsa",
 		.cra_driver_name = "rsa-caam",
diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h
index fd145c46eae1..82645bcf8b27 100644
--- a/drivers/crypto/caam/caampkc.h
+++ b/drivers/crypto/caam/caampkc.h
@@ -95,6 +95,14 @@ struct caam_rsa_ctx {
 	struct device *dev;
 };
 
+/**
+ * caam_rsa_req_ctx - per request context.
+ * @src: input scatterlist (stripped of leading zeros)
+ */
+struct caam_rsa_req_ctx {
+	struct scatterlist src[2];
+};
+
 /**
  * rsa_edesc - s/w-extended rsa descriptor
  * @src_nents     : number of segments in input scatterlist
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index e4cc636e1104..538c01f428c1 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -322,9 +322,9 @@ static int caam_remove(struct platform_device *pdev)
 
 	/*
 	 * De-initialize RNG state handles initialized by this driver.
-	 * In case of DPAA 2.x, RNG is managed by MC firmware.
+	 * In case of SoCs with Management Complex, RNG is managed by MC f/w.
 	 */
-	if (!caam_dpaa2 && ctrlpriv->rng4_sh_init)
+	if (!ctrlpriv->mc_en && ctrlpriv->rng4_sh_init)
 		deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init);
 
 	/* Shut down debug views */
@@ -396,11 +396,56 @@ static void kick_trng(struct platform_device *pdev, int ent_delay)
 	clrsetbits_32(&r4tst->rtmctl, RTMCTL_PRGM, RTMCTL_SAMP_MODE_RAW_ES_SC);
 }
 
+static int caam_get_era_from_hw(struct caam_ctrl __iomem *ctrl)
+{
+	static const struct {
+		u16 ip_id;
+		u8 maj_rev;
+		u8 era;
+	} id[] = {
+		{0x0A10, 1, 1},
+		{0x0A10, 2, 2},
+		{0x0A12, 1, 3},
+		{0x0A14, 1, 3},
+		{0x0A14, 2, 4},
+		{0x0A16, 1, 4},
+		{0x0A10, 3, 4},
+		{0x0A11, 1, 4},
+		{0x0A18, 1, 4},
+		{0x0A11, 2, 5},
+		{0x0A12, 2, 5},
+		{0x0A13, 1, 5},
+		{0x0A1C, 1, 5}
+	};
+	u32 ccbvid, id_ms;
+	u8 maj_rev, era;
+	u16 ip_id;
+	int i;
+
+	ccbvid = rd_reg32(&ctrl->perfmon.ccb_id);
+	era = (ccbvid & CCBVID_ERA_MASK) >> CCBVID_ERA_SHIFT;
+	if (era)	/* This is '0' prior to CAAM ERA-6 */
+		return era;
+
+	id_ms = rd_reg32(&ctrl->perfmon.caam_id_ms);
+	ip_id = (id_ms & SECVID_MS_IPID_MASK) >> SECVID_MS_IPID_SHIFT;
+	maj_rev = (id_ms & SECVID_MS_MAJ_REV_MASK) >> SECVID_MS_MAJ_REV_SHIFT;
+
+	for (i = 0; i < ARRAY_SIZE(id); i++)
+		if (id[i].ip_id == ip_id && id[i].maj_rev == maj_rev)
+			return id[i].era;
+
+	return -ENOTSUPP;
+}
+
 /**
  * caam_get_era() - Return the ERA of the SEC on SoC, based
- * on "sec-era" propery in the DTS. This property is updated by u-boot.
+ * on "sec-era" optional property in the DTS. This property is updated
+ * by u-boot.
+ * In case this property is not passed an attempt to retrieve the CAAM
+ * era via register reads will be made.
  **/
-int caam_get_era(void)
+static int caam_get_era(struct caam_ctrl __iomem *ctrl)
 {
 	struct device_node *caam_node;
 	int ret;
@@ -410,9 +455,11 @@ int caam_get_era(void)
 	ret = of_property_read_u32(caam_node, "fsl,sec-era", &prop);
 	of_node_put(caam_node);
 
-	return ret ? -ENOTSUPP : prop;
+	if (!ret)
+		return prop;
+	else
+		return caam_get_era_from_hw(ctrl);
 }
-EXPORT_SYMBOL(caam_get_era);
 
 static const struct of_device_id caam_match[] = {
 	{
@@ -571,11 +618,15 @@ static int caam_probe(struct platform_device *pdev)
 	/*
 	 * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel,
 	 * long pointers in master configuration register.
-	 * In case of DPAA 2.x, Management Complex firmware performs
+	 * In case of SoCs with Management Complex, MC f/w performs
 	 * the configuration.
 	 */
 	caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2);
-	if (!caam_dpaa2)
+	np = of_find_compatible_node(NULL, NULL, "fsl,qoriq-mc");
+	ctrlpriv->mc_en = !!np;
+	of_node_put(np);
+
+	if (!ctrlpriv->mc_en)
 		clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR,
 			      MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF |
 			      MCFGR_WDENABLE | MCFGR_LARGE_BURST |
@@ -623,7 +674,7 @@ static int caam_probe(struct platform_device *pdev)
 		goto iounmap_ctrl;
 	}
 
-	ctrlpriv->era = caam_get_era();
+	ctrlpriv->era = caam_get_era(ctrl);
 
 	ret = of_platform_populate(nprop, caam_match, NULL, dev);
 	if (ret) {
@@ -686,9 +737,9 @@ static int caam_probe(struct platform_device *pdev)
 	/*
 	 * If SEC has RNG version >= 4 and RNG state handle has not been
 	 * already instantiated, do RNG instantiation
-	 * In case of DPAA 2.x, RNG is managed by MC firmware.
+	 * In case of SoCs with Management Complex, RNG is managed by MC f/w.
 	 */
-	if (!caam_dpaa2 &&
+	if (!ctrlpriv->mc_en &&
 	    (cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) {
 		ctrlpriv->rng4_sh_init =
 			rd_reg32(&ctrl->r4tst[0].rdsta);
@@ -757,9 +808,8 @@ static int caam_probe(struct platform_device *pdev)
 	/* Report "alive" for developer to see */
 	dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id,
 		 ctrlpriv->era);
-	dev_info(dev, "job rings = %d, qi = %d, dpaa2 = %s\n",
-		 ctrlpriv->total_jobrs, ctrlpriv->qi_present,
-		 caam_dpaa2 ? "yes" : "no");
+	dev_info(dev, "job rings = %d, qi = %d\n",
+		 ctrlpriv->total_jobrs, ctrlpriv->qi_present);
 
 #ifdef CONFIG_DEBUG_FS
 	debugfs_create_file("rq_dequeued", S_IRUSR | S_IRGRP | S_IROTH,
diff --git a/drivers/crypto/caam/ctrl.h b/drivers/crypto/caam/ctrl.h
index be693a2cc25e..f3ecd67922a7 100644
--- a/drivers/crypto/caam/ctrl.h
+++ b/drivers/crypto/caam/ctrl.h
@@ -9,8 +9,6 @@
 #define CTRL_H
 
 /* Prototypes for backend-level services exposed to APIs */
-int caam_get_era(void);
-
 extern bool caam_dpaa2;
 
 #endif /* CTRL_H */
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index 7696a774a362..babc78abd155 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -82,6 +82,7 @@ struct caam_drv_private {
 	 */
 	u8 total_jobrs;		/* Total Job Rings in device */
 	u8 qi_present;		/* Nonzero if QI present in device */
+	u8 mc_en;		/* Nonzero if MC f/w is active */
 	int secvio_irq;		/* Security violation interrupt number */
 	int virt_en;		/* Virtualization enabled in CAAM */
 	int era;		/* CAAM Era (internal HW revision) */
diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c
index b9480828da38..67f7f8c42c93 100644
--- a/drivers/crypto/caam/qi.c
+++ b/drivers/crypto/caam/qi.c
@@ -657,9 +657,8 @@ static int init_cgr(struct device *qidev)
 {
 	int ret;
 	struct qm_mcc_initcgr opts;
-	const u64 cpus = *(u64 *)qman_affine_cpus();
-	const int num_cpus = hweight64(cpus);
-	const u64 val = num_cpus * MAX_RSP_FQ_BACKLOG_PER_CPU;
+	const u64 val = (u64)cpumask_weight(qman_affine_cpus()) *
+			MAX_RSP_FQ_BACKLOG_PER_CPU;
 
 	ret = qman_alloc_cgrid(&qipriv.cgr.cgrid);
 	if (ret) {
diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h
index fee363865d88..4fb91ba39c36 100644
--- a/drivers/crypto/caam/regs.h
+++ b/drivers/crypto/caam/regs.h
@@ -312,11 +312,17 @@ struct caam_perfmon {
 
 	/* Component Instantiation Parameters			fe0-fff */
 	u32 rtic_id;		/* RVID - RTIC Version ID	*/
+#define CCBVID_ERA_MASK		0xff000000
+#define CCBVID_ERA_SHIFT	24
 	u32 ccb_id;		/* CCBVID - CCB Version ID	*/
 	u32 cha_id_ms;		/* CHAVID - CHA Version ID Most Significant*/
 	u32 cha_id_ls;		/* CHAVID - CHA Version ID Least Significant*/
 	u32 cha_num_ms;		/* CHANUM - CHA Number Most Significant	*/
 	u32 cha_num_ls;		/* CHANUM - CHA Number Least Significant*/
+#define SECVID_MS_IPID_MASK	0xffff0000
+#define SECVID_MS_IPID_SHIFT	16
+#define SECVID_MS_MAJ_REV_MASK	0x0000ff00
+#define SECVID_MS_MAJ_REV_SHIFT	8
 	u32 caam_id_ms;		/* CAAMVID - CAAM Version ID MS	*/
 	u32 caam_id_ls;		/* CAAMVID - CAAM Version ID LS	*/
 };
diff --git a/drivers/crypto/cavium/zip/common.h b/drivers/crypto/cavium/zip/common.h
index dc451e0a43c5..58fb3ed6e644 100644
--- a/drivers/crypto/cavium/zip/common.h
+++ b/drivers/crypto/cavium/zip/common.h
@@ -46,8 +46,10 @@
 #ifndef __COMMON_H__
 #define __COMMON_H__
 
+#include <linux/delay.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
+#include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/pci.h>
@@ -149,6 +151,25 @@ struct zip_operation {
 	u32   sizeofzops;
 };
 
+static inline int zip_poll_result(union zip_zres_s *result)
+{
+	int retries = 1000;
+
+	while (!result->s.compcode) {
+		if (!--retries) {
+			pr_err("ZIP ERR: request timed out");
+			return -ETIMEDOUT;
+		}
+		udelay(10);
+		/*
+		 * Force re-reading of compcode which is updated
+		 * by the ZIP coprocessor.
+		 */
+		rmb();
+	}
+	return 0;
+}
+
 /* error messages */
 #define zip_err(fmt, args...) pr_err("ZIP ERR:%s():%d: " \
 			      fmt "\n", __func__, __LINE__, ## args)
diff --git a/drivers/crypto/cavium/zip/zip_crypto.c b/drivers/crypto/cavium/zip/zip_crypto.c
index 8df4d26cf9d4..b92b6e7e100f 100644
--- a/drivers/crypto/cavium/zip/zip_crypto.c
+++ b/drivers/crypto/cavium/zip/zip_crypto.c
@@ -124,7 +124,7 @@ int zip_compress(const u8 *src, unsigned int slen,
 		 struct zip_kernel_ctx *zip_ctx)
 {
 	struct zip_operation  *zip_ops   = NULL;
-	struct zip_state      zip_state;
+	struct zip_state      *zip_state;
 	struct zip_device     *zip = NULL;
 	int ret;
 
@@ -135,20 +135,23 @@ int zip_compress(const u8 *src, unsigned int slen,
 	if (!zip)
 		return -ENODEV;
 
-	memset(&zip_state, 0, sizeof(struct zip_state));
+	zip_state = kzalloc(sizeof(*zip_state), GFP_ATOMIC);
+	if (!zip_state)
+		return -ENOMEM;
+
 	zip_ops = &zip_ctx->zip_comp;
 
 	zip_ops->input_len  = slen;
 	zip_ops->output_len = *dlen;
 	memcpy(zip_ops->input, src, slen);
 
-	ret = zip_deflate(zip_ops, &zip_state, zip);
+	ret = zip_deflate(zip_ops, zip_state, zip);
 
 	if (!ret) {
 		*dlen = zip_ops->output_len;
 		memcpy(dst, zip_ops->output, *dlen);
 	}
-
+	kfree(zip_state);
 	return ret;
 }
 
@@ -157,7 +160,7 @@ int zip_decompress(const u8 *src, unsigned int slen,
 		   struct zip_kernel_ctx *zip_ctx)
 {
 	struct zip_operation  *zip_ops   = NULL;
-	struct zip_state      zip_state;
+	struct zip_state      *zip_state;
 	struct zip_device     *zip = NULL;
 	int ret;
 
@@ -168,7 +171,10 @@ int zip_decompress(const u8 *src, unsigned int slen,
 	if (!zip)
 		return -ENODEV;
 
-	memset(&zip_state, 0, sizeof(struct zip_state));
+	zip_state = kzalloc(sizeof(*zip_state), GFP_ATOMIC);
+	if (!zip_state)
+		return -ENOMEM;
+
 	zip_ops = &zip_ctx->zip_decomp;
 	memcpy(zip_ops->input, src, slen);
 
@@ -179,13 +185,13 @@ int zip_decompress(const u8 *src, unsigned int slen,
 	zip_ops->input_len  = slen;
 	zip_ops->output_len = *dlen;
 
-	ret = zip_inflate(zip_ops, &zip_state, zip);
+	ret = zip_inflate(zip_ops, zip_state, zip);
 
 	if (!ret) {
 		*dlen = zip_ops->output_len;
 		memcpy(dst, zip_ops->output, *dlen);
 	}
-
+	kfree(zip_state);
 	return ret;
 }
 
diff --git a/drivers/crypto/cavium/zip/zip_deflate.c b/drivers/crypto/cavium/zip/zip_deflate.c
index 9a944b8c1e29..d7133f857d67 100644
--- a/drivers/crypto/cavium/zip/zip_deflate.c
+++ b/drivers/crypto/cavium/zip/zip_deflate.c
@@ -129,8 +129,8 @@ int zip_deflate(struct zip_operation *zip_ops, struct zip_state *s,
 	/* Stats update for compression requests submitted */
 	atomic64_inc(&zip_dev->stats.comp_req_submit);
 
-	while (!result_ptr->s.compcode)
-		continue;
+	/* Wait for completion or error */
+	zip_poll_result(result_ptr);
 
 	/* Stats update for compression requests completed */
 	atomic64_inc(&zip_dev->stats.comp_req_complete);
diff --git a/drivers/crypto/cavium/zip/zip_device.c b/drivers/crypto/cavium/zip/zip_device.c
index ccf21fb91513..f174ec29ed69 100644
--- a/drivers/crypto/cavium/zip/zip_device.c
+++ b/drivers/crypto/cavium/zip/zip_device.c
@@ -87,12 +87,12 @@ u32 zip_load_instr(union zip_inst_s *instr,
 	 * Distribute the instructions between the enabled queues based on
 	 * the CPU id.
 	 */
-	if (smp_processor_id() % 2 == 0)
+	if (raw_smp_processor_id() % 2 == 0)
 		queue = 0;
 	else
 		queue = 1;
 
-	zip_dbg("CPU Core: %d Queue number:%d", smp_processor_id(), queue);
+	zip_dbg("CPU Core: %d Queue number:%d", raw_smp_processor_id(), queue);
 
 	/* Take cmd buffer lock */
 	spin_lock(&zip_dev->iq[queue].lock);
diff --git a/drivers/crypto/cavium/zip/zip_inflate.c b/drivers/crypto/cavium/zip/zip_inflate.c
index 50cbdd83dbf2..7e0d73e2f89e 100644
--- a/drivers/crypto/cavium/zip/zip_inflate.c
+++ b/drivers/crypto/cavium/zip/zip_inflate.c
@@ -143,8 +143,8 @@ int zip_inflate(struct zip_operation *zip_ops, struct zip_state *s,
 	/* Decompression requests submitted stats update */
 	atomic64_inc(&zip_dev->stats.decomp_req_submit);
 
-	while (!result_ptr->s.compcode)
-		continue;
+	/* Wait for completion or error */
+	zip_poll_result(result_ptr);
 
 	/* Decompression requests completed stats update */
 	atomic64_inc(&zip_dev->stats.decomp_req_complete);
diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c
index 1cd8aa488185..be055b9547f6 100644
--- a/drivers/crypto/cavium/zip/zip_main.c
+++ b/drivers/crypto/cavium/zip/zip_main.c
@@ -113,7 +113,7 @@ struct zip_device *zip_get_device(int node)
  */
 int zip_get_node_id(void)
 {
-	return cpu_to_node(smp_processor_id());
+	return cpu_to_node(raw_smp_processor_id());
 }
 
 /* Initializes the ZIP h/w sub-system */
@@ -469,6 +469,8 @@ static int zip_show_stats(struct seq_file *s, void *unused)
 	struct zip_stats  *st;
 
 	for (index = 0; index < MAX_ZIP_DEVICES; index++) {
+		u64 pending = 0;
+
 		if (zip_dev[index]) {
 			zip = zip_dev[index];
 			st  = &zip->stats;
@@ -476,16 +478,15 @@ static int zip_show_stats(struct seq_file *s, void *unused)
 			/* Get all the pending requests */
 			for (q = 0; q < ZIP_NUM_QUEUES; q++) {
 				val = zip_reg_read((zip->reg_base +
-						    ZIP_DBG_COREX_STA(q)));
-				val = (val >> 32);
-				val = val & 0xffffff;
-				atomic64_add(val, &st->pending_req);
+						    ZIP_DBG_QUEX_STA(q)));
+				pending += val >> 32 & 0xffffff;
 			}
 
-			avg_chunk = (atomic64_read(&st->comp_in_bytes) /
-				     atomic64_read(&st->comp_req_complete));
-			avg_cr = (atomic64_read(&st->comp_in_bytes) /
-				  atomic64_read(&st->comp_out_bytes));
+			val = atomic64_read(&st->comp_req_complete);
+			avg_chunk = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0;
+
+			val = atomic64_read(&st->comp_out_bytes);
+			avg_cr = (val) ? atomic64_read(&st->comp_in_bytes) / val : 0;
 			seq_printf(s, "        ZIP Device %d Stats\n"
 				      "-----------------------------------\n"
 				      "Comp Req Submitted        : \t%lld\n"
@@ -513,10 +514,7 @@ static int zip_show_stats(struct seq_file *s, void *unused)
 				       (u64)atomic64_read(&st->decomp_in_bytes),
 				       (u64)atomic64_read(&st->decomp_out_bytes),
 				       (u64)atomic64_read(&st->decomp_bad_reqs),
-				       (u64)atomic64_read(&st->pending_req));
-
-			/* Reset pending requests  count */
-			atomic64_set(&st->pending_req, 0);
+				       pending);
 		}
 	}
 	return 0;
diff --git a/drivers/crypto/cavium/zip/zip_main.h b/drivers/crypto/cavium/zip/zip_main.h
index 64e051f60784..e1e4fa92ce80 100644
--- a/drivers/crypto/cavium/zip/zip_main.h
+++ b/drivers/crypto/cavium/zip/zip_main.h
@@ -74,7 +74,6 @@ struct zip_stats {
 	atomic64_t    comp_req_complete;
 	atomic64_t    decomp_req_submit;
 	atomic64_t    decomp_req_complete;
-	atomic64_t    pending_req;
 	atomic64_t    comp_in_bytes;
 	atomic64_t    comp_out_bytes;
 	atomic64_t    decomp_in_bytes;
diff --git a/drivers/crypto/cavium/zip/zip_regs.h b/drivers/crypto/cavium/zip/zip_regs.h
index d0be682305c1..874e0236c87e 100644
--- a/drivers/crypto/cavium/zip/zip_regs.h
+++ b/drivers/crypto/cavium/zip/zip_regs.h
@@ -443,7 +443,7 @@ union zip_corex_bist_status {
 
 static inline u64 ZIP_COREX_BIST_STATUS(u64 param1)
 {
-	if (((param1 <= 1)))
+	if (param1 <= 1)
 		return 0x0520ull + (param1 & 1) * 0x8ull;
 	pr_err("ZIP_COREX_BIST_STATUS: %llu\n", param1);
 	return 0;
@@ -537,7 +537,7 @@ union zip_dbg_corex_inst {
 
 static inline u64 ZIP_DBG_COREX_INST(u64 param1)
 {
-	if (((param1 <= 1)))
+	if (param1 <= 1)
 		return 0x0640ull + (param1 & 1) * 0x8ull;
 	pr_err("ZIP_DBG_COREX_INST: %llu\n", param1);
 	return 0;
@@ -568,7 +568,7 @@ union zip_dbg_corex_sta {
 
 static inline u64 ZIP_DBG_COREX_STA(u64 param1)
 {
-	if (((param1 <= 1)))
+	if (param1 <= 1)
 		return 0x0680ull + (param1 & 1) * 0x8ull;
 	pr_err("ZIP_DBG_COREX_STA: %llu\n", param1);
 	return 0;
@@ -599,7 +599,7 @@ union zip_dbg_quex_sta {
 
 static inline u64 ZIP_DBG_QUEX_STA(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x1800ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_DBG_QUEX_STA: %llu\n", param1);
 	return 0;
@@ -817,7 +817,7 @@ union zip_msix_pbax {
 
 static inline u64 ZIP_MSIX_PBAX(u64 param1)
 {
-	if (((param1 == 0)))
+	if (param1 == 0)
 		return 0x0000838000FF0000ull;
 	pr_err("ZIP_MSIX_PBAX: %llu\n", param1);
 	return 0;
@@ -846,7 +846,7 @@ union zip_msix_vecx_addr {
 
 static inline u64 ZIP_MSIX_VECX_ADDR(u64 param1)
 {
-	if (((param1 <= 17)))
+	if (param1 <= 17)
 		return 0x0000838000F00000ull + (param1 & 31) * 0x10ull;
 	pr_err("ZIP_MSIX_VECX_ADDR: %llu\n", param1);
 	return 0;
@@ -875,7 +875,7 @@ union zip_msix_vecx_ctl {
 
 static inline u64 ZIP_MSIX_VECX_CTL(u64 param1)
 {
-	if (((param1 <= 17)))
+	if (param1 <= 17)
 		return 0x0000838000F00008ull + (param1 & 31) * 0x10ull;
 	pr_err("ZIP_MSIX_VECX_CTL: %llu\n", param1);
 	return 0;
@@ -900,7 +900,7 @@ union zip_quex_done {
 
 static inline u64 ZIP_QUEX_DONE(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x2000ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_DONE: %llu\n", param1);
 	return 0;
@@ -925,7 +925,7 @@ union zip_quex_done_ack {
 
 static inline u64 ZIP_QUEX_DONE_ACK(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x2200ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_DONE_ACK: %llu\n", param1);
 	return 0;
@@ -950,7 +950,7 @@ union zip_quex_done_ena_w1c {
 
 static inline u64 ZIP_QUEX_DONE_ENA_W1C(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x2600ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_DONE_ENA_W1C: %llu\n", param1);
 	return 0;
@@ -975,7 +975,7 @@ union zip_quex_done_ena_w1s {
 
 static inline u64 ZIP_QUEX_DONE_ENA_W1S(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x2400ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_DONE_ENA_W1S: %llu\n", param1);
 	return 0;
@@ -1004,7 +1004,7 @@ union zip_quex_done_wait {
 
 static inline u64 ZIP_QUEX_DONE_WAIT(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x2800ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_DONE_WAIT: %llu\n", param1);
 	return 0;
@@ -1029,7 +1029,7 @@ union zip_quex_doorbell {
 
 static inline u64 ZIP_QUEX_DOORBELL(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x4000ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_DOORBELL: %llu\n", param1);
 	return 0;
@@ -1058,7 +1058,7 @@ union zip_quex_err_ena_w1c {
 
 static inline u64 ZIP_QUEX_ERR_ENA_W1C(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x3600ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_ERR_ENA_W1C: %llu\n", param1);
 	return 0;
@@ -1087,7 +1087,7 @@ union zip_quex_err_ena_w1s {
 
 static inline u64 ZIP_QUEX_ERR_ENA_W1S(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x3400ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_ERR_ENA_W1S: %llu\n", param1);
 	return 0;
@@ -1120,7 +1120,7 @@ union zip_quex_err_int {
 
 static inline u64 ZIP_QUEX_ERR_INT(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x3000ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_ERR_INT: %llu\n", param1);
 	return 0;
@@ -1150,7 +1150,7 @@ union zip_quex_err_int_w1s {
 
 static inline u64 ZIP_QUEX_ERR_INT_W1S(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x3200ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_ERR_INT_W1S: %llu\n", param1);
 	return 0;
@@ -1179,7 +1179,7 @@ union zip_quex_gcfg {
 
 static inline u64 ZIP_QUEX_GCFG(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x1A00ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_GCFG: %llu\n", param1);
 	return 0;
@@ -1204,7 +1204,7 @@ union zip_quex_map {
 
 static inline u64 ZIP_QUEX_MAP(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x1400ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_MAP: %llu\n", param1);
 	return 0;
@@ -1236,7 +1236,7 @@ union zip_quex_sbuf_addr {
 
 static inline u64 ZIP_QUEX_SBUF_ADDR(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x1000ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_SBUF_ADDR: %llu\n", param1);
 	return 0;
@@ -1276,7 +1276,7 @@ union zip_quex_sbuf_ctl {
 
 static inline u64 ZIP_QUEX_SBUF_CTL(u64 param1)
 {
-	if (((param1 <= 7)))
+	if (param1 <= 7)
 		return 0x1200ull + (param1 & 7) * 0x8ull;
 	pr_err("ZIP_QUEX_SBUF_CTL: %llu\n", param1);
 	return 0;
diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c
index d95ec526587a..ff478d826d7d 100644
--- a/drivers/crypto/ccp/psp-dev.c
+++ b/drivers/crypto/ccp/psp-dev.c
@@ -22,11 +22,17 @@
 #include <linux/delay.h>
 #include <linux/hw_random.h>
 #include <linux/ccp.h>
+#include <linux/firmware.h>
 
 #include "sp-dev.h"
 #include "psp-dev.h"
 
+#define SEV_VERSION_GREATER_OR_EQUAL(_maj, _min)	\
+		((psp_master->api_major) >= _maj &&	\
+		 (psp_master->api_minor) >= _min)
+
 #define DEVICE_NAME	"sev"
+#define SEV_FW_FILE	"amd/sev.fw"
 
 static DEFINE_MUTEX(sev_cmd_mutex);
 static struct sev_misc_dev *misc_dev;
@@ -112,6 +118,8 @@ static int sev_cmd_buffer_len(int cmd)
 	case SEV_CMD_RECEIVE_UPDATE_DATA:	return sizeof(struct sev_data_receive_update_data);
 	case SEV_CMD_RECEIVE_UPDATE_VMSA:	return sizeof(struct sev_data_receive_update_vmsa);
 	case SEV_CMD_LAUNCH_UPDATE_SECRET:	return sizeof(struct sev_data_launch_secret);
+	case SEV_CMD_DOWNLOAD_FIRMWARE:		return sizeof(struct sev_data_download_firmware);
+	case SEV_CMD_GET_ID:			return sizeof(struct sev_data_get_id);
 	default:				return 0;
 	}
 
@@ -378,6 +386,79 @@ void *psp_copy_user_blob(u64 __user uaddr, u32 len)
 }
 EXPORT_SYMBOL_GPL(psp_copy_user_blob);
 
+static int sev_get_api_version(void)
+{
+	struct sev_user_data_status *status;
+	int error, ret;
+
+	status = &psp_master->status_cmd_buf;
+	ret = sev_platform_status(status, &error);
+	if (ret) {
+		dev_err(psp_master->dev,
+			"SEV: failed to get status. Error: %#x\n", error);
+		return 1;
+	}
+
+	psp_master->api_major = status->api_major;
+	psp_master->api_minor = status->api_minor;
+	psp_master->build = status->build;
+
+	return 0;
+}
+
+/* Don't fail if SEV FW couldn't be updated. Continue with existing SEV FW */
+static int sev_update_firmware(struct device *dev)
+{
+	struct sev_data_download_firmware *data;
+	const struct firmware *firmware;
+	int ret, error, order;
+	struct page *p;
+	u64 data_size;
+
+	ret = request_firmware(&firmware, SEV_FW_FILE, dev);
+	if (ret < 0)
+		return -1;
+
+	/*
+	 * SEV FW expects the physical address given to it to be 32
+	 * byte aligned. Memory allocated has structure placed at the
+	 * beginning followed by the firmware being passed to the SEV
+	 * FW. Allocate enough memory for data structure + alignment
+	 * padding + SEV FW.
+	 */
+	data_size = ALIGN(sizeof(struct sev_data_download_firmware), 32);
+
+	order = get_order(firmware->size + data_size);
+	p = alloc_pages(GFP_KERNEL, order);
+	if (!p) {
+		ret = -1;
+		goto fw_err;
+	}
+
+	/*
+	 * Copy firmware data to a kernel allocated contiguous
+	 * memory region.
+	 */
+	data = page_address(p);
+	memcpy(page_address(p) + data_size, firmware->data, firmware->size);
+
+	data->address = __psp_pa(page_address(p) + data_size);
+	data->len = firmware->size;
+
+	ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error);
+	if (ret)
+		dev_dbg(dev, "Failed to update SEV firmware: %#x\n", error);
+	else
+		dev_info(dev, "SEV firmware update successful\n");
+
+	__free_pages(p, order);
+
+fw_err:
+	release_firmware(firmware);
+
+	return ret;
+}
+
 static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp)
 {
 	struct sev_user_data_pek_cert_import input;
@@ -430,6 +511,46 @@ static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp)
 	return ret;
 }
 
+static int sev_ioctl_do_get_id(struct sev_issue_cmd *argp)
+{
+	struct sev_data_get_id *data;
+	u64 data_size, user_size;
+	void *id_blob, *mem;
+	int ret;
+
+	/* SEV GET_ID available from SEV API v0.16 and up */
+	if (!SEV_VERSION_GREATER_OR_EQUAL(0, 16))
+		return -ENOTSUPP;
+
+	/* SEV FW expects the buffer it fills with the ID to be
+	 * 8-byte aligned. Memory allocated should be enough to
+	 * hold data structure + alignment padding + memory
+	 * where SEV FW writes the ID.
+	 */
+	data_size = ALIGN(sizeof(struct sev_data_get_id), 8);
+	user_size = sizeof(struct sev_user_data_get_id);
+
+	mem = kzalloc(data_size + user_size, GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	data = mem;
+	id_blob = mem + data_size;
+
+	data->address = __psp_pa(id_blob);
+	data->len = user_size;
+
+	ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, data, &argp->error);
+	if (!ret) {
+		if (copy_to_user((void __user *)argp->data, id_blob, data->len))
+			ret = -EFAULT;
+	}
+
+	kfree(mem);
+
+	return ret;
+}
+
 static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp)
 {
 	struct sev_user_data_pdh_cert_export input;
@@ -567,6 +688,9 @@ static long sev_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
 	case SEV_PDH_CERT_EXPORT:
 		ret = sev_ioctl_do_pdh_export(&input);
 		break;
+	case SEV_GET_ID:
+		ret = sev_ioctl_do_get_id(&input);
+		break;
 	default:
 		ret = -EINVAL;
 		goto out;
@@ -750,7 +874,6 @@ EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user);
 
 void psp_pci_init(void)
 {
-	struct sev_user_data_status *status;
 	struct sp_device *sp;
 	int error, rc;
 
@@ -760,6 +883,13 @@ void psp_pci_init(void)
 
 	psp_master = sp->psp_data;
 
+	if (sev_get_api_version())
+		goto err;
+
+	if (SEV_VERSION_GREATER_OR_EQUAL(0, 15) &&
+	    sev_update_firmware(psp_master->dev) == 0)
+		sev_get_api_version();
+
 	/* Initialize the platform */
 	rc = sev_platform_init(&error);
 	if (rc) {
@@ -767,16 +897,9 @@ void psp_pci_init(void)
 		goto err;
 	}
 
-	/* Display SEV firmware version */
-	status = &psp_master->status_cmd_buf;
-	rc = sev_platform_status(status, &error);
-	if (rc) {
-		dev_err(sp->dev, "SEV: failed to get status error %#x\n", error);
-		goto err;
-	}
+	dev_info(sp->dev, "SEV API:%d.%d build:%d\n", psp_master->api_major,
+		 psp_master->api_minor, psp_master->build);
 
-	dev_info(sp->dev, "SEV API:%d.%d build:%d\n", status->api_major,
-		 status->api_minor, status->build);
 	return;
 
 err:
diff --git a/drivers/crypto/ccp/psp-dev.h b/drivers/crypto/ccp/psp-dev.h
index c81f0b11287a..c7e9098a233c 100644
--- a/drivers/crypto/ccp/psp-dev.h
+++ b/drivers/crypto/ccp/psp-dev.h
@@ -78,6 +78,10 @@ struct psp_device {
 	struct sev_misc_dev *sev_misc;
 	struct sev_user_data_status status_cmd_buf;
 	struct sev_data_init init_cmd_buf;
+
+	u8 api_major;
+	u8 api_minor;
+	u8 build;
 };
 
 #endif /* __PSP_DEV_H */
diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c
index df98f7afe645..d2810c183b73 100644
--- a/drivers/crypto/ccree/cc_cipher.c
+++ b/drivers/crypto/ccree/cc_cipher.c
@@ -42,6 +42,7 @@ struct cc_cipher_ctx {
 	int cipher_mode;
 	int flow_mode;
 	unsigned int flags;
+	bool hw_key;
 	struct cc_user_key_info user;
 	struct cc_hw_key_info hw;
 	struct crypto_shash *shash_tfm;
@@ -49,6 +50,13 @@ struct cc_cipher_ctx {
 
 static void cc_cipher_complete(struct device *dev, void *cc_req, int err);
 
+static inline bool cc_is_hw_key(struct crypto_tfm *tfm)
+{
+	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
+
+	return ctx_p->hw_key;
+}
+
 static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size)
 {
 	switch (ctx_p->flow_mode) {
@@ -211,7 +219,7 @@ struct tdes_keys {
 	u8	key3[DES_KEY_SIZE];
 };
 
-static enum cc_hw_crypto_key hw_key_to_cc_hw_key(int slot_num)
+static enum cc_hw_crypto_key cc_slot_to_hw_key(int slot_num)
 {
 	switch (slot_num) {
 	case 0:
@@ -226,69 +234,100 @@ static enum cc_hw_crypto_key hw_key_to_cc_hw_key(int slot_num)
 	return END_OF_KEYS;
 }
 
-static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key,
-			    unsigned int keylen)
+static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key,
+			     unsigned int keylen)
 {
 	struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm);
 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
 	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
-	u32 tmp[DES3_EDE_EXPKEY_WORDS];
-	struct cc_crypto_alg *cc_alg =
-			container_of(tfm->__crt_alg, struct cc_crypto_alg,
-				     skcipher_alg.base);
-	unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize;
+	struct cc_hkey_info hki;
 
-	dev_dbg(dev, "Setting key in context @%p for %s. keylen=%u\n",
+	dev_dbg(dev, "Setting HW key in context @%p for %s. keylen=%u\n",
 		ctx_p, crypto_tfm_alg_name(tfm), keylen);
 	dump_byte_array("key", (u8 *)key, keylen);
 
 	/* STAT_PHASE_0: Init and sanity checks */
 
+	/* This check the size of the hardware key token */
+	if (keylen != sizeof(hki)) {
+		dev_err(dev, "Unsupported HW key size %d.\n", keylen);
+		crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	if (ctx_p->flow_mode != S_DIN_to_AES) {
+		dev_err(dev, "HW key not supported for non-AES flows\n");
+		return -EINVAL;
+	}
+
+	memcpy(&hki, key, keylen);
+
+	/* The real key len for crypto op is the size of the HW key
+	 * referenced by the HW key slot, not the hardware key token
+	 */
+	keylen = hki.keylen;
+
 	if (validate_keys_sizes(ctx_p, keylen)) {
 		dev_err(dev, "Unsupported key size %d.\n", keylen);
 		crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 		return -EINVAL;
 	}
 
-	if (cc_is_hw_key(tfm)) {
-		/* setting HW key slots */
-		struct arm_hw_key_info *hki = (struct arm_hw_key_info *)key;
+	ctx_p->hw.key1_slot = cc_slot_to_hw_key(hki.hw_key1);
+	if (ctx_p->hw.key1_slot == END_OF_KEYS) {
+		dev_err(dev, "Unsupported hw key1 number (%d)\n", hki.hw_key1);
+		return -EINVAL;
+	}
 
-		if (ctx_p->flow_mode != S_DIN_to_AES) {
-			dev_err(dev, "HW key not supported for non-AES flows\n");
+	if (ctx_p->cipher_mode == DRV_CIPHER_XTS ||
+	    ctx_p->cipher_mode == DRV_CIPHER_ESSIV ||
+	    ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER) {
+		if (hki.hw_key1 == hki.hw_key2) {
+			dev_err(dev, "Illegal hw key numbers (%d,%d)\n",
+				hki.hw_key1, hki.hw_key2);
 			return -EINVAL;
 		}
-
-		ctx_p->hw.key1_slot = hw_key_to_cc_hw_key(hki->hw_key1);
-		if (ctx_p->hw.key1_slot == END_OF_KEYS) {
-			dev_err(dev, "Unsupported hw key1 number (%d)\n",
-				hki->hw_key1);
+		ctx_p->hw.key2_slot = cc_slot_to_hw_key(hki.hw_key2);
+		if (ctx_p->hw.key2_slot == END_OF_KEYS) {
+			dev_err(dev, "Unsupported hw key2 number (%d)\n",
+				hki.hw_key2);
 			return -EINVAL;
 		}
+	}
 
-		if (ctx_p->cipher_mode == DRV_CIPHER_XTS ||
-		    ctx_p->cipher_mode == DRV_CIPHER_ESSIV ||
-		    ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER) {
-			if (hki->hw_key1 == hki->hw_key2) {
-				dev_err(dev, "Illegal hw key numbers (%d,%d)\n",
-					hki->hw_key1, hki->hw_key2);
-				return -EINVAL;
-			}
-			ctx_p->hw.key2_slot =
-				hw_key_to_cc_hw_key(hki->hw_key2);
-			if (ctx_p->hw.key2_slot == END_OF_KEYS) {
-				dev_err(dev, "Unsupported hw key2 number (%d)\n",
-					hki->hw_key2);
-				return -EINVAL;
-			}
-		}
+	ctx_p->keylen = keylen;
+	ctx_p->hw_key = true;
+	dev_dbg(dev, "cc_is_hw_key ret 0");
+
+	return 0;
+}
+
+static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key,
+			    unsigned int keylen)
+{
+	struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm);
+	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
+	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
+	u32 tmp[DES3_EDE_EXPKEY_WORDS];
+	struct cc_crypto_alg *cc_alg =
+			container_of(tfm->__crt_alg, struct cc_crypto_alg,
+				     skcipher_alg.base);
+	unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize;
+
+	dev_dbg(dev, "Setting key in context @%p for %s. keylen=%u\n",
+		ctx_p, crypto_tfm_alg_name(tfm), keylen);
+	dump_byte_array("key", (u8 *)key, keylen);
 
-		ctx_p->keylen = keylen;
-		dev_dbg(dev, "cc_is_hw_key ret 0");
+	/* STAT_PHASE_0: Init and sanity checks */
 
-		return 0;
+	if (validate_keys_sizes(ctx_p, keylen)) {
+		dev_err(dev, "Unsupported key size %d.\n", keylen);
+		crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
 	}
 
+	ctx_p->hw_key = false;
+
 	/*
 	 * Verify DES weak keys
 	 * Note that we're dropping the expanded key since the
@@ -734,6 +773,241 @@ static int cc_cipher_decrypt(struct skcipher_request *req)
 
 /* Block cipher alg */
 static const struct cc_alg_template skcipher_algs[] = {
+	{
+		.name = "xts(paes)",
+		.driver_name = "xts-paes-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_XTS,
+		.flow_mode = S_DIN_to_AES,
+		.min_hw_rev = CC_HW_REV_630,
+	},
+	{
+		.name = "xts512(paes)",
+		.driver_name = "xts-paes-du512-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_XTS,
+		.flow_mode = S_DIN_to_AES,
+		.data_unit = 512,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "xts4096(paes)",
+		.driver_name = "xts-paes-du4096-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_XTS,
+		.flow_mode = S_DIN_to_AES,
+		.data_unit = 4096,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "essiv(paes)",
+		.driver_name = "essiv-paes-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_ESSIV,
+		.flow_mode = S_DIN_to_AES,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "essiv512(paes)",
+		.driver_name = "essiv-paes-du512-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_ESSIV,
+		.flow_mode = S_DIN_to_AES,
+		.data_unit = 512,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "essiv4096(paes)",
+		.driver_name = "essiv-paes-du4096-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_ESSIV,
+		.flow_mode = S_DIN_to_AES,
+		.data_unit = 4096,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "bitlocker(paes)",
+		.driver_name = "bitlocker-paes-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_BITLOCKER,
+		.flow_mode = S_DIN_to_AES,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "bitlocker512(paes)",
+		.driver_name = "bitlocker-paes-du512-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_BITLOCKER,
+		.flow_mode = S_DIN_to_AES,
+		.data_unit = 512,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "bitlocker4096(paes)",
+		.driver_name = "bitlocker-paes-du4096-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize =  CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_BITLOCKER,
+		.flow_mode = S_DIN_to_AES,
+		.data_unit = 4096,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "ecb(paes)",
+		.driver_name = "ecb-paes-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = 0,
+			},
+		.cipher_mode = DRV_CIPHER_ECB,
+		.flow_mode = S_DIN_to_AES,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "cbc(paes)",
+		.driver_name = "cbc-paes-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+		},
+		.cipher_mode = DRV_CIPHER_CBC,
+		.flow_mode = S_DIN_to_AES,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "ofb(paes)",
+		.driver_name = "ofb-paes-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_OFB,
+		.flow_mode = S_DIN_to_AES,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "cts1(cbc(paes))",
+		.driver_name = "cts1-cbc-paes-ccree",
+		.blocksize = AES_BLOCK_SIZE,
+		.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_CBC_CTS,
+		.flow_mode = S_DIN_to_AES,
+		.min_hw_rev = CC_HW_REV_712,
+	},
+	{
+		.name = "ctr(paes)",
+		.driver_name = "ctr-paes-ccree",
+		.blocksize = 1,
+		.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.template_skcipher = {
+			.setkey = cc_cipher_sethkey,
+			.encrypt = cc_cipher_encrypt,
+			.decrypt = cc_cipher_decrypt,
+			.min_keysize = CC_HW_KEY_SIZE,
+			.max_keysize = CC_HW_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+			},
+		.cipher_mode = DRV_CIPHER_CTR,
+		.flow_mode = S_DIN_to_AES,
+		.min_hw_rev = CC_HW_REV_712,
+	},
 	{
 		.name = "xts(aes)",
 		.driver_name = "xts-aes-ccree",
diff --git a/drivers/crypto/ccree/cc_cipher.h b/drivers/crypto/ccree/cc_cipher.h
index 2a2a6f46c515..68444cfa936b 100644
--- a/drivers/crypto/ccree/cc_cipher.h
+++ b/drivers/crypto/ccree/cc_cipher.h
@@ -13,18 +13,6 @@
 #include "cc_driver.h"
 #include "cc_buffer_mgr.h"
 
-/* Crypto cipher flags */
-#define CC_CRYPTO_CIPHER_KEY_KFDE0	BIT(0)
-#define CC_CRYPTO_CIPHER_KEY_KFDE1	BIT(1)
-#define CC_CRYPTO_CIPHER_KEY_KFDE2	BIT(2)
-#define CC_CRYPTO_CIPHER_KEY_KFDE3	BIT(3)
-#define CC_CRYPTO_CIPHER_DU_SIZE_512B	BIT(4)
-
-#define CC_CRYPTO_CIPHER_KEY_KFDE_MASK (CC_CRYPTO_CIPHER_KEY_KFDE0 | \
-					CC_CRYPTO_CIPHER_KEY_KFDE1 | \
-					CC_CRYPTO_CIPHER_KEY_KFDE2 | \
-					CC_CRYPTO_CIPHER_KEY_KFDE3)
-
 struct cipher_req_ctx {
 	struct async_gen_req_ctx gen_ctx;
 	enum cc_req_dma_buf_type dma_buf_type;
@@ -42,18 +30,12 @@ int cc_cipher_alloc(struct cc_drvdata *drvdata);
 
 int cc_cipher_free(struct cc_drvdata *drvdata);
 
-struct arm_hw_key_info {
-	int hw_key1;
-	int hw_key2;
-};
+struct cc_hkey_info {
+	u16 keylen;
+	u8 hw_key1;
+	u8 hw_key2;
+} __packed;
 
-/*
- * This is a stub function that will replaced when we
- * implement secure keys
- */
-static inline bool cc_is_hw_key(struct crypto_tfm *tfm)
-{
-	return false;
-}
+#define CC_HW_KEY_SIZE sizeof(struct cc_hkey_info)
 
 #endif /*__CC_CIPHER_H__*/
diff --git a/drivers/crypto/ccree/cc_debugfs.c b/drivers/crypto/ccree/cc_debugfs.c
index 08f8db489cf0..5ca184e42483 100644
--- a/drivers/crypto/ccree/cc_debugfs.c
+++ b/drivers/crypto/ccree/cc_debugfs.c
@@ -26,7 +26,8 @@ struct cc_debugfs_ctx {
 static struct dentry *cc_debugfs_dir;
 
 static struct debugfs_reg32 debug_regs[] = {
-	CC_DEBUG_REG(HOST_SIGNATURE),
+	{ .name = "SIGNATURE" }, /* Must be 0th */
+	{ .name = "VERSION" }, /* Must be 1st */
 	CC_DEBUG_REG(HOST_IRR),
 	CC_DEBUG_REG(HOST_POWER_DOWN_EN),
 	CC_DEBUG_REG(AXIM_MON_ERR),
@@ -34,7 +35,6 @@ static struct debugfs_reg32 debug_regs[] = {
 	CC_DEBUG_REG(HOST_IMR),
 	CC_DEBUG_REG(AXIM_CFG),
 	CC_DEBUG_REG(AXIM_CACHE_PARAMS),
-	CC_DEBUG_REG(HOST_VERSION),
 	CC_DEBUG_REG(GPR_HOST),
 	CC_DEBUG_REG(AXIM_MON_COMP),
 };
@@ -58,6 +58,9 @@ int cc_debugfs_init(struct cc_drvdata *drvdata)
 	struct debugfs_regset32 *regset;
 	struct dentry *file;
 
+	debug_regs[0].offset = drvdata->sig_offset;
+	debug_regs[1].offset = drvdata->ver_offset;
+
 	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
 		return -ENOMEM;
diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c
index 89ce013ae093..bd974fef05e4 100644
--- a/drivers/crypto/ccree/cc_driver.c
+++ b/drivers/crypto/ccree/cc_driver.c
@@ -168,14 +168,14 @@ int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe)
 	val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));
 
 	if (is_probe)
-		dev_info(dev, "Cache params previous: 0x%08X\n", val);
+		dev_dbg(dev, "Cache params previous: 0x%08X\n", val);
 
 	cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), cache_params);
 	val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));
 
 	if (is_probe)
-		dev_info(dev, "Cache params current: 0x%08X (expect: 0x%08X)\n",
-			 val, cache_params);
+		dev_dbg(dev, "Cache params current: 0x%08X (expect: 0x%08X)\n",
+			val, cache_params);
 
 	return 0;
 }
@@ -190,6 +190,7 @@ static int init_cc_resources(struct platform_device *plat_dev)
 	u64 dma_mask;
 	const struct cc_hw_data *hw_rev;
 	const struct of_device_id *dev_id;
+	struct clk *clk;
 	int rc = 0;
 
 	new_drvdata = devm_kzalloc(dev, sizeof(*new_drvdata), GFP_KERNEL);
@@ -207,15 +208,36 @@ static int init_cc_resources(struct platform_device *plat_dev)
 	if (hw_rev->rev >= CC_HW_REV_712) {
 		new_drvdata->hash_len_sz = HASH_LEN_SIZE_712;
 		new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP);
+		new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_712);
+		new_drvdata->ver_offset = CC_REG(HOST_VERSION_712);
 	} else {
 		new_drvdata->hash_len_sz = HASH_LEN_SIZE_630;
 		new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP8);
+		new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_630);
+		new_drvdata->ver_offset = CC_REG(HOST_VERSION_630);
 	}
 
 	platform_set_drvdata(plat_dev, new_drvdata);
 	new_drvdata->plat_dev = plat_dev;
 
-	new_drvdata->clk = of_clk_get(np, 0);
+	clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(clk))
+		switch (PTR_ERR(clk)) {
+		/* Clock is optional so this might be fine */
+		case -ENOENT:
+			break;
+
+		/* Clock not available, let's try again soon */
+		case -EPROBE_DEFER:
+			return -EPROBE_DEFER;
+
+		default:
+			dev_err(dev, "Error getting clock: %ld\n",
+				PTR_ERR(clk));
+			return PTR_ERR(clk);
+		}
+	new_drvdata->clk = clk;
+
 	new_drvdata->coherent = of_dma_is_coherent(np);
 
 	/* Get device resources */
@@ -265,7 +287,7 @@ static int init_cc_resources(struct platform_device *plat_dev)
 	}
 
 	if (rc) {
-		dev_err(dev, "Failed in dma_set_mask, mask=%pad\n", &dma_mask);
+		dev_err(dev, "Failed in dma_set_mask, mask=%llx\n", dma_mask);
 		return rc;
 	}
 
@@ -276,7 +298,7 @@ static int init_cc_resources(struct platform_device *plat_dev)
 	}
 
 	/* Verify correct mapping */
-	signature_val = cc_ioread(new_drvdata, CC_REG(HOST_SIGNATURE));
+	signature_val = cc_ioread(new_drvdata, new_drvdata->sig_offset);
 	if (signature_val != hw_rev->sig) {
 		dev_err(dev, "Invalid CC signature: SIGNATURE=0x%08X != expected=0x%08X\n",
 			signature_val, hw_rev->sig);
@@ -287,7 +309,7 @@ static int init_cc_resources(struct platform_device *plat_dev)
 
 	/* Display HW versions */
 	dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X, Driver version %s\n",
-		 hw_rev->name, cc_ioread(new_drvdata, CC_REG(HOST_VERSION)),
+		 hw_rev->name, cc_ioread(new_drvdata, new_drvdata->ver_offset),
 		 DRV_MODULE_VERSION);
 
 	rc = init_cc_regs(new_drvdata, true);
diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h
index 2048fdeb9579..95f82b2d1e70 100644
--- a/drivers/crypto/ccree/cc_driver.h
+++ b/drivers/crypto/ccree/cc_driver.h
@@ -129,6 +129,8 @@ struct cc_drvdata {
 	enum cc_hw_rev hw_rev;
 	u32 hash_len_sz;
 	u32 axim_mon_offset;
+	u32 sig_offset;
+	u32 ver_offset;
 };
 
 struct cc_crypto_alg {
diff --git a/drivers/crypto/ccree/cc_host_regs.h b/drivers/crypto/ccree/cc_host_regs.h
index f51001898ca1..616b2e1c41ba 100644
--- a/drivers/crypto/ccree/cc_host_regs.h
+++ b/drivers/crypto/ccree/cc_host_regs.h
@@ -45,7 +45,8 @@
 #define CC_HOST_ICR_DSCRPTR_WATERMARK_QUEUE0_CLEAR_BIT_SIZE	0x1UL
 #define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SHIFT	0x17UL
 #define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SIZE	0x1UL
-#define CC_HOST_SIGNATURE_REG_OFFSET	0xA24UL
+#define CC_HOST_SIGNATURE_712_REG_OFFSET	0xA24UL
+#define CC_HOST_SIGNATURE_630_REG_OFFSET	0xAC8UL
 #define CC_HOST_SIGNATURE_VALUE_BIT_SHIFT	0x0UL
 #define CC_HOST_SIGNATURE_VALUE_BIT_SIZE	0x20UL
 #define CC_HOST_BOOT_REG_OFFSET	0xA28UL
@@ -105,7 +106,8 @@
 #define CC_HOST_BOOT_ONLY_ENCRYPT_LOCAL_BIT_SIZE	0x1UL
 #define CC_HOST_BOOT_AES_EXISTS_LOCAL_BIT_SHIFT	0x1EUL
 #define CC_HOST_BOOT_AES_EXISTS_LOCAL_BIT_SIZE	0x1UL
-#define CC_HOST_VERSION_REG_OFFSET	0xA40UL
+#define CC_HOST_VERSION_712_REG_OFFSET	0xA40UL
+#define CC_HOST_VERSION_630_REG_OFFSET	0xAD8UL
 #define CC_HOST_VERSION_VALUE_BIT_SHIFT	0x0UL
 #define CC_HOST_VERSION_VALUE_BIT_SIZE	0x20UL
 #define CC_HOST_KFDE0_VALID_REG_OFFSET	0xA60UL
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index 59fe6631e73e..b916c4eb608c 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -203,13 +203,8 @@ static inline void chcr_handle_aead_resp(struct aead_request *req,
 					 int err)
 {
 	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
-	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
-	struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
 
-	chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
-	if (reqctx->b0_dma)
-		dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma,
-				 reqctx->b0_len, DMA_BIDIRECTIONAL);
+	chcr_aead_common_exit(req);
 	if (reqctx->verify == VERIFY_SW) {
 		chcr_verify_tag(req, input, &err);
 		reqctx->verify = VERIFY_HW;
@@ -638,7 +633,6 @@ static int chcr_sg_ent_in_wr(struct scatterlist *src,
 		src = sg_next(src);
 		srcskip = 0;
 	}
-
 	if (sg_dma_len(dst) == dstskip) {
 		dst = sg_next(dst);
 		dstskip = 0;
@@ -688,6 +682,7 @@ static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
 	int err;
 
 	SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
+
 	skcipher_request_set_tfm(subreq, cipher);
 	skcipher_request_set_callback(subreq, flags, NULL, NULL);
 	skcipher_request_set_crypt(subreq, src, dst,
@@ -760,13 +755,13 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
 
 	nents = sg_nents_xlen(reqctx->dstsg,  wrparam->bytes, CHCR_DST_SG_SIZE,
 			      reqctx->dst_ofst);
-	dst_size = get_space_for_phys_dsgl(nents + 1);
+	dst_size = get_space_for_phys_dsgl(nents);
 	kctx_len = roundup(ablkctx->enckey_len, 16);
 	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
 	nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes,
 				  CHCR_SRC_SG_SIZE, reqctx->src_ofst);
-	temp = reqctx->imm ? roundup(IV + wrparam->req->nbytes, 16) :
-				     (sgl_len(nents + MIN_CIPHER_SG) * 8);
+	temp = reqctx->imm ? roundup(wrparam->bytes, 16) :
+				     (sgl_len(nents) * 8);
 	transhdr_len += temp;
 	transhdr_len = roundup(transhdr_len, 16);
 	skb = alloc_skb(SGE_MAX_WR_LEN, flags);
@@ -788,7 +783,7 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
 							 ablkctx->ciph_mode,
 							 0, 0, IV >> 1);
 	chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
-							  0, 0, dst_size);
+							  0, 1, dst_size);
 
 	chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
 	if ((reqctx->op == CHCR_DECRYPT_OP) &&
@@ -818,8 +813,8 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
 	chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid);
 
 	atomic_inc(&adap->chcr_stats.cipher_rqst);
-	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len
-		+(reqctx->imm ? (IV + wrparam->bytes) : 0);
+	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + IV
+		+ (reqctx->imm ? (wrparam->bytes) : 0);
 	create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0,
 		    transhdr_len, temp,
 			ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
@@ -1022,7 +1017,7 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv,
 	ret = crypto_cipher_setkey(cipher, key, keylen);
 	if (ret)
 		goto out;
-	/*H/W sends the encrypted IV in dsgl when AADIVDROP bit is 0*/
+	crypto_cipher_encrypt_one(cipher, iv, iv);
 	for (i = 0; i < round8; i++)
 		gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
 
@@ -1113,16 +1108,8 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
 		goto complete;
 	}
 
-	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
-					    c_ctx(tfm)->tx_qidx))) {
-		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
-			err = -EBUSY;
-			goto unmap;
-		}
-
-	}
 	if (!reqctx->imm) {
-		bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1,
+		bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 0,
 					  CIP_SPACE_LEFT(ablkctx->enckey_len),
 					  reqctx->src_ofst, reqctx->dst_ofst);
 		if ((bytes + reqctx->processed) >= req->nbytes)
@@ -1133,11 +1120,7 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
 		/*CTR mode counter overfloa*/
 		bytes  = req->nbytes - reqctx->processed;
 	}
-	dma_sync_single_for_cpu(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
-				reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
 	err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
-	dma_sync_single_for_device(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
-				   reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
 	if (err)
 		goto unmap;
 
@@ -1212,7 +1195,6 @@ static int process_cipher(struct ablkcipher_request *req,
 
 		dnents = sg_nents_xlen(req->dst, req->nbytes,
 				       CHCR_DST_SG_SIZE, 0);
-		dnents += 1; // IV
 		phys_dsgl = get_space_for_phys_dsgl(dnents);
 		kctx_len = roundup(ablkctx->enckey_len, 16);
 		transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
@@ -1225,8 +1207,7 @@ static int process_cipher(struct ablkcipher_request *req,
 	}
 
 	if (!reqctx->imm) {
-		bytes = chcr_sg_ent_in_wr(req->src, req->dst,
-					  MIN_CIPHER_SG,
+		bytes = chcr_sg_ent_in_wr(req->src, req->dst, 0,
 					  CIP_SPACE_LEFT(ablkctx->enckey_len),
 					  0, 0);
 		if ((bytes + reqctx->processed) >= req->nbytes)
@@ -1293,13 +1274,14 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req)
 {
 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 	struct sk_buff *skb = NULL;
-	int err;
+	int err, isfull = 0;
 	struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
 
 	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
 					    c_ctx(tfm)->tx_qidx))) {
+		isfull = 1;
 		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
-			return -EBUSY;
+			return -ENOSPC;
 	}
 
 	err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx],
@@ -1309,7 +1291,7 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req)
 	skb->dev = u_ctx->lldi.ports[0];
 	set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
 	chcr_send_wr(skb);
-	return -EINPROGRESS;
+	return isfull ? -EBUSY : -EINPROGRESS;
 }
 
 static int chcr_aes_decrypt(struct ablkcipher_request *req)
@@ -1317,12 +1299,13 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req)
 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 	struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
 	struct sk_buff *skb = NULL;
-	int err;
+	int err, isfull = 0;
 
 	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
 					    c_ctx(tfm)->tx_qidx))) {
+		isfull = 1;
 		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
-			return -EBUSY;
+			return -ENOSPC;
 	}
 
 	 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx],
@@ -1332,7 +1315,7 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req)
 	skb->dev = u_ctx->lldi.ports[0];
 	set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
 	chcr_send_wr(skb);
-	return -EINPROGRESS;
+	return isfull ? -EBUSY : -EINPROGRESS;
 }
 
 static int chcr_device_init(struct chcr_context *ctx)
@@ -1574,14 +1557,15 @@ static int chcr_ahash_update(struct ahash_request *req)
 	u8 remainder = 0, bs;
 	unsigned int nbytes = req->nbytes;
 	struct hash_wr_param params;
-	int error;
+	int error, isfull = 0;
 
 	bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
 	u_ctx = ULD_CTX(h_ctx(rtfm));
 	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
 					    h_ctx(rtfm)->tx_qidx))) {
+		isfull = 1;
 		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
-			return -EBUSY;
+			return -ENOSPC;
 	}
 
 	if (nbytes + req_ctx->reqlen >= bs) {
@@ -1633,7 +1617,7 @@ static int chcr_ahash_update(struct ahash_request *req)
 	set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
 	chcr_send_wr(skb);
 
-	return -EINPROGRESS;
+	return isfull ? -EBUSY : -EINPROGRESS;
 unmap:
 	chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
 	return error;
@@ -1710,15 +1694,16 @@ static int chcr_ahash_finup(struct ahash_request *req)
 	struct sk_buff *skb;
 	struct hash_wr_param params;
 	u8  bs;
-	int error;
+	int error, isfull = 0;
 
 	bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
 	u_ctx = ULD_CTX(h_ctx(rtfm));
 
 	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
 					    h_ctx(rtfm)->tx_qidx))) {
+		isfull = 1;
 		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
-			return -EBUSY;
+			return -ENOSPC;
 	}
 	chcr_init_hctx_per_wr(req_ctx);
 	error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
@@ -1777,7 +1762,7 @@ static int chcr_ahash_finup(struct ahash_request *req)
 	set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
 	chcr_send_wr(skb);
 
-	return -EINPROGRESS;
+	return isfull ? -EBUSY : -EINPROGRESS;
 unmap:
 	chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
 	return error;
@@ -1791,7 +1776,7 @@ static int chcr_ahash_digest(struct ahash_request *req)
 	struct sk_buff *skb;
 	struct hash_wr_param params;
 	u8  bs;
-	int error;
+	int error, isfull = 0;
 
 	rtfm->init(req);
 	bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
@@ -1799,8 +1784,9 @@ static int chcr_ahash_digest(struct ahash_request *req)
 	u_ctx = ULD_CTX(h_ctx(rtfm));
 	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
 					    h_ctx(rtfm)->tx_qidx))) {
+		isfull = 1;
 		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
-			return -EBUSY;
+			return -ENOSPC;
 	}
 
 	chcr_init_hctx_per_wr(req_ctx);
@@ -1856,7 +1842,7 @@ static int chcr_ahash_digest(struct ahash_request *req)
 	skb->dev = u_ctx->lldi.ports[0];
 	set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
 	chcr_send_wr(skb);
-	return -EINPROGRESS;
+	return isfull ? -EBUSY : -EINPROGRESS;
 unmap:
 	chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
 	return error;
@@ -1875,11 +1861,6 @@ static int chcr_ahash_continue(struct ahash_request *req)
 
 	bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
 	u_ctx = ULD_CTX(h_ctx(rtfm));
-	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
-					    h_ctx(rtfm)->tx_qidx))) {
-		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
-			return -EBUSY;
-	}
 	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
 	params.kctx_len = roundup(params.alg_prm.result_size, 16);
 	if (is_hmac(crypto_ahash_tfm(rtfm))) {
@@ -2192,22 +2173,35 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
 	}
 }
 
-static int chcr_aead_common_init(struct aead_request *req,
-				 unsigned short op_type)
+inline void chcr_aead_common_exit(struct aead_request *req)
+{
+	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
+
+	chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
+}
+
+static int chcr_aead_common_init(struct aead_request *req)
 {
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
 	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
-	int error = -EINVAL;
 	unsigned int authsize = crypto_aead_authsize(tfm);
+	int error = -EINVAL;
 
 	/* validate key size */
 	if (aeadctx->enckey_len == 0)
 		goto err;
-	if (op_type && req->cryptlen < authsize)
+	if (reqctx->op && req->cryptlen < authsize)
 		goto err;
+	if (reqctx->b0_len)
+		reqctx->scratch_pad = reqctx->iv + IV;
+	else
+		reqctx->scratch_pad = NULL;
+
 	error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
-				  op_type);
+				  reqctx->op);
 	if (error) {
 		error = -ENOMEM;
 		goto err;
@@ -2244,7 +2238,7 @@ static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
 	aead_request_set_tfm(subreq, aeadctx->sw_cipher);
 	aead_request_set_callback(subreq, req->base.flags,
 				  req->base.complete, req->base.data);
-	 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
+	aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
 				 req->iv);
 	 aead_request_set_ad(subreq, req->assoclen);
 	return op_type ? crypto_aead_decrypt(subreq) :
@@ -2253,8 +2247,7 @@ static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
 
 static struct sk_buff *create_authenc_wr(struct aead_request *req,
 					 unsigned short qid,
-					 int size,
-					 unsigned short op_type)
+					 int size)
 {
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
@@ -2278,18 +2271,20 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
 	if (req->cryptlen == 0)
 		return NULL;
 
-	reqctx->b0_dma = 0;
+	reqctx->b0_len = 0;
+	error = chcr_aead_common_init(req);
+	if (error)
+		return ERR_PTR(error);
+
 	if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL ||
-	subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
+		subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
 		null = 1;
 		assoclen = 0;
+		reqctx->aad_nents = 0;
 	}
-	error = chcr_aead_common_init(req, op_type);
-	if (error)
-		return ERR_PTR(error);
 	dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
 	dnents += sg_nents_xlen(req->dst, req->cryptlen +
-		(op_type ? -authsize : authsize), CHCR_DST_SG_SIZE,
+		(reqctx->op ? -authsize : authsize), CHCR_DST_SG_SIZE,
 		req->assoclen);
 	dnents += MIN_AUTH_SG; // For IV
 
@@ -2306,11 +2301,10 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
 	transhdr_len = roundup(transhdr_len, 16);
 
 	if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
-				    transhdr_len, op_type)) {
+				    transhdr_len, reqctx->op)) {
 		atomic_inc(&adap->chcr_stats.fallback);
-		chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
-				    op_type);
-		return ERR_PTR(chcr_aead_fallback(req, op_type));
+		chcr_aead_common_exit(req);
+		return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
 	}
 	skb = alloc_skb(SGE_MAX_WR_LEN, flags);
 	if (!skb) {
@@ -2320,7 +2314,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
 
 	chcr_req = __skb_put_zero(skb, transhdr_len);
 
-	temp  = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
+	temp  = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
 
 	/*
 	 * Input order	is AAD,IV and Payload. where IV should be included as
@@ -2344,8 +2338,8 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
 		temp = CHCR_SCMD_CIPHER_MODE_AES_CTR;
 	else
 		temp = CHCR_SCMD_CIPHER_MODE_AES_CBC;
-	chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
-					(op_type == CHCR_ENCRYPT_OP) ? 1 : 0,
+	chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op,
+					(reqctx->op == CHCR_ENCRYPT_OP) ? 1 : 0,
 					temp,
 					actx->auth_mode, aeadctx->hmac_ctrl,
 					IV >> 1);
@@ -2353,7 +2347,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
 					 0, 0, dst_size);
 
 	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
-	if (op_type == CHCR_ENCRYPT_OP ||
+	if (reqctx->op == CHCR_ENCRYPT_OP ||
 		subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
 		subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL)
 		memcpy(chcr_req->key_ctx.key, aeadctx->key,
@@ -2376,20 +2370,18 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
 	}
 	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
 	ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
-	chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
-	chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
+	chcr_add_aead_dst_ent(req, phys_cpl, assoclen, qid);
+	chcr_add_aead_src_ent(req, ulptx, assoclen);
 	atomic_inc(&adap->chcr_stats.cipher_rqst);
 	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
 		kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0);
 	create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
 		   transhdr_len, temp, 0);
 	reqctx->skb = skb;
-	reqctx->op = op_type;
 
 	return skb;
 err:
-	chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
-			    op_type);
+	chcr_aead_common_exit(req);
 
 	return ERR_PTR(error);
 }
@@ -2408,11 +2400,14 @@ int chcr_aead_dma_map(struct device *dev,
 				-authsize : authsize);
 	if (!req->cryptlen || !dst_size)
 		return 0;
-	reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV,
+	reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len),
 					DMA_BIDIRECTIONAL);
 	if (dma_mapping_error(dev, reqctx->iv_dma))
 		return -ENOMEM;
-
+	if (reqctx->b0_len)
+		reqctx->b0_dma = reqctx->iv_dma + IV;
+	else
+		reqctx->b0_dma = 0;
 	if (req->src == req->dst) {
 		error = dma_map_sg(dev, req->src, sg_nents(req->src),
 				   DMA_BIDIRECTIONAL);
@@ -2452,7 +2447,7 @@ void chcr_aead_dma_unmap(struct device *dev,
 	if (!req->cryptlen || !dst_size)
 		return;
 
-	dma_unmap_single(dev, reqctx->iv_dma, IV,
+	dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len),
 					DMA_BIDIRECTIONAL);
 	if (req->src == req->dst) {
 		dma_unmap_sg(dev, req->src, sg_nents(req->src),
@@ -2467,8 +2462,7 @@ void chcr_aead_dma_unmap(struct device *dev,
 
 void chcr_add_aead_src_ent(struct aead_request *req,
 			   struct ulptx_sgl *ulptx,
-			   unsigned int assoclen,
-			   unsigned short op_type)
+			   unsigned int assoclen)
 {
 	struct ulptx_walk ulp_walk;
 	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
@@ -2476,7 +2470,7 @@ void chcr_add_aead_src_ent(struct aead_request *req,
 	if (reqctx->imm) {
 		u8 *buf = (u8 *)ulptx;
 
-		if (reqctx->b0_dma) {
+		if (reqctx->b0_len) {
 			memcpy(buf, reqctx->scratch_pad, reqctx->b0_len);
 			buf += reqctx->b0_len;
 		}
@@ -2489,7 +2483,7 @@ void chcr_add_aead_src_ent(struct aead_request *req,
 				   buf, req->cryptlen, req->assoclen);
 	} else {
 		ulptx_walk_init(&ulp_walk, ulptx);
-		if (reqctx->b0_dma)
+		if (reqctx->b0_len)
 			ulptx_walk_add_page(&ulp_walk, reqctx->b0_len,
 					    &reqctx->b0_dma);
 		ulptx_walk_add_sg(&ulp_walk, req->src, assoclen, 0);
@@ -2503,7 +2497,6 @@ void chcr_add_aead_src_ent(struct aead_request *req,
 void chcr_add_aead_dst_ent(struct aead_request *req,
 			   struct cpl_rx_phys_dsgl *phys_cpl,
 			   unsigned int assoclen,
-			   unsigned short op_type,
 			   unsigned short qid)
 {
 	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
@@ -2513,32 +2506,30 @@ void chcr_add_aead_dst_ent(struct aead_request *req,
 	u32 temp;
 
 	dsgl_walk_init(&dsgl_walk, phys_cpl);
-	if (reqctx->b0_dma)
+	if (reqctx->b0_len)
 		dsgl_walk_add_page(&dsgl_walk, reqctx->b0_len, &reqctx->b0_dma);
 	dsgl_walk_add_sg(&dsgl_walk, req->dst, assoclen, 0);
 	dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma);
-	temp = req->cryptlen + (op_type ? -authsize : authsize);
+	temp = req->cryptlen + (reqctx->op ? -authsize : authsize);
 	dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, req->assoclen);
 	dsgl_walk_end(&dsgl_walk, qid);
 }
 
 void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
-			     struct ulptx_sgl *ulptx,
+			     void *ulptx,
 			     struct  cipher_wr_param *wrparam)
 {
 	struct ulptx_walk ulp_walk;
 	struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
+	u8 *buf = ulptx;
 
+	memcpy(buf, reqctx->iv, IV);
+	buf += IV;
 	if (reqctx->imm) {
-		u8 *buf = (u8 *)ulptx;
-
-		memcpy(buf, reqctx->iv, IV);
-		buf += IV;
 		sg_pcopy_to_buffer(req->src, sg_nents(req->src),
 				   buf, wrparam->bytes, reqctx->processed);
 	} else {
-		ulptx_walk_init(&ulp_walk, ulptx);
-		ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma);
+		ulptx_walk_init(&ulp_walk, (struct ulptx_sgl *)buf);
 		ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes,
 				  reqctx->src_ofst);
 		reqctx->srcsg = ulp_walk.last_sg;
@@ -2556,7 +2547,6 @@ void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
 	struct dsgl_walk dsgl_walk;
 
 	dsgl_walk_init(&dsgl_walk, phys_cpl);
-	dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma);
 	dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes,
 			 reqctx->dst_ofst);
 	reqctx->dstsg = dsgl_walk.last_sg;
@@ -2630,12 +2620,6 @@ int chcr_cipher_dma_map(struct device *dev,
 			struct ablkcipher_request *req)
 {
 	int error;
-	struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
-
-	reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV,
-					DMA_BIDIRECTIONAL);
-	if (dma_mapping_error(dev, reqctx->iv_dma))
-		return -ENOMEM;
 
 	if (req->src == req->dst) {
 		error = dma_map_sg(dev, req->src, sg_nents(req->src),
@@ -2658,17 +2642,12 @@ int chcr_cipher_dma_map(struct device *dev,
 
 	return 0;
 err:
-	dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
 	return -ENOMEM;
 }
 
 void chcr_cipher_dma_unmap(struct device *dev,
 			   struct ablkcipher_request *req)
 {
-	struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
-
-	dma_unmap_single(dev, reqctx->iv_dma, IV,
-					DMA_BIDIRECTIONAL);
 	if (req->src == req->dst) {
 		dma_unmap_sg(dev, req->src, sg_nents(req->src),
 				   DMA_BIDIRECTIONAL);
@@ -2738,7 +2717,8 @@ static inline int crypto_ccm_check_iv(const u8 *iv)
 static int ccm_format_packet(struct aead_request *req,
 			     struct chcr_aead_ctx *aeadctx,
 			     unsigned int sub_type,
-			     unsigned short op_type)
+			     unsigned short op_type,
+			     unsigned int assoclen)
 {
 	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
 	int rc = 0;
@@ -2748,13 +2728,13 @@ static int ccm_format_packet(struct aead_request *req,
 		memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3);
 		memcpy(reqctx->iv + 4, req->iv, 8);
 		memset(reqctx->iv + 12, 0, 4);
-		*((unsigned short *)(reqctx->scratch_pad + 16)) =
-			htons(req->assoclen - 8);
 	} else {
 		memcpy(reqctx->iv, req->iv, 16);
-		*((unsigned short *)(reqctx->scratch_pad + 16)) =
-			htons(req->assoclen);
 	}
+	if (assoclen)
+		*((unsigned short *)(reqctx->scratch_pad + 16)) =
+				htons(assoclen);
+
 	generate_b0(req, aeadctx, op_type);
 	/* zero the ctr value */
 	memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1);
@@ -2836,8 +2816,7 @@ static int aead_ccm_validate_input(unsigned short op_type,
 
 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
 					  unsigned short qid,
-					  int size,
-					  unsigned short op_type)
+					  int size)
 {
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
@@ -2855,22 +2834,20 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
 		GFP_ATOMIC;
 	struct adapter *adap = padap(a_ctx(tfm)->dev);
 
-	reqctx->b0_dma = 0;
 	sub_type = get_aead_subtype(tfm);
 	if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
 		assoclen -= 8;
-	error = chcr_aead_common_init(req, op_type);
+	reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0);
+	error = chcr_aead_common_init(req);
 	if (error)
 		return ERR_PTR(error);
 
-
-	reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0);
-	error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type);
+	error = aead_ccm_validate_input(reqctx->op, req, aeadctx, sub_type);
 	if (error)
 		goto err;
 	dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
 	dnents += sg_nents_xlen(req->dst, req->cryptlen
-			+ (op_type ? -authsize : authsize),
+			+ (reqctx->op ? -authsize : authsize),
 			CHCR_DST_SG_SIZE, req->assoclen);
 	dnents += MIN_CCM_SG; // For IV and B0
 	dst_size = get_space_for_phys_dsgl(dnents);
@@ -2886,11 +2863,10 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
 	transhdr_len = roundup(transhdr_len, 16);
 
 	if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE -
-				    reqctx->b0_len, transhdr_len, op_type)) {
+				    reqctx->b0_len, transhdr_len, reqctx->op)) {
 		atomic_inc(&adap->chcr_stats.fallback);
-		chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
-				    op_type);
-		return ERR_PTR(chcr_aead_fallback(req, op_type));
+		chcr_aead_common_exit(req);
+		return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
 	}
 	skb = alloc_skb(SGE_MAX_WR_LEN,  flags);
 
@@ -2901,7 +2877,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
 
 	chcr_req = (struct chcr_wr *) __skb_put_zero(skb, transhdr_len);
 
-	fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type);
+	fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, reqctx->op);
 
 	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
 	memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
@@ -2910,21 +2886,11 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
 
 	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
 	ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
-	error = ccm_format_packet(req, aeadctx, sub_type, op_type);
+	error = ccm_format_packet(req, aeadctx, sub_type, reqctx->op, assoclen);
 	if (error)
 		goto dstmap_fail;
-
-	reqctx->b0_dma = dma_map_single(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev,
-					&reqctx->scratch_pad, reqctx->b0_len,
-					DMA_BIDIRECTIONAL);
-	if (dma_mapping_error(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev,
-			      reqctx->b0_dma)) {
-		error = -ENOMEM;
-		goto dstmap_fail;
-	}
-
-	chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
-	chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
+	chcr_add_aead_dst_ent(req, phys_cpl, assoclen, qid);
+	chcr_add_aead_src_ent(req, ulptx, assoclen);
 
 	atomic_inc(&adap->chcr_stats.aead_rqst);
 	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
@@ -2933,20 +2899,18 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
 	create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0,
 		    transhdr_len, temp, 0);
 	reqctx->skb = skb;
-	reqctx->op = op_type;
 
 	return skb;
 dstmap_fail:
 	kfree_skb(skb);
 err:
-	chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type);
+	chcr_aead_common_exit(req);
 	return ERR_PTR(error);
 }
 
 static struct sk_buff *create_gcm_wr(struct aead_request *req,
 				     unsigned short qid,
-				     int size,
-				     unsigned short op_type)
+				     int size)
 {
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
@@ -2966,13 +2930,13 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
 	if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
 		assoclen = req->assoclen - 8;
 
-	reqctx->b0_dma = 0;
-	error = chcr_aead_common_init(req, op_type);
+	reqctx->b0_len = 0;
+	error = chcr_aead_common_init(req);
 	if (error)
 		return ERR_PTR(error);
 	dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
 	dnents += sg_nents_xlen(req->dst, req->cryptlen +
-				(op_type ? -authsize : authsize),
+				(reqctx->op ? -authsize : authsize),
 				CHCR_DST_SG_SIZE, req->assoclen);
 	dnents += MIN_GCM_SG; // For IV
 	dst_size = get_space_for_phys_dsgl(dnents);
@@ -2986,11 +2950,11 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
 	transhdr_len += temp;
 	transhdr_len = roundup(transhdr_len, 16);
 	if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
-			    transhdr_len, op_type)) {
+			    transhdr_len, reqctx->op)) {
+
 		atomic_inc(&adap->chcr_stats.fallback);
-		chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
-				    op_type);
-		return ERR_PTR(chcr_aead_fallback(req, op_type));
+		chcr_aead_common_exit(req);
+		return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
 	}
 	skb = alloc_skb(SGE_MAX_WR_LEN, flags);
 	if (!skb) {
@@ -3001,7 +2965,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
 	chcr_req = __skb_put_zero(skb, transhdr_len);
 
 	//Offset of tag from end
-	temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
+	temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
 	chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
 					a_ctx(tfm)->dev->rx_channel_id, 2,
 					(assoclen + 1));
@@ -3014,7 +2978,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
 			FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1,
 						temp, temp);
 	chcr_req->sec_cpl.seqno_numivs =
-			FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
+			FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, (reqctx->op ==
 					CHCR_ENCRYPT_OP) ? 1 : 0,
 					CHCR_SCMD_CIPHER_MODE_AES_GCM,
 					CHCR_SCMD_AUTH_MODE_GHASH,
@@ -3040,19 +3004,18 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
 	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
 	ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
 
-	chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
-	chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
+	chcr_add_aead_dst_ent(req, phys_cpl, assoclen, qid);
+	chcr_add_aead_src_ent(req, ulptx, assoclen);
 	atomic_inc(&adap->chcr_stats.aead_rqst);
 	temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
 		kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0);
 	create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
 		    transhdr_len, temp, reqctx->verify);
 	reqctx->skb = skb;
-	reqctx->op = op_type;
 	return skb;
 
 err:
-	chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type);
+	chcr_aead_common_exit(req);
 	return ERR_PTR(error);
 }
 
@@ -3461,6 +3424,7 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
 	}
 	{
 		SHASH_DESC_ON_STACK(shash, base_hash);
+
 		shash->tfm = base_hash;
 		shash->flags = crypto_shash_get_flags(base_hash);
 		bs = crypto_shash_blocksize(base_hash);
@@ -3585,13 +3549,13 @@ static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
 }
 
 static int chcr_aead_op(struct aead_request *req,
-			unsigned short op_type,
 			int size,
 			create_wr_t create_wr_fn)
 {
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	struct uld_ctx *u_ctx;
 	struct sk_buff *skb;
+	int isfull = 0;
 
 	if (!a_ctx(tfm)->dev) {
 		pr_err("chcr : %s : No crypto device.\n", __func__);
@@ -3600,13 +3564,13 @@ static int chcr_aead_op(struct aead_request *req,
 	u_ctx = ULD_CTX(a_ctx(tfm));
 	if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
 				   a_ctx(tfm)->tx_qidx)) {
+		isfull = 1;
 		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
-			return -EBUSY;
+			return -ENOSPC;
 	}
 
 	/* Form a WR from req */
-	skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size,
-			   op_type);
+	skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size);
 
 	if (IS_ERR(skb) || !skb)
 		return PTR_ERR(skb);
@@ -3614,7 +3578,7 @@ static int chcr_aead_op(struct aead_request *req,
 	skb->dev = u_ctx->lldi.ports[0];
 	set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx);
 	chcr_send_wr(skb);
-	return -EINPROGRESS;
+	return isfull ? -EBUSY : -EINPROGRESS;
 }
 
 static int chcr_aead_encrypt(struct aead_request *req)
@@ -3623,21 +3587,19 @@ static int chcr_aead_encrypt(struct aead_request *req)
 	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
 
 	reqctx->verify = VERIFY_HW;
+	reqctx->op = CHCR_ENCRYPT_OP;
 
 	switch (get_aead_subtype(tfm)) {
 	case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
 	case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
 	case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
 	case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
-		return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
-				    create_authenc_wr);
+		return chcr_aead_op(req, 0, create_authenc_wr);
 	case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
 	case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
-		return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
-				    create_aead_ccm_wr);
+		return chcr_aead_op(req, 0, create_aead_ccm_wr);
 	default:
-		return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
-				    create_gcm_wr);
+		return chcr_aead_op(req, 0, create_gcm_wr);
 	}
 }
 
@@ -3655,21 +3617,18 @@ static int chcr_aead_decrypt(struct aead_request *req)
 		size = 0;
 		reqctx->verify = VERIFY_HW;
 	}
-
+	reqctx->op = CHCR_DECRYPT_OP;
 	switch (get_aead_subtype(tfm)) {
 	case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
 	case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
 	case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
 	case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
-		return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
-				    create_authenc_wr);
+		return chcr_aead_op(req, size, create_authenc_wr);
 	case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
 	case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
-		return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
-				    create_aead_ccm_wr);
+		return chcr_aead_op(req, size, create_aead_ccm_wr);
 	default:
-		return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
-				    create_gcm_wr);
+		return chcr_aead_op(req, size, create_gcm_wr);
 	}
 }
 
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
index dba3dff1e209..1871500309e2 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -146,7 +146,7 @@
 	 kctx_len)
 #define CIPHER_TRANSHDR_SIZE(kctx_len, sge_pairs) \
 	(TRANSHDR_SIZE((kctx_len)) + (sge_pairs) +\
-	 sizeof(struct cpl_rx_phys_dsgl))
+	 sizeof(struct cpl_rx_phys_dsgl) + AES_BLOCK_SIZE)
 #define HASH_TRANSHDR_SIZE(kctx_len)\
 	(TRANSHDR_SIZE(kctx_len) + DUMMY_BYTES)
 
@@ -259,7 +259,6 @@
 					ULP_TX_SC_MORE_V((immdatalen)))
 #define MAX_NK 8
 #define MAX_DSGL_ENT			32
-#define MIN_CIPHER_SG			1 /* IV */
 #define MIN_AUTH_SG			1 /* IV */
 #define MIN_GCM_SG			1 /* IV */
 #define MIN_DIGEST_SG			1 /*Partial Buffer*/
diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h
index 1a20424e18c6..de3a9c085daf 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -56,7 +56,7 @@
 #define MAX_SALT                4
 #define CIP_WR_MIN_LEN (sizeof(struct chcr_wr) + \
 		    sizeof(struct cpl_rx_phys_dsgl) + \
-		    sizeof(struct ulptx_sgl))
+		    sizeof(struct ulptx_sgl) + 16) //IV
 
 #define HASH_WR_MIN_LEN (sizeof(struct chcr_wr) + \
 			DUMMY_BYTES + \
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index c8e8972af283..54835cb109e5 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -190,8 +190,8 @@ struct chcr_aead_reqctx {
 	short int dst_nents;
 	u16 imm;
 	u16 verify;
-	u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
-	unsigned char scratch_pad[MAX_SCRATCH_PAD_SIZE];
+	u8 iv[CHCR_MAX_CRYPTO_IV_LEN + MAX_SCRATCH_PAD_SIZE];
+	u8 *scratch_pad;
 };
 
 struct ulptx_walk {
@@ -295,7 +295,6 @@ struct chcr_blkcipher_req_ctx {
 	unsigned int src_ofst;
 	unsigned int dst_ofst;
 	unsigned int op;
-	dma_addr_t iv_dma;
 	u16 imm;
 	u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
 };
@@ -312,8 +311,7 @@ struct chcr_alg_template {
 
 typedef struct sk_buff *(*create_wr_t)(struct aead_request *req,
 				       unsigned short qid,
-				       int size,
-				       unsigned short op_type);
+				       int size);
 
 void chcr_verify_tag(struct aead_request *req, u8 *input, int *err);
 int chcr_aead_dma_map(struct device *dev, struct aead_request *req,
@@ -322,12 +320,12 @@ void chcr_aead_dma_unmap(struct device *dev, struct aead_request *req,
 			 unsigned short op_type);
 void chcr_add_aead_dst_ent(struct aead_request *req,
 			   struct cpl_rx_phys_dsgl *phys_cpl,
-			   unsigned int assoclen, unsigned short op_type,
+			   unsigned int assoclen,
 			   unsigned short qid);
 void chcr_add_aead_src_ent(struct aead_request *req, struct ulptx_sgl *ulptx,
-			   unsigned int assoclen, unsigned short op_type);
+			   unsigned int assoclen);
 void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
-			     struct ulptx_sgl *ulptx,
+			     void *ulptx,
 			     struct  cipher_wr_param *wrparam);
 int chcr_cipher_dma_map(struct device *dev, struct ablkcipher_request *req);
 void chcr_cipher_dma_unmap(struct device *dev, struct ablkcipher_request *req);
@@ -340,4 +338,5 @@ void chcr_add_hash_src_ent(struct ahash_request *req, struct ulptx_sgl *ulptx,
 			   struct hash_wr_param *param);
 int chcr_hash_dma_map(struct device *dev, struct ahash_request *req);
 void chcr_hash_dma_unmap(struct device *dev, struct ahash_request *req);
+void chcr_aead_common_exit(struct aead_request *req);
 #endif /* __CHCR_CRYPTO_H__ */
diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c
index 8e0aa3f175c9..461b97e2f1fd 100644
--- a/drivers/crypto/chelsio/chcr_ipsec.c
+++ b/drivers/crypto/chelsio/chcr_ipsec.c
@@ -346,18 +346,23 @@ inline void *copy_cpltx_pktxt(struct sk_buff *skb,
 				struct net_device *dev,
 				void *pos)
 {
+	struct cpl_tx_pkt_core *cpl;
+	struct sge_eth_txq *q;
 	struct adapter *adap;
 	struct port_info *pi;
-	struct sge_eth_txq *q;
-	struct cpl_tx_pkt_core *cpl;
-	u64 cntrl = 0;
 	u32 ctrl0, qidx;
+	u64 cntrl = 0;
+	int left;
 
 	pi = netdev_priv(dev);
 	adap = pi->adapter;
 	qidx = skb->queue_mapping;
 	q = &adap->sge.ethtxq[qidx + pi->first_qset];
 
+	left = (void *)q->q.stat - pos;
+	if (!left)
+		pos = q->q.desc;
+
 	cpl = (struct cpl_tx_pkt_core *)pos;
 
 	cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
@@ -382,18 +387,17 @@ inline void *copy_key_cpltx_pktxt(struct sk_buff *skb,
 				void *pos,
 				struct ipsec_sa_entry *sa_entry)
 {
-	struct adapter *adap;
-	struct port_info *pi;
-	struct sge_eth_txq *q;
-	unsigned int len, qidx;
 	struct _key_ctx *key_ctx;
 	int left, eoq, key_len;
+	struct sge_eth_txq *q;
+	struct adapter *adap;
+	struct port_info *pi;
+	unsigned int qidx;
 
 	pi = netdev_priv(dev);
 	adap = pi->adapter;
 	qidx = skb->queue_mapping;
 	q = &adap->sge.ethtxq[qidx + pi->first_qset];
-	len = sa_entry->enckey_len + sizeof(struct cpl_tx_pkt_core);
 	key_len = sa_entry->kctx_len;
 
 	/* end of queue, reset pos to start of queue */
@@ -411,19 +415,14 @@ inline void *copy_key_cpltx_pktxt(struct sk_buff *skb,
 	pos += sizeof(struct _key_ctx);
 	left -= sizeof(struct _key_ctx);
 
-	if (likely(len <= left)) {
+	if (likely(key_len <= left)) {
 		memcpy(key_ctx->key, sa_entry->key, key_len);
 		pos += key_len;
 	} else {
-		if (key_len <= left) {
-			memcpy(pos, sa_entry->key, key_len);
-			pos += key_len;
-		} else {
-			memcpy(pos, sa_entry->key, left);
-			memcpy(q->q.desc, sa_entry->key + left,
-			       key_len - left);
-			pos = (u8 *)q->q.desc + (key_len - left);
-		}
+		memcpy(pos, sa_entry->key, left);
+		memcpy(q->q.desc, sa_entry->key + left,
+		       key_len - left);
+		pos = (u8 *)q->q.desc + (key_len - left);
 	}
 	/* Copy CPL TX PKT XT */
 	pos = copy_cpltx_pktxt(skb, dev, pos);
diff --git a/drivers/crypto/chelsio/chtls/chtls.h b/drivers/crypto/chelsio/chtls/chtls.h
index f4b8f1ec0061..a53a0e6ba024 100644
--- a/drivers/crypto/chelsio/chtls/chtls.h
+++ b/drivers/crypto/chelsio/chtls/chtls.h
@@ -67,11 +67,6 @@ enum {
 	CPL_RET_UNKNOWN_TID = 4    /* unexpected unknown TID */
 };
 
-#define TLS_RCV_ST_READ_HEADER		0xF0
-#define TLS_RCV_ST_READ_BODY		0xF1
-#define TLS_RCV_ST_READ_DONE		0xF2
-#define TLS_RCV_ST_READ_NB		0xF3
-
 #define LISTEN_INFO_HASH_SIZE 32
 #define RSPQ_HASH_BITS 5
 struct listen_info {
@@ -149,6 +144,7 @@ struct chtls_dev {
 	struct list_head rcu_node;
 	struct list_head na_node;
 	unsigned int send_page_order;
+	int max_host_sndbuf;
 	struct key_map kmap;
 };
 
@@ -278,6 +274,7 @@ struct tlsrx_cmp_hdr {
 #define TLSRX_HDR_PKT_MAC_ERROR_F        TLSRX_HDR_PKT_MAC_ERROR_V(1U)
 
 #define TLSRX_HDR_PKT_ERROR_M           0x1F
+#define CONTENT_TYPE_ERROR		0x7F
 
 struct ulp_mem_rw {
 	__be32 cmd;
@@ -347,8 +344,8 @@ enum {
 	ULPCB_FLAG_HOLD      = 1 << 3,	/* skb not ready for Tx yet */
 	ULPCB_FLAG_COMPL     = 1 << 4,	/* request WR completion */
 	ULPCB_FLAG_URG       = 1 << 5,	/* urgent data */
-	ULPCB_FLAG_TLS_ND    = 1 << 6, /* payload of zero length */
-	ULPCB_FLAG_NO_HDR    = 1 << 7, /* not a ofld wr */
+	ULPCB_FLAG_TLS_HDR   = 1 << 6,  /* payload with tls hdr */
+	ULPCB_FLAG_NO_HDR    = 1 << 7,  /* not a ofld wr */
 };
 
 /* The ULP mode/submode of an skbuff */
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c b/drivers/crypto/chelsio/chtls/chtls_cm.c
index 82a473a0cefa..2bb6f0380758 100644
--- a/drivers/crypto/chelsio/chtls/chtls_cm.c
+++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
@@ -1537,6 +1537,10 @@ static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)
 	struct sock *sk;
 
 	sk = lookup_tid(cdev->tids, hwtid);
+	if (unlikely(!sk)) {
+		pr_err("can't find conn. for hwtid %u.\n", hwtid);
+		return -EINVAL;
+	}
 	skb_dst_set(skb, NULL);
 	process_cpl_msg(chtls_recv_data, sk, skb);
 	return 0;
@@ -1585,6 +1589,10 @@ static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)
 	struct sock *sk;
 
 	sk = lookup_tid(cdev->tids, hwtid);
+	if (unlikely(!sk)) {
+		pr_err("can't find conn. for hwtid %u.\n", hwtid);
+		return -EINVAL;
+	}
 	skb_dst_set(skb, NULL);
 	process_cpl_msg(chtls_recv_pdu, sk, skb);
 	return 0;
@@ -1600,12 +1608,14 @@ static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)
 
 static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
 {
-	struct cpl_rx_tls_cmp *cmp_cpl = cplhdr(skb);
+	struct tlsrx_cmp_hdr *tls_hdr_pkt;
+	struct cpl_rx_tls_cmp *cmp_cpl;
 	struct sk_buff *skb_rec;
 	struct chtls_sock *csk;
 	struct chtls_hws *tlsk;
 	struct tcp_sock *tp;
 
+	cmp_cpl = cplhdr(skb);
 	csk = rcu_dereference_sk_user_data(sk);
 	tlsk = &csk->tlshws;
 	tp = tcp_sk(sk);
@@ -1615,16 +1625,18 @@ static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
 
 	skb_reset_transport_header(skb);
 	__skb_pull(skb, sizeof(*cmp_cpl));
+	tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;
+	if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)
+		tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
 	if (!skb->data_len)
-		__skb_trim(skb, CPL_RX_TLS_CMP_LENGTH_G
-				(ntohl(cmp_cpl->pdulength_length)));
+		__skb_trim(skb, TLS_HEADER_LENGTH);
 
 	tp->rcv_nxt +=
 		CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));
 
+	ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;
 	skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);
 	if (!skb_rec) {
-		ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_ND;
 		__skb_queue_tail(&sk->sk_receive_queue, skb);
 	} else {
 		chtls_set_hdrlen(skb, tlsk->pldlen);
@@ -1646,6 +1658,10 @@ static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)
 	struct sock *sk;
 
 	sk = lookup_tid(cdev->tids, hwtid);
+	if (unlikely(!sk)) {
+		pr_err("can't find conn. for hwtid %u.\n", hwtid);
+		return -EINVAL;
+	}
 	skb_dst_set(skb, NULL);
 	process_cpl_msg(chtls_rx_hdr, sk, skb);
 
@@ -2105,6 +2121,10 @@ static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)
 	struct sock *sk;
 
 	sk = lookup_tid(cdev->tids, hwtid);
+	if (unlikely(!sk)) {
+		pr_err("can't find conn. for hwtid %u.\n", hwtid);
+		return -EINVAL;
+	}
 	process_cpl_msg(chtls_rx_ack, sk, skb);
 
 	return 0;
diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c b/drivers/crypto/chelsio/chtls/chtls_hw.c
index 54a13aa99121..55d50140f9e5 100644
--- a/drivers/crypto/chelsio/chtls/chtls_hw.c
+++ b/drivers/crypto/chelsio/chtls/chtls_hw.c
@@ -213,7 +213,7 @@ static int chtls_key_info(struct chtls_sock *csk,
 			  struct _key_ctx *kctx,
 			  u32 keylen, u32 optname)
 {
-	unsigned char key[CHCR_KEYCTX_CIPHER_KEY_SIZE_256];
+	unsigned char key[AES_KEYSIZE_128];
 	struct tls12_crypto_info_aes_gcm_128 *gcm_ctx;
 	unsigned char ghash_h[AEAD_H_SIZE];
 	struct crypto_cipher *cipher;
@@ -228,10 +228,6 @@ static int chtls_key_info(struct chtls_sock *csk,
 
 	if (keylen == AES_KEYSIZE_128) {
 		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
-	} else if (keylen == AES_KEYSIZE_192) {
-		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
-	} else if (keylen == AES_KEYSIZE_256) {
-		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
 	} else {
 		pr_err("GCM: Invalid key length %d\n", keylen);
 		return -EINVAL;
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c b/drivers/crypto/chelsio/chtls/chtls_io.c
index 5a75be43950f..51fc6821cbbf 100644
--- a/drivers/crypto/chelsio/chtls/chtls_io.c
+++ b/drivers/crypto/chelsio/chtls/chtls_io.c
@@ -907,11 +907,83 @@ static int chtls_skb_copy_to_page_nocache(struct sock *sk,
 }
 
 /* Read TLS header to find content type and data length */
-static u16 tls_header_read(struct tls_hdr *thdr, struct iov_iter *from)
+static int tls_header_read(struct tls_hdr *thdr, struct iov_iter *from)
 {
 	if (copy_from_iter(thdr, sizeof(*thdr), from) != sizeof(*thdr))
 		return -EFAULT;
-	return (__force u16)cpu_to_be16(thdr->length);
+	return (__force int)cpu_to_be16(thdr->length);
+}
+
+static int csk_mem_free(struct chtls_dev *cdev, struct sock *sk)
+{
+	return (cdev->max_host_sndbuf - sk->sk_wmem_queued);
+}
+
+static int csk_wait_memory(struct chtls_dev *cdev,
+			   struct sock *sk, long *timeo_p)
+{
+	DEFINE_WAIT_FUNC(wait, woken_wake_function);
+	int sndbuf, err = 0;
+	long current_timeo;
+	long vm_wait = 0;
+	bool noblock;
+
+	current_timeo = *timeo_p;
+	noblock = (*timeo_p ? false : true);
+	sndbuf = cdev->max_host_sndbuf;
+	if (csk_mem_free(cdev, sk)) {
+		current_timeo = (prandom_u32() % (HZ / 5)) + 2;
+		vm_wait = (prandom_u32() % (HZ / 5)) + 2;
+	}
+
+	add_wait_queue(sk_sleep(sk), &wait);
+	while (1) {
+		sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
+
+		if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
+			goto do_error;
+		if (!*timeo_p) {
+			if (noblock)
+				set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
+			goto do_nonblock;
+		}
+		if (signal_pending(current))
+			goto do_interrupted;
+		sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
+		if (csk_mem_free(cdev, sk) && !vm_wait)
+			break;
+
+		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
+		sk->sk_write_pending++;
+		sk_wait_event(sk, &current_timeo, sk->sk_err ||
+			      (sk->sk_shutdown & SEND_SHUTDOWN) ||
+			      (csk_mem_free(cdev, sk) && !vm_wait), &wait);
+		sk->sk_write_pending--;
+
+		if (vm_wait) {
+			vm_wait -= current_timeo;
+			current_timeo = *timeo_p;
+			if (current_timeo != MAX_SCHEDULE_TIMEOUT) {
+				current_timeo -= vm_wait;
+				if (current_timeo < 0)
+					current_timeo = 0;
+			}
+			vm_wait = 0;
+		}
+		*timeo_p = current_timeo;
+	}
+do_rm_wq:
+	remove_wait_queue(sk_sleep(sk), &wait);
+	return err;
+do_error:
+	err = -EPIPE;
+	goto do_rm_wq;
+do_nonblock:
+	err = -EAGAIN;
+	goto do_rm_wq;
+do_interrupted:
+	err = sock_intr_errno(*timeo_p);
+	goto do_rm_wq;
 }
 
 int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
@@ -952,6 +1024,8 @@ int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 			copy = mss - skb->len;
 			skb->ip_summed = CHECKSUM_UNNECESSARY;
 		}
+		if (!csk_mem_free(cdev, sk))
+			goto wait_for_sndbuf;
 
 		if (is_tls_tx(csk) && !csk->tlshws.txleft) {
 			struct tls_hdr hdr;
@@ -1009,9 +1083,10 @@ int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 			int off = TCP_OFF(sk);
 			bool merge;
 
-			if (page)
-				pg_size <<= compound_order(page);
+			if (!page)
+				goto wait_for_memory;
 
+			pg_size <<= compound_order(page);
 			if (off < pg_size &&
 			    skb_can_coalesce(skb, i, page, off)) {
 				merge = 1;
@@ -1099,8 +1174,10 @@ int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 		if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND)
 			push_frames_if_head(sk);
 		continue;
+wait_for_sndbuf:
+		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 wait_for_memory:
-		err = sk_stream_wait_memory(sk, &timeo);
+		err = csk_wait_memory(cdev, sk, &timeo);
 		if (err)
 			goto do_error;
 	}
@@ -1131,6 +1208,7 @@ int chtls_sendpage(struct sock *sk, struct page *page,
 		   int offset, size_t size, int flags)
 {
 	struct chtls_sock *csk;
+	struct chtls_dev *cdev;
 	int mss, err, copied;
 	struct tcp_sock *tp;
 	long timeo;
@@ -1138,6 +1216,7 @@ int chtls_sendpage(struct sock *sk, struct page *page,
 	tp = tcp_sk(sk);
 	copied = 0;
 	csk = rcu_dereference_sk_user_data(sk);
+	cdev = csk->cdev;
 	timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 
 	err = sk_stream_wait_connect(sk, &timeo);
@@ -1152,10 +1231,11 @@ int chtls_sendpage(struct sock *sk, struct page *page,
 		struct sk_buff *skb = skb_peek_tail(&csk->txq);
 		int copy, i;
 
-		copy = mss - skb->len;
 		if (!skb || (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) ||
-		    copy <= 0) {
+		    (copy = mss - skb->len) <= 0) {
 new_buf:
+			if (!csk_mem_free(cdev, sk))
+				goto wait_for_sndbuf;
 
 			if (is_tls_tx(csk)) {
 				skb = get_record_skb(sk,
@@ -1167,7 +1247,7 @@ int chtls_sendpage(struct sock *sk, struct page *page,
 				skb = get_tx_skb(sk, 0);
 			}
 			if (!skb)
-				goto do_error;
+				goto wait_for_memory;
 			copy = mss;
 		}
 		if (copy > size)
@@ -1206,8 +1286,12 @@ int chtls_sendpage(struct sock *sk, struct page *page,
 		if (unlikely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND))
 			push_frames_if_head(sk);
 		continue;
-
+wait_for_sndbuf:
 		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
+wait_for_memory:
+		err = csk_wait_memory(cdev, sk, &timeo);
+		if (err)
+			goto do_error;
 	}
 out:
 	csk_reset_flag(csk, CSK_TX_MORE_DATA);
@@ -1409,7 +1493,7 @@ static int chtls_pt_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
 			break;
 		chtls_cleanup_rbuf(sk, copied);
 		sk_wait_data(sk, &timeo, NULL);
-			continue;
+		continue;
 found_ok_skb:
 		if (!skb->len) {
 			skb_dst_set(skb, NULL);
@@ -1449,31 +1533,13 @@ static int chtls_pt_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
 				}
 			}
 		}
-		if (hws->rstate == TLS_RCV_ST_READ_BODY) {
-			if (skb_copy_datagram_msg(skb, offset,
-						  msg, avail)) {
-				if (!copied) {
-					copied = -EFAULT;
-					break;
-				}
-			}
-		} else {
-			struct tlsrx_cmp_hdr *tls_hdr_pkt =
-				(struct tlsrx_cmp_hdr *)skb->data;
-
-			if ((tls_hdr_pkt->res_to_mac_error &
-			    TLSRX_HDR_PKT_ERROR_M))
-				tls_hdr_pkt->type = 0x7F;
-
-			/* CMP pld len is for recv seq */
-			hws->rcvpld = skb->hdr_len;
-			if (skb_copy_datagram_msg(skb, offset, msg, avail)) {
-				if (!copied) {
-					copied = -EFAULT;
-					break;
-				}
+		if (skb_copy_datagram_msg(skb, offset, msg, avail)) {
+			if (!copied) {
+				copied = -EFAULT;
+				break;
 			}
 		}
+
 		copied += avail;
 		len -= avail;
 		hws->copied_seq += avail;
@@ -1481,32 +1547,20 @@ static int chtls_pt_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
 		if (tp->urg_data && after(tp->copied_seq, tp->urg_seq))
 			tp->urg_data = 0;
 
-		if (hws->rstate == TLS_RCV_ST_READ_BODY &&
-		    (avail + offset) >= skb->len) {
+		if ((avail + offset) >= skb->len) {
 			if (likely(skb))
 				chtls_free_skb(sk, skb);
 			buffers_freed++;
-			hws->rstate = TLS_RCV_ST_READ_HEADER;
-			atomic_inc(&adap->chcr_stats.tls_pdu_rx);
-			tp->copied_seq += hws->rcvpld;
+			if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_TLS_HDR) {
+				tp->copied_seq += skb->len;
+				hws->rcvpld = skb->hdr_len;
+			} else {
+				tp->copied_seq += hws->rcvpld;
+			}
 			hws->copied_seq = 0;
 			if (copied >= target &&
 			    !skb_peek(&sk->sk_receive_queue))
 				break;
-		} else {
-			if (likely(skb)) {
-				if (ULP_SKB_CB(skb)->flags &
-				    ULPCB_FLAG_TLS_ND)
-					hws->rstate =
-						TLS_RCV_ST_READ_HEADER;
-				else
-					hws->rstate =
-						TLS_RCV_ST_READ_BODY;
-				chtls_free_skb(sk, skb);
-			}
-			buffers_freed++;
-			tp->copied_seq += avail;
-			hws->copied_seq = 0;
 		}
 	} while (len > 0);
 
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c b/drivers/crypto/chelsio/chtls/chtls_main.c
index 007c45c38fc7..9b07f9165658 100644
--- a/drivers/crypto/chelsio/chtls/chtls_main.c
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -216,7 +216,6 @@ static void *chtls_uld_add(const struct cxgb4_lld_info *info)
 	cdev->lldi = lldi;
 	cdev->pdev = lldi->pdev;
 	cdev->tids = lldi->tids;
-	cdev->ports = (struct net_device **)(cdev + 1);
 	cdev->ports = lldi->ports;
 	cdev->mtus = lldi->mtus;
 	cdev->tids = lldi->tids;
@@ -239,6 +238,7 @@ static void *chtls_uld_add(const struct cxgb4_lld_info *info)
 	spin_lock_init(&cdev->idr_lock);
 	cdev->send_page_order = min_t(uint, get_order(32768),
 				      send_page_order);
+	cdev->max_host_sndbuf = 48 * 1024;
 
 	if (lldi->vr->key.size)
 		if (chtls_init_kmap(cdev, lldi))
@@ -250,7 +250,7 @@ static void *chtls_uld_add(const struct cxgb4_lld_info *info)
 
 	return cdev;
 out_rspq_skb:
-	for (j = 0; j <= i; j++)
+	for (j = 0; j < i; j++)
 		kfree_skb(cdev->rspq_skb_cache[j]);
 	kfree_skb(cdev->askb);
 out_skb:
@@ -441,7 +441,7 @@ static int chtls_uld_rx_handler(void *handle, const __be64 *rsp,
 static int do_chtls_getsockopt(struct sock *sk, char __user *optval,
 			       int __user *optlen)
 {
-	struct tls_crypto_info crypto_info;
+	struct tls_crypto_info crypto_info = { 0 };
 
 	crypto_info.version = TLS_1_2_VERSION;
 	if (copy_to_user(optval, &crypto_info, sizeof(struct tls_crypto_info)))
@@ -491,9 +491,13 @@ static int do_chtls_setsockopt(struct sock *sk, int optname,
 
 	switch (tmp_crypto_info.cipher_type) {
 	case TLS_CIPHER_AES_GCM_128: {
-		rc = copy_from_user(crypto_info, optval,
-				    sizeof(struct
-					   tls12_crypto_info_aes_gcm_128));
+		/* Obtain version and type from previous copy */
+		crypto_info[0] = tmp_crypto_info;
+		/* Now copy the following data */
+		rc = copy_from_user((char *)crypto_info + sizeof(*crypto_info),
+				optval + sizeof(*crypto_info),
+				sizeof(struct tls12_crypto_info_aes_gcm_128)
+				- sizeof(*crypto_info));
 
 		if (rc) {
 			rc = -EFAULT;
diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c
index 86f5f459762e..2cfabb99cb6e 100644
--- a/drivers/crypto/exynos-rng.c
+++ b/drivers/crypto/exynos-rng.c
@@ -319,8 +319,7 @@ static int exynos_rng_remove(struct platform_device *pdev)
 
 static int __maybe_unused exynos_rng_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct exynos_rng_dev *rng = platform_get_drvdata(pdev);
+	struct exynos_rng_dev *rng = dev_get_drvdata(dev);
 	int ret;
 
 	/* If we were never seeded then after resume it will be the same */
@@ -350,8 +349,7 @@ static int __maybe_unused exynos_rng_suspend(struct device *dev)
 
 static int __maybe_unused exynos_rng_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct exynos_rng_dev *rng = platform_get_drvdata(pdev);
+	struct exynos_rng_dev *rng = dev_get_drvdata(dev);
 	int ret;
 
 	/* Never seeded so nothing to do */
diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c
index d4a81be0d7d2..4e86f864a952 100644
--- a/drivers/crypto/inside-secure/safexcel.c
+++ b/drivers/crypto/inside-secure/safexcel.c
@@ -20,6 +20,7 @@
 #include <linux/platform_device.h>
 #include <linux/workqueue.h>
 
+#include <crypto/internal/aead.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/skcipher.h>
 
@@ -152,8 +153,8 @@ static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
 	       EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
 	writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL);
 
-	memset(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_RAM, 0,
-	       EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32));
+	memset_io(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_RAM, 0,
+		  EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32));
 
 	eip197_write_firmware(priv, fw[FW_IFPP], EIP197_PE_ICE_FPP_CTRL,
 			      EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN);
@@ -352,6 +353,7 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
 	/* H/W capabilities selection */
 	val = EIP197_FUNCTION_RSVD;
 	val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY;
+	val |= EIP197_PROTOCOL_ENCRYPT_HASH | EIP197_PROTOCOL_HASH_DECRYPT;
 	val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC;
 	val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1;
 	val |= EIP197_ALG_SHA2 | EIP197_ALG_HMAC_SHA2;
@@ -537,6 +539,27 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
 	       EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
 }
 
+inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
+				       struct safexcel_result_desc *rdesc)
+{
+	if (likely(!rdesc->result_data.error_code))
+		return 0;
+
+	if (rdesc->result_data.error_code & 0x407f) {
+		/* Fatal error (bits 0-7, 14) */
+		dev_err(priv->dev,
+			"cipher: result: result descriptor error (%d)\n",
+			rdesc->result_data.error_code);
+		return -EIO;
+	} else if (rdesc->result_data.error_code == BIT(9)) {
+		/* Authentication failed */
+		return -EBADMSG;
+	}
+
+	/* All other non-fatal errors */
+	return -EINVAL;
+}
+
 void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
 {
 	struct safexcel_command_desc *cdesc;
@@ -770,6 +793,9 @@ static struct safexcel_alg_template *safexcel_algs[] = {
 	&safexcel_alg_hmac_sha1,
 	&safexcel_alg_hmac_sha224,
 	&safexcel_alg_hmac_sha256,
+	&safexcel_alg_authenc_hmac_sha1_cbc_aes,
+	&safexcel_alg_authenc_hmac_sha224_cbc_aes,
+	&safexcel_alg_authenc_hmac_sha256_cbc_aes,
 };
 
 static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
@@ -781,6 +807,8 @@ static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
 
 		if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
 			ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
+		else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
+			ret = crypto_register_aead(&safexcel_algs[i]->alg.aead);
 		else
 			ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
 
@@ -794,6 +822,8 @@ static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
 	for (j = 0; j < i; j++) {
 		if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
 			crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
+		else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)
+			crypto_unregister_aead(&safexcel_algs[j]->alg.aead);
 		else
 			crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
 	}
@@ -808,6 +838,8 @@ static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
 	for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
 		if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
 			crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
+		else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
+			crypto_unregister_aead(&safexcel_algs[i]->alg.aead);
 		else
 			crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
 	}
diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h
index b470a849721f..8b3ee9b59f53 100644
--- a/drivers/crypto/inside-secure/safexcel.h
+++ b/drivers/crypto/inside-secure/safexcel.h
@@ -11,8 +11,10 @@
 #ifndef __SAFEXCEL_H__
 #define __SAFEXCEL_H__
 
+#include <crypto/aead.h>
 #include <crypto/algapi.h>
 #include <crypto/internal/hash.h>
+#include <crypto/sha.h>
 #include <crypto/skcipher.h>
 
 #define EIP197_HIA_VERSION_LE			0xca35
@@ -20,7 +22,7 @@
 
 /* Static configuration */
 #define EIP197_DEFAULT_RING_SIZE		400
-#define EIP197_MAX_TOKENS			5
+#define EIP197_MAX_TOKENS			8
 #define EIP197_MAX_RINGS			4
 #define EIP197_FETCH_COUNT			1
 #define EIP197_MAX_BATCH_SZ			64
@@ -28,6 +30,17 @@
 #define EIP197_GFP_FLAGS(base)	((base).flags & CRYPTO_TFM_REQ_MAY_SLEEP ? \
 				 GFP_KERNEL : GFP_ATOMIC)
 
+/* Custom on-stack requests (for invalidation) */
+#define EIP197_SKCIPHER_REQ_SIZE	sizeof(struct skcipher_request) + \
+					sizeof(struct safexcel_cipher_req)
+#define EIP197_AHASH_REQ_SIZE		sizeof(struct ahash_request) + \
+					sizeof(struct safexcel_ahash_req)
+#define EIP197_AEAD_REQ_SIZE		sizeof(struct aead_request) + \
+					sizeof(struct safexcel_cipher_req)
+#define EIP197_REQUEST_ON_STACK(name, type, size) \
+	char __##name##_desc[size] CRYPTO_MINALIGN_ATTR; \
+	struct type##_request *name = (void *)__##name##_desc
+
 /* Register base offsets */
 #define EIP197_HIA_AIC(priv)		((priv)->base + (priv)->offsets.hia_aic)
 #define EIP197_HIA_AIC_G(priv)		((priv)->base + (priv)->offsets.hia_aic_g)
@@ -274,7 +287,7 @@ struct safexcel_context_record {
 	u32 control0;
 	u32 control1;
 
-	__le32 data[12];
+	__le32 data[24];
 } __packed;
 
 /* control0 */
@@ -286,8 +299,8 @@ struct safexcel_context_record {
 #define CONTEXT_CONTROL_TYPE_CRYPTO_IN		0x5
 #define CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT	0x6
 #define CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN	0x7
-#define CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT	0x14
-#define CONTEXT_CONTROL_TYPE_HASH_DECRYPT_OUT	0x15
+#define CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT	0xe
+#define CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN	0xf
 #define CONTEXT_CONTROL_RESTART_HASH		BIT(4)
 #define CONTEXT_CONTROL_NO_FINISH_HASH		BIT(5)
 #define CONTEXT_CONTROL_SIZE(n)			((n) << 8)
@@ -391,11 +404,15 @@ struct safexcel_token {
 	u8 opcode:4;
 } __packed;
 
+#define EIP197_TOKEN_HASH_RESULT_VERIFY		BIT(16)
+
 #define EIP197_TOKEN_STAT_LAST_HASH		BIT(0)
 #define EIP197_TOKEN_STAT_LAST_PACKET		BIT(1)
 #define EIP197_TOKEN_OPCODE_DIRECTION		0x0
 #define EIP197_TOKEN_OPCODE_INSERT		0x2
 #define EIP197_TOKEN_OPCODE_NOOP		EIP197_TOKEN_OPCODE_INSERT
+#define EIP197_TOKEN_OPCODE_RETRIEVE		0x4
+#define EIP197_TOKEN_OPCODE_VERIFY		0xd
 #define EIP197_TOKEN_OPCODE_BYPASS		GENMASK(3, 0)
 
 static inline void eip197_noop_token(struct safexcel_token *token)
@@ -479,6 +496,7 @@ struct safexcel_ring {
 
 enum safexcel_alg_type {
 	SAFEXCEL_ALG_TYPE_SKCIPHER,
+	SAFEXCEL_ALG_TYPE_AEAD,
 	SAFEXCEL_ALG_TYPE_AHASH,
 };
 
@@ -581,6 +599,16 @@ struct safexcel_context {
 	bool exit_inv;
 };
 
+struct safexcel_ahash_export_state {
+	u64 len;
+	u64 processed;
+
+	u32 digest;
+
+	u32 state[SHA256_DIGEST_SIZE / sizeof(u32)];
+	u8 cache[SHA256_BLOCK_SIZE];
+};
+
 /*
  * Template structure to describe the algorithms in order to register them.
  * It also has the purpose to contain our private structure and is actually
@@ -591,6 +619,7 @@ struct safexcel_alg_template {
 	enum safexcel_alg_type type;
 	union {
 		struct skcipher_alg skcipher;
+		struct aead_alg aead;
 		struct ahash_alg ahash;
 	} alg;
 };
@@ -601,6 +630,8 @@ struct safexcel_inv_result {
 };
 
 void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring);
+int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
+				struct safexcel_result_desc *rdesc);
 void safexcel_complete(struct safexcel_crypto_priv *priv, int ring);
 int safexcel_invalidate_cache(struct crypto_async_request *async,
 			      struct safexcel_crypto_priv *priv,
@@ -625,6 +656,8 @@ struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *pri
 						bool first, bool last,
 						dma_addr_t data, u32 len);
 void safexcel_inv_complete(struct crypto_async_request *req, int error);
+int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen,
+			 void *istate, void *ostate);
 
 /* available algorithms */
 extern struct safexcel_alg_template safexcel_alg_ecb_aes;
@@ -635,5 +668,8 @@ extern struct safexcel_alg_template safexcel_alg_sha256;
 extern struct safexcel_alg_template safexcel_alg_hmac_sha1;
 extern struct safexcel_alg_template safexcel_alg_hmac_sha224;
 extern struct safexcel_alg_template safexcel_alg_hmac_sha256;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes;
 
 #endif
diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c
index bafb60505fab..6bb60fda2043 100644
--- a/drivers/crypto/inside-secure/safexcel_cipher.c
+++ b/drivers/crypto/inside-secure/safexcel_cipher.c
@@ -12,8 +12,12 @@
 #include <linux/dma-mapping.h>
 #include <linux/dmapool.h>
 
+#include <crypto/aead.h>
 #include <crypto/aes.h>
+#include <crypto/authenc.h>
+#include <crypto/sha.h>
 #include <crypto/skcipher.h>
+#include <crypto/internal/aead.h>
 #include <crypto/internal/skcipher.h>
 
 #include "safexcel.h"
@@ -28,9 +32,16 @@ struct safexcel_cipher_ctx {
 	struct safexcel_crypto_priv *priv;
 
 	u32 mode;
+	bool aead;
 
 	__le32 key[8];
 	unsigned int key_len;
+
+	/* All the below is AEAD specific */
+	u32 alg;
+	u32 state_sz;
+	u32 ipad[SHA256_DIGEST_SIZE / sizeof(u32)];
+	u32 opad[SHA256_DIGEST_SIZE / sizeof(u32)];
 };
 
 struct safexcel_cipher_req {
@@ -38,18 +49,16 @@ struct safexcel_cipher_req {
 	bool needs_inv;
 };
 
-static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx,
-				  struct crypto_async_request *async,
-				  struct safexcel_command_desc *cdesc,
-				  u32 length)
+static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
+				    struct safexcel_command_desc *cdesc,
+				    u32 length)
 {
-	struct skcipher_request *req = skcipher_request_cast(async);
 	struct safexcel_token *token;
 	unsigned offset = 0;
 
 	if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
 		offset = AES_BLOCK_SIZE / sizeof(u32);
-		memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_SIZE);
+		memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
 
 		cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
 	}
@@ -65,8 +74,64 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx,
 				EIP197_TOKEN_INS_TYPE_OUTPUT;
 }
 
-static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key,
-			       unsigned int len)
+static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
+				struct safexcel_command_desc *cdesc,
+				enum safexcel_cipher_direction direction,
+				u32 cryptlen, u32 assoclen, u32 digestsize)
+{
+	struct safexcel_token *token;
+	unsigned offset = 0;
+
+	if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
+		offset = AES_BLOCK_SIZE / sizeof(u32);
+		memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
+
+		cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
+	}
+
+	token = (struct safexcel_token *)(cdesc->control_data.token + offset);
+
+	if (direction == SAFEXCEL_DECRYPT)
+		cryptlen -= digestsize;
+
+	token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
+	token[0].packet_length = assoclen;
+	token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH |
+				EIP197_TOKEN_INS_TYPE_OUTPUT;
+
+	token[1].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
+	token[1].packet_length = cryptlen;
+	token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
+	token[1].instructions = EIP197_TOKEN_INS_LAST |
+				EIP197_TOKEN_INS_TYPE_CRYTO |
+				EIP197_TOKEN_INS_TYPE_HASH |
+				EIP197_TOKEN_INS_TYPE_OUTPUT;
+
+	if (direction == SAFEXCEL_ENCRYPT) {
+		token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
+		token[2].packet_length = digestsize;
+		token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
+				EIP197_TOKEN_STAT_LAST_PACKET;
+		token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
+					EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
+	} else {
+		token[2].opcode = EIP197_TOKEN_OPCODE_RETRIEVE;
+		token[2].packet_length = digestsize;
+		token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
+				EIP197_TOKEN_STAT_LAST_PACKET;
+		token[2].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
+
+		token[3].opcode = EIP197_TOKEN_OPCODE_VERIFY;
+		token[3].packet_length = digestsize |
+					 EIP197_TOKEN_HASH_RESULT_VERIFY;
+		token[3].stat = EIP197_TOKEN_STAT_LAST_HASH |
+				EIP197_TOKEN_STAT_LAST_PACKET;
+		token[3].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT;
+	}
+}
+
+static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
+					const u8 *key, unsigned int len)
 {
 	struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
@@ -98,41 +163,123 @@ static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key,
 	return 0;
 }
 
+static int safexcel_aead_aes_setkey(struct crypto_aead *ctfm, const u8 *key,
+				    unsigned int len)
+{
+	struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct safexcel_ahash_export_state istate, ostate;
+	struct safexcel_crypto_priv *priv = ctx->priv;
+	struct crypto_authenc_keys keys;
+
+	if (crypto_authenc_extractkeys(&keys, key, len) != 0)
+		goto badkey;
+
+	if (keys.enckeylen > sizeof(ctx->key))
+		goto badkey;
+
+	/* Encryption key */
+	if (priv->version == EIP197 && ctx->base.ctxr_dma &&
+	    memcmp(ctx->key, keys.enckey, keys.enckeylen))
+		ctx->base.needs_inv = true;
+
+	/* Auth key */
+	switch (ctx->alg) {
+	case CONTEXT_CONTROL_CRYPTO_ALG_SHA1:
+		if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey,
+					 keys.authkeylen, &istate, &ostate))
+			goto badkey;
+		break;
+	case CONTEXT_CONTROL_CRYPTO_ALG_SHA224:
+		if (safexcel_hmac_setkey("safexcel-sha224", keys.authkey,
+					 keys.authkeylen, &istate, &ostate))
+			goto badkey;
+		break;
+	case CONTEXT_CONTROL_CRYPTO_ALG_SHA256:
+		if (safexcel_hmac_setkey("safexcel-sha256", keys.authkey,
+					 keys.authkeylen, &istate, &ostate))
+			goto badkey;
+		break;
+	default:
+		dev_err(priv->dev, "aead: unsupported hash algorithm\n");
+		goto badkey;
+	}
+
+	crypto_aead_set_flags(ctfm, crypto_aead_get_flags(ctfm) &
+				    CRYPTO_TFM_RES_MASK);
+
+	if (priv->version == EIP197 && ctx->base.ctxr_dma &&
+	    (memcmp(ctx->ipad, istate.state, ctx->state_sz) ||
+	     memcmp(ctx->opad, ostate.state, ctx->state_sz)))
+		ctx->base.needs_inv = true;
+
+	/* Now copy the keys into the context */
+	memcpy(ctx->key, keys.enckey, keys.enckeylen);
+	ctx->key_len = keys.enckeylen;
+
+	memcpy(ctx->ipad, &istate.state, ctx->state_sz);
+	memcpy(ctx->opad, &ostate.state, ctx->state_sz);
+
+	memzero_explicit(&keys, sizeof(keys));
+	return 0;
+
+badkey:
+	crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+	memzero_explicit(&keys, sizeof(keys));
+	return -EINVAL;
+}
+
 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
 				    struct crypto_async_request *async,
+				    struct safexcel_cipher_req *sreq,
 				    struct safexcel_command_desc *cdesc)
 {
 	struct safexcel_crypto_priv *priv = ctx->priv;
-	struct skcipher_request *req = skcipher_request_cast(async);
-	struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
 	int ctrl_size;
 
-	if (sreq->direction == SAFEXCEL_ENCRYPT)
+	if (ctx->aead) {
+		if (sreq->direction == SAFEXCEL_ENCRYPT)
+			cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT;
+		else
+			cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN;
+	} else {
 		cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT;
-	else
-		cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_IN;
+
+		/* The decryption control type is a combination of the
+		 * encryption type and CONTEXT_CONTROL_TYPE_NULL_IN, for all
+		 * types.
+		 */
+		if (sreq->direction == SAFEXCEL_DECRYPT)
+			cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_NULL_IN;
+	}
 
 	cdesc->control_data.control0 |= CONTEXT_CONTROL_KEY_EN;
 	cdesc->control_data.control1 |= ctx->mode;
 
+	if (ctx->aead)
+		cdesc->control_data.control0 |= CONTEXT_CONTROL_DIGEST_HMAC |
+						ctx->alg;
+
 	switch (ctx->key_len) {
 	case AES_KEYSIZE_128:
 		cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES128;
-		ctrl_size = 4;
 		break;
 	case AES_KEYSIZE_192:
 		cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES192;
-		ctrl_size = 6;
 		break;
 	case AES_KEYSIZE_256:
 		cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES256;
-		ctrl_size = 8;
 		break;
 	default:
 		dev_err(priv->dev, "aes keysize not supported: %u\n",
 			ctx->key_len);
 		return -EINVAL;
 	}
+
+	ctrl_size = ctx->key_len / sizeof(u32);
+	if (ctx->aead)
+		/* Take in account the ipad+opad digests */
+		ctrl_size += ctx->state_sz / sizeof(u32) * 2;
 	cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(ctrl_size);
 
 	return 0;
@@ -140,9 +287,12 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
 
 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring,
 				      struct crypto_async_request *async,
+				      struct scatterlist *src,
+				      struct scatterlist *dst,
+				      unsigned int cryptlen,
+				      struct safexcel_cipher_req *sreq,
 				      bool *should_complete, int *ret)
 {
-	struct skcipher_request *req = skcipher_request_cast(async);
 	struct safexcel_result_desc *rdesc;
 	int ndesc = 0;
 
@@ -158,12 +308,8 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
 			break;
 		}
 
-		if (rdesc->result_data.error_code) {
-			dev_err(priv->dev,
-				"cipher: result: result descriptor error (%d)\n",
-				rdesc->result_data.error_code);
-			*ret = -EIO;
-		}
+		if (likely(!*ret))
+			*ret = safexcel_rdesc_check_errors(priv, rdesc);
 
 		ndesc++;
 	} while (!rdesc->last_seg);
@@ -171,16 +317,16 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
 	safexcel_complete(priv, ring);
 	spin_unlock_bh(&priv->ring[ring].egress_lock);
 
-	if (req->src == req->dst) {
-		dma_unmap_sg(priv->dev, req->src,
-			     sg_nents_for_len(req->src, req->cryptlen),
+	if (src == dst) {
+		dma_unmap_sg(priv->dev, src,
+			     sg_nents_for_len(src, cryptlen),
 			     DMA_BIDIRECTIONAL);
 	} else {
-		dma_unmap_sg(priv->dev, req->src,
-			     sg_nents_for_len(req->src, req->cryptlen),
+		dma_unmap_sg(priv->dev, src,
+			     sg_nents_for_len(src, cryptlen),
 			     DMA_TO_DEVICE);
-		dma_unmap_sg(priv->dev, req->dst,
-			     sg_nents_for_len(req->dst, req->cryptlen),
+		dma_unmap_sg(priv->dev, dst,
+			     sg_nents_for_len(dst, cryptlen),
 			     DMA_FROM_DEVICE);
 	}
 
@@ -189,39 +335,43 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
 	return ndesc;
 }
 
-static int safexcel_aes_send(struct crypto_async_request *async,
-			     int ring, struct safexcel_request *request,
-			     int *commands, int *results)
+static int safexcel_aes_send(struct crypto_async_request *base, int ring,
+			     struct safexcel_request *request,
+			     struct safexcel_cipher_req *sreq,
+			     struct scatterlist *src, struct scatterlist *dst,
+			     unsigned int cryptlen, unsigned int assoclen,
+			     unsigned int digestsize, u8 *iv, int *commands,
+			     int *results)
 {
-	struct skcipher_request *req = skcipher_request_cast(async);
-	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 	struct safexcel_crypto_priv *priv = ctx->priv;
 	struct safexcel_command_desc *cdesc;
 	struct safexcel_result_desc *rdesc;
 	struct scatterlist *sg;
-	int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = req->cryptlen;
+	unsigned int totlen = cryptlen + assoclen;
+	int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = totlen;
 	int i, ret = 0;
 
-	if (req->src == req->dst) {
-		nr_src = dma_map_sg(priv->dev, req->src,
-				    sg_nents_for_len(req->src, req->cryptlen),
+	if (src == dst) {
+		nr_src = dma_map_sg(priv->dev, src,
+				    sg_nents_for_len(src, totlen),
 				    DMA_BIDIRECTIONAL);
 		nr_dst = nr_src;
 		if (!nr_src)
 			return -EINVAL;
 	} else {
-		nr_src = dma_map_sg(priv->dev, req->src,
-				    sg_nents_for_len(req->src, req->cryptlen),
+		nr_src = dma_map_sg(priv->dev, src,
+				    sg_nents_for_len(src, totlen),
 				    DMA_TO_DEVICE);
 		if (!nr_src)
 			return -EINVAL;
 
-		nr_dst = dma_map_sg(priv->dev, req->dst,
-				    sg_nents_for_len(req->dst, req->cryptlen),
+		nr_dst = dma_map_sg(priv->dev, dst,
+				    sg_nents_for_len(dst, totlen),
 				    DMA_FROM_DEVICE);
 		if (!nr_dst) {
-			dma_unmap_sg(priv->dev, req->src,
-				     sg_nents_for_len(req->src, req->cryptlen),
+			dma_unmap_sg(priv->dev, src,
+				     sg_nents_for_len(src, totlen),
 				     DMA_TO_DEVICE);
 			return -EINVAL;
 		}
@@ -229,10 +379,17 @@ static int safexcel_aes_send(struct crypto_async_request *async,
 
 	memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);
 
+	if (ctx->aead) {
+		memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
+		       ctx->ipad, ctx->state_sz);
+		memcpy(ctx->base.ctxr->data + (ctx->key_len + ctx->state_sz) / sizeof(u32),
+		       ctx->opad, ctx->state_sz);
+	}
+
 	spin_lock_bh(&priv->ring[ring].egress_lock);
 
 	/* command descriptors */
-	for_each_sg(req->src, sg, nr_src, i) {
+	for_each_sg(src, sg, nr_src, i) {
 		int len = sg_dma_len(sg);
 
 		/* Do not overflow the request */
@@ -240,7 +397,7 @@ static int safexcel_aes_send(struct crypto_async_request *async,
 			len = queued;
 
 		cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, !(queued - len),
-					   sg_dma_address(sg), len, req->cryptlen,
+					   sg_dma_address(sg), len, totlen,
 					   ctx->base.ctxr_dma);
 		if (IS_ERR(cdesc)) {
 			/* No space left in the command descriptor ring */
@@ -250,8 +407,14 @@ static int safexcel_aes_send(struct crypto_async_request *async,
 		n_cdesc++;
 
 		if (n_cdesc == 1) {
-			safexcel_context_control(ctx, async, cdesc);
-			safexcel_cipher_token(ctx, async, cdesc, req->cryptlen);
+			safexcel_context_control(ctx, base, sreq, cdesc);
+			if (ctx->aead)
+				safexcel_aead_token(ctx, iv, cdesc,
+						    sreq->direction, cryptlen,
+						    assoclen, digestsize);
+			else
+				safexcel_skcipher_token(ctx, iv, cdesc,
+							cryptlen);
 		}
 
 		queued -= len;
@@ -260,7 +423,7 @@ static int safexcel_aes_send(struct crypto_async_request *async,
 	}
 
 	/* result descriptors */
-	for_each_sg(req->dst, sg, nr_dst, i) {
+	for_each_sg(dst, sg, nr_dst, i) {
 		bool first = !i, last = (i == nr_dst - 1);
 		u32 len = sg_dma_len(sg);
 
@@ -276,7 +439,7 @@ static int safexcel_aes_send(struct crypto_async_request *async,
 
 	spin_unlock_bh(&priv->ring[ring].egress_lock);
 
-	request->req = &req->base;
+	request->req = base;
 
 	*commands = n_cdesc;
 	*results = n_rdesc;
@@ -291,16 +454,16 @@ static int safexcel_aes_send(struct crypto_async_request *async,
 
 	spin_unlock_bh(&priv->ring[ring].egress_lock);
 
-	if (req->src == req->dst) {
-		dma_unmap_sg(priv->dev, req->src,
-			     sg_nents_for_len(req->src, req->cryptlen),
+	if (src == dst) {
+		dma_unmap_sg(priv->dev, src,
+			     sg_nents_for_len(src, totlen),
 			     DMA_BIDIRECTIONAL);
 	} else {
-		dma_unmap_sg(priv->dev, req->src,
-			     sg_nents_for_len(req->src, req->cryptlen),
+		dma_unmap_sg(priv->dev, src,
+			     sg_nents_for_len(src, totlen),
 			     DMA_TO_DEVICE);
-		dma_unmap_sg(priv->dev, req->dst,
-			     sg_nents_for_len(req->dst, req->cryptlen),
+		dma_unmap_sg(priv->dev, dst,
+			     sg_nents_for_len(dst, totlen),
 			     DMA_FROM_DEVICE);
 	}
 
@@ -309,11 +472,10 @@ static int safexcel_aes_send(struct crypto_async_request *async,
 
 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 				      int ring,
-				      struct crypto_async_request *async,
+				      struct crypto_async_request *base,
 				      bool *should_complete, int *ret)
 {
-	struct skcipher_request *req = skcipher_request_cast(async);
-	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 	struct safexcel_result_desc *rdesc;
 	int ndesc = 0, enq_ret;
 
@@ -354,7 +516,7 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 	ctx->base.ring = ring;
 
 	spin_lock_bh(&priv->ring[ring].queue_lock);
-	enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
+	enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
 	spin_unlock_bh(&priv->ring[ring].queue_lock);
 
 	if (enq_ret != -EINPROGRESS)
@@ -368,9 +530,10 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 	return ndesc;
 }
 
-static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
-				  struct crypto_async_request *async,
-				  bool *should_complete, int *ret)
+static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv,
+					   int ring,
+					   struct crypto_async_request *async,
+					   bool *should_complete, int *ret)
 {
 	struct skcipher_request *req = skcipher_request_cast(async);
 	struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
@@ -381,24 +544,48 @@ static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
 		err = safexcel_handle_inv_result(priv, ring, async,
 						 should_complete, ret);
 	} else {
-		err = safexcel_handle_req_result(priv, ring, async,
+		err = safexcel_handle_req_result(priv, ring, async, req->src,
+						 req->dst, req->cryptlen, sreq,
 						 should_complete, ret);
 	}
 
 	return err;
 }
 
-static int safexcel_cipher_send_inv(struct crypto_async_request *async,
+static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv,
+				       int ring,
+				       struct crypto_async_request *async,
+				       bool *should_complete, int *ret)
+{
+	struct aead_request *req = aead_request_cast(async);
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct safexcel_cipher_req *sreq = aead_request_ctx(req);
+	int err;
+
+	if (sreq->needs_inv) {
+		sreq->needs_inv = false;
+		err = safexcel_handle_inv_result(priv, ring, async,
+						 should_complete, ret);
+	} else {
+		err = safexcel_handle_req_result(priv, ring, async, req->src,
+						 req->dst,
+						 req->cryptlen + crypto_aead_authsize(tfm),
+						 sreq, should_complete, ret);
+	}
+
+	return err;
+}
+
+static int safexcel_cipher_send_inv(struct crypto_async_request *base,
 				    int ring, struct safexcel_request *request,
 				    int *commands, int *results)
 {
-	struct skcipher_request *req = skcipher_request_cast(async);
-	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 	struct safexcel_crypto_priv *priv = ctx->priv;
 	int ret;
 
-	ret = safexcel_invalidate_cache(async, priv,
-					ctx->base.ctxr_dma, ring, request);
+	ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring,
+					request);
 	if (unlikely(ret))
 		return ret;
 
@@ -408,9 +595,9 @@ static int safexcel_cipher_send_inv(struct crypto_async_request *async,
 	return 0;
 }
 
-static int safexcel_send(struct crypto_async_request *async,
-			 int ring, struct safexcel_request *request,
-			 int *commands, int *results)
+static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
+				  struct safexcel_request *request,
+				  int *commands, int *results)
 {
 	struct skcipher_request *req = skcipher_request_cast(async);
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
@@ -421,59 +608,108 @@ static int safexcel_send(struct crypto_async_request *async,
 	BUG_ON(priv->version == EIP97 && sreq->needs_inv);
 
 	if (sreq->needs_inv)
-		ret = safexcel_cipher_send_inv(async, ring, request,
-					       commands, results);
+		ret = safexcel_cipher_send_inv(async, ring, request, commands,
+					       results);
+	else
+		ret = safexcel_aes_send(async, ring, request, sreq, req->src,
+					req->dst, req->cryptlen, 0, 0, req->iv,
+					commands, results);
+	return ret;
+}
+
+static int safexcel_aead_send(struct crypto_async_request *async, int ring,
+			      struct safexcel_request *request, int *commands,
+			      int *results)
+{
+	struct aead_request *req = aead_request_cast(async);
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+	struct safexcel_cipher_req *sreq = aead_request_ctx(req);
+	struct safexcel_crypto_priv *priv = ctx->priv;
+	int ret;
+
+	BUG_ON(priv->version == EIP97 && sreq->needs_inv);
+
+	if (sreq->needs_inv)
+		ret = safexcel_cipher_send_inv(async, ring, request, commands,
+					       results);
 	else
-		ret = safexcel_aes_send(async, ring, request,
+		ret = safexcel_aes_send(async, ring, request, sreq, req->src,
+					req->dst, req->cryptlen, req->assoclen,
+					crypto_aead_authsize(tfm), req->iv,
 					commands, results);
 	return ret;
 }
 
-static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm)
+static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm,
+				    struct crypto_async_request *base,
+				    struct safexcel_cipher_req *sreq,
+				    struct safexcel_inv_result *result)
 {
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct safexcel_crypto_priv *priv = ctx->priv;
-	SKCIPHER_REQUEST_ON_STACK(req, __crypto_skcipher_cast(tfm));
-	struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
-	struct safexcel_inv_result result = {};
 	int ring = ctx->base.ring;
 
-	memset(req, 0, sizeof(struct skcipher_request));
+	init_completion(&result->completion);
 
-	/* create invalidation request */
-	init_completion(&result.completion);
-	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
-				      safexcel_inv_complete, &result);
-
-	skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm));
-	ctx = crypto_tfm_ctx(req->base.tfm);
+	ctx = crypto_tfm_ctx(base->tfm);
 	ctx->base.exit_inv = true;
 	sreq->needs_inv = true;
 
 	spin_lock_bh(&priv->ring[ring].queue_lock);
-	crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
+	crypto_enqueue_request(&priv->ring[ring].queue, base);
 	spin_unlock_bh(&priv->ring[ring].queue_lock);
 
 	queue_work(priv->ring[ring].workqueue,
 		   &priv->ring[ring].work_data.work);
 
-	wait_for_completion(&result.completion);
+	wait_for_completion(&result->completion);
 
-	if (result.error) {
+	if (result->error) {
 		dev_warn(priv->dev,
 			"cipher: sync: invalidate: completion error %d\n",
-			 result.error);
-		return result.error;
+			 result->error);
+		return result->error;
 	}
 
 	return 0;
 }
 
-static int safexcel_aes(struct skcipher_request *req,
-			enum safexcel_cipher_direction dir, u32 mode)
+static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm)
 {
-	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+	EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE);
 	struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
+	struct safexcel_inv_result result = {};
+
+	memset(req, 0, sizeof(struct skcipher_request));
+
+	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+				      safexcel_inv_complete, &result);
+	skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm));
+
+	return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
+}
+
+static int safexcel_aead_exit_inv(struct crypto_tfm *tfm)
+{
+	EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE);
+	struct safexcel_cipher_req *sreq = aead_request_ctx(req);
+	struct safexcel_inv_result result = {};
+
+	memset(req, 0, sizeof(struct aead_request));
+
+	aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+				  safexcel_inv_complete, &result);
+	aead_request_set_tfm(req, __crypto_aead_cast(tfm));
+
+	return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
+}
+
+static int safexcel_aes(struct crypto_async_request *base,
+			struct safexcel_cipher_req *sreq,
+			enum safexcel_cipher_direction dir, u32 mode)
+{
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 	struct safexcel_crypto_priv *priv = ctx->priv;
 	int ret, ring;
 
@@ -489,7 +725,7 @@ static int safexcel_aes(struct skcipher_request *req,
 	} else {
 		ctx->base.ring = safexcel_select_ring(priv);
 		ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
-						 EIP197_GFP_FLAGS(req->base),
+						 EIP197_GFP_FLAGS(*base),
 						 &ctx->base.ctxr_dma);
 		if (!ctx->base.ctxr)
 			return -ENOMEM;
@@ -498,7 +734,7 @@ static int safexcel_aes(struct skcipher_request *req,
 	ring = ctx->base.ring;
 
 	spin_lock_bh(&priv->ring[ring].queue_lock);
-	ret = crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
+	ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
 	spin_unlock_bh(&priv->ring[ring].queue_lock);
 
 	queue_work(priv->ring[ring].workqueue,
@@ -509,14 +745,14 @@ static int safexcel_aes(struct skcipher_request *req,
 
 static int safexcel_ecb_aes_encrypt(struct skcipher_request *req)
 {
-	return safexcel_aes(req, SAFEXCEL_ENCRYPT,
-			    CONTEXT_CONTROL_CRYPTO_MODE_ECB);
+	return safexcel_aes(&req->base, skcipher_request_ctx(req),
+			    SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB);
 }
 
 static int safexcel_ecb_aes_decrypt(struct skcipher_request *req)
 {
-	return safexcel_aes(req, SAFEXCEL_DECRYPT,
-			    CONTEXT_CONTROL_CRYPTO_MODE_ECB);
+	return safexcel_aes(&req->base, skcipher_request_ctx(req),
+			    SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB);
 }
 
 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
@@ -526,34 +762,64 @@ static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
 		container_of(tfm->__crt_alg, struct safexcel_alg_template,
 			     alg.skcipher.base);
 
-	ctx->priv = tmpl->priv;
-	ctx->base.send = safexcel_send;
-	ctx->base.handle_result = safexcel_handle_result;
-
 	crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
 				    sizeof(struct safexcel_cipher_req));
 
+	ctx->priv = tmpl->priv;
+
+	ctx->base.send = safexcel_skcipher_send;
+	ctx->base.handle_result = safexcel_skcipher_handle_result;
 	return 0;
 }
 
-static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
+static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm)
 {
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
-	struct safexcel_crypto_priv *priv = ctx->priv;
-	int ret;
 
-	memzero_explicit(ctx->key, 8 * sizeof(u32));
+	memzero_explicit(ctx->key, sizeof(ctx->key));
 
 	/* context not allocated, skip invalidation */
 	if (!ctx->base.ctxr)
+		return -ENOMEM;
+
+	memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data));
+	return 0;
+}
+
+static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
+{
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct safexcel_crypto_priv *priv = ctx->priv;
+	int ret;
+
+	if (safexcel_cipher_cra_exit(tfm))
 		return;
 
-	memzero_explicit(ctx->base.ctxr->data, 8 * sizeof(u32));
+	if (priv->version == EIP197) {
+		ret = safexcel_skcipher_exit_inv(tfm);
+		if (ret)
+			dev_warn(priv->dev, "skcipher: invalidation error %d\n",
+				 ret);
+	} else {
+		dma_pool_free(priv->context_pool, ctx->base.ctxr,
+			      ctx->base.ctxr_dma);
+	}
+}
+
+static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
+{
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct safexcel_crypto_priv *priv = ctx->priv;
+	int ret;
+
+	if (safexcel_cipher_cra_exit(tfm))
+		return;
 
 	if (priv->version == EIP197) {
-		ret = safexcel_cipher_exit_inv(tfm);
+		ret = safexcel_aead_exit_inv(tfm);
 		if (ret)
-			dev_warn(priv->dev, "cipher: invalidation error %d\n", ret);
+			dev_warn(priv->dev, "aead: invalidation error %d\n",
+				 ret);
 	} else {
 		dma_pool_free(priv->context_pool, ctx->base.ctxr,
 			      ctx->base.ctxr_dma);
@@ -563,7 +829,7 @@ static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
 struct safexcel_alg_template safexcel_alg_ecb_aes = {
 	.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
 	.alg.skcipher = {
-		.setkey = safexcel_aes_setkey,
+		.setkey = safexcel_skcipher_aes_setkey,
 		.encrypt = safexcel_ecb_aes_encrypt,
 		.decrypt = safexcel_ecb_aes_decrypt,
 		.min_keysize = AES_MIN_KEY_SIZE,
@@ -586,20 +852,20 @@ struct safexcel_alg_template safexcel_alg_ecb_aes = {
 
 static int safexcel_cbc_aes_encrypt(struct skcipher_request *req)
 {
-	return safexcel_aes(req, SAFEXCEL_ENCRYPT,
-			    CONTEXT_CONTROL_CRYPTO_MODE_CBC);
+	return safexcel_aes(&req->base, skcipher_request_ctx(req),
+			    SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC);
 }
 
 static int safexcel_cbc_aes_decrypt(struct skcipher_request *req)
 {
-	return safexcel_aes(req, SAFEXCEL_DECRYPT,
-			    CONTEXT_CONTROL_CRYPTO_MODE_CBC);
+	return safexcel_aes(&req->base, skcipher_request_ctx(req),
+			    SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC);
 }
 
 struct safexcel_alg_template safexcel_alg_cbc_aes = {
 	.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
 	.alg.skcipher = {
-		.setkey = safexcel_aes_setkey,
+		.setkey = safexcel_skcipher_aes_setkey,
 		.encrypt = safexcel_cbc_aes_encrypt,
 		.decrypt = safexcel_cbc_aes_decrypt,
 		.min_keysize = AES_MIN_KEY_SIZE,
@@ -620,3 +886,139 @@ struct safexcel_alg_template safexcel_alg_cbc_aes = {
 		},
 	},
 };
+
+static int safexcel_aead_encrypt(struct aead_request *req)
+{
+	struct safexcel_cipher_req *creq = aead_request_ctx(req);
+
+	return safexcel_aes(&req->base, creq, SAFEXCEL_ENCRYPT,
+			    CONTEXT_CONTROL_CRYPTO_MODE_CBC);
+}
+
+static int safexcel_aead_decrypt(struct aead_request *req)
+{
+	struct safexcel_cipher_req *creq = aead_request_ctx(req);
+
+	return safexcel_aes(&req->base, creq, SAFEXCEL_DECRYPT,
+			    CONTEXT_CONTROL_CRYPTO_MODE_CBC);
+}
+
+static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
+{
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct safexcel_alg_template *tmpl =
+		container_of(tfm->__crt_alg, struct safexcel_alg_template,
+			     alg.aead.base);
+
+	crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
+				sizeof(struct safexcel_cipher_req));
+
+	ctx->priv = tmpl->priv;
+
+	ctx->aead = true;
+	ctx->base.send = safexcel_aead_send;
+	ctx->base.handle_result = safexcel_aead_handle_result;
+	return 0;
+}
+
+static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm)
+{
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	safexcel_aead_cra_init(tfm);
+	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
+	ctx->state_sz = SHA1_DIGEST_SIZE;
+	return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
+	.type = SAFEXCEL_ALG_TYPE_AEAD,
+	.alg.aead = {
+		.setkey = safexcel_aead_aes_setkey,
+		.encrypt = safexcel_aead_encrypt,
+		.decrypt = safexcel_aead_decrypt,
+		.ivsize = AES_BLOCK_SIZE,
+		.maxauthsize = SHA1_DIGEST_SIZE,
+		.base = {
+			.cra_name = "authenc(hmac(sha1),cbc(aes))",
+			.cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
+				     CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+			.cra_alignmask = 0,
+			.cra_init = safexcel_aead_sha1_cra_init,
+			.cra_exit = safexcel_aead_cra_exit,
+			.cra_module = THIS_MODULE,
+		},
+	},
+};
+
+static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm)
+{
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	safexcel_aead_cra_init(tfm);
+	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
+	ctx->state_sz = SHA256_DIGEST_SIZE;
+	return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
+	.type = SAFEXCEL_ALG_TYPE_AEAD,
+	.alg.aead = {
+		.setkey = safexcel_aead_aes_setkey,
+		.encrypt = safexcel_aead_encrypt,
+		.decrypt = safexcel_aead_decrypt,
+		.ivsize = AES_BLOCK_SIZE,
+		.maxauthsize = SHA256_DIGEST_SIZE,
+		.base = {
+			.cra_name = "authenc(hmac(sha256),cbc(aes))",
+			.cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
+				     CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+			.cra_alignmask = 0,
+			.cra_init = safexcel_aead_sha256_cra_init,
+			.cra_exit = safexcel_aead_cra_exit,
+			.cra_module = THIS_MODULE,
+		},
+	},
+};
+
+static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm)
+{
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	safexcel_aead_cra_init(tfm);
+	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
+	ctx->state_sz = SHA256_DIGEST_SIZE;
+	return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
+	.type = SAFEXCEL_ALG_TYPE_AEAD,
+	.alg.aead = {
+		.setkey = safexcel_aead_aes_setkey,
+		.encrypt = safexcel_aead_encrypt,
+		.decrypt = safexcel_aead_decrypt,
+		.ivsize = AES_BLOCK_SIZE,
+		.maxauthsize = SHA224_DIGEST_SIZE,
+		.base = {
+			.cra_name = "authenc(hmac(sha224),cbc(aes))",
+			.cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
+				     CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+			.cra_alignmask = 0,
+			.cra_init = safexcel_aead_sha224_cra_init,
+			.cra_exit = safexcel_aead_cra_exit,
+			.cra_module = THIS_MODULE,
+		},
+	},
+};
diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c
index 317b9e480312..d138d6b8fec5 100644
--- a/drivers/crypto/inside-secure/safexcel_hash.c
+++ b/drivers/crypto/inside-secure/safexcel_hash.c
@@ -50,16 +50,6 @@ struct safexcel_ahash_req {
 	u8 cache_next[SHA256_BLOCK_SIZE] __aligned(sizeof(u32));
 };
 
-struct safexcel_ahash_export_state {
-	u64 len;
-	u64 processed;
-
-	u32 digest;
-
-	u32 state[SHA256_DIGEST_SIZE / sizeof(u32)];
-	u8 cache[SHA256_BLOCK_SIZE];
-};
-
 static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
 				u32 input_length, u32 result_length)
 {
@@ -146,11 +136,8 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
 		dev_err(priv->dev,
 			"hash: result: could not retrieve the result descriptor\n");
 		*ret = PTR_ERR(rdesc);
-	} else if (rdesc->result_data.error_code) {
-		dev_err(priv->dev,
-			"hash: result: result descriptor error (%d)\n",
-			rdesc->result_data.error_code);
-		*ret = -EINVAL;
+	} else {
+		*ret = safexcel_rdesc_check_errors(priv, rdesc);
 	}
 
 	safexcel_complete(priv, ring);
@@ -480,7 +467,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
 {
 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct safexcel_crypto_priv *priv = ctx->priv;
-	AHASH_REQUEST_ON_STACK(req, __crypto_ahash_cast(tfm));
+	EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);
 	struct safexcel_ahash_req *rctx = ahash_request_ctx(req);
 	struct safexcel_inv_result result = {};
 	int ring = ctx->base.ring;
@@ -912,8 +899,8 @@ static int safexcel_hmac_init_iv(struct ahash_request *areq,
 	return crypto_ahash_export(areq, state);
 }
 
-static int safexcel_hmac_setkey(const char *alg, const u8 *key,
-				unsigned int keylen, void *istate, void *ostate)
+int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen,
+			 void *istate, void *ostate)
 {
 	struct ahash_request *areq;
 	struct crypto_ahash *tfm;
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index 1e87637c412d..36afd6d8753c 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -334,7 +334,7 @@ static int wait_for_csb(struct nx842_workmem *wmem,
 		return -EPROTO;
 	case CSB_CC_SEQUENCE:
 		/* should not happen, we don't use chained CRBs */
-		CSB_ERR(csb, "CRB seqeunce number error");
+		CSB_ERR(csb, "CRB sequence number error");
 		return -EPROTO;
 	case CSB_CC_UNKNOWN_CODE:
 		CSB_ERR(csb, "Unknown subfunction code");
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index ad02aa63b519..d1a1c74fb56a 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -1087,7 +1087,7 @@ static void omap_sham_finish_req(struct ahash_request *req, int err)
 
 	if (test_bit(FLAGS_SGS_COPIED, &dd->flags))
 		free_pages((unsigned long)sg_virt(ctx->sg),
-			   get_order(ctx->sg->length));
+			   get_order(ctx->sg->length + ctx->bufcnt));
 
 	if (test_bit(FLAGS_SGS_ALLOCED, &dd->flags))
 		kfree(ctx->sg);
diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c
index a4df966adbf6..321d5e2ac833 100644
--- a/drivers/crypto/picoxcell_crypto.c
+++ b/drivers/crypto/picoxcell_crypto.c
@@ -1169,8 +1169,7 @@ static void spacc_spacc_complete(unsigned long data)
 #ifdef CONFIG_PM
 static int spacc_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct spacc_engine *engine = platform_get_drvdata(pdev);
+	struct spacc_engine *engine = dev_get_drvdata(dev);
 
 	/*
 	 * We only support standby mode. All we have to do is gate the clock to
@@ -1184,8 +1183,7 @@ static int spacc_suspend(struct device *dev)
 
 static int spacc_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct spacc_engine *engine = platform_get_drvdata(pdev);
+	struct spacc_engine *engine = dev_get_drvdata(dev);
 
 	return clk_enable(engine->clk);
 }
diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/qat/qat_c3xxx/adf_drv.c
index f172171668ee..ba197f34c252 100644
--- a/drivers/crypto/qat/qat_c3xxx/adf_drv.c
+++ b/drivers/crypto/qat/qat_c3xxx/adf_drv.c
@@ -329,5 +329,7 @@ module_exit(adfdrv_release);
 
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_AUTHOR("Intel");
+MODULE_FIRMWARE(ADF_C3XXX_FW);
+MODULE_FIRMWARE(ADF_C3XXX_MMP);
 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
 MODULE_VERSION(ADF_DRV_VERSION);
diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c
index 58a984c9c3ec..59a5a0df50b6 100644
--- a/drivers/crypto/qat/qat_c62x/adf_drv.c
+++ b/drivers/crypto/qat/qat_c62x/adf_drv.c
@@ -329,5 +329,7 @@ module_exit(adfdrv_release);
 
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_AUTHOR("Intel");
+MODULE_FIRMWARE(ADF_C62X_FW);
+MODULE_FIRMWARE(ADF_C62X_MMP);
 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
 MODULE_VERSION(ADF_DRV_VERSION);
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
index 2ce01f010c74..be5c5a988ca5 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
@@ -332,5 +332,6 @@ module_exit(adfdrv_release);
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_AUTHOR("Intel");
 MODULE_FIRMWARE(ADF_DH895XCC_FW);
+MODULE_FIRMWARE(ADF_DH895XCC_MMP);
 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
 MODULE_VERSION(ADF_DRV_VERSION);
diff --git a/drivers/crypto/vmx/aes.c b/drivers/crypto/vmx/aes.c
index 96072b9b55c4..d7316f7a3a69 100644
--- a/drivers/crypto/vmx/aes.c
+++ b/drivers/crypto/vmx/aes.c
@@ -48,8 +48,6 @@ static int p8_aes_init(struct crypto_tfm *tfm)
 		       alg, PTR_ERR(fallback));
 		return PTR_ERR(fallback);
 	}
-	printk(KERN_INFO "Using '%s' as fallback implementation.\n",
-	       crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback));
 
 	crypto_cipher_set_flags(fallback,
 				crypto_cipher_get_flags((struct
diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c
index 7394d35d5936..5285ece4f33a 100644
--- a/drivers/crypto/vmx/aes_cbc.c
+++ b/drivers/crypto/vmx/aes_cbc.c
@@ -52,9 +52,6 @@ static int p8_aes_cbc_init(struct crypto_tfm *tfm)
 		       alg, PTR_ERR(fallback));
 		return PTR_ERR(fallback);
 	}
-	printk(KERN_INFO "Using '%s' as fallback implementation.\n",
-		crypto_skcipher_driver_name(fallback));
-
 
 	crypto_skcipher_set_flags(
 		fallback,
diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
index fc60d00a2e84..cd777c75291d 100644
--- a/drivers/crypto/vmx/aes_ctr.c
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -50,8 +50,6 @@ static int p8_aes_ctr_init(struct crypto_tfm *tfm)
 		       alg, PTR_ERR(fallback));
 		return PTR_ERR(fallback);
 	}
-	printk(KERN_INFO "Using '%s' as fallback implementation.\n",
-		crypto_skcipher_driver_name(fallback));
 
 	crypto_skcipher_set_flags(
 		fallback,
diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c
index 8cd6e62e4c90..8bd9aff0f55f 100644
--- a/drivers/crypto/vmx/aes_xts.c
+++ b/drivers/crypto/vmx/aes_xts.c
@@ -53,8 +53,6 @@ static int p8_aes_xts_init(struct crypto_tfm *tfm)
 			alg, PTR_ERR(fallback));
 		return PTR_ERR(fallback);
 	}
-	printk(KERN_INFO "Using '%s' as fallback implementation.\n",
-		crypto_skcipher_driver_name(fallback));
 
 	crypto_skcipher_set_flags(
 		fallback,
diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl
index 0b4a293b8a1e..d6a9f63d65ba 100644
--- a/drivers/crypto/vmx/aesp8-ppc.pl
+++ b/drivers/crypto/vmx/aesp8-ppc.pl
@@ -1,12 +1,51 @@
 #! /usr/bin/env perl
-# Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
+# SPDX-License-Identifier: GPL-2.0
+
+# This code is taken from CRYPTOGAMs[1] and is included here using the option
+# in the license to distribute the code under the GPL. Therefore this program
+# is free software; you can redistribute it and/or modify it under the terms of
+# the GNU General Public License version 2 as published by the Free Software
+# Foundation.
 #
-# Licensed under the OpenSSL license (the "License").  You may not use
-# this file except in compliance with the License.  You can obtain a copy
-# in the file LICENSE in the source distribution or at
-# https://www.openssl.org/source/license.html
+# [1] https://www.openssl.org/~appro/cryptogams/
 
+# Copyright (c) 2006-2017, CRYPTOGAMS by <appro@openssl.org>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#       * Redistributions of source code must retain copyright notices,
+#         this list of conditions and the following disclaimer.
+#
+#       * Redistributions in binary form must reproduce the above
+#         copyright notice, this list of conditions and the following
+#         disclaimer in the documentation and/or other materials
+#         provided with the distribution.
 #
+#       * Neither the name of the CRYPTOGAMS nor the names of its
+#         copyright holder and contributors may be used to endorse or
+#         promote products derived from this software without specific
+#         prior written permission.
+#
+# ALTERNATIVELY, provided that this notice is retained in full, this
+# product may be distributed under the terms of the GNU General Public
+# License (GPL), in which case the provisions of the GPL apply INSTEAD OF
+# those given above.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 # ====================================================================
 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
 # project. The module is, however, dual licensed under OpenSSL and
diff --git a/drivers/crypto/vmx/ghash.c b/drivers/crypto/vmx/ghash.c
index 27a94a119009..1c4b5b889fba 100644
--- a/drivers/crypto/vmx/ghash.c
+++ b/drivers/crypto/vmx/ghash.c
@@ -64,8 +64,6 @@ static int p8_ghash_init_tfm(struct crypto_tfm *tfm)
 		       alg, PTR_ERR(fallback));
 		return PTR_ERR(fallback);
 	}
-	printk(KERN_INFO "Using '%s' as fallback implementation.\n",
-	       crypto_tfm_alg_driver_name(crypto_shash_tfm(fallback)));
 
 	crypto_shash_set_flags(fallback,
 			       crypto_shash_get_flags((struct crypto_shash
diff --git a/drivers/crypto/vmx/ghashp8-ppc.pl b/drivers/crypto/vmx/ghashp8-ppc.pl
index d8429cb71f02..f746af271460 100644
--- a/drivers/crypto/vmx/ghashp8-ppc.pl
+++ b/drivers/crypto/vmx/ghashp8-ppc.pl
@@ -1,5 +1,14 @@
 #!/usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0
+
+# This code is taken from the OpenSSL project but the author (Andy Polyakov)
+# has relicensed it under the GPLv2. Therefore this program is free software;
+# you can redistribute it and/or modify it under the terms of the GNU General
+# Public License version 2 as published by the Free Software Foundation.
 #
+# The original headers, including the original license headers, are
+# included below for completeness.
+
 # ====================================================================
 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
 # project. The module is, however, dual licensed under OpenSSL and
diff --git a/drivers/dax/super.c b/drivers/dax/super.c
index 2b2332b605e4..88672b6f6252 100644
--- a/drivers/dax/super.c
+++ b/drivers/dax/super.c
@@ -86,6 +86,7 @@ int __bdev_dax_supported(struct super_block *sb, int blocksize)
 {
 	struct block_device *bdev = sb->s_bdev;
 	struct dax_device *dax_dev;
+	bool dax_enabled = false;
 	pgoff_t pgoff;
 	int err, id;
 	void *kaddr;
@@ -134,14 +135,21 @@ int __bdev_dax_supported(struct super_block *sb, int blocksize)
 		 * on being able to do (page_address(pfn_to_page())).
 		 */
 		WARN_ON(IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API));
+		dax_enabled = true;
 	} else if (pfn_t_devmap(pfn)) {
-		/* pass */;
-	} else {
+		struct dev_pagemap *pgmap;
+
+		pgmap = get_dev_pagemap(pfn_t_to_pfn(pfn), NULL);
+		if (pgmap && pgmap->type == MEMORY_DEVICE_FS_DAX)
+			dax_enabled = true;
+		put_dev_pagemap(pgmap);
+	}
+
+	if (!dax_enabled) {
 		pr_debug("VFS (%s): error: dax support not enabled\n",
 				sb->s_id);
 		return -EOPNOTSUPP;
 	}
-
 	return 0;
 }
 EXPORT_SYMBOL_GPL(__bdev_dax_supported);
@@ -282,6 +290,16 @@ size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
 }
 EXPORT_SYMBOL_GPL(dax_copy_from_iter);
 
+size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
+		size_t bytes, struct iov_iter *i)
+{
+	if (!dax_alive(dax_dev))
+		return 0;
+
+	return dax_dev->ops->copy_to_iter(dax_dev, pgoff, addr, bytes, i);
+}
+EXPORT_SYMBOL_GPL(dax_copy_to_iter);
+
 #ifdef CONFIG_ARCH_HAS_PMEM_API
 void arch_wb_cache_pmem(void *addr, size_t size);
 void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
diff --git a/drivers/dca/dca-core.c b/drivers/dca/dca-core.c
index 7afbb28d6a0f..1bc5ffb338c8 100644
--- a/drivers/dca/dca-core.c
+++ b/drivers/dca/dca-core.c
@@ -270,7 +270,7 @@ EXPORT_SYMBOL_GPL(dca_remove_requester);
  * @dev - the device that wants dca service
  * @cpu - the cpuid as returned by get_cpu()
  */
-u8 dca_common_get_tag(struct device *dev, int cpu)
+static u8 dca_common_get_tag(struct device *dev, int cpu)
 {
 	struct dca_provider *dca;
 	u8 tag;
diff --git a/drivers/dma-buf/sync_debug.h b/drivers/dma-buf/sync_debug.h
index d615a89f774c..05e33f937ad0 100644
--- a/drivers/dma-buf/sync_debug.h
+++ b/drivers/dma-buf/sync_debug.h
@@ -62,8 +62,6 @@ struct sync_pt {
 	struct rb_node node;
 };
 
-#ifdef CONFIG_SW_SYNC
-
 extern const struct file_operations sw_sync_debugfs_fops;
 
 void sync_timeline_debug_add(struct sync_timeline *obj);
@@ -72,12 +70,4 @@ void sync_file_debug_add(struct sync_file *fence);
 void sync_file_debug_remove(struct sync_file *fence);
 void sync_dump(void);
 
-#else
-# define sync_timeline_debug_add(obj)
-# define sync_timeline_debug_remove(obj)
-# define sync_file_debug_add(fence)
-# define sync_file_debug_remove(fence)
-# define sync_dump()
-#endif
-
 #endif /* _LINUX_SYNC_H */
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 6d61cd023633..ca1680afa20a 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -151,13 +151,6 @@ config DMA_JZ4780
 	  If you have a board based on such a SoC and wish to use DMA for
 	  devices which can use the DMA controller, say Y or M here.
 
-config DMA_OMAP
-	tristate "OMAP DMA support"
-	depends on ARCH_OMAP || COMPILE_TEST
-	select DMA_ENGINE
-	select DMA_VIRTUAL_CHANNELS
-	select TI_DMA_CROSSBAR if (SOC_DRA7XX || COMPILE_TEST)
-
 config DMA_SA11X0
 	tristate "SA-11x0 DMA support"
 	depends on ARCH_SA1100 || COMPILE_TEST
@@ -574,28 +567,6 @@ config TIMB_DMA
 	help
 	  Enable support for the Timberdale FPGA DMA engine.
 
-config TI_CPPI41
-	tristate "CPPI 4.1 DMA support"
-	depends on (ARCH_OMAP || ARCH_DAVINCI_DA8XX)
-	select DMA_ENGINE
-	help
-	  The Communications Port Programming Interface (CPPI) 4.1 DMA engine
-	  is currently used by the USB driver on AM335x and DA8xx platforms.
-
-config TI_DMA_CROSSBAR
-	bool
-
-config TI_EDMA
-	bool "TI EDMA support"
-	depends on ARCH_DAVINCI || ARCH_OMAP || ARCH_KEYSTONE || COMPILE_TEST
-	select DMA_ENGINE
-	select DMA_VIRTUAL_CHANNELS
-	select TI_DMA_CROSSBAR if (ARCH_OMAP || COMPILE_TEST)
-	default n
-	help
-	  Enable support for the TI EDMA controller. This DMA
-	  engine is found on TI DaVinci and AM33xx parts.
-
 config XGENE_DMA
 	tristate "APM X-Gene DMA support"
 	depends on ARCH_XGENE || COMPILE_TEST
@@ -653,6 +624,8 @@ source "drivers/dma/hsu/Kconfig"
 
 source "drivers/dma/sh/Kconfig"
 
+source "drivers/dma/ti/Kconfig"
+
 # clients
 comment "DMA Clients"
 	depends on DMA_ENGINE
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index 0f62a4d49aab..203a99d68315 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -24,7 +24,6 @@ obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o
 obj-$(CONFIG_DMA_BCM2835) += bcm2835-dma.o
 obj-$(CONFIG_DMA_JZ4740) += dma-jz4740.o
 obj-$(CONFIG_DMA_JZ4780) += dma-jz4780.o
-obj-$(CONFIG_DMA_OMAP) += omap-dma.o
 obj-$(CONFIG_DMA_SA11X0) += sa11x0-dma.o
 obj-$(CONFIG_DMA_SUN4I) += sun4i-dma.o
 obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o
@@ -69,13 +68,11 @@ obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
 obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o
 obj-$(CONFIG_TEGRA210_ADMA) += tegra210-adma.o
 obj-$(CONFIG_TIMB_DMA) += timb_dma.o
-obj-$(CONFIG_TI_CPPI41) += cppi41.o
-obj-$(CONFIG_TI_DMA_CROSSBAR) += ti-dma-crossbar.o
-obj-$(CONFIG_TI_EDMA) += edma.o
 obj-$(CONFIG_XGENE_DMA) += xgene-dma.o
 obj-$(CONFIG_ZX_DMA) += zx_dma.o
 obj-$(CONFIG_ST_FDMA) += st_fdma.o
 
 obj-y += mediatek/
 obj-y += qcom/
+obj-y += ti/
 obj-y += xilinx/
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index a861b5b4d443..75f38d19fcbe 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -2041,8 +2041,7 @@ static void at_dma_shutdown(struct platform_device *pdev)
 
 static int at_dma_prepare(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct at_dma *atdma = platform_get_drvdata(pdev);
+	struct at_dma *atdma = dev_get_drvdata(dev);
 	struct dma_chan *chan, *_chan;
 
 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
@@ -2076,8 +2075,7 @@ static void atc_suspend_cyclic(struct at_dma_chan *atchan)
 
 static int at_dma_suspend_noirq(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct at_dma *atdma = platform_get_drvdata(pdev);
+	struct at_dma *atdma = dev_get_drvdata(dev);
 	struct dma_chan *chan, *_chan;
 
 	/* preserve data */
@@ -2118,8 +2116,7 @@ static void atc_resume_cyclic(struct at_dma_chan *atchan)
 
 static int at_dma_resume_noirq(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct at_dma *atdma = platform_get_drvdata(pdev);
+	struct at_dma *atdma = dev_get_drvdata(dev);
 	struct dma_chan *chan, *_chan;
 
 	/* bring back DMA controller */
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
index 94236ec9d410..4bf72561667c 100644
--- a/drivers/dma/at_xdmac.c
+++ b/drivers/dma/at_xdmac.c
@@ -1833,8 +1833,7 @@ static void at_xdmac_free_chan_resources(struct dma_chan *chan)
 #ifdef CONFIG_PM
 static int atmel_xdmac_prepare(struct device *dev)
 {
-	struct platform_device	*pdev = to_platform_device(dev);
-	struct at_xdmac		*atxdmac = platform_get_drvdata(pdev);
+	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);
 	struct dma_chan		*chan, *_chan;
 
 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {
@@ -1853,8 +1852,7 @@ static int atmel_xdmac_prepare(struct device *dev)
 #ifdef CONFIG_PM_SLEEP
 static int atmel_xdmac_suspend(struct device *dev)
 {
-	struct platform_device	*pdev = to_platform_device(dev);
-	struct at_xdmac		*atxdmac = platform_get_drvdata(pdev);
+	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);
 	struct dma_chan		*chan, *_chan;
 
 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {
@@ -1878,8 +1876,7 @@ static int atmel_xdmac_suspend(struct device *dev)
 
 static int atmel_xdmac_resume(struct device *dev)
 {
-	struct platform_device	*pdev = to_platform_device(dev);
-	struct at_xdmac		*atxdmac = platform_get_drvdata(pdev);
+	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);
 	struct at_xdmac_chan	*atchan;
 	struct dma_chan		*chan, *_chan;
 	int			i;
diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c
deleted file mode 100644
index d9bee65a18a4..000000000000
--- a/drivers/dma/cppi41.c
+++ /dev/null
@@ -1,1223 +0,0 @@
-#include <linux/delay.h>
-#include <linux/dmaengine.h>
-#include <linux/dma-mapping.h>
-#include <linux/platform_device.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/slab.h>
-#include <linux/of_dma.h>
-#include <linux/of_irq.h>
-#include <linux/dmapool.h>
-#include <linux/interrupt.h>
-#include <linux/of_address.h>
-#include <linux/pm_runtime.h>
-#include "dmaengine.h"
-
-#define DESC_TYPE	27
-#define DESC_TYPE_HOST	0x10
-#define DESC_TYPE_TEARD	0x13
-
-#define TD_DESC_IS_RX	(1 << 16)
-#define TD_DESC_DMA_NUM	10
-
-#define DESC_LENGTH_BITS_NUM	21
-
-#define DESC_TYPE_USB	(5 << 26)
-#define DESC_PD_COMPLETE	(1 << 31)
-
-/* DMA engine */
-#define DMA_TDFDQ	4
-#define DMA_TXGCR(x)	(0x800 + (x) * 0x20)
-#define DMA_RXGCR(x)	(0x808 + (x) * 0x20)
-#define RXHPCRA0		4
-
-#define GCR_CHAN_ENABLE		(1 << 31)
-#define GCR_TEARDOWN		(1 << 30)
-#define GCR_STARV_RETRY		(1 << 24)
-#define GCR_DESC_TYPE_HOST	(1 << 14)
-
-/* DMA scheduler */
-#define DMA_SCHED_CTRL		0
-#define DMA_SCHED_CTRL_EN	(1 << 31)
-#define DMA_SCHED_WORD(x)	((x) * 4 + 0x800)
-
-#define SCHED_ENTRY0_CHAN(x)	((x) << 0)
-#define SCHED_ENTRY0_IS_RX	(1 << 7)
-
-#define SCHED_ENTRY1_CHAN(x)	((x) << 8)
-#define SCHED_ENTRY1_IS_RX	(1 << 15)
-
-#define SCHED_ENTRY2_CHAN(x)	((x) << 16)
-#define SCHED_ENTRY2_IS_RX	(1 << 23)
-
-#define SCHED_ENTRY3_CHAN(x)	((x) << 24)
-#define SCHED_ENTRY3_IS_RX	(1 << 31)
-
-/* Queue manager */
-/* 4 KiB of memory for descriptors, 2 for each endpoint */
-#define ALLOC_DECS_NUM		128
-#define DESCS_AREAS		1
-#define TOTAL_DESCS_NUM		(ALLOC_DECS_NUM * DESCS_AREAS)
-#define QMGR_SCRATCH_SIZE	(TOTAL_DESCS_NUM * 4)
-
-#define QMGR_LRAM0_BASE		0x80
-#define QMGR_LRAM_SIZE		0x84
-#define QMGR_LRAM1_BASE		0x88
-#define QMGR_MEMBASE(x)		(0x1000 + (x) * 0x10)
-#define QMGR_MEMCTRL(x)		(0x1004 + (x) * 0x10)
-#define QMGR_MEMCTRL_IDX_SH	16
-#define QMGR_MEMCTRL_DESC_SH	8
-
-#define QMGR_PEND(x)	(0x90 + (x) * 4)
-
-#define QMGR_PENDING_SLOT_Q(x)	(x / 32)
-#define QMGR_PENDING_BIT_Q(x)	(x % 32)
-
-#define QMGR_QUEUE_A(n)	(0x2000 + (n) * 0x10)
-#define QMGR_QUEUE_B(n)	(0x2004 + (n) * 0x10)
-#define QMGR_QUEUE_C(n)	(0x2008 + (n) * 0x10)
-#define QMGR_QUEUE_D(n)	(0x200c + (n) * 0x10)
-
-/* Packet Descriptor */
-#define PD2_ZERO_LENGTH		(1 << 19)
-
-struct cppi41_channel {
-	struct dma_chan chan;
-	struct dma_async_tx_descriptor txd;
-	struct cppi41_dd *cdd;
-	struct cppi41_desc *desc;
-	dma_addr_t desc_phys;
-	void __iomem *gcr_reg;
-	int is_tx;
-	u32 residue;
-
-	unsigned int q_num;
-	unsigned int q_comp_num;
-	unsigned int port_num;
-
-	unsigned td_retry;
-	unsigned td_queued:1;
-	unsigned td_seen:1;
-	unsigned td_desc_seen:1;
-
-	struct list_head node;		/* Node for pending list */
-};
-
-struct cppi41_desc {
-	u32 pd0;
-	u32 pd1;
-	u32 pd2;
-	u32 pd3;
-	u32 pd4;
-	u32 pd5;
-	u32 pd6;
-	u32 pd7;
-} __aligned(32);
-
-struct chan_queues {
-	u16 submit;
-	u16 complete;
-};
-
-struct cppi41_dd {
-	struct dma_device ddev;
-
-	void *qmgr_scratch;
-	dma_addr_t scratch_phys;
-
-	struct cppi41_desc *cd;
-	dma_addr_t descs_phys;
-	u32 first_td_desc;
-	struct cppi41_channel *chan_busy[ALLOC_DECS_NUM];
-
-	void __iomem *ctrl_mem;
-	void __iomem *sched_mem;
-	void __iomem *qmgr_mem;
-	unsigned int irq;
-	const struct chan_queues *queues_rx;
-	const struct chan_queues *queues_tx;
-	struct chan_queues td_queue;
-	u16 first_completion_queue;
-	u16 qmgr_num_pend;
-	u32 n_chans;
-	u8 platform;
-
-	struct list_head pending;	/* Pending queued transfers */
-	spinlock_t lock;		/* Lock for pending list */
-
-	/* context for suspend/resume */
-	unsigned int dma_tdfdq;
-
-	bool is_suspended;
-};
-
-static struct chan_queues am335x_usb_queues_tx[] = {
-	/* USB0 ENDP 1 */
-	[ 0] = { .submit = 32, .complete =  93},
-	[ 1] = { .submit = 34, .complete =  94},
-	[ 2] = { .submit = 36, .complete =  95},
-	[ 3] = { .submit = 38, .complete =  96},
-	[ 4] = { .submit = 40, .complete =  97},
-	[ 5] = { .submit = 42, .complete =  98},
-	[ 6] = { .submit = 44, .complete =  99},
-	[ 7] = { .submit = 46, .complete = 100},
-	[ 8] = { .submit = 48, .complete = 101},
-	[ 9] = { .submit = 50, .complete = 102},
-	[10] = { .submit = 52, .complete = 103},
-	[11] = { .submit = 54, .complete = 104},
-	[12] = { .submit = 56, .complete = 105},
-	[13] = { .submit = 58, .complete = 106},
-	[14] = { .submit = 60, .complete = 107},
-
-	/* USB1 ENDP1 */
-	[15] = { .submit = 62, .complete = 125},
-	[16] = { .submit = 64, .complete = 126},
-	[17] = { .submit = 66, .complete = 127},
-	[18] = { .submit = 68, .complete = 128},
-	[19] = { .submit = 70, .complete = 129},
-	[20] = { .submit = 72, .complete = 130},
-	[21] = { .submit = 74, .complete = 131},
-	[22] = { .submit = 76, .complete = 132},
-	[23] = { .submit = 78, .complete = 133},
-	[24] = { .submit = 80, .complete = 134},
-	[25] = { .submit = 82, .complete = 135},
-	[26] = { .submit = 84, .complete = 136},
-	[27] = { .submit = 86, .complete = 137},
-	[28] = { .submit = 88, .complete = 138},
-	[29] = { .submit = 90, .complete = 139},
-};
-
-static const struct chan_queues am335x_usb_queues_rx[] = {
-	/* USB0 ENDP 1 */
-	[ 0] = { .submit =  1, .complete = 109},
-	[ 1] = { .submit =  2, .complete = 110},
-	[ 2] = { .submit =  3, .complete = 111},
-	[ 3] = { .submit =  4, .complete = 112},
-	[ 4] = { .submit =  5, .complete = 113},
-	[ 5] = { .submit =  6, .complete = 114},
-	[ 6] = { .submit =  7, .complete = 115},
-	[ 7] = { .submit =  8, .complete = 116},
-	[ 8] = { .submit =  9, .complete = 117},
-	[ 9] = { .submit = 10, .complete = 118},
-	[10] = { .submit = 11, .complete = 119},
-	[11] = { .submit = 12, .complete = 120},
-	[12] = { .submit = 13, .complete = 121},
-	[13] = { .submit = 14, .complete = 122},
-	[14] = { .submit = 15, .complete = 123},
-
-	/* USB1 ENDP 1 */
-	[15] = { .submit = 16, .complete = 141},
-	[16] = { .submit = 17, .complete = 142},
-	[17] = { .submit = 18, .complete = 143},
-	[18] = { .submit = 19, .complete = 144},
-	[19] = { .submit = 20, .complete = 145},
-	[20] = { .submit = 21, .complete = 146},
-	[21] = { .submit = 22, .complete = 147},
-	[22] = { .submit = 23, .complete = 148},
-	[23] = { .submit = 24, .complete = 149},
-	[24] = { .submit = 25, .complete = 150},
-	[25] = { .submit = 26, .complete = 151},
-	[26] = { .submit = 27, .complete = 152},
-	[27] = { .submit = 28, .complete = 153},
-	[28] = { .submit = 29, .complete = 154},
-	[29] = { .submit = 30, .complete = 155},
-};
-
-static const struct chan_queues da8xx_usb_queues_tx[] = {
-	[0] = { .submit =  16, .complete = 24},
-	[1] = { .submit =  18, .complete = 24},
-	[2] = { .submit =  20, .complete = 24},
-	[3] = { .submit =  22, .complete = 24},
-};
-
-static const struct chan_queues da8xx_usb_queues_rx[] = {
-	[0] = { .submit =  1, .complete = 26},
-	[1] = { .submit =  3, .complete = 26},
-	[2] = { .submit =  5, .complete = 26},
-	[3] = { .submit =  7, .complete = 26},
-};
-
-struct cppi_glue_infos {
-	const struct chan_queues *queues_rx;
-	const struct chan_queues *queues_tx;
-	struct chan_queues td_queue;
-	u16 first_completion_queue;
-	u16 qmgr_num_pend;
-};
-
-static struct cppi41_channel *to_cpp41_chan(struct dma_chan *c)
-{
-	return container_of(c, struct cppi41_channel, chan);
-}
-
-static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc)
-{
-	struct cppi41_channel *c;
-	u32 descs_size;
-	u32 desc_num;
-
-	descs_size = sizeof(struct cppi41_desc) * ALLOC_DECS_NUM;
-
-	if (!((desc >= cdd->descs_phys) &&
-			(desc < (cdd->descs_phys + descs_size)))) {
-		return NULL;
-	}
-
-	desc_num = (desc - cdd->descs_phys) / sizeof(struct cppi41_desc);
-	BUG_ON(desc_num >= ALLOC_DECS_NUM);
-	c = cdd->chan_busy[desc_num];
-	cdd->chan_busy[desc_num] = NULL;
-
-	/* Usecount for chan_busy[], paired with push_desc_queue() */
-	pm_runtime_put(cdd->ddev.dev);
-
-	return c;
-}
-
-static void cppi_writel(u32 val, void *__iomem *mem)
-{
-	__raw_writel(val, mem);
-}
-
-static u32 cppi_readl(void *__iomem *mem)
-{
-	return __raw_readl(mem);
-}
-
-static u32 pd_trans_len(u32 val)
-{
-	return val & ((1 << (DESC_LENGTH_BITS_NUM + 1)) - 1);
-}
-
-static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num)
-{
-	u32 desc;
-
-	desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num));
-	desc &= ~0x1f;
-	return desc;
-}
-
-static irqreturn_t cppi41_irq(int irq, void *data)
-{
-	struct cppi41_dd *cdd = data;
-	u16 first_completion_queue = cdd->first_completion_queue;
-	u16 qmgr_num_pend = cdd->qmgr_num_pend;
-	struct cppi41_channel *c;
-	int i;
-
-	for (i = QMGR_PENDING_SLOT_Q(first_completion_queue); i < qmgr_num_pend;
-			i++) {
-		u32 val;
-		u32 q_num;
-
-		val = cppi_readl(cdd->qmgr_mem + QMGR_PEND(i));
-		if (i == QMGR_PENDING_SLOT_Q(first_completion_queue) && val) {
-			u32 mask;
-			/* set corresponding bit for completetion Q 93 */
-			mask = 1 << QMGR_PENDING_BIT_Q(first_completion_queue);
-			/* not set all bits for queues less than Q 93 */
-			mask--;
-			/* now invert and keep only Q 93+ set */
-			val &= ~mask;
-		}
-
-		if (val)
-			__iormb();
-
-		while (val) {
-			u32 desc, len;
-
-			/*
-			 * This should never trigger, see the comments in
-			 * push_desc_queue()
-			 */
-			WARN_ON(cdd->is_suspended);
-
-			q_num = __fls(val);
-			val &= ~(1 << q_num);
-			q_num += 32 * i;
-			desc = cppi41_pop_desc(cdd, q_num);
-			c = desc_to_chan(cdd, desc);
-			if (WARN_ON(!c)) {
-				pr_err("%s() q %d desc %08x\n", __func__,
-						q_num, desc);
-				continue;
-			}
-
-			if (c->desc->pd2 & PD2_ZERO_LENGTH)
-				len = 0;
-			else
-				len = pd_trans_len(c->desc->pd0);
-
-			c->residue = pd_trans_len(c->desc->pd6) - len;
-			dma_cookie_complete(&c->txd);
-			dmaengine_desc_get_callback_invoke(&c->txd, NULL);
-		}
-	}
-	return IRQ_HANDLED;
-}
-
-static dma_cookie_t cppi41_tx_submit(struct dma_async_tx_descriptor *tx)
-{
-	dma_cookie_t cookie;
-
-	cookie = dma_cookie_assign(tx);
-
-	return cookie;
-}
-
-static int cppi41_dma_alloc_chan_resources(struct dma_chan *chan)
-{
-	struct cppi41_channel *c = to_cpp41_chan(chan);
-	struct cppi41_dd *cdd = c->cdd;
-	int error;
-
-	error = pm_runtime_get_sync(cdd->ddev.dev);
-	if (error < 0) {
-		dev_err(cdd->ddev.dev, "%s pm runtime get: %i\n",
-			__func__, error);
-		pm_runtime_put_noidle(cdd->ddev.dev);
-
-		return error;
-	}
-
-	dma_cookie_init(chan);
-	dma_async_tx_descriptor_init(&c->txd, chan);
-	c->txd.tx_submit = cppi41_tx_submit;
-
-	if (!c->is_tx)
-		cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
-
-	pm_runtime_mark_last_busy(cdd->ddev.dev);
-	pm_runtime_put_autosuspend(cdd->ddev.dev);
-
-	return 0;
-}
-
-static void cppi41_dma_free_chan_resources(struct dma_chan *chan)
-{
-	struct cppi41_channel *c = to_cpp41_chan(chan);
-	struct cppi41_dd *cdd = c->cdd;
-	int error;
-
-	error = pm_runtime_get_sync(cdd->ddev.dev);
-	if (error < 0) {
-		pm_runtime_put_noidle(cdd->ddev.dev);
-
-		return;
-	}
-
-	WARN_ON(!list_empty(&cdd->pending));
-
-	pm_runtime_mark_last_busy(cdd->ddev.dev);
-	pm_runtime_put_autosuspend(cdd->ddev.dev);
-}
-
-static enum dma_status cppi41_dma_tx_status(struct dma_chan *chan,
-	dma_cookie_t cookie, struct dma_tx_state *txstate)
-{
-	struct cppi41_channel *c = to_cpp41_chan(chan);
-	enum dma_status ret;
-
-	ret = dma_cookie_status(chan, cookie, txstate);
-
-	dma_set_residue(txstate, c->residue);
-
-	return ret;
-}
-
-static void push_desc_queue(struct cppi41_channel *c)
-{
-	struct cppi41_dd *cdd = c->cdd;
-	u32 desc_num;
-	u32 desc_phys;
-	u32 reg;
-
-	c->residue = 0;
-
-	reg = GCR_CHAN_ENABLE;
-	if (!c->is_tx) {
-		reg |= GCR_STARV_RETRY;
-		reg |= GCR_DESC_TYPE_HOST;
-		reg |= c->q_comp_num;
-	}
-
-	cppi_writel(reg, c->gcr_reg);
-
-	/*
-	 * We don't use writel() but __raw_writel() so we have to make sure
-	 * that the DMA descriptor in coherent memory made to the main memory
-	 * before starting the dma engine.
-	 */
-	__iowmb();
-
-	/*
-	 * DMA transfers can take at least 200ms to complete with USB mass
-	 * storage connected. To prevent autosuspend timeouts, we must use
-	 * pm_runtime_get/put() when chan_busy[] is modified. This will get
-	 * cleared in desc_to_chan() or cppi41_stop_chan() depending on the
-	 * outcome of the transfer.
-	 */
-	pm_runtime_get(cdd->ddev.dev);
-
-	desc_phys = lower_32_bits(c->desc_phys);
-	desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
-	WARN_ON(cdd->chan_busy[desc_num]);
-	cdd->chan_busy[desc_num] = c;
-
-	reg = (sizeof(struct cppi41_desc) - 24) / 4;
-	reg |= desc_phys;
-	cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));
-}
-
-/*
- * Caller must hold cdd->lock to prevent push_desc_queue()
- * getting called out of order. We have both cppi41_dma_issue_pending()
- * and cppi41_runtime_resume() call this function.
- */
-static void cppi41_run_queue(struct cppi41_dd *cdd)
-{
-	struct cppi41_channel *c, *_c;
-
-	list_for_each_entry_safe(c, _c, &cdd->pending, node) {
-		push_desc_queue(c);
-		list_del(&c->node);
-	}
-}
-
-static void cppi41_dma_issue_pending(struct dma_chan *chan)
-{
-	struct cppi41_channel *c = to_cpp41_chan(chan);
-	struct cppi41_dd *cdd = c->cdd;
-	unsigned long flags;
-	int error;
-
-	error = pm_runtime_get(cdd->ddev.dev);
-	if ((error != -EINPROGRESS) && error < 0) {
-		pm_runtime_put_noidle(cdd->ddev.dev);
-		dev_err(cdd->ddev.dev, "Failed to pm_runtime_get: %i\n",
-			error);
-
-		return;
-	}
-
-	spin_lock_irqsave(&cdd->lock, flags);
-	list_add_tail(&c->node, &cdd->pending);
-	if (!cdd->is_suspended)
-		cppi41_run_queue(cdd);
-	spin_unlock_irqrestore(&cdd->lock, flags);
-
-	pm_runtime_mark_last_busy(cdd->ddev.dev);
-	pm_runtime_put_autosuspend(cdd->ddev.dev);
-}
-
-static u32 get_host_pd0(u32 length)
-{
-	u32 reg;
-
-	reg = DESC_TYPE_HOST << DESC_TYPE;
-	reg |= length;
-
-	return reg;
-}
-
-static u32 get_host_pd1(struct cppi41_channel *c)
-{
-	u32 reg;
-
-	reg = 0;
-
-	return reg;
-}
-
-static u32 get_host_pd2(struct cppi41_channel *c)
-{
-	u32 reg;
-
-	reg = DESC_TYPE_USB;
-	reg |= c->q_comp_num;
-
-	return reg;
-}
-
-static u32 get_host_pd3(u32 length)
-{
-	u32 reg;
-
-	/* PD3 = packet size */
-	reg = length;
-
-	return reg;
-}
-
-static u32 get_host_pd6(u32 length)
-{
-	u32 reg;
-
-	/* PD6 buffer size */
-	reg = DESC_PD_COMPLETE;
-	reg |= length;
-
-	return reg;
-}
-
-static u32 get_host_pd4_or_7(u32 addr)
-{
-	u32 reg;
-
-	reg = addr;
-
-	return reg;
-}
-
-static u32 get_host_pd5(void)
-{
-	u32 reg;
-
-	reg = 0;
-
-	return reg;
-}
-
-static struct dma_async_tx_descriptor *cppi41_dma_prep_slave_sg(
-	struct dma_chan *chan, struct scatterlist *sgl, unsigned sg_len,
-	enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
-{
-	struct cppi41_channel *c = to_cpp41_chan(chan);
-	struct cppi41_desc *d;
-	struct scatterlist *sg;
-	unsigned int i;
-
-	d = c->desc;
-	for_each_sg(sgl, sg, sg_len, i) {
-		u32 addr;
-		u32 len;
-
-		/* We need to use more than one desc once musb supports sg */
-		addr = lower_32_bits(sg_dma_address(sg));
-		len = sg_dma_len(sg);
-
-		d->pd0 = get_host_pd0(len);
-		d->pd1 = get_host_pd1(c);
-		d->pd2 = get_host_pd2(c);
-		d->pd3 = get_host_pd3(len);
-		d->pd4 = get_host_pd4_or_7(addr);
-		d->pd5 = get_host_pd5();
-		d->pd6 = get_host_pd6(len);
-		d->pd7 = get_host_pd4_or_7(addr);
-
-		d++;
-	}
-
-	return &c->txd;
-}
-
-static void cppi41_compute_td_desc(struct cppi41_desc *d)
-{
-	d->pd0 = DESC_TYPE_TEARD << DESC_TYPE;
-}
-
-static int cppi41_tear_down_chan(struct cppi41_channel *c)
-{
-	struct dmaengine_result abort_result;
-	struct cppi41_dd *cdd = c->cdd;
-	struct cppi41_desc *td;
-	u32 reg;
-	u32 desc_phys;
-	u32 td_desc_phys;
-
-	td = cdd->cd;
-	td += cdd->first_td_desc;
-
-	td_desc_phys = cdd->descs_phys;
-	td_desc_phys += cdd->first_td_desc * sizeof(struct cppi41_desc);
-
-	if (!c->td_queued) {
-		cppi41_compute_td_desc(td);
-		__iowmb();
-
-		reg = (sizeof(struct cppi41_desc) - 24) / 4;
-		reg |= td_desc_phys;
-		cppi_writel(reg, cdd->qmgr_mem +
-				QMGR_QUEUE_D(cdd->td_queue.submit));
-
-		reg = GCR_CHAN_ENABLE;
-		if (!c->is_tx) {
-			reg |= GCR_STARV_RETRY;
-			reg |= GCR_DESC_TYPE_HOST;
-			reg |= cdd->td_queue.complete;
-		}
-		reg |= GCR_TEARDOWN;
-		cppi_writel(reg, c->gcr_reg);
-		c->td_queued = 1;
-		c->td_retry = 500;
-	}
-
-	if (!c->td_seen || !c->td_desc_seen) {
-
-		desc_phys = cppi41_pop_desc(cdd, cdd->td_queue.complete);
-		if (!desc_phys && c->is_tx)
-			desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);
-
-		if (desc_phys == c->desc_phys) {
-			c->td_desc_seen = 1;
-
-		} else if (desc_phys == td_desc_phys) {
-			u32 pd0;
-
-			__iormb();
-			pd0 = td->pd0;
-			WARN_ON((pd0 >> DESC_TYPE) != DESC_TYPE_TEARD);
-			WARN_ON(!c->is_tx && !(pd0 & TD_DESC_IS_RX));
-			WARN_ON((pd0 & 0x1f) != c->port_num);
-			c->td_seen = 1;
-		} else if (desc_phys) {
-			WARN_ON_ONCE(1);
-		}
-	}
-	c->td_retry--;
-	/*
-	 * If the TX descriptor / channel is in use, the caller needs to poke
-	 * his TD bit multiple times. After that he hardware releases the
-	 * transfer descriptor followed by TD descriptor. Waiting seems not to
-	 * cause any difference.
-	 * RX seems to be thrown out right away. However once the TearDown
-	 * descriptor gets through we are done. If we have seens the transfer
-	 * descriptor before the TD we fetch it from enqueue, it has to be
-	 * there waiting for us.
-	 */
-	if (!c->td_seen && c->td_retry) {
-		udelay(1);
-		return -EAGAIN;
-	}
-	WARN_ON(!c->td_retry);
-
-	if (!c->td_desc_seen) {
-		desc_phys = cppi41_pop_desc(cdd, c->q_num);
-		if (!desc_phys)
-			desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);
-		WARN_ON(!desc_phys);
-	}
-
-	c->td_queued = 0;
-	c->td_seen = 0;
-	c->td_desc_seen = 0;
-	cppi_writel(0, c->gcr_reg);
-
-	/* Invoke the callback to do the necessary clean-up */
-	abort_result.result = DMA_TRANS_ABORTED;
-	dma_cookie_complete(&c->txd);
-	dmaengine_desc_get_callback_invoke(&c->txd, &abort_result);
-
-	return 0;
-}
-
-static int cppi41_stop_chan(struct dma_chan *chan)
-{
-	struct cppi41_channel *c = to_cpp41_chan(chan);
-	struct cppi41_dd *cdd = c->cdd;
-	u32 desc_num;
-	u32 desc_phys;
-	int ret;
-
-	desc_phys = lower_32_bits(c->desc_phys);
-	desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
-	if (!cdd->chan_busy[desc_num])
-		return 0;
-
-	ret = cppi41_tear_down_chan(c);
-	if (ret)
-		return ret;
-
-	WARN_ON(!cdd->chan_busy[desc_num]);
-	cdd->chan_busy[desc_num] = NULL;
-
-	/* Usecount for chan_busy[], paired with push_desc_queue() */
-	pm_runtime_put(cdd->ddev.dev);
-
-	return 0;
-}
-
-static int cppi41_add_chans(struct device *dev, struct cppi41_dd *cdd)
-{
-	struct cppi41_channel *cchan, *chans;
-	int i;
-	u32 n_chans = cdd->n_chans;
-
-	/*
-	 * The channels can only be used as TX or as RX. So we add twice
-	 * that much dma channels because USB can only do RX or TX.
-	 */
-	n_chans *= 2;
-
-	chans = devm_kcalloc(dev, n_chans, sizeof(*chans), GFP_KERNEL);
-	if (!chans)
-		return -ENOMEM;
-
-	for (i = 0; i < n_chans; i++) {
-		cchan = &chans[i];
-
-		cchan->cdd = cdd;
-		if (i & 1) {
-			cchan->gcr_reg = cdd->ctrl_mem + DMA_TXGCR(i >> 1);
-			cchan->is_tx = 1;
-		} else {
-			cchan->gcr_reg = cdd->ctrl_mem + DMA_RXGCR(i >> 1);
-			cchan->is_tx = 0;
-		}
-		cchan->port_num = i >> 1;
-		cchan->desc = &cdd->cd[i];
-		cchan->desc_phys = cdd->descs_phys;
-		cchan->desc_phys += i * sizeof(struct cppi41_desc);
-		cchan->chan.device = &cdd->ddev;
-		list_add_tail(&cchan->chan.device_node, &cdd->ddev.channels);
-	}
-	cdd->first_td_desc = n_chans;
-
-	return 0;
-}
-
-static void purge_descs(struct device *dev, struct cppi41_dd *cdd)
-{
-	unsigned int mem_decs;
-	int i;
-
-	mem_decs = ALLOC_DECS_NUM * sizeof(struct cppi41_desc);
-
-	for (i = 0; i < DESCS_AREAS; i++) {
-
-		cppi_writel(0, cdd->qmgr_mem + QMGR_MEMBASE(i));
-		cppi_writel(0, cdd->qmgr_mem + QMGR_MEMCTRL(i));
-
-		dma_free_coherent(dev, mem_decs, cdd->cd,
-				cdd->descs_phys);
-	}
-}
-
-static void disable_sched(struct cppi41_dd *cdd)
-{
-	cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
-}
-
-static void deinit_cppi41(struct device *dev, struct cppi41_dd *cdd)
-{
-	disable_sched(cdd);
-
-	purge_descs(dev, cdd);
-
-	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
-	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
-	dma_free_coherent(dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch,
-			cdd->scratch_phys);
-}
-
-static int init_descs(struct device *dev, struct cppi41_dd *cdd)
-{
-	unsigned int desc_size;
-	unsigned int mem_decs;
-	int i;
-	u32 reg;
-	u32 idx;
-
-	BUILD_BUG_ON(sizeof(struct cppi41_desc) &
-			(sizeof(struct cppi41_desc) - 1));
-	BUILD_BUG_ON(sizeof(struct cppi41_desc) < 32);
-	BUILD_BUG_ON(ALLOC_DECS_NUM < 32);
-
-	desc_size = sizeof(struct cppi41_desc);
-	mem_decs = ALLOC_DECS_NUM * desc_size;
-
-	idx = 0;
-	for (i = 0; i < DESCS_AREAS; i++) {
-
-		reg = idx << QMGR_MEMCTRL_IDX_SH;
-		reg |= (ilog2(desc_size) - 5) << QMGR_MEMCTRL_DESC_SH;
-		reg |= ilog2(ALLOC_DECS_NUM) - 5;
-
-		BUILD_BUG_ON(DESCS_AREAS != 1);
-		cdd->cd = dma_alloc_coherent(dev, mem_decs,
-				&cdd->descs_phys, GFP_KERNEL);
-		if (!cdd->cd)
-			return -ENOMEM;
-
-		cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));
-		cppi_writel(reg, cdd->qmgr_mem + QMGR_MEMCTRL(i));
-
-		idx += ALLOC_DECS_NUM;
-	}
-	return 0;
-}
-
-static void init_sched(struct cppi41_dd *cdd)
-{
-	unsigned ch;
-	unsigned word;
-	u32 reg;
-
-	word = 0;
-	cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
-	for (ch = 0; ch < cdd->n_chans; ch += 2) {
-
-		reg = SCHED_ENTRY0_CHAN(ch);
-		reg |= SCHED_ENTRY1_CHAN(ch) | SCHED_ENTRY1_IS_RX;
-
-		reg |= SCHED_ENTRY2_CHAN(ch + 1);
-		reg |= SCHED_ENTRY3_CHAN(ch + 1) | SCHED_ENTRY3_IS_RX;
-		cppi_writel(reg, cdd->sched_mem + DMA_SCHED_WORD(word));
-		word++;
-	}
-	reg = cdd->n_chans * 2 - 1;
-	reg |= DMA_SCHED_CTRL_EN;
-	cppi_writel(reg, cdd->sched_mem + DMA_SCHED_CTRL);
-}
-
-static int init_cppi41(struct device *dev, struct cppi41_dd *cdd)
-{
-	int ret;
-
-	BUILD_BUG_ON(QMGR_SCRATCH_SIZE > ((1 << 14) - 1));
-	cdd->qmgr_scratch = dma_alloc_coherent(dev, QMGR_SCRATCH_SIZE,
-			&cdd->scratch_phys, GFP_KERNEL);
-	if (!cdd->qmgr_scratch)
-		return -ENOMEM;
-
-	cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);
-	cppi_writel(TOTAL_DESCS_NUM, cdd->qmgr_mem + QMGR_LRAM_SIZE);
-	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
-
-	ret = init_descs(dev, cdd);
-	if (ret)
-		goto err_td;
-
-	cppi_writel(cdd->td_queue.submit, cdd->ctrl_mem + DMA_TDFDQ);
-	init_sched(cdd);
-
-	return 0;
-err_td:
-	deinit_cppi41(dev, cdd);
-	return ret;
-}
-
-static struct platform_driver cpp41_dma_driver;
-/*
- * The param format is:
- * X Y
- * X: Port
- * Y: 0 = RX else TX
- */
-#define INFO_PORT	0
-#define INFO_IS_TX	1
-
-static bool cpp41_dma_filter_fn(struct dma_chan *chan, void *param)
-{
-	struct cppi41_channel *cchan;
-	struct cppi41_dd *cdd;
-	const struct chan_queues *queues;
-	u32 *num = param;
-
-	if (chan->device->dev->driver != &cpp41_dma_driver.driver)
-		return false;
-
-	cchan = to_cpp41_chan(chan);
-
-	if (cchan->port_num != num[INFO_PORT])
-		return false;
-
-	if (cchan->is_tx && !num[INFO_IS_TX])
-		return false;
-	cdd = cchan->cdd;
-	if (cchan->is_tx)
-		queues = cdd->queues_tx;
-	else
-		queues = cdd->queues_rx;
-
-	BUILD_BUG_ON(ARRAY_SIZE(am335x_usb_queues_rx) !=
-		     ARRAY_SIZE(am335x_usb_queues_tx));
-	if (WARN_ON(cchan->port_num >= ARRAY_SIZE(am335x_usb_queues_rx)))
-		return false;
-
-	cchan->q_num = queues[cchan->port_num].submit;
-	cchan->q_comp_num = queues[cchan->port_num].complete;
-	return true;
-}
-
-static struct of_dma_filter_info cpp41_dma_info = {
-	.filter_fn = cpp41_dma_filter_fn,
-};
-
-static struct dma_chan *cppi41_dma_xlate(struct of_phandle_args *dma_spec,
-		struct of_dma *ofdma)
-{
-	int count = dma_spec->args_count;
-	struct of_dma_filter_info *info = ofdma->of_dma_data;
-
-	if (!info || !info->filter_fn)
-		return NULL;
-
-	if (count != 2)
-		return NULL;
-
-	return dma_request_channel(info->dma_cap, info->filter_fn,
-			&dma_spec->args[0]);
-}
-
-static const struct cppi_glue_infos am335x_usb_infos = {
-	.queues_rx = am335x_usb_queues_rx,
-	.queues_tx = am335x_usb_queues_tx,
-	.td_queue = { .submit = 31, .complete = 0 },
-	.first_completion_queue = 93,
-	.qmgr_num_pend = 5,
-};
-
-static const struct cppi_glue_infos da8xx_usb_infos = {
-	.queues_rx = da8xx_usb_queues_rx,
-	.queues_tx = da8xx_usb_queues_tx,
-	.td_queue = { .submit = 31, .complete = 0 },
-	.first_completion_queue = 24,
-	.qmgr_num_pend = 2,
-};
-
-static const struct of_device_id cppi41_dma_ids[] = {
-	{ .compatible = "ti,am3359-cppi41", .data = &am335x_usb_infos},
-	{ .compatible = "ti,da830-cppi41", .data = &da8xx_usb_infos},
-	{},
-};
-MODULE_DEVICE_TABLE(of, cppi41_dma_ids);
-
-static const struct cppi_glue_infos *get_glue_info(struct device *dev)
-{
-	const struct of_device_id *of_id;
-
-	of_id = of_match_node(cppi41_dma_ids, dev->of_node);
-	if (!of_id)
-		return NULL;
-	return of_id->data;
-}
-
-#define CPPI41_DMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
-				BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
-				BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
-				BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
-
-static int cppi41_dma_probe(struct platform_device *pdev)
-{
-	struct cppi41_dd *cdd;
-	struct device *dev = &pdev->dev;
-	const struct cppi_glue_infos *glue_info;
-	struct resource *mem;
-	int index;
-	int irq;
-	int ret;
-
-	glue_info = get_glue_info(dev);
-	if (!glue_info)
-		return -EINVAL;
-
-	cdd = devm_kzalloc(&pdev->dev, sizeof(*cdd), GFP_KERNEL);
-	if (!cdd)
-		return -ENOMEM;
-
-	dma_cap_set(DMA_SLAVE, cdd->ddev.cap_mask);
-	cdd->ddev.device_alloc_chan_resources = cppi41_dma_alloc_chan_resources;
-	cdd->ddev.device_free_chan_resources = cppi41_dma_free_chan_resources;
-	cdd->ddev.device_tx_status = cppi41_dma_tx_status;
-	cdd->ddev.device_issue_pending = cppi41_dma_issue_pending;
-	cdd->ddev.device_prep_slave_sg = cppi41_dma_prep_slave_sg;
-	cdd->ddev.device_terminate_all = cppi41_stop_chan;
-	cdd->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
-	cdd->ddev.src_addr_widths = CPPI41_DMA_BUSWIDTHS;
-	cdd->ddev.dst_addr_widths = CPPI41_DMA_BUSWIDTHS;
-	cdd->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
-	cdd->ddev.dev = dev;
-	INIT_LIST_HEAD(&cdd->ddev.channels);
-	cpp41_dma_info.dma_cap = cdd->ddev.cap_mask;
-
-	index = of_property_match_string(dev->of_node,
-					 "reg-names", "controller");
-	if (index < 0)
-		return index;
-
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, index);
-	cdd->ctrl_mem = devm_ioremap_resource(dev, mem);
-	if (IS_ERR(cdd->ctrl_mem))
-		return PTR_ERR(cdd->ctrl_mem);
-
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, index + 1);
-	cdd->sched_mem = devm_ioremap_resource(dev, mem);
-	if (IS_ERR(cdd->sched_mem))
-		return PTR_ERR(cdd->sched_mem);
-
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, index + 2);
-	cdd->qmgr_mem = devm_ioremap_resource(dev, mem);
-	if (IS_ERR(cdd->qmgr_mem))
-		return PTR_ERR(cdd->qmgr_mem);
-
-	spin_lock_init(&cdd->lock);
-	INIT_LIST_HEAD(&cdd->pending);
-
-	platform_set_drvdata(pdev, cdd);
-
-	pm_runtime_enable(dev);
-	pm_runtime_set_autosuspend_delay(dev, 100);
-	pm_runtime_use_autosuspend(dev);
-	ret = pm_runtime_get_sync(dev);
-	if (ret < 0)
-		goto err_get_sync;
-
-	cdd->queues_rx = glue_info->queues_rx;
-	cdd->queues_tx = glue_info->queues_tx;
-	cdd->td_queue = glue_info->td_queue;
-	cdd->qmgr_num_pend = glue_info->qmgr_num_pend;
-	cdd->first_completion_queue = glue_info->first_completion_queue;
-
-	ret = of_property_read_u32(dev->of_node,
-				   "#dma-channels", &cdd->n_chans);
-	if (ret)
-		goto err_get_n_chans;
-
-	ret = init_cppi41(dev, cdd);
-	if (ret)
-		goto err_init_cppi;
-
-	ret = cppi41_add_chans(dev, cdd);
-	if (ret)
-		goto err_chans;
-
-	irq = irq_of_parse_and_map(dev->of_node, 0);
-	if (!irq) {
-		ret = -EINVAL;
-		goto err_chans;
-	}
-
-	ret = devm_request_irq(&pdev->dev, irq, cppi41_irq, IRQF_SHARED,
-			dev_name(dev), cdd);
-	if (ret)
-		goto err_chans;
-	cdd->irq = irq;
-
-	ret = dma_async_device_register(&cdd->ddev);
-	if (ret)
-		goto err_chans;
-
-	ret = of_dma_controller_register(dev->of_node,
-			cppi41_dma_xlate, &cpp41_dma_info);
-	if (ret)
-		goto err_of;
-
-	pm_runtime_mark_last_busy(dev);
-	pm_runtime_put_autosuspend(dev);
-
-	return 0;
-err_of:
-	dma_async_device_unregister(&cdd->ddev);
-err_chans:
-	deinit_cppi41(dev, cdd);
-err_init_cppi:
-	pm_runtime_dont_use_autosuspend(dev);
-err_get_n_chans:
-err_get_sync:
-	pm_runtime_put_sync(dev);
-	pm_runtime_disable(dev);
-	return ret;
-}
-
-static int cppi41_dma_remove(struct platform_device *pdev)
-{
-	struct cppi41_dd *cdd = platform_get_drvdata(pdev);
-	int error;
-
-	error = pm_runtime_get_sync(&pdev->dev);
-	if (error < 0)
-		dev_err(&pdev->dev, "%s could not pm_runtime_get: %i\n",
-			__func__, error);
-	of_dma_controller_free(pdev->dev.of_node);
-	dma_async_device_unregister(&cdd->ddev);
-
-	devm_free_irq(&pdev->dev, cdd->irq, cdd);
-	deinit_cppi41(&pdev->dev, cdd);
-	pm_runtime_dont_use_autosuspend(&pdev->dev);
-	pm_runtime_put_sync(&pdev->dev);
-	pm_runtime_disable(&pdev->dev);
-	return 0;
-}
-
-static int __maybe_unused cppi41_suspend(struct device *dev)
-{
-	struct cppi41_dd *cdd = dev_get_drvdata(dev);
-
-	cdd->dma_tdfdq = cppi_readl(cdd->ctrl_mem + DMA_TDFDQ);
-	disable_sched(cdd);
-
-	return 0;
-}
-
-static int __maybe_unused cppi41_resume(struct device *dev)
-{
-	struct cppi41_dd *cdd = dev_get_drvdata(dev);
-	struct cppi41_channel *c;
-	int i;
-
-	for (i = 0; i < DESCS_AREAS; i++)
-		cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));
-
-	list_for_each_entry(c, &cdd->ddev.channels, chan.device_node)
-		if (!c->is_tx)
-			cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
-
-	init_sched(cdd);
-
-	cppi_writel(cdd->dma_tdfdq, cdd->ctrl_mem + DMA_TDFDQ);
-	cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);
-	cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE);
-	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
-
-	return 0;
-}
-
-static int __maybe_unused cppi41_runtime_suspend(struct device *dev)
-{
-	struct cppi41_dd *cdd = dev_get_drvdata(dev);
-	unsigned long flags;
-
-	spin_lock_irqsave(&cdd->lock, flags);
-	cdd->is_suspended = true;
-	WARN_ON(!list_empty(&cdd->pending));
-	spin_unlock_irqrestore(&cdd->lock, flags);
-
-	return 0;
-}
-
-static int __maybe_unused cppi41_runtime_resume(struct device *dev)
-{
-	struct cppi41_dd *cdd = dev_get_drvdata(dev);
-	unsigned long flags;
-
-	spin_lock_irqsave(&cdd->lock, flags);
-	cdd->is_suspended = false;
-	cppi41_run_queue(cdd);
-	spin_unlock_irqrestore(&cdd->lock, flags);
-
-	return 0;
-}
-
-static const struct dev_pm_ops cppi41_pm_ops = {
-	SET_LATE_SYSTEM_SLEEP_PM_OPS(cppi41_suspend, cppi41_resume)
-	SET_RUNTIME_PM_OPS(cppi41_runtime_suspend,
-			   cppi41_runtime_resume,
-			   NULL)
-};
-
-static struct platform_driver cpp41_dma_driver = {
-	.probe  = cppi41_dma_probe,
-	.remove = cppi41_dma_remove,
-	.driver = {
-		.name = "cppi41-dma-engine",
-		.pm = &cppi41_pm_ops,
-		.of_match_table = of_match_ptr(cppi41_dma_ids),
-	},
-};
-
-module_platform_driver(cpp41_dma_driver);
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Sebastian Andrzej Siewior <bigeasy@linutronix.de>");
diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c
index 2419fe524daa..15b2453d2647 100644
--- a/drivers/dma/dma-axi-dmac.c
+++ b/drivers/dma/dma-axi-dmac.c
@@ -687,7 +687,7 @@ static int axi_dmac_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_unregister_device;
 
-	ret = request_irq(dmac->irq, axi_dmac_interrupt_handler, 0,
+	ret = request_irq(dmac->irq, axi_dmac_interrupt_handler, IRQF_SHARED,
 		dev_name(&pdev->dev), dmac);
 	if (ret)
 		goto err_unregister_of;
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index b9339524d5bd..aa1712beb0cc 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -468,6 +468,8 @@ static int dmatest_func(void *data)
 	unsigned long long	total_len = 0;
 	u8			align = 0;
 	bool			is_memset = false;
+	dma_addr_t		*srcs;
+	dma_addr_t		*dma_pq;
 
 	set_freezable();
 
@@ -551,6 +553,14 @@ static int dmatest_func(void *data)
 
 	set_user_nice(current, 10);
 
+	srcs = kcalloc(src_cnt, sizeof(dma_addr_t), GFP_KERNEL);
+	if (!srcs)
+		goto err_dstbuf;
+
+	dma_pq = kcalloc(dst_cnt, sizeof(dma_addr_t), GFP_KERNEL);
+	if (!dma_pq)
+		goto err_srcs_array;
+
 	/*
 	 * src and dst buffers are freed by ourselves below
 	 */
@@ -561,7 +571,6 @@ static int dmatest_func(void *data)
 	       && !(params->iterations && total_tests >= params->iterations)) {
 		struct dma_async_tx_descriptor *tx = NULL;
 		struct dmaengine_unmap_data *um;
-		dma_addr_t srcs[src_cnt];
 		dma_addr_t *dsts;
 		unsigned int src_off, dst_off, len;
 
@@ -676,8 +685,6 @@ static int dmatest_func(void *data)
 						      srcs, src_cnt,
 						      len, flags);
 		else if (thread->type == DMA_PQ) {
-			dma_addr_t dma_pq[dst_cnt];
-
 			for (i = 0; i < dst_cnt; i++)
 				dma_pq[i] = dsts[i] + dst_off;
 			tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
@@ -779,6 +786,9 @@ static int dmatest_func(void *data)
 	runtime = ktime_to_us(ktime);
 
 	ret = 0;
+	kfree(dma_pq);
+err_srcs_array:
+	kfree(srcs);
 err_dstbuf:
 	for (i = 0; thread->udsts[i]; i++)
 		kfree(thread->udsts[i]);
diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c
index bc31fe802061..f62dd0944908 100644
--- a/drivers/dma/dw/platform.c
+++ b/drivers/dma/dw/platform.c
@@ -293,8 +293,7 @@ MODULE_DEVICE_TABLE(acpi, dw_dma_acpi_id_table);
 
 static int dw_suspend_late(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct dw_dma_chip *chip = platform_get_drvdata(pdev);
+	struct dw_dma_chip *chip = dev_get_drvdata(dev);
 
 	dw_dma_disable(chip);
 	clk_disable_unprepare(chip->clk);
@@ -304,8 +303,7 @@ static int dw_suspend_late(struct device *dev)
 
 static int dw_resume_early(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct dw_dma_chip *chip = platform_get_drvdata(pdev);
+	struct dw_dma_chip *chip = dev_get_drvdata(dev);
 	int ret;
 
 	ret = clk_prepare_enable(chip->clk);
diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c
deleted file mode 100644
index 85ea92fcea54..000000000000
--- a/drivers/dma/edma.c
+++ /dev/null
@@ -1,2568 +0,0 @@
-/*
- * TI EDMA DMA engine driver
- *
- * Copyright 2012 Texas Instruments
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/dmaengine.h>
-#include <linux/dma-mapping.h>
-#include <linux/edma.h>
-#include <linux/err.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/list.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-#include <linux/spinlock.h>
-#include <linux/of.h>
-#include <linux/of_dma.h>
-#include <linux/of_irq.h>
-#include <linux/of_address.h>
-#include <linux/of_device.h>
-#include <linux/pm_runtime.h>
-
-#include <linux/platform_data/edma.h>
-
-#include "dmaengine.h"
-#include "virt-dma.h"
-
-/* Offsets matching "struct edmacc_param" */
-#define PARM_OPT		0x00
-#define PARM_SRC		0x04
-#define PARM_A_B_CNT		0x08
-#define PARM_DST		0x0c
-#define PARM_SRC_DST_BIDX	0x10
-#define PARM_LINK_BCNTRLD	0x14
-#define PARM_SRC_DST_CIDX	0x18
-#define PARM_CCNT		0x1c
-
-#define PARM_SIZE		0x20
-
-/* Offsets for EDMA CC global channel registers and their shadows */
-#define SH_ER			0x00	/* 64 bits */
-#define SH_ECR			0x08	/* 64 bits */
-#define SH_ESR			0x10	/* 64 bits */
-#define SH_CER			0x18	/* 64 bits */
-#define SH_EER			0x20	/* 64 bits */
-#define SH_EECR			0x28	/* 64 bits */
-#define SH_EESR			0x30	/* 64 bits */
-#define SH_SER			0x38	/* 64 bits */
-#define SH_SECR			0x40	/* 64 bits */
-#define SH_IER			0x50	/* 64 bits */
-#define SH_IECR			0x58	/* 64 bits */
-#define SH_IESR			0x60	/* 64 bits */
-#define SH_IPR			0x68	/* 64 bits */
-#define SH_ICR			0x70	/* 64 bits */
-#define SH_IEVAL		0x78
-#define SH_QER			0x80
-#define SH_QEER			0x84
-#define SH_QEECR		0x88
-#define SH_QEESR		0x8c
-#define SH_QSER			0x90
-#define SH_QSECR		0x94
-#define SH_SIZE			0x200
-
-/* Offsets for EDMA CC global registers */
-#define EDMA_REV		0x0000
-#define EDMA_CCCFG		0x0004
-#define EDMA_QCHMAP		0x0200	/* 8 registers */
-#define EDMA_DMAQNUM		0x0240	/* 8 registers (4 on OMAP-L1xx) */
-#define EDMA_QDMAQNUM		0x0260
-#define EDMA_QUETCMAP		0x0280
-#define EDMA_QUEPRI		0x0284
-#define EDMA_EMR		0x0300	/* 64 bits */
-#define EDMA_EMCR		0x0308	/* 64 bits */
-#define EDMA_QEMR		0x0310
-#define EDMA_QEMCR		0x0314
-#define EDMA_CCERR		0x0318
-#define EDMA_CCERRCLR		0x031c
-#define EDMA_EEVAL		0x0320
-#define EDMA_DRAE		0x0340	/* 4 x 64 bits*/
-#define EDMA_QRAE		0x0380	/* 4 registers */
-#define EDMA_QUEEVTENTRY	0x0400	/* 2 x 16 registers */
-#define EDMA_QSTAT		0x0600	/* 2 registers */
-#define EDMA_QWMTHRA		0x0620
-#define EDMA_QWMTHRB		0x0624
-#define EDMA_CCSTAT		0x0640
-
-#define EDMA_M			0x1000	/* global channel registers */
-#define EDMA_ECR		0x1008
-#define EDMA_ECRH		0x100C
-#define EDMA_SHADOW0		0x2000	/* 4 shadow regions */
-#define EDMA_PARM		0x4000	/* PaRAM entries */
-
-#define PARM_OFFSET(param_no)	(EDMA_PARM + ((param_no) << 5))
-
-#define EDMA_DCHMAP		0x0100  /* 64 registers */
-
-/* CCCFG register */
-#define GET_NUM_DMACH(x)	(x & 0x7) /* bits 0-2 */
-#define GET_NUM_QDMACH(x)	((x & 0x70) >> 4) /* bits 4-6 */
-#define GET_NUM_PAENTRY(x)	((x & 0x7000) >> 12) /* bits 12-14 */
-#define GET_NUM_EVQUE(x)	((x & 0x70000) >> 16) /* bits 16-18 */
-#define GET_NUM_REGN(x)		((x & 0x300000) >> 20) /* bits 20-21 */
-#define CHMAP_EXIST		BIT(24)
-
-/* CCSTAT register */
-#define EDMA_CCSTAT_ACTV	BIT(4)
-
-/*
- * Max of 20 segments per channel to conserve PaRAM slots
- * Also note that MAX_NR_SG should be atleast the no.of periods
- * that are required for ASoC, otherwise DMA prep calls will
- * fail. Today davinci-pcm is the only user of this driver and
- * requires atleast 17 slots, so we setup the default to 20.
- */
-#define MAX_NR_SG		20
-#define EDMA_MAX_SLOTS		MAX_NR_SG
-#define EDMA_DESCRIPTORS	16
-
-#define EDMA_CHANNEL_ANY		-1	/* for edma_alloc_channel() */
-#define EDMA_SLOT_ANY			-1	/* for edma_alloc_slot() */
-#define EDMA_CONT_PARAMS_ANY		 1001
-#define EDMA_CONT_PARAMS_FIXED_EXACT	 1002
-#define EDMA_CONT_PARAMS_FIXED_NOT_EXACT 1003
-
-/* PaRAM slots are laid out like this */
-struct edmacc_param {
-	u32 opt;
-	u32 src;
-	u32 a_b_cnt;
-	u32 dst;
-	u32 src_dst_bidx;
-	u32 link_bcntrld;
-	u32 src_dst_cidx;
-	u32 ccnt;
-} __packed;
-
-/* fields in edmacc_param.opt */
-#define SAM		BIT(0)
-#define DAM		BIT(1)
-#define SYNCDIM		BIT(2)
-#define STATIC		BIT(3)
-#define EDMA_FWID	(0x07 << 8)
-#define TCCMODE		BIT(11)
-#define EDMA_TCC(t)	((t) << 12)
-#define TCINTEN		BIT(20)
-#define ITCINTEN	BIT(21)
-#define TCCHEN		BIT(22)
-#define ITCCHEN		BIT(23)
-
-struct edma_pset {
-	u32				len;
-	dma_addr_t			addr;
-	struct edmacc_param		param;
-};
-
-struct edma_desc {
-	struct virt_dma_desc		vdesc;
-	struct list_head		node;
-	enum dma_transfer_direction	direction;
-	int				cyclic;
-	int				absync;
-	int				pset_nr;
-	struct edma_chan		*echan;
-	int				processed;
-
-	/*
-	 * The following 4 elements are used for residue accounting.
-	 *
-	 * - processed_stat: the number of SG elements we have traversed
-	 * so far to cover accounting. This is updated directly to processed
-	 * during edma_callback and is always <= processed, because processed
-	 * refers to the number of pending transfer (programmed to EDMA
-	 * controller), where as processed_stat tracks number of transfers
-	 * accounted for so far.
-	 *
-	 * - residue: The amount of bytes we have left to transfer for this desc
-	 *
-	 * - residue_stat: The residue in bytes of data we have covered
-	 * so far for accounting. This is updated directly to residue
-	 * during callbacks to keep it current.
-	 *
-	 * - sg_len: Tracks the length of the current intermediate transfer,
-	 * this is required to update the residue during intermediate transfer
-	 * completion callback.
-	 */
-	int				processed_stat;
-	u32				sg_len;
-	u32				residue;
-	u32				residue_stat;
-
-	struct edma_pset		pset[0];
-};
-
-struct edma_cc;
-
-struct edma_tc {
-	struct device_node		*node;
-	u16				id;
-};
-
-struct edma_chan {
-	struct virt_dma_chan		vchan;
-	struct list_head		node;
-	struct edma_desc		*edesc;
-	struct edma_cc			*ecc;
-	struct edma_tc			*tc;
-	int				ch_num;
-	bool				alloced;
-	bool				hw_triggered;
-	int				slot[EDMA_MAX_SLOTS];
-	int				missed;
-	struct dma_slave_config		cfg;
-};
-
-struct edma_cc {
-	struct device			*dev;
-	struct edma_soc_info		*info;
-	void __iomem			*base;
-	int				id;
-	bool				legacy_mode;
-
-	/* eDMA3 resource information */
-	unsigned			num_channels;
-	unsigned			num_qchannels;
-	unsigned			num_region;
-	unsigned			num_slots;
-	unsigned			num_tc;
-	bool				chmap_exist;
-	enum dma_event_q		default_queue;
-
-	unsigned int			ccint;
-	unsigned int			ccerrint;
-
-	/*
-	 * The slot_inuse bit for each PaRAM slot is clear unless the slot is
-	 * in use by Linux or if it is allocated to be used by DSP.
-	 */
-	unsigned long *slot_inuse;
-
-	struct dma_device		dma_slave;
-	struct dma_device		*dma_memcpy;
-	struct edma_chan		*slave_chans;
-	struct edma_tc			*tc_list;
-	int				dummy_slot;
-};
-
-/* dummy param set used to (re)initialize parameter RAM slots */
-static const struct edmacc_param dummy_paramset = {
-	.link_bcntrld = 0xffff,
-	.ccnt = 1,
-};
-
-#define EDMA_BINDING_LEGACY	0
-#define EDMA_BINDING_TPCC	1
-static const u32 edma_binding_type[] = {
-	[EDMA_BINDING_LEGACY] = EDMA_BINDING_LEGACY,
-	[EDMA_BINDING_TPCC] = EDMA_BINDING_TPCC,
-};
-
-static const struct of_device_id edma_of_ids[] = {
-	{
-		.compatible = "ti,edma3",
-		.data = &edma_binding_type[EDMA_BINDING_LEGACY],
-	},
-	{
-		.compatible = "ti,edma3-tpcc",
-		.data = &edma_binding_type[EDMA_BINDING_TPCC],
-	},
-	{}
-};
-MODULE_DEVICE_TABLE(of, edma_of_ids);
-
-static const struct of_device_id edma_tptc_of_ids[] = {
-	{ .compatible = "ti,edma3-tptc", },
-	{}
-};
-MODULE_DEVICE_TABLE(of, edma_tptc_of_ids);
-
-static inline unsigned int edma_read(struct edma_cc *ecc, int offset)
-{
-	return (unsigned int)__raw_readl(ecc->base + offset);
-}
-
-static inline void edma_write(struct edma_cc *ecc, int offset, int val)
-{
-	__raw_writel(val, ecc->base + offset);
-}
-
-static inline void edma_modify(struct edma_cc *ecc, int offset, unsigned and,
-			       unsigned or)
-{
-	unsigned val = edma_read(ecc, offset);
-
-	val &= and;
-	val |= or;
-	edma_write(ecc, offset, val);
-}
-
-static inline void edma_and(struct edma_cc *ecc, int offset, unsigned and)
-{
-	unsigned val = edma_read(ecc, offset);
-
-	val &= and;
-	edma_write(ecc, offset, val);
-}
-
-static inline void edma_or(struct edma_cc *ecc, int offset, unsigned or)
-{
-	unsigned val = edma_read(ecc, offset);
-
-	val |= or;
-	edma_write(ecc, offset, val);
-}
-
-static inline unsigned int edma_read_array(struct edma_cc *ecc, int offset,
-					   int i)
-{
-	return edma_read(ecc, offset + (i << 2));
-}
-
-static inline void edma_write_array(struct edma_cc *ecc, int offset, int i,
-				    unsigned val)
-{
-	edma_write(ecc, offset + (i << 2), val);
-}
-
-static inline void edma_modify_array(struct edma_cc *ecc, int offset, int i,
-				     unsigned and, unsigned or)
-{
-	edma_modify(ecc, offset + (i << 2), and, or);
-}
-
-static inline void edma_or_array(struct edma_cc *ecc, int offset, int i,
-				 unsigned or)
-{
-	edma_or(ecc, offset + (i << 2), or);
-}
-
-static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j,
-				  unsigned or)
-{
-	edma_or(ecc, offset + ((i * 2 + j) << 2), or);
-}
-
-static inline void edma_write_array2(struct edma_cc *ecc, int offset, int i,
-				     int j, unsigned val)
-{
-	edma_write(ecc, offset + ((i * 2 + j) << 2), val);
-}
-
-static inline unsigned int edma_shadow0_read(struct edma_cc *ecc, int offset)
-{
-	return edma_read(ecc, EDMA_SHADOW0 + offset);
-}
-
-static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc,
-						   int offset, int i)
-{
-	return edma_read(ecc, EDMA_SHADOW0 + offset + (i << 2));
-}
-
-static inline void edma_shadow0_write(struct edma_cc *ecc, int offset,
-				      unsigned val)
-{
-	edma_write(ecc, EDMA_SHADOW0 + offset, val);
-}
-
-static inline void edma_shadow0_write_array(struct edma_cc *ecc, int offset,
-					    int i, unsigned val)
-{
-	edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val);
-}
-
-static inline unsigned int edma_param_read(struct edma_cc *ecc, int offset,
-					   int param_no)
-{
-	return edma_read(ecc, EDMA_PARM + offset + (param_no << 5));
-}
-
-static inline void edma_param_write(struct edma_cc *ecc, int offset,
-				    int param_no, unsigned val)
-{
-	edma_write(ecc, EDMA_PARM + offset + (param_no << 5), val);
-}
-
-static inline void edma_param_modify(struct edma_cc *ecc, int offset,
-				     int param_no, unsigned and, unsigned or)
-{
-	edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or);
-}
-
-static inline void edma_param_and(struct edma_cc *ecc, int offset, int param_no,
-				  unsigned and)
-{
-	edma_and(ecc, EDMA_PARM + offset + (param_no << 5), and);
-}
-
-static inline void edma_param_or(struct edma_cc *ecc, int offset, int param_no,
-				 unsigned or)
-{
-	edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or);
-}
-
-static inline void edma_set_bits(int offset, int len, unsigned long *p)
-{
-	for (; len > 0; len--)
-		set_bit(offset + (len - 1), p);
-}
-
-static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
-					  int priority)
-{
-	int bit = queue_no * 4;
-
-	edma_modify(ecc, EDMA_QUEPRI, ~(0x7 << bit), ((priority & 0x7) << bit));
-}
-
-static void edma_set_chmap(struct edma_chan *echan, int slot)
-{
-	struct edma_cc *ecc = echan->ecc;
-	int channel = EDMA_CHAN_SLOT(echan->ch_num);
-
-	if (ecc->chmap_exist) {
-		slot = EDMA_CHAN_SLOT(slot);
-		edma_write_array(ecc, EDMA_DCHMAP, channel, (slot << 5));
-	}
-}
-
-static void edma_setup_interrupt(struct edma_chan *echan, bool enable)
-{
-	struct edma_cc *ecc = echan->ecc;
-	int channel = EDMA_CHAN_SLOT(echan->ch_num);
-
-	if (enable) {
-		edma_shadow0_write_array(ecc, SH_ICR, channel >> 5,
-					 BIT(channel & 0x1f));
-		edma_shadow0_write_array(ecc, SH_IESR, channel >> 5,
-					 BIT(channel & 0x1f));
-	} else {
-		edma_shadow0_write_array(ecc, SH_IECR, channel >> 5,
-					 BIT(channel & 0x1f));
-	}
-}
-
-/*
- * paRAM slot management functions
- */
-static void edma_write_slot(struct edma_cc *ecc, unsigned slot,
-			    const struct edmacc_param *param)
-{
-	slot = EDMA_CHAN_SLOT(slot);
-	if (slot >= ecc->num_slots)
-		return;
-	memcpy_toio(ecc->base + PARM_OFFSET(slot), param, PARM_SIZE);
-}
-
-static int edma_read_slot(struct edma_cc *ecc, unsigned slot,
-			   struct edmacc_param *param)
-{
-	slot = EDMA_CHAN_SLOT(slot);
-	if (slot >= ecc->num_slots)
-		return -EINVAL;
-	memcpy_fromio(param, ecc->base + PARM_OFFSET(slot), PARM_SIZE);
-
-	return 0;
-}
-
-/**
- * edma_alloc_slot - allocate DMA parameter RAM
- * @ecc: pointer to edma_cc struct
- * @slot: specific slot to allocate; negative for "any unused slot"
- *
- * This allocates a parameter RAM slot, initializing it to hold a
- * dummy transfer.  Slots allocated using this routine have not been
- * mapped to a hardware DMA channel, and will normally be used by
- * linking to them from a slot associated with a DMA channel.
- *
- * Normal use is to pass EDMA_SLOT_ANY as the @slot, but specific
- * slots may be allocated on behalf of DSP firmware.
- *
- * Returns the number of the slot, else negative errno.
- */
-static int edma_alloc_slot(struct edma_cc *ecc, int slot)
-{
-	if (slot >= 0) {
-		slot = EDMA_CHAN_SLOT(slot);
-		/* Requesting entry paRAM slot for a HW triggered channel. */
-		if (ecc->chmap_exist && slot < ecc->num_channels)
-			slot = EDMA_SLOT_ANY;
-	}
-
-	if (slot < 0) {
-		if (ecc->chmap_exist)
-			slot = 0;
-		else
-			slot = ecc->num_channels;
-		for (;;) {
-			slot = find_next_zero_bit(ecc->slot_inuse,
-						  ecc->num_slots,
-						  slot);
-			if (slot == ecc->num_slots)
-				return -ENOMEM;
-			if (!test_and_set_bit(slot, ecc->slot_inuse))
-				break;
-		}
-	} else if (slot >= ecc->num_slots) {
-		return -EINVAL;
-	} else if (test_and_set_bit(slot, ecc->slot_inuse)) {
-		return -EBUSY;
-	}
-
-	edma_write_slot(ecc, slot, &dummy_paramset);
-
-	return EDMA_CTLR_CHAN(ecc->id, slot);
-}
-
-static void edma_free_slot(struct edma_cc *ecc, unsigned slot)
-{
-	slot = EDMA_CHAN_SLOT(slot);
-	if (slot >= ecc->num_slots)
-		return;
-
-	edma_write_slot(ecc, slot, &dummy_paramset);
-	clear_bit(slot, ecc->slot_inuse);
-}
-
-/**
- * edma_link - link one parameter RAM slot to another
- * @ecc: pointer to edma_cc struct
- * @from: parameter RAM slot originating the link
- * @to: parameter RAM slot which is the link target
- *
- * The originating slot should not be part of any active DMA transfer.
- */
-static void edma_link(struct edma_cc *ecc, unsigned from, unsigned to)
-{
-	if (unlikely(EDMA_CTLR(from) != EDMA_CTLR(to)))
-		dev_warn(ecc->dev, "Ignoring eDMA instance for linking\n");
-
-	from = EDMA_CHAN_SLOT(from);
-	to = EDMA_CHAN_SLOT(to);
-	if (from >= ecc->num_slots || to >= ecc->num_slots)
-		return;
-
-	edma_param_modify(ecc, PARM_LINK_BCNTRLD, from, 0xffff0000,
-			  PARM_OFFSET(to));
-}
-
-/**
- * edma_get_position - returns the current transfer point
- * @ecc: pointer to edma_cc struct
- * @slot: parameter RAM slot being examined
- * @dst:  true selects the dest position, false the source
- *
- * Returns the position of the current active slot
- */
-static dma_addr_t edma_get_position(struct edma_cc *ecc, unsigned slot,
-				    bool dst)
-{
-	u32 offs;
-
-	slot = EDMA_CHAN_SLOT(slot);
-	offs = PARM_OFFSET(slot);
-	offs += dst ? PARM_DST : PARM_SRC;
-
-	return edma_read(ecc, offs);
-}
-
-/*
- * Channels with event associations will be triggered by their hardware
- * events, and channels without such associations will be triggered by
- * software.  (At this writing there is no interface for using software
- * triggers except with channels that don't support hardware triggers.)
- */
-static void edma_start(struct edma_chan *echan)
-{
-	struct edma_cc *ecc = echan->ecc;
-	int channel = EDMA_CHAN_SLOT(echan->ch_num);
-	int j = (channel >> 5);
-	unsigned int mask = BIT(channel & 0x1f);
-
-	if (!echan->hw_triggered) {
-		/* EDMA channels without event association */
-		dev_dbg(ecc->dev, "ESR%d %08x\n", j,
-			edma_shadow0_read_array(ecc, SH_ESR, j));
-		edma_shadow0_write_array(ecc, SH_ESR, j, mask);
-	} else {
-		/* EDMA channel with event association */
-		dev_dbg(ecc->dev, "ER%d %08x\n", j,
-			edma_shadow0_read_array(ecc, SH_ER, j));
-		/* Clear any pending event or error */
-		edma_write_array(ecc, EDMA_ECR, j, mask);
-		edma_write_array(ecc, EDMA_EMCR, j, mask);
-		/* Clear any SER */
-		edma_shadow0_write_array(ecc, SH_SECR, j, mask);
-		edma_shadow0_write_array(ecc, SH_EESR, j, mask);
-		dev_dbg(ecc->dev, "EER%d %08x\n", j,
-			edma_shadow0_read_array(ecc, SH_EER, j));
-	}
-}
-
-static void edma_stop(struct edma_chan *echan)
-{
-	struct edma_cc *ecc = echan->ecc;
-	int channel = EDMA_CHAN_SLOT(echan->ch_num);
-	int j = (channel >> 5);
-	unsigned int mask = BIT(channel & 0x1f);
-
-	edma_shadow0_write_array(ecc, SH_EECR, j, mask);
-	edma_shadow0_write_array(ecc, SH_ECR, j, mask);
-	edma_shadow0_write_array(ecc, SH_SECR, j, mask);
-	edma_write_array(ecc, EDMA_EMCR, j, mask);
-
-	/* clear possibly pending completion interrupt */
-	edma_shadow0_write_array(ecc, SH_ICR, j, mask);
-
-	dev_dbg(ecc->dev, "EER%d %08x\n", j,
-		edma_shadow0_read_array(ecc, SH_EER, j));
-
-	/* REVISIT:  consider guarding against inappropriate event
-	 * chaining by overwriting with dummy_paramset.
-	 */
-}
-
-/*
- * Temporarily disable EDMA hardware events on the specified channel,
- * preventing them from triggering new transfers
- */
-static void edma_pause(struct edma_chan *echan)
-{
-	int channel = EDMA_CHAN_SLOT(echan->ch_num);
-	unsigned int mask = BIT(channel & 0x1f);
-
-	edma_shadow0_write_array(echan->ecc, SH_EECR, channel >> 5, mask);
-}
-
-/* Re-enable EDMA hardware events on the specified channel.  */
-static void edma_resume(struct edma_chan *echan)
-{
-	int channel = EDMA_CHAN_SLOT(echan->ch_num);
-	unsigned int mask = BIT(channel & 0x1f);
-
-	edma_shadow0_write_array(echan->ecc, SH_EESR, channel >> 5, mask);
-}
-
-static void edma_trigger_channel(struct edma_chan *echan)
-{
-	struct edma_cc *ecc = echan->ecc;
-	int channel = EDMA_CHAN_SLOT(echan->ch_num);
-	unsigned int mask = BIT(channel & 0x1f);
-
-	edma_shadow0_write_array(ecc, SH_ESR, (channel >> 5), mask);
-
-	dev_dbg(ecc->dev, "ESR%d %08x\n", (channel >> 5),
-		edma_shadow0_read_array(ecc, SH_ESR, (channel >> 5)));
-}
-
-static void edma_clean_channel(struct edma_chan *echan)
-{
-	struct edma_cc *ecc = echan->ecc;
-	int channel = EDMA_CHAN_SLOT(echan->ch_num);
-	int j = (channel >> 5);
-	unsigned int mask = BIT(channel & 0x1f);
-
-	dev_dbg(ecc->dev, "EMR%d %08x\n", j, edma_read_array(ecc, EDMA_EMR, j));
-	edma_shadow0_write_array(ecc, SH_ECR, j, mask);
-	/* Clear the corresponding EMR bits */
-	edma_write_array(ecc, EDMA_EMCR, j, mask);
-	/* Clear any SER */
-	edma_shadow0_write_array(ecc, SH_SECR, j, mask);
-	edma_write(ecc, EDMA_CCERRCLR, BIT(16) | BIT(1) | BIT(0));
-}
-
-/* Move channel to a specific event queue */
-static void edma_assign_channel_eventq(struct edma_chan *echan,
-				       enum dma_event_q eventq_no)
-{
-	struct edma_cc *ecc = echan->ecc;
-	int channel = EDMA_CHAN_SLOT(echan->ch_num);
-	int bit = (channel & 0x7) * 4;
-
-	/* default to low priority queue */
-	if (eventq_no == EVENTQ_DEFAULT)
-		eventq_no = ecc->default_queue;
-	if (eventq_no >= ecc->num_tc)
-		return;
-
-	eventq_no &= 7;
-	edma_modify_array(ecc, EDMA_DMAQNUM, (channel >> 3), ~(0x7 << bit),
-			  eventq_no << bit);
-}
-
-static int edma_alloc_channel(struct edma_chan *echan,
-			      enum dma_event_q eventq_no)
-{
-	struct edma_cc *ecc = echan->ecc;
-	int channel = EDMA_CHAN_SLOT(echan->ch_num);
-
-	/* ensure access through shadow region 0 */
-	edma_or_array2(ecc, EDMA_DRAE, 0, channel >> 5, BIT(channel & 0x1f));
-
-	/* ensure no events are pending */
-	edma_stop(echan);
-
-	edma_setup_interrupt(echan, true);
-
-	edma_assign_channel_eventq(echan, eventq_no);
-
-	return 0;
-}
-
-static void edma_free_channel(struct edma_chan *echan)
-{
-	/* ensure no events are pending */
-	edma_stop(echan);
-	/* REVISIT should probably take out of shadow region 0 */
-	edma_setup_interrupt(echan, false);
-}
-
-static inline struct edma_cc *to_edma_cc(struct dma_device *d)
-{
-	return container_of(d, struct edma_cc, dma_slave);
-}
-
-static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
-{
-	return container_of(c, struct edma_chan, vchan.chan);
-}
-
-static inline struct edma_desc *to_edma_desc(struct dma_async_tx_descriptor *tx)
-{
-	return container_of(tx, struct edma_desc, vdesc.tx);
-}
-
-static void edma_desc_free(struct virt_dma_desc *vdesc)
-{
-	kfree(container_of(vdesc, struct edma_desc, vdesc));
-}
-
-/* Dispatch a queued descriptor to the controller (caller holds lock) */
-static void edma_execute(struct edma_chan *echan)
-{
-	struct edma_cc *ecc = echan->ecc;
-	struct virt_dma_desc *vdesc;
-	struct edma_desc *edesc;
-	struct device *dev = echan->vchan.chan.device->dev;
-	int i, j, left, nslots;
-
-	if (!echan->edesc) {
-		/* Setup is needed for the first transfer */
-		vdesc = vchan_next_desc(&echan->vchan);
-		if (!vdesc)
-			return;
-		list_del(&vdesc->node);
-		echan->edesc = to_edma_desc(&vdesc->tx);
-	}
-
-	edesc = echan->edesc;
-
-	/* Find out how many left */
-	left = edesc->pset_nr - edesc->processed;
-	nslots = min(MAX_NR_SG, left);
-	edesc->sg_len = 0;
-
-	/* Write descriptor PaRAM set(s) */
-	for (i = 0; i < nslots; i++) {
-		j = i + edesc->processed;
-		edma_write_slot(ecc, echan->slot[i], &edesc->pset[j].param);
-		edesc->sg_len += edesc->pset[j].len;
-		dev_vdbg(dev,
-			 "\n pset[%d]:\n"
-			 "  chnum\t%d\n"
-			 "  slot\t%d\n"
-			 "  opt\t%08x\n"
-			 "  src\t%08x\n"
-			 "  dst\t%08x\n"
-			 "  abcnt\t%08x\n"
-			 "  ccnt\t%08x\n"
-			 "  bidx\t%08x\n"
-			 "  cidx\t%08x\n"
-			 "  lkrld\t%08x\n",
-			 j, echan->ch_num, echan->slot[i],
-			 edesc->pset[j].param.opt,
-			 edesc->pset[j].param.src,
-			 edesc->pset[j].param.dst,
-			 edesc->pset[j].param.a_b_cnt,
-			 edesc->pset[j].param.ccnt,
-			 edesc->pset[j].param.src_dst_bidx,
-			 edesc->pset[j].param.src_dst_cidx,
-			 edesc->pset[j].param.link_bcntrld);
-		/* Link to the previous slot if not the last set */
-		if (i != (nslots - 1))
-			edma_link(ecc, echan->slot[i], echan->slot[i + 1]);
-	}
-
-	edesc->processed += nslots;
-
-	/*
-	 * If this is either the last set in a set of SG-list transactions
-	 * then setup a link to the dummy slot, this results in all future
-	 * events being absorbed and that's OK because we're done
-	 */
-	if (edesc->processed == edesc->pset_nr) {
-		if (edesc->cyclic)
-			edma_link(ecc, echan->slot[nslots - 1], echan->slot[1]);
-		else
-			edma_link(ecc, echan->slot[nslots - 1],
-				  echan->ecc->dummy_slot);
-	}
-
-	if (echan->missed) {
-		/*
-		 * This happens due to setup times between intermediate
-		 * transfers in long SG lists which have to be broken up into
-		 * transfers of MAX_NR_SG
-		 */
-		dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
-		edma_clean_channel(echan);
-		edma_stop(echan);
-		edma_start(echan);
-		edma_trigger_channel(echan);
-		echan->missed = 0;
-	} else if (edesc->processed <= MAX_NR_SG) {
-		dev_dbg(dev, "first transfer starting on channel %d\n",
-			echan->ch_num);
-		edma_start(echan);
-	} else {
-		dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
-			echan->ch_num, edesc->processed);
-		edma_resume(echan);
-	}
-}
-
-static int edma_terminate_all(struct dma_chan *chan)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-	unsigned long flags;
-	LIST_HEAD(head);
-
-	spin_lock_irqsave(&echan->vchan.lock, flags);
-
-	/*
-	 * Stop DMA activity: we assume the callback will not be called
-	 * after edma_dma() returns (even if it does, it will see
-	 * echan->edesc is NULL and exit.)
-	 */
-	if (echan->edesc) {
-		edma_stop(echan);
-		/* Move the cyclic channel back to default queue */
-		if (!echan->tc && echan->edesc->cyclic)
-			edma_assign_channel_eventq(echan, EVENTQ_DEFAULT);
-
-		vchan_terminate_vdesc(&echan->edesc->vdesc);
-		echan->edesc = NULL;
-	}
-
-	vchan_get_all_descriptors(&echan->vchan, &head);
-	spin_unlock_irqrestore(&echan->vchan.lock, flags);
-	vchan_dma_desc_free_list(&echan->vchan, &head);
-
-	return 0;
-}
-
-static void edma_synchronize(struct dma_chan *chan)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-
-	vchan_synchronize(&echan->vchan);
-}
-
-static int edma_slave_config(struct dma_chan *chan,
-	struct dma_slave_config *cfg)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-
-	if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
-	    cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
-		return -EINVAL;
-
-	if (cfg->src_maxburst > chan->device->max_burst ||
-	    cfg->dst_maxburst > chan->device->max_burst)
-		return -EINVAL;
-
-	memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
-
-	return 0;
-}
-
-static int edma_dma_pause(struct dma_chan *chan)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-
-	if (!echan->edesc)
-		return -EINVAL;
-
-	edma_pause(echan);
-	return 0;
-}
-
-static int edma_dma_resume(struct dma_chan *chan)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-
-	edma_resume(echan);
-	return 0;
-}
-
-/*
- * A PaRAM set configuration abstraction used by other modes
- * @chan: Channel who's PaRAM set we're configuring
- * @pset: PaRAM set to initialize and setup.
- * @src_addr: Source address of the DMA
- * @dst_addr: Destination address of the DMA
- * @burst: In units of dev_width, how much to send
- * @dev_width: How much is the dev_width
- * @dma_length: Total length of the DMA transfer
- * @direction: Direction of the transfer
- */
-static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
-			    dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
-			    unsigned int acnt, unsigned int dma_length,
-			    enum dma_transfer_direction direction)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-	struct device *dev = chan->device->dev;
-	struct edmacc_param *param = &epset->param;
-	int bcnt, ccnt, cidx;
-	int src_bidx, dst_bidx, src_cidx, dst_cidx;
-	int absync;
-
-	/* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
-	if (!burst)
-		burst = 1;
-	/*
-	 * If the maxburst is equal to the fifo width, use
-	 * A-synced transfers. This allows for large contiguous
-	 * buffer transfers using only one PaRAM set.
-	 */
-	if (burst == 1) {
-		/*
-		 * For the A-sync case, bcnt and ccnt are the remainder
-		 * and quotient respectively of the division of:
-		 * (dma_length / acnt) by (SZ_64K -1). This is so
-		 * that in case bcnt over flows, we have ccnt to use.
-		 * Note: In A-sync tranfer only, bcntrld is used, but it
-		 * only applies for sg_dma_len(sg) >= SZ_64K.
-		 * In this case, the best way adopted is- bccnt for the
-		 * first frame will be the remainder below. Then for
-		 * every successive frame, bcnt will be SZ_64K-1. This
-		 * is assured as bcntrld = 0xffff in end of function.
-		 */
-		absync = false;
-		ccnt = dma_length / acnt / (SZ_64K - 1);
-		bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
-		/*
-		 * If bcnt is non-zero, we have a remainder and hence an
-		 * extra frame to transfer, so increment ccnt.
-		 */
-		if (bcnt)
-			ccnt++;
-		else
-			bcnt = SZ_64K - 1;
-		cidx = acnt;
-	} else {
-		/*
-		 * If maxburst is greater than the fifo address_width,
-		 * use AB-synced transfers where A count is the fifo
-		 * address_width and B count is the maxburst. In this
-		 * case, we are limited to transfers of C count frames
-		 * of (address_width * maxburst) where C count is limited
-		 * to SZ_64K-1. This places an upper bound on the length
-		 * of an SG segment that can be handled.
-		 */
-		absync = true;
-		bcnt = burst;
-		ccnt = dma_length / (acnt * bcnt);
-		if (ccnt > (SZ_64K - 1)) {
-			dev_err(dev, "Exceeded max SG segment size\n");
-			return -EINVAL;
-		}
-		cidx = acnt * bcnt;
-	}
-
-	epset->len = dma_length;
-
-	if (direction == DMA_MEM_TO_DEV) {
-		src_bidx = acnt;
-		src_cidx = cidx;
-		dst_bidx = 0;
-		dst_cidx = 0;
-		epset->addr = src_addr;
-	} else if (direction == DMA_DEV_TO_MEM)  {
-		src_bidx = 0;
-		src_cidx = 0;
-		dst_bidx = acnt;
-		dst_cidx = cidx;
-		epset->addr = dst_addr;
-	} else if (direction == DMA_MEM_TO_MEM)  {
-		src_bidx = acnt;
-		src_cidx = cidx;
-		dst_bidx = acnt;
-		dst_cidx = cidx;
-	} else {
-		dev_err(dev, "%s: direction not implemented yet\n", __func__);
-		return -EINVAL;
-	}
-
-	param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
-	/* Configure A or AB synchronized transfers */
-	if (absync)
-		param->opt |= SYNCDIM;
-
-	param->src = src_addr;
-	param->dst = dst_addr;
-
-	param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
-	param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
-
-	param->a_b_cnt = bcnt << 16 | acnt;
-	param->ccnt = ccnt;
-	/*
-	 * Only time when (bcntrld) auto reload is required is for
-	 * A-sync case, and in this case, a requirement of reload value
-	 * of SZ_64K-1 only is assured. 'link' is initially set to NULL
-	 * and then later will be populated by edma_execute.
-	 */
-	param->link_bcntrld = 0xffffffff;
-	return absync;
-}
-
-static struct dma_async_tx_descriptor *edma_prep_slave_sg(
-	struct dma_chan *chan, struct scatterlist *sgl,
-	unsigned int sg_len, enum dma_transfer_direction direction,
-	unsigned long tx_flags, void *context)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-	struct device *dev = chan->device->dev;
-	struct edma_desc *edesc;
-	dma_addr_t src_addr = 0, dst_addr = 0;
-	enum dma_slave_buswidth dev_width;
-	u32 burst;
-	struct scatterlist *sg;
-	int i, nslots, ret;
-
-	if (unlikely(!echan || !sgl || !sg_len))
-		return NULL;
-
-	if (direction == DMA_DEV_TO_MEM) {
-		src_addr = echan->cfg.src_addr;
-		dev_width = echan->cfg.src_addr_width;
-		burst = echan->cfg.src_maxburst;
-	} else if (direction == DMA_MEM_TO_DEV) {
-		dst_addr = echan->cfg.dst_addr;
-		dev_width = echan->cfg.dst_addr_width;
-		burst = echan->cfg.dst_maxburst;
-	} else {
-		dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
-		return NULL;
-	}
-
-	if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
-		dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
-		return NULL;
-	}
-
-	edesc = kzalloc(sizeof(*edesc) + sg_len * sizeof(edesc->pset[0]),
-			GFP_ATOMIC);
-	if (!edesc)
-		return NULL;
-
-	edesc->pset_nr = sg_len;
-	edesc->residue = 0;
-	edesc->direction = direction;
-	edesc->echan = echan;
-
-	/* Allocate a PaRAM slot, if needed */
-	nslots = min_t(unsigned, MAX_NR_SG, sg_len);
-
-	for (i = 0; i < nslots; i++) {
-		if (echan->slot[i] < 0) {
-			echan->slot[i] =
-				edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
-			if (echan->slot[i] < 0) {
-				kfree(edesc);
-				dev_err(dev, "%s: Failed to allocate slot\n",
-					__func__);
-				return NULL;
-			}
-		}
-	}
-
-	/* Configure PaRAM sets for each SG */
-	for_each_sg(sgl, sg, sg_len, i) {
-		/* Get address for each SG */
-		if (direction == DMA_DEV_TO_MEM)
-			dst_addr = sg_dma_address(sg);
-		else
-			src_addr = sg_dma_address(sg);
-
-		ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
-				       dst_addr, burst, dev_width,
-				       sg_dma_len(sg), direction);
-		if (ret < 0) {
-			kfree(edesc);
-			return NULL;
-		}
-
-		edesc->absync = ret;
-		edesc->residue += sg_dma_len(sg);
-
-		if (i == sg_len - 1)
-			/* Enable completion interrupt */
-			edesc->pset[i].param.opt |= TCINTEN;
-		else if (!((i+1) % MAX_NR_SG))
-			/*
-			 * Enable early completion interrupt for the
-			 * intermediateset. In this case the driver will be
-			 * notified when the paRAM set is submitted to TC. This
-			 * will allow more time to set up the next set of slots.
-			 */
-			edesc->pset[i].param.opt |= (TCINTEN | TCCMODE);
-	}
-	edesc->residue_stat = edesc->residue;
-
-	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
-}
-
-static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
-	struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
-	size_t len, unsigned long tx_flags)
-{
-	int ret, nslots;
-	struct edma_desc *edesc;
-	struct device *dev = chan->device->dev;
-	struct edma_chan *echan = to_edma_chan(chan);
-	unsigned int width, pset_len, array_size;
-
-	if (unlikely(!echan || !len))
-		return NULL;
-
-	/* Align the array size (acnt block) with the transfer properties */
-	switch (__ffs((src | dest | len))) {
-	case 0:
-		array_size = SZ_32K - 1;
-		break;
-	case 1:
-		array_size = SZ_32K - 2;
-		break;
-	default:
-		array_size = SZ_32K - 4;
-		break;
-	}
-
-	if (len < SZ_64K) {
-		/*
-		 * Transfer size less than 64K can be handled with one paRAM
-		 * slot and with one burst.
-		 * ACNT = length
-		 */
-		width = len;
-		pset_len = len;
-		nslots = 1;
-	} else {
-		/*
-		 * Transfer size bigger than 64K will be handled with maximum of
-		 * two paRAM slots.
-		 * slot1: (full_length / 32767) times 32767 bytes bursts.
-		 *	  ACNT = 32767, length1: (full_length / 32767) * 32767
-		 * slot2: the remaining amount of data after slot1.
-		 *	  ACNT = full_length - length1, length2 = ACNT
-		 *
-		 * When the full_length is multibple of 32767 one slot can be
-		 * used to complete the transfer.
-		 */
-		width = array_size;
-		pset_len = rounddown(len, width);
-		/* One slot is enough for lengths multiple of (SZ_32K -1) */
-		if (unlikely(pset_len == len))
-			nslots = 1;
-		else
-			nslots = 2;
-	}
-
-	edesc = kzalloc(sizeof(*edesc) + nslots * sizeof(edesc->pset[0]),
-			GFP_ATOMIC);
-	if (!edesc)
-		return NULL;
-
-	edesc->pset_nr = nslots;
-	edesc->residue = edesc->residue_stat = len;
-	edesc->direction = DMA_MEM_TO_MEM;
-	edesc->echan = echan;
-
-	ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
-			       width, pset_len, DMA_MEM_TO_MEM);
-	if (ret < 0) {
-		kfree(edesc);
-		return NULL;
-	}
-
-	edesc->absync = ret;
-
-	edesc->pset[0].param.opt |= ITCCHEN;
-	if (nslots == 1) {
-		/* Enable transfer complete interrupt */
-		edesc->pset[0].param.opt |= TCINTEN;
-	} else {
-		/* Enable transfer complete chaining for the first slot */
-		edesc->pset[0].param.opt |= TCCHEN;
-
-		if (echan->slot[1] < 0) {
-			echan->slot[1] = edma_alloc_slot(echan->ecc,
-							 EDMA_SLOT_ANY);
-			if (echan->slot[1] < 0) {
-				kfree(edesc);
-				dev_err(dev, "%s: Failed to allocate slot\n",
-					__func__);
-				return NULL;
-			}
-		}
-		dest += pset_len;
-		src += pset_len;
-		pset_len = width = len % array_size;
-
-		ret = edma_config_pset(chan, &edesc->pset[1], src, dest, 1,
-				       width, pset_len, DMA_MEM_TO_MEM);
-		if (ret < 0) {
-			kfree(edesc);
-			return NULL;
-		}
-
-		edesc->pset[1].param.opt |= ITCCHEN;
-		edesc->pset[1].param.opt |= TCINTEN;
-	}
-
-	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
-}
-
-static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
-	struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
-	size_t period_len, enum dma_transfer_direction direction,
-	unsigned long tx_flags)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-	struct device *dev = chan->device->dev;
-	struct edma_desc *edesc;
-	dma_addr_t src_addr, dst_addr;
-	enum dma_slave_buswidth dev_width;
-	bool use_intermediate = false;
-	u32 burst;
-	int i, ret, nslots;
-
-	if (unlikely(!echan || !buf_len || !period_len))
-		return NULL;
-
-	if (direction == DMA_DEV_TO_MEM) {
-		src_addr = echan->cfg.src_addr;
-		dst_addr = buf_addr;
-		dev_width = echan->cfg.src_addr_width;
-		burst = echan->cfg.src_maxburst;
-	} else if (direction == DMA_MEM_TO_DEV) {
-		src_addr = buf_addr;
-		dst_addr = echan->cfg.dst_addr;
-		dev_width = echan->cfg.dst_addr_width;
-		burst = echan->cfg.dst_maxburst;
-	} else {
-		dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
-		return NULL;
-	}
-
-	if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
-		dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
-		return NULL;
-	}
-
-	if (unlikely(buf_len % period_len)) {
-		dev_err(dev, "Period should be multiple of Buffer length\n");
-		return NULL;
-	}
-
-	nslots = (buf_len / period_len) + 1;
-
-	/*
-	 * Cyclic DMA users such as audio cannot tolerate delays introduced
-	 * by cases where the number of periods is more than the maximum
-	 * number of SGs the EDMA driver can handle at a time. For DMA types
-	 * such as Slave SGs, such delays are tolerable and synchronized,
-	 * but the synchronization is difficult to achieve with Cyclic and
-	 * cannot be guaranteed, so we error out early.
-	 */
-	if (nslots > MAX_NR_SG) {
-		/*
-		 * If the burst and period sizes are the same, we can put
-		 * the full buffer into a single period and activate
-		 * intermediate interrupts. This will produce interrupts
-		 * after each burst, which is also after each desired period.
-		 */
-		if (burst == period_len) {
-			period_len = buf_len;
-			nslots = 2;
-			use_intermediate = true;
-		} else {
-			return NULL;
-		}
-	}
-
-	edesc = kzalloc(sizeof(*edesc) + nslots * sizeof(edesc->pset[0]),
-			GFP_ATOMIC);
-	if (!edesc)
-		return NULL;
-
-	edesc->cyclic = 1;
-	edesc->pset_nr = nslots;
-	edesc->residue = edesc->residue_stat = buf_len;
-	edesc->direction = direction;
-	edesc->echan = echan;
-
-	dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
-		__func__, echan->ch_num, nslots, period_len, buf_len);
-
-	for (i = 0; i < nslots; i++) {
-		/* Allocate a PaRAM slot, if needed */
-		if (echan->slot[i] < 0) {
-			echan->slot[i] =
-				edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
-			if (echan->slot[i] < 0) {
-				kfree(edesc);
-				dev_err(dev, "%s: Failed to allocate slot\n",
-					__func__);
-				return NULL;
-			}
-		}
-
-		if (i == nslots - 1) {
-			memcpy(&edesc->pset[i], &edesc->pset[0],
-			       sizeof(edesc->pset[0]));
-			break;
-		}
-
-		ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
-				       dst_addr, burst, dev_width, period_len,
-				       direction);
-		if (ret < 0) {
-			kfree(edesc);
-			return NULL;
-		}
-
-		if (direction == DMA_DEV_TO_MEM)
-			dst_addr += period_len;
-		else
-			src_addr += period_len;
-
-		dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
-		dev_vdbg(dev,
-			"\n pset[%d]:\n"
-			"  chnum\t%d\n"
-			"  slot\t%d\n"
-			"  opt\t%08x\n"
-			"  src\t%08x\n"
-			"  dst\t%08x\n"
-			"  abcnt\t%08x\n"
-			"  ccnt\t%08x\n"
-			"  bidx\t%08x\n"
-			"  cidx\t%08x\n"
-			"  lkrld\t%08x\n",
-			i, echan->ch_num, echan->slot[i],
-			edesc->pset[i].param.opt,
-			edesc->pset[i].param.src,
-			edesc->pset[i].param.dst,
-			edesc->pset[i].param.a_b_cnt,
-			edesc->pset[i].param.ccnt,
-			edesc->pset[i].param.src_dst_bidx,
-			edesc->pset[i].param.src_dst_cidx,
-			edesc->pset[i].param.link_bcntrld);
-
-		edesc->absync = ret;
-
-		/*
-		 * Enable period interrupt only if it is requested
-		 */
-		if (tx_flags & DMA_PREP_INTERRUPT) {
-			edesc->pset[i].param.opt |= TCINTEN;
-
-			/* Also enable intermediate interrupts if necessary */
-			if (use_intermediate)
-				edesc->pset[i].param.opt |= ITCINTEN;
-		}
-	}
-
-	/* Place the cyclic channel to highest priority queue */
-	if (!echan->tc)
-		edma_assign_channel_eventq(echan, EVENTQ_0);
-
-	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
-}
-
-static void edma_completion_handler(struct edma_chan *echan)
-{
-	struct device *dev = echan->vchan.chan.device->dev;
-	struct edma_desc *edesc;
-
-	spin_lock(&echan->vchan.lock);
-	edesc = echan->edesc;
-	if (edesc) {
-		if (edesc->cyclic) {
-			vchan_cyclic_callback(&edesc->vdesc);
-			spin_unlock(&echan->vchan.lock);
-			return;
-		} else if (edesc->processed == edesc->pset_nr) {
-			edesc->residue = 0;
-			edma_stop(echan);
-			vchan_cookie_complete(&edesc->vdesc);
-			echan->edesc = NULL;
-
-			dev_dbg(dev, "Transfer completed on channel %d\n",
-				echan->ch_num);
-		} else {
-			dev_dbg(dev, "Sub transfer completed on channel %d\n",
-				echan->ch_num);
-
-			edma_pause(echan);
-
-			/* Update statistics for tx_status */
-			edesc->residue -= edesc->sg_len;
-			edesc->residue_stat = edesc->residue;
-			edesc->processed_stat = edesc->processed;
-		}
-		edma_execute(echan);
-	}
-
-	spin_unlock(&echan->vchan.lock);
-}
-
-/* eDMA interrupt handler */
-static irqreturn_t dma_irq_handler(int irq, void *data)
-{
-	struct edma_cc *ecc = data;
-	int ctlr;
-	u32 sh_ier;
-	u32 sh_ipr;
-	u32 bank;
-
-	ctlr = ecc->id;
-	if (ctlr < 0)
-		return IRQ_NONE;
-
-	dev_vdbg(ecc->dev, "dma_irq_handler\n");
-
-	sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 0);
-	if (!sh_ipr) {
-		sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 1);
-		if (!sh_ipr)
-			return IRQ_NONE;
-		sh_ier = edma_shadow0_read_array(ecc, SH_IER, 1);
-		bank = 1;
-	} else {
-		sh_ier = edma_shadow0_read_array(ecc, SH_IER, 0);
-		bank = 0;
-	}
-
-	do {
-		u32 slot;
-		u32 channel;
-
-		slot = __ffs(sh_ipr);
-		sh_ipr &= ~(BIT(slot));
-
-		if (sh_ier & BIT(slot)) {
-			channel = (bank << 5) | slot;
-			/* Clear the corresponding IPR bits */
-			edma_shadow0_write_array(ecc, SH_ICR, bank, BIT(slot));
-			edma_completion_handler(&ecc->slave_chans[channel]);
-		}
-	} while (sh_ipr);
-
-	edma_shadow0_write(ecc, SH_IEVAL, 1);
-	return IRQ_HANDLED;
-}
-
-static void edma_error_handler(struct edma_chan *echan)
-{
-	struct edma_cc *ecc = echan->ecc;
-	struct device *dev = echan->vchan.chan.device->dev;
-	struct edmacc_param p;
-	int err;
-
-	if (!echan->edesc)
-		return;
-
-	spin_lock(&echan->vchan.lock);
-
-	err = edma_read_slot(ecc, echan->slot[0], &p);
-
-	/*
-	 * Issue later based on missed flag which will be sure
-	 * to happen as:
-	 * (1) we finished transmitting an intermediate slot and
-	 *     edma_execute is coming up.
-	 * (2) or we finished current transfer and issue will
-	 *     call edma_execute.
-	 *
-	 * Important note: issuing can be dangerous here and
-	 * lead to some nasty recursion when we are in a NULL
-	 * slot. So we avoid doing so and set the missed flag.
-	 */
-	if (err || (p.a_b_cnt == 0 && p.ccnt == 0)) {
-		dev_dbg(dev, "Error on null slot, setting miss\n");
-		echan->missed = 1;
-	} else {
-		/*
-		 * The slot is already programmed but the event got
-		 * missed, so its safe to issue it here.
-		 */
-		dev_dbg(dev, "Missed event, TRIGGERING\n");
-		edma_clean_channel(echan);
-		edma_stop(echan);
-		edma_start(echan);
-		edma_trigger_channel(echan);
-	}
-	spin_unlock(&echan->vchan.lock);
-}
-
-static inline bool edma_error_pending(struct edma_cc *ecc)
-{
-	if (edma_read_array(ecc, EDMA_EMR, 0) ||
-	    edma_read_array(ecc, EDMA_EMR, 1) ||
-	    edma_read(ecc, EDMA_QEMR) || edma_read(ecc, EDMA_CCERR))
-		return true;
-
-	return false;
-}
-
-/* eDMA error interrupt handler */
-static irqreturn_t dma_ccerr_handler(int irq, void *data)
-{
-	struct edma_cc *ecc = data;
-	int i, j;
-	int ctlr;
-	unsigned int cnt = 0;
-	unsigned int val;
-
-	ctlr = ecc->id;
-	if (ctlr < 0)
-		return IRQ_NONE;
-
-	dev_vdbg(ecc->dev, "dma_ccerr_handler\n");
-
-	if (!edma_error_pending(ecc)) {
-		/*
-		 * The registers indicate no pending error event but the irq
-		 * handler has been called.
-		 * Ask eDMA to re-evaluate the error registers.
-		 */
-		dev_err(ecc->dev, "%s: Error interrupt without error event!\n",
-			__func__);
-		edma_write(ecc, EDMA_EEVAL, 1);
-		return IRQ_NONE;
-	}
-
-	while (1) {
-		/* Event missed register(s) */
-		for (j = 0; j < 2; j++) {
-			unsigned long emr;
-
-			val = edma_read_array(ecc, EDMA_EMR, j);
-			if (!val)
-				continue;
-
-			dev_dbg(ecc->dev, "EMR%d 0x%08x\n", j, val);
-			emr = val;
-			for (i = find_next_bit(&emr, 32, 0); i < 32;
-			     i = find_next_bit(&emr, 32, i + 1)) {
-				int k = (j << 5) + i;
-
-				/* Clear the corresponding EMR bits */
-				edma_write_array(ecc, EDMA_EMCR, j, BIT(i));
-				/* Clear any SER */
-				edma_shadow0_write_array(ecc, SH_SECR, j,
-							 BIT(i));
-				edma_error_handler(&ecc->slave_chans[k]);
-			}
-		}
-
-		val = edma_read(ecc, EDMA_QEMR);
-		if (val) {
-			dev_dbg(ecc->dev, "QEMR 0x%02x\n", val);
-			/* Not reported, just clear the interrupt reason. */
-			edma_write(ecc, EDMA_QEMCR, val);
-			edma_shadow0_write(ecc, SH_QSECR, val);
-		}
-
-		val = edma_read(ecc, EDMA_CCERR);
-		if (val) {
-			dev_warn(ecc->dev, "CCERR 0x%08x\n", val);
-			/* Not reported, just clear the interrupt reason. */
-			edma_write(ecc, EDMA_CCERRCLR, val);
-		}
-
-		if (!edma_error_pending(ecc))
-			break;
-		cnt++;
-		if (cnt > 10)
-			break;
-	}
-	edma_write(ecc, EDMA_EEVAL, 1);
-	return IRQ_HANDLED;
-}
-
-/* Alloc channel resources */
-static int edma_alloc_chan_resources(struct dma_chan *chan)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-	struct edma_cc *ecc = echan->ecc;
-	struct device *dev = ecc->dev;
-	enum dma_event_q eventq_no = EVENTQ_DEFAULT;
-	int ret;
-
-	if (echan->tc) {
-		eventq_no = echan->tc->id;
-	} else if (ecc->tc_list) {
-		/* memcpy channel */
-		echan->tc = &ecc->tc_list[ecc->info->default_queue];
-		eventq_no = echan->tc->id;
-	}
-
-	ret = edma_alloc_channel(echan, eventq_no);
-	if (ret)
-		return ret;
-
-	echan->slot[0] = edma_alloc_slot(ecc, echan->ch_num);
-	if (echan->slot[0] < 0) {
-		dev_err(dev, "Entry slot allocation failed for channel %u\n",
-			EDMA_CHAN_SLOT(echan->ch_num));
-		ret = echan->slot[0];
-		goto err_slot;
-	}
-
-	/* Set up channel -> slot mapping for the entry slot */
-	edma_set_chmap(echan, echan->slot[0]);
-	echan->alloced = true;
-
-	dev_dbg(dev, "Got eDMA channel %d for virt channel %d (%s trigger)\n",
-		EDMA_CHAN_SLOT(echan->ch_num), chan->chan_id,
-		echan->hw_triggered ? "HW" : "SW");
-
-	return 0;
-
-err_slot:
-	edma_free_channel(echan);
-	return ret;
-}
-
-/* Free channel resources */
-static void edma_free_chan_resources(struct dma_chan *chan)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-	struct device *dev = echan->ecc->dev;
-	int i;
-
-	/* Terminate transfers */
-	edma_stop(echan);
-
-	vchan_free_chan_resources(&echan->vchan);
-
-	/* Free EDMA PaRAM slots */
-	for (i = 0; i < EDMA_MAX_SLOTS; i++) {
-		if (echan->slot[i] >= 0) {
-			edma_free_slot(echan->ecc, echan->slot[i]);
-			echan->slot[i] = -1;
-		}
-	}
-
-	/* Set entry slot to the dummy slot */
-	edma_set_chmap(echan, echan->ecc->dummy_slot);
-
-	/* Free EDMA channel */
-	if (echan->alloced) {
-		edma_free_channel(echan);
-		echan->alloced = false;
-	}
-
-	echan->tc = NULL;
-	echan->hw_triggered = false;
-
-	dev_dbg(dev, "Free eDMA channel %d for virt channel %d\n",
-		EDMA_CHAN_SLOT(echan->ch_num), chan->chan_id);
-}
-
-/* Send pending descriptor to hardware */
-static void edma_issue_pending(struct dma_chan *chan)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-	unsigned long flags;
-
-	spin_lock_irqsave(&echan->vchan.lock, flags);
-	if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
-		edma_execute(echan);
-	spin_unlock_irqrestore(&echan->vchan.lock, flags);
-}
-
-/*
- * This limit exists to avoid a possible infinite loop when waiting for proof
- * that a particular transfer is completed. This limit can be hit if there
- * are large bursts to/from slow devices or the CPU is never able to catch
- * the DMA hardware idle. On an AM335x transfering 48 bytes from the UART
- * RX-FIFO, as many as 55 loops have been seen.
- */
-#define EDMA_MAX_TR_WAIT_LOOPS 1000
-
-static u32 edma_residue(struct edma_desc *edesc)
-{
-	bool dst = edesc->direction == DMA_DEV_TO_MEM;
-	int loop_count = EDMA_MAX_TR_WAIT_LOOPS;
-	struct edma_chan *echan = edesc->echan;
-	struct edma_pset *pset = edesc->pset;
-	dma_addr_t done, pos;
-	int i;
-
-	/*
-	 * We always read the dst/src position from the first RamPar
-	 * pset. That's the one which is active now.
-	 */
-	pos = edma_get_position(echan->ecc, echan->slot[0], dst);
-
-	/*
-	 * "pos" may represent a transfer request that is still being
-	 * processed by the EDMACC or EDMATC. We will busy wait until
-	 * any one of the situations occurs:
-	 *   1. the DMA hardware is idle
-	 *   2. a new transfer request is setup
-	 *   3. we hit the loop limit
-	 */
-	while (edma_read(echan->ecc, EDMA_CCSTAT) & EDMA_CCSTAT_ACTV) {
-		/* check if a new transfer request is setup */
-		if (edma_get_position(echan->ecc,
-				      echan->slot[0], dst) != pos) {
-			break;
-		}
-
-		if (!--loop_count) {
-			dev_dbg_ratelimited(echan->vchan.chan.device->dev,
-				"%s: timeout waiting for PaRAM update\n",
-				__func__);
-			break;
-		}
-
-		cpu_relax();
-	}
-
-	/*
-	 * Cyclic is simple. Just subtract pset[0].addr from pos.
-	 *
-	 * We never update edesc->residue in the cyclic case, so we
-	 * can tell the remaining room to the end of the circular
-	 * buffer.
-	 */
-	if (edesc->cyclic) {
-		done = pos - pset->addr;
-		edesc->residue_stat = edesc->residue - done;
-		return edesc->residue_stat;
-	}
-
-	/*
-	 * For SG operation we catch up with the last processed
-	 * status.
-	 */
-	pset += edesc->processed_stat;
-
-	for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
-		/*
-		 * If we are inside this pset address range, we know
-		 * this is the active one. Get the current delta and
-		 * stop walking the psets.
-		 */
-		if (pos >= pset->addr && pos < pset->addr + pset->len)
-			return edesc->residue_stat - (pos - pset->addr);
-
-		/* Otherwise mark it done and update residue_stat. */
-		edesc->processed_stat++;
-		edesc->residue_stat -= pset->len;
-	}
-	return edesc->residue_stat;
-}
-
-/* Check request completion status */
-static enum dma_status edma_tx_status(struct dma_chan *chan,
-				      dma_cookie_t cookie,
-				      struct dma_tx_state *txstate)
-{
-	struct edma_chan *echan = to_edma_chan(chan);
-	struct virt_dma_desc *vdesc;
-	enum dma_status ret;
-	unsigned long flags;
-
-	ret = dma_cookie_status(chan, cookie, txstate);
-	if (ret == DMA_COMPLETE || !txstate)
-		return ret;
-
-	spin_lock_irqsave(&echan->vchan.lock, flags);
-	if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
-		txstate->residue = edma_residue(echan->edesc);
-	else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
-		txstate->residue = to_edma_desc(&vdesc->tx)->residue;
-	spin_unlock_irqrestore(&echan->vchan.lock, flags);
-
-	return ret;
-}
-
-static bool edma_is_memcpy_channel(int ch_num, s32 *memcpy_channels)
-{
-	if (!memcpy_channels)
-		return false;
-	while (*memcpy_channels != -1) {
-		if (*memcpy_channels == ch_num)
-			return true;
-		memcpy_channels++;
-	}
-	return false;
-}
-
-#define EDMA_DMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
-				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
-				 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
-				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
-
-static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode)
-{
-	struct dma_device *s_ddev = &ecc->dma_slave;
-	struct dma_device *m_ddev = NULL;
-	s32 *memcpy_channels = ecc->info->memcpy_channels;
-	int i, j;
-
-	dma_cap_zero(s_ddev->cap_mask);
-	dma_cap_set(DMA_SLAVE, s_ddev->cap_mask);
-	dma_cap_set(DMA_CYCLIC, s_ddev->cap_mask);
-	if (ecc->legacy_mode && !memcpy_channels) {
-		dev_warn(ecc->dev,
-			 "Legacy memcpy is enabled, things might not work\n");
-
-		dma_cap_set(DMA_MEMCPY, s_ddev->cap_mask);
-		s_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy;
-		s_ddev->directions = BIT(DMA_MEM_TO_MEM);
-	}
-
-	s_ddev->device_prep_slave_sg = edma_prep_slave_sg;
-	s_ddev->device_prep_dma_cyclic = edma_prep_dma_cyclic;
-	s_ddev->device_alloc_chan_resources = edma_alloc_chan_resources;
-	s_ddev->device_free_chan_resources = edma_free_chan_resources;
-	s_ddev->device_issue_pending = edma_issue_pending;
-	s_ddev->device_tx_status = edma_tx_status;
-	s_ddev->device_config = edma_slave_config;
-	s_ddev->device_pause = edma_dma_pause;
-	s_ddev->device_resume = edma_dma_resume;
-	s_ddev->device_terminate_all = edma_terminate_all;
-	s_ddev->device_synchronize = edma_synchronize;
-
-	s_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS;
-	s_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
-	s_ddev->directions |= (BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV));
-	s_ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
-	s_ddev->max_burst = SZ_32K - 1; /* CIDX: 16bit signed */
-
-	s_ddev->dev = ecc->dev;
-	INIT_LIST_HEAD(&s_ddev->channels);
-
-	if (memcpy_channels) {
-		m_ddev = devm_kzalloc(ecc->dev, sizeof(*m_ddev), GFP_KERNEL);
-		if (!m_ddev) {
-			dev_warn(ecc->dev, "memcpy is disabled due to OoM\n");
-			memcpy_channels = NULL;
-			goto ch_setup;
-		}
-		ecc->dma_memcpy = m_ddev;
-
-		dma_cap_zero(m_ddev->cap_mask);
-		dma_cap_set(DMA_MEMCPY, m_ddev->cap_mask);
-
-		m_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy;
-		m_ddev->device_alloc_chan_resources = edma_alloc_chan_resources;
-		m_ddev->device_free_chan_resources = edma_free_chan_resources;
-		m_ddev->device_issue_pending = edma_issue_pending;
-		m_ddev->device_tx_status = edma_tx_status;
-		m_ddev->device_config = edma_slave_config;
-		m_ddev->device_pause = edma_dma_pause;
-		m_ddev->device_resume = edma_dma_resume;
-		m_ddev->device_terminate_all = edma_terminate_all;
-		m_ddev->device_synchronize = edma_synchronize;
-
-		m_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS;
-		m_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
-		m_ddev->directions = BIT(DMA_MEM_TO_MEM);
-		m_ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
-
-		m_ddev->dev = ecc->dev;
-		INIT_LIST_HEAD(&m_ddev->channels);
-	} else if (!ecc->legacy_mode) {
-		dev_info(ecc->dev, "memcpy is disabled\n");
-	}
-
-ch_setup:
-	for (i = 0; i < ecc->num_channels; i++) {
-		struct edma_chan *echan = &ecc->slave_chans[i];
-		echan->ch_num = EDMA_CTLR_CHAN(ecc->id, i);
-		echan->ecc = ecc;
-		echan->vchan.desc_free = edma_desc_free;
-
-		if (m_ddev && edma_is_memcpy_channel(i, memcpy_channels))
-			vchan_init(&echan->vchan, m_ddev);
-		else
-			vchan_init(&echan->vchan, s_ddev);
-
-		INIT_LIST_HEAD(&echan->node);
-		for (j = 0; j < EDMA_MAX_SLOTS; j++)
-			echan->slot[j] = -1;
-	}
-}
-
-static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata,
-			      struct edma_cc *ecc)
-{
-	int i;
-	u32 value, cccfg;
-	s8 (*queue_priority_map)[2];
-
-	/* Decode the eDMA3 configuration from CCCFG register */
-	cccfg = edma_read(ecc, EDMA_CCCFG);
-
-	value = GET_NUM_REGN(cccfg);
-	ecc->num_region = BIT(value);
-
-	value = GET_NUM_DMACH(cccfg);
-	ecc->num_channels = BIT(value + 1);
-
-	value = GET_NUM_QDMACH(cccfg);
-	ecc->num_qchannels = value * 2;
-
-	value = GET_NUM_PAENTRY(cccfg);
-	ecc->num_slots = BIT(value + 4);
-
-	value = GET_NUM_EVQUE(cccfg);
-	ecc->num_tc = value + 1;
-
-	ecc->chmap_exist = (cccfg & CHMAP_EXIST) ? true : false;
-
-	dev_dbg(dev, "eDMA3 CC HW configuration (cccfg: 0x%08x):\n", cccfg);
-	dev_dbg(dev, "num_region: %u\n", ecc->num_region);
-	dev_dbg(dev, "num_channels: %u\n", ecc->num_channels);
-	dev_dbg(dev, "num_qchannels: %u\n", ecc->num_qchannels);
-	dev_dbg(dev, "num_slots: %u\n", ecc->num_slots);
-	dev_dbg(dev, "num_tc: %u\n", ecc->num_tc);
-	dev_dbg(dev, "chmap_exist: %s\n", ecc->chmap_exist ? "yes" : "no");
-
-	/* Nothing need to be done if queue priority is provided */
-	if (pdata->queue_priority_mapping)
-		return 0;
-
-	/*
-	 * Configure TC/queue priority as follows:
-	 * Q0 - priority 0
-	 * Q1 - priority 1
-	 * Q2 - priority 2
-	 * ...
-	 * The meaning of priority numbers: 0 highest priority, 7 lowest
-	 * priority. So Q0 is the highest priority queue and the last queue has
-	 * the lowest priority.
-	 */
-	queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, sizeof(s8),
-					  GFP_KERNEL);
-	if (!queue_priority_map)
-		return -ENOMEM;
-
-	for (i = 0; i < ecc->num_tc; i++) {
-		queue_priority_map[i][0] = i;
-		queue_priority_map[i][1] = i;
-	}
-	queue_priority_map[i][0] = -1;
-	queue_priority_map[i][1] = -1;
-
-	pdata->queue_priority_mapping = queue_priority_map;
-	/* Default queue has the lowest priority */
-	pdata->default_queue = i - 1;
-
-	return 0;
-}
-
-#if IS_ENABLED(CONFIG_OF)
-static int edma_xbar_event_map(struct device *dev, struct edma_soc_info *pdata,
-			       size_t sz)
-{
-	const char pname[] = "ti,edma-xbar-event-map";
-	struct resource res;
-	void __iomem *xbar;
-	s16 (*xbar_chans)[2];
-	size_t nelm = sz / sizeof(s16);
-	u32 shift, offset, mux;
-	int ret, i;
-
-	xbar_chans = devm_kcalloc(dev, nelm + 2, sizeof(s16), GFP_KERNEL);
-	if (!xbar_chans)
-		return -ENOMEM;
-
-	ret = of_address_to_resource(dev->of_node, 1, &res);
-	if (ret)
-		return -ENOMEM;
-
-	xbar = devm_ioremap(dev, res.start, resource_size(&res));
-	if (!xbar)
-		return -ENOMEM;
-
-	ret = of_property_read_u16_array(dev->of_node, pname, (u16 *)xbar_chans,
-					 nelm);
-	if (ret)
-		return -EIO;
-
-	/* Invalidate last entry for the other user of this mess */
-	nelm >>= 1;
-	xbar_chans[nelm][0] = -1;
-	xbar_chans[nelm][1] = -1;
-
-	for (i = 0; i < nelm; i++) {
-		shift = (xbar_chans[i][1] & 0x03) << 3;
-		offset = xbar_chans[i][1] & 0xfffffffc;
-		mux = readl(xbar + offset);
-		mux &= ~(0xff << shift);
-		mux |= xbar_chans[i][0] << shift;
-		writel(mux, (xbar + offset));
-	}
-
-	pdata->xbar_chans = (const s16 (*)[2]) xbar_chans;
-	return 0;
-}
-
-static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev,
-						     bool legacy_mode)
-{
-	struct edma_soc_info *info;
-	struct property *prop;
-	int sz, ret;
-
-	info = devm_kzalloc(dev, sizeof(struct edma_soc_info), GFP_KERNEL);
-	if (!info)
-		return ERR_PTR(-ENOMEM);
-
-	if (legacy_mode) {
-		prop = of_find_property(dev->of_node, "ti,edma-xbar-event-map",
-					&sz);
-		if (prop) {
-			ret = edma_xbar_event_map(dev, info, sz);
-			if (ret)
-				return ERR_PTR(ret);
-		}
-		return info;
-	}
-
-	/* Get the list of channels allocated to be used for memcpy */
-	prop = of_find_property(dev->of_node, "ti,edma-memcpy-channels", &sz);
-	if (prop) {
-		const char pname[] = "ti,edma-memcpy-channels";
-		size_t nelm = sz / sizeof(s32);
-		s32 *memcpy_ch;
-
-		memcpy_ch = devm_kcalloc(dev, nelm + 1, sizeof(s32),
-					 GFP_KERNEL);
-		if (!memcpy_ch)
-			return ERR_PTR(-ENOMEM);
-
-		ret = of_property_read_u32_array(dev->of_node, pname,
-						 (u32 *)memcpy_ch, nelm);
-		if (ret)
-			return ERR_PTR(ret);
-
-		memcpy_ch[nelm] = -1;
-		info->memcpy_channels = memcpy_ch;
-	}
-
-	prop = of_find_property(dev->of_node, "ti,edma-reserved-slot-ranges",
-				&sz);
-	if (prop) {
-		const char pname[] = "ti,edma-reserved-slot-ranges";
-		u32 (*tmp)[2];
-		s16 (*rsv_slots)[2];
-		size_t nelm = sz / sizeof(*tmp);
-		struct edma_rsv_info *rsv_info;
-		int i;
-
-		if (!nelm)
-			return info;
-
-		tmp = kcalloc(nelm, sizeof(*tmp), GFP_KERNEL);
-		if (!tmp)
-			return ERR_PTR(-ENOMEM);
-
-		rsv_info = devm_kzalloc(dev, sizeof(*rsv_info), GFP_KERNEL);
-		if (!rsv_info) {
-			kfree(tmp);
-			return ERR_PTR(-ENOMEM);
-		}
-
-		rsv_slots = devm_kcalloc(dev, nelm + 1, sizeof(*rsv_slots),
-					 GFP_KERNEL);
-		if (!rsv_slots) {
-			kfree(tmp);
-			return ERR_PTR(-ENOMEM);
-		}
-
-		ret = of_property_read_u32_array(dev->of_node, pname,
-						 (u32 *)tmp, nelm * 2);
-		if (ret) {
-			kfree(tmp);
-			return ERR_PTR(ret);
-		}
-
-		for (i = 0; i < nelm; i++) {
-			rsv_slots[i][0] = tmp[i][0];
-			rsv_slots[i][1] = tmp[i][1];
-		}
-		rsv_slots[nelm][0] = -1;
-		rsv_slots[nelm][1] = -1;
-
-		info->rsv = rsv_info;
-		info->rsv->rsv_slots = (const s16 (*)[2])rsv_slots;
-
-		kfree(tmp);
-	}
-
-	return info;
-}
-
-static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec,
-				      struct of_dma *ofdma)
-{
-	struct edma_cc *ecc = ofdma->of_dma_data;
-	struct dma_chan *chan = NULL;
-	struct edma_chan *echan;
-	int i;
-
-	if (!ecc || dma_spec->args_count < 1)
-		return NULL;
-
-	for (i = 0; i < ecc->num_channels; i++) {
-		echan = &ecc->slave_chans[i];
-		if (echan->ch_num == dma_spec->args[0]) {
-			chan = &echan->vchan.chan;
-			break;
-		}
-	}
-
-	if (!chan)
-		return NULL;
-
-	if (echan->ecc->legacy_mode && dma_spec->args_count == 1)
-		goto out;
-
-	if (!echan->ecc->legacy_mode && dma_spec->args_count == 2 &&
-	    dma_spec->args[1] < echan->ecc->num_tc) {
-		echan->tc = &echan->ecc->tc_list[dma_spec->args[1]];
-		goto out;
-	}
-
-	return NULL;
-out:
-	/* The channel is going to be used as HW synchronized */
-	echan->hw_triggered = true;
-	return dma_get_slave_channel(chan);
-}
-#else
-static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev,
-						     bool legacy_mode)
-{
-	return ERR_PTR(-EINVAL);
-}
-
-static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec,
-				      struct of_dma *ofdma)
-{
-	return NULL;
-}
-#endif
-
-static int edma_probe(struct platform_device *pdev)
-{
-	struct edma_soc_info	*info = pdev->dev.platform_data;
-	s8			(*queue_priority_mapping)[2];
-	int			i, off, ln;
-	const s16		(*rsv_slots)[2];
-	const s16		(*xbar_chans)[2];
-	int			irq;
-	char			*irq_name;
-	struct resource		*mem;
-	struct device_node	*node = pdev->dev.of_node;
-	struct device		*dev = &pdev->dev;
-	struct edma_cc		*ecc;
-	bool			legacy_mode = true;
-	int ret;
-
-	if (node) {
-		const struct of_device_id *match;
-
-		match = of_match_node(edma_of_ids, node);
-		if (match && (*(u32 *)match->data) == EDMA_BINDING_TPCC)
-			legacy_mode = false;
-
-		info = edma_setup_info_from_dt(dev, legacy_mode);
-		if (IS_ERR(info)) {
-			dev_err(dev, "failed to get DT data\n");
-			return PTR_ERR(info);
-		}
-	}
-
-	if (!info)
-		return -ENODEV;
-
-	pm_runtime_enable(dev);
-	ret = pm_runtime_get_sync(dev);
-	if (ret < 0) {
-		dev_err(dev, "pm_runtime_get_sync() failed\n");
-		return ret;
-	}
-
-	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
-	if (ret)
-		return ret;
-
-	ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL);
-	if (!ecc)
-		return -ENOMEM;
-
-	ecc->dev = dev;
-	ecc->id = pdev->id;
-	ecc->legacy_mode = legacy_mode;
-	/* When booting with DT the pdev->id is -1 */
-	if (ecc->id < 0)
-		ecc->id = 0;
-
-	mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "edma3_cc");
-	if (!mem) {
-		dev_dbg(dev, "mem resource not found, using index 0\n");
-		mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-		if (!mem) {
-			dev_err(dev, "no mem resource?\n");
-			return -ENODEV;
-		}
-	}
-	ecc->base = devm_ioremap_resource(dev, mem);
-	if (IS_ERR(ecc->base))
-		return PTR_ERR(ecc->base);
-
-	platform_set_drvdata(pdev, ecc);
-
-	/* Get eDMA3 configuration from IP */
-	ret = edma_setup_from_hw(dev, info, ecc);
-	if (ret)
-		return ret;
-
-	/* Allocate memory based on the information we got from the IP */
-	ecc->slave_chans = devm_kcalloc(dev, ecc->num_channels,
-					sizeof(*ecc->slave_chans), GFP_KERNEL);
-	if (!ecc->slave_chans)
-		return -ENOMEM;
-
-	ecc->slot_inuse = devm_kcalloc(dev, BITS_TO_LONGS(ecc->num_slots),
-				       sizeof(unsigned long), GFP_KERNEL);
-	if (!ecc->slot_inuse)
-		return -ENOMEM;
-
-	ecc->default_queue = info->default_queue;
-
-	for (i = 0; i < ecc->num_slots; i++)
-		edma_write_slot(ecc, i, &dummy_paramset);
-
-	if (info->rsv) {
-		/* Set the reserved slots in inuse list */
-		rsv_slots = info->rsv->rsv_slots;
-		if (rsv_slots) {
-			for (i = 0; rsv_slots[i][0] != -1; i++) {
-				off = rsv_slots[i][0];
-				ln = rsv_slots[i][1];
-				edma_set_bits(off, ln, ecc->slot_inuse);
-			}
-		}
-	}
-
-	/* Clear the xbar mapped channels in unused list */
-	xbar_chans = info->xbar_chans;
-	if (xbar_chans) {
-		for (i = 0; xbar_chans[i][1] != -1; i++) {
-			off = xbar_chans[i][1];
-		}
-	}
-
-	irq = platform_get_irq_byname(pdev, "edma3_ccint");
-	if (irq < 0 && node)
-		irq = irq_of_parse_and_map(node, 0);
-
-	if (irq >= 0) {
-		irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccint",
-					  dev_name(dev));
-		ret = devm_request_irq(dev, irq, dma_irq_handler, 0, irq_name,
-				       ecc);
-		if (ret) {
-			dev_err(dev, "CCINT (%d) failed --> %d\n", irq, ret);
-			return ret;
-		}
-		ecc->ccint = irq;
-	}
-
-	irq = platform_get_irq_byname(pdev, "edma3_ccerrint");
-	if (irq < 0 && node)
-		irq = irq_of_parse_and_map(node, 2);
-
-	if (irq >= 0) {
-		irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccerrint",
-					  dev_name(dev));
-		ret = devm_request_irq(dev, irq, dma_ccerr_handler, 0, irq_name,
-				       ecc);
-		if (ret) {
-			dev_err(dev, "CCERRINT (%d) failed --> %d\n", irq, ret);
-			return ret;
-		}
-		ecc->ccerrint = irq;
-	}
-
-	ecc->dummy_slot = edma_alloc_slot(ecc, EDMA_SLOT_ANY);
-	if (ecc->dummy_slot < 0) {
-		dev_err(dev, "Can't allocate PaRAM dummy slot\n");
-		return ecc->dummy_slot;
-	}
-
-	queue_priority_mapping = info->queue_priority_mapping;
-
-	if (!ecc->legacy_mode) {
-		int lowest_priority = 0;
-		struct of_phandle_args tc_args;
-
-		ecc->tc_list = devm_kcalloc(dev, ecc->num_tc,
-					    sizeof(*ecc->tc_list), GFP_KERNEL);
-		if (!ecc->tc_list)
-			return -ENOMEM;
-
-		for (i = 0;; i++) {
-			ret = of_parse_phandle_with_fixed_args(node, "ti,tptcs",
-							       1, i, &tc_args);
-			if (ret || i == ecc->num_tc)
-				break;
-
-			ecc->tc_list[i].node = tc_args.np;
-			ecc->tc_list[i].id = i;
-			queue_priority_mapping[i][1] = tc_args.args[0];
-			if (queue_priority_mapping[i][1] > lowest_priority) {
-				lowest_priority = queue_priority_mapping[i][1];
-				info->default_queue = i;
-			}
-		}
-	}
-
-	/* Event queue priority mapping */
-	for (i = 0; queue_priority_mapping[i][0] != -1; i++)
-		edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
-					      queue_priority_mapping[i][1]);
-
-	for (i = 0; i < ecc->num_region; i++) {
-		edma_write_array2(ecc, EDMA_DRAE, i, 0, 0x0);
-		edma_write_array2(ecc, EDMA_DRAE, i, 1, 0x0);
-		edma_write_array(ecc, EDMA_QRAE, i, 0x0);
-	}
-	ecc->info = info;
-
-	/* Init the dma device and channels */
-	edma_dma_init(ecc, legacy_mode);
-
-	for (i = 0; i < ecc->num_channels; i++) {
-		/* Assign all channels to the default queue */
-		edma_assign_channel_eventq(&ecc->slave_chans[i],
-					   info->default_queue);
-		/* Set entry slot to the dummy slot */
-		edma_set_chmap(&ecc->slave_chans[i], ecc->dummy_slot);
-	}
-
-	ecc->dma_slave.filter.map = info->slave_map;
-	ecc->dma_slave.filter.mapcnt = info->slavecnt;
-	ecc->dma_slave.filter.fn = edma_filter_fn;
-
-	ret = dma_async_device_register(&ecc->dma_slave);
-	if (ret) {
-		dev_err(dev, "slave ddev registration failed (%d)\n", ret);
-		goto err_reg1;
-	}
-
-	if (ecc->dma_memcpy) {
-		ret = dma_async_device_register(ecc->dma_memcpy);
-		if (ret) {
-			dev_err(dev, "memcpy ddev registration failed (%d)\n",
-				ret);
-			dma_async_device_unregister(&ecc->dma_slave);
-			goto err_reg1;
-		}
-	}
-
-	if (node)
-		of_dma_controller_register(node, of_edma_xlate, ecc);
-
-	dev_info(dev, "TI EDMA DMA engine driver\n");
-
-	return 0;
-
-err_reg1:
-	edma_free_slot(ecc, ecc->dummy_slot);
-	return ret;
-}
-
-static void edma_cleanupp_vchan(struct dma_device *dmadev)
-{
-	struct edma_chan *echan, *_echan;
-
-	list_for_each_entry_safe(echan, _echan,
-			&dmadev->channels, vchan.chan.device_node) {
-		list_del(&echan->vchan.chan.device_node);
-		tasklet_kill(&echan->vchan.task);
-	}
-}
-
-static int edma_remove(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct edma_cc *ecc = dev_get_drvdata(dev);
-
-	devm_free_irq(dev, ecc->ccint, ecc);
-	devm_free_irq(dev, ecc->ccerrint, ecc);
-
-	edma_cleanupp_vchan(&ecc->dma_slave);
-
-	if (dev->of_node)
-		of_dma_controller_free(dev->of_node);
-	dma_async_device_unregister(&ecc->dma_slave);
-	if (ecc->dma_memcpy)
-		dma_async_device_unregister(ecc->dma_memcpy);
-	edma_free_slot(ecc, ecc->dummy_slot);
-
-	return 0;
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int edma_pm_suspend(struct device *dev)
-{
-	struct edma_cc *ecc = dev_get_drvdata(dev);
-	struct edma_chan *echan = ecc->slave_chans;
-	int i;
-
-	for (i = 0; i < ecc->num_channels; i++) {
-		if (echan[i].alloced)
-			edma_setup_interrupt(&echan[i], false);
-	}
-
-	return 0;
-}
-
-static int edma_pm_resume(struct device *dev)
-{
-	struct edma_cc *ecc = dev_get_drvdata(dev);
-	struct edma_chan *echan = ecc->slave_chans;
-	int i;
-	s8 (*queue_priority_mapping)[2];
-
-	/* re initialize dummy slot to dummy param set */
-	edma_write_slot(ecc, ecc->dummy_slot, &dummy_paramset);
-
-	queue_priority_mapping = ecc->info->queue_priority_mapping;
-
-	/* Event queue priority mapping */
-	for (i = 0; queue_priority_mapping[i][0] != -1; i++)
-		edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
-					      queue_priority_mapping[i][1]);
-
-	for (i = 0; i < ecc->num_channels; i++) {
-		if (echan[i].alloced) {
-			/* ensure access through shadow region 0 */
-			edma_or_array2(ecc, EDMA_DRAE, 0, i >> 5,
-				       BIT(i & 0x1f));
-
-			edma_setup_interrupt(&echan[i], true);
-
-			/* Set up channel -> slot mapping for the entry slot */
-			edma_set_chmap(&echan[i], echan[i].slot[0]);
-		}
-	}
-
-	return 0;
-}
-#endif
-
-static const struct dev_pm_ops edma_pm_ops = {
-	SET_LATE_SYSTEM_SLEEP_PM_OPS(edma_pm_suspend, edma_pm_resume)
-};
-
-static struct platform_driver edma_driver = {
-	.probe		= edma_probe,
-	.remove		= edma_remove,
-	.driver = {
-		.name	= "edma",
-		.pm	= &edma_pm_ops,
-		.of_match_table = edma_of_ids,
-	},
-};
-
-static int edma_tptc_probe(struct platform_device *pdev)
-{
-	pm_runtime_enable(&pdev->dev);
-	return pm_runtime_get_sync(&pdev->dev);
-}
-
-static struct platform_driver edma_tptc_driver = {
-	.probe		= edma_tptc_probe,
-	.driver = {
-		.name	= "edma3-tptc",
-		.of_match_table = edma_tptc_of_ids,
-	},
-};
-
-bool edma_filter_fn(struct dma_chan *chan, void *param)
-{
-	bool match = false;
-
-	if (chan->device->dev->driver == &edma_driver.driver) {
-		struct edma_chan *echan = to_edma_chan(chan);
-		unsigned ch_req = *(unsigned *)param;
-		if (ch_req == echan->ch_num) {
-			/* The channel is going to be used as HW synchronized */
-			echan->hw_triggered = true;
-			match = true;
-		}
-	}
-	return match;
-}
-EXPORT_SYMBOL(edma_filter_fn);
-
-static int edma_init(void)
-{
-	int ret;
-
-	ret = platform_driver_register(&edma_tptc_driver);
-	if (ret)
-		return ret;
-
-	return platform_driver_register(&edma_driver);
-}
-subsys_initcall(edma_init);
-
-static void __exit edma_exit(void)
-{
-	platform_driver_unregister(&edma_driver);
-	platform_driver_unregister(&edma_tptc_driver);
-}
-module_exit(edma_exit);
-
-MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
-MODULE_DESCRIPTION("TI EDMA DMA engine driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index 3eaece888e75..1117b5123a6f 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -1328,8 +1328,7 @@ static int fsldma_of_remove(struct platform_device *op)
 #ifdef CONFIG_PM
 static int fsldma_suspend_late(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct fsldma_device *fdev = platform_get_drvdata(pdev);
+	struct fsldma_device *fdev = dev_get_drvdata(dev);
 	struct fsldma_chan *chan;
 	int i;
 
@@ -1360,8 +1359,7 @@ static int fsldma_suspend_late(struct device *dev)
 
 static int fsldma_resume_early(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct fsldma_device *fdev = platform_get_drvdata(pdev);
+	struct fsldma_device *fdev = dev_get_drvdata(dev);
 	struct fsldma_chan *chan;
 	u32 mode;
 	int i;
diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c
index 1953e57505f4..e5c911200bdb 100644
--- a/drivers/dma/idma64.c
+++ b/drivers/dma/idma64.c
@@ -670,8 +670,7 @@ static int idma64_platform_remove(struct platform_device *pdev)
 
 static int idma64_pm_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct idma64_chip *chip = platform_get_drvdata(pdev);
+	struct idma64_chip *chip = dev_get_drvdata(dev);
 
 	idma64_off(chip->idma64);
 	return 0;
@@ -679,8 +678,7 @@ static int idma64_pm_suspend(struct device *dev)
 
 static int idma64_pm_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct idma64_chip *chip = platform_get_drvdata(pdev);
+	struct idma64_chip *chip = dev_get_drvdata(dev);
 
 	idma64_on(chip->idma64);
 	return 0;
diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c
index 715b39ae5a46..75b6ff0415ee 100644
--- a/drivers/dma/imx-dma.c
+++ b/drivers/dma/imx-dma.c
@@ -1,19 +1,13 @@
-/*
- * drivers/dma/imx-dma.c
- *
- * This file contains a driver for the Freescale i.MX DMA engine
- * found on i.MX1/21/27
- *
- * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
- * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com>
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// drivers/dma/imx-dma.c
+//
+// This file contains a driver for the Freescale i.MX DMA engine
+// found on i.MX1/21/27
+//
+// Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
+// Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com>
+
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/types.h>
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
index ccd03c3cedfe..f077992635c2 100644
--- a/drivers/dma/imx-sdma.c
+++ b/drivers/dma/imx-sdma.c
@@ -1,21 +1,14 @@
-/*
- * drivers/dma/imx-sdma.c
- *
- * This file contains a driver for the Freescale Smart DMA engine
- *
- * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
- *
- * Based on code from Freescale:
- *
- * Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// drivers/dma/imx-sdma.c
+//
+// This file contains a driver for the Freescale Smart DMA engine
+//
+// Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
+//
+// Based on code from Freescale:
+//
+// Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved.
 
 #include <linux/init.h>
 #include <linux/iopoll.h>
diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c
index 41d167921fab..ae5182ff0128 100644
--- a/drivers/dma/mxs-dma.c
+++ b/drivers/dma/mxs-dma.c
@@ -1,12 +1,8 @@
-/*
- * Copyright 2011 Freescale Semiconductor, Inc. All Rights Reserved.
- *
- * Refer to drivers/dma/imx-sdma.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2011 Freescale Semiconductor, Inc. All Rights Reserved.
+//
+// Refer to drivers/dma/imx-sdma.c
 
 #include <linux/init.h>
 #include <linux/types.h>
diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c
deleted file mode 100644
index d21c19822feb..000000000000
--- a/drivers/dma/omap-dma.c
+++ /dev/null
@@ -1,1668 +0,0 @@
-/*
- * OMAP DMAengine support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#include <linux/delay.h>
-#include <linux/dmaengine.h>
-#include <linux/dma-mapping.h>
-#include <linux/dmapool.h>
-#include <linux/err.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/list.h>
-#include <linux/module.h>
-#include <linux/omap-dma.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-#include <linux/spinlock.h>
-#include <linux/of_dma.h>
-#include <linux/of_device.h>
-
-#include "virt-dma.h"
-
-#define OMAP_SDMA_REQUESTS	127
-#define OMAP_SDMA_CHANNELS	32
-
-struct omap_dmadev {
-	struct dma_device ddev;
-	spinlock_t lock;
-	void __iomem *base;
-	const struct omap_dma_reg *reg_map;
-	struct omap_system_dma_plat_info *plat;
-	bool legacy;
-	bool ll123_supported;
-	struct dma_pool *desc_pool;
-	unsigned dma_requests;
-	spinlock_t irq_lock;
-	uint32_t irq_enable_mask;
-	struct omap_chan **lch_map;
-};
-
-struct omap_chan {
-	struct virt_dma_chan vc;
-	void __iomem *channel_base;
-	const struct omap_dma_reg *reg_map;
-	uint32_t ccr;
-
-	struct dma_slave_config	cfg;
-	unsigned dma_sig;
-	bool cyclic;
-	bool paused;
-	bool running;
-
-	int dma_ch;
-	struct omap_desc *desc;
-	unsigned sgidx;
-};
-
-#define DESC_NXT_SV_REFRESH	(0x1 << 24)
-#define DESC_NXT_SV_REUSE	(0x2 << 24)
-#define DESC_NXT_DV_REFRESH	(0x1 << 26)
-#define DESC_NXT_DV_REUSE	(0x2 << 26)
-#define DESC_NTYPE_TYPE2	(0x2 << 29)
-
-/* Type 2 descriptor with Source or Destination address update */
-struct omap_type2_desc {
-	uint32_t next_desc;
-	uint32_t en;
-	uint32_t addr; /* src or dst */
-	uint16_t fn;
-	uint16_t cicr;
-	int16_t cdei;
-	int16_t csei;
-	int32_t cdfi;
-	int32_t csfi;
-} __packed;
-
-struct omap_sg {
-	dma_addr_t addr;
-	uint32_t en;		/* number of elements (24-bit) */
-	uint32_t fn;		/* number of frames (16-bit) */
-	int32_t fi;		/* for double indexing */
-	int16_t ei;		/* for double indexing */
-
-	/* Linked list */
-	struct omap_type2_desc *t2_desc;
-	dma_addr_t t2_desc_paddr;
-};
-
-struct omap_desc {
-	struct virt_dma_desc vd;
-	bool using_ll;
-	enum dma_transfer_direction dir;
-	dma_addr_t dev_addr;
-
-	int32_t fi;		/* for OMAP_DMA_SYNC_PACKET / double indexing */
-	int16_t ei;		/* for double indexing */
-	uint8_t es;		/* CSDP_DATA_TYPE_xxx */
-	uint32_t ccr;		/* CCR value */
-	uint16_t clnk_ctrl;	/* CLNK_CTRL value */
-	uint16_t cicr;		/* CICR value */
-	uint32_t csdp;		/* CSDP value */
-
-	unsigned sglen;
-	struct omap_sg sg[0];
-};
-
-enum {
-	CAPS_0_SUPPORT_LL123	= BIT(20),	/* Linked List type1/2/3 */
-	CAPS_0_SUPPORT_LL4	= BIT(21),	/* Linked List type4 */
-
-	CCR_FS			= BIT(5),
-	CCR_READ_PRIORITY	= BIT(6),
-	CCR_ENABLE		= BIT(7),
-	CCR_AUTO_INIT		= BIT(8),	/* OMAP1 only */
-	CCR_REPEAT		= BIT(9),	/* OMAP1 only */
-	CCR_OMAP31_DISABLE	= BIT(10),	/* OMAP1 only */
-	CCR_SUSPEND_SENSITIVE	= BIT(8),	/* OMAP2+ only */
-	CCR_RD_ACTIVE		= BIT(9),	/* OMAP2+ only */
-	CCR_WR_ACTIVE		= BIT(10),	/* OMAP2+ only */
-	CCR_SRC_AMODE_CONSTANT	= 0 << 12,
-	CCR_SRC_AMODE_POSTINC	= 1 << 12,
-	CCR_SRC_AMODE_SGLIDX	= 2 << 12,
-	CCR_SRC_AMODE_DBLIDX	= 3 << 12,
-	CCR_DST_AMODE_CONSTANT	= 0 << 14,
-	CCR_DST_AMODE_POSTINC	= 1 << 14,
-	CCR_DST_AMODE_SGLIDX	= 2 << 14,
-	CCR_DST_AMODE_DBLIDX	= 3 << 14,
-	CCR_CONSTANT_FILL	= BIT(16),
-	CCR_TRANSPARENT_COPY	= BIT(17),
-	CCR_BS			= BIT(18),
-	CCR_SUPERVISOR		= BIT(22),
-	CCR_PREFETCH		= BIT(23),
-	CCR_TRIGGER_SRC		= BIT(24),
-	CCR_BUFFERING_DISABLE	= BIT(25),
-	CCR_WRITE_PRIORITY	= BIT(26),
-	CCR_SYNC_ELEMENT	= 0,
-	CCR_SYNC_FRAME		= CCR_FS,
-	CCR_SYNC_BLOCK		= CCR_BS,
-	CCR_SYNC_PACKET		= CCR_BS | CCR_FS,
-
-	CSDP_DATA_TYPE_8	= 0,
-	CSDP_DATA_TYPE_16	= 1,
-	CSDP_DATA_TYPE_32	= 2,
-	CSDP_SRC_PORT_EMIFF	= 0 << 2, /* OMAP1 only */
-	CSDP_SRC_PORT_EMIFS	= 1 << 2, /* OMAP1 only */
-	CSDP_SRC_PORT_OCP_T1	= 2 << 2, /* OMAP1 only */
-	CSDP_SRC_PORT_TIPB	= 3 << 2, /* OMAP1 only */
-	CSDP_SRC_PORT_OCP_T2	= 4 << 2, /* OMAP1 only */
-	CSDP_SRC_PORT_MPUI	= 5 << 2, /* OMAP1 only */
-	CSDP_SRC_PACKED		= BIT(6),
-	CSDP_SRC_BURST_1	= 0 << 7,
-	CSDP_SRC_BURST_16	= 1 << 7,
-	CSDP_SRC_BURST_32	= 2 << 7,
-	CSDP_SRC_BURST_64	= 3 << 7,
-	CSDP_DST_PORT_EMIFF	= 0 << 9, /* OMAP1 only */
-	CSDP_DST_PORT_EMIFS	= 1 << 9, /* OMAP1 only */
-	CSDP_DST_PORT_OCP_T1	= 2 << 9, /* OMAP1 only */
-	CSDP_DST_PORT_TIPB	= 3 << 9, /* OMAP1 only */
-	CSDP_DST_PORT_OCP_T2	= 4 << 9, /* OMAP1 only */
-	CSDP_DST_PORT_MPUI	= 5 << 9, /* OMAP1 only */
-	CSDP_DST_PACKED		= BIT(13),
-	CSDP_DST_BURST_1	= 0 << 14,
-	CSDP_DST_BURST_16	= 1 << 14,
-	CSDP_DST_BURST_32	= 2 << 14,
-	CSDP_DST_BURST_64	= 3 << 14,
-	CSDP_WRITE_NON_POSTED	= 0 << 16,
-	CSDP_WRITE_POSTED	= 1 << 16,
-	CSDP_WRITE_LAST_NON_POSTED = 2 << 16,
-
-	CICR_TOUT_IE		= BIT(0),	/* OMAP1 only */
-	CICR_DROP_IE		= BIT(1),
-	CICR_HALF_IE		= BIT(2),
-	CICR_FRAME_IE		= BIT(3),
-	CICR_LAST_IE		= BIT(4),
-	CICR_BLOCK_IE		= BIT(5),
-	CICR_PKT_IE		= BIT(7),	/* OMAP2+ only */
-	CICR_TRANS_ERR_IE	= BIT(8),	/* OMAP2+ only */
-	CICR_SUPERVISOR_ERR_IE	= BIT(10),	/* OMAP2+ only */
-	CICR_MISALIGNED_ERR_IE	= BIT(11),	/* OMAP2+ only */
-	CICR_DRAIN_IE		= BIT(12),	/* OMAP2+ only */
-	CICR_SUPER_BLOCK_IE	= BIT(14),	/* OMAP2+ only */
-
-	CLNK_CTRL_ENABLE_LNK	= BIT(15),
-
-	CDP_DST_VALID_INC	= 0 << 0,
-	CDP_DST_VALID_RELOAD	= 1 << 0,
-	CDP_DST_VALID_REUSE	= 2 << 0,
-	CDP_SRC_VALID_INC	= 0 << 2,
-	CDP_SRC_VALID_RELOAD	= 1 << 2,
-	CDP_SRC_VALID_REUSE	= 2 << 2,
-	CDP_NTYPE_TYPE1		= 1 << 4,
-	CDP_NTYPE_TYPE2		= 2 << 4,
-	CDP_NTYPE_TYPE3		= 3 << 4,
-	CDP_TMODE_NORMAL	= 0 << 8,
-	CDP_TMODE_LLIST		= 1 << 8,
-	CDP_FAST		= BIT(10),
-};
-
-static const unsigned es_bytes[] = {
-	[CSDP_DATA_TYPE_8] = 1,
-	[CSDP_DATA_TYPE_16] = 2,
-	[CSDP_DATA_TYPE_32] = 4,
-};
-
-static struct of_dma_filter_info omap_dma_info = {
-	.filter_fn = omap_dma_filter_fn,
-};
-
-static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
-{
-	return container_of(d, struct omap_dmadev, ddev);
-}
-
-static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
-{
-	return container_of(c, struct omap_chan, vc.chan);
-}
-
-static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
-{
-	return container_of(t, struct omap_desc, vd.tx);
-}
-
-static void omap_dma_desc_free(struct virt_dma_desc *vd)
-{
-	struct omap_desc *d = to_omap_dma_desc(&vd->tx);
-
-	if (d->using_ll) {
-		struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
-		int i;
-
-		for (i = 0; i < d->sglen; i++) {
-			if (d->sg[i].t2_desc)
-				dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
-					      d->sg[i].t2_desc_paddr);
-		}
-	}
-
-	kfree(d);
-}
-
-static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
-				     enum dma_transfer_direction dir, bool last)
-{
-	struct omap_sg *sg = &d->sg[idx];
-	struct omap_type2_desc *t2_desc = sg->t2_desc;
-
-	if (idx)
-		d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
-	if (last)
-		t2_desc->next_desc = 0xfffffffc;
-
-	t2_desc->en = sg->en;
-	t2_desc->addr = sg->addr;
-	t2_desc->fn = sg->fn & 0xffff;
-	t2_desc->cicr = d->cicr;
-	if (!last)
-		t2_desc->cicr &= ~CICR_BLOCK_IE;
-
-	switch (dir) {
-	case DMA_DEV_TO_MEM:
-		t2_desc->cdei = sg->ei;
-		t2_desc->csei = d->ei;
-		t2_desc->cdfi = sg->fi;
-		t2_desc->csfi = d->fi;
-
-		t2_desc->en |= DESC_NXT_DV_REFRESH;
-		t2_desc->en |= DESC_NXT_SV_REUSE;
-		break;
-	case DMA_MEM_TO_DEV:
-		t2_desc->cdei = d->ei;
-		t2_desc->csei = sg->ei;
-		t2_desc->cdfi = d->fi;
-		t2_desc->csfi = sg->fi;
-
-		t2_desc->en |= DESC_NXT_SV_REFRESH;
-		t2_desc->en |= DESC_NXT_DV_REUSE;
-		break;
-	default:
-		return;
-	}
-
-	t2_desc->en |= DESC_NTYPE_TYPE2;
-}
-
-static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
-{
-	switch (type) {
-	case OMAP_DMA_REG_16BIT:
-		writew_relaxed(val, addr);
-		break;
-	case OMAP_DMA_REG_2X16BIT:
-		writew_relaxed(val, addr);
-		writew_relaxed(val >> 16, addr + 2);
-		break;
-	case OMAP_DMA_REG_32BIT:
-		writel_relaxed(val, addr);
-		break;
-	default:
-		WARN_ON(1);
-	}
-}
-
-static unsigned omap_dma_read(unsigned type, void __iomem *addr)
-{
-	unsigned val;
-
-	switch (type) {
-	case OMAP_DMA_REG_16BIT:
-		val = readw_relaxed(addr);
-		break;
-	case OMAP_DMA_REG_2X16BIT:
-		val = readw_relaxed(addr);
-		val |= readw_relaxed(addr + 2) << 16;
-		break;
-	case OMAP_DMA_REG_32BIT:
-		val = readl_relaxed(addr);
-		break;
-	default:
-		WARN_ON(1);
-		val = 0;
-	}
-
-	return val;
-}
-
-static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
-{
-	const struct omap_dma_reg *r = od->reg_map + reg;
-
-	WARN_ON(r->stride);
-
-	omap_dma_write(val, r->type, od->base + r->offset);
-}
-
-static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
-{
-	const struct omap_dma_reg *r = od->reg_map + reg;
-
-	WARN_ON(r->stride);
-
-	return omap_dma_read(r->type, od->base + r->offset);
-}
-
-static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
-{
-	const struct omap_dma_reg *r = c->reg_map + reg;
-
-	omap_dma_write(val, r->type, c->channel_base + r->offset);
-}
-
-static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
-{
-	const struct omap_dma_reg *r = c->reg_map + reg;
-
-	return omap_dma_read(r->type, c->channel_base + r->offset);
-}
-
-static void omap_dma_clear_csr(struct omap_chan *c)
-{
-	if (dma_omap1())
-		omap_dma_chan_read(c, CSR);
-	else
-		omap_dma_chan_write(c, CSR, ~0);
-}
-
-static unsigned omap_dma_get_csr(struct omap_chan *c)
-{
-	unsigned val = omap_dma_chan_read(c, CSR);
-
-	if (!dma_omap1())
-		omap_dma_chan_write(c, CSR, val);
-
-	return val;
-}
-
-static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
-	unsigned lch)
-{
-	c->channel_base = od->base + od->plat->channel_stride * lch;
-
-	od->lch_map[lch] = c;
-}
-
-static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
-{
-	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
-	uint16_t cicr = d->cicr;
-
-	if (__dma_omap15xx(od->plat->dma_attr))
-		omap_dma_chan_write(c, CPC, 0);
-	else
-		omap_dma_chan_write(c, CDAC, 0);
-
-	omap_dma_clear_csr(c);
-
-	if (d->using_ll) {
-		uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
-
-		if (d->dir == DMA_DEV_TO_MEM)
-			cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
-		else
-			cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
-		omap_dma_chan_write(c, CDP, cdp);
-
-		omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
-		omap_dma_chan_write(c, CCDN, 0);
-		omap_dma_chan_write(c, CCFN, 0xffff);
-		omap_dma_chan_write(c, CCEN, 0xffffff);
-
-		cicr &= ~CICR_BLOCK_IE;
-	} else if (od->ll123_supported) {
-		omap_dma_chan_write(c, CDP, 0);
-	}
-
-	/* Enable interrupts */
-	omap_dma_chan_write(c, CICR, cicr);
-
-	/* Enable channel */
-	omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
-
-	c->running = true;
-}
-
-static void omap_dma_drain_chan(struct omap_chan *c)
-{
-	int i;
-	u32 val;
-
-	/* Wait for sDMA FIFO to drain */
-	for (i = 0; ; i++) {
-		val = omap_dma_chan_read(c, CCR);
-		if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
-			break;
-
-		if (i > 100)
-			break;
-
-		udelay(5);
-	}
-
-	if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
-		dev_err(c->vc.chan.device->dev,
-			"DMA drain did not complete on lch %d\n",
-			c->dma_ch);
-}
-
-static int omap_dma_stop(struct omap_chan *c)
-{
-	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
-	uint32_t val;
-
-	/* disable irq */
-	omap_dma_chan_write(c, CICR, 0);
-
-	omap_dma_clear_csr(c);
-
-	val = omap_dma_chan_read(c, CCR);
-	if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
-		uint32_t sysconfig;
-
-		sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
-		val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
-		val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
-		omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
-
-		val = omap_dma_chan_read(c, CCR);
-		val &= ~CCR_ENABLE;
-		omap_dma_chan_write(c, CCR, val);
-
-		if (!(c->ccr & CCR_BUFFERING_DISABLE))
-			omap_dma_drain_chan(c);
-
-		omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
-	} else {
-		if (!(val & CCR_ENABLE))
-			return -EINVAL;
-
-		val &= ~CCR_ENABLE;
-		omap_dma_chan_write(c, CCR, val);
-
-		if (!(c->ccr & CCR_BUFFERING_DISABLE))
-			omap_dma_drain_chan(c);
-	}
-
-	mb();
-
-	if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
-		val = omap_dma_chan_read(c, CLNK_CTRL);
-
-		if (dma_omap1())
-			val |= 1 << 14; /* set the STOP_LNK bit */
-		else
-			val &= ~CLNK_CTRL_ENABLE_LNK;
-
-		omap_dma_chan_write(c, CLNK_CTRL, val);
-	}
-	c->running = false;
-	return 0;
-}
-
-static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
-{
-	struct omap_sg *sg = d->sg + c->sgidx;
-	unsigned cxsa, cxei, cxfi;
-
-	if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
-		cxsa = CDSA;
-		cxei = CDEI;
-		cxfi = CDFI;
-	} else {
-		cxsa = CSSA;
-		cxei = CSEI;
-		cxfi = CSFI;
-	}
-
-	omap_dma_chan_write(c, cxsa, sg->addr);
-	omap_dma_chan_write(c, cxei, sg->ei);
-	omap_dma_chan_write(c, cxfi, sg->fi);
-	omap_dma_chan_write(c, CEN, sg->en);
-	omap_dma_chan_write(c, CFN, sg->fn);
-
-	omap_dma_start(c, d);
-	c->sgidx++;
-}
-
-static void omap_dma_start_desc(struct omap_chan *c)
-{
-	struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
-	struct omap_desc *d;
-	unsigned cxsa, cxei, cxfi;
-
-	if (!vd) {
-		c->desc = NULL;
-		return;
-	}
-
-	list_del(&vd->node);
-
-	c->desc = d = to_omap_dma_desc(&vd->tx);
-	c->sgidx = 0;
-
-	/*
-	 * This provides the necessary barrier to ensure data held in
-	 * DMA coherent memory is visible to the DMA engine prior to
-	 * the transfer starting.
-	 */
-	mb();
-
-	omap_dma_chan_write(c, CCR, d->ccr);
-	if (dma_omap1())
-		omap_dma_chan_write(c, CCR2, d->ccr >> 16);
-
-	if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
-		cxsa = CSSA;
-		cxei = CSEI;
-		cxfi = CSFI;
-	} else {
-		cxsa = CDSA;
-		cxei = CDEI;
-		cxfi = CDFI;
-	}
-
-	omap_dma_chan_write(c, cxsa, d->dev_addr);
-	omap_dma_chan_write(c, cxei, d->ei);
-	omap_dma_chan_write(c, cxfi, d->fi);
-	omap_dma_chan_write(c, CSDP, d->csdp);
-	omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
-
-	omap_dma_start_sg(c, d);
-}
-
-static void omap_dma_callback(int ch, u16 status, void *data)
-{
-	struct omap_chan *c = data;
-	struct omap_desc *d;
-	unsigned long flags;
-
-	spin_lock_irqsave(&c->vc.lock, flags);
-	d = c->desc;
-	if (d) {
-		if (c->cyclic) {
-			vchan_cyclic_callback(&d->vd);
-		} else if (d->using_ll || c->sgidx == d->sglen) {
-			omap_dma_start_desc(c);
-			vchan_cookie_complete(&d->vd);
-		} else {
-			omap_dma_start_sg(c, d);
-		}
-	}
-	spin_unlock_irqrestore(&c->vc.lock, flags);
-}
-
-static irqreturn_t omap_dma_irq(int irq, void *devid)
-{
-	struct omap_dmadev *od = devid;
-	unsigned status, channel;
-
-	spin_lock(&od->irq_lock);
-
-	status = omap_dma_glbl_read(od, IRQSTATUS_L1);
-	status &= od->irq_enable_mask;
-	if (status == 0) {
-		spin_unlock(&od->irq_lock);
-		return IRQ_NONE;
-	}
-
-	while ((channel = ffs(status)) != 0) {
-		unsigned mask, csr;
-		struct omap_chan *c;
-
-		channel -= 1;
-		mask = BIT(channel);
-		status &= ~mask;
-
-		c = od->lch_map[channel];
-		if (c == NULL) {
-			/* This should never happen */
-			dev_err(od->ddev.dev, "invalid channel %u\n", channel);
-			continue;
-		}
-
-		csr = omap_dma_get_csr(c);
-		omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
-
-		omap_dma_callback(channel, csr, c);
-	}
-
-	spin_unlock(&od->irq_lock);
-
-	return IRQ_HANDLED;
-}
-
-static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
-{
-	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
-	struct omap_chan *c = to_omap_dma_chan(chan);
-	struct device *dev = od->ddev.dev;
-	int ret;
-
-	if (od->legacy) {
-		ret = omap_request_dma(c->dma_sig, "DMA engine",
-				       omap_dma_callback, c, &c->dma_ch);
-	} else {
-		ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,
-				       &c->dma_ch);
-	}
-
-	dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
-
-	if (ret >= 0) {
-		omap_dma_assign(od, c, c->dma_ch);
-
-		if (!od->legacy) {
-			unsigned val;
-
-			spin_lock_irq(&od->irq_lock);
-			val = BIT(c->dma_ch);
-			omap_dma_glbl_write(od, IRQSTATUS_L1, val);
-			od->irq_enable_mask |= val;
-			omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
-
-			val = omap_dma_glbl_read(od, IRQENABLE_L0);
-			val &= ~BIT(c->dma_ch);
-			omap_dma_glbl_write(od, IRQENABLE_L0, val);
-			spin_unlock_irq(&od->irq_lock);
-		}
-	}
-
-	if (dma_omap1()) {
-		if (__dma_omap16xx(od->plat->dma_attr)) {
-			c->ccr = CCR_OMAP31_DISABLE;
-			/* Duplicate what plat-omap/dma.c does */
-			c->ccr |= c->dma_ch + 1;
-		} else {
-			c->ccr = c->dma_sig & 0x1f;
-		}
-	} else {
-		c->ccr = c->dma_sig & 0x1f;
-		c->ccr |= (c->dma_sig & ~0x1f) << 14;
-	}
-	if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
-		c->ccr |= CCR_BUFFERING_DISABLE;
-
-	return ret;
-}
-
-static void omap_dma_free_chan_resources(struct dma_chan *chan)
-{
-	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
-	struct omap_chan *c = to_omap_dma_chan(chan);
-
-	if (!od->legacy) {
-		spin_lock_irq(&od->irq_lock);
-		od->irq_enable_mask &= ~BIT(c->dma_ch);
-		omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
-		spin_unlock_irq(&od->irq_lock);
-	}
-
-	c->channel_base = NULL;
-	od->lch_map[c->dma_ch] = NULL;
-	vchan_free_chan_resources(&c->vc);
-	omap_free_dma(c->dma_ch);
-
-	dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
-		c->dma_sig);
-	c->dma_sig = 0;
-}
-
-static size_t omap_dma_sg_size(struct omap_sg *sg)
-{
-	return sg->en * sg->fn;
-}
-
-static size_t omap_dma_desc_size(struct omap_desc *d)
-{
-	unsigned i;
-	size_t size;
-
-	for (size = i = 0; i < d->sglen; i++)
-		size += omap_dma_sg_size(&d->sg[i]);
-
-	return size * es_bytes[d->es];
-}
-
-static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
-{
-	unsigned i;
-	size_t size, es_size = es_bytes[d->es];
-
-	for (size = i = 0; i < d->sglen; i++) {
-		size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
-
-		if (size)
-			size += this_size;
-		else if (addr >= d->sg[i].addr &&
-			 addr < d->sg[i].addr + this_size)
-			size += d->sg[i].addr + this_size - addr;
-	}
-	return size;
-}
-
-/*
- * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
- * read before the DMA controller finished disabling the channel.
- */
-static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
-{
-	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
-	uint32_t val;
-
-	val = omap_dma_chan_read(c, reg);
-	if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
-		val = omap_dma_chan_read(c, reg);
-
-	return val;
-}
-
-static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
-{
-	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
-	dma_addr_t addr, cdac;
-
-	if (__dma_omap15xx(od->plat->dma_attr)) {
-		addr = omap_dma_chan_read(c, CPC);
-	} else {
-		addr = omap_dma_chan_read_3_3(c, CSAC);
-		cdac = omap_dma_chan_read_3_3(c, CDAC);
-
-		/*
-		 * CDAC == 0 indicates that the DMA transfer on the channel has
-		 * not been started (no data has been transferred so far).
-		 * Return the programmed source start address in this case.
-		 */
-		if (cdac == 0)
-			addr = omap_dma_chan_read(c, CSSA);
-	}
-
-	if (dma_omap1())
-		addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
-
-	return addr;
-}
-
-static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
-{
-	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
-	dma_addr_t addr;
-
-	if (__dma_omap15xx(od->plat->dma_attr)) {
-		addr = omap_dma_chan_read(c, CPC);
-	} else {
-		addr = omap_dma_chan_read_3_3(c, CDAC);
-
-		/*
-		 * CDAC == 0 indicates that the DMA transfer on the channel
-		 * has not been started (no data has been transferred so
-		 * far).  Return the programmed destination start address in
-		 * this case.
-		 */
-		if (addr == 0)
-			addr = omap_dma_chan_read(c, CDSA);
-	}
-
-	if (dma_omap1())
-		addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
-
-	return addr;
-}
-
-static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
-	dma_cookie_t cookie, struct dma_tx_state *txstate)
-{
-	struct omap_chan *c = to_omap_dma_chan(chan);
-	struct virt_dma_desc *vd;
-	enum dma_status ret;
-	unsigned long flags;
-
-	ret = dma_cookie_status(chan, cookie, txstate);
-
-	if (!c->paused && c->running) {
-		uint32_t ccr = omap_dma_chan_read(c, CCR);
-		/*
-		 * The channel is no longer active, set the return value
-		 * accordingly
-		 */
-		if (!(ccr & CCR_ENABLE))
-			ret = DMA_COMPLETE;
-	}
-
-	if (ret == DMA_COMPLETE || !txstate)
-		return ret;
-
-	spin_lock_irqsave(&c->vc.lock, flags);
-	vd = vchan_find_desc(&c->vc, cookie);
-	if (vd) {
-		txstate->residue = omap_dma_desc_size(to_omap_dma_desc(&vd->tx));
-	} else if (c->desc && c->desc->vd.tx.cookie == cookie) {
-		struct omap_desc *d = c->desc;
-		dma_addr_t pos;
-
-		if (d->dir == DMA_MEM_TO_DEV)
-			pos = omap_dma_get_src_pos(c);
-		else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
-			pos = omap_dma_get_dst_pos(c);
-		else
-			pos = 0;
-
-		txstate->residue = omap_dma_desc_size_pos(d, pos);
-	} else {
-		txstate->residue = 0;
-	}
-	if (ret == DMA_IN_PROGRESS && c->paused)
-		ret = DMA_PAUSED;
-	spin_unlock_irqrestore(&c->vc.lock, flags);
-
-	return ret;
-}
-
-static void omap_dma_issue_pending(struct dma_chan *chan)
-{
-	struct omap_chan *c = to_omap_dma_chan(chan);
-	unsigned long flags;
-
-	spin_lock_irqsave(&c->vc.lock, flags);
-	if (vchan_issue_pending(&c->vc) && !c->desc)
-		omap_dma_start_desc(c);
-	spin_unlock_irqrestore(&c->vc.lock, flags);
-}
-
-static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
-	struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
-	enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
-{
-	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
-	struct omap_chan *c = to_omap_dma_chan(chan);
-	enum dma_slave_buswidth dev_width;
-	struct scatterlist *sgent;
-	struct omap_desc *d;
-	dma_addr_t dev_addr;
-	unsigned i, es, en, frame_bytes;
-	bool ll_failed = false;
-	u32 burst;
-	u32 port_window, port_window_bytes;
-
-	if (dir == DMA_DEV_TO_MEM) {
-		dev_addr = c->cfg.src_addr;
-		dev_width = c->cfg.src_addr_width;
-		burst = c->cfg.src_maxburst;
-		port_window = c->cfg.src_port_window_size;
-	} else if (dir == DMA_MEM_TO_DEV) {
-		dev_addr = c->cfg.dst_addr;
-		dev_width = c->cfg.dst_addr_width;
-		burst = c->cfg.dst_maxburst;
-		port_window = c->cfg.dst_port_window_size;
-	} else {
-		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
-		return NULL;
-	}
-
-	/* Bus width translates to the element size (ES) */
-	switch (dev_width) {
-	case DMA_SLAVE_BUSWIDTH_1_BYTE:
-		es = CSDP_DATA_TYPE_8;
-		break;
-	case DMA_SLAVE_BUSWIDTH_2_BYTES:
-		es = CSDP_DATA_TYPE_16;
-		break;
-	case DMA_SLAVE_BUSWIDTH_4_BYTES:
-		es = CSDP_DATA_TYPE_32;
-		break;
-	default: /* not reached */
-		return NULL;
-	}
-
-	/* Now allocate and setup the descriptor. */
-	d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC);
-	if (!d)
-		return NULL;
-
-	d->dir = dir;
-	d->dev_addr = dev_addr;
-	d->es = es;
-
-	/* When the port_window is used, one frame must cover the window */
-	if (port_window) {
-		burst = port_window;
-		port_window_bytes = port_window * es_bytes[es];
-
-		d->ei = 1;
-		/*
-		 * One frame covers the port_window and by  configure
-		 * the source frame index to be -1 * (port_window - 1)
-		 * we instruct the sDMA that after a frame is processed
-		 * it should move back to the start of the window.
-		 */
-		d->fi = -(port_window_bytes - 1);
-	}
-
-	d->ccr = c->ccr | CCR_SYNC_FRAME;
-	if (dir == DMA_DEV_TO_MEM) {
-		d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
-
-		d->ccr |= CCR_DST_AMODE_POSTINC;
-		if (port_window) {
-			d->ccr |= CCR_SRC_AMODE_DBLIDX;
-
-			if (port_window_bytes >= 64)
-				d->csdp |= CSDP_SRC_BURST_64;
-			else if (port_window_bytes >= 32)
-				d->csdp |= CSDP_SRC_BURST_32;
-			else if (port_window_bytes >= 16)
-				d->csdp |= CSDP_SRC_BURST_16;
-
-		} else {
-			d->ccr |= CCR_SRC_AMODE_CONSTANT;
-		}
-	} else {
-		d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
-
-		d->ccr |= CCR_SRC_AMODE_POSTINC;
-		if (port_window) {
-			d->ccr |= CCR_DST_AMODE_DBLIDX;
-
-			if (port_window_bytes >= 64)
-				d->csdp |= CSDP_DST_BURST_64;
-			else if (port_window_bytes >= 32)
-				d->csdp |= CSDP_DST_BURST_32;
-			else if (port_window_bytes >= 16)
-				d->csdp |= CSDP_DST_BURST_16;
-		} else {
-			d->ccr |= CCR_DST_AMODE_CONSTANT;
-		}
-	}
-
-	d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
-	d->csdp |= es;
-
-	if (dma_omap1()) {
-		d->cicr |= CICR_TOUT_IE;
-
-		if (dir == DMA_DEV_TO_MEM)
-			d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
-		else
-			d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
-	} else {
-		if (dir == DMA_DEV_TO_MEM)
-			d->ccr |= CCR_TRIGGER_SRC;
-
-		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
-
-		if (port_window)
-			d->csdp |= CSDP_WRITE_LAST_NON_POSTED;
-	}
-	if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
-		d->clnk_ctrl = c->dma_ch;
-
-	/*
-	 * Build our scatterlist entries: each contains the address,
-	 * the number of elements (EN) in each frame, and the number of
-	 * frames (FN).  Number of bytes for this entry = ES * EN * FN.
-	 *
-	 * Burst size translates to number of elements with frame sync.
-	 * Note: DMA engine defines burst to be the number of dev-width
-	 * transfers.
-	 */
-	en = burst;
-	frame_bytes = es_bytes[es] * en;
-
-	if (sglen >= 2)
-		d->using_ll = od->ll123_supported;
-
-	for_each_sg(sgl, sgent, sglen, i) {
-		struct omap_sg *osg = &d->sg[i];
-
-		osg->addr = sg_dma_address(sgent);
-		osg->en = en;
-		osg->fn = sg_dma_len(sgent) / frame_bytes;
-
-		if (d->using_ll) {
-			osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
-						      &osg->t2_desc_paddr);
-			if (!osg->t2_desc) {
-				dev_err(chan->device->dev,
-					"t2_desc[%d] allocation failed\n", i);
-				ll_failed = true;
-				d->using_ll = false;
-				continue;
-			}
-
-			omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
-		}
-	}
-
-	d->sglen = sglen;
-
-	/* Release the dma_pool entries if one allocation failed */
-	if (ll_failed) {
-		for (i = 0; i < d->sglen; i++) {
-			struct omap_sg *osg = &d->sg[i];
-
-			if (osg->t2_desc) {
-				dma_pool_free(od->desc_pool, osg->t2_desc,
-					      osg->t2_desc_paddr);
-				osg->t2_desc = NULL;
-			}
-		}
-	}
-
-	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
-}
-
-static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
-	struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
-	size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
-{
-	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
-	struct omap_chan *c = to_omap_dma_chan(chan);
-	enum dma_slave_buswidth dev_width;
-	struct omap_desc *d;
-	dma_addr_t dev_addr;
-	unsigned es;
-	u32 burst;
-
-	if (dir == DMA_DEV_TO_MEM) {
-		dev_addr = c->cfg.src_addr;
-		dev_width = c->cfg.src_addr_width;
-		burst = c->cfg.src_maxburst;
-	} else if (dir == DMA_MEM_TO_DEV) {
-		dev_addr = c->cfg.dst_addr;
-		dev_width = c->cfg.dst_addr_width;
-		burst = c->cfg.dst_maxburst;
-	} else {
-		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
-		return NULL;
-	}
-
-	/* Bus width translates to the element size (ES) */
-	switch (dev_width) {
-	case DMA_SLAVE_BUSWIDTH_1_BYTE:
-		es = CSDP_DATA_TYPE_8;
-		break;
-	case DMA_SLAVE_BUSWIDTH_2_BYTES:
-		es = CSDP_DATA_TYPE_16;
-		break;
-	case DMA_SLAVE_BUSWIDTH_4_BYTES:
-		es = CSDP_DATA_TYPE_32;
-		break;
-	default: /* not reached */
-		return NULL;
-	}
-
-	/* Now allocate and setup the descriptor. */
-	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
-	if (!d)
-		return NULL;
-
-	d->dir = dir;
-	d->dev_addr = dev_addr;
-	d->fi = burst;
-	d->es = es;
-	d->sg[0].addr = buf_addr;
-	d->sg[0].en = period_len / es_bytes[es];
-	d->sg[0].fn = buf_len / period_len;
-	d->sglen = 1;
-
-	d->ccr = c->ccr;
-	if (dir == DMA_DEV_TO_MEM)
-		d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
-	else
-		d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
-
-	d->cicr = CICR_DROP_IE;
-	if (flags & DMA_PREP_INTERRUPT)
-		d->cicr |= CICR_FRAME_IE;
-
-	d->csdp = es;
-
-	if (dma_omap1()) {
-		d->cicr |= CICR_TOUT_IE;
-
-		if (dir == DMA_DEV_TO_MEM)
-			d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
-		else
-			d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
-	} else {
-		if (burst)
-			d->ccr |= CCR_SYNC_PACKET;
-		else
-			d->ccr |= CCR_SYNC_ELEMENT;
-
-		if (dir == DMA_DEV_TO_MEM) {
-			d->ccr |= CCR_TRIGGER_SRC;
-			d->csdp |= CSDP_DST_PACKED;
-		} else {
-			d->csdp |= CSDP_SRC_PACKED;
-		}
-
-		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
-
-		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
-	}
-
-	if (__dma_omap15xx(od->plat->dma_attr))
-		d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
-	else
-		d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
-
-	c->cyclic = true;
-
-	return vchan_tx_prep(&c->vc, &d->vd, flags);
-}
-
-static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
-	struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
-	size_t len, unsigned long tx_flags)
-{
-	struct omap_chan *c = to_omap_dma_chan(chan);
-	struct omap_desc *d;
-	uint8_t data_type;
-
-	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
-	if (!d)
-		return NULL;
-
-	data_type = __ffs((src | dest | len));
-	if (data_type > CSDP_DATA_TYPE_32)
-		data_type = CSDP_DATA_TYPE_32;
-
-	d->dir = DMA_MEM_TO_MEM;
-	d->dev_addr = src;
-	d->fi = 0;
-	d->es = data_type;
-	d->sg[0].en = len / BIT(data_type);
-	d->sg[0].fn = 1;
-	d->sg[0].addr = dest;
-	d->sglen = 1;
-	d->ccr = c->ccr;
-	d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
-
-	d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
-
-	d->csdp = data_type;
-
-	if (dma_omap1()) {
-		d->cicr |= CICR_TOUT_IE;
-		d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
-	} else {
-		d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
-		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
-		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
-	}
-
-	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
-}
-
-static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
-	struct dma_chan *chan, struct dma_interleaved_template *xt,
-	unsigned long flags)
-{
-	struct omap_chan *c = to_omap_dma_chan(chan);
-	struct omap_desc *d;
-	struct omap_sg *sg;
-	uint8_t data_type;
-	size_t src_icg, dst_icg;
-
-	/* Slave mode is not supported */
-	if (is_slave_direction(xt->dir))
-		return NULL;
-
-	if (xt->frame_size != 1 || xt->numf == 0)
-		return NULL;
-
-	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
-	if (!d)
-		return NULL;
-
-	data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
-	if (data_type > CSDP_DATA_TYPE_32)
-		data_type = CSDP_DATA_TYPE_32;
-
-	sg = &d->sg[0];
-	d->dir = DMA_MEM_TO_MEM;
-	d->dev_addr = xt->src_start;
-	d->es = data_type;
-	sg->en = xt->sgl[0].size / BIT(data_type);
-	sg->fn = xt->numf;
-	sg->addr = xt->dst_start;
-	d->sglen = 1;
-	d->ccr = c->ccr;
-
-	src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
-	dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
-	if (src_icg) {
-		d->ccr |= CCR_SRC_AMODE_DBLIDX;
-		d->ei = 1;
-		d->fi = src_icg;
-	} else if (xt->src_inc) {
-		d->ccr |= CCR_SRC_AMODE_POSTINC;
-		d->fi = 0;
-	} else {
-		dev_err(chan->device->dev,
-			"%s: SRC constant addressing is not supported\n",
-			__func__);
-		kfree(d);
-		return NULL;
-	}
-
-	if (dst_icg) {
-		d->ccr |= CCR_DST_AMODE_DBLIDX;
-		sg->ei = 1;
-		sg->fi = dst_icg;
-	} else if (xt->dst_inc) {
-		d->ccr |= CCR_DST_AMODE_POSTINC;
-		sg->fi = 0;
-	} else {
-		dev_err(chan->device->dev,
-			"%s: DST constant addressing is not supported\n",
-			__func__);
-		kfree(d);
-		return NULL;
-	}
-
-	d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
-
-	d->csdp = data_type;
-
-	if (dma_omap1()) {
-		d->cicr |= CICR_TOUT_IE;
-		d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
-	} else {
-		d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
-		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
-		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
-	}
-
-	return vchan_tx_prep(&c->vc, &d->vd, flags);
-}
-
-static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
-{
-	struct omap_chan *c = to_omap_dma_chan(chan);
-
-	if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
-	    cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
-		return -EINVAL;
-
-	if (cfg->src_maxburst > chan->device->max_burst ||
-	    cfg->dst_maxburst > chan->device->max_burst)
-		return -EINVAL;
-
-	memcpy(&c->cfg, cfg, sizeof(c->cfg));
-
-	return 0;
-}
-
-static int omap_dma_terminate_all(struct dma_chan *chan)
-{
-	struct omap_chan *c = to_omap_dma_chan(chan);
-	unsigned long flags;
-	LIST_HEAD(head);
-
-	spin_lock_irqsave(&c->vc.lock, flags);
-
-	/*
-	 * Stop DMA activity: we assume the callback will not be called
-	 * after omap_dma_stop() returns (even if it does, it will see
-	 * c->desc is NULL and exit.)
-	 */
-	if (c->desc) {
-		vchan_terminate_vdesc(&c->desc->vd);
-		c->desc = NULL;
-		/* Avoid stopping the dma twice */
-		if (!c->paused)
-			omap_dma_stop(c);
-	}
-
-	c->cyclic = false;
-	c->paused = false;
-
-	vchan_get_all_descriptors(&c->vc, &head);
-	spin_unlock_irqrestore(&c->vc.lock, flags);
-	vchan_dma_desc_free_list(&c->vc, &head);
-
-	return 0;
-}
-
-static void omap_dma_synchronize(struct dma_chan *chan)
-{
-	struct omap_chan *c = to_omap_dma_chan(chan);
-
-	vchan_synchronize(&c->vc);
-}
-
-static int omap_dma_pause(struct dma_chan *chan)
-{
-	struct omap_chan *c = to_omap_dma_chan(chan);
-	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
-	unsigned long flags;
-	int ret = -EINVAL;
-	bool can_pause = false;
-
-	spin_lock_irqsave(&od->irq_lock, flags);
-
-	if (!c->desc)
-		goto out;
-
-	if (c->cyclic)
-		can_pause = true;
-
-	/*
-	 * We do not allow DMA_MEM_TO_DEV transfers to be paused.
-	 * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer:
-	 * "When a channel is disabled during a transfer, the channel undergoes
-	 * an abort, unless it is hardware-source-synchronized …".
-	 * A source-synchronised channel is one where the fetching of data is
-	 * under control of the device. In other words, a device-to-memory
-	 * transfer. So, a destination-synchronised channel (which would be a
-	 * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE
-	 * bit is cleared.
-	 * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel
-	 * aborts immediately after completion of current read/write
-	 * transactions and then the FIFO is cleaned up." The term "cleaned up"
-	 * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE
-	 * are both clear _before_ disabling the channel, otherwise data loss
-	 * will occur.
-	 * The problem is that if the channel is active, then device activity
-	 * can result in DMA activity starting between reading those as both
-	 * clear and the write to DMA_CCR to clear the enable bit hitting the
-	 * hardware. If the DMA hardware can't drain the data in its FIFO to the
-	 * destination, then data loss "might" occur (say if we write to an UART
-	 * and the UART is not accepting any further data).
-	 */
-	else if (c->desc->dir == DMA_DEV_TO_MEM)
-		can_pause = true;
-
-	if (can_pause && !c->paused) {
-		ret = omap_dma_stop(c);
-		if (!ret)
-			c->paused = true;
-	}
-out:
-	spin_unlock_irqrestore(&od->irq_lock, flags);
-
-	return ret;
-}
-
-static int omap_dma_resume(struct dma_chan *chan)
-{
-	struct omap_chan *c = to_omap_dma_chan(chan);
-	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
-	unsigned long flags;
-	int ret = -EINVAL;
-
-	spin_lock_irqsave(&od->irq_lock, flags);
-
-	if (c->paused && c->desc) {
-		mb();
-
-		/* Restore channel link register */
-		omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
-
-		omap_dma_start(c, c->desc);
-		c->paused = false;
-		ret = 0;
-	}
-	spin_unlock_irqrestore(&od->irq_lock, flags);
-
-	return ret;
-}
-
-static int omap_dma_chan_init(struct omap_dmadev *od)
-{
-	struct omap_chan *c;
-
-	c = kzalloc(sizeof(*c), GFP_KERNEL);
-	if (!c)
-		return -ENOMEM;
-
-	c->reg_map = od->reg_map;
-	c->vc.desc_free = omap_dma_desc_free;
-	vchan_init(&c->vc, &od->ddev);
-
-	return 0;
-}
-
-static void omap_dma_free(struct omap_dmadev *od)
-{
-	while (!list_empty(&od->ddev.channels)) {
-		struct omap_chan *c = list_first_entry(&od->ddev.channels,
-			struct omap_chan, vc.chan.device_node);
-
-		list_del(&c->vc.chan.device_node);
-		tasklet_kill(&c->vc.task);
-		kfree(c);
-	}
-}
-
-#define OMAP_DMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
-				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
-				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
-
-static int omap_dma_probe(struct platform_device *pdev)
-{
-	struct omap_dmadev *od;
-	struct resource *res;
-	int rc, i, irq;
-	u32 lch_count;
-
-	od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
-	if (!od)
-		return -ENOMEM;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	od->base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(od->base))
-		return PTR_ERR(od->base);
-
-	od->plat = omap_get_plat_info();
-	if (!od->plat)
-		return -EPROBE_DEFER;
-
-	od->reg_map = od->plat->reg_map;
-
-	dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
-	dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
-	dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
-	dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
-	od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
-	od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
-	od->ddev.device_tx_status = omap_dma_tx_status;
-	od->ddev.device_issue_pending = omap_dma_issue_pending;
-	od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
-	od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
-	od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
-	od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
-	od->ddev.device_config = omap_dma_slave_config;
-	od->ddev.device_pause = omap_dma_pause;
-	od->ddev.device_resume = omap_dma_resume;
-	od->ddev.device_terminate_all = omap_dma_terminate_all;
-	od->ddev.device_synchronize = omap_dma_synchronize;
-	od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
-	od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
-	od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
-	od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
-	od->ddev.max_burst = SZ_16M - 1; /* CCEN: 24bit unsigned */
-	od->ddev.dev = &pdev->dev;
-	INIT_LIST_HEAD(&od->ddev.channels);
-	spin_lock_init(&od->lock);
-	spin_lock_init(&od->irq_lock);
-
-	/* Number of DMA requests */
-	od->dma_requests = OMAP_SDMA_REQUESTS;
-	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
-						      "dma-requests",
-						      &od->dma_requests)) {
-		dev_info(&pdev->dev,
-			 "Missing dma-requests property, using %u.\n",
-			 OMAP_SDMA_REQUESTS);
-	}
-
-	/* Number of available logical channels */
-	if (!pdev->dev.of_node) {
-		lch_count = od->plat->dma_attr->lch_count;
-		if (unlikely(!lch_count))
-			lch_count = OMAP_SDMA_CHANNELS;
-	} else if (of_property_read_u32(pdev->dev.of_node, "dma-channels",
-					&lch_count)) {
-		dev_info(&pdev->dev,
-			 "Missing dma-channels property, using %u.\n",
-			 OMAP_SDMA_CHANNELS);
-		lch_count = OMAP_SDMA_CHANNELS;
-	}
-
-	od->lch_map = devm_kcalloc(&pdev->dev, lch_count, sizeof(*od->lch_map),
-				   GFP_KERNEL);
-	if (!od->lch_map)
-		return -ENOMEM;
-
-	for (i = 0; i < od->dma_requests; i++) {
-		rc = omap_dma_chan_init(od);
-		if (rc) {
-			omap_dma_free(od);
-			return rc;
-		}
-	}
-
-	irq = platform_get_irq(pdev, 1);
-	if (irq <= 0) {
-		dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
-		od->legacy = true;
-	} else {
-		/* Disable all interrupts */
-		od->irq_enable_mask = 0;
-		omap_dma_glbl_write(od, IRQENABLE_L1, 0);
-
-		rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
-				      IRQF_SHARED, "omap-dma-engine", od);
-		if (rc)
-			return rc;
-	}
-
-	if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
-		od->ll123_supported = true;
-
-	od->ddev.filter.map = od->plat->slave_map;
-	od->ddev.filter.mapcnt = od->plat->slavecnt;
-	od->ddev.filter.fn = omap_dma_filter_fn;
-
-	if (od->ll123_supported) {
-		od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
-						&pdev->dev,
-						sizeof(struct omap_type2_desc),
-						4, 0);
-		if (!od->desc_pool) {
-			dev_err(&pdev->dev,
-				"unable to allocate descriptor pool\n");
-			od->ll123_supported = false;
-		}
-	}
-
-	rc = dma_async_device_register(&od->ddev);
-	if (rc) {
-		pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
-			rc);
-		omap_dma_free(od);
-		return rc;
-	}
-
-	platform_set_drvdata(pdev, od);
-
-	if (pdev->dev.of_node) {
-		omap_dma_info.dma_cap = od->ddev.cap_mask;
-
-		/* Device-tree DMA controller registration */
-		rc = of_dma_controller_register(pdev->dev.of_node,
-				of_dma_simple_xlate, &omap_dma_info);
-		if (rc) {
-			pr_warn("OMAP-DMA: failed to register DMA controller\n");
-			dma_async_device_unregister(&od->ddev);
-			omap_dma_free(od);
-		}
-	}
-
-	dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
-		 od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
-
-	return rc;
-}
-
-static int omap_dma_remove(struct platform_device *pdev)
-{
-	struct omap_dmadev *od = platform_get_drvdata(pdev);
-	int irq;
-
-	if (pdev->dev.of_node)
-		of_dma_controller_free(pdev->dev.of_node);
-
-	irq = platform_get_irq(pdev, 1);
-	devm_free_irq(&pdev->dev, irq, od);
-
-	dma_async_device_unregister(&od->ddev);
-
-	if (!od->legacy) {
-		/* Disable all interrupts */
-		omap_dma_glbl_write(od, IRQENABLE_L0, 0);
-	}
-
-	if (od->ll123_supported)
-		dma_pool_destroy(od->desc_pool);
-
-	omap_dma_free(od);
-
-	return 0;
-}
-
-static const struct of_device_id omap_dma_match[] = {
-	{ .compatible = "ti,omap2420-sdma", },
-	{ .compatible = "ti,omap2430-sdma", },
-	{ .compatible = "ti,omap3430-sdma", },
-	{ .compatible = "ti,omap3630-sdma", },
-	{ .compatible = "ti,omap4430-sdma", },
-	{},
-};
-MODULE_DEVICE_TABLE(of, omap_dma_match);
-
-static struct platform_driver omap_dma_driver = {
-	.probe	= omap_dma_probe,
-	.remove	= omap_dma_remove,
-	.driver = {
-		.name = "omap-dma-engine",
-		.of_match_table = of_match_ptr(omap_dma_match),
-	},
-};
-
-bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
-{
-	if (chan->device->dev->driver == &omap_dma_driver.driver) {
-		struct omap_dmadev *od = to_omap_dma_dev(chan->device);
-		struct omap_chan *c = to_omap_dma_chan(chan);
-		unsigned req = *(unsigned *)param;
-
-		if (req <= od->dma_requests) {
-			c->dma_sig = req;
-			return true;
-		}
-	}
-	return false;
-}
-EXPORT_SYMBOL_GPL(omap_dma_filter_fn);
-
-static int omap_dma_init(void)
-{
-	return platform_driver_register(&omap_dma_driver);
-}
-subsys_initcall(omap_dma_init);
-
-static void __exit omap_dma_exit(void)
-{
-	platform_driver_unregister(&omap_dma_driver);
-}
-module_exit(omap_dma_exit);
-
-MODULE_AUTHOR("Russell King");
-MODULE_LICENSE("GPL");
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index de1fd59fe136..f802bd3b0481 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -27,6 +27,7 @@
 #include <linux/of_dma.h>
 #include <linux/err.h>
 #include <linux/pm_runtime.h>
+#include <linux/bug.h>
 
 #include "dmaengine.h"
 #define PL330_MAX_CHAN		8
@@ -1094,51 +1095,96 @@ static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
 	return off;
 }
 
-static inline int _ldst_devtomem(struct pl330_dmac *pl330, unsigned dry_run,
-				 u8 buf[], const struct _xfer_spec *pxs,
-				 int cyc)
+static u32 _emit_load(unsigned int dry_run, u8 buf[],
+	enum pl330_cond cond, enum dma_transfer_direction direction,
+	u8 peri)
 {
 	int off = 0;
-	enum pl330_cond cond;
 
-	if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
-		cond = BURST;
-	else
-		cond = SINGLE;
+	switch (direction) {
+	case DMA_MEM_TO_MEM:
+		/* fall through */
+	case DMA_MEM_TO_DEV:
+		off += _emit_LD(dry_run, &buf[off], cond);
+		break;
 
-	while (cyc--) {
-		off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
-		off += _emit_LDP(dry_run, &buf[off], cond, pxs->desc->peri);
-		off += _emit_ST(dry_run, &buf[off], ALWAYS);
+	case DMA_DEV_TO_MEM:
+		if (cond == ALWAYS) {
+			off += _emit_LDP(dry_run, &buf[off], SINGLE,
+				peri);
+			off += _emit_LDP(dry_run, &buf[off], BURST,
+				peri);
+		} else {
+			off += _emit_LDP(dry_run, &buf[off], cond,
+				peri);
+		}
+		break;
+
+	default:
+		/* this code should be unreachable */
+		WARN_ON(1);
+		break;
+	}
+
+	return off;
+}
+
+static inline u32 _emit_store(unsigned int dry_run, u8 buf[],
+	enum pl330_cond cond, enum dma_transfer_direction direction,
+	u8 peri)
+{
+	int off = 0;
+
+	switch (direction) {
+	case DMA_MEM_TO_MEM:
+		/* fall through */
+	case DMA_DEV_TO_MEM:
+		off += _emit_ST(dry_run, &buf[off], cond);
+		break;
 
-		if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
-			off += _emit_FLUSHP(dry_run, &buf[off],
-					    pxs->desc->peri);
+	case DMA_MEM_TO_DEV:
+		if (cond == ALWAYS) {
+			off += _emit_STP(dry_run, &buf[off], SINGLE,
+				peri);
+			off += _emit_STP(dry_run, &buf[off], BURST,
+				peri);
+		} else {
+			off += _emit_STP(dry_run, &buf[off], cond,
+				peri);
+		}
+		break;
+
+	default:
+		/* this code should be unreachable */
+		WARN_ON(1);
+		break;
 	}
 
 	return off;
 }
 
-static inline int _ldst_memtodev(struct pl330_dmac *pl330,
+static inline int _ldst_peripheral(struct pl330_dmac *pl330,
 				 unsigned dry_run, u8 buf[],
-				 const struct _xfer_spec *pxs, int cyc)
+				 const struct _xfer_spec *pxs, int cyc,
+				 enum pl330_cond cond)
 {
 	int off = 0;
-	enum pl330_cond cond;
 
 	if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
 		cond = BURST;
-	else
-		cond = SINGLE;
 
+	/*
+	 * do FLUSHP at beginning to clear any stale dma requests before the
+	 * first WFP.
+	 */
+	if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
+		off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
 	while (cyc--) {
 		off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
-		off += _emit_LD(dry_run, &buf[off], ALWAYS);
-		off += _emit_STP(dry_run, &buf[off], cond, pxs->desc->peri);
-
-		if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
-			off += _emit_FLUSHP(dry_run, &buf[off],
-					    pxs->desc->peri);
+		off += _emit_load(dry_run, &buf[off], cond, pxs->desc->rqtype,
+			pxs->desc->peri);
+		off += _emit_store(dry_run, &buf[off], cond, pxs->desc->rqtype,
+			pxs->desc->peri);
 	}
 
 	return off;
@@ -1148,19 +1194,65 @@ static int _bursts(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[],
 		const struct _xfer_spec *pxs, int cyc)
 {
 	int off = 0;
+	enum pl330_cond cond = BRST_LEN(pxs->ccr) > 1 ? BURST : SINGLE;
 
 	switch (pxs->desc->rqtype) {
 	case DMA_MEM_TO_DEV:
-		off += _ldst_memtodev(pl330, dry_run, &buf[off], pxs, cyc);
-		break;
+		/* fall through */
 	case DMA_DEV_TO_MEM:
-		off += _ldst_devtomem(pl330, dry_run, &buf[off], pxs, cyc);
+		off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs, cyc,
+			cond);
 		break;
+
 	case DMA_MEM_TO_MEM:
 		off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
 		break;
+
 	default:
-		off += 0x40000000; /* Scare off the Client */
+		/* this code should be unreachable */
+		WARN_ON(1);
+		break;
+	}
+
+	return off;
+}
+
+/*
+ * transfer dregs with single transfers to peripheral, or a reduced size burst
+ * for mem-to-mem.
+ */
+static int _dregs(struct pl330_dmac *pl330, unsigned int dry_run, u8 buf[],
+		const struct _xfer_spec *pxs, int transfer_length)
+{
+	int off = 0;
+	int dregs_ccr;
+
+	if (transfer_length == 0)
+		return off;
+
+	switch (pxs->desc->rqtype) {
+	case DMA_MEM_TO_DEV:
+		/* fall through */
+	case DMA_DEV_TO_MEM:
+		off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs,
+			transfer_length, SINGLE);
+		break;
+
+	case DMA_MEM_TO_MEM:
+		dregs_ccr = pxs->ccr;
+		dregs_ccr &= ~((0xf << CC_SRCBRSTLEN_SHFT) |
+			(0xf << CC_DSTBRSTLEN_SHFT));
+		dregs_ccr |= (((transfer_length - 1) & 0xf) <<
+			CC_SRCBRSTLEN_SHFT);
+		dregs_ccr |= (((transfer_length - 1) & 0xf) <<
+			CC_DSTBRSTLEN_SHFT);
+		off += _emit_MOV(dry_run, &buf[off], CCR, dregs_ccr);
+		off += _ldst_memtomem(dry_run, &buf[off], pxs, 1);
+		break;
+
+	default:
+		/* this code should be unreachable */
+		WARN_ON(1);
 		break;
 	}
 
@@ -1256,6 +1348,8 @@ static inline int _setup_loops(struct pl330_dmac *pl330,
 	struct pl330_xfer *x = &pxs->desc->px;
 	u32 ccr = pxs->ccr;
 	unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
+	int num_dregs = (x->bytes - BURST_TO_BYTE(bursts, ccr)) /
+		BRST_SIZE(ccr);
 	int off = 0;
 
 	while (bursts) {
@@ -1263,6 +1357,7 @@ static inline int _setup_loops(struct pl330_dmac *pl330,
 		off += _loop(pl330, dry_run, &buf[off], &c, pxs);
 		bursts -= c;
 	}
+	off += _dregs(pl330, dry_run, &buf[off], pxs, num_dregs);
 
 	return off;
 }
@@ -1294,7 +1389,6 @@ static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run,
 		      struct _xfer_spec *pxs)
 {
 	struct _pl330_req *req = &thrd->req[index];
-	struct pl330_xfer *x;
 	u8 *buf = req->mc_cpu;
 	int off = 0;
 
@@ -1303,11 +1397,6 @@ static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run,
 	/* DMAMOV CCR, ccr */
 	off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
 
-	x = &pxs->desc->px;
-	/* Error if xfer length is not aligned at burst size */
-	if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr)))
-		return -EINVAL;
-
 	off += _setup_xfer(pl330, dry_run, &buf[off], pxs);
 
 	/* DMASEV peripheral/event */
@@ -1365,6 +1454,20 @@ static int pl330_submit_req(struct pl330_thread *thrd,
 	u32 ccr;
 	int ret = 0;
 
+	switch (desc->rqtype) {
+	case DMA_MEM_TO_DEV:
+		break;
+
+	case DMA_DEV_TO_MEM:
+		break;
+
+	case DMA_MEM_TO_MEM:
+		break;
+
+	default:
+		return -ENOTSUPP;
+	}
+
 	if (pl330->state == DYING
 		|| pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
 		dev_info(thrd->dmac->ddma.dev, "%s:%d\n",
@@ -2106,6 +2209,18 @@ static bool pl330_prep_slave_fifo(struct dma_pl330_chan *pch,
 	return true;
 }
 
+static int fixup_burst_len(int max_burst_len, int quirks)
+{
+	if (quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
+		return 1;
+	else if (max_burst_len > PL330_MAX_BURST)
+		return PL330_MAX_BURST;
+	else if (max_burst_len < 1)
+		return 1;
+	else
+		return max_burst_len;
+}
+
 static int pl330_config(struct dma_chan *chan,
 			struct dma_slave_config *slave_config)
 {
@@ -2117,15 +2232,15 @@ static int pl330_config(struct dma_chan *chan,
 			pch->fifo_addr = slave_config->dst_addr;
 		if (slave_config->dst_addr_width)
 			pch->burst_sz = __ffs(slave_config->dst_addr_width);
-		if (slave_config->dst_maxburst)
-			pch->burst_len = slave_config->dst_maxburst;
+		pch->burst_len = fixup_burst_len(slave_config->dst_maxburst,
+			pch->dmac->quirks);
 	} else if (slave_config->direction == DMA_DEV_TO_MEM) {
 		if (slave_config->src_addr)
 			pch->fifo_addr = slave_config->src_addr;
 		if (slave_config->src_addr_width)
 			pch->burst_sz = __ffs(slave_config->src_addr_width);
-		if (slave_config->src_maxburst)
-			pch->burst_len = slave_config->src_maxburst;
+		pch->burst_len = fixup_burst_len(slave_config->src_maxburst,
+			pch->dmac->quirks);
 	}
 
 	return 0;
@@ -2175,33 +2290,6 @@ static int pl330_terminate_all(struct dma_chan *chan)
 	return 0;
 }
 
-/*
- * We don't support DMA_RESUME command because of hardware
- * limitations, so after pausing the channel we cannot restore
- * it to active state. We have to terminate channel and setup
- * DMA transfer again. This pause feature was implemented to
- * allow safely read residue before channel termination.
- */
-static int pl330_pause(struct dma_chan *chan)
-{
-	struct dma_pl330_chan *pch = to_pchan(chan);
-	struct pl330_dmac *pl330 = pch->dmac;
-	unsigned long flags;
-
-	pm_runtime_get_sync(pl330->ddma.dev);
-	spin_lock_irqsave(&pch->lock, flags);
-
-	spin_lock(&pl330->lock);
-	_stop(pch->thread);
-	spin_unlock(&pl330->lock);
-
-	spin_unlock_irqrestore(&pch->lock, flags);
-	pm_runtime_mark_last_busy(pl330->ddma.dev);
-	pm_runtime_put_autosuspend(pl330->ddma.dev);
-
-	return 0;
-}
-
 static void pl330_free_chan_resources(struct dma_chan *chan)
 {
 	struct dma_pl330_chan *pch = to_pchan(chan);
@@ -2519,14 +2607,8 @@ static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
 	burst_len >>= desc->rqcfg.brst_size;
 
 	/* src/dst_burst_len can't be more than 16 */
-	if (burst_len > 16)
-		burst_len = 16;
-
-	while (burst_len > 1) {
-		if (!(len % (burst_len << desc->rqcfg.brst_size)))
-			break;
-		burst_len--;
-	}
+	if (burst_len > PL330_MAX_BURST)
+		burst_len = PL330_MAX_BURST;
 
 	return burst_len;
 }
@@ -2598,7 +2680,7 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
 
 		desc->rqtype = direction;
 		desc->rqcfg.brst_size = pch->burst_sz;
-		desc->rqcfg.brst_len = 1;
+		desc->rqcfg.brst_len = pch->burst_len;
 		desc->bytes_requested = period_len;
 		fill_px(&desc->px, dst, src, period_len);
 
@@ -2743,7 +2825,7 @@ pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		}
 
 		desc->rqcfg.brst_size = pch->burst_sz;
-		desc->rqcfg.brst_len = 1;
+		desc->rqcfg.brst_len = pch->burst_len;
 		desc->rqtype = direction;
 		desc->bytes_requested = sg_dma_len(sg);
 	}
@@ -2918,7 +3000,6 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
 	pd->device_tx_status = pl330_tx_status;
 	pd->device_prep_slave_sg = pl330_prep_slave_sg;
 	pd->device_config = pl330_config;
-	pd->device_pause = pl330_pause;
 	pd->device_terminate_all = pl330_terminate_all;
 	pd->device_issue_pending = pl330_issue_pending;
 	pd->src_addr_widths = PL330_DMA_BUSWIDTHS;
diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c
index 4a828c18099a..1617715aa6e0 100644
--- a/drivers/dma/qcom/bam_dma.c
+++ b/drivers/dma/qcom/bam_dma.c
@@ -451,6 +451,7 @@ static void bam_reset_channel(struct bam_chan *bchan)
 /**
  * bam_chan_init_hw - Initialize channel hardware
  * @bchan: bam channel
+ * @dir: DMA transfer direction
  *
  * This function resets and initializes the BAM channel
  */
@@ -673,7 +674,7 @@ static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan,
 				remainder = 0;
 			}
 
-			async_desc->length += desc->size;
+			async_desc->length += le16_to_cpu(desc->size);
 			desc++;
 		} while (remainder > 0);
 	}
@@ -687,7 +688,7 @@ static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan,
 
 /**
  * bam_dma_terminate_all - terminate all transactions on a channel
- * @bchan: bam dma channel
+ * @chan: bam dma channel
  *
  * Dequeues and frees all transactions
  * No callbacks are done
@@ -918,7 +919,8 @@ static enum dma_status bam_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 				continue;
 
 			for (i = 0; i < async_desc->num_desc; i++)
-				residue += async_desc->curr_desc[i].size;
+				residue += le16_to_cpu(
+						async_desc->curr_desc[i].size);
 		}
 	}
 
@@ -958,7 +960,7 @@ static void bam_apply_new_config(struct bam_chan *bchan,
 
 /**
  * bam_start_dma - start next transaction
- * @bchan - bam dma channel
+ * @bchan: bam dma channel
  */
 static void bam_start_dma(struct bam_chan *bchan)
 {
diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c
index 963cc5228d05..43d4b00b8138 100644
--- a/drivers/dma/qcom/hidma.c
+++ b/drivers/dma/qcom/hidma.c
@@ -616,8 +616,7 @@ static irqreturn_t hidma_chirq_handler_msi(int chirq, void *arg)
 static ssize_t hidma_show_values(struct device *dev,
 				 struct device_attribute *attr, char *buf)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct hidma_dev *mdev = platform_get_drvdata(pdev);
+	struct hidma_dev *mdev = dev_get_drvdata(dev);
 
 	buf[0] = 0;
 
diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c
index 000c7019ca7d..d64edeb6771a 100644
--- a/drivers/dma/qcom/hidma_mgmt.c
+++ b/drivers/dma/qcom/hidma_mgmt.c
@@ -398,7 +398,7 @@ static int __init hidma_mgmt_of_populate_channels(struct device_node *np)
 		}
 		of_node_get(child);
 		new_pdev->dev.of_node = child;
-		of_dma_configure(&new_pdev->dev, child);
+		of_dma_configure(&new_pdev->dev, child, true);
 		/*
 		 * It is assumed that calling of_msi_configure is safe on
 		 * platforms with or without MSI support.
diff --git a/drivers/dma/qcom/hidma_mgmt_sys.c b/drivers/dma/qcom/hidma_mgmt_sys.c
index d61f1068a34b..cbb89eafd844 100644
--- a/drivers/dma/qcom/hidma_mgmt_sys.c
+++ b/drivers/dma/qcom/hidma_mgmt_sys.c
@@ -107,8 +107,7 @@ static struct hidma_mgmt_fileinfo hidma_mgmt_files[] = {
 static ssize_t show_values(struct device *dev, struct device_attribute *attr,
 			   char *buf)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct hidma_mgmt_dev *mdev = platform_get_drvdata(pdev);
+	struct hidma_mgmt_dev *mdev = dev_get_drvdata(dev);
 	unsigned int i;
 
 	buf[0] = 0;
@@ -125,8 +124,7 @@ static ssize_t show_values(struct device *dev, struct device_attribute *attr,
 static ssize_t set_values(struct device *dev, struct device_attribute *attr,
 			  const char *buf, size_t count)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct hidma_mgmt_dev *mdev = platform_get_drvdata(pdev);
+	struct hidma_mgmt_dev *mdev = dev_get_drvdata(dev);
 	unsigned long tmp;
 	unsigned int i;
 	int rc;
diff --git a/drivers/dma/sh/shdmac.c b/drivers/dma/sh/shdmac.c
index c94ffab0d25c..04a74e0a95b7 100644
--- a/drivers/dma/sh/shdmac.c
+++ b/drivers/dma/sh/shdmac.c
@@ -443,7 +443,6 @@ static bool sh_dmae_reset(struct sh_dmae_device *shdev)
 	return ret;
 }
 
-#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE)
 static irqreturn_t sh_dmae_err(int irq, void *data)
 {
 	struct sh_dmae_device *shdev = data;
@@ -454,7 +453,6 @@ static irqreturn_t sh_dmae_err(int irq, void *data)
 	sh_dmae_reset(shdev);
 	return IRQ_HANDLED;
 }
-#endif
 
 static bool sh_dmae_desc_completed(struct shdma_chan *schan,
 				   struct shdma_desc *sdesc)
@@ -686,11 +684,8 @@ static int sh_dmae_probe(struct platform_device *pdev)
 	const struct sh_dmae_pdata *pdata;
 	unsigned long chan_flag[SH_DMAE_MAX_CHANNELS] = {};
 	int chan_irq[SH_DMAE_MAX_CHANNELS];
-#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE)
 	unsigned long irqflags = 0;
-	int errirq;
-#endif
-	int err, i, irq_cnt = 0, irqres = 0, irq_cap = 0;
+	int err, errirq, i, irq_cnt = 0, irqres = 0, irq_cap = 0;
 	struct sh_dmae_device *shdev;
 	struct dma_device *dma_dev;
 	struct resource *chan, *dmars, *errirq_res, *chanirq_res;
@@ -792,33 +787,32 @@ static int sh_dmae_probe(struct platform_device *pdev)
 	if (err)
 		goto rst_err;
 
-#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE)
-	chanirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 1);
+	if (IS_ENABLED(CONFIG_CPU_SH4) || IS_ENABLED(CONFIG_ARCH_RENESAS)) {
+		chanirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 1);
 
-	if (!chanirq_res)
-		chanirq_res = errirq_res;
-	else
-		irqres++;
+		if (!chanirq_res)
+			chanirq_res = errirq_res;
+		else
+			irqres++;
 
-	if (chanirq_res == errirq_res ||
-	    (errirq_res->flags & IORESOURCE_BITS) == IORESOURCE_IRQ_SHAREABLE)
-		irqflags = IRQF_SHARED;
+		if (chanirq_res == errirq_res ||
+		    (errirq_res->flags & IORESOURCE_BITS) == IORESOURCE_IRQ_SHAREABLE)
+			irqflags = IRQF_SHARED;
 
-	errirq = errirq_res->start;
+		errirq = errirq_res->start;
 
-	err = devm_request_irq(&pdev->dev, errirq, sh_dmae_err, irqflags,
-			       "DMAC Address Error", shdev);
-	if (err) {
-		dev_err(&pdev->dev,
-			"DMA failed requesting irq #%d, error %d\n",
-			errirq, err);
-		goto eirq_err;
+		err = devm_request_irq(&pdev->dev, errirq, sh_dmae_err,
+				       irqflags, "DMAC Address Error", shdev);
+		if (err) {
+			dev_err(&pdev->dev,
+				"DMA failed requesting irq #%d, error %d\n",
+				errirq, err);
+			goto eirq_err;
+		}
+	} else {
+		chanirq_res = errirq_res;
 	}
 
-#else
-	chanirq_res = errirq_res;
-#endif /* CONFIG_CPU_SH4 || CONFIG_ARCH_SHMOBILE */
-
 	if (chanirq_res->start == chanirq_res->end &&
 	    !platform_get_resource(pdev, IORESOURCE_IRQ, 1)) {
 		/* Special case - all multiplexed */
@@ -884,9 +878,7 @@ static int sh_dmae_probe(struct platform_device *pdev)
 chan_probe_err:
 	sh_dmae_chan_remove(shdev);
 
-#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE)
 eirq_err:
-#endif
 rst_err:
 	spin_lock_irq(&sh_dmae_lock);
 	list_del_rcu(&shdev->node);
diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c
index b106e8a60af6..ab69f835ea02 100644
--- a/drivers/dma/sprd-dma.c
+++ b/drivers/dma/sprd-dma.c
@@ -6,6 +6,7 @@
 
 #include <linux/clk.h>
 #include <linux/dma-mapping.h>
+#include <linux/dma/sprd-dma.h>
 #include <linux/errno.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -116,57 +117,21 @@
 #define SPRD_DMA_SRC_TRSF_STEP_OFFSET	0
 #define SPRD_DMA_TRSF_STEP_MASK		GENMASK(15, 0)
 
+/* define the DMA transfer step type */
+#define SPRD_DMA_NONE_STEP		0
+#define SPRD_DMA_BYTE_STEP		1
+#define SPRD_DMA_SHORT_STEP		2
+#define SPRD_DMA_WORD_STEP		4
+#define SPRD_DMA_DWORD_STEP		8
+
 #define SPRD_DMA_SOFTWARE_UID		0
 
-/*
- * enum sprd_dma_req_mode: define the DMA request mode
- * @SPRD_DMA_FRAG_REQ: fragment request mode
- * @SPRD_DMA_BLK_REQ: block request mode
- * @SPRD_DMA_TRANS_REQ: transaction request mode
- * @SPRD_DMA_LIST_REQ: link-list request mode
- *
- * We have 4 types request mode: fragment mode, block mode, transaction mode
- * and linklist mode. One transaction can contain several blocks, one block can
- * contain several fragments. Link-list mode means we can save several DMA
- * configuration into one reserved memory, then DMA can fetch each DMA
- * configuration automatically to start transfer.
- */
-enum sprd_dma_req_mode {
-	SPRD_DMA_FRAG_REQ,
-	SPRD_DMA_BLK_REQ,
-	SPRD_DMA_TRANS_REQ,
-	SPRD_DMA_LIST_REQ,
-};
-
-/*
- * enum sprd_dma_int_type: define the DMA interrupt type
- * @SPRD_DMA_NO_INT: do not need generate DMA interrupts.
- * @SPRD_DMA_FRAG_INT: fragment done interrupt when one fragment request
- * is done.
- * @SPRD_DMA_BLK_INT: block done interrupt when one block request is done.
- * @SPRD_DMA_BLK_FRAG_INT: block and fragment interrupt when one fragment
- * or one block request is done.
- * @SPRD_DMA_TRANS_INT: tansaction done interrupt when one transaction
- * request is done.
- * @SPRD_DMA_TRANS_FRAG_INT: transaction and fragment interrupt when one
- * transaction request or fragment request is done.
- * @SPRD_DMA_TRANS_BLK_INT: transaction and block interrupt when one
- * transaction request or block request is done.
- * @SPRD_DMA_LIST_INT: link-list done interrupt when one link-list request
- * is done.
- * @SPRD_DMA_CFGERR_INT: configure error interrupt when configuration is
- * incorrect.
- */
-enum sprd_dma_int_type {
-	SPRD_DMA_NO_INT,
-	SPRD_DMA_FRAG_INT,
-	SPRD_DMA_BLK_INT,
-	SPRD_DMA_BLK_FRAG_INT,
-	SPRD_DMA_TRANS_INT,
-	SPRD_DMA_TRANS_FRAG_INT,
-	SPRD_DMA_TRANS_BLK_INT,
-	SPRD_DMA_LIST_INT,
-	SPRD_DMA_CFGERR_INT,
+/* dma data width values */
+enum sprd_dma_datawidth {
+	SPRD_DMA_DATAWIDTH_1_BYTE,
+	SPRD_DMA_DATAWIDTH_2_BYTES,
+	SPRD_DMA_DATAWIDTH_4_BYTES,
+	SPRD_DMA_DATAWIDTH_8_BYTES,
 };
 
 /* dma channel hardware configuration */
@@ -199,6 +164,7 @@ struct sprd_dma_desc {
 struct sprd_dma_chn {
 	struct virt_dma_chan	vc;
 	void __iomem		*chn_base;
+	struct dma_slave_config	slave_cfg;
 	u32			chn_num;
 	u32			dev_id;
 	struct sprd_dma_desc	*cur_desc;
@@ -587,52 +553,97 @@ static void sprd_dma_issue_pending(struct dma_chan *chan)
 	spin_unlock_irqrestore(&schan->vc.lock, flags);
 }
 
-static int sprd_dma_config(struct dma_chan *chan, struct sprd_dma_desc *sdesc,
-			   dma_addr_t dest, dma_addr_t src, size_t len)
+static int sprd_dma_get_datawidth(enum dma_slave_buswidth buswidth)
+{
+	switch (buswidth) {
+	case DMA_SLAVE_BUSWIDTH_1_BYTE:
+	case DMA_SLAVE_BUSWIDTH_2_BYTES:
+	case DMA_SLAVE_BUSWIDTH_4_BYTES:
+	case DMA_SLAVE_BUSWIDTH_8_BYTES:
+		return ffs(buswidth) - 1;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int sprd_dma_get_step(enum dma_slave_buswidth buswidth)
+{
+	switch (buswidth) {
+	case DMA_SLAVE_BUSWIDTH_1_BYTE:
+	case DMA_SLAVE_BUSWIDTH_2_BYTES:
+	case DMA_SLAVE_BUSWIDTH_4_BYTES:
+	case DMA_SLAVE_BUSWIDTH_8_BYTES:
+		return buswidth;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int sprd_dma_fill_desc(struct dma_chan *chan,
+			      struct sprd_dma_desc *sdesc,
+			      dma_addr_t src, dma_addr_t dst, u32 len,
+			      enum dma_transfer_direction dir,
+			      unsigned long flags,
+			      struct dma_slave_config *slave_cfg)
 {
 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan);
+	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 	struct sprd_dma_chn_hw *hw = &sdesc->chn_hw;
-	u32 datawidth, src_step, des_step, fragment_len;
-	u32 block_len, req_mode, irq_mode, transcation_len;
-	u32 fix_mode = 0, fix_en = 0;
-
-	if (IS_ALIGNED(len, 4)) {
-		datawidth = 2;
-		src_step = 4;
-		des_step = 4;
-	} else if (IS_ALIGNED(len, 2)) {
-		datawidth = 1;
-		src_step = 2;
-		des_step = 2;
+	u32 req_mode = (flags >> SPRD_DMA_REQ_SHIFT) & SPRD_DMA_REQ_MODE_MASK;
+	u32 int_mode = flags & SPRD_DMA_INT_MASK;
+	int src_datawidth, dst_datawidth, src_step, dst_step;
+	u32 temp, fix_mode = 0, fix_en = 0;
+
+	if (dir == DMA_MEM_TO_DEV) {
+		src_step = sprd_dma_get_step(slave_cfg->src_addr_width);
+		if (src_step < 0) {
+			dev_err(sdev->dma_dev.dev, "invalid source step\n");
+			return src_step;
+		}
+		dst_step = SPRD_DMA_NONE_STEP;
 	} else {
-		datawidth = 0;
-		src_step = 1;
-		des_step = 1;
+		dst_step = sprd_dma_get_step(slave_cfg->dst_addr_width);
+		if (dst_step < 0) {
+			dev_err(sdev->dma_dev.dev, "invalid destination step\n");
+			return dst_step;
+		}
+		src_step = SPRD_DMA_NONE_STEP;
 	}
 
-	fragment_len = SPRD_DMA_MEMCPY_MIN_SIZE;
-	if (len <= SPRD_DMA_BLK_LEN_MASK) {
-		block_len = len;
-		transcation_len = 0;
-		req_mode = SPRD_DMA_BLK_REQ;
-		irq_mode = SPRD_DMA_BLK_INT;
-	} else {
-		block_len = SPRD_DMA_MEMCPY_MIN_SIZE;
-		transcation_len = len;
-		req_mode = SPRD_DMA_TRANS_REQ;
-		irq_mode = SPRD_DMA_TRANS_INT;
+	src_datawidth = sprd_dma_get_datawidth(slave_cfg->src_addr_width);
+	if (src_datawidth < 0) {
+		dev_err(sdev->dma_dev.dev, "invalid source datawidth\n");
+		return src_datawidth;
 	}
 
+	dst_datawidth = sprd_dma_get_datawidth(slave_cfg->dst_addr_width);
+	if (dst_datawidth < 0) {
+		dev_err(sdev->dma_dev.dev, "invalid destination datawidth\n");
+		return dst_datawidth;
+	}
+
+	if (slave_cfg->slave_id)
+		schan->dev_id = slave_cfg->slave_id;
+
 	hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET;
-	hw->wrap_ptr = (u32)((src >> SPRD_DMA_HIGH_ADDR_OFFSET) &
-			     SPRD_DMA_HIGH_ADDR_MASK);
-	hw->wrap_to = (u32)((dest >> SPRD_DMA_HIGH_ADDR_OFFSET) &
-			    SPRD_DMA_HIGH_ADDR_MASK);
 
-	hw->src_addr = (u32)(src & SPRD_DMA_LOW_ADDR_MASK);
-	hw->des_addr = (u32)(dest & SPRD_DMA_LOW_ADDR_MASK);
+	/*
+	 * wrap_ptr and wrap_to will save the high 4 bits source address and
+	 * destination address.
+	 */
+	hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK;
+	hw->wrap_to = (dst >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK;
+	hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK;
+	hw->des_addr = dst & SPRD_DMA_LOW_ADDR_MASK;
 
-	if ((src_step != 0 && des_step != 0) || (src_step | des_step) == 0) {
+	/*
+	 * If the src step and dst step both are 0 or both are not 0, that means
+	 * we can not enable the fix mode. If one is 0 and another one is not,
+	 * we can enable the fix mode.
+	 */
+	if ((src_step != 0 && dst_step != 0) || (src_step | dst_step) == 0) {
 		fix_en = 0;
 	} else {
 		fix_en = 1;
@@ -642,87 +653,119 @@ static int sprd_dma_config(struct dma_chan *chan, struct sprd_dma_desc *sdesc,
 			fix_mode = 0;
 	}
 
-	hw->frg_len = datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET |
-		datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET |
-		req_mode << SPRD_DMA_REQ_MODE_OFFSET |
-		fix_mode << SPRD_DMA_FIX_SEL_OFFSET |
-		fix_en << SPRD_DMA_FIX_EN_OFFSET |
-		(fragment_len & SPRD_DMA_FRG_LEN_MASK);
-	hw->blk_len = block_len & SPRD_DMA_BLK_LEN_MASK;
-
-	hw->intc = SPRD_DMA_CFG_ERR_INT_EN;
-
-	switch (irq_mode) {
-	case SPRD_DMA_NO_INT:
-		break;
-
-	case SPRD_DMA_FRAG_INT:
-		hw->intc |= SPRD_DMA_FRAG_INT_EN;
-		break;
+	hw->intc = int_mode | SPRD_DMA_CFG_ERR_INT_EN;
 
-	case SPRD_DMA_BLK_INT:
-		hw->intc |= SPRD_DMA_BLK_INT_EN;
-		break;
+	temp = src_datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET;
+	temp |= dst_datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET;
+	temp |= req_mode << SPRD_DMA_REQ_MODE_OFFSET;
+	temp |= fix_mode << SPRD_DMA_FIX_SEL_OFFSET;
+	temp |= fix_en << SPRD_DMA_FIX_EN_OFFSET;
+	temp |= slave_cfg->src_maxburst & SPRD_DMA_FRG_LEN_MASK;
+	hw->frg_len = temp;
 
-	case SPRD_DMA_BLK_FRAG_INT:
-		hw->intc |= SPRD_DMA_BLK_INT_EN | SPRD_DMA_FRAG_INT_EN;
-		break;
+	hw->blk_len = len & SPRD_DMA_BLK_LEN_MASK;
+	hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK;
 
-	case SPRD_DMA_TRANS_INT:
-		hw->intc |= SPRD_DMA_TRANS_INT_EN;
-		break;
+	temp = (dst_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET;
+	temp |= (src_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET;
+	hw->trsf_step = temp;
 
-	case SPRD_DMA_TRANS_FRAG_INT:
-		hw->intc |= SPRD_DMA_TRANS_INT_EN | SPRD_DMA_FRAG_INT_EN;
-		break;
+	hw->frg_step = 0;
+	hw->src_blk_step = 0;
+	hw->des_blk_step = 0;
+	return 0;
+}
 
-	case SPRD_DMA_TRANS_BLK_INT:
-		hw->intc |= SPRD_DMA_TRANS_INT_EN | SPRD_DMA_BLK_INT_EN;
-		break;
+static struct dma_async_tx_descriptor *
+sprd_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
+			 size_t len, unsigned long flags)
+{
+	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
+	struct sprd_dma_desc *sdesc;
+	struct sprd_dma_chn_hw *hw;
+	enum sprd_dma_datawidth datawidth;
+	u32 step, temp;
 
-	case SPRD_DMA_LIST_INT:
-		hw->intc |= SPRD_DMA_LIST_INT_EN;
-		break;
+	sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT);
+	if (!sdesc)
+		return NULL;
 
-	case SPRD_DMA_CFGERR_INT:
-		hw->intc |= SPRD_DMA_CFG_ERR_INT_EN;
-		break;
+	hw = &sdesc->chn_hw;
 
-	default:
-		dev_err(sdev->dma_dev.dev, "invalid irq mode\n");
-		return -EINVAL;
+	hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET;
+	hw->intc = SPRD_DMA_TRANS_INT | SPRD_DMA_CFG_ERR_INT_EN;
+	hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK;
+	hw->des_addr = dest & SPRD_DMA_LOW_ADDR_MASK;
+	hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) &
+		SPRD_DMA_HIGH_ADDR_MASK;
+	hw->wrap_to = (dest >> SPRD_DMA_HIGH_ADDR_OFFSET) &
+		SPRD_DMA_HIGH_ADDR_MASK;
+
+	if (IS_ALIGNED(len, 8)) {
+		datawidth = SPRD_DMA_DATAWIDTH_8_BYTES;
+		step = SPRD_DMA_DWORD_STEP;
+	} else if (IS_ALIGNED(len, 4)) {
+		datawidth = SPRD_DMA_DATAWIDTH_4_BYTES;
+		step = SPRD_DMA_WORD_STEP;
+	} else if (IS_ALIGNED(len, 2)) {
+		datawidth = SPRD_DMA_DATAWIDTH_2_BYTES;
+		step = SPRD_DMA_SHORT_STEP;
+	} else {
+		datawidth = SPRD_DMA_DATAWIDTH_1_BYTE;
+		step = SPRD_DMA_BYTE_STEP;
 	}
 
-	if (transcation_len == 0)
-		hw->trsc_len = block_len & SPRD_DMA_TRSC_LEN_MASK;
-	else
-		hw->trsc_len = transcation_len & SPRD_DMA_TRSC_LEN_MASK;
+	temp = datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET;
+	temp |= datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET;
+	temp |= SPRD_DMA_TRANS_REQ << SPRD_DMA_REQ_MODE_OFFSET;
+	temp |= len & SPRD_DMA_FRG_LEN_MASK;
+	hw->frg_len = temp;
 
-	hw->trsf_step = (des_step & SPRD_DMA_TRSF_STEP_MASK) <<
-			SPRD_DMA_DEST_TRSF_STEP_OFFSET |
-			(src_step & SPRD_DMA_TRSF_STEP_MASK) <<
-			SPRD_DMA_SRC_TRSF_STEP_OFFSET;
+	hw->blk_len = len & SPRD_DMA_BLK_LEN_MASK;
+	hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK;
 
-	hw->frg_step = 0;
-	hw->src_blk_step = 0;
-	hw->des_blk_step = 0;
-	hw->src_blk_step = 0;
-	return 0;
+	temp = (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET;
+	temp |= (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET;
+	hw->trsf_step = temp;
+
+	return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);
 }
 
 static struct dma_async_tx_descriptor *
-sprd_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
-			 size_t len, unsigned long flags)
+sprd_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
+		       unsigned int sglen, enum dma_transfer_direction dir,
+		       unsigned long flags, void *context)
 {
 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
+	struct dma_slave_config *slave_cfg = &schan->slave_cfg;
+	dma_addr_t src = 0, dst = 0;
 	struct sprd_dma_desc *sdesc;
-	int ret;
+	struct scatterlist *sg;
+	u32 len = 0;
+	int ret, i;
+
+	/* TODO: now we only support one sg for each DMA configuration. */
+	if (!is_slave_direction(dir) || sglen > 1)
+		return NULL;
 
 	sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT);
 	if (!sdesc)
 		return NULL;
 
-	ret = sprd_dma_config(chan, sdesc, dest, src, len);
+	for_each_sg(sgl, sg, sglen, i) {
+		len = sg_dma_len(sg);
+
+		if (dir == DMA_MEM_TO_DEV) {
+			src = sg_dma_address(sg);
+			dst = slave_cfg->dst_addr;
+		} else {
+			src = slave_cfg->src_addr;
+			dst = sg_dma_address(sg);
+		}
+	}
+
+	ret = sprd_dma_fill_desc(chan, sdesc, src, dst, len, dir, flags,
+				 slave_cfg);
 	if (ret) {
 		kfree(sdesc);
 		return NULL;
@@ -731,6 +774,19 @@ sprd_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 	return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);
 }
 
+static int sprd_dma_slave_config(struct dma_chan *chan,
+				 struct dma_slave_config *config)
+{
+	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
+	struct dma_slave_config *slave_cfg = &schan->slave_cfg;
+
+	if (!is_slave_direction(config->direction))
+		return -EINVAL;
+
+	memcpy(slave_cfg, config, sizeof(*config));
+	return 0;
+}
+
 static int sprd_dma_pause(struct dma_chan *chan)
 {
 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
@@ -842,10 +898,9 @@ static int sprd_dma_probe(struct platform_device *pdev)
 	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	sdev->glb_base = devm_ioremap_nocache(&pdev->dev, res->start,
-					      resource_size(res));
-	if (!sdev->glb_base)
-		return -ENOMEM;
+	sdev->glb_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(sdev->glb_base))
+		return PTR_ERR(sdev->glb_base);
 
 	dma_cap_set(DMA_MEMCPY, sdev->dma_dev.cap_mask);
 	sdev->total_chns = chn_count;
@@ -858,6 +913,8 @@ static int sprd_dma_probe(struct platform_device *pdev)
 	sdev->dma_dev.device_tx_status = sprd_dma_tx_status;
 	sdev->dma_dev.device_issue_pending = sprd_dma_issue_pending;
 	sdev->dma_dev.device_prep_dma_memcpy = sprd_dma_prep_dma_memcpy;
+	sdev->dma_dev.device_prep_slave_sg = sprd_dma_prep_slave_sg;
+	sdev->dma_dev.device_config = sprd_dma_slave_config;
 	sdev->dma_dev.device_pause = sprd_dma_pause;
 	sdev->dma_dev.device_resume = sprd_dma_resume;
 	sdev->dma_dev.device_terminate_all = sprd_dma_terminate_all;
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index c2b089af0420..1bc149af990e 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -2889,8 +2889,7 @@ static int __init d40_dmaengine_init(struct d40_base *base,
 #ifdef CONFIG_PM_SLEEP
 static int dma40_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct d40_base *base = platform_get_drvdata(pdev);
+	struct d40_base *base = dev_get_drvdata(dev);
 	int ret;
 
 	ret = pm_runtime_force_suspend(dev);
@@ -2904,8 +2903,7 @@ static int dma40_suspend(struct device *dev)
 
 static int dma40_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct d40_base *base = platform_get_drvdata(pdev);
+	struct d40_base *base = dev_get_drvdata(dev);
 	int ret = 0;
 
 	if (base->lcpa_regulator) {
@@ -2970,8 +2968,7 @@ static void d40_save_restore_registers(struct d40_base *base, bool save)
 
 static int dma40_runtime_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct d40_base *base = platform_get_drvdata(pdev);
+	struct d40_base *base = dev_get_drvdata(dev);
 
 	d40_save_restore_registers(base, true);
 
@@ -2985,8 +2982,7 @@ static int dma40_runtime_suspend(struct device *dev)
 
 static int dma40_runtime_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct d40_base *base = platform_get_drvdata(pdev);
+	struct d40_base *base = dev_get_drvdata(dev);
 
 	d40_save_restore_registers(base, false);
 
diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c
index daa1602eb9f5..9dc450b7ace6 100644
--- a/drivers/dma/stm32-mdma.c
+++ b/drivers/dma/stm32-mdma.c
@@ -252,13 +252,17 @@ struct stm32_mdma_hwdesc {
 	u32 cmdr;
 } __aligned(64);
 
+struct stm32_mdma_desc_node {
+	struct stm32_mdma_hwdesc *hwdesc;
+	dma_addr_t hwdesc_phys;
+};
+
 struct stm32_mdma_desc {
 	struct virt_dma_desc vdesc;
 	u32 ccr;
-	struct stm32_mdma_hwdesc *hwdesc;
-	dma_addr_t hwdesc_phys;
 	bool cyclic;
 	u32 count;
+	struct stm32_mdma_desc_node node[];
 };
 
 struct stm32_mdma_chan {
@@ -344,30 +348,42 @@ static struct stm32_mdma_desc *stm32_mdma_alloc_desc(
 		struct stm32_mdma_chan *chan, u32 count)
 {
 	struct stm32_mdma_desc *desc;
+	int i;
 
-	desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
+	desc = kzalloc(offsetof(typeof(*desc), node[count]), GFP_NOWAIT);
 	if (!desc)
 		return NULL;
 
-	desc->hwdesc = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT,
-				      &desc->hwdesc_phys);
-	if (!desc->hwdesc) {
-		dev_err(chan2dev(chan), "Failed to allocate descriptor\n");
-		kfree(desc);
-		return NULL;
+	for (i = 0; i < count; i++) {
+		desc->node[i].hwdesc =
+			dma_pool_alloc(chan->desc_pool, GFP_NOWAIT,
+				       &desc->node[i].hwdesc_phys);
+		if (!desc->node[i].hwdesc)
+			goto err;
 	}
 
 	desc->count = count;
 
 	return desc;
+
+err:
+	dev_err(chan2dev(chan), "Failed to allocate descriptor\n");
+	while (--i >= 0)
+		dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
+			      desc->node[i].hwdesc_phys);
+	kfree(desc);
+	return NULL;
 }
 
 static void stm32_mdma_desc_free(struct virt_dma_desc *vdesc)
 {
 	struct stm32_mdma_desc *desc = to_stm32_mdma_desc(vdesc);
 	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(vdesc->tx.chan);
+	int i;
 
-	dma_pool_free(chan->desc_pool, desc->hwdesc, desc->hwdesc_phys);
+	for (i = 0; i < desc->count; i++)
+		dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
+			      desc->node[i].hwdesc_phys);
 	kfree(desc);
 }
 
@@ -410,13 +426,10 @@ static enum dma_slave_buswidth stm32_mdma_get_max_width(dma_addr_t addr,
 static u32 stm32_mdma_get_best_burst(u32 buf_len, u32 tlen, u32 max_burst,
 				     enum dma_slave_buswidth width)
 {
-	u32 best_burst = max_burst;
-	u32 burst_len = best_burst * width;
+	u32 best_burst;
 
-	while ((burst_len > 0) && (tlen % burst_len)) {
-		best_burst = best_burst >> 1;
-		burst_len = best_burst * width;
-	}
+	best_burst = min((u32)1 << __ffs(tlen | buf_len),
+			 max_burst * width) / width;
 
 	return (best_burst > 0) ? best_burst : 1;
 }
@@ -669,18 +682,18 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan,
 }
 
 static void stm32_mdma_dump_hwdesc(struct stm32_mdma_chan *chan,
-				   struct stm32_mdma_hwdesc *hwdesc)
+				   struct stm32_mdma_desc_node *node)
 {
-	dev_dbg(chan2dev(chan), "hwdesc:  0x%p\n", hwdesc);
-	dev_dbg(chan2dev(chan), "CTCR:    0x%08x\n", hwdesc->ctcr);
-	dev_dbg(chan2dev(chan), "CBNDTR:  0x%08x\n", hwdesc->cbndtr);
-	dev_dbg(chan2dev(chan), "CSAR:    0x%08x\n", hwdesc->csar);
-	dev_dbg(chan2dev(chan), "CDAR:    0x%08x\n", hwdesc->cdar);
-	dev_dbg(chan2dev(chan), "CBRUR:   0x%08x\n", hwdesc->cbrur);
-	dev_dbg(chan2dev(chan), "CLAR:    0x%08x\n", hwdesc->clar);
-	dev_dbg(chan2dev(chan), "CTBR:    0x%08x\n", hwdesc->ctbr);
-	dev_dbg(chan2dev(chan), "CMAR:    0x%08x\n", hwdesc->cmar);
-	dev_dbg(chan2dev(chan), "CMDR:    0x%08x\n\n", hwdesc->cmdr);
+	dev_dbg(chan2dev(chan), "hwdesc:  %pad\n", &node->hwdesc_phys);
+	dev_dbg(chan2dev(chan), "CTCR:    0x%08x\n", node->hwdesc->ctcr);
+	dev_dbg(chan2dev(chan), "CBNDTR:  0x%08x\n", node->hwdesc->cbndtr);
+	dev_dbg(chan2dev(chan), "CSAR:    0x%08x\n", node->hwdesc->csar);
+	dev_dbg(chan2dev(chan), "CDAR:    0x%08x\n", node->hwdesc->cdar);
+	dev_dbg(chan2dev(chan), "CBRUR:   0x%08x\n", node->hwdesc->cbrur);
+	dev_dbg(chan2dev(chan), "CLAR:    0x%08x\n", node->hwdesc->clar);
+	dev_dbg(chan2dev(chan), "CTBR:    0x%08x\n", node->hwdesc->ctbr);
+	dev_dbg(chan2dev(chan), "CMAR:    0x%08x\n", node->hwdesc->cmar);
+	dev_dbg(chan2dev(chan), "CMDR:    0x%08x\n\n", node->hwdesc->cmdr);
 }
 
 static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan,
@@ -694,7 +707,7 @@ static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan,
 	struct stm32_mdma_hwdesc *hwdesc;
 	u32 next = count + 1;
 
-	hwdesc = &desc->hwdesc[count];
+	hwdesc = desc->node[count].hwdesc;
 	hwdesc->ctcr = ctcr;
 	hwdesc->cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK |
 			STM32_MDMA_CBNDTR_BRDUM |
@@ -704,19 +717,20 @@ static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan,
 	hwdesc->csar = src_addr;
 	hwdesc->cdar = dst_addr;
 	hwdesc->cbrur = 0;
-	hwdesc->clar = desc->hwdesc_phys + next * sizeof(*hwdesc);
 	hwdesc->ctbr = ctbr;
 	hwdesc->cmar = config->mask_addr;
 	hwdesc->cmdr = config->mask_data;
 
 	if (is_last) {
 		if (is_cyclic)
-			hwdesc->clar = desc->hwdesc_phys;
+			hwdesc->clar = desc->node[0].hwdesc_phys;
 		else
 			hwdesc->clar = 0;
+	} else {
+		hwdesc->clar = desc->node[next].hwdesc_phys;
 	}
 
-	stm32_mdma_dump_hwdesc(chan, hwdesc);
+	stm32_mdma_dump_hwdesc(chan, &desc->node[count]);
 }
 
 static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan,
@@ -780,7 +794,7 @@ stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl,
 {
 	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
 	struct stm32_mdma_desc *desc;
-	int ret;
+	int i, ret;
 
 	/*
 	 * Once DMA is in setup cyclic mode the channel we cannot assign this
@@ -806,7 +820,9 @@ stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl,
 	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 
 xfer_setup_err:
-	dma_pool_free(chan->desc_pool, &desc->hwdesc, desc->hwdesc_phys);
+	for (i = 0; i < desc->count; i++)
+		dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
+			      desc->node[i].hwdesc_phys);
 	kfree(desc);
 	return NULL;
 }
@@ -895,7 +911,9 @@ stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr,
 	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 
 xfer_setup_err:
-	dma_pool_free(chan->desc_pool, &desc->hwdesc, desc->hwdesc_phys);
+	for (i = 0; i < desc->count; i++)
+		dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
+			      desc->node[i].hwdesc_phys);
 	kfree(desc);
 	return NULL;
 }
@@ -1009,7 +1027,7 @@ stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src,
 			ctcr |= STM32_MDMA_CTCR_PKE;
 
 		/* Prepare hardware descriptor */
-		hwdesc = desc->hwdesc;
+		hwdesc = desc->node[0].hwdesc;
 		hwdesc->ctcr = ctcr;
 		hwdesc->cbndtr = cbndtr;
 		hwdesc->csar = src;
@@ -1020,7 +1038,7 @@ stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src,
 		hwdesc->cmar = 0;
 		hwdesc->cmdr = 0;
 
-		stm32_mdma_dump_hwdesc(chan, hwdesc);
+		stm32_mdma_dump_hwdesc(chan, &desc->node[0]);
 	} else {
 		/* Setup a LLI transfer */
 		ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_LINKED_LIST) |
@@ -1120,7 +1138,7 @@ static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan)
 	}
 
 	chan->desc = to_stm32_mdma_desc(vdesc);
-	hwdesc = chan->desc->hwdesc;
+	hwdesc = chan->desc->node[0].hwdesc;
 	chan->curr_hwdesc = 0;
 
 	stm32_mdma_write(dmadev, STM32_MDMA_CCR(id), chan->desc->ccr);
@@ -1198,7 +1216,7 @@ static int stm32_mdma_resume(struct dma_chan *c)
 	unsigned long flags;
 	u32 status, reg;
 
-	hwdesc = &chan->desc->hwdesc[chan->curr_hwdesc];
+	hwdesc = chan->desc->node[chan->curr_hwdesc].hwdesc;
 
 	spin_lock_irqsave(&chan->vchan.lock, flags);
 
@@ -1268,13 +1286,13 @@ static size_t stm32_mdma_desc_residue(struct stm32_mdma_chan *chan,
 				      u32 curr_hwdesc)
 {
 	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
+	struct stm32_mdma_hwdesc *hwdesc = desc->node[0].hwdesc;
 	u32 cbndtr, residue, modulo, burst_size;
 	int i;
 
 	residue = 0;
 	for (i = curr_hwdesc + 1; i < desc->count; i++) {
-		struct stm32_mdma_hwdesc *hwdesc = &desc->hwdesc[i];
-
+		hwdesc = desc->node[i].hwdesc;
 		residue += STM32_MDMA_CBNDTR_BNDT(hwdesc->cbndtr);
 	}
 	cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id));
@@ -1503,7 +1521,7 @@ static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec,
 
 	c = dma_get_any_slave_channel(&dmadev->ddev);
 	if (!c) {
-		dev_err(mdma2dev(dmadev), "No more channel avalaible\n");
+		dev_err(mdma2dev(dmadev), "No more channels available\n");
 		return NULL;
 	}
 
diff --git a/drivers/dma/ti-dma-crossbar.c b/drivers/dma/ti-dma-crossbar.c
deleted file mode 100644
index 9272b173c746..000000000000
--- a/drivers/dma/ti-dma-crossbar.c
+++ /dev/null
@@ -1,478 +0,0 @@
-/*
- *  Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com
- *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
-#include <linux/slab.h>
-#include <linux/err.h>
-#include <linux/init.h>
-#include <linux/list.h>
-#include <linux/io.h>
-#include <linux/of_address.h>
-#include <linux/of_device.h>
-#include <linux/of_dma.h>
-
-#define TI_XBAR_DRA7		0
-#define TI_XBAR_AM335X		1
-static const u32 ti_xbar_type[] = {
-	[TI_XBAR_DRA7] = TI_XBAR_DRA7,
-	[TI_XBAR_AM335X] = TI_XBAR_AM335X,
-};
-
-static const struct of_device_id ti_dma_xbar_match[] = {
-	{
-		.compatible = "ti,dra7-dma-crossbar",
-		.data = &ti_xbar_type[TI_XBAR_DRA7],
-	},
-	{
-		.compatible = "ti,am335x-edma-crossbar",
-		.data = &ti_xbar_type[TI_XBAR_AM335X],
-	},
-	{},
-};
-
-/* Crossbar on AM335x/AM437x family */
-#define TI_AM335X_XBAR_LINES	64
-
-struct ti_am335x_xbar_data {
-	void __iomem *iomem;
-
-	struct dma_router dmarouter;
-
-	u32 xbar_events; /* maximum number of events to select in xbar */
-	u32 dma_requests; /* number of DMA requests on eDMA */
-};
-
-struct ti_am335x_xbar_map {
-	u16 dma_line;
-	u8 mux_val;
-};
-
-static inline void ti_am335x_xbar_write(void __iomem *iomem, int event, u8 val)
-{
-	/*
-	 * TPCC_EVT_MUX_60_63 register layout is different than the
-	 * rest, in the sense, that event 63 is mapped to lowest byte
-	 * and event 60 is mapped to highest, handle it separately.
-	 */
-	if (event >= 60 && event <= 63)
-		writeb_relaxed(val, iomem + (63 - event % 4));
-	else
-		writeb_relaxed(val, iomem + event);
-}
-
-static void ti_am335x_xbar_free(struct device *dev, void *route_data)
-{
-	struct ti_am335x_xbar_data *xbar = dev_get_drvdata(dev);
-	struct ti_am335x_xbar_map *map = route_data;
-
-	dev_dbg(dev, "Unmapping XBAR event %u on channel %u\n",
-		map->mux_val, map->dma_line);
-
-	ti_am335x_xbar_write(xbar->iomem, map->dma_line, 0);
-	kfree(map);
-}
-
-static void *ti_am335x_xbar_route_allocate(struct of_phandle_args *dma_spec,
-					   struct of_dma *ofdma)
-{
-	struct platform_device *pdev = of_find_device_by_node(ofdma->of_node);
-	struct ti_am335x_xbar_data *xbar = platform_get_drvdata(pdev);
-	struct ti_am335x_xbar_map *map;
-
-	if (dma_spec->args_count != 3)
-		return ERR_PTR(-EINVAL);
-
-	if (dma_spec->args[2] >= xbar->xbar_events) {
-		dev_err(&pdev->dev, "Invalid XBAR event number: %d\n",
-			dma_spec->args[2]);
-		return ERR_PTR(-EINVAL);
-	}
-
-	if (dma_spec->args[0] >= xbar->dma_requests) {
-		dev_err(&pdev->dev, "Invalid DMA request line number: %d\n",
-			dma_spec->args[0]);
-		return ERR_PTR(-EINVAL);
-	}
-
-	/* The of_node_put() will be done in the core for the node */
-	dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0);
-	if (!dma_spec->np) {
-		dev_err(&pdev->dev, "Can't get DMA master\n");
-		return ERR_PTR(-EINVAL);
-	}
-
-	map = kzalloc(sizeof(*map), GFP_KERNEL);
-	if (!map) {
-		of_node_put(dma_spec->np);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	map->dma_line = (u16)dma_spec->args[0];
-	map->mux_val = (u8)dma_spec->args[2];
-
-	dma_spec->args[2] = 0;
-	dma_spec->args_count = 2;
-
-	dev_dbg(&pdev->dev, "Mapping XBAR event%u to DMA%u\n",
-		map->mux_val, map->dma_line);
-
-	ti_am335x_xbar_write(xbar->iomem, map->dma_line, map->mux_val);
-
-	return map;
-}
-
-static const struct of_device_id ti_am335x_master_match[] = {
-	{ .compatible = "ti,edma3-tpcc", },
-	{},
-};
-
-static int ti_am335x_xbar_probe(struct platform_device *pdev)
-{
-	struct device_node *node = pdev->dev.of_node;
-	const struct of_device_id *match;
-	struct device_node *dma_node;
-	struct ti_am335x_xbar_data *xbar;
-	struct resource *res;
-	void __iomem *iomem;
-	int i, ret;
-
-	if (!node)
-		return -ENODEV;
-
-	xbar = devm_kzalloc(&pdev->dev, sizeof(*xbar), GFP_KERNEL);
-	if (!xbar)
-		return -ENOMEM;
-
-	dma_node = of_parse_phandle(node, "dma-masters", 0);
-	if (!dma_node) {
-		dev_err(&pdev->dev, "Can't get DMA master node\n");
-		return -ENODEV;
-	}
-
-	match = of_match_node(ti_am335x_master_match, dma_node);
-	if (!match) {
-		dev_err(&pdev->dev, "DMA master is not supported\n");
-		of_node_put(dma_node);
-		return -EINVAL;
-	}
-
-	if (of_property_read_u32(dma_node, "dma-requests",
-				 &xbar->dma_requests)) {
-		dev_info(&pdev->dev,
-			 "Missing XBAR output information, using %u.\n",
-			 TI_AM335X_XBAR_LINES);
-		xbar->dma_requests = TI_AM335X_XBAR_LINES;
-	}
-	of_node_put(dma_node);
-
-	if (of_property_read_u32(node, "dma-requests", &xbar->xbar_events)) {
-		dev_info(&pdev->dev,
-			 "Missing XBAR input information, using %u.\n",
-			 TI_AM335X_XBAR_LINES);
-		xbar->xbar_events = TI_AM335X_XBAR_LINES;
-	}
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	iomem = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(iomem))
-		return PTR_ERR(iomem);
-
-	xbar->iomem = iomem;
-
-	xbar->dmarouter.dev = &pdev->dev;
-	xbar->dmarouter.route_free = ti_am335x_xbar_free;
-
-	platform_set_drvdata(pdev, xbar);
-
-	/* Reset the crossbar */
-	for (i = 0; i < xbar->dma_requests; i++)
-		ti_am335x_xbar_write(xbar->iomem, i, 0);
-
-	ret = of_dma_router_register(node, ti_am335x_xbar_route_allocate,
-				     &xbar->dmarouter);
-
-	return ret;
-}
-
-/* Crossbar on DRA7xx family */
-#define TI_DRA7_XBAR_OUTPUTS	127
-#define TI_DRA7_XBAR_INPUTS	256
-
-struct ti_dra7_xbar_data {
-	void __iomem *iomem;
-
-	struct dma_router dmarouter;
-	struct mutex mutex;
-	unsigned long *dma_inuse;
-
-	u16 safe_val; /* Value to rest the crossbar lines */
-	u32 xbar_requests; /* number of DMA requests connected to XBAR */
-	u32 dma_requests; /* number of DMA requests forwarded to DMA */
-	u32 dma_offset;
-};
-
-struct ti_dra7_xbar_map {
-	u16 xbar_in;
-	int xbar_out;
-};
-
-static inline void ti_dra7_xbar_write(void __iomem *iomem, int xbar, u16 val)
-{
-	writew_relaxed(val, iomem + (xbar * 2));
-}
-
-static void ti_dra7_xbar_free(struct device *dev, void *route_data)
-{
-	struct ti_dra7_xbar_data *xbar = dev_get_drvdata(dev);
-	struct ti_dra7_xbar_map *map = route_data;
-
-	dev_dbg(dev, "Unmapping XBAR%u (was routed to %d)\n",
-		map->xbar_in, map->xbar_out);
-
-	ti_dra7_xbar_write(xbar->iomem, map->xbar_out, xbar->safe_val);
-	mutex_lock(&xbar->mutex);
-	clear_bit(map->xbar_out, xbar->dma_inuse);
-	mutex_unlock(&xbar->mutex);
-	kfree(map);
-}
-
-static void *ti_dra7_xbar_route_allocate(struct of_phandle_args *dma_spec,
-					 struct of_dma *ofdma)
-{
-	struct platform_device *pdev = of_find_device_by_node(ofdma->of_node);
-	struct ti_dra7_xbar_data *xbar = platform_get_drvdata(pdev);
-	struct ti_dra7_xbar_map *map;
-
-	if (dma_spec->args[0] >= xbar->xbar_requests) {
-		dev_err(&pdev->dev, "Invalid XBAR request number: %d\n",
-			dma_spec->args[0]);
-		return ERR_PTR(-EINVAL);
-	}
-
-	/* The of_node_put() will be done in the core for the node */
-	dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0);
-	if (!dma_spec->np) {
-		dev_err(&pdev->dev, "Can't get DMA master\n");
-		return ERR_PTR(-EINVAL);
-	}
-
-	map = kzalloc(sizeof(*map), GFP_KERNEL);
-	if (!map) {
-		of_node_put(dma_spec->np);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	mutex_lock(&xbar->mutex);
-	map->xbar_out = find_first_zero_bit(xbar->dma_inuse,
-					    xbar->dma_requests);
-	if (map->xbar_out == xbar->dma_requests) {
-		mutex_unlock(&xbar->mutex);
-		dev_err(&pdev->dev, "Run out of free DMA requests\n");
-		kfree(map);
-		return ERR_PTR(-ENOMEM);
-	}
-	set_bit(map->xbar_out, xbar->dma_inuse);
-	mutex_unlock(&xbar->mutex);
-
-	map->xbar_in = (u16)dma_spec->args[0];
-
-	dma_spec->args[0] = map->xbar_out + xbar->dma_offset;
-
-	dev_dbg(&pdev->dev, "Mapping XBAR%u to DMA%d\n",
-		map->xbar_in, map->xbar_out);
-
-	ti_dra7_xbar_write(xbar->iomem, map->xbar_out, map->xbar_in);
-
-	return map;
-}
-
-#define TI_XBAR_EDMA_OFFSET	0
-#define TI_XBAR_SDMA_OFFSET	1
-static const u32 ti_dma_offset[] = {
-	[TI_XBAR_EDMA_OFFSET] = 0,
-	[TI_XBAR_SDMA_OFFSET] = 1,
-};
-
-static const struct of_device_id ti_dra7_master_match[] = {
-	{
-		.compatible = "ti,omap4430-sdma",
-		.data = &ti_dma_offset[TI_XBAR_SDMA_OFFSET],
-	},
-	{
-		.compatible = "ti,edma3",
-		.data = &ti_dma_offset[TI_XBAR_EDMA_OFFSET],
-	},
-	{
-		.compatible = "ti,edma3-tpcc",
-		.data = &ti_dma_offset[TI_XBAR_EDMA_OFFSET],
-	},
-	{},
-};
-
-static inline void ti_dra7_xbar_reserve(int offset, int len, unsigned long *p)
-{
-	for (; len > 0; len--)
-		set_bit(offset + (len - 1), p);
-}
-
-static int ti_dra7_xbar_probe(struct platform_device *pdev)
-{
-	struct device_node *node = pdev->dev.of_node;
-	const struct of_device_id *match;
-	struct device_node *dma_node;
-	struct ti_dra7_xbar_data *xbar;
-	struct property *prop;
-	struct resource *res;
-	u32 safe_val;
-	int sz;
-	void __iomem *iomem;
-	int i, ret;
-
-	if (!node)
-		return -ENODEV;
-
-	xbar = devm_kzalloc(&pdev->dev, sizeof(*xbar), GFP_KERNEL);
-	if (!xbar)
-		return -ENOMEM;
-
-	dma_node = of_parse_phandle(node, "dma-masters", 0);
-	if (!dma_node) {
-		dev_err(&pdev->dev, "Can't get DMA master node\n");
-		return -ENODEV;
-	}
-
-	match = of_match_node(ti_dra7_master_match, dma_node);
-	if (!match) {
-		dev_err(&pdev->dev, "DMA master is not supported\n");
-		of_node_put(dma_node);
-		return -EINVAL;
-	}
-
-	if (of_property_read_u32(dma_node, "dma-requests",
-				 &xbar->dma_requests)) {
-		dev_info(&pdev->dev,
-			 "Missing XBAR output information, using %u.\n",
-			 TI_DRA7_XBAR_OUTPUTS);
-		xbar->dma_requests = TI_DRA7_XBAR_OUTPUTS;
-	}
-	of_node_put(dma_node);
-
-	xbar->dma_inuse = devm_kcalloc(&pdev->dev,
-				       BITS_TO_LONGS(xbar->dma_requests),
-				       sizeof(unsigned long), GFP_KERNEL);
-	if (!xbar->dma_inuse)
-		return -ENOMEM;
-
-	if (of_property_read_u32(node, "dma-requests", &xbar->xbar_requests)) {
-		dev_info(&pdev->dev,
-			 "Missing XBAR input information, using %u.\n",
-			 TI_DRA7_XBAR_INPUTS);
-		xbar->xbar_requests = TI_DRA7_XBAR_INPUTS;
-	}
-
-	if (!of_property_read_u32(node, "ti,dma-safe-map", &safe_val))
-		xbar->safe_val = (u16)safe_val;
-
-
-	prop = of_find_property(node, "ti,reserved-dma-request-ranges", &sz);
-	if (prop) {
-		const char pname[] = "ti,reserved-dma-request-ranges";
-		u32 (*rsv_events)[2];
-		size_t nelm = sz / sizeof(*rsv_events);
-		int i;
-
-		if (!nelm)
-			return -EINVAL;
-
-		rsv_events = kcalloc(nelm, sizeof(*rsv_events), GFP_KERNEL);
-		if (!rsv_events)
-			return -ENOMEM;
-
-		ret = of_property_read_u32_array(node, pname, (u32 *)rsv_events,
-						 nelm * 2);
-		if (ret)
-			return ret;
-
-		for (i = 0; i < nelm; i++) {
-			ti_dra7_xbar_reserve(rsv_events[i][0], rsv_events[i][1],
-					     xbar->dma_inuse);
-		}
-		kfree(rsv_events);
-	}
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	iomem = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(iomem))
-		return PTR_ERR(iomem);
-
-	xbar->iomem = iomem;
-
-	xbar->dmarouter.dev = &pdev->dev;
-	xbar->dmarouter.route_free = ti_dra7_xbar_free;
-	xbar->dma_offset = *(u32 *)match->data;
-
-	mutex_init(&xbar->mutex);
-	platform_set_drvdata(pdev, xbar);
-
-	/* Reset the crossbar */
-	for (i = 0; i < xbar->dma_requests; i++) {
-		if (!test_bit(i, xbar->dma_inuse))
-			ti_dra7_xbar_write(xbar->iomem, i, xbar->safe_val);
-	}
-
-	ret = of_dma_router_register(node, ti_dra7_xbar_route_allocate,
-				     &xbar->dmarouter);
-	if (ret) {
-		/* Restore the defaults for the crossbar */
-		for (i = 0; i < xbar->dma_requests; i++) {
-			if (!test_bit(i, xbar->dma_inuse))
-				ti_dra7_xbar_write(xbar->iomem, i, i);
-		}
-	}
-
-	return ret;
-}
-
-static int ti_dma_xbar_probe(struct platform_device *pdev)
-{
-	const struct of_device_id *match;
-	int ret;
-
-	match = of_match_node(ti_dma_xbar_match, pdev->dev.of_node);
-	if (unlikely(!match))
-		return -EINVAL;
-
-	switch (*(u32 *)match->data) {
-	case TI_XBAR_DRA7:
-		ret = ti_dra7_xbar_probe(pdev);
-		break;
-	case TI_XBAR_AM335X:
-		ret = ti_am335x_xbar_probe(pdev);
-		break;
-	default:
-		dev_err(&pdev->dev, "Unsupported crossbar\n");
-		ret = -ENODEV;
-		break;
-	}
-
-	return ret;
-}
-
-static struct platform_driver ti_dma_xbar_driver = {
-	.driver = {
-		.name = "ti-dma-crossbar",
-		.of_match_table = of_match_ptr(ti_dma_xbar_match),
-	},
-	.probe	= ti_dma_xbar_probe,
-};
-
-static int omap_dmaxbar_init(void)
-{
-	return platform_driver_register(&ti_dma_xbar_driver);
-}
-arch_initcall(omap_dmaxbar_init);
diff --git a/drivers/dma/ti/Kconfig b/drivers/dma/ti/Kconfig
new file mode 100644
index 000000000000..e5e74e1361dc
--- /dev/null
+++ b/drivers/dma/ti/Kconfig
@@ -0,0 +1,37 @@
+#
+# Texas Instruments DMA drivers
+#
+
+config TI_CPPI41
+	tristate "Texas Instruments CPPI 4.1 DMA support"
+	depends on (ARCH_OMAP || ARCH_DAVINCI_DA8XX)
+	select DMA_ENGINE
+	help
+	  The Communications Port Programming Interface (CPPI) 4.1 DMA engine
+	  is currently used by the USB driver on AM335x and DA8xx platforms.
+
+config TI_EDMA
+	tristate "Texas Instruments EDMA support"
+	depends on ARCH_DAVINCI || ARCH_OMAP || ARCH_KEYSTONE || COMPILE_TEST
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
+	select TI_DMA_CROSSBAR if (ARCH_OMAP || COMPILE_TEST)
+	default y
+	help
+	  Enable support for the TI EDMA (Enhanced DMA) controller. This DMA
+	  engine is found on TI DaVinci, AM33xx, AM43xx, DRA7xx and Keystone 2
+	  parts.
+
+config DMA_OMAP
+	tristate "Texas Instruments sDMA (omap-dma) support"
+	depends on ARCH_OMAP || COMPILE_TEST
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
+	select TI_DMA_CROSSBAR if (SOC_DRA7XX || COMPILE_TEST)
+	default y
+	help
+	  Enable support for the TI sDMA (System DMA or DMA4) controller. This
+	  DMA engine is found on OMAP and DRA7xx parts.
+
+config TI_DMA_CROSSBAR
+	bool
diff --git a/drivers/dma/ti/Makefile b/drivers/dma/ti/Makefile
new file mode 100644
index 000000000000..113e59ec9c32
--- /dev/null
+++ b/drivers/dma/ti/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_TI_CPPI41) += cppi41.o
+obj-$(CONFIG_TI_EDMA) += edma.o
+obj-$(CONFIG_DMA_OMAP) += omap-dma.o
+obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o
diff --git a/drivers/dma/ti/cppi41.c b/drivers/dma/ti/cppi41.c
new file mode 100644
index 000000000000..1497da367710
--- /dev/null
+++ b/drivers/dma/ti/cppi41.c
@@ -0,0 +1,1223 @@
+#include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/of_dma.h>
+#include <linux/of_irq.h>
+#include <linux/dmapool.h>
+#include <linux/interrupt.h>
+#include <linux/of_address.h>
+#include <linux/pm_runtime.h>
+#include "../dmaengine.h"
+
+#define DESC_TYPE	27
+#define DESC_TYPE_HOST	0x10
+#define DESC_TYPE_TEARD	0x13
+
+#define TD_DESC_IS_RX	(1 << 16)
+#define TD_DESC_DMA_NUM	10
+
+#define DESC_LENGTH_BITS_NUM	21
+
+#define DESC_TYPE_USB	(5 << 26)
+#define DESC_PD_COMPLETE	(1 << 31)
+
+/* DMA engine */
+#define DMA_TDFDQ	4
+#define DMA_TXGCR(x)	(0x800 + (x) * 0x20)
+#define DMA_RXGCR(x)	(0x808 + (x) * 0x20)
+#define RXHPCRA0		4
+
+#define GCR_CHAN_ENABLE		(1 << 31)
+#define GCR_TEARDOWN		(1 << 30)
+#define GCR_STARV_RETRY		(1 << 24)
+#define GCR_DESC_TYPE_HOST	(1 << 14)
+
+/* DMA scheduler */
+#define DMA_SCHED_CTRL		0
+#define DMA_SCHED_CTRL_EN	(1 << 31)
+#define DMA_SCHED_WORD(x)	((x) * 4 + 0x800)
+
+#define SCHED_ENTRY0_CHAN(x)	((x) << 0)
+#define SCHED_ENTRY0_IS_RX	(1 << 7)
+
+#define SCHED_ENTRY1_CHAN(x)	((x) << 8)
+#define SCHED_ENTRY1_IS_RX	(1 << 15)
+
+#define SCHED_ENTRY2_CHAN(x)	((x) << 16)
+#define SCHED_ENTRY2_IS_RX	(1 << 23)
+
+#define SCHED_ENTRY3_CHAN(x)	((x) << 24)
+#define SCHED_ENTRY3_IS_RX	(1 << 31)
+
+/* Queue manager */
+/* 4 KiB of memory for descriptors, 2 for each endpoint */
+#define ALLOC_DECS_NUM		128
+#define DESCS_AREAS		1
+#define TOTAL_DESCS_NUM		(ALLOC_DECS_NUM * DESCS_AREAS)
+#define QMGR_SCRATCH_SIZE	(TOTAL_DESCS_NUM * 4)
+
+#define QMGR_LRAM0_BASE		0x80
+#define QMGR_LRAM_SIZE		0x84
+#define QMGR_LRAM1_BASE		0x88
+#define QMGR_MEMBASE(x)		(0x1000 + (x) * 0x10)
+#define QMGR_MEMCTRL(x)		(0x1004 + (x) * 0x10)
+#define QMGR_MEMCTRL_IDX_SH	16
+#define QMGR_MEMCTRL_DESC_SH	8
+
+#define QMGR_PEND(x)	(0x90 + (x) * 4)
+
+#define QMGR_PENDING_SLOT_Q(x)	(x / 32)
+#define QMGR_PENDING_BIT_Q(x)	(x % 32)
+
+#define QMGR_QUEUE_A(n)	(0x2000 + (n) * 0x10)
+#define QMGR_QUEUE_B(n)	(0x2004 + (n) * 0x10)
+#define QMGR_QUEUE_C(n)	(0x2008 + (n) * 0x10)
+#define QMGR_QUEUE_D(n)	(0x200c + (n) * 0x10)
+
+/* Packet Descriptor */
+#define PD2_ZERO_LENGTH		(1 << 19)
+
+struct cppi41_channel {
+	struct dma_chan chan;
+	struct dma_async_tx_descriptor txd;
+	struct cppi41_dd *cdd;
+	struct cppi41_desc *desc;
+	dma_addr_t desc_phys;
+	void __iomem *gcr_reg;
+	int is_tx;
+	u32 residue;
+
+	unsigned int q_num;
+	unsigned int q_comp_num;
+	unsigned int port_num;
+
+	unsigned td_retry;
+	unsigned td_queued:1;
+	unsigned td_seen:1;
+	unsigned td_desc_seen:1;
+
+	struct list_head node;		/* Node for pending list */
+};
+
+struct cppi41_desc {
+	u32 pd0;
+	u32 pd1;
+	u32 pd2;
+	u32 pd3;
+	u32 pd4;
+	u32 pd5;
+	u32 pd6;
+	u32 pd7;
+} __aligned(32);
+
+struct chan_queues {
+	u16 submit;
+	u16 complete;
+};
+
+struct cppi41_dd {
+	struct dma_device ddev;
+
+	void *qmgr_scratch;
+	dma_addr_t scratch_phys;
+
+	struct cppi41_desc *cd;
+	dma_addr_t descs_phys;
+	u32 first_td_desc;
+	struct cppi41_channel *chan_busy[ALLOC_DECS_NUM];
+
+	void __iomem *ctrl_mem;
+	void __iomem *sched_mem;
+	void __iomem *qmgr_mem;
+	unsigned int irq;
+	const struct chan_queues *queues_rx;
+	const struct chan_queues *queues_tx;
+	struct chan_queues td_queue;
+	u16 first_completion_queue;
+	u16 qmgr_num_pend;
+	u32 n_chans;
+	u8 platform;
+
+	struct list_head pending;	/* Pending queued transfers */
+	spinlock_t lock;		/* Lock for pending list */
+
+	/* context for suspend/resume */
+	unsigned int dma_tdfdq;
+
+	bool is_suspended;
+};
+
+static struct chan_queues am335x_usb_queues_tx[] = {
+	/* USB0 ENDP 1 */
+	[ 0] = { .submit = 32, .complete =  93},
+	[ 1] = { .submit = 34, .complete =  94},
+	[ 2] = { .submit = 36, .complete =  95},
+	[ 3] = { .submit = 38, .complete =  96},
+	[ 4] = { .submit = 40, .complete =  97},
+	[ 5] = { .submit = 42, .complete =  98},
+	[ 6] = { .submit = 44, .complete =  99},
+	[ 7] = { .submit = 46, .complete = 100},
+	[ 8] = { .submit = 48, .complete = 101},
+	[ 9] = { .submit = 50, .complete = 102},
+	[10] = { .submit = 52, .complete = 103},
+	[11] = { .submit = 54, .complete = 104},
+	[12] = { .submit = 56, .complete = 105},
+	[13] = { .submit = 58, .complete = 106},
+	[14] = { .submit = 60, .complete = 107},
+
+	/* USB1 ENDP1 */
+	[15] = { .submit = 62, .complete = 125},
+	[16] = { .submit = 64, .complete = 126},
+	[17] = { .submit = 66, .complete = 127},
+	[18] = { .submit = 68, .complete = 128},
+	[19] = { .submit = 70, .complete = 129},
+	[20] = { .submit = 72, .complete = 130},
+	[21] = { .submit = 74, .complete = 131},
+	[22] = { .submit = 76, .complete = 132},
+	[23] = { .submit = 78, .complete = 133},
+	[24] = { .submit = 80, .complete = 134},
+	[25] = { .submit = 82, .complete = 135},
+	[26] = { .submit = 84, .complete = 136},
+	[27] = { .submit = 86, .complete = 137},
+	[28] = { .submit = 88, .complete = 138},
+	[29] = { .submit = 90, .complete = 139},
+};
+
+static const struct chan_queues am335x_usb_queues_rx[] = {
+	/* USB0 ENDP 1 */
+	[ 0] = { .submit =  1, .complete = 109},
+	[ 1] = { .submit =  2, .complete = 110},
+	[ 2] = { .submit =  3, .complete = 111},
+	[ 3] = { .submit =  4, .complete = 112},
+	[ 4] = { .submit =  5, .complete = 113},
+	[ 5] = { .submit =  6, .complete = 114},
+	[ 6] = { .submit =  7, .complete = 115},
+	[ 7] = { .submit =  8, .complete = 116},
+	[ 8] = { .submit =  9, .complete = 117},
+	[ 9] = { .submit = 10, .complete = 118},
+	[10] = { .submit = 11, .complete = 119},
+	[11] = { .submit = 12, .complete = 120},
+	[12] = { .submit = 13, .complete = 121},
+	[13] = { .submit = 14, .complete = 122},
+	[14] = { .submit = 15, .complete = 123},
+
+	/* USB1 ENDP 1 */
+	[15] = { .submit = 16, .complete = 141},
+	[16] = { .submit = 17, .complete = 142},
+	[17] = { .submit = 18, .complete = 143},
+	[18] = { .submit = 19, .complete = 144},
+	[19] = { .submit = 20, .complete = 145},
+	[20] = { .submit = 21, .complete = 146},
+	[21] = { .submit = 22, .complete = 147},
+	[22] = { .submit = 23, .complete = 148},
+	[23] = { .submit = 24, .complete = 149},
+	[24] = { .submit = 25, .complete = 150},
+	[25] = { .submit = 26, .complete = 151},
+	[26] = { .submit = 27, .complete = 152},
+	[27] = { .submit = 28, .complete = 153},
+	[28] = { .submit = 29, .complete = 154},
+	[29] = { .submit = 30, .complete = 155},
+};
+
+static const struct chan_queues da8xx_usb_queues_tx[] = {
+	[0] = { .submit =  16, .complete = 24},
+	[1] = { .submit =  18, .complete = 24},
+	[2] = { .submit =  20, .complete = 24},
+	[3] = { .submit =  22, .complete = 24},
+};
+
+static const struct chan_queues da8xx_usb_queues_rx[] = {
+	[0] = { .submit =  1, .complete = 26},
+	[1] = { .submit =  3, .complete = 26},
+	[2] = { .submit =  5, .complete = 26},
+	[3] = { .submit =  7, .complete = 26},
+};
+
+struct cppi_glue_infos {
+	const struct chan_queues *queues_rx;
+	const struct chan_queues *queues_tx;
+	struct chan_queues td_queue;
+	u16 first_completion_queue;
+	u16 qmgr_num_pend;
+};
+
+static struct cppi41_channel *to_cpp41_chan(struct dma_chan *c)
+{
+	return container_of(c, struct cppi41_channel, chan);
+}
+
+static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc)
+{
+	struct cppi41_channel *c;
+	u32 descs_size;
+	u32 desc_num;
+
+	descs_size = sizeof(struct cppi41_desc) * ALLOC_DECS_NUM;
+
+	if (!((desc >= cdd->descs_phys) &&
+			(desc < (cdd->descs_phys + descs_size)))) {
+		return NULL;
+	}
+
+	desc_num = (desc - cdd->descs_phys) / sizeof(struct cppi41_desc);
+	BUG_ON(desc_num >= ALLOC_DECS_NUM);
+	c = cdd->chan_busy[desc_num];
+	cdd->chan_busy[desc_num] = NULL;
+
+	/* Usecount for chan_busy[], paired with push_desc_queue() */
+	pm_runtime_put(cdd->ddev.dev);
+
+	return c;
+}
+
+static void cppi_writel(u32 val, void *__iomem *mem)
+{
+	__raw_writel(val, mem);
+}
+
+static u32 cppi_readl(void *__iomem *mem)
+{
+	return __raw_readl(mem);
+}
+
+static u32 pd_trans_len(u32 val)
+{
+	return val & ((1 << (DESC_LENGTH_BITS_NUM + 1)) - 1);
+}
+
+static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num)
+{
+	u32 desc;
+
+	desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num));
+	desc &= ~0x1f;
+	return desc;
+}
+
+static irqreturn_t cppi41_irq(int irq, void *data)
+{
+	struct cppi41_dd *cdd = data;
+	u16 first_completion_queue = cdd->first_completion_queue;
+	u16 qmgr_num_pend = cdd->qmgr_num_pend;
+	struct cppi41_channel *c;
+	int i;
+
+	for (i = QMGR_PENDING_SLOT_Q(first_completion_queue); i < qmgr_num_pend;
+			i++) {
+		u32 val;
+		u32 q_num;
+
+		val = cppi_readl(cdd->qmgr_mem + QMGR_PEND(i));
+		if (i == QMGR_PENDING_SLOT_Q(first_completion_queue) && val) {
+			u32 mask;
+			/* set corresponding bit for completetion Q 93 */
+			mask = 1 << QMGR_PENDING_BIT_Q(first_completion_queue);
+			/* not set all bits for queues less than Q 93 */
+			mask--;
+			/* now invert and keep only Q 93+ set */
+			val &= ~mask;
+		}
+
+		if (val)
+			__iormb();
+
+		while (val) {
+			u32 desc, len;
+
+			/*
+			 * This should never trigger, see the comments in
+			 * push_desc_queue()
+			 */
+			WARN_ON(cdd->is_suspended);
+
+			q_num = __fls(val);
+			val &= ~(1 << q_num);
+			q_num += 32 * i;
+			desc = cppi41_pop_desc(cdd, q_num);
+			c = desc_to_chan(cdd, desc);
+			if (WARN_ON(!c)) {
+				pr_err("%s() q %d desc %08x\n", __func__,
+						q_num, desc);
+				continue;
+			}
+
+			if (c->desc->pd2 & PD2_ZERO_LENGTH)
+				len = 0;
+			else
+				len = pd_trans_len(c->desc->pd0);
+
+			c->residue = pd_trans_len(c->desc->pd6) - len;
+			dma_cookie_complete(&c->txd);
+			dmaengine_desc_get_callback_invoke(&c->txd, NULL);
+		}
+	}
+	return IRQ_HANDLED;
+}
+
+static dma_cookie_t cppi41_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+	dma_cookie_t cookie;
+
+	cookie = dma_cookie_assign(tx);
+
+	return cookie;
+}
+
+static int cppi41_dma_alloc_chan_resources(struct dma_chan *chan)
+{
+	struct cppi41_channel *c = to_cpp41_chan(chan);
+	struct cppi41_dd *cdd = c->cdd;
+	int error;
+
+	error = pm_runtime_get_sync(cdd->ddev.dev);
+	if (error < 0) {
+		dev_err(cdd->ddev.dev, "%s pm runtime get: %i\n",
+			__func__, error);
+		pm_runtime_put_noidle(cdd->ddev.dev);
+
+		return error;
+	}
+
+	dma_cookie_init(chan);
+	dma_async_tx_descriptor_init(&c->txd, chan);
+	c->txd.tx_submit = cppi41_tx_submit;
+
+	if (!c->is_tx)
+		cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
+
+	pm_runtime_mark_last_busy(cdd->ddev.dev);
+	pm_runtime_put_autosuspend(cdd->ddev.dev);
+
+	return 0;
+}
+
+static void cppi41_dma_free_chan_resources(struct dma_chan *chan)
+{
+	struct cppi41_channel *c = to_cpp41_chan(chan);
+	struct cppi41_dd *cdd = c->cdd;
+	int error;
+
+	error = pm_runtime_get_sync(cdd->ddev.dev);
+	if (error < 0) {
+		pm_runtime_put_noidle(cdd->ddev.dev);
+
+		return;
+	}
+
+	WARN_ON(!list_empty(&cdd->pending));
+
+	pm_runtime_mark_last_busy(cdd->ddev.dev);
+	pm_runtime_put_autosuspend(cdd->ddev.dev);
+}
+
+static enum dma_status cppi41_dma_tx_status(struct dma_chan *chan,
+	dma_cookie_t cookie, struct dma_tx_state *txstate)
+{
+	struct cppi41_channel *c = to_cpp41_chan(chan);
+	enum dma_status ret;
+
+	ret = dma_cookie_status(chan, cookie, txstate);
+
+	dma_set_residue(txstate, c->residue);
+
+	return ret;
+}
+
+static void push_desc_queue(struct cppi41_channel *c)
+{
+	struct cppi41_dd *cdd = c->cdd;
+	u32 desc_num;
+	u32 desc_phys;
+	u32 reg;
+
+	c->residue = 0;
+
+	reg = GCR_CHAN_ENABLE;
+	if (!c->is_tx) {
+		reg |= GCR_STARV_RETRY;
+		reg |= GCR_DESC_TYPE_HOST;
+		reg |= c->q_comp_num;
+	}
+
+	cppi_writel(reg, c->gcr_reg);
+
+	/*
+	 * We don't use writel() but __raw_writel() so we have to make sure
+	 * that the DMA descriptor in coherent memory made to the main memory
+	 * before starting the dma engine.
+	 */
+	__iowmb();
+
+	/*
+	 * DMA transfers can take at least 200ms to complete with USB mass
+	 * storage connected. To prevent autosuspend timeouts, we must use
+	 * pm_runtime_get/put() when chan_busy[] is modified. This will get
+	 * cleared in desc_to_chan() or cppi41_stop_chan() depending on the
+	 * outcome of the transfer.
+	 */
+	pm_runtime_get(cdd->ddev.dev);
+
+	desc_phys = lower_32_bits(c->desc_phys);
+	desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
+	WARN_ON(cdd->chan_busy[desc_num]);
+	cdd->chan_busy[desc_num] = c;
+
+	reg = (sizeof(struct cppi41_desc) - 24) / 4;
+	reg |= desc_phys;
+	cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));
+}
+
+/*
+ * Caller must hold cdd->lock to prevent push_desc_queue()
+ * getting called out of order. We have both cppi41_dma_issue_pending()
+ * and cppi41_runtime_resume() call this function.
+ */
+static void cppi41_run_queue(struct cppi41_dd *cdd)
+{
+	struct cppi41_channel *c, *_c;
+
+	list_for_each_entry_safe(c, _c, &cdd->pending, node) {
+		push_desc_queue(c);
+		list_del(&c->node);
+	}
+}
+
+static void cppi41_dma_issue_pending(struct dma_chan *chan)
+{
+	struct cppi41_channel *c = to_cpp41_chan(chan);
+	struct cppi41_dd *cdd = c->cdd;
+	unsigned long flags;
+	int error;
+
+	error = pm_runtime_get(cdd->ddev.dev);
+	if ((error != -EINPROGRESS) && error < 0) {
+		pm_runtime_put_noidle(cdd->ddev.dev);
+		dev_err(cdd->ddev.dev, "Failed to pm_runtime_get: %i\n",
+			error);
+
+		return;
+	}
+
+	spin_lock_irqsave(&cdd->lock, flags);
+	list_add_tail(&c->node, &cdd->pending);
+	if (!cdd->is_suspended)
+		cppi41_run_queue(cdd);
+	spin_unlock_irqrestore(&cdd->lock, flags);
+
+	pm_runtime_mark_last_busy(cdd->ddev.dev);
+	pm_runtime_put_autosuspend(cdd->ddev.dev);
+}
+
+static u32 get_host_pd0(u32 length)
+{
+	u32 reg;
+
+	reg = DESC_TYPE_HOST << DESC_TYPE;
+	reg |= length;
+
+	return reg;
+}
+
+static u32 get_host_pd1(struct cppi41_channel *c)
+{
+	u32 reg;
+
+	reg = 0;
+
+	return reg;
+}
+
+static u32 get_host_pd2(struct cppi41_channel *c)
+{
+	u32 reg;
+
+	reg = DESC_TYPE_USB;
+	reg |= c->q_comp_num;
+
+	return reg;
+}
+
+static u32 get_host_pd3(u32 length)
+{
+	u32 reg;
+
+	/* PD3 = packet size */
+	reg = length;
+
+	return reg;
+}
+
+static u32 get_host_pd6(u32 length)
+{
+	u32 reg;
+
+	/* PD6 buffer size */
+	reg = DESC_PD_COMPLETE;
+	reg |= length;
+
+	return reg;
+}
+
+static u32 get_host_pd4_or_7(u32 addr)
+{
+	u32 reg;
+
+	reg = addr;
+
+	return reg;
+}
+
+static u32 get_host_pd5(void)
+{
+	u32 reg;
+
+	reg = 0;
+
+	return reg;
+}
+
+static struct dma_async_tx_descriptor *cppi41_dma_prep_slave_sg(
+	struct dma_chan *chan, struct scatterlist *sgl, unsigned sg_len,
+	enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
+{
+	struct cppi41_channel *c = to_cpp41_chan(chan);
+	struct cppi41_desc *d;
+	struct scatterlist *sg;
+	unsigned int i;
+
+	d = c->desc;
+	for_each_sg(sgl, sg, sg_len, i) {
+		u32 addr;
+		u32 len;
+
+		/* We need to use more than one desc once musb supports sg */
+		addr = lower_32_bits(sg_dma_address(sg));
+		len = sg_dma_len(sg);
+
+		d->pd0 = get_host_pd0(len);
+		d->pd1 = get_host_pd1(c);
+		d->pd2 = get_host_pd2(c);
+		d->pd3 = get_host_pd3(len);
+		d->pd4 = get_host_pd4_or_7(addr);
+		d->pd5 = get_host_pd5();
+		d->pd6 = get_host_pd6(len);
+		d->pd7 = get_host_pd4_or_7(addr);
+
+		d++;
+	}
+
+	return &c->txd;
+}
+
+static void cppi41_compute_td_desc(struct cppi41_desc *d)
+{
+	d->pd0 = DESC_TYPE_TEARD << DESC_TYPE;
+}
+
+static int cppi41_tear_down_chan(struct cppi41_channel *c)
+{
+	struct dmaengine_result abort_result;
+	struct cppi41_dd *cdd = c->cdd;
+	struct cppi41_desc *td;
+	u32 reg;
+	u32 desc_phys;
+	u32 td_desc_phys;
+
+	td = cdd->cd;
+	td += cdd->first_td_desc;
+
+	td_desc_phys = cdd->descs_phys;
+	td_desc_phys += cdd->first_td_desc * sizeof(struct cppi41_desc);
+
+	if (!c->td_queued) {
+		cppi41_compute_td_desc(td);
+		__iowmb();
+
+		reg = (sizeof(struct cppi41_desc) - 24) / 4;
+		reg |= td_desc_phys;
+		cppi_writel(reg, cdd->qmgr_mem +
+				QMGR_QUEUE_D(cdd->td_queue.submit));
+
+		reg = GCR_CHAN_ENABLE;
+		if (!c->is_tx) {
+			reg |= GCR_STARV_RETRY;
+			reg |= GCR_DESC_TYPE_HOST;
+			reg |= cdd->td_queue.complete;
+		}
+		reg |= GCR_TEARDOWN;
+		cppi_writel(reg, c->gcr_reg);
+		c->td_queued = 1;
+		c->td_retry = 500;
+	}
+
+	if (!c->td_seen || !c->td_desc_seen) {
+
+		desc_phys = cppi41_pop_desc(cdd, cdd->td_queue.complete);
+		if (!desc_phys && c->is_tx)
+			desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);
+
+		if (desc_phys == c->desc_phys) {
+			c->td_desc_seen = 1;
+
+		} else if (desc_phys == td_desc_phys) {
+			u32 pd0;
+
+			__iormb();
+			pd0 = td->pd0;
+			WARN_ON((pd0 >> DESC_TYPE) != DESC_TYPE_TEARD);
+			WARN_ON(!c->is_tx && !(pd0 & TD_DESC_IS_RX));
+			WARN_ON((pd0 & 0x1f) != c->port_num);
+			c->td_seen = 1;
+		} else if (desc_phys) {
+			WARN_ON_ONCE(1);
+		}
+	}
+	c->td_retry--;
+	/*
+	 * If the TX descriptor / channel is in use, the caller needs to poke
+	 * his TD bit multiple times. After that he hardware releases the
+	 * transfer descriptor followed by TD descriptor. Waiting seems not to
+	 * cause any difference.
+	 * RX seems to be thrown out right away. However once the TearDown
+	 * descriptor gets through we are done. If we have seens the transfer
+	 * descriptor before the TD we fetch it from enqueue, it has to be
+	 * there waiting for us.
+	 */
+	if (!c->td_seen && c->td_retry) {
+		udelay(1);
+		return -EAGAIN;
+	}
+	WARN_ON(!c->td_retry);
+
+	if (!c->td_desc_seen) {
+		desc_phys = cppi41_pop_desc(cdd, c->q_num);
+		if (!desc_phys)
+			desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);
+		WARN_ON(!desc_phys);
+	}
+
+	c->td_queued = 0;
+	c->td_seen = 0;
+	c->td_desc_seen = 0;
+	cppi_writel(0, c->gcr_reg);
+
+	/* Invoke the callback to do the necessary clean-up */
+	abort_result.result = DMA_TRANS_ABORTED;
+	dma_cookie_complete(&c->txd);
+	dmaengine_desc_get_callback_invoke(&c->txd, &abort_result);
+
+	return 0;
+}
+
+static int cppi41_stop_chan(struct dma_chan *chan)
+{
+	struct cppi41_channel *c = to_cpp41_chan(chan);
+	struct cppi41_dd *cdd = c->cdd;
+	u32 desc_num;
+	u32 desc_phys;
+	int ret;
+
+	desc_phys = lower_32_bits(c->desc_phys);
+	desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
+	if (!cdd->chan_busy[desc_num])
+		return 0;
+
+	ret = cppi41_tear_down_chan(c);
+	if (ret)
+		return ret;
+
+	WARN_ON(!cdd->chan_busy[desc_num]);
+	cdd->chan_busy[desc_num] = NULL;
+
+	/* Usecount for chan_busy[], paired with push_desc_queue() */
+	pm_runtime_put(cdd->ddev.dev);
+
+	return 0;
+}
+
+static int cppi41_add_chans(struct device *dev, struct cppi41_dd *cdd)
+{
+	struct cppi41_channel *cchan, *chans;
+	int i;
+	u32 n_chans = cdd->n_chans;
+
+	/*
+	 * The channels can only be used as TX or as RX. So we add twice
+	 * that much dma channels because USB can only do RX or TX.
+	 */
+	n_chans *= 2;
+
+	chans = devm_kcalloc(dev, n_chans, sizeof(*chans), GFP_KERNEL);
+	if (!chans)
+		return -ENOMEM;
+
+	for (i = 0; i < n_chans; i++) {
+		cchan = &chans[i];
+
+		cchan->cdd = cdd;
+		if (i & 1) {
+			cchan->gcr_reg = cdd->ctrl_mem + DMA_TXGCR(i >> 1);
+			cchan->is_tx = 1;
+		} else {
+			cchan->gcr_reg = cdd->ctrl_mem + DMA_RXGCR(i >> 1);
+			cchan->is_tx = 0;
+		}
+		cchan->port_num = i >> 1;
+		cchan->desc = &cdd->cd[i];
+		cchan->desc_phys = cdd->descs_phys;
+		cchan->desc_phys += i * sizeof(struct cppi41_desc);
+		cchan->chan.device = &cdd->ddev;
+		list_add_tail(&cchan->chan.device_node, &cdd->ddev.channels);
+	}
+	cdd->first_td_desc = n_chans;
+
+	return 0;
+}
+
+static void purge_descs(struct device *dev, struct cppi41_dd *cdd)
+{
+	unsigned int mem_decs;
+	int i;
+
+	mem_decs = ALLOC_DECS_NUM * sizeof(struct cppi41_desc);
+
+	for (i = 0; i < DESCS_AREAS; i++) {
+
+		cppi_writel(0, cdd->qmgr_mem + QMGR_MEMBASE(i));
+		cppi_writel(0, cdd->qmgr_mem + QMGR_MEMCTRL(i));
+
+		dma_free_coherent(dev, mem_decs, cdd->cd,
+				cdd->descs_phys);
+	}
+}
+
+static void disable_sched(struct cppi41_dd *cdd)
+{
+	cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
+}
+
+static void deinit_cppi41(struct device *dev, struct cppi41_dd *cdd)
+{
+	disable_sched(cdd);
+
+	purge_descs(dev, cdd);
+
+	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
+	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
+	dma_free_coherent(dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch,
+			cdd->scratch_phys);
+}
+
+static int init_descs(struct device *dev, struct cppi41_dd *cdd)
+{
+	unsigned int desc_size;
+	unsigned int mem_decs;
+	int i;
+	u32 reg;
+	u32 idx;
+
+	BUILD_BUG_ON(sizeof(struct cppi41_desc) &
+			(sizeof(struct cppi41_desc) - 1));
+	BUILD_BUG_ON(sizeof(struct cppi41_desc) < 32);
+	BUILD_BUG_ON(ALLOC_DECS_NUM < 32);
+
+	desc_size = sizeof(struct cppi41_desc);
+	mem_decs = ALLOC_DECS_NUM * desc_size;
+
+	idx = 0;
+	for (i = 0; i < DESCS_AREAS; i++) {
+
+		reg = idx << QMGR_MEMCTRL_IDX_SH;
+		reg |= (ilog2(desc_size) - 5) << QMGR_MEMCTRL_DESC_SH;
+		reg |= ilog2(ALLOC_DECS_NUM) - 5;
+
+		BUILD_BUG_ON(DESCS_AREAS != 1);
+		cdd->cd = dma_alloc_coherent(dev, mem_decs,
+				&cdd->descs_phys, GFP_KERNEL);
+		if (!cdd->cd)
+			return -ENOMEM;
+
+		cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));
+		cppi_writel(reg, cdd->qmgr_mem + QMGR_MEMCTRL(i));
+
+		idx += ALLOC_DECS_NUM;
+	}
+	return 0;
+}
+
+static void init_sched(struct cppi41_dd *cdd)
+{
+	unsigned ch;
+	unsigned word;
+	u32 reg;
+
+	word = 0;
+	cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
+	for (ch = 0; ch < cdd->n_chans; ch += 2) {
+
+		reg = SCHED_ENTRY0_CHAN(ch);
+		reg |= SCHED_ENTRY1_CHAN(ch) | SCHED_ENTRY1_IS_RX;
+
+		reg |= SCHED_ENTRY2_CHAN(ch + 1);
+		reg |= SCHED_ENTRY3_CHAN(ch + 1) | SCHED_ENTRY3_IS_RX;
+		cppi_writel(reg, cdd->sched_mem + DMA_SCHED_WORD(word));
+		word++;
+	}
+	reg = cdd->n_chans * 2 - 1;
+	reg |= DMA_SCHED_CTRL_EN;
+	cppi_writel(reg, cdd->sched_mem + DMA_SCHED_CTRL);
+}
+
+static int init_cppi41(struct device *dev, struct cppi41_dd *cdd)
+{
+	int ret;
+
+	BUILD_BUG_ON(QMGR_SCRATCH_SIZE > ((1 << 14) - 1));
+	cdd->qmgr_scratch = dma_alloc_coherent(dev, QMGR_SCRATCH_SIZE,
+			&cdd->scratch_phys, GFP_KERNEL);
+	if (!cdd->qmgr_scratch)
+		return -ENOMEM;
+
+	cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);
+	cppi_writel(TOTAL_DESCS_NUM, cdd->qmgr_mem + QMGR_LRAM_SIZE);
+	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
+
+	ret = init_descs(dev, cdd);
+	if (ret)
+		goto err_td;
+
+	cppi_writel(cdd->td_queue.submit, cdd->ctrl_mem + DMA_TDFDQ);
+	init_sched(cdd);
+
+	return 0;
+err_td:
+	deinit_cppi41(dev, cdd);
+	return ret;
+}
+
+static struct platform_driver cpp41_dma_driver;
+/*
+ * The param format is:
+ * X Y
+ * X: Port
+ * Y: 0 = RX else TX
+ */
+#define INFO_PORT	0
+#define INFO_IS_TX	1
+
+static bool cpp41_dma_filter_fn(struct dma_chan *chan, void *param)
+{
+	struct cppi41_channel *cchan;
+	struct cppi41_dd *cdd;
+	const struct chan_queues *queues;
+	u32 *num = param;
+
+	if (chan->device->dev->driver != &cpp41_dma_driver.driver)
+		return false;
+
+	cchan = to_cpp41_chan(chan);
+
+	if (cchan->port_num != num[INFO_PORT])
+		return false;
+
+	if (cchan->is_tx && !num[INFO_IS_TX])
+		return false;
+	cdd = cchan->cdd;
+	if (cchan->is_tx)
+		queues = cdd->queues_tx;
+	else
+		queues = cdd->queues_rx;
+
+	BUILD_BUG_ON(ARRAY_SIZE(am335x_usb_queues_rx) !=
+		     ARRAY_SIZE(am335x_usb_queues_tx));
+	if (WARN_ON(cchan->port_num >= ARRAY_SIZE(am335x_usb_queues_rx)))
+		return false;
+
+	cchan->q_num = queues[cchan->port_num].submit;
+	cchan->q_comp_num = queues[cchan->port_num].complete;
+	return true;
+}
+
+static struct of_dma_filter_info cpp41_dma_info = {
+	.filter_fn = cpp41_dma_filter_fn,
+};
+
+static struct dma_chan *cppi41_dma_xlate(struct of_phandle_args *dma_spec,
+		struct of_dma *ofdma)
+{
+	int count = dma_spec->args_count;
+	struct of_dma_filter_info *info = ofdma->of_dma_data;
+
+	if (!info || !info->filter_fn)
+		return NULL;
+
+	if (count != 2)
+		return NULL;
+
+	return dma_request_channel(info->dma_cap, info->filter_fn,
+			&dma_spec->args[0]);
+}
+
+static const struct cppi_glue_infos am335x_usb_infos = {
+	.queues_rx = am335x_usb_queues_rx,
+	.queues_tx = am335x_usb_queues_tx,
+	.td_queue = { .submit = 31, .complete = 0 },
+	.first_completion_queue = 93,
+	.qmgr_num_pend = 5,
+};
+
+static const struct cppi_glue_infos da8xx_usb_infos = {
+	.queues_rx = da8xx_usb_queues_rx,
+	.queues_tx = da8xx_usb_queues_tx,
+	.td_queue = { .submit = 31, .complete = 0 },
+	.first_completion_queue = 24,
+	.qmgr_num_pend = 2,
+};
+
+static const struct of_device_id cppi41_dma_ids[] = {
+	{ .compatible = "ti,am3359-cppi41", .data = &am335x_usb_infos},
+	{ .compatible = "ti,da830-cppi41", .data = &da8xx_usb_infos},
+	{},
+};
+MODULE_DEVICE_TABLE(of, cppi41_dma_ids);
+
+static const struct cppi_glue_infos *get_glue_info(struct device *dev)
+{
+	const struct of_device_id *of_id;
+
+	of_id = of_match_node(cppi41_dma_ids, dev->of_node);
+	if (!of_id)
+		return NULL;
+	return of_id->data;
+}
+
+#define CPPI41_DMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
+				BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
+				BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
+				BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
+
+static int cppi41_dma_probe(struct platform_device *pdev)
+{
+	struct cppi41_dd *cdd;
+	struct device *dev = &pdev->dev;
+	const struct cppi_glue_infos *glue_info;
+	struct resource *mem;
+	int index;
+	int irq;
+	int ret;
+
+	glue_info = get_glue_info(dev);
+	if (!glue_info)
+		return -EINVAL;
+
+	cdd = devm_kzalloc(&pdev->dev, sizeof(*cdd), GFP_KERNEL);
+	if (!cdd)
+		return -ENOMEM;
+
+	dma_cap_set(DMA_SLAVE, cdd->ddev.cap_mask);
+	cdd->ddev.device_alloc_chan_resources = cppi41_dma_alloc_chan_resources;
+	cdd->ddev.device_free_chan_resources = cppi41_dma_free_chan_resources;
+	cdd->ddev.device_tx_status = cppi41_dma_tx_status;
+	cdd->ddev.device_issue_pending = cppi41_dma_issue_pending;
+	cdd->ddev.device_prep_slave_sg = cppi41_dma_prep_slave_sg;
+	cdd->ddev.device_terminate_all = cppi41_stop_chan;
+	cdd->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+	cdd->ddev.src_addr_widths = CPPI41_DMA_BUSWIDTHS;
+	cdd->ddev.dst_addr_widths = CPPI41_DMA_BUSWIDTHS;
+	cdd->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+	cdd->ddev.dev = dev;
+	INIT_LIST_HEAD(&cdd->ddev.channels);
+	cpp41_dma_info.dma_cap = cdd->ddev.cap_mask;
+
+	index = of_property_match_string(dev->of_node,
+					 "reg-names", "controller");
+	if (index < 0)
+		return index;
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, index);
+	cdd->ctrl_mem = devm_ioremap_resource(dev, mem);
+	if (IS_ERR(cdd->ctrl_mem))
+		return PTR_ERR(cdd->ctrl_mem);
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, index + 1);
+	cdd->sched_mem = devm_ioremap_resource(dev, mem);
+	if (IS_ERR(cdd->sched_mem))
+		return PTR_ERR(cdd->sched_mem);
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, index + 2);
+	cdd->qmgr_mem = devm_ioremap_resource(dev, mem);
+	if (IS_ERR(cdd->qmgr_mem))
+		return PTR_ERR(cdd->qmgr_mem);
+
+	spin_lock_init(&cdd->lock);
+	INIT_LIST_HEAD(&cdd->pending);
+
+	platform_set_drvdata(pdev, cdd);
+
+	pm_runtime_enable(dev);
+	pm_runtime_set_autosuspend_delay(dev, 100);
+	pm_runtime_use_autosuspend(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0)
+		goto err_get_sync;
+
+	cdd->queues_rx = glue_info->queues_rx;
+	cdd->queues_tx = glue_info->queues_tx;
+	cdd->td_queue = glue_info->td_queue;
+	cdd->qmgr_num_pend = glue_info->qmgr_num_pend;
+	cdd->first_completion_queue = glue_info->first_completion_queue;
+
+	ret = of_property_read_u32(dev->of_node,
+				   "#dma-channels", &cdd->n_chans);
+	if (ret)
+		goto err_get_n_chans;
+
+	ret = init_cppi41(dev, cdd);
+	if (ret)
+		goto err_init_cppi;
+
+	ret = cppi41_add_chans(dev, cdd);
+	if (ret)
+		goto err_chans;
+
+	irq = irq_of_parse_and_map(dev->of_node, 0);
+	if (!irq) {
+		ret = -EINVAL;
+		goto err_chans;
+	}
+
+	ret = devm_request_irq(&pdev->dev, irq, cppi41_irq, IRQF_SHARED,
+			dev_name(dev), cdd);
+	if (ret)
+		goto err_chans;
+	cdd->irq = irq;
+
+	ret = dma_async_device_register(&cdd->ddev);
+	if (ret)
+		goto err_chans;
+
+	ret = of_dma_controller_register(dev->of_node,
+			cppi41_dma_xlate, &cpp41_dma_info);
+	if (ret)
+		goto err_of;
+
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
+
+	return 0;
+err_of:
+	dma_async_device_unregister(&cdd->ddev);
+err_chans:
+	deinit_cppi41(dev, cdd);
+err_init_cppi:
+	pm_runtime_dont_use_autosuspend(dev);
+err_get_n_chans:
+err_get_sync:
+	pm_runtime_put_sync(dev);
+	pm_runtime_disable(dev);
+	return ret;
+}
+
+static int cppi41_dma_remove(struct platform_device *pdev)
+{
+	struct cppi41_dd *cdd = platform_get_drvdata(pdev);
+	int error;
+
+	error = pm_runtime_get_sync(&pdev->dev);
+	if (error < 0)
+		dev_err(&pdev->dev, "%s could not pm_runtime_get: %i\n",
+			__func__, error);
+	of_dma_controller_free(pdev->dev.of_node);
+	dma_async_device_unregister(&cdd->ddev);
+
+	devm_free_irq(&pdev->dev, cdd->irq, cdd);
+	deinit_cppi41(&pdev->dev, cdd);
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	return 0;
+}
+
+static int __maybe_unused cppi41_suspend(struct device *dev)
+{
+	struct cppi41_dd *cdd = dev_get_drvdata(dev);
+
+	cdd->dma_tdfdq = cppi_readl(cdd->ctrl_mem + DMA_TDFDQ);
+	disable_sched(cdd);
+
+	return 0;
+}
+
+static int __maybe_unused cppi41_resume(struct device *dev)
+{
+	struct cppi41_dd *cdd = dev_get_drvdata(dev);
+	struct cppi41_channel *c;
+	int i;
+
+	for (i = 0; i < DESCS_AREAS; i++)
+		cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));
+
+	list_for_each_entry(c, &cdd->ddev.channels, chan.device_node)
+		if (!c->is_tx)
+			cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
+
+	init_sched(cdd);
+
+	cppi_writel(cdd->dma_tdfdq, cdd->ctrl_mem + DMA_TDFDQ);
+	cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);
+	cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE);
+	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
+
+	return 0;
+}
+
+static int __maybe_unused cppi41_runtime_suspend(struct device *dev)
+{
+	struct cppi41_dd *cdd = dev_get_drvdata(dev);
+	unsigned long flags;
+
+	spin_lock_irqsave(&cdd->lock, flags);
+	cdd->is_suspended = true;
+	WARN_ON(!list_empty(&cdd->pending));
+	spin_unlock_irqrestore(&cdd->lock, flags);
+
+	return 0;
+}
+
+static int __maybe_unused cppi41_runtime_resume(struct device *dev)
+{
+	struct cppi41_dd *cdd = dev_get_drvdata(dev);
+	unsigned long flags;
+
+	spin_lock_irqsave(&cdd->lock, flags);
+	cdd->is_suspended = false;
+	cppi41_run_queue(cdd);
+	spin_unlock_irqrestore(&cdd->lock, flags);
+
+	return 0;
+}
+
+static const struct dev_pm_ops cppi41_pm_ops = {
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(cppi41_suspend, cppi41_resume)
+	SET_RUNTIME_PM_OPS(cppi41_runtime_suspend,
+			   cppi41_runtime_resume,
+			   NULL)
+};
+
+static struct platform_driver cpp41_dma_driver = {
+	.probe  = cppi41_dma_probe,
+	.remove = cppi41_dma_remove,
+	.driver = {
+		.name = "cppi41-dma-engine",
+		.pm = &cppi41_pm_ops,
+		.of_match_table = of_match_ptr(cppi41_dma_ids),
+	},
+};
+
+module_platform_driver(cpp41_dma_driver);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Sebastian Andrzej Siewior <bigeasy@linutronix.de>");
diff --git a/drivers/dma/ti/dma-crossbar.c b/drivers/dma/ti/dma-crossbar.c
new file mode 100644
index 000000000000..9272b173c746
--- /dev/null
+++ b/drivers/dma/ti/dma-crossbar.c
@@ -0,0 +1,478 @@
+/*
+ *  Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com
+ *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+#include <linux/slab.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/list.h>
+#include <linux/io.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_dma.h>
+
+#define TI_XBAR_DRA7		0
+#define TI_XBAR_AM335X		1
+static const u32 ti_xbar_type[] = {
+	[TI_XBAR_DRA7] = TI_XBAR_DRA7,
+	[TI_XBAR_AM335X] = TI_XBAR_AM335X,
+};
+
+static const struct of_device_id ti_dma_xbar_match[] = {
+	{
+		.compatible = "ti,dra7-dma-crossbar",
+		.data = &ti_xbar_type[TI_XBAR_DRA7],
+	},
+	{
+		.compatible = "ti,am335x-edma-crossbar",
+		.data = &ti_xbar_type[TI_XBAR_AM335X],
+	},
+	{},
+};
+
+/* Crossbar on AM335x/AM437x family */
+#define TI_AM335X_XBAR_LINES	64
+
+struct ti_am335x_xbar_data {
+	void __iomem *iomem;
+
+	struct dma_router dmarouter;
+
+	u32 xbar_events; /* maximum number of events to select in xbar */
+	u32 dma_requests; /* number of DMA requests on eDMA */
+};
+
+struct ti_am335x_xbar_map {
+	u16 dma_line;
+	u8 mux_val;
+};
+
+static inline void ti_am335x_xbar_write(void __iomem *iomem, int event, u8 val)
+{
+	/*
+	 * TPCC_EVT_MUX_60_63 register layout is different than the
+	 * rest, in the sense, that event 63 is mapped to lowest byte
+	 * and event 60 is mapped to highest, handle it separately.
+	 */
+	if (event >= 60 && event <= 63)
+		writeb_relaxed(val, iomem + (63 - event % 4));
+	else
+		writeb_relaxed(val, iomem + event);
+}
+
+static void ti_am335x_xbar_free(struct device *dev, void *route_data)
+{
+	struct ti_am335x_xbar_data *xbar = dev_get_drvdata(dev);
+	struct ti_am335x_xbar_map *map = route_data;
+
+	dev_dbg(dev, "Unmapping XBAR event %u on channel %u\n",
+		map->mux_val, map->dma_line);
+
+	ti_am335x_xbar_write(xbar->iomem, map->dma_line, 0);
+	kfree(map);
+}
+
+static void *ti_am335x_xbar_route_allocate(struct of_phandle_args *dma_spec,
+					   struct of_dma *ofdma)
+{
+	struct platform_device *pdev = of_find_device_by_node(ofdma->of_node);
+	struct ti_am335x_xbar_data *xbar = platform_get_drvdata(pdev);
+	struct ti_am335x_xbar_map *map;
+
+	if (dma_spec->args_count != 3)
+		return ERR_PTR(-EINVAL);
+
+	if (dma_spec->args[2] >= xbar->xbar_events) {
+		dev_err(&pdev->dev, "Invalid XBAR event number: %d\n",
+			dma_spec->args[2]);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (dma_spec->args[0] >= xbar->dma_requests) {
+		dev_err(&pdev->dev, "Invalid DMA request line number: %d\n",
+			dma_spec->args[0]);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* The of_node_put() will be done in the core for the node */
+	dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0);
+	if (!dma_spec->np) {
+		dev_err(&pdev->dev, "Can't get DMA master\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	map = kzalloc(sizeof(*map), GFP_KERNEL);
+	if (!map) {
+		of_node_put(dma_spec->np);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	map->dma_line = (u16)dma_spec->args[0];
+	map->mux_val = (u8)dma_spec->args[2];
+
+	dma_spec->args[2] = 0;
+	dma_spec->args_count = 2;
+
+	dev_dbg(&pdev->dev, "Mapping XBAR event%u to DMA%u\n",
+		map->mux_val, map->dma_line);
+
+	ti_am335x_xbar_write(xbar->iomem, map->dma_line, map->mux_val);
+
+	return map;
+}
+
+static const struct of_device_id ti_am335x_master_match[] = {
+	{ .compatible = "ti,edma3-tpcc", },
+	{},
+};
+
+static int ti_am335x_xbar_probe(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	const struct of_device_id *match;
+	struct device_node *dma_node;
+	struct ti_am335x_xbar_data *xbar;
+	struct resource *res;
+	void __iomem *iomem;
+	int i, ret;
+
+	if (!node)
+		return -ENODEV;
+
+	xbar = devm_kzalloc(&pdev->dev, sizeof(*xbar), GFP_KERNEL);
+	if (!xbar)
+		return -ENOMEM;
+
+	dma_node = of_parse_phandle(node, "dma-masters", 0);
+	if (!dma_node) {
+		dev_err(&pdev->dev, "Can't get DMA master node\n");
+		return -ENODEV;
+	}
+
+	match = of_match_node(ti_am335x_master_match, dma_node);
+	if (!match) {
+		dev_err(&pdev->dev, "DMA master is not supported\n");
+		of_node_put(dma_node);
+		return -EINVAL;
+	}
+
+	if (of_property_read_u32(dma_node, "dma-requests",
+				 &xbar->dma_requests)) {
+		dev_info(&pdev->dev,
+			 "Missing XBAR output information, using %u.\n",
+			 TI_AM335X_XBAR_LINES);
+		xbar->dma_requests = TI_AM335X_XBAR_LINES;
+	}
+	of_node_put(dma_node);
+
+	if (of_property_read_u32(node, "dma-requests", &xbar->xbar_events)) {
+		dev_info(&pdev->dev,
+			 "Missing XBAR input information, using %u.\n",
+			 TI_AM335X_XBAR_LINES);
+		xbar->xbar_events = TI_AM335X_XBAR_LINES;
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	iomem = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(iomem))
+		return PTR_ERR(iomem);
+
+	xbar->iomem = iomem;
+
+	xbar->dmarouter.dev = &pdev->dev;
+	xbar->dmarouter.route_free = ti_am335x_xbar_free;
+
+	platform_set_drvdata(pdev, xbar);
+
+	/* Reset the crossbar */
+	for (i = 0; i < xbar->dma_requests; i++)
+		ti_am335x_xbar_write(xbar->iomem, i, 0);
+
+	ret = of_dma_router_register(node, ti_am335x_xbar_route_allocate,
+				     &xbar->dmarouter);
+
+	return ret;
+}
+
+/* Crossbar on DRA7xx family */
+#define TI_DRA7_XBAR_OUTPUTS	127
+#define TI_DRA7_XBAR_INPUTS	256
+
+struct ti_dra7_xbar_data {
+	void __iomem *iomem;
+
+	struct dma_router dmarouter;
+	struct mutex mutex;
+	unsigned long *dma_inuse;
+
+	u16 safe_val; /* Value to rest the crossbar lines */
+	u32 xbar_requests; /* number of DMA requests connected to XBAR */
+	u32 dma_requests; /* number of DMA requests forwarded to DMA */
+	u32 dma_offset;
+};
+
+struct ti_dra7_xbar_map {
+	u16 xbar_in;
+	int xbar_out;
+};
+
+static inline void ti_dra7_xbar_write(void __iomem *iomem, int xbar, u16 val)
+{
+	writew_relaxed(val, iomem + (xbar * 2));
+}
+
+static void ti_dra7_xbar_free(struct device *dev, void *route_data)
+{
+	struct ti_dra7_xbar_data *xbar = dev_get_drvdata(dev);
+	struct ti_dra7_xbar_map *map = route_data;
+
+	dev_dbg(dev, "Unmapping XBAR%u (was routed to %d)\n",
+		map->xbar_in, map->xbar_out);
+
+	ti_dra7_xbar_write(xbar->iomem, map->xbar_out, xbar->safe_val);
+	mutex_lock(&xbar->mutex);
+	clear_bit(map->xbar_out, xbar->dma_inuse);
+	mutex_unlock(&xbar->mutex);
+	kfree(map);
+}
+
+static void *ti_dra7_xbar_route_allocate(struct of_phandle_args *dma_spec,
+					 struct of_dma *ofdma)
+{
+	struct platform_device *pdev = of_find_device_by_node(ofdma->of_node);
+	struct ti_dra7_xbar_data *xbar = platform_get_drvdata(pdev);
+	struct ti_dra7_xbar_map *map;
+
+	if (dma_spec->args[0] >= xbar->xbar_requests) {
+		dev_err(&pdev->dev, "Invalid XBAR request number: %d\n",
+			dma_spec->args[0]);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* The of_node_put() will be done in the core for the node */
+	dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0);
+	if (!dma_spec->np) {
+		dev_err(&pdev->dev, "Can't get DMA master\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	map = kzalloc(sizeof(*map), GFP_KERNEL);
+	if (!map) {
+		of_node_put(dma_spec->np);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	mutex_lock(&xbar->mutex);
+	map->xbar_out = find_first_zero_bit(xbar->dma_inuse,
+					    xbar->dma_requests);
+	if (map->xbar_out == xbar->dma_requests) {
+		mutex_unlock(&xbar->mutex);
+		dev_err(&pdev->dev, "Run out of free DMA requests\n");
+		kfree(map);
+		return ERR_PTR(-ENOMEM);
+	}
+	set_bit(map->xbar_out, xbar->dma_inuse);
+	mutex_unlock(&xbar->mutex);
+
+	map->xbar_in = (u16)dma_spec->args[0];
+
+	dma_spec->args[0] = map->xbar_out + xbar->dma_offset;
+
+	dev_dbg(&pdev->dev, "Mapping XBAR%u to DMA%d\n",
+		map->xbar_in, map->xbar_out);
+
+	ti_dra7_xbar_write(xbar->iomem, map->xbar_out, map->xbar_in);
+
+	return map;
+}
+
+#define TI_XBAR_EDMA_OFFSET	0
+#define TI_XBAR_SDMA_OFFSET	1
+static const u32 ti_dma_offset[] = {
+	[TI_XBAR_EDMA_OFFSET] = 0,
+	[TI_XBAR_SDMA_OFFSET] = 1,
+};
+
+static const struct of_device_id ti_dra7_master_match[] = {
+	{
+		.compatible = "ti,omap4430-sdma",
+		.data = &ti_dma_offset[TI_XBAR_SDMA_OFFSET],
+	},
+	{
+		.compatible = "ti,edma3",
+		.data = &ti_dma_offset[TI_XBAR_EDMA_OFFSET],
+	},
+	{
+		.compatible = "ti,edma3-tpcc",
+		.data = &ti_dma_offset[TI_XBAR_EDMA_OFFSET],
+	},
+	{},
+};
+
+static inline void ti_dra7_xbar_reserve(int offset, int len, unsigned long *p)
+{
+	for (; len > 0; len--)
+		set_bit(offset + (len - 1), p);
+}
+
+static int ti_dra7_xbar_probe(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	const struct of_device_id *match;
+	struct device_node *dma_node;
+	struct ti_dra7_xbar_data *xbar;
+	struct property *prop;
+	struct resource *res;
+	u32 safe_val;
+	int sz;
+	void __iomem *iomem;
+	int i, ret;
+
+	if (!node)
+		return -ENODEV;
+
+	xbar = devm_kzalloc(&pdev->dev, sizeof(*xbar), GFP_KERNEL);
+	if (!xbar)
+		return -ENOMEM;
+
+	dma_node = of_parse_phandle(node, "dma-masters", 0);
+	if (!dma_node) {
+		dev_err(&pdev->dev, "Can't get DMA master node\n");
+		return -ENODEV;
+	}
+
+	match = of_match_node(ti_dra7_master_match, dma_node);
+	if (!match) {
+		dev_err(&pdev->dev, "DMA master is not supported\n");
+		of_node_put(dma_node);
+		return -EINVAL;
+	}
+
+	if (of_property_read_u32(dma_node, "dma-requests",
+				 &xbar->dma_requests)) {
+		dev_info(&pdev->dev,
+			 "Missing XBAR output information, using %u.\n",
+			 TI_DRA7_XBAR_OUTPUTS);
+		xbar->dma_requests = TI_DRA7_XBAR_OUTPUTS;
+	}
+	of_node_put(dma_node);
+
+	xbar->dma_inuse = devm_kcalloc(&pdev->dev,
+				       BITS_TO_LONGS(xbar->dma_requests),
+				       sizeof(unsigned long), GFP_KERNEL);
+	if (!xbar->dma_inuse)
+		return -ENOMEM;
+
+	if (of_property_read_u32(node, "dma-requests", &xbar->xbar_requests)) {
+		dev_info(&pdev->dev,
+			 "Missing XBAR input information, using %u.\n",
+			 TI_DRA7_XBAR_INPUTS);
+		xbar->xbar_requests = TI_DRA7_XBAR_INPUTS;
+	}
+
+	if (!of_property_read_u32(node, "ti,dma-safe-map", &safe_val))
+		xbar->safe_val = (u16)safe_val;
+
+
+	prop = of_find_property(node, "ti,reserved-dma-request-ranges", &sz);
+	if (prop) {
+		const char pname[] = "ti,reserved-dma-request-ranges";
+		u32 (*rsv_events)[2];
+		size_t nelm = sz / sizeof(*rsv_events);
+		int i;
+
+		if (!nelm)
+			return -EINVAL;
+
+		rsv_events = kcalloc(nelm, sizeof(*rsv_events), GFP_KERNEL);
+		if (!rsv_events)
+			return -ENOMEM;
+
+		ret = of_property_read_u32_array(node, pname, (u32 *)rsv_events,
+						 nelm * 2);
+		if (ret)
+			return ret;
+
+		for (i = 0; i < nelm; i++) {
+			ti_dra7_xbar_reserve(rsv_events[i][0], rsv_events[i][1],
+					     xbar->dma_inuse);
+		}
+		kfree(rsv_events);
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	iomem = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(iomem))
+		return PTR_ERR(iomem);
+
+	xbar->iomem = iomem;
+
+	xbar->dmarouter.dev = &pdev->dev;
+	xbar->dmarouter.route_free = ti_dra7_xbar_free;
+	xbar->dma_offset = *(u32 *)match->data;
+
+	mutex_init(&xbar->mutex);
+	platform_set_drvdata(pdev, xbar);
+
+	/* Reset the crossbar */
+	for (i = 0; i < xbar->dma_requests; i++) {
+		if (!test_bit(i, xbar->dma_inuse))
+			ti_dra7_xbar_write(xbar->iomem, i, xbar->safe_val);
+	}
+
+	ret = of_dma_router_register(node, ti_dra7_xbar_route_allocate,
+				     &xbar->dmarouter);
+	if (ret) {
+		/* Restore the defaults for the crossbar */
+		for (i = 0; i < xbar->dma_requests; i++) {
+			if (!test_bit(i, xbar->dma_inuse))
+				ti_dra7_xbar_write(xbar->iomem, i, i);
+		}
+	}
+
+	return ret;
+}
+
+static int ti_dma_xbar_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *match;
+	int ret;
+
+	match = of_match_node(ti_dma_xbar_match, pdev->dev.of_node);
+	if (unlikely(!match))
+		return -EINVAL;
+
+	switch (*(u32 *)match->data) {
+	case TI_XBAR_DRA7:
+		ret = ti_dra7_xbar_probe(pdev);
+		break;
+	case TI_XBAR_AM335X:
+		ret = ti_am335x_xbar_probe(pdev);
+		break;
+	default:
+		dev_err(&pdev->dev, "Unsupported crossbar\n");
+		ret = -ENODEV;
+		break;
+	}
+
+	return ret;
+}
+
+static struct platform_driver ti_dma_xbar_driver = {
+	.driver = {
+		.name = "ti-dma-crossbar",
+		.of_match_table = of_match_ptr(ti_dma_xbar_match),
+	},
+	.probe	= ti_dma_xbar_probe,
+};
+
+static int omap_dmaxbar_init(void)
+{
+	return platform_driver_register(&ti_dma_xbar_driver);
+}
+arch_initcall(omap_dmaxbar_init);
diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c
new file mode 100644
index 000000000000..93a5cbf13319
--- /dev/null
+++ b/drivers/dma/ti/edma.c
@@ -0,0 +1,2568 @@
+/*
+ * TI EDMA DMA engine driver
+ *
+ * Copyright 2012 Texas Instruments
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/edma.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/of_irq.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/pm_runtime.h>
+
+#include <linux/platform_data/edma.h>
+
+#include "../dmaengine.h"
+#include "../virt-dma.h"
+
+/* Offsets matching "struct edmacc_param" */
+#define PARM_OPT		0x00
+#define PARM_SRC		0x04
+#define PARM_A_B_CNT		0x08
+#define PARM_DST		0x0c
+#define PARM_SRC_DST_BIDX	0x10
+#define PARM_LINK_BCNTRLD	0x14
+#define PARM_SRC_DST_CIDX	0x18
+#define PARM_CCNT		0x1c
+
+#define PARM_SIZE		0x20
+
+/* Offsets for EDMA CC global channel registers and their shadows */
+#define SH_ER			0x00	/* 64 bits */
+#define SH_ECR			0x08	/* 64 bits */
+#define SH_ESR			0x10	/* 64 bits */
+#define SH_CER			0x18	/* 64 bits */
+#define SH_EER			0x20	/* 64 bits */
+#define SH_EECR			0x28	/* 64 bits */
+#define SH_EESR			0x30	/* 64 bits */
+#define SH_SER			0x38	/* 64 bits */
+#define SH_SECR			0x40	/* 64 bits */
+#define SH_IER			0x50	/* 64 bits */
+#define SH_IECR			0x58	/* 64 bits */
+#define SH_IESR			0x60	/* 64 bits */
+#define SH_IPR			0x68	/* 64 bits */
+#define SH_ICR			0x70	/* 64 bits */
+#define SH_IEVAL		0x78
+#define SH_QER			0x80
+#define SH_QEER			0x84
+#define SH_QEECR		0x88
+#define SH_QEESR		0x8c
+#define SH_QSER			0x90
+#define SH_QSECR		0x94
+#define SH_SIZE			0x200
+
+/* Offsets for EDMA CC global registers */
+#define EDMA_REV		0x0000
+#define EDMA_CCCFG		0x0004
+#define EDMA_QCHMAP		0x0200	/* 8 registers */
+#define EDMA_DMAQNUM		0x0240	/* 8 registers (4 on OMAP-L1xx) */
+#define EDMA_QDMAQNUM		0x0260
+#define EDMA_QUETCMAP		0x0280
+#define EDMA_QUEPRI		0x0284
+#define EDMA_EMR		0x0300	/* 64 bits */
+#define EDMA_EMCR		0x0308	/* 64 bits */
+#define EDMA_QEMR		0x0310
+#define EDMA_QEMCR		0x0314
+#define EDMA_CCERR		0x0318
+#define EDMA_CCERRCLR		0x031c
+#define EDMA_EEVAL		0x0320
+#define EDMA_DRAE		0x0340	/* 4 x 64 bits*/
+#define EDMA_QRAE		0x0380	/* 4 registers */
+#define EDMA_QUEEVTENTRY	0x0400	/* 2 x 16 registers */
+#define EDMA_QSTAT		0x0600	/* 2 registers */
+#define EDMA_QWMTHRA		0x0620
+#define EDMA_QWMTHRB		0x0624
+#define EDMA_CCSTAT		0x0640
+
+#define EDMA_M			0x1000	/* global channel registers */
+#define EDMA_ECR		0x1008
+#define EDMA_ECRH		0x100C
+#define EDMA_SHADOW0		0x2000	/* 4 shadow regions */
+#define EDMA_PARM		0x4000	/* PaRAM entries */
+
+#define PARM_OFFSET(param_no)	(EDMA_PARM + ((param_no) << 5))
+
+#define EDMA_DCHMAP		0x0100  /* 64 registers */
+
+/* CCCFG register */
+#define GET_NUM_DMACH(x)	(x & 0x7) /* bits 0-2 */
+#define GET_NUM_QDMACH(x)	((x & 0x70) >> 4) /* bits 4-6 */
+#define GET_NUM_PAENTRY(x)	((x & 0x7000) >> 12) /* bits 12-14 */
+#define GET_NUM_EVQUE(x)	((x & 0x70000) >> 16) /* bits 16-18 */
+#define GET_NUM_REGN(x)		((x & 0x300000) >> 20) /* bits 20-21 */
+#define CHMAP_EXIST		BIT(24)
+
+/* CCSTAT register */
+#define EDMA_CCSTAT_ACTV	BIT(4)
+
+/*
+ * Max of 20 segments per channel to conserve PaRAM slots
+ * Also note that MAX_NR_SG should be atleast the no.of periods
+ * that are required for ASoC, otherwise DMA prep calls will
+ * fail. Today davinci-pcm is the only user of this driver and
+ * requires atleast 17 slots, so we setup the default to 20.
+ */
+#define MAX_NR_SG		20
+#define EDMA_MAX_SLOTS		MAX_NR_SG
+#define EDMA_DESCRIPTORS	16
+
+#define EDMA_CHANNEL_ANY		-1	/* for edma_alloc_channel() */
+#define EDMA_SLOT_ANY			-1	/* for edma_alloc_slot() */
+#define EDMA_CONT_PARAMS_ANY		 1001
+#define EDMA_CONT_PARAMS_FIXED_EXACT	 1002
+#define EDMA_CONT_PARAMS_FIXED_NOT_EXACT 1003
+
+/* PaRAM slots are laid out like this */
+struct edmacc_param {
+	u32 opt;
+	u32 src;
+	u32 a_b_cnt;
+	u32 dst;
+	u32 src_dst_bidx;
+	u32 link_bcntrld;
+	u32 src_dst_cidx;
+	u32 ccnt;
+} __packed;
+
+/* fields in edmacc_param.opt */
+#define SAM		BIT(0)
+#define DAM		BIT(1)
+#define SYNCDIM		BIT(2)
+#define STATIC		BIT(3)
+#define EDMA_FWID	(0x07 << 8)
+#define TCCMODE		BIT(11)
+#define EDMA_TCC(t)	((t) << 12)
+#define TCINTEN		BIT(20)
+#define ITCINTEN	BIT(21)
+#define TCCHEN		BIT(22)
+#define ITCCHEN		BIT(23)
+
+struct edma_pset {
+	u32				len;
+	dma_addr_t			addr;
+	struct edmacc_param		param;
+};
+
+struct edma_desc {
+	struct virt_dma_desc		vdesc;
+	struct list_head		node;
+	enum dma_transfer_direction	direction;
+	int				cyclic;
+	int				absync;
+	int				pset_nr;
+	struct edma_chan		*echan;
+	int				processed;
+
+	/*
+	 * The following 4 elements are used for residue accounting.
+	 *
+	 * - processed_stat: the number of SG elements we have traversed
+	 * so far to cover accounting. This is updated directly to processed
+	 * during edma_callback and is always <= processed, because processed
+	 * refers to the number of pending transfer (programmed to EDMA
+	 * controller), where as processed_stat tracks number of transfers
+	 * accounted for so far.
+	 *
+	 * - residue: The amount of bytes we have left to transfer for this desc
+	 *
+	 * - residue_stat: The residue in bytes of data we have covered
+	 * so far for accounting. This is updated directly to residue
+	 * during callbacks to keep it current.
+	 *
+	 * - sg_len: Tracks the length of the current intermediate transfer,
+	 * this is required to update the residue during intermediate transfer
+	 * completion callback.
+	 */
+	int				processed_stat;
+	u32				sg_len;
+	u32				residue;
+	u32				residue_stat;
+
+	struct edma_pset		pset[0];
+};
+
+struct edma_cc;
+
+struct edma_tc {
+	struct device_node		*node;
+	u16				id;
+};
+
+struct edma_chan {
+	struct virt_dma_chan		vchan;
+	struct list_head		node;
+	struct edma_desc		*edesc;
+	struct edma_cc			*ecc;
+	struct edma_tc			*tc;
+	int				ch_num;
+	bool				alloced;
+	bool				hw_triggered;
+	int				slot[EDMA_MAX_SLOTS];
+	int				missed;
+	struct dma_slave_config		cfg;
+};
+
+struct edma_cc {
+	struct device			*dev;
+	struct edma_soc_info		*info;
+	void __iomem			*base;
+	int				id;
+	bool				legacy_mode;
+
+	/* eDMA3 resource information */
+	unsigned			num_channels;
+	unsigned			num_qchannels;
+	unsigned			num_region;
+	unsigned			num_slots;
+	unsigned			num_tc;
+	bool				chmap_exist;
+	enum dma_event_q		default_queue;
+
+	unsigned int			ccint;
+	unsigned int			ccerrint;
+
+	/*
+	 * The slot_inuse bit for each PaRAM slot is clear unless the slot is
+	 * in use by Linux or if it is allocated to be used by DSP.
+	 */
+	unsigned long *slot_inuse;
+
+	struct dma_device		dma_slave;
+	struct dma_device		*dma_memcpy;
+	struct edma_chan		*slave_chans;
+	struct edma_tc			*tc_list;
+	int				dummy_slot;
+};
+
+/* dummy param set used to (re)initialize parameter RAM slots */
+static const struct edmacc_param dummy_paramset = {
+	.link_bcntrld = 0xffff,
+	.ccnt = 1,
+};
+
+#define EDMA_BINDING_LEGACY	0
+#define EDMA_BINDING_TPCC	1
+static const u32 edma_binding_type[] = {
+	[EDMA_BINDING_LEGACY] = EDMA_BINDING_LEGACY,
+	[EDMA_BINDING_TPCC] = EDMA_BINDING_TPCC,
+};
+
+static const struct of_device_id edma_of_ids[] = {
+	{
+		.compatible = "ti,edma3",
+		.data = &edma_binding_type[EDMA_BINDING_LEGACY],
+	},
+	{
+		.compatible = "ti,edma3-tpcc",
+		.data = &edma_binding_type[EDMA_BINDING_TPCC],
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, edma_of_ids);
+
+static const struct of_device_id edma_tptc_of_ids[] = {
+	{ .compatible = "ti,edma3-tptc", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, edma_tptc_of_ids);
+
+static inline unsigned int edma_read(struct edma_cc *ecc, int offset)
+{
+	return (unsigned int)__raw_readl(ecc->base + offset);
+}
+
+static inline void edma_write(struct edma_cc *ecc, int offset, int val)
+{
+	__raw_writel(val, ecc->base + offset);
+}
+
+static inline void edma_modify(struct edma_cc *ecc, int offset, unsigned and,
+			       unsigned or)
+{
+	unsigned val = edma_read(ecc, offset);
+
+	val &= and;
+	val |= or;
+	edma_write(ecc, offset, val);
+}
+
+static inline void edma_and(struct edma_cc *ecc, int offset, unsigned and)
+{
+	unsigned val = edma_read(ecc, offset);
+
+	val &= and;
+	edma_write(ecc, offset, val);
+}
+
+static inline void edma_or(struct edma_cc *ecc, int offset, unsigned or)
+{
+	unsigned val = edma_read(ecc, offset);
+
+	val |= or;
+	edma_write(ecc, offset, val);
+}
+
+static inline unsigned int edma_read_array(struct edma_cc *ecc, int offset,
+					   int i)
+{
+	return edma_read(ecc, offset + (i << 2));
+}
+
+static inline void edma_write_array(struct edma_cc *ecc, int offset, int i,
+				    unsigned val)
+{
+	edma_write(ecc, offset + (i << 2), val);
+}
+
+static inline void edma_modify_array(struct edma_cc *ecc, int offset, int i,
+				     unsigned and, unsigned or)
+{
+	edma_modify(ecc, offset + (i << 2), and, or);
+}
+
+static inline void edma_or_array(struct edma_cc *ecc, int offset, int i,
+				 unsigned or)
+{
+	edma_or(ecc, offset + (i << 2), or);
+}
+
+static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j,
+				  unsigned or)
+{
+	edma_or(ecc, offset + ((i * 2 + j) << 2), or);
+}
+
+static inline void edma_write_array2(struct edma_cc *ecc, int offset, int i,
+				     int j, unsigned val)
+{
+	edma_write(ecc, offset + ((i * 2 + j) << 2), val);
+}
+
+static inline unsigned int edma_shadow0_read(struct edma_cc *ecc, int offset)
+{
+	return edma_read(ecc, EDMA_SHADOW0 + offset);
+}
+
+static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc,
+						   int offset, int i)
+{
+	return edma_read(ecc, EDMA_SHADOW0 + offset + (i << 2));
+}
+
+static inline void edma_shadow0_write(struct edma_cc *ecc, int offset,
+				      unsigned val)
+{
+	edma_write(ecc, EDMA_SHADOW0 + offset, val);
+}
+
+static inline void edma_shadow0_write_array(struct edma_cc *ecc, int offset,
+					    int i, unsigned val)
+{
+	edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val);
+}
+
+static inline unsigned int edma_param_read(struct edma_cc *ecc, int offset,
+					   int param_no)
+{
+	return edma_read(ecc, EDMA_PARM + offset + (param_no << 5));
+}
+
+static inline void edma_param_write(struct edma_cc *ecc, int offset,
+				    int param_no, unsigned val)
+{
+	edma_write(ecc, EDMA_PARM + offset + (param_no << 5), val);
+}
+
+static inline void edma_param_modify(struct edma_cc *ecc, int offset,
+				     int param_no, unsigned and, unsigned or)
+{
+	edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or);
+}
+
+static inline void edma_param_and(struct edma_cc *ecc, int offset, int param_no,
+				  unsigned and)
+{
+	edma_and(ecc, EDMA_PARM + offset + (param_no << 5), and);
+}
+
+static inline void edma_param_or(struct edma_cc *ecc, int offset, int param_no,
+				 unsigned or)
+{
+	edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or);
+}
+
+static inline void edma_set_bits(int offset, int len, unsigned long *p)
+{
+	for (; len > 0; len--)
+		set_bit(offset + (len - 1), p);
+}
+
+static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
+					  int priority)
+{
+	int bit = queue_no * 4;
+
+	edma_modify(ecc, EDMA_QUEPRI, ~(0x7 << bit), ((priority & 0x7) << bit));
+}
+
+static void edma_set_chmap(struct edma_chan *echan, int slot)
+{
+	struct edma_cc *ecc = echan->ecc;
+	int channel = EDMA_CHAN_SLOT(echan->ch_num);
+
+	if (ecc->chmap_exist) {
+		slot = EDMA_CHAN_SLOT(slot);
+		edma_write_array(ecc, EDMA_DCHMAP, channel, (slot << 5));
+	}
+}
+
+static void edma_setup_interrupt(struct edma_chan *echan, bool enable)
+{
+	struct edma_cc *ecc = echan->ecc;
+	int channel = EDMA_CHAN_SLOT(echan->ch_num);
+
+	if (enable) {
+		edma_shadow0_write_array(ecc, SH_ICR, channel >> 5,
+					 BIT(channel & 0x1f));
+		edma_shadow0_write_array(ecc, SH_IESR, channel >> 5,
+					 BIT(channel & 0x1f));
+	} else {
+		edma_shadow0_write_array(ecc, SH_IECR, channel >> 5,
+					 BIT(channel & 0x1f));
+	}
+}
+
+/*
+ * paRAM slot management functions
+ */
+static void edma_write_slot(struct edma_cc *ecc, unsigned slot,
+			    const struct edmacc_param *param)
+{
+	slot = EDMA_CHAN_SLOT(slot);
+	if (slot >= ecc->num_slots)
+		return;
+	memcpy_toio(ecc->base + PARM_OFFSET(slot), param, PARM_SIZE);
+}
+
+static int edma_read_slot(struct edma_cc *ecc, unsigned slot,
+			   struct edmacc_param *param)
+{
+	slot = EDMA_CHAN_SLOT(slot);
+	if (slot >= ecc->num_slots)
+		return -EINVAL;
+	memcpy_fromio(param, ecc->base + PARM_OFFSET(slot), PARM_SIZE);
+
+	return 0;
+}
+
+/**
+ * edma_alloc_slot - allocate DMA parameter RAM
+ * @ecc: pointer to edma_cc struct
+ * @slot: specific slot to allocate; negative for "any unused slot"
+ *
+ * This allocates a parameter RAM slot, initializing it to hold a
+ * dummy transfer.  Slots allocated using this routine have not been
+ * mapped to a hardware DMA channel, and will normally be used by
+ * linking to them from a slot associated with a DMA channel.
+ *
+ * Normal use is to pass EDMA_SLOT_ANY as the @slot, but specific
+ * slots may be allocated on behalf of DSP firmware.
+ *
+ * Returns the number of the slot, else negative errno.
+ */
+static int edma_alloc_slot(struct edma_cc *ecc, int slot)
+{
+	if (slot >= 0) {
+		slot = EDMA_CHAN_SLOT(slot);
+		/* Requesting entry paRAM slot for a HW triggered channel. */
+		if (ecc->chmap_exist && slot < ecc->num_channels)
+			slot = EDMA_SLOT_ANY;
+	}
+
+	if (slot < 0) {
+		if (ecc->chmap_exist)
+			slot = 0;
+		else
+			slot = ecc->num_channels;
+		for (;;) {
+			slot = find_next_zero_bit(ecc->slot_inuse,
+						  ecc->num_slots,
+						  slot);
+			if (slot == ecc->num_slots)
+				return -ENOMEM;
+			if (!test_and_set_bit(slot, ecc->slot_inuse))
+				break;
+		}
+	} else if (slot >= ecc->num_slots) {
+		return -EINVAL;
+	} else if (test_and_set_bit(slot, ecc->slot_inuse)) {
+		return -EBUSY;
+	}
+
+	edma_write_slot(ecc, slot, &dummy_paramset);
+
+	return EDMA_CTLR_CHAN(ecc->id, slot);
+}
+
+static void edma_free_slot(struct edma_cc *ecc, unsigned slot)
+{
+	slot = EDMA_CHAN_SLOT(slot);
+	if (slot >= ecc->num_slots)
+		return;
+
+	edma_write_slot(ecc, slot, &dummy_paramset);
+	clear_bit(slot, ecc->slot_inuse);
+}
+
+/**
+ * edma_link - link one parameter RAM slot to another
+ * @ecc: pointer to edma_cc struct
+ * @from: parameter RAM slot originating the link
+ * @to: parameter RAM slot which is the link target
+ *
+ * The originating slot should not be part of any active DMA transfer.
+ */
+static void edma_link(struct edma_cc *ecc, unsigned from, unsigned to)
+{
+	if (unlikely(EDMA_CTLR(from) != EDMA_CTLR(to)))
+		dev_warn(ecc->dev, "Ignoring eDMA instance for linking\n");
+
+	from = EDMA_CHAN_SLOT(from);
+	to = EDMA_CHAN_SLOT(to);
+	if (from >= ecc->num_slots || to >= ecc->num_slots)
+		return;
+
+	edma_param_modify(ecc, PARM_LINK_BCNTRLD, from, 0xffff0000,
+			  PARM_OFFSET(to));
+}
+
+/**
+ * edma_get_position - returns the current transfer point
+ * @ecc: pointer to edma_cc struct
+ * @slot: parameter RAM slot being examined
+ * @dst:  true selects the dest position, false the source
+ *
+ * Returns the position of the current active slot
+ */
+static dma_addr_t edma_get_position(struct edma_cc *ecc, unsigned slot,
+				    bool dst)
+{
+	u32 offs;
+
+	slot = EDMA_CHAN_SLOT(slot);
+	offs = PARM_OFFSET(slot);
+	offs += dst ? PARM_DST : PARM_SRC;
+
+	return edma_read(ecc, offs);
+}
+
+/*
+ * Channels with event associations will be triggered by their hardware
+ * events, and channels without such associations will be triggered by
+ * software.  (At this writing there is no interface for using software
+ * triggers except with channels that don't support hardware triggers.)
+ */
+static void edma_start(struct edma_chan *echan)
+{
+	struct edma_cc *ecc = echan->ecc;
+	int channel = EDMA_CHAN_SLOT(echan->ch_num);
+	int j = (channel >> 5);
+	unsigned int mask = BIT(channel & 0x1f);
+
+	if (!echan->hw_triggered) {
+		/* EDMA channels without event association */
+		dev_dbg(ecc->dev, "ESR%d %08x\n", j,
+			edma_shadow0_read_array(ecc, SH_ESR, j));
+		edma_shadow0_write_array(ecc, SH_ESR, j, mask);
+	} else {
+		/* EDMA channel with event association */
+		dev_dbg(ecc->dev, "ER%d %08x\n", j,
+			edma_shadow0_read_array(ecc, SH_ER, j));
+		/* Clear any pending event or error */
+		edma_write_array(ecc, EDMA_ECR, j, mask);
+		edma_write_array(ecc, EDMA_EMCR, j, mask);
+		/* Clear any SER */
+		edma_shadow0_write_array(ecc, SH_SECR, j, mask);
+		edma_shadow0_write_array(ecc, SH_EESR, j, mask);
+		dev_dbg(ecc->dev, "EER%d %08x\n", j,
+			edma_shadow0_read_array(ecc, SH_EER, j));
+	}
+}
+
+static void edma_stop(struct edma_chan *echan)
+{
+	struct edma_cc *ecc = echan->ecc;
+	int channel = EDMA_CHAN_SLOT(echan->ch_num);
+	int j = (channel >> 5);
+	unsigned int mask = BIT(channel & 0x1f);
+
+	edma_shadow0_write_array(ecc, SH_EECR, j, mask);
+	edma_shadow0_write_array(ecc, SH_ECR, j, mask);
+	edma_shadow0_write_array(ecc, SH_SECR, j, mask);
+	edma_write_array(ecc, EDMA_EMCR, j, mask);
+
+	/* clear possibly pending completion interrupt */
+	edma_shadow0_write_array(ecc, SH_ICR, j, mask);
+
+	dev_dbg(ecc->dev, "EER%d %08x\n", j,
+		edma_shadow0_read_array(ecc, SH_EER, j));
+
+	/* REVISIT:  consider guarding against inappropriate event
+	 * chaining by overwriting with dummy_paramset.
+	 */
+}
+
+/*
+ * Temporarily disable EDMA hardware events on the specified channel,
+ * preventing them from triggering new transfers
+ */
+static void edma_pause(struct edma_chan *echan)
+{
+	int channel = EDMA_CHAN_SLOT(echan->ch_num);
+	unsigned int mask = BIT(channel & 0x1f);
+
+	edma_shadow0_write_array(echan->ecc, SH_EECR, channel >> 5, mask);
+}
+
+/* Re-enable EDMA hardware events on the specified channel.  */
+static void edma_resume(struct edma_chan *echan)
+{
+	int channel = EDMA_CHAN_SLOT(echan->ch_num);
+	unsigned int mask = BIT(channel & 0x1f);
+
+	edma_shadow0_write_array(echan->ecc, SH_EESR, channel >> 5, mask);
+}
+
+static void edma_trigger_channel(struct edma_chan *echan)
+{
+	struct edma_cc *ecc = echan->ecc;
+	int channel = EDMA_CHAN_SLOT(echan->ch_num);
+	unsigned int mask = BIT(channel & 0x1f);
+
+	edma_shadow0_write_array(ecc, SH_ESR, (channel >> 5), mask);
+
+	dev_dbg(ecc->dev, "ESR%d %08x\n", (channel >> 5),
+		edma_shadow0_read_array(ecc, SH_ESR, (channel >> 5)));
+}
+
+static void edma_clean_channel(struct edma_chan *echan)
+{
+	struct edma_cc *ecc = echan->ecc;
+	int channel = EDMA_CHAN_SLOT(echan->ch_num);
+	int j = (channel >> 5);
+	unsigned int mask = BIT(channel & 0x1f);
+
+	dev_dbg(ecc->dev, "EMR%d %08x\n", j, edma_read_array(ecc, EDMA_EMR, j));
+	edma_shadow0_write_array(ecc, SH_ECR, j, mask);
+	/* Clear the corresponding EMR bits */
+	edma_write_array(ecc, EDMA_EMCR, j, mask);
+	/* Clear any SER */
+	edma_shadow0_write_array(ecc, SH_SECR, j, mask);
+	edma_write(ecc, EDMA_CCERRCLR, BIT(16) | BIT(1) | BIT(0));
+}
+
+/* Move channel to a specific event queue */
+static void edma_assign_channel_eventq(struct edma_chan *echan,
+				       enum dma_event_q eventq_no)
+{
+	struct edma_cc *ecc = echan->ecc;
+	int channel = EDMA_CHAN_SLOT(echan->ch_num);
+	int bit = (channel & 0x7) * 4;
+
+	/* default to low priority queue */
+	if (eventq_no == EVENTQ_DEFAULT)
+		eventq_no = ecc->default_queue;
+	if (eventq_no >= ecc->num_tc)
+		return;
+
+	eventq_no &= 7;
+	edma_modify_array(ecc, EDMA_DMAQNUM, (channel >> 3), ~(0x7 << bit),
+			  eventq_no << bit);
+}
+
+static int edma_alloc_channel(struct edma_chan *echan,
+			      enum dma_event_q eventq_no)
+{
+	struct edma_cc *ecc = echan->ecc;
+	int channel = EDMA_CHAN_SLOT(echan->ch_num);
+
+	/* ensure access through shadow region 0 */
+	edma_or_array2(ecc, EDMA_DRAE, 0, channel >> 5, BIT(channel & 0x1f));
+
+	/* ensure no events are pending */
+	edma_stop(echan);
+
+	edma_setup_interrupt(echan, true);
+
+	edma_assign_channel_eventq(echan, eventq_no);
+
+	return 0;
+}
+
+static void edma_free_channel(struct edma_chan *echan)
+{
+	/* ensure no events are pending */
+	edma_stop(echan);
+	/* REVISIT should probably take out of shadow region 0 */
+	edma_setup_interrupt(echan, false);
+}
+
+static inline struct edma_cc *to_edma_cc(struct dma_device *d)
+{
+	return container_of(d, struct edma_cc, dma_slave);
+}
+
+static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
+{
+	return container_of(c, struct edma_chan, vchan.chan);
+}
+
+static inline struct edma_desc *to_edma_desc(struct dma_async_tx_descriptor *tx)
+{
+	return container_of(tx, struct edma_desc, vdesc.tx);
+}
+
+static void edma_desc_free(struct virt_dma_desc *vdesc)
+{
+	kfree(container_of(vdesc, struct edma_desc, vdesc));
+}
+
+/* Dispatch a queued descriptor to the controller (caller holds lock) */
+static void edma_execute(struct edma_chan *echan)
+{
+	struct edma_cc *ecc = echan->ecc;
+	struct virt_dma_desc *vdesc;
+	struct edma_desc *edesc;
+	struct device *dev = echan->vchan.chan.device->dev;
+	int i, j, left, nslots;
+
+	if (!echan->edesc) {
+		/* Setup is needed for the first transfer */
+		vdesc = vchan_next_desc(&echan->vchan);
+		if (!vdesc)
+			return;
+		list_del(&vdesc->node);
+		echan->edesc = to_edma_desc(&vdesc->tx);
+	}
+
+	edesc = echan->edesc;
+
+	/* Find out how many left */
+	left = edesc->pset_nr - edesc->processed;
+	nslots = min(MAX_NR_SG, left);
+	edesc->sg_len = 0;
+
+	/* Write descriptor PaRAM set(s) */
+	for (i = 0; i < nslots; i++) {
+		j = i + edesc->processed;
+		edma_write_slot(ecc, echan->slot[i], &edesc->pset[j].param);
+		edesc->sg_len += edesc->pset[j].len;
+		dev_vdbg(dev,
+			 "\n pset[%d]:\n"
+			 "  chnum\t%d\n"
+			 "  slot\t%d\n"
+			 "  opt\t%08x\n"
+			 "  src\t%08x\n"
+			 "  dst\t%08x\n"
+			 "  abcnt\t%08x\n"
+			 "  ccnt\t%08x\n"
+			 "  bidx\t%08x\n"
+			 "  cidx\t%08x\n"
+			 "  lkrld\t%08x\n",
+			 j, echan->ch_num, echan->slot[i],
+			 edesc->pset[j].param.opt,
+			 edesc->pset[j].param.src,
+			 edesc->pset[j].param.dst,
+			 edesc->pset[j].param.a_b_cnt,
+			 edesc->pset[j].param.ccnt,
+			 edesc->pset[j].param.src_dst_bidx,
+			 edesc->pset[j].param.src_dst_cidx,
+			 edesc->pset[j].param.link_bcntrld);
+		/* Link to the previous slot if not the last set */
+		if (i != (nslots - 1))
+			edma_link(ecc, echan->slot[i], echan->slot[i + 1]);
+	}
+
+	edesc->processed += nslots;
+
+	/*
+	 * If this is either the last set in a set of SG-list transactions
+	 * then setup a link to the dummy slot, this results in all future
+	 * events being absorbed and that's OK because we're done
+	 */
+	if (edesc->processed == edesc->pset_nr) {
+		if (edesc->cyclic)
+			edma_link(ecc, echan->slot[nslots - 1], echan->slot[1]);
+		else
+			edma_link(ecc, echan->slot[nslots - 1],
+				  echan->ecc->dummy_slot);
+	}
+
+	if (echan->missed) {
+		/*
+		 * This happens due to setup times between intermediate
+		 * transfers in long SG lists which have to be broken up into
+		 * transfers of MAX_NR_SG
+		 */
+		dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
+		edma_clean_channel(echan);
+		edma_stop(echan);
+		edma_start(echan);
+		edma_trigger_channel(echan);
+		echan->missed = 0;
+	} else if (edesc->processed <= MAX_NR_SG) {
+		dev_dbg(dev, "first transfer starting on channel %d\n",
+			echan->ch_num);
+		edma_start(echan);
+	} else {
+		dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
+			echan->ch_num, edesc->processed);
+		edma_resume(echan);
+	}
+}
+
+static int edma_terminate_all(struct dma_chan *chan)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+	unsigned long flags;
+	LIST_HEAD(head);
+
+	spin_lock_irqsave(&echan->vchan.lock, flags);
+
+	/*
+	 * Stop DMA activity: we assume the callback will not be called
+	 * after edma_dma() returns (even if it does, it will see
+	 * echan->edesc is NULL and exit.)
+	 */
+	if (echan->edesc) {
+		edma_stop(echan);
+		/* Move the cyclic channel back to default queue */
+		if (!echan->tc && echan->edesc->cyclic)
+			edma_assign_channel_eventq(echan, EVENTQ_DEFAULT);
+
+		vchan_terminate_vdesc(&echan->edesc->vdesc);
+		echan->edesc = NULL;
+	}
+
+	vchan_get_all_descriptors(&echan->vchan, &head);
+	spin_unlock_irqrestore(&echan->vchan.lock, flags);
+	vchan_dma_desc_free_list(&echan->vchan, &head);
+
+	return 0;
+}
+
+static void edma_synchronize(struct dma_chan *chan)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+
+	vchan_synchronize(&echan->vchan);
+}
+
+static int edma_slave_config(struct dma_chan *chan,
+	struct dma_slave_config *cfg)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+
+	if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
+	    cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
+		return -EINVAL;
+
+	if (cfg->src_maxburst > chan->device->max_burst ||
+	    cfg->dst_maxburst > chan->device->max_burst)
+		return -EINVAL;
+
+	memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
+
+	return 0;
+}
+
+static int edma_dma_pause(struct dma_chan *chan)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+
+	if (!echan->edesc)
+		return -EINVAL;
+
+	edma_pause(echan);
+	return 0;
+}
+
+static int edma_dma_resume(struct dma_chan *chan)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+
+	edma_resume(echan);
+	return 0;
+}
+
+/*
+ * A PaRAM set configuration abstraction used by other modes
+ * @chan: Channel who's PaRAM set we're configuring
+ * @pset: PaRAM set to initialize and setup.
+ * @src_addr: Source address of the DMA
+ * @dst_addr: Destination address of the DMA
+ * @burst: In units of dev_width, how much to send
+ * @dev_width: How much is the dev_width
+ * @dma_length: Total length of the DMA transfer
+ * @direction: Direction of the transfer
+ */
+static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
+			    dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
+			    unsigned int acnt, unsigned int dma_length,
+			    enum dma_transfer_direction direction)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+	struct device *dev = chan->device->dev;
+	struct edmacc_param *param = &epset->param;
+	int bcnt, ccnt, cidx;
+	int src_bidx, dst_bidx, src_cidx, dst_cidx;
+	int absync;
+
+	/* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
+	if (!burst)
+		burst = 1;
+	/*
+	 * If the maxburst is equal to the fifo width, use
+	 * A-synced transfers. This allows for large contiguous
+	 * buffer transfers using only one PaRAM set.
+	 */
+	if (burst == 1) {
+		/*
+		 * For the A-sync case, bcnt and ccnt are the remainder
+		 * and quotient respectively of the division of:
+		 * (dma_length / acnt) by (SZ_64K -1). This is so
+		 * that in case bcnt over flows, we have ccnt to use.
+		 * Note: In A-sync tranfer only, bcntrld is used, but it
+		 * only applies for sg_dma_len(sg) >= SZ_64K.
+		 * In this case, the best way adopted is- bccnt for the
+		 * first frame will be the remainder below. Then for
+		 * every successive frame, bcnt will be SZ_64K-1. This
+		 * is assured as bcntrld = 0xffff in end of function.
+		 */
+		absync = false;
+		ccnt = dma_length / acnt / (SZ_64K - 1);
+		bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
+		/*
+		 * If bcnt is non-zero, we have a remainder and hence an
+		 * extra frame to transfer, so increment ccnt.
+		 */
+		if (bcnt)
+			ccnt++;
+		else
+			bcnt = SZ_64K - 1;
+		cidx = acnt;
+	} else {
+		/*
+		 * If maxburst is greater than the fifo address_width,
+		 * use AB-synced transfers where A count is the fifo
+		 * address_width and B count is the maxburst. In this
+		 * case, we are limited to transfers of C count frames
+		 * of (address_width * maxburst) where C count is limited
+		 * to SZ_64K-1. This places an upper bound on the length
+		 * of an SG segment that can be handled.
+		 */
+		absync = true;
+		bcnt = burst;
+		ccnt = dma_length / (acnt * bcnt);
+		if (ccnt > (SZ_64K - 1)) {
+			dev_err(dev, "Exceeded max SG segment size\n");
+			return -EINVAL;
+		}
+		cidx = acnt * bcnt;
+	}
+
+	epset->len = dma_length;
+
+	if (direction == DMA_MEM_TO_DEV) {
+		src_bidx = acnt;
+		src_cidx = cidx;
+		dst_bidx = 0;
+		dst_cidx = 0;
+		epset->addr = src_addr;
+	} else if (direction == DMA_DEV_TO_MEM)  {
+		src_bidx = 0;
+		src_cidx = 0;
+		dst_bidx = acnt;
+		dst_cidx = cidx;
+		epset->addr = dst_addr;
+	} else if (direction == DMA_MEM_TO_MEM)  {
+		src_bidx = acnt;
+		src_cidx = cidx;
+		dst_bidx = acnt;
+		dst_cidx = cidx;
+	} else {
+		dev_err(dev, "%s: direction not implemented yet\n", __func__);
+		return -EINVAL;
+	}
+
+	param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
+	/* Configure A or AB synchronized transfers */
+	if (absync)
+		param->opt |= SYNCDIM;
+
+	param->src = src_addr;
+	param->dst = dst_addr;
+
+	param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
+	param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
+
+	param->a_b_cnt = bcnt << 16 | acnt;
+	param->ccnt = ccnt;
+	/*
+	 * Only time when (bcntrld) auto reload is required is for
+	 * A-sync case, and in this case, a requirement of reload value
+	 * of SZ_64K-1 only is assured. 'link' is initially set to NULL
+	 * and then later will be populated by edma_execute.
+	 */
+	param->link_bcntrld = 0xffffffff;
+	return absync;
+}
+
+static struct dma_async_tx_descriptor *edma_prep_slave_sg(
+	struct dma_chan *chan, struct scatterlist *sgl,
+	unsigned int sg_len, enum dma_transfer_direction direction,
+	unsigned long tx_flags, void *context)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+	struct device *dev = chan->device->dev;
+	struct edma_desc *edesc;
+	dma_addr_t src_addr = 0, dst_addr = 0;
+	enum dma_slave_buswidth dev_width;
+	u32 burst;
+	struct scatterlist *sg;
+	int i, nslots, ret;
+
+	if (unlikely(!echan || !sgl || !sg_len))
+		return NULL;
+
+	if (direction == DMA_DEV_TO_MEM) {
+		src_addr = echan->cfg.src_addr;
+		dev_width = echan->cfg.src_addr_width;
+		burst = echan->cfg.src_maxburst;
+	} else if (direction == DMA_MEM_TO_DEV) {
+		dst_addr = echan->cfg.dst_addr;
+		dev_width = echan->cfg.dst_addr_width;
+		burst = echan->cfg.dst_maxburst;
+	} else {
+		dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
+		return NULL;
+	}
+
+	if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
+		dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
+		return NULL;
+	}
+
+	edesc = kzalloc(sizeof(*edesc) + sg_len * sizeof(edesc->pset[0]),
+			GFP_ATOMIC);
+	if (!edesc)
+		return NULL;
+
+	edesc->pset_nr = sg_len;
+	edesc->residue = 0;
+	edesc->direction = direction;
+	edesc->echan = echan;
+
+	/* Allocate a PaRAM slot, if needed */
+	nslots = min_t(unsigned, MAX_NR_SG, sg_len);
+
+	for (i = 0; i < nslots; i++) {
+		if (echan->slot[i] < 0) {
+			echan->slot[i] =
+				edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
+			if (echan->slot[i] < 0) {
+				kfree(edesc);
+				dev_err(dev, "%s: Failed to allocate slot\n",
+					__func__);
+				return NULL;
+			}
+		}
+	}
+
+	/* Configure PaRAM sets for each SG */
+	for_each_sg(sgl, sg, sg_len, i) {
+		/* Get address for each SG */
+		if (direction == DMA_DEV_TO_MEM)
+			dst_addr = sg_dma_address(sg);
+		else
+			src_addr = sg_dma_address(sg);
+
+		ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
+				       dst_addr, burst, dev_width,
+				       sg_dma_len(sg), direction);
+		if (ret < 0) {
+			kfree(edesc);
+			return NULL;
+		}
+
+		edesc->absync = ret;
+		edesc->residue += sg_dma_len(sg);
+
+		if (i == sg_len - 1)
+			/* Enable completion interrupt */
+			edesc->pset[i].param.opt |= TCINTEN;
+		else if (!((i+1) % MAX_NR_SG))
+			/*
+			 * Enable early completion interrupt for the
+			 * intermediateset. In this case the driver will be
+			 * notified when the paRAM set is submitted to TC. This
+			 * will allow more time to set up the next set of slots.
+			 */
+			edesc->pset[i].param.opt |= (TCINTEN | TCCMODE);
+	}
+	edesc->residue_stat = edesc->residue;
+
+	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
+}
+
+static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
+	struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
+	size_t len, unsigned long tx_flags)
+{
+	int ret, nslots;
+	struct edma_desc *edesc;
+	struct device *dev = chan->device->dev;
+	struct edma_chan *echan = to_edma_chan(chan);
+	unsigned int width, pset_len, array_size;
+
+	if (unlikely(!echan || !len))
+		return NULL;
+
+	/* Align the array size (acnt block) with the transfer properties */
+	switch (__ffs((src | dest | len))) {
+	case 0:
+		array_size = SZ_32K - 1;
+		break;
+	case 1:
+		array_size = SZ_32K - 2;
+		break;
+	default:
+		array_size = SZ_32K - 4;
+		break;
+	}
+
+	if (len < SZ_64K) {
+		/*
+		 * Transfer size less than 64K can be handled with one paRAM
+		 * slot and with one burst.
+		 * ACNT = length
+		 */
+		width = len;
+		pset_len = len;
+		nslots = 1;
+	} else {
+		/*
+		 * Transfer size bigger than 64K will be handled with maximum of
+		 * two paRAM slots.
+		 * slot1: (full_length / 32767) times 32767 bytes bursts.
+		 *	  ACNT = 32767, length1: (full_length / 32767) * 32767
+		 * slot2: the remaining amount of data after slot1.
+		 *	  ACNT = full_length - length1, length2 = ACNT
+		 *
+		 * When the full_length is multibple of 32767 one slot can be
+		 * used to complete the transfer.
+		 */
+		width = array_size;
+		pset_len = rounddown(len, width);
+		/* One slot is enough for lengths multiple of (SZ_32K -1) */
+		if (unlikely(pset_len == len))
+			nslots = 1;
+		else
+			nslots = 2;
+	}
+
+	edesc = kzalloc(sizeof(*edesc) + nslots * sizeof(edesc->pset[0]),
+			GFP_ATOMIC);
+	if (!edesc)
+		return NULL;
+
+	edesc->pset_nr = nslots;
+	edesc->residue = edesc->residue_stat = len;
+	edesc->direction = DMA_MEM_TO_MEM;
+	edesc->echan = echan;
+
+	ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
+			       width, pset_len, DMA_MEM_TO_MEM);
+	if (ret < 0) {
+		kfree(edesc);
+		return NULL;
+	}
+
+	edesc->absync = ret;
+
+	edesc->pset[0].param.opt |= ITCCHEN;
+	if (nslots == 1) {
+		/* Enable transfer complete interrupt */
+		edesc->pset[0].param.opt |= TCINTEN;
+	} else {
+		/* Enable transfer complete chaining for the first slot */
+		edesc->pset[0].param.opt |= TCCHEN;
+
+		if (echan->slot[1] < 0) {
+			echan->slot[1] = edma_alloc_slot(echan->ecc,
+							 EDMA_SLOT_ANY);
+			if (echan->slot[1] < 0) {
+				kfree(edesc);
+				dev_err(dev, "%s: Failed to allocate slot\n",
+					__func__);
+				return NULL;
+			}
+		}
+		dest += pset_len;
+		src += pset_len;
+		pset_len = width = len % array_size;
+
+		ret = edma_config_pset(chan, &edesc->pset[1], src, dest, 1,
+				       width, pset_len, DMA_MEM_TO_MEM);
+		if (ret < 0) {
+			kfree(edesc);
+			return NULL;
+		}
+
+		edesc->pset[1].param.opt |= ITCCHEN;
+		edesc->pset[1].param.opt |= TCINTEN;
+	}
+
+	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
+}
+
+static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
+	struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
+	size_t period_len, enum dma_transfer_direction direction,
+	unsigned long tx_flags)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+	struct device *dev = chan->device->dev;
+	struct edma_desc *edesc;
+	dma_addr_t src_addr, dst_addr;
+	enum dma_slave_buswidth dev_width;
+	bool use_intermediate = false;
+	u32 burst;
+	int i, ret, nslots;
+
+	if (unlikely(!echan || !buf_len || !period_len))
+		return NULL;
+
+	if (direction == DMA_DEV_TO_MEM) {
+		src_addr = echan->cfg.src_addr;
+		dst_addr = buf_addr;
+		dev_width = echan->cfg.src_addr_width;
+		burst = echan->cfg.src_maxburst;
+	} else if (direction == DMA_MEM_TO_DEV) {
+		src_addr = buf_addr;
+		dst_addr = echan->cfg.dst_addr;
+		dev_width = echan->cfg.dst_addr_width;
+		burst = echan->cfg.dst_maxburst;
+	} else {
+		dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
+		return NULL;
+	}
+
+	if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
+		dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
+		return NULL;
+	}
+
+	if (unlikely(buf_len % period_len)) {
+		dev_err(dev, "Period should be multiple of Buffer length\n");
+		return NULL;
+	}
+
+	nslots = (buf_len / period_len) + 1;
+
+	/*
+	 * Cyclic DMA users such as audio cannot tolerate delays introduced
+	 * by cases where the number of periods is more than the maximum
+	 * number of SGs the EDMA driver can handle at a time. For DMA types
+	 * such as Slave SGs, such delays are tolerable and synchronized,
+	 * but the synchronization is difficult to achieve with Cyclic and
+	 * cannot be guaranteed, so we error out early.
+	 */
+	if (nslots > MAX_NR_SG) {
+		/*
+		 * If the burst and period sizes are the same, we can put
+		 * the full buffer into a single period and activate
+		 * intermediate interrupts. This will produce interrupts
+		 * after each burst, which is also after each desired period.
+		 */
+		if (burst == period_len) {
+			period_len = buf_len;
+			nslots = 2;
+			use_intermediate = true;
+		} else {
+			return NULL;
+		}
+	}
+
+	edesc = kzalloc(sizeof(*edesc) + nslots * sizeof(edesc->pset[0]),
+			GFP_ATOMIC);
+	if (!edesc)
+		return NULL;
+
+	edesc->cyclic = 1;
+	edesc->pset_nr = nslots;
+	edesc->residue = edesc->residue_stat = buf_len;
+	edesc->direction = direction;
+	edesc->echan = echan;
+
+	dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
+		__func__, echan->ch_num, nslots, period_len, buf_len);
+
+	for (i = 0; i < nslots; i++) {
+		/* Allocate a PaRAM slot, if needed */
+		if (echan->slot[i] < 0) {
+			echan->slot[i] =
+				edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
+			if (echan->slot[i] < 0) {
+				kfree(edesc);
+				dev_err(dev, "%s: Failed to allocate slot\n",
+					__func__);
+				return NULL;
+			}
+		}
+
+		if (i == nslots - 1) {
+			memcpy(&edesc->pset[i], &edesc->pset[0],
+			       sizeof(edesc->pset[0]));
+			break;
+		}
+
+		ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
+				       dst_addr, burst, dev_width, period_len,
+				       direction);
+		if (ret < 0) {
+			kfree(edesc);
+			return NULL;
+		}
+
+		if (direction == DMA_DEV_TO_MEM)
+			dst_addr += period_len;
+		else
+			src_addr += period_len;
+
+		dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
+		dev_vdbg(dev,
+			"\n pset[%d]:\n"
+			"  chnum\t%d\n"
+			"  slot\t%d\n"
+			"  opt\t%08x\n"
+			"  src\t%08x\n"
+			"  dst\t%08x\n"
+			"  abcnt\t%08x\n"
+			"  ccnt\t%08x\n"
+			"  bidx\t%08x\n"
+			"  cidx\t%08x\n"
+			"  lkrld\t%08x\n",
+			i, echan->ch_num, echan->slot[i],
+			edesc->pset[i].param.opt,
+			edesc->pset[i].param.src,
+			edesc->pset[i].param.dst,
+			edesc->pset[i].param.a_b_cnt,
+			edesc->pset[i].param.ccnt,
+			edesc->pset[i].param.src_dst_bidx,
+			edesc->pset[i].param.src_dst_cidx,
+			edesc->pset[i].param.link_bcntrld);
+
+		edesc->absync = ret;
+
+		/*
+		 * Enable period interrupt only if it is requested
+		 */
+		if (tx_flags & DMA_PREP_INTERRUPT) {
+			edesc->pset[i].param.opt |= TCINTEN;
+
+			/* Also enable intermediate interrupts if necessary */
+			if (use_intermediate)
+				edesc->pset[i].param.opt |= ITCINTEN;
+		}
+	}
+
+	/* Place the cyclic channel to highest priority queue */
+	if (!echan->tc)
+		edma_assign_channel_eventq(echan, EVENTQ_0);
+
+	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
+}
+
+static void edma_completion_handler(struct edma_chan *echan)
+{
+	struct device *dev = echan->vchan.chan.device->dev;
+	struct edma_desc *edesc;
+
+	spin_lock(&echan->vchan.lock);
+	edesc = echan->edesc;
+	if (edesc) {
+		if (edesc->cyclic) {
+			vchan_cyclic_callback(&edesc->vdesc);
+			spin_unlock(&echan->vchan.lock);
+			return;
+		} else if (edesc->processed == edesc->pset_nr) {
+			edesc->residue = 0;
+			edma_stop(echan);
+			vchan_cookie_complete(&edesc->vdesc);
+			echan->edesc = NULL;
+
+			dev_dbg(dev, "Transfer completed on channel %d\n",
+				echan->ch_num);
+		} else {
+			dev_dbg(dev, "Sub transfer completed on channel %d\n",
+				echan->ch_num);
+
+			edma_pause(echan);
+
+			/* Update statistics for tx_status */
+			edesc->residue -= edesc->sg_len;
+			edesc->residue_stat = edesc->residue;
+			edesc->processed_stat = edesc->processed;
+		}
+		edma_execute(echan);
+	}
+
+	spin_unlock(&echan->vchan.lock);
+}
+
+/* eDMA interrupt handler */
+static irqreturn_t dma_irq_handler(int irq, void *data)
+{
+	struct edma_cc *ecc = data;
+	int ctlr;
+	u32 sh_ier;
+	u32 sh_ipr;
+	u32 bank;
+
+	ctlr = ecc->id;
+	if (ctlr < 0)
+		return IRQ_NONE;
+
+	dev_vdbg(ecc->dev, "dma_irq_handler\n");
+
+	sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 0);
+	if (!sh_ipr) {
+		sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 1);
+		if (!sh_ipr)
+			return IRQ_NONE;
+		sh_ier = edma_shadow0_read_array(ecc, SH_IER, 1);
+		bank = 1;
+	} else {
+		sh_ier = edma_shadow0_read_array(ecc, SH_IER, 0);
+		bank = 0;
+	}
+
+	do {
+		u32 slot;
+		u32 channel;
+
+		slot = __ffs(sh_ipr);
+		sh_ipr &= ~(BIT(slot));
+
+		if (sh_ier & BIT(slot)) {
+			channel = (bank << 5) | slot;
+			/* Clear the corresponding IPR bits */
+			edma_shadow0_write_array(ecc, SH_ICR, bank, BIT(slot));
+			edma_completion_handler(&ecc->slave_chans[channel]);
+		}
+	} while (sh_ipr);
+
+	edma_shadow0_write(ecc, SH_IEVAL, 1);
+	return IRQ_HANDLED;
+}
+
+static void edma_error_handler(struct edma_chan *echan)
+{
+	struct edma_cc *ecc = echan->ecc;
+	struct device *dev = echan->vchan.chan.device->dev;
+	struct edmacc_param p;
+	int err;
+
+	if (!echan->edesc)
+		return;
+
+	spin_lock(&echan->vchan.lock);
+
+	err = edma_read_slot(ecc, echan->slot[0], &p);
+
+	/*
+	 * Issue later based on missed flag which will be sure
+	 * to happen as:
+	 * (1) we finished transmitting an intermediate slot and
+	 *     edma_execute is coming up.
+	 * (2) or we finished current transfer and issue will
+	 *     call edma_execute.
+	 *
+	 * Important note: issuing can be dangerous here and
+	 * lead to some nasty recursion when we are in a NULL
+	 * slot. So we avoid doing so and set the missed flag.
+	 */
+	if (err || (p.a_b_cnt == 0 && p.ccnt == 0)) {
+		dev_dbg(dev, "Error on null slot, setting miss\n");
+		echan->missed = 1;
+	} else {
+		/*
+		 * The slot is already programmed but the event got
+		 * missed, so its safe to issue it here.
+		 */
+		dev_dbg(dev, "Missed event, TRIGGERING\n");
+		edma_clean_channel(echan);
+		edma_stop(echan);
+		edma_start(echan);
+		edma_trigger_channel(echan);
+	}
+	spin_unlock(&echan->vchan.lock);
+}
+
+static inline bool edma_error_pending(struct edma_cc *ecc)
+{
+	if (edma_read_array(ecc, EDMA_EMR, 0) ||
+	    edma_read_array(ecc, EDMA_EMR, 1) ||
+	    edma_read(ecc, EDMA_QEMR) || edma_read(ecc, EDMA_CCERR))
+		return true;
+
+	return false;
+}
+
+/* eDMA error interrupt handler */
+static irqreturn_t dma_ccerr_handler(int irq, void *data)
+{
+	struct edma_cc *ecc = data;
+	int i, j;
+	int ctlr;
+	unsigned int cnt = 0;
+	unsigned int val;
+
+	ctlr = ecc->id;
+	if (ctlr < 0)
+		return IRQ_NONE;
+
+	dev_vdbg(ecc->dev, "dma_ccerr_handler\n");
+
+	if (!edma_error_pending(ecc)) {
+		/*
+		 * The registers indicate no pending error event but the irq
+		 * handler has been called.
+		 * Ask eDMA to re-evaluate the error registers.
+		 */
+		dev_err(ecc->dev, "%s: Error interrupt without error event!\n",
+			__func__);
+		edma_write(ecc, EDMA_EEVAL, 1);
+		return IRQ_NONE;
+	}
+
+	while (1) {
+		/* Event missed register(s) */
+		for (j = 0; j < 2; j++) {
+			unsigned long emr;
+
+			val = edma_read_array(ecc, EDMA_EMR, j);
+			if (!val)
+				continue;
+
+			dev_dbg(ecc->dev, "EMR%d 0x%08x\n", j, val);
+			emr = val;
+			for (i = find_next_bit(&emr, 32, 0); i < 32;
+			     i = find_next_bit(&emr, 32, i + 1)) {
+				int k = (j << 5) + i;
+
+				/* Clear the corresponding EMR bits */
+				edma_write_array(ecc, EDMA_EMCR, j, BIT(i));
+				/* Clear any SER */
+				edma_shadow0_write_array(ecc, SH_SECR, j,
+							 BIT(i));
+				edma_error_handler(&ecc->slave_chans[k]);
+			}
+		}
+
+		val = edma_read(ecc, EDMA_QEMR);
+		if (val) {
+			dev_dbg(ecc->dev, "QEMR 0x%02x\n", val);
+			/* Not reported, just clear the interrupt reason. */
+			edma_write(ecc, EDMA_QEMCR, val);
+			edma_shadow0_write(ecc, SH_QSECR, val);
+		}
+
+		val = edma_read(ecc, EDMA_CCERR);
+		if (val) {
+			dev_warn(ecc->dev, "CCERR 0x%08x\n", val);
+			/* Not reported, just clear the interrupt reason. */
+			edma_write(ecc, EDMA_CCERRCLR, val);
+		}
+
+		if (!edma_error_pending(ecc))
+			break;
+		cnt++;
+		if (cnt > 10)
+			break;
+	}
+	edma_write(ecc, EDMA_EEVAL, 1);
+	return IRQ_HANDLED;
+}
+
+/* Alloc channel resources */
+static int edma_alloc_chan_resources(struct dma_chan *chan)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+	struct edma_cc *ecc = echan->ecc;
+	struct device *dev = ecc->dev;
+	enum dma_event_q eventq_no = EVENTQ_DEFAULT;
+	int ret;
+
+	if (echan->tc) {
+		eventq_no = echan->tc->id;
+	} else if (ecc->tc_list) {
+		/* memcpy channel */
+		echan->tc = &ecc->tc_list[ecc->info->default_queue];
+		eventq_no = echan->tc->id;
+	}
+
+	ret = edma_alloc_channel(echan, eventq_no);
+	if (ret)
+		return ret;
+
+	echan->slot[0] = edma_alloc_slot(ecc, echan->ch_num);
+	if (echan->slot[0] < 0) {
+		dev_err(dev, "Entry slot allocation failed for channel %u\n",
+			EDMA_CHAN_SLOT(echan->ch_num));
+		ret = echan->slot[0];
+		goto err_slot;
+	}
+
+	/* Set up channel -> slot mapping for the entry slot */
+	edma_set_chmap(echan, echan->slot[0]);
+	echan->alloced = true;
+
+	dev_dbg(dev, "Got eDMA channel %d for virt channel %d (%s trigger)\n",
+		EDMA_CHAN_SLOT(echan->ch_num), chan->chan_id,
+		echan->hw_triggered ? "HW" : "SW");
+
+	return 0;
+
+err_slot:
+	edma_free_channel(echan);
+	return ret;
+}
+
+/* Free channel resources */
+static void edma_free_chan_resources(struct dma_chan *chan)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+	struct device *dev = echan->ecc->dev;
+	int i;
+
+	/* Terminate transfers */
+	edma_stop(echan);
+
+	vchan_free_chan_resources(&echan->vchan);
+
+	/* Free EDMA PaRAM slots */
+	for (i = 0; i < EDMA_MAX_SLOTS; i++) {
+		if (echan->slot[i] >= 0) {
+			edma_free_slot(echan->ecc, echan->slot[i]);
+			echan->slot[i] = -1;
+		}
+	}
+
+	/* Set entry slot to the dummy slot */
+	edma_set_chmap(echan, echan->ecc->dummy_slot);
+
+	/* Free EDMA channel */
+	if (echan->alloced) {
+		edma_free_channel(echan);
+		echan->alloced = false;
+	}
+
+	echan->tc = NULL;
+	echan->hw_triggered = false;
+
+	dev_dbg(dev, "Free eDMA channel %d for virt channel %d\n",
+		EDMA_CHAN_SLOT(echan->ch_num), chan->chan_id);
+}
+
+/* Send pending descriptor to hardware */
+static void edma_issue_pending(struct dma_chan *chan)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+	unsigned long flags;
+
+	spin_lock_irqsave(&echan->vchan.lock, flags);
+	if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
+		edma_execute(echan);
+	spin_unlock_irqrestore(&echan->vchan.lock, flags);
+}
+
+/*
+ * This limit exists to avoid a possible infinite loop when waiting for proof
+ * that a particular transfer is completed. This limit can be hit if there
+ * are large bursts to/from slow devices or the CPU is never able to catch
+ * the DMA hardware idle. On an AM335x transfering 48 bytes from the UART
+ * RX-FIFO, as many as 55 loops have been seen.
+ */
+#define EDMA_MAX_TR_WAIT_LOOPS 1000
+
+static u32 edma_residue(struct edma_desc *edesc)
+{
+	bool dst = edesc->direction == DMA_DEV_TO_MEM;
+	int loop_count = EDMA_MAX_TR_WAIT_LOOPS;
+	struct edma_chan *echan = edesc->echan;
+	struct edma_pset *pset = edesc->pset;
+	dma_addr_t done, pos;
+	int i;
+
+	/*
+	 * We always read the dst/src position from the first RamPar
+	 * pset. That's the one which is active now.
+	 */
+	pos = edma_get_position(echan->ecc, echan->slot[0], dst);
+
+	/*
+	 * "pos" may represent a transfer request that is still being
+	 * processed by the EDMACC or EDMATC. We will busy wait until
+	 * any one of the situations occurs:
+	 *   1. the DMA hardware is idle
+	 *   2. a new transfer request is setup
+	 *   3. we hit the loop limit
+	 */
+	while (edma_read(echan->ecc, EDMA_CCSTAT) & EDMA_CCSTAT_ACTV) {
+		/* check if a new transfer request is setup */
+		if (edma_get_position(echan->ecc,
+				      echan->slot[0], dst) != pos) {
+			break;
+		}
+
+		if (!--loop_count) {
+			dev_dbg_ratelimited(echan->vchan.chan.device->dev,
+				"%s: timeout waiting for PaRAM update\n",
+				__func__);
+			break;
+		}
+
+		cpu_relax();
+	}
+
+	/*
+	 * Cyclic is simple. Just subtract pset[0].addr from pos.
+	 *
+	 * We never update edesc->residue in the cyclic case, so we
+	 * can tell the remaining room to the end of the circular
+	 * buffer.
+	 */
+	if (edesc->cyclic) {
+		done = pos - pset->addr;
+		edesc->residue_stat = edesc->residue - done;
+		return edesc->residue_stat;
+	}
+
+	/*
+	 * For SG operation we catch up with the last processed
+	 * status.
+	 */
+	pset += edesc->processed_stat;
+
+	for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
+		/*
+		 * If we are inside this pset address range, we know
+		 * this is the active one. Get the current delta and
+		 * stop walking the psets.
+		 */
+		if (pos >= pset->addr && pos < pset->addr + pset->len)
+			return edesc->residue_stat - (pos - pset->addr);
+
+		/* Otherwise mark it done and update residue_stat. */
+		edesc->processed_stat++;
+		edesc->residue_stat -= pset->len;
+	}
+	return edesc->residue_stat;
+}
+
+/* Check request completion status */
+static enum dma_status edma_tx_status(struct dma_chan *chan,
+				      dma_cookie_t cookie,
+				      struct dma_tx_state *txstate)
+{
+	struct edma_chan *echan = to_edma_chan(chan);
+	struct virt_dma_desc *vdesc;
+	enum dma_status ret;
+	unsigned long flags;
+
+	ret = dma_cookie_status(chan, cookie, txstate);
+	if (ret == DMA_COMPLETE || !txstate)
+		return ret;
+
+	spin_lock_irqsave(&echan->vchan.lock, flags);
+	if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
+		txstate->residue = edma_residue(echan->edesc);
+	else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
+		txstate->residue = to_edma_desc(&vdesc->tx)->residue;
+	spin_unlock_irqrestore(&echan->vchan.lock, flags);
+
+	return ret;
+}
+
+static bool edma_is_memcpy_channel(int ch_num, s32 *memcpy_channels)
+{
+	if (!memcpy_channels)
+		return false;
+	while (*memcpy_channels != -1) {
+		if (*memcpy_channels == ch_num)
+			return true;
+		memcpy_channels++;
+	}
+	return false;
+}
+
+#define EDMA_DMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
+
+static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode)
+{
+	struct dma_device *s_ddev = &ecc->dma_slave;
+	struct dma_device *m_ddev = NULL;
+	s32 *memcpy_channels = ecc->info->memcpy_channels;
+	int i, j;
+
+	dma_cap_zero(s_ddev->cap_mask);
+	dma_cap_set(DMA_SLAVE, s_ddev->cap_mask);
+	dma_cap_set(DMA_CYCLIC, s_ddev->cap_mask);
+	if (ecc->legacy_mode && !memcpy_channels) {
+		dev_warn(ecc->dev,
+			 "Legacy memcpy is enabled, things might not work\n");
+
+		dma_cap_set(DMA_MEMCPY, s_ddev->cap_mask);
+		s_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy;
+		s_ddev->directions = BIT(DMA_MEM_TO_MEM);
+	}
+
+	s_ddev->device_prep_slave_sg = edma_prep_slave_sg;
+	s_ddev->device_prep_dma_cyclic = edma_prep_dma_cyclic;
+	s_ddev->device_alloc_chan_resources = edma_alloc_chan_resources;
+	s_ddev->device_free_chan_resources = edma_free_chan_resources;
+	s_ddev->device_issue_pending = edma_issue_pending;
+	s_ddev->device_tx_status = edma_tx_status;
+	s_ddev->device_config = edma_slave_config;
+	s_ddev->device_pause = edma_dma_pause;
+	s_ddev->device_resume = edma_dma_resume;
+	s_ddev->device_terminate_all = edma_terminate_all;
+	s_ddev->device_synchronize = edma_synchronize;
+
+	s_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS;
+	s_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
+	s_ddev->directions |= (BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV));
+	s_ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+	s_ddev->max_burst = SZ_32K - 1; /* CIDX: 16bit signed */
+
+	s_ddev->dev = ecc->dev;
+	INIT_LIST_HEAD(&s_ddev->channels);
+
+	if (memcpy_channels) {
+		m_ddev = devm_kzalloc(ecc->dev, sizeof(*m_ddev), GFP_KERNEL);
+		if (!m_ddev) {
+			dev_warn(ecc->dev, "memcpy is disabled due to OoM\n");
+			memcpy_channels = NULL;
+			goto ch_setup;
+		}
+		ecc->dma_memcpy = m_ddev;
+
+		dma_cap_zero(m_ddev->cap_mask);
+		dma_cap_set(DMA_MEMCPY, m_ddev->cap_mask);
+
+		m_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy;
+		m_ddev->device_alloc_chan_resources = edma_alloc_chan_resources;
+		m_ddev->device_free_chan_resources = edma_free_chan_resources;
+		m_ddev->device_issue_pending = edma_issue_pending;
+		m_ddev->device_tx_status = edma_tx_status;
+		m_ddev->device_config = edma_slave_config;
+		m_ddev->device_pause = edma_dma_pause;
+		m_ddev->device_resume = edma_dma_resume;
+		m_ddev->device_terminate_all = edma_terminate_all;
+		m_ddev->device_synchronize = edma_synchronize;
+
+		m_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS;
+		m_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
+		m_ddev->directions = BIT(DMA_MEM_TO_MEM);
+		m_ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+
+		m_ddev->dev = ecc->dev;
+		INIT_LIST_HEAD(&m_ddev->channels);
+	} else if (!ecc->legacy_mode) {
+		dev_info(ecc->dev, "memcpy is disabled\n");
+	}
+
+ch_setup:
+	for (i = 0; i < ecc->num_channels; i++) {
+		struct edma_chan *echan = &ecc->slave_chans[i];
+		echan->ch_num = EDMA_CTLR_CHAN(ecc->id, i);
+		echan->ecc = ecc;
+		echan->vchan.desc_free = edma_desc_free;
+
+		if (m_ddev && edma_is_memcpy_channel(i, memcpy_channels))
+			vchan_init(&echan->vchan, m_ddev);
+		else
+			vchan_init(&echan->vchan, s_ddev);
+
+		INIT_LIST_HEAD(&echan->node);
+		for (j = 0; j < EDMA_MAX_SLOTS; j++)
+			echan->slot[j] = -1;
+	}
+}
+
+static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata,
+			      struct edma_cc *ecc)
+{
+	int i;
+	u32 value, cccfg;
+	s8 (*queue_priority_map)[2];
+
+	/* Decode the eDMA3 configuration from CCCFG register */
+	cccfg = edma_read(ecc, EDMA_CCCFG);
+
+	value = GET_NUM_REGN(cccfg);
+	ecc->num_region = BIT(value);
+
+	value = GET_NUM_DMACH(cccfg);
+	ecc->num_channels = BIT(value + 1);
+
+	value = GET_NUM_QDMACH(cccfg);
+	ecc->num_qchannels = value * 2;
+
+	value = GET_NUM_PAENTRY(cccfg);
+	ecc->num_slots = BIT(value + 4);
+
+	value = GET_NUM_EVQUE(cccfg);
+	ecc->num_tc = value + 1;
+
+	ecc->chmap_exist = (cccfg & CHMAP_EXIST) ? true : false;
+
+	dev_dbg(dev, "eDMA3 CC HW configuration (cccfg: 0x%08x):\n", cccfg);
+	dev_dbg(dev, "num_region: %u\n", ecc->num_region);
+	dev_dbg(dev, "num_channels: %u\n", ecc->num_channels);
+	dev_dbg(dev, "num_qchannels: %u\n", ecc->num_qchannels);
+	dev_dbg(dev, "num_slots: %u\n", ecc->num_slots);
+	dev_dbg(dev, "num_tc: %u\n", ecc->num_tc);
+	dev_dbg(dev, "chmap_exist: %s\n", ecc->chmap_exist ? "yes" : "no");
+
+	/* Nothing need to be done if queue priority is provided */
+	if (pdata->queue_priority_mapping)
+		return 0;
+
+	/*
+	 * Configure TC/queue priority as follows:
+	 * Q0 - priority 0
+	 * Q1 - priority 1
+	 * Q2 - priority 2
+	 * ...
+	 * The meaning of priority numbers: 0 highest priority, 7 lowest
+	 * priority. So Q0 is the highest priority queue and the last queue has
+	 * the lowest priority.
+	 */
+	queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, sizeof(s8),
+					  GFP_KERNEL);
+	if (!queue_priority_map)
+		return -ENOMEM;
+
+	for (i = 0; i < ecc->num_tc; i++) {
+		queue_priority_map[i][0] = i;
+		queue_priority_map[i][1] = i;
+	}
+	queue_priority_map[i][0] = -1;
+	queue_priority_map[i][1] = -1;
+
+	pdata->queue_priority_mapping = queue_priority_map;
+	/* Default queue has the lowest priority */
+	pdata->default_queue = i - 1;
+
+	return 0;
+}
+
+#if IS_ENABLED(CONFIG_OF)
+static int edma_xbar_event_map(struct device *dev, struct edma_soc_info *pdata,
+			       size_t sz)
+{
+	const char pname[] = "ti,edma-xbar-event-map";
+	struct resource res;
+	void __iomem *xbar;
+	s16 (*xbar_chans)[2];
+	size_t nelm = sz / sizeof(s16);
+	u32 shift, offset, mux;
+	int ret, i;
+
+	xbar_chans = devm_kcalloc(dev, nelm + 2, sizeof(s16), GFP_KERNEL);
+	if (!xbar_chans)
+		return -ENOMEM;
+
+	ret = of_address_to_resource(dev->of_node, 1, &res);
+	if (ret)
+		return -ENOMEM;
+
+	xbar = devm_ioremap(dev, res.start, resource_size(&res));
+	if (!xbar)
+		return -ENOMEM;
+
+	ret = of_property_read_u16_array(dev->of_node, pname, (u16 *)xbar_chans,
+					 nelm);
+	if (ret)
+		return -EIO;
+
+	/* Invalidate last entry for the other user of this mess */
+	nelm >>= 1;
+	xbar_chans[nelm][0] = -1;
+	xbar_chans[nelm][1] = -1;
+
+	for (i = 0; i < nelm; i++) {
+		shift = (xbar_chans[i][1] & 0x03) << 3;
+		offset = xbar_chans[i][1] & 0xfffffffc;
+		mux = readl(xbar + offset);
+		mux &= ~(0xff << shift);
+		mux |= xbar_chans[i][0] << shift;
+		writel(mux, (xbar + offset));
+	}
+
+	pdata->xbar_chans = (const s16 (*)[2]) xbar_chans;
+	return 0;
+}
+
+static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev,
+						     bool legacy_mode)
+{
+	struct edma_soc_info *info;
+	struct property *prop;
+	int sz, ret;
+
+	info = devm_kzalloc(dev, sizeof(struct edma_soc_info), GFP_KERNEL);
+	if (!info)
+		return ERR_PTR(-ENOMEM);
+
+	if (legacy_mode) {
+		prop = of_find_property(dev->of_node, "ti,edma-xbar-event-map",
+					&sz);
+		if (prop) {
+			ret = edma_xbar_event_map(dev, info, sz);
+			if (ret)
+				return ERR_PTR(ret);
+		}
+		return info;
+	}
+
+	/* Get the list of channels allocated to be used for memcpy */
+	prop = of_find_property(dev->of_node, "ti,edma-memcpy-channels", &sz);
+	if (prop) {
+		const char pname[] = "ti,edma-memcpy-channels";
+		size_t nelm = sz / sizeof(s32);
+		s32 *memcpy_ch;
+
+		memcpy_ch = devm_kcalloc(dev, nelm + 1, sizeof(s32),
+					 GFP_KERNEL);
+		if (!memcpy_ch)
+			return ERR_PTR(-ENOMEM);
+
+		ret = of_property_read_u32_array(dev->of_node, pname,
+						 (u32 *)memcpy_ch, nelm);
+		if (ret)
+			return ERR_PTR(ret);
+
+		memcpy_ch[nelm] = -1;
+		info->memcpy_channels = memcpy_ch;
+	}
+
+	prop = of_find_property(dev->of_node, "ti,edma-reserved-slot-ranges",
+				&sz);
+	if (prop) {
+		const char pname[] = "ti,edma-reserved-slot-ranges";
+		u32 (*tmp)[2];
+		s16 (*rsv_slots)[2];
+		size_t nelm = sz / sizeof(*tmp);
+		struct edma_rsv_info *rsv_info;
+		int i;
+
+		if (!nelm)
+			return info;
+
+		tmp = kcalloc(nelm, sizeof(*tmp), GFP_KERNEL);
+		if (!tmp)
+			return ERR_PTR(-ENOMEM);
+
+		rsv_info = devm_kzalloc(dev, sizeof(*rsv_info), GFP_KERNEL);
+		if (!rsv_info) {
+			kfree(tmp);
+			return ERR_PTR(-ENOMEM);
+		}
+
+		rsv_slots = devm_kcalloc(dev, nelm + 1, sizeof(*rsv_slots),
+					 GFP_KERNEL);
+		if (!rsv_slots) {
+			kfree(tmp);
+			return ERR_PTR(-ENOMEM);
+		}
+
+		ret = of_property_read_u32_array(dev->of_node, pname,
+						 (u32 *)tmp, nelm * 2);
+		if (ret) {
+			kfree(tmp);
+			return ERR_PTR(ret);
+		}
+
+		for (i = 0; i < nelm; i++) {
+			rsv_slots[i][0] = tmp[i][0];
+			rsv_slots[i][1] = tmp[i][1];
+		}
+		rsv_slots[nelm][0] = -1;
+		rsv_slots[nelm][1] = -1;
+
+		info->rsv = rsv_info;
+		info->rsv->rsv_slots = (const s16 (*)[2])rsv_slots;
+
+		kfree(tmp);
+	}
+
+	return info;
+}
+
+static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec,
+				      struct of_dma *ofdma)
+{
+	struct edma_cc *ecc = ofdma->of_dma_data;
+	struct dma_chan *chan = NULL;
+	struct edma_chan *echan;
+	int i;
+
+	if (!ecc || dma_spec->args_count < 1)
+		return NULL;
+
+	for (i = 0; i < ecc->num_channels; i++) {
+		echan = &ecc->slave_chans[i];
+		if (echan->ch_num == dma_spec->args[0]) {
+			chan = &echan->vchan.chan;
+			break;
+		}
+	}
+
+	if (!chan)
+		return NULL;
+
+	if (echan->ecc->legacy_mode && dma_spec->args_count == 1)
+		goto out;
+
+	if (!echan->ecc->legacy_mode && dma_spec->args_count == 2 &&
+	    dma_spec->args[1] < echan->ecc->num_tc) {
+		echan->tc = &echan->ecc->tc_list[dma_spec->args[1]];
+		goto out;
+	}
+
+	return NULL;
+out:
+	/* The channel is going to be used as HW synchronized */
+	echan->hw_triggered = true;
+	return dma_get_slave_channel(chan);
+}
+#else
+static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev,
+						     bool legacy_mode)
+{
+	return ERR_PTR(-EINVAL);
+}
+
+static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec,
+				      struct of_dma *ofdma)
+{
+	return NULL;
+}
+#endif
+
+static int edma_probe(struct platform_device *pdev)
+{
+	struct edma_soc_info	*info = pdev->dev.platform_data;
+	s8			(*queue_priority_mapping)[2];
+	int			i, off, ln;
+	const s16		(*rsv_slots)[2];
+	const s16		(*xbar_chans)[2];
+	int			irq;
+	char			*irq_name;
+	struct resource		*mem;
+	struct device_node	*node = pdev->dev.of_node;
+	struct device		*dev = &pdev->dev;
+	struct edma_cc		*ecc;
+	bool			legacy_mode = true;
+	int ret;
+
+	if (node) {
+		const struct of_device_id *match;
+
+		match = of_match_node(edma_of_ids, node);
+		if (match && (*(u32 *)match->data) == EDMA_BINDING_TPCC)
+			legacy_mode = false;
+
+		info = edma_setup_info_from_dt(dev, legacy_mode);
+		if (IS_ERR(info)) {
+			dev_err(dev, "failed to get DT data\n");
+			return PTR_ERR(info);
+		}
+	}
+
+	if (!info)
+		return -ENODEV;
+
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		dev_err(dev, "pm_runtime_get_sync() failed\n");
+		return ret;
+	}
+
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (ret)
+		return ret;
+
+	ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL);
+	if (!ecc)
+		return -ENOMEM;
+
+	ecc->dev = dev;
+	ecc->id = pdev->id;
+	ecc->legacy_mode = legacy_mode;
+	/* When booting with DT the pdev->id is -1 */
+	if (ecc->id < 0)
+		ecc->id = 0;
+
+	mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "edma3_cc");
+	if (!mem) {
+		dev_dbg(dev, "mem resource not found, using index 0\n");
+		mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		if (!mem) {
+			dev_err(dev, "no mem resource?\n");
+			return -ENODEV;
+		}
+	}
+	ecc->base = devm_ioremap_resource(dev, mem);
+	if (IS_ERR(ecc->base))
+		return PTR_ERR(ecc->base);
+
+	platform_set_drvdata(pdev, ecc);
+
+	/* Get eDMA3 configuration from IP */
+	ret = edma_setup_from_hw(dev, info, ecc);
+	if (ret)
+		return ret;
+
+	/* Allocate memory based on the information we got from the IP */
+	ecc->slave_chans = devm_kcalloc(dev, ecc->num_channels,
+					sizeof(*ecc->slave_chans), GFP_KERNEL);
+	if (!ecc->slave_chans)
+		return -ENOMEM;
+
+	ecc->slot_inuse = devm_kcalloc(dev, BITS_TO_LONGS(ecc->num_slots),
+				       sizeof(unsigned long), GFP_KERNEL);
+	if (!ecc->slot_inuse)
+		return -ENOMEM;
+
+	ecc->default_queue = info->default_queue;
+
+	for (i = 0; i < ecc->num_slots; i++)
+		edma_write_slot(ecc, i, &dummy_paramset);
+
+	if (info->rsv) {
+		/* Set the reserved slots in inuse list */
+		rsv_slots = info->rsv->rsv_slots;
+		if (rsv_slots) {
+			for (i = 0; rsv_slots[i][0] != -1; i++) {
+				off = rsv_slots[i][0];
+				ln = rsv_slots[i][1];
+				edma_set_bits(off, ln, ecc->slot_inuse);
+			}
+		}
+	}
+
+	/* Clear the xbar mapped channels in unused list */
+	xbar_chans = info->xbar_chans;
+	if (xbar_chans) {
+		for (i = 0; xbar_chans[i][1] != -1; i++) {
+			off = xbar_chans[i][1];
+		}
+	}
+
+	irq = platform_get_irq_byname(pdev, "edma3_ccint");
+	if (irq < 0 && node)
+		irq = irq_of_parse_and_map(node, 0);
+
+	if (irq >= 0) {
+		irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccint",
+					  dev_name(dev));
+		ret = devm_request_irq(dev, irq, dma_irq_handler, 0, irq_name,
+				       ecc);
+		if (ret) {
+			dev_err(dev, "CCINT (%d) failed --> %d\n", irq, ret);
+			return ret;
+		}
+		ecc->ccint = irq;
+	}
+
+	irq = platform_get_irq_byname(pdev, "edma3_ccerrint");
+	if (irq < 0 && node)
+		irq = irq_of_parse_and_map(node, 2);
+
+	if (irq >= 0) {
+		irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccerrint",
+					  dev_name(dev));
+		ret = devm_request_irq(dev, irq, dma_ccerr_handler, 0, irq_name,
+				       ecc);
+		if (ret) {
+			dev_err(dev, "CCERRINT (%d) failed --> %d\n", irq, ret);
+			return ret;
+		}
+		ecc->ccerrint = irq;
+	}
+
+	ecc->dummy_slot = edma_alloc_slot(ecc, EDMA_SLOT_ANY);
+	if (ecc->dummy_slot < 0) {
+		dev_err(dev, "Can't allocate PaRAM dummy slot\n");
+		return ecc->dummy_slot;
+	}
+
+	queue_priority_mapping = info->queue_priority_mapping;
+
+	if (!ecc->legacy_mode) {
+		int lowest_priority = 0;
+		struct of_phandle_args tc_args;
+
+		ecc->tc_list = devm_kcalloc(dev, ecc->num_tc,
+					    sizeof(*ecc->tc_list), GFP_KERNEL);
+		if (!ecc->tc_list)
+			return -ENOMEM;
+
+		for (i = 0;; i++) {
+			ret = of_parse_phandle_with_fixed_args(node, "ti,tptcs",
+							       1, i, &tc_args);
+			if (ret || i == ecc->num_tc)
+				break;
+
+			ecc->tc_list[i].node = tc_args.np;
+			ecc->tc_list[i].id = i;
+			queue_priority_mapping[i][1] = tc_args.args[0];
+			if (queue_priority_mapping[i][1] > lowest_priority) {
+				lowest_priority = queue_priority_mapping[i][1];
+				info->default_queue = i;
+			}
+		}
+	}
+
+	/* Event queue priority mapping */
+	for (i = 0; queue_priority_mapping[i][0] != -1; i++)
+		edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
+					      queue_priority_mapping[i][1]);
+
+	for (i = 0; i < ecc->num_region; i++) {
+		edma_write_array2(ecc, EDMA_DRAE, i, 0, 0x0);
+		edma_write_array2(ecc, EDMA_DRAE, i, 1, 0x0);
+		edma_write_array(ecc, EDMA_QRAE, i, 0x0);
+	}
+	ecc->info = info;
+
+	/* Init the dma device and channels */
+	edma_dma_init(ecc, legacy_mode);
+
+	for (i = 0; i < ecc->num_channels; i++) {
+		/* Assign all channels to the default queue */
+		edma_assign_channel_eventq(&ecc->slave_chans[i],
+					   info->default_queue);
+		/* Set entry slot to the dummy slot */
+		edma_set_chmap(&ecc->slave_chans[i], ecc->dummy_slot);
+	}
+
+	ecc->dma_slave.filter.map = info->slave_map;
+	ecc->dma_slave.filter.mapcnt = info->slavecnt;
+	ecc->dma_slave.filter.fn = edma_filter_fn;
+
+	ret = dma_async_device_register(&ecc->dma_slave);
+	if (ret) {
+		dev_err(dev, "slave ddev registration failed (%d)\n", ret);
+		goto err_reg1;
+	}
+
+	if (ecc->dma_memcpy) {
+		ret = dma_async_device_register(ecc->dma_memcpy);
+		if (ret) {
+			dev_err(dev, "memcpy ddev registration failed (%d)\n",
+				ret);
+			dma_async_device_unregister(&ecc->dma_slave);
+			goto err_reg1;
+		}
+	}
+
+	if (node)
+		of_dma_controller_register(node, of_edma_xlate, ecc);
+
+	dev_info(dev, "TI EDMA DMA engine driver\n");
+
+	return 0;
+
+err_reg1:
+	edma_free_slot(ecc, ecc->dummy_slot);
+	return ret;
+}
+
+static void edma_cleanupp_vchan(struct dma_device *dmadev)
+{
+	struct edma_chan *echan, *_echan;
+
+	list_for_each_entry_safe(echan, _echan,
+			&dmadev->channels, vchan.chan.device_node) {
+		list_del(&echan->vchan.chan.device_node);
+		tasklet_kill(&echan->vchan.task);
+	}
+}
+
+static int edma_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct edma_cc *ecc = dev_get_drvdata(dev);
+
+	devm_free_irq(dev, ecc->ccint, ecc);
+	devm_free_irq(dev, ecc->ccerrint, ecc);
+
+	edma_cleanupp_vchan(&ecc->dma_slave);
+
+	if (dev->of_node)
+		of_dma_controller_free(dev->of_node);
+	dma_async_device_unregister(&ecc->dma_slave);
+	if (ecc->dma_memcpy)
+		dma_async_device_unregister(ecc->dma_memcpy);
+	edma_free_slot(ecc, ecc->dummy_slot);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int edma_pm_suspend(struct device *dev)
+{
+	struct edma_cc *ecc = dev_get_drvdata(dev);
+	struct edma_chan *echan = ecc->slave_chans;
+	int i;
+
+	for (i = 0; i < ecc->num_channels; i++) {
+		if (echan[i].alloced)
+			edma_setup_interrupt(&echan[i], false);
+	}
+
+	return 0;
+}
+
+static int edma_pm_resume(struct device *dev)
+{
+	struct edma_cc *ecc = dev_get_drvdata(dev);
+	struct edma_chan *echan = ecc->slave_chans;
+	int i;
+	s8 (*queue_priority_mapping)[2];
+
+	/* re initialize dummy slot to dummy param set */
+	edma_write_slot(ecc, ecc->dummy_slot, &dummy_paramset);
+
+	queue_priority_mapping = ecc->info->queue_priority_mapping;
+
+	/* Event queue priority mapping */
+	for (i = 0; queue_priority_mapping[i][0] != -1; i++)
+		edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
+					      queue_priority_mapping[i][1]);
+
+	for (i = 0; i < ecc->num_channels; i++) {
+		if (echan[i].alloced) {
+			/* ensure access through shadow region 0 */
+			edma_or_array2(ecc, EDMA_DRAE, 0, i >> 5,
+				       BIT(i & 0x1f));
+
+			edma_setup_interrupt(&echan[i], true);
+
+			/* Set up channel -> slot mapping for the entry slot */
+			edma_set_chmap(&echan[i], echan[i].slot[0]);
+		}
+	}
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops edma_pm_ops = {
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(edma_pm_suspend, edma_pm_resume)
+};
+
+static struct platform_driver edma_driver = {
+	.probe		= edma_probe,
+	.remove		= edma_remove,
+	.driver = {
+		.name	= "edma",
+		.pm	= &edma_pm_ops,
+		.of_match_table = edma_of_ids,
+	},
+};
+
+static int edma_tptc_probe(struct platform_device *pdev)
+{
+	pm_runtime_enable(&pdev->dev);
+	return pm_runtime_get_sync(&pdev->dev);
+}
+
+static struct platform_driver edma_tptc_driver = {
+	.probe		= edma_tptc_probe,
+	.driver = {
+		.name	= "edma3-tptc",
+		.of_match_table = edma_tptc_of_ids,
+	},
+};
+
+bool edma_filter_fn(struct dma_chan *chan, void *param)
+{
+	bool match = false;
+
+	if (chan->device->dev->driver == &edma_driver.driver) {
+		struct edma_chan *echan = to_edma_chan(chan);
+		unsigned ch_req = *(unsigned *)param;
+		if (ch_req == echan->ch_num) {
+			/* The channel is going to be used as HW synchronized */
+			echan->hw_triggered = true;
+			match = true;
+		}
+	}
+	return match;
+}
+EXPORT_SYMBOL(edma_filter_fn);
+
+static int edma_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&edma_tptc_driver);
+	if (ret)
+		return ret;
+
+	return platform_driver_register(&edma_driver);
+}
+subsys_initcall(edma_init);
+
+static void __exit edma_exit(void)
+{
+	platform_driver_unregister(&edma_driver);
+	platform_driver_unregister(&edma_tptc_driver);
+}
+module_exit(edma_exit);
+
+MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
+MODULE_DESCRIPTION("TI EDMA DMA engine driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/ti/omap-dma.c b/drivers/dma/ti/omap-dma.c
new file mode 100644
index 000000000000..b73fb51fbc81
--- /dev/null
+++ b/drivers/dma/ti/omap-dma.c
@@ -0,0 +1,1668 @@
+/*
+ * OMAP DMAengine support
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/omap-dma.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/of_dma.h>
+#include <linux/of_device.h>
+
+#include "../virt-dma.h"
+
+#define OMAP_SDMA_REQUESTS	127
+#define OMAP_SDMA_CHANNELS	32
+
+struct omap_dmadev {
+	struct dma_device ddev;
+	spinlock_t lock;
+	void __iomem *base;
+	const struct omap_dma_reg *reg_map;
+	struct omap_system_dma_plat_info *plat;
+	bool legacy;
+	bool ll123_supported;
+	struct dma_pool *desc_pool;
+	unsigned dma_requests;
+	spinlock_t irq_lock;
+	uint32_t irq_enable_mask;
+	struct omap_chan **lch_map;
+};
+
+struct omap_chan {
+	struct virt_dma_chan vc;
+	void __iomem *channel_base;
+	const struct omap_dma_reg *reg_map;
+	uint32_t ccr;
+
+	struct dma_slave_config	cfg;
+	unsigned dma_sig;
+	bool cyclic;
+	bool paused;
+	bool running;
+
+	int dma_ch;
+	struct omap_desc *desc;
+	unsigned sgidx;
+};
+
+#define DESC_NXT_SV_REFRESH	(0x1 << 24)
+#define DESC_NXT_SV_REUSE	(0x2 << 24)
+#define DESC_NXT_DV_REFRESH	(0x1 << 26)
+#define DESC_NXT_DV_REUSE	(0x2 << 26)
+#define DESC_NTYPE_TYPE2	(0x2 << 29)
+
+/* Type 2 descriptor with Source or Destination address update */
+struct omap_type2_desc {
+	uint32_t next_desc;
+	uint32_t en;
+	uint32_t addr; /* src or dst */
+	uint16_t fn;
+	uint16_t cicr;
+	int16_t cdei;
+	int16_t csei;
+	int32_t cdfi;
+	int32_t csfi;
+} __packed;
+
+struct omap_sg {
+	dma_addr_t addr;
+	uint32_t en;		/* number of elements (24-bit) */
+	uint32_t fn;		/* number of frames (16-bit) */
+	int32_t fi;		/* for double indexing */
+	int16_t ei;		/* for double indexing */
+
+	/* Linked list */
+	struct omap_type2_desc *t2_desc;
+	dma_addr_t t2_desc_paddr;
+};
+
+struct omap_desc {
+	struct virt_dma_desc vd;
+	bool using_ll;
+	enum dma_transfer_direction dir;
+	dma_addr_t dev_addr;
+
+	int32_t fi;		/* for OMAP_DMA_SYNC_PACKET / double indexing */
+	int16_t ei;		/* for double indexing */
+	uint8_t es;		/* CSDP_DATA_TYPE_xxx */
+	uint32_t ccr;		/* CCR value */
+	uint16_t clnk_ctrl;	/* CLNK_CTRL value */
+	uint16_t cicr;		/* CICR value */
+	uint32_t csdp;		/* CSDP value */
+
+	unsigned sglen;
+	struct omap_sg sg[0];
+};
+
+enum {
+	CAPS_0_SUPPORT_LL123	= BIT(20),	/* Linked List type1/2/3 */
+	CAPS_0_SUPPORT_LL4	= BIT(21),	/* Linked List type4 */
+
+	CCR_FS			= BIT(5),
+	CCR_READ_PRIORITY	= BIT(6),
+	CCR_ENABLE		= BIT(7),
+	CCR_AUTO_INIT		= BIT(8),	/* OMAP1 only */
+	CCR_REPEAT		= BIT(9),	/* OMAP1 only */
+	CCR_OMAP31_DISABLE	= BIT(10),	/* OMAP1 only */
+	CCR_SUSPEND_SENSITIVE	= BIT(8),	/* OMAP2+ only */
+	CCR_RD_ACTIVE		= BIT(9),	/* OMAP2+ only */
+	CCR_WR_ACTIVE		= BIT(10),	/* OMAP2+ only */
+	CCR_SRC_AMODE_CONSTANT	= 0 << 12,
+	CCR_SRC_AMODE_POSTINC	= 1 << 12,
+	CCR_SRC_AMODE_SGLIDX	= 2 << 12,
+	CCR_SRC_AMODE_DBLIDX	= 3 << 12,
+	CCR_DST_AMODE_CONSTANT	= 0 << 14,
+	CCR_DST_AMODE_POSTINC	= 1 << 14,
+	CCR_DST_AMODE_SGLIDX	= 2 << 14,
+	CCR_DST_AMODE_DBLIDX	= 3 << 14,
+	CCR_CONSTANT_FILL	= BIT(16),
+	CCR_TRANSPARENT_COPY	= BIT(17),
+	CCR_BS			= BIT(18),
+	CCR_SUPERVISOR		= BIT(22),
+	CCR_PREFETCH		= BIT(23),
+	CCR_TRIGGER_SRC		= BIT(24),
+	CCR_BUFFERING_DISABLE	= BIT(25),
+	CCR_WRITE_PRIORITY	= BIT(26),
+	CCR_SYNC_ELEMENT	= 0,
+	CCR_SYNC_FRAME		= CCR_FS,
+	CCR_SYNC_BLOCK		= CCR_BS,
+	CCR_SYNC_PACKET		= CCR_BS | CCR_FS,
+
+	CSDP_DATA_TYPE_8	= 0,
+	CSDP_DATA_TYPE_16	= 1,
+	CSDP_DATA_TYPE_32	= 2,
+	CSDP_SRC_PORT_EMIFF	= 0 << 2, /* OMAP1 only */
+	CSDP_SRC_PORT_EMIFS	= 1 << 2, /* OMAP1 only */
+	CSDP_SRC_PORT_OCP_T1	= 2 << 2, /* OMAP1 only */
+	CSDP_SRC_PORT_TIPB	= 3 << 2, /* OMAP1 only */
+	CSDP_SRC_PORT_OCP_T2	= 4 << 2, /* OMAP1 only */
+	CSDP_SRC_PORT_MPUI	= 5 << 2, /* OMAP1 only */
+	CSDP_SRC_PACKED		= BIT(6),
+	CSDP_SRC_BURST_1	= 0 << 7,
+	CSDP_SRC_BURST_16	= 1 << 7,
+	CSDP_SRC_BURST_32	= 2 << 7,
+	CSDP_SRC_BURST_64	= 3 << 7,
+	CSDP_DST_PORT_EMIFF	= 0 << 9, /* OMAP1 only */
+	CSDP_DST_PORT_EMIFS	= 1 << 9, /* OMAP1 only */
+	CSDP_DST_PORT_OCP_T1	= 2 << 9, /* OMAP1 only */
+	CSDP_DST_PORT_TIPB	= 3 << 9, /* OMAP1 only */
+	CSDP_DST_PORT_OCP_T2	= 4 << 9, /* OMAP1 only */
+	CSDP_DST_PORT_MPUI	= 5 << 9, /* OMAP1 only */
+	CSDP_DST_PACKED		= BIT(13),
+	CSDP_DST_BURST_1	= 0 << 14,
+	CSDP_DST_BURST_16	= 1 << 14,
+	CSDP_DST_BURST_32	= 2 << 14,
+	CSDP_DST_BURST_64	= 3 << 14,
+	CSDP_WRITE_NON_POSTED	= 0 << 16,
+	CSDP_WRITE_POSTED	= 1 << 16,
+	CSDP_WRITE_LAST_NON_POSTED = 2 << 16,
+
+	CICR_TOUT_IE		= BIT(0),	/* OMAP1 only */
+	CICR_DROP_IE		= BIT(1),
+	CICR_HALF_IE		= BIT(2),
+	CICR_FRAME_IE		= BIT(3),
+	CICR_LAST_IE		= BIT(4),
+	CICR_BLOCK_IE		= BIT(5),
+	CICR_PKT_IE		= BIT(7),	/* OMAP2+ only */
+	CICR_TRANS_ERR_IE	= BIT(8),	/* OMAP2+ only */
+	CICR_SUPERVISOR_ERR_IE	= BIT(10),	/* OMAP2+ only */
+	CICR_MISALIGNED_ERR_IE	= BIT(11),	/* OMAP2+ only */
+	CICR_DRAIN_IE		= BIT(12),	/* OMAP2+ only */
+	CICR_SUPER_BLOCK_IE	= BIT(14),	/* OMAP2+ only */
+
+	CLNK_CTRL_ENABLE_LNK	= BIT(15),
+
+	CDP_DST_VALID_INC	= 0 << 0,
+	CDP_DST_VALID_RELOAD	= 1 << 0,
+	CDP_DST_VALID_REUSE	= 2 << 0,
+	CDP_SRC_VALID_INC	= 0 << 2,
+	CDP_SRC_VALID_RELOAD	= 1 << 2,
+	CDP_SRC_VALID_REUSE	= 2 << 2,
+	CDP_NTYPE_TYPE1		= 1 << 4,
+	CDP_NTYPE_TYPE2		= 2 << 4,
+	CDP_NTYPE_TYPE3		= 3 << 4,
+	CDP_TMODE_NORMAL	= 0 << 8,
+	CDP_TMODE_LLIST		= 1 << 8,
+	CDP_FAST		= BIT(10),
+};
+
+static const unsigned es_bytes[] = {
+	[CSDP_DATA_TYPE_8] = 1,
+	[CSDP_DATA_TYPE_16] = 2,
+	[CSDP_DATA_TYPE_32] = 4,
+};
+
+static struct of_dma_filter_info omap_dma_info = {
+	.filter_fn = omap_dma_filter_fn,
+};
+
+static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
+{
+	return container_of(d, struct omap_dmadev, ddev);
+}
+
+static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
+{
+	return container_of(c, struct omap_chan, vc.chan);
+}
+
+static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
+{
+	return container_of(t, struct omap_desc, vd.tx);
+}
+
+static void omap_dma_desc_free(struct virt_dma_desc *vd)
+{
+	struct omap_desc *d = to_omap_dma_desc(&vd->tx);
+
+	if (d->using_ll) {
+		struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
+		int i;
+
+		for (i = 0; i < d->sglen; i++) {
+			if (d->sg[i].t2_desc)
+				dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
+					      d->sg[i].t2_desc_paddr);
+		}
+	}
+
+	kfree(d);
+}
+
+static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
+				     enum dma_transfer_direction dir, bool last)
+{
+	struct omap_sg *sg = &d->sg[idx];
+	struct omap_type2_desc *t2_desc = sg->t2_desc;
+
+	if (idx)
+		d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
+	if (last)
+		t2_desc->next_desc = 0xfffffffc;
+
+	t2_desc->en = sg->en;
+	t2_desc->addr = sg->addr;
+	t2_desc->fn = sg->fn & 0xffff;
+	t2_desc->cicr = d->cicr;
+	if (!last)
+		t2_desc->cicr &= ~CICR_BLOCK_IE;
+
+	switch (dir) {
+	case DMA_DEV_TO_MEM:
+		t2_desc->cdei = sg->ei;
+		t2_desc->csei = d->ei;
+		t2_desc->cdfi = sg->fi;
+		t2_desc->csfi = d->fi;
+
+		t2_desc->en |= DESC_NXT_DV_REFRESH;
+		t2_desc->en |= DESC_NXT_SV_REUSE;
+		break;
+	case DMA_MEM_TO_DEV:
+		t2_desc->cdei = d->ei;
+		t2_desc->csei = sg->ei;
+		t2_desc->cdfi = d->fi;
+		t2_desc->csfi = sg->fi;
+
+		t2_desc->en |= DESC_NXT_SV_REFRESH;
+		t2_desc->en |= DESC_NXT_DV_REUSE;
+		break;
+	default:
+		return;
+	}
+
+	t2_desc->en |= DESC_NTYPE_TYPE2;
+}
+
+static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
+{
+	switch (type) {
+	case OMAP_DMA_REG_16BIT:
+		writew_relaxed(val, addr);
+		break;
+	case OMAP_DMA_REG_2X16BIT:
+		writew_relaxed(val, addr);
+		writew_relaxed(val >> 16, addr + 2);
+		break;
+	case OMAP_DMA_REG_32BIT:
+		writel_relaxed(val, addr);
+		break;
+	default:
+		WARN_ON(1);
+	}
+}
+
+static unsigned omap_dma_read(unsigned type, void __iomem *addr)
+{
+	unsigned val;
+
+	switch (type) {
+	case OMAP_DMA_REG_16BIT:
+		val = readw_relaxed(addr);
+		break;
+	case OMAP_DMA_REG_2X16BIT:
+		val = readw_relaxed(addr);
+		val |= readw_relaxed(addr + 2) << 16;
+		break;
+	case OMAP_DMA_REG_32BIT:
+		val = readl_relaxed(addr);
+		break;
+	default:
+		WARN_ON(1);
+		val = 0;
+	}
+
+	return val;
+}
+
+static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
+{
+	const struct omap_dma_reg *r = od->reg_map + reg;
+
+	WARN_ON(r->stride);
+
+	omap_dma_write(val, r->type, od->base + r->offset);
+}
+
+static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
+{
+	const struct omap_dma_reg *r = od->reg_map + reg;
+
+	WARN_ON(r->stride);
+
+	return omap_dma_read(r->type, od->base + r->offset);
+}
+
+static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
+{
+	const struct omap_dma_reg *r = c->reg_map + reg;
+
+	omap_dma_write(val, r->type, c->channel_base + r->offset);
+}
+
+static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
+{
+	const struct omap_dma_reg *r = c->reg_map + reg;
+
+	return omap_dma_read(r->type, c->channel_base + r->offset);
+}
+
+static void omap_dma_clear_csr(struct omap_chan *c)
+{
+	if (dma_omap1())
+		omap_dma_chan_read(c, CSR);
+	else
+		omap_dma_chan_write(c, CSR, ~0);
+}
+
+static unsigned omap_dma_get_csr(struct omap_chan *c)
+{
+	unsigned val = omap_dma_chan_read(c, CSR);
+
+	if (!dma_omap1())
+		omap_dma_chan_write(c, CSR, val);
+
+	return val;
+}
+
+static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
+	unsigned lch)
+{
+	c->channel_base = od->base + od->plat->channel_stride * lch;
+
+	od->lch_map[lch] = c;
+}
+
+static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
+{
+	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
+	uint16_t cicr = d->cicr;
+
+	if (__dma_omap15xx(od->plat->dma_attr))
+		omap_dma_chan_write(c, CPC, 0);
+	else
+		omap_dma_chan_write(c, CDAC, 0);
+
+	omap_dma_clear_csr(c);
+
+	if (d->using_ll) {
+		uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
+
+		if (d->dir == DMA_DEV_TO_MEM)
+			cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
+		else
+			cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
+		omap_dma_chan_write(c, CDP, cdp);
+
+		omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
+		omap_dma_chan_write(c, CCDN, 0);
+		omap_dma_chan_write(c, CCFN, 0xffff);
+		omap_dma_chan_write(c, CCEN, 0xffffff);
+
+		cicr &= ~CICR_BLOCK_IE;
+	} else if (od->ll123_supported) {
+		omap_dma_chan_write(c, CDP, 0);
+	}
+
+	/* Enable interrupts */
+	omap_dma_chan_write(c, CICR, cicr);
+
+	/* Enable channel */
+	omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
+
+	c->running = true;
+}
+
+static void omap_dma_drain_chan(struct omap_chan *c)
+{
+	int i;
+	u32 val;
+
+	/* Wait for sDMA FIFO to drain */
+	for (i = 0; ; i++) {
+		val = omap_dma_chan_read(c, CCR);
+		if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
+			break;
+
+		if (i > 100)
+			break;
+
+		udelay(5);
+	}
+
+	if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
+		dev_err(c->vc.chan.device->dev,
+			"DMA drain did not complete on lch %d\n",
+			c->dma_ch);
+}
+
+static int omap_dma_stop(struct omap_chan *c)
+{
+	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
+	uint32_t val;
+
+	/* disable irq */
+	omap_dma_chan_write(c, CICR, 0);
+
+	omap_dma_clear_csr(c);
+
+	val = omap_dma_chan_read(c, CCR);
+	if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
+		uint32_t sysconfig;
+
+		sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
+		val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
+		val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
+		omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
+
+		val = omap_dma_chan_read(c, CCR);
+		val &= ~CCR_ENABLE;
+		omap_dma_chan_write(c, CCR, val);
+
+		if (!(c->ccr & CCR_BUFFERING_DISABLE))
+			omap_dma_drain_chan(c);
+
+		omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
+	} else {
+		if (!(val & CCR_ENABLE))
+			return -EINVAL;
+
+		val &= ~CCR_ENABLE;
+		omap_dma_chan_write(c, CCR, val);
+
+		if (!(c->ccr & CCR_BUFFERING_DISABLE))
+			omap_dma_drain_chan(c);
+	}
+
+	mb();
+
+	if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
+		val = omap_dma_chan_read(c, CLNK_CTRL);
+
+		if (dma_omap1())
+			val |= 1 << 14; /* set the STOP_LNK bit */
+		else
+			val &= ~CLNK_CTRL_ENABLE_LNK;
+
+		omap_dma_chan_write(c, CLNK_CTRL, val);
+	}
+	c->running = false;
+	return 0;
+}
+
+static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
+{
+	struct omap_sg *sg = d->sg + c->sgidx;
+	unsigned cxsa, cxei, cxfi;
+
+	if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
+		cxsa = CDSA;
+		cxei = CDEI;
+		cxfi = CDFI;
+	} else {
+		cxsa = CSSA;
+		cxei = CSEI;
+		cxfi = CSFI;
+	}
+
+	omap_dma_chan_write(c, cxsa, sg->addr);
+	omap_dma_chan_write(c, cxei, sg->ei);
+	omap_dma_chan_write(c, cxfi, sg->fi);
+	omap_dma_chan_write(c, CEN, sg->en);
+	omap_dma_chan_write(c, CFN, sg->fn);
+
+	omap_dma_start(c, d);
+	c->sgidx++;
+}
+
+static void omap_dma_start_desc(struct omap_chan *c)
+{
+	struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
+	struct omap_desc *d;
+	unsigned cxsa, cxei, cxfi;
+
+	if (!vd) {
+		c->desc = NULL;
+		return;
+	}
+
+	list_del(&vd->node);
+
+	c->desc = d = to_omap_dma_desc(&vd->tx);
+	c->sgidx = 0;
+
+	/*
+	 * This provides the necessary barrier to ensure data held in
+	 * DMA coherent memory is visible to the DMA engine prior to
+	 * the transfer starting.
+	 */
+	mb();
+
+	omap_dma_chan_write(c, CCR, d->ccr);
+	if (dma_omap1())
+		omap_dma_chan_write(c, CCR2, d->ccr >> 16);
+
+	if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
+		cxsa = CSSA;
+		cxei = CSEI;
+		cxfi = CSFI;
+	} else {
+		cxsa = CDSA;
+		cxei = CDEI;
+		cxfi = CDFI;
+	}
+
+	omap_dma_chan_write(c, cxsa, d->dev_addr);
+	omap_dma_chan_write(c, cxei, d->ei);
+	omap_dma_chan_write(c, cxfi, d->fi);
+	omap_dma_chan_write(c, CSDP, d->csdp);
+	omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
+
+	omap_dma_start_sg(c, d);
+}
+
+static void omap_dma_callback(int ch, u16 status, void *data)
+{
+	struct omap_chan *c = data;
+	struct omap_desc *d;
+	unsigned long flags;
+
+	spin_lock_irqsave(&c->vc.lock, flags);
+	d = c->desc;
+	if (d) {
+		if (c->cyclic) {
+			vchan_cyclic_callback(&d->vd);
+		} else if (d->using_ll || c->sgidx == d->sglen) {
+			omap_dma_start_desc(c);
+			vchan_cookie_complete(&d->vd);
+		} else {
+			omap_dma_start_sg(c, d);
+		}
+	}
+	spin_unlock_irqrestore(&c->vc.lock, flags);
+}
+
+static irqreturn_t omap_dma_irq(int irq, void *devid)
+{
+	struct omap_dmadev *od = devid;
+	unsigned status, channel;
+
+	spin_lock(&od->irq_lock);
+
+	status = omap_dma_glbl_read(od, IRQSTATUS_L1);
+	status &= od->irq_enable_mask;
+	if (status == 0) {
+		spin_unlock(&od->irq_lock);
+		return IRQ_NONE;
+	}
+
+	while ((channel = ffs(status)) != 0) {
+		unsigned mask, csr;
+		struct omap_chan *c;
+
+		channel -= 1;
+		mask = BIT(channel);
+		status &= ~mask;
+
+		c = od->lch_map[channel];
+		if (c == NULL) {
+			/* This should never happen */
+			dev_err(od->ddev.dev, "invalid channel %u\n", channel);
+			continue;
+		}
+
+		csr = omap_dma_get_csr(c);
+		omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
+
+		omap_dma_callback(channel, csr, c);
+	}
+
+	spin_unlock(&od->irq_lock);
+
+	return IRQ_HANDLED;
+}
+
+static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
+{
+	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
+	struct omap_chan *c = to_omap_dma_chan(chan);
+	struct device *dev = od->ddev.dev;
+	int ret;
+
+	if (od->legacy) {
+		ret = omap_request_dma(c->dma_sig, "DMA engine",
+				       omap_dma_callback, c, &c->dma_ch);
+	} else {
+		ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,
+				       &c->dma_ch);
+	}
+
+	dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
+
+	if (ret >= 0) {
+		omap_dma_assign(od, c, c->dma_ch);
+
+		if (!od->legacy) {
+			unsigned val;
+
+			spin_lock_irq(&od->irq_lock);
+			val = BIT(c->dma_ch);
+			omap_dma_glbl_write(od, IRQSTATUS_L1, val);
+			od->irq_enable_mask |= val;
+			omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
+
+			val = omap_dma_glbl_read(od, IRQENABLE_L0);
+			val &= ~BIT(c->dma_ch);
+			omap_dma_glbl_write(od, IRQENABLE_L0, val);
+			spin_unlock_irq(&od->irq_lock);
+		}
+	}
+
+	if (dma_omap1()) {
+		if (__dma_omap16xx(od->plat->dma_attr)) {
+			c->ccr = CCR_OMAP31_DISABLE;
+			/* Duplicate what plat-omap/dma.c does */
+			c->ccr |= c->dma_ch + 1;
+		} else {
+			c->ccr = c->dma_sig & 0x1f;
+		}
+	} else {
+		c->ccr = c->dma_sig & 0x1f;
+		c->ccr |= (c->dma_sig & ~0x1f) << 14;
+	}
+	if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
+		c->ccr |= CCR_BUFFERING_DISABLE;
+
+	return ret;
+}
+
+static void omap_dma_free_chan_resources(struct dma_chan *chan)
+{
+	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
+	struct omap_chan *c = to_omap_dma_chan(chan);
+
+	if (!od->legacy) {
+		spin_lock_irq(&od->irq_lock);
+		od->irq_enable_mask &= ~BIT(c->dma_ch);
+		omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
+		spin_unlock_irq(&od->irq_lock);
+	}
+
+	c->channel_base = NULL;
+	od->lch_map[c->dma_ch] = NULL;
+	vchan_free_chan_resources(&c->vc);
+	omap_free_dma(c->dma_ch);
+
+	dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
+		c->dma_sig);
+	c->dma_sig = 0;
+}
+
+static size_t omap_dma_sg_size(struct omap_sg *sg)
+{
+	return sg->en * sg->fn;
+}
+
+static size_t omap_dma_desc_size(struct omap_desc *d)
+{
+	unsigned i;
+	size_t size;
+
+	for (size = i = 0; i < d->sglen; i++)
+		size += omap_dma_sg_size(&d->sg[i]);
+
+	return size * es_bytes[d->es];
+}
+
+static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
+{
+	unsigned i;
+	size_t size, es_size = es_bytes[d->es];
+
+	for (size = i = 0; i < d->sglen; i++) {
+		size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
+
+		if (size)
+			size += this_size;
+		else if (addr >= d->sg[i].addr &&
+			 addr < d->sg[i].addr + this_size)
+			size += d->sg[i].addr + this_size - addr;
+	}
+	return size;
+}
+
+/*
+ * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
+ * read before the DMA controller finished disabling the channel.
+ */
+static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
+{
+	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
+	uint32_t val;
+
+	val = omap_dma_chan_read(c, reg);
+	if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
+		val = omap_dma_chan_read(c, reg);
+
+	return val;
+}
+
+static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
+{
+	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
+	dma_addr_t addr, cdac;
+
+	if (__dma_omap15xx(od->plat->dma_attr)) {
+		addr = omap_dma_chan_read(c, CPC);
+	} else {
+		addr = omap_dma_chan_read_3_3(c, CSAC);
+		cdac = omap_dma_chan_read_3_3(c, CDAC);
+
+		/*
+		 * CDAC == 0 indicates that the DMA transfer on the channel has
+		 * not been started (no data has been transferred so far).
+		 * Return the programmed source start address in this case.
+		 */
+		if (cdac == 0)
+			addr = omap_dma_chan_read(c, CSSA);
+	}
+
+	if (dma_omap1())
+		addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
+
+	return addr;
+}
+
+static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
+{
+	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
+	dma_addr_t addr;
+
+	if (__dma_omap15xx(od->plat->dma_attr)) {
+		addr = omap_dma_chan_read(c, CPC);
+	} else {
+		addr = omap_dma_chan_read_3_3(c, CDAC);
+
+		/*
+		 * CDAC == 0 indicates that the DMA transfer on the channel
+		 * has not been started (no data has been transferred so
+		 * far).  Return the programmed destination start address in
+		 * this case.
+		 */
+		if (addr == 0)
+			addr = omap_dma_chan_read(c, CDSA);
+	}
+
+	if (dma_omap1())
+		addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
+
+	return addr;
+}
+
+static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
+	dma_cookie_t cookie, struct dma_tx_state *txstate)
+{
+	struct omap_chan *c = to_omap_dma_chan(chan);
+	struct virt_dma_desc *vd;
+	enum dma_status ret;
+	unsigned long flags;
+
+	ret = dma_cookie_status(chan, cookie, txstate);
+
+	if (!c->paused && c->running) {
+		uint32_t ccr = omap_dma_chan_read(c, CCR);
+		/*
+		 * The channel is no longer active, set the return value
+		 * accordingly
+		 */
+		if (!(ccr & CCR_ENABLE))
+			ret = DMA_COMPLETE;
+	}
+
+	if (ret == DMA_COMPLETE || !txstate)
+		return ret;
+
+	spin_lock_irqsave(&c->vc.lock, flags);
+	vd = vchan_find_desc(&c->vc, cookie);
+	if (vd) {
+		txstate->residue = omap_dma_desc_size(to_omap_dma_desc(&vd->tx));
+	} else if (c->desc && c->desc->vd.tx.cookie == cookie) {
+		struct omap_desc *d = c->desc;
+		dma_addr_t pos;
+
+		if (d->dir == DMA_MEM_TO_DEV)
+			pos = omap_dma_get_src_pos(c);
+		else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
+			pos = omap_dma_get_dst_pos(c);
+		else
+			pos = 0;
+
+		txstate->residue = omap_dma_desc_size_pos(d, pos);
+	} else {
+		txstate->residue = 0;
+	}
+	if (ret == DMA_IN_PROGRESS && c->paused)
+		ret = DMA_PAUSED;
+	spin_unlock_irqrestore(&c->vc.lock, flags);
+
+	return ret;
+}
+
+static void omap_dma_issue_pending(struct dma_chan *chan)
+{
+	struct omap_chan *c = to_omap_dma_chan(chan);
+	unsigned long flags;
+
+	spin_lock_irqsave(&c->vc.lock, flags);
+	if (vchan_issue_pending(&c->vc) && !c->desc)
+		omap_dma_start_desc(c);
+	spin_unlock_irqrestore(&c->vc.lock, flags);
+}
+
+static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
+	struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
+	enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
+{
+	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
+	struct omap_chan *c = to_omap_dma_chan(chan);
+	enum dma_slave_buswidth dev_width;
+	struct scatterlist *sgent;
+	struct omap_desc *d;
+	dma_addr_t dev_addr;
+	unsigned i, es, en, frame_bytes;
+	bool ll_failed = false;
+	u32 burst;
+	u32 port_window, port_window_bytes;
+
+	if (dir == DMA_DEV_TO_MEM) {
+		dev_addr = c->cfg.src_addr;
+		dev_width = c->cfg.src_addr_width;
+		burst = c->cfg.src_maxburst;
+		port_window = c->cfg.src_port_window_size;
+	} else if (dir == DMA_MEM_TO_DEV) {
+		dev_addr = c->cfg.dst_addr;
+		dev_width = c->cfg.dst_addr_width;
+		burst = c->cfg.dst_maxburst;
+		port_window = c->cfg.dst_port_window_size;
+	} else {
+		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
+		return NULL;
+	}
+
+	/* Bus width translates to the element size (ES) */
+	switch (dev_width) {
+	case DMA_SLAVE_BUSWIDTH_1_BYTE:
+		es = CSDP_DATA_TYPE_8;
+		break;
+	case DMA_SLAVE_BUSWIDTH_2_BYTES:
+		es = CSDP_DATA_TYPE_16;
+		break;
+	case DMA_SLAVE_BUSWIDTH_4_BYTES:
+		es = CSDP_DATA_TYPE_32;
+		break;
+	default: /* not reached */
+		return NULL;
+	}
+
+	/* Now allocate and setup the descriptor. */
+	d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC);
+	if (!d)
+		return NULL;
+
+	d->dir = dir;
+	d->dev_addr = dev_addr;
+	d->es = es;
+
+	/* When the port_window is used, one frame must cover the window */
+	if (port_window) {
+		burst = port_window;
+		port_window_bytes = port_window * es_bytes[es];
+
+		d->ei = 1;
+		/*
+		 * One frame covers the port_window and by  configure
+		 * the source frame index to be -1 * (port_window - 1)
+		 * we instruct the sDMA that after a frame is processed
+		 * it should move back to the start of the window.
+		 */
+		d->fi = -(port_window_bytes - 1);
+	}
+
+	d->ccr = c->ccr | CCR_SYNC_FRAME;
+	if (dir == DMA_DEV_TO_MEM) {
+		d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
+
+		d->ccr |= CCR_DST_AMODE_POSTINC;
+		if (port_window) {
+			d->ccr |= CCR_SRC_AMODE_DBLIDX;
+
+			if (port_window_bytes >= 64)
+				d->csdp |= CSDP_SRC_BURST_64;
+			else if (port_window_bytes >= 32)
+				d->csdp |= CSDP_SRC_BURST_32;
+			else if (port_window_bytes >= 16)
+				d->csdp |= CSDP_SRC_BURST_16;
+
+		} else {
+			d->ccr |= CCR_SRC_AMODE_CONSTANT;
+		}
+	} else {
+		d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
+
+		d->ccr |= CCR_SRC_AMODE_POSTINC;
+		if (port_window) {
+			d->ccr |= CCR_DST_AMODE_DBLIDX;
+
+			if (port_window_bytes >= 64)
+				d->csdp |= CSDP_DST_BURST_64;
+			else if (port_window_bytes >= 32)
+				d->csdp |= CSDP_DST_BURST_32;
+			else if (port_window_bytes >= 16)
+				d->csdp |= CSDP_DST_BURST_16;
+		} else {
+			d->ccr |= CCR_DST_AMODE_CONSTANT;
+		}
+	}
+
+	d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
+	d->csdp |= es;
+
+	if (dma_omap1()) {
+		d->cicr |= CICR_TOUT_IE;
+
+		if (dir == DMA_DEV_TO_MEM)
+			d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
+		else
+			d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
+	} else {
+		if (dir == DMA_DEV_TO_MEM)
+			d->ccr |= CCR_TRIGGER_SRC;
+
+		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
+
+		if (port_window)
+			d->csdp |= CSDP_WRITE_LAST_NON_POSTED;
+	}
+	if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
+		d->clnk_ctrl = c->dma_ch;
+
+	/*
+	 * Build our scatterlist entries: each contains the address,
+	 * the number of elements (EN) in each frame, and the number of
+	 * frames (FN).  Number of bytes for this entry = ES * EN * FN.
+	 *
+	 * Burst size translates to number of elements with frame sync.
+	 * Note: DMA engine defines burst to be the number of dev-width
+	 * transfers.
+	 */
+	en = burst;
+	frame_bytes = es_bytes[es] * en;
+
+	if (sglen >= 2)
+		d->using_ll = od->ll123_supported;
+
+	for_each_sg(sgl, sgent, sglen, i) {
+		struct omap_sg *osg = &d->sg[i];
+
+		osg->addr = sg_dma_address(sgent);
+		osg->en = en;
+		osg->fn = sg_dma_len(sgent) / frame_bytes;
+
+		if (d->using_ll) {
+			osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
+						      &osg->t2_desc_paddr);
+			if (!osg->t2_desc) {
+				dev_err(chan->device->dev,
+					"t2_desc[%d] allocation failed\n", i);
+				ll_failed = true;
+				d->using_ll = false;
+				continue;
+			}
+
+			omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
+		}
+	}
+
+	d->sglen = sglen;
+
+	/* Release the dma_pool entries if one allocation failed */
+	if (ll_failed) {
+		for (i = 0; i < d->sglen; i++) {
+			struct omap_sg *osg = &d->sg[i];
+
+			if (osg->t2_desc) {
+				dma_pool_free(od->desc_pool, osg->t2_desc,
+					      osg->t2_desc_paddr);
+				osg->t2_desc = NULL;
+			}
+		}
+	}
+
+	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
+}
+
+static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
+	struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
+	size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
+{
+	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
+	struct omap_chan *c = to_omap_dma_chan(chan);
+	enum dma_slave_buswidth dev_width;
+	struct omap_desc *d;
+	dma_addr_t dev_addr;
+	unsigned es;
+	u32 burst;
+
+	if (dir == DMA_DEV_TO_MEM) {
+		dev_addr = c->cfg.src_addr;
+		dev_width = c->cfg.src_addr_width;
+		burst = c->cfg.src_maxburst;
+	} else if (dir == DMA_MEM_TO_DEV) {
+		dev_addr = c->cfg.dst_addr;
+		dev_width = c->cfg.dst_addr_width;
+		burst = c->cfg.dst_maxburst;
+	} else {
+		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
+		return NULL;
+	}
+
+	/* Bus width translates to the element size (ES) */
+	switch (dev_width) {
+	case DMA_SLAVE_BUSWIDTH_1_BYTE:
+		es = CSDP_DATA_TYPE_8;
+		break;
+	case DMA_SLAVE_BUSWIDTH_2_BYTES:
+		es = CSDP_DATA_TYPE_16;
+		break;
+	case DMA_SLAVE_BUSWIDTH_4_BYTES:
+		es = CSDP_DATA_TYPE_32;
+		break;
+	default: /* not reached */
+		return NULL;
+	}
+
+	/* Now allocate and setup the descriptor. */
+	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
+	if (!d)
+		return NULL;
+
+	d->dir = dir;
+	d->dev_addr = dev_addr;
+	d->fi = burst;
+	d->es = es;
+	d->sg[0].addr = buf_addr;
+	d->sg[0].en = period_len / es_bytes[es];
+	d->sg[0].fn = buf_len / period_len;
+	d->sglen = 1;
+
+	d->ccr = c->ccr;
+	if (dir == DMA_DEV_TO_MEM)
+		d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
+	else
+		d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
+
+	d->cicr = CICR_DROP_IE;
+	if (flags & DMA_PREP_INTERRUPT)
+		d->cicr |= CICR_FRAME_IE;
+
+	d->csdp = es;
+
+	if (dma_omap1()) {
+		d->cicr |= CICR_TOUT_IE;
+
+		if (dir == DMA_DEV_TO_MEM)
+			d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
+		else
+			d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
+	} else {
+		if (burst)
+			d->ccr |= CCR_SYNC_PACKET;
+		else
+			d->ccr |= CCR_SYNC_ELEMENT;
+
+		if (dir == DMA_DEV_TO_MEM) {
+			d->ccr |= CCR_TRIGGER_SRC;
+			d->csdp |= CSDP_DST_PACKED;
+		} else {
+			d->csdp |= CSDP_SRC_PACKED;
+		}
+
+		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
+
+		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
+	}
+
+	if (__dma_omap15xx(od->plat->dma_attr))
+		d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
+	else
+		d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
+
+	c->cyclic = true;
+
+	return vchan_tx_prep(&c->vc, &d->vd, flags);
+}
+
+static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
+	struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
+	size_t len, unsigned long tx_flags)
+{
+	struct omap_chan *c = to_omap_dma_chan(chan);
+	struct omap_desc *d;
+	uint8_t data_type;
+
+	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
+	if (!d)
+		return NULL;
+
+	data_type = __ffs((src | dest | len));
+	if (data_type > CSDP_DATA_TYPE_32)
+		data_type = CSDP_DATA_TYPE_32;
+
+	d->dir = DMA_MEM_TO_MEM;
+	d->dev_addr = src;
+	d->fi = 0;
+	d->es = data_type;
+	d->sg[0].en = len / BIT(data_type);
+	d->sg[0].fn = 1;
+	d->sg[0].addr = dest;
+	d->sglen = 1;
+	d->ccr = c->ccr;
+	d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
+
+	d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
+
+	d->csdp = data_type;
+
+	if (dma_omap1()) {
+		d->cicr |= CICR_TOUT_IE;
+		d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
+	} else {
+		d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
+		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
+		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
+	}
+
+	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
+}
+
+static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
+	struct dma_chan *chan, struct dma_interleaved_template *xt,
+	unsigned long flags)
+{
+	struct omap_chan *c = to_omap_dma_chan(chan);
+	struct omap_desc *d;
+	struct omap_sg *sg;
+	uint8_t data_type;
+	size_t src_icg, dst_icg;
+
+	/* Slave mode is not supported */
+	if (is_slave_direction(xt->dir))
+		return NULL;
+
+	if (xt->frame_size != 1 || xt->numf == 0)
+		return NULL;
+
+	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
+	if (!d)
+		return NULL;
+
+	data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
+	if (data_type > CSDP_DATA_TYPE_32)
+		data_type = CSDP_DATA_TYPE_32;
+
+	sg = &d->sg[0];
+	d->dir = DMA_MEM_TO_MEM;
+	d->dev_addr = xt->src_start;
+	d->es = data_type;
+	sg->en = xt->sgl[0].size / BIT(data_type);
+	sg->fn = xt->numf;
+	sg->addr = xt->dst_start;
+	d->sglen = 1;
+	d->ccr = c->ccr;
+
+	src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
+	dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
+	if (src_icg) {
+		d->ccr |= CCR_SRC_AMODE_DBLIDX;
+		d->ei = 1;
+		d->fi = src_icg;
+	} else if (xt->src_inc) {
+		d->ccr |= CCR_SRC_AMODE_POSTINC;
+		d->fi = 0;
+	} else {
+		dev_err(chan->device->dev,
+			"%s: SRC constant addressing is not supported\n",
+			__func__);
+		kfree(d);
+		return NULL;
+	}
+
+	if (dst_icg) {
+		d->ccr |= CCR_DST_AMODE_DBLIDX;
+		sg->ei = 1;
+		sg->fi = dst_icg;
+	} else if (xt->dst_inc) {
+		d->ccr |= CCR_DST_AMODE_POSTINC;
+		sg->fi = 0;
+	} else {
+		dev_err(chan->device->dev,
+			"%s: DST constant addressing is not supported\n",
+			__func__);
+		kfree(d);
+		return NULL;
+	}
+
+	d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
+
+	d->csdp = data_type;
+
+	if (dma_omap1()) {
+		d->cicr |= CICR_TOUT_IE;
+		d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
+	} else {
+		d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
+		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
+		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
+	}
+
+	return vchan_tx_prep(&c->vc, &d->vd, flags);
+}
+
+static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
+{
+	struct omap_chan *c = to_omap_dma_chan(chan);
+
+	if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
+	    cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
+		return -EINVAL;
+
+	if (cfg->src_maxburst > chan->device->max_burst ||
+	    cfg->dst_maxburst > chan->device->max_burst)
+		return -EINVAL;
+
+	memcpy(&c->cfg, cfg, sizeof(c->cfg));
+
+	return 0;
+}
+
+static int omap_dma_terminate_all(struct dma_chan *chan)
+{
+	struct omap_chan *c = to_omap_dma_chan(chan);
+	unsigned long flags;
+	LIST_HEAD(head);
+
+	spin_lock_irqsave(&c->vc.lock, flags);
+
+	/*
+	 * Stop DMA activity: we assume the callback will not be called
+	 * after omap_dma_stop() returns (even if it does, it will see
+	 * c->desc is NULL and exit.)
+	 */
+	if (c->desc) {
+		vchan_terminate_vdesc(&c->desc->vd);
+		c->desc = NULL;
+		/* Avoid stopping the dma twice */
+		if (!c->paused)
+			omap_dma_stop(c);
+	}
+
+	c->cyclic = false;
+	c->paused = false;
+
+	vchan_get_all_descriptors(&c->vc, &head);
+	spin_unlock_irqrestore(&c->vc.lock, flags);
+	vchan_dma_desc_free_list(&c->vc, &head);
+
+	return 0;
+}
+
+static void omap_dma_synchronize(struct dma_chan *chan)
+{
+	struct omap_chan *c = to_omap_dma_chan(chan);
+
+	vchan_synchronize(&c->vc);
+}
+
+static int omap_dma_pause(struct dma_chan *chan)
+{
+	struct omap_chan *c = to_omap_dma_chan(chan);
+	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
+	unsigned long flags;
+	int ret = -EINVAL;
+	bool can_pause = false;
+
+	spin_lock_irqsave(&od->irq_lock, flags);
+
+	if (!c->desc)
+		goto out;
+
+	if (c->cyclic)
+		can_pause = true;
+
+	/*
+	 * We do not allow DMA_MEM_TO_DEV transfers to be paused.
+	 * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer:
+	 * "When a channel is disabled during a transfer, the channel undergoes
+	 * an abort, unless it is hardware-source-synchronized …".
+	 * A source-synchronised channel is one where the fetching of data is
+	 * under control of the device. In other words, a device-to-memory
+	 * transfer. So, a destination-synchronised channel (which would be a
+	 * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE
+	 * bit is cleared.
+	 * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel
+	 * aborts immediately after completion of current read/write
+	 * transactions and then the FIFO is cleaned up." The term "cleaned up"
+	 * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE
+	 * are both clear _before_ disabling the channel, otherwise data loss
+	 * will occur.
+	 * The problem is that if the channel is active, then device activity
+	 * can result in DMA activity starting between reading those as both
+	 * clear and the write to DMA_CCR to clear the enable bit hitting the
+	 * hardware. If the DMA hardware can't drain the data in its FIFO to the
+	 * destination, then data loss "might" occur (say if we write to an UART
+	 * and the UART is not accepting any further data).
+	 */
+	else if (c->desc->dir == DMA_DEV_TO_MEM)
+		can_pause = true;
+
+	if (can_pause && !c->paused) {
+		ret = omap_dma_stop(c);
+		if (!ret)
+			c->paused = true;
+	}
+out:
+	spin_unlock_irqrestore(&od->irq_lock, flags);
+
+	return ret;
+}
+
+static int omap_dma_resume(struct dma_chan *chan)
+{
+	struct omap_chan *c = to_omap_dma_chan(chan);
+	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
+	unsigned long flags;
+	int ret = -EINVAL;
+
+	spin_lock_irqsave(&od->irq_lock, flags);
+
+	if (c->paused && c->desc) {
+		mb();
+
+		/* Restore channel link register */
+		omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
+
+		omap_dma_start(c, c->desc);
+		c->paused = false;
+		ret = 0;
+	}
+	spin_unlock_irqrestore(&od->irq_lock, flags);
+
+	return ret;
+}
+
+static int omap_dma_chan_init(struct omap_dmadev *od)
+{
+	struct omap_chan *c;
+
+	c = kzalloc(sizeof(*c), GFP_KERNEL);
+	if (!c)
+		return -ENOMEM;
+
+	c->reg_map = od->reg_map;
+	c->vc.desc_free = omap_dma_desc_free;
+	vchan_init(&c->vc, &od->ddev);
+
+	return 0;
+}
+
+static void omap_dma_free(struct omap_dmadev *od)
+{
+	while (!list_empty(&od->ddev.channels)) {
+		struct omap_chan *c = list_first_entry(&od->ddev.channels,
+			struct omap_chan, vc.chan.device_node);
+
+		list_del(&c->vc.chan.device_node);
+		tasklet_kill(&c->vc.task);
+		kfree(c);
+	}
+}
+
+#define OMAP_DMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
+
+static int omap_dma_probe(struct platform_device *pdev)
+{
+	struct omap_dmadev *od;
+	struct resource *res;
+	int rc, i, irq;
+	u32 lch_count;
+
+	od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
+	if (!od)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	od->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(od->base))
+		return PTR_ERR(od->base);
+
+	od->plat = omap_get_plat_info();
+	if (!od->plat)
+		return -EPROBE_DEFER;
+
+	od->reg_map = od->plat->reg_map;
+
+	dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
+	dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
+	dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
+	dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
+	od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
+	od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
+	od->ddev.device_tx_status = omap_dma_tx_status;
+	od->ddev.device_issue_pending = omap_dma_issue_pending;
+	od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
+	od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
+	od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
+	od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
+	od->ddev.device_config = omap_dma_slave_config;
+	od->ddev.device_pause = omap_dma_pause;
+	od->ddev.device_resume = omap_dma_resume;
+	od->ddev.device_terminate_all = omap_dma_terminate_all;
+	od->ddev.device_synchronize = omap_dma_synchronize;
+	od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
+	od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
+	od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+	od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+	od->ddev.max_burst = SZ_16M - 1; /* CCEN: 24bit unsigned */
+	od->ddev.dev = &pdev->dev;
+	INIT_LIST_HEAD(&od->ddev.channels);
+	spin_lock_init(&od->lock);
+	spin_lock_init(&od->irq_lock);
+
+	/* Number of DMA requests */
+	od->dma_requests = OMAP_SDMA_REQUESTS;
+	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
+						      "dma-requests",
+						      &od->dma_requests)) {
+		dev_info(&pdev->dev,
+			 "Missing dma-requests property, using %u.\n",
+			 OMAP_SDMA_REQUESTS);
+	}
+
+	/* Number of available logical channels */
+	if (!pdev->dev.of_node) {
+		lch_count = od->plat->dma_attr->lch_count;
+		if (unlikely(!lch_count))
+			lch_count = OMAP_SDMA_CHANNELS;
+	} else if (of_property_read_u32(pdev->dev.of_node, "dma-channels",
+					&lch_count)) {
+		dev_info(&pdev->dev,
+			 "Missing dma-channels property, using %u.\n",
+			 OMAP_SDMA_CHANNELS);
+		lch_count = OMAP_SDMA_CHANNELS;
+	}
+
+	od->lch_map = devm_kcalloc(&pdev->dev, lch_count, sizeof(*od->lch_map),
+				   GFP_KERNEL);
+	if (!od->lch_map)
+		return -ENOMEM;
+
+	for (i = 0; i < od->dma_requests; i++) {
+		rc = omap_dma_chan_init(od);
+		if (rc) {
+			omap_dma_free(od);
+			return rc;
+		}
+	}
+
+	irq = platform_get_irq(pdev, 1);
+	if (irq <= 0) {
+		dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
+		od->legacy = true;
+	} else {
+		/* Disable all interrupts */
+		od->irq_enable_mask = 0;
+		omap_dma_glbl_write(od, IRQENABLE_L1, 0);
+
+		rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
+				      IRQF_SHARED, "omap-dma-engine", od);
+		if (rc)
+			return rc;
+	}
+
+	if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
+		od->ll123_supported = true;
+
+	od->ddev.filter.map = od->plat->slave_map;
+	od->ddev.filter.mapcnt = od->plat->slavecnt;
+	od->ddev.filter.fn = omap_dma_filter_fn;
+
+	if (od->ll123_supported) {
+		od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
+						&pdev->dev,
+						sizeof(struct omap_type2_desc),
+						4, 0);
+		if (!od->desc_pool) {
+			dev_err(&pdev->dev,
+				"unable to allocate descriptor pool\n");
+			od->ll123_supported = false;
+		}
+	}
+
+	rc = dma_async_device_register(&od->ddev);
+	if (rc) {
+		pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
+			rc);
+		omap_dma_free(od);
+		return rc;
+	}
+
+	platform_set_drvdata(pdev, od);
+
+	if (pdev->dev.of_node) {
+		omap_dma_info.dma_cap = od->ddev.cap_mask;
+
+		/* Device-tree DMA controller registration */
+		rc = of_dma_controller_register(pdev->dev.of_node,
+				of_dma_simple_xlate, &omap_dma_info);
+		if (rc) {
+			pr_warn("OMAP-DMA: failed to register DMA controller\n");
+			dma_async_device_unregister(&od->ddev);
+			omap_dma_free(od);
+		}
+	}
+
+	dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
+		 od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
+
+	return rc;
+}
+
+static int omap_dma_remove(struct platform_device *pdev)
+{
+	struct omap_dmadev *od = platform_get_drvdata(pdev);
+	int irq;
+
+	if (pdev->dev.of_node)
+		of_dma_controller_free(pdev->dev.of_node);
+
+	irq = platform_get_irq(pdev, 1);
+	devm_free_irq(&pdev->dev, irq, od);
+
+	dma_async_device_unregister(&od->ddev);
+
+	if (!od->legacy) {
+		/* Disable all interrupts */
+		omap_dma_glbl_write(od, IRQENABLE_L0, 0);
+	}
+
+	if (od->ll123_supported)
+		dma_pool_destroy(od->desc_pool);
+
+	omap_dma_free(od);
+
+	return 0;
+}
+
+static const struct of_device_id omap_dma_match[] = {
+	{ .compatible = "ti,omap2420-sdma", },
+	{ .compatible = "ti,omap2430-sdma", },
+	{ .compatible = "ti,omap3430-sdma", },
+	{ .compatible = "ti,omap3630-sdma", },
+	{ .compatible = "ti,omap4430-sdma", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, omap_dma_match);
+
+static struct platform_driver omap_dma_driver = {
+	.probe	= omap_dma_probe,
+	.remove	= omap_dma_remove,
+	.driver = {
+		.name = "omap-dma-engine",
+		.of_match_table = of_match_ptr(omap_dma_match),
+	},
+};
+
+bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
+{
+	if (chan->device->dev->driver == &omap_dma_driver.driver) {
+		struct omap_dmadev *od = to_omap_dma_dev(chan->device);
+		struct omap_chan *c = to_omap_dma_chan(chan);
+		unsigned req = *(unsigned *)param;
+
+		if (req <= od->dma_requests) {
+			c->dma_sig = req;
+			return true;
+		}
+	}
+	return false;
+}
+EXPORT_SYMBOL_GPL(omap_dma_filter_fn);
+
+static int omap_dma_init(void)
+{
+	return platform_driver_register(&omap_dma_driver);
+}
+subsys_initcall(omap_dma_init);
+
+static void __exit omap_dma_exit(void)
+{
+	platform_driver_unregister(&omap_dma_driver);
+}
+module_exit(omap_dma_exit);
+
+MODULE_AUTHOR("Russell King");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c
index 4d8c7b9078fd..eb45af71d3a3 100644
--- a/drivers/dma/txx9dmac.c
+++ b/drivers/dma/txx9dmac.c
@@ -1244,8 +1244,7 @@ static void txx9dmac_shutdown(struct platform_device *pdev)
 
 static int txx9dmac_suspend_noirq(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct txx9dmac_dev *ddev = platform_get_drvdata(pdev);
+	struct txx9dmac_dev *ddev = dev_get_drvdata(dev);
 
 	txx9dmac_off(ddev);
 	return 0;
@@ -1253,9 +1252,8 @@ static int txx9dmac_suspend_noirq(struct device *dev)
 
 static int txx9dmac_resume_noirq(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct txx9dmac_dev *ddev = platform_get_drvdata(pdev);
-	struct txx9dmac_platform_data *pdata = dev_get_platdata(&pdev->dev);
+	struct txx9dmac_dev *ddev = dev_get_drvdata(dev);
+	struct txx9dmac_platform_data *pdata = dev_get_platdata(dev);
 	u32 mcr;
 
 	mcr = TXX9_DMA_MCR_MSTEN | MCR_LE;
diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig
index da2da53bca6d..57304b2e989f 100644
--- a/drivers/edac/Kconfig
+++ b/drivers/edac/Kconfig
@@ -232,6 +232,7 @@ config EDAC_SBRIDGE
 config EDAC_SKX
 	tristate "Intel Skylake server Integrated MC"
 	depends on PCI && X86_64 && X86_MCE_INTEL && PCI_MMCONFIG
+	depends on ACPI_NFIT || !ACPI_NFIT # if ACPI_NFIT=m, EDAC_SKX can't be y
 	select DMI
 	help
 	  Support for error detection and correction the Intel
@@ -374,7 +375,7 @@ config EDAC_THUNDERX
 
 config EDAC_ALTERA
 	bool "Altera SOCFPGA ECC"
-	depends on EDAC=y && ARCH_SOCFPGA
+	depends on EDAC=y && (ARCH_SOCFPGA || ARCH_STRATIX10)
 	help
 	  Support for error detection and correction on the
 	  Altera SOCs. This must be selected for SDRAM ECC.
diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c
index 11d6419788c2..d0d5c4dbe097 100644
--- a/drivers/edac/altera_edac.c
+++ b/drivers/edac/altera_edac.c
@@ -1,20 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
+ *  Copyright (C) 2017-2018, Intel Corporation. All rights reserved
  *  Copyright Altera Corporation (C) 2014-2016. All rights reserved.
  *  Copyright 2011-2012 Calxeda, Inc.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * Adapted from the highbank_mc_edac driver.
  */
 
 #include <asm/cacheflush.h>
@@ -26,6 +14,7 @@
 #include <linux/irqchip/chained_irq.h>
 #include <linux/kernel.h>
 #include <linux/mfd/syscon.h>
+#include <linux/notifier.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
@@ -80,6 +69,25 @@ static const struct altr_sdram_prv_data a10_data = {
 	.ue_set_mask        = A10_DIAGINT_TDERRA_MASK,
 };
 
+static const struct altr_sdram_prv_data s10_data = {
+	.ecc_ctrl_offset    = S10_ECCCTRL1_OFST,
+	.ecc_ctl_en_mask    = A10_ECCCTRL1_ECC_EN,
+	.ecc_stat_offset    = S10_INTSTAT_OFST,
+	.ecc_stat_ce_mask   = A10_INTSTAT_SBEERR,
+	.ecc_stat_ue_mask   = A10_INTSTAT_DBEERR,
+	.ecc_saddr_offset   = S10_SERRADDR_OFST,
+	.ecc_daddr_offset   = S10_DERRADDR_OFST,
+	.ecc_irq_en_offset  = S10_ERRINTEN_OFST,
+	.ecc_irq_en_mask    = A10_ECC_IRQ_EN_MASK,
+	.ecc_irq_clr_offset = S10_INTSTAT_OFST,
+	.ecc_irq_clr_mask   = (A10_INTSTAT_SBEERR | A10_INTSTAT_DBEERR),
+	.ecc_cnt_rst_offset = S10_ECCCTRL1_OFST,
+	.ecc_cnt_rst_mask   = A10_ECC_CNT_RESET_MASK,
+	.ce_ue_trgr_offset  = S10_DIAGINTTEST_OFST,
+	.ce_set_mask        = A10_DIAGINT_TSERRA_MASK,
+	.ue_set_mask        = A10_DIAGINT_TDERRA_MASK,
+};
+
 /*********************** EDAC Memory Controller Functions ****************/
 
 /* The SDRAM controller uses the EDAC Memory Controller framework.       */
@@ -231,6 +239,7 @@ static unsigned long get_total_mem(void)
 static const struct of_device_id altr_sdram_ctrl_of_match[] = {
 	{ .compatible = "altr,sdram-edac", .data = &c5_data},
 	{ .compatible = "altr,sdram-edac-a10", .data = &a10_data},
+	{ .compatible = "altr,sdram-edac-s10", .data = &s10_data},
 	{},
 };
 MODULE_DEVICE_TABLE(of, altr_sdram_ctrl_of_match);
@@ -477,6 +486,292 @@ static int altr_sdram_remove(struct platform_device *pdev)
 	return 0;
 }
 
+/**************** Stratix 10 EDAC Memory Controller Functions ************/
+
+/**
+ * s10_protected_reg_write
+ * Write to a protected SMC register.
+ * @context: Not used.
+ * @reg: Address of register
+ * @value: Value to write
+ * Return: INTEL_SIP_SMC_STATUS_OK (0) on success
+ *	   INTEL_SIP_SMC_REG_ERROR on error
+ *	   INTEL_SIP_SMC_RETURN_UNKNOWN_FUNCTION if not supported
+ */
+static int s10_protected_reg_write(void *context, unsigned int reg,
+				   unsigned int val)
+{
+	struct arm_smccc_res result;
+
+	arm_smccc_smc(INTEL_SIP_SMC_REG_WRITE, reg, val, 0, 0,
+		      0, 0, 0, &result);
+
+	return (int)result.a0;
+}
+
+/**
+ * s10_protected_reg_read
+ * Read the status of a protected SMC register
+ * @context: Not used.
+ * @reg: Address of register
+ * @value: Value read.
+ * Return: INTEL_SIP_SMC_STATUS_OK (0) on success
+ *	   INTEL_SIP_SMC_REG_ERROR on error
+ *	   INTEL_SIP_SMC_RETURN_UNKNOWN_FUNCTION if not supported
+ */
+static int s10_protected_reg_read(void *context, unsigned int reg,
+				  unsigned int *val)
+{
+	struct arm_smccc_res result;
+
+	arm_smccc_smc(INTEL_SIP_SMC_REG_READ, reg, 0, 0, 0,
+		      0, 0, 0, &result);
+
+	*val = (unsigned int)result.a1;
+
+	return (int)result.a0;
+}
+
+static bool s10_sdram_writeable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case S10_ECCCTRL1_OFST:
+	case S10_ERRINTEN_OFST:
+	case S10_INTMODE_OFST:
+	case S10_INTSTAT_OFST:
+	case S10_DIAGINTTEST_OFST:
+	case S10_SYSMGR_ECC_INTMASK_VAL_OFST:
+	case S10_SYSMGR_ECC_INTMASK_SET_OFST:
+	case S10_SYSMGR_ECC_INTMASK_CLR_OFST:
+		return true;
+	}
+	return false;
+}
+
+static bool s10_sdram_readable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case S10_ECCCTRL1_OFST:
+	case S10_ERRINTEN_OFST:
+	case S10_INTMODE_OFST:
+	case S10_INTSTAT_OFST:
+	case S10_DERRADDR_OFST:
+	case S10_SERRADDR_OFST:
+	case S10_DIAGINTTEST_OFST:
+	case S10_SYSMGR_ECC_INTMASK_VAL_OFST:
+	case S10_SYSMGR_ECC_INTMASK_SET_OFST:
+	case S10_SYSMGR_ECC_INTMASK_CLR_OFST:
+	case S10_SYSMGR_ECC_INTSTAT_SERR_OFST:
+	case S10_SYSMGR_ECC_INTSTAT_DERR_OFST:
+		return true;
+	}
+	return false;
+}
+
+static bool s10_sdram_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case S10_ECCCTRL1_OFST:
+	case S10_ERRINTEN_OFST:
+	case S10_INTMODE_OFST:
+	case S10_INTSTAT_OFST:
+	case S10_DERRADDR_OFST:
+	case S10_SERRADDR_OFST:
+	case S10_DIAGINTTEST_OFST:
+	case S10_SYSMGR_ECC_INTMASK_VAL_OFST:
+	case S10_SYSMGR_ECC_INTMASK_SET_OFST:
+	case S10_SYSMGR_ECC_INTMASK_CLR_OFST:
+	case S10_SYSMGR_ECC_INTSTAT_SERR_OFST:
+	case S10_SYSMGR_ECC_INTSTAT_DERR_OFST:
+		return true;
+	}
+	return false;
+}
+
+static const struct regmap_config s10_sdram_regmap_cfg = {
+	.name = "s10_ddr",
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xffffffff,
+	.writeable_reg = s10_sdram_writeable_reg,
+	.readable_reg = s10_sdram_readable_reg,
+	.volatile_reg = s10_sdram_volatile_reg,
+	.reg_read = s10_protected_reg_read,
+	.reg_write = s10_protected_reg_write,
+	.use_single_rw = true,
+};
+
+static int altr_s10_sdram_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *id;
+	struct edac_mc_layer layers[2];
+	struct mem_ctl_info *mci;
+	struct altr_sdram_mc_data *drvdata;
+	const struct altr_sdram_prv_data *priv;
+	struct regmap *regmap;
+	struct dimm_info *dimm;
+	u32 read_reg;
+	int irq, ret = 0;
+	unsigned long mem_size;
+
+	id = of_match_device(altr_sdram_ctrl_of_match, &pdev->dev);
+	if (!id)
+		return -ENODEV;
+
+	/* Grab specific offsets and masks for Stratix10 */
+	priv = of_match_node(altr_sdram_ctrl_of_match,
+			     pdev->dev.of_node)->data;
+
+	regmap = devm_regmap_init(&pdev->dev, NULL, (void *)priv,
+				  &s10_sdram_regmap_cfg);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	/* Validate the SDRAM controller has ECC enabled */
+	if (regmap_read(regmap, priv->ecc_ctrl_offset, &read_reg) ||
+	    ((read_reg & priv->ecc_ctl_en_mask) != priv->ecc_ctl_en_mask)) {
+		edac_printk(KERN_ERR, EDAC_MC,
+			    "No ECC/ECC disabled [0x%08X]\n", read_reg);
+		return -ENODEV;
+	}
+
+	/* Grab memory size from device tree. */
+	mem_size = get_total_mem();
+	if (!mem_size) {
+		edac_printk(KERN_ERR, EDAC_MC, "Unable to calculate memory size\n");
+		return -ENODEV;
+	}
+
+	/* Ensure the SDRAM Interrupt is disabled */
+	if (regmap_update_bits(regmap, priv->ecc_irq_en_offset,
+			       priv->ecc_irq_en_mask, 0)) {
+		edac_printk(KERN_ERR, EDAC_MC,
+			    "Error disabling SDRAM ECC IRQ\n");
+		return -ENODEV;
+	}
+
+	/* Toggle to clear the SDRAM Error count */
+	if (regmap_update_bits(regmap, priv->ecc_cnt_rst_offset,
+			       priv->ecc_cnt_rst_mask,
+			       priv->ecc_cnt_rst_mask)) {
+		edac_printk(KERN_ERR, EDAC_MC,
+			    "Error clearing SDRAM ECC count\n");
+		return -ENODEV;
+	}
+
+	if (regmap_update_bits(regmap, priv->ecc_cnt_rst_offset,
+			       priv->ecc_cnt_rst_mask, 0)) {
+		edac_printk(KERN_ERR, EDAC_MC,
+			    "Error clearing SDRAM ECC count\n");
+		return -ENODEV;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		edac_printk(KERN_ERR, EDAC_MC,
+			    "No irq %d in DT\n", irq);
+		return -ENODEV;
+	}
+
+	layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
+	layers[0].size = 1;
+	layers[0].is_virt_csrow = true;
+	layers[1].type = EDAC_MC_LAYER_CHANNEL;
+	layers[1].size = 1;
+	layers[1].is_virt_csrow = false;
+	mci = edac_mc_alloc(0, ARRAY_SIZE(layers), layers,
+			    sizeof(struct altr_sdram_mc_data));
+	if (!mci)
+		return -ENOMEM;
+
+	mci->pdev = &pdev->dev;
+	drvdata = mci->pvt_info;
+	drvdata->mc_vbase = regmap;
+	drvdata->data = priv;
+	platform_set_drvdata(pdev, mci);
+
+	if (!devres_open_group(&pdev->dev, NULL, GFP_KERNEL)) {
+		edac_printk(KERN_ERR, EDAC_MC,
+			    "Unable to get managed device resource\n");
+		ret = -ENOMEM;
+		goto free;
+	}
+
+	mci->mtype_cap = MEM_FLAG_DDR3;
+	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
+	mci->edac_cap = EDAC_FLAG_SECDED;
+	mci->mod_name = EDAC_MOD_STR;
+	mci->ctl_name = dev_name(&pdev->dev);
+	mci->scrub_mode = SCRUB_SW_SRC;
+	mci->dev_name = dev_name(&pdev->dev);
+
+	dimm = *mci->dimms;
+	dimm->nr_pages = ((mem_size - 1) >> PAGE_SHIFT) + 1;
+	dimm->grain = 8;
+	dimm->dtype = DEV_X8;
+	dimm->mtype = MEM_DDR3;
+	dimm->edac_mode = EDAC_SECDED;
+
+	ret = edac_mc_add_mc(mci);
+	if (ret < 0)
+		goto err;
+
+	ret = devm_request_irq(&pdev->dev, irq, altr_sdram_mc_err_handler,
+			       IRQF_SHARED, dev_name(&pdev->dev), mci);
+	if (ret < 0) {
+		edac_mc_printk(mci, KERN_ERR,
+			       "Unable to request irq %d\n", irq);
+		ret = -ENODEV;
+		goto err2;
+	}
+
+	if (regmap_write(regmap, S10_SYSMGR_ECC_INTMASK_CLR_OFST,
+			 S10_DDR0_IRQ_MASK)) {
+		edac_printk(KERN_ERR, EDAC_MC,
+			    "Error clearing SDRAM ECC count\n");
+		return -ENODEV;
+	}
+
+	if (regmap_update_bits(drvdata->mc_vbase, priv->ecc_irq_en_offset,
+			       priv->ecc_irq_en_mask, priv->ecc_irq_en_mask)) {
+		edac_mc_printk(mci, KERN_ERR,
+			       "Error enabling SDRAM ECC IRQ\n");
+		ret = -ENODEV;
+		goto err2;
+	}
+
+	altr_sdr_mc_create_debugfs_nodes(mci);
+
+	devres_close_group(&pdev->dev, NULL);
+
+	return 0;
+
+err2:
+	edac_mc_del_mc(&pdev->dev);
+err:
+	devres_release_group(&pdev->dev, NULL);
+free:
+	edac_mc_free(mci);
+	edac_printk(KERN_ERR, EDAC_MC,
+		    "EDAC Probe Failed; Error %d\n", ret);
+
+	return ret;
+}
+
+static int altr_s10_sdram_remove(struct platform_device *pdev)
+{
+	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
+
+	edac_mc_del_mc(&pdev->dev);
+	edac_mc_free(mci);
+	platform_set_drvdata(pdev, NULL);
+
+	return 0;
+}
+
+/************** </Stratix10 EDAC Memory Controller Functions> ***********/
+
 /*
  * If you want to suspend, need to disable EDAC by removing it
  * from the device tree or defconfig.
@@ -508,6 +803,20 @@ static struct platform_driver altr_sdram_edac_driver = {
 
 module_platform_driver(altr_sdram_edac_driver);
 
+static struct platform_driver altr_s10_sdram_edac_driver = {
+	.probe = altr_s10_sdram_probe,
+	.remove = altr_s10_sdram_remove,
+	.driver = {
+		.name = "altr_s10_sdram_edac",
+#ifdef CONFIG_PM
+		.pm = &altr_sdram_pm_ops,
+#endif
+		.of_match_table = altr_sdram_ctrl_of_match,
+	},
+};
+
+module_platform_driver(altr_s10_sdram_edac_driver);
+
 /************************* EDAC Parent Probe *************************/
 
 static const struct of_device_id altr_edac_device_of_match[];
@@ -1106,7 +1415,7 @@ static void *ocram_alloc_mem(size_t size, void **other)
 
 static void ocram_free_mem(void *p, size_t size, void *other)
 {
-	gen_pool_free((struct gen_pool *)other, (u32)p, size);
+	gen_pool_free((struct gen_pool *)other, (unsigned long)p, size);
 }
 
 static const struct edac_device_prv_data ocramecc_data = {
@@ -1925,6 +2234,171 @@ static struct platform_driver altr_edac_a10_driver = {
 };
 module_platform_driver(altr_edac_a10_driver);
 
+/************** Stratix 10 EDAC Device Controller Functions> ************/
+
+#define to_s10edac(p, m) container_of(p, struct altr_stratix10_edac, m)
+
+/*
+ * The double bit error is handled through SError which is fatal. This is
+ * called as a panic notifier to printout ECC error info as part of the panic.
+ */
+static int s10_edac_dberr_handler(struct notifier_block *this,
+				  unsigned long event, void *ptr)
+{
+	struct altr_stratix10_edac *edac = to_s10edac(this, panic_notifier);
+	int err_addr, dberror;
+
+	s10_protected_reg_read(edac, S10_SYSMGR_ECC_INTSTAT_DERR_OFST,
+			       &dberror);
+	/* Remember the UE Errors for a reboot */
+	s10_protected_reg_write(edac, S10_SYSMGR_UE_VAL_OFST, dberror);
+	if (dberror & S10_DDR0_IRQ_MASK) {
+		s10_protected_reg_read(edac, S10_DERRADDR_OFST, &err_addr);
+		/* Remember the UE Error address */
+		s10_protected_reg_write(edac, S10_SYSMGR_UE_ADDR_OFST,
+					err_addr);
+		edac_printk(KERN_ERR, EDAC_MC,
+			    "EDAC: [Uncorrectable errors @ 0x%08X]\n\n",
+			    err_addr);
+	}
+
+	return NOTIFY_DONE;
+}
+
+static void altr_edac_s10_irq_handler(struct irq_desc *desc)
+{
+	struct altr_stratix10_edac *edac = irq_desc_get_handler_data(desc);
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	int irq = irq_desc_get_irq(desc);
+	int bit, sm_offset, irq_status;
+
+	sm_offset = S10_SYSMGR_ECC_INTSTAT_SERR_OFST;
+
+	chained_irq_enter(chip, desc);
+
+	s10_protected_reg_read(NULL, sm_offset, &irq_status);
+
+	for_each_set_bit(bit, (unsigned long *)&irq_status, 32) {
+		irq = irq_linear_revmap(edac->domain, bit);
+		if (irq)
+			generic_handle_irq(irq);
+	}
+
+	chained_irq_exit(chip, desc);
+}
+
+static void s10_eccmgr_irq_mask(struct irq_data *d)
+{
+	struct altr_stratix10_edac *edac = irq_data_get_irq_chip_data(d);
+
+	s10_protected_reg_write(edac, S10_SYSMGR_ECC_INTMASK_SET_OFST,
+				BIT(d->hwirq));
+}
+
+static void s10_eccmgr_irq_unmask(struct irq_data *d)
+{
+	struct altr_stratix10_edac *edac = irq_data_get_irq_chip_data(d);
+
+	s10_protected_reg_write(edac, S10_SYSMGR_ECC_INTMASK_CLR_OFST,
+				BIT(d->hwirq));
+}
+
+static int s10_eccmgr_irqdomain_map(struct irq_domain *d, unsigned int irq,
+				    irq_hw_number_t hwirq)
+{
+	struct altr_stratix10_edac *edac = d->host_data;
+
+	irq_set_chip_and_handler(irq, &edac->irq_chip, handle_simple_irq);
+	irq_set_chip_data(irq, edac);
+	irq_set_noprobe(irq);
+
+	return 0;
+}
+
+static const struct irq_domain_ops s10_eccmgr_ic_ops = {
+	.map = s10_eccmgr_irqdomain_map,
+	.xlate = irq_domain_xlate_twocell,
+};
+
+static int altr_edac_s10_probe(struct platform_device *pdev)
+{
+	struct altr_stratix10_edac *edac;
+	struct device_node *child;
+	int dberror, err_addr;
+
+	edac = devm_kzalloc(&pdev->dev, sizeof(*edac), GFP_KERNEL);
+	if (!edac)
+		return -ENOMEM;
+
+	edac->dev = &pdev->dev;
+	platform_set_drvdata(pdev, edac);
+	INIT_LIST_HEAD(&edac->s10_ecc_devices);
+
+	edac->irq_chip.name = pdev->dev.of_node->name;
+	edac->irq_chip.irq_mask = s10_eccmgr_irq_mask;
+	edac->irq_chip.irq_unmask = s10_eccmgr_irq_unmask;
+	edac->domain = irq_domain_add_linear(pdev->dev.of_node, 64,
+					     &s10_eccmgr_ic_ops, edac);
+	if (!edac->domain) {
+		dev_err(&pdev->dev, "Error adding IRQ domain\n");
+		return -ENOMEM;
+	}
+
+	edac->sb_irq = platform_get_irq(pdev, 0);
+	if (edac->sb_irq < 0) {
+		dev_err(&pdev->dev, "No SBERR IRQ resource\n");
+		return edac->sb_irq;
+	}
+
+	irq_set_chained_handler_and_data(edac->sb_irq,
+					 altr_edac_s10_irq_handler,
+					 edac);
+
+	edac->panic_notifier.notifier_call = s10_edac_dberr_handler;
+	atomic_notifier_chain_register(&panic_notifier_list,
+				       &edac->panic_notifier);
+
+	/* Printout a message if uncorrectable error previously. */
+	s10_protected_reg_read(edac, S10_SYSMGR_UE_VAL_OFST, &dberror);
+	if (dberror) {
+		s10_protected_reg_read(edac, S10_SYSMGR_UE_ADDR_OFST,
+				       &err_addr);
+		edac_printk(KERN_ERR, EDAC_DEVICE,
+			    "Previous Boot UE detected[0x%X] @ 0x%X\n",
+			    dberror, err_addr);
+		/* Reset the sticky registers */
+		s10_protected_reg_write(edac, S10_SYSMGR_UE_VAL_OFST, 0);
+		s10_protected_reg_write(edac, S10_SYSMGR_UE_ADDR_OFST, 0);
+	}
+
+	for_each_child_of_node(pdev->dev.of_node, child) {
+		if (!of_device_is_available(child))
+			continue;
+
+		if (of_device_is_compatible(child, "altr,sdram-edac-s10"))
+			of_platform_populate(pdev->dev.of_node,
+					     altr_sdram_ctrl_of_match,
+					     NULL, &pdev->dev);
+	}
+
+	return 0;
+}
+
+static const struct of_device_id altr_edac_s10_of_match[] = {
+	{ .compatible = "altr,socfpga-s10-ecc-manager" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, altr_edac_s10_of_match);
+
+static struct platform_driver altr_edac_s10_driver = {
+	.probe =  altr_edac_s10_probe,
+	.driver = {
+		.name = "socfpga_s10_ecc_manager",
+		.of_match_table = altr_edac_s10_of_match,
+	},
+};
+module_platform_driver(altr_edac_s10_driver);
+
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Thor Thayer");
 MODULE_DESCRIPTION("EDAC Driver for Altera Memories");
diff --git a/drivers/edac/altera_edac.h b/drivers/edac/altera_edac.h
index cbc96290f743..81f0554e09de 100644
--- a/drivers/edac/altera_edac.h
+++ b/drivers/edac/altera_edac.h
@@ -1,23 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
- *
+ * Copyright (C) 2017-2018, Intel Corporation
  * Copyright (C) 2015 Altera Corporation
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #ifndef _ALTERA_EDAC_H
 #define _ALTERA_EDAC_H
 
+#include <linux/arm-smccc.h>
 #include <linux/edac.h>
 #include <linux/types.h>
 
@@ -94,6 +84,7 @@
 /* SDRAM Controller Address Width Register */
 #define CV_DRAMADDRW               0xFFC2502C
 #define A10_DRAMADDRW              0xFFCFA0A8
+#define S10_DRAMADDRW              0xF80110E0
 
 /* SDRAM Controller Address Widths Field Register */
 #define DRAMADDRW_COLBIT_MASK      0x001F
@@ -115,6 +106,7 @@
 /* SDRAM Controller Interface Data Width Register */
 #define CV_DRAMIFWIDTH             0xFFC25030
 #define A10_DRAMIFWIDTH            0xFFCFB008
+#define S10_DRAMIFWIDTH            0xF8011008
 
 /* SDRAM Controller Interface Data Width Defines */
 #define CV_DRAMIFWIDTH_16B_ECC     24
@@ -164,6 +156,34 @@
 #define A10_INTMASK_CLR_OFST       0x10
 #define A10_DDR0_IRQ_MASK          BIT(17)
 
+/************* Stratix10 Defines **************/
+
+/* SDRAM Controller EccCtrl Register */
+#define S10_ECCCTRL1_OFST          0xF8011100
+
+/* SDRAM Controller DRAM IRQ Register */
+#define S10_ERRINTEN_OFST          0xF8011110
+
+/* SDRAM Interrupt Mode Register */
+#define S10_INTMODE_OFST           0xF801111C
+
+/* SDRAM Controller Error Status Register */
+#define S10_INTSTAT_OFST           0xF8011120
+
+/* SDRAM Controller ECC Error Address Register */
+#define S10_DERRADDR_OFST          0xF801112C
+#define S10_SERRADDR_OFST          0xF8011130
+
+/* SDRAM Controller ECC Diagnostic Register */
+#define S10_DIAGINTTEST_OFST       0xF8011124
+
+/* SDRAM Single Bit Error Count Compare Set Register */
+#define S10_SERRCNTREG_OFST        0xF801113C
+
+/* Sticky registers for Uncorrected Errors */
+#define S10_SYSMGR_UE_VAL_OFST     0xFFD12220
+#define S10_SYSMGR_UE_ADDR_OFST    0xFFD12224
+
 struct altr_sdram_prv_data {
 	int ecc_ctrl_offset;
 	int ecc_ctl_en_mask;
@@ -296,6 +316,18 @@ struct altr_sdram_mc_data {
 /* A10 ECC Controller memory initialization timeout */
 #define ALTR_A10_ECC_INIT_WATCHDOG_10US      10000
 
+/************* Stratix10 Defines **************/
+
+/* Stratix10 ECC Manager Defines */
+#define S10_SYSMGR_ECC_INTMASK_VAL_OFST   0xFFD12090
+#define S10_SYSMGR_ECC_INTMASK_SET_OFST   0xFFD12094
+#define S10_SYSMGR_ECC_INTMASK_CLR_OFST   0xFFD12098
+
+#define S10_SYSMGR_ECC_INTSTAT_SERR_OFST  0xFFD1209C
+#define S10_SYSMGR_ECC_INTSTAT_DERR_OFST  0xFFD120A0
+
+#define S10_DDR0_IRQ_MASK                 BIT(16)
+
 struct altr_edac_device_dev;
 
 struct edac_device_prv_data {
@@ -340,4 +372,78 @@ struct altr_arria10_edac {
 	struct list_head	a10_ecc_devices;
 };
 
+/*
+ * Functions specified by ARM SMC Calling convention:
+ *
+ * FAST call executes atomic operations, returns when the requested operation
+ * has completed.
+ * STD call starts a operation which can be preempted by a non-secure
+ * interrupt. The call can return before the requested operation has
+ * completed.
+ *
+ * a0..a7 is used as register names in the descriptions below, on arm32
+ * that translates to r0..r7 and on arm64 to w0..w7.
+ */
+
+#define INTEL_SIP_SMC_STD_CALL_VAL(func_num) \
+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_STD_CALL, ARM_SMCCC_SMC_64, \
+	ARM_SMCCC_OWNER_SIP, (func_num))
+
+#define INTEL_SIP_SMC_FAST_CALL_VAL(func_num) \
+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \
+	ARM_SMCCC_OWNER_SIP, (func_num))
+
+#define INTEL_SIP_SMC_RETURN_UNKNOWN_FUNCTION		0xFFFFFFFF
+#define INTEL_SIP_SMC_STATUS_OK				0x0
+#define INTEL_SIP_SMC_REG_ERROR				0x5
+
+/*
+ * Request INTEL_SIP_SMC_REG_READ
+ *
+ * Read a protected register using SMCCC
+ *
+ * Call register usage:
+ * a0: INTEL_SIP_SMC_REG_READ.
+ * a1: register address.
+ * a2-7: not used.
+ *
+ * Return status:
+ * a0: INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_REG_ERROR, or
+ *     INTEL_SIP_SMC_RETURN_UNKNOWN_FUNCTION
+ * a1: Value in the register
+ * a2-3: not used.
+ */
+#define INTEL_SIP_SMC_FUNCID_REG_READ 7
+#define INTEL_SIP_SMC_REG_READ \
+	INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_REG_READ)
+
+/*
+ * Request INTEL_SIP_SMC_REG_WRITE
+ *
+ * Write a protected register using SMCCC
+ *
+ * Call register usage:
+ * a0: INTEL_SIP_SMC_REG_WRITE.
+ * a1: register address
+ * a2: value to program into register.
+ * a3-7: not used.
+ *
+ * Return status:
+ * a0: INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_REG_ERROR, or
+ *     INTEL_SIP_SMC_RETURN_UNKNOWN_FUNCTION
+ * a1-3: not used.
+ */
+#define INTEL_SIP_SMC_FUNCID_REG_WRITE 8
+#define INTEL_SIP_SMC_REG_WRITE \
+	INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_REG_WRITE)
+
+struct altr_stratix10_edac {
+	struct device		*dev;
+	int sb_irq;
+	struct irq_domain	*domain;
+	struct irq_chip		irq_chip;
+	struct list_head	s10_ecc_devices;
+	struct notifier_block	panic_notifier;
+};
+
 #endif	/* #ifndef _ALTERA_EDAC_H */
diff --git a/drivers/edac/ghes_edac.c b/drivers/edac/ghes_edac.c
index 68b6ee18bea6..473aeec4b1da 100644
--- a/drivers/edac/ghes_edac.c
+++ b/drivers/edac/ghes_edac.c
@@ -91,6 +91,7 @@ static void ghes_edac_dmidecode(const struct dmi_header *dh, void *arg)
 		struct dimm_info *dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms,
 						       mci->n_layers,
 						       dimm_fill->count, 0, 0);
+		u16 rdr_mask = BIT(7) | BIT(13);
 
 		if (entry->size == 0xffff) {
 			pr_info("Can't get DIMM%i size\n",
@@ -99,22 +100,21 @@ static void ghes_edac_dmidecode(const struct dmi_header *dh, void *arg)
 		} else if (entry->size == 0x7fff) {
 			dimm->nr_pages = MiB_TO_PAGES(entry->extended_size);
 		} else {
-			if (entry->size & 1 << 15)
-				dimm->nr_pages = MiB_TO_PAGES((entry->size &
-							       0x7fff) << 10);
+			if (entry->size & BIT(15))
+				dimm->nr_pages = MiB_TO_PAGES((entry->size & 0x7fff) << 10);
 			else
 				dimm->nr_pages = MiB_TO_PAGES(entry->size);
 		}
 
 		switch (entry->memory_type) {
 		case 0x12:
-			if (entry->type_detail & 1 << 13)
+			if (entry->type_detail & BIT(13))
 				dimm->mtype = MEM_RDDR;
 			else
 				dimm->mtype = MEM_DDR;
 			break;
 		case 0x13:
-			if (entry->type_detail & 1 << 13)
+			if (entry->type_detail & BIT(13))
 				dimm->mtype = MEM_RDDR2;
 			else
 				dimm->mtype = MEM_DDR2;
@@ -123,20 +123,29 @@ static void ghes_edac_dmidecode(const struct dmi_header *dh, void *arg)
 			dimm->mtype = MEM_FB_DDR2;
 			break;
 		case 0x18:
-			if (entry->type_detail & 1 << 13)
+			if (entry->type_detail & BIT(12))
+				dimm->mtype = MEM_NVDIMM;
+			else if (entry->type_detail & BIT(13))
 				dimm->mtype = MEM_RDDR3;
 			else
 				dimm->mtype = MEM_DDR3;
 			break;
+		case 0x1a:
+			if (entry->type_detail & BIT(12))
+				dimm->mtype = MEM_NVDIMM;
+			else if (entry->type_detail & BIT(13))
+				dimm->mtype = MEM_RDDR4;
+			else
+				dimm->mtype = MEM_DDR4;
+			break;
 		default:
-			if (entry->type_detail & 1 << 6)
+			if (entry->type_detail & BIT(6))
 				dimm->mtype = MEM_RMBS;
-			else if ((entry->type_detail & ((1 << 7) | (1 << 13)))
-				 == ((1 << 7) | (1 << 13)))
+			else if ((entry->type_detail & rdr_mask) == rdr_mask)
 				dimm->mtype = MEM_RDR;
-			else if (entry->type_detail & 1 << 7)
+			else if (entry->type_detail & BIT(7))
 				dimm->mtype = MEM_SDR;
-			else if (entry->type_detail & 1 << 9)
+			else if (entry->type_detail & BIT(9))
 				dimm->mtype = MEM_EDO;
 			else
 				dimm->mtype = MEM_UNKNOWN;
@@ -172,8 +181,7 @@ static void ghes_edac_dmidecode(const struct dmi_header *dh, void *arg)
 	}
 }
 
-void ghes_edac_report_mem_error(struct ghes *ghes, int sev,
-				struct cper_sec_mem_err *mem_err)
+void ghes_edac_report_mem_error(int sev, struct cper_sec_mem_err *mem_err)
 {
 	enum hw_event_mc_err_type type;
 	struct edac_raw_error_desc *e;
@@ -183,10 +191,8 @@ void ghes_edac_report_mem_error(struct ghes *ghes, int sev,
 	char *p;
 	u8 grain_bits;
 
-	if (!pvt) {
-		pr_err("Internal error: Can't find EDAC structure\n");
+	if (!pvt)
 		return;
-	}
 
 	/*
 	 * We can do the locking below because GHES defers error processing
@@ -434,12 +440,16 @@ int ghes_edac_register(struct ghes *ghes, struct device *dev)
 	struct mem_ctl_info *mci;
 	struct edac_mc_layer layers[1];
 	struct ghes_edac_dimm_fill dimm_fill;
-	int idx;
+	int idx = -1;
 
-	/* Check if safe to enable on this system */
-	idx = acpi_match_platform_list(plat_list);
-	if (!force_load && idx < 0)
-		return 0;
+	if (IS_ENABLED(CONFIG_X86)) {
+		/* Check if safe to enable on this system */
+		idx = acpi_match_platform_list(plat_list);
+		if (!force_load && idx < 0)
+			return -ENODEV;
+	} else {
+		idx = 0;
+	}
 
 	/*
 	 * We have only one logical memory controller to which all DIMMs belong.
@@ -519,6 +529,9 @@ void ghes_edac_unregister(struct ghes *ghes)
 {
 	struct mem_ctl_info *mci;
 
+	if (!ghes_pvt)
+		return;
+
 	mci = ghes_pvt->mci;
 	edac_mc_del_mc(mci->pdev);
 	edac_mc_free(mci);
diff --git a/drivers/edac/i7core_edac.c b/drivers/edac/i7core_edac.c
index 8c5540160a23..4d0ea3563d47 100644
--- a/drivers/edac/i7core_edac.c
+++ b/drivers/edac/i7core_edac.c
@@ -1743,7 +1743,7 @@ static void i7core_mce_output_error(struct mem_ctl_info *mci,
 		err = "write parity error";
 		break;
 	case 19:
-		err = "redundacy loss";
+		err = "redundancy loss";
 		break;
 	case 20:
 		err = "reserved";
diff --git a/drivers/firmware/arm_scmi/base.c b/drivers/firmware/arm_scmi/base.c
index 0d3806c0d432..9dff33ea6416 100644
--- a/drivers/firmware/arm_scmi/base.c
+++ b/drivers/firmware/arm_scmi/base.c
@@ -26,7 +26,7 @@ struct scmi_msg_resp_base_attributes {
  * scmi_base_attributes_get() - gets the implementation details
  *	that are associated with the base protocol.
  *
- * @handle - SCMI entity handle
+ * @handle: SCMI entity handle
  *
  * Return: 0 on success, else appropriate SCMI error.
  */
@@ -37,7 +37,7 @@ static int scmi_base_attributes_get(const struct scmi_handle *handle)
 	struct scmi_msg_resp_base_attributes *attr_info;
 	struct scmi_revision_info *rev = handle->version;
 
-	ret = scmi_one_xfer_init(handle, PROTOCOL_ATTRIBUTES,
+	ret = scmi_xfer_get_init(handle, PROTOCOL_ATTRIBUTES,
 				 SCMI_PROTOCOL_BASE, 0, sizeof(*attr_info), &t);
 	if (ret)
 		return ret;
@@ -49,15 +49,16 @@ static int scmi_base_attributes_get(const struct scmi_handle *handle)
 		rev->num_agents = attr_info->num_agents;
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
+
 	return ret;
 }
 
 /**
  * scmi_base_vendor_id_get() - gets vendor/subvendor identifier ASCII string.
  *
- * @handle - SCMI entity handle
- * @sub_vendor - specify true if sub-vendor ID is needed
+ * @handle: SCMI entity handle
+ * @sub_vendor: specify true if sub-vendor ID is needed
  *
  * Return: 0 on success, else appropriate SCMI error.
  */
@@ -80,7 +81,7 @@ scmi_base_vendor_id_get(const struct scmi_handle *handle, bool sub_vendor)
 		size = ARRAY_SIZE(rev->vendor_id);
 	}
 
-	ret = scmi_one_xfer_init(handle, cmd, SCMI_PROTOCOL_BASE, 0, size, &t);
+	ret = scmi_xfer_get_init(handle, cmd, SCMI_PROTOCOL_BASE, 0, size, &t);
 	if (ret)
 		return ret;
 
@@ -88,7 +89,8 @@ scmi_base_vendor_id_get(const struct scmi_handle *handle, bool sub_vendor)
 	if (!ret)
 		memcpy(vendor_id, t->rx.buf, size);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
+
 	return ret;
 }
 
@@ -97,7 +99,7 @@ scmi_base_vendor_id_get(const struct scmi_handle *handle, bool sub_vendor)
  *	implementation 32-bit version. The format of the version number is
  *	vendor-specific
  *
- * @handle - SCMI entity handle
+ * @handle: SCMI entity handle
  *
  * Return: 0 on success, else appropriate SCMI error.
  */
@@ -109,7 +111,7 @@ scmi_base_implementation_version_get(const struct scmi_handle *handle)
 	struct scmi_xfer *t;
 	struct scmi_revision_info *rev = handle->version;
 
-	ret = scmi_one_xfer_init(handle, BASE_DISCOVER_IMPLEMENT_VERSION,
+	ret = scmi_xfer_get_init(handle, BASE_DISCOVER_IMPLEMENT_VERSION,
 				 SCMI_PROTOCOL_BASE, 0, sizeof(*impl_ver), &t);
 	if (ret)
 		return ret;
@@ -120,7 +122,8 @@ scmi_base_implementation_version_get(const struct scmi_handle *handle)
 		rev->impl_ver = le32_to_cpu(*impl_ver);
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
+
 	return ret;
 }
 
@@ -128,8 +131,8 @@ scmi_base_implementation_version_get(const struct scmi_handle *handle)
  * scmi_base_implementation_list_get() - gets the list of protocols it is
  *	OSPM is allowed to access
  *
- * @handle - SCMI entity handle
- * @protocols_imp - pointer to hold the list of protocol identifiers
+ * @handle: SCMI entity handle
+ * @protocols_imp: pointer to hold the list of protocol identifiers
  *
  * Return: 0 on success, else appropriate SCMI error.
  */
@@ -143,7 +146,7 @@ static int scmi_base_implementation_list_get(const struct scmi_handle *handle,
 	u32 tot_num_ret = 0, loop_num_ret;
 	struct device *dev = handle->dev;
 
-	ret = scmi_one_xfer_init(handle, BASE_DISCOVER_LIST_PROTOCOLS,
+	ret = scmi_xfer_get_init(handle, BASE_DISCOVER_LIST_PROTOCOLS,
 				 SCMI_PROTOCOL_BASE, sizeof(*num_skip), 0, &t);
 	if (ret)
 		return ret;
@@ -172,16 +175,17 @@ static int scmi_base_implementation_list_get(const struct scmi_handle *handle,
 		tot_num_ret += loop_num_ret;
 	} while (loop_num_ret);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
+
 	return ret;
 }
 
 /**
  * scmi_base_discover_agent_get() - discover the name of an agent
  *
- * @handle - SCMI entity handle
- * @id - Agent identifier
- * @name - Agent identifier ASCII string
+ * @handle: SCMI entity handle
+ * @id: Agent identifier
+ * @name: Agent identifier ASCII string
  *
  * An agent id of 0 is reserved to identify the platform itself.
  * Generally operating system is represented as "OSPM"
@@ -194,7 +198,7 @@ static int scmi_base_discover_agent_get(const struct scmi_handle *handle,
 	int ret;
 	struct scmi_xfer *t;
 
-	ret = scmi_one_xfer_init(handle, BASE_DISCOVER_AGENT,
+	ret = scmi_xfer_get_init(handle, BASE_DISCOVER_AGENT,
 				 SCMI_PROTOCOL_BASE, sizeof(__le32),
 				 SCMI_MAX_STR_SIZE, &t);
 	if (ret)
@@ -206,7 +210,8 @@ static int scmi_base_discover_agent_get(const struct scmi_handle *handle,
 	if (!ret)
 		memcpy(name, t->rx.buf, SCMI_MAX_STR_SIZE);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
+
 	return ret;
 }
 
diff --git a/drivers/firmware/arm_scmi/bus.c b/drivers/firmware/arm_scmi/bus.c
index f2760a596c28..472c88ae1c0f 100644
--- a/drivers/firmware/arm_scmi/bus.c
+++ b/drivers/firmware/arm_scmi/bus.c
@@ -125,13 +125,13 @@ scmi_device_create(struct device_node *np, struct device *parent, int protocol)
 	int id, retval;
 	struct scmi_device *scmi_dev;
 
-	id = ida_simple_get(&scmi_bus_id, 1, 0, GFP_KERNEL);
-	if (id < 0)
-		return NULL;
-
 	scmi_dev = kzalloc(sizeof(*scmi_dev), GFP_KERNEL);
 	if (!scmi_dev)
-		goto no_mem;
+		return NULL;
+
+	id = ida_simple_get(&scmi_bus_id, 1, 0, GFP_KERNEL);
+	if (id < 0)
+		goto free_mem;
 
 	scmi_dev->id = id;
 	scmi_dev->protocol_id = protocol;
@@ -141,13 +141,15 @@ scmi_device_create(struct device_node *np, struct device *parent, int protocol)
 	dev_set_name(&scmi_dev->dev, "scmi_dev.%d", id);
 
 	retval = device_register(&scmi_dev->dev);
-	if (!retval)
-		return scmi_dev;
+	if (retval)
+		goto put_dev;
 
+	return scmi_dev;
+put_dev:
 	put_device(&scmi_dev->dev);
-	kfree(scmi_dev);
-no_mem:
 	ida_simple_remove(&scmi_bus_id, id);
+free_mem:
+	kfree(scmi_dev);
 	return NULL;
 }
 
@@ -171,9 +173,9 @@ int scmi_protocol_register(int protocol_id, scmi_prot_init_fn_t fn)
 	spin_lock(&protocol_lock);
 	ret = idr_alloc(&scmi_protocols, fn, protocol_id, protocol_id + 1,
 			GFP_ATOMIC);
+	spin_unlock(&protocol_lock);
 	if (ret != protocol_id)
 		pr_err("unable to allocate SCMI idr slot, err %d\n", ret);
-	spin_unlock(&protocol_lock);
 
 	return ret;
 }
diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c
index 2b90606452a2..e4119eb34986 100644
--- a/drivers/firmware/arm_scmi/clock.c
+++ b/drivers/firmware/arm_scmi/clock.c
@@ -77,7 +77,7 @@ static int scmi_clock_protocol_attributes_get(const struct scmi_handle *handle,
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_clock_protocol_attributes *attr;
 
-	ret = scmi_one_xfer_init(handle, PROTOCOL_ATTRIBUTES,
+	ret = scmi_xfer_get_init(handle, PROTOCOL_ATTRIBUTES,
 				 SCMI_PROTOCOL_CLOCK, 0, sizeof(*attr), &t);
 	if (ret)
 		return ret;
@@ -90,7 +90,7 @@ static int scmi_clock_protocol_attributes_get(const struct scmi_handle *handle,
 		ci->max_async_req = attr->max_async_req;
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -101,7 +101,7 @@ static int scmi_clock_attributes_get(const struct scmi_handle *handle,
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_clock_attributes *attr;
 
-	ret = scmi_one_xfer_init(handle, CLOCK_ATTRIBUTES, SCMI_PROTOCOL_CLOCK,
+	ret = scmi_xfer_get_init(handle, CLOCK_ATTRIBUTES, SCMI_PROTOCOL_CLOCK,
 				 sizeof(clk_id), sizeof(*attr), &t);
 	if (ret)
 		return ret;
@@ -115,7 +115,7 @@ static int scmi_clock_attributes_get(const struct scmi_handle *handle,
 	else
 		clk->name[0] = '\0';
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -132,7 +132,7 @@ scmi_clock_describe_rates_get(const struct scmi_handle *handle, u32 clk_id,
 	struct scmi_msg_clock_describe_rates *clk_desc;
 	struct scmi_msg_resp_clock_describe_rates *rlist;
 
-	ret = scmi_one_xfer_init(handle, CLOCK_DESCRIBE_RATES,
+	ret = scmi_xfer_get_init(handle, CLOCK_DESCRIBE_RATES,
 				 SCMI_PROTOCOL_CLOCK, sizeof(*clk_desc), 0, &t);
 	if (ret)
 		return ret;
@@ -186,7 +186,7 @@ scmi_clock_describe_rates_get(const struct scmi_handle *handle, u32 clk_id,
 		clk->list.num_rates = tot_rate_cnt;
 
 err:
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -196,7 +196,7 @@ scmi_clock_rate_get(const struct scmi_handle *handle, u32 clk_id, u64 *value)
 	int ret;
 	struct scmi_xfer *t;
 
-	ret = scmi_one_xfer_init(handle, CLOCK_RATE_GET, SCMI_PROTOCOL_CLOCK,
+	ret = scmi_xfer_get_init(handle, CLOCK_RATE_GET, SCMI_PROTOCOL_CLOCK,
 				 sizeof(__le32), sizeof(u64), &t);
 	if (ret)
 		return ret;
@@ -211,7 +211,7 @@ scmi_clock_rate_get(const struct scmi_handle *handle, u32 clk_id, u64 *value)
 		*value |= (u64)le32_to_cpu(*(pval + 1)) << 32;
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -222,7 +222,7 @@ static int scmi_clock_rate_set(const struct scmi_handle *handle, u32 clk_id,
 	struct scmi_xfer *t;
 	struct scmi_clock_set_rate *cfg;
 
-	ret = scmi_one_xfer_init(handle, CLOCK_RATE_SET, SCMI_PROTOCOL_CLOCK,
+	ret = scmi_xfer_get_init(handle, CLOCK_RATE_SET, SCMI_PROTOCOL_CLOCK,
 				 sizeof(*cfg), 0, &t);
 	if (ret)
 		return ret;
@@ -235,7 +235,7 @@ static int scmi_clock_rate_set(const struct scmi_handle *handle, u32 clk_id,
 
 	ret = scmi_do_xfer(handle, t);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -246,7 +246,7 @@ scmi_clock_config_set(const struct scmi_handle *handle, u32 clk_id, u32 config)
 	struct scmi_xfer *t;
 	struct scmi_clock_set_config *cfg;
 
-	ret = scmi_one_xfer_init(handle, CLOCK_CONFIG_SET, SCMI_PROTOCOL_CLOCK,
+	ret = scmi_xfer_get_init(handle, CLOCK_CONFIG_SET, SCMI_PROTOCOL_CLOCK,
 				 sizeof(*cfg), 0, &t);
 	if (ret)
 		return ret;
@@ -257,7 +257,7 @@ scmi_clock_config_set(const struct scmi_handle *handle, u32 clk_id, u32 config)
 
 	ret = scmi_do_xfer(handle, t);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h
index 0c30234f9098..937a930ce87d 100644
--- a/drivers/firmware/arm_scmi/common.h
+++ b/drivers/firmware/arm_scmi/common.h
@@ -7,6 +7,7 @@
  * Copyright (C) 2018 ARM Ltd.
  */
 
+#include <linux/bitfield.h>
 #include <linux/completion.h>
 #include <linux/device.h>
 #include <linux/errno.h>
@@ -14,10 +15,10 @@
 #include <linux/scmi_protocol.h>
 #include <linux/types.h>
 
-#define PROTOCOL_REV_MINOR_BITS	16
-#define PROTOCOL_REV_MINOR_MASK	((1U << PROTOCOL_REV_MINOR_BITS) - 1)
-#define PROTOCOL_REV_MAJOR(x)	((x) >> PROTOCOL_REV_MINOR_BITS)
-#define PROTOCOL_REV_MINOR(x)	((x) & PROTOCOL_REV_MINOR_MASK)
+#define PROTOCOL_REV_MINOR_MASK	GENMASK(15, 0)
+#define PROTOCOL_REV_MAJOR_MASK	GENMASK(31, 16)
+#define PROTOCOL_REV_MAJOR(x)	(u16)(FIELD_GET(PROTOCOL_REV_MAJOR_MASK, (x)))
+#define PROTOCOL_REV_MINOR(x)	(u16)(FIELD_GET(PROTOCOL_REV_MINOR_MASK, (x)))
 #define MAX_PROTOCOLS_IMP	16
 #define MAX_OPPS		16
 
@@ -50,8 +51,11 @@ struct scmi_msg_resp_prot_version {
  * @id: The identifier of the command being sent
  * @protocol_id: The identifier of the protocol used to send @id command
  * @seq: The token to identify the message. when a message/command returns,
- *       the platform returns the whole message header unmodified including
- *	 the token.
+ *	the platform returns the whole message header unmodified including
+ *	the token
+ * @status: Status of the transfer once it's complete
+ * @poll_completion: Indicate if the transfer needs to be polled for
+ *	completion or interrupt mode is used
  */
 struct scmi_msg_hdr {
 	u8 id;
@@ -82,18 +86,16 @@ struct scmi_msg {
  *	buffer for the rx path as we use for the tx path.
  * @done: completion event
  */
-
 struct scmi_xfer {
-	void *con_priv;
 	struct scmi_msg_hdr hdr;
 	struct scmi_msg tx;
 	struct scmi_msg rx;
 	struct completion done;
 };
 
-void scmi_one_xfer_put(const struct scmi_handle *h, struct scmi_xfer *xfer);
+void scmi_xfer_put(const struct scmi_handle *h, struct scmi_xfer *xfer);
 int scmi_do_xfer(const struct scmi_handle *h, struct scmi_xfer *xfer);
-int scmi_one_xfer_init(const struct scmi_handle *h, u8 msg_id, u8 prot_id,
+int scmi_xfer_get_init(const struct scmi_handle *h, u8 msg_id, u8 prot_id,
 		       size_t tx_size, size_t rx_size, struct scmi_xfer **p);
 int scmi_handle_put(const struct scmi_handle *handle);
 struct scmi_handle *scmi_handle_get(struct device *dev);
diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c
index 2455be8cbc4f..8f952f2f1a29 100644
--- a/drivers/firmware/arm_scmi/driver.c
+++ b/drivers/firmware/arm_scmi/driver.c
@@ -29,16 +29,12 @@
 
 #include "common.h"
 
-#define MSG_ID_SHIFT		0
-#define MSG_ID_MASK		0xff
-#define MSG_TYPE_SHIFT		8
-#define MSG_TYPE_MASK		0x3
-#define MSG_PROTOCOL_ID_SHIFT	10
-#define MSG_PROTOCOL_ID_MASK	0xff
-#define MSG_TOKEN_ID_SHIFT	18
-#define MSG_TOKEN_ID_MASK	0x3ff
-#define MSG_XTRACT_TOKEN(header)	\
-	(((header) >> MSG_TOKEN_ID_SHIFT) & MSG_TOKEN_ID_MASK)
+#define MSG_ID_MASK		GENMASK(7, 0)
+#define MSG_TYPE_MASK		GENMASK(9, 8)
+#define MSG_PROTOCOL_ID_MASK	GENMASK(17, 10)
+#define MSG_TOKEN_ID_MASK	GENMASK(27, 18)
+#define MSG_XTRACT_TOKEN(hdr)	FIELD_GET(MSG_TOKEN_ID_MASK, (hdr))
+#define MSG_TOKEN_MAX		(MSG_XTRACT_TOKEN(MSG_TOKEN_ID_MASK) + 1)
 
 enum scmi_error_codes {
 	SCMI_SUCCESS = 0,	/* Success */
@@ -55,7 +51,7 @@ enum scmi_error_codes {
 	SCMI_ERR_MAX
 };
 
-/* List of all  SCMI devices active in system */
+/* List of all SCMI devices active in system */
 static LIST_HEAD(scmi_list);
 /* Protection for the entire list */
 static DEFINE_MUTEX(scmi_list_mutex);
@@ -72,7 +68,6 @@ static DEFINE_MUTEX(scmi_list_mutex);
 struct scmi_xfers_info {
 	struct scmi_xfer *xfer_block;
 	unsigned long *xfer_alloc_table;
-	/* protect transfer allocation */
 	spinlock_t xfer_lock;
 };
 
@@ -98,6 +93,7 @@ struct scmi_desc {
  * @payload: Transmit/Receive mailbox channel payload area
  * @dev: Reference to device in the SCMI hierarchy corresponding to this
  *	 channel
+ * @handle: Pointer to SCMI entity handle
  */
 struct scmi_chan_info {
 	struct mbox_client cl;
@@ -108,7 +104,7 @@ struct scmi_chan_info {
 };
 
 /**
- * struct scmi_info - Structure representing a  SCMI instance
+ * struct scmi_info - Structure representing a SCMI instance
  *
  * @dev: Device pointer
  * @desc: SoC description for this instance
@@ -117,9 +113,9 @@ struct scmi_chan_info {
  *	implementation version and (sub-)vendor identification.
  * @minfo: Message info
  * @tx_idr: IDR object to map protocol id to channel info pointer
- * @protocols_imp: list of protocols implemented, currently maximum of
+ * @protocols_imp: List of protocols implemented, currently maximum of
  *	MAX_PROTOCOLS_IMP elements allocated by the base protocol
- * @node: list head
+ * @node: List head
  * @users: Number of users of this instance
  */
 struct scmi_info {
@@ -225,9 +221,7 @@ static void scmi_rx_callback(struct mbox_client *cl, void *m)
 
 	xfer_id = MSG_XTRACT_TOKEN(ioread32(&mem->msg_header));
 
-	/*
-	 * Are we even expecting this?
-	 */
+	/* Are we even expecting this? */
 	if (!test_bit(xfer_id, minfo->xfer_alloc_table)) {
 		dev_err(dev, "message for %d is not expected!\n", xfer_id);
 		return;
@@ -252,12 +246,14 @@ static void scmi_rx_callback(struct mbox_client *cl, void *m)
  *
  * @hdr: pointer to header containing all the information on message id,
  *	protocol id and sequence id.
+ *
+ * Return: 32-bit packed command header to be sent to the platform.
  */
 static inline u32 pack_scmi_header(struct scmi_msg_hdr *hdr)
 {
-	return ((hdr->id & MSG_ID_MASK) << MSG_ID_SHIFT) |
-	   ((hdr->seq & MSG_TOKEN_ID_MASK) << MSG_TOKEN_ID_SHIFT) |
-	   ((hdr->protocol_id & MSG_PROTOCOL_ID_MASK) << MSG_PROTOCOL_ID_SHIFT);
+	return FIELD_PREP(MSG_ID_MASK, hdr->id) |
+		FIELD_PREP(MSG_TOKEN_ID_MASK, hdr->seq) |
+		FIELD_PREP(MSG_PROTOCOL_ID_MASK, hdr->protocol_id);
 }
 
 /**
@@ -286,9 +282,9 @@ static void scmi_tx_prepare(struct mbox_client *cl, void *m)
 }
 
 /**
- * scmi_one_xfer_get() - Allocate one message
+ * scmi_xfer_get() - Allocate one message
  *
- * @handle: SCMI entity handle
+ * @handle: Pointer to SCMI entity handle
  *
  * Helper function which is used by various command functions that are
  * exposed to clients of this driver for allocating a message traffic event.
@@ -299,7 +295,7 @@ static void scmi_tx_prepare(struct mbox_client *cl, void *m)
  *
  * Return: 0 if all went fine, else corresponding error.
  */
-static struct scmi_xfer *scmi_one_xfer_get(const struct scmi_handle *handle)
+static struct scmi_xfer *scmi_xfer_get(const struct scmi_handle *handle)
 {
 	u16 xfer_id;
 	struct scmi_xfer *xfer;
@@ -328,14 +324,14 @@ static struct scmi_xfer *scmi_one_xfer_get(const struct scmi_handle *handle)
 }
 
 /**
- * scmi_one_xfer_put() - Release a message
+ * scmi_xfer_put() - Release a message
  *
- * @minfo: transfer info pointer
- * @xfer: message that was reserved by scmi_one_xfer_get
+ * @handle: Pointer to SCMI entity handle
+ * @xfer: message that was reserved by scmi_xfer_get
  *
  * This holds a spinlock to maintain integrity of internal data structures.
  */
-void scmi_one_xfer_put(const struct scmi_handle *handle, struct scmi_xfer *xfer)
+void scmi_xfer_put(const struct scmi_handle *handle, struct scmi_xfer *xfer)
 {
 	unsigned long flags;
 	struct scmi_info *info = handle_to_scmi_info(handle);
@@ -378,12 +374,12 @@ static bool scmi_xfer_done_no_timeout(const struct scmi_chan_info *cinfo,
 /**
  * scmi_do_xfer() - Do one transfer
  *
- * @info: Pointer to SCMI entity information
+ * @handle: Pointer to SCMI entity handle
  * @xfer: Transfer to initiate and wait for response
  *
  * Return: -ETIMEDOUT in case of no response, if transmit error,
- *   return corresponding error, else if all goes well,
- *   return 0.
+ *	return corresponding error, else if all goes well,
+ *	return 0.
  */
 int scmi_do_xfer(const struct scmi_handle *handle, struct scmi_xfer *xfer)
 {
@@ -440,22 +436,22 @@ int scmi_do_xfer(const struct scmi_handle *handle, struct scmi_xfer *xfer)
 }
 
 /**
- * scmi_one_xfer_init() - Allocate and initialise one message
+ * scmi_xfer_get_init() - Allocate and initialise one message
  *
- * @handle: SCMI entity handle
+ * @handle: Pointer to SCMI entity handle
  * @msg_id: Message identifier
- * @msg_prot_id: Protocol identifier for the message
+ * @prot_id: Protocol identifier for the message
  * @tx_size: transmit message size
  * @rx_size: receive message size
  * @p: pointer to the allocated and initialised message
  *
- * This function allocates the message using @scmi_one_xfer_get and
+ * This function allocates the message using @scmi_xfer_get and
  * initialise the header.
  *
  * Return: 0 if all went fine with @p pointing to message, else
  *	corresponding error.
  */
-int scmi_one_xfer_init(const struct scmi_handle *handle, u8 msg_id, u8 prot_id,
+int scmi_xfer_get_init(const struct scmi_handle *handle, u8 msg_id, u8 prot_id,
 		       size_t tx_size, size_t rx_size, struct scmi_xfer **p)
 {
 	int ret;
@@ -468,7 +464,7 @@ int scmi_one_xfer_init(const struct scmi_handle *handle, u8 msg_id, u8 prot_id,
 	    tx_size > info->desc->max_msg_size)
 		return -ERANGE;
 
-	xfer = scmi_one_xfer_get(handle);
+	xfer = scmi_xfer_get(handle);
 	if (IS_ERR(xfer)) {
 		ret = PTR_ERR(xfer);
 		dev_err(dev, "failed to get free message slot(%d)\n", ret);
@@ -482,13 +478,16 @@ int scmi_one_xfer_init(const struct scmi_handle *handle, u8 msg_id, u8 prot_id,
 	xfer->hdr.poll_completion = false;
 
 	*p = xfer;
+
 	return 0;
 }
 
 /**
  * scmi_version_get() - command to get the revision of the SCMI entity
  *
- * @handle: Handle to SCMI entity information
+ * @handle: Pointer to SCMI entity handle
+ * @protocol: Protocol identifier for the message
+ * @version: Holds returned version of protocol.
  *
  * Updates the SCMI information in the internal data structure.
  *
@@ -501,7 +500,7 @@ int scmi_version_get(const struct scmi_handle *handle, u8 protocol,
 	__le32 *rev_info;
 	struct scmi_xfer *t;
 
-	ret = scmi_one_xfer_init(handle, PROTOCOL_VERSION, protocol, 0,
+	ret = scmi_xfer_get_init(handle, PROTOCOL_VERSION, protocol, 0,
 				 sizeof(*version), &t);
 	if (ret)
 		return ret;
@@ -512,7 +511,7 @@ int scmi_version_get(const struct scmi_handle *handle, u8 protocol,
 		*version = le32_to_cpu(*rev_info);
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -540,12 +539,12 @@ scmi_is_protocol_implemented(const struct scmi_handle *handle, u8 prot_id)
 }
 
 /**
- * scmi_handle_get() - Get the  SCMI handle for a device
+ * scmi_handle_get() - Get the SCMI handle for a device
  *
  * @dev: pointer to device for which we want SCMI handle
  *
  * NOTE: The function does not track individual clients of the framework
- * and is expected to be maintained by caller of  SCMI protocol library.
+ * and is expected to be maintained by caller of SCMI protocol library.
  * scmi_handle_put must be balanced with successful scmi_handle_get
  *
  * Return: pointer to handle if successful, NULL on error
@@ -576,7 +575,7 @@ struct scmi_handle *scmi_handle_get(struct device *dev)
  * @handle: handle acquired by scmi_handle_get
  *
  * NOTE: The function does not track individual clients of the framework
- * and is expected to be maintained by caller of  SCMI protocol library.
+ * and is expected to be maintained by caller of SCMI protocol library.
  * scmi_handle_put must be balanced with successful scmi_handle_get
  *
  * Return: 0 is successfully released
@@ -599,7 +598,7 @@ int scmi_handle_put(const struct scmi_handle *handle)
 }
 
 static const struct scmi_desc scmi_generic_desc = {
-	.max_rx_timeout_ms = 30,	/* we may increase this if required */
+	.max_rx_timeout_ms = 30,	/* We may increase this if required */
 	.max_msg = 20,		/* Limited by MBOX_TX_QUEUE_LEN */
 	.max_msg_size = 128,
 };
@@ -621,9 +620,9 @@ static int scmi_xfer_info_init(struct scmi_info *sinfo)
 	struct scmi_xfers_info *info = &sinfo->minfo;
 
 	/* Pre-allocated messages, no more than what hdr.seq can support */
-	if (WARN_ON(desc->max_msg >= (MSG_TOKEN_ID_MASK + 1))) {
-		dev_err(dev, "Maximum message of %d exceeds supported %d\n",
-			desc->max_msg, MSG_TOKEN_ID_MASK + 1);
+	if (WARN_ON(desc->max_msg >= MSG_TOKEN_MAX)) {
+		dev_err(dev, "Maximum message of %d exceeds supported %ld\n",
+			desc->max_msg, MSG_TOKEN_MAX);
 		return -EINVAL;
 	}
 
@@ -637,8 +636,6 @@ static int scmi_xfer_info_init(struct scmi_info *sinfo)
 	if (!info->xfer_alloc_table)
 		return -ENOMEM;
 
-	bitmap_zero(info->xfer_alloc_table, desc->max_msg);
-
 	/* Pre-initialize the buffer pointer to pre-allocated buffers */
 	for (i = 0, xfer = info->xfer_block; i < desc->max_msg; i++, xfer++) {
 		xfer->rx.buf = devm_kcalloc(dev, sizeof(u8), desc->max_msg_size,
@@ -690,11 +687,12 @@ static int scmi_remove(struct platform_device *pdev)
 		list_del(&info->node);
 	mutex_unlock(&scmi_list_mutex);
 
-	if (!ret) {
-		/* Safe to free channels since no more users */
-		ret = idr_for_each(idr, scmi_mbox_free_channel, idr);
-		idr_destroy(&info->tx_idr);
-	}
+	if (ret)
+		return ret;
+
+	/* Safe to free channels since no more users */
+	ret = idr_for_each(idr, scmi_mbox_free_channel, idr);
+	idr_destroy(&info->tx_idr);
 
 	return ret;
 }
@@ -841,7 +839,8 @@ static int scmi_probe(struct platform_device *pdev)
 		if (of_property_read_u32(child, "reg", &prot_id))
 			continue;
 
-		prot_id &= MSG_PROTOCOL_ID_MASK;
+		if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id))
+			dev_err(dev, "Out of range protocol %d\n", prot_id);
 
 		if (!scmi_is_protocol_implemented(handle, prot_id)) {
 			dev_err(dev, "SCMI protocol %d not implemented\n",
diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c
index 987c64d19801..2a219b1261b1 100644
--- a/drivers/firmware/arm_scmi/perf.c
+++ b/drivers/firmware/arm_scmi/perf.c
@@ -115,7 +115,7 @@ static int scmi_perf_attributes_get(const struct scmi_handle *handle,
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_perf_attributes *attr;
 
-	ret = scmi_one_xfer_init(handle, PROTOCOL_ATTRIBUTES,
+	ret = scmi_xfer_get_init(handle, PROTOCOL_ATTRIBUTES,
 				 SCMI_PROTOCOL_PERF, 0, sizeof(*attr), &t);
 	if (ret)
 		return ret;
@@ -133,7 +133,7 @@ static int scmi_perf_attributes_get(const struct scmi_handle *handle,
 		pi->stats_size = le32_to_cpu(attr->stats_size);
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -145,7 +145,7 @@ scmi_perf_domain_attributes_get(const struct scmi_handle *handle, u32 domain,
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_perf_domain_attributes *attr;
 
-	ret = scmi_one_xfer_init(handle, PERF_DOMAIN_ATTRIBUTES,
+	ret = scmi_xfer_get_init(handle, PERF_DOMAIN_ATTRIBUTES,
 				 SCMI_PROTOCOL_PERF, sizeof(domain),
 				 sizeof(*attr), &t);
 	if (ret)
@@ -171,7 +171,7 @@ scmi_perf_domain_attributes_get(const struct scmi_handle *handle, u32 domain,
 		memcpy(dom_info->name, attr->name, SCMI_MAX_STR_SIZE);
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -194,7 +194,7 @@ scmi_perf_describe_levels_get(const struct scmi_handle *handle, u32 domain,
 	struct scmi_msg_perf_describe_levels *dom_info;
 	struct scmi_msg_resp_perf_describe_levels *level_info;
 
-	ret = scmi_one_xfer_init(handle, PERF_DESCRIBE_LEVELS,
+	ret = scmi_xfer_get_init(handle, PERF_DESCRIBE_LEVELS,
 				 SCMI_PROTOCOL_PERF, sizeof(*dom_info), 0, &t);
 	if (ret)
 		return ret;
@@ -237,7 +237,7 @@ scmi_perf_describe_levels_get(const struct scmi_handle *handle, u32 domain,
 	} while (num_returned && num_remaining);
 
 	perf_dom->opp_count = tot_opp_cnt;
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 
 	sort(perf_dom->opp, tot_opp_cnt, sizeof(*opp), opp_cmp_func, NULL);
 	return ret;
@@ -250,7 +250,7 @@ static int scmi_perf_limits_set(const struct scmi_handle *handle, u32 domain,
 	struct scmi_xfer *t;
 	struct scmi_perf_set_limits *limits;
 
-	ret = scmi_one_xfer_init(handle, PERF_LIMITS_SET, SCMI_PROTOCOL_PERF,
+	ret = scmi_xfer_get_init(handle, PERF_LIMITS_SET, SCMI_PROTOCOL_PERF,
 				 sizeof(*limits), 0, &t);
 	if (ret)
 		return ret;
@@ -262,7 +262,7 @@ static int scmi_perf_limits_set(const struct scmi_handle *handle, u32 domain,
 
 	ret = scmi_do_xfer(handle, t);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -273,7 +273,7 @@ static int scmi_perf_limits_get(const struct scmi_handle *handle, u32 domain,
 	struct scmi_xfer *t;
 	struct scmi_perf_get_limits *limits;
 
-	ret = scmi_one_xfer_init(handle, PERF_LIMITS_GET, SCMI_PROTOCOL_PERF,
+	ret = scmi_xfer_get_init(handle, PERF_LIMITS_GET, SCMI_PROTOCOL_PERF,
 				 sizeof(__le32), 0, &t);
 	if (ret)
 		return ret;
@@ -288,7 +288,7 @@ static int scmi_perf_limits_get(const struct scmi_handle *handle, u32 domain,
 		*min_perf = le32_to_cpu(limits->min_level);
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -299,7 +299,7 @@ static int scmi_perf_level_set(const struct scmi_handle *handle, u32 domain,
 	struct scmi_xfer *t;
 	struct scmi_perf_set_level *lvl;
 
-	ret = scmi_one_xfer_init(handle, PERF_LEVEL_SET, SCMI_PROTOCOL_PERF,
+	ret = scmi_xfer_get_init(handle, PERF_LEVEL_SET, SCMI_PROTOCOL_PERF,
 				 sizeof(*lvl), 0, &t);
 	if (ret)
 		return ret;
@@ -311,7 +311,7 @@ static int scmi_perf_level_set(const struct scmi_handle *handle, u32 domain,
 
 	ret = scmi_do_xfer(handle, t);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -321,7 +321,7 @@ static int scmi_perf_level_get(const struct scmi_handle *handle, u32 domain,
 	int ret;
 	struct scmi_xfer *t;
 
-	ret = scmi_one_xfer_init(handle, PERF_LEVEL_GET, SCMI_PROTOCOL_PERF,
+	ret = scmi_xfer_get_init(handle, PERF_LEVEL_GET, SCMI_PROTOCOL_PERF,
 				 sizeof(u32), sizeof(u32), &t);
 	if (ret)
 		return ret;
@@ -333,7 +333,7 @@ static int scmi_perf_level_get(const struct scmi_handle *handle, u32 domain,
 	if (!ret)
 		*level = le32_to_cpu(*(__le32 *)t->rx.buf);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -349,8 +349,8 @@ static int scmi_dev_domain_id(struct device *dev)
 	return clkspec.args[0];
 }
 
-static int scmi_dvfs_add_opps_to_device(const struct scmi_handle *handle,
-					struct device *dev)
+static int scmi_dvfs_device_opps_add(const struct scmi_handle *handle,
+				     struct device *dev)
 {
 	int idx, ret, domain;
 	unsigned long freq;
@@ -383,7 +383,7 @@ static int scmi_dvfs_add_opps_to_device(const struct scmi_handle *handle,
 	return 0;
 }
 
-static int scmi_dvfs_get_transition_latency(const struct scmi_handle *handle,
+static int scmi_dvfs_transition_latency_get(const struct scmi_handle *handle,
 					    struct device *dev)
 {
 	struct perf_dom_info *dom;
@@ -432,8 +432,8 @@ static struct scmi_perf_ops perf_ops = {
 	.level_set = scmi_perf_level_set,
 	.level_get = scmi_perf_level_get,
 	.device_domain_id = scmi_dev_domain_id,
-	.get_transition_latency = scmi_dvfs_get_transition_latency,
-	.add_opps_to_device = scmi_dvfs_add_opps_to_device,
+	.transition_latency_get = scmi_dvfs_transition_latency_get,
+	.device_opps_add = scmi_dvfs_device_opps_add,
 	.freq_set = scmi_dvfs_freq_set,
 	.freq_get = scmi_dvfs_freq_get,
 };
diff --git a/drivers/firmware/arm_scmi/power.c b/drivers/firmware/arm_scmi/power.c
index 087c2876cdf2..cfa033b05aed 100644
--- a/drivers/firmware/arm_scmi/power.c
+++ b/drivers/firmware/arm_scmi/power.c
@@ -63,7 +63,7 @@ static int scmi_power_attributes_get(const struct scmi_handle *handle,
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_power_attributes *attr;
 
-	ret = scmi_one_xfer_init(handle, PROTOCOL_ATTRIBUTES,
+	ret = scmi_xfer_get_init(handle, PROTOCOL_ATTRIBUTES,
 				 SCMI_PROTOCOL_POWER, 0, sizeof(*attr), &t);
 	if (ret)
 		return ret;
@@ -78,7 +78,7 @@ static int scmi_power_attributes_get(const struct scmi_handle *handle,
 		pi->stats_size = le32_to_cpu(attr->stats_size);
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -90,7 +90,7 @@ scmi_power_domain_attributes_get(const struct scmi_handle *handle, u32 domain,
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_power_domain_attributes *attr;
 
-	ret = scmi_one_xfer_init(handle, POWER_DOMAIN_ATTRIBUTES,
+	ret = scmi_xfer_get_init(handle, POWER_DOMAIN_ATTRIBUTES,
 				 SCMI_PROTOCOL_POWER, sizeof(domain),
 				 sizeof(*attr), &t);
 	if (ret)
@@ -109,7 +109,7 @@ scmi_power_domain_attributes_get(const struct scmi_handle *handle, u32 domain,
 		memcpy(dom_info->name, attr->name, SCMI_MAX_STR_SIZE);
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -120,7 +120,7 @@ scmi_power_state_set(const struct scmi_handle *handle, u32 domain, u32 state)
 	struct scmi_xfer *t;
 	struct scmi_power_set_state *st;
 
-	ret = scmi_one_xfer_init(handle, POWER_STATE_SET, SCMI_PROTOCOL_POWER,
+	ret = scmi_xfer_get_init(handle, POWER_STATE_SET, SCMI_PROTOCOL_POWER,
 				 sizeof(*st), 0, &t);
 	if (ret)
 		return ret;
@@ -132,7 +132,7 @@ scmi_power_state_set(const struct scmi_handle *handle, u32 domain, u32 state)
 
 	ret = scmi_do_xfer(handle, t);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -142,7 +142,7 @@ scmi_power_state_get(const struct scmi_handle *handle, u32 domain, u32 *state)
 	int ret;
 	struct scmi_xfer *t;
 
-	ret = scmi_one_xfer_init(handle, POWER_STATE_GET, SCMI_PROTOCOL_POWER,
+	ret = scmi_xfer_get_init(handle, POWER_STATE_GET, SCMI_PROTOCOL_POWER,
 				 sizeof(u32), sizeof(u32), &t);
 	if (ret)
 		return ret;
@@ -153,7 +153,7 @@ scmi_power_state_get(const struct scmi_handle *handle, u32 domain, u32 *state)
 	if (!ret)
 		*state = le32_to_cpu(*(__le32 *)t->rx.buf);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
diff --git a/drivers/firmware/arm_scmi/sensors.c b/drivers/firmware/arm_scmi/sensors.c
index bbb469fea0ed..27f2092b9882 100644
--- a/drivers/firmware/arm_scmi/sensors.c
+++ b/drivers/firmware/arm_scmi/sensors.c
@@ -79,7 +79,7 @@ static int scmi_sensor_attributes_get(const struct scmi_handle *handle,
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_sensor_attributes *attr;
 
-	ret = scmi_one_xfer_init(handle, PROTOCOL_ATTRIBUTES,
+	ret = scmi_xfer_get_init(handle, PROTOCOL_ATTRIBUTES,
 				 SCMI_PROTOCOL_SENSOR, 0, sizeof(*attr), &t);
 	if (ret)
 		return ret;
@@ -95,7 +95,7 @@ static int scmi_sensor_attributes_get(const struct scmi_handle *handle,
 		si->reg_size = le32_to_cpu(attr->reg_size);
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -108,7 +108,7 @@ static int scmi_sensor_description_get(const struct scmi_handle *handle,
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_sensor_description *buf;
 
-	ret = scmi_one_xfer_init(handle, SENSOR_DESCRIPTION_GET,
+	ret = scmi_xfer_get_init(handle, SENSOR_DESCRIPTION_GET,
 				 SCMI_PROTOCOL_SENSOR, sizeof(__le32), 0, &t);
 	if (ret)
 		return ret;
@@ -150,7 +150,7 @@ static int scmi_sensor_description_get(const struct scmi_handle *handle,
 		 */
 	} while (num_returned && num_remaining);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -162,7 +162,7 @@ scmi_sensor_configuration_set(const struct scmi_handle *handle, u32 sensor_id)
 	struct scmi_xfer *t;
 	struct scmi_msg_set_sensor_config *cfg;
 
-	ret = scmi_one_xfer_init(handle, SENSOR_CONFIG_SET,
+	ret = scmi_xfer_get_init(handle, SENSOR_CONFIG_SET,
 				 SCMI_PROTOCOL_SENSOR, sizeof(*cfg), 0, &t);
 	if (ret)
 		return ret;
@@ -173,7 +173,7 @@ scmi_sensor_configuration_set(const struct scmi_handle *handle, u32 sensor_id)
 
 	ret = scmi_do_xfer(handle, t);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -185,7 +185,7 @@ static int scmi_sensor_trip_point_set(const struct scmi_handle *handle,
 	struct scmi_xfer *t;
 	struct scmi_msg_set_sensor_trip_point *trip;
 
-	ret = scmi_one_xfer_init(handle, SENSOR_TRIP_POINT_SET,
+	ret = scmi_xfer_get_init(handle, SENSOR_TRIP_POINT_SET,
 				 SCMI_PROTOCOL_SENSOR, sizeof(*trip), 0, &t);
 	if (ret)
 		return ret;
@@ -198,7 +198,7 @@ static int scmi_sensor_trip_point_set(const struct scmi_handle *handle,
 
 	ret = scmi_do_xfer(handle, t);
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
@@ -209,7 +209,7 @@ static int scmi_sensor_reading_get(const struct scmi_handle *handle,
 	struct scmi_xfer *t;
 	struct scmi_msg_sensor_reading_get *sensor;
 
-	ret = scmi_one_xfer_init(handle, SENSOR_READING_GET,
+	ret = scmi_xfer_get_init(handle, SENSOR_READING_GET,
 				 SCMI_PROTOCOL_SENSOR, sizeof(*sensor),
 				 sizeof(u64), &t);
 	if (ret)
@@ -227,7 +227,7 @@ static int scmi_sensor_reading_get(const struct scmi_handle *handle,
 		*value |= (u64)le32_to_cpu(*(pval + 1)) << 32;
 	}
 
-	scmi_one_xfer_put(handle, t);
+	scmi_xfer_put(handle, t);
 	return ret;
 }
 
diff --git a/drivers/firmware/broadcom/bcm47xx_nvram.c b/drivers/firmware/broadcom/bcm47xx_nvram.c
index 0b631e5b5b84..d25f080fcb0d 100644
--- a/drivers/firmware/broadcom/bcm47xx_nvram.c
+++ b/drivers/firmware/broadcom/bcm47xx_nvram.c
@@ -36,7 +36,7 @@ struct nvram_header {
 
 static char nvram_buf[NVRAM_SPACE];
 static size_t nvram_len;
-static const u32 nvram_sizes[] = {0x8000, 0xF000, 0x10000};
+static const u32 nvram_sizes[] = {0x6000, 0x8000, 0xF000, 0x10000};
 
 static u32 find_nvram_size(void __iomem *end)
 {
diff --git a/drivers/firmware/dmi-id.c b/drivers/firmware/dmi-id.c
index 951b6c79f166..624a11cb07e2 100644
--- a/drivers/firmware/dmi-id.c
+++ b/drivers/firmware/dmi-id.c
@@ -47,6 +47,7 @@ DEFINE_DMI_ATTR_WITH_SHOW(product_name,		0444, DMI_PRODUCT_NAME);
 DEFINE_DMI_ATTR_WITH_SHOW(product_version,	0444, DMI_PRODUCT_VERSION);
 DEFINE_DMI_ATTR_WITH_SHOW(product_serial,	0400, DMI_PRODUCT_SERIAL);
 DEFINE_DMI_ATTR_WITH_SHOW(product_uuid,		0400, DMI_PRODUCT_UUID);
+DEFINE_DMI_ATTR_WITH_SHOW(product_sku,		0444, DMI_PRODUCT_SKU);
 DEFINE_DMI_ATTR_WITH_SHOW(product_family,	0444, DMI_PRODUCT_FAMILY);
 DEFINE_DMI_ATTR_WITH_SHOW(board_vendor,		0444, DMI_BOARD_VENDOR);
 DEFINE_DMI_ATTR_WITH_SHOW(board_name,		0444, DMI_BOARD_NAME);
@@ -193,6 +194,7 @@ static void __init dmi_id_init_attr_table(void)
 	ADD_DMI_ATTR(product_serial,    DMI_PRODUCT_SERIAL);
 	ADD_DMI_ATTR(product_uuid,      DMI_PRODUCT_UUID);
 	ADD_DMI_ATTR(product_family,    DMI_PRODUCT_FAMILY);
+	ADD_DMI_ATTR(product_sku,       DMI_PRODUCT_SKU);
 	ADD_DMI_ATTR(board_vendor,      DMI_BOARD_VENDOR);
 	ADD_DMI_ATTR(board_name,        DMI_BOARD_NAME);
 	ADD_DMI_ATTR(board_version,     DMI_BOARD_VERSION);
diff --git a/drivers/firmware/dmi_scan.c b/drivers/firmware/dmi_scan.c
index 54e66adef252..f2483548cde9 100644
--- a/drivers/firmware/dmi_scan.c
+++ b/drivers/firmware/dmi_scan.c
@@ -447,6 +447,7 @@ static void __init dmi_decode(const struct dmi_header *dm, void *dummy)
 		dmi_save_ident(dm, DMI_PRODUCT_VERSION, 6);
 		dmi_save_ident(dm, DMI_PRODUCT_SERIAL, 7);
 		dmi_save_uuid(dm, DMI_PRODUCT_UUID, 8);
+		dmi_save_ident(dm, DMI_PRODUCT_SKU, 25);
 		dmi_save_ident(dm, DMI_PRODUCT_FAMILY, 26);
 		break;
 	case 2:		/* Base Board Information */
diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig
index 3098410abad8..781a4a337557 100644
--- a/drivers/firmware/efi/Kconfig
+++ b/drivers/firmware/efi/Kconfig
@@ -174,6 +174,11 @@ config UEFI_CPER_ARM
 	depends on UEFI_CPER && ( ARM || ARM64 )
 	default y
 
+config UEFI_CPER_X86
+	bool
+	depends on UEFI_CPER && X86
+	default y
+
 config EFI_DEV_PATH_PARSER
 	bool
 	depends on ACPI
diff --git a/drivers/firmware/efi/Makefile b/drivers/firmware/efi/Makefile
index cb805374f4bc..5f9f5039de50 100644
--- a/drivers/firmware/efi/Makefile
+++ b/drivers/firmware/efi/Makefile
@@ -31,3 +31,4 @@ obj-$(CONFIG_ARM)			+= $(arm-obj-y)
 obj-$(CONFIG_ARM64)			+= $(arm-obj-y)
 obj-$(CONFIG_EFI_CAPSULE_LOADER)	+= capsule-loader.o
 obj-$(CONFIG_UEFI_CPER_ARM)		+= cper-arm.o
+obj-$(CONFIG_UEFI_CPER_X86)		+= cper-x86.o
diff --git a/drivers/firmware/efi/apple-properties.c b/drivers/firmware/efi/apple-properties.c
index adaa9a3714b9..60a95719ecb8 100644
--- a/drivers/firmware/efi/apple-properties.c
+++ b/drivers/firmware/efi/apple-properties.c
@@ -13,6 +13,9 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Note, all properties are considered as u8 arrays.
+ * To get a value of any of them the caller must use device_property_read_u8_array().
  */
 
 #define pr_fmt(fmt) "apple-properties: " fmt
@@ -96,12 +99,13 @@ static void __init unmarshal_key_value_pairs(struct dev_header *dev_header,
 		entry[i].name = key;
 		entry[i].length = val_len - sizeof(val_len);
 		entry[i].is_array = !!entry[i].length;
-		entry[i].pointer.raw_data = ptr + key_len + sizeof(val_len);
+		entry[i].type = DEV_PROP_U8;
+		entry[i].pointer.u8_data = ptr + key_len + sizeof(val_len);
 
 		if (dump_properties) {
 			dev_info(dev, "property: %s\n", entry[i].name);
 			print_hex_dump(KERN_INFO, pr_fmt(), DUMP_PREFIX_OFFSET,
-				16, 1, entry[i].pointer.raw_data,
+				16, 1, entry[i].pointer.u8_data,
 				entry[i].length, true);
 		}
 
diff --git a/drivers/firmware/efi/arm-init.c b/drivers/firmware/efi/arm-init.c
index 80d1a885def5..b5214c143fee 100644
--- a/drivers/firmware/efi/arm-init.c
+++ b/drivers/firmware/efi/arm-init.c
@@ -193,7 +193,7 @@ static __init void reserve_regions(void)
 	 * uses its own memory map instead.
 	 */
 	memblock_dump_all();
-	memblock_remove(0, (phys_addr_t)ULLONG_MAX);
+	memblock_remove(0, PHYS_ADDR_MAX);
 
 	for_each_efi_memory_desc(md) {
 		paddr = md->phys_addr;
diff --git a/drivers/firmware/efi/capsule-loader.c b/drivers/firmware/efi/capsule-loader.c
index e456f4602df1..96688986da56 100644
--- a/drivers/firmware/efi/capsule-loader.c
+++ b/drivers/firmware/efi/capsule-loader.c
@@ -134,10 +134,16 @@ static ssize_t efi_capsule_submit_update(struct capsule_info *cap_info)
 
 	/* Indicate capsule binary uploading is done */
 	cap_info->index = NO_FURTHER_WRITE_ACTION;
-	pr_info("Successfully upload capsule file with reboot type '%s'\n",
-		!cap_info->reset_type ? "RESET_COLD" :
-		cap_info->reset_type == 1 ? "RESET_WARM" :
-		"RESET_SHUTDOWN");
+
+	if (cap_info->header.flags & EFI_CAPSULE_PERSIST_ACROSS_RESET) {
+		pr_info("Successfully uploaded capsule file with reboot type '%s'\n",
+			!cap_info->reset_type ? "RESET_COLD" :
+			cap_info->reset_type == 1 ? "RESET_WARM" :
+			"RESET_SHUTDOWN");
+	} else {
+		pr_info("Successfully processed capsule file\n");
+	}
+
 	return 0;
 }
 
diff --git a/drivers/firmware/efi/cper-arm.c b/drivers/firmware/efi/cper-arm.c
index 698e5c8e0c8d..502811344e81 100644
--- a/drivers/firmware/efi/cper-arm.c
+++ b/drivers/firmware/efi/cper-arm.c
@@ -30,8 +30,6 @@
 #include <acpi/ghes.h>
 #include <ras/ras_event.h>
 
-#define INDENT_SP	" "
-
 static const char * const arm_reg_ctx_strs[] = {
 	"AArch32 general purpose registers",
 	"AArch32 EL1 context registers",
@@ -283,7 +281,7 @@ void cper_print_proc_arm(const char *pfx,
 			pfx, proc->psci_state);
 	}
 
-	snprintf(newpfx, sizeof(newpfx), "%s%s", pfx, INDENT_SP);
+	snprintf(newpfx, sizeof(newpfx), "%s ", pfx);
 
 	err_info = (struct cper_arm_err_info *)(proc + 1);
 	for (i = 0; i < proc->err_info_num; i++) {
@@ -310,7 +308,7 @@ void cper_print_proc_arm(const char *pfx,
 		if (err_info->validation_bits & CPER_ARM_INFO_VALID_ERR_INFO) {
 			printk("%serror_info: 0x%016llx\n", newpfx,
 			       err_info->error_info);
-			snprintf(infopfx, sizeof(infopfx), "%s%s", newpfx, INDENT_SP);
+			snprintf(infopfx, sizeof(infopfx), "%s ", newpfx);
 			cper_print_arm_err_info(infopfx, err_info->type,
 						err_info->error_info);
 		}
diff --git a/drivers/firmware/efi/cper-x86.c b/drivers/firmware/efi/cper-x86.c
new file mode 100644
index 000000000000..2531de49f56c
--- /dev/null
+++ b/drivers/firmware/efi/cper-x86.c
@@ -0,0 +1,356 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2018, Advanced Micro Devices, Inc.
+
+#include <linux/cper.h>
+
+/*
+ * We don't need a "CPER_IA" prefix since these are all locally defined.
+ * This will save us a lot of line space.
+ */
+#define VALID_LAPIC_ID			BIT_ULL(0)
+#define VALID_CPUID_INFO		BIT_ULL(1)
+#define VALID_PROC_ERR_INFO_NUM(bits)	(((bits) & GENMASK_ULL(7, 2)) >> 2)
+#define VALID_PROC_CXT_INFO_NUM(bits)	(((bits) & GENMASK_ULL(13, 8)) >> 8)
+
+#define INFO_ERR_STRUCT_TYPE_CACHE					\
+	GUID_INIT(0xA55701F5, 0xE3EF, 0x43DE, 0xAC, 0x72, 0x24, 0x9B,	\
+		  0x57, 0x3F, 0xAD, 0x2C)
+#define INFO_ERR_STRUCT_TYPE_TLB					\
+	GUID_INIT(0xFC06B535, 0x5E1F, 0x4562, 0x9F, 0x25, 0x0A, 0x3B,	\
+		  0x9A, 0xDB, 0x63, 0xC3)
+#define INFO_ERR_STRUCT_TYPE_BUS					\
+	GUID_INIT(0x1CF3F8B3, 0xC5B1, 0x49a2, 0xAA, 0x59, 0x5E, 0xEF,	\
+		  0x92, 0xFF, 0xA6, 0x3C)
+#define INFO_ERR_STRUCT_TYPE_MS						\
+	GUID_INIT(0x48AB7F57, 0xDC34, 0x4f6c, 0xA7, 0xD3, 0xB0, 0xB5,	\
+		  0xB0, 0xA7, 0x43, 0x14)
+
+#define INFO_VALID_CHECK_INFO		BIT_ULL(0)
+#define INFO_VALID_TARGET_ID		BIT_ULL(1)
+#define INFO_VALID_REQUESTOR_ID		BIT_ULL(2)
+#define INFO_VALID_RESPONDER_ID		BIT_ULL(3)
+#define INFO_VALID_IP			BIT_ULL(4)
+
+#define CHECK_VALID_TRANS_TYPE		BIT_ULL(0)
+#define CHECK_VALID_OPERATION		BIT_ULL(1)
+#define CHECK_VALID_LEVEL		BIT_ULL(2)
+#define CHECK_VALID_PCC			BIT_ULL(3)
+#define CHECK_VALID_UNCORRECTED		BIT_ULL(4)
+#define CHECK_VALID_PRECISE_IP		BIT_ULL(5)
+#define CHECK_VALID_RESTARTABLE_IP	BIT_ULL(6)
+#define CHECK_VALID_OVERFLOW		BIT_ULL(7)
+
+#define CHECK_VALID_BUS_PART_TYPE	BIT_ULL(8)
+#define CHECK_VALID_BUS_TIME_OUT	BIT_ULL(9)
+#define CHECK_VALID_BUS_ADDR_SPACE	BIT_ULL(10)
+
+#define CHECK_VALID_BITS(check)		(((check) & GENMASK_ULL(15, 0)))
+#define CHECK_TRANS_TYPE(check)		(((check) & GENMASK_ULL(17, 16)) >> 16)
+#define CHECK_OPERATION(check)		(((check) & GENMASK_ULL(21, 18)) >> 18)
+#define CHECK_LEVEL(check)		(((check) & GENMASK_ULL(24, 22)) >> 22)
+#define CHECK_PCC			BIT_ULL(25)
+#define CHECK_UNCORRECTED		BIT_ULL(26)
+#define CHECK_PRECISE_IP		BIT_ULL(27)
+#define CHECK_RESTARTABLE_IP		BIT_ULL(28)
+#define CHECK_OVERFLOW			BIT_ULL(29)
+
+#define CHECK_BUS_PART_TYPE(check)	(((check) & GENMASK_ULL(31, 30)) >> 30)
+#define CHECK_BUS_TIME_OUT		BIT_ULL(32)
+#define CHECK_BUS_ADDR_SPACE(check)	(((check) & GENMASK_ULL(34, 33)) >> 33)
+
+#define CHECK_VALID_MS_ERR_TYPE		BIT_ULL(0)
+#define CHECK_VALID_MS_PCC		BIT_ULL(1)
+#define CHECK_VALID_MS_UNCORRECTED	BIT_ULL(2)
+#define CHECK_VALID_MS_PRECISE_IP	BIT_ULL(3)
+#define CHECK_VALID_MS_RESTARTABLE_IP	BIT_ULL(4)
+#define CHECK_VALID_MS_OVERFLOW		BIT_ULL(5)
+
+#define CHECK_MS_ERR_TYPE(check)	(((check) & GENMASK_ULL(18, 16)) >> 16)
+#define CHECK_MS_PCC			BIT_ULL(19)
+#define CHECK_MS_UNCORRECTED		BIT_ULL(20)
+#define CHECK_MS_PRECISE_IP		BIT_ULL(21)
+#define CHECK_MS_RESTARTABLE_IP		BIT_ULL(22)
+#define CHECK_MS_OVERFLOW		BIT_ULL(23)
+
+#define CTX_TYPE_MSR			1
+#define CTX_TYPE_MMREG			7
+
+enum err_types {
+	ERR_TYPE_CACHE = 0,
+	ERR_TYPE_TLB,
+	ERR_TYPE_BUS,
+	ERR_TYPE_MS,
+	N_ERR_TYPES
+};
+
+static enum err_types cper_get_err_type(const guid_t *err_type)
+{
+	if (guid_equal(err_type, &INFO_ERR_STRUCT_TYPE_CACHE))
+		return ERR_TYPE_CACHE;
+	else if (guid_equal(err_type, &INFO_ERR_STRUCT_TYPE_TLB))
+		return ERR_TYPE_TLB;
+	else if (guid_equal(err_type, &INFO_ERR_STRUCT_TYPE_BUS))
+		return ERR_TYPE_BUS;
+	else if (guid_equal(err_type, &INFO_ERR_STRUCT_TYPE_MS))
+		return ERR_TYPE_MS;
+	else
+		return N_ERR_TYPES;
+}
+
+static const char * const ia_check_trans_type_strs[] = {
+	"Instruction",
+	"Data Access",
+	"Generic",
+};
+
+static const char * const ia_check_op_strs[] = {
+	"generic error",
+	"generic read",
+	"generic write",
+	"data read",
+	"data write",
+	"instruction fetch",
+	"prefetch",
+	"eviction",
+	"snoop",
+};
+
+static const char * const ia_check_bus_part_type_strs[] = {
+	"Local Processor originated request",
+	"Local Processor responded to request",
+	"Local Processor observed",
+	"Generic",
+};
+
+static const char * const ia_check_bus_addr_space_strs[] = {
+	"Memory Access",
+	"Reserved",
+	"I/O",
+	"Other Transaction",
+};
+
+static const char * const ia_check_ms_error_type_strs[] = {
+	"No Error",
+	"Unclassified",
+	"Microcode ROM Parity Error",
+	"External Error",
+	"FRC Error",
+	"Internal Unclassified",
+};
+
+static const char * const ia_reg_ctx_strs[] = {
+	"Unclassified Data",
+	"MSR Registers (Machine Check and other MSRs)",
+	"32-bit Mode Execution Context",
+	"64-bit Mode Execution Context",
+	"FXSAVE Context",
+	"32-bit Mode Debug Registers (DR0-DR7)",
+	"64-bit Mode Debug Registers (DR0-DR7)",
+	"Memory Mapped Registers",
+};
+
+static inline void print_bool(char *str, const char *pfx, u64 check, u64 bit)
+{
+	printk("%s%s: %s\n", pfx, str, (check & bit) ? "true" : "false");
+}
+
+static void print_err_info_ms(const char *pfx, u16 validation_bits, u64 check)
+{
+	if (validation_bits & CHECK_VALID_MS_ERR_TYPE) {
+		u8 err_type = CHECK_MS_ERR_TYPE(check);
+
+		printk("%sError Type: %u, %s\n", pfx, err_type,
+		       err_type < ARRAY_SIZE(ia_check_ms_error_type_strs) ?
+		       ia_check_ms_error_type_strs[err_type] : "unknown");
+	}
+
+	if (validation_bits & CHECK_VALID_MS_PCC)
+		print_bool("Processor Context Corrupt", pfx, check, CHECK_MS_PCC);
+
+	if (validation_bits & CHECK_VALID_MS_UNCORRECTED)
+		print_bool("Uncorrected", pfx, check, CHECK_MS_UNCORRECTED);
+
+	if (validation_bits & CHECK_VALID_MS_PRECISE_IP)
+		print_bool("Precise IP", pfx, check, CHECK_MS_PRECISE_IP);
+
+	if (validation_bits & CHECK_VALID_MS_RESTARTABLE_IP)
+		print_bool("Restartable IP", pfx, check, CHECK_MS_RESTARTABLE_IP);
+
+	if (validation_bits & CHECK_VALID_MS_OVERFLOW)
+		print_bool("Overflow", pfx, check, CHECK_MS_OVERFLOW);
+}
+
+static void print_err_info(const char *pfx, u8 err_type, u64 check)
+{
+	u16 validation_bits = CHECK_VALID_BITS(check);
+
+	/*
+	 * The MS Check structure varies a lot from the others, so use a
+	 * separate function for decoding.
+	 */
+	if (err_type == ERR_TYPE_MS)
+		return print_err_info_ms(pfx, validation_bits, check);
+
+	if (validation_bits & CHECK_VALID_TRANS_TYPE) {
+		u8 trans_type = CHECK_TRANS_TYPE(check);
+
+		printk("%sTransaction Type: %u, %s\n", pfx, trans_type,
+		       trans_type < ARRAY_SIZE(ia_check_trans_type_strs) ?
+		       ia_check_trans_type_strs[trans_type] : "unknown");
+	}
+
+	if (validation_bits & CHECK_VALID_OPERATION) {
+		u8 op = CHECK_OPERATION(check);
+
+		/*
+		 * CACHE has more operation types than TLB or BUS, though the
+		 * name and the order are the same.
+		 */
+		u8 max_ops = (err_type == ERR_TYPE_CACHE) ? 9 : 7;
+
+		printk("%sOperation: %u, %s\n", pfx, op,
+		       op < max_ops ? ia_check_op_strs[op] : "unknown");
+	}
+
+	if (validation_bits & CHECK_VALID_LEVEL)
+		printk("%sLevel: %llu\n", pfx, CHECK_LEVEL(check));
+
+	if (validation_bits & CHECK_VALID_PCC)
+		print_bool("Processor Context Corrupt", pfx, check, CHECK_PCC);
+
+	if (validation_bits & CHECK_VALID_UNCORRECTED)
+		print_bool("Uncorrected", pfx, check, CHECK_UNCORRECTED);
+
+	if (validation_bits & CHECK_VALID_PRECISE_IP)
+		print_bool("Precise IP", pfx, check, CHECK_PRECISE_IP);
+
+	if (validation_bits & CHECK_VALID_RESTARTABLE_IP)
+		print_bool("Restartable IP", pfx, check, CHECK_RESTARTABLE_IP);
+
+	if (validation_bits & CHECK_VALID_OVERFLOW)
+		print_bool("Overflow", pfx, check, CHECK_OVERFLOW);
+
+	if (err_type != ERR_TYPE_BUS)
+		return;
+
+	if (validation_bits & CHECK_VALID_BUS_PART_TYPE) {
+		u8 part_type = CHECK_BUS_PART_TYPE(check);
+
+		printk("%sParticipation Type: %u, %s\n", pfx, part_type,
+		       part_type < ARRAY_SIZE(ia_check_bus_part_type_strs) ?
+		       ia_check_bus_part_type_strs[part_type] : "unknown");
+	}
+
+	if (validation_bits & CHECK_VALID_BUS_TIME_OUT)
+		print_bool("Time Out", pfx, check, CHECK_BUS_TIME_OUT);
+
+	if (validation_bits & CHECK_VALID_BUS_ADDR_SPACE) {
+		u8 addr_space = CHECK_BUS_ADDR_SPACE(check);
+
+		printk("%sAddress Space: %u, %s\n", pfx, addr_space,
+		       addr_space < ARRAY_SIZE(ia_check_bus_addr_space_strs) ?
+		       ia_check_bus_addr_space_strs[addr_space] : "unknown");
+	}
+}
+
+void cper_print_proc_ia(const char *pfx, const struct cper_sec_proc_ia *proc)
+{
+	int i;
+	struct cper_ia_err_info *err_info;
+	struct cper_ia_proc_ctx *ctx_info;
+	char newpfx[64], infopfx[64];
+	u8 err_type;
+
+	if (proc->validation_bits & VALID_LAPIC_ID)
+		printk("%sLocal APIC_ID: 0x%llx\n", pfx, proc->lapic_id);
+
+	if (proc->validation_bits & VALID_CPUID_INFO) {
+		printk("%sCPUID Info:\n", pfx);
+		print_hex_dump(pfx, "", DUMP_PREFIX_OFFSET, 16, 4, proc->cpuid,
+			       sizeof(proc->cpuid), 0);
+	}
+
+	snprintf(newpfx, sizeof(newpfx), "%s ", pfx);
+
+	err_info = (struct cper_ia_err_info *)(proc + 1);
+	for (i = 0; i < VALID_PROC_ERR_INFO_NUM(proc->validation_bits); i++) {
+		printk("%sError Information Structure %d:\n", pfx, i);
+
+		err_type = cper_get_err_type(&err_info->err_type);
+		printk("%sError Structure Type: %s\n", newpfx,
+		       err_type < ARRAY_SIZE(cper_proc_error_type_strs) ?
+		       cper_proc_error_type_strs[err_type] : "unknown");
+
+		if (err_type >= N_ERR_TYPES) {
+			printk("%sError Structure Type: %pUl\n", newpfx,
+			       &err_info->err_type);
+		}
+
+		if (err_info->validation_bits & INFO_VALID_CHECK_INFO) {
+			printk("%sCheck Information: 0x%016llx\n", newpfx,
+			       err_info->check_info);
+
+			if (err_type < N_ERR_TYPES) {
+				snprintf(infopfx, sizeof(infopfx), "%s ",
+					 newpfx);
+
+				print_err_info(infopfx, err_type,
+					       err_info->check_info);
+			}
+		}
+
+		if (err_info->validation_bits & INFO_VALID_TARGET_ID) {
+			printk("%sTarget Identifier: 0x%016llx\n",
+			       newpfx, err_info->target_id);
+		}
+
+		if (err_info->validation_bits & INFO_VALID_REQUESTOR_ID) {
+			printk("%sRequestor Identifier: 0x%016llx\n",
+			       newpfx, err_info->requestor_id);
+		}
+
+		if (err_info->validation_bits & INFO_VALID_RESPONDER_ID) {
+			printk("%sResponder Identifier: 0x%016llx\n",
+			       newpfx, err_info->responder_id);
+		}
+
+		if (err_info->validation_bits & INFO_VALID_IP) {
+			printk("%sInstruction Pointer: 0x%016llx\n",
+			       newpfx, err_info->ip);
+		}
+
+		err_info++;
+	}
+
+	ctx_info = (struct cper_ia_proc_ctx *)err_info;
+	for (i = 0; i < VALID_PROC_CXT_INFO_NUM(proc->validation_bits); i++) {
+		int size = sizeof(*ctx_info) + ctx_info->reg_arr_size;
+		int groupsize = 4;
+
+		printk("%sContext Information Structure %d:\n", pfx, i);
+
+		printk("%sRegister Context Type: %s\n", newpfx,
+		       ctx_info->reg_ctx_type < ARRAY_SIZE(ia_reg_ctx_strs) ?
+		       ia_reg_ctx_strs[ctx_info->reg_ctx_type] : "unknown");
+
+		printk("%sRegister Array Size: 0x%04x\n", newpfx,
+		       ctx_info->reg_arr_size);
+
+		if (ctx_info->reg_ctx_type == CTX_TYPE_MSR) {
+			groupsize = 8; /* MSRs are 8 bytes wide. */
+			printk("%sMSR Address: 0x%08x\n", newpfx,
+			       ctx_info->msr_addr);
+		}
+
+		if (ctx_info->reg_ctx_type == CTX_TYPE_MMREG) {
+			printk("%sMM Register Address: 0x%016llx\n", newpfx,
+			       ctx_info->mm_reg_addr);
+		}
+
+		printk("%sRegister Array:\n", newpfx);
+		print_hex_dump(newpfx, "", DUMP_PREFIX_OFFSET, 16, groupsize,
+			       (ctx_info + 1), ctx_info->reg_arr_size, 0);
+
+		ctx_info = (struct cper_ia_proc_ctx *)((long)ctx_info + size);
+	}
+}
diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c
index c165933ebf38..3bf0dca378a6 100644
--- a/drivers/firmware/efi/cper.c
+++ b/drivers/firmware/efi/cper.c
@@ -37,8 +37,6 @@
 #include <acpi/ghes.h>
 #include <ras/ras_event.h>
 
-#define INDENT_SP	" "
-
 static char rcd_decode_str[CPER_REC_LEN];
 
 /*
@@ -433,7 +431,7 @@ cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata
 	if (gdata->validation_bits & CPER_SEC_VALID_FRU_TEXT)
 		printk("%s""fru_text: %.20s\n", pfx, gdata->fru_text);
 
-	snprintf(newpfx, sizeof(newpfx), "%s%s", pfx, INDENT_SP);
+	snprintf(newpfx, sizeof(newpfx), "%s ", pfx);
 	if (guid_equal(sec_type, &CPER_SEC_PROC_GENERIC)) {
 		struct cper_sec_proc_generic *proc_err = acpi_hest_get_payload(gdata);
 
@@ -469,6 +467,16 @@ cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata
 			cper_print_proc_arm(newpfx, arm_err);
 		else
 			goto err_section_too_small;
+#endif
+#if defined(CONFIG_UEFI_CPER_X86)
+	} else if (guid_equal(sec_type, &CPER_SEC_PROC_IA)) {
+		struct cper_sec_proc_ia *ia_err = acpi_hest_get_payload(gdata);
+
+		printk("%ssection_type: IA32/X64 processor error\n", newpfx);
+		if (gdata->error_data_length >= sizeof(*ia_err))
+			cper_print_proc_ia(newpfx, ia_err);
+		else
+			goto err_section_too_small;
 #endif
 	} else {
 		const void *err = acpi_hest_get_payload(gdata);
@@ -500,7 +508,7 @@ void cper_estatus_print(const char *pfx,
 		       "It has been corrected by h/w "
 		       "and requires no further action");
 	printk("%s""event severity: %s\n", pfx, cper_severity_str(severity));
-	snprintf(newpfx, sizeof(newpfx), "%s%s", pfx, INDENT_SP);
+	snprintf(newpfx, sizeof(newpfx), "%s ", pfx);
 
 	apei_estatus_for_each_section(estatus, gdata) {
 		cper_estatus_print_section(newpfx, gdata, sec_no);
diff --git a/drivers/firmware/efi/libstub/secureboot.c b/drivers/firmware/efi/libstub/secureboot.c
index 8f07eb414c00..72d9dfbebf08 100644
--- a/drivers/firmware/efi/libstub/secureboot.c
+++ b/drivers/firmware/efi/libstub/secureboot.c
@@ -30,6 +30,9 @@ static const efi_char16_t shim_MokSBState_name[] = L"MokSBState";
 
 /*
  * Determine whether we're in secure boot mode.
+ *
+ * Please keep the logic in sync with
+ * arch/x86/xen/efi.c:xen_efi_get_secureboot().
  */
 enum efi_secureboot_mode efi_get_secureboot(efi_system_table_t *sys_table_arg)
 {
diff --git a/drivers/firmware/efi/libstub/tpm.c b/drivers/firmware/efi/libstub/tpm.c
index 9d08cea3f1b0..caa37a6dd9d4 100644
--- a/drivers/firmware/efi/libstub/tpm.c
+++ b/drivers/firmware/efi/libstub/tpm.c
@@ -59,7 +59,7 @@ void efi_enable_reset_attack_mitigation(efi_system_table_t *sys_table_arg)
 
 #endif
 
-void efi_retrieve_tpm2_eventlog_1_2(efi_system_table_t *sys_table_arg)
+static void efi_retrieve_tpm2_eventlog_1_2(efi_system_table_t *sys_table_arg)
 {
 	efi_guid_t tcg2_guid = EFI_TCG2_PROTOCOL_GUID;
 	efi_guid_t linux_eventlog_guid = LINUX_EFI_TPM_EVENT_LOG_GUID;
diff --git a/drivers/firmware/google/Kconfig b/drivers/firmware/google/Kconfig
index f16b381a569c..a456a000048b 100644
--- a/drivers/firmware/google/Kconfig
+++ b/drivers/firmware/google/Kconfig
@@ -55,6 +55,14 @@ config GOOGLE_MEMCONSOLE_X86_LEGACY
 	  the EBDA on Google servers.  If found, this log is exported to
 	  userland in the file /sys/firmware/log.
 
+config GOOGLE_FRAMEBUFFER_COREBOOT
+	tristate "Coreboot Framebuffer"
+	depends on FB_SIMPLE
+	depends on GOOGLE_COREBOOT_TABLE
+	help
+	  This option enables the kernel to search for a framebuffer in
+	  the coreboot table.  If found, it is registered with simplefb.
+
 config GOOGLE_MEMCONSOLE_COREBOOT
 	tristate "Firmware Memory Console"
 	depends on GOOGLE_COREBOOT_TABLE
diff --git a/drivers/firmware/google/Makefile b/drivers/firmware/google/Makefile
index dcd3675efcfc..d0b3fba96194 100644
--- a/drivers/firmware/google/Makefile
+++ b/drivers/firmware/google/Makefile
@@ -4,6 +4,7 @@ obj-$(CONFIG_GOOGLE_SMI)		+= gsmi.o
 obj-$(CONFIG_GOOGLE_COREBOOT_TABLE)        += coreboot_table.o
 obj-$(CONFIG_GOOGLE_COREBOOT_TABLE_ACPI)   += coreboot_table-acpi.o
 obj-$(CONFIG_GOOGLE_COREBOOT_TABLE_OF)     += coreboot_table-of.o
+obj-$(CONFIG_GOOGLE_FRAMEBUFFER_COREBOOT)  += framebuffer-coreboot.o
 obj-$(CONFIG_GOOGLE_MEMCONSOLE)            += memconsole.o
 obj-$(CONFIG_GOOGLE_MEMCONSOLE_COREBOOT)   += memconsole-coreboot.o
 obj-$(CONFIG_GOOGLE_MEMCONSOLE_X86_LEGACY) += memconsole-x86-legacy.o
diff --git a/drivers/firmware/google/coreboot_table-acpi.c b/drivers/firmware/google/coreboot_table-acpi.c
index fb98db2d20e2..77197fe3d42f 100644
--- a/drivers/firmware/google/coreboot_table-acpi.c
+++ b/drivers/firmware/google/coreboot_table-acpi.c
@@ -53,7 +53,7 @@ static int coreboot_table_acpi_probe(struct platform_device *pdev)
 	if (!ptr)
 		return -ENOMEM;
 
-	return coreboot_table_init(ptr);
+	return coreboot_table_init(&pdev->dev, ptr);
 }
 
 static int coreboot_table_acpi_remove(struct platform_device *pdev)
diff --git a/drivers/firmware/google/coreboot_table-of.c b/drivers/firmware/google/coreboot_table-of.c
index 727acdc83e83..f15bf404c579 100644
--- a/drivers/firmware/google/coreboot_table-of.c
+++ b/drivers/firmware/google/coreboot_table-of.c
@@ -34,7 +34,7 @@ static int coreboot_table_of_probe(struct platform_device *pdev)
 	if (!ptr)
 		return -ENOMEM;
 
-	return coreboot_table_init(ptr);
+	return coreboot_table_init(&pdev->dev, ptr);
 }
 
 static int coreboot_table_of_remove(struct platform_device *pdev)
diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c
index 0019d3ec18dd..19db5709ae28 100644
--- a/drivers/firmware/google/coreboot_table.c
+++ b/drivers/firmware/google/coreboot_table.c
@@ -4,6 +4,7 @@
  * Module providing coreboot table access.
  *
  * Copyright 2017 Google Inc.
+ * Copyright 2017 Samuel Holland <samuel@sholland.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License v2.0 as published by
@@ -15,37 +16,96 @@
  * GNU General Public License for more details.
  */
 
+#include <linux/device.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/slab.h>
 
 #include "coreboot_table.h"
 
-struct coreboot_table_entry {
-	u32 tag;
-	u32 size;
-};
+#define CB_DEV(d) container_of(d, struct coreboot_device, dev)
+#define CB_DRV(d) container_of(d, struct coreboot_driver, drv)
 
 static struct coreboot_table_header __iomem *ptr_header;
 
-/*
- * This function parses the coreboot table for an entry that contains the base
- * address of the given entry tag. The coreboot table consists of a header
- * directly followed by a number of small, variable-sized entries, which each
- * contain an identifying tag and their length as the first two fields.
- */
-int coreboot_table_find(int tag, void *data, size_t data_size)
+static int coreboot_bus_match(struct device *dev, struct device_driver *drv)
 {
-	struct coreboot_table_header header;
-	struct coreboot_table_entry entry;
-	void *ptr_entry;
-	int i;
+	struct coreboot_device *device = CB_DEV(dev);
+	struct coreboot_driver *driver = CB_DRV(drv);
 
-	if (!ptr_header)
-		return -EPROBE_DEFER;
+	return device->entry.tag == driver->tag;
+}
 
+static int coreboot_bus_probe(struct device *dev)
+{
+	int ret = -ENODEV;
+	struct coreboot_device *device = CB_DEV(dev);
+	struct coreboot_driver *driver = CB_DRV(dev->driver);
+
+	if (driver->probe)
+		ret = driver->probe(device);
+
+	return ret;
+}
+
+static int coreboot_bus_remove(struct device *dev)
+{
+	int ret = 0;
+	struct coreboot_device *device = CB_DEV(dev);
+	struct coreboot_driver *driver = CB_DRV(dev->driver);
+
+	if (driver->remove)
+		ret = driver->remove(device);
+
+	return ret;
+}
+
+static struct bus_type coreboot_bus_type = {
+	.name		= "coreboot",
+	.match		= coreboot_bus_match,
+	.probe		= coreboot_bus_probe,
+	.remove		= coreboot_bus_remove,
+};
+
+static int __init coreboot_bus_init(void)
+{
+	return bus_register(&coreboot_bus_type);
+}
+module_init(coreboot_bus_init);
+
+static void coreboot_device_release(struct device *dev)
+{
+	struct coreboot_device *device = CB_DEV(dev);
+
+	kfree(device);
+}
+
+int coreboot_driver_register(struct coreboot_driver *driver)
+{
+	driver->drv.bus = &coreboot_bus_type;
+
+	return driver_register(&driver->drv);
+}
+EXPORT_SYMBOL(coreboot_driver_register);
+
+void coreboot_driver_unregister(struct coreboot_driver *driver)
+{
+	driver_unregister(&driver->drv);
+}
+EXPORT_SYMBOL(coreboot_driver_unregister);
+
+int coreboot_table_init(struct device *dev, void __iomem *ptr)
+{
+	int i, ret;
+	void *ptr_entry;
+	struct coreboot_device *device;
+	struct coreboot_table_entry entry;
+	struct coreboot_table_header header;
+
+	ptr_header = ptr;
 	memcpy_fromio(&header, ptr_header, sizeof(header));
 
 	if (strncmp(header.signature, "LBIO", sizeof(header.signature))) {
@@ -54,37 +114,41 @@ int coreboot_table_find(int tag, void *data, size_t data_size)
 	}
 
 	ptr_entry = (void *)ptr_header + header.header_bytes;
-
 	for (i = 0; i < header.table_entries; i++) {
 		memcpy_fromio(&entry, ptr_entry, sizeof(entry));
-		if (entry.tag == tag) {
-			if (data_size < entry.size)
-				return -EINVAL;
 
-			memcpy_fromio(data, ptr_entry, entry.size);
+		device = kzalloc(sizeof(struct device) + entry.size, GFP_KERNEL);
+		if (!device) {
+			ret = -ENOMEM;
+			break;
+		}
+
+		dev_set_name(&device->dev, "coreboot%d", i);
+		device->dev.parent = dev;
+		device->dev.bus = &coreboot_bus_type;
+		device->dev.release = coreboot_device_release;
+		memcpy_fromio(&device->entry, ptr_entry, entry.size);
 
-			return 0;
+		ret = device_register(&device->dev);
+		if (ret) {
+			put_device(&device->dev);
+			break;
 		}
 
 		ptr_entry += entry.size;
 	}
 
-	return -ENOENT;
-}
-EXPORT_SYMBOL(coreboot_table_find);
-
-int coreboot_table_init(void __iomem *ptr)
-{
-	ptr_header = ptr;
-
-	return 0;
+	return ret;
 }
 EXPORT_SYMBOL(coreboot_table_init);
 
 int coreboot_table_exit(void)
 {
-	if (ptr_header)
+	if (ptr_header) {
+		bus_unregister(&coreboot_bus_type);
 		iounmap(ptr_header);
+		ptr_header = NULL;
+	}
 
 	return 0;
 }
diff --git a/drivers/firmware/google/coreboot_table.h b/drivers/firmware/google/coreboot_table.h
index 6eff1ae0c5d3..8ad95a94481b 100644
--- a/drivers/firmware/google/coreboot_table.h
+++ b/drivers/firmware/google/coreboot_table.h
@@ -3,7 +3,9 @@
  *
  * Internal header for coreboot table access.
  *
+ * Copyright 2014 Gerd Hoffmann <kraxel@redhat.com>
  * Copyright 2017 Google Inc.
+ * Copyright 2017 Samuel Holland <samuel@sholland.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License v2.0 as published by
@@ -20,14 +22,6 @@
 
 #include <linux/io.h>
 
-/* List of coreboot entry structures that is used */
-struct lb_cbmem_ref {
-	uint32_t tag;
-	uint32_t size;
-
-	uint64_t cbmem_addr;
-};
-
 /* Coreboot table header structure */
 struct coreboot_table_header {
 	char signature[4];
@@ -38,11 +32,67 @@ struct coreboot_table_header {
 	u32 table_entries;
 };
 
-/* Retrieve coreboot table entry with tag *tag* and copy it to data */
-int coreboot_table_find(int tag, void *data, size_t data_size);
+/* List of coreboot entry structures that is used */
+/* Generic */
+struct coreboot_table_entry {
+	u32 tag;
+	u32 size;
+};
+
+/* Points to a CBMEM entry */
+struct lb_cbmem_ref {
+	u32 tag;
+	u32 size;
+
+	u64 cbmem_addr;
+};
+
+/* Describes framebuffer setup by coreboot */
+struct lb_framebuffer {
+	u32 tag;
+	u32 size;
+
+	u64 physical_address;
+	u32 x_resolution;
+	u32 y_resolution;
+	u32 bytes_per_line;
+	u8  bits_per_pixel;
+	u8  red_mask_pos;
+	u8  red_mask_size;
+	u8  green_mask_pos;
+	u8  green_mask_size;
+	u8  blue_mask_pos;
+	u8  blue_mask_size;
+	u8  reserved_mask_pos;
+	u8  reserved_mask_size;
+};
+
+/* A device, additionally with information from coreboot. */
+struct coreboot_device {
+	struct device dev;
+	union {
+		struct coreboot_table_entry entry;
+		struct lb_cbmem_ref cbmem_ref;
+		struct lb_framebuffer framebuffer;
+	};
+};
+
+/* A driver for handling devices described in coreboot tables. */
+struct coreboot_driver {
+	int (*probe)(struct coreboot_device *);
+	int (*remove)(struct coreboot_device *);
+	struct device_driver drv;
+	u32 tag;
+};
+
+/* Register a driver that uses the data from a coreboot table. */
+int coreboot_driver_register(struct coreboot_driver *driver);
+
+/* Unregister a driver that uses the data from a coreboot table. */
+void coreboot_driver_unregister(struct coreboot_driver *driver);
 
 /* Initialize coreboot table module given a pointer to iomem */
-int coreboot_table_init(void __iomem *ptr);
+int coreboot_table_init(struct device *dev, void __iomem *ptr);
 
 /* Cleanup coreboot table module */
 int coreboot_table_exit(void);
diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/google/framebuffer-coreboot.c
new file mode 100644
index 000000000000..b8b49c067157
--- /dev/null
+++ b/drivers/firmware/google/framebuffer-coreboot.c
@@ -0,0 +1,115 @@
+/*
+ * framebuffer-coreboot.c
+ *
+ * Memory based framebuffer accessed through coreboot table.
+ *
+ * Copyright 2012-2013 David Herrmann <dh.herrmann@gmail.com>
+ * Copyright 2017 Google Inc.
+ * Copyright 2017 Samuel Holland <samuel@sholland.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License v2.0 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/platform_data/simplefb.h>
+#include <linux/platform_device.h>
+
+#include "coreboot_table.h"
+
+#define CB_TAG_FRAMEBUFFER 0x12
+
+static const struct simplefb_format formats[] = SIMPLEFB_FORMATS;
+
+static int framebuffer_probe(struct coreboot_device *dev)
+{
+	int i;
+	u32 length;
+	struct lb_framebuffer *fb = &dev->framebuffer;
+	struct platform_device *pdev;
+	struct resource res;
+	struct simplefb_platform_data pdata = {
+		.width = fb->x_resolution,
+		.height = fb->y_resolution,
+		.stride = fb->bytes_per_line,
+		.format = NULL,
+	};
+
+	for (i = 0; i < ARRAY_SIZE(formats); ++i) {
+		if (fb->bits_per_pixel     == formats[i].bits_per_pixel &&
+		    fb->red_mask_pos       == formats[i].red.offset &&
+		    fb->red_mask_size      == formats[i].red.length &&
+		    fb->green_mask_pos     == formats[i].green.offset &&
+		    fb->green_mask_size    == formats[i].green.length &&
+		    fb->blue_mask_pos      == formats[i].blue.offset &&
+		    fb->blue_mask_size     == formats[i].blue.length &&
+		    fb->reserved_mask_pos  == formats[i].transp.offset &&
+		    fb->reserved_mask_size == formats[i].transp.length)
+			pdata.format = formats[i].name;
+	}
+	if (!pdata.format)
+		return -ENODEV;
+
+	memset(&res, 0, sizeof(res));
+	res.flags = IORESOURCE_MEM | IORESOURCE_BUSY;
+	res.name = "Coreboot Framebuffer";
+	res.start = fb->physical_address;
+	length = PAGE_ALIGN(fb->y_resolution * fb->bytes_per_line);
+	res.end = res.start + length - 1;
+	if (res.end <= res.start)
+		return -EINVAL;
+
+	pdev = platform_device_register_resndata(&dev->dev,
+						 "simple-framebuffer", 0,
+						 &res, 1, &pdata,
+						 sizeof(pdata));
+	if (IS_ERR(pdev))
+		pr_warn("coreboot: could not register framebuffer\n");
+	else
+		dev_set_drvdata(&dev->dev, pdev);
+
+	return PTR_ERR_OR_ZERO(pdev);
+}
+
+static int framebuffer_remove(struct coreboot_device *dev)
+{
+	struct platform_device *pdev = dev_get_drvdata(&dev->dev);
+
+	platform_device_unregister(pdev);
+
+	return 0;
+}
+
+static struct coreboot_driver framebuffer_driver = {
+	.probe = framebuffer_probe,
+	.remove = framebuffer_remove,
+	.drv = {
+		.name = "framebuffer",
+	},
+	.tag = CB_TAG_FRAMEBUFFER,
+};
+
+static int __init coreboot_framebuffer_init(void)
+{
+	return coreboot_driver_register(&framebuffer_driver);
+}
+
+static void coreboot_framebuffer_exit(void)
+{
+	coreboot_driver_unregister(&framebuffer_driver);
+}
+
+module_init(coreboot_framebuffer_init);
+module_exit(coreboot_framebuffer_exit);
+
+MODULE_AUTHOR("Samuel Holland <samuel@sholland.org>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/firmware/google/memconsole-coreboot.c b/drivers/firmware/google/memconsole-coreboot.c
index 52738887735c..b29e10757bfb 100644
--- a/drivers/firmware/google/memconsole-coreboot.c
+++ b/drivers/firmware/google/memconsole-coreboot.c
@@ -15,9 +15,9 @@
  * GNU General Public License for more details.
  */
 
+#include <linux/device.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/platform_device.h>
 
 #include "memconsole.h"
 #include "coreboot_table.h"
@@ -73,18 +73,19 @@ static ssize_t memconsole_coreboot_read(char *buf, loff_t pos, size_t count)
 	return done;
 }
 
-static int memconsole_coreboot_init(phys_addr_t physaddr)
+static int memconsole_probe(struct coreboot_device *dev)
 {
 	struct cbmem_cons __iomem *tmp_cbmc;
 
-	tmp_cbmc = memremap(physaddr, sizeof(*tmp_cbmc), MEMREMAP_WB);
+	tmp_cbmc = memremap(dev->cbmem_ref.cbmem_addr,
+			    sizeof(*tmp_cbmc), MEMREMAP_WB);
 
 	if (!tmp_cbmc)
 		return -ENOMEM;
 
 	/* Read size only once to prevent overrun attack through /dev/mem. */
 	cbmem_console_size = tmp_cbmc->size_dont_access_after_boot;
-	cbmem_console = memremap(physaddr,
+	cbmem_console = memremap(dev->cbmem_ref.cbmem_addr,
 				 cbmem_console_size + sizeof(*cbmem_console),
 				 MEMREMAP_WB);
 	memunmap(tmp_cbmc);
@@ -93,26 +94,11 @@ static int memconsole_coreboot_init(phys_addr_t physaddr)
 		return -ENOMEM;
 
 	memconsole_setup(memconsole_coreboot_read);
-	return 0;
-}
-
-static int memconsole_probe(struct platform_device *pdev)
-{
-	int ret;
-	struct lb_cbmem_ref entry;
-
-	ret = coreboot_table_find(CB_TAG_CBMEM_CONSOLE, &entry, sizeof(entry));
-	if (ret)
-		return ret;
-
-	ret = memconsole_coreboot_init(entry.cbmem_addr);
-	if (ret)
-		return ret;
 
 	return memconsole_sysfs_init();
 }
 
-static int memconsole_remove(struct platform_device *pdev)
+static int memconsole_remove(struct coreboot_device *dev)
 {
 	memconsole_exit();
 
@@ -122,28 +108,27 @@ static int memconsole_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static struct platform_driver memconsole_driver = {
+static struct coreboot_driver memconsole_driver = {
 	.probe = memconsole_probe,
 	.remove = memconsole_remove,
-	.driver = {
+	.drv = {
 		.name = "memconsole",
 	},
+	.tag = CB_TAG_CBMEM_CONSOLE,
 };
 
-static int __init platform_memconsole_init(void)
+static void coreboot_memconsole_exit(void)
 {
-	struct platform_device *pdev;
-
-	pdev = platform_device_register_simple("memconsole", -1, NULL, 0);
-	if (IS_ERR(pdev))
-		return PTR_ERR(pdev);
-
-	platform_driver_register(&memconsole_driver);
+	coreboot_driver_unregister(&memconsole_driver);
+}
 
-	return 0;
+static int __init coreboot_memconsole_init(void)
+{
+	return coreboot_driver_register(&memconsole_driver);
 }
 
-module_init(platform_memconsole_init);
+module_exit(coreboot_memconsole_exit);
+module_init(coreboot_memconsole_init);
 
 MODULE_AUTHOR("Google, Inc.");
 MODULE_LICENSE("GPL");
diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/google/vpd.c
index e4b40f2b4627..e9db895916c3 100644
--- a/drivers/firmware/google/vpd.c
+++ b/drivers/firmware/google/vpd.c
@@ -286,20 +286,15 @@ static int vpd_sections_init(phys_addr_t physaddr)
 	return 0;
 }
 
-static int vpd_probe(struct platform_device *pdev)
+static int vpd_probe(struct coreboot_device *dev)
 {
 	int ret;
-	struct lb_cbmem_ref entry;
-
-	ret = coreboot_table_find(CB_TAG_VPD, &entry, sizeof(entry));
-	if (ret)
-		return ret;
 
 	vpd_kobj = kobject_create_and_add("vpd", firmware_kobj);
 	if (!vpd_kobj)
 		return -ENOMEM;
 
-	ret = vpd_sections_init(entry.cbmem_addr);
+	ret = vpd_sections_init(dev->cbmem_ref.cbmem_addr);
 	if (ret) {
 		kobject_put(vpd_kobj);
 		return ret;
@@ -308,7 +303,7 @@ static int vpd_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int vpd_remove(struct platform_device *pdev)
+static int vpd_remove(struct coreboot_device *dev)
 {
 	vpd_section_destroy(&ro_vpd);
 	vpd_section_destroy(&rw_vpd);
@@ -318,41 +313,27 @@ static int vpd_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static struct platform_driver vpd_driver = {
+static struct coreboot_driver vpd_driver = {
 	.probe = vpd_probe,
 	.remove = vpd_remove,
-	.driver = {
+	.drv = {
 		.name = "vpd",
 	},
+	.tag = CB_TAG_VPD,
 };
 
-static struct platform_device *vpd_pdev;
-
-static int __init vpd_platform_init(void)
+static int __init coreboot_vpd_init(void)
 {
-	int ret;
-
-	ret = platform_driver_register(&vpd_driver);
-	if (ret)
-		return ret;
-
-	vpd_pdev = platform_device_register_simple("vpd", -1, NULL, 0);
-	if (IS_ERR(vpd_pdev)) {
-		platform_driver_unregister(&vpd_driver);
-		return PTR_ERR(vpd_pdev);
-	}
-
-	return 0;
+	return coreboot_driver_register(&vpd_driver);
 }
 
-static void __exit vpd_platform_exit(void)
+static void __exit coreboot_vpd_exit(void)
 {
-	platform_device_unregister(vpd_pdev);
-	platform_driver_unregister(&vpd_driver);
+	coreboot_driver_unregister(&vpd_driver);
 }
 
-module_init(vpd_platform_init);
-module_exit(vpd_platform_exit);
+module_init(coreboot_vpd_init);
+module_exit(coreboot_vpd_exit);
 
 MODULE_AUTHOR("Google, Inc.");
 MODULE_LICENSE("GPL");
diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c
index 5a7d693009ef..e778af766fae 100644
--- a/drivers/firmware/qcom_scm.c
+++ b/drivers/firmware/qcom_scm.c
@@ -603,6 +603,9 @@ static const struct of_device_id qcom_scm_dt_match[] = {
 	{ .compatible = "qcom,scm-msm8996",
 	  .data = NULL, /* no clocks */
 	},
+	{ .compatible = "qcom,scm-ipq4019",
+	  .data = NULL, /* no clocks */
+	},
 	{ .compatible = "qcom,scm",
 	  .data = (void *)(SCM_HAS_CORE_CLK
 			   | SCM_HAS_IFACE_CLK
diff --git a/drivers/firmware/ti_sci.c b/drivers/firmware/ti_sci.c
index 5229036dcfbf..b74a533ef35b 100644
--- a/drivers/firmware/ti_sci.c
+++ b/drivers/firmware/ti_sci.c
@@ -1,17 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Texas Instruments System Control Interface Protocol Driver
  *
  * Copyright (C) 2015-2016 Texas Instruments Incorporated - http://www.ti.com/
  *	Nishanth Menon
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #define pr_fmt(fmt) "%s: " fmt, __func__
diff --git a/drivers/firmware/ti_sci.h b/drivers/firmware/ti_sci.h
index 9b611e9e6f6d..12bf316b68df 100644
--- a/drivers/firmware/ti_sci.h
+++ b/drivers/firmware/ti_sci.h
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: BSD-3-Clause
 /*
  * Texas Instruments System Control Interface (TISCI) Protocol
  *
@@ -6,35 +7,6 @@
  * See: http://processors.wiki.ti.com/index.php/TISCI for details
  *
  * Copyright (C)  2015-2016 Texas Instruments Incorporated - http://www.ti.com/
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *   Redistributions of source code must retain the above copyright
- *   notice, this list of conditions and the following disclaimer.
- *
- *   Redistributions in binary form must reproduce the above copyright
- *   notice, this list of conditions and the following disclaimer in the
- *   documentation and/or other materials provided with the
- *   distribution.
- *
- *   Neither the name of Texas Instruments Incorporated nor the names of
- *   its contributors may be used to endorse or promote products derived
- *   from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
  */
 
 #ifndef __TI_SCI_H
diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig
index f47ef848bcd0..ee9c5420c47f 100644
--- a/drivers/fpga/Kconfig
+++ b/drivers/fpga/Kconfig
@@ -53,7 +53,6 @@ config FPGA_MGR_ALTERA_CVP
 config FPGA_MGR_ZYNQ_FPGA
 	tristate "Xilinx Zynq FPGA"
 	depends on ARCH_ZYNQ || COMPILE_TEST
-	depends on HAS_DMA
 	help
 	  FPGA manager driver support for Xilinx Zynq FPGAs.
 
@@ -70,6 +69,13 @@ config FPGA_MGR_ICE40_SPI
 	help
 	  FPGA manager driver support for Lattice iCE40 FPGAs over SPI.
 
+config FPGA_MGR_MACHXO2_SPI
+	tristate "Lattice MachXO2 SPI"
+	depends on SPI
+	help
+	  FPGA manager driver support for Lattice MachXO2 configuration
+	  over slave SPI interface.
+
 config FPGA_MGR_TS73XX
 	tristate "Technologic Systems TS-73xx SBC FPGA Manager"
 	depends on ARCH_EP93XX && MACH_TS72XX
diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile
index 3cb276a0f88d..f9803dad6919 100644
--- a/drivers/fpga/Makefile
+++ b/drivers/fpga/Makefile
@@ -10,6 +10,7 @@ obj-$(CONFIG_FPGA)			+= fpga-mgr.o
 obj-$(CONFIG_FPGA_MGR_ALTERA_CVP)	+= altera-cvp.o
 obj-$(CONFIG_FPGA_MGR_ALTERA_PS_SPI)	+= altera-ps-spi.o
 obj-$(CONFIG_FPGA_MGR_ICE40_SPI)	+= ice40-spi.o
+obj-$(CONFIG_FPGA_MGR_MACHXO2_SPI)	+= machxo2-spi.o
 obj-$(CONFIG_FPGA_MGR_SOCFPGA)		+= socfpga.o
 obj-$(CONFIG_FPGA_MGR_SOCFPGA_A10)	+= socfpga-a10.o
 obj-$(CONFIG_FPGA_MGR_TS73XX)		+= ts73xx-fpga.o
diff --git a/drivers/fpga/altera-cvp.c b/drivers/fpga/altera-cvp.c
index 77b04e4b3254..dd4edd8f22ce 100644
--- a/drivers/fpga/altera-cvp.c
+++ b/drivers/fpga/altera-cvp.c
@@ -401,6 +401,7 @@ static int altera_cvp_probe(struct pci_dev *pdev,
 			    const struct pci_device_id *dev_id)
 {
 	struct altera_cvp_conf *conf;
+	struct fpga_manager *mgr;
 	u16 cmd, val;
 	int ret;
 
@@ -452,16 +453,24 @@ static int altera_cvp_probe(struct pci_dev *pdev,
 	snprintf(conf->mgr_name, sizeof(conf->mgr_name), "%s @%s",
 		 ALTERA_CVP_MGR_NAME, pci_name(pdev));
 
-	ret = fpga_mgr_register(&pdev->dev, conf->mgr_name,
-				&altera_cvp_ops, conf);
-	if (ret)
+	mgr = fpga_mgr_create(&pdev->dev, conf->mgr_name,
+			      &altera_cvp_ops, conf);
+	if (!mgr)
+		return -ENOMEM;
+
+	pci_set_drvdata(pdev, mgr);
+
+	ret = fpga_mgr_register(mgr);
+	if (ret) {
+		fpga_mgr_free(mgr);
 		goto err_unmap;
+	}
 
 	ret = driver_create_file(&altera_cvp_driver.driver,
 				 &driver_attr_chkcfg);
 	if (ret) {
 		dev_err(&pdev->dev, "Can't create sysfs chkcfg file\n");
-		fpga_mgr_unregister(&pdev->dev);
+		fpga_mgr_unregister(mgr);
 		goto err_unmap;
 	}
 
@@ -483,7 +492,7 @@ static void altera_cvp_remove(struct pci_dev *pdev)
 	u16 cmd;
 
 	driver_remove_file(&altera_cvp_driver.driver, &driver_attr_chkcfg);
-	fpga_mgr_unregister(&pdev->dev);
+	fpga_mgr_unregister(mgr);
 	pci_iounmap(pdev, conf->map);
 	pci_release_region(pdev, CVP_BAR);
 	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
diff --git a/drivers/fpga/altera-fpga2sdram.c b/drivers/fpga/altera-fpga2sdram.c
index d4eeb74388da..23660ccd634b 100644
--- a/drivers/fpga/altera-fpga2sdram.c
+++ b/drivers/fpga/altera-fpga2sdram.c
@@ -1,19 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * FPGA to SDRAM Bridge Driver for Altera SoCFPGA Devices
  *
  *  Copyright (C) 2013-2016 Altera Corporation, All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 /*
@@ -106,6 +95,7 @@ static int alt_fpga_bridge_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct alt_fpga2sdram_data *priv;
+	struct fpga_bridge *br;
 	u32 enable;
 	struct regmap *sysmgr;
 	int ret = 0;
@@ -131,10 +121,18 @@ static int alt_fpga_bridge_probe(struct platform_device *pdev)
 	/* Get f2s bridge configuration saved in handoff register */
 	regmap_read(sysmgr, SYSMGR_ISWGRP_HANDOFF3, &priv->mask);
 
-	ret = fpga_bridge_register(dev, F2S_BRIDGE_NAME,
-				   &altera_fpga2sdram_br_ops, priv);
-	if (ret)
+	br = fpga_bridge_create(dev, F2S_BRIDGE_NAME,
+				&altera_fpga2sdram_br_ops, priv);
+	if (!br)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, br);
+
+	ret = fpga_bridge_register(br);
+	if (ret) {
+		fpga_bridge_free(br);
 		return ret;
+	}
 
 	dev_info(dev, "driver initialized with handoff %08x\n", priv->mask);
 
@@ -146,7 +144,7 @@ static int alt_fpga_bridge_probe(struct platform_device *pdev)
 				 (enable ? "enabling" : "disabling"));
 			ret = _alt_fpga2sdram_enable_set(priv, enable);
 			if (ret) {
-				fpga_bridge_unregister(&pdev->dev);
+				fpga_bridge_unregister(br);
 				return ret;
 			}
 		}
@@ -157,7 +155,9 @@ static int alt_fpga_bridge_probe(struct platform_device *pdev)
 
 static int alt_fpga_bridge_remove(struct platform_device *pdev)
 {
-	fpga_bridge_unregister(&pdev->dev);
+	struct fpga_bridge *br = platform_get_drvdata(pdev);
+
+	fpga_bridge_unregister(br);
 
 	return 0;
 }
diff --git a/drivers/fpga/altera-freeze-bridge.c b/drivers/fpga/altera-freeze-bridge.c
index 6159cfcf78a2..ffd586c48ecf 100644
--- a/drivers/fpga/altera-freeze-bridge.c
+++ b/drivers/fpga/altera-freeze-bridge.c
@@ -1,19 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * FPGA Freeze Bridge Controller
  *
  *  Copyright (C) 2016 Altera Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <linux/delay.h>
 #include <linux/io.h>
@@ -221,8 +210,10 @@ static int altera_freeze_br_probe(struct platform_device *pdev)
 	struct device_node *np = pdev->dev.of_node;
 	void __iomem *base_addr;
 	struct altera_freeze_br_data *priv;
+	struct fpga_bridge *br;
 	struct resource *res;
 	u32 status, revision;
+	int ret;
 
 	if (!np)
 		return -ENODEV;
@@ -254,13 +245,27 @@ static int altera_freeze_br_probe(struct platform_device *pdev)
 
 	priv->base_addr = base_addr;
 
-	return fpga_bridge_register(dev, FREEZE_BRIDGE_NAME,
-				    &altera_freeze_br_br_ops, priv);
+	br = fpga_bridge_create(dev, FREEZE_BRIDGE_NAME,
+				&altera_freeze_br_br_ops, priv);
+	if (!br)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, br);
+
+	ret = fpga_bridge_register(br);
+	if (ret) {
+		fpga_bridge_free(br);
+		return ret;
+	}
+
+	return 0;
 }
 
 static int altera_freeze_br_remove(struct platform_device *pdev)
 {
-	fpga_bridge_unregister(&pdev->dev);
+	struct fpga_bridge *br = platform_get_drvdata(pdev);
+
+	fpga_bridge_unregister(br);
 
 	return 0;
 }
diff --git a/drivers/fpga/altera-hps2fpga.c b/drivers/fpga/altera-hps2fpga.c
index 406d2f10741f..a974d3f60321 100644
--- a/drivers/fpga/altera-hps2fpga.c
+++ b/drivers/fpga/altera-hps2fpga.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * FPGA to/from HPS Bridge Driver for Altera SoCFPGA Devices
  *
@@ -6,18 +7,6 @@
  * Includes this patch from the mailing list:
  *   fpga: altera-hps2fpga: fix HPS2FPGA bridge visibility to L3 masters
  *   Signed-off-by: Anatolij Gustschin <agust@denx.de>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 /*
@@ -139,6 +128,7 @@ static int alt_fpga_bridge_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct altera_hps2fpga_data *priv;
 	const struct of_device_id *of_id;
+	struct fpga_bridge *br;
 	u32 enable;
 	int ret;
 
@@ -190,11 +180,24 @@ static int alt_fpga_bridge_probe(struct platform_device *pdev)
 		}
 	}
 
-	ret = fpga_bridge_register(dev, priv->name, &altera_hps2fpga_br_ops,
-				   priv);
-err:
+	br = fpga_bridge_create(dev, priv->name, &altera_hps2fpga_br_ops, priv);
+	if (!br) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	platform_set_drvdata(pdev, br);
+
+	ret = fpga_bridge_register(br);
 	if (ret)
-		clk_disable_unprepare(priv->clk);
+		goto err_free;
+
+	return 0;
+
+err_free:
+	fpga_bridge_free(br);
+err:
+	clk_disable_unprepare(priv->clk);
 
 	return ret;
 }
@@ -204,7 +207,7 @@ static int alt_fpga_bridge_remove(struct platform_device *pdev)
 	struct fpga_bridge *bridge = platform_get_drvdata(pdev);
 	struct altera_hps2fpga_data *priv = bridge->priv;
 
-	fpga_bridge_unregister(&pdev->dev);
+	fpga_bridge_unregister(bridge);
 
 	clk_disable_unprepare(priv->clk);
 
diff --git a/drivers/fpga/altera-pr-ip-core-plat.c b/drivers/fpga/altera-pr-ip-core-plat.c
index 8fb36b8b4648..b293d83143f1 100644
--- a/drivers/fpga/altera-pr-ip-core-plat.c
+++ b/drivers/fpga/altera-pr-ip-core-plat.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Driver for Altera Partial Reconfiguration IP Core
  *
@@ -5,18 +6,6 @@
  *
  * Based on socfpga-a10.c Copyright (C) 2015-2016 Altera Corporation
  *  by Alan Tull <atull@opensource.altera.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <linux/fpga/altera-pr-ip-core.h>
 #include <linux/module.h>
diff --git a/drivers/fpga/altera-pr-ip-core.c b/drivers/fpga/altera-pr-ip-core.c
index a7b31f9797ce..65e0b6a2c031 100644
--- a/drivers/fpga/altera-pr-ip-core.c
+++ b/drivers/fpga/altera-pr-ip-core.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Driver for Altera Partial Reconfiguration IP Core
  *
@@ -5,18 +6,6 @@
  *
  * Based on socfpga-a10.c Copyright (C) 2015-2016 Altera Corporation
  *  by Alan Tull <atull@opensource.altera.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <linux/delay.h>
 #include <linux/fpga/altera-pr-ip-core.h>
@@ -187,6 +176,8 @@ static const struct fpga_manager_ops alt_pr_ops = {
 int alt_pr_register(struct device *dev, void __iomem *reg_base)
 {
 	struct alt_pr_priv *priv;
+	struct fpga_manager *mgr;
+	int ret;
 	u32 val;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
@@ -201,15 +192,27 @@ int alt_pr_register(struct device *dev, void __iomem *reg_base)
 		(val & ALT_PR_CSR_STATUS_MSK) >> ALT_PR_CSR_STATUS_SFT,
 		(int)(val & ALT_PR_CSR_PR_START));
 
-	return fpga_mgr_register(dev, dev_name(dev), &alt_pr_ops, priv);
+	mgr = fpga_mgr_create(dev, dev_name(dev), &alt_pr_ops, priv);
+	if (!mgr)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, mgr);
+
+	ret = fpga_mgr_register(mgr);
+	if (ret)
+		fpga_mgr_free(mgr);
+
+	return ret;
 }
 EXPORT_SYMBOL_GPL(alt_pr_register);
 
 int alt_pr_unregister(struct device *dev)
 {
+	struct fpga_manager *mgr = dev_get_drvdata(dev);
+
 	dev_dbg(dev, "%s\n", __func__);
 
-	fpga_mgr_unregister(dev);
+	fpga_mgr_unregister(mgr);
 
 	return 0;
 }
diff --git a/drivers/fpga/altera-ps-spi.c b/drivers/fpga/altera-ps-spi.c
index 06d212a3d49d..24b25c626036 100644
--- a/drivers/fpga/altera-ps-spi.c
+++ b/drivers/fpga/altera-ps-spi.c
@@ -238,6 +238,8 @@ static int altera_ps_probe(struct spi_device *spi)
 {
 	struct altera_ps_conf *conf;
 	const struct of_device_id *of_id;
+	struct fpga_manager *mgr;
+	int ret;
 
 	conf = devm_kzalloc(&spi->dev, sizeof(*conf), GFP_KERNEL);
 	if (!conf)
@@ -273,13 +275,25 @@ static int altera_ps_probe(struct spi_device *spi)
 	snprintf(conf->mgr_name, sizeof(conf->mgr_name), "%s %s",
 		 dev_driver_string(&spi->dev), dev_name(&spi->dev));
 
-	return fpga_mgr_register(&spi->dev, conf->mgr_name,
-				 &altera_ps_ops, conf);
+	mgr = fpga_mgr_create(&spi->dev, conf->mgr_name,
+			      &altera_ps_ops, conf);
+	if (!mgr)
+		return -ENOMEM;
+
+	spi_set_drvdata(spi, mgr);
+
+	ret = fpga_mgr_register(mgr);
+	if (ret)
+		fpga_mgr_free(mgr);
+
+	return ret;
 }
 
 static int altera_ps_remove(struct spi_device *spi)
 {
-	fpga_mgr_unregister(&spi->dev);
+	struct fpga_manager *mgr = spi_get_drvdata(spi);
+
+	fpga_mgr_unregister(mgr);
 
 	return 0;
 }
diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c
index 31bd2c59c305..24b8f98b73ec 100644
--- a/drivers/fpga/fpga-bridge.c
+++ b/drivers/fpga/fpga-bridge.c
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * FPGA Bridge Framework Driver
  *
  *  Copyright (C) 2013-2016 Altera Corporation, All Rights Reserved.
  *  Copyright (C) 2017 Intel Corporation
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <linux/fpga/fpga-bridge.h>
 #include <linux/idr.h>
@@ -132,6 +121,7 @@ static int fpga_bridge_dev_match(struct device *dev, const void *data)
 /**
  * fpga_bridge_get - get an exclusive reference to a fpga bridge
  * @dev:	parent device that fpga bridge was registered with
+ * @info:	fpga manager info
  *
  * Given a device, get an exclusive reference to a fpga bridge.
  *
@@ -328,28 +318,29 @@ static struct attribute *fpga_bridge_attrs[] = {
 ATTRIBUTE_GROUPS(fpga_bridge);
 
 /**
- * fpga_bridge_register - register a fpga bridge driver
+ * fpga_bridge_create - create and initialize a struct fpga_bridge
  * @dev:	FPGA bridge device from pdev
  * @name:	FPGA bridge name
  * @br_ops:	pointer to structure of fpga bridge ops
  * @priv:	FPGA bridge private data
  *
- * Return: 0 for success, error code otherwise.
+ * Return: struct fpga_bridge or NULL
  */
-int fpga_bridge_register(struct device *dev, const char *name,
-			 const struct fpga_bridge_ops *br_ops, void *priv)
+struct fpga_bridge *fpga_bridge_create(struct device *dev, const char *name,
+				       const struct fpga_bridge_ops *br_ops,
+				       void *priv)
 {
 	struct fpga_bridge *bridge;
 	int id, ret = 0;
 
 	if (!name || !strlen(name)) {
 		dev_err(dev, "Attempt to register with no name!\n");
-		return -EINVAL;
+		return NULL;
 	}
 
 	bridge = kzalloc(sizeof(*bridge), GFP_KERNEL);
 	if (!bridge)
-		return -ENOMEM;
+		return NULL;
 
 	id = ida_simple_get(&fpga_bridge_ida, 0, 0, GFP_KERNEL);
 	if (id < 0) {
@@ -370,40 +361,62 @@ int fpga_bridge_register(struct device *dev, const char *name,
 	bridge->dev.parent = dev;
 	bridge->dev.of_node = dev->of_node;
 	bridge->dev.id = id;
-	dev_set_drvdata(dev, bridge);
 
 	ret = dev_set_name(&bridge->dev, "br%d", id);
 	if (ret)
 		goto error_device;
 
-	ret = device_add(&bridge->dev);
-	if (ret)
-		goto error_device;
-
-	of_platform_populate(dev->of_node, NULL, NULL, dev);
-
-	dev_info(bridge->dev.parent, "fpga bridge [%s] registered\n",
-		 bridge->name);
-
-	return 0;
+	return bridge;
 
 error_device:
 	ida_simple_remove(&fpga_bridge_ida, id);
 error_kfree:
 	kfree(bridge);
 
-	return ret;
+	return NULL;
 }
-EXPORT_SYMBOL_GPL(fpga_bridge_register);
+EXPORT_SYMBOL_GPL(fpga_bridge_create);
 
 /**
- * fpga_bridge_unregister - unregister a fpga bridge driver
- * @dev: FPGA bridge device from pdev
+ * fpga_bridge_free - free a fpga bridge and its id
+ * @bridge:	FPGA bridge struct created by fpga_bridge_create
  */
-void fpga_bridge_unregister(struct device *dev)
+void fpga_bridge_free(struct fpga_bridge *bridge)
 {
-	struct fpga_bridge *bridge = dev_get_drvdata(dev);
+	ida_simple_remove(&fpga_bridge_ida, bridge->dev.id);
+	kfree(bridge);
+}
+EXPORT_SYMBOL_GPL(fpga_bridge_free);
 
+/**
+ * fpga_bridge_register - register a fpga bridge
+ * @bridge:	FPGA bridge struct created by fpga_bridge_create
+ *
+ * Return: 0 for success, error code otherwise.
+ */
+int fpga_bridge_register(struct fpga_bridge *bridge)
+{
+	struct device *dev = &bridge->dev;
+	int ret;
+
+	ret = device_add(dev);
+	if (ret)
+		return ret;
+
+	of_platform_populate(dev->of_node, NULL, NULL, dev);
+
+	dev_info(dev->parent, "fpga bridge [%s] registered\n", bridge->name);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(fpga_bridge_register);
+
+/**
+ * fpga_bridge_unregister - unregister and free a fpga bridge
+ * @bridge:	FPGA bridge struct created by fpga_bridge_create
+ */
+void fpga_bridge_unregister(struct fpga_bridge *bridge)
+{
 	/*
 	 * If the low level driver provides a method for putting bridge into
 	 * a desired state upon unregister, do it.
@@ -419,8 +432,7 @@ static void fpga_bridge_dev_release(struct device *dev)
 {
 	struct fpga_bridge *bridge = to_fpga_bridge(dev);
 
-	ida_simple_remove(&fpga_bridge_ida, bridge->dev.id);
-	kfree(bridge);
+	fpga_bridge_free(bridge);
 }
 
 static int __init fpga_bridge_dev_init(void)
diff --git a/drivers/fpga/fpga-mgr.c b/drivers/fpga/fpga-mgr.c
index 9939d2cbc9a6..c1564cf827fe 100644
--- a/drivers/fpga/fpga-mgr.c
+++ b/drivers/fpga/fpga-mgr.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * FPGA Manager Core
  *
@@ -6,18 +7,6 @@
  *
  * With code from the mailing list:
  * Copyright (C) 2013 Xilinx, Inc.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <linux/firmware.h>
 #include <linux/fpga/fpga-mgr.h>
@@ -32,6 +21,12 @@
 static DEFINE_IDA(fpga_mgr_ida);
 static struct class *fpga_mgr_class;
 
+/**
+ * fpga_image_info_alloc - Allocate a FPGA image info struct
+ * @dev: owning device
+ *
+ * Return: struct fpga_image_info or NULL
+ */
 struct fpga_image_info *fpga_image_info_alloc(struct device *dev)
 {
 	struct fpga_image_info *info;
@@ -50,6 +45,10 @@ struct fpga_image_info *fpga_image_info_alloc(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(fpga_image_info_alloc);
 
+/**
+ * fpga_image_info_free - Free a FPGA image info struct
+ * @info: FPGA image info struct to free
+ */
 void fpga_image_info_free(struct fpga_image_info *info)
 {
 	struct device *dev;
@@ -234,7 +233,7 @@ static int fpga_mgr_buf_load_mapped(struct fpga_manager *mgr,
 /**
  * fpga_mgr_buf_load - load fpga from image in buffer
  * @mgr:	fpga manager
- * @flags:	flags setting fpga confuration modes
+ * @info:	fpga image info
  * @buf:	buffer contain fpga image
  * @count:	byte count of buf
  *
@@ -343,6 +342,16 @@ static int fpga_mgr_firmware_load(struct fpga_manager *mgr,
 	return ret;
 }
 
+/**
+ * fpga_mgr_load - load FPGA from scatter/gather table, buffer, or firmware
+ * @mgr:	fpga manager
+ * @info:	fpga image information.
+ *
+ * Load the FPGA from an image which is indicated in @info.  If successful, the
+ * FPGA ends up in operating mode.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
 int fpga_mgr_load(struct fpga_manager *mgr, struct fpga_image_info *info)
 {
 	if (info->sgt)
@@ -429,11 +438,9 @@ static int fpga_mgr_dev_match(struct device *dev, const void *data)
 }
 
 /**
- * fpga_mgr_get - get a reference to a fpga mgr
+ * fpga_mgr_get - Given a device, get a reference to a fpga mgr.
  * @dev:	parent device that fpga mgr was registered with
  *
- * Given a device, get a reference to a fpga mgr.
- *
  * Return: fpga manager struct or IS_ERR() condition containing error code.
  */
 struct fpga_manager *fpga_mgr_get(struct device *dev)
@@ -453,10 +460,9 @@ static int fpga_mgr_of_node_match(struct device *dev, const void *data)
 }
 
 /**
- * of_fpga_mgr_get - get a reference to a fpga mgr
- * @node:	device node
+ * of_fpga_mgr_get - Given a device node, get a reference to a fpga mgr.
  *
- * Given a device node, get a reference to a fpga mgr.
+ * @node:	device node
  *
  * Return: fpga manager struct or IS_ERR() condition containing error code.
  */
@@ -489,7 +495,10 @@ EXPORT_SYMBOL_GPL(fpga_mgr_put);
  * @mgr:	fpga manager
  *
  * Given a pointer to FPGA Manager (from fpga_mgr_get() or
- * of_fpga_mgr_put()) attempt to get the mutex.
+ * of_fpga_mgr_put()) attempt to get the mutex. The user should call
+ * fpga_mgr_lock() and verify that it returns 0 before attempting to
+ * program the FPGA.  Likewise, the user should call fpga_mgr_unlock
+ * when done programming the FPGA.
  *
  * Return: 0 for success or -EBUSY
  */
@@ -505,7 +514,7 @@ int fpga_mgr_lock(struct fpga_manager *mgr)
 EXPORT_SYMBOL_GPL(fpga_mgr_lock);
 
 /**
- * fpga_mgr_unlock - Unlock FPGA manager
+ * fpga_mgr_unlock - Unlock FPGA manager after done programming
  * @mgr:	fpga manager
  */
 void fpga_mgr_unlock(struct fpga_manager *mgr)
@@ -515,17 +524,17 @@ void fpga_mgr_unlock(struct fpga_manager *mgr)
 EXPORT_SYMBOL_GPL(fpga_mgr_unlock);
 
 /**
- * fpga_mgr_register - register a low level fpga manager driver
+ * fpga_mgr_create - create and initialize a FPGA manager struct
  * @dev:	fpga manager device from pdev
  * @name:	fpga manager name
  * @mops:	pointer to structure of fpga manager ops
  * @priv:	fpga manager private data
  *
- * Return: 0 on success, negative error code otherwise.
+ * Return: pointer to struct fpga_manager or NULL
  */
-int fpga_mgr_register(struct device *dev, const char *name,
-		      const struct fpga_manager_ops *mops,
-		      void *priv)
+struct fpga_manager *fpga_mgr_create(struct device *dev, const char *name,
+				     const struct fpga_manager_ops *mops,
+				     void *priv)
 {
 	struct fpga_manager *mgr;
 	int id, ret;
@@ -534,17 +543,17 @@ int fpga_mgr_register(struct device *dev, const char *name,
 	    !mops->write_init || (!mops->write && !mops->write_sg) ||
 	    (mops->write && mops->write_sg)) {
 		dev_err(dev, "Attempt to register without fpga_manager_ops\n");
-		return -EINVAL;
+		return NULL;
 	}
 
 	if (!name || !strlen(name)) {
 		dev_err(dev, "Attempt to register with no name!\n");
-		return -EINVAL;
+		return NULL;
 	}
 
 	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
 	if (!mgr)
-		return -ENOMEM;
+		return NULL;
 
 	id = ida_simple_get(&fpga_mgr_ida, 0, 0, GFP_KERNEL);
 	if (id < 0) {
@@ -558,25 +567,56 @@ int fpga_mgr_register(struct device *dev, const char *name,
 	mgr->mops = mops;
 	mgr->priv = priv;
 
-	/*
-	 * Initialize framework state by requesting low level driver read state
-	 * from device.  FPGA may be in reset mode or may have been programmed
-	 * by bootloader or EEPROM.
-	 */
-	mgr->state = mgr->mops->state(mgr);
-
 	device_initialize(&mgr->dev);
 	mgr->dev.class = fpga_mgr_class;
 	mgr->dev.groups = mops->groups;
 	mgr->dev.parent = dev;
 	mgr->dev.of_node = dev->of_node;
 	mgr->dev.id = id;
-	dev_set_drvdata(dev, mgr);
 
 	ret = dev_set_name(&mgr->dev, "fpga%d", id);
 	if (ret)
 		goto error_device;
 
+	return mgr;
+
+error_device:
+	ida_simple_remove(&fpga_mgr_ida, id);
+error_kfree:
+	kfree(mgr);
+
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(fpga_mgr_create);
+
+/**
+ * fpga_mgr_free - deallocate a FPGA manager
+ * @mgr:	fpga manager struct created by fpga_mgr_create
+ */
+void fpga_mgr_free(struct fpga_manager *mgr)
+{
+	ida_simple_remove(&fpga_mgr_ida, mgr->dev.id);
+	kfree(mgr);
+}
+EXPORT_SYMBOL_GPL(fpga_mgr_free);
+
+/**
+ * fpga_mgr_register - register a FPGA manager
+ * @mgr:	fpga manager struct created by fpga_mgr_create
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int fpga_mgr_register(struct fpga_manager *mgr)
+{
+	int ret;
+
+	/*
+	 * Initialize framework state by requesting low level driver read state
+	 * from device.  FPGA may be in reset mode or may have been programmed
+	 * by bootloader or EEPROM.
+	 */
+	mgr->state = mgr->mops->state(mgr);
+
 	ret = device_add(&mgr->dev);
 	if (ret)
 		goto error_device;
@@ -586,22 +626,18 @@ int fpga_mgr_register(struct device *dev, const char *name,
 	return 0;
 
 error_device:
-	ida_simple_remove(&fpga_mgr_ida, id);
-error_kfree:
-	kfree(mgr);
+	ida_simple_remove(&fpga_mgr_ida, mgr->dev.id);
 
 	return ret;
 }
 EXPORT_SYMBOL_GPL(fpga_mgr_register);
 
 /**
- * fpga_mgr_unregister - unregister a low level fpga manager driver
- * @dev:	fpga manager device from pdev
+ * fpga_mgr_unregister - unregister and free a FPGA manager
+ * @mgr:	fpga manager struct
  */
-void fpga_mgr_unregister(struct device *dev)
+void fpga_mgr_unregister(struct fpga_manager *mgr)
 {
-	struct fpga_manager *mgr = dev_get_drvdata(dev);
-
 	dev_info(&mgr->dev, "%s %s\n", __func__, mgr->name);
 
 	/*
@@ -619,8 +655,7 @@ static void fpga_mgr_dev_release(struct device *dev)
 {
 	struct fpga_manager *mgr = to_fpga_manager(dev);
 
-	ida_simple_remove(&fpga_mgr_ida, mgr->dev.id);
-	kfree(mgr);
+	fpga_mgr_free(mgr);
 }
 
 static int __init fpga_mgr_class_init(void)
diff --git a/drivers/fpga/fpga-region.c b/drivers/fpga/fpga-region.c
index edab2a2e03ef..6d214d75c7be 100644
--- a/drivers/fpga/fpga-region.c
+++ b/drivers/fpga/fpga-region.c
@@ -1,22 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * FPGA Region - Device Tree support for FPGA programming under Linux
  *
  *  Copyright (C) 2013-2016 Altera Corporation
  *  Copyright (C) 2017 Intel Corporation
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-
 #include <linux/fpga/fpga-bridge.h>
 #include <linux/fpga/fpga-mgr.h>
 #include <linux/fpga/fpga-region.h>
@@ -93,8 +81,16 @@ static void fpga_region_put(struct fpga_region *region)
 
 /**
  * fpga_region_program_fpga - program FPGA
+ *
  * @region: FPGA region
+ *
  * Program an FPGA using fpga image info (region->info).
+ * If the region has a get_bridges function, the exclusive reference for the
+ * bridges will be held if programming succeeds.  This is intended to prevent
+ * reprogramming the region until the caller considers it safe to do so.
+ * The caller will need to call fpga_bridges_put() before attempting to
+ * reprogram the region.
+ *
  * Return 0 for success or negative error code.
  */
 int fpga_region_program_fpga(struct fpga_region *region)
@@ -162,45 +158,86 @@ int fpga_region_program_fpga(struct fpga_region *region)
 }
 EXPORT_SYMBOL_GPL(fpga_region_program_fpga);
 
-int fpga_region_register(struct device *dev, struct fpga_region *region)
+/**
+ * fpga_region_create - alloc and init a struct fpga_region
+ * @dev: device parent
+ * @mgr: manager that programs this region
+ * @get_bridges: optional function to get bridges to a list
+ *
+ * Return: struct fpga_region or NULL
+ */
+struct fpga_region
+*fpga_region_create(struct device *dev,
+		    struct fpga_manager *mgr,
+		    int (*get_bridges)(struct fpga_region *))
 {
+	struct fpga_region *region;
 	int id, ret = 0;
 
+	region = kzalloc(sizeof(*region), GFP_KERNEL);
+	if (!region)
+		return NULL;
+
 	id = ida_simple_get(&fpga_region_ida, 0, 0, GFP_KERNEL);
 	if (id < 0)
-		return id;
+		goto err_free;
 
+	region->mgr = mgr;
+	region->get_bridges = get_bridges;
 	mutex_init(&region->mutex);
 	INIT_LIST_HEAD(&region->bridge_list);
+
 	device_initialize(&region->dev);
-	region->dev.groups = region->groups;
 	region->dev.class = fpga_region_class;
 	region->dev.parent = dev;
 	region->dev.of_node = dev->of_node;
 	region->dev.id = id;
-	dev_set_drvdata(dev, region);
 
 	ret = dev_set_name(&region->dev, "region%d", id);
 	if (ret)
 		goto err_remove;
 
-	ret = device_add(&region->dev);
-	if (ret)
-		goto err_remove;
-
-	return 0;
+	return region;
 
 err_remove:
 	ida_simple_remove(&fpga_region_ida, id);
-	return ret;
+err_free:
+	kfree(region);
+
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(fpga_region_create);
+
+/**
+ * fpga_region_free - free a struct fpga_region
+ * @region: FPGA region created by fpga_region_create
+ */
+void fpga_region_free(struct fpga_region *region)
+{
+	ida_simple_remove(&fpga_region_ida, region->dev.id);
+	kfree(region);
+}
+EXPORT_SYMBOL_GPL(fpga_region_free);
+
+/**
+ * fpga_region_register - register a FPGA region
+ * @region: FPGA region created by fpga_region_create
+ * Return: 0 or -errno
+ */
+int fpga_region_register(struct fpga_region *region)
+{
+	return device_add(&region->dev);
+
 }
 EXPORT_SYMBOL_GPL(fpga_region_register);
 
-int fpga_region_unregister(struct fpga_region *region)
+/**
+ * fpga_region_unregister - unregister and free a FPGA region
+ * @region: FPGA region
+ */
+void fpga_region_unregister(struct fpga_region *region)
 {
 	device_unregister(&region->dev);
-
-	return 0;
 }
 EXPORT_SYMBOL_GPL(fpga_region_unregister);
 
@@ -208,7 +245,7 @@ static void fpga_region_dev_release(struct device *dev)
 {
 	struct fpga_region *region = to_fpga_region(dev);
 
-	ida_simple_remove(&fpga_region_ida, region->dev.id);
+	fpga_region_free(region);
 }
 
 /**
diff --git a/drivers/fpga/ice40-spi.c b/drivers/fpga/ice40-spi.c
index 7fca82023062..5981c7ee7a7d 100644
--- a/drivers/fpga/ice40-spi.c
+++ b/drivers/fpga/ice40-spi.c
@@ -133,6 +133,7 @@ static int ice40_fpga_probe(struct spi_device *spi)
 {
 	struct device *dev = &spi->dev;
 	struct ice40_fpga_priv *priv;
+	struct fpga_manager *mgr;
 	int ret;
 
 	priv = devm_kzalloc(&spi->dev, sizeof(*priv), GFP_KERNEL);
@@ -174,14 +175,26 @@ static int ice40_fpga_probe(struct spi_device *spi)
 		return ret;
 	}
 
-	/* Register with the FPGA manager */
-	return fpga_mgr_register(dev, "Lattice iCE40 FPGA Manager",
-				 &ice40_fpga_ops, priv);
+	mgr = fpga_mgr_create(dev, "Lattice iCE40 FPGA Manager",
+			      &ice40_fpga_ops, priv);
+	if (!mgr)
+		return -ENOMEM;
+
+	spi_set_drvdata(spi, mgr);
+
+	ret = fpga_mgr_register(mgr);
+	if (ret)
+		fpga_mgr_free(mgr);
+
+	return ret;
 }
 
 static int ice40_fpga_remove(struct spi_device *spi)
 {
-	fpga_mgr_unregister(&spi->dev);
+	struct fpga_manager *mgr = spi_get_drvdata(spi);
+
+	fpga_mgr_unregister(mgr);
+
 	return 0;
 }
 
diff --git a/drivers/fpga/machxo2-spi.c b/drivers/fpga/machxo2-spi.c
new file mode 100644
index 000000000000..a582e0000c97
--- /dev/null
+++ b/drivers/fpga/machxo2-spi.c
@@ -0,0 +1,415 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Lattice MachXO2 Slave SPI Driver
+ *
+ * Manage Lattice FPGA firmware that is loaded over SPI using
+ * the slave serial configuration interface.
+ *
+ * Copyright (C) 2018 Paolo Pisati <p.pisati@gmail.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/fpga/fpga-mgr.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/spi/spi.h>
+
+/* MachXO2 Programming Guide - sysCONFIG Programming Commands */
+#define IDCODE_PUB		{0xe0, 0x00, 0x00, 0x00}
+#define ISC_ENABLE		{0xc6, 0x08, 0x00, 0x00}
+#define ISC_ERASE		{0x0e, 0x04, 0x00, 0x00}
+#define ISC_PROGRAMDONE		{0x5e, 0x00, 0x00, 0x00}
+#define LSC_INITADDRESS		{0x46, 0x00, 0x00, 0x00}
+#define LSC_PROGINCRNV		{0x70, 0x00, 0x00, 0x01}
+#define LSC_READ_STATUS		{0x3c, 0x00, 0x00, 0x00}
+#define LSC_REFRESH		{0x79, 0x00, 0x00, 0x00}
+
+/*
+ * Max CCLK in Slave SPI mode according to 'MachXO2 Family Data
+ * Sheet' sysCONFIG Port Timing Specifications (3-36)
+ */
+#define MACHXO2_MAX_SPEED		66000000
+
+#define MACHXO2_LOW_DELAY_USEC		5
+#define MACHXO2_HIGH_DELAY_USEC		200
+#define MACHXO2_REFRESH_USEC		4800
+#define MACHXO2_MAX_BUSY_LOOP		128
+#define MACHXO2_MAX_REFRESH_LOOP	16
+
+#define MACHXO2_PAGE_SIZE		16
+#define MACHXO2_BUF_SIZE		(MACHXO2_PAGE_SIZE + 4)
+
+/* Status register bits, errors and error mask */
+#define BUSY	12
+#define DONE	8
+#define DVER	27
+#define ENAB	9
+#define ERRBITS	23
+#define ERRMASK	7
+#define FAIL	13
+
+#define ENOERR	0 /* no error */
+#define EID	1
+#define ECMD	2
+#define ECRC	3
+#define EPREAM	4 /* preamble error */
+#define EABRT	5 /* abort error */
+#define EOVERFL	6 /* overflow error */
+#define ESDMEOF	7 /* SDM EOF */
+
+static inline u8 get_err(unsigned long *status)
+{
+	return (*status >> ERRBITS) & ERRMASK;
+}
+
+static int get_status(struct spi_device *spi, unsigned long *status)
+{
+	struct spi_message msg;
+	struct spi_transfer rx, tx;
+	static const u8 cmd[] = LSC_READ_STATUS;
+	int ret;
+
+	memset(&rx, 0, sizeof(rx));
+	memset(&tx, 0, sizeof(tx));
+	tx.tx_buf = cmd;
+	tx.len = sizeof(cmd);
+	rx.rx_buf = status;
+	rx.len = 4;
+	spi_message_init(&msg);
+	spi_message_add_tail(&tx, &msg);
+	spi_message_add_tail(&rx, &msg);
+	ret = spi_sync(spi, &msg);
+	if (ret)
+		return ret;
+
+	*status = be32_to_cpu(*status);
+
+	return 0;
+}
+
+#ifdef DEBUG
+static const char *get_err_string(u8 err)
+{
+	switch (err) {
+	case ENOERR:	return "No Error";
+	case EID:	return "ID ERR";
+	case ECMD:	return "CMD ERR";
+	case ECRC:	return "CRC ERR";
+	case EPREAM:	return "Preamble ERR";
+	case EABRT:	return "Abort ERR";
+	case EOVERFL:	return "Overflow ERR";
+	case ESDMEOF:	return "SDM EOF";
+	}
+
+	return "Default switch case";
+}
+#endif
+
+static void dump_status_reg(unsigned long *status)
+{
+#ifdef DEBUG
+	pr_debug("machxo2 status: 0x%08lX - done=%d, cfgena=%d, busy=%d, fail=%d, devver=%d, err=%s\n",
+		 *status, test_bit(DONE, status), test_bit(ENAB, status),
+		 test_bit(BUSY, status), test_bit(FAIL, status),
+		 test_bit(DVER, status), get_err_string(get_err(status)));
+#endif
+}
+
+static int wait_until_not_busy(struct spi_device *spi)
+{
+	unsigned long status;
+	int ret, loop = 0;
+
+	do {
+		ret = get_status(spi, &status);
+		if (ret)
+			return ret;
+		if (++loop >= MACHXO2_MAX_BUSY_LOOP)
+			return -EBUSY;
+	} while (test_bit(BUSY, &status));
+
+	return 0;
+}
+
+static int machxo2_cleanup(struct fpga_manager *mgr)
+{
+	struct spi_device *spi = mgr->priv;
+	struct spi_message msg;
+	struct spi_transfer tx[2];
+	static const u8 erase[] = ISC_ERASE;
+	static const u8 refresh[] = LSC_REFRESH;
+	int ret;
+
+	memset(tx, 0, sizeof(tx));
+	spi_message_init(&msg);
+	tx[0].tx_buf = &erase;
+	tx[0].len = sizeof(erase);
+	spi_message_add_tail(&tx[0], &msg);
+	ret = spi_sync(spi, &msg);
+	if (ret)
+		goto fail;
+
+	ret = wait_until_not_busy(spi);
+	if (ret)
+		goto fail;
+
+	spi_message_init(&msg);
+	tx[1].tx_buf = &refresh;
+	tx[1].len = sizeof(refresh);
+	tx[1].delay_usecs = MACHXO2_REFRESH_USEC;
+	spi_message_add_tail(&tx[1], &msg);
+	ret = spi_sync(spi, &msg);
+	if (ret)
+		goto fail;
+
+	return 0;
+fail:
+	dev_err(&mgr->dev, "Cleanup failed\n");
+
+	return ret;
+}
+
+static enum fpga_mgr_states machxo2_spi_state(struct fpga_manager *mgr)
+{
+	struct spi_device *spi = mgr->priv;
+	unsigned long status;
+
+	get_status(spi, &status);
+	if (!test_bit(BUSY, &status) && test_bit(DONE, &status) &&
+	    get_err(&status) == ENOERR)
+		return FPGA_MGR_STATE_OPERATING;
+
+	return FPGA_MGR_STATE_UNKNOWN;
+}
+
+static int machxo2_write_init(struct fpga_manager *mgr,
+			      struct fpga_image_info *info,
+			      const char *buf, size_t count)
+{
+	struct spi_device *spi = mgr->priv;
+	struct spi_message msg;
+	struct spi_transfer tx[3];
+	static const u8 enable[] = ISC_ENABLE;
+	static const u8 erase[] = ISC_ERASE;
+	static const u8 initaddr[] = LSC_INITADDRESS;
+	unsigned long status;
+	int ret;
+
+	if ((info->flags & FPGA_MGR_PARTIAL_RECONFIG)) {
+		dev_err(&mgr->dev,
+			"Partial reconfiguration is not supported\n");
+		return -ENOTSUPP;
+	}
+
+	get_status(spi, &status);
+	dump_status_reg(&status);
+	memset(tx, 0, sizeof(tx));
+	spi_message_init(&msg);
+	tx[0].tx_buf = &enable;
+	tx[0].len = sizeof(enable);
+	tx[0].delay_usecs = MACHXO2_LOW_DELAY_USEC;
+	spi_message_add_tail(&tx[0], &msg);
+
+	tx[1].tx_buf = &erase;
+	tx[1].len = sizeof(erase);
+	spi_message_add_tail(&tx[1], &msg);
+	ret = spi_sync(spi, &msg);
+	if (ret)
+		goto fail;
+
+	ret = wait_until_not_busy(spi);
+	if (ret)
+		goto fail;
+
+	get_status(spi, &status);
+	if (test_bit(FAIL, &status))
+		goto fail;
+	dump_status_reg(&status);
+
+	spi_message_init(&msg);
+	tx[2].tx_buf = &initaddr;
+	tx[2].len = sizeof(initaddr);
+	spi_message_add_tail(&tx[2], &msg);
+	ret = spi_sync(spi, &msg);
+	if (ret)
+		goto fail;
+
+	get_status(spi, &status);
+	dump_status_reg(&status);
+
+	return 0;
+fail:
+	dev_err(&mgr->dev, "Error during FPGA init.\n");
+
+	return ret;
+}
+
+static int machxo2_write(struct fpga_manager *mgr, const char *buf,
+			 size_t count)
+{
+	struct spi_device *spi = mgr->priv;
+	struct spi_message msg;
+	struct spi_transfer tx;
+	static const u8 progincr[] = LSC_PROGINCRNV;
+	u8 payload[MACHXO2_BUF_SIZE];
+	unsigned long status;
+	int i, ret;
+
+	if (count % MACHXO2_PAGE_SIZE != 0) {
+		dev_err(&mgr->dev, "Malformed payload.\n");
+		return -EINVAL;
+	}
+	get_status(spi, &status);
+	dump_status_reg(&status);
+	memcpy(payload, &progincr, sizeof(progincr));
+	for (i = 0; i < count; i += MACHXO2_PAGE_SIZE) {
+		memcpy(&payload[sizeof(progincr)], &buf[i], MACHXO2_PAGE_SIZE);
+		memset(&tx, 0, sizeof(tx));
+		spi_message_init(&msg);
+		tx.tx_buf = payload;
+		tx.len = MACHXO2_BUF_SIZE;
+		tx.delay_usecs = MACHXO2_HIGH_DELAY_USEC;
+		spi_message_add_tail(&tx, &msg);
+		ret = spi_sync(spi, &msg);
+		if (ret) {
+			dev_err(&mgr->dev, "Error loading the bitstream.\n");
+			return ret;
+		}
+	}
+	get_status(spi, &status);
+	dump_status_reg(&status);
+
+	return 0;
+}
+
+static int machxo2_write_complete(struct fpga_manager *mgr,
+				  struct fpga_image_info *info)
+{
+	struct spi_device *spi = mgr->priv;
+	struct spi_message msg;
+	struct spi_transfer tx[2];
+	static const u8 progdone[] = ISC_PROGRAMDONE;
+	static const u8 refresh[] = LSC_REFRESH;
+	unsigned long status;
+	int ret, refreshloop = 0;
+
+	memset(tx, 0, sizeof(tx));
+	spi_message_init(&msg);
+	tx[0].tx_buf = &progdone;
+	tx[0].len = sizeof(progdone);
+	spi_message_add_tail(&tx[0], &msg);
+	ret = spi_sync(spi, &msg);
+	if (ret)
+		goto fail;
+	ret = wait_until_not_busy(spi);
+	if (ret)
+		goto fail;
+
+	get_status(spi, &status);
+	dump_status_reg(&status);
+	if (!test_bit(DONE, &status)) {
+		machxo2_cleanup(mgr);
+		goto fail;
+	}
+
+	do {
+		spi_message_init(&msg);
+		tx[1].tx_buf = &refresh;
+		tx[1].len = sizeof(refresh);
+		tx[1].delay_usecs = MACHXO2_REFRESH_USEC;
+		spi_message_add_tail(&tx[1], &msg);
+		ret = spi_sync(spi, &msg);
+		if (ret)
+			goto fail;
+
+		/* check refresh status */
+		get_status(spi, &status);
+		dump_status_reg(&status);
+		if (!test_bit(BUSY, &status) && test_bit(DONE, &status) &&
+		    get_err(&status) == ENOERR)
+			break;
+		if (++refreshloop == MACHXO2_MAX_REFRESH_LOOP) {
+			machxo2_cleanup(mgr);
+			goto fail;
+		}
+	} while (1);
+
+	get_status(spi, &status);
+	dump_status_reg(&status);
+
+	return 0;
+fail:
+	dev_err(&mgr->dev, "Refresh failed.\n");
+
+	return ret;
+}
+
+static const struct fpga_manager_ops machxo2_ops = {
+	.state = machxo2_spi_state,
+	.write_init = machxo2_write_init,
+	.write = machxo2_write,
+	.write_complete = machxo2_write_complete,
+};
+
+static int machxo2_spi_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct fpga_manager *mgr;
+	int ret;
+
+	if (spi->max_speed_hz > MACHXO2_MAX_SPEED) {
+		dev_err(dev, "Speed is too high\n");
+		return -EINVAL;
+	}
+
+	mgr = fpga_mgr_create(dev, "Lattice MachXO2 SPI FPGA Manager",
+			      &machxo2_ops, spi);
+	if (!mgr)
+		return -ENOMEM;
+
+	spi_set_drvdata(spi, mgr);
+
+	ret = fpga_mgr_register(mgr);
+	if (ret)
+		fpga_mgr_free(mgr);
+
+	return ret;
+}
+
+static int machxo2_spi_remove(struct spi_device *spi)
+{
+	struct fpga_manager *mgr = spi_get_drvdata(spi);
+
+	fpga_mgr_unregister(mgr);
+
+	return 0;
+}
+
+static const struct of_device_id of_match[] = {
+	{ .compatible = "lattice,machxo2-slave-spi", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, of_match);
+
+static const struct spi_device_id lattice_ids[] = {
+	{ "machxo2-slave-spi", 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(spi, lattice_ids);
+
+static struct spi_driver machxo2_spi_driver = {
+	.driver = {
+		.name = "machxo2-slave-spi",
+		.of_match_table = of_match_ptr(of_match),
+	},
+	.probe = machxo2_spi_probe,
+	.remove = machxo2_spi_remove,
+	.id_table = lattice_ids,
+};
+
+module_spi_driver(machxo2_spi_driver)
+
+MODULE_AUTHOR("Paolo Pisati <p.pisati@gmail.com>");
+MODULE_DESCRIPTION("Load Lattice FPGA firmware over SPI");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/fpga/of-fpga-region.c b/drivers/fpga/of-fpga-region.c
index 119ff75522f1..35fabb8083fb 100644
--- a/drivers/fpga/of-fpga-region.c
+++ b/drivers/fpga/of-fpga-region.c
@@ -1,22 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * FPGA Region - Device Tree support for FPGA programming under Linux
  *
  *  Copyright (C) 2013-2016 Altera Corporation
  *  Copyright (C) 2017 Intel Corporation
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-
 #include <linux/fpga/fpga-bridge.h>
 #include <linux/fpga/fpga-mgr.h>
 #include <linux/fpga/fpga-region.h>
@@ -422,27 +410,25 @@ static int of_fpga_region_probe(struct platform_device *pdev)
 	if (IS_ERR(mgr))
 		return -EPROBE_DEFER;
 
-	region = devm_kzalloc(dev, sizeof(*region), GFP_KERNEL);
+	region = fpga_region_create(dev, mgr, of_fpga_region_get_bridges);
 	if (!region) {
 		ret = -ENOMEM;
 		goto eprobe_mgr_put;
 	}
 
-	region->mgr = mgr;
-
-	/* Specify how to get bridges for this type of region. */
-	region->get_bridges = of_fpga_region_get_bridges;
-
-	ret = fpga_region_register(dev, region);
+	ret = fpga_region_register(region);
 	if (ret)
-		goto eprobe_mgr_put;
+		goto eprobe_free;
 
 	of_platform_populate(np, fpga_region_of_match, NULL, &region->dev);
+	dev_set_drvdata(dev, region);
 
 	dev_info(dev, "FPGA Region probed\n");
 
 	return 0;
 
+eprobe_free:
+	fpga_region_free(region);
 eprobe_mgr_put:
 	fpga_mgr_put(mgr);
 	return ret;
diff --git a/drivers/fpga/socfpga-a10.c b/drivers/fpga/socfpga-a10.c
index a46e343a5b72..be30c48eb6e4 100644
--- a/drivers/fpga/socfpga-a10.c
+++ b/drivers/fpga/socfpga-a10.c
@@ -1,21 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * FPGA Manager Driver for Altera Arria10 SoCFPGA
  *
  * Copyright (C) 2015-2016 Altera Corporation
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-
 #include <linux/clk.h>
 #include <linux/device.h>
 #include <linux/delay.h>
@@ -482,6 +470,7 @@ static int socfpga_a10_fpga_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct a10_fpga_priv *priv;
 	void __iomem *reg_base;
+	struct fpga_manager *mgr;
 	struct resource *res;
 	int ret;
 
@@ -519,9 +508,16 @@ static int socfpga_a10_fpga_probe(struct platform_device *pdev)
 		return -EBUSY;
 	}
 
-	ret = fpga_mgr_register(dev, "SoCFPGA Arria10 FPGA Manager",
-				 &socfpga_a10_fpga_mgr_ops, priv);
+	mgr = fpga_mgr_create(dev, "SoCFPGA Arria10 FPGA Manager",
+			      &socfpga_a10_fpga_mgr_ops, priv);
+	if (!mgr)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, mgr);
+
+	ret = fpga_mgr_register(mgr);
 	if (ret) {
+		fpga_mgr_free(mgr);
 		clk_disable_unprepare(priv->clk);
 		return ret;
 	}
@@ -534,7 +530,7 @@ static int socfpga_a10_fpga_remove(struct platform_device *pdev)
 	struct fpga_manager *mgr = platform_get_drvdata(pdev);
 	struct a10_fpga_priv *priv = mgr->priv;
 
-	fpga_mgr_unregister(&pdev->dev);
+	fpga_mgr_unregister(mgr);
 	clk_disable_unprepare(priv->clk);
 
 	return 0;
diff --git a/drivers/fpga/socfpga.c b/drivers/fpga/socfpga.c
index b6672e66cda6..959d71f26896 100644
--- a/drivers/fpga/socfpga.c
+++ b/drivers/fpga/socfpga.c
@@ -1,19 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * FPGA Manager Driver for Altera SOCFPGA
  *
  *  Copyright (C) 2013-2015 Altera Corporation
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <linux/completion.h>
 #include <linux/delay.h>
@@ -555,6 +544,7 @@ static int socfpga_fpga_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct socfpga_fpga_priv *priv;
+	struct fpga_manager *mgr;
 	struct resource *res;
 	int ret;
 
@@ -581,13 +571,25 @@ static int socfpga_fpga_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	return fpga_mgr_register(dev, "Altera SOCFPGA FPGA Manager",
-				 &socfpga_fpga_ops, priv);
+	mgr = fpga_mgr_create(dev, "Altera SOCFPGA FPGA Manager",
+			      &socfpga_fpga_ops, priv);
+	if (!mgr)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, mgr);
+
+	ret = fpga_mgr_register(mgr);
+	if (ret)
+		fpga_mgr_free(mgr);
+
+	return ret;
 }
 
 static int socfpga_fpga_remove(struct platform_device *pdev)
 {
-	fpga_mgr_unregister(&pdev->dev);
+	struct fpga_manager *mgr = platform_get_drvdata(pdev);
+
+	fpga_mgr_unregister(mgr);
 
 	return 0;
 }
diff --git a/drivers/fpga/ts73xx-fpga.c b/drivers/fpga/ts73xx-fpga.c
index f6a96b42e2ca..08efd1895b1b 100644
--- a/drivers/fpga/ts73xx-fpga.c
+++ b/drivers/fpga/ts73xx-fpga.c
@@ -116,7 +116,9 @@ static int ts73xx_fpga_probe(struct platform_device *pdev)
 {
 	struct device *kdev = &pdev->dev;
 	struct ts73xx_fpga_priv *priv;
+	struct fpga_manager *mgr;
 	struct resource *res;
+	int ret;
 
 	priv = devm_kzalloc(kdev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -131,13 +133,25 @@ static int ts73xx_fpga_probe(struct platform_device *pdev)
 		return PTR_ERR(priv->io_base);
 	}
 
-	return fpga_mgr_register(kdev, "TS-73xx FPGA Manager",
-				 &ts73xx_fpga_ops, priv);
+	mgr = fpga_mgr_create(kdev, "TS-73xx FPGA Manager",
+			      &ts73xx_fpga_ops, priv);
+	if (!mgr)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, mgr);
+
+	ret = fpga_mgr_register(mgr);
+	if (ret)
+		fpga_mgr_free(mgr);
+
+	return ret;
 }
 
 static int ts73xx_fpga_remove(struct platform_device *pdev)
 {
-	fpga_mgr_unregister(&pdev->dev);
+	struct fpga_manager *mgr = platform_get_drvdata(pdev);
+
+	fpga_mgr_unregister(mgr);
 
 	return 0;
 }
diff --git a/drivers/fpga/xilinx-pr-decoupler.c b/drivers/fpga/xilinx-pr-decoupler.c
index 0d7743089414..07ba1539e82c 100644
--- a/drivers/fpga/xilinx-pr-decoupler.c
+++ b/drivers/fpga/xilinx-pr-decoupler.c
@@ -94,6 +94,7 @@ MODULE_DEVICE_TABLE(of, xlnx_pr_decoupler_of_match);
 static int xlnx_pr_decoupler_probe(struct platform_device *pdev)
 {
 	struct xlnx_pr_decoupler_data *priv;
+	struct fpga_bridge *br;
 	int err;
 	struct resource *res;
 
@@ -120,16 +121,27 @@ static int xlnx_pr_decoupler_probe(struct platform_device *pdev)
 
 	clk_disable(priv->clk);
 
-	err = fpga_bridge_register(&pdev->dev, "Xilinx PR Decoupler",
-				   &xlnx_pr_decoupler_br_ops, priv);
+	br = fpga_bridge_create(&pdev->dev, "Xilinx PR Decoupler",
+				&xlnx_pr_decoupler_br_ops, priv);
+	if (!br) {
+		err = -ENOMEM;
+		goto err_clk;
+	}
+
+	platform_set_drvdata(pdev, br);
 
+	err = fpga_bridge_register(br);
 	if (err) {
 		dev_err(&pdev->dev, "unable to register Xilinx PR Decoupler");
-		clk_unprepare(priv->clk);
-		return err;
+		goto err_clk;
 	}
 
 	return 0;
+
+err_clk:
+	clk_unprepare(priv->clk);
+
+	return err;
 }
 
 static int xlnx_pr_decoupler_remove(struct platform_device *pdev)
@@ -137,7 +149,7 @@ static int xlnx_pr_decoupler_remove(struct platform_device *pdev)
 	struct fpga_bridge *bridge = platform_get_drvdata(pdev);
 	struct xlnx_pr_decoupler_data *p = bridge->priv;
 
-	fpga_bridge_unregister(&pdev->dev);
+	fpga_bridge_unregister(bridge);
 
 	clk_unprepare(p->clk);
 
diff --git a/drivers/fpga/xilinx-spi.c b/drivers/fpga/xilinx-spi.c
index 9b62a4c2a3df..8d1945966533 100644
--- a/drivers/fpga/xilinx-spi.c
+++ b/drivers/fpga/xilinx-spi.c
@@ -143,6 +143,8 @@ static const struct fpga_manager_ops xilinx_spi_ops = {
 static int xilinx_spi_probe(struct spi_device *spi)
 {
 	struct xilinx_spi_conf *conf;
+	struct fpga_manager *mgr;
+	int ret;
 
 	conf = devm_kzalloc(&spi->dev, sizeof(*conf), GFP_KERNEL);
 	if (!conf)
@@ -165,13 +167,25 @@ static int xilinx_spi_probe(struct spi_device *spi)
 		return PTR_ERR(conf->done);
 	}
 
-	return fpga_mgr_register(&spi->dev, "Xilinx Slave Serial FPGA Manager",
-				 &xilinx_spi_ops, conf);
+	mgr = fpga_mgr_create(&spi->dev, "Xilinx Slave Serial FPGA Manager",
+			      &xilinx_spi_ops, conf);
+	if (!mgr)
+		return -ENOMEM;
+
+	spi_set_drvdata(spi, mgr);
+
+	ret = fpga_mgr_register(mgr);
+	if (ret)
+		fpga_mgr_free(mgr);
+
+	return ret;
 }
 
 static int xilinx_spi_remove(struct spi_device *spi)
 {
-	fpga_mgr_unregister(&spi->dev);
+	struct fpga_manager *mgr = spi_get_drvdata(spi);
+
+	fpga_mgr_unregister(mgr);
 
 	return 0;
 }
diff --git a/drivers/fpga/zynq-fpga.c b/drivers/fpga/zynq-fpga.c
index 70b15b303471..3110e00121ca 100644
--- a/drivers/fpga/zynq-fpga.c
+++ b/drivers/fpga/zynq-fpga.c
@@ -558,6 +558,7 @@ static int zynq_fpga_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct zynq_fpga_priv *priv;
+	struct fpga_manager *mgr;
 	struct resource *res;
 	int err;
 
@@ -613,10 +614,17 @@ static int zynq_fpga_probe(struct platform_device *pdev)
 
 	clk_disable(priv->clk);
 
-	err = fpga_mgr_register(dev, "Xilinx Zynq FPGA Manager",
-				&zynq_fpga_ops, priv);
+	mgr = fpga_mgr_create(dev, "Xilinx Zynq FPGA Manager",
+			      &zynq_fpga_ops, priv);
+	if (!mgr)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, mgr);
+
+	err = fpga_mgr_register(mgr);
 	if (err) {
 		dev_err(dev, "unable to register FPGA manager\n");
+		fpga_mgr_free(mgr);
 		clk_unprepare(priv->clk);
 		return err;
 	}
@@ -632,7 +640,7 @@ static int zynq_fpga_remove(struct platform_device *pdev)
 	mgr = platform_get_drvdata(pdev);
 	priv = mgr->priv;
 
-	fpga_mgr_unregister(&pdev->dev);
+	fpga_mgr_unregister(mgr);
 
 	clk_unprepare(priv->clk);
 
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index b960f6f35abd..71c0ab46f216 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -22,6 +22,18 @@ menuconfig GPIOLIB
 
 if GPIOLIB
 
+config GPIOLIB_FASTPATH_LIMIT
+	int "Maximum number of GPIOs for fast path"
+	range 32 512
+	default 512
+	help
+	   This adjusts the point at which certain APIs will switch from
+	   using a stack allocated buffer to a dynamically allocated buffer.
+
+	   You shouldn't need to change this unless you really need to
+	   optimize either stack space or performance. Change this carefully
+	   since setting an incorrect value could cause stack corruption.
+
 config OF_GPIO
 	def_bool y
 	depends on OF
diff --git a/drivers/gpio/gpio-104-dio-48e.c b/drivers/gpio/gpio-104-dio-48e.c
index 31e22c93e844..9c4e07fcb74b 100644
--- a/drivers/gpio/gpio-104-dio-48e.c
+++ b/drivers/gpio/gpio-104-dio-48e.c
@@ -188,7 +188,7 @@ static int dio48e_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask,
 {
 	struct dio48e_gpio *const dio48egpio = gpiochip_get_data(chip);
 	size_t i;
-	const size_t ports[] = { 0, 1, 2, 4, 5, 6 };
+	static const size_t ports[] = { 0, 1, 2, 4, 5, 6 };
 	const unsigned int gpio_reg_size = 8;
 	unsigned int bits_offset;
 	size_t word_index;
diff --git a/drivers/gpio/gpio-104-idi-48.c b/drivers/gpio/gpio-104-idi-48.c
index f35632609379..2c9738adb3a6 100644
--- a/drivers/gpio/gpio-104-idi-48.c
+++ b/drivers/gpio/gpio-104-idi-48.c
@@ -94,7 +94,7 @@ static int idi_48_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask,
 {
 	struct idi_48_gpio *const idi48gpio = gpiochip_get_data(chip);
 	size_t i;
-	const size_t ports[] = { 0, 1, 2, 4, 5, 6 };
+	static const size_t ports[] = { 0, 1, 2, 4, 5, 6 };
 	const unsigned int gpio_reg_size = 8;
 	unsigned int bits_offset;
 	size_t word_index;
diff --git a/drivers/gpio/gpio-74xx-mmio.c b/drivers/gpio/gpio-74xx-mmio.c
index 0475e8ec96d0..49616ec815ee 100644
--- a/drivers/gpio/gpio-74xx-mmio.c
+++ b/drivers/gpio/gpio-74xx-mmio.c
@@ -105,27 +105,22 @@ static int mmio_74xx_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
 
 static int mmio_74xx_gpio_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *of_id;
 	struct mmio_74xx_gpio_priv *priv;
 	struct resource *res;
 	void __iomem *dat;
 	int err;
 
-	of_id = of_match_device(mmio_74xx_gpio_ids, &pdev->dev);
-	if (!of_id)
-		return -ENODEV;
-
 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
+	priv->flags = (uintptr_t)of_device_get_match_data(&pdev->dev);
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	dat = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(dat))
 		return PTR_ERR(dat);
 
-	priv->flags = (uintptr_t) of_id->data;
-
 	err = bgpio_init(&priv->gc, &pdev->dev,
 			 DIV_ROUND_UP(MMIO_74XX_BIT_CNT(priv->flags), 8),
 			 dat, NULL, NULL, NULL, NULL, 0);
diff --git a/drivers/gpio/gpio-aspeed.c b/drivers/gpio/gpio-aspeed.c
index 6f693b7d5220..5e89f1c74a33 100644
--- a/drivers/gpio/gpio-aspeed.c
+++ b/drivers/gpio/gpio-aspeed.c
@@ -54,6 +54,8 @@ struct aspeed_gpio {
 	u8 *offset_timer;
 	unsigned int timer_users[4];
 	struct clk *clk;
+
+	u32 *dcache;
 };
 
 struct aspeed_gpio_bank {
@@ -231,12 +233,13 @@ static void __aspeed_gpio_set(struct gpio_chip *gc, unsigned int offset,
 	u32 reg;
 
 	addr = bank_val_reg(gpio, bank, GPIO_DATA);
-	reg = ioread32(addr);
+	reg = gpio->dcache[GPIO_BANK(offset)];
 
 	if (val)
 		reg |= GPIO_BIT(offset);
 	else
 		reg &= ~GPIO_BIT(offset);
+	gpio->dcache[GPIO_BANK(offset)] = reg;
 
 	iowrite32(reg, addr);
 }
@@ -287,11 +290,10 @@ static int aspeed_gpio_dir_out(struct gpio_chip *gc,
 
 	spin_lock_irqsave(&gpio->lock, flags);
 
+	__aspeed_gpio_set(gc, offset, val);
 	reg = ioread32(bank_val_reg(gpio, bank, GPIO_DIR));
 	iowrite32(reg | GPIO_BIT(offset), bank_val_reg(gpio, bank, GPIO_DIR));
 
-	__aspeed_gpio_set(gc, offset, val);
-
 	spin_unlock_irqrestore(&gpio->lock, flags);
 
 	return 0;
@@ -852,7 +854,7 @@ static int __init aspeed_gpio_probe(struct platform_device *pdev)
 	const struct of_device_id *gpio_id;
 	struct aspeed_gpio *gpio;
 	struct resource *res;
-	int rc;
+	int rc, i, banks;
 
 	gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL);
 	if (!gpio)
@@ -893,6 +895,20 @@ static int __init aspeed_gpio_probe(struct platform_device *pdev)
 	gpio->chip.base = -1;
 	gpio->chip.irq.need_valid_mask = true;
 
+	/* Allocate a cache of the output registers */
+	banks = gpio->config->nr_gpios >> 5;
+	gpio->dcache = devm_kzalloc(&pdev->dev,
+				    sizeof(u32) * banks, GFP_KERNEL);
+	if (!gpio->dcache)
+		return -ENOMEM;
+
+	/* Populate it with initial values read from the HW */
+	for (i = 0; i < banks; i++) {
+		const struct aspeed_gpio_bank *bank = &aspeed_gpio_banks[i];
+		gpio->dcache[i] = ioread32(gpio->base + bank->val_regs +
+					   GPIO_DATA);
+	}
+
 	rc = devm_gpiochip_add_data(&pdev->dev, &gpio->chip, gpio);
 	if (rc < 0)
 		return rc;
diff --git a/drivers/gpio/gpio-dwapb.c b/drivers/gpio/gpio-dwapb.c
index 226977f78482..7a2de3de6571 100644
--- a/drivers/gpio/gpio-dwapb.c
+++ b/drivers/gpio/gpio-dwapb.c
@@ -441,14 +441,19 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio,
 	irq_gc->chip_types[1].handler = handle_edge_irq;
 
 	if (!pp->irq_shared) {
-		irq_set_chained_handler_and_data(pp->irq, dwapb_irq_handler,
-						 gpio);
+		int i;
+
+		for (i = 0; i < pp->ngpio; i++) {
+			if (pp->irq[i] >= 0)
+				irq_set_chained_handler_and_data(pp->irq[i],
+						dwapb_irq_handler, gpio);
+		}
 	} else {
 		/*
 		 * Request a shared IRQ since where MFD would have devices
 		 * using the same irq pin
 		 */
-		err = devm_request_irq(gpio->dev, pp->irq,
+		err = devm_request_irq(gpio->dev, pp->irq[0],
 				       dwapb_irq_handler_mfd,
 				       IRQF_SHARED, "gpio-dwapb-mfd", gpio);
 		if (err) {
@@ -524,7 +529,7 @@ static int dwapb_gpio_add_port(struct dwapb_gpio *gpio,
 	if (pp->idx == 0)
 		port->gc.set_config = dwapb_gpio_set_config;
 
-	if (pp->irq)
+	if (pp->has_irq)
 		dwapb_configure_irqs(gpio, port, pp);
 
 	err = gpiochip_add_data(&port->gc, port);
@@ -535,7 +540,7 @@ static int dwapb_gpio_add_port(struct dwapb_gpio *gpio,
 		port->is_registered = true;
 
 	/* Add GPIO-signaled ACPI event support */
-	if (pp->irq)
+	if (pp->has_irq)
 		acpi_gpiochip_request_interrupts(&port->gc);
 
 	return err;
@@ -557,7 +562,7 @@ dwapb_gpio_get_pdata(struct device *dev)
 	struct dwapb_platform_data *pdata;
 	struct dwapb_port_property *pp;
 	int nports;
-	int i;
+	int i, j;
 
 	nports = device_get_child_node_count(dev);
 	if (nports == 0)
@@ -575,6 +580,8 @@ dwapb_gpio_get_pdata(struct device *dev)
 
 	i = 0;
 	device_for_each_child_node(dev, fwnode)  {
+		struct device_node *np = NULL;
+
 		pp = &pdata->properties[i++];
 		pp->fwnode = fwnode;
 
@@ -594,23 +601,35 @@ dwapb_gpio_get_pdata(struct device *dev)
 			pp->ngpio = 32;
 		}
 
+		pp->irq_shared	= false;
+		pp->gpio_base	= -1;
+
 		/*
 		 * Only port A can provide interrupts in all configurations of
 		 * the IP.
 		 */
-		if (dev->of_node && pp->idx == 0 &&
-			fwnode_property_read_bool(fwnode,
+		if (pp->idx != 0)
+			continue;
+
+		if (dev->of_node && fwnode_property_read_bool(fwnode,
 						  "interrupt-controller")) {
-			pp->irq = irq_of_parse_and_map(to_of_node(fwnode), 0);
-			if (!pp->irq)
-				dev_warn(dev, "no irq for port%d\n", pp->idx);
+			np = to_of_node(fwnode);
 		}
 
-		if (has_acpi_companion(dev) && pp->idx == 0)
-			pp->irq = platform_get_irq(to_platform_device(dev), 0);
+		for (j = 0; j < pp->ngpio; j++) {
+			pp->irq[j] = -ENXIO;
 
-		pp->irq_shared	= false;
-		pp->gpio_base	= -1;
+			if (np)
+				pp->irq[j] = of_irq_get(np, j);
+			else if (has_acpi_companion(dev))
+				pp->irq[j] = platform_get_irq(to_platform_device(dev), j);
+
+			if (pp->irq[j] >= 0)
+				pp->has_irq = true;
+		}
+
+		if (!pp->has_irq)
+			dev_warn(dev, "no irq for port%d\n", pp->idx);
 	}
 
 	return pdata;
@@ -684,13 +703,7 @@ static int dwapb_gpio_probe(struct platform_device *pdev)
 
 	gpio->flags = 0;
 	if (dev->of_node) {
-		const struct of_device_id *of_devid;
-
-		of_devid = of_match_device(dwapb_of_match, dev);
-		if (of_devid) {
-			if (of_devid->data)
-				gpio->flags = (uintptr_t)of_devid->data;
-		}
+		gpio->flags = (uintptr_t)of_device_get_match_data(dev);
 	} else if (has_acpi_companion(dev)) {
 		const struct acpi_device_id *acpi_id;
 
diff --git a/drivers/gpio/gpio-eic-sprd.c b/drivers/gpio/gpio-eic-sprd.c
index de7dd939c043..e0d6a0a7bc69 100644
--- a/drivers/gpio/gpio-eic-sprd.c
+++ b/drivers/gpio/gpio-eic-sprd.c
@@ -300,6 +300,7 @@ static int sprd_eic_irq_set_type(struct irq_data *data, unsigned int flow_type)
 	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
 	struct sprd_eic *sprd_eic = gpiochip_get_data(chip);
 	u32 offset = irqd_to_hwirq(data);
+	int state;
 
 	switch (sprd_eic->type) {
 	case SPRD_EIC_DEBOUNCE:
@@ -310,6 +311,17 @@ static int sprd_eic_irq_set_type(struct irq_data *data, unsigned int flow_type)
 		case IRQ_TYPE_LEVEL_LOW:
 			sprd_eic_update(chip, offset, SPRD_EIC_DBNC_IEV, 0);
 			break;
+		case IRQ_TYPE_EDGE_RISING:
+		case IRQ_TYPE_EDGE_FALLING:
+		case IRQ_TYPE_EDGE_BOTH:
+			state = sprd_eic_get(chip, offset);
+			if (state)
+				sprd_eic_update(chip, offset,
+						SPRD_EIC_DBNC_IEV, 0);
+			else
+				sprd_eic_update(chip, offset,
+						SPRD_EIC_DBNC_IEV, 1);
+			break;
 		default:
 			return -ENOTSUPP;
 		}
@@ -324,6 +336,17 @@ static int sprd_eic_irq_set_type(struct irq_data *data, unsigned int flow_type)
 		case IRQ_TYPE_LEVEL_LOW:
 			sprd_eic_update(chip, offset, SPRD_EIC_LATCH_INTPOL, 1);
 			break;
+		case IRQ_TYPE_EDGE_RISING:
+		case IRQ_TYPE_EDGE_FALLING:
+		case IRQ_TYPE_EDGE_BOTH:
+			state = sprd_eic_get(chip, offset);
+			if (state)
+				sprd_eic_update(chip, offset,
+						SPRD_EIC_LATCH_INTPOL, 0);
+			else
+				sprd_eic_update(chip, offset,
+						SPRD_EIC_LATCH_INTPOL, 1);
+			break;
 		default:
 			return -ENOTSUPP;
 		}
@@ -405,6 +428,55 @@ static int sprd_eic_irq_set_type(struct irq_data *data, unsigned int flow_type)
 	return 0;
 }
 
+static void sprd_eic_toggle_trigger(struct gpio_chip *chip, unsigned int irq,
+				    unsigned int offset)
+{
+	struct sprd_eic *sprd_eic = gpiochip_get_data(chip);
+	struct irq_data *data = irq_get_irq_data(irq);
+	u32 trigger = irqd_get_trigger_type(data);
+	int state, post_state;
+
+	/*
+	 * The debounce EIC and latch EIC can only support level trigger, so we
+	 * can toggle the level trigger to emulate the edge trigger.
+	 */
+	if ((sprd_eic->type != SPRD_EIC_DEBOUNCE &&
+	     sprd_eic->type != SPRD_EIC_LATCH) ||
+	    !(trigger & IRQ_TYPE_EDGE_BOTH))
+		return;
+
+	sprd_eic_irq_mask(data);
+	state = sprd_eic_get(chip, offset);
+
+retry:
+	switch (sprd_eic->type) {
+	case SPRD_EIC_DEBOUNCE:
+		if (state)
+			sprd_eic_update(chip, offset, SPRD_EIC_DBNC_IEV, 0);
+		else
+			sprd_eic_update(chip, offset, SPRD_EIC_DBNC_IEV, 1);
+		break;
+	case SPRD_EIC_LATCH:
+		if (state)
+			sprd_eic_update(chip, offset, SPRD_EIC_LATCH_INTPOL, 0);
+		else
+			sprd_eic_update(chip, offset, SPRD_EIC_LATCH_INTPOL, 1);
+		break;
+	default:
+		sprd_eic_irq_unmask(data);
+		return;
+	}
+
+	post_state = sprd_eic_get(chip, offset);
+	if (state != post_state) {
+		dev_warn(chip->parent, "EIC level was changed.\n");
+		state = post_state;
+		goto retry;
+	}
+
+	sprd_eic_irq_unmask(data);
+}
+
 static int sprd_eic_match_chip_by_type(struct gpio_chip *chip, void *data)
 {
 	enum sprd_eic_type type = *(enum sprd_eic_type *)data;
@@ -448,6 +520,7 @@ static void sprd_eic_handle_one_type(struct gpio_chip *chip)
 					bank * SPRD_EIC_PER_BANK_NR + n);
 
 			generic_handle_irq(girq);
+			sprd_eic_toggle_trigger(chip, girq, n);
 		}
 	}
 }
diff --git a/drivers/gpio/gpio-ge.c b/drivers/gpio/gpio-ge.c
index 1fe2d3418f2f..636952769bc8 100644
--- a/drivers/gpio/gpio-ge.c
+++ b/drivers/gpio/gpio-ge.c
@@ -52,8 +52,6 @@ MODULE_DEVICE_TABLE(of, gef_gpio_ids);
 
 static int __init gef_gpio_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *of_id =
-		of_match_device(gef_gpio_ids, &pdev->dev);
 	struct gpio_chip *gc;
 	void __iomem *regs;
 	int ret;
@@ -82,7 +80,7 @@ static int __init gef_gpio_probe(struct platform_device *pdev)
 	}
 
 	gc->base = -1;
-	gc->ngpio = (u16)(uintptr_t)of_id->data;
+	gc->ngpio = (u16)(uintptr_t)of_device_get_match_data(&pdev->dev);
 	gc->of_gpio_n_cells = 2;
 	gc->of_node = pdev->dev.of_node;
 
diff --git a/drivers/gpio/gpio-gpio-mm.c b/drivers/gpio/gpio-gpio-mm.c
index d496cc56c2a2..b56ff2efbf36 100644
--- a/drivers/gpio/gpio-gpio-mm.c
+++ b/drivers/gpio/gpio-gpio-mm.c
@@ -177,7 +177,7 @@ static int gpiomm_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask,
 {
 	struct gpiomm_gpio *const gpiommgpio = gpiochip_get_data(chip);
 	size_t i;
-	const size_t ports[] = { 0, 1, 2, 4, 5, 6 };
+	static const size_t ports[] = { 0, 1, 2, 4, 5, 6 };
 	const unsigned int gpio_reg_size = 8;
 	unsigned int bits_offset;
 	size_t word_index;
diff --git a/drivers/gpio/gpio-ingenic.c b/drivers/gpio/gpio-ingenic.c
index 15fb2bc796a8..e738e384a5ca 100644
--- a/drivers/gpio/gpio-ingenic.c
+++ b/drivers/gpio/gpio-ingenic.c
@@ -285,8 +285,6 @@ MODULE_DEVICE_TABLE(of, ingenic_gpio_of_match);
 static int ingenic_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	const struct of_device_id *of_id = of_match_device(
-			ingenic_gpio_of_match, dev);
 	struct ingenic_gpio_chip *jzgc;
 	u32 bank;
 	int err;
@@ -323,7 +321,7 @@ static int ingenic_gpio_probe(struct platform_device *pdev)
 	jzgc->gc.parent = dev;
 	jzgc->gc.of_node = dev->of_node;
 	jzgc->gc.owner = THIS_MODULE;
-	jzgc->version = (enum jz_version)of_id->data;
+	jzgc->version = (enum jz_version)of_device_get_match_data(dev);
 
 	jzgc->gc.set = ingenic_gpio_set;
 	jzgc->gc.get = ingenic_gpio_get;
diff --git a/drivers/gpio/gpio-loongson.c b/drivers/gpio/gpio-loongson.c
index 92c4fe7b2677..16cfbe9e72fe 100644
--- a/drivers/gpio/gpio-loongson.c
+++ b/drivers/gpio/gpio-loongson.c
@@ -17,9 +17,11 @@
 #include <linux/module.h>
 #include <linux/spinlock.h>
 #include <linux/err.h>
+#include <linux/gpio/driver.h>
+#include <linux/platform_device.h>
+#include <linux/bitops.h>
 #include <asm/types.h>
 #include <loongson.h>
-#include <linux/gpio.h>
 
 #define STLS2F_N_GPIO		4
 #define STLS3A_N_GPIO		16
@@ -30,86 +32,108 @@
 #define LOONGSON_N_GPIO	STLS2F_N_GPIO
 #endif
 
+/*
+ * Offset into the register where we read lines, we write them from offset 0.
+ * This offset is the only thing that stand between us and using
+ * GPIO_GENERIC.
+ */
 #define LOONGSON_GPIO_IN_OFFSET	16
 
 static DEFINE_SPINLOCK(gpio_lock);
 
-static int loongson_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
+static int loongson_gpio_get_value(struct gpio_chip *chip, unsigned gpio)
 {
-	u32 temp;
-	u32 mask;
+	u32 val;
 
 	spin_lock(&gpio_lock);
-	mask = 1 << gpio;
-	temp = LOONGSON_GPIOIE;
-	temp |= mask;
-	LOONGSON_GPIOIE = temp;
+	val = LOONGSON_GPIODATA;
 	spin_unlock(&gpio_lock);
 
-	return 0;
+	return !!(val & BIT(gpio + LOONGSON_GPIO_IN_OFFSET));
 }
 
-static int loongson_gpio_direction_output(struct gpio_chip *chip,
-		unsigned gpio, int level)
+static void loongson_gpio_set_value(struct gpio_chip *chip,
+		unsigned gpio, int value)
+{
+	u32 val;
+
+	spin_lock(&gpio_lock);
+	val = LOONGSON_GPIODATA;
+	if (value)
+		val |= BIT(gpio);
+	else
+		val &= ~BIT(gpio);
+	LOONGSON_GPIODATA = val;
+	spin_unlock(&gpio_lock);
+}
+
+static int loongson_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
 {
 	u32 temp;
-	u32 mask;
 
-	gpio_set_value(gpio, level);
 	spin_lock(&gpio_lock);
-	mask = 1 << gpio;
 	temp = LOONGSON_GPIOIE;
-	temp &= (~mask);
+	temp |= BIT(gpio);
 	LOONGSON_GPIOIE = temp;
 	spin_unlock(&gpio_lock);
 
 	return 0;
 }
 
-static int loongson_gpio_get_value(struct gpio_chip *chip, unsigned gpio)
+static int loongson_gpio_direction_output(struct gpio_chip *chip,
+		unsigned gpio, int level)
 {
-	u32 val;
-	u32 mask;
+	u32 temp;
 
-	mask = 1 << (gpio + LOONGSON_GPIO_IN_OFFSET);
+	loongson_gpio_set_value(chip, gpio, level);
 	spin_lock(&gpio_lock);
-	val = LOONGSON_GPIODATA;
+	temp = LOONGSON_GPIOIE;
+	temp &= ~BIT(gpio);
+	LOONGSON_GPIOIE = temp;
 	spin_unlock(&gpio_lock);
 
-	return (val & mask) != 0;
+	return 0;
 }
 
-static void loongson_gpio_set_value(struct gpio_chip *chip,
-		unsigned gpio, int value)
+static int loongson_gpio_probe(struct platform_device *pdev)
 {
-	u32 val;
-	u32 mask;
-
-	mask = 1 << gpio;
-
-	spin_lock(&gpio_lock);
-	val = LOONGSON_GPIODATA;
-	if (value)
-		val |= mask;
-	else
-		val &= (~mask);
-	LOONGSON_GPIODATA = val;
-	spin_unlock(&gpio_lock);
+	struct gpio_chip *gc;
+	struct device *dev = &pdev->dev;
+
+	gc = devm_kzalloc(dev, sizeof(*gc), GFP_KERNEL);
+	if (!gc)
+		return -ENOMEM;
+
+	gc->label = "loongson-gpio-chip";
+	gc->base = 0;
+	gc->ngpio = LOONGSON_N_GPIO;
+	gc->get = loongson_gpio_get_value;
+	gc->set = loongson_gpio_set_value;
+	gc->direction_input = loongson_gpio_direction_input;
+	gc->direction_output = loongson_gpio_direction_output;
+
+	return gpiochip_add_data(gc, NULL);
 }
 
-static struct gpio_chip loongson_chip = {
-	.label                  = "Loongson-gpio-chip",
-	.direction_input        = loongson_gpio_direction_input,
-	.get                    = loongson_gpio_get_value,
-	.direction_output       = loongson_gpio_direction_output,
-	.set                    = loongson_gpio_set_value,
-	.base			= 0,
-	.ngpio                  = LOONGSON_N_GPIO,
-	.can_sleep		= false,
+static struct platform_driver loongson_gpio_driver = {
+	.driver = {
+		.name = "loongson-gpio",
+	},
+	.probe = loongson_gpio_probe,
 };
 
 static int __init loongson_gpio_setup(void)
 {
-	return gpiochip_add_data(&loongson_chip, NULL);
+	struct platform_device *pdev;
+	int ret;
+
+	ret = platform_driver_register(&loongson_gpio_driver);
+	if (ret) {
+		pr_err("error registering loongson GPIO driver\n");
+		return ret;
+	}
+
+	pdev = platform_device_register_simple("loongson-gpio", -1, NULL, 0);
+	return PTR_ERR_OR_ZERO(pdev);
 }
 postcore_initcall(loongson_gpio_setup);
diff --git a/drivers/gpio/gpio-lp3943.c b/drivers/gpio/gpio-lp3943.c
index 6dc6725403ec..c3a3b9b7b553 100644
--- a/drivers/gpio/gpio-lp3943.c
+++ b/drivers/gpio/gpio-lp3943.c
@@ -12,7 +12,7 @@
 
 #include <linux/bitops.h>
 #include <linux/err.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/i2c.h>
 #include <linux/mfd/lp3943.h>
 #include <linux/module.h>
diff --git a/drivers/gpio/gpio-lp873x.c b/drivers/gpio/gpio-lp873x.c
index df0ad2cef0d2..801995dd9b26 100644
--- a/drivers/gpio/gpio-lp873x.c
+++ b/drivers/gpio/gpio-lp873x.c
@@ -14,7 +14,7 @@
  * Based on the TPS65218 driver
  */
 
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
diff --git a/drivers/gpio/gpio-lpc32xx.c b/drivers/gpio/gpio-lpc32xx.c
index 92b3ae2a6735..aa74cc4d8b14 100644
--- a/drivers/gpio/gpio-lpc32xx.c
+++ b/drivers/gpio/gpio-lpc32xx.c
@@ -20,9 +20,8 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/errno.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/platform_device.h>
 #include <linux/module.h>
 
diff --git a/drivers/gpio/gpio-lynxpoint.c b/drivers/gpio/gpio-lynxpoint.c
index 1e557b10d73e..b5b5e500e72c 100644
--- a/drivers/gpio/gpio-lynxpoint.c
+++ b/drivers/gpio/gpio-lynxpoint.c
@@ -25,7 +25,7 @@
 #include <linux/types.h>
 #include <linux/bitops.h>
 #include <linux/interrupt.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/slab.h>
 #include <linux/acpi.h>
 #include <linux/platform_device.h>
diff --git a/drivers/gpio/gpio-max730x.c b/drivers/gpio/gpio-max730x.c
index 946d09195598..198a36b07773 100644
--- a/drivers/gpio/gpio-max730x.c
+++ b/drivers/gpio/gpio-max730x.c
@@ -35,7 +35,7 @@
 #include <linux/platform_device.h>
 #include <linux/mutex.h>
 #include <linux/spi/max7301.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/slab.h>
 
 /*
diff --git a/drivers/gpio/gpio-mc33880.c b/drivers/gpio/gpio-mc33880.c
index 0f0df7956264..18a5a58d634a 100644
--- a/drivers/gpio/gpio-mc33880.c
+++ b/drivers/gpio/gpio-mc33880.c
@@ -24,7 +24,7 @@
 #include <linux/mutex.h>
 #include <linux/spi/spi.h>
 #include <linux/spi/mc33880.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/slab.h>
 #include <linux/module.h>
 
diff --git a/drivers/gpio/gpio-mc9s08dz60.c b/drivers/gpio/gpio-mc9s08dz60.c
index 2fcad5b9cca5..d8d846d2189a 100644
--- a/drivers/gpio/gpio-mc9s08dz60.c
+++ b/drivers/gpio/gpio-mc9s08dz60.c
@@ -18,7 +18,7 @@
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/i2c.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 
 #define GPIO_GROUP_NUM 2
 #define GPIO_NUM_PER_GROUP 8
diff --git a/drivers/gpio/gpio-ml-ioh.c b/drivers/gpio/gpio-ml-ioh.c
index b3678bd1c120..e2bee27eb526 100644
--- a/drivers/gpio/gpio-ml-ioh.c
+++ b/drivers/gpio/gpio-ml-ioh.c
@@ -18,7 +18,7 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/pci.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 
diff --git a/drivers/gpio/gpio-mm-lantiq.c b/drivers/gpio/gpio-mm-lantiq.c
index b1cf76dd84ba..b0754fe69e77 100644
--- a/drivers/gpio/gpio-mm-lantiq.c
+++ b/drivers/gpio/gpio-mm-lantiq.c
@@ -11,7 +11,7 @@
 #include <linux/types.h>
 #include <linux/platform_device.h>
 #include <linux/mutex.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/of.h>
 #include <linux/of_gpio.h>
 #include <linux/io.h>
diff --git a/drivers/gpio/gpio-mockup.c b/drivers/gpio/gpio-mockup.c
index 76c2fe91a901..d66b7a768ecd 100644
--- a/drivers/gpio/gpio-mockup.c
+++ b/drivers/gpio/gpio-mockup.c
@@ -1,15 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * GPIO Testing Device Driver
  *
  * Copyright (C) 2014  Kamlakant Patel <kamlakant.patel@broadcom.com>
  * Copyright (C) 2015-2016  Bamvor Jian Zhang <bamv2005@gmail.com>
  * Copyright (C) 2017 Bartosz Golaszewski <brgl@bgdev.pl>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- *
  */
 
 #include <linux/init.h>
diff --git a/drivers/gpio/gpio-msic.c b/drivers/gpio/gpio-msic.c
index 6cb67595d15f..3b34dbecef99 100644
--- a/drivers/gpio/gpio-msic.c
+++ b/drivers/gpio/gpio-msic.c
@@ -24,7 +24,7 @@
 #include <linux/slab.h>
 #include <linux/interrupt.h>
 #include <linux/init.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/platform_device.h>
 #include <linux/mfd/intel_msic.h>
 
diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c
index 45c65f805fd6..6e02148c208b 100644
--- a/drivers/gpio/gpio-mvebu.c
+++ b/drivers/gpio/gpio-mvebu.c
@@ -36,7 +36,8 @@
 #include <linux/bitops.h>
 #include <linux/clk.h>
 #include <linux/err.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/consumer.h>
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/irq.h>
@@ -51,8 +52,6 @@
 #include <linux/regmap.h>
 #include <linux/slab.h>
 
-#include "gpiolib.h"
-
 /*
  * GPIO unit register offsets.
  */
@@ -608,19 +607,16 @@ static int mvebu_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm)
 	if (mvpwm->gpiod) {
 		ret = -EBUSY;
 	} else {
-		desc = gpio_to_desc(mvchip->chip.base + pwm->hwpwm);
-		if (!desc) {
-			ret = -ENODEV;
+		desc = gpiochip_request_own_desc(&mvchip->chip,
+						 pwm->hwpwm, "mvebu-pwm");
+		if (IS_ERR(desc)) {
+			ret = PTR_ERR(desc);
 			goto out;
 		}
 
-		ret = gpiod_request(desc, "mvebu-pwm");
-		if (ret)
-			goto out;
-
 		ret = gpiod_direction_output(desc, 0);
 		if (ret) {
-			gpiod_free(desc);
+			gpiochip_free_own_desc(desc);
 			goto out;
 		}
 
@@ -637,7 +633,7 @@ static void mvebu_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm)
 	unsigned long flags;
 
 	spin_lock_irqsave(&mvpwm->lock, flags);
-	gpiod_free(mvpwm->gpiod);
+	gpiochip_free_own_desc(mvpwm->gpiod);
 	mvpwm->gpiod = NULL;
 	spin_unlock_irqrestore(&mvpwm->lock, flags);
 }
diff --git a/drivers/gpio/gpio-mxc.c b/drivers/gpio/gpio-mxc.c
index 5245a2fe62ae..2f2829966d4c 100644
--- a/drivers/gpio/gpio-mxc.c
+++ b/drivers/gpio/gpio-mxc.c
@@ -1,25 +1,13 @@
-/*
- * MXC GPIO support. (c) 2008 Daniel Mack <daniel@caiaq.de>
- * Copyright 2008 Juergen Beisert, kernel@pengutronix.de
- *
- * Based on code from Freescale Semiconductor,
- * Authors: Daniel Mack, Juergen Beisert.
- * Copyright (C) 2004-2010 Freescale Semiconductor, Inc. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
+// SPDX-License-Identifier: GPL-2.0+
+//
+// MXC GPIO support. (c) 2008 Daniel Mack <daniel@caiaq.de>
+// Copyright 2008 Juergen Beisert, kernel@pengutronix.de
+//
+// Based on code from Freescale Semiconductor,
+// Authors: Daniel Mack, Juergen Beisert.
+// Copyright (C) 2004-2010 Freescale Semiconductor, Inc. All Rights Reserved.
+
+#include <linux/clk.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -30,8 +18,6 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/gpio/driver.h>
-/* FIXME: for gpio_get_value() replace this with direct register read */
-#include <linux/gpio.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/bug.h>
@@ -62,6 +48,7 @@ struct mxc_gpio_hwdata {
 struct mxc_gpio_port {
 	struct list_head node;
 	void __iomem *base;
+	struct clk *clk;
 	int irq;
 	int irq_high;
 	struct irq_domain *domain;
@@ -174,7 +161,6 @@ static int gpio_set_irq_type(struct irq_data *d, u32 type)
 	struct mxc_gpio_port *port = gc->private;
 	u32 bit, val;
 	u32 gpio_idx = d->hwirq;
-	u32 gpio = port->gc.base + gpio_idx;
 	int edge;
 	void __iomem *reg = port->base;
 
@@ -190,13 +176,13 @@ static int gpio_set_irq_type(struct irq_data *d, u32 type)
 		if (GPIO_EDGE_SEL >= 0) {
 			edge = GPIO_INT_BOTH_EDGES;
 		} else {
-			val = gpio_get_value(gpio);
+			val = port->gc.get(&port->gc, gpio_idx);
 			if (val) {
 				edge = GPIO_INT_LOW_LEV;
-				pr_debug("mxc: set GPIO %d to low trigger\n", gpio);
+				pr_debug("mxc: set GPIO %d to low trigger\n", gpio_idx);
 			} else {
 				edge = GPIO_INT_HIGH_LEV;
-				pr_debug("mxc: set GPIO %d to high trigger\n", gpio);
+				pr_debug("mxc: set GPIO %d to high trigger\n", gpio_idx);
 			}
 			port->both_edges |= 1 << gpio_idx;
 		}
@@ -437,6 +423,17 @@ static int mxc_gpio_probe(struct platform_device *pdev)
 	if (port->irq < 0)
 		return port->irq;
 
+	/* the controller clock is optional */
+	port->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(port->clk))
+		port->clk = NULL;
+
+	err = clk_prepare_enable(port->clk);
+	if (err) {
+		dev_err(&pdev->dev, "Unable to enable clock.\n");
+		return err;
+	}
+
 	/* disable the interrupt and clear the status */
 	writel(0, port->base + GPIO_IMR);
 	writel(~0, port->base + GPIO_ISR);
@@ -505,6 +502,7 @@ static int mxc_gpio_probe(struct platform_device *pdev)
 out_irqdomain_remove:
 	irq_domain_remove(port->domain);
 out_bgio:
+	clk_disable_unprepare(port->clk);
 	dev_info(&pdev->dev, "%s failed with errno %d\n", __func__, err);
 	return err;
 }
diff --git a/drivers/gpio/gpio-mxs.c b/drivers/gpio/gpio-mxs.c
index 435def22445d..e2831ee70cdc 100644
--- a/drivers/gpio/gpio-mxs.c
+++ b/drivers/gpio/gpio-mxs.c
@@ -1,24 +1,10 @@
-/*
- * MXC GPIO support. (c) 2008 Daniel Mack <daniel@caiaq.de>
- * Copyright 2008 Juergen Beisert, kernel@pengutronix.de
- *
- * Based on code from Freescale,
- * Copyright (C) 2004-2010 Freescale Semiconductor, Inc. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA  02110-1301, USA.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// MXC GPIO support. (c) 2008 Daniel Mack <daniel@caiaq.de>
+// Copyright 2008 Juergen Beisert, kernel@pengutronix.de
+//
+// Based on code from Freescale,
+// Copyright (C) 2004-2010 Freescale Semiconductor, Inc. All Rights Reserved.
 
 #include <linux/err.h>
 #include <linux/init.h>
@@ -290,8 +276,6 @@ MODULE_DEVICE_TABLE(of, mxs_gpio_dt_ids);
 
 static int mxs_gpio_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *of_id =
-			of_match_device(mxs_gpio_dt_ids, &pdev->dev);
 	struct device_node *np = pdev->dev.of_node;
 	struct device_node *parent;
 	static void __iomem *base;
@@ -306,7 +290,7 @@ static int mxs_gpio_probe(struct platform_device *pdev)
 	port->id = of_alias_get_id(np, "gpio");
 	if (port->id < 0)
 		return port->id;
-	port->devid = (enum mxs_gpio_id) of_id->data;
+	port->devid = (enum mxs_gpio_id)of_device_get_match_data(&pdev->dev);
 	port->dev = &pdev->dev;
 	port->irq = platform_get_irq(pdev, 0);
 	if (port->irq < 0)
diff --git a/drivers/gpio/gpio-octeon.c b/drivers/gpio/gpio-octeon.c
index 96a8a8cb2729..1b19c88ea7bb 100644
--- a/drivers/gpio/gpio-octeon.c
+++ b/drivers/gpio/gpio-octeon.c
@@ -9,7 +9,7 @@
 #include <linux/platform_device.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/io.h>
 
 #include <asm/octeon/octeon.h>
diff --git a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c
index 35971a341c40..d1afedf4dcbf 100644
--- a/drivers/gpio/gpio-omap.c
+++ b/drivers/gpio/gpio-omap.c
@@ -24,7 +24,7 @@
 #include <linux/pm.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/bitops.h>
 #include <linux/platform_data/gpio-omap.h>
 
diff --git a/drivers/gpio/gpio-palmas.c b/drivers/gpio/gpio-palmas.c
index 3d818195e351..05b0cd5dcf11 100644
--- a/drivers/gpio/gpio-palmas.c
+++ b/drivers/gpio/gpio-palmas.c
@@ -18,7 +18,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/mfd/palmas.h>
@@ -159,13 +159,9 @@ static int palmas_gpio_probe(struct platform_device *pdev)
 	struct palmas_platform_data *palmas_pdata;
 	struct palmas_gpio *palmas_gpio;
 	int ret;
-	const struct of_device_id *match;
 	const struct palmas_device_data *dev_data;
 
-	match = of_match_device(of_palmas_gpio_match, &pdev->dev);
-	if (!match)
-		return -ENODEV;
-	dev_data = match->data;
+	dev_data = of_device_get_match_data(&pdev->dev);
 	if (!dev_data)
 		dev_data = &palmas_dev_data;
 
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index d2ead4b1cf61..c55ad157e820 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -12,7 +12,7 @@
  */
 
 #include <linux/acpi.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/init.h>
@@ -25,29 +25,44 @@
 
 #include <asm/unaligned.h>
 
-#define PCA953X_INPUT		0
-#define PCA953X_OUTPUT		1
-#define PCA953X_INVERT		2
-#define PCA953X_DIRECTION	3
+#define PCA953X_INPUT		0x00
+#define PCA953X_OUTPUT		0x01
+#define PCA953X_INVERT		0x02
+#define PCA953X_DIRECTION	0x03
 
 #define REG_ADDR_AI		0x80
 
-#define PCA957X_IN		0
-#define PCA957X_INVRT		1
-#define PCA957X_BKEN		2
-#define PCA957X_PUPD		3
-#define PCA957X_CFG		4
-#define PCA957X_OUT		5
-#define PCA957X_MSK		6
-#define PCA957X_INTS		7
-
-#define PCAL953X_IN_LATCH	34
-#define PCAL953X_INT_MASK	37
-#define PCAL953X_INT_STAT	38
+#define PCA957X_IN		0x00
+#define PCA957X_INVRT		0x01
+#define PCA957X_BKEN		0x02
+#define PCA957X_PUPD		0x03
+#define PCA957X_CFG		0x04
+#define PCA957X_OUT		0x05
+#define PCA957X_MSK		0x06
+#define PCA957X_INTS		0x07
+
+#define PCAL953X_OUT_STRENGTH	0x20
+#define PCAL953X_IN_LATCH	0x22
+#define PCAL953X_PULL_EN	0x23
+#define PCAL953X_PULL_SEL	0x24
+#define PCAL953X_INT_MASK	0x25
+#define PCAL953X_INT_STAT	0x26
+#define PCAL953X_OUT_CONF	0x27
+
+#define PCAL6524_INT_EDGE	0x28
+#define PCAL6524_INT_CLR	0x2a
+#define PCAL6524_IN_STATUS	0x2b
+#define PCAL6524_OUT_INDCONF	0x2c
+#define PCAL6524_DEBOUNCE	0x2d
 
 #define PCA_GPIO_MASK		0x00FF
+
+#define PCAL_GPIO_MASK		0x1f
+#define PCAL_PINCTRL_MASK	0xe0
+
 #define PCA_INT			0x0100
 #define PCA_PCAL		0x0200
+#define PCA_LATCH_INT (PCA_PCAL | PCA_INT)
 #define PCA953X_TYPE		0x1000
 #define PCA957X_TYPE		0x2000
 #define PCA_TYPE_MASK		0xF000
@@ -207,9 +222,11 @@ static int pca957x_write_regs_16(struct pca953x_chip *chip, int reg, u8 *val)
 static int pca953x_write_regs_24(struct pca953x_chip *chip, int reg, u8 *val)
 {
 	int bank_shift = fls((chip->gpio_chip.ngpio - 1) / BANK_SZ);
+	int addr = (reg & PCAL_GPIO_MASK) << bank_shift;
+	int pinctrl = (reg & PCAL_PINCTRL_MASK) << 1;
 
 	return i2c_smbus_write_i2c_block_data(chip->client,
-					      (reg << bank_shift) | REG_ADDR_AI,
+					      pinctrl | addr | REG_ADDR_AI,
 					      NBANK(chip), val);
 }
 
@@ -249,9 +266,11 @@ static int pca953x_read_regs_16(struct pca953x_chip *chip, int reg, u8 *val)
 static int pca953x_read_regs_24(struct pca953x_chip *chip, int reg, u8 *val)
 {
 	int bank_shift = fls((chip->gpio_chip.ngpio - 1) / BANK_SZ);
+	int addr = (reg & PCAL_GPIO_MASK) << bank_shift;
+	int pinctrl = (reg & PCAL_PINCTRL_MASK) << 1;
 
 	return i2c_smbus_read_i2c_block_data(chip->client,
-					     (reg << bank_shift) | REG_ADDR_AI,
+					     pinctrl | addr | REG_ADDR_AI,
 					     NBANK(chip), val);
 }
 
@@ -522,6 +541,15 @@ static int pca953x_irq_set_type(struct irq_data *d, unsigned int type)
 	return 0;
 }
 
+static void pca953x_irq_shutdown(struct irq_data *d)
+{
+	struct pca953x_chip *chip = irq_data_get_irq_chip_data(d);
+	u8 mask = 1 << (d->hwirq % BANK_SZ);
+
+	chip->irq_trig_raise[d->hwirq / BANK_SZ] &= ~mask;
+	chip->irq_trig_fall[d->hwirq / BANK_SZ] &= ~mask;
+}
+
 static struct irq_chip pca953x_irq_chip = {
 	.name			= "pca953x",
 	.irq_mask		= pca953x_irq_mask,
@@ -529,6 +557,7 @@ static struct irq_chip pca953x_irq_chip = {
 	.irq_bus_lock		= pca953x_irq_bus_lock,
 	.irq_bus_sync_unlock	= pca953x_irq_bus_sync_unlock,
 	.irq_set_type		= pca953x_irq_set_type,
+	.irq_shutdown		= pca953x_irq_shutdown,
 };
 
 static bool pca953x_irq_pending(struct pca953x_chip *chip, u8 *pending)
@@ -810,13 +839,11 @@ static int pca953x_probe(struct i2c_client *client,
 		chip->driver_data = i2c_id->driver_data;
 	} else {
 		const struct acpi_device_id *acpi_id;
-		const struct of_device_id *match;
+		struct device *dev = &client->dev;
 
-		match = of_match_device(pca953x_dt_ids, &client->dev);
-		if (match) {
-			chip->driver_data = (int)(uintptr_t)match->data;
-		} else {
-			acpi_id = acpi_match_device(pca953x_acpi_ids, &client->dev);
+		chip->driver_data = (uintptr_t)of_device_get_match_data(dev);
+		if (!chip->driver_data) {
+			acpi_id = acpi_match_device(pca953x_acpi_ids, dev);
 			if (!acpi_id) {
 				ret = -ENODEV;
 				goto err_exit;
@@ -936,8 +963,8 @@ static const struct of_device_id pca953x_dt_ids[] = {
 	{ .compatible = "nxp,pca9575", .data = OF_957X(16, PCA_INT), },
 	{ .compatible = "nxp,pca9698", .data = OF_953X(40, 0), },
 
-	{ .compatible = "nxp,pcal6524", .data = OF_953X(24, PCA_INT), },
-	{ .compatible = "nxp,pcal9555a", .data = OF_953X(16, PCA_INT), },
+	{ .compatible = "nxp,pcal6524", .data = OF_953X(24, PCA_LATCH_INT), },
+	{ .compatible = "nxp,pcal9555a", .data = OF_953X(16, PCA_LATCH_INT), },
 
 	{ .compatible = "maxim,max7310", .data = OF_953X( 8, 0), },
 	{ .compatible = "maxim,max7312", .data = OF_953X(16, PCA_INT), },
diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c
index 38fbb420c6cd..adf72dda25a2 100644
--- a/drivers/gpio/gpio-pcf857x.c
+++ b/drivers/gpio/gpio-pcf857x.c
@@ -18,7 +18,7 @@
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/i2c.h>
 #include <linux/platform_data/pcf857x.h>
 #include <linux/interrupt.h>
diff --git a/drivers/gpio/gpio-pch.c b/drivers/gpio/gpio-pch.c
index 68c6d0c5a6d1..ffce0ab912ed 100644
--- a/drivers/gpio/gpio-pch.c
+++ b/drivers/gpio/gpio-pch.c
@@ -17,7 +17,7 @@
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/pci.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/slab.h>
diff --git a/drivers/gpio/gpio-pl061.c b/drivers/gpio/gpio-pl061.c
index b70974cb9ef1..2afd9de84a0d 100644
--- a/drivers/gpio/gpio-pl061.c
+++ b/drivers/gpio/gpio-pl061.c
@@ -20,7 +20,7 @@
 #include <linux/irq.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/bitops.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/device.h>
 #include <linux/amba/bus.h>
 #include <linux/slab.h>
diff --git a/drivers/gpio/gpio-pmic-eic-sprd.c b/drivers/gpio/gpio-pmic-eic-sprd.c
index 66d68d991162..29e044ff4b17 100644
--- a/drivers/gpio/gpio-pmic-eic-sprd.c
+++ b/drivers/gpio/gpio-pmic-eic-sprd.c
@@ -178,6 +178,14 @@ static int sprd_pmic_eic_irq_set_type(struct irq_data *data,
 	case IRQ_TYPE_LEVEL_LOW:
 		pmic_eic->reg[REG_IEV] = 0;
 		break;
+	case IRQ_TYPE_EDGE_RISING:
+	case IRQ_TYPE_EDGE_FALLING:
+	case IRQ_TYPE_EDGE_BOTH:
+		/*
+		 * Will set the trigger level according to current EIC level
+		 * in irq_bus_sync_unlock() interface, so here nothing to do.
+		 */
+		break;
 	default:
 		return -ENOTSUPP;
 	}
@@ -197,11 +205,22 @@ static void sprd_pmic_eic_bus_sync_unlock(struct irq_data *data)
 {
 	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
 	struct sprd_pmic_eic *pmic_eic = gpiochip_get_data(chip);
+	u32 trigger = irqd_get_trigger_type(data);
 	u32 offset = irqd_to_hwirq(data);
+	int state;
 
 	/* Set irq type */
-	sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IEV,
-			     pmic_eic->reg[REG_IEV]);
+	if (trigger & IRQ_TYPE_EDGE_BOTH) {
+		state = sprd_pmic_eic_get(chip, offset);
+		if (state)
+			sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IEV, 0);
+		else
+			sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IEV, 1);
+	} else {
+		sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IEV,
+				     pmic_eic->reg[REG_IEV]);
+	}
+
 	/* Set irq unmask */
 	sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IE,
 			     pmic_eic->reg[REG_IE]);
@@ -212,6 +231,35 @@ static void sprd_pmic_eic_bus_sync_unlock(struct irq_data *data)
 	mutex_unlock(&pmic_eic->buslock);
 }
 
+static void sprd_pmic_eic_toggle_trigger(struct gpio_chip *chip,
+					 unsigned int irq, unsigned int offset)
+{
+	u32 trigger = irq_get_trigger_type(irq);
+	int state, post_state;
+
+	if (!(trigger & IRQ_TYPE_EDGE_BOTH))
+		return;
+
+	state = sprd_pmic_eic_get(chip, offset);
+retry:
+	if (state)
+		sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IEV, 0);
+	else
+		sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IEV, 1);
+
+	post_state = sprd_pmic_eic_get(chip, offset);
+	if (state != post_state) {
+		dev_warn(chip->parent, "PMIC EIC level was changed.\n");
+		state = post_state;
+		goto retry;
+	}
+
+	/* Set irq unmask */
+	sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IE, 1);
+	/* Generate trigger start pulse for debounce EIC */
+	sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_TRIG, 1);
+}
+
 static irqreturn_t sprd_pmic_eic_irq_handler(int irq, void *data)
 {
 	struct sprd_pmic_eic *pmic_eic = data;
@@ -233,6 +281,12 @@ static irqreturn_t sprd_pmic_eic_irq_handler(int irq, void *data)
 
 		girq = irq_find_mapping(chip->irq.domain, n);
 		handle_nested_irq(girq);
+
+		/*
+		 * The PMIC EIC can only support level trigger, so we can
+		 * toggle the level trigger to emulate the edge trigger.
+		 */
+		sprd_pmic_eic_toggle_trigger(chip, girq, n);
 	}
 
 	return IRQ_HANDLED;
diff --git a/drivers/gpio/gpio-pxa.c b/drivers/gpio/gpio-pxa.c
index f480fb896963..1e66f808051c 100644
--- a/drivers/gpio/gpio-pxa.c
+++ b/drivers/gpio/gpio-pxa.c
@@ -14,7 +14,7 @@
 #include <linux/module.h>
 #include <linux/clk.h>
 #include <linux/err.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/gpio-pxa.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -579,15 +579,9 @@ static int pxa_gpio_probe_dt(struct platform_device *pdev,
 			     struct pxa_gpio_chip *pchip)
 {
 	int nr_gpios;
-	const struct of_device_id *of_id =
-				of_match_device(pxa_gpio_dt_ids, &pdev->dev);
 	const struct pxa_gpio_id *gpio_id;
 
-	if (!of_id || !of_id->data) {
-		dev_err(&pdev->dev, "Failed to find gpio controller\n");
-		return -EFAULT;
-	}
-	gpio_id = of_id->data;
+	gpio_id = of_device_get_match_data(&pdev->dev);
 	gpio_type = gpio_id->type;
 
 	nr_gpios = gpio_id->gpio_nums;
diff --git a/drivers/gpio/gpio-stmpe.c b/drivers/gpio/gpio-stmpe.c
index f8d7d1cd8488..8d6a5a7e612d 100644
--- a/drivers/gpio/gpio-stmpe.c
+++ b/drivers/gpio/gpio-stmpe.c
@@ -363,13 +363,15 @@ static struct irq_chip stmpe_gpio_irq_chip = {
 	.irq_set_type		= stmpe_gpio_irq_set_type,
 };
 
+#define MAX_GPIOS 24
+
 static irqreturn_t stmpe_gpio_irq(int irq, void *dev)
 {
 	struct stmpe_gpio *stmpe_gpio = dev;
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	u8 statmsbreg;
 	int num_banks = DIV_ROUND_UP(stmpe->num_gpios, 8);
-	u8 status[num_banks];
+	u8 status[DIV_ROUND_UP(MAX_GPIOS, 8)];
 	int ret;
 	int i;
 
@@ -434,6 +436,11 @@ static int stmpe_gpio_probe(struct platform_device *pdev)
 	struct stmpe_gpio *stmpe_gpio;
 	int ret, irq;
 
+	if (stmpe->num_gpios > MAX_GPIOS) {
+		dev_err(&pdev->dev, "Need to increase maximum GPIO number\n");
+		return -EINVAL;
+	}
+
 	stmpe_gpio = kzalloc(sizeof(*stmpe_gpio), GFP_KERNEL);
 	if (!stmpe_gpio)
 		return -ENOMEM;
diff --git a/drivers/gpio/gpio-syscon.c b/drivers/gpio/gpio-syscon.c
index 537cec7583fc..8b0a69c5ba88 100644
--- a/drivers/gpio/gpio-syscon.c
+++ b/drivers/gpio/gpio-syscon.c
@@ -182,20 +182,15 @@ MODULE_DEVICE_TABLE(of, syscon_gpio_ids);
 static int syscon_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	const struct of_device_id *of_id;
 	struct syscon_gpio_priv *priv;
 	struct device_node *np = dev->of_node;
 	int ret;
 
-	of_id = of_match_device(syscon_gpio_ids, dev);
-	if (!of_id)
-		return -ENODEV;
-
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	priv->data = of_id->data;
+	priv->data = of_device_get_match_data(dev);
 
 	if (priv->data->compatible) {
 		priv->syscon = syscon_regmap_lookup_by_compatible(
@@ -205,6 +200,8 @@ static int syscon_gpio_probe(struct platform_device *pdev)
 	} else {
 		priv->syscon =
 			syscon_regmap_lookup_by_phandle(np, "gpio,syscon-dev");
+		if (IS_ERR(priv->syscon) && np->parent)
+			priv->syscon = syscon_node_to_regmap(np->parent);
 		if (IS_ERR(priv->syscon))
 			return PTR_ERR(priv->syscon);
 
diff --git a/drivers/gpio/gpio-ts4900.c b/drivers/gpio/gpio-ts4900.c
index 5bd21725e604..1da8d0586329 100644
--- a/drivers/gpio/gpio-ts4900.c
+++ b/drivers/gpio/gpio-ts4900.c
@@ -128,15 +128,10 @@ MODULE_DEVICE_TABLE(of, ts4900_gpio_of_match_table);
 static int ts4900_gpio_probe(struct i2c_client *client,
 			const struct i2c_device_id *id)
 {
-	const struct of_device_id *match;
 	struct ts4900_gpio_priv *priv;
 	u32 ngpio;
 	int ret;
 
-	match = of_match_device(ts4900_gpio_of_match_table, &client->dev);
-	if (!match)
-		return -EINVAL;
-
 	if (of_property_read_u32(client->dev.of_node, "ngpios", &ngpio))
 		ngpio = DEFAULT_PIN_NUMBER;
 
@@ -148,7 +143,7 @@ static int ts4900_gpio_probe(struct i2c_client *client,
 	priv->gpio_chip.label = "ts4900-gpio";
 	priv->gpio_chip.ngpio = ngpio;
 	priv->gpio_chip.parent = &client->dev;
-	priv->input_bit = (uintptr_t)match->data;
+	priv->input_bit = (uintptr_t)of_device_get_match_data(&client->dev);
 
 	priv->regmap = devm_regmap_init_i2c(client, &ts4900_regmap_config);
 	if (IS_ERR(priv->regmap)) {
diff --git a/drivers/gpio/gpio-vf610.c b/drivers/gpio/gpio-vf610.c
index 4610cc2938ad..d4ad6d0e02a2 100644
--- a/drivers/gpio/gpio-vf610.c
+++ b/drivers/gpio/gpio-vf610.c
@@ -254,8 +254,6 @@ static struct irq_chip vf610_gpio_irq_chip = {
 
 static int vf610_gpio_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *of_id = of_match_device(vf610_gpio_dt_ids,
-							   &pdev->dev);
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
 	struct vf610_gpio_port *port;
@@ -267,7 +265,7 @@ static int vf610_gpio_probe(struct platform_device *pdev)
 	if (!port)
 		return -ENOMEM;
 
-	port->sdata = of_id->data;
+	port->sdata = of_device_get_match_data(dev);
 	iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	port->base = devm_ioremap_resource(dev, iores);
 	if (IS_ERR(port->base))
diff --git a/drivers/gpio/gpio-xlp.c b/drivers/gpio/gpio-xlp.c
index e74bd43a6974..8e4275eaa7d7 100644
--- a/drivers/gpio/gpio-xlp.c
+++ b/drivers/gpio/gpio-xlp.c
@@ -322,14 +322,7 @@ static int xlp_gpio_probe(struct platform_device *pdev)
 		return irq;
 
 	if (pdev->dev.of_node) {
-		const struct of_device_id *of_id;
-
-		of_id = of_match_device(xlp_gpio_of_ids, &pdev->dev);
-		if (!of_id) {
-			dev_err(&pdev->dev, "Unable to match OF ID\n");
-			return -ENODEV;
-		}
-		soc_type = (uintptr_t) of_id->data;
+		soc_type = (uintptr_t)of_device_get_match_data(&pdev->dev);
 	} else {
 		const struct acpi_device_id *acpi_id;
 
diff --git a/drivers/gpio/gpio-xra1403.c b/drivers/gpio/gpio-xra1403.c
index 8d4c8e99b251..8711a7907568 100644
--- a/drivers/gpio/gpio-xra1403.c
+++ b/drivers/gpio/gpio-xra1403.c
@@ -39,6 +39,7 @@
 #define XRA_REIR  0x10 /* Input Rising Edge Interrupt Enable */
 #define XRA_FEIR  0x12 /* Input Falling Edge Interrupt Enable */
 #define XRA_IFR   0x14 /* Input Filter Enable/Disable */
+#define XRA_LAST  0x15 /* Bounds */
 
 struct xra1403 {
 	struct gpio_chip  chip;
@@ -50,7 +51,7 @@ static const struct regmap_config xra1403_regmap_cfg = {
 		.pad_bits = 1,
 		.val_bits = 8,
 
-		.max_register = XRA_IFR | 0x01,
+		.max_register = XRA_LAST,
 };
 
 static unsigned int to_reg(unsigned int reg, unsigned int offset)
@@ -126,21 +127,16 @@ static void xra1403_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 {
 	int reg;
 	struct xra1403 *xra = gpiochip_get_data(chip);
-	int *value;
+	int value[XRA_LAST];
 	int i;
 	unsigned int gcr;
 	unsigned int gsr;
 
-	value = kmalloc_array(xra1403_regmap_cfg.max_register, sizeof(*value),
-				GFP_KERNEL);
-	if (!value)
-		return;
-
 	seq_puts(s, "xra reg:");
-	for (reg = 0; reg <= xra1403_regmap_cfg.max_register; reg++)
+	for (reg = 0; reg <= XRA_LAST; reg++)
 		seq_printf(s, " %2.2x", reg);
 	seq_puts(s, "\n  value:");
-	for (reg = 0; reg < xra1403_regmap_cfg.max_register; reg++) {
+	for (reg = 0; reg < XRA_LAST; reg++) {
 		regmap_read(xra->regmap, reg, &value[reg]);
 		seq_printf(s, " %2.2x", value[reg]);
 	}
@@ -159,7 +155,6 @@ static void xra1403_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 			   (gcr & BIT(i)) ? "in" : "out",
 			   (gsr & BIT(i)) ? "hi" : "lo");
 	}
-	kfree(value);
 }
 #else
 #define xra1403_dbg_show NULL
diff --git a/drivers/gpio/gpio-zynq.c b/drivers/gpio/gpio-zynq.c
index 75ee877e5cd5..3f5fcdd5a429 100644
--- a/drivers/gpio/gpio-zynq.c
+++ b/drivers/gpio/gpio-zynq.c
@@ -665,10 +665,8 @@ static void zynq_gpio_restore_context(struct zynq_gpio *gpio)
 
 static int __maybe_unused zynq_gpio_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	int irq = platform_get_irq(pdev, 0);
-	struct irq_data *data = irq_get_irq_data(irq);
-	struct zynq_gpio *gpio = platform_get_drvdata(pdev);
+	struct zynq_gpio *gpio = dev_get_drvdata(dev);
+	struct irq_data *data = irq_get_irq_data(gpio->irq);
 
 	if (!irqd_is_wakeup_set(data)) {
 		zynq_gpio_save_context(gpio);
@@ -680,10 +678,8 @@ static int __maybe_unused zynq_gpio_suspend(struct device *dev)
 
 static int __maybe_unused zynq_gpio_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	int irq = platform_get_irq(pdev, 0);
-	struct irq_data *data = irq_get_irq_data(irq);
-	struct zynq_gpio *gpio = platform_get_drvdata(pdev);
+	struct zynq_gpio *gpio = dev_get_drvdata(dev);
+	struct irq_data *data = irq_get_irq_data(gpio->irq);
 	int ret;
 
 	if (!irqd_is_wakeup_set(data)) {
@@ -831,7 +827,7 @@ static int zynq_gpio_probe(struct platform_device *pdev)
 	chip->free = zynq_gpio_free;
 	chip->direction_input = zynq_gpio_dir_in;
 	chip->direction_output = zynq_gpio_dir_out;
-	chip->base = -1;
+	chip->base = of_alias_get_id(pdev->dev.of_node, "gpio");
 	chip->ngpio = gpio->p_data->ngpio;
 
 	/* Retrieve GPIO clock */
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index 586d15137c03..28d968088131 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -210,11 +210,8 @@ static struct gpio_desc *of_find_regulator_gpio(struct device *dev, const char *
 	if (!con_id)
 		return ERR_PTR(-ENOENT);
 
-	for (i = 0; i < ARRAY_SIZE(whitelist); i++)
-		if (!strcmp(con_id, whitelist[i]))
-			break;
-
-	if (i == ARRAY_SIZE(whitelist))
+	i = match_string(whitelist, ARRAY_SIZE(whitelist), con_id);
+	if (i < 0)
 		return ERR_PTR(-ENOENT);
 
 	desc = of_get_named_gpiod_flags(np, con_id, 0, of_flags);
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index d8ccb500872f..2ef4597ea31b 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -61,6 +61,11 @@ static struct bus_type gpio_bus_type = {
 	.name = "gpio",
 };
 
+/*
+ * Number of GPIOs to use for the fast path in set array
+ */
+#define FASTPATH_NGPIO CONFIG_GPIOLIB_FASTPATH_LIMIT
+
 /* gpio_lock prevents conflicts during gpio_desc[] table updates.
  * While any GPIO is requested, its gpio_chip is not removable;
  * each GPIO's "requested" flag serves as a lock and refcount.
@@ -71,6 +76,9 @@ static DEFINE_MUTEX(gpio_lookup_lock);
 static LIST_HEAD(gpio_lookup_list);
 LIST_HEAD(gpio_devices);
 
+static DEFINE_MUTEX(gpio_machine_hogs_mutex);
+static LIST_HEAD(gpio_machine_hogs);
+
 static void gpiochip_free_hogs(struct gpio_chip *chip);
 static int gpiochip_add_irqchip(struct gpio_chip *gpiochip,
 				struct lock_class_key *lock_key,
@@ -450,12 +458,11 @@ static long linehandle_ioctl(struct file *filep, unsigned int cmd,
 			vals[i] = !!ghd.values[i];
 
 		/* Reuse the array setting function */
-		gpiod_set_array_value_complex(false,
+		return gpiod_set_array_value_complex(false,
 					      true,
 					      lh->numdescs,
 					      lh->descs,
 					      vals);
-		return 0;
 	}
 	return -EINVAL;
 }
@@ -1172,6 +1179,41 @@ static int gpiochip_setup_dev(struct gpio_device *gdev)
 	return status;
 }
 
+static void gpiochip_machine_hog(struct gpio_chip *chip, struct gpiod_hog *hog)
+{
+	struct gpio_desc *desc;
+	int rv;
+
+	desc = gpiochip_get_desc(chip, hog->chip_hwnum);
+	if (IS_ERR(desc)) {
+		pr_err("%s: unable to get GPIO desc: %ld\n",
+		       __func__, PTR_ERR(desc));
+		return;
+	}
+
+	if (test_bit(FLAG_IS_HOGGED, &desc->flags))
+		return;
+
+	rv = gpiod_hog(desc, hog->line_name, hog->lflags, hog->dflags);
+	if (rv)
+		pr_err("%s: unable to hog GPIO line (%s:%u): %d\n",
+		       __func__, chip->label, hog->chip_hwnum, rv);
+}
+
+static void machine_gpiochip_add(struct gpio_chip *chip)
+{
+	struct gpiod_hog *hog;
+
+	mutex_lock(&gpio_machine_hogs_mutex);
+
+	list_for_each_entry(hog, &gpio_machine_hogs, list) {
+		if (!strcmp(chip->label, hog->chip_label))
+			gpiochip_machine_hog(chip, hog);
+	}
+
+	mutex_unlock(&gpio_machine_hogs_mutex);
+}
+
 static void gpiochip_setup_devs(void)
 {
 	struct gpio_device *gdev;
@@ -1244,6 +1286,10 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
 		goto err_free_descs;
 	}
 
+	if (chip->ngpio > FASTPATH_NGPIO)
+		chip_warn(chip, "line cnt %u is greater than fast path cnt %u\n",
+		chip->ngpio, FASTPATH_NGPIO);
+
 	gdev->label = kstrdup_const(chip->label ?: "unknown", GFP_KERNEL);
 	if (!gdev->label) {
 		status = -ENOMEM;
@@ -1327,6 +1373,8 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
 
 	acpi_gpiochip_add(chip);
 
+	machine_gpiochip_add(chip);
+
 	/*
 	 * By first adding the chardev, and then adding the device,
 	 * we get a device node entry in sysfs under
@@ -2078,6 +2126,11 @@ EXPORT_SYMBOL_GPL(gpiochip_generic_config);
  * @pctldev: the pin controller to map to
  * @gpio_offset: the start offset in the current gpio_chip number space
  * @pin_group: name of the pin group inside the pin controller
+ *
+ * Calling this function directly from a DeviceTree-supported
+ * pinctrl driver is DEPRECATED. Please see Section 2.1 of
+ * Documentation/devicetree/bindings/gpio/gpio.txt on how to
+ * bind pinctrl and gpio drivers via the "gpio-ranges" property.
  */
 int gpiochip_add_pingroup_range(struct gpio_chip *chip,
 			struct pinctrl_dev *pctldev,
@@ -2131,6 +2184,11 @@ EXPORT_SYMBOL_GPL(gpiochip_add_pingroup_range);
  *
  * Returns:
  * 0 on success, or a negative error-code on failure.
+ *
+ * Calling this function directly from a DeviceTree-supported
+ * pinctrl driver is DEPRECATED. Please see Section 2.1 of
+ * Documentation/devicetree/bindings/gpio/gpio.txt on how to
+ * bind pinctrl and gpio drivers via the "gpio-ranges" property.
  */
 int gpiochip_add_pin_range(struct gpio_chip *chip, const char *pinctl_name,
 			   unsigned int gpio_offset, unsigned int pin_offset,
@@ -2719,16 +2777,28 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep,
 
 	while (i < array_size) {
 		struct gpio_chip *chip = desc_array[i]->gdev->chip;
-		unsigned long mask[BITS_TO_LONGS(chip->ngpio)];
-		unsigned long bits[BITS_TO_LONGS(chip->ngpio)];
+		unsigned long fastpath[2 * BITS_TO_LONGS(FASTPATH_NGPIO)];
+		unsigned long *mask, *bits;
 		int first, j, ret;
 
+		if (likely(chip->ngpio <= FASTPATH_NGPIO)) {
+			mask = fastpath;
+		} else {
+			mask = kmalloc_array(2 * BITS_TO_LONGS(chip->ngpio),
+					   sizeof(*mask),
+					   can_sleep ? GFP_KERNEL : GFP_ATOMIC);
+			if (!mask)
+				return -ENOMEM;
+		}
+
+		bits = mask + BITS_TO_LONGS(chip->ngpio);
+		bitmap_zero(mask, chip->ngpio);
+
 		if (!can_sleep)
 			WARN_ON(chip->can_sleep);
 
 		/* collect all inputs belonging to the same chip */
 		first = i;
-		memset(mask, 0, sizeof(mask));
 		do {
 			const struct gpio_desc *desc = desc_array[i];
 			int hwgpio = gpio_chip_hwgpio(desc);
@@ -2739,8 +2809,11 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep,
 			 (desc_array[i]->gdev->chip == chip));
 
 		ret = gpio_chip_get_multiple(chip, mask, bits);
-		if (ret)
+		if (ret) {
+			if (mask != fastpath)
+				kfree(mask);
 			return ret;
+		}
 
 		for (j = first; j < i; j++) {
 			const struct gpio_desc *desc = desc_array[j];
@@ -2752,6 +2825,9 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep,
 			value_array[j] = value;
 			trace_gpio_value(desc_to_gpio(desc), 1, value);
 		}
+
+		if (mask != fastpath)
+			kfree(mask);
 	}
 	return 0;
 }
@@ -2935,7 +3011,7 @@ static void gpio_chip_set_multiple(struct gpio_chip *chip,
 	}
 }
 
-void gpiod_set_array_value_complex(bool raw, bool can_sleep,
+int gpiod_set_array_value_complex(bool raw, bool can_sleep,
 				   unsigned int array_size,
 				   struct gpio_desc **desc_array,
 				   int *value_array)
@@ -2944,14 +3020,26 @@ void gpiod_set_array_value_complex(bool raw, bool can_sleep,
 
 	while (i < array_size) {
 		struct gpio_chip *chip = desc_array[i]->gdev->chip;
-		unsigned long mask[BITS_TO_LONGS(chip->ngpio)];
-		unsigned long bits[BITS_TO_LONGS(chip->ngpio)];
+		unsigned long fastpath[2 * BITS_TO_LONGS(FASTPATH_NGPIO)];
+		unsigned long *mask, *bits;
 		int count = 0;
 
+		if (likely(chip->ngpio <= FASTPATH_NGPIO)) {
+			mask = fastpath;
+		} else {
+			mask = kmalloc_array(2 * BITS_TO_LONGS(chip->ngpio),
+					   sizeof(*mask),
+					   can_sleep ? GFP_KERNEL : GFP_ATOMIC);
+			if (!mask)
+				return -ENOMEM;
+		}
+
+		bits = mask + BITS_TO_LONGS(chip->ngpio);
+		bitmap_zero(mask, chip->ngpio);
+
 		if (!can_sleep)
 			WARN_ON(chip->can_sleep);
 
-		memset(mask, 0, sizeof(mask));
 		do {
 			struct gpio_desc *desc = desc_array[i];
 			int hwgpio = gpio_chip_hwgpio(desc);
@@ -2982,7 +3070,11 @@ void gpiod_set_array_value_complex(bool raw, bool can_sleep,
 		/* push collected bits to outputs */
 		if (count != 0)
 			gpio_chip_set_multiple(chip, mask, bits);
+
+		if (mask != fastpath)
+			kfree(mask);
 	}
+	return 0;
 }
 
 /**
@@ -3057,13 +3149,13 @@ EXPORT_SYMBOL_GPL(gpiod_set_value);
  * This function should be called from contexts where we cannot sleep, and will
  * complain if the GPIO chip functions potentially sleep.
  */
-void gpiod_set_raw_array_value(unsigned int array_size,
+int gpiod_set_raw_array_value(unsigned int array_size,
 			 struct gpio_desc **desc_array, int *value_array)
 {
 	if (!desc_array)
-		return;
-	gpiod_set_array_value_complex(true, false, array_size, desc_array,
-				      value_array);
+		return -EINVAL;
+	return gpiod_set_array_value_complex(true, false, array_size,
+					desc_array, value_array);
 }
 EXPORT_SYMBOL_GPL(gpiod_set_raw_array_value);
 
@@ -3383,14 +3475,14 @@ EXPORT_SYMBOL_GPL(gpiod_set_value_cansleep);
  *
  * This function is to be called from contexts that can sleep.
  */
-void gpiod_set_raw_array_value_cansleep(unsigned int array_size,
+int gpiod_set_raw_array_value_cansleep(unsigned int array_size,
 					struct gpio_desc **desc_array,
 					int *value_array)
 {
 	might_sleep_if(extra_checks);
 	if (!desc_array)
-		return;
-	gpiod_set_array_value_complex(true, true, array_size, desc_array,
+		return -EINVAL;
+	return gpiod_set_array_value_complex(true, true, array_size, desc_array,
 				      value_array);
 }
 EXPORT_SYMBOL_GPL(gpiod_set_raw_array_value_cansleep);
@@ -3463,6 +3555,33 @@ void gpiod_remove_lookup_table(struct gpiod_lookup_table *table)
 }
 EXPORT_SYMBOL_GPL(gpiod_remove_lookup_table);
 
+/**
+ * gpiod_add_hogs() - register a set of GPIO hogs from machine code
+ * @hogs: table of gpio hog entries with a zeroed sentinel at the end
+ */
+void gpiod_add_hogs(struct gpiod_hog *hogs)
+{
+	struct gpio_chip *chip;
+	struct gpiod_hog *hog;
+
+	mutex_lock(&gpio_machine_hogs_mutex);
+
+	for (hog = &hogs[0]; hog->chip_label; hog++) {
+		list_add_tail(&hog->list, &gpio_machine_hogs);
+
+		/*
+		 * The chip may have been registered earlier, so check if it
+		 * exists and, if so, try to hog the line now.
+		 */
+		chip = find_chip_by_name(hog->chip_label);
+		if (chip)
+			gpiochip_machine_hog(chip, hog);
+	}
+
+	mutex_unlock(&gpio_machine_hogs_mutex);
+}
+EXPORT_SYMBOL_GPL(gpiod_add_hogs);
+
 static struct gpiod_lookup_table *gpiod_find_lookup_table(struct device *dev)
 {
 	const char *dev_id = dev ? dev_name(dev) : NULL;
diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
index ad456b6f9d8b..1a8e20363861 100644
--- a/drivers/gpio/gpiolib.h
+++ b/drivers/gpio/gpiolib.h
@@ -188,7 +188,7 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep,
 				  unsigned int array_size,
 				  struct gpio_desc **desc_array,
 				  int *value_array);
-void gpiod_set_array_value_complex(bool raw, bool can_sleep,
+int gpiod_set_array_value_complex(bool raw, bool can_sleep,
 				   unsigned int array_size,
 				   struct gpio_desc **desc_array,
 				   int *value_array);
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index deeefa7a1773..2a72d2feb76d 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -49,16 +49,17 @@ config DRM_DEBUG_MM
 
 	  If in doubt, say "N".
 
-config DRM_DEBUG_MM_SELFTEST
-	tristate "kselftests for DRM range manager (struct drm_mm)"
+config DRM_DEBUG_SELFTEST
+	tristate "kselftests for DRM"
 	depends on DRM
 	depends on DEBUG_KERNEL
 	select PRIME_NUMBERS
 	select DRM_LIB_RANDOM
+	select DRM_KMS_HELPER
 	default n
 	help
-	  This option provides a kernel module that can be used to test
-	  the DRM range manager (drm_mm) and its API. This option is not
+	  This option provides kernel modules that can be used to run
+	  various selftests on parts of the DRM api. This option is not
 	  useful for distributions or general kernels, but only for kernel
 	  developers working on DRM and associated drivers.
 
@@ -267,6 +268,8 @@ source "drivers/gpu/drm/amd/amdkfd/Kconfig"
 
 source "drivers/gpu/drm/imx/Kconfig"
 
+source "drivers/gpu/drm/v3d/Kconfig"
+
 source "drivers/gpu/drm/vc4/Kconfig"
 
 source "drivers/gpu/drm/etnaviv/Kconfig"
@@ -289,6 +292,8 @@ source "drivers/gpu/drm/pl111/Kconfig"
 
 source "drivers/gpu/drm/tve200/Kconfig"
 
+source "drivers/gpu/drm/xen/Kconfig"
+
 # Keep legacy drivers last
 
 menuconfig DRM_LEGACY
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 50093ff4479b..ef9f3dab287f 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -43,7 +43,7 @@ drm_kms_helper-$(CONFIG_DRM_KMS_CMA_HELPER) += drm_fb_cma_helper.o
 drm_kms_helper-$(CONFIG_DRM_DP_AUX_CHARDEV) += drm_dp_aux_dev.o
 
 obj-$(CONFIG_DRM_KMS_HELPER) += drm_kms_helper.o
-obj-$(CONFIG_DRM_DEBUG_MM_SELFTEST) += selftests/
+obj-$(CONFIG_DRM_DEBUG_SELFTEST) += selftests/
 
 obj-$(CONFIG_DRM)	+= drm.o
 obj-$(CONFIG_DRM_MIPI_DSI) += drm_mipi_dsi.o
@@ -61,6 +61,7 @@ obj-$(CONFIG_DRM_MGA)	+= mga/
 obj-$(CONFIG_DRM_I810)	+= i810/
 obj-$(CONFIG_DRM_I915)	+= i915/
 obj-$(CONFIG_DRM_MGAG200) += mgag200/
+obj-$(CONFIG_DRM_V3D)  += v3d/
 obj-$(CONFIG_DRM_VC4)  += vc4/
 obj-$(CONFIG_DRM_CIRRUS_QEMU) += cirrus/
 obj-$(CONFIG_DRM_SIS)   += sis/
@@ -103,3 +104,4 @@ obj-$(CONFIG_DRM_MXSFB)	+= mxsfb/
 obj-$(CONFIG_DRM_TINYDRM) += tinydrm/
 obj-$(CONFIG_DRM_PL111) += pl111/
 obj-$(CONFIG_DRM_TVE200) += tve200/
+obj-$(CONFIG_DRM_XEN) += xen/
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 2ca2b5154d52..bfd332c95b61 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -56,13 +56,18 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
-	ci_smc.o ci_dpm.o dce_v8_0.o gfx_v7_0.o cik_sdma.o uvd_v4_2.o vce_v2_0.o \
-	amdgpu_amdkfd_gfx_v7.o
+	ci_smc.o ci_dpm.o dce_v8_0.o gfx_v7_0.o cik_sdma.o uvd_v4_2.o vce_v2_0.o
 
 amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
 
 amdgpu-y += \
-	vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o
+	vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o \
+	vega20_reg_init.o
+
+# add DF block
+amdgpu-y += \
+	df_v1_7.o \
+	df_v3_6.o
 
 # add GMC block
 amdgpu-y += \
@@ -126,11 +131,20 @@ amdgpu-y += \
 	vcn_v1_0.o
 
 # add amdkfd interfaces
+amdgpu-y += amdgpu_amdkfd.o
+
+ifneq ($(CONFIG_HSA_AMD),)
 amdgpu-y += \
-	 amdgpu_amdkfd.o \
 	 amdgpu_amdkfd_fence.o \
 	 amdgpu_amdkfd_gpuvm.o \
-	 amdgpu_amdkfd_gfx_v8.o
+	 amdgpu_amdkfd_gfx_v8.o \
+	 amdgpu_amdkfd_gfx_v9.o
+
+ifneq ($(CONFIG_DRM_AMDGPU_CIK),)
+amdgpu-y += amdgpu_amdkfd_gfx_v7.o
+endif
+
+endif
 
 # add cgs
 amdgpu-y += amdgpu_cgs.o
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index c8b605f3dc05..a59c07590cee 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -129,6 +129,7 @@ extern int amdgpu_lbpw;
 extern int amdgpu_compute_multipipe;
 extern int amdgpu_gpu_recovery;
 extern int amdgpu_emu_mode;
+extern uint amdgpu_smu_memory_pool_size;
 
 #ifdef CONFIG_DRM_AMDGPU_SI
 extern int amdgpu_si_support;
@@ -137,6 +138,7 @@ extern int amdgpu_si_support;
 extern int amdgpu_cik_support;
 #endif
 
+#define AMDGPU_SG_THRESHOLD			(256*1024*1024)
 #define AMDGPU_DEFAULT_GTT_SIZE_MB		3072ULL /* 3GB by default */
 #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS	        3000
 #define AMDGPU_MAX_USEC_TIMEOUT			100000	/* 100 ms */
@@ -222,10 +224,10 @@ enum amdgpu_kiq_irq {
 	AMDGPU_CP_KIQ_IRQ_LAST
 };
 
-int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev,
+int amdgpu_device_ip_set_clockgating_state(void *dev,
 					   enum amd_ip_block_type block_type,
 					   enum amd_clockgating_state state);
-int amdgpu_device_ip_set_powergating_state(struct amdgpu_device *adev,
+int amdgpu_device_ip_set_powergating_state(void *dev,
 					   enum amd_ip_block_type block_type,
 					   enum amd_powergating_state state);
 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
@@ -681,6 +683,8 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, unsigned ring_id);
 
 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
+void amdgpu_ctx_mgr_entity_cleanup(struct amdgpu_ctx_mgr *mgr);
+void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr);
 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
 
 
@@ -771,9 +775,18 @@ struct amdgpu_rlc {
 	u32 starting_offsets_start;
 	u32 reg_list_format_size_bytes;
 	u32 reg_list_size_bytes;
+	u32 reg_list_format_direct_reg_list_length;
+	u32 save_restore_list_cntl_size_bytes;
+	u32 save_restore_list_gpm_size_bytes;
+	u32 save_restore_list_srm_size_bytes;
 
 	u32 *register_list_format;
 	u32 *register_restore;
+	u8 *save_restore_list_cntl;
+	u8 *save_restore_list_gpm;
+	u8 *save_restore_list_srm;
+
+	bool is_rlc_v2_1;
 };
 
 #define AMDGPU_MAX_COMPUTE_QUEUES KGD_MAX_QUEUES
@@ -867,6 +880,8 @@ struct amdgpu_gfx_config {
 
 	/* gfx configure feature */
 	uint32_t double_offchip_lds_buf;
+	/* cached value of DB_DEBUG2 */
+	uint32_t db_debug2;
 };
 
 struct amdgpu_cu_info {
@@ -938,6 +953,12 @@ struct amdgpu_gfx {
 	uint32_t			ce_feature_version;
 	uint32_t			pfp_feature_version;
 	uint32_t			rlc_feature_version;
+	uint32_t			rlc_srlc_fw_version;
+	uint32_t			rlc_srlc_feature_version;
+	uint32_t			rlc_srlg_fw_version;
+	uint32_t			rlc_srlg_feature_version;
+	uint32_t			rlc_srls_fw_version;
+	uint32_t			rlc_srls_feature_version;
 	uint32_t			mec_feature_version;
 	uint32_t			mec2_feature_version;
 	struct amdgpu_ring		gfx_ring[AMDGPU_MAX_GFX_RINGS];
@@ -1204,6 +1225,8 @@ struct amdgpu_asic_funcs {
 	/* invalidate hdp read cache */
 	void (*invalidate_hdp)(struct amdgpu_device *adev,
 			       struct amdgpu_ring *ring);
+	/* check if the asic needs a full reset of if soft reset will work */
+	bool (*need_full_reset)(struct amdgpu_device *adev);
 };
 
 /*
@@ -1368,7 +1391,19 @@ struct amdgpu_nbio_funcs {
 	void (*detect_hw_virt)(struct amdgpu_device *adev);
 };
 
-
+struct amdgpu_df_funcs {
+	void (*init)(struct amdgpu_device *adev);
+	void (*enable_broadcast_mode)(struct amdgpu_device *adev,
+				      bool enable);
+	u32 (*get_fb_channel_number)(struct amdgpu_device *adev);
+	u32 (*get_hbm_channel_number)(struct amdgpu_device *adev);
+	void (*update_medium_grain_clock_gating)(struct amdgpu_device *adev,
+						 bool enable);
+	void (*get_clockgating_state)(struct amdgpu_device *adev,
+				      u32 *flags);
+	void (*enable_ecc_force_par_wr_rmw)(struct amdgpu_device *adev,
+					    bool enable);
+};
 /* Define the HW IP blocks will be used in driver , add more if necessary */
 enum amd_hw_ip_block_type {
 	GC_HWIP = 1,
@@ -1398,6 +1433,7 @@ enum amd_hw_ip_block_type {
 struct amd_powerplay {
 	void *pp_handle;
 	const struct amd_pm_funcs *pp_funcs;
+	uint32_t pp_feature;
 };
 
 #define AMDGPU_RESET_MAGIC_NUM 64
@@ -1590,6 +1626,7 @@ struct amdgpu_device {
 	uint32_t 		*reg_offset[MAX_HWIP][HWIP_MAX_INSTANCE];
 
 	const struct amdgpu_nbio_funcs	*nbio_funcs;
+	const struct amdgpu_df_funcs	*df_funcs;
 
 	/* delayed work_func for deferring clockgating during resume */
 	struct delayed_work     late_init_work;
@@ -1764,6 +1801,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_asic_get_config_memsize(adev) (adev)->asic_funcs->get_config_memsize((adev))
 #define amdgpu_asic_flush_hdp(adev, r) (adev)->asic_funcs->flush_hdp((adev), (r))
 #define amdgpu_asic_invalidate_hdp(adev, r) (adev)->asic_funcs->invalidate_hdp((adev), (r))
+#define amdgpu_asic_need_full_reset(adev) (adev)->asic_funcs->need_full_reset((adev))
 #define amdgpu_gmc_flush_gpu_tlb(adev, vmid) (adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid))
 #define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))
 #define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid))
@@ -1790,6 +1828,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_ring_emit_rreg(r, d) (r)->funcs->emit_rreg((r), (d))
 #define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v))
 #define amdgpu_ring_emit_reg_wait(r, d, v, m) (r)->funcs->emit_reg_wait((r), (d), (v), (m))
+#define amdgpu_ring_emit_reg_write_reg_wait(r, d0, d1, v, m) (r)->funcs->emit_reg_write_reg_wait((r), (d0), (d1), (v), (m))
 #define amdgpu_ring_emit_tmz(r, b) (r)->funcs->emit_tmz((r), (b))
 #define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
 #define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
index a29362f9ef41..428e5eb3444f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
@@ -290,12 +290,11 @@ static int acp_hw_init(void *handle)
 	else if (r)
 		return r;
 
-	r = cgs_get_pci_resource(adev->acp.cgs_device, CGS_RESOURCE_TYPE_MMIO,
-			0x5289, 0, &acp_base);
-	if (r == -ENODEV)
-		return 0;
-	else if (r)
-		return r;
+	if (adev->rmmio_size == 0 || adev->rmmio_size < 0x5289)
+		return -EINVAL;
+
+	acp_base = adev->rmmio_base;
+
 	if (adev->asic_type != CHIP_STONEY) {
 		adev->acp.acp_genpd = kzalloc(sizeof(struct acp_pm_domain), GFP_KERNEL);
 		if (adev->acp.acp_genpd == NULL)
@@ -513,7 +512,7 @@ static int acp_hw_fini(void *handle)
 	if (adev->acp.acp_genpd) {
 		for (i = 0; i < ACP_DEVS ; i++) {
 			dev = get_mfd_cell_dev(adev->acp.acp_cell[i].name, i);
-			ret = pm_genpd_remove_device(&adev->acp.acp_genpd->gpd, dev);
+			ret = pm_genpd_remove_device(dev);
 			/* If removal fails, dont giveup and try rest */
 			if (ret)
 				dev_err(dev, "remove dev from genpd failed\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 4d36203ffb11..8f6f45567bfa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -50,15 +50,21 @@ int amdgpu_amdkfd_init(void)
 		kgd2kfd = NULL;
 	}
 
+
 #elif defined(CONFIG_HSA_AMD)
+
 	ret = kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd);
 	if (ret)
 		kgd2kfd = NULL;
 
 #else
+	kgd2kfd = NULL;
 	ret = -ENOENT;
 #endif
+
+#if defined(CONFIG_HSA_AMD_MODULE) || defined(CONFIG_HSA_AMD)
 	amdgpu_amdkfd_gpuvm_init_mem_limits();
+#endif
 
 	return ret;
 }
@@ -92,8 +98,12 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
 	case CHIP_POLARIS11:
 		kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions();
 		break;
+	case CHIP_VEGA10:
+	case CHIP_RAVEN:
+		kfd2kgd = amdgpu_amdkfd_gfx_9_0_get_functions();
+		break;
 	default:
-		dev_dbg(adev->dev, "kfd not supported on this ASIC\n");
+		dev_info(adev->dev, "kfd not supported on this ASIC\n");
 		return;
 	}
 
@@ -175,6 +185,28 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
 				&gpu_resources.doorbell_physical_address,
 				&gpu_resources.doorbell_aperture_size,
 				&gpu_resources.doorbell_start_offset);
+		if (adev->asic_type >= CHIP_VEGA10) {
+			/* On SOC15 the BIF is involved in routing
+			 * doorbells using the low 12 bits of the
+			 * address. Communicate the assignments to
+			 * KFD. KFD uses two doorbell pages per
+			 * process in case of 64-bit doorbells so we
+			 * can use each doorbell assignment twice.
+			 */
+			gpu_resources.sdma_doorbell[0][0] =
+				AMDGPU_DOORBELL64_sDMA_ENGINE0;
+			gpu_resources.sdma_doorbell[0][1] =
+				AMDGPU_DOORBELL64_sDMA_ENGINE0 + 0x200;
+			gpu_resources.sdma_doorbell[1][0] =
+				AMDGPU_DOORBELL64_sDMA_ENGINE1;
+			gpu_resources.sdma_doorbell[1][1] =
+				AMDGPU_DOORBELL64_sDMA_ENGINE1 + 0x200;
+			/* Doorbells 0x0f0-0ff and 0x2f0-2ff are reserved for
+			 * SDMA, IH and VCN. So don't use them for the CP.
+			 */
+			gpu_resources.reserved_doorbell_mask = 0x1f0;
+			gpu_resources.reserved_doorbell_val  = 0x0f0;
+		}
 
 		kgd2kfd->device_init(adev->kfd, &gpu_resources);
 	}
@@ -217,13 +249,19 @@ int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
 	struct amdgpu_bo *bo = NULL;
+	struct amdgpu_bo_param bp;
 	int r;
 	uint64_t gpu_addr_tmp = 0;
 	void *cpu_ptr_tmp = NULL;
 
-	r = amdgpu_bo_create(adev, size, PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
-			     AMDGPU_GEM_CREATE_CPU_GTT_USWC, ttm_bo_type_kernel,
-			     NULL, &bo);
+	memset(&bp, 0, sizeof(bp));
+	bp.size = size;
+	bp.byte_align = PAGE_SIZE;
+	bp.domain = AMDGPU_GEM_DOMAIN_GTT;
+	bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
+	bp.type = ttm_bo_type_kernel;
+	bp.resv = NULL;
+	r = amdgpu_bo_create(adev, &bp, &bo);
 	if (r) {
 		dev_err(adev->dev,
 			"failed to allocate BO for amdkfd (%d)\n", r);
@@ -432,3 +470,44 @@ bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid)
 
 	return false;
 }
+
+#if !defined(CONFIG_HSA_AMD_MODULE) && !defined(CONFIG_HSA_AMD)
+bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
+{
+	return false;
+}
+
+void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo)
+{
+}
+
+void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
+					struct amdgpu_vm *vm)
+{
+}
+
+struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
+{
+	return NULL;
+}
+
+int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm)
+{
+	return 0;
+}
+
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void)
+{
+	return NULL;
+}
+
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void)
+{
+	return NULL;
+}
+
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void)
+{
+	return NULL;
+}
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index c2c2bea731e0..a8418a3f4e9d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -28,6 +28,7 @@
 #include <linux/types.h>
 #include <linux/mm.h>
 #include <linux/mmu_context.h>
+#include <linux/workqueue.h>
 #include <kgd_kfd_interface.h>
 #include <drm/ttm/ttm_execbuf_util.h>
 #include "amdgpu_sync.h"
@@ -59,7 +60,9 @@ struct kgd_mem {
 
 	uint32_t mapping_flags;
 
+	atomic_t invalid;
 	struct amdkfd_process_info *process_info;
+	struct page **user_pages;
 
 	struct amdgpu_sync sync;
 
@@ -84,6 +87,9 @@ struct amdkfd_process_info {
 	struct list_head vm_list_head;
 	/* List head for all KFD BOs that belong to a KFD process. */
 	struct list_head kfd_bo_list;
+	/* List of userptr BOs that are valid or invalid */
+	struct list_head userptr_valid_list;
+	struct list_head userptr_inval_list;
 	/* Lock to protect kfd_bo_list */
 	struct mutex lock;
 
@@ -91,6 +97,11 @@ struct amdkfd_process_info {
 	unsigned int n_vms;
 	/* Eviction Fence */
 	struct amdgpu_amdkfd_fence *eviction_fence;
+
+	/* MMU-notifier related fields */
+	atomic_t evicted_bos;
+	struct delayed_work restore_userptr_work;
+	struct pid *pid;
 };
 
 int amdgpu_amdkfd_init(void);
@@ -104,12 +115,14 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
 void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
 void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);
 
+int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
 int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
 				uint32_t vmid, uint64_t gpu_addr,
 				uint32_t *ib_cmd, uint32_t ib_len);
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void);
 
 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
 
@@ -143,14 +156,14 @@ uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
 
 /* GPUVM API */
 int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,
-					  void **process_info,
-					  struct dma_fence **ef);
+					void **process_info,
+					struct dma_fence **ef);
 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
-					   struct file *filp,
-					   void **vm, void **process_info,
-					   struct dma_fence **ef);
+					struct file *filp,
+					void **vm, void **process_info,
+					struct dma_fence **ef);
 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
-				    struct amdgpu_vm *vm);
+				struct amdgpu_vm *vm);
 void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm);
 uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index ea54e53172b9..0ff36d45a597 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -98,8 +98,6 @@ static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
 					unsigned int vmid);
 
-static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
-				uint32_t hpd_size, uint64_t hpd_gpu_addr);
 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
 			uint32_t queue_id, uint32_t __user *wptr,
@@ -183,7 +181,6 @@ static const struct kfd2kgd_calls kfd2kgd = {
 	.free_pasid = amdgpu_pasid_free,
 	.program_sh_mem_settings = kgd_program_sh_mem_settings,
 	.set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
-	.init_pipeline = kgd_init_pipeline,
 	.init_interrupts = kgd_init_interrupts,
 	.hqd_load = kgd_hqd_load,
 	.hqd_sdma_load = kgd_hqd_sdma_load,
@@ -309,13 +306,6 @@ static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
 	return 0;
 }
 
-static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
-				uint32_t hpd_size, uint64_t hpd_gpu_addr)
-{
-	/* amdgpu owns the per-pipe state */
-	return 0;
-}
-
 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index 89264c9a5e9f..6ef9762b4b00 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -57,8 +57,6 @@ static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
 		uint32_t sh_mem_bases);
 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
 		unsigned int vmid);
-static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
-		uint32_t hpd_size, uint64_t hpd_gpu_addr);
 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
 			uint32_t queue_id, uint32_t __user *wptr,
@@ -141,7 +139,6 @@ static const struct kfd2kgd_calls kfd2kgd = {
 	.free_pasid = amdgpu_pasid_free,
 	.program_sh_mem_settings = kgd_program_sh_mem_settings,
 	.set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
-	.init_pipeline = kgd_init_pipeline,
 	.init_interrupts = kgd_init_interrupts,
 	.hqd_load = kgd_hqd_load,
 	.hqd_sdma_load = kgd_hqd_sdma_load,
@@ -270,13 +267,6 @@ static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
 	return 0;
 }
 
-static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
-				uint32_t hpd_size, uint64_t hpd_gpu_addr)
-{
-	/* amdgpu owns the per-pipe state */
-	return 0;
-}
-
 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
new file mode 100644
index 000000000000..f0c0d3953f69
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -0,0 +1,1043 @@
+/*
+ * Copyright 2014-2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#define pr_fmt(fmt) "kfd2kgd: " fmt
+
+#include <linux/module.h>
+#include <linux/fdtable.h>
+#include <linux/uaccess.h>
+#include <linux/firmware.h>
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include "amdgpu_amdkfd.h"
+#include "amdgpu_ucode.h"
+#include "soc15_hw_ip.h"
+#include "gc/gc_9_0_offset.h"
+#include "gc/gc_9_0_sh_mask.h"
+#include "vega10_enum.h"
+#include "sdma0/sdma0_4_0_offset.h"
+#include "sdma0/sdma0_4_0_sh_mask.h"
+#include "sdma1/sdma1_4_0_offset.h"
+#include "sdma1/sdma1_4_0_sh_mask.h"
+#include "athub/athub_1_0_offset.h"
+#include "athub/athub_1_0_sh_mask.h"
+#include "oss/osssys_4_0_offset.h"
+#include "oss/osssys_4_0_sh_mask.h"
+#include "soc15_common.h"
+#include "v9_structs.h"
+#include "soc15.h"
+#include "soc15d.h"
+
+/* HACK: MMHUB and GC both have VM-related register with the same
+ * names but different offsets. Define the MMHUB register we need here
+ * with a prefix. A proper solution would be to move the functions
+ * programming these registers into gfx_v9_0.c and mmhub_v1_0.c
+ * respectively.
+ */
+#define mmMMHUB_VM_INVALIDATE_ENG16_REQ				0x06f3
+#define mmMMHUB_VM_INVALIDATE_ENG16_REQ_BASE_IDX		0
+
+#define mmMMHUB_VM_INVALIDATE_ENG16_ACK				0x0705
+#define mmMMHUB_VM_INVALIDATE_ENG16_ACK_BASE_IDX		0
+
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32		0x072b
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX	0
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32		0x072c
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX	0
+
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32		0x074b
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_BASE_IDX	0
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32		0x074c
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_BASE_IDX	0
+
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32		0x076b
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_BASE_IDX	0
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32		0x076c
+#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_BASE_IDX	0
+
+#define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_LO32		0x0727
+#define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_LO32_BASE_IDX	0
+#define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_HI32		0x0728
+#define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_HI32_BASE_IDX	0
+
+#define V9_PIPE_PER_MEC		(4)
+#define V9_QUEUES_PER_PIPE_MEC	(8)
+
+enum hqd_dequeue_request_type {
+	NO_ACTION = 0,
+	DRAIN_PIPE,
+	RESET_WAVES
+};
+
+/*
+ * Register access functions
+ */
+
+static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
+		uint32_t sh_mem_config,
+		uint32_t sh_mem_ape1_base, uint32_t sh_mem_ape1_limit,
+		uint32_t sh_mem_bases);
+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+		unsigned int vmid);
+static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
+static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+			uint32_t queue_id, uint32_t __user *wptr,
+			uint32_t wptr_shift, uint32_t wptr_mask,
+			struct mm_struct *mm);
+static int kgd_hqd_dump(struct kgd_dev *kgd,
+			uint32_t pipe_id, uint32_t queue_id,
+			uint32_t (**dump)[2], uint32_t *n_regs);
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+			     uint32_t __user *wptr, struct mm_struct *mm);
+static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+			     uint32_t engine_id, uint32_t queue_id,
+			     uint32_t (**dump)[2], uint32_t *n_regs);
+static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+		uint32_t pipe_id, uint32_t queue_id);
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+				enum kfd_preempt_type reset_type,
+				unsigned int utimeout, uint32_t pipe_id,
+				uint32_t queue_id);
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+				unsigned int utimeout);
+static int kgd_address_watch_disable(struct kgd_dev *kgd);
+static int kgd_address_watch_execute(struct kgd_dev *kgd,
+					unsigned int watch_point_id,
+					uint32_t cntl_val,
+					uint32_t addr_hi,
+					uint32_t addr_lo);
+static int kgd_wave_control_execute(struct kgd_dev *kgd,
+					uint32_t gfx_index_val,
+					uint32_t sq_cmd);
+static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
+					unsigned int watch_point_id,
+					unsigned int reg_offset);
+
+static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
+		uint8_t vmid);
+static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+		uint8_t vmid);
+static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
+		uint32_t page_table_base);
+static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
+static void set_scratch_backing_va(struct kgd_dev *kgd,
+					uint64_t va, uint32_t vmid);
+static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid);
+static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid);
+
+/* Because of REG_GET_FIELD() being used, we put this function in the
+ * asic specific file.
+ */
+static int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
+		struct tile_config *config)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+	config->gb_addr_config = adev->gfx.config.gb_addr_config;
+
+	config->tile_config_ptr = adev->gfx.config.tile_mode_array;
+	config->num_tile_configs =
+			ARRAY_SIZE(adev->gfx.config.tile_mode_array);
+	config->macro_tile_config_ptr =
+			adev->gfx.config.macrotile_mode_array;
+	config->num_macro_tile_configs =
+			ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
+
+	return 0;
+}
+
+static const struct kfd2kgd_calls kfd2kgd = {
+	.init_gtt_mem_allocation = alloc_gtt_mem,
+	.free_gtt_mem = free_gtt_mem,
+	.get_local_mem_info = get_local_mem_info,
+	.get_gpu_clock_counter = get_gpu_clock_counter,
+	.get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+	.alloc_pasid = amdgpu_pasid_alloc,
+	.free_pasid = amdgpu_pasid_free,
+	.program_sh_mem_settings = kgd_program_sh_mem_settings,
+	.set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
+	.init_interrupts = kgd_init_interrupts,
+	.hqd_load = kgd_hqd_load,
+	.hqd_sdma_load = kgd_hqd_sdma_load,
+	.hqd_dump = kgd_hqd_dump,
+	.hqd_sdma_dump = kgd_hqd_sdma_dump,
+	.hqd_is_occupied = kgd_hqd_is_occupied,
+	.hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
+	.hqd_destroy = kgd_hqd_destroy,
+	.hqd_sdma_destroy = kgd_hqd_sdma_destroy,
+	.address_watch_disable = kgd_address_watch_disable,
+	.address_watch_execute = kgd_address_watch_execute,
+	.wave_control_execute = kgd_wave_control_execute,
+	.address_watch_get_offset = kgd_address_watch_get_offset,
+	.get_atc_vmid_pasid_mapping_pasid =
+			get_atc_vmid_pasid_mapping_pasid,
+	.get_atc_vmid_pasid_mapping_valid =
+			get_atc_vmid_pasid_mapping_valid,
+	.get_fw_version = get_fw_version,
+	.set_scratch_backing_va = set_scratch_backing_va,
+	.get_tile_config = amdgpu_amdkfd_get_tile_config,
+	.get_cu_info = get_cu_info,
+	.get_vram_usage = amdgpu_amdkfd_get_vram_usage,
+	.create_process_vm = amdgpu_amdkfd_gpuvm_create_process_vm,
+	.acquire_process_vm = amdgpu_amdkfd_gpuvm_acquire_process_vm,
+	.destroy_process_vm = amdgpu_amdkfd_gpuvm_destroy_process_vm,
+	.get_process_page_dir = amdgpu_amdkfd_gpuvm_get_process_page_dir,
+	.set_vm_context_page_table_base = set_vm_context_page_table_base,
+	.alloc_memory_of_gpu = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu,
+	.free_memory_of_gpu = amdgpu_amdkfd_gpuvm_free_memory_of_gpu,
+	.map_memory_to_gpu = amdgpu_amdkfd_gpuvm_map_memory_to_gpu,
+	.unmap_memory_to_gpu = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu,
+	.sync_memory = amdgpu_amdkfd_gpuvm_sync_memory,
+	.map_gtt_bo_to_kernel = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel,
+	.restore_process_bos = amdgpu_amdkfd_gpuvm_restore_process_bos,
+	.invalidate_tlbs = invalidate_tlbs,
+	.invalidate_tlbs_vmid = invalidate_tlbs_vmid,
+	.submit_ib = amdgpu_amdkfd_submit_ib,
+};
+
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void)
+{
+	return (struct kfd2kgd_calls *)&kfd2kgd;
+}
+
+static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
+{
+	return (struct amdgpu_device *)kgd;
+}
+
+static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
+			uint32_t queue, uint32_t vmid)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+	mutex_lock(&adev->srbm_mutex);
+	soc15_grbm_select(adev, mec, pipe, queue, vmid);
+}
+
+static void unlock_srbm(struct kgd_dev *kgd)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+	soc15_grbm_select(adev, 0, 0, 0, 0);
+	mutex_unlock(&adev->srbm_mutex);
+}
+
+static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
+				uint32_t queue_id)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+	uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+	uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+
+	lock_srbm(kgd, mec, pipe, queue_id, 0);
+}
+
+static uint32_t get_queue_mask(struct amdgpu_device *adev,
+			       uint32_t pipe_id, uint32_t queue_id)
+{
+	unsigned int bit = (pipe_id * adev->gfx.mec.num_queue_per_pipe +
+			    queue_id) & 31;
+
+	return ((uint32_t)1) << bit;
+}
+
+static void release_queue(struct kgd_dev *kgd)
+{
+	unlock_srbm(kgd);
+}
+
+static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
+					uint32_t sh_mem_config,
+					uint32_t sh_mem_ape1_base,
+					uint32_t sh_mem_ape1_limit,
+					uint32_t sh_mem_bases)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+	lock_srbm(kgd, 0, 0, 0, vmid);
+
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config);
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases);
+	/* APE1 no longer exists on GFX9 */
+
+	unlock_srbm(kgd);
+}
+
+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+					unsigned int vmid)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+	/*
+	 * We have to assume that there is no outstanding mapping.
+	 * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
+	 * a mapping is in progress or because a mapping finished
+	 * and the SW cleared it.
+	 * So the protocol is to always wait & clear.
+	 */
+	uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
+			ATC_VMID0_PASID_MAPPING__VALID_MASK;
+
+	/*
+	 * need to do this twice, once for gfx and once for mmhub
+	 * for ATC add 16 to VMID for mmhub, for IH different registers.
+	 * ATC_VMID0..15 registers are separate from ATC_VMID16..31.
+	 */
+
+	WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid,
+	       pasid_mapping);
+
+	while (!(RREG32(SOC15_REG_OFFSET(
+				ATHUB, 0,
+				mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) &
+		 (1U << vmid)))
+		cpu_relax();
+
+	WREG32(SOC15_REG_OFFSET(ATHUB, 0,
+				mmATC_VMID_PASID_MAPPING_UPDATE_STATUS),
+	       1U << vmid);
+
+	/* Mapping vmid to pasid also for IH block */
+	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid,
+	       pasid_mapping);
+
+	WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID16_PASID_MAPPING) + vmid,
+	       pasid_mapping);
+
+	while (!(RREG32(SOC15_REG_OFFSET(
+				ATHUB, 0,
+				mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) &
+		 (1U << (vmid + 16))))
+		cpu_relax();
+
+	WREG32(SOC15_REG_OFFSET(ATHUB, 0,
+				mmATC_VMID_PASID_MAPPING_UPDATE_STATUS),
+	       1U << (vmid + 16));
+
+	/* Mapping vmid to pasid also for IH block */
+	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid,
+	       pasid_mapping);
+	return 0;
+}
+
+/* TODO - RING0 form of field is obsolete, seems to date back to SI
+ * but still works
+ */
+
+static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	uint32_t mec;
+	uint32_t pipe;
+
+	mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+	pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+
+	lock_srbm(kgd, mec, pipe, 0, 0);
+
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL),
+		CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
+		CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
+
+	unlock_srbm(kgd);
+
+	return 0;
+}
+
+static uint32_t get_sdma_base_addr(struct amdgpu_device *adev,
+				unsigned int engine_id,
+				unsigned int queue_id)
+{
+	uint32_t base[2] = {
+		SOC15_REG_OFFSET(SDMA0, 0,
+				 mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL,
+		SOC15_REG_OFFSET(SDMA1, 0,
+				 mmSDMA1_RLC0_RB_CNTL) - mmSDMA1_RLC0_RB_CNTL
+	};
+	uint32_t retval;
+
+	retval = base[engine_id] + queue_id * (mmSDMA0_RLC1_RB_CNTL -
+					       mmSDMA0_RLC0_RB_CNTL);
+
+	pr_debug("sdma base address: 0x%x\n", retval);
+
+	return retval;
+}
+
+static inline struct v9_mqd *get_mqd(void *mqd)
+{
+	return (struct v9_mqd *)mqd;
+}
+
+static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)
+{
+	return (struct v9_sdma_mqd *)mqd;
+}
+
+static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+			uint32_t queue_id, uint32_t __user *wptr,
+			uint32_t wptr_shift, uint32_t wptr_mask,
+			struct mm_struct *mm)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	struct v9_mqd *m;
+	uint32_t *mqd_hqd;
+	uint32_t reg, hqd_base, data;
+
+	m = get_mqd(mqd);
+
+	acquire_queue(kgd, pipe_id, queue_id);
+
+	/* HIQ is set during driver init period with vmid set to 0*/
+	if (m->cp_hqd_vmid == 0) {
+		uint32_t value, mec, pipe;
+
+		mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+		pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+
+		pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
+			mec, pipe, queue_id);
+		value = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS));
+		value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
+			((mec << 5) | (pipe << 3) | queue_id | 0x80));
+		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), value);
+	}
+
+	/* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
+	mqd_hqd = &m->cp_mqd_base_addr_lo;
+	hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
+
+	for (reg = hqd_base;
+	     reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
+		WREG32(reg, mqd_hqd[reg - hqd_base]);
+
+
+	/* Activate doorbell logic before triggering WPTR poll. */
+	data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
+			     CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data);
+
+	if (wptr) {
+		/* Don't read wptr with get_user because the user
+		 * context may not be accessible (if this function
+		 * runs in a work queue). Instead trigger a one-shot
+		 * polling read from memory in the CP. This assumes
+		 * that wptr is GPU-accessible in the queue's VMID via
+		 * ATC or SVM. WPTR==RPTR before starting the poll so
+		 * the CP starts fetching new commands from the right
+		 * place.
+		 *
+		 * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit
+		 * tricky. Assume that the queue didn't overflow. The
+		 * number of valid bits in the 32-bit RPTR depends on
+		 * the queue size. The remaining bits are taken from
+		 * the saved 64-bit WPTR. If the WPTR wrapped, add the
+		 * queue size.
+		 */
+		uint32_t queue_size =
+			2 << REG_GET_FIELD(m->cp_hqd_pq_control,
+					   CP_HQD_PQ_CONTROL, QUEUE_SIZE);
+		uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1);
+
+		if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr)
+			guessed_wptr += queue_size;
+		guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1);
+		guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32;
+
+		WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO),
+		       lower_32_bits(guessed_wptr));
+		WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI),
+		       upper_32_bits(guessed_wptr));
+		WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
+		       lower_32_bits((uintptr_t)wptr));
+		WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
+		       upper_32_bits((uintptr_t)wptr));
+		WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL1),
+		       get_queue_mask(adev, pipe_id, queue_id));
+	}
+
+	/* Start the EOP fetcher */
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_RPTR),
+	       REG_SET_FIELD(m->cp_hqd_eop_rptr,
+			     CP_HQD_EOP_RPTR, INIT_FETCHER, 1));
+
+	data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), data);
+
+	release_queue(kgd);
+
+	return 0;
+}
+
+static int kgd_hqd_dump(struct kgd_dev *kgd,
+			uint32_t pipe_id, uint32_t queue_id,
+			uint32_t (**dump)[2], uint32_t *n_regs)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	uint32_t i = 0, reg;
+#define HQD_N_REGS 56
+#define DUMP_REG(addr) do {				\
+		if (WARN_ON_ONCE(i >= HQD_N_REGS))	\
+			break;				\
+		(*dump)[i][0] = (addr) << 2;		\
+		(*dump)[i++][1] = RREG32(addr);		\
+	} while (0)
+
+	*dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+	if (*dump == NULL)
+		return -ENOMEM;
+
+	acquire_queue(kgd, pipe_id, queue_id);
+
+	for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
+	     reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
+		DUMP_REG(reg);
+
+	release_queue(kgd);
+
+	WARN_ON_ONCE(i != HQD_N_REGS);
+	*n_regs = i;
+
+	return 0;
+}
+
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+			     uint32_t __user *wptr, struct mm_struct *mm)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	struct v9_sdma_mqd *m;
+	uint32_t sdma_base_addr, sdmax_gfx_context_cntl;
+	unsigned long end_jiffies;
+	uint32_t data;
+	uint64_t data64;
+	uint64_t __user *wptr64 = (uint64_t __user *)wptr;
+
+	m = get_sdma_mqd(mqd);
+	sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id,
+					    m->sdma_queue_id);
+	sdmax_gfx_context_cntl = m->sdma_engine_id ?
+		SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GFX_CONTEXT_CNTL) :
+		SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_CONTEXT_CNTL);
+
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+		m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK));
+
+	end_jiffies = msecs_to_jiffies(2000) + jiffies;
+	while (true) {
+		data = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+		if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
+			break;
+		if (time_after(jiffies, end_jiffies))
+			return -ETIME;
+		usleep_range(500, 1000);
+	}
+	data = RREG32(sdmax_gfx_context_cntl);
+	data = REG_SET_FIELD(data, SDMA0_GFX_CONTEXT_CNTL,
+			     RESUME_CTX, 0);
+	WREG32(sdmax_gfx_context_cntl, data);
+
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL_OFFSET,
+	       m->sdmax_rlcx_doorbell_offset);
+
+	data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL,
+			     ENABLE, 1);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdmax_rlcx_rb_rptr);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI,
+				m->sdmax_rlcx_rb_rptr_hi);
+
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 1);
+	if (read_user_wptr(mm, wptr64, data64)) {
+		WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
+		       lower_32_bits(data64));
+		WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI,
+		       upper_32_bits(data64));
+	} else {
+		WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
+		       m->sdmax_rlcx_rb_rptr);
+		WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI,
+		       m->sdmax_rlcx_rb_rptr_hi);
+	}
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 0);
+
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI,
+			m->sdmax_rlcx_rb_base_hi);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO,
+			m->sdmax_rlcx_rb_rptr_addr_lo);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,
+			m->sdmax_rlcx_rb_rptr_addr_hi);
+
+	data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL,
+			     RB_ENABLE, 1);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data);
+
+	return 0;
+}
+
+static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+			     uint32_t engine_id, uint32_t queue_id,
+			     uint32_t (**dump)[2], uint32_t *n_regs)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	uint32_t sdma_base_addr = get_sdma_base_addr(adev, engine_id, queue_id);
+	uint32_t i = 0, reg;
+#undef HQD_N_REGS
+#define HQD_N_REGS (19+6+7+10)
+
+	*dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+	if (*dump == NULL)
+		return -ENOMEM;
+
+	for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++)
+		DUMP_REG(sdma_base_addr + reg);
+	for (reg = mmSDMA0_RLC0_STATUS; reg <= mmSDMA0_RLC0_CSA_ADDR_HI; reg++)
+		DUMP_REG(sdma_base_addr + reg);
+	for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN;
+	     reg <= mmSDMA0_RLC0_MINOR_PTR_UPDATE; reg++)
+		DUMP_REG(sdma_base_addr + reg);
+	for (reg = mmSDMA0_RLC0_MIDCMD_DATA0;
+	     reg <= mmSDMA0_RLC0_MIDCMD_CNTL; reg++)
+		DUMP_REG(sdma_base_addr + reg);
+
+	WARN_ON_ONCE(i != HQD_N_REGS);
+	*n_regs = i;
+
+	return 0;
+}
+
+static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+				uint32_t pipe_id, uint32_t queue_id)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	uint32_t act;
+	bool retval = false;
+	uint32_t low, high;
+
+	acquire_queue(kgd, pipe_id, queue_id);
+	act = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE));
+	if (act) {
+		low = lower_32_bits(queue_address >> 8);
+		high = upper_32_bits(queue_address >> 8);
+
+		if (low == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE)) &&
+		   high == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI)))
+			retval = true;
+	}
+	release_queue(kgd);
+	return retval;
+}
+
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	struct v9_sdma_mqd *m;
+	uint32_t sdma_base_addr;
+	uint32_t sdma_rlc_rb_cntl;
+
+	m = get_sdma_mqd(mqd);
+	sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id,
+					    m->sdma_queue_id);
+
+	sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
+
+	if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)
+		return true;
+
+	return false;
+}
+
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+				enum kfd_preempt_type reset_type,
+				unsigned int utimeout, uint32_t pipe_id,
+				uint32_t queue_id)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	enum hqd_dequeue_request_type type;
+	unsigned long end_jiffies;
+	uint32_t temp;
+	struct v9_mqd *m = get_mqd(mqd);
+
+	acquire_queue(kgd, pipe_id, queue_id);
+
+	if (m->cp_hqd_vmid == 0)
+		WREG32_FIELD15(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0);
+
+	switch (reset_type) {
+	case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
+		type = DRAIN_PIPE;
+		break;
+	case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
+		type = RESET_WAVES;
+		break;
+	default:
+		type = DRAIN_PIPE;
+		break;
+	}
+
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), type);
+
+	end_jiffies = (utimeout * HZ / 1000) + jiffies;
+	while (true) {
+		temp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE));
+		if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
+			break;
+		if (time_after(jiffies, end_jiffies)) {
+			pr_err("cp queue preemption time out.\n");
+			release_queue(kgd);
+			return -ETIME;
+		}
+		usleep_range(500, 1000);
+	}
+
+	release_queue(kgd);
+	return 0;
+}
+
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+				unsigned int utimeout)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	struct v9_sdma_mqd *m;
+	uint32_t sdma_base_addr;
+	uint32_t temp;
+	unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;
+
+	m = get_sdma_mqd(mqd);
+	sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id,
+					    m->sdma_queue_id);
+
+	temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
+	temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK;
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp);
+
+	while (true) {
+		temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+		if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
+			break;
+		if (time_after(jiffies, end_jiffies))
+			return -ETIME;
+		usleep_range(500, 1000);
+	}
+
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+		RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) |
+		SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK);
+
+	m->sdmax_rlcx_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR);
+	m->sdmax_rlcx_rb_rptr_hi =
+		RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI);
+
+	return 0;
+}
+
+static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
+							uint8_t vmid)
+{
+	uint32_t reg;
+	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+	reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
+		     + vmid);
+	return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK;
+}
+
+static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+								uint8_t vmid)
+{
+	uint32_t reg;
+	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+	reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
+		     + vmid);
+	return reg & ATC_VMID0_PASID_MAPPING__PASID_MASK;
+}
+
+static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+	uint32_t req = (1 << vmid) |
+		(0 << VM_INVALIDATE_ENG16_REQ__FLUSH_TYPE__SHIFT) | /* legacy */
+		VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES_MASK |
+		VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0_MASK |
+		VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1_MASK |
+		VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2_MASK |
+		VM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES_MASK;
+
+	mutex_lock(&adev->srbm_mutex);
+
+	/* Use legacy mode tlb invalidation.
+	 *
+	 * Currently on Raven the code below is broken for anything but
+	 * legacy mode due to a MMHUB power gating problem. A workaround
+	 * is for MMHUB to wait until the condition PER_VMID_INVALIDATE_REQ
+	 * == PER_VMID_INVALIDATE_ACK instead of simply waiting for the ack
+	 * bit.
+	 *
+	 * TODO 1: agree on the right set of invalidation registers for
+	 * KFD use. Use the last one for now. Invalidate both GC and
+	 * MMHUB.
+	 *
+	 * TODO 2: support range-based invalidation, requires kfg2kgd
+	 * interface change
+	 */
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_ADDR_RANGE_LO32),
+				0xffffffff);
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_ADDR_RANGE_HI32),
+				0x0000001f);
+
+	WREG32(SOC15_REG_OFFSET(MMHUB, 0,
+				mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_LO32),
+				0xffffffff);
+	WREG32(SOC15_REG_OFFSET(MMHUB, 0,
+				mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_HI32),
+				0x0000001f);
+
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_REQ), req);
+
+	WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_INVALIDATE_ENG16_REQ),
+				req);
+
+	while (!(RREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_ACK)) &
+					(1 << vmid)))
+		cpu_relax();
+
+	while (!(RREG32(SOC15_REG_OFFSET(MMHUB, 0,
+					mmMMHUB_VM_INVALIDATE_ENG16_ACK)) &
+					(1 << vmid)))
+		cpu_relax();
+
+	mutex_unlock(&adev->srbm_mutex);
+
+}
+
+static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid)
+{
+	signed long r;
+	uint32_t seq;
+	struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
+
+	spin_lock(&adev->gfx.kiq.ring_lock);
+	amdgpu_ring_alloc(ring, 12); /* fence + invalidate_tlbs package*/
+	amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
+	amdgpu_ring_write(ring,
+			PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
+			PACKET3_INVALIDATE_TLBS_ALL_HUB(1) |
+			PACKET3_INVALIDATE_TLBS_PASID(pasid) |
+			PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(0)); /* legacy */
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock(&adev->gfx.kiq.ring_lock);
+
+	r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout);
+	if (r < 1) {
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+		return -ETIME;
+	}
+
+	return 0;
+}
+
+static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+	int vmid;
+	struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
+
+	if (ring->ready)
+		return invalidate_tlbs_with_kiq(adev, pasid);
+
+	for (vmid = 0; vmid < 16; vmid++) {
+		if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
+			continue;
+		if (get_atc_vmid_pasid_mapping_valid(kgd, vmid)) {
+			if (get_atc_vmid_pasid_mapping_pasid(kgd, vmid)
+				== pasid) {
+				write_vmid_invalidate_request(kgd, vmid);
+				break;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+	if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
+		pr_err("non kfd vmid %d\n", vmid);
+		return 0;
+	}
+
+	write_vmid_invalidate_request(kgd, vmid);
+	return 0;
+}
+
+static int kgd_address_watch_disable(struct kgd_dev *kgd)
+{
+	return 0;
+}
+
+static int kgd_address_watch_execute(struct kgd_dev *kgd,
+					unsigned int watch_point_id,
+					uint32_t cntl_val,
+					uint32_t addr_hi,
+					uint32_t addr_lo)
+{
+	return 0;
+}
+
+static int kgd_wave_control_execute(struct kgd_dev *kgd,
+					uint32_t gfx_index_val,
+					uint32_t sq_cmd)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	uint32_t data = 0;
+
+	mutex_lock(&adev->grbm_idx_mutex);
+
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), gfx_index_val);
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CMD), sq_cmd);
+
+	data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+		INSTANCE_BROADCAST_WRITES, 1);
+	data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+		SH_BROADCAST_WRITES, 1);
+	data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+		SE_BROADCAST_WRITES, 1);
+
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data);
+	mutex_unlock(&adev->grbm_idx_mutex);
+
+	return 0;
+}
+
+static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
+					unsigned int watch_point_id,
+					unsigned int reg_offset)
+{
+	return 0;
+}
+
+static void set_scratch_backing_va(struct kgd_dev *kgd,
+					uint64_t va, uint32_t vmid)
+{
+	/* No longer needed on GFXv9. The scratch base address is
+	 * passed to the shader by the CP. It's the user mode driver's
+	 * responsibility.
+	 */
+}
+
+/* FIXME: Does this need to be ASIC-specific code? */
+static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+	const union amdgpu_firmware_header *hdr;
+
+	switch (type) {
+	case KGD_ENGINE_PFP:
+		hdr = (const union amdgpu_firmware_header *)adev->gfx.pfp_fw->data;
+		break;
+
+	case KGD_ENGINE_ME:
+		hdr = (const union amdgpu_firmware_header *)adev->gfx.me_fw->data;
+		break;
+
+	case KGD_ENGINE_CE:
+		hdr = (const union amdgpu_firmware_header *)adev->gfx.ce_fw->data;
+		break;
+
+	case KGD_ENGINE_MEC1:
+		hdr = (const union amdgpu_firmware_header *)adev->gfx.mec_fw->data;
+		break;
+
+	case KGD_ENGINE_MEC2:
+		hdr = (const union amdgpu_firmware_header *)adev->gfx.mec2_fw->data;
+		break;
+
+	case KGD_ENGINE_RLC:
+		hdr = (const union amdgpu_firmware_header *)adev->gfx.rlc_fw->data;
+		break;
+
+	case KGD_ENGINE_SDMA1:
+		hdr = (const union amdgpu_firmware_header *)adev->sdma.instance[0].fw->data;
+		break;
+
+	case KGD_ENGINE_SDMA2:
+		hdr = (const union amdgpu_firmware_header *)adev->sdma.instance[1].fw->data;
+		break;
+
+	default:
+		return 0;
+	}
+
+	if (hdr == NULL)
+		return 0;
+
+	/* Only 12 bit in use*/
+	return hdr->common.ucode_version;
+}
+
+static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
+		uint32_t page_table_base)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	uint64_t base = (uint64_t)page_table_base << PAGE_SHIFT |
+		AMDGPU_PTE_VALID;
+
+	if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
+		pr_err("trying to set page table base for wrong VMID %u\n",
+		       vmid);
+		return;
+	}
+
+	/* TODO: take advantage of per-process address space size. For
+	 * now, all processes share the same address space size, like
+	 * on GFX8 and older.
+	 */
+	WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32) + (vmid*2), 0);
+	WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32) + (vmid*2), 0);
+
+	WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32) + (vmid*2),
+			lower_32_bits(adev->vm_manager.max_pfn - 1));
+	WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32) + (vmid*2),
+			upper_32_bits(adev->vm_manager.max_pfn - 1));
+
+	WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) + (vmid*2), lower_32_bits(base));
+	WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) + (vmid*2), upper_32_bits(base));
+
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32) + (vmid*2), 0);
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32) + (vmid*2), 0);
+
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32) + (vmid*2),
+			lower_32_bits(adev->vm_manager.max_pfn - 1));
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32) + (vmid*2),
+			upper_32_bits(adev->vm_manager.max_pfn - 1));
+
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) + (vmid*2), lower_32_bits(base));
+	WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) + (vmid*2), upper_32_bits(base));
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 1d6e1479da38..ff8fd75f7ca5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -23,6 +23,8 @@
 #define pr_fmt(fmt) "kfd2kgd: " fmt
 
 #include <linux/list.h>
+#include <linux/pagemap.h>
+#include <linux/sched/mm.h>
 #include <drm/drmP.h>
 #include "amdgpu_object.h"
 #include "amdgpu_vm.h"
@@ -33,10 +35,20 @@
  */
 #define VI_BO_SIZE_ALIGN (0x8000)
 
+/* BO flag to indicate a KFD userptr BO */
+#define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63)
+
+/* Userptr restore delay, just long enough to allow consecutive VM
+ * changes to accumulate
+ */
+#define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
+
 /* Impose limit on how much memory KFD can use */
 static struct {
 	uint64_t max_system_mem_limit;
+	uint64_t max_userptr_mem_limit;
 	int64_t system_mem_used;
+	int64_t userptr_mem_used;
 	spinlock_t mem_limit_lock;
 } kfd_mem_limit;
 
@@ -57,6 +69,7 @@ static const char * const domain_bit_to_string[] = {
 
 #define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
 
+static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
 
 
 static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
@@ -78,6 +91,7 @@ static bool check_if_add_bo_to_vm(struct amdgpu_vm *avm,
 
 /* Set memory usage limits. Current, limits are
  *  System (kernel) memory - 3/8th System RAM
+ *  Userptr memory - 3/4th System RAM
  */
 void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
 {
@@ -90,8 +104,10 @@ void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
 
 	spin_lock_init(&kfd_mem_limit.mem_limit_lock);
 	kfd_mem_limit.max_system_mem_limit = (mem >> 1) - (mem >> 3);
-	pr_debug("Kernel memory limit %lluM\n",
-		(kfd_mem_limit.max_system_mem_limit >> 20));
+	kfd_mem_limit.max_userptr_mem_limit = mem - (mem >> 2);
+	pr_debug("Kernel memory limit %lluM, userptr limit %lluM\n",
+		(kfd_mem_limit.max_system_mem_limit >> 20),
+		(kfd_mem_limit.max_userptr_mem_limit >> 20));
 }
 
 static int amdgpu_amdkfd_reserve_system_mem_limit(struct amdgpu_device *adev,
@@ -111,6 +127,16 @@ static int amdgpu_amdkfd_reserve_system_mem_limit(struct amdgpu_device *adev,
 			goto err_no_mem;
 		}
 		kfd_mem_limit.system_mem_used += (acc_size + size);
+	} else if (domain == AMDGPU_GEM_DOMAIN_CPU) {
+		if ((kfd_mem_limit.system_mem_used + acc_size >
+			kfd_mem_limit.max_system_mem_limit) ||
+			(kfd_mem_limit.userptr_mem_used + (size + acc_size) >
+			kfd_mem_limit.max_userptr_mem_limit)) {
+			ret = -ENOMEM;
+			goto err_no_mem;
+		}
+		kfd_mem_limit.system_mem_used += acc_size;
+		kfd_mem_limit.userptr_mem_used += size;
 	}
 err_no_mem:
 	spin_unlock(&kfd_mem_limit.mem_limit_lock);
@@ -126,10 +152,16 @@ static void unreserve_system_mem_limit(struct amdgpu_device *adev,
 				       sizeof(struct amdgpu_bo));
 
 	spin_lock(&kfd_mem_limit.mem_limit_lock);
-	if (domain == AMDGPU_GEM_DOMAIN_GTT)
+	if (domain == AMDGPU_GEM_DOMAIN_GTT) {
 		kfd_mem_limit.system_mem_used -= (acc_size + size);
+	} else if (domain == AMDGPU_GEM_DOMAIN_CPU) {
+		kfd_mem_limit.system_mem_used -= acc_size;
+		kfd_mem_limit.userptr_mem_used -= size;
+	}
 	WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
 		  "kfd system memory accounting unbalanced");
+	WARN_ONCE(kfd_mem_limit.userptr_mem_used < 0,
+		  "kfd userptr memory accounting unbalanced");
 
 	spin_unlock(&kfd_mem_limit.mem_limit_lock);
 }
@@ -138,12 +170,17 @@ void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo)
 {
 	spin_lock(&kfd_mem_limit.mem_limit_lock);
 
-	if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT) {
+	if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) {
+		kfd_mem_limit.system_mem_used -= bo->tbo.acc_size;
+		kfd_mem_limit.userptr_mem_used -= amdgpu_bo_size(bo);
+	} else if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT) {
 		kfd_mem_limit.system_mem_used -=
 			(bo->tbo.acc_size + amdgpu_bo_size(bo));
 	}
 	WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
 		  "kfd system memory accounting unbalanced");
+	WARN_ONCE(kfd_mem_limit.userptr_mem_used < 0,
+		  "kfd userptr memory accounting unbalanced");
 
 	spin_unlock(&kfd_mem_limit.mem_limit_lock);
 }
@@ -506,7 +543,8 @@ static void remove_bo_from_vm(struct amdgpu_device *adev,
 }
 
 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
-				struct amdkfd_process_info *process_info)
+				struct amdkfd_process_info *process_info,
+				bool userptr)
 {
 	struct ttm_validate_buffer *entry = &mem->validate_list;
 	struct amdgpu_bo *bo = mem->bo;
@@ -515,10 +553,95 @@ static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
 	entry->shared = true;
 	entry->bo = &bo->tbo;
 	mutex_lock(&process_info->lock);
-	list_add_tail(&entry->head, &process_info->kfd_bo_list);
+	if (userptr)
+		list_add_tail(&entry->head, &process_info->userptr_valid_list);
+	else
+		list_add_tail(&entry->head, &process_info->kfd_bo_list);
 	mutex_unlock(&process_info->lock);
 }
 
+/* Initializes user pages. It registers the MMU notifier and validates
+ * the userptr BO in the GTT domain.
+ *
+ * The BO must already be on the userptr_valid_list. Otherwise an
+ * eviction and restore may happen that leaves the new BO unmapped
+ * with the user mode queues running.
+ *
+ * Takes the process_info->lock to protect against concurrent restore
+ * workers.
+ *
+ * Returns 0 for success, negative errno for errors.
+ */
+static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
+			   uint64_t user_addr)
+{
+	struct amdkfd_process_info *process_info = mem->process_info;
+	struct amdgpu_bo *bo = mem->bo;
+	struct ttm_operation_ctx ctx = { true, false };
+	int ret = 0;
+
+	mutex_lock(&process_info->lock);
+
+	ret = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, user_addr, 0);
+	if (ret) {
+		pr_err("%s: Failed to set userptr: %d\n", __func__, ret);
+		goto out;
+	}
+
+	ret = amdgpu_mn_register(bo, user_addr);
+	if (ret) {
+		pr_err("%s: Failed to register MMU notifier: %d\n",
+		       __func__, ret);
+		goto out;
+	}
+
+	/* If no restore worker is running concurrently, user_pages
+	 * should not be allocated
+	 */
+	WARN(mem->user_pages, "Leaking user_pages array");
+
+	mem->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,
+					   sizeof(struct page *),
+					   GFP_KERNEL | __GFP_ZERO);
+	if (!mem->user_pages) {
+		pr_err("%s: Failed to allocate pages array\n", __func__);
+		ret = -ENOMEM;
+		goto unregister_out;
+	}
+
+	ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, mem->user_pages);
+	if (ret) {
+		pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
+		goto free_out;
+	}
+
+	amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, mem->user_pages);
+
+	ret = amdgpu_bo_reserve(bo, true);
+	if (ret) {
+		pr_err("%s: Failed to reserve BO\n", __func__);
+		goto release_out;
+	}
+	amdgpu_ttm_placement_from_domain(bo, mem->domain);
+	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+	if (ret)
+		pr_err("%s: failed to validate BO\n", __func__);
+	amdgpu_bo_unreserve(bo);
+
+release_out:
+	if (ret)
+		release_pages(mem->user_pages, bo->tbo.ttm->num_pages);
+free_out:
+	kvfree(mem->user_pages);
+	mem->user_pages = NULL;
+unregister_out:
+	if (ret)
+		amdgpu_mn_unregister(bo);
+out:
+	mutex_unlock(&process_info->lock);
+	return ret;
+}
+
 /* Reserving a BO and its page table BOs must happen atomically to
  * avoid deadlocks. Some operations update multiple VMs at once. Track
  * all the reservation info in a context structure. Optionally a sync
@@ -748,7 +871,8 @@ static int update_gpuvm_pte(struct amdgpu_device *adev,
 }
 
 static int map_bo_to_gpuvm(struct amdgpu_device *adev,
-		struct kfd_bo_va_list *entry, struct amdgpu_sync *sync)
+		struct kfd_bo_va_list *entry, struct amdgpu_sync *sync,
+		bool no_update_pte)
 {
 	int ret;
 
@@ -762,6 +886,9 @@ static int map_bo_to_gpuvm(struct amdgpu_device *adev,
 		return ret;
 	}
 
+	if (no_update_pte)
+		return 0;
+
 	ret = update_gpuvm_pte(adev, entry, sync);
 	if (ret) {
 		pr_err("update_gpuvm_pte() failed\n");
@@ -820,6 +947,8 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
 		mutex_init(&info->lock);
 		INIT_LIST_HEAD(&info->vm_list_head);
 		INIT_LIST_HEAD(&info->kfd_bo_list);
+		INIT_LIST_HEAD(&info->userptr_valid_list);
+		INIT_LIST_HEAD(&info->userptr_inval_list);
 
 		info->eviction_fence =
 			amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
@@ -830,6 +959,11 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
 			goto create_evict_fence_fail;
 		}
 
+		info->pid = get_task_pid(current->group_leader, PIDTYPE_PID);
+		atomic_set(&info->evicted_bos, 0);
+		INIT_DELAYED_WORK(&info->restore_userptr_work,
+				  amdgpu_amdkfd_restore_userptr_worker);
+
 		*process_info = info;
 		*ef = dma_fence_get(&info->eviction_fence->base);
 	}
@@ -872,6 +1006,7 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
 		dma_fence_put(*ef);
 		*ef = NULL;
 		*process_info = NULL;
+		put_pid(info->pid);
 create_evict_fence_fail:
 		mutex_destroy(&info->lock);
 		kfree(info);
@@ -967,8 +1102,12 @@ void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
 	/* Release per-process resources when last compute VM is destroyed */
 	if (!process_info->n_vms) {
 		WARN_ON(!list_empty(&process_info->kfd_bo_list));
+		WARN_ON(!list_empty(&process_info->userptr_valid_list));
+		WARN_ON(!list_empty(&process_info->userptr_inval_list));
 
 		dma_fence_put(&process_info->eviction_fence->base);
+		cancel_delayed_work_sync(&process_info->restore_userptr_work);
+		put_pid(process_info->pid);
 		mutex_destroy(&process_info->lock);
 		kfree(process_info);
 	}
@@ -1003,9 +1142,11 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
 	struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
+	uint64_t user_addr = 0;
 	struct amdgpu_bo *bo;
+	struct amdgpu_bo_param bp;
 	int byte_align;
-	u32 alloc_domain;
+	u32 domain, alloc_domain;
 	u64 alloc_flags;
 	uint32_t mapping_flags;
 	int ret;
@@ -1014,14 +1155,21 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 	 * Check on which domain to allocate BO
 	 */
 	if (flags & ALLOC_MEM_FLAGS_VRAM) {
-		alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
+		domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
 		alloc_flags = AMDGPU_GEM_CREATE_VRAM_CLEARED;
 		alloc_flags |= (flags & ALLOC_MEM_FLAGS_PUBLIC) ?
 			AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED :
 			AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
 	} else if (flags & ALLOC_MEM_FLAGS_GTT) {
-		alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
+		domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
+		alloc_flags = 0;
+	} else if (flags & ALLOC_MEM_FLAGS_USERPTR) {
+		domain = AMDGPU_GEM_DOMAIN_GTT;
+		alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
 		alloc_flags = 0;
+		if (!offset || !*offset)
+			return -EINVAL;
+		user_addr = *offset;
 	} else {
 		return -EINVAL;
 	}
@@ -1069,8 +1217,14 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 	pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s\n",
 			va, size, domain_string(alloc_domain));
 
-	ret = amdgpu_bo_create(adev, size, byte_align,
-				alloc_domain, alloc_flags, ttm_bo_type_device, NULL, &bo);
+	memset(&bp, 0, sizeof(bp));
+	bp.size = size;
+	bp.byte_align = byte_align;
+	bp.domain = alloc_domain;
+	bp.flags = alloc_flags;
+	bp.type = ttm_bo_type_device;
+	bp.resv = NULL;
+	ret = amdgpu_bo_create(adev, &bp, &bo);
 	if (ret) {
 		pr_debug("Failed to create BO on domain %s. ret %d\n",
 				domain_string(alloc_domain), ret);
@@ -1078,18 +1232,34 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 	}
 	bo->kfd_bo = *mem;
 	(*mem)->bo = bo;
+	if (user_addr)
+		bo->flags |= AMDGPU_AMDKFD_USERPTR_BO;
 
 	(*mem)->va = va;
-	(*mem)->domain = alloc_domain;
+	(*mem)->domain = domain;
 	(*mem)->mapped_to_gpu_memory = 0;
 	(*mem)->process_info = avm->process_info;
-	add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info);
+	add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);
+
+	if (user_addr) {
+		ret = init_user_pages(*mem, current->mm, user_addr);
+		if (ret) {
+			mutex_lock(&avm->process_info->lock);
+			list_del(&(*mem)->validate_list.head);
+			mutex_unlock(&avm->process_info->lock);
+			goto allocate_init_user_pages_failed;
+		}
+	}
 
 	if (offset)
 		*offset = amdgpu_bo_mmap_offset(bo);
 
 	return 0;
 
+allocate_init_user_pages_failed:
+	amdgpu_bo_unref(&bo);
+	/* Don't unreserve system mem limit twice */
+	goto err_reserve_system_mem;
 err_bo_create:
 	unreserve_system_mem_limit(adev, size, alloc_domain);
 err_reserve_system_mem:
@@ -1122,12 +1292,24 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
 	 * be freed anyway
 	 */
 
+	/* No more MMU notifiers */
+	amdgpu_mn_unregister(mem->bo);
+
 	/* Make sure restore workers don't access the BO any more */
 	bo_list_entry = &mem->validate_list;
 	mutex_lock(&process_info->lock);
 	list_del(&bo_list_entry->head);
 	mutex_unlock(&process_info->lock);
 
+	/* Free user pages if necessary */
+	if (mem->user_pages) {
+		pr_debug("%s: Freeing user_pages array\n", __func__);
+		if (mem->user_pages[0])
+			release_pages(mem->user_pages,
+					mem->bo->tbo.ttm->num_pages);
+		kvfree(mem->user_pages);
+	}
+
 	ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
 	if (unlikely(ret))
 		return ret;
@@ -1173,21 +1355,32 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
 	struct kfd_bo_va_list *bo_va_entry = NULL;
 	struct kfd_bo_va_list *bo_va_entry_aql = NULL;
 	unsigned long bo_size;
-
-	/* Make sure restore is not running concurrently.
-	 */
-	mutex_lock(&mem->process_info->lock);
-
-	mutex_lock(&mem->lock);
+	bool is_invalid_userptr = false;
 
 	bo = mem->bo;
-
 	if (!bo) {
 		pr_err("Invalid BO when mapping memory to GPU\n");
-		ret = -EINVAL;
-		goto out;
+		return -EINVAL;
 	}
 
+	/* Make sure restore is not running concurrently. Since we
+	 * don't map invalid userptr BOs, we rely on the next restore
+	 * worker to do the mapping
+	 */
+	mutex_lock(&mem->process_info->lock);
+
+	/* Lock mmap-sem. If we find an invalid userptr BO, we can be
+	 * sure that the MMU notifier is no longer running
+	 * concurrently and the queues are actually stopped
+	 */
+	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
+		down_write(&current->mm->mmap_sem);
+		is_invalid_userptr = atomic_read(&mem->invalid);
+		up_write(&current->mm->mmap_sem);
+	}
+
+	mutex_lock(&mem->lock);
+
 	domain = mem->domain;
 	bo_size = bo->tbo.mem.size;
 
@@ -1200,6 +1393,14 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
 	if (unlikely(ret))
 		goto out;
 
+	/* Userptr can be marked as "not invalid", but not actually be
+	 * validated yet (still in the system domain). In that case
+	 * the queues are still stopped and we can leave mapping for
+	 * the next restore worker
+	 */
+	if (bo->tbo.mem.mem_type == TTM_PL_SYSTEM)
+		is_invalid_userptr = true;
+
 	if (check_if_add_bo_to_vm(avm, mem)) {
 		ret = add_bo_to_vm(adev, mem, avm, false,
 				&bo_va_entry);
@@ -1217,7 +1418,8 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
 			goto add_bo_to_vm_failed;
 	}
 
-	if (mem->mapped_to_gpu_memory == 0) {
+	if (mem->mapped_to_gpu_memory == 0 &&
+	    !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
 		/* Validate BO only once. The eviction fence gets added to BO
 		 * the first time it is mapped. Validate will wait for all
 		 * background evictions to complete.
@@ -1235,7 +1437,8 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
 					entry->va, entry->va + bo_size,
 					entry);
 
-			ret = map_bo_to_gpuvm(adev, entry, ctx.sync);
+			ret = map_bo_to_gpuvm(adev, entry, ctx.sync,
+					      is_invalid_userptr);
 			if (ret) {
 				pr_err("Failed to map radeon bo to gpuvm\n");
 				goto map_bo_to_gpuvm_failed;
@@ -1418,6 +1621,337 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
 	return ret;
 }
 
+/* Evict a userptr BO by stopping the queues if necessary
+ *
+ * Runs in MMU notifier, may be in RECLAIM_FS context. This means it
+ * cannot do any memory allocations, and cannot take any locks that
+ * are held elsewhere while allocating memory. Therefore this is as
+ * simple as possible, using atomic counters.
+ *
+ * It doesn't do anything to the BO itself. The real work happens in
+ * restore, where we get updated page addresses. This function only
+ * ensures that GPU access to the BO is stopped.
+ */
+int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem,
+				struct mm_struct *mm)
+{
+	struct amdkfd_process_info *process_info = mem->process_info;
+	int invalid, evicted_bos;
+	int r = 0;
+
+	invalid = atomic_inc_return(&mem->invalid);
+	evicted_bos = atomic_inc_return(&process_info->evicted_bos);
+	if (evicted_bos == 1) {
+		/* First eviction, stop the queues */
+		r = kgd2kfd->quiesce_mm(mm);
+		if (r)
+			pr_err("Failed to quiesce KFD\n");
+		schedule_delayed_work(&process_info->restore_userptr_work,
+			msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
+	}
+
+	return r;
+}
+
+/* Update invalid userptr BOs
+ *
+ * Moves invalidated (evicted) userptr BOs from userptr_valid_list to
+ * userptr_inval_list and updates user pages for all BOs that have
+ * been invalidated since their last update.
+ */
+static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
+				     struct mm_struct *mm)
+{
+	struct kgd_mem *mem, *tmp_mem;
+	struct amdgpu_bo *bo;
+	struct ttm_operation_ctx ctx = { false, false };
+	int invalid, ret;
+
+	/* Move all invalidated BOs to the userptr_inval_list and
+	 * release their user pages by migration to the CPU domain
+	 */
+	list_for_each_entry_safe(mem, tmp_mem,
+				 &process_info->userptr_valid_list,
+				 validate_list.head) {
+		if (!atomic_read(&mem->invalid))
+			continue; /* BO is still valid */
+
+		bo = mem->bo;
+
+		if (amdgpu_bo_reserve(bo, true))
+			return -EAGAIN;
+		amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
+		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+		amdgpu_bo_unreserve(bo);
+		if (ret) {
+			pr_err("%s: Failed to invalidate userptr BO\n",
+			       __func__);
+			return -EAGAIN;
+		}
+
+		list_move_tail(&mem->validate_list.head,
+			       &process_info->userptr_inval_list);
+	}
+
+	if (list_empty(&process_info->userptr_inval_list))
+		return 0; /* All evicted userptr BOs were freed */
+
+	/* Go through userptr_inval_list and update any invalid user_pages */
+	list_for_each_entry(mem, &process_info->userptr_inval_list,
+			    validate_list.head) {
+		invalid = atomic_read(&mem->invalid);
+		if (!invalid)
+			/* BO hasn't been invalidated since the last
+			 * revalidation attempt. Keep its BO list.
+			 */
+			continue;
+
+		bo = mem->bo;
+
+		if (!mem->user_pages) {
+			mem->user_pages =
+				kvmalloc_array(bo->tbo.ttm->num_pages,
+						 sizeof(struct page *),
+						 GFP_KERNEL | __GFP_ZERO);
+			if (!mem->user_pages) {
+				pr_err("%s: Failed to allocate pages array\n",
+				       __func__);
+				return -ENOMEM;
+			}
+		} else if (mem->user_pages[0]) {
+			release_pages(mem->user_pages, bo->tbo.ttm->num_pages);
+		}
+
+		/* Get updated user pages */
+		ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
+						   mem->user_pages);
+		if (ret) {
+			mem->user_pages[0] = NULL;
+			pr_info("%s: Failed to get user pages: %d\n",
+				__func__, ret);
+			/* Pretend it succeeded. It will fail later
+			 * with a VM fault if the GPU tries to access
+			 * it. Better than hanging indefinitely with
+			 * stalled user mode queues.
+			 */
+		}
+
+		/* Mark the BO as valid unless it was invalidated
+		 * again concurrently
+		 */
+		if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid)
+			return -EAGAIN;
+	}
+
+	return 0;
+}
+
+/* Validate invalid userptr BOs
+ *
+ * Validates BOs on the userptr_inval_list, and moves them back to the
+ * userptr_valid_list. Also updates GPUVM page tables with new page
+ * addresses and waits for the page table updates to complete.
+ */
+static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
+{
+	struct amdgpu_bo_list_entry *pd_bo_list_entries;
+	struct list_head resv_list, duplicates;
+	struct ww_acquire_ctx ticket;
+	struct amdgpu_sync sync;
+
+	struct amdgpu_vm *peer_vm;
+	struct kgd_mem *mem, *tmp_mem;
+	struct amdgpu_bo *bo;
+	struct ttm_operation_ctx ctx = { false, false };
+	int i, ret;
+
+	pd_bo_list_entries = kcalloc(process_info->n_vms,
+				     sizeof(struct amdgpu_bo_list_entry),
+				     GFP_KERNEL);
+	if (!pd_bo_list_entries) {
+		pr_err("%s: Failed to allocate PD BO list entries\n", __func__);
+		return -ENOMEM;
+	}
+
+	INIT_LIST_HEAD(&resv_list);
+	INIT_LIST_HEAD(&duplicates);
+
+	/* Get all the page directory BOs that need to be reserved */
+	i = 0;
+	list_for_each_entry(peer_vm, &process_info->vm_list_head,
+			    vm_list_node)
+		amdgpu_vm_get_pd_bo(peer_vm, &resv_list,
+				    &pd_bo_list_entries[i++]);
+	/* Add the userptr_inval_list entries to resv_list */
+	list_for_each_entry(mem, &process_info->userptr_inval_list,
+			    validate_list.head) {
+		list_add_tail(&mem->resv_list.head, &resv_list);
+		mem->resv_list.bo = mem->validate_list.bo;
+		mem->resv_list.shared = mem->validate_list.shared;
+	}
+
+	/* Reserve all BOs and page tables for validation */
+	ret = ttm_eu_reserve_buffers(&ticket, &resv_list, false, &duplicates);
+	WARN(!list_empty(&duplicates), "Duplicates should be empty");
+	if (ret)
+		goto out;
+
+	amdgpu_sync_create(&sync);
+
+	/* Avoid triggering eviction fences when unmapping invalid
+	 * userptr BOs (waits for all fences, doesn't use
+	 * FENCE_OWNER_VM)
+	 */
+	list_for_each_entry(peer_vm, &process_info->vm_list_head,
+			    vm_list_node)
+		amdgpu_amdkfd_remove_eviction_fence(peer_vm->root.base.bo,
+						process_info->eviction_fence,
+						NULL, NULL);
+
+	ret = process_validate_vms(process_info);
+	if (ret)
+		goto unreserve_out;
+
+	/* Validate BOs and update GPUVM page tables */
+	list_for_each_entry_safe(mem, tmp_mem,
+				 &process_info->userptr_inval_list,
+				 validate_list.head) {
+		struct kfd_bo_va_list *bo_va_entry;
+
+		bo = mem->bo;
+
+		/* Copy pages array and validate the BO if we got user pages */
+		if (mem->user_pages[0]) {
+			amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
+						     mem->user_pages);
+			amdgpu_ttm_placement_from_domain(bo, mem->domain);
+			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+			if (ret) {
+				pr_err("%s: failed to validate BO\n", __func__);
+				goto unreserve_out;
+			}
+		}
+
+		/* Validate succeeded, now the BO owns the pages, free
+		 * our copy of the pointer array. Put this BO back on
+		 * the userptr_valid_list. If we need to revalidate
+		 * it, we need to start from scratch.
+		 */
+		kvfree(mem->user_pages);
+		mem->user_pages = NULL;
+		list_move_tail(&mem->validate_list.head,
+			       &process_info->userptr_valid_list);
+
+		/* Update mapping. If the BO was not validated
+		 * (because we couldn't get user pages), this will
+		 * clear the page table entries, which will result in
+		 * VM faults if the GPU tries to access the invalid
+		 * memory.
+		 */
+		list_for_each_entry(bo_va_entry, &mem->bo_va_list, bo_list) {
+			if (!bo_va_entry->is_mapped)
+				continue;
+
+			ret = update_gpuvm_pte((struct amdgpu_device *)
+					       bo_va_entry->kgd_dev,
+					       bo_va_entry, &sync);
+			if (ret) {
+				pr_err("%s: update PTE failed\n", __func__);
+				/* make sure this gets validated again */
+				atomic_inc(&mem->invalid);
+				goto unreserve_out;
+			}
+		}
+	}
+
+	/* Update page directories */
+	ret = process_update_pds(process_info, &sync);
+
+unreserve_out:
+	list_for_each_entry(peer_vm, &process_info->vm_list_head,
+			    vm_list_node)
+		amdgpu_bo_fence(peer_vm->root.base.bo,
+				&process_info->eviction_fence->base, true);
+	ttm_eu_backoff_reservation(&ticket, &resv_list);
+	amdgpu_sync_wait(&sync, false);
+	amdgpu_sync_free(&sync);
+out:
+	kfree(pd_bo_list_entries);
+
+	return ret;
+}
+
+/* Worker callback to restore evicted userptr BOs
+ *
+ * Tries to update and validate all userptr BOs. If successful and no
+ * concurrent evictions happened, the queues are restarted. Otherwise,
+ * reschedule for another attempt later.
+ */
+static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct amdkfd_process_info *process_info =
+		container_of(dwork, struct amdkfd_process_info,
+			     restore_userptr_work);
+	struct task_struct *usertask;
+	struct mm_struct *mm;
+	int evicted_bos;
+
+	evicted_bos = atomic_read(&process_info->evicted_bos);
+	if (!evicted_bos)
+		return;
+
+	/* Reference task and mm in case of concurrent process termination */
+	usertask = get_pid_task(process_info->pid, PIDTYPE_PID);
+	if (!usertask)
+		return;
+	mm = get_task_mm(usertask);
+	if (!mm) {
+		put_task_struct(usertask);
+		return;
+	}
+
+	mutex_lock(&process_info->lock);
+
+	if (update_invalid_user_pages(process_info, mm))
+		goto unlock_out;
+	/* userptr_inval_list can be empty if all evicted userptr BOs
+	 * have been freed. In that case there is nothing to validate
+	 * and we can just restart the queues.
+	 */
+	if (!list_empty(&process_info->userptr_inval_list)) {
+		if (atomic_read(&process_info->evicted_bos) != evicted_bos)
+			goto unlock_out; /* Concurrent eviction, try again */
+
+		if (validate_invalid_user_pages(process_info))
+			goto unlock_out;
+	}
+	/* Final check for concurrent evicton and atomic update. If
+	 * another eviction happens after successful update, it will
+	 * be a first eviction that calls quiesce_mm. The eviction
+	 * reference counting inside KFD will handle this case.
+	 */
+	if (atomic_cmpxchg(&process_info->evicted_bos, evicted_bos, 0) !=
+	    evicted_bos)
+		goto unlock_out;
+	evicted_bos = 0;
+	if (kgd2kfd->resume_mm(mm)) {
+		pr_err("%s: Failed to resume KFD\n", __func__);
+		/* No recovery from this failure. Probably the CP is
+		 * hanging. No point trying again.
+		 */
+	}
+unlock_out:
+	mutex_unlock(&process_info->lock);
+	mmput(mm);
+	put_task_struct(usertask);
+
+	/* If validation failed, reschedule another attempt */
+	if (evicted_bos)
+		schedule_delayed_work(&process_info->restore_userptr_work,
+			msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
+}
+
 /** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
  *   KFD process identified by process_info
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index a0f48cb9b8f0..236915849cfe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -322,3 +322,47 @@ int amdgpu_atomfirmware_get_clock_info(struct amdgpu_device *adev)
 
 	return ret;
 }
+
+union gfx_info {
+	struct  atom_gfx_info_v2_4 v24;
+};
+
+int amdgpu_atomfirmware_get_gfx_info(struct amdgpu_device *adev)
+{
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	int index;
+	uint8_t frev, crev;
+	uint16_t data_offset;
+
+	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+					    gfx_info);
+	if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,
+				   &frev, &crev, &data_offset)) {
+		union gfx_info *gfx_info = (union gfx_info *)
+			(mode_info->atom_context->bios + data_offset);
+		switch (crev) {
+		case 4:
+			adev->gfx.config.max_shader_engines = gfx_info->v24.gc_num_se;
+			adev->gfx.config.max_cu_per_sh = gfx_info->v24.gc_num_cu_per_sh;
+			adev->gfx.config.max_sh_per_se = gfx_info->v24.gc_num_sh_per_se;
+			adev->gfx.config.max_backends_per_se = gfx_info->v24.gc_num_rb_per_se;
+			adev->gfx.config.max_texture_channel_caches = gfx_info->v24.gc_num_tccs;
+			adev->gfx.config.max_gprs = le16_to_cpu(gfx_info->v24.gc_num_gprs);
+			adev->gfx.config.max_gs_threads = gfx_info->v24.gc_num_max_gs_thds;
+			adev->gfx.config.gs_vgt_table_depth = gfx_info->v24.gc_gs_table_depth;
+			adev->gfx.config.gs_prim_buffer_depth =
+				le16_to_cpu(gfx_info->v24.gc_gsprim_buff_depth);
+			adev->gfx.config.double_offchip_lds_buf =
+				gfx_info->v24.gc_double_offchip_lds_buffer;
+			adev->gfx.cu_info.wave_front_size = le16_to_cpu(gfx_info->v24.gc_wave_size);
+			adev->gfx.cu_info.max_waves_per_simd = le16_to_cpu(gfx_info->v24.gc_max_waves_per_simd);
+			adev->gfx.cu_info.max_scratch_slots_per_cu = gfx_info->v24.gc_max_scratch_slots_per_cu;
+			adev->gfx.cu_info.lds_size = le16_to_cpu(gfx_info->v24.gc_lds_size);
+			return 0;
+		default:
+			return -EINVAL;
+		}
+
+	}
+	return -EINVAL;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
index 7689c961c4ef..20f158fd3b76 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
@@ -30,5 +30,6 @@ int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev);
 int amdgpu_atomfirmware_get_vram_width(struct amdgpu_device *adev);
 int amdgpu_atomfirmware_get_vram_type(struct amdgpu_device *adev);
 int amdgpu_atomfirmware_get_clock_info(struct amdgpu_device *adev);
+int amdgpu_atomfirmware_get_gfx_info(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
index 1ae5ae8c45a4..1bcb2b247335 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
@@ -550,7 +550,7 @@ static int amdgpu_atpx_init(void)
  * look up whether we are the integrated or discrete GPU (all asics).
  * Returns the client id.
  */
-static int amdgpu_atpx_get_client_id(struct pci_dev *pdev)
+static enum vga_switcheroo_client_id amdgpu_atpx_get_client_id(struct pci_dev *pdev)
 {
 	if (amdgpu_atpx_priv.dhandle == ACPI_HANDLE(&pdev->dev))
 		return VGA_SWITCHEROO_IGD;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
index 02b849be083b..19cfff31f2e1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
@@ -75,13 +75,20 @@ static void amdgpu_benchmark_move(struct amdgpu_device *adev, unsigned size,
 {
 	struct amdgpu_bo *dobj = NULL;
 	struct amdgpu_bo *sobj = NULL;
+	struct amdgpu_bo_param bp;
 	uint64_t saddr, daddr;
 	int r, n;
 	int time;
 
+	memset(&bp, 0, sizeof(bp));
+	bp.size = size;
+	bp.byte_align = PAGE_SIZE;
+	bp.domain = sdomain;
+	bp.flags = 0;
+	bp.type = ttm_bo_type_kernel;
+	bp.resv = NULL;
 	n = AMDGPU_BENCHMARK_ITERATIONS;
-	r = amdgpu_bo_create(adev, size, PAGE_SIZE,sdomain, 0,
-			     ttm_bo_type_kernel, NULL, &sobj);
+	r = amdgpu_bo_create(adev, &bp, &sobj);
 	if (r) {
 		goto out_cleanup;
 	}
@@ -93,8 +100,8 @@ static void amdgpu_benchmark_move(struct amdgpu_device *adev, unsigned size,
 	if (r) {
 		goto out_cleanup;
 	}
-	r = amdgpu_bo_create(adev, size, PAGE_SIZE, ddomain, 0,
-			     ttm_bo_type_kernel, NULL, &dobj);
+	bp.domain = ddomain;
+	r = amdgpu_bo_create(adev, &bp, &dobj);
 	if (r) {
 		goto out_cleanup;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index 71a57b2f7f04..e950730f1933 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -23,7 +23,6 @@
  */
 #include <linux/list.h>
 #include <linux/slab.h>
-#include <linux/pci.h>
 #include <drm/drmP.h>
 #include <linux/firmware.h>
 #include <drm/amdgpu_drm.h>
@@ -109,121 +108,6 @@ static void amdgpu_cgs_write_ind_register(struct cgs_device *cgs_device,
 	WARN(1, "Invalid indirect register space");
 }
 
-static int amdgpu_cgs_get_pci_resource(struct cgs_device *cgs_device,
-				       enum cgs_resource_type resource_type,
-				       uint64_t size,
-				       uint64_t offset,
-				       uint64_t *resource_base)
-{
-	CGS_FUNC_ADEV;
-
-	if (resource_base == NULL)
-		return -EINVAL;
-
-	switch (resource_type) {
-	case CGS_RESOURCE_TYPE_MMIO:
-		if (adev->rmmio_size == 0)
-			return -ENOENT;
-		if ((offset + size) > adev->rmmio_size)
-			return -EINVAL;
-		*resource_base = adev->rmmio_base;
-		return 0;
-	case CGS_RESOURCE_TYPE_DOORBELL:
-		if (adev->doorbell.size == 0)
-			return -ENOENT;
-		if ((offset + size) > adev->doorbell.size)
-			return -EINVAL;
-		*resource_base = adev->doorbell.base;
-		return 0;
-	case CGS_RESOURCE_TYPE_FB:
-	case CGS_RESOURCE_TYPE_IO:
-	case CGS_RESOURCE_TYPE_ROM:
-	default:
-		return -EINVAL;
-	}
-}
-
-static const void *amdgpu_cgs_atom_get_data_table(struct cgs_device *cgs_device,
-						  unsigned table, uint16_t *size,
-						  uint8_t *frev, uint8_t *crev)
-{
-	CGS_FUNC_ADEV;
-	uint16_t data_start;
-
-	if (amdgpu_atom_parse_data_header(
-		    adev->mode_info.atom_context, table, size,
-		    frev, crev, &data_start))
-		return (uint8_t*)adev->mode_info.atom_context->bios +
-			data_start;
-
-	return NULL;
-}
-
-static int amdgpu_cgs_atom_get_cmd_table_revs(struct cgs_device *cgs_device, unsigned table,
-					      uint8_t *frev, uint8_t *crev)
-{
-	CGS_FUNC_ADEV;
-
-	if (amdgpu_atom_parse_cmd_header(
-		    adev->mode_info.atom_context, table,
-		    frev, crev))
-		return 0;
-
-	return -EINVAL;
-}
-
-static int amdgpu_cgs_atom_exec_cmd_table(struct cgs_device *cgs_device, unsigned table,
-					  void *args)
-{
-	CGS_FUNC_ADEV;
-
-	return amdgpu_atom_execute_table(
-		adev->mode_info.atom_context, table, args);
-}
-
-static int amdgpu_cgs_set_clockgating_state(struct cgs_device *cgs_device,
-				  enum amd_ip_block_type block_type,
-				  enum amd_clockgating_state state)
-{
-	CGS_FUNC_ADEV;
-	int i, r = -1;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_blocks[i].status.valid)
-			continue;
-
-		if (adev->ip_blocks[i].version->type == block_type) {
-			r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
-								(void *)adev,
-									state);
-			break;
-		}
-	}
-	return r;
-}
-
-static int amdgpu_cgs_set_powergating_state(struct cgs_device *cgs_device,
-				  enum amd_ip_block_type block_type,
-				  enum amd_powergating_state state)
-{
-	CGS_FUNC_ADEV;
-	int i, r = -1;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_blocks[i].status.valid)
-			continue;
-
-		if (adev->ip_blocks[i].version->type == block_type) {
-			r = adev->ip_blocks[i].version->funcs->set_powergating_state(
-								(void *)adev,
-									state);
-			break;
-		}
-	}
-	return r;
-}
-
-
 static uint32_t fw_type_convert(struct cgs_device *cgs_device, uint32_t fw_type)
 {
 	CGS_FUNC_ADEV;
@@ -271,18 +155,6 @@ static uint32_t fw_type_convert(struct cgs_device *cgs_device, uint32_t fw_type)
 	return result;
 }
 
-static int amdgpu_cgs_rel_firmware(struct cgs_device *cgs_device, enum cgs_ucode_id type)
-{
-	CGS_FUNC_ADEV;
-	if ((CGS_UCODE_ID_SMU == type) || (CGS_UCODE_ID_SMU_SK == type)) {
-		release_firmware(adev->pm.fw);
-		adev->pm.fw = NULL;
-		return 0;
-	}
-	/* cannot release other firmware because they are not created by cgs */
-	return -EINVAL;
-}
-
 static uint16_t amdgpu_get_firmware_version(struct cgs_device *cgs_device,
 					enum cgs_ucode_id type)
 {
@@ -326,34 +198,6 @@ static uint16_t amdgpu_get_firmware_version(struct cgs_device *cgs_device,
 	return fw_version;
 }
 
-static int amdgpu_cgs_enter_safe_mode(struct cgs_device *cgs_device,
-					bool en)
-{
-	CGS_FUNC_ADEV;
-
-	if (adev->gfx.rlc.funcs->enter_safe_mode == NULL ||
-		adev->gfx.rlc.funcs->exit_safe_mode == NULL)
-		return 0;
-
-	if (en)
-		adev->gfx.rlc.funcs->enter_safe_mode(adev);
-	else
-		adev->gfx.rlc.funcs->exit_safe_mode(adev);
-
-	return 0;
-}
-
-static void amdgpu_cgs_lock_grbm_idx(struct cgs_device *cgs_device,
-					bool lock)
-{
-	CGS_FUNC_ADEV;
-
-	if (lock)
-		mutex_lock(&adev->grbm_idx_mutex);
-	else
-		mutex_unlock(&adev->grbm_idx_mutex);
-}
-
 static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 					enum cgs_ucode_id type,
 					struct cgs_firmware_info *info)
@@ -541,6 +385,9 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 			case CHIP_POLARIS12:
 				strcpy(fw_name, "amdgpu/polaris12_smc.bin");
 				break;
+			case CHIP_VEGAM:
+				strcpy(fw_name, "amdgpu/vegam_smc.bin");
+				break;
 			case CHIP_VEGA10:
 				if ((adev->pdev->device == 0x687f) &&
 					((adev->pdev->revision == 0xc0) ||
@@ -553,6 +400,9 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 			case CHIP_VEGA12:
 				strcpy(fw_name, "amdgpu/vega12_smc.bin");
 				break;
+			case CHIP_VEGA20:
+				strcpy(fw_name, "amdgpu/vega20_smc.bin");
+				break;
 			default:
 				DRM_ERROR("SMC firmware not supported\n");
 				return -EINVAL;
@@ -598,97 +448,12 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 	return 0;
 }
 
-static int amdgpu_cgs_is_virtualization_enabled(void *cgs_device)
-{
-	CGS_FUNC_ADEV;
-	return amdgpu_sriov_vf(adev);
-}
-
-static int amdgpu_cgs_get_active_displays_info(struct cgs_device *cgs_device,
-					  struct cgs_display_info *info)
-{
-	CGS_FUNC_ADEV;
-	struct cgs_mode_info *mode_info;
-
-	if (info == NULL)
-		return -EINVAL;
-
-	mode_info = info->mode_info;
-	if (mode_info)
-		/* if the displays are off, vblank time is max */
-		mode_info->vblank_time_us = 0xffffffff;
-
-	if (!amdgpu_device_has_dc_support(adev)) {
-		struct amdgpu_crtc *amdgpu_crtc;
-		struct drm_device *ddev = adev->ddev;
-		struct drm_crtc *crtc;
-		uint32_t line_time_us, vblank_lines;
-
-		if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) {
-			list_for_each_entry(crtc,
-					&ddev->mode_config.crtc_list, head) {
-				amdgpu_crtc = to_amdgpu_crtc(crtc);
-				if (crtc->enabled) {
-					info->active_display_mask |= (1 << amdgpu_crtc->crtc_id);
-					info->display_count++;
-				}
-				if (mode_info != NULL &&
-					crtc->enabled && amdgpu_crtc->enabled &&
-					amdgpu_crtc->hw_mode.clock) {
-					line_time_us = (amdgpu_crtc->hw_mode.crtc_htotal * 1000) /
-								amdgpu_crtc->hw_mode.clock;
-					vblank_lines = amdgpu_crtc->hw_mode.crtc_vblank_end -
-								amdgpu_crtc->hw_mode.crtc_vdisplay +
-								(amdgpu_crtc->v_border * 2);
-					mode_info->vblank_time_us = vblank_lines * line_time_us;
-					mode_info->refresh_rate = drm_mode_vrefresh(&amdgpu_crtc->hw_mode);
-					/* we have issues with mclk switching with refresh rates
-					 * over 120 hz on the non-DC code.
-					 */
-					if (mode_info->refresh_rate > 120)
-						mode_info->vblank_time_us = 0;
-					mode_info = NULL;
-				}
-			}
-		}
-	} else {
-		info->display_count = adev->pm.pm_display_cfg.num_display;
-		if (mode_info != NULL) {
-			mode_info->vblank_time_us = adev->pm.pm_display_cfg.min_vblank_time;
-			mode_info->refresh_rate = adev->pm.pm_display_cfg.vrefresh;
-		}
-	}
-	return 0;
-}
-
-
-static int amdgpu_cgs_notify_dpm_enabled(struct cgs_device *cgs_device, bool enabled)
-{
-	CGS_FUNC_ADEV;
-
-	adev->pm.dpm_enabled = enabled;
-
-	return 0;
-}
-
 static const struct cgs_ops amdgpu_cgs_ops = {
 	.read_register = amdgpu_cgs_read_register,
 	.write_register = amdgpu_cgs_write_register,
 	.read_ind_register = amdgpu_cgs_read_ind_register,
 	.write_ind_register = amdgpu_cgs_write_ind_register,
-	.get_pci_resource = amdgpu_cgs_get_pci_resource,
-	.atom_get_data_table = amdgpu_cgs_atom_get_data_table,
-	.atom_get_cmd_table_revs = amdgpu_cgs_atom_get_cmd_table_revs,
-	.atom_exec_cmd_table = amdgpu_cgs_atom_exec_cmd_table,
 	.get_firmware_info = amdgpu_cgs_get_firmware_info,
-	.rel_firmware = amdgpu_cgs_rel_firmware,
-	.set_powergating_state = amdgpu_cgs_set_powergating_state,
-	.set_clockgating_state = amdgpu_cgs_set_clockgating_state,
-	.get_active_displays_info = amdgpu_cgs_get_active_displays_info,
-	.notify_dpm_enabled = amdgpu_cgs_notify_dpm_enabled,
-	.is_virtualization_enabled = amdgpu_cgs_is_virtualization_enabled,
-	.enter_safe_mode = amdgpu_cgs_enter_safe_mode,
-	.lock_grbm_idx = amdgpu_cgs_lock_grbm_idx,
 };
 
 struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index 96501ff0e55b..8e66851eb427 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -691,7 +691,7 @@ static int amdgpu_connector_lvds_get_modes(struct drm_connector *connector)
 	return ret;
 }
 
-static int amdgpu_connector_lvds_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status amdgpu_connector_lvds_mode_valid(struct drm_connector *connector,
 					     struct drm_display_mode *mode)
 {
 	struct drm_encoder *encoder = amdgpu_connector_best_single_encoder(connector);
@@ -843,7 +843,7 @@ static int amdgpu_connector_vga_get_modes(struct drm_connector *connector)
 	return ret;
 }
 
-static int amdgpu_connector_vga_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status amdgpu_connector_vga_mode_valid(struct drm_connector *connector,
 					    struct drm_display_mode *mode)
 {
 	struct drm_device *dev = connector->dev;
@@ -1172,7 +1172,7 @@ static void amdgpu_connector_dvi_force(struct drm_connector *connector)
 		amdgpu_connector->use_digital = true;
 }
 
-static int amdgpu_connector_dvi_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status amdgpu_connector_dvi_mode_valid(struct drm_connector *connector,
 					    struct drm_display_mode *mode)
 {
 	struct drm_device *dev = connector->dev;
@@ -1448,7 +1448,7 @@ amdgpu_connector_dp_detect(struct drm_connector *connector, bool force)
 	return ret;
 }
 
-static int amdgpu_connector_dp_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status amdgpu_connector_dp_mode_valid(struct drm_connector *connector,
 					   struct drm_display_mode *mode)
 {
 	struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index dc34b50e6b29..9c1d491d742e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -382,8 +382,7 @@ static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
 
 	p->bytes_moved += ctx.bytes_moved;
 	if (adev->gmc.visible_vram_size < adev->gmc.real_vram_size &&
-	    bo->tbo.mem.mem_type == TTM_PL_VRAM &&
-	    bo->tbo.mem.start < adev->gmc.visible_vram_size >> PAGE_SHIFT)
+	    amdgpu_bo_in_cpu_visible_vram(bo))
 		p->bytes_moved_vis += ctx.bytes_moved;
 
 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
@@ -411,7 +410,6 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
 		struct amdgpu_bo_list_entry *candidate = p->evictable;
 		struct amdgpu_bo *bo = candidate->robj;
 		struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
-		u64 initial_bytes_moved, bytes_moved;
 		bool update_bytes_moved_vis;
 		uint32_t other;
 
@@ -435,18 +433,14 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
 			continue;
 
 		/* Good we can try to move this BO somewhere else */
-		amdgpu_ttm_placement_from_domain(bo, other);
 		update_bytes_moved_vis =
 			adev->gmc.visible_vram_size < adev->gmc.real_vram_size &&
-			bo->tbo.mem.mem_type == TTM_PL_VRAM &&
-			bo->tbo.mem.start < adev->gmc.visible_vram_size >> PAGE_SHIFT;
-		initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
+			amdgpu_bo_in_cpu_visible_vram(bo);
+		amdgpu_ttm_placement_from_domain(bo, other);
 		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
-		bytes_moved = atomic64_read(&adev->num_bytes_moved) -
-			initial_bytes_moved;
-		p->bytes_moved += bytes_moved;
+		p->bytes_moved += ctx.bytes_moved;
 		if (update_bytes_moved_vis)
-			p->bytes_moved_vis += bytes_moved;
+			p->bytes_moved_vis += ctx.bytes_moved;
 
 		if (unlikely(r))
 			break;
@@ -536,7 +530,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 	if (p->bo_list) {
 		amdgpu_bo_list_get_list(p->bo_list, &p->validated);
 		if (p->bo_list->first_userptr != p->bo_list->num_entries)
-			p->mn = amdgpu_mn_get(p->adev);
+			p->mn = amdgpu_mn_get(p->adev, AMDGPU_MN_TYPE_GFX);
 	}
 
 	INIT_LIST_HEAD(&duplicates);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 3fabf9f97022..c5bb36275e93 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -91,7 +91,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
 			continue;
 
 		r = drm_sched_entity_init(&ring->sched, &ctx->rings[i].entity,
-					  rq, amdgpu_sched_jobs, &ctx->guilty);
+					  rq, &ctx->guilty);
 		if (r)
 			goto failed;
 	}
@@ -111,8 +111,9 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
 	return r;
 }
 
-static void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
+static void amdgpu_ctx_fini(struct kref *ref)
 {
+	struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount);
 	struct amdgpu_device *adev = ctx->adev;
 	unsigned i, j;
 
@@ -125,13 +126,11 @@ static void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
 	kfree(ctx->fences);
 	ctx->fences = NULL;
 
-	for (i = 0; i < adev->num_rings; i++)
-		drm_sched_entity_fini(&adev->rings[i]->sched,
-				      &ctx->rings[i].entity);
-
 	amdgpu_queue_mgr_fini(adev, &ctx->queue_mgr);
 
 	mutex_destroy(&ctx->lock);
+
+	kfree(ctx);
 }
 
 static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
@@ -170,12 +169,20 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
 static void amdgpu_ctx_do_release(struct kref *ref)
 {
 	struct amdgpu_ctx *ctx;
+	u32 i;
 
 	ctx = container_of(ref, struct amdgpu_ctx, refcount);
 
-	amdgpu_ctx_fini(ctx);
+	for (i = 0; i < ctx->adev->num_rings; i++) {
 
-	kfree(ctx);
+		if (ctx->adev->rings[i] == &ctx->adev->gfx.kiq.ring)
+			continue;
+
+		drm_sched_entity_fini(&ctx->adev->rings[i]->sched,
+			&ctx->rings[i].entity);
+	}
+
+	amdgpu_ctx_fini(ref);
 }
 
 static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id)
@@ -437,16 +444,72 @@ void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
 	idr_init(&mgr->ctx_handles);
 }
 
+void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr)
+{
+	struct amdgpu_ctx *ctx;
+	struct idr *idp;
+	uint32_t id, i;
+
+	idp = &mgr->ctx_handles;
+
+	idr_for_each_entry(idp, ctx, id) {
+
+		if (!ctx->adev)
+			return;
+
+		for (i = 0; i < ctx->adev->num_rings; i++) {
+
+			if (ctx->adev->rings[i] == &ctx->adev->gfx.kiq.ring)
+				continue;
+
+			if (kref_read(&ctx->refcount) == 1)
+				drm_sched_entity_do_release(&ctx->adev->rings[i]->sched,
+						  &ctx->rings[i].entity);
+			else
+				DRM_ERROR("ctx %p is still alive\n", ctx);
+		}
+	}
+}
+
+void amdgpu_ctx_mgr_entity_cleanup(struct amdgpu_ctx_mgr *mgr)
+{
+	struct amdgpu_ctx *ctx;
+	struct idr *idp;
+	uint32_t id, i;
+
+	idp = &mgr->ctx_handles;
+
+	idr_for_each_entry(idp, ctx, id) {
+
+		if (!ctx->adev)
+			return;
+
+		for (i = 0; i < ctx->adev->num_rings; i++) {
+
+			if (ctx->adev->rings[i] == &ctx->adev->gfx.kiq.ring)
+				continue;
+
+			if (kref_read(&ctx->refcount) == 1)
+				drm_sched_entity_cleanup(&ctx->adev->rings[i]->sched,
+					&ctx->rings[i].entity);
+			else
+				DRM_ERROR("ctx %p is still alive\n", ctx);
+		}
+	}
+}
+
 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
 {
 	struct amdgpu_ctx *ctx;
 	struct idr *idp;
 	uint32_t id;
 
+	amdgpu_ctx_mgr_entity_cleanup(mgr);
+
 	idp = &mgr->ctx_handles;
 
 	idr_for_each_entry(idp, ctx, id) {
-		if (kref_put(&ctx->refcount, amdgpu_ctx_do_release) != 1)
+		if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1)
 			DRM_ERROR("ctx %p is still alive\n", ctx);
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index 448d69fe3756..f5fb93795a69 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -28,8 +28,13 @@
 #include <linux/debugfs.h>
 #include "amdgpu.h"
 
-/*
- * Debugfs
+/**
+ * amdgpu_debugfs_add_files - Add simple debugfs entries
+ *
+ * @adev:  Device to attach debugfs entries to
+ * @files:  Array of function callbacks that respond to reads
+ * @nfiles: Number of callbacks to register
+ *
  */
 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
 			     const struct drm_info_list *files,
@@ -64,7 +69,33 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
 
 #if defined(CONFIG_DEBUG_FS)
 
-
+/**
+ * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
+ *
+ * @read: True if reading
+ * @f: open file handle
+ * @buf: User buffer to write/read to
+ * @size: Number of bytes to write/read
+ * @pos:  Offset to seek to
+ *
+ * This debugfs entry has special meaning on the offset being sought.
+ * Various bits have different meanings:
+ *
+ * Bit 62:  Indicates a GRBM bank switch is needed
+ * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
+ * 			zero)
+ * Bits 24..33: The SE or ME selector if needed
+ * Bits 34..43: The SH (or SA) or PIPE selector if needed
+ * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
+ *
+ * Bit 23:  Indicates that the PM power gating lock should be held
+ * 			This is necessary to read registers that might be
+ * 			unreliable during a power gating transistion.
+ *
+ * The lower bits are the BYTE offset of the register to read.  This
+ * allows reading multiple registers in a single call and having
+ * the returned size reflect that.
+ */
 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
 		char __user *buf, size_t size, loff_t *pos)
 {
@@ -164,19 +195,37 @@ static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
 	return result;
 }
 
-
+/**
+ * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
+ */
 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 					size_t size, loff_t *pos)
 {
 	return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
 }
 
+/**
+ * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
+ */
 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 					 size_t size, loff_t *pos)
 {
 	return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 }
 
+
+/**
+ * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
+ *
+ * @f: open file handle
+ * @buf: User buffer to store read data in
+ * @size: Number of bytes to read
+ * @pos:  Offset to seek to
+ *
+ * The lower bits are the BYTE offset of the register to read.  This
+ * allows reading multiple registers in a single call and having
+ * the returned size reflect that.
+ */
 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 					size_t size, loff_t *pos)
 {
@@ -204,6 +253,18 @@ static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 	return result;
 }
 
+/**
+ * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
+ *
+ * @f: open file handle
+ * @buf: User buffer to write data from
+ * @size: Number of bytes to write
+ * @pos:  Offset to seek to
+ *
+ * The lower bits are the BYTE offset of the register to write.  This
+ * allows writing multiple registers in a single call and having
+ * the returned size reflect that.
+ */
 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
 					 size_t size, loff_t *pos)
 {
@@ -232,6 +293,18 @@ static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user
 	return result;
 }
 
+/**
+ * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
+ *
+ * @f: open file handle
+ * @buf: User buffer to store read data in
+ * @size: Number of bytes to read
+ * @pos:  Offset to seek to
+ *
+ * The lower bits are the BYTE offset of the register to read.  This
+ * allows reading multiple registers in a single call and having
+ * the returned size reflect that.
+ */
 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 					size_t size, loff_t *pos)
 {
@@ -259,6 +332,18 @@ static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 	return result;
 }
 
+/**
+ * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
+ *
+ * @f: open file handle
+ * @buf: User buffer to write data from
+ * @size: Number of bytes to write
+ * @pos:  Offset to seek to
+ *
+ * The lower bits are the BYTE offset of the register to write.  This
+ * allows writing multiple registers in a single call and having
+ * the returned size reflect that.
+ */
 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
 					 size_t size, loff_t *pos)
 {
@@ -287,6 +372,18 @@ static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user
 	return result;
 }
 
+/**
+ * amdgpu_debugfs_regs_smc_read - Read from a SMC register
+ *
+ * @f: open file handle
+ * @buf: User buffer to store read data in
+ * @size: Number of bytes to read
+ * @pos:  Offset to seek to
+ *
+ * The lower bits are the BYTE offset of the register to read.  This
+ * allows reading multiple registers in a single call and having
+ * the returned size reflect that.
+ */
 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 					size_t size, loff_t *pos)
 {
@@ -314,6 +411,18 @@ static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 	return result;
 }
 
+/**
+ * amdgpu_debugfs_regs_smc_write - Write to a SMC register
+ *
+ * @f: open file handle
+ * @buf: User buffer to write data from
+ * @size: Number of bytes to write
+ * @pos:  Offset to seek to
+ *
+ * The lower bits are the BYTE offset of the register to write.  This
+ * allows writing multiple registers in a single call and having
+ * the returned size reflect that.
+ */
 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
 					 size_t size, loff_t *pos)
 {
@@ -342,6 +451,20 @@ static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *
 	return result;
 }
 
+/**
+ * amdgpu_debugfs_gca_config_read - Read from gfx config data
+ *
+ * @f: open file handle
+ * @buf: User buffer to store read data in
+ * @size: Number of bytes to read
+ * @pos:  Offset to seek to
+ *
+ * This file is used to access configuration data in a somewhat
+ * stable fashion.  The format is a series of DWORDs with the first
+ * indicating which revision it is.  New content is appended to the
+ * end so that older software can still read the data.
+ */
+
 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 					size_t size, loff_t *pos)
 {
@@ -418,6 +541,19 @@ static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 	return result;
 }
 
+/**
+ * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
+ *
+ * @f: open file handle
+ * @buf: User buffer to store read data in
+ * @size: Number of bytes to read
+ * @pos:  Offset to seek to
+ *
+ * The offset is treated as the BYTE address of one of the sensors
+ * enumerated in amd/include/kgd_pp_interface.h under the
+ * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
+ * you would use the offset 3 * 4 = 12.
+ */
 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 					size_t size, loff_t *pos)
 {
@@ -428,7 +564,7 @@ static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 	if (size & 3 || *pos & 0x3)
 		return -EINVAL;
 
-	if (amdgpu_dpm == 0)
+	if (!adev->pm.dpm_enabled)
 		return -EINVAL;
 
 	/* convert offset to sensor number */
@@ -457,6 +593,27 @@ static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 	return !r ? outsize : r;
 }
 
+/** amdgpu_debugfs_wave_read - Read WAVE STATUS data
+ *
+ * @f: open file handle
+ * @buf: User buffer to store read data in
+ * @size: Number of bytes to read
+ * @pos:  Offset to seek to
+ *
+ * The offset being sought changes which wave that the status data
+ * will be returned for.  The bits are used as follows:
+ *
+ * Bits 0..6: 	Byte offset into data
+ * Bits 7..14:	SE selector
+ * Bits 15..22:	SH/SA selector
+ * Bits 23..30: CU/{WGP+SIMD} selector
+ * Bits 31..36: WAVE ID selector
+ * Bits 37..44: SIMD ID selector
+ *
+ * The returned data begins with one DWORD of version information
+ * Followed by WAVE STATUS registers relevant to the GFX IP version
+ * being used.  See gfx_v8_0_read_wave_data() for an example output.
+ */
 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
 					size_t size, loff_t *pos)
 {
@@ -507,6 +664,28 @@ static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
 	return result;
 }
 
+/** amdgpu_debugfs_gpr_read - Read wave gprs
+ *
+ * @f: open file handle
+ * @buf: User buffer to store read data in
+ * @size: Number of bytes to read
+ * @pos:  Offset to seek to
+ *
+ * The offset being sought changes which wave that the status data
+ * will be returned for.  The bits are used as follows:
+ *
+ * Bits 0..11:	Byte offset into data
+ * Bits 12..19:	SE selector
+ * Bits 20..27:	SH/SA selector
+ * Bits 28..35: CU/{WGP+SIMD} selector
+ * Bits 36..43: WAVE ID selector
+ * Bits 37..44: SIMD ID selector
+ * Bits 52..59: Thread selector
+ * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
+ *
+ * The return data comes from the SGPR or VGPR register bank for
+ * the selected operational unit.
+ */
 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
 					size_t size, loff_t *pos)
 {
@@ -637,6 +816,12 @@ static const char *debugfs_regs_names[] = {
 	"amdgpu_gpr",
 };
 
+/**
+ * amdgpu_debugfs_regs_init -	Initialize debugfs entries that provide
+ * 								register access.
+ *
+ * @adev: The device to attach the debugfs entries to
+ */
 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
 {
 	struct drm_minor *minor = adev->ddev->primary;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 34af664b9f93..290e279abf0d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -83,8 +83,10 @@ static const char *amdgpu_asic_name[] = {
 	"POLARIS10",
 	"POLARIS11",
 	"POLARIS12",
+	"VEGAM",
 	"VEGA10",
 	"VEGA12",
+	"VEGA20",
 	"RAVEN",
 	"LAST",
 };
@@ -690,6 +692,8 @@ void amdgpu_device_gart_location(struct amdgpu_device *adev,
 {
 	u64 size_af, size_bf;
 
+	mc->gart_size += adev->pm.smu_prv_buffer_size;
+
 	size_af = adev->gmc.mc_mask - mc->vram_end;
 	size_bf = mc->vram_start;
 	if (size_bf > size_af) {
@@ -907,6 +911,46 @@ static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
 	}
 }
 
+static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
+{
+	struct sysinfo si;
+	bool is_os_64 = (sizeof(void *) == 8) ? true : false;
+	uint64_t total_memory;
+	uint64_t dram_size_seven_GB = 0x1B8000000;
+	uint64_t dram_size_three_GB = 0xB8000000;
+
+	if (amdgpu_smu_memory_pool_size == 0)
+		return;
+
+	if (!is_os_64) {
+		DRM_WARN("Not 64-bit OS, feature not supported\n");
+		goto def_value;
+	}
+	si_meminfo(&si);
+	total_memory = (uint64_t)si.totalram * si.mem_unit;
+
+	if ((amdgpu_smu_memory_pool_size == 1) ||
+		(amdgpu_smu_memory_pool_size == 2)) {
+		if (total_memory < dram_size_three_GB)
+			goto def_value1;
+	} else if ((amdgpu_smu_memory_pool_size == 4) ||
+		(amdgpu_smu_memory_pool_size == 8)) {
+		if (total_memory < dram_size_seven_GB)
+			goto def_value1;
+	} else {
+		DRM_WARN("Smu memory pool size not supported\n");
+		goto def_value;
+	}
+	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
+
+	return;
+
+def_value1:
+	DRM_WARN("No enough system memory\n");
+def_value:
+	adev->pm.smu_prv_buffer_size = 0;
+}
+
 /**
  * amdgpu_device_check_arguments - validate module params
  *
@@ -948,6 +992,8 @@ static void amdgpu_device_check_arguments(struct amdgpu_device *adev)
 		amdgpu_vm_fragment_size = -1;
 	}
 
+	amdgpu_device_check_smu_prv_buffer_size(adev);
+
 	amdgpu_device_check_vm_size(adev);
 
 	amdgpu_device_check_block_size(adev);
@@ -1039,10 +1085,11 @@ static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
  * the hardware IP specified.
  * Returns the error code from the last instance.
  */
-int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev,
+int amdgpu_device_ip_set_clockgating_state(void *dev,
 					   enum amd_ip_block_type block_type,
 					   enum amd_clockgating_state state)
 {
+	struct amdgpu_device *adev = dev;
 	int i, r = 0;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
@@ -1072,10 +1119,11 @@ int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev,
  * the hardware IP specified.
  * Returns the error code from the last instance.
  */
-int amdgpu_device_ip_set_powergating_state(struct amdgpu_device *adev,
+int amdgpu_device_ip_set_powergating_state(void *dev,
 					   enum amd_ip_block_type block_type,
 					   enum amd_powergating_state state)
 {
+	struct amdgpu_device *adev = dev;
 	int i, r = 0;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
@@ -1320,9 +1368,10 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
 	case CHIP_TOPAZ:
 	case CHIP_TONGA:
 	case CHIP_FIJI:
-	case CHIP_POLARIS11:
 	case CHIP_POLARIS10:
+	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 	case CHIP_CARRIZO:
 	case CHIP_STONEY:
 #ifdef CONFIG_DRM_AMDGPU_SI
@@ -1339,6 +1388,7 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
 	case CHIP_KABINI:
 	case CHIP_MULLINS:
 #endif
+	case CHIP_VEGA20:
 	default:
 		return 0;
 	case CHIP_VEGA10:
@@ -1428,9 +1478,10 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
 	case CHIP_TOPAZ:
 	case CHIP_TONGA:
 	case CHIP_FIJI:
-	case CHIP_POLARIS11:
 	case CHIP_POLARIS10:
+	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 	case CHIP_CARRIZO:
 	case CHIP_STONEY:
 		if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
@@ -1472,6 +1523,7 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
 #endif
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 	case CHIP_RAVEN:
 		if (adev->asic_type == CHIP_RAVEN)
 			adev->family = AMDGPU_FAMILY_RV;
@@ -1499,6 +1551,8 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
 			return -EAGAIN;
 	}
 
+	adev->powerplay.pp_feature = amdgpu_pp_feature_mask;
+
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
 			DRM_ERROR("disabled ip block: %d <%s>\n",
@@ -1654,12 +1708,17 @@ static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev)
 	if (amdgpu_emu_mode == 1)
 		return 0;
 
+	r = amdgpu_ib_ring_tests(adev);
+	if (r)
+		DRM_ERROR("ib ring test failed (%d).\n", r);
+
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if (!adev->ip_blocks[i].status.valid)
 			continue;
 		/* skip CG for VCE/UVD, it's handled specially */
 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
+		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
 		    adev->ip_blocks[i].version->funcs->set_clockgating_state) {
 			/* enable clockgating to save power */
 			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
@@ -1704,8 +1763,8 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
 		}
 	}
 
-	mod_delayed_work(system_wq, &adev->late_init_work,
-			msecs_to_jiffies(AMDGPU_RESUME_MS));
+	queue_delayed_work(system_wq, &adev->late_init_work,
+			   msecs_to_jiffies(AMDGPU_RESUME_MS));
 
 	amdgpu_device_fill_reset_magic(adev);
 
@@ -1759,6 +1818,7 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
 
 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
 			adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
+			adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
 			adev->ip_blocks[i].version->funcs->set_clockgating_state) {
 			/* ungate blocks before hw fini so that we can shutdown the blocks safely */
 			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
@@ -1850,6 +1910,12 @@ int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
 	if (amdgpu_sriov_vf(adev))
 		amdgpu_virt_request_full_gpu(adev, false);
 
+	/* ungate SMC block powergating */
+	if (adev->powerplay.pp_feature & PP_GFXOFF_MASK)
+		amdgpu_device_ip_set_powergating_state(adev,
+						       AMD_IP_BLOCK_TYPE_SMC,
+						       AMD_CG_STATE_UNGATE);
+
 	/* ungate SMC block first */
 	r = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
 						   AMD_CG_STATE_UNGATE);
@@ -2086,16 +2152,15 @@ bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
 	case CHIP_MULLINS:
 	case CHIP_CARRIZO:
 	case CHIP_STONEY:
-	case CHIP_POLARIS11:
 	case CHIP_POLARIS10:
+	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 	case CHIP_TONGA:
 	case CHIP_FIJI:
-#if defined(CONFIG_DRM_AMD_DC_PRE_VEGA)
-		return amdgpu_dc != 0;
-#endif
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 	case CHIP_RAVEN:
 #endif
@@ -2375,10 +2440,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		goto failed;
 	}
 
-	r = amdgpu_ib_ring_tests(adev);
-	if (r)
-		DRM_ERROR("ib ring test failed (%d).\n", r);
-
 	if (amdgpu_sriov_vf(adev))
 		amdgpu_virt_init_data_exchange(adev);
 
@@ -2539,7 +2600,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 	/* unpin the front buffers and cursors */
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-		struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
+		struct drm_framebuffer *fb = crtc->primary->fb;
 		struct amdgpu_bo *robj;
 
 		if (amdgpu_crtc->cursor_bo) {
@@ -2551,10 +2612,10 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 			}
 		}
 
-		if (rfb == NULL || rfb->obj == NULL) {
+		if (fb == NULL || fb->obj[0] == NULL) {
 			continue;
 		}
-		robj = gem_to_amdgpu_bo(rfb->obj);
+		robj = gem_to_amdgpu_bo(fb->obj[0]);
 		/* don't unpin kernel fb objects */
 		if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
 			r = amdgpu_bo_reserve(robj, true);
@@ -2640,11 +2701,6 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 	}
 	amdgpu_fence_driver_resume(adev);
 
-	if (resume) {
-		r = amdgpu_ib_ring_tests(adev);
-		if (r)
-			DRM_ERROR("ib ring test failed (%d).\n", r);
-	}
 
 	r = amdgpu_device_ip_late_init(adev);
 	if (r)
@@ -2736,6 +2792,9 @@ static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
 	if (amdgpu_sriov_vf(adev))
 		return true;
 
+	if (amdgpu_asic_need_full_reset(adev))
+		return true;
+
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if (!adev->ip_blocks[i].status.valid)
 			continue;
@@ -2792,6 +2851,9 @@ static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
 {
 	int i;
 
+	if (amdgpu_asic_need_full_reset(adev))
+		return true;
+
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if (!adev->ip_blocks[i].status.valid)
 			continue;
@@ -3087,20 +3149,19 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
 
 	/* now we are okay to resume SMC/CP/SDMA */
 	r = amdgpu_device_ip_reinit_late_sriov(adev);
-	amdgpu_virt_release_full_gpu(adev, true);
 	if (r)
 		goto error;
 
 	amdgpu_irq_gpu_reset_resume_helper(adev);
 	r = amdgpu_ib_ring_tests(adev);
 
+error:
+	amdgpu_virt_release_full_gpu(adev, true);
 	if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
 		atomic_inc(&adev->vram_lost_counter);
 		r = amdgpu_device_handle_vram_lost(adev);
 	}
 
-error:
-
 	return r;
 }
 
@@ -3117,7 +3178,6 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 			      struct amdgpu_job *job, bool force)
 {
-	struct drm_atomic_state *state = NULL;
 	int i, r, resched;
 
 	if (!force && !amdgpu_device_ip_check_soft_reset(adev)) {
@@ -3140,10 +3200,6 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 	/* block TTM */
 	resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
 
-	/* store modesetting */
-	if (amdgpu_device_has_dc_support(adev))
-		state = drm_atomic_helper_suspend(adev->ddev);
-
 	/* block all schedulers and reset given job's ring */
 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 		struct amdgpu_ring *ring = adev->rings[i];
@@ -3183,10 +3239,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 		kthread_unpark(ring->sched.thread);
 	}
 
-	if (amdgpu_device_has_dc_support(adev)) {
-		if (drm_atomic_helper_resume(adev->ddev, state))
-			dev_info(adev->dev, "drm resume failed:%d\n", r);
-	} else {
+	if (!amdgpu_device_has_dc_support(adev)) {
 		drm_helper_resume_force_mode(adev->ddev);
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index 93f700ab1bfb..76ee8e04ff11 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -35,6 +35,7 @@
 #include <linux/pm_runtime.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_fb_helper.h>
 
 static void amdgpu_display_flip_callback(struct dma_fence *f,
@@ -151,8 +152,6 @@ int amdgpu_display_crtc_page_flip_target(struct drm_crtc *crtc,
 	struct drm_device *dev = crtc->dev;
 	struct amdgpu_device *adev = dev->dev_private;
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-	struct amdgpu_framebuffer *old_amdgpu_fb;
-	struct amdgpu_framebuffer *new_amdgpu_fb;
 	struct drm_gem_object *obj;
 	struct amdgpu_flip_work *work;
 	struct amdgpu_bo *new_abo;
@@ -174,15 +173,13 @@ int amdgpu_display_crtc_page_flip_target(struct drm_crtc *crtc,
 	work->async = (page_flip_flags & DRM_MODE_PAGE_FLIP_ASYNC) != 0;
 
 	/* schedule unpin of the old buffer */
-	old_amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
-	obj = old_amdgpu_fb->obj;
+	obj = crtc->primary->fb->obj[0];
 
 	/* take a reference to the old object */
 	work->old_abo = gem_to_amdgpu_bo(obj);
 	amdgpu_bo_ref(work->old_abo);
 
-	new_amdgpu_fb = to_amdgpu_framebuffer(fb);
-	obj = new_amdgpu_fb->obj;
+	obj = fb->obj[0];
 	new_abo = gem_to_amdgpu_bo(obj);
 
 	/* pin the new buffer */
@@ -192,7 +189,7 @@ int amdgpu_display_crtc_page_flip_target(struct drm_crtc *crtc,
 		goto cleanup;
 	}
 
-	r = amdgpu_bo_pin(new_abo, amdgpu_display_framebuffer_domains(adev), &base);
+	r = amdgpu_bo_pin(new_abo, amdgpu_display_supported_domains(adev), &base);
 	if (unlikely(r != 0)) {
 		DRM_ERROR("failed to pin new abo buffer before flip\n");
 		goto unreserve;
@@ -482,31 +479,12 @@ bool amdgpu_display_ddc_probe(struct amdgpu_connector *amdgpu_connector,
 	return true;
 }
 
-static void amdgpu_display_user_framebuffer_destroy(struct drm_framebuffer *fb)
-{
-	struct amdgpu_framebuffer *amdgpu_fb = to_amdgpu_framebuffer(fb);
-
-	drm_gem_object_put_unlocked(amdgpu_fb->obj);
-	drm_framebuffer_cleanup(fb);
-	kfree(amdgpu_fb);
-}
-
-static int amdgpu_display_user_framebuffer_create_handle(
-			struct drm_framebuffer *fb,
-			struct drm_file *file_priv,
-			unsigned int *handle)
-{
-	struct amdgpu_framebuffer *amdgpu_fb = to_amdgpu_framebuffer(fb);
-
-	return drm_gem_handle_create(file_priv, amdgpu_fb->obj, handle);
-}
-
 static const struct drm_framebuffer_funcs amdgpu_fb_funcs = {
-	.destroy = amdgpu_display_user_framebuffer_destroy,
-	.create_handle = amdgpu_display_user_framebuffer_create_handle,
+	.destroy = drm_gem_fb_destroy,
+	.create_handle = drm_gem_fb_create_handle,
 };
 
-uint32_t amdgpu_display_framebuffer_domains(struct amdgpu_device *adev)
+uint32_t amdgpu_display_supported_domains(struct amdgpu_device *adev)
 {
 	uint32_t domain = AMDGPU_GEM_DOMAIN_VRAM;
 
@@ -526,11 +504,11 @@ int amdgpu_display_framebuffer_init(struct drm_device *dev,
 				    struct drm_gem_object *obj)
 {
 	int ret;
-	rfb->obj = obj;
+	rfb->base.obj[0] = obj;
 	drm_helper_mode_fill_fb_struct(dev, &rfb->base, mode_cmd);
 	ret = drm_framebuffer_init(dev, &rfb->base, &amdgpu_fb_funcs);
 	if (ret) {
-		rfb->obj = NULL;
+		rfb->base.obj[0] = NULL;
 		return ret;
 	}
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h
index 2b11d808f297..f66e3e3fef0a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h
@@ -23,7 +23,7 @@
 #ifndef __AMDGPU_DISPLAY_H__
 #define __AMDGPU_DISPLAY_H__
 
-uint32_t amdgpu_display_framebuffer_domains(struct amdgpu_device *adev);
+uint32_t amdgpu_display_supported_domains(struct amdgpu_device *adev);
 struct drm_framebuffer *
 amdgpu_display_user_framebuffer_create(struct drm_device *dev,
 				       struct drm_file *file_priv,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
index e997ebbe43ea..def1010ac05e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
@@ -115,6 +115,26 @@ void amdgpu_dpm_print_ps_status(struct amdgpu_device *adev,
 	pr_cont("\n");
 }
 
+void amdgpu_dpm_get_active_displays(struct amdgpu_device *adev)
+{
+	struct drm_device *ddev = adev->ddev;
+	struct drm_crtc *crtc;
+	struct amdgpu_crtc *amdgpu_crtc;
+
+	adev->pm.dpm.new_active_crtcs = 0;
+	adev->pm.dpm.new_active_crtc_count = 0;
+	if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) {
+		list_for_each_entry(crtc,
+				    &ddev->mode_config.crtc_list, head) {
+			amdgpu_crtc = to_amdgpu_crtc(crtc);
+			if (amdgpu_crtc->enabled) {
+				adev->pm.dpm.new_active_crtcs |= (1 << amdgpu_crtc->crtc_id);
+				adev->pm.dpm.new_active_crtc_count++;
+			}
+		}
+	}
+}
+
 
 u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index 643d008410c6..dd6203a0a6b7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -52,8 +52,6 @@ enum amdgpu_dpm_event_src {
 	AMDGPU_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL = 4
 };
 
-#define SCLK_DEEP_SLEEP_MASK 0x8
-
 struct amdgpu_ps {
 	u32 caps; /* vbios flags */
 	u32 class; /* vbios flags */
@@ -349,12 +347,6 @@ enum amdgpu_pcie_gen {
 		((adev)->powerplay.pp_funcs->set_clockgating_by_smu(\
 			(adev)->powerplay.pp_handle, msg_id))
 
-#define amdgpu_dpm_notify_smu_memory_info(adev, virtual_addr_low, \
-			virtual_addr_hi, mc_addr_low, mc_addr_hi, size) \
-		((adev)->powerplay.pp_funcs->notify_smu_memory_info)( \
-			(adev)->powerplay.pp_handle, virtual_addr_low, \
-			virtual_addr_hi, mc_addr_low, mc_addr_hi, size)
-
 #define amdgpu_dpm_get_power_profile_mode(adev, buf) \
 		((adev)->powerplay.pp_funcs->get_power_profile_mode(\
 			(adev)->powerplay.pp_handle, buf))
@@ -445,6 +437,8 @@ struct amdgpu_pm {
 	uint32_t                pcie_gen_mask;
 	uint32_t                pcie_mlw_mask;
 	struct amd_pp_display_configuration pm_display_cfg;/* set by dc */
+	uint32_t                smu_prv_buffer_size;
+	struct amdgpu_bo        *smu_prv_buffer;
 };
 
 #define R600_SSTU_DFLT                               0
@@ -482,6 +476,7 @@ void amdgpu_dpm_print_ps_status(struct amdgpu_device *adev,
 				struct amdgpu_ps *rps);
 u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev);
 u32 amdgpu_dpm_get_vrefresh(struct amdgpu_device *adev);
+void amdgpu_dpm_get_active_displays(struct amdgpu_device *adev);
 bool amdgpu_is_uvd_state(u32 class, u32 class2);
 void amdgpu_calculate_u_and_p(u32 i, u32 r_c, u32 p_b,
 			      u32 *p, u32 *u);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 0b19482b36b8..b0bf2f24da48 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -75,9 +75,10 @@
  * - 3.23.0 - Add query for VRAM lost counter
  * - 3.24.0 - Add high priority compute support for gfx9
  * - 3.25.0 - Add support for sensor query info (stable pstate sclk/mclk).
+ * - 3.26.0 - GFX9: Process AMDGPU_IB_FLAG_TC_WB_NOT_INVALIDATE.
  */
 #define KMS_DRIVER_MAJOR	3
-#define KMS_DRIVER_MINOR	25
+#define KMS_DRIVER_MINOR	26
 #define KMS_DRIVER_PATCHLEVEL	0
 
 int amdgpu_vram_limit = 0;
@@ -121,7 +122,7 @@ uint amdgpu_pg_mask = 0xffffffff;
 uint amdgpu_sdma_phase_quantum = 32;
 char *amdgpu_disable_cu = NULL;
 char *amdgpu_virtual_display = NULL;
-uint amdgpu_pp_feature_mask = 0xffffbfff;
+uint amdgpu_pp_feature_mask = 0xffff3fff; /* gfxoff (bit 15) disabled by default */
 int amdgpu_ngg = 0;
 int amdgpu_prim_buf_per_se = 0;
 int amdgpu_pos_buf_per_se = 0;
@@ -132,6 +133,7 @@ int amdgpu_lbpw = -1;
 int amdgpu_compute_multipipe = -1;
 int amdgpu_gpu_recovery = -1; /* auto */
 int amdgpu_emu_mode = 0;
+uint amdgpu_smu_memory_pool_size = 0;
 
 MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
 module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -316,6 +318,11 @@ MODULE_PARM_DESC(cik_support, "CIK support (1 = enabled (default), 0 = disabled)
 module_param_named(cik_support, amdgpu_cik_support, int, 0444);
 #endif
 
+MODULE_PARM_DESC(smu_memory_pool_size,
+	"reserve gtt for smu debug usage, 0 = disable,"
+		"0x1 = 256Mbyte, 0x2 = 512Mbyte, 0x4 = 1 Gbyte, 0x8 = 2GByte");
+module_param_named(smu_memory_pool_size, amdgpu_smu_memory_pool_size, uint, 0444);
+
 static const struct pci_device_id pciidlist[] = {
 #ifdef  CONFIG_DRM_AMDGPU_SI
 	{0x1002, 0x6780, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
@@ -534,6 +541,9 @@ static const struct pci_device_id pciidlist[] = {
 	{0x1002, 0x6995, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
 	{0x1002, 0x6997, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
 	{0x1002, 0x699F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
+	/* VEGAM */
+	{0x1002, 0x694C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGAM},
+	{0x1002, 0x694E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGAM},
 	/* Vega 10 */
 	{0x1002, 0x6860, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10},
 	{0x1002, 0x6861, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA10},
@@ -550,6 +560,13 @@ static const struct pci_device_id pciidlist[] = {
 	{0x1002, 0x69A2, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA12},
 	{0x1002, 0x69A3, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA12},
 	{0x1002, 0x69AF, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA12},
+	/* Vega 20 */
+	{0x1002, 0x66A0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA20|AMD_EXP_HW_SUPPORT},
+	{0x1002, 0x66A1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA20|AMD_EXP_HW_SUPPORT},
+	{0x1002, 0x66A2, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA20|AMD_EXP_HW_SUPPORT},
+	{0x1002, 0x66A3, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA20|AMD_EXP_HW_SUPPORT},
+	{0x1002, 0x66A7, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA20|AMD_EXP_HW_SUPPORT},
+	{0x1002, 0x66AF, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VEGA20|AMD_EXP_HW_SUPPORT},
 	/* Raven */
 	{0x1002, 0x15dd, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RAVEN|AMD_IS_APU},
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index 12063019751b..bc5fd8ebab5d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -137,7 +137,7 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
 	/* need to align pitch with crtc limits */
 	mode_cmd->pitches[0] = amdgpu_align_pitch(adev, mode_cmd->width, cpp,
 						  fb_tiled);
-	domain = amdgpu_display_framebuffer_domains(adev);
+	domain = amdgpu_display_supported_domains(adev);
 
 	height = ALIGN(mode_cmd->height, 8);
 	size = mode_cmd->pitches[0] * height;
@@ -292,9 +292,9 @@ static int amdgpu_fbdev_destroy(struct drm_device *dev, struct amdgpu_fbdev *rfb
 
 	drm_fb_helper_unregister_fbi(&rfbdev->helper);
 
-	if (rfb->obj) {
-		amdgpufb_destroy_pinned_object(rfb->obj);
-		rfb->obj = NULL;
+	if (rfb->base.obj[0]) {
+		amdgpufb_destroy_pinned_object(rfb->base.obj[0]);
+		rfb->base.obj[0] = NULL;
 		drm_framebuffer_unregister_private(&rfb->base);
 		drm_framebuffer_cleanup(&rfb->base);
 	}
@@ -377,7 +377,7 @@ int amdgpu_fbdev_total_size(struct amdgpu_device *adev)
 	if (!adev->mode_info.rfbdev)
 		return 0;
 
-	robj = gem_to_amdgpu_bo(adev->mode_info.rfbdev->rfb.obj);
+	robj = gem_to_amdgpu_bo(adev->mode_info.rfbdev->rfb.base.obj[0]);
 	size += amdgpu_bo_size(robj);
 	return size;
 }
@@ -386,7 +386,7 @@ bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj)
 {
 	if (!adev->mode_info.rfbdev)
 		return false;
-	if (robj == gem_to_amdgpu_bo(adev->mode_info.rfbdev->rfb.obj))
+	if (robj == gem_to_amdgpu_bo(adev->mode_info.rfbdev->rfb.base.obj[0]))
 		return true;
 	return false;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 97449e06a242..39ec6b8890a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -131,7 +131,8 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
  * Emits a fence command on the requested ring (all asics).
  * Returns 0 on success, -ENOMEM on failure.
  */
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f)
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
+		      unsigned flags)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_fence *fence;
@@ -149,7 +150,7 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f)
 		       adev->fence_context + ring->idx,
 		       seq);
 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
-			       seq, AMDGPU_FENCE_FLAG_INT);
+			       seq, flags | AMDGPU_FENCE_FLAG_INT);
 
 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
 	/* This function can't be called concurrently anyway, otherwise
@@ -375,14 +376,14 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
 	struct amdgpu_device *adev = ring->adev;
 	uint64_t index;
 
-	if (ring != &adev->uvd.ring) {
+	if (ring != &adev->uvd.inst[ring->me].ring) {
 		ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];
 		ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);
 	} else {
 		/* put fence directly behind firmware */
 		index = ALIGN(adev->uvd.fw->size, 8);
-		ring->fence_drv.cpu_addr = adev->uvd.cpu_addr + index;
-		ring->fence_drv.gpu_addr = adev->uvd.gpu_addr + index;
+		ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;
+		ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;
 	}
 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));
 	amdgpu_irq_get(adev, irq_src, irq_type);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index cf0f186c6092..17d6b9fb6d77 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -113,12 +113,17 @@ int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev)
 	int r;
 
 	if (adev->gart.robj == NULL) {
-		r = amdgpu_bo_create(adev, adev->gart.table_size, PAGE_SIZE,
-				     AMDGPU_GEM_DOMAIN_VRAM,
-				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-				     ttm_bo_type_kernel, NULL,
-				     &adev->gart.robj);
+		struct amdgpu_bo_param bp;
+
+		memset(&bp, 0, sizeof(bp));
+		bp.size = adev->gart.table_size;
+		bp.byte_align = PAGE_SIZE;
+		bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
+		bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+			AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+		bp.type = ttm_bo_type_kernel;
+		bp.resv = NULL;
+		r = amdgpu_bo_create(adev, &bp, &adev->gart.robj);
 		if (r) {
 			return r;
 		}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 46b9ea4e6103..2c8e27370284 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -48,17 +48,25 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 			     struct drm_gem_object **obj)
 {
 	struct amdgpu_bo *bo;
+	struct amdgpu_bo_param bp;
 	int r;
 
+	memset(&bp, 0, sizeof(bp));
 	*obj = NULL;
 	/* At least align on page size */
 	if (alignment < PAGE_SIZE) {
 		alignment = PAGE_SIZE;
 	}
 
+	bp.size = size;
+	bp.byte_align = alignment;
+	bp.type = type;
+	bp.resv = resv;
+	bp.preferred_domain = initial_domain;
 retry:
-	r = amdgpu_bo_create(adev, size, alignment, initial_domain,
-			     flags, type, resv, &bo);
+	bp.flags = flags;
+	bp.domain = initial_domain;
+	r = amdgpu_bo_create(adev, &bp, &bo);
 	if (r) {
 		if (r != -ERESTARTSYS) {
 			if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
@@ -221,12 +229,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
 		return -EINVAL;
 
 	/* reject invalid gem domains */
-	if (args->in.domains & ~(AMDGPU_GEM_DOMAIN_CPU |
-				 AMDGPU_GEM_DOMAIN_GTT |
-				 AMDGPU_GEM_DOMAIN_VRAM |
-				 AMDGPU_GEM_DOMAIN_GDS |
-				 AMDGPU_GEM_DOMAIN_GWS |
-				 AMDGPU_GEM_DOMAIN_OA))
+	if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK)
 		return -EINVAL;
 
 	/* create a gem object to contain this object in */
@@ -771,16 +774,23 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 }
 
 #if defined(CONFIG_DEBUG_FS)
+
+#define amdgpu_debugfs_gem_bo_print_flag(m, bo, flag)	\
+	if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) {	\
+		seq_printf((m), " " #flag);		\
+	}
+
 static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)
 {
 	struct drm_gem_object *gobj = ptr;
 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
 	struct seq_file *m = data;
 
+	struct dma_buf_attachment *attachment;
+	struct dma_buf *dma_buf;
 	unsigned domain;
 	const char *placement;
 	unsigned pin_count;
-	uint64_t offset;
 
 	domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
 	switch (domain) {
@@ -798,13 +808,27 @@ static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)
 	seq_printf(m, "\t0x%08x: %12ld byte %s",
 		   id, amdgpu_bo_size(bo), placement);
 
-	offset = READ_ONCE(bo->tbo.mem.start);
-	if (offset != AMDGPU_BO_INVALID_OFFSET)
-		seq_printf(m, " @ 0x%010Lx", offset);
-
 	pin_count = READ_ONCE(bo->pin_count);
 	if (pin_count)
 		seq_printf(m, " pin count %d", pin_count);
+
+	dma_buf = READ_ONCE(bo->gem_base.dma_buf);
+	attachment = READ_ONCE(bo->gem_base.import_attach);
+
+	if (attachment)
+		seq_printf(m, " imported from %p", dma_buf);
+	else if (dma_buf)
+		seq_printf(m, " exported as %p", dma_buf);
+
+	amdgpu_debugfs_gem_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
+	amdgpu_debugfs_gem_bo_print_flag(m, bo, NO_CPU_ACCESS);
+	amdgpu_debugfs_gem_bo_print_flag(m, bo, CPU_GTT_USWC);
+	amdgpu_debugfs_gem_bo_print_flag(m, bo, VRAM_CLEARED);
+	amdgpu_debugfs_gem_bo_print_flag(m, bo, SHADOW);
+	amdgpu_debugfs_gem_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
+	amdgpu_debugfs_gem_bo_print_flag(m, bo, VM_ALWAYS_VALID);
+	amdgpu_debugfs_gem_bo_print_flag(m, bo, EXPLICIT_SYNC);
+
 	seq_printf(m, "\n");
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 311589e02d17..f70eeed9ed76 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -127,6 +127,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	struct amdgpu_vm *vm;
 	uint64_t fence_ctx;
 	uint32_t status = 0, alloc_size;
+	unsigned fence_flags = 0;
 
 	unsigned i;
 	int r = 0;
@@ -227,7 +228,10 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 #endif
 		amdgpu_asic_invalidate_hdp(adev, ring);
 
-	r = amdgpu_fence_emit(ring, f);
+	if (ib->flags & AMDGPU_IB_FLAG_TC_WB_NOT_INVALIDATE)
+		fence_flags |= AMDGPU_FENCE_FLAG_TC_WB_ONLY;
+
+	r = amdgpu_fence_emit(ring, f, fence_flags);
 	if (r) {
 		dev_err(adev->dev, "failed to emit fence (%d)\n", r);
 		if (job && job->vmid)
@@ -242,7 +246,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	/* wrap the last IB with fence */
 	if (job && job->uf_addr) {
 		amdgpu_ring_emit_fence(ring, job->uf_addr, job->uf_sequence,
-				       AMDGPU_FENCE_FLAG_64BIT);
+				       fence_flags | AMDGPU_FENCE_FLAG_64BIT);
 	}
 
 	if (patch_offset != ~0 && ring->funcs->patch_cond_exec)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 4b7824d30e73..91517b166a3b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -31,6 +31,7 @@
 #include "amdgpu_sched.h"
 #include "amdgpu_uvd.h"
 #include "amdgpu_vce.h"
+#include "atom.h"
 
 #include <linux/vga_switcheroo.h>
 #include <linux/slab.h>
@@ -214,6 +215,18 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info,
 		fw_info->ver = adev->gfx.rlc_fw_version;
 		fw_info->feature = adev->gfx.rlc_feature_version;
 		break;
+	case AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_CNTL:
+		fw_info->ver = adev->gfx.rlc_srlc_fw_version;
+		fw_info->feature = adev->gfx.rlc_srlc_feature_version;
+		break;
+	case AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_GPM_MEM:
+		fw_info->ver = adev->gfx.rlc_srlg_fw_version;
+		fw_info->feature = adev->gfx.rlc_srlg_feature_version;
+		break;
+	case AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_SRM_MEM:
+		fw_info->ver = adev->gfx.rlc_srls_fw_version;
+		fw_info->feature = adev->gfx.rlc_srls_feature_version;
+		break;
 	case AMDGPU_INFO_FW_GFX_MEC:
 		if (query_fw->index == 0) {
 			fw_info->ver = adev->gfx.mec_fw_version;
@@ -273,12 +286,15 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 	struct drm_crtc *crtc;
 	uint32_t ui32 = 0;
 	uint64_t ui64 = 0;
-	int i, found;
+	int i, j, found;
 	int ui32_size = sizeof(ui32);
 
 	if (!info->return_size || !info->return_pointer)
 		return -EINVAL;
 
+	/* Ensure IB tests are run on ring */
+	flush_delayed_work(&adev->late_init_work);
+
 	switch (info->query) {
 	case AMDGPU_INFO_ACCEL_WORKING:
 		ui32 = adev->accel_working;
@@ -332,7 +348,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 			break;
 		case AMDGPU_HW_IP_UVD:
 			type = AMD_IP_BLOCK_TYPE_UVD;
-			ring_mask = adev->uvd.ring.ready ? 1 : 0;
+			for (i = 0; i < adev->uvd.num_uvd_inst; i++)
+				ring_mask |= ((adev->uvd.inst[i].ring.ready ? 1 : 0) << i);
 			ib_start_alignment = AMDGPU_GPU_PAGE_SIZE;
 			ib_size_alignment = 16;
 			break;
@@ -345,8 +362,11 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 			break;
 		case AMDGPU_HW_IP_UVD_ENC:
 			type = AMD_IP_BLOCK_TYPE_UVD;
-			for (i = 0; i < adev->uvd.num_enc_rings; i++)
-				ring_mask |= ((adev->uvd.ring_enc[i].ready ? 1 : 0) << i);
+			for (i = 0; i < adev->uvd.num_uvd_inst; i++)
+				for (j = 0; j < adev->uvd.num_enc_rings; j++)
+					ring_mask |=
+					((adev->uvd.inst[i].ring_enc[j].ready ? 1 : 0) <<
+					(j + i * adev->uvd.num_enc_rings));
 			ib_start_alignment = AMDGPU_GPU_PAGE_SIZE;
 			ib_size_alignment = 1;
 			break;
@@ -701,10 +721,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		}
 	}
 	case AMDGPU_INFO_SENSOR: {
-		struct pp_gpu_power query = {0};
-		int query_size = sizeof(query);
-
-		if (amdgpu_dpm == 0)
+		if (!adev->pm.dpm_enabled)
 			return -ENOENT;
 
 		switch (info->sensor_info.type) {
@@ -746,10 +763,10 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 			/* get average GPU power */
 			if (amdgpu_dpm_read_sensor(adev,
 						   AMDGPU_PP_SENSOR_GPU_POWER,
-						   (void *)&query, &query_size)) {
+						   (void *)&ui32, &ui32_size)) {
 				return -EINVAL;
 			}
-			ui32 = query.average_gpu_power >> 8;
+			ui32 >>= 8;
 			break;
 		case AMDGPU_INFO_SENSOR_VDDNB:
 			/* get VDDNB in millivolts */
@@ -913,8 +930,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 		return;
 
 	pm_runtime_get_sync(dev->dev);
-
-	amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
+	amdgpu_ctx_mgr_entity_fini(&fpriv->ctx_mgr);
 
 	if (adev->asic_type != CHIP_RAVEN) {
 		amdgpu_uvd_free_handles(adev, file_priv);
@@ -935,6 +951,8 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 	pd = amdgpu_bo_ref(fpriv->vm.root.base.bo);
 
 	amdgpu_vm_fini(adev, &fpriv->vm);
+	amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
+
 	if (pasid)
 		amdgpu_pasid_free_delayed(pd->tbo.resv, pasid);
 	amdgpu_bo_unref(&pd);
@@ -1088,6 +1106,7 @@ static int amdgpu_debugfs_firmware_info(struct seq_file *m, void *data)
 	struct amdgpu_device *adev = dev->dev_private;
 	struct drm_amdgpu_info_firmware fw_info;
 	struct drm_amdgpu_query_fw query_fw;
+	struct atom_context *ctx = adev->mode_info.atom_context;
 	int ret, i;
 
 	/* VCE */
@@ -1146,6 +1165,30 @@ static int amdgpu_debugfs_firmware_info(struct seq_file *m, void *data)
 	seq_printf(m, "RLC feature version: %u, firmware version: 0x%08x\n",
 		   fw_info.feature, fw_info.ver);
 
+	/* RLC SAVE RESTORE LIST CNTL */
+	query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_CNTL;
+	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+	if (ret)
+		return ret;
+	seq_printf(m, "RLC SRLC feature version: %u, firmware version: 0x%08x\n",
+		   fw_info.feature, fw_info.ver);
+
+	/* RLC SAVE RESTORE LIST GPM MEM */
+	query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_GPM_MEM;
+	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+	if (ret)
+		return ret;
+	seq_printf(m, "RLC SRLG feature version: %u, firmware version: 0x%08x\n",
+		   fw_info.feature, fw_info.ver);
+
+	/* RLC SAVE RESTORE LIST SRM MEM */
+	query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_SRM_MEM;
+	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+	if (ret)
+		return ret;
+	seq_printf(m, "RLC SRLS feature version: %u, firmware version: 0x%08x\n",
+		   fw_info.feature, fw_info.ver);
+
 	/* MEC */
 	query_fw.fw_type = AMDGPU_INFO_FW_GFX_MEC;
 	query_fw.index = 0;
@@ -1210,6 +1253,9 @@ static int amdgpu_debugfs_firmware_info(struct seq_file *m, void *data)
 	seq_printf(m, "VCN feature version: %u, firmware version: 0x%08x\n",
 		   fw_info.feature, fw_info.ver);
 
+
+	seq_printf(m, "VBIOS version: %s\n", ctx->vbios_version);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
index bd67f4cb8e6c..83e344fbb50a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
@@ -36,12 +36,14 @@
 #include <drm/drm.h>
 
 #include "amdgpu.h"
+#include "amdgpu_amdkfd.h"
 
 struct amdgpu_mn {
 	/* constant after initialisation */
 	struct amdgpu_device	*adev;
 	struct mm_struct	*mm;
 	struct mmu_notifier	mn;
+	enum amdgpu_mn_type	type;
 
 	/* only used on destruction */
 	struct work_struct	work;
@@ -185,7 +187,7 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
 }
 
 /**
- * amdgpu_mn_invalidate_range_start - callback to notify about mm change
+ * amdgpu_mn_invalidate_range_start_gfx - callback to notify about mm change
  *
  * @mn: our notifier
  * @mn: the mm this callback is about
@@ -195,10 +197,10 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
  * We block for all BOs between start and end to be idle and
  * unmap them by move them into system domain again.
  */
-static void amdgpu_mn_invalidate_range_start(struct mmu_notifier *mn,
-					     struct mm_struct *mm,
-					     unsigned long start,
-					     unsigned long end)
+static void amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
+						 struct mm_struct *mm,
+						 unsigned long start,
+						 unsigned long end)
 {
 	struct amdgpu_mn *rmn = container_of(mn, struct amdgpu_mn, mn);
 	struct interval_tree_node *it;
@@ -219,6 +221,49 @@ static void amdgpu_mn_invalidate_range_start(struct mmu_notifier *mn,
 	}
 }
 
+/**
+ * amdgpu_mn_invalidate_range_start_hsa - callback to notify about mm change
+ *
+ * @mn: our notifier
+ * @mn: the mm this callback is about
+ * @start: start of updated range
+ * @end: end of updated range
+ *
+ * We temporarily evict all BOs between start and end. This
+ * necessitates evicting all user-mode queues of the process. The BOs
+ * are restorted in amdgpu_mn_invalidate_range_end_hsa.
+ */
+static void amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
+						 struct mm_struct *mm,
+						 unsigned long start,
+						 unsigned long end)
+{
+	struct amdgpu_mn *rmn = container_of(mn, struct amdgpu_mn, mn);
+	struct interval_tree_node *it;
+
+	/* notification is exclusive, but interval is inclusive */
+	end -= 1;
+
+	amdgpu_mn_read_lock(rmn);
+
+	it = interval_tree_iter_first(&rmn->objects, start, end);
+	while (it) {
+		struct amdgpu_mn_node *node;
+		struct amdgpu_bo *bo;
+
+		node = container_of(it, struct amdgpu_mn_node, it);
+		it = interval_tree_iter_next(it, start, end);
+
+		list_for_each_entry(bo, &node->bos, mn_list) {
+			struct kgd_mem *mem = bo->kfd_bo;
+
+			if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
+							 start, end))
+				amdgpu_amdkfd_evict_userptr(mem, mm);
+		}
+	}
+}
+
 /**
  * amdgpu_mn_invalidate_range_end - callback to notify about mm change
  *
@@ -239,23 +284,39 @@ static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
 	amdgpu_mn_read_unlock(rmn);
 }
 
-static const struct mmu_notifier_ops amdgpu_mn_ops = {
-	.release = amdgpu_mn_release,
-	.invalidate_range_start = amdgpu_mn_invalidate_range_start,
-	.invalidate_range_end = amdgpu_mn_invalidate_range_end,
+static const struct mmu_notifier_ops amdgpu_mn_ops[] = {
+	[AMDGPU_MN_TYPE_GFX] = {
+		.release = amdgpu_mn_release,
+		.invalidate_range_start = amdgpu_mn_invalidate_range_start_gfx,
+		.invalidate_range_end = amdgpu_mn_invalidate_range_end,
+	},
+	[AMDGPU_MN_TYPE_HSA] = {
+		.release = amdgpu_mn_release,
+		.invalidate_range_start = amdgpu_mn_invalidate_range_start_hsa,
+		.invalidate_range_end = amdgpu_mn_invalidate_range_end,
+	},
 };
 
+/* Low bits of any reasonable mm pointer will be unused due to struct
+ * alignment. Use these bits to make a unique key from the mm pointer
+ * and notifier type.
+ */
+#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
+
 /**
  * amdgpu_mn_get - create notifier context
  *
  * @adev: amdgpu device pointer
+ * @type: type of MMU notifier context
  *
  * Creates a notifier context for current->mm.
  */
-struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
+struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
+				enum amdgpu_mn_type type)
 {
 	struct mm_struct *mm = current->mm;
 	struct amdgpu_mn *rmn;
+	unsigned long key = AMDGPU_MN_KEY(mm, type);
 	int r;
 
 	mutex_lock(&adev->mn_lock);
@@ -264,8 +325,8 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
 		return ERR_PTR(-EINTR);
 	}
 
-	hash_for_each_possible(adev->mn_hash, rmn, node, (unsigned long)mm)
-		if (rmn->mm == mm)
+	hash_for_each_possible(adev->mn_hash, rmn, node, key)
+		if (AMDGPU_MN_KEY(rmn->mm, rmn->type) == key)
 			goto release_locks;
 
 	rmn = kzalloc(sizeof(*rmn), GFP_KERNEL);
@@ -276,8 +337,9 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
 
 	rmn->adev = adev;
 	rmn->mm = mm;
-	rmn->mn.ops = &amdgpu_mn_ops;
 	init_rwsem(&rmn->lock);
+	rmn->type = type;
+	rmn->mn.ops = &amdgpu_mn_ops[type];
 	rmn->objects = RB_ROOT_CACHED;
 	mutex_init(&rmn->read_lock);
 	atomic_set(&rmn->recursion, 0);
@@ -286,7 +348,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
 	if (r)
 		goto free_rmn;
 
-	hash_add(adev->mn_hash, &rmn->node, (unsigned long)mm);
+	hash_add(adev->mn_hash, &rmn->node, AMDGPU_MN_KEY(mm, type));
 
 release_locks:
 	up_write(&mm->mmap_sem);
@@ -315,15 +377,21 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
 {
 	unsigned long end = addr + amdgpu_bo_size(bo) - 1;
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	enum amdgpu_mn_type type =
+		bo->kfd_bo ? AMDGPU_MN_TYPE_HSA : AMDGPU_MN_TYPE_GFX;
 	struct amdgpu_mn *rmn;
-	struct amdgpu_mn_node *node = NULL;
+	struct amdgpu_mn_node *node = NULL, *new_node;
 	struct list_head bos;
 	struct interval_tree_node *it;
 
-	rmn = amdgpu_mn_get(adev);
+	rmn = amdgpu_mn_get(adev, type);
 	if (IS_ERR(rmn))
 		return PTR_ERR(rmn);
 
+	new_node = kmalloc(sizeof(*new_node), GFP_KERNEL);
+	if (!new_node)
+		return -ENOMEM;
+
 	INIT_LIST_HEAD(&bos);
 
 	down_write(&rmn->lock);
@@ -337,13 +405,10 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
 		list_splice(&node->bos, &bos);
 	}
 
-	if (!node) {
-		node = kmalloc(sizeof(struct amdgpu_mn_node), GFP_KERNEL);
-		if (!node) {
-			up_write(&rmn->lock);
-			return -ENOMEM;
-		}
-	}
+	if (!node)
+		node = new_node;
+	else
+		kfree(new_node);
 
 	bo->mn = rmn;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
index d0095a3793b8..eb0f432f78fe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
@@ -29,16 +29,23 @@
  */
 struct amdgpu_mn;
 
+enum amdgpu_mn_type {
+	AMDGPU_MN_TYPE_GFX,
+	AMDGPU_MN_TYPE_HSA,
+};
+
 #if defined(CONFIG_MMU_NOTIFIER)
 void amdgpu_mn_lock(struct amdgpu_mn *mn);
 void amdgpu_mn_unlock(struct amdgpu_mn *mn);
-struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev);
+struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
+				enum amdgpu_mn_type type);
 int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
 void amdgpu_mn_unregister(struct amdgpu_bo *bo);
 #else
 static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {}
 static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {}
-static inline struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
+static inline struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
+					      enum amdgpu_mn_type type)
 {
 	return NULL;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index d6416ee52e32..b9e9e8b02fb7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -308,7 +308,6 @@ struct amdgpu_display_funcs {
 
 struct amdgpu_framebuffer {
 	struct drm_framebuffer base;
-	struct drm_gem_object *obj;
 
 	/* caching for later use */
 	uint64_t address;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 6d08cde8443c..6a9e46ae7f0a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -191,14 +191,21 @@ int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
 			      u32 domain, struct amdgpu_bo **bo_ptr,
 			      u64 *gpu_addr, void **cpu_addr)
 {
+	struct amdgpu_bo_param bp;
 	bool free = false;
 	int r;
 
+	memset(&bp, 0, sizeof(bp));
+	bp.size = size;
+	bp.byte_align = align;
+	bp.domain = domain;
+	bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+		AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+	bp.type = ttm_bo_type_kernel;
+	bp.resv = NULL;
+
 	if (!*bo_ptr) {
-		r = amdgpu_bo_create(adev, size, align, domain,
-				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-				     ttm_bo_type_kernel, NULL, bo_ptr);
+		r = amdgpu_bo_create(adev, &bp, bo_ptr);
 		if (r) {
 			dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
 				r);
@@ -341,27 +348,25 @@ static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
 	return false;
 }
 
-static int amdgpu_bo_do_create(struct amdgpu_device *adev, unsigned long size,
-			       int byte_align, u32 domain,
-			       u64 flags, enum ttm_bo_type type,
-			       struct reservation_object *resv,
+static int amdgpu_bo_do_create(struct amdgpu_device *adev,
+			       struct amdgpu_bo_param *bp,
 			       struct amdgpu_bo **bo_ptr)
 {
 	struct ttm_operation_ctx ctx = {
-		.interruptible = (type != ttm_bo_type_kernel),
+		.interruptible = (bp->type != ttm_bo_type_kernel),
 		.no_wait_gpu = false,
-		.resv = resv,
+		.resv = bp->resv,
 		.flags = TTM_OPT_FLAG_ALLOW_RES_EVICT
 	};
 	struct amdgpu_bo *bo;
-	unsigned long page_align;
+	unsigned long page_align, size = bp->size;
 	size_t acc_size;
 	int r;
 
-	page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
+	page_align = roundup(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;
 	size = ALIGN(size, PAGE_SIZE);
 
-	if (!amdgpu_bo_validate_size(adev, size, domain))
+	if (!amdgpu_bo_validate_size(adev, size, bp->domain))
 		return -ENOMEM;
 
 	*bo_ptr = NULL;
@@ -375,18 +380,14 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, unsigned long size,
 	drm_gem_private_object_init(adev->ddev, &bo->gem_base, size);
 	INIT_LIST_HEAD(&bo->shadow_list);
 	INIT_LIST_HEAD(&bo->va);
-	bo->preferred_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
-					 AMDGPU_GEM_DOMAIN_GTT |
-					 AMDGPU_GEM_DOMAIN_CPU |
-					 AMDGPU_GEM_DOMAIN_GDS |
-					 AMDGPU_GEM_DOMAIN_GWS |
-					 AMDGPU_GEM_DOMAIN_OA);
+	bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :
+		bp->domain;
 	bo->allowed_domains = bo->preferred_domains;
-	if (type != ttm_bo_type_kernel &&
+	if (bp->type != ttm_bo_type_kernel &&
 	    bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
 		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
 
-	bo->flags = flags;
+	bo->flags = bp->flags;
 
 #ifdef CONFIG_X86_32
 	/* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
@@ -417,11 +418,13 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, unsigned long size,
 #endif
 
 	bo->tbo.bdev = &adev->mman.bdev;
-	amdgpu_ttm_placement_from_domain(bo, domain);
+	amdgpu_ttm_placement_from_domain(bo, bp->domain);
+	if (bp->type == ttm_bo_type_kernel)
+		bo->tbo.priority = 1;
 
-	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
+	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type,
 				 &bo->placement, page_align, &ctx, acc_size,
-				 NULL, resv, &amdgpu_ttm_bo_destroy);
+				 NULL, bp->resv, &amdgpu_ttm_bo_destroy);
 	if (unlikely(r != 0))
 		return r;
 
@@ -433,10 +436,7 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, unsigned long size,
 	else
 		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
 
-	if (type == ttm_bo_type_kernel)
-		bo->tbo.priority = 1;
-
-	if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
+	if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
 	    bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
 		struct dma_fence *fence;
 
@@ -449,20 +449,20 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, unsigned long size,
 		bo->tbo.moving = dma_fence_get(fence);
 		dma_fence_put(fence);
 	}
-	if (!resv)
+	if (!bp->resv)
 		amdgpu_bo_unreserve(bo);
 	*bo_ptr = bo;
 
 	trace_amdgpu_bo_create(bo);
 
 	/* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
-	if (type == ttm_bo_type_device)
+	if (bp->type == ttm_bo_type_device)
 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 
 	return 0;
 
 fail_unreserve:
-	if (!resv)
+	if (!bp->resv)
 		ww_mutex_unlock(&bo->tbo.resv->lock);
 	amdgpu_bo_unref(&bo);
 	return r;
@@ -472,16 +472,22 @@ static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
 				   unsigned long size, int byte_align,
 				   struct amdgpu_bo *bo)
 {
+	struct amdgpu_bo_param bp;
 	int r;
 
 	if (bo->shadow)
 		return 0;
 
-	r = amdgpu_bo_do_create(adev, size, byte_align, AMDGPU_GEM_DOMAIN_GTT,
-				AMDGPU_GEM_CREATE_CPU_GTT_USWC |
-				AMDGPU_GEM_CREATE_SHADOW,
-				ttm_bo_type_kernel,
-				bo->tbo.resv, &bo->shadow);
+	memset(&bp, 0, sizeof(bp));
+	bp.size = size;
+	bp.byte_align = byte_align;
+	bp.domain = AMDGPU_GEM_DOMAIN_GTT;
+	bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+		AMDGPU_GEM_CREATE_SHADOW;
+	bp.type = ttm_bo_type_kernel;
+	bp.resv = bo->tbo.resv;
+
+	r = amdgpu_bo_do_create(adev, &bp, &bo->shadow);
 	if (!r) {
 		bo->shadow->parent = amdgpu_bo_ref(bo);
 		mutex_lock(&adev->shadow_list_lock);
@@ -492,28 +498,26 @@ static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
 	return r;
 }
 
-int amdgpu_bo_create(struct amdgpu_device *adev, unsigned long size,
-		     int byte_align, u32 domain,
-		     u64 flags, enum ttm_bo_type type,
-		     struct reservation_object *resv,
+int amdgpu_bo_create(struct amdgpu_device *adev,
+		     struct amdgpu_bo_param *bp,
 		     struct amdgpu_bo **bo_ptr)
 {
-	uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW;
+	u64 flags = bp->flags;
 	int r;
 
-	r = amdgpu_bo_do_create(adev, size, byte_align, domain,
-				parent_flags, type, resv, bo_ptr);
+	bp->flags = bp->flags & ~AMDGPU_GEM_CREATE_SHADOW;
+	r = amdgpu_bo_do_create(adev, bp, bo_ptr);
 	if (r)
 		return r;
 
 	if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_need_backup(adev)) {
-		if (!resv)
+		if (!bp->resv)
 			WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv,
 							NULL));
 
-		r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
+		r = amdgpu_bo_create_shadow(adev, bp->size, bp->byte_align, (*bo_ptr));
 
-		if (!resv)
+		if (!bp->resv)
 			reservation_object_unlock((*bo_ptr)->tbo.resv);
 
 		if (r)
@@ -689,8 +693,21 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 		return -EINVAL;
 
 	/* A shared bo cannot be migrated to VRAM */
-	if (bo->prime_shared_count && (domain == AMDGPU_GEM_DOMAIN_VRAM))
-		return -EINVAL;
+	if (bo->prime_shared_count) {
+		if (domain & AMDGPU_GEM_DOMAIN_GTT)
+			domain = AMDGPU_GEM_DOMAIN_GTT;
+		else
+			return -EINVAL;
+	}
+
+	/* This assumes only APU display buffers are pinned with (VRAM|GTT).
+	 * See function amdgpu_display_supported_domains()
+	 */
+	if (domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) {
+		domain = AMDGPU_GEM_DOMAIN_VRAM;
+		if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD)
+			domain = AMDGPU_GEM_DOMAIN_GTT;
+	}
 
 	if (bo->pin_count) {
 		uint32_t mem_type = bo->tbo.mem.mem_type;
@@ -838,6 +855,13 @@ int amdgpu_bo_init(struct amdgpu_device *adev)
 	return amdgpu_ttm_init(adev);
 }
 
+int amdgpu_bo_late_init(struct amdgpu_device *adev)
+{
+	amdgpu_ttm_late_init(adev);
+
+	return 0;
+}
+
 void amdgpu_bo_fini(struct amdgpu_device *adev)
 {
 	amdgpu_ttm_fini(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 546f77cb7882..540e03fa159f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -33,6 +33,16 @@
 
 #define AMDGPU_BO_INVALID_OFFSET	LONG_MAX
 
+struct amdgpu_bo_param {
+	unsigned long			size;
+	int				byte_align;
+	u32				domain;
+	u32				preferred_domain;
+	u64				flags;
+	enum ttm_bo_type		type;
+	struct reservation_object	*resv;
+};
+
 /* bo virtual addresses in a vm */
 struct amdgpu_bo_va_mapping {
 	struct amdgpu_bo_va		*bo_va;
@@ -195,6 +205,27 @@ static inline bool amdgpu_bo_gpu_accessible(struct amdgpu_bo *bo)
 	}
 }
 
+/**
+ * amdgpu_bo_in_cpu_visible_vram - check if BO is (partly) in visible VRAM
+ */
+static inline bool amdgpu_bo_in_cpu_visible_vram(struct amdgpu_bo *bo)
+{
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	unsigned fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
+	struct drm_mm_node *node = bo->tbo.mem.mm_node;
+	unsigned long pages_left;
+
+	if (bo->tbo.mem.mem_type != TTM_PL_VRAM)
+		return false;
+
+	for (pages_left = bo->tbo.mem.num_pages; pages_left;
+	     pages_left -= node->size, node++)
+		if (node->start < fpfn)
+			return true;
+
+	return false;
+}
+
 /**
  * amdgpu_bo_explicit_sync - return whether the bo is explicitly synced
  */
@@ -203,10 +234,8 @@ static inline bool amdgpu_bo_explicit_sync(struct amdgpu_bo *bo)
 	return bo->flags & AMDGPU_GEM_CREATE_EXPLICIT_SYNC;
 }
 
-int amdgpu_bo_create(struct amdgpu_device *adev, unsigned long size,
-		     int byte_align, u32 domain,
-		     u64 flags, enum ttm_bo_type type,
-		     struct reservation_object *resv,
+int amdgpu_bo_create(struct amdgpu_device *adev,
+		     struct amdgpu_bo_param *bp,
 		     struct amdgpu_bo **bo_ptr);
 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
 			      unsigned long size, int align,
@@ -230,6 +259,7 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 int amdgpu_bo_unpin(struct amdgpu_bo *bo);
 int amdgpu_bo_evict_vram(struct amdgpu_device *adev);
 int amdgpu_bo_init(struct amdgpu_device *adev);
+int amdgpu_bo_late_init(struct amdgpu_device *adev);
 void amdgpu_bo_fini(struct amdgpu_device *adev);
 int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo,
 				struct vm_area_struct *vma);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index 361975cf45a9..b455da487782 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -77,6 +77,37 @@ void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev)
 	}
 }
 
+/**
+ * DOC: power_dpm_state
+ *
+ * This is a legacy interface and is only provided for backwards compatibility.
+ * The amdgpu driver provides a sysfs API for adjusting certain power
+ * related parameters.  The file power_dpm_state is used for this.
+ * It accepts the following arguments:
+ * - battery
+ * - balanced
+ * - performance
+ *
+ * battery
+ *
+ * On older GPUs, the vbios provided a special power state for battery
+ * operation.  Selecting battery switched to this state.  This is no
+ * longer provided on newer GPUs so the option does nothing in that case.
+ *
+ * balanced
+ *
+ * On older GPUs, the vbios provided a special power state for balanced
+ * operation.  Selecting balanced switched to this state.  This is no
+ * longer provided on newer GPUs so the option does nothing in that case.
+ *
+ * performance
+ *
+ * On older GPUs, the vbios provided a special power state for performance
+ * operation.  Selecting performance switched to this state.  This is no
+ * longer provided on newer GPUs so the option does nothing in that case.
+ *
+ */
+
 static ssize_t amdgpu_get_dpm_state(struct device *dev,
 				    struct device_attribute *attr,
 				    char *buf)
@@ -131,6 +162,59 @@ static ssize_t amdgpu_set_dpm_state(struct device *dev,
 	return count;
 }
 
+
+/**
+ * DOC: power_dpm_force_performance_level
+ *
+ * The amdgpu driver provides a sysfs API for adjusting certain power
+ * related parameters.  The file power_dpm_force_performance_level is
+ * used for this.  It accepts the following arguments:
+ * - auto
+ * - low
+ * - high
+ * - manual
+ * - GPU fan
+ * - profile_standard
+ * - profile_min_sclk
+ * - profile_min_mclk
+ * - profile_peak
+ *
+ * auto
+ *
+ * When auto is selected, the driver will attempt to dynamically select
+ * the optimal power profile for current conditions in the driver.
+ *
+ * low
+ *
+ * When low is selected, the clocks are forced to the lowest power state.
+ *
+ * high
+ *
+ * When high is selected, the clocks are forced to the highest power state.
+ *
+ * manual
+ *
+ * When manual is selected, the user can manually adjust which power states
+ * are enabled for each clock domain via the sysfs pp_dpm_mclk, pp_dpm_sclk,
+ * and pp_dpm_pcie files and adjust the power state transition heuristics
+ * via the pp_power_profile_mode sysfs file.
+ *
+ * profile_standard
+ * profile_min_sclk
+ * profile_min_mclk
+ * profile_peak
+ *
+ * When the profiling modes are selected, clock and power gating are
+ * disabled and the clocks are set for different profiling cases. This
+ * mode is recommended for profiling specific work loads where you do
+ * not want clock or power gating for clock fluctuation to interfere
+ * with your results. profile_standard sets the clocks to a fixed clock
+ * level which varies from asic to asic.  profile_min_sclk forces the sclk
+ * to the lowest level.  profile_min_mclk forces the mclk to the lowest level.
+ * profile_peak sets all clocks (mclk, sclk, pcie) to the highest levels.
+ *
+ */
+
 static ssize_t amdgpu_get_dpm_forced_performance_level(struct device *dev,
 						struct device_attribute *attr,
 								char *buf)
@@ -324,6 +408,17 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev,
 	return count;
 }
 
+/**
+ * DOC: pp_table
+ *
+ * The amdgpu driver provides a sysfs API for uploading new powerplay
+ * tables.  The file pp_table is used for this.  Reading the file
+ * will dump the current power play table.  Writing to the file
+ * will attempt to upload a new powerplay table and re-initialize
+ * powerplay using that new table.
+ *
+ */
+
 static ssize_t amdgpu_get_pp_table(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
@@ -360,6 +455,29 @@ static ssize_t amdgpu_set_pp_table(struct device *dev,
 	return count;
 }
 
+/**
+ * DOC: pp_od_clk_voltage
+ *
+ * The amdgpu driver provides a sysfs API for adjusting the clocks and voltages
+ * in each power level within a power state.  The pp_od_clk_voltage is used for
+ * this.
+ *
+ * Reading the file will display:
+ * - a list of engine clock levels and voltages labeled OD_SCLK
+ * - a list of memory clock levels and voltages labeled OD_MCLK
+ * - a list of valid ranges for sclk, mclk, and voltage labeled OD_RANGE
+ *
+ * To manually adjust these settings, first select manual using
+ * power_dpm_force_performance_level. Enter a new value for each
+ * level by writing a string that contains "s/m level clock voltage" to
+ * the file.  E.g., "s 1 500 820" will update sclk level 1 to be 500 MHz
+ * at 820 mV; "m 0 350 810" will update mclk level 0 to be 350 MHz at
+ * 810 mV.  When you have edited all of the states as needed, write
+ * "c" (commit) to the file to commit your changes.  If you want to reset to the
+ * default power levels, write "r" (reset) to the file to reset them.
+ *
+ */
+
 static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev,
 		struct device_attribute *attr,
 		const char *buf,
@@ -437,6 +555,7 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev,
 	if (adev->powerplay.pp_funcs->print_clock_levels) {
 		size = amdgpu_dpm_print_clock_levels(adev, OD_SCLK, buf);
 		size += amdgpu_dpm_print_clock_levels(adev, OD_MCLK, buf+size);
+		size += amdgpu_dpm_print_clock_levels(adev, OD_RANGE, buf+size);
 		return size;
 	} else {
 		return snprintf(buf, PAGE_SIZE, "\n");
@@ -444,6 +563,23 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev,
 
 }
 
+/**
+ * DOC: pp_dpm_sclk pp_dpm_mclk pp_dpm_pcie
+ *
+ * The amdgpu driver provides a sysfs API for adjusting what power levels
+ * are enabled for a given power state.  The files pp_dpm_sclk, pp_dpm_mclk,
+ * and pp_dpm_pcie are used for this.
+ *
+ * Reading back the files will show you the available power levels within
+ * the power state and the clock information for those levels.
+ *
+ * To manually adjust these states, first select manual using
+ * power_dpm_force_performance_level.
+ * Secondly,Enter a new value for each level by inputing a string that
+ * contains " echo xx xx xx > pp_dpm_sclk/mclk/pcie"
+ * E.g., echo 4 5 6 to > pp_dpm_sclk will enable sclk levels 4, 5, and 6.
+ */
+
 static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
@@ -466,23 +602,27 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
 	struct amdgpu_device *adev = ddev->dev_private;
 	int ret;
 	long level;
-	uint32_t i, mask = 0;
-	char sub_str[2];
+	uint32_t mask = 0;
+	char *sub_str = NULL;
+	char *tmp;
+	char buf_cpy[count];
+	const char delimiter[3] = {' ', '\n', '\0'};
 
-	for (i = 0; i < strlen(buf); i++) {
-		if (*(buf + i) == '\n')
-			continue;
-		sub_str[0] = *(buf + i);
-		sub_str[1] = '\0';
-		ret = kstrtol(sub_str, 0, &level);
+	memcpy(buf_cpy, buf, count+1);
+	tmp = buf_cpy;
+	while (tmp[0]) {
+		sub_str =  strsep(&tmp, delimiter);
+		if (strlen(sub_str)) {
+			ret = kstrtol(sub_str, 0, &level);
 
-		if (ret) {
-			count = -EINVAL;
-			goto fail;
-		}
-		mask |= 1 << level;
+			if (ret) {
+				count = -EINVAL;
+				goto fail;
+			}
+			mask |= 1 << level;
+		} else
+			break;
 	}
-
 	if (adev->powerplay.pp_funcs->force_clock_level)
 		amdgpu_dpm_force_clock_level(adev, PP_SCLK, mask);
 
@@ -512,21 +652,26 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev,
 	struct amdgpu_device *adev = ddev->dev_private;
 	int ret;
 	long level;
-	uint32_t i, mask = 0;
-	char sub_str[2];
+	uint32_t mask = 0;
+	char *sub_str = NULL;
+	char *tmp;
+	char buf_cpy[count];
+	const char delimiter[3] = {' ', '\n', '\0'};
 
-	for (i = 0; i < strlen(buf); i++) {
-		if (*(buf + i) == '\n')
-			continue;
-		sub_str[0] = *(buf + i);
-		sub_str[1] = '\0';
-		ret = kstrtol(sub_str, 0, &level);
+	memcpy(buf_cpy, buf, count+1);
+	tmp = buf_cpy;
+	while (tmp[0]) {
+		sub_str =  strsep(&tmp, delimiter);
+		if (strlen(sub_str)) {
+			ret = kstrtol(sub_str, 0, &level);
 
-		if (ret) {
-			count = -EINVAL;
-			goto fail;
-		}
-		mask |= 1 << level;
+			if (ret) {
+				count = -EINVAL;
+				goto fail;
+			}
+			mask |= 1 << level;
+		} else
+			break;
 	}
 	if (adev->powerplay.pp_funcs->force_clock_level)
 		amdgpu_dpm_force_clock_level(adev, PP_MCLK, mask);
@@ -557,21 +702,27 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev,
 	struct amdgpu_device *adev = ddev->dev_private;
 	int ret;
 	long level;
-	uint32_t i, mask = 0;
-	char sub_str[2];
+	uint32_t mask = 0;
+	char *sub_str = NULL;
+	char *tmp;
+	char buf_cpy[count];
+	const char delimiter[3] = {' ', '\n', '\0'};
 
-	for (i = 0; i < strlen(buf); i++) {
-		if (*(buf + i) == '\n')
-			continue;
-		sub_str[0] = *(buf + i);
-		sub_str[1] = '\0';
-		ret = kstrtol(sub_str, 0, &level);
+	memcpy(buf_cpy, buf, count+1);
+	tmp = buf_cpy;
 
-		if (ret) {
-			count = -EINVAL;
-			goto fail;
-		}
-		mask |= 1 << level;
+	while (tmp[0]) {
+		sub_str =  strsep(&tmp, delimiter);
+		if (strlen(sub_str)) {
+			ret = kstrtol(sub_str, 0, &level);
+
+			if (ret) {
+				count = -EINVAL;
+				goto fail;
+			}
+			mask |= 1 << level;
+		} else
+			break;
 	}
 	if (adev->powerplay.pp_funcs->force_clock_level)
 		amdgpu_dpm_force_clock_level(adev, PP_PCIE, mask);
@@ -668,6 +819,26 @@ static ssize_t amdgpu_set_pp_mclk_od(struct device *dev,
 	return count;
 }
 
+/**
+ * DOC: pp_power_profile_mode
+ *
+ * The amdgpu driver provides a sysfs API for adjusting the heuristics
+ * related to switching between power levels in a power state.  The file
+ * pp_power_profile_mode is used for this.
+ *
+ * Reading this file outputs a list of all of the predefined power profiles
+ * and the relevant heuristics settings for that profile.
+ *
+ * To select a profile or create a custom profile, first select manual using
+ * power_dpm_force_performance_level.  Writing the number of a predefined
+ * profile to pp_power_profile_mode will enable those heuristics.  To
+ * create a custom set of heuristics, write a string of numbers to the file
+ * starting with the number of the custom profile along with a setting
+ * for each heuristic parameter.  Due to differences across asic families
+ * the heuristic parameters vary from family to family.
+ *
+ */
+
 static ssize_t amdgpu_get_pp_power_profile_mode(struct device *dev,
 		struct device_attribute *attr,
 		char *buf)
@@ -1020,8 +1191,8 @@ static ssize_t amdgpu_hwmon_show_power_avg(struct device *dev,
 {
 	struct amdgpu_device *adev = dev_get_drvdata(dev);
 	struct drm_device *ddev = adev->ddev;
-	struct pp_gpu_power query = {0};
-	int r, size = sizeof(query);
+	u32 query = 0;
+	int r, size = sizeof(u32);
 	unsigned uw;
 
 	/* Can't get power when the card is off */
@@ -1041,7 +1212,7 @@ static ssize_t amdgpu_hwmon_show_power_avg(struct device *dev,
 		return r;
 
 	/* convert to microwatts */
-	uw = (query.average_gpu_power >> 8) * 1000000;
+	uw = (query >> 8) * 1000000 + (query & 0xff) * 1000;
 
 	return snprintf(buf, PAGE_SIZE, "%u\n", uw);
 }
@@ -1109,6 +1280,46 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev,
 	return count;
 }
 
+
+/**
+ * DOC: hwmon
+ *
+ * The amdgpu driver exposes the following sensor interfaces:
+ * - GPU temperature (via the on-die sensor)
+ * - GPU voltage
+ * - Northbridge voltage (APUs only)
+ * - GPU power
+ * - GPU fan
+ *
+ * hwmon interfaces for GPU temperature:
+ * - temp1_input: the on die GPU temperature in millidegrees Celsius
+ * - temp1_crit: temperature critical max value in millidegrees Celsius
+ * - temp1_crit_hyst: temperature hysteresis for critical limit in millidegrees Celsius
+ *
+ * hwmon interfaces for GPU voltage:
+ * - in0_input: the voltage on the GPU in millivolts
+ * - in1_input: the voltage on the Northbridge in millivolts
+ *
+ * hwmon interfaces for GPU power:
+ * - power1_average: average power used by the GPU in microWatts
+ * - power1_cap_min: minimum cap supported in microWatts
+ * - power1_cap_max: maximum cap supported in microWatts
+ * - power1_cap: selected power cap in microWatts
+ *
+ * hwmon interfaces for GPU fan:
+ * - pwm1: pulse width modulation fan level (0-255)
+ * - pwm1_enable: pulse width modulation fan control method
+ *                0: no fan speed control
+ *                1: manual fan speed control using pwm interface
+ *                2: automatic fan speed control
+ * - pwm1_min: pulse width modulation fan control minimum level (0)
+ * - pwm1_max: pulse width modulation fan control maximum level (255)
+ * - fan1_input: fan speed in RPM
+ *
+ * You can use hwmon tools like sensors to view this information on your system.
+ *
+ */
+
 static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, amdgpu_hwmon_show_temp, NULL, 0);
 static SENSOR_DEVICE_ATTR(temp1_crit, S_IRUGO, amdgpu_hwmon_show_temp_thresh, NULL, 0);
 static SENSOR_DEVICE_ATTR(temp1_crit_hyst, S_IRUGO, amdgpu_hwmon_show_temp_thresh, NULL, 1);
@@ -1153,19 +1364,14 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
 	struct amdgpu_device *adev = dev_get_drvdata(dev);
 	umode_t effective_mode = attr->mode;
 
-	/* handle non-powerplay limitations */
-	if (!adev->powerplay.pp_handle) {
-		/* Skip fan attributes if fan is not present */
-		if (adev->pm.no_fan &&
-		    (attr == &sensor_dev_attr_pwm1.dev_attr.attr ||
-		     attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr ||
-		     attr == &sensor_dev_attr_pwm1_max.dev_attr.attr ||
-		     attr == &sensor_dev_attr_pwm1_min.dev_attr.attr))
-			return 0;
-		/* requires powerplay */
-		if (attr == &sensor_dev_attr_fan1_input.dev_attr.attr)
-			return 0;
-	}
+
+	/* Skip fan attributes if fan is not present */
+	if (adev->pm.no_fan && (attr == &sensor_dev_attr_pwm1.dev_attr.attr ||
+	    attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr ||
+	    attr == &sensor_dev_attr_pwm1_max.dev_attr.attr ||
+	    attr == &sensor_dev_attr_pwm1_min.dev_attr.attr ||
+	    attr == &sensor_dev_attr_fan1_input.dev_attr.attr))
+		return 0;
 
 	/* Skip limit attributes if DPM is not enabled */
 	if (!adev->pm.dpm_enabled &&
@@ -1658,9 +1864,6 @@ void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev)
 
 void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
 {
-	struct drm_device *ddev = adev->ddev;
-	struct drm_crtc *crtc;
-	struct amdgpu_crtc *amdgpu_crtc;
 	int i = 0;
 
 	if (!adev->pm.dpm_enabled)
@@ -1676,21 +1879,25 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
 	}
 
 	if (adev->powerplay.pp_funcs->dispatch_tasks) {
+		if (!amdgpu_device_has_dc_support(adev)) {
+			mutex_lock(&adev->pm.mutex);
+			amdgpu_dpm_get_active_displays(adev);
+			adev->pm.pm_display_cfg.num_display = adev->pm.dpm.new_active_crtcs;
+			adev->pm.pm_display_cfg.vrefresh = amdgpu_dpm_get_vrefresh(adev);
+			adev->pm.pm_display_cfg.min_vblank_time = amdgpu_dpm_get_vblank_time(adev);
+			/* we have issues with mclk switching with refresh rates over 120 hz on the non-DC code. */
+			if (adev->pm.pm_display_cfg.vrefresh > 120)
+				adev->pm.pm_display_cfg.min_vblank_time = 0;
+			if (adev->powerplay.pp_funcs->display_configuration_change)
+				adev->powerplay.pp_funcs->display_configuration_change(
+								adev->powerplay.pp_handle,
+								&adev->pm.pm_display_cfg);
+			mutex_unlock(&adev->pm.mutex);
+		}
 		amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, NULL);
 	} else {
 		mutex_lock(&adev->pm.mutex);
-		adev->pm.dpm.new_active_crtcs = 0;
-		adev->pm.dpm.new_active_crtc_count = 0;
-		if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) {
-			list_for_each_entry(crtc,
-					    &ddev->mode_config.crtc_list, head) {
-				amdgpu_crtc = to_amdgpu_crtc(crtc);
-				if (amdgpu_crtc->enabled) {
-					adev->pm.dpm.new_active_crtcs |= (1 << amdgpu_crtc->crtc_id);
-					adev->pm.dpm.new_active_crtc_count++;
-				}
-			}
-		}
+		amdgpu_dpm_get_active_displays(adev);
 		/* update battery/ac status */
 		if (power_supply_is_system_supplied() > 0)
 			adev->pm.dpm.ac_power = true;
@@ -1711,7 +1918,7 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
 static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *adev)
 {
 	uint32_t value;
-	struct pp_gpu_power query = {0};
+	uint32_t query = 0;
 	int size;
 
 	/* sanity check PP is enabled */
@@ -1734,17 +1941,9 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a
 		seq_printf(m, "\t%u mV (VDDGFX)\n", value);
 	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VDDNB, (void *)&value, &size))
 		seq_printf(m, "\t%u mV (VDDNB)\n", value);
-	size = sizeof(query);
-	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_POWER, (void *)&query, &size)) {
-		seq_printf(m, "\t%u.%u W (VDDC)\n", query.vddc_power >> 8,
-				query.vddc_power & 0xff);
-		seq_printf(m, "\t%u.%u W (VDDCI)\n", query.vddci_power >> 8,
-				query.vddci_power & 0xff);
-		seq_printf(m, "\t%u.%u W (max GPU)\n", query.max_gpu_power >> 8,
-				query.max_gpu_power & 0xff);
-		seq_printf(m, "\t%u.%u W (average GPU)\n", query.average_gpu_power >> 8,
-				query.average_gpu_power & 0xff);
-	}
+	size = sizeof(uint32_t);
+	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_POWER, (void *)&query, &size))
+		seq_printf(m, "\t%u.%u W (average GPU)\n", query >> 8, query & 0xff);
 	size = sizeof(value);
 	seq_printf(m, "\n");
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
index 4b584cb75bf4..4683626b065f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
@@ -102,12 +102,18 @@ amdgpu_gem_prime_import_sg_table(struct drm_device *dev,
 	struct reservation_object *resv = attach->dmabuf->resv;
 	struct amdgpu_device *adev = dev->dev_private;
 	struct amdgpu_bo *bo;
+	struct amdgpu_bo_param bp;
 	int ret;
 
+	memset(&bp, 0, sizeof(bp));
+	bp.size = attach->dmabuf->size;
+	bp.byte_align = PAGE_SIZE;
+	bp.domain = AMDGPU_GEM_DOMAIN_CPU;
+	bp.flags = 0;
+	bp.type = ttm_bo_type_sg;
+	bp.resv = resv;
 	ww_mutex_lock(&resv->lock, NULL);
-	ret = amdgpu_bo_create(adev, attach->dmabuf->size, PAGE_SIZE,
-			       AMDGPU_GEM_DOMAIN_CPU, 0, ttm_bo_type_sg,
-			       resv, &bo);
+	ret = amdgpu_bo_create(adev, &bp, &bo);
 	if (ret)
 		goto error;
 
@@ -209,7 +215,7 @@ static int amdgpu_gem_begin_cpu_access(struct dma_buf *dma_buf,
 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(dma_buf->priv);
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct ttm_operation_ctx ctx = { true, false };
-	u32 domain = amdgpu_display_framebuffer_domains(adev);
+	u32 domain = amdgpu_display_supported_domains(adev);
 	int ret;
 	bool reads = (direction == DMA_BIDIRECTIONAL ||
 		      direction == DMA_FROM_DEVICE);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index c7d43e064fc7..9f1a5bd39ae8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -52,6 +52,7 @@ static int psp_sw_init(void *handle)
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 		psp_v3_1_set_psp_funcs(psp);
 		break;
 	case CHIP_RAVEN:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
index 262c1267249e..8af16e81c7d4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
@@ -66,6 +66,8 @@ static int amdgpu_identity_map(struct amdgpu_device *adev,
 			       u32 ring,
 			       struct amdgpu_ring **out_ring)
 {
+	u32 instance;
+
 	switch (mapper->hw_ip) {
 	case AMDGPU_HW_IP_GFX:
 		*out_ring = &adev->gfx.gfx_ring[ring];
@@ -77,13 +79,16 @@ static int amdgpu_identity_map(struct amdgpu_device *adev,
 		*out_ring = &adev->sdma.instance[ring].ring;
 		break;
 	case AMDGPU_HW_IP_UVD:
-		*out_ring = &adev->uvd.ring;
+		instance = ring;
+		*out_ring = &adev->uvd.inst[instance].ring;
 		break;
 	case AMDGPU_HW_IP_VCE:
 		*out_ring = &adev->vce.ring[ring];
 		break;
 	case AMDGPU_HW_IP_UVD_ENC:
-		*out_ring = &adev->uvd.ring_enc[ring];
+		instance = ring / adev->uvd.num_enc_rings;
+		*out_ring =
+		&adev->uvd.inst[instance].ring_enc[ring%adev->uvd.num_enc_rings];
 		break;
 	case AMDGPU_HW_IP_VCN_DEC:
 		*out_ring = &adev->vcn.ring_dec;
@@ -240,13 +245,14 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
 		ip_num_rings = adev->sdma.num_instances;
 		break;
 	case AMDGPU_HW_IP_UVD:
-		ip_num_rings = 1;
+		ip_num_rings = adev->uvd.num_uvd_inst;
 		break;
 	case AMDGPU_HW_IP_VCE:
 		ip_num_rings = adev->vce.num_rings;
 		break;
 	case AMDGPU_HW_IP_UVD_ENC:
-		ip_num_rings = adev->uvd.num_enc_rings;
+		ip_num_rings =
+			adev->uvd.num_enc_rings * adev->uvd.num_uvd_inst;
 		break;
 	case AMDGPU_HW_IP_VCN_DEC:
 		ip_num_rings = 1;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index d5f526f38e50..c6850b629d0e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -362,6 +362,7 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring)
 
 	dma_fence_put(ring->vmid_wait);
 	ring->vmid_wait = NULL;
+	ring->me = 0;
 
 	ring->adev->rings[ring->idx] = NULL;
 }
@@ -459,6 +460,26 @@ void amdgpu_ring_lru_touch(struct amdgpu_device *adev, struct amdgpu_ring *ring)
 	spin_unlock(&adev->ring_lru_list_lock);
 }
 
+/**
+ * amdgpu_ring_emit_reg_write_reg_wait_helper - ring helper
+ *
+ * @adev: amdgpu_device pointer
+ * @reg0: register to write
+ * @reg1: register to wait on
+ * @ref: reference value to write/wait on
+ * @mask: mask to wait on
+ *
+ * Helper for rings that don't support write and wait in a
+ * single oneshot packet.
+ */
+void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,
+						uint32_t reg0, uint32_t reg1,
+						uint32_t ref, uint32_t mask)
+{
+	amdgpu_ring_emit_wreg(ring, reg0, ref);
+	amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
+}
+
 /*
  * Debugfs info
  */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 1a5911882657..1513124c5659 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -29,7 +29,7 @@
 #include <drm/drm_print.h>
 
 /* max number of rings */
-#define AMDGPU_MAX_RINGS		18
+#define AMDGPU_MAX_RINGS		21
 #define AMDGPU_MAX_GFX_RINGS		1
 #define AMDGPU_MAX_COMPUTE_RINGS	8
 #define AMDGPU_MAX_VCE_RINGS		3
@@ -42,6 +42,7 @@
 
 #define AMDGPU_FENCE_FLAG_64BIT         (1 << 0)
 #define AMDGPU_FENCE_FLAG_INT           (1 << 1)
+#define AMDGPU_FENCE_FLAG_TC_WB_ONLY    (1 << 2)
 
 enum amdgpu_ring_type {
 	AMDGPU_RING_TYPE_GFX,
@@ -90,7 +91,8 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
 				   unsigned irq_type);
 void amdgpu_fence_driver_suspend(struct amdgpu_device *adev);
 void amdgpu_fence_driver_resume(struct amdgpu_device *adev);
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence);
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence,
+		      unsigned flags);
 int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s);
 void amdgpu_fence_process(struct amdgpu_ring *ring);
 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring);
@@ -154,6 +156,9 @@ struct amdgpu_ring_funcs {
 	void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
 	void (*emit_reg_wait)(struct amdgpu_ring *ring, uint32_t reg,
 			      uint32_t val, uint32_t mask);
+	void (*emit_reg_write_reg_wait)(struct amdgpu_ring *ring,
+					uint32_t reg0, uint32_t reg1,
+					uint32_t ref, uint32_t mask);
 	void (*emit_tmz)(struct amdgpu_ring *ring, bool start);
 	/* priority functions */
 	void (*set_priority) (struct amdgpu_ring *ring,
@@ -228,6 +233,10 @@ int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type,
 			int *blacklist, int num_blacklist,
 			bool lru_pipe_order, struct amdgpu_ring **ring);
 void amdgpu_ring_lru_touch(struct amdgpu_device *adev, struct amdgpu_ring *ring);
+void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,
+						uint32_t reg0, uint32_t val0,
+						uint32_t reg1, uint32_t val1);
+
 static inline void amdgpu_ring_clear_ring(struct amdgpu_ring *ring)
 {
 	int i = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
index 2dbe87591f81..d167e8ab76d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
@@ -33,6 +33,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 	struct amdgpu_bo *vram_obj = NULL;
 	struct amdgpu_bo **gtt_obj = NULL;
+	struct amdgpu_bo_param bp;
 	uint64_t gart_addr, vram_addr;
 	unsigned n, size;
 	int i, r;
@@ -58,9 +59,15 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 		r = 1;
 		goto out_cleanup;
 	}
-
-	r = amdgpu_bo_create(adev, size, PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, 0,
-			     ttm_bo_type_kernel, NULL, &vram_obj);
+	memset(&bp, 0, sizeof(bp));
+	bp.size = size;
+	bp.byte_align = PAGE_SIZE;
+	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
+	bp.flags = 0;
+	bp.type = ttm_bo_type_kernel;
+	bp.resv = NULL;
+
+	r = amdgpu_bo_create(adev, &bp, &vram_obj);
 	if (r) {
 		DRM_ERROR("Failed to create VRAM object\n");
 		goto out_cleanup;
@@ -79,9 +86,8 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 		void **vram_start, **vram_end;
 		struct dma_fence *fence = NULL;
 
-		r = amdgpu_bo_create(adev, size, PAGE_SIZE,
-				     AMDGPU_GEM_DOMAIN_GTT, 0,
-				     ttm_bo_type_kernel, NULL, gtt_obj + i);
+		bp.domain = AMDGPU_GEM_DOMAIN_GTT;
+		r = amdgpu_bo_create(adev, &bp, gtt_obj + i);
 		if (r) {
 			DRM_ERROR("Failed to create GTT object %d\n", i);
 			goto out_lclean;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 532263ab6e16..e96e26d3f3b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -275,7 +275,7 @@ TRACE_EVENT(amdgpu_vm_bo_unmap,
 			     ),
 
 	    TP_fast_assign(
-			   __entry->bo = bo_va->base.bo;
+			   __entry->bo = bo_va ? bo_va->base.bo : NULL;
 			   __entry->start = mapping->start;
 			   __entry->last = mapping->last;
 			   __entry->offset = mapping->offset;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 205da3ff9cd0..e93a0a237dc3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -63,16 +63,44 @@ static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);
 /*
  * Global memory.
  */
+
+/**
+ * amdgpu_ttm_mem_global_init - Initialize and acquire reference to
+ * memory object
+ *
+ * @ref: Object for initialization.
+ *
+ * This is called by drm_global_item_ref() when an object is being
+ * initialized.
+ */
 static int amdgpu_ttm_mem_global_init(struct drm_global_reference *ref)
 {
 	return ttm_mem_global_init(ref->object);
 }
 
+/**
+ * amdgpu_ttm_mem_global_release - Drop reference to a memory object
+ *
+ * @ref: Object being removed
+ *
+ * This is called by drm_global_item_unref() when an object is being
+ * released.
+ */
 static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref)
 {
 	ttm_mem_global_release(ref->object);
 }
 
+/**
+ * amdgpu_ttm_global_init - Initialize global TTM memory reference
+ * 							structures.
+ *
+ * @adev:  	AMDGPU device for which the global structures need to be
+ *			registered.
+ *
+ * This is called as part of the AMDGPU ttm init from amdgpu_ttm_init()
+ * during bring up.
+ */
 static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 {
 	struct drm_global_reference *global_ref;
@@ -80,7 +108,9 @@ static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 	struct drm_sched_rq *rq;
 	int r;
 
+	/* ensure reference is false in case init fails */
 	adev->mman.mem_global_referenced = false;
+
 	global_ref = &adev->mman.mem_global_ref;
 	global_ref->global_type = DRM_GLOBAL_TTM_MEM;
 	global_ref->size = sizeof(struct ttm_mem_global);
@@ -111,7 +141,7 @@ static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 	ring = adev->mman.buffer_funcs_ring;
 	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
 	r = drm_sched_entity_init(&ring->sched, &adev->mman.entity,
-				  rq, amdgpu_sched_jobs, NULL);
+				  rq, NULL);
 	if (r) {
 		DRM_ERROR("Failed setting up TTM BO move run queue.\n");
 		goto error_entity;
@@ -146,6 +176,18 @@ static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
 	return 0;
 }
 
+/**
+ * amdgpu_init_mem_type - 	Initialize a memory manager for a specific
+ * 							type of memory request.
+ *
+ * @bdev:	The TTM BO device object (contains a reference to
+ * 			amdgpu_device)
+ * @type:	The type of memory requested
+ * @man:
+ *
+ * This is called by ttm_bo_init_mm() when a buffer object is being
+ * initialized.
+ */
 static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 				struct ttm_mem_type_manager *man)
 {
@@ -161,6 +203,7 @@ static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 		man->default_caching = TTM_PL_FLAG_CACHED;
 		break;
 	case TTM_PL_TT:
+		/* GTT memory  */
 		man->func = &amdgpu_gtt_mgr_func;
 		man->gpu_offset = adev->gmc.gart_start;
 		man->available_caching = TTM_PL_MASK_CACHING;
@@ -193,6 +236,14 @@ static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 	return 0;
 }
 
+/**
+ * amdgpu_evict_flags - Compute placement flags
+ *
+ * @bo: The buffer object to evict
+ * @placement: Possible destination(s) for evicted BO
+ *
+ * Fill in placement data when ttm_bo_evict() is called
+ */
 static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 				struct ttm_placement *placement)
 {
@@ -204,12 +255,14 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 		.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
 	};
 
+	/* Don't handle scatter gather BOs */
 	if (bo->type == ttm_bo_type_sg) {
 		placement->num_placement = 0;
 		placement->num_busy_placement = 0;
 		return;
 	}
 
+	/* Object isn't an AMDGPU object so ignore */
 	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) {
 		placement->placement = &placements;
 		placement->busy_placement = &placements;
@@ -217,26 +270,16 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 		placement->num_busy_placement = 1;
 		return;
 	}
+
 	abo = ttm_to_amdgpu_bo(bo);
 	switch (bo->mem.mem_type) {
 	case TTM_PL_VRAM:
 		if (!adev->mman.buffer_funcs_enabled) {
+			/* Move to system memory */
 			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
 		} else if (adev->gmc.visible_vram_size < adev->gmc.real_vram_size &&
-			   !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) {
-			unsigned fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
-			struct drm_mm_node *node = bo->mem.mm_node;
-			unsigned long pages_left;
-
-			for (pages_left = bo->mem.num_pages;
-			     pages_left;
-			     pages_left -= node->size, node++) {
-				if (node->start < fpfn)
-					break;
-			}
-
-			if (!pages_left)
-				goto gtt;
+			   !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
+			   amdgpu_bo_in_cpu_visible_vram(abo)) {
 
 			/* Try evicting to the CPU inaccessible part of VRAM
 			 * first, but only set GTT as busy placement, so this
@@ -245,12 +288,12 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 			 */
 			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
 							 AMDGPU_GEM_DOMAIN_GTT);
-			abo->placements[0].fpfn = fpfn;
+			abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
 			abo->placements[0].lpfn = 0;
 			abo->placement.busy_placement = &abo->placements[1];
 			abo->placement.num_busy_placement = 1;
 		} else {
-gtt:
+			/* Move to GTT memory */
 			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
 		}
 		break;
@@ -261,6 +304,15 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 	*placement = abo->placement;
 }
 
+/**
+ * amdgpu_verify_access - Verify access for a mmap call
+ *
+ * @bo:		The buffer object to map
+ * @filp:	The file pointer from the process performing the mmap
+ *
+ * This is called by ttm_bo_mmap() to verify whether a process
+ * has the right to mmap a BO to their process space.
+ */
 static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 {
 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
@@ -278,6 +330,15 @@ static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 					  filp->private_data);
 }
 
+/**
+ * amdgpu_move_null - Register memory for a buffer object
+ *
+ * @bo:			The bo to assign the memory to
+ * @new_mem:	The memory to be assigned.
+ *
+ * Assign the memory from new_mem to the memory of the buffer object
+ * bo.
+ */
 static void amdgpu_move_null(struct ttm_buffer_object *bo,
 			     struct ttm_mem_reg *new_mem)
 {
@@ -288,6 +349,10 @@ static void amdgpu_move_null(struct ttm_buffer_object *bo,
 	new_mem->mm_node = NULL;
 }
 
+/**
+ * amdgpu_mm_node_addr -	Compute the GPU relative offset of a GTT
+ * 							buffer.
+ */
 static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
 				    struct drm_mm_node *mm_node,
 				    struct ttm_mem_reg *mem)
@@ -302,9 +367,10 @@ static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
 }
 
 /**
- * amdgpu_find_mm_node - Helper function finds the drm_mm_node
- *  corresponding to @offset. It also modifies the offset to be
- *  within the drm_mm_node returned
+ * amdgpu_find_mm_node -	Helper function finds the drm_mm_node
+ *  						corresponding to @offset. It also modifies
+ * 							the offset to be within the drm_mm_node
+ * 							returned
  */
 static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
 					       unsigned long *offset)
@@ -443,7 +509,12 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 	return r;
 }
 
-
+/**
+ * amdgpu_move_blit - Copy an entire buffer to another buffer
+ *
+ * This is a helper called by amdgpu_bo_move() and
+ * amdgpu_move_vram_ram() to help move buffers to and from VRAM.
+ */
 static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 			    bool evict, bool no_wait_gpu,
 			    struct ttm_mem_reg *new_mem,
@@ -478,6 +549,11 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 	return r;
 }
 
+/**
+ * amdgpu_move_vram_ram - Copy VRAM buffer to RAM buffer
+ *
+ * Called by amdgpu_bo_move().
+ */
 static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
 				struct ttm_operation_ctx *ctx,
 				struct ttm_mem_reg *new_mem)
@@ -490,6 +566,8 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
 	int r;
 
 	adev = amdgpu_ttm_adev(bo->bdev);
+
+	/* create space/pages for new_mem in GTT space */
 	tmp_mem = *new_mem;
 	tmp_mem.mm_node = NULL;
 	placement.num_placement = 1;
@@ -504,25 +582,36 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
 		return r;
 	}
 
+	/* set caching flags */
 	r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
 
+	/* Bind the memory to the GTT space */
 	r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
+
+	/* blit VRAM to GTT */
 	r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, &tmp_mem, old_mem);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
+
+	/* move BO (in tmp_mem) to new_mem */
 	r = ttm_bo_move_ttm(bo, ctx, new_mem);
 out_cleanup:
 	ttm_bo_mem_put(bo, &tmp_mem);
 	return r;
 }
 
+/**
+ * amdgpu_move_ram_vram - Copy buffer from RAM to VRAM
+ *
+ * Called by amdgpu_bo_move().
+ */
 static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
 				struct ttm_operation_ctx *ctx,
 				struct ttm_mem_reg *new_mem)
@@ -535,6 +624,8 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
 	int r;
 
 	adev = amdgpu_ttm_adev(bo->bdev);
+
+	/* make space in GTT for old_mem buffer */
 	tmp_mem = *new_mem;
 	tmp_mem.mm_node = NULL;
 	placement.num_placement = 1;
@@ -548,10 +639,14 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
 	if (unlikely(r)) {
 		return r;
 	}
+
+	/* move/bind old memory to GTT space */
 	r = ttm_bo_move_ttm(bo, ctx, &tmp_mem);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
+
+	/* copy to VRAM */
 	r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, new_mem, old_mem);
 	if (unlikely(r)) {
 		goto out_cleanup;
@@ -561,6 +656,11 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
 	return r;
 }
 
+/**
+ * amdgpu_bo_move - Move a buffer object to a new memory location
+ *
+ * Called by ttm_bo_handle_move_mem()
+ */
 static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 			  struct ttm_operation_ctx *ctx,
 			  struct ttm_mem_reg *new_mem)
@@ -626,6 +726,11 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 	return 0;
 }
 
+/**
+ * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
+ *
+ * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
+ */
 static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 {
 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
@@ -695,7 +800,7 @@ struct amdgpu_ttm_tt {
 	struct ttm_dma_tt	ttm;
 	u64			offset;
 	uint64_t		userptr;
-	struct mm_struct	*usermm;
+	struct task_struct	*usertask;
 	uint32_t		userflags;
 	spinlock_t              guptasklock;
 	struct list_head        guptasks;
@@ -703,17 +808,29 @@ struct amdgpu_ttm_tt {
 	uint32_t		last_set_pages;
 };
 
+/**
+ * amdgpu_ttm_tt_get_user_pages - 	Pin pages of memory pointed to
+ * 									by a USERPTR pointer to memory
+ *
+ * Called by amdgpu_gem_userptr_ioctl() and amdgpu_cs_parser_bos().
+ * This provides a wrapper around the get_user_pages() call to provide
+ * device accessible pages that back user memory.
+ */
 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
 {
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
+	struct mm_struct *mm = gtt->usertask->mm;
 	unsigned int flags = 0;
 	unsigned pinned = 0;
 	int r;
 
+	if (!mm) /* Happens during process shutdown */
+		return -ESRCH;
+
 	if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
 		flags |= FOLL_WRITE;
 
-	down_read(&current->mm->mmap_sem);
+	down_read(&mm->mmap_sem);
 
 	if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
 		/* check that we only use anonymous memory
@@ -721,13 +838,14 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
 		unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
 		struct vm_area_struct *vma;
 
-		vma = find_vma(gtt->usermm, gtt->userptr);
+		vma = find_vma(mm, gtt->userptr);
 		if (!vma || vma->vm_file || vma->vm_end < end) {
-			up_read(&current->mm->mmap_sem);
+			up_read(&mm->mmap_sem);
 			return -EPERM;
 		}
 	}
 
+	/* loop enough times using contiguous pages of memory */
 	do {
 		unsigned num_pages = ttm->num_pages - pinned;
 		uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
@@ -739,7 +857,12 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
 		list_add(&guptask.list, &gtt->guptasks);
 		spin_unlock(&gtt->guptasklock);
 
-		r = get_user_pages(userptr, num_pages, flags, p, NULL);
+		if (mm == current->mm)
+			r = get_user_pages(userptr, num_pages, flags, p, NULL);
+		else
+			r = get_user_pages_remote(gtt->usertask,
+					mm, userptr, num_pages,
+					flags, p, NULL, NULL);
 
 		spin_lock(&gtt->guptasklock);
 		list_del(&guptask.list);
@@ -752,15 +875,23 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
 
 	} while (pinned < ttm->num_pages);
 
-	up_read(&current->mm->mmap_sem);
+	up_read(&mm->mmap_sem);
 	return 0;
 
 release_pages:
 	release_pages(pages, pinned);
-	up_read(&current->mm->mmap_sem);
+	up_read(&mm->mmap_sem);
 	return r;
 }
 
+/**
+ * amdgpu_ttm_tt_set_user_pages - 	Copy pages in, putting old pages
+ * 									as necessary.
+ *
+ * Called by amdgpu_cs_list_validate().  This creates the page list
+ * that backs user memory and will ultimately be mapped into the device
+ * address space.
+ */
 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
 {
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
@@ -775,6 +906,11 @@ void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
 	}
 }
 
+/**
+ * amdgpu_ttm_tt_mark_user_page - Mark pages as dirty
+ *
+ * Called while unpinning userptr pages
+ */
 void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm)
 {
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
@@ -793,7 +929,12 @@ void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm)
 	}
 }
 
-/* prepare the sg table with the user pages */
+/**
+ * amdgpu_ttm_tt_pin_userptr - 	prepare the sg table with the
+ * 								user pages
+ *
+ * Called by amdgpu_ttm_backend_bind()
+ **/
 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
@@ -805,17 +946,20 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
 	enum dma_data_direction direction = write ?
 		DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 
+	/* Allocate an SG array and squash pages into it */
 	r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
 				      ttm->num_pages << PAGE_SHIFT,
 				      GFP_KERNEL);
 	if (r)
 		goto release_sg;
 
+	/* Map SG to device */
 	r = -ENOMEM;
 	nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
 	if (nents != ttm->sg->nents)
 		goto release_sg;
 
+	/* convert SG to linear array of pages and dma addresses */
 	drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
 					 gtt->ttm.dma_address, ttm->num_pages);
 
@@ -826,6 +970,9 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
 	return r;
 }
 
+/**
+ * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
+ */
 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
@@ -839,14 +986,60 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
 	if (!ttm->sg->sgl)
 		return;
 
-	/* free the sg table and pages again */
+	/* unmap the pages mapped to the device */
 	dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
 
+	/* mark the pages as dirty */
 	amdgpu_ttm_tt_mark_user_pages(ttm);
 
 	sg_free_table(ttm->sg);
 }
 
+int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
+				struct ttm_buffer_object *tbo,
+				uint64_t flags)
+{
+	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
+	struct ttm_tt *ttm = tbo->ttm;
+	struct amdgpu_ttm_tt *gtt = (void *)ttm;
+	int r;
+
+	if (abo->flags & AMDGPU_GEM_CREATE_MQD_GFX9) {
+		uint64_t page_idx = 1;
+
+		r = amdgpu_gart_bind(adev, gtt->offset, page_idx,
+				ttm->pages, gtt->ttm.dma_address, flags);
+		if (r)
+			goto gart_bind_fail;
+
+		/* Patch mtype of the second part BO */
+		flags &=  ~AMDGPU_PTE_MTYPE_MASK;
+		flags |= AMDGPU_PTE_MTYPE(AMDGPU_MTYPE_NC);
+
+		r = amdgpu_gart_bind(adev,
+				gtt->offset + (page_idx << PAGE_SHIFT),
+				ttm->num_pages - page_idx,
+				&ttm->pages[page_idx],
+				&(gtt->ttm.dma_address[page_idx]), flags);
+	} else {
+		r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
+				     ttm->pages, gtt->ttm.dma_address, flags);
+	}
+
+gart_bind_fail:
+	if (r)
+		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
+			  ttm->num_pages, gtt->offset);
+
+	return r;
+}
+
+/**
+ * amdgpu_ttm_backend_bind - Bind GTT memory
+ *
+ * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
+ * This handles binding GTT memory to the device address space.
+ */
 static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
 				   struct ttm_mem_reg *bo_mem)
 {
@@ -877,7 +1070,10 @@ static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
 		return 0;
 	}
 
+	/* compute PTE flags relevant to this BO memory */
 	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);
+
+	/* bind pages into GART page tables */
 	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
 	r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
 		ttm->pages, gtt->ttm.dma_address, flags);
@@ -888,6 +1084,9 @@ static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
 	return r;
 }
 
+/**
+ * amdgpu_ttm_alloc_gart - Allocate GART memory for buffer object
+ */
 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
@@ -903,6 +1102,7 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
 	    amdgpu_gtt_mgr_has_gart_addr(&bo->mem))
 		return 0;
 
+	/* allocate GTT space */
 	tmp = bo->mem;
 	tmp.mm_node = NULL;
 	placement.num_placement = 1;
@@ -918,10 +1118,12 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
 	if (unlikely(r))
 		return r;
 
+	/* compute PTE flags for this buffer object */
 	flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
+
+	/* Bind pages */
 	gtt->offset = (u64)tmp.start << PAGE_SHIFT;
-	r = amdgpu_gart_bind(adev, gtt->offset, bo->ttm->num_pages,
-			     bo->ttm->pages, gtt->ttm.dma_address, flags);
+	r = amdgpu_ttm_gart_bind(adev, bo, flags);
 	if (unlikely(r)) {
 		ttm_bo_mem_put(bo, &tmp);
 		return r;
@@ -935,31 +1137,40 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
 	return 0;
 }
 
+/**
+ * amdgpu_ttm_recover_gart - Rebind GTT pages
+ *
+ * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
+ * rebind GTT pages during a GPU reset.
+ */
 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
-	struct amdgpu_ttm_tt *gtt = (void *)tbo->ttm;
 	uint64_t flags;
 	int r;
 
-	if (!gtt)
+	if (!tbo->ttm)
 		return 0;
 
-	flags = amdgpu_ttm_tt_pte_flags(adev, &gtt->ttm.ttm, &tbo->mem);
-	r = amdgpu_gart_bind(adev, gtt->offset, gtt->ttm.ttm.num_pages,
-			     gtt->ttm.ttm.pages, gtt->ttm.dma_address, flags);
-	if (r)
-		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
-			  gtt->ttm.ttm.num_pages, gtt->offset);
+	flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);
+	r = amdgpu_ttm_gart_bind(adev, tbo, flags);
+
 	return r;
 }
 
+/**
+ * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
+ *
+ * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
+ * ttm_tt_destroy().
+ */
 static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 	int r;
 
+	/* if the pages have userptr pinning then clear that first */
 	if (gtt->userptr)
 		amdgpu_ttm_tt_unpin_userptr(ttm);
 
@@ -978,6 +1189,9 @@ static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)
 {
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 
+	if (gtt->usertask)
+		put_task_struct(gtt->usertask);
+
 	ttm_dma_tt_fini(&gtt->ttm);
 	kfree(gtt);
 }
@@ -988,6 +1202,13 @@ static struct ttm_backend_func amdgpu_backend_func = {
 	.destroy = &amdgpu_ttm_backend_destroy,
 };
 
+/**
+ * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
+ *
+ * @bo: The buffer object to create a GTT ttm_tt object around
+ *
+ * Called by ttm_tt_create().
+ */
 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
 					   uint32_t page_flags)
 {
@@ -1001,6 +1222,8 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
 		return NULL;
 	}
 	gtt->ttm.ttm.func = &amdgpu_backend_func;
+
+	/* allocate space for the uninitialized page entries */
 	if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {
 		kfree(gtt);
 		return NULL;
@@ -1008,6 +1231,12 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
 	return &gtt->ttm.ttm;
 }
 
+/**
+ * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
+ *
+ * Map the pages of a ttm_tt object to an address space visible
+ * to the underlying device.
+ */
 static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,
 			struct ttm_operation_ctx *ctx)
 {
@@ -1015,6 +1244,7 @@ static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
 
+	/* user pages are bound by amdgpu_ttm_tt_pin_userptr() */
 	if (gtt && gtt->userptr) {
 		ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
 		if (!ttm->sg)
@@ -1039,9 +1269,17 @@ static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,
 	}
 #endif
 
+	/* fall back to generic helper to populate the page array
+	 * and map them to the device */
 	return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);
 }
 
+/**
+ * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
+ *
+ * Unmaps pages of a ttm_tt object from the device address space and
+ * unpopulates the page array backing it.
+ */
 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
 {
 	struct amdgpu_device *adev;
@@ -1067,9 +1305,21 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
 	}
 #endif
 
+	/* fall back to generic helper to unmap and unpopulate array */
 	ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);
 }
 
+/**
+ * amdgpu_ttm_tt_set_userptr -	Initialize userptr GTT ttm_tt
+ * 								for the current task
+ *
+ * @ttm: The ttm_tt object to bind this userptr object to
+ * @addr:  The address in the current tasks VM space to use
+ * @flags: Requirements of userptr object.
+ *
+ * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages
+ * to current task
+ */
 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
 			      uint32_t flags)
 {
@@ -1079,8 +1329,13 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
 		return -EINVAL;
 
 	gtt->userptr = addr;
-	gtt->usermm = current->mm;
 	gtt->userflags = flags;
+
+	if (gtt->usertask)
+		put_task_struct(gtt->usertask);
+	gtt->usertask = current->group_leader;
+	get_task_struct(gtt->usertask);
+
 	spin_lock_init(&gtt->guptasklock);
 	INIT_LIST_HEAD(&gtt->guptasks);
 	atomic_set(&gtt->mmu_invalidations, 0);
@@ -1089,6 +1344,9 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
 	return 0;
 }
 
+/**
+ * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
+ */
 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
 {
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
@@ -1096,9 +1354,18 @@ struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
 	if (gtt == NULL)
 		return NULL;
 
-	return gtt->usermm;
+	if (gtt->usertask == NULL)
+		return NULL;
+
+	return gtt->usertask->mm;
 }
 
+/**
+ * amdgpu_ttm_tt_affect_userptr -	Determine if a ttm_tt object lays
+ * 									inside an address range for the
+ * 									current task.
+ *
+ */
 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
 				  unsigned long end)
 {
@@ -1109,10 +1376,16 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
 	if (gtt == NULL || !gtt->userptr)
 		return false;
 
+	/* Return false if no part of the ttm_tt object lies within
+	 * the range
+	 */
 	size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;
 	if (gtt->userptr > end || gtt->userptr + size <= start)
 		return false;
 
+	/* Search the lists of tasks that hold this mapping and see
+	 * if current is one of them.  If it is return false.
+	 */
 	spin_lock(&gtt->guptasklock);
 	list_for_each_entry(entry, &gtt->guptasks, list) {
 		if (entry->task == current) {
@@ -1127,6 +1400,10 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
 	return true;
 }
 
+/**
+ * amdgpu_ttm_tt_userptr_invalidated -	Has the ttm_tt object been
+ * 										invalidated?
+ */
 bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
 				       int *last_invalidated)
 {
@@ -1137,6 +1414,12 @@ bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
 	return prev_invalidated != *last_invalidated;
 }
 
+/**
+ * amdgpu_ttm_tt_userptr_needs_pages -	Have the pages backing this
+ * 										ttm_tt object been invalidated
+ * 										since the last time they've
+ * 										been set?
+ */
 bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
 {
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
@@ -1147,6 +1430,9 @@ bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
 	return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;
 }
 
+/**
+ * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
+ */
 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
 {
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
@@ -1157,6 +1443,12 @@ bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
 	return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
 }
 
+/**
+ * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
+ *
+ * @ttm: The ttm_tt object to compute the flags for
+ * @mem: The memory registry backing this ttm_tt object
+ */
 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
 				 struct ttm_mem_reg *mem)
 {
@@ -1181,6 +1473,16 @@ uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
 	return flags;
 }
 
+/**
+ * amdgpu_ttm_bo_eviction_valuable -	Check to see if we can evict
+ * 										a buffer object.
+ *
+ * Return true if eviction is sensible.  Called by
+ * ttm_mem_evict_first() on behalf of ttm_bo_mem_force_space()
+ * which tries to evict buffer objects until it can find space
+ * for a new object and by ttm_bo_force_list_clean() which is
+ * used to clean out a memory space.
+ */
 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 					    const struct ttm_place *place)
 {
@@ -1227,6 +1529,19 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 	return ttm_bo_eviction_valuable(bo, place);
 }
 
+/**
+ * amdgpu_ttm_access_memory -	Read or Write memory that backs a
+ * 								buffer object.
+ *
+ * @bo:  The buffer object to read/write
+ * @offset:  Offset into buffer object
+ * @buf:  Secondary buffer to write/read from
+ * @len: Length in bytes of access
+ * @write:  true if writing
+ *
+ * This is used to access VRAM that backs a buffer object via MMIO
+ * access for debugging purposes.
+ */
 static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
 				    unsigned long offset,
 				    void *buf, int len, int write)
@@ -1329,6 +1644,7 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
 {
 	struct ttm_operation_ctx ctx = { false, false };
+	struct amdgpu_bo_param bp;
 	int r = 0;
 	int i;
 	u64 vram_size = adev->gmc.visible_vram_size;
@@ -1336,17 +1652,21 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
 	u64 size = adev->fw_vram_usage.size;
 	struct amdgpu_bo *bo;
 
+	memset(&bp, 0, sizeof(bp));
+	bp.size = adev->fw_vram_usage.size;
+	bp.byte_align = PAGE_SIZE;
+	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
+	bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+		AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+	bp.type = ttm_bo_type_kernel;
+	bp.resv = NULL;
 	adev->fw_vram_usage.va = NULL;
 	adev->fw_vram_usage.reserved_bo = NULL;
 
 	if (adev->fw_vram_usage.size > 0 &&
 		adev->fw_vram_usage.size <= vram_size) {
 
-		r = amdgpu_bo_create(adev, adev->fw_vram_usage.size, PAGE_SIZE,
-				     AMDGPU_GEM_DOMAIN_VRAM,
-				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-				     ttm_bo_type_kernel, NULL,
+		r = amdgpu_bo_create(adev, &bp,
 				     &adev->fw_vram_usage.reserved_bo);
 		if (r)
 			goto error_create;
@@ -1398,13 +1718,22 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
 	adev->fw_vram_usage.reserved_bo = NULL;
 	return r;
 }
-
+/**
+ * amdgpu_ttm_init -	Init the memory management (ttm) as well as
+ * 						various gtt/vram related fields.
+ *
+ * This initializes all of the memory space pools that the TTM layer
+ * will need such as the GTT space (system memory mapped to the device),
+ * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
+ * can be mapped per VMID.
+ */
 int amdgpu_ttm_init(struct amdgpu_device *adev)
 {
 	uint64_t gtt_size;
 	int r;
 	u64 vis_vram_limit;
 
+	/* initialize global references for vram/gtt */
 	r = amdgpu_ttm_global_init(adev);
 	if (r) {
 		return r;
@@ -1425,6 +1754,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	/* We opt to avoid OOM on system pages allocations */
 	adev->mman.bdev.no_retry = true;
 
+	/* Initialize VRAM pool with all of VRAM divided into pages */
 	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
 				adev->gmc.real_vram_size >> PAGE_SHIFT);
 	if (r) {
@@ -1454,15 +1784,23 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 		return r;
 	}
 
-	r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE,
-				    AMDGPU_GEM_DOMAIN_VRAM,
-				    &adev->stolen_vga_memory,
-				    NULL, NULL);
-	if (r)
-		return r;
+	/* allocate memory as required for VGA
+	 * This is used for VGA emulation and pre-OS scanout buffers to
+	 * avoid display artifacts while transitioning between pre-OS
+	 * and driver.  */
+	if (adev->gmc.stolen_size) {
+		r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE,
+					    AMDGPU_GEM_DOMAIN_VRAM,
+					    &adev->stolen_vga_memory,
+					    NULL, NULL);
+		if (r)
+			return r;
+	}
 	DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
 		 (unsigned) (adev->gmc.real_vram_size / (1024 * 1024)));
 
+	/* Compute GTT size, either bsaed on 3/4th the size of RAM size
+	 * or whatever the user passed on module init */
 	if (amdgpu_gtt_size == -1) {
 		struct sysinfo si;
 
@@ -1473,6 +1811,8 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	}
 	else
 		gtt_size = (uint64_t)amdgpu_gtt_size << 20;
+
+	/* Initialize GTT memory pool */
 	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
 	if (r) {
 		DRM_ERROR("Failed initializing GTT heap.\n");
@@ -1481,6 +1821,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
 		 (unsigned)(gtt_size / (1024 * 1024)));
 
+	/* Initialize various on-chip memory pools */
 	adev->gds.mem.total_size = adev->gds.mem.total_size << AMDGPU_GDS_SHIFT;
 	adev->gds.mem.gfx_partition_size = adev->gds.mem.gfx_partition_size << AMDGPU_GDS_SHIFT;
 	adev->gds.mem.cs_partition_size = adev->gds.mem.cs_partition_size << AMDGPU_GDS_SHIFT;
@@ -1520,6 +1861,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 		}
 	}
 
+	/* Register debugfs entries for amdgpu_ttm */
 	r = amdgpu_ttm_debugfs_init(adev);
 	if (r) {
 		DRM_ERROR("Failed to init debugfs\n");
@@ -1528,13 +1870,25 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	return 0;
 }
 
+/**
+ * amdgpu_ttm_late_init -	Handle any late initialization for
+ * 							amdgpu_ttm
+ */
+void amdgpu_ttm_late_init(struct amdgpu_device *adev)
+{
+	/* return the VGA stolen memory (if any) back to VRAM */
+	amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
+}
+
+/**
+ * amdgpu_ttm_fini - De-initialize the TTM memory pools
+ */
 void amdgpu_ttm_fini(struct amdgpu_device *adev)
 {
 	if (!adev->mman.initialized)
 		return;
 
 	amdgpu_ttm_debugfs_fini(adev);
-	amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
 	amdgpu_ttm_fw_reserve_vram_fini(adev);
 	if (adev->mman.aper_base_kaddr)
 		iounmap(adev->mman.aper_base_kaddr);
@@ -1856,6 +2210,11 @@ static const struct drm_info_list amdgpu_ttm_debugfs_list[] = {
 #endif
 };
 
+/**
+ * amdgpu_ttm_vram_read - Linear read access to VRAM
+ *
+ * Accesses VRAM via MMIO for debugging purposes.
+ */
 static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
 				    size_t size, loff_t *pos)
 {
@@ -1895,6 +2254,11 @@ static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
 	return result;
 }
 
+/**
+ * amdgpu_ttm_vram_write - Linear write access to VRAM
+ *
+ * Accesses VRAM via MMIO for debugging purposes.
+ */
 static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
 				    size_t size, loff_t *pos)
 {
@@ -1943,6 +2307,9 @@ static const struct file_operations amdgpu_ttm_vram_fops = {
 
 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
 
+/**
+ * amdgpu_ttm_gtt_read - Linear read access to GTT memory
+ */
 static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf,
 				   size_t size, loff_t *pos)
 {
@@ -1990,6 +2357,13 @@ static const struct file_operations amdgpu_ttm_gtt_fops = {
 
 #endif
 
+/**
+ * amdgpu_iomem_read - Virtual read access to GPU mapped memory
+ *
+ * This function is used to read memory that has been mapped to the
+ * GPU and the known addresses are not physical addresses but instead
+ * bus addresses (e.g., what you'd put in an IB or ring buffer).
+ */
 static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
 				 size_t size, loff_t *pos)
 {
@@ -1998,6 +2372,7 @@ static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
 	ssize_t result = 0;
 	int r;
 
+	/* retrieve the IOMMU domain if any for this device */
 	dom = iommu_get_domain_for_dev(adev->dev);
 
 	while (size) {
@@ -2010,6 +2385,10 @@ static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
 
 		bytes = bytes < size ? bytes : size;
 
+		/* Translate the bus address to a physical address.  If
+		 * the domain is NULL it means there is no IOMMU active
+		 * and the address translation is the identity
+		 */
 		addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
 
 		pfn = addr >> PAGE_SHIFT;
@@ -2034,6 +2413,13 @@ static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
 	return result;
 }
 
+/**
+ * amdgpu_iomem_write - Virtual write access to GPU mapped memory
+ *
+ * This function is used to write memory that has been mapped to the
+ * GPU and the known addresses are not physical addresses but instead
+ * bus addresses (e.g., what you'd put in an IB or ring buffer).
+ */
 static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
 				 size_t size, loff_t *pos)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 6ea7de863041..e969c879d87e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -77,6 +77,7 @@ uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man);
 uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man);
 
 int amdgpu_ttm_init(struct amdgpu_device *adev);
+void amdgpu_ttm_late_init(struct amdgpu_device *adev);
 void amdgpu_ttm_fini(struct amdgpu_device *adev);
 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev,
 					bool enable);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 5916cc25e28b..f55f72a37ca8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -161,8 +161,38 @@ void amdgpu_ucode_print_rlc_hdr(const struct common_firmware_header *hdr)
 			  le32_to_cpu(rlc_hdr->reg_list_format_separate_array_offset_bytes));
 		DRM_DEBUG("reg_list_separate_size_bytes: %u\n",
 			  le32_to_cpu(rlc_hdr->reg_list_separate_size_bytes));
-		DRM_DEBUG("reg_list_separate_size_bytes: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_list_separate_size_bytes));
+		DRM_DEBUG("reg_list_separate_array_offset_bytes: %u\n",
+			  le32_to_cpu(rlc_hdr->reg_list_separate_array_offset_bytes));
+		if (version_minor == 1) {
+			const struct rlc_firmware_header_v2_1 *v2_1 =
+				container_of(rlc_hdr, struct rlc_firmware_header_v2_1, v2_0);
+			DRM_DEBUG("reg_list_format_direct_reg_list_length: %u\n",
+				  le32_to_cpu(v2_1->reg_list_format_direct_reg_list_length));
+			DRM_DEBUG("save_restore_list_cntl_ucode_ver: %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_cntl_ucode_ver));
+			DRM_DEBUG("save_restore_list_cntl_feature_ver: %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_cntl_feature_ver));
+			DRM_DEBUG("save_restore_list_cntl_size_bytes %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_cntl_size_bytes));
+			DRM_DEBUG("save_restore_list_cntl_offset_bytes: %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_cntl_offset_bytes));
+			DRM_DEBUG("save_restore_list_gpm_ucode_ver: %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_gpm_ucode_ver));
+			DRM_DEBUG("save_restore_list_gpm_feature_ver: %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_gpm_feature_ver));
+			DRM_DEBUG("save_restore_list_gpm_size_bytes %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_gpm_size_bytes));
+			DRM_DEBUG("save_restore_list_gpm_offset_bytes: %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_gpm_offset_bytes));
+			DRM_DEBUG("save_restore_list_srm_ucode_ver: %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_srm_ucode_ver));
+			DRM_DEBUG("save_restore_list_srm_feature_ver: %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_srm_feature_ver));
+			DRM_DEBUG("save_restore_list_srm_size_bytes %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_srm_size_bytes));
+			DRM_DEBUG("save_restore_list_srm_offset_bytes: %u\n",
+				  le32_to_cpu(v2_1->save_restore_list_srm_offset_bytes));
+		}
 	} else {
 		DRM_ERROR("Unknown RLC ucode version: %u.%u\n", version_major, version_minor);
 	}
@@ -265,6 +295,7 @@ amdgpu_ucode_get_load_type(struct amdgpu_device *adev, int load_type)
 	case CHIP_POLARIS10:
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 		if (!load_type)
 			return AMDGPU_FW_LOAD_DIRECT;
 		else
@@ -276,6 +307,8 @@ amdgpu_ucode_get_load_type(struct amdgpu_device *adev, int load_type)
 			return AMDGPU_FW_LOAD_DIRECT;
 		else
 			return AMDGPU_FW_LOAD_PSP;
+	case CHIP_VEGA20:
+		return AMDGPU_FW_LOAD_DIRECT;
 	default:
 		DRM_ERROR("Unknown firmware load type\n");
 	}
@@ -307,7 +340,10 @@ static int amdgpu_ucode_init_single_fw(struct amdgpu_device *adev,
 	    (ucode->ucode_id != AMDGPU_UCODE_ID_CP_MEC1 &&
 	     ucode->ucode_id != AMDGPU_UCODE_ID_CP_MEC2 &&
 	     ucode->ucode_id != AMDGPU_UCODE_ID_CP_MEC1_JT &&
-	     ucode->ucode_id != AMDGPU_UCODE_ID_CP_MEC2_JT)) {
+	     ucode->ucode_id != AMDGPU_UCODE_ID_CP_MEC2_JT &&
+	     ucode->ucode_id != AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL &&
+	     ucode->ucode_id != AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM &&
+	     ucode->ucode_id != AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM)) {
 		ucode->ucode_size = le32_to_cpu(header->ucode_size_bytes);
 
 		memcpy(ucode->kaddr, (void *)((uint8_t *)ucode->fw->data +
@@ -329,6 +365,18 @@ static int amdgpu_ucode_init_single_fw(struct amdgpu_device *adev,
 					      le32_to_cpu(header->ucode_array_offset_bytes) +
 					      le32_to_cpu(cp_hdr->jt_offset) * 4),
 		       ucode->ucode_size);
+	} else if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL) {
+		ucode->ucode_size = adev->gfx.rlc.save_restore_list_cntl_size_bytes;
+		memcpy(ucode->kaddr, adev->gfx.rlc.save_restore_list_cntl,
+		       ucode->ucode_size);
+	} else if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM) {
+		ucode->ucode_size = adev->gfx.rlc.save_restore_list_gpm_size_bytes;
+		memcpy(ucode->kaddr, adev->gfx.rlc.save_restore_list_gpm,
+		       ucode->ucode_size);
+	} else if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM) {
+		ucode->ucode_size = adev->gfx.rlc.save_restore_list_srm_size_bytes;
+		memcpy(ucode->kaddr, adev->gfx.rlc.save_restore_list_srm,
+		       ucode->ucode_size);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
index 30b5500dc152..08e38579af24 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
@@ -98,6 +98,24 @@ struct rlc_firmware_header_v2_0 {
 	uint32_t reg_list_separate_array_offset_bytes; /* payload offset from the start of the header */
 };
 
+/* version_major=2, version_minor=1 */
+struct rlc_firmware_header_v2_1 {
+	struct rlc_firmware_header_v2_0 v2_0;
+	uint32_t reg_list_format_direct_reg_list_length; /* length of direct reg list format array */
+	uint32_t save_restore_list_cntl_ucode_ver;
+	uint32_t save_restore_list_cntl_feature_ver;
+	uint32_t save_restore_list_cntl_size_bytes;
+	uint32_t save_restore_list_cntl_offset_bytes;
+	uint32_t save_restore_list_gpm_ucode_ver;
+	uint32_t save_restore_list_gpm_feature_ver;
+	uint32_t save_restore_list_gpm_size_bytes;
+	uint32_t save_restore_list_gpm_offset_bytes;
+	uint32_t save_restore_list_srm_ucode_ver;
+	uint32_t save_restore_list_srm_feature_ver;
+	uint32_t save_restore_list_srm_size_bytes;
+	uint32_t save_restore_list_srm_offset_bytes;
+};
+
 /* version_major=1, version_minor=0 */
 struct sdma_firmware_header_v1_0 {
 	struct common_firmware_header header;
@@ -148,6 +166,7 @@ union amdgpu_firmware_header {
 	struct gfx_firmware_header_v1_0 gfx;
 	struct rlc_firmware_header_v1_0 rlc;
 	struct rlc_firmware_header_v2_0 rlc_v2_0;
+	struct rlc_firmware_header_v2_1 rlc_v2_1;
 	struct sdma_firmware_header_v1_0 sdma;
 	struct sdma_firmware_header_v1_1 sdma_v1_1;
 	struct gpu_info_firmware_header_v1_0 gpu_info;
@@ -168,6 +187,9 @@ enum AMDGPU_UCODE_ID {
 	AMDGPU_UCODE_ID_CP_MEC2,
 	AMDGPU_UCODE_ID_CP_MEC2_JT,
 	AMDGPU_UCODE_ID_RLC_G,
+	AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL,
+	AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM,
+	AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM,
 	AMDGPU_UCODE_ID_STORAGE,
 	AMDGPU_UCODE_ID_SMC,
 	AMDGPU_UCODE_ID_UVD,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index 627542b22ae4..bcf68f80bbf0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -66,15 +66,18 @@
 #define FIRMWARE_POLARIS10	"amdgpu/polaris10_uvd.bin"
 #define FIRMWARE_POLARIS11	"amdgpu/polaris11_uvd.bin"
 #define FIRMWARE_POLARIS12	"amdgpu/polaris12_uvd.bin"
+#define FIRMWARE_VEGAM		"amdgpu/vegam_uvd.bin"
 
 #define FIRMWARE_VEGA10		"amdgpu/vega10_uvd.bin"
 #define FIRMWARE_VEGA12		"amdgpu/vega12_uvd.bin"
+#define FIRMWARE_VEGA20		"amdgpu/vega20_uvd.bin"
 
-#define mmUVD_GPCOM_VCPU_DATA0_VEGA10 (0x03c4 + 0x7e00)
-#define mmUVD_GPCOM_VCPU_DATA1_VEGA10 (0x03c5 + 0x7e00)
-#define mmUVD_GPCOM_VCPU_CMD_VEGA10 (0x03c3 + 0x7e00)
-#define mmUVD_NO_OP_VEGA10 (0x03ff + 0x7e00)
-#define mmUVD_ENGINE_CNTL_VEGA10 (0x03c6 + 0x7e00)
+/* These are common relative offsets for all asics, from uvd_7_0_offset.h,  */
+#define UVD_GPCOM_VCPU_CMD		0x03c3
+#define UVD_GPCOM_VCPU_DATA0	0x03c4
+#define UVD_GPCOM_VCPU_DATA1	0x03c5
+#define UVD_NO_OP				0x03ff
+#define UVD_BASE_SI				0x3800
 
 /**
  * amdgpu_uvd_cs_ctx - Command submission parser context
@@ -109,9 +112,11 @@ MODULE_FIRMWARE(FIRMWARE_STONEY);
 MODULE_FIRMWARE(FIRMWARE_POLARIS10);
 MODULE_FIRMWARE(FIRMWARE_POLARIS11);
 MODULE_FIRMWARE(FIRMWARE_POLARIS12);
+MODULE_FIRMWARE(FIRMWARE_VEGAM);
 
 MODULE_FIRMWARE(FIRMWARE_VEGA10);
 MODULE_FIRMWARE(FIRMWARE_VEGA12);
+MODULE_FIRMWARE(FIRMWARE_VEGA20);
 
 static void amdgpu_uvd_idle_work_handler(struct work_struct *work);
 
@@ -123,9 +128,9 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 	const char *fw_name;
 	const struct common_firmware_header *hdr;
 	unsigned version_major, version_minor, family_id;
-	int i, r;
+	int i, j, r;
 
-	INIT_DELAYED_WORK(&adev->uvd.idle_work, amdgpu_uvd_idle_work_handler);
+	INIT_DELAYED_WORK(&adev->uvd.inst->idle_work, amdgpu_uvd_idle_work_handler);
 
 	switch (adev->asic_type) {
 #ifdef CONFIG_DRM_AMDGPU_CIK
@@ -172,6 +177,12 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 	case CHIP_VEGA12:
 		fw_name = FIRMWARE_VEGA12;
 		break;
+	case CHIP_VEGAM:
+		fw_name = FIRMWARE_VEGAM;
+		break;
+	case CHIP_VEGA20:
+		fw_name = FIRMWARE_VEGA20;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -226,28 +237,30 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
 		bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 
-	r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE,
-				    AMDGPU_GEM_DOMAIN_VRAM, &adev->uvd.vcpu_bo,
-				    &adev->uvd.gpu_addr, &adev->uvd.cpu_addr);
-	if (r) {
-		dev_err(adev->dev, "(%d) failed to allocate UVD bo\n", r);
-		return r;
-	}
+	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
 
-	ring = &adev->uvd.ring;
-	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
-	r = drm_sched_entity_init(&ring->sched, &adev->uvd.entity,
-				  rq, amdgpu_sched_jobs, NULL);
-	if (r != 0) {
-		DRM_ERROR("Failed setting up UVD run queue.\n");
-		return r;
-	}
+		r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE,
+					    AMDGPU_GEM_DOMAIN_VRAM, &adev->uvd.inst[j].vcpu_bo,
+					    &adev->uvd.inst[j].gpu_addr, &adev->uvd.inst[j].cpu_addr);
+		if (r) {
+			dev_err(adev->dev, "(%d) failed to allocate UVD bo\n", r);
+			return r;
+		}
 
-	for (i = 0; i < adev->uvd.max_handles; ++i) {
-		atomic_set(&adev->uvd.handles[i], 0);
-		adev->uvd.filp[i] = NULL;
-	}
+		ring = &adev->uvd.inst[j].ring;
+		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+		r = drm_sched_entity_init(&ring->sched, &adev->uvd.inst[j].entity,
+					  rq, NULL);
+		if (r != 0) {
+			DRM_ERROR("Failed setting up UVD(%d) run queue.\n", j);
+			return r;
+		}
 
+		for (i = 0; i < adev->uvd.max_handles; ++i) {
+			atomic_set(&adev->uvd.inst[j].handles[i], 0);
+			adev->uvd.inst[j].filp[i] = NULL;
+		}
+	}
 	/* from uvd v5.0 HW addressing capacity increased to 64 bits */
 	if (!amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0))
 		adev->uvd.address_64_bit = true;
@@ -274,20 +287,22 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 
 int amdgpu_uvd_sw_fini(struct amdgpu_device *adev)
 {
-	int i;
-	kfree(adev->uvd.saved_bo);
+	int i, j;
 
-	drm_sched_entity_fini(&adev->uvd.ring.sched, &adev->uvd.entity);
+	for (j = 0; j < adev->uvd.num_uvd_inst; ++j) {
+		kfree(adev->uvd.inst[j].saved_bo);
 
-	amdgpu_bo_free_kernel(&adev->uvd.vcpu_bo,
-			      &adev->uvd.gpu_addr,
-			      (void **)&adev->uvd.cpu_addr);
+		drm_sched_entity_fini(&adev->uvd.inst[j].ring.sched, &adev->uvd.inst[j].entity);
 
-	amdgpu_ring_fini(&adev->uvd.ring);
+		amdgpu_bo_free_kernel(&adev->uvd.inst[j].vcpu_bo,
+				      &adev->uvd.inst[j].gpu_addr,
+				      (void **)&adev->uvd.inst[j].cpu_addr);
 
-	for (i = 0; i < AMDGPU_MAX_UVD_ENC_RINGS; ++i)
-		amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
+		amdgpu_ring_fini(&adev->uvd.inst[j].ring);
 
+		for (i = 0; i < AMDGPU_MAX_UVD_ENC_RINGS; ++i)
+			amdgpu_ring_fini(&adev->uvd.inst[j].ring_enc[i]);
+	}
 	release_firmware(adev->uvd.fw);
 
 	return 0;
@@ -297,32 +312,33 @@ int amdgpu_uvd_suspend(struct amdgpu_device *adev)
 {
 	unsigned size;
 	void *ptr;
-	int i;
-
-	if (adev->uvd.vcpu_bo == NULL)
-		return 0;
+	int i, j;
 
-	cancel_delayed_work_sync(&adev->uvd.idle_work);
+	for (j = 0; j < adev->uvd.num_uvd_inst; ++j) {
+		if (adev->uvd.inst[j].vcpu_bo == NULL)
+			continue;
 
-	/* only valid for physical mode */
-	if (adev->asic_type < CHIP_POLARIS10) {
-		for (i = 0; i < adev->uvd.max_handles; ++i)
-			if (atomic_read(&adev->uvd.handles[i]))
-				break;
+		cancel_delayed_work_sync(&adev->uvd.inst[j].idle_work);
 
-		if (i == adev->uvd.max_handles)
-			return 0;
-	}
+		/* only valid for physical mode */
+		if (adev->asic_type < CHIP_POLARIS10) {
+			for (i = 0; i < adev->uvd.max_handles; ++i)
+				if (atomic_read(&adev->uvd.inst[j].handles[i]))
+					break;
 
-	size = amdgpu_bo_size(adev->uvd.vcpu_bo);
-	ptr = adev->uvd.cpu_addr;
+			if (i == adev->uvd.max_handles)
+				continue;
+		}
 
-	adev->uvd.saved_bo = kmalloc(size, GFP_KERNEL);
-	if (!adev->uvd.saved_bo)
-		return -ENOMEM;
+		size = amdgpu_bo_size(adev->uvd.inst[j].vcpu_bo);
+		ptr = adev->uvd.inst[j].cpu_addr;
 
-	memcpy_fromio(adev->uvd.saved_bo, ptr, size);
+		adev->uvd.inst[j].saved_bo = kmalloc(size, GFP_KERNEL);
+		if (!adev->uvd.inst[j].saved_bo)
+			return -ENOMEM;
 
+		memcpy_fromio(adev->uvd.inst[j].saved_bo, ptr, size);
+	}
 	return 0;
 }
 
@@ -330,59 +346,65 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
 {
 	unsigned size;
 	void *ptr;
+	int i;
 
-	if (adev->uvd.vcpu_bo == NULL)
-		return -EINVAL;
+	for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
+		if (adev->uvd.inst[i].vcpu_bo == NULL)
+			return -EINVAL;
 
-	size = amdgpu_bo_size(adev->uvd.vcpu_bo);
-	ptr = adev->uvd.cpu_addr;
+		size = amdgpu_bo_size(adev->uvd.inst[i].vcpu_bo);
+		ptr = adev->uvd.inst[i].cpu_addr;
 
-	if (adev->uvd.saved_bo != NULL) {
-		memcpy_toio(ptr, adev->uvd.saved_bo, size);
-		kfree(adev->uvd.saved_bo);
-		adev->uvd.saved_bo = NULL;
-	} else {
-		const struct common_firmware_header *hdr;
-		unsigned offset;
-
-		hdr = (const struct common_firmware_header *)adev->uvd.fw->data;
-		if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
-			offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
-			memcpy_toio(adev->uvd.cpu_addr, adev->uvd.fw->data + offset,
-				    le32_to_cpu(hdr->ucode_size_bytes));
-			size -= le32_to_cpu(hdr->ucode_size_bytes);
-			ptr += le32_to_cpu(hdr->ucode_size_bytes);
+		if (adev->uvd.inst[i].saved_bo != NULL) {
+			memcpy_toio(ptr, adev->uvd.inst[i].saved_bo, size);
+			kfree(adev->uvd.inst[i].saved_bo);
+			adev->uvd.inst[i].saved_bo = NULL;
+		} else {
+			const struct common_firmware_header *hdr;
+			unsigned offset;
+
+			hdr = (const struct common_firmware_header *)adev->uvd.fw->data;
+			if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
+				offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
+				memcpy_toio(adev->uvd.inst[i].cpu_addr, adev->uvd.fw->data + offset,
+					    le32_to_cpu(hdr->ucode_size_bytes));
+				size -= le32_to_cpu(hdr->ucode_size_bytes);
+				ptr += le32_to_cpu(hdr->ucode_size_bytes);
+			}
+			memset_io(ptr, 0, size);
+			/* to restore uvd fence seq */
+			amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring);
 		}
-		memset_io(ptr, 0, size);
-		/* to restore uvd fence seq */
-		amdgpu_fence_driver_force_completion(&adev->uvd.ring);
 	}
-
 	return 0;
 }
 
 void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
 {
-	struct amdgpu_ring *ring = &adev->uvd.ring;
-	int i, r;
+	struct amdgpu_ring *ring;
+	int i, j, r;
 
-	for (i = 0; i < adev->uvd.max_handles; ++i) {
-		uint32_t handle = atomic_read(&adev->uvd.handles[i]);
-		if (handle != 0 && adev->uvd.filp[i] == filp) {
-			struct dma_fence *fence;
-
-			r = amdgpu_uvd_get_destroy_msg(ring, handle,
-						       false, &fence);
-			if (r) {
-				DRM_ERROR("Error destroying UVD (%d)!\n", r);
-				continue;
-			}
+	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
+		ring = &adev->uvd.inst[j].ring;
+
+		for (i = 0; i < adev->uvd.max_handles; ++i) {
+			uint32_t handle = atomic_read(&adev->uvd.inst[j].handles[i]);
+			if (handle != 0 && adev->uvd.inst[j].filp[i] == filp) {
+				struct dma_fence *fence;
+
+				r = amdgpu_uvd_get_destroy_msg(ring, handle,
+							       false, &fence);
+				if (r) {
+					DRM_ERROR("Error destroying UVD(%d) %d!\n", j, r);
+					continue;
+				}
 
-			dma_fence_wait(fence, false);
-			dma_fence_put(fence);
+				dma_fence_wait(fence, false);
+				dma_fence_put(fence);
 
-			adev->uvd.filp[i] = NULL;
-			atomic_set(&adev->uvd.handles[i], 0);
+				adev->uvd.inst[j].filp[i] = NULL;
+				atomic_set(&adev->uvd.inst[j].handles[i], 0);
+			}
 		}
 	}
 }
@@ -657,15 +679,16 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx,
 	void *ptr;
 	long r;
 	int i;
+	uint32_t ip_instance = ctx->parser->job->ring->me;
 
 	if (offset & 0x3F) {
-		DRM_ERROR("UVD messages must be 64 byte aligned!\n");
+		DRM_ERROR("UVD(%d) messages must be 64 byte aligned!\n", ip_instance);
 		return -EINVAL;
 	}
 
 	r = amdgpu_bo_kmap(bo, &ptr);
 	if (r) {
-		DRM_ERROR("Failed mapping the UVD message (%ld)!\n", r);
+		DRM_ERROR("Failed mapping the UVD(%d) message (%ld)!\n", ip_instance, r);
 		return r;
 	}
 
@@ -675,7 +698,7 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx,
 	handle = msg[2];
 
 	if (handle == 0) {
-		DRM_ERROR("Invalid UVD handle!\n");
+		DRM_ERROR("Invalid UVD(%d) handle!\n", ip_instance);
 		return -EINVAL;
 	}
 
@@ -686,18 +709,18 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx,
 
 		/* try to alloc a new handle */
 		for (i = 0; i < adev->uvd.max_handles; ++i) {
-			if (atomic_read(&adev->uvd.handles[i]) == handle) {
-				DRM_ERROR("Handle 0x%x already in use!\n", handle);
+			if (atomic_read(&adev->uvd.inst[ip_instance].handles[i]) == handle) {
+				DRM_ERROR("(%d)Handle 0x%x already in use!\n", ip_instance, handle);
 				return -EINVAL;
 			}
 
-			if (!atomic_cmpxchg(&adev->uvd.handles[i], 0, handle)) {
-				adev->uvd.filp[i] = ctx->parser->filp;
+			if (!atomic_cmpxchg(&adev->uvd.inst[ip_instance].handles[i], 0, handle)) {
+				adev->uvd.inst[ip_instance].filp[i] = ctx->parser->filp;
 				return 0;
 			}
 		}
 
-		DRM_ERROR("No more free UVD handles!\n");
+		DRM_ERROR("No more free UVD(%d) handles!\n", ip_instance);
 		return -ENOSPC;
 
 	case 1:
@@ -709,27 +732,27 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx,
 
 		/* validate the handle */
 		for (i = 0; i < adev->uvd.max_handles; ++i) {
-			if (atomic_read(&adev->uvd.handles[i]) == handle) {
-				if (adev->uvd.filp[i] != ctx->parser->filp) {
-					DRM_ERROR("UVD handle collision detected!\n");
+			if (atomic_read(&adev->uvd.inst[ip_instance].handles[i]) == handle) {
+				if (adev->uvd.inst[ip_instance].filp[i] != ctx->parser->filp) {
+					DRM_ERROR("UVD(%d) handle collision detected!\n", ip_instance);
 					return -EINVAL;
 				}
 				return 0;
 			}
 		}
 
-		DRM_ERROR("Invalid UVD handle 0x%x!\n", handle);
+		DRM_ERROR("Invalid UVD(%d) handle 0x%x!\n", ip_instance, handle);
 		return -ENOENT;
 
 	case 2:
 		/* it's a destroy msg, free the handle */
 		for (i = 0; i < adev->uvd.max_handles; ++i)
-			atomic_cmpxchg(&adev->uvd.handles[i], handle, 0);
+			atomic_cmpxchg(&adev->uvd.inst[ip_instance].handles[i], handle, 0);
 		amdgpu_bo_kunmap(bo);
 		return 0;
 
 	default:
-		DRM_ERROR("Illegal UVD message type (%d)!\n", msg_type);
+		DRM_ERROR("Illegal UVD(%d) message type (%d)!\n", ip_instance, msg_type);
 		return -EINVAL;
 	}
 	BUG();
@@ -800,7 +823,7 @@ static int amdgpu_uvd_cs_pass2(struct amdgpu_uvd_cs_ctx *ctx)
 		}
 
 		if ((cmd == 0 || cmd == 0x3) &&
-		    (start >> 28) != (ctx->parser->adev->uvd.gpu_addr >> 28)) {
+		    (start >> 28) != (ctx->parser->adev->uvd.inst->gpu_addr >> 28)) {
 			DRM_ERROR("msg/fb buffer %LX-%LX out of 256MB segment!\n",
 				  start, end);
 			return -EINVAL;
@@ -968,6 +991,8 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 	uint64_t addr;
 	long r;
 	int i;
+	unsigned offset_idx = 0;
+	unsigned offset[3] = { UVD_BASE_SI, 0, 0 };
 
 	amdgpu_bo_kunmap(bo);
 	amdgpu_bo_unpin(bo);
@@ -987,17 +1012,16 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 		goto err;
 
 	if (adev->asic_type >= CHIP_VEGA10) {
-		data[0] = PACKET0(mmUVD_GPCOM_VCPU_DATA0_VEGA10, 0);
-		data[1] = PACKET0(mmUVD_GPCOM_VCPU_DATA1_VEGA10, 0);
-		data[2] = PACKET0(mmUVD_GPCOM_VCPU_CMD_VEGA10, 0);
-		data[3] = PACKET0(mmUVD_NO_OP_VEGA10, 0);
-	} else {
-		data[0] = PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0);
-		data[1] = PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0);
-		data[2] = PACKET0(mmUVD_GPCOM_VCPU_CMD, 0);
-		data[3] = PACKET0(mmUVD_NO_OP, 0);
+		offset_idx = 1 + ring->me;
+		offset[1] = adev->reg_offset[UVD_HWIP][0][1];
+		offset[2] = adev->reg_offset[UVD_HWIP][1][1];
 	}
 
+	data[0] = PACKET0(offset[offset_idx] + UVD_GPCOM_VCPU_DATA0, 0);
+	data[1] = PACKET0(offset[offset_idx] + UVD_GPCOM_VCPU_DATA1, 0);
+	data[2] = PACKET0(offset[offset_idx] + UVD_GPCOM_VCPU_CMD, 0);
+	data[3] = PACKET0(offset[offset_idx] + UVD_NO_OP, 0);
+
 	ib = &job->ibs[0];
 	addr = amdgpu_bo_gpu_offset(bo);
 	ib->ptr[0] = data[0];
@@ -1033,7 +1057,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 		if (r)
 			goto err_free;
 
-		r = amdgpu_job_submit(job, ring, &adev->uvd.entity,
+		r = amdgpu_job_submit(job, ring, &adev->uvd.inst[ring->me].entity,
 				      AMDGPU_FENCE_OWNER_UNDEFINED, &f);
 		if (r)
 			goto err_free;
@@ -1121,8 +1145,15 @@ int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 static void amdgpu_uvd_idle_work_handler(struct work_struct *work)
 {
 	struct amdgpu_device *adev =
-		container_of(work, struct amdgpu_device, uvd.idle_work.work);
-	unsigned fences = amdgpu_fence_count_emitted(&adev->uvd.ring);
+		container_of(work, struct amdgpu_device, uvd.inst->idle_work.work);
+	unsigned fences = 0, i, j;
+
+	for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {
+		fences += amdgpu_fence_count_emitted(&adev->uvd.inst[i].ring);
+		for (j = 0; j < adev->uvd.num_enc_rings; ++j) {
+			fences += amdgpu_fence_count_emitted(&adev->uvd.inst[i].ring_enc[j]);
+		}
+	}
 
 	if (fences == 0) {
 		if (adev->pm.dpm_enabled) {
@@ -1136,7 +1167,7 @@ static void amdgpu_uvd_idle_work_handler(struct work_struct *work)
 							       AMD_CG_STATE_GATE);
 		}
 	} else {
-		schedule_delayed_work(&adev->uvd.idle_work, UVD_IDLE_TIMEOUT);
+		schedule_delayed_work(&adev->uvd.inst->idle_work, UVD_IDLE_TIMEOUT);
 	}
 }
 
@@ -1148,7 +1179,7 @@ void amdgpu_uvd_ring_begin_use(struct amdgpu_ring *ring)
 	if (amdgpu_sriov_vf(adev))
 		return;
 
-	set_clocks = !cancel_delayed_work_sync(&adev->uvd.idle_work);
+	set_clocks = !cancel_delayed_work_sync(&adev->uvd.inst->idle_work);
 	if (set_clocks) {
 		if (adev->pm.dpm_enabled) {
 			amdgpu_dpm_enable_uvd(adev, true);
@@ -1165,7 +1196,7 @@ void amdgpu_uvd_ring_begin_use(struct amdgpu_ring *ring)
 void amdgpu_uvd_ring_end_use(struct amdgpu_ring *ring)
 {
 	if (!amdgpu_sriov_vf(ring->adev))
-		schedule_delayed_work(&ring->adev->uvd.idle_work, UVD_IDLE_TIMEOUT);
+		schedule_delayed_work(&ring->adev->uvd.inst->idle_work, UVD_IDLE_TIMEOUT);
 }
 
 /**
@@ -1179,27 +1210,28 @@ int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct dma_fence *fence;
 	long r;
+	uint32_t ip_instance = ring->me;
 
 	r = amdgpu_uvd_get_create_msg(ring, 1, NULL);
 	if (r) {
-		DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r);
+		DRM_ERROR("amdgpu: (%d)failed to get create msg (%ld).\n", ip_instance, r);
 		goto error;
 	}
 
 	r = amdgpu_uvd_get_destroy_msg(ring, 1, true, &fence);
 	if (r) {
-		DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r);
+		DRM_ERROR("amdgpu: (%d)failed to get destroy ib (%ld).\n", ip_instance, r);
 		goto error;
 	}
 
 	r = dma_fence_wait_timeout(fence, false, timeout);
 	if (r == 0) {
-		DRM_ERROR("amdgpu: IB test timed out.\n");
+		DRM_ERROR("amdgpu: (%d)IB test timed out.\n", ip_instance);
 		r = -ETIMEDOUT;
 	} else if (r < 0) {
-		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
+		DRM_ERROR("amdgpu: (%d)fence wait failed (%ld).\n", ip_instance, r);
 	} else {
-		DRM_DEBUG("ib test on ring %d succeeded\n",  ring->idx);
+		DRM_DEBUG("ib test on (%d)ring %d succeeded\n", ip_instance, ring->idx);
 		r = 0;
 	}
 
@@ -1227,7 +1259,7 @@ uint32_t amdgpu_uvd_used_handles(struct amdgpu_device *adev)
 		 * necessarily linear. So we need to count
 		 * all non-zero handles.
 		 */
-		if (atomic_read(&adev->uvd.handles[i]))
+		if (atomic_read(&adev->uvd.inst->handles[i]))
 			used_handles++;
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
index 32ea20b99e53..b1579fba134c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
@@ -31,30 +31,37 @@
 #define AMDGPU_UVD_SESSION_SIZE		(50*1024)
 #define AMDGPU_UVD_FIRMWARE_OFFSET	256
 
+#define AMDGPU_MAX_UVD_INSTANCES			2
+
 #define AMDGPU_UVD_FIRMWARE_SIZE(adev)    \
 	(AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(((const struct common_firmware_header *)(adev)->uvd.fw->data)->ucode_size_bytes) + \
 			       8) - AMDGPU_UVD_FIRMWARE_OFFSET)
 
-struct amdgpu_uvd {
+struct amdgpu_uvd_inst {
 	struct amdgpu_bo	*vcpu_bo;
 	void			*cpu_addr;
 	uint64_t		gpu_addr;
-	unsigned		fw_version;
 	void			*saved_bo;
-	unsigned		max_handles;
 	atomic_t		handles[AMDGPU_MAX_UVD_HANDLES];
 	struct drm_file		*filp[AMDGPU_MAX_UVD_HANDLES];
 	struct delayed_work	idle_work;
-	const struct firmware	*fw;	/* UVD firmware */
 	struct amdgpu_ring	ring;
 	struct amdgpu_ring	ring_enc[AMDGPU_MAX_UVD_ENC_RINGS];
 	struct amdgpu_irq_src	irq;
-	bool			address_64_bit;
-	bool			use_ctx_buf;
 	struct drm_sched_entity entity;
 	struct drm_sched_entity entity_enc;
 	uint32_t                srbm_soft_reset;
+};
+
+struct amdgpu_uvd {
+	const struct firmware	*fw;	/* UVD firmware */
+	unsigned		fw_version;
+	unsigned		max_handles;
 	unsigned		num_enc_rings;
+	uint8_t		num_uvd_inst;
+	bool			address_64_bit;
+	bool			use_ctx_buf;
+	struct amdgpu_uvd_inst		inst[AMDGPU_MAX_UVD_INSTANCES];
 };
 
 int amdgpu_uvd_sw_init(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index a33804bd3314..23d960ec1cf2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -51,11 +51,13 @@
 #define FIRMWARE_FIJI		"amdgpu/fiji_vce.bin"
 #define FIRMWARE_STONEY		"amdgpu/stoney_vce.bin"
 #define FIRMWARE_POLARIS10	"amdgpu/polaris10_vce.bin"
-#define FIRMWARE_POLARIS11         "amdgpu/polaris11_vce.bin"
-#define FIRMWARE_POLARIS12         "amdgpu/polaris12_vce.bin"
+#define FIRMWARE_POLARIS11	"amdgpu/polaris11_vce.bin"
+#define FIRMWARE_POLARIS12	"amdgpu/polaris12_vce.bin"
+#define FIRMWARE_VEGAM		"amdgpu/vegam_vce.bin"
 
 #define FIRMWARE_VEGA10		"amdgpu/vega10_vce.bin"
 #define FIRMWARE_VEGA12		"amdgpu/vega12_vce.bin"
+#define FIRMWARE_VEGA20		"amdgpu/vega20_vce.bin"
 
 #ifdef CONFIG_DRM_AMDGPU_CIK
 MODULE_FIRMWARE(FIRMWARE_BONAIRE);
@@ -71,9 +73,11 @@ MODULE_FIRMWARE(FIRMWARE_STONEY);
 MODULE_FIRMWARE(FIRMWARE_POLARIS10);
 MODULE_FIRMWARE(FIRMWARE_POLARIS11);
 MODULE_FIRMWARE(FIRMWARE_POLARIS12);
+MODULE_FIRMWARE(FIRMWARE_VEGAM);
 
 MODULE_FIRMWARE(FIRMWARE_VEGA10);
 MODULE_FIRMWARE(FIRMWARE_VEGA12);
+MODULE_FIRMWARE(FIRMWARE_VEGA20);
 
 static void amdgpu_vce_idle_work_handler(struct work_struct *work);
 
@@ -132,12 +136,18 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
 	case CHIP_POLARIS12:
 		fw_name = FIRMWARE_POLARIS12;
 		break;
+	case CHIP_VEGAM:
+		fw_name = FIRMWARE_VEGAM;
+		break;
 	case CHIP_VEGA10:
 		fw_name = FIRMWARE_VEGA10;
 		break;
 	case CHIP_VEGA12:
 		fw_name = FIRMWARE_VEGA12;
 		break;
+	case CHIP_VEGA20:
+		fw_name = FIRMWARE_VEGA20;
+		break;
 
 	default:
 		return -EINVAL;
@@ -181,7 +191,7 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
 	ring = &adev->vce.ring[0];
 	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
 	r = drm_sched_entity_init(&ring->sched, &adev->vce.entity,
-				  rq, amdgpu_sched_jobs, NULL);
+				  rq, NULL);
 	if (r != 0) {
 		DRM_ERROR("Failed setting up VCE run queue.\n");
 		return r;
@@ -755,6 +765,18 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
 			if (r)
 				goto out;
 			break;
+
+		case 0x0500000d: /* MV buffer */
+			r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3,
+							idx + 2, 0, 0);
+			if (r)
+				goto out;
+
+			r = amdgpu_vce_validate_bo(p, ib_idx, idx + 8,
+							idx + 7, 0, 0);
+			if (r)
+				goto out;
+			break;
 		}
 
 		idx += len / 4;
@@ -860,6 +882,18 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
 				goto out;
 			break;
 
+		case 0x0500000d: /* MV buffer */
+			r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3,
+							idx + 2, *size, 0);
+			if (r)
+				goto out;
+
+			r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 8,
+							idx + 7, *size / 12, 0);
+			if (r)
+				goto out;
+			break;
+
 		default:
 			DRM_ERROR("invalid VCE command (0x%x)!\n", cmd);
 			r = -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index 58e495330b38..8851bcdfc260 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -105,7 +105,7 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
 	ring = &adev->vcn.ring_dec;
 	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
 	r = drm_sched_entity_init(&ring->sched, &adev->vcn.entity_dec,
-				  rq, amdgpu_sched_jobs, NULL);
+				  rq, NULL);
 	if (r != 0) {
 		DRM_ERROR("Failed setting up VCN dec run queue.\n");
 		return r;
@@ -114,7 +114,7 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
 	ring = &adev->vcn.ring_enc[0];
 	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
 	r = drm_sched_entity_init(&ring->sched, &adev->vcn.entity_enc,
-				  rq, amdgpu_sched_jobs, NULL);
+				  rq, NULL);
 	if (r != 0) {
 		DRM_ERROR("Failed setting up VCN enc run queue.\n");
 		return r;
@@ -205,13 +205,18 @@ static void amdgpu_vcn_idle_work_handler(struct work_struct *work)
 	struct amdgpu_device *adev =
 		container_of(work, struct amdgpu_device, vcn.idle_work.work);
 	unsigned fences = amdgpu_fence_count_emitted(&adev->vcn.ring_dec);
+	unsigned i;
+
+	for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
+		fences += amdgpu_fence_count_emitted(&adev->vcn.ring_enc[i]);
+	}
 
 	if (fences == 0) {
-		if (adev->pm.dpm_enabled) {
-			/* might be used when with pg/cg
+		if (adev->pm.dpm_enabled)
 			amdgpu_dpm_enable_uvd(adev, false);
-			*/
-		}
+		else
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN,
+							       AMD_PG_STATE_GATE);
 	} else {
 		schedule_delayed_work(&adev->vcn.idle_work, VCN_IDLE_TIMEOUT);
 	}
@@ -223,9 +228,11 @@ void amdgpu_vcn_ring_begin_use(struct amdgpu_ring *ring)
 	bool set_clocks = !cancel_delayed_work_sync(&adev->vcn.idle_work);
 
 	if (set_clocks && adev->pm.dpm_enabled) {
-		/* might be used when with pg/cg
-		amdgpu_dpm_enable_uvd(adev, true);
-		*/
+		if (adev->pm.dpm_enabled)
+			amdgpu_dpm_enable_uvd(adev, true);
+		else
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN,
+							       AMD_PG_STATE_UNGATE);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
index 2fd7db891689..181e6afa9847 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
@@ -45,6 +45,17 @@
 #define VCN_ENC_CMD_REG_WRITE		0x0000000b
 #define VCN_ENC_CMD_REG_WAIT		0x0000000c
 
+enum engine_status_constants {
+	UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON = 0x2AAAA0,
+	UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON = 0x00000002,
+	UVD_STATUS__UVD_BUSY = 0x00000004,
+	GB_ADDR_CONFIG_DEFAULT = 0x26010011,
+	UVD_STATUS__IDLE = 0x2,
+	UVD_STATUS__BUSY = 0x5,
+	UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF = 0x1,
+	UVD_STATUS__RBC_BUSY = 0x1,
+};
+
 struct amdgpu_vcn {
 	struct amdgpu_bo	*vcpu_bo;
 	void			*cpu_addr;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index da55a78d7380..ccba88cc8c54 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -94,6 +94,34 @@ struct amdgpu_prt_cb {
 	struct dma_fence_cb cb;
 };
 
+static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
+				   struct amdgpu_vm *vm,
+				   struct amdgpu_bo *bo)
+{
+	base->vm = vm;
+	base->bo = bo;
+	INIT_LIST_HEAD(&base->bo_list);
+	INIT_LIST_HEAD(&base->vm_status);
+
+	if (!bo)
+		return;
+	list_add_tail(&base->bo_list, &bo->va);
+
+	if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
+		return;
+
+	if (bo->preferred_domains &
+	    amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type))
+		return;
+
+	/*
+	 * we checked all the prerequisites, but it looks like this per vm bo
+	 * is currently evicted. add the bo to the evicted list to make sure it
+	 * is validated on next vm use to avoid fault.
+	 * */
+	list_move_tail(&base->vm_status, &vm->evicted);
+}
+
 /**
  * amdgpu_vm_level_shift - return the addr shift for each level
  *
@@ -196,24 +224,16 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			      void *param)
 {
 	struct ttm_bo_global *glob = adev->mman.bdev.glob;
-	int r;
+	struct amdgpu_vm_bo_base *bo_base, *tmp;
+	int r = 0;
 
-	spin_lock(&vm->status_lock);
-	while (!list_empty(&vm->evicted)) {
-		struct amdgpu_vm_bo_base *bo_base;
-		struct amdgpu_bo *bo;
+	list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
+		struct amdgpu_bo *bo = bo_base->bo;
 
-		bo_base = list_first_entry(&vm->evicted,
-					   struct amdgpu_vm_bo_base,
-					   vm_status);
-		spin_unlock(&vm->status_lock);
-
-		bo = bo_base->bo;
-		BUG_ON(!bo);
 		if (bo->parent) {
 			r = validate(param, bo);
 			if (r)
-				return r;
+				break;
 
 			spin_lock(&glob->lru_lock);
 			ttm_bo_move_to_lru_tail(&bo->tbo);
@@ -222,22 +242,29 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			spin_unlock(&glob->lru_lock);
 		}
 
-		if (bo->tbo.type == ttm_bo_type_kernel &&
-		    vm->use_cpu_for_update) {
-			r = amdgpu_bo_kmap(bo, NULL);
-			if (r)
-				return r;
-		}
-
-		spin_lock(&vm->status_lock);
-		if (bo->tbo.type != ttm_bo_type_kernel)
+		if (bo->tbo.type != ttm_bo_type_kernel) {
+			spin_lock(&vm->moved_lock);
 			list_move(&bo_base->vm_status, &vm->moved);
-		else
+			spin_unlock(&vm->moved_lock);
+		} else {
 			list_move(&bo_base->vm_status, &vm->relocated);
+		}
 	}
-	spin_unlock(&vm->status_lock);
 
-	return 0;
+	spin_lock(&glob->lru_lock);
+	list_for_each_entry(bo_base, &vm->idle, vm_status) {
+		struct amdgpu_bo *bo = bo_base->bo;
+
+		if (!bo->parent)
+			continue;
+
+		ttm_bo_move_to_lru_tail(&bo->tbo);
+		if (bo->shadow)
+			ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
+	}
+	spin_unlock(&glob->lru_lock);
+
+	return r;
 }
 
 /**
@@ -249,13 +276,7 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  */
 bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 {
-	bool ready;
-
-	spin_lock(&vm->status_lock);
-	ready = list_empty(&vm->evicted);
-	spin_unlock(&vm->status_lock);
-
-	return ready;
+	return list_empty(&vm->evicted);
 }
 
 /**
@@ -412,11 +433,16 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 		struct amdgpu_bo *pt;
 
 		if (!entry->base.bo) {
-			r = amdgpu_bo_create(adev,
-					     amdgpu_vm_bo_size(adev, level),
-					     AMDGPU_GPU_PAGE_SIZE,
-					     AMDGPU_GEM_DOMAIN_VRAM, flags,
-					     ttm_bo_type_kernel, resv, &pt);
+			struct amdgpu_bo_param bp;
+
+			memset(&bp, 0, sizeof(bp));
+			bp.size = amdgpu_vm_bo_size(adev, level);
+			bp.byte_align = AMDGPU_GPU_PAGE_SIZE;
+			bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
+			bp.flags = flags;
+			bp.type = ttm_bo_type_kernel;
+			bp.resv = resv;
+			r = amdgpu_bo_create(adev, &bp, &pt);
 			if (r)
 				return r;
 
@@ -441,12 +467,8 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 			*/
 			pt->parent = amdgpu_bo_ref(parent->base.bo);
 
-			entry->base.vm = vm;
-			entry->base.bo = pt;
-			list_add_tail(&entry->base.bo_list, &pt->va);
-			spin_lock(&vm->status_lock);
-			list_add(&entry->base.vm_status, &vm->relocated);
-			spin_unlock(&vm->status_lock);
+			amdgpu_vm_bo_base_init(&entry->base, vm, pt);
+			list_move(&entry->base.vm_status, &vm->relocated);
 		}
 
 		if (level < AMDGPU_VM_PTB) {
@@ -628,7 +650,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_
 		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
 
 	if (vm_flush_needed || pasid_mapping_needed) {
-		r = amdgpu_fence_emit(ring, &fence);
+		r = amdgpu_fence_emit(ring, &fence, 0);
 		if (r)
 			return r;
 	}
@@ -893,10 +915,8 @@ static void amdgpu_vm_invalidate_level(struct amdgpu_device *adev,
 		if (!entry->base.bo)
 			continue;
 
-		spin_lock(&vm->status_lock);
-		if (list_empty(&entry->base.vm_status))
-			list_add(&entry->base.vm_status, &vm->relocated);
-		spin_unlock(&vm->status_lock);
+		if (!entry->base.moved)
+			list_move(&entry->base.vm_status, &vm->relocated);
 		amdgpu_vm_invalidate_level(adev, vm, entry, level + 1);
 	}
 }
@@ -926,6 +946,14 @@ int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 	params.adev = adev;
 
 	if (vm->use_cpu_for_update) {
+		struct amdgpu_vm_bo_base *bo_base;
+
+		list_for_each_entry(bo_base, &vm->relocated, vm_status) {
+			r = amdgpu_bo_kmap(bo_base->bo, NULL);
+			if (unlikely(r))
+				return r;
+		}
+
 		r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
 		if (unlikely(r))
 			return r;
@@ -941,7 +969,6 @@ int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 		params.func = amdgpu_vm_do_set_ptes;
 	}
 
-	spin_lock(&vm->status_lock);
 	while (!list_empty(&vm->relocated)) {
 		struct amdgpu_vm_bo_base *bo_base, *parent;
 		struct amdgpu_vm_pt *pt, *entry;
@@ -950,14 +977,12 @@ int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 		bo_base = list_first_entry(&vm->relocated,
 					   struct amdgpu_vm_bo_base,
 					   vm_status);
-		list_del_init(&bo_base->vm_status);
-		spin_unlock(&vm->status_lock);
+		bo_base->moved = false;
+		list_move(&bo_base->vm_status, &vm->idle);
 
 		bo = bo_base->bo->parent;
-		if (!bo) {
-			spin_lock(&vm->status_lock);
+		if (!bo)
 			continue;
-		}
 
 		parent = list_first_entry(&bo->va, struct amdgpu_vm_bo_base,
 					  bo_list);
@@ -966,12 +991,10 @@ int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 
 		amdgpu_vm_update_pde(&params, vm, pt, entry);
 
-		spin_lock(&vm->status_lock);
 		if (!vm->use_cpu_for_update &&
 		    (ndw - params.ib->length_dw) < 32)
 			break;
 	}
-	spin_unlock(&vm->status_lock);
 
 	if (vm->use_cpu_for_update) {
 		/* Flush HDP */
@@ -1074,9 +1097,7 @@ static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
 		if (entry->huge) {
 			/* Add the entry to the relocated list to update it. */
 			entry->huge = false;
-			spin_lock(&p->vm->status_lock);
 			list_move(&entry->base.vm_status, &p->vm->relocated);
-			spin_unlock(&p->vm->status_lock);
 		}
 		return;
 	}
@@ -1555,9 +1576,22 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 		amdgpu_asic_flush_hdp(adev, NULL);
 	}
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->moved_lock);
 	list_del_init(&bo_va->base.vm_status);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->moved_lock);
+
+	/* If the BO is not in its preferred location add it back to
+	 * the evicted list so that it gets validated again on the
+	 * next command submission.
+	 */
+	if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
+		uint32_t mem_type = bo->tbo.mem.mem_type;
+
+		if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type)))
+			list_add_tail(&bo_va->base.vm_status, &vm->evicted);
+		else
+			list_add(&bo_va->base.vm_status, &vm->idle);
+	}
 
 	list_splice_init(&bo_va->invalids, &bo_va->valids);
 	bo_va->cleared = clear;
@@ -1766,19 +1800,18 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 			   struct amdgpu_vm *vm)
 {
+	struct amdgpu_bo_va *bo_va, *tmp;
+	struct list_head moved;
 	bool clear;
-	int r = 0;
-
-	spin_lock(&vm->status_lock);
-	while (!list_empty(&vm->moved)) {
-		struct amdgpu_bo_va *bo_va;
-		struct reservation_object *resv;
+	int r;
 
-		bo_va = list_first_entry(&vm->moved,
-			struct amdgpu_bo_va, base.vm_status);
-		spin_unlock(&vm->status_lock);
+	INIT_LIST_HEAD(&moved);
+	spin_lock(&vm->moved_lock);
+	list_splice_init(&vm->moved, &moved);
+	spin_unlock(&vm->moved_lock);
 
-		resv = bo_va->base.bo->tbo.resv;
+	list_for_each_entry_safe(bo_va, tmp, &moved, base.vm_status) {
+		struct reservation_object *resv = bo_va->base.bo->tbo.resv;
 
 		/* Per VM BOs never need to bo cleared in the page tables */
 		if (resv == vm->root.base.bo->tbo.resv)
@@ -1791,17 +1824,19 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 			clear = true;
 
 		r = amdgpu_vm_bo_update(adev, bo_va, clear);
-		if (r)
+		if (r) {
+			spin_lock(&vm->moved_lock);
+			list_splice(&moved, &vm->moved);
+			spin_unlock(&vm->moved_lock);
 			return r;
+		}
 
 		if (!clear && resv != vm->root.base.bo->tbo.resv)
 			reservation_object_unlock(resv);
 
-		spin_lock(&vm->status_lock);
 	}
-	spin_unlock(&vm->status_lock);
 
-	return r;
+	return 0;
 }
 
 /**
@@ -1827,36 +1862,12 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
 	if (bo_va == NULL) {
 		return NULL;
 	}
-	bo_va->base.vm = vm;
-	bo_va->base.bo = bo;
-	INIT_LIST_HEAD(&bo_va->base.bo_list);
-	INIT_LIST_HEAD(&bo_va->base.vm_status);
+	amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
 
 	bo_va->ref_count = 1;
 	INIT_LIST_HEAD(&bo_va->valids);
 	INIT_LIST_HEAD(&bo_va->invalids);
 
-	if (!bo)
-		return bo_va;
-
-	list_add_tail(&bo_va->base.bo_list, &bo->va);
-
-	if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
-		return bo_va;
-
-	if (bo->preferred_domains &
-	    amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type))
-		return bo_va;
-
-	/*
-	 * We checked all the prerequisites, but it looks like this per VM BO
-	 * is currently evicted. add the BO to the evicted list to make sure it
-	 * is validated on next VM use to avoid fault.
-	 * */
-	spin_lock(&vm->status_lock);
-	list_move_tail(&bo_va->base.vm_status, &vm->evicted);
-	spin_unlock(&vm->status_lock);
-
 	return bo_va;
 }
 
@@ -1884,11 +1895,11 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
 	if (mapping->flags & AMDGPU_PTE_PRT)
 		amdgpu_vm_prt_get(adev);
 
-	if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
-		spin_lock(&vm->status_lock);
-		if (list_empty(&bo_va->base.vm_status))
-			list_add(&bo_va->base.vm_status, &vm->moved);
-		spin_unlock(&vm->status_lock);
+	if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
+	    !bo_va->base.moved) {
+		spin_lock(&vm->moved_lock);
+		list_move(&bo_va->base.vm_status, &vm->moved);
+		spin_unlock(&vm->moved_lock);
 	}
 	trace_amdgpu_vm_bo_map(bo_va, mapping);
 }
@@ -2198,9 +2209,9 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
 
 	list_del(&bo_va->base.bo_list);
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->moved_lock);
 	list_del(&bo_va->base.vm_status);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->moved_lock);
 
 	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
 		list_del(&mapping->list);
@@ -2234,33 +2245,34 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
 {
 	struct amdgpu_vm_bo_base *bo_base;
 
+	/* shadow bo doesn't have bo base, its validation needs its parent */
+	if (bo->parent && bo->parent->shadow == bo)
+		bo = bo->parent;
+
 	list_for_each_entry(bo_base, &bo->va, bo_list) {
 		struct amdgpu_vm *vm = bo_base->vm;
+		bool was_moved = bo_base->moved;
 
 		bo_base->moved = true;
 		if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
-			spin_lock(&bo_base->vm->status_lock);
 			if (bo->tbo.type == ttm_bo_type_kernel)
 				list_move(&bo_base->vm_status, &vm->evicted);
 			else
 				list_move_tail(&bo_base->vm_status,
 					       &vm->evicted);
-			spin_unlock(&bo_base->vm->status_lock);
 			continue;
 		}
 
-		if (bo->tbo.type == ttm_bo_type_kernel) {
-			spin_lock(&bo_base->vm->status_lock);
-			if (list_empty(&bo_base->vm_status))
-				list_add(&bo_base->vm_status, &vm->relocated);
-			spin_unlock(&bo_base->vm->status_lock);
+		if (was_moved)
 			continue;
-		}
 
-		spin_lock(&bo_base->vm->status_lock);
-		if (list_empty(&bo_base->vm_status))
-			list_add(&bo_base->vm_status, &vm->moved);
-		spin_unlock(&bo_base->vm->status_lock);
+		if (bo->tbo.type == ttm_bo_type_kernel) {
+			list_move(&bo_base->vm_status, &vm->relocated);
+		} else {
+			spin_lock(&bo_base->vm->moved_lock);
+			list_move(&bo_base->vm_status, &vm->moved);
+			spin_unlock(&bo_base->vm->moved_lock);
+		}
 	}
 }
 
@@ -2355,6 +2367,8 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t vm_size,
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		   int vm_context, unsigned int pasid)
 {
+	struct amdgpu_bo_param bp;
+	struct amdgpu_bo *root;
 	const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
 		AMDGPU_VM_PTE_COUNT(adev) * 8);
 	unsigned ring_instance;
@@ -2367,10 +2381,11 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	vm->va = RB_ROOT_CACHED;
 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
 		vm->reserved_vmid[i] = NULL;
-	spin_lock_init(&vm->status_lock);
 	INIT_LIST_HEAD(&vm->evicted);
 	INIT_LIST_HEAD(&vm->relocated);
+	spin_lock_init(&vm->moved_lock);
 	INIT_LIST_HEAD(&vm->moved);
+	INIT_LIST_HEAD(&vm->idle);
 	INIT_LIST_HEAD(&vm->freed);
 
 	/* create scheduler entity for page table updates */
@@ -2380,7 +2395,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	ring = adev->vm_manager.vm_pte_rings[ring_instance];
 	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
 	r = drm_sched_entity_init(&ring->sched, &vm->entity,
-				  rq, amdgpu_sched_jobs, NULL);
+				  rq, NULL);
 	if (r)
 		return r;
 
@@ -2409,24 +2424,28 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		flags |= AMDGPU_GEM_CREATE_SHADOW;
 
 	size = amdgpu_vm_bo_size(adev, adev->vm_manager.root_level);
-	r = amdgpu_bo_create(adev, size, align, AMDGPU_GEM_DOMAIN_VRAM, flags,
-			     ttm_bo_type_kernel, NULL, &vm->root.base.bo);
+	memset(&bp, 0, sizeof(bp));
+	bp.size = size;
+	bp.byte_align = align;
+	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
+	bp.flags = flags;
+	bp.type = ttm_bo_type_kernel;
+	bp.resv = NULL;
+	r = amdgpu_bo_create(adev, &bp, &root);
 	if (r)
 		goto error_free_sched_entity;
 
-	r = amdgpu_bo_reserve(vm->root.base.bo, true);
+	r = amdgpu_bo_reserve(root, true);
 	if (r)
 		goto error_free_root;
 
-	r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo,
+	r = amdgpu_vm_clear_bo(adev, vm, root,
 			       adev->vm_manager.root_level,
 			       vm->pte_support_ats);
 	if (r)
 		goto error_unreserve;
 
-	vm->root.base.vm = vm;
-	list_add_tail(&vm->root.base.bo_list, &vm->root.base.bo->va);
-	list_add_tail(&vm->root.base.vm_status, &vm->evicted);
+	amdgpu_vm_bo_base_init(&vm->root.base, vm, root);
 	amdgpu_bo_unreserve(vm->root.base.bo);
 
 	if (pasid) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 30f080364c97..061b99a18cb8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -75,11 +75,12 @@ struct amdgpu_bo_list_entry;
 /* PDE Block Fragment Size for VEGA10 */
 #define AMDGPU_PDE_BFS(a)	((uint64_t)a << 59)
 
-/* VEGA10 only */
+
+/* For GFX9 */
 #define AMDGPU_PTE_MTYPE(a)    ((uint64_t)a << 57)
 #define AMDGPU_PTE_MTYPE_MASK	AMDGPU_PTE_MTYPE(3ULL)
 
-/* For Raven */
+#define AMDGPU_MTYPE_NC 0
 #define AMDGPU_MTYPE_CC 2
 
 #define AMDGPU_PTE_DEFAULT_ATC  (AMDGPU_PTE_SYSTEM      \
@@ -167,9 +168,6 @@ struct amdgpu_vm {
 	/* tree of virtual addresses mapped */
 	struct rb_root_cached	va;
 
-	/* protecting invalidated */
-	spinlock_t		status_lock;
-
 	/* BOs who needs a validation */
 	struct list_head	evicted;
 
@@ -178,6 +176,10 @@ struct amdgpu_vm {
 
 	/* BOs moved, but not yet updated in the PT */
 	struct list_head	moved;
+	spinlock_t		moved_lock;
+
+	/* All BOs of this VM not currently in the state machine */
+	struct list_head	idle;
 
 	/* BO mappings freed, but not yet updated in the PT */
 	struct list_head	freed;
@@ -186,9 +188,6 @@ struct amdgpu_vm {
 	struct amdgpu_vm_pt     root;
 	struct dma_fence	*last_update;
 
-	/* protecting freed */
-	spinlock_t		freed_lock;
-
 	/* Scheduler entity for page table updates */
 	struct drm_sched_entity	entity;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index 47ef3e6e7178..a266dcf5daed 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -5903,7 +5903,7 @@ static int ci_dpm_init(struct amdgpu_device *adev)
 	pi->pcie_dpm_key_disabled = 0;
 	pi->thermal_sclk_dpm_enabled = 0;
 
-	if (amdgpu_pp_feature_mask & SCLK_DEEP_SLEEP_MASK)
+	if (adev->powerplay.pp_feature & PP_SCLK_DEEP_SLEEP_MASK)
 		pi->caps_sclk_ds = true;
 	else
 		pi->caps_sclk_ds = false;
@@ -6255,7 +6255,7 @@ static int ci_dpm_late_init(void *handle)
 	int ret;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	if (!amdgpu_dpm)
+	if (!adev->pm.dpm_enabled)
 		return 0;
 
 	/* init the sysfs and debugfs files late */
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 0df22030e713..8ff4c60d1b59 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -1735,6 +1735,12 @@ static void cik_invalidate_hdp(struct amdgpu_device *adev,
 	}
 }
 
+static bool cik_need_full_reset(struct amdgpu_device *adev)
+{
+	/* change this when we support soft reset */
+	return true;
+}
+
 static const struct amdgpu_asic_funcs cik_asic_funcs =
 {
 	.read_disabled_bios = &cik_read_disabled_bios,
@@ -1748,6 +1754,7 @@ static const struct amdgpu_asic_funcs cik_asic_funcs =
 	.get_config_memsize = &cik_get_config_memsize,
 	.flush_hdp = &cik_flush_hdp,
 	.invalidate_hdp = &cik_invalidate_hdp,
+	.need_full_reset = &cik_need_full_reset,
 };
 
 static int cik_common_early_init(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index 452f88ea46a2..ada241bfeee9 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -1823,7 +1823,6 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct amdgpu_device *adev = dev->dev_private;
-	struct amdgpu_framebuffer *amdgpu_fb;
 	struct drm_framebuffer *target_fb;
 	struct drm_gem_object *obj;
 	struct amdgpu_bo *abo;
@@ -1842,18 +1841,15 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 		return 0;
 	}
 
-	if (atomic) {
-		amdgpu_fb = to_amdgpu_framebuffer(fb);
+	if (atomic)
 		target_fb = fb;
-	} else {
-		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
+	else
 		target_fb = crtc->primary->fb;
-	}
 
 	/* If atomic, assume fb object is pinned & idle & fenced and
 	 * just update base pointers
 	 */
-	obj = amdgpu_fb->obj;
+	obj = target_fb->obj[0];
 	abo = gem_to_amdgpu_bo(obj);
 	r = amdgpu_bo_reserve(abo, false);
 	if (unlikely(r != 0))
@@ -2043,8 +2039,7 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(mmMASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
-		amdgpu_fb = to_amdgpu_framebuffer(fb);
-		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		abo = gem_to_amdgpu_bo(fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r != 0))
 			return r;
@@ -2526,11 +2521,9 @@ static void dce_v10_0_crtc_disable(struct drm_crtc *crtc)
 	dce_v10_0_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
 	if (crtc->primary->fb) {
 		int r;
-		struct amdgpu_framebuffer *amdgpu_fb;
 		struct amdgpu_bo *abo;
 
-		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
-		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		abo = gem_to_amdgpu_bo(crtc->primary->fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r))
 			DRM_ERROR("failed to reserve abo before unpin\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index a7c1c584a191..a5b96eac3033 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -173,6 +173,7 @@ static void dce_v11_0_init_golden_registers(struct amdgpu_device *adev)
 							ARRAY_SIZE(polaris11_golden_settings_a11));
 		break;
 	case CHIP_POLARIS10:
+	case CHIP_VEGAM:
 		amdgpu_device_program_register_sequence(adev,
 							polaris10_golden_settings_a11,
 							ARRAY_SIZE(polaris10_golden_settings_a11));
@@ -473,6 +474,7 @@ static int dce_v11_0_get_num_crtc (struct amdgpu_device *adev)
 		num_crtc = 2;
 		break;
 	case CHIP_POLARIS10:
+	case CHIP_VEGAM:
 		num_crtc = 6;
 		break;
 	case CHIP_POLARIS11:
@@ -1445,6 +1447,7 @@ static int dce_v11_0_audio_init(struct amdgpu_device *adev)
 		adev->mode_info.audio.num_pins = 7;
 		break;
 	case CHIP_POLARIS10:
+	case CHIP_VEGAM:
 		adev->mode_info.audio.num_pins = 8;
 		break;
 	case CHIP_POLARIS11:
@@ -1862,7 +1865,6 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct amdgpu_device *adev = dev->dev_private;
-	struct amdgpu_framebuffer *amdgpu_fb;
 	struct drm_framebuffer *target_fb;
 	struct drm_gem_object *obj;
 	struct amdgpu_bo *abo;
@@ -1881,18 +1883,15 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
 		return 0;
 	}
 
-	if (atomic) {
-		amdgpu_fb = to_amdgpu_framebuffer(fb);
+	if (atomic)
 		target_fb = fb;
-	} else {
-		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
+	else
 		target_fb = crtc->primary->fb;
-	}
 
 	/* If atomic, assume fb object is pinned & idle & fenced and
 	 * just update base pointers
 	 */
-	obj = amdgpu_fb->obj;
+	obj = target_fb->obj[0];
 	abo = gem_to_amdgpu_bo(obj);
 	r = amdgpu_bo_reserve(abo, false);
 	if (unlikely(r != 0))
@@ -2082,8 +2081,7 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(mmCRTC_MASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
-		amdgpu_fb = to_amdgpu_framebuffer(fb);
-		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		abo = gem_to_amdgpu_bo(fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r != 0))
 			return r;
@@ -2253,7 +2251,8 @@ static u32 dce_v11_0_pick_pll(struct drm_crtc *crtc)
 
 	if ((adev->asic_type == CHIP_POLARIS10) ||
 	    (adev->asic_type == CHIP_POLARIS11) ||
-	    (adev->asic_type == CHIP_POLARIS12)) {
+	    (adev->asic_type == CHIP_POLARIS12) ||
+	    (adev->asic_type == CHIP_VEGAM)) {
 		struct amdgpu_encoder *amdgpu_encoder =
 			to_amdgpu_encoder(amdgpu_crtc->encoder);
 		struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv;
@@ -2601,11 +2600,9 @@ static void dce_v11_0_crtc_disable(struct drm_crtc *crtc)
 	dce_v11_0_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
 	if (crtc->primary->fb) {
 		int r;
-		struct amdgpu_framebuffer *amdgpu_fb;
 		struct amdgpu_bo *abo;
 
-		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
-		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		abo = gem_to_amdgpu_bo(crtc->primary->fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r))
 			DRM_ERROR("failed to reserve abo before unpin\n");
@@ -2673,7 +2670,8 @@ static int dce_v11_0_crtc_mode_set(struct drm_crtc *crtc,
 
 	if ((adev->asic_type == CHIP_POLARIS10) ||
 	    (adev->asic_type == CHIP_POLARIS11) ||
-	    (adev->asic_type == CHIP_POLARIS12)) {
+	    (adev->asic_type == CHIP_POLARIS12) ||
+	    (adev->asic_type == CHIP_VEGAM)) {
 		struct amdgpu_encoder *amdgpu_encoder =
 			to_amdgpu_encoder(amdgpu_crtc->encoder);
 		int encoder_mode =
@@ -2830,6 +2828,7 @@ static int dce_v11_0_early_init(void *handle)
 		adev->mode_info.num_dig = 9;
 		break;
 	case CHIP_POLARIS10:
+	case CHIP_VEGAM:
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 6;
 		break;
@@ -2949,7 +2948,8 @@ static int dce_v11_0_hw_init(void *handle)
 	amdgpu_atombios_encoder_init_dig(adev);
 	if ((adev->asic_type == CHIP_POLARIS10) ||
 	    (adev->asic_type == CHIP_POLARIS11) ||
-	    (adev->asic_type == CHIP_POLARIS12)) {
+	    (adev->asic_type == CHIP_POLARIS12) ||
+	    (adev->asic_type == CHIP_VEGAM)) {
 		amdgpu_atombios_crtc_set_dce_clock(adev, adev->clock.default_dispclk,
 						   DCE_CLOCK_TYPE_DISPCLK, ATOM_GCK_DFS);
 		amdgpu_atombios_crtc_set_dce_clock(adev, 0,
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index 9f67b7fd3487..394cc1e8fe20 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -1780,7 +1780,6 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct amdgpu_device *adev = dev->dev_private;
-	struct amdgpu_framebuffer *amdgpu_fb;
 	struct drm_framebuffer *target_fb;
 	struct drm_gem_object *obj;
 	struct amdgpu_bo *abo;
@@ -1798,18 +1797,15 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
 		return 0;
 	}
 
-	if (atomic) {
-		amdgpu_fb = to_amdgpu_framebuffer(fb);
+	if (atomic)
 		target_fb = fb;
-	} else {
-		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
+	else
 		target_fb = crtc->primary->fb;
-	}
 
 	/* If atomic, assume fb object is pinned & idle & fenced and
 	 * just update base pointers
 	 */
-	obj = amdgpu_fb->obj;
+	obj = target_fb->obj[0];
 	abo = gem_to_amdgpu_bo(obj);
 	r = amdgpu_bo_reserve(abo, false);
 	if (unlikely(r != 0))
@@ -1978,8 +1974,7 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(mmMASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
-		amdgpu_fb = to_amdgpu_framebuffer(fb);
-		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		abo = gem_to_amdgpu_bo(fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r != 0))
 			return r;
@@ -2414,11 +2409,9 @@ static void dce_v6_0_crtc_disable(struct drm_crtc *crtc)
 	dce_v6_0_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
 	if (crtc->primary->fb) {
 		int r;
-		struct amdgpu_framebuffer *amdgpu_fb;
 		struct amdgpu_bo *abo;
 
-		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
-		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		abo = gem_to_amdgpu_bo(crtc->primary->fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r))
 			DRM_ERROR("failed to reserve abo before unpin\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index f55422cbd77a..c9b9ab8f1b05 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -1754,7 +1754,6 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct amdgpu_device *adev = dev->dev_private;
-	struct amdgpu_framebuffer *amdgpu_fb;
 	struct drm_framebuffer *target_fb;
 	struct drm_gem_object *obj;
 	struct amdgpu_bo *abo;
@@ -1773,18 +1772,15 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 		return 0;
 	}
 
-	if (atomic) {
-		amdgpu_fb = to_amdgpu_framebuffer(fb);
+	if (atomic)
 		target_fb = fb;
-	} else {
-		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
+	else
 		target_fb = crtc->primary->fb;
-	}
 
 	/* If atomic, assume fb object is pinned & idle & fenced and
 	 * just update base pointers
 	 */
-	obj = amdgpu_fb->obj;
+	obj = target_fb->obj[0];
 	abo = gem_to_amdgpu_bo(obj);
 	r = amdgpu_bo_reserve(abo, false);
 	if (unlikely(r != 0))
@@ -1955,8 +1951,7 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(mmMASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
-		amdgpu_fb = to_amdgpu_framebuffer(fb);
-		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		abo = gem_to_amdgpu_bo(fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r != 0))
 			return r;
@@ -2430,11 +2425,9 @@ static void dce_v8_0_crtc_disable(struct drm_crtc *crtc)
 	dce_v8_0_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
 	if (crtc->primary->fb) {
 		int r;
-		struct amdgpu_framebuffer *amdgpu_fb;
 		struct amdgpu_bo *abo;
 
-		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
-		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		abo = gem_to_amdgpu_bo(crtc->primary->fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r))
 			DRM_ERROR("failed to reserve abo before unpin\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
index b51f05dc9582..dbf2ccd0c744 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -168,11 +168,9 @@ static void dce_virtual_crtc_disable(struct drm_crtc *crtc)
 	dce_virtual_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
 	if (crtc->primary->fb) {
 		int r;
-		struct amdgpu_framebuffer *amdgpu_fb;
 		struct amdgpu_bo *abo;
 
-		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
-		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		abo = gem_to_amdgpu_bo(crtc->primary->fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r))
 			DRM_ERROR("failed to reserve abo before unpin\n");
@@ -329,7 +327,7 @@ static int dce_virtual_get_modes(struct drm_connector *connector)
 	return 0;
 }
 
-static int dce_virtual_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status dce_virtual_mode_valid(struct drm_connector *connector,
 				  struct drm_display_mode *mode)
 {
 	return MODE_OK;
@@ -462,8 +460,9 @@ static int dce_virtual_hw_init(void *handle)
 		break;
 	case CHIP_CARRIZO:
 	case CHIP_STONEY:
-	case CHIP_POLARIS11:
 	case CHIP_POLARIS10:
+	case CHIP_POLARIS11:
+	case CHIP_VEGAM:
 		dce_v11_0_disable_dce(adev);
 		break;
 	case CHIP_TOPAZ:
@@ -474,6 +473,7 @@ static int dce_virtual_hw_init(void *handle)
 		break;
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 		break;
 	default:
 		DRM_ERROR("Virtual display unsupported ASIC type: 0x%X\n", adev->asic_type);
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v1_7.c b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
new file mode 100644
index 000000000000..9935371db7ce
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "df_v1_7.h"
+
+#include "df/df_1_7_default.h"
+#include "df/df_1_7_offset.h"
+#include "df/df_1_7_sh_mask.h"
+
+static u32 df_v1_7_channel_number[] = {1, 2, 0, 4, 0, 8, 0, 16, 2};
+
+static void df_v1_7_init (struct amdgpu_device *adev)
+{
+}
+
+static void df_v1_7_enable_broadcast_mode(struct amdgpu_device *adev,
+                                          bool enable)
+{
+	u32 tmp;
+
+	if (enable) {
+		tmp = RREG32_SOC15(DF, 0, mmFabricConfigAccessControl);
+		tmp &= ~FabricConfigAccessControl__CfgRegInstAccEn_MASK;
+		WREG32_SOC15(DF, 0, mmFabricConfigAccessControl, tmp);
+	} else
+		WREG32_SOC15(DF, 0, mmFabricConfigAccessControl,
+			     mmFabricConfigAccessControl_DEFAULT);
+}
+
+static u32 df_v1_7_get_fb_channel_number(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+	tmp = RREG32_SOC15(DF, 0, mmDF_CS_AON0_DramBaseAddress0);
+	tmp &= DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK;
+	tmp >>= DF_CS_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
+
+	return tmp;
+}
+
+static u32 df_v1_7_get_hbm_channel_number(struct amdgpu_device *adev)
+{
+	int fb_channel_number;
+
+	fb_channel_number = adev->df_funcs->get_fb_channel_number(adev);
+
+	return df_v1_7_channel_number[fb_channel_number];
+}
+
+static void df_v1_7_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+						     bool enable)
+{
+	u32 tmp;
+
+	/* Put DF on broadcast mode */
+	adev->df_funcs->enable_broadcast_mode(adev, true);
+
+	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_DF_MGCG)) {
+		tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
+		tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
+		tmp |= DF_V1_7_MGCG_ENABLE_15_CYCLE_DELAY;
+		WREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater, tmp);
+	} else {
+		tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
+		tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
+		tmp |= DF_V1_7_MGCG_DISABLE;
+		WREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater, tmp);
+	}
+
+	/* Exit boradcast mode */
+	adev->df_funcs->enable_broadcast_mode(adev, false);
+}
+
+static void df_v1_7_get_clockgating_state(struct amdgpu_device *adev,
+					  u32 *flags)
+{
+	u32 tmp;
+
+	/* AMD_CG_SUPPORT_DF_MGCG */
+	tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
+	if (tmp & DF_V1_7_MGCG_ENABLE_15_CYCLE_DELAY)
+		*flags |= AMD_CG_SUPPORT_DF_MGCG;
+}
+
+static void df_v1_7_enable_ecc_force_par_wr_rmw(struct amdgpu_device *adev,
+						bool enable)
+{
+	WREG32_FIELD15(DF, 0, DF_CS_AON0_CoherentSlaveModeCtrlA0,
+		       ForceParWrRMW, enable);
+}
+
+const struct amdgpu_df_funcs df_v1_7_funcs = {
+	.init = df_v1_7_init,
+	.enable_broadcast_mode = df_v1_7_enable_broadcast_mode,
+	.get_fb_channel_number = df_v1_7_get_fb_channel_number,
+	.get_hbm_channel_number = df_v1_7_get_hbm_channel_number,
+	.update_medium_grain_clock_gating = df_v1_7_update_medium_grain_clock_gating,
+	.get_clockgating_state = df_v1_7_get_clockgating_state,
+	.enable_ecc_force_par_wr_rmw = df_v1_7_enable_ecc_force_par_wr_rmw,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v1_7.h b/drivers/gpu/drm/amd/amdgpu/df_v1_7.h
new file mode 100644
index 000000000000..74621104c487
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/df_v1_7.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __DF_V1_7_H__
+#define __DF_V1_7_H__
+
+#include "soc15_common.h"
+enum DF_V1_7_MGCG
+{
+	DF_V1_7_MGCG_DISABLE = 0,
+	DF_V1_7_MGCG_ENABLE_00_CYCLE_DELAY =1,
+	DF_V1_7_MGCG_ENABLE_01_CYCLE_DELAY =2,
+	DF_V1_7_MGCG_ENABLE_15_CYCLE_DELAY =13,
+	DF_V1_7_MGCG_ENABLE_31_CYCLE_DELAY =14,
+	DF_V1_7_MGCG_ENABLE_63_CYCLE_DELAY =15
+};
+
+extern const struct amdgpu_df_funcs df_v1_7_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
new file mode 100644
index 000000000000..60608b3df881
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "df_v3_6.h"
+
+#include "df/df_3_6_default.h"
+#include "df/df_3_6_offset.h"
+#include "df/df_3_6_sh_mask.h"
+
+static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
+				       16, 32, 0, 0, 0, 2, 4, 8};
+
+static void df_v3_6_init(struct amdgpu_device *adev)
+{
+}
+
+static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
+					  bool enable)
+{
+	u32 tmp;
+
+	if (enable) {
+		tmp = RREG32_SOC15(DF, 0, mmFabricConfigAccessControl);
+		tmp &= ~FabricConfigAccessControl__CfgRegInstAccEn_MASK;
+		WREG32_SOC15(DF, 0, mmFabricConfigAccessControl, tmp);
+	} else
+		WREG32_SOC15(DF, 0, mmFabricConfigAccessControl,
+			     mmFabricConfigAccessControl_DEFAULT);
+}
+
+static u32 df_v3_6_get_fb_channel_number(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+	tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DramBaseAddress0);
+	tmp &= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK;
+	tmp >>= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
+
+	return tmp;
+}
+
+static u32 df_v3_6_get_hbm_channel_number(struct amdgpu_device *adev)
+{
+	int fb_channel_number;
+
+	fb_channel_number = adev->df_funcs->get_fb_channel_number(adev);
+	if (fb_channel_number > ARRAY_SIZE(df_v3_6_channel_number))
+		fb_channel_number = 0;
+
+	return df_v3_6_channel_number[fb_channel_number];
+}
+
+static void df_v3_6_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+						     bool enable)
+{
+	u32 tmp;
+
+	/* Put DF on broadcast mode */
+	adev->df_funcs->enable_broadcast_mode(adev, true);
+
+	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_DF_MGCG)) {
+		tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
+		tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
+		tmp |= DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY;
+		WREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater, tmp);
+	} else {
+		tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
+		tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
+		tmp |= DF_V3_6_MGCG_DISABLE;
+		WREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater, tmp);
+	}
+
+	/* Exit broadcast mode */
+	adev->df_funcs->enable_broadcast_mode(adev, false);
+}
+
+static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
+					  u32 *flags)
+{
+	u32 tmp;
+
+	/* AMD_CG_SUPPORT_DF_MGCG */
+	tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
+	if (tmp & DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY)
+		*flags |= AMD_CG_SUPPORT_DF_MGCG;
+}
+
+const struct amdgpu_df_funcs df_v3_6_funcs = {
+	.init = df_v3_6_init,
+	.enable_broadcast_mode = df_v3_6_enable_broadcast_mode,
+	.get_fb_channel_number = df_v3_6_get_fb_channel_number,
+	.get_hbm_channel_number = df_v3_6_get_hbm_channel_number,
+	.update_medium_grain_clock_gating =
+			df_v3_6_update_medium_grain_clock_gating,
+	.get_clockgating_state = df_v3_6_get_clockgating_state,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
new file mode 100644
index 000000000000..e79c58e5efcb
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __DF_V3_6_H__
+#define __DF_V3_6_H__
+
+#include "soc15_common.h"
+
+enum DF_V3_6_MGCG {
+	DF_V3_6_MGCG_DISABLE = 0,
+	DF_V3_6_MGCG_ENABLE_00_CYCLE_DELAY = 1,
+	DF_V3_6_MGCG_ENABLE_01_CYCLE_DELAY = 2,
+	DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY = 13,
+	DF_V3_6_MGCG_ENABLE_31_CYCLE_DELAY = 14,
+	DF_V3_6_MGCG_ENABLE_63_CYCLE_DELAY = 15
+};
+
+extern const struct amdgpu_df_funcs df_v3_6_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index e14263fca1c9..818874b13c99 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -125,18 +125,6 @@ MODULE_FIRMWARE("amdgpu/fiji_mec.bin");
 MODULE_FIRMWARE("amdgpu/fiji_mec2.bin");
 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin");
 
-MODULE_FIRMWARE("amdgpu/polaris11_ce.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_me.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_mec.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin");
-MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin");
-
 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin");
@@ -149,6 +137,18 @@ MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin");
 
+MODULE_FIRMWARE("amdgpu/polaris11_ce.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_me.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_mec.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin");
+MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin");
+
 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin");
@@ -161,6 +161,13 @@ MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin");
 
+MODULE_FIRMWARE("amdgpu/vegam_ce.bin");
+MODULE_FIRMWARE("amdgpu/vegam_pfp.bin");
+MODULE_FIRMWARE("amdgpu/vegam_me.bin");
+MODULE_FIRMWARE("amdgpu/vegam_mec.bin");
+MODULE_FIRMWARE("amdgpu/vegam_mec2.bin");
+MODULE_FIRMWARE("amdgpu/vegam_rlc.bin");
+
 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
 {
 	{mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},
@@ -292,6 +299,37 @@ static const u32 tonga_mgcg_cgcg_init[] =
 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 };
 
+static const u32 golden_settings_vegam_a11[] =
+{
+	mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
+	mmCB_HW_CONTROL_2, 0x0f000000, 0x0d000000,
+	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x3a00161a,
+	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002e,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
+	mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
+	mmSQ_CONFIG, 0x07f80000, 0x01180000,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
+	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
+	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+};
+
+static const u32 vegam_golden_common_all[] =
+{
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
+	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
+};
+
 static const u32 golden_settings_polaris11_a11[] =
 {
 	mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208,
@@ -712,6 +750,14 @@ static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
 							tonga_golden_common_all,
 							ARRAY_SIZE(tonga_golden_common_all));
 		break;
+	case CHIP_VEGAM:
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_vegam_a11,
+							ARRAY_SIZE(golden_settings_vegam_a11));
+		amdgpu_device_program_register_sequence(adev,
+							vegam_golden_common_all,
+							ARRAY_SIZE(vegam_golden_common_all));
+		break;
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
 		amdgpu_device_program_register_sequence(adev,
@@ -918,17 +964,20 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
 	case CHIP_FIJI:
 		chip_name = "fiji";
 		break;
-	case CHIP_POLARIS11:
-		chip_name = "polaris11";
+	case CHIP_STONEY:
+		chip_name = "stoney";
 		break;
 	case CHIP_POLARIS10:
 		chip_name = "polaris10";
 		break;
+	case CHIP_POLARIS11:
+		chip_name = "polaris11";
+		break;
 	case CHIP_POLARIS12:
 		chip_name = "polaris12";
 		break;
-	case CHIP_STONEY:
-		chip_name = "stoney";
+	case CHIP_VEGAM:
+		chip_name = "vegam";
 		break;
 	default:
 		BUG();
@@ -1770,6 +1819,7 @@ static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
 		gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN;
 		break;
 	case CHIP_POLARIS10:
+	case CHIP_VEGAM:
 		ret = amdgpu_atombios_get_gfx_info(adev);
 		if (ret)
 			return ret;
@@ -1957,12 +2007,13 @@ static int gfx_v8_0_sw_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	switch (adev->asic_type) {
-	case CHIP_FIJI:
 	case CHIP_TONGA:
+	case CHIP_CARRIZO:
+	case CHIP_FIJI:
+	case CHIP_POLARIS10:
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
-	case CHIP_POLARIS10:
-	case CHIP_CARRIZO:
+	case CHIP_VEGAM:
 		adev->gfx.mec.num_mec = 2;
 		break;
 	case CHIP_TOPAZ:
@@ -2323,6 +2374,7 @@ static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
 
 		break;
 	case CHIP_FIJI:
+	case CHIP_VEGAM:
 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
@@ -3504,6 +3556,7 @@ gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
 {
 	switch (adev->asic_type) {
 	case CHIP_FIJI:
+	case CHIP_VEGAM:
 		*rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
 			  RB_XSEL2(1) | PKR_MAP(2) |
 			  PKR_XSEL(1) | PKR_YSEL(1) |
@@ -4071,7 +4124,8 @@ static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
 		gfx_v8_0_init_power_gating(adev);
 		WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
 	} else if ((adev->asic_type == CHIP_POLARIS11) ||
-		   (adev->asic_type == CHIP_POLARIS12)) {
+		   (adev->asic_type == CHIP_POLARIS12) ||
+		   (adev->asic_type == CHIP_VEGAM)) {
 		gfx_v8_0_init_csb(adev);
 		gfx_v8_0_init_save_restore_list(adev);
 		gfx_v8_0_enable_save_restore_machine(adev);
@@ -4146,7 +4200,8 @@ static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
 	WREG32(mmRLC_CGCG_CGLS_CTRL, tmp);
 	if (adev->asic_type == CHIP_POLARIS11 ||
 	    adev->asic_type == CHIP_POLARIS10 ||
-	    adev->asic_type == CHIP_POLARIS12) {
+	    adev->asic_type == CHIP_POLARIS12 ||
+	    adev->asic_type == CHIP_VEGAM) {
 		tmp = RREG32(mmRLC_CGCG_CGLS_CTRL_3D);
 		tmp &= ~0x3;
 		WREG32(mmRLC_CGCG_CGLS_CTRL_3D, tmp);
@@ -5498,7 +5553,8 @@ static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *ade
 						       bool enable)
 {
 	if ((adev->asic_type == CHIP_POLARIS11) ||
-	    (adev->asic_type == CHIP_POLARIS12))
+	    (adev->asic_type == CHIP_POLARIS12) ||
+	    (adev->asic_type == CHIP_VEGAM))
 		/* Send msg to SMU via Powerplay */
 		amdgpu_device_ip_set_powergating_state(adev,
 						       AMD_IP_BLOCK_TYPE_SMC,
@@ -5588,6 +5644,7 @@ static int gfx_v8_0_set_powergating_state(void *handle,
 		break;
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
 			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
 		else
@@ -6154,6 +6211,7 @@ static int gfx_v8_0_set_clockgating_state(void *handle,
 	case CHIP_POLARIS10:
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 		gfx_v8_0_polaris_update_gfx_clock_gating(adev, state);
 		break;
 	default:
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 9d39fd5b1822..d7530fdfaad5 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -27,6 +27,7 @@
 #include "amdgpu_gfx.h"
 #include "soc15.h"
 #include "soc15d.h"
+#include "amdgpu_atomfirmware.h"
 
 #include "gc/gc_9_0_offset.h"
 #include "gc/gc_9_0_sh_mask.h"
@@ -41,7 +42,6 @@
 #define GFX9_MEC_HPD_SIZE 2048
 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
-#define GFX9_RLC_FORMAT_DIRECT_REG_LIST_LENGTH 34
 
 #define mmPWR_MISC_CNTL_STATUS					0x0183
 #define mmPWR_MISC_CNTL_STATUS_BASE_IDX				0
@@ -64,6 +64,13 @@ MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
 
+MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
+MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
+MODULE_FIRMWARE("amdgpu/vega20_me.bin");
+MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
+MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
+MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
+
 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
 MODULE_FIRMWARE("amdgpu/raven_me.bin");
@@ -73,29 +80,22 @@ MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
 
 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
 {
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
 };
 
 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
@@ -109,6 +109,20 @@ static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800)
 };
 
+static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
+{
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
+};
+
 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
 {
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
@@ -185,6 +199,30 @@ static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000)
 };
 
+static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
+{
+	mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
+	mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
+	mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
+	mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
+	mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
+	mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
+	mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
+	mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
+};
+
+static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
+{
+	mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
+	mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
+	mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
+	mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
+	mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
+	mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
+	mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
+	mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
+};
+
 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
@@ -218,6 +256,14 @@ static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 						golden_settings_gc_9_2_1_vg12,
 						ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
 		break;
+	case CHIP_VEGA20:
+		soc15_program_register_sequence(adev,
+						golden_settings_gc_9_0,
+						ARRAY_SIZE(golden_settings_gc_9_0));
+		soc15_program_register_sequence(adev,
+						golden_settings_gc_9_0_vg20,
+						ARRAY_SIZE(golden_settings_gc_9_0_vg20));
+		break;
 	case CHIP_RAVEN:
 		soc15_program_register_sequence(adev,
 						 golden_settings_gc_9_1,
@@ -401,6 +447,27 @@ static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
 	kfree(adev->gfx.rlc.register_list_format);
 }
 
+static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
+{
+	const struct rlc_firmware_header_v2_1 *rlc_hdr;
+
+	rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
+	adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
+	adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
+	adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
+	adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
+	adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
+	adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
+	adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
+	adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
+	adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
+	adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
+	adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
+	adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
+	adev->gfx.rlc.reg_list_format_direct_reg_list_length =
+			le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
+}
+
 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 {
 	const char *chip_name;
@@ -412,6 +479,8 @@ static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 	const struct rlc_firmware_header_v2_0 *rlc_hdr;
 	unsigned int *tmp = NULL;
 	unsigned int i = 0;
+	uint16_t version_major;
+	uint16_t version_minor;
 
 	DRM_DEBUG("\n");
 
@@ -422,6 +491,9 @@ static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 	case CHIP_VEGA12:
 		chip_name = "vega12";
 		break;
+	case CHIP_VEGA20:
+		chip_name = "vega20";
+		break;
 	case CHIP_RAVEN:
 		chip_name = "raven";
 		break;
@@ -468,6 +540,12 @@ static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 		goto out;
 	err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
 	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
+
+	version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
+	version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
+	if (version_major == 2 && version_minor == 1)
+		adev->gfx.rlc.is_rlc_v2_1 = true;
+
 	adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
 	adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
 	adev->gfx.rlc.save_and_restore_offset =
@@ -508,6 +586,9 @@ static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 	for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
 		adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
 
+	if (adev->gfx.rlc.is_rlc_v2_1)
+		gfx_v9_0_init_rlc_ext_microcode(adev);
+
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
 	err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
 	if (err)
@@ -566,6 +647,26 @@ static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 		adev->firmware.fw_size +=
 			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 
+		if (adev->gfx.rlc.is_rlc_v2_1) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
+
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
+
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
+		}
+
 		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
 		info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
 		info->fw = adev->gfx.mec_fw;
@@ -1013,9 +1114,10 @@ static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
 	.select_me_pipe_q = &gfx_v9_0_select_me_pipe_q
 };
 
-static void gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
+static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
 {
 	u32 gb_addr_config;
+	int err;
 
 	adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
 
@@ -1037,6 +1139,20 @@ static void gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
 		gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
 		DRM_INFO("fix gfx.config for vega12\n");
 		break;
+	case CHIP_VEGA20:
+		adev->gfx.config.max_hw_contexts = 8;
+		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
+		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
+		gb_addr_config &= ~0xf3e777ff;
+		gb_addr_config |= 0x22014042;
+		/* check vbios table if gpu info is not available */
+		err = amdgpu_atomfirmware_get_gfx_info(adev);
+		if (err)
+			return err;
+		break;
 	case CHIP_RAVEN:
 		adev->gfx.config.max_hw_contexts = 8;
 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
@@ -1086,6 +1202,8 @@ static void gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
 					adev->gfx.config.gb_addr_config,
 					GB_ADDR_CONFIG,
 					PIPE_INTERLEAVE_SIZE));
+
+	return 0;
 }
 
 static int gfx_v9_0_ngg_create_buf(struct amdgpu_device *adev,
@@ -1319,6 +1437,7 @@ static int gfx_v9_0_sw_init(void *handle)
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 	case CHIP_RAVEN:
 		adev->gfx.mec.num_mec = 2;
 		break;
@@ -1446,7 +1565,9 @@ static int gfx_v9_0_sw_init(void *handle)
 
 	adev->gfx.ce_ram_size = 0x8000;
 
-	gfx_v9_0_gpu_early_init(adev);
+	r = gfx_v9_0_gpu_early_init(adev);
+	if (r)
+		return r;
 
 	r = gfx_v9_0_ngg_init(adev);
 	if (r)
@@ -1600,6 +1721,7 @@ static void gfx_v9_0_gpu_init(struct amdgpu_device *adev)
 
 	gfx_v9_0_setup_rb(adev);
 	gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
+	adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
 
 	/* XXX SH_MEM regs */
 	/* where to put LDS, scratch, GPUVM in FSA64 space */
@@ -1616,7 +1738,10 @@ static void gfx_v9_0_gpu_init(struct amdgpu_device *adev)
 			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
 					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
 			WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, tmp);
-			tmp = adev->gmc.shared_aperture_start >> 48;
+			tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
+				(adev->gmc.private_aperture_start >> 48));
+			tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
+				(adev->gmc.shared_aperture_start >> 48));
 			WREG32_SOC15(GC, 0, mmSH_MEM_BASES, tmp);
 		}
 	}
@@ -1708,55 +1833,42 @@ static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
 			adev->gfx.rlc.clear_state_size);
 }
 
-static void gfx_v9_0_parse_ind_reg_list(int *register_list_format,
+static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
 				int indirect_offset,
 				int list_size,
 				int *unique_indirect_regs,
 				int *unique_indirect_reg_count,
-				int max_indirect_reg_count,
 				int *indirect_start_offsets,
-				int *indirect_start_offsets_count,
-				int max_indirect_start_offsets_count)
+				int *indirect_start_offsets_count)
 {
 	int idx;
-	bool new_entry = true;
 
 	for (; indirect_offset < list_size; indirect_offset++) {
+		indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
+		*indirect_start_offsets_count = *indirect_start_offsets_count + 1;
 
-		if (new_entry) {
-			new_entry = false;
-			indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
-			*indirect_start_offsets_count = *indirect_start_offsets_count + 1;
-			BUG_ON(*indirect_start_offsets_count >= max_indirect_start_offsets_count);
-		}
+		while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
+			indirect_offset += 2;
 
-		if (register_list_format[indirect_offset] == 0xFFFFFFFF) {
-			new_entry = true;
-			continue;
-		}
+			/* look for the matching indice */
+			for (idx = 0; idx < *unique_indirect_reg_count; idx++) {
+				if (unique_indirect_regs[idx] ==
+					register_list_format[indirect_offset] ||
+					!unique_indirect_regs[idx])
+					break;
+			}
 
-		indirect_offset += 2;
+			BUG_ON(idx >= *unique_indirect_reg_count);
 
-		/* look for the matching indice */
-		for (idx = 0; idx < *unique_indirect_reg_count; idx++) {
-			if (unique_indirect_regs[idx] ==
-				register_list_format[indirect_offset])
-				break;
-		}
+			if (!unique_indirect_regs[idx])
+				unique_indirect_regs[idx] = register_list_format[indirect_offset];
 
-		if (idx >= *unique_indirect_reg_count) {
-			unique_indirect_regs[*unique_indirect_reg_count] =
-				register_list_format[indirect_offset];
-			idx = *unique_indirect_reg_count;
-			*unique_indirect_reg_count = *unique_indirect_reg_count + 1;
-			BUG_ON(*unique_indirect_reg_count >= max_indirect_reg_count);
+			indirect_offset++;
 		}
-
-		register_list_format[indirect_offset] = idx;
 	}
 }
 
-static int gfx_v9_0_init_rlc_save_restore_list(struct amdgpu_device *adev)
+static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
 {
 	int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
 	int unique_indirect_reg_count = 0;
@@ -1765,7 +1877,7 @@ static int gfx_v9_0_init_rlc_save_restore_list(struct amdgpu_device *adev)
 	int indirect_start_offsets_count = 0;
 
 	int list_size = 0;
-	int i = 0;
+	int i = 0, j = 0;
 	u32 tmp = 0;
 
 	u32 *register_list_format =
@@ -1776,15 +1888,14 @@ static int gfx_v9_0_init_rlc_save_restore_list(struct amdgpu_device *adev)
 		adev->gfx.rlc.reg_list_format_size_bytes);
 
 	/* setup unique_indirect_regs array and indirect_start_offsets array */
-	gfx_v9_0_parse_ind_reg_list(register_list_format,
-				GFX9_RLC_FORMAT_DIRECT_REG_LIST_LENGTH,
-				adev->gfx.rlc.reg_list_format_size_bytes >> 2,
-				unique_indirect_regs,
-				&unique_indirect_reg_count,
-				ARRAY_SIZE(unique_indirect_regs),
-				indirect_start_offsets,
-				&indirect_start_offsets_count,
-				ARRAY_SIZE(indirect_start_offsets));
+	unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
+	gfx_v9_1_parse_ind_reg_list(register_list_format,
+				    adev->gfx.rlc.reg_list_format_direct_reg_list_length,
+				    adev->gfx.rlc.reg_list_format_size_bytes >> 2,
+				    unique_indirect_regs,
+				    &unique_indirect_reg_count,
+				    indirect_start_offsets,
+				    &indirect_start_offsets_count);
 
 	/* enable auto inc in case it is disabled */
 	tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
@@ -1798,19 +1909,37 @@ static int gfx_v9_0_init_rlc_save_restore_list(struct amdgpu_device *adev)
 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
 			adev->gfx.rlc.register_restore[i]);
 
-	/* load direct register */
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 0);
-	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
-		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
-			adev->gfx.rlc.register_restore[i]);
-
 	/* load indirect register */
 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
 		adev->gfx.rlc.reg_list_format_start);
-	for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++)
+
+	/* direct register portion */
+	for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
 			register_list_format[i]);
 
+	/* indirect register portion */
+	while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
+		if (register_list_format[i] == 0xFFFFFFFF) {
+			WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
+			continue;
+		}
+
+		WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
+		WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
+
+		for (j = 0; j < unique_indirect_reg_count; j++) {
+			if (register_list_format[i] == unique_indirect_regs[j]) {
+				WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
+				break;
+			}
+		}
+
+		BUG_ON(j >= unique_indirect_reg_count);
+
+		i++;
+	}
+
 	/* set save/restore list size */
 	list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
 	list_size = list_size >> 1;
@@ -1823,14 +1952,19 @@ static int gfx_v9_0_init_rlc_save_restore_list(struct amdgpu_device *adev)
 		adev->gfx.rlc.starting_offsets_start);
 	for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
-			indirect_start_offsets[i]);
+		       indirect_start_offsets[i]);
 
 	/* load unique indirect regs*/
 	for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
-		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) + i,
-			unique_indirect_regs[i] & 0x3FFFF);
-		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) + i,
-			unique_indirect_regs[i] >> 20);
+		if (unique_indirect_regs[i] != 0) {
+			WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
+			       + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
+			       unique_indirect_regs[i] & 0x3FFFF);
+
+			WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
+			       + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
+			       unique_indirect_regs[i] >> 20);
+		}
 	}
 
 	kfree(register_list_format);
@@ -2010,6 +2144,9 @@ static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *ad
 
 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
 {
+	if (!adev->gfx.rlc.is_rlc_v2_1)
+		return;
+
 	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
 			      AMD_PG_SUPPORT_GFX_SMG |
 			      AMD_PG_SUPPORT_GFX_DMG |
@@ -2017,27 +2154,12 @@ static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
 			      AMD_PG_SUPPORT_GDS |
 			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
 		gfx_v9_0_init_csb(adev);
-		gfx_v9_0_init_rlc_save_restore_list(adev);
+		gfx_v9_1_init_rlc_save_restore_list(adev);
 		gfx_v9_0_enable_save_restore_machine(adev);
 
-		if (adev->asic_type == CHIP_RAVEN) {
-			WREG32(mmRLC_JUMP_TABLE_RESTORE,
-				adev->gfx.rlc.cp_table_gpu_addr >> 8);
-			gfx_v9_0_init_gfx_power_gating(adev);
-
-			if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
-				gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
-				gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
-			} else {
-				gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
-				gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
-			}
-
-			if (adev->pg_flags & AMD_PG_SUPPORT_CP)
-				gfx_v9_0_enable_cp_power_gating(adev, true);
-			else
-				gfx_v9_0_enable_cp_power_gating(adev, false);
-		}
+		WREG32(mmRLC_JUMP_TABLE_RESTORE,
+		       adev->gfx.rlc.cp_table_gpu_addr >> 8);
+		gfx_v9_0_init_gfx_power_gating(adev);
 	}
 }
 
@@ -3061,6 +3183,9 @@ static int gfx_v9_0_hw_fini(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int i;
 
+	amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_GFX,
+					       AMD_PG_STATE_UNGATE);
+
 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
 
@@ -3279,6 +3404,11 @@ static int gfx_v9_0_late_init(void *handle)
 	if (r)
 		return r;
 
+	r = amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_GFX,
+						   AMD_PG_STATE_GATE);
+	if (r)
+		return r;
+
 	return 0;
 }
 
@@ -3339,8 +3469,7 @@ static void gfx_v9_0_exit_rlc_safe_mode(struct amdgpu_device *adev)
 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
 						bool enable)
 {
-	/* TODO: double check if we need to perform under safe mdoe */
-	/* gfx_v9_0_enter_rlc_safe_mode(adev); */
+	gfx_v9_0_enter_rlc_safe_mode(adev);
 
 	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
 		gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
@@ -3351,7 +3480,7 @@ static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
 		gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
 	}
 
-	/* gfx_v9_0_exit_rlc_safe_mode(adev); */
+	gfx_v9_0_exit_rlc_safe_mode(adev);
 }
 
 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
@@ -3605,6 +3734,7 @@ static int gfx_v9_0_set_clockgating_state(void *handle,
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 	case CHIP_RAVEN:
 		gfx_v9_0_update_gfx_clock_gating(adev,
 						 state == AMD_CG_STATE_GATE ? true : false);
@@ -3742,7 +3872,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 	}
 
 	amdgpu_ring_write(ring, header);
-BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
+	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
 	amdgpu_ring_write(ring,
 #ifdef __BIG_ENDIAN
 		(2 << 0) |
@@ -3774,13 +3904,16 @@ static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 {
 	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
 	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
+	bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
 
 	/* RELEASE_MEM - flush caches, send int */
 	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
-	amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
-				 EOP_TC_ACTION_EN |
-				 EOP_TC_WB_ACTION_EN |
-				 EOP_TC_MD_ACTION_EN |
+	amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
+					       EOP_TC_NC_ACTION_EN) :
+					      (EOP_TCL1_ACTION_EN |
+					       EOP_TC_ACTION_EN |
+					       EOP_TC_WB_ACTION_EN |
+					       EOP_TC_MD_ACTION_EN)) |
 				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
 				 EVENT_INDEX(5)));
 	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
@@ -4137,6 +4270,20 @@ static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 	gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
 }
 
+static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
+						  uint32_t reg0, uint32_t reg1,
+						  uint32_t ref, uint32_t mask)
+{
+	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
+
+	if (amdgpu_sriov_vf(ring->adev))
+		gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
+				      ref, mask, 0x20);
+	else
+		amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
+							   ref, mask);
+}
+
 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
 						 enum amdgpu_interrupt_state state)
 {
@@ -4458,6 +4605,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
 	.emit_tmz = gfx_v9_0_ring_emit_tmz,
 	.emit_wreg = gfx_v9_0_ring_emit_wreg,
 	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
 };
 
 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
@@ -4492,6 +4640,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
 	.set_priority = gfx_v9_0_ring_set_priority_compute,
 	.emit_wreg = gfx_v9_0_ring_emit_wreg,
 	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
 };
 
 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
@@ -4522,6 +4671,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
 	.emit_rreg = gfx_v9_0_ring_emit_rreg,
 	.emit_wreg = gfx_v9_0_ring_emit_wreg,
 	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
 };
 
 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
@@ -4577,6 +4727,7 @@ static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 	case CHIP_RAVEN:
 		adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
 		break;
@@ -4686,6 +4837,7 @@ static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
 
 	cu_info->number = active_cu_number;
 	cu_info->ao_cu_mask = ao_cu_mask;
+	cu_info->simd_per_cu = NUM_SIMD_PER_CU;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index 5617cf62c566..79f9ac29019b 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -819,12 +819,33 @@ static int gmc_v6_0_late_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	amdgpu_bo_late_init(adev);
+
 	if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
 		return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
 	else
 		return 0;
 }
 
+static unsigned gmc_v6_0_get_vbios_fb_size(struct amdgpu_device *adev)
+{
+	u32 d1vga_control = RREG32(mmD1VGA_CONTROL);
+	unsigned size;
+
+	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
+		size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */
+	} else {
+		u32 viewport = RREG32(mmVIEWPORT_SIZE);
+		size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
+			REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) *
+			4);
+	}
+	/* return 0 if the pre-OS buffer uses up most of vram */
+	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))
+		return 0;
+	return size;
+}
+
 static int gmc_v6_0_sw_init(void *handle)
 {
 	int r;
@@ -851,8 +872,6 @@ static int gmc_v6_0_sw_init(void *handle)
 
 	adev->gmc.mc_mask = 0xffffffffffULL;
 
-	adev->gmc.stolen_size = 256 * 1024;
-
 	adev->need_dma32 = false;
 	dma_bits = adev->need_dma32 ? 32 : 40;
 	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
@@ -878,6 +897,8 @@ static int gmc_v6_0_sw_init(void *handle)
 	if (r)
 		return r;
 
+	adev->gmc.stolen_size = gmc_v6_0_get_vbios_fb_size(adev);
+
 	r = amdgpu_bo_init(adev);
 	if (r)
 		return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 80054f36e487..7147bfe25a23 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -958,12 +958,33 @@ static int gmc_v7_0_late_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	amdgpu_bo_late_init(adev);
+
 	if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
 		return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
 	else
 		return 0;
 }
 
+static unsigned gmc_v7_0_get_vbios_fb_size(struct amdgpu_device *adev)
+{
+	u32 d1vga_control = RREG32(mmD1VGA_CONTROL);
+	unsigned size;
+
+	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
+		size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */
+	} else {
+		u32 viewport = RREG32(mmVIEWPORT_SIZE);
+		size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
+			REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) *
+			4);
+	}
+	/* return 0 if the pre-OS buffer uses up most of vram */
+	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))
+		return 0;
+	return size;
+}
+
 static int gmc_v7_0_sw_init(void *handle)
 {
 	int r;
@@ -998,8 +1019,6 @@ static int gmc_v7_0_sw_init(void *handle)
 	 */
 	adev->gmc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
 
-	adev->gmc.stolen_size = 256 * 1024;
-
 	/* set DMA mask + need_dma32 flags.
 	 * PCIE - can handle 40-bits.
 	 * IGP - can handle 40-bits
@@ -1030,6 +1049,8 @@ static int gmc_v7_0_sw_init(void *handle)
 	if (r)
 		return r;
 
+	adev->gmc.stolen_size = gmc_v7_0_get_vbios_fb_size(adev);
+
 	/* Memory manager */
 	r = amdgpu_bo_init(adev);
 	if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index d71d4cb68f9c..1edbe6b477b5 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -138,6 +138,7 @@ static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
 		break;
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 		amdgpu_device_program_register_sequence(adev,
 							golden_settings_polaris11_a11,
 							ARRAY_SIZE(golden_settings_polaris11_a11));
@@ -231,6 +232,7 @@ static int gmc_v8_0_init_microcode(struct amdgpu_device *adev)
 	case CHIP_FIJI:
 	case CHIP_CARRIZO:
 	case CHIP_STONEY:
+	case CHIP_VEGAM:
 		return 0;
 	default: BUG();
 	}
@@ -567,9 +569,10 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
 	/* set the gart size */
 	if (amdgpu_gart_size == -1) {
 		switch (adev->asic_type) {
-		case CHIP_POLARIS11: /* all engines support GPUVM */
 		case CHIP_POLARIS10: /* all engines support GPUVM */
+		case CHIP_POLARIS11: /* all engines support GPUVM */
 		case CHIP_POLARIS12: /* all engines support GPUVM */
+		case CHIP_VEGAM:     /* all engines support GPUVM */
 		default:
 			adev->gmc.gart_size = 256ULL << 20;
 			break;
@@ -1049,12 +1052,33 @@ static int gmc_v8_0_late_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	amdgpu_bo_late_init(adev);
+
 	if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
 		return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
 	else
 		return 0;
 }
 
+static unsigned gmc_v8_0_get_vbios_fb_size(struct amdgpu_device *adev)
+{
+	u32 d1vga_control = RREG32(mmD1VGA_CONTROL);
+	unsigned size;
+
+	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
+		size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */
+	} else {
+		u32 viewport = RREG32(mmVIEWPORT_SIZE);
+		size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
+			REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) *
+			4);
+	}
+	/* return 0 if the pre-OS buffer uses up most of vram */
+	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))
+		return 0;
+	return size;
+}
+
 #define mmMC_SEQ_MISC0_FIJI 0xA71
 
 static int gmc_v8_0_sw_init(void *handle)
@@ -1068,7 +1092,8 @@ static int gmc_v8_0_sw_init(void *handle)
 	} else {
 		u32 tmp;
 
-		if (adev->asic_type == CHIP_FIJI)
+		if ((adev->asic_type == CHIP_FIJI) ||
+		    (adev->asic_type == CHIP_VEGAM))
 			tmp = RREG32(mmMC_SEQ_MISC0_FIJI);
 		else
 			tmp = RREG32(mmMC_SEQ_MISC0);
@@ -1096,8 +1121,6 @@ static int gmc_v8_0_sw_init(void *handle)
 	 */
 	adev->gmc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
 
-	adev->gmc.stolen_size = 256 * 1024;
-
 	/* set DMA mask + need_dma32 flags.
 	 * PCIE - can handle 40-bits.
 	 * IGP - can handle 40-bits
@@ -1128,6 +1151,8 @@ static int gmc_v8_0_sw_init(void *handle)
 	if (r)
 		return r;
 
+	adev->gmc.stolen_size = gmc_v8_0_get_vbios_fb_size(adev);
+
 	/* Memory manager */
 	r = amdgpu_bo_init(adev);
 	if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index e687363900bb..3c0a85d4e4ab 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -43,19 +43,13 @@
 #include "gfxhub_v1_0.h"
 #include "mmhub_v1_0.h"
 
-#define mmDF_CS_AON0_DramBaseAddress0                                                                  0x0044
-#define mmDF_CS_AON0_DramBaseAddress0_BASE_IDX                                                         0
-//DF_CS_AON0_DramBaseAddress0
-#define DF_CS_AON0_DramBaseAddress0__AddrRngVal__SHIFT                                                        0x0
-#define DF_CS_AON0_DramBaseAddress0__LgcyMmioHoleEn__SHIFT                                                    0x1
-#define DF_CS_AON0_DramBaseAddress0__IntLvNumChan__SHIFT                                                      0x4
-#define DF_CS_AON0_DramBaseAddress0__IntLvAddrSel__SHIFT                                                      0x8
-#define DF_CS_AON0_DramBaseAddress0__DramBaseAddr__SHIFT                                                      0xc
-#define DF_CS_AON0_DramBaseAddress0__AddrRngVal_MASK                                                          0x00000001L
-#define DF_CS_AON0_DramBaseAddress0__LgcyMmioHoleEn_MASK                                                      0x00000002L
-#define DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK                                                        0x000000F0L
-#define DF_CS_AON0_DramBaseAddress0__IntLvAddrSel_MASK                                                        0x00000700L
-#define DF_CS_AON0_DramBaseAddress0__DramBaseAddr_MASK                                                        0xFFFFF000L
+/* add these here since we already include dce12 headers and these are for DCN */
+#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
+#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
 
 /* XXX Move this macro to VEGA10 header file, which is like vid.h for VI.*/
 #define AMDGPU_NUM_OF_VMIDS			8
@@ -385,11 +379,9 @@ static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid),
 			      upper_32_bits(pd_addr));
 
-	amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_req + eng, req);
-
-	/* wait for the invalidate to complete */
-	amdgpu_ring_emit_reg_wait(ring, hub->vm_inv_eng0_ack + eng,
-				  1 << vmid, 1 << vmid);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
 
 	return pd_addr;
 }
@@ -556,8 +548,7 @@ static int gmc_v9_0_early_init(void *handle)
 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
 	adev->gmc.shared_aperture_end =
 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
-	adev->gmc.private_aperture_start =
-		adev->gmc.shared_aperture_end + 1;
+	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
 	adev->gmc.private_aperture_end =
 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
 
@@ -659,6 +650,11 @@ static int gmc_v9_0_late_init(void *handle)
 	unsigned i;
 	int r;
 
+	/*
+	 * TODO - Uncomment once GART corruption issue is fixed.
+	 */
+	/* amdgpu_bo_late_init(adev); */
+
 	for(i = 0; i < adev->num_rings; ++i) {
 		struct amdgpu_ring *ring = adev->rings[i];
 		unsigned vmhub = ring->funcs->vmhub;
@@ -679,6 +675,7 @@ static int gmc_v9_0_late_init(void *handle)
 			DRM_INFO("ECC is active.\n");
 		} else if (r == 0) {
 			DRM_INFO("ECC is not present.\n");
+			adev->df_funcs->enable_ecc_force_par_wr_rmw(adev, false);
 		} else {
 			DRM_ERROR("gmc_v9_0_ecc_available() failed. r: %d\n", r);
 			return r;
@@ -697,10 +694,7 @@ static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
 	amdgpu_device_vram_location(adev, &adev->gmc, base);
 	amdgpu_device_gart_location(adev, mc);
 	/* base offset of vram pages */
-	if (adev->flags & AMD_IS_APU)
-		adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
-	else
-		adev->vm_manager.vram_base_offset = 0;
+	adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
 }
 
 /**
@@ -714,7 +708,6 @@ static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
  */
 static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
 {
-	u32 tmp;
 	int chansize, numchan;
 	int r;
 
@@ -727,39 +720,7 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
 		else
 			chansize = 128;
 
-		tmp = RREG32_SOC15(DF, 0, mmDF_CS_AON0_DramBaseAddress0);
-		tmp &= DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK;
-		tmp >>= DF_CS_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
-		switch (tmp) {
-		case 0:
-		default:
-			numchan = 1;
-			break;
-		case 1:
-			numchan = 2;
-			break;
-		case 2:
-			numchan = 0;
-			break;
-		case 3:
-			numchan = 4;
-			break;
-		case 4:
-			numchan = 0;
-			break;
-		case 5:
-			numchan = 8;
-			break;
-		case 6:
-			numchan = 0;
-			break;
-		case 7:
-			numchan = 16;
-			break;
-		case 8:
-			numchan = 2;
-			break;
-		}
+		numchan = adev->df_funcs->get_hbm_channel_number(adev);
 		adev->gmc.vram_width = numchan * chansize;
 	}
 
@@ -792,6 +753,7 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
 		switch (adev->asic_type) {
 		case CHIP_VEGA10:  /* all engines support GPUVM */
 		case CHIP_VEGA12:  /* all engines support GPUVM */
+		case CHIP_VEGA20:
 		default:
 			adev->gmc.gart_size = 512ULL << 20;
 			break;
@@ -826,6 +788,52 @@ static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
 	return amdgpu_gart_table_vram_alloc(adev);
 }
 
+static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
+{
+#if 0
+	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
+#endif
+	unsigned size;
+
+	/*
+	 * TODO Remove once GART corruption is resolved
+	 * Check related code in gmc_v9_0_sw_fini
+	 * */
+	size = 9 * 1024 * 1024;
+
+#if 0
+	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
+		size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */
+	} else {
+		u32 viewport;
+
+		switch (adev->asic_type) {
+		case CHIP_RAVEN:
+			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
+			size = (REG_GET_FIELD(viewport,
+					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
+				REG_GET_FIELD(viewport,
+					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
+				4);
+			break;
+		case CHIP_VEGA10:
+		case CHIP_VEGA12:
+		default:
+			viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
+			size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
+				REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
+				4);
+			break;
+		}
+	}
+	/* return 0 if the pre-OS buffer uses up most of vram */
+	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))
+		return 0;
+
+#endif
+	return size;
+}
+
 static int gmc_v9_0_sw_init(void *handle)
 {
 	int r;
@@ -851,6 +859,7 @@ static int gmc_v9_0_sw_init(void *handle)
 		break;
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 		/*
 		 * To fulfill 4-level page support,
 		 * vm size is 256TB (48bit), maximum size of Vega10,
@@ -877,12 +886,6 @@ static int gmc_v9_0_sw_init(void *handle)
 	 */
 	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
 
-	/*
-	 * It needs to reserve 8M stolen memory for vega10
-	 * TODO: Figure out how to avoid that...
-	 */
-	adev->gmc.stolen_size = 8 * 1024 * 1024;
-
 	/* set DMA mask + need_dma32 flags.
 	 * PCIE - can handle 44-bits.
 	 * IGP - can handle 44-bits
@@ -907,6 +910,8 @@ static int gmc_v9_0_sw_init(void *handle)
 	if (r)
 		return r;
 
+	adev->gmc.stolen_size = gmc_v9_0_get_vbios_fb_size(adev);
+
 	/* Memory manager */
 	r = amdgpu_bo_init(adev);
 	if (r)
@@ -950,6 +955,18 @@ static int gmc_v9_0_sw_fini(void *handle)
 	amdgpu_gem_force_release(adev);
 	amdgpu_vm_manager_fini(adev);
 	gmc_v9_0_gart_fini(adev);
+
+	/*
+	* TODO:
+	* Currently there is a bug where some memory client outside
+	* of the driver writes to first 8M of VRAM on S3 resume,
+	* this overrides GART which by default gets placed in first 8M and
+	* causes VM_FAULTS once GTT is accessed.
+	* Keep the stolen memory reservation until the while this is not solved.
+	* Also check code in gmc_v9_0_get_vbios_fb_size and gmc_v9_0_late_init
+	*/
+	amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
+
 	amdgpu_bo_fini(adev);
 
 	return 0;
@@ -960,6 +977,7 @@ static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
+	case CHIP_VEGA20:
 		soc15_program_register_sequence(adev,
 						golden_settings_mmhub_1_0_0,
 						ARRAY_SIZE(golden_settings_mmhub_1_0_0));
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index 26ba984ab2b7..17f7f074cedc 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -2817,7 +2817,7 @@ static int kv_dpm_init(struct amdgpu_device *adev)
 		pi->caps_tcp_ramping = true;
 	}
 
-	if (amdgpu_pp_feature_mask & SCLK_DEEP_SLEEP_MASK)
+	if (adev->powerplay.pp_feature & PP_SCLK_DEEP_SLEEP_MASK)
 		pi->caps_sclk_ds = true;
 	else
 		pi->caps_sclk_ds = false;
@@ -2974,7 +2974,7 @@ static int kv_dpm_late_init(void *handle)
 	/* powerdown unused blocks for now */
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	if (!amdgpu_dpm)
+	if (!adev->pm.dpm_enabled)
 		return 0;
 
 	kv_dpm_powergate_acp(adev, true);
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
index 43f925773b57..3d53c4413f13 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
@@ -734,6 +734,7 @@ int mmhub_v1_0_set_clockgating(struct amdgpu_device *adev,
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 	case CHIP_RAVEN:
 		mmhub_v1_0_update_medium_grain_clock_gating(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index 493348672475..078f70faedcb 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -260,8 +260,10 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
 	} while (timeout > 1);
 
 flr_done:
-	if (locked)
+	if (locked) {
+		adev->in_gpu_reset = 0;
 		mutex_unlock(&adev->lock_reset);
+	}
 
 	/* Trigger recovery for world switch failure if no TDR */
 	if (amdgpu_lockup_timeout == 0)
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
index df34dc79d444..365517c0121e 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
@@ -34,10 +34,19 @@
 #define smnCPM_CONTROL                                                                                  0x11180460
 #define smnPCIE_CNTL2                                                                                   0x11180070
 
+/* vega20 */
+#define mmRCC_DEV0_EPF0_STRAP0_VG20                                                                         0x0011
+#define mmRCC_DEV0_EPF0_STRAP0_VG20_BASE_IDX                                                                2
+
 static u32 nbio_v7_0_get_rev_id(struct amdgpu_device *adev)
 {
         u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0);
 
+	if (adev->asic_type == CHIP_VEGA20)
+		tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0_VG20);
+	else
+		tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0);
+
 	tmp &= RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK;
 	tmp >>= RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT;
 
@@ -75,10 +84,14 @@ static void nbio_v7_0_sdma_doorbell_range(struct amdgpu_device *adev, int instan
 			SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA1_DOORBELL_RANGE);
 
 	u32 doorbell_range = RREG32(reg);
+	u32 range = 2;
+
+	if (adev->asic_type == CHIP_VEGA20)
+		range = 8;
 
 	if (use_doorbell) {
 		doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, OFFSET, doorbell_index);
-		doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, 2);
+		doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, range);
 	} else
 		doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, 0);
 
@@ -133,6 +146,9 @@ static void nbio_v7_0_update_medium_grain_clock_gating(struct amdgpu_device *ade
 {
 	uint32_t def, data;
 
+	if (adev->asic_type == CHIP_VEGA20)
+		return;
+
 	/* NBIF_MGCG_CTRL_LCLK */
 	def = data = RREG32_PCIE(smnNBIF_MGCG_CTRL_LCLK);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
index 8da6da90b1c9..0cf48d26c676 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
@@ -40,11 +40,20 @@ enum psp_gfx_crtl_cmd_id
     GFX_CTRL_CMD_ID_INIT_GPCOM_RING = 0x00020000,   /* initialize GPCOM ring */
     GFX_CTRL_CMD_ID_DESTROY_RINGS   = 0x00030000,   /* destroy rings */
     GFX_CTRL_CMD_ID_CAN_INIT_RINGS  = 0x00040000,   /* is it allowed to initialized the rings */
+    GFX_CTRL_CMD_ID_ENABLE_INT      = 0x00050000,   /* enable PSP-to-Gfx interrupt */
+    GFX_CTRL_CMD_ID_DISABLE_INT     = 0x00060000,   /* disable PSP-to-Gfx interrupt */
+    GFX_CTRL_CMD_ID_MODE1_RST       = 0x00070000,   /* trigger the Mode 1 reset */
 
     GFX_CTRL_CMD_ID_MAX             = 0x000F0000,   /* max command ID */
 };
 
 
+/*-----------------------------------------------------------------------------
+    NOTE:   All physical addresses used in this interface are actually
+            GPU Virtual Addresses.
+*/
+
+
 /* Control registers of the TEE Gfx interface. These are located in
 *  SRBM-to-PSP mailbox registers (total 8 registers).
 */
@@ -55,8 +64,8 @@ struct psp_gfx_ctrl
     volatile uint32_t   rbi_rptr;         /* +8   Read pointer (index) of RBI ring */
     volatile uint32_t   gpcom_wptr;       /* +12  Write pointer (index) of GPCOM ring */
     volatile uint32_t   gpcom_rptr;       /* +16  Read pointer (index) of GPCOM ring */
-    volatile uint32_t   ring_addr_lo;     /* +20  bits [31:0] of physical address of ring buffer */
-    volatile uint32_t   ring_addr_hi;     /* +24  bits [63:32] of physical address of ring buffer */
+    volatile uint32_t   ring_addr_lo;     /* +20  bits [31:0] of GPU Virtual of ring buffer (VMID=0)*/
+    volatile uint32_t   ring_addr_hi;     /* +24  bits [63:32] of GPU Virtual of ring buffer (VMID=0) */
     volatile uint32_t   ring_buf_size;    /* +28  Ring buffer size (in bytes) */
 
 };
@@ -78,6 +87,8 @@ enum psp_gfx_cmd_id
     GFX_CMD_ID_LOAD_ASD     = 0x00000004,   /* load ASD Driver */
     GFX_CMD_ID_SETUP_TMR    = 0x00000005,   /* setup TMR region */
     GFX_CMD_ID_LOAD_IP_FW   = 0x00000006,   /* load HW IP FW */
+    GFX_CMD_ID_DESTROY_TMR  = 0x00000007,   /* destroy TMR region */
+    GFX_CMD_ID_SAVE_RESTORE = 0x00000008,   /* save/restore HW IP FW */
 
 };
 
@@ -85,11 +96,11 @@ enum psp_gfx_cmd_id
 /* Command to load Trusted Application binary into PSP OS. */
 struct psp_gfx_cmd_load_ta
 {
-    uint32_t        app_phy_addr_lo;        /* bits [31:0] of the physical address of the TA binary (must be 4 KB aligned) */
-    uint32_t        app_phy_addr_hi;        /* bits [63:32] of the physical address of the TA binary */
+    uint32_t        app_phy_addr_lo;        /* bits [31:0] of the GPU Virtual address of the TA binary (must be 4 KB aligned) */
+    uint32_t        app_phy_addr_hi;        /* bits [63:32] of the GPU Virtual address of the TA binary */
     uint32_t        app_len;                /* length of the TA binary in bytes */
-    uint32_t        cmd_buf_phy_addr_lo;    /* bits [31:0] of the physical address of CMD buffer (must be 4 KB aligned) */
-    uint32_t        cmd_buf_phy_addr_hi;    /* bits [63:32] of the physical address of CMD buffer */
+    uint32_t        cmd_buf_phy_addr_lo;    /* bits [31:0] of the GPU Virtual address of CMD buffer (must be 4 KB aligned) */
+    uint32_t        cmd_buf_phy_addr_hi;    /* bits [63:32] of the GPU Virtual address of CMD buffer */
     uint32_t        cmd_buf_len;            /* length of the CMD buffer in bytes; must be multiple of 4 KB */
 
     /* Note: CmdBufLen can be set to 0. In this case no persistent CMD buffer is provided
@@ -111,8 +122,8 @@ struct psp_gfx_cmd_unload_ta
 */
 struct psp_gfx_buf_desc
 {
-    uint32_t        buf_phy_addr_lo;       /* bits [31:0] of physical address of the buffer (must be 4 KB aligned) */
-    uint32_t        buf_phy_addr_hi;       /* bits [63:32] of physical address of the buffer */
+    uint32_t        buf_phy_addr_lo;       /* bits [31:0] of GPU Virtual address of the buffer (must be 4 KB aligned) */
+    uint32_t        buf_phy_addr_hi;       /* bits [63:32] of GPU Virtual address of the buffer */
     uint32_t        buf_size;              /* buffer size in bytes (must be multiple of 4 KB and no bigger than 64 MB) */
 
 };
@@ -145,8 +156,8 @@ struct psp_gfx_cmd_invoke_cmd
 /* Command to setup TMR region. */
 struct psp_gfx_cmd_setup_tmr
 {
-    uint32_t        buf_phy_addr_lo;       /* bits [31:0] of physical address of TMR buffer (must be 4 KB aligned) */
-    uint32_t        buf_phy_addr_hi;       /* bits [63:32] of physical address of TMR buffer */
+    uint32_t        buf_phy_addr_lo;       /* bits [31:0] of GPU Virtual address of TMR buffer (must be 4 KB aligned) */
+    uint32_t        buf_phy_addr_hi;       /* bits [63:32] of GPU Virtual address of TMR buffer */
     uint32_t        buf_size;              /* buffer size in bytes (must be multiple of 4 KB) */
 
 };
@@ -174,18 +185,32 @@ enum psp_gfx_fw_type
     GFX_FW_TYPE_ISP         = 16,
     GFX_FW_TYPE_ACP         = 17,
     GFX_FW_TYPE_SMU         = 18,
+    GFX_FW_TYPE_MMSCH       = 19,
+    GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM        = 20,
+    GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM        = 21,
+    GFX_FW_TYPE_RLC_RESTORE_LIST_CNTL           = 22,
+    GFX_FW_TYPE_MAX         = 23
 };
 
 /* Command to load HW IP FW. */
 struct psp_gfx_cmd_load_ip_fw
 {
-    uint32_t                fw_phy_addr_lo;    /* bits [31:0] of physical address of FW location (must be 4 KB aligned) */
-    uint32_t                fw_phy_addr_hi;    /* bits [63:32] of physical address of FW location */
+    uint32_t                fw_phy_addr_lo;    /* bits [31:0] of GPU Virtual address of FW location (must be 4 KB aligned) */
+    uint32_t                fw_phy_addr_hi;    /* bits [63:32] of GPU Virtual address of FW location */
     uint32_t                fw_size;           /* FW buffer size in bytes */
     enum psp_gfx_fw_type    fw_type;           /* FW type */
 
 };
 
+/* Command to save/restore HW IP FW. */
+struct psp_gfx_cmd_save_restore_ip_fw
+{
+    uint32_t                save_fw;              /* if set, command is used for saving fw otherwise for resetoring*/
+    uint32_t                save_restore_addr_lo; /* bits [31:0] of FB address of GART memory used as save/restore buffer (must be 4 KB aligned) */
+    uint32_t                save_restore_addr_hi; /* bits [63:32] of FB address of GART memory used as save/restore buffer */
+    uint32_t                buf_size;             /* Size of the save/restore buffer in bytes */
+    enum psp_gfx_fw_type    fw_type;              /* FW type */
+};
 
 /* All GFX ring buffer commands. */
 union psp_gfx_commands
@@ -195,7 +220,7 @@ union psp_gfx_commands
     struct psp_gfx_cmd_invoke_cmd       cmd_invoke_cmd;
     struct psp_gfx_cmd_setup_tmr        cmd_setup_tmr;
     struct psp_gfx_cmd_load_ip_fw       cmd_load_ip_fw;
-
+    struct psp_gfx_cmd_save_restore_ip_fw cmd_save_restore_ip_fw;
 };
 
 
@@ -226,8 +251,8 @@ struct psp_gfx_cmd_resp
 
     /* These fields are used for RBI only. They are all 0 in GPCOM commands
     */
-    uint32_t        resp_buf_addr_lo;   /* +12 bits [31:0] of physical address of response buffer (must be 4 KB aligned) */
-    uint32_t        resp_buf_addr_hi;   /* +16 bits [63:32] of physical address of response buffer */
+    uint32_t        resp_buf_addr_lo;   /* +12 bits [31:0] of GPU Virtual address of response buffer (must be 4 KB aligned) */
+    uint32_t        resp_buf_addr_hi;   /* +16 bits [63:32] of GPU Virtual address of response buffer */
     uint32_t        resp_offset;        /* +20 offset within response buffer */
     uint32_t        resp_buf_size;      /* +24 total size of the response buffer in bytes */
 
@@ -251,19 +276,19 @@ struct psp_gfx_cmd_resp
 /* Structure of the Ring Buffer Frame */
 struct psp_gfx_rb_frame
 {
-    uint32_t    cmd_buf_addr_lo;    /* +0  bits [31:0] of physical address of command buffer (must be 4 KB aligned) */
-    uint32_t    cmd_buf_addr_hi;    /* +4  bits [63:32] of physical address of command buffer */
+    uint32_t    cmd_buf_addr_lo;    /* +0  bits [31:0] of GPU Virtual address of command buffer (must be 4 KB aligned) */
+    uint32_t    cmd_buf_addr_hi;    /* +4  bits [63:32] of GPU Virtual address of command buffer */
     uint32_t    cmd_buf_size;       /* +8  command buffer size in bytes */
-    uint32_t    fence_addr_lo;      /* +12 bits [31:0] of physical address of Fence for this frame */
-    uint32_t    fence_addr_hi;      /* +16 bits [63:32] of physical address of Fence for this frame */
+    uint32_t    fence_addr_lo;      /* +12 bits [31:0] of GPU Virtual address of Fence for this frame */
+    uint32_t    fence_addr_hi;      /* +16 bits [63:32] of GPU Virtual address of Fence for this frame */
     uint32_t    fence_value;        /* +20 Fence value */
     uint32_t    sid_lo;             /* +24 bits [31:0] of SID value (used only for RBI frames) */
     uint32_t    sid_hi;             /* +28 bits [63:32] of SID value (used only for RBI frames) */
     uint8_t     vmid;               /* +32 VMID value used for mapping of all addresses for this frame */
     uint8_t     frame_type;         /* +33 1: destory context frame, 0: all other frames; used only for RBI frames */
     uint8_t     reserved1[2];       /* +34 reserved, must be 0 */
-    uint32_t    reserved2[7];       /* +40 reserved, must be 0 */
-    /* total 64 bytes */
+    uint32_t    reserved2[7];       /* +36 reserved, must be 0 */
+                /* total 64 bytes */
 };
 
 #endif /* _PSP_TEE_GFX_IF_H_ */
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
index 8873d833a7f7..0ff136d02d9b 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
@@ -70,6 +70,15 @@ psp_v10_0_get_fw_type(struct amdgpu_firmware_info *ucode, enum psp_gfx_fw_type *
 	case AMDGPU_UCODE_ID_RLC_G:
 		*type = GFX_FW_TYPE_RLC_G;
 		break;
+	case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
+		*type = GFX_FW_TYPE_RLC_RESTORE_LIST_CNTL;
+		break;
+	case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
+		*type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
+		break;
+	case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
+		*type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
+		break;
 	case AMDGPU_UCODE_ID_SMC:
 		*type = GFX_FW_TYPE_SMU;
 		break;
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index 196e75def1f2..0c768e388ace 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -41,6 +41,9 @@ MODULE_FIRMWARE("amdgpu/vega10_sos.bin");
 MODULE_FIRMWARE("amdgpu/vega10_asd.bin");
 MODULE_FIRMWARE("amdgpu/vega12_sos.bin");
 MODULE_FIRMWARE("amdgpu/vega12_asd.bin");
+MODULE_FIRMWARE("amdgpu/vega20_sos.bin");
+MODULE_FIRMWARE("amdgpu/vega20_asd.bin");
+
 
 #define smnMP1_FIRMWARE_FLAGS 0x3010028
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index be20a387d961..aa9ab299fd32 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -62,6 +62,8 @@ MODULE_FIRMWARE("amdgpu/polaris11_sdma.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_sdma1.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_sdma.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_sdma1.bin");
+MODULE_FIRMWARE("amdgpu/vegam_sdma.bin");
+MODULE_FIRMWARE("amdgpu/vegam_sdma1.bin");
 
 
 static const u32 sdma_offsets[SDMA_MAX_INSTANCE] =
@@ -209,6 +211,7 @@ static void sdma_v3_0_init_golden_registers(struct amdgpu_device *adev)
 		break;
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 		amdgpu_device_program_register_sequence(adev,
 							golden_settings_polaris11_a11,
 							ARRAY_SIZE(golden_settings_polaris11_a11));
@@ -275,15 +278,18 @@ static int sdma_v3_0_init_microcode(struct amdgpu_device *adev)
 	case CHIP_FIJI:
 		chip_name = "fiji";
 		break;
-	case CHIP_POLARIS11:
-		chip_name = "polaris11";
-		break;
 	case CHIP_POLARIS10:
 		chip_name = "polaris10";
 		break;
+	case CHIP_POLARIS11:
+		chip_name = "polaris11";
+		break;
 	case CHIP_POLARIS12:
 		chip_name = "polaris12";
 		break;
+	case CHIP_VEGAM:
+		chip_name = "vegam";
+		break;
 	case CHIP_CARRIZO:
 		chip_name = "carrizo";
 		break;
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 399f876f9cad..ca53b3fba422 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -42,6 +42,8 @@ MODULE_FIRMWARE("amdgpu/vega10_sdma.bin");
 MODULE_FIRMWARE("amdgpu/vega10_sdma1.bin");
 MODULE_FIRMWARE("amdgpu/vega12_sdma.bin");
 MODULE_FIRMWARE("amdgpu/vega12_sdma1.bin");
+MODULE_FIRMWARE("amdgpu/vega20_sdma.bin");
+MODULE_FIRMWARE("amdgpu/vega20_sdma1.bin");
 MODULE_FIRMWARE("amdgpu/raven_sdma.bin");
 
 #define SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK  0x000000F8L
@@ -107,6 +109,28 @@ static const struct soc15_reg_golden golden_settings_sdma_4_1[] =
 	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003c0)
 };
 
+static const struct soc15_reg_golden golden_settings_sdma_4_2[] =
+{
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831d07),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff0, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003c0),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831d07),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CLK_CTRL, 0xffffffff, 0x3f000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_PAGE, 0x000003ff, 0x000003c0)
+};
+
 static const struct soc15_reg_golden golden_settings_sdma_rv1[] =
 {
 	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00000002),
@@ -139,6 +163,11 @@ static void sdma_v4_0_init_golden_registers(struct amdgpu_device *adev)
 						golden_settings_sdma_vg12,
 						ARRAY_SIZE(golden_settings_sdma_vg12));
 		break;
+	case CHIP_VEGA20:
+		soc15_program_register_sequence(adev,
+						golden_settings_sdma_4_2,
+						ARRAY_SIZE(golden_settings_sdma_4_2));
+		break;
 	case CHIP_RAVEN:
 		soc15_program_register_sequence(adev,
 						 golden_settings_sdma_4_1,
@@ -182,6 +211,9 @@ static int sdma_v4_0_init_microcode(struct amdgpu_device *adev)
 	case CHIP_VEGA12:
 		chip_name = "vega12";
 		break;
+	case CHIP_VEGA20:
+		chip_name = "vega20";
+		break;
 	case CHIP_RAVEN:
 		chip_name = "raven";
 		break;
@@ -360,6 +392,31 @@ static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring,
 
 }
 
+static void sdma_v4_0_wait_reg_mem(struct amdgpu_ring *ring,
+				   int mem_space, int hdp,
+				   uint32_t addr0, uint32_t addr1,
+				   uint32_t ref, uint32_t mask,
+				   uint32_t inv)
+{
+	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
+			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(hdp) |
+			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(mem_space) |
+			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
+	if (mem_space) {
+		/* memory */
+		amdgpu_ring_write(ring, addr0);
+		amdgpu_ring_write(ring, addr1);
+	} else {
+		/* registers */
+		amdgpu_ring_write(ring, addr0 << 2);
+		amdgpu_ring_write(ring, addr1 << 2);
+	}
+	amdgpu_ring_write(ring, ref); /* reference */
+	amdgpu_ring_write(ring, mask); /* mask */
+	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
+			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(inv)); /* retry count, poll interval */
+}
+
 /**
  * sdma_v4_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
  *
@@ -378,15 +435,10 @@ static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 	else
 		ref_and_mask = nbio_hf_reg->ref_and_mask_sdma1;
 
-	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
-			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
-			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
-	amdgpu_ring_write(ring, (adev->nbio_funcs->get_hdp_flush_done_offset(adev)) << 2);
-	amdgpu_ring_write(ring, (adev->nbio_funcs->get_hdp_flush_req_offset(adev)) << 2);
-	amdgpu_ring_write(ring, ref_and_mask); /* reference */
-	amdgpu_ring_write(ring, ref_and_mask); /* mask */
-	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
-			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
+	sdma_v4_0_wait_reg_mem(ring, 0, 1,
+			       adev->nbio_funcs->get_hdp_flush_done_offset(adev),
+			       adev->nbio_funcs->get_hdp_flush_req_offset(adev),
+			       ref_and_mask, ref_and_mask, 10);
 }
 
 /**
@@ -1114,16 +1166,10 @@ static void sdma_v4_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 	uint64_t addr = ring->fence_drv.gpu_addr;
 
 	/* wait for idle */
-	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
-			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
-			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
-			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
-	amdgpu_ring_write(ring, addr & 0xfffffffc);
-	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
-	amdgpu_ring_write(ring, seq); /* reference */
-	amdgpu_ring_write(ring, 0xffffffff); /* mask */
-	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
-			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
+	sdma_v4_0_wait_reg_mem(ring, 1, 0,
+			       addr & 0xfffffffc,
+			       upper_32_bits(addr) & 0xffffffff,
+			       seq, 0xffffffff, 4);
 }
 
 
@@ -1154,15 +1200,7 @@ static void sdma_v4_0_ring_emit_wreg(struct amdgpu_ring *ring,
 static void sdma_v4_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 					 uint32_t val, uint32_t mask)
 {
-	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
-			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
-			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
-	amdgpu_ring_write(ring, reg << 2);
-	amdgpu_ring_write(ring, 0);
-	amdgpu_ring_write(ring, val); /* reference */
-	amdgpu_ring_write(ring, mask); /* mask */
-	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
-			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
+	sdma_v4_0_wait_reg_mem(ring, 0, 0, reg, 0, val, mask, 10);
 }
 
 static int sdma_v4_0_early_init(void *handle)
@@ -1510,6 +1548,7 @@ static int sdma_v4_0_set_clockgating_state(void *handle,
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 	case CHIP_RAVEN:
 		sdma_v4_0_update_medium_grain_clock_gating(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
@@ -1605,6 +1644,7 @@ static const struct amdgpu_ring_funcs sdma_v4_0_ring_funcs = {
 	.pad_ib = sdma_v4_0_ring_pad_ib,
 	.emit_wreg = sdma_v4_0_ring_emit_wreg,
 	.emit_reg_wait = sdma_v4_0_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
 };
 
 static void sdma_v4_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index a675ec6d2811..c364ef94cc36 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -1252,6 +1252,12 @@ static void si_invalidate_hdp(struct amdgpu_device *adev,
 	}
 }
 
+static bool si_need_full_reset(struct amdgpu_device *adev)
+{
+	/* change this when we support soft reset */
+	return true;
+}
+
 static int si_get_pcie_lanes(struct amdgpu_device *adev)
 {
 	u32 link_width_cntl;
@@ -1332,6 +1338,7 @@ static const struct amdgpu_asic_funcs si_asic_funcs =
 	.get_config_memsize = &si_get_config_memsize,
 	.flush_hdp = &si_flush_hdp,
 	.invalidate_hdp = &si_invalidate_hdp,
+	.need_full_reset = &si_need_full_reset,
 };
 
 static uint32_t si_get_rev_id(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index 797d505bf9ee..b12d7c9d42a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -7580,7 +7580,7 @@ static int si_dpm_late_init(void *handle)
 	int ret;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	if (!amdgpu_dpm)
+	if (!adev->pm.dpm_enabled)
 		return 0;
 
 	ret = si_set_temperature_range(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 51cf8a30f6c2..68b4a22a8892 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -41,8 +41,6 @@
 #include "sdma1/sdma1_4_0_offset.h"
 #include "hdp/hdp_4_0_offset.h"
 #include "hdp/hdp_4_0_sh_mask.h"
-#include "mp/mp_9_0_offset.h"
-#include "mp/mp_9_0_sh_mask.h"
 #include "smuio/smuio_9_0_offset.h"
 #include "smuio/smuio_9_0_sh_mask.h"
 
@@ -52,6 +50,8 @@
 #include "gmc_v9_0.h"
 #include "gfxhub_v1_0.h"
 #include "mmhub_v1_0.h"
+#include "df_v1_7.h"
+#include "df_v3_6.h"
 #include "vega10_ih.h"
 #include "sdma_v4_0.h"
 #include "uvd_v7_0.h"
@@ -60,33 +60,6 @@
 #include "dce_virtual.h"
 #include "mxgpu_ai.h"
 
-#define mmFabricConfigAccessControl                                                                    0x0410
-#define mmFabricConfigAccessControl_BASE_IDX                                                           0
-#define mmFabricConfigAccessControl_DEFAULT                                      0x00000000
-//FabricConfigAccessControl
-#define FabricConfigAccessControl__CfgRegInstAccEn__SHIFT                                                     0x0
-#define FabricConfigAccessControl__CfgRegInstAccRegLock__SHIFT                                                0x1
-#define FabricConfigAccessControl__CfgRegInstID__SHIFT                                                        0x10
-#define FabricConfigAccessControl__CfgRegInstAccEn_MASK                                                       0x00000001L
-#define FabricConfigAccessControl__CfgRegInstAccRegLock_MASK                                                  0x00000002L
-#define FabricConfigAccessControl__CfgRegInstID_MASK                                                          0x00FF0000L
-
-
-#define mmDF_PIE_AON0_DfGlobalClkGater                                                                 0x00fc
-#define mmDF_PIE_AON0_DfGlobalClkGater_BASE_IDX                                                        0
-//DF_PIE_AON0_DfGlobalClkGater
-#define DF_PIE_AON0_DfGlobalClkGater__MGCGMode__SHIFT                                                         0x0
-#define DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK                                                           0x0000000FL
-
-enum {
-	DF_MGCG_DISABLE = 0,
-	DF_MGCG_ENABLE_00_CYCLE_DELAY =1,
-	DF_MGCG_ENABLE_01_CYCLE_DELAY =2,
-	DF_MGCG_ENABLE_15_CYCLE_DELAY =13,
-	DF_MGCG_ENABLE_31_CYCLE_DELAY =14,
-	DF_MGCG_ENABLE_63_CYCLE_DELAY =15
-};
-
 #define mmMP0_MISC_CGTT_CTRL0                                                                   0x01b9
 #define mmMP0_MISC_CGTT_CTRL0_BASE_IDX                                                          0
 #define mmMP0_MISC_LIGHT_SLEEP_CTRL                                                             0x01ba
@@ -313,6 +286,7 @@ static struct soc15_allowed_register_entry soc15_allowed_read_registers[] = {
 	{ SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STALLED_STAT1)},
 	{ SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STATUS)},
 	{ SOC15_REG_ENTRY(GC, 0, mmGB_ADDR_CONFIG)},
+	{ SOC15_REG_ENTRY(GC, 0, mmDB_DEBUG2)},
 };
 
 static uint32_t soc15_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
@@ -341,6 +315,8 @@ static uint32_t soc15_get_register_value(struct amdgpu_device *adev,
 	} else {
 		if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG))
 			return adev->gfx.config.gb_addr_config;
+		else if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG2))
+			return adev->gfx.config.db_debug2;
 		return RREG32(reg_offset);
 	}
 }
@@ -512,15 +488,24 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev)
 	case CHIP_RAVEN:
 		vega10_reg_base_init(adev);
 		break;
+	case CHIP_VEGA20:
+		vega20_reg_base_init(adev);
+		break;
 	default:
 		return -EINVAL;
 	}
 
 	if (adev->flags & AMD_IS_APU)
 		adev->nbio_funcs = &nbio_v7_0_funcs;
+	else if (adev->asic_type == CHIP_VEGA20)
+		adev->nbio_funcs = &nbio_v7_0_funcs;
 	else
 		adev->nbio_funcs = &nbio_v6_1_funcs;
 
+	if (adev->asic_type == CHIP_VEGA20)
+		adev->df_funcs = &df_v3_6_funcs;
+	else
+		adev->df_funcs = &df_v1_7_funcs;
 	adev->nbio_funcs->detect_hw_virt(adev);
 
 	if (amdgpu_sriov_vf(adev))
@@ -529,12 +514,15 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev)
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 		amdgpu_device_ip_block_add(adev, &vega10_common_ip_block);
 		amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block);
 		amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block);
-		amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block);
-		if (!amdgpu_sriov_vf(adev))
-			amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
+		if (adev->asic_type != CHIP_VEGA20) {
+			amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block);
+			if (!amdgpu_sriov_vf(adev))
+				amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
+		}
 		if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
 			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
@@ -593,6 +581,12 @@ static void soc15_invalidate_hdp(struct amdgpu_device *adev,
 			HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1);
 }
 
+static bool soc15_need_full_reset(struct amdgpu_device *adev)
+{
+	/* change this when we implement soft reset */
+	return true;
+}
+
 static const struct amdgpu_asic_funcs soc15_asic_funcs =
 {
 	.read_disabled_bios = &soc15_read_disabled_bios,
@@ -606,6 +600,7 @@ static const struct amdgpu_asic_funcs soc15_asic_funcs =
 	.get_config_memsize = &soc15_get_config_memsize,
 	.flush_hdp = &soc15_flush_hdp,
 	.invalidate_hdp = &soc15_invalidate_hdp,
+	.need_full_reset = &soc15_need_full_reset,
 };
 
 static int soc15_common_early_init(void *handle)
@@ -675,6 +670,27 @@ static int soc15_common_early_init(void *handle)
 		adev->pg_flags = 0;
 		adev->external_rev_id = adev->rev_id + 0x14;
 		break;
+	case CHIP_VEGA20:
+		adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
+			AMD_CG_SUPPORT_GFX_MGLS |
+			AMD_CG_SUPPORT_GFX_CGCG |
+			AMD_CG_SUPPORT_GFX_CGLS |
+			AMD_CG_SUPPORT_GFX_3D_CGCG |
+			AMD_CG_SUPPORT_GFX_3D_CGLS |
+			AMD_CG_SUPPORT_GFX_CP_LS |
+			AMD_CG_SUPPORT_MC_LS |
+			AMD_CG_SUPPORT_MC_MGCG |
+			AMD_CG_SUPPORT_SDMA_MGCG |
+			AMD_CG_SUPPORT_SDMA_LS |
+			AMD_CG_SUPPORT_BIF_MGCG |
+			AMD_CG_SUPPORT_BIF_LS |
+			AMD_CG_SUPPORT_HDP_MGCG |
+			AMD_CG_SUPPORT_ROM_MGCG |
+			AMD_CG_SUPPORT_VCE_MGCG |
+			AMD_CG_SUPPORT_UVD_MGCG;
+		adev->pg_flags = 0;
+		adev->external_rev_id = adev->rev_id + 0x28;
+		break;
 	case CHIP_RAVEN:
 		adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
 			AMD_CG_SUPPORT_GFX_MGLS |
@@ -694,8 +710,15 @@ static int soc15_common_early_init(void *handle)
 			AMD_CG_SUPPORT_MC_MGCG |
 			AMD_CG_SUPPORT_MC_LS |
 			AMD_CG_SUPPORT_SDMA_MGCG |
-			AMD_CG_SUPPORT_SDMA_LS;
-		adev->pg_flags = AMD_PG_SUPPORT_SDMA;
+			AMD_CG_SUPPORT_SDMA_LS |
+			AMD_CG_SUPPORT_VCN_MGCG;
+
+		adev->pg_flags = AMD_PG_SUPPORT_SDMA | AMD_PG_SUPPORT_VCN;
+
+		if (adev->powerplay.pp_feature & PP_GFXOFF_MASK)
+			adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
+				AMD_PG_SUPPORT_CP |
+				AMD_PG_SUPPORT_RLC_SMU_HS;
 
 		adev->external_rev_id = 0x1;
 		break;
@@ -871,32 +894,6 @@ static void soc15_update_rom_medium_grain_clock_gating(struct amdgpu_device *ade
 		WREG32(SOC15_REG_OFFSET(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0), data);
 }
 
-static void soc15_update_df_medium_grain_clock_gating(struct amdgpu_device *adev,
-						       bool enable)
-{
-	uint32_t data;
-
-	/* Put DF on broadcast mode */
-	data = RREG32(SOC15_REG_OFFSET(DF, 0, mmFabricConfigAccessControl));
-	data &= ~FabricConfigAccessControl__CfgRegInstAccEn_MASK;
-	WREG32(SOC15_REG_OFFSET(DF, 0, mmFabricConfigAccessControl), data);
-
-	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_DF_MGCG)) {
-		data = RREG32(SOC15_REG_OFFSET(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater));
-		data &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
-		data |= DF_MGCG_ENABLE_15_CYCLE_DELAY;
-		WREG32(SOC15_REG_OFFSET(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater), data);
-	} else {
-		data = RREG32(SOC15_REG_OFFSET(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater));
-		data &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
-		data |= DF_MGCG_DISABLE;
-		WREG32(SOC15_REG_OFFSET(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater), data);
-	}
-
-	WREG32(SOC15_REG_OFFSET(DF, 0, mmFabricConfigAccessControl),
-	       mmFabricConfigAccessControl_DEFAULT);
-}
-
 static int soc15_common_set_clockgating_state(void *handle,
 					    enum amd_clockgating_state state)
 {
@@ -908,6 +905,7 @@ static int soc15_common_set_clockgating_state(void *handle,
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 		adev->nbio_funcs->update_medium_grain_clock_gating(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
 		adev->nbio_funcs->update_medium_grain_light_sleep(adev,
@@ -920,7 +918,7 @@ static int soc15_common_set_clockgating_state(void *handle,
 				state == AMD_CG_STATE_GATE ? true : false);
 		soc15_update_rom_medium_grain_clock_gating(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
-		soc15_update_df_medium_grain_clock_gating(adev,
+		adev->df_funcs->update_medium_grain_clock_gating(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
 		break;
 	case CHIP_RAVEN:
@@ -973,10 +971,7 @@ static void soc15_common_get_clockgating_state(void *handle, u32 *flags)
 	if (!(data & CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK))
 		*flags |= AMD_CG_SUPPORT_ROM_MGCG;
 
-	/* AMD_CG_SUPPORT_DF_MGCG */
-	data = RREG32(SOC15_REG_OFFSET(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater));
-	if (data & DF_MGCG_ENABLE_15_CYCLE_DELAY)
-		*flags |= AMD_CG_SUPPORT_DF_MGCG;
+	adev->df_funcs->get_clockgating_state(adev, flags);
 }
 
 static int soc15_common_set_powergating_state(void *handle,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.h b/drivers/gpu/drm/amd/amdgpu/soc15.h
index f70da8a29f86..1f714b7af520 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.h
@@ -55,5 +55,6 @@ void soc15_program_register_sequence(struct amdgpu_device *adev,
 					     const u32 array_size);
 
 int vega10_reg_base_init(struct amdgpu_device *adev);
+int vega20_reg_base_init(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
index def865067edd..0942f492d2e1 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
@@ -47,6 +47,21 @@
 #define WREG32_SOC15_OFFSET(ip, inst, reg, offset, value) \
 	WREG32((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) + offset, value)
 
+#define SOC15_WAIT_ON_RREG(ip, inst, reg, expected_value, mask, ret) \
+	do {							\
+		uint32_t tmp_ = RREG32(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg); \
+		uint32_t loop = adev->usec_timeout;		\
+		while ((tmp_ & (mask)) != (expected_value)) {	\
+			udelay(2);				\
+			tmp_ = RREG32(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg); \
+			loop--;					\
+			if (!loop) {				\
+				ret = -ETIMEDOUT;		\
+				break;				\
+			}					\
+		}						\
+	} while (0)
+
 #endif
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15d.h b/drivers/gpu/drm/amd/amdgpu/soc15d.h
index 7f408f85fdb6..8dc29107228f 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15d.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15d.h
@@ -159,6 +159,7 @@
 #define		EOP_TC_WB_ACTION_EN                     (1 << 15) /* L2 */
 #define		EOP_TCL1_ACTION_EN                      (1 << 16)
 #define		EOP_TC_ACTION_EN                        (1 << 17) /* L2 */
+#define		EOP_TC_NC_ACTION_EN			(1 << 19)
 #define		EOP_TC_MD_ACTION_EN			(1 << 21) /* L2 metadata */
 
 #define		DATA_SEL(x)                             ((x) << 29)
@@ -268,6 +269,11 @@
 			 * x=1: tmz_end
 			 */
 
+#define	PACKET3_INVALIDATE_TLBS				0x98
+#              define PACKET3_INVALIDATE_TLBS_DST_SEL(x)     ((x) << 0)
+#              define PACKET3_INVALIDATE_TLBS_ALL_HUB(x)     ((x) << 4)
+#              define PACKET3_INVALIDATE_TLBS_PASID(x)       ((x) << 5)
+#              define PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(x)  ((x) << 29)
 #define PACKET3_SET_RESOURCES				0xA0
 /* 1. header
  * 2. CONTROL
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index 948bb9437757..6fed3d7797a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -93,6 +93,7 @@ static void uvd_v4_2_ring_set_wptr(struct amdgpu_ring *ring)
 static int uvd_v4_2_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	adev->uvd.num_uvd_inst = 1;
 
 	uvd_v4_2_set_ring_funcs(adev);
 	uvd_v4_2_set_irq_funcs(adev);
@@ -107,7 +108,7 @@ static int uvd_v4_2_sw_init(void *handle)
 	int r;
 
 	/* UVD TRAP */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 124, &adev->uvd.irq);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 124, &adev->uvd.inst->irq);
 	if (r)
 		return r;
 
@@ -119,9 +120,9 @@ static int uvd_v4_2_sw_init(void *handle)
 	if (r)
 		return r;
 
-	ring = &adev->uvd.ring;
+	ring = &adev->uvd.inst->ring;
 	sprintf(ring->name, "uvd");
-	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
+	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.inst->irq, 0);
 
 	return r;
 }
@@ -150,7 +151,7 @@ static void uvd_v4_2_enable_mgcg(struct amdgpu_device *adev,
 static int uvd_v4_2_hw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	struct amdgpu_ring *ring = &adev->uvd.inst->ring;
 	uint32_t tmp;
 	int r;
 
@@ -208,7 +209,7 @@ static int uvd_v4_2_hw_init(void *handle)
 static int uvd_v4_2_hw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	struct amdgpu_ring *ring = &adev->uvd.inst->ring;
 
 	if (RREG32(mmUVD_STATUS) != 0)
 		uvd_v4_2_stop(adev);
@@ -251,7 +252,7 @@ static int uvd_v4_2_resume(void *handle)
  */
 static int uvd_v4_2_start(struct amdgpu_device *adev)
 {
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	struct amdgpu_ring *ring = &adev->uvd.inst->ring;
 	uint32_t rb_bufsz;
 	int i, j, r;
 	u32 tmp;
@@ -523,6 +524,18 @@ static void uvd_v4_2_ring_emit_ib(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, ib->length_dw);
 }
 
+static void uvd_v4_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+	int i;
+
+	WARN_ON(ring->wptr % 2 || count % 2);
+
+	for (i = 0; i < count / 2; i++) {
+		amdgpu_ring_write(ring, PACKET0(mmUVD_NO_OP, 0));
+		amdgpu_ring_write(ring, 0);
+	}
+}
+
 /**
  * uvd_v4_2_mc_resume - memory controller programming
  *
@@ -536,7 +549,7 @@ static void uvd_v4_2_mc_resume(struct amdgpu_device *adev)
 	uint32_t size;
 
 	/* programm the VCPU memory controller bits 0-27 */
-	addr = (adev->uvd.gpu_addr + AMDGPU_UVD_FIRMWARE_OFFSET) >> 3;
+	addr = (adev->uvd.inst->gpu_addr + AMDGPU_UVD_FIRMWARE_OFFSET) >> 3;
 	size = AMDGPU_UVD_FIRMWARE_SIZE(adev) >> 3;
 	WREG32(mmUVD_VCPU_CACHE_OFFSET0, addr);
 	WREG32(mmUVD_VCPU_CACHE_SIZE0, size);
@@ -553,11 +566,11 @@ static void uvd_v4_2_mc_resume(struct amdgpu_device *adev)
 	WREG32(mmUVD_VCPU_CACHE_SIZE2, size);
 
 	/* bits 28-31 */
-	addr = (adev->uvd.gpu_addr >> 28) & 0xF;
+	addr = (adev->uvd.inst->gpu_addr >> 28) & 0xF;
 	WREG32(mmUVD_LMI_ADDR_EXT, (addr << 12) | (addr << 0));
 
 	/* bits 32-39 */
-	addr = (adev->uvd.gpu_addr >> 32) & 0xFF;
+	addr = (adev->uvd.inst->gpu_addr >> 32) & 0xFF;
 	WREG32(mmUVD_LMI_EXT40_ADDR, addr | (0x9 << 16) | (0x1 << 31));
 
 	WREG32(mmUVD_UDEC_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
@@ -664,7 +677,7 @@ static int uvd_v4_2_process_interrupt(struct amdgpu_device *adev,
 				      struct amdgpu_iv_entry *entry)
 {
 	DRM_DEBUG("IH: UVD TRAP\n");
-	amdgpu_fence_process(&adev->uvd.ring);
+	amdgpu_fence_process(&adev->uvd.inst->ring);
 	return 0;
 }
 
@@ -688,7 +701,7 @@ static int uvd_v4_2_set_powergating_state(void *handle,
 
 	if (state == AMD_PG_STATE_GATE) {
 		uvd_v4_2_stop(adev);
-		if (adev->pg_flags & AMD_PG_SUPPORT_UVD && amdgpu_dpm == 0) {
+		if (adev->pg_flags & AMD_PG_SUPPORT_UVD && !adev->pm.dpm_enabled) {
 			if (!(RREG32_SMC(ixCURRENT_PG_STATUS) &
 				CURRENT_PG_STATUS__UVD_PG_STATUS_MASK)) {
 				WREG32(mmUVD_PGFSM_CONFIG, (UVD_PGFSM_CONFIG__UVD_PGFSM_FSM_ADDR_MASK   |
@@ -699,7 +712,7 @@ static int uvd_v4_2_set_powergating_state(void *handle,
 		}
 		return 0;
 	} else {
-		if (adev->pg_flags & AMD_PG_SUPPORT_UVD && amdgpu_dpm == 0) {
+		if (adev->pg_flags & AMD_PG_SUPPORT_UVD && !adev->pm.dpm_enabled) {
 			if (RREG32_SMC(ixCURRENT_PG_STATUS) &
 				CURRENT_PG_STATUS__UVD_PG_STATUS_MASK) {
 				WREG32(mmUVD_PGFSM_CONFIG, (UVD_PGFSM_CONFIG__UVD_PGFSM_FSM_ADDR_MASK   |
@@ -732,7 +745,6 @@ static const struct amd_ip_funcs uvd_v4_2_ip_funcs = {
 static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = {
 	.type = AMDGPU_RING_TYPE_UVD,
 	.align_mask = 0xf,
-	.nop = PACKET0(mmUVD_NO_OP, 0),
 	.support_64bit_ptrs = false,
 	.get_rptr = uvd_v4_2_ring_get_rptr,
 	.get_wptr = uvd_v4_2_ring_get_wptr,
@@ -745,7 +757,7 @@ static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = {
 	.emit_fence = uvd_v4_2_ring_emit_fence,
 	.test_ring = uvd_v4_2_ring_test_ring,
 	.test_ib = amdgpu_uvd_ring_test_ib,
-	.insert_nop = amdgpu_ring_insert_nop,
+	.insert_nop = uvd_v4_2_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
@@ -753,7 +765,7 @@ static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = {
 
 static void uvd_v4_2_set_ring_funcs(struct amdgpu_device *adev)
 {
-	adev->uvd.ring.funcs = &uvd_v4_2_ring_funcs;
+	adev->uvd.inst->ring.funcs = &uvd_v4_2_ring_funcs;
 }
 
 static const struct amdgpu_irq_src_funcs uvd_v4_2_irq_funcs = {
@@ -763,8 +775,8 @@ static const struct amdgpu_irq_src_funcs uvd_v4_2_irq_funcs = {
 
 static void uvd_v4_2_set_irq_funcs(struct amdgpu_device *adev)
 {
-	adev->uvd.irq.num_types = 1;
-	adev->uvd.irq.funcs = &uvd_v4_2_irq_funcs;
+	adev->uvd.inst->irq.num_types = 1;
+	adev->uvd.inst->irq.funcs = &uvd_v4_2_irq_funcs;
 }
 
 const struct amdgpu_ip_block_version uvd_v4_2_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index 6445d55e7d5a..341ee6d55ce8 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -89,6 +89,7 @@ static void uvd_v5_0_ring_set_wptr(struct amdgpu_ring *ring)
 static int uvd_v5_0_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	adev->uvd.num_uvd_inst = 1;
 
 	uvd_v5_0_set_ring_funcs(adev);
 	uvd_v5_0_set_irq_funcs(adev);
@@ -103,7 +104,7 @@ static int uvd_v5_0_sw_init(void *handle)
 	int r;
 
 	/* UVD TRAP */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 124, &adev->uvd.irq);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 124, &adev->uvd.inst->irq);
 	if (r)
 		return r;
 
@@ -115,9 +116,9 @@ static int uvd_v5_0_sw_init(void *handle)
 	if (r)
 		return r;
 
-	ring = &adev->uvd.ring;
+	ring = &adev->uvd.inst->ring;
 	sprintf(ring->name, "uvd");
-	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
+	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.inst->irq, 0);
 
 	return r;
 }
@@ -144,7 +145,7 @@ static int uvd_v5_0_sw_fini(void *handle)
 static int uvd_v5_0_hw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	struct amdgpu_ring *ring = &adev->uvd.inst->ring;
 	uint32_t tmp;
 	int r;
 
@@ -204,7 +205,7 @@ static int uvd_v5_0_hw_init(void *handle)
 static int uvd_v5_0_hw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	struct amdgpu_ring *ring = &adev->uvd.inst->ring;
 
 	if (RREG32(mmUVD_STATUS) != 0)
 		uvd_v5_0_stop(adev);
@@ -253,9 +254,9 @@ static void uvd_v5_0_mc_resume(struct amdgpu_device *adev)
 
 	/* programm memory controller bits 0-27 */
 	WREG32(mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
-			lower_32_bits(adev->uvd.gpu_addr));
+			lower_32_bits(adev->uvd.inst->gpu_addr));
 	WREG32(mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
-			upper_32_bits(adev->uvd.gpu_addr));
+			upper_32_bits(adev->uvd.inst->gpu_addr));
 
 	offset = AMDGPU_UVD_FIRMWARE_OFFSET;
 	size = AMDGPU_UVD_FIRMWARE_SIZE(adev);
@@ -287,7 +288,7 @@ static void uvd_v5_0_mc_resume(struct amdgpu_device *adev)
  */
 static int uvd_v5_0_start(struct amdgpu_device *adev)
 {
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	struct amdgpu_ring *ring = &adev->uvd.inst->ring;
 	uint32_t rb_bufsz, tmp;
 	uint32_t lmi_swap_cntl;
 	uint32_t mp_swap_cntl;
@@ -540,6 +541,18 @@ static void uvd_v5_0_ring_emit_ib(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, ib->length_dw);
 }
 
+static void uvd_v5_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+	int i;
+
+	WARN_ON(ring->wptr % 2 || count % 2);
+
+	for (i = 0; i < count / 2; i++) {
+		amdgpu_ring_write(ring, PACKET0(mmUVD_NO_OP, 0));
+		amdgpu_ring_write(ring, 0);
+	}
+}
+
 static bool uvd_v5_0_is_idle(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -586,7 +599,7 @@ static int uvd_v5_0_process_interrupt(struct amdgpu_device *adev,
 				      struct amdgpu_iv_entry *entry)
 {
 	DRM_DEBUG("IH: UVD TRAP\n");
-	amdgpu_fence_process(&adev->uvd.ring);
+	amdgpu_fence_process(&adev->uvd.inst->ring);
 	return 0;
 }
 
@@ -840,7 +853,6 @@ static const struct amd_ip_funcs uvd_v5_0_ip_funcs = {
 static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = {
 	.type = AMDGPU_RING_TYPE_UVD,
 	.align_mask = 0xf,
-	.nop = PACKET0(mmUVD_NO_OP, 0),
 	.support_64bit_ptrs = false,
 	.get_rptr = uvd_v5_0_ring_get_rptr,
 	.get_wptr = uvd_v5_0_ring_get_wptr,
@@ -853,7 +865,7 @@ static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = {
 	.emit_fence = uvd_v5_0_ring_emit_fence,
 	.test_ring = uvd_v5_0_ring_test_ring,
 	.test_ib = amdgpu_uvd_ring_test_ib,
-	.insert_nop = amdgpu_ring_insert_nop,
+	.insert_nop = uvd_v5_0_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
@@ -861,7 +873,7 @@ static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = {
 
 static void uvd_v5_0_set_ring_funcs(struct amdgpu_device *adev)
 {
-	adev->uvd.ring.funcs = &uvd_v5_0_ring_funcs;
+	adev->uvd.inst->ring.funcs = &uvd_v5_0_ring_funcs;
 }
 
 static const struct amdgpu_irq_src_funcs uvd_v5_0_irq_funcs = {
@@ -871,8 +883,8 @@ static const struct amdgpu_irq_src_funcs uvd_v5_0_irq_funcs = {
 
 static void uvd_v5_0_set_irq_funcs(struct amdgpu_device *adev)
 {
-	adev->uvd.irq.num_types = 1;
-	adev->uvd.irq.funcs = &uvd_v5_0_irq_funcs;
+	adev->uvd.inst->irq.num_types = 1;
+	adev->uvd.inst->irq.funcs = &uvd_v5_0_irq_funcs;
 }
 
 const struct amdgpu_ip_block_version uvd_v5_0_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index f26f515db2fb..bfddf97dd13e 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -62,7 +62,7 @@ static void uvd_v6_0_enable_mgcg(struct amdgpu_device *adev,
 static inline bool uvd_v6_0_enc_support(struct amdgpu_device *adev)
 {
 	return ((adev->asic_type >= CHIP_POLARIS10) &&
-			(adev->asic_type <= CHIP_POLARIS12) &&
+			(adev->asic_type <= CHIP_VEGAM) &&
 			(!adev->uvd.fw_version || adev->uvd.fw_version >= FW_1_130_16));
 }
 
@@ -91,7 +91,7 @@ static uint64_t uvd_v6_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	if (ring == &adev->uvd.ring_enc[0])
+	if (ring == &adev->uvd.inst->ring_enc[0])
 		return RREG32(mmUVD_RB_RPTR);
 	else
 		return RREG32(mmUVD_RB_RPTR2);
@@ -121,7 +121,7 @@ static uint64_t uvd_v6_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	if (ring == &adev->uvd.ring_enc[0])
+	if (ring == &adev->uvd.inst->ring_enc[0])
 		return RREG32(mmUVD_RB_WPTR);
 	else
 		return RREG32(mmUVD_RB_WPTR2);
@@ -152,7 +152,7 @@ static void uvd_v6_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	if (ring == &adev->uvd.ring_enc[0])
+	if (ring == &adev->uvd.inst->ring_enc[0])
 		WREG32(mmUVD_RB_WPTR,
 			lower_32_bits(ring->wptr));
 	else
@@ -375,6 +375,7 @@ static int uvd_v6_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 static int uvd_v6_0_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	adev->uvd.num_uvd_inst = 1;
 
 	if (!(adev->flags & AMD_IS_APU) &&
 	    (RREG32_SMC(ixCC_HARVEST_FUSES) & CC_HARVEST_FUSES__UVD_DISABLE_MASK))
@@ -399,14 +400,14 @@ static int uvd_v6_0_sw_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	/* UVD TRAP */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 124, &adev->uvd.irq);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 124, &adev->uvd.inst->irq);
 	if (r)
 		return r;
 
 	/* UVD ENC TRAP */
 	if (uvd_v6_0_enc_support(adev)) {
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
-			r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, i + 119, &adev->uvd.irq);
+			r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, i + 119, &adev->uvd.inst->irq);
 			if (r)
 				return r;
 		}
@@ -418,18 +419,18 @@ static int uvd_v6_0_sw_init(void *handle)
 
 	if (!uvd_v6_0_enc_support(adev)) {
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i)
-			adev->uvd.ring_enc[i].funcs = NULL;
+			adev->uvd.inst->ring_enc[i].funcs = NULL;
 
-		adev->uvd.irq.num_types = 1;
+		adev->uvd.inst->irq.num_types = 1;
 		adev->uvd.num_enc_rings = 0;
 
 		DRM_INFO("UVD ENC is disabled\n");
 	} else {
 		struct drm_sched_rq *rq;
-		ring = &adev->uvd.ring_enc[0];
+		ring = &adev->uvd.inst->ring_enc[0];
 		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
-		r = drm_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
-					  rq, amdgpu_sched_jobs, NULL);
+		r = drm_sched_entity_init(&ring->sched, &adev->uvd.inst->entity_enc,
+					  rq, NULL);
 		if (r) {
 			DRM_ERROR("Failed setting up UVD ENC run queue.\n");
 			return r;
@@ -440,17 +441,17 @@ static int uvd_v6_0_sw_init(void *handle)
 	if (r)
 		return r;
 
-	ring = &adev->uvd.ring;
+	ring = &adev->uvd.inst->ring;
 	sprintf(ring->name, "uvd");
-	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
+	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.inst->irq, 0);
 	if (r)
 		return r;
 
 	if (uvd_v6_0_enc_support(adev)) {
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
-			ring = &adev->uvd.ring_enc[i];
+			ring = &adev->uvd.inst->ring_enc[i];
 			sprintf(ring->name, "uvd_enc%d", i);
-			r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
+			r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.inst->irq, 0);
 			if (r)
 				return r;
 		}
@@ -469,10 +470,10 @@ static int uvd_v6_0_sw_fini(void *handle)
 		return r;
 
 	if (uvd_v6_0_enc_support(adev)) {
-		drm_sched_entity_fini(&adev->uvd.ring_enc[0].sched, &adev->uvd.entity_enc);
+		drm_sched_entity_fini(&adev->uvd.inst->ring_enc[0].sched, &adev->uvd.inst->entity_enc);
 
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i)
-			amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
+			amdgpu_ring_fini(&adev->uvd.inst->ring_enc[i]);
 	}
 
 	return amdgpu_uvd_sw_fini(adev);
@@ -488,7 +489,7 @@ static int uvd_v6_0_sw_fini(void *handle)
 static int uvd_v6_0_hw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	struct amdgpu_ring *ring = &adev->uvd.inst->ring;
 	uint32_t tmp;
 	int i, r;
 
@@ -532,7 +533,7 @@ static int uvd_v6_0_hw_init(void *handle)
 
 	if (uvd_v6_0_enc_support(adev)) {
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
-			ring = &adev->uvd.ring_enc[i];
+			ring = &adev->uvd.inst->ring_enc[i];
 			ring->ready = true;
 			r = amdgpu_ring_test_ring(ring);
 			if (r) {
@@ -563,7 +564,7 @@ static int uvd_v6_0_hw_init(void *handle)
 static int uvd_v6_0_hw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	struct amdgpu_ring *ring = &adev->uvd.inst->ring;
 
 	if (RREG32(mmUVD_STATUS) != 0)
 		uvd_v6_0_stop(adev);
@@ -611,9 +612,9 @@ static void uvd_v6_0_mc_resume(struct amdgpu_device *adev)
 
 	/* programm memory controller bits 0-27 */
 	WREG32(mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
-			lower_32_bits(adev->uvd.gpu_addr));
+			lower_32_bits(adev->uvd.inst->gpu_addr));
 	WREG32(mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
-			upper_32_bits(adev->uvd.gpu_addr));
+			upper_32_bits(adev->uvd.inst->gpu_addr));
 
 	offset = AMDGPU_UVD_FIRMWARE_OFFSET;
 	size = AMDGPU_UVD_FIRMWARE_SIZE(adev);
@@ -726,7 +727,7 @@ static void cz_set_uvd_clock_gating_branches(struct amdgpu_device *adev,
  */
 static int uvd_v6_0_start(struct amdgpu_device *adev)
 {
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	struct amdgpu_ring *ring = &adev->uvd.inst->ring;
 	uint32_t rb_bufsz, tmp;
 	uint32_t lmi_swap_cntl;
 	uint32_t mp_swap_cntl;
@@ -866,14 +867,14 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
 	WREG32_FIELD(UVD_RBC_RB_CNTL, RB_NO_FETCH, 0);
 
 	if (uvd_v6_0_enc_support(adev)) {
-		ring = &adev->uvd.ring_enc[0];
+		ring = &adev->uvd.inst->ring_enc[0];
 		WREG32(mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
 		WREG32(mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
 		WREG32(mmUVD_RB_BASE_LO, ring->gpu_addr);
 		WREG32(mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
 		WREG32(mmUVD_RB_SIZE, ring->ring_size / 4);
 
-		ring = &adev->uvd.ring_enc[1];
+		ring = &adev->uvd.inst->ring_enc[1];
 		WREG32(mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
 		WREG32(mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
 		WREG32(mmUVD_RB_BASE_LO2, ring->gpu_addr);
@@ -963,6 +964,16 @@ static void uvd_v6_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_TRAP);
 }
 
+/**
+ * uvd_v6_0_ring_emit_hdp_flush - skip HDP flushing
+ *
+ * @ring: amdgpu_ring pointer
+ */
+static void uvd_v6_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
+{
+	/* The firmware doesn't seem to like touching registers at this point. */
+}
+
 /**
  * uvd_v6_0_ring_test_ring - register write test
  *
@@ -1089,6 +1100,18 @@ static void uvd_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, 0xE);
 }
 
+static void uvd_v6_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+	int i;
+
+	WARN_ON(ring->wptr % 2 || count % 2);
+
+	for (i = 0; i < count / 2; i++) {
+		amdgpu_ring_write(ring, PACKET0(mmUVD_NO_OP, 0));
+		amdgpu_ring_write(ring, 0);
+	}
+}
+
 static void uvd_v6_0_enc_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 {
 	uint32_t seq = ring->fence_drv.sync_seq;
@@ -1148,10 +1171,10 @@ static bool uvd_v6_0_check_soft_reset(void *handle)
 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_UVD, 1);
 
 	if (srbm_soft_reset) {
-		adev->uvd.srbm_soft_reset = srbm_soft_reset;
+		adev->uvd.inst->srbm_soft_reset = srbm_soft_reset;
 		return true;
 	} else {
-		adev->uvd.srbm_soft_reset = 0;
+		adev->uvd.inst->srbm_soft_reset = 0;
 		return false;
 	}
 }
@@ -1160,7 +1183,7 @@ static int uvd_v6_0_pre_soft_reset(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	if (!adev->uvd.srbm_soft_reset)
+	if (!adev->uvd.inst->srbm_soft_reset)
 		return 0;
 
 	uvd_v6_0_stop(adev);
@@ -1172,9 +1195,9 @@ static int uvd_v6_0_soft_reset(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	u32 srbm_soft_reset;
 
-	if (!adev->uvd.srbm_soft_reset)
+	if (!adev->uvd.inst->srbm_soft_reset)
 		return 0;
-	srbm_soft_reset = adev->uvd.srbm_soft_reset;
+	srbm_soft_reset = adev->uvd.inst->srbm_soft_reset;
 
 	if (srbm_soft_reset) {
 		u32 tmp;
@@ -1202,7 +1225,7 @@ static int uvd_v6_0_post_soft_reset(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	if (!adev->uvd.srbm_soft_reset)
+	if (!adev->uvd.inst->srbm_soft_reset)
 		return 0;
 
 	mdelay(5);
@@ -1228,17 +1251,17 @@ static int uvd_v6_0_process_interrupt(struct amdgpu_device *adev,
 
 	switch (entry->src_id) {
 	case 124:
-		amdgpu_fence_process(&adev->uvd.ring);
+		amdgpu_fence_process(&adev->uvd.inst->ring);
 		break;
 	case 119:
 		if (likely(uvd_v6_0_enc_support(adev)))
-			amdgpu_fence_process(&adev->uvd.ring_enc[0]);
+			amdgpu_fence_process(&adev->uvd.inst->ring_enc[0]);
 		else
 			int_handled = false;
 		break;
 	case 120:
 		if (likely(uvd_v6_0_enc_support(adev)))
-			amdgpu_fence_process(&adev->uvd.ring_enc[1]);
+			amdgpu_fence_process(&adev->uvd.inst->ring_enc[1]);
 		else
 			int_handled = false;
 		break;
@@ -1521,22 +1544,22 @@ static const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
 static const struct amdgpu_ring_funcs uvd_v6_0_ring_phys_funcs = {
 	.type = AMDGPU_RING_TYPE_UVD,
 	.align_mask = 0xf,
-	.nop = PACKET0(mmUVD_NO_OP, 0),
 	.support_64bit_ptrs = false,
 	.get_rptr = uvd_v6_0_ring_get_rptr,
 	.get_wptr = uvd_v6_0_ring_get_wptr,
 	.set_wptr = uvd_v6_0_ring_set_wptr,
 	.parse_cs = amdgpu_uvd_ring_parse_cs,
 	.emit_frame_size =
-		6 + 6 + /* hdp flush / invalidate */
+		6 + /* hdp invalidate */
 		10 + /* uvd_v6_0_ring_emit_pipeline_sync */
 		14, /* uvd_v6_0_ring_emit_fence x1 no user fence */
 	.emit_ib_size = 8, /* uvd_v6_0_ring_emit_ib */
 	.emit_ib = uvd_v6_0_ring_emit_ib,
 	.emit_fence = uvd_v6_0_ring_emit_fence,
+	.emit_hdp_flush = uvd_v6_0_ring_emit_hdp_flush,
 	.test_ring = uvd_v6_0_ring_test_ring,
 	.test_ib = amdgpu_uvd_ring_test_ib,
-	.insert_nop = amdgpu_ring_insert_nop,
+	.insert_nop = uvd_v6_0_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
@@ -1552,7 +1575,7 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
 	.get_wptr = uvd_v6_0_ring_get_wptr,
 	.set_wptr = uvd_v6_0_ring_set_wptr,
 	.emit_frame_size =
-		6 + 6 + /* hdp flush / invalidate */
+		6 + /* hdp invalidate */
 		10 + /* uvd_v6_0_ring_emit_pipeline_sync */
 		VI_FLUSH_GPU_TLB_NUM_WREG * 6 + 8 + /* uvd_v6_0_ring_emit_vm_flush */
 		14 + 14, /* uvd_v6_0_ring_emit_fence x2 vm fence */
@@ -1561,6 +1584,7 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
 	.emit_fence = uvd_v6_0_ring_emit_fence,
 	.emit_vm_flush = uvd_v6_0_ring_emit_vm_flush,
 	.emit_pipeline_sync = uvd_v6_0_ring_emit_pipeline_sync,
+	.emit_hdp_flush = uvd_v6_0_ring_emit_hdp_flush,
 	.test_ring = uvd_v6_0_ring_test_ring,
 	.test_ib = amdgpu_uvd_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
@@ -1600,10 +1624,10 @@ static const struct amdgpu_ring_funcs uvd_v6_0_enc_ring_vm_funcs = {
 static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
 {
 	if (adev->asic_type >= CHIP_POLARIS10) {
-		adev->uvd.ring.funcs = &uvd_v6_0_ring_vm_funcs;
+		adev->uvd.inst->ring.funcs = &uvd_v6_0_ring_vm_funcs;
 		DRM_INFO("UVD is enabled in VM mode\n");
 	} else {
-		adev->uvd.ring.funcs = &uvd_v6_0_ring_phys_funcs;
+		adev->uvd.inst->ring.funcs = &uvd_v6_0_ring_phys_funcs;
 		DRM_INFO("UVD is enabled in physical mode\n");
 	}
 }
@@ -1613,7 +1637,7 @@ static void uvd_v6_0_set_enc_ring_funcs(struct amdgpu_device *adev)
 	int i;
 
 	for (i = 0; i < adev->uvd.num_enc_rings; ++i)
-		adev->uvd.ring_enc[i].funcs = &uvd_v6_0_enc_ring_vm_funcs;
+		adev->uvd.inst->ring_enc[i].funcs = &uvd_v6_0_enc_ring_vm_funcs;
 
 	DRM_INFO("UVD ENC is enabled in VM mode\n");
 }
@@ -1626,11 +1650,11 @@ static const struct amdgpu_irq_src_funcs uvd_v6_0_irq_funcs = {
 static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev)
 {
 	if (uvd_v6_0_enc_support(adev))
-		adev->uvd.irq.num_types = adev->uvd.num_enc_rings + 1;
+		adev->uvd.inst->irq.num_types = adev->uvd.num_enc_rings + 1;
 	else
-		adev->uvd.irq.num_types = 1;
+		adev->uvd.inst->irq.num_types = 1;
 
-	adev->uvd.irq.funcs = &uvd_v6_0_irq_funcs;
+	adev->uvd.inst->irq.funcs = &uvd_v6_0_irq_funcs;
 }
 
 const struct amdgpu_ip_block_version uvd_v6_0_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index eddc57f3b72a..57d32f21b3a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -40,6 +40,8 @@
 #include "mmhub/mmhub_1_0_offset.h"
 #include "mmhub/mmhub_1_0_sh_mask.h"
 
+#define UVD7_MAX_HW_INSTANCES_VEGA20			2
+
 static void uvd_v7_0_set_ring_funcs(struct amdgpu_device *adev);
 static void uvd_v7_0_set_enc_ring_funcs(struct amdgpu_device *adev);
 static void uvd_v7_0_set_irq_funcs(struct amdgpu_device *adev);
@@ -47,6 +49,11 @@ static int uvd_v7_0_start(struct amdgpu_device *adev);
 static void uvd_v7_0_stop(struct amdgpu_device *adev);
 static int uvd_v7_0_sriov_start(struct amdgpu_device *adev);
 
+static int amdgpu_ih_clientid_uvds[] = {
+	SOC15_IH_CLIENTID_UVD,
+	SOC15_IH_CLIENTID_UVD1
+};
+
 /**
  * uvd_v7_0_ring_get_rptr - get read pointer
  *
@@ -58,7 +65,7 @@ static uint64_t uvd_v7_0_ring_get_rptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
+	return RREG32_SOC15(UVD, ring->me, mmUVD_RBC_RB_RPTR);
 }
 
 /**
@@ -72,10 +79,10 @@ static uint64_t uvd_v7_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	if (ring == &adev->uvd.ring_enc[0])
-		return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR);
+	if (ring == &adev->uvd.inst[ring->me].ring_enc[0])
+		return RREG32_SOC15(UVD, ring->me, mmUVD_RB_RPTR);
 	else
-		return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2);
+		return RREG32_SOC15(UVD, ring->me, mmUVD_RB_RPTR2);
 }
 
 /**
@@ -89,7 +96,7 @@ static uint64_t uvd_v7_0_ring_get_wptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR);
+	return RREG32_SOC15(UVD, ring->me, mmUVD_RBC_RB_WPTR);
 }
 
 /**
@@ -106,10 +113,10 @@ static uint64_t uvd_v7_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
 	if (ring->use_doorbell)
 		return adev->wb.wb[ring->wptr_offs];
 
-	if (ring == &adev->uvd.ring_enc[0])
-		return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
+	if (ring == &adev->uvd.inst[ring->me].ring_enc[0])
+		return RREG32_SOC15(UVD, ring->me, mmUVD_RB_WPTR);
 	else
-		return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
+		return RREG32_SOC15(UVD, ring->me, mmUVD_RB_WPTR2);
 }
 
 /**
@@ -123,7 +130,7 @@ static void uvd_v7_0_ring_set_wptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
+	WREG32_SOC15(UVD, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
 }
 
 /**
@@ -144,11 +151,11 @@ static void uvd_v7_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
 		return;
 	}
 
-	if (ring == &adev->uvd.ring_enc[0])
-		WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR,
+	if (ring == &adev->uvd.inst[ring->me].ring_enc[0])
+		WREG32_SOC15(UVD, ring->me, mmUVD_RB_WPTR,
 			lower_32_bits(ring->wptr));
 	else
-		WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2,
+		WREG32_SOC15(UVD, ring->me, mmUVD_RB_WPTR2,
 			lower_32_bits(ring->wptr));
 }
 
@@ -170,8 +177,8 @@ static int uvd_v7_0_enc_ring_test_ring(struct amdgpu_ring *ring)
 
 	r = amdgpu_ring_alloc(ring, 16);
 	if (r) {
-		DRM_ERROR("amdgpu: uvd enc failed to lock ring %d (%d).\n",
-			  ring->idx, r);
+		DRM_ERROR("amdgpu: uvd enc failed to lock (%d)ring %d (%d).\n",
+			  ring->me, ring->idx, r);
 		return r;
 	}
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
@@ -184,11 +191,11 @@ static int uvd_v7_0_enc_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
-			 ring->idx, i);
+		DRM_DEBUG("(%d)ring test on %d succeeded in %d usecs\n",
+			 ring->me, ring->idx, i);
 	} else {
-		DRM_ERROR("amdgpu: ring %d test failed\n",
-			  ring->idx);
+		DRM_ERROR("amdgpu: (%d)ring %d test failed\n",
+			  ring->me, ring->idx);
 		r = -ETIMEDOUT;
 	}
 
@@ -342,24 +349,24 @@ static int uvd_v7_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 	r = uvd_v7_0_enc_get_create_msg(ring, 1, NULL);
 	if (r) {
-		DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r);
+		DRM_ERROR("amdgpu: (%d)failed to get create msg (%ld).\n", ring->me, r);
 		goto error;
 	}
 
 	r = uvd_v7_0_enc_get_destroy_msg(ring, 1, true, &fence);
 	if (r) {
-		DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r);
+		DRM_ERROR("amdgpu: (%d)failed to get destroy ib (%ld).\n", ring->me, r);
 		goto error;
 	}
 
 	r = dma_fence_wait_timeout(fence, false, timeout);
 	if (r == 0) {
-		DRM_ERROR("amdgpu: IB test timed out.\n");
+		DRM_ERROR("amdgpu: (%d)IB test timed out.\n", ring->me);
 		r = -ETIMEDOUT;
 	} else if (r < 0) {
-		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
+		DRM_ERROR("amdgpu: (%d)fence wait failed (%ld).\n", ring->me, r);
 	} else {
-		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on (%d)ring %d succeeded\n", ring->me, ring->idx);
 		r = 0;
 	}
 error:
@@ -370,6 +377,10 @@ static int uvd_v7_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 static int uvd_v7_0_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	if (adev->asic_type == CHIP_VEGA20)
+		adev->uvd.num_uvd_inst = UVD7_MAX_HW_INSTANCES_VEGA20;
+	else
+		adev->uvd.num_uvd_inst = 1;
 
 	if (amdgpu_sriov_vf(adev))
 		adev->uvd.num_enc_rings = 1;
@@ -386,19 +397,21 @@ static int uvd_v7_0_sw_init(void *handle)
 {
 	struct amdgpu_ring *ring;
 	struct drm_sched_rq *rq;
-	int i, r;
+	int i, j, r;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	/* UVD TRAP */
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UVD, 124, &adev->uvd.irq);
-	if (r)
-		return r;
-
-	/* UVD ENC TRAP */
-	for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
-		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UVD, i + 119, &adev->uvd.irq);
+	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
+		/* UVD TRAP */
+		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_uvds[j], 124, &adev->uvd.inst[j].irq);
 		if (r)
 			return r;
+
+		/* UVD ENC TRAP */
+		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
+			r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_uvds[j], i + 119, &adev->uvd.inst[j].irq);
+			if (r)
+				return r;
+		}
 	}
 
 	r = amdgpu_uvd_sw_init(adev);
@@ -415,43 +428,48 @@ static int uvd_v7_0_sw_init(void *handle)
 		DRM_INFO("PSP loading UVD firmware\n");
 	}
 
-	ring = &adev->uvd.ring_enc[0];
-	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
-	r = drm_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
-				  rq, amdgpu_sched_jobs, NULL);
-	if (r) {
-		DRM_ERROR("Failed setting up UVD ENC run queue.\n");
-		return r;
+	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
+		ring = &adev->uvd.inst[j].ring_enc[0];
+		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+		r = drm_sched_entity_init(&ring->sched, &adev->uvd.inst[j].entity_enc,
+					  rq, NULL);
+		if (r) {
+			DRM_ERROR("(%d)Failed setting up UVD ENC run queue.\n", j);
+			return r;
+		}
 	}
 
 	r = amdgpu_uvd_resume(adev);
 	if (r)
 		return r;
-	if (!amdgpu_sriov_vf(adev)) {
-		ring = &adev->uvd.ring;
-		sprintf(ring->name, "uvd");
-		r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
-		if (r)
-			return r;
-	}
 
-	for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
-		ring = &adev->uvd.ring_enc[i];
-		sprintf(ring->name, "uvd_enc%d", i);
-		if (amdgpu_sriov_vf(adev)) {
-			ring->use_doorbell = true;
-
-			/* currently only use the first enconding ring for
-			 * sriov, so set unused location for other unused rings.
-			 */
-			if (i == 0)
-				ring->doorbell_index = AMDGPU_DOORBELL64_UVD_RING0_1 * 2;
-			else
-				ring->doorbell_index = AMDGPU_DOORBELL64_UVD_RING2_3 * 2 + 1;
+	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
+		if (!amdgpu_sriov_vf(adev)) {
+			ring = &adev->uvd.inst[j].ring;
+			sprintf(ring->name, "uvd<%d>", j);
+			r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.inst[j].irq, 0);
+			if (r)
+				return r;
+		}
+
+		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
+			ring = &adev->uvd.inst[j].ring_enc[i];
+			sprintf(ring->name, "uvd_enc%d<%d>", i, j);
+			if (amdgpu_sriov_vf(adev)) {
+				ring->use_doorbell = true;
+
+				/* currently only use the first enconding ring for
+				 * sriov, so set unused location for other unused rings.
+				 */
+				if (i == 0)
+					ring->doorbell_index = AMDGPU_DOORBELL64_UVD_RING0_1 * 2;
+				else
+					ring->doorbell_index = AMDGPU_DOORBELL64_UVD_RING2_3 * 2 + 1;
+			}
+			r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.inst[j].irq, 0);
+			if (r)
+				return r;
 		}
-		r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
-		if (r)
-			return r;
 	}
 
 	r = amdgpu_virt_alloc_mm_table(adev);
@@ -463,7 +481,7 @@ static int uvd_v7_0_sw_init(void *handle)
 
 static int uvd_v7_0_sw_fini(void *handle)
 {
-	int i, r;
+	int i, j, r;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	amdgpu_virt_free_mm_table(adev);
@@ -472,11 +490,12 @@ static int uvd_v7_0_sw_fini(void *handle)
 	if (r)
 		return r;
 
-	drm_sched_entity_fini(&adev->uvd.ring_enc[0].sched, &adev->uvd.entity_enc);
-
-	for (i = 0; i < adev->uvd.num_enc_rings; ++i)
-		amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
+	for (j = 0; j < adev->uvd.num_uvd_inst; ++j) {
+		drm_sched_entity_fini(&adev->uvd.inst[j].ring_enc[0].sched, &adev->uvd.inst[j].entity_enc);
 
+		for (i = 0; i < adev->uvd.num_enc_rings; ++i)
+			amdgpu_ring_fini(&adev->uvd.inst[j].ring_enc[i]);
+	}
 	return amdgpu_uvd_sw_fini(adev);
 }
 
@@ -490,9 +509,9 @@ static int uvd_v7_0_sw_fini(void *handle)
 static int uvd_v7_0_hw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	struct amdgpu_ring *ring;
 	uint32_t tmp;
-	int i, r;
+	int i, j, r;
 
 	if (amdgpu_sriov_vf(adev))
 		r = uvd_v7_0_sriov_start(adev);
@@ -501,57 +520,60 @@ static int uvd_v7_0_hw_init(void *handle)
 	if (r)
 		goto done;
 
-	if (!amdgpu_sriov_vf(adev)) {
-		ring->ready = true;
-		r = amdgpu_ring_test_ring(ring);
-		if (r) {
-			ring->ready = false;
-			goto done;
+	for (j = 0; j < adev->uvd.num_uvd_inst; ++j) {
+		ring = &adev->uvd.inst[j].ring;
+
+		if (!amdgpu_sriov_vf(adev)) {
+			ring->ready = true;
+			r = amdgpu_ring_test_ring(ring);
+			if (r) {
+				ring->ready = false;
+				goto done;
+			}
+
+			r = amdgpu_ring_alloc(ring, 10);
+			if (r) {
+				DRM_ERROR("amdgpu: (%d)ring failed to lock UVD ring (%d).\n", j, r);
+				goto done;
+			}
+
+			tmp = PACKET0(SOC15_REG_OFFSET(UVD, j,
+				mmUVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL), 0);
+			amdgpu_ring_write(ring, tmp);
+			amdgpu_ring_write(ring, 0xFFFFF);
+
+			tmp = PACKET0(SOC15_REG_OFFSET(UVD, j,
+				mmUVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL), 0);
+			amdgpu_ring_write(ring, tmp);
+			amdgpu_ring_write(ring, 0xFFFFF);
+
+			tmp = PACKET0(SOC15_REG_OFFSET(UVD, j,
+				mmUVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL), 0);
+			amdgpu_ring_write(ring, tmp);
+			amdgpu_ring_write(ring, 0xFFFFF);
+
+			/* Clear timeout status bits */
+			amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, j,
+				mmUVD_SEMA_TIMEOUT_STATUS), 0));
+			amdgpu_ring_write(ring, 0x8);
+
+			amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, j,
+				mmUVD_SEMA_CNTL), 0));
+			amdgpu_ring_write(ring, 3);
+
+			amdgpu_ring_commit(ring);
 		}
 
-		r = amdgpu_ring_alloc(ring, 10);
-		if (r) {
-			DRM_ERROR("amdgpu: ring failed to lock UVD ring (%d).\n", r);
-			goto done;
+		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
+			ring = &adev->uvd.inst[j].ring_enc[i];
+			ring->ready = true;
+			r = amdgpu_ring_test_ring(ring);
+			if (r) {
+				ring->ready = false;
+				goto done;
+			}
 		}
-
-		tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0,
-			mmUVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL), 0);
-		amdgpu_ring_write(ring, tmp);
-		amdgpu_ring_write(ring, 0xFFFFF);
-
-		tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0,
-			mmUVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL), 0);
-		amdgpu_ring_write(ring, tmp);
-		amdgpu_ring_write(ring, 0xFFFFF);
-
-		tmp = PACKET0(SOC15_REG_OFFSET(UVD, 0,
-			mmUVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL), 0);
-		amdgpu_ring_write(ring, tmp);
-		amdgpu_ring_write(ring, 0xFFFFF);
-
-		/* Clear timeout status bits */
-		amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0,
-			mmUVD_SEMA_TIMEOUT_STATUS), 0));
-		amdgpu_ring_write(ring, 0x8);
-
-		amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0,
-			mmUVD_SEMA_CNTL), 0));
-		amdgpu_ring_write(ring, 3);
-
-		amdgpu_ring_commit(ring);
 	}
-
-	for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
-		ring = &adev->uvd.ring_enc[i];
-		ring->ready = true;
-		r = amdgpu_ring_test_ring(ring);
-		if (r) {
-			ring->ready = false;
-			goto done;
-		}
-	}
-
 done:
 	if (!r)
 		DRM_INFO("UVD and UVD ENC initialized successfully.\n");
@@ -569,7 +591,7 @@ static int uvd_v7_0_hw_init(void *handle)
 static int uvd_v7_0_hw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	int i;
 
 	if (!amdgpu_sriov_vf(adev))
 		uvd_v7_0_stop(adev);
@@ -578,7 +600,8 @@ static int uvd_v7_0_hw_fini(void *handle)
 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
 	}
 
-	ring->ready = false;
+	for (i = 0; i < adev->uvd.num_uvd_inst; ++i)
+		adev->uvd.inst[i].ring.ready = false;
 
 	return 0;
 }
@@ -618,48 +641,51 @@ static void uvd_v7_0_mc_resume(struct amdgpu_device *adev)
 {
 	uint32_t size = AMDGPU_UVD_FIRMWARE_SIZE(adev);
 	uint32_t offset;
+	int i;
 
-	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
-			lower_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
-		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
-			upper_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
-		offset = 0;
-	} else {
-		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
-			lower_32_bits(adev->uvd.gpu_addr));
-		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
-			upper_32_bits(adev->uvd.gpu_addr));
-		offset = size;
-	}
+	for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {
+		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+			WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
+				lower_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
+			WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
+				upper_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
+			offset = 0;
+		} else {
+			WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
+				lower_32_bits(adev->uvd.inst[i].gpu_addr));
+			WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
+				upper_32_bits(adev->uvd.inst[i].gpu_addr));
+			offset = size;
+		}
 
-	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
-				AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
-	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
-
-	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
-			lower_32_bits(adev->uvd.gpu_addr + offset));
-	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
-			upper_32_bits(adev->uvd.gpu_addr + offset));
-	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, (1 << 21));
-	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_UVD_HEAP_SIZE);
-
-	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
-			lower_32_bits(adev->uvd.gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
-	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
-			upper_32_bits(adev->uvd.gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
-	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, (2 << 21));
-	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2,
-			AMDGPU_UVD_STACK_SIZE + (AMDGPU_UVD_SESSION_SIZE * 40));
-
-	WREG32_SOC15(UVD, 0, mmUVD_UDEC_ADDR_CONFIG,
-			adev->gfx.config.gb_addr_config);
-	WREG32_SOC15(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG,
-			adev->gfx.config.gb_addr_config);
-	WREG32_SOC15(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG,
-			adev->gfx.config.gb_addr_config);
-
-	WREG32_SOC15(UVD, 0, mmUVD_GP_SCRATCH4, adev->uvd.max_handles);
+		WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_OFFSET0,
+					AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
+		WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_SIZE0, size);
+
+		WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
+				lower_32_bits(adev->uvd.inst[i].gpu_addr + offset));
+		WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
+				upper_32_bits(adev->uvd.inst[i].gpu_addr + offset));
+		WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_OFFSET1, (1 << 21));
+		WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_UVD_HEAP_SIZE);
+
+		WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
+				lower_32_bits(adev->uvd.inst[i].gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
+		WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
+				upper_32_bits(adev->uvd.inst[i].gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
+		WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_OFFSET2, (2 << 21));
+		WREG32_SOC15(UVD, i, mmUVD_VCPU_CACHE_SIZE2,
+				AMDGPU_UVD_STACK_SIZE + (AMDGPU_UVD_SESSION_SIZE * 40));
+
+		WREG32_SOC15(UVD, i, mmUVD_UDEC_ADDR_CONFIG,
+				adev->gfx.config.gb_addr_config);
+		WREG32_SOC15(UVD, i, mmUVD_UDEC_DB_ADDR_CONFIG,
+				adev->gfx.config.gb_addr_config);
+		WREG32_SOC15(UVD, i, mmUVD_UDEC_DBW_ADDR_CONFIG,
+				adev->gfx.config.gb_addr_config);
+
+		WREG32_SOC15(UVD, i, mmUVD_GP_SCRATCH4, adev->uvd.max_handles);
+	}
 }
 
 static int uvd_v7_0_mmsch_start(struct amdgpu_device *adev,
@@ -669,6 +695,7 @@ static int uvd_v7_0_mmsch_start(struct amdgpu_device *adev,
 	uint64_t addr = table->gpu_addr;
 	struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)table->cpu_addr;
 	uint32_t size;
+	int i;
 
 	size = header->header_size + header->vce_table_size + header->uvd_table_size;
 
@@ -688,11 +715,12 @@ static int uvd_v7_0_mmsch_start(struct amdgpu_device *adev,
 	/* 4, set resp to zero */
 	WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP, 0);
 
-	WDOORBELL32(adev->uvd.ring_enc[0].doorbell_index, 0);
-	adev->wb.wb[adev->uvd.ring_enc[0].wptr_offs] = 0;
-	adev->uvd.ring_enc[0].wptr = 0;
-	adev->uvd.ring_enc[0].wptr_old = 0;
-
+	for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {
+		WDOORBELL32(adev->uvd.inst[i].ring_enc[0].doorbell_index, 0);
+		adev->wb.wb[adev->uvd.inst[i].ring_enc[0].wptr_offs] = 0;
+		adev->uvd.inst[i].ring_enc[0].wptr = 0;
+		adev->uvd.inst[i].ring_enc[0].wptr_old = 0;
+	}
 	/* 5, kick off the initialization and wait until VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero */
 	WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_HOST, 0x10000001);
 
@@ -725,6 +753,7 @@ static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)
 	struct mmsch_v1_0_cmd_end end = { {0} };
 	uint32_t *init_table = adev->virt.mm_table.cpu_addr;
 	struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)init_table;
+	uint8_t i = 0;
 
 	direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;
 	direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
@@ -742,120 +771,121 @@ static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)
 
 		init_table += header->uvd_table_offset;
 
-		ring = &adev->uvd.ring;
-		ring->wptr = 0;
-		size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4);
-
-		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS),
-						   0xFFFFFFFF, 0x00000004);
-		/* mc resume*/
-		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
-						    lower_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
-			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
-						    upper_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
-			offset = 0;
-		} else {
-			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
-						    lower_32_bits(adev->uvd.gpu_addr));
-			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
-						    upper_32_bits(adev->uvd.gpu_addr));
-			offset = size;
+		for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {
+			ring = &adev->uvd.inst[i].ring;
+			ring->wptr = 0;
+			size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4);
+
+			MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS),
+							   0xFFFFFFFF, 0x00000004);
+			/* mc resume*/
+			if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+				MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
+							    lower_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
+				MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
+							    upper_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
+				offset = 0;
+			} else {
+				MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
+							    lower_32_bits(adev->uvd.inst[i].gpu_addr));
+				MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
+							    upper_32_bits(adev->uvd.inst[i].gpu_addr));
+				offset = size;
+			}
+
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET0),
+						    AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE0), size);
+
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
+						    lower_32_bits(adev->uvd.inst[i].gpu_addr + offset));
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
+						    upper_32_bits(adev->uvd.inst[i].gpu_addr + offset));
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET1), (1 << 21));
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_UVD_HEAP_SIZE);
+
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
+						    lower_32_bits(adev->uvd.inst[i].gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
+						    upper_32_bits(adev->uvd.inst[i].gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET2), (2 << 21));
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE2),
+						    AMDGPU_UVD_STACK_SIZE + (AMDGPU_UVD_SESSION_SIZE * 40));
+
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_GP_SCRATCH4), adev->uvd.max_handles);
+			/* mc resume end*/
+
+			/* disable clock gating */
+			MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_CGC_CTRL),
+							   ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK, 0);
+
+			/* disable interupt */
+			MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_MASTINT_EN),
+							   ~UVD_MASTINT_EN__VCPU_EN_MASK, 0);
+
+			/* stall UMC and register bus before resetting VCPU */
+			MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_CTRL2),
+							   ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
+							   UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
+
+			/* put LMI, VCPU, RBC etc... into reset */
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_SOFT_RESET),
+						    (uint32_t)(UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
+							       UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
+							       UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
+							       UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
+							       UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
+							       UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
+							       UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
+							       UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK));
+
+			/* initialize UVD memory controller */
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_CTRL),
+						    (uint32_t)((0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
+							       UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
+							       UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
+							       UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
+							       UVD_LMI_CTRL__REQ_MODE_MASK |
+							       0x00100000L));
+
+			/* take all subblocks out of reset, except VCPU */
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_SOFT_RESET),
+						    UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
+
+			/* enable VCPU clock */
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CNTL),
+						    UVD_VCPU_CNTL__CLK_EN_MASK);
+
+			/* enable master interrupt */
+			MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_MASTINT_EN),
+							   ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
+							   (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
+
+			/* clear the bit 4 of UVD_STATUS */
+			MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS),
+							   ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT), 0);
+
+			/* force RBC into idle state */
+			size = order_base_2(ring->ring_size);
+			tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, size);
+			tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_RBC_RB_CNTL), tmp);
+
+			ring = &adev->uvd.inst[i].ring_enc[0];
+			ring->wptr = 0;
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_LO), ring->gpu_addr);
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_HI), upper_32_bits(ring->gpu_addr));
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_RB_SIZE), ring->ring_size / 4);
+
+			/* boot up the VCPU */
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_SOFT_RESET), 0);
+
+			/* enable UMC */
+			MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_CTRL2),
+											   ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, 0);
+
+			MMSCH_V1_0_INSERT_DIRECT_POLL(SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS), 0x02, 0x02);
 		}
-
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0),
-					    AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size);
-
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
-					    lower_32_bits(adev->uvd.gpu_addr + offset));
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
-					    upper_32_bits(adev->uvd.gpu_addr + offset));
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), (1 << 21));
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_UVD_HEAP_SIZE);
-
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
-					    lower_32_bits(adev->uvd.gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
-					    upper_32_bits(adev->uvd.gpu_addr + offset + AMDGPU_UVD_HEAP_SIZE));
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), (2 << 21));
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE2),
-					    AMDGPU_UVD_STACK_SIZE + (AMDGPU_UVD_SESSION_SIZE * 40));
-
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH4), adev->uvd.max_handles);
-		/* mc resume end*/
-
-		/* disable clock gating */
-		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_CGC_CTRL),
-						   ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK, 0);
-
-		/* disable interupt */
-		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
-						   ~UVD_MASTINT_EN__VCPU_EN_MASK, 0);
-
-		/* stall UMC and register bus before resetting VCPU */
-		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
-						   ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
-						   UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
-
-		/* put LMI, VCPU, RBC etc... into reset */
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
-					    (uint32_t)(UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
-						       UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
-						       UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
-						       UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
-						       UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
-						       UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
-						       UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
-						       UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK));
-
-		/* initialize UVD memory controller */
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL),
-					    (uint32_t)((0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
-						       UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
-						       UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
-						       UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
-						       UVD_LMI_CTRL__REQ_MODE_MASK |
-						       0x00100000L));
-
-		/* take all subblocks out of reset, except VCPU */
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
-					    UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
-
-		/* enable VCPU clock */
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL),
-					    UVD_VCPU_CNTL__CLK_EN_MASK);
-
-		/* enable master interrupt */
-		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
-						   ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
-						   (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
-
-		/* clear the bit 4 of UVD_STATUS */
-		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS),
-						   ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT), 0);
-
-		/* force RBC into idle state */
-		size = order_base_2(ring->ring_size);
-		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, size);
-		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), tmp);
-
-		ring = &adev->uvd.ring_enc[0];
-		ring->wptr = 0;
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_LO), ring->gpu_addr);
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_HI), upper_32_bits(ring->gpu_addr));
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_SIZE), ring->ring_size / 4);
-
-		/* boot up the VCPU */
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0);
-
-		/* enable UMC */
-		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
-										   ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, 0);
-
-		MMSCH_V1_0_INSERT_DIRECT_POLL(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0x02, 0x02);
-
 		/* add end packet */
 		memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
 		table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;
@@ -874,15 +904,17 @@ static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)
  */
 static int uvd_v7_0_start(struct amdgpu_device *adev)
 {
-	struct amdgpu_ring *ring = &adev->uvd.ring;
+	struct amdgpu_ring *ring;
 	uint32_t rb_bufsz, tmp;
 	uint32_t lmi_swap_cntl;
 	uint32_t mp_swap_cntl;
-	int i, j, r;
+	int i, j, k, r;
 
-	/* disable DPG */
-	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 0,
-			~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
+	for (k = 0; k < adev->uvd.num_uvd_inst; ++k) {
+		/* disable DPG */
+		WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_POWER_STATUS), 0,
+				~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
+	}
 
 	/* disable byte swapping */
 	lmi_swap_cntl = 0;
@@ -890,157 +922,159 @@ static int uvd_v7_0_start(struct amdgpu_device *adev)
 
 	uvd_v7_0_mc_resume(adev);
 
-	/* disable clock gating */
-	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_CGC_CTRL), 0,
-			~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK);
-
-	/* disable interupt */
-	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
-			~UVD_MASTINT_EN__VCPU_EN_MASK);
-
-	/* stall UMC and register bus before resetting VCPU */
-	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
-			UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
-			~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
-	mdelay(1);
-
-	/* put LMI, VCPU, RBC etc... into reset */
-	WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
-		UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
-		UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
-		UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
-		UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
-		UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
-		UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
-		UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
-		UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
-	mdelay(5);
+	for (k = 0; k < adev->uvd.num_uvd_inst; ++k) {
+		ring = &adev->uvd.inst[k].ring;
+		/* disable clock gating */
+		WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_CGC_CTRL), 0,
+				~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK);
 
-	/* initialize UVD memory controller */
-	WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL,
-		(0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
-		UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
-		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
-		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
-		UVD_LMI_CTRL__REQ_MODE_MASK |
-		0x00100000L);
+		/* disable interupt */
+		WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_MASTINT_EN), 0,
+				~UVD_MASTINT_EN__VCPU_EN_MASK);
+
+		/* stall UMC and register bus before resetting VCPU */
+		WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_LMI_CTRL2),
+				UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
+				~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
+		mdelay(1);
+
+		/* put LMI, VCPU, RBC etc... into reset */
+		WREG32_SOC15(UVD, k, mmUVD_SOFT_RESET,
+			UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
+			UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
+			UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
+			UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
+			UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
+			UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
+			UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
+			UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
+		mdelay(5);
+
+		/* initialize UVD memory controller */
+		WREG32_SOC15(UVD, k, mmUVD_LMI_CTRL,
+			(0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
+			UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
+			UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
+			UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
+			UVD_LMI_CTRL__REQ_MODE_MASK |
+			0x00100000L);
 
 #ifdef __BIG_ENDIAN
-	/* swap (8 in 32) RB and IB */
-	lmi_swap_cntl = 0xa;
-	mp_swap_cntl = 0;
+		/* swap (8 in 32) RB and IB */
+		lmi_swap_cntl = 0xa;
+		mp_swap_cntl = 0;
 #endif
-	WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
-	WREG32_SOC15(UVD, 0, mmUVD_MP_SWAP_CNTL, mp_swap_cntl);
-
-	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0, 0x40c2040);
-	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA1, 0x0);
-	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0, 0x40c2040);
-	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB1, 0x0);
-	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_ALU, 0);
-	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX, 0x88);
-
-	/* take all subblocks out of reset, except VCPU */
-	WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
-			UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
-	mdelay(5);
+		WREG32_SOC15(UVD, k, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
+		WREG32_SOC15(UVD, k, mmUVD_MP_SWAP_CNTL, mp_swap_cntl);
 
-	/* enable VCPU clock */
-	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL,
-			UVD_VCPU_CNTL__CLK_EN_MASK);
+		WREG32_SOC15(UVD, k, mmUVD_MPC_SET_MUXA0, 0x40c2040);
+		WREG32_SOC15(UVD, k, mmUVD_MPC_SET_MUXA1, 0x0);
+		WREG32_SOC15(UVD, k, mmUVD_MPC_SET_MUXB0, 0x40c2040);
+		WREG32_SOC15(UVD, k, mmUVD_MPC_SET_MUXB1, 0x0);
+		WREG32_SOC15(UVD, k, mmUVD_MPC_SET_ALU, 0);
+		WREG32_SOC15(UVD, k, mmUVD_MPC_SET_MUX, 0x88);
 
-	/* enable UMC */
-	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
-			~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
+		/* take all subblocks out of reset, except VCPU */
+		WREG32_SOC15(UVD, k, mmUVD_SOFT_RESET,
+				UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
+		mdelay(5);
 
-	/* boot up the VCPU */
-	WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET, 0);
-	mdelay(10);
+		/* enable VCPU clock */
+		WREG32_SOC15(UVD, k, mmUVD_VCPU_CNTL,
+				UVD_VCPU_CNTL__CLK_EN_MASK);
 
-	for (i = 0; i < 10; ++i) {
-		uint32_t status;
+		/* enable UMC */
+		WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_LMI_CTRL2), 0,
+				~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 
-		for (j = 0; j < 100; ++j) {
-			status = RREG32_SOC15(UVD, 0, mmUVD_STATUS);
+		/* boot up the VCPU */
+		WREG32_SOC15(UVD, k, mmUVD_SOFT_RESET, 0);
+		mdelay(10);
+
+		for (i = 0; i < 10; ++i) {
+			uint32_t status;
+
+			for (j = 0; j < 100; ++j) {
+				status = RREG32_SOC15(UVD, k, mmUVD_STATUS);
+				if (status & 2)
+					break;
+				mdelay(10);
+			}
+			r = 0;
 			if (status & 2)
 				break;
+
+			DRM_ERROR("UVD(%d) not responding, trying to reset the VCPU!!!\n", k);
+			WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_SOFT_RESET),
+					UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
+					~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 			mdelay(10);
+			WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_SOFT_RESET), 0,
+					~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
+			mdelay(10);
+			r = -1;
 		}
-		r = 0;
-		if (status & 2)
-			break;
-
-		DRM_ERROR("UVD not responding, trying to reset the VCPU!!!\n");
-		WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
-				UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
-				~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
-		mdelay(10);
-		WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
-				~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
-		mdelay(10);
-		r = -1;
-	}
-
-	if (r) {
-		DRM_ERROR("UVD not responding, giving up!!!\n");
-		return r;
-	}
-	/* enable master interrupt */
-	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
-		(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
-		~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
-
-	/* clear the bit 4 of UVD_STATUS */
-	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0,
-			~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
-
-	/* force RBC into idle state */
-	rb_bufsz = order_base_2(ring->ring_size);
-	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
-	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
-	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
-	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
-	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
-	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
-	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
-
-	/* set the write pointer delay */
-	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
-
-	/* set the wb address */
-	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
-			(upper_32_bits(ring->gpu_addr) >> 2));
-
-	/* programm the RB_BASE for ring buffer */
-	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
-			lower_32_bits(ring->gpu_addr));
-	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
-			upper_32_bits(ring->gpu_addr));
-
-	/* Initialize the ring buffer's read and write pointers */
-	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
-
-	ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
-	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
-			lower_32_bits(ring->wptr));
 
-	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), 0,
-			~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
-
-	ring = &adev->uvd.ring_enc[0];
-	WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
-	WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
-	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
-	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
-	WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
+		if (r) {
+			DRM_ERROR("UVD(%d) not responding, giving up!!!\n", k);
+			return r;
+		}
+		/* enable master interrupt */
+		WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_MASTINT_EN),
+			(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
+			~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
 
-	ring = &adev->uvd.ring_enc[1];
-	WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
-	WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
-	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
-	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
-	WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
+		/* clear the bit 4 of UVD_STATUS */
+		WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_STATUS), 0,
+				~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
 
+		/* force RBC into idle state */
+		rb_bufsz = order_base_2(ring->ring_size);
+		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
+		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
+		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
+		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
+		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
+		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
+		WREG32_SOC15(UVD, k, mmUVD_RBC_RB_CNTL, tmp);
+
+		/* set the write pointer delay */
+		WREG32_SOC15(UVD, k, mmUVD_RBC_RB_WPTR_CNTL, 0);
+
+		/* set the wb address */
+		WREG32_SOC15(UVD, k, mmUVD_RBC_RB_RPTR_ADDR,
+				(upper_32_bits(ring->gpu_addr) >> 2));
+
+		/* programm the RB_BASE for ring buffer */
+		WREG32_SOC15(UVD, k, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
+				lower_32_bits(ring->gpu_addr));
+		WREG32_SOC15(UVD, k, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
+				upper_32_bits(ring->gpu_addr));
+
+		/* Initialize the ring buffer's read and write pointers */
+		WREG32_SOC15(UVD, k, mmUVD_RBC_RB_RPTR, 0);
+
+		ring->wptr = RREG32_SOC15(UVD, k, mmUVD_RBC_RB_RPTR);
+		WREG32_SOC15(UVD, k, mmUVD_RBC_RB_WPTR,
+				lower_32_bits(ring->wptr));
+
+		WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_RBC_RB_CNTL), 0,
+				~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
+
+		ring = &adev->uvd.inst[k].ring_enc[0];
+		WREG32_SOC15(UVD, k, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
+		WREG32_SOC15(UVD, k, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
+		WREG32_SOC15(UVD, k, mmUVD_RB_BASE_LO, ring->gpu_addr);
+		WREG32_SOC15(UVD, k, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
+		WREG32_SOC15(UVD, k, mmUVD_RB_SIZE, ring->ring_size / 4);
+
+		ring = &adev->uvd.inst[k].ring_enc[1];
+		WREG32_SOC15(UVD, k, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
+		WREG32_SOC15(UVD, k, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
+		WREG32_SOC15(UVD, k, mmUVD_RB_BASE_LO2, ring->gpu_addr);
+		WREG32_SOC15(UVD, k, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
+		WREG32_SOC15(UVD, k, mmUVD_RB_SIZE2, ring->ring_size / 4);
+	}
 	return 0;
 }
 
@@ -1053,26 +1087,30 @@ static int uvd_v7_0_start(struct amdgpu_device *adev)
  */
 static void uvd_v7_0_stop(struct amdgpu_device *adev)
 {
-	/* force RBC into idle state */
-	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, 0x11010101);
-
-	/* Stall UMC and register bus before resetting VCPU */
-	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
-			UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
-			~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
-	mdelay(1);
-
-	/* put VCPU into reset */
-	WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
-			UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
-	mdelay(5);
+	uint8_t i = 0;
 
-	/* disable VCPU clock */
-	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL, 0x0);
+	for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {
+		/* force RBC into idle state */
+		WREG32_SOC15(UVD, i, mmUVD_RBC_RB_CNTL, 0x11010101);
 
-	/* Unstall UMC and register bus */
-	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
-			~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
+		/* Stall UMC and register bus before resetting VCPU */
+		WREG32_P(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_CTRL2),
+				UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
+				~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
+		mdelay(1);
+
+		/* put VCPU into reset */
+		WREG32_SOC15(UVD, i, mmUVD_SOFT_RESET,
+				UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
+		mdelay(5);
+
+		/* disable VCPU clock */
+		WREG32_SOC15(UVD, i, mmUVD_VCPU_CNTL, 0x0);
+
+		/* Unstall UMC and register bus */
+		WREG32_P(SOC15_REG_OFFSET(UVD, i, mmUVD_LMI_CTRL2), 0,
+				~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
+	}
 }
 
 /**
@@ -1091,26 +1129,26 @@ static void uvd_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq
 	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_CONTEXT_ID), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_CONTEXT_ID), 0));
 	amdgpu_ring_write(ring, seq);
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA0), 0));
 	amdgpu_ring_write(ring, addr & 0xffffffff);
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA1), 0));
 	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_CMD), 0));
 	amdgpu_ring_write(ring, 0);
 
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA0), 0));
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA1), 0));
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_CMD), 0));
 	amdgpu_ring_write(ring, 2);
 }
 
@@ -1135,6 +1173,16 @@ static void uvd_v7_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_TRAP);
 }
 
+/**
+ * uvd_v7_0_ring_emit_hdp_flush - skip HDP flushing
+ *
+ * @ring: amdgpu_ring pointer
+ */
+static void uvd_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
+{
+	/* The firmware doesn't seem to like touching registers at this point. */
+}
+
 /**
  * uvd_v7_0_ring_test_ring - register write test
  *
@@ -1149,30 +1197,30 @@ static int uvd_v7_0_ring_test_ring(struct amdgpu_ring *ring)
 	unsigned i;
 	int r;
 
-	WREG32_SOC15(UVD, 0, mmUVD_CONTEXT_ID, 0xCAFEDEAD);
+	WREG32_SOC15(UVD, ring->me, mmUVD_CONTEXT_ID, 0xCAFEDEAD);
 	r = amdgpu_ring_alloc(ring, 3);
 	if (r) {
-		DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
-			  ring->idx, r);
+		DRM_ERROR("amdgpu: (%d)cp failed to lock ring %d (%d).\n",
+			  ring->me, ring->idx, r);
 		return r;
 	}
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_CONTEXT_ID), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_CONTEXT_ID), 0));
 	amdgpu_ring_write(ring, 0xDEADBEEF);
 	amdgpu_ring_commit(ring);
 	for (i = 0; i < adev->usec_timeout; i++) {
-		tmp = RREG32_SOC15(UVD, 0, mmUVD_CONTEXT_ID);
+		tmp = RREG32_SOC15(UVD, ring->me, mmUVD_CONTEXT_ID);
 		if (tmp == 0xDEADBEEF)
 			break;
 		DRM_UDELAY(1);
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
-			 ring->idx, i);
+		DRM_DEBUG("(%d)ring test on %d succeeded in %d usecs\n",
+			 ring->me, ring->idx, i);
 	} else {
-		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
-			  ring->idx, tmp);
+		DRM_ERROR("(%d)amdgpu: ring %d test failed (0x%08X)\n",
+			  ring->me, ring->idx, tmp);
 		r = -EINVAL;
 	}
 	return r;
@@ -1193,17 +1241,17 @@ static void uvd_v7_0_ring_emit_ib(struct amdgpu_ring *ring,
 	struct amdgpu_device *adev = ring->adev;
 
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_VMID), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_LMI_RBC_IB_VMID), 0));
 	amdgpu_ring_write(ring, vmid);
 
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_LOW), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_LMI_RBC_IB_64BIT_BAR_LOW), 0));
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH), 0));
 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_IB_SIZE), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_RBC_IB_SIZE), 0));
 	amdgpu_ring_write(ring, ib->length_dw);
 }
 
@@ -1231,13 +1279,13 @@ static void uvd_v7_0_ring_emit_wreg(struct amdgpu_ring *ring,
 	struct amdgpu_device *adev = ring->adev;
 
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA0), 0));
 	amdgpu_ring_write(ring, reg << 2);
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA1), 0));
 	amdgpu_ring_write(ring, val);
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_CMD), 0));
 	amdgpu_ring_write(ring, 8);
 }
 
@@ -1247,16 +1295,16 @@ static void uvd_v7_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 	struct amdgpu_device *adev = ring->adev;
 
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA0), 0));
 	amdgpu_ring_write(ring, reg << 2);
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_DATA1), 0));
 	amdgpu_ring_write(ring, val);
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH8), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GP_SCRATCH8), 0));
 	amdgpu_ring_write(ring, mask);
 	amdgpu_ring_write(ring,
-		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
+		PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_GPCOM_VCPU_CMD), 0));
 	amdgpu_ring_write(ring, 12);
 }
 
@@ -1277,12 +1325,15 @@ static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 
 static void uvd_v7_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
-	int i;
 	struct amdgpu_device *adev = ring->adev;
+	int i;
 
-	for (i = 0; i < count; i++)
-		amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0));
+	WARN_ON(ring->wptr % 2 || count % 2);
 
+	for (i = 0; i < count / 2; i++) {
+		amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, ring->me, mmUVD_NO_OP), 0));
+		amdgpu_ring_write(ring, 0);
+	}
 }
 
 static void uvd_v7_0_enc_ring_insert_end(struct amdgpu_ring *ring)
@@ -1349,16 +1400,16 @@ static bool uvd_v7_0_check_soft_reset(void *handle)
 
 	if (REG_GET_FIELD(tmp, SRBM_STATUS, UVD_RQ_PENDING) ||
 	    REG_GET_FIELD(tmp, SRBM_STATUS, UVD_BUSY) ||
-	    (RREG32_SOC15(UVD, 0, mmUVD_STATUS) &
+	    (RREG32_SOC15(UVD, ring->me, mmUVD_STATUS) &
 		    AMDGPU_UVD_STATUS_BUSY_MASK))
 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
 				SRBM_SOFT_RESET, SOFT_RESET_UVD, 1);
 
 	if (srbm_soft_reset) {
-		adev->uvd.srbm_soft_reset = srbm_soft_reset;
+		adev->uvd.inst[ring->me].srbm_soft_reset = srbm_soft_reset;
 		return true;
 	} else {
-		adev->uvd.srbm_soft_reset = 0;
+		adev->uvd.inst[ring->me].srbm_soft_reset = 0;
 		return false;
 	}
 }
@@ -1367,7 +1418,7 @@ static int uvd_v7_0_pre_soft_reset(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	if (!adev->uvd.srbm_soft_reset)
+	if (!adev->uvd.inst[ring->me].srbm_soft_reset)
 		return 0;
 
 	uvd_v7_0_stop(adev);
@@ -1379,9 +1430,9 @@ static int uvd_v7_0_soft_reset(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	u32 srbm_soft_reset;
 
-	if (!adev->uvd.srbm_soft_reset)
+	if (!adev->uvd.inst[ring->me].srbm_soft_reset)
 		return 0;
-	srbm_soft_reset = adev->uvd.srbm_soft_reset;
+	srbm_soft_reset = adev->uvd.inst[ring->me].srbm_soft_reset;
 
 	if (srbm_soft_reset) {
 		u32 tmp;
@@ -1409,7 +1460,7 @@ static int uvd_v7_0_post_soft_reset(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	if (!adev->uvd.srbm_soft_reset)
+	if (!adev->uvd.inst[ring->me].srbm_soft_reset)
 		return 0;
 
 	mdelay(5);
@@ -1431,17 +1482,32 @@ static int uvd_v7_0_process_interrupt(struct amdgpu_device *adev,
 				      struct amdgpu_irq_src *source,
 				      struct amdgpu_iv_entry *entry)
 {
+	uint32_t ip_instance;
+
+	switch (entry->client_id) {
+	case SOC15_IH_CLIENTID_UVD:
+		ip_instance = 0;
+		break;
+	case SOC15_IH_CLIENTID_UVD1:
+		ip_instance = 1;
+		break;
+	default:
+		DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
+		return 0;
+	}
+
 	DRM_DEBUG("IH: UVD TRAP\n");
+
 	switch (entry->src_id) {
 	case 124:
-		amdgpu_fence_process(&adev->uvd.ring);
+		amdgpu_fence_process(&adev->uvd.inst[ip_instance].ring);
 		break;
 	case 119:
-		amdgpu_fence_process(&adev->uvd.ring_enc[0]);
+		amdgpu_fence_process(&adev->uvd.inst[ip_instance].ring_enc[0]);
 		break;
 	case 120:
 		if (!amdgpu_sriov_vf(adev))
-			amdgpu_fence_process(&adev->uvd.ring_enc[1]);
+			amdgpu_fence_process(&adev->uvd.inst[ip_instance].ring_enc[1]);
 		break;
 	default:
 		DRM_ERROR("Unhandled interrupt: %d %d\n",
@@ -1457,9 +1523,9 @@ static void uvd_v7_0_set_sw_clock_gating(struct amdgpu_device *adev)
 {
 	uint32_t data, data1, data2, suvd_flags;
 
-	data = RREG32_SOC15(UVD, 0, mmUVD_CGC_CTRL);
-	data1 = RREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_GATE);
-	data2 = RREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_CTRL);
+	data = RREG32_SOC15(UVD, ring->me, mmUVD_CGC_CTRL);
+	data1 = RREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_GATE);
+	data2 = RREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_CTRL);
 
 	data &= ~(UVD_CGC_CTRL__CLK_OFF_DELAY_MASK |
 		  UVD_CGC_CTRL__CLK_GATE_DLY_TIMER_MASK);
@@ -1503,18 +1569,18 @@ static void uvd_v7_0_set_sw_clock_gating(struct amdgpu_device *adev)
 			UVD_SUVD_CGC_CTRL__SDB_MODE_MASK);
 	data1 |= suvd_flags;
 
-	WREG32_SOC15(UVD, 0, mmUVD_CGC_CTRL, data);
-	WREG32_SOC15(UVD, 0, mmUVD_CGC_GATE, 0);
-	WREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_GATE, data1);
-	WREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_CTRL, data2);
+	WREG32_SOC15(UVD, ring->me, mmUVD_CGC_CTRL, data);
+	WREG32_SOC15(UVD, ring->me, mmUVD_CGC_GATE, 0);
+	WREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_GATE, data1);
+	WREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_CTRL, data2);
 }
 
 static void uvd_v7_0_set_hw_clock_gating(struct amdgpu_device *adev)
 {
 	uint32_t data, data1, cgc_flags, suvd_flags;
 
-	data = RREG32_SOC15(UVD, 0, mmUVD_CGC_GATE);
-	data1 = RREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_GATE);
+	data = RREG32_SOC15(UVD, ring->me, mmUVD_CGC_GATE);
+	data1 = RREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_GATE);
 
 	cgc_flags = UVD_CGC_GATE__SYS_MASK |
 		UVD_CGC_GATE__UDEC_MASK |
@@ -1546,8 +1612,8 @@ static void uvd_v7_0_set_hw_clock_gating(struct amdgpu_device *adev)
 	data |= cgc_flags;
 	data1 |= suvd_flags;
 
-	WREG32_SOC15(UVD, 0, mmUVD_CGC_GATE, data);
-	WREG32_SOC15(UVD, 0, mmUVD_SUVD_CGC_GATE, data1);
+	WREG32_SOC15(UVD, ring->me, mmUVD_CGC_GATE, data);
+	WREG32_SOC15(UVD, ring->me, mmUVD_SUVD_CGC_GATE, data1);
 }
 
 static void uvd_v7_0_set_bypass_mode(struct amdgpu_device *adev, bool enable)
@@ -1606,7 +1672,7 @@ static int uvd_v7_0_set_powergating_state(void *handle,
 	if (!(adev->pg_flags & AMD_PG_SUPPORT_UVD))
 		return 0;
 
-	WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, UVD_POWER_STATUS__UVD_PG_EN_MASK);
+	WREG32_SOC15(UVD, ring->me, mmUVD_POWER_STATUS, UVD_POWER_STATUS__UVD_PG_EN_MASK);
 
 	if (state == AMD_PG_STATE_GATE) {
 		uvd_v7_0_stop(adev);
@@ -1647,14 +1713,13 @@ const struct amd_ip_funcs uvd_v7_0_ip_funcs = {
 static const struct amdgpu_ring_funcs uvd_v7_0_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_UVD,
 	.align_mask = 0xf,
-	.nop = PACKET0(0x81ff, 0),
 	.support_64bit_ptrs = false,
 	.vmhub = AMDGPU_MMHUB,
 	.get_rptr = uvd_v7_0_ring_get_rptr,
 	.get_wptr = uvd_v7_0_ring_get_wptr,
 	.set_wptr = uvd_v7_0_ring_set_wptr,
 	.emit_frame_size =
-		6 + 6 + /* hdp flush / invalidate */
+		6 + /* hdp invalidate */
 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
 		8 + /* uvd_v7_0_ring_emit_vm_flush */
@@ -1663,6 +1728,7 @@ static const struct amdgpu_ring_funcs uvd_v7_0_ring_vm_funcs = {
 	.emit_ib = uvd_v7_0_ring_emit_ib,
 	.emit_fence = uvd_v7_0_ring_emit_fence,
 	.emit_vm_flush = uvd_v7_0_ring_emit_vm_flush,
+	.emit_hdp_flush = uvd_v7_0_ring_emit_hdp_flush,
 	.test_ring = uvd_v7_0_ring_test_ring,
 	.test_ib = amdgpu_uvd_ring_test_ib,
 	.insert_nop = uvd_v7_0_ring_insert_nop,
@@ -1671,6 +1737,7 @@ static const struct amdgpu_ring_funcs uvd_v7_0_ring_vm_funcs = {
 	.end_use = amdgpu_uvd_ring_end_use,
 	.emit_wreg = uvd_v7_0_ring_emit_wreg,
 	.emit_reg_wait = uvd_v7_0_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
 };
 
 static const struct amdgpu_ring_funcs uvd_v7_0_enc_ring_vm_funcs = {
@@ -1702,22 +1769,32 @@ static const struct amdgpu_ring_funcs uvd_v7_0_enc_ring_vm_funcs = {
 	.end_use = amdgpu_uvd_ring_end_use,
 	.emit_wreg = uvd_v7_0_enc_ring_emit_wreg,
 	.emit_reg_wait = uvd_v7_0_enc_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
 };
 
 static void uvd_v7_0_set_ring_funcs(struct amdgpu_device *adev)
 {
-	adev->uvd.ring.funcs = &uvd_v7_0_ring_vm_funcs;
-	DRM_INFO("UVD is enabled in VM mode\n");
+	int i;
+
+	for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
+		adev->uvd.inst[i].ring.funcs = &uvd_v7_0_ring_vm_funcs;
+		adev->uvd.inst[i].ring.me = i;
+		DRM_INFO("UVD(%d) is enabled in VM mode\n", i);
+	}
 }
 
 static void uvd_v7_0_set_enc_ring_funcs(struct amdgpu_device *adev)
 {
-	int i;
+	int i, j;
 
-	for (i = 0; i < adev->uvd.num_enc_rings; ++i)
-		adev->uvd.ring_enc[i].funcs = &uvd_v7_0_enc_ring_vm_funcs;
+	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
+		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
+			adev->uvd.inst[j].ring_enc[i].funcs = &uvd_v7_0_enc_ring_vm_funcs;
+			adev->uvd.inst[j].ring_enc[i].me = j;
+		}
 
-	DRM_INFO("UVD ENC is enabled in VM mode\n");
+		DRM_INFO("UVD(%d) ENC is enabled in VM mode\n", j);
+	}
 }
 
 static const struct amdgpu_irq_src_funcs uvd_v7_0_irq_funcs = {
@@ -1727,8 +1804,12 @@ static const struct amdgpu_irq_src_funcs uvd_v7_0_irq_funcs = {
 
 static void uvd_v7_0_set_irq_funcs(struct amdgpu_device *adev)
 {
-	adev->uvd.irq.num_types = adev->uvd.num_enc_rings + 1;
-	adev->uvd.irq.funcs = &uvd_v7_0_irq_funcs;
+	int i;
+
+	for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
+		adev->uvd.inst[i].irq.num_types = adev->uvd.num_enc_rings + 1;
+		adev->uvd.inst[i].irq.funcs = &uvd_v7_0_irq_funcs;
+	}
 }
 
 const struct amdgpu_ip_block_version uvd_v7_0_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index 428d1928e44e..0999c843f623 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -388,7 +388,8 @@ static unsigned vce_v3_0_get_harvest_config(struct amdgpu_device *adev)
 	default:
 		if ((adev->asic_type == CHIP_POLARIS10) ||
 		    (adev->asic_type == CHIP_POLARIS11) ||
-		    (adev->asic_type == CHIP_POLARIS12))
+		    (adev->asic_type == CHIP_POLARIS12) ||
+		    (adev->asic_type == CHIP_VEGAM))
 			return AMDGPU_VCE_HARVEST_VCE1;
 
 		return 0;
@@ -467,8 +468,8 @@ static int vce_v3_0_hw_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	vce_v3_0_override_vce_clock_gating(adev, true);
-	if (!(adev->flags & AMD_IS_APU))
-		amdgpu_asic_set_vce_clocks(adev, 10000, 10000);
+
+	amdgpu_asic_set_vce_clocks(adev, 10000, 10000);
 
 	for (i = 0; i < adev->vce.num_rings; i++)
 		adev->vce.ring[i].ready = false;
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index 73fd48d6c756..8fd1b742985a 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -1081,6 +1081,7 @@ static const struct amdgpu_ring_funcs vce_v4_0_ring_vm_funcs = {
 	.end_use = amdgpu_vce_ring_end_use,
 	.emit_wreg = vce_v4_0_emit_wreg,
 	.emit_reg_wait = vce_v4_0_emit_reg_wait,
+	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
 };
 
 static void vce_v4_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
index 8c132673bc79..110b294ebed3 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
@@ -35,7 +35,6 @@
 #include "mmhub/mmhub_9_1_offset.h"
 #include "mmhub/mmhub_9_1_sh_mask.h"
 
-static int vcn_v1_0_start(struct amdgpu_device *adev);
 static int vcn_v1_0_stop(struct amdgpu_device *adev);
 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev);
 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev);
@@ -146,10 +145,6 @@ static int vcn_v1_0_hw_init(void *handle)
 	struct amdgpu_ring *ring = &adev->vcn.ring_dec;
 	int i, r;
 
-	r = vcn_v1_0_start(adev);
-	if (r)
-		goto done;
-
 	ring->ready = true;
 	r = amdgpu_ring_test_ring(ring);
 	if (r) {
@@ -185,11 +180,9 @@ static int vcn_v1_0_hw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	struct amdgpu_ring *ring = &adev->vcn.ring_dec;
-	int r;
 
-	r = vcn_v1_0_stop(adev);
-	if (r)
-		return r;
+	if (RREG32_SOC15(VCN, 0, mmUVD_STATUS))
+		vcn_v1_0_stop(adev);
 
 	ring->ready = false;
 
@@ -288,14 +281,14 @@ static void vcn_v1_0_mc_resume(struct amdgpu_device *adev)
  *
  * Disable clock gating for VCN block
  */
-static void vcn_v1_0_disable_clock_gating(struct amdgpu_device *adev, bool sw)
+static void vcn_v1_0_disable_clock_gating(struct amdgpu_device *adev)
 {
 	uint32_t data;
 
 	/* JPEG disable CGC */
 	data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
 
-	if (sw)
+	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 		data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 	else
 		data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE_MASK;
@@ -310,7 +303,7 @@ static void vcn_v1_0_disable_clock_gating(struct amdgpu_device *adev, bool sw)
 
 	/* UVD disable CGC */
 	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
-	if (sw)
+	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 	else
 		data &= ~ UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
@@ -415,13 +408,13 @@ static void vcn_v1_0_disable_clock_gating(struct amdgpu_device *adev, bool sw)
  *
  * Enable clock gating for VCN block
  */
-static void vcn_v1_0_enable_clock_gating(struct amdgpu_device *adev, bool sw)
+static void vcn_v1_0_enable_clock_gating(struct amdgpu_device *adev)
 {
 	uint32_t data = 0;
 
 	/* enable JPEG CGC */
 	data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
-	if (sw)
+	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 		data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 	else
 		data |= 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
@@ -435,7 +428,7 @@ static void vcn_v1_0_enable_clock_gating(struct amdgpu_device *adev, bool sw)
 
 	/* enable UVD CGC */
 	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
-	if (sw)
+	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 	else
 		data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
@@ -480,6 +473,94 @@ static void vcn_v1_0_enable_clock_gating(struct amdgpu_device *adev, bool sw)
 	WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 }
 
+static void vcn_1_0_disable_static_power_gating(struct amdgpu_device *adev)
+{
+	uint32_t data = 0;
+	int ret;
+
+	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
+		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
+			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
+
+		WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
+		SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON, 0xFFFFFF, ret);
+	} else {
+		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
+			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
+			| 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
+			| 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
+			| 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
+			| 1 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
+			| 1 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
+			| 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
+			| 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
+			| 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
+			| 1 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
+		WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
+		SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, 0,  0xFFFFFFFF, ret);
+	}
+
+	/* polling UVD_PGFSM_STATUS to confirm UVDM_PWR_STATUS , UVDU_PWR_STATUS are 0 (power on) */
+
+	data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
+	data &= ~0x103;
+	if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
+		data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON | UVD_POWER_STATUS__UVD_PG_EN_MASK;
+
+	WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
+}
+
+static void vcn_1_0_enable_static_power_gating(struct amdgpu_device *adev)
+{
+	uint32_t data = 0;
+	int ret;
+
+	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
+		/* Before power off, this indicator has to be turned on */
+		data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
+		data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
+		data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
+		WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
+
+
+		data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
+			| 2 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
+
+		WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
+
+		data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
+			| 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
+			| 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
+			| 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
+			| 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
+			| 2 << UVD_PGFSM_STATUS__UVDIL_PWR_STATUS__SHIFT
+			| 2 << UVD_PGFSM_STATUS__UVDIR_PWR_STATUS__SHIFT
+			| 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
+			| 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
+			| 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
+			| 2 << UVD_PGFSM_STATUS__UVDW_PWR_STATUS__SHIFT);
+		SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, data, 0xFFFFFFFF, ret);
+	}
+}
+
 /**
  * vcn_v1_0_start - start VCN block
  *
@@ -499,8 +580,9 @@ static int vcn_v1_0_start(struct amdgpu_device *adev)
 
 	vcn_v1_0_mc_resume(adev);
 
+	vcn_1_0_disable_static_power_gating(adev);
 	/* disable clock gating */
-	vcn_v1_0_disable_clock_gating(adev, true);
+	vcn_v1_0_disable_clock_gating(adev);
 
 	/* disable interupt */
 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
@@ -680,16 +762,45 @@ static int vcn_v1_0_stop(struct amdgpu_device *adev)
 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
 			~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 
-	/* enable clock gating */
-	vcn_v1_0_enable_clock_gating(adev, true);
+	WREG32_SOC15(VCN, 0, mmUVD_STATUS, 0);
 
+	vcn_v1_0_enable_clock_gating(adev);
+	vcn_1_0_enable_static_power_gating(adev);
 	return 0;
 }
 
+bool vcn_v1_0_is_idle(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return (RREG32_SOC15(VCN, 0, mmUVD_STATUS) == 0x2);
+}
+
+int vcn_v1_0_wait_for_idle(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int ret = 0;
+
+	SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, 0x2, 0x2, ret);
+
+	return ret;
+}
+
 static int vcn_v1_0_set_clockgating_state(void *handle,
 					  enum amd_clockgating_state state)
 {
-	/* needed for driver unload*/
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
+
+	if (enable) {
+		/* wait for STATUS to clear */
+		if (vcn_v1_0_is_idle(handle))
+			return -EBUSY;
+		vcn_v1_0_enable_clock_gating(adev);
+	} else {
+		/* disable HW gating and enable Sw gating */
+		vcn_v1_0_disable_clock_gating(adev);
+	}
 	return 0;
 }
 
@@ -1048,16 +1159,36 @@ static int vcn_v1_0_process_interrupt(struct amdgpu_device *adev,
 	return 0;
 }
 
-static void vcn_v1_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+static void vcn_v1_0_dec_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
-	int i;
 	struct amdgpu_device *adev = ring->adev;
+	int i;
 
-	for (i = 0; i < count; i++)
-		amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0));
+	WARN_ON(ring->wptr % 2 || count % 2);
 
+	for (i = 0; i < count / 2; i++) {
+		amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0));
+		amdgpu_ring_write(ring, 0);
+	}
 }
 
+static int vcn_v1_0_set_powergating_state(void *handle,
+					  enum amd_powergating_state state)
+{
+	/* This doesn't actually powergate the VCN block.
+	 * That's done in the dpm code via the SMC.  This
+	 * just re-inits the block as necessary.  The actual
+	 * gating still happens in the dpm code.  We should
+	 * revisit this when there is a cleaner line between
+	 * the smc and the hw blocks
+	 */
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (state == AMD_PG_STATE_GATE)
+		return vcn_v1_0_stop(adev);
+	else
+		return vcn_v1_0_start(adev);
+}
 
 static const struct amd_ip_funcs vcn_v1_0_ip_funcs = {
 	.name = "vcn_v1_0",
@@ -1069,20 +1200,19 @@ static const struct amd_ip_funcs vcn_v1_0_ip_funcs = {
 	.hw_fini = vcn_v1_0_hw_fini,
 	.suspend = vcn_v1_0_suspend,
 	.resume = vcn_v1_0_resume,
-	.is_idle = NULL /* vcn_v1_0_is_idle */,
-	.wait_for_idle = NULL /* vcn_v1_0_wait_for_idle */,
+	.is_idle = vcn_v1_0_is_idle,
+	.wait_for_idle = vcn_v1_0_wait_for_idle,
 	.check_soft_reset = NULL /* vcn_v1_0_check_soft_reset */,
 	.pre_soft_reset = NULL /* vcn_v1_0_pre_soft_reset */,
 	.soft_reset = NULL /* vcn_v1_0_soft_reset */,
 	.post_soft_reset = NULL /* vcn_v1_0_post_soft_reset */,
 	.set_clockgating_state = vcn_v1_0_set_clockgating_state,
-	.set_powergating_state = NULL /* vcn_v1_0_set_powergating_state */,
+	.set_powergating_state = vcn_v1_0_set_powergating_state,
 };
 
 static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_DEC,
 	.align_mask = 0xf,
-	.nop = PACKET0(0x81ff, 0),
 	.support_64bit_ptrs = false,
 	.vmhub = AMDGPU_MMHUB,
 	.get_rptr = vcn_v1_0_dec_ring_get_rptr,
@@ -1101,7 +1231,7 @@ static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
 	.emit_vm_flush = vcn_v1_0_dec_ring_emit_vm_flush,
 	.test_ring = amdgpu_vcn_dec_ring_test_ring,
 	.test_ib = amdgpu_vcn_dec_ring_test_ib,
-	.insert_nop = vcn_v1_0_ring_insert_nop,
+	.insert_nop = vcn_v1_0_dec_ring_insert_nop,
 	.insert_start = vcn_v1_0_dec_ring_insert_start,
 	.insert_end = vcn_v1_0_dec_ring_insert_end,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
@@ -1109,6 +1239,7 @@ static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
 	.end_use = amdgpu_vcn_ring_end_use,
 	.emit_wreg = vcn_v1_0_dec_ring_emit_wreg,
 	.emit_reg_wait = vcn_v1_0_dec_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
 };
 
 static const struct amdgpu_ring_funcs vcn_v1_0_enc_ring_vm_funcs = {
@@ -1139,6 +1270,7 @@ static const struct amdgpu_ring_funcs vcn_v1_0_enc_ring_vm_funcs = {
 	.end_use = amdgpu_vcn_ring_end_use,
 	.emit_wreg = vcn_v1_0_enc_ring_emit_wreg,
 	.emit_reg_wait = vcn_v1_0_enc_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
 };
 
 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c b/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c
new file mode 100644
index 000000000000..52778de93ab0
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "soc15.h"
+
+#include "soc15_common.h"
+#include "soc15_hw_ip.h"
+#include "vega20_ip_offset.h"
+
+int vega20_reg_base_init(struct amdgpu_device *adev)
+{
+	/* HW has more IP blocks,  only initialized the blocke beend by our driver  */
+	uint32_t i;
+	for (i = 0 ; i < MAX_INSTANCE ; ++i) {
+		adev->reg_offset[GC_HWIP][i] = (uint32_t *)(&(GC_BASE.instance[i]));
+		adev->reg_offset[HDP_HWIP][i] = (uint32_t *)(&(HDP_BASE.instance[i]));
+		adev->reg_offset[MMHUB_HWIP][i] = (uint32_t *)(&(MMHUB_BASE.instance[i]));
+		adev->reg_offset[ATHUB_HWIP][i] = (uint32_t *)(&(ATHUB_BASE.instance[i]));
+		adev->reg_offset[NBIO_HWIP][i] = (uint32_t *)(&(NBIO_BASE.instance[i]));
+		adev->reg_offset[MP0_HWIP][i] = (uint32_t *)(&(MP0_BASE.instance[i]));
+		adev->reg_offset[UVD_HWIP][i] = (uint32_t *)(&(UVD_BASE.instance[i]));
+		adev->reg_offset[VCE_HWIP][i] = (uint32_t *)(&(VCE_BASE.instance[i]));
+		adev->reg_offset[DF_HWIP][i] = (uint32_t *)(&(DF_BASE.instance[i]));
+		adev->reg_offset[DCE_HWIP][i] = (uint32_t *)(&(DCE_BASE.instance[i]));
+		adev->reg_offset[OSSSYS_HWIP][i] = (uint32_t *)(&(OSSSYS_BASE.instance[i]));
+		adev->reg_offset[SDMA0_HWIP][i] = (uint32_t *)(&(SDMA0_BASE.instance[i]));
+		adev->reg_offset[SDMA1_HWIP][i] = (uint32_t *)(&(SDMA1_BASE.instance[i]));
+		adev->reg_offset[SMUIO_HWIP][i] = (uint32_t *)(&(SMUIO_BASE.instance[i]));
+	}
+	return 0;
+}
+
+
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 126f1276d347..4ac1288ab7df 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -305,9 +305,10 @@ static void vi_init_golden_registers(struct amdgpu_device *adev)
 							stoney_mgcg_cgcg_init,
 							ARRAY_SIZE(stoney_mgcg_cgcg_init));
 		break;
-	case CHIP_POLARIS11:
 	case CHIP_POLARIS10:
+	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 	default:
 		break;
 	}
@@ -728,33 +729,59 @@ static int vi_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
 		return r;
 
 	tmp = RREG32_SMC(cntl_reg);
-	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
-		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
+
+	if (adev->flags & AMD_IS_APU)
+		tmp &= ~CG_DCLK_CNTL__DCLK_DIVIDER_MASK;
+	else
+		tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
+				CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
 	tmp |= dividers.post_divider;
 	WREG32_SMC(cntl_reg, tmp);
 
 	for (i = 0; i < 100; i++) {
-		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
-			break;
+		tmp = RREG32_SMC(status_reg);
+		if (adev->flags & AMD_IS_APU) {
+			if (tmp & 0x10000)
+				break;
+		} else {
+			if (tmp & CG_DCLK_STATUS__DCLK_STATUS_MASK)
+				break;
+		}
 		mdelay(10);
 	}
 	if (i == 100)
 		return -ETIMEDOUT;
-
 	return 0;
 }
 
+#define ixGNB_CLK1_DFS_CNTL 0xD82200F0
+#define ixGNB_CLK1_STATUS   0xD822010C
+#define ixGNB_CLK2_DFS_CNTL 0xD8220110
+#define ixGNB_CLK2_STATUS   0xD822012C
+#define ixGNB_CLK3_DFS_CNTL 0xD8220130
+#define ixGNB_CLK3_STATUS   0xD822014C
+
 static int vi_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
 {
 	int r;
 
-	r = vi_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
-	if (r)
-		return r;
+	if (adev->flags & AMD_IS_APU) {
+		r = vi_set_uvd_clock(adev, vclk, ixGNB_CLK2_DFS_CNTL, ixGNB_CLK2_STATUS);
+		if (r)
+			return r;
 
-	r = vi_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
-	if (r)
-		return r;
+		r = vi_set_uvd_clock(adev, dclk, ixGNB_CLK1_DFS_CNTL, ixGNB_CLK1_STATUS);
+		if (r)
+			return r;
+	} else {
+		r = vi_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
+		if (r)
+			return r;
+
+		r = vi_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
+		if (r)
+			return r;
+	}
 
 	return 0;
 }
@@ -764,6 +791,22 @@ static int vi_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
 	int r, i;
 	struct atom_clock_dividers dividers;
 	u32 tmp;
+	u32 reg_ctrl;
+	u32 reg_status;
+	u32 status_mask;
+	u32 reg_mask;
+
+	if (adev->flags & AMD_IS_APU) {
+		reg_ctrl = ixGNB_CLK3_DFS_CNTL;
+		reg_status = ixGNB_CLK3_STATUS;
+		status_mask = 0x00010000;
+		reg_mask = CG_ECLK_CNTL__ECLK_DIVIDER_MASK;
+	} else {
+		reg_ctrl = ixCG_ECLK_CNTL;
+		reg_status = ixCG_ECLK_STATUS;
+		status_mask = CG_ECLK_STATUS__ECLK_STATUS_MASK;
+		reg_mask = CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK | CG_ECLK_CNTL__ECLK_DIVIDER_MASK;
+	}
 
 	r = amdgpu_atombios_get_clock_dividers(adev,
 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
@@ -772,24 +815,25 @@ static int vi_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
 		return r;
 
 	for (i = 0; i < 100; i++) {
-		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
+		if (RREG32_SMC(reg_status) & status_mask)
 			break;
 		mdelay(10);
 	}
+
 	if (i == 100)
 		return -ETIMEDOUT;
 
-	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
-	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
-		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
+	tmp = RREG32_SMC(reg_ctrl);
+	tmp &= ~reg_mask;
 	tmp |= dividers.post_divider;
-	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
+	WREG32_SMC(reg_ctrl, tmp);
 
 	for (i = 0; i < 100; i++) {
-		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
+		if (RREG32_SMC(reg_status) & status_mask)
 			break;
 		mdelay(10);
 	}
+
 	if (i == 100)
 		return -ETIMEDOUT;
 
@@ -876,6 +920,27 @@ static void vi_invalidate_hdp(struct amdgpu_device *adev,
 	}
 }
 
+static bool vi_need_full_reset(struct amdgpu_device *adev)
+{
+	switch (adev->asic_type) {
+	case CHIP_CARRIZO:
+	case CHIP_STONEY:
+		/* CZ has hang issues with full reset at the moment */
+		return false;
+	case CHIP_FIJI:
+	case CHIP_TONGA:
+		/* XXX: soft reset should work on fiji and tonga */
+		return true;
+	case CHIP_POLARIS10:
+	case CHIP_POLARIS11:
+	case CHIP_POLARIS12:
+	case CHIP_TOPAZ:
+	default:
+		/* change this when we support soft reset */
+		return true;
+	}
+}
+
 static const struct amdgpu_asic_funcs vi_asic_funcs =
 {
 	.read_disabled_bios = &vi_read_disabled_bios,
@@ -889,6 +954,7 @@ static const struct amdgpu_asic_funcs vi_asic_funcs =
 	.get_config_memsize = &vi_get_config_memsize,
 	.flush_hdp = &vi_flush_hdp,
 	.invalidate_hdp = &vi_invalidate_hdp,
+	.need_full_reset = &vi_need_full_reset,
 };
 
 #define CZ_REV_BRISTOL(rev)	 \
@@ -1031,6 +1097,30 @@ static int vi_common_early_init(void *handle)
 		adev->pg_flags = 0;
 		adev->external_rev_id = adev->rev_id + 0x64;
 		break;
+	case CHIP_VEGAM:
+		adev->cg_flags = 0;
+			/*AMD_CG_SUPPORT_GFX_MGCG |
+			AMD_CG_SUPPORT_GFX_RLC_LS |
+			AMD_CG_SUPPORT_GFX_CP_LS |
+			AMD_CG_SUPPORT_GFX_CGCG |
+			AMD_CG_SUPPORT_GFX_CGLS |
+			AMD_CG_SUPPORT_GFX_3D_CGCG |
+			AMD_CG_SUPPORT_GFX_3D_CGLS |
+			AMD_CG_SUPPORT_SDMA_MGCG |
+			AMD_CG_SUPPORT_SDMA_LS |
+			AMD_CG_SUPPORT_BIF_MGCG |
+			AMD_CG_SUPPORT_BIF_LS |
+			AMD_CG_SUPPORT_HDP_MGCG |
+			AMD_CG_SUPPORT_HDP_LS |
+			AMD_CG_SUPPORT_ROM_MGCG |
+			AMD_CG_SUPPORT_MC_MGCG |
+			AMD_CG_SUPPORT_MC_LS |
+			AMD_CG_SUPPORT_DRM_LS |
+			AMD_CG_SUPPORT_UVD_MGCG |
+			AMD_CG_SUPPORT_VCE_MGCG;*/
+		adev->pg_flags = 0;
+		adev->external_rev_id = adev->rev_id + 0x6E;
+		break;
 	case CHIP_CARRIZO:
 		adev->cg_flags = AMD_CG_SUPPORT_UVD_MGCG |
 			AMD_CG_SUPPORT_GFX_MGCG |
@@ -1422,6 +1512,7 @@ static int vi_common_set_clockgating_state(void *handle,
 	case CHIP_POLARIS10:
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 		vi_common_set_clockgating_state_by_smu(adev, state);
 	default:
 		break;
@@ -1551,9 +1642,10 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
 			amdgpu_device_ip_block_add(adev, &vce_v3_0_ip_block);
 		}
 		break;
-	case CHIP_POLARIS11:
 	case CHIP_POLARIS10:
+	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
 		amdgpu_device_ip_block_add(adev, &gmc_v8_1_ip_block);
 		amdgpu_device_ip_block_add(adev, &tonga_ih_ip_block);
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
index 0d0242240c47..ffd096fffc1c 100644
--- a/drivers/gpu/drm/amd/amdkfd/Makefile
+++ b/drivers/gpu/drm/amd/amdkfd/Makefile
@@ -30,12 +30,14 @@ amdkfd-y	:= kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \
 		kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \
 		kfd_process.o kfd_queue.o kfd_mqd_manager.o \
 		kfd_mqd_manager_cik.o kfd_mqd_manager_vi.o \
+		kfd_mqd_manager_v9.o \
 		kfd_kernel_queue.o kfd_kernel_queue_cik.o \
-		kfd_kernel_queue_vi.o kfd_packet_manager.o \
-		kfd_process_queue_manager.o kfd_device_queue_manager.o \
-		kfd_device_queue_manager_cik.o kfd_device_queue_manager_vi.o \
+		kfd_kernel_queue_vi.o kfd_kernel_queue_v9.o \
+		kfd_packet_manager.o kfd_process_queue_manager.o \
+		kfd_device_queue_manager.o kfd_device_queue_manager_cik.o \
+		kfd_device_queue_manager_vi.o kfd_device_queue_manager_v9.o \
 		kfd_interrupt.o kfd_events.o cik_event_interrupt.o \
-		kfd_dbgdev.o kfd_dbgmgr.o kfd_crat.o
+		kfd_int_process_v9.o kfd_dbgdev.o kfd_dbgmgr.o kfd_crat.o
 
 ifneq ($(CONFIG_AMD_IOMMU_V2),)
 amdkfd-y += kfd_iommu.o
diff --git a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
index 3d5ccb3755d4..49df6c791cfc 100644
--- a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
+++ b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
@@ -27,18 +27,28 @@
 static bool cik_event_interrupt_isr(struct kfd_dev *dev,
 					const uint32_t *ih_ring_entry)
 {
-	unsigned int pasid;
 	const struct cik_ih_ring_entry *ihre =
 			(const struct cik_ih_ring_entry *)ih_ring_entry;
+	unsigned int vmid, pasid;
+
+	/* Only handle interrupts from KFD VMIDs */
+	vmid  = (ihre->ring_id & 0x0000ff00) >> 8;
+	if (vmid < dev->vm_info.first_vmid_kfd ||
+	    vmid > dev->vm_info.last_vmid_kfd)
+		return 0;
 
+	/* If there is no valid PASID, it's likely a firmware bug */
 	pasid = (ihre->ring_id & 0xffff0000) >> 16;
+	if (WARN_ONCE(pasid == 0, "FW bug: No PASID in KFD interrupt"))
+		return 0;
 
-	/* Do not process in ISR, just request it to be forwarded to WQ. */
-	return (pasid != 0) &&
-		(ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE ||
+	/* Interrupt types we care about: various signals and faults.
+	 * They will be forwarded to a work queue (see below).
+	 */
+	return ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE ||
 		ihre->source_id == CIK_INTSRC_SDMA_TRAP ||
 		ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG ||
-		ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE);
+		ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE;
 }
 
 static void cik_event_interrupt_wq(struct kfd_dev *dev,
diff --git a/drivers/gpu/drm/amd/amdkfd/cik_regs.h b/drivers/gpu/drm/amd/amdkfd/cik_regs.h
index 48769d12dd7b..37ce6dd65391 100644
--- a/drivers/gpu/drm/amd/amdkfd/cik_regs.h
+++ b/drivers/gpu/drm/amd/amdkfd/cik_regs.h
@@ -33,7 +33,8 @@
 #define	APE1_MTYPE(x)					((x) << 7)
 
 /* valid for both DEFAULT_MTYPE and APE1_MTYPE */
-#define	MTYPE_CACHED					0
+#define	MTYPE_CACHED_NV					0
+#define	MTYPE_CACHED					1
 #define	MTYPE_NONCACHED					3
 
 #define	DEFAULT_CP_HQD_PERSISTENT_STATE			(0x33U << 8)
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
new file mode 100644
index 000000000000..f68aef02fc1f
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
@@ -0,0 +1,560 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+static const uint32_t cwsr_trap_gfx8_hex[] = {
+	0xbf820001, 0xbf820125,
+	0xb8f4f802, 0x89748674,
+	0xb8f5f803, 0x8675ff75,
+	0x00000400, 0xbf850011,
+	0xc00a1e37, 0x00000000,
+	0xbf8c007f, 0x87777978,
+	0xbf840002, 0xb974f802,
+	0xbe801d78, 0xb8f5f803,
+	0x8675ff75, 0x000001ff,
+	0xbf850002, 0x80708470,
+	0x82718071, 0x8671ff71,
+	0x0000ffff, 0xb974f802,
+	0xbe801f70, 0xb8f5f803,
+	0x8675ff75, 0x00000100,
+	0xbf840006, 0xbefa0080,
+	0xb97a0203, 0x8671ff71,
+	0x0000ffff, 0x80f08870,
+	0x82f18071, 0xbefa0080,
+	0xb97a0283, 0xbef60068,
+	0xbef70069, 0xb8fa1c07,
+	0x8e7a9c7a, 0x87717a71,
+	0xb8fa03c7, 0x8e7a9b7a,
+	0x87717a71, 0xb8faf807,
+	0x867aff7a, 0x00007fff,
+	0xb97af807, 0xbef2007e,
+	0xbef3007f, 0xbefe0180,
+	0xbf900004, 0x877a8474,
+	0xb97af802, 0xbf8e0002,
+	0xbf88fffe, 0xbef8007e,
+	0x8679ff7f, 0x0000ffff,
+	0x8779ff79, 0x00040000,
+	0xbefa0080, 0xbefb00ff,
+	0x00807fac, 0x867aff7f,
+	0x08000000, 0x8f7a837a,
+	0x877b7a7b, 0x867aff7f,
+	0x70000000, 0x8f7a817a,
+	0x877b7a7b, 0xbeef007c,
+	0xbeee0080, 0xb8ee2a05,
+	0x806e816e, 0x8e6e8a6e,
+	0xb8fa1605, 0x807a817a,
+	0x8e7a867a, 0x806e7a6e,
+	0xbefa0084, 0xbefa00ff,
+	0x01000000, 0xbefe007c,
+	0xbefc006e, 0xc0611bfc,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611c3c,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611c7c,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611cbc,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611cfc,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611d3c,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xb8f5f803,
+	0xbefe007c, 0xbefc006e,
+	0xc0611d7c, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0611dbc, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0611dfc, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xb8eff801, 0xbefe007c,
+	0xbefc006e, 0xc0611bfc,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611b3c,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611b7c,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0x867aff7f,
+	0x04000000, 0xbef30080,
+	0x8773737a, 0xb8ee2a05,
+	0x806e816e, 0x8e6e8a6e,
+	0xb8f51605, 0x80758175,
+	0x8e758475, 0x8e7a8275,
+	0xbefa00ff, 0x01000000,
+	0xbef60178, 0x80786e78,
+	0x82798079, 0xbefc0080,
+	0xbe802b00, 0xbe822b02,
+	0xbe842b04, 0xbe862b06,
+	0xbe882b08, 0xbe8a2b0a,
+	0xbe8c2b0c, 0xbe8e2b0e,
+	0xc06b003c, 0x00000000,
+	0xc06b013c, 0x00000010,
+	0xc06b023c, 0x00000020,
+	0xc06b033c, 0x00000030,
+	0x8078c078, 0x82798079,
+	0x807c907c, 0xbf0a757c,
+	0xbf85ffeb, 0xbef80176,
+	0xbeee0080, 0xbefe00c1,
+	0xbeff00c1, 0xbefa00ff,
+	0x01000000, 0xe0724000,
+	0x6e1e0000, 0xe0724100,
+	0x6e1e0100, 0xe0724200,
+	0x6e1e0200, 0xe0724300,
+	0x6e1e0300, 0xbefe00c1,
+	0xbeff00c1, 0xb8f54306,
+	0x8675c175, 0xbf84002c,
+	0xbf8a0000, 0x867aff73,
+	0x04000000, 0xbf840028,
+	0x8e758675, 0x8e758275,
+	0xbefa0075, 0xb8ee2a05,
+	0x806e816e, 0x8e6e8a6e,
+	0xb8fa1605, 0x807a817a,
+	0x8e7a867a, 0x806e7a6e,
+	0x806eff6e, 0x00000080,
+	0xbefa00ff, 0x01000000,
+	0xbefc0080, 0xd28c0002,
+	0x000100c1, 0xd28d0003,
+	0x000204c1, 0xd1060002,
+	0x00011103, 0x7e0602ff,
+	0x00000200, 0xbefc00ff,
+	0x00010000, 0xbe80007b,
+	0x867bff7b, 0xff7fffff,
+	0x877bff7b, 0x00058000,
+	0xd8ec0000, 0x00000002,
+	0xbf8c007f, 0xe0765000,
+	0x6e1e0002, 0x32040702,
+	0xd0c9006a, 0x0000eb02,
+	0xbf87fff7, 0xbefb0000,
+	0xbeee00ff, 0x00000400,
+	0xbefe00c1, 0xbeff00c1,
+	0xb8f52a05, 0x80758175,
+	0x8e758275, 0x8e7a8875,
+	0xbefa00ff, 0x01000000,
+	0xbefc0084, 0xbf0a757c,
+	0xbf840015, 0xbf11017c,
+	0x8075ff75, 0x00001000,
+	0x7e000300, 0x7e020301,
+	0x7e040302, 0x7e060303,
+	0xe0724000, 0x6e1e0000,
+	0xe0724100, 0x6e1e0100,
+	0xe0724200, 0x6e1e0200,
+	0xe0724300, 0x6e1e0300,
+	0x807c847c, 0x806eff6e,
+	0x00000400, 0xbf0a757c,
+	0xbf85ffef, 0xbf9c0000,
+	0xbf8200ca, 0xbef8007e,
+	0x8679ff7f, 0x0000ffff,
+	0x8779ff79, 0x00040000,
+	0xbefa0080, 0xbefb00ff,
+	0x00807fac, 0x8676ff7f,
+	0x08000000, 0x8f768376,
+	0x877b767b, 0x8676ff7f,
+	0x70000000, 0x8f768176,
+	0x877b767b, 0x8676ff7f,
+	0x04000000, 0xbf84001e,
+	0xbefe00c1, 0xbeff00c1,
+	0xb8f34306, 0x8673c173,
+	0xbf840019, 0x8e738673,
+	0x8e738273, 0xbefa0073,
+	0xb8f22a05, 0x80728172,
+	0x8e728a72, 0xb8f61605,
+	0x80768176, 0x8e768676,
+	0x80727672, 0x8072ff72,
+	0x00000080, 0xbefa00ff,
+	0x01000000, 0xbefc0080,
+	0xe0510000, 0x721e0000,
+	0xe0510100, 0x721e0000,
+	0x807cff7c, 0x00000200,
+	0x8072ff72, 0x00000200,
+	0xbf0a737c, 0xbf85fff6,
+	0xbef20080, 0xbefe00c1,
+	0xbeff00c1, 0xb8f32a05,
+	0x80738173, 0x8e738273,
+	0x8e7a8873, 0xbefa00ff,
+	0x01000000, 0xbef60072,
+	0x8072ff72, 0x00000400,
+	0xbefc0084, 0xbf11087c,
+	0x8073ff73, 0x00008000,
+	0xe0524000, 0x721e0000,
+	0xe0524100, 0x721e0100,
+	0xe0524200, 0x721e0200,
+	0xe0524300, 0x721e0300,
+	0xbf8c0f70, 0x7e000300,
+	0x7e020301, 0x7e040302,
+	0x7e060303, 0x807c847c,
+	0x8072ff72, 0x00000400,
+	0xbf0a737c, 0xbf85ffee,
+	0xbf9c0000, 0xe0524000,
+	0x761e0000, 0xe0524100,
+	0x761e0100, 0xe0524200,
+	0x761e0200, 0xe0524300,
+	0x761e0300, 0xb8f22a05,
+	0x80728172, 0x8e728a72,
+	0xb8f61605, 0x80768176,
+	0x8e768676, 0x80727672,
+	0x80f2c072, 0xb8f31605,
+	0x80738173, 0x8e738473,
+	0x8e7a8273, 0xbefa00ff,
+	0x01000000, 0xbefc0073,
+	0xc031003c, 0x00000072,
+	0x80f2c072, 0xbf8c007f,
+	0x80fc907c, 0xbe802d00,
+	0xbe822d02, 0xbe842d04,
+	0xbe862d06, 0xbe882d08,
+	0xbe8a2d0a, 0xbe8c2d0c,
+	0xbe8e2d0e, 0xbf06807c,
+	0xbf84fff1, 0xb8f22a05,
+	0x80728172, 0x8e728a72,
+	0xb8f61605, 0x80768176,
+	0x8e768676, 0x80727672,
+	0xbefa0084, 0xbefa00ff,
+	0x01000000, 0xc0211cfc,
+	0x00000072, 0x80728472,
+	0xc0211c3c, 0x00000072,
+	0x80728472, 0xc0211c7c,
+	0x00000072, 0x80728472,
+	0xc0211bbc, 0x00000072,
+	0x80728472, 0xc0211bfc,
+	0x00000072, 0x80728472,
+	0xc0211d3c, 0x00000072,
+	0x80728472, 0xc0211d7c,
+	0x00000072, 0x80728472,
+	0xc0211a3c, 0x00000072,
+	0x80728472, 0xc0211a7c,
+	0x00000072, 0x80728472,
+	0xc0211dfc, 0x00000072,
+	0x80728472, 0xc0211b3c,
+	0x00000072, 0x80728472,
+	0xc0211b7c, 0x00000072,
+	0x80728472, 0xbf8c007f,
+	0xbefc0073, 0xbefe006e,
+	0xbeff006f, 0x867375ff,
+	0x000003ff, 0xb9734803,
+	0x867375ff, 0xfffff800,
+	0x8f738b73, 0xb973a2c3,
+	0xb977f801, 0x8673ff71,
+	0xf0000000, 0x8f739c73,
+	0x8e739073, 0xbef60080,
+	0x87767376, 0x8673ff71,
+	0x08000000, 0x8f739b73,
+	0x8e738f73, 0x87767376,
+	0x8673ff74, 0x00800000,
+	0x8f739773, 0xb976f807,
+	0x8671ff71, 0x0000ffff,
+	0x86fe7e7e, 0x86ea6a6a,
+	0xb974f802, 0xbf8a0000,
+	0x95807370, 0xbf810000,
+};
+
+
+static const uint32_t cwsr_trap_gfx9_hex[] = {
+	0xbf820001, 0xbf82015a,
+	0xb8f8f802, 0x89788678,
+	0xb8f1f803, 0x866eff71,
+	0x00000400, 0xbf850034,
+	0x866eff71, 0x00000800,
+	0xbf850003, 0x866eff71,
+	0x00000100, 0xbf840008,
+	0x866eff78, 0x00002000,
+	0xbf840001, 0xbf810000,
+	0x8778ff78, 0x00002000,
+	0x80ec886c, 0x82ed806d,
+	0xb8eef807, 0x866fff6e,
+	0x001f8000, 0x8e6f8b6f,
+	0x8977ff77, 0xfc000000,
+	0x87776f77, 0x896eff6e,
+	0x001f8000, 0xb96ef807,
+	0xb8f0f812, 0xb8f1f813,
+	0x8ef08870, 0xc0071bb8,
+	0x00000000, 0xbf8cc07f,
+	0xc0071c38, 0x00000008,
+	0xbf8cc07f, 0x86ee6e6e,
+	0xbf840001, 0xbe801d6e,
+	0xb8f1f803, 0x8671ff71,
+	0x000001ff, 0xbf850002,
+	0x806c846c, 0x826d806d,
+	0x866dff6d, 0x0000ffff,
+	0x8f6e8b77, 0x866eff6e,
+	0x001f8000, 0xb96ef807,
+	0x86fe7e7e, 0x86ea6a6a,
+	0xb978f802, 0xbe801f6c,
+	0x866dff6d, 0x0000ffff,
+	0xbef00080, 0xb9700283,
+	0xb8f02407, 0x8e709c70,
+	0x876d706d, 0xb8f003c7,
+	0x8e709b70, 0x876d706d,
+	0xb8f0f807, 0x8670ff70,
+	0x00007fff, 0xb970f807,
+	0xbeee007e, 0xbeef007f,
+	0xbefe0180, 0xbf900004,
+	0x87708478, 0xb970f802,
+	0xbf8e0002, 0xbf88fffe,
+	0xb8f02a05, 0x80708170,
+	0x8e708a70, 0xb8f11605,
+	0x80718171, 0x8e718671,
+	0x80707170, 0x80707e70,
+	0x8271807f, 0x8671ff71,
+	0x0000ffff, 0xc0471cb8,
+	0x00000040, 0xbf8cc07f,
+	0xc04b1d38, 0x00000048,
+	0xbf8cc07f, 0xc0431e78,
+	0x00000058, 0xbf8cc07f,
+	0xc0471eb8, 0x0000005c,
+	0xbf8cc07f, 0xbef4007e,
+	0x8675ff7f, 0x0000ffff,
+	0x8775ff75, 0x00040000,
+	0xbef60080, 0xbef700ff,
+	0x00807fac, 0x8670ff7f,
+	0x08000000, 0x8f708370,
+	0x87777077, 0x8670ff7f,
+	0x70000000, 0x8f708170,
+	0x87777077, 0xbefb007c,
+	0xbefa0080, 0xb8fa2a05,
+	0x807a817a, 0x8e7a8a7a,
+	0xb8f01605, 0x80708170,
+	0x8e708670, 0x807a707a,
+	0xbef60084, 0xbef600ff,
+	0x01000000, 0xbefe007c,
+	0xbefc007a, 0xc0611efa,
+	0x0000007c, 0xbf8cc07f,
+	0x807a847a, 0xbefc007e,
+	0xbefe007c, 0xbefc007a,
+	0xc0611b3a, 0x0000007c,
+	0xbf8cc07f, 0x807a847a,
+	0xbefc007e, 0xbefe007c,
+	0xbefc007a, 0xc0611b7a,
+	0x0000007c, 0xbf8cc07f,
+	0x807a847a, 0xbefc007e,
+	0xbefe007c, 0xbefc007a,
+	0xc0611bba, 0x0000007c,
+	0xbf8cc07f, 0x807a847a,
+	0xbefc007e, 0xbefe007c,
+	0xbefc007a, 0xc0611bfa,
+	0x0000007c, 0xbf8cc07f,
+	0x807a847a, 0xbefc007e,
+	0xbefe007c, 0xbefc007a,
+	0xc0611e3a, 0x0000007c,
+	0xbf8cc07f, 0x807a847a,
+	0xbefc007e, 0xb8f1f803,
+	0xbefe007c, 0xbefc007a,
+	0xc0611c7a, 0x0000007c,
+	0xbf8cc07f, 0x807a847a,
+	0xbefc007e, 0xbefe007c,
+	0xbefc007a, 0xc0611a3a,
+	0x0000007c, 0xbf8cc07f,
+	0x807a847a, 0xbefc007e,
+	0xbefe007c, 0xbefc007a,
+	0xc0611a7a, 0x0000007c,
+	0xbf8cc07f, 0x807a847a,
+	0xbefc007e, 0xb8fbf801,
+	0xbefe007c, 0xbefc007a,
+	0xc0611efa, 0x0000007c,
+	0xbf8cc07f, 0x807a847a,
+	0xbefc007e, 0x8670ff7f,
+	0x04000000, 0xbeef0080,
+	0x876f6f70, 0xb8fa2a05,
+	0x807a817a, 0x8e7a8a7a,
+	0xb8f11605, 0x80718171,
+	0x8e718471, 0x8e768271,
+	0xbef600ff, 0x01000000,
+	0xbef20174, 0x80747a74,
+	0x82758075, 0xbefc0080,
+	0xbf800000, 0xbe802b00,
+	0xbe822b02, 0xbe842b04,
+	0xbe862b06, 0xbe882b08,
+	0xbe8a2b0a, 0xbe8c2b0c,
+	0xbe8e2b0e, 0xc06b003a,
+	0x00000000, 0xbf8cc07f,
+	0xc06b013a, 0x00000010,
+	0xbf8cc07f, 0xc06b023a,
+	0x00000020, 0xbf8cc07f,
+	0xc06b033a, 0x00000030,
+	0xbf8cc07f, 0x8074c074,
+	0x82758075, 0x807c907c,
+	0xbf0a717c, 0xbf85ffe7,
+	0xbef40172, 0xbefa0080,
+	0xbefe00c1, 0xbeff00c1,
+	0xbee80080, 0xbee90080,
+	0xbef600ff, 0x01000000,
+	0xe0724000, 0x7a1d0000,
+	0xe0724100, 0x7a1d0100,
+	0xe0724200, 0x7a1d0200,
+	0xe0724300, 0x7a1d0300,
+	0xbefe00c1, 0xbeff00c1,
+	0xb8f14306, 0x8671c171,
+	0xbf84002c, 0xbf8a0000,
+	0x8670ff6f, 0x04000000,
+	0xbf840028, 0x8e718671,
+	0x8e718271, 0xbef60071,
+	0xb8fa2a05, 0x807a817a,
+	0x8e7a8a7a, 0xb8f01605,
+	0x80708170, 0x8e708670,
+	0x807a707a, 0x807aff7a,
+	0x00000080, 0xbef600ff,
+	0x01000000, 0xbefc0080,
+	0xd28c0002, 0x000100c1,
+	0xd28d0003, 0x000204c1,
+	0xd1060002, 0x00011103,
+	0x7e0602ff, 0x00000200,
+	0xbefc00ff, 0x00010000,
+	0xbe800077, 0x8677ff77,
+	0xff7fffff, 0x8777ff77,
+	0x00058000, 0xd8ec0000,
+	0x00000002, 0xbf8cc07f,
+	0xe0765000, 0x7a1d0002,
+	0x68040702, 0xd0c9006a,
+	0x0000e302, 0xbf87fff7,
+	0xbef70000, 0xbefa00ff,
+	0x00000400, 0xbefe00c1,
+	0xbeff00c1, 0xb8f12a05,
+	0x80718171, 0x8e718271,
+	0x8e768871, 0xbef600ff,
+	0x01000000, 0xbefc0084,
+	0xbf0a717c, 0xbf840015,
+	0xbf11017c, 0x8071ff71,
+	0x00001000, 0x7e000300,
+	0x7e020301, 0x7e040302,
+	0x7e060303, 0xe0724000,
+	0x7a1d0000, 0xe0724100,
+	0x7a1d0100, 0xe0724200,
+	0x7a1d0200, 0xe0724300,
+	0x7a1d0300, 0x807c847c,
+	0x807aff7a, 0x00000400,
+	0xbf0a717c, 0xbf85ffef,
+	0xbf9c0000, 0xbf8200d9,
+	0xbef4007e, 0x8675ff7f,
+	0x0000ffff, 0x8775ff75,
+	0x00040000, 0xbef60080,
+	0xbef700ff, 0x00807fac,
+	0x866eff7f, 0x08000000,
+	0x8f6e836e, 0x87776e77,
+	0x866eff7f, 0x70000000,
+	0x8f6e816e, 0x87776e77,
+	0x866eff7f, 0x04000000,
+	0xbf84001e, 0xbefe00c1,
+	0xbeff00c1, 0xb8ef4306,
+	0x866fc16f, 0xbf840019,
+	0x8e6f866f, 0x8e6f826f,
+	0xbef6006f, 0xb8f82a05,
+	0x80788178, 0x8e788a78,
+	0xb8ee1605, 0x806e816e,
+	0x8e6e866e, 0x80786e78,
+	0x8078ff78, 0x00000080,
+	0xbef600ff, 0x01000000,
+	0xbefc0080, 0xe0510000,
+	0x781d0000, 0xe0510100,
+	0x781d0000, 0x807cff7c,
+	0x00000200, 0x8078ff78,
+	0x00000200, 0xbf0a6f7c,
+	0xbf85fff6, 0xbef80080,
+	0xbefe00c1, 0xbeff00c1,
+	0xb8ef2a05, 0x806f816f,
+	0x8e6f826f, 0x8e76886f,
+	0xbef600ff, 0x01000000,
+	0xbeee0078, 0x8078ff78,
+	0x00000400, 0xbefc0084,
+	0xbf11087c, 0x806fff6f,
+	0x00008000, 0xe0524000,
+	0x781d0000, 0xe0524100,
+	0x781d0100, 0xe0524200,
+	0x781d0200, 0xe0524300,
+	0x781d0300, 0xbf8c0f70,
+	0x7e000300, 0x7e020301,
+	0x7e040302, 0x7e060303,
+	0x807c847c, 0x8078ff78,
+	0x00000400, 0xbf0a6f7c,
+	0xbf85ffee, 0xbf9c0000,
+	0xe0524000, 0x6e1d0000,
+	0xe0524100, 0x6e1d0100,
+	0xe0524200, 0x6e1d0200,
+	0xe0524300, 0x6e1d0300,
+	0xb8f82a05, 0x80788178,
+	0x8e788a78, 0xb8ee1605,
+	0x806e816e, 0x8e6e866e,
+	0x80786e78, 0x80f8c078,
+	0xb8ef1605, 0x806f816f,
+	0x8e6f846f, 0x8e76826f,
+	0xbef600ff, 0x01000000,
+	0xbefc006f, 0xc031003a,
+	0x00000078, 0x80f8c078,
+	0xbf8cc07f, 0x80fc907c,
+	0xbf800000, 0xbe802d00,
+	0xbe822d02, 0xbe842d04,
+	0xbe862d06, 0xbe882d08,
+	0xbe8a2d0a, 0xbe8c2d0c,
+	0xbe8e2d0e, 0xbf06807c,
+	0xbf84fff0, 0xb8f82a05,
+	0x80788178, 0x8e788a78,
+	0xb8ee1605, 0x806e816e,
+	0x8e6e866e, 0x80786e78,
+	0xbef60084, 0xbef600ff,
+	0x01000000, 0xc0211bfa,
+	0x00000078, 0x80788478,
+	0xc0211b3a, 0x00000078,
+	0x80788478, 0xc0211b7a,
+	0x00000078, 0x80788478,
+	0xc0211eba, 0x00000078,
+	0x80788478, 0xc0211efa,
+	0x00000078, 0x80788478,
+	0xc0211c3a, 0x00000078,
+	0x80788478, 0xc0211c7a,
+	0x00000078, 0x80788478,
+	0xc0211a3a, 0x00000078,
+	0x80788478, 0xc0211a7a,
+	0x00000078, 0x80788478,
+	0xc0211cfa, 0x00000078,
+	0x80788478, 0xbf8cc07f,
+	0xbefc006f, 0xbefe007a,
+	0xbeff007b, 0x866f71ff,
+	0x000003ff, 0xb96f4803,
+	0x866f71ff, 0xfffff800,
+	0x8f6f8b6f, 0xb96fa2c3,
+	0xb973f801, 0xb8ee2a05,
+	0x806e816e, 0x8e6e8a6e,
+	0xb8ef1605, 0x806f816f,
+	0x8e6f866f, 0x806e6f6e,
+	0x806e746e, 0x826f8075,
+	0x866fff6f, 0x0000ffff,
+	0xc0071cb7, 0x00000040,
+	0xc00b1d37, 0x00000048,
+	0xc0031e77, 0x00000058,
+	0xc0071eb7, 0x0000005c,
+	0xbf8cc07f, 0x866fff6d,
+	0xf0000000, 0x8f6f9c6f,
+	0x8e6f906f, 0xbeee0080,
+	0x876e6f6e, 0x866fff6d,
+	0x08000000, 0x8f6f9b6f,
+	0x8e6f8f6f, 0x876e6f6e,
+	0x866fff70, 0x00800000,
+	0x8f6f976f, 0xb96ef807,
+	0x866dff6d, 0x0000ffff,
+	0x86fe7e7e, 0x86ea6a6a,
+	0xb970f802, 0xbf8a0000,
+	0x95806f6c, 0xbf810000,
+};
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
index 997a383dcb8b..a2a04bb64096 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
@@ -20,9 +20,12 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#if 0
-HW (VI) source code for CWSR trap handler
-#Version 18 + multiple trap handler
+/* To compile this assembly code:
+ * PROJECT=vi ./sp3 cwsr_trap_handler_gfx8.asm -hex tmp.hex
+ */
+
+/* HW (VI) source code for CWSR trap handler */
+/* Version 18 + multiple trap handler */
 
 // this performance-optimal version was originally from Seven Xu at SRDC
 
@@ -98,6 +101,7 @@ var SWIZZLE_EN                      =   0                   //whether we use swi
 /**************************************************************************/
 var SQ_WAVE_STATUS_INST_ATC_SHIFT  = 23
 var SQ_WAVE_STATUS_INST_ATC_MASK   = 0x00800000
+var SQ_WAVE_STATUS_SPI_PRIO_SHIFT  = 1
 var SQ_WAVE_STATUS_SPI_PRIO_MASK   = 0x00000006
 
 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT    = 12
@@ -149,7 +153,7 @@ var s_save_spi_init_lo              =   exec_lo
 var s_save_spi_init_hi              =   exec_hi
 
                                                 //tba_lo and tba_hi need to be saved/restored
-var s_save_pc_lo            =   ttmp0           //{TTMP1, TTMP0} = {3??h0,pc_rewind[3:0], HT[0],trapID[7:0], PC[47:0]}
+var s_save_pc_lo            =   ttmp0           //{TTMP1, TTMP0} = {3'h0,pc_rewind[3:0], HT[0],trapID[7:0], PC[47:0]}
 var s_save_pc_hi            =   ttmp1
 var s_save_exec_lo          =   ttmp2
 var s_save_exec_hi          =   ttmp3
@@ -319,6 +323,10 @@ end
         s_sendmsg   sendmsg(MSG_SAVEWAVE)  //send SPI a message and wait for SPI's write to EXEC
     end
 
+    // Set SPI_PRIO=2 to avoid starving instruction fetch in the waves we're waiting for.
+    s_or_b32 s_save_tmp, s_save_status, (2 << SQ_WAVE_STATUS_SPI_PRIO_SHIFT)
+    s_setreg_b32 hwreg(HW_REG_STATUS), s_save_tmp
+
   L_SLEEP:
     s_sleep 0x2                // sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause SQ hang, since the 7,8th wave could not get arbit to exec inst, while other waves are stuck into the sleep-loop and waiting for wrexec!=0
 
@@ -1007,8 +1015,6 @@ end
 
     s_waitcnt       lgkmcnt(0)                                                                                      //from now on, it is safe to restore STATUS and IB_STS
 
-    s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff      //pc[47:32]        //Do it here in order not to affect STATUS
-
     //for normal save & restore, the saved PC points to the next inst to execute, no adjustment needs to be made, otherwise:
     if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
         s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 8            //pc[31:0]+8     //two back-to-back s_trap are used (first for save and second for restore)
@@ -1044,6 +1050,7 @@ end
     s_lshr_b32      s_restore_m0, s_restore_m0, SQ_WAVE_STATUS_INST_ATC_SHIFT
     s_setreg_b32    hwreg(HW_REG_IB_STS),   s_restore_tmp
 
+    s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff      //pc[47:32]        //Do it here in order not to affect STATUS
     s_and_b64    exec, exec, exec  // Restore STATUS.EXECZ, not writable by s_setreg_b32
     s_and_b64    vcc, vcc, vcc  // Restore STATUS.VCCZ, not writable by s_setreg_b32
     s_setreg_b32    hwreg(HW_REG_STATUS),   s_restore_status     // SCC is included, which is changed by previous salu
@@ -1127,258 +1134,3 @@ end
 function get_hwreg_size_bytes
     return 128 //HWREG size 128 bytes
 end
-
-
-#endif
-
-static const uint32_t cwsr_trap_gfx8_hex[] = {
-	0xbf820001, 0xbf820123,
-	0xb8f4f802, 0x89748674,
-	0xb8f5f803, 0x8675ff75,
-	0x00000400, 0xbf850011,
-	0xc00a1e37, 0x00000000,
-	0xbf8c007f, 0x87777978,
-	0xbf840002, 0xb974f802,
-	0xbe801d78, 0xb8f5f803,
-	0x8675ff75, 0x000001ff,
-	0xbf850002, 0x80708470,
-	0x82718071, 0x8671ff71,
-	0x0000ffff, 0xb974f802,
-	0xbe801f70, 0xb8f5f803,
-	0x8675ff75, 0x00000100,
-	0xbf840006, 0xbefa0080,
-	0xb97a0203, 0x8671ff71,
-	0x0000ffff, 0x80f08870,
-	0x82f18071, 0xbefa0080,
-	0xb97a0283, 0xbef60068,
-	0xbef70069, 0xb8fa1c07,
-	0x8e7a9c7a, 0x87717a71,
-	0xb8fa03c7, 0x8e7a9b7a,
-	0x87717a71, 0xb8faf807,
-	0x867aff7a, 0x00007fff,
-	0xb97af807, 0xbef2007e,
-	0xbef3007f, 0xbefe0180,
-	0xbf900004, 0xbf8e0002,
-	0xbf88fffe, 0xbef8007e,
-	0x8679ff7f, 0x0000ffff,
-	0x8779ff79, 0x00040000,
-	0xbefa0080, 0xbefb00ff,
-	0x00807fac, 0x867aff7f,
-	0x08000000, 0x8f7a837a,
-	0x877b7a7b, 0x867aff7f,
-	0x70000000, 0x8f7a817a,
-	0x877b7a7b, 0xbeef007c,
-	0xbeee0080, 0xb8ee2a05,
-	0x806e816e, 0x8e6e8a6e,
-	0xb8fa1605, 0x807a817a,
-	0x8e7a867a, 0x806e7a6e,
-	0xbefa0084, 0xbefa00ff,
-	0x01000000, 0xbefe007c,
-	0xbefc006e, 0xc0611bfc,
-	0x0000007c, 0x806e846e,
-	0xbefc007e, 0xbefe007c,
-	0xbefc006e, 0xc0611c3c,
-	0x0000007c, 0x806e846e,
-	0xbefc007e, 0xbefe007c,
-	0xbefc006e, 0xc0611c7c,
-	0x0000007c, 0x806e846e,
-	0xbefc007e, 0xbefe007c,
-	0xbefc006e, 0xc0611cbc,
-	0x0000007c, 0x806e846e,
-	0xbefc007e, 0xbefe007c,
-	0xbefc006e, 0xc0611cfc,
-	0x0000007c, 0x806e846e,
-	0xbefc007e, 0xbefe007c,
-	0xbefc006e, 0xc0611d3c,
-	0x0000007c, 0x806e846e,
-	0xbefc007e, 0xb8f5f803,
-	0xbefe007c, 0xbefc006e,
-	0xc0611d7c, 0x0000007c,
-	0x806e846e, 0xbefc007e,
-	0xbefe007c, 0xbefc006e,
-	0xc0611dbc, 0x0000007c,
-	0x806e846e, 0xbefc007e,
-	0xbefe007c, 0xbefc006e,
-	0xc0611dfc, 0x0000007c,
-	0x806e846e, 0xbefc007e,
-	0xb8eff801, 0xbefe007c,
-	0xbefc006e, 0xc0611bfc,
-	0x0000007c, 0x806e846e,
-	0xbefc007e, 0xbefe007c,
-	0xbefc006e, 0xc0611b3c,
-	0x0000007c, 0x806e846e,
-	0xbefc007e, 0xbefe007c,
-	0xbefc006e, 0xc0611b7c,
-	0x0000007c, 0x806e846e,
-	0xbefc007e, 0x867aff7f,
-	0x04000000, 0xbef30080,
-	0x8773737a, 0xb8ee2a05,
-	0x806e816e, 0x8e6e8a6e,
-	0xb8f51605, 0x80758175,
-	0x8e758475, 0x8e7a8275,
-	0xbefa00ff, 0x01000000,
-	0xbef60178, 0x80786e78,
-	0x82798079, 0xbefc0080,
-	0xbe802b00, 0xbe822b02,
-	0xbe842b04, 0xbe862b06,
-	0xbe882b08, 0xbe8a2b0a,
-	0xbe8c2b0c, 0xbe8e2b0e,
-	0xc06b003c, 0x00000000,
-	0xc06b013c, 0x00000010,
-	0xc06b023c, 0x00000020,
-	0xc06b033c, 0x00000030,
-	0x8078c078, 0x82798079,
-	0x807c907c, 0xbf0a757c,
-	0xbf85ffeb, 0xbef80176,
-	0xbeee0080, 0xbefe00c1,
-	0xbeff00c1, 0xbefa00ff,
-	0x01000000, 0xe0724000,
-	0x6e1e0000, 0xe0724100,
-	0x6e1e0100, 0xe0724200,
-	0x6e1e0200, 0xe0724300,
-	0x6e1e0300, 0xbefe00c1,
-	0xbeff00c1, 0xb8f54306,
-	0x8675c175, 0xbf84002c,
-	0xbf8a0000, 0x867aff73,
-	0x04000000, 0xbf840028,
-	0x8e758675, 0x8e758275,
-	0xbefa0075, 0xb8ee2a05,
-	0x806e816e, 0x8e6e8a6e,
-	0xb8fa1605, 0x807a817a,
-	0x8e7a867a, 0x806e7a6e,
-	0x806eff6e, 0x00000080,
-	0xbefa00ff, 0x01000000,
-	0xbefc0080, 0xd28c0002,
-	0x000100c1, 0xd28d0003,
-	0x000204c1, 0xd1060002,
-	0x00011103, 0x7e0602ff,
-	0x00000200, 0xbefc00ff,
-	0x00010000, 0xbe80007b,
-	0x867bff7b, 0xff7fffff,
-	0x877bff7b, 0x00058000,
-	0xd8ec0000, 0x00000002,
-	0xbf8c007f, 0xe0765000,
-	0x6e1e0002, 0x32040702,
-	0xd0c9006a, 0x0000eb02,
-	0xbf87fff7, 0xbefb0000,
-	0xbeee00ff, 0x00000400,
-	0xbefe00c1, 0xbeff00c1,
-	0xb8f52a05, 0x80758175,
-	0x8e758275, 0x8e7a8875,
-	0xbefa00ff, 0x01000000,
-	0xbefc0084, 0xbf0a757c,
-	0xbf840015, 0xbf11017c,
-	0x8075ff75, 0x00001000,
-	0x7e000300, 0x7e020301,
-	0x7e040302, 0x7e060303,
-	0xe0724000, 0x6e1e0000,
-	0xe0724100, 0x6e1e0100,
-	0xe0724200, 0x6e1e0200,
-	0xe0724300, 0x6e1e0300,
-	0x807c847c, 0x806eff6e,
-	0x00000400, 0xbf0a757c,
-	0xbf85ffef, 0xbf9c0000,
-	0xbf8200ca, 0xbef8007e,
-	0x8679ff7f, 0x0000ffff,
-	0x8779ff79, 0x00040000,
-	0xbefa0080, 0xbefb00ff,
-	0x00807fac, 0x8676ff7f,
-	0x08000000, 0x8f768376,
-	0x877b767b, 0x8676ff7f,
-	0x70000000, 0x8f768176,
-	0x877b767b, 0x8676ff7f,
-	0x04000000, 0xbf84001e,
-	0xbefe00c1, 0xbeff00c1,
-	0xb8f34306, 0x8673c173,
-	0xbf840019, 0x8e738673,
-	0x8e738273, 0xbefa0073,
-	0xb8f22a05, 0x80728172,
-	0x8e728a72, 0xb8f61605,
-	0x80768176, 0x8e768676,
-	0x80727672, 0x8072ff72,
-	0x00000080, 0xbefa00ff,
-	0x01000000, 0xbefc0080,
-	0xe0510000, 0x721e0000,
-	0xe0510100, 0x721e0000,
-	0x807cff7c, 0x00000200,
-	0x8072ff72, 0x00000200,
-	0xbf0a737c, 0xbf85fff6,
-	0xbef20080, 0xbefe00c1,
-	0xbeff00c1, 0xb8f32a05,
-	0x80738173, 0x8e738273,
-	0x8e7a8873, 0xbefa00ff,
-	0x01000000, 0xbef60072,
-	0x8072ff72, 0x00000400,
-	0xbefc0084, 0xbf11087c,
-	0x8073ff73, 0x00008000,
-	0xe0524000, 0x721e0000,
-	0xe0524100, 0x721e0100,
-	0xe0524200, 0x721e0200,
-	0xe0524300, 0x721e0300,
-	0xbf8c0f70, 0x7e000300,
-	0x7e020301, 0x7e040302,
-	0x7e060303, 0x807c847c,
-	0x8072ff72, 0x00000400,
-	0xbf0a737c, 0xbf85ffee,
-	0xbf9c0000, 0xe0524000,
-	0x761e0000, 0xe0524100,
-	0x761e0100, 0xe0524200,
-	0x761e0200, 0xe0524300,
-	0x761e0300, 0xb8f22a05,
-	0x80728172, 0x8e728a72,
-	0xb8f61605, 0x80768176,
-	0x8e768676, 0x80727672,
-	0x80f2c072, 0xb8f31605,
-	0x80738173, 0x8e738473,
-	0x8e7a8273, 0xbefa00ff,
-	0x01000000, 0xbefc0073,
-	0xc031003c, 0x00000072,
-	0x80f2c072, 0xbf8c007f,
-	0x80fc907c, 0xbe802d00,
-	0xbe822d02, 0xbe842d04,
-	0xbe862d06, 0xbe882d08,
-	0xbe8a2d0a, 0xbe8c2d0c,
-	0xbe8e2d0e, 0xbf06807c,
-	0xbf84fff1, 0xb8f22a05,
-	0x80728172, 0x8e728a72,
-	0xb8f61605, 0x80768176,
-	0x8e768676, 0x80727672,
-	0xbefa0084, 0xbefa00ff,
-	0x01000000, 0xc0211cfc,
-	0x00000072, 0x80728472,
-	0xc0211c3c, 0x00000072,
-	0x80728472, 0xc0211c7c,
-	0x00000072, 0x80728472,
-	0xc0211bbc, 0x00000072,
-	0x80728472, 0xc0211bfc,
-	0x00000072, 0x80728472,
-	0xc0211d3c, 0x00000072,
-	0x80728472, 0xc0211d7c,
-	0x00000072, 0x80728472,
-	0xc0211a3c, 0x00000072,
-	0x80728472, 0xc0211a7c,
-	0x00000072, 0x80728472,
-	0xc0211dfc, 0x00000072,
-	0x80728472, 0xc0211b3c,
-	0x00000072, 0x80728472,
-	0xc0211b7c, 0x00000072,
-	0x80728472, 0xbf8c007f,
-	0x8671ff71, 0x0000ffff,
-	0xbefc0073, 0xbefe006e,
-	0xbeff006f, 0x867375ff,
-	0x000003ff, 0xb9734803,
-	0x867375ff, 0xfffff800,
-	0x8f738b73, 0xb973a2c3,
-	0xb977f801, 0x8673ff71,
-	0xf0000000, 0x8f739c73,
-	0x8e739073, 0xbef60080,
-	0x87767376, 0x8673ff71,
-	0x08000000, 0x8f739b73,
-	0x8e738f73, 0x87767376,
-	0x8673ff74, 0x00800000,
-	0x8f739773, 0xb976f807,
-	0x86fe7e7e, 0x86ea6a6a,
-	0xb974f802, 0xbf8a0000,
-	0x95807370, 0xbf810000,
-};
-
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
new file mode 100644
index 000000000000..998be96be736
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
@@ -0,0 +1,1214 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/* To compile this assembly code:
+ * PROJECT=greenland ./sp3 cwsr_trap_handler_gfx9.asm -hex tmp.hex
+ */
+
+/* HW (GFX9) source code for CWSR trap handler */
+/* Version 18 + multiple trap handler */
+
+// this performance-optimal version was originally from Seven Xu at SRDC
+
+// Revison #18	 --...
+/* Rev History
+** #1. Branch from gc dv.   //gfxip/gfx9/main/src/test/suites/block/cs/sr/cs_trap_handler.sp3#1,#50, #51, #52-53(Skip, Already Fixed by PV), #54-56(merged),#57-58(mergerd, skiped-already fixed by PV)
+** #4. SR Memory Layout:
+**			 1. VGPR-SGPR-HWREG-{LDS}
+**			 2. tba_hi.bits.26 - reconfigured as the first wave in tg bits, for defer Save LDS for a threadgroup.. performance concern..
+** #5. Update: 1. Accurate g8sr_ts_save_d timestamp
+** #6. Update: 1. Fix s_barrier usage; 2. VGPR s/r using swizzle buffer?(NoNeed, already matched the swizzle pattern, more investigation)
+** #7. Update: 1. don't barrier if noLDS
+** #8. Branch: 1. Branch to ver#0, which is very similar to gc dv version
+**	       2. Fix SQ issue by s_sleep 2
+** #9. Update: 1. Fix scc restore failed issue, restore wave_status at last
+**	       2. optimize s_buffer save by burst 16sgprs...
+** #10. Update 1. Optimize restore sgpr by busrt 16 sgprs.
+** #11. Update 1. Add 2 more timestamp for debug version
+** #12. Update 1. Add VGPR SR using DWx4, some case improve and some case drop performance
+** #13. Integ  1. Always use MUBUF for PV trap shader...
+** #14. Update 1. s_buffer_store soft clause...
+** #15. Update 1. PERF - sclar write with glc:0/mtype0 to allow L2 combine. perf improvement a lot.
+** #16. Update 1. PRRF - UNROLL LDS_DMA got 2500cycle save in IP tree
+** #17. Update 1. FUNC - LDS_DMA has issues while ATC, replace with ds_read/buffer_store for save part[TODO restore part]
+**	       2. PERF - Save LDS before save VGPR to cover LDS save long latency...
+** #18. Update 1. FUNC - Implicitly estore STATUS.VCCZ, which is not writable by s_setreg_b32
+**	       2. FUNC - Handle non-CWSR traps
+*/
+
+var G8SR_WDMEM_HWREG_OFFSET = 0
+var G8SR_WDMEM_SGPR_OFFSET  = 128  // in bytes
+
+// Keep definition same as the app shader, These 2 time stamps are part of the app shader... Should before any Save and after restore.
+
+var G8SR_DEBUG_TIMESTAMP = 0
+var G8SR_DEBUG_TS_SAVE_D_OFFSET = 40*4	// ts_save_d timestamp offset relative to SGPR_SR_memory_offset
+var s_g8sr_ts_save_s	= s[34:35]   // save start
+var s_g8sr_ts_sq_save_msg  = s[36:37]	// The save shader send SAVEWAVE msg to spi
+var s_g8sr_ts_spi_wrexec   = s[38:39]	// the SPI write the sr address to SQ
+var s_g8sr_ts_save_d	= s[40:41]   // save end
+var s_g8sr_ts_restore_s = s[42:43]   // restore start
+var s_g8sr_ts_restore_d = s[44:45]   // restore end
+
+var G8SR_VGPR_SR_IN_DWX4 = 0
+var G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4 = 0x00100000	 // DWx4 stride is 4*4Bytes
+var G8SR_RESTORE_BUF_RSRC_WORD1_STRIDE_DWx4  = G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4
+
+
+/*************************************************************************/
+/*		    control on how to run the shader			 */
+/*************************************************************************/
+//any hack that needs to be made to run this code in EMU (either because various EMU code are not ready or no compute save & restore in EMU run)
+var EMU_RUN_HACK		    =	0
+var EMU_RUN_HACK_RESTORE_NORMAL	    =	0
+var EMU_RUN_HACK_SAVE_NORMAL_EXIT   =	0
+var EMU_RUN_HACK_SAVE_SINGLE_WAVE   =	0
+var EMU_RUN_HACK_SAVE_FIRST_TIME    =	0		    //for interrupted restore in which the first save is through EMU_RUN_HACK
+var SAVE_LDS			    =	1
+var WG_BASE_ADDR_LO		    =	0x9000a000
+var WG_BASE_ADDR_HI		    =	0x0
+var WAVE_SPACE			    =	0x5000		    //memory size that each wave occupies in workgroup state mem
+var CTX_SAVE_CONTROL		    =	0x0
+var CTX_RESTORE_CONTROL		    =	CTX_SAVE_CONTROL
+var SIM_RUN_HACK		    =	0		    //any hack that needs to be made to run this code in SIM (either because various RTL code are not ready or no compute save & restore in RTL run)
+var SGPR_SAVE_USE_SQC		    =	1		    //use SQC D$ to do the write
+var USE_MTBUF_INSTEAD_OF_MUBUF	    =	0		    //because TC EMU currently asserts on 0 of // overload DFMT field to carry 4 more bits of stride for MUBUF opcodes
+var SWIZZLE_EN			    =	0		    //whether we use swizzled buffer addressing
+var ACK_SQC_STORE		    =	1		    //workaround for suspected SQC store bug causing incorrect stores under concurrency
+
+/**************************************************************************/
+/*			variables					  */
+/**************************************************************************/
+var SQ_WAVE_STATUS_INST_ATC_SHIFT  = 23
+var SQ_WAVE_STATUS_INST_ATC_MASK   = 0x00800000
+var SQ_WAVE_STATUS_SPI_PRIO_SHIFT  = 1
+var SQ_WAVE_STATUS_SPI_PRIO_MASK   = 0x00000006
+var SQ_WAVE_STATUS_HALT_MASK       = 0x2000
+
+var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT	= 12
+var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE	= 9
+var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT	= 8
+var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE	= 6
+var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT	= 24
+var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE	= 3			//FIXME	 sq.blk still has 4 bits at this time while SQ programming guide has 3 bits
+
+var SQ_WAVE_TRAPSTS_SAVECTX_MASK    =	0x400
+var SQ_WAVE_TRAPSTS_EXCE_MASK	    =	0x1FF			// Exception mask
+var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT   =	10
+var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK   =	0x100
+var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT  =	8
+var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK	=   0x3FF
+var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT	=   0x0
+var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE	=   10
+var SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK	=   0xFFFFF800
+var SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT	=   11
+var SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE	=   21
+var SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK	=   0x800
+
+var SQ_WAVE_IB_STS_RCNT_SHIFT		=   16			//FIXME
+var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT	=   15			//FIXME
+var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK	= 0x1F8000
+var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG	= 0x00007FFF	//FIXME
+
+var SQ_BUF_RSRC_WORD1_ATC_SHIFT	    =	24
+var SQ_BUF_RSRC_WORD3_MTYPE_SHIFT   =	27
+
+var TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT	=   26			// bits [31:26] unused by SPI debug data
+var TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK	=   0xFC000000
+
+/*	Save	    */
+var S_SAVE_BUF_RSRC_WORD1_STRIDE	=   0x00040000		//stride is 4 bytes
+var S_SAVE_BUF_RSRC_WORD3_MISC		=   0x00807FAC		//SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14] when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE
+
+var S_SAVE_SPI_INIT_ATC_MASK		=   0x08000000		//bit[27]: ATC bit
+var S_SAVE_SPI_INIT_ATC_SHIFT		=   27
+var S_SAVE_SPI_INIT_MTYPE_MASK		=   0x70000000		//bit[30:28]: Mtype
+var S_SAVE_SPI_INIT_MTYPE_SHIFT		=   28
+var S_SAVE_SPI_INIT_FIRST_WAVE_MASK	=   0x04000000		//bit[26]: FirstWaveInTG
+var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT	=   26
+
+var S_SAVE_PC_HI_RCNT_SHIFT		=   28			//FIXME	 check with Brian to ensure all fields other than PC[47:0] can be used
+var S_SAVE_PC_HI_RCNT_MASK		=   0xF0000000		//FIXME
+var S_SAVE_PC_HI_FIRST_REPLAY_SHIFT	=   27			//FIXME
+var S_SAVE_PC_HI_FIRST_REPLAY_MASK	=   0x08000000		//FIXME
+
+var s_save_spi_init_lo		    =	exec_lo
+var s_save_spi_init_hi		    =	exec_hi
+
+var s_save_pc_lo	    =	ttmp0		//{TTMP1, TTMP0} = {3'h0,pc_rewind[3:0], HT[0],trapID[7:0], PC[47:0]}
+var s_save_pc_hi	    =	ttmp1
+var s_save_exec_lo	    =	ttmp2
+var s_save_exec_hi	    =	ttmp3
+var s_save_tmp		    =	ttmp4
+var s_save_trapsts	    =	ttmp5		//not really used until the end of the SAVE routine
+var s_save_xnack_mask_lo    =	ttmp6
+var s_save_xnack_mask_hi    =	ttmp7
+var s_save_buf_rsrc0	    =	ttmp8
+var s_save_buf_rsrc1	    =	ttmp9
+var s_save_buf_rsrc2	    =	ttmp10
+var s_save_buf_rsrc3	    =	ttmp11
+var s_save_status	    =	ttmp12
+var s_save_mem_offset	    =	ttmp14
+var s_save_alloc_size	    =	s_save_trapsts		//conflict
+var s_save_m0		    =	ttmp15
+var s_save_ttmps_lo	    =	s_save_tmp		//no conflict
+var s_save_ttmps_hi	    =	s_save_trapsts		//no conflict
+
+/*	Restore	    */
+var S_RESTORE_BUF_RSRC_WORD1_STRIDE	    =	S_SAVE_BUF_RSRC_WORD1_STRIDE
+var S_RESTORE_BUF_RSRC_WORD3_MISC	    =	S_SAVE_BUF_RSRC_WORD3_MISC
+
+var S_RESTORE_SPI_INIT_ATC_MASK		    =	0x08000000	    //bit[27]: ATC bit
+var S_RESTORE_SPI_INIT_ATC_SHIFT	    =	27
+var S_RESTORE_SPI_INIT_MTYPE_MASK	    =	0x70000000	    //bit[30:28]: Mtype
+var S_RESTORE_SPI_INIT_MTYPE_SHIFT	    =	28
+var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK	    =	0x04000000	    //bit[26]: FirstWaveInTG
+var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT	    =	26
+
+var S_RESTORE_PC_HI_RCNT_SHIFT		    =	S_SAVE_PC_HI_RCNT_SHIFT
+var S_RESTORE_PC_HI_RCNT_MASK		    =	S_SAVE_PC_HI_RCNT_MASK
+var S_RESTORE_PC_HI_FIRST_REPLAY_SHIFT	    =	S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+var S_RESTORE_PC_HI_FIRST_REPLAY_MASK	    =	S_SAVE_PC_HI_FIRST_REPLAY_MASK
+
+var s_restore_spi_init_lo		    =	exec_lo
+var s_restore_spi_init_hi		    =	exec_hi
+
+var s_restore_mem_offset	=   ttmp12
+var s_restore_alloc_size	=   ttmp3
+var s_restore_tmp		=   ttmp2
+var s_restore_mem_offset_save	=   s_restore_tmp	//no conflict
+
+var s_restore_m0	    =	s_restore_alloc_size	//no conflict
+
+var s_restore_mode	    =	ttmp7
+
+var s_restore_pc_lo	    =	ttmp0
+var s_restore_pc_hi	    =	ttmp1
+var s_restore_exec_lo	    =	ttmp14
+var s_restore_exec_hi	    = 	ttmp15
+var s_restore_status	    =	ttmp4
+var s_restore_trapsts	    =	ttmp5
+var s_restore_xnack_mask_lo =	xnack_mask_lo
+var s_restore_xnack_mask_hi =	xnack_mask_hi
+var s_restore_buf_rsrc0	    =	ttmp8
+var s_restore_buf_rsrc1	    =	ttmp9
+var s_restore_buf_rsrc2	    =	ttmp10
+var s_restore_buf_rsrc3	    =	ttmp11
+var s_restore_ttmps_lo	    =	s_restore_tmp		//no conflict
+var s_restore_ttmps_hi	    =	s_restore_alloc_size	//no conflict
+
+/**************************************************************************/
+/*			trap handler entry points			  */
+/**************************************************************************/
+/* Shader Main*/
+
+shader main
+  asic(GFX9)
+  type(CS)
+
+
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))		    //hack to use trap_id for determining save/restore
+	//FIXME VCCZ un-init assertion s_getreg_b32	s_save_status, hwreg(HW_REG_STATUS)	    //save STATUS since we will change SCC
+	s_and_b32 s_save_tmp, s_save_pc_hi, 0xffff0000		    //change SCC
+	s_cmp_eq_u32 s_save_tmp, 0x007e0000			    //Save: trap_id = 0x7e. Restore: trap_id = 0x7f.
+	s_cbranch_scc0 L_JUMP_TO_RESTORE			    //do not need to recover STATUS here  since we are going to RESTORE
+	//FIXME	 s_setreg_b32	hwreg(HW_REG_STATUS),	s_save_status	    //need to recover STATUS since we are going to SAVE
+	s_branch L_SKIP_RESTORE					    //NOT restore, SAVE actually
+    else
+	s_branch L_SKIP_RESTORE					    //NOT restore. might be a regular trap or save
+    end
+
+L_JUMP_TO_RESTORE:
+    s_branch L_RESTORE						    //restore
+
+L_SKIP_RESTORE:
+
+    s_getreg_b32    s_save_status, hwreg(HW_REG_STATUS)				    //save STATUS since we will change SCC
+    s_andn2_b32	    s_save_status, s_save_status, SQ_WAVE_STATUS_SPI_PRIO_MASK	    //check whether this is for save
+    s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+    s_and_b32       ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK    //check whether this is for save
+    s_cbranch_scc1  L_SAVE					//this is the operation for save
+
+    // *********    Handle non-CWSR traps	*******************
+if (!EMU_RUN_HACK)
+    // Illegal instruction is a non-maskable exception which blocks context save.
+    // Halt the wavefront and return from the trap.
+    s_and_b32       ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK
+    s_cbranch_scc1  L_HALT_WAVE
+
+    // If STATUS.MEM_VIOL is asserted then we cannot fetch from the TMA.
+    // Instead, halt the wavefront and return from the trap.
+    s_and_b32       ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK
+    s_cbranch_scc0  L_FETCH_2ND_TRAP
+
+L_HALT_WAVE:
+    // If STATUS.HALT is set then this fault must come from SQC instruction fetch.
+    // We cannot prevent further faults so just terminate the wavefront.
+    s_and_b32       ttmp2, s_save_status, SQ_WAVE_STATUS_HALT_MASK
+    s_cbranch_scc0  L_NOT_ALREADY_HALTED
+    s_endpgm
+L_NOT_ALREADY_HALTED:
+    s_or_b32        s_save_status, s_save_status, SQ_WAVE_STATUS_HALT_MASK
+
+    // If the PC points to S_ENDPGM then context save will fail if STATUS.HALT is set.
+    // Rewind the PC to prevent this from occurring. The debugger compensates for this.
+    s_sub_u32       ttmp0, ttmp0, 0x8
+    s_subb_u32      ttmp1, ttmp1, 0x0
+
+L_FETCH_2ND_TRAP:
+    // Preserve and clear scalar XNACK state before issuing scalar reads.
+    // Save IB_STS.FIRST_REPLAY[15] and IB_STS.RCNT[20:16] into unused space ttmp11[31:26].
+    s_getreg_b32    ttmp2, hwreg(HW_REG_IB_STS)
+    s_and_b32       ttmp3, ttmp2, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
+    s_lshl_b32      ttmp3, ttmp3, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
+    s_andn2_b32     ttmp11, ttmp11, TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK
+    s_or_b32        ttmp11, ttmp11, ttmp3
+
+    s_andn2_b32     ttmp2, ttmp2, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
+    s_setreg_b32    hwreg(HW_REG_IB_STS), ttmp2
+
+    // Read second-level TBA/TMA from first-level TMA and jump if available.
+    // ttmp[2:5] and ttmp12 can be used (others hold SPI-initialized debug data)
+    // ttmp12 holds SQ_WAVE_STATUS
+    s_getreg_b32    ttmp4, hwreg(HW_REG_SQ_SHADER_TMA_LO)
+    s_getreg_b32    ttmp5, hwreg(HW_REG_SQ_SHADER_TMA_HI)
+    s_lshl_b64      [ttmp4, ttmp5], [ttmp4, ttmp5], 0x8
+    s_load_dwordx2  [ttmp2, ttmp3], [ttmp4, ttmp5], 0x0 glc:1 // second-level TBA
+    s_waitcnt       lgkmcnt(0)
+    s_load_dwordx2  [ttmp4, ttmp5], [ttmp4, ttmp5], 0x8 glc:1 // second-level TMA
+    s_waitcnt       lgkmcnt(0)
+    s_and_b64       [ttmp2, ttmp3], [ttmp2, ttmp3], [ttmp2, ttmp3]
+    s_cbranch_scc0  L_NO_NEXT_TRAP // second-level trap handler not been set
+    s_setpc_b64     [ttmp2, ttmp3] // jump to second-level trap handler
+
+L_NO_NEXT_TRAP:
+    s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+    s_and_b32	    s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCE_MASK // Check whether it is an exception
+    s_cbranch_scc1  L_EXCP_CASE	  // Exception, jump back to the shader program directly.
+    s_add_u32	    ttmp0, ttmp0, 4   // S_TRAP case, add 4 to ttmp0
+    s_addc_u32	ttmp1, ttmp1, 0
+L_EXCP_CASE:
+    s_and_b32	ttmp1, ttmp1, 0xFFFF
+
+    // Restore SQ_WAVE_IB_STS.
+    s_lshr_b32      ttmp2, ttmp11, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
+    s_and_b32       ttmp2, ttmp2, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
+    s_setreg_b32    hwreg(HW_REG_IB_STS), ttmp2
+
+    // Restore SQ_WAVE_STATUS.
+    s_and_b64       exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
+    s_and_b64       vcc, vcc, vcc    // Restore STATUS.VCCZ, not writable by s_setreg_b32
+    s_setreg_b32    hwreg(HW_REG_STATUS), s_save_status
+
+    s_rfe_b64       [ttmp0, ttmp1]
+end
+    // *********	End handling of non-CWSR traps	 *******************
+
+/**************************************************************************/
+/*			save routine					  */
+/**************************************************************************/
+
+L_SAVE:
+
+if G8SR_DEBUG_TIMESTAMP
+	s_memrealtime	s_g8sr_ts_save_s
+	s_waitcnt lgkmcnt(0)	     //FIXME, will cause xnack??
+end
+
+    s_and_b32	    s_save_pc_hi, s_save_pc_hi, 0x0000ffff    //pc[47:32]
+
+    s_mov_b32	    s_save_tmp, 0							    //clear saveCtx bit
+    s_setreg_b32    hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp	    //clear saveCtx bit
+
+    s_getreg_b32    s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_RCNT_SHIFT, SQ_WAVE_IB_STS_RCNT_SIZE)		    //save RCNT
+    s_lshl_b32	    s_save_tmp, s_save_tmp, S_SAVE_PC_HI_RCNT_SHIFT
+    s_or_b32	    s_save_pc_hi, s_save_pc_hi, s_save_tmp
+    s_getreg_b32    s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT, SQ_WAVE_IB_STS_FIRST_REPLAY_SIZE)   //save FIRST_REPLAY
+    s_lshl_b32	    s_save_tmp, s_save_tmp, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+    s_or_b32	    s_save_pc_hi, s_save_pc_hi, s_save_tmp
+    s_getreg_b32    s_save_tmp, hwreg(HW_REG_IB_STS)					    //clear RCNT and FIRST_REPLAY in IB_STS
+    s_and_b32	    s_save_tmp, s_save_tmp, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG
+
+    s_setreg_b32    hwreg(HW_REG_IB_STS), s_save_tmp
+
+    /*	    inform SPI the readiness and wait for SPI's go signal */
+    s_mov_b32	    s_save_exec_lo, exec_lo						    //save EXEC and use EXEC for the go signal from SPI
+    s_mov_b32	    s_save_exec_hi, exec_hi
+    s_mov_b64	    exec,   0x0								    //clear EXEC to get ready to receive
+
+if G8SR_DEBUG_TIMESTAMP
+	s_memrealtime  s_g8sr_ts_sq_save_msg
+	s_waitcnt lgkmcnt(0)
+end
+
+    if (EMU_RUN_HACK)
+
+    else
+	s_sendmsg   sendmsg(MSG_SAVEWAVE)  //send SPI a message and wait for SPI's write to EXEC
+    end
+
+    // Set SPI_PRIO=2 to avoid starving instruction fetch in the waves we're waiting for.
+    s_or_b32 s_save_tmp, s_save_status, (2 << SQ_WAVE_STATUS_SPI_PRIO_SHIFT)
+    s_setreg_b32 hwreg(HW_REG_STATUS), s_save_tmp
+
+  L_SLEEP:
+    s_sleep 0x2		       // sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause SQ hang, since the 7,8th wave could not get arbit to exec inst, while other waves are stuck into the sleep-loop and waiting for wrexec!=0
+
+    if (EMU_RUN_HACK)
+
+    else
+	s_cbranch_execz L_SLEEP
+    end
+
+if G8SR_DEBUG_TIMESTAMP
+	s_memrealtime  s_g8sr_ts_spi_wrexec
+	s_waitcnt lgkmcnt(0)
+end
+
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_SINGLE_WAVE))
+	//calculate wd_addr using absolute thread id
+	v_readlane_b32 s_save_tmp, v9, 0
+	s_lshr_b32 s_save_tmp, s_save_tmp, 6
+	s_mul_i32 s_save_tmp, s_save_tmp, WAVE_SPACE
+	s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+	s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+	s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL
+    else
+    end
+    if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_SINGLE_WAVE))
+	s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+	s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+	s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL
+    else
+    end
+
+    // Save trap temporaries 6-11, 13-15 initialized by SPI debug dispatch logic
+    // ttmp SR memory offset : size(VGPR)+size(SGPR)+0x40
+    get_vgpr_size_bytes(s_save_ttmps_lo)
+    get_sgpr_size_bytes(s_save_ttmps_hi)
+    s_add_u32	    s_save_ttmps_lo, s_save_ttmps_lo, s_save_ttmps_hi
+    s_add_u32	    s_save_ttmps_lo, s_save_ttmps_lo, s_save_spi_init_lo
+    s_addc_u32	    s_save_ttmps_hi, s_save_spi_init_hi, 0x0
+    s_and_b32	    s_save_ttmps_hi, s_save_ttmps_hi, 0xFFFF
+    s_store_dwordx2 [ttmp6, ttmp7], [s_save_ttmps_lo, s_save_ttmps_hi], 0x40 glc:1
+    ack_sqc_store_workaround()
+    s_store_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_save_ttmps_lo, s_save_ttmps_hi], 0x48 glc:1
+    ack_sqc_store_workaround()
+    s_store_dword   ttmp13, [s_save_ttmps_lo, s_save_ttmps_hi], 0x58 glc:1
+    ack_sqc_store_workaround()
+    s_store_dwordx2 [ttmp14, ttmp15], [s_save_ttmps_lo, s_save_ttmps_hi], 0x5C glc:1
+    ack_sqc_store_workaround()
+
+    /*	    setup Resource Contants    */
+    s_mov_b32	    s_save_buf_rsrc0,	s_save_spi_init_lo							//base_addr_lo
+    s_and_b32	    s_save_buf_rsrc1,	s_save_spi_init_hi, 0x0000FFFF						//base_addr_hi
+    s_or_b32	    s_save_buf_rsrc1,	s_save_buf_rsrc1,  S_SAVE_BUF_RSRC_WORD1_STRIDE
+    s_mov_b32	    s_save_buf_rsrc2,	0									//NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited
+    s_mov_b32	    s_save_buf_rsrc3,	S_SAVE_BUF_RSRC_WORD3_MISC
+    s_and_b32	    s_save_tmp,		s_save_spi_init_hi, S_SAVE_SPI_INIT_ATC_MASK
+    s_lshr_b32	    s_save_tmp,		s_save_tmp, (S_SAVE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT)	    //get ATC bit into position
+    s_or_b32	    s_save_buf_rsrc3,	s_save_buf_rsrc3,  s_save_tmp						//or ATC
+    s_and_b32	    s_save_tmp,		s_save_spi_init_hi, S_SAVE_SPI_INIT_MTYPE_MASK
+    s_lshr_b32	    s_save_tmp,		s_save_tmp, (S_SAVE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT)	    //get MTYPE bits into position
+    s_or_b32	    s_save_buf_rsrc3,	s_save_buf_rsrc3,  s_save_tmp						//or MTYPE
+
+    //FIXME  right now s_save_m0/s_save_mem_offset use tma_lo/tma_hi  (might need to save them before using them?)
+    s_mov_b32	    s_save_m0,		m0								    //save M0
+
+    /*	    global mem offset		*/
+    s_mov_b32	    s_save_mem_offset,	0x0									//mem offset initial value = 0
+
+
+
+
+    /*	    save HW registers	*/
+    //////////////////////////////
+
+  L_SAVE_HWREG:
+	// HWREG SR memory offset : size(VGPR)+size(SGPR)
+       get_vgpr_size_bytes(s_save_mem_offset)
+       get_sgpr_size_bytes(s_save_tmp)
+       s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
+
+
+    s_mov_b32	    s_save_buf_rsrc2, 0x4				//NUM_RECORDS	in bytes
+    if (SWIZZLE_EN)
+	s_add_u32	s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0			    //FIXME need to use swizzle to enable bounds checking?
+    else
+	s_mov_b32	s_save_buf_rsrc2,  0x1000000				    //NUM_RECORDS in bytes
+    end
+
+
+    write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)			//M0
+
+    if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_FIRST_TIME))
+	s_add_u32 s_save_pc_lo, s_save_pc_lo, 4		    //pc[31:0]+4
+	s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0	    //carry bit over
+    end
+
+    write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset)		    //PC
+    write_hwreg_to_mem(s_save_pc_hi, s_save_buf_rsrc0, s_save_mem_offset)
+    write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset)		//EXEC
+    write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)
+    write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset)		//STATUS
+
+    //s_save_trapsts conflicts with s_save_alloc_size
+    s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+    write_hwreg_to_mem(s_save_trapsts, s_save_buf_rsrc0, s_save_mem_offset)		//TRAPSTS
+
+    write_hwreg_to_mem(xnack_mask_lo, s_save_buf_rsrc0, s_save_mem_offset)	    //XNACK_MASK_LO
+    write_hwreg_to_mem(xnack_mask_hi, s_save_buf_rsrc0, s_save_mem_offset)	    //XNACK_MASK_HI
+
+    //use s_save_tmp would introduce conflict here between s_save_tmp and s_save_buf_rsrc2
+    s_getreg_b32    s_save_m0, hwreg(HW_REG_MODE)						    //MODE
+    write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
+
+
+
+    /*	    the first wave in the threadgroup	 */
+    s_and_b32	    s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK	// extract fisrt wave bit
+    s_mov_b32	     s_save_exec_hi, 0x0
+    s_or_b32	     s_save_exec_hi, s_save_tmp, s_save_exec_hi				 // save first wave bit to s_save_exec_hi.bits[26]
+
+
+    /*		save SGPRs	*/
+	// Save SGPR before LDS save, then the s0 to s4 can be used during LDS save...
+    //////////////////////////////
+
+    // SGPR SR memory offset : size(VGPR)
+    get_vgpr_size_bytes(s_save_mem_offset)
+    // TODO, change RSRC word to rearrange memory layout for SGPRS
+
+    s_getreg_b32    s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE)		//spgr_size
+    s_add_u32	    s_save_alloc_size, s_save_alloc_size, 1
+    s_lshl_b32	    s_save_alloc_size, s_save_alloc_size, 4			    //Number of SGPRs = (sgpr_size + 1) * 16   (non-zero value)
+
+    if (SGPR_SAVE_USE_SQC)
+	s_lshl_b32	s_save_buf_rsrc2,   s_save_alloc_size, 2		    //NUM_RECORDS in bytes
+    else
+	s_lshl_b32	s_save_buf_rsrc2,   s_save_alloc_size, 8		    //NUM_RECORDS in bytes (64 threads)
+    end
+
+    if (SWIZZLE_EN)
+	s_add_u32	s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0			    //FIXME need to use swizzle to enable bounds checking?
+    else
+	s_mov_b32	s_save_buf_rsrc2,  0x1000000				    //NUM_RECORDS in bytes
+    end
+
+
+    // backup s_save_buf_rsrc0,1 to s_save_pc_lo/hi, since write_16sgpr_to_mem function will change the rsrc0
+    //s_mov_b64 s_save_pc_lo, s_save_buf_rsrc0
+    s_mov_b64 s_save_xnack_mask_lo, s_save_buf_rsrc0
+    s_add_u32 s_save_buf_rsrc0, s_save_buf_rsrc0, s_save_mem_offset
+    s_addc_u32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0
+
+    s_mov_b32	    m0, 0x0			    //SGPR initial index value =0
+    s_nop	    0x0				    //Manually inserted wait states
+  L_SAVE_SGPR_LOOP:
+    // SGPR is allocated in 16 SGPR granularity
+    s_movrels_b64   s0, s0     //s0 = s[0+m0], s1 = s[1+m0]
+    s_movrels_b64   s2, s2     //s2 = s[2+m0], s3 = s[3+m0]
+    s_movrels_b64   s4, s4     //s4 = s[4+m0], s5 = s[5+m0]
+    s_movrels_b64   s6, s6     //s6 = s[6+m0], s7 = s[7+m0]
+    s_movrels_b64   s8, s8     //s8 = s[8+m0], s9 = s[9+m0]
+    s_movrels_b64   s10, s10   //s10 = s[10+m0], s11 = s[11+m0]
+    s_movrels_b64   s12, s12   //s12 = s[12+m0], s13 = s[13+m0]
+    s_movrels_b64   s14, s14   //s14 = s[14+m0], s15 = s[15+m0]
+
+    write_16sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset) //PV: the best performance should be using s_buffer_store_dwordx4
+    s_add_u32	    m0, m0, 16							    //next sgpr index
+    s_cmp_lt_u32    m0, s_save_alloc_size					    //scc = (m0 < s_save_alloc_size) ? 1 : 0
+    s_cbranch_scc1  L_SAVE_SGPR_LOOP					//SGPR save is complete?
+    // restore s_save_buf_rsrc0,1
+    //s_mov_b64 s_save_buf_rsrc0, s_save_pc_lo
+    s_mov_b64 s_save_buf_rsrc0, s_save_xnack_mask_lo
+
+
+
+
+    /*		save first 4 VGPR, then LDS save could use   */
+	// each wave will alloc 4 vgprs at least...
+    /////////////////////////////////////////////////////////////////////////////////////
+
+    s_mov_b32	    s_save_mem_offset, 0
+    s_mov_b32	    exec_lo, 0xFFFFFFFF						    //need every thread from now on
+    s_mov_b32	    exec_hi, 0xFFFFFFFF
+    s_mov_b32	    xnack_mask_lo, 0x0
+    s_mov_b32	    xnack_mask_hi, 0x0
+
+    if (SWIZZLE_EN)
+	s_add_u32	s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0			    //FIXME need to use swizzle to enable bounds checking?
+    else
+	s_mov_b32	s_save_buf_rsrc2,  0x1000000				    //NUM_RECORDS in bytes
+    end
+
+
+    // VGPR Allocated in 4-GPR granularity
+
+if G8SR_VGPR_SR_IN_DWX4
+	// the const stride for DWx4 is 4*4 bytes
+	s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+	s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4  // const stride to 4*4 bytes
+
+	buffer_store_dwordx4 v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+
+	s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+	s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE  // reset const stride to 4 bytes
+else
+	buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+	buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256
+	buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*2
+	buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*3
+end
+
+
+
+    /*		save LDS	*/
+    //////////////////////////////
+
+  L_SAVE_LDS:
+
+	// Change EXEC to all threads...
+    s_mov_b32	    exec_lo, 0xFFFFFFFF	  //need every thread from now on
+    s_mov_b32	    exec_hi, 0xFFFFFFFF
+
+    s_getreg_b32    s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)		    //lds_size
+    s_and_b32	    s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF		    //lds_size is zero?
+    s_cbranch_scc0  L_SAVE_LDS_DONE									       //no lds used? jump to L_SAVE_DONE
+
+    s_barrier		    //LDS is used? wait for other waves in the same TG
+    s_and_b32	    s_save_tmp, s_save_exec_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK		       //exec is still used here
+    s_cbranch_scc0  L_SAVE_LDS_DONE
+
+	// first wave do LDS save;
+
+    s_lshl_b32	    s_save_alloc_size, s_save_alloc_size, 6			    //LDS size in dwords = lds_size * 64dw
+    s_lshl_b32	    s_save_alloc_size, s_save_alloc_size, 2			    //LDS size in bytes
+    s_mov_b32	    s_save_buf_rsrc2,  s_save_alloc_size			    //NUM_RECORDS in bytes
+
+    // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
+    //
+    get_vgpr_size_bytes(s_save_mem_offset)
+    get_sgpr_size_bytes(s_save_tmp)
+    s_add_u32  s_save_mem_offset, s_save_mem_offset, s_save_tmp
+    s_add_u32 s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes()
+
+
+    if (SWIZZLE_EN)
+	s_add_u32	s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0	      //FIXME need to use swizzle to enable bounds checking?
+    else
+	s_mov_b32	s_save_buf_rsrc2,  0x1000000		      //NUM_RECORDS in bytes
+    end
+
+    s_mov_b32	    m0, 0x0						  //lds_offset initial value = 0
+
+
+var LDS_DMA_ENABLE = 0
+var UNROLL = 0
+if UNROLL==0 && LDS_DMA_ENABLE==1
+	s_mov_b32  s3, 256*2
+	s_nop 0
+	s_nop 0
+	s_nop 0
+  L_SAVE_LDS_LOOP:
+	//TODO: looks the 2 buffer_store/load clause for s/r will hurt performance.???
+    if (SAVE_LDS)     //SPI always alloc LDS space in 128DW granularity
+	    buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1		// first 64DW
+	    buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:256 // second 64DW
+    end
+
+    s_add_u32	    m0, m0, s3						//every buffer_store_lds does 256 bytes
+    s_add_u32	    s_save_mem_offset, s_save_mem_offset, s3				//mem offset increased by 256 bytes
+    s_cmp_lt_u32    m0, s_save_alloc_size						//scc=(m0 < s_save_alloc_size) ? 1 : 0
+    s_cbranch_scc1  L_SAVE_LDS_LOOP							//LDS save is complete?
+
+elsif LDS_DMA_ENABLE==1 && UNROLL==1 // UNROOL	, has ichace miss
+      // store from higest LDS address to lowest
+      s_mov_b32	 s3, 256*2
+      s_sub_u32	 m0, s_save_alloc_size, s3
+      s_add_u32 s_save_mem_offset, s_save_mem_offset, m0
+      s_lshr_b32 s_save_alloc_size, s_save_alloc_size, 9   // how many 128 trunks...
+      s_sub_u32 s_save_alloc_size, 128, s_save_alloc_size   // store from higheset addr to lowest
+      s_mul_i32 s_save_alloc_size, s_save_alloc_size, 6*4   // PC offset increment,  each LDS save block cost 6*4 Bytes instruction
+      s_add_u32 s_save_alloc_size, s_save_alloc_size, 3*4   //2is the below 2 inst...//s_addc and s_setpc
+      s_nop 0
+      s_nop 0
+      s_nop 0	//pad 3 dw to let LDS_DMA align with 64Bytes
+      s_getpc_b64 s[0:1]			      // reuse s[0:1], since s[0:1] already saved
+      s_add_u32	  s0, s0,s_save_alloc_size
+      s_addc_u32  s1, s1, 0
+      s_setpc_b64 s[0:1]
+
+
+       for var i =0; i< 128; i++
+	    // be careful to make here a 64Byte aligned address, which could improve performance...
+	    buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:0		// first 64DW
+	    buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:256		  // second 64DW
+
+	if i!=127
+	s_sub_u32  m0, m0, s3	   // use a sgpr to shrink 2DW-inst to 1DW inst to improve performance , i.e.  pack more LDS_DMA inst to one Cacheline
+	    s_sub_u32  s_save_mem_offset, s_save_mem_offset,  s3
+	    end
+       end
+
+else   // BUFFER_STORE
+      v_mbcnt_lo_u32_b32 v2, 0xffffffff, 0x0
+      v_mbcnt_hi_u32_b32 v3, 0xffffffff, v2	// tid
+      v_mul_i32_i24 v2, v3, 8	// tid*8
+      v_mov_b32 v3, 256*2
+      s_mov_b32 m0, 0x10000
+      s_mov_b32 s0, s_save_buf_rsrc3
+      s_and_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0xFF7FFFFF	  // disable add_tid
+      s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0x58000   //DFMT
+
+L_SAVE_LDS_LOOP_VECTOR:
+      ds_read_b64 v[0:1], v2	//x =LDS[a], byte address
+      s_waitcnt lgkmcnt(0)
+      buffer_store_dwordx2  v[0:1], v2, s_save_buf_rsrc0, s_save_mem_offset offen:1  glc:1  slc:1
+//	s_waitcnt vmcnt(0)
+//	v_add_u32 v2, vcc[0:1], v2, v3
+      v_add_u32 v2, v2, v3
+      v_cmp_lt_u32 vcc[0:1], v2, s_save_alloc_size
+      s_cbranch_vccnz L_SAVE_LDS_LOOP_VECTOR
+
+      // restore rsrc3
+      s_mov_b32 s_save_buf_rsrc3, s0
+
+end
+
+L_SAVE_LDS_DONE:
+
+
+    /*		save VGPRs  - set the Rest VGPRs	*/
+    //////////////////////////////////////////////////////////////////////////////////////
+  L_SAVE_VGPR:
+    // VGPR SR memory offset: 0
+    // TODO rearrange the RSRC words to use swizzle for VGPR save...
+
+    s_mov_b32	    s_save_mem_offset, (0+256*4)				    // for the rest VGPRs
+    s_mov_b32	    exec_lo, 0xFFFFFFFF						    //need every thread from now on
+    s_mov_b32	    exec_hi, 0xFFFFFFFF
+
+    s_getreg_b32    s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)		    //vpgr_size
+    s_add_u32	    s_save_alloc_size, s_save_alloc_size, 1
+    s_lshl_b32	    s_save_alloc_size, s_save_alloc_size, 2			    //Number of VGPRs = (vgpr_size + 1) * 4    (non-zero value)	  //FIXME for GFX, zero is possible
+    s_lshl_b32	    s_save_buf_rsrc2,  s_save_alloc_size, 8			    //NUM_RECORDS in bytes (64 threads*4)
+    if (SWIZZLE_EN)
+	s_add_u32	s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0			    //FIXME need to use swizzle to enable bounds checking?
+    else
+	s_mov_b32	s_save_buf_rsrc2,  0x1000000				    //NUM_RECORDS in bytes
+    end
+
+
+    // VGPR Allocated in 4-GPR granularity
+
+if G8SR_VGPR_SR_IN_DWX4
+	// the const stride for DWx4 is 4*4 bytes
+	s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+	s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4  // const stride to 4*4 bytes
+
+	s_mov_b32	  m0, 4	    // skip first 4 VGPRs
+	s_cmp_lt_u32	  m0, s_save_alloc_size
+	s_cbranch_scc0	  L_SAVE_VGPR_LOOP_END	    // no more vgprs
+
+	s_set_gpr_idx_on  m0, 0x1   // This will change M0
+	s_add_u32	  s_save_alloc_size, s_save_alloc_size, 0x1000	// because above inst change m0
+L_SAVE_VGPR_LOOP:
+	v_mov_b32	  v0, v0   // v0 = v[0+m0]
+	v_mov_b32	  v1, v1
+	v_mov_b32	  v2, v2
+	v_mov_b32	  v3, v3
+
+
+	buffer_store_dwordx4 v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+	s_add_u32	  m0, m0, 4
+	s_add_u32	  s_save_mem_offset, s_save_mem_offset, 256*4
+	s_cmp_lt_u32	  m0, s_save_alloc_size
+    s_cbranch_scc1  L_SAVE_VGPR_LOOP						    //VGPR save is complete?
+    s_set_gpr_idx_off
+L_SAVE_VGPR_LOOP_END:
+
+	s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+	s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE  // reset const stride to 4 bytes
+else
+    // VGPR store using dw burst
+    s_mov_b32	      m0, 0x4	//VGPR initial index value =0
+    s_cmp_lt_u32      m0, s_save_alloc_size
+    s_cbranch_scc0    L_SAVE_VGPR_END
+
+
+    s_set_gpr_idx_on	m0, 0x1 //M0[7:0] = M0[7:0] and M0[15:12] = 0x1
+    s_add_u32	    s_save_alloc_size, s_save_alloc_size, 0x1000		    //add 0x1000 since we compare m0 against it later
+
+  L_SAVE_VGPR_LOOP:
+    v_mov_b32	    v0, v0		//v0 = v[0+m0]
+    v_mov_b32	    v1, v1		//v0 = v[0+m0]
+    v_mov_b32	    v2, v2		//v0 = v[0+m0]
+    v_mov_b32	    v3, v3		//v0 = v[0+m0]
+
+    if(USE_MTBUF_INSTEAD_OF_MUBUF)
+	tbuffer_store_format_x v0, v0, s_save_buf_rsrc0, s_save_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+    else
+	buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+	buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256
+	buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*2
+	buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*3
+    end
+
+    s_add_u32	    m0, m0, 4							    //next vgpr index
+    s_add_u32	    s_save_mem_offset, s_save_mem_offset, 256*4			    //every buffer_store_dword does 256 bytes
+    s_cmp_lt_u32    m0, s_save_alloc_size					    //scc = (m0 < s_save_alloc_size) ? 1 : 0
+    s_cbranch_scc1  L_SAVE_VGPR_LOOP						    //VGPR save is complete?
+    s_set_gpr_idx_off
+end
+
+L_SAVE_VGPR_END:
+
+
+
+
+
+
+    /*	   S_PGM_END_SAVED  */				    //FIXME  graphics ONLY
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_NORMAL_EXIT))
+	s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff    //pc[47:32]
+	s_add_u32 s_save_pc_lo, s_save_pc_lo, 4		    //pc[31:0]+4
+	s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0	    //carry bit over
+	s_rfe_b64 s_save_pc_lo				    //Return to the main shader program
+    else
+    end
+
+// Save Done timestamp
+if G8SR_DEBUG_TIMESTAMP
+	s_memrealtime	s_g8sr_ts_save_d
+	// SGPR SR memory offset : size(VGPR)
+	get_vgpr_size_bytes(s_save_mem_offset)
+	s_add_u32 s_save_mem_offset, s_save_mem_offset, G8SR_DEBUG_TS_SAVE_D_OFFSET
+	s_waitcnt lgkmcnt(0)	     //FIXME, will cause xnack??
+	// Need reset rsrc2??
+	s_mov_b32 m0, s_save_mem_offset
+	s_mov_b32 s_save_buf_rsrc2,  0x1000000					//NUM_RECORDS in bytes
+	s_buffer_store_dwordx2 s_g8sr_ts_save_d, s_save_buf_rsrc0, m0	    glc:1
+end
+
+
+    s_branch	L_END_PGM
+
+
+
+/**************************************************************************/
+/*			restore routine					  */
+/**************************************************************************/
+
+L_RESTORE:
+    /*	    Setup Resource Contants    */
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+	//calculate wd_addr using absolute thread id
+	v_readlane_b32 s_restore_tmp, v9, 0
+	s_lshr_b32 s_restore_tmp, s_restore_tmp, 6
+	s_mul_i32 s_restore_tmp, s_restore_tmp, WAVE_SPACE
+	s_add_i32 s_restore_spi_init_lo, s_restore_tmp, WG_BASE_ADDR_LO
+	s_mov_b32 s_restore_spi_init_hi, WG_BASE_ADDR_HI
+	s_and_b32 s_restore_spi_init_hi, s_restore_spi_init_hi, CTX_RESTORE_CONTROL
+    else
+    end
+
+if G8SR_DEBUG_TIMESTAMP
+	s_memrealtime	s_g8sr_ts_restore_s
+	s_waitcnt lgkmcnt(0)	     //FIXME, will cause xnack??
+	// tma_lo/hi are sgpr 110, 111, which will not used for 112 SGPR allocated case...
+	s_mov_b32 s_restore_pc_lo, s_g8sr_ts_restore_s[0]
+	s_mov_b32 s_restore_pc_hi, s_g8sr_ts_restore_s[1]   //backup ts to ttmp0/1, sicne exec will be finally restored..
+end
+
+
+
+    s_mov_b32	    s_restore_buf_rsrc0,    s_restore_spi_init_lo							    //base_addr_lo
+    s_and_b32	    s_restore_buf_rsrc1,    s_restore_spi_init_hi, 0x0000FFFF						    //base_addr_hi
+    s_or_b32	    s_restore_buf_rsrc1,    s_restore_buf_rsrc1,  S_RESTORE_BUF_RSRC_WORD1_STRIDE
+    s_mov_b32	    s_restore_buf_rsrc2,    0										    //NUM_RECORDS initial value = 0 (in bytes)
+    s_mov_b32	    s_restore_buf_rsrc3,    S_RESTORE_BUF_RSRC_WORD3_MISC
+    s_and_b32	    s_restore_tmp,	    s_restore_spi_init_hi, S_RESTORE_SPI_INIT_ATC_MASK
+    s_lshr_b32	    s_restore_tmp,	    s_restore_tmp, (S_RESTORE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT)	    //get ATC bit into position
+    s_or_b32	    s_restore_buf_rsrc3,    s_restore_buf_rsrc3,  s_restore_tmp						    //or ATC
+    s_and_b32	    s_restore_tmp,	    s_restore_spi_init_hi, S_RESTORE_SPI_INIT_MTYPE_MASK
+    s_lshr_b32	    s_restore_tmp,	    s_restore_tmp, (S_RESTORE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT)   //get MTYPE bits into position
+    s_or_b32	    s_restore_buf_rsrc3,    s_restore_buf_rsrc3,  s_restore_tmp						    //or MTYPE
+
+    /*	    global mem offset		*/
+//  s_mov_b32	    s_restore_mem_offset, 0x0				    //mem offset initial value = 0
+
+    /*	    the first wave in the threadgroup	 */
+    s_and_b32	    s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK
+    s_cbranch_scc0  L_RESTORE_VGPR
+
+    /*		restore LDS	*/
+    //////////////////////////////
+  L_RESTORE_LDS:
+
+    s_mov_b32	    exec_lo, 0xFFFFFFFF							    //need every thread from now on   //be consistent with SAVE although can be moved ahead
+    s_mov_b32	    exec_hi, 0xFFFFFFFF
+
+    s_getreg_b32    s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)		//lds_size
+    s_and_b32	    s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF		    //lds_size is zero?
+    s_cbranch_scc0  L_RESTORE_VGPR							    //no lds used? jump to L_RESTORE_VGPR
+    s_lshl_b32	    s_restore_alloc_size, s_restore_alloc_size, 6			    //LDS size in dwords = lds_size * 64dw
+    s_lshl_b32	    s_restore_alloc_size, s_restore_alloc_size, 2			    //LDS size in bytes
+    s_mov_b32	    s_restore_buf_rsrc2,    s_restore_alloc_size			    //NUM_RECORDS in bytes
+
+    // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
+    //
+    get_vgpr_size_bytes(s_restore_mem_offset)
+    get_sgpr_size_bytes(s_restore_tmp)
+    s_add_u32  s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+    s_add_u32  s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes()	     //FIXME, Check if offset overflow???
+
+
+    if (SWIZZLE_EN)
+	s_add_u32	s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0			    //FIXME need to use swizzle to enable bounds checking?
+    else
+	s_mov_b32	s_restore_buf_rsrc2,  0x1000000					    //NUM_RECORDS in bytes
+    end
+    s_mov_b32	    m0, 0x0								    //lds_offset initial value = 0
+
+  L_RESTORE_LDS_LOOP:
+    if (SAVE_LDS)
+	buffer_load_dword   v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1		       // first 64DW
+	buffer_load_dword   v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 offset:256	       // second 64DW
+    end
+    s_add_u32	    m0, m0, 256*2						// 128 DW
+    s_add_u32	    s_restore_mem_offset, s_restore_mem_offset, 256*2		//mem offset increased by 128DW
+    s_cmp_lt_u32    m0, s_restore_alloc_size					//scc=(m0 < s_restore_alloc_size) ? 1 : 0
+    s_cbranch_scc1  L_RESTORE_LDS_LOOP							    //LDS restore is complete?
+
+
+    /*		restore VGPRs	    */
+    //////////////////////////////
+  L_RESTORE_VGPR:
+	// VGPR SR memory offset : 0
+    s_mov_b32	    s_restore_mem_offset, 0x0
+    s_mov_b32	    exec_lo, 0xFFFFFFFF							    //need every thread from now on   //be consistent with SAVE although can be moved ahead
+    s_mov_b32	    exec_hi, 0xFFFFFFFF
+
+    s_getreg_b32    s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)	//vpgr_size
+    s_add_u32	    s_restore_alloc_size, s_restore_alloc_size, 1
+    s_lshl_b32	    s_restore_alloc_size, s_restore_alloc_size, 2			    //Number of VGPRs = (vgpr_size + 1) * 4    (non-zero value)
+    s_lshl_b32	    s_restore_buf_rsrc2,  s_restore_alloc_size, 8			    //NUM_RECORDS in bytes (64 threads*4)
+    if (SWIZZLE_EN)
+	s_add_u32	s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0			    //FIXME need to use swizzle to enable bounds checking?
+    else
+	s_mov_b32	s_restore_buf_rsrc2,  0x1000000					    //NUM_RECORDS in bytes
+    end
+
+if G8SR_VGPR_SR_IN_DWX4
+     get_vgpr_size_bytes(s_restore_mem_offset)
+     s_sub_u32	       s_restore_mem_offset, s_restore_mem_offset, 256*4
+
+     // the const stride for DWx4 is 4*4 bytes
+     s_and_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+     s_or_b32  s_restore_buf_rsrc1, s_restore_buf_rsrc1, G8SR_RESTORE_BUF_RSRC_WORD1_STRIDE_DWx4  // const stride to 4*4 bytes
+
+     s_mov_b32	       m0, s_restore_alloc_size
+     s_set_gpr_idx_on  m0, 0x8	  // Note.. This will change m0
+
+L_RESTORE_VGPR_LOOP:
+     buffer_load_dwordx4 v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+     s_waitcnt vmcnt(0)
+     s_sub_u32	       m0, m0, 4
+     v_mov_b32	       v0, v0	// v[0+m0] = v0
+     v_mov_b32	       v1, v1
+     v_mov_b32	       v2, v2
+     v_mov_b32	       v3, v3
+     s_sub_u32	       s_restore_mem_offset, s_restore_mem_offset, 256*4
+     s_cmp_eq_u32      m0, 0x8000
+     s_cbranch_scc0    L_RESTORE_VGPR_LOOP
+     s_set_gpr_idx_off
+
+     s_and_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+     s_or_b32  s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE  // const stride to 4*4 bytes
+
+else
+    // VGPR load using dw burst
+    s_mov_b32	    s_restore_mem_offset_save, s_restore_mem_offset	// restore start with v1, v0 will be the last
+    s_add_u32	    s_restore_mem_offset, s_restore_mem_offset, 256*4
+    s_mov_b32	    m0, 4				//VGPR initial index value = 1
+    s_set_gpr_idx_on  m0, 0x8			    //M0[7:0] = M0[7:0] and M0[15:12] = 0x8
+    s_add_u32	    s_restore_alloc_size, s_restore_alloc_size, 0x8000			    //add 0x8000 since we compare m0 against it later
+
+  L_RESTORE_VGPR_LOOP:
+    if(USE_MTBUF_INSTEAD_OF_MUBUF)
+	tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+    else
+	buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+	buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256
+	buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*2
+	buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*3
+    end
+    s_waitcnt	    vmcnt(0)								    //ensure data ready
+    v_mov_b32	    v0, v0								    //v[0+m0] = v0
+    v_mov_b32	    v1, v1
+    v_mov_b32	    v2, v2
+    v_mov_b32	    v3, v3
+    s_add_u32	    m0, m0, 4								    //next vgpr index
+    s_add_u32	    s_restore_mem_offset, s_restore_mem_offset, 256*4				//every buffer_load_dword does 256 bytes
+    s_cmp_lt_u32    m0, s_restore_alloc_size						    //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+    s_cbranch_scc1  L_RESTORE_VGPR_LOOP							    //VGPR restore (except v0) is complete?
+    s_set_gpr_idx_off
+											    /* VGPR restore on v0 */
+    if(USE_MTBUF_INSTEAD_OF_MUBUF)
+	tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+    else
+	buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1
+	buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1 offset:256
+	buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1 offset:256*2
+	buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1 offset:256*3
+    end
+
+end
+
+    /*		restore SGPRs	    */
+    //////////////////////////////
+
+    // SGPR SR memory offset : size(VGPR)
+    get_vgpr_size_bytes(s_restore_mem_offset)
+    get_sgpr_size_bytes(s_restore_tmp)
+    s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+    s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 16*4	   // restore SGPR from S[n] to S[0], by 16 sgprs group
+    // TODO, change RSRC word to rearrange memory layout for SGPRS
+
+    s_getreg_b32    s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE)		    //spgr_size
+    s_add_u32	    s_restore_alloc_size, s_restore_alloc_size, 1
+    s_lshl_b32	    s_restore_alloc_size, s_restore_alloc_size, 4			    //Number of SGPRs = (sgpr_size + 1) * 16   (non-zero value)
+
+    if (SGPR_SAVE_USE_SQC)
+	s_lshl_b32	s_restore_buf_rsrc2,	s_restore_alloc_size, 2			    //NUM_RECORDS in bytes
+    else
+	s_lshl_b32	s_restore_buf_rsrc2,	s_restore_alloc_size, 8			    //NUM_RECORDS in bytes (64 threads)
+    end
+    if (SWIZZLE_EN)
+	s_add_u32	s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0			    //FIXME need to use swizzle to enable bounds checking?
+    else
+	s_mov_b32	s_restore_buf_rsrc2,  0x1000000					    //NUM_RECORDS in bytes
+    end
+
+    s_mov_b32 m0, s_restore_alloc_size
+
+ L_RESTORE_SGPR_LOOP:
+    read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)	 //PV: further performance improvement can be made
+    s_waitcnt	    lgkmcnt(0)								    //ensure data ready
+
+    s_sub_u32 m0, m0, 16    // Restore from S[n] to S[0]
+    s_nop 0 // hazard SALU M0=> S_MOVREL
+
+    s_movreld_b64   s0, s0	//s[0+m0] = s0
+    s_movreld_b64   s2, s2
+    s_movreld_b64   s4, s4
+    s_movreld_b64   s6, s6
+    s_movreld_b64   s8, s8
+    s_movreld_b64   s10, s10
+    s_movreld_b64   s12, s12
+    s_movreld_b64   s14, s14
+
+    s_cmp_eq_u32    m0, 0		//scc = (m0 < s_restore_alloc_size) ? 1 : 0
+    s_cbranch_scc0  L_RESTORE_SGPR_LOOP		    //SGPR restore (except s0) is complete?
+
+    /*	    restore HW registers    */
+    //////////////////////////////
+  L_RESTORE_HWREG:
+
+
+if G8SR_DEBUG_TIMESTAMP
+      s_mov_b32 s_g8sr_ts_restore_s[0], s_restore_pc_lo
+      s_mov_b32 s_g8sr_ts_restore_s[1], s_restore_pc_hi
+end
+
+    // HWREG SR memory offset : size(VGPR)+size(SGPR)
+    get_vgpr_size_bytes(s_restore_mem_offset)
+    get_sgpr_size_bytes(s_restore_tmp)
+    s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+
+
+    s_mov_b32	    s_restore_buf_rsrc2, 0x4						    //NUM_RECORDS   in bytes
+    if (SWIZZLE_EN)
+	s_add_u32	s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0			    //FIXME need to use swizzle to enable bounds checking?
+    else
+	s_mov_b32	s_restore_buf_rsrc2,  0x1000000					    //NUM_RECORDS in bytes
+    end
+
+    read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset)		    //M0
+    read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset)		//PC
+    read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
+    read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset)		    //EXEC
+    read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
+    read_hwreg_from_mem(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset)		    //STATUS
+    read_hwreg_from_mem(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset)		    //TRAPSTS
+    read_hwreg_from_mem(xnack_mask_lo, s_restore_buf_rsrc0, s_restore_mem_offset)		    //XNACK_MASK_LO
+    read_hwreg_from_mem(xnack_mask_hi, s_restore_buf_rsrc0, s_restore_mem_offset)		    //XNACK_MASK_HI
+    read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset)		//MODE
+
+    s_waitcnt	    lgkmcnt(0)											    //from now on, it is safe to restore STATUS and IB_STS
+
+    //for normal save & restore, the saved PC points to the next inst to execute, no adjustment needs to be made, otherwise:
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+	s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 8		 //pc[31:0]+8	  //two back-to-back s_trap are used (first for save and second for restore)
+	s_addc_u32  s_restore_pc_hi, s_restore_pc_hi, 0x0	 //carry bit over
+    end
+    if ((EMU_RUN_HACK) && (EMU_RUN_HACK_RESTORE_NORMAL))
+	s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 4		 //pc[31:0]+4	  // save is hack through s_trap but restore is normal
+	s_addc_u32  s_restore_pc_hi, s_restore_pc_hi, 0x0	 //carry bit over
+    end
+
+    s_mov_b32	    m0,		s_restore_m0
+    s_mov_b32	    exec_lo,	s_restore_exec_lo
+    s_mov_b32	    exec_hi,	s_restore_exec_hi
+
+    s_and_b32	    s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts
+    s_setreg_b32    hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0
+    s_and_b32	    s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts
+    s_lshr_b32	    s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT
+    s_setreg_b32    hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0
+    //s_setreg_b32  hwreg(HW_REG_TRAPSTS),  s_restore_trapsts	   //don't overwrite SAVECTX bit as it may be set through external SAVECTX during restore
+    s_setreg_b32    hwreg(HW_REG_MODE),	    s_restore_mode
+
+    // Restore trap temporaries 6-11, 13-15 initialized by SPI debug dispatch logic
+    // ttmp SR memory offset : size(VGPR)+size(SGPR)+0x40
+    get_vgpr_size_bytes(s_restore_ttmps_lo)
+    get_sgpr_size_bytes(s_restore_ttmps_hi)
+    s_add_u32	    s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_ttmps_hi
+    s_add_u32	    s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_buf_rsrc0
+    s_addc_u32	    s_restore_ttmps_hi, s_restore_buf_rsrc1, 0x0
+    s_and_b32	    s_restore_ttmps_hi, s_restore_ttmps_hi, 0xFFFF
+    s_load_dwordx2  [ttmp6, ttmp7], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x40 glc:1
+    s_load_dwordx4  [ttmp8, ttmp9, ttmp10, ttmp11], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x48 glc:1
+    s_load_dword    ttmp13, [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x58 glc:1
+    s_load_dwordx2  [ttmp14, ttmp15], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x5C glc:1
+    s_waitcnt	    lgkmcnt(0)
+
+    //reuse s_restore_m0 as a temp register
+    s_and_b32	    s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_RCNT_MASK
+    s_lshr_b32	    s_restore_m0, s_restore_m0, S_SAVE_PC_HI_RCNT_SHIFT
+    s_lshl_b32	    s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_RCNT_SHIFT
+    s_mov_b32	    s_restore_tmp, 0x0										    //IB_STS is zero
+    s_or_b32	    s_restore_tmp, s_restore_tmp, s_restore_m0
+    s_and_b32	    s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_FIRST_REPLAY_MASK
+    s_lshr_b32	    s_restore_m0, s_restore_m0, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+    s_lshl_b32	    s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT
+    s_or_b32	    s_restore_tmp, s_restore_tmp, s_restore_m0
+    s_and_b32	    s_restore_m0, s_restore_status, SQ_WAVE_STATUS_INST_ATC_MASK
+    s_lshr_b32	    s_restore_m0, s_restore_m0, SQ_WAVE_STATUS_INST_ATC_SHIFT
+    s_setreg_b32    hwreg(HW_REG_IB_STS),   s_restore_tmp
+
+    s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff	//pc[47:32]	   //Do it here in order not to affect STATUS
+    s_and_b64	 exec, exec, exec  // Restore STATUS.EXECZ, not writable by s_setreg_b32
+    s_and_b64	 vcc, vcc, vcc	// Restore STATUS.VCCZ, not writable by s_setreg_b32
+    s_setreg_b32    hwreg(HW_REG_STATUS),   s_restore_status	 // SCC is included, which is changed by previous salu
+
+    s_barrier							//barrier to ensure the readiness of LDS before access attempts from any other wave in the same TG //FIXME not performance-optimal at this time
+
+if G8SR_DEBUG_TIMESTAMP
+    s_memrealtime s_g8sr_ts_restore_d
+    s_waitcnt lgkmcnt(0)
+end
+
+//  s_rfe_b64 s_restore_pc_lo					//Return to the main shader program and resume execution
+    s_rfe_restore_b64  s_restore_pc_lo, s_restore_m0		// s_restore_m0[0] is used to set STATUS.inst_atc
+
+
+/**************************************************************************/
+/*			the END						  */
+/**************************************************************************/
+L_END_PGM:
+    s_endpgm
+
+end
+
+
+/**************************************************************************/
+/*			the helper functions				  */
+/**************************************************************************/
+
+//Only for save hwreg to mem
+function write_hwreg_to_mem(s, s_rsrc, s_mem_offset)
+	s_mov_b32 exec_lo, m0			//assuming exec_lo is not needed anymore from this point on
+	s_mov_b32 m0, s_mem_offset
+	s_buffer_store_dword s, s_rsrc, m0	glc:1
+	ack_sqc_store_workaround()
+	s_add_u32	s_mem_offset, s_mem_offset, 4
+	s_mov_b32   m0, exec_lo
+end
+
+
+// HWREG are saved before SGPRs, so all HWREG could be use.
+function write_16sgpr_to_mem(s, s_rsrc, s_mem_offset)
+
+	s_buffer_store_dwordx4 s[0], s_rsrc, 0	glc:1
+	ack_sqc_store_workaround()
+	s_buffer_store_dwordx4 s[4], s_rsrc, 16	 glc:1
+	ack_sqc_store_workaround()
+	s_buffer_store_dwordx4 s[8], s_rsrc, 32	 glc:1
+	ack_sqc_store_workaround()
+	s_buffer_store_dwordx4 s[12], s_rsrc, 48 glc:1
+	ack_sqc_store_workaround()
+	s_add_u32	s_rsrc[0], s_rsrc[0], 4*16
+	s_addc_u32	s_rsrc[1], s_rsrc[1], 0x0	      // +scc
+end
+
+
+function read_hwreg_from_mem(s, s_rsrc, s_mem_offset)
+    s_buffer_load_dword s, s_rsrc, s_mem_offset	    glc:1
+    s_add_u32	    s_mem_offset, s_mem_offset, 4
+end
+
+function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset)
+    s_buffer_load_dwordx16 s, s_rsrc, s_mem_offset	glc:1
+    s_sub_u32	    s_mem_offset, s_mem_offset, 4*16
+end
+
+
+
+function get_lds_size_bytes(s_lds_size_byte)
+    // SQ LDS granularity is 64DW, while PGM_RSRC2.lds_size is in granularity 128DW
+    s_getreg_b32   s_lds_size_byte, hwreg(HW_REG_LDS_ALLOC, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)		// lds_size
+    s_lshl_b32	   s_lds_size_byte, s_lds_size_byte, 8			    //LDS size in dwords = lds_size * 64 *4Bytes    // granularity 64DW
+end
+
+function get_vgpr_size_bytes(s_vgpr_size_byte)
+    s_getreg_b32   s_vgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)	 //vpgr_size
+    s_add_u32	   s_vgpr_size_byte, s_vgpr_size_byte, 1
+    s_lshl_b32	   s_vgpr_size_byte, s_vgpr_size_byte, (2+8) //Number of VGPRs = (vgpr_size + 1) * 4 * 64 * 4	(non-zero value)   //FIXME for GFX, zero is possible
+end
+
+function get_sgpr_size_bytes(s_sgpr_size_byte)
+    s_getreg_b32   s_sgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE)	 //spgr_size
+    s_add_u32	   s_sgpr_size_byte, s_sgpr_size_byte, 1
+    s_lshl_b32	   s_sgpr_size_byte, s_sgpr_size_byte, 6 //Number of SGPRs = (sgpr_size + 1) * 16 *4   (non-zero value)
+end
+
+function get_hwreg_size_bytes
+    return 128 //HWREG size 128 bytes
+end
+
+function ack_sqc_store_workaround
+    if ACK_SQC_STORE
+        s_waitcnt lgkmcnt(0)
+    end
+end
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 59808a39ecf4..f64c5551cdba 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -233,7 +233,7 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
 	pr_debug("Queue Size: 0x%llX, %u\n",
 			q_properties->queue_size, args->ring_size);
 
-	pr_debug("Queue r/w Pointers: %p, %p\n",
+	pr_debug("Queue r/w Pointers: %px, %px\n",
 			q_properties->read_ptr,
 			q_properties->write_ptr);
 
@@ -292,8 +292,16 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 
 
 	/* Return gpu_id as doorbell offset for mmap usage */
-	args->doorbell_offset = (KFD_MMAP_DOORBELL_MASK | args->gpu_id);
+	args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
+	args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
 	args->doorbell_offset <<= PAGE_SHIFT;
+	if (KFD_IS_SOC15(dev->device_info->asic_family))
+		/* On SOC15 ASICs, doorbell allocation must be
+		 * per-device, and independent from the per-process
+		 * queue_id. Return the doorbell offset within the
+		 * doorbell aperture to user mode.
+		 */
+		args->doorbell_offset |= q_properties.doorbell_off;
 
 	mutex_unlock(&p->mutex);
 
@@ -1296,8 +1304,8 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
 		return -EINVAL;
 	}
 
-	devices_arr = kmalloc(args->n_devices * sizeof(*devices_arr),
-			      GFP_KERNEL);
+	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
+				    GFP_KERNEL);
 	if (!devices_arr)
 		return -ENOMEM;
 
@@ -1405,8 +1413,8 @@ static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
 		return -EINVAL;
 	}
 
-	devices_arr = kmalloc(args->n_devices * sizeof(*devices_arr),
-			      GFP_KERNEL);
+	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
+				    GFP_KERNEL);
 	if (!devices_arr)
 		return -ENOMEM;
 
@@ -1645,23 +1653,33 @@ static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
 {
 	struct kfd_process *process;
+	struct kfd_dev *dev = NULL;
+	unsigned long vm_pgoff;
+	unsigned int gpu_id;
 
 	process = kfd_get_process(current);
 	if (IS_ERR(process))
 		return PTR_ERR(process);
 
-	if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) ==
-			KFD_MMAP_DOORBELL_MASK) {
-		vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK;
-		return kfd_doorbell_mmap(process, vma);
-	} else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) ==
-			KFD_MMAP_EVENTS_MASK) {
-		vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK;
+	vm_pgoff = vma->vm_pgoff;
+	vma->vm_pgoff = KFD_MMAP_OFFSET_VALUE_GET(vm_pgoff);
+	gpu_id = KFD_MMAP_GPU_ID_GET(vm_pgoff);
+	if (gpu_id)
+		dev = kfd_device_by_id(gpu_id);
+
+	switch (vm_pgoff & KFD_MMAP_TYPE_MASK) {
+	case KFD_MMAP_TYPE_DOORBELL:
+		if (!dev)
+			return -ENODEV;
+		return kfd_doorbell_mmap(dev, process, vma);
+
+	case KFD_MMAP_TYPE_EVENTS:
 		return kfd_event_mmap(process, vma);
-	} else if ((vma->vm_pgoff & KFD_MMAP_RESERVED_MEM_MASK) ==
-			KFD_MMAP_RESERVED_MEM_MASK) {
-		vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_RESERVED_MEM_MASK;
-		return kfd_reserved_mem_mmap(process, vma);
+
+	case KFD_MMAP_TYPE_RESERVED_MEM:
+		if (!dev)
+			return -ENODEV;
+		return kfd_reserved_mem_mmap(dev, process, vma);
 	}
 
 	return -EFAULT;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index 4f126ef6139b..296b3f230280 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -132,6 +132,9 @@ static struct kfd_gpu_cache_info carrizo_cache_info[] = {
 #define fiji_cache_info  carrizo_cache_info
 #define polaris10_cache_info carrizo_cache_info
 #define polaris11_cache_info carrizo_cache_info
+/* TODO - check & update Vega10 cache details */
+#define vega10_cache_info carrizo_cache_info
+#define raven_cache_info carrizo_cache_info
 
 static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
 		struct crat_subtype_computeunit *cu)
@@ -603,6 +606,14 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
 		pcache_info = polaris11_cache_info;
 		num_of_cache_types = ARRAY_SIZE(polaris11_cache_info);
 		break;
+	case CHIP_VEGA10:
+		pcache_info = vega10_cache_info;
+		num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
+		break;
+	case CHIP_RAVEN:
+		pcache_info = raven_cache_info;
+		num_of_cache_types = ARRAY_SIZE(raven_cache_info);
+		break;
 	default:
 		return -EINVAL;
 	}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 3346699960dd..7ee6cec2c060 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -20,16 +20,13 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
-#include <linux/amd-iommu.h>
-#endif
 #include <linux/bsearch.h>
 #include <linux/pci.h>
 #include <linux/slab.h>
 #include "kfd_priv.h"
 #include "kfd_device_queue_manager.h"
 #include "kfd_pm4_headers_vi.h"
-#include "cwsr_trap_handler_gfx8.asm"
+#include "cwsr_trap_handler.h"
 #include "kfd_iommu.h"
 
 #define MQD_SIZE_ALIGNED 768
@@ -41,6 +38,7 @@ static const struct kfd_device_info kaveri_device_info = {
 	.max_pasid_bits = 16,
 	/* max num of queues for KV.TODO should be a dynamic value */
 	.max_no_of_hqd	= 24,
+	.doorbell_size  = 4,
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
@@ -55,6 +53,7 @@ static const struct kfd_device_info carrizo_device_info = {
 	.max_pasid_bits = 16,
 	/* max num of queues for CZ.TODO should be a dynamic value */
 	.max_no_of_hqd	= 24,
+	.doorbell_size  = 4,
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
@@ -70,6 +69,7 @@ static const struct kfd_device_info hawaii_device_info = {
 	.max_pasid_bits = 16,
 	/* max num of queues for KV.TODO should be a dynamic value */
 	.max_no_of_hqd	= 24,
+	.doorbell_size  = 4,
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
@@ -83,6 +83,7 @@ static const struct kfd_device_info tonga_device_info = {
 	.asic_family = CHIP_TONGA,
 	.max_pasid_bits = 16,
 	.max_no_of_hqd  = 24,
+	.doorbell_size  = 4,
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
@@ -96,6 +97,7 @@ static const struct kfd_device_info tonga_vf_device_info = {
 	.asic_family = CHIP_TONGA,
 	.max_pasid_bits = 16,
 	.max_no_of_hqd  = 24,
+	.doorbell_size  = 4,
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
@@ -109,6 +111,7 @@ static const struct kfd_device_info fiji_device_info = {
 	.asic_family = CHIP_FIJI,
 	.max_pasid_bits = 16,
 	.max_no_of_hqd  = 24,
+	.doorbell_size  = 4,
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
@@ -122,6 +125,7 @@ static const struct kfd_device_info fiji_vf_device_info = {
 	.asic_family = CHIP_FIJI,
 	.max_pasid_bits = 16,
 	.max_no_of_hqd  = 24,
+	.doorbell_size  = 4,
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
@@ -136,6 +140,7 @@ static const struct kfd_device_info polaris10_device_info = {
 	.asic_family = CHIP_POLARIS10,
 	.max_pasid_bits = 16,
 	.max_no_of_hqd  = 24,
+	.doorbell_size  = 4,
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
@@ -149,6 +154,7 @@ static const struct kfd_device_info polaris10_vf_device_info = {
 	.asic_family = CHIP_POLARIS10,
 	.max_pasid_bits = 16,
 	.max_no_of_hqd  = 24,
+	.doorbell_size  = 4,
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
@@ -162,6 +168,7 @@ static const struct kfd_device_info polaris11_device_info = {
 	.asic_family = CHIP_POLARIS11,
 	.max_pasid_bits = 16,
 	.max_no_of_hqd  = 24,
+	.doorbell_size  = 4,
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
@@ -171,6 +178,34 @@ static const struct kfd_device_info polaris11_device_info = {
 	.needs_pci_atomics = true,
 };
 
+static const struct kfd_device_info vega10_device_info = {
+	.asic_family = CHIP_VEGA10,
+	.max_pasid_bits = 16,
+	.max_no_of_hqd  = 24,
+	.doorbell_size  = 8,
+	.ih_ring_entry_size = 8 * sizeof(uint32_t),
+	.event_interrupt_class = &event_interrupt_class_v9,
+	.num_of_watch_points = 4,
+	.mqd_size_aligned = MQD_SIZE_ALIGNED,
+	.supports_cwsr = true,
+	.needs_iommu_device = false,
+	.needs_pci_atomics = false,
+};
+
+static const struct kfd_device_info vega10_vf_device_info = {
+	.asic_family = CHIP_VEGA10,
+	.max_pasid_bits = 16,
+	.max_no_of_hqd  = 24,
+	.doorbell_size  = 8,
+	.ih_ring_entry_size = 8 * sizeof(uint32_t),
+	.event_interrupt_class = &event_interrupt_class_v9,
+	.num_of_watch_points = 4,
+	.mqd_size_aligned = MQD_SIZE_ALIGNED,
+	.supports_cwsr = true,
+	.needs_iommu_device = false,
+	.needs_pci_atomics = false,
+};
+
 
 struct kfd_deviceid {
 	unsigned short did;
@@ -250,6 +285,15 @@ static const struct kfd_deviceid supported_devices[] = {
 	{ 0x67EB, &polaris11_device_info },	/* Polaris11 */
 	{ 0x67EF, &polaris11_device_info },	/* Polaris11 */
 	{ 0x67FF, &polaris11_device_info },	/* Polaris11 */
+	{ 0x6860, &vega10_device_info },	/* Vega10 */
+	{ 0x6861, &vega10_device_info },	/* Vega10 */
+	{ 0x6862, &vega10_device_info },	/* Vega10 */
+	{ 0x6863, &vega10_device_info },	/* Vega10 */
+	{ 0x6864, &vega10_device_info },	/* Vega10 */
+	{ 0x6867, &vega10_device_info },	/* Vega10 */
+	{ 0x6868, &vega10_device_info },	/* Vega10 */
+	{ 0x686C, &vega10_vf_device_info },	/* Vega10  vf*/
+	{ 0x687F, &vega10_device_info },	/* Vega10 */
 };
 
 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
@@ -279,7 +323,7 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
 	struct pci_dev *pdev, const struct kfd2kgd_calls *f2g)
 {
 	struct kfd_dev *kfd;
-
+	int ret;
 	const struct kfd_device_info *device_info =
 					lookup_device_info(pdev->device);
 
@@ -288,19 +332,18 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
 		return NULL;
 	}
 
-	if (device_info->needs_pci_atomics) {
-		/* Allow BIF to recode atomics to PCIe 3.0
-		 * AtomicOps. 32 and 64-bit requests are possible and
-		 * must be supported.
-		 */
-		if (pci_enable_atomic_ops_to_root(pdev,
-				PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
-				PCI_EXP_DEVCAP2_ATOMIC_COMP64) < 0) {
-			dev_info(kfd_device,
-				"skipped device %x:%x, PCI rejects atomics",
-				 pdev->vendor, pdev->device);
-			return NULL;
-		}
+	/* Allow BIF to recode atomics to PCIe 3.0 AtomicOps.
+	 * 32 and 64-bit requests are possible and must be
+	 * supported.
+	 */
+	ret = pci_enable_atomic_ops_to_root(pdev,
+			PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
+			PCI_EXP_DEVCAP2_ATOMIC_COMP64);
+	if (device_info->needs_pci_atomics && ret < 0) {
+		dev_info(kfd_device,
+			 "skipped device %x:%x, PCI rejects atomics\n",
+			 pdev->vendor, pdev->device);
+		return NULL;
 	}
 
 	kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);
@@ -323,10 +366,16 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
 static void kfd_cwsr_init(struct kfd_dev *kfd)
 {
 	if (cwsr_enable && kfd->device_info->supports_cwsr) {
-		BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE);
+		if (kfd->device_info->asic_family < CHIP_VEGA10) {
+			BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE);
+			kfd->cwsr_isa = cwsr_trap_gfx8_hex;
+			kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex);
+		} else {
+			BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE);
+			kfd->cwsr_isa = cwsr_trap_gfx9_hex;
+			kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex);
+		}
 
-		kfd->cwsr_isa = cwsr_trap_gfx8_hex;
-		kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex);
 		kfd->cwsr_enabled = true;
 	}
 }
@@ -541,6 +590,44 @@ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
 	spin_unlock(&kfd->interrupt_lock);
 }
 
+int kgd2kfd_quiesce_mm(struct mm_struct *mm)
+{
+	struct kfd_process *p;
+	int r;
+
+	/* Because we are called from arbitrary context (workqueue) as opposed
+	 * to process context, kfd_process could attempt to exit while we are
+	 * running so the lookup function increments the process ref count.
+	 */
+	p = kfd_lookup_process_by_mm(mm);
+	if (!p)
+		return -ESRCH;
+
+	r = kfd_process_evict_queues(p);
+
+	kfd_unref_process(p);
+	return r;
+}
+
+int kgd2kfd_resume_mm(struct mm_struct *mm)
+{
+	struct kfd_process *p;
+	int r;
+
+	/* Because we are called from arbitrary context (workqueue) as opposed
+	 * to process context, kfd_process could attempt to exit while we are
+	 * running so the lookup function increments the process ref count.
+	 */
+	p = kfd_lookup_process_by_mm(mm);
+	if (!p)
+		return -ESRCH;
+
+	r = kfd_process_restore_queues(p);
+
+	kfd_unref_process(p);
+	return r;
+}
+
 /** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will
  *   prepare for safe eviction of KFD BOs that belong to the specified
  *   process.
@@ -652,7 +739,7 @@ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
 	if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size)
 		return -ENOMEM;
 
-	*mem_obj = kmalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
+	*mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
 	if ((*mem_obj) == NULL)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index d55d29d31da4..668ad07ebe1f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -110,6 +110,57 @@ void program_sh_mem_settings(struct device_queue_manager *dqm,
 						qpd->sh_mem_bases);
 }
 
+static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q)
+{
+	struct kfd_dev *dev = qpd->dqm->dev;
+
+	if (!KFD_IS_SOC15(dev->device_info->asic_family)) {
+		/* On pre-SOC15 chips we need to use the queue ID to
+		 * preserve the user mode ABI.
+		 */
+		q->doorbell_id = q->properties.queue_id;
+	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
+		/* For SDMA queues on SOC15, use static doorbell
+		 * assignments based on the engine and queue.
+		 */
+		q->doorbell_id = dev->shared_resources.sdma_doorbell
+			[q->properties.sdma_engine_id]
+			[q->properties.sdma_queue_id];
+	} else {
+		/* For CP queues on SOC15 reserve a free doorbell ID */
+		unsigned int found;
+
+		found = find_first_zero_bit(qpd->doorbell_bitmap,
+					    KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
+		if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
+			pr_debug("No doorbells available");
+			return -EBUSY;
+		}
+		set_bit(found, qpd->doorbell_bitmap);
+		q->doorbell_id = found;
+	}
+
+	q->properties.doorbell_off =
+		kfd_doorbell_id_to_offset(dev, q->process,
+					  q->doorbell_id);
+
+	return 0;
+}
+
+static void deallocate_doorbell(struct qcm_process_device *qpd,
+				struct queue *q)
+{
+	unsigned int old;
+	struct kfd_dev *dev = qpd->dqm->dev;
+
+	if (!KFD_IS_SOC15(dev->device_info->asic_family) ||
+	    q->properties.type == KFD_QUEUE_TYPE_SDMA)
+		return;
+
+	old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
+	WARN_ON(!old);
+}
+
 static int allocate_vmid(struct device_queue_manager *dqm,
 			struct qcm_process_device *qpd,
 			struct queue *q)
@@ -145,15 +196,19 @@ static int allocate_vmid(struct device_queue_manager *dqm,
 static int flush_texture_cache_nocpsch(struct kfd_dev *kdev,
 				struct qcm_process_device *qpd)
 {
-	uint32_t len;
+	const struct packet_manager_funcs *pmf = qpd->dqm->packets.pmf;
+	int ret;
 
 	if (!qpd->ib_kaddr)
 		return -ENOMEM;
 
-	len = pm_create_release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
+	ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
+	if (ret)
+		return ret;
 
 	return kdev->kfd2kgd->submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid,
-				qpd->ib_base, (uint32_t *)qpd->ib_kaddr, len);
+				qpd->ib_base, (uint32_t *)qpd->ib_kaddr,
+				pmf->release_mem_size / sizeof(uint32_t));
 }
 
 static void deallocate_vmid(struct device_queue_manager *dqm,
@@ -301,10 +356,14 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 	if (retval)
 		return retval;
 
+	retval = allocate_doorbell(qpd, q);
+	if (retval)
+		goto out_deallocate_hqd;
+
 	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
 	if (retval)
-		goto out_deallocate_hqd;
+		goto out_deallocate_doorbell;
 
 	pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
 			q->pipe, q->queue);
@@ -324,6 +383,8 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 
 out_uninit_mqd:
 	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+out_deallocate_doorbell:
+	deallocate_doorbell(qpd, q);
 out_deallocate_hqd:
 	deallocate_hqd(dqm, q);
 
@@ -357,6 +418,8 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 	}
 	dqm->total_queue_count--;
 
+	deallocate_doorbell(qpd, q);
+
 	retval = mqd->destroy_mqd(mqd, q->mqd,
 				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 				KFD_UNMAP_LATENCY_MS,
@@ -861,6 +924,10 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 	q->properties.sdma_queue_id = q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
 	q->properties.sdma_engine_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
 
+	retval = allocate_doorbell(qpd, q);
+	if (retval)
+		goto out_deallocate_sdma_queue;
+
 	pr_debug("SDMA id is:    %d\n", q->sdma_id);
 	pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
 	pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
@@ -869,7 +936,7 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
 	if (retval)
-		goto out_deallocate_sdma_queue;
+		goto out_deallocate_doorbell;
 
 	retval = mqd->load_mqd(mqd, q->mqd, 0, 0, &q->properties, NULL);
 	if (retval)
@@ -879,6 +946,8 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 
 out_uninit_mqd:
 	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+out_deallocate_doorbell:
+	deallocate_doorbell(qpd, q);
 out_deallocate_sdma_queue:
 	deallocate_sdma_queue(dqm, q->sdma_id);
 
@@ -1070,12 +1139,17 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 		q->properties.sdma_engine_id =
 			q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
 	}
+
+	retval = allocate_doorbell(qpd, q);
+	if (retval)
+		goto out_deallocate_sdma_queue;
+
 	mqd = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
 
 	if (!mqd) {
 		retval = -ENOMEM;
-		goto out_deallocate_sdma_queue;
+		goto out_deallocate_doorbell;
 	}
 	/*
 	 * Eviction state logic: we only mark active queues as evicted
@@ -1093,7 +1167,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
 	if (retval)
-		goto out_deallocate_sdma_queue;
+		goto out_deallocate_doorbell;
 
 	list_add(&q->list, &qpd->queues_list);
 	qpd->queue_count++;
@@ -1117,6 +1191,8 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	mutex_unlock(&dqm->lock);
 	return retval;
 
+out_deallocate_doorbell:
+	deallocate_doorbell(qpd, q);
 out_deallocate_sdma_queue:
 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 		deallocate_sdma_queue(dqm, q->sdma_id);
@@ -1257,6 +1333,8 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 		goto failed;
 	}
 
+	deallocate_doorbell(qpd, q);
+
 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
 		dqm->sdma_queue_count--;
 		deallocate_sdma_queue(dqm, q->sdma_id);
@@ -1308,7 +1386,10 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 				   void __user *alternate_aperture_base,
 				   uint64_t alternate_aperture_size)
 {
-	bool retval;
+	bool retval = true;
+
+	if (!dqm->asic_ops.set_cache_memory_policy)
+		return retval;
 
 	mutex_lock(&dqm->lock);
 
@@ -1577,6 +1658,11 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 	case CHIP_POLARIS11:
 		device_queue_manager_init_vi_tonga(&dqm->asic_ops);
 		break;
+
+	case CHIP_VEGA10:
+	case CHIP_RAVEN:
+		device_queue_manager_init_v9(&dqm->asic_ops);
+		break;
 	default:
 		WARN(1, "Unexpected ASIC family %u",
 		     dev->device_info->asic_family);
@@ -1627,6 +1713,18 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
 	int pipe, queue;
 	int r = 0;
 
+	r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
+		KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE, &dump, &n_regs);
+	if (!r) {
+		seq_printf(m, "  HIQ on MEC %d Pipe %d Queue %d\n",
+				KFD_CIK_HIQ_PIPE/get_pipes_per_mec(dqm)+1,
+				KFD_CIK_HIQ_PIPE%get_pipes_per_mec(dqm),
+				KFD_CIK_HIQ_QUEUE);
+		seq_reg_dump(m, dump, n_regs);
+
+		kfree(dump);
+	}
+
 	for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
 		int pipe_offset = pipe * get_queues_per_pipe(dqm);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 412beff3281d..59a6b1956932 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -200,6 +200,8 @@ void device_queue_manager_init_vi(
 		struct device_queue_manager_asic_ops *asic_ops);
 void device_queue_manager_init_vi_tonga(
 		struct device_queue_manager_asic_ops *asic_ops);
+void device_queue_manager_init_v9(
+		struct device_queue_manager_asic_ops *asic_ops);
 void program_sh_mem_settings(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd);
 unsigned int get_queues_num(struct device_queue_manager *dqm);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
new file mode 100644
index 000000000000..79e5bcf6367c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2016-2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "kfd_device_queue_manager.h"
+#include "vega10_enum.h"
+#include "gc/gc_9_0_offset.h"
+#include "gc/gc_9_0_sh_mask.h"
+#include "sdma0/sdma0_4_0_sh_mask.h"
+
+static int update_qpd_v9(struct device_queue_manager *dqm,
+			 struct qcm_process_device *qpd);
+static void init_sdma_vm_v9(struct device_queue_manager *dqm, struct queue *q,
+			    struct qcm_process_device *qpd);
+
+void device_queue_manager_init_v9(
+	struct device_queue_manager_asic_ops *asic_ops)
+{
+	asic_ops->update_qpd = update_qpd_v9;
+	asic_ops->init_sdma_vm = init_sdma_vm_v9;
+}
+
+static uint32_t compute_sh_mem_bases_64bit(struct kfd_process_device *pdd)
+{
+	uint32_t shared_base = pdd->lds_base >> 48;
+	uint32_t private_base = pdd->scratch_base >> 48;
+
+	return (shared_base << SH_MEM_BASES__SHARED_BASE__SHIFT) |
+		private_base;
+}
+
+static int update_qpd_v9(struct device_queue_manager *dqm,
+			 struct qcm_process_device *qpd)
+{
+	struct kfd_process_device *pdd;
+
+	pdd = qpd_to_pdd(qpd);
+
+	/* check if sh_mem_config register already configured */
+	if (qpd->sh_mem_config == 0) {
+		qpd->sh_mem_config =
+				SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
+					SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
+		if (vega10_noretry &&
+		    !dqm->dev->device_info->needs_iommu_device)
+			qpd->sh_mem_config |=
+				1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
+
+		qpd->sh_mem_ape1_limit = 0;
+		qpd->sh_mem_ape1_base = 0;
+	}
+
+	qpd->sh_mem_bases = compute_sh_mem_bases_64bit(pdd);
+
+	pr_debug("sh_mem_bases 0x%X\n", qpd->sh_mem_bases);
+
+	return 0;
+}
+
+static void init_sdma_vm_v9(struct device_queue_manager *dqm, struct queue *q,
+			    struct qcm_process_device *qpd)
+{
+	/* Not needed on SDMAv4 any more */
+	q->properties.sdma_vm_addr = 0;
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
index ebb4da14e3df..c3744d89352c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
@@ -33,7 +33,6 @@
 
 static DEFINE_IDA(doorbell_ida);
 static unsigned int max_doorbell_slices;
-#define KFD_SIZE_OF_DOORBELL_IN_BYTES 4
 
 /*
  * Each device exposes a doorbell aperture, a PCI MMIO aperture that
@@ -50,9 +49,9 @@ static unsigned int max_doorbell_slices;
  */
 
 /* # of doorbell bytes allocated for each process. */
-static inline size_t doorbell_process_allocation(void)
+size_t kfd_doorbell_process_slice(struct kfd_dev *kfd)
 {
-	return roundup(KFD_SIZE_OF_DOORBELL_IN_BYTES *
+	return roundup(kfd->device_info->doorbell_size *
 			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
 			PAGE_SIZE);
 }
@@ -72,16 +71,16 @@ int kfd_doorbell_init(struct kfd_dev *kfd)
 
 	doorbell_start_offset =
 			roundup(kfd->shared_resources.doorbell_start_offset,
-					doorbell_process_allocation());
+					kfd_doorbell_process_slice(kfd));
 
 	doorbell_aperture_size =
 			rounddown(kfd->shared_resources.doorbell_aperture_size,
-					doorbell_process_allocation());
+					kfd_doorbell_process_slice(kfd));
 
 	if (doorbell_aperture_size > doorbell_start_offset)
 		doorbell_process_limit =
 			(doorbell_aperture_size - doorbell_start_offset) /
-						doorbell_process_allocation();
+						kfd_doorbell_process_slice(kfd);
 	else
 		return -ENOSPC;
 
@@ -95,7 +94,7 @@ int kfd_doorbell_init(struct kfd_dev *kfd)
 	kfd->doorbell_id_offset = doorbell_start_offset / sizeof(u32);
 
 	kfd->doorbell_kernel_ptr = ioremap(kfd->doorbell_base,
-						doorbell_process_allocation());
+					   kfd_doorbell_process_slice(kfd));
 
 	if (!kfd->doorbell_kernel_ptr)
 		return -ENOMEM;
@@ -127,21 +126,16 @@ void kfd_doorbell_fini(struct kfd_dev *kfd)
 		iounmap(kfd->doorbell_kernel_ptr);
 }
 
-int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma)
+int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
+		      struct vm_area_struct *vma)
 {
 	phys_addr_t address;
-	struct kfd_dev *dev;
 
 	/*
 	 * For simplicitly we only allow mapping of the entire doorbell
 	 * allocation of a single device & process.
 	 */
-	if (vma->vm_end - vma->vm_start != doorbell_process_allocation())
-		return -EINVAL;
-
-	/* Find kfd device according to gpu id */
-	dev = kfd_device_by_id(vma->vm_pgoff);
-	if (!dev)
+	if (vma->vm_end - vma->vm_start != kfd_doorbell_process_slice(dev))
 		return -EINVAL;
 
 	/* Calculate physical address of doorbell */
@@ -158,19 +152,19 @@ int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma)
 		 "     vm_flags            == 0x%04lX\n"
 		 "     size                == 0x%04lX\n",
 		 (unsigned long long) vma->vm_start, address, vma->vm_flags,
-		 doorbell_process_allocation());
+		 kfd_doorbell_process_slice(dev));
 
 
 	return io_remap_pfn_range(vma,
 				vma->vm_start,
 				address >> PAGE_SHIFT,
-				doorbell_process_allocation(),
+				kfd_doorbell_process_slice(dev),
 				vma->vm_page_prot);
 }
 
 
 /* get kernel iomem pointer for a doorbell */
-u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
+void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
 					unsigned int *doorbell_off)
 {
 	u32 inx;
@@ -185,6 +179,8 @@ u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
 	if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
 		return NULL;
 
+	inx *= kfd->device_info->doorbell_size / sizeof(u32);
+
 	/*
 	 * Calculating the kernel doorbell offset using the first
 	 * doorbell page.
@@ -210,7 +206,7 @@ void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr)
 	mutex_unlock(&kfd->doorbell_mutex);
 }
 
-inline void write_kernel_doorbell(u32 __iomem *db, u32 value)
+void write_kernel_doorbell(void __iomem *db, u32 value)
 {
 	if (db) {
 		writel(value, db);
@@ -218,30 +214,37 @@ inline void write_kernel_doorbell(u32 __iomem *db, u32 value)
 	}
 }
 
-/*
- * queue_ids are in the range [0,MAX_PROCESS_QUEUES) and are mapped 1:1
- * to doorbells with the process's doorbell page
- */
-unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,
+void write_kernel_doorbell64(void __iomem *db, u64 value)
+{
+	if (db) {
+		WARN(((unsigned long)db & 7) != 0,
+		     "Unaligned 64-bit doorbell");
+		writeq(value, (u64 __iomem *)db);
+		pr_debug("writing %llu to doorbell address %p\n", value, db);
+	}
+}
+
+unsigned int kfd_doorbell_id_to_offset(struct kfd_dev *kfd,
 					struct kfd_process *process,
-					unsigned int queue_id)
+					unsigned int doorbell_id)
 {
 	/*
 	 * doorbell_id_offset accounts for doorbells taken by KGD.
-	 * index * doorbell_process_allocation/sizeof(u32) adjusts to
-	 * the process's doorbells.
+	 * index * kfd_doorbell_process_slice/sizeof(u32) adjusts to
+	 * the process's doorbells. The offset returned is in dword
+	 * units regardless of the ASIC-dependent doorbell size.
 	 */
 	return kfd->doorbell_id_offset +
 		process->doorbell_index
-		* doorbell_process_allocation() / sizeof(u32) +
-		queue_id;
+		* kfd_doorbell_process_slice(kfd) / sizeof(u32) +
+		doorbell_id * kfd->device_info->doorbell_size / sizeof(u32);
 }
 
 uint64_t kfd_get_number_elems(struct kfd_dev *kfd)
 {
 	uint64_t num_of_elems = (kfd->shared_resources.doorbell_aperture_size -
 				kfd->shared_resources.doorbell_start_offset) /
-					doorbell_process_allocation() + 1;
+					kfd_doorbell_process_slice(kfd) + 1;
 
 	return num_of_elems;
 
@@ -251,7 +254,7 @@ phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
 					struct kfd_process *process)
 {
 	return dev->doorbell_base +
-		process->doorbell_index * doorbell_process_allocation();
+		process->doorbell_index * kfd_doorbell_process_slice(dev);
 }
 
 int kfd_alloc_process_doorbells(struct kfd_process *process)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index 4890a90f1e44..5562e94e786a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -345,7 +345,7 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
 	case KFD_EVENT_TYPE_DEBUG:
 		ret = create_signal_event(devkfd, p, ev);
 		if (!ret) {
-			*event_page_offset = KFD_MMAP_EVENTS_MASK;
+			*event_page_offset = KFD_MMAP_TYPE_EVENTS;
 			*event_page_offset <<= PAGE_SHIFT;
 			*event_slot_index = ev->event_id;
 		}
@@ -496,7 +496,7 @@ void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
 			pr_debug_ratelimited("Partial ID invalid: %u (%u valid bits)\n",
 					     partial_id, valid_id_bits);
 
-		if (p->signal_event_count < KFD_SIGNAL_EVENT_LIMIT/2) {
+		if (p->signal_event_count < KFD_SIGNAL_EVENT_LIMIT / 64) {
 			/* With relatively few events, it's faster to
 			 * iterate over the event IDR
 			 */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
index 66852de410c8..97d5423c5673 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
@@ -275,23 +275,35 @@
  * for FLAT_* / S_LOAD operations.
  */
 
-#define MAKE_GPUVM_APP_BASE(gpu_num) \
+#define MAKE_GPUVM_APP_BASE_VI(gpu_num) \
 	(((uint64_t)(gpu_num) << 61) + 0x1000000000000L)
 
 #define MAKE_GPUVM_APP_LIMIT(base, size) \
 	(((uint64_t)(base) & 0xFFFFFF0000000000UL) + (size) - 1)
 
-#define MAKE_SCRATCH_APP_BASE() \
+#define MAKE_SCRATCH_APP_BASE_VI() \
 	(((uint64_t)(0x1UL) << 61) + 0x100000000L)
 
 #define MAKE_SCRATCH_APP_LIMIT(base) \
 	(((uint64_t)base & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF)
 
-#define MAKE_LDS_APP_BASE() \
+#define MAKE_LDS_APP_BASE_VI() \
 	(((uint64_t)(0x1UL) << 61) + 0x0)
 #define MAKE_LDS_APP_LIMIT(base) \
 	(((uint64_t)(base) & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF)
 
+/* On GFXv9 the LDS and scratch apertures are programmed independently
+ * using the high 16 bits of the 64-bit virtual address. They must be
+ * in the hole, which will be the case as long as the high 16 bits are
+ * not 0.
+ *
+ * The aperture sizes are still 4GB implicitly.
+ *
+ * A GPUVM aperture is not applicable on GFXv9.
+ */
+#define MAKE_LDS_APP_BASE_V9() ((uint64_t)(0x1UL) << 48)
+#define MAKE_SCRATCH_APP_BASE_V9() ((uint64_t)(0x2UL) << 48)
+
 /* User mode manages most of the SVM aperture address space. The low
  * 16MB are reserved for kernel use (CWSR trap handler and kernel IB
  * for now).
@@ -300,6 +312,55 @@
 #define SVM_CWSR_BASE (SVM_USER_BASE - KFD_CWSR_TBA_TMA_SIZE)
 #define SVM_IB_BASE   (SVM_CWSR_BASE - PAGE_SIZE)
 
+static void kfd_init_apertures_vi(struct kfd_process_device *pdd, uint8_t id)
+{
+	/*
+	 * node id couldn't be 0 - the three MSB bits of
+	 * aperture shoudn't be 0
+	 */
+	pdd->lds_base = MAKE_LDS_APP_BASE_VI();
+	pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
+
+	if (!pdd->dev->device_info->needs_iommu_device) {
+		/* dGPUs: SVM aperture starting at 0
+		 * with small reserved space for kernel.
+		 * Set them to CANONICAL addresses.
+		 */
+		pdd->gpuvm_base = SVM_USER_BASE;
+		pdd->gpuvm_limit =
+			pdd->dev->shared_resources.gpuvm_size - 1;
+	} else {
+		/* set them to non CANONICAL addresses, and no SVM is
+		 * allocated.
+		 */
+		pdd->gpuvm_base = MAKE_GPUVM_APP_BASE_VI(id + 1);
+		pdd->gpuvm_limit = MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base,
+				pdd->dev->shared_resources.gpuvm_size);
+	}
+
+	pdd->scratch_base = MAKE_SCRATCH_APP_BASE_VI();
+	pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
+}
+
+static void kfd_init_apertures_v9(struct kfd_process_device *pdd, uint8_t id)
+{
+	pdd->lds_base = MAKE_LDS_APP_BASE_V9();
+	pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
+
+	/* Raven needs SVM to support graphic handle, etc. Leave the small
+	 * reserved space before SVM on Raven as well, even though we don't
+	 * have to.
+	 * Set gpuvm_base and gpuvm_limit to CANONICAL addresses so that they
+	 * are used in Thunk to reserve SVM.
+	 */
+	pdd->gpuvm_base = SVM_USER_BASE;
+	pdd->gpuvm_limit =
+		pdd->dev->shared_resources.gpuvm_size - 1;
+
+	pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9();
+	pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
+}
+
 int kfd_init_apertures(struct kfd_process *process)
 {
 	uint8_t id  = 0;
@@ -307,9 +368,7 @@ int kfd_init_apertures(struct kfd_process *process)
 	struct kfd_process_device *pdd;
 
 	/*Iterating over all devices*/
-	while (kfd_topology_enum_kfd_devices(id, &dev) == 0 &&
-		id < NUM_OF_SUPPORTED_GPUS) {
-
+	while (kfd_topology_enum_kfd_devices(id, &dev) == 0) {
 		if (!dev) {
 			id++; /* Skip non GPU devices */
 			continue;
@@ -318,7 +377,7 @@ int kfd_init_apertures(struct kfd_process *process)
 		pdd = kfd_create_process_device_data(dev, process);
 		if (!pdd) {
 			pr_err("Failed to create process device data\n");
-			return -1;
+			return -ENOMEM;
 		}
 		/*
 		 * For 64 bit process apertures will be statically reserved in
@@ -330,32 +389,30 @@ int kfd_init_apertures(struct kfd_process *process)
 			pdd->gpuvm_base = pdd->gpuvm_limit = 0;
 			pdd->scratch_base = pdd->scratch_limit = 0;
 		} else {
-			/* Same LDS and scratch apertures can be used
-			 * on all GPUs. This allows using more dGPUs
-			 * than placement options for apertures.
-			 */
-			pdd->lds_base = MAKE_LDS_APP_BASE();
-			pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
-
-			pdd->scratch_base = MAKE_SCRATCH_APP_BASE();
-			pdd->scratch_limit =
-				MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
+			switch (dev->device_info->asic_family) {
+			case CHIP_KAVERI:
+			case CHIP_HAWAII:
+			case CHIP_CARRIZO:
+			case CHIP_TONGA:
+			case CHIP_FIJI:
+			case CHIP_POLARIS10:
+			case CHIP_POLARIS11:
+				kfd_init_apertures_vi(pdd, id);
+				break;
+			case CHIP_VEGA10:
+			case CHIP_RAVEN:
+				kfd_init_apertures_v9(pdd, id);
+				break;
+			default:
+				WARN(1, "Unexpected ASIC family %u",
+				     dev->device_info->asic_family);
+				return -EINVAL;
+			}
 
-			if (dev->device_info->needs_iommu_device) {
-				/* APUs: GPUVM aperture in
-				 * non-canonical address space
-				 */
-				pdd->gpuvm_base = MAKE_GPUVM_APP_BASE(id + 1);
-				pdd->gpuvm_limit = MAKE_GPUVM_APP_LIMIT(
-					pdd->gpuvm_base,
-					dev->shared_resources.gpuvm_size);
-			} else {
-				/* dGPUs: SVM aperture starting at 0
-				 * with small reserved space for kernel
+			if (!dev->device_info->needs_iommu_device) {
+				/* dGPUs: the reserved space for kernel
+				 * before SVM
 				 */
-				pdd->gpuvm_base = SVM_USER_BASE;
-				pdd->gpuvm_limit =
-					dev->shared_resources.gpuvm_size - 1;
 				pdd->qpd.cwsr_base = SVM_CWSR_BASE;
 				pdd->qpd.ib_base = SVM_IB_BASE;
 			}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
new file mode 100644
index 000000000000..37029baa3346
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2016-2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "kfd_priv.h"
+#include "kfd_events.h"
+#include "soc15_int.h"
+
+
+static bool event_interrupt_isr_v9(struct kfd_dev *dev,
+					const uint32_t *ih_ring_entry)
+{
+	uint16_t source_id, client_id, pasid, vmid;
+	const uint32_t *data = ih_ring_entry;
+
+	/* Only handle interrupts from KFD VMIDs */
+	vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
+	if (vmid < dev->vm_info.first_vmid_kfd ||
+	    vmid > dev->vm_info.last_vmid_kfd)
+		return 0;
+
+	/* If there is no valid PASID, it's likely a firmware bug */
+	pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
+	if (WARN_ONCE(pasid == 0, "FW bug: No PASID in KFD interrupt"))
+		return 0;
+
+	source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
+	client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
+
+	pr_debug("client id 0x%x, source id %d, pasid 0x%x. raw data:\n",
+		 client_id, source_id, pasid);
+	pr_debug("%8X, %8X, %8X, %8X, %8X, %8X, %8X, %8X.\n",
+		 data[0], data[1], data[2], data[3],
+		 data[4], data[5], data[6], data[7]);
+
+	/* Interrupt types we care about: various signals and faults.
+	 * They will be forwarded to a work queue (see below).
+	 */
+	return source_id == SOC15_INTSRC_CP_END_OF_PIPE ||
+		source_id == SOC15_INTSRC_SDMA_TRAP ||
+		source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
+		source_id == SOC15_INTSRC_CP_BAD_OPCODE;
+}
+
+static void event_interrupt_wq_v9(struct kfd_dev *dev,
+					const uint32_t *ih_ring_entry)
+{
+	uint16_t source_id, client_id, pasid, vmid;
+	uint32_t context_id;
+
+	source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
+	client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
+	pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
+	vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
+	context_id = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry);
+
+	if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
+		kfd_signal_event_interrupt(pasid, context_id, 32);
+	else if (source_id == SOC15_INTSRC_SDMA_TRAP)
+		kfd_signal_event_interrupt(pasid, context_id & 0xfffffff, 28);
+	else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG)
+		kfd_signal_event_interrupt(pasid, context_id & 0xffffff, 24);
+	else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE)
+		kfd_signal_hw_exception_event(pasid);
+	else if (client_id == SOC15_IH_CLIENTID_VMC ||
+		 client_id == SOC15_IH_CLIENTID_UTCL2) {
+		/* TODO */
+	}
+}
+
+const struct kfd_event_interrupt_class event_interrupt_class_v9 = {
+	.interrupt_isr = event_interrupt_isr_v9,
+	.interrupt_wq = event_interrupt_wq_v9,
+};
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
index 035c351f47c5..db6d9336b80d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
@@ -139,10 +139,12 @@ static void interrupt_wq(struct work_struct *work)
 {
 	struct kfd_dev *dev = container_of(work, struct kfd_dev,
 						interrupt_work);
+	uint32_t ih_ring_entry[KFD_MAX_RING_ENTRY_SIZE];
 
-	uint32_t ih_ring_entry[DIV_ROUND_UP(
-				dev->device_info->ih_ring_entry_size,
-				sizeof(uint32_t))];
+	if (dev->device_info->ih_ring_entry_size > sizeof(ih_ring_entry)) {
+		dev_err_once(kfd_chardev(), "Ring entry too small\n");
+		return;
+	}
 
 	while (dequeue_ih_ring_entry(dev, ih_ring_entry))
 		dev->device_info->event_interrupt_class->interrupt_wq(dev,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
index 69f496485331..476951d8c91c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
@@ -99,7 +99,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 	kq->rptr_kernel = kq->rptr_mem->cpu_ptr;
 	kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr;
 
-	retval = kfd_gtt_sa_allocate(dev, sizeof(*kq->wptr_kernel),
+	retval = kfd_gtt_sa_allocate(dev, dev->device_info->doorbell_size,
 					&kq->wptr_mem);
 
 	if (retval != 0)
@@ -208,6 +208,7 @@ static int acquire_packet_buffer(struct kernel_queue *kq,
 	size_t available_size;
 	size_t queue_size_dwords;
 	uint32_t wptr, rptr;
+	uint64_t wptr64;
 	unsigned int *queue_address;
 
 	/* When rptr == wptr, the buffer is empty.
@@ -216,7 +217,8 @@ static int acquire_packet_buffer(struct kernel_queue *kq,
 	 * the opposite. So we can only use up to queue_size_dwords - 1 dwords.
 	 */
 	rptr = *kq->rptr_kernel;
-	wptr = *kq->wptr_kernel;
+	wptr = kq->pending_wptr;
+	wptr64 = kq->pending_wptr64;
 	queue_address = (unsigned int *)kq->pq_kernel_addr;
 	queue_size_dwords = kq->queue->properties.queue_size / 4;
 
@@ -232,29 +234,33 @@ static int acquire_packet_buffer(struct kernel_queue *kq,
 		 * make sure calling functions know
 		 * acquire_packet_buffer() failed
 		 */
-		*buffer_ptr = NULL;
-		return -ENOMEM;
+		goto err_no_space;
 	}
 
 	if (wptr + packet_size_in_dwords >= queue_size_dwords) {
 		/* make sure after rolling back to position 0, there is
 		 * still enough space.
 		 */
-		if (packet_size_in_dwords >= rptr) {
-			*buffer_ptr = NULL;
-			return -ENOMEM;
-		}
+		if (packet_size_in_dwords >= rptr)
+			goto err_no_space;
+
 		/* fill nops, roll back and start at position 0 */
 		while (wptr > 0) {
 			queue_address[wptr] = kq->nop_packet;
 			wptr = (wptr + 1) % queue_size_dwords;
+			wptr64++;
 		}
 	}
 
 	*buffer_ptr = &queue_address[wptr];
 	kq->pending_wptr = wptr + packet_size_in_dwords;
+	kq->pending_wptr64 = wptr64 + packet_size_in_dwords;
 
 	return 0;
+
+err_no_space:
+	*buffer_ptr = NULL;
+	return -ENOMEM;
 }
 
 static void submit_packet(struct kernel_queue *kq)
@@ -270,14 +276,18 @@ static void submit_packet(struct kernel_queue *kq)
 	pr_debug("\n");
 #endif
 
-	*kq->wptr_kernel = kq->pending_wptr;
-	write_kernel_doorbell(kq->queue->properties.doorbell_ptr,
-				kq->pending_wptr);
+	kq->ops_asic_specific.submit_packet(kq);
 }
 
 static void rollback_packet(struct kernel_queue *kq)
 {
-	kq->pending_wptr = *kq->queue->properties.write_ptr;
+	if (kq->dev->device_info->doorbell_size == 8) {
+		kq->pending_wptr64 = *kq->wptr64_kernel;
+		kq->pending_wptr = *kq->wptr_kernel %
+			(kq->queue->properties.queue_size / 4);
+	} else {
+		kq->pending_wptr = *kq->wptr_kernel;
+	}
 }
 
 struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
@@ -308,6 +318,11 @@ struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
 	case CHIP_HAWAII:
 		kernel_queue_init_cik(&kq->ops_asic_specific);
 		break;
+
+	case CHIP_VEGA10:
+	case CHIP_RAVEN:
+		kernel_queue_init_v9(&kq->ops_asic_specific);
+		break;
 	default:
 		WARN(1, "Unexpected ASIC family %u",
 		     dev->device_info->asic_family);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
index 594053136ee4..97aff2041a5d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
@@ -72,6 +72,7 @@ struct kernel_queue {
 	struct kfd_dev		*dev;
 	struct mqd_manager	*mqd;
 	struct queue		*queue;
+	uint64_t		pending_wptr64;
 	uint32_t		pending_wptr;
 	unsigned int		nop_packet;
 
@@ -79,7 +80,10 @@ struct kernel_queue {
 	uint32_t		*rptr_kernel;
 	uint64_t		rptr_gpu_addr;
 	struct kfd_mem_obj	*wptr_mem;
-	uint32_t		*wptr_kernel;
+	union {
+		uint64_t	*wptr64_kernel;
+		uint32_t	*wptr_kernel;
+	};
 	uint64_t		wptr_gpu_addr;
 	struct kfd_mem_obj	*pq;
 	uint64_t		pq_gpu_addr;
@@ -97,5 +101,6 @@ struct kernel_queue {
 
 void kernel_queue_init_cik(struct kernel_queue_ops *ops);
 void kernel_queue_init_vi(struct kernel_queue_ops *ops);
+void kernel_queue_init_v9(struct kernel_queue_ops *ops);
 
 #endif /* KFD_KERNEL_QUEUE_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_cik.c
index a90eb440b1fb..19e54acb4125 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_cik.c
@@ -26,11 +26,13 @@
 static bool initialize_cik(struct kernel_queue *kq, struct kfd_dev *dev,
 			enum kfd_queue_type type, unsigned int queue_size);
 static void uninitialize_cik(struct kernel_queue *kq);
+static void submit_packet_cik(struct kernel_queue *kq);
 
 void kernel_queue_init_cik(struct kernel_queue_ops *ops)
 {
 	ops->initialize = initialize_cik;
 	ops->uninitialize = uninitialize_cik;
+	ops->submit_packet = submit_packet_cik;
 }
 
 static bool initialize_cik(struct kernel_queue *kq, struct kfd_dev *dev,
@@ -42,3 +44,10 @@ static bool initialize_cik(struct kernel_queue *kq, struct kfd_dev *dev,
 static void uninitialize_cik(struct kernel_queue *kq)
 {
 }
+
+static void submit_packet_cik(struct kernel_queue *kq)
+{
+	*kq->wptr_kernel = kq->pending_wptr;
+	write_kernel_doorbell(kq->queue->properties.doorbell_ptr,
+				kq->pending_wptr);
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c
new file mode 100644
index 000000000000..684a3bf07efd
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c
@@ -0,0 +1,340 @@
+/*
+ * Copyright 2016-2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "kfd_kernel_queue.h"
+#include "kfd_device_queue_manager.h"
+#include "kfd_pm4_headers_ai.h"
+#include "kfd_pm4_opcodes.h"
+
+static bool initialize_v9(struct kernel_queue *kq, struct kfd_dev *dev,
+			enum kfd_queue_type type, unsigned int queue_size);
+static void uninitialize_v9(struct kernel_queue *kq);
+static void submit_packet_v9(struct kernel_queue *kq);
+
+void kernel_queue_init_v9(struct kernel_queue_ops *ops)
+{
+	ops->initialize = initialize_v9;
+	ops->uninitialize = uninitialize_v9;
+	ops->submit_packet = submit_packet_v9;
+}
+
+static bool initialize_v9(struct kernel_queue *kq, struct kfd_dev *dev,
+			enum kfd_queue_type type, unsigned int queue_size)
+{
+	int retval;
+
+	retval = kfd_gtt_sa_allocate(dev, PAGE_SIZE, &kq->eop_mem);
+	if (retval)
+		return false;
+
+	kq->eop_gpu_addr = kq->eop_mem->gpu_addr;
+	kq->eop_kernel_addr = kq->eop_mem->cpu_ptr;
+
+	memset(kq->eop_kernel_addr, 0, PAGE_SIZE);
+
+	return true;
+}
+
+static void uninitialize_v9(struct kernel_queue *kq)
+{
+	kfd_gtt_sa_free(kq->dev, kq->eop_mem);
+}
+
+static void submit_packet_v9(struct kernel_queue *kq)
+{
+	*kq->wptr64_kernel = kq->pending_wptr64;
+	write_kernel_doorbell64(kq->queue->properties.doorbell_ptr,
+				kq->pending_wptr64);
+}
+
+static int pm_map_process_v9(struct packet_manager *pm,
+		uint32_t *buffer, struct qcm_process_device *qpd)
+{
+	struct pm4_mes_map_process *packet;
+	uint64_t vm_page_table_base_addr =
+		(uint64_t)(qpd->page_table_base) << 12;
+
+	packet = (struct pm4_mes_map_process *)buffer;
+	memset(buffer, 0, sizeof(struct pm4_mes_map_process));
+
+	packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,
+					sizeof(struct pm4_mes_map_process));
+	packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
+	packet->bitfields2.process_quantum = 1;
+	packet->bitfields2.pasid = qpd->pqm->process->pasid;
+	packet->bitfields14.gds_size = qpd->gds_size;
+	packet->bitfields14.num_gws = qpd->num_gws;
+	packet->bitfields14.num_oac = qpd->num_oac;
+	packet->bitfields14.sdma_enable = 1;
+	packet->bitfields14.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;
+
+	packet->sh_mem_config = qpd->sh_mem_config;
+	packet->sh_mem_bases = qpd->sh_mem_bases;
+	packet->sq_shader_tba_lo = lower_32_bits(qpd->tba_addr >> 8);
+	packet->sq_shader_tba_hi = upper_32_bits(qpd->tba_addr >> 8);
+	packet->sq_shader_tma_lo = lower_32_bits(qpd->tma_addr >> 8);
+	packet->sq_shader_tma_hi = upper_32_bits(qpd->tma_addr >> 8);
+
+	packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
+	packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
+
+	packet->vm_context_page_table_base_addr_lo32 =
+			lower_32_bits(vm_page_table_base_addr);
+	packet->vm_context_page_table_base_addr_hi32 =
+			upper_32_bits(vm_page_table_base_addr);
+
+	return 0;
+}
+
+static int pm_runlist_v9(struct packet_manager *pm, uint32_t *buffer,
+			uint64_t ib, size_t ib_size_in_dwords, bool chain)
+{
+	struct pm4_mes_runlist *packet;
+
+	int concurrent_proc_cnt = 0;
+	struct kfd_dev *kfd = pm->dqm->dev;
+
+	/* Determine the number of processes to map together to HW:
+	 * it can not exceed the number of VMIDs available to the
+	 * scheduler, and it is determined by the smaller of the number
+	 * of processes in the runlist and kfd module parameter
+	 * hws_max_conc_proc.
+	 * Note: the arbitration between the number of VMIDs and
+	 * hws_max_conc_proc has been done in
+	 * kgd2kfd_device_init().
+	 */
+	concurrent_proc_cnt = min(pm->dqm->processes_count,
+			kfd->max_proc_per_quantum);
+
+	packet = (struct pm4_mes_runlist *)buffer;
+
+	memset(buffer, 0, sizeof(struct pm4_mes_runlist));
+	packet->header.u32All = pm_build_pm4_header(IT_RUN_LIST,
+						sizeof(struct pm4_mes_runlist));
+
+	packet->bitfields4.ib_size = ib_size_in_dwords;
+	packet->bitfields4.chain = chain ? 1 : 0;
+	packet->bitfields4.offload_polling = 0;
+	packet->bitfields4.valid = 1;
+	packet->bitfields4.process_cnt = concurrent_proc_cnt;
+	packet->ordinal2 = lower_32_bits(ib);
+	packet->ib_base_hi = upper_32_bits(ib);
+
+	return 0;
+}
+
+static int pm_map_queues_v9(struct packet_manager *pm, uint32_t *buffer,
+		struct queue *q, bool is_static)
+{
+	struct pm4_mes_map_queues *packet;
+	bool use_static = is_static;
+
+	packet = (struct pm4_mes_map_queues *)buffer;
+	memset(buffer, 0, sizeof(struct pm4_mes_map_queues));
+
+	packet->header.u32All = pm_build_pm4_header(IT_MAP_QUEUES,
+					sizeof(struct pm4_mes_map_queues));
+	packet->bitfields2.alloc_format =
+		alloc_format__mes_map_queues__one_per_pipe_vi;
+	packet->bitfields2.num_queues = 1;
+	packet->bitfields2.queue_sel =
+		queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi;
+
+	packet->bitfields2.engine_sel =
+		engine_sel__mes_map_queues__compute_vi;
+	packet->bitfields2.queue_type =
+		queue_type__mes_map_queues__normal_compute_vi;
+
+	switch (q->properties.type) {
+	case KFD_QUEUE_TYPE_COMPUTE:
+		if (use_static)
+			packet->bitfields2.queue_type =
+		queue_type__mes_map_queues__normal_latency_static_queue_vi;
+		break;
+	case KFD_QUEUE_TYPE_DIQ:
+		packet->bitfields2.queue_type =
+			queue_type__mes_map_queues__debug_interface_queue_vi;
+		break;
+	case KFD_QUEUE_TYPE_SDMA:
+		packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
+				engine_sel__mes_map_queues__sdma0_vi;
+		use_static = false; /* no static queues under SDMA */
+		break;
+	default:
+		WARN(1, "queue type %d", q->properties.type);
+		return -EINVAL;
+	}
+	packet->bitfields3.doorbell_offset =
+			q->properties.doorbell_off;
+
+	packet->mqd_addr_lo =
+			lower_32_bits(q->gart_mqd_addr);
+
+	packet->mqd_addr_hi =
+			upper_32_bits(q->gart_mqd_addr);
+
+	packet->wptr_addr_lo =
+			lower_32_bits((uint64_t)q->properties.write_ptr);
+
+	packet->wptr_addr_hi =
+			upper_32_bits((uint64_t)q->properties.write_ptr);
+
+	return 0;
+}
+
+static int pm_unmap_queues_v9(struct packet_manager *pm, uint32_t *buffer,
+			enum kfd_queue_type type,
+			enum kfd_unmap_queues_filter filter,
+			uint32_t filter_param, bool reset,
+			unsigned int sdma_engine)
+{
+	struct pm4_mes_unmap_queues *packet;
+
+	packet = (struct pm4_mes_unmap_queues *)buffer;
+	memset(buffer, 0, sizeof(struct pm4_mes_unmap_queues));
+
+	packet->header.u32All = pm_build_pm4_header(IT_UNMAP_QUEUES,
+					sizeof(struct pm4_mes_unmap_queues));
+	switch (type) {
+	case KFD_QUEUE_TYPE_COMPUTE:
+	case KFD_QUEUE_TYPE_DIQ:
+		packet->bitfields2.engine_sel =
+			engine_sel__mes_unmap_queues__compute;
+		break;
+	case KFD_QUEUE_TYPE_SDMA:
+		packet->bitfields2.engine_sel =
+			engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
+		break;
+	default:
+		WARN(1, "queue type %d", type);
+		return -EINVAL;
+	}
+
+	if (reset)
+		packet->bitfields2.action =
+			action__mes_unmap_queues__reset_queues;
+	else
+		packet->bitfields2.action =
+			action__mes_unmap_queues__preempt_queues;
+
+	switch (filter) {
+	case KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE:
+		packet->bitfields2.queue_sel =
+			queue_sel__mes_unmap_queues__perform_request_on_specified_queues;
+		packet->bitfields2.num_queues = 1;
+		packet->bitfields3b.doorbell_offset0 = filter_param;
+		break;
+	case KFD_UNMAP_QUEUES_FILTER_BY_PASID:
+		packet->bitfields2.queue_sel =
+			queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;
+		packet->bitfields3a.pasid = filter_param;
+		break;
+	case KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES:
+		packet->bitfields2.queue_sel =
+			queue_sel__mes_unmap_queues__unmap_all_queues;
+		break;
+	case KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES:
+		/* in this case, we do not preempt static queues */
+		packet->bitfields2.queue_sel =
+			queue_sel__mes_unmap_queues__unmap_all_non_static_queues;
+		break;
+	default:
+		WARN(1, "filter %d", filter);
+		return -EINVAL;
+	}
+
+	return 0;
+
+}
+
+static int pm_query_status_v9(struct packet_manager *pm, uint32_t *buffer,
+			uint64_t fence_address,	uint32_t fence_value)
+{
+	struct pm4_mes_query_status *packet;
+
+	packet = (struct pm4_mes_query_status *)buffer;
+	memset(buffer, 0, sizeof(struct pm4_mes_query_status));
+
+
+	packet->header.u32All = pm_build_pm4_header(IT_QUERY_STATUS,
+					sizeof(struct pm4_mes_query_status));
+
+	packet->bitfields2.context_id = 0;
+	packet->bitfields2.interrupt_sel =
+			interrupt_sel__mes_query_status__completion_status;
+	packet->bitfields2.command =
+			command__mes_query_status__fence_only_after_write_ack;
+
+	packet->addr_hi = upper_32_bits((uint64_t)fence_address);
+	packet->addr_lo = lower_32_bits((uint64_t)fence_address);
+	packet->data_hi = upper_32_bits((uint64_t)fence_value);
+	packet->data_lo = lower_32_bits((uint64_t)fence_value);
+
+	return 0;
+}
+
+
+static int pm_release_mem_v9(uint64_t gpu_addr, uint32_t *buffer)
+{
+	struct pm4_mec_release_mem *packet;
+
+	packet = (struct pm4_mec_release_mem *)buffer;
+	memset(buffer, 0, sizeof(struct pm4_mec_release_mem));
+
+	packet->header.u32All = pm_build_pm4_header(IT_RELEASE_MEM,
+					sizeof(struct pm4_mec_release_mem));
+
+	packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
+	packet->bitfields2.event_index = event_index__mec_release_mem__end_of_pipe;
+	packet->bitfields2.tcl1_action_ena = 1;
+	packet->bitfields2.tc_action_ena = 1;
+	packet->bitfields2.cache_policy = cache_policy__mec_release_mem__lru;
+
+	packet->bitfields3.data_sel = data_sel__mec_release_mem__send_32_bit_low;
+	packet->bitfields3.int_sel =
+		int_sel__mec_release_mem__send_interrupt_after_write_confirm;
+
+	packet->bitfields4.address_lo_32b = (gpu_addr & 0xffffffff) >> 2;
+	packet->address_hi = upper_32_bits(gpu_addr);
+
+	packet->data_lo = 0;
+
+	return 0;
+}
+
+const struct packet_manager_funcs kfd_v9_pm_funcs = {
+	.map_process		= pm_map_process_v9,
+	.runlist		= pm_runlist_v9,
+	.set_resources		= pm_set_resources_vi,
+	.map_queues		= pm_map_queues_v9,
+	.unmap_queues		= pm_unmap_queues_v9,
+	.query_status		= pm_query_status_v9,
+	.release_mem		= pm_release_mem_v9,
+	.map_process_size	= sizeof(struct pm4_mes_map_process),
+	.runlist_size		= sizeof(struct pm4_mes_runlist),
+	.set_resources_size	= sizeof(struct pm4_mes_set_resources),
+	.map_queues_size	= sizeof(struct pm4_mes_map_queues),
+	.unmap_queues_size	= sizeof(struct pm4_mes_unmap_queues),
+	.query_status_size	= sizeof(struct pm4_mes_query_status),
+	.release_mem_size	= sizeof(struct pm4_mec_release_mem)
+};
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c
index f1d48281e322..bf20c6d32ef3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c
@@ -22,15 +22,20 @@
  */
 
 #include "kfd_kernel_queue.h"
+#include "kfd_device_queue_manager.h"
+#include "kfd_pm4_headers_vi.h"
+#include "kfd_pm4_opcodes.h"
 
 static bool initialize_vi(struct kernel_queue *kq, struct kfd_dev *dev,
 			enum kfd_queue_type type, unsigned int queue_size);
 static void uninitialize_vi(struct kernel_queue *kq);
+static void submit_packet_vi(struct kernel_queue *kq);
 
 void kernel_queue_init_vi(struct kernel_queue_ops *ops)
 {
 	ops->initialize = initialize_vi;
 	ops->uninitialize = uninitialize_vi;
+	ops->submit_packet = submit_packet_vi;
 }
 
 static bool initialize_vi(struct kernel_queue *kq, struct kfd_dev *dev,
@@ -54,3 +59,317 @@ static void uninitialize_vi(struct kernel_queue *kq)
 {
 	kfd_gtt_sa_free(kq->dev, kq->eop_mem);
 }
+
+static void submit_packet_vi(struct kernel_queue *kq)
+{
+	*kq->wptr_kernel = kq->pending_wptr;
+	write_kernel_doorbell(kq->queue->properties.doorbell_ptr,
+				kq->pending_wptr);
+}
+
+unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size)
+{
+	union PM4_MES_TYPE_3_HEADER header;
+
+	header.u32All = 0;
+	header.opcode = opcode;
+	header.count = packet_size / 4 - 2;
+	header.type = PM4_TYPE_3;
+
+	return header.u32All;
+}
+
+static int pm_map_process_vi(struct packet_manager *pm, uint32_t *buffer,
+				struct qcm_process_device *qpd)
+{
+	struct pm4_mes_map_process *packet;
+
+	packet = (struct pm4_mes_map_process *)buffer;
+
+	memset(buffer, 0, sizeof(struct pm4_mes_map_process));
+
+	packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,
+					sizeof(struct pm4_mes_map_process));
+	packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
+	packet->bitfields2.process_quantum = 1;
+	packet->bitfields2.pasid = qpd->pqm->process->pasid;
+	packet->bitfields3.page_table_base = qpd->page_table_base;
+	packet->bitfields10.gds_size = qpd->gds_size;
+	packet->bitfields10.num_gws = qpd->num_gws;
+	packet->bitfields10.num_oac = qpd->num_oac;
+	packet->bitfields10.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;
+
+	packet->sh_mem_config = qpd->sh_mem_config;
+	packet->sh_mem_bases = qpd->sh_mem_bases;
+	packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
+	packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
+
+	packet->sh_hidden_private_base_vmid = qpd->sh_hidden_private_base;
+
+	packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
+	packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
+
+	return 0;
+}
+
+static int pm_runlist_vi(struct packet_manager *pm, uint32_t *buffer,
+			uint64_t ib, size_t ib_size_in_dwords, bool chain)
+{
+	struct pm4_mes_runlist *packet;
+	int concurrent_proc_cnt = 0;
+	struct kfd_dev *kfd = pm->dqm->dev;
+
+	if (WARN_ON(!ib))
+		return -EFAULT;
+
+	/* Determine the number of processes to map together to HW:
+	 * it can not exceed the number of VMIDs available to the
+	 * scheduler, and it is determined by the smaller of the number
+	 * of processes in the runlist and kfd module parameter
+	 * hws_max_conc_proc.
+	 * Note: the arbitration between the number of VMIDs and
+	 * hws_max_conc_proc has been done in
+	 * kgd2kfd_device_init().
+	 */
+	concurrent_proc_cnt = min(pm->dqm->processes_count,
+			kfd->max_proc_per_quantum);
+
+	packet = (struct pm4_mes_runlist *)buffer;
+
+	memset(buffer, 0, sizeof(struct pm4_mes_runlist));
+	packet->header.u32All = pm_build_pm4_header(IT_RUN_LIST,
+						sizeof(struct pm4_mes_runlist));
+
+	packet->bitfields4.ib_size = ib_size_in_dwords;
+	packet->bitfields4.chain = chain ? 1 : 0;
+	packet->bitfields4.offload_polling = 0;
+	packet->bitfields4.valid = 1;
+	packet->bitfields4.process_cnt = concurrent_proc_cnt;
+	packet->ordinal2 = lower_32_bits(ib);
+	packet->bitfields3.ib_base_hi = upper_32_bits(ib);
+
+	return 0;
+}
+
+int pm_set_resources_vi(struct packet_manager *pm, uint32_t *buffer,
+				struct scheduling_resources *res)
+{
+	struct pm4_mes_set_resources *packet;
+
+	packet = (struct pm4_mes_set_resources *)buffer;
+	memset(buffer, 0, sizeof(struct pm4_mes_set_resources));
+
+	packet->header.u32All = pm_build_pm4_header(IT_SET_RESOURCES,
+					sizeof(struct pm4_mes_set_resources));
+
+	packet->bitfields2.queue_type =
+			queue_type__mes_set_resources__hsa_interface_queue_hiq;
+	packet->bitfields2.vmid_mask = res->vmid_mask;
+	packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY_MS / 100;
+	packet->bitfields7.oac_mask = res->oac_mask;
+	packet->bitfields8.gds_heap_base = res->gds_heap_base;
+	packet->bitfields8.gds_heap_size = res->gds_heap_size;
+
+	packet->gws_mask_lo = lower_32_bits(res->gws_mask);
+	packet->gws_mask_hi = upper_32_bits(res->gws_mask);
+
+	packet->queue_mask_lo = lower_32_bits(res->queue_mask);
+	packet->queue_mask_hi = upper_32_bits(res->queue_mask);
+
+	return 0;
+}
+
+static int pm_map_queues_vi(struct packet_manager *pm, uint32_t *buffer,
+		struct queue *q, bool is_static)
+{
+	struct pm4_mes_map_queues *packet;
+	bool use_static = is_static;
+
+	packet = (struct pm4_mes_map_queues *)buffer;
+	memset(buffer, 0, sizeof(struct pm4_mes_map_queues));
+
+	packet->header.u32All = pm_build_pm4_header(IT_MAP_QUEUES,
+					sizeof(struct pm4_mes_map_queues));
+	packet->bitfields2.alloc_format =
+		alloc_format__mes_map_queues__one_per_pipe_vi;
+	packet->bitfields2.num_queues = 1;
+	packet->bitfields2.queue_sel =
+		queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi;
+
+	packet->bitfields2.engine_sel =
+		engine_sel__mes_map_queues__compute_vi;
+	packet->bitfields2.queue_type =
+		queue_type__mes_map_queues__normal_compute_vi;
+
+	switch (q->properties.type) {
+	case KFD_QUEUE_TYPE_COMPUTE:
+		if (use_static)
+			packet->bitfields2.queue_type =
+		queue_type__mes_map_queues__normal_latency_static_queue_vi;
+		break;
+	case KFD_QUEUE_TYPE_DIQ:
+		packet->bitfields2.queue_type =
+			queue_type__mes_map_queues__debug_interface_queue_vi;
+		break;
+	case KFD_QUEUE_TYPE_SDMA:
+		packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
+				engine_sel__mes_map_queues__sdma0_vi;
+		use_static = false; /* no static queues under SDMA */
+		break;
+	default:
+		WARN(1, "queue type %d", q->properties.type);
+		return -EINVAL;
+	}
+	packet->bitfields3.doorbell_offset =
+			q->properties.doorbell_off;
+
+	packet->mqd_addr_lo =
+			lower_32_bits(q->gart_mqd_addr);
+
+	packet->mqd_addr_hi =
+			upper_32_bits(q->gart_mqd_addr);
+
+	packet->wptr_addr_lo =
+			lower_32_bits((uint64_t)q->properties.write_ptr);
+
+	packet->wptr_addr_hi =
+			upper_32_bits((uint64_t)q->properties.write_ptr);
+
+	return 0;
+}
+
+static int pm_unmap_queues_vi(struct packet_manager *pm, uint32_t *buffer,
+			enum kfd_queue_type type,
+			enum kfd_unmap_queues_filter filter,
+			uint32_t filter_param, bool reset,
+			unsigned int sdma_engine)
+{
+	struct pm4_mes_unmap_queues *packet;
+
+	packet = (struct pm4_mes_unmap_queues *)buffer;
+	memset(buffer, 0, sizeof(struct pm4_mes_unmap_queues));
+
+	packet->header.u32All = pm_build_pm4_header(IT_UNMAP_QUEUES,
+					sizeof(struct pm4_mes_unmap_queues));
+	switch (type) {
+	case KFD_QUEUE_TYPE_COMPUTE:
+	case KFD_QUEUE_TYPE_DIQ:
+		packet->bitfields2.engine_sel =
+			engine_sel__mes_unmap_queues__compute;
+		break;
+	case KFD_QUEUE_TYPE_SDMA:
+		packet->bitfields2.engine_sel =
+			engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
+		break;
+	default:
+		WARN(1, "queue type %d", type);
+		return -EINVAL;
+	}
+
+	if (reset)
+		packet->bitfields2.action =
+			action__mes_unmap_queues__reset_queues;
+	else
+		packet->bitfields2.action =
+			action__mes_unmap_queues__preempt_queues;
+
+	switch (filter) {
+	case KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE:
+		packet->bitfields2.queue_sel =
+			queue_sel__mes_unmap_queues__perform_request_on_specified_queues;
+		packet->bitfields2.num_queues = 1;
+		packet->bitfields3b.doorbell_offset0 = filter_param;
+		break;
+	case KFD_UNMAP_QUEUES_FILTER_BY_PASID:
+		packet->bitfields2.queue_sel =
+			queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;
+		packet->bitfields3a.pasid = filter_param;
+		break;
+	case KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES:
+		packet->bitfields2.queue_sel =
+			queue_sel__mes_unmap_queues__unmap_all_queues;
+		break;
+	case KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES:
+		/* in this case, we do not preempt static queues */
+		packet->bitfields2.queue_sel =
+			queue_sel__mes_unmap_queues__unmap_all_non_static_queues;
+		break;
+	default:
+		WARN(1, "filter %d", filter);
+		return -EINVAL;
+	}
+
+	return 0;
+
+}
+
+static int pm_query_status_vi(struct packet_manager *pm, uint32_t *buffer,
+			uint64_t fence_address,	uint32_t fence_value)
+{
+	struct pm4_mes_query_status *packet;
+
+	packet = (struct pm4_mes_query_status *)buffer;
+	memset(buffer, 0, sizeof(struct pm4_mes_query_status));
+
+	packet->header.u32All = pm_build_pm4_header(IT_QUERY_STATUS,
+					sizeof(struct pm4_mes_query_status));
+
+	packet->bitfields2.context_id = 0;
+	packet->bitfields2.interrupt_sel =
+			interrupt_sel__mes_query_status__completion_status;
+	packet->bitfields2.command =
+			command__mes_query_status__fence_only_after_write_ack;
+
+	packet->addr_hi = upper_32_bits((uint64_t)fence_address);
+	packet->addr_lo = lower_32_bits((uint64_t)fence_address);
+	packet->data_hi = upper_32_bits((uint64_t)fence_value);
+	packet->data_lo = lower_32_bits((uint64_t)fence_value);
+
+	return 0;
+}
+
+static int pm_release_mem_vi(uint64_t gpu_addr, uint32_t *buffer)
+{
+	struct pm4_mec_release_mem *packet;
+
+	packet = (struct pm4_mec_release_mem *)buffer;
+	memset(buffer, 0, sizeof(*packet));
+
+	packet->header.u32All = pm_build_pm4_header(IT_RELEASE_MEM,
+						 sizeof(*packet));
+
+	packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
+	packet->bitfields2.event_index = event_index___release_mem__end_of_pipe;
+	packet->bitfields2.tcl1_action_ena = 1;
+	packet->bitfields2.tc_action_ena = 1;
+	packet->bitfields2.cache_policy = cache_policy___release_mem__lru;
+	packet->bitfields2.atc = 0;
+
+	packet->bitfields3.data_sel = data_sel___release_mem__send_32_bit_low;
+	packet->bitfields3.int_sel =
+		int_sel___release_mem__send_interrupt_after_write_confirm;
+
+	packet->bitfields4.address_lo_32b = (gpu_addr & 0xffffffff) >> 2;
+	packet->address_hi = upper_32_bits(gpu_addr);
+
+	packet->data_lo = 0;
+
+	return 0;
+}
+
+const struct packet_manager_funcs kfd_vi_pm_funcs = {
+	.map_process		= pm_map_process_vi,
+	.runlist		= pm_runlist_vi,
+	.set_resources		= pm_set_resources_vi,
+	.map_queues		= pm_map_queues_vi,
+	.unmap_queues		= pm_unmap_queues_vi,
+	.query_status		= pm_query_status_vi,
+	.release_mem		= pm_release_mem_vi,
+	.map_process_size	= sizeof(struct pm4_mes_map_process),
+	.runlist_size		= sizeof(struct pm4_mes_runlist),
+	.set_resources_size	= sizeof(struct pm4_mes_set_resources),
+	.map_queues_size	= sizeof(struct pm4_mes_map_queues),
+	.unmap_queues_size	= sizeof(struct pm4_mes_unmap_queues),
+	.query_status_size	= sizeof(struct pm4_mes_query_status),
+	.release_mem_size	= sizeof(struct pm4_mec_release_mem)
+};
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
index e0c07d24d251..76bf2dc8aec4 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
@@ -43,6 +43,8 @@ static const struct kgd2kfd_calls kgd2kfd = {
 	.interrupt	= kgd2kfd_interrupt,
 	.suspend	= kgd2kfd_suspend,
 	.resume		= kgd2kfd_resume,
+	.quiesce_mm	= kgd2kfd_quiesce_mm,
+	.resume_mm	= kgd2kfd_resume_mm,
 	.schedule_evict_and_restore_process =
 			  kgd2kfd_schedule_evict_and_restore_process,
 };
@@ -81,6 +83,11 @@ module_param(ignore_crat, int, 0444);
 MODULE_PARM_DESC(ignore_crat,
 	"Ignore CRAT table during KFD initialization (0 = use CRAT (default), 1 = ignore CRAT)");
 
+int vega10_noretry;
+module_param_named(noretry, vega10_noretry, int, 0644);
+MODULE_PARM_DESC(noretry,
+	"Set sh_mem_config.retry_disable on Vega10 (0 = retry enabled (default), 1 = retry disabled)");
+
 static int amdkfd_init_completed;
 
 int kgd2kfd_init(unsigned int interface_version,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
index ee7061e1c466..4b8eb506642b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
@@ -38,6 +38,9 @@ struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
 	case CHIP_POLARIS10:
 	case CHIP_POLARIS11:
 		return mqd_manager_init_vi_tonga(type, dev);
+	case CHIP_VEGA10:
+	case CHIP_RAVEN:
+		return mqd_manager_init_v9(type, dev);
 	default:
 		WARN(1, "Unexpected ASIC family %u",
 		     dev->device_info->asic_family);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index c00c325ed3c9..06eaa218eba6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -79,10 +79,6 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
 	m->cp_mqd_base_addr_lo        = lower_32_bits(addr);
 	m->cp_mqd_base_addr_hi        = upper_32_bits(addr);
 
-	m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN;
-	/* Although WinKFD writes this, I suspect it should not be necessary */
-	m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE;
-
 	m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
 				QUANTUM_DURATION(10);
 
@@ -412,7 +408,7 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
 	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
 		return NULL;
 
-	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
+	mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
 	if (!mqd)
 		return NULL;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
new file mode 100644
index 000000000000..684054ff02cd
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -0,0 +1,443 @@
+/*
+ * Copyright 2016-2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include "kfd_priv.h"
+#include "kfd_mqd_manager.h"
+#include "v9_structs.h"
+#include "gc/gc_9_0_offset.h"
+#include "gc/gc_9_0_sh_mask.h"
+#include "sdma0/sdma0_4_0_sh_mask.h"
+
+static inline struct v9_mqd *get_mqd(void *mqd)
+{
+	return (struct v9_mqd *)mqd;
+}
+
+static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)
+{
+	return (struct v9_sdma_mqd *)mqd;
+}
+
+static int init_mqd(struct mqd_manager *mm, void **mqd,
+			struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+			struct queue_properties *q)
+{
+	int retval;
+	uint64_t addr;
+	struct v9_mqd *m;
+	struct kfd_dev *kfd = mm->dev;
+
+	/* From V9,  for CWSR, the control stack is located on the next page
+	 * boundary after the mqd, we will use the gtt allocation function
+	 * instead of sub-allocation function.
+	 */
+	if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
+		*mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
+		if (!*mqd_mem_obj)
+			return -ENOMEM;
+		retval = kfd->kfd2kgd->init_gtt_mem_allocation(kfd->kgd,
+			ALIGN(q->ctl_stack_size, PAGE_SIZE) +
+				ALIGN(sizeof(struct v9_mqd), PAGE_SIZE),
+			&((*mqd_mem_obj)->gtt_mem),
+			&((*mqd_mem_obj)->gpu_addr),
+			(void *)&((*mqd_mem_obj)->cpu_ptr));
+	} else
+		retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct v9_mqd),
+				mqd_mem_obj);
+	if (retval != 0)
+		return -ENOMEM;
+
+	m = (struct v9_mqd *) (*mqd_mem_obj)->cpu_ptr;
+	addr = (*mqd_mem_obj)->gpu_addr;
+
+	memset(m, 0, sizeof(struct v9_mqd));
+
+	m->header = 0xC0310800;
+	m->compute_pipelinestat_enable = 1;
+	m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
+
+	m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK |
+			0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT;
+
+	m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT;
+
+	m->cp_mqd_base_addr_lo        = lower_32_bits(addr);
+	m->cp_mqd_base_addr_hi        = upper_32_bits(addr);
+
+	m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT |
+			1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
+			10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
+
+	m->cp_hqd_pipe_priority = 1;
+	m->cp_hqd_queue_priority = 15;
+
+	if (q->format == KFD_QUEUE_FORMAT_AQL) {
+		m->cp_hqd_aql_control =
+			1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
+	}
+
+	if (q->tba_addr) {
+		m->compute_pgm_rsrc2 |=
+			(1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
+	}
+
+	if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address) {
+		m->cp_hqd_persistent_state |=
+			(1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
+		m->cp_hqd_ctx_save_base_addr_lo =
+			lower_32_bits(q->ctx_save_restore_area_address);
+		m->cp_hqd_ctx_save_base_addr_hi =
+			upper_32_bits(q->ctx_save_restore_area_address);
+		m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;
+		m->cp_hqd_cntl_stack_size = q->ctl_stack_size;
+		m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;
+		m->cp_hqd_wg_state_offset = q->ctl_stack_size;
+	}
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+	retval = mm->update_mqd(mm, m, q);
+
+	return retval;
+}
+
+static int load_mqd(struct mqd_manager *mm, void *mqd,
+			uint32_t pipe_id, uint32_t queue_id,
+			struct queue_properties *p, struct mm_struct *mms)
+{
+	/* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
+	uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
+
+	return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
+					  (uint32_t __user *)p->write_ptr,
+					  wptr_shift, 0, mms);
+}
+
+static int update_mqd(struct mqd_manager *mm, void *mqd,
+		      struct queue_properties *q)
+{
+	struct v9_mqd *m;
+
+	m = get_mqd(mqd);
+
+	m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT;
+	m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
+	pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
+
+	m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
+	m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
+
+	m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+	m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+	m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
+	m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
+
+	m->cp_hqd_pq_doorbell_control =
+		q->doorbell_off <<
+			CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+	pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
+			m->cp_hqd_pq_doorbell_control);
+
+	m->cp_hqd_ib_control =
+		3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT |
+		1 << CP_HQD_IB_CONTROL__IB_EXE_DISABLE__SHIFT;
+
+	/*
+	 * HW does not clamp this field correctly. Maximum EOP queue size
+	 * is constrained by per-SE EOP done signal count, which is 8-bit.
+	 * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit
+	 * more than (EOP entry count - 1) so a queue size of 0x800 dwords
+	 * is safe, giving a maximum field value of 0xA.
+	 */
+	m->cp_hqd_eop_control = min(0xA,
+		order_base_2(q->eop_ring_buffer_size / 4) - 1);
+	m->cp_hqd_eop_base_addr_lo =
+			lower_32_bits(q->eop_ring_buffer_address >> 8);
+	m->cp_hqd_eop_base_addr_hi =
+			upper_32_bits(q->eop_ring_buffer_address >> 8);
+
+	m->cp_hqd_iq_timer = 0;
+
+	m->cp_hqd_vmid = q->vmid;
+
+	if (q->format == KFD_QUEUE_FORMAT_AQL) {
+		m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
+				2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT |
+				1 << CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT |
+				1 << CP_HQD_PQ_CONTROL__WPP_CLAMP_EN__SHIFT;
+		m->cp_hqd_pq_doorbell_control |= 1 <<
+			CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT;
+	}
+	if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address)
+		m->cp_hqd_ctx_save_control = 0;
+
+	q->is_active = (q->queue_size > 0 &&
+			q->queue_address != 0 &&
+			q->queue_percent > 0 &&
+			!q->is_evicted);
+
+	return 0;
+}
+
+
+static int destroy_mqd(struct mqd_manager *mm, void *mqd,
+			enum kfd_preempt_type type,
+			unsigned int timeout, uint32_t pipe_id,
+			uint32_t queue_id)
+{
+	return mm->dev->kfd2kgd->hqd_destroy
+		(mm->dev->kgd, mqd, type, timeout,
+		pipe_id, queue_id);
+}
+
+static void uninit_mqd(struct mqd_manager *mm, void *mqd,
+			struct kfd_mem_obj *mqd_mem_obj)
+{
+	struct kfd_dev *kfd = mm->dev;
+
+	if (mqd_mem_obj->gtt_mem) {
+		kfd->kfd2kgd->free_gtt_mem(kfd->kgd, mqd_mem_obj->gtt_mem);
+		kfree(mqd_mem_obj);
+	} else {
+		kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+	}
+}
+
+static bool is_occupied(struct mqd_manager *mm, void *mqd,
+			uint64_t queue_address,	uint32_t pipe_id,
+			uint32_t queue_id)
+{
+	return mm->dev->kfd2kgd->hqd_is_occupied(
+		mm->dev->kgd, queue_address,
+		pipe_id, queue_id);
+}
+
+static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
+			struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+			struct queue_properties *q)
+{
+	struct v9_mqd *m;
+	int retval = init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
+
+	if (retval != 0)
+		return retval;
+
+	m = get_mqd(*mqd);
+
+	m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
+			1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
+
+	return retval;
+}
+
+static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
+			struct queue_properties *q)
+{
+	struct v9_mqd *m;
+	int retval = update_mqd(mm, mqd, q);
+
+	if (retval != 0)
+		return retval;
+
+	/* TODO: what's the point? update_mqd already does this. */
+	m = get_mqd(mqd);
+	m->cp_hqd_vmid = q->vmid;
+	return retval;
+}
+
+static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+		struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+		struct queue_properties *q)
+{
+	int retval;
+	struct v9_sdma_mqd *m;
+
+
+	retval = kfd_gtt_sa_allocate(mm->dev,
+			sizeof(struct v9_sdma_mqd),
+			mqd_mem_obj);
+
+	if (retval != 0)
+		return -ENOMEM;
+
+	m = (struct v9_sdma_mqd *) (*mqd_mem_obj)->cpu_ptr;
+
+	memset(m, 0, sizeof(struct v9_sdma_mqd));
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = (*mqd_mem_obj)->gpu_addr;
+
+	retval = mm->update_mqd(mm, m, q);
+
+	return retval;
+}
+
+static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		struct kfd_mem_obj *mqd_mem_obj)
+{
+	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+}
+
+static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		uint32_t pipe_id, uint32_t queue_id,
+		struct queue_properties *p, struct mm_struct *mms)
+{
+	return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
+					       (uint32_t __user *)p->write_ptr,
+					       mms);
+}
+
+#define SDMA_RLC_DUMMY_DEFAULT 0xf
+
+static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		struct queue_properties *q)
+{
+	struct v9_sdma_mqd *m;
+
+	m = get_sdma_mqd(mqd);
+	m->sdmax_rlcx_rb_cntl = order_base_2(q->queue_size / 4)
+		<< SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
+		q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
+		1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
+		6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
+
+	m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
+	m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
+	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+	m->sdmax_rlcx_doorbell_offset =
+		q->doorbell_off << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;
+
+	m->sdma_engine_id = q->sdma_engine_id;
+	m->sdma_queue_id = q->sdma_queue_id;
+	m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT;
+
+	q->is_active = (q->queue_size > 0 &&
+			q->queue_address != 0 &&
+			q->queue_percent > 0 &&
+			!q->is_evicted);
+
+	return 0;
+}
+
+/*
+ *  * preempt type here is ignored because there is only one way
+ *  * to preempt sdma queue
+ */
+static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		enum kfd_preempt_type type,
+		unsigned int timeout, uint32_t pipe_id,
+		uint32_t queue_id)
+{
+	return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
+}
+
+static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
+		uint64_t queue_address, uint32_t pipe_id,
+		uint32_t queue_id)
+{
+	return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
+}
+
+#if defined(CONFIG_DEBUG_FS)
+
+static int debugfs_show_mqd(struct seq_file *m, void *data)
+{
+	seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
+		     data, sizeof(struct v9_mqd), false);
+	return 0;
+}
+
+static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
+{
+	seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
+		     data, sizeof(struct v9_sdma_mqd), false);
+	return 0;
+}
+
+#endif
+
+struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
+		struct kfd_dev *dev)
+{
+	struct mqd_manager *mqd;
+
+	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
+		return NULL;
+
+	mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
+	if (!mqd)
+		return NULL;
+
+	mqd->dev = dev;
+
+	switch (type) {
+	case KFD_MQD_TYPE_CP:
+	case KFD_MQD_TYPE_COMPUTE:
+		mqd->init_mqd = init_mqd;
+		mqd->uninit_mqd = uninit_mqd;
+		mqd->load_mqd = load_mqd;
+		mqd->update_mqd = update_mqd;
+		mqd->destroy_mqd = destroy_mqd;
+		mqd->is_occupied = is_occupied;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
+		break;
+	case KFD_MQD_TYPE_HIQ:
+		mqd->init_mqd = init_mqd_hiq;
+		mqd->uninit_mqd = uninit_mqd;
+		mqd->load_mqd = load_mqd;
+		mqd->update_mqd = update_mqd_hiq;
+		mqd->destroy_mqd = destroy_mqd;
+		mqd->is_occupied = is_occupied;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
+		break;
+	case KFD_MQD_TYPE_SDMA:
+		mqd->init_mqd = init_mqd_sdma;
+		mqd->uninit_mqd = uninit_mqd_sdma;
+		mqd->load_mqd = load_mqd_sdma;
+		mqd->update_mqd = update_mqd_sdma;
+		mqd->destroy_mqd = destroy_mqd_sdma;
+		mqd->is_occupied = is_occupied_sdma;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
+#endif
+		break;
+	default:
+		kfree(mqd);
+		return NULL;
+	}
+
+	return mqd;
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 89e4242e43e7..481307b8b4db 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -394,7 +394,7 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
 		return NULL;
 
-	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
+	mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
 	if (!mqd)
 		return NULL;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
index 89ba4c670ec5..c317feb43f69 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
@@ -26,8 +26,6 @@
 #include "kfd_device_queue_manager.h"
 #include "kfd_kernel_queue.h"
 #include "kfd_priv.h"
-#include "kfd_pm4_headers_vi.h"
-#include "kfd_pm4_opcodes.h"
 
 static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes,
 				unsigned int buffer_size_bytes)
@@ -39,18 +37,6 @@ static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes,
 	*wptr = temp;
 }
 
-static unsigned int build_pm4_header(unsigned int opcode, size_t packet_size)
-{
-	union PM4_MES_TYPE_3_HEADER header;
-
-	header.u32All = 0;
-	header.opcode = opcode;
-	header.count = packet_size / 4 - 2;
-	header.type = PM4_TYPE_3;
-
-	return header.u32All;
-}
-
 static void pm_calc_rlib_size(struct packet_manager *pm,
 				unsigned int *rlib_size,
 				bool *over_subscription)
@@ -80,9 +66,9 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
 		pr_debug("Over subscribed runlist\n");
 	}
 
-	map_queue_size = sizeof(struct pm4_mes_map_queues);
+	map_queue_size = pm->pmf->map_queues_size;
 	/* calculate run list ib allocation size */
-	*rlib_size = process_count * sizeof(struct pm4_mes_map_process) +
+	*rlib_size = process_count * pm->pmf->map_process_size +
 		     queue_count * map_queue_size;
 
 	/*
@@ -90,7 +76,7 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
 	 * when over subscription
 	 */
 	if (*over_subscription)
-		*rlib_size += sizeof(struct pm4_mes_runlist);
+		*rlib_size += pm->pmf->runlist_size;
 
 	pr_debug("runlist ib size %d\n", *rlib_size);
 }
@@ -108,12 +94,14 @@ static int pm_allocate_runlist_ib(struct packet_manager *pm,
 
 	pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription);
 
+	mutex_lock(&pm->lock);
+
 	retval = kfd_gtt_sa_allocate(pm->dqm->dev, *rl_buffer_size,
 					&pm->ib_buffer_obj);
 
 	if (retval) {
 		pr_err("Failed to allocate runlist IB\n");
-		return retval;
+		goto out;
 	}
 
 	*(void **)rl_buffer = pm->ib_buffer_obj->cpu_ptr;
@@ -121,138 +109,10 @@ static int pm_allocate_runlist_ib(struct packet_manager *pm,
 
 	memset(*rl_buffer, 0, *rl_buffer_size);
 	pm->allocated = true;
-	return retval;
-}
-
-static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer,
-			uint64_t ib, size_t ib_size_in_dwords, bool chain)
-{
-	struct pm4_mes_runlist *packet;
-	int concurrent_proc_cnt = 0;
-	struct kfd_dev *kfd = pm->dqm->dev;
-
-	if (WARN_ON(!ib))
-		return -EFAULT;
-
-	/* Determine the number of processes to map together to HW:
-	 * it can not exceed the number of VMIDs available to the
-	 * scheduler, and it is determined by the smaller of the number
-	 * of processes in the runlist and kfd module parameter
-	 * hws_max_conc_proc.
-	 * Note: the arbitration between the number of VMIDs and
-	 * hws_max_conc_proc has been done in
-	 * kgd2kfd_device_init().
-	 */
-	concurrent_proc_cnt = min(pm->dqm->processes_count,
-			kfd->max_proc_per_quantum);
-
-	packet = (struct pm4_mes_runlist *)buffer;
-
-	memset(buffer, 0, sizeof(struct pm4_mes_runlist));
-	packet->header.u32All = build_pm4_header(IT_RUN_LIST,
-						sizeof(struct pm4_mes_runlist));
-
-	packet->bitfields4.ib_size = ib_size_in_dwords;
-	packet->bitfields4.chain = chain ? 1 : 0;
-	packet->bitfields4.offload_polling = 0;
-	packet->bitfields4.valid = 1;
-	packet->bitfields4.process_cnt = concurrent_proc_cnt;
-	packet->ordinal2 = lower_32_bits(ib);
-	packet->bitfields3.ib_base_hi = upper_32_bits(ib);
-
-	return 0;
-}
-
-static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,
-				struct qcm_process_device *qpd)
-{
-	struct pm4_mes_map_process *packet;
-
-	packet = (struct pm4_mes_map_process *)buffer;
 
-	memset(buffer, 0, sizeof(struct pm4_mes_map_process));
-
-	packet->header.u32All = build_pm4_header(IT_MAP_PROCESS,
-					sizeof(struct pm4_mes_map_process));
-	packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
-	packet->bitfields2.process_quantum = 1;
-	packet->bitfields2.pasid = qpd->pqm->process->pasid;
-	packet->bitfields3.page_table_base = qpd->page_table_base;
-	packet->bitfields10.gds_size = qpd->gds_size;
-	packet->bitfields10.num_gws = qpd->num_gws;
-	packet->bitfields10.num_oac = qpd->num_oac;
-	packet->bitfields10.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;
-
-	packet->sh_mem_config = qpd->sh_mem_config;
-	packet->sh_mem_bases = qpd->sh_mem_bases;
-	packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
-	packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
-
-	packet->sh_hidden_private_base_vmid = qpd->sh_hidden_private_base;
-
-	packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
-	packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
-
-	return 0;
-}
-
-static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer,
-		struct queue *q, bool is_static)
-{
-	struct pm4_mes_map_queues *packet;
-	bool use_static = is_static;
-
-	packet = (struct pm4_mes_map_queues *)buffer;
-	memset(buffer, 0, sizeof(struct pm4_mes_map_queues));
-
-	packet->header.u32All = build_pm4_header(IT_MAP_QUEUES,
-						sizeof(struct pm4_mes_map_queues));
-	packet->bitfields2.alloc_format =
-		alloc_format__mes_map_queues__one_per_pipe_vi;
-	packet->bitfields2.num_queues = 1;
-	packet->bitfields2.queue_sel =
-		queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi;
-
-	packet->bitfields2.engine_sel =
-		engine_sel__mes_map_queues__compute_vi;
-	packet->bitfields2.queue_type =
-		queue_type__mes_map_queues__normal_compute_vi;
-
-	switch (q->properties.type) {
-	case KFD_QUEUE_TYPE_COMPUTE:
-		if (use_static)
-			packet->bitfields2.queue_type =
-		queue_type__mes_map_queues__normal_latency_static_queue_vi;
-		break;
-	case KFD_QUEUE_TYPE_DIQ:
-		packet->bitfields2.queue_type =
-			queue_type__mes_map_queues__debug_interface_queue_vi;
-		break;
-	case KFD_QUEUE_TYPE_SDMA:
-		packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
-				engine_sel__mes_map_queues__sdma0_vi;
-		use_static = false; /* no static queues under SDMA */
-		break;
-	default:
-		WARN(1, "queue type %d", q->properties.type);
-		return -EINVAL;
-	}
-	packet->bitfields3.doorbell_offset =
-			q->properties.doorbell_off;
-
-	packet->mqd_addr_lo =
-			lower_32_bits(q->gart_mqd_addr);
-
-	packet->mqd_addr_hi =
-			upper_32_bits(q->gart_mqd_addr);
-
-	packet->wptr_addr_lo =
-			lower_32_bits((uint64_t)q->properties.write_ptr);
-
-	packet->wptr_addr_hi =
-			upper_32_bits((uint64_t)q->properties.write_ptr);
-
-	return 0;
+out:
+	mutex_unlock(&pm->lock);
+	return retval;
 }
 
 static int pm_create_runlist_ib(struct packet_manager *pm,
@@ -292,12 +152,12 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 			return -ENOMEM;
 		}
 
-		retval = pm_create_map_process(pm, &rl_buffer[rl_wptr], qpd);
+		retval = pm->pmf->map_process(pm, &rl_buffer[rl_wptr], qpd);
 		if (retval)
 			return retval;
 
 		proccesses_mapped++;
-		inc_wptr(&rl_wptr, sizeof(struct pm4_mes_map_process),
+		inc_wptr(&rl_wptr, pm->pmf->map_process_size,
 				alloc_size_bytes);
 
 		list_for_each_entry(kq, &qpd->priv_queue_list, list) {
@@ -307,7 +167,7 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 			pr_debug("static_queue, mapping kernel q %d, is debug status %d\n",
 				kq->queue->queue, qpd->is_debug);
 
-			retval = pm_create_map_queue(pm,
+			retval = pm->pmf->map_queues(pm,
 						&rl_buffer[rl_wptr],
 						kq->queue,
 						qpd->is_debug);
@@ -315,7 +175,7 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 				return retval;
 
 			inc_wptr(&rl_wptr,
-				sizeof(struct pm4_mes_map_queues),
+				pm->pmf->map_queues_size,
 				alloc_size_bytes);
 		}
 
@@ -326,7 +186,7 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 			pr_debug("static_queue, mapping user queue %d, is debug status %d\n",
 				q->queue, qpd->is_debug);
 
-			retval = pm_create_map_queue(pm,
+			retval = pm->pmf->map_queues(pm,
 						&rl_buffer[rl_wptr],
 						q,
 						qpd->is_debug);
@@ -335,7 +195,7 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 				return retval;
 
 			inc_wptr(&rl_wptr,
-				sizeof(struct pm4_mes_map_queues),
+				pm->pmf->map_queues_size,
 				alloc_size_bytes);
 		}
 	}
@@ -343,7 +203,7 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 	pr_debug("Finished map process and queues to runlist\n");
 
 	if (is_over_subscription)
-		retval = pm_create_runlist(pm, &rl_buffer[rl_wptr],
+		retval = pm->pmf->runlist(pm, &rl_buffer[rl_wptr],
 					*rl_gpu_addr,
 					alloc_size_bytes / sizeof(uint32_t),
 					true);
@@ -355,45 +215,29 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 	return retval;
 }
 
-/* pm_create_release_mem - Create a RELEASE_MEM packet and return the size
- *     of this packet
- *     @gpu_addr - GPU address of the packet. It's a virtual address.
- *     @buffer - buffer to fill up with the packet. It's a CPU kernel pointer
- *     Return - length of the packet
- */
-uint32_t pm_create_release_mem(uint64_t gpu_addr, uint32_t *buffer)
-{
-	struct pm4_mec_release_mem *packet;
-
-	WARN_ON(!buffer);
-
-	packet = (struct pm4_mec_release_mem *)buffer;
-	memset(buffer, 0, sizeof(*packet));
-
-	packet->header.u32All = build_pm4_header(IT_RELEASE_MEM,
-						 sizeof(*packet));
-
-	packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
-	packet->bitfields2.event_index = event_index___release_mem__end_of_pipe;
-	packet->bitfields2.tcl1_action_ena = 1;
-	packet->bitfields2.tc_action_ena = 1;
-	packet->bitfields2.cache_policy = cache_policy___release_mem__lru;
-	packet->bitfields2.atc = 0;
-
-	packet->bitfields3.data_sel = data_sel___release_mem__send_32_bit_low;
-	packet->bitfields3.int_sel =
-		int_sel___release_mem__send_interrupt_after_write_confirm;
-
-	packet->bitfields4.address_lo_32b = (gpu_addr & 0xffffffff) >> 2;
-	packet->address_hi = upper_32_bits(gpu_addr);
-
-	packet->data_lo = 0;
-
-	return sizeof(*packet) / sizeof(unsigned int);
-}
-
 int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
 {
+	switch (dqm->dev->device_info->asic_family) {
+	case CHIP_KAVERI:
+	case CHIP_HAWAII:
+		/* PM4 packet structures on CIK are the same as on VI */
+	case CHIP_CARRIZO:
+	case CHIP_TONGA:
+	case CHIP_FIJI:
+	case CHIP_POLARIS10:
+	case CHIP_POLARIS11:
+		pm->pmf = &kfd_vi_pm_funcs;
+		break;
+	case CHIP_VEGA10:
+	case CHIP_RAVEN:
+		pm->pmf = &kfd_v9_pm_funcs;
+		break;
+	default:
+		WARN(1, "Unexpected ASIC family %u",
+		     dqm->dev->device_info->asic_family);
+		return -EINVAL;
+	}
+
 	pm->dqm = dqm;
 	mutex_init(&pm->lock);
 	pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ);
@@ -415,38 +259,25 @@ void pm_uninit(struct packet_manager *pm)
 int pm_send_set_resources(struct packet_manager *pm,
 				struct scheduling_resources *res)
 {
-	struct pm4_mes_set_resources *packet;
+	uint32_t *buffer, size;
 	int retval = 0;
 
+	size = pm->pmf->set_resources_size;
 	mutex_lock(&pm->lock);
 	pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
-					sizeof(*packet) / sizeof(uint32_t),
-					(unsigned int **)&packet);
-	if (!packet) {
+					size / sizeof(uint32_t),
+					(unsigned int **)&buffer);
+	if (!buffer) {
 		pr_err("Failed to allocate buffer on kernel queue\n");
 		retval = -ENOMEM;
 		goto out;
 	}
 
-	memset(packet, 0, sizeof(struct pm4_mes_set_resources));
-	packet->header.u32All = build_pm4_header(IT_SET_RESOURCES,
-					sizeof(struct pm4_mes_set_resources));
-
-	packet->bitfields2.queue_type =
-			queue_type__mes_set_resources__hsa_interface_queue_hiq;
-	packet->bitfields2.vmid_mask = res->vmid_mask;
-	packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY_MS / 100;
-	packet->bitfields7.oac_mask = res->oac_mask;
-	packet->bitfields8.gds_heap_base = res->gds_heap_base;
-	packet->bitfields8.gds_heap_size = res->gds_heap_size;
-
-	packet->gws_mask_lo = lower_32_bits(res->gws_mask);
-	packet->gws_mask_hi = upper_32_bits(res->gws_mask);
-
-	packet->queue_mask_lo = lower_32_bits(res->queue_mask);
-	packet->queue_mask_hi = upper_32_bits(res->queue_mask);
-
-	pm->priv_queue->ops.submit_packet(pm->priv_queue);
+	retval = pm->pmf->set_resources(pm, buffer, res);
+	if (!retval)
+		pm->priv_queue->ops.submit_packet(pm->priv_queue);
+	else
+		pm->priv_queue->ops.rollback_packet(pm->priv_queue);
 
 out:
 	mutex_unlock(&pm->lock);
@@ -468,7 +299,7 @@ int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues)
 
 	pr_debug("runlist IB address: 0x%llX\n", rl_gpu_ib_addr);
 
-	packet_size_dwords = sizeof(struct pm4_mes_runlist) / sizeof(uint32_t);
+	packet_size_dwords = pm->pmf->runlist_size / sizeof(uint32_t);
 	mutex_lock(&pm->lock);
 
 	retval = pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
@@ -476,7 +307,7 @@ int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues)
 	if (retval)
 		goto fail_acquire_packet_buffer;
 
-	retval = pm_create_runlist(pm, rl_buffer, rl_gpu_ib_addr,
+	retval = pm->pmf->runlist(pm, rl_buffer, rl_gpu_ib_addr,
 					rl_ib_size / sizeof(uint32_t), false);
 	if (retval)
 		goto fail_create_runlist;
@@ -499,37 +330,29 @@ int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues)
 int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
 			uint32_t fence_value)
 {
-	int retval;
-	struct pm4_mes_query_status *packet;
+	uint32_t *buffer, size;
+	int retval = 0;
 
 	if (WARN_ON(!fence_address))
 		return -EFAULT;
 
+	size = pm->pmf->query_status_size;
 	mutex_lock(&pm->lock);
-	retval = pm->priv_queue->ops.acquire_packet_buffer(
-			pm->priv_queue,
-			sizeof(struct pm4_mes_query_status) / sizeof(uint32_t),
-			(unsigned int **)&packet);
-	if (retval)
-		goto fail_acquire_packet_buffer;
-
-	packet->header.u32All = build_pm4_header(IT_QUERY_STATUS,
-					sizeof(struct pm4_mes_query_status));
-
-	packet->bitfields2.context_id = 0;
-	packet->bitfields2.interrupt_sel =
-			interrupt_sel__mes_query_status__completion_status;
-	packet->bitfields2.command =
-			command__mes_query_status__fence_only_after_write_ack;
-
-	packet->addr_hi = upper_32_bits((uint64_t)fence_address);
-	packet->addr_lo = lower_32_bits((uint64_t)fence_address);
-	packet->data_hi = upper_32_bits((uint64_t)fence_value);
-	packet->data_lo = lower_32_bits((uint64_t)fence_value);
+	pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
+			size / sizeof(uint32_t), (unsigned int **)&buffer);
+	if (!buffer) {
+		pr_err("Failed to allocate buffer on kernel queue\n");
+		retval = -ENOMEM;
+		goto out;
+	}
 
-	pm->priv_queue->ops.submit_packet(pm->priv_queue);
+	retval = pm->pmf->query_status(pm, buffer, fence_address, fence_value);
+	if (!retval)
+		pm->priv_queue->ops.submit_packet(pm->priv_queue);
+	else
+		pm->priv_queue->ops.rollback_packet(pm->priv_queue);
 
-fail_acquire_packet_buffer:
+out:
 	mutex_unlock(&pm->lock);
 	return retval;
 }
@@ -539,82 +362,27 @@ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
 			uint32_t filter_param, bool reset,
 			unsigned int sdma_engine)
 {
-	int retval;
-	uint32_t *buffer;
-	struct pm4_mes_unmap_queues *packet;
+	uint32_t *buffer, size;
+	int retval = 0;
 
+	size = pm->pmf->unmap_queues_size;
 	mutex_lock(&pm->lock);
-	retval = pm->priv_queue->ops.acquire_packet_buffer(
-			pm->priv_queue,
-			sizeof(struct pm4_mes_unmap_queues) / sizeof(uint32_t),
-			&buffer);
-	if (retval)
-		goto err_acquire_packet_buffer;
-
-	packet = (struct pm4_mes_unmap_queues *)buffer;
-	memset(buffer, 0, sizeof(struct pm4_mes_unmap_queues));
-	pr_debug("static_queue: unmapping queues: filter is %d , reset is %d , type is %d\n",
-		filter, reset, type);
-	packet->header.u32All = build_pm4_header(IT_UNMAP_QUEUES,
-					sizeof(struct pm4_mes_unmap_queues));
-	switch (type) {
-	case KFD_QUEUE_TYPE_COMPUTE:
-	case KFD_QUEUE_TYPE_DIQ:
-		packet->bitfields2.engine_sel =
-			engine_sel__mes_unmap_queues__compute;
-		break;
-	case KFD_QUEUE_TYPE_SDMA:
-		packet->bitfields2.engine_sel =
-			engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
-		break;
-	default:
-		WARN(1, "queue type %d", type);
-		retval = -EINVAL;
-		goto err_invalid;
+	pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
+			size / sizeof(uint32_t), (unsigned int **)&buffer);
+	if (!buffer) {
+		pr_err("Failed to allocate buffer on kernel queue\n");
+		retval = -ENOMEM;
+		goto out;
 	}
 
-	if (reset)
-		packet->bitfields2.action =
-				action__mes_unmap_queues__reset_queues;
+	retval = pm->pmf->unmap_queues(pm, buffer, type, filter, filter_param,
+				       reset, sdma_engine);
+	if (!retval)
+		pm->priv_queue->ops.submit_packet(pm->priv_queue);
 	else
-		packet->bitfields2.action =
-				action__mes_unmap_queues__preempt_queues;
-
-	switch (filter) {
-	case KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE:
-		packet->bitfields2.queue_sel =
-				queue_sel__mes_unmap_queues__perform_request_on_specified_queues;
-		packet->bitfields2.num_queues = 1;
-		packet->bitfields3b.doorbell_offset0 = filter_param;
-		break;
-	case KFD_UNMAP_QUEUES_FILTER_BY_PASID:
-		packet->bitfields2.queue_sel =
-				queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;
-		packet->bitfields3a.pasid = filter_param;
-		break;
-	case KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES:
-		packet->bitfields2.queue_sel =
-				queue_sel__mes_unmap_queues__unmap_all_queues;
-		break;
-	case KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES:
-		/* in this case, we do not preempt static queues */
-		packet->bitfields2.queue_sel =
-				queue_sel__mes_unmap_queues__unmap_all_non_static_queues;
-		break;
-	default:
-		WARN(1, "filter %d", filter);
-		retval = -EINVAL;
-		goto err_invalid;
-	}
+		pm->priv_queue->ops.rollback_packet(pm->priv_queue);
 
-	pm->priv_queue->ops.submit_packet(pm->priv_queue);
-
-	mutex_unlock(&pm->lock);
-	return 0;
-
-err_invalid:
-	pm->priv_queue->ops.rollback_packet(pm->priv_queue);
-err_acquire_packet_buffer:
+out:
 	mutex_unlock(&pm->lock);
 	return retval;
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h
new file mode 100644
index 000000000000..f2bcf5c092ea
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h
@@ -0,0 +1,583 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef F32_MES_PM4_PACKETS_H
+#define F32_MES_PM4_PACKETS_H
+
+#ifndef PM4_MES_HEADER_DEFINED
+#define PM4_MES_HEADER_DEFINED
+union PM4_MES_TYPE_3_HEADER {
+	struct {
+		uint32_t reserved1 : 8; /* < reserved */
+		uint32_t opcode    : 8; /* < IT opcode */
+		uint32_t count     : 14;/* < number of DWORDs - 1 in the
+					 *   information body.
+					 */
+		uint32_t type      : 2; /* < packet identifier.
+					 *   It should be 3 for type 3 packets
+					 */
+	};
+	uint32_t u32All;
+};
+#endif /* PM4_MES_HEADER_DEFINED */
+
+/*--------------------MES_SET_RESOURCES--------------------*/
+
+#ifndef PM4_MES_SET_RESOURCES_DEFINED
+#define PM4_MES_SET_RESOURCES_DEFINED
+enum mes_set_resources_queue_type_enum {
+	queue_type__mes_set_resources__kernel_interface_queue_kiq = 0,
+	queue_type__mes_set_resources__hsa_interface_queue_hiq = 1,
+	queue_type__mes_set_resources__hsa_debug_interface_queue = 4
+};
+
+
+struct pm4_mes_set_resources {
+	union {
+		union PM4_MES_TYPE_3_HEADER	header;		/* header */
+		uint32_t			ordinal1;
+	};
+
+	union {
+		struct {
+			uint32_t vmid_mask:16;
+			uint32_t unmap_latency:8;
+			uint32_t reserved1:5;
+			enum mes_set_resources_queue_type_enum queue_type:3;
+		} bitfields2;
+		uint32_t ordinal2;
+	};
+
+	uint32_t queue_mask_lo;
+	uint32_t queue_mask_hi;
+	uint32_t gws_mask_lo;
+	uint32_t gws_mask_hi;
+
+	union {
+		struct {
+			uint32_t oac_mask:16;
+			uint32_t reserved2:16;
+		} bitfields7;
+		uint32_t ordinal7;
+	};
+
+	union {
+		struct {
+		uint32_t gds_heap_base:6;
+		uint32_t reserved3:5;
+		uint32_t gds_heap_size:6;
+		uint32_t reserved4:15;
+		} bitfields8;
+		uint32_t ordinal8;
+	};
+
+};
+#endif
+
+/*--------------------MES_RUN_LIST--------------------*/
+
+#ifndef PM4_MES_RUN_LIST_DEFINED
+#define PM4_MES_RUN_LIST_DEFINED
+
+struct pm4_mes_runlist {
+	union {
+		union PM4_MES_TYPE_3_HEADER header; /* header */
+		uint32_t ordinal1;
+	};
+
+	union {
+		struct {
+			uint32_t reserved1:2;
+			uint32_t ib_base_lo:30;
+		} bitfields2;
+		uint32_t ordinal2;
+	};
+
+	uint32_t ib_base_hi;
+
+	union {
+		struct {
+			uint32_t ib_size:20;
+			uint32_t chain:1;
+			uint32_t offload_polling:1;
+			uint32_t reserved2:1;
+			uint32_t valid:1;
+			uint32_t process_cnt:4;
+			uint32_t reserved3:4;
+		} bitfields4;
+		uint32_t ordinal4;
+	};
+
+};
+#endif
+
+/*--------------------MES_MAP_PROCESS--------------------*/
+
+#ifndef PM4_MES_MAP_PROCESS_DEFINED
+#define PM4_MES_MAP_PROCESS_DEFINED
+
+struct pm4_mes_map_process {
+	union {
+		union PM4_MES_TYPE_3_HEADER header;	/* header */
+		uint32_t ordinal1;
+	};
+
+	union {
+		struct {
+			uint32_t pasid:16;
+			uint32_t reserved1:8;
+			uint32_t diq_enable:1;
+			uint32_t process_quantum:7;
+		} bitfields2;
+		uint32_t ordinal2;
+	};
+
+	uint32_t vm_context_page_table_base_addr_lo32;
+
+	uint32_t vm_context_page_table_base_addr_hi32;
+
+	uint32_t sh_mem_bases;
+
+	uint32_t sh_mem_config;
+
+	uint32_t sq_shader_tba_lo;
+
+	uint32_t sq_shader_tba_hi;
+
+	uint32_t sq_shader_tma_lo;
+
+	uint32_t sq_shader_tma_hi;
+
+	uint32_t reserved6;
+
+	uint32_t gds_addr_lo;
+
+	uint32_t gds_addr_hi;
+
+	union {
+		struct {
+			uint32_t num_gws:6;
+			uint32_t reserved7:1;
+			uint32_t sdma_enable:1;
+			uint32_t num_oac:4;
+			uint32_t reserved8:4;
+			uint32_t gds_size:6;
+			uint32_t num_queues:10;
+		} bitfields14;
+		uint32_t ordinal14;
+	};
+
+	uint32_t completion_signal_lo;
+
+	uint32_t completion_signal_hi;
+
+};
+
+#endif
+
+/*--------------------MES_MAP_PROCESS_VM--------------------*/
+
+#ifndef PM4_MES_MAP_PROCESS_VM_DEFINED
+#define PM4_MES_MAP_PROCESS_VM_DEFINED
+
+struct PM4_MES_MAP_PROCESS_VM {
+	union {
+		union PM4_MES_TYPE_3_HEADER header;	/* header */
+		uint32_t ordinal1;
+	};
+
+	uint32_t reserved1;
+
+	uint32_t vm_context_cntl;
+
+	uint32_t reserved2;
+
+	uint32_t vm_context_page_table_end_addr_lo32;
+
+	uint32_t vm_context_page_table_end_addr_hi32;
+
+	uint32_t vm_context_page_table_start_addr_lo32;
+
+	uint32_t vm_context_page_table_start_addr_hi32;
+
+	uint32_t reserved3;
+
+	uint32_t reserved4;
+
+	uint32_t reserved5;
+
+	uint32_t reserved6;
+
+	uint32_t reserved7;
+
+	uint32_t reserved8;
+
+	uint32_t completion_signal_lo32;
+
+	uint32_t completion_signal_hi32;
+
+};
+#endif
+
+/*--------------------MES_MAP_QUEUES--------------------*/
+
+#ifndef PM4_MES_MAP_QUEUES_VI_DEFINED
+#define PM4_MES_MAP_QUEUES_VI_DEFINED
+enum mes_map_queues_queue_sel_enum {
+	queue_sel__mes_map_queues__map_to_specified_queue_slots_vi = 0,
+queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi = 1
+};
+
+enum mes_map_queues_queue_type_enum {
+	queue_type__mes_map_queues__normal_compute_vi = 0,
+	queue_type__mes_map_queues__debug_interface_queue_vi = 1,
+	queue_type__mes_map_queues__normal_latency_static_queue_vi = 2,
+queue_type__mes_map_queues__low_latency_static_queue_vi = 3
+};
+
+enum mes_map_queues_alloc_format_enum {
+	alloc_format__mes_map_queues__one_per_pipe_vi = 0,
+alloc_format__mes_map_queues__all_on_one_pipe_vi = 1
+};
+
+enum mes_map_queues_engine_sel_enum {
+	engine_sel__mes_map_queues__compute_vi = 0,
+	engine_sel__mes_map_queues__sdma0_vi = 2,
+	engine_sel__mes_map_queues__sdma1_vi = 3
+};
+
+
+struct pm4_mes_map_queues {
+	union {
+		union PM4_MES_TYPE_3_HEADER   header;            /* header */
+		uint32_t            ordinal1;
+	};
+
+	union {
+		struct {
+			uint32_t reserved1:4;
+			enum mes_map_queues_queue_sel_enum queue_sel:2;
+			uint32_t reserved2:15;
+			enum mes_map_queues_queue_type_enum queue_type:3;
+			enum mes_map_queues_alloc_format_enum alloc_format:2;
+			enum mes_map_queues_engine_sel_enum engine_sel:3;
+			uint32_t num_queues:3;
+		} bitfields2;
+		uint32_t ordinal2;
+	};
+
+	union {
+		struct {
+			uint32_t reserved3:1;
+			uint32_t check_disable:1;
+			uint32_t doorbell_offset:26;
+			uint32_t reserved4:4;
+		} bitfields3;
+		uint32_t ordinal3;
+	};
+
+	uint32_t mqd_addr_lo;
+	uint32_t mqd_addr_hi;
+	uint32_t wptr_addr_lo;
+	uint32_t wptr_addr_hi;
+};
+#endif
+
+/*--------------------MES_QUERY_STATUS--------------------*/
+
+#ifndef PM4_MES_QUERY_STATUS_DEFINED
+#define PM4_MES_QUERY_STATUS_DEFINED
+enum mes_query_status_interrupt_sel_enum {
+	interrupt_sel__mes_query_status__completion_status = 0,
+	interrupt_sel__mes_query_status__process_status = 1,
+	interrupt_sel__mes_query_status__queue_status = 2
+};
+
+enum mes_query_status_command_enum {
+	command__mes_query_status__interrupt_only = 0,
+	command__mes_query_status__fence_only_immediate = 1,
+	command__mes_query_status__fence_only_after_write_ack = 2,
+	command__mes_query_status__fence_wait_for_write_ack_send_interrupt = 3
+};
+
+enum mes_query_status_engine_sel_enum {
+	engine_sel__mes_query_status__compute = 0,
+	engine_sel__mes_query_status__sdma0_queue = 2,
+	engine_sel__mes_query_status__sdma1_queue = 3
+};
+
+struct pm4_mes_query_status {
+	union {
+		union PM4_MES_TYPE_3_HEADER   header;            /* header */
+		uint32_t            ordinal1;
+	};
+
+	union {
+		struct {
+			uint32_t context_id:28;
+			enum mes_query_status_interrupt_sel_enum	interrupt_sel:2;
+			enum mes_query_status_command_enum command:2;
+		} bitfields2;
+		uint32_t ordinal2;
+	};
+
+	union {
+		struct {
+			uint32_t pasid:16;
+			uint32_t reserved1:16;
+		} bitfields3a;
+		struct {
+			uint32_t reserved2:2;
+			uint32_t doorbell_offset:26;
+			enum mes_query_status_engine_sel_enum engine_sel:3;
+			uint32_t reserved3:1;
+		} bitfields3b;
+		uint32_t ordinal3;
+	};
+
+	uint32_t addr_lo;
+	uint32_t addr_hi;
+	uint32_t data_lo;
+	uint32_t data_hi;
+};
+#endif
+
+/*--------------------MES_UNMAP_QUEUES--------------------*/
+
+#ifndef PM4_MES_UNMAP_QUEUES_DEFINED
+#define PM4_MES_UNMAP_QUEUES_DEFINED
+enum mes_unmap_queues_action_enum {
+	action__mes_unmap_queues__preempt_queues = 0,
+	action__mes_unmap_queues__reset_queues = 1,
+	action__mes_unmap_queues__disable_process_queues = 2,
+	action__mes_unmap_queues__reserved = 3
+};
+
+enum mes_unmap_queues_queue_sel_enum {
+	queue_sel__mes_unmap_queues__perform_request_on_specified_queues = 0,
+	queue_sel__mes_unmap_queues__perform_request_on_pasid_queues = 1,
+	queue_sel__mes_unmap_queues__unmap_all_queues = 2,
+	queue_sel__mes_unmap_queues__unmap_all_non_static_queues = 3
+};
+
+enum mes_unmap_queues_engine_sel_enum {
+	engine_sel__mes_unmap_queues__compute = 0,
+	engine_sel__mes_unmap_queues__sdma0 = 2,
+	engine_sel__mes_unmap_queues__sdmal = 3
+};
+
+struct pm4_mes_unmap_queues {
+	union {
+		union PM4_MES_TYPE_3_HEADER   header;            /* header */
+		uint32_t            ordinal1;
+	};
+
+	union {
+		struct {
+			enum mes_unmap_queues_action_enum action:2;
+			uint32_t reserved1:2;
+			enum mes_unmap_queues_queue_sel_enum queue_sel:2;
+			uint32_t reserved2:20;
+			enum mes_unmap_queues_engine_sel_enum engine_sel:3;
+			uint32_t num_queues:3;
+		} bitfields2;
+		uint32_t ordinal2;
+	};
+
+	union {
+		struct {
+			uint32_t pasid:16;
+			uint32_t reserved3:16;
+		} bitfields3a;
+		struct {
+			uint32_t reserved4:2;
+			uint32_t doorbell_offset0:26;
+			int32_t reserved5:4;
+		} bitfields3b;
+		uint32_t ordinal3;
+	};
+
+	union {
+	struct {
+			uint32_t reserved6:2;
+			uint32_t doorbell_offset1:26;
+			uint32_t reserved7:4;
+		} bitfields4;
+		uint32_t ordinal4;
+	};
+
+	union {
+		struct {
+			uint32_t reserved8:2;
+			uint32_t doorbell_offset2:26;
+			uint32_t reserved9:4;
+		} bitfields5;
+		uint32_t ordinal5;
+	};
+
+	union {
+		struct {
+			uint32_t reserved10:2;
+			uint32_t doorbell_offset3:26;
+			uint32_t reserved11:4;
+		} bitfields6;
+		uint32_t ordinal6;
+	};
+};
+#endif
+
+#ifndef PM4_MEC_RELEASE_MEM_DEFINED
+#define PM4_MEC_RELEASE_MEM_DEFINED
+
+enum mec_release_mem_event_index_enum {
+	event_index__mec_release_mem__end_of_pipe = 5,
+	event_index__mec_release_mem__shader_done = 6
+};
+
+enum mec_release_mem_cache_policy_enum {
+	cache_policy__mec_release_mem__lru = 0,
+	cache_policy__mec_release_mem__stream = 1
+};
+
+enum mec_release_mem_pq_exe_status_enum {
+	pq_exe_status__mec_release_mem__default = 0,
+	pq_exe_status__mec_release_mem__phase_update = 1
+};
+
+enum mec_release_mem_dst_sel_enum {
+	dst_sel__mec_release_mem__memory_controller = 0,
+	dst_sel__mec_release_mem__tc_l2 = 1,
+	dst_sel__mec_release_mem__queue_write_pointer_register = 2,
+	dst_sel__mec_release_mem__queue_write_pointer_poll_mask_bit = 3
+};
+
+enum mec_release_mem_int_sel_enum {
+	int_sel__mec_release_mem__none = 0,
+	int_sel__mec_release_mem__send_interrupt_only = 1,
+	int_sel__mec_release_mem__send_interrupt_after_write_confirm = 2,
+	int_sel__mec_release_mem__send_data_after_write_confirm = 3,
+	int_sel__mec_release_mem__unconditionally_send_int_ctxid = 4,
+	int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_32_bit_compare = 5,
+	int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_64_bit_compare = 6
+};
+
+enum mec_release_mem_data_sel_enum {
+	data_sel__mec_release_mem__none = 0,
+	data_sel__mec_release_mem__send_32_bit_low = 1,
+	data_sel__mec_release_mem__send_64_bit_data = 2,
+	data_sel__mec_release_mem__send_gpu_clock_counter = 3,
+	data_sel__mec_release_mem__send_cp_perfcounter_hi_lo = 4,
+	data_sel__mec_release_mem__store_gds_data_to_memory = 5
+};
+
+struct pm4_mec_release_mem {
+	union {
+		union PM4_MES_TYPE_3_HEADER header;     /*header */
+		unsigned int ordinal1;
+	};
+
+	union {
+		struct {
+			unsigned int event_type:6;
+			unsigned int reserved1:2;
+			enum mec_release_mem_event_index_enum event_index:4;
+			unsigned int tcl1_vol_action_ena:1;
+			unsigned int tc_vol_action_ena:1;
+			unsigned int reserved2:1;
+			unsigned int tc_wb_action_ena:1;
+			unsigned int tcl1_action_ena:1;
+			unsigned int tc_action_ena:1;
+			uint32_t reserved3:1;
+			uint32_t tc_nc_action_ena:1;
+			uint32_t tc_wc_action_ena:1;
+			uint32_t tc_md_action_ena:1;
+			uint32_t reserved4:3;
+			enum mec_release_mem_cache_policy_enum cache_policy:2;
+			uint32_t reserved5:2;
+			enum mec_release_mem_pq_exe_status_enum pq_exe_status:1;
+			uint32_t reserved6:2;
+		} bitfields2;
+		unsigned int ordinal2;
+	};
+
+	union {
+		struct {
+			uint32_t reserved7:16;
+			enum mec_release_mem_dst_sel_enum dst_sel:2;
+			uint32_t reserved8:6;
+			enum mec_release_mem_int_sel_enum int_sel:3;
+			uint32_t reserved9:2;
+			enum mec_release_mem_data_sel_enum data_sel:3;
+		} bitfields3;
+		unsigned int ordinal3;
+	};
+
+	union {
+		struct {
+			uint32_t reserved10:2;
+			unsigned int address_lo_32b:30;
+		} bitfields4;
+		struct {
+			uint32_t reserved11:3;
+			uint32_t address_lo_64b:29;
+		} bitfields4b;
+		uint32_t reserved12;
+		unsigned int ordinal4;
+	};
+
+	union {
+		uint32_t address_hi;
+		uint32_t reserved13;
+		uint32_t ordinal5;
+	};
+
+	union {
+		uint32_t data_lo;
+		uint32_t cmp_data_lo;
+		struct {
+			uint32_t dw_offset:16;
+			uint32_t num_dwords:16;
+		} bitfields6c;
+		uint32_t reserved14;
+		uint32_t ordinal6;
+	};
+
+	union {
+		uint32_t data_hi;
+		uint32_t cmp_data_hi;
+		uint32_t reserved15;
+		uint32_t reserved16;
+		uint32_t ordinal7;
+	};
+
+	uint32_t int_ctxid;
+
+};
+
+#endif
+
+enum {
+	CACHE_FLUSH_AND_INV_TS_EVENT = 0x00000014
+};
+#endif
+
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 96a9cc0f02c9..5e3990bb4c4b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -39,11 +39,37 @@
 
 #include "amd_shared.h"
 
+#define KFD_MAX_RING_ENTRY_SIZE	8
+
 #define KFD_SYSFS_FILE_MODE 0444
 
-#define KFD_MMAP_DOORBELL_MASK 0x8000000000000ull
-#define KFD_MMAP_EVENTS_MASK 0x4000000000000ull
-#define KFD_MMAP_RESERVED_MEM_MASK 0x2000000000000ull
+/* GPU ID hash width in bits */
+#define KFD_GPU_ID_HASH_WIDTH 16
+
+/* Use upper bits of mmap offset to store KFD driver specific information.
+ * BITS[63:62] - Encode MMAP type
+ * BITS[61:46] - Encode gpu_id. To identify to which GPU the offset belongs to
+ * BITS[45:0]  - MMAP offset value
+ *
+ * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these
+ *  defines are w.r.t to PAGE_SIZE
+ */
+#define KFD_MMAP_TYPE_SHIFT	(62 - PAGE_SHIFT)
+#define KFD_MMAP_TYPE_MASK	(0x3ULL << KFD_MMAP_TYPE_SHIFT)
+#define KFD_MMAP_TYPE_DOORBELL	(0x3ULL << KFD_MMAP_TYPE_SHIFT)
+#define KFD_MMAP_TYPE_EVENTS	(0x2ULL << KFD_MMAP_TYPE_SHIFT)
+#define KFD_MMAP_TYPE_RESERVED_MEM	(0x1ULL << KFD_MMAP_TYPE_SHIFT)
+
+#define KFD_MMAP_GPU_ID_SHIFT (46 - PAGE_SHIFT)
+#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
+				<< KFD_MMAP_GPU_ID_SHIFT)
+#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\
+				& KFD_MMAP_GPU_ID_MASK)
+#define KFD_MMAP_GPU_ID_GET(offset)    ((offset & KFD_MMAP_GPU_ID_MASK) \
+				>> KFD_MMAP_GPU_ID_SHIFT)
+
+#define KFD_MMAP_OFFSET_VALUE_MASK	(0x3FFFFFFFFFFFULL >> PAGE_SHIFT)
+#define KFD_MMAP_OFFSET_VALUE_GET(offset) (offset & KFD_MMAP_OFFSET_VALUE_MASK)
 
 /*
  * When working with cp scheduler we should assign the HIQ manually or via
@@ -55,9 +81,6 @@
 #define KFD_CIK_HIQ_PIPE 4
 #define KFD_CIK_HIQ_QUEUE 0
 
-/* GPU ID hash width in bits */
-#define KFD_GPU_ID_HASH_WIDTH 16
-
 /* Macro for allocating structures */
 #define kfd_alloc_struct(ptr_to_struct)	\
 	((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))
@@ -116,6 +139,11 @@ extern int debug_largebar;
  */
 extern int ignore_crat;
 
+/*
+ * Set sh_mem_config.retry_disable on Vega10
+ */
+extern int vega10_noretry;
+
 /**
  * enum kfd_sched_policy
  *
@@ -148,6 +176,8 @@ enum cache_policy {
 	cache_policy_noncoherent
 };
 
+#define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10)
+
 struct kfd_event_interrupt_class {
 	bool (*interrupt_isr)(struct kfd_dev *dev,
 				const uint32_t *ih_ring_entry);
@@ -160,6 +190,7 @@ struct kfd_device_info {
 	const struct kfd_event_interrupt_class *event_interrupt_class;
 	unsigned int max_pasid_bits;
 	unsigned int max_no_of_hqd;
+	unsigned int doorbell_size;
 	size_t ih_ring_entry_size;
 	uint8_t num_of_watch_points;
 	uint16_t mqd_size_aligned;
@@ -173,6 +204,7 @@ struct kfd_mem_obj {
 	uint32_t range_end;
 	uint64_t gpu_addr;
 	uint32_t *cpu_ptr;
+	void *gtt_mem;
 };
 
 struct kfd_vmid_info {
@@ -364,7 +396,7 @@ struct queue_properties {
 	uint32_t queue_percent;
 	uint32_t *read_ptr;
 	uint32_t *write_ptr;
-	uint32_t __iomem *doorbell_ptr;
+	void __iomem *doorbell_ptr;
 	uint32_t doorbell_off;
 	bool is_interop;
 	bool is_evicted;
@@ -427,6 +459,7 @@ struct queue {
 	uint32_t queue;
 
 	unsigned int sdma_id;
+	unsigned int doorbell_id;
 
 	struct kfd_process	*process;
 	struct kfd_dev		*device;
@@ -501,6 +534,9 @@ struct qcm_process_device {
 	/* IB memory */
 	uint64_t ib_base;
 	void *ib_kaddr;
+
+	/* doorbell resources per process per device */
+	unsigned long *doorbell_bitmap;
 };
 
 /* KFD Memory Eviction */
@@ -512,6 +548,8 @@ struct qcm_process_device {
 /* Approx. time before evicting the process again */
 #define PROCESS_ACTIVE_TIME_MS 10
 
+int kgd2kfd_quiesce_mm(struct mm_struct *mm);
+int kgd2kfd_resume_mm(struct mm_struct *mm);
 int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
 					       struct dma_fence *fence);
 
@@ -681,6 +719,8 @@ struct kfd_process *kfd_get_process(const struct task_struct *);
 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
 void kfd_unref_process(struct kfd_process *p);
+int kfd_process_evict_queues(struct kfd_process *p);
+int kfd_process_restore_queues(struct kfd_process *p);
 void kfd_suspend_all_processes(void);
 int kfd_resume_all_processes(void);
 
@@ -693,7 +733,7 @@ struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
 							struct kfd_process *p);
 
-int kfd_reserved_mem_mmap(struct kfd_process *process,
+int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
 			  struct vm_area_struct *vma);
 
 /* KFD process API for creating and translating handles */
@@ -721,17 +761,20 @@ unsigned int kfd_pasid_alloc(void);
 void kfd_pasid_free(unsigned int pasid);
 
 /* Doorbells */
+size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
 int kfd_doorbell_init(struct kfd_dev *kfd);
 void kfd_doorbell_fini(struct kfd_dev *kfd);
-int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma);
-u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
+int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
+		      struct vm_area_struct *vma);
+void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
 					unsigned int *doorbell_off);
 void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
 u32 read_kernel_doorbell(u32 __iomem *db);
-void write_kernel_doorbell(u32 __iomem *db, u32 value);
-unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,
+void write_kernel_doorbell(void __iomem *db, u32 value);
+void write_kernel_doorbell64(void __iomem *db, u64 value);
+unsigned int kfd_doorbell_id_to_offset(struct kfd_dev *kfd,
 					struct kfd_process *process,
-					unsigned int queue_id);
+					unsigned int doorbell_id);
 phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
 					struct kfd_process *process);
 int kfd_alloc_process_doorbells(struct kfd_process *process);
@@ -788,6 +831,8 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 		struct kfd_dev *dev);
 struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
 		struct kfd_dev *dev);
+struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
+		struct kfd_dev *dev);
 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
 void device_queue_manager_uninit(struct device_queue_manager *dqm);
 struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
@@ -832,8 +877,42 @@ struct packet_manager {
 	bool allocated;
 	struct kfd_mem_obj *ib_buffer_obj;
 	unsigned int ib_size_bytes;
+
+	const struct packet_manager_funcs *pmf;
+};
+
+struct packet_manager_funcs {
+	/* Support ASIC-specific packet formats for PM4 packets */
+	int (*map_process)(struct packet_manager *pm, uint32_t *buffer,
+			struct qcm_process_device *qpd);
+	int (*runlist)(struct packet_manager *pm, uint32_t *buffer,
+			uint64_t ib, size_t ib_size_in_dwords, bool chain);
+	int (*set_resources)(struct packet_manager *pm, uint32_t *buffer,
+			struct scheduling_resources *res);
+	int (*map_queues)(struct packet_manager *pm, uint32_t *buffer,
+			struct queue *q, bool is_static);
+	int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,
+			enum kfd_queue_type type,
+			enum kfd_unmap_queues_filter mode,
+			uint32_t filter_param, bool reset,
+			unsigned int sdma_engine);
+	int (*query_status)(struct packet_manager *pm, uint32_t *buffer,
+			uint64_t fence_address,	uint32_t fence_value);
+	int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);
+
+	/* Packet sizes */
+	int map_process_size;
+	int runlist_size;
+	int set_resources_size;
+	int map_queues_size;
+	int unmap_queues_size;
+	int query_status_size;
+	int release_mem_size;
 };
 
+extern const struct packet_manager_funcs kfd_vi_pm_funcs;
+extern const struct packet_manager_funcs kfd_v9_pm_funcs;
+
 int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
 void pm_uninit(struct packet_manager *pm);
 int pm_send_set_resources(struct packet_manager *pm,
@@ -849,12 +928,17 @@ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
 
 void pm_release_ib(struct packet_manager *pm);
 
-uint32_t pm_create_release_mem(uint64_t gpu_addr, uint32_t *buffer);
+/* Following PM funcs can be shared among VI and AI */
+unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);
+int pm_set_resources_vi(struct packet_manager *pm, uint32_t *buffer,
+				struct scheduling_resources *res);
 
 uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
 
 /* Events */
 extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
+extern const struct kfd_event_interrupt_class event_interrupt_class_v9;
+
 extern const struct kfd_device_global_init_class device_global_init_class_cik;
 
 void kfd_event_init_process(struct kfd_process *p);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 1711ad0642f7..1d80b4f7c681 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -332,6 +332,7 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
 			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
 				get_order(KFD_CWSR_TBA_TMA_SIZE));
 
+		kfree(pdd->qpd.doorbell_bitmap);
 		idr_destroy(&pdd->alloc_idr);
 
 		kfree(pdd);
@@ -451,7 +452,8 @@ static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
 		if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
 			continue;
 
-		offset = (dev->id | KFD_MMAP_RESERVED_MEM_MASK) << PAGE_SHIFT;
+		offset = (KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id))
+			<< PAGE_SHIFT;
 		qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
 			KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
 			MAP_SHARED, offset);
@@ -585,6 +587,31 @@ static struct kfd_process *create_process(const struct task_struct *thread,
 	return ERR_PTR(err);
 }
 
+static int init_doorbell_bitmap(struct qcm_process_device *qpd,
+			struct kfd_dev *dev)
+{
+	unsigned int i;
+
+	if (!KFD_IS_SOC15(dev->device_info->asic_family))
+		return 0;
+
+	qpd->doorbell_bitmap =
+		kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
+				     BITS_PER_BYTE), GFP_KERNEL);
+	if (!qpd->doorbell_bitmap)
+		return -ENOMEM;
+
+	/* Mask out any reserved doorbells */
+	for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS; i++)
+		if ((dev->shared_resources.reserved_doorbell_mask & i) ==
+		    dev->shared_resources.reserved_doorbell_val) {
+			set_bit(i, qpd->doorbell_bitmap);
+			pr_debug("reserved doorbell 0x%03x\n", i);
+		}
+
+	return 0;
+}
+
 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
 							struct kfd_process *p)
 {
@@ -606,6 +633,12 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
 	if (!pdd)
 		return NULL;
 
+	if (init_doorbell_bitmap(&pdd->qpd, dev)) {
+		pr_err("Failed to init doorbell for process\n");
+		kfree(pdd);
+		return NULL;
+	}
+
 	pdd->dev = dev;
 	INIT_LIST_HEAD(&pdd->qpd.queues_list);
 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
@@ -808,7 +841,7 @@ struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
  * Eviction is reference-counted per process-device. This means multiple
  * evictions from different sources can be nested safely.
  */
-static int process_evict_queues(struct kfd_process *p)
+int kfd_process_evict_queues(struct kfd_process *p)
 {
 	struct kfd_process_device *pdd;
 	int r = 0;
@@ -844,7 +877,7 @@ static int process_evict_queues(struct kfd_process *p)
 }
 
 /* process_restore_queues - Restore all user queues of a process */
-static  int process_restore_queues(struct kfd_process *p)
+int kfd_process_restore_queues(struct kfd_process *p)
 {
 	struct kfd_process_device *pdd;
 	int r, ret = 0;
@@ -886,7 +919,7 @@ static void evict_process_worker(struct work_struct *work)
 	flush_delayed_work(&p->restore_work);
 
 	pr_debug("Started evicting pasid %d\n", p->pasid);
-	ret = process_evict_queues(p);
+	ret = kfd_process_evict_queues(p);
 	if (!ret) {
 		dma_fence_signal(p->ef);
 		dma_fence_put(p->ef);
@@ -946,7 +979,7 @@ static void restore_process_worker(struct work_struct *work)
 		return;
 	}
 
-	ret = process_restore_queues(p);
+	ret = kfd_process_restore_queues(p);
 	if (!ret)
 		pr_debug("Finished restoring pasid %d\n", p->pasid);
 	else
@@ -963,7 +996,7 @@ void kfd_suspend_all_processes(void)
 		cancel_delayed_work_sync(&p->eviction_work);
 		cancel_delayed_work_sync(&p->restore_work);
 
-		if (process_evict_queues(p))
+		if (kfd_process_evict_queues(p))
 			pr_err("Failed to suspend process %d\n", p->pasid);
 		dma_fence_signal(p->ef);
 		dma_fence_put(p->ef);
@@ -989,15 +1022,12 @@ int kfd_resume_all_processes(void)
 	return ret;
 }
 
-int kfd_reserved_mem_mmap(struct kfd_process *process,
+int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
 			  struct vm_area_struct *vma)
 {
-	struct kfd_dev *dev = kfd_device_by_id(vma->vm_pgoff);
 	struct kfd_process_device *pdd;
 	struct qcm_process_device *qpd;
 
-	if (!dev)
-		return -EINVAL;
 	if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
 		pr_err("Incorrect CWSR mapping size.\n");
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 7817e327ea6d..d65ce0436b31 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -119,9 +119,6 @@ static int create_cp_queue(struct process_queue_manager *pqm,
 	/* Doorbell initialized in user space*/
 	q_properties->doorbell_ptr = NULL;
 
-	q_properties->doorbell_off =
-			kfd_queue_id_to_doorbell(dev, pqm->process, qid);
-
 	/* let DQM handle it*/
 	q_properties->vmid = 0;
 	q_properties->queue_id = qid;
@@ -244,10 +241,20 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 	}
 
 	if (retval != 0) {
-		pr_err("DQM create queue failed\n");
+		pr_err("Pasid %d DQM create queue %d failed. ret %d\n",
+			pqm->process->pasid, type, retval);
 		goto err_create_queue;
 	}
 
+	if (q)
+		/* Return the doorbell offset within the doorbell page
+		 * to the caller so it can be passed up to user mode
+		 * (in bytes).
+		 */
+		properties->doorbell_off =
+			(q->properties.doorbell_off * sizeof(uint32_t)) &
+			(kfd_doorbell_process_slice(dev) - 1);
+
 	pr_debug("PQM After DQM create queue\n");
 
 	list_add(&pqn->process_queue_list, &pqm->queues);
@@ -313,8 +320,11 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
 		dqm = pqn->q->device->dqm;
 		retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
 		if (retval) {
-			pr_debug("Destroy queue failed, returned %d\n", retval);
-			goto err_destroy_queue;
+			pr_err("Pasid %d destroy queue %d failed, ret %d\n",
+				pqm->process->pasid,
+				pqn->q->properties.queue_id, retval);
+			if (retval != -ETIME)
+				goto err_destroy_queue;
 		}
 		uninit_queue(pqn->q);
 	}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
index a5315d4f1c95..6dcd621e5b71 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
@@ -36,8 +36,8 @@ void print_queue_properties(struct queue_properties *q)
 	pr_debug("Queue Address: 0x%llX\n", q->queue_address);
 	pr_debug("Queue Id: %u\n", q->queue_id);
 	pr_debug("Queue Process Vmid: %u\n", q->vmid);
-	pr_debug("Queue Read Pointer: 0x%p\n", q->read_ptr);
-	pr_debug("Queue Write Pointer: 0x%p\n", q->write_ptr);
+	pr_debug("Queue Read Pointer: 0x%px\n", q->read_ptr);
+	pr_debug("Queue Write Pointer: 0x%px\n", q->write_ptr);
 	pr_debug("Queue Doorbell Pointer: 0x%p\n", q->doorbell_ptr);
 	pr_debug("Queue Doorbell Offset: %u\n", q->doorbell_off);
 }
@@ -53,8 +53,8 @@ void print_queue(struct queue *q)
 	pr_debug("Queue Address: 0x%llX\n", q->properties.queue_address);
 	pr_debug("Queue Id: %u\n", q->properties.queue_id);
 	pr_debug("Queue Process Vmid: %u\n", q->properties.vmid);
-	pr_debug("Queue Read Pointer: 0x%p\n", q->properties.read_ptr);
-	pr_debug("Queue Write Pointer: 0x%p\n", q->properties.write_ptr);
+	pr_debug("Queue Read Pointer: 0x%px\n", q->properties.read_ptr);
+	pr_debug("Queue Write Pointer: 0x%px\n", q->properties.write_ptr);
 	pr_debug("Queue Doorbell Pointer: 0x%p\n", q->properties.doorbell_ptr);
 	pr_debug("Queue Doorbell Offset: %u\n", q->properties.doorbell_off);
 	pr_debug("Queue MQD Address: 0x%p\n", q->mqd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index ac28abc94e57..bc95d4dfee2e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1239,6 +1239,12 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
 			HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
 			HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
 		break;
+	case CHIP_VEGA10:
+	case CHIP_RAVEN:
+		dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 <<
+			HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
+			HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
+		break;
 	default:
 		WARN(1, "Unexpected ASIC family %u",
 		     dev->gpu->device_info->asic_family);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
index eb54cfcaf039..7d9c3f948dff 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
@@ -45,6 +45,7 @@
 
 #define HSA_CAP_DOORBELL_TYPE_PRE_1_0		0x0
 #define HSA_CAP_DOORBELL_TYPE_1_0		0x1
+#define HSA_CAP_DOORBELL_TYPE_2_0		0x2
 #define HSA_CAP_AQL_QUEUE_DOUBLE_MAP		0x00004000
 
 struct kfd_node_properties {
diff --git a/drivers/gpu/drm/amd/amdkfd/soc15_int.h b/drivers/gpu/drm/amd/amdkfd/soc15_int.h
new file mode 100644
index 000000000000..0bc0b25cb410
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/soc15_int.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2016-2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef HSA_SOC15_INT_H_INCLUDED
+#define HSA_SOC15_INT_H_INCLUDED
+
+#include "soc15_ih_clientid.h"
+
+#define SOC15_INTSRC_CP_END_OF_PIPE	181
+#define SOC15_INTSRC_CP_BAD_OPCODE	183
+#define SOC15_INTSRC_SQ_INTERRUPT_MSG	239
+#define SOC15_INTSRC_VMC_FAULT		0
+#define SOC15_INTSRC_SDMA_TRAP		224
+
+
+#define SOC15_CLIENT_ID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) & 0xff)
+#define SOC15_SOURCE_ID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) >> 8 & 0xff)
+#define SOC15_RING_ID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) >> 16 & 0xff)
+#define SOC15_VMID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) >> 24 & 0xf)
+#define SOC15_VMID_TYPE_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) >> 31 & 0x1)
+#define SOC15_PASID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[3]) & 0xffff)
+#define SOC15_CONTEXT_ID0_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[4]))
+#define SOC15_CONTEXT_ID1_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[5]))
+#define SOC15_CONTEXT_ID2_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[6]))
+#define SOC15_CONTEXT_ID3_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[7]))
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig
index 5b124a67404c..d5d4586e6176 100644
--- a/drivers/gpu/drm/amd/display/Kconfig
+++ b/drivers/gpu/drm/amd/display/Kconfig
@@ -9,14 +9,6 @@ config DRM_AMD_DC
 	  support for AMDGPU. This adds required support for Vega and
 	  Raven ASICs.
 
-config DRM_AMD_DC_PRE_VEGA
-	bool "DC support for Polaris and older ASICs"
-	default y
-	help
-	  Choose this option to enable the new DC support for older asics
-	  by default. This includes Polaris, Carrizo, Tonga, Bonaire,
-	  and Hawaii.
-
 config DRM_AMD_DC_FBC
 	bool "AMD FBC - Enable Frame Buffer Compression"
 	depends on DRM_AMD_DC
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index 1dd1142246c2..1ce10bc2d37b 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -433,11 +433,6 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 
 	init_data.dce_environment = DCE_ENV_PRODUCTION_DRV;
 
-	if (amdgpu_dc_log)
-		init_data.log_mask = DC_DEFAULT_LOG_MASK;
-	else
-		init_data.log_mask = DC_MIN_LOG_MASK;
-
 	/*
 	 * TODO debug why this doesn't work on Raven
 	 */
@@ -649,18 +644,6 @@ amdgpu_dm_find_first_crtc_matching_connector(struct drm_atomic_state *state,
 static int dm_resume(void *handle)
 {
 	struct amdgpu_device *adev = handle;
-	struct amdgpu_display_manager *dm = &adev->dm;
-	int ret = 0;
-
-	/* power on hardware */
-	dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0);
-
-	ret = amdgpu_dm_display_resume(adev);
-	return ret;
-}
-
-int amdgpu_dm_display_resume(struct amdgpu_device *adev)
-{
 	struct drm_device *ddev = adev->ddev;
 	struct amdgpu_display_manager *dm = &adev->dm;
 	struct amdgpu_dm_connector *aconnector;
@@ -671,10 +654,12 @@ int amdgpu_dm_display_resume(struct amdgpu_device *adev)
 	struct drm_plane *plane;
 	struct drm_plane_state *new_plane_state;
 	struct dm_plane_state *dm_new_plane_state;
-
-	int ret = 0;
+	int ret;
 	int i;
 
+	/* power on hardware */
+	dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0);
+
 	/* program HPD filter */
 	dc_resume(dm->dc);
 
@@ -688,8 +673,7 @@ int amdgpu_dm_display_resume(struct amdgpu_device *adev)
 	amdgpu_dm_irq_resume_early(adev);
 
 	/* Do detection*/
-	list_for_each_entry(connector,
-			&ddev->mode_config.connector_list, head) {
+	list_for_each_entry(connector, &ddev->mode_config.connector_list, head) {
 		aconnector = to_amdgpu_dm_connector(connector);
 
 		/*
@@ -711,7 +695,7 @@ int amdgpu_dm_display_resume(struct amdgpu_device *adev)
 	}
 
 	/* Force mode set in atomic comit */
-	for_each_new_crtc_in_state(adev->dm.cached_state, crtc, new_crtc_state, i)
+	for_each_new_crtc_in_state(dm->cached_state, crtc, new_crtc_state, i)
 		new_crtc_state->active_changed = true;
 
 	/*
@@ -719,7 +703,7 @@ int amdgpu_dm_display_resume(struct amdgpu_device *adev)
 	 * them here, since they were duplicated as part of the suspend
 	 * procedure.
 	 */
-	for_each_new_crtc_in_state(adev->dm.cached_state, crtc, new_crtc_state, i) {
+	for_each_new_crtc_in_state(dm->cached_state, crtc, new_crtc_state, i) {
 		dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
 		if (dm_new_crtc_state->stream) {
 			WARN_ON(kref_read(&dm_new_crtc_state->stream->refcount) > 1);
@@ -728,7 +712,7 @@ int amdgpu_dm_display_resume(struct amdgpu_device *adev)
 		}
 	}
 
-	for_each_new_plane_in_state(adev->dm.cached_state, plane, new_plane_state, i) {
+	for_each_new_plane_in_state(dm->cached_state, plane, new_plane_state, i) {
 		dm_new_plane_state = to_dm_plane_state(new_plane_state);
 		if (dm_new_plane_state->dc_state) {
 			WARN_ON(kref_read(&dm_new_plane_state->dc_state->refcount) > 1);
@@ -737,9 +721,9 @@ int amdgpu_dm_display_resume(struct amdgpu_device *adev)
 		}
 	}
 
-	ret = drm_atomic_helper_resume(ddev, adev->dm.cached_state);
+	ret = drm_atomic_helper_resume(ddev, dm->cached_state);
 
-	adev->dm.cached_state = NULL;
+	dm->cached_state = NULL;
 
 	amdgpu_dm_irq_resume_late(adev);
 
@@ -927,6 +911,7 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector)
 		drm_mode_connector_update_edid_property(connector, NULL);
 		aconnector->num_modes = 0;
 		aconnector->dc_sink = NULL;
+		aconnector->edid = NULL;
 	}
 
 	mutex_unlock(&dev->mode_config.mutex);
@@ -1131,6 +1116,7 @@ static int dce110_register_irq_handlers(struct amdgpu_device *adev)
 
 	if (adev->asic_type == CHIP_VEGA10 ||
 	    adev->asic_type == CHIP_VEGA12 ||
+	    adev->asic_type == CHIP_VEGA20 ||
 	    adev->asic_type == CHIP_RAVEN)
 		client_id = SOC15_IH_CLIENTID_DCE;
 
@@ -1529,8 +1515,10 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS10:
 	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 		if (dce110_register_irq_handlers(dm->adev)) {
 			DRM_ERROR("DM: Failed to initialize IRQ\n");
 			goto fail;
@@ -1549,7 +1537,7 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 		break;
 #endif
 	default:
-		DRM_ERROR("Usupported ASIC type: 0x%X\n", adev->asic_type);
+		DRM_ERROR("Unsupported ASIC type: 0x%X\n", adev->asic_type);
 		goto fail;
 	}
 
@@ -1657,7 +1645,6 @@ static ssize_t s3_debug_store(struct device *device,
 	if (ret == 0) {
 		if (s3_state) {
 			dm_resume(adev);
-			amdgpu_dm_display_resume(adev);
 			drm_kms_helper_hotplug_event(adev->ddev);
 		} else
 			dm_suspend(adev);
@@ -1722,6 +1709,7 @@ static int dm_early_init(void *handle)
 		adev->mode_info.plane_type = dm_plane_type_default;
 		break;
 	case CHIP_POLARIS10:
+	case CHIP_VEGAM:
 		adev->mode_info.num_crtc = 6;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 6;
@@ -1729,6 +1717,7 @@ static int dm_early_init(void *handle)
 		break;
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
+	case CHIP_VEGA20:
 		adev->mode_info.num_crtc = 6;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 6;
@@ -1743,7 +1732,7 @@ static int dm_early_init(void *handle)
 		break;
 #endif
 	default:
-		DRM_ERROR("Usupported ASIC type: 0x%X\n", adev->asic_type);
+		DRM_ERROR("Unsupported ASIC type: 0x%X\n", adev->asic_type);
 		return -EINVAL;
 	}
 
@@ -1848,7 +1837,7 @@ static bool fill_rects_from_plane_state(const struct drm_plane_state *state,
 static int get_fb_info(const struct amdgpu_framebuffer *amdgpu_fb,
 		       uint64_t *tiling_flags)
 {
-	struct amdgpu_bo *rbo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+	struct amdgpu_bo *rbo = gem_to_amdgpu_bo(amdgpu_fb->base.obj[0]);
 	int r = amdgpu_bo_reserve(rbo, false);
 
 	if (unlikely(r)) {
@@ -1977,6 +1966,7 @@ static int fill_plane_attributes_from_fb(struct amdgpu_device *adev,
 
 	if (adev->asic_type == CHIP_VEGA10 ||
 	    adev->asic_type == CHIP_VEGA12 ||
+	    adev->asic_type == CHIP_VEGA20 ||
 	    adev->asic_type == CHIP_RAVEN) {
 		/* Fill GFX9 params */
 		plane_state->tiling_info.gfx9.num_pipes =
@@ -2017,7 +2007,6 @@ static int fill_plane_attributes(struct amdgpu_device *adev,
 	const struct amdgpu_framebuffer *amdgpu_fb =
 		to_amdgpu_framebuffer(plane_state->fb);
 	const struct drm_crtc *crtc = plane_state->crtc;
-	struct dc_transfer_func *input_tf;
 	int ret = 0;
 
 	if (!fill_rects_from_plane_state(plane_state, dc_plane_state))
@@ -2031,13 +2020,6 @@ static int fill_plane_attributes(struct amdgpu_device *adev,
 	if (ret)
 		return ret;
 
-	input_tf = dc_create_transfer_func();
-
-	if (input_tf == NULL)
-		return -ENOMEM;
-
-	dc_plane_state->in_transfer_func = input_tf;
-
 	/*
 	 * Always set input transfer function, since plane state is refreshed
 	 * every time.
@@ -2206,7 +2188,6 @@ fill_stream_properties_from_drm_display_mode(struct dc_stream_state *stream,
 					     const struct drm_connector *connector)
 {
 	struct dc_crtc_timing *timing_out = &stream->timing;
-	struct dc_transfer_func *tf = dc_create_transfer_func();
 
 	memset(timing_out, 0, sizeof(struct dc_crtc_timing));
 
@@ -2250,9 +2231,8 @@ fill_stream_properties_from_drm_display_mode(struct dc_stream_state *stream,
 
 	stream->output_color_space = get_output_color_space(timing_out);
 
-	tf->type = TF_TYPE_PREDEFINED;
-	tf->tf = TRANSFER_FUNCTION_SRGB;
-	stream->out_transfer_func = tf;
+	stream->out_transfer_func->type = TF_TYPE_PREDEFINED;
+	stream->out_transfer_func->tf = TRANSFER_FUNCTION_SRGB;
 }
 
 static void fill_audio_info(struct audio_info *audio_info,
@@ -2488,6 +2468,9 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector,
 
 	update_stream_signal(stream);
 
+	if (dm_state && dm_state->freesync_capable)
+		stream->ignore_msa_timing_param = true;
+
 	return stream;
 }
 
@@ -2710,18 +2693,15 @@ static void amdgpu_dm_connector_destroy(struct drm_connector *connector)
 	const struct dc_link *link = aconnector->dc_link;
 	struct amdgpu_device *adev = connector->dev->dev_private;
 	struct amdgpu_display_manager *dm = &adev->dm;
+
 #if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) ||\
 	defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE)
 
 	if ((link->connector_signal & (SIGNAL_TYPE_EDP | SIGNAL_TYPE_LVDS)) &&
-	    link->type != dc_connection_none) {
-		amdgpu_dm_register_backlight_device(dm);
-
-		if (dm->backlight_dev) {
-			backlight_device_unregister(dm->backlight_dev);
-			dm->backlight_dev = NULL;
-		}
-
+	    link->type != dc_connection_none &&
+	    dm->backlight_dev) {
+		backlight_device_unregister(dm->backlight_dev);
+		dm->backlight_dev = NULL;
 	}
 #endif
 	drm_connector_unregister(connector);
@@ -2855,7 +2835,7 @@ static void handle_edid_mgmt(struct amdgpu_dm_connector *aconnector)
 	create_eml_sink(aconnector);
 }
 
-int amdgpu_dm_connector_mode_valid(struct drm_connector *connector,
+enum drm_mode_status amdgpu_dm_connector_mode_valid(struct drm_connector *connector,
 				   struct drm_display_mode *mode)
 {
 	int result = MODE_ERROR;
@@ -3058,8 +3038,7 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane,
 	}
 
 	afb = to_amdgpu_framebuffer(new_state->fb);
-
-	obj = afb->obj;
+	obj = new_state->fb->obj[0];
 	rbo = gem_to_amdgpu_bo(obj);
 	adev = amdgpu_ttm_adev(rbo->tbo.bdev);
 	r = amdgpu_bo_reserve(rbo, false);
@@ -3067,12 +3046,11 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane,
 		return r;
 
 	if (plane->type != DRM_PLANE_TYPE_CURSOR)
-		domain = amdgpu_display_framebuffer_domains(adev);
+		domain = amdgpu_display_supported_domains(adev);
 	else
 		domain = AMDGPU_GEM_DOMAIN_VRAM;
 
 	r = amdgpu_bo_pin(rbo, domain, &afb->address);
-
 	amdgpu_bo_unreserve(rbo);
 
 	if (unlikely(r != 0)) {
@@ -3123,14 +3101,12 @@ static void dm_plane_helper_cleanup_fb(struct drm_plane *plane,
 				       struct drm_plane_state *old_state)
 {
 	struct amdgpu_bo *rbo;
-	struct amdgpu_framebuffer *afb;
 	int r;
 
 	if (!old_state->fb)
 		return;
 
-	afb = to_amdgpu_framebuffer(old_state->fb);
-	rbo = gem_to_amdgpu_bo(afb->obj);
+	rbo = gem_to_amdgpu_bo(old_state->fb->obj[0]);
 	r = amdgpu_bo_reserve(rbo, false);
 	if (unlikely(r)) {
 		DRM_ERROR("failed to reserve rbo before unpin\n");
@@ -3773,7 +3749,7 @@ static void remove_stream(struct amdgpu_device *adev,
 static int get_cursor_position(struct drm_plane *plane, struct drm_crtc *crtc,
 			       struct dc_cursor_position *position)
 {
-	struct amdgpu_crtc *amdgpu_crtc = amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	int x, y;
 	int xorigin = 0, yorigin = 0;
 
@@ -3905,7 +3881,7 @@ static void amdgpu_dm_do_flip(struct drm_crtc *crtc,
 	int r, vpos, hpos;
 	struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
 	struct amdgpu_framebuffer *afb = to_amdgpu_framebuffer(fb);
-	struct amdgpu_bo *abo = gem_to_amdgpu_bo(afb->obj);
+	struct amdgpu_bo *abo = gem_to_amdgpu_bo(fb->obj[0]);
 	struct amdgpu_device *adev = crtc->dev->dev_private;
 	bool async_flip = (crtc->state->pageflip_flags & DRM_MODE_PAGE_FLIP_ASYNC) != 0;
 	struct dc_flip_addrs addr = { {0} };
@@ -3986,6 +3962,96 @@ static void amdgpu_dm_do_flip(struct drm_crtc *crtc,
 	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
 }
 
+/*
+ * TODO this whole function needs to go
+ *
+ * dc_surface_update is needlessly complex. See if we can just replace this
+ * with a dc_plane_state and follow the atomic model a bit more closely here.
+ */
+static bool commit_planes_to_stream(
+		struct dc *dc,
+		struct dc_plane_state **plane_states,
+		uint8_t new_plane_count,
+		struct dm_crtc_state *dm_new_crtc_state,
+		struct dm_crtc_state *dm_old_crtc_state,
+		struct dc_state *state)
+{
+	/* no need to dynamically allocate this. it's pretty small */
+	struct dc_surface_update updates[MAX_SURFACES];
+	struct dc_flip_addrs *flip_addr;
+	struct dc_plane_info *plane_info;
+	struct dc_scaling_info *scaling_info;
+	int i;
+	struct dc_stream_state *dc_stream = dm_new_crtc_state->stream;
+	struct dc_stream_update *stream_update =
+			kzalloc(sizeof(struct dc_stream_update), GFP_KERNEL);
+
+	if (!stream_update) {
+		BREAK_TO_DEBUGGER();
+		return false;
+	}
+
+	flip_addr = kcalloc(MAX_SURFACES, sizeof(struct dc_flip_addrs),
+			    GFP_KERNEL);
+	plane_info = kcalloc(MAX_SURFACES, sizeof(struct dc_plane_info),
+			     GFP_KERNEL);
+	scaling_info = kcalloc(MAX_SURFACES, sizeof(struct dc_scaling_info),
+			       GFP_KERNEL);
+
+	if (!flip_addr || !plane_info || !scaling_info) {
+		kfree(flip_addr);
+		kfree(plane_info);
+		kfree(scaling_info);
+		kfree(stream_update);
+		return false;
+	}
+
+	memset(updates, 0, sizeof(updates));
+
+	stream_update->src = dc_stream->src;
+	stream_update->dst = dc_stream->dst;
+	stream_update->out_transfer_func = dc_stream->out_transfer_func;
+
+	for (i = 0; i < new_plane_count; i++) {
+		updates[i].surface = plane_states[i];
+		updates[i].gamma =
+			(struct dc_gamma *)plane_states[i]->gamma_correction;
+		updates[i].in_transfer_func = plane_states[i]->in_transfer_func;
+		flip_addr[i].address = plane_states[i]->address;
+		flip_addr[i].flip_immediate = plane_states[i]->flip_immediate;
+		plane_info[i].color_space = plane_states[i]->color_space;
+		plane_info[i].format = plane_states[i]->format;
+		plane_info[i].plane_size = plane_states[i]->plane_size;
+		plane_info[i].rotation = plane_states[i]->rotation;
+		plane_info[i].horizontal_mirror = plane_states[i]->horizontal_mirror;
+		plane_info[i].stereo_format = plane_states[i]->stereo_format;
+		plane_info[i].tiling_info = plane_states[i]->tiling_info;
+		plane_info[i].visible = plane_states[i]->visible;
+		plane_info[i].per_pixel_alpha = plane_states[i]->per_pixel_alpha;
+		plane_info[i].dcc = plane_states[i]->dcc;
+		scaling_info[i].scaling_quality = plane_states[i]->scaling_quality;
+		scaling_info[i].src_rect = plane_states[i]->src_rect;
+		scaling_info[i].dst_rect = plane_states[i]->dst_rect;
+		scaling_info[i].clip_rect = plane_states[i]->clip_rect;
+
+		updates[i].flip_addr = &flip_addr[i];
+		updates[i].plane_info = &plane_info[i];
+		updates[i].scaling_info = &scaling_info[i];
+	}
+
+	dc_commit_updates_for_stream(
+			dc,
+			updates,
+			new_plane_count,
+			dc_stream, stream_update, plane_states, state);
+
+	kfree(flip_addr);
+	kfree(plane_info);
+	kfree(scaling_info);
+	kfree(stream_update);
+	return true;
+}
+
 static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 				    struct drm_device *dev,
 				    struct amdgpu_display_manager *dm,
@@ -4001,6 +4067,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 	struct drm_crtc_state *new_pcrtc_state =
 			drm_atomic_get_new_crtc_state(state, pcrtc);
 	struct dm_crtc_state *acrtc_state = to_dm_crtc_state(new_pcrtc_state);
+	struct dm_crtc_state *dm_old_crtc_state =
+			to_dm_crtc_state(drm_atomic_get_old_crtc_state(state, pcrtc));
 	struct dm_atomic_state *dm_state = to_dm_atomic_state(state);
 	int planes_count = 0;
 	unsigned long flags;
@@ -4037,7 +4105,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 		}
 		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
 
-		if (!pflip_needed) {
+		if (!pflip_needed || plane->type == DRM_PLANE_TYPE_OVERLAY) {
 			WARN_ON(!dm_new_plane_state->dc_state);
 
 			plane_states_constructed[planes_count] = dm_new_plane_state->dc_state;
@@ -4079,10 +4147,12 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 			spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags);
 		}
 
-		if (false == dc_commit_planes_to_stream(dm->dc,
+
+		if (false == commit_planes_to_stream(dm->dc,
 							plane_states_constructed,
 							planes_count,
-							dc_stream_attach,
+							acrtc_state,
+							dm_old_crtc_state,
 							dm_state->context))
 			dm_error("%s: Failed to attach plane!\n", __func__);
 	} else {
@@ -4307,8 +4377,10 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 		struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc);
 		struct dc_stream_status *status = NULL;
 
-		if (acrtc)
+		if (acrtc) {
 			new_crtc_state = drm_atomic_get_new_crtc_state(state, &acrtc->base);
+			old_crtc_state = drm_atomic_get_old_crtc_state(state, &acrtc->base);
+		}
 
 		/* Skip any modesets/resets */
 		if (!acrtc || drm_atomic_crtc_needs_modeset(new_crtc_state))
@@ -4331,11 +4403,12 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 		WARN_ON(!status->plane_count);
 
 		/*TODO How it works with MPO ?*/
-		if (!dc_commit_planes_to_stream(
+		if (!commit_planes_to_stream(
 				dm->dc,
 				status->plane_states,
 				status->plane_count,
-				dm_new_crtc_state->stream,
+				dm_new_crtc_state,
+				to_dm_crtc_state(old_crtc_state),
 				dm_state->context))
 			dm_error("%s: Failed to update stream scaling!\n", __func__);
 	}
@@ -4578,7 +4651,7 @@ static int dm_update_crtcs_state(struct dc *dc,
 		if (aconnector && enable) {
 			// Make sure fake sink is created in plug-in scenario
 			new_con_state = drm_atomic_get_connector_state(state,
- 								    &aconnector->base);
+								    &aconnector->base);
 
 			if (IS_ERR(new_con_state)) {
 				ret = PTR_ERR_OR_ZERO(new_con_state);
@@ -4755,7 +4828,8 @@ static int dm_update_planes_state(struct dc *dc,
 
 		/* Remove any changed/removed planes */
 		if (!enable) {
-			if (pflip_needed)
+			if (pflip_needed &&
+			    plane->type != DRM_PLANE_TYPE_OVERLAY)
 				continue;
 
 			if (!old_plane_crtc)
@@ -4802,7 +4876,8 @@ static int dm_update_planes_state(struct dc *dc,
 			if (!dm_new_crtc_state->stream)
 				continue;
 
-			if (pflip_needed)
+			if (pflip_needed &&
+			    plane->type != DRM_PLANE_TYPE_OVERLAY)
 				continue;
 
 			WARN_ON(dm_new_plane_state->dc_state);
@@ -5009,17 +5084,24 @@ void amdgpu_dm_add_sink_to_freesync_module(struct drm_connector *connector,
 					   struct edid *edid)
 {
 	int i;
-	uint64_t val_capable;
 	bool edid_check_required;
 	struct detailed_timing *timing;
 	struct detailed_non_pixel *data;
 	struct detailed_data_monitor_range *range;
 	struct amdgpu_dm_connector *amdgpu_dm_connector =
 			to_amdgpu_dm_connector(connector);
+	struct dm_connector_state *dm_con_state;
 
 	struct drm_device *dev = connector->dev;
 	struct amdgpu_device *adev = dev->dev_private;
 
+	if (!connector->state) {
+		DRM_ERROR("%s - Connector has no state", __func__);
+		return;
+	}
+
+	dm_con_state = to_dm_connector_state(connector->state);
+
 	edid_check_required = false;
 	if (!amdgpu_dm_connector->dc_sink) {
 		DRM_ERROR("dc_sink NULL, could not add free_sync module.\n");
@@ -5038,7 +5120,7 @@ void amdgpu_dm_add_sink_to_freesync_module(struct drm_connector *connector,
 						amdgpu_dm_connector);
 		}
 	}
-	val_capable = 0;
+	dm_con_state->freesync_capable = false;
 	if (edid_check_required == true && (edid->version > 1 ||
 	   (edid->version == 1 && edid->revision > 1))) {
 		for (i = 0; i < 4; i++) {
@@ -5074,7 +5156,7 @@ void amdgpu_dm_add_sink_to_freesync_module(struct drm_connector *connector,
 					amdgpu_dm_connector->min_vfreq * 1000000;
 			amdgpu_dm_connector->caps.max_refresh_in_micro_hz =
 					amdgpu_dm_connector->max_vfreq * 1000000;
-				val_capable = 1;
+			dm_con_state->freesync_capable = true;
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
index b68400c1154b..d5aa89ad5571 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
@@ -28,7 +28,6 @@
 
 #include <drm/drmP.h>
 #include <drm/drm_atomic.h>
-#include "dc.h"
 
 /*
  * This file contains the definition for amdgpu_display_manager
@@ -53,6 +52,7 @@
 struct amdgpu_device;
 struct drm_device;
 struct amdgpu_dm_irq_handler_data;
+struct dc;
 
 struct amdgpu_dm_prev_state {
 	struct drm_framebuffer *fb;
@@ -220,6 +220,7 @@ struct dm_connector_state {
 	uint8_t underscan_hborder;
 	bool underscan_enable;
 	struct mod_freesync_user_enable user_enable;
+	bool freesync_capable;
 };
 
 #define to_dm_connector_state(x)\
@@ -246,7 +247,7 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
 				     struct dc_link *link,
 				     int link_index);
 
-int amdgpu_dm_connector_mode_valid(struct drm_connector *connector,
+enum drm_mode_status amdgpu_dm_connector_mode_valid(struct drm_connector *connector,
 				   struct drm_display_mode *mode);
 
 void dm_restore_drm_connector_state(struct drm_device *dev,
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
index 25f064c01038..b329393307e5 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
@@ -25,6 +25,7 @@
 
 #include "amdgpu_mode.h"
 #include "amdgpu_dm.h"
+#include "dc.h"
 #include "modules/color/color_gamma.h"
 
 #define MAX_DRM_LUT_VALUE 0xFFFF
@@ -87,9 +88,9 @@ static void __drm_lut_to_dc_gamma(struct drm_color_lut *lut,
 			g = drm_color_lut_extract(lut[i].green, 16);
 			b = drm_color_lut_extract(lut[i].blue, 16);
 
-			gamma->entries.red[i] = dal_fixed31_32_from_int(r);
-			gamma->entries.green[i] = dal_fixed31_32_from_int(g);
-			gamma->entries.blue[i] = dal_fixed31_32_from_int(b);
+			gamma->entries.red[i] = dc_fixpt_from_int(r);
+			gamma->entries.green[i] = dc_fixpt_from_int(g);
+			gamma->entries.blue[i] = dc_fixpt_from_int(b);
 		}
 		return;
 	}
@@ -100,9 +101,9 @@ static void __drm_lut_to_dc_gamma(struct drm_color_lut *lut,
 		g = drm_color_lut_extract(lut[i].green, 16);
 		b = drm_color_lut_extract(lut[i].blue, 16);
 
-		gamma->entries.red[i] = dal_fixed31_32_from_fraction(r, MAX_DRM_LUT_VALUE);
-		gamma->entries.green[i] = dal_fixed31_32_from_fraction(g, MAX_DRM_LUT_VALUE);
-		gamma->entries.blue[i] = dal_fixed31_32_from_fraction(b, MAX_DRM_LUT_VALUE);
+		gamma->entries.red[i] = dc_fixpt_from_fraction(r, MAX_DRM_LUT_VALUE);
+		gamma->entries.green[i] = dc_fixpt_from_fraction(g, MAX_DRM_LUT_VALUE);
+		gamma->entries.blue[i] = dc_fixpt_from_fraction(b, MAX_DRM_LUT_VALUE);
 	}
 }
 
@@ -207,7 +208,7 @@ void amdgpu_dm_set_ctm(struct dm_crtc_state *crtc)
 	for (i = 0; i < 12; i++) {
 		/* Skip 4th element */
 		if (i % 4 == 3) {
-			stream->gamut_remap_matrix.matrix[i] = dal_fixed31_32_zero;
+			stream->gamut_remap_matrix.matrix[i] = dc_fixpt_zero;
 			continue;
 		}
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index ca0b08bfa2cf..bd449351803f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -330,11 +330,6 @@ bool dm_helpers_dp_mst_send_payload_allocation(
 	return true;
 }
 
-bool dm_helpers_dc_conn_log(struct dc_context *ctx, struct log_entry *entry, enum dc_log_type event)
-{
-	return true;
-}
-
 void dm_dtn_log_begin(struct dc_context *ctx)
 {}
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
index 89342b48be6b..0229c7edb8ad 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
@@ -37,8 +37,17 @@
 
 unsigned long long dm_get_timestamp(struct dc_context *ctx)
 {
-	/* TODO: return actual timestamp */
-	return 0;
+	struct timespec64 time;
+
+	getrawmonotonic64(&time);
+	return timespec64_to_ns(&time);
+}
+
+unsigned long long dm_get_elapse_time_in_ns(struct dc_context *ctx,
+		unsigned long long current_time_stamp,
+		unsigned long long last_time_stamp)
+{
+	return current_time_stamp - last_time_stamp;
 }
 
 void dm_perf_trace_timestamp(const char *func_name, unsigned int line)
diff --git a/drivers/gpu/drm/amd/display/dc/basics/Makefile b/drivers/gpu/drm/amd/display/dc/basics/Makefile
index bca33bd9a0d2..b49ea96b5dae 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/basics/Makefile
@@ -24,7 +24,7 @@
 # It provides the general basic services required by other DAL
 # subcomponents.
 
-BASICS = conversion.o fixpt31_32.o fixpt32_32.o \
+BASICS = conversion.o fixpt31_32.o \
 	logger.o log_helpers.o vector.o
 
 AMD_DAL_BASICS = $(addprefix $(AMDDALPATH)/dc/basics/,$(BASICS))
diff --git a/drivers/gpu/drm/amd/display/dc/basics/conversion.c b/drivers/gpu/drm/amd/display/dc/basics/conversion.c
index 310964915a83..50b47f11875c 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/conversion.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/conversion.c
@@ -41,22 +41,22 @@ uint16_t fixed_point_to_int_frac(
 
 	uint16_t result;
 
-	uint16_t d = (uint16_t)dal_fixed31_32_floor(
-		dal_fixed31_32_abs(
+	uint16_t d = (uint16_t)dc_fixpt_floor(
+		dc_fixpt_abs(
 			arg));
 
 	if (d <= (uint16_t)(1 << integer_bits) - (1 / (uint16_t)divisor))
-		numerator = (uint16_t)dal_fixed31_32_round(
-			dal_fixed31_32_mul_int(
+		numerator = (uint16_t)dc_fixpt_round(
+			dc_fixpt_mul_int(
 				arg,
 				divisor));
 	else {
-		numerator = dal_fixed31_32_floor(
-			dal_fixed31_32_sub(
-				dal_fixed31_32_from_int(
+		numerator = dc_fixpt_floor(
+			dc_fixpt_sub(
+				dc_fixpt_from_int(
 					1LL << integer_bits),
-				dal_fixed31_32_recip(
-					dal_fixed31_32_from_int(
+				dc_fixpt_recip(
+					dc_fixpt_from_int(
 						divisor))));
 	}
 
@@ -66,8 +66,8 @@ uint16_t fixed_point_to_int_frac(
 		result = (uint16_t)(
 		(1 << (integer_bits + fractional_bits + 1)) + numerator);
 
-	if ((result != 0) && dal_fixed31_32_lt(
-		arg, dal_fixed31_32_zero))
+	if ((result != 0) && dc_fixpt_lt(
+		arg, dc_fixpt_zero))
 		result |= 1 << (integer_bits + fractional_bits);
 
 	return result;
@@ -84,15 +84,15 @@ void convert_float_matrix(
 	uint32_t buffer_size)
 {
 	const struct fixed31_32 min_2_13 =
-		dal_fixed31_32_from_fraction(S2D13_MIN, DIVIDER);
+		dc_fixpt_from_fraction(S2D13_MIN, DIVIDER);
 	const struct fixed31_32 max_2_13 =
-		dal_fixed31_32_from_fraction(S2D13_MAX, DIVIDER);
+		dc_fixpt_from_fraction(S2D13_MAX, DIVIDER);
 	uint32_t i;
 
 	for (i = 0; i < buffer_size; ++i) {
 		uint32_t reg_value =
 				fixed_point_to_int_frac(
-					dal_fixed31_32_clamp(
+					dc_fixpt_clamp(
 						flt[i],
 						min_2_13,
 						max_2_13),
diff --git a/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c b/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c
index 8a9bba879207..e61dd97d0928 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c
@@ -26,13 +26,13 @@
 #include "dm_services.h"
 #include "include/fixed31_32.h"
 
-static inline uint64_t abs_i64(
-	int64_t arg)
+static inline unsigned long long abs_i64(
+	long long arg)
 {
 	if (arg > 0)
-		return (uint64_t)arg;
+		return (unsigned long long)arg;
 	else
-		return (uint64_t)(-arg);
+		return (unsigned long long)(-arg);
 }
 
 /*
@@ -40,12 +40,12 @@ static inline uint64_t abs_i64(
  * result = dividend / divisor
  * *remainder = dividend % divisor
  */
-static inline uint64_t complete_integer_division_u64(
-	uint64_t dividend,
-	uint64_t divisor,
-	uint64_t *remainder)
+static inline unsigned long long complete_integer_division_u64(
+	unsigned long long dividend,
+	unsigned long long divisor,
+	unsigned long long *remainder)
 {
-	uint64_t result;
+	unsigned long long result;
 
 	ASSERT(divisor);
 
@@ -64,30 +64,28 @@ static inline uint64_t complete_integer_division_u64(
 #define GET_FRACTIONAL_PART(x) \
 	(FRACTIONAL_PART_MASK & (x))
 
-struct fixed31_32 dal_fixed31_32_from_fraction(
-	int64_t numerator,
-	int64_t denominator)
+struct fixed31_32 dc_fixpt_from_fraction(long long numerator, long long denominator)
 {
 	struct fixed31_32 res;
 
 	bool arg1_negative = numerator < 0;
 	bool arg2_negative = denominator < 0;
 
-	uint64_t arg1_value = arg1_negative ? -numerator : numerator;
-	uint64_t arg2_value = arg2_negative ? -denominator : denominator;
+	unsigned long long arg1_value = arg1_negative ? -numerator : numerator;
+	unsigned long long arg2_value = arg2_negative ? -denominator : denominator;
 
-	uint64_t remainder;
+	unsigned long long remainder;
 
 	/* determine integer part */
 
-	uint64_t res_value = complete_integer_division_u64(
+	unsigned long long res_value = complete_integer_division_u64(
 		arg1_value, arg2_value, &remainder);
 
 	ASSERT(res_value <= LONG_MAX);
 
 	/* determine fractional part */
 	{
-		uint32_t i = FIXED31_32_BITS_PER_FRACTIONAL_PART;
+		unsigned int i = FIXED31_32_BITS_PER_FRACTIONAL_PART;
 
 		do {
 			remainder <<= 1;
@@ -103,14 +101,14 @@ struct fixed31_32 dal_fixed31_32_from_fraction(
 
 	/* round up LSB */
 	{
-		uint64_t summand = (remainder << 1) >= arg2_value;
+		unsigned long long summand = (remainder << 1) >= arg2_value;
 
 		ASSERT(res_value <= LLONG_MAX - summand);
 
 		res_value += summand;
 	}
 
-	res.value = (int64_t)res_value;
+	res.value = (long long)res_value;
 
 	if (arg1_negative ^ arg2_negative)
 		res.value = -res.value;
@@ -118,79 +116,23 @@ struct fixed31_32 dal_fixed31_32_from_fraction(
 	return res;
 }
 
-struct fixed31_32 dal_fixed31_32_from_int_nonconst(
-	int64_t arg)
-{
-	struct fixed31_32 res;
-
-	ASSERT((LONG_MIN <= arg) && (arg <= LONG_MAX));
-
-	res.value = arg << FIXED31_32_BITS_PER_FRACTIONAL_PART;
-
-	return res;
-}
-
-struct fixed31_32 dal_fixed31_32_shl(
-	struct fixed31_32 arg,
-	uint8_t shift)
-{
-	struct fixed31_32 res;
-
-	ASSERT(((arg.value >= 0) && (arg.value <= LLONG_MAX >> shift)) ||
-		((arg.value < 0) && (arg.value >= LLONG_MIN >> shift)));
-
-	res.value = arg.value << shift;
-
-	return res;
-}
-
-struct fixed31_32 dal_fixed31_32_add(
-	struct fixed31_32 arg1,
-	struct fixed31_32 arg2)
-{
-	struct fixed31_32 res;
-
-	ASSERT(((arg1.value >= 0) && (LLONG_MAX - arg1.value >= arg2.value)) ||
-		((arg1.value < 0) && (LLONG_MIN - arg1.value <= arg2.value)));
-
-	res.value = arg1.value + arg2.value;
-
-	return res;
-}
-
-struct fixed31_32 dal_fixed31_32_sub(
-	struct fixed31_32 arg1,
-	struct fixed31_32 arg2)
-{
-	struct fixed31_32 res;
-
-	ASSERT(((arg2.value >= 0) && (LLONG_MIN + arg2.value <= arg1.value)) ||
-		((arg2.value < 0) && (LLONG_MAX + arg2.value >= arg1.value)));
-
-	res.value = arg1.value - arg2.value;
-
-	return res;
-}
-
-struct fixed31_32 dal_fixed31_32_mul(
-	struct fixed31_32 arg1,
-	struct fixed31_32 arg2)
+struct fixed31_32 dc_fixpt_mul(struct fixed31_32 arg1, struct fixed31_32 arg2)
 {
 	struct fixed31_32 res;
 
 	bool arg1_negative = arg1.value < 0;
 	bool arg2_negative = arg2.value < 0;
 
-	uint64_t arg1_value = arg1_negative ? -arg1.value : arg1.value;
-	uint64_t arg2_value = arg2_negative ? -arg2.value : arg2.value;
+	unsigned long long arg1_value = arg1_negative ? -arg1.value : arg1.value;
+	unsigned long long arg2_value = arg2_negative ? -arg2.value : arg2.value;
 
-	uint64_t arg1_int = GET_INTEGER_PART(arg1_value);
-	uint64_t arg2_int = GET_INTEGER_PART(arg2_value);
+	unsigned long long arg1_int = GET_INTEGER_PART(arg1_value);
+	unsigned long long arg2_int = GET_INTEGER_PART(arg2_value);
 
-	uint64_t arg1_fra = GET_FRACTIONAL_PART(arg1_value);
-	uint64_t arg2_fra = GET_FRACTIONAL_PART(arg2_value);
+	unsigned long long arg1_fra = GET_FRACTIONAL_PART(arg1_value);
+	unsigned long long arg2_fra = GET_FRACTIONAL_PART(arg2_value);
 
-	uint64_t tmp;
+	unsigned long long tmp;
 
 	res.value = arg1_int * arg2_int;
 
@@ -200,22 +142,22 @@ struct fixed31_32 dal_fixed31_32_mul(
 
 	tmp = arg1_int * arg2_fra;
 
-	ASSERT(tmp <= (uint64_t)(LLONG_MAX - res.value));
+	ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
 
 	res.value += tmp;
 
 	tmp = arg2_int * arg1_fra;
 
-	ASSERT(tmp <= (uint64_t)(LLONG_MAX - res.value));
+	ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
 
 	res.value += tmp;
 
 	tmp = arg1_fra * arg2_fra;
 
 	tmp = (tmp >> FIXED31_32_BITS_PER_FRACTIONAL_PART) +
-		(tmp >= (uint64_t)dal_fixed31_32_half.value);
+		(tmp >= (unsigned long long)dc_fixpt_half.value);
 
-	ASSERT(tmp <= (uint64_t)(LLONG_MAX - res.value));
+	ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
 
 	res.value += tmp;
 
@@ -225,18 +167,17 @@ struct fixed31_32 dal_fixed31_32_mul(
 	return res;
 }
 
-struct fixed31_32 dal_fixed31_32_sqr(
-	struct fixed31_32 arg)
+struct fixed31_32 dc_fixpt_sqr(struct fixed31_32 arg)
 {
 	struct fixed31_32 res;
 
-	uint64_t arg_value = abs_i64(arg.value);
+	unsigned long long arg_value = abs_i64(arg.value);
 
-	uint64_t arg_int = GET_INTEGER_PART(arg_value);
+	unsigned long long arg_int = GET_INTEGER_PART(arg_value);
 
-	uint64_t arg_fra = GET_FRACTIONAL_PART(arg_value);
+	unsigned long long arg_fra = GET_FRACTIONAL_PART(arg_value);
 
-	uint64_t tmp;
+	unsigned long long tmp;
 
 	res.value = arg_int * arg_int;
 
@@ -246,28 +187,27 @@ struct fixed31_32 dal_fixed31_32_sqr(
 
 	tmp = arg_int * arg_fra;
 
-	ASSERT(tmp <= (uint64_t)(LLONG_MAX - res.value));
+	ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
 
 	res.value += tmp;
 
-	ASSERT(tmp <= (uint64_t)(LLONG_MAX - res.value));
+	ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
 
 	res.value += tmp;
 
 	tmp = arg_fra * arg_fra;
 
 	tmp = (tmp >> FIXED31_32_BITS_PER_FRACTIONAL_PART) +
-		(tmp >= (uint64_t)dal_fixed31_32_half.value);
+		(tmp >= (unsigned long long)dc_fixpt_half.value);
 
-	ASSERT(tmp <= (uint64_t)(LLONG_MAX - res.value));
+	ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
 
 	res.value += tmp;
 
 	return res;
 }
 
-struct fixed31_32 dal_fixed31_32_recip(
-	struct fixed31_32 arg)
+struct fixed31_32 dc_fixpt_recip(struct fixed31_32 arg)
 {
 	/*
 	 * @note
@@ -276,41 +216,40 @@ struct fixed31_32 dal_fixed31_32_recip(
 
 	ASSERT(arg.value);
 
-	return dal_fixed31_32_from_fraction(
-		dal_fixed31_32_one.value,
+	return dc_fixpt_from_fraction(
+		dc_fixpt_one.value,
 		arg.value);
 }
 
-struct fixed31_32 dal_fixed31_32_sinc(
-	struct fixed31_32 arg)
+struct fixed31_32 dc_fixpt_sinc(struct fixed31_32 arg)
 {
 	struct fixed31_32 square;
 
-	struct fixed31_32 res = dal_fixed31_32_one;
+	struct fixed31_32 res = dc_fixpt_one;
 
-	int32_t n = 27;
+	int n = 27;
 
 	struct fixed31_32 arg_norm = arg;
 
-	if (dal_fixed31_32_le(
-		dal_fixed31_32_two_pi,
-		dal_fixed31_32_abs(arg))) {
-		arg_norm = dal_fixed31_32_sub(
+	if (dc_fixpt_le(
+		dc_fixpt_two_pi,
+		dc_fixpt_abs(arg))) {
+		arg_norm = dc_fixpt_sub(
 			arg_norm,
-			dal_fixed31_32_mul_int(
-				dal_fixed31_32_two_pi,
-				(int32_t)div64_s64(
+			dc_fixpt_mul_int(
+				dc_fixpt_two_pi,
+				(int)div64_s64(
 					arg_norm.value,
-					dal_fixed31_32_two_pi.value)));
+					dc_fixpt_two_pi.value)));
 	}
 
-	square = dal_fixed31_32_sqr(arg_norm);
+	square = dc_fixpt_sqr(arg_norm);
 
 	do {
-		res = dal_fixed31_32_sub(
-			dal_fixed31_32_one,
-			dal_fixed31_32_div_int(
-				dal_fixed31_32_mul(
+		res = dc_fixpt_sub(
+			dc_fixpt_one,
+			dc_fixpt_div_int(
+				dc_fixpt_mul(
 					square,
 					res),
 				n * (n - 1)));
@@ -319,37 +258,35 @@ struct fixed31_32 dal_fixed31_32_sinc(
 	} while (n > 2);
 
 	if (arg.value != arg_norm.value)
-		res = dal_fixed31_32_div(
-			dal_fixed31_32_mul(res, arg_norm),
+		res = dc_fixpt_div(
+			dc_fixpt_mul(res, arg_norm),
 			arg);
 
 	return res;
 }
 
-struct fixed31_32 dal_fixed31_32_sin(
-	struct fixed31_32 arg)
+struct fixed31_32 dc_fixpt_sin(struct fixed31_32 arg)
 {
-	return dal_fixed31_32_mul(
+	return dc_fixpt_mul(
 		arg,
-		dal_fixed31_32_sinc(arg));
+		dc_fixpt_sinc(arg));
 }
 
-struct fixed31_32 dal_fixed31_32_cos(
-	struct fixed31_32 arg)
+struct fixed31_32 dc_fixpt_cos(struct fixed31_32 arg)
 {
 	/* TODO implement argument normalization */
 
-	const struct fixed31_32 square = dal_fixed31_32_sqr(arg);
+	const struct fixed31_32 square = dc_fixpt_sqr(arg);
 
-	struct fixed31_32 res = dal_fixed31_32_one;
+	struct fixed31_32 res = dc_fixpt_one;
 
-	int32_t n = 26;
+	int n = 26;
 
 	do {
-		res = dal_fixed31_32_sub(
-			dal_fixed31_32_one,
-			dal_fixed31_32_div_int(
-				dal_fixed31_32_mul(
+		res = dc_fixpt_sub(
+			dc_fixpt_one,
+			dc_fixpt_div_int(
+				dc_fixpt_mul(
 					square,
 					res),
 				n * (n - 1)));
@@ -367,37 +304,35 @@ struct fixed31_32 dal_fixed31_32_cos(
  *
  * Calculated as Taylor series.
  */
-static struct fixed31_32 fixed31_32_exp_from_taylor_series(
-	struct fixed31_32 arg)
+static struct fixed31_32 fixed31_32_exp_from_taylor_series(struct fixed31_32 arg)
 {
-	uint32_t n = 9;
+	unsigned int n = 9;
 
-	struct fixed31_32 res = dal_fixed31_32_from_fraction(
+	struct fixed31_32 res = dc_fixpt_from_fraction(
 		n + 2,
 		n + 1);
 	/* TODO find correct res */
 
-	ASSERT(dal_fixed31_32_lt(arg, dal_fixed31_32_one));
+	ASSERT(dc_fixpt_lt(arg, dc_fixpt_one));
 
 	do
-		res = dal_fixed31_32_add(
-			dal_fixed31_32_one,
-			dal_fixed31_32_div_int(
-				dal_fixed31_32_mul(
+		res = dc_fixpt_add(
+			dc_fixpt_one,
+			dc_fixpt_div_int(
+				dc_fixpt_mul(
 					arg,
 					res),
 				n));
 	while (--n != 1);
 
-	return dal_fixed31_32_add(
-		dal_fixed31_32_one,
-		dal_fixed31_32_mul(
+	return dc_fixpt_add(
+		dc_fixpt_one,
+		dc_fixpt_mul(
 			arg,
 			res));
 }
 
-struct fixed31_32 dal_fixed31_32_exp(
-	struct fixed31_32 arg)
+struct fixed31_32 dc_fixpt_exp(struct fixed31_32 arg)
 {
 	/*
 	 * @brief
@@ -406,44 +341,43 @@ struct fixed31_32 dal_fixed31_32_exp(
 	 * where m = round(x / ln(2)), r = x - m * ln(2)
 	 */
 
-	if (dal_fixed31_32_le(
-		dal_fixed31_32_ln2_div_2,
-		dal_fixed31_32_abs(arg))) {
-		int32_t m = dal_fixed31_32_round(
-			dal_fixed31_32_div(
+	if (dc_fixpt_le(
+		dc_fixpt_ln2_div_2,
+		dc_fixpt_abs(arg))) {
+		int m = dc_fixpt_round(
+			dc_fixpt_div(
 				arg,
-				dal_fixed31_32_ln2));
+				dc_fixpt_ln2));
 
-		struct fixed31_32 r = dal_fixed31_32_sub(
+		struct fixed31_32 r = dc_fixpt_sub(
 			arg,
-			dal_fixed31_32_mul_int(
-				dal_fixed31_32_ln2,
+			dc_fixpt_mul_int(
+				dc_fixpt_ln2,
 				m));
 
 		ASSERT(m != 0);
 
-		ASSERT(dal_fixed31_32_lt(
-			dal_fixed31_32_abs(r),
-			dal_fixed31_32_one));
+		ASSERT(dc_fixpt_lt(
+			dc_fixpt_abs(r),
+			dc_fixpt_one));
 
 		if (m > 0)
-			return dal_fixed31_32_shl(
+			return dc_fixpt_shl(
 				fixed31_32_exp_from_taylor_series(r),
-				(uint8_t)m);
+				(unsigned char)m);
 		else
-			return dal_fixed31_32_div_int(
+			return dc_fixpt_div_int(
 				fixed31_32_exp_from_taylor_series(r),
 				1LL << -m);
 	} else if (arg.value != 0)
 		return fixed31_32_exp_from_taylor_series(arg);
 	else
-		return dal_fixed31_32_one;
+		return dc_fixpt_one;
 }
 
-struct fixed31_32 dal_fixed31_32_log(
-	struct fixed31_32 arg)
+struct fixed31_32 dc_fixpt_log(struct fixed31_32 arg)
 {
-	struct fixed31_32 res = dal_fixed31_32_neg(dal_fixed31_32_one);
+	struct fixed31_32 res = dc_fixpt_neg(dc_fixpt_one);
 	/* TODO improve 1st estimation */
 
 	struct fixed31_32 error;
@@ -453,15 +387,15 @@ struct fixed31_32 dal_fixed31_32_log(
 	/* TODO if arg is zero, return -INF */
 
 	do {
-		struct fixed31_32 res1 = dal_fixed31_32_add(
-			dal_fixed31_32_sub(
+		struct fixed31_32 res1 = dc_fixpt_add(
+			dc_fixpt_sub(
 				res,
-				dal_fixed31_32_one),
-			dal_fixed31_32_div(
+				dc_fixpt_one),
+			dc_fixpt_div(
 				arg,
-				dal_fixed31_32_exp(res)));
+				dc_fixpt_exp(res)));
 
-		error = dal_fixed31_32_sub(
+		error = dc_fixpt_sub(
 			res,
 			res1);
 
@@ -472,78 +406,23 @@ struct fixed31_32 dal_fixed31_32_log(
 	return res;
 }
 
-struct fixed31_32 dal_fixed31_32_pow(
-	struct fixed31_32 arg1,
-	struct fixed31_32 arg2)
-{
-	return dal_fixed31_32_exp(
-		dal_fixed31_32_mul(
-			dal_fixed31_32_log(arg1),
-			arg2));
-}
-
-int32_t dal_fixed31_32_floor(
-	struct fixed31_32 arg)
-{
-	uint64_t arg_value = abs_i64(arg.value);
-
-	if (arg.value >= 0)
-		return (int32_t)GET_INTEGER_PART(arg_value);
-	else
-		return -(int32_t)GET_INTEGER_PART(arg_value);
-}
-
-int32_t dal_fixed31_32_round(
-	struct fixed31_32 arg)
-{
-	uint64_t arg_value = abs_i64(arg.value);
-
-	const int64_t summand = dal_fixed31_32_half.value;
-
-	ASSERT(LLONG_MAX - (int64_t)arg_value >= summand);
-
-	arg_value += summand;
-
-	if (arg.value >= 0)
-		return (int32_t)GET_INTEGER_PART(arg_value);
-	else
-		return -(int32_t)GET_INTEGER_PART(arg_value);
-}
-
-int32_t dal_fixed31_32_ceil(
-	struct fixed31_32 arg)
-{
-	uint64_t arg_value = abs_i64(arg.value);
-
-	const int64_t summand = dal_fixed31_32_one.value -
-		dal_fixed31_32_epsilon.value;
-
-	ASSERT(LLONG_MAX - (int64_t)arg_value >= summand);
-
-	arg_value += summand;
-
-	if (arg.value >= 0)
-		return (int32_t)GET_INTEGER_PART(arg_value);
-	else
-		return -(int32_t)GET_INTEGER_PART(arg_value);
-}
 
 /* this function is a generic helper to translate fixed point value to
  * specified integer format that will consist of integer_bits integer part and
  * fractional_bits fractional part. For example it is used in
- * dal_fixed31_32_u2d19 to receive 2 bits integer part and 19 bits fractional
+ * dc_fixpt_u2d19 to receive 2 bits integer part and 19 bits fractional
  * part in 32 bits. It is used in hw programming (scaler)
  */
 
-static inline uint32_t ux_dy(
-	int64_t value,
-	uint32_t integer_bits,
-	uint32_t fractional_bits)
+static inline unsigned int ux_dy(
+	long long value,
+	unsigned int integer_bits,
+	unsigned int fractional_bits)
 {
 	/* 1. create mask of integer part */
-	uint32_t result = (1 << integer_bits) - 1;
+	unsigned int result = (1 << integer_bits) - 1;
 	/* 2. mask out fractional part */
-	uint32_t fractional_part = FRACTIONAL_PART_MASK & value;
+	unsigned int fractional_part = FRACTIONAL_PART_MASK & value;
 	/* 3. shrink fixed point integer part to be of integer_bits width*/
 	result &= GET_INTEGER_PART(value);
 	/* 4. make space for fractional part to be filled in after integer */
@@ -554,13 +433,13 @@ static inline uint32_t ux_dy(
 	return result | fractional_part;
 }
 
-static inline uint32_t clamp_ux_dy(
-	int64_t value,
-	uint32_t integer_bits,
-	uint32_t fractional_bits,
-	uint32_t min_clamp)
+static inline unsigned int clamp_ux_dy(
+	long long value,
+	unsigned int integer_bits,
+	unsigned int fractional_bits,
+	unsigned int min_clamp)
 {
-	uint32_t truncated_val = ux_dy(value, integer_bits, fractional_bits);
+	unsigned int truncated_val = ux_dy(value, integer_bits, fractional_bits);
 
 	if (value >= (1LL << (integer_bits + FIXED31_32_BITS_PER_FRACTIONAL_PART)))
 		return (1 << (integer_bits + fractional_bits)) - 1;
@@ -570,35 +449,30 @@ static inline uint32_t clamp_ux_dy(
 		return min_clamp;
 }
 
-uint32_t dal_fixed31_32_u2d19(
-	struct fixed31_32 arg)
+unsigned int dc_fixpt_u2d19(struct fixed31_32 arg)
 {
 	return ux_dy(arg.value, 2, 19);
 }
 
-uint32_t dal_fixed31_32_u0d19(
-	struct fixed31_32 arg)
+unsigned int dc_fixpt_u0d19(struct fixed31_32 arg)
 {
 	return ux_dy(arg.value, 0, 19);
 }
 
-uint32_t dal_fixed31_32_clamp_u0d14(
-	struct fixed31_32 arg)
+unsigned int dc_fixpt_clamp_u0d14(struct fixed31_32 arg)
 {
 	return clamp_ux_dy(arg.value, 0, 14, 1);
 }
 
-uint32_t dal_fixed31_32_clamp_u0d10(
-	struct fixed31_32 arg)
+unsigned int dc_fixpt_clamp_u0d10(struct fixed31_32 arg)
 {
 	return clamp_ux_dy(arg.value, 0, 10, 1);
 }
 
-int32_t dal_fixed31_32_s4d19(
-	struct fixed31_32 arg)
+int dc_fixpt_s4d19(struct fixed31_32 arg)
 {
 	if (arg.value < 0)
-		return -(int32_t)ux_dy(dal_fixed31_32_abs(arg).value, 4, 19);
+		return -(int)ux_dy(dc_fixpt_abs(arg).value, 4, 19);
 	else
 		return ux_dy(arg.value, 4, 19);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/basics/fixpt32_32.c b/drivers/gpu/drm/amd/display/dc/basics/fixpt32_32.c
deleted file mode 100644
index 4d3aaa82a07b..000000000000
--- a/drivers/gpu/drm/amd/display/dc/basics/fixpt32_32.c
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dm_services.h"
-#include "include/fixed32_32.h"
-
-static uint64_t u64_div(uint64_t n, uint64_t d)
-{
-	uint32_t i = 0;
-	uint64_t r;
-	uint64_t q = div64_u64_rem(n, d, &r);
-
-	for (i = 0; i < 32; ++i) {
-		uint64_t sbit = q & (1ULL<<63);
-
-		r <<= 1;
-		r |= sbit ? 1 : 0;
-		q <<= 1;
-		if (r >= d) {
-			r -= d;
-			q |= 1;
-		}
-	}
-
-	if (2*r >= d)
-		q += 1;
-	return q;
-}
-
-struct fixed32_32 dal_fixed32_32_from_fraction(uint32_t n, uint32_t d)
-{
-	struct fixed32_32 fx;
-
-	fx.value = u64_div((uint64_t)n << 32, (uint64_t)d << 32);
-	return fx;
-}
-
-struct fixed32_32 dal_fixed32_32_add(
-	struct fixed32_32 lhs,
-	struct fixed32_32 rhs)
-{
-	struct fixed32_32 fx = {lhs.value + rhs.value};
-
-	ASSERT(fx.value >= rhs.value);
-	return fx;
-}
-
-struct fixed32_32 dal_fixed32_32_add_int(struct fixed32_32 lhs, uint32_t rhs)
-{
-	struct fixed32_32 fx = {lhs.value + ((uint64_t)rhs << 32)};
-
-	ASSERT(fx.value >= (uint64_t)rhs << 32);
-	return fx;
-
-}
-struct fixed32_32 dal_fixed32_32_sub(
-	struct fixed32_32 lhs,
-	struct fixed32_32 rhs)
-{
-	struct fixed32_32 fx;
-
-	ASSERT(lhs.value >= rhs.value);
-	fx.value = lhs.value - rhs.value;
-	return fx;
-}
-
-struct fixed32_32 dal_fixed32_32_sub_int(struct fixed32_32 lhs, uint32_t rhs)
-{
-	struct fixed32_32 fx;
-
-	ASSERT(lhs.value >= ((uint64_t)rhs<<32));
-	fx.value = lhs.value - ((uint64_t)rhs<<32);
-	return fx;
-}
-
-struct fixed32_32 dal_fixed32_32_mul(
-	struct fixed32_32 lhs,
-	struct fixed32_32 rhs)
-{
-	struct fixed32_32 fx;
-	uint64_t lhs_int = lhs.value>>32;
-	uint64_t lhs_frac = (uint32_t)lhs.value;
-	uint64_t rhs_int = rhs.value>>32;
-	uint64_t rhs_frac = (uint32_t)rhs.value;
-	uint64_t ahbh = lhs_int * rhs_int;
-	uint64_t ahbl = lhs_int * rhs_frac;
-	uint64_t albh = lhs_frac * rhs_int;
-	uint64_t albl = lhs_frac * rhs_frac;
-
-	ASSERT((ahbh>>32) == 0);
-
-	fx.value = (ahbh<<32) + ahbl + albh + (albl>>32);
-	return fx;
-
-}
-
-struct fixed32_32 dal_fixed32_32_mul_int(struct fixed32_32 lhs, uint32_t rhs)
-{
-	struct fixed32_32 fx;
-	uint64_t lhsi = (lhs.value>>32) * (uint64_t)rhs;
-	uint64_t lhsf;
-
-	ASSERT((lhsi>>32) == 0);
-	lhsf = ((uint32_t)lhs.value) * (uint64_t)rhs;
-	ASSERT((lhsi<<32) + lhsf >= lhsf);
-	fx.value = (lhsi<<32) + lhsf;
-	return fx;
-}
-
-struct fixed32_32 dal_fixed32_32_div(
-	struct fixed32_32 lhs,
-	struct fixed32_32 rhs)
-{
-	struct fixed32_32 fx;
-
-	fx.value = u64_div(lhs.value, rhs.value);
-	return fx;
-}
-
-struct fixed32_32 dal_fixed32_32_div_int(struct fixed32_32 lhs, uint32_t rhs)
-{
-	struct fixed32_32 fx;
-
-	fx.value = u64_div(lhs.value, (uint64_t)rhs << 32);
-	return fx;
-}
-
-uint32_t dal_fixed32_32_ceil(struct fixed32_32 v)
-{
-	ASSERT((uint32_t)v.value ? (v.value >> 32) + 1 >= 1 : true);
-	return (v.value>>32) + ((uint32_t)v.value ? 1 : 0);
-}
-
-uint32_t dal_fixed32_32_round(struct fixed32_32 v)
-{
-	ASSERT(v.value + (1ULL<<31) >= (1ULL<<31));
-	return (v.value + (1ULL<<31))>>32;
-}
-
diff --git a/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c b/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c
index 854678a0c54b..021451549ff7 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c
@@ -94,7 +94,6 @@ void dc_conn_log(struct dc_context *ctx,
 			dm_logger_append(&entry, "%2.2X ", hex_data[i]);
 
 	dm_logger_append(&entry, "^\n");
-	dm_helpers_dc_conn_log(ctx, &entry, event);
 
 fail:
 	dm_logger_close(&entry);
diff --git a/drivers/gpu/drm/amd/display/dc/basics/logger.c b/drivers/gpu/drm/amd/display/dc/basics/logger.c
index 31bee054f43a..738a818d58d1 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/logger.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/logger.c
@@ -61,7 +61,7 @@ static const struct dc_log_type_info log_type_info_tbl[] = {
 		{LOG_EVENT_UNDERFLOW,       "Underflow"},
 		{LOG_IF_TRACE,              "InterfaceTrace"},
 		{LOG_DTN,                   "DTN"},
-		{LOG_PROFILING,             "Profiling"}
+		{LOG_DISPLAYSTATS,          "DisplayStats"}
 };
 
 
@@ -402,3 +402,4 @@ void dm_logger_close(struct log_entry *entry)
 		entry->max_buf_bytes = 0;
 	}
 }
+
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
index 10a5807a7e8b..b8cef7af3c4a 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
@@ -1330,6 +1330,9 @@ static enum bp_result bios_parser_get_firmware_info(
 			case 2:
 				result = get_firmware_info_v3_2(bp, info);
 				break;
+			case 3:
+				result = get_firmware_info_v3_2(bp, info);
+				break;
 			default:
 				break;
 			}
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
index 4b5fdd577848..651e1fd4622f 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
@@ -24,7 +24,7 @@
  */
 
 #include "dm_services.h"
-
+#include "amdgpu.h"
 #include "atom.h"
 
 #include "include/bios_parser_interface.h"
@@ -35,16 +35,16 @@
 #include "bios_parser_types_internal.h"
 
 #define EXEC_BIOS_CMD_TABLE(command, params)\
-	(cgs_atom_exec_cmd_table(bp->base.ctx->cgs_device, \
+	(amdgpu_atom_execute_table(((struct amdgpu_device *)bp->base.ctx->driver_context)->mode_info.atom_context, \
 		GetIndexIntoMasterTable(COMMAND, command), \
-		&params) == 0)
+		(uint32_t *)&params) == 0)
 
 #define BIOS_CMD_TABLE_REVISION(command, frev, crev)\
-	cgs_atom_get_cmd_table_revs(bp->base.ctx->cgs_device, \
+	amdgpu_atom_parse_cmd_header(((struct amdgpu_device *)bp->base.ctx->driver_context)->mode_info.atom_context, \
 		GetIndexIntoMasterTable(COMMAND, command), &frev, &crev)
 
 #define BIOS_CMD_TABLE_PARA_REVISION(command)\
-	bios_cmd_table_para_revision(bp->base.ctx->cgs_device, \
+	bios_cmd_table_para_revision(bp->base.ctx->driver_context, \
 		GetIndexIntoMasterTable(COMMAND, command))
 
 static void init_dig_encoder_control(struct bios_parser *bp);
@@ -82,16 +82,18 @@ void dal_bios_parser_init_cmd_tbl(struct bios_parser *bp)
 	init_set_dce_clock(bp);
 }
 
-static uint32_t bios_cmd_table_para_revision(void *cgs_device,
+static uint32_t bios_cmd_table_para_revision(void *dev,
 					     uint32_t index)
 {
+	struct amdgpu_device *adev = dev;
 	uint8_t frev, crev;
 
-	if (cgs_atom_get_cmd_table_revs(cgs_device,
+	if (amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context,
 					index,
-					&frev, &crev) != 0)
+					&frev, &crev))
+		return crev;
+	else
 		return 0;
-	return crev;
 }
 
 /*******************************************************************************
@@ -368,7 +370,7 @@ static void init_transmitter_control(struct bios_parser *bp)
 	uint8_t crev;
 
 	if (BIOS_CMD_TABLE_REVISION(UNIPHYTransmitterControl,
-			frev, crev) != 0)
+			frev, crev) == false)
 		BREAK_TO_DEBUGGER();
 	switch (crev) {
 	case 2:
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
index 3f63f712c8a4..752b08a42d3e 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
@@ -26,14 +26,18 @@
 #include "dm_services.h"
 
 #include "ObjectID.h"
-#include "atomfirmware.h"
 
+#include "atomfirmware.h"
+#include "atom.h"
 #include "include/bios_parser_interface.h"
 
 #include "command_table2.h"
 #include "command_table_helper2.h"
 #include "bios_parser_helper.h"
 #include "bios_parser_types_internal2.h"
+#include "amdgpu.h"
+
+
 #define DC_LOGGER \
 	bp->base.ctx->logger
 
@@ -43,16 +47,16 @@
 		->FieldName)-(char *)0)/sizeof(uint16_t))
 
 #define EXEC_BIOS_CMD_TABLE(fname, params)\
-	(cgs_atom_exec_cmd_table(bp->base.ctx->cgs_device, \
+	(amdgpu_atom_execute_table(((struct amdgpu_device *)bp->base.ctx->driver_context)->mode_info.atom_context, \
 		GET_INDEX_INTO_MASTER_TABLE(command, fname), \
-		&params) == 0)
+		(uint32_t *)&params) == 0)
 
 #define BIOS_CMD_TABLE_REVISION(fname, frev, crev)\
-	cgs_atom_get_cmd_table_revs(bp->base.ctx->cgs_device, \
+	amdgpu_atom_parse_cmd_header(((struct amdgpu_device *)bp->base.ctx->driver_context)->mode_info.atom_context, \
 		GET_INDEX_INTO_MASTER_TABLE(command, fname), &frev, &crev)
 
 #define BIOS_CMD_TABLE_PARA_REVISION(fname)\
-	bios_cmd_table_para_revision(bp->base.ctx->cgs_device, \
+	bios_cmd_table_para_revision(bp->base.ctx->driver_context, \
 			GET_INDEX_INTO_MASTER_TABLE(command, fname))
 
 static void init_dig_encoder_control(struct bios_parser *bp);
@@ -86,16 +90,18 @@ void dal_firmware_parser_init_cmd_tbl(struct bios_parser *bp)
 	init_get_smu_clock_info(bp);
 }
 
-static uint32_t bios_cmd_table_para_revision(void *cgs_device,
+static uint32_t bios_cmd_table_para_revision(void *dev,
 					     uint32_t index)
 {
+	struct amdgpu_device *adev = dev;
 	uint8_t frev, crev;
 
-	if (cgs_atom_get_cmd_table_revs(cgs_device,
+	if (amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context,
 					index,
-					&frev, &crev) != 0)
+					&frev, &crev))
+		return crev;
+	else
 		return 0;
-	return crev;
 }
 
 /******************************************************************************
@@ -201,7 +207,7 @@ static void init_transmitter_control(struct bios_parser *bp)
 	uint8_t frev;
 	uint8_t crev;
 
-	if (BIOS_CMD_TABLE_REVISION(dig1transmittercontrol, frev, crev) != 0)
+	if (BIOS_CMD_TABLE_REVISION(dig1transmittercontrol, frev, crev) == false)
 		BREAK_TO_DEBUGGER();
 	switch (crev) {
 	case 6:
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c
index 2979358c6a55..253bbb1eea60 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c
@@ -51,6 +51,7 @@ bool dal_bios_parser_init_cmd_tbl_helper(
 		return true;
 
 	case DCE_VERSION_11_2:
+	case DCE_VERSION_11_22:
 		*h = dal_cmd_tbl_helper_dce112_get_table();
 		return true;
 
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
index 9a4d30dd4969..bbbcef566c55 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
@@ -52,6 +52,7 @@ bool dal_bios_parser_init_cmd_tbl_helper2(
 		return true;
 
 	case DCE_VERSION_11_2:
+	case DCE_VERSION_11_22:
 		*h = dal_cmd_tbl_helper_dce112_get_table2();
 		return true;
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/calcs_logger.h b/drivers/gpu/drm/amd/display/dc/calcs/calcs_logger.h
new file mode 100644
index 000000000000..fc3f98fb09ea
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/calcs/calcs_logger.h
@@ -0,0 +1,579 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _CALCS_CALCS_LOGGER_H_
+#define _CALCS_CALCS_LOGGER_H_
+#define DC_LOGGER \
+	logger
+
+static void print_bw_calcs_dceip(struct dal_logger *logger, const struct bw_calcs_dceip *dceip)
+{
+
+	DC_LOG_BANDWIDTH_CALCS("#####################################################################");
+	DC_LOG_BANDWIDTH_CALCS("struct bw_calcs_dceip");
+	DC_LOG_BANDWIDTH_CALCS("#####################################################################");
+	DC_LOG_BANDWIDTH_CALCS("	[enum]   bw_calcs_version version %d", dceip->version);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] large_cursor: %d", dceip->large_cursor);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] dmif_pipe_en_fbc_chunk_tracker: %d", dceip->dmif_pipe_en_fbc_chunk_tracker);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] display_write_back_supported: %d", dceip->display_write_back_supported);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] argb_compression_support: %d", dceip->argb_compression_support);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] pre_downscaler_enabled: %d", dceip->pre_downscaler_enabled);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] underlay_downscale_prefetch_enabled: %d",
+				dceip->underlay_downscale_prefetch_enabled);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] graphics_lb_nodownscaling_multi_line_prefetching: %d",
+				dceip->graphics_lb_nodownscaling_multi_line_prefetching);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] limit_excessive_outstanding_dmif_requests: %d",
+				dceip->limit_excessive_outstanding_dmif_requests);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] cursor_max_outstanding_group_num: %d",
+				dceip->cursor_max_outstanding_group_num);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] lines_interleaved_into_lb: %d", dceip->lines_interleaved_into_lb);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] low_power_tiling_mode: %d", dceip->low_power_tiling_mode);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] chunk_width: %d", dceip->chunk_width);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] number_of_graphics_pipes: %d", dceip->number_of_graphics_pipes);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] number_of_underlay_pipes: %d", dceip->number_of_underlay_pipes);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] max_dmif_buffer_allocated: %d", dceip->max_dmif_buffer_allocated);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] graphics_dmif_size: %d", dceip->graphics_dmif_size);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] underlay_luma_dmif_size: %d", dceip->underlay_luma_dmif_size);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] underlay_chroma_dmif_size: %d", dceip->underlay_chroma_dmif_size);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] scatter_gather_lines_of_pte_prefetching_in_linear_mode: %d",
+				dceip->scatter_gather_lines_of_pte_prefetching_in_linear_mode);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] display_write_back420_luma_mcifwr_buffer_size: %d",
+				dceip->display_write_back420_luma_mcifwr_buffer_size);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] display_write_back420_chroma_mcifwr_buffer_size: %d",
+				dceip->display_write_back420_chroma_mcifwr_buffer_size);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] scatter_gather_pte_request_rows_in_tiling_mode: %d",
+				dceip->scatter_gather_pte_request_rows_in_tiling_mode);
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] underlay_vscaler_efficiency10_bit_per_component: %d",
+				bw_fixed_to_int(dceip->underlay_vscaler_efficiency10_bit_per_component));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] underlay_vscaler_efficiency12_bit_per_component: %d",
+				bw_fixed_to_int(dceip->underlay_vscaler_efficiency12_bit_per_component));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] graphics_vscaler_efficiency6_bit_per_component: %d",
+				bw_fixed_to_int(dceip->graphics_vscaler_efficiency6_bit_per_component));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] graphics_vscaler_efficiency8_bit_per_component: %d",
+				bw_fixed_to_int(dceip->graphics_vscaler_efficiency8_bit_per_component));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] graphics_vscaler_efficiency10_bit_per_component: %d",
+				bw_fixed_to_int(dceip->graphics_vscaler_efficiency10_bit_per_component));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] graphics_vscaler_efficiency12_bit_per_component: %d",
+				bw_fixed_to_int(dceip->graphics_vscaler_efficiency12_bit_per_component));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] alpha_vscaler_efficiency: %d",
+				bw_fixed_to_int(dceip->alpha_vscaler_efficiency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] lb_write_pixels_per_dispclk: %d",
+				bw_fixed_to_int(dceip->lb_write_pixels_per_dispclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] lb_size_per_component444: %d",
+				bw_fixed_to_int(dceip->lb_size_per_component444));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] stutter_and_dram_clock_state_change_gated_before_cursor: %d",
+				bw_fixed_to_int(dceip->stutter_and_dram_clock_state_change_gated_before_cursor));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] underlay420_luma_lb_size_per_component: %d",
+				bw_fixed_to_int(dceip->underlay420_luma_lb_size_per_component));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] underlay420_chroma_lb_size_per_component: %d",
+				bw_fixed_to_int(dceip->underlay420_chroma_lb_size_per_component));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] underlay422_lb_size_per_component: %d",
+				bw_fixed_to_int(dceip->underlay422_lb_size_per_component));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] cursor_chunk_width: %d", bw_fixed_to_int(dceip->cursor_chunk_width));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] cursor_dcp_buffer_lines: %d",
+				bw_fixed_to_int(dceip->cursor_dcp_buffer_lines));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] underlay_maximum_width_efficient_for_tiling: %d",
+				bw_fixed_to_int(dceip->underlay_maximum_width_efficient_for_tiling));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] underlay_maximum_height_efficient_for_tiling: %d",
+				bw_fixed_to_int(dceip->underlay_maximum_height_efficient_for_tiling));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] peak_pte_request_to_eviction_ratio_limiting_multiple_displays_or_single_rotated_display: %d",
+				bw_fixed_to_int(dceip->peak_pte_request_to_eviction_ratio_limiting_multiple_displays_or_single_rotated_display));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] peak_pte_request_to_eviction_ratio_limiting_single_display_no_rotation: %d",
+				bw_fixed_to_int(dceip->peak_pte_request_to_eviction_ratio_limiting_single_display_no_rotation));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] minimum_outstanding_pte_request_limit: %d",
+				bw_fixed_to_int(dceip->minimum_outstanding_pte_request_limit));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] maximum_total_outstanding_pte_requests_allowed_by_saw: %d",
+				bw_fixed_to_int(dceip->maximum_total_outstanding_pte_requests_allowed_by_saw));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] linear_mode_line_request_alternation_slice: %d",
+				bw_fixed_to_int(dceip->linear_mode_line_request_alternation_slice));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] request_efficiency: %d", bw_fixed_to_int(dceip->request_efficiency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dispclk_per_request: %d", bw_fixed_to_int(dceip->dispclk_per_request));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dispclk_ramping_factor: %d",
+				bw_fixed_to_int(dceip->dispclk_ramping_factor));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] display_pipe_throughput_factor: %d",
+				bw_fixed_to_int(dceip->display_pipe_throughput_factor));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mcifwr_all_surfaces_burst_time: %d",
+				bw_fixed_to_int(dceip->mcifwr_all_surfaces_burst_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dmif_request_buffer_size: %d",
+				bw_fixed_to_int(dceip->dmif_request_buffer_size));
+
+
+}
+
+static void print_bw_calcs_vbios(struct dal_logger *logger, const struct bw_calcs_vbios *vbios)
+{
+
+	DC_LOG_BANDWIDTH_CALCS("#####################################################################");
+	DC_LOG_BANDWIDTH_CALCS("struct bw_calcs_vbios vbios");
+	DC_LOG_BANDWIDTH_CALCS("#####################################################################");
+	DC_LOG_BANDWIDTH_CALCS("	[enum] bw_defines memory_type: %d", vbios->memory_type);
+	DC_LOG_BANDWIDTH_CALCS("	[enum] bw_defines memory_type: %d", vbios->memory_type);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] dram_channel_width_in_bits: %d", vbios->dram_channel_width_in_bits);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] number_of_dram_channels: %d", vbios->number_of_dram_channels);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] number_of_dram_banks: %d", vbios->number_of_dram_banks);
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] low_yclk: %d", bw_fixed_to_int(vbios->low_yclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mid_yclk: %d", bw_fixed_to_int(vbios->mid_yclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] high_yclk: %d", bw_fixed_to_int(vbios->high_yclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] low_sclk: %d", bw_fixed_to_int(vbios->low_sclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mid1_sclk: %d", bw_fixed_to_int(vbios->mid1_sclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mid2_sclk: %d", bw_fixed_to_int(vbios->mid2_sclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mid3_sclk: %d", bw_fixed_to_int(vbios->mid3_sclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mid4_sclk: %d", bw_fixed_to_int(vbios->mid4_sclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mid5_sclk: %d", bw_fixed_to_int(vbios->mid5_sclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mid6_sclk: %d", bw_fixed_to_int(vbios->mid6_sclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] high_sclk: %d", bw_fixed_to_int(vbios->high_sclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] low_voltage_max_dispclk: %d",
+				bw_fixed_to_int(vbios->low_voltage_max_dispclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mid_voltage_max_dispclk;: %d",
+				bw_fixed_to_int(vbios->mid_voltage_max_dispclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] high_voltage_max_dispclk;: %d",
+				bw_fixed_to_int(vbios->high_voltage_max_dispclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] low_voltage_max_phyclk: %d",
+				bw_fixed_to_int(vbios->low_voltage_max_phyclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mid_voltage_max_phyclk: %d",
+				bw_fixed_to_int(vbios->mid_voltage_max_phyclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] high_voltage_max_phyclk: %d",
+				bw_fixed_to_int(vbios->high_voltage_max_phyclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] data_return_bus_width: %d", bw_fixed_to_int(vbios->data_return_bus_width));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] trc: %d", bw_fixed_to_int(vbios->trc));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dmifmc_urgent_latency: %d", bw_fixed_to_int(vbios->dmifmc_urgent_latency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] stutter_self_refresh_exit_latency: %d",
+				bw_fixed_to_int(vbios->stutter_self_refresh_exit_latency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] stutter_self_refresh_entry_latency: %d",
+				bw_fixed_to_int(vbios->stutter_self_refresh_entry_latency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] nbp_state_change_latency: %d",
+				bw_fixed_to_int(vbios->nbp_state_change_latency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mcifwrmc_urgent_latency: %d",
+				bw_fixed_to_int(vbios->mcifwrmc_urgent_latency));
+	DC_LOG_BANDWIDTH_CALCS("	[bool] scatter_gather_enable: %d", vbios->scatter_gather_enable);
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] down_spread_percentage: %d",
+				bw_fixed_to_int(vbios->down_spread_percentage));
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] cursor_width: %d", vbios->cursor_width);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] average_compression_rate: %d", vbios->average_compression_rate);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] number_of_request_slots_gmc_reserves_for_dmif_per_channel: %d",
+				vbios->number_of_request_slots_gmc_reserves_for_dmif_per_channel);
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] blackout_duration: %d", bw_fixed_to_int(vbios->blackout_duration));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] maximum_blackout_recovery_time: %d",
+				bw_fixed_to_int(vbios->maximum_blackout_recovery_time));
+
+
+}
+
+static void print_bw_calcs_data(struct dal_logger *logger, struct bw_calcs_data *data)
+{
+
+	int i, j, k;
+
+	DC_LOG_BANDWIDTH_CALCS("#####################################################################");
+	DC_LOG_BANDWIDTH_CALCS("struct bw_calcs_data data");
+	DC_LOG_BANDWIDTH_CALCS("#####################################################################");
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] number_of_displays: %d", data->number_of_displays);
+	DC_LOG_BANDWIDTH_CALCS("	[enum] bw_defines underlay_surface_type: %d", data->underlay_surface_type);
+	DC_LOG_BANDWIDTH_CALCS("	[enum] bw_defines panning_and_bezel_adjustment: %d",
+				data->panning_and_bezel_adjustment);
+	DC_LOG_BANDWIDTH_CALCS("	[enum] bw_defines graphics_tiling_mode: %d", data->graphics_tiling_mode);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] graphics_lb_bpc: %d", data->graphics_lb_bpc);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] underlay_lb_bpc: %d", data->underlay_lb_bpc);
+	DC_LOG_BANDWIDTH_CALCS("	[enum] bw_defines underlay_tiling_mode: %d", data->underlay_tiling_mode);
+	DC_LOG_BANDWIDTH_CALCS("	[enum] bw_defines d0_underlay_mode: %d", data->d0_underlay_mode);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] d1_display_write_back_dwb_enable: %d", data->d1_display_write_back_dwb_enable);
+	DC_LOG_BANDWIDTH_CALCS("	[enum] bw_defines d1_underlay_mode: %d", data->d1_underlay_mode);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] cpup_state_change_enable: %d", data->cpup_state_change_enable);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] cpuc_state_change_enable: %d", data->cpuc_state_change_enable);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] nbp_state_change_enable: %d", data->nbp_state_change_enable);
+	DC_LOG_BANDWIDTH_CALCS("	[bool] stutter_mode_enable: %d", data->stutter_mode_enable);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] y_clk_level: %d", data->y_clk_level);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] sclk_level: %d", data->sclk_level);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] number_of_underlay_surfaces: %d", data->number_of_underlay_surfaces);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] number_of_dram_wrchannels: %d", data->number_of_dram_wrchannels);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] chunk_request_delay: %d", data->chunk_request_delay);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] number_of_dram_channels: %d", data->number_of_dram_channels);
+	DC_LOG_BANDWIDTH_CALCS("	[enum] bw_defines underlay_micro_tile_mode: %d", data->underlay_micro_tile_mode);
+	DC_LOG_BANDWIDTH_CALCS("	[enum] bw_defines graphics_micro_tile_mode: %d", data->graphics_micro_tile_mode);
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] max_phyclk: %d", bw_fixed_to_int(data->max_phyclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dram_efficiency: %d", bw_fixed_to_int(data->dram_efficiency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] src_width_after_surface_type: %d",
+				bw_fixed_to_int(data->src_width_after_surface_type));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] src_height_after_surface_type: %d",
+				bw_fixed_to_int(data->src_height_after_surface_type));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] hsr_after_surface_type: %d",
+				bw_fixed_to_int(data->hsr_after_surface_type));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] vsr_after_surface_type: %d", bw_fixed_to_int(data->vsr_after_surface_type));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] src_width_after_rotation: %d",
+				bw_fixed_to_int(data->src_width_after_rotation));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] src_height_after_rotation: %d",
+				bw_fixed_to_int(data->src_height_after_rotation));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] hsr_after_rotation: %d", bw_fixed_to_int(data->hsr_after_rotation));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] vsr_after_rotation: %d", bw_fixed_to_int(data->vsr_after_rotation));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] source_height_pixels: %d", bw_fixed_to_int(data->source_height_pixels));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] hsr_after_stereo: %d", bw_fixed_to_int(data->hsr_after_stereo));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] vsr_after_stereo: %d", bw_fixed_to_int(data->vsr_after_stereo));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] source_width_in_lb: %d", bw_fixed_to_int(data->source_width_in_lb));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] lb_line_pitch: %d", bw_fixed_to_int(data->lb_line_pitch));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] underlay_maximum_source_efficient_for_tiling: %d",
+				bw_fixed_to_int(data->underlay_maximum_source_efficient_for_tiling));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] num_lines_at_frame_start: %d",
+				bw_fixed_to_int(data->num_lines_at_frame_start));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] min_dmif_size_in_time: %d", bw_fixed_to_int(data->min_dmif_size_in_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] min_mcifwr_size_in_time: %d",
+				bw_fixed_to_int(data->min_mcifwr_size_in_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_requests_for_dmif_size: %d",
+				bw_fixed_to_int(data->total_requests_for_dmif_size));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] peak_pte_request_to_eviction_ratio_limiting: %d",
+				bw_fixed_to_int(data->peak_pte_request_to_eviction_ratio_limiting));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] useful_pte_per_pte_request: %d",
+				bw_fixed_to_int(data->useful_pte_per_pte_request));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] scatter_gather_pte_request_rows: %d",
+				bw_fixed_to_int(data->scatter_gather_pte_request_rows));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] scatter_gather_row_height: %d",
+				bw_fixed_to_int(data->scatter_gather_row_height));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] scatter_gather_pte_requests_in_vblank: %d",
+				bw_fixed_to_int(data->scatter_gather_pte_requests_in_vblank));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] inefficient_linear_pitch_in_bytes: %d",
+				bw_fixed_to_int(data->inefficient_linear_pitch_in_bytes));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] cursor_total_data: %d", bw_fixed_to_int(data->cursor_total_data));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] cursor_total_request_groups: %d",
+				bw_fixed_to_int(data->cursor_total_request_groups));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] scatter_gather_total_pte_requests: %d",
+				bw_fixed_to_int(data->scatter_gather_total_pte_requests));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] scatter_gather_total_pte_request_groups: %d",
+				bw_fixed_to_int(data->scatter_gather_total_pte_request_groups));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] tile_width_in_pixels: %d", bw_fixed_to_int(data->tile_width_in_pixels));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dmif_total_number_of_data_request_page_close_open: %d",
+				bw_fixed_to_int(data->dmif_total_number_of_data_request_page_close_open));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mcifwr_total_number_of_data_request_page_close_open: %d",
+				bw_fixed_to_int(data->mcifwr_total_number_of_data_request_page_close_open));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] bytes_per_page_close_open: %d",
+				bw_fixed_to_int(data->bytes_per_page_close_open));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mcifwr_total_page_close_open_time: %d",
+				bw_fixed_to_int(data->mcifwr_total_page_close_open_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_requests_for_adjusted_dmif_size: %d",
+				bw_fixed_to_int(data->total_requests_for_adjusted_dmif_size));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_dmifmc_urgent_trips: %d",
+				bw_fixed_to_int(data->total_dmifmc_urgent_trips));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_dmifmc_urgent_latency: %d",
+				bw_fixed_to_int(data->total_dmifmc_urgent_latency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_display_reads_required_data: %d",
+				bw_fixed_to_int(data->total_display_reads_required_data));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_display_reads_required_dram_access_data: %d",
+				bw_fixed_to_int(data->total_display_reads_required_dram_access_data));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_display_writes_required_data: %d",
+				bw_fixed_to_int(data->total_display_writes_required_data));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_display_writes_required_dram_access_data: %d",
+				bw_fixed_to_int(data->total_display_writes_required_dram_access_data));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] display_reads_required_data: %d",
+				bw_fixed_to_int(data->display_reads_required_data));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] display_reads_required_dram_access_data: %d",
+				bw_fixed_to_int(data->display_reads_required_dram_access_data));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dmif_total_page_close_open_time: %d",
+				bw_fixed_to_int(data->dmif_total_page_close_open_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] min_cursor_memory_interface_buffer_size_in_time: %d",
+				bw_fixed_to_int(data->min_cursor_memory_interface_buffer_size_in_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] min_read_buffer_size_in_time: %d",
+				bw_fixed_to_int(data->min_read_buffer_size_in_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] display_reads_time_for_data_transfer: %d",
+				bw_fixed_to_int(data->display_reads_time_for_data_transfer));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] display_writes_time_for_data_transfer: %d",
+				bw_fixed_to_int(data->display_writes_time_for_data_transfer));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dmif_required_dram_bandwidth: %d",
+				bw_fixed_to_int(data->dmif_required_dram_bandwidth));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mcifwr_required_dram_bandwidth: %d",
+				bw_fixed_to_int(data->mcifwr_required_dram_bandwidth));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] required_dmifmc_urgent_latency_for_page_close_open: %d",
+				bw_fixed_to_int(data->required_dmifmc_urgent_latency_for_page_close_open));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] required_mcifmcwr_urgent_latency: %d",
+				bw_fixed_to_int(data->required_mcifmcwr_urgent_latency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] required_dram_bandwidth_gbyte_per_second: %d",
+				bw_fixed_to_int(data->required_dram_bandwidth_gbyte_per_second));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dram_bandwidth: %d", bw_fixed_to_int(data->dram_bandwidth));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dmif_required_sclk: %d", bw_fixed_to_int(data->dmif_required_sclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mcifwr_required_sclk: %d", bw_fixed_to_int(data->mcifwr_required_sclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] required_sclk: %d", bw_fixed_to_int(data->required_sclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] downspread_factor: %d", bw_fixed_to_int(data->downspread_factor));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] v_scaler_efficiency: %d", bw_fixed_to_int(data->v_scaler_efficiency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] scaler_limits_factor: %d", bw_fixed_to_int(data->scaler_limits_factor));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] display_pipe_pixel_throughput: %d",
+				bw_fixed_to_int(data->display_pipe_pixel_throughput));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_dispclk_required_with_ramping: %d",
+				bw_fixed_to_int(data->total_dispclk_required_with_ramping));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_dispclk_required_without_ramping: %d",
+				bw_fixed_to_int(data->total_dispclk_required_without_ramping));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_read_request_bandwidth: %d",
+				bw_fixed_to_int(data->total_read_request_bandwidth));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_write_request_bandwidth: %d",
+				bw_fixed_to_int(data->total_write_request_bandwidth));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dispclk_required_for_total_read_request_bandwidth: %d",
+				bw_fixed_to_int(data->dispclk_required_for_total_read_request_bandwidth));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_dispclk_required_with_ramping_with_request_bandwidth: %d",
+				bw_fixed_to_int(data->total_dispclk_required_with_ramping_with_request_bandwidth));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_dispclk_required_without_ramping_with_request_bandwidth: %d",
+				bw_fixed_to_int(data->total_dispclk_required_without_ramping_with_request_bandwidth));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dispclk: %d", bw_fixed_to_int(data->dispclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] blackout_recovery_time: %d", bw_fixed_to_int(data->blackout_recovery_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] min_pixels_per_data_fifo_entry: %d",
+				bw_fixed_to_int(data->min_pixels_per_data_fifo_entry));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] sclk_deep_sleep: %d", bw_fixed_to_int(data->sclk_deep_sleep));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] chunk_request_time: %d", bw_fixed_to_int(data->chunk_request_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] cursor_request_time: %d", bw_fixed_to_int(data->cursor_request_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] line_source_pixels_transfer_time: %d",
+				bw_fixed_to_int(data->line_source_pixels_transfer_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dmifdram_access_efficiency: %d",
+				bw_fixed_to_int(data->dmifdram_access_efficiency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mcifwrdram_access_efficiency: %d",
+				bw_fixed_to_int(data->mcifwrdram_access_efficiency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_average_bandwidth_no_compression: %d",
+				bw_fixed_to_int(data->total_average_bandwidth_no_compression));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_average_bandwidth: %d",
+				bw_fixed_to_int(data->total_average_bandwidth));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] total_stutter_cycle_duration: %d",
+				bw_fixed_to_int(data->total_stutter_cycle_duration));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] stutter_burst_time: %d", bw_fixed_to_int(data->stutter_burst_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] time_in_self_refresh: %d", bw_fixed_to_int(data->time_in_self_refresh));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] stutter_efficiency: %d", bw_fixed_to_int(data->stutter_efficiency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] worst_number_of_trips_to_memory: %d",
+				bw_fixed_to_int(data->worst_number_of_trips_to_memory));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] immediate_flip_time: %d", bw_fixed_to_int(data->immediate_flip_time));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] latency_for_non_dmif_clients: %d",
+				bw_fixed_to_int(data->latency_for_non_dmif_clients));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] latency_for_non_mcifwr_clients: %d",
+				bw_fixed_to_int(data->latency_for_non_mcifwr_clients));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dmifmc_urgent_latency_supported_in_high_sclk_and_yclk: %d",
+				bw_fixed_to_int(data->dmifmc_urgent_latency_supported_in_high_sclk_and_yclk));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] nbp_state_dram_speed_change_margin: %d",
+				bw_fixed_to_int(data->nbp_state_dram_speed_change_margin));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] display_reads_time_for_data_transfer_and_urgent_latency: %d",
+				bw_fixed_to_int(data->display_reads_time_for_data_transfer_and_urgent_latency));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dram_speed_change_margin: %d",
+				bw_fixed_to_int(data->dram_speed_change_margin));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] min_vblank_dram_speed_change_margin: %d",
+				bw_fixed_to_int(data->min_vblank_dram_speed_change_margin));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] min_stutter_refresh_duration: %d",
+				bw_fixed_to_int(data->min_stutter_refresh_duration));
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] total_stutter_dmif_buffer_size: %d", data->total_stutter_dmif_buffer_size);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] total_bytes_requested: %d", data->total_bytes_requested);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] min_stutter_dmif_buffer_size: %d", data->min_stutter_dmif_buffer_size);
+	DC_LOG_BANDWIDTH_CALCS("	[uint32_t] num_stutter_bursts: %d", data->num_stutter_bursts);
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] v_blank_nbp_state_dram_speed_change_latency_supported: %d",
+				bw_fixed_to_int(data->v_blank_nbp_state_dram_speed_change_latency_supported));
+	DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] nbp_state_dram_speed_change_latency_supported: %d",
+				bw_fixed_to_int(data->nbp_state_dram_speed_change_latency_supported));
+
+	for (i = 0; i < maximum_number_of_surfaces; i++) {
+		DC_LOG_BANDWIDTH_CALCS("	[bool] fbc_en[%d]:%d\n", i, data->fbc_en[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[bool] lpt_en[%d]:%d", i, data->lpt_en[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[bool] displays_match_flag[%d]:%d", i, data->displays_match_flag[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[bool] use_alpha[%d]:%d", i, data->use_alpha[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[bool] orthogonal_rotation[%d]:%d", i, data->orthogonal_rotation[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[bool] enable[%d]:%d", i, data->enable[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[bool] access_one_channel_only[%d]:%d", i, data->access_one_channel_only[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[bool] scatter_gather_enable_for_pipe[%d]:%d",
+					i, data->scatter_gather_enable_for_pipe[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[bool] interlace_mode[%d]:%d",
+					i, data->interlace_mode[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[bool] display_pstate_change_enable[%d]:%d",
+					i, data->display_pstate_change_enable[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[bool] line_buffer_prefetch[%d]:%d", i, data->line_buffer_prefetch[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[uint32_t] bytes_per_pixel[%d]:%d", i, data->bytes_per_pixel[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[uint32_t] max_chunks_non_fbc_mode[%d]:%d",
+					i, data->max_chunks_non_fbc_mode[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[uint32_t] lb_bpc[%d]:%d", i, data->lb_bpc[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[uint32_t] output_bpphdmi[%d]:%d", i, data->output_bpphdmi[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[uint32_t] output_bppdp4_lane_hbr[%d]:%d", i, data->output_bppdp4_lane_hbr[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[uint32_t] output_bppdp4_lane_hbr2[%d]:%d",
+					i, data->output_bppdp4_lane_hbr2[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[uint32_t] output_bppdp4_lane_hbr3[%d]:%d",
+					i, data->output_bppdp4_lane_hbr3[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[enum] bw_defines stereo_mode[%d]:%d", i, data->stereo_mode[i]);
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dmif_buffer_transfer_time[%d]:%d",
+					i, bw_fixed_to_int(data->dmif_buffer_transfer_time[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] displays_with_same_mode[%d]:%d",
+					i, bw_fixed_to_int(data->displays_with_same_mode[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] stutter_dmif_buffer_size[%d]:%d",
+					i, bw_fixed_to_int(data->stutter_dmif_buffer_size[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] stutter_refresh_duration[%d]:%d",
+					i, bw_fixed_to_int(data->stutter_refresh_duration[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] stutter_exit_watermark[%d]:%d",
+					i, bw_fixed_to_int(data->stutter_exit_watermark[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] stutter_entry_watermark[%d]:%d",
+					i, bw_fixed_to_int(data->stutter_entry_watermark[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] h_total[%d]:%d", i, bw_fixed_to_int(data->h_total[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] v_total[%d]:%d", i, bw_fixed_to_int(data->v_total[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] pixel_rate[%d]:%d", i, bw_fixed_to_int(data->pixel_rate[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] src_width[%d]:%d", i, bw_fixed_to_int(data->src_width[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] pitch_in_pixels[%d]:%d",
+					i, bw_fixed_to_int(data->pitch_in_pixels[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] pitch_in_pixels_after_surface_type[%d]:%d",
+					i, bw_fixed_to_int(data->pitch_in_pixels_after_surface_type[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] src_height[%d]:%d", i, bw_fixed_to_int(data->src_height[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] scale_ratio[%d]:%d", i, bw_fixed_to_int(data->scale_ratio[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] h_taps[%d]:%d", i, bw_fixed_to_int(data->h_taps[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] v_taps[%d]:%d", i, bw_fixed_to_int(data->v_taps[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] h_scale_ratio[%d]:%d", i, bw_fixed_to_int(data->h_scale_ratio[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] v_scale_ratio[%d]:%d", i, bw_fixed_to_int(data->v_scale_ratio[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] rotation_angle[%d]:%d",
+					i, bw_fixed_to_int(data->rotation_angle[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] compression_rate[%d]:%d",
+					i, bw_fixed_to_int(data->compression_rate[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] hsr[%d]:%d", i, bw_fixed_to_int(data->hsr[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] vsr[%d]:%d", i, bw_fixed_to_int(data->vsr[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] source_width_rounded_up_to_chunks[%d]:%d",
+					i, bw_fixed_to_int(data->source_width_rounded_up_to_chunks[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] source_width_pixels[%d]:%d",
+					i, bw_fixed_to_int(data->source_width_pixels[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] source_height_rounded_up_to_chunks[%d]:%d",
+					i, bw_fixed_to_int(data->source_height_rounded_up_to_chunks[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] display_bandwidth[%d]:%d",
+					i, bw_fixed_to_int(data->display_bandwidth[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] request_bandwidth[%d]:%d",
+					i, bw_fixed_to_int(data->request_bandwidth[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] bytes_per_request[%d]:%d",
+					i, bw_fixed_to_int(data->bytes_per_request[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] useful_bytes_per_request[%d]:%d",
+					i, bw_fixed_to_int(data->useful_bytes_per_request[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] lines_interleaved_in_mem_access[%d]:%d",
+					i, bw_fixed_to_int(data->lines_interleaved_in_mem_access[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] latency_hiding_lines[%d]:%d",
+					i, bw_fixed_to_int(data->latency_hiding_lines[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] lb_partitions[%d]:%d",
+					i, bw_fixed_to_int(data->lb_partitions[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] lb_partitions_max[%d]:%d",
+					i, bw_fixed_to_int(data->lb_partitions_max[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dispclk_required_with_ramping[%d]:%d",
+					i, bw_fixed_to_int(data->dispclk_required_with_ramping[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dispclk_required_without_ramping[%d]:%d",
+					i, bw_fixed_to_int(data->dispclk_required_without_ramping[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] data_buffer_size[%d]:%d",
+					i, bw_fixed_to_int(data->data_buffer_size[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] outstanding_chunk_request_limit[%d]:%d",
+					i, bw_fixed_to_int(data->outstanding_chunk_request_limit[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] urgent_watermark[%d]:%d",
+					i, bw_fixed_to_int(data->urgent_watermark[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] nbp_state_change_watermark[%d]:%d",
+					i, bw_fixed_to_int(data->nbp_state_change_watermark[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] v_filter_init[%d]:%d", i, bw_fixed_to_int(data->v_filter_init[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] stutter_cycle_duration[%d]:%d",
+					i, bw_fixed_to_int(data->stutter_cycle_duration[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] average_bandwidth[%d]:%d",
+					i, bw_fixed_to_int(data->average_bandwidth[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] average_bandwidth_no_compression[%d]:%d",
+					i, bw_fixed_to_int(data->average_bandwidth_no_compression[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] scatter_gather_pte_request_limit[%d]:%d",
+					i, bw_fixed_to_int(data->scatter_gather_pte_request_limit[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] lb_size_per_component[%d]:%d",
+					i, bw_fixed_to_int(data->lb_size_per_component[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] memory_chunk_size_in_bytes[%d]:%d",
+					i, bw_fixed_to_int(data->memory_chunk_size_in_bytes[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] pipe_chunk_size_in_bytes[%d]:%d",
+					i, bw_fixed_to_int(data->pipe_chunk_size_in_bytes[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] number_of_trips_to_memory_for_getting_apte_row[%d]:%d",
+					i, bw_fixed_to_int(data->number_of_trips_to_memory_for_getting_apte_row[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] adjusted_data_buffer_size[%d]:%d",
+					i, bw_fixed_to_int(data->adjusted_data_buffer_size[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] adjusted_data_buffer_size_in_memory[%d]:%d",
+					i, bw_fixed_to_int(data->adjusted_data_buffer_size_in_memory[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] pixels_per_data_fifo_entry[%d]:%d",
+					i, bw_fixed_to_int(data->pixels_per_data_fifo_entry[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] scatter_gather_pte_requests_in_row[%d]:%d",
+					i, bw_fixed_to_int(data->scatter_gather_pte_requests_in_row[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] pte_request_per_chunk[%d]:%d",
+					i, bw_fixed_to_int(data->pte_request_per_chunk[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] scatter_gather_page_width[%d]:%d",
+					i, bw_fixed_to_int(data->scatter_gather_page_width[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] scatter_gather_page_height[%d]:%d",
+					i, bw_fixed_to_int(data->scatter_gather_page_height[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] lb_lines_in_per_line_out_in_beginning_of_frame[%d]:%d",
+					i, bw_fixed_to_int(data->lb_lines_in_per_line_out_in_beginning_of_frame[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] lb_lines_in_per_line_out_in_middle_of_frame[%d]:%d",
+					i, bw_fixed_to_int(data->lb_lines_in_per_line_out_in_middle_of_frame[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] cursor_width_pixels[%d]:%d",
+					i, bw_fixed_to_int(data->cursor_width_pixels[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] minimum_latency_hiding[%d]:%d",
+					i, bw_fixed_to_int(data->minimum_latency_hiding[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] maximum_latency_hiding[%d]:%d",
+					i, bw_fixed_to_int(data->maximum_latency_hiding[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] minimum_latency_hiding_with_cursor[%d]:%d",
+					i, bw_fixed_to_int(data->minimum_latency_hiding_with_cursor[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] maximum_latency_hiding_with_cursor[%d]:%d",
+					i, bw_fixed_to_int(data->maximum_latency_hiding_with_cursor[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] src_pixels_for_first_output_pixel[%d]:%d",
+					i, bw_fixed_to_int(data->src_pixels_for_first_output_pixel[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] src_pixels_for_last_output_pixel[%d]:%d",
+					i, bw_fixed_to_int(data->src_pixels_for_last_output_pixel[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] src_data_for_first_output_pixel[%d]:%d",
+					i, bw_fixed_to_int(data->src_data_for_first_output_pixel[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] src_data_for_last_output_pixel[%d]:%d",
+					i, bw_fixed_to_int(data->src_data_for_last_output_pixel[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] active_time[%d]:%d", i, bw_fixed_to_int(data->active_time[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] horizontal_blank_and_chunk_granularity_factor[%d]:%d",
+					i, bw_fixed_to_int(data->horizontal_blank_and_chunk_granularity_factor[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] cursor_latency_hiding[%d]:%d",
+					i, bw_fixed_to_int(data->cursor_latency_hiding[i]));
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] v_blank_dram_speed_change_margin[%d]:%d",
+					i, bw_fixed_to_int(data->v_blank_dram_speed_change_margin[i]));
+		}
+
+	for (i = 0; i < maximum_number_of_surfaces; i++) {
+		for (j = 0; j < 3; j++) {
+			for (k = 0; k < 8; k++) {
+
+				DC_LOG_BANDWIDTH_CALCS("\n	[bw_fixed] line_source_transfer_time[%d][%d][%d]:%d",
+					i, j, k, bw_fixed_to_int(data->line_source_transfer_time[i][j][k]));
+				DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dram_speed_change_line_source_transfer_time[%d][%d][%d]:%d",
+					i, j, k,
+					bw_fixed_to_int(data->dram_speed_change_line_source_transfer_time[i][j][k]));
+			}
+		}
+	}
+
+	for (i = 0; i < 3; i++) {
+		for (j = 0; j < 8; j++) {
+
+			DC_LOG_BANDWIDTH_CALCS("\n	[uint32_t] num_displays_with_margin[%d][%d]:%d",
+					i, j, data->num_displays_with_margin[i][j]);
+			DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dmif_burst_time[%d][%d]:%d",
+					i, j, bw_fixed_to_int(data->dmif_burst_time[i][j]));
+			DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] mcifwr_burst_time[%d][%d]:%d",
+					i, j, bw_fixed_to_int(data->mcifwr_burst_time[i][j]));
+			DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] min_dram_speed_change_margin[%d][%d]:%d",
+					i, j, bw_fixed_to_int(data->min_dram_speed_change_margin[i][j]));
+			DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dispclk_required_for_dram_speed_change[%d][%d]:%d",
+					i, j, bw_fixed_to_int(data->dispclk_required_for_dram_speed_change[i][j]));
+			DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] blackout_duration_margin[%d][%d]:%d",
+					i, j, bw_fixed_to_int(data->blackout_duration_margin[i][j]));
+			DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dispclk_required_for_blackout_duration[%d][%d]:%d",
+					i, j, bw_fixed_to_int(data->dispclk_required_for_blackout_duration[i][j]));
+			DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dispclk_required_for_blackout_recovery[%d][%d]:%d",
+					i, j, bw_fixed_to_int(data->dispclk_required_for_blackout_recovery[i][j]));
+		}
+	}
+
+	for (i = 0; i < 6; i++) {
+		DC_LOG_BANDWIDTH_CALCS("	[bw_fixed] dmif_required_sclk_for_urgent_latency[%d]:%d",
+					i, bw_fixed_to_int(data->dmif_required_sclk_for_urgent_latency[i]));
+	}
+}
+;
+
+#endif /* _CALCS_CALCS_LOGGER_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/custom_float.c b/drivers/gpu/drm/amd/display/dc/calcs/custom_float.c
index 7243c37f569e..31d167bc548f 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/custom_float.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/custom_float.c
@@ -36,41 +36,41 @@ static bool build_custom_float(
 	uint32_t exp_offset = (1 << (format->exponenta_bits - 1)) - 1;
 
 	const struct fixed31_32 mantissa_constant_plus_max_fraction =
-		dal_fixed31_32_from_fraction(
+		dc_fixpt_from_fraction(
 			(1LL << (format->mantissa_bits + 1)) - 1,
 			1LL << format->mantissa_bits);
 
 	struct fixed31_32 mantiss;
 
-	if (dal_fixed31_32_eq(
+	if (dc_fixpt_eq(
 		value,
-		dal_fixed31_32_zero)) {
+		dc_fixpt_zero)) {
 		*negative = false;
 		*mantissa = 0;
 		*exponenta = 0;
 		return true;
 	}
 
-	if (dal_fixed31_32_lt(
+	if (dc_fixpt_lt(
 		value,
-		dal_fixed31_32_zero)) {
+		dc_fixpt_zero)) {
 		*negative = format->sign;
-		value = dal_fixed31_32_neg(value);
+		value = dc_fixpt_neg(value);
 	} else {
 		*negative = false;
 	}
 
-	if (dal_fixed31_32_lt(
+	if (dc_fixpt_lt(
 		value,
-		dal_fixed31_32_one)) {
+		dc_fixpt_one)) {
 		uint32_t i = 1;
 
 		do {
-			value = dal_fixed31_32_shl(value, 1);
+			value = dc_fixpt_shl(value, 1);
 			++i;
-		} while (dal_fixed31_32_lt(
+		} while (dc_fixpt_lt(
 			value,
-			dal_fixed31_32_one));
+			dc_fixpt_one));
 
 		--i;
 
@@ -81,15 +81,15 @@ static bool build_custom_float(
 		}
 
 		*exponenta = exp_offset - i;
-	} else if (dal_fixed31_32_le(
+	} else if (dc_fixpt_le(
 		mantissa_constant_plus_max_fraction,
 		value)) {
 		uint32_t i = 1;
 
 		do {
-			value = dal_fixed31_32_shr(value, 1);
+			value = dc_fixpt_shr(value, 1);
 			++i;
-		} while (dal_fixed31_32_lt(
+		} while (dc_fixpt_lt(
 			mantissa_constant_plus_max_fraction,
 			value));
 
@@ -98,23 +98,23 @@ static bool build_custom_float(
 		*exponenta = exp_offset;
 	}
 
-	mantiss = dal_fixed31_32_sub(
+	mantiss = dc_fixpt_sub(
 		value,
-		dal_fixed31_32_one);
+		dc_fixpt_one);
 
-	if (dal_fixed31_32_lt(
+	if (dc_fixpt_lt(
 			mantiss,
-			dal_fixed31_32_zero) ||
-		dal_fixed31_32_lt(
-			dal_fixed31_32_one,
+			dc_fixpt_zero) ||
+		dc_fixpt_lt(
+			dc_fixpt_one,
 			mantiss))
-		mantiss = dal_fixed31_32_zero;
+		mantiss = dc_fixpt_zero;
 	else
-		mantiss = dal_fixed31_32_shl(
+		mantiss = dc_fixpt_shl(
 			mantiss,
 			format->mantissa_bits);
 
-	*mantissa = dal_fixed31_32_floor(mantiss);
+	*mantissa = dc_fixpt_floor(mantiss);
 
 	return true;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
index 0cbab81ab304..2c4e8f0cb2dc 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
@@ -28,6 +28,7 @@
 #include "dc.h"
 #include "core_types.h"
 #include "dal_asic_id.h"
+#include "calcs_logger.h"
 
 /*
  * NOTE:
@@ -52,11 +53,14 @@ static enum bw_calcs_version bw_calcs_version_from_asic_id(struct hw_asic_id asi
 		return BW_CALCS_VERSION_CARRIZO;
 
 	case FAMILY_VI:
+		if (ASIC_REV_IS_POLARIS12_V(asic_id.hw_internal_rev))
+			return BW_CALCS_VERSION_POLARIS12;
 		if (ASIC_REV_IS_POLARIS10_P(asic_id.hw_internal_rev))
 			return BW_CALCS_VERSION_POLARIS10;
-		if (ASIC_REV_IS_POLARIS11_M(asic_id.hw_internal_rev) ||
-				ASIC_REV_IS_POLARIS12_V(asic_id.hw_internal_rev))
+		if (ASIC_REV_IS_POLARIS11_M(asic_id.hw_internal_rev))
 			return BW_CALCS_VERSION_POLARIS11;
+		if (ASIC_REV_IS_VEGAM(asic_id.hw_internal_rev))
+			return BW_CALCS_VERSION_VEGAM;
 		return BW_CALCS_VERSION_INVALID;
 
 	case FAMILY_AI:
@@ -2145,6 +2149,9 @@ void bw_calcs_init(struct bw_calcs_dceip *bw_dceip,
 		dceip.mcifwr_all_surfaces_burst_time = bw_int_to_fixed(0); /* todo: this is a bug*/
 		break;
 	case BW_CALCS_VERSION_POLARIS10:
+		/* TODO: Treat VEGAM the same as P10 for now
+		 * Need to tune the para for VEGAM if needed */
+	case BW_CALCS_VERSION_VEGAM:
 		vbios.memory_type = bw_def_gddr5;
 		vbios.dram_channel_width_in_bits = 32;
 		vbios.number_of_dram_channels = asic_id.vram_width / vbios.dram_channel_width_in_bits;
@@ -2373,6 +2380,122 @@ void bw_calcs_init(struct bw_calcs_dceip *bw_dceip,
 		dceip.scatter_gather_pte_request_rows_in_tiling_mode = 2;
 		dceip.mcifwr_all_surfaces_burst_time = bw_int_to_fixed(0);
 		break;
+	case BW_CALCS_VERSION_POLARIS12:
+		vbios.memory_type = bw_def_gddr5;
+		vbios.dram_channel_width_in_bits = 32;
+		vbios.number_of_dram_channels = asic_id.vram_width / vbios.dram_channel_width_in_bits;
+		vbios.number_of_dram_banks = 8;
+		vbios.high_yclk = bw_int_to_fixed(6000);
+		vbios.mid_yclk = bw_int_to_fixed(3200);
+		vbios.low_yclk = bw_int_to_fixed(1000);
+		vbios.low_sclk = bw_int_to_fixed(678);
+		vbios.mid1_sclk = bw_int_to_fixed(864);
+		vbios.mid2_sclk = bw_int_to_fixed(900);
+		vbios.mid3_sclk = bw_int_to_fixed(920);
+		vbios.mid4_sclk = bw_int_to_fixed(940);
+		vbios.mid5_sclk = bw_int_to_fixed(960);
+		vbios.mid6_sclk = bw_int_to_fixed(980);
+		vbios.high_sclk = bw_int_to_fixed(1049);
+		vbios.low_voltage_max_dispclk = bw_int_to_fixed(459);
+		vbios.mid_voltage_max_dispclk = bw_int_to_fixed(654);
+		vbios.high_voltage_max_dispclk = bw_int_to_fixed(1108);
+		vbios.low_voltage_max_phyclk = bw_int_to_fixed(540);
+		vbios.mid_voltage_max_phyclk = bw_int_to_fixed(810);
+		vbios.high_voltage_max_phyclk = bw_int_to_fixed(810);
+		vbios.data_return_bus_width = bw_int_to_fixed(32);
+		vbios.trc = bw_int_to_fixed(48);
+		if (vbios.number_of_dram_channels == 2) // 64-bit
+			vbios.dmifmc_urgent_latency = bw_int_to_fixed(4);
+		else
+			vbios.dmifmc_urgent_latency = bw_int_to_fixed(3);
+		vbios.stutter_self_refresh_exit_latency = bw_int_to_fixed(5);
+		vbios.stutter_self_refresh_entry_latency = bw_int_to_fixed(0);
+		vbios.nbp_state_change_latency = bw_int_to_fixed(250);
+		vbios.mcifwrmc_urgent_latency = bw_int_to_fixed(10);
+		vbios.scatter_gather_enable = false;
+		vbios.down_spread_percentage = bw_frc_to_fixed(5, 10);
+		vbios.cursor_width = 32;
+		vbios.average_compression_rate = 4;
+		vbios.number_of_request_slots_gmc_reserves_for_dmif_per_channel = 256;
+		vbios.blackout_duration = bw_int_to_fixed(0); /* us */
+		vbios.maximum_blackout_recovery_time = bw_int_to_fixed(0);
+
+		dceip.max_average_percent_of_ideal_port_bw_display_can_use_in_normal_system_operation = 100;
+		dceip.max_average_percent_of_ideal_drambw_display_can_use_in_normal_system_operation = 100;
+		dceip.percent_of_ideal_port_bw_received_after_urgent_latency = 100;
+		dceip.large_cursor = false;
+		dceip.dmif_request_buffer_size = bw_int_to_fixed(768);
+		dceip.dmif_pipe_en_fbc_chunk_tracker = false;
+		dceip.cursor_max_outstanding_group_num = 1;
+		dceip.lines_interleaved_into_lb = 2;
+		dceip.chunk_width = 256;
+		dceip.number_of_graphics_pipes = 5;
+		dceip.number_of_underlay_pipes = 0;
+		dceip.low_power_tiling_mode = 0;
+		dceip.display_write_back_supported = true;
+		dceip.argb_compression_support = true;
+		dceip.underlay_vscaler_efficiency6_bit_per_component =
+			bw_frc_to_fixed(35556, 10000);
+		dceip.underlay_vscaler_efficiency8_bit_per_component =
+			bw_frc_to_fixed(34286, 10000);
+		dceip.underlay_vscaler_efficiency10_bit_per_component =
+			bw_frc_to_fixed(32, 10);
+		dceip.underlay_vscaler_efficiency12_bit_per_component =
+			bw_int_to_fixed(3);
+		dceip.graphics_vscaler_efficiency6_bit_per_component =
+			bw_frc_to_fixed(35, 10);
+		dceip.graphics_vscaler_efficiency8_bit_per_component =
+			bw_frc_to_fixed(34286, 10000);
+		dceip.graphics_vscaler_efficiency10_bit_per_component =
+			bw_frc_to_fixed(32, 10);
+		dceip.graphics_vscaler_efficiency12_bit_per_component =
+			bw_int_to_fixed(3);
+		dceip.alpha_vscaler_efficiency = bw_int_to_fixed(3);
+		dceip.max_dmif_buffer_allocated = 4;
+		dceip.graphics_dmif_size = 12288;
+		dceip.underlay_luma_dmif_size = 19456;
+		dceip.underlay_chroma_dmif_size = 23552;
+		dceip.pre_downscaler_enabled = true;
+		dceip.underlay_downscale_prefetch_enabled = true;
+		dceip.lb_write_pixels_per_dispclk = bw_int_to_fixed(1);
+		dceip.lb_size_per_component444 = bw_int_to_fixed(245952);
+		dceip.graphics_lb_nodownscaling_multi_line_prefetching = true;
+		dceip.stutter_and_dram_clock_state_change_gated_before_cursor =
+			bw_int_to_fixed(1);
+		dceip.underlay420_luma_lb_size_per_component = bw_int_to_fixed(
+			82176);
+		dceip.underlay420_chroma_lb_size_per_component =
+			bw_int_to_fixed(164352);
+		dceip.underlay422_lb_size_per_component = bw_int_to_fixed(
+			82176);
+		dceip.cursor_chunk_width = bw_int_to_fixed(64);
+		dceip.cursor_dcp_buffer_lines = bw_int_to_fixed(4);
+		dceip.underlay_maximum_width_efficient_for_tiling =
+			bw_int_to_fixed(1920);
+		dceip.underlay_maximum_height_efficient_for_tiling =
+			bw_int_to_fixed(1080);
+		dceip.peak_pte_request_to_eviction_ratio_limiting_multiple_displays_or_single_rotated_display =
+			bw_frc_to_fixed(3, 10);
+		dceip.peak_pte_request_to_eviction_ratio_limiting_single_display_no_rotation =
+			bw_int_to_fixed(25);
+		dceip.minimum_outstanding_pte_request_limit = bw_int_to_fixed(
+			2);
+		dceip.maximum_total_outstanding_pte_requests_allowed_by_saw =
+			bw_int_to_fixed(128);
+		dceip.limit_excessive_outstanding_dmif_requests = true;
+		dceip.linear_mode_line_request_alternation_slice =
+			bw_int_to_fixed(64);
+		dceip.scatter_gather_lines_of_pte_prefetching_in_linear_mode =
+			32;
+		dceip.display_write_back420_luma_mcifwr_buffer_size = 12288;
+		dceip.display_write_back420_chroma_mcifwr_buffer_size = 8192;
+		dceip.request_efficiency = bw_frc_to_fixed(8, 10);
+		dceip.dispclk_per_request = bw_int_to_fixed(2);
+		dceip.dispclk_ramping_factor = bw_frc_to_fixed(105, 100);
+		dceip.display_pipe_throughput_factor = bw_frc_to_fixed(105, 100);
+		dceip.scatter_gather_pte_request_rows_in_tiling_mode = 2;
+		dceip.mcifwr_all_surfaces_burst_time = bw_int_to_fixed(0);
+		break;
 	case BW_CALCS_VERSION_STONEY:
 		vbios.memory_type = bw_def_gddr5;
 		vbios.dram_channel_width_in_bits = 64;
@@ -2815,6 +2938,19 @@ static void populate_initial_data(
 				data->bytes_per_pixel[num_displays + 4] = 4;
 				break;
 			}
+		} else if (pipe[i].stream->dst.width != 0 &&
+					pipe[i].stream->dst.height != 0 &&
+					pipe[i].stream->src.width != 0 &&
+					pipe[i].stream->src.height != 0) {
+			data->src_width[num_displays + 4] = bw_int_to_fixed(pipe[i].stream->src.width);
+			data->pitch_in_pixels[num_displays + 4] = data->src_width[num_displays + 4];
+			data->src_height[num_displays + 4] = bw_int_to_fixed(pipe[i].stream->src.height);
+			data->h_taps[num_displays + 4] = pipe[i].stream->src.width == pipe[i].stream->dst.width ? bw_int_to_fixed(1) : bw_int_to_fixed(2);
+			data->v_taps[num_displays + 4] = pipe[i].stream->src.height == pipe[i].stream->dst.height ? bw_int_to_fixed(1) : bw_int_to_fixed(2);
+			data->h_scale_ratio[num_displays + 4] = bw_frc_to_fixed(pipe[i].stream->src.width, pipe[i].stream->dst.width);
+			data->v_scale_ratio[num_displays + 4] = bw_frc_to_fixed(pipe[i].stream->src.height, pipe[i].stream->dst.height);
+			data->rotation_angle[num_displays + 4] = bw_int_to_fixed(0);
+			data->bytes_per_pixel[num_displays + 4] = 4;
 		} else {
 			data->src_width[num_displays + 4] = bw_int_to_fixed(pipe[i].stream->timing.h_addressable);
 			data->pitch_in_pixels[num_displays + 4] = data->src_width[num_displays + 4];
@@ -2873,6 +3009,11 @@ bool bw_calcs(struct dc_context *ctx,
 		struct bw_fixed mid_yclk = vbios->mid_yclk;
 		struct bw_fixed low_yclk = vbios->low_yclk;
 
+		if (ctx->dc->debug.bandwidth_calcs_trace) {
+			print_bw_calcs_dceip(ctx->logger, dceip);
+			print_bw_calcs_vbios(ctx->logger, vbios);
+			print_bw_calcs_data(ctx->logger, data);
+		}
 		calculate_bandwidth(dceip, vbios, data);
 
 		yclk_lvl = data->y_clk_level;
@@ -2968,7 +3109,33 @@ bool bw_calcs(struct dc_context *ctx,
 			bw_fixed_to_int(bw_mul(data->
 				stutter_exit_watermark[9], bw_int_to_fixed(1000)));
 
-
+		calcs_output->stutter_entry_wm_ns[0].a_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[4], bw_int_to_fixed(1000)));
+		calcs_output->stutter_entry_wm_ns[1].a_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[5], bw_int_to_fixed(1000)));
+		calcs_output->stutter_entry_wm_ns[2].a_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[6], bw_int_to_fixed(1000)));
+		if (ctx->dc->caps.max_slave_planes) {
+			calcs_output->stutter_entry_wm_ns[3].a_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[0], bw_int_to_fixed(1000)));
+			calcs_output->stutter_entry_wm_ns[4].a_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[1], bw_int_to_fixed(1000)));
+		} else {
+			calcs_output->stutter_entry_wm_ns[3].a_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[7], bw_int_to_fixed(1000)));
+			calcs_output->stutter_entry_wm_ns[4].a_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[8], bw_int_to_fixed(1000)));
+		}
+		calcs_output->stutter_entry_wm_ns[5].a_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[9], bw_int_to_fixed(1000)));
 
 		calcs_output->urgent_wm_ns[0].a_mark =
 			bw_fixed_to_int(bw_mul(data->
@@ -3063,7 +3230,33 @@ bool bw_calcs(struct dc_context *ctx,
 				bw_fixed_to_int(bw_mul(data->
 					stutter_exit_watermark[9], bw_int_to_fixed(1000)));
 
-
+		calcs_output->stutter_entry_wm_ns[0].b_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[4], bw_int_to_fixed(1000)));
+		calcs_output->stutter_entry_wm_ns[1].b_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[5], bw_int_to_fixed(1000)));
+		calcs_output->stutter_entry_wm_ns[2].b_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[6], bw_int_to_fixed(1000)));
+		if (ctx->dc->caps.max_slave_planes) {
+			calcs_output->stutter_entry_wm_ns[3].b_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[0], bw_int_to_fixed(1000)));
+			calcs_output->stutter_entry_wm_ns[4].b_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[1], bw_int_to_fixed(1000)));
+		} else {
+			calcs_output->stutter_entry_wm_ns[3].b_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[7], bw_int_to_fixed(1000)));
+			calcs_output->stutter_entry_wm_ns[4].b_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[8], bw_int_to_fixed(1000)));
+		}
+		calcs_output->stutter_entry_wm_ns[5].b_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[9], bw_int_to_fixed(1000)));
 
 			calcs_output->urgent_wm_ns[0].b_mark =
 				bw_fixed_to_int(bw_mul(data->
@@ -3156,6 +3349,34 @@ bool bw_calcs(struct dc_context *ctx,
 				bw_fixed_to_int(bw_mul(data->
 					stutter_exit_watermark[9], bw_int_to_fixed(1000)));
 
+		calcs_output->stutter_entry_wm_ns[0].c_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[4], bw_int_to_fixed(1000)));
+		calcs_output->stutter_entry_wm_ns[1].c_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[5], bw_int_to_fixed(1000)));
+		calcs_output->stutter_entry_wm_ns[2].c_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[6], bw_int_to_fixed(1000)));
+		if (ctx->dc->caps.max_slave_planes) {
+			calcs_output->stutter_entry_wm_ns[3].c_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[0], bw_int_to_fixed(1000)));
+			calcs_output->stutter_entry_wm_ns[4].c_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[1], bw_int_to_fixed(1000)));
+		} else {
+			calcs_output->stutter_entry_wm_ns[3].c_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[7], bw_int_to_fixed(1000)));
+			calcs_output->stutter_entry_wm_ns[4].c_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[8], bw_int_to_fixed(1000)));
+		}
+		calcs_output->stutter_entry_wm_ns[5].c_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[9], bw_int_to_fixed(1000)));
+
 			calcs_output->urgent_wm_ns[0].c_mark =
 				bw_fixed_to_int(bw_mul(data->
 					urgent_watermark[4], bw_int_to_fixed(1000)));
@@ -3260,6 +3481,33 @@ bool bw_calcs(struct dc_context *ctx,
 			bw_fixed_to_int(bw_mul(data->
 				stutter_exit_watermark[9], bw_int_to_fixed(1000)));
 
+		calcs_output->stutter_entry_wm_ns[0].d_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[4], bw_int_to_fixed(1000)));
+		calcs_output->stutter_entry_wm_ns[1].d_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[5], bw_int_to_fixed(1000)));
+		calcs_output->stutter_entry_wm_ns[2].d_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[6], bw_int_to_fixed(1000)));
+		if (ctx->dc->caps.max_slave_planes) {
+			calcs_output->stutter_entry_wm_ns[3].d_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[0], bw_int_to_fixed(1000)));
+			calcs_output->stutter_entry_wm_ns[4].d_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[1], bw_int_to_fixed(1000)));
+		} else {
+			calcs_output->stutter_entry_wm_ns[3].d_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[7], bw_int_to_fixed(1000)));
+			calcs_output->stutter_entry_wm_ns[4].d_mark =
+				bw_fixed_to_int(bw_mul(data->
+					stutter_entry_watermark[8], bw_int_to_fixed(1000)));
+		}
+		calcs_output->stutter_entry_wm_ns[5].d_mark =
+			bw_fixed_to_int(bw_mul(data->
+				stutter_entry_watermark[9], bw_int_to_fixed(1000)));
 
 		calcs_output->urgent_wm_ns[0].d_mark =
 			bw_fixed_to_int(bw_mul(data->
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
index 4bb43a371292..49a4ea45466d 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
@@ -873,14 +873,14 @@ bool dcn_validate_bandwidth(
 			}
 
 			if (pipe->plane_state->rotation % 2 == 0) {
-				ASSERT(pipe->plane_res.scl_data.ratios.horz.value != dal_fixed31_32_one.value
+				ASSERT(pipe->plane_res.scl_data.ratios.horz.value != dc_fixpt_one.value
 					|| v->scaler_rec_out_width[input_idx] == v->viewport_width[input_idx]);
-				ASSERT(pipe->plane_res.scl_data.ratios.vert.value != dal_fixed31_32_one.value
+				ASSERT(pipe->plane_res.scl_data.ratios.vert.value != dc_fixpt_one.value
 					|| v->scaler_recout_height[input_idx] == v->viewport_height[input_idx]);
 			} else {
-				ASSERT(pipe->plane_res.scl_data.ratios.horz.value != dal_fixed31_32_one.value
+				ASSERT(pipe->plane_res.scl_data.ratios.horz.value != dc_fixpt_one.value
 					|| v->scaler_recout_height[input_idx] == v->viewport_width[input_idx]);
-				ASSERT(pipe->plane_res.scl_data.ratios.vert.value != dal_fixed31_32_one.value
+				ASSERT(pipe->plane_res.scl_data.ratios.vert.value != dc_fixpt_one.value
 					|| v->scaler_rec_out_width[input_idx] == v->viewport_height[input_idx]);
 			}
 			v->dcc_enable[input_idx] = pipe->plane_state->dcc.enable ? dcn_bw_yes : dcn_bw_no;
@@ -1459,39 +1459,39 @@ void dcn_bw_notify_pplib_of_wm_ranges(struct dc *dc)
 void dcn_bw_sync_calcs_and_dml(struct dc *dc)
 {
 	kernel_fpu_begin();
-	DC_LOG_BANDWIDTH_CALCS("sr_exit_time: %d ns\n"
-			"sr_enter_plus_exit_time: %d ns\n"
-			"urgent_latency: %d ns\n"
-			"write_back_latency: %d ns\n"
-			"percent_of_ideal_drambw_received_after_urg_latency: %d %\n"
+	DC_LOG_BANDWIDTH_CALCS("sr_exit_time: %f ns\n"
+			"sr_enter_plus_exit_time: %f ns\n"
+			"urgent_latency: %f ns\n"
+			"write_back_latency: %f ns\n"
+			"percent_of_ideal_drambw_received_after_urg_latency: %f %%\n"
 			"max_request_size: %d bytes\n"
-			"dcfclkv_max0p9: %d kHz\n"
-			"dcfclkv_nom0p8: %d kHz\n"
-			"dcfclkv_mid0p72: %d kHz\n"
-			"dcfclkv_min0p65: %d kHz\n"
-			"max_dispclk_vmax0p9: %d kHz\n"
-			"max_dispclk_vnom0p8: %d kHz\n"
-			"max_dispclk_vmid0p72: %d kHz\n"
-			"max_dispclk_vmin0p65: %d kHz\n"
-			"max_dppclk_vmax0p9: %d kHz\n"
-			"max_dppclk_vnom0p8: %d kHz\n"
-			"max_dppclk_vmid0p72: %d kHz\n"
-			"max_dppclk_vmin0p65: %d kHz\n"
-			"socclk: %d kHz\n"
-			"fabric_and_dram_bandwidth_vmax0p9: %d MB/s\n"
-			"fabric_and_dram_bandwidth_vnom0p8: %d MB/s\n"
-			"fabric_and_dram_bandwidth_vmid0p72: %d MB/s\n"
-			"fabric_and_dram_bandwidth_vmin0p65: %d MB/s\n"
-			"phyclkv_max0p9: %d kHz\n"
-			"phyclkv_nom0p8: %d kHz\n"
-			"phyclkv_mid0p72: %d kHz\n"
-			"phyclkv_min0p65: %d kHz\n"
-			"downspreading: %d %\n"
+			"dcfclkv_max0p9: %f kHz\n"
+			"dcfclkv_nom0p8: %f kHz\n"
+			"dcfclkv_mid0p72: %f kHz\n"
+			"dcfclkv_min0p65: %f kHz\n"
+			"max_dispclk_vmax0p9: %f kHz\n"
+			"max_dispclk_vnom0p8: %f kHz\n"
+			"max_dispclk_vmid0p72: %f kHz\n"
+			"max_dispclk_vmin0p65: %f kHz\n"
+			"max_dppclk_vmax0p9: %f kHz\n"
+			"max_dppclk_vnom0p8: %f kHz\n"
+			"max_dppclk_vmid0p72: %f kHz\n"
+			"max_dppclk_vmin0p65: %f kHz\n"
+			"socclk: %f kHz\n"
+			"fabric_and_dram_bandwidth_vmax0p9: %f MB/s\n"
+			"fabric_and_dram_bandwidth_vnom0p8: %f MB/s\n"
+			"fabric_and_dram_bandwidth_vmid0p72: %f MB/s\n"
+			"fabric_and_dram_bandwidth_vmin0p65: %f MB/s\n"
+			"phyclkv_max0p9: %f kHz\n"
+			"phyclkv_nom0p8: %f kHz\n"
+			"phyclkv_mid0p72: %f kHz\n"
+			"phyclkv_min0p65: %f kHz\n"
+			"downspreading: %f %%\n"
 			"round_trip_ping_latency_cycles: %d DCFCLK Cycles\n"
 			"urgent_out_of_order_return_per_channel: %d Bytes\n"
 			"number_of_channels: %d\n"
 			"vmm_page_size: %d Bytes\n"
-			"dram_clock_change_latency: %d ns\n"
+			"dram_clock_change_latency: %f ns\n"
 			"return_bus_width: %d Bytes\n",
 			dc->dcn_soc->sr_exit_time * 1000,
 			dc->dcn_soc->sr_enter_plus_exit_time * 1000,
@@ -1527,11 +1527,11 @@ void dcn_bw_sync_calcs_and_dml(struct dc *dc)
 			dc->dcn_soc->vmm_page_size,
 			dc->dcn_soc->dram_clock_change_latency * 1000,
 			dc->dcn_soc->return_bus_width);
-	DC_LOG_BANDWIDTH_CALCS("rob_buffer_size_in_kbyte: %d\n"
-			"det_buffer_size_in_kbyte: %d\n"
-			"dpp_output_buffer_pixels: %d\n"
-			"opp_output_buffer_lines: %d\n"
-			"pixel_chunk_size_in_kbyte: %d\n"
+	DC_LOG_BANDWIDTH_CALCS("rob_buffer_size_in_kbyte: %f\n"
+			"det_buffer_size_in_kbyte: %f\n"
+			"dpp_output_buffer_pixels: %f\n"
+			"opp_output_buffer_lines: %f\n"
+			"pixel_chunk_size_in_kbyte: %f\n"
 			"pte_enable: %d\n"
 			"pte_chunk_size: %d kbytes\n"
 			"meta_chunk_size: %d kbytes\n"
@@ -1550,13 +1550,13 @@ void dcn_bw_sync_calcs_and_dml(struct dc *dc)
 			"max_pscl_tolb_throughput: %d pixels/dppclk\n"
 			"max_lb_tovscl_throughput: %d pixels/dppclk\n"
 			"max_vscl_tohscl_throughput: %d pixels/dppclk\n"
-			"max_hscl_ratio: %d\n"
-			"max_vscl_ratio: %d\n"
+			"max_hscl_ratio: %f\n"
+			"max_vscl_ratio: %f\n"
 			"max_hscl_taps: %d\n"
 			"max_vscl_taps: %d\n"
 			"pte_buffer_size_in_requests: %d\n"
-			"dispclk_ramping_margin: %d %\n"
-			"under_scan_factor: %d %\n"
+			"dispclk_ramping_margin: %f %%\n"
+			"under_scan_factor: %f %%\n"
 			"max_inter_dcn_tile_repeaters: %d\n"
 			"can_vstartup_lines_exceed_vsync_plus_back_porch_lines_minus_one: %d\n"
 			"bug_forcing_luma_and_chroma_request_to_same_size_fixed: %d\n"
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 9cd3566def8d..644b2187507b 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -936,95 +936,6 @@ bool dc_post_update_surfaces_to_stream(struct dc *dc)
 	return true;
 }
 
-/*
- * TODO this whole function needs to go
- *
- * dc_surface_update is needlessly complex. See if we can just replace this
- * with a dc_plane_state and follow the atomic model a bit more closely here.
- */
-bool dc_commit_planes_to_stream(
-		struct dc *dc,
-		struct dc_plane_state **plane_states,
-		uint8_t new_plane_count,
-		struct dc_stream_state *dc_stream,
-		struct dc_state *state)
-{
-	/* no need to dynamically allocate this. it's pretty small */
-	struct dc_surface_update updates[MAX_SURFACES];
-	struct dc_flip_addrs *flip_addr;
-	struct dc_plane_info *plane_info;
-	struct dc_scaling_info *scaling_info;
-	int i;
-	struct dc_stream_update *stream_update =
-			kzalloc(sizeof(struct dc_stream_update), GFP_KERNEL);
-
-	if (!stream_update) {
-		BREAK_TO_DEBUGGER();
-		return false;
-	}
-
-	flip_addr = kcalloc(MAX_SURFACES, sizeof(struct dc_flip_addrs),
-			    GFP_KERNEL);
-	plane_info = kcalloc(MAX_SURFACES, sizeof(struct dc_plane_info),
-			     GFP_KERNEL);
-	scaling_info = kcalloc(MAX_SURFACES, sizeof(struct dc_scaling_info),
-			       GFP_KERNEL);
-
-	if (!flip_addr || !plane_info || !scaling_info) {
-		kfree(flip_addr);
-		kfree(plane_info);
-		kfree(scaling_info);
-		kfree(stream_update);
-		return false;
-	}
-
-	memset(updates, 0, sizeof(updates));
-
-	stream_update->src = dc_stream->src;
-	stream_update->dst = dc_stream->dst;
-	stream_update->out_transfer_func = dc_stream->out_transfer_func;
-
-	for (i = 0; i < new_plane_count; i++) {
-		updates[i].surface = plane_states[i];
-		updates[i].gamma =
-			(struct dc_gamma *)plane_states[i]->gamma_correction;
-		updates[i].in_transfer_func = plane_states[i]->in_transfer_func;
-		flip_addr[i].address = plane_states[i]->address;
-		flip_addr[i].flip_immediate = plane_states[i]->flip_immediate;
-		plane_info[i].color_space = plane_states[i]->color_space;
-		plane_info[i].input_tf = plane_states[i]->input_tf;
-		plane_info[i].format = plane_states[i]->format;
-		plane_info[i].plane_size = plane_states[i]->plane_size;
-		plane_info[i].rotation = plane_states[i]->rotation;
-		plane_info[i].horizontal_mirror = plane_states[i]->horizontal_mirror;
-		plane_info[i].stereo_format = plane_states[i]->stereo_format;
-		plane_info[i].tiling_info = plane_states[i]->tiling_info;
-		plane_info[i].visible = plane_states[i]->visible;
-		plane_info[i].per_pixel_alpha = plane_states[i]->per_pixel_alpha;
-		plane_info[i].dcc = plane_states[i]->dcc;
-		scaling_info[i].scaling_quality = plane_states[i]->scaling_quality;
-		scaling_info[i].src_rect = plane_states[i]->src_rect;
-		scaling_info[i].dst_rect = plane_states[i]->dst_rect;
-		scaling_info[i].clip_rect = plane_states[i]->clip_rect;
-
-		updates[i].flip_addr = &flip_addr[i];
-		updates[i].plane_info = &plane_info[i];
-		updates[i].scaling_info = &scaling_info[i];
-	}
-
-	dc_commit_updates_for_stream(
-			dc,
-			updates,
-			new_plane_count,
-			dc_stream, stream_update, plane_states, state);
-
-	kfree(flip_addr);
-	kfree(plane_info);
-	kfree(scaling_info);
-	kfree(stream_update);
-	return true;
-}
-
 struct dc_state *dc_create_state(void)
 {
 	struct dc_state *context = kzalloc(sizeof(struct dc_state),
@@ -1107,9 +1018,6 @@ static enum surface_update_type get_plane_info_update_type(const struct dc_surfa
 	if (u->plane_info->color_space != u->surface->color_space)
 		update_flags->bits.color_space_change = 1;
 
-	if (u->plane_info->input_tf != u->surface->input_tf)
-		update_flags->bits.input_tf_change = 1;
-
 	if (u->plane_info->horizontal_mirror != u->surface->horizontal_mirror)
 		update_flags->bits.horizontal_mirror_change = 1;
 
@@ -1243,12 +1151,20 @@ static enum surface_update_type det_surface_update(const struct dc *dc,
 	if (u->input_csc_color_matrix)
 		update_flags->bits.input_csc_change = 1;
 
-	if (update_flags->bits.in_transfer_func_change
-			|| update_flags->bits.input_csc_change) {
+	if (u->coeff_reduction_factor)
+		update_flags->bits.coeff_reduction_change = 1;
+
+	if (update_flags->bits.in_transfer_func_change) {
 		type = UPDATE_TYPE_MED;
 		elevate_update_type(&overall_type, type);
 	}
 
+	if (update_flags->bits.input_csc_change
+			|| update_flags->bits.coeff_reduction_change) {
+		type = UPDATE_TYPE_FULL;
+		elevate_update_type(&overall_type, type);
+	}
+
 	return overall_type;
 }
 
@@ -1297,7 +1213,7 @@ enum surface_update_type dc_check_update_surfaces_for_stream(
 	type = check_update_surfaces_for_stream(dc, updates, surface_count, stream_update, stream_status);
 	if (type == UPDATE_TYPE_FULL)
 		for (i = 0; i < surface_count; i++)
-			updates[i].surface->update_flags.bits.full_update = 1;
+			updates[i].surface->update_flags.raw = 0xFFFFFFFF;
 
 	return type;
 }
@@ -1375,6 +1291,12 @@ static void commit_planes_for_stream(struct dc *dc,
 					pipe_ctx->stream_res.abm->funcs->set_abm_level(
 							pipe_ctx->stream_res.abm, stream->abm_level);
 			}
+
+			if (stream_update && stream_update->periodic_fn_vsync_delta &&
+					pipe_ctx->stream_res.tg->funcs->program_vline_interrupt)
+				pipe_ctx->stream_res.tg->funcs->program_vline_interrupt(
+						pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing,
+						pipe_ctx->stream->periodic_fn_vsync_delta);
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_debug.c b/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
index 5a552cb3f8a7..267c76766dea 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
@@ -36,8 +36,9 @@
 #include "hw_sequencer.h"
 
 #include "resource.h"
-#define DC_LOGGER \
-	logger
+
+#define DC_LOGGER_INIT(logger)
+
 
 #define SURFACE_TRACE(...) do {\
 		if (dc->debug.surface_trace) \
@@ -60,8 +61,7 @@ void pre_surface_trace(
 		int surface_count)
 {
 	int i;
-	struct dc  *core_dc = dc;
-	struct dal_logger *logger =  core_dc->ctx->logger;
+	DC_LOGGER_INIT(dc->ctx->logger);
 
 	for (i = 0; i < surface_count; i++) {
 		const struct dc_plane_state *plane_state = plane_states[i];
@@ -72,8 +72,8 @@ void pre_surface_trace(
 				"plane_state->visible = %d;\n"
 				"plane_state->flip_immediate = %d;\n"
 				"plane_state->address.type = %d;\n"
-				"plane_state->address.grph.addr.quad_part = 0x%X;\n"
-				"plane_state->address.grph.meta_addr.quad_part = 0x%X;\n"
+				"plane_state->address.grph.addr.quad_part = 0x%llX;\n"
+				"plane_state->address.grph.meta_addr.quad_part = 0x%llX;\n"
 				"plane_state->scaling_quality.h_taps = %d;\n"
 				"plane_state->scaling_quality.v_taps = %d;\n"
 				"plane_state->scaling_quality.h_taps_c = %d;\n"
@@ -155,7 +155,6 @@ void pre_surface_trace(
 				"plane_state->tiling_info.gfx8.pipe_config = %d;\n"
 				"plane_state->tiling_info.gfx8.array_mode = %d;\n"
 				"plane_state->color_space = %d;\n"
-				"plane_state->input_tf = %d;\n"
 				"plane_state->dcc.enable = %d;\n"
 				"plane_state->format = %d;\n"
 				"plane_state->rotation = %d;\n"
@@ -163,7 +162,6 @@ void pre_surface_trace(
 				plane_state->tiling_info.gfx8.pipe_config,
 				plane_state->tiling_info.gfx8.array_mode,
 				plane_state->color_space,
-				plane_state->input_tf,
 				plane_state->dcc.enable,
 				plane_state->format,
 				plane_state->rotation,
@@ -183,8 +181,7 @@ void update_surface_trace(
 		int surface_count)
 {
 	int i;
-	struct dc  *core_dc = dc;
-	struct dal_logger *logger =  core_dc->ctx->logger;
+	DC_LOGGER_INIT(dc->ctx->logger);
 
 	for (i = 0; i < surface_count; i++) {
 		const struct dc_surface_update *update = &updates[i];
@@ -192,8 +189,8 @@ void update_surface_trace(
 		SURFACE_TRACE("Update %d\n", i);
 		if (update->flip_addr) {
 			SURFACE_TRACE("flip_addr->address.type = %d;\n"
-					"flip_addr->address.grph.addr.quad_part = 0x%X;\n"
-					"flip_addr->address.grph.meta_addr.quad_part = 0x%X;\n"
+					"flip_addr->address.grph.addr.quad_part = 0x%llX;\n"
+					"flip_addr->address.grph.meta_addr.quad_part = 0x%llX;\n"
 					"flip_addr->flip_immediate = %d;\n",
 					update->flip_addr->address.type,
 					update->flip_addr->address.grph.addr.quad_part,
@@ -204,16 +201,15 @@ void update_surface_trace(
 		if (update->plane_info) {
 			SURFACE_TRACE(
 					"plane_info->color_space = %d;\n"
-					"plane_info->input_tf = %d;\n"
 					"plane_info->format = %d;\n"
 					"plane_info->plane_size.grph.surface_pitch = %d;\n"
 					"plane_info->plane_size.grph.surface_size.height = %d;\n"
 					"plane_info->plane_size.grph.surface_size.width = %d;\n"
 					"plane_info->plane_size.grph.surface_size.x = %d;\n"
 					"plane_info->plane_size.grph.surface_size.y = %d;\n"
-					"plane_info->rotation = %d;\n",
+					"plane_info->rotation = %d;\n"
+					"plane_info->stereo_format = %d;\n",
 					update->plane_info->color_space,
-					update->plane_info->input_tf,
 					update->plane_info->format,
 					update->plane_info->plane_size.grph.surface_pitch,
 					update->plane_info->plane_size.grph.surface_size.height,
@@ -303,8 +299,7 @@ void update_surface_trace(
 
 void post_surface_trace(struct dc *dc)
 {
-	struct dc  *core_dc = dc;
-	struct dal_logger *logger =  core_dc->ctx->logger;
+	DC_LOGGER_INIT(dc->ctx->logger);
 
 	SURFACE_TRACE("post surface process.\n");
 
@@ -316,10 +311,10 @@ void context_timing_trace(
 {
 	int i;
 	struct dc  *core_dc = dc;
-	struct dal_logger *logger =  core_dc->ctx->logger;
 	int h_pos[MAX_PIPES], v_pos[MAX_PIPES];
 	struct crtc_position position;
 	unsigned int underlay_idx = core_dc->res_pool->underlay_pipe_index;
+	DC_LOGGER_INIT(dc->ctx->logger);
 
 
 	for (i = 0; i < core_dc->res_pool->pipe_count; i++) {
@@ -354,9 +349,7 @@ void context_clock_trace(
 		struct dc_state *context)
 {
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
-	struct dc  *core_dc = dc;
-	struct dal_logger *logger =  core_dc->ctx->logger;
-
+	DC_LOGGER_INIT(dc->ctx->logger);
 	CLOCK_TRACE("Current: dispclk_khz:%d  max_dppclk_khz:%d  dcfclk_khz:%d\n"
 			"dcfclk_deep_sleep_khz:%d  fclk_khz:%d  socclk_khz:%d\n",
 			context->bw.dcn.calc_clk.dispclk_khz,
@@ -371,6 +364,7 @@ void context_clock_trace(
 			context->bw.dcn.calc_clk.dppclk_khz,
 			context->bw.dcn.calc_clk.dcfclk_khz,
 			context->bw.dcn.calc_clk.dcfclk_deep_sleep_khz,
-			context->bw.dcn.calc_clk.fclk_khz);
+			context->bw.dcn.calc_clk.fclk_khz,
+			context->bw.dcn.calc_clk.socclk_khz);
 #endif
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index ebc96b720083..83d121510ef5 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -208,6 +208,7 @@ void color_space_to_black_color(
 	case COLOR_SPACE_YCBCR709:
 	case COLOR_SPACE_YCBCR601_LIMITED:
 	case COLOR_SPACE_YCBCR709_LIMITED:
+	case COLOR_SPACE_2020_YCBCR:
 		*black_color = black_color_format[BLACK_COLOR_FORMAT_YUV_CV];
 		break;
 
@@ -216,7 +217,25 @@ void color_space_to_black_color(
 			black_color_format[BLACK_COLOR_FORMAT_RGB_LIMITED];
 		break;
 
-	default:
+	/**
+	 * Remove default and add case for all color space
+	 * so when we forget to add new color space
+	 * compiler will give a warning
+	 */
+	case COLOR_SPACE_UNKNOWN:
+	case COLOR_SPACE_SRGB:
+	case COLOR_SPACE_XR_RGB:
+	case COLOR_SPACE_MSREF_SCRGB:
+	case COLOR_SPACE_XV_YCC_709:
+	case COLOR_SPACE_XV_YCC_601:
+	case COLOR_SPACE_2020_RGB_FULLRANGE:
+	case COLOR_SPACE_2020_RGB_LIMITEDRANGE:
+	case COLOR_SPACE_ADOBERGB:
+	case COLOR_SPACE_DCIP3:
+	case COLOR_SPACE_DISPLAYNATIVE:
+	case COLOR_SPACE_DOLBYVISION:
+	case COLOR_SPACE_APPCTRL:
+	case COLOR_SPACE_CUSTOMPOINTS:
 		/* fefault is sRGB black (full range). */
 		*black_color =
 			black_color_format[BLACK_COLOR_FORMAT_RGB_FULLRANGE];
@@ -230,6 +249,9 @@ bool hwss_wait_for_blank_complete(
 {
 	int counter;
 
+	/* Not applicable if the pipe is not primary, save 300ms of boot time */
+	if (!tg->funcs->is_blanked)
+		return true;
 	for (counter = 0; counter < 100; counter++) {
 		if (tg->funcs->is_blanked(tg))
 			break;
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
index 6d1c4981a185..2fa521812d23 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
@@ -45,8 +45,9 @@
 #include "dce/dce_11_0_d.h"
 #include "dce/dce_11_0_enum.h"
 #include "dce/dce_11_0_sh_mask.h"
-#define DC_LOGGER \
-	dc_ctx->logger
+
+#define DC_LOGGER_INIT(logger)
+
 
 #define LINK_INFO(...) \
 	DC_LOG_HW_HOTPLUG(  \
@@ -468,6 +469,13 @@ static void link_disconnect_sink(struct dc_link *link)
 	link->dpcd_sink_count = 0;
 }
 
+static void link_disconnect_remap(struct dc_sink *prev_sink, struct dc_link *link)
+{
+	dc_sink_release(link->local_sink);
+	link->local_sink = prev_sink;
+}
+
+
 static bool detect_dp(
 	struct dc_link *link,
 	struct display_sink_capability *sink_caps,
@@ -550,6 +558,17 @@ static bool detect_dp(
 	return true;
 }
 
+static bool is_same_edid(struct dc_edid *old_edid, struct dc_edid *new_edid)
+{
+	if (old_edid->length != new_edid->length)
+		return false;
+
+	if (new_edid->length == 0)
+		return false;
+
+	return (memcmp(old_edid->raw_edid, new_edid->raw_edid, new_edid->length) == 0);
+}
+
 bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 {
 	struct dc_sink_init_data sink_init_data = { 0 };
@@ -557,11 +576,15 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 	uint8_t i;
 	bool converter_disable_audio = false;
 	struct audio_support *aud_support = &link->dc->res_pool->audio_support;
+	bool same_edid = false;
 	enum dc_edid_status edid_status;
 	struct dc_context *dc_ctx = link->ctx;
 	struct dc_sink *sink = NULL;
+	struct dc_sink *prev_sink = NULL;
+	struct dpcd_caps prev_dpcd_caps;
+	bool same_dpcd = true;
 	enum dc_connection_type new_connection_type = dc_connection_none;
-
+	DC_LOGGER_INIT(link->ctx->logger);
 	if (link->connector_signal == SIGNAL_TYPE_VIRTUAL)
 		return false;
 
@@ -574,6 +597,11 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 			link->local_sink)
 		return true;
 
+	prev_sink = link->local_sink;
+	if (prev_sink != NULL) {
+		dc_sink_retain(prev_sink);
+		memcpy(&prev_dpcd_caps, &link->dpcd_caps, sizeof(struct dpcd_caps));
+	}
 	link_disconnect_sink(link);
 
 	if (new_connection_type != dc_connection_none) {
@@ -615,14 +643,25 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 				link,
 				&sink_caps,
 				&converter_disable_audio,
-				aud_support, reason))
+				aud_support, reason)) {
+				if (prev_sink != NULL)
+					dc_sink_release(prev_sink);
 				return false;
+			}
 
+			// Check if dpcp block is the same
+			if (prev_sink != NULL) {
+				if (memcmp(&link->dpcd_caps, &prev_dpcd_caps, sizeof(struct dpcd_caps)))
+					same_dpcd = false;
+			}
 			/* Active dongle downstream unplug */
 			if (link->type == dc_connection_active_dongle
 					&& link->dpcd_caps.sink_count.
-					bits.SINK_COUNT == 0)
+					bits.SINK_COUNT == 0) {
+				if (prev_sink != NULL)
+					dc_sink_release(prev_sink);
 				return true;
+			}
 
 			if (link->type == dc_connection_mst_branch) {
 				LINK_INFO("link=%d, mst branch is now Connected\n",
@@ -630,9 +669,11 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 				/* Need to setup mst link_cap struct here
 				 * otherwise dc_link_detect() will leave mst link_cap
 				 * empty which leads to allocate_mst_payload() has "0"
-				 * pbn_per_slot value leading to exception on dal_fixed31_32_div()
+				 * pbn_per_slot value leading to exception on dc_fixpt_div()
 				 */
 				link->verified_link_cap = link->reported_link_cap;
+				if (prev_sink != NULL)
+					dc_sink_release(prev_sink);
 				return false;
 			}
 
@@ -642,6 +683,8 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 		default:
 			DC_ERROR("Invalid connector type! signal:%d\n",
 				link->connector_signal);
+			if (prev_sink != NULL)
+				dc_sink_release(prev_sink);
 			return false;
 		} /* switch() */
 
@@ -664,6 +707,8 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 		sink = dc_sink_create(&sink_init_data);
 		if (!sink) {
 			DC_ERROR("Failed to create sink!\n");
+			if (prev_sink != NULL)
+				dc_sink_release(prev_sink);
 			return false;
 		}
 
@@ -687,22 +732,33 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 			break;
 		}
 
-		if (link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT &&
-			sink_caps.transaction_type ==
-			DDC_TRANSACTION_TYPE_I2C_OVER_AUX) {
-			/*
-			 * TODO debug why Dell 2413 doesn't like
-			 *  two link trainings
-			 */
+		// Check if edid is the same
+		if ((prev_sink != NULL) && ((edid_status == EDID_THE_SAME) || (edid_status == EDID_OK)))
+			same_edid = is_same_edid(&prev_sink->dc_edid, &sink->dc_edid);
 
-			/* deal with non-mst cases */
-			dp_hbr_verify_link_cap(link, &link->reported_link_cap);
-		}
+		// If both edid and dpcd are the same, then discard new sink and revert back to original sink
+		if ((same_edid) && (same_dpcd)) {
+			link_disconnect_remap(prev_sink, link);
+			sink = prev_sink;
+			prev_sink = NULL;
+		} else {
+			if (link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT &&
+					sink_caps.transaction_type ==
+						DDC_TRANSACTION_TYPE_I2C_OVER_AUX) {
+				/*
+				 * TODO debug why Dell 2413 doesn't like
+				 *  two link trainings
+				 */
 
-		/* HDMI-DVI Dongle */
-		if (sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A &&
-				!sink->edid_caps.edid_hdmi)
-			sink->sink_signal = SIGNAL_TYPE_DVI_SINGLE_LINK;
+				/* deal with non-mst cases */
+				dp_hbr_verify_link_cap(link, &link->reported_link_cap);
+			}
+
+			/* HDMI-DVI Dongle */
+			if (sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A &&
+					!sink->edid_caps.edid_hdmi)
+				sink->sink_signal = SIGNAL_TYPE_DVI_SINGLE_LINK;
+		}
 
 		/* Connectivity log: detection */
 		for (i = 0; i < sink->dc_edid.length / EDID_BLOCK_SIZE; i++) {
@@ -761,10 +817,14 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 		sink_caps.signal = SIGNAL_TYPE_NONE;
 	}
 
-	LINK_INFO("link=%d, dc_sink_in=%p is now %s\n",
+	LINK_INFO("link=%d, dc_sink_in=%p is now %s prev_sink=%p dpcd same=%d edid same=%d\n",
 		link->link_index, sink,
 		(sink_caps.signal == SIGNAL_TYPE_NONE ?
-			"Disconnected":"Connected"));
+			"Disconnected":"Connected"), prev_sink,
+			same_dpcd, same_edid);
+
+	if (prev_sink != NULL)
+		dc_sink_release(prev_sink);
 
 	return true;
 }
@@ -927,6 +987,7 @@ static bool construct(
 	struct integrated_info info = {{{ 0 }}};
 	struct dc_bios *bios = init_params->dc->ctx->dc_bios;
 	const struct dc_vbios_funcs *bp_funcs = bios->funcs;
+	DC_LOGGER_INIT(dc_ctx->logger);
 
 	link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
 	link->irq_source_hpd_rx = DC_IRQ_SOURCE_INVALID;
@@ -1135,7 +1196,8 @@ static void dpcd_configure_panel_mode(
 {
 	union dpcd_edp_config edp_config_set;
 	bool panel_mode_edp = false;
-	struct dc_context *dc_ctx = link->ctx;
+	DC_LOGGER_INIT(link->ctx->logger);
+
 	memset(&edp_config_set, '\0', sizeof(union dpcd_edp_config));
 
 	if (DP_PANEL_MODE_DEFAULT != panel_mode) {
@@ -1183,16 +1245,21 @@ static void enable_stream_features(struct pipe_ctx *pipe_ctx)
 {
 	struct dc_stream_state *stream = pipe_ctx->stream;
 	struct dc_link *link = stream->sink->link;
-	union down_spread_ctrl downspread;
+	union down_spread_ctrl old_downspread;
+	union down_spread_ctrl new_downspread;
 
 	core_link_read_dpcd(link, DP_DOWNSPREAD_CTRL,
-			&downspread.raw, sizeof(downspread));
+			&old_downspread.raw, sizeof(old_downspread));
 
-	downspread.bits.IGNORE_MSA_TIMING_PARAM =
+	new_downspread.raw = old_downspread.raw;
+
+	new_downspread.bits.IGNORE_MSA_TIMING_PARAM =
 			(stream->ignore_msa_timing_param) ? 1 : 0;
 
-	core_link_write_dpcd(link, DP_DOWNSPREAD_CTRL,
-			&downspread.raw, sizeof(downspread));
+	if (new_downspread.raw != old_downspread.raw) {
+		core_link_write_dpcd(link, DP_DOWNSPREAD_CTRL,
+			&new_downspread.raw, sizeof(new_downspread));
+	}
 }
 
 static enum dc_status enable_link_dp(
@@ -1843,9 +1910,22 @@ static void disable_link(struct dc_link *link, enum signal_type signal)
 
 static bool dp_active_dongle_validate_timing(
 		const struct dc_crtc_timing *timing,
-		const struct dc_dongle_caps *dongle_caps)
+		const struct dpcd_caps *dpcd_caps)
 {
 	unsigned int required_pix_clk = timing->pix_clk_khz;
+	const struct dc_dongle_caps *dongle_caps = &dpcd_caps->dongle_caps;
+
+	switch (dpcd_caps->dongle_type) {
+	case DISPLAY_DONGLE_DP_VGA_CONVERTER:
+	case DISPLAY_DONGLE_DP_DVI_CONVERTER:
+	case DISPLAY_DONGLE_DP_DVI_DONGLE:
+		if (timing->pixel_encoding == PIXEL_ENCODING_RGB)
+			return true;
+		else
+			return false;
+	default:
+		break;
+	}
 
 	if (dongle_caps->dongle_type != DISPLAY_DONGLE_DP_HDMI_CONVERTER ||
 		dongle_caps->extendedCapValid == false)
@@ -1911,7 +1991,7 @@ enum dc_status dc_link_validate_mode_timing(
 		const struct dc_crtc_timing *timing)
 {
 	uint32_t max_pix_clk = stream->sink->dongle_max_pix_clk;
-	struct dc_dongle_caps *dongle_caps = &link->dpcd_caps.dongle_caps;
+	struct dpcd_caps *dpcd_caps = &link->dpcd_caps;
 
 	/* A hack to avoid failing any modes for EDID override feature on
 	 * topology change such as lower quality cable for DP or different dongle
@@ -1924,7 +2004,7 @@ enum dc_status dc_link_validate_mode_timing(
 		return DC_EXCEED_DONGLE_CAP;
 
 	/* Active Dongle*/
-	if (!dp_active_dongle_validate_timing(timing, dongle_caps))
+	if (!dp_active_dongle_validate_timing(timing, dpcd_caps))
 		return DC_EXCEED_DONGLE_CAP;
 
 	switch (stream->signal) {
@@ -1950,10 +2030,10 @@ bool dc_link_set_backlight_level(const struct dc_link *link, uint32_t level,
 	struct dc  *core_dc = link->ctx->dc;
 	struct abm *abm = core_dc->res_pool->abm;
 	struct dmcu *dmcu = core_dc->res_pool->dmcu;
-	struct dc_context *dc_ctx = link->ctx;
 	unsigned int controller_id = 0;
 	bool use_smooth_brightness = true;
 	int i;
+	DC_LOGGER_INIT(link->ctx->logger);
 
 	if ((dmcu == NULL) ||
 		(abm == NULL) ||
@@ -1961,7 +2041,7 @@ bool dc_link_set_backlight_level(const struct dc_link *link, uint32_t level,
 		return false;
 
 	if (stream) {
-		if (stream->bl_pwm_level == 0)
+		if (stream->bl_pwm_level == EDP_BACKLIGHT_RAMP_DISABLE_LEVEL)
 			frame_ramp = 0;
 
 		((struct dc_stream_state *)stream)->bl_pwm_level = level;
@@ -2038,10 +2118,10 @@ static struct fixed31_32 get_pbn_per_slot(struct dc_stream_state *stream)
 			&stream->sink->link->cur_link_settings;
 	uint32_t link_rate_in_mbps =
 			link_settings->link_rate * LINK_RATE_REF_FREQ_IN_MHZ;
-	struct fixed31_32 mbps = dal_fixed31_32_from_int(
+	struct fixed31_32 mbps = dc_fixpt_from_int(
 			link_rate_in_mbps * link_settings->lane_count);
 
-	return dal_fixed31_32_div_int(mbps, 54);
+	return dc_fixpt_div_int(mbps, 54);
 }
 
 static int get_color_depth(enum dc_color_depth color_depth)
@@ -2082,7 +2162,7 @@ static struct fixed31_32 get_pbn_from_timing(struct pipe_ctx *pipe_ctx)
 	numerator = 64 * PEAK_FACTOR_X1000;
 	denominator = 54 * 8 * 1000 * 1000;
 	kbps *= numerator;
-	peak_kbps = dal_fixed31_32_from_fraction(kbps, denominator);
+	peak_kbps = dc_fixpt_from_fraction(kbps, denominator);
 
 	return peak_kbps;
 }
@@ -2149,8 +2229,8 @@ static enum dc_status allocate_mst_payload(struct pipe_ctx *pipe_ctx)
 	struct fixed31_32 avg_time_slots_per_mtp;
 	struct fixed31_32 pbn;
 	struct fixed31_32 pbn_per_slot;
-	struct dc_context *dc_ctx = link->ctx;
 	uint8_t i;
+	DC_LOGGER_INIT(link->ctx->logger);
 
 	/* enable_link_dp_mst already check link->enabled_stream_count
 	 * and stream is in link->stream[]. This is called during set mode,
@@ -2178,11 +2258,11 @@ static enum dc_status allocate_mst_payload(struct pipe_ctx *pipe_ctx)
 			link->mst_stream_alloc_table.stream_count);
 
 	for (i = 0; i < MAX_CONTROLLER_NUM; i++) {
-		DC_LOG_MST("stream_enc[%d]: 0x%x      "
+		DC_LOG_MST("stream_enc[%d]: %p      "
 		"stream[%d].vcp_id: %d      "
 		"stream[%d].slot_count: %d\n",
 		i,
-		link->mst_stream_alloc_table.stream_allocations[i].stream_enc,
+		(void *) link->mst_stream_alloc_table.stream_allocations[i].stream_enc,
 		i,
 		link->mst_stream_alloc_table.stream_allocations[i].vcp_id,
 		i,
@@ -2209,7 +2289,7 @@ static enum dc_status allocate_mst_payload(struct pipe_ctx *pipe_ctx)
 	/* slot X.Y for only current stream */
 	pbn_per_slot = get_pbn_per_slot(stream);
 	pbn = get_pbn_from_timing(pipe_ctx);
-	avg_time_slots_per_mtp = dal_fixed31_32_div(pbn, pbn_per_slot);
+	avg_time_slots_per_mtp = dc_fixpt_div(pbn, pbn_per_slot);
 
 	stream_encoder->funcs->set_mst_bandwidth(
 		stream_encoder,
@@ -2226,10 +2306,10 @@ static enum dc_status deallocate_mst_payload(struct pipe_ctx *pipe_ctx)
 	struct link_encoder *link_encoder = link->link_enc;
 	struct stream_encoder *stream_encoder = pipe_ctx->stream_res.stream_enc;
 	struct dp_mst_stream_allocation_table proposed_table = {0};
-	struct fixed31_32 avg_time_slots_per_mtp = dal_fixed31_32_from_int(0);
+	struct fixed31_32 avg_time_slots_per_mtp = dc_fixpt_from_int(0);
 	uint8_t i;
 	bool mst_mode = (link->type == dc_connection_mst_branch);
-	struct dc_context *dc_ctx = link->ctx;
+	DC_LOGGER_INIT(link->ctx->logger);
 
 	/* deallocate_mst_payload is called before disable link. When mode or
 	 * disable/enable monitor, new stream is created which is not in link
@@ -2268,11 +2348,11 @@ static enum dc_status deallocate_mst_payload(struct pipe_ctx *pipe_ctx)
 			link->mst_stream_alloc_table.stream_count);
 
 	for (i = 0; i < MAX_CONTROLLER_NUM; i++) {
-		DC_LOG_MST("stream_enc[%d]: 0x%x      "
+		DC_LOG_MST("stream_enc[%d]: %p      "
 		"stream[%d].vcp_id: %d      "
 		"stream[%d].slot_count: %d\n",
 		i,
-		link->mst_stream_alloc_table.stream_allocations[i].stream_enc,
+		(void *) link->mst_stream_alloc_table.stream_allocations[i].stream_enc,
 		i,
 		link->mst_stream_alloc_table.stream_allocations[i].vcp_id,
 		i,
@@ -2302,8 +2382,8 @@ void core_link_enable_stream(
 		struct pipe_ctx *pipe_ctx)
 {
 	struct dc  *core_dc = pipe_ctx->stream->ctx->dc;
-	struct dc_context *dc_ctx = pipe_ctx->stream->ctx;
 	enum dc_status status;
+	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
 
 	/* eDP lit up by bios already, no need to enable again. */
 	if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP &&
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
index 3b5053570229..7d609c71394b 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
@@ -1378,8 +1378,8 @@ static uint32_t bandwidth_in_kbps_from_timing(
 {
 	uint32_t bits_per_channel = 0;
 	uint32_t kbps;
-	switch (timing->display_color_depth) {
 
+	switch (timing->display_color_depth) {
 	case COLOR_DEPTH_666:
 		bits_per_channel = 6;
 		break;
@@ -1401,14 +1401,20 @@ static uint32_t bandwidth_in_kbps_from_timing(
 	default:
 		break;
 	}
+
 	ASSERT(bits_per_channel != 0);
 
 	kbps = timing->pix_clk_khz;
 	kbps *= bits_per_channel;
 
-	if (timing->flags.Y_ONLY != 1)
+	if (timing->flags.Y_ONLY != 1) {
 		/*Only YOnly make reduce bandwidth by 1/3 compares to RGB*/
 		kbps *= 3;
+		if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR420)
+			kbps /= 2;
+		else if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR422)
+			kbps = kbps * 2 / 3;
+	}
 
 	return kbps;
 
@@ -2278,6 +2284,8 @@ static bool retrieve_link_cap(struct dc_link *link)
 	union edp_configuration_cap edp_config_cap;
 	union dp_downstream_port_present ds_port = { 0 };
 	enum dc_status status = DC_ERROR_UNEXPECTED;
+	uint32_t read_dpcd_retry_cnt = 3;
+	int i;
 
 	memset(dpcd_data, '\0', sizeof(dpcd_data));
 	memset(&down_strm_port_count,
@@ -2285,11 +2293,15 @@ static bool retrieve_link_cap(struct dc_link *link)
 	memset(&edp_config_cap, '\0',
 		sizeof(union edp_configuration_cap));
 
-	status = core_link_read_dpcd(
-			link,
-			DP_DPCD_REV,
-			dpcd_data,
-			sizeof(dpcd_data));
+	for (i = 0; i < read_dpcd_retry_cnt; i++) {
+		status = core_link_read_dpcd(
+				link,
+				DP_DPCD_REV,
+				dpcd_data,
+				sizeof(dpcd_data));
+		if (status == DC_OK)
+			break;
+	}
 
 	if (status != DC_OK) {
 		dm_error("%s: Read dpcd data failed.\n", __func__);
@@ -2376,6 +2388,10 @@ bool detect_dp_sink_caps(struct dc_link *link)
 void detect_edp_sink_caps(struct dc_link *link)
 {
 	retrieve_link_cap(link);
+
+	if (link->reported_link_cap.link_rate == LINK_RATE_UNKNOWN)
+		link->reported_link_cap.link_rate = LINK_RATE_HIGH2;
+
 	link->verified_link_cap = link->reported_link_cap;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
index 7c866a7d5e77..82cd1d6e6e59 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
@@ -11,8 +11,6 @@
 #include "dc_link_dp.h"
 #include "dc_link_ddc.h"
 #include "dm_helpers.h"
-#include "dce/dce_link_encoder.h"
-#include "dce/dce_stream_encoder.h"
 #include "dpcd_defs.h"
 
 enum dc_status core_link_read_dpcd(
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index ba3487e97361..751f3ac9d921 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -45,8 +45,9 @@
 #include "dcn10/dcn10_resource.h"
 #endif
 #include "dce120/dce120_resource.h"
-#define DC_LOGGER \
-	ctx->logger
+
+#define DC_LOGGER_INIT(logger)
+
 enum dce_version resource_parse_asic_id(struct hw_asic_id asic_id)
 {
 	enum dce_version dc_version = DCE_VERSION_UNKNOWN;
@@ -78,6 +79,8 @@ enum dce_version resource_parse_asic_id(struct hw_asic_id asic_id)
 				ASIC_REV_IS_POLARIS12_V(asic_id.hw_internal_rev)) {
 			dc_version = DCE_VERSION_11_2;
 		}
+		if (ASIC_REV_IS_VEGAM(asic_id.hw_internal_rev))
+			dc_version = DCE_VERSION_11_22;
 		break;
 	case FAMILY_AI:
 		dc_version = DCE_VERSION_12_0;
@@ -124,6 +127,7 @@ struct resource_pool *dc_create_resource_pool(
 			num_virtual_links, dc, asic_id);
 		break;
 	case DCE_VERSION_11_2:
+	case DCE_VERSION_11_22:
 		res_pool = dce112_create_resource_pool(
 			num_virtual_links, dc);
 		break;
@@ -492,9 +496,9 @@ static void calculate_viewport(struct pipe_ctx *pipe_ctx)
 	data->viewport_c.x = data->viewport.x / vpc_div;
 	data->viewport_c.y = data->viewport.y / vpc_div;
 	data->inits.h_c = (data->viewport.x % vpc_div) != 0 ?
-			dal_fixed31_32_half : dal_fixed31_32_zero;
+			dc_fixpt_half : dc_fixpt_zero;
 	data->inits.v_c = (data->viewport.y % vpc_div) != 0 ?
-			dal_fixed31_32_half : dal_fixed31_32_zero;
+			dc_fixpt_half : dc_fixpt_zero;
 	/* Round up, assume original video size always even dimensions */
 	data->viewport_c.width = (data->viewport.width + vpc_div - 1) / vpc_div;
 	data->viewport_c.height = (data->viewport.height + vpc_div - 1) / vpc_div;
@@ -623,10 +627,10 @@ static void calculate_scaling_ratios(struct pipe_ctx *pipe_ctx)
 			pipe_ctx->plane_state->rotation == ROTATION_ANGLE_270)
 		rect_swap_helper(&surf_src);
 
-	pipe_ctx->plane_res.scl_data.ratios.horz = dal_fixed31_32_from_fraction(
+	pipe_ctx->plane_res.scl_data.ratios.horz = dc_fixpt_from_fraction(
 					surf_src.width,
 					plane_state->dst_rect.width);
-	pipe_ctx->plane_res.scl_data.ratios.vert = dal_fixed31_32_from_fraction(
+	pipe_ctx->plane_res.scl_data.ratios.vert = dc_fixpt_from_fraction(
 					surf_src.height,
 					plane_state->dst_rect.height);
 
@@ -648,6 +652,14 @@ static void calculate_scaling_ratios(struct pipe_ctx *pipe_ctx)
 		pipe_ctx->plane_res.scl_data.ratios.horz_c.value /= 2;
 		pipe_ctx->plane_res.scl_data.ratios.vert_c.value /= 2;
 	}
+	pipe_ctx->plane_res.scl_data.ratios.horz = dc_fixpt_truncate(
+			pipe_ctx->plane_res.scl_data.ratios.horz, 19);
+	pipe_ctx->plane_res.scl_data.ratios.vert = dc_fixpt_truncate(
+			pipe_ctx->plane_res.scl_data.ratios.vert, 19);
+	pipe_ctx->plane_res.scl_data.ratios.horz_c = dc_fixpt_truncate(
+			pipe_ctx->plane_res.scl_data.ratios.horz_c, 19);
+	pipe_ctx->plane_res.scl_data.ratios.vert_c = dc_fixpt_truncate(
+			pipe_ctx->plane_res.scl_data.ratios.vert_c, 19);
 }
 
 static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *recout_skip)
@@ -684,32 +696,33 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 	 * 	init_bot = init + scaling_ratio
 	 * 	init_c = init + truncated_vp_c_offset(from calculate viewport)
 	 */
-	data->inits.h = dal_fixed31_32_div_int(
-			dal_fixed31_32_add_int(data->ratios.horz, data->taps.h_taps + 1), 2);
+	data->inits.h = dc_fixpt_truncate(dc_fixpt_div_int(
+			dc_fixpt_add_int(data->ratios.horz, data->taps.h_taps + 1), 2), 19);
 
-	data->inits.h_c = dal_fixed31_32_add(data->inits.h_c, dal_fixed31_32_div_int(
-			dal_fixed31_32_add_int(data->ratios.horz_c, data->taps.h_taps_c + 1), 2));
+	data->inits.h_c = dc_fixpt_truncate(dc_fixpt_add(data->inits.h_c, dc_fixpt_div_int(
+			dc_fixpt_add_int(data->ratios.horz_c, data->taps.h_taps_c + 1), 2)), 19);
 
-	data->inits.v = dal_fixed31_32_div_int(
-			dal_fixed31_32_add_int(data->ratios.vert, data->taps.v_taps + 1), 2);
+	data->inits.v = dc_fixpt_truncate(dc_fixpt_div_int(
+			dc_fixpt_add_int(data->ratios.vert, data->taps.v_taps + 1), 2), 19);
+
+	data->inits.v_c = dc_fixpt_truncate(dc_fixpt_add(data->inits.v_c, dc_fixpt_div_int(
+			dc_fixpt_add_int(data->ratios.vert_c, data->taps.v_taps_c + 1), 2)), 19);
 
-	data->inits.v_c = dal_fixed31_32_add(data->inits.v_c, dal_fixed31_32_div_int(
-			dal_fixed31_32_add_int(data->ratios.vert_c, data->taps.v_taps_c + 1), 2));
 
 
 	/* Adjust for viewport end clip-off */
 	if ((data->viewport.x + data->viewport.width) < (src.x + src.width) && !flip_horz_scan_dir) {
 		int vp_clip = src.x + src.width - data->viewport.width - data->viewport.x;
-		int int_part = dal_fixed31_32_floor(
-				dal_fixed31_32_sub(data->inits.h, data->ratios.horz));
+		int int_part = dc_fixpt_floor(
+				dc_fixpt_sub(data->inits.h, data->ratios.horz));
 
 		int_part = int_part > 0 ? int_part : 0;
 		data->viewport.width += int_part < vp_clip ? int_part : vp_clip;
 	}
 	if ((data->viewport.y + data->viewport.height) < (src.y + src.height) && !flip_vert_scan_dir) {
 		int vp_clip = src.y + src.height - data->viewport.height - data->viewport.y;
-		int int_part = dal_fixed31_32_floor(
-				dal_fixed31_32_sub(data->inits.v, data->ratios.vert));
+		int int_part = dc_fixpt_floor(
+				dc_fixpt_sub(data->inits.v, data->ratios.vert));
 
 		int_part = int_part > 0 ? int_part : 0;
 		data->viewport.height += int_part < vp_clip ? int_part : vp_clip;
@@ -717,8 +730,8 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 	if ((data->viewport_c.x + data->viewport_c.width) < (src.x + src.width) / vpc_div && !flip_horz_scan_dir) {
 		int vp_clip = (src.x + src.width) / vpc_div -
 				data->viewport_c.width - data->viewport_c.x;
-		int int_part = dal_fixed31_32_floor(
-				dal_fixed31_32_sub(data->inits.h_c, data->ratios.horz_c));
+		int int_part = dc_fixpt_floor(
+				dc_fixpt_sub(data->inits.h_c, data->ratios.horz_c));
 
 		int_part = int_part > 0 ? int_part : 0;
 		data->viewport_c.width += int_part < vp_clip ? int_part : vp_clip;
@@ -726,8 +739,8 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 	if ((data->viewport_c.y + data->viewport_c.height) < (src.y + src.height) / vpc_div && !flip_vert_scan_dir) {
 		int vp_clip = (src.y + src.height) / vpc_div -
 				data->viewport_c.height - data->viewport_c.y;
-		int int_part = dal_fixed31_32_floor(
-				dal_fixed31_32_sub(data->inits.v_c, data->ratios.vert_c));
+		int int_part = dc_fixpt_floor(
+				dc_fixpt_sub(data->inits.v_c, data->ratios.vert_c));
 
 		int_part = int_part > 0 ? int_part : 0;
 		data->viewport_c.height += int_part < vp_clip ? int_part : vp_clip;
@@ -737,9 +750,9 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 	if (data->viewport.x && !flip_horz_scan_dir) {
 		int int_part;
 
-		data->inits.h = dal_fixed31_32_add(data->inits.h, dal_fixed31_32_mul_int(
+		data->inits.h = dc_fixpt_add(data->inits.h, dc_fixpt_mul_int(
 				data->ratios.horz, recout_skip->width));
-		int_part = dal_fixed31_32_floor(data->inits.h) - data->viewport.x;
+		int_part = dc_fixpt_floor(data->inits.h) - data->viewport.x;
 		if (int_part < data->taps.h_taps) {
 			int int_adj = data->viewport.x >= (data->taps.h_taps - int_part) ?
 						(data->taps.h_taps - int_part) : data->viewport.x;
@@ -752,15 +765,15 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 			int_part = data->taps.h_taps;
 		}
 		data->inits.h.value &= 0xffffffff;
-		data->inits.h = dal_fixed31_32_add_int(data->inits.h, int_part);
+		data->inits.h = dc_fixpt_add_int(data->inits.h, int_part);
 	}
 
 	if (data->viewport_c.x && !flip_horz_scan_dir) {
 		int int_part;
 
-		data->inits.h_c = dal_fixed31_32_add(data->inits.h_c, dal_fixed31_32_mul_int(
+		data->inits.h_c = dc_fixpt_add(data->inits.h_c, dc_fixpt_mul_int(
 				data->ratios.horz_c, recout_skip->width));
-		int_part = dal_fixed31_32_floor(data->inits.h_c) - data->viewport_c.x;
+		int_part = dc_fixpt_floor(data->inits.h_c) - data->viewport_c.x;
 		if (int_part < data->taps.h_taps_c) {
 			int int_adj = data->viewport_c.x >= (data->taps.h_taps_c - int_part) ?
 					(data->taps.h_taps_c - int_part) : data->viewport_c.x;
@@ -773,15 +786,15 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 			int_part = data->taps.h_taps_c;
 		}
 		data->inits.h_c.value &= 0xffffffff;
-		data->inits.h_c = dal_fixed31_32_add_int(data->inits.h_c, int_part);
+		data->inits.h_c = dc_fixpt_add_int(data->inits.h_c, int_part);
 	}
 
 	if (data->viewport.y && !flip_vert_scan_dir) {
 		int int_part;
 
-		data->inits.v = dal_fixed31_32_add(data->inits.v, dal_fixed31_32_mul_int(
+		data->inits.v = dc_fixpt_add(data->inits.v, dc_fixpt_mul_int(
 				data->ratios.vert, recout_skip->height));
-		int_part = dal_fixed31_32_floor(data->inits.v) - data->viewport.y;
+		int_part = dc_fixpt_floor(data->inits.v) - data->viewport.y;
 		if (int_part < data->taps.v_taps) {
 			int int_adj = data->viewport.y >= (data->taps.v_taps - int_part) ?
 						(data->taps.v_taps - int_part) : data->viewport.y;
@@ -794,15 +807,15 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 			int_part = data->taps.v_taps;
 		}
 		data->inits.v.value &= 0xffffffff;
-		data->inits.v = dal_fixed31_32_add_int(data->inits.v, int_part);
+		data->inits.v = dc_fixpt_add_int(data->inits.v, int_part);
 	}
 
 	if (data->viewport_c.y && !flip_vert_scan_dir) {
 		int int_part;
 
-		data->inits.v_c = dal_fixed31_32_add(data->inits.v_c, dal_fixed31_32_mul_int(
+		data->inits.v_c = dc_fixpt_add(data->inits.v_c, dc_fixpt_mul_int(
 				data->ratios.vert_c, recout_skip->height));
-		int_part = dal_fixed31_32_floor(data->inits.v_c) - data->viewport_c.y;
+		int_part = dc_fixpt_floor(data->inits.v_c) - data->viewport_c.y;
 		if (int_part < data->taps.v_taps_c) {
 			int int_adj = data->viewport_c.y >= (data->taps.v_taps_c - int_part) ?
 					(data->taps.v_taps_c - int_part) : data->viewport_c.y;
@@ -815,12 +828,12 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 			int_part = data->taps.v_taps_c;
 		}
 		data->inits.v_c.value &= 0xffffffff;
-		data->inits.v_c = dal_fixed31_32_add_int(data->inits.v_c, int_part);
+		data->inits.v_c = dc_fixpt_add_int(data->inits.v_c, int_part);
 	}
 
 	/* Interlaced inits based on final vert inits */
-	data->inits.v_bot = dal_fixed31_32_add(data->inits.v, data->ratios.vert);
-	data->inits.v_c_bot = dal_fixed31_32_add(data->inits.v_c, data->ratios.vert_c);
+	data->inits.v_bot = dc_fixpt_add(data->inits.v, data->ratios.vert);
+	data->inits.v_c_bot = dc_fixpt_add(data->inits.v_c, data->ratios.vert_c);
 
 	if (pipe_ctx->plane_state->rotation == ROTATION_ANGLE_90 ||
 			pipe_ctx->plane_state->rotation == ROTATION_ANGLE_270) {
@@ -835,7 +848,7 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx)
 	struct dc_crtc_timing *timing = &pipe_ctx->stream->timing;
 	struct view recout_skip = { 0 };
 	bool res = false;
-	struct dc_context *ctx = pipe_ctx->stream->ctx;
+	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
 	/* Important: scaling ratio calculation requires pixel format,
 	 * lb depth calculation requires recout and taps require scaling ratios.
 	 * Inits require viewport, taps, ratios and recout of split pipe
@@ -843,6 +856,9 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx)
 	pipe_ctx->plane_res.scl_data.format = convert_pixel_format_to_dalsurface(
 			pipe_ctx->plane_state->format);
 
+	if (pipe_ctx->stream->timing.flags.INTERLACE)
+		pipe_ctx->stream->dst.height *= 2;
+
 	calculate_scaling_ratios(pipe_ctx);
 
 	calculate_viewport(pipe_ctx);
@@ -863,6 +879,8 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx)
 
 	pipe_ctx->plane_res.scl_data.h_active = timing->h_addressable + timing->h_border_left + timing->h_border_right;
 	pipe_ctx->plane_res.scl_data.v_active = timing->v_addressable + timing->v_border_top + timing->v_border_bottom;
+	if (pipe_ctx->stream->timing.flags.INTERLACE)
+		pipe_ctx->plane_res.scl_data.v_active *= 2;
 
 
 	/* Taps calculations */
@@ -908,6 +926,9 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx)
 				plane_state->dst_rect.x,
 				plane_state->dst_rect.y);
 
+	if (pipe_ctx->stream->timing.flags.INTERLACE)
+		pipe_ctx->stream->dst.height /= 2;
+
 	return res;
 }
 
@@ -1294,6 +1315,19 @@ bool dc_add_all_planes_for_stream(
 }
 
 
+static bool is_hdr_static_meta_changed(struct dc_stream_state *cur_stream,
+	struct dc_stream_state *new_stream)
+{
+	if (cur_stream == NULL)
+		return true;
+
+	if (memcmp(&cur_stream->hdr_static_metadata,
+			&new_stream->hdr_static_metadata,
+			sizeof(struct dc_info_packet)) != 0)
+		return true;
+
+	return false;
+}
 
 static bool is_timing_changed(struct dc_stream_state *cur_stream,
 		struct dc_stream_state *new_stream)
@@ -1329,6 +1363,9 @@ static bool are_stream_backends_same(
 	if (is_timing_changed(stream_a, stream_b))
 		return false;
 
+	if (is_hdr_static_meta_changed(stream_a, stream_b))
+		return false;
+
 	return true;
 }
 
@@ -1599,18 +1636,6 @@ enum dc_status dc_remove_stream_from_ctx(
 	return DC_OK;
 }
 
-static void copy_pipe_ctx(
-	const struct pipe_ctx *from_pipe_ctx, struct pipe_ctx *to_pipe_ctx)
-{
-	struct dc_plane_state *plane_state = to_pipe_ctx->plane_state;
-	struct dc_stream_state *stream = to_pipe_ctx->stream;
-
-	*to_pipe_ctx = *from_pipe_ctx;
-	to_pipe_ctx->stream = stream;
-	if (plane_state != NULL)
-		to_pipe_ctx->plane_state = plane_state;
-}
-
 static struct dc_stream_state *find_pll_sharable_stream(
 		struct dc_stream_state *stream_needs_pll,
 		struct dc_state *context)
@@ -1703,7 +1728,7 @@ enum dc_status resource_map_pool_resources(
 		pipe_idx = acquire_first_split_pipe(&context->res_ctx, pool, stream);
 #endif
 
-	if (pipe_idx < 0)
+	if (pipe_idx < 0 || context->res_ctx.pipe_ctx[pipe_idx].stream_res.tg == NULL)
 		return DC_NO_CONTROLLER_RESOURCE;
 
 	pipe_ctx = &context->res_ctx.pipe_ctx[pipe_idx];
@@ -1752,26 +1777,6 @@ enum dc_status resource_map_pool_resources(
 	return DC_ERROR_UNEXPECTED;
 }
 
-/* first stream in the context is used to populate the rest */
-void validate_guaranteed_copy_streams(
-		struct dc_state *context,
-		int max_streams)
-{
-	int i;
-
-	for (i = 1; i < max_streams; i++) {
-		context->streams[i] = context->streams[0];
-
-		copy_pipe_ctx(&context->res_ctx.pipe_ctx[0],
-			      &context->res_ctx.pipe_ctx[i]);
-		context->res_ctx.pipe_ctx[i].stream =
-				context->res_ctx.pipe_ctx[0].stream;
-
-		dc_stream_retain(context->streams[i]);
-		context->stream_count++;
-	}
-}
-
 void dc_resource_state_copy_construct_current(
 		const struct dc *dc,
 		struct dc_state *dst_ctx)
@@ -1798,9 +1803,9 @@ enum dc_status dc_validate_global_state(
 		return DC_ERROR_UNEXPECTED;
 
 	if (dc->res_pool->funcs->validate_global) {
-			result = dc->res_pool->funcs->validate_global(dc, new_ctx);
-			if (result != DC_OK)
-				return result;
+		result = dc->res_pool->funcs->validate_global(dc, new_ctx);
+		if (result != DC_OK)
+			return result;
 	}
 
 	for (i = 0; i < new_ctx->stream_count; i++) {
@@ -1843,7 +1848,7 @@ enum dc_status dc_validate_global_state(
 }
 
 static void patch_gamut_packet_checksum(
-		struct encoder_info_packet *gamut_packet)
+		struct dc_info_packet *gamut_packet)
 {
 	/* For gamut we recalc checksum */
 	if (gamut_packet->valid) {
@@ -1862,12 +1867,11 @@ static void patch_gamut_packet_checksum(
 }
 
 static void set_avi_info_frame(
-		struct encoder_info_packet *info_packet,
+		struct dc_info_packet *info_packet,
 		struct pipe_ctx *pipe_ctx)
 {
 	struct dc_stream_state *stream = pipe_ctx->stream;
 	enum dc_color_space color_space = COLOR_SPACE_UNKNOWN;
-	struct info_frame info_frame = { {0} };
 	uint32_t pixel_encoding = 0;
 	enum scanning_type scan_type = SCANNING_TYPE_NODATA;
 	enum dc_aspect_ratio aspect = ASPECT_RATIO_NO_DATA;
@@ -1877,22 +1881,24 @@ static void set_avi_info_frame(
 	unsigned int cn0_cn1_value = 0;
 	uint8_t *check_sum = NULL;
 	uint8_t byte_index = 0;
-	union hdmi_info_packet *hdmi_info = &info_frame.avi_info_packet.info_packet_hdmi;
+	union hdmi_info_packet hdmi_info;
 	union display_content_support support = {0};
 	unsigned int vic = pipe_ctx->stream->timing.vic;
 	enum dc_timing_3d_format format;
 
+	memset(&hdmi_info, 0, sizeof(union hdmi_info_packet));
+
 	color_space = pipe_ctx->stream->output_color_space;
 	if (color_space == COLOR_SPACE_UNKNOWN)
 		color_space = (stream->timing.pixel_encoding == PIXEL_ENCODING_RGB) ?
 			COLOR_SPACE_SRGB:COLOR_SPACE_YCBCR709;
 
 	/* Initialize header */
-	hdmi_info->bits.header.info_frame_type = HDMI_INFOFRAME_TYPE_AVI;
+	hdmi_info.bits.header.info_frame_type = HDMI_INFOFRAME_TYPE_AVI;
 	/* InfoFrameVersion_3 is defined by CEA861F (Section 6.4), but shall
 	* not be used in HDMI 2.0 (Section 10.1) */
-	hdmi_info->bits.header.version = 2;
-	hdmi_info->bits.header.length = HDMI_AVI_INFOFRAME_SIZE;
+	hdmi_info.bits.header.version = 2;
+	hdmi_info.bits.header.length = HDMI_AVI_INFOFRAME_SIZE;
 
 	/*
 	 * IDO-defined (Y2,Y1,Y0 = 1,1,1) shall not be used by devices built
@@ -1918,39 +1924,39 @@ static void set_avi_info_frame(
 
 	/* Y0_Y1_Y2 : The pixel encoding */
 	/* H14b AVI InfoFrame has extension on Y-field from 2 bits to 3 bits */
-	hdmi_info->bits.Y0_Y1_Y2 = pixel_encoding;
+	hdmi_info.bits.Y0_Y1_Y2 = pixel_encoding;
 
 	/* A0 = 1 Active Format Information valid */
-	hdmi_info->bits.A0 = ACTIVE_FORMAT_VALID;
+	hdmi_info.bits.A0 = ACTIVE_FORMAT_VALID;
 
 	/* B0, B1 = 3; Bar info data is valid */
-	hdmi_info->bits.B0_B1 = BAR_INFO_BOTH_VALID;
+	hdmi_info.bits.B0_B1 = BAR_INFO_BOTH_VALID;
 
-	hdmi_info->bits.SC0_SC1 = PICTURE_SCALING_UNIFORM;
+	hdmi_info.bits.SC0_SC1 = PICTURE_SCALING_UNIFORM;
 
 	/* S0, S1 : Underscan / Overscan */
 	/* TODO: un-hardcode scan type */
 	scan_type = SCANNING_TYPE_UNDERSCAN;
-	hdmi_info->bits.S0_S1 = scan_type;
+	hdmi_info.bits.S0_S1 = scan_type;
 
 	/* C0, C1 : Colorimetry */
 	if (color_space == COLOR_SPACE_YCBCR709 ||
 			color_space == COLOR_SPACE_YCBCR709_LIMITED)
-		hdmi_info->bits.C0_C1 = COLORIMETRY_ITU709;
+		hdmi_info.bits.C0_C1 = COLORIMETRY_ITU709;
 	else if (color_space == COLOR_SPACE_YCBCR601 ||
 			color_space == COLOR_SPACE_YCBCR601_LIMITED)
-		hdmi_info->bits.C0_C1 = COLORIMETRY_ITU601;
+		hdmi_info.bits.C0_C1 = COLORIMETRY_ITU601;
 	else {
-		hdmi_info->bits.C0_C1 = COLORIMETRY_NO_DATA;
+		hdmi_info.bits.C0_C1 = COLORIMETRY_NO_DATA;
 	}
 	if (color_space == COLOR_SPACE_2020_RGB_FULLRANGE ||
 			color_space == COLOR_SPACE_2020_RGB_LIMITEDRANGE ||
 			color_space == COLOR_SPACE_2020_YCBCR) {
-		hdmi_info->bits.EC0_EC2 = COLORIMETRYEX_BT2020RGBYCBCR;
-		hdmi_info->bits.C0_C1   = COLORIMETRY_EXTENDED;
+		hdmi_info.bits.EC0_EC2 = COLORIMETRYEX_BT2020RGBYCBCR;
+		hdmi_info.bits.C0_C1   = COLORIMETRY_EXTENDED;
 	} else if (color_space == COLOR_SPACE_ADOBERGB) {
-		hdmi_info->bits.EC0_EC2 = COLORIMETRYEX_ADOBERGB;
-		hdmi_info->bits.C0_C1   = COLORIMETRY_EXTENDED;
+		hdmi_info.bits.EC0_EC2 = COLORIMETRYEX_ADOBERGB;
+		hdmi_info.bits.C0_C1   = COLORIMETRY_EXTENDED;
 	}
 
 	/* TODO: un-hardcode aspect ratio */
@@ -1959,18 +1965,18 @@ static void set_avi_info_frame(
 	switch (aspect) {
 	case ASPECT_RATIO_4_3:
 	case ASPECT_RATIO_16_9:
-		hdmi_info->bits.M0_M1 = aspect;
+		hdmi_info.bits.M0_M1 = aspect;
 		break;
 
 	case ASPECT_RATIO_NO_DATA:
 	case ASPECT_RATIO_64_27:
 	case ASPECT_RATIO_256_135:
 	default:
-		hdmi_info->bits.M0_M1 = 0;
+		hdmi_info.bits.M0_M1 = 0;
 	}
 
 	/* Active Format Aspect ratio - same as Picture Aspect Ratio. */
-	hdmi_info->bits.R0_R3 = ACTIVE_FORMAT_ASPECT_RATIO_SAME_AS_PICTURE;
+	hdmi_info.bits.R0_R3 = ACTIVE_FORMAT_ASPECT_RATIO_SAME_AS_PICTURE;
 
 	/* TODO: un-hardcode cn0_cn1 and itc */
 
@@ -2013,8 +2019,8 @@ static void set_avi_info_frame(
 				}
 			}
 		}
-		hdmi_info->bits.CN0_CN1 = cn0_cn1_value;
-		hdmi_info->bits.ITC = itc_value;
+		hdmi_info.bits.CN0_CN1 = cn0_cn1_value;
+		hdmi_info.bits.ITC = itc_value;
 	}
 
 	/* TODO : We should handle YCC quantization */
@@ -2023,19 +2029,19 @@ static void set_avi_info_frame(
 			stream->sink->edid_caps.qy_bit == 1) {
 		if (color_space == COLOR_SPACE_SRGB ||
 			color_space == COLOR_SPACE_2020_RGB_FULLRANGE) {
-			hdmi_info->bits.Q0_Q1   = RGB_QUANTIZATION_FULL_RANGE;
-			hdmi_info->bits.YQ0_YQ1 = YYC_QUANTIZATION_FULL_RANGE;
+			hdmi_info.bits.Q0_Q1   = RGB_QUANTIZATION_FULL_RANGE;
+			hdmi_info.bits.YQ0_YQ1 = YYC_QUANTIZATION_FULL_RANGE;
 		} else if (color_space == COLOR_SPACE_SRGB_LIMITED ||
 					color_space == COLOR_SPACE_2020_RGB_LIMITEDRANGE) {
-			hdmi_info->bits.Q0_Q1   = RGB_QUANTIZATION_LIMITED_RANGE;
-			hdmi_info->bits.YQ0_YQ1 = YYC_QUANTIZATION_LIMITED_RANGE;
+			hdmi_info.bits.Q0_Q1   = RGB_QUANTIZATION_LIMITED_RANGE;
+			hdmi_info.bits.YQ0_YQ1 = YYC_QUANTIZATION_LIMITED_RANGE;
 		} else {
-			hdmi_info->bits.Q0_Q1   = RGB_QUANTIZATION_DEFAULT_RANGE;
-			hdmi_info->bits.YQ0_YQ1 = YYC_QUANTIZATION_LIMITED_RANGE;
+			hdmi_info.bits.Q0_Q1   = RGB_QUANTIZATION_DEFAULT_RANGE;
+			hdmi_info.bits.YQ0_YQ1 = YYC_QUANTIZATION_LIMITED_RANGE;
 		}
 	} else {
-		hdmi_info->bits.Q0_Q1   = RGB_QUANTIZATION_DEFAULT_RANGE;
-		hdmi_info->bits.YQ0_YQ1   = YYC_QUANTIZATION_LIMITED_RANGE;
+		hdmi_info.bits.Q0_Q1   = RGB_QUANTIZATION_DEFAULT_RANGE;
+		hdmi_info.bits.YQ0_YQ1   = YYC_QUANTIZATION_LIMITED_RANGE;
 	}
 
 	///VIC
@@ -2060,51 +2066,49 @@ static void set_avi_info_frame(
 			break;
 		}
 	}
-	hdmi_info->bits.VIC0_VIC7 = vic;
+	hdmi_info.bits.VIC0_VIC7 = vic;
 
 	/* pixel repetition
 	 * PR0 - PR3 start from 0 whereas pHwPathMode->mode.timing.flags.pixel
 	 * repetition start from 1 */
-	hdmi_info->bits.PR0_PR3 = 0;
+	hdmi_info.bits.PR0_PR3 = 0;
 
 	/* Bar Info
 	 * barTop:    Line Number of End of Top Bar.
 	 * barBottom: Line Number of Start of Bottom Bar.
 	 * barLeft:   Pixel Number of End of Left Bar.
 	 * barRight:  Pixel Number of Start of Right Bar. */
-	hdmi_info->bits.bar_top = stream->timing.v_border_top;
-	hdmi_info->bits.bar_bottom = (stream->timing.v_total
+	hdmi_info.bits.bar_top = stream->timing.v_border_top;
+	hdmi_info.bits.bar_bottom = (stream->timing.v_total
 			- stream->timing.v_border_bottom + 1);
-	hdmi_info->bits.bar_left  = stream->timing.h_border_left;
-	hdmi_info->bits.bar_right = (stream->timing.h_total
+	hdmi_info.bits.bar_left  = stream->timing.h_border_left;
+	hdmi_info.bits.bar_right = (stream->timing.h_total
 			- stream->timing.h_border_right + 1);
 
 	/* check_sum - Calculate AFMT_AVI_INFO0 ~ AFMT_AVI_INFO3 */
-	check_sum = &info_frame.avi_info_packet.info_packet_hdmi.packet_raw_data.sb[0];
+	check_sum = &hdmi_info.packet_raw_data.sb[0];
 
 	*check_sum = HDMI_INFOFRAME_TYPE_AVI + HDMI_AVI_INFOFRAME_SIZE + 2;
 
 	for (byte_index = 1; byte_index <= HDMI_AVI_INFOFRAME_SIZE; byte_index++)
-		*check_sum += hdmi_info->packet_raw_data.sb[byte_index];
+		*check_sum += hdmi_info.packet_raw_data.sb[byte_index];
 
 	/* one byte complement */
 	*check_sum = (uint8_t) (0x100 - *check_sum);
 
 	/* Store in hw_path_mode */
-	info_packet->hb0 = hdmi_info->packet_raw_data.hb0;
-	info_packet->hb1 = hdmi_info->packet_raw_data.hb1;
-	info_packet->hb2 = hdmi_info->packet_raw_data.hb2;
+	info_packet->hb0 = hdmi_info.packet_raw_data.hb0;
+	info_packet->hb1 = hdmi_info.packet_raw_data.hb1;
+	info_packet->hb2 = hdmi_info.packet_raw_data.hb2;
 
-	for (byte_index = 0; byte_index < sizeof(info_frame.avi_info_packet.
-				info_packet_hdmi.packet_raw_data.sb); byte_index++)
-		info_packet->sb[byte_index] = info_frame.avi_info_packet.
-				info_packet_hdmi.packet_raw_data.sb[byte_index];
+	for (byte_index = 0; byte_index < sizeof(hdmi_info.packet_raw_data.sb); byte_index++)
+		info_packet->sb[byte_index] = hdmi_info.packet_raw_data.sb[byte_index];
 
 	info_packet->valid = true;
 }
 
 static void set_vendor_info_packet(
-		struct encoder_info_packet *info_packet,
+		struct dc_info_packet *info_packet,
 		struct dc_stream_state *stream)
 {
 	uint32_t length = 0;
@@ -2217,7 +2221,7 @@ static void set_vendor_info_packet(
 }
 
 static void set_spd_info_packet(
-		struct encoder_info_packet *info_packet,
+		struct dc_info_packet *info_packet,
 		struct dc_stream_state *stream)
 {
 	/* SPD info packet for FreeSync */
@@ -2338,104 +2342,19 @@ static void set_spd_info_packet(
 }
 
 static void set_hdr_static_info_packet(
-		struct encoder_info_packet *info_packet,
+		struct dc_info_packet *info_packet,
 		struct dc_stream_state *stream)
 {
-	uint16_t i = 0;
-	enum signal_type signal = stream->signal;
-	uint32_t data;
+	/* HDR Static Metadata info packet for HDR10 */
 
-	if (!stream->hdr_static_metadata.hdr_supported)
+	if (!stream->hdr_static_metadata.valid)
 		return;
 
-	if (dc_is_hdmi_signal(signal)) {
-		info_packet->valid = true;
-
-		info_packet->hb0 = 0x87;
-		info_packet->hb1 = 0x01;
-		info_packet->hb2 = 0x1A;
-		i = 1;
-	} else if (dc_is_dp_signal(signal)) {
-		info_packet->valid = true;
-
-		info_packet->hb0 = 0x00;
-		info_packet->hb1 = 0x87;
-		info_packet->hb2 = 0x1D;
-		info_packet->hb3 = (0x13 << 2);
-		i = 2;
-	}
-
-	data = stream->hdr_static_metadata.is_hdr;
-	info_packet->sb[i++] = data ? 0x02 : 0x00;
-	info_packet->sb[i++] = 0x00;
-
-	data = stream->hdr_static_metadata.chromaticity_green_x / 2;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	data = stream->hdr_static_metadata.chromaticity_green_y / 2;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	data = stream->hdr_static_metadata.chromaticity_blue_x / 2;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	data = stream->hdr_static_metadata.chromaticity_blue_y / 2;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	data = stream->hdr_static_metadata.chromaticity_red_x / 2;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	data = stream->hdr_static_metadata.chromaticity_red_y / 2;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	data = stream->hdr_static_metadata.chromaticity_white_point_x / 2;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	data = stream->hdr_static_metadata.chromaticity_white_point_y / 2;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	data = stream->hdr_static_metadata.max_luminance;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	data = stream->hdr_static_metadata.min_luminance;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	data = stream->hdr_static_metadata.maximum_content_light_level;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	data = stream->hdr_static_metadata.maximum_frame_average_light_level;
-	info_packet->sb[i++] = data & 0xFF;
-	info_packet->sb[i++] = (data & 0xFF00) >> 8;
-
-	if (dc_is_hdmi_signal(signal)) {
-		uint32_t checksum = 0;
-
-		checksum += info_packet->hb0;
-		checksum += info_packet->hb1;
-		checksum += info_packet->hb2;
-
-		for (i = 1; i <= info_packet->hb2; i++)
-			checksum += info_packet->sb[i];
-
-		info_packet->sb[0] = 0x100 - checksum;
-	} else if (dc_is_dp_signal(signal)) {
-		info_packet->sb[0] = 0x01;
-		info_packet->sb[1] = 0x1A;
-	}
+	*info_packet = stream->hdr_static_metadata;
 }
 
 static void set_vsc_info_packet(
-		struct encoder_info_packet *info_packet,
+		struct dc_info_packet *info_packet,
 		struct dc_stream_state *stream)
 {
 	unsigned int vscPacketRevision = 0;
@@ -2650,6 +2569,8 @@ bool pipe_need_reprogram(
 	if (is_timing_changed(pipe_ctx_old->stream, pipe_ctx->stream))
 		return true;
 
+	if (is_hdr_static_meta_changed(pipe_ctx_old->stream, pipe_ctx->stream))
+		return true;
 
 	return false;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index ce0747ed0f00..3732a1de9d6c 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -101,14 +101,16 @@ static void construct(struct dc_stream_state *stream,
 	stream->status.link = stream->sink->link;
 
 	update_stream_signal(stream);
+
+	stream->out_transfer_func = dc_create_transfer_func();
+	stream->out_transfer_func->type = TF_TYPE_BYPASS;
 }
 
 static void destruct(struct dc_stream_state *stream)
 {
 	dc_sink_release(stream->sink);
 	if (stream->out_transfer_func != NULL) {
-		dc_transfer_func_release(
-				stream->out_transfer_func);
+		dc_transfer_func_release(stream->out_transfer_func);
 		stream->out_transfer_func = NULL;
 	}
 }
@@ -176,6 +178,7 @@ bool dc_stream_set_cursor_attributes(
 	int i;
 	struct dc  *core_dc;
 	struct resource_context *res_ctx;
+	struct pipe_ctx *pipe_to_program = NULL;
 
 	if (NULL == stream) {
 		dm_error("DC: dc_stream is NULL!\n");
@@ -203,9 +206,17 @@ bool dc_stream_set_cursor_attributes(
 		if (pipe_ctx->top_pipe && pipe_ctx->plane_state != pipe_ctx->top_pipe->plane_state)
 			continue;
 
+		if (!pipe_to_program) {
+			pipe_to_program = pipe_ctx;
+			core_dc->hwss.pipe_control_lock(core_dc, pipe_to_program, true);
+		}
 
 		core_dc->hwss.set_cursor_attribute(pipe_ctx);
 	}
+
+	if (pipe_to_program)
+		core_dc->hwss.pipe_control_lock(core_dc, pipe_to_program, false);
+
 	return true;
 }
 
@@ -216,6 +227,7 @@ bool dc_stream_set_cursor_position(
 	int i;
 	struct dc  *core_dc;
 	struct resource_context *res_ctx;
+	struct pipe_ctx *pipe_to_program = NULL;
 
 	if (NULL == stream) {
 		dm_error("DC: dc_stream is NULL!\n");
@@ -241,9 +253,17 @@ bool dc_stream_set_cursor_position(
 				!pipe_ctx->plane_res.ipp)
 			continue;
 
+		if (!pipe_to_program) {
+			pipe_to_program = pipe_ctx;
+			core_dc->hwss.pipe_control_lock(core_dc, pipe_to_program, true);
+		}
+
 		core_dc->hwss.set_cursor_position(pipe_ctx);
 	}
 
+	if (pipe_to_program)
+		core_dc->hwss.pipe_control_lock(core_dc, pipe_to_program, false);
+
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
index 132eef3826e2..68a71adeb12e 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
@@ -38,6 +38,12 @@
 static void construct(struct dc_context *ctx, struct dc_plane_state *plane_state)
 {
 	plane_state->ctx = ctx;
+
+	plane_state->gamma_correction = dc_create_gamma();
+	plane_state->gamma_correction->is_identity = true;
+
+	plane_state->in_transfer_func = dc_create_transfer_func();
+	plane_state->in_transfer_func->type = TF_TYPE_BYPASS;
 }
 
 static void destruct(struct dc_plane_state *plane_state)
@@ -175,7 +181,7 @@ void dc_transfer_func_release(struct dc_transfer_func *tf)
 	kref_put(&tf->refcount, dc_transfer_func_free);
 }
 
-struct dc_transfer_func *dc_create_transfer_func(void)
+struct dc_transfer_func *dc_create_transfer_func()
 {
 	struct dc_transfer_func *tf = kvzalloc(sizeof(*tf), GFP_KERNEL);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index fa4b3c8b3bb7..9cfde0ccf4e9 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -38,7 +38,7 @@
 #include "inc/compressor.h"
 #include "dml/display_mode_lib.h"
 
-#define DC_VER "3.1.38"
+#define DC_VER "3.1.44"
 
 #define MAX_SURFACES 3
 #define MAX_STREAMS 6
@@ -75,6 +75,7 @@ struct dc_caps {
 	bool dynamic_audio;
 	bool is_apu;
 	bool dual_link_dvi;
+	bool post_blend_color_processing;
 };
 
 struct dc_dcc_surface_param {
@@ -202,6 +203,8 @@ struct dc_debug {
 	bool timing_trace;
 	bool clock_trace;
 	bool validation_trace;
+	bool bandwidth_calcs_trace;
+	int max_downscale_src_width;
 
 	/* stutter efficiency related */
 	bool disable_stutter;
@@ -238,6 +241,8 @@ struct dc_debug {
 	bool az_endpoint_mute_only;
 	bool always_use_regamma;
 	bool p010_mpo_support;
+	bool recovery_enabled;
+
 };
 struct dc_state;
 struct resource_pool;
@@ -332,20 +337,6 @@ enum {
 	TRANSFER_FUNC_POINTS = 1025
 };
 
-// Moved here from color module for linux
-enum color_transfer_func {
-	transfer_func_unknown,
-	transfer_func_srgb,
-	transfer_func_bt709,
-	transfer_func_pq2084,
-	transfer_func_pq2084_interim,
-	transfer_func_linear_0_1,
-	transfer_func_linear_0_125,
-	transfer_func_dolbyvision,
-	transfer_func_gamma_22,
-	transfer_func_gamma_26
-};
-
 struct dc_hdr_static_metadata {
 	/* display chromaticities and white point in units of 0.00001 */
 	unsigned int chromaticity_green_x;
@@ -361,9 +352,6 @@ struct dc_hdr_static_metadata {
 	uint32_t max_luminance;
 	uint32_t maximum_content_light_level;
 	uint32_t maximum_frame_average_light_level;
-
-	bool hdr_supported;
-	bool is_hdr;
 };
 
 enum dc_transfer_func_type {
@@ -419,7 +407,6 @@ union surface_update_flags {
 		/* Medium updates */
 		uint32_t dcc_change:1;
 		uint32_t color_space_change:1;
-		uint32_t input_tf_change:1;
 		uint32_t horizontal_mirror_change:1;
 		uint32_t per_pixel_alpha_change:1;
 		uint32_t rotation_change:1;
@@ -428,6 +415,7 @@ union surface_update_flags {
 		uint32_t position_change:1;
 		uint32_t in_transfer_func_change:1;
 		uint32_t input_csc_change:1;
+		uint32_t coeff_reduction_change:1;
 		uint32_t output_tf_change:1;
 		uint32_t pixel_format_change:1;
 
@@ -460,7 +448,7 @@ struct dc_plane_state {
 	struct dc_gamma *gamma_correction;
 	struct dc_transfer_func *in_transfer_func;
 	struct dc_bias_and_scale *bias_and_scale;
-	struct csc_transform input_csc_color_matrix;
+	struct dc_csc_transform input_csc_color_matrix;
 	struct fixed31_32 coeff_reduction_factor;
 	uint32_t sdr_white_level;
 
@@ -468,7 +456,6 @@ struct dc_plane_state {
 	struct dc_hdr_static_metadata hdr_static_ctx;
 
 	enum dc_color_space color_space;
-	enum color_transfer_func input_tf;
 
 	enum surface_pixel_format format;
 	enum dc_rotation_angle rotation;
@@ -498,7 +485,6 @@ struct dc_plane_info {
 	enum dc_rotation_angle rotation;
 	enum plane_stereo_format stereo_format;
 	enum dc_color_space color_space;
-	enum color_transfer_func input_tf;
 	unsigned int sdr_white_level;
 	bool horizontal_mirror;
 	bool visible;
@@ -517,19 +503,18 @@ struct dc_surface_update {
 	struct dc_plane_state *surface;
 
 	/* isr safe update parameters.  null means no updates */
-	struct dc_flip_addrs *flip_addr;
-	struct dc_plane_info *plane_info;
-	struct dc_scaling_info *scaling_info;
+	const struct dc_flip_addrs *flip_addr;
+	const struct dc_plane_info *plane_info;
+	const struct dc_scaling_info *scaling_info;
 
 	/* following updates require alloc/sleep/spin that is not isr safe,
 	 * null means no updates
 	 */
-	struct dc_gamma *gamma;
-	enum color_transfer_func color_input_tf;
-	struct dc_transfer_func *in_transfer_func;
+	const struct dc_gamma *gamma;
+	const struct dc_transfer_func *in_transfer_func;
 
-	struct csc_transform *input_csc_color_matrix;
-	struct fixed31_32 *coeff_reduction_factor;
+	const struct dc_csc_transform *input_csc_color_matrix;
+	const struct fixed31_32 *coeff_reduction_factor;
 };
 
 /*
@@ -699,6 +684,7 @@ struct dc_cursor {
 	struct dc_cursor_attributes attributes;
 };
 
+
 /*******************************************************************************
  * Interrupt interfaces
  ******************************************************************************/
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
index 2726b02e006b..90bccd5ccaa2 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
@@ -26,6 +26,8 @@
 #ifndef DC_DP_TYPES_H
 #define DC_DP_TYPES_H
 
+#include "os_types.h"
+
 enum dc_lane_count {
 	LANE_COUNT_UNKNOWN = 0,
 	LANE_COUNT_ONE = 1,
diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c
index 48e1fcf53d43..bd0fda0ceb91 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c
@@ -117,6 +117,65 @@ uint32_t generic_reg_get5(const struct dc_context *ctx, uint32_t addr,
 	return reg_val;
 }
 
+uint32_t generic_reg_get6(const struct dc_context *ctx, uint32_t addr,
+		uint8_t shift1, uint32_t mask1, uint32_t *field_value1,
+		uint8_t shift2, uint32_t mask2, uint32_t *field_value2,
+		uint8_t shift3, uint32_t mask3, uint32_t *field_value3,
+		uint8_t shift4, uint32_t mask4, uint32_t *field_value4,
+		uint8_t shift5, uint32_t mask5, uint32_t *field_value5,
+		uint8_t shift6, uint32_t mask6, uint32_t *field_value6)
+{
+	uint32_t reg_val = dm_read_reg(ctx, addr);
+	*field_value1 = get_reg_field_value_ex(reg_val, mask1, shift1);
+	*field_value2 = get_reg_field_value_ex(reg_val, mask2, shift2);
+	*field_value3 = get_reg_field_value_ex(reg_val, mask3, shift3);
+	*field_value4 = get_reg_field_value_ex(reg_val, mask4, shift4);
+	*field_value5 = get_reg_field_value_ex(reg_val, mask5, shift5);
+	*field_value6 = get_reg_field_value_ex(reg_val, mask6, shift6);
+	return reg_val;
+}
+
+uint32_t generic_reg_get7(const struct dc_context *ctx, uint32_t addr,
+		uint8_t shift1, uint32_t mask1, uint32_t *field_value1,
+		uint8_t shift2, uint32_t mask2, uint32_t *field_value2,
+		uint8_t shift3, uint32_t mask3, uint32_t *field_value3,
+		uint8_t shift4, uint32_t mask4, uint32_t *field_value4,
+		uint8_t shift5, uint32_t mask5, uint32_t *field_value5,
+		uint8_t shift6, uint32_t mask6, uint32_t *field_value6,
+		uint8_t shift7, uint32_t mask7, uint32_t *field_value7)
+{
+	uint32_t reg_val = dm_read_reg(ctx, addr);
+	*field_value1 = get_reg_field_value_ex(reg_val, mask1, shift1);
+	*field_value2 = get_reg_field_value_ex(reg_val, mask2, shift2);
+	*field_value3 = get_reg_field_value_ex(reg_val, mask3, shift3);
+	*field_value4 = get_reg_field_value_ex(reg_val, mask4, shift4);
+	*field_value5 = get_reg_field_value_ex(reg_val, mask5, shift5);
+	*field_value6 = get_reg_field_value_ex(reg_val, mask6, shift6);
+	*field_value7 = get_reg_field_value_ex(reg_val, mask7, shift7);
+	return reg_val;
+}
+
+uint32_t generic_reg_get8(const struct dc_context *ctx, uint32_t addr,
+		uint8_t shift1, uint32_t mask1, uint32_t *field_value1,
+		uint8_t shift2, uint32_t mask2, uint32_t *field_value2,
+		uint8_t shift3, uint32_t mask3, uint32_t *field_value3,
+		uint8_t shift4, uint32_t mask4, uint32_t *field_value4,
+		uint8_t shift5, uint32_t mask5, uint32_t *field_value5,
+		uint8_t shift6, uint32_t mask6, uint32_t *field_value6,
+		uint8_t shift7, uint32_t mask7, uint32_t *field_value7,
+		uint8_t shift8, uint32_t mask8, uint32_t *field_value8)
+{
+	uint32_t reg_val = dm_read_reg(ctx, addr);
+	*field_value1 = get_reg_field_value_ex(reg_val, mask1, shift1);
+	*field_value2 = get_reg_field_value_ex(reg_val, mask2, shift2);
+	*field_value3 = get_reg_field_value_ex(reg_val, mask3, shift3);
+	*field_value4 = get_reg_field_value_ex(reg_val, mask4, shift4);
+	*field_value5 = get_reg_field_value_ex(reg_val, mask5, shift5);
+	*field_value6 = get_reg_field_value_ex(reg_val, mask6, shift6);
+	*field_value7 = get_reg_field_value_ex(reg_val, mask7, shift7);
+	*field_value8 = get_reg_field_value_ex(reg_val, mask8, shift8);
+	return reg_val;
+}
 /* note:  va version of this is pretty bad idea, since there is a output parameter pass by pointer
  * compiler won't be able to check for size match and is prone to stack corruption type of bugs
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
index b83a7dc2f5a9..b1f70579d61b 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
@@ -423,6 +423,11 @@ enum dc_gamma_type {
 	GAMMA_CS_TFM_1D = 3,
 };
 
+struct dc_csc_transform {
+	uint16_t matrix[12];
+	bool enable_adjustment;
+};
+
 struct dc_gamma {
 	struct kref refcount;
 	enum dc_gamma_type type;
diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h
index dc34515ef01f..8a716baa1203 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_link.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_link.h
@@ -51,6 +51,14 @@ struct link_mst_stream_allocation_table {
 	struct link_mst_stream_allocation stream_allocations[MAX_CONTROLLER_NUM];
 };
 
+struct time_stamp {
+	uint64_t edp_poweroff;
+	uint64_t edp_poweron;
+};
+
+struct link_trace {
+	struct time_stamp time_stamp;
+};
 /*
  * A link contains one or more sinks and their connected status.
  * The currently active signal type (HDMI, DP-SST, DP-MST) is also reported.
@@ -114,6 +122,7 @@ struct dc_link {
 
 	struct dc_link_status link_status;
 
+	struct link_trace link_trace;
 };
 
 const struct dc_link_status *dc_link_get_status(const struct dc_link *dc_link);
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index d017df56b2ba..d7e6d53bb383 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -58,18 +58,20 @@ struct dc_stream_state {
 
 	struct freesync_context freesync_ctx;
 
-	struct dc_hdr_static_metadata hdr_static_metadata;
+	struct dc_info_packet hdr_static_metadata;
 	struct dc_transfer_func *out_transfer_func;
 	struct colorspace_transform gamut_remap_matrix;
-	struct csc_transform csc_color_matrix;
+	struct dc_csc_transform csc_color_matrix;
 
 	enum dc_color_space output_color_space;
 	enum dc_dither_option dither_option;
 
 	enum view_3d_format view_format;
-	enum color_transfer_func output_tf;
 
 	bool ignore_msa_timing_param;
+
+	unsigned long long periodic_fn_vsync_delta;
+
 	/* TODO: custom INFO packets */
 	/* TODO: ABM info (DMCU) */
 	/* PSR info */
@@ -110,9 +112,10 @@ struct dc_stream_update {
 	struct rect src;
 	struct rect dst;
 	struct dc_transfer_func *out_transfer_func;
-	struct dc_hdr_static_metadata *hdr_static_metadata;
-	enum color_transfer_func color_output_tf;
+	struct dc_info_packet *hdr_static_metadata;
 	unsigned int *abm_level;
+
+	unsigned long long *periodic_fn_vsync_delta;
 };
 
 bool dc_is_stream_unchanged(
@@ -131,13 +134,6 @@ bool dc_is_stream_scaling_unchanged(
  *   This does not trigger a flip.  No surface address is programmed.
  */
 
-bool dc_commit_planes_to_stream(
-		struct dc *dc,
-		struct dc_plane_state **plane_states,
-		uint8_t new_plane_count,
-		struct dc_stream_state *dc_stream,
-		struct dc_state *state);
-
 void dc_commit_updates_for_stream(struct dc *dc,
 		struct dc_surface_update *srf_updates,
 		int surface_count,
@@ -208,14 +204,6 @@ bool dc_add_all_planes_for_stream(
 
 enum dc_status dc_validate_stream(struct dc *dc, struct dc_stream_state *stream);
 
-/*
- * This function takes a stream and checks if it is guaranteed to be supported.
- * Guaranteed means that MAX_COFUNC similar streams are supported.
- *
- * After this call:
- *   No hardware is programmed for call.  Only validation is done.
- */
-
 /*
  * Set up streams and links associated to drive sinks
  * The streams parameter is an absolute set of all active streams.
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index 9441305d3ab5..76df2534c4a4 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -25,7 +25,7 @@
 #ifndef DC_TYPES_H_
 #define DC_TYPES_H_
 
-#include "fixed32_32.h"
+#include "os_types.h"
 #include "fixed31_32.h"
 #include "irq_types.h"
 #include "dc_dp_types.h"
@@ -370,12 +370,6 @@ struct dc_csc_adjustments {
 	struct fixed31_32 hue;
 };
 
-enum {
-	MAX_LANES = 2,
-	MAX_COFUNC_PATH = 6,
-	LAYER_INDEX_PRIMARY = -1,
-};
-
 enum dpcd_downstream_port_max_bpc {
 	DOWN_STREAM_MAX_8BPC = 0,
 	DOWN_STREAM_MAX_10BPC,
@@ -530,6 +524,15 @@ struct vrr_params {
 	uint32_t frame_counter;
 };
 
+struct dc_info_packet {
+	bool valid;
+	uint8_t hb0;
+	uint8_t hb1;
+	uint8_t hb2;
+	uint8_t hb3;
+	uint8_t sb[32];
+};
+
 #define DC_PLANE_UPDATE_TIMES_MAX 10
 
 struct dc_plane_flip_time {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
index fe92a1222803..29294db1a96b 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
@@ -26,7 +26,7 @@
 #include "dce_abm.h"
 #include "dm_services.h"
 #include "reg_helper.h"
-#include "fixed32_32.h"
+#include "fixed31_32.h"
 #include "dc.h"
 
 #include "atom.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
index 6d5cdcdc8ec9..7f6d724686f1 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
@@ -33,8 +33,9 @@
 
 #define CTX \
 	aud->base.ctx
-#define DC_LOGGER \
-	aud->base.ctx->logger
+
+#define DC_LOGGER_INIT()
+
 #define REG(reg)\
 	(aud->regs->reg)
 
@@ -348,8 +349,8 @@ static void set_audio_latency(
 
 void dce_aud_az_enable(struct audio *audio)
 {
-	struct dce_audio *aud = DCE_AUD(audio);
 	uint32_t value = AZ_REG_READ(AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL);
+	DC_LOGGER_INIT();
 
 	set_reg_field_value(value, 1,
 			    AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL,
@@ -371,7 +372,7 @@ void dce_aud_az_enable(struct audio *audio)
 void dce_aud_az_disable(struct audio *audio)
 {
 	uint32_t value;
-	struct dce_audio *aud = DCE_AUD(audio);
+	DC_LOGGER_INIT();
 
 	value = AZ_REG_READ(AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL);
 	set_reg_field_value(value, 1,
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
index 0aa2cda60890..599c7ab6befe 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
@@ -41,8 +41,9 @@
 
 #define CTX \
 	clk_src->base.ctx
-#define DC_LOGGER \
-	calc_pll_cs->ctx->logger
+
+#define DC_LOGGER_INIT()
+
 #undef FN
 #define FN(reg_name, field_name) \
 	clk_src->cs_shift->field_name, clk_src->cs_mask->field_name
@@ -467,7 +468,7 @@ static uint32_t dce110_get_pix_clk_dividers_helper (
 {
 	uint32_t field = 0;
 	uint32_t pll_calc_error = MAX_PLL_CALC_ERROR;
-	struct calc_pll_clock_source *calc_pll_cs = &clk_src->calc_pll;
+	DC_LOGGER_INIT();
 	/* Check if reference clock is external (not pcie/xtalin)
 	* HW Dce80 spec:
 	* 00 - PCIE_REFCLK, 01 - XTALIN,    02 - GENERICA,    03 - GENERICB
@@ -557,8 +558,8 @@ static uint32_t dce110_get_pix_clk_dividers(
 		struct pll_settings *pll_settings)
 {
 	struct dce110_clk_src *clk_src = TO_DCE110_CLK_SRC(cs);
-	struct calc_pll_clock_source *calc_pll_cs = &clk_src->calc_pll;
 	uint32_t pll_calc_error = MAX_PLL_CALC_ERROR;
+	DC_LOGGER_INIT();
 
 	if (pix_clk_params == NULL || pll_settings == NULL
 			|| pix_clk_params->requested_pix_clk == 0) {
@@ -589,6 +590,7 @@ static uint32_t dce110_get_pix_clk_dividers(
 			pll_settings, pix_clk_params);
 		break;
 	case DCE_VERSION_11_2:
+	case DCE_VERSION_11_22:
 	case DCE_VERSION_12_0:
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 	case DCN_VERSION_1_0:
@@ -655,12 +657,12 @@ static uint32_t dce110_get_d_to_pixel_rate_in_hz(
 			return 0;
 		}
 
-		pix_rate = dal_fixed31_32_from_int(clk_src->ref_freq_khz);
-		pix_rate = dal_fixed31_32_mul_int(pix_rate, 1000);
-		pix_rate = dal_fixed31_32_mul_int(pix_rate, phase);
-		pix_rate = dal_fixed31_32_div_int(pix_rate, modulo);
+		pix_rate = dc_fixpt_from_int(clk_src->ref_freq_khz);
+		pix_rate = dc_fixpt_mul_int(pix_rate, 1000);
+		pix_rate = dc_fixpt_mul_int(pix_rate, phase);
+		pix_rate = dc_fixpt_div_int(pix_rate, modulo);
 
-		return dal_fixed31_32_round(pix_rate);
+		return dc_fixpt_round(pix_rate);
 	} else {
 		return dce110_get_dp_pixel_rate_from_combo_phy_pll(cs, pix_clk_params, pll_settings);
 	}
@@ -709,12 +711,12 @@ static bool calculate_ss(
 		const struct spread_spectrum_data *ss_data,
 		struct delta_sigma_data *ds_data)
 {
-	struct fixed32_32 fb_div;
-	struct fixed32_32 ss_amount;
-	struct fixed32_32 ss_nslip_amount;
-	struct fixed32_32 ss_ds_frac_amount;
-	struct fixed32_32 ss_step_size;
-	struct fixed32_32 modulation_time;
+	struct fixed31_32 fb_div;
+	struct fixed31_32 ss_amount;
+	struct fixed31_32 ss_nslip_amount;
+	struct fixed31_32 ss_ds_frac_amount;
+	struct fixed31_32 ss_step_size;
+	struct fixed31_32 modulation_time;
 
 	if (ds_data == NULL)
 		return false;
@@ -729,42 +731,42 @@ static bool calculate_ss(
 
 	/* compute SS_AMOUNT_FBDIV & SS_AMOUNT_NFRAC_SLIP & SS_AMOUNT_DSFRAC*/
 	/* 6 decimal point support in fractional feedback divider */
-	fb_div  = dal_fixed32_32_from_fraction(
+	fb_div  = dc_fixpt_from_fraction(
 		pll_settings->fract_feedback_divider, 1000000);
-	fb_div = dal_fixed32_32_add_int(fb_div, pll_settings->feedback_divider);
+	fb_div = dc_fixpt_add_int(fb_div, pll_settings->feedback_divider);
 
 	ds_data->ds_frac_amount = 0;
 	/*spreadSpectrumPercentage is in the unit of .01%,
 	 * so have to divided by 100 * 100*/
-	ss_amount = dal_fixed32_32_mul(
-		fb_div, dal_fixed32_32_from_fraction(ss_data->percentage,
+	ss_amount = dc_fixpt_mul(
+		fb_div, dc_fixpt_from_fraction(ss_data->percentage,
 					100 * ss_data->percentage_divider));
-	ds_data->feedback_amount = dal_fixed32_32_floor(ss_amount);
+	ds_data->feedback_amount = dc_fixpt_floor(ss_amount);
 
-	ss_nslip_amount = dal_fixed32_32_sub(ss_amount,
-		dal_fixed32_32_from_int(ds_data->feedback_amount));
-	ss_nslip_amount = dal_fixed32_32_mul_int(ss_nslip_amount, 10);
-	ds_data->nfrac_amount = dal_fixed32_32_floor(ss_nslip_amount);
+	ss_nslip_amount = dc_fixpt_sub(ss_amount,
+		dc_fixpt_from_int(ds_data->feedback_amount));
+	ss_nslip_amount = dc_fixpt_mul_int(ss_nslip_amount, 10);
+	ds_data->nfrac_amount = dc_fixpt_floor(ss_nslip_amount);
 
-	ss_ds_frac_amount = dal_fixed32_32_sub(ss_nslip_amount,
-		dal_fixed32_32_from_int(ds_data->nfrac_amount));
-	ss_ds_frac_amount = dal_fixed32_32_mul_int(ss_ds_frac_amount, 65536);
-	ds_data->ds_frac_amount = dal_fixed32_32_floor(ss_ds_frac_amount);
+	ss_ds_frac_amount = dc_fixpt_sub(ss_nslip_amount,
+		dc_fixpt_from_int(ds_data->nfrac_amount));
+	ss_ds_frac_amount = dc_fixpt_mul_int(ss_ds_frac_amount, 65536);
+	ds_data->ds_frac_amount = dc_fixpt_floor(ss_ds_frac_amount);
 
 	/* compute SS_STEP_SIZE_DSFRAC */
-	modulation_time = dal_fixed32_32_from_fraction(
+	modulation_time = dc_fixpt_from_fraction(
 		pll_settings->reference_freq * 1000,
 		pll_settings->reference_divider * ss_data->modulation_freq_hz);
 
 	if (ss_data->flags.CENTER_SPREAD)
-		modulation_time = dal_fixed32_32_div_int(modulation_time, 4);
+		modulation_time = dc_fixpt_div_int(modulation_time, 4);
 	else
-		modulation_time = dal_fixed32_32_div_int(modulation_time, 2);
+		modulation_time = dc_fixpt_div_int(modulation_time, 2);
 
-	ss_step_size = dal_fixed32_32_div(ss_amount, modulation_time);
+	ss_step_size = dc_fixpt_div(ss_amount, modulation_time);
 	/* SS_STEP_SIZE_DSFRAC_DEC = Int(SS_STEP_SIZE * 2 ^ 16 * 10)*/
-	ss_step_size = dal_fixed32_32_mul_int(ss_step_size, 65536 * 10);
-	ds_data->ds_frac_size =  dal_fixed32_32_floor(ss_step_size);
+	ss_step_size = dc_fixpt_mul_int(ss_step_size, 65536 * 10);
+	ds_data->ds_frac_size =  dc_fixpt_floor(ss_step_size);
 
 	return true;
 }
@@ -978,6 +980,7 @@ static bool dce110_program_pix_clk(
 
 		break;
 	case DCE_VERSION_11_2:
+	case DCE_VERSION_11_22:
 	case DCE_VERSION_12_0:
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 	case DCN_VERSION_1_0:
@@ -1054,7 +1057,7 @@ static void get_ss_info_from_atombios(
 	struct spread_spectrum_info *ss_info_cur;
 	struct spread_spectrum_data *ss_data_cur;
 	uint32_t i;
-	struct calc_pll_clock_source *calc_pll_cs = &clk_src->calc_pll;
+	DC_LOGGER_INIT();
 	if (ss_entries_num == NULL) {
 		DC_LOG_SYNC(
 			"Invalid entry !!!\n");
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c
index 78e6beb6cf26..8a581c67bf2d 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c
@@ -26,7 +26,7 @@
 #include "dce_clocks.h"
 #include "dm_services.h"
 #include "reg_helper.h"
-#include "fixed32_32.h"
+#include "fixed31_32.h"
 #include "bios_parser_interface.h"
 #include "dc.h"
 #include "dmcu.h"
@@ -35,7 +35,7 @@
 #endif
 #include "core_types.h"
 #include "dc_types.h"
-
+#include "dal_asic_id.h"
 
 #define TO_DCE_CLOCKS(clocks)\
 	container_of(clocks, struct dce_disp_clk, base)
@@ -228,19 +228,19 @@ static int dce_clocks_get_dp_ref_freq(struct display_clock *clk)
 	 generated according to average value (case as with previous ASICs)
 	  */
 	if (clk_dce->ss_on_dprefclk && clk_dce->dprefclk_ss_divider != 0) {
-		struct fixed32_32 ss_percentage = dal_fixed32_32_div_int(
-				dal_fixed32_32_from_fraction(
+		struct fixed31_32 ss_percentage = dc_fixpt_div_int(
+				dc_fixpt_from_fraction(
 						clk_dce->dprefclk_ss_percentage,
 						clk_dce->dprefclk_ss_divider), 200);
-		struct fixed32_32 adj_dp_ref_clk_khz;
+		struct fixed31_32 adj_dp_ref_clk_khz;
 
-		ss_percentage = dal_fixed32_32_sub(dal_fixed32_32_one,
+		ss_percentage = dc_fixpt_sub(dc_fixpt_one,
 								ss_percentage);
 		adj_dp_ref_clk_khz =
-			dal_fixed32_32_mul_int(
+			dc_fixpt_mul_int(
 				ss_percentage,
 				dp_ref_clk_khz);
-		dp_ref_clk_khz = dal_fixed32_32_floor(adj_dp_ref_clk_khz);
+		dp_ref_clk_khz = dc_fixpt_floor(adj_dp_ref_clk_khz);
 	}
 
 	return dp_ref_clk_khz;
@@ -256,19 +256,19 @@ static int dce_clocks_get_dp_ref_freq_wrkaround(struct display_clock *clk)
 	int dp_ref_clk_khz = 600000;
 
 	if (clk_dce->ss_on_dprefclk && clk_dce->dprefclk_ss_divider != 0) {
-		struct fixed32_32 ss_percentage = dal_fixed32_32_div_int(
-				dal_fixed32_32_from_fraction(
+		struct fixed31_32 ss_percentage = dc_fixpt_div_int(
+				dc_fixpt_from_fraction(
 						clk_dce->dprefclk_ss_percentage,
 						clk_dce->dprefclk_ss_divider), 200);
-		struct fixed32_32 adj_dp_ref_clk_khz;
+		struct fixed31_32 adj_dp_ref_clk_khz;
 
-		ss_percentage = dal_fixed32_32_sub(dal_fixed32_32_one,
+		ss_percentage = dc_fixpt_sub(dc_fixpt_one,
 								ss_percentage);
 		adj_dp_ref_clk_khz =
-			dal_fixed32_32_mul_int(
+			dc_fixpt_mul_int(
 				ss_percentage,
 				dp_ref_clk_khz);
-		dp_ref_clk_khz = dal_fixed32_32_floor(adj_dp_ref_clk_khz);
+		dp_ref_clk_khz = dc_fixpt_floor(adj_dp_ref_clk_khz);
 	}
 
 	return dp_ref_clk_khz;
@@ -413,9 +413,12 @@ static int dce112_set_clock(
 	/*VBIOS will determine DPREFCLK frequency, so we don't set it*/
 	dce_clk_params.target_clock_frequency = 0;
 	dce_clk_params.clock_type = DCECLOCK_TYPE_DPREFCLK;
-	dce_clk_params.flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK =
+	if (!ASICREV_IS_VEGA20_P(clk->ctx->asic_id.hw_internal_rev))
+		dce_clk_params.flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK =
 			(dce_clk_params.pll_id ==
 					CLOCK_SOURCE_COMBO_DISPLAY_PLL0);
+	else
+		dce_clk_params.flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK = false;
 
 	bp->funcs->set_dce_clock(bp, &dce_clk_params);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
index 2ee3d9bf1062..a576b8bbb3cd 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
@@ -28,7 +28,7 @@
 #include "dce_dmcu.h"
 #include "dm_services.h"
 #include "reg_helper.h"
-#include "fixed32_32.h"
+#include "fixed31_32.h"
 #include "dc.h"
 
 #define TO_DCE_DMCU(dmcu)\
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.c b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.c
index 487724345d9d..0275d6d60da4 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.c
@@ -53,7 +53,8 @@ void dce_pipe_control_lock(struct dc *dc,
 	struct dce_hwseq *hws = dc->hwseq;
 
 	/* Not lock pipe when blank */
-	if (lock && pipe->stream_res.tg->funcs->is_blanked(pipe->stream_res.tg))
+	if (lock && pipe->stream_res.tg->funcs->is_blanked &&
+	    pipe->stream_res.tg->funcs->is_blanked(pipe->stream_res.tg))
 		return;
 
 	val = REG_GET_4(BLND_V_UPDATE_LOCK[pipe->stream_res.tg->inst],
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
index d737e911971b..5d9506b3d46b 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
@@ -195,13 +195,13 @@ static void dce_ipp_program_input_lut(
 
 	for (i = 0; i < gamma->num_entries; i++) {
 		REG_SET(DC_LUT_SEQ_COLOR, 0, DC_LUT_SEQ_COLOR,
-				dal_fixed31_32_round(
+				dc_fixpt_round(
 					gamma->entries.red[i]));
 		REG_SET(DC_LUT_SEQ_COLOR, 0, DC_LUT_SEQ_COLOR,
-				dal_fixed31_32_round(
+				dc_fixpt_round(
 					gamma->entries.green[i]));
 		REG_SET(DC_LUT_SEQ_COLOR, 0, DC_LUT_SEQ_COLOR,
-				dal_fixed31_32_round(
+				dc_fixpt_round(
 					gamma->entries.blue[i]));
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
index 8167cad7bcf7..dbe3b26b6d9e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
@@ -113,6 +113,7 @@ static const struct link_encoder_funcs dce110_lnk_enc_funcs = {
 	.connect_dig_be_to_fe = dce110_link_encoder_connect_dig_be_to_fe,
 	.enable_hpd = dce110_link_encoder_enable_hpd,
 	.disable_hpd = dce110_link_encoder_disable_hpd,
+	.is_dig_enabled = dce110_is_dig_enabled,
 	.destroy = dce110_link_encoder_destroy
 };
 
@@ -535,8 +536,9 @@ void dce110_psr_program_secondary_packet(struct link_encoder *enc,
 		DP_SEC_GSP0_PRIORITY, 1);
 }
 
-static bool is_dig_enabled(const struct dce110_link_encoder *enc110)
+bool dce110_is_dig_enabled(struct link_encoder *enc)
 {
+	struct dce110_link_encoder *enc110 = TO_DCE110_LINK_ENC(enc);
 	uint32_t value;
 
 	REG_GET(DIG_BE_EN_CNTL, DIG_ENABLE, &value);
@@ -1031,7 +1033,7 @@ void dce110_link_encoder_disable_output(
 	struct bp_transmitter_control cntl = { 0 };
 	enum bp_result result;
 
-	if (!is_dig_enabled(enc110)) {
+	if (!dce110_is_dig_enabled(enc)) {
 		/* OF_SKIP_POWER_DOWN_INACTIVE_ENCODER */
 		return;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h
index 0ec3433d34b6..347069461a22 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h
@@ -263,4 +263,6 @@ void dce110_psr_program_dp_dphy_fast_training(struct link_encoder *enc,
 void dce110_psr_program_secondary_packet(struct link_encoder *enc,
 			unsigned int sdp_transmit_line_num_deadline);
 
+bool dce110_is_dig_enabled(struct link_encoder *enc);
+
 #endif /* __DC_LINK_ENCODER__DCE110_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c
index 0790f25c7b3b..b235a75355b8 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c
@@ -174,6 +174,25 @@ static void program_urgency_watermark(
 		URGENCY_HIGH_WATERMARK, urgency_high_wm);
 }
 
+static void dce120_program_urgency_watermark(
+	struct dce_mem_input *dce_mi,
+	uint32_t wm_select,
+	uint32_t urgency_low_wm,
+	uint32_t urgency_high_wm)
+{
+	REG_UPDATE(DPG_WATERMARK_MASK_CONTROL,
+		URGENCY_WATERMARK_MASK, wm_select);
+
+	REG_SET_2(DPG_PIPE_URGENCY_CONTROL, 0,
+		URGENCY_LOW_WATERMARK, urgency_low_wm,
+		URGENCY_HIGH_WATERMARK, urgency_high_wm);
+
+	REG_SET_2(DPG_PIPE_URGENT_LEVEL_CONTROL, 0,
+		URGENT_LEVEL_LOW_WATERMARK, urgency_low_wm,
+		URGENT_LEVEL_HIGH_WATERMARK, urgency_high_wm);
+
+}
+
 static void program_nbp_watermark(
 	struct dce_mem_input *dce_mi,
 	uint32_t wm_select,
@@ -206,6 +225,25 @@ static void program_nbp_watermark(
 	}
 }
 
+static void dce120_program_stutter_watermark(
+	struct dce_mem_input *dce_mi,
+	uint32_t wm_select,
+	uint32_t stutter_mark,
+	uint32_t stutter_entry)
+{
+	REG_UPDATE(DPG_WATERMARK_MASK_CONTROL,
+		STUTTER_EXIT_SELF_REFRESH_WATERMARK_MASK, wm_select);
+
+	if (REG(DPG_PIPE_STUTTER_CONTROL2))
+		REG_UPDATE_2(DPG_PIPE_STUTTER_CONTROL2,
+				STUTTER_EXIT_SELF_REFRESH_WATERMARK, stutter_mark,
+				STUTTER_ENTER_SELF_REFRESH_WATERMARK, stutter_entry);
+	else
+		REG_UPDATE_2(DPG_PIPE_STUTTER_CONTROL,
+				STUTTER_EXIT_SELF_REFRESH_WATERMARK, stutter_mark,
+				STUTTER_ENTER_SELF_REFRESH_WATERMARK, stutter_entry);
+}
+
 static void program_stutter_watermark(
 	struct dce_mem_input *dce_mi,
 	uint32_t wm_select,
@@ -225,7 +263,8 @@ static void program_stutter_watermark(
 static void dce_mi_program_display_marks(
 	struct mem_input *mi,
 	struct dce_watermarks nbp,
-	struct dce_watermarks stutter,
+	struct dce_watermarks stutter_exit,
+	struct dce_watermarks stutter_enter,
 	struct dce_watermarks urgent,
 	uint32_t total_dest_line_time_ns)
 {
@@ -243,13 +282,14 @@ static void dce_mi_program_display_marks(
 	program_nbp_watermark(dce_mi, 2, nbp.a_mark); /* set a */
 	program_nbp_watermark(dce_mi, 1, nbp.d_mark); /* set d */
 
-	program_stutter_watermark(dce_mi, 2, stutter.a_mark); /* set a */
-	program_stutter_watermark(dce_mi, 1, stutter.d_mark); /* set d */
+	program_stutter_watermark(dce_mi, 2, stutter_exit.a_mark); /* set a */
+	program_stutter_watermark(dce_mi, 1, stutter_exit.d_mark); /* set d */
 }
 
-static void dce120_mi_program_display_marks(struct mem_input *mi,
+static void dce112_mi_program_display_marks(struct mem_input *mi,
 	struct dce_watermarks nbp,
-	struct dce_watermarks stutter,
+	struct dce_watermarks stutter_exit,
+	struct dce_watermarks stutter_entry,
 	struct dce_watermarks urgent,
 	uint32_t total_dest_line_time_ns)
 {
@@ -273,10 +313,43 @@ static void dce120_mi_program_display_marks(struct mem_input *mi,
 	program_nbp_watermark(dce_mi, 2, nbp.c_mark); /* set c */
 	program_nbp_watermark(dce_mi, 3, nbp.d_mark); /* set d */
 
-	program_stutter_watermark(dce_mi, 0, stutter.a_mark); /* set a */
-	program_stutter_watermark(dce_mi, 1, stutter.b_mark); /* set b */
-	program_stutter_watermark(dce_mi, 2, stutter.c_mark); /* set c */
-	program_stutter_watermark(dce_mi, 3, stutter.d_mark); /* set d */
+	program_stutter_watermark(dce_mi, 0, stutter_exit.a_mark); /* set a */
+	program_stutter_watermark(dce_mi, 1, stutter_exit.b_mark); /* set b */
+	program_stutter_watermark(dce_mi, 2, stutter_exit.c_mark); /* set c */
+	program_stutter_watermark(dce_mi, 3, stutter_exit.d_mark); /* set d */
+}
+
+static void dce120_mi_program_display_marks(struct mem_input *mi,
+	struct dce_watermarks nbp,
+	struct dce_watermarks stutter_exit,
+	struct dce_watermarks stutter_entry,
+	struct dce_watermarks urgent,
+	uint32_t total_dest_line_time_ns)
+{
+	struct dce_mem_input *dce_mi = TO_DCE_MEM_INPUT(mi);
+	uint32_t stutter_en = mi->ctx->dc->debug.disable_stutter ? 0 : 1;
+
+	dce120_program_urgency_watermark(dce_mi, 0, /* set a */
+			urgent.a_mark, total_dest_line_time_ns);
+	dce120_program_urgency_watermark(dce_mi, 1, /* set b */
+			urgent.b_mark, total_dest_line_time_ns);
+	dce120_program_urgency_watermark(dce_mi, 2, /* set c */
+			urgent.c_mark, total_dest_line_time_ns);
+	dce120_program_urgency_watermark(dce_mi, 3, /* set d */
+			urgent.d_mark, total_dest_line_time_ns);
+
+	REG_UPDATE_2(DPG_PIPE_STUTTER_CONTROL,
+		STUTTER_ENABLE, stutter_en,
+		STUTTER_IGNORE_FBC, 1);
+	program_nbp_watermark(dce_mi, 0, nbp.a_mark); /* set a */
+	program_nbp_watermark(dce_mi, 1, nbp.b_mark); /* set b */
+	program_nbp_watermark(dce_mi, 2, nbp.c_mark); /* set c */
+	program_nbp_watermark(dce_mi, 3, nbp.d_mark); /* set d */
+
+	dce120_program_stutter_watermark(dce_mi, 0, stutter_exit.a_mark, stutter_entry.a_mark); /* set a */
+	dce120_program_stutter_watermark(dce_mi, 1, stutter_exit.b_mark, stutter_entry.b_mark); /* set b */
+	dce120_program_stutter_watermark(dce_mi, 2, stutter_exit.c_mark, stutter_entry.c_mark); /* set c */
+	dce120_program_stutter_watermark(dce_mi, 3, stutter_exit.d_mark, stutter_entry.d_mark); /* set d */
 }
 
 static void program_tiling(
@@ -694,6 +767,18 @@ void dce112_mem_input_construct(
 	const struct dce_mem_input_registers *regs,
 	const struct dce_mem_input_shift *mi_shift,
 	const struct dce_mem_input_mask *mi_mask)
+{
+	dce_mem_input_construct(dce_mi, ctx, inst, regs, mi_shift, mi_mask);
+	dce_mi->base.funcs->mem_input_program_display_marks = dce112_mi_program_display_marks;
+}
+
+void dce120_mem_input_construct(
+	struct dce_mem_input *dce_mi,
+	struct dc_context *ctx,
+	int inst,
+	const struct dce_mem_input_registers *regs,
+	const struct dce_mem_input_shift *mi_shift,
+	const struct dce_mem_input_mask *mi_mask)
 {
 	dce_mem_input_construct(dce_mi, ctx, inst, regs, mi_shift, mi_mask);
 	dce_mi->base.funcs->mem_input_program_display_marks = dce120_mi_program_display_marks;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h
index 05d39c0cbe87..d15b0d7f47fc 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h
@@ -106,6 +106,7 @@ struct dce_mem_input_registers {
 	uint32_t DPG_PIPE_ARBITRATION_CONTROL1;
 	uint32_t DPG_WATERMARK_MASK_CONTROL;
 	uint32_t DPG_PIPE_URGENCY_CONTROL;
+	uint32_t DPG_PIPE_URGENT_LEVEL_CONTROL;
 	uint32_t DPG_PIPE_NB_PSTATE_CHANGE_CONTROL;
 	uint32_t DPG_PIPE_LOW_POWER_CONTROL;
 	uint32_t DPG_PIPE_STUTTER_CONTROL;
@@ -213,6 +214,11 @@ struct dce_mem_input_registers {
 
 #define MI_DCE12_DMIF_PG_MASK_SH_LIST(mask_sh, blk)\
 	SFB(blk, DPG_PIPE_STUTTER_CONTROL2, STUTTER_EXIT_SELF_REFRESH_WATERMARK, mask_sh),\
+	SFB(blk, DPG_PIPE_STUTTER_CONTROL2, STUTTER_ENTER_SELF_REFRESH_WATERMARK, mask_sh),\
+	SFB(blk, DPG_PIPE_URGENT_LEVEL_CONTROL, URGENT_LEVEL_LOW_WATERMARK, mask_sh),\
+	SFB(blk, DPG_PIPE_URGENT_LEVEL_CONTROL, URGENT_LEVEL_HIGH_WATERMARK, mask_sh),\
+	SFB(blk, DPG_PIPE_URGENCY_CONTROL, URGENCY_LOW_WATERMARK, mask_sh),\
+	SFB(blk, DPG_PIPE_URGENCY_CONTROL, URGENCY_HIGH_WATERMARK, mask_sh),\
 	SFB(blk, DPG_WATERMARK_MASK_CONTROL, PSTATE_CHANGE_WATERMARK_MASK, mask_sh),\
 	SFB(blk, DPG_PIPE_LOW_POWER_CONTROL, PSTATE_CHANGE_ENABLE, mask_sh),\
 	SFB(blk, DPG_PIPE_LOW_POWER_CONTROL, PSTATE_CHANGE_URGENT_DURING_REQUEST, mask_sh),\
@@ -286,6 +292,8 @@ struct dce_mem_input_registers {
 	type STUTTER_EXIT_SELF_REFRESH_WATERMARK_MASK; \
 	type URGENCY_LOW_WATERMARK; \
 	type URGENCY_HIGH_WATERMARK; \
+	type URGENT_LEVEL_LOW_WATERMARK;\
+	type URGENT_LEVEL_HIGH_WATERMARK;\
 	type NB_PSTATE_CHANGE_ENABLE; \
 	type NB_PSTATE_CHANGE_URGENT_DURING_REQUEST; \
 	type NB_PSTATE_CHANGE_NOT_SELF_REFRESH_DURING_REQUEST; \
@@ -297,6 +305,7 @@ struct dce_mem_input_registers {
 	type STUTTER_ENABLE; \
 	type STUTTER_IGNORE_FBC; \
 	type STUTTER_EXIT_SELF_REFRESH_WATERMARK; \
+	type STUTTER_ENTER_SELF_REFRESH_WATERMARK; \
 	type DMIF_BUFFERS_ALLOCATED; \
 	type DMIF_BUFFERS_ALLOCATION_COMPLETED; \
 	type ENABLE; /* MC_HUB_RDREQ_DMIF_LIMIT */\
@@ -344,4 +353,12 @@ void dce112_mem_input_construct(
 	const struct dce_mem_input_shift *mi_shift,
 	const struct dce_mem_input_mask *mi_mask);
 
+void dce120_mem_input_construct(
+	struct dce_mem_input *dce_mi,
+	struct dc_context *ctx,
+	int inst,
+	const struct dce_mem_input_registers *regs,
+	const struct dce_mem_input_shift *mi_shift,
+	const struct dce_mem_input_mask *mi_mask);
+
 #endif /*__DCE_MEM_INPUT_H__*/
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_scl_filters.c b/drivers/gpu/drm/amd/display/dc/dce/dce_scl_filters.c
index 6243450b41b7..48862bebf29e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_scl_filters.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_scl_filters.c
@@ -1014,11 +1014,11 @@ static const uint16_t filter_8tap_64p_183[264] = {
 
 const uint16_t *get_filter_3tap_16p(struct fixed31_32 ratio)
 {
-	if (ratio.value < dal_fixed31_32_one.value)
+	if (ratio.value < dc_fixpt_one.value)
 		return filter_3tap_16p_upscale;
-	else if (ratio.value < dal_fixed31_32_from_fraction(4, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
 		return filter_3tap_16p_117;
-	else if (ratio.value < dal_fixed31_32_from_fraction(5, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
 		return filter_3tap_16p_150;
 	else
 		return filter_3tap_16p_183;
@@ -1026,11 +1026,11 @@ const uint16_t *get_filter_3tap_16p(struct fixed31_32 ratio)
 
 const uint16_t *get_filter_3tap_64p(struct fixed31_32 ratio)
 {
-	if (ratio.value < dal_fixed31_32_one.value)
+	if (ratio.value < dc_fixpt_one.value)
 		return filter_3tap_64p_upscale;
-	else if (ratio.value < dal_fixed31_32_from_fraction(4, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
 		return filter_3tap_64p_117;
-	else if (ratio.value < dal_fixed31_32_from_fraction(5, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
 		return filter_3tap_64p_150;
 	else
 		return filter_3tap_64p_183;
@@ -1038,11 +1038,11 @@ const uint16_t *get_filter_3tap_64p(struct fixed31_32 ratio)
 
 const uint16_t *get_filter_4tap_16p(struct fixed31_32 ratio)
 {
-	if (ratio.value < dal_fixed31_32_one.value)
+	if (ratio.value < dc_fixpt_one.value)
 		return filter_4tap_16p_upscale;
-	else if (ratio.value < dal_fixed31_32_from_fraction(4, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
 		return filter_4tap_16p_117;
-	else if (ratio.value < dal_fixed31_32_from_fraction(5, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
 		return filter_4tap_16p_150;
 	else
 		return filter_4tap_16p_183;
@@ -1050,11 +1050,11 @@ const uint16_t *get_filter_4tap_16p(struct fixed31_32 ratio)
 
 const uint16_t *get_filter_4tap_64p(struct fixed31_32 ratio)
 {
-	if (ratio.value < dal_fixed31_32_one.value)
+	if (ratio.value < dc_fixpt_one.value)
 		return filter_4tap_64p_upscale;
-	else if (ratio.value < dal_fixed31_32_from_fraction(4, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
 		return filter_4tap_64p_117;
-	else if (ratio.value < dal_fixed31_32_from_fraction(5, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
 		return filter_4tap_64p_150;
 	else
 		return filter_4tap_64p_183;
@@ -1062,11 +1062,11 @@ const uint16_t *get_filter_4tap_64p(struct fixed31_32 ratio)
 
 const uint16_t *get_filter_5tap_64p(struct fixed31_32 ratio)
 {
-	if (ratio.value < dal_fixed31_32_one.value)
+	if (ratio.value < dc_fixpt_one.value)
 		return filter_5tap_64p_upscale;
-	else if (ratio.value < dal_fixed31_32_from_fraction(4, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
 		return filter_5tap_64p_117;
-	else if (ratio.value < dal_fixed31_32_from_fraction(5, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
 		return filter_5tap_64p_150;
 	else
 		return filter_5tap_64p_183;
@@ -1074,11 +1074,11 @@ const uint16_t *get_filter_5tap_64p(struct fixed31_32 ratio)
 
 const uint16_t *get_filter_6tap_64p(struct fixed31_32 ratio)
 {
-	if (ratio.value < dal_fixed31_32_one.value)
+	if (ratio.value < dc_fixpt_one.value)
 		return filter_6tap_64p_upscale;
-	else if (ratio.value < dal_fixed31_32_from_fraction(4, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
 		return filter_6tap_64p_117;
-	else if (ratio.value < dal_fixed31_32_from_fraction(5, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
 		return filter_6tap_64p_150;
 	else
 		return filter_6tap_64p_183;
@@ -1086,11 +1086,11 @@ const uint16_t *get_filter_6tap_64p(struct fixed31_32 ratio)
 
 const uint16_t *get_filter_7tap_64p(struct fixed31_32 ratio)
 {
-	if (ratio.value < dal_fixed31_32_one.value)
+	if (ratio.value < dc_fixpt_one.value)
 		return filter_7tap_64p_upscale;
-	else if (ratio.value < dal_fixed31_32_from_fraction(4, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
 		return filter_7tap_64p_117;
-	else if (ratio.value < dal_fixed31_32_from_fraction(5, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
 		return filter_7tap_64p_150;
 	else
 		return filter_7tap_64p_183;
@@ -1098,11 +1098,11 @@ const uint16_t *get_filter_7tap_64p(struct fixed31_32 ratio)
 
 const uint16_t *get_filter_8tap_64p(struct fixed31_32 ratio)
 {
-	if (ratio.value < dal_fixed31_32_one.value)
+	if (ratio.value < dc_fixpt_one.value)
 		return filter_8tap_64p_upscale;
-	else if (ratio.value < dal_fixed31_32_from_fraction(4, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
 		return filter_8tap_64p_117;
-	else if (ratio.value < dal_fixed31_32_from_fraction(5, 3).value)
+	else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
 		return filter_8tap_64p_150;
 	else
 		return filter_8tap_64p_183;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
index 162f6a6c4208..0a6d483dc046 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
@@ -26,27 +26,10 @@
 #include "dc_bios_types.h"
 #include "dce_stream_encoder.h"
 #include "reg_helper.h"
+#include "hw_shared.h"
+
 #define DC_LOGGER \
 		enc110->base.ctx->logger
-enum DP_PIXEL_ENCODING {
-DP_PIXEL_ENCODING_RGB444                 = 0x00000000,
-DP_PIXEL_ENCODING_YCBCR422               = 0x00000001,
-DP_PIXEL_ENCODING_YCBCR444               = 0x00000002,
-DP_PIXEL_ENCODING_RGB_WIDE_GAMUT         = 0x00000003,
-DP_PIXEL_ENCODING_Y_ONLY                 = 0x00000004,
-DP_PIXEL_ENCODING_YCBCR420               = 0x00000005,
-DP_PIXEL_ENCODING_RESERVED               = 0x00000006,
-};
-
-
-enum DP_COMPONENT_DEPTH {
-DP_COMPONENT_DEPTH_6BPC                  = 0x00000000,
-DP_COMPONENT_DEPTH_8BPC                  = 0x00000001,
-DP_COMPONENT_DEPTH_10BPC                 = 0x00000002,
-DP_COMPONENT_DEPTH_12BPC                 = 0x00000003,
-DP_COMPONENT_DEPTH_16BPC                 = 0x00000004,
-DP_COMPONENT_DEPTH_RESERVED              = 0x00000005,
-};
 
 
 #define REG(reg)\
@@ -80,7 +63,7 @@ enum {
 static void dce110_update_generic_info_packet(
 	struct dce110_stream_encoder *enc110,
 	uint32_t packet_index,
-	const struct encoder_info_packet *info_packet)
+	const struct dc_info_packet *info_packet)
 {
 	uint32_t regval;
 	/* TODOFPGA Figure out a proper number for max_retries polling for lock
@@ -196,7 +179,7 @@ static void dce110_update_generic_info_packet(
 static void dce110_update_hdmi_info_packet(
 	struct dce110_stream_encoder *enc110,
 	uint32_t packet_index,
-	const struct encoder_info_packet *info_packet)
+	const struct dc_info_packet *info_packet)
 {
 	uint32_t cont, send, line;
 
@@ -314,11 +297,11 @@ static void dce110_stream_encoder_dp_set_stream_attribute(
 	switch (crtc_timing->pixel_encoding) {
 	case PIXEL_ENCODING_YCBCR422:
 		REG_UPDATE(DP_PIXEL_FORMAT, DP_PIXEL_ENCODING,
-				DP_PIXEL_ENCODING_YCBCR422);
+				DP_PIXEL_ENCODING_TYPE_YCBCR422);
 		break;
 	case PIXEL_ENCODING_YCBCR444:
 		REG_UPDATE(DP_PIXEL_FORMAT, DP_PIXEL_ENCODING,
-				DP_PIXEL_ENCODING_YCBCR444);
+				DP_PIXEL_ENCODING_TYPE_YCBCR444);
 
 		if (crtc_timing->flags.Y_ONLY)
 			if (crtc_timing->display_color_depth != COLOR_DEPTH_666)
@@ -326,7 +309,7 @@ static void dce110_stream_encoder_dp_set_stream_attribute(
 				 * Color depth of Y-only could be
 				 * 8, 10, 12, 16 bits */
 				REG_UPDATE(DP_PIXEL_FORMAT, DP_PIXEL_ENCODING,
-						DP_PIXEL_ENCODING_Y_ONLY);
+						DP_PIXEL_ENCODING_TYPE_Y_ONLY);
 		/* Note: DP_MSA_MISC1 bit 7 is the indicator
 		 * of Y-only mode.
 		 * This bit is set in HW if register
@@ -334,7 +317,7 @@ static void dce110_stream_encoder_dp_set_stream_attribute(
 		break;
 	case PIXEL_ENCODING_YCBCR420:
 		REG_UPDATE(DP_PIXEL_FORMAT, DP_PIXEL_ENCODING,
-				DP_PIXEL_ENCODING_YCBCR420);
+				DP_PIXEL_ENCODING_TYPE_YCBCR420);
 		if (enc110->se_mask->DP_VID_M_DOUBLE_VALUE_EN)
 			REG_UPDATE(DP_VID_TIMING, DP_VID_M_DOUBLE_VALUE_EN, 1);
 
@@ -345,7 +328,7 @@ static void dce110_stream_encoder_dp_set_stream_attribute(
 		break;
 	default:
 		REG_UPDATE(DP_PIXEL_FORMAT, DP_PIXEL_ENCODING,
-				DP_PIXEL_ENCODING_RGB444);
+				DP_PIXEL_ENCODING_TYPE_RGB444);
 		break;
 	}
 
@@ -363,20 +346,20 @@ static void dce110_stream_encoder_dp_set_stream_attribute(
 		break;
 	case COLOR_DEPTH_888:
 		REG_UPDATE(DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH,
-				DP_COMPONENT_DEPTH_8BPC);
+				DP_COMPONENT_PIXEL_DEPTH_8BPC);
 		break;
 	case COLOR_DEPTH_101010:
 		REG_UPDATE(DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH,
-				DP_COMPONENT_DEPTH_10BPC);
+				DP_COMPONENT_PIXEL_DEPTH_10BPC);
 
 		break;
 	case COLOR_DEPTH_121212:
 		REG_UPDATE(DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH,
-				DP_COMPONENT_DEPTH_12BPC);
+				DP_COMPONENT_PIXEL_DEPTH_12BPC);
 		break;
 	default:
 		REG_UPDATE(DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH,
-				DP_COMPONENT_DEPTH_6BPC);
+				DP_COMPONENT_PIXEL_DEPTH_6BPC);
 		break;
 	}
 
@@ -700,11 +683,11 @@ static void dce110_stream_encoder_set_mst_bandwidth(
 	struct fixed31_32 avg_time_slots_per_mtp)
 {
 	struct dce110_stream_encoder *enc110 = DCE110STRENC_FROM_STRENC(enc);
-	uint32_t x = dal_fixed31_32_floor(
+	uint32_t x = dc_fixpt_floor(
 		avg_time_slots_per_mtp);
-	uint32_t y = dal_fixed31_32_ceil(
-		dal_fixed31_32_shl(
-			dal_fixed31_32_sub_int(
+	uint32_t y = dc_fixpt_ceil(
+		dc_fixpt_shl(
+			dc_fixpt_sub_int(
 				avg_time_slots_per_mtp,
 				x),
 			26));
@@ -836,7 +819,7 @@ static void dce110_stream_encoder_update_dp_info_packets(
 	const struct encoder_info_frame *info_frame)
 {
 	struct dce110_stream_encoder *enc110 = DCE110STRENC_FROM_STRENC(enc);
-	uint32_t value = REG_READ(DP_SEC_CNTL);
+	uint32_t value = 0;
 
 	if (info_frame->vsc.valid)
 		dce110_update_generic_info_packet(
@@ -870,6 +853,7 @@ static void dce110_stream_encoder_update_dp_info_packets(
 	* Therefore we need to enable master bit
 	* if at least on of the fields is not 0
 	*/
+	value = REG_READ(DP_SEC_CNTL);
 	if (value)
 		REG_UPDATE(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, 1);
 }
@@ -879,7 +863,7 @@ static void dce110_stream_encoder_stop_dp_info_packets(
 {
 	/* stop generic packets on DP */
 	struct dce110_stream_encoder *enc110 = DCE110STRENC_FROM_STRENC(enc);
-	uint32_t value = REG_READ(DP_SEC_CNTL);
+	uint32_t value = 0;
 
 	if (enc110->se_mask->DP_SEC_AVI_ENABLE) {
 		REG_SET_7(DP_SEC_CNTL, 0,
@@ -892,25 +876,10 @@ static void dce110_stream_encoder_stop_dp_info_packets(
 			DP_SEC_STREAM_ENABLE, 0);
 	}
 
-#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
-	if (enc110->se_mask->DP_SEC_GSP7_ENABLE) {
-		REG_SET_10(DP_SEC_CNTL, 0,
-			DP_SEC_GSP0_ENABLE, 0,
-			DP_SEC_GSP1_ENABLE, 0,
-			DP_SEC_GSP2_ENABLE, 0,
-			DP_SEC_GSP3_ENABLE, 0,
-			DP_SEC_GSP4_ENABLE, 0,
-			DP_SEC_GSP5_ENABLE, 0,
-			DP_SEC_GSP6_ENABLE, 0,
-			DP_SEC_GSP7_ENABLE, 0,
-			DP_SEC_MPG_ENABLE, 0,
-			DP_SEC_STREAM_ENABLE, 0);
-	}
-#endif
 	/* this register shared with audio info frame.
 	 * therefore we need to keep master enabled
 	 * if at least one of the fields is not 0 */
-
+	value = REG_READ(DP_SEC_CNTL);
 	if (value)
 		REG_UPDATE(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, 1);
 
@@ -1513,7 +1482,7 @@ static void dce110_se_disable_dp_audio(
 	struct stream_encoder *enc)
 {
 	struct dce110_stream_encoder *enc110 = DCE110STRENC_FROM_STRENC(enc);
-	uint32_t value = REG_READ(DP_SEC_CNTL);
+	uint32_t value = 0;
 
 	/* Disable Audio packets */
 	REG_UPDATE_5(DP_SEC_CNTL,
@@ -1525,6 +1494,7 @@ static void dce110_se_disable_dp_audio(
 
 	/* This register shared with encoder info frame. Therefore we need to
 	keep master enabled if at least on of the fields is not 0 */
+	value = REG_READ(DP_SEC_CNTL);
 	if (value != 0)
 		REG_UPDATE(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, 1);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
index 832c5daada35..a02e719d7794 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
@@ -41,7 +41,7 @@
 #define DC_LOGGER \
 	xfm_dce->base.ctx->logger
 
-#define IDENTITY_RATIO(ratio) (dal_fixed31_32_u2d19(ratio) == (1 << 19))
+#define IDENTITY_RATIO(ratio) (dc_fixpt_u2d19(ratio) == (1 << 19))
 #define GAMUT_MATRIX_SIZE 12
 #define SCL_PHASES 16
 
@@ -256,27 +256,27 @@ static void calculate_inits(
 	struct fixed31_32 v_init;
 
 	inits->h_int_scale_ratio =
-		dal_fixed31_32_u2d19(data->ratios.horz) << 5;
+		dc_fixpt_u2d19(data->ratios.horz) << 5;
 	inits->v_int_scale_ratio =
-		dal_fixed31_32_u2d19(data->ratios.vert) << 5;
+		dc_fixpt_u2d19(data->ratios.vert) << 5;
 
 	h_init =
-		dal_fixed31_32_div_int(
-			dal_fixed31_32_add(
+		dc_fixpt_div_int(
+			dc_fixpt_add(
 				data->ratios.horz,
-				dal_fixed31_32_from_int(data->taps.h_taps + 1)),
+				dc_fixpt_from_int(data->taps.h_taps + 1)),
 				2);
-	inits->h_init.integer = dal_fixed31_32_floor(h_init);
-	inits->h_init.fraction = dal_fixed31_32_u0d19(h_init) << 5;
+	inits->h_init.integer = dc_fixpt_floor(h_init);
+	inits->h_init.fraction = dc_fixpt_u0d19(h_init) << 5;
 
 	v_init =
-		dal_fixed31_32_div_int(
-			dal_fixed31_32_add(
+		dc_fixpt_div_int(
+			dc_fixpt_add(
 				data->ratios.vert,
-				dal_fixed31_32_from_int(data->taps.v_taps + 1)),
+				dc_fixpt_from_int(data->taps.v_taps + 1)),
 				2);
-	inits->v_init.integer = dal_fixed31_32_floor(v_init);
-	inits->v_init.fraction = dal_fixed31_32_u0d19(v_init) << 5;
+	inits->v_init.integer = dc_fixpt_floor(v_init);
+	inits->v_init.fraction = dc_fixpt_u0d19(v_init) << 5;
 }
 
 static void program_scl_ratios_inits(
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
index 3092f76bdb75..38ec0d609297 100644
--- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
@@ -733,38 +733,6 @@ enum dc_status dce100_add_stream_to_ctx(
 	return result;
 }
 
-enum dc_status dce100_validate_guaranteed(
-		struct dc  *dc,
-		struct dc_stream_state *dc_stream,
-		struct dc_state *context)
-{
-	enum dc_status result = DC_ERROR_UNEXPECTED;
-
-	context->streams[0] = dc_stream;
-	dc_stream_retain(context->streams[0]);
-	context->stream_count++;
-
-	result = resource_map_pool_resources(dc, context, dc_stream);
-
-	if (result == DC_OK)
-		result = resource_map_clock_resources(dc, context, dc_stream);
-
-	if (result == DC_OK)
-		result = build_mapped_resource(dc, context, dc_stream);
-
-	if (result == DC_OK) {
-		validate_guaranteed_copy_streams(
-				context, dc->caps.max_streams);
-		result = resource_build_scaling_params_for_context(dc, context);
-	}
-
-	if (result == DC_OK)
-		if (!dce100_validate_bandwidth(dc, context))
-			result = DC_FAIL_BANDWIDTH_VALIDATE;
-
-	return result;
-}
-
 static void dce100_destroy_resource_pool(struct resource_pool **pool)
 {
 	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
@@ -786,7 +754,6 @@ enum dc_status dce100_validate_plane(const struct dc_plane_state *plane_state, s
 static const struct resource_funcs dce100_res_pool_funcs = {
 	.destroy = dce100_destroy_resource_pool,
 	.link_enc_create = dce100_link_encoder_create,
-	.validate_guaranteed = dce100_validate_guaranteed,
 	.validate_bandwidth = dce100_validate_bandwidth,
 	.validate_plane = dce100_validate_plane,
 	.add_stream_to_ctx = dce100_add_stream_to_ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
index d0575999f172..a92fb0aa2ff3 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
@@ -70,8 +70,9 @@
 
 #define CTX \
 	hws->ctx
-#define DC_LOGGER \
-	ctx->logger
+
+#define DC_LOGGER_INIT()
+
 #define REG(reg)\
 	hws->regs->reg
 
@@ -279,7 +280,9 @@ dce110_set_input_transfer_func(struct pipe_ctx *pipe_ctx,
 	build_prescale_params(&prescale_params, plane_state);
 	ipp->funcs->ipp_program_prescale(ipp, &prescale_params);
 
-	if (plane_state->gamma_correction && dce_use_lut(plane_state->format))
+	if (plane_state->gamma_correction &&
+			!plane_state->gamma_correction->is_identity &&
+			dce_use_lut(plane_state->format))
 		ipp->funcs->ipp_program_input_lut(ipp, plane_state->gamma_correction);
 
 	if (tf == NULL) {
@@ -506,19 +509,19 @@ dce110_translate_regamma_to_hw_format(const struct dc_transfer_func *output_tf,
 	rgb_resulted[hw_points - 1].green = output_tf->tf_pts.green[start_index];
 	rgb_resulted[hw_points - 1].blue = output_tf->tf_pts.blue[start_index];
 
-	arr_points[0].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-					     dal_fixed31_32_from_int(region_start));
-	arr_points[1].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-					     dal_fixed31_32_from_int(region_end));
+	arr_points[0].x = dc_fixpt_pow(dc_fixpt_from_int(2),
+					     dc_fixpt_from_int(region_start));
+	arr_points[1].x = dc_fixpt_pow(dc_fixpt_from_int(2),
+					     dc_fixpt_from_int(region_end));
 
 	y_r = rgb_resulted[0].red;
 	y_g = rgb_resulted[0].green;
 	y_b = rgb_resulted[0].blue;
 
-	y1_min = dal_fixed31_32_min(y_r, dal_fixed31_32_min(y_g, y_b));
+	y1_min = dc_fixpt_min(y_r, dc_fixpt_min(y_g, y_b));
 
 	arr_points[0].y = y1_min;
-	arr_points[0].slope = dal_fixed31_32_div(arr_points[0].y,
+	arr_points[0].slope = dc_fixpt_div(arr_points[0].y,
 						 arr_points[0].x);
 
 	y_r = rgb_resulted[hw_points - 1].red;
@@ -528,21 +531,21 @@ dce110_translate_regamma_to_hw_format(const struct dc_transfer_func *output_tf,
 	/* see comment above, m_arrPoints[1].y should be the Y value for the
 	 * region end (m_numOfHwPoints), not last HW point(m_numOfHwPoints - 1)
 	 */
-	y3_max = dal_fixed31_32_max(y_r, dal_fixed31_32_max(y_g, y_b));
+	y3_max = dc_fixpt_max(y_r, dc_fixpt_max(y_g, y_b));
 
 	arr_points[1].y = y3_max;
 
-	arr_points[1].slope = dal_fixed31_32_zero;
+	arr_points[1].slope = dc_fixpt_zero;
 
 	if (output_tf->tf == TRANSFER_FUNCTION_PQ) {
 		/* for PQ, we want to have a straight line from last HW X point,
 		 * and the slope to be such that we hit 1.0 at 10000 nits.
 		 */
-		const struct fixed31_32 end_value = dal_fixed31_32_from_int(125);
+		const struct fixed31_32 end_value = dc_fixpt_from_int(125);
 
-		arr_points[1].slope = dal_fixed31_32_div(
-				dal_fixed31_32_sub(dal_fixed31_32_one, arr_points[1].y),
-				dal_fixed31_32_sub(end_value, arr_points[1].x));
+		arr_points[1].slope = dc_fixpt_div(
+				dc_fixpt_sub(dc_fixpt_one, arr_points[1].y),
+				dc_fixpt_sub(end_value, arr_points[1].x));
 	}
 
 	regamma_params->hw_points_num = hw_points;
@@ -566,16 +569,16 @@ dce110_translate_regamma_to_hw_format(const struct dc_transfer_func *output_tf,
 	i = 1;
 
 	while (i != hw_points + 1) {
-		if (dal_fixed31_32_lt(rgb_plus_1->red, rgb->red))
+		if (dc_fixpt_lt(rgb_plus_1->red, rgb->red))
 			rgb_plus_1->red = rgb->red;
-		if (dal_fixed31_32_lt(rgb_plus_1->green, rgb->green))
+		if (dc_fixpt_lt(rgb_plus_1->green, rgb->green))
 			rgb_plus_1->green = rgb->green;
-		if (dal_fixed31_32_lt(rgb_plus_1->blue, rgb->blue))
+		if (dc_fixpt_lt(rgb_plus_1->blue, rgb->blue))
 			rgb_plus_1->blue = rgb->blue;
 
-		rgb->delta_red = dal_fixed31_32_sub(rgb_plus_1->red, rgb->red);
-		rgb->delta_green = dal_fixed31_32_sub(rgb_plus_1->green, rgb->green);
-		rgb->delta_blue = dal_fixed31_32_sub(rgb_plus_1->blue, rgb->blue);
+		rgb->delta_red = dc_fixpt_sub(rgb_plus_1->red, rgb->red);
+		rgb->delta_green = dc_fixpt_sub(rgb_plus_1->green, rgb->green);
+		rgb->delta_blue = dc_fixpt_sub(rgb_plus_1->blue, rgb->blue);
 
 		++rgb_plus_1;
 		++rgb;
@@ -851,6 +854,28 @@ void hwss_edp_power_control(
 
 	if (power_up != is_panel_powered_on(hwseq)) {
 		/* Send VBIOS command to prompt eDP panel power */
+		if (power_up) {
+			unsigned long long current_ts = dm_get_timestamp(ctx);
+			unsigned long long duration_in_ms =
+					dm_get_elapse_time_in_ns(
+							ctx,
+							current_ts,
+							div64_u64(link->link_trace.time_stamp.edp_poweroff, 1000000));
+			unsigned long long wait_time_ms = 0;
+
+			/* max 500ms from LCDVDD off to on */
+			if (link->link_trace.time_stamp.edp_poweroff == 0)
+				wait_time_ms = 500;
+			else if (duration_in_ms < 500)
+				wait_time_ms = 500 - duration_in_ms;
+
+			if (wait_time_ms) {
+				msleep(wait_time_ms);
+				dm_output_to_console("%s: wait %lld ms to power on eDP.\n",
+						__func__, wait_time_ms);
+			}
+
+		}
 
 		DC_LOG_HW_RESUME_S3(
 				"%s: Panel Power action: %s\n",
@@ -864,9 +889,14 @@ void hwss_edp_power_control(
 		cntl.coherent = false;
 		cntl.lanes_number = LANE_COUNT_FOUR;
 		cntl.hpd_sel = link->link_enc->hpd_source;
-
 		bp_result = link_transmitter_control(ctx->dc_bios, &cntl);
 
+		if (!power_up)
+			/*save driver power off time stamp*/
+			link->link_trace.time_stamp.edp_poweroff = dm_get_timestamp(ctx);
+		else
+			link->link_trace.time_stamp.edp_poweron = dm_get_timestamp(ctx);
+
 		if (bp_result != BP_RESULT_OK)
 			DC_LOG_ERROR(
 					"%s: Panel Power bp_result: %d\n",
@@ -1011,7 +1041,7 @@ void dce110_unblank_stream(struct pipe_ctx *pipe_ctx,
 
 	if (link->local_sink && link->local_sink->sink_signal == SIGNAL_TYPE_EDP) {
 		link->dc->hwss.edp_backlight_control(link, true);
-		stream->bl_pwm_level = 0;
+		stream->bl_pwm_level = EDP_BACKLIGHT_RAMP_DISABLE_LEVEL;
 	}
 }
 void dce110_blank_stream(struct pipe_ctx *pipe_ctx)
@@ -1203,7 +1233,7 @@ static void program_scaler(const struct dc *dc,
 		&pipe_ctx->plane_res.scl_data);
 }
 
-static enum dc_status dce110_prog_pixclk_crtc_otg(
+static enum dc_status dce110_enable_stream_timing(
 		struct pipe_ctx *pipe_ctx,
 		struct dc_state *context,
 		struct dc *dc)
@@ -1269,7 +1299,7 @@ static enum dc_status apply_single_controller_ctx_to_hw(
 			pipe_ctx[pipe_ctx->pipe_idx];
 
 	/*  */
-	dc->hwss.prog_pixclk_crtc_otg(pipe_ctx, context, dc);
+	dc->hwss.enable_stream_timing(pipe_ctx, context, dc);
 
 	/* FPGA does not program backend */
 	if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
@@ -1441,6 +1471,17 @@ static void disable_vga_and_power_gate_all_controllers(
 	}
 }
 
+static struct dc_link *get_link_for_edp(struct dc *dc)
+{
+	int i;
+
+	for (i = 0; i < dc->link_count; i++) {
+		if (dc->links[i]->connector_signal == SIGNAL_TYPE_EDP)
+			return dc->links[i];
+	}
+	return NULL;
+}
+
 static struct dc_link *get_link_for_edp_not_in_use(
 		struct dc *dc,
 		struct dc_state *context)
@@ -1475,20 +1516,21 @@ static struct dc_link *get_link_for_edp_not_in_use(
  */
 void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context)
 {
-	struct dc_bios *dcb = dc->ctx->dc_bios;
-
-	/* vbios already light up eDP, so we can leverage vbios and skip eDP
-	 * programming
-	 */
-	bool can_eDP_fast_boot_optimize =
-			(dcb->funcs->get_vga_enabled_displays(dc->ctx->dc_bios) == ATOM_DISPLAY_LCD1_ACTIVE);
-
-	/* if OS doesn't light up eDP and eDP link is available, we want to disable */
 	struct dc_link *edp_link_to_turnoff = NULL;
+	struct dc_link *edp_link = get_link_for_edp(dc);
+	bool can_eDP_fast_boot_optimize = false;
+
+	if (edp_link) {
+		can_eDP_fast_boot_optimize =
+				edp_link->link_enc->funcs->is_dig_enabled(edp_link->link_enc);
+	}
 
 	if (can_eDP_fast_boot_optimize) {
 		edp_link_to_turnoff = get_link_for_edp_not_in_use(dc, context);
 
+		/* if OS doesn't light up eDP and eDP link is available, we want to disable
+		 * If resume from S4/S5, should optimization.
+		 */
 		if (!edp_link_to_turnoff)
 			dc->apply_edp_fast_boot_optimization = true;
 	}
@@ -1544,6 +1586,7 @@ static void dce110_set_displaymarks(
 			pipe_ctx->plane_res.mi,
 			context->bw.dce.nbp_state_change_wm_ns[num_pipes],
 			context->bw.dce.stutter_exit_wm_ns[num_pipes],
+			context->bw.dce.stutter_entry_wm_ns[num_pipes],
 			context->bw.dce.urgent_wm_ns[num_pipes],
 			total_dest_line_time_ns);
 		if (i == underlay_idx) {
@@ -1569,6 +1612,7 @@ static void set_safe_displaymarks(
 		MAX_WATERMARK, MAX_WATERMARK, MAX_WATERMARK, MAX_WATERMARK };
 	struct dce_watermarks nbp_marks = {
 		SAFE_NBP_MARK, SAFE_NBP_MARK, SAFE_NBP_MARK, SAFE_NBP_MARK };
+	struct dce_watermarks min_marks = { 0, 0, 0, 0};
 
 	for (i = 0; i < MAX_PIPES; i++) {
 		if (res_ctx->pipe_ctx[i].stream == NULL || res_ctx->pipe_ctx[i].plane_res.mi == NULL)
@@ -1578,6 +1622,7 @@ static void set_safe_displaymarks(
 				res_ctx->pipe_ctx[i].plane_res.mi,
 				nbp_marks,
 				max_marks,
+				min_marks,
 				max_marks,
 				MAX_WATERMARK);
 
@@ -1803,6 +1848,9 @@ static bool should_enable_fbc(struct dc *dc,
 		}
 	}
 
+	/* Pipe context should be found */
+	ASSERT(pipe_ctx);
+
 	/* Only supports eDP */
 	if (pipe_ctx->stream->sink->link->connector_signal != SIGNAL_TYPE_EDP)
 		return false;
@@ -2221,74 +2269,6 @@ static void program_gamut_remap(struct pipe_ctx *pipe_ctx)
 
 	pipe_ctx->plane_res.xfm->funcs->transform_set_gamut_remap(pipe_ctx->plane_res.xfm, &adjust);
 }
-
-/**
- * TODO REMOVE, USE UPDATE INSTEAD
- */
-static void set_plane_config(
-	const struct dc *dc,
-	struct pipe_ctx *pipe_ctx,
-	struct resource_context *res_ctx)
-{
-	struct mem_input *mi = pipe_ctx->plane_res.mi;
-	struct dc_plane_state *plane_state = pipe_ctx->plane_state;
-	struct xfm_grph_csc_adjustment adjust;
-	struct out_csc_color_matrix tbl_entry;
-	unsigned int i;
-
-	memset(&adjust, 0, sizeof(adjust));
-	memset(&tbl_entry, 0, sizeof(tbl_entry));
-	adjust.gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_BYPASS;
-
-	dce_enable_fe_clock(dc->hwseq, mi->inst, true);
-
-	set_default_colors(pipe_ctx);
-	if (pipe_ctx->stream->csc_color_matrix.enable_adjustment == true) {
-		tbl_entry.color_space =
-			pipe_ctx->stream->output_color_space;
-
-		for (i = 0; i < 12; i++)
-			tbl_entry.regval[i] =
-			pipe_ctx->stream->csc_color_matrix.matrix[i];
-
-		pipe_ctx->plane_res.xfm->funcs->opp_set_csc_adjustment
-				(pipe_ctx->plane_res.xfm, &tbl_entry);
-	}
-
-	if (pipe_ctx->stream->gamut_remap_matrix.enable_remap == true) {
-		adjust.gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_SW;
-
-		for (i = 0; i < CSC_TEMPERATURE_MATRIX_SIZE; i++)
-			adjust.temperature_matrix[i] =
-				pipe_ctx->stream->gamut_remap_matrix.matrix[i];
-	}
-
-	pipe_ctx->plane_res.xfm->funcs->transform_set_gamut_remap(pipe_ctx->plane_res.xfm, &adjust);
-
-	pipe_ctx->plane_res.scl_data.lb_params.alpha_en = pipe_ctx->bottom_pipe != 0;
-	program_scaler(dc, pipe_ctx);
-
-	program_surface_visibility(dc, pipe_ctx);
-
-	mi->funcs->mem_input_program_surface_config(
-			mi,
-			plane_state->format,
-			&plane_state->tiling_info,
-			&plane_state->plane_size,
-			plane_state->rotation,
-			NULL,
-			false);
-	if (mi->funcs->set_blank)
-		mi->funcs->set_blank(mi, pipe_ctx->plane_state->visible);
-
-	if (dc->config.gpu_vm_support)
-		mi->funcs->mem_input_program_pte_vm(
-				pipe_ctx->plane_res.mi,
-				plane_state->format,
-				&plane_state->tiling_info,
-				plane_state->rotation);
-}
-
 static void update_plane_addr(const struct dc *dc,
 		struct pipe_ctx *pipe_ctx)
 {
@@ -2699,8 +2679,11 @@ static void dce110_program_front_end_for_pipe(
 	struct dc_plane_state *plane_state = pipe_ctx->plane_state;
 	struct xfm_grph_csc_adjustment adjust;
 	struct out_csc_color_matrix tbl_entry;
+#if defined(CONFIG_DRM_AMD_DC_FBC)
+	unsigned int underlay_idx = dc->res_pool->underlay_pipe_index;
+#endif
 	unsigned int i;
-	struct dc_context *ctx = dc->ctx;
+	DC_LOGGER_INIT();
 	memset(&tbl_entry, 0, sizeof(tbl_entry));
 
 	if (dc->current_state)
@@ -2740,7 +2723,9 @@ static void dce110_program_front_end_for_pipe(
 	program_scaler(dc, pipe_ctx);
 
 #if defined(CONFIG_DRM_AMD_DC_FBC)
-	if (dc->fbc_compressor && old_pipe->stream) {
+	/* fbc not applicable on Underlay pipe */
+	if (dc->fbc_compressor && old_pipe->stream &&
+	    pipe_ctx->pipe_idx != underlay_idx) {
 		if (plane_state->tiling_info.gfx8.array_mode == DC_ARRAY_LINEAR_GENERAL)
 			dc->fbc_compressor->funcs->disable_fbc(dc->fbc_compressor);
 		else
@@ -2776,13 +2761,13 @@ static void dce110_program_front_end_for_pipe(
 		dc->hwss.set_output_transfer_func(pipe_ctx, pipe_ctx->stream);
 
 	DC_LOG_SURFACE(
-			"Pipe:%d 0x%x: addr hi:0x%x, "
+			"Pipe:%d %p: addr hi:0x%x, "
 			"addr low:0x%x, "
 			"src: %d, %d, %d,"
 			" %d; dst: %d, %d, %d, %d;"
 			"clip: %d, %d, %d, %d\n",
 			pipe_ctx->pipe_idx,
-			pipe_ctx->plane_state,
+			(void *) pipe_ctx->plane_state,
 			pipe_ctx->plane_state->address.grph.addr.high_part,
 			pipe_ctx->plane_state->address.grph.addr.low_part,
 			pipe_ctx->plane_state->src_rect.x,
@@ -2970,7 +2955,6 @@ static const struct hw_sequencer_funcs dce110_funcs = {
 	.init_hw = init_hw,
 	.apply_ctx_to_hw = dce110_apply_ctx_to_hw,
 	.apply_ctx_for_surface = dce110_apply_ctx_for_surface,
-	.set_plane_config = set_plane_config,
 	.update_plane_addr = update_plane_addr,
 	.update_pending_status = dce110_update_pending_status,
 	.set_input_transfer_func = dce110_set_input_transfer_func,
@@ -2993,7 +2977,7 @@ static const struct hw_sequencer_funcs dce110_funcs = {
 	.get_position = get_position,
 	.set_static_screen_control = set_static_screen_control,
 	.reset_hw_ctx_wrap = dce110_reset_hw_ctx_wrap,
-	.prog_pixclk_crtc_otg = dce110_prog_pixclk_crtc_otg,
+	.enable_stream_timing = dce110_enable_stream_timing,
 	.setup_stereo = NULL,
 	.set_avmute = dce110_set_avmute,
 	.wait_for_mpcc_disconnect = dce110_wait_for_mpcc_disconnect,
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
index 7bab8c6d2a73..0564c8e31252 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
@@ -923,6 +923,7 @@ void dce_mem_input_v_program_display_marks(
 	struct mem_input *mem_input,
 	struct dce_watermarks nbp,
 	struct dce_watermarks stutter,
+	struct dce_watermarks stutter_enter,
 	struct dce_watermarks urgent,
 	uint32_t total_dest_line_time_ns)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
index b1f14be20fdf..ee33786bdef6 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
@@ -930,38 +930,6 @@ static enum dc_status dce110_add_stream_to_ctx(
 	return result;
 }
 
-static enum dc_status dce110_validate_guaranteed(
-		struct dc *dc,
-		struct dc_stream_state *dc_stream,
-		struct dc_state *context)
-{
-	enum dc_status result = DC_ERROR_UNEXPECTED;
-
-	context->streams[0] = dc_stream;
-	dc_stream_retain(context->streams[0]);
-	context->stream_count++;
-
-	result = resource_map_pool_resources(dc, context, dc_stream);
-
-	if (result == DC_OK)
-		result = resource_map_clock_resources(dc, context, dc_stream);
-
-	if (result == DC_OK)
-		result = build_mapped_resource(dc, context, dc_stream);
-
-	if (result == DC_OK) {
-		validate_guaranteed_copy_streams(
-				context, dc->caps.max_streams);
-		result = resource_build_scaling_params_for_context(dc, context);
-	}
-
-	if (result == DC_OK)
-		if (!dce110_validate_bandwidth(dc, context))
-			result = DC_FAIL_BANDWIDTH_VALIDATE;
-
-	return result;
-}
-
 static struct pipe_ctx *dce110_acquire_underlay(
 		struct dc_state *context,
 		const struct resource_pool *pool,
@@ -1036,7 +1004,6 @@ static void dce110_destroy_resource_pool(struct resource_pool **pool)
 static const struct resource_funcs dce110_res_pool_funcs = {
 	.destroy = dce110_destroy_resource_pool,
 	.link_enc_create = dce110_link_encoder_create,
-	.validate_guaranteed = dce110_validate_guaranteed,
 	.validate_bandwidth = dce110_validate_bandwidth,
 	.validate_plane = dce110_validate_plane,
 	.acquire_idle_pipe_for_layer = dce110_acquire_underlay,
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c
index be7153924a70..1b2fe0df347f 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c
@@ -431,14 +431,6 @@ void dce110_timing_generator_set_drr(
 			0,
 			CRTC_V_TOTAL_CONTROL,
 			CRTC_SET_V_TOTAL_MIN_MASK);
-		set_reg_field_value(v_total_min,
-				0,
-				CRTC_V_TOTAL_MIN,
-				CRTC_V_TOTAL_MIN);
-		set_reg_field_value(v_total_max,
-				0,
-				CRTC_V_TOTAL_MAX,
-				CRTC_V_TOTAL_MAX);
 		set_reg_field_value(v_total_cntl,
 				0,
 				CRTC_V_TOTAL_CONTROL,
@@ -447,6 +439,14 @@ void dce110_timing_generator_set_drr(
 				0,
 				CRTC_V_TOTAL_CONTROL,
 				CRTC_V_TOTAL_MAX_SEL);
+		set_reg_field_value(v_total_min,
+				0,
+				CRTC_V_TOTAL_MIN,
+				CRTC_V_TOTAL_MIN);
+		set_reg_field_value(v_total_max,
+				0,
+				CRTC_V_TOTAL_MAX,
+				CRTC_V_TOTAL_MAX);
 		set_reg_field_value(v_total_cntl,
 				0,
 				CRTC_V_TOTAL_CONTROL,
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c
index 8ad04816e7d3..a3cef60380ed 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c
@@ -648,12 +648,6 @@ static void dce110_timing_generator_v_disable_vga(
 	return;
 }
 
-static bool dce110_tg_v_is_blanked(struct timing_generator *tg)
-{
-	/* Signal comes from the primary pipe, underlay is never blanked. */
-	return false;
-}
-
 /** ********************************************************************************************
  *
  * DCE11 Timing Generator Constructor / Destructor
@@ -670,7 +664,6 @@ static const struct timing_generator_funcs dce110_tg_v_funcs = {
 		.set_early_control = dce110_timing_generator_v_set_early_control,
 		.wait_for_state = dce110_timing_generator_v_wait_for_state,
 		.set_blank = dce110_timing_generator_v_set_blank,
-		.is_blanked = dce110_tg_v_is_blanked,
 		.set_colors = dce110_timing_generator_v_set_colors,
 		.set_overscan_blank_color =
 				dce110_timing_generator_v_set_overscan_color_black,
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c
index 8ba3c12fc608..a7dce060204f 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c
@@ -373,13 +373,13 @@ static void calculate_inits(
 	struct rect *chroma_viewport)
 {
 	inits->h_int_scale_ratio_luma =
-		dal_fixed31_32_u2d19(data->ratios.horz) << 5;
+		dc_fixpt_u2d19(data->ratios.horz) << 5;
 	inits->v_int_scale_ratio_luma =
-		dal_fixed31_32_u2d19(data->ratios.vert) << 5;
+		dc_fixpt_u2d19(data->ratios.vert) << 5;
 	inits->h_int_scale_ratio_chroma =
-		dal_fixed31_32_u2d19(data->ratios.horz_c) << 5;
+		dc_fixpt_u2d19(data->ratios.horz_c) << 5;
 	inits->v_int_scale_ratio_chroma =
-		dal_fixed31_32_u2d19(data->ratios.vert_c) << 5;
+		dc_fixpt_u2d19(data->ratios.vert_c) << 5;
 
 	inits->h_init_luma.integer = 1;
 	inits->v_init_luma.integer = 1;
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
index cd1e3f72c44e..00c0a1ef15eb 100644
--- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
@@ -430,7 +430,7 @@ static struct stream_encoder *dce112_stream_encoder_create(
 
 	if (!enc110)
 		return NULL;
-	
+
 	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
 					&stream_enc_regs[eng_id],
 					&se_shift, &se_mask);
@@ -867,38 +867,6 @@ enum dc_status dce112_add_stream_to_ctx(
 	return result;
 }
 
-enum dc_status dce112_validate_guaranteed(
-		struct dc *dc,
-		struct dc_stream_state *stream,
-		struct dc_state *context)
-{
-	enum dc_status result = DC_ERROR_UNEXPECTED;
-
-	context->streams[0] = stream;
-	dc_stream_retain(context->streams[0]);
-	context->stream_count++;
-
-	result = resource_map_pool_resources(dc, context, stream);
-
-	if (result == DC_OK)
-		result = resource_map_phy_clock_resources(dc, context, stream);
-
-	if (result == DC_OK)
-		result = build_mapped_resource(dc, context, stream);
-
-	if (result == DC_OK) {
-		validate_guaranteed_copy_streams(
-				context, dc->caps.max_streams);
-		result = resource_build_scaling_params_for_context(dc, context);
-	}
-
-	if (result == DC_OK)
-		if (!dce112_validate_bandwidth(dc, context))
-			result = DC_FAIL_BANDWIDTH_VALIDATE;
-
-	return result;
-}
-
 enum dc_status dce112_validate_global(
 		struct dc *dc,
 		struct dc_state *context)
@@ -921,7 +889,6 @@ static void dce112_destroy_resource_pool(struct resource_pool **pool)
 static const struct resource_funcs dce112_res_pool_funcs = {
 	.destroy = dce112_destroy_resource_pool,
 	.link_enc_create = dce112_link_encoder_create,
-	.validate_guaranteed = dce112_validate_guaranteed,
 	.validate_bandwidth = dce112_validate_bandwidth,
 	.validate_plane = dce100_validate_plane,
 	.add_stream_to_ctx = dce112_add_stream_to_ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.h b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.h
index d5c19d34eb0a..95a403396219 100644
--- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.h
@@ -42,11 +42,6 @@ enum dc_status dce112_validate_with_context(
 		struct dc_state *context,
 		struct dc_state *old_context);
 
-enum dc_status dce112_validate_guaranteed(
-		struct dc *dc,
-		struct dc_stream_state *dc_stream,
-		struct dc_state *context);
-
 bool dce112_validate_bandwidth(
 	struct dc *dc,
 	struct dc_state *context);
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
index 4659a4bfabaa..2d58daccc005 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
@@ -652,7 +652,7 @@ static struct mem_input *dce120_mem_input_create(
 		return NULL;
 	}
 
-	dce112_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
+	dce120_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
 	return &dce_mi->base;
 }
 
@@ -684,7 +684,6 @@ static void dce120_destroy_resource_pool(struct resource_pool **pool)
 static const struct resource_funcs dce120_res_pool_funcs = {
 	.destroy = dce120_destroy_resource_pool,
 	.link_enc_create = dce120_link_encoder_create,
-	.validate_guaranteed = dce112_validate_guaranteed,
 	.validate_bandwidth = dce112_validate_bandwidth,
 	.validate_plane = dce100_validate_plane,
 	.add_stream_to_ctx = dce112_add_stream_to_ctx
@@ -815,14 +814,25 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc)
 	dm_pp_notify_wm_clock_changes(dc->ctx, &clk_ranges);
 }
 
+static uint32_t read_pipe_fuses(struct dc_context *ctx)
+{
+	uint32_t value = dm_read_reg_soc15(ctx, mmCC_DC_PIPE_DIS, 0);
+	/* VG20 support max 6 pipes */
+	value = value & 0x3f;
+	return value;
+}
+
 static bool construct(
 	uint8_t num_virtual_links,
 	struct dc *dc,
 	struct dce110_resource_pool *pool)
 {
 	unsigned int i;
+	int j;
 	struct dc_context *ctx = dc->ctx;
 	struct irq_service_init_data irq_init_data;
+	bool harvest_enabled = ASICREV_IS_VEGA20_P(ctx->asic_id.hw_internal_rev);
+	uint32_t pipe_fuses;
 
 	ctx->dc_bios->regs = &bios_regs;
 
@@ -916,28 +926,41 @@ static bool construct(
 	if (!pool->base.irqs)
 		goto irqs_create_fail;
 
+	/* retrieve valid pipe fuses */
+	if (harvest_enabled)
+		pipe_fuses = read_pipe_fuses(ctx);
+
+	/* index to valid pipe resource */
+	j = 0;
 	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.timing_generators[i] =
+		if (harvest_enabled) {
+			if ((pipe_fuses & (1 << i)) != 0) {
+				dm_error("DC: skip invalid pipe %d!\n", i);
+				continue;
+			}
+		}
+
+		pool->base.timing_generators[j] =
 				dce120_timing_generator_create(
 					ctx,
 					i,
 					&dce120_tg_offsets[i]);
-		if (pool->base.timing_generators[i] == NULL) {
+		if (pool->base.timing_generators[j] == NULL) {
 			BREAK_TO_DEBUGGER();
 			dm_error("DC: failed to create tg!\n");
 			goto controller_create_fail;
 		}
 
-		pool->base.mis[i] = dce120_mem_input_create(ctx, i);
+		pool->base.mis[j] = dce120_mem_input_create(ctx, i);
 
-		if (pool->base.mis[i] == NULL) {
+		if (pool->base.mis[j] == NULL) {
 			BREAK_TO_DEBUGGER();
 			dm_error(
 				"DC: failed to create memory input!\n");
 			goto controller_create_fail;
 		}
 
-		pool->base.ipps[i] = dce120_ipp_create(ctx, i);
+		pool->base.ipps[j] = dce120_ipp_create(ctx, i);
 		if (pool->base.ipps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
 			dm_error(
@@ -945,7 +968,7 @@ static bool construct(
 			goto controller_create_fail;
 		}
 
-		pool->base.transforms[i] = dce120_transform_create(ctx, i);
+		pool->base.transforms[j] = dce120_transform_create(ctx, i);
 		if (pool->base.transforms[i] == NULL) {
 			BREAK_TO_DEBUGGER();
 			dm_error(
@@ -953,16 +976,23 @@ static bool construct(
 			goto res_create_fail;
 		}
 
-		pool->base.opps[i] = dce120_opp_create(
+		pool->base.opps[j] = dce120_opp_create(
 			ctx,
 			i);
-		if (pool->base.opps[i] == NULL) {
+		if (pool->base.opps[j] == NULL) {
 			BREAK_TO_DEBUGGER();
 			dm_error(
 				"DC: failed to create output pixel processor!\n");
 		}
+
+		/* check next valid pipe */
+		j++;
 	}
 
+	/* valid pipe num */
+	pool->base.pipe_count = j;
+	pool->base.timing_generator_count = j;
+
 	if (!resource_construct(num_virtual_links, dc, &pool->base,
 			 &res_create_funcs))
 		goto res_create_fail;
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
index 7bee78172d85..2ea490f8482e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
+++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
@@ -570,18 +570,18 @@ void dce120_timing_generator_set_drr(
 				0x180);
 
 	} else {
-		CRTC_REG_UPDATE(
-				CRTC0_CRTC_V_TOTAL_MIN,
-				CRTC_V_TOTAL_MIN, 0);
-		CRTC_REG_UPDATE(
-				CRTC0_CRTC_V_TOTAL_MAX,
-				CRTC_V_TOTAL_MAX, 0);
 		CRTC_REG_SET_N(CRTC0_CRTC_V_TOTAL_CONTROL, 5,
 				FD(CRTC0_CRTC_V_TOTAL_CONTROL__CRTC_V_TOTAL_MIN_SEL), 0,
 				FD(CRTC0_CRTC_V_TOTAL_CONTROL__CRTC_V_TOTAL_MAX_SEL), 0,
 				FD(CRTC0_CRTC_V_TOTAL_CONTROL__CRTC_FORCE_LOCK_ON_EVENT), 0,
 				FD(CRTC0_CRTC_V_TOTAL_CONTROL__CRTC_FORCE_LOCK_TO_MASTER_VSYNC), 0,
 				FD(CRTC0_CRTC_V_TOTAL_CONTROL__CRTC_SET_V_TOTAL_MIN_MASK), 0);
+		CRTC_REG_UPDATE(
+				CRTC0_CRTC_V_TOTAL_MIN,
+				CRTC_V_TOTAL_MIN, 0);
+		CRTC_REG_UPDATE(
+				CRTC0_CRTC_V_TOTAL_MAX,
+				CRTC_V_TOTAL_MAX, 0);
 		CRTC_REG_UPDATE(
 				CRTC0_CRTC_STATIC_SCREEN_CONTROL,
 				CRTC_STATIC_SCREEN_EVENT_MASK,
diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
index 5d854a37a978..48a068964722 100644
--- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
@@ -691,23 +691,6 @@ static void destruct(struct dce110_resource_pool *pool)
 	}
 }
 
-static enum dc_status build_mapped_resource(
-		const struct dc *dc,
-		struct dc_state *context,
-		struct dc_stream_state *stream)
-{
-	struct pipe_ctx *pipe_ctx = resource_get_head_pipe_for_stream(&context->res_ctx, stream);
-
-	if (!pipe_ctx)
-		return DC_ERROR_UNEXPECTED;
-
-	dce110_resource_build_pipe_hw_param(pipe_ctx);
-
-	resource_build_info_frame(pipe_ctx);
-
-	return DC_OK;
-}
-
 bool dce80_validate_bandwidth(
 	struct dc *dc,
 	struct dc_state *context)
@@ -749,37 +732,6 @@ enum dc_status dce80_validate_global(
 	return DC_OK;
 }
 
-enum dc_status dce80_validate_guaranteed(
-		struct dc *dc,
-		struct dc_stream_state *dc_stream,
-		struct dc_state *context)
-{
-	enum dc_status result = DC_ERROR_UNEXPECTED;
-
-	context->streams[0] = dc_stream;
-	dc_stream_retain(context->streams[0]);
-	context->stream_count++;
-
-	result = resource_map_pool_resources(dc, context, dc_stream);
-
-	if (result == DC_OK)
-		result = resource_map_clock_resources(dc, context, dc_stream);
-
-	if (result == DC_OK)
-		result = build_mapped_resource(dc, context, dc_stream);
-
-	if (result == DC_OK) {
-		validate_guaranteed_copy_streams(
-				context, dc->caps.max_streams);
-		result = resource_build_scaling_params_for_context(dc, context);
-	}
-
-	if (result == DC_OK)
-		result = dce80_validate_bandwidth(dc, context);
-
-	return result;
-}
-
 static void dce80_destroy_resource_pool(struct resource_pool **pool)
 {
 	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
@@ -792,7 +744,6 @@ static void dce80_destroy_resource_pool(struct resource_pool **pool)
 static const struct resource_funcs dce80_res_pool_funcs = {
 	.destroy = dce80_destroy_resource_pool,
 	.link_enc_create = dce80_link_encoder_create,
-	.validate_guaranteed = dce80_validate_guaranteed,
 	.validate_bandwidth = dce80_validate_bandwidth,
 	.validate_plane = dce100_validate_plane,
 	.add_stream_to_ctx = dce100_add_stream_to_ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
index 5469bdfe19f3..84f52c63d95c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
@@ -26,7 +26,7 @@ DCN10 = dcn10_resource.o dcn10_ipp.o dcn10_hw_sequencer.o \
 		dcn10_dpp.o dcn10_opp.o dcn10_optc.o \
 		dcn10_hubp.o dcn10_mpc.o \
 		dcn10_dpp_dscl.o dcn10_dpp_cm.o dcn10_cm_common.o \
-		dcn10_hubbub.o
+		dcn10_hubbub.o dcn10_stream_encoder.o dcn10_link_encoder.o
 
 AMD_DAL_DCN10 = $(addprefix $(AMDDALPATH)/dc/dcn10/,$(DCN10))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
index 881a1bff94d2..5d95a997fd9f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
@@ -169,7 +169,7 @@ bool cm_helper_convert_to_custom_float(
 	}
 
 	if (fixpoint == true)
-		arr_points[1].custom_float_y = dal_fixed31_32_clamp_u0d14(arr_points[1].y);
+		arr_points[1].custom_float_y = dc_fixpt_clamp_u0d14(arr_points[1].y);
 	else if (!convert_to_custom_float_format(arr_points[1].y, &fmt,
 		&arr_points[1].custom_float_y)) {
 		BREAK_TO_DEBUGGER();
@@ -327,19 +327,19 @@ bool cm_helper_translate_curve_to_hw_format(
 	rgb_resulted[hw_points - 1].green = output_tf->tf_pts.green[start_index];
 	rgb_resulted[hw_points - 1].blue = output_tf->tf_pts.blue[start_index];
 
-	arr_points[0].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-					     dal_fixed31_32_from_int(region_start));
-	arr_points[1].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-					     dal_fixed31_32_from_int(region_end));
+	arr_points[0].x = dc_fixpt_pow(dc_fixpt_from_int(2),
+					     dc_fixpt_from_int(region_start));
+	arr_points[1].x = dc_fixpt_pow(dc_fixpt_from_int(2),
+					     dc_fixpt_from_int(region_end));
 
 	y_r = rgb_resulted[0].red;
 	y_g = rgb_resulted[0].green;
 	y_b = rgb_resulted[0].blue;
 
-	y1_min = dal_fixed31_32_min(y_r, dal_fixed31_32_min(y_g, y_b));
+	y1_min = dc_fixpt_min(y_r, dc_fixpt_min(y_g, y_b));
 
 	arr_points[0].y = y1_min;
-	arr_points[0].slope = dal_fixed31_32_div(arr_points[0].y, arr_points[0].x);
+	arr_points[0].slope = dc_fixpt_div(arr_points[0].y, arr_points[0].x);
 	y_r = rgb_resulted[hw_points - 1].red;
 	y_g = rgb_resulted[hw_points - 1].green;
 	y_b = rgb_resulted[hw_points - 1].blue;
@@ -347,35 +347,35 @@ bool cm_helper_translate_curve_to_hw_format(
 	/* see comment above, m_arrPoints[1].y should be the Y value for the
 	 * region end (m_numOfHwPoints), not last HW point(m_numOfHwPoints - 1)
 	 */
-	y3_max = dal_fixed31_32_max(y_r, dal_fixed31_32_max(y_g, y_b));
+	y3_max = dc_fixpt_max(y_r, dc_fixpt_max(y_g, y_b));
 
 	arr_points[1].y = y3_max;
 
-	arr_points[1].slope = dal_fixed31_32_zero;
+	arr_points[1].slope = dc_fixpt_zero;
 
 	if (output_tf->tf == TRANSFER_FUNCTION_PQ) {
 		/* for PQ, we want to have a straight line from last HW X point,
 		 * and the slope to be such that we hit 1.0 at 10000 nits.
 		 */
 		const struct fixed31_32 end_value =
-				dal_fixed31_32_from_int(125);
+				dc_fixpt_from_int(125);
 
-		arr_points[1].slope = dal_fixed31_32_div(
-			dal_fixed31_32_sub(dal_fixed31_32_one, arr_points[1].y),
-			dal_fixed31_32_sub(end_value, arr_points[1].x));
+		arr_points[1].slope = dc_fixpt_div(
+			dc_fixpt_sub(dc_fixpt_one, arr_points[1].y),
+			dc_fixpt_sub(end_value, arr_points[1].x));
 	}
 
 	lut_params->hw_points_num = hw_points;
 
-	i = 1;
-	for (k = 0; k < MAX_REGIONS_NUMBER && i < MAX_REGIONS_NUMBER; k++) {
+	k = 0;
+	for (i = 1; i < MAX_REGIONS_NUMBER; i++) {
 		if (seg_distr[k] != -1) {
 			lut_params->arr_curve_points[k].segments_num =
 					seg_distr[k];
 			lut_params->arr_curve_points[i].offset =
 					lut_params->arr_curve_points[k].offset + (1 << seg_distr[k]);
 		}
-		i++;
+		k++;
 	}
 
 	if (seg_distr[k] != -1)
@@ -386,24 +386,24 @@ bool cm_helper_translate_curve_to_hw_format(
 
 	i = 1;
 	while (i != hw_points + 1) {
-		if (dal_fixed31_32_lt(rgb_plus_1->red, rgb->red))
+		if (dc_fixpt_lt(rgb_plus_1->red, rgb->red))
 			rgb_plus_1->red = rgb->red;
-		if (dal_fixed31_32_lt(rgb_plus_1->green, rgb->green))
+		if (dc_fixpt_lt(rgb_plus_1->green, rgb->green))
 			rgb_plus_1->green = rgb->green;
-		if (dal_fixed31_32_lt(rgb_plus_1->blue, rgb->blue))
+		if (dc_fixpt_lt(rgb_plus_1->blue, rgb->blue))
 			rgb_plus_1->blue = rgb->blue;
 
-		rgb->delta_red   = dal_fixed31_32_sub(rgb_plus_1->red,   rgb->red);
-		rgb->delta_green = dal_fixed31_32_sub(rgb_plus_1->green, rgb->green);
-		rgb->delta_blue  = dal_fixed31_32_sub(rgb_plus_1->blue,  rgb->blue);
+		rgb->delta_red   = dc_fixpt_sub(rgb_plus_1->red,   rgb->red);
+		rgb->delta_green = dc_fixpt_sub(rgb_plus_1->green, rgb->green);
+		rgb->delta_blue  = dc_fixpt_sub(rgb_plus_1->blue,  rgb->blue);
 
 		if (fixpoint == true) {
-			rgb->delta_red_reg   = dal_fixed31_32_clamp_u0d10(rgb->delta_red);
-			rgb->delta_green_reg = dal_fixed31_32_clamp_u0d10(rgb->delta_green);
-			rgb->delta_blue_reg  = dal_fixed31_32_clamp_u0d10(rgb->delta_blue);
-			rgb->red_reg         = dal_fixed31_32_clamp_u0d14(rgb->red);
-			rgb->green_reg       = dal_fixed31_32_clamp_u0d14(rgb->green);
-			rgb->blue_reg        = dal_fixed31_32_clamp_u0d14(rgb->blue);
+			rgb->delta_red_reg   = dc_fixpt_clamp_u0d10(rgb->delta_red);
+			rgb->delta_green_reg = dc_fixpt_clamp_u0d10(rgb->delta_green);
+			rgb->delta_blue_reg  = dc_fixpt_clamp_u0d10(rgb->delta_blue);
+			rgb->red_reg         = dc_fixpt_clamp_u0d14(rgb->red);
+			rgb->green_reg       = dc_fixpt_clamp_u0d14(rgb->green);
+			rgb->blue_reg        = dc_fixpt_clamp_u0d14(rgb->blue);
 		}
 
 		++rgb_plus_1;
@@ -489,19 +489,19 @@ bool cm_helper_translate_curve_to_degamma_hw_format(
 	rgb_resulted[hw_points - 1].green = output_tf->tf_pts.green[start_index];
 	rgb_resulted[hw_points - 1].blue = output_tf->tf_pts.blue[start_index];
 
-	arr_points[0].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-					     dal_fixed31_32_from_int(region_start));
-	arr_points[1].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-					     dal_fixed31_32_from_int(region_end));
+	arr_points[0].x = dc_fixpt_pow(dc_fixpt_from_int(2),
+					     dc_fixpt_from_int(region_start));
+	arr_points[1].x = dc_fixpt_pow(dc_fixpt_from_int(2),
+					     dc_fixpt_from_int(region_end));
 
 	y_r = rgb_resulted[0].red;
 	y_g = rgb_resulted[0].green;
 	y_b = rgb_resulted[0].blue;
 
-	y1_min = dal_fixed31_32_min(y_r, dal_fixed31_32_min(y_g, y_b));
+	y1_min = dc_fixpt_min(y_r, dc_fixpt_min(y_g, y_b));
 
 	arr_points[0].y = y1_min;
-	arr_points[0].slope = dal_fixed31_32_div(arr_points[0].y, arr_points[0].x);
+	arr_points[0].slope = dc_fixpt_div(arr_points[0].y, arr_points[0].x);
 	y_r = rgb_resulted[hw_points - 1].red;
 	y_g = rgb_resulted[hw_points - 1].green;
 	y_b = rgb_resulted[hw_points - 1].blue;
@@ -509,35 +509,35 @@ bool cm_helper_translate_curve_to_degamma_hw_format(
 	/* see comment above, m_arrPoints[1].y should be the Y value for the
 	 * region end (m_numOfHwPoints), not last HW point(m_numOfHwPoints - 1)
 	 */
-	y3_max = dal_fixed31_32_max(y_r, dal_fixed31_32_max(y_g, y_b));
+	y3_max = dc_fixpt_max(y_r, dc_fixpt_max(y_g, y_b));
 
 	arr_points[1].y = y3_max;
 
-	arr_points[1].slope = dal_fixed31_32_zero;
+	arr_points[1].slope = dc_fixpt_zero;
 
 	if (output_tf->tf == TRANSFER_FUNCTION_PQ) {
 		/* for PQ, we want to have a straight line from last HW X point,
 		 * and the slope to be such that we hit 1.0 at 10000 nits.
 		 */
 		const struct fixed31_32 end_value =
-				dal_fixed31_32_from_int(125);
+				dc_fixpt_from_int(125);
 
-		arr_points[1].slope = dal_fixed31_32_div(
-			dal_fixed31_32_sub(dal_fixed31_32_one, arr_points[1].y),
-			dal_fixed31_32_sub(end_value, arr_points[1].x));
+		arr_points[1].slope = dc_fixpt_div(
+			dc_fixpt_sub(dc_fixpt_one, arr_points[1].y),
+			dc_fixpt_sub(end_value, arr_points[1].x));
 	}
 
 	lut_params->hw_points_num = hw_points;
 
-	i = 1;
-	for (k = 0; k < MAX_REGIONS_NUMBER && i < MAX_REGIONS_NUMBER; k++) {
+	k = 0;
+	for (i = 1; i < MAX_REGIONS_NUMBER; i++) {
 		if (seg_distr[k] != -1) {
 			lut_params->arr_curve_points[k].segments_num =
 					seg_distr[k];
 			lut_params->arr_curve_points[i].offset =
 					lut_params->arr_curve_points[k].offset + (1 << seg_distr[k]);
 		}
-		i++;
+		k++;
 	}
 
 	if (seg_distr[k] != -1)
@@ -548,16 +548,16 @@ bool cm_helper_translate_curve_to_degamma_hw_format(
 
 	i = 1;
 	while (i != hw_points + 1) {
-		if (dal_fixed31_32_lt(rgb_plus_1->red, rgb->red))
+		if (dc_fixpt_lt(rgb_plus_1->red, rgb->red))
 			rgb_plus_1->red = rgb->red;
-		if (dal_fixed31_32_lt(rgb_plus_1->green, rgb->green))
+		if (dc_fixpt_lt(rgb_plus_1->green, rgb->green))
 			rgb_plus_1->green = rgb->green;
-		if (dal_fixed31_32_lt(rgb_plus_1->blue, rgb->blue))
+		if (dc_fixpt_lt(rgb_plus_1->blue, rgb->blue))
 			rgb_plus_1->blue = rgb->blue;
 
-		rgb->delta_red   = dal_fixed31_32_sub(rgb_plus_1->red,   rgb->red);
-		rgb->delta_green = dal_fixed31_32_sub(rgb_plus_1->green, rgb->green);
-		rgb->delta_blue  = dal_fixed31_32_sub(rgb_plus_1->blue,  rgb->blue);
+		rgb->delta_red   = dc_fixpt_sub(rgb_plus_1->red,   rgb->red);
+		rgb->delta_green = dc_fixpt_sub(rgb_plus_1->green, rgb->green);
+		rgb->delta_blue  = dc_fixpt_sub(rgb_plus_1->blue,  rgb->blue);
 
 		++rgb_plus_1;
 		++rgb;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
index e305c28c98de..46a35c7f01df 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
@@ -98,6 +98,30 @@ enum gamut_remap_select {
 	GAMUT_REMAP_COMB_COEFF
 };
 
+void dpp_read_state(struct dpp *dpp_base,
+		struct dcn_dpp_state *s)
+{
+	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
+
+	REG_GET(CM_IGAM_CONTROL,
+			CM_IGAM_LUT_MODE, &s->igam_lut_mode);
+	REG_GET(CM_IGAM_CONTROL,
+			CM_IGAM_INPUT_FORMAT, &s->igam_input_format);
+	REG_GET(CM_DGAM_CONTROL,
+			CM_DGAM_LUT_MODE, &s->dgam_lut_mode);
+	REG_GET(CM_RGAM_CONTROL,
+			CM_RGAM_LUT_MODE, &s->rgam_lut_mode);
+	REG_GET(CM_GAMUT_REMAP_CONTROL,
+			CM_GAMUT_REMAP_MODE, &s->gamut_remap_mode);
+
+	s->gamut_remap_c11_c12 = REG_READ(CM_GAMUT_REMAP_C11_C12);
+	s->gamut_remap_c13_c14 = REG_READ(CM_GAMUT_REMAP_C13_C14);
+	s->gamut_remap_c21_c22 = REG_READ(CM_GAMUT_REMAP_C21_C22);
+	s->gamut_remap_c23_c24 = REG_READ(CM_GAMUT_REMAP_C23_C24);
+	s->gamut_remap_c31_c32 = REG_READ(CM_GAMUT_REMAP_C31_C32);
+	s->gamut_remap_c33_c34 = REG_READ(CM_GAMUT_REMAP_C33_C34);
+}
+
 /* Program gamut remap in bypass mode */
 void dpp_set_gamut_remap_bypass(struct dcn10_dpp *dpp)
 {
@@ -106,7 +130,7 @@ void dpp_set_gamut_remap_bypass(struct dcn10_dpp *dpp)
 	/* Gamut remap in bypass */
 }
 
-#define IDENTITY_RATIO(ratio) (dal_fixed31_32_u2d19(ratio) == (1 << 19))
+#define IDENTITY_RATIO(ratio) (dc_fixpt_u2d19(ratio) == (1 << 19))
 
 
 bool dpp_get_optimal_number_of_taps(
@@ -121,6 +145,18 @@ bool dpp_get_optimal_number_of_taps(
 	else
 		pixel_width = scl_data->viewport.width;
 
+	/* Some ASICs does not support  FP16 scaling, so we reject modes require this*/
+	if (scl_data->viewport.width  != scl_data->h_active &&
+		scl_data->viewport.height != scl_data->v_active &&
+		dpp->caps->dscl_data_proc_format == DSCL_DATA_PRCESSING_FIXED_FORMAT &&
+		scl_data->format == PIXEL_FORMAT_FP16)
+		return false;
+
+	if (scl_data->viewport.width > scl_data->h_active &&
+		dpp->ctx->dc->debug.max_downscale_src_width != 0 &&
+		scl_data->viewport.width > dpp->ctx->dc->debug.max_downscale_src_width)
+		return false;
+
 	/* TODO: add lb check */
 
 	/* No support for programming ratio of 4, drop to 3.99999.. */
@@ -257,7 +293,7 @@ void dpp1_cnv_setup (
 		struct dpp *dpp_base,
 		enum surface_pixel_format format,
 		enum expansion_mode mode,
-		struct csc_transform input_csc_color_matrix,
+		struct dc_csc_transform input_csc_color_matrix,
 		enum dc_color_space input_color_space)
 {
 	uint32_t pixel_format;
@@ -416,7 +452,7 @@ void dpp1_set_cursor_position(
 	if (src_x_offset >= (int)param->viewport_width)
 		cur_en = 0;  /* not visible beyond right edge*/
 
-	if (src_x_offset + (int)width < 0)
+	if (src_x_offset + (int)width <= 0)
 		cur_en = 0;  /* not visible beyond left edge*/
 
 	REG_UPDATE(CURSOR0_CONTROL,
@@ -443,6 +479,7 @@ void dpp1_dppclk_control(
 }
 
 static const struct dpp_funcs dcn10_dpp_funcs = {
+		.dpp_read_state = dpp_read_state,
 		.dpp_reset = dpp_reset,
 		.dpp_set_scaler = dpp1_dscl_set_scaler_manual_scale,
 		.dpp_get_optimal_number_of_taps = dpp_get_optimal_number_of_taps,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
index 17b062a8f88a..5944a3ba0409 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
@@ -44,6 +44,10 @@
 #define TF_REG_LIST_DCN(id) \
 	SRI(CM_GAMUT_REMAP_CONTROL, CM, id),\
 	SRI(CM_GAMUT_REMAP_C11_C12, CM, id),\
+	SRI(CM_GAMUT_REMAP_C13_C14, CM, id),\
+	SRI(CM_GAMUT_REMAP_C21_C22, CM, id),\
+	SRI(CM_GAMUT_REMAP_C23_C24, CM, id),\
+	SRI(CM_GAMUT_REMAP_C31_C32, CM, id),\
 	SRI(CM_GAMUT_REMAP_C33_C34, CM, id),\
 	SRI(DSCL_EXT_OVERSCAN_LEFT_RIGHT, DSCL, id), \
 	SRI(DSCL_EXT_OVERSCAN_TOP_BOTTOM, DSCL, id), \
@@ -108,6 +112,8 @@
 	SRI(CM_DGAM_LUT_DATA, CM, id), \
 	SRI(CM_CONTROL, CM, id), \
 	SRI(CM_DGAM_CONTROL, CM, id), \
+	SRI(CM_TEST_DEBUG_INDEX, CM, id), \
+	SRI(CM_TEST_DEBUG_DATA, CM, id), \
 	SRI(FORMAT_CONTROL, CNVC_CFG, id), \
 	SRI(CNVC_SURFACE_PIXEL_FORMAT, CNVC_CFG, id), \
 	SRI(CURSOR0_CONTROL, CNVC_CUR, id), \
@@ -175,6 +181,14 @@
 	TF_SF(CM0_CM_GAMUT_REMAP_CONTROL, CM_GAMUT_REMAP_MODE, mask_sh),\
 	TF_SF(CM0_CM_GAMUT_REMAP_C11_C12, CM_GAMUT_REMAP_C11, mask_sh),\
 	TF_SF(CM0_CM_GAMUT_REMAP_C11_C12, CM_GAMUT_REMAP_C12, mask_sh),\
+	TF_SF(CM0_CM_GAMUT_REMAP_C13_C14, CM_GAMUT_REMAP_C13, mask_sh),\
+	TF_SF(CM0_CM_GAMUT_REMAP_C13_C14, CM_GAMUT_REMAP_C14, mask_sh),\
+	TF_SF(CM0_CM_GAMUT_REMAP_C21_C22, CM_GAMUT_REMAP_C21, mask_sh),\
+	TF_SF(CM0_CM_GAMUT_REMAP_C21_C22, CM_GAMUT_REMAP_C22, mask_sh),\
+	TF_SF(CM0_CM_GAMUT_REMAP_C23_C24, CM_GAMUT_REMAP_C23, mask_sh),\
+	TF_SF(CM0_CM_GAMUT_REMAP_C23_C24, CM_GAMUT_REMAP_C24, mask_sh),\
+	TF_SF(CM0_CM_GAMUT_REMAP_C31_C32, CM_GAMUT_REMAP_C31, mask_sh),\
+	TF_SF(CM0_CM_GAMUT_REMAP_C31_C32, CM_GAMUT_REMAP_C32, mask_sh),\
 	TF_SF(CM0_CM_GAMUT_REMAP_C33_C34, CM_GAMUT_REMAP_C33, mask_sh),\
 	TF_SF(CM0_CM_GAMUT_REMAP_C33_C34, CM_GAMUT_REMAP_C34, mask_sh),\
 	TF_SF(DSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT, EXT_OVERSCAN_LEFT, mask_sh),\
@@ -300,6 +314,7 @@
 	TF_SF(CM0_CM_DGAM_LUT_INDEX, CM_DGAM_LUT_INDEX, mask_sh), \
 	TF_SF(CM0_CM_DGAM_LUT_DATA, CM_DGAM_LUT_DATA, mask_sh), \
 	TF_SF(CM0_CM_DGAM_CONTROL, CM_DGAM_LUT_MODE, mask_sh), \
+	TF_SF(CM0_CM_TEST_DEBUG_INDEX, CM_TEST_DEBUG_INDEX, mask_sh), \
 	TF_SF(CNVC_CFG0_FORMAT_CONTROL, CNVC_BYPASS, mask_sh), \
 	TF2_SF(CNVC_CFG0, FORMAT_CONTROL__ALPHA_EN, mask_sh), \
 	TF_SF(CNVC_CFG0_FORMAT_CONTROL, FORMAT_EXPANSION_MODE, mask_sh), \
@@ -417,6 +432,41 @@
 	TF_SF(CURSOR0_CURSOR_CONTROL, CURSOR_ENABLE, mask_sh), \
 	TF_SF(DPP_TOP0_DPP_CONTROL, DPPCLK_RATE_CONTROL, mask_sh)
 
+/*
+ *
+	DCN1 CM debug status register definition
+
+	register :ID9_CM_STATUS do
+	implement_ref :cm
+	map to:  :cmdebugind, at: j
+	width 32
+	disclosure   NEVER
+
+		field :ID9_VUPDATE_CFG, [0], R
+		field :ID9_IGAM_LUT_MODE, [2..1], R
+		field :ID9_BNS_BYPASS, [3], R
+		field :ID9_ICSC_MODE, [5..4], R
+		field :ID9_DGAM_LUT_MODE, [8..6], R
+		field :ID9_HDR_BYPASS, [9], R
+		field :ID9_GAMUT_REMAP_MODE, [11..10], R
+		field :ID9_RGAM_LUT_MODE, [14..12], R
+		#1 free bit
+		field :ID9_OCSC_MODE, [18..16], R
+		field :ID9_DENORM_MODE, [21..19], R
+		field :ID9_ROUND_TRUNC_MODE, [25..22], R
+		field :ID9_DITHER_EN, [26], R
+		field :ID9_DITHER_MODE, [28..27], R
+	end
+*/
+
+#define TF_DEBUG_REG_LIST_SH_DCN10 \
+	.CM_TEST_DEBUG_DATA_ID9_ICSC_MODE = 4, \
+	.CM_TEST_DEBUG_DATA_ID9_OCSC_MODE = 16
+
+#define TF_DEBUG_REG_LIST_MASK_DCN10 \
+	.CM_TEST_DEBUG_DATA_ID9_ICSC_MODE = 0x30, \
+	.CM_TEST_DEBUG_DATA_ID9_OCSC_MODE = 0x70000
+
 #define TF_REG_FIELD_LIST(type) \
 	type EXT_OVERSCAN_LEFT; \
 	type EXT_OVERSCAN_RIGHT; \
@@ -486,6 +536,14 @@
 	type CM_GAMUT_REMAP_MODE; \
 	type CM_GAMUT_REMAP_C11; \
 	type CM_GAMUT_REMAP_C12; \
+	type CM_GAMUT_REMAP_C13; \
+	type CM_GAMUT_REMAP_C14; \
+	type CM_GAMUT_REMAP_C21; \
+	type CM_GAMUT_REMAP_C22; \
+	type CM_GAMUT_REMAP_C23; \
+	type CM_GAMUT_REMAP_C24; \
+	type CM_GAMUT_REMAP_C31; \
+	type CM_GAMUT_REMAP_C32; \
 	type CM_GAMUT_REMAP_C33; \
 	type CM_GAMUT_REMAP_C34; \
 	type CM_COMA_C11; \
@@ -1010,6 +1068,9 @@
 	type CUR0_EXPANSION_MODE; \
 	type CUR0_ENABLE; \
 	type CM_BYPASS; \
+	type CM_TEST_DEBUG_INDEX; \
+	type CM_TEST_DEBUG_DATA_ID9_ICSC_MODE; \
+	type CM_TEST_DEBUG_DATA_ID9_OCSC_MODE;\
 	type FORMAT_CONTROL__ALPHA_EN; \
 	type CUR0_COLOR0; \
 	type CUR0_COLOR1; \
@@ -1054,6 +1115,10 @@ struct dcn_dpp_mask {
 	uint32_t RECOUT_SIZE; \
 	uint32_t CM_GAMUT_REMAP_CONTROL; \
 	uint32_t CM_GAMUT_REMAP_C11_C12; \
+	uint32_t CM_GAMUT_REMAP_C13_C14; \
+	uint32_t CM_GAMUT_REMAP_C21_C22; \
+	uint32_t CM_GAMUT_REMAP_C23_C24; \
+	uint32_t CM_GAMUT_REMAP_C31_C32; \
 	uint32_t CM_GAMUT_REMAP_C33_C34; \
 	uint32_t CM_COMA_C11_C12; \
 	uint32_t CM_COMA_C33_C34; \
@@ -1255,6 +1320,8 @@ struct dcn_dpp_mask {
 	uint32_t CM_IGAM_LUT_RW_CONTROL; \
 	uint32_t CM_IGAM_LUT_RW_INDEX; \
 	uint32_t CM_IGAM_LUT_SEQ_COLOR; \
+	uint32_t CM_TEST_DEBUG_INDEX; \
+	uint32_t CM_TEST_DEBUG_DATA; \
 	uint32_t FORMAT_CONTROL; \
 	uint32_t CNVC_SURFACE_PIXEL_FORMAT; \
 	uint32_t CURSOR_CONTROL; \
@@ -1289,8 +1356,8 @@ struct dcn10_dpp {
 
 enum dcn10_input_csc_select {
 	INPUT_CSC_SELECT_BYPASS = 0,
-	INPUT_CSC_SELECT_ICSC,
-	INPUT_CSC_SELECT_COMA
+	INPUT_CSC_SELECT_ICSC = 1,
+	INPUT_CSC_SELECT_COMA = 2
 };
 
 void dpp1_set_cursor_attributes(
@@ -1364,6 +1431,9 @@ bool dpp_get_optimal_number_of_taps(
 		struct scaler_data *scl_data,
 		const struct scaling_taps *in_taps);
 
+void dpp_read_state(struct dpp *dpp_base,
+		struct dcn_dpp_state *s);
+
 void dpp_reset(struct dpp *dpp_base);
 
 void dpp1_cm_program_regamma_lut(
@@ -1408,7 +1478,7 @@ void dpp1_cnv_setup (
 		struct dpp *dpp_base,
 		enum surface_pixel_format format,
 		enum expansion_mode mode,
-		struct csc_transform input_csc_color_matrix,
+		struct dc_csc_transform input_csc_color_matrix,
 		enum dc_color_space input_color_space);
 
 void dpp1_full_bypass(struct dpp *dpp_base);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
index fb32975e4b67..116977eb24e2 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
@@ -216,41 +216,55 @@ static void dpp1_cm_program_color_matrix(
 		struct dcn10_dpp *dpp,
 		const uint16_t *regval)
 {
-	uint32_t mode;
+	uint32_t ocsc_mode;
+	uint32_t cur_mode;
 	struct color_matrices_reg gam_regs;
 
-	REG_GET(CM_OCSC_CONTROL, CM_OCSC_MODE, &mode);
-
 	if (regval == NULL) {
 		BREAK_TO_DEBUGGER();
 		return;
 	}
-	mode = 4;
+
+	/* determine which CSC matrix (ocsc or comb) we are using
+	 * currently.  select the alternate set to double buffer
+	 * the CSC update so CSC is updated on frame boundary
+	 */
+	REG_SET(CM_TEST_DEBUG_INDEX, 0,
+			CM_TEST_DEBUG_INDEX, 9);
+
+	REG_GET(CM_TEST_DEBUG_DATA,
+			CM_TEST_DEBUG_DATA_ID9_OCSC_MODE, &cur_mode);
+
+	if (cur_mode != 4)
+		ocsc_mode = 4;
+	else
+		ocsc_mode = 5;
+
+
 	gam_regs.shifts.csc_c11 = dpp->tf_shift->CM_OCSC_C11;
 	gam_regs.masks.csc_c11  = dpp->tf_mask->CM_OCSC_C11;
 	gam_regs.shifts.csc_c12 = dpp->tf_shift->CM_OCSC_C12;
 	gam_regs.masks.csc_c12 = dpp->tf_mask->CM_OCSC_C12;
 
-	if (mode == 4) {
+	if (ocsc_mode == 4) {
 
 		gam_regs.csc_c11_c12 = REG(CM_OCSC_C11_C12);
 		gam_regs.csc_c33_c34 = REG(CM_OCSC_C33_C34);
 
-		cm_helper_program_color_matrices(
-				dpp->base.ctx,
-				regval,
-				&gam_regs);
-
 	} else {
 
 		gam_regs.csc_c11_c12 = REG(CM_COMB_C11_C12);
 		gam_regs.csc_c33_c34 = REG(CM_COMB_C33_C34);
 
-		cm_helper_program_color_matrices(
-				dpp->base.ctx,
-				regval,
-				&gam_regs);
 	}
+
+	cm_helper_program_color_matrices(
+			dpp->base.ctx,
+			regval,
+			&gam_regs);
+
+	REG_SET(CM_OCSC_CONTROL, 0, CM_OCSC_MODE, ocsc_mode);
+
 }
 
 void dpp1_cm_set_output_csc_default(
@@ -260,15 +274,14 @@ void dpp1_cm_set_output_csc_default(
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
 	const uint16_t *regval = NULL;
 	int arr_size;
-	uint32_t ocsc_mode = 4;
 
 	regval = find_color_matrix(colorspace, &arr_size);
 	if (regval == NULL) {
 		BREAK_TO_DEBUGGER();
 		return;
 	}
+
 	dpp1_cm_program_color_matrix(dpp, regval);
-	REG_SET(CM_OCSC_CONTROL, 0, CM_OCSC_MODE, ocsc_mode);
 }
 
 static void dpp1_cm_get_reg_field(
@@ -329,9 +342,8 @@ void dpp1_cm_set_output_csc_adjustment(
 		const uint16_t *regval)
 {
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
-	uint32_t ocsc_mode = 4;
+
 	dpp1_cm_program_color_matrix(dpp, regval);
-	REG_SET(CM_OCSC_CONTROL, 0, CM_OCSC_MODE, ocsc_mode);
 }
 
 void dpp1_cm_power_on_regamma_lut(struct dpp *dpp_base,
@@ -437,17 +449,18 @@ void dpp1_cm_program_regamma_lutb_settings(
 void dpp1_program_input_csc(
 		struct dpp *dpp_base,
 		enum dc_color_space color_space,
-		enum dcn10_input_csc_select select,
+		enum dcn10_input_csc_select input_select,
 		const struct out_csc_color_matrix *tbl_entry)
 {
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
 	int i;
 	int arr_size = sizeof(dcn10_input_csc_matrix)/sizeof(struct dcn10_input_csc_matrix);
 	const uint16_t *regval = NULL;
-	uint32_t selection = 1;
+	uint32_t cur_select = 0;
+	enum dcn10_input_csc_select select;
 	struct color_matrices_reg gam_regs;
 
-	if (select == INPUT_CSC_SELECT_BYPASS) {
+	if (input_select == INPUT_CSC_SELECT_BYPASS) {
 		REG_SET(CM_ICSC_CONTROL, 0, CM_ICSC_MODE, 0);
 		return;
 	}
@@ -467,36 +480,45 @@ void dpp1_program_input_csc(
 		regval = tbl_entry->regval;
 	}
 
-	if (select == INPUT_CSC_SELECT_COMA)
-		selection = 2;
-	REG_SET(CM_ICSC_CONTROL, 0,
-			CM_ICSC_MODE, selection);
+	/* determine which CSC matrix (icsc or coma) we are using
+	 * currently.  select the alternate set to double buffer
+	 * the CSC update so CSC is updated on frame boundary
+	 */
+	REG_SET(CM_TEST_DEBUG_INDEX, 0,
+			CM_TEST_DEBUG_INDEX, 9);
+
+	REG_GET(CM_TEST_DEBUG_DATA,
+			CM_TEST_DEBUG_DATA_ID9_ICSC_MODE, &cur_select);
+
+	if (cur_select != INPUT_CSC_SELECT_ICSC)
+		select = INPUT_CSC_SELECT_ICSC;
+	else
+		select = INPUT_CSC_SELECT_COMA;
 
 	gam_regs.shifts.csc_c11 = dpp->tf_shift->CM_ICSC_C11;
 	gam_regs.masks.csc_c11  = dpp->tf_mask->CM_ICSC_C11;
 	gam_regs.shifts.csc_c12 = dpp->tf_shift->CM_ICSC_C12;
 	gam_regs.masks.csc_c12 = dpp->tf_mask->CM_ICSC_C12;
 
-
 	if (select == INPUT_CSC_SELECT_ICSC) {
 
 		gam_regs.csc_c11_c12 = REG(CM_ICSC_C11_C12);
 		gam_regs.csc_c33_c34 = REG(CM_ICSC_C33_C34);
 
-		cm_helper_program_color_matrices(
-				dpp->base.ctx,
-				regval,
-				&gam_regs);
 	} else {
 
 		gam_regs.csc_c11_c12 = REG(CM_COMA_C11_C12);
 		gam_regs.csc_c33_c34 = REG(CM_COMA_C33_C34);
 
-		cm_helper_program_color_matrices(
-				dpp->base.ctx,
-				regval,
-				&gam_regs);
 	}
+
+	cm_helper_program_color_matrices(
+			dpp->base.ctx,
+			regval,
+			&gam_regs);
+
+	REG_SET(CM_ICSC_CONTROL, 0,
+				CM_ICSC_MODE, select);
 }
 
 //keep here for now, decide multi dce support later
@@ -789,13 +811,13 @@ void dpp1_program_input_lut(
 	REG_UPDATE(CM_IGAM_LUT_RW_INDEX, CM_IGAM_LUT_RW_INDEX, 0);
 	for (i = 0; i < gamma->num_entries; i++) {
 		REG_SET(CM_IGAM_LUT_SEQ_COLOR, 0, CM_IGAM_LUT_SEQ_COLOR,
-				dal_fixed31_32_round(
+				dc_fixpt_round(
 					gamma->entries.red[i]));
 		REG_SET(CM_IGAM_LUT_SEQ_COLOR, 0, CM_IGAM_LUT_SEQ_COLOR,
-				dal_fixed31_32_round(
+				dc_fixpt_round(
 					gamma->entries.green[i]));
 		REG_SET(CM_IGAM_LUT_SEQ_COLOR, 0, CM_IGAM_LUT_SEQ_COLOR,
-				dal_fixed31_32_round(
+				dc_fixpt_round(
 					gamma->entries.blue[i]));
 	}
 	// Power off LUT memory
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
index 3eb824debf43..4ddd6273d5a5 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
@@ -169,7 +169,7 @@ static enum dscl_mode_sel dpp1_dscl_get_dscl_mode(
 		const struct scaler_data *data,
 		bool dbg_always_scale)
 {
-	const long long one = dal_fixed31_32_one.value;
+	const long long one = dc_fixpt_one.value;
 
 	if (dpp_base->caps->dscl_data_proc_format == DSCL_DATA_PRCESSING_FIXED_FORMAT) {
 		/* DSCL is processing data in fixed format */
@@ -464,8 +464,8 @@ static enum lb_memory_config dpp1_dscl_find_lb_memory_config(struct dcn10_dpp *d
 	int num_part_y, num_part_c;
 	int vtaps = scl_data->taps.v_taps;
 	int vtaps_c = scl_data->taps.v_taps_c;
-	int ceil_vratio = dal_fixed31_32_ceil(scl_data->ratios.vert);
-	int ceil_vratio_c = dal_fixed31_32_ceil(scl_data->ratios.vert_c);
+	int ceil_vratio = dc_fixpt_ceil(scl_data->ratios.vert);
+	int ceil_vratio_c = dc_fixpt_ceil(scl_data->ratios.vert_c);
 	enum lb_memory_config mem_cfg = LB_MEMORY_CONFIG_0;
 
 	if (dpp->base.ctx->dc->debug.use_max_lb)
@@ -565,52 +565,52 @@ static void dpp1_dscl_set_manual_ratio_init(
 	uint32_t init_int = 0;
 
 	REG_SET(SCL_HORZ_FILTER_SCALE_RATIO, 0,
-			SCL_H_SCALE_RATIO, dal_fixed31_32_u2d19(data->ratios.horz) << 5);
+			SCL_H_SCALE_RATIO, dc_fixpt_u2d19(data->ratios.horz) << 5);
 
 	REG_SET(SCL_VERT_FILTER_SCALE_RATIO, 0,
-			SCL_V_SCALE_RATIO, dal_fixed31_32_u2d19(data->ratios.vert) << 5);
+			SCL_V_SCALE_RATIO, dc_fixpt_u2d19(data->ratios.vert) << 5);
 
 	REG_SET(SCL_HORZ_FILTER_SCALE_RATIO_C, 0,
-			SCL_H_SCALE_RATIO_C, dal_fixed31_32_u2d19(data->ratios.horz_c) << 5);
+			SCL_H_SCALE_RATIO_C, dc_fixpt_u2d19(data->ratios.horz_c) << 5);
 
 	REG_SET(SCL_VERT_FILTER_SCALE_RATIO_C, 0,
-			SCL_V_SCALE_RATIO_C, dal_fixed31_32_u2d19(data->ratios.vert_c) << 5);
+			SCL_V_SCALE_RATIO_C, dc_fixpt_u2d19(data->ratios.vert_c) << 5);
 
 	/*
 	 * 0.24 format for fraction, first five bits zeroed
 	 */
-	init_frac = dal_fixed31_32_u0d19(data->inits.h) << 5;
-	init_int = dal_fixed31_32_floor(data->inits.h);
+	init_frac = dc_fixpt_u0d19(data->inits.h) << 5;
+	init_int = dc_fixpt_floor(data->inits.h);
 	REG_SET_2(SCL_HORZ_FILTER_INIT, 0,
 		SCL_H_INIT_FRAC, init_frac,
 		SCL_H_INIT_INT, init_int);
 
-	init_frac = dal_fixed31_32_u0d19(data->inits.h_c) << 5;
-	init_int = dal_fixed31_32_floor(data->inits.h_c);
+	init_frac = dc_fixpt_u0d19(data->inits.h_c) << 5;
+	init_int = dc_fixpt_floor(data->inits.h_c);
 	REG_SET_2(SCL_HORZ_FILTER_INIT_C, 0,
 		SCL_H_INIT_FRAC_C, init_frac,
 		SCL_H_INIT_INT_C, init_int);
 
-	init_frac = dal_fixed31_32_u0d19(data->inits.v) << 5;
-	init_int = dal_fixed31_32_floor(data->inits.v);
+	init_frac = dc_fixpt_u0d19(data->inits.v) << 5;
+	init_int = dc_fixpt_floor(data->inits.v);
 	REG_SET_2(SCL_VERT_FILTER_INIT, 0,
 		SCL_V_INIT_FRAC, init_frac,
 		SCL_V_INIT_INT, init_int);
 
-	init_frac = dal_fixed31_32_u0d19(data->inits.v_bot) << 5;
-	init_int = dal_fixed31_32_floor(data->inits.v_bot);
+	init_frac = dc_fixpt_u0d19(data->inits.v_bot) << 5;
+	init_int = dc_fixpt_floor(data->inits.v_bot);
 	REG_SET_2(SCL_VERT_FILTER_INIT_BOT, 0,
 		SCL_V_INIT_FRAC_BOT, init_frac,
 		SCL_V_INIT_INT_BOT, init_int);
 
-	init_frac = dal_fixed31_32_u0d19(data->inits.v_c) << 5;
-	init_int = dal_fixed31_32_floor(data->inits.v_c);
+	init_frac = dc_fixpt_u0d19(data->inits.v_c) << 5;
+	init_int = dc_fixpt_floor(data->inits.v_c);
 	REG_SET_2(SCL_VERT_FILTER_INIT_C, 0,
 		SCL_V_INIT_FRAC_C, init_frac,
 		SCL_V_INIT_INT_C, init_int);
 
-	init_frac = dal_fixed31_32_u0d19(data->inits.v_c_bot) << 5;
-	init_int = dal_fixed31_32_floor(data->inits.v_c_bot);
+	init_frac = dc_fixpt_u0d19(data->inits.v_c_bot) << 5;
+	init_int = dc_fixpt_floor(data->inits.v_c_bot);
 	REG_SET_2(SCL_VERT_FILTER_INIT_BOT_C, 0,
 		SCL_V_INIT_FRAC_BOT_C, init_frac,
 		SCL_V_INIT_INT_BOT_C, init_int);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
index 738f67ffd1b4..943143efbb82 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
@@ -476,8 +476,235 @@ void hubbub1_toggle_watermark_change_req(struct hubbub *hubbub)
 			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, watermark_change_req);
 }
 
+void hubbub1_soft_reset(struct hubbub *hubbub, bool reset)
+{
+	uint32_t reset_en = reset ? 1 : 0;
+
+	REG_UPDATE(DCHUBBUB_SOFT_RESET,
+			DCHUBBUB_GLOBAL_SOFT_RESET, reset_en);
+}
+
+static bool hubbub1_dcc_support_swizzle(
+		enum swizzle_mode_values swizzle,
+		unsigned int bytes_per_element,
+		enum segment_order *segment_order_horz,
+		enum segment_order *segment_order_vert)
+{
+	bool standard_swizzle = false;
+	bool display_swizzle = false;
+
+	switch (swizzle) {
+	case DC_SW_4KB_S:
+	case DC_SW_64KB_S:
+	case DC_SW_VAR_S:
+	case DC_SW_4KB_S_X:
+	case DC_SW_64KB_S_X:
+	case DC_SW_VAR_S_X:
+		standard_swizzle = true;
+		break;
+	case DC_SW_4KB_D:
+	case DC_SW_64KB_D:
+	case DC_SW_VAR_D:
+	case DC_SW_4KB_D_X:
+	case DC_SW_64KB_D_X:
+	case DC_SW_VAR_D_X:
+		display_swizzle = true;
+		break;
+	default:
+		break;
+	}
+
+	if (bytes_per_element == 1 && standard_swizzle) {
+		*segment_order_horz = segment_order__contiguous;
+		*segment_order_vert = segment_order__na;
+		return true;
+	}
+	if (bytes_per_element == 2 && standard_swizzle) {
+		*segment_order_horz = segment_order__non_contiguous;
+		*segment_order_vert = segment_order__contiguous;
+		return true;
+	}
+	if (bytes_per_element == 4 && standard_swizzle) {
+		*segment_order_horz = segment_order__non_contiguous;
+		*segment_order_vert = segment_order__contiguous;
+		return true;
+	}
+	if (bytes_per_element == 8 && standard_swizzle) {
+		*segment_order_horz = segment_order__na;
+		*segment_order_vert = segment_order__contiguous;
+		return true;
+	}
+	if (bytes_per_element == 8 && display_swizzle) {
+		*segment_order_horz = segment_order__contiguous;
+		*segment_order_vert = segment_order__non_contiguous;
+		return true;
+	}
+
+	return false;
+}
+
+static bool hubbub1_dcc_support_pixel_format(
+		enum surface_pixel_format format,
+		unsigned int *bytes_per_element)
+{
+	/* DML: get_bytes_per_element */
+	switch (format) {
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB1555:
+	case SURFACE_PIXEL_FORMAT_GRPH_RGB565:
+		*bytes_per_element = 2;
+		return true;
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB8888:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR8888:
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010:
+		*bytes_per_element = 4;
+		return true;
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616:
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F:
+		*bytes_per_element = 8;
+		return true;
+	default:
+		return false;
+	}
+}
+
+static void hubbub1_get_blk256_size(unsigned int *blk256_width, unsigned int *blk256_height,
+		unsigned int bytes_per_element)
+{
+	/* copied from DML.  might want to refactor DML to leverage from DML */
+	/* DML : get_blk256_size */
+	if (bytes_per_element == 1) {
+		*blk256_width = 16;
+		*blk256_height = 16;
+	} else if (bytes_per_element == 2) {
+		*blk256_width = 16;
+		*blk256_height = 8;
+	} else if (bytes_per_element == 4) {
+		*blk256_width = 8;
+		*blk256_height = 8;
+	} else if (bytes_per_element == 8) {
+		*blk256_width = 8;
+		*blk256_height = 4;
+	}
+}
+
+static void hubbub1_det_request_size(
+		unsigned int height,
+		unsigned int width,
+		unsigned int bpe,
+		bool *req128_horz_wc,
+		bool *req128_vert_wc)
+{
+	unsigned int detile_buf_size = 164 * 1024;  /* 164KB for DCN1.0 */
+
+	unsigned int blk256_height = 0;
+	unsigned int blk256_width = 0;
+	unsigned int swath_bytes_horz_wc, swath_bytes_vert_wc;
+
+	hubbub1_get_blk256_size(&blk256_width, &blk256_height, bpe);
+
+	swath_bytes_horz_wc = height * blk256_height * bpe;
+	swath_bytes_vert_wc = width * blk256_width * bpe;
+
+	*req128_horz_wc = (2 * swath_bytes_horz_wc <= detile_buf_size) ?
+			false : /* full 256B request */
+			true; /* half 128b request */
+
+	*req128_vert_wc = (2 * swath_bytes_vert_wc <= detile_buf_size) ?
+			false : /* full 256B request */
+			true; /* half 128b request */
+}
+
+static bool hubbub1_get_dcc_compression_cap(struct hubbub *hubbub,
+		const struct dc_dcc_surface_param *input,
+		struct dc_surface_dcc_cap *output)
+{
+	struct dc *dc = hubbub->ctx->dc;
+	/* implement section 1.6.2.1 of DCN1_Programming_Guide.docx */
+	enum dcc_control dcc_control;
+	unsigned int bpe;
+	enum segment_order segment_order_horz, segment_order_vert;
+	bool req128_horz_wc, req128_vert_wc;
+
+	memset(output, 0, sizeof(*output));
+
+	if (dc->debug.disable_dcc == DCC_DISABLE)
+		return false;
+
+	if (!hubbub->funcs->dcc_support_pixel_format(input->format, &bpe))
+		return false;
+
+	if (!hubbub->funcs->dcc_support_swizzle(input->swizzle_mode, bpe,
+			&segment_order_horz, &segment_order_vert))
+		return false;
+
+	hubbub1_det_request_size(input->surface_size.height,  input->surface_size.width,
+			bpe, &req128_horz_wc, &req128_vert_wc);
+
+	if (!req128_horz_wc && !req128_vert_wc) {
+		dcc_control = dcc_control__256_256_xxx;
+	} else if (input->scan == SCAN_DIRECTION_HORIZONTAL) {
+		if (!req128_horz_wc)
+			dcc_control = dcc_control__256_256_xxx;
+		else if (segment_order_horz == segment_order__contiguous)
+			dcc_control = dcc_control__128_128_xxx;
+		else
+			dcc_control = dcc_control__256_64_64;
+	} else if (input->scan == SCAN_DIRECTION_VERTICAL) {
+		if (!req128_vert_wc)
+			dcc_control = dcc_control__256_256_xxx;
+		else if (segment_order_vert == segment_order__contiguous)
+			dcc_control = dcc_control__128_128_xxx;
+		else
+			dcc_control = dcc_control__256_64_64;
+	} else {
+		if ((req128_horz_wc &&
+			segment_order_horz == segment_order__non_contiguous) ||
+			(req128_vert_wc &&
+			segment_order_vert == segment_order__non_contiguous))
+			/* access_dir not known, must use most constraining */
+			dcc_control = dcc_control__256_64_64;
+		else
+			/* reg128 is true for either horz and vert
+			 * but segment_order is contiguous
+			 */
+			dcc_control = dcc_control__128_128_xxx;
+	}
+
+	if (dc->debug.disable_dcc == DCC_HALF_REQ_DISALBE &&
+		dcc_control != dcc_control__256_256_xxx)
+		return false;
+
+	switch (dcc_control) {
+	case dcc_control__256_256_xxx:
+		output->grph.rgb.max_uncompressed_blk_size = 256;
+		output->grph.rgb.max_compressed_blk_size = 256;
+		output->grph.rgb.independent_64b_blks = false;
+		break;
+	case dcc_control__128_128_xxx:
+		output->grph.rgb.max_uncompressed_blk_size = 128;
+		output->grph.rgb.max_compressed_blk_size = 128;
+		output->grph.rgb.independent_64b_blks = false;
+		break;
+	case dcc_control__256_64_64:
+		output->grph.rgb.max_uncompressed_blk_size = 256;
+		output->grph.rgb.max_compressed_blk_size = 64;
+		output->grph.rgb.independent_64b_blks = true;
+		break;
+	}
+
+	output->capable = true;
+	output->const_color_support = false;
+
+	return true;
+}
+
 static const struct hubbub_funcs hubbub1_funcs = {
-	.update_dchub = hubbub1_update_dchub
+	.update_dchub = hubbub1_update_dchub,
+	.dcc_support_swizzle = hubbub1_dcc_support_swizzle,
+	.dcc_support_pixel_format = hubbub1_dcc_support_pixel_format,
+	.get_dcc_compression_cap = hubbub1_get_dcc_compression_cap,
 };
 
 void hubbub1_construct(struct hubbub *hubbub,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
index a16e908821a0..6315a0e6b0d6 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
@@ -27,6 +27,7 @@
 #define __DC_HUBBUB_DCN10_H__
 
 #include "core_types.h"
+#include "dchubbub.h"
 
 #define HUBHUB_REG_LIST_DCN()\
 	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A),\
@@ -47,7 +48,8 @@
 	SR(DCHUBBUB_ARB_DF_REQ_OUTSTAND),\
 	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
 	SR(DCHUBBUB_TEST_DEBUG_INDEX), \
-	SR(DCHUBBUB_TEST_DEBUG_DATA)
+	SR(DCHUBBUB_TEST_DEBUG_DATA),\
+	SR(DCHUBBUB_SOFT_RESET)
 
 #define HUBBUB_SR_WATERMARK_REG_LIST()\
 	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A),\
@@ -104,6 +106,7 @@ struct dcn_hubbub_registers {
 	uint32_t DCHUBBUB_SDPIF_AGP_BOT;
 	uint32_t DCHUBBUB_SDPIF_AGP_TOP;
 	uint32_t DCHUBBUB_CRC_CTRL;
+	uint32_t DCHUBBUB_SOFT_RESET;
 };
 
 /* set field name */
@@ -113,6 +116,7 @@ struct dcn_hubbub_registers {
 
 #define HUBBUB_MASK_SH_LIST_DCN(mask_sh)\
 		HUBBUB_SF(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_ENABLE, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_SOFT_RESET, DCHUBBUB_GLOBAL_SOFT_RESET, mask_sh), \
 		HUBBUB_SF(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL, DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, mask_sh), \
 		HUBBUB_SF(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL, DCHUBBUB_ARB_WATERMARK_CHANGE_DONE_INTERRUPT_DISABLE, mask_sh), \
 		HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_VALUE, mask_sh), \
@@ -142,6 +146,7 @@ struct dcn_hubbub_registers {
 		type DCHUBBUB_ARB_SAT_LEVEL;\
 		type DCHUBBUB_ARB_MIN_REQ_OUTSTAND;\
 		type DCHUBBUB_GLOBAL_TIMER_REFDIV;\
+		type DCHUBBUB_GLOBAL_SOFT_RESET; \
 		type SDPIF_FB_TOP;\
 		type SDPIF_FB_BASE;\
 		type SDPIF_FB_OFFSET;\
@@ -173,12 +178,6 @@ struct dcn_hubbub_wm {
 	struct dcn_hubbub_wm_set sets[4];
 };
 
-struct hubbub_funcs {
-	void (*update_dchub)(
-			struct hubbub *hubbub,
-			struct dchub_init_data *dh_data);
-};
-
 struct hubbub {
 	const struct hubbub_funcs *funcs;
 	struct dc_context *ctx;
@@ -206,6 +205,7 @@ void hubbub1_toggle_watermark_change_req(
 void hubbub1_wm_read_state(struct hubbub *hubbub,
 		struct dcn_hubbub_wm *wm);
 
+void hubbub1_soft_reset(struct hubbub *hubbub, bool reset);
 void hubbub1_construct(struct hubbub *hubbub,
 	struct dc_context *ctx,
 	const struct dcn_hubbub_registers *hubbub_regs,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
index 39b72f696ae9..d2ab78b35a7a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
@@ -78,6 +78,27 @@ static void hubp1_disconnect(struct hubp *hubp)
 			CURSOR_ENABLE, 0);
 }
 
+static void hubp1_disable_control(struct hubp *hubp, bool disable_hubp)
+{
+	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
+	uint32_t disable = disable_hubp ? 1 : 0;
+
+	REG_UPDATE(DCHUBP_CNTL,
+			HUBP_DISABLE, disable);
+}
+
+static unsigned int hubp1_get_underflow_status(struct hubp *hubp)
+{
+	uint32_t hubp_underflow = 0;
+	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
+
+	REG_GET(DCHUBP_CNTL,
+		HUBP_UNDERFLOW_STATUS,
+		&hubp_underflow);
+
+	return hubp_underflow;
+}
+
 static void hubp1_set_hubp_blank_en(struct hubp *hubp, bool blank)
 {
 	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
@@ -146,6 +167,9 @@ void hubp1_program_size_and_rotation(
 	 * 444 or 420 luma
 	 */
 	if (format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
+		ASSERT(plane_size->video.chroma_pitch != 0);
+		/* Chroma pitch zero can cause system hang! */
+
 		pitch = plane_size->video.luma_pitch - 1;
 		meta_pitch = dcc->video.meta_pitch_l - 1;
 		pitch_c = plane_size->video.chroma_pitch - 1;
@@ -535,11 +559,13 @@ void hubp1_program_deadline(
 	REG_SET(VBLANK_PARAMETERS_3, 0,
 		REFCYC_PER_META_CHUNK_VBLANK_L, dlg_attr->refcyc_per_meta_chunk_vblank_l);
 
-	REG_SET(NOM_PARAMETERS_0, 0,
-		DST_Y_PER_PTE_ROW_NOM_L, dlg_attr->dst_y_per_pte_row_nom_l);
+	if (REG(NOM_PARAMETERS_0))
+		REG_SET(NOM_PARAMETERS_0, 0,
+			DST_Y_PER_PTE_ROW_NOM_L, dlg_attr->dst_y_per_pte_row_nom_l);
 
-	REG_SET(NOM_PARAMETERS_1, 0,
-		REFCYC_PER_PTE_GROUP_NOM_L, dlg_attr->refcyc_per_pte_group_nom_l);
+	if (REG(NOM_PARAMETERS_1))
+		REG_SET(NOM_PARAMETERS_1, 0,
+			REFCYC_PER_PTE_GROUP_NOM_L, dlg_attr->refcyc_per_pte_group_nom_l);
 
 	REG_SET(NOM_PARAMETERS_4, 0,
 		DST_Y_PER_META_ROW_NOM_L, dlg_attr->dst_y_per_meta_row_nom_l);
@@ -568,11 +594,13 @@ void hubp1_program_deadline(
 	REG_SET(VBLANK_PARAMETERS_4, 0,
 		REFCYC_PER_META_CHUNK_VBLANK_C, dlg_attr->refcyc_per_meta_chunk_vblank_c);
 
-	REG_SET(NOM_PARAMETERS_2, 0,
-		DST_Y_PER_PTE_ROW_NOM_C, dlg_attr->dst_y_per_pte_row_nom_c);
+	if (REG(NOM_PARAMETERS_2))
+		REG_SET(NOM_PARAMETERS_2, 0,
+			DST_Y_PER_PTE_ROW_NOM_C, dlg_attr->dst_y_per_pte_row_nom_c);
 
-	REG_SET(NOM_PARAMETERS_3, 0,
-		REFCYC_PER_PTE_GROUP_NOM_C, dlg_attr->refcyc_per_pte_group_nom_c);
+	if (REG(NOM_PARAMETERS_3))
+		REG_SET(NOM_PARAMETERS_3, 0,
+			REFCYC_PER_PTE_GROUP_NOM_C, dlg_attr->refcyc_per_pte_group_nom_c);
 
 	REG_SET(NOM_PARAMETERS_6, 0,
 		DST_Y_PER_META_ROW_NOM_C, dlg_attr->dst_y_per_meta_row_nom_c);
@@ -609,6 +637,13 @@ void hubp1_program_deadline(
 	REG_SET(DCN_SURF1_TTU_CNTL1, 0,
 		REFCYC_PER_REQ_DELIVERY_PRE,
 		ttu_attr->refcyc_per_req_delivery_pre_c);
+
+	REG_SET_3(DCN_CUR0_TTU_CNTL0, 0,
+		REFCYC_PER_REQ_DELIVERY, ttu_attr->refcyc_per_req_delivery_cur0,
+		QoS_LEVEL_FIXED, ttu_attr->qos_level_fixed_cur0,
+		QoS_RAMP_DISABLE, ttu_attr->qos_ramp_disable_cur0);
+	REG_SET(DCN_CUR0_TTU_CNTL1, 0,
+		REFCYC_PER_REQ_DELIVERY_PRE, ttu_attr->refcyc_per_req_delivery_pre_cur0);
 }
 
 static void hubp1_setup(
@@ -752,9 +787,159 @@ void min_set_viewport(
 		  PRI_VIEWPORT_Y_START_C, viewport_c->y);
 }
 
-void hubp1_read_state(struct dcn10_hubp *hubp1,
-		struct dcn_hubp_state *s)
+void hubp1_read_state(struct hubp *hubp)
 {
+	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
+	struct dcn_hubp_state *s = &hubp1->state;
+	struct _vcs_dpi_display_dlg_regs_st *dlg_attr = &s->dlg_attr;
+	struct _vcs_dpi_display_ttu_regs_st *ttu_attr = &s->ttu_attr;
+	struct _vcs_dpi_display_rq_regs_st *rq_regs = &s->rq_regs;
+
+	/* Requester */
+	REG_GET(HUBPRET_CONTROL,
+			DET_BUF_PLANE1_BASE_ADDRESS, &rq_regs->plane1_base_address);
+	REG_GET_4(DCN_EXPANSION_MODE,
+			DRQ_EXPANSION_MODE, &rq_regs->drq_expansion_mode,
+			PRQ_EXPANSION_MODE, &rq_regs->prq_expansion_mode,
+			MRQ_EXPANSION_MODE, &rq_regs->mrq_expansion_mode,
+			CRQ_EXPANSION_MODE, &rq_regs->crq_expansion_mode);
+	REG_GET_8(DCHUBP_REQ_SIZE_CONFIG,
+		CHUNK_SIZE, &rq_regs->rq_regs_l.chunk_size,
+		MIN_CHUNK_SIZE, &rq_regs->rq_regs_l.min_chunk_size,
+		META_CHUNK_SIZE, &rq_regs->rq_regs_l.meta_chunk_size,
+		MIN_META_CHUNK_SIZE, &rq_regs->rq_regs_l.min_meta_chunk_size,
+		DPTE_GROUP_SIZE, &rq_regs->rq_regs_l.dpte_group_size,
+		MPTE_GROUP_SIZE, &rq_regs->rq_regs_l.mpte_group_size,
+		SWATH_HEIGHT, &rq_regs->rq_regs_l.swath_height,
+		PTE_ROW_HEIGHT_LINEAR, &rq_regs->rq_regs_l.pte_row_height_linear);
+	REG_GET_8(DCHUBP_REQ_SIZE_CONFIG_C,
+		CHUNK_SIZE_C, &rq_regs->rq_regs_c.chunk_size,
+		MIN_CHUNK_SIZE_C, &rq_regs->rq_regs_c.min_chunk_size,
+		META_CHUNK_SIZE_C, &rq_regs->rq_regs_c.meta_chunk_size,
+		MIN_META_CHUNK_SIZE_C, &rq_regs->rq_regs_c.min_meta_chunk_size,
+		DPTE_GROUP_SIZE_C, &rq_regs->rq_regs_c.dpte_group_size,
+		MPTE_GROUP_SIZE_C, &rq_regs->rq_regs_c.mpte_group_size,
+		SWATH_HEIGHT_C, &rq_regs->rq_regs_c.swath_height,
+		PTE_ROW_HEIGHT_LINEAR_C, &rq_regs->rq_regs_c.pte_row_height_linear);
+
+	/* DLG - Per hubp */
+	REG_GET_2(BLANK_OFFSET_0,
+		REFCYC_H_BLANK_END, &dlg_attr->refcyc_h_blank_end,
+		DLG_V_BLANK_END, &dlg_attr->dlg_vblank_end);
+
+	REG_GET(BLANK_OFFSET_1,
+		MIN_DST_Y_NEXT_START, &dlg_attr->min_dst_y_next_start);
+
+	REG_GET(DST_DIMENSIONS,
+		REFCYC_PER_HTOTAL, &dlg_attr->refcyc_per_htotal);
+
+	REG_GET_2(DST_AFTER_SCALER,
+		REFCYC_X_AFTER_SCALER, &dlg_attr->refcyc_x_after_scaler,
+		DST_Y_AFTER_SCALER, &dlg_attr->dst_y_after_scaler);
+
+	if (REG(PREFETCH_SETTINS))
+		REG_GET_2(PREFETCH_SETTINS,
+			DST_Y_PREFETCH, &dlg_attr->dst_y_prefetch,
+			VRATIO_PREFETCH, &dlg_attr->vratio_prefetch);
+	else
+		REG_GET_2(PREFETCH_SETTINGS,
+			DST_Y_PREFETCH, &dlg_attr->dst_y_prefetch,
+			VRATIO_PREFETCH, &dlg_attr->vratio_prefetch);
+
+	REG_GET_2(VBLANK_PARAMETERS_0,
+		DST_Y_PER_VM_VBLANK, &dlg_attr->dst_y_per_vm_vblank,
+		DST_Y_PER_ROW_VBLANK, &dlg_attr->dst_y_per_row_vblank);
+
+	REG_GET(REF_FREQ_TO_PIX_FREQ,
+		REF_FREQ_TO_PIX_FREQ, &dlg_attr->ref_freq_to_pix_freq);
+
+	/* DLG - Per luma/chroma */
+	REG_GET(VBLANK_PARAMETERS_1,
+		REFCYC_PER_PTE_GROUP_VBLANK_L, &dlg_attr->refcyc_per_pte_group_vblank_l);
+
+	REG_GET(VBLANK_PARAMETERS_3,
+		REFCYC_PER_META_CHUNK_VBLANK_L, &dlg_attr->refcyc_per_meta_chunk_vblank_l);
+
+	if (REG(NOM_PARAMETERS_0))
+		REG_GET(NOM_PARAMETERS_0,
+			DST_Y_PER_PTE_ROW_NOM_L, &dlg_attr->dst_y_per_pte_row_nom_l);
+
+	if (REG(NOM_PARAMETERS_1))
+		REG_GET(NOM_PARAMETERS_1,
+			REFCYC_PER_PTE_GROUP_NOM_L, &dlg_attr->refcyc_per_pte_group_nom_l);
+
+	REG_GET(NOM_PARAMETERS_4,
+		DST_Y_PER_META_ROW_NOM_L, &dlg_attr->dst_y_per_meta_row_nom_l);
+
+	REG_GET(NOM_PARAMETERS_5,
+		REFCYC_PER_META_CHUNK_NOM_L, &dlg_attr->refcyc_per_meta_chunk_nom_l);
+
+	REG_GET_2(PER_LINE_DELIVERY_PRE,
+		REFCYC_PER_LINE_DELIVERY_PRE_L, &dlg_attr->refcyc_per_line_delivery_pre_l,
+		REFCYC_PER_LINE_DELIVERY_PRE_C, &dlg_attr->refcyc_per_line_delivery_pre_c);
+
+	REG_GET_2(PER_LINE_DELIVERY,
+		REFCYC_PER_LINE_DELIVERY_L, &dlg_attr->refcyc_per_line_delivery_l,
+		REFCYC_PER_LINE_DELIVERY_C, &dlg_attr->refcyc_per_line_delivery_c);
+
+	if (REG(PREFETCH_SETTINS_C))
+		REG_GET(PREFETCH_SETTINS_C,
+			VRATIO_PREFETCH_C, &dlg_attr->vratio_prefetch_c);
+	else
+		REG_GET(PREFETCH_SETTINGS_C,
+			VRATIO_PREFETCH_C, &dlg_attr->vratio_prefetch_c);
+
+	REG_GET(VBLANK_PARAMETERS_2,
+		REFCYC_PER_PTE_GROUP_VBLANK_C, &dlg_attr->refcyc_per_pte_group_vblank_c);
+
+	REG_GET(VBLANK_PARAMETERS_4,
+		REFCYC_PER_META_CHUNK_VBLANK_C, &dlg_attr->refcyc_per_meta_chunk_vblank_c);
+
+	if (REG(NOM_PARAMETERS_2))
+		REG_GET(NOM_PARAMETERS_2,
+			DST_Y_PER_PTE_ROW_NOM_C, &dlg_attr->dst_y_per_pte_row_nom_c);
+
+	if (REG(NOM_PARAMETERS_3))
+		REG_GET(NOM_PARAMETERS_3,
+			REFCYC_PER_PTE_GROUP_NOM_C, &dlg_attr->refcyc_per_pte_group_nom_c);
+
+	REG_GET(NOM_PARAMETERS_6,
+		DST_Y_PER_META_ROW_NOM_C, &dlg_attr->dst_y_per_meta_row_nom_c);
+
+	REG_GET(NOM_PARAMETERS_7,
+		REFCYC_PER_META_CHUNK_NOM_C, &dlg_attr->refcyc_per_meta_chunk_nom_c);
+
+	/* TTU - per hubp */
+	REG_GET_2(DCN_TTU_QOS_WM,
+		QoS_LEVEL_LOW_WM, &ttu_attr->qos_level_low_wm,
+		QoS_LEVEL_HIGH_WM, &ttu_attr->qos_level_high_wm);
+
+	REG_GET_2(DCN_GLOBAL_TTU_CNTL,
+		MIN_TTU_VBLANK, &ttu_attr->min_ttu_vblank,
+		QoS_LEVEL_FLIP, &ttu_attr->qos_level_flip);
+
+	/* TTU - per luma/chroma */
+	/* Assumed surf0 is luma and 1 is chroma */
+
+	REG_GET_3(DCN_SURF0_TTU_CNTL0,
+		REFCYC_PER_REQ_DELIVERY, &ttu_attr->refcyc_per_req_delivery_l,
+		QoS_LEVEL_FIXED, &ttu_attr->qos_level_fixed_l,
+		QoS_RAMP_DISABLE, &ttu_attr->qos_ramp_disable_l);
+
+	REG_GET(DCN_SURF0_TTU_CNTL1,
+		REFCYC_PER_REQ_DELIVERY_PRE,
+		&ttu_attr->refcyc_per_req_delivery_pre_l);
+
+	REG_GET_3(DCN_SURF1_TTU_CNTL0,
+		REFCYC_PER_REQ_DELIVERY, &ttu_attr->refcyc_per_req_delivery_c,
+		QoS_LEVEL_FIXED, &ttu_attr->qos_level_fixed_c,
+		QoS_RAMP_DISABLE, &ttu_attr->qos_ramp_disable_c);
+
+	REG_GET(DCN_SURF1_TTU_CNTL1,
+		REFCYC_PER_REQ_DELIVERY_PRE,
+		&ttu_attr->refcyc_per_req_delivery_pre_c);
+
+	/* Rest of hubp */
 	REG_GET(DCSURF_SURFACE_CONFIG,
 			SURFACE_PIXEL_FORMAT, &s->pixel_format);
 
@@ -890,14 +1075,14 @@ void hubp1_cursor_set_position(
 	ASSERT(param->h_scale_ratio.value);
 
 	if (param->h_scale_ratio.value)
-		dst_x_offset = dal_fixed31_32_floor(dal_fixed31_32_div(
-				dal_fixed31_32_from_int(dst_x_offset),
+		dst_x_offset = dc_fixpt_floor(dc_fixpt_div(
+				dc_fixpt_from_int(dst_x_offset),
 				param->h_scale_ratio));
 
 	if (src_x_offset >= (int)param->viewport_width)
 		cur_en = 0;  /* not visible beyond right edge*/
 
-	if (src_x_offset + (int)hubp->curs_attr.width < 0)
+	if (src_x_offset + (int)hubp->curs_attr.width <= 0)
 		cur_en = 0;  /* not visible beyond left edge*/
 
 	if (cur_en && REG_READ(CURSOR_SURFACE_ADDRESS) == 0)
@@ -952,6 +1137,10 @@ static struct hubp_funcs dcn10_hubp_funcs = {
 	.hubp_disconnect = hubp1_disconnect,
 	.hubp_clk_cntl = hubp1_clk_cntl,
 	.hubp_vtg_sel = hubp1_vtg_sel,
+	.hubp_read_state = hubp1_read_state,
+	.hubp_disable_control =  hubp1_disable_control,
+	.hubp_get_underflow_status = hubp1_get_underflow_status,
+
 };
 
 /*****************************************/
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
index 4a3703e12ea1..af384034398f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
@@ -30,6 +30,7 @@
 #define TO_DCN10_HUBP(hubp)\
 	container_of(hubp, struct dcn10_hubp, base)
 
+/* Register address initialization macro for all ASICs (including those with reduced functionality) */
 #define HUBP_REG_LIST_DCN(id)\
 	SRI(DCHUBP_CNTL, HUBP, id),\
 	SRI(HUBPREQ_DEBUG_DB, HUBP, id),\
@@ -78,16 +79,12 @@
 	SRI(REF_FREQ_TO_PIX_FREQ, HUBPREQ, id),\
 	SRI(VBLANK_PARAMETERS_1, HUBPREQ, id),\
 	SRI(VBLANK_PARAMETERS_3, HUBPREQ, id),\
-	SRI(NOM_PARAMETERS_0, HUBPREQ, id),\
-	SRI(NOM_PARAMETERS_1, HUBPREQ, id),\
 	SRI(NOM_PARAMETERS_4, HUBPREQ, id),\
 	SRI(NOM_PARAMETERS_5, HUBPREQ, id),\
 	SRI(PER_LINE_DELIVERY_PRE, HUBPREQ, id),\
 	SRI(PER_LINE_DELIVERY, HUBPREQ, id),\
 	SRI(VBLANK_PARAMETERS_2, HUBPREQ, id),\
 	SRI(VBLANK_PARAMETERS_4, HUBPREQ, id),\
-	SRI(NOM_PARAMETERS_2, HUBPREQ, id),\
-	SRI(NOM_PARAMETERS_3, HUBPREQ, id),\
 	SRI(NOM_PARAMETERS_6, HUBPREQ, id),\
 	SRI(NOM_PARAMETERS_7, HUBPREQ, id),\
 	SRI(DCN_TTU_QOS_WM, HUBPREQ, id),\
@@ -96,11 +93,21 @@
 	SRI(DCN_SURF0_TTU_CNTL1, HUBPREQ, id),\
 	SRI(DCN_SURF1_TTU_CNTL0, HUBPREQ, id),\
 	SRI(DCN_SURF1_TTU_CNTL1, HUBPREQ, id),\
-	SRI(DCN_VM_MX_L1_TLB_CNTL, HUBPREQ, id),\
+	SRI(DCN_CUR0_TTU_CNTL0, HUBPREQ, id),\
+	SRI(DCN_CUR0_TTU_CNTL1, HUBPREQ, id),\
 	SRI(HUBP_CLK_CNTL, HUBP, id)
 
+/* Register address initialization macro for ASICs with VM */
+#define HUBP_REG_LIST_DCN_VM(id)\
+	SRI(NOM_PARAMETERS_0, HUBPREQ, id),\
+	SRI(NOM_PARAMETERS_1, HUBPREQ, id),\
+	SRI(NOM_PARAMETERS_2, HUBPREQ, id),\
+	SRI(NOM_PARAMETERS_3, HUBPREQ, id),\
+	SRI(DCN_VM_MX_L1_TLB_CNTL, HUBPREQ, id)
+
 #define HUBP_REG_LIST_DCN10(id)\
 	HUBP_REG_LIST_DCN(id),\
+	HUBP_REG_LIST_DCN_VM(id),\
 	SRI(PREFETCH_SETTINS, HUBPREQ, id),\
 	SRI(PREFETCH_SETTINS_C, HUBPREQ, id),\
 	SRI(DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB, HUBPREQ, id),\
@@ -198,6 +205,8 @@
 	uint32_t DCN_SURF0_TTU_CNTL1; \
 	uint32_t DCN_SURF1_TTU_CNTL0; \
 	uint32_t DCN_SURF1_TTU_CNTL1; \
+	uint32_t DCN_CUR0_TTU_CNTL0; \
+	uint32_t DCN_CUR0_TTU_CNTL1; \
 	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB; \
 	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB; \
 	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB; \
@@ -237,12 +246,14 @@
 #define HUBP_SF(reg_name, field_name, post_fix)\
 	.field_name = reg_name ## __ ## field_name ## post_fix
 
+/* Mask/shift struct generation macro for all ASICs (including those with reduced functionality) */
 #define HUBP_MASK_SH_LIST_DCN(mask_sh)\
 	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_BLANK_EN, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_TTU_DISABLE, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_UNDERFLOW_STATUS, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_NO_OUTSTANDING_REQ, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_VTG_SEL, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_DISABLE, mask_sh),\
 	HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_PIPES, mask_sh),\
 	HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_BANKS, mask_sh),\
 	HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, PIPE_INTERLEAVE, mask_sh),\
@@ -335,8 +346,6 @@
 	HUBP_SF(HUBPREQ0_REF_FREQ_TO_PIX_FREQ, REF_FREQ_TO_PIX_FREQ, mask_sh),\
 	HUBP_SF(HUBPREQ0_VBLANK_PARAMETERS_1, REFCYC_PER_PTE_GROUP_VBLANK_L, mask_sh),\
 	HUBP_SF(HUBPREQ0_VBLANK_PARAMETERS_3, REFCYC_PER_META_CHUNK_VBLANK_L, mask_sh),\
-	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_0, DST_Y_PER_PTE_ROW_NOM_L, mask_sh),\
-	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_1, REFCYC_PER_PTE_GROUP_NOM_L, mask_sh),\
 	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_4, DST_Y_PER_META_ROW_NOM_L, mask_sh),\
 	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_5, REFCYC_PER_META_CHUNK_NOM_L, mask_sh),\
 	HUBP_SF(HUBPREQ0_PER_LINE_DELIVERY_PRE, REFCYC_PER_LINE_DELIVERY_PRE_L, mask_sh),\
@@ -345,8 +354,6 @@
 	HUBP_SF(HUBPREQ0_PER_LINE_DELIVERY, REFCYC_PER_LINE_DELIVERY_C, mask_sh),\
 	HUBP_SF(HUBPREQ0_VBLANK_PARAMETERS_2, REFCYC_PER_PTE_GROUP_VBLANK_C, mask_sh),\
 	HUBP_SF(HUBPREQ0_VBLANK_PARAMETERS_4, REFCYC_PER_META_CHUNK_VBLANK_C, mask_sh),\
-	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_2, DST_Y_PER_PTE_ROW_NOM_C, mask_sh),\
-	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_3, REFCYC_PER_PTE_GROUP_NOM_C, mask_sh),\
 	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_6, DST_Y_PER_META_ROW_NOM_C, mask_sh),\
 	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_7, REFCYC_PER_META_CHUNK_NOM_C, mask_sh),\
 	HUBP_SF(HUBPREQ0_DCN_TTU_QOS_WM, QoS_LEVEL_LOW_WM, mask_sh),\
@@ -357,12 +364,24 @@
 	HUBP_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL0, QoS_LEVEL_FIXED, mask_sh),\
 	HUBP_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL0, QoS_RAMP_DISABLE, mask_sh),\
 	HUBP_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL1, REFCYC_PER_REQ_DELIVERY_PRE, mask_sh),\
+	HUBP_SF(HUBP0_HUBP_CLK_CNTL, HUBP_CLOCK_ENABLE, mask_sh)
+
+/* Mask/shift struct generation macro for ASICs with VM */
+#define HUBP_MASK_SH_LIST_DCN_VM(mask_sh)\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_0, DST_Y_PER_PTE_ROW_NOM_L, mask_sh),\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_1, REFCYC_PER_PTE_GROUP_NOM_L, mask_sh),\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_2, DST_Y_PER_PTE_ROW_NOM_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_3, REFCYC_PER_PTE_GROUP_NOM_C, mask_sh),\
 	HUBP_SF(HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, mask_sh),\
 	HUBP_SF(HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, mask_sh),\
-	HUBP_SF(HUBP0_HUBP_CLK_CNTL, HUBP_CLOCK_ENABLE, mask_sh)
+	HUBP_SF(HUBPREQ0_DCN_CUR0_TTU_CNTL0, REFCYC_PER_REQ_DELIVERY, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_CUR0_TTU_CNTL0, QoS_LEVEL_FIXED, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_CUR0_TTU_CNTL0, QoS_RAMP_DISABLE, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_CUR0_TTU_CNTL1, REFCYC_PER_REQ_DELIVERY_PRE, mask_sh)
 
 #define HUBP_MASK_SH_LIST_DCN10(mask_sh)\
 	HUBP_MASK_SH_LIST_DCN(mask_sh),\
+	HUBP_MASK_SH_LIST_DCN_VM(mask_sh),\
 	HUBP_SF(HUBPREQ0_PREFETCH_SETTINS, DST_Y_PREFETCH, mask_sh),\
 	HUBP_SF(HUBPREQ0_PREFETCH_SETTINS, VRATIO_PREFETCH, mask_sh),\
 	HUBP_SF(HUBPREQ0_PREFETCH_SETTINS_C, VRATIO_PREFETCH_C, mask_sh),\
@@ -403,6 +422,7 @@
 
 #define DCN_HUBP_REG_FIELD_LIST(type) \
 	type HUBP_BLANK_EN;\
+	type HUBP_DISABLE;\
 	type HUBP_TTU_DISABLE;\
 	type HUBP_NO_OUTSTANDING_REQ;\
 	type HUBP_VTG_SEL;\
@@ -601,8 +621,29 @@ struct dcn_mi_mask {
 	DCN_HUBP_REG_FIELD_LIST(uint32_t);
 };
 
+struct dcn_hubp_state {
+	struct _vcs_dpi_display_dlg_regs_st dlg_attr;
+	struct _vcs_dpi_display_ttu_regs_st ttu_attr;
+	struct _vcs_dpi_display_rq_regs_st rq_regs;
+	uint32_t pixel_format;
+	uint32_t inuse_addr_hi;
+	uint32_t viewport_width;
+	uint32_t viewport_height;
+	uint32_t rotation_angle;
+	uint32_t h_mirror_en;
+	uint32_t sw_mode;
+	uint32_t dcc_en;
+	uint32_t blank_en;
+	uint32_t underflow_status;
+	uint32_t ttu_disable;
+	uint32_t min_ttu_vblank;
+	uint32_t qos_level_low_wm;
+	uint32_t qos_level_high_wm;
+};
+
 struct dcn10_hubp {
 	struct hubp base;
+	struct dcn_hubp_state state;
 	const struct dcn_mi_registers *hubp_regs;
 	const struct dcn_mi_shift *hubp_shift;
 	const struct dcn_mi_mask *hubp_mask;
@@ -680,26 +721,9 @@ void dcn10_hubp_construct(
 	const struct dcn_mi_shift *hubp_shift,
 	const struct dcn_mi_mask *hubp_mask);
 
-
-struct dcn_hubp_state {
-	uint32_t pixel_format;
-	uint32_t inuse_addr_hi;
-	uint32_t viewport_width;
-	uint32_t viewport_height;
-	uint32_t rotation_angle;
-	uint32_t h_mirror_en;
-	uint32_t sw_mode;
-	uint32_t dcc_en;
-	uint32_t blank_en;
-	uint32_t underflow_status;
-	uint32_t ttu_disable;
-	uint32_t min_ttu_vblank;
-	uint32_t qos_level_low_wm;
-	uint32_t qos_level_high_wm;
-};
-void hubp1_read_state(struct dcn10_hubp *hubp1,
-		struct dcn_hubp_state *s);
+void hubp1_read_state(struct hubp *hubp);
 
 enum cursor_pitch hubp1_get_cursor_pitch(unsigned int pitch);
 
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
index 8b0f6b8a5627..f8e0576af6e0 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
@@ -45,8 +45,8 @@
 #include "dcn10_hubbub.h"
 #include "dcn10_cm_common.h"
 
-#define DC_LOGGER \
-	ctx->logger
+#define DC_LOGGER_INIT(logger)
+
 #define CTX \
 	hws->ctx
 #define REG(reg)\
@@ -56,16 +56,17 @@
 #define FN(reg_name, field_name) \
 	hws->shifts->field_name, hws->masks->field_name
 
+/*print is 17 wide, first two characters are spaces*/
 #define DTN_INFO_MICRO_SEC(ref_cycle) \
 	print_microsec(dc_ctx, ref_cycle)
 
 void print_microsec(struct dc_context *dc_ctx, uint32_t ref_cycle)
 {
-	static const uint32_t ref_clk_mhz = 48;
-	static const unsigned int frac = 10;
+	const uint32_t ref_clk_mhz = dc_ctx->dc->res_pool->ref_clock_inKhz / 1000;
+	static const unsigned int frac = 1000;
 	uint32_t us_x10 = (ref_cycle * frac) / ref_clk_mhz;
 
-	DTN_INFO("%d.%d \t ",
+	DTN_INFO("  %11d.%03d",
 			us_x10 / frac,
 			us_x10 % frac);
 }
@@ -92,14 +93,14 @@ void dcn10_log_hubbub_state(struct dc *dc)
 
 	hubbub1_wm_read_state(dc->res_pool->hubbub, &wm);
 
-	DTN_INFO("HUBBUB WM: \t data_urgent \t pte_meta_urgent \t "
-			"sr_enter \t sr_exit \t dram_clk_change \n");
+	DTN_INFO("HUBBUB WM:      data_urgent  pte_meta_urgent"
+			"         sr_enter          sr_exit  dram_clk_change\n");
 
 	for (i = 0; i < 4; i++) {
 		struct dcn_hubbub_wm_set *s;
 
 		s = &wm.sets[i];
-		DTN_INFO("WM_Set[%d]:\t ", s->wm_set);
+		DTN_INFO("WM_Set[%d]:", s->wm_set);
 		DTN_INFO_MICRO_SEC(s->data_urgent);
 		DTN_INFO_MICRO_SEC(s->pte_meta_urgent);
 		DTN_INFO_MICRO_SEC(s->sr_enter);
@@ -111,6 +112,121 @@ void dcn10_log_hubbub_state(struct dc *dc)
 	DTN_INFO("\n");
 }
 
+static void dcn10_log_hubp_states(struct dc *dc)
+{
+	struct dc_context *dc_ctx = dc->ctx;
+	struct resource_pool *pool = dc->res_pool;
+	int i;
+
+	DTN_INFO("HUBP:  format  addr_hi  width  height"
+			"  rot  mir  sw_mode  dcc_en  blank_en  ttu_dis  underflow"
+			"   min_ttu_vblank       qos_low_wm      qos_high_wm\n");
+	for (i = 0; i < pool->pipe_count; i++) {
+		struct hubp *hubp = pool->hubps[i];
+		struct dcn_hubp_state *s = &(TO_DCN10_HUBP(hubp)->state);
+
+		hubp->funcs->hubp_read_state(hubp);
+
+		if (!s->blank_en) {
+			DTN_INFO("[%2d]:  %5xh  %6xh  %5d  %6d  %2xh  %2xh  %6xh"
+					"  %6d  %8d  %7d  %8xh",
+					hubp->inst,
+					s->pixel_format,
+					s->inuse_addr_hi,
+					s->viewport_width,
+					s->viewport_height,
+					s->rotation_angle,
+					s->h_mirror_en,
+					s->sw_mode,
+					s->dcc_en,
+					s->blank_en,
+					s->ttu_disable,
+					s->underflow_status);
+			DTN_INFO_MICRO_SEC(s->min_ttu_vblank);
+			DTN_INFO_MICRO_SEC(s->qos_level_low_wm);
+			DTN_INFO_MICRO_SEC(s->qos_level_high_wm);
+			DTN_INFO("\n");
+		}
+	}
+
+	DTN_INFO("\n=========RQ========\n");
+	DTN_INFO("HUBP:  drq_exp_m  prq_exp_m  mrq_exp_m  crq_exp_m  plane1_ba  L:chunk_s  min_chu_s  meta_ch_s"
+		"  min_m_c_s  dpte_gr_s  mpte_gr_s  swath_hei  pte_row_h  C:chunk_s  min_chu_s  meta_ch_s"
+		"  min_m_c_s  dpte_gr_s  mpte_gr_s  swath_hei  pte_row_h\n");
+	for (i = 0; i < pool->pipe_count; i++) {
+		struct dcn_hubp_state *s = &(TO_DCN10_HUBP(pool->hubps[i])->state);
+		struct _vcs_dpi_display_rq_regs_st *rq_regs = &s->rq_regs;
+
+		if (!s->blank_en)
+			DTN_INFO("[%2d]:  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh\n",
+				pool->hubps[i]->inst, rq_regs->drq_expansion_mode, rq_regs->prq_expansion_mode, rq_regs->mrq_expansion_mode,
+				rq_regs->crq_expansion_mode, rq_regs->plane1_base_address, rq_regs->rq_regs_l.chunk_size,
+				rq_regs->rq_regs_l.min_chunk_size, rq_regs->rq_regs_l.meta_chunk_size,
+				rq_regs->rq_regs_l.min_meta_chunk_size, rq_regs->rq_regs_l.dpte_group_size,
+				rq_regs->rq_regs_l.mpte_group_size, rq_regs->rq_regs_l.swath_height,
+				rq_regs->rq_regs_l.pte_row_height_linear, rq_regs->rq_regs_c.chunk_size, rq_regs->rq_regs_c.min_chunk_size,
+				rq_regs->rq_regs_c.meta_chunk_size, rq_regs->rq_regs_c.min_meta_chunk_size,
+				rq_regs->rq_regs_c.dpte_group_size, rq_regs->rq_regs_c.mpte_group_size,
+				rq_regs->rq_regs_c.swath_height, rq_regs->rq_regs_c.pte_row_height_linear);
+	}
+
+	DTN_INFO("========DLG========\n");
+	DTN_INFO("HUBP:  rc_hbe     dlg_vbe    min_d_y_n  rc_per_ht  rc_x_a_s "
+			"  dst_y_a_s  dst_y_pf   dst_y_vvb  dst_y_rvb  dst_y_vfl  dst_y_rfl  rf_pix_fq"
+			"  vratio_pf  vrat_pf_c  rc_pg_vbl  rc_pg_vbc  rc_mc_vbl  rc_mc_vbc  rc_pg_fll"
+			"  rc_pg_flc  rc_mc_fll  rc_mc_flc  pr_nom_l   pr_nom_c   rc_pg_nl   rc_pg_nc "
+			"  mr_nom_l   mr_nom_c   rc_mc_nl   rc_mc_nc   rc_ld_pl   rc_ld_pc   rc_ld_l  "
+			"  rc_ld_c    cha_cur0   ofst_cur1  cha_cur1   vr_af_vc0  ddrq_limt  x_rt_dlay"
+			"  x_rp_dlay  x_rr_sfl\n");
+	for (i = 0; i < pool->pipe_count; i++) {
+		struct dcn_hubp_state *s = &(TO_DCN10_HUBP(pool->hubps[i])->state);
+		struct _vcs_dpi_display_dlg_regs_st *dlg_regs = &s->dlg_attr;
+
+		if (!s->blank_en)
+			DTN_INFO("[%2d]:  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh"
+				"%  8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh"
+				"  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh\n",
+				pool->hubps[i]->inst, dlg_regs->refcyc_h_blank_end, dlg_regs->dlg_vblank_end, dlg_regs->min_dst_y_next_start,
+				dlg_regs->refcyc_per_htotal, dlg_regs->refcyc_x_after_scaler, dlg_regs->dst_y_after_scaler,
+				dlg_regs->dst_y_prefetch, dlg_regs->dst_y_per_vm_vblank, dlg_regs->dst_y_per_row_vblank,
+				dlg_regs->dst_y_per_vm_flip, dlg_regs->dst_y_per_row_flip, dlg_regs->ref_freq_to_pix_freq,
+				dlg_regs->vratio_prefetch, dlg_regs->vratio_prefetch_c, dlg_regs->refcyc_per_pte_group_vblank_l,
+				dlg_regs->refcyc_per_pte_group_vblank_c, dlg_regs->refcyc_per_meta_chunk_vblank_l,
+				dlg_regs->refcyc_per_meta_chunk_vblank_c, dlg_regs->refcyc_per_pte_group_flip_l,
+				dlg_regs->refcyc_per_pte_group_flip_c, dlg_regs->refcyc_per_meta_chunk_flip_l,
+				dlg_regs->refcyc_per_meta_chunk_flip_c, dlg_regs->dst_y_per_pte_row_nom_l,
+				dlg_regs->dst_y_per_pte_row_nom_c, dlg_regs->refcyc_per_pte_group_nom_l,
+				dlg_regs->refcyc_per_pte_group_nom_c, dlg_regs->dst_y_per_meta_row_nom_l,
+				dlg_regs->dst_y_per_meta_row_nom_c, dlg_regs->refcyc_per_meta_chunk_nom_l,
+				dlg_regs->refcyc_per_meta_chunk_nom_c, dlg_regs->refcyc_per_line_delivery_pre_l,
+				dlg_regs->refcyc_per_line_delivery_pre_c, dlg_regs->refcyc_per_line_delivery_l,
+				dlg_regs->refcyc_per_line_delivery_c, dlg_regs->chunk_hdl_adjust_cur0, dlg_regs->dst_y_offset_cur1,
+				dlg_regs->chunk_hdl_adjust_cur1, dlg_regs->vready_after_vcount0, dlg_regs->dst_y_delta_drq_limit,
+				dlg_regs->xfc_reg_transfer_delay, dlg_regs->xfc_reg_precharge_delay,
+				dlg_regs->xfc_reg_remote_surface_flip_latency);
+	}
+
+	DTN_INFO("========TTU========\n");
+	DTN_INFO("HUBP:  qos_ll_wm  qos_lh_wm  mn_ttu_vb  qos_l_flp  rc_rd_p_l  rc_rd_l    rc_rd_p_c"
+			"  rc_rd_c    rc_rd_c0   rc_rd_pc0  rc_rd_c1   rc_rd_pc1  qos_lf_l   qos_rds_l"
+			"  qos_lf_c   qos_rds_c  qos_lf_c0  qos_rds_c0 qos_lf_c1  qos_rds_c1\n");
+	for (i = 0; i < pool->pipe_count; i++) {
+		struct dcn_hubp_state *s = &(TO_DCN10_HUBP(pool->hubps[i])->state);
+		struct _vcs_dpi_display_ttu_regs_st *ttu_regs = &s->ttu_attr;
+
+		if (!s->blank_en)
+			DTN_INFO("[%2d]:  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh  %8xh\n",
+				pool->hubps[i]->inst, ttu_regs->qos_level_low_wm, ttu_regs->qos_level_high_wm, ttu_regs->min_ttu_vblank,
+				ttu_regs->qos_level_flip, ttu_regs->refcyc_per_req_delivery_pre_l, ttu_regs->refcyc_per_req_delivery_l,
+				ttu_regs->refcyc_per_req_delivery_pre_c, ttu_regs->refcyc_per_req_delivery_c, ttu_regs->refcyc_per_req_delivery_cur0,
+				ttu_regs->refcyc_per_req_delivery_pre_cur0, ttu_regs->refcyc_per_req_delivery_cur1,
+				ttu_regs->refcyc_per_req_delivery_pre_cur1, ttu_regs->qos_level_fixed_l, ttu_regs->qos_ramp_disable_l,
+				ttu_regs->qos_level_fixed_c, ttu_regs->qos_ramp_disable_c, ttu_regs->qos_level_fixed_cur0,
+				ttu_regs->qos_ramp_disable_cur0, ttu_regs->qos_level_fixed_cur1, ttu_regs->qos_ramp_disable_cur1);
+	}
+	DTN_INFO("\n");
+}
+
 void dcn10_log_hw_state(struct dc *dc)
 {
 	struct dc_context *dc_ctx = dc->ctx;
@@ -121,41 +237,64 @@ void dcn10_log_hw_state(struct dc *dc)
 
 	dcn10_log_hubbub_state(dc);
 
-	DTN_INFO("HUBP:\t format \t addr_hi \t width \t height \t "
-			"rotation \t mirror \t  sw_mode \t "
-			"dcc_en \t blank_en \t ttu_dis \t underflow \t "
-			"min_ttu_vblank \t qos_low_wm \t qos_high_wm \n");
+	dcn10_log_hubp_states(dc);
 
+	DTN_INFO("DPP:    IGAM format  IGAM mode    DGAM mode    RGAM mode"
+			"  GAMUT mode  C11 C12   C13 C14   C21 C22   C23 C24   "
+			"C31 C32   C33 C34\n");
 	for (i = 0; i < pool->pipe_count; i++) {
-		struct hubp *hubp = pool->hubps[i];
-		struct dcn_hubp_state s;
-
-		hubp1_read_state(TO_DCN10_HUBP(hubp), &s);
-
-		DTN_INFO("[%d]:\t %xh \t %xh \t %d \t %d \t "
-				"%xh \t %xh \t %xh \t "
-				"%d \t %d \t %d \t %xh \t",
-				hubp->inst,
-				s.pixel_format,
-				s.inuse_addr_hi,
-				s.viewport_width,
-				s.viewport_height,
-				s.rotation_angle,
-				s.h_mirror_en,
-				s.sw_mode,
-				s.dcc_en,
-				s.blank_en,
-				s.ttu_disable,
-				s.underflow_status);
-		DTN_INFO_MICRO_SEC(s.min_ttu_vblank);
-		DTN_INFO_MICRO_SEC(s.qos_level_low_wm);
-		DTN_INFO_MICRO_SEC(s.qos_level_high_wm);
+		struct dpp *dpp = pool->dpps[i];
+		struct dcn_dpp_state s;
+
+		dpp->funcs->dpp_read_state(dpp, &s);
+
+		DTN_INFO("[%2d]:  %11xh  %-11s  %-11s  %-11s"
+				"%8x    %08xh %08xh %08xh %08xh %08xh %08xh",
+				dpp->inst,
+				s.igam_input_format,
+				(s.igam_lut_mode == 0) ? "BypassFixed" :
+					((s.igam_lut_mode == 1) ? "BypassFloat" :
+					((s.igam_lut_mode == 2) ? "RAM" :
+					((s.igam_lut_mode == 3) ? "RAM" :
+								 "Unknown"))),
+				(s.dgam_lut_mode == 0) ? "Bypass" :
+					((s.dgam_lut_mode == 1) ? "sRGB" :
+					((s.dgam_lut_mode == 2) ? "Ycc" :
+					((s.dgam_lut_mode == 3) ? "RAM" :
+					((s.dgam_lut_mode == 4) ? "RAM" :
+								 "Unknown")))),
+				(s.rgam_lut_mode == 0) ? "Bypass" :
+					((s.rgam_lut_mode == 1) ? "sRGB" :
+					((s.rgam_lut_mode == 2) ? "Ycc" :
+					((s.rgam_lut_mode == 3) ? "RAM" :
+					((s.rgam_lut_mode == 4) ? "RAM" :
+								 "Unknown")))),
+				s.gamut_remap_mode,
+				s.gamut_remap_c11_c12,
+				s.gamut_remap_c13_c14,
+				s.gamut_remap_c21_c22,
+				s.gamut_remap_c23_c24,
+				s.gamut_remap_c31_c32,
+				s.gamut_remap_c33_c34);
 		DTN_INFO("\n");
 	}
 	DTN_INFO("\n");
 
-	DTN_INFO("OTG:\t v_bs \t v_be \t v_ss \t v_se \t vpol \t vmax \t vmin \t "
-			"h_bs \t h_be \t h_ss \t h_se \t hpol \t htot \t vtot \t underflow\n");
+	DTN_INFO("MPCC:  OPP  DPP  MPCCBOT  MODE  ALPHA_MODE  PREMULT  OVERLAP_ONLY  IDLE\n");
+	for (i = 0; i < pool->pipe_count; i++) {
+		struct mpcc_state s = {0};
+
+		pool->mpc->funcs->read_mpcc_state(pool->mpc, i, &s);
+		if (s.opp_id != 0xf)
+			DTN_INFO("[%2d]:  %2xh  %2xh  %6xh  %4d  %10d  %7d  %12d  %4d\n",
+				i, s.opp_id, s.dpp_id, s.bot_mpcc_id,
+				s.mode, s.alpha_mode, s.pre_multiplied_alpha, s.overlap_only,
+				s.idle);
+	}
+	DTN_INFO("\n");
+
+	DTN_INFO("OTG:  v_bs  v_be  v_ss  v_se  vpol  vmax  vmin  vmax_sel  vmin_sel"
+			"  h_bs  h_be  h_ss  h_se  hpol  htot  vtot  underflow\n");
 
 	for (i = 0; i < pool->timing_generator_count; i++) {
 		struct timing_generator *tg = pool->timing_generators[i];
@@ -167,9 +306,8 @@ void dcn10_log_hw_state(struct dc *dc)
 		if ((s.otg_enabled & 1) == 0)
 			continue;
 
-		DTN_INFO("[%d]:\t %d \t %d \t %d \t %d \t "
-				"%d \t %d \t %d \t %d \t %d \t %d \t "
-				"%d \t %d \t %d \t %d \t %d \t ",
+		DTN_INFO("[%d]: %5d %5d %5d %5d %5d %5d %5d %9d %9d %5d %5d %5d"
+				" %5d %5d %5d %5d  %9d\n",
 				tg->inst,
 				s.v_blank_start,
 				s.v_blank_end,
@@ -178,6 +316,8 @@ void dcn10_log_hw_state(struct dc *dc)
 				s.v_sync_a_pol,
 				s.v_total_max,
 				s.v_total_min,
+				s.v_total_max_sel,
+				s.v_total_min_sel,
 				s.h_blank_start,
 				s.h_blank_end,
 				s.h_sync_a_start,
@@ -186,10 +326,25 @@ void dcn10_log_hw_state(struct dc *dc)
 				s.h_total,
 				s.v_total,
 				s.underflow_occurred_status);
-		DTN_INFO("\n");
+
+		// Clear underflow for debug purposes
+		// We want to keep underflow sticky bit on for the longevity tests outside of test environment.
+		// This function is called only from Windows or Diags test environment, hence it's safe to clear
+		// it from here without affecting the original intent.
+		tg->funcs->clear_optc_underflow(tg);
 	}
 	DTN_INFO("\n");
 
+	DTN_INFO("\nCALCULATED Clocks: dcfclk_khz:%d  dcfclk_deep_sleep_khz:%d  dispclk_khz:%d\n"
+		"dppclk_khz:%d  max_supported_dppclk_khz:%d  fclk_khz:%d  socclk_khz:%d\n\n",
+			dc->current_state->bw.dcn.calc_clk.dcfclk_khz,
+			dc->current_state->bw.dcn.calc_clk.dcfclk_deep_sleep_khz,
+			dc->current_state->bw.dcn.calc_clk.dispclk_khz,
+			dc->current_state->bw.dcn.calc_clk.dppclk_khz,
+			dc->current_state->bw.dcn.calc_clk.max_supported_dppclk_khz,
+			dc->current_state->bw.dcn.calc_clk.fclk_khz,
+			dc->current_state->bw.dcn.calc_clk.socclk_khz);
+
 	log_mpc_crc(dc);
 
 	DTN_INFO_END();
@@ -354,7 +509,7 @@ static void power_on_plane(
 	struct dce_hwseq *hws,
 	int plane_id)
 {
-	struct dc_context *ctx = hws->ctx;
+	DC_LOGGER_INIT(hws->ctx->logger);
 	if (REG(DC_IP_REQUEST_CNTL)) {
 		REG_SET(DC_IP_REQUEST_CNTL, 0,
 				IP_REQUEST_EN, 1);
@@ -461,7 +616,7 @@ static void false_optc_underflow_wa(
 		tg->funcs->clear_optc_underflow(tg);
 }
 
-static enum dc_status dcn10_prog_pixclk_crtc_otg(
+static enum dc_status dcn10_enable_stream_timing(
 		struct pipe_ctx *pipe_ctx,
 		struct dc_state *context,
 		struct dc *dc)
@@ -553,7 +708,7 @@ static void reset_back_end_for_pipe(
 		struct dc_state *context)
 {
 	int i;
-	struct dc_context *ctx = dc->ctx;
+	DC_LOGGER_INIT(dc->ctx->logger);
 	if (pipe_ctx->stream_res.stream_enc == NULL) {
 		pipe_ctx->stream = NULL;
 		return;
@@ -603,6 +758,90 @@ static void reset_back_end_for_pipe(
 					pipe_ctx->pipe_idx, pipe_ctx->stream_res.tg->inst);
 }
 
+static bool dcn10_hw_wa_force_recovery(struct dc *dc)
+{
+	struct hubp *hubp ;
+	unsigned int i;
+	bool need_recover = true;
+
+	if (!dc->debug.recovery_enabled)
+		return false;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx =
+			&dc->current_state->res_ctx.pipe_ctx[i];
+		if (pipe_ctx != NULL) {
+			hubp = pipe_ctx->plane_res.hubp;
+			if (hubp != NULL) {
+				if (hubp->funcs->hubp_get_underflow_status(hubp) != 0) {
+					/* one pipe underflow, we will reset all the pipes*/
+					need_recover = true;
+				}
+			}
+		}
+	}
+	if (!need_recover)
+		return false;
+	/*
+	DCHUBP_CNTL:HUBP_BLANK_EN=1
+	DCHUBBUB_SOFT_RESET:DCHUBBUB_GLOBAL_SOFT_RESET=1
+	DCHUBP_CNTL:HUBP_DISABLE=1
+	DCHUBP_CNTL:HUBP_DISABLE=0
+	DCHUBBUB_SOFT_RESET:DCHUBBUB_GLOBAL_SOFT_RESET=0
+	DCSURF_PRIMARY_SURFACE_ADDRESS
+	DCHUBP_CNTL:HUBP_BLANK_EN=0
+	*/
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx =
+			&dc->current_state->res_ctx.pipe_ctx[i];
+		if (pipe_ctx != NULL) {
+			hubp = pipe_ctx->plane_res.hubp;
+			/*DCHUBP_CNTL:HUBP_BLANK_EN=1*/
+			if (hubp != NULL)
+				hubp->funcs->set_hubp_blank_en(hubp, true);
+		}
+	}
+	/*DCHUBBUB_SOFT_RESET:DCHUBBUB_GLOBAL_SOFT_RESET=1*/
+	hubbub1_soft_reset(dc->res_pool->hubbub, true);
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx =
+			&dc->current_state->res_ctx.pipe_ctx[i];
+		if (pipe_ctx != NULL) {
+			hubp = pipe_ctx->plane_res.hubp;
+			/*DCHUBP_CNTL:HUBP_DISABLE=1*/
+			if (hubp != NULL)
+				hubp->funcs->hubp_disable_control(hubp, true);
+		}
+	}
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx =
+			&dc->current_state->res_ctx.pipe_ctx[i];
+		if (pipe_ctx != NULL) {
+			hubp = pipe_ctx->plane_res.hubp;
+			/*DCHUBP_CNTL:HUBP_DISABLE=0*/
+			if (hubp != NULL)
+				hubp->funcs->hubp_disable_control(hubp, true);
+		}
+	}
+	/*DCHUBBUB_SOFT_RESET:DCHUBBUB_GLOBAL_SOFT_RESET=0*/
+	hubbub1_soft_reset(dc->res_pool->hubbub, false);
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx =
+			&dc->current_state->res_ctx.pipe_ctx[i];
+		if (pipe_ctx != NULL) {
+			hubp = pipe_ctx->plane_res.hubp;
+			/*DCHUBP_CNTL:HUBP_BLANK_EN=0*/
+			if (hubp != NULL)
+				hubp->funcs->set_hubp_blank_en(hubp, true);
+		}
+	}
+	return true;
+
+}
+
+
 static void dcn10_verify_allow_pstate_change_high(struct dc *dc)
 {
 	static bool should_log_hw_state; /* prevent hw state log by default */
@@ -611,13 +850,17 @@ static void dcn10_verify_allow_pstate_change_high(struct dc *dc)
 		if (should_log_hw_state) {
 			dcn10_log_hw_state(dc);
 		}
-
 		BREAK_TO_DEBUGGER();
+		if (dcn10_hw_wa_force_recovery(dc)) {
+		/*check again*/
+			if (!hubbub1_verify_allow_pstate_change_high(dc->res_pool->hubbub))
+				BREAK_TO_DEBUGGER();
+		}
 	}
 }
 
 /* trigger HW to start disconnect plane from stream on the next vsync */
-static void plane_atomic_disconnect(struct dc *dc, struct pipe_ctx *pipe_ctx)
+void hwss1_plane_atomic_disconnect(struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
 	struct hubp *hubp = pipe_ctx->plane_res.hubp;
 	int dpp_id = pipe_ctx->plane_res.dpp->inst;
@@ -649,7 +892,7 @@ static void plane_atomic_power_down(struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
 	struct dce_hwseq *hws = dc->hwseq;
 	struct dpp *dpp = pipe_ctx->plane_res.dpp;
-	struct dc_context *ctx = dc->ctx;
+	DC_LOGGER_INIT(dc->ctx->logger);
 
 	if (REG(DC_IP_REQUEST_CNTL)) {
 		REG_SET(DC_IP_REQUEST_CNTL, 0,
@@ -699,7 +942,7 @@ static void plane_atomic_disable(struct dc *dc, struct pipe_ctx *pipe_ctx)
 
 static void dcn10_disable_plane(struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
-	struct dc_context *ctx = dc->ctx;
+	DC_LOGGER_INIT(dc->ctx->logger);
 
 	if (!pipe_ctx->plane_res.hubp || pipe_ctx->plane_res.hubp->power_gated)
 		return;
@@ -800,7 +1043,7 @@ static void dcn10_init_hw(struct dc *dc)
 		dc->res_pool->opps[i]->mpcc_disconnect_pending[pipe_ctx->plane_res.mpcc_inst] = true;
 		pipe_ctx->stream_res.opp = dc->res_pool->opps[i];
 
-		plane_atomic_disconnect(dc, pipe_ctx);
+		hwss1_plane_atomic_disconnect(dc, pipe_ctx);
 	}
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
@@ -945,9 +1188,8 @@ static bool dcn10_set_input_transfer_func(struct pipe_ctx *pipe_ctx,
 		tf = plane_state->in_transfer_func;
 
 	if (plane_state->gamma_correction &&
-		plane_state->gamma_correction->is_identity)
-		dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_BYPASS);
-	else if (plane_state->gamma_correction && dce_use_lut(plane_state->format))
+		!plane_state->gamma_correction->is_identity
+			&& dce_use_lut(plane_state->format))
 		dpp_base->funcs->dpp_program_input_lut(dpp_base, plane_state->gamma_correction);
 
 	if (tf == NULL)
@@ -1433,7 +1675,7 @@ static void program_csc_matrix(struct pipe_ctx *pipe_ctx,
 	}
 }
 
-static void program_output_csc(struct dc *dc,
+static void dcn10_program_output_csc(struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
 		enum dc_color_space colorspace,
 		uint16_t *matrix,
@@ -1542,22 +1784,22 @@ static uint16_t fixed_point_to_int_frac(
 
 	uint16_t result;
 
-	uint16_t d = (uint16_t)dal_fixed31_32_floor(
-		dal_fixed31_32_abs(
+	uint16_t d = (uint16_t)dc_fixpt_floor(
+		dc_fixpt_abs(
 			arg));
 
 	if (d <= (uint16_t)(1 << integer_bits) - (1 / (uint16_t)divisor))
-		numerator = (uint16_t)dal_fixed31_32_floor(
-			dal_fixed31_32_mul_int(
+		numerator = (uint16_t)dc_fixpt_floor(
+			dc_fixpt_mul_int(
 				arg,
 				divisor));
 	else {
-		numerator = dal_fixed31_32_floor(
-			dal_fixed31_32_sub(
-				dal_fixed31_32_from_int(
+		numerator = dc_fixpt_floor(
+			dc_fixpt_sub(
+				dc_fixpt_from_int(
 					1LL << integer_bits),
-				dal_fixed31_32_recip(
-					dal_fixed31_32_from_int(
+				dc_fixpt_recip(
+					dc_fixpt_from_int(
 						divisor))));
 	}
 
@@ -1567,8 +1809,8 @@ static uint16_t fixed_point_to_int_frac(
 		result = (uint16_t)(
 		(1 << (integer_bits + fractional_bits + 1)) + numerator);
 
-	if ((result != 0) && dal_fixed31_32_lt(
-		arg, dal_fixed31_32_zero))
+	if ((result != 0) && dc_fixpt_lt(
+		arg, dc_fixpt_zero))
 		result |= 1 << (integer_bits + fractional_bits);
 
 	return result;
@@ -1582,8 +1824,8 @@ void build_prescale_params(struct  dc_bias_and_scale *bias_and_scale,
 			&& plane_state->input_csc_color_matrix.enable_adjustment
 			&& plane_state->coeff_reduction_factor.value != 0) {
 		bias_and_scale->scale_blue = fixed_point_to_int_frac(
-			dal_fixed31_32_mul(plane_state->coeff_reduction_factor,
-					dal_fixed31_32_from_fraction(256, 255)),
+			dc_fixpt_mul(plane_state->coeff_reduction_factor,
+					dc_fixpt_from_fraction(256, 255)),
 				2,
 				13);
 		bias_and_scale->scale_red = bias_and_scale->scale_blue;
@@ -1623,6 +1865,8 @@ static void update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
 	struct mpc *mpc = dc->res_pool->mpc;
 	struct mpc_tree *mpc_tree_params = &(pipe_ctx->stream_res.opp->mpc_tree_params);
 
+
+
 	/* TODO: proper fix once fpga works */
 
 	if (dc->debug.surface_visual_confirm)
@@ -1649,6 +1893,7 @@ static void update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
 			pipe_ctx->stream->output_color_space)
 					&& per_pixel_alpha;
 
+
 	/*
 	 * TODO: remove hack
 	 * Note: currently there is a bug in init_hw such that
@@ -1659,6 +1904,12 @@ static void update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
 	 */
 	mpcc_id = hubp->inst;
 
+	/* If there is no full update, don't need to touch MPC tree*/
+	if (!pipe_ctx->plane_state->update_flags.bits.full_update) {
+		mpc->funcs->update_blending(mpc, &blnd_cfg, mpcc_id);
+		return;
+	}
+
 	/* check if this MPCC is already being used */
 	new_mpcc = mpc->funcs->get_mpcc_for_dpp(mpc_tree_params, mpcc_id);
 	/* remove MPCC if being used */
@@ -1777,7 +2028,7 @@ static void update_dchubp_dpp(
 		/*gamut remap*/
 		program_gamut_remap(pipe_ctx);
 
-		program_output_csc(dc,
+		dc->hwss.program_output_csc(dc,
 				pipe_ctx,
 				pipe_ctx->stream->output_color_space,
 				pipe_ctx->stream->csc_color_matrix.matrix,
@@ -1810,9 +2061,9 @@ static void update_dchubp_dpp(
 		hubp->funcs->set_blank(hubp, false);
 }
 
-static void dcn10_otg_blank(
+static void dcn10_blank_pixel_data(
 		struct dc *dc,
-		struct stream_resource stream_res,
+		struct stream_resource *stream_res,
 		struct dc_stream_state *stream,
 		bool blank)
 {
@@ -1823,27 +2074,27 @@ static void dcn10_otg_blank(
 	color_space = stream->output_color_space;
 	color_space_to_black_color(dc, color_space, &black_color);
 
-	if (stream_res.tg->funcs->set_blank_color)
-		stream_res.tg->funcs->set_blank_color(
-				stream_res.tg,
+	if (stream_res->tg->funcs->set_blank_color)
+		stream_res->tg->funcs->set_blank_color(
+				stream_res->tg,
 				&black_color);
 
 	if (!blank) {
-		if (stream_res.tg->funcs->set_blank)
-			stream_res.tg->funcs->set_blank(stream_res.tg, blank);
-		if (stream_res.abm)
-			stream_res.abm->funcs->set_abm_level(stream_res.abm, stream->abm_level);
+		if (stream_res->tg->funcs->set_blank)
+			stream_res->tg->funcs->set_blank(stream_res->tg, blank);
+		if (stream_res->abm)
+			stream_res->abm->funcs->set_abm_level(stream_res->abm, stream->abm_level);
 	} else if (blank) {
-		if (stream_res.abm)
-			stream_res.abm->funcs->set_abm_immediate_disable(stream_res.abm);
-		if (stream_res.tg->funcs->set_blank)
-			stream_res.tg->funcs->set_blank(stream_res.tg, blank);
+		if (stream_res->abm)
+			stream_res->abm->funcs->set_abm_immediate_disable(stream_res->abm);
+		if (stream_res->tg->funcs->set_blank)
+			stream_res->tg->funcs->set_blank(stream_res->tg, blank);
 	}
 }
 
 static void set_hdr_multiplier(struct pipe_ctx *pipe_ctx)
 {
-	struct fixed31_32 multiplier = dal_fixed31_32_from_fraction(
+	struct fixed31_32 multiplier = dc_fixpt_from_fraction(
 			pipe_ctx->plane_state->sdr_white_level, 80);
 	uint32_t hw_mult = 0x1f000; // 1.0 default multiplier
 	struct custom_float_format fmt;
@@ -1876,7 +2127,7 @@ static void program_all_pipe_in_tree(
 		pipe_ctx->stream_res.tg->funcs->program_global_sync(
 				pipe_ctx->stream_res.tg);
 
-		dcn10_otg_blank(dc, pipe_ctx->stream_res,
+		dc->hwss.blank_pixel_data(dc, &pipe_ctx->stream_res,
 				pipe_ctx->stream, blank);
 	}
 
@@ -1983,9 +2234,9 @@ static void dcn10_apply_ctx_for_surface(
 	bool removed_pipe[4] = { false };
 	unsigned int ref_clk_mhz = dc->res_pool->ref_clock_inKhz/1000;
 	bool program_water_mark = false;
-	struct dc_context *ctx = dc->ctx;
 	struct pipe_ctx *top_pipe_to_program =
 			find_top_pipe_for_stream(dc, context, stream);
+	DC_LOGGER_INIT(dc->ctx->logger);
 
 	if (!top_pipe_to_program)
 		return;
@@ -1996,7 +2247,7 @@ static void dcn10_apply_ctx_for_surface(
 
 	if (num_planes == 0) {
 		/* OTG blank before remove all front end */
-		dcn10_otg_blank(dc, top_pipe_to_program->stream_res, top_pipe_to_program->stream, true);
+		dc->hwss.blank_pixel_data(dc, &top_pipe_to_program->stream_res, top_pipe_to_program->stream, true);
 	}
 
 	/* Disconnect unused mpcc */
@@ -2027,7 +2278,7 @@ static void dcn10_apply_ctx_for_surface(
 			old_pipe_ctx->plane_state &&
 			old_pipe_ctx->stream_res.tg == tg) {
 
-			plane_atomic_disconnect(dc, old_pipe_ctx);
+			hwss1_plane_atomic_disconnect(dc, old_pipe_ctx);
 			removed_pipe[i] = true;
 
 			DC_LOG_DC(
@@ -2335,15 +2586,6 @@ static void set_static_screen_control(struct pipe_ctx **pipe_ctx,
 			set_static_screen_control(pipe_ctx[i]->stream_res.tg, value);
 }
 
-static void set_plane_config(
-	const struct dc *dc,
-	struct pipe_ctx *pipe_ctx,
-	struct resource_context *res_ctx)
-{
-	/* TODO */
-	program_gamut_remap(pipe_ctx);
-}
-
 static void dcn10_config_stereo_parameters(
 		struct dc_stream_state *stream, struct crtc_stereo_flags *flags)
 {
@@ -2521,12 +2763,12 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 	.init_hw = dcn10_init_hw,
 	.apply_ctx_to_hw = dce110_apply_ctx_to_hw,
 	.apply_ctx_for_surface = dcn10_apply_ctx_for_surface,
-	.set_plane_config = set_plane_config,
 	.update_plane_addr = dcn10_update_plane_addr,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.set_input_transfer_func = dcn10_set_input_transfer_func,
 	.set_output_transfer_func = dcn10_set_output_transfer_func,
+	.program_output_csc = dcn10_program_output_csc,
 	.power_down = dce110_power_down,
 	.enable_accelerated_mode = dce110_enable_accelerated_mode,
 	.enable_timing_synchronization = dcn10_enable_timing_synchronization,
@@ -2538,10 +2780,11 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 	.blank_stream = dce110_blank_stream,
 	.enable_display_power_gating = dcn10_dummy_display_power_gating,
 	.disable_plane = dcn10_disable_plane,
+	.blank_pixel_data = dcn10_blank_pixel_data,
 	.pipe_control_lock = dcn10_pipe_control_lock,
 	.set_bandwidth = dcn10_set_bandwidth,
 	.reset_hw_ctx_wrap = reset_hw_ctx_wrap,
-	.prog_pixclk_crtc_otg = dcn10_prog_pixclk_crtc_otg,
+	.enable_stream_timing = dcn10_enable_stream_timing,
 	.set_drr = set_drr,
 	.get_position = get_position,
 	.set_static_screen_control = set_static_screen_control,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
index 6c526b5095d9..44f734b73f9e 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
@@ -37,4 +37,6 @@ extern void fill_display_configs(
 
 bool is_rgb_cspace(enum dc_color_space output_color_space);
 
+void hwss1_plane_atomic_disconnect(struct dc *dc, struct pipe_ctx *pipe_ctx);
+
 #endif /* __DC_HWSS_DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
new file mode 100644
index 000000000000..21fa40ac0786
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
@@ -0,0 +1,1362 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "reg_helper.h"
+
+#include "core_types.h"
+#include "link_encoder.h"
+#include "dcn10_link_encoder.h"
+#include "stream_encoder.h"
+#include "i2caux_interface.h"
+#include "dc_bios_types.h"
+
+#include "gpio_service_interface.h"
+
+#define CTX \
+	enc10->base.ctx
+#define DC_LOGGER \
+	enc10->base.ctx->logger
+
+#define REG(reg)\
+	(enc10->link_regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+	enc10->link_shift->field_name, enc10->link_mask->field_name
+
+
+/*
+ * @brief
+ * Trigger Source Select
+ * ASIC-dependent, actual values for register programming
+ */
+#define DCN10_DIG_FE_SOURCE_SELECT_INVALID 0x0
+#define DCN10_DIG_FE_SOURCE_SELECT_DIGA 0x1
+#define DCN10_DIG_FE_SOURCE_SELECT_DIGB 0x2
+#define DCN10_DIG_FE_SOURCE_SELECT_DIGC 0x4
+#define DCN10_DIG_FE_SOURCE_SELECT_DIGD 0x08
+#define DCN10_DIG_FE_SOURCE_SELECT_DIGE 0x10
+#define DCN10_DIG_FE_SOURCE_SELECT_DIGF 0x20
+#define DCN10_DIG_FE_SOURCE_SELECT_DIGG 0x40
+
+enum {
+	DP_MST_UPDATE_MAX_RETRY = 50
+};
+
+
+
+static void aux_initialize(struct dcn10_link_encoder *enc10);
+
+
+static const struct link_encoder_funcs dcn10_lnk_enc_funcs = {
+	.validate_output_with_stream =
+		dcn10_link_encoder_validate_output_with_stream,
+	.hw_init = dcn10_link_encoder_hw_init,
+	.setup = dcn10_link_encoder_setup,
+	.enable_tmds_output = dcn10_link_encoder_enable_tmds_output,
+	.enable_dp_output = dcn10_link_encoder_enable_dp_output,
+	.enable_dp_mst_output = dcn10_link_encoder_enable_dp_mst_output,
+	.disable_output = dcn10_link_encoder_disable_output,
+	.dp_set_lane_settings = dcn10_link_encoder_dp_set_lane_settings,
+	.dp_set_phy_pattern = dcn10_link_encoder_dp_set_phy_pattern,
+	.update_mst_stream_allocation_table =
+		dcn10_link_encoder_update_mst_stream_allocation_table,
+	.psr_program_dp_dphy_fast_training =
+			dcn10_psr_program_dp_dphy_fast_training,
+	.psr_program_secondary_packet = dcn10_psr_program_secondary_packet,
+	.connect_dig_be_to_fe = dcn10_link_encoder_connect_dig_be_to_fe,
+	.enable_hpd = dcn10_link_encoder_enable_hpd,
+	.disable_hpd = dcn10_link_encoder_disable_hpd,
+	.is_dig_enabled = dcn10_is_dig_enabled,
+	.destroy = dcn10_link_encoder_destroy
+};
+
+static enum bp_result link_transmitter_control(
+	struct dcn10_link_encoder *enc10,
+	struct bp_transmitter_control *cntl)
+{
+	enum bp_result result;
+	struct dc_bios *bp = enc10->base.ctx->dc_bios;
+
+	result = bp->funcs->transmitter_control(bp, cntl);
+
+	return result;
+}
+
+static void enable_phy_bypass_mode(
+	struct dcn10_link_encoder *enc10,
+	bool enable)
+{
+	/* This register resides in DP back end block;
+	 * transmitter is used for the offset
+	 */
+	REG_UPDATE(DP_DPHY_CNTL, DPHY_BYPASS, enable);
+
+}
+
+static void disable_prbs_symbols(
+	struct dcn10_link_encoder *enc10,
+	bool disable)
+{
+	/* This register resides in DP back end block;
+	 * transmitter is used for the offset
+	 */
+	REG_UPDATE_4(DP_DPHY_CNTL,
+			DPHY_ATEST_SEL_LANE0, disable,
+			DPHY_ATEST_SEL_LANE1, disable,
+			DPHY_ATEST_SEL_LANE2, disable,
+			DPHY_ATEST_SEL_LANE3, disable);
+}
+
+static void disable_prbs_mode(
+	struct dcn10_link_encoder *enc10)
+{
+	REG_UPDATE(DP_DPHY_PRBS_CNTL, DPHY_PRBS_EN, 0);
+}
+
+static void program_pattern_symbols(
+	struct dcn10_link_encoder *enc10,
+	uint16_t pattern_symbols[8])
+{
+	/* This register resides in DP back end block;
+	 * transmitter is used for the offset
+	 */
+	REG_SET_3(DP_DPHY_SYM0, 0,
+			DPHY_SYM1, pattern_symbols[0],
+			DPHY_SYM2, pattern_symbols[1],
+			DPHY_SYM3, pattern_symbols[2]);
+
+	/* This register resides in DP back end block;
+	 * transmitter is used for the offset
+	 */
+	REG_SET_3(DP_DPHY_SYM1, 0,
+			DPHY_SYM4, pattern_symbols[3],
+			DPHY_SYM5, pattern_symbols[4],
+			DPHY_SYM6, pattern_symbols[5]);
+
+	/* This register resides in DP back end block;
+	 * transmitter is used for the offset
+	 */
+	REG_SET_2(DP_DPHY_SYM2, 0,
+			DPHY_SYM7, pattern_symbols[6],
+			DPHY_SYM8, pattern_symbols[7]);
+}
+
+static void set_dp_phy_pattern_d102(
+	struct dcn10_link_encoder *enc10)
+{
+	/* Disable PHY Bypass mode to setup the test pattern */
+	enable_phy_bypass_mode(enc10, false);
+
+	/* For 10-bit PRBS or debug symbols
+	 * please use the following sequence:
+	 *
+	 * Enable debug symbols on the lanes
+	 */
+	disable_prbs_symbols(enc10, true);
+
+	/* Disable PRBS mode */
+	disable_prbs_mode(enc10);
+
+	/* Program debug symbols to be output */
+	{
+		uint16_t pattern_symbols[8] = {
+			0x2AA, 0x2AA, 0x2AA, 0x2AA,
+			0x2AA, 0x2AA, 0x2AA, 0x2AA
+		};
+
+		program_pattern_symbols(enc10, pattern_symbols);
+	}
+
+	/* Enable phy bypass mode to enable the test pattern */
+
+	enable_phy_bypass_mode(enc10, true);
+}
+
+static void set_link_training_complete(
+	struct dcn10_link_encoder *enc10,
+	bool complete)
+{
+	/* This register resides in DP back end block;
+	 * transmitter is used for the offset
+	 */
+	REG_UPDATE(DP_LINK_CNTL, DP_LINK_TRAINING_COMPLETE, complete);
+
+}
+
+void dcn10_link_encoder_set_dp_phy_pattern_training_pattern(
+	struct link_encoder *enc,
+	uint32_t index)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	/* Write Training Pattern */
+
+	REG_WRITE(DP_DPHY_TRAINING_PATTERN_SEL, index);
+
+	/* Set HW Register Training Complete to false */
+
+	set_link_training_complete(enc10, false);
+
+	/* Disable PHY Bypass mode to output Training Pattern */
+
+	enable_phy_bypass_mode(enc10, false);
+
+	/* Disable PRBS mode */
+	disable_prbs_mode(enc10);
+}
+
+static void setup_panel_mode(
+	struct dcn10_link_encoder *enc10,
+	enum dp_panel_mode panel_mode)
+{
+	uint32_t value;
+
+	ASSERT(REG(DP_DPHY_INTERNAL_CTRL));
+	value = REG_READ(DP_DPHY_INTERNAL_CTRL);
+
+	switch (panel_mode) {
+	case DP_PANEL_MODE_EDP:
+		value = 0x1;
+		break;
+	case DP_PANEL_MODE_SPECIAL:
+		value = 0x11;
+		break;
+	default:
+		value = 0x0;
+		break;
+	}
+
+	REG_WRITE(DP_DPHY_INTERNAL_CTRL, value);
+}
+
+static void set_dp_phy_pattern_symbol_error(
+	struct dcn10_link_encoder *enc10)
+{
+	/* Disable PHY Bypass mode to setup the test pattern */
+	enable_phy_bypass_mode(enc10, false);
+
+	/* program correct panel mode*/
+	setup_panel_mode(enc10, DP_PANEL_MODE_DEFAULT);
+
+	/* A PRBS23 pattern is used for most DP electrical measurements. */
+
+	/* Enable PRBS symbols on the lanes */
+	disable_prbs_symbols(enc10, false);
+
+	/* For PRBS23 Set bit DPHY_PRBS_SEL=1 and Set bit DPHY_PRBS_EN=1 */
+	REG_UPDATE_2(DP_DPHY_PRBS_CNTL,
+			DPHY_PRBS_SEL, 1,
+			DPHY_PRBS_EN, 1);
+
+	/* Enable phy bypass mode to enable the test pattern */
+	enable_phy_bypass_mode(enc10, true);
+}
+
+static void set_dp_phy_pattern_prbs7(
+	struct dcn10_link_encoder *enc10)
+{
+	/* Disable PHY Bypass mode to setup the test pattern */
+	enable_phy_bypass_mode(enc10, false);
+
+	/* A PRBS7 pattern is used for most DP electrical measurements. */
+
+	/* Enable PRBS symbols on the lanes */
+	disable_prbs_symbols(enc10, false);
+
+	/* For PRBS7 Set bit DPHY_PRBS_SEL=0 and Set bit DPHY_PRBS_EN=1 */
+	REG_UPDATE_2(DP_DPHY_PRBS_CNTL,
+			DPHY_PRBS_SEL, 0,
+			DPHY_PRBS_EN, 1);
+
+	/* Enable phy bypass mode to enable the test pattern */
+	enable_phy_bypass_mode(enc10, true);
+}
+
+static void set_dp_phy_pattern_80bit_custom(
+	struct dcn10_link_encoder *enc10,
+	const uint8_t *pattern)
+{
+	/* Disable PHY Bypass mode to setup the test pattern */
+	enable_phy_bypass_mode(enc10, false);
+
+	/* Enable debug symbols on the lanes */
+
+	disable_prbs_symbols(enc10, true);
+
+	/* Enable PHY bypass mode to enable the test pattern */
+	/* TODO is it really needed ? */
+
+	enable_phy_bypass_mode(enc10, true);
+
+	/* Program 80 bit custom pattern */
+	{
+		uint16_t pattern_symbols[8];
+
+		pattern_symbols[0] =
+			((pattern[1] & 0x03) << 8) | pattern[0];
+		pattern_symbols[1] =
+			((pattern[2] & 0x0f) << 6) | ((pattern[1] >> 2) & 0x3f);
+		pattern_symbols[2] =
+			((pattern[3] & 0x3f) << 4) | ((pattern[2] >> 4) & 0x0f);
+		pattern_symbols[3] =
+			(pattern[4] << 2) | ((pattern[3] >> 6) & 0x03);
+		pattern_symbols[4] =
+			((pattern[6] & 0x03) << 8) | pattern[5];
+		pattern_symbols[5] =
+			((pattern[7] & 0x0f) << 6) | ((pattern[6] >> 2) & 0x3f);
+		pattern_symbols[6] =
+			((pattern[8] & 0x3f) << 4) | ((pattern[7] >> 4) & 0x0f);
+		pattern_symbols[7] =
+			(pattern[9] << 2) | ((pattern[8] >> 6) & 0x03);
+
+		program_pattern_symbols(enc10, pattern_symbols);
+	}
+
+	/* Enable phy bypass mode to enable the test pattern */
+
+	enable_phy_bypass_mode(enc10, true);
+}
+
+static void set_dp_phy_pattern_hbr2_compliance_cp2520_2(
+	struct dcn10_link_encoder *enc10,
+	unsigned int cp2520_pattern)
+{
+
+	/* previously there is a register DP_HBR2_EYE_PATTERN
+	 * that is enabled to get the pattern.
+	 * But it does not work with the latest spec change,
+	 * so we are programming the following registers manually.
+	 *
+	 * The following settings have been confirmed
+	 * by Nick Chorney and Sandra Liu
+	 */
+
+	/* Disable PHY Bypass mode to setup the test pattern */
+
+	enable_phy_bypass_mode(enc10, false);
+
+	/* Setup DIG encoder in DP SST mode */
+	enc10->base.funcs->setup(&enc10->base, SIGNAL_TYPE_DISPLAY_PORT);
+
+	/* ensure normal panel mode. */
+	setup_panel_mode(enc10, DP_PANEL_MODE_DEFAULT);
+
+	/* no vbid after BS (SR)
+	 * DP_LINK_FRAMING_CNTL changed history Sandra Liu
+	 * 11000260 / 11000104 / 110000FC
+	 */
+	REG_UPDATE_3(DP_LINK_FRAMING_CNTL,
+			DP_IDLE_BS_INTERVAL, 0xFC,
+			DP_VBID_DISABLE, 1,
+			DP_VID_ENHANCED_FRAME_MODE, 1);
+
+	/* swap every BS with SR */
+	REG_UPDATE(DP_DPHY_SCRAM_CNTL, DPHY_SCRAMBLER_BS_COUNT, 0);
+
+	/* select cp2520 patterns */
+	if (REG(DP_DPHY_HBR2_PATTERN_CONTROL))
+		REG_UPDATE(DP_DPHY_HBR2_PATTERN_CONTROL,
+				DP_DPHY_HBR2_PATTERN_CONTROL, cp2520_pattern);
+	else
+		/* pre-DCE11 can only generate CP2520 pattern 2 */
+		ASSERT(cp2520_pattern == 2);
+
+	/* set link training complete */
+	set_link_training_complete(enc10, true);
+
+	/* disable video stream */
+	REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, 0);
+
+	/* Disable PHY Bypass mode to setup the test pattern */
+	enable_phy_bypass_mode(enc10, false);
+}
+
+static void set_dp_phy_pattern_passthrough_mode(
+	struct dcn10_link_encoder *enc10,
+	enum dp_panel_mode panel_mode)
+{
+	/* program correct panel mode */
+	setup_panel_mode(enc10, panel_mode);
+
+	/* restore LINK_FRAMING_CNTL and DPHY_SCRAMBLER_BS_COUNT
+	 * in case we were doing HBR2 compliance pattern before
+	 */
+	REG_UPDATE_3(DP_LINK_FRAMING_CNTL,
+			DP_IDLE_BS_INTERVAL, 0x2000,
+			DP_VBID_DISABLE, 0,
+			DP_VID_ENHANCED_FRAME_MODE, 1);
+
+	REG_UPDATE(DP_DPHY_SCRAM_CNTL, DPHY_SCRAMBLER_BS_COUNT, 0x1FF);
+
+	/* set link training complete */
+	set_link_training_complete(enc10, true);
+
+	/* Disable PHY Bypass mode to setup the test pattern */
+	enable_phy_bypass_mode(enc10, false);
+
+	/* Disable PRBS mode */
+	disable_prbs_mode(enc10);
+}
+
+/* return value is bit-vector */
+static uint8_t get_frontend_source(
+	enum engine_id engine)
+{
+	switch (engine) {
+	case ENGINE_ID_DIGA:
+		return DCN10_DIG_FE_SOURCE_SELECT_DIGA;
+	case ENGINE_ID_DIGB:
+		return DCN10_DIG_FE_SOURCE_SELECT_DIGB;
+	case ENGINE_ID_DIGC:
+		return DCN10_DIG_FE_SOURCE_SELECT_DIGC;
+	case ENGINE_ID_DIGD:
+		return DCN10_DIG_FE_SOURCE_SELECT_DIGD;
+	case ENGINE_ID_DIGE:
+		return DCN10_DIG_FE_SOURCE_SELECT_DIGE;
+	case ENGINE_ID_DIGF:
+		return DCN10_DIG_FE_SOURCE_SELECT_DIGF;
+	case ENGINE_ID_DIGG:
+		return DCN10_DIG_FE_SOURCE_SELECT_DIGG;
+	default:
+		ASSERT_CRITICAL(false);
+		return DCN10_DIG_FE_SOURCE_SELECT_INVALID;
+	}
+}
+
+static void configure_encoder(
+	struct dcn10_link_encoder *enc10,
+	const struct dc_link_settings *link_settings)
+{
+	/* set number of lanes */
+
+	REG_SET(DP_CONFIG, 0,
+			DP_UDI_LANES, link_settings->lane_count - LANE_COUNT_ONE);
+
+	/* setup scrambler */
+	REG_UPDATE(DP_DPHY_SCRAM_CNTL, DPHY_SCRAMBLER_ADVANCE, 1);
+}
+
+void dcn10_psr_program_dp_dphy_fast_training(struct link_encoder *enc,
+			bool exit_link_training_required)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+
+	if (exit_link_training_required)
+		REG_UPDATE(DP_DPHY_FAST_TRAINING,
+				DPHY_RX_FAST_TRAINING_CAPABLE, 1);
+	else {
+		REG_UPDATE(DP_DPHY_FAST_TRAINING,
+				DPHY_RX_FAST_TRAINING_CAPABLE, 0);
+		/*In DCE 11, we are able to pre-program a Force SR register
+		 * to be able to trigger SR symbol after 5 idle patterns
+		 * transmitted. Upon PSR Exit, DMCU can trigger
+		 * DPHY_LOAD_BS_COUNT_START = 1. Upon writing 1 to
+		 * DPHY_LOAD_BS_COUNT_START and the internal counter
+		 * reaches DPHY_LOAD_BS_COUNT, the next BS symbol will be
+		 * replaced by SR symbol once.
+		 */
+
+		REG_UPDATE(DP_DPHY_BS_SR_SWAP_CNTL, DPHY_LOAD_BS_COUNT, 0x5);
+	}
+}
+
+void dcn10_psr_program_secondary_packet(struct link_encoder *enc,
+			unsigned int sdp_transmit_line_num_deadline)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+
+	REG_UPDATE_2(DP_SEC_CNTL1,
+		DP_SEC_GSP0_LINE_NUM, sdp_transmit_line_num_deadline,
+		DP_SEC_GSP0_PRIORITY, 1);
+}
+
+bool dcn10_is_dig_enabled(struct link_encoder *enc)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	uint32_t value;
+
+	REG_GET(DIG_BE_EN_CNTL, DIG_ENABLE, &value);
+	return value;
+}
+
+static void link_encoder_disable(struct dcn10_link_encoder *enc10)
+{
+	/* reset training pattern */
+	REG_SET(DP_DPHY_TRAINING_PATTERN_SEL, 0,
+			DPHY_TRAINING_PATTERN_SEL, 0);
+
+	/* reset training complete */
+	REG_UPDATE(DP_LINK_CNTL, DP_LINK_TRAINING_COMPLETE, 0);
+
+	/* reset panel mode */
+	setup_panel_mode(enc10, DP_PANEL_MODE_DEFAULT);
+}
+
+static void hpd_initialize(
+	struct dcn10_link_encoder *enc10)
+{
+	/* Associate HPD with DIG_BE */
+	enum hpd_source_id hpd_source = enc10->base.hpd_source;
+
+	REG_UPDATE(DIG_BE_CNTL, DIG_HPD_SELECT, hpd_source);
+}
+
+bool dcn10_link_encoder_validate_dvi_output(
+	const struct dcn10_link_encoder *enc10,
+	enum signal_type connector_signal,
+	enum signal_type signal,
+	const struct dc_crtc_timing *crtc_timing)
+{
+	uint32_t max_pixel_clock = TMDS_MAX_PIXEL_CLOCK;
+
+	if (signal == SIGNAL_TYPE_DVI_DUAL_LINK)
+		max_pixel_clock *= 2;
+
+	/* This handles the case of HDMI downgrade to DVI we don't want to
+	 * we don't want to cap the pixel clock if the DDI is not DVI.
+	 */
+	if (connector_signal != SIGNAL_TYPE_DVI_DUAL_LINK &&
+			connector_signal != SIGNAL_TYPE_DVI_SINGLE_LINK)
+		max_pixel_clock = enc10->base.features.max_hdmi_pixel_clock;
+
+	/* DVI only support RGB pixel encoding */
+	if (crtc_timing->pixel_encoding != PIXEL_ENCODING_RGB)
+		return false;
+
+	/*connect DVI via adpater's HDMI connector*/
+	if ((connector_signal == SIGNAL_TYPE_DVI_SINGLE_LINK ||
+		connector_signal == SIGNAL_TYPE_HDMI_TYPE_A) &&
+		signal != SIGNAL_TYPE_HDMI_TYPE_A &&
+		crtc_timing->pix_clk_khz > TMDS_MAX_PIXEL_CLOCK)
+		return false;
+	if (crtc_timing->pix_clk_khz < TMDS_MIN_PIXEL_CLOCK)
+		return false;
+
+	if (crtc_timing->pix_clk_khz > max_pixel_clock)
+		return false;
+
+	/* DVI supports 6/8bpp single-link and 10/16bpp dual-link */
+	switch (crtc_timing->display_color_depth) {
+	case COLOR_DEPTH_666:
+	case COLOR_DEPTH_888:
+	break;
+	case COLOR_DEPTH_101010:
+	case COLOR_DEPTH_161616:
+		if (signal != SIGNAL_TYPE_DVI_DUAL_LINK)
+			return false;
+	break;
+	default:
+		return false;
+	}
+
+	return true;
+}
+
+static bool dcn10_link_encoder_validate_hdmi_output(
+	const struct dcn10_link_encoder *enc10,
+	const struct dc_crtc_timing *crtc_timing,
+	int adjusted_pix_clk_khz)
+{
+	enum dc_color_depth max_deep_color =
+			enc10->base.features.max_hdmi_deep_color;
+
+	if (max_deep_color < crtc_timing->display_color_depth)
+		return false;
+
+	if (crtc_timing->display_color_depth < COLOR_DEPTH_888)
+		return false;
+	if (adjusted_pix_clk_khz < TMDS_MIN_PIXEL_CLOCK)
+		return false;
+
+	if ((adjusted_pix_clk_khz == 0) ||
+		(adjusted_pix_clk_khz > enc10->base.features.max_hdmi_pixel_clock))
+		return false;
+
+	/* DCE11 HW does not support 420 */
+	if (!enc10->base.features.ycbcr420_supported &&
+			crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR420)
+		return false;
+
+	if (!enc10->base.features.flags.bits.HDMI_6GB_EN &&
+		adjusted_pix_clk_khz >= 300000)
+		return false;
+	return true;
+}
+
+bool dcn10_link_encoder_validate_dp_output(
+	const struct dcn10_link_encoder *enc10,
+	const struct dc_crtc_timing *crtc_timing)
+{
+	/* default RGB only */
+	if (crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB)
+		return true;
+
+	if (enc10->base.features.flags.bits.IS_YCBCR_CAPABLE)
+		return true;
+
+	/* for DCE 8.x or later DP Y-only feature,
+	 * we need ASIC cap + FeatureSupportDPYonly, not support 666
+	 */
+	if (crtc_timing->flags.Y_ONLY &&
+		enc10->base.features.flags.bits.IS_YCBCR_CAPABLE &&
+		crtc_timing->display_color_depth != COLOR_DEPTH_666)
+		return true;
+
+	return false;
+}
+
+void dcn10_link_encoder_construct(
+	struct dcn10_link_encoder *enc10,
+	const struct encoder_init_data *init_data,
+	const struct encoder_feature_support *enc_features,
+	const struct dcn10_link_enc_registers *link_regs,
+	const struct dcn10_link_enc_aux_registers *aux_regs,
+	const struct dcn10_link_enc_hpd_registers *hpd_regs,
+	const struct dcn10_link_enc_shift *link_shift,
+	const struct dcn10_link_enc_mask *link_mask)
+{
+	struct bp_encoder_cap_info bp_cap_info = {0};
+	const struct dc_vbios_funcs *bp_funcs = init_data->ctx->dc_bios->funcs;
+	enum bp_result result = BP_RESULT_OK;
+
+	enc10->base.funcs = &dcn10_lnk_enc_funcs;
+	enc10->base.ctx = init_data->ctx;
+	enc10->base.id = init_data->encoder;
+
+	enc10->base.hpd_source = init_data->hpd_source;
+	enc10->base.connector = init_data->connector;
+
+	enc10->base.preferred_engine = ENGINE_ID_UNKNOWN;
+
+	enc10->base.features = *enc_features;
+
+	enc10->base.transmitter = init_data->transmitter;
+
+	/* set the flag to indicate whether driver poll the I2C data pin
+	 * while doing the DP sink detect
+	 */
+
+/*	if (dal_adapter_service_is_feature_supported(as,
+		FEATURE_DP_SINK_DETECT_POLL_DATA_PIN))
+		enc10->base.features.flags.bits.
+			DP_SINK_DETECT_POLL_DATA_PIN = true;*/
+
+	enc10->base.output_signals =
+		SIGNAL_TYPE_DVI_SINGLE_LINK |
+		SIGNAL_TYPE_DVI_DUAL_LINK |
+		SIGNAL_TYPE_LVDS |
+		SIGNAL_TYPE_DISPLAY_PORT |
+		SIGNAL_TYPE_DISPLAY_PORT_MST |
+		SIGNAL_TYPE_EDP |
+		SIGNAL_TYPE_HDMI_TYPE_A;
+
+	/* For DCE 8.0 and 8.1, by design, UNIPHY is hardwired to DIG_BE.
+	 * SW always assign DIG_FE 1:1 mapped to DIG_FE for non-MST UNIPHY.
+	 * SW assign DIG_FE to non-MST UNIPHY first and MST last. So prefer
+	 * DIG is per UNIPHY and used by SST DP, eDP, HDMI, DVI and LVDS.
+	 * Prefer DIG assignment is decided by board design.
+	 * For DCE 8.0, there are only max 6 UNIPHYs, we assume board design
+	 * and VBIOS will filter out 7 UNIPHY for DCE 8.0.
+	 * By this, adding DIGG should not hurt DCE 8.0.
+	 * This will let DCE 8.1 share DCE 8.0 as much as possible
+	 */
+
+	enc10->link_regs = link_regs;
+	enc10->aux_regs = aux_regs;
+	enc10->hpd_regs = hpd_regs;
+	enc10->link_shift = link_shift;
+	enc10->link_mask = link_mask;
+
+	switch (enc10->base.transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		enc10->base.preferred_engine = ENGINE_ID_DIGA;
+	break;
+	case TRANSMITTER_UNIPHY_B:
+		enc10->base.preferred_engine = ENGINE_ID_DIGB;
+	break;
+	case TRANSMITTER_UNIPHY_C:
+		enc10->base.preferred_engine = ENGINE_ID_DIGC;
+	break;
+	case TRANSMITTER_UNIPHY_D:
+		enc10->base.preferred_engine = ENGINE_ID_DIGD;
+	break;
+	case TRANSMITTER_UNIPHY_E:
+		enc10->base.preferred_engine = ENGINE_ID_DIGE;
+	break;
+	case TRANSMITTER_UNIPHY_F:
+		enc10->base.preferred_engine = ENGINE_ID_DIGF;
+	break;
+	case TRANSMITTER_UNIPHY_G:
+		enc10->base.preferred_engine = ENGINE_ID_DIGG;
+	break;
+	default:
+		ASSERT_CRITICAL(false);
+		enc10->base.preferred_engine = ENGINE_ID_UNKNOWN;
+	}
+
+	/* default to one to mirror Windows behavior */
+	enc10->base.features.flags.bits.HDMI_6GB_EN = 1;
+
+	result = bp_funcs->get_encoder_cap_info(enc10->base.ctx->dc_bios,
+						enc10->base.id, &bp_cap_info);
+
+	/* Override features with DCE-specific values */
+	if (result == BP_RESULT_OK) {
+		enc10->base.features.flags.bits.IS_HBR2_CAPABLE =
+				bp_cap_info.DP_HBR2_EN;
+		enc10->base.features.flags.bits.IS_HBR3_CAPABLE =
+				bp_cap_info.DP_HBR3_EN;
+		enc10->base.features.flags.bits.HDMI_6GB_EN = bp_cap_info.HDMI_6GB_EN;
+	} else {
+		DC_LOG_WARNING("%s: Failed to get encoder_cap_info from VBIOS with error code %d!\n",
+				__func__,
+				result);
+	}
+}
+
+bool dcn10_link_encoder_validate_output_with_stream(
+	struct link_encoder *enc,
+	const struct dc_stream_state *stream)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	bool is_valid;
+
+	switch (stream->signal) {
+	case SIGNAL_TYPE_DVI_SINGLE_LINK:
+	case SIGNAL_TYPE_DVI_DUAL_LINK:
+		is_valid = dcn10_link_encoder_validate_dvi_output(
+			enc10,
+			stream->sink->link->connector_signal,
+			stream->signal,
+			&stream->timing);
+	break;
+	case SIGNAL_TYPE_HDMI_TYPE_A:
+		is_valid = dcn10_link_encoder_validate_hdmi_output(
+				enc10,
+				&stream->timing,
+				stream->phy_pix_clk);
+	break;
+	case SIGNAL_TYPE_DISPLAY_PORT:
+	case SIGNAL_TYPE_DISPLAY_PORT_MST:
+		is_valid = dcn10_link_encoder_validate_dp_output(
+					enc10, &stream->timing);
+	break;
+	case SIGNAL_TYPE_EDP:
+		is_valid = (stream->timing.pixel_encoding == PIXEL_ENCODING_RGB) ? true : false;
+	break;
+	case SIGNAL_TYPE_VIRTUAL:
+		is_valid = true;
+		break;
+	default:
+		is_valid = false;
+	break;
+	}
+
+	return is_valid;
+}
+
+void dcn10_link_encoder_hw_init(
+	struct link_encoder *enc)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	struct bp_transmitter_control cntl = { 0 };
+	enum bp_result result;
+
+	cntl.action = TRANSMITTER_CONTROL_INIT;
+	cntl.engine_id = ENGINE_ID_UNKNOWN;
+	cntl.transmitter = enc10->base.transmitter;
+	cntl.connector_obj_id = enc10->base.connector;
+	cntl.lanes_number = LANE_COUNT_FOUR;
+	cntl.coherent = false;
+	cntl.hpd_sel = enc10->base.hpd_source;
+
+	if (enc10->base.connector.id == CONNECTOR_ID_EDP)
+		cntl.signal = SIGNAL_TYPE_EDP;
+
+	result = link_transmitter_control(enc10, &cntl);
+
+	if (result != BP_RESULT_OK) {
+		DC_LOG_ERROR("%s: Failed to execute VBIOS command table!\n",
+			__func__);
+		BREAK_TO_DEBUGGER();
+		return;
+	}
+
+	if (enc10->base.connector.id == CONNECTOR_ID_LVDS) {
+		cntl.action = TRANSMITTER_CONTROL_BACKLIGHT_BRIGHTNESS;
+
+		result = link_transmitter_control(enc10, &cntl);
+
+		ASSERT(result == BP_RESULT_OK);
+
+	}
+	aux_initialize(enc10);
+
+	/* reinitialize HPD.
+	 * hpd_initialize() will pass DIG_FE id to HW context.
+	 * All other routine within HW context will use fe_engine_offset
+	 * as DIG_FE id even caller pass DIG_FE id.
+	 * So this routine must be called first.
+	 */
+	hpd_initialize(enc10);
+}
+
+void dcn10_link_encoder_destroy(struct link_encoder **enc)
+{
+	kfree(TO_DCN10_LINK_ENC(*enc));
+	*enc = NULL;
+}
+
+void dcn10_link_encoder_setup(
+	struct link_encoder *enc,
+	enum signal_type signal)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+
+	switch (signal) {
+	case SIGNAL_TYPE_EDP:
+	case SIGNAL_TYPE_DISPLAY_PORT:
+		/* DP SST */
+		REG_UPDATE(DIG_BE_CNTL, DIG_MODE, 0);
+		break;
+	case SIGNAL_TYPE_LVDS:
+		/* LVDS */
+		REG_UPDATE(DIG_BE_CNTL, DIG_MODE, 1);
+		break;
+	case SIGNAL_TYPE_DVI_SINGLE_LINK:
+	case SIGNAL_TYPE_DVI_DUAL_LINK:
+		/* TMDS-DVI */
+		REG_UPDATE(DIG_BE_CNTL, DIG_MODE, 2);
+		break;
+	case SIGNAL_TYPE_HDMI_TYPE_A:
+		/* TMDS-HDMI */
+		REG_UPDATE(DIG_BE_CNTL, DIG_MODE, 3);
+		break;
+	case SIGNAL_TYPE_DISPLAY_PORT_MST:
+		/* DP MST */
+		REG_UPDATE(DIG_BE_CNTL, DIG_MODE, 5);
+		break;
+	default:
+		ASSERT_CRITICAL(false);
+		/* invalid mode ! */
+		break;
+	}
+
+}
+
+/* TODO: still need depth or just pass in adjusted pixel clock? */
+void dcn10_link_encoder_enable_tmds_output(
+	struct link_encoder *enc,
+	enum clock_source_id clock_source,
+	enum dc_color_depth color_depth,
+	enum signal_type signal,
+	uint32_t pixel_clock)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	struct bp_transmitter_control cntl = { 0 };
+	enum bp_result result;
+
+	/* Enable the PHY */
+
+	cntl.action = TRANSMITTER_CONTROL_ENABLE;
+	cntl.engine_id = enc->preferred_engine;
+	cntl.transmitter = enc10->base.transmitter;
+	cntl.pll_id = clock_source;
+	cntl.signal = signal;
+	if (cntl.signal == SIGNAL_TYPE_DVI_DUAL_LINK)
+		cntl.lanes_number = 8;
+	else
+		cntl.lanes_number = 4;
+
+	cntl.hpd_sel = enc10->base.hpd_source;
+
+	cntl.pixel_clock = pixel_clock;
+	cntl.color_depth = color_depth;
+
+	result = link_transmitter_control(enc10, &cntl);
+
+	if (result != BP_RESULT_OK) {
+		DC_LOG_ERROR("%s: Failed to execute VBIOS command table!\n",
+			__func__);
+		BREAK_TO_DEBUGGER();
+	}
+}
+
+/* enables DP PHY output */
+void dcn10_link_encoder_enable_dp_output(
+	struct link_encoder *enc,
+	const struct dc_link_settings *link_settings,
+	enum clock_source_id clock_source)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	struct bp_transmitter_control cntl = { 0 };
+	enum bp_result result;
+
+	/* Enable the PHY */
+
+	/* number_of_lanes is used for pixel clock adjust,
+	 * but it's not passed to asic_control.
+	 * We need to set number of lanes manually.
+	 */
+	configure_encoder(enc10, link_settings);
+
+	cntl.action = TRANSMITTER_CONTROL_ENABLE;
+	cntl.engine_id = enc->preferred_engine;
+	cntl.transmitter = enc10->base.transmitter;
+	cntl.pll_id = clock_source;
+	cntl.signal = SIGNAL_TYPE_DISPLAY_PORT;
+	cntl.lanes_number = link_settings->lane_count;
+	cntl.hpd_sel = enc10->base.hpd_source;
+	cntl.pixel_clock = link_settings->link_rate
+						* LINK_RATE_REF_FREQ_IN_KHZ;
+	/* TODO: check if undefined works */
+	cntl.color_depth = COLOR_DEPTH_UNDEFINED;
+
+	result = link_transmitter_control(enc10, &cntl);
+
+	if (result != BP_RESULT_OK) {
+		DC_LOG_ERROR("%s: Failed to execute VBIOS command table!\n",
+			__func__);
+		BREAK_TO_DEBUGGER();
+	}
+}
+
+/* enables DP PHY output in MST mode */
+void dcn10_link_encoder_enable_dp_mst_output(
+	struct link_encoder *enc,
+	const struct dc_link_settings *link_settings,
+	enum clock_source_id clock_source)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	struct bp_transmitter_control cntl = { 0 };
+	enum bp_result result;
+
+	/* Enable the PHY */
+
+	/* number_of_lanes is used for pixel clock adjust,
+	 * but it's not passed to asic_control.
+	 * We need to set number of lanes manually.
+	 */
+	configure_encoder(enc10, link_settings);
+
+	cntl.action = TRANSMITTER_CONTROL_ENABLE;
+	cntl.engine_id = ENGINE_ID_UNKNOWN;
+	cntl.transmitter = enc10->base.transmitter;
+	cntl.pll_id = clock_source;
+	cntl.signal = SIGNAL_TYPE_DISPLAY_PORT_MST;
+	cntl.lanes_number = link_settings->lane_count;
+	cntl.hpd_sel = enc10->base.hpd_source;
+	cntl.pixel_clock = link_settings->link_rate
+						* LINK_RATE_REF_FREQ_IN_KHZ;
+	/* TODO: check if undefined works */
+	cntl.color_depth = COLOR_DEPTH_UNDEFINED;
+
+	result = link_transmitter_control(enc10, &cntl);
+
+	if (result != BP_RESULT_OK) {
+		DC_LOG_ERROR("%s: Failed to execute VBIOS command table!\n",
+			__func__);
+		BREAK_TO_DEBUGGER();
+	}
+}
+/*
+ * @brief
+ * Disable transmitter and its encoder
+ */
+void dcn10_link_encoder_disable_output(
+	struct link_encoder *enc,
+	enum signal_type signal)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	struct bp_transmitter_control cntl = { 0 };
+	enum bp_result result;
+
+	if (!dcn10_is_dig_enabled(enc)) {
+		/* OF_SKIP_POWER_DOWN_INACTIVE_ENCODER */
+		return;
+	}
+	/* Power-down RX and disable GPU PHY should be paired.
+	 * Disabling PHY without powering down RX may cause
+	 * symbol lock loss, on which we will get DP Sink interrupt.
+	 */
+
+	/* There is a case for the DP active dongles
+	 * where we want to disable the PHY but keep RX powered,
+	 * for those we need to ignore DP Sink interrupt
+	 * by checking lane count that has been set
+	 * on the last do_enable_output().
+	 */
+
+	/* disable transmitter */
+	cntl.action = TRANSMITTER_CONTROL_DISABLE;
+	cntl.transmitter = enc10->base.transmitter;
+	cntl.hpd_sel = enc10->base.hpd_source;
+	cntl.signal = signal;
+	cntl.connector_obj_id = enc10->base.connector;
+
+	result = link_transmitter_control(enc10, &cntl);
+
+	if (result != BP_RESULT_OK) {
+		DC_LOG_ERROR("%s: Failed to execute VBIOS command table!\n",
+			__func__);
+		BREAK_TO_DEBUGGER();
+		return;
+	}
+
+	/* disable encoder */
+	if (dc_is_dp_signal(signal))
+		link_encoder_disable(enc10);
+}
+
+void dcn10_link_encoder_dp_set_lane_settings(
+	struct link_encoder *enc,
+	const struct link_training_settings *link_settings)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	union dpcd_training_lane_set training_lane_set = { { 0 } };
+	int32_t lane = 0;
+	struct bp_transmitter_control cntl = { 0 };
+
+	if (!link_settings) {
+		BREAK_TO_DEBUGGER();
+		return;
+	}
+
+	cntl.action = TRANSMITTER_CONTROL_SET_VOLTAGE_AND_PREEMPASIS;
+	cntl.transmitter = enc10->base.transmitter;
+	cntl.connector_obj_id = enc10->base.connector;
+	cntl.lanes_number = link_settings->link_settings.lane_count;
+	cntl.hpd_sel = enc10->base.hpd_source;
+	cntl.pixel_clock = link_settings->link_settings.link_rate *
+						LINK_RATE_REF_FREQ_IN_KHZ;
+
+	for (lane = 0; lane < link_settings->link_settings.lane_count; lane++) {
+		/* translate lane settings */
+
+		training_lane_set.bits.VOLTAGE_SWING_SET =
+			link_settings->lane_settings[lane].VOLTAGE_SWING;
+		training_lane_set.bits.PRE_EMPHASIS_SET =
+			link_settings->lane_settings[lane].PRE_EMPHASIS;
+
+		/* post cursor 2 setting only applies to HBR2 link rate */
+		if (link_settings->link_settings.link_rate == LINK_RATE_HIGH2) {
+			/* this is passed to VBIOS
+			 * to program post cursor 2 level
+			 */
+			training_lane_set.bits.POST_CURSOR2_SET =
+				link_settings->lane_settings[lane].POST_CURSOR2;
+		}
+
+		cntl.lane_select = lane;
+		cntl.lane_settings = training_lane_set.raw;
+
+		/* call VBIOS table to set voltage swing and pre-emphasis */
+		link_transmitter_control(enc10, &cntl);
+	}
+}
+
+/* set DP PHY test and training patterns */
+void dcn10_link_encoder_dp_set_phy_pattern(
+	struct link_encoder *enc,
+	const struct encoder_set_dp_phy_pattern_param *param)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+
+	switch (param->dp_phy_pattern) {
+	case DP_TEST_PATTERN_TRAINING_PATTERN1:
+		dcn10_link_encoder_set_dp_phy_pattern_training_pattern(enc, 0);
+		break;
+	case DP_TEST_PATTERN_TRAINING_PATTERN2:
+		dcn10_link_encoder_set_dp_phy_pattern_training_pattern(enc, 1);
+		break;
+	case DP_TEST_PATTERN_TRAINING_PATTERN3:
+		dcn10_link_encoder_set_dp_phy_pattern_training_pattern(enc, 2);
+		break;
+	case DP_TEST_PATTERN_TRAINING_PATTERN4:
+		dcn10_link_encoder_set_dp_phy_pattern_training_pattern(enc, 3);
+		break;
+	case DP_TEST_PATTERN_D102:
+		set_dp_phy_pattern_d102(enc10);
+		break;
+	case DP_TEST_PATTERN_SYMBOL_ERROR:
+		set_dp_phy_pattern_symbol_error(enc10);
+		break;
+	case DP_TEST_PATTERN_PRBS7:
+		set_dp_phy_pattern_prbs7(enc10);
+		break;
+	case DP_TEST_PATTERN_80BIT_CUSTOM:
+		set_dp_phy_pattern_80bit_custom(
+			enc10, param->custom_pattern);
+		break;
+	case DP_TEST_PATTERN_CP2520_1:
+		set_dp_phy_pattern_hbr2_compliance_cp2520_2(enc10, 1);
+		break;
+	case DP_TEST_PATTERN_CP2520_2:
+		set_dp_phy_pattern_hbr2_compliance_cp2520_2(enc10, 2);
+		break;
+	case DP_TEST_PATTERN_CP2520_3:
+		set_dp_phy_pattern_hbr2_compliance_cp2520_2(enc10, 3);
+		break;
+	case DP_TEST_PATTERN_VIDEO_MODE: {
+		set_dp_phy_pattern_passthrough_mode(
+			enc10, param->dp_panel_mode);
+		break;
+	}
+
+	default:
+		/* invalid phy pattern */
+		ASSERT_CRITICAL(false);
+		break;
+	}
+}
+
+static void fill_stream_allocation_row_info(
+	const struct link_mst_stream_allocation *stream_allocation,
+	uint32_t *src,
+	uint32_t *slots)
+{
+	const struct stream_encoder *stream_enc = stream_allocation->stream_enc;
+
+	if (stream_enc) {
+		*src = stream_enc->id;
+		*slots = stream_allocation->slot_count;
+	} else {
+		*src = 0;
+		*slots = 0;
+	}
+}
+
+/* programs DP MST VC payload allocation */
+void dcn10_link_encoder_update_mst_stream_allocation_table(
+	struct link_encoder *enc,
+	const struct link_mst_stream_allocation_table *table)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	uint32_t value0 = 0;
+	uint32_t value1 = 0;
+	uint32_t value2 = 0;
+	uint32_t slots = 0;
+	uint32_t src = 0;
+	uint32_t retries = 0;
+
+	/* For CZ, there are only 3 pipes. So Virtual channel is up 3.*/
+
+	/* --- Set MSE Stream Attribute -
+	 * Setup VC Payload Table on Tx Side,
+	 * Issue allocation change trigger
+	 * to commit payload on both tx and rx side
+	 */
+
+	/* we should clean-up table each time */
+
+	if (table->stream_count >= 1) {
+		fill_stream_allocation_row_info(
+			&table->stream_allocations[0],
+			&src,
+			&slots);
+	} else {
+		src = 0;
+		slots = 0;
+	}
+
+	REG_UPDATE_2(DP_MSE_SAT0,
+			DP_MSE_SAT_SRC0, src,
+			DP_MSE_SAT_SLOT_COUNT0, slots);
+
+	if (table->stream_count >= 2) {
+		fill_stream_allocation_row_info(
+			&table->stream_allocations[1],
+			&src,
+			&slots);
+	} else {
+		src = 0;
+		slots = 0;
+	}
+
+	REG_UPDATE_2(DP_MSE_SAT0,
+			DP_MSE_SAT_SRC1, src,
+			DP_MSE_SAT_SLOT_COUNT1, slots);
+
+	if (table->stream_count >= 3) {
+		fill_stream_allocation_row_info(
+			&table->stream_allocations[2],
+			&src,
+			&slots);
+	} else {
+		src = 0;
+		slots = 0;
+	}
+
+	REG_UPDATE_2(DP_MSE_SAT1,
+			DP_MSE_SAT_SRC2, src,
+			DP_MSE_SAT_SLOT_COUNT2, slots);
+
+	if (table->stream_count >= 4) {
+		fill_stream_allocation_row_info(
+			&table->stream_allocations[3],
+			&src,
+			&slots);
+	} else {
+		src = 0;
+		slots = 0;
+	}
+
+	REG_UPDATE_2(DP_MSE_SAT1,
+			DP_MSE_SAT_SRC3, src,
+			DP_MSE_SAT_SLOT_COUNT3, slots);
+
+	/* --- wait for transaction finish */
+
+	/* send allocation change trigger (ACT) ?
+	 * this step first sends the ACT,
+	 * then double buffers the SAT into the hardware
+	 * making the new allocation active on the DP MST mode link
+	 */
+
+	/* DP_MSE_SAT_UPDATE:
+	 * 0 - No Action
+	 * 1 - Update SAT with trigger
+	 * 2 - Update SAT without trigger
+	 */
+	REG_UPDATE(DP_MSE_SAT_UPDATE,
+			DP_MSE_SAT_UPDATE, 1);
+
+	/* wait for update to complete
+	 * (i.e. DP_MSE_SAT_UPDATE field is reset to 0)
+	 * then wait for the transmission
+	 * of at least 16 MTP headers on immediate local link.
+	 * i.e. DP_MSE_16_MTP_KEEPOUT field (read only) is reset to 0
+	 * a value of 1 indicates that DP MST mode
+	 * is in the 16 MTP keepout region after a VC has been added.
+	 * MST stream bandwidth (VC rate) can be configured
+	 * after this bit is cleared
+	 */
+	do {
+		udelay(10);
+
+		value0 = REG_READ(DP_MSE_SAT_UPDATE);
+
+		REG_GET(DP_MSE_SAT_UPDATE,
+				DP_MSE_SAT_UPDATE, &value1);
+
+		REG_GET(DP_MSE_SAT_UPDATE,
+				DP_MSE_16_MTP_KEEPOUT, &value2);
+
+		/* bit field DP_MSE_SAT_UPDATE is set to 1 already */
+		if (!value1 && !value2)
+			break;
+		++retries;
+	} while (retries < DP_MST_UPDATE_MAX_RETRY);
+}
+
+void dcn10_link_encoder_connect_dig_be_to_fe(
+	struct link_encoder *enc,
+	enum engine_id engine,
+	bool connect)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	uint32_t field;
+
+	if (engine != ENGINE_ID_UNKNOWN) {
+
+		REG_GET(DIG_BE_CNTL, DIG_FE_SOURCE_SELECT, &field);
+
+		if (connect)
+			field |= get_frontend_source(engine);
+		else
+			field &= ~get_frontend_source(engine);
+
+		REG_UPDATE(DIG_BE_CNTL, DIG_FE_SOURCE_SELECT, field);
+	}
+}
+
+
+#define HPD_REG(reg)\
+	(enc10->hpd_regs->reg)
+
+#define HPD_REG_READ(reg_name) \
+		dm_read_reg(CTX, HPD_REG(reg_name))
+
+#define HPD_REG_UPDATE_N(reg_name, n, ...)	\
+		generic_reg_update_ex(CTX, \
+				HPD_REG(reg_name), \
+				HPD_REG_READ(reg_name), \
+				n, __VA_ARGS__)
+
+#define HPD_REG_UPDATE(reg_name, field, val)	\
+		HPD_REG_UPDATE_N(reg_name, 1, \
+				FN(reg_name, field), val)
+
+void dcn10_link_encoder_enable_hpd(struct link_encoder *enc)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+
+	HPD_REG_UPDATE(DC_HPD_CONTROL,
+			DC_HPD_EN, 1);
+}
+
+void dcn10_link_encoder_disable_hpd(struct link_encoder *enc)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+
+	HPD_REG_UPDATE(DC_HPD_CONTROL,
+			DC_HPD_EN, 0);
+}
+
+
+#define AUX_REG(reg)\
+	(enc10->aux_regs->reg)
+
+#define AUX_REG_READ(reg_name) \
+		dm_read_reg(CTX, AUX_REG(reg_name))
+
+#define AUX_REG_UPDATE_N(reg_name, n, ...)	\
+		generic_reg_update_ex(CTX, \
+				AUX_REG(reg_name), \
+				AUX_REG_READ(reg_name), \
+				n, __VA_ARGS__)
+
+#define AUX_REG_UPDATE(reg_name, field, val)	\
+		AUX_REG_UPDATE_N(reg_name, 1, \
+				FN(reg_name, field), val)
+
+#define AUX_REG_UPDATE_2(reg, f1, v1, f2, v2)	\
+		AUX_REG_UPDATE_N(reg, 2,\
+				FN(reg, f1), v1,\
+				FN(reg, f2), v2)
+
+static void aux_initialize(
+	struct dcn10_link_encoder *enc10)
+{
+	enum hpd_source_id hpd_source = enc10->base.hpd_source;
+
+	AUX_REG_UPDATE_2(AUX_CONTROL,
+			AUX_HPD_SEL, hpd_source,
+			AUX_LS_READ_EN, 0);
+
+	/* 1/4 window (the maximum allowed) */
+	AUX_REG_UPDATE(AUX_DPHY_RX_CONTROL0,
+			AUX_RX_RECEIVE_WINDOW, 1);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
new file mode 100644
index 000000000000..2a97cdb2cfbb
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
@@ -0,0 +1,330 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_LINK_ENCODER__DCN10_H__
+#define __DC_LINK_ENCODER__DCN10_H__
+
+#include "link_encoder.h"
+
+#define TO_DCN10_LINK_ENC(link_encoder)\
+	container_of(link_encoder, struct dcn10_link_encoder, base)
+
+
+#define AUX_REG_LIST(id)\
+	SRI(AUX_CONTROL, DP_AUX, id), \
+	SRI(AUX_DPHY_RX_CONTROL0, DP_AUX, id)
+
+#define HPD_REG_LIST(id)\
+	SRI(DC_HPD_CONTROL, HPD, id)
+
+#define LE_DCN_COMMON_REG_LIST(id) \
+	SRI(DIG_BE_CNTL, DIG, id), \
+	SRI(DIG_BE_EN_CNTL, DIG, id), \
+	SRI(DP_CONFIG, DP, id), \
+	SRI(DP_DPHY_CNTL, DP, id), \
+	SRI(DP_DPHY_PRBS_CNTL, DP, id), \
+	SRI(DP_DPHY_SCRAM_CNTL, DP, id),\
+	SRI(DP_DPHY_SYM0, DP, id), \
+	SRI(DP_DPHY_SYM1, DP, id), \
+	SRI(DP_DPHY_SYM2, DP, id), \
+	SRI(DP_DPHY_TRAINING_PATTERN_SEL, DP, id), \
+	SRI(DP_LINK_CNTL, DP, id), \
+	SRI(DP_LINK_FRAMING_CNTL, DP, id), \
+	SRI(DP_MSE_SAT0, DP, id), \
+	SRI(DP_MSE_SAT1, DP, id), \
+	SRI(DP_MSE_SAT2, DP, id), \
+	SRI(DP_MSE_SAT_UPDATE, DP, id), \
+	SRI(DP_SEC_CNTL, DP, id), \
+	SRI(DP_VID_STREAM_CNTL, DP, id), \
+	SRI(DP_DPHY_FAST_TRAINING, DP, id), \
+	SRI(DP_SEC_CNTL1, DP, id), \
+	SRI(DP_DPHY_BS_SR_SWAP_CNTL, DP, id), \
+	SRI(DP_DPHY_INTERNAL_CTRL, DP, id), \
+	SRI(DP_DPHY_HBR2_PATTERN_CONTROL, DP, id)
+
+#define LE_DCN10_REG_LIST(id)\
+	LE_DCN_COMMON_REG_LIST(id)
+
+struct dcn10_link_enc_aux_registers {
+	uint32_t AUX_CONTROL;
+	uint32_t AUX_DPHY_RX_CONTROL0;
+};
+
+struct dcn10_link_enc_hpd_registers {
+	uint32_t DC_HPD_CONTROL;
+};
+
+struct dcn10_link_enc_registers {
+	uint32_t DIG_BE_CNTL;
+	uint32_t DIG_BE_EN_CNTL;
+	uint32_t DP_CONFIG;
+	uint32_t DP_DPHY_CNTL;
+	uint32_t DP_DPHY_INTERNAL_CTRL;
+	uint32_t DP_DPHY_PRBS_CNTL;
+	uint32_t DP_DPHY_SCRAM_CNTL;
+	uint32_t DP_DPHY_SYM0;
+	uint32_t DP_DPHY_SYM1;
+	uint32_t DP_DPHY_SYM2;
+	uint32_t DP_DPHY_TRAINING_PATTERN_SEL;
+	uint32_t DP_LINK_CNTL;
+	uint32_t DP_LINK_FRAMING_CNTL;
+	uint32_t DP_MSE_SAT0;
+	uint32_t DP_MSE_SAT1;
+	uint32_t DP_MSE_SAT2;
+	uint32_t DP_MSE_SAT_UPDATE;
+	uint32_t DP_SEC_CNTL;
+	uint32_t DP_VID_STREAM_CNTL;
+	uint32_t DP_DPHY_FAST_TRAINING;
+	uint32_t DP_DPHY_BS_SR_SWAP_CNTL;
+	uint32_t DP_DPHY_HBR2_PATTERN_CONTROL;
+	uint32_t DP_SEC_CNTL1;
+};
+
+#define LE_SF(reg_name, field_name, post_fix)\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define LINK_ENCODER_MASK_SH_LIST_DCN10(mask_sh)\
+	LE_SF(DIG0_DIG_BE_EN_CNTL, DIG_ENABLE, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CNTL, DIG_HPD_SELECT, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CNTL, DIG_MODE, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CNTL, DIG_FE_SOURCE_SELECT, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_BYPASS, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE0, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE1, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE2, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE3, mask_sh),\
+	LE_SF(DP0_DP_DPHY_PRBS_CNTL, DPHY_PRBS_EN, mask_sh),\
+	LE_SF(DP0_DP_DPHY_PRBS_CNTL, DPHY_PRBS_SEL, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM0, DPHY_SYM1, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM0, DPHY_SYM2, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM0, DPHY_SYM3, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM1, DPHY_SYM4, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM1, DPHY_SYM5, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM1, DPHY_SYM6, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM2, DPHY_SYM7, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM2, DPHY_SYM8, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SCRAM_CNTL, DPHY_SCRAMBLER_BS_COUNT, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SCRAM_CNTL, DPHY_SCRAMBLER_ADVANCE, mask_sh),\
+	LE_SF(DP0_DP_DPHY_FAST_TRAINING, DPHY_RX_FAST_TRAINING_CAPABLE, mask_sh),\
+	LE_SF(DP0_DP_DPHY_BS_SR_SWAP_CNTL, DPHY_LOAD_BS_COUNT, mask_sh),\
+	LE_SF(DP0_DP_DPHY_TRAINING_PATTERN_SEL, DPHY_TRAINING_PATTERN_SEL, mask_sh),\
+	LE_SF(DP0_DP_DPHY_HBR2_PATTERN_CONTROL, DP_DPHY_HBR2_PATTERN_CONTROL, mask_sh),\
+	LE_SF(DP0_DP_LINK_CNTL, DP_LINK_TRAINING_COMPLETE, mask_sh),\
+	LE_SF(DP0_DP_LINK_FRAMING_CNTL, DP_IDLE_BS_INTERVAL, mask_sh),\
+	LE_SF(DP0_DP_LINK_FRAMING_CNTL, DP_VBID_DISABLE, mask_sh),\
+	LE_SF(DP0_DP_LINK_FRAMING_CNTL, DP_VID_ENHANCED_FRAME_MODE, mask_sh),\
+	LE_SF(DP0_DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, mask_sh),\
+	LE_SF(DP0_DP_CONFIG, DP_UDI_LANES, mask_sh),\
+	LE_SF(DP0_DP_SEC_CNTL1, DP_SEC_GSP0_LINE_NUM, mask_sh),\
+	LE_SF(DP0_DP_SEC_CNTL1, DP_SEC_GSP0_PRIORITY, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT0, DP_MSE_SAT_SRC0, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT0, DP_MSE_SAT_SRC1, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT0, DP_MSE_SAT_SLOT_COUNT0, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT0, DP_MSE_SAT_SLOT_COUNT1, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT1, DP_MSE_SAT_SRC2, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT1, DP_MSE_SAT_SRC3, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT1, DP_MSE_SAT_SLOT_COUNT2, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT1, DP_MSE_SAT_SLOT_COUNT3, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT_UPDATE, DP_MSE_SAT_UPDATE, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT_UPDATE, DP_MSE_16_MTP_KEEPOUT, mask_sh),\
+	LE_SF(DP_AUX0_AUX_CONTROL, AUX_HPD_SEL, mask_sh),\
+	LE_SF(DP_AUX0_AUX_CONTROL, AUX_LS_READ_EN, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_RECEIVE_WINDOW, mask_sh),\
+	LE_SF(HPD0_DC_HPD_CONTROL, DC_HPD_EN, mask_sh)
+
+#define DCN_LINK_ENCODER_REG_FIELD_LIST(type) \
+	type DIG_ENABLE;\
+	type DIG_HPD_SELECT;\
+	type DIG_MODE;\
+	type DIG_FE_SOURCE_SELECT;\
+	type DPHY_BYPASS;\
+	type DPHY_ATEST_SEL_LANE0;\
+	type DPHY_ATEST_SEL_LANE1;\
+	type DPHY_ATEST_SEL_LANE2;\
+	type DPHY_ATEST_SEL_LANE3;\
+	type DPHY_PRBS_EN;\
+	type DPHY_PRBS_SEL;\
+	type DPHY_SYM1;\
+	type DPHY_SYM2;\
+	type DPHY_SYM3;\
+	type DPHY_SYM4;\
+	type DPHY_SYM5;\
+	type DPHY_SYM6;\
+	type DPHY_SYM7;\
+	type DPHY_SYM8;\
+	type DPHY_SCRAMBLER_BS_COUNT;\
+	type DPHY_SCRAMBLER_ADVANCE;\
+	type DPHY_RX_FAST_TRAINING_CAPABLE;\
+	type DPHY_LOAD_BS_COUNT;\
+	type DPHY_TRAINING_PATTERN_SEL;\
+	type DP_DPHY_HBR2_PATTERN_CONTROL;\
+	type DP_LINK_TRAINING_COMPLETE;\
+	type DP_IDLE_BS_INTERVAL;\
+	type DP_VBID_DISABLE;\
+	type DP_VID_ENHANCED_FRAME_MODE;\
+	type DP_VID_STREAM_ENABLE;\
+	type DP_UDI_LANES;\
+	type DP_SEC_GSP0_LINE_NUM;\
+	type DP_SEC_GSP0_PRIORITY;\
+	type DP_MSE_SAT_SRC0;\
+	type DP_MSE_SAT_SRC1;\
+	type DP_MSE_SAT_SRC2;\
+	type DP_MSE_SAT_SRC3;\
+	type DP_MSE_SAT_SLOT_COUNT0;\
+	type DP_MSE_SAT_SLOT_COUNT1;\
+	type DP_MSE_SAT_SLOT_COUNT2;\
+	type DP_MSE_SAT_SLOT_COUNT3;\
+	type DP_MSE_SAT_UPDATE;\
+	type DP_MSE_16_MTP_KEEPOUT;\
+	type AUX_HPD_SEL;\
+	type AUX_LS_READ_EN;\
+	type AUX_RX_RECEIVE_WINDOW;\
+	type DC_HPD_EN
+
+struct dcn10_link_enc_shift {
+	DCN_LINK_ENCODER_REG_FIELD_LIST(uint8_t);
+};
+
+struct dcn10_link_enc_mask {
+	DCN_LINK_ENCODER_REG_FIELD_LIST(uint32_t);
+};
+
+struct dcn10_link_encoder {
+	struct link_encoder base;
+	const struct dcn10_link_enc_registers *link_regs;
+	const struct dcn10_link_enc_aux_registers *aux_regs;
+	const struct dcn10_link_enc_hpd_registers *hpd_regs;
+	const struct dcn10_link_enc_shift *link_shift;
+	const struct dcn10_link_enc_mask *link_mask;
+};
+
+
+void dcn10_link_encoder_construct(
+	struct dcn10_link_encoder *enc10,
+	const struct encoder_init_data *init_data,
+	const struct encoder_feature_support *enc_features,
+	const struct dcn10_link_enc_registers *link_regs,
+	const struct dcn10_link_enc_aux_registers *aux_regs,
+	const struct dcn10_link_enc_hpd_registers *hpd_regs,
+	const struct dcn10_link_enc_shift *link_shift,
+	const struct dcn10_link_enc_mask *link_mask);
+
+bool dcn10_link_encoder_validate_dvi_output(
+	const struct dcn10_link_encoder *enc10,
+	enum signal_type connector_signal,
+	enum signal_type signal,
+	const struct dc_crtc_timing *crtc_timing);
+
+bool dcn10_link_encoder_validate_rgb_output(
+	const struct dcn10_link_encoder *enc10,
+	const struct dc_crtc_timing *crtc_timing);
+
+bool dcn10_link_encoder_validate_dp_output(
+	const struct dcn10_link_encoder *enc10,
+	const struct dc_crtc_timing *crtc_timing);
+
+bool dcn10_link_encoder_validate_wireless_output(
+	const struct dcn10_link_encoder *enc10,
+	const struct dc_crtc_timing *crtc_timing);
+
+bool dcn10_link_encoder_validate_output_with_stream(
+	struct link_encoder *enc,
+	const struct dc_stream_state *stream);
+
+/****************** HW programming ************************/
+
+/* initialize HW */  /* why do we initialze aux in here? */
+void dcn10_link_encoder_hw_init(struct link_encoder *enc);
+
+void dcn10_link_encoder_destroy(struct link_encoder **enc);
+
+/* program DIG_MODE in DIG_BE */
+/* TODO can this be combined with enable_output? */
+void dcn10_link_encoder_setup(
+	struct link_encoder *enc,
+	enum signal_type signal);
+
+/* enables TMDS PHY output */
+/* TODO: still need depth or just pass in adjusted pixel clock? */
+void dcn10_link_encoder_enable_tmds_output(
+	struct link_encoder *enc,
+	enum clock_source_id clock_source,
+	enum dc_color_depth color_depth,
+	enum signal_type signal,
+	uint32_t pixel_clock);
+
+/* enables DP PHY output */
+void dcn10_link_encoder_enable_dp_output(
+	struct link_encoder *enc,
+	const struct dc_link_settings *link_settings,
+	enum clock_source_id clock_source);
+
+/* enables DP PHY output in MST mode */
+void dcn10_link_encoder_enable_dp_mst_output(
+	struct link_encoder *enc,
+	const struct dc_link_settings *link_settings,
+	enum clock_source_id clock_source);
+
+/* disable PHY output */
+void dcn10_link_encoder_disable_output(
+	struct link_encoder *enc,
+	enum signal_type signal);
+
+/* set DP lane settings */
+void dcn10_link_encoder_dp_set_lane_settings(
+	struct link_encoder *enc,
+	const struct link_training_settings *link_settings);
+
+void dcn10_link_encoder_dp_set_phy_pattern(
+	struct link_encoder *enc,
+	const struct encoder_set_dp_phy_pattern_param *param);
+
+/* programs DP MST VC payload allocation */
+void dcn10_link_encoder_update_mst_stream_allocation_table(
+	struct link_encoder *enc,
+	const struct link_mst_stream_allocation_table *table);
+
+void dcn10_link_encoder_connect_dig_be_to_fe(
+	struct link_encoder *enc,
+	enum engine_id engine,
+	bool connect);
+
+void dcn10_link_encoder_set_dp_phy_pattern_training_pattern(
+	struct link_encoder *enc,
+	uint32_t index);
+
+void dcn10_link_encoder_enable_hpd(struct link_encoder *enc);
+
+void dcn10_link_encoder_disable_hpd(struct link_encoder *enc);
+
+void dcn10_psr_program_dp_dphy_fast_training(struct link_encoder *enc,
+			bool exit_link_training_required);
+
+void dcn10_psr_program_secondary_packet(struct link_encoder *enc,
+			unsigned int sdp_transmit_line_num_deadline);
+
+bool dcn10_is_dig_enabled(struct link_encoder *enc);
+
+#endif /* __DC_LINK_ENCODER__DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
index 179890b1a8c4..9ca51ae46de7 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
@@ -65,6 +65,7 @@ static void mpc1_update_blending(
 	int mpcc_id)
 {
 	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
+	struct mpcc *mpcc = mpc1_get_mpcc(mpc, mpcc_id);
 
 	REG_UPDATE_5(MPCC_CONTROL[mpcc_id],
 			MPCC_ALPHA_BLND_MODE,		blnd_cfg->alpha_mode,
@@ -74,6 +75,7 @@ static void mpc1_update_blending(
 			MPCC_GLOBAL_GAIN,		blnd_cfg->global_gain);
 
 	mpc1_set_bg_color(mpc, &blnd_cfg->black_color, mpcc_id);
+	mpcc->blnd_cfg = *blnd_cfg;
 }
 
 void mpc1_update_stereo_mix(
@@ -235,8 +237,7 @@ struct mpcc *mpc1_insert_plane(
 	}
 
 	/* update the blending configuration */
-	new_mpcc->blnd_cfg = *blnd_cfg;
-	mpc->funcs->update_blending(mpc, &new_mpcc->blnd_cfg, mpcc_id);
+	mpc->funcs->update_blending(mpc, blnd_cfg, mpcc_id);
 
 	/* update the stereo mix settings, if provided */
 	if (sm_cfg != NULL) {
@@ -409,7 +410,26 @@ void mpc1_init_mpcc_list_from_hw(
 	}
 }
 
+void mpc1_read_mpcc_state(
+		struct mpc *mpc,
+		int mpcc_inst,
+		struct mpcc_state *s)
+{
+	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
+
+	REG_GET(MPCC_OPP_ID[mpcc_inst], MPCC_OPP_ID, &s->opp_id);
+	REG_GET(MPCC_TOP_SEL[mpcc_inst], MPCC_TOP_SEL, &s->dpp_id);
+	REG_GET(MPCC_BOT_SEL[mpcc_inst], MPCC_BOT_SEL, &s->bot_mpcc_id);
+	REG_GET_4(MPCC_CONTROL[mpcc_inst], MPCC_MODE, &s->mode,
+			MPCC_ALPHA_BLND_MODE, &s->alpha_mode,
+			MPCC_ALPHA_MULTIPLIED_MODE, &s->pre_multiplied_alpha,
+			MPCC_BLND_ACTIVE_OVERLAP_ONLY, &s->overlap_only);
+	REG_GET_2(MPCC_STATUS[mpcc_inst], MPCC_IDLE, &s->idle,
+			MPCC_BUSY, &s->busy);
+}
+
 const struct mpc_funcs dcn10_mpc_funcs = {
+	.read_mpcc_state = mpc1_read_mpcc_state,
 	.insert_plane = mpc1_insert_plane,
 	.remove_mpcc = mpc1_remove_mpcc,
 	.mpc_init = mpc1_mpc_init,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.h
index 267a2995ef6e..d3d16c4cbea3 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.h
@@ -183,4 +183,9 @@ struct mpcc *mpc1_get_mpcc_for_dpp(
 	struct mpc_tree *tree,
 	int dpp_id);
 
+void mpc1_read_mpcc_state(
+		struct mpc *mpc,
+		int mpcc_inst,
+		struct mpcc_state *s);
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
index 4bf64d1b2c60..f2fbce0e3fc5 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
@@ -93,6 +93,81 @@ static void optc1_disable_stereo(struct timing_generator *optc)
 		OTG_3D_STRUCTURE_STEREO_SEL_OVR, 0);
 }
 
+static uint32_t get_start_vline(struct timing_generator *optc, const struct dc_crtc_timing *dc_crtc_timing)
+{
+	struct dc_crtc_timing patched_crtc_timing;
+	int vesa_sync_start;
+	int asic_blank_end;
+	int interlace_factor;
+	int vertical_line_start;
+
+	patched_crtc_timing = *dc_crtc_timing;
+	optc1_apply_front_porch_workaround(optc, &patched_crtc_timing);
+
+	vesa_sync_start = patched_crtc_timing.h_addressable +
+			patched_crtc_timing.h_border_right +
+			patched_crtc_timing.h_front_porch;
+
+	asic_blank_end = patched_crtc_timing.h_total -
+			vesa_sync_start -
+			patched_crtc_timing.h_border_left;
+
+	interlace_factor = patched_crtc_timing.flags.INTERLACE ? 2 : 1;
+
+	vesa_sync_start = patched_crtc_timing.v_addressable +
+			patched_crtc_timing.v_border_bottom +
+			patched_crtc_timing.v_front_porch;
+
+	asic_blank_end = (patched_crtc_timing.v_total -
+			vesa_sync_start -
+			patched_crtc_timing.v_border_top)
+			* interlace_factor;
+
+	vertical_line_start = asic_blank_end - optc->dlg_otg_param.vstartup_start + 1;
+	if (vertical_line_start < 0) {
+		ASSERT(0);
+		vertical_line_start = 0;
+	}
+
+	return vertical_line_start;
+}
+
+void optc1_program_vline_interrupt(
+		struct timing_generator *optc,
+		const struct dc_crtc_timing *dc_crtc_timing,
+		unsigned long long vsync_delta)
+{
+
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+	unsigned long long req_delta_tens_of_usec = div64_u64((vsync_delta + 9999), 10000);
+	unsigned long long pix_clk_hundreds_khz = div64_u64((dc_crtc_timing->pix_clk_khz + 99), 100);
+	uint32_t req_delta_lines = (uint32_t) div64_u64(
+			(req_delta_tens_of_usec * pix_clk_hundreds_khz + dc_crtc_timing->h_total - 1),
+								dc_crtc_timing->h_total);
+
+	uint32_t vsync_line = get_start_vline(optc, dc_crtc_timing);
+	uint32_t start_line = 0;
+	uint32_t endLine = 0;
+
+	if (req_delta_lines != 0)
+		req_delta_lines--;
+
+	if (req_delta_lines > vsync_line)
+		start_line = dc_crtc_timing->v_total - (req_delta_lines - vsync_line) - 1;
+	else
+		start_line = vsync_line - req_delta_lines;
+
+	endLine = start_line + 2;
+
+	if (endLine >= dc_crtc_timing->v_total)
+		endLine = 2;
+
+	REG_SET_2(OTG_VERTICAL_INTERRUPT0_POSITION, 0,
+			OTG_VERTICAL_INTERRUPT0_LINE_START, start_line,
+			OTG_VERTICAL_INTERRUPT0_LINE_END, endLine);
+}
+
 /**
  * program_timing_generator   used by mode timing set
  * Program CRTC Timing Registers - OTG_H_*, OTG_V_*, Pixel repetition.
@@ -285,7 +360,7 @@ void optc1_program_timing(
 
 }
 
-static void optc1_set_blank_data_double_buffer(struct timing_generator *optc, bool enable)
+void optc1_set_blank_data_double_buffer(struct timing_generator *optc, bool enable)
 {
 	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
@@ -780,17 +855,17 @@ void optc1_set_drr(
 				OTG_SET_V_TOTAL_MIN_MASK_EN, 0,
 				OTG_SET_V_TOTAL_MIN_MASK, 0);
 	} else {
-		REG_SET(OTG_V_TOTAL_MIN, 0,
-			OTG_V_TOTAL_MIN, 0);
-
-		REG_SET(OTG_V_TOTAL_MAX, 0,
-			OTG_V_TOTAL_MAX, 0);
-
 		REG_UPDATE_4(OTG_V_TOTAL_CONTROL,
 				OTG_SET_V_TOTAL_MIN_MASK, 0,
 				OTG_V_TOTAL_MIN_SEL, 0,
 				OTG_V_TOTAL_MAX_SEL, 0,
 				OTG_FORCE_LOCK_ON_EVENT, 0);
+
+		REG_SET(OTG_V_TOTAL_MIN, 0,
+			OTG_V_TOTAL_MIN, 0);
+
+		REG_SET(OTG_V_TOTAL_MAX, 0,
+			OTG_V_TOTAL_MAX, 0);
 	}
 }
 
@@ -1154,6 +1229,12 @@ void optc1_read_otg_state(struct optc *optc1,
 	REG_GET(OTG_V_TOTAL_MIN,
 			OTG_V_TOTAL_MIN, &s->v_total_min);
 
+	REG_GET(OTG_V_TOTAL_CONTROL,
+			OTG_V_TOTAL_MAX_SEL, &s->v_total_max_sel);
+
+	REG_GET(OTG_V_TOTAL_CONTROL,
+			OTG_V_TOTAL_MIN_SEL, &s->v_total_min_sel);
+
 	REG_GET_2(OTG_V_SYNC_A,
 			OTG_V_SYNC_A_START, &s->v_sync_a_start,
 			OTG_V_SYNC_A_END, &s->v_sync_a_end);
@@ -1176,20 +1257,20 @@ void optc1_read_otg_state(struct optc *optc1,
 			OPTC_UNDERFLOW_OCCURRED_STATUS, &s->underflow_occurred_status);
 }
 
-static void optc1_clear_optc_underflow(struct timing_generator *optc)
+void optc1_clear_optc_underflow(struct timing_generator *optc)
 {
 	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	REG_UPDATE(OPTC_INPUT_GLOBAL_CONTROL, OPTC_UNDERFLOW_CLEAR, 1);
 }
 
-static void optc1_tg_init(struct timing_generator *optc)
+void optc1_tg_init(struct timing_generator *optc)
 {
 	optc1_set_blank_data_double_buffer(optc, true);
 	optc1_clear_optc_underflow(optc);
 }
 
-static bool optc1_is_tg_enabled(struct timing_generator *optc)
+bool optc1_is_tg_enabled(struct timing_generator *optc)
 {
 	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 	uint32_t otg_enabled = 0;
@@ -1200,7 +1281,7 @@ static bool optc1_is_tg_enabled(struct timing_generator *optc)
 
 }
 
-static bool optc1_is_optc_underflow_occurred(struct timing_generator *optc)
+bool optc1_is_optc_underflow_occurred(struct timing_generator *optc)
 {
 	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 	uint32_t underflow_occurred = 0;
@@ -1215,6 +1296,7 @@ static bool optc1_is_optc_underflow_occurred(struct timing_generator *optc)
 static const struct timing_generator_funcs dcn10_tg_funcs = {
 		.validate_timing = optc1_validate_timing,
 		.program_timing = optc1_program_timing,
+		.program_vline_interrupt = optc1_program_vline_interrupt,
 		.program_global_sync = optc1_program_global_sync,
 		.enable_crtc = optc1_enable_crtc,
 		.disable_crtc = optc1_disable_crtc,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
index d25e7bf0d0d7..c62052f46460 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
@@ -65,6 +65,8 @@
 	SRI(OTG_NOM_VERT_POSITION, OTG, inst),\
 	SRI(OTG_BLACK_COLOR, OTG, inst),\
 	SRI(OTG_CLOCK_CONTROL, OTG, inst),\
+	SRI(OTG_VERTICAL_INTERRUPT0_CONTROL, OTG, inst),\
+	SRI(OTG_VERTICAL_INTERRUPT0_POSITION, OTG, inst),\
 	SRI(OTG_VERTICAL_INTERRUPT2_CONTROL, OTG, inst),\
 	SRI(OTG_VERTICAL_INTERRUPT2_POSITION, OTG, inst),\
 	SRI(OPTC_INPUT_CLOCK_CONTROL, ODM, inst),\
@@ -124,6 +126,8 @@ struct dcn_optc_registers {
 	uint32_t OTG_TEST_PATTERN_CONTROL;
 	uint32_t OTG_TEST_PATTERN_COLOR;
 	uint32_t OTG_CLOCK_CONTROL;
+	uint32_t OTG_VERTICAL_INTERRUPT0_CONTROL;
+	uint32_t OTG_VERTICAL_INTERRUPT0_POSITION;
 	uint32_t OTG_VERTICAL_INTERRUPT2_CONTROL;
 	uint32_t OTG_VERTICAL_INTERRUPT2_POSITION;
 	uint32_t OPTC_INPUT_CLOCK_CONTROL;
@@ -206,6 +210,9 @@ struct dcn_optc_registers {
 	SF(OTG0_OTG_CLOCK_CONTROL, OTG_CLOCK_EN, mask_sh),\
 	SF(OTG0_OTG_CLOCK_CONTROL, OTG_CLOCK_ON, mask_sh),\
 	SF(OTG0_OTG_CLOCK_CONTROL, OTG_CLOCK_GATE_DIS, mask_sh),\
+	SF(OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL, OTG_VERTICAL_INTERRUPT0_INT_ENABLE, mask_sh),\
+	SF(OTG0_OTG_VERTICAL_INTERRUPT0_POSITION, OTG_VERTICAL_INTERRUPT0_LINE_START, mask_sh),\
+	SF(OTG0_OTG_VERTICAL_INTERRUPT0_POSITION, OTG_VERTICAL_INTERRUPT0_LINE_END, mask_sh),\
 	SF(OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL, OTG_VERTICAL_INTERRUPT2_INT_ENABLE, mask_sh),\
 	SF(OTG0_OTG_VERTICAL_INTERRUPT2_POSITION, OTG_VERTICAL_INTERRUPT2_LINE_START, mask_sh),\
 	SF(ODM0_OPTC_INPUT_CLOCK_CONTROL, OPTC_INPUT_CLK_EN, mask_sh),\
@@ -323,6 +330,9 @@ struct dcn_optc_registers {
 	type OTG_CLOCK_EN;\
 	type OTG_CLOCK_ON;\
 	type OTG_CLOCK_GATE_DIS;\
+	type OTG_VERTICAL_INTERRUPT0_INT_ENABLE;\
+	type OTG_VERTICAL_INTERRUPT0_LINE_START;\
+	type OTG_VERTICAL_INTERRUPT0_LINE_END;\
 	type OTG_VERTICAL_INTERRUPT2_INT_ENABLE;\
 	type OTG_VERTICAL_INTERRUPT2_LINE_START;\
 	type OPTC_INPUT_CLK_EN;\
@@ -396,6 +406,8 @@ struct dcn_otg_state {
 	uint32_t v_total;
 	uint32_t v_total_max;
 	uint32_t v_total_min;
+	uint32_t v_total_min_sel;
+	uint32_t v_total_max_sel;
 	uint32_t v_sync_a_start;
 	uint32_t v_sync_a_end;
 	uint32_t h_blank_start;
@@ -420,6 +432,10 @@ void optc1_program_timing(
 	const struct dc_crtc_timing *dc_crtc_timing,
 	bool use_vbios);
 
+void optc1_program_vline_interrupt(struct timing_generator *optc,
+		const struct dc_crtc_timing *dc_crtc_timing,
+		unsigned long long vsync_delta);
+
 void optc1_program_global_sync(
 		struct timing_generator *optc);
 
@@ -481,4 +497,14 @@ void optc1_program_stereo(struct timing_generator *optc,
 
 bool optc1_is_stereo_left_eye(struct timing_generator *optc);
 
+void optc1_clear_optc_underflow(struct timing_generator *optc);
+
+void optc1_tg_init(struct timing_generator *optc);
+
+bool optc1_is_tg_enabled(struct timing_generator *optc);
+
+bool optc1_is_optc_underflow_occurred(struct timing_generator *optc);
+
+void optc1_set_blank_data_double_buffer(struct timing_generator *optc, bool enable);
+
 #endif /* __DC_TIMING_GENERATOR_DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
index 02bd664aed3e..df5cb2d1d164 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
@@ -38,8 +38,8 @@
 #include "dcn10/dcn10_hw_sequencer.h"
 #include "dce110/dce110_hw_sequencer.h"
 #include "dcn10/dcn10_opp.h"
-#include "dce/dce_link_encoder.h"
-#include "dce/dce_stream_encoder.h"
+#include "dcn10/dcn10_link_encoder.h"
+#include "dcn10/dcn10_stream_encoder.h"
 #include "dce/dce_clocks.h"
 #include "dce/dce_clock_source.h"
 #include "dce/dce_audio.h"
@@ -166,36 +166,22 @@ static const struct dce_abm_mask abm_mask = {
 
 #define stream_enc_regs(id)\
 [id] = {\
-	SE_DCN_REG_LIST(id),\
-	.TMDS_CNTL = 0,\
-	.AFMT_AVI_INFO0 = 0,\
-	.AFMT_AVI_INFO1 = 0,\
-	.AFMT_AVI_INFO2 = 0,\
-	.AFMT_AVI_INFO3 = 0,\
+	SE_DCN_REG_LIST(id)\
 }
 
-static const struct dce110_stream_enc_registers stream_enc_regs[] = {
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
 	stream_enc_regs(0),
 	stream_enc_regs(1),
 	stream_enc_regs(2),
 	stream_enc_regs(3),
 };
 
-static const struct dce_stream_encoder_shift se_shift = {
+static const struct dcn10_stream_encoder_shift se_shift = {
 		SE_COMMON_MASK_SH_LIST_DCN10(__SHIFT)
 };
 
-static const struct dce_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN10(_MASK),
-		.AFMT_GENERIC0_UPDATE = 0,
-		.AFMT_GENERIC2_UPDATE = 0,
-		.DP_DYN_RANGE = 0,
-		.DP_YCBCR_RANGE = 0,
-		.HDMI_AVI_INFO_SEND = 0,
-		.HDMI_AVI_INFO_CONT = 0,
-		.HDMI_AVI_INFO_LINE = 0,
-		.DP_SEC_AVI_ENABLE = 0,
-		.AFMT_AVI_INFO_VERSION = 0
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN10(_MASK)
 };
 
 #define audio_regs(id)\
@@ -228,13 +214,11 @@ static const struct dce_aduio_mask audio_mask = {
 	AUX_REG_LIST(id)\
 }
 
-static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
 		aux_regs(0),
 		aux_regs(1),
 		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4),
-		aux_regs(5)
+		aux_regs(3)
 };
 
 #define hpd_regs(id)\
@@ -242,13 +226,11 @@ static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
 	HPD_REG_LIST(id)\
 }
 
-static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
 		hpd_regs(0),
 		hpd_regs(1),
 		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4),
-		hpd_regs(5)
+		hpd_regs(3)
 };
 
 #define link_regs(id)\
@@ -257,14 +239,19 @@ static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
 	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
 }
 
-static const struct dce110_link_enc_registers link_enc_regs[] = {
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
 	link_regs(0),
 	link_regs(1),
 	link_regs(2),
-	link_regs(3),
-	link_regs(4),
-	link_regs(5),
-	link_regs(6),
+	link_regs(3)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+		LINK_ENCODER_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+		LINK_ENCODER_MASK_SH_LIST_DCN10(_MASK)
 };
 
 #define ipp_regs(id)\
@@ -320,11 +307,14 @@ static const struct dcn_dpp_registers tf_regs[] = {
 };
 
 static const struct dcn_dpp_shift tf_shift = {
-	TF_REG_LIST_SH_MASK_DCN10(__SHIFT)
+	TF_REG_LIST_SH_MASK_DCN10(__SHIFT),
+	TF_DEBUG_REG_LIST_SH_DCN10
+
 };
 
 static const struct dcn_dpp_mask tf_mask = {
 	TF_REG_LIST_SH_MASK_DCN10(_MASK),
+	TF_DEBUG_REG_LIST_MASK_DCN10
 };
 
 static const struct dcn_mpc_registers mpc_regs = {
@@ -457,6 +447,8 @@ static const struct dc_debug debug_defaults_drv = {
 		.vsr_support = true,
 		.performance_trace = false,
 		.az_endpoint_mute_only = true,
+		.recovery_enabled = false, /*enable this by default after testing.*/
+		.max_downscale_src_width = 3840,
 };
 
 static const struct dc_debug debug_defaults_diags = {
@@ -592,20 +584,22 @@ static const struct encoder_feature_support link_enc_feature = {
 struct link_encoder *dcn10_link_encoder_create(
 	const struct encoder_init_data *enc_init_data)
 {
-	struct dce110_link_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL);
+	struct dcn10_link_encoder *enc10 =
+		kzalloc(sizeof(struct dcn10_link_encoder), GFP_KERNEL);
 
-	if (!enc110)
+	if (!enc10)
 		return NULL;
 
-	dce110_link_encoder_construct(enc110,
+	dcn10_link_encoder_construct(enc10,
 				      enc_init_data,
 				      &link_enc_feature,
 				      &link_enc_regs[enc_init_data->transmitter],
 				      &link_enc_aux_regs[enc_init_data->channel - 1],
-				      &link_enc_hpd_regs[enc_init_data->hpd_source]);
+				      &link_enc_hpd_regs[enc_init_data->hpd_source],
+				      &le_shift,
+				      &le_mask);
 
-	return &enc110->base;
+	return &enc10->base;
 }
 
 struct clock_source *dcn10_clock_source_create(
@@ -650,16 +644,16 @@ static struct stream_encoder *dcn10_stream_encoder_create(
 	enum engine_id eng_id,
 	struct dc_context *ctx)
 {
-	struct dce110_stream_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL);
+	struct dcn10_stream_encoder *enc1 =
+		kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
 
-	if (!enc110)
+	if (!enc1)
 		return NULL;
 
-	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
+	dcn10_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id,
 					&stream_enc_regs[eng_id],
 					&se_shift, &se_mask);
-	return &enc110->base;
+	return &enc1->base;
 }
 
 static const struct dce_hwseq_registers hwseq_reg = {
@@ -918,36 +912,6 @@ enum dc_status dcn10_add_stream_to_ctx(
 	return result;
 }
 
-enum dc_status dcn10_validate_guaranteed(
-		struct dc *dc,
-		struct dc_stream_state *dc_stream,
-		struct dc_state *context)
-{
-	enum dc_status result = DC_ERROR_UNEXPECTED;
-
-	context->streams[0] = dc_stream;
-	dc_stream_retain(context->streams[0]);
-	context->stream_count++;
-
-	result = resource_map_pool_resources(dc, context, dc_stream);
-
-	if (result == DC_OK)
-		result = resource_map_phy_clock_resources(dc, context, dc_stream);
-
-	if (result == DC_OK)
-		result = build_mapped_resource(dc, context, dc_stream);
-
-	if (result == DC_OK) {
-		validate_guaranteed_copy_streams(
-				context, dc->caps.max_streams);
-		result = resource_build_scaling_params_for_context(dc, context);
-	}
-	if (result == DC_OK && !dcn_validate_bandwidth(dc, context))
-		return DC_FAIL_BANDWIDTH_VALIDATE;
-
-	return result;
-}
-
 static struct pipe_ctx *dcn10_acquire_idle_pipe_for_layer(
 		struct dc_state *context,
 		const struct resource_pool *pool,
@@ -978,235 +942,16 @@ static struct pipe_ctx *dcn10_acquire_idle_pipe_for_layer(
 	return idle_pipe;
 }
 
-enum dcc_control {
-	dcc_control__256_256_xxx,
-	dcc_control__128_128_xxx,
-	dcc_control__256_64_64,
-};
-
-enum segment_order {
-	segment_order__na,
-	segment_order__contiguous,
-	segment_order__non_contiguous,
-};
-
-static bool dcc_support_pixel_format(
-		enum surface_pixel_format format,
-		unsigned int *bytes_per_element)
-{
-	/* DML: get_bytes_per_element */
-	switch (format) {
-	case SURFACE_PIXEL_FORMAT_GRPH_ARGB1555:
-	case SURFACE_PIXEL_FORMAT_GRPH_RGB565:
-		*bytes_per_element = 2;
-		return true;
-	case SURFACE_PIXEL_FORMAT_GRPH_ARGB8888:
-	case SURFACE_PIXEL_FORMAT_GRPH_ABGR8888:
-	case SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010:
-	case SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010:
-		*bytes_per_element = 4;
-		return true;
-	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616:
-	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F:
-	case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F:
-		*bytes_per_element = 8;
-		return true;
-	default:
-		return false;
-	}
-}
-
-static bool dcc_support_swizzle(
-		enum swizzle_mode_values swizzle,
-		unsigned int bytes_per_element,
-		enum segment_order *segment_order_horz,
-		enum segment_order *segment_order_vert)
-{
-	bool standard_swizzle = false;
-	bool display_swizzle = false;
-
-	switch (swizzle) {
-	case DC_SW_4KB_S:
-	case DC_SW_64KB_S:
-	case DC_SW_VAR_S:
-	case DC_SW_4KB_S_X:
-	case DC_SW_64KB_S_X:
-	case DC_SW_VAR_S_X:
-		standard_swizzle = true;
-		break;
-	case DC_SW_4KB_D:
-	case DC_SW_64KB_D:
-	case DC_SW_VAR_D:
-	case DC_SW_4KB_D_X:
-	case DC_SW_64KB_D_X:
-	case DC_SW_VAR_D_X:
-		display_swizzle = true;
-		break;
-	default:
-		break;
-	}
-
-	if (bytes_per_element == 1 && standard_swizzle) {
-		*segment_order_horz = segment_order__contiguous;
-		*segment_order_vert = segment_order__na;
-		return true;
-	}
-	if (bytes_per_element == 2 && standard_swizzle) {
-		*segment_order_horz = segment_order__non_contiguous;
-		*segment_order_vert = segment_order__contiguous;
-		return true;
-	}
-	if (bytes_per_element == 4 && standard_swizzle) {
-		*segment_order_horz = segment_order__non_contiguous;
-		*segment_order_vert = segment_order__contiguous;
-		return true;
-	}
-	if (bytes_per_element == 8 && standard_swizzle) {
-		*segment_order_horz = segment_order__na;
-		*segment_order_vert = segment_order__contiguous;
-		return true;
-	}
-	if (bytes_per_element == 8 && display_swizzle) {
-		*segment_order_horz = segment_order__contiguous;
-		*segment_order_vert = segment_order__non_contiguous;
-		return true;
-	}
-
-	return false;
-}
-
-static void get_blk256_size(unsigned int *blk256_width, unsigned int *blk256_height,
-		unsigned int bytes_per_element)
-{
-	/* copied from DML.  might want to refactor DML to leverage from DML */
-	/* DML : get_blk256_size */
-	if (bytes_per_element == 1) {
-		*blk256_width = 16;
-		*blk256_height = 16;
-	} else if (bytes_per_element == 2) {
-		*blk256_width = 16;
-		*blk256_height = 8;
-	} else if (bytes_per_element == 4) {
-		*blk256_width = 8;
-		*blk256_height = 8;
-	} else if (bytes_per_element == 8) {
-		*blk256_width = 8;
-		*blk256_height = 4;
-	}
-}
-
-static void det_request_size(
-		unsigned int height,
-		unsigned int width,
-		unsigned int bpe,
-		bool *req128_horz_wc,
-		bool *req128_vert_wc)
-{
-	unsigned int detile_buf_size = 164 * 1024;  /* 164KB for DCN1.0 */
-
-	unsigned int blk256_height = 0;
-	unsigned int blk256_width = 0;
-	unsigned int swath_bytes_horz_wc, swath_bytes_vert_wc;
-
-	get_blk256_size(&blk256_width, &blk256_height, bpe);
-
-	swath_bytes_horz_wc = height * blk256_height * bpe;
-	swath_bytes_vert_wc = width * blk256_width * bpe;
-
-	*req128_horz_wc = (2 * swath_bytes_horz_wc <= detile_buf_size) ?
-			false : /* full 256B request */
-			true; /* half 128b request */
-
-	*req128_vert_wc = (2 * swath_bytes_vert_wc <= detile_buf_size) ?
-			false : /* full 256B request */
-			true; /* half 128b request */
-}
-
-static bool get_dcc_compression_cap(const struct dc *dc,
+static bool dcn10_get_dcc_compression_cap(const struct dc *dc,
 		const struct dc_dcc_surface_param *input,
 		struct dc_surface_dcc_cap *output)
 {
-	/* implement section 1.6.2.1 of DCN1_Programming_Guide.docx */
-	enum dcc_control dcc_control;
-	unsigned int bpe;
-	enum segment_order segment_order_horz, segment_order_vert;
-	bool req128_horz_wc, req128_vert_wc;
-
-	memset(output, 0, sizeof(*output));
-
-	if (dc->debug.disable_dcc == DCC_DISABLE)
-		return false;
-
-	if (!dcc_support_pixel_format(input->format,
-			&bpe))
-		return false;
-
-	if (!dcc_support_swizzle(input->swizzle_mode, bpe,
-			&segment_order_horz, &segment_order_vert))
-		return false;
-
-	det_request_size(input->surface_size.height,  input->surface_size.width,
-			bpe, &req128_horz_wc, &req128_vert_wc);
-
-	if (!req128_horz_wc && !req128_vert_wc) {
-		dcc_control = dcc_control__256_256_xxx;
-	} else if (input->scan == SCAN_DIRECTION_HORIZONTAL) {
-		if (!req128_horz_wc)
-			dcc_control = dcc_control__256_256_xxx;
-		else if (segment_order_horz == segment_order__contiguous)
-			dcc_control = dcc_control__128_128_xxx;
-		else
-			dcc_control = dcc_control__256_64_64;
-	} else if (input->scan == SCAN_DIRECTION_VERTICAL) {
-		if (!req128_vert_wc)
-			dcc_control = dcc_control__256_256_xxx;
-		else if (segment_order_vert == segment_order__contiguous)
-			dcc_control = dcc_control__128_128_xxx;
-		else
-			dcc_control = dcc_control__256_64_64;
-	} else {
-		if ((req128_horz_wc &&
-			segment_order_horz == segment_order__non_contiguous) ||
-			(req128_vert_wc &&
-			segment_order_vert == segment_order__non_contiguous))
-			/* access_dir not known, must use most constraining */
-			dcc_control = dcc_control__256_64_64;
-		else
-			/* reg128 is true for either horz and vert
-			 * but segment_order is contiguous
-			 */
-			dcc_control = dcc_control__128_128_xxx;
-	}
-
-	if (dc->debug.disable_dcc == DCC_HALF_REQ_DISALBE &&
-		dcc_control != dcc_control__256_256_xxx)
-		return false;
-
-	switch (dcc_control) {
-	case dcc_control__256_256_xxx:
-		output->grph.rgb.max_uncompressed_blk_size = 256;
-		output->grph.rgb.max_compressed_blk_size = 256;
-		output->grph.rgb.independent_64b_blks = false;
-		break;
-	case dcc_control__128_128_xxx:
-		output->grph.rgb.max_uncompressed_blk_size = 128;
-		output->grph.rgb.max_compressed_blk_size = 128;
-		output->grph.rgb.independent_64b_blks = false;
-		break;
-	case dcc_control__256_64_64:
-		output->grph.rgb.max_uncompressed_blk_size = 256;
-		output->grph.rgb.max_compressed_blk_size = 64;
-		output->grph.rgb.independent_64b_blks = true;
-		break;
-	}
-
-	output->capable = true;
-	output->const_color_support = false;
-
-	return true;
+	return dc->res_pool->hubbub->funcs->get_dcc_compression_cap(
+			dc->res_pool->hubbub,
+			input,
+			output);
 }
 
-
 static void dcn10_destroy_resource_pool(struct resource_pool **pool)
 {
 	struct dcn10_resource_pool *dcn10_pool = TO_DCN10_RES_POOL(*pool);
@@ -1227,13 +972,12 @@ static enum dc_status dcn10_validate_plane(const struct dc_plane_state *plane_st
 }
 
 static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = get_dcc_compression_cap
+	.get_dcc_compression_cap = dcn10_get_dcc_compression_cap
 };
 
 static struct resource_funcs dcn10_res_pool_funcs = {
 	.destroy = dcn10_destroy_resource_pool,
 	.link_enc_create = dcn10_link_encoder_create,
-	.validate_guaranteed = dcn10_validate_guaranteed,
 	.validate_bandwidth = dcn_validate_bandwidth,
 	.acquire_idle_pipe_for_layer = dcn10_acquire_idle_pipe_for_layer,
 	.validate_plane = dcn10_validate_plane,
@@ -1282,6 +1026,7 @@ static bool construct(
 	dc->caps.max_cursor_size = 256;
 	dc->caps.max_slave_planes = 1;
 	dc->caps.is_apu = true;
+	dc->caps.post_blend_color_processing = false;
 
 	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
 		dc->debug = debug_defaults_drv;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
new file mode 100644
index 000000000000..653b7b2efe2e
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
@@ -0,0 +1,1490 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#include "dc_bios_types.h"
+#include "dcn10_stream_encoder.h"
+#include "reg_helper.h"
+#include "hw_shared.h"
+
+#define DC_LOGGER \
+		enc1->base.ctx->logger
+
+
+#define REG(reg)\
+	(enc1->regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+	enc1->se_shift->field_name, enc1->se_mask->field_name
+
+#define VBI_LINE_0 0
+#define DP_BLANK_MAX_RETRY 20
+#define HDMI_CLOCK_CHANNEL_RATE_MORE_340M 340000
+
+
+enum {
+	DP_MST_UPDATE_MAX_RETRY = 50
+};
+
+#define CTX \
+	enc1->base.ctx
+
+void enc1_update_generic_info_packet(
+	struct dcn10_stream_encoder *enc1,
+	uint32_t packet_index,
+	const struct dc_info_packet *info_packet)
+{
+	uint32_t regval;
+	/* TODOFPGA Figure out a proper number for max_retries polling for lock
+	 * use 50 for now.
+	 */
+	uint32_t max_retries = 50;
+
+	/*we need turn on clock before programming AFMT block*/
+	REG_UPDATE(AFMT_CNTL, AFMT_AUDIO_CLOCK_EN, 1);
+
+	if (packet_index >= 8)
+		ASSERT(0);
+
+	/* poll dig_update_lock is not locked -> asic internal signal
+	 * assume otg master lock will unlock it
+	 */
+/*		REG_WAIT(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_LOCK_STATUS,
+			0, 10, max_retries);*/
+
+	/* check if HW reading GSP memory */
+	REG_WAIT(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_CONFLICT,
+			0, 10, max_retries);
+
+	/* HW does is not reading GSP memory not reading too long ->
+	 * something wrong. clear GPS memory access and notify?
+	 * hw SW is writing to GSP memory
+	 */
+	REG_UPDATE(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_CONFLICT_CLR, 1);
+
+	/* choose which generic packet to use */
+	regval = REG_READ(AFMT_VBI_PACKET_CONTROL);
+	REG_UPDATE(AFMT_VBI_PACKET_CONTROL,
+			AFMT_GENERIC_INDEX, packet_index);
+
+	/* write generic packet header
+	 * (4th byte is for GENERIC0 only)
+	 */
+	REG_SET_4(AFMT_GENERIC_HDR, 0,
+			AFMT_GENERIC_HB0, info_packet->hb0,
+			AFMT_GENERIC_HB1, info_packet->hb1,
+			AFMT_GENERIC_HB2, info_packet->hb2,
+			AFMT_GENERIC_HB3, info_packet->hb3);
+
+	/* write generic packet contents
+	 * (we never use last 4 bytes)
+	 * there are 8 (0-7) mmDIG0_AFMT_GENERIC0_x registers
+	 */
+	{
+		const uint32_t *content =
+			(const uint32_t *) &info_packet->sb[0];
+
+		REG_WRITE(AFMT_GENERIC_0, *content++);
+		REG_WRITE(AFMT_GENERIC_1, *content++);
+		REG_WRITE(AFMT_GENERIC_2, *content++);
+		REG_WRITE(AFMT_GENERIC_3, *content++);
+		REG_WRITE(AFMT_GENERIC_4, *content++);
+		REG_WRITE(AFMT_GENERIC_5, *content++);
+		REG_WRITE(AFMT_GENERIC_6, *content++);
+		REG_WRITE(AFMT_GENERIC_7, *content);
+	}
+
+	switch (packet_index) {
+	case 0:
+		REG_UPDATE(AFMT_VBI_PACKET_CONTROL1,
+				AFMT_GENERIC0_FRAME_UPDATE, 1);
+		break;
+	case 1:
+		REG_UPDATE(AFMT_VBI_PACKET_CONTROL1,
+				AFMT_GENERIC1_FRAME_UPDATE, 1);
+		break;
+	case 2:
+		REG_UPDATE(AFMT_VBI_PACKET_CONTROL1,
+				AFMT_GENERIC2_FRAME_UPDATE, 1);
+		break;
+	case 3:
+		REG_UPDATE(AFMT_VBI_PACKET_CONTROL1,
+				AFMT_GENERIC3_FRAME_UPDATE, 1);
+		break;
+	case 4:
+		REG_UPDATE(AFMT_VBI_PACKET_CONTROL1,
+				AFMT_GENERIC4_FRAME_UPDATE, 1);
+		break;
+	case 5:
+		REG_UPDATE(AFMT_VBI_PACKET_CONTROL1,
+				AFMT_GENERIC5_FRAME_UPDATE, 1);
+		break;
+	case 6:
+		REG_UPDATE(AFMT_VBI_PACKET_CONTROL1,
+				AFMT_GENERIC6_FRAME_UPDATE, 1);
+		break;
+	case 7:
+		REG_UPDATE(AFMT_VBI_PACKET_CONTROL1,
+				AFMT_GENERIC7_FRAME_UPDATE, 1);
+		break;
+	default:
+		break;
+	}
+}
+
+static void enc1_update_hdmi_info_packet(
+	struct dcn10_stream_encoder *enc1,
+	uint32_t packet_index,
+	const struct dc_info_packet *info_packet)
+{
+	uint32_t cont, send, line;
+
+	if (info_packet->valid) {
+		enc1_update_generic_info_packet(
+			enc1,
+			packet_index,
+			info_packet);
+
+		/* enable transmission of packet(s) -
+		 * packet transmission begins on the next frame
+		 */
+		cont = 1;
+		/* send packet(s) every frame */
+		send = 1;
+		/* select line number to send packets on */
+		line = 2;
+	} else {
+		cont = 0;
+		send = 0;
+		line = 0;
+	}
+
+	/* choose which generic packet control to use */
+	switch (packet_index) {
+	case 0:
+		REG_UPDATE_3(HDMI_GENERIC_PACKET_CONTROL0,
+				HDMI_GENERIC0_CONT, cont,
+				HDMI_GENERIC0_SEND, send,
+				HDMI_GENERIC0_LINE, line);
+		break;
+	case 1:
+		REG_UPDATE_3(HDMI_GENERIC_PACKET_CONTROL0,
+				HDMI_GENERIC1_CONT, cont,
+				HDMI_GENERIC1_SEND, send,
+				HDMI_GENERIC1_LINE, line);
+		break;
+	case 2:
+		REG_UPDATE_3(HDMI_GENERIC_PACKET_CONTROL1,
+				HDMI_GENERIC0_CONT, cont,
+				HDMI_GENERIC0_SEND, send,
+				HDMI_GENERIC0_LINE, line);
+		break;
+	case 3:
+		REG_UPDATE_3(HDMI_GENERIC_PACKET_CONTROL1,
+				HDMI_GENERIC1_CONT, cont,
+				HDMI_GENERIC1_SEND, send,
+				HDMI_GENERIC1_LINE, line);
+		break;
+	case 4:
+		REG_UPDATE_3(HDMI_GENERIC_PACKET_CONTROL2,
+				HDMI_GENERIC0_CONT, cont,
+				HDMI_GENERIC0_SEND, send,
+				HDMI_GENERIC0_LINE, line);
+		break;
+	case 5:
+		REG_UPDATE_3(HDMI_GENERIC_PACKET_CONTROL2,
+				HDMI_GENERIC1_CONT, cont,
+				HDMI_GENERIC1_SEND, send,
+				HDMI_GENERIC1_LINE, line);
+		break;
+	case 6:
+		REG_UPDATE_3(HDMI_GENERIC_PACKET_CONTROL3,
+				HDMI_GENERIC0_CONT, cont,
+				HDMI_GENERIC0_SEND, send,
+				HDMI_GENERIC0_LINE, line);
+		break;
+	case 7:
+		REG_UPDATE_3(HDMI_GENERIC_PACKET_CONTROL3,
+				HDMI_GENERIC1_CONT, cont,
+				HDMI_GENERIC1_SEND, send,
+				HDMI_GENERIC1_LINE, line);
+		break;
+	default:
+		/* invalid HW packet index */
+		DC_LOG_WARNING(
+			"Invalid HW packet index: %s()\n",
+			__func__);
+		return;
+	}
+}
+
+/* setup stream encoder in dp mode */
+void enc1_stream_encoder_dp_set_stream_attribute(
+	struct stream_encoder *enc,
+	struct dc_crtc_timing *crtc_timing,
+	enum dc_color_space output_color_space)
+{
+	uint32_t h_active_start;
+	uint32_t v_active_start;
+	uint32_t misc0 = 0;
+	uint32_t misc1 = 0;
+	uint32_t h_blank;
+	uint32_t h_back_porch;
+	uint8_t synchronous_clock = 0; /* asynchronous mode */
+	uint8_t colorimetry_bpc;
+	uint8_t dynamic_range_rgb = 0; /*full range*/
+	uint8_t dynamic_range_ycbcr = 1; /*bt709*/
+
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	REG_UPDATE(DP_DB_CNTL, DP_DB_DISABLE, 1);
+
+	/* set pixel encoding */
+	switch (crtc_timing->pixel_encoding) {
+	case PIXEL_ENCODING_YCBCR422:
+		REG_UPDATE(DP_PIXEL_FORMAT, DP_PIXEL_ENCODING,
+				DP_PIXEL_ENCODING_TYPE_YCBCR422);
+		break;
+	case PIXEL_ENCODING_YCBCR444:
+		REG_UPDATE(DP_PIXEL_FORMAT, DP_PIXEL_ENCODING,
+				DP_PIXEL_ENCODING_TYPE_YCBCR444);
+
+		if (crtc_timing->flags.Y_ONLY)
+			if (crtc_timing->display_color_depth != COLOR_DEPTH_666)
+				/* HW testing only, no use case yet.
+				 * Color depth of Y-only could be
+				 * 8, 10, 12, 16 bits
+				 */
+				REG_UPDATE(DP_PIXEL_FORMAT, DP_PIXEL_ENCODING,
+						DP_PIXEL_ENCODING_TYPE_Y_ONLY);
+		/* Note: DP_MSA_MISC1 bit 7 is the indicator
+		 * of Y-only mode.
+		 * This bit is set in HW if register
+		 * DP_PIXEL_ENCODING is programmed to 0x4
+		 */
+		break;
+	case PIXEL_ENCODING_YCBCR420:
+		REG_UPDATE(DP_PIXEL_FORMAT, DP_PIXEL_ENCODING,
+				DP_PIXEL_ENCODING_TYPE_YCBCR420);
+		REG_UPDATE(DP_VID_TIMING, DP_VID_N_MUL, 1);
+		break;
+	default:
+		REG_UPDATE(DP_PIXEL_FORMAT, DP_PIXEL_ENCODING,
+				DP_PIXEL_ENCODING_TYPE_RGB444);
+		break;
+	}
+
+	misc1 = REG_READ(DP_MSA_MISC);
+
+	/* set color depth */
+
+	switch (crtc_timing->display_color_depth) {
+	case COLOR_DEPTH_666:
+		REG_UPDATE(DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH,
+				0);
+		break;
+	case COLOR_DEPTH_888:
+		REG_UPDATE(DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH,
+				DP_COMPONENT_PIXEL_DEPTH_8BPC);
+		break;
+	case COLOR_DEPTH_101010:
+		REG_UPDATE(DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH,
+				DP_COMPONENT_PIXEL_DEPTH_10BPC);
+
+		break;
+	case COLOR_DEPTH_121212:
+		REG_UPDATE(DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH,
+				DP_COMPONENT_PIXEL_DEPTH_12BPC);
+		break;
+	default:
+		REG_UPDATE(DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH,
+				DP_COMPONENT_PIXEL_DEPTH_6BPC);
+		break;
+	}
+
+	/* set dynamic range and YCbCr range */
+
+	switch (crtc_timing->display_color_depth) {
+	case COLOR_DEPTH_666:
+		colorimetry_bpc = 0;
+		break;
+	case COLOR_DEPTH_888:
+		colorimetry_bpc = 1;
+		break;
+	case COLOR_DEPTH_101010:
+		colorimetry_bpc = 2;
+		break;
+	case COLOR_DEPTH_121212:
+		colorimetry_bpc = 3;
+		break;
+	default:
+		colorimetry_bpc = 0;
+		break;
+	}
+
+	misc0 = misc0 | synchronous_clock;
+	misc0 = colorimetry_bpc << 5;
+
+	switch (output_color_space) {
+	case COLOR_SPACE_SRGB:
+		misc0 = misc0 | 0x0;
+		misc1 = misc1 & ~0x80; /* bit7 = 0*/
+		dynamic_range_rgb = 0; /*full range*/
+		break;
+	case COLOR_SPACE_SRGB_LIMITED:
+		misc0 = misc0 | 0x8; /* bit3=1 */
+		misc1 = misc1 & ~0x80; /* bit7 = 0*/
+		dynamic_range_rgb = 1; /*limited range*/
+		break;
+	case COLOR_SPACE_YCBCR601:
+	case COLOR_SPACE_YCBCR601_LIMITED:
+		misc0 = misc0 | 0x8; /* bit3=1, bit4=0 */
+		misc1 = misc1 & ~0x80; /* bit7 = 0*/
+		dynamic_range_ycbcr = 0; /*bt601*/
+		if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422)
+			misc0 = misc0 | 0x2; /* bit2=0, bit1=1 */
+		else if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR444)
+			misc0 = misc0 | 0x4; /* bit2=1, bit1=0 */
+		break;
+	case COLOR_SPACE_YCBCR709:
+	case COLOR_SPACE_YCBCR709_LIMITED:
+		misc0 = misc0 | 0x18; /* bit3=1, bit4=1 */
+		misc1 = misc1 & ~0x80; /* bit7 = 0*/
+		dynamic_range_ycbcr = 1; /*bt709*/
+		if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422)
+			misc0 = misc0 | 0x2; /* bit2=0, bit1=1 */
+		else if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR444)
+			misc0 = misc0 | 0x4; /* bit2=1, bit1=0 */
+		break;
+	case COLOR_SPACE_2020_RGB_LIMITEDRANGE:
+		dynamic_range_rgb = 1; /*limited range*/
+		break;
+	case COLOR_SPACE_2020_RGB_FULLRANGE:
+	case COLOR_SPACE_2020_YCBCR:
+	case COLOR_SPACE_XR_RGB:
+	case COLOR_SPACE_MSREF_SCRGB:
+	case COLOR_SPACE_ADOBERGB:
+	case COLOR_SPACE_DCIP3:
+	case COLOR_SPACE_XV_YCC_709:
+	case COLOR_SPACE_XV_YCC_601:
+	case COLOR_SPACE_DISPLAYNATIVE:
+	case COLOR_SPACE_DOLBYVISION:
+	case COLOR_SPACE_APPCTRL:
+	case COLOR_SPACE_CUSTOMPOINTS:
+	case COLOR_SPACE_UNKNOWN:
+		/* do nothing */
+		break;
+	}
+
+	REG_SET(DP_MSA_COLORIMETRY, 0, DP_MSA_MISC0, misc0);
+	REG_WRITE(DP_MSA_MISC, misc1);   /* MSA_MISC1 */
+
+	/* dcn new register
+	 * dc_crtc_timing is vesa dmt struct. data from edid
+	 */
+	REG_SET_2(DP_MSA_TIMING_PARAM1, 0,
+			DP_MSA_HTOTAL, crtc_timing->h_total,
+			DP_MSA_VTOTAL, crtc_timing->v_total);
+
+	/* calculate from vesa timing parameters
+	 * h_active_start related to leading edge of sync
+	 */
+
+	h_blank = crtc_timing->h_total - crtc_timing->h_border_left -
+			crtc_timing->h_addressable - crtc_timing->h_border_right;
+
+	h_back_porch = h_blank - crtc_timing->h_front_porch -
+			crtc_timing->h_sync_width;
+
+	/* start at beginning of left border */
+	h_active_start = crtc_timing->h_sync_width + h_back_porch;
+
+
+	v_active_start = crtc_timing->v_total - crtc_timing->v_border_top -
+			crtc_timing->v_addressable - crtc_timing->v_border_bottom -
+			crtc_timing->v_front_porch;
+
+
+	/* start at beginning of left border */
+	REG_SET_2(DP_MSA_TIMING_PARAM2, 0,
+		DP_MSA_HSTART, h_active_start,
+		DP_MSA_VSTART, v_active_start);
+
+	REG_SET_4(DP_MSA_TIMING_PARAM3, 0,
+			DP_MSA_HSYNCWIDTH,
+			crtc_timing->h_sync_width,
+			DP_MSA_HSYNCPOLARITY,
+			!crtc_timing->flags.HSYNC_POSITIVE_POLARITY,
+			DP_MSA_VSYNCWIDTH,
+			crtc_timing->v_sync_width,
+			DP_MSA_VSYNCPOLARITY,
+			!crtc_timing->flags.VSYNC_POSITIVE_POLARITY);
+
+	/* HWDITH include border or overscan */
+	REG_SET_2(DP_MSA_TIMING_PARAM4, 0,
+		DP_MSA_HWIDTH, crtc_timing->h_border_left +
+		crtc_timing->h_addressable + crtc_timing->h_border_right,
+		DP_MSA_VHEIGHT, crtc_timing->v_border_top +
+		crtc_timing->v_addressable + crtc_timing->v_border_bottom);
+}
+
+static void enc1_stream_encoder_set_stream_attribute_helper(
+		struct dcn10_stream_encoder *enc1,
+		struct dc_crtc_timing *crtc_timing)
+{
+	switch (crtc_timing->pixel_encoding) {
+	case PIXEL_ENCODING_YCBCR422:
+		REG_UPDATE(DIG_FE_CNTL, TMDS_PIXEL_ENCODING, 1);
+		break;
+	default:
+		REG_UPDATE(DIG_FE_CNTL, TMDS_PIXEL_ENCODING, 0);
+		break;
+	}
+	REG_UPDATE(DIG_FE_CNTL, TMDS_COLOR_FORMAT, 0);
+}
+
+/* setup stream encoder in hdmi mode */
+void enc1_stream_encoder_hdmi_set_stream_attribute(
+	struct stream_encoder *enc,
+	struct dc_crtc_timing *crtc_timing,
+	int actual_pix_clk_khz,
+	bool enable_audio)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	struct bp_encoder_control cntl = {0};
+
+	cntl.action = ENCODER_CONTROL_SETUP;
+	cntl.engine_id = enc1->base.id;
+	cntl.signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	cntl.enable_dp_audio = enable_audio;
+	cntl.pixel_clock = actual_pix_clk_khz;
+	cntl.lanes_number = LANE_COUNT_FOUR;
+
+	if (enc1->base.bp->funcs->encoder_control(
+			enc1->base.bp, &cntl) != BP_RESULT_OK)
+		return;
+
+	enc1_stream_encoder_set_stream_attribute_helper(enc1, crtc_timing);
+
+	/* setup HDMI engine */
+	REG_UPDATE_5(HDMI_CONTROL,
+		HDMI_PACKET_GEN_VERSION, 1,
+		HDMI_KEEPOUT_MODE, 1,
+		HDMI_DEEP_COLOR_ENABLE, 0,
+		HDMI_DATA_SCRAMBLE_EN, 0,
+		HDMI_CLOCK_CHANNEL_RATE, 0);
+
+
+	switch (crtc_timing->display_color_depth) {
+	case COLOR_DEPTH_888:
+		REG_UPDATE(HDMI_CONTROL, HDMI_DEEP_COLOR_DEPTH, 0);
+		break;
+	case COLOR_DEPTH_101010:
+		if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422) {
+			REG_UPDATE_2(HDMI_CONTROL,
+					HDMI_DEEP_COLOR_DEPTH, 1,
+					HDMI_DEEP_COLOR_ENABLE, 0);
+		} else {
+			REG_UPDATE_2(HDMI_CONTROL,
+					HDMI_DEEP_COLOR_DEPTH, 1,
+					HDMI_DEEP_COLOR_ENABLE, 1);
+			}
+		break;
+	case COLOR_DEPTH_121212:
+		if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422) {
+			REG_UPDATE_2(HDMI_CONTROL,
+					HDMI_DEEP_COLOR_DEPTH, 2,
+					HDMI_DEEP_COLOR_ENABLE, 0);
+		} else {
+			REG_UPDATE_2(HDMI_CONTROL,
+					HDMI_DEEP_COLOR_DEPTH, 2,
+					HDMI_DEEP_COLOR_ENABLE, 1);
+			}
+		break;
+	case COLOR_DEPTH_161616:
+		REG_UPDATE_2(HDMI_CONTROL,
+				HDMI_DEEP_COLOR_DEPTH, 3,
+				HDMI_DEEP_COLOR_ENABLE, 1);
+		break;
+	default:
+		break;
+	}
+
+	if (actual_pix_clk_khz >= HDMI_CLOCK_CHANNEL_RATE_MORE_340M) {
+		/* enable HDMI data scrambler
+		 * HDMI_CLOCK_CHANNEL_RATE_MORE_340M
+		 * Clock channel frequency is 1/4 of character rate.
+		 */
+		REG_UPDATE_2(HDMI_CONTROL,
+			HDMI_DATA_SCRAMBLE_EN, 1,
+			HDMI_CLOCK_CHANNEL_RATE, 1);
+	} else if (crtc_timing->flags.LTE_340MCSC_SCRAMBLE) {
+
+		/* TODO: New feature for DCE11, still need to implement */
+
+		/* enable HDMI data scrambler
+		 * HDMI_CLOCK_CHANNEL_FREQ_EQUAL_TO_CHAR_RATE
+		 * Clock channel frequency is the same
+		 * as character rate
+		 */
+		REG_UPDATE_2(HDMI_CONTROL,
+			HDMI_DATA_SCRAMBLE_EN, 1,
+			HDMI_CLOCK_CHANNEL_RATE, 0);
+	}
+
+
+	REG_UPDATE_3(HDMI_VBI_PACKET_CONTROL,
+		HDMI_GC_CONT, 1,
+		HDMI_GC_SEND, 1,
+		HDMI_NULL_SEND, 1);
+
+	/* following belongs to audio */
+	REG_UPDATE(HDMI_INFOFRAME_CONTROL0, HDMI_AUDIO_INFO_SEND, 1);
+
+	REG_UPDATE(AFMT_INFOFRAME_CONTROL0, AFMT_AUDIO_INFO_UPDATE, 1);
+
+	REG_UPDATE(HDMI_INFOFRAME_CONTROL1, HDMI_AUDIO_INFO_LINE,
+				VBI_LINE_0 + 2);
+
+	REG_UPDATE(HDMI_GC, HDMI_GC_AVMUTE, 0);
+}
+
+/* setup stream encoder in dvi mode */
+void enc1_stream_encoder_dvi_set_stream_attribute(
+	struct stream_encoder *enc,
+	struct dc_crtc_timing *crtc_timing,
+	bool is_dual_link)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	struct bp_encoder_control cntl = {0};
+
+	cntl.action = ENCODER_CONTROL_SETUP;
+	cntl.engine_id = enc1->base.id;
+	cntl.signal = is_dual_link ?
+			SIGNAL_TYPE_DVI_DUAL_LINK : SIGNAL_TYPE_DVI_SINGLE_LINK;
+	cntl.enable_dp_audio = false;
+	cntl.pixel_clock = crtc_timing->pix_clk_khz;
+	cntl.lanes_number = (is_dual_link) ? LANE_COUNT_EIGHT : LANE_COUNT_FOUR;
+
+	if (enc1->base.bp->funcs->encoder_control(
+			enc1->base.bp, &cntl) != BP_RESULT_OK)
+		return;
+
+	ASSERT(crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB);
+	ASSERT(crtc_timing->display_color_depth == COLOR_DEPTH_888);
+	enc1_stream_encoder_set_stream_attribute_helper(enc1, crtc_timing);
+}
+
+void enc1_stream_encoder_set_mst_bandwidth(
+	struct stream_encoder *enc,
+	struct fixed31_32 avg_time_slots_per_mtp)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	uint32_t x = dc_fixpt_floor(
+		avg_time_slots_per_mtp);
+	uint32_t y = dc_fixpt_ceil(
+		dc_fixpt_shl(
+			dc_fixpt_sub_int(
+				avg_time_slots_per_mtp,
+				x),
+			26));
+
+	REG_SET_2(DP_MSE_RATE_CNTL, 0,
+		DP_MSE_RATE_X, x,
+		DP_MSE_RATE_Y, y);
+
+	/* wait for update to be completed on the link */
+	/* i.e. DP_MSE_RATE_UPDATE_PENDING field (read only) */
+	/* is reset to 0 (not pending) */
+	REG_WAIT(DP_MSE_RATE_UPDATE, DP_MSE_RATE_UPDATE_PENDING,
+			0,
+			10, DP_MST_UPDATE_MAX_RETRY);
+}
+
+static void enc1_stream_encoder_update_hdmi_info_packets(
+	struct stream_encoder *enc,
+	const struct encoder_info_frame *info_frame)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	/* for bring up, disable dp double  TODO */
+	REG_UPDATE(HDMI_DB_CONTROL, HDMI_DB_DISABLE, 1);
+
+	enc1_update_hdmi_info_packet(enc1, 0, &info_frame->avi);
+	enc1_update_hdmi_info_packet(enc1, 1, &info_frame->vendor);
+	enc1_update_hdmi_info_packet(enc1, 2, &info_frame->gamut);
+	enc1_update_hdmi_info_packet(enc1, 3, &info_frame->spd);
+	enc1_update_hdmi_info_packet(enc1, 4, &info_frame->hdrsmd);
+}
+
+static void enc1_stream_encoder_stop_hdmi_info_packets(
+	struct stream_encoder *enc)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	/* stop generic packets 0 & 1 on HDMI */
+	REG_SET_6(HDMI_GENERIC_PACKET_CONTROL0, 0,
+		HDMI_GENERIC1_CONT, 0,
+		HDMI_GENERIC1_LINE, 0,
+		HDMI_GENERIC1_SEND, 0,
+		HDMI_GENERIC0_CONT, 0,
+		HDMI_GENERIC0_LINE, 0,
+		HDMI_GENERIC0_SEND, 0);
+
+	/* stop generic packets 2 & 3 on HDMI */
+	REG_SET_6(HDMI_GENERIC_PACKET_CONTROL1, 0,
+		HDMI_GENERIC0_CONT, 0,
+		HDMI_GENERIC0_LINE, 0,
+		HDMI_GENERIC0_SEND, 0,
+		HDMI_GENERIC1_CONT, 0,
+		HDMI_GENERIC1_LINE, 0,
+		HDMI_GENERIC1_SEND, 0);
+
+	/* stop generic packets 2 & 3 on HDMI */
+	REG_SET_6(HDMI_GENERIC_PACKET_CONTROL2, 0,
+		HDMI_GENERIC0_CONT, 0,
+		HDMI_GENERIC0_LINE, 0,
+		HDMI_GENERIC0_SEND, 0,
+		HDMI_GENERIC1_CONT, 0,
+		HDMI_GENERIC1_LINE, 0,
+		HDMI_GENERIC1_SEND, 0);
+
+	REG_SET_6(HDMI_GENERIC_PACKET_CONTROL3, 0,
+		HDMI_GENERIC0_CONT, 0,
+		HDMI_GENERIC0_LINE, 0,
+		HDMI_GENERIC0_SEND, 0,
+		HDMI_GENERIC1_CONT, 0,
+		HDMI_GENERIC1_LINE, 0,
+		HDMI_GENERIC1_SEND, 0);
+}
+
+void enc1_stream_encoder_update_dp_info_packets(
+	struct stream_encoder *enc,
+	const struct encoder_info_frame *info_frame)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	uint32_t value = 0;
+
+	if (info_frame->vsc.valid)
+		enc1_update_generic_info_packet(
+					enc1,
+					0,  /* packetIndex */
+					&info_frame->vsc);
+
+	if (info_frame->spd.valid)
+		enc1_update_generic_info_packet(
+				enc1,
+				2,  /* packetIndex */
+				&info_frame->spd);
+
+	if (info_frame->hdrsmd.valid)
+		enc1_update_generic_info_packet(
+				enc1,
+				3,  /* packetIndex */
+				&info_frame->hdrsmd);
+
+	/* enable/disable transmission of packet(s).
+	 * If enabled, packet transmission begins on the next frame
+	 */
+	REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP0_ENABLE, info_frame->vsc.valid);
+	REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP2_ENABLE, info_frame->spd.valid);
+	REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP3_ENABLE, info_frame->hdrsmd.valid);
+
+
+	/* This bit is the master enable bit.
+	 * When enabling secondary stream engine,
+	 * this master bit must also be set.
+	 * This register shared with audio info frame.
+	 * Therefore we need to enable master bit
+	 * if at least on of the fields is not 0
+	 */
+	value = REG_READ(DP_SEC_CNTL);
+	if (value)
+		REG_UPDATE(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, 1);
+}
+
+void enc1_stream_encoder_stop_dp_info_packets(
+	struct stream_encoder *enc)
+{
+	/* stop generic packets on DP */
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	uint32_t value = 0;
+
+	REG_SET_10(DP_SEC_CNTL, 0,
+		DP_SEC_GSP0_ENABLE, 0,
+		DP_SEC_GSP1_ENABLE, 0,
+		DP_SEC_GSP2_ENABLE, 0,
+		DP_SEC_GSP3_ENABLE, 0,
+		DP_SEC_GSP4_ENABLE, 0,
+		DP_SEC_GSP5_ENABLE, 0,
+		DP_SEC_GSP6_ENABLE, 0,
+		DP_SEC_GSP7_ENABLE, 0,
+		DP_SEC_MPG_ENABLE, 0,
+		DP_SEC_STREAM_ENABLE, 0);
+
+	/* this register shared with audio info frame.
+	 * therefore we need to keep master enabled
+	 * if at least one of the fields is not 0 */
+	value = REG_READ(DP_SEC_CNTL);
+	if (value)
+		REG_UPDATE(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, 1);
+
+}
+
+void enc1_stream_encoder_dp_blank(
+	struct stream_encoder *enc)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	uint32_t retries = 0;
+	uint32_t  reg1 = 0;
+	uint32_t max_retries = DP_BLANK_MAX_RETRY * 10;
+
+	/* Note: For CZ, we are changing driver default to disable
+	 * stream deferred to next VBLANK. If results are positive, we
+	 * will make the same change to all DCE versions. There are a
+	 * handful of panels that cannot handle disable stream at
+	 * HBLANK and will result in a white line flash across the
+	 * screen on stream disable.
+	 */
+	REG_GET(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, &reg1);
+	if ((reg1 & 0x1) == 0)
+		/*stream not enabled*/
+		return;
+	/* Specify the video stream disable point
+	 * (2 = start of the next vertical blank)
+	 */
+	REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_DIS_DEFER, 2);
+	/* Larger delay to wait until VBLANK - use max retry of
+	 * 10us*3000=30ms. This covers 16.6ms of typical 60 Hz mode +
+	 * a little more because we may not trust delay accuracy.
+	 */
+	max_retries = DP_BLANK_MAX_RETRY * 150;
+
+	/* disable DP stream */
+	REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, 0);
+
+	/* the encoder stops sending the video stream
+	 * at the start of the vertical blanking.
+	 * Poll for DP_VID_STREAM_STATUS == 0
+	 */
+
+	REG_WAIT(DP_VID_STREAM_CNTL, DP_VID_STREAM_STATUS,
+			0,
+			10, max_retries);
+
+	ASSERT(retries <= max_retries);
+
+	/* Tell the DP encoder to ignore timing from CRTC, must be done after
+	 * the polling. If we set DP_STEER_FIFO_RESET before DP stream blank is
+	 * complete, stream status will be stuck in video stream enabled state,
+	 * i.e. DP_VID_STREAM_STATUS stuck at 1.
+	 */
+
+	REG_UPDATE(DP_STEER_FIFO, DP_STEER_FIFO_RESET, true);
+}
+
+/* output video stream to link encoder */
+void enc1_stream_encoder_dp_unblank(
+	struct stream_encoder *enc,
+	const struct encoder_unblank_param *param)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	if (param->link_settings.link_rate != LINK_RATE_UNKNOWN) {
+		uint32_t n_vid = 0x8000;
+		uint32_t m_vid;
+
+		/* M / N = Fstream / Flink
+		 * m_vid / n_vid = pixel rate / link rate
+		 */
+
+		uint64_t m_vid_l = n_vid;
+
+		m_vid_l *= param->pixel_clk_khz;
+		m_vid_l = div_u64(m_vid_l,
+			param->link_settings.link_rate
+				* LINK_RATE_REF_FREQ_IN_KHZ);
+
+		m_vid = (uint32_t) m_vid_l;
+
+		/* enable auto measurement */
+
+		REG_UPDATE(DP_VID_TIMING, DP_VID_M_N_GEN_EN, 0);
+
+		/* auto measurement need 1 full 0x8000 symbol cycle to kick in,
+		 * therefore program initial value for Mvid and Nvid
+		 */
+
+		REG_UPDATE(DP_VID_N, DP_VID_N, n_vid);
+
+		REG_UPDATE(DP_VID_M, DP_VID_M, m_vid);
+
+		REG_UPDATE(DP_VID_TIMING, DP_VID_M_N_GEN_EN, 1);
+	}
+
+	/* set DIG_START to 0x1 to resync FIFO */
+
+	REG_UPDATE(DIG_FE_CNTL, DIG_START, 1);
+
+	/* switch DP encoder to CRTC data */
+
+	REG_UPDATE(DP_STEER_FIFO, DP_STEER_FIFO_RESET, 0);
+
+	/* wait 100us for DIG/DP logic to prime
+	 * (i.e. a few video lines)
+	 */
+	udelay(100);
+
+	/* the hardware would start sending video at the start of the next DP
+	 * frame (i.e. rising edge of the vblank).
+	 * NOTE: We used to program DP_VID_STREAM_DIS_DEFER = 2 here, but this
+	 * register has no effect on enable transition! HW always guarantees
+	 * VID_STREAM enable at start of next frame, and this is not
+	 * programmable
+	 */
+
+	REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, true);
+}
+
+void enc1_stream_encoder_set_avmute(
+	struct stream_encoder *enc,
+	bool enable)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	unsigned int value = enable ? 1 : 0;
+
+	REG_UPDATE(HDMI_GC, HDMI_GC_AVMUTE, value);
+}
+
+
+#define DP_SEC_AUD_N__DP_SEC_AUD_N__DEFAULT 0x8000
+#define DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE__AUTO_CALC 1
+
+#include "include/audio_types.h"
+
+/**
+* speakersToChannels
+*
+* @brief
+*  translate speakers to channels
+*
+*  FL  - Front Left
+*  FR  - Front Right
+*  RL  - Rear Left
+*  RR  - Rear Right
+*  RC  - Rear Center
+*  FC  - Front Center
+*  FLC - Front Left Center
+*  FRC - Front Right Center
+*  RLC - Rear Left Center
+*  RRC - Rear Right Center
+*  LFE - Low Freq Effect
+*
+*               FC
+*          FLC      FRC
+*    FL                    FR
+*
+*                    LFE
+*              ()
+*
+*
+*    RL                    RR
+*          RLC      RRC
+*               RC
+*
+*             ch  8   7   6   5   4   3   2   1
+* 0b00000011      -   -   -   -   -   -   FR  FL
+* 0b00000111      -   -   -   -   -   LFE FR  FL
+* 0b00001011      -   -   -   -   FC  -   FR  FL
+* 0b00001111      -   -   -   -   FC  LFE FR  FL
+* 0b00010011      -   -   -   RC  -   -   FR  FL
+* 0b00010111      -   -   -   RC  -   LFE FR  FL
+* 0b00011011      -   -   -   RC  FC  -   FR  FL
+* 0b00011111      -   -   -   RC  FC  LFE FR  FL
+* 0b00110011      -   -   RR  RL  -   -   FR  FL
+* 0b00110111      -   -   RR  RL  -   LFE FR  FL
+* 0b00111011      -   -   RR  RL  FC  -   FR  FL
+* 0b00111111      -   -   RR  RL  FC  LFE FR  FL
+* 0b01110011      -   RC  RR  RL  -   -   FR  FL
+* 0b01110111      -   RC  RR  RL  -   LFE FR  FL
+* 0b01111011      -   RC  RR  RL  FC  -   FR  FL
+* 0b01111111      -   RC  RR  RL  FC  LFE FR  FL
+* 0b11110011      RRC RLC RR  RL  -   -   FR  FL
+* 0b11110111      RRC RLC RR  RL  -   LFE FR  FL
+* 0b11111011      RRC RLC RR  RL  FC  -   FR  FL
+* 0b11111111      RRC RLC RR  RL  FC  LFE FR  FL
+* 0b11000011      FRC FLC -   -   -   -   FR  FL
+* 0b11000111      FRC FLC -   -   -   LFE FR  FL
+* 0b11001011      FRC FLC -   -   FC  -   FR  FL
+* 0b11001111      FRC FLC -   -   FC  LFE FR  FL
+* 0b11010011      FRC FLC -   RC  -   -   FR  FL
+* 0b11010111      FRC FLC -   RC  -   LFE FR  FL
+* 0b11011011      FRC FLC -   RC  FC  -   FR  FL
+* 0b11011111      FRC FLC -   RC  FC  LFE FR  FL
+* 0b11110011      FRC FLC RR  RL  -   -   FR  FL
+* 0b11110111      FRC FLC RR  RL  -   LFE FR  FL
+* 0b11111011      FRC FLC RR  RL  FC  -   FR  FL
+* 0b11111111      FRC FLC RR  RL  FC  LFE FR  FL
+*
+* @param
+*  speakers - speaker information as it comes from CEA audio block
+*/
+/* translate speakers to channels */
+
+union audio_cea_channels {
+	uint8_t all;
+	struct audio_cea_channels_bits {
+		uint32_t FL:1;
+		uint32_t FR:1;
+		uint32_t LFE:1;
+		uint32_t FC:1;
+		uint32_t RL_RC:1;
+		uint32_t RR:1;
+		uint32_t RC_RLC_FLC:1;
+		uint32_t RRC_FRC:1;
+	} channels;
+};
+
+struct audio_clock_info {
+	/* pixel clock frequency*/
+	uint32_t pixel_clock_in_10khz;
+	/* N - 32KHz audio */
+	uint32_t n_32khz;
+	/* CTS - 32KHz audio*/
+	uint32_t cts_32khz;
+	uint32_t n_44khz;
+	uint32_t cts_44khz;
+	uint32_t n_48khz;
+	uint32_t cts_48khz;
+};
+
+/* 25.2MHz/1.001*/
+/* 25.2MHz/1.001*/
+/* 25.2MHz*/
+/* 27MHz */
+/* 27MHz*1.001*/
+/* 27MHz*1.001*/
+/* 54MHz*/
+/* 54MHz*1.001*/
+/* 74.25MHz/1.001*/
+/* 74.25MHz*/
+/* 148.5MHz/1.001*/
+/* 148.5MHz*/
+
+static const struct audio_clock_info audio_clock_info_table[16] = {
+	{2517, 4576, 28125, 7007, 31250, 6864, 28125},
+	{2518, 4576, 28125, 7007, 31250, 6864, 28125},
+	{2520, 4096, 25200, 6272, 28000, 6144, 25200},
+	{2700, 4096, 27000, 6272, 30000, 6144, 27000},
+	{2702, 4096, 27027, 6272, 30030, 6144, 27027},
+	{2703, 4096, 27027, 6272, 30030, 6144, 27027},
+	{5400, 4096, 54000, 6272, 60000, 6144, 54000},
+	{5405, 4096, 54054, 6272, 60060, 6144, 54054},
+	{7417, 11648, 210937, 17836, 234375, 11648, 140625},
+	{7425, 4096, 74250, 6272, 82500, 6144, 74250},
+	{14835, 11648, 421875, 8918, 234375, 5824, 140625},
+	{14850, 4096, 148500, 6272, 165000, 6144, 148500},
+	{29670, 5824, 421875, 4459, 234375, 5824, 281250},
+	{29700, 3072, 222750, 4704, 247500, 5120, 247500},
+	{59340, 5824, 843750, 8918, 937500, 5824, 562500},
+	{59400, 3072, 445500, 9408, 990000, 6144, 594000}
+};
+
+static const struct audio_clock_info audio_clock_info_table_36bpc[14] = {
+	{2517,  9152,  84375,  7007,  48875,  9152,  56250},
+	{2518,  9152,  84375,  7007,  48875,  9152,  56250},
+	{2520,  4096,  37800,  6272,  42000,  6144,  37800},
+	{2700,  4096,  40500,  6272,  45000,  6144,  40500},
+	{2702,  8192,  81081,  6272,  45045,  8192,  54054},
+	{2703,  8192,  81081,  6272,  45045,  8192,  54054},
+	{5400,  4096,  81000,  6272,  90000,  6144,  81000},
+	{5405,  4096,  81081,  6272,  90090,  6144,  81081},
+	{7417, 11648, 316406, 17836, 351562, 11648, 210937},
+	{7425, 4096, 111375,  6272, 123750,  6144, 111375},
+	{14835, 11648, 632812, 17836, 703125, 11648, 421875},
+	{14850, 4096, 222750,  6272, 247500,  6144, 222750},
+	{29670, 5824, 632812,  8918, 703125,  5824, 421875},
+	{29700, 4096, 445500,  4704, 371250,  5120, 371250}
+};
+
+static const struct audio_clock_info audio_clock_info_table_48bpc[14] = {
+	{2517,  4576,  56250,  7007,  62500,  6864,  56250},
+	{2518,  4576,  56250,  7007,  62500,  6864,  56250},
+	{2520,  4096,  50400,  6272,  56000,  6144,  50400},
+	{2700,  4096,  54000,  6272,  60000,  6144,  54000},
+	{2702,  4096,  54054,  6267,  60060,  8192,  54054},
+	{2703,  4096,  54054,  6272,  60060,  8192,  54054},
+	{5400,  4096, 108000,  6272, 120000,  6144, 108000},
+	{5405,  4096, 108108,  6272, 120120,  6144, 108108},
+	{7417, 11648, 421875, 17836, 468750, 11648, 281250},
+	{7425,  4096, 148500,  6272, 165000,  6144, 148500},
+	{14835, 11648, 843750,  8918, 468750, 11648, 281250},
+	{14850, 4096, 297000,  6272, 330000,  6144, 297000},
+	{29670, 5824, 843750,  4459, 468750,  5824, 562500},
+	{29700, 3072, 445500,  4704, 495000,  5120, 495000}
+
+
+};
+
+static union audio_cea_channels speakers_to_channels(
+	struct audio_speaker_flags speaker_flags)
+{
+	union audio_cea_channels cea_channels = {0};
+
+	/* these are one to one */
+	cea_channels.channels.FL = speaker_flags.FL_FR;
+	cea_channels.channels.FR = speaker_flags.FL_FR;
+	cea_channels.channels.LFE = speaker_flags.LFE;
+	cea_channels.channels.FC = speaker_flags.FC;
+
+	/* if Rear Left and Right exist move RC speaker to channel 7
+	 * otherwise to channel 5
+	 */
+	if (speaker_flags.RL_RR) {
+		cea_channels.channels.RL_RC = speaker_flags.RL_RR;
+		cea_channels.channels.RR = speaker_flags.RL_RR;
+		cea_channels.channels.RC_RLC_FLC = speaker_flags.RC;
+	} else {
+		cea_channels.channels.RL_RC = speaker_flags.RC;
+	}
+
+	/* FRONT Left Right Center and REAR Left Right Center are exclusive */
+	if (speaker_flags.FLC_FRC) {
+		cea_channels.channels.RC_RLC_FLC = speaker_flags.FLC_FRC;
+		cea_channels.channels.RRC_FRC = speaker_flags.FLC_FRC;
+	} else {
+		cea_channels.channels.RC_RLC_FLC = speaker_flags.RLC_RRC;
+		cea_channels.channels.RRC_FRC = speaker_flags.RLC_RRC;
+	}
+
+	return cea_channels;
+}
+
+static uint32_t calc_max_audio_packets_per_line(
+	const struct audio_crtc_info *crtc_info)
+{
+	uint32_t max_packets_per_line;
+
+	max_packets_per_line =
+		crtc_info->h_total - crtc_info->h_active;
+
+	if (crtc_info->pixel_repetition)
+		max_packets_per_line *= crtc_info->pixel_repetition;
+
+	/* for other hdmi features */
+	max_packets_per_line -= 58;
+	/* for Control Period */
+	max_packets_per_line -= 16;
+	/* Number of Audio Packets per Line */
+	max_packets_per_line /= 32;
+
+	return max_packets_per_line;
+}
+
+static void get_audio_clock_info(
+	enum dc_color_depth color_depth,
+	uint32_t crtc_pixel_clock_in_khz,
+	uint32_t actual_pixel_clock_in_khz,
+	struct audio_clock_info *audio_clock_info)
+{
+	const struct audio_clock_info *clock_info;
+	uint32_t index;
+	uint32_t crtc_pixel_clock_in_10khz = crtc_pixel_clock_in_khz / 10;
+	uint32_t audio_array_size;
+
+	switch (color_depth) {
+	case COLOR_DEPTH_161616:
+		clock_info = audio_clock_info_table_48bpc;
+		audio_array_size = ARRAY_SIZE(
+				audio_clock_info_table_48bpc);
+		break;
+	case COLOR_DEPTH_121212:
+		clock_info = audio_clock_info_table_36bpc;
+		audio_array_size = ARRAY_SIZE(
+				audio_clock_info_table_36bpc);
+		break;
+	default:
+		clock_info = audio_clock_info_table;
+		audio_array_size = ARRAY_SIZE(
+				audio_clock_info_table);
+		break;
+	}
+
+	if (clock_info != NULL) {
+		/* search for exact pixel clock in table */
+		for (index = 0; index < audio_array_size; index++) {
+			if (clock_info[index].pixel_clock_in_10khz >
+				crtc_pixel_clock_in_10khz)
+				break;  /* not match */
+			else if (clock_info[index].pixel_clock_in_10khz ==
+					crtc_pixel_clock_in_10khz) {
+				/* match found */
+				*audio_clock_info = clock_info[index];
+				return;
+			}
+		}
+	}
+
+	/* not found */
+	if (actual_pixel_clock_in_khz == 0)
+		actual_pixel_clock_in_khz = crtc_pixel_clock_in_khz;
+
+	/* See HDMI spec  the table entry under
+	 *  pixel clock of "Other". */
+	audio_clock_info->pixel_clock_in_10khz =
+			actual_pixel_clock_in_khz / 10;
+	audio_clock_info->cts_32khz = actual_pixel_clock_in_khz;
+	audio_clock_info->cts_44khz = actual_pixel_clock_in_khz;
+	audio_clock_info->cts_48khz = actual_pixel_clock_in_khz;
+
+	audio_clock_info->n_32khz = 4096;
+	audio_clock_info->n_44khz = 6272;
+	audio_clock_info->n_48khz = 6144;
+}
+
+static void enc1_se_audio_setup(
+	struct stream_encoder *enc,
+	unsigned int az_inst,
+	struct audio_info *audio_info)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	uint32_t speakers = 0;
+	uint32_t channels = 0;
+
+	ASSERT(audio_info);
+	if (audio_info == NULL)
+		/* This should not happen.it does so we don't get BSOD*/
+		return;
+
+	speakers = audio_info->flags.info.ALLSPEAKERS;
+	channels = speakers_to_channels(audio_info->flags.speaker_flags).all;
+
+	/* setup the audio stream source select (audio -> dig mapping) */
+	REG_SET(AFMT_AUDIO_SRC_CONTROL, 0, AFMT_AUDIO_SRC_SELECT, az_inst);
+
+	/* Channel allocation */
+	REG_UPDATE(AFMT_AUDIO_PACKET_CONTROL2, AFMT_AUDIO_CHANNEL_ENABLE, channels);
+}
+
+static void enc1_se_setup_hdmi_audio(
+	struct stream_encoder *enc,
+	const struct audio_crtc_info *crtc_info)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	struct audio_clock_info audio_clock_info = {0};
+	uint32_t max_packets_per_line;
+
+	/* For now still do calculation, although this field is ignored when
+	 * above HDMI_PACKET_GEN_VERSION set to 1
+	 */
+	max_packets_per_line = calc_max_audio_packets_per_line(crtc_info);
+
+	/* HDMI_AUDIO_PACKET_CONTROL */
+	REG_UPDATE_2(HDMI_AUDIO_PACKET_CONTROL,
+			HDMI_AUDIO_PACKETS_PER_LINE, max_packets_per_line,
+			HDMI_AUDIO_DELAY_EN, 1);
+
+	/* AFMT_AUDIO_PACKET_CONTROL */
+	REG_UPDATE(AFMT_AUDIO_PACKET_CONTROL, AFMT_60958_CS_UPDATE, 1);
+
+	/* AFMT_AUDIO_PACKET_CONTROL2 */
+	REG_UPDATE_2(AFMT_AUDIO_PACKET_CONTROL2,
+			AFMT_AUDIO_LAYOUT_OVRD, 0,
+			AFMT_60958_OSF_OVRD, 0);
+
+	/* HDMI_ACR_PACKET_CONTROL */
+	REG_UPDATE_3(HDMI_ACR_PACKET_CONTROL,
+			HDMI_ACR_AUTO_SEND, 1,
+			HDMI_ACR_SOURCE, 0,
+			HDMI_ACR_AUDIO_PRIORITY, 0);
+
+	/* Program audio clock sample/regeneration parameters */
+	get_audio_clock_info(crtc_info->color_depth,
+			     crtc_info->requested_pixel_clock,
+			     crtc_info->calculated_pixel_clock,
+			     &audio_clock_info);
+	DC_LOG_HW_AUDIO(
+			"\n%s:Input::requested_pixel_clock = %d"	\
+			"calculated_pixel_clock = %d \n", __func__,	\
+			crtc_info->requested_pixel_clock,		\
+			crtc_info->calculated_pixel_clock);
+
+	/* HDMI_ACR_32_0__HDMI_ACR_CTS_32_MASK */
+	REG_UPDATE(HDMI_ACR_32_0, HDMI_ACR_CTS_32, audio_clock_info.cts_32khz);
+
+	/* HDMI_ACR_32_1__HDMI_ACR_N_32_MASK */
+	REG_UPDATE(HDMI_ACR_32_1, HDMI_ACR_N_32, audio_clock_info.n_32khz);
+
+	/* HDMI_ACR_44_0__HDMI_ACR_CTS_44_MASK */
+	REG_UPDATE(HDMI_ACR_44_0, HDMI_ACR_CTS_44, audio_clock_info.cts_44khz);
+
+	/* HDMI_ACR_44_1__HDMI_ACR_N_44_MASK */
+	REG_UPDATE(HDMI_ACR_44_1, HDMI_ACR_N_44, audio_clock_info.n_44khz);
+
+	/* HDMI_ACR_48_0__HDMI_ACR_CTS_48_MASK */
+	REG_UPDATE(HDMI_ACR_48_0, HDMI_ACR_CTS_48, audio_clock_info.cts_48khz);
+
+	/* HDMI_ACR_48_1__HDMI_ACR_N_48_MASK */
+	REG_UPDATE(HDMI_ACR_48_1, HDMI_ACR_N_48, audio_clock_info.n_48khz);
+
+	/* Video driver cannot know in advance which sample rate will
+	 * be used by HD Audio driver
+	 * HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE field is
+	 * programmed below in interruppt callback
+	 */
+
+	/* AFMT_60958_0__AFMT_60958_CS_CHANNEL_NUMBER_L_MASK &
+	 * AFMT_60958_0__AFMT_60958_CS_CLOCK_ACCURACY_MASK
+	 */
+	REG_UPDATE_2(AFMT_60958_0,
+			AFMT_60958_CS_CHANNEL_NUMBER_L, 1,
+			AFMT_60958_CS_CLOCK_ACCURACY, 0);
+
+	/* AFMT_60958_1 AFMT_60958_CS_CHALNNEL_NUMBER_R */
+	REG_UPDATE(AFMT_60958_1, AFMT_60958_CS_CHANNEL_NUMBER_R, 2);
+
+	/* AFMT_60958_2 now keep this settings until
+	 * Programming guide comes out
+	 */
+	REG_UPDATE_6(AFMT_60958_2,
+			AFMT_60958_CS_CHANNEL_NUMBER_2, 3,
+			AFMT_60958_CS_CHANNEL_NUMBER_3, 4,
+			AFMT_60958_CS_CHANNEL_NUMBER_4, 5,
+			AFMT_60958_CS_CHANNEL_NUMBER_5, 6,
+			AFMT_60958_CS_CHANNEL_NUMBER_6, 7,
+			AFMT_60958_CS_CHANNEL_NUMBER_7, 8);
+}
+
+static void enc1_se_setup_dp_audio(
+	struct stream_encoder *enc)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	/* --- DP Audio packet configurations --- */
+
+	/* ATP Configuration */
+	REG_SET(DP_SEC_AUD_N, 0,
+			DP_SEC_AUD_N, DP_SEC_AUD_N__DP_SEC_AUD_N__DEFAULT);
+
+	/* Async/auto-calc timestamp mode */
+	REG_SET(DP_SEC_TIMESTAMP, 0, DP_SEC_TIMESTAMP_MODE,
+			DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE__AUTO_CALC);
+
+	/* --- The following are the registers
+	 *  copied from the SetupHDMI ---
+	 */
+
+	/* AFMT_AUDIO_PACKET_CONTROL */
+	REG_UPDATE(AFMT_AUDIO_PACKET_CONTROL, AFMT_60958_CS_UPDATE, 1);
+
+	/* AFMT_AUDIO_PACKET_CONTROL2 */
+	/* Program the ATP and AIP next */
+	REG_UPDATE_2(AFMT_AUDIO_PACKET_CONTROL2,
+			AFMT_AUDIO_LAYOUT_OVRD, 0,
+			AFMT_60958_OSF_OVRD, 0);
+
+	/* AFMT_INFOFRAME_CONTROL0 */
+	REG_UPDATE(AFMT_INFOFRAME_CONTROL0, AFMT_AUDIO_INFO_UPDATE, 1);
+
+	/* AFMT_60958_0__AFMT_60958_CS_CLOCK_ACCURACY_MASK */
+	REG_UPDATE(AFMT_60958_0, AFMT_60958_CS_CLOCK_ACCURACY, 0);
+}
+
+static void enc1_se_enable_audio_clock(
+	struct stream_encoder *enc,
+	bool enable)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	if (REG(AFMT_CNTL) == 0)
+		return;   /* DCE8/10 does not have this register */
+
+	REG_UPDATE(AFMT_CNTL, AFMT_AUDIO_CLOCK_EN, !!enable);
+
+	/* wait for AFMT clock to turn on,
+	 * expectation: this should complete in 1-2 reads
+	 *
+	 * REG_WAIT(AFMT_CNTL, AFMT_AUDIO_CLOCK_ON, !!enable, 1, 10);
+	 *
+	 * TODO: wait for clock_on does not work well. May need HW
+	 * program sequence. But audio seems work normally even without wait
+	 * for clock_on status change
+	 */
+}
+
+static void enc1_se_enable_dp_audio(
+	struct stream_encoder *enc)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	/* Enable Audio packets */
+	REG_UPDATE(DP_SEC_CNTL, DP_SEC_ASP_ENABLE, 1);
+
+	/* Program the ATP and AIP next */
+	REG_UPDATE_2(DP_SEC_CNTL,
+			DP_SEC_ATP_ENABLE, 1,
+			DP_SEC_AIP_ENABLE, 1);
+
+	/* Program STREAM_ENABLE after all the other enables. */
+	REG_UPDATE(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, 1);
+}
+
+static void enc1_se_disable_dp_audio(
+	struct stream_encoder *enc)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	uint32_t value = 0;
+
+	/* Disable Audio packets */
+	REG_UPDATE_5(DP_SEC_CNTL,
+			DP_SEC_ASP_ENABLE, 0,
+			DP_SEC_ATP_ENABLE, 0,
+			DP_SEC_AIP_ENABLE, 0,
+			DP_SEC_ACM_ENABLE, 0,
+			DP_SEC_STREAM_ENABLE, 0);
+
+	/* This register shared with encoder info frame. Therefore we need to
+	 * keep master enabled if at least on of the fields is not 0
+	 */
+	value = REG_READ(DP_SEC_CNTL);
+	if (value != 0)
+		REG_UPDATE(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, 1);
+
+}
+
+void enc1_se_audio_mute_control(
+	struct stream_encoder *enc,
+	bool mute)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	REG_UPDATE(AFMT_AUDIO_PACKET_CONTROL, AFMT_AUDIO_SAMPLE_SEND, !mute);
+}
+
+void enc1_se_dp_audio_setup(
+	struct stream_encoder *enc,
+	unsigned int az_inst,
+	struct audio_info *info)
+{
+	enc1_se_audio_setup(enc, az_inst, info);
+}
+
+void enc1_se_dp_audio_enable(
+	struct stream_encoder *enc)
+{
+	enc1_se_enable_audio_clock(enc, true);
+	enc1_se_setup_dp_audio(enc);
+	enc1_se_enable_dp_audio(enc);
+}
+
+void enc1_se_dp_audio_disable(
+	struct stream_encoder *enc)
+{
+	enc1_se_disable_dp_audio(enc);
+	enc1_se_enable_audio_clock(enc, false);
+}
+
+void enc1_se_hdmi_audio_setup(
+	struct stream_encoder *enc,
+	unsigned int az_inst,
+	struct audio_info *info,
+	struct audio_crtc_info *audio_crtc_info)
+{
+	enc1_se_enable_audio_clock(enc, true);
+	enc1_se_setup_hdmi_audio(enc, audio_crtc_info);
+	enc1_se_audio_setup(enc, az_inst, info);
+}
+
+void enc1_se_hdmi_audio_disable(
+	struct stream_encoder *enc)
+{
+	enc1_se_enable_audio_clock(enc, false);
+}
+
+
+void enc1_setup_stereo_sync(
+	struct stream_encoder *enc,
+	int tg_inst, bool enable)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	REG_UPDATE(DIG_FE_CNTL, DIG_STEREOSYNC_SELECT, tg_inst);
+	REG_UPDATE(DIG_FE_CNTL, DIG_STEREOSYNC_GATE_EN, !enable);
+}
+
+
+static const struct stream_encoder_funcs dcn10_str_enc_funcs = {
+	.dp_set_stream_attribute =
+		enc1_stream_encoder_dp_set_stream_attribute,
+	.hdmi_set_stream_attribute =
+		enc1_stream_encoder_hdmi_set_stream_attribute,
+	.dvi_set_stream_attribute =
+		enc1_stream_encoder_dvi_set_stream_attribute,
+	.set_mst_bandwidth =
+		enc1_stream_encoder_set_mst_bandwidth,
+	.update_hdmi_info_packets =
+		enc1_stream_encoder_update_hdmi_info_packets,
+	.stop_hdmi_info_packets =
+		enc1_stream_encoder_stop_hdmi_info_packets,
+	.update_dp_info_packets =
+		enc1_stream_encoder_update_dp_info_packets,
+	.stop_dp_info_packets =
+		enc1_stream_encoder_stop_dp_info_packets,
+	.dp_blank =
+		enc1_stream_encoder_dp_blank,
+	.dp_unblank =
+		enc1_stream_encoder_dp_unblank,
+	.audio_mute_control = enc1_se_audio_mute_control,
+
+	.dp_audio_setup = enc1_se_dp_audio_setup,
+	.dp_audio_enable = enc1_se_dp_audio_enable,
+	.dp_audio_disable = enc1_se_dp_audio_disable,
+
+	.hdmi_audio_setup = enc1_se_hdmi_audio_setup,
+	.hdmi_audio_disable = enc1_se_hdmi_audio_disable,
+	.setup_stereo_sync  = enc1_setup_stereo_sync,
+	.set_avmute = enc1_stream_encoder_set_avmute,
+};
+
+void dcn10_stream_encoder_construct(
+	struct dcn10_stream_encoder *enc1,
+	struct dc_context *ctx,
+	struct dc_bios *bp,
+	enum engine_id eng_id,
+	const struct dcn10_stream_enc_registers *regs,
+	const struct dcn10_stream_encoder_shift *se_shift,
+	const struct dcn10_stream_encoder_mask *se_mask)
+{
+	enc1->base.funcs = &dcn10_str_enc_funcs;
+	enc1->base.ctx = ctx;
+	enc1->base.id = eng_id;
+	enc1->base.bp = bp;
+	enc1->regs = regs;
+	enc1->se_shift = se_shift;
+	enc1->se_mask = se_mask;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
new file mode 100644
index 000000000000..6b3e4ded155b
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
@@ -0,0 +1,524 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_STREAM_ENCODER_DCN10_H__
+#define __DC_STREAM_ENCODER_DCN10_H__
+
+#include "stream_encoder.h"
+
+#define DCN10STRENC_FROM_STRENC(stream_encoder)\
+	container_of(stream_encoder, struct dcn10_stream_encoder, base)
+
+#define SE_COMMON_DCN_REG_LIST(id) \
+	SRI(AFMT_CNTL, DIG, id), \
+	SRI(AFMT_GENERIC_0, DIG, id), \
+	SRI(AFMT_GENERIC_1, DIG, id), \
+	SRI(AFMT_GENERIC_2, DIG, id), \
+	SRI(AFMT_GENERIC_3, DIG, id), \
+	SRI(AFMT_GENERIC_4, DIG, id), \
+	SRI(AFMT_GENERIC_5, DIG, id), \
+	SRI(AFMT_GENERIC_6, DIG, id), \
+	SRI(AFMT_GENERIC_7, DIG, id), \
+	SRI(AFMT_GENERIC_HDR, DIG, id), \
+	SRI(AFMT_INFOFRAME_CONTROL0, DIG, id), \
+	SRI(AFMT_VBI_PACKET_CONTROL, DIG, id), \
+	SRI(AFMT_VBI_PACKET_CONTROL1, DIG, id), \
+	SRI(AFMT_AUDIO_PACKET_CONTROL, DIG, id), \
+	SRI(AFMT_AUDIO_PACKET_CONTROL2, DIG, id), \
+	SRI(AFMT_AUDIO_SRC_CONTROL, DIG, id), \
+	SRI(AFMT_60958_0, DIG, id), \
+	SRI(AFMT_60958_1, DIG, id), \
+	SRI(AFMT_60958_2, DIG, id), \
+	SRI(DIG_FE_CNTL, DIG, id), \
+	SRI(HDMI_CONTROL, DIG, id), \
+	SRI(HDMI_DB_CONTROL, DIG, id), \
+	SRI(HDMI_GC, DIG, id), \
+	SRI(HDMI_GENERIC_PACKET_CONTROL0, DIG, id), \
+	SRI(HDMI_GENERIC_PACKET_CONTROL1, DIG, id), \
+	SRI(HDMI_GENERIC_PACKET_CONTROL2, DIG, id), \
+	SRI(HDMI_GENERIC_PACKET_CONTROL3, DIG, id), \
+	SRI(HDMI_INFOFRAME_CONTROL0, DIG, id), \
+	SRI(HDMI_INFOFRAME_CONTROL1, DIG, id), \
+	SRI(HDMI_VBI_PACKET_CONTROL, DIG, id), \
+	SRI(HDMI_AUDIO_PACKET_CONTROL, DIG, id),\
+	SRI(HDMI_ACR_PACKET_CONTROL, DIG, id),\
+	SRI(HDMI_ACR_32_0, DIG, id),\
+	SRI(HDMI_ACR_32_1, DIG, id),\
+	SRI(HDMI_ACR_44_0, DIG, id),\
+	SRI(HDMI_ACR_44_1, DIG, id),\
+	SRI(HDMI_ACR_48_0, DIG, id),\
+	SRI(HDMI_ACR_48_1, DIG, id),\
+	SRI(DP_DB_CNTL, DP, id), \
+	SRI(DP_MSA_MISC, DP, id), \
+	SRI(DP_MSA_COLORIMETRY, DP, id), \
+	SRI(DP_MSA_TIMING_PARAM1, DP, id), \
+	SRI(DP_MSA_TIMING_PARAM2, DP, id), \
+	SRI(DP_MSA_TIMING_PARAM3, DP, id), \
+	SRI(DP_MSA_TIMING_PARAM4, DP, id), \
+	SRI(DP_MSE_RATE_CNTL, DP, id), \
+	SRI(DP_MSE_RATE_UPDATE, DP, id), \
+	SRI(DP_PIXEL_FORMAT, DP, id), \
+	SRI(DP_SEC_CNTL, DP, id), \
+	SRI(DP_STEER_FIFO, DP, id), \
+	SRI(DP_VID_M, DP, id), \
+	SRI(DP_VID_N, DP, id), \
+	SRI(DP_VID_STREAM_CNTL, DP, id), \
+	SRI(DP_VID_TIMING, DP, id), \
+	SRI(DP_SEC_AUD_N, DP, id), \
+	SRI(DP_SEC_TIMESTAMP, DP, id)
+
+#define SE_DCN_REG_LIST(id)\
+	SE_COMMON_DCN_REG_LIST(id)
+
+
+struct dcn10_stream_enc_registers {
+	uint32_t AFMT_CNTL;
+	uint32_t AFMT_AVI_INFO0;
+	uint32_t AFMT_AVI_INFO1;
+	uint32_t AFMT_AVI_INFO2;
+	uint32_t AFMT_AVI_INFO3;
+	uint32_t AFMT_GENERIC_0;
+	uint32_t AFMT_GENERIC_1;
+	uint32_t AFMT_GENERIC_2;
+	uint32_t AFMT_GENERIC_3;
+	uint32_t AFMT_GENERIC_4;
+	uint32_t AFMT_GENERIC_5;
+	uint32_t AFMT_GENERIC_6;
+	uint32_t AFMT_GENERIC_7;
+	uint32_t AFMT_GENERIC_HDR;
+	uint32_t AFMT_INFOFRAME_CONTROL0;
+	uint32_t AFMT_VBI_PACKET_CONTROL;
+	uint32_t AFMT_VBI_PACKET_CONTROL1;
+	uint32_t AFMT_AUDIO_PACKET_CONTROL;
+	uint32_t AFMT_AUDIO_PACKET_CONTROL2;
+	uint32_t AFMT_AUDIO_SRC_CONTROL;
+	uint32_t AFMT_60958_0;
+	uint32_t AFMT_60958_1;
+	uint32_t AFMT_60958_2;
+	uint32_t DIG_FE_CNTL;
+	uint32_t DP_MSE_RATE_CNTL;
+	uint32_t DP_MSE_RATE_UPDATE;
+	uint32_t DP_PIXEL_FORMAT;
+	uint32_t DP_SEC_CNTL;
+	uint32_t DP_STEER_FIFO;
+	uint32_t DP_VID_M;
+	uint32_t DP_VID_N;
+	uint32_t DP_VID_STREAM_CNTL;
+	uint32_t DP_VID_TIMING;
+	uint32_t DP_SEC_AUD_N;
+	uint32_t DP_SEC_TIMESTAMP;
+	uint32_t HDMI_CONTROL;
+	uint32_t HDMI_GC;
+	uint32_t HDMI_GENERIC_PACKET_CONTROL0;
+	uint32_t HDMI_GENERIC_PACKET_CONTROL1;
+	uint32_t HDMI_GENERIC_PACKET_CONTROL2;
+	uint32_t HDMI_GENERIC_PACKET_CONTROL3;
+	uint32_t HDMI_GENERIC_PACKET_CONTROL4;
+	uint32_t HDMI_GENERIC_PACKET_CONTROL5;
+	uint32_t HDMI_INFOFRAME_CONTROL0;
+	uint32_t HDMI_INFOFRAME_CONTROL1;
+	uint32_t HDMI_VBI_PACKET_CONTROL;
+	uint32_t HDMI_AUDIO_PACKET_CONTROL;
+	uint32_t HDMI_ACR_PACKET_CONTROL;
+	uint32_t HDMI_ACR_32_0;
+	uint32_t HDMI_ACR_32_1;
+	uint32_t HDMI_ACR_44_0;
+	uint32_t HDMI_ACR_44_1;
+	uint32_t HDMI_ACR_48_0;
+	uint32_t HDMI_ACR_48_1;
+	uint32_t DP_DB_CNTL;
+	uint32_t DP_MSA_MISC;
+	uint32_t DP_MSA_COLORIMETRY;
+	uint32_t DP_MSA_TIMING_PARAM1;
+	uint32_t DP_MSA_TIMING_PARAM2;
+	uint32_t DP_MSA_TIMING_PARAM3;
+	uint32_t DP_MSA_TIMING_PARAM4;
+	uint32_t HDMI_DB_CONTROL;
+};
+
+
+#define SE_SF(reg_name, field_name, post_fix)\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define SE_COMMON_MASK_SH_LIST_SOC_BASE(mask_sh)\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_INDEX, mask_sh),\
+	SE_SF(DIG0_AFMT_GENERIC_HDR, AFMT_GENERIC_HB0, mask_sh),\
+	SE_SF(DIG0_AFMT_GENERIC_HDR, AFMT_GENERIC_HB1, mask_sh),\
+	SE_SF(DIG0_AFMT_GENERIC_HDR, AFMT_GENERIC_HB2, mask_sh),\
+	SE_SF(DIG0_AFMT_GENERIC_HDR, AFMT_GENERIC_HB3, mask_sh),\
+	SE_SF(DP0_DP_PIXEL_FORMAT, DP_PIXEL_ENCODING, mask_sh),\
+	SE_SF(DP0_DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, HDMI_PACKET_GEN_VERSION, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, HDMI_KEEPOUT_MODE, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, HDMI_DEEP_COLOR_ENABLE, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, HDMI_DEEP_COLOR_DEPTH, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, HDMI_DATA_SCRAMBLE_EN, mask_sh),\
+	SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_GC_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_GC_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_NULL_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_INFOFRAME_CONTROL0, HDMI_AUDIO_INFO_SEND, mask_sh),\
+	SE_SF(DIG0_AFMT_INFOFRAME_CONTROL0, AFMT_AUDIO_INFO_UPDATE, mask_sh),\
+	SE_SF(DIG0_HDMI_INFOFRAME_CONTROL1, HDMI_AUDIO_INFO_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GC, HDMI_GC_AVMUTE, mask_sh),\
+	SE_SF(DP0_DP_MSE_RATE_CNTL, DP_MSE_RATE_X, mask_sh),\
+	SE_SF(DP0_DP_MSE_RATE_CNTL, DP_MSE_RATE_Y, mask_sh),\
+	SE_SF(DP0_DP_MSE_RATE_UPDATE, DP_MSE_RATE_UPDATE_PENDING, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP0_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP1_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP2_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP3_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_MPG_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_VID_STREAM_CNTL, DP_VID_STREAM_DIS_DEFER, mask_sh),\
+	SE_SF(DP0_DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_VID_STREAM_CNTL, DP_VID_STREAM_STATUS, mask_sh),\
+	SE_SF(DP0_DP_STEER_FIFO, DP_STEER_FIFO_RESET, mask_sh),\
+	SE_SF(DP0_DP_VID_TIMING, DP_VID_M_N_GEN_EN, mask_sh),\
+	SE_SF(DP0_DP_VID_N, DP_VID_N, mask_sh),\
+	SE_SF(DP0_DP_VID_M, DP_VID_M, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CNTL, DIG_START, mask_sh),\
+	SE_SF(DIG0_AFMT_AUDIO_SRC_CONTROL, AFMT_AUDIO_SRC_SELECT, mask_sh),\
+	SE_SF(DIG0_AFMT_AUDIO_PACKET_CONTROL2, AFMT_AUDIO_CHANNEL_ENABLE, mask_sh),\
+	SE_SF(DIG0_HDMI_AUDIO_PACKET_CONTROL, HDMI_AUDIO_PACKETS_PER_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_AUDIO_PACKET_CONTROL, HDMI_AUDIO_DELAY_EN, mask_sh),\
+	SE_SF(DIG0_AFMT_AUDIO_PACKET_CONTROL, AFMT_60958_CS_UPDATE, mask_sh),\
+	SE_SF(DIG0_AFMT_AUDIO_PACKET_CONTROL2, AFMT_AUDIO_LAYOUT_OVRD, mask_sh),\
+	SE_SF(DIG0_AFMT_AUDIO_PACKET_CONTROL2, AFMT_60958_OSF_OVRD, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_PACKET_CONTROL, HDMI_ACR_AUTO_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_PACKET_CONTROL, HDMI_ACR_SOURCE, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_PACKET_CONTROL, HDMI_ACR_AUDIO_PRIORITY, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_32_0, HDMI_ACR_CTS_32, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_32_1, HDMI_ACR_N_32, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_44_0, HDMI_ACR_CTS_44, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_44_1, HDMI_ACR_N_44, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_48_0, HDMI_ACR_CTS_48, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_48_1, HDMI_ACR_N_48, mask_sh),\
+	SE_SF(DIG0_AFMT_60958_0, AFMT_60958_CS_CHANNEL_NUMBER_L, mask_sh),\
+	SE_SF(DIG0_AFMT_60958_0, AFMT_60958_CS_CLOCK_ACCURACY, mask_sh),\
+	SE_SF(DIG0_AFMT_60958_1, AFMT_60958_CS_CHANNEL_NUMBER_R, mask_sh),\
+	SE_SF(DIG0_AFMT_60958_2, AFMT_60958_CS_CHANNEL_NUMBER_2, mask_sh),\
+	SE_SF(DIG0_AFMT_60958_2, AFMT_60958_CS_CHANNEL_NUMBER_3, mask_sh),\
+	SE_SF(DIG0_AFMT_60958_2, AFMT_60958_CS_CHANNEL_NUMBER_4, mask_sh),\
+	SE_SF(DIG0_AFMT_60958_2, AFMT_60958_CS_CHANNEL_NUMBER_5, mask_sh),\
+	SE_SF(DIG0_AFMT_60958_2, AFMT_60958_CS_CHANNEL_NUMBER_6, mask_sh),\
+	SE_SF(DIG0_AFMT_60958_2, AFMT_60958_CS_CHANNEL_NUMBER_7, mask_sh),\
+	SE_SF(DP0_DP_SEC_AUD_N, DP_SEC_AUD_N, mask_sh),\
+	SE_SF(DP0_DP_SEC_TIMESTAMP, DP_SEC_TIMESTAMP_MODE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_ASP_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_ATP_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_AIP_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_ACM_ENABLE, mask_sh),\
+	SE_SF(DIG0_AFMT_AUDIO_PACKET_CONTROL, AFMT_AUDIO_SAMPLE_SEND, mask_sh),\
+	SE_SF(DIG0_AFMT_CNTL, AFMT_AUDIO_CLOCK_EN, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, HDMI_CLOCK_CHANNEL_RATE, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CNTL, TMDS_PIXEL_ENCODING, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CNTL, TMDS_COLOR_FORMAT, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CNTL, DIG_STEREOSYNC_SELECT, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CNTL, DIG_STEREOSYNC_GATE_EN, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_LOCK_STATUS, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_CONFLICT, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_CONFLICT_CLR, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC0_FRAME_UPDATE_PENDING, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC1_FRAME_UPDATE_PENDING, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC2_FRAME_UPDATE_PENDING, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC3_FRAME_UPDATE_PENDING, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC4_FRAME_UPDATE_PENDING, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC5_FRAME_UPDATE_PENDING, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC6_FRAME_UPDATE_PENDING, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC7_FRAME_UPDATE_PENDING, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC0_FRAME_UPDATE, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC1_FRAME_UPDATE, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC2_FRAME_UPDATE, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC3_FRAME_UPDATE, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC4_FRAME_UPDATE, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC5_FRAME_UPDATE, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC6_FRAME_UPDATE, mask_sh),\
+	SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC7_FRAME_UPDATE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP4_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP5_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP6_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP7_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_DB_CNTL, DP_DB_DISABLE, mask_sh),\
+	SE_SF(DP0_DP_MSA_COLORIMETRY, DP_MSA_MISC0, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM1, DP_MSA_HTOTAL, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM1, DP_MSA_VTOTAL, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM2, DP_MSA_HSTART, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM2, DP_MSA_VSTART, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM3, DP_MSA_HSYNCWIDTH, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM3, DP_MSA_HSYNCPOLARITY, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM3, DP_MSA_VSYNCWIDTH, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM3, DP_MSA_VSYNCPOLARITY, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM4, DP_MSA_HWIDTH, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM4, DP_MSA_VHEIGHT, mask_sh),\
+	SE_SF(DIG0_HDMI_DB_CONTROL, HDMI_DB_DISABLE, mask_sh),\
+	SE_SF(DP0_DP_VID_TIMING, DP_VID_N_MUL, mask_sh)
+
+#define SE_COMMON_MASK_SH_LIST_SOC(mask_sh)\
+	SE_COMMON_MASK_SH_LIST_SOC_BASE(mask_sh)
+
+#define SE_COMMON_MASK_SH_LIST_DCN10(mask_sh)\
+	SE_COMMON_MASK_SH_LIST_SOC(mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC0_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC0_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC0_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC1_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC1_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC1_LINE, mask_sh)
+
+
+#define SE_REG_FIELD_LIST_DCN1_0(type) \
+	type AFMT_GENERIC_INDEX;\
+	type AFMT_GENERIC_HB0;\
+	type AFMT_GENERIC_HB1;\
+	type AFMT_GENERIC_HB2;\
+	type AFMT_GENERIC_HB3;\
+	type AFMT_GENERIC_LOCK_STATUS;\
+	type AFMT_GENERIC_CONFLICT;\
+	type AFMT_GENERIC_CONFLICT_CLR;\
+	type AFMT_GENERIC0_FRAME_UPDATE_PENDING;\
+	type AFMT_GENERIC1_FRAME_UPDATE_PENDING;\
+	type AFMT_GENERIC2_FRAME_UPDATE_PENDING;\
+	type AFMT_GENERIC3_FRAME_UPDATE_PENDING;\
+	type AFMT_GENERIC4_FRAME_UPDATE_PENDING;\
+	type AFMT_GENERIC5_FRAME_UPDATE_PENDING;\
+	type AFMT_GENERIC6_FRAME_UPDATE_PENDING;\
+	type AFMT_GENERIC7_FRAME_UPDATE_PENDING;\
+	type AFMT_GENERIC0_FRAME_UPDATE;\
+	type AFMT_GENERIC1_FRAME_UPDATE;\
+	type AFMT_GENERIC2_FRAME_UPDATE;\
+	type AFMT_GENERIC3_FRAME_UPDATE;\
+	type AFMT_GENERIC4_FRAME_UPDATE;\
+	type AFMT_GENERIC5_FRAME_UPDATE;\
+	type AFMT_GENERIC6_FRAME_UPDATE;\
+	type AFMT_GENERIC7_FRAME_UPDATE;\
+	type HDMI_GENERIC0_CONT;\
+	type HDMI_GENERIC0_SEND;\
+	type HDMI_GENERIC0_LINE;\
+	type HDMI_GENERIC1_CONT;\
+	type HDMI_GENERIC1_SEND;\
+	type HDMI_GENERIC1_LINE;\
+	type HDMI_GENERIC2_CONT;\
+	type HDMI_GENERIC2_SEND;\
+	type HDMI_GENERIC2_LINE;\
+	type HDMI_GENERIC3_CONT;\
+	type HDMI_GENERIC3_SEND;\
+	type HDMI_GENERIC3_LINE;\
+	type HDMI_GENERIC4_CONT;\
+	type HDMI_GENERIC4_SEND;\
+	type HDMI_GENERIC4_LINE;\
+	type HDMI_GENERIC5_CONT;\
+	type HDMI_GENERIC5_SEND;\
+	type HDMI_GENERIC5_LINE;\
+	type HDMI_GENERIC6_CONT;\
+	type HDMI_GENERIC6_SEND;\
+	type HDMI_GENERIC6_LINE;\
+	type HDMI_GENERIC7_CONT;\
+	type HDMI_GENERIC7_SEND;\
+	type HDMI_GENERIC7_LINE;\
+	type DP_PIXEL_ENCODING;\
+	type DP_COMPONENT_DEPTH;\
+	type HDMI_PACKET_GEN_VERSION;\
+	type HDMI_KEEPOUT_MODE;\
+	type HDMI_DEEP_COLOR_ENABLE;\
+	type HDMI_CLOCK_CHANNEL_RATE;\
+	type HDMI_DEEP_COLOR_DEPTH;\
+	type HDMI_GC_CONT;\
+	type HDMI_GC_SEND;\
+	type HDMI_NULL_SEND;\
+	type HDMI_DATA_SCRAMBLE_EN;\
+	type HDMI_AUDIO_INFO_SEND;\
+	type AFMT_AUDIO_INFO_UPDATE;\
+	type HDMI_AUDIO_INFO_LINE;\
+	type HDMI_GC_AVMUTE;\
+	type DP_MSE_RATE_X;\
+	type DP_MSE_RATE_Y;\
+	type DP_MSE_RATE_UPDATE_PENDING;\
+	type DP_SEC_GSP0_ENABLE;\
+	type DP_SEC_STREAM_ENABLE;\
+	type DP_SEC_GSP1_ENABLE;\
+	type DP_SEC_GSP2_ENABLE;\
+	type DP_SEC_GSP3_ENABLE;\
+	type DP_SEC_GSP4_ENABLE;\
+	type DP_SEC_GSP5_ENABLE;\
+	type DP_SEC_GSP6_ENABLE;\
+	type DP_SEC_GSP7_ENABLE;\
+	type DP_SEC_MPG_ENABLE;\
+	type DP_VID_STREAM_DIS_DEFER;\
+	type DP_VID_STREAM_ENABLE;\
+	type DP_VID_STREAM_STATUS;\
+	type DP_STEER_FIFO_RESET;\
+	type DP_VID_M_N_GEN_EN;\
+	type DP_VID_N;\
+	type DP_VID_M;\
+	type DIG_START;\
+	type AFMT_AUDIO_SRC_SELECT;\
+	type AFMT_AUDIO_CHANNEL_ENABLE;\
+	type HDMI_AUDIO_PACKETS_PER_LINE;\
+	type HDMI_AUDIO_DELAY_EN;\
+	type AFMT_60958_CS_UPDATE;\
+	type AFMT_AUDIO_LAYOUT_OVRD;\
+	type AFMT_60958_OSF_OVRD;\
+	type HDMI_ACR_AUTO_SEND;\
+	type HDMI_ACR_SOURCE;\
+	type HDMI_ACR_AUDIO_PRIORITY;\
+	type HDMI_ACR_CTS_32;\
+	type HDMI_ACR_N_32;\
+	type HDMI_ACR_CTS_44;\
+	type HDMI_ACR_N_44;\
+	type HDMI_ACR_CTS_48;\
+	type HDMI_ACR_N_48;\
+	type AFMT_60958_CS_CHANNEL_NUMBER_L;\
+	type AFMT_60958_CS_CLOCK_ACCURACY;\
+	type AFMT_60958_CS_CHANNEL_NUMBER_R;\
+	type AFMT_60958_CS_CHANNEL_NUMBER_2;\
+	type AFMT_60958_CS_CHANNEL_NUMBER_3;\
+	type AFMT_60958_CS_CHANNEL_NUMBER_4;\
+	type AFMT_60958_CS_CHANNEL_NUMBER_5;\
+	type AFMT_60958_CS_CHANNEL_NUMBER_6;\
+	type AFMT_60958_CS_CHANNEL_NUMBER_7;\
+	type DP_SEC_AUD_N;\
+	type DP_SEC_TIMESTAMP_MODE;\
+	type DP_SEC_ASP_ENABLE;\
+	type DP_SEC_ATP_ENABLE;\
+	type DP_SEC_AIP_ENABLE;\
+	type DP_SEC_ACM_ENABLE;\
+	type AFMT_AUDIO_SAMPLE_SEND;\
+	type AFMT_AUDIO_CLOCK_EN;\
+	type TMDS_PIXEL_ENCODING;\
+	type TMDS_COLOR_FORMAT;\
+	type DIG_STEREOSYNC_SELECT;\
+	type DIG_STEREOSYNC_GATE_EN;\
+	type DP_DB_DISABLE;\
+	type DP_MSA_MISC0;\
+	type DP_MSA_HTOTAL;\
+	type DP_MSA_VTOTAL;\
+	type DP_MSA_HSTART;\
+	type DP_MSA_VSTART;\
+	type DP_MSA_HSYNCWIDTH;\
+	type DP_MSA_HSYNCPOLARITY;\
+	type DP_MSA_VSYNCWIDTH;\
+	type DP_MSA_VSYNCPOLARITY;\
+	type DP_MSA_HWIDTH;\
+	type DP_MSA_VHEIGHT;\
+	type HDMI_DB_DISABLE;\
+	type DP_VID_N_MUL;\
+	type DP_VID_M_DOUBLE_VALUE_EN
+
+struct dcn10_stream_encoder_shift {
+	SE_REG_FIELD_LIST_DCN1_0(uint8_t);
+};
+
+struct dcn10_stream_encoder_mask {
+	SE_REG_FIELD_LIST_DCN1_0(uint32_t);
+};
+
+struct dcn10_stream_encoder {
+	struct stream_encoder base;
+	const struct dcn10_stream_enc_registers *regs;
+	const struct dcn10_stream_encoder_shift *se_shift;
+	const struct dcn10_stream_encoder_mask *se_mask;
+};
+
+void dcn10_stream_encoder_construct(
+	struct dcn10_stream_encoder *enc1,
+	struct dc_context *ctx,
+	struct dc_bios *bp,
+	enum engine_id eng_id,
+	const struct dcn10_stream_enc_registers *regs,
+	const struct dcn10_stream_encoder_shift *se_shift,
+	const struct dcn10_stream_encoder_mask *se_mask);
+
+void enc1_update_generic_info_packet(
+	struct dcn10_stream_encoder *enc1,
+	uint32_t packet_index,
+	const struct dc_info_packet *info_packet);
+
+void enc1_stream_encoder_dp_set_stream_attribute(
+	struct stream_encoder *enc,
+	struct dc_crtc_timing *crtc_timing,
+	enum dc_color_space output_color_space);
+
+void enc1_stream_encoder_hdmi_set_stream_attribute(
+	struct stream_encoder *enc,
+	struct dc_crtc_timing *crtc_timing,
+	int actual_pix_clk_khz,
+	bool enable_audio);
+
+void enc1_stream_encoder_dvi_set_stream_attribute(
+	struct stream_encoder *enc,
+	struct dc_crtc_timing *crtc_timing,
+	bool is_dual_link);
+
+void enc1_stream_encoder_set_mst_bandwidth(
+	struct stream_encoder *enc,
+	struct fixed31_32 avg_time_slots_per_mtp);
+
+void enc1_stream_encoder_update_dp_info_packets(
+	struct stream_encoder *enc,
+	const struct encoder_info_frame *info_frame);
+
+void enc1_stream_encoder_stop_dp_info_packets(
+	struct stream_encoder *enc);
+
+void enc1_stream_encoder_dp_blank(
+	struct stream_encoder *enc);
+
+void enc1_stream_encoder_dp_unblank(
+	struct stream_encoder *enc,
+	const struct encoder_unblank_param *param);
+
+void enc1_setup_stereo_sync(
+	struct stream_encoder *enc,
+	int tg_inst, bool enable);
+
+void enc1_stream_encoder_set_avmute(
+	struct stream_encoder *enc,
+	bool enable);
+
+void enc1_se_audio_mute_control(
+	struct stream_encoder *enc,
+	bool mute);
+
+void enc1_se_dp_audio_setup(
+	struct stream_encoder *enc,
+	unsigned int az_inst,
+	struct audio_info *info);
+
+void enc1_se_dp_audio_enable(
+	struct stream_encoder *enc);
+
+void enc1_se_dp_audio_disable(
+	struct stream_encoder *enc);
+
+void enc1_se_hdmi_audio_setup(
+	struct stream_encoder *enc,
+	unsigned int az_inst,
+	struct audio_info *info,
+	struct audio_crtc_info *audio_crtc_info);
+
+void enc1_se_hdmi_audio_disable(
+	struct stream_encoder *enc);
+
+#endif /* __DC_STREAM_ENCODER_DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dm_services.h b/drivers/gpu/drm/amd/display/dc/dm_services.h
index 22e7ee7dcd26..4ff9b2bba178 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_services.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_services.h
@@ -341,6 +341,10 @@ bool dm_dmcu_set_pipe(struct dc_context *ctx, unsigned int controller_id);
 
 unsigned long long dm_get_timestamp(struct dc_context *ctx);
 
+unsigned long long dm_get_elapse_time_in_ns(struct dc_context *ctx,
+		unsigned long long current_time_stamp,
+		unsigned long long last_time_stamp);
+
 /*
  * performance tracing
  */
@@ -351,10 +355,6 @@ void dm_perf_trace_timestamp(const char *func_name, unsigned int line);
 /*
  * Debug and verification hooks
  */
-bool dm_helpers_dc_conn_log(
-		struct dc_context *ctx,
-		struct log_entry *entry,
-		enum dc_log_type event);
 
 void dm_dtn_log_begin(struct dc_context *ctx);
 void dm_dtn_log_append_v(struct dc_context *ctx, const char *msg, ...);
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h
index b1ad3553f900..47c19f8fe7d1 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h
@@ -108,4 +108,17 @@ enum output_standard {
 	dm_std_uninitialized = 0, dm_std_cvtr2, dm_std_cvt
 };
 
+enum mpc_combine_affinity {
+	dm_mpc_always_when_possible,
+	dm_mpc_reduce_voltage,
+	dm_mpc_reduce_voltage_and_clocks
+};
+
+enum self_refresh_affinity {
+	dm_try_to_allow_self_refresh_and_mclk_switch,
+	dm_allow_self_refresh_and_mclk_switch,
+	dm_allow_self_refresh,
+	dm_neither_self_refresh_nor_mclk_switch
+};
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
index c109b2c34c8f..fd9d97aab071 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
@@ -26,75 +26,89 @@
 #include "display_mode_lib.h"
 #include "dc_features.h"
 
+static const struct _vcs_dpi_ip_params_st dcn1_0_ip = {
+	.rob_buffer_size_kbytes = 64,
+	.det_buffer_size_kbytes = 164,
+	.dpte_buffer_size_in_pte_reqs = 42,
+	.dpp_output_buffer_pixels = 2560,
+	.opp_output_buffer_lines = 1,
+	.pixel_chunk_size_kbytes = 8,
+	.pte_enable = 1,
+	.pte_chunk_size_kbytes = 2,
+	.meta_chunk_size_kbytes = 2,
+	.writeback_chunk_size_kbytes = 2,
+	.line_buffer_size_bits = 589824,
+	.max_line_buffer_lines = 12,
+	.IsLineBufferBppFixed = 0,
+	.LineBufferFixedBpp = -1,
+	.writeback_luma_buffer_size_kbytes = 12,
+	.writeback_chroma_buffer_size_kbytes = 8,
+	.max_num_dpp = 4,
+	.max_num_wb = 2,
+	.max_dchub_pscl_bw_pix_per_clk = 4,
+	.max_pscl_lb_bw_pix_per_clk = 2,
+	.max_lb_vscl_bw_pix_per_clk = 4,
+	.max_vscl_hscl_bw_pix_per_clk = 4,
+	.max_hscl_ratio = 4,
+	.max_vscl_ratio = 4,
+	.hscl_mults = 4,
+	.vscl_mults = 4,
+	.max_hscl_taps = 8,
+	.max_vscl_taps = 8,
+	.dispclk_ramp_margin_percent = 1,
+	.underscan_factor = 1.10,
+	.min_vblank_lines = 14,
+	.dppclk_delay_subtotal = 90,
+	.dispclk_delay_subtotal = 42,
+	.dcfclk_cstate_latency = 10,
+	.max_inter_dcn_tile_repeaters = 8,
+	.can_vstartup_lines_exceed_vsync_plus_back_porch_lines_minus_one = 0,
+	.bug_forcing_LC_req_same_size_fixed = 0,
+};
+
+static const struct _vcs_dpi_soc_bounding_box_st dcn1_0_soc = {
+	.sr_exit_time_us = 9.0,
+	.sr_enter_plus_exit_time_us = 11.0,
+	.urgent_latency_us = 4.0,
+	.writeback_latency_us = 12.0,
+	.ideal_dram_bw_after_urgent_percent = 80.0,
+	.max_request_size_bytes = 256,
+	.downspread_percent = 0.5,
+	.dram_page_open_time_ns = 50.0,
+	.dram_rw_turnaround_time_ns = 17.5,
+	.dram_return_buffer_per_channel_bytes = 8192,
+	.round_trip_ping_latency_dcfclk_cycles = 128,
+	.urgent_out_of_order_return_per_channel_bytes = 256,
+	.channel_interleave_bytes = 256,
+	.num_banks = 8,
+	.num_chans = 2,
+	.vmm_page_size_bytes = 4096,
+	.dram_clock_change_latency_us = 17.0,
+	.writeback_dram_clock_change_latency_us = 23.0,
+	.return_bus_width_bytes = 64,
+};
+
 static void set_soc_bounding_box(struct _vcs_dpi_soc_bounding_box_st *soc, enum dml_project project)
 {
-	if (project == DML_PROJECT_RAVEN1) {
-		soc->sr_exit_time_us = 9.0;
-		soc->sr_enter_plus_exit_time_us = 11.0;
-		soc->urgent_latency_us = 4.0;
-		soc->writeback_latency_us = 12.0;
-		soc->ideal_dram_bw_after_urgent_percent = 80.0;
-		soc->max_request_size_bytes = 256;
-		soc->downspread_percent = 0.5;
-		soc->dram_page_open_time_ns = 50.0;
-		soc->dram_rw_turnaround_time_ns = 17.5;
-		soc->dram_return_buffer_per_channel_bytes = 8192;
-		soc->round_trip_ping_latency_dcfclk_cycles = 128;
-		soc->urgent_out_of_order_return_per_channel_bytes = 256;
-		soc->channel_interleave_bytes = 256;
-		soc->num_banks = 8;
-		soc->num_chans = 2;
-		soc->vmm_page_size_bytes = 4096;
-		soc->dram_clock_change_latency_us = 17.0;
-		soc->writeback_dram_clock_change_latency_us = 23.0;
-		soc->return_bus_width_bytes = 64;
-	} else {
-		BREAK_TO_DEBUGGER(); /* Invalid Project Specified */
+	switch (project) {
+	case DML_PROJECT_RAVEN1:
+		*soc = dcn1_0_soc;
+		break;
+	default:
+		ASSERT(0);
+		break;
 	}
 }
 
 static void set_ip_params(struct _vcs_dpi_ip_params_st *ip, enum dml_project project)
 {
-	if (project == DML_PROJECT_RAVEN1) {
-		ip->rob_buffer_size_kbytes = 64;
-		ip->det_buffer_size_kbytes = 164;
-		ip->dpte_buffer_size_in_pte_reqs = 42;
-		ip->dpp_output_buffer_pixels = 2560;
-		ip->opp_output_buffer_lines = 1;
-		ip->pixel_chunk_size_kbytes = 8;
-		ip->pte_enable = 1;
-		ip->pte_chunk_size_kbytes = 2;
-		ip->meta_chunk_size_kbytes = 2;
-		ip->writeback_chunk_size_kbytes = 2;
-		ip->line_buffer_size_bits = 589824;
-		ip->max_line_buffer_lines = 12;
-		ip->IsLineBufferBppFixed = 0;
-		ip->LineBufferFixedBpp = -1;
-		ip->writeback_luma_buffer_size_kbytes = 12;
-		ip->writeback_chroma_buffer_size_kbytes = 8;
-		ip->max_num_dpp = 4;
-		ip->max_num_wb = 2;
-		ip->max_dchub_pscl_bw_pix_per_clk = 4;
-		ip->max_pscl_lb_bw_pix_per_clk = 2;
-		ip->max_lb_vscl_bw_pix_per_clk = 4;
-		ip->max_vscl_hscl_bw_pix_per_clk = 4;
-		ip->max_hscl_ratio = 4;
-		ip->max_vscl_ratio = 4;
-		ip->hscl_mults = 4;
-		ip->vscl_mults = 4;
-		ip->max_hscl_taps = 8;
-		ip->max_vscl_taps = 8;
-		ip->dispclk_ramp_margin_percent = 1;
-		ip->underscan_factor = 1.10;
-		ip->min_vblank_lines = 14;
-		ip->dppclk_delay_subtotal = 90;
-		ip->dispclk_delay_subtotal = 42;
-		ip->dcfclk_cstate_latency = 10;
-		ip->max_inter_dcn_tile_repeaters = 8;
-		ip->can_vstartup_lines_exceed_vsync_plus_back_porch_lines_minus_one = 0;
-		ip->bug_forcing_LC_req_same_size_fixed = 0;
-	} else {
-		BREAK_TO_DEBUGGER(); /* Invalid Project Specified */
+	switch (project) {
+	case DML_PROJECT_RAVEN1:
+		*ip = dcn1_0_ip;
+		break;
+	default:
+		ASSERT(0);
+		break;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
index 09affa16cc43..7fa0375939ae 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
@@ -25,39 +25,39 @@
 #ifndef __DISPLAY_MODE_STRUCTS_H__
 #define __DISPLAY_MODE_STRUCTS_H__
 
-typedef struct _vcs_dpi_voltage_scaling_st	voltage_scaling_st;
-typedef struct _vcs_dpi_soc_bounding_box_st	soc_bounding_box_st;
-typedef struct _vcs_dpi_ip_params_st	ip_params_st;
-typedef struct _vcs_dpi_display_pipe_source_params_st	display_pipe_source_params_st;
-typedef struct _vcs_dpi_display_output_params_st	display_output_params_st;
-typedef struct _vcs_dpi_display_bandwidth_st	display_bandwidth_st;
-typedef struct _vcs_dpi_scaler_ratio_depth_st	scaler_ratio_depth_st;
-typedef struct _vcs_dpi_scaler_taps_st	scaler_taps_st;
-typedef struct _vcs_dpi_display_pipe_dest_params_st	display_pipe_dest_params_st;
-typedef struct _vcs_dpi_display_pipe_params_st	display_pipe_params_st;
-typedef struct _vcs_dpi_display_clocks_and_cfg_st	display_clocks_and_cfg_st;
-typedef struct _vcs_dpi_display_e2e_pipe_params_st	display_e2e_pipe_params_st;
-typedef struct _vcs_dpi_dchub_buffer_sizing_st	dchub_buffer_sizing_st;
-typedef struct _vcs_dpi_watermarks_perf_st	watermarks_perf_st;
-typedef struct _vcs_dpi_cstate_pstate_watermarks_st	cstate_pstate_watermarks_st;
-typedef struct _vcs_dpi_wm_calc_pipe_params_st	wm_calc_pipe_params_st;
-typedef struct _vcs_dpi_vratio_pre_st	vratio_pre_st;
-typedef struct _vcs_dpi_display_data_rq_misc_params_st	display_data_rq_misc_params_st;
-typedef struct _vcs_dpi_display_data_rq_sizing_params_st	display_data_rq_sizing_params_st;
-typedef struct _vcs_dpi_display_data_rq_dlg_params_st	display_data_rq_dlg_params_st;
-typedef struct _vcs_dpi_display_cur_rq_dlg_params_st	display_cur_rq_dlg_params_st;
-typedef struct _vcs_dpi_display_rq_dlg_params_st	display_rq_dlg_params_st;
-typedef struct _vcs_dpi_display_rq_sizing_params_st	display_rq_sizing_params_st;
-typedef struct _vcs_dpi_display_rq_misc_params_st	display_rq_misc_params_st;
-typedef struct _vcs_dpi_display_rq_params_st	display_rq_params_st;
-typedef struct _vcs_dpi_display_dlg_regs_st	display_dlg_regs_st;
-typedef struct _vcs_dpi_display_ttu_regs_st	display_ttu_regs_st;
-typedef struct _vcs_dpi_display_data_rq_regs_st	display_data_rq_regs_st;
-typedef struct _vcs_dpi_display_rq_regs_st	display_rq_regs_st;
-typedef struct _vcs_dpi_display_dlg_sys_params_st	display_dlg_sys_params_st;
-typedef struct _vcs_dpi_display_dlg_prefetch_param_st	display_dlg_prefetch_param_st;
-typedef struct _vcs_dpi_display_pipe_clock_st	display_pipe_clock_st;
-typedef struct _vcs_dpi_display_arb_params_st	display_arb_params_st;
+typedef struct _vcs_dpi_voltage_scaling_st voltage_scaling_st;
+typedef struct _vcs_dpi_soc_bounding_box_st soc_bounding_box_st;
+typedef struct _vcs_dpi_ip_params_st ip_params_st;
+typedef struct _vcs_dpi_display_pipe_source_params_st display_pipe_source_params_st;
+typedef struct _vcs_dpi_display_output_params_st display_output_params_st;
+typedef struct _vcs_dpi_display_bandwidth_st display_bandwidth_st;
+typedef struct _vcs_dpi_scaler_ratio_depth_st scaler_ratio_depth_st;
+typedef struct _vcs_dpi_scaler_taps_st scaler_taps_st;
+typedef struct _vcs_dpi_display_pipe_dest_params_st display_pipe_dest_params_st;
+typedef struct _vcs_dpi_display_pipe_params_st display_pipe_params_st;
+typedef struct _vcs_dpi_display_clocks_and_cfg_st display_clocks_and_cfg_st;
+typedef struct _vcs_dpi_display_e2e_pipe_params_st display_e2e_pipe_params_st;
+typedef struct _vcs_dpi_dchub_buffer_sizing_st dchub_buffer_sizing_st;
+typedef struct _vcs_dpi_watermarks_perf_st watermarks_perf_st;
+typedef struct _vcs_dpi_cstate_pstate_watermarks_st cstate_pstate_watermarks_st;
+typedef struct _vcs_dpi_wm_calc_pipe_params_st wm_calc_pipe_params_st;
+typedef struct _vcs_dpi_vratio_pre_st vratio_pre_st;
+typedef struct _vcs_dpi_display_data_rq_misc_params_st display_data_rq_misc_params_st;
+typedef struct _vcs_dpi_display_data_rq_sizing_params_st display_data_rq_sizing_params_st;
+typedef struct _vcs_dpi_display_data_rq_dlg_params_st display_data_rq_dlg_params_st;
+typedef struct _vcs_dpi_display_cur_rq_dlg_params_st display_cur_rq_dlg_params_st;
+typedef struct _vcs_dpi_display_rq_dlg_params_st display_rq_dlg_params_st;
+typedef struct _vcs_dpi_display_rq_sizing_params_st display_rq_sizing_params_st;
+typedef struct _vcs_dpi_display_rq_misc_params_st display_rq_misc_params_st;
+typedef struct _vcs_dpi_display_rq_params_st display_rq_params_st;
+typedef struct _vcs_dpi_display_dlg_regs_st display_dlg_regs_st;
+typedef struct _vcs_dpi_display_ttu_regs_st display_ttu_regs_st;
+typedef struct _vcs_dpi_display_data_rq_regs_st display_data_rq_regs_st;
+typedef struct _vcs_dpi_display_rq_regs_st display_rq_regs_st;
+typedef struct _vcs_dpi_display_dlg_sys_params_st display_dlg_sys_params_st;
+typedef struct _vcs_dpi_display_dlg_prefetch_param_st display_dlg_prefetch_param_st;
+typedef struct _vcs_dpi_display_pipe_clock_st display_pipe_clock_st;
+typedef struct _vcs_dpi_display_arb_params_st display_arb_params_st;
 
 struct _vcs_dpi_voltage_scaling_st {
 	int state;
@@ -72,89 +72,107 @@ struct _vcs_dpi_voltage_scaling_st {
 	double dppclk_mhz;
 };
 
-struct	_vcs_dpi_soc_bounding_box_st	{
-	double	sr_exit_time_us;
-	double	sr_enter_plus_exit_time_us;
-	double	urgent_latency_us;
-	double	writeback_latency_us;
-	double	ideal_dram_bw_after_urgent_percent;
-	unsigned int	max_request_size_bytes;
-	double	downspread_percent;
-	double	dram_page_open_time_ns;
-	double	dram_rw_turnaround_time_ns;
-	double	dram_return_buffer_per_channel_bytes;
-	double	dram_channel_width_bytes;
+struct _vcs_dpi_soc_bounding_box_st {
+	double sr_exit_time_us;
+	double sr_enter_plus_exit_time_us;
+	double urgent_latency_us;
+	double urgent_latency_pixel_data_only_us;
+	double urgent_latency_pixel_mixed_with_vm_data_us;
+	double urgent_latency_vm_data_only_us;
+	double writeback_latency_us;
+	double ideal_dram_bw_after_urgent_percent;
+	double pct_ideal_dram_sdp_bw_after_urgent_pixel_only; // PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly
+	double pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm;
+	double pct_ideal_dram_sdp_bw_after_urgent_vm_only;
+	double max_avg_sdp_bw_use_normal_percent;
+	double max_avg_dram_bw_use_normal_percent;
+	unsigned int max_request_size_bytes;
+	double downspread_percent;
+	double dram_page_open_time_ns;
+	double dram_rw_turnaround_time_ns;
+	double dram_return_buffer_per_channel_bytes;
+	double dram_channel_width_bytes;
 	double fabric_datapath_to_dcn_data_return_bytes;
 	double dcn_downspread_percent;
 	double dispclk_dppclk_vco_speed_mhz;
 	double dfs_vco_period_ps;
-	unsigned int	round_trip_ping_latency_dcfclk_cycles;
-	unsigned int	urgent_out_of_order_return_per_channel_bytes;
-	unsigned int	channel_interleave_bytes;
-	unsigned int	num_banks;
-	unsigned int	num_chans;
-	unsigned int	vmm_page_size_bytes;
-	double	dram_clock_change_latency_us;
-	double	writeback_dram_clock_change_latency_us;
-	unsigned int	return_bus_width_bytes;
-	unsigned int	voltage_override;
-	double	xfc_bus_transport_time_us;
-	double	xfc_xbuf_latency_tolerance_us;
+	unsigned int urgent_out_of_order_return_per_channel_pixel_only_bytes;
+	unsigned int urgent_out_of_order_return_per_channel_pixel_and_vm_bytes;
+	unsigned int urgent_out_of_order_return_per_channel_vm_only_bytes;
+	unsigned int round_trip_ping_latency_dcfclk_cycles;
+	unsigned int urgent_out_of_order_return_per_channel_bytes;
+	unsigned int channel_interleave_bytes;
+	unsigned int num_banks;
+	unsigned int num_chans;
+	unsigned int vmm_page_size_bytes;
+	double dram_clock_change_latency_us;
+	double writeback_dram_clock_change_latency_us;
+	unsigned int return_bus_width_bytes;
+	unsigned int voltage_override;
+	double xfc_bus_transport_time_us;
+	double xfc_xbuf_latency_tolerance_us;
+	int use_urgent_burst_bw;
 	struct _vcs_dpi_voltage_scaling_st clock_limits[7];
 };
 
-struct	_vcs_dpi_ip_params_st	{
-	unsigned int	max_inter_dcn_tile_repeaters;
-	unsigned int	num_dsc;
-	unsigned int	odm_capable;
-	unsigned int	rob_buffer_size_kbytes;
-	unsigned int	det_buffer_size_kbytes;
-	unsigned int	dpte_buffer_size_in_pte_reqs;
-	unsigned int	pde_proc_buffer_size_64k_reqs;
-	unsigned int	dpp_output_buffer_pixels;
-	unsigned int	opp_output_buffer_lines;
-	unsigned int	pixel_chunk_size_kbytes;
-	unsigned char	pte_enable;
-	unsigned int	pte_chunk_size_kbytes;
-	unsigned int	meta_chunk_size_kbytes;
-	unsigned int	writeback_chunk_size_kbytes;
-	unsigned int	line_buffer_size_bits;
-	unsigned int	max_line_buffer_lines;
-	unsigned int	writeback_luma_buffer_size_kbytes;
-	unsigned int	writeback_chroma_buffer_size_kbytes;
-	unsigned int	writeback_chroma_line_buffer_width_pixels;
-	unsigned int	max_page_table_levels;
-	unsigned int	max_num_dpp;
-	unsigned int	max_num_otg;
-	unsigned int	cursor_chunk_size;
-	unsigned int	cursor_buffer_size;
-	unsigned int	max_num_wb;
-	unsigned int	max_dchub_pscl_bw_pix_per_clk;
-	unsigned int	max_pscl_lb_bw_pix_per_clk;
-	unsigned int	max_lb_vscl_bw_pix_per_clk;
-	unsigned int	max_vscl_hscl_bw_pix_per_clk;
-	double	max_hscl_ratio;
-	double	max_vscl_ratio;
-	unsigned int	hscl_mults;
-	unsigned int	vscl_mults;
-	unsigned int	max_hscl_taps;
-	unsigned int	max_vscl_taps;
-	unsigned int	xfc_supported;
-	unsigned int	xfc_fill_constant_bytes;
-	double	dispclk_ramp_margin_percent;
-	double	xfc_fill_bw_overhead_percent;
-	double	underscan_factor;
-	unsigned int	min_vblank_lines;
-	unsigned int	dppclk_delay_subtotal;
-	unsigned int	dispclk_delay_subtotal;
-	unsigned int	dcfclk_cstate_latency;
-	unsigned int	dppclk_delay_scl;
-	unsigned int	dppclk_delay_scl_lb_only;
-	unsigned int	dppclk_delay_cnvc_formatter;
-	unsigned int	dppclk_delay_cnvc_cursor;
-	unsigned int	is_line_buffer_bpp_fixed;
-	unsigned int	line_buffer_fixed_bpp;
-	unsigned int	dcc_supported;
+struct _vcs_dpi_ip_params_st {
+	bool gpuvm_enable;
+	bool hostvm_enable;
+	unsigned int gpuvm_max_page_table_levels;
+	unsigned int hostvm_max_page_table_levels;
+	unsigned int hostvm_cached_page_table_levels;
+	unsigned int pte_group_size_bytes;
+	unsigned int max_inter_dcn_tile_repeaters;
+	unsigned int num_dsc;
+	unsigned int odm_capable;
+	unsigned int rob_buffer_size_kbytes;
+	unsigned int det_buffer_size_kbytes;
+	unsigned int dpte_buffer_size_in_pte_reqs;
+	unsigned int pde_proc_buffer_size_64k_reqs;
+	unsigned int dpp_output_buffer_pixels;
+	unsigned int opp_output_buffer_lines;
+	unsigned int pixel_chunk_size_kbytes;
+	unsigned char pte_enable;
+	unsigned int pte_chunk_size_kbytes;
+	unsigned int meta_chunk_size_kbytes;
+	unsigned int writeback_chunk_size_kbytes;
+	unsigned int line_buffer_size_bits;
+	unsigned int max_line_buffer_lines;
+	unsigned int writeback_luma_buffer_size_kbytes;
+	unsigned int writeback_chroma_buffer_size_kbytes;
+	unsigned int writeback_chroma_line_buffer_width_pixels;
+	unsigned int max_page_table_levels;
+	unsigned int max_num_dpp;
+	unsigned int max_num_otg;
+	unsigned int cursor_chunk_size;
+	unsigned int cursor_buffer_size;
+	unsigned int max_num_wb;
+	unsigned int max_dchub_pscl_bw_pix_per_clk;
+	unsigned int max_pscl_lb_bw_pix_per_clk;
+	unsigned int max_lb_vscl_bw_pix_per_clk;
+	unsigned int max_vscl_hscl_bw_pix_per_clk;
+	double max_hscl_ratio;
+	double max_vscl_ratio;
+	unsigned int hscl_mults;
+	unsigned int vscl_mults;
+	unsigned int max_hscl_taps;
+	unsigned int max_vscl_taps;
+	unsigned int xfc_supported;
+	unsigned int xfc_fill_constant_bytes;
+	double dispclk_ramp_margin_percent;
+	double xfc_fill_bw_overhead_percent;
+	double underscan_factor;
+	unsigned int min_vblank_lines;
+	unsigned int dppclk_delay_subtotal;
+	unsigned int dispclk_delay_subtotal;
+	unsigned int dcfclk_cstate_latency;
+	unsigned int dppclk_delay_scl;
+	unsigned int dppclk_delay_scl_lb_only;
+	unsigned int dppclk_delay_cnvc_formatter;
+	unsigned int dppclk_delay_cnvc_cursor;
+	unsigned int is_line_buffer_bpp_fixed;
+	unsigned int line_buffer_fixed_bpp;
+	unsigned int dcc_supported;
 
 	unsigned int IsLineBufferBppFixed;
 	unsigned int LineBufferFixedBpp;
@@ -169,41 +187,45 @@ struct _vcs_dpi_display_xfc_params_st {
 	int xfc_slv_chunk_size_bytes;
 };
 
-struct	_vcs_dpi_display_pipe_source_params_st	{
-	int	source_format;
-	unsigned char	dcc;
-	unsigned int	dcc_override;
-	unsigned int	dcc_rate;
-	unsigned char	dcc_use_global;
-	unsigned char	vm;
-	unsigned char	vm_levels_force_en;
-	unsigned int	vm_levels_force;
-	int	source_scan;
-	int	sw_mode;
-	int	macro_tile_size;
-	unsigned char	is_display_sw;
-	unsigned int	viewport_width;
-	unsigned int	viewport_height;
-	unsigned int	viewport_y_y;
-	unsigned int	viewport_y_c;
-	unsigned int	viewport_width_c;
-	unsigned int	viewport_height_c;
-	unsigned int	data_pitch;
-	unsigned int	data_pitch_c;
-	unsigned int	meta_pitch;
-	unsigned int	meta_pitch_c;
-	unsigned int	cur0_src_width;
-	int	cur0_bpp;
-	unsigned int	cur1_src_width;
-	int	cur1_bpp;
-	int	num_cursors;
-	unsigned char	is_hsplit;
-	unsigned char	dynamic_metadata_enable;
-	unsigned int	dynamic_metadata_lines_before_active;
-	unsigned int	dynamic_metadata_xmit_bytes;
-	unsigned int	hsplit_grp;
-	unsigned char	xfc_enable;
-	unsigned char	xfc_slave;
+struct _vcs_dpi_display_pipe_source_params_st {
+	int source_format;
+	unsigned char dcc;
+	unsigned int dcc_override;
+	unsigned int dcc_rate;
+	unsigned char dcc_use_global;
+	unsigned char vm;
+	bool gpuvm;    // gpuvm enabled
+	bool hostvm;    // hostvm enabled
+	bool gpuvm_levels_force_en;
+	unsigned int gpuvm_levels_force;
+	bool hostvm_levels_force_en;
+	unsigned int hostvm_levels_force;
+	int source_scan;
+	int sw_mode;
+	int macro_tile_size;
+	unsigned char is_display_sw;
+	unsigned int viewport_width;
+	unsigned int viewport_height;
+	unsigned int viewport_y_y;
+	unsigned int viewport_y_c;
+	unsigned int viewport_width_c;
+	unsigned int viewport_height_c;
+	unsigned int data_pitch;
+	unsigned int data_pitch_c;
+	unsigned int meta_pitch;
+	unsigned int meta_pitch_c;
+	unsigned int cur0_src_width;
+	int cur0_bpp;
+	unsigned int cur1_src_width;
+	int cur1_bpp;
+	int num_cursors;
+	unsigned char is_hsplit;
+	unsigned char dynamic_metadata_enable;
+	unsigned int dynamic_metadata_lines_before_active;
+	unsigned int dynamic_metadata_xmit_bytes;
+	unsigned int hsplit_grp;
+	unsigned char xfc_enable;
+	unsigned char xfc_slave;
 	struct _vcs_dpi_display_xfc_params_st xfc_params;
 };
 struct writeback_st {
@@ -215,338 +237,339 @@ struct writeback_st {
 	int wb_vtaps_luma;
 	int wb_htaps_chroma;
 	int wb_vtaps_chroma;
-	int wb_hratio;
-	int wb_vratio;
-};
-
-struct	_vcs_dpi_display_output_params_st	{
-	int	dp_lanes;
-	int	output_bpp;
-	int	dsc_enable;
-	int	wb_enable;
-	int	opp_input_bpc;
-	int	output_type;
-	int	output_format;
-	int	output_standard;
-	int	dsc_slices;
+	double wb_hratio;
+	double wb_vratio;
+};
+
+struct _vcs_dpi_display_output_params_st {
+	int dp_lanes;
+	int output_bpp;
+	int dsc_enable;
+	int wb_enable;
+	int num_active_wb;
+	int output_bpc;
+	int output_type;
+	int output_format;
+	int output_standard;
+	int dsc_slices;
 	struct writeback_st wb;
 };
 
-struct	_vcs_dpi_display_bandwidth_st	{
-	double	total_bw_consumed_gbps;
-	double	guaranteed_urgent_return_bw_gbps;
-};
-
-struct	_vcs_dpi_scaler_ratio_depth_st	{
-	double	hscl_ratio;
-	double	vscl_ratio;
-	double	hscl_ratio_c;
-	double	vscl_ratio_c;
-	double	vinit;
-	double	vinit_c;
-	double	vinit_bot;
-	double	vinit_bot_c;
-	int	lb_depth;
-	int	scl_enable;
-};
-
-struct	_vcs_dpi_scaler_taps_st	{
-	unsigned int	htaps;
-	unsigned int	vtaps;
-	unsigned int	htaps_c;
-	unsigned int	vtaps_c;
-};
-
-struct	_vcs_dpi_display_pipe_dest_params_st	{
-	unsigned int	recout_width;
-	unsigned int	recout_height;
-	unsigned int	full_recout_width;
-	unsigned int	full_recout_height;
-	unsigned int	hblank_start;
-	unsigned int	hblank_end;
-	unsigned int	vblank_start;
-	unsigned int	vblank_end;
-	unsigned int	htotal;
-	unsigned int	vtotal;
-	unsigned int	vactive;
-	unsigned int	hactive;
-	unsigned int	vstartup_start;
-	unsigned int	vupdate_offset;
-	unsigned int	vupdate_width;
-	unsigned int	vready_offset;
-	unsigned char	interlaced;
-	unsigned char	underscan;
-	double	pixel_rate_mhz;
-	unsigned char	synchronized_vblank_all_planes;
-	unsigned char	otg_inst;
-	unsigned char	odm_split_cnt;
-	unsigned char	odm_combine;
-};
-
-struct	_vcs_dpi_display_pipe_params_st	{
-	display_pipe_source_params_st	src;
-	display_pipe_dest_params_st	dest;
-	scaler_ratio_depth_st	scale_ratio_depth;
-	scaler_taps_st	scale_taps;
-};
-
-struct	_vcs_dpi_display_clocks_and_cfg_st	{
-	int	voltage;
-	double	dppclk_mhz;
-	double	refclk_mhz;
-	double	dispclk_mhz;
-	double	dcfclk_mhz;
-	double	socclk_mhz;
-};
-
-struct	_vcs_dpi_display_e2e_pipe_params_st	{
-	display_pipe_params_st	pipe;
-	display_output_params_st	dout;
-	display_clocks_and_cfg_st	clks_cfg;
-};
-
-struct	_vcs_dpi_dchub_buffer_sizing_st	{
-	unsigned int	swath_width_y;
-	unsigned int	swath_height_y;
-	unsigned int	swath_height_c;
-	unsigned int	detail_buffer_size_y;
-};
-
-struct	_vcs_dpi_watermarks_perf_st	{
-	double	stutter_eff_in_active_region_percent;
-	double	urgent_latency_supported_us;
-	double	non_urgent_latency_supported_us;
-	double	dram_clock_change_margin_us;
-	double	dram_access_eff_percent;
-};
-
-struct	_vcs_dpi_cstate_pstate_watermarks_st	{
-	double	cstate_exit_us;
-	double	cstate_enter_plus_exit_us;
-	double	pstate_change_us;
-};
-
-struct	_vcs_dpi_wm_calc_pipe_params_st	{
-	unsigned int	num_dpp;
-	int	voltage;
-	int	output_type;
-	double	dcfclk_mhz;
-	double	socclk_mhz;
-	double	dppclk_mhz;
-	double	pixclk_mhz;
-	unsigned char	interlace_en;
-	unsigned char	pte_enable;
-	unsigned char	dcc_enable;
-	double	dcc_rate;
-	double	bytes_per_pixel_c;
-	double	bytes_per_pixel_y;
-	unsigned int	swath_width_y;
-	unsigned int	swath_height_y;
-	unsigned int	swath_height_c;
-	unsigned int	det_buffer_size_y;
-	double	h_ratio;
-	double	v_ratio;
-	unsigned int	h_taps;
-	unsigned int	h_total;
-	unsigned int	v_total;
-	unsigned int	v_active;
-	unsigned int	e2e_index;
-	double	display_pipe_line_delivery_time;
-	double	read_bw;
-	unsigned int	lines_in_det_y;
-	unsigned int	lines_in_det_y_rounded_down_to_swath;
-	double	full_det_buffering_time;
-	double	dcfclk_deepsleep_mhz_per_plane;
-};
-
-struct	_vcs_dpi_vratio_pre_st	{
-	double	vratio_pre_l;
-	double	vratio_pre_c;
-};
-
-struct	_vcs_dpi_display_data_rq_misc_params_st	{
-	unsigned int	full_swath_bytes;
-	unsigned int	stored_swath_bytes;
-	unsigned int	blk256_height;
-	unsigned int	blk256_width;
-	unsigned int	req_height;
-	unsigned int	req_width;
-};
-
-struct	_vcs_dpi_display_data_rq_sizing_params_st	{
-	unsigned int	chunk_bytes;
-	unsigned int	min_chunk_bytes;
-	unsigned int	meta_chunk_bytes;
-	unsigned int	min_meta_chunk_bytes;
-	unsigned int	mpte_group_bytes;
-	unsigned int	dpte_group_bytes;
-};
-
-struct	_vcs_dpi_display_data_rq_dlg_params_st	{
-	unsigned int	swath_width_ub;
-	unsigned int	swath_height;
-	unsigned int	req_per_swath_ub;
-	unsigned int	meta_pte_bytes_per_frame_ub;
-	unsigned int	dpte_req_per_row_ub;
-	unsigned int	dpte_groups_per_row_ub;
-	unsigned int	dpte_row_height;
-	unsigned int	dpte_bytes_per_row_ub;
-	unsigned int	meta_chunks_per_row_ub;
-	unsigned int	meta_req_per_row_ub;
-	unsigned int	meta_row_height;
-	unsigned int	meta_bytes_per_row_ub;
-};
-
-struct	_vcs_dpi_display_cur_rq_dlg_params_st	{
-	unsigned char	enable;
-	unsigned int	swath_height;
-	unsigned int	req_per_line;
-};
-
-struct	_vcs_dpi_display_rq_dlg_params_st	{
-	display_data_rq_dlg_params_st	rq_l;
-	display_data_rq_dlg_params_st	rq_c;
-	display_cur_rq_dlg_params_st	rq_cur0;
-};
-
-struct	_vcs_dpi_display_rq_sizing_params_st	{
-	display_data_rq_sizing_params_st	rq_l;
-	display_data_rq_sizing_params_st	rq_c;
-};
-
-struct	_vcs_dpi_display_rq_misc_params_st	{
-	display_data_rq_misc_params_st	rq_l;
-	display_data_rq_misc_params_st	rq_c;
-};
-
-struct	_vcs_dpi_display_rq_params_st	{
-	unsigned char	yuv420;
-	unsigned char	yuv420_10bpc;
-	display_rq_misc_params_st	misc;
-	display_rq_sizing_params_st	sizing;
-	display_rq_dlg_params_st	dlg;
-};
-
-struct	_vcs_dpi_display_dlg_regs_st	{
-	unsigned int	refcyc_h_blank_end;
-	unsigned int	dlg_vblank_end;
-	unsigned int	min_dst_y_next_start;
-	unsigned int	refcyc_per_htotal;
-	unsigned int	refcyc_x_after_scaler;
-	unsigned int	dst_y_after_scaler;
-	unsigned int	dst_y_prefetch;
-	unsigned int	dst_y_per_vm_vblank;
-	unsigned int	dst_y_per_row_vblank;
-	unsigned int	dst_y_per_vm_flip;
-	unsigned int	dst_y_per_row_flip;
-	unsigned int	ref_freq_to_pix_freq;
-	unsigned int	vratio_prefetch;
-	unsigned int	vratio_prefetch_c;
-	unsigned int	refcyc_per_pte_group_vblank_l;
-	unsigned int	refcyc_per_pte_group_vblank_c;
-	unsigned int	refcyc_per_meta_chunk_vblank_l;
-	unsigned int	refcyc_per_meta_chunk_vblank_c;
-	unsigned int	refcyc_per_pte_group_flip_l;
-	unsigned int	refcyc_per_pte_group_flip_c;
-	unsigned int	refcyc_per_meta_chunk_flip_l;
-	unsigned int	refcyc_per_meta_chunk_flip_c;
-	unsigned int	dst_y_per_pte_row_nom_l;
-	unsigned int	dst_y_per_pte_row_nom_c;
-	unsigned int	refcyc_per_pte_group_nom_l;
-	unsigned int	refcyc_per_pte_group_nom_c;
-	unsigned int	dst_y_per_meta_row_nom_l;
-	unsigned int	dst_y_per_meta_row_nom_c;
-	unsigned int	refcyc_per_meta_chunk_nom_l;
-	unsigned int	refcyc_per_meta_chunk_nom_c;
-	unsigned int	refcyc_per_line_delivery_pre_l;
-	unsigned int	refcyc_per_line_delivery_pre_c;
-	unsigned int	refcyc_per_line_delivery_l;
-	unsigned int	refcyc_per_line_delivery_c;
-	unsigned int	chunk_hdl_adjust_cur0;
-	unsigned int	chunk_hdl_adjust_cur1;
-	unsigned int	vready_after_vcount0;
-	unsigned int	dst_y_offset_cur0;
-	unsigned int	dst_y_offset_cur1;
-	unsigned int	xfc_reg_transfer_delay;
-	unsigned int	xfc_reg_precharge_delay;
-	unsigned int	xfc_reg_remote_surface_flip_latency;
-	unsigned int	xfc_reg_prefetch_margin;
-	unsigned int	dst_y_delta_drq_limit;
-};
-
-struct	_vcs_dpi_display_ttu_regs_st	{
-	unsigned int	qos_level_low_wm;
-	unsigned int	qos_level_high_wm;
-	unsigned int	min_ttu_vblank;
-	unsigned int	qos_level_flip;
-	unsigned int	refcyc_per_req_delivery_l;
-	unsigned int	refcyc_per_req_delivery_c;
-	unsigned int	refcyc_per_req_delivery_cur0;
-	unsigned int	refcyc_per_req_delivery_cur1;
-	unsigned int	refcyc_per_req_delivery_pre_l;
-	unsigned int	refcyc_per_req_delivery_pre_c;
-	unsigned int	refcyc_per_req_delivery_pre_cur0;
-	unsigned int	refcyc_per_req_delivery_pre_cur1;
-	unsigned int	qos_level_fixed_l;
-	unsigned int	qos_level_fixed_c;
-	unsigned int	qos_level_fixed_cur0;
-	unsigned int	qos_level_fixed_cur1;
-	unsigned int	qos_ramp_disable_l;
-	unsigned int	qos_ramp_disable_c;
-	unsigned int	qos_ramp_disable_cur0;
-	unsigned int	qos_ramp_disable_cur1;
-};
-
-struct	_vcs_dpi_display_data_rq_regs_st	{
-	unsigned int	chunk_size;
-	unsigned int	min_chunk_size;
-	unsigned int	meta_chunk_size;
-	unsigned int	min_meta_chunk_size;
-	unsigned int	dpte_group_size;
-	unsigned int	mpte_group_size;
-	unsigned int	swath_height;
-	unsigned int	pte_row_height_linear;
-};
-
-struct	_vcs_dpi_display_rq_regs_st	{
-	display_data_rq_regs_st	rq_regs_l;
-	display_data_rq_regs_st	rq_regs_c;
-	unsigned int	drq_expansion_mode;
-	unsigned int	prq_expansion_mode;
-	unsigned int	mrq_expansion_mode;
-	unsigned int	crq_expansion_mode;
-	unsigned int	plane1_base_address;
-};
-
-struct	_vcs_dpi_display_dlg_sys_params_st	{
-	double	t_mclk_wm_us;
-	double	t_urg_wm_us;
-	double	t_sr_wm_us;
-	double	t_extra_us;
-	double	mem_trip_us;
-	double	t_srx_delay_us;
-	double	deepsleep_dcfclk_mhz;
-	double	total_flip_bw;
-	unsigned int	total_flip_bytes;
-};
-
-struct	_vcs_dpi_display_dlg_prefetch_param_st	{
-	double	prefetch_bw;
-	unsigned int	flip_bytes;
-};
-
-struct	_vcs_dpi_display_pipe_clock_st	{
-	double	dcfclk_mhz;
-	double	dispclk_mhz;
-	double	socclk_mhz;
-	double	dscclk_mhz[6];
-	double	dppclk_mhz[6];
-};
-
-struct	_vcs_dpi_display_arb_params_st	{
-	int	max_req_outstanding;
-	int	min_req_outstanding;
-	int	sat_level_us;
+struct _vcs_dpi_display_bandwidth_st {
+	double total_bw_consumed_gbps;
+	double guaranteed_urgent_return_bw_gbps;
+};
+
+struct _vcs_dpi_scaler_ratio_depth_st {
+	double hscl_ratio;
+	double vscl_ratio;
+	double hscl_ratio_c;
+	double vscl_ratio_c;
+	double vinit;
+	double vinit_c;
+	double vinit_bot;
+	double vinit_bot_c;
+	int lb_depth;
+	int scl_enable;
+};
+
+struct _vcs_dpi_scaler_taps_st {
+	unsigned int htaps;
+	unsigned int vtaps;
+	unsigned int htaps_c;
+	unsigned int vtaps_c;
+};
+
+struct _vcs_dpi_display_pipe_dest_params_st {
+	unsigned int recout_width;
+	unsigned int recout_height;
+	unsigned int full_recout_width;
+	unsigned int full_recout_height;
+	unsigned int hblank_start;
+	unsigned int hblank_end;
+	unsigned int vblank_start;
+	unsigned int vblank_end;
+	unsigned int htotal;
+	unsigned int vtotal;
+	unsigned int vactive;
+	unsigned int hactive;
+	unsigned int vstartup_start;
+	unsigned int vupdate_offset;
+	unsigned int vupdate_width;
+	unsigned int vready_offset;
+	unsigned char interlaced;
+	unsigned char underscan;
+	double pixel_rate_mhz;
+	unsigned char synchronized_vblank_all_planes;
+	unsigned char otg_inst;
+	unsigned char odm_split_cnt;
+	unsigned char odm_combine;
+};
+
+struct _vcs_dpi_display_pipe_params_st {
+	display_pipe_source_params_st src;
+	display_pipe_dest_params_st dest;
+	scaler_ratio_depth_st scale_ratio_depth;
+	scaler_taps_st scale_taps;
+};
+
+struct _vcs_dpi_display_clocks_and_cfg_st {
+	int voltage;
+	double dppclk_mhz;
+	double refclk_mhz;
+	double dispclk_mhz;
+	double dcfclk_mhz;
+	double socclk_mhz;
+};
+
+struct _vcs_dpi_display_e2e_pipe_params_st {
+	display_pipe_params_st pipe;
+	display_output_params_st dout;
+	display_clocks_and_cfg_st clks_cfg;
+};
+
+struct _vcs_dpi_dchub_buffer_sizing_st {
+	unsigned int swath_width_y;
+	unsigned int swath_height_y;
+	unsigned int swath_height_c;
+	unsigned int detail_buffer_size_y;
+};
+
+struct _vcs_dpi_watermarks_perf_st {
+	double stutter_eff_in_active_region_percent;
+	double urgent_latency_supported_us;
+	double non_urgent_latency_supported_us;
+	double dram_clock_change_margin_us;
+	double dram_access_eff_percent;
+};
+
+struct _vcs_dpi_cstate_pstate_watermarks_st {
+	double cstate_exit_us;
+	double cstate_enter_plus_exit_us;
+	double pstate_change_us;
+};
+
+struct _vcs_dpi_wm_calc_pipe_params_st {
+	unsigned int num_dpp;
+	int voltage;
+	int output_type;
+	double dcfclk_mhz;
+	double socclk_mhz;
+	double dppclk_mhz;
+	double pixclk_mhz;
+	unsigned char interlace_en;
+	unsigned char pte_enable;
+	unsigned char dcc_enable;
+	double dcc_rate;
+	double bytes_per_pixel_c;
+	double bytes_per_pixel_y;
+	unsigned int swath_width_y;
+	unsigned int swath_height_y;
+	unsigned int swath_height_c;
+	unsigned int det_buffer_size_y;
+	double h_ratio;
+	double v_ratio;
+	unsigned int h_taps;
+	unsigned int h_total;
+	unsigned int v_total;
+	unsigned int v_active;
+	unsigned int e2e_index;
+	double display_pipe_line_delivery_time;
+	double read_bw;
+	unsigned int lines_in_det_y;
+	unsigned int lines_in_det_y_rounded_down_to_swath;
+	double full_det_buffering_time;
+	double dcfclk_deepsleep_mhz_per_plane;
+};
+
+struct _vcs_dpi_vratio_pre_st {
+	double vratio_pre_l;
+	double vratio_pre_c;
+};
+
+struct _vcs_dpi_display_data_rq_misc_params_st {
+	unsigned int full_swath_bytes;
+	unsigned int stored_swath_bytes;
+	unsigned int blk256_height;
+	unsigned int blk256_width;
+	unsigned int req_height;
+	unsigned int req_width;
+};
+
+struct _vcs_dpi_display_data_rq_sizing_params_st {
+	unsigned int chunk_bytes;
+	unsigned int min_chunk_bytes;
+	unsigned int meta_chunk_bytes;
+	unsigned int min_meta_chunk_bytes;
+	unsigned int mpte_group_bytes;
+	unsigned int dpte_group_bytes;
+};
+
+struct _vcs_dpi_display_data_rq_dlg_params_st {
+	unsigned int swath_width_ub;
+	unsigned int swath_height;
+	unsigned int req_per_swath_ub;
+	unsigned int meta_pte_bytes_per_frame_ub;
+	unsigned int dpte_req_per_row_ub;
+	unsigned int dpte_groups_per_row_ub;
+	unsigned int dpte_row_height;
+	unsigned int dpte_bytes_per_row_ub;
+	unsigned int meta_chunks_per_row_ub;
+	unsigned int meta_req_per_row_ub;
+	unsigned int meta_row_height;
+	unsigned int meta_bytes_per_row_ub;
+};
+
+struct _vcs_dpi_display_cur_rq_dlg_params_st {
+	unsigned char enable;
+	unsigned int swath_height;
+	unsigned int req_per_line;
+};
+
+struct _vcs_dpi_display_rq_dlg_params_st {
+	display_data_rq_dlg_params_st rq_l;
+	display_data_rq_dlg_params_st rq_c;
+	display_cur_rq_dlg_params_st rq_cur0;
+};
+
+struct _vcs_dpi_display_rq_sizing_params_st {
+	display_data_rq_sizing_params_st rq_l;
+	display_data_rq_sizing_params_st rq_c;
+};
+
+struct _vcs_dpi_display_rq_misc_params_st {
+	display_data_rq_misc_params_st rq_l;
+	display_data_rq_misc_params_st rq_c;
+};
+
+struct _vcs_dpi_display_rq_params_st {
+	unsigned char yuv420;
+	unsigned char yuv420_10bpc;
+	display_rq_misc_params_st misc;
+	display_rq_sizing_params_st sizing;
+	display_rq_dlg_params_st dlg;
+};
+
+struct _vcs_dpi_display_dlg_regs_st {
+	unsigned int refcyc_h_blank_end;
+	unsigned int dlg_vblank_end;
+	unsigned int min_dst_y_next_start;
+	unsigned int refcyc_per_htotal;
+	unsigned int refcyc_x_after_scaler;
+	unsigned int dst_y_after_scaler;
+	unsigned int dst_y_prefetch;
+	unsigned int dst_y_per_vm_vblank;
+	unsigned int dst_y_per_row_vblank;
+	unsigned int dst_y_per_vm_flip;
+	unsigned int dst_y_per_row_flip;
+	unsigned int ref_freq_to_pix_freq;
+	unsigned int vratio_prefetch;
+	unsigned int vratio_prefetch_c;
+	unsigned int refcyc_per_pte_group_vblank_l;
+	unsigned int refcyc_per_pte_group_vblank_c;
+	unsigned int refcyc_per_meta_chunk_vblank_l;
+	unsigned int refcyc_per_meta_chunk_vblank_c;
+	unsigned int refcyc_per_pte_group_flip_l;
+	unsigned int refcyc_per_pte_group_flip_c;
+	unsigned int refcyc_per_meta_chunk_flip_l;
+	unsigned int refcyc_per_meta_chunk_flip_c;
+	unsigned int dst_y_per_pte_row_nom_l;
+	unsigned int dst_y_per_pte_row_nom_c;
+	unsigned int refcyc_per_pte_group_nom_l;
+	unsigned int refcyc_per_pte_group_nom_c;
+	unsigned int dst_y_per_meta_row_nom_l;
+	unsigned int dst_y_per_meta_row_nom_c;
+	unsigned int refcyc_per_meta_chunk_nom_l;
+	unsigned int refcyc_per_meta_chunk_nom_c;
+	unsigned int refcyc_per_line_delivery_pre_l;
+	unsigned int refcyc_per_line_delivery_pre_c;
+	unsigned int refcyc_per_line_delivery_l;
+	unsigned int refcyc_per_line_delivery_c;
+	unsigned int chunk_hdl_adjust_cur0;
+	unsigned int chunk_hdl_adjust_cur1;
+	unsigned int vready_after_vcount0;
+	unsigned int dst_y_offset_cur0;
+	unsigned int dst_y_offset_cur1;
+	unsigned int xfc_reg_transfer_delay;
+	unsigned int xfc_reg_precharge_delay;
+	unsigned int xfc_reg_remote_surface_flip_latency;
+	unsigned int xfc_reg_prefetch_margin;
+	unsigned int dst_y_delta_drq_limit;
+};
+
+struct _vcs_dpi_display_ttu_regs_st {
+	unsigned int qos_level_low_wm;
+	unsigned int qos_level_high_wm;
+	unsigned int min_ttu_vblank;
+	unsigned int qos_level_flip;
+	unsigned int refcyc_per_req_delivery_l;
+	unsigned int refcyc_per_req_delivery_c;
+	unsigned int refcyc_per_req_delivery_cur0;
+	unsigned int refcyc_per_req_delivery_cur1;
+	unsigned int refcyc_per_req_delivery_pre_l;
+	unsigned int refcyc_per_req_delivery_pre_c;
+	unsigned int refcyc_per_req_delivery_pre_cur0;
+	unsigned int refcyc_per_req_delivery_pre_cur1;
+	unsigned int qos_level_fixed_l;
+	unsigned int qos_level_fixed_c;
+	unsigned int qos_level_fixed_cur0;
+	unsigned int qos_level_fixed_cur1;
+	unsigned int qos_ramp_disable_l;
+	unsigned int qos_ramp_disable_c;
+	unsigned int qos_ramp_disable_cur0;
+	unsigned int qos_ramp_disable_cur1;
+};
+
+struct _vcs_dpi_display_data_rq_regs_st {
+	unsigned int chunk_size;
+	unsigned int min_chunk_size;
+	unsigned int meta_chunk_size;
+	unsigned int min_meta_chunk_size;
+	unsigned int dpte_group_size;
+	unsigned int mpte_group_size;
+	unsigned int swath_height;
+	unsigned int pte_row_height_linear;
+};
+
+struct _vcs_dpi_display_rq_regs_st {
+	display_data_rq_regs_st rq_regs_l;
+	display_data_rq_regs_st rq_regs_c;
+	unsigned int drq_expansion_mode;
+	unsigned int prq_expansion_mode;
+	unsigned int mrq_expansion_mode;
+	unsigned int crq_expansion_mode;
+	unsigned int plane1_base_address;
+};
+
+struct _vcs_dpi_display_dlg_sys_params_st {
+	double t_mclk_wm_us;
+	double t_urg_wm_us;
+	double t_sr_wm_us;
+	double t_extra_us;
+	double mem_trip_us;
+	double t_srx_delay_us;
+	double deepsleep_dcfclk_mhz;
+	double total_flip_bw;
+	unsigned int total_flip_bytes;
+};
+
+struct _vcs_dpi_display_dlg_prefetch_param_st {
+	double prefetch_bw;
+	unsigned int flip_bytes;
+};
+
+struct _vcs_dpi_display_pipe_clock_st {
+	double dcfclk_mhz;
+	double dispclk_mhz;
+	double socclk_mhz;
+	double dscclk_mhz[6];
+	double dppclk_mhz[6];
+};
+
+struct _vcs_dpi_display_arb_params_st {
+	int max_req_outstanding;
+	int min_req_outstanding;
+	int sat_level_us;
 };
 
 #endif /*__DISPLAY_MODE_STRUCTS_H__*/
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h b/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
index f9cf08357989..e8ce08567cd8 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
@@ -35,6 +35,16 @@ static inline double dml_min(double a, double b)
 	return (double) dcn_bw_min2(a, b);
 }
 
+static inline double dml_min3(double a, double b, double c)
+{
+	return dml_min(dml_min(a, b), c);
+}
+
+static inline double dml_min4(double a, double b, double c, double d)
+{
+	return dml_min(dml_min(a, b), dml_min(c, d));
+}
+
 static inline double dml_max(double a, double b)
 {
 	return (double) dcn_bw_max2(a, b);
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
index 87b580fa4bc9..0caee3523017 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
@@ -75,6 +75,7 @@ bool dal_hw_factory_init(
 		return true;
 	case DCE_VERSION_11_0:
 	case DCE_VERSION_11_2:
+	case DCE_VERSION_11_22:
 		dal_hw_factory_dce110_init(factory);
 		return true;
 	case DCE_VERSION_12_0:
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
index 0ae8ace25739..55c707488541 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
@@ -72,6 +72,7 @@ bool dal_hw_translate_init(
 	case DCE_VERSION_10_0:
 	case DCE_VERSION_11_0:
 	case DCE_VERSION_11_2:
+	case DCE_VERSION_11_22:
 		dal_hw_translate_dce110_init(translate);
 		return true;
 	case DCE_VERSION_12_0:
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.c b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.c
index abd0095ced30..b7256f595052 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.c
@@ -527,7 +527,7 @@ static void construct(
 	REG_GET(MICROSECOND_TIME_BASE_DIV, XTAL_REF_DIV, &xtal_ref_div);
 
 	if (xtal_ref_div == 0) {
-		DC_LOG_WARNING("Invalid base timer divider\n",
+		DC_LOG_WARNING("Invalid base timer divider [%s]\n",
 				__func__);
 		xtal_ref_div = 2;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/i2caux.c b/drivers/gpu/drm/amd/display/dc/i2caux/i2caux.c
index 5cbf6626b8d4..14dc8c94d862 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/i2caux.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/i2caux.c
@@ -83,6 +83,7 @@ struct i2caux *dal_i2caux_create(
 	case DCE_VERSION_8_3:
 		return dal_i2caux_dce80_create(ctx);
 	case DCE_VERSION_11_2:
+	case DCE_VERSION_11_22:
 		return dal_i2caux_dce112_create(ctx);
 	case DCE_VERSION_11_0:
 		return dal_i2caux_dce110_create(ctx);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
index 8c51ad70cace..a94942d4e66b 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
@@ -95,11 +95,6 @@ struct resource_funcs {
 	struct link_encoder *(*link_enc_create)(
 			const struct encoder_init_data *init);
 
-	enum dc_status (*validate_guaranteed)(
-					struct dc *dc,
-					struct dc_stream_state *stream,
-					struct dc_state *context);
-
 	bool (*validate_bandwidth)(
 					struct dc *dc,
 					struct dc_state *context);
@@ -250,6 +245,7 @@ struct dce_bw_output {
 	bool all_displays_in_sync;
 	struct dce_watermarks urgent_wm_ns[MAX_PIPES];
 	struct dce_watermarks stutter_exit_wm_ns[MAX_PIPES];
+	struct dce_watermarks stutter_entry_wm_ns[MAX_PIPES];
 	struct dce_watermarks nbp_state_change_wm_ns[MAX_PIPES];
 	int sclk_khz;
 	int sclk_deep_sleep_khz;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dce_calcs.h b/drivers/gpu/drm/amd/display/dc/inc/dce_calcs.h
index a9bfe9ff8ce6..eece165206f9 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dce_calcs.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/dce_calcs.h
@@ -42,6 +42,8 @@ enum bw_calcs_version {
 	BW_CALCS_VERSION_CARRIZO,
 	BW_CALCS_VERSION_POLARIS10,
 	BW_CALCS_VERSION_POLARIS11,
+	BW_CALCS_VERSION_POLARIS12,
+	BW_CALCS_VERSION_VEGAM,
 	BW_CALCS_VERSION_STONEY,
 	BW_CALCS_VERSION_VEGA10
 };
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
new file mode 100644
index 000000000000..02f757dd70d4
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DAL_DCHUBBUB_H__
+#define __DAL_DCHUBBUB_H__
+
+
+enum dcc_control {
+	dcc_control__256_256_xxx,
+	dcc_control__128_128_xxx,
+	dcc_control__256_64_64,
+};
+
+enum segment_order {
+	segment_order__na,
+	segment_order__contiguous,
+	segment_order__non_contiguous,
+};
+
+
+struct hubbub_funcs {
+	void (*update_dchub)(
+			struct hubbub *hubbub,
+			struct dchub_init_data *dh_data);
+
+	bool (*get_dcc_compression_cap)(struct hubbub *hubbub,
+			const struct dc_dcc_surface_param *input,
+			struct dc_surface_dcc_cap *output);
+
+	bool (*dcc_support_swizzle)(
+			enum swizzle_mode_values swizzle,
+			unsigned int bytes_per_element,
+			enum segment_order *segment_order_horz,
+			enum segment_order *segment_order_vert);
+
+	bool (*dcc_support_pixel_format)(
+			enum surface_pixel_format format,
+			unsigned int *bytes_per_element);
+};
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
index 99995608b620..582458f028f8 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
@@ -44,7 +44,23 @@ struct dpp_grph_csc_adjustment {
 	enum graphics_gamut_adjust_type gamut_adjust_type;
 };
 
+struct dcn_dpp_state {
+	uint32_t igam_lut_mode;
+	uint32_t igam_input_format;
+	uint32_t dgam_lut_mode;
+	uint32_t rgam_lut_mode;
+	uint32_t gamut_remap_mode;
+	uint32_t gamut_remap_c11_c12;
+	uint32_t gamut_remap_c13_c14;
+	uint32_t gamut_remap_c21_c22;
+	uint32_t gamut_remap_c23_c24;
+	uint32_t gamut_remap_c31_c32;
+	uint32_t gamut_remap_c33_c34;
+};
+
 struct dpp_funcs {
+	void (*dpp_read_state)(struct dpp *dpp, struct dcn_dpp_state *s);
+
 	void (*dpp_reset)(struct dpp *dpp);
 
 	void (*dpp_set_scaler)(struct dpp *dpp,
@@ -117,7 +133,7 @@ struct dpp_funcs {
 			struct dpp *dpp_base,
 			enum surface_pixel_format format,
 			enum expansion_mode mode,
-			struct csc_transform input_csc_color_matrix,
+			struct dc_csc_transform input_csc_color_matrix,
 			enum dc_color_space input_color_space);
 
 	void (*dpp_full_bypass)(struct dpp *dpp_base);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
index 9ced254e652c..97df82cddf82 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
@@ -56,7 +56,6 @@ struct hubp {
 	bool power_gated;
 };
 
-
 struct hubp_funcs {
 	void (*hubp_setup)(
 			struct hubp *hubp,
@@ -121,6 +120,9 @@ struct hubp_funcs {
 
 	void (*hubp_clk_cntl)(struct hubp *hubp, bool enable);
 	void (*hubp_vtg_sel)(struct hubp *hubp, uint32_t otg_inst);
+	void (*hubp_read_state)(struct hubp *hubp);
+	void (*hubp_disable_control)(struct hubp *hubp, bool disable_hubp);
+	unsigned int (*hubp_get_underflow_status)(struct hubp *hubp);
 
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
index b22158190262..cf7433ebf91a 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
@@ -140,11 +140,6 @@ enum opp_regamma {
 	OPP_REGAMMA_USER
 };
 
-struct csc_transform {
-	uint16_t matrix[12];
-	bool enable_adjustment;
-};
-
 struct dc_bias_and_scale {
 	uint16_t scale_red;
 	uint16_t bias_red;
@@ -191,4 +186,9 @@ enum controller_dp_test_pattern {
 	CONTROLLER_DP_TEST_PATTERN_COLORSQUARES_CEA
 };
 
+enum dc_lut_mode {
+	LUT_BYPASS,
+	LUT_RAM_A,
+	LUT_RAM_B
+};
 #endif /* __DAL_HW_SHARED_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/ipp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/ipp.h
index 2109eac20a3d..b2fa4c4cd920 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/ipp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/ipp.h
@@ -87,7 +87,7 @@ struct ipp_funcs {
 		struct input_pixel_processor *ipp,
 		enum surface_pixel_format format,
 		enum expansion_mode mode,
-		struct csc_transform input_csc_color_matrix,
+		struct dc_csc_transform input_csc_color_matrix,
 		enum dc_color_space input_color_space);
 
 	/* DCE function to setup IPP.  TODO: see if we can consolidate to setup */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
index 54d8a1386142..cf6df2e7beb2 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
@@ -149,6 +149,7 @@ struct link_encoder_funcs {
 		bool connect);
 	void (*enable_hpd)(struct link_encoder *enc);
 	void (*disable_hpd)(struct link_encoder *enc);
+	bool (*is_dig_enabled)(struct link_encoder *enc);
 	void (*destroy)(struct link_encoder **enc);
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h
index 3e1e7e6a8792..47f1dc5a43b7 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h
@@ -104,6 +104,7 @@ struct mem_input_funcs {
 		struct mem_input *mem_input,
 		struct dce_watermarks nbp,
 		struct dce_watermarks stutter,
+		struct dce_watermarks stutter_enter,
 		struct dce_watermarks urgent,
 		uint32_t total_dest_line_time_ns);
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
index 23a8d5e53a89..caf74e3c836f 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
@@ -105,7 +105,24 @@ struct mpc {
 	struct mpcc mpcc_array[MAX_MPCC];
 };
 
+struct mpcc_state {
+	uint32_t opp_id;
+	uint32_t dpp_id;
+	uint32_t bot_mpcc_id;
+	uint32_t mode;
+	uint32_t alpha_mode;
+	uint32_t pre_multiplied_alpha;
+	uint32_t overlap_only;
+	uint32_t idle;
+	uint32_t busy;
+};
+
 struct mpc_funcs {
+	void (*read_mpcc_state)(
+			struct mpc *mpc,
+			int mpcc_inst,
+			struct mpcc_state *s);
+
 	/*
 	 * Insert DPP into MPC tree based on specified blending position.
 	 * Only used for planes that are part of blending chain for OPP output
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
index b5db1692393c..cfa7ec9517ae 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
@@ -29,31 +29,40 @@
 #define STREAM_ENCODER_H_
 
 #include "audio_types.h"
+#include "hw_shared.h"
 
 struct dc_bios;
 struct dc_context;
 struct dc_crtc_timing;
 
-struct encoder_info_packet {
-	bool valid;
-	uint8_t hb0;
-	uint8_t hb1;
-	uint8_t hb2;
-	uint8_t hb3;
-	uint8_t sb[32];
+enum dp_pixel_encoding_type {
+	DP_PIXEL_ENCODING_TYPE_RGB444		= 0x00000000,
+	DP_PIXEL_ENCODING_TYPE_YCBCR422		= 0x00000001,
+	DP_PIXEL_ENCODING_TYPE_YCBCR444		= 0x00000002,
+	DP_PIXEL_ENCODING_TYPE_RGB_WIDE_GAMUT	= 0x00000003,
+	DP_PIXEL_ENCODING_TYPE_Y_ONLY		= 0x00000004,
+	DP_PIXEL_ENCODING_TYPE_YCBCR420		= 0x00000005
+};
+
+enum dp_component_depth {
+	DP_COMPONENT_PIXEL_DEPTH_6BPC		= 0x00000000,
+	DP_COMPONENT_PIXEL_DEPTH_8BPC		= 0x00000001,
+	DP_COMPONENT_PIXEL_DEPTH_10BPC		= 0x00000002,
+	DP_COMPONENT_PIXEL_DEPTH_12BPC		= 0x00000003,
+	DP_COMPONENT_PIXEL_DEPTH_16BPC		= 0x00000004
 };
 
 struct encoder_info_frame {
 	/* auxiliary video information */
-	struct encoder_info_packet avi;
-	struct encoder_info_packet gamut;
-	struct encoder_info_packet vendor;
+	struct dc_info_packet avi;
+	struct dc_info_packet gamut;
+	struct dc_info_packet vendor;
 	/* source product description */
-	struct encoder_info_packet spd;
+	struct dc_info_packet spd;
 	/* video stream configuration */
-	struct encoder_info_packet vsc;
+	struct dc_info_packet vsc;
 	/* HDR Static MetaData */
-	struct encoder_info_packet hdrsmd;
+	struct dc_info_packet hdrsmd;
 };
 
 struct encoder_unblank_param {
@@ -147,6 +156,7 @@ struct stream_encoder_funcs {
 
 	void (*set_avmute)(
 		struct stream_encoder *enc, bool enable);
+
 };
 
 #endif /* STREAM_ENCODER_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
index 3217b5bf6c7a..69cb0a105300 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
@@ -140,6 +140,9 @@ struct timing_generator_funcs {
 	void (*program_timing)(struct timing_generator *tg,
 							const struct dc_crtc_timing *timing,
 							bool use_vbios);
+	void (*program_vline_interrupt)(struct timing_generator *optc,
+			const struct dc_crtc_timing *dc_crtc_timing,
+			unsigned long long vsync_delta);
 	bool (*enable_crtc)(struct timing_generator *tg);
 	bool (*disable_crtc)(struct timing_generator *tg);
 	bool (*is_counter_moving)(struct timing_generator *tg);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/transform.h b/drivers/gpu/drm/amd/display/dc/inc/hw/transform.h
index c5b3623bcbd9..fecc80c47c26 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/transform.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/transform.h
@@ -252,7 +252,7 @@ struct transform_funcs {
 			struct transform *xfm_base,
 			enum surface_pixel_format format,
 			enum expansion_mode mode,
-			struct csc_transform input_csc_color_matrix,
+			struct dc_csc_transform input_csc_color_matrix,
 			enum dc_color_space input_color_space);
 
 	void (*ipp_full_bypass)(struct transform *xfm_base);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
index e764cbad881b..63fc6c499789 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
@@ -32,6 +32,8 @@
 #include "inc/hw/link_encoder.h"
 #include "core_status.h"
 
+#define EDP_BACKLIGHT_RAMP_DISABLE_LEVEL 0xFFFFFFFF
+
 enum pipe_gating_control {
 	PIPE_GATING_CONTROL_DISABLE = 0,
 	PIPE_GATING_CONTROL_ENABLE,
@@ -63,6 +65,7 @@ struct dchub_init_data;
 struct dc_static_screen_events;
 struct resource_pool;
 struct resource_context;
+struct stream_resource;
 
 struct hw_sequencer_funcs {
 
@@ -80,11 +83,6 @@ struct hw_sequencer_funcs {
 			int num_planes,
 			struct dc_state *context);
 
-	void (*set_plane_config)(
-			const struct dc *dc,
-			struct pipe_ctx *pipe_ctx,
-			struct resource_context *res_ctx);
-
 	void (*program_gamut_remap)(
 			struct pipe_ctx *pipe_ctx);
 
@@ -93,6 +91,12 @@ struct hw_sequencer_funcs {
 			enum dc_color_space colorspace,
 			uint16_t *matrix);
 
+	void (*program_output_csc)(struct dc *dc,
+			struct pipe_ctx *pipe_ctx,
+			enum dc_color_space colorspace,
+			uint16_t *matrix,
+			int opp_id);
+
 	void (*update_plane_addr)(
 		const struct dc *dc,
 		struct pipe_ctx *pipe_ctx);
@@ -154,6 +158,11 @@ struct hw_sequencer_funcs {
 				struct dc *dc,
 				struct pipe_ctx *pipe,
 				bool lock);
+	void (*blank_pixel_data)(
+			struct dc *dc,
+			struct stream_resource *stream_res,
+			struct dc_stream_state *stream,
+			bool blank);
 
 	void (*set_bandwidth)(
 			struct dc *dc,
@@ -169,7 +178,7 @@ struct hw_sequencer_funcs {
 	void (*set_static_screen_control)(struct pipe_ctx **pipe_ctx,
 			int num_pipes, const struct dc_static_screen_events *events);
 
-	enum dc_status (*prog_pixclk_crtc_otg)(
+	enum dc_status (*enable_stream_timing)(
 			struct pipe_ctx *pipe_ctx,
 			struct dc_state *context,
 			struct dc *dc);
@@ -201,6 +210,7 @@ struct hw_sequencer_funcs {
 
 	void (*set_cursor_position)(struct pipe_ctx *pipe);
 	void (*set_cursor_attribute)(struct pipe_ctx *pipe);
+
 };
 
 void color_space_to_black_color(
diff --git a/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h b/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
index 77eb72874e90..3306e7b0b3e3 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
@@ -183,6 +183,36 @@
 				FN(reg_name, f4), v4, \
 				FN(reg_name, f5), v5)
 
+#define REG_GET_6(reg_name, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6)	\
+		generic_reg_get6(CTX, REG(reg_name), \
+				FN(reg_name, f1), v1, \
+				FN(reg_name, f2), v2, \
+				FN(reg_name, f3), v3, \
+				FN(reg_name, f4), v4, \
+				FN(reg_name, f5), v5, \
+				FN(reg_name, f6), v6)
+
+#define REG_GET_7(reg_name, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7)	\
+		generic_reg_get7(CTX, REG(reg_name), \
+				FN(reg_name, f1), v1, \
+				FN(reg_name, f2), v2, \
+				FN(reg_name, f3), v3, \
+				FN(reg_name, f4), v4, \
+				FN(reg_name, f5), v5, \
+				FN(reg_name, f6), v6, \
+				FN(reg_name, f7), v7)
+
+#define REG_GET_8(reg_name, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8)	\
+		generic_reg_get8(CTX, REG(reg_name), \
+				FN(reg_name, f1), v1, \
+				FN(reg_name, f2), v2, \
+				FN(reg_name, f3), v3, \
+				FN(reg_name, f4), v4, \
+				FN(reg_name, f5), v5, \
+				FN(reg_name, f6), v6, \
+				FN(reg_name, f7), v7, \
+				FN(reg_name, f8), v8)
+
 /* macro to poll and wait for a register field to read back given value */
 
 #define REG_WAIT(reg_name, field, val, delay_between_poll_us, max_try)	\
@@ -389,4 +419,30 @@ uint32_t generic_reg_get5(const struct dc_context *ctx, uint32_t addr,
 		uint8_t shift4, uint32_t mask4, uint32_t *field_value4,
 		uint8_t shift5, uint32_t mask5, uint32_t *field_value5);
 
+uint32_t generic_reg_get6(const struct dc_context *ctx, uint32_t addr,
+		uint8_t shift1, uint32_t mask1, uint32_t *field_value1,
+		uint8_t shift2, uint32_t mask2, uint32_t *field_value2,
+		uint8_t shift3, uint32_t mask3, uint32_t *field_value3,
+		uint8_t shift4, uint32_t mask4, uint32_t *field_value4,
+		uint8_t shift5, uint32_t mask5, uint32_t *field_value5,
+		uint8_t shift6, uint32_t mask6, uint32_t *field_value6);
+
+uint32_t generic_reg_get7(const struct dc_context *ctx, uint32_t addr,
+		uint8_t shift1, uint32_t mask1, uint32_t *field_value1,
+		uint8_t shift2, uint32_t mask2, uint32_t *field_value2,
+		uint8_t shift3, uint32_t mask3, uint32_t *field_value3,
+		uint8_t shift4, uint32_t mask4, uint32_t *field_value4,
+		uint8_t shift5, uint32_t mask5, uint32_t *field_value5,
+		uint8_t shift6, uint32_t mask6, uint32_t *field_value6,
+		uint8_t shift7, uint32_t mask7, uint32_t *field_value7);
+
+uint32_t generic_reg_get8(const struct dc_context *ctx, uint32_t addr,
+		uint8_t shift1, uint32_t mask1, uint32_t *field_value1,
+		uint8_t shift2, uint32_t mask2, uint32_t *field_value2,
+		uint8_t shift3, uint32_t mask3, uint32_t *field_value3,
+		uint8_t shift4, uint32_t mask4, uint32_t *field_value4,
+		uint8_t shift5, uint32_t mask5, uint32_t *field_value5,
+		uint8_t shift6, uint32_t mask6, uint32_t *field_value6,
+		uint8_t shift7, uint32_t mask7, uint32_t *field_value7,
+		uint8_t shift8, uint32_t mask8, uint32_t *field_value8);
 #endif /* DRIVERS_GPU_DRM_AMD_DC_DEV_DC_INC_REG_HELPER_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h
index 5467332faf7b..640a647f4611 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/resource.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h
@@ -139,10 +139,6 @@ bool resource_validate_attach_surfaces(
 		struct dc_state *context,
 		const struct resource_pool *pool);
 
-void validate_guaranteed_copy_streams(
-		struct dc_state *context,
-		int max_streams);
-
 void resource_validate_ctx_update_pointer_after_copy(
 		const struct dc_state *src_ctx,
 		struct dc_state *dst_ctx);
diff --git a/drivers/gpu/drm/amd/display/dc/irq_types.h b/drivers/gpu/drm/amd/display/dc/irq_types.h
index a506c2e939f5..0b5f3a278c22 100644
--- a/drivers/gpu/drm/amd/display/dc/irq_types.h
+++ b/drivers/gpu/drm/amd/display/dc/irq_types.h
@@ -26,6 +26,8 @@
 #ifndef __DAL_IRQ_TYPES_H__
 #define __DAL_IRQ_TYPES_H__
 
+#include "os_types.h"
+
 struct dc_context;
 
 typedef void (*interrupt_handler)(void *);
@@ -135,6 +137,13 @@ enum dc_irq_source {
 	DC_IRQ_SOURCE_VBLANK5,
 	DC_IRQ_SOURCE_VBLANK6,
 
+	DC_IRQ_SOURCE_DC1_VLINE0,
+	DC_IRQ_SOURCE_DC2_VLINE0,
+	DC_IRQ_SOURCE_DC3_VLINE0,
+	DC_IRQ_SOURCE_DC4_VLINE0,
+	DC_IRQ_SOURCE_DC5_VLINE0,
+	DC_IRQ_SOURCE_DC6_VLINE0,
+
 	DAL_IRQ_SOURCES_NUMBER
 };
 
diff --git a/drivers/gpu/drm/amd/display/include/dal_asic_id.h b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
index 9b0a04f99ac8..25029ed42d89 100644
--- a/drivers/gpu/drm/amd/display/include/dal_asic_id.h
+++ b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
@@ -86,6 +86,7 @@
 #define VI_POLARIS10_P_A0 80
 #define VI_POLARIS11_M_A0 90
 #define VI_POLARIS12_V_A0 100
+#define VI_VEGAM_A0 110
 
 #define VI_UNKNOWN 0xFF
 
@@ -98,7 +99,9 @@
 		(eChipRev < VI_POLARIS11_M_A0))
 #define ASIC_REV_IS_POLARIS11_M(eChipRev) ((eChipRev >= VI_POLARIS11_M_A0) &&  \
 		(eChipRev < VI_POLARIS12_V_A0))
-#define ASIC_REV_IS_POLARIS12_V(eChipRev) (eChipRev >= VI_POLARIS12_V_A0)
+#define ASIC_REV_IS_POLARIS12_V(eChipRev) ((eChipRev >= VI_POLARIS12_V_A0) && \
+		(eChipRev < VI_VEGAM_A0))
+#define ASIC_REV_IS_VEGAM(eChipRev) (eChipRev >= VI_VEGAM_A0)
 
 /* DCE11 */
 #define CZ_CARRIZO_A0 0x01
@@ -110,17 +113,19 @@
 	((rev >= STONEY_A0) && (rev < CZ_UNKNOWN))
 
 /* DCE12 */
+#define AI_UNKNOWN 0xFF
 
 #define AI_GREENLAND_P_A0 1
 #define AI_GREENLAND_P_A1 2
 #define AI_UNKNOWN 0xFF
 
 #define AI_VEGA12_P_A0 20
+#define AI_VEGA20_P_A0 40
 #define ASICREV_IS_GREENLAND_M(eChipRev)  (eChipRev < AI_VEGA12_P_A0)
 #define ASICREV_IS_GREENLAND_P(eChipRev)  (eChipRev < AI_VEGA12_P_A0)
 
-#define ASICREV_IS_VEGA12_P(eChipRev) ((eChipRev >= AI_VEGA12_P_A0) && (eChipRev < AI_UNKNOWN))
-#define ASICREV_IS_VEGA12_p(eChipRev) ((eChipRev >= AI_VEGA12_P_A0) && (eChipRev < AI_UNKNOWN))
+#define ASICREV_IS_VEGA12_P(eChipRev) ((eChipRev >= AI_VEGA12_P_A0) && (eChipRev < AI_VEGA20_P_A0))
+#define ASICREV_IS_VEGA20_P(eChipRev) ((eChipRev >= AI_VEGA20_P_A0) && (eChipRev < AI_UNKNOWN))
 
 /* DCN1_0 */
 #define INTERNAL_REV_RAVEN_A0             0x00    /* First spin of Raven */
diff --git a/drivers/gpu/drm/amd/display/include/dal_types.h b/drivers/gpu/drm/amd/display/include/dal_types.h
index fa543965feb5..840142b65f8b 100644
--- a/drivers/gpu/drm/amd/display/include/dal_types.h
+++ b/drivers/gpu/drm/amd/display/include/dal_types.h
@@ -40,6 +40,7 @@ enum dce_version {
 	DCE_VERSION_10_0,
 	DCE_VERSION_11_0,
 	DCE_VERSION_11_2,
+	DCE_VERSION_11_22,
 	DCE_VERSION_12_0,
 	DCE_VERSION_MAX,
 	DCN_VERSION_1_0,
diff --git a/drivers/gpu/drm/amd/display/include/fixed31_32.h b/drivers/gpu/drm/amd/display/include/fixed31_32.h
index 0de258622c12..bb0d4ebba9f0 100644
--- a/drivers/gpu/drm/amd/display/include/fixed31_32.h
+++ b/drivers/gpu/drm/amd/display/include/fixed31_32.h
@@ -26,9 +26,13 @@
 #ifndef __DAL_FIXED31_32_H__
 #define __DAL_FIXED31_32_H__
 
-#include "os_types.h"
-
 #define FIXED31_32_BITS_PER_FRACTIONAL_PART 32
+#ifndef LLONG_MIN
+#define LLONG_MIN (1LL<<63)
+#endif
+#ifndef LLONG_MAX
+#define LLONG_MAX (-1LL>>1)
+#endif
 
 /*
  * @brief
@@ -44,24 +48,25 @@
  */
 
 struct fixed31_32 {
-	int64_t value;
+	long long value;
 };
 
+
 /*
  * @brief
  * Useful constants
  */
 
-static const struct fixed31_32 dal_fixed31_32_zero = { 0 };
-static const struct fixed31_32 dal_fixed31_32_epsilon = { 1LL };
-static const struct fixed31_32 dal_fixed31_32_half = { 0x80000000LL };
-static const struct fixed31_32 dal_fixed31_32_one = { 0x100000000LL };
+static const struct fixed31_32 dc_fixpt_zero = { 0 };
+static const struct fixed31_32 dc_fixpt_epsilon = { 1LL };
+static const struct fixed31_32 dc_fixpt_half = { 0x80000000LL };
+static const struct fixed31_32 dc_fixpt_one = { 0x100000000LL };
 
-static const struct fixed31_32 dal_fixed31_32_pi = { 13493037705LL };
-static const struct fixed31_32 dal_fixed31_32_two_pi = { 26986075409LL };
-static const struct fixed31_32 dal_fixed31_32_e = { 11674931555LL };
-static const struct fixed31_32 dal_fixed31_32_ln2 = { 2977044471LL };
-static const struct fixed31_32 dal_fixed31_32_ln2_div_2 = { 1488522236LL };
+static const struct fixed31_32 dc_fixpt_pi = { 13493037705LL };
+static const struct fixed31_32 dc_fixpt_two_pi = { 26986075409LL };
+static const struct fixed31_32 dc_fixpt_e = { 11674931555LL };
+static const struct fixed31_32 dc_fixpt_ln2 = { 2977044471LL };
+static const struct fixed31_32 dc_fixpt_ln2_div_2 = { 1488522236LL };
 
 /*
  * @brief
@@ -72,24 +77,19 @@ static const struct fixed31_32 dal_fixed31_32_ln2_div_2 = { 1488522236LL };
  * @brief
  * result = numerator / denominator
  */
-struct fixed31_32 dal_fixed31_32_from_fraction(
-	int64_t numerator,
-	int64_t denominator);
+struct fixed31_32 dc_fixpt_from_fraction(long long numerator, long long denominator);
 
 /*
  * @brief
  * result = arg
  */
-struct fixed31_32 dal_fixed31_32_from_int_nonconst(int64_t arg);
-static inline struct fixed31_32 dal_fixed31_32_from_int(int64_t arg)
+static inline struct fixed31_32 dc_fixpt_from_int(int arg)
 {
-	if (__builtin_constant_p(arg)) {
-		struct fixed31_32 res;
-		BUILD_BUG_ON((LONG_MIN > arg) || (arg > LONG_MAX));
-		res.value = arg << FIXED31_32_BITS_PER_FRACTIONAL_PART;
-		return res;
-	} else
-		return dal_fixed31_32_from_int_nonconst(arg);
+	struct fixed31_32 res;
+
+	res.value = (long long) arg << FIXED31_32_BITS_PER_FRACTIONAL_PART;
+
+	return res;
 }
 
 /*
@@ -101,7 +101,7 @@ static inline struct fixed31_32 dal_fixed31_32_from_int(int64_t arg)
  * @brief
  * result = -arg
  */
-static inline struct fixed31_32 dal_fixed31_32_neg(struct fixed31_32 arg)
+static inline struct fixed31_32 dc_fixpt_neg(struct fixed31_32 arg)
 {
 	struct fixed31_32 res;
 
@@ -114,10 +114,10 @@ static inline struct fixed31_32 dal_fixed31_32_neg(struct fixed31_32 arg)
  * @brief
  * result = abs(arg) := (arg >= 0) ? arg : -arg
  */
-static inline struct fixed31_32 dal_fixed31_32_abs(struct fixed31_32 arg)
+static inline struct fixed31_32 dc_fixpt_abs(struct fixed31_32 arg)
 {
 	if (arg.value < 0)
-		return dal_fixed31_32_neg(arg);
+		return dc_fixpt_neg(arg);
 	else
 		return arg;
 }
@@ -131,8 +131,7 @@ static inline struct fixed31_32 dal_fixed31_32_abs(struct fixed31_32 arg)
  * @brief
  * result = arg1 < arg2
  */
-static inline bool dal_fixed31_32_lt(struct fixed31_32 arg1,
-				     struct fixed31_32 arg2)
+static inline bool dc_fixpt_lt(struct fixed31_32 arg1, struct fixed31_32 arg2)
 {
 	return arg1.value < arg2.value;
 }
@@ -141,8 +140,7 @@ static inline bool dal_fixed31_32_lt(struct fixed31_32 arg1,
  * @brief
  * result = arg1 <= arg2
  */
-static inline bool dal_fixed31_32_le(struct fixed31_32 arg1,
-				     struct fixed31_32 arg2)
+static inline bool dc_fixpt_le(struct fixed31_32 arg1, struct fixed31_32 arg2)
 {
 	return arg1.value <= arg2.value;
 }
@@ -151,8 +149,7 @@ static inline bool dal_fixed31_32_le(struct fixed31_32 arg1,
  * @brief
  * result = arg1 == arg2
  */
-static inline bool dal_fixed31_32_eq(struct fixed31_32 arg1,
-				     struct fixed31_32 arg2)
+static inline bool dc_fixpt_eq(struct fixed31_32 arg1, struct fixed31_32 arg2)
 {
 	return arg1.value == arg2.value;
 }
@@ -161,8 +158,7 @@ static inline bool dal_fixed31_32_eq(struct fixed31_32 arg1,
  * @brief
  * result = min(arg1, arg2) := (arg1 <= arg2) ? arg1 : arg2
  */
-static inline struct fixed31_32 dal_fixed31_32_min(struct fixed31_32 arg1,
-						   struct fixed31_32 arg2)
+static inline struct fixed31_32 dc_fixpt_min(struct fixed31_32 arg1, struct fixed31_32 arg2)
 {
 	if (arg1.value <= arg2.value)
 		return arg1;
@@ -174,8 +170,7 @@ static inline struct fixed31_32 dal_fixed31_32_min(struct fixed31_32 arg1,
  * @brief
  * result = max(arg1, arg2) := (arg1 <= arg2) ? arg2 : arg1
  */
-static inline struct fixed31_32 dal_fixed31_32_max(struct fixed31_32 arg1,
-						   struct fixed31_32 arg2)
+static inline struct fixed31_32 dc_fixpt_max(struct fixed31_32 arg1, struct fixed31_32 arg2)
 {
 	if (arg1.value <= arg2.value)
 		return arg2;
@@ -189,14 +184,14 @@ static inline struct fixed31_32 dal_fixed31_32_max(struct fixed31_32 arg1,
  * result = | arg, when min_value < arg < max_value
  *          | max_value, when arg >= max_value
  */
-static inline struct fixed31_32 dal_fixed31_32_clamp(
+static inline struct fixed31_32 dc_fixpt_clamp(
 	struct fixed31_32 arg,
 	struct fixed31_32 min_value,
 	struct fixed31_32 max_value)
 {
-	if (dal_fixed31_32_le(arg, min_value))
+	if (dc_fixpt_le(arg, min_value))
 		return min_value;
-	else if (dal_fixed31_32_le(max_value, arg))
+	else if (dc_fixpt_le(max_value, arg))
 		return max_value;
 	else
 		return arg;
@@ -211,21 +206,30 @@ static inline struct fixed31_32 dal_fixed31_32_clamp(
  * @brief
  * result = arg << shift
  */
-struct fixed31_32 dal_fixed31_32_shl(
-	struct fixed31_32 arg,
-	uint8_t shift);
+static inline struct fixed31_32 dc_fixpt_shl(struct fixed31_32 arg, unsigned char shift)
+{
+	ASSERT(((arg.value >= 0) && (arg.value <= LLONG_MAX >> shift)) ||
+		((arg.value < 0) && (arg.value >= ~(LLONG_MAX >> shift))));
+
+	arg.value = arg.value << shift;
+
+	return arg;
+}
 
 /*
  * @brief
  * result = arg >> shift
  */
-static inline struct fixed31_32 dal_fixed31_32_shr(
-	struct fixed31_32 arg,
-	uint8_t shift)
+static inline struct fixed31_32 dc_fixpt_shr(struct fixed31_32 arg, unsigned char shift)
 {
-	struct fixed31_32 res;
-	res.value = arg.value >> shift;
-	return res;
+	bool negative = arg.value < 0;
+
+	if (negative)
+		arg.value = -arg.value;
+	arg.value = arg.value >> shift;
+	if (negative)
+		arg.value = -arg.value;
+	return arg;
 }
 
 /*
@@ -237,38 +241,50 @@ static inline struct fixed31_32 dal_fixed31_32_shr(
  * @brief
  * result = arg1 + arg2
  */
-struct fixed31_32 dal_fixed31_32_add(
-	struct fixed31_32 arg1,
-	struct fixed31_32 arg2);
+static inline struct fixed31_32 dc_fixpt_add(struct fixed31_32 arg1, struct fixed31_32 arg2)
+{
+	struct fixed31_32 res;
+
+	ASSERT(((arg1.value >= 0) && (LLONG_MAX - arg1.value >= arg2.value)) ||
+		((arg1.value < 0) && (LLONG_MIN - arg1.value <= arg2.value)));
+
+	res.value = arg1.value + arg2.value;
+
+	return res;
+}
 
 /*
  * @brief
  * result = arg1 + arg2
  */
-static inline struct fixed31_32 dal_fixed31_32_add_int(struct fixed31_32 arg1,
-						       int32_t arg2)
+static inline struct fixed31_32 dc_fixpt_add_int(struct fixed31_32 arg1, int arg2)
 {
-	return dal_fixed31_32_add(arg1,
-				  dal_fixed31_32_from_int(arg2));
+	return dc_fixpt_add(arg1, dc_fixpt_from_int(arg2));
 }
 
 /*
  * @brief
  * result = arg1 - arg2
  */
-struct fixed31_32 dal_fixed31_32_sub(
-	struct fixed31_32 arg1,
-	struct fixed31_32 arg2);
+static inline struct fixed31_32 dc_fixpt_sub(struct fixed31_32 arg1, struct fixed31_32 arg2)
+{
+	struct fixed31_32 res;
+
+	ASSERT(((arg2.value >= 0) && (LLONG_MIN + arg2.value <= arg1.value)) ||
+		((arg2.value < 0) && (LLONG_MAX + arg2.value >= arg1.value)));
+
+	res.value = arg1.value - arg2.value;
+
+	return res;
+}
 
 /*
  * @brief
  * result = arg1 - arg2
  */
-static inline struct fixed31_32 dal_fixed31_32_sub_int(struct fixed31_32 arg1,
-						       int32_t arg2)
+static inline struct fixed31_32 dc_fixpt_sub_int(struct fixed31_32 arg1, int arg2)
 {
-	return dal_fixed31_32_sub(arg1,
-				  dal_fixed31_32_from_int(arg2));
+	return dc_fixpt_sub(arg1, dc_fixpt_from_int(arg2));
 }
 
 
@@ -281,49 +297,40 @@ static inline struct fixed31_32 dal_fixed31_32_sub_int(struct fixed31_32 arg1,
  * @brief
  * result = arg1 * arg2
  */
-struct fixed31_32 dal_fixed31_32_mul(
-	struct fixed31_32 arg1,
-	struct fixed31_32 arg2);
+struct fixed31_32 dc_fixpt_mul(struct fixed31_32 arg1, struct fixed31_32 arg2);
 
 
 /*
  * @brief
  * result = arg1 * arg2
  */
-static inline struct fixed31_32 dal_fixed31_32_mul_int(struct fixed31_32 arg1,
-						       int32_t arg2)
+static inline struct fixed31_32 dc_fixpt_mul_int(struct fixed31_32 arg1, int arg2)
 {
-	return dal_fixed31_32_mul(arg1,
-				  dal_fixed31_32_from_int(arg2));
+	return dc_fixpt_mul(arg1, dc_fixpt_from_int(arg2));
 }
 
 /*
  * @brief
  * result = square(arg) := arg * arg
  */
-struct fixed31_32 dal_fixed31_32_sqr(
-	struct fixed31_32 arg);
+struct fixed31_32 dc_fixpt_sqr(struct fixed31_32 arg);
 
 /*
  * @brief
  * result = arg1 / arg2
  */
-static inline struct fixed31_32 dal_fixed31_32_div_int(struct fixed31_32 arg1,
-						       int64_t arg2)
+static inline struct fixed31_32 dc_fixpt_div_int(struct fixed31_32 arg1, long long arg2)
 {
-	return dal_fixed31_32_from_fraction(arg1.value,
-					    dal_fixed31_32_from_int(arg2).value);
+	return dc_fixpt_from_fraction(arg1.value, dc_fixpt_from_int(arg2).value);
 }
 
 /*
  * @brief
  * result = arg1 / arg2
  */
-static inline struct fixed31_32 dal_fixed31_32_div(struct fixed31_32 arg1,
-						   struct fixed31_32 arg2)
+static inline struct fixed31_32 dc_fixpt_div(struct fixed31_32 arg1, struct fixed31_32 arg2)
 {
-	return dal_fixed31_32_from_fraction(arg1.value,
-					    arg2.value);
+	return dc_fixpt_from_fraction(arg1.value, arg2.value);
 }
 
 /*
@@ -338,8 +345,7 @@ static inline struct fixed31_32 dal_fixed31_32_div(struct fixed31_32 arg1,
  * @note
  * No special actions taken in case argument is zero.
  */
-struct fixed31_32 dal_fixed31_32_recip(
-	struct fixed31_32 arg);
+struct fixed31_32 dc_fixpt_recip(struct fixed31_32 arg);
 
 /*
  * @brief
@@ -354,8 +360,7 @@ struct fixed31_32 dal_fixed31_32_recip(
  * Argument specified in radians,
  * internally it's normalized to [-2pi...2pi] range.
  */
-struct fixed31_32 dal_fixed31_32_sinc(
-	struct fixed31_32 arg);
+struct fixed31_32 dc_fixpt_sinc(struct fixed31_32 arg);
 
 /*
  * @brief
@@ -365,8 +370,7 @@ struct fixed31_32 dal_fixed31_32_sinc(
  * Argument specified in radians,
  * internally it's normalized to [-2pi...2pi] range.
  */
-struct fixed31_32 dal_fixed31_32_sin(
-	struct fixed31_32 arg);
+struct fixed31_32 dc_fixpt_sin(struct fixed31_32 arg);
 
 /*
  * @brief
@@ -378,8 +382,7 @@ struct fixed31_32 dal_fixed31_32_sin(
  * passing arguments outside that range
  * will cause incorrect result!
  */
-struct fixed31_32 dal_fixed31_32_cos(
-	struct fixed31_32 arg);
+struct fixed31_32 dc_fixpt_cos(struct fixed31_32 arg);
 
 /*
  * @brief
@@ -393,8 +396,7 @@ struct fixed31_32 dal_fixed31_32_cos(
  * @note
  * Currently, function is verified for abs(arg) <= 1.
  */
-struct fixed31_32 dal_fixed31_32_exp(
-	struct fixed31_32 arg);
+struct fixed31_32 dc_fixpt_exp(struct fixed31_32 arg);
 
 /*
  * @brief
@@ -406,8 +408,7 @@ struct fixed31_32 dal_fixed31_32_exp(
  * Currently, no special actions taken
  * in case of invalid argument(s). Take care!
  */
-struct fixed31_32 dal_fixed31_32_log(
-	struct fixed31_32 arg);
+struct fixed31_32 dc_fixpt_log(struct fixed31_32 arg);
 
 /*
  * @brief
@@ -421,9 +422,13 @@ struct fixed31_32 dal_fixed31_32_log(
  * @note
  * Currently, abs(arg1) should be less than 1. Take care!
  */
-struct fixed31_32 dal_fixed31_32_pow(
-	struct fixed31_32 arg1,
-	struct fixed31_32 arg2);
+static inline struct fixed31_32 dc_fixpt_pow(struct fixed31_32 arg1, struct fixed31_32 arg2)
+{
+	return dc_fixpt_exp(
+		dc_fixpt_mul(
+			dc_fixpt_log(arg1),
+			arg2));
+}
 
 /*
  * @brief
@@ -434,22 +439,56 @@ struct fixed31_32 dal_fixed31_32_pow(
  * @brief
  * result = floor(arg) := greatest integer lower than or equal to arg
  */
-int32_t dal_fixed31_32_floor(
-	struct fixed31_32 arg);
+static inline int dc_fixpt_floor(struct fixed31_32 arg)
+{
+	unsigned long long arg_value = arg.value > 0 ? arg.value : -arg.value;
+
+	if (arg.value >= 0)
+		return (int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
+	else
+		return -(int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
+}
 
 /*
  * @brief
  * result = round(arg) := integer nearest to arg
  */
-int32_t dal_fixed31_32_round(
-	struct fixed31_32 arg);
+static inline int dc_fixpt_round(struct fixed31_32 arg)
+{
+	unsigned long long arg_value = arg.value > 0 ? arg.value : -arg.value;
+
+	const long long summand = dc_fixpt_half.value;
+
+	ASSERT(LLONG_MAX - (long long)arg_value >= summand);
+
+	arg_value += summand;
+
+	if (arg.value >= 0)
+		return (int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
+	else
+		return -(int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
+}
 
 /*
  * @brief
  * result = ceil(arg) := lowest integer greater than or equal to arg
  */
-int32_t dal_fixed31_32_ceil(
-	struct fixed31_32 arg);
+static inline int dc_fixpt_ceil(struct fixed31_32 arg)
+{
+	unsigned long long arg_value = arg.value > 0 ? arg.value : -arg.value;
+
+	const long long summand = dc_fixpt_one.value -
+		dc_fixpt_epsilon.value;
+
+	ASSERT(LLONG_MAX - (long long)arg_value >= summand);
+
+	arg_value += summand;
+
+	if (arg.value >= 0)
+		return (int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
+	else
+		return -(int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
+}
 
 /* the following two function are used in scaler hw programming to convert fixed
  * point value to format 2 bits from integer part and 19 bits from fractional
@@ -457,20 +496,31 @@ int32_t dal_fixed31_32_ceil(
  * fractional
  */
 
-uint32_t dal_fixed31_32_u2d19(
-	struct fixed31_32 arg);
+unsigned int dc_fixpt_u2d19(struct fixed31_32 arg);
+
+unsigned int dc_fixpt_u0d19(struct fixed31_32 arg);
 
-uint32_t dal_fixed31_32_u0d19(
-	struct fixed31_32 arg);
+unsigned int dc_fixpt_clamp_u0d14(struct fixed31_32 arg);
 
+unsigned int dc_fixpt_clamp_u0d10(struct fixed31_32 arg);
 
-uint32_t dal_fixed31_32_clamp_u0d14(
-	struct fixed31_32 arg);
+int dc_fixpt_s4d19(struct fixed31_32 arg);
 
-uint32_t dal_fixed31_32_clamp_u0d10(
-	struct fixed31_32 arg);
+static inline struct fixed31_32 dc_fixpt_truncate(struct fixed31_32 arg, unsigned int frac_bits)
+{
+	bool negative = arg.value < 0;
 
-int32_t dal_fixed31_32_s4d19(
-	struct fixed31_32 arg);
+	if (frac_bits >= FIXED31_32_BITS_PER_FRACTIONAL_PART) {
+		ASSERT(frac_bits == FIXED31_32_BITS_PER_FRACTIONAL_PART);
+		return arg;
+	}
+
+	if (negative)
+		arg.value = -arg.value;
+	arg.value &= (~0LL) << (FIXED31_32_BITS_PER_FRACTIONAL_PART - frac_bits);
+	if (negative)
+		arg.value = -arg.value;
+	return arg;
+}
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/include/fixed32_32.h b/drivers/gpu/drm/amd/display/include/fixed32_32.h
deleted file mode 100644
index 9c70341fe026..000000000000
--- a/drivers/gpu/drm/amd/display/include/fixed32_32.h
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-
-#ifndef __DAL_FIXED32_32_H__
-#define __DAL_FIXED32_32_H__
-
-#include "os_types.h"
-
-struct fixed32_32 {
-	uint64_t value;
-};
-
-static const struct fixed32_32 dal_fixed32_32_zero = { 0 };
-static const struct fixed32_32 dal_fixed32_32_one = { 0x100000000LL };
-static const struct fixed32_32 dal_fixed32_32_half = { 0x80000000LL };
-
-struct fixed32_32 dal_fixed32_32_from_fraction(uint32_t n, uint32_t d);
-static inline struct fixed32_32 dal_fixed32_32_from_int(uint32_t value)
-{
-	struct fixed32_32 fx;
-
-	fx.value = (uint64_t)value<<32;
-	return fx;
-}
-
-struct fixed32_32 dal_fixed32_32_add(
-	struct fixed32_32 lhs,
-	struct fixed32_32 rhs);
-struct fixed32_32 dal_fixed32_32_add_int(
-	struct fixed32_32 lhs,
-	uint32_t rhs);
-struct fixed32_32 dal_fixed32_32_sub(
-	struct fixed32_32 lhs,
-	struct fixed32_32 rhs);
-struct fixed32_32 dal_fixed32_32_sub_int(
-	struct fixed32_32 lhs,
-	uint32_t rhs);
-struct fixed32_32 dal_fixed32_32_mul(
-	struct fixed32_32 lhs,
-	struct fixed32_32 rhs);
-struct fixed32_32 dal_fixed32_32_mul_int(
-	struct fixed32_32 lhs,
-	uint32_t rhs);
-struct fixed32_32 dal_fixed32_32_div(
-	struct fixed32_32 lhs,
-	struct fixed32_32 rhs);
-struct fixed32_32 dal_fixed32_32_div_int(
-	struct fixed32_32 lhs,
-	uint32_t rhs);
-
-static inline struct fixed32_32 dal_fixed32_32_min(struct fixed32_32 lhs,
-						   struct fixed32_32 rhs)
-{
-	return (lhs.value < rhs.value) ? lhs : rhs;
-}
-
-static inline struct fixed32_32 dal_fixed32_32_max(struct fixed32_32 lhs,
-						   struct fixed32_32 rhs)
-{
-	return (lhs.value > rhs.value) ? lhs : rhs;
-}
-
-static inline bool dal_fixed32_32_gt(struct fixed32_32 lhs, struct fixed32_32 rhs)
-{
-	return lhs.value > rhs.value;
-}
-
-static inline bool dal_fixed32_32_gt_int(struct fixed32_32 lhs, uint32_t rhs)
-{
-	return lhs.value > ((uint64_t)rhs<<32);
-}
-
-static inline bool dal_fixed32_32_lt(struct fixed32_32 lhs, struct fixed32_32 rhs)
-{
-	return lhs.value < rhs.value;
-}
-
-static inline bool dal_fixed32_32_lt_int(struct fixed32_32 lhs, uint32_t rhs)
-{
-	return lhs.value < ((uint64_t)rhs<<32);
-}
-
-static inline bool dal_fixed32_32_le(struct fixed32_32 lhs, struct fixed32_32 rhs)
-{
-	return lhs.value <= rhs.value;
-}
-
-static inline bool dal_fixed32_32_le_int(struct fixed32_32 lhs, uint32_t rhs)
-{
-	return lhs.value <= ((uint64_t)rhs<<32);
-}
-
-static inline bool dal_fixed32_32_eq(struct fixed32_32 lhs, struct fixed32_32 rhs)
-{
-	return lhs.value == rhs.value;
-}
-
-uint32_t dal_fixed32_32_ceil(struct fixed32_32 value);
-static inline uint32_t dal_fixed32_32_floor(struct fixed32_32 value)
-{
-	return value.value>>32;
-}
-
-uint32_t dal_fixed32_32_round(struct fixed32_32 value);
-
-#endif
diff --git a/drivers/gpu/drm/amd/display/include/logger_interface.h b/drivers/gpu/drm/amd/display/include/logger_interface.h
index 28dee960d509..dc98d6d4b2bd 100644
--- a/drivers/gpu/drm/amd/display/include/logger_interface.h
+++ b/drivers/gpu/drm/amd/display/include/logger_interface.h
@@ -190,4 +190,13 @@ void context_clock_trace(
 	} \
 } while (0)
 
+#define DISPLAY_STATS_BEGIN(entry) \
+	dm_logger_open(dc->ctx->logger, &entry, LOG_DISPLAYSTATS)
+
+#define DISPLAY_STATS(msg, ...) \
+	dm_logger_append(&log_entry, msg, ##__VA_ARGS__)
+
+#define DISPLAY_STATS_END(entry) \
+	dm_logger_close(&entry)
+
 #endif /* __DAL_LOGGER_INTERFACE_H__ */
diff --git a/drivers/gpu/drm/amd/display/include/logger_types.h b/drivers/gpu/drm/amd/display/include/logger_types.h
index 427796bdc14a..0a540b9897a6 100644
--- a/drivers/gpu/drm/amd/display/include/logger_types.h
+++ b/drivers/gpu/drm/amd/display/include/logger_types.h
@@ -29,39 +29,39 @@
 #include "os_types.h"
 
 #define MAX_NAME_LEN 32
-#define DC_LOG_ERROR(a, ...) dm_logger_write(DC_LOGGER, LOG_ERROR, a, ## __VA_ARGS__)
-#define DC_LOG_WARNING(a, ...) dm_logger_write(DC_LOGGER, LOG_WARNING, a, ## __VA_ARGS__)
-#define DC_LOG_DEBUG(a, ...) dm_logger_write(DC_LOGGER, LOG_DEBUG, a, ## __VA_ARGS__)
-#define DC_LOG_DC(a, ...) dm_logger_write(DC_LOGGER, LOG_DC, a, ## __VA_ARGS__)
-#define DC_LOG_DTN(a, ...) dm_logger_write(DC_LOGGER, LOG_DTN, a, ## __VA_ARGS__)
-#define DC_LOG_SURFACE(a, ...) dm_logger_write(DC_LOGGER, LOG_SURFACE, a, ## __VA_ARGS__)
-#define DC_LOG_HW_HOTPLUG(a, ...) dm_logger_write(DC_LOGGER, LOG_HW_HOTPLUG, a, ## __VA_ARGS__)
-#define DC_LOG_HW_LINK_TRAINING(a, ...) dm_logger_write(DC_LOGGER, LOG_HW_LINK_TRAINING, a, ## __VA_ARGS__)
-#define DC_LOG_HW_SET_MODE(a, ...) dm_logger_write(DC_LOGGER, LOG_HW_SET_MODE, a, ## __VA_ARGS__)
-#define DC_LOG_HW_RESUME_S3(a, ...) dm_logger_write(DC_LOGGER, LOG_HW_RESUME_S3, a, ## __VA_ARGS__)
-#define DC_LOG_HW_AUDIO(a, ...) dm_logger_write(DC_LOGGER, LOG_HW_AUDIO, a, ## __VA_ARGS__)
-#define DC_LOG_HW_HPD_IRQ(a, ...) dm_logger_write(DC_LOGGER, LOG_HW_HPD_IRQ, a, ## __VA_ARGS__)
-#define DC_LOG_MST(a, ...) dm_logger_write(DC_LOGGER, LOG_MST, a, ## __VA_ARGS__)
-#define DC_LOG_SCALER(a, ...) dm_logger_write(DC_LOGGER, LOG_SCALER, a, ## __VA_ARGS__)
-#define DC_LOG_BIOS(a, ...) dm_logger_write(DC_LOGGER, LOG_BIOS, a, ## __VA_ARGS__)
-#define DC_LOG_BANDWIDTH_CALCS(a, ...) dm_logger_write(DC_LOGGER, LOG_BANDWIDTH_CALCS, a, ## __VA_ARGS__)
-#define DC_LOG_BANDWIDTH_VALIDATION(a, ...) dm_logger_write(DC_LOGGER, LOG_BANDWIDTH_VALIDATION, a, ## __VA_ARGS__)
-#define DC_LOG_I2C_AUX(a, ...) dm_logger_write(DC_LOGGER, LOG_I2C_AUX, a, ## __VA_ARGS__)
-#define DC_LOG_SYNC(a, ...) dm_logger_write(DC_LOGGER, LOG_SYNC, a, ## __VA_ARGS__)
-#define DC_LOG_BACKLIGHT(a, ...) dm_logger_write(DC_LOGGER, LOG_BACKLIGHT, a, ## __VA_ARGS__)
-#define DC_LOG_FEATURE_OVERRIDE(a, ...) dm_logger_write(DC_LOGGER, LOG_FEATURE_OVERRIDE, a, ## __VA_ARGS__)
-#define DC_LOG_DETECTION_EDID_PARSER(a, ...) dm_logger_write(DC_LOGGER, LOG_DETECTION_EDID_PARSER, a, ## __VA_ARGS__)
-#define DC_LOG_DETECTION_DP_CAPS(a, ...) dm_logger_write(DC_LOGGER, LOG_DETECTION_DP_CAPS, a, ## __VA_ARGS__)
-#define DC_LOG_RESOURCE(a, ...) dm_logger_write(DC_LOGGER, LOG_RESOURCE, a, ## __VA_ARGS__)
-#define DC_LOG_DML(a, ...) dm_logger_write(DC_LOGGER, LOG_DML, a, ## __VA_ARGS__)
-#define DC_LOG_EVENT_MODE_SET(a, ...) dm_logger_write(DC_LOGGER, LOG_EVENT_MODE_SET, a, ## __VA_ARGS__)
-#define DC_LOG_EVENT_DETECTION(a, ...) dm_logger_write(DC_LOGGER, LOG_EVENT_DETECTION, a, ## __VA_ARGS__)
-#define DC_LOG_EVENT_LINK_TRAINING(a, ...) dm_logger_write(DC_LOGGER, LOG_EVENT_LINK_TRAINING, a, ## __VA_ARGS__)
-#define DC_LOG_EVENT_LINK_LOSS(a, ...) dm_logger_write(DC_LOGGER, LOG_EVENT_LINK_LOSS, a, ## __VA_ARGS__)
-#define DC_LOG_EVENT_UNDERFLOW(a, ...) dm_logger_write(DC_LOGGER, LOG_EVENT_UNDERFLOW, a, ## __VA_ARGS__)
-#define DC_LOG_IF_TRACE(a, ...) dm_logger_write(DC_LOGGER, LOG_IF_TRACE, a, ## __VA_ARGS__)
-#define DC_LOG_PERF_TRACE(a, ...) dm_logger_write(DC_LOGGER, LOG_PERF_TRACE, a, ## __VA_ARGS__)
 
+#define DC_LOG_ERROR(...) DRM_ERROR(__VA_ARGS__)
+#define DC_LOG_WARNING(...) DRM_WARN(__VA_ARGS__)
+#define DC_LOG_DEBUG(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_DC(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_DTN(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_SURFACE(...) pr_debug("[SURFACE]:"__VA_ARGS__)
+#define DC_LOG_HW_HOTPLUG(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_HW_LINK_TRAINING(...) pr_debug("[HW_LINK_TRAINING]:"__VA_ARGS__)
+#define DC_LOG_HW_SET_MODE(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_HW_RESUME_S3(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_HW_AUDIO(...) pr_debug("[HW_AUDIO]:"__VA_ARGS__)
+#define DC_LOG_HW_HPD_IRQ(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_MST(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_SCALER(...) pr_debug("[SCALER]:"__VA_ARGS__)
+#define DC_LOG_BIOS(...) pr_debug("[BIOS]:"__VA_ARGS__)
+#define DC_LOG_BANDWIDTH_CALCS(...) pr_debug("[BANDWIDTH_CALCS]:"__VA_ARGS__)
+#define DC_LOG_BANDWIDTH_VALIDATION(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_I2C_AUX(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_SYNC(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_BACKLIGHT(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_FEATURE_OVERRIDE(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_DETECTION_EDID_PARSER(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_DETECTION_DP_CAPS(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_RESOURCE(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_DML(...) pr_debug("[DML]:"__VA_ARGS__)
+#define DC_LOG_EVENT_MODE_SET(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_EVENT_DETECTION(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_EVENT_LINK_TRAINING(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_EVENT_LINK_LOSS(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_EVENT_UNDERFLOW(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#define DC_LOG_IF_TRACE(...) pr_debug("[IF_TRACE]:"__VA_ARGS__)
+#define DC_LOG_PERF_TRACE(...) DRM_DEBUG_KMS(__VA_ARGS__)
 
 struct dal_logger;
 
@@ -98,7 +98,7 @@ enum dc_log_type {
 	LOG_EVENT_UNDERFLOW,
 	LOG_IF_TRACE,
 	LOG_PERF_TRACE,
-	LOG_PROFILING,
+	LOG_DISPLAYSTATS,
 
 	LOG_SECTION_TOTAL_COUNT
 };
diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
index b3747a019deb..0cd111d59018 100644
--- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
+++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
@@ -43,7 +43,7 @@ static bool de_pq_initialized; /* = false; */
 /* one-time setup of X points */
 void setup_x_points_distribution(void)
 {
-	struct fixed31_32 region_size = dal_fixed31_32_from_int(128);
+	struct fixed31_32 region_size = dc_fixpt_from_int(128);
 	int32_t segment;
 	uint32_t seg_offset;
 	uint32_t index;
@@ -53,8 +53,8 @@ void setup_x_points_distribution(void)
 	coordinates_x[MAX_HW_POINTS + 1].x = region_size;
 
 	for (segment = 6; segment > (6 - NUM_REGIONS); segment--) {
-		region_size = dal_fixed31_32_div_int(region_size, 2);
-		increment = dal_fixed31_32_div_int(region_size,
+		region_size = dc_fixpt_div_int(region_size, 2);
+		increment = dc_fixpt_div_int(region_size,
 						NUM_PTS_IN_REGION);
 		seg_offset = (segment + (NUM_REGIONS - 7)) * NUM_PTS_IN_REGION;
 		coordinates_x[seg_offset].x = region_size;
@@ -62,7 +62,7 @@ void setup_x_points_distribution(void)
 		for (index = seg_offset + 1;
 				index < seg_offset + NUM_PTS_IN_REGION;
 				index++) {
-			coordinates_x[index].x = dal_fixed31_32_add
+			coordinates_x[index].x = dc_fixpt_add
 					(coordinates_x[index-1].x, increment);
 		}
 	}
@@ -72,63 +72,63 @@ static void compute_pq(struct fixed31_32 in_x, struct fixed31_32 *out_y)
 {
 	/* consts for PQ gamma formula. */
 	const struct fixed31_32 m1 =
-		dal_fixed31_32_from_fraction(159301758, 1000000000);
+		dc_fixpt_from_fraction(159301758, 1000000000);
 	const struct fixed31_32 m2 =
-		dal_fixed31_32_from_fraction(7884375, 100000);
+		dc_fixpt_from_fraction(7884375, 100000);
 	const struct fixed31_32 c1 =
-		dal_fixed31_32_from_fraction(8359375, 10000000);
+		dc_fixpt_from_fraction(8359375, 10000000);
 	const struct fixed31_32 c2 =
-		dal_fixed31_32_from_fraction(188515625, 10000000);
+		dc_fixpt_from_fraction(188515625, 10000000);
 	const struct fixed31_32 c3 =
-		dal_fixed31_32_from_fraction(186875, 10000);
+		dc_fixpt_from_fraction(186875, 10000);
 
 	struct fixed31_32 l_pow_m1;
 	struct fixed31_32 base;
 
-	if (dal_fixed31_32_lt(in_x, dal_fixed31_32_zero))
-		in_x = dal_fixed31_32_zero;
+	if (dc_fixpt_lt(in_x, dc_fixpt_zero))
+		in_x = dc_fixpt_zero;
 
-	l_pow_m1 = dal_fixed31_32_pow(in_x, m1);
-	base = dal_fixed31_32_div(
-			dal_fixed31_32_add(c1,
-					(dal_fixed31_32_mul(c2, l_pow_m1))),
-			dal_fixed31_32_add(dal_fixed31_32_one,
-					(dal_fixed31_32_mul(c3, l_pow_m1))));
-	*out_y = dal_fixed31_32_pow(base, m2);
+	l_pow_m1 = dc_fixpt_pow(in_x, m1);
+	base = dc_fixpt_div(
+			dc_fixpt_add(c1,
+					(dc_fixpt_mul(c2, l_pow_m1))),
+			dc_fixpt_add(dc_fixpt_one,
+					(dc_fixpt_mul(c3, l_pow_m1))));
+	*out_y = dc_fixpt_pow(base, m2);
 }
 
 static void compute_de_pq(struct fixed31_32 in_x, struct fixed31_32 *out_y)
 {
 	/* consts for dePQ gamma formula. */
 	const struct fixed31_32 m1 =
-		dal_fixed31_32_from_fraction(159301758, 1000000000);
+		dc_fixpt_from_fraction(159301758, 1000000000);
 	const struct fixed31_32 m2 =
-		dal_fixed31_32_from_fraction(7884375, 100000);
+		dc_fixpt_from_fraction(7884375, 100000);
 	const struct fixed31_32 c1 =
-		dal_fixed31_32_from_fraction(8359375, 10000000);
+		dc_fixpt_from_fraction(8359375, 10000000);
 	const struct fixed31_32 c2 =
-		dal_fixed31_32_from_fraction(188515625, 10000000);
+		dc_fixpt_from_fraction(188515625, 10000000);
 	const struct fixed31_32 c3 =
-		dal_fixed31_32_from_fraction(186875, 10000);
+		dc_fixpt_from_fraction(186875, 10000);
 
 	struct fixed31_32 l_pow_m1;
 	struct fixed31_32 base, div;
 
 
-	if (dal_fixed31_32_lt(in_x, dal_fixed31_32_zero))
-		in_x = dal_fixed31_32_zero;
+	if (dc_fixpt_lt(in_x, dc_fixpt_zero))
+		in_x = dc_fixpt_zero;
 
-	l_pow_m1 = dal_fixed31_32_pow(in_x,
-			dal_fixed31_32_div(dal_fixed31_32_one, m2));
-	base = dal_fixed31_32_sub(l_pow_m1, c1);
+	l_pow_m1 = dc_fixpt_pow(in_x,
+			dc_fixpt_div(dc_fixpt_one, m2));
+	base = dc_fixpt_sub(l_pow_m1, c1);
 
-	if (dal_fixed31_32_lt(base, dal_fixed31_32_zero))
-		base = dal_fixed31_32_zero;
+	if (dc_fixpt_lt(base, dc_fixpt_zero))
+		base = dc_fixpt_zero;
 
-	div = dal_fixed31_32_sub(c2, dal_fixed31_32_mul(c3, l_pow_m1));
+	div = dc_fixpt_sub(c2, dc_fixpt_mul(c3, l_pow_m1));
 
-	*out_y = dal_fixed31_32_pow(dal_fixed31_32_div(base, div),
-			dal_fixed31_32_div(dal_fixed31_32_one, m1));
+	*out_y = dc_fixpt_pow(dc_fixpt_div(base, div),
+			dc_fixpt_div(dc_fixpt_one, m1));
 
 }
 /* one-time pre-compute PQ values - only for sdr_white_level 80 */
@@ -138,14 +138,14 @@ void precompute_pq(void)
 	struct fixed31_32 x;
 	const struct hw_x_point *coord_x = coordinates_x + 32;
 	struct fixed31_32 scaling_factor =
-			dal_fixed31_32_from_fraction(80, 10000);
+			dc_fixpt_from_fraction(80, 10000);
 
 	/* pow function has problems with arguments too small */
 	for (i = 0; i < 32; i++)
-		pq_table[i] = dal_fixed31_32_zero;
+		pq_table[i] = dc_fixpt_zero;
 
 	for (i = 32; i <= MAX_HW_POINTS; i++) {
-		x = dal_fixed31_32_mul(coord_x->x, scaling_factor);
+		x = dc_fixpt_mul(coord_x->x, scaling_factor);
 		compute_pq(x, &pq_table[i]);
 		++coord_x;
 	}
@@ -158,7 +158,7 @@ void precompute_de_pq(void)
 	struct fixed31_32  y;
 	uint32_t begin_index, end_index;
 
-	struct fixed31_32 scaling_factor = dal_fixed31_32_from_int(125);
+	struct fixed31_32 scaling_factor = dc_fixpt_from_int(125);
 
 	/* X points is 2^-25 to 2^7
 	 * De-gamma X is 2^-12 to 2^0 – we are skipping first -12-(-25) = 13 regions
@@ -167,11 +167,11 @@ void precompute_de_pq(void)
 	end_index = begin_index + 12 * NUM_PTS_IN_REGION;
 
 	for (i = 0; i <= begin_index; i++)
-		de_pq_table[i] = dal_fixed31_32_zero;
+		de_pq_table[i] = dc_fixpt_zero;
 
 	for (; i <= end_index; i++) {
 		compute_de_pq(coordinates_x[i].x, &y);
-		de_pq_table[i] = dal_fixed31_32_mul(y, scaling_factor);
+		de_pq_table[i] = dc_fixpt_mul(y, scaling_factor);
 	}
 
 	for (; i <= MAX_HW_POINTS; i++)
@@ -185,25 +185,25 @@ struct dividers {
 
 static void build_coefficients(struct gamma_coefficients *coefficients, bool is_2_4)
 {
-		static const int32_t numerator01[] = { 31308, 180000};
-		static const int32_t numerator02[] = { 12920, 4500};
-		static const int32_t numerator03[] = { 55, 99};
-		static const int32_t numerator04[] = { 55, 99};
-		static const int32_t numerator05[] = { 2400, 2200};
+	static const int32_t numerator01[] = { 31308, 180000};
+	static const int32_t numerator02[] = { 12920, 4500};
+	static const int32_t numerator03[] = { 55, 99};
+	static const int32_t numerator04[] = { 55, 99};
+	static const int32_t numerator05[] = { 2400, 2200};
 
-		uint32_t i = 0;
-		uint32_t index = is_2_4 == true ? 0:1;
+	uint32_t i = 0;
+	uint32_t index = is_2_4 == true ? 0:1;
 
 	do {
-		coefficients->a0[i] = dal_fixed31_32_from_fraction(
+		coefficients->a0[i] = dc_fixpt_from_fraction(
 			numerator01[index], 10000000);
-		coefficients->a1[i] = dal_fixed31_32_from_fraction(
+		coefficients->a1[i] = dc_fixpt_from_fraction(
 			numerator02[index], 1000);
-		coefficients->a2[i] = dal_fixed31_32_from_fraction(
+		coefficients->a2[i] = dc_fixpt_from_fraction(
 			numerator03[index], 1000);
-		coefficients->a3[i] = dal_fixed31_32_from_fraction(
+		coefficients->a3[i] = dc_fixpt_from_fraction(
 			numerator04[index], 1000);
-		coefficients->user_gamma[i] = dal_fixed31_32_from_fraction(
+		coefficients->user_gamma[i] = dc_fixpt_from_fraction(
 			numerator05[index], 1000);
 
 		++i;
@@ -218,33 +218,33 @@ static struct fixed31_32 translate_from_linear_space(
 	struct fixed31_32 a3,
 	struct fixed31_32 gamma)
 {
-	const struct fixed31_32 one = dal_fixed31_32_from_int(1);
+	const struct fixed31_32 one = dc_fixpt_from_int(1);
 
-	if (dal_fixed31_32_lt(one, arg))
+	if (dc_fixpt_lt(one, arg))
 		return one;
 
-	if (dal_fixed31_32_le(arg, dal_fixed31_32_neg(a0)))
-		return dal_fixed31_32_sub(
+	if (dc_fixpt_le(arg, dc_fixpt_neg(a0)))
+		return dc_fixpt_sub(
 			a2,
-			dal_fixed31_32_mul(
-				dal_fixed31_32_add(
+			dc_fixpt_mul(
+				dc_fixpt_add(
 					one,
 					a3),
-				dal_fixed31_32_pow(
-					dal_fixed31_32_neg(arg),
-					dal_fixed31_32_recip(gamma))));
-	else if (dal_fixed31_32_le(a0, arg))
-		return dal_fixed31_32_sub(
-			dal_fixed31_32_mul(
-				dal_fixed31_32_add(
+				dc_fixpt_pow(
+					dc_fixpt_neg(arg),
+					dc_fixpt_recip(gamma))));
+	else if (dc_fixpt_le(a0, arg))
+		return dc_fixpt_sub(
+			dc_fixpt_mul(
+				dc_fixpt_add(
 					one,
 					a3),
-				dal_fixed31_32_pow(
+				dc_fixpt_pow(
 					arg,
-					dal_fixed31_32_recip(gamma))),
+					dc_fixpt_recip(gamma))),
 			a2);
 	else
-		return dal_fixed31_32_mul(
+		return dc_fixpt_mul(
 			arg,
 			a1);
 }
@@ -259,25 +259,25 @@ static struct fixed31_32 translate_to_linear_space(
 {
 	struct fixed31_32 linear;
 
-	a0 = dal_fixed31_32_mul(a0, a1);
-	if (dal_fixed31_32_le(arg, dal_fixed31_32_neg(a0)))
+	a0 = dc_fixpt_mul(a0, a1);
+	if (dc_fixpt_le(arg, dc_fixpt_neg(a0)))
 
-		linear = dal_fixed31_32_neg(
-				 dal_fixed31_32_pow(
-				 dal_fixed31_32_div(
-				 dal_fixed31_32_sub(a2, arg),
-				 dal_fixed31_32_add(
-				 dal_fixed31_32_one, a3)), gamma));
+		linear = dc_fixpt_neg(
+				 dc_fixpt_pow(
+				 dc_fixpt_div(
+				 dc_fixpt_sub(a2, arg),
+				 dc_fixpt_add(
+				 dc_fixpt_one, a3)), gamma));
 
-	else if (dal_fixed31_32_le(dal_fixed31_32_neg(a0), arg) &&
-			 dal_fixed31_32_le(arg, a0))
-		linear = dal_fixed31_32_div(arg, a1);
+	else if (dc_fixpt_le(dc_fixpt_neg(a0), arg) &&
+			 dc_fixpt_le(arg, a0))
+		linear = dc_fixpt_div(arg, a1);
 	else
-		linear =  dal_fixed31_32_pow(
-					dal_fixed31_32_div(
-					dal_fixed31_32_add(a2, arg),
-					dal_fixed31_32_add(
-					dal_fixed31_32_one, a3)), gamma);
+		linear =  dc_fixpt_pow(
+					dc_fixpt_div(
+					dc_fixpt_add(a2, arg),
+					dc_fixpt_add(
+					dc_fixpt_one, a3)), gamma);
 
 	return linear;
 }
@@ -352,8 +352,8 @@ static bool find_software_points(
 				right = axis_x[max_number - 1].b;
 		}
 
-		if (dal_fixed31_32_le(left, hw_point) &&
-			dal_fixed31_32_le(hw_point, right)) {
+		if (dc_fixpt_le(left, hw_point) &&
+			dc_fixpt_le(hw_point, right)) {
 			*index_to_start = i;
 			*index_left = i;
 
@@ -366,7 +366,7 @@ static bool find_software_points(
 
 			return true;
 		} else if ((i == *index_to_start) &&
-			dal_fixed31_32_le(hw_point, left)) {
+			dc_fixpt_le(hw_point, left)) {
 			*index_to_start = i;
 			*index_left = i;
 			*index_right = i;
@@ -375,7 +375,7 @@ static bool find_software_points(
 
 			return true;
 		} else if ((i == max_number - 1) &&
-			dal_fixed31_32_le(right, hw_point)) {
+			dc_fixpt_le(right, hw_point)) {
 			*index_to_start = i;
 			*index_left = i;
 			*index_right = i;
@@ -457,17 +457,17 @@ static bool build_custom_gamma_mapping_coefficients_worker(
 		}
 
 		if (hw_pos == HW_POINT_POSITION_MIDDLE)
-			point->coeff = dal_fixed31_32_div(
-				dal_fixed31_32_sub(
+			point->coeff = dc_fixpt_div(
+				dc_fixpt_sub(
 					coord_x,
 					left_pos),
-				dal_fixed31_32_sub(
+				dc_fixpt_sub(
 					right_pos,
 					left_pos));
 		else if (hw_pos == HW_POINT_POSITION_LEFT)
-			point->coeff = dal_fixed31_32_zero;
+			point->coeff = dc_fixpt_zero;
 		else if (hw_pos == HW_POINT_POSITION_RIGHT)
-			point->coeff = dal_fixed31_32_from_int(2);
+			point->coeff = dc_fixpt_from_int(2);
 		else {
 			BREAK_TO_DEBUGGER();
 			return false;
@@ -502,45 +502,45 @@ static struct fixed31_32 calculate_mapped_value(
 
 	if ((point->left_index < 0) || (point->left_index > max_index)) {
 		BREAK_TO_DEBUGGER();
-		return dal_fixed31_32_zero;
+		return dc_fixpt_zero;
 	}
 
 	if ((point->right_index < 0) || (point->right_index > max_index)) {
 		BREAK_TO_DEBUGGER();
-		return dal_fixed31_32_zero;
+		return dc_fixpt_zero;
 	}
 
 	if (point->pos == HW_POINT_POSITION_MIDDLE)
 		if (channel == CHANNEL_NAME_RED)
-			result = dal_fixed31_32_add(
-				dal_fixed31_32_mul(
+			result = dc_fixpt_add(
+				dc_fixpt_mul(
 					point->coeff,
-					dal_fixed31_32_sub(
+					dc_fixpt_sub(
 						rgb[point->right_index].r,
 						rgb[point->left_index].r)),
 				rgb[point->left_index].r);
 		else if (channel == CHANNEL_NAME_GREEN)
-			result = dal_fixed31_32_add(
-				dal_fixed31_32_mul(
+			result = dc_fixpt_add(
+				dc_fixpt_mul(
 					point->coeff,
-					dal_fixed31_32_sub(
+					dc_fixpt_sub(
 						rgb[point->right_index].g,
 						rgb[point->left_index].g)),
 				rgb[point->left_index].g);
 		else
-			result = dal_fixed31_32_add(
-				dal_fixed31_32_mul(
+			result = dc_fixpt_add(
+				dc_fixpt_mul(
 					point->coeff,
-					dal_fixed31_32_sub(
+					dc_fixpt_sub(
 						rgb[point->right_index].b,
 						rgb[point->left_index].b)),
 				rgb[point->left_index].b);
 	else if (point->pos == HW_POINT_POSITION_LEFT) {
 		BREAK_TO_DEBUGGER();
-		result = dal_fixed31_32_zero;
+		result = dc_fixpt_zero;
 	} else {
 		BREAK_TO_DEBUGGER();
-		result = dal_fixed31_32_one;
+		result = dc_fixpt_one;
 	}
 
 	return result;
@@ -558,7 +558,7 @@ static void build_pq(struct pwl_float_data_ex *rgb_regamma,
 	struct fixed31_32 x;
 	struct fixed31_32 output;
 	struct fixed31_32 scaling_factor =
-			dal_fixed31_32_from_fraction(sdr_white_level, 10000);
+			dc_fixpt_from_fraction(sdr_white_level, 10000);
 
 	if (!pq_initialized && sdr_white_level == 80) {
 		precompute_pq();
@@ -579,15 +579,15 @@ static void build_pq(struct pwl_float_data_ex *rgb_regamma,
 		if (sdr_white_level == 80) {
 			output = pq_table[i];
 		} else {
-			x = dal_fixed31_32_mul(coord_x->x, scaling_factor);
+			x = dc_fixpt_mul(coord_x->x, scaling_factor);
 			compute_pq(x, &output);
 		}
 
 		/* should really not happen? */
-		if (dal_fixed31_32_lt(output, dal_fixed31_32_zero))
-			output = dal_fixed31_32_zero;
-		else if (dal_fixed31_32_lt(dal_fixed31_32_one, output))
-			output = dal_fixed31_32_one;
+		if (dc_fixpt_lt(output, dc_fixpt_zero))
+			output = dc_fixpt_zero;
+		else if (dc_fixpt_lt(dc_fixpt_one, output))
+			output = dc_fixpt_one;
 
 		rgb->r = output;
 		rgb->g = output;
@@ -605,7 +605,7 @@ static void build_de_pq(struct pwl_float_data_ex *de_pq,
 	uint32_t i;
 	struct fixed31_32 output;
 
-	struct fixed31_32 scaling_factor = dal_fixed31_32_from_int(125);
+	struct fixed31_32 scaling_factor = dc_fixpt_from_int(125);
 
 	if (!de_pq_initialized) {
 		precompute_de_pq();
@@ -616,9 +616,9 @@ static void build_de_pq(struct pwl_float_data_ex *de_pq,
 	for (i = 0; i <= hw_points_num; i++) {
 		output = de_pq_table[i];
 		/* should really not happen? */
-		if (dal_fixed31_32_lt(output, dal_fixed31_32_zero))
-			output = dal_fixed31_32_zero;
-		else if (dal_fixed31_32_lt(scaling_factor, output))
+		if (dc_fixpt_lt(output, dc_fixpt_zero))
+			output = dc_fixpt_zero;
+		else if (dc_fixpt_lt(scaling_factor, output))
 			output = scaling_factor;
 		de_pq[i].r = output;
 		de_pq[i].g = output;
@@ -670,9 +670,9 @@ static void build_degamma(struct pwl_float_data_ex *curve,
 	end_index = begin_index + 12 * NUM_PTS_IN_REGION;
 
 	while (i != begin_index) {
-		curve[i].r = dal_fixed31_32_zero;
-		curve[i].g = dal_fixed31_32_zero;
-		curve[i].b = dal_fixed31_32_zero;
+		curve[i].r = dc_fixpt_zero;
+		curve[i].g = dc_fixpt_zero;
+		curve[i].b = dc_fixpt_zero;
 		i++;
 	}
 
@@ -684,19 +684,19 @@ static void build_degamma(struct pwl_float_data_ex *curve,
 		i++;
 	}
 	while (i != hw_points_num + 1) {
-		curve[i].r = dal_fixed31_32_one;
-		curve[i].g = dal_fixed31_32_one;
-		curve[i].b = dal_fixed31_32_one;
+		curve[i].r = dc_fixpt_one;
+		curve[i].g = dc_fixpt_one;
+		curve[i].b = dc_fixpt_one;
 		i++;
 	}
 }
 
-static bool scale_gamma(struct pwl_float_data *pwl_rgb,
+static void scale_gamma(struct pwl_float_data *pwl_rgb,
 		const struct dc_gamma *ramp,
 		struct dividers dividers)
 {
-	const struct fixed31_32 max_driver = dal_fixed31_32_from_int(0xFFFF);
-	const struct fixed31_32 max_os = dal_fixed31_32_from_int(0xFF00);
+	const struct fixed31_32 max_driver = dc_fixpt_from_int(0xFFFF);
+	const struct fixed31_32 max_os = dc_fixpt_from_int(0xFF00);
 	struct fixed31_32 scaler = max_os;
 	uint32_t i;
 	struct pwl_float_data *rgb = pwl_rgb;
@@ -705,9 +705,9 @@ static bool scale_gamma(struct pwl_float_data *pwl_rgb,
 	i = 0;
 
 	do {
-		if (dal_fixed31_32_lt(max_os, ramp->entries.red[i]) ||
-			dal_fixed31_32_lt(max_os, ramp->entries.green[i]) ||
-			dal_fixed31_32_lt(max_os, ramp->entries.blue[i])) {
+		if (dc_fixpt_lt(max_os, ramp->entries.red[i]) ||
+			dc_fixpt_lt(max_os, ramp->entries.green[i]) ||
+			dc_fixpt_lt(max_os, ramp->entries.blue[i])) {
 			scaler = max_driver;
 			break;
 		}
@@ -717,109 +717,170 @@ static bool scale_gamma(struct pwl_float_data *pwl_rgb,
 	i = 0;
 
 	do {
-		rgb->r = dal_fixed31_32_div(
+		rgb->r = dc_fixpt_div(
 			ramp->entries.red[i], scaler);
-		rgb->g = dal_fixed31_32_div(
+		rgb->g = dc_fixpt_div(
 			ramp->entries.green[i], scaler);
-		rgb->b = dal_fixed31_32_div(
+		rgb->b = dc_fixpt_div(
 			ramp->entries.blue[i], scaler);
 
 		++rgb;
 		++i;
 	} while (i != ramp->num_entries);
 
-	rgb->r = dal_fixed31_32_mul(rgb_last->r,
+	rgb->r = dc_fixpt_mul(rgb_last->r,
 			dividers.divider1);
-	rgb->g = dal_fixed31_32_mul(rgb_last->g,
+	rgb->g = dc_fixpt_mul(rgb_last->g,
 			dividers.divider1);
-	rgb->b = dal_fixed31_32_mul(rgb_last->b,
+	rgb->b = dc_fixpt_mul(rgb_last->b,
 			dividers.divider1);
 
 	++rgb;
 
-	rgb->r = dal_fixed31_32_mul(rgb_last->r,
+	rgb->r = dc_fixpt_mul(rgb_last->r,
 			dividers.divider2);
-	rgb->g = dal_fixed31_32_mul(rgb_last->g,
+	rgb->g = dc_fixpt_mul(rgb_last->g,
 			dividers.divider2);
-	rgb->b = dal_fixed31_32_mul(rgb_last->b,
+	rgb->b = dc_fixpt_mul(rgb_last->b,
 			dividers.divider2);
 
 	++rgb;
 
-	rgb->r = dal_fixed31_32_mul(rgb_last->r,
+	rgb->r = dc_fixpt_mul(rgb_last->r,
 			dividers.divider3);
-	rgb->g = dal_fixed31_32_mul(rgb_last->g,
+	rgb->g = dc_fixpt_mul(rgb_last->g,
 			dividers.divider3);
-	rgb->b = dal_fixed31_32_mul(rgb_last->b,
+	rgb->b = dc_fixpt_mul(rgb_last->b,
 			dividers.divider3);
-
-	return true;
 }
 
-static bool scale_gamma_dx(struct pwl_float_data *pwl_rgb,
+static void scale_gamma_dx(struct pwl_float_data *pwl_rgb,
 		const struct dc_gamma *ramp,
 		struct dividers dividers)
 {
 	uint32_t i;
-	struct fixed31_32 min = dal_fixed31_32_zero;
-	struct fixed31_32 max = dal_fixed31_32_one;
+	struct fixed31_32 min = dc_fixpt_zero;
+	struct fixed31_32 max = dc_fixpt_one;
 
-	struct fixed31_32 delta = dal_fixed31_32_zero;
-	struct fixed31_32 offset = dal_fixed31_32_zero;
+	struct fixed31_32 delta = dc_fixpt_zero;
+	struct fixed31_32 offset = dc_fixpt_zero;
 
 	for (i = 0 ; i < ramp->num_entries; i++) {
-		if (dal_fixed31_32_lt(ramp->entries.red[i], min))
+		if (dc_fixpt_lt(ramp->entries.red[i], min))
 			min = ramp->entries.red[i];
 
-		if (dal_fixed31_32_lt(ramp->entries.green[i], min))
+		if (dc_fixpt_lt(ramp->entries.green[i], min))
 			min = ramp->entries.green[i];
 
-		if (dal_fixed31_32_lt(ramp->entries.blue[i], min))
+		if (dc_fixpt_lt(ramp->entries.blue[i], min))
 			min = ramp->entries.blue[i];
 
-		if (dal_fixed31_32_lt(max, ramp->entries.red[i]))
+		if (dc_fixpt_lt(max, ramp->entries.red[i]))
 			max = ramp->entries.red[i];
 
-		if (dal_fixed31_32_lt(max, ramp->entries.green[i]))
+		if (dc_fixpt_lt(max, ramp->entries.green[i]))
 			max = ramp->entries.green[i];
 
-		if (dal_fixed31_32_lt(max, ramp->entries.blue[i]))
+		if (dc_fixpt_lt(max, ramp->entries.blue[i]))
 			max = ramp->entries.blue[i];
 	}
 
-	if (dal_fixed31_32_lt(min, dal_fixed31_32_zero))
-		delta = dal_fixed31_32_neg(min);
+	if (dc_fixpt_lt(min, dc_fixpt_zero))
+		delta = dc_fixpt_neg(min);
 
-	offset = dal_fixed31_32_add(min, max);
+	offset = dc_fixpt_add(min, max);
 
 	for (i = 0 ; i < ramp->num_entries; i++) {
-		pwl_rgb[i].r = dal_fixed31_32_div(
-			dal_fixed31_32_add(
+		pwl_rgb[i].r = dc_fixpt_div(
+			dc_fixpt_add(
 				ramp->entries.red[i], delta), offset);
-		pwl_rgb[i].g = dal_fixed31_32_div(
-			dal_fixed31_32_add(
+		pwl_rgb[i].g = dc_fixpt_div(
+			dc_fixpt_add(
 				ramp->entries.green[i], delta), offset);
-		pwl_rgb[i].b = dal_fixed31_32_div(
-			dal_fixed31_32_add(
+		pwl_rgb[i].b = dc_fixpt_div(
+			dc_fixpt_add(
 				ramp->entries.blue[i], delta), offset);
 
 	}
 
-	pwl_rgb[i].r =  dal_fixed31_32_sub(dal_fixed31_32_mul_int(
+	pwl_rgb[i].r =  dc_fixpt_sub(dc_fixpt_mul_int(
 				pwl_rgb[i-1].r, 2), pwl_rgb[i-2].r);
-	pwl_rgb[i].g =  dal_fixed31_32_sub(dal_fixed31_32_mul_int(
+	pwl_rgb[i].g =  dc_fixpt_sub(dc_fixpt_mul_int(
 				pwl_rgb[i-1].g, 2), pwl_rgb[i-2].g);
-	pwl_rgb[i].b =  dal_fixed31_32_sub(dal_fixed31_32_mul_int(
+	pwl_rgb[i].b =  dc_fixpt_sub(dc_fixpt_mul_int(
 				pwl_rgb[i-1].b, 2), pwl_rgb[i-2].b);
 	++i;
-	pwl_rgb[i].r =  dal_fixed31_32_sub(dal_fixed31_32_mul_int(
+	pwl_rgb[i].r =  dc_fixpt_sub(dc_fixpt_mul_int(
 				pwl_rgb[i-1].r, 2), pwl_rgb[i-2].r);
-	pwl_rgb[i].g =  dal_fixed31_32_sub(dal_fixed31_32_mul_int(
+	pwl_rgb[i].g =  dc_fixpt_sub(dc_fixpt_mul_int(
 				pwl_rgb[i-1].g, 2), pwl_rgb[i-2].g);
-	pwl_rgb[i].b =  dal_fixed31_32_sub(dal_fixed31_32_mul_int(
+	pwl_rgb[i].b =  dc_fixpt_sub(dc_fixpt_mul_int(
 				pwl_rgb[i-1].b, 2), pwl_rgb[i-2].b);
+}
 
-	return true;
+/* todo: all these scale_gamma functions are inherently the same but
+ *  take different structures as params or different format for ramp
+ *  values. We could probably implement it in a more generic fashion
+ */
+static void scale_user_regamma_ramp(struct pwl_float_data *pwl_rgb,
+		const struct regamma_ramp *ramp,
+		struct dividers dividers)
+{
+	unsigned short max_driver = 0xFFFF;
+	unsigned short max_os = 0xFF00;
+	unsigned short scaler = max_os;
+	uint32_t i;
+	struct pwl_float_data *rgb = pwl_rgb;
+	struct pwl_float_data *rgb_last = rgb + GAMMA_RGB_256_ENTRIES - 1;
+
+	i = 0;
+	do {
+		if (ramp->gamma[i] > max_os ||
+				ramp->gamma[i + 256] > max_os ||
+				ramp->gamma[i + 512] > max_os) {
+			scaler = max_driver;
+			break;
+		}
+		i++;
+	} while (i != GAMMA_RGB_256_ENTRIES);
+
+	i = 0;
+	do {
+		rgb->r = dc_fixpt_from_fraction(
+				ramp->gamma[i], scaler);
+		rgb->g = dc_fixpt_from_fraction(
+				ramp->gamma[i + 256], scaler);
+		rgb->b = dc_fixpt_from_fraction(
+				ramp->gamma[i + 512], scaler);
+
+		++rgb;
+		++i;
+	} while (i != GAMMA_RGB_256_ENTRIES);
+
+	rgb->r = dc_fixpt_mul(rgb_last->r,
+			dividers.divider1);
+	rgb->g = dc_fixpt_mul(rgb_last->g,
+			dividers.divider1);
+	rgb->b = dc_fixpt_mul(rgb_last->b,
+			dividers.divider1);
+
+	++rgb;
+
+	rgb->r = dc_fixpt_mul(rgb_last->r,
+			dividers.divider2);
+	rgb->g = dc_fixpt_mul(rgb_last->g,
+			dividers.divider2);
+	rgb->b = dc_fixpt_mul(rgb_last->b,
+			dividers.divider2);
+
+	++rgb;
+
+	rgb->r = dc_fixpt_mul(rgb_last->r,
+			dividers.divider3);
+	rgb->g = dc_fixpt_mul(rgb_last->g,
+			dividers.divider3);
+	rgb->b = dc_fixpt_mul(rgb_last->b,
+			dividers.divider3);
 }
 
 /*
@@ -852,7 +913,7 @@ static void apply_lut_1d(
 	struct fixed31_32 lut2;
 	const int max_lut_index = 4095;
 	const struct fixed31_32 max_lut_index_f =
-			dal_fixed31_32_from_int_nonconst(max_lut_index);
+			dc_fixpt_from_int(max_lut_index);
 	int32_t index = 0, index_next = 0;
 	struct fixed31_32 index_f;
 	struct fixed31_32 delta_lut;
@@ -870,10 +931,10 @@ static void apply_lut_1d(
 			else
 				regamma_y = &tf_pts->blue[i];
 
-			norm_y = dal_fixed31_32_mul(max_lut_index_f,
+			norm_y = dc_fixpt_mul(max_lut_index_f,
 						   *regamma_y);
-			index = dal_fixed31_32_floor(norm_y);
-			index_f = dal_fixed31_32_from_int_nonconst(index);
+			index = dc_fixpt_floor(norm_y);
+			index_f = dc_fixpt_from_int(index);
 
 			if (index < 0 || index > max_lut_index)
 				continue;
@@ -892,11 +953,11 @@ static void apply_lut_1d(
 			}
 
 			// we have everything now, so interpolate
-			delta_lut = dal_fixed31_32_sub(lut2, lut1);
-			delta_index = dal_fixed31_32_sub(norm_y, index_f);
+			delta_lut = dc_fixpt_sub(lut2, lut1);
+			delta_index = dc_fixpt_sub(norm_y, index_f);
 
-			*regamma_y = dal_fixed31_32_add(lut1,
-				dal_fixed31_32_mul(delta_index, delta_lut));
+			*regamma_y = dc_fixpt_add(lut1,
+				dc_fixpt_mul(delta_index, delta_lut));
 		}
 	}
 }
@@ -912,7 +973,7 @@ static void build_evenly_distributed_points(
 	uint32_t i = 0;
 
 	do {
-		struct fixed31_32 value = dal_fixed31_32_from_fraction(i,
+		struct fixed31_32 value = dc_fixpt_from_fraction(i,
 			numberof_points - 1);
 
 		p->r = value;
@@ -923,21 +984,21 @@ static void build_evenly_distributed_points(
 		++i;
 	} while (i != numberof_points);
 
-	p->r = dal_fixed31_32_div(p_last->r, dividers.divider1);
-	p->g = dal_fixed31_32_div(p_last->g, dividers.divider1);
-	p->b = dal_fixed31_32_div(p_last->b, dividers.divider1);
+	p->r = dc_fixpt_div(p_last->r, dividers.divider1);
+	p->g = dc_fixpt_div(p_last->g, dividers.divider1);
+	p->b = dc_fixpt_div(p_last->b, dividers.divider1);
 
 	++p;
 
-	p->r = dal_fixed31_32_div(p_last->r, dividers.divider2);
-	p->g = dal_fixed31_32_div(p_last->g, dividers.divider2);
-	p->b = dal_fixed31_32_div(p_last->b, dividers.divider2);
+	p->r = dc_fixpt_div(p_last->r, dividers.divider2);
+	p->g = dc_fixpt_div(p_last->g, dividers.divider2);
+	p->b = dc_fixpt_div(p_last->b, dividers.divider2);
 
 	++p;
 
-	p->r = dal_fixed31_32_div(p_last->r, dividers.divider3);
-	p->g = dal_fixed31_32_div(p_last->g, dividers.divider3);
-	p->b = dal_fixed31_32_div(p_last->b, dividers.divider3);
+	p->r = dc_fixpt_div(p_last->r, dividers.divider3);
+	p->g = dc_fixpt_div(p_last->g, dividers.divider3);
+	p->b = dc_fixpt_div(p_last->b, dividers.divider3);
 }
 
 static inline void copy_rgb_regamma_to_coordinates_x(
@@ -949,7 +1010,7 @@ static inline void copy_rgb_regamma_to_coordinates_x(
 	uint32_t i = 0;
 	const struct pwl_float_data_ex *rgb_regamma = rgb_ex;
 
-	while (i <= hw_points_num) {
+	while (i <= hw_points_num + 1) {
 		coords->regamma_y_red = rgb_regamma->r;
 		coords->regamma_y_green = rgb_regamma->g;
 		coords->regamma_y_blue = rgb_regamma->b;
@@ -1002,6 +1063,102 @@ static bool calculate_interpolated_hardware_curve(
 	return true;
 }
 
+/* The "old" interpolation uses a complicated scheme to build an array of
+ * coefficients while also using an array of 0-255 normalized to 0-1
+ * Then there's another loop using both of the above + new scaled user ramp
+ * and we concatenate them. It also searches for points of interpolation and
+ * uses enums for positions.
+ *
+ * This function uses a different approach:
+ * user ramp is always applied on X with 0/255, 1/255, 2/255, ..., 255/255
+ * To find index for hwX , we notice the following:
+ * i/255 <= hwX < (i+1)/255  <=> i <= 255*hwX < i+1
+ * See apply_lut_1d which is the same principle, but on 4K entry 1D LUT
+ *
+ * Once the index is known, combined Y is simply:
+ * user_ramp(index) + (hwX-index/255)*(user_ramp(index+1) - user_ramp(index)
+ *
+ * We should switch to this method in all cases, it's simpler and faster
+ * ToDo one day - for now this only applies to ADL regamma to avoid regression
+ * for regular use cases (sRGB and PQ)
+ */
+static void interpolate_user_regamma(uint32_t hw_points_num,
+		struct pwl_float_data *rgb_user,
+		bool apply_degamma,
+		struct dc_transfer_func_distributed_points *tf_pts)
+{
+	uint32_t i;
+	uint32_t color = 0;
+	int32_t index;
+	int32_t index_next;
+	struct fixed31_32 *tf_point;
+	struct fixed31_32 hw_x;
+	struct fixed31_32 norm_factor =
+			dc_fixpt_from_int(255);
+	struct fixed31_32 norm_x;
+	struct fixed31_32 index_f;
+	struct fixed31_32 lut1;
+	struct fixed31_32 lut2;
+	struct fixed31_32 delta_lut;
+	struct fixed31_32 delta_index;
+
+	i = 0;
+	/* fixed_pt library has problems handling too small values */
+	while (i != 32) {
+		tf_pts->red[i] = dc_fixpt_zero;
+		tf_pts->green[i] = dc_fixpt_zero;
+		tf_pts->blue[i] = dc_fixpt_zero;
+		++i;
+	}
+	while (i <= hw_points_num + 1) {
+		for (color = 0; color < 3; color++) {
+			if (color == 0)
+				tf_point = &tf_pts->red[i];
+			else if (color == 1)
+				tf_point = &tf_pts->green[i];
+			else
+				tf_point = &tf_pts->blue[i];
+
+			if (apply_degamma) {
+				if (color == 0)
+					hw_x = coordinates_x[i].regamma_y_red;
+				else if (color == 1)
+					hw_x = coordinates_x[i].regamma_y_green;
+				else
+					hw_x = coordinates_x[i].regamma_y_blue;
+			} else
+				hw_x = coordinates_x[i].x;
+
+			norm_x = dc_fixpt_mul(norm_factor, hw_x);
+			index = dc_fixpt_floor(norm_x);
+			if (index < 0 || index > 255)
+				continue;
+
+			index_f = dc_fixpt_from_int(index);
+			index_next = (index == 255) ? index : index + 1;
+
+			if (color == 0) {
+				lut1 = rgb_user[index].r;
+				lut2 = rgb_user[index_next].r;
+			} else if (color == 1) {
+				lut1 = rgb_user[index].g;
+				lut2 = rgb_user[index_next].g;
+			} else {
+				lut1 = rgb_user[index].b;
+				lut2 = rgb_user[index_next].b;
+			}
+
+			// we have everything now, so interpolate
+			delta_lut = dc_fixpt_sub(lut2, lut1);
+			delta_index = dc_fixpt_sub(norm_x, index_f);
+
+			*tf_point = dc_fixpt_add(lut1,
+				dc_fixpt_mul(delta_index, delta_lut));
+		}
+		++i;
+	}
+}
+
 static void build_new_custom_resulted_curve(
 	uint32_t hw_points_num,
 	struct dc_transfer_func_distributed_points *tf_pts)
@@ -1011,16 +1168,39 @@ static void build_new_custom_resulted_curve(
 	i = 0;
 
 	while (i != hw_points_num + 1) {
-		tf_pts->red[i] = dal_fixed31_32_clamp(
-			tf_pts->red[i], dal_fixed31_32_zero,
-			dal_fixed31_32_one);
-		tf_pts->green[i] = dal_fixed31_32_clamp(
-			tf_pts->green[i], dal_fixed31_32_zero,
-			dal_fixed31_32_one);
-		tf_pts->blue[i] = dal_fixed31_32_clamp(
-			tf_pts->blue[i], dal_fixed31_32_zero,
-			dal_fixed31_32_one);
+		tf_pts->red[i] = dc_fixpt_clamp(
+			tf_pts->red[i], dc_fixpt_zero,
+			dc_fixpt_one);
+		tf_pts->green[i] = dc_fixpt_clamp(
+			tf_pts->green[i], dc_fixpt_zero,
+			dc_fixpt_one);
+		tf_pts->blue[i] = dc_fixpt_clamp(
+			tf_pts->blue[i], dc_fixpt_zero,
+			dc_fixpt_one);
+
+		++i;
+	}
+}
+
+static void apply_degamma_for_user_regamma(struct pwl_float_data_ex *rgb_regamma,
+		uint32_t hw_points_num)
+{
+	uint32_t i;
+
+	struct gamma_coefficients coeff;
+	struct pwl_float_data_ex *rgb = rgb_regamma;
+	const struct hw_x_point *coord_x = coordinates_x;
+
+	build_coefficients(&coeff, true);
 
+	i = 0;
+	while (i != hw_points_num + 1) {
+		rgb->r = translate_from_linear_space_ex(
+				coord_x->x, &coeff, 0);
+		rgb->g = rgb->r;
+		rgb->b = rgb->r;
+		++coord_x;
+		++rgb;
 		++i;
 	}
 }
@@ -1062,6 +1242,7 @@ static bool map_regamma_hw_to_x_user(
 		}
 	}
 
+	/* this should be named differently, all it does is clamp to 0-1 */
 	build_new_custom_resulted_curve(hw_points_num, tf_pts);
 
 	return true;
@@ -1109,9 +1290,9 @@ bool mod_color_calculate_regamma_params(struct dc_transfer_func *output_tf,
 	if (!coeff)
 		goto coeff_alloc_fail;
 
-	dividers.divider1 = dal_fixed31_32_from_fraction(3, 2);
-	dividers.divider2 = dal_fixed31_32_from_int(2);
-	dividers.divider3 = dal_fixed31_32_from_fraction(5, 2);
+	dividers.divider1 = dc_fixpt_from_fraction(3, 2);
+	dividers.divider2 = dc_fixpt_from_int(2);
+	dividers.divider3 = dc_fixpt_from_fraction(5, 2);
 
 	tf = output_tf->tf;
 
@@ -1168,6 +1349,113 @@ bool mod_color_calculate_regamma_params(struct dc_transfer_func *output_tf,
 	return ret;
 }
 
+bool calculate_user_regamma_coeff(struct dc_transfer_func *output_tf,
+		const struct regamma_lut *regamma)
+{
+	struct gamma_coefficients coeff;
+	const struct hw_x_point *coord_x = coordinates_x;
+	uint32_t i = 0;
+
+	do {
+		coeff.a0[i] = dc_fixpt_from_fraction(
+				regamma->coeff.A0[i], 10000000);
+		coeff.a1[i] = dc_fixpt_from_fraction(
+				regamma->coeff.A1[i], 1000);
+		coeff.a2[i] = dc_fixpt_from_fraction(
+				regamma->coeff.A2[i], 1000);
+		coeff.a3[i] = dc_fixpt_from_fraction(
+				regamma->coeff.A3[i], 1000);
+		coeff.user_gamma[i] = dc_fixpt_from_fraction(
+				regamma->coeff.gamma[i], 1000);
+
+		++i;
+	} while (i != 3);
+
+	i = 0;
+	/* fixed_pt library has problems handling too small values */
+	while (i != 32) {
+		output_tf->tf_pts.red[i] = dc_fixpt_zero;
+		output_tf->tf_pts.green[i] = dc_fixpt_zero;
+		output_tf->tf_pts.blue[i] = dc_fixpt_zero;
+		++coord_x;
+		++i;
+	}
+	while (i != MAX_HW_POINTS + 1) {
+		output_tf->tf_pts.red[i] = translate_from_linear_space_ex(
+				coord_x->x, &coeff, 0);
+		output_tf->tf_pts.green[i] = translate_from_linear_space_ex(
+				coord_x->x, &coeff, 1);
+		output_tf->tf_pts.blue[i] = translate_from_linear_space_ex(
+				coord_x->x, &coeff, 2);
+		++coord_x;
+		++i;
+	}
+
+	// this function just clamps output to 0-1
+	build_new_custom_resulted_curve(MAX_HW_POINTS, &output_tf->tf_pts);
+	output_tf->type = TF_TYPE_DISTRIBUTED_POINTS;
+
+	return true;
+}
+
+bool calculate_user_regamma_ramp(struct dc_transfer_func *output_tf,
+		const struct regamma_lut *regamma)
+{
+	struct dc_transfer_func_distributed_points *tf_pts = &output_tf->tf_pts;
+	struct dividers dividers;
+
+	struct pwl_float_data *rgb_user = NULL;
+	struct pwl_float_data_ex *rgb_regamma = NULL;
+	bool ret = false;
+
+	if (regamma == NULL)
+		return false;
+
+	output_tf->type = TF_TYPE_DISTRIBUTED_POINTS;
+
+	rgb_user = kzalloc(sizeof(*rgb_user) * (GAMMA_RGB_256_ENTRIES + _EXTRA_POINTS),
+			GFP_KERNEL);
+	if (!rgb_user)
+		goto rgb_user_alloc_fail;
+
+	rgb_regamma = kzalloc(sizeof(*rgb_regamma) * (MAX_HW_POINTS + _EXTRA_POINTS),
+			GFP_KERNEL);
+	if (!rgb_regamma)
+		goto rgb_regamma_alloc_fail;
+
+	dividers.divider1 = dc_fixpt_from_fraction(3, 2);
+	dividers.divider2 = dc_fixpt_from_int(2);
+	dividers.divider3 = dc_fixpt_from_fraction(5, 2);
+
+	scale_user_regamma_ramp(rgb_user, &regamma->ramp, dividers);
+
+	if (regamma->flags.bits.applyDegamma == 1) {
+		apply_degamma_for_user_regamma(rgb_regamma, MAX_HW_POINTS);
+		copy_rgb_regamma_to_coordinates_x(coordinates_x,
+				MAX_HW_POINTS, rgb_regamma);
+	}
+
+	interpolate_user_regamma(MAX_HW_POINTS, rgb_user,
+			regamma->flags.bits.applyDegamma, tf_pts);
+
+	// no custom HDR curves!
+	tf_pts->end_exponent = 0;
+	tf_pts->x_point_at_y1_red = 1;
+	tf_pts->x_point_at_y1_green = 1;
+	tf_pts->x_point_at_y1_blue = 1;
+
+	// this function just clamps output to 0-1
+	build_new_custom_resulted_curve(MAX_HW_POINTS, tf_pts);
+
+	ret = true;
+
+	kfree(rgb_regamma);
+rgb_regamma_alloc_fail:
+	kvfree(rgb_user);
+rgb_user_alloc_fail:
+	return ret;
+}
+
 bool mod_color_calculate_degamma_params(struct dc_transfer_func *input_tf,
 		const struct dc_gamma *ramp, bool mapUserRamp)
 {
@@ -1208,9 +1496,9 @@ bool mod_color_calculate_degamma_params(struct dc_transfer_func *input_tf,
 	if (!coeff)
 		goto coeff_alloc_fail;
 
-	dividers.divider1 = dal_fixed31_32_from_fraction(3, 2);
-	dividers.divider2 = dal_fixed31_32_from_int(2);
-	dividers.divider3 = dal_fixed31_32_from_fraction(5, 2);
+	dividers.divider1 = dc_fixpt_from_fraction(3, 2);
+	dividers.divider2 = dc_fixpt_from_int(2);
+	dividers.divider3 = dc_fixpt_from_fraction(5, 2);
 
 	tf = input_tf->tf;
 
diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.h b/drivers/gpu/drm/amd/display/modules/color/color_gamma.h
index b7f9bc27d101..b64048991a95 100644
--- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.h
+++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.h
@@ -32,6 +32,47 @@ struct dc_transfer_func_distributed_points;
 struct dc_rgb_fixed;
 enum dc_transfer_func_predefined;
 
+/* For SetRegamma ADL interface support
+ * Must match escape type
+ */
+union regamma_flags {
+	unsigned int raw;
+	struct {
+		unsigned int gammaRampArray       :1;    // RegammaRamp is in use
+		unsigned int gammaFromEdid        :1;    //gamma from edid is in use
+		unsigned int gammaFromEdidEx      :1;    //gamma from edid is in use , but only for Display Id 1.2
+		unsigned int gammaFromUser        :1;    //user custom gamma is used
+		unsigned int coeffFromUser        :1;    //coeff. A0-A3 from user is in use
+		unsigned int coeffFromEdid        :1;    //coeff. A0-A3 from edid is in use
+		unsigned int applyDegamma         :1;    //flag for additional degamma correction in driver
+		unsigned int gammaPredefinedSRGB  :1;    //flag for SRGB gamma
+		unsigned int gammaPredefinedPQ    :1;    //flag for PQ gamma
+		unsigned int gammaPredefinedPQ2084Interim :1;    //flag for PQ gamma, lower max nits
+		unsigned int gammaPredefined36    :1;    //flag for 3.6 gamma
+		unsigned int gammaPredefinedReset :1;    //flag to return to previous gamma
+	} bits;
+};
+
+struct regamma_ramp {
+	unsigned short gamma[256*3];  // gamma ramp packed  in same way as OS windows ,r , g & b
+};
+
+struct regamma_coeff {
+	int    gamma[3];
+	int    A0[3];
+	int    A1[3];
+	int    A2[3];
+	int    A3[3];
+};
+
+struct regamma_lut {
+	union regamma_flags flags;
+	union {
+		struct regamma_ramp ramp;
+		struct regamma_coeff coeff;
+	};
+};
+
 void setup_x_points_distribution(void);
 void precompute_pq(void);
 void precompute_de_pq(void);
@@ -45,9 +86,14 @@ bool mod_color_calculate_degamma_params(struct dc_transfer_func *output_tf,
 bool mod_color_calculate_curve(enum dc_transfer_func_predefined  trans,
 		struct dc_transfer_func_distributed_points *points);
 
-bool  mod_color_calculate_degamma_curve(enum dc_transfer_func_predefined trans,
+bool mod_color_calculate_degamma_curve(enum dc_transfer_func_predefined trans,
 				struct dc_transfer_func_distributed_points *points);
 
+bool calculate_user_regamma_coeff(struct dc_transfer_func *output_tf,
+		const struct regamma_lut *regamma);
+
+bool calculate_user_regamma_ramp(struct dc_transfer_func *output_tf,
+		const struct regamma_lut *regamma);
 
 
 #endif /* COLOR_MOD_COLOR_GAMMA_H_ */
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_stats.h b/drivers/gpu/drm/amd/display/modules/inc/mod_stats.h
index 3230e2adb870..3812094b52e8 100644
--- a/drivers/gpu/drm/amd/display/modules/inc/mod_stats.h
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_stats.h
@@ -46,6 +46,10 @@ void mod_stats_dump(struct mod_stats *mod_stats);
 
 void mod_stats_reset_data(struct mod_stats *mod_stats);
 
+void mod_stats_update_event(struct mod_stats *mod_stats,
+		char *event_string,
+		unsigned int length);
+
 void mod_stats_update_flip(struct mod_stats *mod_stats,
 		unsigned long timestamp_in_ns);
 
diff --git a/drivers/gpu/drm/amd/display/modules/stats/stats.c b/drivers/gpu/drm/amd/display/modules/stats/stats.c
index 041f87b73d5f..3f7d47fdc367 100644
--- a/drivers/gpu/drm/amd/display/modules/stats/stats.c
+++ b/drivers/gpu/drm/amd/display/modules/stats/stats.c
@@ -36,9 +36,14 @@
 #define DAL_STATS_ENTRIES_REGKEY_DEFAULT	0x00350000
 #define DAL_STATS_ENTRIES_REGKEY_MAX		0x01000000
 
+#define DAL_STATS_EVENT_ENTRIES_DEFAULT		0x00000100
+
 #define MOD_STATS_NUM_VSYNCS			5
+#define MOD_STATS_EVENT_STRING_MAX		512
 
 struct stats_time_cache {
+	unsigned int entry_id;
+
 	unsigned long flip_timestamp_in_ns;
 	unsigned long vupdate_timestamp_in_ns;
 
@@ -63,15 +68,26 @@ struct stats_time_cache {
 	unsigned int flags;
 };
 
+struct stats_event_cache {
+	unsigned int entry_id;
+	char event_string[MOD_STATS_EVENT_STRING_MAX];
+};
+
 struct core_stats {
 	struct mod_stats public;
 	struct dc *dc;
 
+	bool enabled;
+	unsigned int entries;
+	unsigned int event_entries;
+	unsigned int entry_id;
+
 	struct stats_time_cache *time;
 	unsigned int index;
 
-	bool enabled;
-	unsigned int entries;
+	struct stats_event_cache *events;
+	unsigned int event_index;
+
 };
 
 #define MOD_STATS_TO_CORE(mod_stats)\
@@ -99,12 +115,12 @@ struct mod_stats *mod_stats_create(struct dc *dc)
 	unsigned int reg_data;
 	int i = 0;
 
+	if (dc == NULL)
+		goto fail_construct;
+
 	core_stats = kzalloc(sizeof(struct core_stats), GFP_KERNEL);
 
 	if (core_stats == NULL)
-		goto fail_alloc_context;
-
-	if (dc == NULL)
 		goto fail_construct;
 
 	core_stats->dc = dc;
@@ -115,33 +131,55 @@ struct mod_stats *mod_stats_create(struct dc *dc)
 			&reg_data, sizeof(unsigned int), &flag))
 		core_stats->enabled = reg_data;
 
-	core_stats->entries = DAL_STATS_ENTRIES_REGKEY_DEFAULT;
-	if (dm_read_persistent_data(dc->ctx, NULL, NULL,
-			DAL_STATS_ENTRIES_REGKEY,
-			&reg_data, sizeof(unsigned int), &flag)) {
-		if (reg_data > DAL_STATS_ENTRIES_REGKEY_MAX)
-			core_stats->entries = DAL_STATS_ENTRIES_REGKEY_MAX;
-		else
-			core_stats->entries = reg_data;
-	}
+	if (core_stats->enabled) {
+		core_stats->entries = DAL_STATS_ENTRIES_REGKEY_DEFAULT;
+		if (dm_read_persistent_data(dc->ctx, NULL, NULL,
+				DAL_STATS_ENTRIES_REGKEY,
+				&reg_data, sizeof(unsigned int), &flag)) {
+			if (reg_data > DAL_STATS_ENTRIES_REGKEY_MAX)
+				core_stats->entries = DAL_STATS_ENTRIES_REGKEY_MAX;
+			else
+				core_stats->entries = reg_data;
+		}
+		core_stats->time = kzalloc(
+			sizeof(struct stats_time_cache) *
+				core_stats->entries,
+						GFP_KERNEL);
 
-	core_stats->time = kzalloc(sizeof(struct stats_time_cache) * core_stats->entries,
-					GFP_KERNEL);
+		if (core_stats->time == NULL)
+			goto fail_construct_time;
 
-	if (core_stats->time == NULL)
-		goto fail_construct;
+		core_stats->event_entries = DAL_STATS_EVENT_ENTRIES_DEFAULT;
+		core_stats->events = kzalloc(
+			sizeof(struct stats_event_cache) *
+				core_stats->event_entries,
+						GFP_KERNEL);
+
+		if (core_stats->events == NULL)
+			goto fail_construct_events;
+
+	} else {
+		core_stats->entries = 0;
+	}
 
 	/* Purposely leave index 0 unused so we don't need special logic to
 	 * handle calculation cases that depend on previous flip data.
 	 */
 	core_stats->index = 1;
+	core_stats->event_index = 0;
+
+	// Keeps track of ordering within the different stats structures
+	core_stats->entry_id = 0;
 
 	return &core_stats->public;
 
-fail_construct:
+fail_construct_events:
+	kfree(core_stats->time);
+
+fail_construct_time:
 	kfree(core_stats);
 
-fail_alloc_context:
+fail_construct:
 	return NULL;
 }
 
@@ -153,6 +191,9 @@ void mod_stats_destroy(struct mod_stats *mod_stats)
 		if (core_stats->time != NULL)
 			kfree(core_stats->time);
 
+		if (core_stats->events != NULL)
+			kfree(core_stats->events);
+
 		kfree(core_stats);
 	}
 }
@@ -163,7 +204,11 @@ void mod_stats_dump(struct mod_stats *mod_stats)
 	struct dal_logger *logger = NULL;
 	struct core_stats *core_stats = NULL;
 	struct stats_time_cache *time = NULL;
+	struct stats_event_cache *events = NULL;
+	unsigned int time_index = 1;
+	unsigned int event_index = 0;
 	unsigned int index = 0;
+	struct log_entry log_entry;
 
 	if (mod_stats == NULL)
 		return;
@@ -172,45 +217,62 @@ void mod_stats_dump(struct mod_stats *mod_stats)
 	dc = core_stats->dc;
 	logger = dc->ctx->logger;
 	time = core_stats->time;
-
-	//LogEntry* pLog = GetLog()->Open(LogMajor_ISR, LogMinor_ISR_FreeSyncSW);
-
-	//if (!pLog->IsDummyEntry())
-	{
-		dm_logger_write(logger, LOG_PROFILING, "==Display Caps==\n");
-		dm_logger_write(logger, LOG_PROFILING, "\n");
-		dm_logger_write(logger, LOG_PROFILING, "\n");
-
-		dm_logger_write(logger, LOG_PROFILING, "==Stats==\n");
-		dm_logger_write(logger, LOG_PROFILING,
-			"render avgRender minWindow midPoint maxWindow vsyncToFlip flipToVsync #vsyncBetweenFlip #frame insertDuration vTotalMin vTotalMax eventTrigs vSyncTime1 vSyncTime2 vSyncTime3 vSyncTime4 vSyncTime5 flags\n");
-
-		for (int i = 0; i < core_stats->index && i < core_stats->entries; i++) {
-			dm_logger_write(logger, LOG_PROFILING,
-					"%u  %u  %u  %u  %u  %u  %u  %u  %u  %u  %u  %u  %u  %u  %u  %u  %u  %u  %u\n",
-					time[i].render_time_in_us,
-					time[i].avg_render_time_in_us_last_ten,
-					time[i].min_window,
-					time[i].lfc_mid_point_in_us,
-					time[i].max_window,
-					time[i].vsync_to_flip_time_in_us,
-					time[i].flip_to_vsync_time_in_us,
-					time[i].num_vsync_between_flips,
-					time[i].num_frames_inserted,
-					time[i].inserted_duration_in_us,
-					time[i].v_total_min,
-					time[i].v_total_max,
-					time[i].event_triggers,
-					time[i].v_sync_time_in_us[0],
-					time[i].v_sync_time_in_us[1],
-					time[i].v_sync_time_in_us[2],
-					time[i].v_sync_time_in_us[3],
-					time[i].v_sync_time_in_us[4],
-					time[i].flags);
+	events = core_stats->events;
+
+	DISPLAY_STATS_BEGIN(log_entry);
+
+	DISPLAY_STATS("==Display Caps==\n");
+
+	DISPLAY_STATS("==Display Stats==\n");
+
+	DISPLAY_STATS("%10s %10s %10s %10s %10s"
+			" %11s %11s %17s %10s %14s"
+			" %10s %10s %10s %10s %10s"
+			" %10s %10s %10s %10s\n",
+		"render", "avgRender",
+		"minWindow", "midPoint", "maxWindow",
+		"vsyncToFlip", "flipToVsync", "vsyncsBetweenFlip",
+		"numFrame", "insertDuration",
+		"vTotalMin", "vTotalMax", "eventTrigs",
+		"vSyncTime1", "vSyncTime2", "vSyncTime3",
+		"vSyncTime4", "vSyncTime5", "flags");
+
+	for (int i = 0; i < core_stats->entry_id; i++) {
+		if (event_index < core_stats->event_index &&
+				i == events[event_index].entry_id) {
+			DISPLAY_STATS("%s\n", events[event_index].event_string);
+			event_index++;
+		} else if (time_index < core_stats->index &&
+				i == time[time_index].entry_id) {
+			DISPLAY_STATS("%10u %10u %10u %10u %10u"
+					" %11u %11u %17u %10u %14u"
+					" %10u %10u %10u %10u %10u"
+					" %10u %10u %10u %10u\n",
+				time[time_index].render_time_in_us,
+				time[time_index].avg_render_time_in_us_last_ten,
+				time[time_index].min_window,
+				time[time_index].lfc_mid_point_in_us,
+				time[time_index].max_window,
+				time[time_index].vsync_to_flip_time_in_us,
+				time[time_index].flip_to_vsync_time_in_us,
+				time[time_index].num_vsync_between_flips,
+				time[time_index].num_frames_inserted,
+				time[time_index].inserted_duration_in_us,
+				time[time_index].v_total_min,
+				time[time_index].v_total_max,
+				time[time_index].event_triggers,
+				time[time_index].v_sync_time_in_us[0],
+				time[time_index].v_sync_time_in_us[1],
+				time[time_index].v_sync_time_in_us[2],
+				time[time_index].v_sync_time_in_us[3],
+				time[time_index].v_sync_time_in_us[4],
+				time[time_index].flags);
+
+			time_index++;
 		}
 	}
-	//GetLog()->Close(pLog);
-	//GetLog()->UnSetLogMask(LogMajor_ISR, LogMinor_ISR_FreeSyncSW);
+
+	DISPLAY_STATS_END(log_entry);
 }
 
 void mod_stats_reset_data(struct mod_stats *mod_stats)
@@ -227,7 +289,46 @@ void mod_stats_reset_data(struct mod_stats *mod_stats)
 	memset(core_stats->time, 0,
 		sizeof(struct stats_time_cache) * core_stats->entries);
 
-	core_stats->index = 0;
+	memset(core_stats->events, 0,
+		sizeof(struct stats_event_cache) * core_stats->event_entries);
+
+	core_stats->index = 1;
+	core_stats->event_index = 0;
+
+	// Keeps track of ordering within the different stats structures
+	core_stats->entry_id = 0;
+}
+
+void mod_stats_update_event(struct mod_stats *mod_stats,
+		char *event_string,
+		unsigned int length)
+{
+	struct core_stats *core_stats = NULL;
+	struct stats_event_cache *events = NULL;
+	unsigned int index = 0;
+	unsigned int copy_length = 0;
+
+	if (mod_stats == NULL)
+		return;
+
+	core_stats = MOD_STATS_TO_CORE(mod_stats);
+
+	if (core_stats->event_index >= core_stats->event_entries)
+		return;
+
+	events = core_stats->events;
+	index = core_stats->event_index;
+
+	copy_length = length;
+	if (length > MOD_STATS_EVENT_STRING_MAX)
+		copy_length = MOD_STATS_EVENT_STRING_MAX;
+
+	memcpy(&events[index].event_string, event_string, copy_length);
+	events[index].event_string[copy_length - 1] = '\0';
+
+	events[index].entry_id = core_stats->entry_id;
+	core_stats->event_index++;
+	core_stats->entry_id++;
 }
 
 void mod_stats_update_flip(struct mod_stats *mod_stats,
@@ -250,7 +351,7 @@ void mod_stats_update_flip(struct mod_stats *mod_stats,
 
 	time[index].flip_timestamp_in_ns = timestamp_in_ns;
 	time[index].render_time_in_us =
-		timestamp_in_ns - time[index - 1].flip_timestamp_in_ns;
+		(timestamp_in_ns - time[index - 1].flip_timestamp_in_ns) / 1000;
 
 	if (index >= 10) {
 		for (unsigned int i = 0; i < 10; i++)
@@ -261,12 +362,16 @@ void mod_stats_update_flip(struct mod_stats *mod_stats,
 
 	if (time[index].num_vsync_between_flips > 0)
 		time[index].vsync_to_flip_time_in_us =
-			timestamp_in_ns - time[index].vupdate_timestamp_in_ns;
+			(timestamp_in_ns -
+				time[index].vupdate_timestamp_in_ns) / 1000;
 	else
 		time[index].vsync_to_flip_time_in_us =
-			timestamp_in_ns - time[index - 1].vupdate_timestamp_in_ns;
+			(timestamp_in_ns -
+				time[index - 1].vupdate_timestamp_in_ns) / 1000;
 
+	time[index].entry_id = core_stats->entry_id;
 	core_stats->index++;
+	core_stats->entry_id++;
 }
 
 void mod_stats_update_vupdate(struct mod_stats *mod_stats,
@@ -275,6 +380,8 @@ void mod_stats_update_vupdate(struct mod_stats *mod_stats,
 	struct core_stats *core_stats = NULL;
 	struct stats_time_cache *time = NULL;
 	unsigned int index = 0;
+	unsigned int num_vsyncs = 0;
+	unsigned int prev_vsync_in_ns = 0;
 
 	if (mod_stats == NULL)
 		return;
@@ -286,14 +393,27 @@ void mod_stats_update_vupdate(struct mod_stats *mod_stats,
 
 	time = core_stats->time;
 	index = core_stats->index;
+	num_vsyncs = time[index].num_vsync_between_flips;
+
+	if (num_vsyncs < MOD_STATS_NUM_VSYNCS) {
+		if (num_vsyncs == 0) {
+			prev_vsync_in_ns =
+				time[index - 1].vupdate_timestamp_in_ns;
+
+			time[index].flip_to_vsync_time_in_us =
+				(timestamp_in_ns -
+					time[index - 1].flip_timestamp_in_ns) /
+					1000;
+		} else {
+			prev_vsync_in_ns =
+				time[index].vupdate_timestamp_in_ns;
+		}
 
-	time[index].vupdate_timestamp_in_ns = timestamp_in_ns;
-	if (time[index].num_vsync_between_flips < MOD_STATS_NUM_VSYNCS)
-		time[index].v_sync_time_in_us[time[index].num_vsync_between_flips] =
-			timestamp_in_ns - time[index - 1].vupdate_timestamp_in_ns;
-	time[index].flip_to_vsync_time_in_us =
-		timestamp_in_ns - time[index - 1].flip_timestamp_in_ns;
+		time[index].v_sync_time_in_us[num_vsyncs] =
+			(timestamp_in_ns - prev_vsync_in_ns) / 1000;
+	}
 
+	time[index].vupdate_timestamp_in_ns = timestamp_in_ns;
 	time[index].num_vsync_between_flips++;
 }
 
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index 9fa3aaef3f33..b178176b72ac 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -92,7 +92,7 @@ enum amd_powergating_state {
 #define AMD_CG_SUPPORT_GFX_3D_CGLS		(1 << 21)
 #define AMD_CG_SUPPORT_DRM_MGCG			(1 << 22)
 #define AMD_CG_SUPPORT_DF_MGCG			(1 << 23)
-
+#define AMD_CG_SUPPORT_VCN_MGCG			(1 << 24)
 /* PG flags */
 #define AMD_PG_SUPPORT_GFX_PG			(1 << 0)
 #define AMD_PG_SUPPORT_GFX_SMG			(1 << 1)
@@ -108,6 +108,27 @@ enum amd_powergating_state {
 #define AMD_PG_SUPPORT_GFX_QUICK_MG		(1 << 11)
 #define AMD_PG_SUPPORT_GFX_PIPELINE		(1 << 12)
 #define AMD_PG_SUPPORT_MMHUB			(1 << 13)
+#define AMD_PG_SUPPORT_VCN			(1 << 14)
+
+enum PP_FEATURE_MASK {
+	PP_SCLK_DPM_MASK = 0x1,
+	PP_MCLK_DPM_MASK = 0x2,
+	PP_PCIE_DPM_MASK = 0x4,
+	PP_SCLK_DEEP_SLEEP_MASK = 0x8,
+	PP_POWER_CONTAINMENT_MASK = 0x10,
+	PP_UVD_HANDSHAKE_MASK = 0x20,
+	PP_SMC_VOLTAGE_CONTROL_MASK = 0x40,
+	PP_VBI_TIME_SUPPORT_MASK = 0x80,
+	PP_ULV_MASK = 0x100,
+	PP_ENABLE_GFX_CG_THRU_SMU = 0x200,
+	PP_CLOCK_STRETCH_MASK = 0x400,
+	PP_OD_FUZZY_FAN_CONTROL_MASK = 0x800,
+	PP_SOCCLK_DPM_MASK = 0x1000,
+	PP_DCEFCLK_DPM_MASK = 0x2000,
+	PP_OVERDRIVE_MASK = 0x4000,
+	PP_GFXOFF_MASK = 0x8000,
+	PP_ACG_MASK = 0x10000,
+};
 
 struct amd_ip_funcs {
 	/* Name of IP block */
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_offset.h
index f730d0629020..b6f74bf4af02 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_offset.h
@@ -2095,6 +2095,18 @@
 #define mmDC_GPIO_AUX_CTRL_2_BASE_IDX                                                                  2
 #define mmDC_GPIO_RXEN                                                                                 0x212f
 #define mmDC_GPIO_RXEN_BASE_IDX                                                                        2
+#define mmDC_GPIO_AUX_CTRL_3                                                                           0x2130
+#define mmDC_GPIO_AUX_CTRL_3_BASE_IDX                                                                  2
+#define mmDC_GPIO_AUX_CTRL_4                                                                           0x2131
+#define mmDC_GPIO_AUX_CTRL_4_BASE_IDX                                                                  2
+#define mmDC_GPIO_AUX_CTRL_5                                                                           0x2132
+#define mmDC_GPIO_AUX_CTRL_5_BASE_IDX                                                                  2
+#define mmAUXI2C_PAD_ALL_PWR_OK                                                                        0x2133
+#define mmAUXI2C_PAD_ALL_PWR_OK_BASE_IDX                                                               2
+#define mmDC_GPIO_PULLUPEN                                                                             0x2134
+#define mmDC_GPIO_PULLUPEN_BASE_IDX                                                                    2
+#define mmDC_GPIO_AUX_CTRL_6                                                                           0x2135
+#define mmDC_GPIO_AUX_CTRL_6_BASE_IDX                                                                  2
 #define mmBPHYC_DAC_MACRO_CNTL                                                                         0x2136
 #define mmBPHYC_DAC_MACRO_CNTL_BASE_IDX                                                                2
 #define mmDAC_MACRO_CNTL_RESERVED0                                                                     0x2136
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_sh_mask.h
index 6d3162c42957..bcd190a3fcdd 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_sh_mask.h
@@ -10971,6 +10971,158 @@
 #define DC_GPIO_RXEN__DC_GPIO_BLON_RXEN_MASK                                                                  0x00100000L
 #define DC_GPIO_RXEN__DC_GPIO_DIGON_RXEN_MASK                                                                 0x00200000L
 #define DC_GPIO_RXEN__DC_GPIO_ENA_BL_RXEN_MASK                                                                0x00400000L
+//DC_GPIO_AUX_CTRL_3
+#define DC_GPIO_AUX_CTRL_3__AUX1_NEN_RTERM__SHIFT                                                             0x0
+#define DC_GPIO_AUX_CTRL_3__AUX2_NEN_RTERM__SHIFT                                                             0x1
+#define DC_GPIO_AUX_CTRL_3__AUX3_NEN_RTERM__SHIFT                                                             0x2
+#define DC_GPIO_AUX_CTRL_3__AUX4_NEN_RTERM__SHIFT                                                             0x3
+#define DC_GPIO_AUX_CTRL_3__AUX5_NEN_RTERM__SHIFT                                                             0x4
+#define DC_GPIO_AUX_CTRL_3__AUX6_NEN_RTERM__SHIFT                                                             0x5
+#define DC_GPIO_AUX_CTRL_3__AUX1_DP_DN_SWAP__SHIFT                                                            0x8
+#define DC_GPIO_AUX_CTRL_3__AUX2_DP_DN_SWAP__SHIFT                                                            0x9
+#define DC_GPIO_AUX_CTRL_3__AUX3_DP_DN_SWAP__SHIFT                                                            0xa
+#define DC_GPIO_AUX_CTRL_3__AUX4_DP_DN_SWAP__SHIFT                                                            0xb
+#define DC_GPIO_AUX_CTRL_3__AUX5_DP_DN_SWAP__SHIFT                                                            0xc
+#define DC_GPIO_AUX_CTRL_3__AUX6_DP_DN_SWAP__SHIFT                                                            0xd
+#define DC_GPIO_AUX_CTRL_3__AUX1_HYS_TUNE__SHIFT                                                              0x10
+#define DC_GPIO_AUX_CTRL_3__AUX2_HYS_TUNE__SHIFT                                                              0x12
+#define DC_GPIO_AUX_CTRL_3__AUX3_HYS_TUNE__SHIFT                                                              0x14
+#define DC_GPIO_AUX_CTRL_3__AUX4_HYS_TUNE__SHIFT                                                              0x16
+#define DC_GPIO_AUX_CTRL_3__AUX5_HYS_TUNE__SHIFT                                                              0x18
+#define DC_GPIO_AUX_CTRL_3__AUX6_HYS_TUNE__SHIFT                                                              0x1a
+#define DC_GPIO_AUX_CTRL_3__AUX1_NEN_RTERM_MASK                                                               0x00000001L
+#define DC_GPIO_AUX_CTRL_3__AUX2_NEN_RTERM_MASK                                                               0x00000002L
+#define DC_GPIO_AUX_CTRL_3__AUX3_NEN_RTERM_MASK                                                               0x00000004L
+#define DC_GPIO_AUX_CTRL_3__AUX4_NEN_RTERM_MASK                                                               0x00000008L
+#define DC_GPIO_AUX_CTRL_3__AUX5_NEN_RTERM_MASK                                                               0x00000010L
+#define DC_GPIO_AUX_CTRL_3__AUX6_NEN_RTERM_MASK                                                               0x00000020L
+#define DC_GPIO_AUX_CTRL_3__AUX1_DP_DN_SWAP_MASK                                                              0x00000100L
+#define DC_GPIO_AUX_CTRL_3__AUX2_DP_DN_SWAP_MASK                                                              0x00000200L
+#define DC_GPIO_AUX_CTRL_3__AUX3_DP_DN_SWAP_MASK                                                              0x00000400L
+#define DC_GPIO_AUX_CTRL_3__AUX4_DP_DN_SWAP_MASK                                                              0x00000800L
+#define DC_GPIO_AUX_CTRL_3__AUX5_DP_DN_SWAP_MASK                                                              0x00001000L
+#define DC_GPIO_AUX_CTRL_3__AUX6_DP_DN_SWAP_MASK                                                              0x00002000L
+#define DC_GPIO_AUX_CTRL_3__AUX1_HYS_TUNE_MASK                                                                0x00030000L
+#define DC_GPIO_AUX_CTRL_3__AUX2_HYS_TUNE_MASK                                                                0x000C0000L
+#define DC_GPIO_AUX_CTRL_3__AUX3_HYS_TUNE_MASK                                                                0x00300000L
+#define DC_GPIO_AUX_CTRL_3__AUX4_HYS_TUNE_MASK                                                                0x00C00000L
+#define DC_GPIO_AUX_CTRL_3__AUX5_HYS_TUNE_MASK                                                                0x03000000L
+#define DC_GPIO_AUX_CTRL_3__AUX6_HYS_TUNE_MASK                                                                0x0C000000L
+//DC_GPIO_AUX_CTRL_4
+#define DC_GPIO_AUX_CTRL_4__AUX1_AUX_CTRL__SHIFT                                                              0x0
+#define DC_GPIO_AUX_CTRL_4__AUX2_AUX_CTRL__SHIFT                                                              0x4
+#define DC_GPIO_AUX_CTRL_4__AUX3_AUX_CTRL__SHIFT                                                              0x8
+#define DC_GPIO_AUX_CTRL_4__AUX4_AUX_CTRL__SHIFT                                                              0xc
+#define DC_GPIO_AUX_CTRL_4__AUX5_AUX_CTRL__SHIFT                                                              0x10
+#define DC_GPIO_AUX_CTRL_4__AUX6_AUX_CTRL__SHIFT                                                              0x14
+#define DC_GPIO_AUX_CTRL_4__AUX1_AUX_CTRL_MASK                                                                0x0000000FL
+#define DC_GPIO_AUX_CTRL_4__AUX2_AUX_CTRL_MASK                                                                0x000000F0L
+#define DC_GPIO_AUX_CTRL_4__AUX3_AUX_CTRL_MASK                                                                0x00000F00L
+#define DC_GPIO_AUX_CTRL_4__AUX4_AUX_CTRL_MASK                                                                0x0000F000L
+#define DC_GPIO_AUX_CTRL_4__AUX5_AUX_CTRL_MASK                                                                0x000F0000L
+#define DC_GPIO_AUX_CTRL_4__AUX6_AUX_CTRL_MASK                                                                0x00F00000L
+//DC_GPIO_AUX_CTRL_5
+#define DC_GPIO_AUX_CTRL_5__AUX1_VOD_TUNE__SHIFT                                                              0x0
+#define DC_GPIO_AUX_CTRL_5__AUX2_VOD_TUNE__SHIFT                                                              0x2
+#define DC_GPIO_AUX_CTRL_5__AUX3_VOD_TUNE__SHIFT                                                              0x4
+#define DC_GPIO_AUX_CTRL_5__AUX4_VOD_TUNE__SHIFT                                                              0x6
+#define DC_GPIO_AUX_CTRL_5__AUX5_VOD_TUNE__SHIFT                                                              0x8
+#define DC_GPIO_AUX_CTRL_5__AUX6_VOD_TUNE__SHIFT                                                              0xa
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD1_I2CMODE__SHIFT                                                           0xc
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD2_I2CMODE__SHIFT                                                           0xd
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD3_I2CMODE__SHIFT                                                           0xe
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD4_I2CMODE__SHIFT                                                           0xf
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD5_I2CMODE__SHIFT                                                           0x10
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD6_I2CMODE__SHIFT                                                           0x11
+#define DC_GPIO_AUX_CTRL_5__DDC1_I2C_VPH_1V2_EN__SHIFT                                                        0x12
+#define DC_GPIO_AUX_CTRL_5__DDC2_I2C_VPH_1V2_EN__SHIFT                                                        0x13
+#define DC_GPIO_AUX_CTRL_5__DDC3_I2C_VPH_1V2_EN__SHIFT                                                        0x14
+#define DC_GPIO_AUX_CTRL_5__DDC4_I2C_VPH_1V2_EN__SHIFT                                                        0x15
+#define DC_GPIO_AUX_CTRL_5__DDC5_I2C_VPH_1V2_EN__SHIFT                                                        0x16
+#define DC_GPIO_AUX_CTRL_5__DDC6_I2C_VPH_1V2_EN__SHIFT                                                        0x17
+#define DC_GPIO_AUX_CTRL_5__DDC1_PAD_I2C_CTRL__SHIFT                                                          0x18
+#define DC_GPIO_AUX_CTRL_5__DDC2_PAD_I2C_CTRL__SHIFT                                                          0x19
+#define DC_GPIO_AUX_CTRL_5__DDC3_PAD_I2C_CTRL__SHIFT                                                          0x1a
+#define DC_GPIO_AUX_CTRL_5__DDC4_PAD_I2C_CTRL__SHIFT                                                          0x1b
+#define DC_GPIO_AUX_CTRL_5__DDC5_PAD_I2C_CTRL__SHIFT                                                          0x1c
+#define DC_GPIO_AUX_CTRL_5__DDC6_PAD_I2C_CTRL__SHIFT                                                          0x1d
+#define DC_GPIO_AUX_CTRL_5__AUX1_VOD_TUNE_MASK                                                                0x00000003L
+#define DC_GPIO_AUX_CTRL_5__AUX2_VOD_TUNE_MASK                                                                0x0000000CL
+#define DC_GPIO_AUX_CTRL_5__AUX3_VOD_TUNE_MASK                                                                0x00000030L
+#define DC_GPIO_AUX_CTRL_5__AUX4_VOD_TUNE_MASK                                                                0x000000C0L
+#define DC_GPIO_AUX_CTRL_5__AUX5_VOD_TUNE_MASK                                                                0x00000300L
+#define DC_GPIO_AUX_CTRL_5__AUX6_VOD_TUNE_MASK                                                                0x00000C00L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD1_I2CMODE_MASK                                                             0x00001000L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD2_I2CMODE_MASK                                                             0x00002000L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD3_I2CMODE_MASK                                                             0x00004000L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD4_I2CMODE_MASK                                                             0x00008000L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD5_I2CMODE_MASK                                                             0x00010000L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD6_I2CMODE_MASK                                                             0x00020000L
+#define DC_GPIO_AUX_CTRL_5__DDC1_I2C_VPH_1V2_EN_MASK                                                          0x00040000L
+#define DC_GPIO_AUX_CTRL_5__DDC2_I2C_VPH_1V2_EN_MASK                                                          0x00080000L
+#define DC_GPIO_AUX_CTRL_5__DDC3_I2C_VPH_1V2_EN_MASK                                                          0x00100000L
+#define DC_GPIO_AUX_CTRL_5__DDC4_I2C_VPH_1V2_EN_MASK                                                          0x00200000L
+#define DC_GPIO_AUX_CTRL_5__DDC5_I2C_VPH_1V2_EN_MASK                                                          0x00400000L
+#define DC_GPIO_AUX_CTRL_5__DDC6_I2C_VPH_1V2_EN_MASK                                                          0x00800000L
+#define DC_GPIO_AUX_CTRL_5__DDC1_PAD_I2C_CTRL_MASK                                                            0x01000000L
+#define DC_GPIO_AUX_CTRL_5__DDC2_PAD_I2C_CTRL_MASK                                                            0x02000000L
+#define DC_GPIO_AUX_CTRL_5__DDC3_PAD_I2C_CTRL_MASK                                                            0x04000000L
+#define DC_GPIO_AUX_CTRL_5__DDC4_PAD_I2C_CTRL_MASK                                                            0x08000000L
+#define DC_GPIO_AUX_CTRL_5__DDC5_PAD_I2C_CTRL_MASK                                                            0x10000000L
+#define DC_GPIO_AUX_CTRL_5__DDC6_PAD_I2C_CTRL_MASK                                                            0x20000000L
+//AUXI2C_PAD_ALL_PWR_OK
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY1_ALL_PWR_OK__SHIFT                                                  0x0
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY2_ALL_PWR_OK__SHIFT                                                  0x1
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY3_ALL_PWR_OK__SHIFT                                                  0x2
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY4_ALL_PWR_OK__SHIFT                                                  0x3
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY5_ALL_PWR_OK__SHIFT                                                  0x4
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY6_ALL_PWR_OK__SHIFT                                                  0x5
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY1_ALL_PWR_OK_MASK                                                    0x00000001L
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY2_ALL_PWR_OK_MASK                                                    0x00000002L
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY3_ALL_PWR_OK_MASK                                                    0x00000004L
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY4_ALL_PWR_OK_MASK                                                    0x00000008L
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY5_ALL_PWR_OK_MASK                                                    0x00000010L
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY6_ALL_PWR_OK_MASK                                                    0x00000020L
+//DC_GPIO_PULLUPEN
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICA_PU_EN__SHIFT                                                       0x0
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICB_PU_EN__SHIFT                                                       0x1
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICC_PU_EN__SHIFT                                                       0x2
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICD_PU_EN__SHIFT                                                       0x3
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICE_PU_EN__SHIFT                                                       0x4
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICF_PU_EN__SHIFT                                                       0x5
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICG_PU_EN__SHIFT                                                       0x6
+#define DC_GPIO_PULLUPEN__DC_GPIO_HSYNCA_PU_EN__SHIFT                                                         0x8
+#define DC_GPIO_PULLUPEN__DC_GPIO_VSYNCA_PU_EN__SHIFT                                                         0x9
+#define DC_GPIO_PULLUPEN__DC_GPIO_HPD1_PU_EN__SHIFT                                                           0xe
+#define DC_GPIO_PULLUPEN__DC_GPIO_BLON_PU_EN__SHIFT                                                           0x14
+#define DC_GPIO_PULLUPEN__DC_GPIO_DIGON_PU_EN__SHIFT                                                          0x15
+#define DC_GPIO_PULLUPEN__DC_GPIO_ENA_BL_PU_EN__SHIFT                                                         0x16
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICA_PU_EN_MASK                                                         0x00000001L
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICB_PU_EN_MASK                                                         0x00000002L
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICC_PU_EN_MASK                                                         0x00000004L
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICD_PU_EN_MASK                                                         0x00000008L
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICE_PU_EN_MASK                                                         0x00000010L
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICF_PU_EN_MASK                                                         0x00000020L
+#define DC_GPIO_PULLUPEN__DC_GPIO_GENERICG_PU_EN_MASK                                                         0x00000040L
+#define DC_GPIO_PULLUPEN__DC_GPIO_HSYNCA_PU_EN_MASK                                                           0x00000100L
+#define DC_GPIO_PULLUPEN__DC_GPIO_VSYNCA_PU_EN_MASK                                                           0x00000200L
+#define DC_GPIO_PULLUPEN__DC_GPIO_HPD1_PU_EN_MASK                                                             0x00004000L
+#define DC_GPIO_PULLUPEN__DC_GPIO_BLON_PU_EN_MASK                                                             0x00100000L
+#define DC_GPIO_PULLUPEN__DC_GPIO_DIGON_PU_EN_MASK                                                            0x00200000L
+#define DC_GPIO_PULLUPEN__DC_GPIO_ENA_BL_PU_EN_MASK                                                           0x00400000L
+//DC_GPIO_AUX_CTRL_6
+#define DC_GPIO_AUX_CTRL_6__AUX1_PAD_RXSEL__SHIFT                                                             0x0
+#define DC_GPIO_AUX_CTRL_6__AUX2_PAD_RXSEL__SHIFT                                                             0x2
+#define DC_GPIO_AUX_CTRL_6__AUX3_PAD_RXSEL__SHIFT                                                             0x4
+#define DC_GPIO_AUX_CTRL_6__AUX4_PAD_RXSEL__SHIFT                                                             0x6
+#define DC_GPIO_AUX_CTRL_6__AUX5_PAD_RXSEL__SHIFT                                                             0x8
+#define DC_GPIO_AUX_CTRL_6__AUX6_PAD_RXSEL__SHIFT                                                             0xa
+#define DC_GPIO_AUX_CTRL_6__AUX1_PAD_RXSEL_MASK                                                               0x00000003L
+#define DC_GPIO_AUX_CTRL_6__AUX2_PAD_RXSEL_MASK                                                               0x0000000CL
+#define DC_GPIO_AUX_CTRL_6__AUX3_PAD_RXSEL_MASK                                                               0x00000030L
+#define DC_GPIO_AUX_CTRL_6__AUX4_PAD_RXSEL_MASK                                                               0x000000C0L
+#define DC_GPIO_AUX_CTRL_6__AUX5_PAD_RXSEL_MASK                                                               0x00000300L
+#define DC_GPIO_AUX_CTRL_6__AUX6_PAD_RXSEL_MASK                                                               0x00000C00L
 //BPHYC_DAC_MACRO_CNTL
 #define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_WHITE_LEVEL__SHIFT                                                    0x0
 #define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_WHITE_FINE_CONTROL__SHIFT                                             0x8
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_offset.h
index 4ccf9681c45d..721c61171045 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_offset.h
@@ -3895,6 +3895,10 @@
 #define mmCM0_CM_MEM_PWR_CTRL_BASE_IDX                                                                 2
 #define mmCM0_CM_MEM_PWR_STATUS                                                                        0x0d33
 #define mmCM0_CM_MEM_PWR_STATUS_BASE_IDX                                                               2
+#define mmCM0_CM_TEST_DEBUG_INDEX                                                                      0x0d35
+#define mmCM0_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM0_CM_TEST_DEBUG_DATA                                                                       0x0d36
+#define mmCM0_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp0_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -4367,7 +4371,10 @@
 #define mmCM1_CM_MEM_PWR_CTRL_BASE_IDX                                                                 2
 #define mmCM1_CM_MEM_PWR_STATUS                                                                        0x0e4e
 #define mmCM1_CM_MEM_PWR_STATUS_BASE_IDX                                                               2
-
+#define mmCM1_CM_TEST_DEBUG_INDEX                                                                      0x0e50
+#define mmCM1_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM1_CM_TEST_DEBUG_DATA                                                                       0x0e51
+#define mmCM1_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 // addressBlock: dce_dc_dpp1_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
 // base address: 0x399c
@@ -4839,7 +4846,10 @@
 #define mmCM2_CM_MEM_PWR_CTRL_BASE_IDX                                                                 2
 #define mmCM2_CM_MEM_PWR_STATUS                                                                        0x0f69
 #define mmCM2_CM_MEM_PWR_STATUS_BASE_IDX                                                               2
-
+#define mmCM2_CM_TEST_DEBUG_INDEX                                                                      0x0f6b
+#define mmCM2_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM2_CM_TEST_DEBUG_DATA                                                                       0x0f6c
+#define mmCM2_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 // addressBlock: dce_dc_dpp2_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
 // base address: 0x3e08
@@ -5311,7 +5321,10 @@
 #define mmCM3_CM_MEM_PWR_CTRL_BASE_IDX                                                                 2
 #define mmCM3_CM_MEM_PWR_STATUS                                                                        0x1084
 #define mmCM3_CM_MEM_PWR_STATUS_BASE_IDX                                                               2
-
+#define mmCM3_CM_TEST_DEBUG_INDEX                                                                      0x1086
+#define mmCM3_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM3_CM_TEST_DEBUG_DATA                                                                       0x1087
+#define mmCM3_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 // addressBlock: dce_dc_dpp3_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
 // base address: 0x4274
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_sh_mask.h
index e2a2f114bd8e..e7c0cad41081 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_sh_mask.h
@@ -14049,6 +14049,14 @@
 #define CM0_CM_MEM_PWR_STATUS__RGAM_MEM_PWR_STATE__SHIFT                                                      0x2
 #define CM0_CM_MEM_PWR_STATUS__SHARED_MEM_PWR_STATE_MASK                                                      0x00000003L
 #define CM0_CM_MEM_PWR_STATUS__RGAM_MEM_PWR_STATE_MASK                                                        0x0000000CL
+//CM0_CM_TEST_DEBUG_INDEX
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX__SHIFT                                                   0x0
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN__SHIFT                                                0x8
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX_MASK                                                     0x000000FFL
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN_MASK                                                  0x00000100L
+//CM0_CM_TEST_DEBUG_DATA
+#define CM0_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA__SHIFT                                                     0x0
+#define CM0_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA_MASK                                                       0xFFFFFFFFL
 
 
 // addressBlock: dce_dc_dpp0_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_default.h b/drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_default.h
new file mode 100644
index 000000000000..9e19e723081b
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_default.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2018  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _df_1_7_DEFAULT_HEADER
+#define _df_1_7_DEFAULT_HEADER
+
+#define mmFabricConfigAccessControl_DEFAULT						0x00000000
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_offset.h b/drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_offset.h
new file mode 100644
index 000000000000..e6044e27a913
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_offset.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2018  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _df_1_7_OFFSET_HEADER
+#define _df_1_7_OFFSET_HEADER
+
+#define mmFabricConfigAccessControl									0x0410
+#define mmFabricConfigAccessControl_BASE_IDX								0
+
+#define mmDF_PIE_AON0_DfGlobalClkGater									0x00fc
+#define mmDF_PIE_AON0_DfGlobalClkGater_BASE_IDX								0
+
+#define mmDF_CS_AON0_DramBaseAddress0									0x0044
+#define mmDF_CS_AON0_DramBaseAddress0_BASE_IDX								0
+
+#define mmDF_CS_AON0_CoherentSlaveModeCtrlA0								0x0214
+#define mmDF_CS_AON0_CoherentSlaveModeCtrlA0_BASE_IDX							0
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_sh_mask.h
new file mode 100644
index 000000000000..a78c99480e2d
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/df/df_1_7_sh_mask.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2018  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _df_1_7_SH_MASK_HEADER
+#define _df_1_7_SH_MASK_HEADER
+
+/* FabricConfigAccessControl */
+#define FabricConfigAccessControl__CfgRegInstAccEn__SHIFT						0x0
+#define FabricConfigAccessControl__CfgRegInstAccRegLock__SHIFT						0x1
+#define FabricConfigAccessControl__CfgRegInstID__SHIFT							0x10
+#define FabricConfigAccessControl__CfgRegInstAccEn_MASK							0x00000001L
+#define FabricConfigAccessControl__CfgRegInstAccRegLock_MASK						0x00000002L
+#define FabricConfigAccessControl__CfgRegInstID_MASK							0x00FF0000L
+
+/* DF_PIE_AON0_DfGlobalClkGater */
+#define DF_PIE_AON0_DfGlobalClkGater__MGCGMode__SHIFT							0x0
+#define DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK							0x0000000FL
+
+/* DF_CS_AON0_DramBaseAddress0 */
+#define DF_CS_AON0_DramBaseAddress0__AddrRngVal__SHIFT							0x0
+#define DF_CS_AON0_DramBaseAddress0__LgcyMmioHoleEn__SHIFT						0x1
+#define DF_CS_AON0_DramBaseAddress0__IntLvNumChan__SHIFT						0x4
+#define DF_CS_AON0_DramBaseAddress0__IntLvAddrSel__SHIFT						0x8
+#define DF_CS_AON0_DramBaseAddress0__DramBaseAddr__SHIFT						0xc
+#define DF_CS_AON0_DramBaseAddress0__AddrRngVal_MASK							0x00000001L
+#define DF_CS_AON0_DramBaseAddress0__LgcyMmioHoleEn_MASK						0x00000002L
+#define DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK							0x000000F0L
+#define DF_CS_AON0_DramBaseAddress0__IntLvAddrSel_MASK							0x00000700L
+#define DF_CS_AON0_DramBaseAddress0__DramBaseAddr_MASK							0xFFFFF000L
+
+//DF_CS_AON0_CoherentSlaveModeCtrlA0
+#define DF_CS_AON0_CoherentSlaveModeCtrlA0__ForceParWrRMW__SHIFT					0x3
+#define DF_CS_AON0_CoherentSlaveModeCtrlA0__ForceParWrRMW_MASK						0x00000008L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_default.h b/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_default.h
new file mode 100644
index 000000000000..e58c207ac980
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_default.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2018  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _df_3_6_DEFAULT_HEADER
+#define _df_3_6_DEFAULT_HEADER
+
+#define mmFabricConfigAccessControl_DEFAULT						0x00000000
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_offset.h b/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_offset.h
new file mode 100644
index 000000000000..a9575db8d7aa
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_offset.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2018  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _df_3_6_OFFSET_HEADER
+#define _df_3_6_OFFSET_HEADER
+
+#define mmFabricConfigAccessControl									0x0410
+#define mmFabricConfigAccessControl_BASE_IDX								0
+
+#define mmDF_PIE_AON0_DfGlobalClkGater									0x00fc
+#define mmDF_PIE_AON0_DfGlobalClkGater_BASE_IDX								0
+
+#define mmDF_CS_UMC_AON0_DramBaseAddress0								0x0044
+#define mmDF_CS_UMC_AON0_DramBaseAddress0_BASE_IDX							0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_sh_mask.h
new file mode 100644
index 000000000000..88f7c69df6b9
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_sh_mask.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2018  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _df_3_6_SH_MASK_HEADER
+#define _df_3_6_SH_MASK_HEADER
+
+/* FabricConfigAccessControl */
+#define FabricConfigAccessControl__CfgRegInstAccEn__SHIFT						0x0
+#define FabricConfigAccessControl__CfgRegInstAccRegLock__SHIFT						0x1
+#define FabricConfigAccessControl__CfgRegInstID__SHIFT							0x10
+#define FabricConfigAccessControl__CfgRegInstAccEn_MASK							0x00000001L
+#define FabricConfigAccessControl__CfgRegInstAccRegLock_MASK						0x00000002L
+#define FabricConfigAccessControl__CfgRegInstID_MASK							0x00FF0000L
+
+/* DF_PIE_AON0_DfGlobalClkGater */
+#define DF_PIE_AON0_DfGlobalClkGater__MGCGMode__SHIFT							0x0
+#define DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK							0x0000000FL
+
+/* DF_CS_AON0_DramBaseAddress0 */
+#define DF_CS_UMC_AON0_DramBaseAddress0__AddrRngVal__SHIFT						0x0
+#define DF_CS_UMC_AON0_DramBaseAddress0__LgcyMmioHoleEn__SHIFT						0x1
+#define DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan__SHIFT						0x4
+#define DF_CS_UMC_AON0_DramBaseAddress0__IntLvAddrSel__SHIFT						0x8
+#define DF_CS_UMC_AON0_DramBaseAddress0__DramBaseAddr__SHIFT						0xc
+#define DF_CS_UMC_AON0_DramBaseAddress0__AddrRngVal_MASK						0x00000001L
+#define DF_CS_UMC_AON0_DramBaseAddress0__LgcyMmioHoleEn_MASK						0x00000002L
+#define DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK						0x000000F0L
+#define DF_CS_UMC_AON0_DramBaseAddress0__IntLvAddrSel_MASK						0x00000700L
+#define DF_CS_UMC_AON0_DramBaseAddress0__DramBaseAddr_MASK						0xFFFFF000L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/atombios.h b/drivers/gpu/drm/amd/include/atombios.h
index f696bbb643ef..7931502fa54f 100644
--- a/drivers/gpu/drm/amd/include/atombios.h
+++ b/drivers/gpu/drm/amd/include/atombios.h
@@ -632,6 +632,13 @@ typedef struct _COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_2
   ULONG ulReserved;
 }COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_2;
 
+typedef struct _COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_3
+{
+  COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4 ulClock;
+  USHORT  usMclk_fcw_frac;                  //fractional divider of fcw = usSclk_fcw_frac/65536
+  USHORT  usMclk_fcw_int;                   //integer divider of fcwc
+}COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_3;
+
 //Input parameter of DynamicMemorySettingsTable
 //when ATOM_COMPUTE_CLOCK_FREQ.ulComputeClockFlag = COMPUTE_MEMORY_PLL_PARAM
 typedef struct _DYNAMICE_MEMORY_SETTINGS_PARAMETER
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index de177ce8ca80..c6c1666ac120 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -1219,6 +1219,41 @@ struct  atom_gfx_info_v2_3 {
   uint32_t rm21_sram_vmin_value;
 };
 
+struct  atom_gfx_info_v2_4 {
+  struct  atom_common_table_header  table_header;
+  uint8_t gfxip_min_ver;
+  uint8_t gfxip_max_ver;
+  uint8_t gc_num_se;
+  uint8_t max_tile_pipes;
+  uint8_t gc_num_cu_per_sh;
+  uint8_t gc_num_sh_per_se;
+  uint8_t gc_num_rb_per_se;
+  uint8_t gc_num_tccs;
+  uint32_t regaddr_cp_dma_src_addr;
+  uint32_t regaddr_cp_dma_src_addr_hi;
+  uint32_t regaddr_cp_dma_dst_addr;
+  uint32_t regaddr_cp_dma_dst_addr_hi;
+  uint32_t regaddr_cp_dma_command;
+  uint32_t regaddr_cp_status;
+  uint32_t regaddr_rlc_gpu_clock_32;
+  uint32_t rlc_gpu_timer_refclk;
+  uint8_t active_cu_per_sh;
+  uint8_t active_rb_per_se;
+  uint16_t gcgoldenoffset;
+  uint16_t gc_num_gprs;
+  uint16_t gc_gsprim_buff_depth;
+  uint16_t gc_parameter_cache_depth;
+  uint16_t gc_wave_size;
+  uint16_t gc_max_waves_per_simd;
+  uint16_t gc_lds_size;
+  uint8_t gc_num_max_gs_thds;
+  uint8_t gc_gs_table_depth;
+  uint8_t gc_double_offchip_lds_buffer;
+  uint8_t gc_max_scratch_slots_per_cu;
+  uint32_t sram_rm_fuses_val;
+  uint32_t sram_custom_rm_fuses_val;
+};
+
 /* 
   ***************************************************************************
     Data Table smu_info  structure
diff --git a/drivers/gpu/drm/amd/include/cgs_common.h b/drivers/gpu/drm/amd/include/cgs_common.h
index f2814ae7ecdd..a69deb3a2ac0 100644
--- a/drivers/gpu/drm/amd/include/cgs_common.h
+++ b/drivers/gpu/drm/amd/include/cgs_common.h
@@ -42,20 +42,6 @@ enum cgs_ind_reg {
 	CGS_IND_REG__AUDIO_ENDPT
 };
 
-/**
- * enum cgs_engine - Engines that can be statically power-gated
- */
-enum cgs_engine {
-	CGS_ENGINE__UVD,
-	CGS_ENGINE__VCE,
-	CGS_ENGINE__VP8,
-	CGS_ENGINE__ACP_DMA,
-	CGS_ENGINE__ACP_DSP0,
-	CGS_ENGINE__ACP_DSP1,
-	CGS_ENGINE__ISP,
-	/* ... */
-};
-
 /*
  * enum cgs_ucode_id - Firmware types for different IPs
  */
@@ -76,17 +62,6 @@ enum cgs_ucode_id {
 	CGS_UCODE_ID_MAXIMUM,
 };
 
-/*
- * enum cgs_resource_type - GPU resource type
- */
-enum cgs_resource_type {
-	CGS_RESOURCE_TYPE_MMIO = 0,
-	CGS_RESOURCE_TYPE_FB,
-	CGS_RESOURCE_TYPE_IO,
-	CGS_RESOURCE_TYPE_DOORBELL,
-	CGS_RESOURCE_TYPE_ROM,
-};
-
 /**
  * struct cgs_firmware_info - Firmware information
  */
@@ -104,17 +79,6 @@ struct cgs_firmware_info {
 	bool			is_kicker;
 };
 
-struct cgs_mode_info {
-	uint32_t		refresh_rate;
-	uint32_t		vblank_time_us;
-};
-
-struct cgs_display_info {
-	uint32_t		display_count;
-	uint32_t		active_display_mask;
-	struct cgs_mode_info *mode_info;
-};
-
 typedef unsigned long cgs_handle_t;
 
 /**
@@ -170,119 +134,18 @@ typedef void (*cgs_write_ind_register_t)(struct cgs_device *cgs_device, enum cgs
 #define CGS_WREG32_FIELD_IND(device, space, reg, field, val)	\
 	cgs_write_ind_register(device, space, ix##reg, (cgs_read_ind_register(device, space, ix##reg) & ~CGS_REG_FIELD_MASK(reg, field)) | (val) << CGS_REG_FIELD_SHIFT(reg, field))
 
-/**
- * cgs_get_pci_resource() - provide access to a device resource (PCI BAR)
- * @cgs_device:	opaque device handle
- * @resource_type:	Type of Resource (MMIO, IO, ROM, FB, DOORBELL)
- * @size:	size of the region
- * @offset:	offset from the start of the region
- * @resource_base:	base address (not including offset) returned
- *
- * Return: 0 on success, -errno otherwise
- */
-typedef int (*cgs_get_pci_resource_t)(struct cgs_device *cgs_device,
-				      enum cgs_resource_type resource_type,
-				      uint64_t size,
-				      uint64_t offset,
-				      uint64_t *resource_base);
-
-/**
- * cgs_atom_get_data_table() - Get a pointer to an ATOM BIOS data table
- * @cgs_device:	opaque device handle
- * @table:	data table index
- * @size:	size of the table (output, may be NULL)
- * @frev:	table format revision (output, may be NULL)
- * @crev:	table content revision (output, may be NULL)
- *
- * Return: Pointer to start of the table, or NULL on failure
- */
-typedef const void *(*cgs_atom_get_data_table_t)(
-	struct cgs_device *cgs_device, unsigned table,
-	uint16_t *size, uint8_t *frev, uint8_t *crev);
-
-/**
- * cgs_atom_get_cmd_table_revs() - Get ATOM BIOS command table revisions
- * @cgs_device:	opaque device handle
- * @table:	data table index
- * @frev:	table format revision (output, may be NULL)
- * @crev:	table content revision (output, may be NULL)
- *
- * Return: 0 on success, -errno otherwise
- */
-typedef int (*cgs_atom_get_cmd_table_revs_t)(struct cgs_device *cgs_device, unsigned table,
-					     uint8_t *frev, uint8_t *crev);
-
-/**
- * cgs_atom_exec_cmd_table() - Execute an ATOM BIOS command table
- * @cgs_device: opaque device handle
- * @table:	command table index
- * @args:	arguments
- *
- * Return: 0 on success, -errno otherwise
- */
-typedef int (*cgs_atom_exec_cmd_table_t)(struct cgs_device *cgs_device,
-					 unsigned table, void *args);
-
-/**
- * cgs_get_firmware_info - Get the firmware information from core driver
- * @cgs_device: opaque device handle
- * @type: the firmware type
- * @info: returend firmware information
- *
- * Return: 0 on success, -errno otherwise
- */
 typedef int (*cgs_get_firmware_info)(struct cgs_device *cgs_device,
 				     enum cgs_ucode_id type,
 				     struct cgs_firmware_info *info);
 
-typedef int (*cgs_rel_firmware)(struct cgs_device *cgs_device,
-					 enum cgs_ucode_id type);
-
-typedef int(*cgs_set_powergating_state)(struct cgs_device *cgs_device,
-				  enum amd_ip_block_type block_type,
-				  enum amd_powergating_state state);
-
-typedef int(*cgs_set_clockgating_state)(struct cgs_device *cgs_device,
-				  enum amd_ip_block_type block_type,
-				  enum amd_clockgating_state state);
-
-typedef int(*cgs_get_active_displays_info)(
-					struct cgs_device *cgs_device,
-					struct cgs_display_info *info);
-
-typedef int (*cgs_notify_dpm_enabled)(struct cgs_device *cgs_device, bool enabled);
-
-typedef int (*cgs_is_virtualization_enabled_t)(void *cgs_device);
-
-typedef int (*cgs_enter_safe_mode)(struct cgs_device *cgs_device, bool en);
-
-typedef void (*cgs_lock_grbm_idx)(struct cgs_device *cgs_device, bool lock);
-
 struct cgs_ops {
 	/* MMIO access */
 	cgs_read_register_t read_register;
 	cgs_write_register_t write_register;
 	cgs_read_ind_register_t read_ind_register;
 	cgs_write_ind_register_t write_ind_register;
-	/* PCI resources */
-	cgs_get_pci_resource_t get_pci_resource;
-	/* ATOM BIOS */
-	cgs_atom_get_data_table_t atom_get_data_table;
-	cgs_atom_get_cmd_table_revs_t atom_get_cmd_table_revs;
-	cgs_atom_exec_cmd_table_t atom_exec_cmd_table;
 	/* Firmware Info */
 	cgs_get_firmware_info get_firmware_info;
-	cgs_rel_firmware rel_firmware;
-	/* cg pg interface*/
-	cgs_set_powergating_state set_powergating_state;
-	cgs_set_clockgating_state set_clockgating_state;
-	/* display manager */
-	cgs_get_active_displays_info get_active_displays_info;
-	/* notify dpm enabled */
-	cgs_notify_dpm_enabled notify_dpm_enabled;
-	cgs_is_virtualization_enabled_t is_virtualization_enabled;
-	cgs_enter_safe_mode enter_safe_mode;
-	cgs_lock_grbm_idx lock_grbm_idx;
 };
 
 struct cgs_os_ops; /* To be define in OS-specific CGS header */
@@ -309,40 +172,7 @@ struct cgs_device
 #define cgs_write_ind_register(dev,space,index,value)		\
 	CGS_CALL(write_ind_register,dev,space,index,value)
 
-#define cgs_atom_get_data_table(dev,table,size,frev,crev)	\
-	CGS_CALL(atom_get_data_table,dev,table,size,frev,crev)
-#define cgs_atom_get_cmd_table_revs(dev,table,frev,crev)	\
-	CGS_CALL(atom_get_cmd_table_revs,dev,table,frev,crev)
-#define cgs_atom_exec_cmd_table(dev,table,args)		\
-	CGS_CALL(atom_exec_cmd_table,dev,table,args)
-
 #define cgs_get_firmware_info(dev, type, info)	\
 	CGS_CALL(get_firmware_info, dev, type, info)
-#define cgs_rel_firmware(dev, type)	\
-	CGS_CALL(rel_firmware, dev, type)
-#define cgs_set_powergating_state(dev, block_type, state)	\
-	CGS_CALL(set_powergating_state, dev, block_type, state)
-#define cgs_set_clockgating_state(dev, block_type, state)	\
-	CGS_CALL(set_clockgating_state, dev, block_type, state)
-#define cgs_notify_dpm_enabled(dev, enabled)	\
-	CGS_CALL(notify_dpm_enabled, dev, enabled)
-
-#define cgs_get_active_displays_info(dev, info)	\
-	CGS_CALL(get_active_displays_info, dev, info)
-
-#define cgs_get_pci_resource(cgs_device, resource_type, size, offset, \
-	resource_base) \
-	CGS_CALL(get_pci_resource, cgs_device, resource_type, size, offset, \
-	resource_base)
-
-#define cgs_is_virtualization_enabled(cgs_device) \
-		CGS_CALL(is_virtualization_enabled, cgs_device)
-
-#define cgs_enter_safe_mode(cgs_device, en) \
-		CGS_CALL(enter_safe_mode, cgs_device, en)
-
-#define cgs_lock_grbm_idx(cgs_device, lock) \
-		CGS_CALL(lock_grbm_idx, cgs_device, lock)
-
 
 #endif /* _CGS_COMMON_H */
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 237289a72bb7..5733fbee07f7 100644
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -100,6 +100,21 @@ struct kgd2kfd_shared_resources {
 	/* Bit n == 1 means Queue n is available for KFD */
 	DECLARE_BITMAP(queue_bitmap, KGD_MAX_QUEUES);
 
+	/* Doorbell assignments (SOC15 and later chips only). Only
+	 * specific doorbells are routed to each SDMA engine. Others
+	 * are routed to IH and VCN. They are not usable by the CP.
+	 *
+	 * Any doorbell number D that satisfies the following condition
+	 * is reserved: (D & reserved_doorbell_mask) == reserved_doorbell_val
+	 *
+	 * KFD currently uses 1024 (= 0x3ff) doorbells per process. If
+	 * doorbells 0x0f0-0x0f7 and 0x2f-0x2f7 are reserved, that means
+	 * mask would be set to 0x1f8 and val set to 0x0f0.
+	 */
+	unsigned int sdma_doorbell[2][2];
+	unsigned int reserved_doorbell_mask;
+	unsigned int reserved_doorbell_val;
+
 	/* Base address of doorbell aperture. */
 	phys_addr_t doorbell_physical_address;
 
@@ -173,8 +188,6 @@ struct tile_config {
  * @set_pasid_vmid_mapping: Exposes pasid/vmid pair to the H/W for no cp
  * scheduling mode. Only used for no cp scheduling mode.
  *
- * @init_pipeline: Initialized the compute pipelines.
- *
  * @hqd_load: Loads the mqd structure to a H/W hqd slot. used only for no cp
  * sceduling mode.
  *
@@ -274,9 +287,6 @@ struct kfd2kgd_calls {
 	int (*set_pasid_vmid_mapping)(struct kgd_dev *kgd, unsigned int pasid,
 					unsigned int vmid);
 
-	int (*init_pipeline)(struct kgd_dev *kgd, uint32_t pipe_id,
-				uint32_t hpd_size, uint64_t hpd_gpu_addr);
-
 	int (*init_interrupts)(struct kgd_dev *kgd, uint32_t pipe_id);
 
 	int (*hqd_load)(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
@@ -382,6 +392,10 @@ struct kfd2kgd_calls {
  *
  * @resume: Notifies amdkfd about a resume action done to a kgd device
  *
+ * @quiesce_mm: Quiesce all user queue access to specified MM address space
+ *
+ * @resume_mm: Resume user queue access to specified MM address space
+ *
  * @schedule_evict_and_restore_process: Schedules work queue that will prepare
  * for safe eviction of KFD BOs that belong to the specified process.
  *
@@ -399,6 +413,8 @@ struct kgd2kfd_calls {
 	void (*interrupt)(struct kfd_dev *kfd, const void *ih_ring_entry);
 	void (*suspend)(struct kfd_dev *kfd);
 	int (*resume)(struct kfd_dev *kfd);
+	int (*quiesce_mm)(struct mm_struct *mm);
+	int (*resume_mm)(struct mm_struct *mm);
 	int (*schedule_evict_and_restore_process)(struct mm_struct *mm,
 			struct dma_fence *fence);
 };
diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
index 5c840c022b52..06f08f34a110 100644
--- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
@@ -94,6 +94,7 @@ enum pp_clock_type {
 	PP_PCIE,
 	OD_SCLK,
 	OD_MCLK,
+	OD_RANGE,
 };
 
 enum amd_pp_sensors {
@@ -149,13 +150,6 @@ struct pp_states_info {
 	uint32_t states[16];
 };
 
-struct pp_gpu_power {
-	uint32_t vddc_power;
-	uint32_t vddci_power;
-	uint32_t max_gpu_power;
-	uint32_t average_gpu_power;
-};
-
 #define PP_GROUP_MASK        0xF0000000
 #define PP_GROUP_SHIFT       28
 
@@ -246,11 +240,6 @@ struct amd_pm_funcs {
 	int (*load_firmware)(void *handle);
 	int (*wait_for_fw_loading_complete)(void *handle);
 	int (*set_clockgating_by_smu)(void *handle, uint32_t msg_id);
-	int (*notify_smu_memory_info)(void *handle, uint32_t virtual_addr_low,
-					uint32_t virtual_addr_hi,
-					uint32_t mc_addr_low,
-					uint32_t mc_addr_hi,
-					uint32_t size);
 	int (*set_power_limit)(void *handle, uint32_t n);
 	int (*get_power_limit)(void *handle, uint32_t *limit, bool default_limit);
 /* export to DC */
diff --git a/drivers/gpu/drm/amd/include/soc15_ih_clientid.h b/drivers/gpu/drm/amd/include/soc15_ih_clientid.h
index a12d4f27cfa4..12e196c15bbe 100644
--- a/drivers/gpu/drm/amd/include/soc15_ih_clientid.h
+++ b/drivers/gpu/drm/amd/include/soc15_ih_clientid.h
@@ -43,6 +43,7 @@ enum soc15_ih_clientid {
 	SOC15_IH_CLIENTID_SE2SH		= 0x0c,
 	SOC15_IH_CLIENTID_SE3SH		= 0x0d,
 	SOC15_IH_CLIENTID_SYSHUB	= 0x0e,
+	SOC15_IH_CLIENTID_UVD1		= 0x0e,
 	SOC15_IH_CLIENTID_THM		= 0x0f,
 	SOC15_IH_CLIENTID_UVD		= 0x10,
 	SOC15_IH_CLIENTID_VCE0		= 0x11,
diff --git a/drivers/gpu/drm/amd/include/v9_structs.h b/drivers/gpu/drm/amd/include/v9_structs.h
index 2fb25abaf7c8..ceaf4932258d 100644
--- a/drivers/gpu/drm/amd/include/v9_structs.h
+++ b/drivers/gpu/drm/amd/include/v9_structs.h
@@ -29,10 +29,10 @@ struct v9_sdma_mqd {
 	uint32_t sdmax_rlcx_rb_base;
 	uint32_t sdmax_rlcx_rb_base_hi;
 	uint32_t sdmax_rlcx_rb_rptr;
+	uint32_t sdmax_rlcx_rb_rptr_hi;
 	uint32_t sdmax_rlcx_rb_wptr;
+	uint32_t sdmax_rlcx_rb_wptr_hi;
 	uint32_t sdmax_rlcx_rb_wptr_poll_cntl;
-	uint32_t sdmax_rlcx_rb_wptr_poll_addr_hi;
-	uint32_t sdmax_rlcx_rb_wptr_poll_addr_lo;
 	uint32_t sdmax_rlcx_rb_rptr_addr_hi;
 	uint32_t sdmax_rlcx_rb_rptr_addr_lo;
 	uint32_t sdmax_rlcx_ib_cntl;
@@ -44,29 +44,29 @@ struct v9_sdma_mqd {
 	uint32_t sdmax_rlcx_skip_cntl;
 	uint32_t sdmax_rlcx_context_status;
 	uint32_t sdmax_rlcx_doorbell;
-	uint32_t sdmax_rlcx_virtual_addr;
-	uint32_t sdmax_rlcx_ape1_cntl;
+	uint32_t sdmax_rlcx_status;
 	uint32_t sdmax_rlcx_doorbell_log;
-	uint32_t reserved_22;
-	uint32_t reserved_23;
-	uint32_t reserved_24;
-	uint32_t reserved_25;
-	uint32_t reserved_26;
-	uint32_t reserved_27;
-	uint32_t reserved_28;
-	uint32_t reserved_29;
-	uint32_t reserved_30;
-	uint32_t reserved_31;
-	uint32_t reserved_32;
-	uint32_t reserved_33;
-	uint32_t reserved_34;
-	uint32_t reserved_35;
-	uint32_t reserved_36;
-	uint32_t reserved_37;
-	uint32_t reserved_38;
-	uint32_t reserved_39;
-	uint32_t reserved_40;
-	uint32_t reserved_41;
+	uint32_t sdmax_rlcx_watermark;
+	uint32_t sdmax_rlcx_doorbell_offset;
+	uint32_t sdmax_rlcx_csa_addr_lo;
+	uint32_t sdmax_rlcx_csa_addr_hi;
+	uint32_t sdmax_rlcx_ib_sub_remain;
+	uint32_t sdmax_rlcx_preempt;
+	uint32_t sdmax_rlcx_dummy_reg;
+	uint32_t sdmax_rlcx_rb_wptr_poll_addr_hi;
+	uint32_t sdmax_rlcx_rb_wptr_poll_addr_lo;
+	uint32_t sdmax_rlcx_rb_aql_cntl;
+	uint32_t sdmax_rlcx_minor_ptr_update;
+	uint32_t sdmax_rlcx_midcmd_data0;
+	uint32_t sdmax_rlcx_midcmd_data1;
+	uint32_t sdmax_rlcx_midcmd_data2;
+	uint32_t sdmax_rlcx_midcmd_data3;
+	uint32_t sdmax_rlcx_midcmd_data4;
+	uint32_t sdmax_rlcx_midcmd_data5;
+	uint32_t sdmax_rlcx_midcmd_data6;
+	uint32_t sdmax_rlcx_midcmd_data7;
+	uint32_t sdmax_rlcx_midcmd_data8;
+	uint32_t sdmax_rlcx_midcmd_cntl;
 	uint32_t reserved_42;
 	uint32_t reserved_43;
 	uint32_t reserved_44;
diff --git a/drivers/gpu/drm/amd/include/vega20_ip_offset.h b/drivers/gpu/drm/amd/include/vega20_ip_offset.h
new file mode 100644
index 000000000000..2a2a9cc8bedb
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/vega20_ip_offset.h
@@ -0,0 +1,1051 @@
+/*
+ * Copyright (C) 2018  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _vega20_ip_offset_HEADER
+#define _vega20_ip_offset_HEADER
+
+#define MAX_INSTANCE                                       6
+#define MAX_SEGMENT                                        6
+
+
+struct IP_BASE_INSTANCE
+{
+    unsigned int segment[MAX_SEGMENT];
+};
+
+struct IP_BASE
+{
+    struct IP_BASE_INSTANCE instance[MAX_INSTANCE];
+};
+
+
+static const struct IP_BASE ATHUB_BASE            ={ { { { 0x00000C20, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE CLK_BASE            ={ { { { 0x00016C00, 0x00016E00, 0x00017000, 0x00017200, 0x0001B000, 0x0001B200 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE DCE_BASE            ={ { { { 0x00000012, 0x000000C0, 0x000034C0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE DF_BASE            ={ { { { 0x00007000, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE FUSE_BASE            ={ { { { 0x00017400, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE GC_BASE            ={ { { { 0x00002000, 0x0000A000, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE HDP_BASE            ={ { { { 0x00000F20, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE MMHUB_BASE            ={ { { { 0x0001A000, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE MP0_BASE            ={ { { { 0x00016000, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE MP1_BASE            ={ { { { 0x00016000, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE NBIO_BASE            ={ { { { 0x00000000, 0x00000014, 0x00000D20, 0x00010400, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE OSSSYS_BASE            ={ { { { 0x000010A0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE SDMA0_BASE            ={ { { { 0x00001260, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE SDMA1_BASE            ={ { { { 0x00001860, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE SMUIO_BASE            ={ { { { 0x00016800, 0x00016A00, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE THM_BASE            ={ { { { 0x00016600, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE UMC_BASE            ={ { { { 0x00014000, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE UVD_BASE            ={ { { { 0x00007800, 0x00007E00, 0, 0, 0, 0 } },
+                                        { { 0, 0x00009000, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+/* Adjust VCE_BASE to make vce_4_1 use vce_4_0 offset header files*/
+static const struct IP_BASE VCE_BASE            ={ { { { 0x00007E00/* 0x00008800 */, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE XDMA_BASE            ={ { { { 0x00003400, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+static const struct IP_BASE RSMU_BASE            ={ { { { 0x00012000, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } },
+                                        { { 0, 0, 0, 0, 0, 0 } } } };
+
+
+#define ATHUB_BASE__INST0_SEG0                     0x00000C20
+#define ATHUB_BASE__INST0_SEG1                     0
+#define ATHUB_BASE__INST0_SEG2                     0
+#define ATHUB_BASE__INST0_SEG3                     0
+#define ATHUB_BASE__INST0_SEG4                     0
+#define ATHUB_BASE__INST0_SEG5                     0
+
+#define ATHUB_BASE__INST1_SEG0                     0
+#define ATHUB_BASE__INST1_SEG1                     0
+#define ATHUB_BASE__INST1_SEG2                     0
+#define ATHUB_BASE__INST1_SEG3                     0
+#define ATHUB_BASE__INST1_SEG4                     0
+#define ATHUB_BASE__INST1_SEG5                     0
+
+#define ATHUB_BASE__INST2_SEG0                     0
+#define ATHUB_BASE__INST2_SEG1                     0
+#define ATHUB_BASE__INST2_SEG2                     0
+#define ATHUB_BASE__INST2_SEG3                     0
+#define ATHUB_BASE__INST2_SEG4                     0
+#define ATHUB_BASE__INST2_SEG5                     0
+
+#define ATHUB_BASE__INST3_SEG0                     0
+#define ATHUB_BASE__INST3_SEG1                     0
+#define ATHUB_BASE__INST3_SEG2                     0
+#define ATHUB_BASE__INST3_SEG3                     0
+#define ATHUB_BASE__INST3_SEG4                     0
+#define ATHUB_BASE__INST3_SEG5                     0
+
+#define ATHUB_BASE__INST4_SEG0                     0
+#define ATHUB_BASE__INST4_SEG1                     0
+#define ATHUB_BASE__INST4_SEG2                     0
+#define ATHUB_BASE__INST4_SEG3                     0
+#define ATHUB_BASE__INST4_SEG4                     0
+#define ATHUB_BASE__INST4_SEG5                     0
+
+#define ATHUB_BASE__INST5_SEG0                     0
+#define ATHUB_BASE__INST5_SEG1                     0
+#define ATHUB_BASE__INST5_SEG2                     0
+#define ATHUB_BASE__INST5_SEG3                     0
+#define ATHUB_BASE__INST5_SEG4                     0
+#define ATHUB_BASE__INST5_SEG5                     0
+
+#define CLK_BASE__INST0_SEG0                       0x00016C00
+#define CLK_BASE__INST0_SEG1                       0x00016E00
+#define CLK_BASE__INST0_SEG2                       0x00017000
+#define CLK_BASE__INST0_SEG3                       0x00017200
+#define CLK_BASE__INST0_SEG4                       0x0001B000
+#define CLK_BASE__INST0_SEG5                       0x0001B200
+
+#define CLK_BASE__INST1_SEG0                       0
+#define CLK_BASE__INST1_SEG1                       0
+#define CLK_BASE__INST1_SEG2                       0
+#define CLK_BASE__INST1_SEG3                       0
+#define CLK_BASE__INST1_SEG4                       0
+#define CLK_BASE__INST1_SEG5                       0
+
+#define CLK_BASE__INST2_SEG0                       0
+#define CLK_BASE__INST2_SEG1                       0
+#define CLK_BASE__INST2_SEG2                       0
+#define CLK_BASE__INST2_SEG3                       0
+#define CLK_BASE__INST2_SEG4                       0
+#define CLK_BASE__INST2_SEG5                       0
+
+#define CLK_BASE__INST3_SEG0                       0
+#define CLK_BASE__INST3_SEG1                       0
+#define CLK_BASE__INST3_SEG2                       0
+#define CLK_BASE__INST3_SEG3                       0
+#define CLK_BASE__INST3_SEG4                       0
+#define CLK_BASE__INST3_SEG5                       0
+
+#define CLK_BASE__INST4_SEG0                       0
+#define CLK_BASE__INST4_SEG1                       0
+#define CLK_BASE__INST4_SEG2                       0
+#define CLK_BASE__INST4_SEG3                       0
+#define CLK_BASE__INST4_SEG4                       0
+#define CLK_BASE__INST4_SEG5                       0
+
+#define CLK_BASE__INST5_SEG0                       0
+#define CLK_BASE__INST5_SEG1                       0
+#define CLK_BASE__INST5_SEG2                       0
+#define CLK_BASE__INST5_SEG3                       0
+#define CLK_BASE__INST5_SEG4                       0
+#define CLK_BASE__INST5_SEG5                       0
+
+#define DCE_BASE__INST0_SEG0                       0x00000012
+#define DCE_BASE__INST0_SEG1                       0x000000C0
+#define DCE_BASE__INST0_SEG2                       0x000034C0
+#define DCE_BASE__INST0_SEG3                       0
+#define DCE_BASE__INST0_SEG4                       0
+#define DCE_BASE__INST0_SEG5                       0
+
+#define DCE_BASE__INST1_SEG0                       0
+#define DCE_BASE__INST1_SEG1                       0
+#define DCE_BASE__INST1_SEG2                       0
+#define DCE_BASE__INST1_SEG3                       0
+#define DCE_BASE__INST1_SEG4                       0
+#define DCE_BASE__INST1_SEG5                       0
+
+#define DCE_BASE__INST2_SEG0                       0
+#define DCE_BASE__INST2_SEG1                       0
+#define DCE_BASE__INST2_SEG2                       0
+#define DCE_BASE__INST2_SEG3                       0
+#define DCE_BASE__INST2_SEG4                       0
+#define DCE_BASE__INST2_SEG5                       0
+
+#define DCE_BASE__INST3_SEG0                       0
+#define DCE_BASE__INST3_SEG1                       0
+#define DCE_BASE__INST3_SEG2                       0
+#define DCE_BASE__INST3_SEG3                       0
+#define DCE_BASE__INST3_SEG4                       0
+#define DCE_BASE__INST3_SEG5                       0
+
+#define DCE_BASE__INST4_SEG0                       0
+#define DCE_BASE__INST4_SEG1                       0
+#define DCE_BASE__INST4_SEG2                       0
+#define DCE_BASE__INST4_SEG3                       0
+#define DCE_BASE__INST4_SEG4                       0
+#define DCE_BASE__INST4_SEG5                       0
+
+#define DCE_BASE__INST5_SEG0                       0
+#define DCE_BASE__INST5_SEG1                       0
+#define DCE_BASE__INST5_SEG2                       0
+#define DCE_BASE__INST5_SEG3                       0
+#define DCE_BASE__INST5_SEG4                       0
+#define DCE_BASE__INST5_SEG5                       0
+
+#define DF_BASE__INST0_SEG0                        0x00007000
+#define DF_BASE__INST0_SEG1                        0
+#define DF_BASE__INST0_SEG2                        0
+#define DF_BASE__INST0_SEG3                        0
+#define DF_BASE__INST0_SEG4                        0
+#define DF_BASE__INST0_SEG5                        0
+
+#define DF_BASE__INST1_SEG0                        0
+#define DF_BASE__INST1_SEG1                        0
+#define DF_BASE__INST1_SEG2                        0
+#define DF_BASE__INST1_SEG3                        0
+#define DF_BASE__INST1_SEG4                        0
+#define DF_BASE__INST1_SEG5                        0
+
+#define DF_BASE__INST2_SEG0                        0
+#define DF_BASE__INST2_SEG1                        0
+#define DF_BASE__INST2_SEG2                        0
+#define DF_BASE__INST2_SEG3                        0
+#define DF_BASE__INST2_SEG4                        0
+#define DF_BASE__INST2_SEG5                        0
+
+#define DF_BASE__INST3_SEG0                        0
+#define DF_BASE__INST3_SEG1                        0
+#define DF_BASE__INST3_SEG2                        0
+#define DF_BASE__INST3_SEG3                        0
+#define DF_BASE__INST3_SEG4                        0
+#define DF_BASE__INST3_SEG5                        0
+
+#define DF_BASE__INST4_SEG0                        0
+#define DF_BASE__INST4_SEG1                        0
+#define DF_BASE__INST4_SEG2                        0
+#define DF_BASE__INST4_SEG3                        0
+#define DF_BASE__INST4_SEG4                        0
+#define DF_BASE__INST4_SEG5                        0
+
+#define DF_BASE__INST5_SEG0                        0
+#define DF_BASE__INST5_SEG1                        0
+#define DF_BASE__INST5_SEG2                        0
+#define DF_BASE__INST5_SEG3                        0
+#define DF_BASE__INST5_SEG4                        0
+#define DF_BASE__INST5_SEG5                        0
+
+#define FUSE_BASE__INST0_SEG0                      0x00017400
+#define FUSE_BASE__INST0_SEG1                      0
+#define FUSE_BASE__INST0_SEG2                      0
+#define FUSE_BASE__INST0_SEG3                      0
+#define FUSE_BASE__INST0_SEG4                      0
+#define FUSE_BASE__INST0_SEG5                      0
+
+#define FUSE_BASE__INST1_SEG0                      0
+#define FUSE_BASE__INST1_SEG1                      0
+#define FUSE_BASE__INST1_SEG2                      0
+#define FUSE_BASE__INST1_SEG3                      0
+#define FUSE_BASE__INST1_SEG4                      0
+#define FUSE_BASE__INST1_SEG5                      0
+
+#define FUSE_BASE__INST2_SEG0                      0
+#define FUSE_BASE__INST2_SEG1                      0
+#define FUSE_BASE__INST2_SEG2                      0
+#define FUSE_BASE__INST2_SEG3                      0
+#define FUSE_BASE__INST2_SEG4                      0
+#define FUSE_BASE__INST2_SEG5                      0
+
+#define FUSE_BASE__INST3_SEG0                      0
+#define FUSE_BASE__INST3_SEG1                      0
+#define FUSE_BASE__INST3_SEG2                      0
+#define FUSE_BASE__INST3_SEG3                      0
+#define FUSE_BASE__INST3_SEG4                      0
+#define FUSE_BASE__INST3_SEG5                      0
+
+#define FUSE_BASE__INST4_SEG0                      0
+#define FUSE_BASE__INST4_SEG1                      0
+#define FUSE_BASE__INST4_SEG2                      0
+#define FUSE_BASE__INST4_SEG3                      0
+#define FUSE_BASE__INST4_SEG4                      0
+#define FUSE_BASE__INST4_SEG5                      0
+
+#define FUSE_BASE__INST5_SEG0                      0
+#define FUSE_BASE__INST5_SEG1                      0
+#define FUSE_BASE__INST5_SEG2                      0
+#define FUSE_BASE__INST5_SEG3                      0
+#define FUSE_BASE__INST5_SEG4                      0
+#define FUSE_BASE__INST5_SEG5                      0
+
+#define GC_BASE__INST0_SEG0                        0x00002000
+#define GC_BASE__INST0_SEG1                        0x0000A000
+#define GC_BASE__INST0_SEG2                        0
+#define GC_BASE__INST0_SEG3                        0
+#define GC_BASE__INST0_SEG4                        0
+#define GC_BASE__INST0_SEG5                        0
+
+#define GC_BASE__INST1_SEG0                        0
+#define GC_BASE__INST1_SEG1                        0
+#define GC_BASE__INST1_SEG2                        0
+#define GC_BASE__INST1_SEG3                        0
+#define GC_BASE__INST1_SEG4                        0
+#define GC_BASE__INST1_SEG5                        0
+
+#define GC_BASE__INST2_SEG0                        0
+#define GC_BASE__INST2_SEG1                        0
+#define GC_BASE__INST2_SEG2                        0
+#define GC_BASE__INST2_SEG3                        0
+#define GC_BASE__INST2_SEG4                        0
+#define GC_BASE__INST2_SEG5                        0
+
+#define GC_BASE__INST3_SEG0                        0
+#define GC_BASE__INST3_SEG1                        0
+#define GC_BASE__INST3_SEG2                        0
+#define GC_BASE__INST3_SEG3                        0
+#define GC_BASE__INST3_SEG4                        0
+#define GC_BASE__INST3_SEG5                        0
+
+#define GC_BASE__INST4_SEG0                        0
+#define GC_BASE__INST4_SEG1                        0
+#define GC_BASE__INST4_SEG2                        0
+#define GC_BASE__INST4_SEG3                        0
+#define GC_BASE__INST4_SEG4                        0
+#define GC_BASE__INST4_SEG5                        0
+
+#define GC_BASE__INST5_SEG0                        0
+#define GC_BASE__INST5_SEG1                        0
+#define GC_BASE__INST5_SEG2                        0
+#define GC_BASE__INST5_SEG3                        0
+#define GC_BASE__INST5_SEG4                        0
+#define GC_BASE__INST5_SEG5                        0
+
+#define HDP_BASE__INST0_SEG0                       0x00000F20
+#define HDP_BASE__INST0_SEG1                       0
+#define HDP_BASE__INST0_SEG2                       0
+#define HDP_BASE__INST0_SEG3                       0
+#define HDP_BASE__INST0_SEG4                       0
+#define HDP_BASE__INST0_SEG5                       0
+
+#define HDP_BASE__INST1_SEG0                       0
+#define HDP_BASE__INST1_SEG1                       0
+#define HDP_BASE__INST1_SEG2                       0
+#define HDP_BASE__INST1_SEG3                       0
+#define HDP_BASE__INST1_SEG4                       0
+#define HDP_BASE__INST1_SEG5                       0
+
+#define HDP_BASE__INST2_SEG0                       0
+#define HDP_BASE__INST2_SEG1                       0
+#define HDP_BASE__INST2_SEG2                       0
+#define HDP_BASE__INST2_SEG3                       0
+#define HDP_BASE__INST2_SEG4                       0
+#define HDP_BASE__INST2_SEG5                       0
+
+#define HDP_BASE__INST3_SEG0                       0
+#define HDP_BASE__INST3_SEG1                       0
+#define HDP_BASE__INST3_SEG2                       0
+#define HDP_BASE__INST3_SEG3                       0
+#define HDP_BASE__INST3_SEG4                       0
+#define HDP_BASE__INST3_SEG5                       0
+
+#define HDP_BASE__INST4_SEG0                       0
+#define HDP_BASE__INST4_SEG1                       0
+#define HDP_BASE__INST4_SEG2                       0
+#define HDP_BASE__INST4_SEG3                       0
+#define HDP_BASE__INST4_SEG4                       0
+#define HDP_BASE__INST4_SEG5                       0
+
+#define HDP_BASE__INST5_SEG0                       0
+#define HDP_BASE__INST5_SEG1                       0
+#define HDP_BASE__INST5_SEG2                       0
+#define HDP_BASE__INST5_SEG3                       0
+#define HDP_BASE__INST5_SEG4                       0
+#define HDP_BASE__INST5_SEG5                       0
+
+#define MMHUB_BASE__INST0_SEG0                     0x0001A000
+#define MMHUB_BASE__INST0_SEG1                     0
+#define MMHUB_BASE__INST0_SEG2                     0
+#define MMHUB_BASE__INST0_SEG3                     0
+#define MMHUB_BASE__INST0_SEG4                     0
+#define MMHUB_BASE__INST0_SEG5                     0
+
+#define MMHUB_BASE__INST1_SEG0                     0
+#define MMHUB_BASE__INST1_SEG1                     0
+#define MMHUB_BASE__INST1_SEG2                     0
+#define MMHUB_BASE__INST1_SEG3                     0
+#define MMHUB_BASE__INST1_SEG4                     0
+#define MMHUB_BASE__INST1_SEG5                     0
+
+#define MMHUB_BASE__INST2_SEG0                     0
+#define MMHUB_BASE__INST2_SEG1                     0
+#define MMHUB_BASE__INST2_SEG2                     0
+#define MMHUB_BASE__INST2_SEG3                     0
+#define MMHUB_BASE__INST2_SEG4                     0
+#define MMHUB_BASE__INST2_SEG5                     0
+
+#define MMHUB_BASE__INST3_SEG0                     0
+#define MMHUB_BASE__INST3_SEG1                     0
+#define MMHUB_BASE__INST3_SEG2                     0
+#define MMHUB_BASE__INST3_SEG3                     0
+#define MMHUB_BASE__INST3_SEG4                     0
+#define MMHUB_BASE__INST3_SEG5                     0
+
+#define MMHUB_BASE__INST4_SEG0                     0
+#define MMHUB_BASE__INST4_SEG1                     0
+#define MMHUB_BASE__INST4_SEG2                     0
+#define MMHUB_BASE__INST4_SEG3                     0
+#define MMHUB_BASE__INST4_SEG4                     0
+#define MMHUB_BASE__INST4_SEG5                     0
+
+#define MMHUB_BASE__INST5_SEG0                     0
+#define MMHUB_BASE__INST5_SEG1                     0
+#define MMHUB_BASE__INST5_SEG2                     0
+#define MMHUB_BASE__INST5_SEG3                     0
+#define MMHUB_BASE__INST5_SEG4                     0
+#define MMHUB_BASE__INST5_SEG5                     0
+
+#define MP0_BASE__INST0_SEG0                       0x00016000
+#define MP0_BASE__INST0_SEG1                       0
+#define MP0_BASE__INST0_SEG2                       0
+#define MP0_BASE__INST0_SEG3                       0
+#define MP0_BASE__INST0_SEG4                       0
+#define MP0_BASE__INST0_SEG5                       0
+
+#define MP0_BASE__INST1_SEG0                       0
+#define MP0_BASE__INST1_SEG1                       0
+#define MP0_BASE__INST1_SEG2                       0
+#define MP0_BASE__INST1_SEG3                       0
+#define MP0_BASE__INST1_SEG4                       0
+#define MP0_BASE__INST1_SEG5                       0
+
+#define MP0_BASE__INST2_SEG0                       0
+#define MP0_BASE__INST2_SEG1                       0
+#define MP0_BASE__INST2_SEG2                       0
+#define MP0_BASE__INST2_SEG3                       0
+#define MP0_BASE__INST2_SEG4                       0
+#define MP0_BASE__INST2_SEG5                       0
+
+#define MP0_BASE__INST3_SEG0                       0
+#define MP0_BASE__INST3_SEG1                       0
+#define MP0_BASE__INST3_SEG2                       0
+#define MP0_BASE__INST3_SEG3                       0
+#define MP0_BASE__INST3_SEG4                       0
+#define MP0_BASE__INST3_SEG5                       0
+
+#define MP0_BASE__INST4_SEG0                       0
+#define MP0_BASE__INST4_SEG1                       0
+#define MP0_BASE__INST4_SEG2                       0
+#define MP0_BASE__INST4_SEG3                       0
+#define MP0_BASE__INST4_SEG4                       0
+#define MP0_BASE__INST4_SEG5                       0
+
+#define MP0_BASE__INST5_SEG0                       0
+#define MP0_BASE__INST5_SEG1                       0
+#define MP0_BASE__INST5_SEG2                       0
+#define MP0_BASE__INST5_SEG3                       0
+#define MP0_BASE__INST5_SEG4                       0
+#define MP0_BASE__INST5_SEG5                       0
+
+#define MP1_BASE__INST0_SEG0                       0x00016000
+#define MP1_BASE__INST0_SEG1                       0
+#define MP1_BASE__INST0_SEG2                       0
+#define MP1_BASE__INST0_SEG3                       0
+#define MP1_BASE__INST0_SEG4                       0
+#define MP1_BASE__INST0_SEG5                       0
+
+#define MP1_BASE__INST1_SEG0                       0
+#define MP1_BASE__INST1_SEG1                       0
+#define MP1_BASE__INST1_SEG2                       0
+#define MP1_BASE__INST1_SEG3                       0
+#define MP1_BASE__INST1_SEG4                       0
+#define MP1_BASE__INST1_SEG5                       0
+
+#define MP1_BASE__INST2_SEG0                       0
+#define MP1_BASE__INST2_SEG1                       0
+#define MP1_BASE__INST2_SEG2                       0
+#define MP1_BASE__INST2_SEG3                       0
+#define MP1_BASE__INST2_SEG4                       0
+#define MP1_BASE__INST2_SEG5                       0
+
+#define MP1_BASE__INST3_SEG0                       0
+#define MP1_BASE__INST3_SEG1                       0
+#define MP1_BASE__INST3_SEG2                       0
+#define MP1_BASE__INST3_SEG3                       0
+#define MP1_BASE__INST3_SEG4                       0
+#define MP1_BASE__INST3_SEG5                       0
+
+#define MP1_BASE__INST4_SEG0                       0
+#define MP1_BASE__INST4_SEG1                       0
+#define MP1_BASE__INST4_SEG2                       0
+#define MP1_BASE__INST4_SEG3                       0
+#define MP1_BASE__INST4_SEG4                       0
+#define MP1_BASE__INST4_SEG5                       0
+
+#define MP1_BASE__INST5_SEG0                       0
+#define MP1_BASE__INST5_SEG1                       0
+#define MP1_BASE__INST5_SEG2                       0
+#define MP1_BASE__INST5_SEG3                       0
+#define MP1_BASE__INST5_SEG4                       0
+#define MP1_BASE__INST5_SEG5                       0
+
+#define NBIO_BASE__INST0_SEG0                      0x00000000
+#define NBIO_BASE__INST0_SEG1                      0x00000014
+#define NBIO_BASE__INST0_SEG2                      0x00000D20
+#define NBIO_BASE__INST0_SEG3                      0x00010400
+#define NBIO_BASE__INST0_SEG4                      0
+#define NBIO_BASE__INST0_SEG5                      0
+
+#define NBIO_BASE__INST1_SEG0                      0
+#define NBIO_BASE__INST1_SEG1                      0
+#define NBIO_BASE__INST1_SEG2                      0
+#define NBIO_BASE__INST1_SEG3                      0
+#define NBIO_BASE__INST1_SEG4                      0
+#define NBIO_BASE__INST1_SEG5                      0
+
+#define NBIO_BASE__INST2_SEG0                      0
+#define NBIO_BASE__INST2_SEG1                      0
+#define NBIO_BASE__INST2_SEG2                      0
+#define NBIO_BASE__INST2_SEG3                      0
+#define NBIO_BASE__INST2_SEG4                      0
+#define NBIO_BASE__INST2_SEG5                      0
+
+#define NBIO_BASE__INST3_SEG0                      0
+#define NBIO_BASE__INST3_SEG1                      0
+#define NBIO_BASE__INST3_SEG2                      0
+#define NBIO_BASE__INST3_SEG3                      0
+#define NBIO_BASE__INST3_SEG4                      0
+#define NBIO_BASE__INST3_SEG5                      0
+
+#define NBIO_BASE__INST4_SEG0                      0
+#define NBIO_BASE__INST4_SEG1                      0
+#define NBIO_BASE__INST4_SEG2                      0
+#define NBIO_BASE__INST4_SEG3                      0
+#define NBIO_BASE__INST4_SEG4                      0
+#define NBIO_BASE__INST4_SEG5                      0
+
+#define NBIO_BASE__INST5_SEG0                      0
+#define NBIO_BASE__INST5_SEG1                      0
+#define NBIO_BASE__INST5_SEG2                      0
+#define NBIO_BASE__INST5_SEG3                      0
+#define NBIO_BASE__INST5_SEG4                      0
+#define NBIO_BASE__INST5_SEG5                      0
+
+#define OSSSYS_BASE__INST0_SEG0                    0x000010A0
+#define OSSSYS_BASE__INST0_SEG1                    0
+#define OSSSYS_BASE__INST0_SEG2                    0
+#define OSSSYS_BASE__INST0_SEG3                    0
+#define OSSSYS_BASE__INST0_SEG4                    0
+#define OSSSYS_BASE__INST0_SEG5                    0
+
+#define OSSSYS_BASE__INST1_SEG0                    0
+#define OSSSYS_BASE__INST1_SEG1                    0
+#define OSSSYS_BASE__INST1_SEG2                    0
+#define OSSSYS_BASE__INST1_SEG3                    0
+#define OSSSYS_BASE__INST1_SEG4                    0
+#define OSSSYS_BASE__INST1_SEG5                    0
+
+#define OSSSYS_BASE__INST2_SEG0                    0
+#define OSSSYS_BASE__INST2_SEG1                    0
+#define OSSSYS_BASE__INST2_SEG2                    0
+#define OSSSYS_BASE__INST2_SEG3                    0
+#define OSSSYS_BASE__INST2_SEG4                    0
+#define OSSSYS_BASE__INST2_SEG5                    0
+
+#define OSSSYS_BASE__INST3_SEG0                    0
+#define OSSSYS_BASE__INST3_SEG1                    0
+#define OSSSYS_BASE__INST3_SEG2                    0
+#define OSSSYS_BASE__INST3_SEG3                    0
+#define OSSSYS_BASE__INST3_SEG4                    0
+#define OSSSYS_BASE__INST3_SEG5                    0
+
+#define OSSSYS_BASE__INST4_SEG0                    0
+#define OSSSYS_BASE__INST4_SEG1                    0
+#define OSSSYS_BASE__INST4_SEG2                    0
+#define OSSSYS_BASE__INST4_SEG3                    0
+#define OSSSYS_BASE__INST4_SEG4                    0
+#define OSSSYS_BASE__INST4_SEG5                    0
+
+#define OSSSYS_BASE__INST5_SEG0                    0
+#define OSSSYS_BASE__INST5_SEG1                    0
+#define OSSSYS_BASE__INST5_SEG2                    0
+#define OSSSYS_BASE__INST5_SEG3                    0
+#define OSSSYS_BASE__INST5_SEG4                    0
+#define OSSSYS_BASE__INST5_SEG5                    0
+
+#define SDMA0_BASE__INST0_SEG0                     0x00001260
+#define SDMA0_BASE__INST0_SEG1                     0
+#define SDMA0_BASE__INST0_SEG2                     0
+#define SDMA0_BASE__INST0_SEG3                     0
+#define SDMA0_BASE__INST0_SEG4                     0
+#define SDMA0_BASE__INST0_SEG5                     0
+
+#define SDMA0_BASE__INST1_SEG0                     0
+#define SDMA0_BASE__INST1_SEG1                     0
+#define SDMA0_BASE__INST1_SEG2                     0
+#define SDMA0_BASE__INST1_SEG3                     0
+#define SDMA0_BASE__INST1_SEG4                     0
+#define SDMA0_BASE__INST1_SEG5                     0
+
+#define SDMA0_BASE__INST2_SEG0                     0
+#define SDMA0_BASE__INST2_SEG1                     0
+#define SDMA0_BASE__INST2_SEG2                     0
+#define SDMA0_BASE__INST2_SEG3                     0
+#define SDMA0_BASE__INST2_SEG4                     0
+#define SDMA0_BASE__INST2_SEG5                     0
+
+#define SDMA0_BASE__INST3_SEG0                     0
+#define SDMA0_BASE__INST3_SEG1                     0
+#define SDMA0_BASE__INST3_SEG2                     0
+#define SDMA0_BASE__INST3_SEG3                     0
+#define SDMA0_BASE__INST3_SEG4                     0
+#define SDMA0_BASE__INST3_SEG5                     0
+
+#define SDMA0_BASE__INST4_SEG0                     0
+#define SDMA0_BASE__INST4_SEG1                     0
+#define SDMA0_BASE__INST4_SEG2                     0
+#define SDMA0_BASE__INST4_SEG3                     0
+#define SDMA0_BASE__INST4_SEG4                     0
+#define SDMA0_BASE__INST4_SEG5                     0
+
+#define SDMA0_BASE__INST5_SEG0                     0
+#define SDMA0_BASE__INST5_SEG1                     0
+#define SDMA0_BASE__INST5_SEG2                     0
+#define SDMA0_BASE__INST5_SEG3                     0
+#define SDMA0_BASE__INST5_SEG4                     0
+#define SDMA0_BASE__INST5_SEG5                     0
+
+#define SDMA1_BASE__INST0_SEG0                     0x00001860
+#define SDMA1_BASE__INST0_SEG1                     0
+#define SDMA1_BASE__INST0_SEG2                     0
+#define SDMA1_BASE__INST0_SEG3                     0
+#define SDMA1_BASE__INST0_SEG4                     0
+#define SDMA1_BASE__INST0_SEG5                     0
+
+#define SDMA1_BASE__INST1_SEG0                     0
+#define SDMA1_BASE__INST1_SEG1                     0
+#define SDMA1_BASE__INST1_SEG2                     0
+#define SDMA1_BASE__INST1_SEG3                     0
+#define SDMA1_BASE__INST1_SEG4                     0
+#define SDMA1_BASE__INST1_SEG5                     0
+
+#define SDMA1_BASE__INST2_SEG0                     0
+#define SDMA1_BASE__INST2_SEG1                     0
+#define SDMA1_BASE__INST2_SEG2                     0
+#define SDMA1_BASE__INST2_SEG3                     0
+#define SDMA1_BASE__INST2_SEG4                     0
+#define SDMA1_BASE__INST2_SEG5                     0
+
+#define SDMA1_BASE__INST3_SEG0                     0
+#define SDMA1_BASE__INST3_SEG1                     0
+#define SDMA1_BASE__INST3_SEG2                     0
+#define SDMA1_BASE__INST3_SEG3                     0
+#define SDMA1_BASE__INST3_SEG4                     0
+#define SDMA1_BASE__INST3_SEG5                     0
+
+#define SDMA1_BASE__INST4_SEG0                     0
+#define SDMA1_BASE__INST4_SEG1                     0
+#define SDMA1_BASE__INST4_SEG2                     0
+#define SDMA1_BASE__INST4_SEG3                     0
+#define SDMA1_BASE__INST4_SEG4                     0
+#define SDMA1_BASE__INST4_SEG5                     0
+
+#define SDMA1_BASE__INST5_SEG0                     0
+#define SDMA1_BASE__INST5_SEG1                     0
+#define SDMA1_BASE__INST5_SEG2                     0
+#define SDMA1_BASE__INST5_SEG3                     0
+#define SDMA1_BASE__INST5_SEG4                     0
+#define SDMA1_BASE__INST5_SEG5                     0
+
+#define SMUIO_BASE__INST0_SEG0                     0x00016800
+#define SMUIO_BASE__INST0_SEG1                     0x00016A00
+#define SMUIO_BASE__INST0_SEG2                     0
+#define SMUIO_BASE__INST0_SEG3                     0
+#define SMUIO_BASE__INST0_SEG4                     0
+#define SMUIO_BASE__INST0_SEG5                     0
+
+#define SMUIO_BASE__INST1_SEG0                     0
+#define SMUIO_BASE__INST1_SEG1                     0
+#define SMUIO_BASE__INST1_SEG2                     0
+#define SMUIO_BASE__INST1_SEG3                     0
+#define SMUIO_BASE__INST1_SEG4                     0
+#define SMUIO_BASE__INST1_SEG5                     0
+
+#define SMUIO_BASE__INST2_SEG0                     0
+#define SMUIO_BASE__INST2_SEG1                     0
+#define SMUIO_BASE__INST2_SEG2                     0
+#define SMUIO_BASE__INST2_SEG3                     0
+#define SMUIO_BASE__INST2_SEG4                     0
+#define SMUIO_BASE__INST2_SEG5                     0
+
+#define SMUIO_BASE__INST3_SEG0                     0
+#define SMUIO_BASE__INST3_SEG1                     0
+#define SMUIO_BASE__INST3_SEG2                     0
+#define SMUIO_BASE__INST3_SEG3                     0
+#define SMUIO_BASE__INST3_SEG4                     0
+#define SMUIO_BASE__INST3_SEG5                     0
+
+#define SMUIO_BASE__INST4_SEG0                     0
+#define SMUIO_BASE__INST4_SEG1                     0
+#define SMUIO_BASE__INST4_SEG2                     0
+#define SMUIO_BASE__INST4_SEG3                     0
+#define SMUIO_BASE__INST4_SEG4                     0
+#define SMUIO_BASE__INST4_SEG5                     0
+
+#define SMUIO_BASE__INST5_SEG0                     0
+#define SMUIO_BASE__INST5_SEG1                     0
+#define SMUIO_BASE__INST5_SEG2                     0
+#define SMUIO_BASE__INST5_SEG3                     0
+#define SMUIO_BASE__INST5_SEG4                     0
+#define SMUIO_BASE__INST5_SEG5                     0
+
+#define THM_BASE__INST0_SEG0                       0x00016600
+#define THM_BASE__INST0_SEG1                       0
+#define THM_BASE__INST0_SEG2                       0
+#define THM_BASE__INST0_SEG3                       0
+#define THM_BASE__INST0_SEG4                       0
+#define THM_BASE__INST0_SEG5                       0
+
+#define THM_BASE__INST1_SEG0                       0
+#define THM_BASE__INST1_SEG1                       0
+#define THM_BASE__INST1_SEG2                       0
+#define THM_BASE__INST1_SEG3                       0
+#define THM_BASE__INST1_SEG4                       0
+#define THM_BASE__INST1_SEG5                       0
+
+#define THM_BASE__INST2_SEG0                       0
+#define THM_BASE__INST2_SEG1                       0
+#define THM_BASE__INST2_SEG2                       0
+#define THM_BASE__INST2_SEG3                       0
+#define THM_BASE__INST2_SEG4                       0
+#define THM_BASE__INST2_SEG5                       0
+
+#define THM_BASE__INST3_SEG0                       0
+#define THM_BASE__INST3_SEG1                       0
+#define THM_BASE__INST3_SEG2                       0
+#define THM_BASE__INST3_SEG3                       0
+#define THM_BASE__INST3_SEG4                       0
+#define THM_BASE__INST3_SEG5                       0
+
+#define THM_BASE__INST4_SEG0                       0
+#define THM_BASE__INST4_SEG1                       0
+#define THM_BASE__INST4_SEG2                       0
+#define THM_BASE__INST4_SEG3                       0
+#define THM_BASE__INST4_SEG4                       0
+#define THM_BASE__INST4_SEG5                       0
+
+#define THM_BASE__INST5_SEG0                       0
+#define THM_BASE__INST5_SEG1                       0
+#define THM_BASE__INST5_SEG2                       0
+#define THM_BASE__INST5_SEG3                       0
+#define THM_BASE__INST5_SEG4                       0
+#define THM_BASE__INST5_SEG5                       0
+
+#define UMC_BASE__INST0_SEG0                       0x00014000
+#define UMC_BASE__INST0_SEG1                       0
+#define UMC_BASE__INST0_SEG2                       0
+#define UMC_BASE__INST0_SEG3                       0
+#define UMC_BASE__INST0_SEG4                       0
+#define UMC_BASE__INST0_SEG5                       0
+
+#define UMC_BASE__INST1_SEG0                       0
+#define UMC_BASE__INST1_SEG1                       0
+#define UMC_BASE__INST1_SEG2                       0
+#define UMC_BASE__INST1_SEG3                       0
+#define UMC_BASE__INST1_SEG4                       0
+#define UMC_BASE__INST1_SEG5                       0
+
+#define UMC_BASE__INST2_SEG0                       0
+#define UMC_BASE__INST2_SEG1                       0
+#define UMC_BASE__INST2_SEG2                       0
+#define UMC_BASE__INST2_SEG3                       0
+#define UMC_BASE__INST2_SEG4                       0
+#define UMC_BASE__INST2_SEG5                       0
+
+#define UMC_BASE__INST3_SEG0                       0
+#define UMC_BASE__INST3_SEG1                       0
+#define UMC_BASE__INST3_SEG2                       0
+#define UMC_BASE__INST3_SEG3                       0
+#define UMC_BASE__INST3_SEG4                       0
+#define UMC_BASE__INST3_SEG5                       0
+
+#define UMC_BASE__INST4_SEG0                       0
+#define UMC_BASE__INST4_SEG1                       0
+#define UMC_BASE__INST4_SEG2                       0
+#define UMC_BASE__INST4_SEG3                       0
+#define UMC_BASE__INST4_SEG4                       0
+#define UMC_BASE__INST4_SEG5                       0
+
+#define UMC_BASE__INST5_SEG0                       0
+#define UMC_BASE__INST5_SEG1                       0
+#define UMC_BASE__INST5_SEG2                       0
+#define UMC_BASE__INST5_SEG3                       0
+#define UMC_BASE__INST5_SEG4                       0
+#define UMC_BASE__INST5_SEG5                       0
+
+#define UVD_BASE__INST0_SEG0                       0x00007800
+#define UVD_BASE__INST0_SEG1                       0x00007E00
+#define UVD_BASE__INST0_SEG2                       0
+#define UVD_BASE__INST0_SEG3                       0
+#define UVD_BASE__INST0_SEG4                       0
+#define UVD_BASE__INST0_SEG5                       0
+
+#define UVD_BASE__INST1_SEG0                       0
+#define UVD_BASE__INST1_SEG1                       0x00009000
+#define UVD_BASE__INST1_SEG2                       0
+#define UVD_BASE__INST1_SEG3                       0
+#define UVD_BASE__INST1_SEG4                       0
+#define UVD_BASE__INST1_SEG5                       0
+
+#define UVD_BASE__INST2_SEG0                       0
+#define UVD_BASE__INST2_SEG1                       0
+#define UVD_BASE__INST2_SEG2                       0
+#define UVD_BASE__INST2_SEG3                       0
+#define UVD_BASE__INST2_SEG4                       0
+#define UVD_BASE__INST2_SEG5                       0
+
+#define UVD_BASE__INST3_SEG0                       0
+#define UVD_BASE__INST3_SEG1                       0
+#define UVD_BASE__INST3_SEG2                       0
+#define UVD_BASE__INST3_SEG3                       0
+#define UVD_BASE__INST3_SEG4                       0
+#define UVD_BASE__INST3_SEG5                       0
+
+#define UVD_BASE__INST4_SEG0                       0
+#define UVD_BASE__INST4_SEG1                       0
+#define UVD_BASE__INST4_SEG2                       0
+#define UVD_BASE__INST4_SEG3                       0
+#define UVD_BASE__INST4_SEG4                       0
+#define UVD_BASE__INST4_SEG5                       0
+
+#define UVD_BASE__INST5_SEG0                       0
+#define UVD_BASE__INST5_SEG1                       0
+#define UVD_BASE__INST5_SEG2                       0
+#define UVD_BASE__INST5_SEG3                       0
+#define UVD_BASE__INST5_SEG4                       0
+#define UVD_BASE__INST5_SEG5                       0
+
+#define VCE_BASE__INST0_SEG0                       0x00008800
+#define VCE_BASE__INST0_SEG1                       0
+#define VCE_BASE__INST0_SEG2                       0
+#define VCE_BASE__INST0_SEG3                       0
+#define VCE_BASE__INST0_SEG4                       0
+#define VCE_BASE__INST0_SEG5                       0
+
+#define VCE_BASE__INST1_SEG0                       0
+#define VCE_BASE__INST1_SEG1                       0
+#define VCE_BASE__INST1_SEG2                       0
+#define VCE_BASE__INST1_SEG3                       0
+#define VCE_BASE__INST1_SEG4                       0
+#define VCE_BASE__INST1_SEG5                       0
+
+#define VCE_BASE__INST2_SEG0                       0
+#define VCE_BASE__INST2_SEG1                       0
+#define VCE_BASE__INST2_SEG2                       0
+#define VCE_BASE__INST2_SEG3                       0
+#define VCE_BASE__INST2_SEG4                       0
+#define VCE_BASE__INST2_SEG5                       0
+
+#define VCE_BASE__INST3_SEG0                       0
+#define VCE_BASE__INST3_SEG1                       0
+#define VCE_BASE__INST3_SEG2                       0
+#define VCE_BASE__INST3_SEG3                       0
+#define VCE_BASE__INST3_SEG4                       0
+#define VCE_BASE__INST3_SEG5                       0
+
+#define VCE_BASE__INST4_SEG0                       0
+#define VCE_BASE__INST4_SEG1                       0
+#define VCE_BASE__INST4_SEG2                       0
+#define VCE_BASE__INST4_SEG3                       0
+#define VCE_BASE__INST4_SEG4                       0
+#define VCE_BASE__INST4_SEG5                       0
+
+#define VCE_BASE__INST5_SEG0                       0
+#define VCE_BASE__INST5_SEG1                       0
+#define VCE_BASE__INST5_SEG2                       0
+#define VCE_BASE__INST5_SEG3                       0
+#define VCE_BASE__INST5_SEG4                       0
+#define VCE_BASE__INST5_SEG5                       0
+
+#define XDMA_BASE__INST0_SEG0                      0x00003400
+#define XDMA_BASE__INST0_SEG1                      0
+#define XDMA_BASE__INST0_SEG2                      0
+#define XDMA_BASE__INST0_SEG3                      0
+#define XDMA_BASE__INST0_SEG4                      0
+#define XDMA_BASE__INST0_SEG5                      0
+
+#define XDMA_BASE__INST1_SEG0                      0
+#define XDMA_BASE__INST1_SEG1                      0
+#define XDMA_BASE__INST1_SEG2                      0
+#define XDMA_BASE__INST1_SEG3                      0
+#define XDMA_BASE__INST1_SEG4                      0
+#define XDMA_BASE__INST1_SEG5                      0
+
+#define XDMA_BASE__INST2_SEG0                      0
+#define XDMA_BASE__INST2_SEG1                      0
+#define XDMA_BASE__INST2_SEG2                      0
+#define XDMA_BASE__INST2_SEG3                      0
+#define XDMA_BASE__INST2_SEG4                      0
+#define XDMA_BASE__INST2_SEG5                      0
+
+#define XDMA_BASE__INST3_SEG0                      0
+#define XDMA_BASE__INST3_SEG1                      0
+#define XDMA_BASE__INST3_SEG2                      0
+#define XDMA_BASE__INST3_SEG3                      0
+#define XDMA_BASE__INST3_SEG4                      0
+#define XDMA_BASE__INST3_SEG5                      0
+
+#define XDMA_BASE__INST4_SEG0                      0
+#define XDMA_BASE__INST4_SEG1                      0
+#define XDMA_BASE__INST4_SEG2                      0
+#define XDMA_BASE__INST4_SEG3                      0
+#define XDMA_BASE__INST4_SEG4                      0
+#define XDMA_BASE__INST4_SEG5                      0
+
+#define XDMA_BASE__INST5_SEG0                      0
+#define XDMA_BASE__INST5_SEG1                      0
+#define XDMA_BASE__INST5_SEG2                      0
+#define XDMA_BASE__INST5_SEG3                      0
+#define XDMA_BASE__INST5_SEG4                      0
+#define XDMA_BASE__INST5_SEG5                      0
+
+#define RSMU_BASE__INST0_SEG0                      0x00012000
+#define RSMU_BASE__INST0_SEG1                      0
+#define RSMU_BASE__INST0_SEG2                      0
+#define RSMU_BASE__INST0_SEG3                      0
+#define RSMU_BASE__INST0_SEG4                      0
+#define RSMU_BASE__INST0_SEG5                      0
+
+#define RSMU_BASE__INST1_SEG0                      0
+#define RSMU_BASE__INST1_SEG1                      0
+#define RSMU_BASE__INST1_SEG2                      0
+#define RSMU_BASE__INST1_SEG3                      0
+#define RSMU_BASE__INST1_SEG4                      0
+#define RSMU_BASE__INST1_SEG5                      0
+
+#define RSMU_BASE__INST2_SEG0                      0
+#define RSMU_BASE__INST2_SEG1                      0
+#define RSMU_BASE__INST2_SEG2                      0
+#define RSMU_BASE__INST2_SEG3                      0
+#define RSMU_BASE__INST2_SEG4                      0
+#define RSMU_BASE__INST2_SEG5                      0
+
+#define RSMU_BASE__INST3_SEG0                      0
+#define RSMU_BASE__INST3_SEG1                      0
+#define RSMU_BASE__INST3_SEG2                      0
+#define RSMU_BASE__INST3_SEG3                      0
+#define RSMU_BASE__INST3_SEG4                      0
+#define RSMU_BASE__INST3_SEG5                      0
+
+#define RSMU_BASE__INST4_SEG0                      0
+#define RSMU_BASE__INST4_SEG1                      0
+#define RSMU_BASE__INST4_SEG2                      0
+#define RSMU_BASE__INST4_SEG3                      0
+#define RSMU_BASE__INST4_SEG4                      0
+#define RSMU_BASE__INST4_SEG5                      0
+
+#define RSMU_BASE__INST5_SEG0                      0
+#define RSMU_BASE__INST5_SEG1                      0
+#define RSMU_BASE__INST5_SEG2                      0
+#define RSMU_BASE__INST5_SEG3                      0
+#define RSMU_BASE__INST5_SEG4                      0
+#define RSMU_BASE__INST5_SEG5                      0
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
index 7e8ad30d98e2..b493369e6d0f 100644
--- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
@@ -25,30 +25,16 @@
 #include <linux/kernel.h>
 #include <linux/gfp.h>
 #include <linux/slab.h>
+#include <linux/firmware.h>
 #include "amd_shared.h"
 #include "amd_powerplay.h"
 #include "power_state.h"
 #include "amdgpu.h"
 #include "hwmgr.h"
 
-#define PP_DPM_DISABLED 0xCCCC
-
-static int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_task task_id,
-		enum amd_pm_state_type *user_state);
 
 static const struct amd_pm_funcs pp_dpm_funcs;
 
-static inline int pp_check(struct pp_hwmgr *hwmgr)
-{
-	if (hwmgr == NULL || hwmgr->smumgr_funcs == NULL)
-		return -EINVAL;
-
-	if (hwmgr->pm_en == 0 || hwmgr->hwmgr_func == NULL)
-		return PP_DPM_DISABLED;
-
-	return 0;
-}
-
 static int amd_powerplay_create(struct amdgpu_device *adev)
 {
 	struct pp_hwmgr *hwmgr;
@@ -61,19 +47,21 @@ static int amd_powerplay_create(struct amdgpu_device *adev)
 		return -ENOMEM;
 
 	hwmgr->adev = adev;
-	hwmgr->pm_en = (amdgpu_dpm != 0 && !amdgpu_sriov_vf(adev)) ? true : false;
+	hwmgr->not_vf = !amdgpu_sriov_vf(adev);
+	hwmgr->pm_en = (amdgpu_dpm && hwmgr->not_vf) ? true : false;
 	hwmgr->device = amdgpu_cgs_create_device(adev);
 	mutex_init(&hwmgr->smu_lock);
 	hwmgr->chip_family = adev->family;
 	hwmgr->chip_id = adev->asic_type;
-	hwmgr->feature_mask = amdgpu_pp_feature_mask;
+	hwmgr->feature_mask = adev->powerplay.pp_feature;
+	hwmgr->display_config = &adev->pm.pm_display_cfg;
 	adev->powerplay.pp_handle = hwmgr;
 	adev->powerplay.pp_funcs = &pp_dpm_funcs;
 	return 0;
 }
 
 
-static int amd_powerplay_destroy(struct amdgpu_device *adev)
+static void amd_powerplay_destroy(struct amdgpu_device *adev)
 {
 	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
 
@@ -82,8 +70,6 @@ static int amd_powerplay_destroy(struct amdgpu_device *adev)
 
 	kfree(hwmgr);
 	hwmgr = NULL;
-
-	return 0;
 }
 
 static int pp_early_init(void *handle)
@@ -109,18 +95,9 @@ static int pp_sw_init(void *handle)
 	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret >= 0) {
-		if (hwmgr->smumgr_funcs->smu_init == NULL)
-			return -EINVAL;
-
-		ret = hwmgr->smumgr_funcs->smu_init(hwmgr);
+	ret = hwmgr_sw_init(hwmgr);
 
-		phm_register_irq_handlers(hwmgr);
-
-		pr_debug("amdgpu: powerplay sw initialized\n");
-	}
+	pr_debug("powerplay sw init %s\n", ret ? "failed" : "successfully");
 
 	return ret;
 }
@@ -129,16 +106,14 @@ static int pp_sw_fini(void *handle)
 {
 	struct amdgpu_device *adev = handle;
 	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
-	int ret = 0;
 
-	ret = pp_check(hwmgr);
-	if (ret >= 0) {
-		if (hwmgr->smumgr_funcs->smu_fini != NULL)
-			hwmgr->smumgr_funcs->smu_fini(hwmgr);
-	}
+	hwmgr_sw_fini(hwmgr);
 
-	if (adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_SMU) {
+		release_firmware(adev->pm.fw);
+		adev->pm.fw = NULL;
 		amdgpu_ucode_fini_bo(adev);
+	}
 
 	return 0;
 }
@@ -152,55 +127,76 @@ static int pp_hw_init(void *handle)
 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
 		amdgpu_ucode_init_bo(adev);
 
-	ret = pp_check(hwmgr);
+	ret = hwmgr_hw_init(hwmgr);
 
-	if (ret >= 0) {
-		if (hwmgr->smumgr_funcs->start_smu == NULL)
-			return -EINVAL;
+	if (ret)
+		pr_err("powerplay hw init failed\n");
 
-		if (hwmgr->smumgr_funcs->start_smu(hwmgr)) {
-			pr_err("smc start failed\n");
-			hwmgr->smumgr_funcs->smu_fini(hwmgr);
-			return -EINVAL;
-		}
-		if (ret == PP_DPM_DISABLED)
-			goto exit;
-		ret = hwmgr_hw_init(hwmgr);
-		if (ret)
-			goto exit;
-	}
 	return ret;
-exit:
-	hwmgr->pm_en = 0;
-	cgs_notify_dpm_enabled(hwmgr->device, false);
-	return 0;
-
 }
 
 static int pp_hw_fini(void *handle)
 {
 	struct amdgpu_device *adev = handle;
 	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
-	int ret = 0;
 
-	ret = pp_check(hwmgr);
-	if (ret == 0)
-		hwmgr_hw_fini(hwmgr);
+	hwmgr_hw_fini(hwmgr);
 
 	return 0;
 }
 
+static void pp_reserve_vram_for_smu(struct amdgpu_device *adev)
+{
+	int r = -EINVAL;
+	void *cpu_ptr = NULL;
+	uint64_t gpu_addr;
+	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
+
+	if (amdgpu_bo_create_kernel(adev, adev->pm.smu_prv_buffer_size,
+						PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+						&adev->pm.smu_prv_buffer,
+						&gpu_addr,
+						&cpu_ptr)) {
+		DRM_ERROR("amdgpu: failed to create smu prv buffer\n");
+		return;
+	}
+
+	if (hwmgr->hwmgr_func->notify_cac_buffer_info)
+		r = hwmgr->hwmgr_func->notify_cac_buffer_info(hwmgr,
+					lower_32_bits((unsigned long)cpu_ptr),
+					upper_32_bits((unsigned long)cpu_ptr),
+					lower_32_bits(gpu_addr),
+					upper_32_bits(gpu_addr),
+					adev->pm.smu_prv_buffer_size);
+
+	if (r) {
+		amdgpu_bo_free_kernel(&adev->pm.smu_prv_buffer, NULL, NULL);
+		adev->pm.smu_prv_buffer = NULL;
+		DRM_ERROR("amdgpu: failed to notify SMU buffer address\n");
+	}
+}
+
 static int pp_late_init(void *handle)
 {
 	struct amdgpu_device *adev = handle;
 	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
-	int ret = 0;
-
-	ret = pp_check(hwmgr);
+	int ret;
 
-	if (ret == 0)
-		pp_dpm_dispatch_tasks(hwmgr,
+	if (hwmgr && hwmgr->pm_en) {
+		mutex_lock(&hwmgr->smu_lock);
+		hwmgr_handle_task(hwmgr,
 					AMD_PP_TASK_COMPLETE_INIT, NULL);
+		mutex_unlock(&hwmgr->smu_lock);
+	}
+	if (adev->pm.smu_prv_buffer_size != 0)
+		pp_reserve_vram_for_smu(adev);
+
+	if (hwmgr->hwmgr_func->gfx_off_control &&
+	    (hwmgr->feature_mask & PP_GFXOFF_MASK)) {
+		ret = hwmgr->hwmgr_func->gfx_off_control(hwmgr, true);
+		if (ret)
+			pr_err("gfx off enabling failed!\n");
+	}
 
 	return 0;
 }
@@ -209,6 +205,8 @@ static void pp_late_fini(void *handle)
 {
 	struct amdgpu_device *adev = handle;
 
+	if (adev->pm.smu_prv_buffer)
+		amdgpu_bo_free_kernel(&adev->pm.smu_prv_buffer, NULL, NULL);
 	amd_powerplay_destroy(adev);
 }
 
@@ -233,12 +231,18 @@ static int pp_set_powergating_state(void *handle,
 {
 	struct amdgpu_device *adev = handle;
 	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
-	int ret = 0;
+	int ret;
 
-	ret = pp_check(hwmgr);
+	if (!hwmgr || !hwmgr->pm_en)
+		return 0;
 
-	if (ret)
-		return ret;
+	if (hwmgr->hwmgr_func->gfx_off_control) {
+		/* Enable/disable GFX off through SMU */
+		ret = hwmgr->hwmgr_func->gfx_off_control(hwmgr,
+							 state == AMD_PG_STATE_GATE);
+		if (ret)
+			pr_err("gfx off control failed!\n");
+	}
 
 	if (hwmgr->hwmgr_func->enable_per_cu_power_gating == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -254,38 +258,16 @@ static int pp_suspend(void *handle)
 {
 	struct amdgpu_device *adev = handle;
 	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
-	int ret = 0;
 
-	ret = pp_check(hwmgr);
-	if (ret == 0)
-		hwmgr_hw_suspend(hwmgr);
-	return 0;
+	return hwmgr_suspend(hwmgr);
 }
 
 static int pp_resume(void *handle)
 {
 	struct amdgpu_device *adev = handle;
 	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
-	int ret;
-
-	ret = pp_check(hwmgr);
-
-	if (ret < 0)
-		return ret;
-
-	if (hwmgr->smumgr_funcs->start_smu == NULL)
-		return -EINVAL;
-
-	if (hwmgr->smumgr_funcs->start_smu(hwmgr)) {
-		pr_err("smc start failed\n");
-		hwmgr->smumgr_funcs->smu_fini(hwmgr);
-		return -EINVAL;
-	}
-
-	if (ret == PP_DPM_DISABLED)
-		return 0;
 
-	return hwmgr_hw_resume(hwmgr);
+	return hwmgr_resume(hwmgr);
 }
 
 static int pp_set_clockgating_state(void *handle,
@@ -334,12 +316,9 @@ static int pp_dpm_fw_loading_complete(void *handle)
 static int pp_set_clockgating_by_smu(void *handle, uint32_t msg_id)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
-
-	ret = pp_check(hwmgr);
 
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->update_clock_gatings == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -362,10 +341,10 @@ static void pp_dpm_en_umd_pstate(struct pp_hwmgr  *hwmgr,
 		if (*level & profile_mode_mask) {
 			hwmgr->saved_dpm_level = hwmgr->dpm_level;
 			hwmgr->en_umd_pstate = true;
-			cgs_set_clockgating_state(hwmgr->device,
+			amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 						AMD_IP_BLOCK_TYPE_GFX,
 						AMD_CG_STATE_UNGATE);
-			cgs_set_powergating_state(hwmgr->device,
+			amdgpu_device_ip_set_powergating_state(hwmgr->adev,
 					AMD_IP_BLOCK_TYPE_GFX,
 					AMD_PG_STATE_UNGATE);
 		}
@@ -375,10 +354,10 @@ static void pp_dpm_en_umd_pstate(struct pp_hwmgr  *hwmgr,
 			if (*level == AMD_DPM_FORCED_LEVEL_PROFILE_EXIT)
 				*level = hwmgr->saved_dpm_level;
 			hwmgr->en_umd_pstate = false;
-			cgs_set_clockgating_state(hwmgr->device,
+			amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 					AMD_IP_BLOCK_TYPE_GFX,
 					AMD_CG_STATE_GATE);
-			cgs_set_powergating_state(hwmgr->device,
+			amdgpu_device_ip_set_powergating_state(hwmgr->adev,
 					AMD_IP_BLOCK_TYPE_GFX,
 					AMD_PG_STATE_GATE);
 		}
@@ -389,12 +368,9 @@ static int pp_dpm_force_performance_level(void *handle,
 					enum amd_dpm_forced_level level)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (level == hwmgr->dpm_level)
 		return 0;
@@ -412,13 +388,10 @@ static enum amd_dpm_forced_level pp_dpm_get_performance_level(
 								void *handle)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
 	enum amd_dpm_forced_level level;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
 	level = hwmgr->dpm_level;
@@ -429,13 +402,10 @@ static enum amd_dpm_forced_level pp_dpm_get_performance_level(
 static uint32_t pp_dpm_get_sclk(void *handle, bool low)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
 	uint32_t clk = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return 0;
 
 	if (hwmgr->hwmgr_func->get_sclk == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -450,13 +420,10 @@ static uint32_t pp_dpm_get_sclk(void *handle, bool low)
 static uint32_t pp_dpm_get_mclk(void *handle, bool low)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
 	uint32_t clk = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return 0;
 
 	if (hwmgr->hwmgr_func->get_mclk == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -471,11 +438,8 @@ static uint32_t pp_dpm_get_mclk(void *handle, bool low)
 static void pp_dpm_powergate_vce(void *handle, bool gate)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
+	if (!hwmgr || !hwmgr->pm_en)
 		return;
 
 	if (hwmgr->hwmgr_func->powergate_vce == NULL) {
@@ -490,11 +454,8 @@ static void pp_dpm_powergate_vce(void *handle, bool gate)
 static void pp_dpm_powergate_uvd(void *handle, bool gate)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
-
-	ret = pp_check(hwmgr);
 
-	if (ret)
+	if (!hwmgr || !hwmgr->pm_en)
 		return;
 
 	if (hwmgr->hwmgr_func->powergate_uvd == NULL) {
@@ -512,10 +473,8 @@ static int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_task task_id,
 	int ret = 0;
 	struct pp_hwmgr *hwmgr = handle;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
 	ret = hwmgr_handle_task(hwmgr, task_id, user_state);
@@ -528,15 +487,9 @@ static enum amd_pm_state_type pp_dpm_get_current_power_state(void *handle)
 {
 	struct pp_hwmgr *hwmgr = handle;
 	struct pp_power_state *state;
-	int ret = 0;
 	enum amd_pm_state_type pm_type;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
-
-	if (hwmgr->current_ps == NULL)
+	if (!hwmgr || !hwmgr->pm_en || !hwmgr->current_ps)
 		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -568,11 +521,8 @@ static enum amd_pm_state_type pp_dpm_get_current_power_state(void *handle)
 static void pp_dpm_set_fan_control_mode(void *handle, uint32_t mode)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
-
-	ret = pp_check(hwmgr);
 
-	if (ret)
+	if (!hwmgr || !hwmgr->pm_en)
 		return;
 
 	if (hwmgr->hwmgr_func->set_fan_control_mode == NULL) {
@@ -587,13 +537,10 @@ static void pp_dpm_set_fan_control_mode(void *handle, uint32_t mode)
 static uint32_t pp_dpm_get_fan_control_mode(void *handle)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
 	uint32_t mode = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return 0;
 
 	if (hwmgr->hwmgr_func->get_fan_control_mode == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -610,10 +557,8 @@ static int pp_dpm_set_fan_speed_percent(void *handle, uint32_t percent)
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->set_fan_speed_percent == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -630,10 +575,8 @@ static int pp_dpm_get_fan_speed_percent(void *handle, uint32_t *speed)
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->get_fan_speed_percent == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -651,10 +594,8 @@ static int pp_dpm_get_fan_speed_rpm(void *handle, uint32_t *rpm)
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->get_fan_speed_rpm == NULL)
 		return -EINVAL;
@@ -670,16 +611,10 @@ static int pp_dpm_get_pp_num_states(void *handle,
 {
 	struct pp_hwmgr *hwmgr = handle;
 	int i;
-	int ret = 0;
 
 	memset(data, 0, sizeof(*data));
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
-
-	if (hwmgr->ps == NULL)
+	if (!hwmgr || !hwmgr->pm_en ||!hwmgr->ps)
 		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -713,15 +648,9 @@ static int pp_dpm_get_pp_num_states(void *handle,
 static int pp_dpm_get_pp_table(void *handle, char **table)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
 	int size = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
-
-	if (!hwmgr->soft_pp_table)
+	if (!hwmgr || !hwmgr->pm_en ||!hwmgr->soft_pp_table)
 		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -736,10 +665,6 @@ static int amd_powerplay_reset(void *handle)
 	struct pp_hwmgr *hwmgr = handle;
 	int ret;
 
-	ret = pp_check(hwmgr);
-	if (ret)
-		return ret;
-
 	ret = hwmgr_hw_fini(hwmgr);
 	if (ret)
 		return ret;
@@ -754,40 +679,38 @@ static int amd_powerplay_reset(void *handle)
 static int pp_dpm_set_pp_table(void *handle, const char *buf, size_t size)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
+	int ret = -ENOMEM;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
 	if (!hwmgr->hardcode_pp_table) {
 		hwmgr->hardcode_pp_table = kmemdup(hwmgr->soft_pp_table,
 						   hwmgr->soft_pp_table_size,
 						   GFP_KERNEL);
-		if (!hwmgr->hardcode_pp_table) {
-			mutex_unlock(&hwmgr->smu_lock);
-			return -ENOMEM;
-		}
+		if (!hwmgr->hardcode_pp_table)
+			goto err;
 	}
 
 	memcpy(hwmgr->hardcode_pp_table, buf, size);
 
 	hwmgr->soft_pp_table = hwmgr->hardcode_pp_table;
-	mutex_unlock(&hwmgr->smu_lock);
 
 	ret = amd_powerplay_reset(handle);
 	if (ret)
-		return ret;
+		goto err;
 
 	if (hwmgr->hwmgr_func->avfs_control) {
 		ret = hwmgr->hwmgr_func->avfs_control(hwmgr, false);
 		if (ret)
-			return ret;
+			goto err;
 	}
-
+	mutex_unlock(&hwmgr->smu_lock);
 	return 0;
+err:
+	mutex_unlock(&hwmgr->smu_lock);
+	return ret;
 }
 
 static int pp_dpm_force_clock_level(void *handle,
@@ -796,10 +719,8 @@ static int pp_dpm_force_clock_level(void *handle,
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->force_clock_level == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -820,10 +741,8 @@ static int pp_dpm_print_clock_levels(void *handle,
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->print_clock_levels == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -840,10 +759,8 @@ static int pp_dpm_get_sclk_od(void *handle)
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->get_sclk_od == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -860,10 +777,8 @@ static int pp_dpm_set_sclk_od(void *handle, uint32_t value)
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->set_sclk_od == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -881,10 +796,8 @@ static int pp_dpm_get_mclk_od(void *handle)
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->get_mclk_od == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -901,10 +814,8 @@ static int pp_dpm_set_mclk_od(void *handle, uint32_t value)
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->set_mclk_od == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -922,11 +833,7 @@ static int pp_dpm_read_sensor(void *handle, int idx,
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-	if (ret)
-		return ret;
-
-	if (value == NULL)
+	if (!hwmgr || !hwmgr->pm_en || !value)
 		return -EINVAL;
 
 	switch (idx) {
@@ -948,14 +855,11 @@ static struct amd_vce_state*
 pp_dpm_get_vce_clock_state(void *handle, unsigned idx)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
-
-	ret = pp_check(hwmgr);
 
-	if (ret)
+	if (!hwmgr || !hwmgr->pm_en)
 		return NULL;
 
-	if (hwmgr && idx < hwmgr->num_vce_state_tables)
+	if (idx < hwmgr->num_vce_state_tables)
 		return &hwmgr->vce_states[idx];
 	return NULL;
 }
@@ -964,7 +868,7 @@ static int pp_get_power_profile_mode(void *handle, char *buf)
 {
 	struct pp_hwmgr *hwmgr = handle;
 
-	if (!buf || pp_check(hwmgr))
+	if (!hwmgr || !hwmgr->pm_en || !buf)
 		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->get_power_profile_mode == NULL) {
@@ -980,12 +884,12 @@ static int pp_set_power_profile_mode(void *handle, long *input, uint32_t size)
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = -EINVAL;
 
-	if (pp_check(hwmgr))
-		return -EINVAL;
+	if (!hwmgr || !hwmgr->pm_en)
+		return ret;
 
 	if (hwmgr->hwmgr_func->set_power_profile_mode == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
-		return -EINVAL;
+		return ret;
 	}
 	mutex_lock(&hwmgr->smu_lock);
 	if (hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL)
@@ -998,7 +902,7 @@ static int pp_odn_edit_dpm_table(void *handle, uint32_t type, long *input, uint3
 {
 	struct pp_hwmgr *hwmgr = handle;
 
-	if (pp_check(hwmgr))
+	if (!hwmgr || !hwmgr->pm_en)
 		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->odn_edit_dpm_table == NULL) {
@@ -1016,7 +920,7 @@ static int pp_dpm_switch_power_profile(void *handle,
 	long workload;
 	uint32_t index;
 
-	if (pp_check(hwmgr))
+	if (!hwmgr || !hwmgr->pm_en)
 		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->set_power_profile_mode == NULL) {
@@ -1048,46 +952,12 @@ static int pp_dpm_switch_power_profile(void *handle,
 	return 0;
 }
 
-static int pp_dpm_notify_smu_memory_info(void *handle,
-					uint32_t virtual_addr_low,
-					uint32_t virtual_addr_hi,
-					uint32_t mc_addr_low,
-					uint32_t mc_addr_hi,
-					uint32_t size)
-{
-	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
-
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
-
-	if (hwmgr->hwmgr_func->notify_cac_buffer_info == NULL) {
-		pr_info("%s was not implemented.\n", __func__);
-		return -EINVAL;
-	}
-
-	mutex_lock(&hwmgr->smu_lock);
-
-	ret = hwmgr->hwmgr_func->notify_cac_buffer_info(hwmgr, virtual_addr_low,
-					virtual_addr_hi, mc_addr_low, mc_addr_hi,
-					size);
-
-	mutex_unlock(&hwmgr->smu_lock);
-
-	return ret;
-}
-
 static int pp_set_power_limit(void *handle, uint32_t limit)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->set_power_limit == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -1104,20 +974,14 @@ static int pp_set_power_limit(void *handle, uint32_t limit)
 	hwmgr->hwmgr_func->set_power_limit(hwmgr, limit);
 	hwmgr->power_limit = limit;
 	mutex_unlock(&hwmgr->smu_lock);
-	return ret;
+	return 0;
 }
 
 static int pp_get_power_limit(void *handle, uint32_t *limit, bool default_limit)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
-
-	ret = pp_check(hwmgr);
 
-	if (ret)
-		return ret;
-
-	if (limit == NULL)
+	if (!hwmgr || !hwmgr->pm_en ||!limit)
 		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -1129,19 +993,16 @@ static int pp_get_power_limit(void *handle, uint32_t *limit, bool default_limit)
 
 	mutex_unlock(&hwmgr->smu_lock);
 
-	return ret;
+	return 0;
 }
 
 static int pp_display_configuration_change(void *handle,
 	const struct amd_pp_display_configuration *display_config)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
 	phm_store_dal_configuration_data(hwmgr, display_config);
@@ -1155,12 +1016,7 @@ static int pp_get_display_power_level(void *handle,
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
-
-	if (output == NULL)
+	if (!hwmgr || !hwmgr->pm_en ||!output)
 		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -1177,10 +1033,8 @@ static int pp_get_current_clocks(void *handle,
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
 
@@ -1225,10 +1079,8 @@ static int pp_get_clock_by_type(void *handle, enum amd_pp_clock_type type, struc
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (clocks == NULL)
 		return -EINVAL;
@@ -1246,11 +1098,7 @@ static int pp_get_clock_by_type_with_latency(void *handle,
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-	if (ret)
-		return ret;
-
-	if (!clocks)
+	if (!hwmgr || !hwmgr->pm_en ||!clocks)
 		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -1266,11 +1114,7 @@ static int pp_get_clock_by_type_with_voltage(void *handle,
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-	if (ret)
-		return ret;
-
-	if (!clocks)
+	if (!hwmgr || !hwmgr->pm_en ||!clocks)
 		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -1287,11 +1131,7 @@ static int pp_set_watermarks_for_clocks_ranges(void *handle,
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-	if (ret)
-		return ret;
-
-	if (!wm_with_clock_ranges)
+	if (!hwmgr || !hwmgr->pm_en ||!wm_with_clock_ranges)
 		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -1308,11 +1148,7 @@ static int pp_display_clock_voltage_request(void *handle,
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-	if (ret)
-		return ret;
-
-	if (!clock)
+	if (!hwmgr || !hwmgr->pm_en ||!clock)
 		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -1328,12 +1164,7 @@ static int pp_get_display_mode_validation_clocks(void *handle,
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
-
-	if (clocks == NULL)
+	if (!hwmgr || !hwmgr->pm_en ||!clocks)
 		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -1348,12 +1179,9 @@ static int pp_get_display_mode_validation_clocks(void *handle,
 static int pp_set_mmhub_powergating_by_smu(void *handle)
 {
 	struct pp_hwmgr *hwmgr = handle;
-	int ret = 0;
 
-	ret = pp_check(hwmgr);
-
-	if (ret)
-		return ret;
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
 
 	if (hwmgr->hwmgr_func->set_mmhub_powergating_by_smu == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
@@ -1390,7 +1218,6 @@ static const struct amd_pm_funcs pp_dpm_funcs = {
 	.get_vce_clock_state = pp_dpm_get_vce_clock_state,
 	.switch_power_profile = pp_dpm_switch_power_profile,
 	.set_clockgating_by_smu = pp_set_clockgating_by_smu,
-	.notify_smu_memory_info = pp_dpm_notify_smu_memory_info,
 	.get_power_profile_mode = pp_get_power_profile_mode,
 	.set_power_profile_mode = pp_set_power_profile_mode,
 	.odn_edit_dpm_table = pp_odn_edit_dpm_table,
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
index ae2e9339dd6b..a0bb921fac22 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
@@ -75,8 +75,7 @@ int phm_set_power_state(struct pp_hwmgr *hwmgr,
 
 int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr)
 {
-	int ret = 1;
-	bool enabled;
+	int ret = -EINVAL;;
 	PHM_FUNC_CHECK(hwmgr);
 
 	if (smum_is_dpm_running(hwmgr)) {
@@ -87,17 +86,12 @@ int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr)
 	if (NULL != hwmgr->hwmgr_func->dynamic_state_management_enable)
 		ret = hwmgr->hwmgr_func->dynamic_state_management_enable(hwmgr);
 
-	enabled = ret == 0;
-
-	cgs_notify_dpm_enabled(hwmgr->device, enabled);
-
 	return ret;
 }
 
 int phm_disable_dynamic_state_management(struct pp_hwmgr *hwmgr)
 {
-	int ret = -1;
-	bool enabled;
+	int ret = -EINVAL;
 
 	PHM_FUNC_CHECK(hwmgr);
 
@@ -109,10 +103,6 @@ int phm_disable_dynamic_state_management(struct pp_hwmgr *hwmgr)
 	if (hwmgr->hwmgr_func->dynamic_state_management_disable)
 		ret = hwmgr->hwmgr_func->dynamic_state_management_disable(hwmgr);
 
-	enabled = ret == 0 ? false : true;
-
-	cgs_notify_dpm_enabled(hwmgr->device, enabled);
-
 	return ret;
 }
 
@@ -142,6 +132,15 @@ int phm_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
+int phm_apply_clock_adjust_rules(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (hwmgr->hwmgr_func->apply_clocks_adjust_rules != NULL)
+		return hwmgr->hwmgr_func->apply_clocks_adjust_rules(hwmgr);
+	return 0;
+}
+
 int phm_powerdown_uvd(struct pp_hwmgr *hwmgr)
 {
 	PHM_FUNC_CHECK(hwmgr);
@@ -171,6 +170,16 @@ int phm_disable_clock_power_gatings(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
+int phm_pre_display_configuration_changed(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (NULL != hwmgr->hwmgr_func->pre_display_config_changed)
+		hwmgr->hwmgr_func->pre_display_config_changed(hwmgr);
+
+	return 0;
+
+}
 
 int phm_display_configuration_changed(struct pp_hwmgr *hwmgr)
 {
@@ -275,13 +284,11 @@ int phm_store_dal_configuration_data(struct pp_hwmgr *hwmgr,
 	if (display_config == NULL)
 		return -EINVAL;
 
-	hwmgr->display_config = *display_config;
-
 	if (NULL != hwmgr->hwmgr_func->set_deep_sleep_dcefclk)
-		hwmgr->hwmgr_func->set_deep_sleep_dcefclk(hwmgr, hwmgr->display_config.min_dcef_deep_sleep_set_clk);
+		hwmgr->hwmgr_func->set_deep_sleep_dcefclk(hwmgr, display_config->min_dcef_deep_sleep_set_clk);
 
-	for (index = 0; index < hwmgr->display_config.num_path_including_non_display; index++) {
-		if (hwmgr->display_config.displays[index].controller_id != 0)
+	for (index = 0; index < display_config->num_path_including_non_display; index++) {
+		if (display_config->displays[index].controller_id != 0)
 			number_of_active_display++;
 	}
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
index 42982055b161..e63bc47dc715 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
@@ -40,6 +40,7 @@ extern const struct pp_smumgr_func iceland_smu_funcs;
 extern const struct pp_smumgr_func tonga_smu_funcs;
 extern const struct pp_smumgr_func fiji_smu_funcs;
 extern const struct pp_smumgr_func polaris10_smu_funcs;
+extern const struct pp_smumgr_func vegam_smu_funcs;
 extern const struct pp_smumgr_func vega10_smu_funcs;
 extern const struct pp_smumgr_func vega12_smu_funcs;
 extern const struct pp_smumgr_func smu10_smu_funcs;
@@ -76,7 +77,7 @@ static void hwmgr_init_workload_prority(struct pp_hwmgr *hwmgr)
 
 int hwmgr_early_init(struct pp_hwmgr *hwmgr)
 {
-	if (hwmgr == NULL)
+	if (!hwmgr)
 		return -EINVAL;
 
 	hwmgr->usec_timeout = AMD_MAX_USEC_TIMEOUT;
@@ -95,7 +96,8 @@ int hwmgr_early_init(struct pp_hwmgr *hwmgr)
 		hwmgr->smumgr_funcs = &ci_smu_funcs;
 		ci_set_asic_special_caps(hwmgr);
 		hwmgr->feature_mask &= ~(PP_VBI_TIME_SUPPORT_MASK |
-					PP_ENABLE_GFX_CG_THRU_SMU);
+					 PP_ENABLE_GFX_CG_THRU_SMU |
+					 PP_GFXOFF_MASK);
 		hwmgr->pp_table_version = PP_TABLE_V0;
 		hwmgr->od_enabled = false;
 		smu7_init_function_pointers(hwmgr);
@@ -103,9 +105,11 @@ int hwmgr_early_init(struct pp_hwmgr *hwmgr)
 	case AMDGPU_FAMILY_CZ:
 		hwmgr->od_enabled = false;
 		hwmgr->smumgr_funcs = &smu8_smu_funcs;
+		hwmgr->feature_mask &= ~PP_GFXOFF_MASK;
 		smu8_init_function_pointers(hwmgr);
 		break;
 	case AMDGPU_FAMILY_VI:
+		hwmgr->feature_mask &= ~PP_GFXOFF_MASK;
 		switch (hwmgr->chip_id) {
 		case CHIP_TOPAZ:
 			hwmgr->smumgr_funcs = &iceland_smu_funcs;
@@ -133,14 +137,21 @@ int hwmgr_early_init(struct pp_hwmgr *hwmgr)
 			polaris_set_asic_special_caps(hwmgr);
 			hwmgr->feature_mask &= ~(PP_UVD_HANDSHAKE_MASK);
 			break;
+		case CHIP_VEGAM:
+			hwmgr->smumgr_funcs = &vegam_smu_funcs;
+			polaris_set_asic_special_caps(hwmgr);
+			hwmgr->feature_mask &= ~(PP_UVD_HANDSHAKE_MASK);
+			break;
 		default:
 			return -EINVAL;
 		}
 		smu7_init_function_pointers(hwmgr);
 		break;
 	case AMDGPU_FAMILY_AI:
+		hwmgr->feature_mask &= ~PP_GFXOFF_MASK;
 		switch (hwmgr->chip_id) {
 		case CHIP_VEGA10:
+		case CHIP_VEGA20:
 			hwmgr->smumgr_funcs = &vega10_smu_funcs;
 			vega10_hwmgr_init(hwmgr);
 			break;
@@ -170,22 +181,58 @@ int hwmgr_early_init(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
+int hwmgr_sw_init(struct pp_hwmgr *hwmgr)
+{
+	if (!hwmgr|| !hwmgr->smumgr_funcs || !hwmgr->smumgr_funcs->smu_init)
+		return -EINVAL;
+
+	phm_register_irq_handlers(hwmgr);
+
+	return hwmgr->smumgr_funcs->smu_init(hwmgr);
+}
+
+
+int hwmgr_sw_fini(struct pp_hwmgr *hwmgr)
+{
+	if (hwmgr && hwmgr->smumgr_funcs && hwmgr->smumgr_funcs->smu_fini)
+		hwmgr->smumgr_funcs->smu_fini(hwmgr);
+
+	return 0;
+}
+
 int hwmgr_hw_init(struct pp_hwmgr *hwmgr)
 {
 	int ret = 0;
 
-	if (hwmgr == NULL)
+	if (!hwmgr || !hwmgr->smumgr_funcs)
 		return -EINVAL;
 
-	if (hwmgr->pptable_func == NULL ||
-	    hwmgr->pptable_func->pptable_init == NULL ||
-	    hwmgr->hwmgr_func->backend_init == NULL)
-		return -EINVAL;
+	if (hwmgr->smumgr_funcs->start_smu) {
+		ret = hwmgr->smumgr_funcs->start_smu(hwmgr);
+		if (ret) {
+			pr_err("smc start failed\n");
+			return -EINVAL;
+		}
+	}
+
+	if (!hwmgr->pm_en)
+		return 0;
+
+	if (!hwmgr->pptable_func ||
+	    !hwmgr->pptable_func->pptable_init ||
+	    !hwmgr->hwmgr_func->backend_init) {
+		hwmgr->pm_en = false;
+		pr_info("dpm not supported \n");
+		return 0;
+	}
 
 	ret = hwmgr->pptable_func->pptable_init(hwmgr);
 	if (ret)
 		goto err;
 
+	((struct amdgpu_device *)hwmgr->adev)->pm.no_fan =
+				hwmgr->thermal_controller.fanInfo.bNoFan;
+
 	ret = hwmgr->hwmgr_func->backend_init(hwmgr);
 	if (ret)
 		goto err1;
@@ -206,6 +253,8 @@ int hwmgr_hw_init(struct pp_hwmgr *hwmgr)
 	if (ret)
 		goto err2;
 
+	((struct amdgpu_device *)hwmgr->adev)->pm.dpm_enabled = true;
+
 	return 0;
 err2:
 	if (hwmgr->hwmgr_func->backend_fini)
@@ -214,14 +263,13 @@ int hwmgr_hw_init(struct pp_hwmgr *hwmgr)
 	if (hwmgr->pptable_func->pptable_fini)
 		hwmgr->pptable_func->pptable_fini(hwmgr);
 err:
-	pr_err("amdgpu: powerplay initialization failed\n");
 	return ret;
 }
 
 int hwmgr_hw_fini(struct pp_hwmgr *hwmgr)
 {
-	if (hwmgr == NULL)
-		return -EINVAL;
+	if (!hwmgr || !hwmgr->pm_en)
+		return 0;
 
 	phm_stop_thermal_controller(hwmgr);
 	psm_set_boot_states(hwmgr);
@@ -236,12 +284,12 @@ int hwmgr_hw_fini(struct pp_hwmgr *hwmgr)
 	return psm_fini_power_state_table(hwmgr);
 }
 
-int hwmgr_hw_suspend(struct pp_hwmgr *hwmgr)
+int hwmgr_suspend(struct pp_hwmgr *hwmgr)
 {
 	int ret = 0;
 
-	if (hwmgr == NULL)
-		return -EINVAL;
+	if (!hwmgr || !hwmgr->pm_en)
+		return 0;
 
 	phm_disable_smc_firmware_ctf(hwmgr);
 	ret = psm_set_boot_states(hwmgr);
@@ -255,13 +303,23 @@ int hwmgr_hw_suspend(struct pp_hwmgr *hwmgr)
 	return ret;
 }
 
-int hwmgr_hw_resume(struct pp_hwmgr *hwmgr)
+int hwmgr_resume(struct pp_hwmgr *hwmgr)
 {
 	int ret = 0;
 
-	if (hwmgr == NULL)
+	if (!hwmgr)
 		return -EINVAL;
 
+	if (hwmgr->smumgr_funcs && hwmgr->smumgr_funcs->start_smu) {
+		if (hwmgr->smumgr_funcs->start_smu(hwmgr)) {
+			pr_err("smc start failed\n");
+			return -EINVAL;
+		}
+	}
+
+	if (!hwmgr->pm_en)
+		return 0;
+
 	ret = phm_setup_asic(hwmgr);
 	if (ret)
 		return ret;
@@ -270,9 +328,6 @@ int hwmgr_hw_resume(struct pp_hwmgr *hwmgr)
 	if (ret)
 		return ret;
 	ret = phm_start_thermal_controller(hwmgr);
-	if (ret)
-		return ret;
-
 	ret |= psm_set_performance_states(hwmgr);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c
index 0f2851b5b368..0af13c154328 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c
@@ -46,7 +46,7 @@ int psm_init_power_state_table(struct pp_hwmgr *hwmgr)
 					  sizeof(struct pp_power_state);
 
 	if (table_entries == 0 || size == 0) {
-		pr_warn("Please check whether power state management is suppported on this asic\n");
+		pr_warn("Please check whether power state management is supported on this asic\n");
 		return 0;
 	}
 
@@ -265,6 +265,15 @@ int psm_adjust_power_state_dynamic(struct pp_hwmgr *hwmgr, bool skip,
 	if (skip)
 		return 0;
 
+	if (!hwmgr->ps)
+		/*
+		 * for vega12/vega20 which does not support power state manager
+		 * DAL clock limits should also be honoured
+		 */
+		phm_apply_clock_adjust_rules(hwmgr);
+
+	phm_pre_display_configuration_changed(hwmgr);
+
 	phm_display_configuration_changed(hwmgr);
 
 	if (hwmgr->ps)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
index c6febbf0bf69..7047e29755c3 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
@@ -23,7 +23,8 @@
 #include "pp_debug.h"
 #include <linux/module.h>
 #include <linux/slab.h>
-
+#include <linux/delay.h>
+#include "atom.h"
 #include "ppatomctrl.h"
 #include "atombios.h"
 #include "cgs_common.h"
@@ -128,7 +129,6 @@ static int atomctrl_set_mc_reg_address_table(
 	return 0;
 }
 
-
 int atomctrl_initialize_mc_reg_table(
 		struct pp_hwmgr *hwmgr,
 		uint8_t module_index,
@@ -141,7 +141,7 @@ int atomctrl_initialize_mc_reg_table(
 	u16 size;
 
 	vram_info = (ATOM_VRAM_INFO_HEADER_V2_1 *)
-		cgs_atom_get_data_table(hwmgr->device,
+		smu_atom_get_data_table(hwmgr->adev,
 				GetIndexIntoMasterTable(DATA, VRAM_Info), &size, &frev, &crev);
 
 	if (module_index >= vram_info->ucNumOfVRAMModule) {
@@ -174,6 +174,8 @@ int atomctrl_set_engine_dram_timings_rv770(
 		uint32_t engine_clock,
 		uint32_t memory_clock)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
+
 	SET_ENGINE_CLOCK_PS_ALLOCATION engine_clock_parameters;
 
 	/* They are both in 10KHz Units. */
@@ -184,9 +186,10 @@ int atomctrl_set_engine_dram_timings_rv770(
 	/* in 10 khz units.*/
 	engine_clock_parameters.sReserved.ulClock =
 		cpu_to_le32(memory_clock & SET_CLOCK_FREQ_MASK);
-	return cgs_atom_exec_cmd_table(hwmgr->device,
+
+	return amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings),
-			&engine_clock_parameters);
+			(uint32_t *)&engine_clock_parameters);
 }
 
 /**
@@ -203,7 +206,7 @@ static ATOM_VOLTAGE_OBJECT_INFO *get_voltage_info_table(void *device)
 	union voltage_object_info *voltage_info;
 
 	voltage_info = (union voltage_object_info *)
-		cgs_atom_get_data_table(device, index,
+		smu_atom_get_data_table(device, index,
 			&size, &frev, &crev);
 
 	if (voltage_info != NULL)
@@ -247,16 +250,16 @@ int atomctrl_get_memory_pll_dividers_si(
 		pp_atomctrl_memory_clock_param *mpll_param,
 		bool strobe_mode)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_1 mpll_parameters;
 	int result;
 
 	mpll_parameters.ulClock = cpu_to_le32(clock_value);
 	mpll_parameters.ucInputFlag = (uint8_t)((strobe_mode) ? 1 : 0);
 
-	result = cgs_atom_exec_cmd_table
-		(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 		 GetIndexIntoMasterTable(COMMAND, ComputeMemoryClockParam),
-		 &mpll_parameters);
+		(uint32_t *)&mpll_parameters);
 
 	if (0 == result) {
 		mpll_param->mpll_fb_divider.clk_frac =
@@ -295,14 +298,15 @@ int atomctrl_get_memory_pll_dividers_si(
 int atomctrl_get_memory_pll_dividers_vi(struct pp_hwmgr *hwmgr,
 		uint32_t clock_value, pp_atomctrl_memory_clock_param *mpll_param)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_2 mpll_parameters;
 	int result;
 
 	mpll_parameters.ulClock.ulClock = cpu_to_le32(clock_value);
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, ComputeMemoryClockParam),
-			&mpll_parameters);
+			(uint32_t *)&mpll_parameters);
 
 	if (!result)
 		mpll_param->mpll_post_divider =
@@ -311,19 +315,49 @@ int atomctrl_get_memory_pll_dividers_vi(struct pp_hwmgr *hwmgr,
 	return result;
 }
 
+int atomctrl_get_memory_pll_dividers_ai(struct pp_hwmgr *hwmgr,
+					uint32_t clock_value,
+					pp_atomctrl_memory_clock_param_ai *mpll_param)
+{
+	struct amdgpu_device *adev = hwmgr->adev;
+	COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_3 mpll_parameters = {{0}, 0, 0};
+	int result;
+
+	mpll_parameters.ulClock.ulClock = cpu_to_le32(clock_value);
+
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
+			GetIndexIntoMasterTable(COMMAND, ComputeMemoryClockParam),
+			(uint32_t *)&mpll_parameters);
+
+	/* VEGAM's mpll takes sometime to finish computing */
+	udelay(10);
+
+	if (!result) {
+		mpll_param->ulMclk_fcw_int =
+			le16_to_cpu(mpll_parameters.usMclk_fcw_int);
+		mpll_param->ulMclk_fcw_frac =
+			le16_to_cpu(mpll_parameters.usMclk_fcw_frac);
+		mpll_param->ulClock =
+			le32_to_cpu(mpll_parameters.ulClock.ulClock);
+		mpll_param->ulPostDiv = mpll_parameters.ulClock.ucPostDiv;
+	}
+
+	return result;
+}
+
 int atomctrl_get_engine_pll_dividers_kong(struct pp_hwmgr *hwmgr,
 					  uint32_t clock_value,
 					  pp_atomctrl_clock_dividers_kong *dividers)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4 pll_parameters;
 	int result;
 
 	pll_parameters.ulClock = cpu_to_le32(clock_value);
 
-	result = cgs_atom_exec_cmd_table
-		(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 		 GetIndexIntoMasterTable(COMMAND, ComputeMemoryEnginePLL),
-		 &pll_parameters);
+		(uint32_t *)&pll_parameters);
 
 	if (0 == result) {
 		dividers->pll_post_divider = pll_parameters.ucPostDiv;
@@ -338,16 +372,16 @@ int atomctrl_get_engine_pll_dividers_vi(
 		uint32_t clock_value,
 		pp_atomctrl_clock_dividers_vi *dividers)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 pll_patameters;
 	int result;
 
 	pll_patameters.ulClock.ulClock = cpu_to_le32(clock_value);
 	pll_patameters.ulClock.ucPostDiv = COMPUTE_GPUCLK_INPUT_FLAG_SCLK;
 
-	result = cgs_atom_exec_cmd_table
-		(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 		 GetIndexIntoMasterTable(COMMAND, ComputeMemoryEnginePLL),
-		 &pll_patameters);
+		(uint32_t *)&pll_patameters);
 
 	if (0 == result) {
 		dividers->pll_post_divider =
@@ -375,16 +409,16 @@ int atomctrl_get_engine_pll_dividers_ai(struct pp_hwmgr *hwmgr,
 		uint32_t clock_value,
 		pp_atomctrl_clock_dividers_ai *dividers)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_7 pll_patameters;
 	int result;
 
 	pll_patameters.ulClock.ulClock = cpu_to_le32(clock_value);
 	pll_patameters.ulClock.ucPostDiv = COMPUTE_GPUCLK_INPUT_FLAG_SCLK;
 
-	result = cgs_atom_exec_cmd_table
-		(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 		 GetIndexIntoMasterTable(COMMAND, ComputeMemoryEnginePLL),
-		 &pll_patameters);
+		(uint32_t *)&pll_patameters);
 
 	if (0 == result) {
 		dividers->usSclk_fcw_frac     = le16_to_cpu(pll_patameters.usSclk_fcw_frac);
@@ -407,6 +441,7 @@ int atomctrl_get_dfs_pll_dividers_vi(
 		uint32_t clock_value,
 		pp_atomctrl_clock_dividers_vi *dividers)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 pll_patameters;
 	int result;
 
@@ -414,10 +449,9 @@ int atomctrl_get_dfs_pll_dividers_vi(
 	pll_patameters.ulClock.ucPostDiv =
 		COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK;
 
-	result = cgs_atom_exec_cmd_table
-		(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 		 GetIndexIntoMasterTable(COMMAND, ComputeMemoryEnginePLL),
-		 &pll_patameters);
+		(uint32_t *)&pll_patameters);
 
 	if (0 == result) {
 		dividers->pll_post_divider =
@@ -452,7 +486,7 @@ uint32_t atomctrl_get_reference_clock(struct pp_hwmgr *hwmgr)
 	uint32_t clock;
 
 	fw_info = (ATOM_FIRMWARE_INFO *)
-		cgs_atom_get_data_table(hwmgr->device,
+		smu_atom_get_data_table(hwmgr->adev,
 			GetIndexIntoMasterTable(DATA, FirmwareInfo),
 			&size, &frev, &crev);
 
@@ -476,7 +510,7 @@ bool atomctrl_is_voltage_controlled_by_gpio_v3(
 		uint8_t voltage_mode)
 {
 	ATOM_VOLTAGE_OBJECT_INFO_V3_1 *voltage_info =
-		(ATOM_VOLTAGE_OBJECT_INFO_V3_1 *)get_voltage_info_table(hwmgr->device);
+		(ATOM_VOLTAGE_OBJECT_INFO_V3_1 *)get_voltage_info_table(hwmgr->adev);
 	bool ret;
 
 	PP_ASSERT_WITH_CODE((NULL != voltage_info),
@@ -495,7 +529,7 @@ int atomctrl_get_voltage_table_v3(
 		pp_atomctrl_voltage_table *voltage_table)
 {
 	ATOM_VOLTAGE_OBJECT_INFO_V3_1 *voltage_info =
-		(ATOM_VOLTAGE_OBJECT_INFO_V3_1 *)get_voltage_info_table(hwmgr->device);
+		(ATOM_VOLTAGE_OBJECT_INFO_V3_1 *)get_voltage_info_table(hwmgr->adev);
 	const ATOM_VOLTAGE_OBJECT_V3 *voltage_object;
 	unsigned int i;
 
@@ -572,7 +606,7 @@ static ATOM_GPIO_PIN_LUT *get_gpio_lookup_table(void *device)
 	void *table_address;
 
 	table_address = (ATOM_GPIO_PIN_LUT *)
-		cgs_atom_get_data_table(device,
+		smu_atom_get_data_table(device,
 				GetIndexIntoMasterTable(DATA, GPIO_Pin_LUT),
 				&size, &frev, &crev);
 
@@ -592,7 +626,7 @@ bool atomctrl_get_pp_assign_pin(
 {
 	bool bRet = false;
 	ATOM_GPIO_PIN_LUT *gpio_lookup_table =
-		get_gpio_lookup_table(hwmgr->device);
+		get_gpio_lookup_table(hwmgr->adev);
 
 	PP_ASSERT_WITH_CODE((NULL != gpio_lookup_table),
 			"Could not find GPIO lookup Table in BIOS.", return false);
@@ -613,7 +647,7 @@ int atomctrl_calculate_voltage_evv_on_sclk(
 		bool debug)
 {
 	ATOM_ASIC_PROFILING_INFO_V3_4 *getASICProfilingInfo;
-
+	struct amdgpu_device *adev = hwmgr->adev;
 	EFUSE_LINEAR_FUNC_PARAM sRO_fuse;
 	EFUSE_LINEAR_FUNC_PARAM sCACm_fuse;
 	EFUSE_LINEAR_FUNC_PARAM sCACb_fuse;
@@ -640,7 +674,7 @@ int atomctrl_calculate_voltage_evv_on_sclk(
 	int result;
 
 	getASICProfilingInfo = (ATOM_ASIC_PROFILING_INFO_V3_4 *)
-			cgs_atom_get_data_table(hwmgr->device,
+			smu_atom_get_data_table(hwmgr->adev,
 					GetIndexIntoMasterTable(DATA, ASIC_ProfilingInfo),
 					NULL, NULL, NULL);
 
@@ -706,9 +740,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
 
 	sOutput_FuseValues.sEfuse = sInput_FuseValues;
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
-			&sOutput_FuseValues);
+			(uint32_t *)&sOutput_FuseValues);
 
 	if (result)
 		return result;
@@ -727,9 +761,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
 
 	sOutput_FuseValues.sEfuse = sInput_FuseValues;
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
-			&sOutput_FuseValues);
+			(uint32_t *)&sOutput_FuseValues);
 
 	if (result)
 		return result;
@@ -747,9 +781,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
 	sInput_FuseValues.ucBitLength = sCACb_fuse.ucEfuseLength;
 	sOutput_FuseValues.sEfuse = sInput_FuseValues;
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
-			&sOutput_FuseValues);
+			(uint32_t *)&sOutput_FuseValues);
 
 	if (result)
 		return result;
@@ -768,9 +802,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
 
 	sOutput_FuseValues.sEfuse = sInput_FuseValues;
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
-			&sOutput_FuseValues);
+			(uint32_t *)&sOutput_FuseValues);
 
 	if (result)
 		return result;
@@ -790,9 +824,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
 
 	sOutput_FuseValues.sEfuse = sInput_FuseValues;
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
-			&sOutput_FuseValues);
+			(uint32_t *)&sOutput_FuseValues);
 	if (result)
 		return result;
 
@@ -811,9 +845,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
 	sInput_FuseValues.ucBitLength = sKv_b_fuse.ucEfuseLength;
 	sOutput_FuseValues.sEfuse = sInput_FuseValues;
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
-			&sOutput_FuseValues);
+			(uint32_t *)&sOutput_FuseValues);
 
 	if (result)
 		return result;
@@ -842,9 +876,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
 
 	sOutput_FuseValues.sEfuse = sInput_FuseValues;
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
-			&sOutput_FuseValues);
+			(uint32_t *)&sOutput_FuseValues);
 
 	if (result)
 		return result;
@@ -1053,8 +1087,9 @@ int atomctrl_get_voltage_evv_on_sclk(
 		uint32_t sclk, uint16_t virtual_voltage_Id,
 		uint16_t *voltage)
 {
-	int result;
+	struct amdgpu_device *adev = hwmgr->adev;
 	GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2 get_voltage_info_param_space;
+	int result;
 
 	get_voltage_info_param_space.ucVoltageType   =
 		voltage_type;
@@ -1065,14 +1100,12 @@ int atomctrl_get_voltage_evv_on_sclk(
 	get_voltage_info_param_space.ulSCLKFreq      =
 		cpu_to_le32(sclk);
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, GetVoltageInfo),
-			&get_voltage_info_param_space);
-
-	if (0 != result)
-		return result;
+			(uint32_t *)&get_voltage_info_param_space);
 
-	*voltage = le16_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 *)
+	*voltage = result ? 0 :
+			le16_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 *)
 				(&get_voltage_info_param_space))->usVoltageLevel);
 
 	return result;
@@ -1088,9 +1121,10 @@ int atomctrl_get_voltage_evv(struct pp_hwmgr *hwmgr,
 			     uint16_t virtual_voltage_id,
 			     uint16_t *voltage)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
+	GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2 get_voltage_info_param_space;
 	int result;
 	int entry_id;
-	GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2 get_voltage_info_param_space;
 
 	/* search for leakage voltage ID 0xff01 ~ 0xff08 and sckl */
 	for (entry_id = 0; entry_id < hwmgr->dyn_state.vddc_dependency_on_sclk->count; entry_id++) {
@@ -1111,9 +1145,9 @@ int atomctrl_get_voltage_evv(struct pp_hwmgr *hwmgr,
 	get_voltage_info_param_space.ulSCLKFreq =
 		cpu_to_le32(hwmgr->dyn_state.vddc_dependency_on_sclk->entries[entry_id].clk);
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, GetVoltageInfo),
-			&get_voltage_info_param_space);
+			(uint32_t *)&get_voltage_info_param_space);
 
 	if (0 != result)
 		return result;
@@ -1135,7 +1169,7 @@ uint32_t atomctrl_get_mpll_reference_clock(struct pp_hwmgr *hwmgr)
 	u16 size;
 
 	fw_info = (ATOM_COMMON_TABLE_HEADER *)
-		cgs_atom_get_data_table(hwmgr->device,
+		smu_atom_get_data_table(hwmgr->adev,
 				GetIndexIntoMasterTable(DATA, FirmwareInfo),
 				&size, &frev, &crev);
 
@@ -1167,7 +1201,7 @@ static ATOM_ASIC_INTERNAL_SS_INFO *asic_internal_ss_get_ss_table(void *device)
 	u16 size;
 
 	table = (ATOM_ASIC_INTERNAL_SS_INFO *)
-		cgs_atom_get_data_table(device,
+		smu_atom_get_data_table(device,
 			GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info),
 			&size, &frev, &crev);
 
@@ -1188,7 +1222,7 @@ static int asic_internal_ss_get_ss_asignment(struct pp_hwmgr *hwmgr,
 
 	memset(ssEntry, 0x00, sizeof(pp_atomctrl_internal_ss_info));
 
-	table = asic_internal_ss_get_ss_table(hwmgr->device);
+	table = asic_internal_ss_get_ss_table(hwmgr->adev);
 
 	if (NULL == table)
 		return -1;
@@ -1260,9 +1294,10 @@ int atomctrl_get_engine_clock_spread_spectrum(
 			ASIC_INTERNAL_ENGINE_SS, engine_clock, ssInfo);
 }
 
-int atomctrl_read_efuse(void *device, uint16_t start_index,
+int atomctrl_read_efuse(struct pp_hwmgr *hwmgr, uint16_t start_index,
 		uint16_t end_index, uint32_t mask, uint32_t *efuse)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	int result;
 	READ_EFUSE_VALUE_PARAMETER efuse_param;
 
@@ -1272,11 +1307,10 @@ int atomctrl_read_efuse(void *device, uint16_t start_index,
 	efuse_param.sEfuse.ucBitLength  = (uint8_t)
 			((end_index - start_index) + 1);
 
-	result = cgs_atom_exec_cmd_table(device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
-			&efuse_param);
-	if (!result)
-		*efuse = le32_to_cpu(efuse_param.ulEfuseValue) & mask;
+			(uint32_t *)&efuse_param);
+	*efuse = result ? 0 : le32_to_cpu(efuse_param.ulEfuseValue) & mask;
 
 	return result;
 }
@@ -1284,6 +1318,7 @@ int atomctrl_read_efuse(void *device, uint16_t start_index,
 int atomctrl_set_ac_timing_ai(struct pp_hwmgr *hwmgr, uint32_t memory_clock,
 			      uint8_t level)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	DYNAMICE_MEMORY_SETTINGS_PARAMETER_V2_1 memory_clock_parameters;
 	int result;
 
@@ -1293,10 +1328,9 @@ int atomctrl_set_ac_timing_ai(struct pp_hwmgr *hwmgr, uint32_t memory_clock,
 		ADJUST_MC_SETTING_PARAM;
 	memory_clock_parameters.asDPMMCReg.ucMclkDPMState = level;
 
-	result = cgs_atom_exec_cmd_table
-		(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 		 GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings),
-		 &memory_clock_parameters);
+		(uint32_t *)&memory_clock_parameters);
 
 	return result;
 }
@@ -1304,7 +1338,7 @@ int atomctrl_set_ac_timing_ai(struct pp_hwmgr *hwmgr, uint32_t memory_clock,
 int atomctrl_get_voltage_evv_on_sclk_ai(struct pp_hwmgr *hwmgr, uint8_t voltage_type,
 				uint32_t sclk, uint16_t virtual_voltage_Id, uint32_t *voltage)
 {
-
+	struct amdgpu_device *adev = hwmgr->adev;
 	int result;
 	GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_3 get_voltage_info_param_space;
 
@@ -1313,15 +1347,12 @@ int atomctrl_get_voltage_evv_on_sclk_ai(struct pp_hwmgr *hwmgr, uint8_t voltage_
 	get_voltage_info_param_space.usVoltageLevel = cpu_to_le16(virtual_voltage_Id);
 	get_voltage_info_param_space.ulSCLKFreq = cpu_to_le32(sclk);
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, GetVoltageInfo),
-			&get_voltage_info_param_space);
+			(uint32_t *)&get_voltage_info_param_space);
 
-	if (0 != result)
-		return result;
-
-	*voltage = le32_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_3 *)
-				(&get_voltage_info_param_space))->ulVoltageLevel);
+	*voltage = result ? 0 :
+		le32_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_3 *)(&get_voltage_info_param_space))->ulVoltageLevel);
 
 	return result;
 }
@@ -1334,7 +1365,7 @@ int atomctrl_get_smc_sclk_range_table(struct pp_hwmgr *hwmgr, struct pp_atom_ctr
 	u16 size;
 
 	ATOM_SMU_INFO_V2_1 *psmu_info =
-		(ATOM_SMU_INFO_V2_1 *)cgs_atom_get_data_table(hwmgr->device,
+		(ATOM_SMU_INFO_V2_1 *)smu_atom_get_data_table(hwmgr->adev,
 			GetIndexIntoMasterTable(DATA, SMU_Info),
 			&size, &frev, &crev);
 
@@ -1362,7 +1393,7 @@ int atomctrl_get_avfs_information(struct pp_hwmgr *hwmgr,
 		return -EINVAL;
 
 	profile = (ATOM_ASIC_PROFILING_INFO_V3_6 *)
-			cgs_atom_get_data_table(hwmgr->device,
+			smu_atom_get_data_table(hwmgr->adev,
 					GetIndexIntoMasterTable(DATA, ASIC_ProfilingInfo),
 					NULL, NULL, NULL);
 	if (!profile)
@@ -1402,7 +1433,7 @@ int  atomctrl_get_svi2_info(struct pp_hwmgr *hwmgr, uint8_t voltage_type,
 				uint16_t *load_line)
 {
 	ATOM_VOLTAGE_OBJECT_INFO_V3_1 *voltage_info =
-		(ATOM_VOLTAGE_OBJECT_INFO_V3_1 *)get_voltage_info_table(hwmgr->device);
+		(ATOM_VOLTAGE_OBJECT_INFO_V3_1 *)get_voltage_info_table(hwmgr->adev);
 
 	const ATOM_VOLTAGE_OBJECT_V3 *voltage_object;
 
@@ -1421,16 +1452,17 @@ int  atomctrl_get_svi2_info(struct pp_hwmgr *hwmgr, uint8_t voltage_type,
 
 int atomctrl_get_leakage_id_from_efuse(struct pp_hwmgr *hwmgr, uint16_t *virtual_voltage_id)
 {
-	int result;
+	struct amdgpu_device *adev = hwmgr->adev;
 	SET_VOLTAGE_PS_ALLOCATION allocation;
 	SET_VOLTAGE_PARAMETERS_V1_3 *voltage_parameters =
 			(SET_VOLTAGE_PARAMETERS_V1_3 *)&allocation.sASICSetVoltage;
+	int result;
 
 	voltage_parameters->ucVoltageMode = ATOM_GET_LEAKAGE_ID;
 
-	result = cgs_atom_exec_cmd_table(hwmgr->device,
+	result = amdgpu_atom_execute_table(adev->mode_info.atom_context,
 			GetIndexIntoMasterTable(COMMAND, SetVoltage),
-			voltage_parameters);
+			(uint32_t *)voltage_parameters);
 
 	*virtual_voltage_id = voltage_parameters->usVoltageLevel;
 
@@ -1453,7 +1485,7 @@ int atomctrl_get_leakage_vddc_base_on_leakage(struct pp_hwmgr *hwmgr,
 	ix = GetIndexIntoMasterTable(DATA, ASIC_ProfilingInfo);
 
 	profile = (ATOM_ASIC_PROFILING_INFO_V2_1 *)
-			cgs_atom_get_data_table(hwmgr->device,
+			smu_atom_get_data_table(hwmgr->adev,
 					ix,
 					NULL, NULL, NULL);
 	if (!profile)
@@ -1498,3 +1530,33 @@ int atomctrl_get_leakage_vddc_base_on_leakage(struct pp_hwmgr *hwmgr,
 
 	return 0;
 }
+
+void atomctrl_get_voltage_range(struct pp_hwmgr *hwmgr, uint32_t *max_vddc,
+							uint32_t *min_vddc)
+{
+	void *profile;
+
+	profile = smu_atom_get_data_table(hwmgr->adev,
+					GetIndexIntoMasterTable(DATA, ASIC_ProfilingInfo),
+					NULL, NULL, NULL);
+
+	if (profile) {
+		switch (hwmgr->chip_id) {
+		case CHIP_TONGA:
+		case CHIP_FIJI:
+			*max_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_3 *)profile)->ulMaxVddc/4);
+			*min_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_3 *)profile)->ulMinVddc/4);
+			return;
+		case CHIP_POLARIS11:
+		case CHIP_POLARIS10:
+		case CHIP_POLARIS12:
+			*max_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_6 *)profile)->ulMaxVddc/100);
+			*min_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_6 *)profile)->ulMinVddc/100);
+			return;
+		default:
+			break;
+		}
+	}
+	*max_vddc = 0;
+	*min_vddc = 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h
index c44a92064cf1..3ee54f182943 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h
@@ -146,6 +146,14 @@ struct pp_atomctrl_memory_clock_param {
 };
 typedef struct pp_atomctrl_memory_clock_param pp_atomctrl_memory_clock_param;
 
+struct pp_atomctrl_memory_clock_param_ai {
+	uint32_t ulClock;
+	uint32_t ulPostDiv;
+	uint16_t ulMclk_fcw_frac;
+	uint16_t ulMclk_fcw_int;
+};
+typedef struct pp_atomctrl_memory_clock_param_ai pp_atomctrl_memory_clock_param_ai;
+
 struct pp_atomctrl_internal_ss_info {
 	uint32_t speed_spectrum_percentage;                      /* in 1/100 percentage */
 	uint32_t speed_spectrum_rate;                            /* in KHz */
@@ -295,10 +303,12 @@ extern bool atomctrl_is_voltage_controlled_by_gpio_v3(struct pp_hwmgr *hwmgr, ui
 extern int atomctrl_get_voltage_table_v3(struct pp_hwmgr *hwmgr, uint8_t voltage_type, uint8_t voltage_mode, pp_atomctrl_voltage_table *voltage_table);
 extern int atomctrl_get_memory_pll_dividers_vi(struct pp_hwmgr *hwmgr,
 		uint32_t clock_value, pp_atomctrl_memory_clock_param *mpll_param);
+extern int atomctrl_get_memory_pll_dividers_ai(struct pp_hwmgr *hwmgr,
+		uint32_t clock_value, pp_atomctrl_memory_clock_param_ai *mpll_param);
 extern int atomctrl_get_engine_pll_dividers_kong(struct pp_hwmgr *hwmgr,
 						 uint32_t clock_value,
 						 pp_atomctrl_clock_dividers_kong *dividers);
-extern int atomctrl_read_efuse(void *device, uint16_t start_index,
+extern int atomctrl_read_efuse(struct pp_hwmgr *hwmgr, uint16_t start_index,
 		uint16_t end_index, uint32_t mask, uint32_t *efuse);
 extern int atomctrl_calculate_voltage_evv_on_sclk(struct pp_hwmgr *hwmgr, uint8_t voltage_type,
 		uint32_t sclk, uint16_t virtual_voltage_Id, uint16_t *voltage, uint16_t dpm_level, bool debug);
@@ -320,5 +330,8 @@ extern int atomctrl_get_leakage_vddc_base_on_leakage(struct pp_hwmgr *hwmgr,
 					uint16_t virtual_voltage_id,
 					uint16_t efuse_voltage_id);
 extern int atomctrl_get_leakage_id_from_efuse(struct pp_hwmgr *hwmgr, uint16_t *virtual_voltage_id);
+
+extern void atomctrl_get_voltage_range(struct pp_hwmgr *hwmgr, uint32_t *max_vddc,
+							uint32_t *min_vddc);
 #endif
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
index ad42caac033e..c97b0e5ba43b 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
@@ -23,9 +23,9 @@
 
 #include "ppatomfwctrl.h"
 #include "atomfirmware.h"
+#include "atom.h"
 #include "pp_debug.h"
 
-
 static const union atom_voltage_object_v4 *pp_atomfwctrl_lookup_voltage_type_v4(
 		const struct atom_voltage_objects_info_v4_1 *voltage_object_info_table,
 		uint8_t voltage_type, uint8_t voltage_mode)
@@ -38,35 +38,34 @@ static const union atom_voltage_object_v4 *pp_atomfwctrl_lookup_voltage_type_v4(
 
 	while (offset < size) {
 		const union atom_voltage_object_v4 *voltage_object =
-				(const union atom_voltage_object_v4 *)(start + offset);
+			(const union atom_voltage_object_v4 *)(start + offset);
 
-        if (voltage_type == voltage_object->gpio_voltage_obj.header.voltage_type &&
-            voltage_mode == voltage_object->gpio_voltage_obj.header.voltage_mode)
-            return voltage_object;
+		if (voltage_type == voltage_object->gpio_voltage_obj.header.voltage_type &&
+		    voltage_mode == voltage_object->gpio_voltage_obj.header.voltage_mode)
+			return voltage_object;
 
-        offset += le16_to_cpu(voltage_object->gpio_voltage_obj.header.object_size);
+		offset += le16_to_cpu(voltage_object->gpio_voltage_obj.header.object_size);
 
-    }
+	}
 
-    return NULL;
+	return NULL;
 }
 
 static struct atom_voltage_objects_info_v4_1 *pp_atomfwctrl_get_voltage_info_table(
 		struct pp_hwmgr *hwmgr)
 {
-    const void *table_address;
-    uint16_t idx;
+	const void *table_address;
+	uint16_t idx;
 
-    idx = GetIndexIntoMasterDataTable(voltageobject_info);
-    table_address =	cgs_atom_get_data_table(hwmgr->device,
-    		idx, NULL, NULL, NULL);
+	idx = GetIndexIntoMasterDataTable(voltageobject_info);
+	table_address = smu_atom_get_data_table(hwmgr->adev,
+						idx, NULL, NULL, NULL);
 
-    PP_ASSERT_WITH_CODE( 
-        table_address,
-        "Error retrieving BIOS Table Address!",
-        return NULL);
+	PP_ASSERT_WITH_CODE(table_address,
+			"Error retrieving BIOS Table Address!",
+			return NULL);
 
-    return (struct atom_voltage_objects_info_v4_1 *)table_address;
+	return (struct atom_voltage_objects_info_v4_1 *)table_address;
 }
 
 /**
@@ -167,7 +166,7 @@ static struct atom_gpio_pin_lut_v2_1 *pp_atomfwctrl_get_gpio_lookup_table(
 	uint16_t idx;
 
 	idx = GetIndexIntoMasterDataTable(gpio_pin_lut);
-	table_address =	cgs_atom_get_data_table(hwmgr->device,
+	table_address =	smu_atom_get_data_table(hwmgr->adev,
 			idx, NULL, NULL, NULL);
 	PP_ASSERT_WITH_CODE(table_address,
 			"Error retrieving BIOS Table Address!",
@@ -248,28 +247,30 @@ int pp_atomfwctrl_get_gpu_pll_dividers_vega10(struct pp_hwmgr *hwmgr,
 		uint32_t clock_type, uint32_t clock_value,
 		struct pp_atomfwctrl_clock_dividers_soc15 *dividers)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	struct compute_gpu_clock_input_parameter_v1_8 pll_parameters;
 	struct compute_gpu_clock_output_parameter_v1_8 *pll_output;
-	int result;
 	uint32_t idx;
 
 	pll_parameters.gpuclock_10khz = (uint32_t)clock_value;
 	pll_parameters.gpu_clock_type = clock_type;
 
 	idx = GetIndexIntoMasterCmdTable(computegpuclockparam);
-	result = cgs_atom_exec_cmd_table(hwmgr->device, idx, &pll_parameters);
-
-	if (!result) {
-		pll_output = (struct compute_gpu_clock_output_parameter_v1_8 *)
-				&pll_parameters;
-		dividers->ulClock = le32_to_cpu(pll_output->gpuclock_10khz);
-		dividers->ulDid = le32_to_cpu(pll_output->dfs_did);
-		dividers->ulPll_fb_mult = le32_to_cpu(pll_output->pll_fb_mult);
-		dividers->ulPll_ss_fbsmult = le32_to_cpu(pll_output->pll_ss_fbsmult);
-		dividers->usPll_ss_slew_frac = le16_to_cpu(pll_output->pll_ss_slew_frac);
-		dividers->ucPll_ss_enable = pll_output->pll_ss_enable;
-	}
-	return result;
+
+	if (amdgpu_atom_execute_table(
+		adev->mode_info.atom_context, idx, (uint32_t *)&pll_parameters))
+		return -EINVAL;
+
+	pll_output = (struct compute_gpu_clock_output_parameter_v1_8 *)
+			&pll_parameters;
+	dividers->ulClock = le32_to_cpu(pll_output->gpuclock_10khz);
+	dividers->ulDid = le32_to_cpu(pll_output->dfs_did);
+	dividers->ulPll_fb_mult = le32_to_cpu(pll_output->pll_fb_mult);
+	dividers->ulPll_ss_fbsmult = le32_to_cpu(pll_output->pll_ss_fbsmult);
+	dividers->usPll_ss_slew_frac = le16_to_cpu(pll_output->pll_ss_slew_frac);
+	dividers->ucPll_ss_enable = pll_output->pll_ss_enable;
+
+	return 0;
 }
 
 int pp_atomfwctrl_get_avfs_information(struct pp_hwmgr *hwmgr,
@@ -283,7 +284,7 @@ int pp_atomfwctrl_get_avfs_information(struct pp_hwmgr *hwmgr,
 
 	idx = GetIndexIntoMasterDataTable(asic_profiling_info);
 	profile = (struct atom_asic_profiling_info_v4_1 *)
-			cgs_atom_get_data_table(hwmgr->device,
+			smu_atom_get_data_table(hwmgr->adev,
 					idx, NULL, NULL, NULL);
 
 	if (!profile)
@@ -467,7 +468,7 @@ int pp_atomfwctrl_get_gpio_information(struct pp_hwmgr *hwmgr,
 
 	idx = GetIndexIntoMasterDataTable(smu_info);
 	info = (struct atom_smu_info_v3_1 *)
-		cgs_atom_get_data_table(hwmgr->device,
+		smu_atom_get_data_table(hwmgr->adev,
 				idx, NULL, NULL, NULL);
 
 	if (!info) {
@@ -487,8 +488,9 @@ int pp_atomfwctrl_get_gpio_information(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
-int pp_atomfwctrl__get_clk_information_by_clkid(struct pp_hwmgr *hwmgr, BIOS_CLKID id, uint32_t *frequency)
+int pp_atomfwctrl_get_clk_information_by_clkid(struct pp_hwmgr *hwmgr, BIOS_CLKID id, uint32_t *frequency)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	struct atom_get_smu_clock_info_parameters_v3_1   parameters;
 	struct atom_get_smu_clock_info_output_parameters_v3_1 *output;
 	uint32_t ix;
@@ -497,13 +499,13 @@ int pp_atomfwctrl__get_clk_information_by_clkid(struct pp_hwmgr *hwmgr, BIOS_CLK
 	parameters.command = GET_SMU_CLOCK_INFO_V3_1_GET_CLOCK_FREQ;
 
 	ix = GetIndexIntoMasterCmdTable(getsmuclockinfo);
-	if (!cgs_atom_exec_cmd_table(hwmgr->device, ix, &parameters)) {
-		output = (struct atom_get_smu_clock_info_output_parameters_v3_1 *)&parameters;
-		*frequency = output->atom_smu_outputclkfreq.smu_clock_freq_hz / 10000;
-	} else {
-		pr_info("Error execute_table getsmuclockinfo!");
-		return -1;
-	}
+
+	if (amdgpu_atom_execute_table(
+		adev->mode_info.atom_context, ix, (uint32_t *)&parameters))
+		return -EINVAL;
+
+	output = (struct atom_get_smu_clock_info_output_parameters_v3_1 *)&parameters;
+	*frequency = output->atom_smu_outputclkfreq.smu_clock_freq_hz / 10000;
 
 	return 0;
 }
@@ -513,11 +515,10 @@ int pp_atomfwctrl_get_vbios_bootup_values(struct pp_hwmgr *hwmgr,
 {
 	struct atom_firmware_info_v3_1 *info = NULL;
 	uint16_t ix;
-	uint32_t frequency = 0;
 
 	ix = GetIndexIntoMasterDataTable(firmwareinfo);
 	info = (struct atom_firmware_info_v3_1 *)
-		cgs_atom_get_data_table(hwmgr->device,
+		smu_atom_get_data_table(hwmgr->adev,
 				ix, NULL, NULL, NULL);
 
 	if (!info) {
@@ -536,12 +537,6 @@ int pp_atomfwctrl_get_vbios_bootup_values(struct pp_hwmgr *hwmgr,
 	boot_values->ulSocClk   = 0;
 	boot_values->ulDCEFClk   = 0;
 
-	if (!pp_atomfwctrl__get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_SOCCLK_ID, &frequency))
-		boot_values->ulSocClk   = frequency;
-
-	if (!pp_atomfwctrl__get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_DCEFCLK_ID, &frequency))
-		boot_values->ulDCEFClk   = frequency;
-
 	return 0;
 }
 
@@ -553,7 +548,7 @@ int pp_atomfwctrl_get_smc_dpm_information(struct pp_hwmgr *hwmgr,
 
 	ix = GetIndexIntoMasterDataTable(smc_dpm_info);
 	info = (struct atom_smc_dpm_info_v4_1 *)
-		cgs_atom_get_data_table(hwmgr->device,
+		smu_atom_get_data_table(hwmgr->adev,
 				ix, NULL, NULL, NULL);
 	if (!info) {
 		pr_info("Error retrieving BIOS Table Address!");
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h
index 8df1e84f27c9..fe10aa4db5e6 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h
@@ -230,6 +230,8 @@ int pp_atomfwctrl_get_vbios_bootup_values(struct pp_hwmgr *hwmgr,
 			struct pp_atomfwctrl_bios_boot_up_values *boot_values);
 int pp_atomfwctrl_get_smc_dpm_information(struct pp_hwmgr *hwmgr,
 			struct pp_atomfwctrl_smc_dpm_parameters *param);
+int pp_atomfwctrl_get_clk_information_by_clkid(struct pp_hwmgr *hwmgr,
+					BIOS_CLKID id, uint32_t *frequency);
 
 #endif
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
index c9eecce5683f..f0d48b183d22 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
@@ -141,7 +141,7 @@ static const void *get_powerplay_table(struct pp_hwmgr *hwmgr)
 
 	if (!table_address) {
 		table_address = (ATOM_Tonga_POWERPLAYTABLE *)
-				cgs_atom_get_data_table(hwmgr->device,
+				smu_atom_get_data_table(hwmgr->adev,
 						index, &size, &frev, &crev);
 		hwmgr->soft_pp_table = table_address;	/*Cache the result in RAM.*/
 		hwmgr->soft_pp_table_size = size;
@@ -728,6 +728,32 @@ static int get_mm_clock_voltage_table(
 	return 0;
 }
 
+static int get_gpio_table(struct pp_hwmgr *hwmgr,
+		struct phm_ppt_v1_gpio_table **pp_tonga_gpio_table,
+		const ATOM_Tonga_GPIO_Table *atom_gpio_table)
+{
+	uint32_t table_size;
+	struct phm_ppt_v1_gpio_table *pp_gpio_table;
+	struct phm_ppt_v1_information *pp_table_information =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	table_size = sizeof(struct phm_ppt_v1_gpio_table);
+	pp_gpio_table = kzalloc(table_size, GFP_KERNEL);
+	if (!pp_gpio_table)
+		return -ENOMEM;
+
+	if (pp_table_information->vdd_dep_on_sclk->count <
+			atom_gpio_table->ucVRHotTriggeredSclkDpmIndex)
+		PP_ASSERT_WITH_CODE(false,
+				"SCLK DPM index for VRHot cannot exceed the total sclk level count!",);
+	else
+		pp_gpio_table->vrhot_triggered_sclk_dpm_index =
+				atom_gpio_table->ucVRHotTriggeredSclkDpmIndex;
+
+	*pp_tonga_gpio_table = pp_gpio_table;
+
+	return 0;
+}
 /**
  * Private Function used during initialization.
  * Initialize clock voltage dependency
@@ -761,11 +787,15 @@ static int init_clock_voltage_dependency(
 	const PPTable_Generic_SubTable_Header *pcie_table =
 		(const PPTable_Generic_SubTable_Header *)(((unsigned long) powerplay_table) +
 		le16_to_cpu(powerplay_table->usPCIETableOffset));
+	const ATOM_Tonga_GPIO_Table *gpio_table =
+		(const ATOM_Tonga_GPIO_Table *)(((unsigned long) powerplay_table) +
+		le16_to_cpu(powerplay_table->usGPIOTableOffset));
 
 	pp_table_information->vdd_dep_on_sclk = NULL;
 	pp_table_information->vdd_dep_on_mclk = NULL;
 	pp_table_information->mm_dep_table = NULL;
 	pp_table_information->pcie_table = NULL;
+	pp_table_information->gpio_table = NULL;
 
 	if (powerplay_table->usMMDependencyTableOffset != 0)
 		result = get_mm_clock_voltage_table(hwmgr,
@@ -810,6 +840,10 @@ static int init_clock_voltage_dependency(
 		result = get_valid_clk(hwmgr, &pp_table_information->valid_sclk_values,
 		pp_table_information->vdd_dep_on_sclk);
 
+	if (!result && gpio_table)
+		result = get_gpio_table(hwmgr, &pp_table_information->gpio_table,
+				gpio_table);
+
 	return result;
 }
 
@@ -1116,6 +1150,9 @@ static int pp_tables_v1_0_uninitialize(struct pp_hwmgr *hwmgr)
 	kfree(pp_table_information->pcie_table);
 	pp_table_information->pcie_table = NULL;
 
+	kfree(pp_table_information->gpio_table);
+	pp_table_information->gpio_table = NULL;
+
 	kfree(hwmgr->pptable);
 	hwmgr->pptable = NULL;
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
index 36ca7c419c90..ce64dfabd34b 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
@@ -837,7 +837,7 @@ static const ATOM_PPLIB_POWERPLAYTABLE *get_powerplay_table(
 			hwmgr->soft_pp_table = &soft_dummy_pp_table[0];
 			hwmgr->soft_pp_table_size = sizeof(soft_dummy_pp_table);
 		} else {
-			table_addr = cgs_atom_get_data_table(hwmgr->device,
+			table_addr = smu_atom_get_data_table(hwmgr->adev,
 					GetIndexIntoMasterTable(DATA, PowerPlayInfo),
 					&size, &frev, &crev);
 			hwmgr->soft_pp_table = table_addr;
@@ -1058,7 +1058,7 @@ static int init_overdrive_limits(struct pp_hwmgr *hwmgr,
 		return 0;
 
 	/* We assume here that fw_info is unchanged if this call fails.*/
-	fw_info = cgs_atom_get_data_table(hwmgr->device,
+	fw_info = smu_atom_get_data_table(hwmgr->adev,
 			 GetIndexIntoMasterTable(DATA, FirmwareInfo),
 			 &size, &frev, &crev);
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c
index 10253b89b3d8..85f84f4d8be5 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c
@@ -34,7 +34,7 @@
 #include "rv_ppsmc.h"
 #include "smu10_hwmgr.h"
 #include "power_state.h"
-#include "pp_soc15.h"
+#include "soc15_common.h"
 
 #define SMU10_MAX_DEEPSLEEP_DIVIDER_ID     5
 #define SMU10_MINIMUM_ENGINE_CLOCK         800   /* 8Mhz, the low boundary of engine clock allowed on this chip */
@@ -42,6 +42,13 @@
 #define SMU10_DISPCLK_BYPASS_THRESHOLD     10000 /* 100Mhz */
 #define SMC_RAM_END                     0x40000
 
+#define mmPWR_MISC_CNTL_STATUS					0x0183
+#define mmPWR_MISC_CNTL_STATUS_BASE_IDX				0
+#define PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN__SHIFT	0x0
+#define PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT		0x1
+#define PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK		0x00000001L
+#define PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK		0x00000006L
+
 static const unsigned long SMU10_Magic = (unsigned long) PHM_Rv_Magic;
 
 
@@ -74,11 +81,15 @@ static int smu10_initialize_dpm_defaults(struct pp_hwmgr *hwmgr)
 	smu10_data->thermal_auto_throttling_treshold = 0;
 	smu10_data->is_nb_dpm_enabled = 1;
 	smu10_data->dpm_flags = 1;
-	smu10_data->gfx_off_controled_by_driver = false;
 	smu10_data->need_min_deep_sleep_dcefclk = true;
 	smu10_data->num_active_display = 0;
 	smu10_data->deep_sleep_dcefclk = 0;
 
+	if (hwmgr->feature_mask & PP_GFXOFF_MASK)
+		smu10_data->gfx_off_controled_by_driver = true;
+	else
+		smu10_data->gfx_off_controled_by_driver = false;
+
 	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 					PHM_PlatformCaps_SclkDeepSleep);
 
@@ -161,7 +172,7 @@ static int smu10_set_clock_limit(struct pp_hwmgr *hwmgr, const void *input)
 	struct PP_Clocks clocks = {0};
 	struct pp_display_clock_request clock_req;
 
-	clocks.dcefClock = hwmgr->display_config.min_dcef_set_clk;
+	clocks.dcefClock = hwmgr->display_config->min_dcef_set_clk;
 	clock_req.clock_type = amd_pp_dcf_clock;
 	clock_req.clock_freq_in_khz = clocks.dcefClock * 10;
 
@@ -206,12 +217,18 @@ static int smu10_set_power_state_tasks(struct pp_hwmgr *hwmgr, const void *input
 static int smu10_init_power_gate_state(struct pp_hwmgr *hwmgr)
 {
 	struct smu10_hwmgr *smu10_data = (struct smu10_hwmgr *)(hwmgr->backend);
+	struct amdgpu_device *adev = hwmgr->adev;
 
 	smu10_data->vcn_power_gated = true;
 	smu10_data->isp_tileA_power_gated = true;
 	smu10_data->isp_tileB_power_gated = true;
 
-	return 0;
+	if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)
+		return smum_send_msg_to_smc_with_parameter(hwmgr,
+							   PPSMC_MSG_SetGfxCGPG,
+							   true);
+	else
+		return 0;
 }
 
 
@@ -237,13 +254,31 @@ static int smu10_power_off_asic(struct pp_hwmgr *hwmgr)
 	return smu10_reset_cc6_data(hwmgr);
 }
 
+static bool smu10_is_gfx_on(struct pp_hwmgr *hwmgr)
+{
+	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
+
+	reg = RREG32_SOC15(PWR, 0, mmPWR_MISC_CNTL_STATUS);
+	if ((reg & PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK) ==
+	    (0x2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT))
+		return true;
+
+	return false;
+}
+
 static int smu10_disable_gfx_off(struct pp_hwmgr *hwmgr)
 {
 	struct smu10_hwmgr *smu10_data = (struct smu10_hwmgr *)(hwmgr->backend);
 
-	if (smu10_data->gfx_off_controled_by_driver)
+	if (smu10_data->gfx_off_controled_by_driver) {
 		smum_send_msg_to_smc(hwmgr, PPSMC_MSG_DisableGfxOff);
 
+		/* confirm gfx is back to "on" state */
+		while (!smu10_is_gfx_on(hwmgr))
+			msleep(1);
+	}
+
 	return 0;
 }
 
@@ -267,6 +302,14 @@ static int smu10_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 	return smu10_enable_gfx_off(hwmgr);
 }
 
+static int smu10_gfx_off_control(struct pp_hwmgr *hwmgr, bool enable)
+{
+	if (enable)
+		return smu10_enable_gfx_off(hwmgr);
+	else
+		return smu10_disable_gfx_off(hwmgr);
+}
+
 static int smu10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 				struct pp_power_state  *prequest_ps,
 			const struct pp_power_state *pcurrent_ps)
@@ -340,7 +383,7 @@ static int smu10_get_clock_voltage_dependency_table(struct pp_hwmgr *hwmgr,
 
 static int smu10_populate_clock_table(struct pp_hwmgr *hwmgr)
 {
-	int result;
+	uint32_t result;
 
 	struct smu10_hwmgr *smu10_data = (struct smu10_hwmgr *)(hwmgr->backend);
 	DpmClocks_t  *table = &(smu10_data->clock_table);
@@ -386,11 +429,11 @@ static int smu10_populate_clock_table(struct pp_hwmgr *hwmgr)
 
 	smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetMinGfxclkFrequency);
 	result = smum_get_argument(hwmgr);
-	smu10_data->gfx_min_freq_limit = result * 100;
+	smu10_data->gfx_min_freq_limit = result / 10 * 1000;
 
 	smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetMaxGfxclkFrequency);
 	result = smum_get_argument(hwmgr);
-	smu10_data->gfx_max_freq_limit = result * 100;
+	smu10_data->gfx_max_freq_limit = result / 10 * 1000;
 
 	return 0;
 }
@@ -436,8 +479,8 @@ static int smu10_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 
 	hwmgr->platform_descriptor.minimumClocksReductionPercentage = 50;
 
-	hwmgr->pstate_sclk = SMU10_UMD_PSTATE_GFXCLK;
-	hwmgr->pstate_mclk = SMU10_UMD_PSTATE_FCLK;
+	hwmgr->pstate_sclk = SMU10_UMD_PSTATE_GFXCLK * 100;
+	hwmgr->pstate_mclk = SMU10_UMD_PSTATE_FCLK * 100;
 
 	return result;
 }
@@ -472,6 +515,8 @@ static int smu10_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
 static int smu10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 				enum amd_dpm_forced_level level)
 {
+	struct smu10_hwmgr *data = hwmgr->backend;
+
 	if (hwmgr->smu_version < 0x1E3700) {
 		pr_info("smu firmware version too old, can not set dpm level\n");
 		return 0;
@@ -482,7 +527,7 @@ static int smu10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 	case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetHardMinGfxClk,
-						SMU10_UMD_PSTATE_PEAK_GFXCLK);
+						data->gfx_max_freq_limit/100);
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetHardMinFclkByFreq,
 						SMU10_UMD_PSTATE_PEAK_FCLK);
@@ -495,7 +540,7 @@ static int smu10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetSoftMaxGfxClk,
-						SMU10_UMD_PSTATE_PEAK_GFXCLK);
+						data->gfx_max_freq_limit/100);
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetSoftMaxFclkByFreq,
 						SMU10_UMD_PSTATE_PEAK_FCLK);
@@ -509,10 +554,10 @@ static int smu10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetHardMinGfxClk,
-						SMU10_UMD_PSTATE_MIN_GFXCLK);
+						data->gfx_min_freq_limit/100);
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetSoftMaxGfxClk,
-						SMU10_UMD_PSTATE_MIN_GFXCLK);
+						data->gfx_min_freq_limit/100);
 		break;
 	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
 		smum_send_msg_to_smc_with_parameter(hwmgr,
@@ -552,10 +597,13 @@ static int smu10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 	case AMD_DPM_FORCED_LEVEL_AUTO:
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetHardMinGfxClk,
-						SMU10_UMD_PSTATE_MIN_GFXCLK);
+						data->gfx_min_freq_limit/100);
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetHardMinFclkByFreq,
+						hwmgr->display_config->num_display > 3 ?
+						SMU10_UMD_PSTATE_PEAK_FCLK :
 						SMU10_UMD_PSTATE_MIN_FCLK);
+
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetHardMinSocclkByFreq,
 						SMU10_UMD_PSTATE_MIN_SOCCLK);
@@ -565,7 +613,7 @@ static int smu10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetSoftMaxGfxClk,
-						SMU10_UMD_PSTATE_PEAK_GFXCLK);
+						data->gfx_max_freq_limit/100);
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetSoftMaxFclkByFreq,
 						SMU10_UMD_PSTATE_PEAK_FCLK);
@@ -579,10 +627,10 @@ static int smu10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 	case AMD_DPM_FORCED_LEVEL_LOW:
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetHardMinGfxClk,
-						SMU10_UMD_PSTATE_MIN_GFXCLK);
+						data->gfx_min_freq_limit/100);
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetSoftMaxGfxClk,
-						SMU10_UMD_PSTATE_MIN_GFXCLK);
+						data->gfx_min_freq_limit/100);
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SetHardMinFclkByFreq,
 						SMU10_UMD_PSTATE_MIN_FCLK);
@@ -699,6 +747,16 @@ static int smu10_set_cpu_power_state(struct pp_hwmgr *hwmgr)
 static int smu10_store_cc6_data(struct pp_hwmgr *hwmgr, uint32_t separation_time,
 			bool cc6_disable, bool pstate_disable, bool pstate_switch_disable)
 {
+	struct smu10_hwmgr *data = (struct smu10_hwmgr *)(hwmgr->backend);
+
+	if (separation_time != data->separation_time ||
+			cc6_disable != data->cc6_disable ||
+			pstate_disable != data->pstate_disable) {
+		data->separation_time = separation_time;
+		data->cc6_disable = cc6_disable;
+		data->pstate_disable = pstate_disable;
+		data->cc6_setting_changed = true;
+	}
 	return 0;
 }
 
@@ -711,6 +769,51 @@ static int smu10_get_dal_power_level(struct pp_hwmgr *hwmgr,
 static int smu10_force_clock_level(struct pp_hwmgr *hwmgr,
 		enum pp_clock_type type, uint32_t mask)
 {
+	struct smu10_hwmgr *data = hwmgr->backend;
+	struct smu10_voltage_dependency_table *mclk_table =
+					data->clock_vol_info.vdd_dep_on_fclk;
+	uint32_t low, high;
+
+	low = mask ? (ffs(mask) - 1) : 0;
+	high = mask ? (fls(mask) - 1) : 0;
+
+	switch (type) {
+	case PP_SCLK:
+		if (low > 2 || high > 2) {
+			pr_info("Currently sclk only support 3 levels on RV\n");
+			return -EINVAL;
+		}
+
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinGfxClk,
+						low == 2 ? data->gfx_max_freq_limit/100 :
+						low == 1 ? SMU10_UMD_PSTATE_GFXCLK :
+						data->gfx_min_freq_limit/100);
+
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxGfxClk,
+						high == 0 ? data->gfx_min_freq_limit/100 :
+						high == 1 ? SMU10_UMD_PSTATE_GFXCLK :
+						data->gfx_max_freq_limit/100);
+		break;
+
+	case PP_MCLK:
+		if (low > mclk_table->count - 1 || high > mclk_table->count - 1)
+			return -EINVAL;
+
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinFclkByFreq,
+						mclk_table->entries[low].clk/100);
+
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxFclkByFreq,
+						mclk_table->entries[high].clk/100);
+		break;
+
+	case PP_PCIE:
+	default:
+		break;
+	}
 	return 0;
 }
 
@@ -720,21 +823,30 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr,
 	struct smu10_hwmgr *data = (struct smu10_hwmgr *)(hwmgr->backend);
 	struct smu10_voltage_dependency_table *mclk_table =
 			data->clock_vol_info.vdd_dep_on_fclk;
-	int i, now, size = 0;
+	uint32_t i, now, size = 0;
 
 	switch (type) {
 	case PP_SCLK:
 		smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetGfxclkFrequency);
 		now = smum_get_argument(hwmgr);
 
+	/* driver only know min/max gfx_clk, Add level 1 for all other gfx clks */
+		if (now == data->gfx_max_freq_limit/100)
+			i = 2;
+		else if (now == data->gfx_min_freq_limit/100)
+			i = 0;
+		else
+			i = 1;
+
 		size += sprintf(buf + size, "0: %uMhz %s\n",
-				data->gfx_min_freq_limit / 100,
-				((data->gfx_min_freq_limit / 100)
-				 == now) ? "*" : "");
+					data->gfx_min_freq_limit/100,
+					i == 0 ? "*" : "");
 		size += sprintf(buf + size, "1: %uMhz %s\n",
-				data->gfx_max_freq_limit / 100,
-				((data->gfx_max_freq_limit / 100)
-				 == now) ? "*" : "");
+					i == 1 ? now : SMU10_UMD_PSTATE_GFXCLK,
+					i == 1 ? "*" : "");
+		size += sprintf(buf + size, "2: %uMhz %s\n",
+					data->gfx_max_freq_limit/100,
+					i == 2 ? "*" : "");
 		break;
 	case PP_MCLK:
 		smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetFclkFrequency);
@@ -947,9 +1059,8 @@ static int smu10_get_max_high_clocks(struct pp_hwmgr *hwmgr, struct amd_pp_simpl
 
 static int smu10_thermal_get_temperature(struct pp_hwmgr *hwmgr)
 {
-	uint32_t reg_offset = soc15_get_register_offset(THM_HWID, 0,
-			mmTHM_TCON_CUR_TMP_BASE_IDX, mmTHM_TCON_CUR_TMP);
-	uint32_t reg_value = cgs_read_register(hwmgr->device, reg_offset);
+	struct amdgpu_device *adev = hwmgr->adev;
+	uint32_t reg_value = RREG32_SOC15(THM, 0, mmTHM_TCON_CUR_TMP);
 	int cur_temp =
 		(reg_value & THM_TCON_CUR_TMP__CUR_TEMP_MASK) >> THM_TCON_CUR_TMP__CUR_TEMP__SHIFT;
 
@@ -993,11 +1104,47 @@ static int smu10_read_sensor(struct pp_hwmgr *hwmgr, int idx,
 	return ret;
 }
 
+static int smu10_set_watermarks_for_clocks_ranges(struct pp_hwmgr *hwmgr,
+		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges)
+{
+	struct smu10_hwmgr *data = hwmgr->backend;
+	Watermarks_t *table = &(data->water_marks_table);
+	int result = 0;
+
+	smu_set_watermarks_for_clocks_ranges(table,wm_with_clock_ranges);
+	smum_smc_table_manager(hwmgr, (uint8_t *)table, (uint16_t)SMU10_WMTABLE, false);
+	data->water_marks_exist = true;
+	return result;
+}
+
+static int smu10_smus_notify_pwe(struct pp_hwmgr *hwmgr)
+{
+
+	return smum_send_msg_to_smc(hwmgr, PPSMC_MSG_SetRccPfcPmeRestoreRegister);
+}
+
 static int smu10_set_mmhub_powergating_by_smu(struct pp_hwmgr *hwmgr)
 {
 	return smum_send_msg_to_smc(hwmgr, PPSMC_MSG_PowerGateMmHub);
 }
 
+static void smu10_powergate_vcn(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	if (bgate) {
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
+						AMD_IP_BLOCK_TYPE_VCN,
+						AMD_PG_STATE_GATE);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+					PPSMC_MSG_PowerDownVcn, 0);
+	} else {
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_PowerUpVcn, 0);
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
+						AMD_IP_BLOCK_TYPE_VCN,
+						AMD_PG_STATE_UNGATE);
+	}
+}
+
 static const struct pp_hwmgr_func smu10_hwmgr_funcs = {
 	.backend_init = smu10_hwmgr_backend_init,
 	.backend_fini = smu10_hwmgr_backend_fini,
@@ -1006,7 +1153,7 @@ static const struct pp_hwmgr_func smu10_hwmgr_funcs = {
 	.force_dpm_level = smu10_dpm_force_dpm_level,
 	.get_power_state_size = smu10_get_power_state_size,
 	.powerdown_uvd = NULL,
-	.powergate_uvd = NULL,
+	.powergate_uvd = smu10_powergate_vcn,
 	.powergate_vce = NULL,
 	.get_mclk = smu10_dpm_get_mclk,
 	.get_sclk = smu10_dpm_get_sclk,
@@ -1022,6 +1169,7 @@ static const struct pp_hwmgr_func smu10_hwmgr_funcs = {
 	.get_current_shallow_sleep_clocks = smu10_get_current_shallow_sleep_clocks,
 	.get_clock_by_type_with_latency = smu10_get_clock_by_type_with_latency,
 	.get_clock_by_type_with_voltage = smu10_get_clock_by_type_with_voltage,
+	.set_watermarks_for_clocks_ranges = smu10_set_watermarks_for_clocks_ranges,
 	.get_max_high_clocks = smu10_get_max_high_clocks,
 	.read_sensor = smu10_read_sensor,
 	.set_active_display_count = smu10_set_active_display_count,
@@ -1032,6 +1180,8 @@ static const struct pp_hwmgr_func smu10_hwmgr_funcs = {
 	.power_state_set = smu10_set_power_state_tasks,
 	.dynamic_state_management_disable = smu10_disable_dpm_tasks,
 	.set_mmhub_powergating_by_smu = smu10_set_mmhub_powergating_by_smu,
+	.smus_notify_pwe = smu10_smus_notify_pwe,
+	.gfx_off_control = smu10_gfx_off_control,
 };
 
 int smu10_init_function_pointers(struct pp_hwmgr *hwmgr)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.h
index 175c3a592b6c..1fb296a996f3 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.h
@@ -290,6 +290,7 @@ struct smu10_hwmgr {
 	bool                           vcn_dpg_mode;
 
 	bool                           gfx_off_controled_by_driver;
+	bool                           water_marks_exist;
 	Watermarks_t                      water_marks_table;
 	struct smu10_clock_voltage_information   clock_vol_info;
 	DpmClocks_t                       clock_table;
@@ -310,11 +311,9 @@ int smu10_init_function_pointers(struct pp_hwmgr *hwmgr);
 #define SMU10_UMD_PSTATE_FCLK                   933
 #define SMU10_UMD_PSTATE_VCE                    0x03C00320
 
-#define SMU10_UMD_PSTATE_PEAK_GFXCLK            1100
 #define SMU10_UMD_PSTATE_PEAK_SOCCLK            757
 #define SMU10_UMD_PSTATE_PEAK_FCLK              1200
 
-#define SMU10_UMD_PSTATE_MIN_GFXCLK             200
 #define SMU10_UMD_PSTATE_MIN_FCLK               400
 #define SMU10_UMD_PSTATE_MIN_SOCCLK             200
 #define SMU10_UMD_PSTATE_MIN_VCE                0x0190012C
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c
index f4cbaee4e2ca..6d72a5600917 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c
@@ -147,20 +147,20 @@ void smu7_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
 	data->uvd_power_gated = bgate;
 
 	if (bgate) {
-		cgs_set_powergating_state(hwmgr->device,
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
 						AMD_IP_BLOCK_TYPE_UVD,
 						AMD_PG_STATE_GATE);
-		cgs_set_clockgating_state(hwmgr->device,
+		amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 				AMD_IP_BLOCK_TYPE_UVD,
 				AMD_CG_STATE_GATE);
 		smu7_update_uvd_dpm(hwmgr, true);
 		smu7_powerdown_uvd(hwmgr);
 	} else {
 		smu7_powerup_uvd(hwmgr);
-		cgs_set_clockgating_state(hwmgr->device,
+		amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 				AMD_IP_BLOCK_TYPE_UVD,
 				AMD_CG_STATE_UNGATE);
-		cgs_set_powergating_state(hwmgr->device,
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
 						AMD_IP_BLOCK_TYPE_UVD,
 						AMD_PG_STATE_UNGATE);
 		smu7_update_uvd_dpm(hwmgr, false);
@@ -175,20 +175,20 @@ void smu7_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
 	data->vce_power_gated = bgate;
 
 	if (bgate) {
-		cgs_set_powergating_state(hwmgr->device,
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
 						AMD_IP_BLOCK_TYPE_VCE,
 						AMD_PG_STATE_GATE);
-		cgs_set_clockgating_state(hwmgr->device,
+		amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 				AMD_IP_BLOCK_TYPE_VCE,
 				AMD_CG_STATE_GATE);
 		smu7_update_vce_dpm(hwmgr, true);
 		smu7_powerdown_vce(hwmgr);
 	} else {
 		smu7_powerup_vce(hwmgr);
-		cgs_set_clockgating_state(hwmgr->device,
+		amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 				AMD_IP_BLOCK_TYPE_VCE,
 				AMD_CG_STATE_UNGATE);
-		cgs_set_powergating_state(hwmgr->device,
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
 						AMD_IP_BLOCK_TYPE_VCE,
 						AMD_PG_STATE_UNGATE);
 		smu7_update_vce_dpm(hwmgr, false);
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
index 18b5b2ff47fe..45e9b8cb169d 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
@@ -61,10 +61,6 @@
 #define SMC_CG_IND_START            0xc0030000
 #define SMC_CG_IND_END              0xc0040000
 
-#define VOLTAGE_SCALE               4
-#define VOLTAGE_VID_OFFSET_SCALE1   625
-#define VOLTAGE_VID_OFFSET_SCALE2   100
-
 #define MEM_FREQ_LOW_LATENCY        25000
 #define MEM_FREQ_HIGH_LATENCY       80000
 
@@ -88,6 +84,14 @@ static const struct profile_mode_setting smu7_profiling[6] =
 					 {0, 0, 0, 0, 0, 0, 0, 0},
 					};
 
+#define PPSMC_MSG_SetVBITimeout_VEGAM    ((uint16_t) 0x310)
+
+#define ixPWR_SVI2_PLANE1_LOAD                     0xC0200280
+#define PWR_SVI2_PLANE1_LOAD__PSI1_MASK                    0x00000020L
+#define PWR_SVI2_PLANE1_LOAD__PSI0_EN_MASK                 0x00000040L
+#define PWR_SVI2_PLANE1_LOAD__PSI1__SHIFT                  0x00000005
+#define PWR_SVI2_PLANE1_LOAD__PSI0_EN__SHIFT               0x00000006
+
 /** Values for the CG_THERMAL_CTRL::DPM_EVENT_SRC field. */
 enum DPM_EVENT_SRC {
 	DPM_EVENT_SRC_ANALOG = 0,
@@ -169,6 +173,13 @@ static int smu7_get_current_pcie_lane_number(struct pp_hwmgr *hwmgr)
 */
 static int smu7_enable_smc_voltage_controller(struct pp_hwmgr *hwmgr)
 {
+	if (hwmgr->chip_id == CHIP_VEGAM) {
+		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device,
+				CGS_IND_REG__SMC, PWR_SVI2_PLANE1_LOAD, PSI1, 0);
+		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device,
+				CGS_IND_REG__SMC, PWR_SVI2_PLANE1_LOAD, PSI0_EN, 0);
+	}
+
 	if (hwmgr->feature_mask & PP_SMC_VOLTAGE_CONTROL_MASK)
 		smum_send_msg_to_smc(hwmgr, PPSMC_MSG_Voltage_Cntl_Enable);
 
@@ -798,32 +809,6 @@ static int smu7_setup_dpm_tables_v1(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-static int smu7_get_voltage_dependency_table(
-			const struct phm_ppt_v1_clock_voltage_dependency_table *allowed_dep_table,
-			struct phm_ppt_v1_clock_voltage_dependency_table *dep_table)
-{
-	uint8_t i = 0;
-	PP_ASSERT_WITH_CODE((0 != allowed_dep_table->count),
-				"Voltage Lookup Table empty",
-				return -EINVAL);
-
-	dep_table->count = allowed_dep_table->count;
-	for (i=0; i<dep_table->count; i++) {
-		dep_table->entries[i].clk = allowed_dep_table->entries[i].clk;
-		dep_table->entries[i].vddInd = allowed_dep_table->entries[i].vddInd;
-		dep_table->entries[i].vdd_offset = allowed_dep_table->entries[i].vdd_offset;
-		dep_table->entries[i].vddc = allowed_dep_table->entries[i].vddc;
-		dep_table->entries[i].vddgfx = allowed_dep_table->entries[i].vddgfx;
-		dep_table->entries[i].vddci = allowed_dep_table->entries[i].vddci;
-		dep_table->entries[i].mvdd = allowed_dep_table->entries[i].mvdd;
-		dep_table->entries[i].phases = allowed_dep_table->entries[i].phases;
-		dep_table->entries[i].cks_enable = allowed_dep_table->entries[i].cks_enable;
-		dep_table->entries[i].cks_voffset = allowed_dep_table->entries[i].cks_voffset;
-	}
-
-	return 0;
-}
-
 static int smu7_odn_initial_default_setting(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
@@ -851,7 +836,7 @@ static int smu7_odn_initial_default_setting(struct pp_hwmgr *hwmgr)
 		entries[i].vddc = dep_sclk_table->entries[i].vddc;
 	}
 
-	smu7_get_voltage_dependency_table(dep_sclk_table,
+	smu_get_voltage_dependency_table_ppt_v1(dep_sclk_table,
 		(struct phm_ppt_v1_clock_voltage_dependency_table *)&(odn_table->vdd_dependency_on_sclk));
 
 	odn_table->odn_memory_clock_dpm_levels.num_of_pl =
@@ -863,12 +848,40 @@ static int smu7_odn_initial_default_setting(struct pp_hwmgr *hwmgr)
 		entries[i].vddc = dep_mclk_table->entries[i].vddc;
 	}
 
-	smu7_get_voltage_dependency_table(dep_mclk_table,
+	smu_get_voltage_dependency_table_ppt_v1(dep_mclk_table,
 		(struct phm_ppt_v1_clock_voltage_dependency_table *)&(odn_table->vdd_dependency_on_mclk));
 
 	return 0;
 }
 
+static void smu7_setup_voltage_range_from_vbios(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_sclk_table;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint32_t min_vddc = 0;
+	uint32_t max_vddc = 0;
+
+	if (!table_info)
+		return;
+
+	dep_sclk_table = table_info->vdd_dep_on_sclk;
+
+	atomctrl_get_voltage_range(hwmgr, &max_vddc, &min_vddc);
+
+	if (min_vddc == 0 || min_vddc > 2000
+		|| min_vddc > dep_sclk_table->entries[0].vddc)
+		min_vddc = dep_sclk_table->entries[0].vddc;
+
+	if (max_vddc == 0 || max_vddc > 2000
+		|| max_vddc < dep_sclk_table->entries[dep_sclk_table->count-1].vddc)
+		max_vddc = dep_sclk_table->entries[dep_sclk_table->count-1].vddc;
+
+	data->odn_dpm_table.min_vddc = min_vddc;
+	data->odn_dpm_table.max_vddc = max_vddc;
+}
+
 static int smu7_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
@@ -887,8 +900,10 @@ static int smu7_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 			sizeof(struct smu7_dpm_table));
 
 	/* initialize ODN table */
-	if (hwmgr->od_enabled)
+	if (hwmgr->od_enabled) {
+		smu7_setup_voltage_range_from_vbios(hwmgr);
 		smu7_odn_initial_default_setting(hwmgr);
+	}
 
 	return 0;
 }
@@ -966,6 +981,22 @@ static int smu7_disable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
+static int smu7_disable_sclk_vce_handshake(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t soft_register_value = 0;
+	uint32_t handshake_disables_offset = data->soft_regs_start
+				+ smum_get_offsetof(hwmgr,
+					SMU_SoftRegisters, HandshakeDisables);
+
+	soft_register_value = cgs_read_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC, handshake_disables_offset);
+	soft_register_value |= SMU7_VCE_SCLK_HANDSHAKE_DISABLE;
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			handshake_disables_offset, soft_register_value);
+	return 0;
+}
+
 static int smu7_disable_handshake_uvd(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
@@ -988,23 +1019,29 @@ static int smu7_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
 	/* enable SCLK dpm */
-	if (!data->sclk_dpm_key_disabled)
+	if (!data->sclk_dpm_key_disabled) {
+		if (hwmgr->chip_id == CHIP_VEGAM)
+			smu7_disable_sclk_vce_handshake(hwmgr);
+
 		PP_ASSERT_WITH_CODE(
 		(0 == smum_send_msg_to_smc(hwmgr, PPSMC_MSG_DPM_Enable)),
 		"Failed to enable SCLK DPM during DPM Start Function!",
 		return -EINVAL);
+	}
 
 	/* enable MCLK dpm */
 	if (0 == data->mclk_dpm_key_disabled) {
 		if (!(hwmgr->feature_mask & PP_UVD_HANDSHAKE_MASK))
 			smu7_disable_handshake_uvd(hwmgr);
+
 		PP_ASSERT_WITH_CODE(
 				(0 == smum_send_msg_to_smc(hwmgr,
 						PPSMC_MSG_MCLKDPM_Enable)),
 				"Failed to enable MCLK DPM during DPM Start Function!",
 				return -EINVAL);
 
-		PHM_WRITE_FIELD(hwmgr->device, MC_SEQ_CNTL_3, CAC_EN, 0x1);
+		if (hwmgr->chip_family != CHIP_VEGAM)
+			PHM_WRITE_FIELD(hwmgr->device, MC_SEQ_CNTL_3, CAC_EN, 0x1);
 
 
 		if (hwmgr->chip_family == AMDGPU_FAMILY_CI) {
@@ -1020,8 +1057,13 @@ static int smu7_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
 			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC1_CNTL, 0x5);
 			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_CPL_CNTL, 0x100005);
 			udelay(10);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC0_CNTL, 0x400005);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC1_CNTL, 0x400005);
+			if (hwmgr->chip_id == CHIP_VEGAM) {
+				cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC0_CNTL, 0x400009);
+				cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC1_CNTL, 0x400009);
+			} else {
+				cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC0_CNTL, 0x400005);
+				cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC1_CNTL, 0x400005);
+			}
 			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_CPL_CNTL, 0x500005);
 		}
 	}
@@ -1230,7 +1272,7 @@ static int smu7_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 
 		tmp_result = smu7_construct_voltage_tables(hwmgr);
 		PP_ASSERT_WITH_CODE((0 == tmp_result),
-				"Failed to contruct voltage tables!",
+				"Failed to construct voltage tables!",
 				result = tmp_result);
 	}
 	smum_initialize_mc_reg_table(hwmgr);
@@ -1262,10 +1304,12 @@ static int smu7_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 	PP_ASSERT_WITH_CODE((0 == tmp_result),
 			"Failed to process firmware header!", result = tmp_result);
 
-	tmp_result = smu7_initial_switch_from_arbf0_to_f1(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to initialize switch from ArbF0 to F1!",
-			result = tmp_result);
+	if (hwmgr->chip_id != CHIP_VEGAM) {
+		tmp_result = smu7_initial_switch_from_arbf0_to_f1(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == tmp_result),
+				"Failed to initialize switch from ArbF0 to F1!",
+				result = tmp_result);
+	}
 
 	result = smu7_setup_default_dpm_tables(hwmgr);
 	PP_ASSERT_WITH_CODE(0 == result,
@@ -2755,6 +2799,9 @@ static int smu7_vblank_too_short(struct pp_hwmgr *hwmgr,
 	case CHIP_POLARIS12:
 		switch_limit_us = data->is_memory_gddr5 ? 190 : 150;
 		break;
+	case CHIP_VEGAM:
+		switch_limit_us = 30;
+		break;
 	default:
 		switch_limit_us = data->is_memory_gddr5 ? 450 : 150;
 		break;
@@ -2778,8 +2825,6 @@ static int smu7_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 	struct PP_Clocks minimum_clocks = {0};
 	bool disable_mclk_switching;
 	bool disable_mclk_switching_for_frame_lock;
-	struct cgs_display_info info = {0};
-	struct cgs_mode_info mode_info = {0};
 	const struct phm_clock_and_voltage_limits *max_limits;
 	uint32_t i;
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
@@ -2788,7 +2833,6 @@ static int smu7_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 	int32_t count;
 	int32_t stable_pstate_sclk = 0, stable_pstate_mclk = 0;
 
-	info.mode_info = &mode_info;
 	data->battery_state = (PP_StateUILabel_Battery ==
 			request_ps->classification.ui_label);
 
@@ -2810,10 +2854,8 @@ static int smu7_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 		}
 	}
 
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	minimum_clocks.engineClock = hwmgr->display_config.min_core_set_clock;
-	minimum_clocks.memoryClock = hwmgr->display_config.min_mem_set_clock;
+	minimum_clocks.engineClock = hwmgr->display_config->min_core_set_clock;
+	minimum_clocks.memoryClock = hwmgr->display_config->min_mem_set_clock;
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_StablePState)) {
@@ -2844,12 +2886,12 @@ static int smu7_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 				    PHM_PlatformCaps_DisableMclkSwitchingForFrameLock);
 
 
-	if (info.display_count == 0)
+	if (hwmgr->display_config->num_display == 0)
 		disable_mclk_switching = false;
 	else
-		disable_mclk_switching = ((1 < info.display_count) ||
+		disable_mclk_switching = ((1 < hwmgr->display_config->num_display) ||
 					  disable_mclk_switching_for_frame_lock ||
-					  smu7_vblank_too_short(hwmgr, mode_info.vblank_time_us));
+					  smu7_vblank_too_short(hwmgr, hwmgr->display_config->min_vblank_time));
 
 	sclk = smu7_ps->performance_levels[0].engine_clock;
 	mclk = smu7_ps->performance_levels[0].memory_clock;
@@ -2958,8 +3000,7 @@ static int smu7_dpm_patch_boot_state(struct pp_hwmgr *hwmgr,
 	/* First retrieve the Boot clocks and VDDC from the firmware info table.
 	 * We assume here that fw_info is unchanged if this call fails.
 	 */
-	fw_info = (ATOM_FIRMWARE_INFO_V2_2 *)cgs_atom_get_data_table(
-			hwmgr->device, index,
+	fw_info = (ATOM_FIRMWARE_INFO_V2_2 *)smu_atom_get_data_table(hwmgr->adev, index,
 			&size, &frev, &crev);
 	if (!fw_info)
 		/* During a test, there is no firmware info table. */
@@ -3367,34 +3408,35 @@ static int smu7_get_pp_table_entry(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
-static int smu7_get_gpu_power(struct pp_hwmgr *hwmgr,
-		struct pp_gpu_power *query)
+static int smu7_get_gpu_power(struct pp_hwmgr *hwmgr, u32 *query)
 {
-	PP_ASSERT_WITH_CODE(!smum_send_msg_to_smc(hwmgr,
-			PPSMC_MSG_PmStatusLogStart),
-			"Failed to start pm status log!",
-			return -1);
+	int i;
+	u32 tmp = 0;
+
+	if (!query)
+		return -EINVAL;
 
-	/* Sampling period from 50ms to 4sec */
-	msleep_interruptible(200);
+	smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_GetCurrPkgPwr, 0);
+	tmp = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
+	*query = tmp;
 
-	PP_ASSERT_WITH_CODE(!smum_send_msg_to_smc(hwmgr,
-			PPSMC_MSG_PmStatusLogSample),
-			"Failed to sample pm status log!",
-			return -1);
+	if (tmp != 0)
+		return 0;
 
-	query->vddc_power = cgs_read_ind_register(hwmgr->device,
-			CGS_IND_REG__SMC,
-			ixSMU_PM_STATUS_40);
-	query->vddci_power = cgs_read_ind_register(hwmgr->device,
-			CGS_IND_REG__SMC,
-			ixSMU_PM_STATUS_49);
-	query->max_gpu_power = cgs_read_ind_register(hwmgr->device,
-			CGS_IND_REG__SMC,
-			ixSMU_PM_STATUS_94);
-	query->average_gpu_power = cgs_read_ind_register(hwmgr->device,
-			CGS_IND_REG__SMC,
-			ixSMU_PM_STATUS_95);
+	smum_send_msg_to_smc(hwmgr, PPSMC_MSG_PmStatusLogStart);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+							ixSMU_PM_STATUS_94, 0);
+
+	for (i = 0; i < 10; i++) {
+		mdelay(1);
+		smum_send_msg_to_smc(hwmgr, PPSMC_MSG_PmStatusLogSample);
+		tmp = cgs_read_ind_register(hwmgr->device,
+						CGS_IND_REG__SMC,
+						ixSMU_PM_STATUS_94);
+		if (tmp != 0)
+			break;
+	}
+	*query = tmp;
 
 	return 0;
 }
@@ -3447,10 +3489,7 @@ static int smu7_read_sensor(struct pp_hwmgr *hwmgr, int idx,
 		*size = 4;
 		return 0;
 	case AMDGPU_PP_SENSOR_GPU_POWER:
-		if (*size < sizeof(struct pp_gpu_power))
-			return -EINVAL;
-		*size = sizeof(struct pp_gpu_power);
-		return smu7_get_gpu_power(hwmgr, (struct pp_gpu_power *)value);
+		return smu7_get_gpu_power(hwmgr, (uint32_t *)value);
 	case AMDGPU_PP_SENSOR_VDDGFX:
 		if ((data->vr_config & 0xff) == 0x2)
 			val_vid = PHM_READ_INDIRECT_FIELD(hwmgr->device,
@@ -3481,7 +3520,6 @@ static int smu7_find_dpm_states_clocks_in_dpm_table(struct pp_hwmgr *hwmgr, cons
 			[smu7_ps->performance_level_count - 1].memory_clock;
 	struct PP_Clocks min_clocks = {0};
 	uint32_t i;
-	struct cgs_display_info info = {0};
 
 	for (i = 0; i < sclk_table->count; i++) {
 		if (sclk == sclk_table->dpm_levels[i].value)
@@ -3508,9 +3546,8 @@ static int smu7_find_dpm_states_clocks_in_dpm_table(struct pp_hwmgr *hwmgr, cons
 	if (i >= mclk_table->count)
 		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
 
-	cgs_get_active_displays_info(hwmgr->device, &info);
 
-	if (data->display_timing.num_existing_displays != info.display_count)
+	if (data->display_timing.num_existing_displays != hwmgr->display_config->num_display)
 		data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK;
 
 	return 0;
@@ -3813,9 +3850,14 @@ static int smu7_notify_smc_display(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
-	if (hwmgr->feature_mask & PP_VBI_TIME_SUPPORT_MASK)
-		smum_send_msg_to_smc_with_parameter(hwmgr,
-			(PPSMC_Msg)PPSMC_MSG_SetVBITimeout, data->frame_time_x2);
+	if (hwmgr->feature_mask & PP_VBI_TIME_SUPPORT_MASK) {
+		if (hwmgr->chip_id == CHIP_VEGAM)
+			smum_send_msg_to_smc_with_parameter(hwmgr,
+					(PPSMC_Msg)PPSMC_MSG_SetVBITimeout_VEGAM, data->frame_time_x2);
+		else
+			smum_send_msg_to_smc_with_parameter(hwmgr,
+					(PPSMC_Msg)PPSMC_MSG_SetVBITimeout, data->frame_time_x2);
+	}
 	return (smum_send_msg_to_smc(hwmgr, (PPSMC_Msg)PPSMC_HasDisplay) == 0) ?  0 : -EINVAL;
 }
 
@@ -3909,15 +3951,8 @@ smu7_notify_smc_display_change(struct pp_hwmgr *hwmgr, bool has_display)
 static int
 smu7_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr)
 {
-	uint32_t num_active_displays = 0;
-	struct cgs_display_info info = {0};
-
-	info.mode_info = NULL;
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	num_active_displays = info.display_count;
-
-	if (num_active_displays > 1 && hwmgr->display_config.multi_monitor_in_sync != true)
+	if (hwmgr->display_config->num_display > 1 &&
+			!hwmgr->display_config->multi_monitor_in_sync)
 		smu7_notify_smc_display_change(hwmgr, false);
 
 	return 0;
@@ -3932,33 +3967,24 @@ smu7_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr)
 static int smu7_program_display_gap(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
-	uint32_t num_active_displays = 0;
 	uint32_t display_gap = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL);
 	uint32_t display_gap2;
 	uint32_t pre_vbi_time_in_us;
 	uint32_t frame_time_in_us;
-	uint32_t ref_clock;
-	uint32_t refresh_rate = 0;
-	struct cgs_display_info info = {0};
-	struct cgs_mode_info mode_info = {0};
+	uint32_t ref_clock, refresh_rate;
 
-	info.mode_info = &mode_info;
-	cgs_get_active_displays_info(hwmgr->device, &info);
-	num_active_displays = info.display_count;
-
-	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL, DISP_GAP, (num_active_displays > 0) ? DISPLAY_GAP_VBLANK_OR_WM : DISPLAY_GAP_IGNORE);
+	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL, DISP_GAP, (hwmgr->display_config->num_display > 0) ? DISPLAY_GAP_VBLANK_OR_WM : DISPLAY_GAP_IGNORE);
 	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL, display_gap);
 
 	ref_clock =  amdgpu_asic_get_xclk((struct amdgpu_device *)hwmgr->adev);
-
-	refresh_rate = mode_info.refresh_rate;
+	refresh_rate = hwmgr->display_config->vrefresh;
 
 	if (0 == refresh_rate)
 		refresh_rate = 60;
 
 	frame_time_in_us = 1000000 / refresh_rate;
 
-	pre_vbi_time_in_us = frame_time_in_us - 200 - mode_info.vblank_time_us;
+	pre_vbi_time_in_us = frame_time_in_us - 200 - hwmgr->display_config->min_vblank_time;
 
 	data->frame_time_x2 = frame_time_in_us * 2 / 100;
 
@@ -4038,17 +4064,14 @@ smu7_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	bool is_update_required = false;
-	struct cgs_display_info info = {0, 0, NULL};
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
 
-	if (data->display_timing.num_existing_displays != info.display_count)
+	if (data->display_timing.num_existing_displays != hwmgr->display_config->num_display)
 		is_update_required = true;
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep)) {
-		if (data->display_timing.min_clock_in_sr != hwmgr->display_config.min_core_set_clock_in_sr &&
+		if (data->display_timing.min_clock_in_sr != hwmgr->display_config->min_core_set_clock_in_sr &&
 			(data->display_timing.min_clock_in_sr >= SMU7_MINIMUM_ENGINE_CLOCK ||
-			hwmgr->display_config.min_core_set_clock_in_sr >= SMU7_MINIMUM_ENGINE_CLOCK))
+			hwmgr->display_config->min_core_set_clock_in_sr >= SMU7_MINIMUM_ENGINE_CLOCK))
 			is_update_required = true;
 	}
 	return is_update_required;
@@ -4103,7 +4126,7 @@ static int smu7_check_states_equal(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
-static int smu7_upload_mc_firmware(struct pp_hwmgr *hwmgr)
+static int smu7_check_mc_firmware(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
@@ -4182,13 +4205,9 @@ static int smu7_read_clock_registers(struct pp_hwmgr *hwmgr)
 static int smu7_get_memory_type(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
-	uint32_t temp;
-
-	temp = cgs_read_register(hwmgr->device, mmMC_SEQ_MISC0);
+	struct amdgpu_device *adev = hwmgr->adev;
 
-	data->is_memory_gddr5 = (MC_SEQ_MISC0_GDDR5_VALUE ==
-			((temp & MC_SEQ_MISC0_GDDR5_MASK) >>
-			 MC_SEQ_MISC0_GDDR5_SHIFT));
+	data->is_memory_gddr5 = (adev->gmc.vram_type == AMDGPU_VRAM_TYPE_GDDR5);
 
 	return 0;
 }
@@ -4236,7 +4255,7 @@ static int smu7_setup_asic_task(struct pp_hwmgr *hwmgr)
 {
 	int tmp_result, result = 0;
 
-	smu7_upload_mc_firmware(hwmgr);
+	smu7_check_mc_firmware(hwmgr);
 
 	tmp_result = smu7_read_clock_registers(hwmgr);
 	PP_ASSERT_WITH_CODE((0 == tmp_result),
@@ -4371,22 +4390,36 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
 		break;
 	case OD_SCLK:
 		if (hwmgr->od_enabled) {
-			size = sprintf(buf, "%s: \n", "OD_SCLK");
+			size = sprintf(buf, "%s:\n", "OD_SCLK");
 			for (i = 0; i < odn_sclk_table->num_of_pl; i++)
-				size += sprintf(buf + size, "%d: %10uMhz %10u mV\n",
-					i, odn_sclk_table->entries[i].clock / 100,
+				size += sprintf(buf + size, "%d: %10uMHz %10umV\n",
+					i, odn_sclk_table->entries[i].clock/100,
 					odn_sclk_table->entries[i].vddc);
 		}
 		break;
 	case OD_MCLK:
 		if (hwmgr->od_enabled) {
-			size = sprintf(buf, "%s: \n", "OD_MCLK");
+			size = sprintf(buf, "%s:\n", "OD_MCLK");
 			for (i = 0; i < odn_mclk_table->num_of_pl; i++)
-				size += sprintf(buf + size, "%d: %10uMhz %10u mV\n",
-					i, odn_mclk_table->entries[i].clock / 100,
+				size += sprintf(buf + size, "%d: %10uMHz %10umV\n",
+					i, odn_mclk_table->entries[i].clock/100,
 					odn_mclk_table->entries[i].vddc);
 		}
 		break;
+	case OD_RANGE:
+		if (hwmgr->od_enabled) {
+			size = sprintf(buf, "%s:\n", "OD_RANGE");
+			size += sprintf(buf + size, "SCLK: %7uMHz %10uMHz\n",
+				data->golden_dpm_table.sclk_table.dpm_levels[0].value/100,
+				hwmgr->platform_descriptor.overdriveLimit.engineClock/100);
+			size += sprintf(buf + size, "MCLK: %7uMHz %10uMHz\n",
+				data->golden_dpm_table.mclk_table.dpm_levels[0].value/100,
+				hwmgr->platform_descriptor.overdriveLimit.memoryClock/100);
+			size += sprintf(buf + size, "VDDC: %7umV %11umV\n",
+				data->odn_dpm_table.min_vddc,
+				data->odn_dpm_table.max_vddc);
+		}
+		break;
 	default:
 		break;
 	}
@@ -4670,36 +4703,27 @@ static bool smu7_check_clk_voltage_valid(struct pp_hwmgr *hwmgr,
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint32_t min_vddc;
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_sclk_table;
-
-	if (table_info == NULL)
-		return false;
-
-	dep_sclk_table = table_info->vdd_dep_on_sclk;
-	min_vddc = dep_sclk_table->entries[0].vddc;
-
-	if (voltage < min_vddc || voltage > 2000) {
-		pr_info("OD voltage is out of range [%d - 2000] mV\n", min_vddc);
+	if (voltage < data->odn_dpm_table.min_vddc || voltage > data->odn_dpm_table.max_vddc) {
+		pr_info("OD voltage is out of range [%d - %d] mV\n",
+						data->odn_dpm_table.min_vddc,
+						data->odn_dpm_table.max_vddc);
 		return false;
 	}
 
 	if (type == PP_OD_EDIT_SCLK_VDDC_TABLE) {
-		if (data->vbios_boot_state.sclk_bootup_value > clk ||
+		if (data->golden_dpm_table.sclk_table.dpm_levels[0].value > clk ||
 			hwmgr->platform_descriptor.overdriveLimit.engineClock < clk) {
 			pr_info("OD engine clock is out of range [%d - %d] MHz\n",
-				data->vbios_boot_state.sclk_bootup_value,
-				hwmgr->platform_descriptor.overdriveLimit.engineClock / 100);
+				data->golden_dpm_table.sclk_table.dpm_levels[0].value/100,
+				hwmgr->platform_descriptor.overdriveLimit.engineClock/100);
 			return false;
 		}
 	} else if (type == PP_OD_EDIT_MCLK_VDDC_TABLE) {
-		if (data->vbios_boot_state.mclk_bootup_value > clk ||
+		if (data->golden_dpm_table.mclk_table.dpm_levels[0].value > clk ||
 			hwmgr->platform_descriptor.overdriveLimit.memoryClock < clk) {
 			pr_info("OD memory clock is out of range [%d - %d] MHz\n",
-				data->vbios_boot_state.mclk_bootup_value/100,
-				hwmgr->platform_descriptor.overdriveLimit.memoryClock / 100);
+				data->golden_dpm_table.mclk_table.dpm_levels[0].value/100,
+				hwmgr->platform_descriptor.overdriveLimit.memoryClock/100);
 			return false;
 		}
 	} else {
@@ -4748,10 +4772,6 @@ static void smu7_check_dpm_table_updated(struct pp_hwmgr *hwmgr)
 			return;
 		}
 	}
-	if (i == dep_table->count && data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_VDDC) {
-		data->need_update_smu7_dpm_table &= ~DPMTABLE_OD_UPDATE_VDDC;
-		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
-	}
 
 	dep_table = table_info->vdd_dep_on_sclk;
 	odn_dep_table = (struct phm_ppt_v1_clock_voltage_dependency_table *)&(odn_table->vdd_dependency_on_sclk);
@@ -4761,9 +4781,9 @@ static void smu7_check_dpm_table_updated(struct pp_hwmgr *hwmgr)
 			return;
 		}
 	}
-	if (i == dep_table->count && data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_VDDC) {
+	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_VDDC) {
 		data->need_update_smu7_dpm_table &= ~DPMTABLE_OD_UPDATE_VDDC;
-		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
+		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_OD_UPDATE_MCLK;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.h
index b8d0bb378595..c91e75db6a8e 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.h
@@ -184,6 +184,8 @@ struct smu7_odn_dpm_table {
 	struct smu7_odn_clock_voltage_dependency_table	vdd_dependency_on_sclk;
 	struct smu7_odn_clock_voltage_dependency_table	vdd_dependency_on_mclk;
 	uint32_t					odn_mclk_min_limit;
+	uint32_t min_vddc;
+	uint32_t max_vddc;
 };
 
 struct profile_mode_setting {
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
index d9e92e306535..c952845833d7 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
@@ -623,6 +623,190 @@ static const struct gpu_pt_config_reg DIDTConfig_Polaris11_Kicker[] =
 	{   0xFFFFFFFF  }  /* End of list */
 };
 
+static const struct gpu_pt_config_reg GCCACConfig_VegaM[] =
+{
+// ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+//      Offset                             Mask                                                Shift                                               Value       Type
+// ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+    // DIDT_SQ
+    //
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00060013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00860013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01060013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01860013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02060013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02860013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03060013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03860013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x04060013, GPU_CONFIGREG_GC_CAC_IND },
+
+    // DIDT_TD
+    //
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x000E0013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x008E0013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x010E0013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x018E0013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x020E0013, GPU_CONFIGREG_GC_CAC_IND },
+
+    // DIDT_TCP
+    //
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00100013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00900013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01100013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01900013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02100013, GPU_CONFIGREG_GC_CAC_IND },
+    {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02900013, GPU_CONFIGREG_GC_CAC_IND },
+
+    {   0xFFFFFFFF  }  // End of list
+};
+
+static const struct gpu_pt_config_reg DIDTConfig_VegaM[] =
+{
+// ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+//      Offset                             Mask                                                Shift                                               Value       Type
+// ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+    // DIDT_SQ
+    //
+    {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT,                  0x0073,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT,                  0x00ab,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0084,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT,                  0x005a,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0067,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT,                  0x0084,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0027,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0046,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT,                 0x00aa,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MIN_POWER_MASK,                      DIDT_SQ_CTRL1__MIN_POWER__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MAX_POWER_MASK,                      DIDT_SQ_CTRL1__MAX_POWER__SHIFT,                    0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__UNUSED_0_MASK,                    DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3853,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_0_MASK,                       DIDT_SQ_CTRL2__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x005a,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_1_MASK,                       DIDT_SQ_CTRL2__UNUSED_1__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_2_MASK,                       DIDT_SQ_CTRL2__UNUSED_2__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x0ebb,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__UNUSED_0_MASK,                  DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x3853,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x3153,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__PHASE_OFFSET_MASK,                   DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__UNUSED_0_MASK,                       DIDT_SQ_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    // DIDT_TD
+    //
+    {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT,                  0x000a,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT,                  0x0010,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0017,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT,                  0x002f,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0046,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT,                  0x005d,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MIN_POWER_MASK,                      DIDT_TD_CTRL1__MIN_POWER__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MAX_POWER_MASK,                      DIDT_TD_CTRL1__MAX_POWER__SHIFT,                    0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__UNUSED_0_MASK,                    DIDT_TD_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0x00ff,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3fff,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_0_MASK,                       DIDT_TD_CTRL2__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x000f,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_1_MASK,                       DIDT_TD_CTRL2__UNUSED_1__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_2_MASK,                       DIDT_TD_CTRL2__UNUSED_2__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__UNUSED_0_MASK,                  DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x0dde,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x0dde,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0009,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0009,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__UNUSED_0_MASK,                       DIDT_TD_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    // DIDT_TCP
+    //
+    {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT,                 0x0004,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT,                 0x0037,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT,                 0x00ff,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT,                 0x0054,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MIN_POWER_MASK,                     DIDT_TCP_CTRL1__MIN_POWER__SHIFT,                   0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MAX_POWER_MASK,                     DIDT_TCP_CTRL1__MAX_POWER__SHIFT,                   0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__UNUSED_0_MASK,                   DIDT_TCP_CTRL_OCP__UNUSED_0__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__OCP_MAX_POWER_MASK,              DIDT_TCP_CTRL_OCP__OCP_MAX_POWER__SHIFT,            0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK,               DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT,             0x3dde,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_0_MASK,                      DIDT_TCP_CTRL2__UNUSED_0__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,      DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,    0x0032,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_1_MASK,                      DIDT_TCP_CTRL2__UNUSED_1__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,      DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,    0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_2_MASK,                      DIDT_TCP_CTRL2__UNUSED_2__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,   DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,  DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT,0x01aa,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__UNUSED_0_MASK,                 DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,      DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,    0x3dde,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,    0x3dde,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK,                DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT,              0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+    {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__UNUSED_0_MASK,                       DIDT_TCP_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+    {   0xFFFFFFFF  }  // End of list
+};
 static int smu7_enable_didt(struct pp_hwmgr *hwmgr, const bool enable)
 {
 	uint32_t en = enable ? 1 : 0;
@@ -740,8 +924,8 @@ int smu7_enable_didt_config(struct pp_hwmgr *hwmgr)
 	    PP_CAP(PHM_PlatformCaps_TDRamping) ||
 	    PP_CAP(PHM_PlatformCaps_TCPRamping)) {
 
-		cgs_enter_safe_mode(hwmgr->device, true);
-		cgs_lock_grbm_idx(hwmgr->device, true);
+		adev->gfx.rlc.funcs->enter_safe_mode(adev);
+		mutex_lock(&adev->grbm_idx_mutex);
 		value = 0;
 		value2 = cgs_read_register(hwmgr->device, mmGRBM_GFX_INDEX);
 		for (count = 0; count < num_se; count++) {
@@ -752,67 +936,80 @@ int smu7_enable_didt_config(struct pp_hwmgr *hwmgr)
 
 			if (hwmgr->chip_id == CHIP_POLARIS10) {
 				result = smu7_program_pt_config_registers(hwmgr, GCCACConfig_Polaris10);
-				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", goto error);
 				result = smu7_program_pt_config_registers(hwmgr, DIDTConfig_Polaris10);
-				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", goto error);
 			} else if (hwmgr->chip_id == CHIP_POLARIS11) {
 				result = smu7_program_pt_config_registers(hwmgr, GCCACConfig_Polaris11);
-				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", goto error);
 				if (hwmgr->is_kicker)
 					result = smu7_program_pt_config_registers(hwmgr, DIDTConfig_Polaris11_Kicker);
 				else
 					result = smu7_program_pt_config_registers(hwmgr, DIDTConfig_Polaris11);
-				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", goto error);
 			} else if (hwmgr->chip_id == CHIP_POLARIS12) {
 				result = smu7_program_pt_config_registers(hwmgr, GCCACConfig_Polaris11);
-				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", goto error);
 				result = smu7_program_pt_config_registers(hwmgr, DIDTConfig_Polaris12);
-				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", goto error);
+			} else if (hwmgr->chip_id == CHIP_VEGAM) {
+				result = smu7_program_pt_config_registers(hwmgr, GCCACConfig_VegaM);
+				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", goto error);
+				result = smu7_program_pt_config_registers(hwmgr, DIDTConfig_VegaM);
+				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", goto error);
 			}
 		}
 		cgs_write_register(hwmgr->device, mmGRBM_GFX_INDEX, value2);
 
 		result = smu7_enable_didt(hwmgr, true);
-		PP_ASSERT_WITH_CODE((result == 0), "EnableDiDt failed.", return result);
+		PP_ASSERT_WITH_CODE((result == 0), "EnableDiDt failed.", goto error);
 
 		if (hwmgr->chip_id == CHIP_POLARIS11) {
 			result = smum_send_msg_to_smc(hwmgr,
 						(uint16_t)(PPSMC_MSG_EnableDpmDidt));
 			PP_ASSERT_WITH_CODE((0 == result),
-					"Failed to enable DPM DIDT.", return result);
+					"Failed to enable DPM DIDT.", goto error);
 		}
-		cgs_lock_grbm_idx(hwmgr->device, false);
-		cgs_enter_safe_mode(hwmgr->device, false);
+		mutex_unlock(&adev->grbm_idx_mutex);
+		adev->gfx.rlc.funcs->exit_safe_mode(adev);
 	}
 
 	return 0;
+error:
+	mutex_unlock(&adev->grbm_idx_mutex);
+	adev->gfx.rlc.funcs->exit_safe_mode(adev);
+	return result;
 }
 
 int smu7_disable_didt_config(struct pp_hwmgr *hwmgr)
 {
 	int result;
+	struct amdgpu_device *adev = hwmgr->adev;
 
 	if (PP_CAP(PHM_PlatformCaps_SQRamping) ||
 	    PP_CAP(PHM_PlatformCaps_DBRamping) ||
 	    PP_CAP(PHM_PlatformCaps_TDRamping) ||
 	    PP_CAP(PHM_PlatformCaps_TCPRamping)) {
 
-		cgs_enter_safe_mode(hwmgr->device, true);
+		adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
 		result = smu7_enable_didt(hwmgr, false);
 		PP_ASSERT_WITH_CODE((result == 0),
 				"Post DIDT enable clock gating failed.",
-				return result);
+				goto error);
 		if (hwmgr->chip_id == CHIP_POLARIS11) {
 			result = smum_send_msg_to_smc(hwmgr,
 						(uint16_t)(PPSMC_MSG_DisableDpmDidt));
 			PP_ASSERT_WITH_CODE((0 == result),
-					"Failed to disable DPM DIDT.", return result);
+					"Failed to disable DPM DIDT.", goto error);
 		}
-		cgs_enter_safe_mode(hwmgr->device, false);
+		adev->gfx.rlc.funcs->exit_safe_mode(adev);
 	}
 
 	return 0;
+error:
+	adev->gfx.rlc.funcs->exit_safe_mode(adev);
+	return result;
 }
 
 int smu7_enable_smc_cac(struct pp_hwmgr *hwmgr)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c
index 7b26607c646a..50690c72b2ea 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c
@@ -314,8 +314,7 @@ static int smu8_get_system_info_data(struct pp_hwmgr *hwmgr)
 	uint8_t frev, crev;
 	uint16_t size;
 
-	info = (ATOM_INTEGRATED_SYSTEM_INFO_V1_9 *) cgs_atom_get_data_table(
-			hwmgr->device,
+	info = (ATOM_INTEGRATED_SYSTEM_INFO_V1_9 *)smu_atom_get_data_table(hwmgr->adev,
 			GetIndexIntoMasterTable(DATA, IntegratedSystemInfo),
 			&size, &frev, &crev);
 
@@ -694,7 +693,7 @@ static int smu8_update_sclk_limit(struct pp_hwmgr *hwmgr)
 	else
 		data->sclk_dpm.soft_max_clk  = table->entries[table->count - 1].clk;
 
-	clock = hwmgr->display_config.min_core_set_clock;
+	clock = hwmgr->display_config->min_core_set_clock;
 	if (clock == 0)
 		pr_debug("min_core_set_clock not set\n");
 
@@ -749,7 +748,7 @@ static int smu8_set_deep_sleep_sclk_threshold(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 				PHM_PlatformCaps_SclkDeepSleep)) {
-		uint32_t clks = hwmgr->display_config.min_core_set_clock_in_sr;
+		uint32_t clks = hwmgr->display_config->min_core_set_clock_in_sr;
 		if (clks == 0)
 			clks = SMU8_MIN_DEEP_SLEEP_SCLK;
 
@@ -1041,25 +1040,21 @@ static int smu8_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 	struct smu8_hwmgr *data = hwmgr->backend;
 	struct PP_Clocks clocks = {0, 0, 0, 0};
 	bool force_high;
-	uint32_t  num_of_active_displays = 0;
-	struct cgs_display_info info = {0};
 
 	smu8_ps->need_dfs_bypass = true;
 
 	data->battery_state = (PP_StateUILabel_Battery == prequest_ps->classification.ui_label);
 
-	clocks.memoryClock = hwmgr->display_config.min_mem_set_clock != 0 ?
-				hwmgr->display_config.min_mem_set_clock :
+	clocks.memoryClock = hwmgr->display_config->min_mem_set_clock != 0 ?
+				hwmgr->display_config->min_mem_set_clock :
 				data->sys_info.nbp_memory_clock[1];
 
-	cgs_get_active_displays_info(hwmgr->device, &info);
-	num_of_active_displays = info.display_count;
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
 		clocks.memoryClock = hwmgr->dyn_state.max_clock_voltage_on_ac.mclk;
 
 	force_high = (clocks.memoryClock > data->sys_info.nbp_memory_clock[SMU8_NUM_NBPMEMORYCLOCK - 1])
-			|| (num_of_active_displays >= 3);
+			|| (hwmgr->display_config->num_display >= 3);
 
 	smu8_ps->action = smu8_current_ps->action;
 
@@ -1897,20 +1892,20 @@ static void smu8_dpm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
 	data->uvd_power_gated = bgate;
 
 	if (bgate) {
-		cgs_set_powergating_state(hwmgr->device,
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
 						AMD_IP_BLOCK_TYPE_UVD,
 						AMD_PG_STATE_GATE);
-		cgs_set_clockgating_state(hwmgr->device,
+		amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 						AMD_IP_BLOCK_TYPE_UVD,
 						AMD_CG_STATE_GATE);
 		smu8_dpm_update_uvd_dpm(hwmgr, true);
 		smu8_dpm_powerdown_uvd(hwmgr);
 	} else {
 		smu8_dpm_powerup_uvd(hwmgr);
-		cgs_set_clockgating_state(hwmgr->device,
+		amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 						AMD_IP_BLOCK_TYPE_UVD,
 						AMD_CG_STATE_UNGATE);
-		cgs_set_powergating_state(hwmgr->device,
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
 						AMD_IP_BLOCK_TYPE_UVD,
 						AMD_PG_STATE_UNGATE);
 		smu8_dpm_update_uvd_dpm(hwmgr, false);
@@ -1923,12 +1918,10 @@ static void smu8_dpm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
 	struct smu8_hwmgr *data = hwmgr->backend;
 
 	if (bgate) {
-		cgs_set_powergating_state(
-					hwmgr->device,
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
 					AMD_IP_BLOCK_TYPE_VCE,
 					AMD_PG_STATE_GATE);
-		cgs_set_clockgating_state(
-					hwmgr->device,
+		amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 					AMD_IP_BLOCK_TYPE_VCE,
 					AMD_CG_STATE_GATE);
 		smu8_enable_disable_vce_dpm(hwmgr, false);
@@ -1937,12 +1930,10 @@ static void smu8_dpm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
 	} else {
 		smu8_dpm_powerup_vce(hwmgr);
 		data->vce_power_gated = false;
-		cgs_set_clockgating_state(
-					hwmgr->device,
+		amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 					AMD_IP_BLOCK_TYPE_VCE,
 					AMD_CG_STATE_UNGATE);
-		cgs_set_powergating_state(
-					hwmgr->device,
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
 					AMD_IP_BLOCK_TYPE_VCE,
 					AMD_PG_STATE_UNGATE);
 		smu8_dpm_update_vce_dpm(hwmgr);
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c
index 598122854ab5..93a3d022ba47 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c
@@ -24,6 +24,7 @@
 #include "pp_debug.h"
 #include "ppatomctrl.h"
 #include "ppsmc.h"
+#include "atom.h"
 
 uint8_t convert_to_vid(uint16_t vddc)
 {
@@ -608,3 +609,100 @@ int smu9_register_irq_handlers(struct pp_hwmgr *hwmgr)
 
 	return 0;
 }
+
+void *smu_atom_get_data_table(void *dev, uint32_t table, uint16_t *size,
+						uint8_t *frev, uint8_t *crev)
+{
+	struct amdgpu_device *adev = dev;
+	uint16_t data_start;
+
+	if (amdgpu_atom_parse_data_header(
+		    adev->mode_info.atom_context, table, size,
+		    frev, crev, &data_start))
+		return (uint8_t *)adev->mode_info.atom_context->bios +
+			data_start;
+
+	return NULL;
+}
+
+int smu_get_voltage_dependency_table_ppt_v1(
+			const struct phm_ppt_v1_clock_voltage_dependency_table *allowed_dep_table,
+			struct phm_ppt_v1_clock_voltage_dependency_table *dep_table)
+{
+	uint8_t i = 0;
+	PP_ASSERT_WITH_CODE((0 != allowed_dep_table->count),
+				"Voltage Lookup Table empty",
+				return -EINVAL);
+
+	dep_table->count = allowed_dep_table->count;
+	for (i=0; i<dep_table->count; i++) {
+		dep_table->entries[i].clk = allowed_dep_table->entries[i].clk;
+		dep_table->entries[i].vddInd = allowed_dep_table->entries[i].vddInd;
+		dep_table->entries[i].vdd_offset = allowed_dep_table->entries[i].vdd_offset;
+		dep_table->entries[i].vddc = allowed_dep_table->entries[i].vddc;
+		dep_table->entries[i].vddgfx = allowed_dep_table->entries[i].vddgfx;
+		dep_table->entries[i].vddci = allowed_dep_table->entries[i].vddci;
+		dep_table->entries[i].mvdd = allowed_dep_table->entries[i].mvdd;
+		dep_table->entries[i].phases = allowed_dep_table->entries[i].phases;
+		dep_table->entries[i].cks_enable = allowed_dep_table->entries[i].cks_enable;
+		dep_table->entries[i].cks_voffset = allowed_dep_table->entries[i].cks_voffset;
+	}
+
+	return 0;
+}
+
+int smu_set_watermarks_for_clocks_ranges(void *wt_table,
+		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges)
+{
+	uint32_t i;
+	struct watermarks *table = wt_table;
+
+	if (!table || !wm_with_clock_ranges)
+		return -EINVAL;
+
+	if (wm_with_clock_ranges->num_wm_sets_dmif > 4 || wm_with_clock_ranges->num_wm_sets_mcif > 4)
+		return -EINVAL;
+
+	for (i = 0; i < wm_with_clock_ranges->num_wm_sets_dmif; i++) {
+		table->WatermarkRow[1][i].MinClock =
+			cpu_to_le16((uint16_t)
+			(wm_with_clock_ranges->wm_sets_dmif[i].wm_min_dcefclk_in_khz) /
+			100);
+		table->WatermarkRow[1][i].MaxClock =
+			cpu_to_le16((uint16_t)
+			(wm_with_clock_ranges->wm_sets_dmif[i].wm_max_dcefclk_in_khz) /
+			100);
+		table->WatermarkRow[1][i].MinUclk =
+			cpu_to_le16((uint16_t)
+			(wm_with_clock_ranges->wm_sets_dmif[i].wm_min_memclk_in_khz) /
+			100);
+		table->WatermarkRow[1][i].MaxUclk =
+			cpu_to_le16((uint16_t)
+			(wm_with_clock_ranges->wm_sets_dmif[i].wm_max_memclk_in_khz) /
+			100);
+		table->WatermarkRow[1][i].WmSetting = (uint8_t)
+				wm_with_clock_ranges->wm_sets_dmif[i].wm_set_id;
+	}
+
+	for (i = 0; i < wm_with_clock_ranges->num_wm_sets_mcif; i++) {
+		table->WatermarkRow[0][i].MinClock =
+			cpu_to_le16((uint16_t)
+			(wm_with_clock_ranges->wm_sets_mcif[i].wm_min_socclk_in_khz) /
+			100);
+		table->WatermarkRow[0][i].MaxClock =
+			cpu_to_le16((uint16_t)
+			(wm_with_clock_ranges->wm_sets_mcif[i].wm_max_socclk_in_khz) /
+			100);
+		table->WatermarkRow[0][i].MinUclk =
+			cpu_to_le16((uint16_t)
+			(wm_with_clock_ranges->wm_sets_mcif[i].wm_min_memclk_in_khz) /
+			100);
+		table->WatermarkRow[0][i].MaxUclk =
+			cpu_to_le16((uint16_t)
+			(wm_with_clock_ranges->wm_sets_mcif[i].wm_max_memclk_in_khz) /
+			100);
+		table->WatermarkRow[0][i].WmSetting = (uint8_t)
+				wm_with_clock_ranges->wm_sets_mcif[i].wm_set_id;
+	}
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.h
index d37d16e4b613..916cc01e7652 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.h
@@ -26,10 +26,27 @@
 struct pp_atomctrl_voltage_table;
 struct pp_hwmgr;
 struct phm_ppt_v1_voltage_lookup_table;
+struct Watermarks_t;
+struct pp_wm_sets_with_clock_ranges_soc15;
 
 uint8_t convert_to_vid(uint16_t vddc);
 uint16_t convert_to_vddc(uint8_t vid);
 
+struct watermark_row_generic_t {
+	uint16_t MinClock;
+	uint16_t MaxClock;
+	uint16_t MinUclk;
+	uint16_t MaxUclk;
+
+	uint8_t  WmSetting;
+	uint8_t  Padding[3];
+};
+
+struct watermarks {
+	struct watermark_row_generic_t WatermarkRow[2][4];
+	uint32_t     padding[7];
+};
+
 extern int phm_wait_for_register_unequal(struct pp_hwmgr *hwmgr,
 					uint32_t index,
 					uint32_t value, uint32_t mask);
@@ -82,6 +99,16 @@ int phm_irq_process(struct amdgpu_device *adev,
 
 int smu9_register_irq_handlers(struct pp_hwmgr *hwmgr);
 
+void *smu_atom_get_data_table(void *dev, uint32_t table, uint16_t *size,
+						uint8_t *frev, uint8_t *crev);
+
+int smu_get_voltage_dependency_table_ppt_v1(
+	const struct phm_ppt_v1_clock_voltage_dependency_table *allowed_dep_table,
+		struct phm_ppt_v1_clock_voltage_dependency_table *dep_table);
+
+int smu_set_watermarks_for_clocks_ranges(void *wt_table,
+		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges);
+
 #define PHM_FIELD_SHIFT(reg, field) reg##__##field##__SHIFT
 #define PHM_FIELD_MASK(reg, field) reg##__##field##_MASK
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
index 7cbb56ba6fab..d156b7bb92ae 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
@@ -36,7 +36,7 @@
 #include "smu9.h"
 #include "smu9_driver_if.h"
 #include "vega10_inc.h"
-#include "pp_soc15.h"
+#include "soc15_common.h"
 #include "pppcielanes.h"
 #include "vega10_hwmgr.h"
 #include "vega10_processpptables.h"
@@ -51,10 +51,6 @@
 #include "smuio/smuio_9_0_offset.h"
 #include "smuio/smuio_9_0_sh_mask.h"
 
-#define VOLTAGE_SCALE  4
-#define VOLTAGE_VID_OFFSET_SCALE1   625
-#define VOLTAGE_VID_OFFSET_SCALE2   100
-
 #define HBM_MEMORY_CHANNEL_WIDTH    128
 
 static const uint32_t channel_number[] = {1, 2, 0, 4, 0, 8, 0, 16, 2};
@@ -79,8 +75,6 @@ static const uint32_t channel_number[] = {1, 2, 0, 4, 0, 8, 0, 16, 2};
 #define DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK                                                        0x000000F0L
 #define DF_CS_AON0_DramBaseAddress0__IntLvAddrSel_MASK                                                        0x00000700L
 #define DF_CS_AON0_DramBaseAddress0__DramBaseAddr_MASK                                                        0xFFFFF000L
-static int vega10_force_clock_level(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, uint32_t mask);
 
 static const ULONG PhwVega10_Magic = (ULONG)(PHM_VIslands_Magic);
 
@@ -291,6 +285,48 @@ static int vega10_set_features_platform_caps(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
+static int vega10_odn_initial_default_setting(struct pp_hwmgr *hwmgr)
+{
+	struct vega10_hwmgr *data = hwmgr->backend;
+	struct phm_ppt_v2_information *table_info =
+			(struct phm_ppt_v2_information *)(hwmgr->pptable);
+	struct vega10_odn_dpm_table *odn_table = &(data->odn_dpm_table);
+	struct vega10_odn_vddc_lookup_table *od_lookup_table;
+	struct phm_ppt_v1_voltage_lookup_table *vddc_lookup_table;
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_table[3];
+	struct phm_ppt_v1_clock_voltage_dependency_table *od_table[3];
+	uint32_t i;
+
+	od_lookup_table = &odn_table->vddc_lookup_table;
+	vddc_lookup_table = table_info->vddc_lookup_table;
+
+	for (i = 0; i < vddc_lookup_table->count; i++)
+		od_lookup_table->entries[i].us_vdd = vddc_lookup_table->entries[i].us_vdd;
+
+	od_lookup_table->count = vddc_lookup_table->count;
+
+	dep_table[0] = table_info->vdd_dep_on_sclk;
+	dep_table[1] = table_info->vdd_dep_on_mclk;
+	dep_table[2] = table_info->vdd_dep_on_socclk;
+	od_table[0] = (struct phm_ppt_v1_clock_voltage_dependency_table *)&odn_table->vdd_dep_on_sclk;
+	od_table[1] = (struct phm_ppt_v1_clock_voltage_dependency_table *)&odn_table->vdd_dep_on_mclk;
+	od_table[2] = (struct phm_ppt_v1_clock_voltage_dependency_table *)&odn_table->vdd_dep_on_socclk;
+
+	for (i = 0; i < 3; i++)
+		smu_get_voltage_dependency_table_ppt_v1(dep_table[i], od_table[i]);
+
+	if (odn_table->max_vddc == 0 || odn_table->max_vddc > 2000)
+		odn_table->max_vddc = dep_table[0]->entries[dep_table[0]->count - 1].vddc;
+	if (odn_table->min_vddc == 0 || odn_table->min_vddc > 2000)
+		odn_table->min_vddc = dep_table[0]->entries[0].vddc;
+
+	i = od_table[2]->count - 1;
+	od_table[2]->entries[i].clk = hwmgr->platform_descriptor.overdriveLimit.memoryClock;
+	od_table[2]->entries[i].vddc = odn_table->max_vddc;
+
+	return 0;
+}
+
 static void vega10_init_dpm_defaults(struct pp_hwmgr *hwmgr)
 {
 	struct vega10_hwmgr *data = hwmgr->backend;
@@ -427,7 +463,6 @@ static void vega10_init_dpm_defaults(struct pp_hwmgr *hwmgr)
 		/* ACG firmware has major version 5 */
 	if ((hwmgr->smu_version & 0xff000000) == 0x5000000)
 		data->smu_features[GNLD_ACG].supported = true;
-
 	if (data->registry_data.didt_support)
 		data->smu_features[GNLD_DIDT].supported = true;
 
@@ -754,7 +789,6 @@ static int vega10_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 	uint32_t config_telemetry = 0;
 	struct pp_atomfwctrl_voltage_table vol_table;
 	struct amdgpu_device *adev = hwmgr->adev;
-	uint32_t reg;
 
 	data = kzalloc(sizeof(struct vega10_hwmgr), GFP_KERNEL);
 	if (data == NULL)
@@ -860,10 +894,7 @@ static int vega10_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 			advanceFanControlParameters.usFanPWMMinLimit *
 			hwmgr->thermal_controller.fanInfo.ulMaxRPM / 100;
 
-	reg = soc15_get_register_offset(DF_HWID, 0,
-			mmDF_CS_AON0_DramBaseAddress0_BASE_IDX,
-			mmDF_CS_AON0_DramBaseAddress0);
-	data->mem_channels = (cgs_read_register(hwmgr->device, reg) &
+	data->mem_channels = (RREG32_SOC15(DF, 0, mmDF_CS_AON0_DramBaseAddress0) &
 			DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK) >>
 			DF_CS_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
 	PP_ASSERT_WITH_CODE(data->mem_channels < ARRAY_SIZE(channel_number),
@@ -1370,48 +1401,6 @@ static int vega10_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 	memcpy(&(data->golden_dpm_table), &(data->dpm_table),
 			sizeof(struct vega10_dpm_table));
 
-	if (PP_CAP(PHM_PlatformCaps_ODNinACSupport) ||
-	    PP_CAP(PHM_PlatformCaps_ODNinDCSupport)) {
-		data->odn_dpm_table.odn_core_clock_dpm_levels.num_of_pl =
-						data->dpm_table.gfx_table.count;
-		for (i = 0; i < data->dpm_table.gfx_table.count; i++) {
-			data->odn_dpm_table.odn_core_clock_dpm_levels.entries[i].clock =
-					data->dpm_table.gfx_table.dpm_levels[i].value;
-			data->odn_dpm_table.odn_core_clock_dpm_levels.entries[i].enabled = true;
-		}
-
-		data->odn_dpm_table.vdd_dependency_on_sclk.count =
-				dep_gfx_table->count;
-		for (i = 0; i < dep_gfx_table->count; i++) {
-			data->odn_dpm_table.vdd_dependency_on_sclk.entries[i].clk =
-					dep_gfx_table->entries[i].clk;
-			data->odn_dpm_table.vdd_dependency_on_sclk.entries[i].vddInd =
-					dep_gfx_table->entries[i].vddInd;
-			data->odn_dpm_table.vdd_dependency_on_sclk.entries[i].cks_enable =
-					dep_gfx_table->entries[i].cks_enable;
-			data->odn_dpm_table.vdd_dependency_on_sclk.entries[i].cks_voffset =
-					dep_gfx_table->entries[i].cks_voffset;
-		}
-
-		data->odn_dpm_table.odn_memory_clock_dpm_levels.num_of_pl =
-						data->dpm_table.mem_table.count;
-		for (i = 0; i < data->dpm_table.mem_table.count; i++) {
-			data->odn_dpm_table.odn_memory_clock_dpm_levels.entries[i].clock =
-					data->dpm_table.mem_table.dpm_levels[i].value;
-			data->odn_dpm_table.odn_memory_clock_dpm_levels.entries[i].enabled = true;
-		}
-
-		data->odn_dpm_table.vdd_dependency_on_mclk.count = dep_mclk_table->count;
-		for (i = 0; i < dep_mclk_table->count; i++) {
-			data->odn_dpm_table.vdd_dependency_on_mclk.entries[i].clk =
-					dep_mclk_table->entries[i].clk;
-			data->odn_dpm_table.vdd_dependency_on_mclk.entries[i].vddInd =
-					dep_mclk_table->entries[i].vddInd;
-			data->odn_dpm_table.vdd_dependency_on_mclk.entries[i].vddci =
-					dep_mclk_table->entries[i].vddci;
-		}
-	}
-
 	return 0;
 }
 
@@ -1514,18 +1503,18 @@ static int vega10_populate_single_gfx_level(struct pp_hwmgr *hwmgr,
 {
 	struct phm_ppt_v2_information *table_info =
 			(struct phm_ppt_v2_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_on_sclk =
-			table_info->vdd_dep_on_sclk;
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_on_sclk;
 	struct vega10_hwmgr *data = hwmgr->backend;
 	struct pp_atomfwctrl_clock_dividers_soc15 dividers;
 	uint32_t gfx_max_clock =
 			hwmgr->platform_descriptor.overdriveLimit.engineClock;
 	uint32_t i = 0;
 
-	if (data->apply_overdrive_next_settings_mask &
-			DPMTABLE_OD_UPDATE_VDDC)
+	if (hwmgr->od_enabled)
 		dep_on_sclk = (struct phm_ppt_v1_clock_voltage_dependency_table *)
-						&(data->odn_dpm_table.vdd_dependency_on_sclk);
+						&(data->odn_dpm_table.vdd_dep_on_sclk);
+	else
+		dep_on_sclk = table_info->vdd_dep_on_sclk;
 
 	PP_ASSERT_WITH_CODE(dep_on_sclk,
 			"Invalid SOC_VDD-GFX_CLK Dependency Table!",
@@ -1577,23 +1566,32 @@ static int vega10_populate_single_soc_level(struct pp_hwmgr *hwmgr,
 		uint32_t soc_clock, uint8_t *current_soc_did,
 		uint8_t *current_vol_index)
 {
+	struct vega10_hwmgr *data = hwmgr->backend;
 	struct phm_ppt_v2_information *table_info =
 			(struct phm_ppt_v2_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_on_soc =
-			table_info->vdd_dep_on_socclk;
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_on_soc;
 	struct pp_atomfwctrl_clock_dividers_soc15 dividers;
 	uint32_t i;
 
-	PP_ASSERT_WITH_CODE(dep_on_soc,
-			"Invalid SOC_VDD-SOC_CLK Dependency Table!",
-			return -EINVAL);
-	for (i = 0; i < dep_on_soc->count; i++) {
-		if (dep_on_soc->entries[i].clk == soc_clock)
-			break;
+	if (hwmgr->od_enabled) {
+		dep_on_soc = (struct phm_ppt_v1_clock_voltage_dependency_table *)
+						&data->odn_dpm_table.vdd_dep_on_socclk;
+		for (i = 0; i < dep_on_soc->count; i++) {
+			if (dep_on_soc->entries[i].clk >= soc_clock)
+				break;
+		}
+	} else {
+		dep_on_soc = table_info->vdd_dep_on_socclk;
+		for (i = 0; i < dep_on_soc->count; i++) {
+			if (dep_on_soc->entries[i].clk == soc_clock)
+				break;
+		}
 	}
+
 	PP_ASSERT_WITH_CODE(dep_on_soc->count > i,
 			"Cannot find SOC_CLK in SOC_VDD-SOC_CLK Dependency Table",
 			return -EINVAL);
+
 	PP_ASSERT_WITH_CODE(!pp_atomfwctrl_get_gpu_pll_dividers_vega10(hwmgr,
 			COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
 			soc_clock, &dividers),
@@ -1602,22 +1600,6 @@ static int vega10_populate_single_soc_level(struct pp_hwmgr *hwmgr,
 
 	*current_soc_did = (uint8_t)dividers.ulDid;
 	*current_vol_index = (uint8_t)(dep_on_soc->entries[i].vddInd);
-
-	return 0;
-}
-
-uint16_t vega10_locate_vddc_given_clock(struct pp_hwmgr *hwmgr,
-		uint32_t clk,
-		struct phm_ppt_v1_clock_voltage_dependency_table *dep_table)
-{
-	uint16_t i;
-
-	for (i = 0; i < dep_table->count; i++) {
-		if (dep_table->entries[i].clk == clk)
-			return dep_table->entries[i].vddc;
-	}
-
-	pr_info("[LocateVddcGivenClock] Cannot locate SOC Vddc for this clock!");
 	return 0;
 }
 
@@ -1631,8 +1613,6 @@ static int vega10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
 	struct vega10_hwmgr *data = hwmgr->backend;
 	struct phm_ppt_v2_information *table_info =
 			(struct phm_ppt_v2_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_table =
-			table_info->vdd_dep_on_socclk;
 	PPTable_t *pp_table = &(data->smc_state_table.pp_table);
 	struct vega10_single_dpm_table *dpm_table = &(data->dpm_table.gfx_table);
 	int result = 0;
@@ -1663,11 +1643,6 @@ static int vega10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
 
 	dpm_table = &(data->dpm_table.soc_table);
 	for (i = 0; i < dpm_table->count; i++) {
-		pp_table->SocVid[i] =
-				(uint8_t)convert_to_vid(
-				vega10_locate_vddc_given_clock(hwmgr,
-						dpm_table->dpm_levels[i].value,
-						dep_table));
 		result = vega10_populate_single_soc_level(hwmgr,
 				dpm_table->dpm_levels[i].value,
 				&(pp_table->SocclkDid[i]),
@@ -1678,7 +1653,6 @@ static int vega10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
 
 	j = i - 1;
 	while (i < NUM_SOCCLK_DPM_LEVELS) {
-		pp_table->SocVid[i] = pp_table->SocVid[j];
 		result = vega10_populate_single_soc_level(hwmgr,
 				dpm_table->dpm_levels[j].value,
 				&(pp_table->SocclkDid[i]),
@@ -1691,6 +1665,32 @@ static int vega10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
 	return result;
 }
 
+static void vega10_populate_vddc_soc_levels(struct pp_hwmgr *hwmgr)
+{
+	struct vega10_hwmgr *data = hwmgr->backend;
+	PPTable_t *pp_table = &(data->smc_state_table.pp_table);
+	struct phm_ppt_v2_information *table_info = hwmgr->pptable;
+	struct phm_ppt_v1_voltage_lookup_table *vddc_lookup_table;
+
+	uint8_t soc_vid = 0;
+	uint32_t i, max_vddc_level;
+
+	if (hwmgr->od_enabled)
+		vddc_lookup_table = (struct phm_ppt_v1_voltage_lookup_table *)&data->odn_dpm_table.vddc_lookup_table;
+	else
+		vddc_lookup_table = table_info->vddc_lookup_table;
+
+	max_vddc_level = vddc_lookup_table->count;
+	for (i = 0; i < max_vddc_level; i++) {
+		soc_vid = (uint8_t)convert_to_vid(vddc_lookup_table->entries[i].us_vdd);
+		pp_table->SocVid[i] = soc_vid;
+	}
+	while (i < MAX_REGULAR_DPM_NUMBER) {
+		pp_table->SocVid[i] = soc_vid;
+		i++;
+	}
+}
+
 /**
  * @brief Populates single SMC GFXCLK structure using the provided clock.
  *
@@ -1705,25 +1705,25 @@ static int vega10_populate_single_memory_level(struct pp_hwmgr *hwmgr,
 	struct vega10_hwmgr *data = hwmgr->backend;
 	struct phm_ppt_v2_information *table_info =
 			(struct phm_ppt_v2_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_on_mclk =
-			table_info->vdd_dep_on_mclk;
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_on_mclk;
 	struct pp_atomfwctrl_clock_dividers_soc15 dividers;
 	uint32_t mem_max_clock =
 			hwmgr->platform_descriptor.overdriveLimit.memoryClock;
 	uint32_t i = 0;
 
-	if (data->apply_overdrive_next_settings_mask &
-			DPMTABLE_OD_UPDATE_VDDC)
+	if (hwmgr->od_enabled)
 		dep_on_mclk = (struct phm_ppt_v1_clock_voltage_dependency_table *)
-					&data->odn_dpm_table.vdd_dependency_on_mclk;
+					&data->odn_dpm_table.vdd_dep_on_mclk;
+	else
+		dep_on_mclk = table_info->vdd_dep_on_mclk;
 
 	PP_ASSERT_WITH_CODE(dep_on_mclk,
 			"Invalid SOC_VDD-UCLK Dependency Table!",
 			return -EINVAL);
 
-	if (data->need_update_dpm_table & DPMTABLE_OD_UPDATE_MCLK)
+	if (data->need_update_dpm_table & DPMTABLE_OD_UPDATE_MCLK) {
 		mem_clock = mem_clock > mem_max_clock ? mem_max_clock : mem_clock;
-	else {
+	} else {
 		for (i = 0; i < dep_on_mclk->count; i++) {
 			if (dep_on_mclk->entries[i].clk == mem_clock)
 				break;
@@ -2067,6 +2067,9 @@ static int vega10_populate_avfs_parameters(struct pp_hwmgr *hwmgr)
 	if (data->smu_features[GNLD_AVFS].supported) {
 		result = pp_atomfwctrl_get_avfs_information(hwmgr, &avfs_params);
 		if (!result) {
+			data->odn_dpm_table.max_vddc = avfs_params.ulMaxVddc;
+			data->odn_dpm_table.min_vddc = avfs_params.ulMinVddc;
+
 			pp_table->MinVoltageVid = (uint8_t)
 					convert_to_vid((uint16_t)(avfs_params.ulMinVddc));
 			pp_table->MaxVoltageVid = (uint8_t)
@@ -2345,6 +2348,22 @@ static int vega10_avfs_enable(struct pp_hwmgr *hwmgr, bool enable)
 	return 0;
 }
 
+static int vega10_update_avfs(struct pp_hwmgr *hwmgr)
+{
+	struct vega10_hwmgr *data = hwmgr->backend;
+
+	if (data->need_update_dpm_table & DPMTABLE_OD_UPDATE_VDDC) {
+		vega10_avfs_enable(hwmgr, false);
+	} else if (data->need_update_dpm_table) {
+		vega10_avfs_enable(hwmgr, false);
+		vega10_avfs_enable(hwmgr, true);
+	} else {
+		vega10_avfs_enable(hwmgr, true);
+	}
+
+	return 0;
+}
+
 static int vega10_populate_and_upload_avfs_fuse_override(struct pp_hwmgr *hwmgr)
 {
 	int result = 0;
@@ -2406,6 +2425,10 @@ static int vega10_init_smc_table(struct pp_hwmgr *hwmgr)
 			"Failed to setup default DPM tables!",
 			return result);
 
+	/* initialize ODN table */
+	if (hwmgr->od_enabled)
+		vega10_odn_initial_default_setting(hwmgr);
+
 	pp_atomfwctrl_get_voltage_table_v4(hwmgr, VOLTAGE_TYPE_VDDC,
 			VOLTAGE_OBJ_SVID2,  &voltage_table);
 	pp_table->MaxVidStep = voltage_table.max_vid_step;
@@ -2452,6 +2475,8 @@ static int vega10_init_smc_table(struct pp_hwmgr *hwmgr)
 			"Failed to initialize Memory Level!",
 			return result);
 
+	vega10_populate_vddc_soc_levels(hwmgr);
+
 	result = vega10_populate_all_display_clock_levels(hwmgr);
 	PP_ASSERT_WITH_CODE(!result,
 			"Failed to initialize Display Level!",
@@ -2481,6 +2506,12 @@ static int vega10_init_smc_table(struct pp_hwmgr *hwmgr)
 		data->vbios_boot_state.mvddc    = boot_up_values.usMvddc;
 		data->vbios_boot_state.gfx_clock = boot_up_values.ulGfxClk;
 		data->vbios_boot_state.mem_clock = boot_up_values.ulUClk;
+		pp_atomfwctrl_get_clk_information_by_clkid(hwmgr,
+				SMU9_SYSPLL0_SOCCLK_ID, &boot_up_values.ulSocClk);
+
+		pp_atomfwctrl_get_clk_information_by_clkid(hwmgr,
+				SMU9_SYSPLL0_DCEFCLK_ID, &boot_up_values.ulDCEFClk);
+
 		data->vbios_boot_state.soc_clock = boot_up_values.ulSocClk;
 		data->vbios_boot_state.dcef_clock = boot_up_values.ulDCEFClk;
 		if (0 != boot_up_values.usVddc) {
@@ -2829,7 +2860,7 @@ static int vega10_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 
 	tmp_result = vega10_construct_voltage_tables(hwmgr);
 	PP_ASSERT_WITH_CODE(!tmp_result,
-			"Failed to contruct voltage tables!",
+			"Failed to construct voltage tables!",
 			result = tmp_result);
 
 	tmp_result = vega10_init_smc_table(hwmgr);
@@ -3028,7 +3059,6 @@ static int vega10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 	bool disable_mclk_switching_for_frame_lock;
 	bool disable_mclk_switching_for_vr;
 	bool force_mclk_high;
-	struct cgs_display_info info = {0};
 	const struct phm_clock_and_voltage_limits *max_limits;
 	uint32_t i;
 	struct vega10_hwmgr *data = hwmgr->backend;
@@ -3063,11 +3093,9 @@ static int vega10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 		}
 	}
 
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
 	/* result = PHM_CheckVBlankTime(hwmgr, &vblankTooShort);*/
-	minimum_clocks.engineClock = hwmgr->display_config.min_core_set_clock;
-	minimum_clocks.memoryClock = hwmgr->display_config.min_mem_set_clock;
+	minimum_clocks.engineClock = hwmgr->display_config->min_core_set_clock;
+	minimum_clocks.memoryClock = hwmgr->display_config->min_mem_set_clock;
 
 	if (PP_CAP(PHM_PlatformCaps_StablePState)) {
 		stable_pstate_sclk_dpm_percentage =
@@ -3107,10 +3135,10 @@ static int vega10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 		PP_CAP(PHM_PlatformCaps_DisableMclkSwitchForVR);
 	force_mclk_high = PP_CAP(PHM_PlatformCaps_ForceMclkHigh);
 
-	if (info.display_count == 0)
+	if (hwmgr->display_config->num_display == 0)
 		disable_mclk_switching = false;
 	else
-		disable_mclk_switching = (info.display_count > 1) ||
+		disable_mclk_switching = (hwmgr->display_config->num_display > 1) ||
 			disable_mclk_switching_for_frame_lock ||
 			disable_mclk_switching_for_vr ||
 			force_mclk_high;
@@ -3171,87 +3199,11 @@ static int vega10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 
 static int vega10_find_dpm_states_clocks_in_dpm_table(struct pp_hwmgr *hwmgr, const void *input)
 {
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	const struct vega10_power_state *vega10_ps =
-			cast_const_phw_vega10_power_state(states->pnew_state);
 	struct vega10_hwmgr *data = hwmgr->backend;
-	struct vega10_single_dpm_table *sclk_table =
-			&(data->dpm_table.gfx_table);
-	uint32_t sclk = vega10_ps->performance_levels
-			[vega10_ps->performance_level_count - 1].gfx_clock;
-	struct vega10_single_dpm_table *mclk_table =
-			&(data->dpm_table.mem_table);
-	uint32_t mclk = vega10_ps->performance_levels
-			[vega10_ps->performance_level_count - 1].mem_clock;
-	struct PP_Clocks min_clocks = {0};
-	uint32_t i;
-	struct cgs_display_info info = {0};
-
-	data->need_update_dpm_table = 0;
-
-	if (PP_CAP(PHM_PlatformCaps_ODNinACSupport) ||
-	    PP_CAP(PHM_PlatformCaps_ODNinDCSupport)) {
-		for (i = 0; i < sclk_table->count; i++) {
-			if (sclk == sclk_table->dpm_levels[i].value)
-				break;
-		}
-
-		if (!(data->apply_overdrive_next_settings_mask &
-				DPMTABLE_OD_UPDATE_SCLK) && i >= sclk_table->count) {
-			/* Check SCLK in DAL's minimum clocks
-			 * in case DeepSleep divider update is required.
-			 */
-			if (data->display_timing.min_clock_in_sr !=
-					min_clocks.engineClockInSR &&
-				(min_clocks.engineClockInSR >=
-						VEGA10_MINIMUM_ENGINE_CLOCK ||
-					data->display_timing.min_clock_in_sr >=
-						VEGA10_MINIMUM_ENGINE_CLOCK))
-				data->need_update_dpm_table |= DPMTABLE_UPDATE_SCLK;
-		}
-
-		cgs_get_active_displays_info(hwmgr->device, &info);
-
-		if (data->display_timing.num_existing_displays !=
-				info.display_count)
-			data->need_update_dpm_table |= DPMTABLE_UPDATE_MCLK;
-	} else {
-		for (i = 0; i < sclk_table->count; i++) {
-			if (sclk == sclk_table->dpm_levels[i].value)
-				break;
-		}
-
-		if (i >= sclk_table->count)
-			data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
-		else {
-			/* Check SCLK in DAL's minimum clocks
-			 * in case DeepSleep divider update is required.
-			 */
-			if (data->display_timing.min_clock_in_sr !=
-					min_clocks.engineClockInSR &&
-				(min_clocks.engineClockInSR >=
-						VEGA10_MINIMUM_ENGINE_CLOCK ||
-					data->display_timing.min_clock_in_sr >=
-						VEGA10_MINIMUM_ENGINE_CLOCK))
-				data->need_update_dpm_table |= DPMTABLE_UPDATE_SCLK;
-		}
 
-		for (i = 0; i < mclk_table->count; i++) {
-			if (mclk == mclk_table->dpm_levels[i].value)
-				break;
-		}
-
-		cgs_get_active_displays_info(hwmgr->device, &info);
+	if (data->display_timing.num_existing_displays != hwmgr->display_config->num_display)
+		data->need_update_dpm_table |= DPMTABLE_UPDATE_MCLK;
 
-		if (i >= mclk_table->count)
-			data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
-
-		if (data->display_timing.num_existing_displays !=
-				info.display_count ||
-				i >= mclk_table->count)
-			data->need_update_dpm_table |= DPMTABLE_UPDATE_MCLK;
-	}
 	return 0;
 }
 
@@ -3259,194 +3211,29 @@ static int vega10_populate_and_upload_sclk_mclk_dpm_levels(
 		struct pp_hwmgr *hwmgr, const void *input)
 {
 	int result = 0;
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	const struct vega10_power_state *vega10_ps =
-			cast_const_phw_vega10_power_state(states->pnew_state);
 	struct vega10_hwmgr *data = hwmgr->backend;
-	uint32_t sclk = vega10_ps->performance_levels
-			[vega10_ps->performance_level_count - 1].gfx_clock;
-	uint32_t mclk = vega10_ps->performance_levels
-			[vega10_ps->performance_level_count - 1].mem_clock;
-	struct vega10_dpm_table *dpm_table = &data->dpm_table;
-	struct vega10_dpm_table *golden_dpm_table =
-			&data->golden_dpm_table;
-	uint32_t dpm_count, clock_percent;
-	uint32_t i;
 
-	if (PP_CAP(PHM_PlatformCaps_ODNinACSupport) ||
-	    PP_CAP(PHM_PlatformCaps_ODNinDCSupport)) {
-
-		if (!data->need_update_dpm_table &&
-			!data->apply_optimized_settings &&
-			!data->apply_overdrive_next_settings_mask)
-			return 0;
-
-		if (data->apply_overdrive_next_settings_mask &
-				DPMTABLE_OD_UPDATE_SCLK) {
-			for (dpm_count = 0;
-					dpm_count < dpm_table->gfx_table.count;
-					dpm_count++) {
-				dpm_table->gfx_table.dpm_levels[dpm_count].enabled =
-					data->odn_dpm_table.odn_core_clock_dpm_levels.entries[dpm_count].enabled;
-				dpm_table->gfx_table.dpm_levels[dpm_count].value =
-					data->odn_dpm_table.odn_core_clock_dpm_levels.entries[dpm_count].clock;
-			}
-		}
-
-		if (data->apply_overdrive_next_settings_mask &
-				DPMTABLE_OD_UPDATE_MCLK) {
-			for (dpm_count = 0;
-					dpm_count < dpm_table->mem_table.count;
-					dpm_count++) {
-				dpm_table->mem_table.dpm_levels[dpm_count].enabled =
-					data->odn_dpm_table.odn_memory_clock_dpm_levels.entries[dpm_count].enabled;
-				dpm_table->mem_table.dpm_levels[dpm_count].value =
-					data->odn_dpm_table.odn_memory_clock_dpm_levels.entries[dpm_count].clock;
-			}
-		}
-
-		if ((data->need_update_dpm_table & DPMTABLE_UPDATE_SCLK) ||
-			data->apply_optimized_settings ||
-			(data->apply_overdrive_next_settings_mask &
-					DPMTABLE_OD_UPDATE_SCLK)) {
-			result = vega10_populate_all_graphic_levels(hwmgr);
-			PP_ASSERT_WITH_CODE(!result,
-					"Failed to populate SCLK during PopulateNewDPMClocksStates Function!",
-					return result);
-		}
-
-		if ((data->need_update_dpm_table & DPMTABLE_UPDATE_MCLK) ||
-			(data->apply_overdrive_next_settings_mask &
-					DPMTABLE_OD_UPDATE_MCLK)){
-			result = vega10_populate_all_memory_levels(hwmgr);
-			PP_ASSERT_WITH_CODE(!result,
-					"Failed to populate MCLK during PopulateNewDPMClocksStates Function!",
-					return result);
-		}
-	} else {
-		if (!data->need_update_dpm_table &&
-				!data->apply_optimized_settings)
-			return 0;
-
-		if (data->need_update_dpm_table & DPMTABLE_OD_UPDATE_SCLK &&
-				data->smu_features[GNLD_DPM_GFXCLK].supported) {
-				dpm_table->
-				gfx_table.dpm_levels[dpm_table->gfx_table.count - 1].
-				value = sclk;
-				if (hwmgr->od_enabled) {
-					/* Need to do calculation based on the golden DPM table
-					 * as the Heatmap GPU Clock axis is also based on
-					 * the default values
-					 */
-					PP_ASSERT_WITH_CODE(
-							golden_dpm_table->gfx_table.dpm_levels
-							[golden_dpm_table->gfx_table.count - 1].value,
-							"Divide by 0!",
-							return -1);
-
-					dpm_count = dpm_table->gfx_table.count < 2 ?
-							0 : dpm_table->gfx_table.count - 2;
-					for (i = dpm_count; i > 1; i--) {
-						if (sclk > golden_dpm_table->gfx_table.dpm_levels
-							[golden_dpm_table->gfx_table.count - 1].value) {
-							clock_percent =
-								((sclk - golden_dpm_table->gfx_table.dpm_levels
-								[golden_dpm_table->gfx_table.count - 1].value) *
-								100) /
-								golden_dpm_table->gfx_table.dpm_levels
-								[golden_dpm_table->gfx_table.count - 1].value;
-
-							dpm_table->gfx_table.dpm_levels[i].value =
-								golden_dpm_table->gfx_table.dpm_levels[i].value +
-								(golden_dpm_table->gfx_table.dpm_levels[i].value *
-								clock_percent) / 100;
-						} else if (golden_dpm_table->
-								gfx_table.dpm_levels[dpm_table->gfx_table.count-1].value >
-								sclk) {
-							clock_percent =
-								((golden_dpm_table->gfx_table.dpm_levels
-								[golden_dpm_table->gfx_table.count - 1].value -
-								sclk) *	100) /
-								golden_dpm_table->gfx_table.dpm_levels
-								[golden_dpm_table->gfx_table.count-1].value;
-
-							dpm_table->gfx_table.dpm_levels[i].value =
-								golden_dpm_table->gfx_table.dpm_levels[i].value -
-								(golden_dpm_table->gfx_table.dpm_levels[i].value *
-								clock_percent) / 100;
-						} else
-							dpm_table->gfx_table.dpm_levels[i].value =
-								golden_dpm_table->gfx_table.dpm_levels[i].value;
-					}
-				}
-			}
+	if (!data->need_update_dpm_table)
+		return 0;
 
-		if (data->need_update_dpm_table & DPMTABLE_OD_UPDATE_MCLK &&
-				data->smu_features[GNLD_DPM_UCLK].supported) {
-			dpm_table->
-			mem_table.dpm_levels[dpm_table->mem_table.count - 1].
-			value = mclk;
+	if (data->need_update_dpm_table &
+			(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK + DPMTABLE_UPDATE_SOCCLK)) {
+		result = vega10_populate_all_graphic_levels(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to populate SCLK during PopulateNewDPMClocksStates Function!",
+				return result);
+	}
 
-			if (hwmgr->od_enabled) {
-				PP_ASSERT_WITH_CODE(
-					golden_dpm_table->mem_table.dpm_levels
-					[golden_dpm_table->mem_table.count - 1].value,
-					"Divide by 0!",
-					return -1);
+	if (data->need_update_dpm_table &
+			(DPMTABLE_OD_UPDATE_MCLK + DPMTABLE_UPDATE_MCLK)) {
+		result = vega10_populate_all_memory_levels(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to populate MCLK during PopulateNewDPMClocksStates Function!",
+				return result);
+	}
 
-				dpm_count = dpm_table->mem_table.count < 2 ?
-						0 : dpm_table->mem_table.count - 2;
-				for (i = dpm_count; i > 1; i--) {
-					if (mclk > golden_dpm_table->mem_table.dpm_levels
-						[golden_dpm_table->mem_table.count-1].value) {
-						clock_percent = ((mclk -
-							golden_dpm_table->mem_table.dpm_levels
-							[golden_dpm_table->mem_table.count-1].value) *
-							100) /
-							golden_dpm_table->mem_table.dpm_levels
-							[golden_dpm_table->mem_table.count-1].value;
-
-						dpm_table->mem_table.dpm_levels[i].value =
-							golden_dpm_table->mem_table.dpm_levels[i].value +
-							(golden_dpm_table->mem_table.dpm_levels[i].value *
-							clock_percent) / 100;
-					} else if (golden_dpm_table->mem_table.dpm_levels
-							[dpm_table->mem_table.count-1].value > mclk) {
-						clock_percent = ((golden_dpm_table->mem_table.dpm_levels
-							[golden_dpm_table->mem_table.count-1].value - mclk) *
-							100) /
-							golden_dpm_table->mem_table.dpm_levels
-							[golden_dpm_table->mem_table.count-1].value;
-
-						dpm_table->mem_table.dpm_levels[i].value =
-							golden_dpm_table->mem_table.dpm_levels[i].value -
-							(golden_dpm_table->mem_table.dpm_levels[i].value *
-							clock_percent) / 100;
-					} else
-						dpm_table->mem_table.dpm_levels[i].value =
-							golden_dpm_table->mem_table.dpm_levels[i].value;
-				}
-			}
-		}
+	vega10_populate_vddc_soc_levels(hwmgr);
 
-		if ((data->need_update_dpm_table &
-			(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK)) ||
-			data->apply_optimized_settings) {
-			result = vega10_populate_all_graphic_levels(hwmgr);
-			PP_ASSERT_WITH_CODE(!result,
-					"Failed to populate SCLK during PopulateNewDPMClocksStates Function!",
-					return result);
-		}
-
-		if (data->need_update_dpm_table &
-				(DPMTABLE_OD_UPDATE_MCLK + DPMTABLE_UPDATE_MCLK)) {
-			result = vega10_populate_all_memory_levels(hwmgr);
-			PP_ASSERT_WITH_CODE(!result,
-					"Failed to populate MCLK during PopulateNewDPMClocksStates Function!",
-					return result);
-		}
-	}
 	return result;
 }
 
@@ -3742,8 +3529,9 @@ static int vega10_set_power_state_tasks(struct pp_hwmgr *hwmgr,
 	PP_ASSERT_WITH_CODE(!result,
 			"Failed to upload PPtable!", return result);
 
-	data->apply_optimized_settings = false;
-	data->apply_overdrive_next_settings_mask = 0;
+	vega10_update_avfs(hwmgr);
+
+	data->need_update_dpm_table &= DPMTABLE_OD_UPDATE_VDDC;
 
 	return 0;
 }
@@ -3793,16 +3581,18 @@ static uint32_t vega10_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low)
 }
 
 static int vega10_get_gpu_power(struct pp_hwmgr *hwmgr,
-		struct pp_gpu_power *query)
+		uint32_t *query)
 {
 	uint32_t value;
 
+	if (!query)
+		return -EINVAL;
+
 	smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetCurrPkgPwr);
 	value = smum_get_argument(hwmgr);
 
-	/* power value is an integer */
-	memset(query, 0, sizeof *query);
-	query->average_gpu_power = value << 8;
+	/* SMC returning actual watts, keep consistent with legacy asics, low 8 bit as 8 fractional bits */
+	*query = value << 8;
 
 	return 0;
 }
@@ -3810,22 +3600,18 @@ static int vega10_get_gpu_power(struct pp_hwmgr *hwmgr,
 static int vega10_read_sensor(struct pp_hwmgr *hwmgr, int idx,
 			      void *value, int *size)
 {
-	uint32_t sclk_idx, mclk_idx, activity_percent = 0;
+	struct amdgpu_device *adev = hwmgr->adev;
+	uint32_t sclk_mhz, mclk_idx, activity_percent = 0;
 	struct vega10_hwmgr *data = hwmgr->backend;
 	struct vega10_dpm_table *dpm_table = &data->dpm_table;
 	int ret = 0;
-	uint32_t reg, val_vid;
+	uint32_t val_vid;
 
 	switch (idx) {
 	case AMDGPU_PP_SENSOR_GFX_SCLK:
-		smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetCurrentGfxclkIndex);
-		sclk_idx = smum_get_argument(hwmgr);
-		if (sclk_idx <  dpm_table->gfx_table.count) {
-			*((uint32_t *)value) = dpm_table->gfx_table.dpm_levels[sclk_idx].value;
-			*size = 4;
-		} else {
-			ret = -EINVAL;
-		}
+		smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetAverageGfxclkActualFrequency);
+		sclk_mhz = smum_get_argument(hwmgr);
+		*((uint32_t *)value) = sclk_mhz * 100;
 		break;
 	case AMDGPU_PP_SENSOR_GFX_MCLK:
 		smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetCurrentUclkIndex);
@@ -3856,18 +3642,10 @@ static int vega10_read_sensor(struct pp_hwmgr *hwmgr, int idx,
 		*size = 4;
 		break;
 	case AMDGPU_PP_SENSOR_GPU_POWER:
-		if (*size < sizeof(struct pp_gpu_power))
-			ret = -EINVAL;
-		else {
-			*size = sizeof(struct pp_gpu_power);
-			ret = vega10_get_gpu_power(hwmgr, (struct pp_gpu_power *)value);
-		}
+		ret = vega10_get_gpu_power(hwmgr, (uint32_t *)value);
 		break;
 	case AMDGPU_PP_SENSOR_VDDGFX:
-		reg = soc15_get_register_offset(SMUIO_HWID, 0,
-			mmSMUSVI0_PLANE0_CURRENTVID_BASE_IDX,
-			mmSMUSVI0_PLANE0_CURRENTVID);
-		val_vid = (cgs_read_register(hwmgr->device, reg) &
+		val_vid = (RREG32_SOC15(SMUIO, 0, mmSMUSVI0_PLANE0_CURRENTVID) &
 			SMUSVI0_PLANE0_CURRENTVID__CURRENT_SVI0_PLANE0_VID_MASK) >>
 			SMUSVI0_PLANE0_CURRENTVID__CURRENT_SVI0_PLANE0_VID__SHIFT;
 		*((uint32_t *)value) = (uint32_t)convert_to_vddc((uint8_t)val_vid);
@@ -3956,26 +3734,18 @@ static int vega10_notify_smc_display_config_after_ps_adjustment(
 			(struct phm_ppt_v2_information *)hwmgr->pptable;
 	struct phm_ppt_v1_clock_voltage_dependency_table *mclk_table = table_info->vdd_dep_on_mclk;
 	uint32_t idx;
-	uint32_t num_active_disps = 0;
-	struct cgs_display_info info = {0};
 	struct PP_Clocks min_clocks = {0};
 	uint32_t i;
 	struct pp_display_clock_request clock_req;
 
-	info.mode_info = NULL;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	num_active_disps = info.display_count;
-
-	if (num_active_disps > 1)
+	if (hwmgr->display_config->num_display > 1)
 		vega10_notify_smc_display_change(hwmgr, false);
 	else
 		vega10_notify_smc_display_change(hwmgr, true);
 
-	min_clocks.dcefClock = hwmgr->display_config.min_dcef_set_clk;
-	min_clocks.dcefClockInSR = hwmgr->display_config.min_dcef_deep_sleep_set_clk;
-	min_clocks.memoryClock = hwmgr->display_config.min_mem_set_clock;
+	min_clocks.dcefClock = hwmgr->display_config->min_dcef_set_clk;
+	min_clocks.dcefClockInSR = hwmgr->display_config->min_dcef_deep_sleep_set_clk;
+	min_clocks.memoryClock = hwmgr->display_config->min_mem_set_clock;
 
 	for (i = 0; i < dpm_table->count; i++) {
 		if (dpm_table->dpm_levels[i].value == min_clocks.dcefClock)
@@ -4120,6 +3890,47 @@ static void vega10_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
 	}
 }
 
+static int vega10_force_clock_level(struct pp_hwmgr *hwmgr,
+		enum pp_clock_type type, uint32_t mask)
+{
+	struct vega10_hwmgr *data = hwmgr->backend;
+
+	switch (type) {
+	case PP_SCLK:
+		data->smc_state_table.gfx_boot_level = mask ? (ffs(mask) - 1) : 0;
+		data->smc_state_table.gfx_max_level = mask ? (fls(mask) - 1) : 0;
+
+		PP_ASSERT_WITH_CODE(!vega10_upload_dpm_bootup_level(hwmgr),
+			"Failed to upload boot level to lowest!",
+			return -EINVAL);
+
+		PP_ASSERT_WITH_CODE(!vega10_upload_dpm_max_level(hwmgr),
+			"Failed to upload dpm max level to highest!",
+			return -EINVAL);
+		break;
+
+	case PP_MCLK:
+		data->smc_state_table.mem_boot_level = mask ? (ffs(mask) - 1) : 0;
+		data->smc_state_table.mem_max_level = mask ? (fls(mask) - 1) : 0;
+
+		PP_ASSERT_WITH_CODE(!vega10_upload_dpm_bootup_level(hwmgr),
+			"Failed to upload boot level to lowest!",
+			return -EINVAL);
+
+		PP_ASSERT_WITH_CODE(!vega10_upload_dpm_max_level(hwmgr),
+			"Failed to upload dpm max level to highest!",
+			return -EINVAL);
+
+		break;
+
+	case PP_PCIE:
+	default:
+		break;
+	}
+
+	return 0;
+}
+
 static int vega10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 				enum amd_dpm_forced_level level)
 {
@@ -4356,97 +4167,15 @@ static int vega10_set_watermarks_for_clocks_ranges(struct pp_hwmgr *hwmgr,
 	struct vega10_hwmgr *data = hwmgr->backend;
 	Watermarks_t *table = &(data->smc_state_table.water_marks_table);
 	int result = 0;
-	uint32_t i;
 
 	if (!data->registry_data.disable_water_mark) {
-		for (i = 0; i < wm_with_clock_ranges->num_wm_sets_dmif; i++) {
-			table->WatermarkRow[WM_DCEFCLK][i].MinClock =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_dmif[i].wm_min_dcefclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_DCEFCLK][i].MaxClock =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_dmif[i].wm_max_dcefclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_DCEFCLK][i].MinUclk =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_dmif[i].wm_min_memclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_DCEFCLK][i].MaxUclk =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_dmif[i].wm_max_memclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_DCEFCLK][i].WmSetting = (uint8_t)
-					wm_with_clock_ranges->wm_sets_dmif[i].wm_set_id;
-		}
-
-		for (i = 0; i < wm_with_clock_ranges->num_wm_sets_mcif; i++) {
-			table->WatermarkRow[WM_SOCCLK][i].MinClock =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_mcif[i].wm_min_socclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_SOCCLK][i].MaxClock =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_mcif[i].wm_max_socclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_SOCCLK][i].MinUclk =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_mcif[i].wm_min_memclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_SOCCLK][i].MaxUclk =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_mcif[i].wm_max_memclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_SOCCLK][i].WmSetting = (uint8_t)
-					wm_with_clock_ranges->wm_sets_mcif[i].wm_set_id;
-		}
+		smu_set_watermarks_for_clocks_ranges(table, wm_with_clock_ranges);
 		data->water_marks_bitmap = WaterMarksExist;
 	}
 
 	return result;
 }
 
-static int vega10_force_clock_level(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, uint32_t mask)
-{
-	struct vega10_hwmgr *data = hwmgr->backend;
-
-	switch (type) {
-	case PP_SCLK:
-		data->smc_state_table.gfx_boot_level = mask ? (ffs(mask) - 1) : 0;
-		data->smc_state_table.gfx_max_level = mask ? (fls(mask) - 1) : 0;
-
-		PP_ASSERT_WITH_CODE(!vega10_upload_dpm_bootup_level(hwmgr),
-			"Failed to upload boot level to lowest!",
-			return -EINVAL);
-
-		PP_ASSERT_WITH_CODE(!vega10_upload_dpm_max_level(hwmgr),
-			"Failed to upload dpm max level to highest!",
-			return -EINVAL);
-		break;
-
-	case PP_MCLK:
-		data->smc_state_table.mem_boot_level = mask ? (ffs(mask) - 1) : 0;
-		data->smc_state_table.mem_max_level = mask ? (fls(mask) - 1) : 0;
-
-		PP_ASSERT_WITH_CODE(!vega10_upload_dpm_bootup_level(hwmgr),
-			"Failed to upload boot level to lowest!",
-			return -EINVAL);
-
-		PP_ASSERT_WITH_CODE(!vega10_upload_dpm_max_level(hwmgr),
-			"Failed to upload dpm max level to highest!",
-			return -EINVAL);
-
-		break;
-
-	case PP_PCIE:
-	default:
-		break;
-	}
-
-	return 0;
-}
-
 static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
 		enum pp_clock_type type, char *buf)
 {
@@ -4454,6 +4183,8 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
 	struct vega10_single_dpm_table *sclk_table = &(data->dpm_table.gfx_table);
 	struct vega10_single_dpm_table *mclk_table = &(data->dpm_table.mem_table);
 	struct vega10_pcie_table *pcie_table = &(data->dpm_table.pcie_table);
+	struct vega10_odn_clock_voltage_dependency_table *podn_vdd_dep = NULL;
+
 	int i, now, size = 0;
 
 	switch (type) {
@@ -4492,6 +4223,40 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
 					(pcie_table->pcie_gen[i] == 2) ? "8.0GT/s, x16" : "",
 					(i == now) ? "*" : "");
 		break;
+	case OD_SCLK:
+		if (hwmgr->od_enabled) {
+			size = sprintf(buf, "%s:\n", "OD_SCLK");
+			podn_vdd_dep = &data->odn_dpm_table.vdd_dep_on_sclk;
+			for (i = 0; i < podn_vdd_dep->count; i++)
+				size += sprintf(buf + size, "%d: %10uMhz %10umV\n",
+					i, podn_vdd_dep->entries[i].clk / 100,
+						podn_vdd_dep->entries[i].vddc);
+		}
+		break;
+	case OD_MCLK:
+		if (hwmgr->od_enabled) {
+			size = sprintf(buf, "%s:\n", "OD_MCLK");
+			podn_vdd_dep = &data->odn_dpm_table.vdd_dep_on_mclk;
+			for (i = 0; i < podn_vdd_dep->count; i++)
+				size += sprintf(buf + size, "%d: %10uMhz %10umV\n",
+					i, podn_vdd_dep->entries[i].clk/100,
+						podn_vdd_dep->entries[i].vddc);
+		}
+		break;
+	case OD_RANGE:
+		if (hwmgr->od_enabled) {
+			size = sprintf(buf, "%s:\n", "OD_RANGE");
+			size += sprintf(buf + size, "SCLK: %7uMHz %10uMHz\n",
+				data->golden_dpm_table.gfx_table.dpm_levels[0].value/100,
+				hwmgr->platform_descriptor.overdriveLimit.engineClock/100);
+			size += sprintf(buf + size, "MCLK: %7uMHz %10uMHz\n",
+				data->golden_dpm_table.mem_table.dpm_levels[0].value/100,
+				hwmgr->platform_descriptor.overdriveLimit.memoryClock/100);
+			size += sprintf(buf + size, "VDDC: %7umV %11umV\n",
+				data->odn_dpm_table.min_vddc,
+				data->odn_dpm_table.max_vddc);
+		}
+		break;
 	default:
 		break;
 	}
@@ -4501,10 +4266,8 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
 static int vega10_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
 {
 	struct vega10_hwmgr *data = hwmgr->backend;
-	int result = 0;
-	uint32_t num_turned_on_displays = 1;
 	Watermarks_t *wm_table = &(data->smc_state_table.water_marks_table);
-	struct cgs_display_info info = {0};
+	int result = 0;
 
 	if ((data->water_marks_bitmap & WaterMarksExist) &&
 			!(data->water_marks_bitmap & WaterMarksLoaded)) {
@@ -4514,10 +4277,8 @@ static int vega10_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
 	}
 
 	if (data->water_marks_bitmap & WaterMarksLoaded) {
-		cgs_get_active_displays_info(hwmgr->device, &info);
-		num_turned_on_displays = info.display_count;
 		smum_send_msg_to_smc_with_parameter(hwmgr,
-			PPSMC_MSG_NumOfDisplays, num_turned_on_displays);
+			PPSMC_MSG_NumOfDisplays, hwmgr->display_config->num_display);
 	}
 
 	return result;
@@ -4603,15 +4364,12 @@ vega10_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmg
 {
 	struct vega10_hwmgr *data = hwmgr->backend;
 	bool is_update_required = false;
-	struct cgs_display_info info = {0, 0, NULL};
 
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	if (data->display_timing.num_existing_displays != info.display_count)
+	if (data->display_timing.num_existing_displays != hwmgr->display_config->num_display)
 		is_update_required = true;
 
 	if (PP_CAP(PHM_PlatformCaps_SclkDeepSleep)) {
-		if (data->display_timing.min_clock_in_sr != hwmgr->display_config.min_core_set_clock_in_sr)
+		if (data->display_timing.min_clock_in_sr != hwmgr->display_config->min_core_set_clock_in_sr)
 			is_update_required = true;
 	}
 
@@ -4886,6 +4644,200 @@ static int vega10_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, ui
 	return 0;
 }
 
+
+static bool vega10_check_clk_voltage_valid(struct pp_hwmgr *hwmgr,
+					enum PP_OD_DPM_TABLE_COMMAND type,
+					uint32_t clk,
+					uint32_t voltage)
+{
+	struct vega10_hwmgr *data = hwmgr->backend;
+	struct vega10_odn_dpm_table *odn_table = &(data->odn_dpm_table);
+	struct vega10_single_dpm_table *golden_table;
+
+	if (voltage < odn_table->min_vddc || voltage > odn_table->max_vddc) {
+		pr_info("OD voltage is out of range [%d - %d] mV\n", odn_table->min_vddc, odn_table->max_vddc);
+		return false;
+	}
+
+	if (type == PP_OD_EDIT_SCLK_VDDC_TABLE) {
+		golden_table = &(data->golden_dpm_table.gfx_table);
+		if (golden_table->dpm_levels[0].value > clk ||
+			hwmgr->platform_descriptor.overdriveLimit.engineClock < clk) {
+			pr_info("OD engine clock is out of range [%d - %d] MHz\n",
+				golden_table->dpm_levels[0].value/100,
+				hwmgr->platform_descriptor.overdriveLimit.engineClock/100);
+			return false;
+		}
+	} else if (type == PP_OD_EDIT_MCLK_VDDC_TABLE) {
+		golden_table = &(data->golden_dpm_table.mem_table);
+		if (golden_table->dpm_levels[0].value > clk ||
+			hwmgr->platform_descriptor.overdriveLimit.memoryClock < clk) {
+			pr_info("OD memory clock is out of range [%d - %d] MHz\n",
+				golden_table->dpm_levels[0].value/100,
+				hwmgr->platform_descriptor.overdriveLimit.memoryClock/100);
+			return false;
+		}
+	} else {
+		return false;
+	}
+
+	return true;
+}
+
+static void vega10_check_dpm_table_updated(struct pp_hwmgr *hwmgr)
+{
+	struct vega10_hwmgr *data = hwmgr->backend;
+	struct vega10_odn_dpm_table *odn_table = &(data->odn_dpm_table);
+	struct phm_ppt_v2_information *table_info = hwmgr->pptable;
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_table;
+	struct phm_ppt_v1_clock_voltage_dependency_table *odn_dep_table;
+	uint32_t i;
+
+	dep_table = table_info->vdd_dep_on_mclk;
+	odn_dep_table = (struct phm_ppt_v1_clock_voltage_dependency_table *)&(odn_table->vdd_dep_on_mclk);
+
+	for (i = 0; i < dep_table->count; i++) {
+		if (dep_table->entries[i].vddc != odn_dep_table->entries[i].vddc) {
+			data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_VDDC | DPMTABLE_OD_UPDATE_MCLK;
+			return;
+		}
+	}
+
+	dep_table = table_info->vdd_dep_on_sclk;
+	odn_dep_table = (struct phm_ppt_v1_clock_voltage_dependency_table *)&(odn_table->vdd_dep_on_sclk);
+	for (i = 0; i < dep_table->count; i++) {
+		if (dep_table->entries[i].vddc != odn_dep_table->entries[i].vddc) {
+			data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_VDDC | DPMTABLE_OD_UPDATE_SCLK;
+			return;
+		}
+	}
+
+	if (data->need_update_dpm_table & DPMTABLE_OD_UPDATE_VDDC) {
+		data->need_update_dpm_table &= ~DPMTABLE_OD_UPDATE_VDDC;
+		data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_OD_UPDATE_MCLK;
+	}
+}
+
+static void vega10_odn_update_soc_table(struct pp_hwmgr *hwmgr,
+						enum PP_OD_DPM_TABLE_COMMAND type)
+{
+	struct vega10_hwmgr *data = hwmgr->backend;
+	struct phm_ppt_v2_information *table_info = hwmgr->pptable;
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_table = table_info->vdd_dep_on_socclk;
+	struct vega10_single_dpm_table *dpm_table = &data->golden_dpm_table.soc_table;
+
+	struct vega10_odn_clock_voltage_dependency_table *podn_vdd_dep_on_socclk =
+							&data->odn_dpm_table.vdd_dep_on_socclk;
+	struct vega10_odn_vddc_lookup_table *od_vddc_lookup_table = &data->odn_dpm_table.vddc_lookup_table;
+
+	struct vega10_odn_clock_voltage_dependency_table *podn_vdd_dep;
+	uint8_t i, j;
+
+	if (type == PP_OD_EDIT_SCLK_VDDC_TABLE) {
+		podn_vdd_dep = &data->odn_dpm_table.vdd_dep_on_sclk;
+		for (i = 0; i < podn_vdd_dep->count - 1; i++)
+			od_vddc_lookup_table->entries[i].us_vdd = podn_vdd_dep->entries[i].vddc;
+		if (od_vddc_lookup_table->entries[i].us_vdd < podn_vdd_dep->entries[i].vddc)
+			od_vddc_lookup_table->entries[i].us_vdd = podn_vdd_dep->entries[i].vddc;
+	} else if (type == PP_OD_EDIT_MCLK_VDDC_TABLE) {
+		podn_vdd_dep = &data->odn_dpm_table.vdd_dep_on_mclk;
+		for (i = 0; i < dpm_table->count; i++) {
+			for (j = 0; j < od_vddc_lookup_table->count; j++) {
+				if (od_vddc_lookup_table->entries[j].us_vdd >
+					podn_vdd_dep->entries[i].vddc)
+					break;
+			}
+			if (j == od_vddc_lookup_table->count) {
+				od_vddc_lookup_table->entries[j-1].us_vdd =
+					podn_vdd_dep->entries[i].vddc;
+				data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_VDDC;
+			}
+			podn_vdd_dep->entries[i].vddInd = j;
+		}
+		dpm_table = &data->dpm_table.soc_table;
+		for (i = 0; i < dep_table->count; i++) {
+			if (dep_table->entries[i].vddInd == podn_vdd_dep->entries[dep_table->count-1].vddInd &&
+					dep_table->entries[i].clk < podn_vdd_dep->entries[dep_table->count-1].clk) {
+				data->need_update_dpm_table |= DPMTABLE_UPDATE_SOCCLK;
+				podn_vdd_dep_on_socclk->entries[i].clk = podn_vdd_dep->entries[dep_table->count-1].clk;
+				dpm_table->dpm_levels[i].value = podn_vdd_dep_on_socclk->entries[i].clk;
+			}
+		}
+		if (podn_vdd_dep_on_socclk->entries[podn_vdd_dep_on_socclk->count - 1].clk <
+					podn_vdd_dep->entries[dep_table->count-1].clk) {
+			data->need_update_dpm_table |= DPMTABLE_UPDATE_SOCCLK;
+			podn_vdd_dep_on_socclk->entries[podn_vdd_dep_on_socclk->count - 1].clk = podn_vdd_dep->entries[dep_table->count-1].clk;
+			dpm_table->dpm_levels[podn_vdd_dep_on_socclk->count - 1].value = podn_vdd_dep->entries[dep_table->count-1].clk;
+		}
+		if (podn_vdd_dep_on_socclk->entries[podn_vdd_dep_on_socclk->count - 1].vddInd <
+					podn_vdd_dep->entries[dep_table->count-1].vddInd) {
+			data->need_update_dpm_table |= DPMTABLE_UPDATE_SOCCLK;
+			podn_vdd_dep_on_socclk->entries[podn_vdd_dep_on_socclk->count - 1].vddInd = podn_vdd_dep->entries[dep_table->count-1].vddInd;
+		}
+	}
+}
+
+static int vega10_odn_edit_dpm_table(struct pp_hwmgr *hwmgr,
+					enum PP_OD_DPM_TABLE_COMMAND type,
+					long *input, uint32_t size)
+{
+	struct vega10_hwmgr *data = hwmgr->backend;
+	struct vega10_odn_clock_voltage_dependency_table *podn_vdd_dep_table;
+	struct vega10_single_dpm_table *dpm_table;
+
+	uint32_t input_clk;
+	uint32_t input_vol;
+	uint32_t input_level;
+	uint32_t i;
+
+	PP_ASSERT_WITH_CODE(input, "NULL user input for clock and voltage",
+				return -EINVAL);
+
+	if (!hwmgr->od_enabled) {
+		pr_info("OverDrive feature not enabled\n");
+		return -EINVAL;
+	}
+
+	if (PP_OD_EDIT_SCLK_VDDC_TABLE == type) {
+		dpm_table = &data->dpm_table.gfx_table;
+		podn_vdd_dep_table = &data->odn_dpm_table.vdd_dep_on_sclk;
+		data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
+	} else if (PP_OD_EDIT_MCLK_VDDC_TABLE == type) {
+		dpm_table = &data->dpm_table.mem_table;
+		podn_vdd_dep_table = &data->odn_dpm_table.vdd_dep_on_mclk;
+		data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
+	} else if (PP_OD_RESTORE_DEFAULT_TABLE == type) {
+		memcpy(&(data->dpm_table), &(data->golden_dpm_table), sizeof(struct vega10_dpm_table));
+		vega10_odn_initial_default_setting(hwmgr);
+		return 0;
+	} else if (PP_OD_COMMIT_DPM_TABLE == type) {
+		vega10_check_dpm_table_updated(hwmgr);
+		return 0;
+	} else {
+		return -EINVAL;
+	}
+
+	for (i = 0; i < size; i += 3) {
+		if (i + 3 > size || input[i] >= podn_vdd_dep_table->count) {
+			pr_info("invalid clock voltage input\n");
+			return 0;
+		}
+		input_level = input[i];
+		input_clk = input[i+1] * 100;
+		input_vol = input[i+2];
+
+		if (vega10_check_clk_voltage_valid(hwmgr, type, input_clk, input_vol)) {
+			dpm_table->dpm_levels[input_level].value = input_clk;
+			podn_vdd_dep_table->entries[input_level].clk = input_clk;
+			podn_vdd_dep_table->entries[input_level].vddc = input_vol;
+		} else {
+			return -EINVAL;
+		}
+	}
+	vega10_odn_update_soc_table(hwmgr, type);
+	return 0;
+}
+
 static const struct pp_hwmgr_func vega10_hwmgr_funcs = {
 	.backend_init = vega10_hwmgr_backend_init,
 	.backend_fini = vega10_hwmgr_backend_fini,
@@ -4944,6 +4896,7 @@ static const struct pp_hwmgr_func vega10_hwmgr_funcs = {
 	.get_power_profile_mode = vega10_get_power_profile_mode,
 	.set_power_profile_mode = vega10_set_power_profile_mode,
 	.set_power_limit = vega10_set_power_limit,
+	.odn_edit_dpm_table = vega10_odn_edit_dpm_table,
 };
 
 int vega10_enable_smc_features(struct pp_hwmgr *hwmgr,
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h
index 5339ea1f3dce..aadd6cbc7e85 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h
@@ -282,15 +282,21 @@ struct vega10_registry_data {
 
 struct vega10_odn_clock_voltage_dependency_table {
 	uint32_t count;
-	struct phm_ppt_v1_clock_voltage_dependency_record
-		entries[MAX_REGULAR_DPM_NUMBER];
+	struct phm_ppt_v1_clock_voltage_dependency_record entries[MAX_REGULAR_DPM_NUMBER];
+};
+
+struct vega10_odn_vddc_lookup_table {
+	uint32_t count;
+	struct phm_ppt_v1_voltage_lookup_record entries[MAX_REGULAR_DPM_NUMBER];
 };
 
 struct vega10_odn_dpm_table {
-	struct phm_odn_clock_levels		odn_core_clock_dpm_levels;
-	struct phm_odn_clock_levels		odn_memory_clock_dpm_levels;
-	struct vega10_odn_clock_voltage_dependency_table		vdd_dependency_on_sclk;
-	struct vega10_odn_clock_voltage_dependency_table		vdd_dependency_on_mclk;
+	struct vega10_odn_clock_voltage_dependency_table vdd_dep_on_sclk;
+	struct vega10_odn_clock_voltage_dependency_table vdd_dep_on_mclk;
+	struct vega10_odn_clock_voltage_dependency_table vdd_dep_on_socclk;
+	struct vega10_odn_vddc_lookup_table vddc_lookup_table;
+	uint32_t max_vddc;
+	uint32_t min_vddc;
 };
 
 struct vega10_odn_fan_table {
@@ -301,8 +307,8 @@ struct vega10_odn_fan_table {
 };
 
 struct vega10_hwmgr {
-	struct vega10_dpm_table			dpm_table;
-	struct vega10_dpm_table			golden_dpm_table;
+	struct vega10_dpm_table          dpm_table;
+	struct vega10_dpm_table          golden_dpm_table;
 	struct vega10_registry_data      registry_data;
 	struct vega10_vbios_boot_state   vbios_boot_state;
 	struct vega10_mclk_latency_table mclk_latency_table;
@@ -368,12 +374,8 @@ struct vega10_hwmgr {
 	bool                           need_long_memory_training;
 
 	/* Internal settings to apply the application power optimization parameters */
-	bool                           apply_optimized_settings;
 	uint32_t                       disable_dpm_mask;
 
-	/* ---- Overdrive next setting ---- */
-	uint32_t                       apply_overdrive_next_settings_mask;
-
 	/* ---- SMU9 ---- */
 	struct smu_features            smu_features[GNLD_FEATURES_MAX];
 	struct vega10_smc_state_table  smc_state_table;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c
index ba63faefc61f..a9efd8554fbc 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c
@@ -27,7 +27,7 @@
 #include "vega10_ppsmc.h"
 #include "vega10_inc.h"
 #include "pp_debug.h"
-#include "pp_soc15.h"
+#include "soc15_common.h"
 
 static const struct vega10_didt_config_reg SEDiDtTuningCtrlConfig_Vega10[] =
 {
@@ -888,36 +888,36 @@ static void vega10_didt_set_mask(struct pp_hwmgr *hwmgr, const bool enable)
 	if (PP_CAP(PHM_PlatformCaps_DiDtEDCEnable)) {
 		if (PP_CAP(PHM_PlatformCaps_SQRamping)) {
 			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_EDC_CTRL);
-			data = CGS_REG_SET_FIELD(data, DIDT_SQ_EDC_CTRL, EDC_EN, en);
-			data = CGS_REG_SET_FIELD(data, DIDT_SQ_EDC_CTRL, EDC_SW_RST, ~en);
+			data = REG_SET_FIELD(data, DIDT_SQ_EDC_CTRL, EDC_EN, en);
+			data = REG_SET_FIELD(data, DIDT_SQ_EDC_CTRL, EDC_SW_RST, ~en);
 			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_EDC_CTRL, data);
 		}
 
 		if (PP_CAP(PHM_PlatformCaps_DBRamping)) {
 			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_EDC_CTRL);
-			data = CGS_REG_SET_FIELD(data, DIDT_DB_EDC_CTRL, EDC_EN, en);
-			data = CGS_REG_SET_FIELD(data, DIDT_DB_EDC_CTRL, EDC_SW_RST, ~en);
+			data = REG_SET_FIELD(data, DIDT_DB_EDC_CTRL, EDC_EN, en);
+			data = REG_SET_FIELD(data, DIDT_DB_EDC_CTRL, EDC_SW_RST, ~en);
 			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_EDC_CTRL, data);
 		}
 
 		if (PP_CAP(PHM_PlatformCaps_TDRamping)) {
 			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_EDC_CTRL);
-			data = CGS_REG_SET_FIELD(data, DIDT_TD_EDC_CTRL, EDC_EN, en);
-			data = CGS_REG_SET_FIELD(data, DIDT_TD_EDC_CTRL, EDC_SW_RST, ~en);
+			data = REG_SET_FIELD(data, DIDT_TD_EDC_CTRL, EDC_EN, en);
+			data = REG_SET_FIELD(data, DIDT_TD_EDC_CTRL, EDC_SW_RST, ~en);
 			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_EDC_CTRL, data);
 		}
 
 		if (PP_CAP(PHM_PlatformCaps_TCPRamping)) {
 			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_EDC_CTRL);
-			data = CGS_REG_SET_FIELD(data, DIDT_TCP_EDC_CTRL, EDC_EN, en);
-			data = CGS_REG_SET_FIELD(data, DIDT_TCP_EDC_CTRL, EDC_SW_RST, ~en);
+			data = REG_SET_FIELD(data, DIDT_TCP_EDC_CTRL, EDC_EN, en);
+			data = REG_SET_FIELD(data, DIDT_TCP_EDC_CTRL, EDC_SW_RST, ~en);
 			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_EDC_CTRL, data);
 		}
 
 		if (PP_CAP(PHM_PlatformCaps_DBRRamping)) {
 			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DBR_EDC_CTRL);
-			data = CGS_REG_SET_FIELD(data, DIDT_DBR_EDC_CTRL, EDC_EN, en);
-			data = CGS_REG_SET_FIELD(data, DIDT_DBR_EDC_CTRL, EDC_SW_RST, ~en);
+			data = REG_SET_FIELD(data, DIDT_DBR_EDC_CTRL, EDC_EN, en);
+			data = REG_SET_FIELD(data, DIDT_DBR_EDC_CTRL, EDC_SW_RST, ~en);
 			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DBR_EDC_CTRL, data);
 		}
 	}
@@ -930,20 +930,18 @@ static void vega10_didt_set_mask(struct pp_hwmgr *hwmgr, const bool enable)
 
 static int vega10_enable_cac_driving_se_didt_config(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	int result;
 	uint32_t num_se = 0, count, data;
-	struct amdgpu_device *adev = hwmgr->adev;
-	uint32_t reg;
 
 	num_se = adev->gfx.config.max_shader_engines;
 
-	cgs_enter_safe_mode(hwmgr->device, true);
+	adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
-	cgs_lock_grbm_idx(hwmgr->device, true);
-	reg = soc15_get_register_offset(GC_HWID, 0, mmGRBM_GFX_INDEX_BASE_IDX, mmGRBM_GFX_INDEX);
+	mutex_lock(&adev->grbm_idx_mutex);
 	for (count = 0; count < num_se; count++) {
 		data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | ( count << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
-		cgs_write_register(hwmgr->device, reg, data);
+		WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, data);
 
 		result =  vega10_program_didt_config_registers(hwmgr, SEDiDtStallCtrlConfig_vega10, VEGA10_CONFIGREG_DIDT);
 		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtStallPatternConfig_vega10, VEGA10_CONFIGREG_DIDT);
@@ -958,43 +956,43 @@ static int vega10_enable_cac_driving_se_didt_config(struct pp_hwmgr *hwmgr)
 		if (0 != result)
 			break;
 	}
-	cgs_write_register(hwmgr->device, reg, 0xE0000000);
-	cgs_lock_grbm_idx(hwmgr->device, false);
+	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xE0000000);
+	mutex_unlock(&adev->grbm_idx_mutex);
 
 	vega10_didt_set_mask(hwmgr, true);
 
-	cgs_enter_safe_mode(hwmgr->device, false);
+	adev->gfx.rlc.funcs->exit_safe_mode(adev);
 
 	return 0;
 }
 
 static int vega10_disable_cac_driving_se_didt_config(struct pp_hwmgr *hwmgr)
 {
-	cgs_enter_safe_mode(hwmgr->device, true);
+	struct amdgpu_device *adev = hwmgr->adev;
+
+	adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
 	vega10_didt_set_mask(hwmgr, false);
 
-	cgs_enter_safe_mode(hwmgr->device, false);
+	adev->gfx.rlc.funcs->exit_safe_mode(adev);
 
 	return 0;
 }
 
 static int vega10_enable_psm_gc_didt_config(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	int result;
 	uint32_t num_se = 0, count, data;
-	struct amdgpu_device *adev = hwmgr->adev;
-	uint32_t reg;
 
 	num_se = adev->gfx.config.max_shader_engines;
 
-	cgs_enter_safe_mode(hwmgr->device, true);
+	adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
-	cgs_lock_grbm_idx(hwmgr->device, true);
-	reg = soc15_get_register_offset(GC_HWID, 0, mmGRBM_GFX_INDEX_BASE_IDX, mmGRBM_GFX_INDEX);
+	mutex_lock(&adev->grbm_idx_mutex);
 	for (count = 0; count < num_se; count++) {
 		data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | ( count << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
-		cgs_write_register(hwmgr->device, reg, data);
+		WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, data);
 
 		result = vega10_program_didt_config_registers(hwmgr, SEDiDtStallCtrlConfig_vega10, VEGA10_CONFIGREG_DIDT);
 		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtStallPatternConfig_vega10, VEGA10_CONFIGREG_DIDT);
@@ -1003,12 +1001,12 @@ static int vega10_enable_psm_gc_didt_config(struct pp_hwmgr *hwmgr)
 		if (0 != result)
 			break;
 	}
-	cgs_write_register(hwmgr->device, reg, 0xE0000000);
-	cgs_lock_grbm_idx(hwmgr->device, false);
+	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xE0000000);
+	mutex_unlock(&adev->grbm_idx_mutex);
 
 	vega10_didt_set_mask(hwmgr, true);
 
-	cgs_enter_safe_mode(hwmgr->device, false);
+	adev->gfx.rlc.funcs->exit_safe_mode(adev);
 
 	vega10_program_gc_didt_config_registers(hwmgr, GCDiDtDroopCtrlConfig_vega10);
 	if (PP_CAP(PHM_PlatformCaps_GCEDC))
@@ -1022,13 +1020,14 @@ static int vega10_enable_psm_gc_didt_config(struct pp_hwmgr *hwmgr)
 
 static int vega10_disable_psm_gc_didt_config(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	uint32_t data;
 
-	cgs_enter_safe_mode(hwmgr->device, true);
+	adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
 	vega10_didt_set_mask(hwmgr, false);
 
-	cgs_enter_safe_mode(hwmgr->device, false);
+	adev->gfx.rlc.funcs->exit_safe_mode(adev);
 
 	if (PP_CAP(PHM_PlatformCaps_GCEDC)) {
 		data = 0x00000000;
@@ -1043,20 +1042,18 @@ static int vega10_disable_psm_gc_didt_config(struct pp_hwmgr *hwmgr)
 
 static int vega10_enable_se_edc_config(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	int result;
 	uint32_t num_se = 0, count, data;
-	struct amdgpu_device *adev = hwmgr->adev;
-	uint32_t reg;
 
 	num_se = adev->gfx.config.max_shader_engines;
 
-	cgs_enter_safe_mode(hwmgr->device, true);
+	adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
-	cgs_lock_grbm_idx(hwmgr->device, true);
-	reg = soc15_get_register_offset(GC_HWID, 0, mmGRBM_GFX_INDEX_BASE_IDX, mmGRBM_GFX_INDEX);
+	mutex_lock(&adev->grbm_idx_mutex);
 	for (count = 0; count < num_se; count++) {
 		data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | ( count << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
-		cgs_write_register(hwmgr->device, reg, data);
+		WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, data);
 		result = vega10_program_didt_config_registers(hwmgr, SEDiDtWeightConfig_Vega10, VEGA10_CONFIGREG_DIDT);
 		result |= vega10_program_didt_config_registers(hwmgr, SEEDCStallPatternConfig_Vega10, VEGA10_CONFIGREG_DIDT);
 		result |= vega10_program_didt_config_registers(hwmgr, SEEDCStallDelayConfig_Vega10, VEGA10_CONFIGREG_DIDT);
@@ -1067,46 +1064,46 @@ static int vega10_enable_se_edc_config(struct pp_hwmgr *hwmgr)
 		if (0 != result)
 			break;
 	}
-	cgs_write_register(hwmgr->device, reg, 0xE0000000);
-	cgs_lock_grbm_idx(hwmgr->device, false);
+	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xE0000000);
+	mutex_unlock(&adev->grbm_idx_mutex);
 
 	vega10_didt_set_mask(hwmgr, true);
 
-	cgs_enter_safe_mode(hwmgr->device, false);
+	adev->gfx.rlc.funcs->exit_safe_mode(adev);
 
 	return 0;
 }
 
 static int vega10_disable_se_edc_config(struct pp_hwmgr *hwmgr)
 {
-	cgs_enter_safe_mode(hwmgr->device, true);
+	struct amdgpu_device *adev = hwmgr->adev;
+
+	adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
 	vega10_didt_set_mask(hwmgr, false);
 
-	cgs_enter_safe_mode(hwmgr->device, false);
+	adev->gfx.rlc.funcs->exit_safe_mode(adev);
 
 	return 0;
 }
 
 static int vega10_enable_psm_gc_edc_config(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	int result;
 	uint32_t num_se = 0;
 	uint32_t count, data;
-	struct amdgpu_device *adev = hwmgr->adev;
-	uint32_t reg;
 
 	num_se = adev->gfx.config.max_shader_engines;
 
-	cgs_enter_safe_mode(hwmgr->device, true);
+	adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
 	vega10_program_gc_didt_config_registers(hwmgr, AvfsPSMResetConfig_vega10);
 
-	cgs_lock_grbm_idx(hwmgr->device, true);
-	reg = soc15_get_register_offset(GC_HWID, 0, mmGRBM_GFX_INDEX_BASE_IDX, mmGRBM_GFX_INDEX);
+	mutex_lock(&adev->grbm_idx_mutex);
 	for (count = 0; count < num_se; count++) {
 		data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | ( count << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
-		cgs_write_register(hwmgr->device, reg, data);
+		WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, data);
 		result |= vega10_program_didt_config_registers(hwmgr, PSMSEEDCStallPatternConfig_Vega10, VEGA10_CONFIGREG_DIDT);
 		result |= vega10_program_didt_config_registers(hwmgr, PSMSEEDCStallDelayConfig_Vega10, VEGA10_CONFIGREG_DIDT);
 		result |= vega10_program_didt_config_registers(hwmgr, PSMSEEDCCtrlResetConfig_Vega10, VEGA10_CONFIGREG_DIDT);
@@ -1115,12 +1112,12 @@ static int vega10_enable_psm_gc_edc_config(struct pp_hwmgr *hwmgr)
 		if (0 != result)
 			break;
 	}
-	cgs_write_register(hwmgr->device, reg, 0xE0000000);
-	cgs_lock_grbm_idx(hwmgr->device, false);
+	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xE0000000);
+	mutex_unlock(&adev->grbm_idx_mutex);
 
 	vega10_didt_set_mask(hwmgr, true);
 
-	cgs_enter_safe_mode(hwmgr->device, false);
+	adev->gfx.rlc.funcs->exit_safe_mode(adev);
 
 	vega10_program_gc_didt_config_registers(hwmgr, PSMGCEDCDroopCtrlConfig_vega10);
 
@@ -1137,13 +1134,14 @@ static int vega10_enable_psm_gc_edc_config(struct pp_hwmgr *hwmgr)
 
 static int vega10_disable_psm_gc_edc_config(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	uint32_t data;
 
-	cgs_enter_safe_mode(hwmgr->device, true);
+	adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
 	vega10_didt_set_mask(hwmgr, false);
 
-	cgs_enter_safe_mode(hwmgr->device, false);
+	adev->gfx.rlc.funcs->exit_safe_mode(adev);
 
 	if (PP_CAP(PHM_PlatformCaps_GCEDC)) {
 		data = 0x00000000;
@@ -1158,15 +1156,14 @@ static int vega10_disable_psm_gc_edc_config(struct pp_hwmgr *hwmgr)
 
 static int vega10_enable_se_edc_force_stall_config(struct pp_hwmgr *hwmgr)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 	int result;
 
-	cgs_enter_safe_mode(hwmgr->device, true);
+	adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
-	cgs_lock_grbm_idx(hwmgr->device, true);
-	reg = soc15_get_register_offset(GC_HWID, 0, mmGRBM_GFX_INDEX_BASE_IDX, mmGRBM_GFX_INDEX);
-	cgs_write_register(hwmgr->device, reg, 0xE0000000);
-	cgs_lock_grbm_idx(hwmgr->device, false);
+	mutex_lock(&adev->grbm_idx_mutex);
+	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xE0000000);
+	mutex_unlock(&adev->grbm_idx_mutex);
 
 	result = vega10_program_didt_config_registers(hwmgr, SEEDCForceStallPatternConfig_Vega10, VEGA10_CONFIGREG_DIDT);
 	result |= vega10_program_didt_config_registers(hwmgr, SEEDCCtrlForceStallConfig_Vega10, VEGA10_CONFIGREG_DIDT);
@@ -1175,7 +1172,7 @@ static int vega10_enable_se_edc_force_stall_config(struct pp_hwmgr *hwmgr)
 
 	vega10_didt_set_mask(hwmgr, false);
 
-	cgs_enter_safe_mode(hwmgr->device, false);
+	adev->gfx.rlc.funcs->exit_safe_mode(adev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c
index c61d0744860d..0768d259c07c 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c
@@ -52,7 +52,7 @@ static const void *get_powerplay_table(struct pp_hwmgr *hwmgr)
 
 	if (!table_address) {
 		table_address = (ATOM_Vega10_POWERPLAYTABLE *)
-				cgs_atom_get_data_table(hwmgr->device, index,
+				smu_atom_get_data_table(hwmgr->adev, index,
 						&size, &frev, &crev);
 
 		hwmgr->soft_pp_table = table_address;	/*Cache the result in RAM.*/
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c
index 9f18226a56ea..aa044c1955fe 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c
@@ -25,7 +25,7 @@
 #include "vega10_hwmgr.h"
 #include "vega10_ppsmc.h"
 #include "vega10_inc.h"
-#include "pp_soc15.h"
+#include "soc15_common.h"
 #include "pp_debug.h"
 
 static int vega10_get_current_rpm(struct pp_hwmgr *hwmgr, uint32_t *current_rpm)
@@ -89,6 +89,7 @@ int vega10_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr,
 
 int vega10_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	struct vega10_hwmgr *data = hwmgr->backend;
 	uint32_t tach_period;
 	uint32_t crystal_clock_freq;
@@ -100,10 +101,8 @@ int vega10_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed)
 	if (data->smu_features[GNLD_FAN_CONTROL].supported) {
 		result = vega10_get_current_rpm(hwmgr, speed);
 	} else {
-		uint32_t reg = soc15_get_register_offset(THM_HWID, 0,
-				mmCG_TACH_STATUS_BASE_IDX, mmCG_TACH_STATUS);
 		tach_period =
-			CGS_REG_GET_FIELD(cgs_read_register(hwmgr->device, reg),
+			REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_TACH_STATUS),
 					  CG_TACH_STATUS,
 					  TACH_PERIOD);
 
@@ -127,26 +126,23 @@ int vega10_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed)
 */
 int vega10_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
 {
-	uint32_t reg;
-
-	reg = soc15_get_register_offset(THM_HWID, 0,
-			mmCG_FDO_CTRL2_BASE_IDX, mmCG_FDO_CTRL2);
+	struct amdgpu_device *adev = hwmgr->adev;
 
 	if (hwmgr->fan_ctrl_is_in_default_mode) {
 		hwmgr->fan_ctrl_default_mode =
-			CGS_REG_GET_FIELD(cgs_read_register(hwmgr->device, reg),
+			REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
 				CG_FDO_CTRL2, FDO_PWM_MODE);
 		hwmgr->tmin =
-			CGS_REG_GET_FIELD(cgs_read_register(hwmgr->device, reg),
+			REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
 				CG_FDO_CTRL2, TMIN);
 		hwmgr->fan_ctrl_is_in_default_mode = false;
 	}
 
-	cgs_write_register(hwmgr->device, reg,
-			CGS_REG_SET_FIELD(cgs_read_register(hwmgr->device, reg),
+	WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2,
+			REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
 				CG_FDO_CTRL2, TMIN, 0));
-	cgs_write_register(hwmgr->device, reg,
-			CGS_REG_SET_FIELD(cgs_read_register(hwmgr->device, reg),
+	WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2,
+			REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
 				CG_FDO_CTRL2, FDO_PWM_MODE, mode));
 
 	return 0;
@@ -159,18 +155,15 @@ int vega10_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
 */
 int vega10_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr)
 {
-	uint32_t reg;
-
-	reg = soc15_get_register_offset(THM_HWID, 0,
-			mmCG_FDO_CTRL2_BASE_IDX, mmCG_FDO_CTRL2);
+	struct amdgpu_device *adev = hwmgr->adev;
 
 	if (!hwmgr->fan_ctrl_is_in_default_mode) {
-		cgs_write_register(hwmgr->device, reg,
-			CGS_REG_SET_FIELD(cgs_read_register(hwmgr->device, reg),
+		WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2,
+			REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
 				CG_FDO_CTRL2, FDO_PWM_MODE,
 				hwmgr->fan_ctrl_default_mode));
-		cgs_write_register(hwmgr->device, reg,
-			CGS_REG_SET_FIELD(cgs_read_register(hwmgr->device, reg),
+		WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2,
+			REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
 				CG_FDO_CTRL2, TMIN,
 				hwmgr->tmin << CG_FDO_CTRL2__TMIN__SHIFT));
 		hwmgr->fan_ctrl_is_in_default_mode = true;
@@ -257,10 +250,10 @@ int vega10_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr)
 int vega10_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr,
 		uint32_t speed)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	uint32_t duty100;
 	uint32_t duty;
 	uint64_t tmp64;
-	uint32_t reg;
 
 	if (hwmgr->thermal_controller.fanInfo.bNoFan)
 		return 0;
@@ -271,10 +264,7 @@ int vega10_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr,
 	if (PP_CAP(PHM_PlatformCaps_MicrocodeFanControl))
 		vega10_fan_ctrl_stop_smc_fan_control(hwmgr);
 
-	reg = soc15_get_register_offset(THM_HWID, 0,
-			mmCG_FDO_CTRL1_BASE_IDX, mmCG_FDO_CTRL1);
-
-	duty100 = CGS_REG_GET_FIELD(cgs_read_register(hwmgr->device, reg),
+	duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1),
 				    CG_FDO_CTRL1, FMAX_DUTY100);
 
 	if (duty100 == 0)
@@ -284,10 +274,8 @@ int vega10_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr,
 	do_div(tmp64, 100);
 	duty = (uint32_t)tmp64;
 
-	reg = soc15_get_register_offset(THM_HWID, 0,
-			mmCG_FDO_CTRL0_BASE_IDX, mmCG_FDO_CTRL0);
-	cgs_write_register(hwmgr->device, reg,
-		CGS_REG_SET_FIELD(cgs_read_register(hwmgr->device, reg),
+	WREG32_SOC15(THM, 0, mmCG_FDO_CTRL0,
+		REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL0),
 			CG_FDO_CTRL0, FDO_STATIC_DUTY, duty));
 
 	return vega10_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
@@ -317,10 +305,10 @@ int vega10_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr)
 */
 int vega10_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	uint32_t tach_period;
 	uint32_t crystal_clock_freq;
 	int result = 0;
-	uint32_t reg;
 
 	if (hwmgr->thermal_controller.fanInfo.bNoFan ||
 	    (speed < hwmgr->thermal_controller.fanInfo.ulMinRPM) ||
@@ -333,10 +321,8 @@ int vega10_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed)
 	if (!result) {
 		crystal_clock_freq = amdgpu_asic_get_xclk((struct amdgpu_device *)hwmgr->adev);
 		tach_period = 60 * crystal_clock_freq * 10000 / (8 * speed);
-		reg = soc15_get_register_offset(THM_HWID, 0,
-				mmCG_TACH_STATUS_BASE_IDX, mmCG_TACH_STATUS);
-		cgs_write_register(hwmgr->device, reg,
-				CGS_REG_SET_FIELD(cgs_read_register(hwmgr->device, reg),
+		WREG32_SOC15(THM, 0, mmCG_TACH_STATUS,
+				REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_TACH_STATUS),
 					CG_TACH_STATUS, TACH_PERIOD,
 					tach_period));
 	}
@@ -350,13 +336,10 @@ int vega10_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed)
 */
 int vega10_thermal_get_temperature(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	int temp;
-	uint32_t reg;
 
-	reg = soc15_get_register_offset(THM_HWID, 0,
-			mmCG_MULT_THERMAL_STATUS_BASE_IDX,  mmCG_MULT_THERMAL_STATUS);
-
-	temp = cgs_read_register(hwmgr->device, reg);
+	temp = RREG32_SOC15(THM, 0, mmCG_MULT_THERMAL_STATUS);
 
 	temp = (temp & CG_MULT_THERMAL_STATUS__CTF_TEMP_MASK) >>
 			CG_MULT_THERMAL_STATUS__CTF_TEMP__SHIFT;
@@ -379,11 +362,12 @@ int vega10_thermal_get_temperature(struct pp_hwmgr *hwmgr)
 static int vega10_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
 		struct PP_TemperatureRange *range)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	int low = VEGA10_THERMAL_MINIMUM_ALERT_TEMP *
 			PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
 	int high = VEGA10_THERMAL_MAXIMUM_ALERT_TEMP *
 			PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
-	uint32_t val, reg;
+	uint32_t val;
 
 	if (low < range->min)
 		low = range->min;
@@ -393,20 +377,17 @@ static int vega10_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
 	if (low > high)
 		return -EINVAL;
 
-	reg = soc15_get_register_offset(THM_HWID, 0,
-			mmTHM_THERMAL_INT_CTRL_BASE_IDX, mmTHM_THERMAL_INT_CTRL);
-
-	val = cgs_read_register(hwmgr->device, reg);
+	val = RREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_CTRL);
 
-	val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, MAX_IH_CREDIT, 5);
-	val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, THERM_IH_HW_ENA, 1);
-	val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTH, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
-	val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTL, (low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, MAX_IH_CREDIT, 5);
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, THERM_IH_HW_ENA, 1);
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTH, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTL, (low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
 	val &= (~THM_THERMAL_INT_CTRL__THERM_TRIGGER_MASK_MASK) &
 			(~THM_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK) &
 			(~THM_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK);
 
-	cgs_write_register(hwmgr->device, reg, val);
+	WREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_CTRL, val);
 
 	return 0;
 }
@@ -418,21 +399,17 @@ static int vega10_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
 */
 static int vega10_thermal_initialize(struct pp_hwmgr *hwmgr)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 
 	if (hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution) {
-		reg = soc15_get_register_offset(THM_HWID, 0,
-				mmCG_TACH_CTRL_BASE_IDX, mmCG_TACH_CTRL);
-		cgs_write_register(hwmgr->device, reg,
-			CGS_REG_SET_FIELD(cgs_read_register(hwmgr->device, reg),
+		WREG32_SOC15(THM, 0, mmCG_TACH_CTRL,
+			REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_TACH_CTRL),
 				CG_TACH_CTRL, EDGE_PER_REV,
 				hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution - 1));
 	}
 
-	reg = soc15_get_register_offset(THM_HWID, 0,
-			mmCG_FDO_CTRL2_BASE_IDX, mmCG_FDO_CTRL2);
-	cgs_write_register(hwmgr->device, reg,
-		CGS_REG_SET_FIELD(cgs_read_register(hwmgr->device, reg),
+	WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2,
+		REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
 			CG_FDO_CTRL2, TACH_PWM_RESP_RATE, 0x28));
 
 	return 0;
@@ -445,9 +422,9 @@ static int vega10_thermal_initialize(struct pp_hwmgr *hwmgr)
 */
 static int vega10_thermal_enable_alert(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	struct vega10_hwmgr *data = hwmgr->backend;
 	uint32_t val = 0;
-	uint32_t reg;
 
 	if (data->smu_features[GNLD_FW_CTF].supported) {
 		if (data->smu_features[GNLD_FW_CTF].enabled)
@@ -465,8 +442,7 @@ static int vega10_thermal_enable_alert(struct pp_hwmgr *hwmgr)
 	val |= (1 << THM_THERMAL_INT_ENA__THERM_INTL_CLR__SHIFT);
 	val |= (1 << THM_THERMAL_INT_ENA__THERM_TRIGGER_CLR__SHIFT);
 
-	reg = soc15_get_register_offset(THM_HWID, 0, mmTHM_THERMAL_INT_ENA_BASE_IDX, mmTHM_THERMAL_INT_ENA);
-	cgs_write_register(hwmgr->device, reg, val);
+	WREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_ENA, val);
 
 	return 0;
 }
@@ -477,8 +453,8 @@ static int vega10_thermal_enable_alert(struct pp_hwmgr *hwmgr)
 */
 int vega10_thermal_disable_alert(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	struct vega10_hwmgr *data = hwmgr->backend;
-	uint32_t reg;
 
 	if (data->smu_features[GNLD_FW_CTF].supported) {
 		if (!data->smu_features[GNLD_FW_CTF].enabled)
@@ -493,8 +469,7 @@ int vega10_thermal_disable_alert(struct pp_hwmgr *hwmgr)
 		data->smu_features[GNLD_FW_CTF].enabled = false;
 	}
 
-	reg = soc15_get_register_offset(THM_HWID, 0, mmTHM_THERMAL_INT_ENA_BASE_IDX, mmTHM_THERMAL_INT_ENA);
-	cgs_write_register(hwmgr->device, reg, 0);
+	WREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_ENA, 0);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c
index 200de46bd06b..782e2098824d 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c
@@ -34,7 +34,6 @@
 #include "atomfirmware.h"
 #include "cgs_common.h"
 #include "vega12_inc.h"
-#include "pp_soc15.h"
 #include "pppcielanes.h"
 #include "vega12_hwmgr.h"
 #include "vega12_processpptables.h"
@@ -546,6 +545,7 @@ static int vega12_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 			return -EINVAL);
 
 		dpm_table->dpm_levels[i].value = clock;
+		dpm_table->dpm_levels[i].enabled = true;
 	}
 
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
@@ -565,6 +565,7 @@ static int vega12_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 			return -EINVAL);
 
 		dpm_table->dpm_levels[i].value = clock;
+		dpm_table->dpm_levels[i].enabled = true;
 	}
 
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
@@ -585,6 +586,7 @@ static int vega12_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 			return -EINVAL);
 
 		dpm_table->dpm_levels[i].value = clock;
+		dpm_table->dpm_levels[i].enabled = true;
 	}
 
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
@@ -605,6 +607,7 @@ static int vega12_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 		return -EINVAL);
 
 		dpm_table->dpm_levels[i].value = clock;
+		dpm_table->dpm_levels[i].enabled = true;
 	}
 
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
@@ -625,6 +628,7 @@ static int vega12_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 			return -EINVAL);
 
 		dpm_table->dpm_levels[i].value = clock;
+		dpm_table->dpm_levels[i].enabled = true;
 	}
 
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
@@ -645,6 +649,7 @@ static int vega12_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 		return -EINVAL);
 
 		dpm_table->dpm_levels[i].value = clock;
+		dpm_table->dpm_levels[i].enabled = true;
 	}
 
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
@@ -666,6 +671,7 @@ static int vega12_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 			return -EINVAL);
 
 		dpm_table->dpm_levels[i].value = clock;
+		dpm_table->dpm_levels[i].enabled = true;
 	}
 
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
@@ -686,6 +692,7 @@ static int vega12_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 			return -EINVAL);
 
 		dpm_table->dpm_levels[i].value = clock;
+		dpm_table->dpm_levels[i].enabled = true;
 	}
 
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
@@ -706,6 +713,7 @@ static int vega12_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 			return -EINVAL);
 
 		dpm_table->dpm_levels[i].value = clock;
+		dpm_table->dpm_levels[i].enabled = true;
 	}
 
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
@@ -726,6 +734,7 @@ static int vega12_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 			return -EINVAL);
 
 		dpm_table->dpm_levels[i].value = clock;
+		dpm_table->dpm_levels[i].enabled = true;
 	}
 
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
@@ -992,15 +1001,55 @@ static uint32_t vega12_find_highest_dpm_level(
 
 static int vega12_upload_dpm_min_level(struct pp_hwmgr *hwmgr)
 {
+	struct vega12_hwmgr *data = hwmgr->backend;
+	if (data->smc_state_table.gfx_boot_level !=
+			data->dpm_table.gfx_table.dpm_state.soft_min_level) {
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+			PPSMC_MSG_SetSoftMinByFreq,
+			PPCLK_GFXCLK<<16 | data->dpm_table.gfx_table.dpm_levels[data->smc_state_table.gfx_boot_level].value);
+		data->dpm_table.gfx_table.dpm_state.soft_min_level =
+				data->smc_state_table.gfx_boot_level;
+	}
+
+	if (data->smc_state_table.mem_boot_level !=
+			data->dpm_table.mem_table.dpm_state.soft_min_level) {
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+			PPSMC_MSG_SetSoftMinByFreq,
+			PPCLK_UCLK<<16 | data->dpm_table.mem_table.dpm_levels[data->smc_state_table.mem_boot_level].value);
+		data->dpm_table.mem_table.dpm_state.soft_min_level =
+				data->smc_state_table.mem_boot_level;
+	}
+
 	return 0;
+
 }
 
 static int vega12_upload_dpm_max_level(struct pp_hwmgr *hwmgr)
 {
+	struct vega12_hwmgr *data = hwmgr->backend;
+	if (data->smc_state_table.gfx_max_level !=
+		data->dpm_table.gfx_table.dpm_state.soft_max_level) {
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+			PPSMC_MSG_SetSoftMaxByFreq,
+			/* plus the vale by 1 to align the resolution */
+			PPCLK_GFXCLK<<16 | (data->dpm_table.gfx_table.dpm_levels[data->smc_state_table.gfx_max_level].value + 1));
+		data->dpm_table.gfx_table.dpm_state.soft_max_level =
+				data->smc_state_table.gfx_max_level;
+	}
+
+	if (data->smc_state_table.mem_max_level !=
+		data->dpm_table.mem_table.dpm_state.soft_max_level) {
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+			PPSMC_MSG_SetSoftMaxByFreq,
+			/* plus the vale by 1 to align the resolution */
+			PPCLK_UCLK<<16 | (data->dpm_table.mem_table.dpm_levels[data->smc_state_table.mem_max_level].value + 1));
+		data->dpm_table.mem_table.dpm_state.soft_max_level =
+				data->smc_state_table.mem_max_level;
+	}
+
 	return 0;
 }
 
-
 int vega12_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable)
 {
 	struct vega12_hwmgr *data =
@@ -1064,8 +1113,7 @@ static uint32_t vega12_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low)
 	return (mem_clk * 100);
 }
 
-static int vega12_get_gpu_power(struct pp_hwmgr *hwmgr,
-		struct pp_gpu_power *query)
+static int vega12_get_gpu_power(struct pp_hwmgr *hwmgr, uint32_t *query)
 {
 #if 0
 	uint32_t value;
@@ -1077,7 +1125,7 @@ static int vega12_get_gpu_power(struct pp_hwmgr *hwmgr,
 
 	vega12_read_arg_from_smc(hwmgr, &value);
 	/* power value is an integer */
-	query->average_gpu_power = value << 8;
+	*query = value << 8;
 #endif
 	return 0;
 }
@@ -1186,12 +1234,8 @@ static int vega12_read_sensor(struct pp_hwmgr *hwmgr, int idx,
 		*size = 4;
 		break;
 	case AMDGPU_PP_SENSOR_GPU_POWER:
-		if (*size < sizeof(struct pp_gpu_power))
-			ret = -EINVAL;
-		else {
-			*size = sizeof(struct pp_gpu_power);
-			ret = vega12_get_gpu_power(hwmgr, (struct pp_gpu_power *)value);
-		}
+		ret = vega12_get_gpu_power(hwmgr, (uint32_t *)value);
+
 		break;
 	default:
 		ret = -EINVAL;
@@ -1260,23 +1304,18 @@ static int vega12_notify_smc_display_config_after_ps_adjustment(
 {
 	struct vega12_hwmgr *data =
 			(struct vega12_hwmgr *)(hwmgr->backend);
-	uint32_t num_active_disps = 0;
-	struct cgs_display_info info = {0};
 	struct PP_Clocks min_clocks = {0};
 	struct pp_display_clock_request clock_req;
 	uint32_t clk_request;
 
-	info.mode_info = NULL;
-	cgs_get_active_displays_info(hwmgr->device, &info);
-	num_active_disps = info.display_count;
-	if (num_active_disps > 1)
+	if (hwmgr->display_config->num_display > 1)
 		vega12_notify_smc_display_change(hwmgr, false);
 	else
 		vega12_notify_smc_display_change(hwmgr, true);
 
-	min_clocks.dcefClock = hwmgr->display_config.min_dcef_set_clk;
-	min_clocks.dcefClockInSR = hwmgr->display_config.min_dcef_deep_sleep_set_clk;
-	min_clocks.memoryClock = hwmgr->display_config.min_mem_set_clock;
+	min_clocks.dcefClock = hwmgr->display_config->min_dcef_set_clk;
+	min_clocks.dcefClockInSR = hwmgr->display_config->min_dcef_deep_sleep_set_clk;
+	min_clocks.memoryClock = hwmgr->display_config->min_mem_set_clock;
 
 	if (data->smu_features[GNLD_DPM_DCEFCLK].supported) {
 		clock_req.clock_type = amd_pp_dcef_clock;
@@ -1832,9 +1871,7 @@ static int vega12_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
 {
 	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
 	int result = 0;
-	uint32_t num_turned_on_displays = 1;
 	Watermarks_t *wm_table = &(data->smc_state_table.water_marks_table);
-	struct cgs_display_info info = {0};
 
 	if ((data->water_marks_bitmap & WaterMarksExist) &&
 			!(data->water_marks_bitmap & WaterMarksLoaded)) {
@@ -1846,12 +1883,9 @@ static int vega12_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
 
 	if ((data->water_marks_bitmap & WaterMarksExist) &&
 		data->smu_features[GNLD_DPM_DCEFCLK].supported &&
-		data->smu_features[GNLD_DPM_SOCCLK].supported) {
-		cgs_get_active_displays_info(hwmgr->device, &info);
-		num_turned_on_displays = info.display_count;
+		data->smu_features[GNLD_DPM_SOCCLK].supported)
 		smum_send_msg_to_smc_with_parameter(hwmgr,
-			PPSMC_MSG_NumOfDisplays, num_turned_on_displays);
-	}
+			PPSMC_MSG_NumOfDisplays, hwmgr->display_config->num_display);
 
 	return result;
 }
@@ -1894,15 +1928,12 @@ vega12_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmg
 {
 	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
 	bool is_update_required = false;
-	struct cgs_display_info info = {0, 0, NULL};
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
 
-	if (data->display_timing.num_existing_displays != info.display_count)
+	if (data->display_timing.num_existing_displays != hwmgr->display_config->num_display)
 		is_update_required = true;
 
 	if (data->registry_data.gfx_clk_deep_sleep_support) {
-		if (data->display_timing.min_clock_in_sr != hwmgr->display_config.min_core_set_clock_in_sr)
+		if (data->display_timing.min_clock_in_sr != hwmgr->display_config->min_core_set_clock_in_sr)
 			is_update_required = true;
 	}
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h
index bc98b1df3b65..e81ded1ec198 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h
@@ -33,7 +33,7 @@
 #define WaterMarksExist  1
 #define WaterMarksLoaded 2
 
-#define VG12_PSUEDO_NUM_GFXCLK_DPM_LEVELS   8
+#define VG12_PSUEDO_NUM_GFXCLK_DPM_LEVELS   16
 #define VG12_PSUEDO_NUM_SOCCLK_DPM_LEVELS   8
 #define VG12_PSUEDO_NUM_DCEFCLK_DPM_LEVELS  8
 #define VG12_PSUEDO_NUM_UCLK_DPM_LEVELS     4
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.c
index b34113f45904..888ddca902d8 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.c
@@ -51,7 +51,7 @@ static const void *get_powerplay_table(struct pp_hwmgr *hwmgr)
 
 	if (!table_address) {
 		table_address = (ATOM_Vega12_POWERPLAYTABLE *)
-				cgs_atom_get_data_table(hwmgr->device, index,
+				smu_atom_get_data_table(hwmgr->adev, index,
 						&size, &frev, &crev);
 
 		hwmgr->soft_pp_table = table_address;	/*Cache the result in RAM.*/
@@ -224,6 +224,11 @@ static int append_vbios_pptable(struct pp_hwmgr *hwmgr, PPTable_t *ppsmc_pptable
 	ppsmc_pptable->AcgGfxclkSpreadPercent = smc_dpm_table.acggfxclkspreadpercent;
 	ppsmc_pptable->AcgGfxclkSpreadFreq = smc_dpm_table.acggfxclkspreadfreq;
 
+	/* 0xFFFF will disable the ACG feature */
+	if (!(hwmgr->feature_mask & PP_ACG_MASK)) {
+		ppsmc_pptable->AcgThresholdFreqHigh = 0xFFFF;
+		ppsmc_pptable->AcgThresholdFreqLow = 0xFFFF;
+	}
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.c
index df0fa815cd6e..cfd9e6ccb790 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.c
@@ -26,7 +26,7 @@
 #include "vega12_smumgr.h"
 #include "vega12_ppsmc.h"
 #include "vega12_inc.h"
-#include "pp_soc15.h"
+#include "soc15_common.h"
 #include "pp_debug.h"
 
 static int vega12_get_current_rpm(struct pp_hwmgr *hwmgr, uint32_t *current_rpm)
@@ -147,13 +147,10 @@ int vega12_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr)
 */
 int vega12_thermal_get_temperature(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	int temp = 0;
-	uint32_t reg;
 
-	reg = soc15_get_register_offset(THM_HWID, 0,
-			mmCG_MULT_THERMAL_STATUS_BASE_IDX,  mmCG_MULT_THERMAL_STATUS);
-
-	temp = cgs_read_register(hwmgr->device, reg);
+	temp = RREG32_SOC15(THM, 0, mmCG_MULT_THERMAL_STATUS);
 
 	temp = (temp & CG_MULT_THERMAL_STATUS__CTF_TEMP_MASK) >>
 			CG_MULT_THERMAL_STATUS__CTF_TEMP__SHIFT;
@@ -175,11 +172,12 @@ int vega12_thermal_get_temperature(struct pp_hwmgr *hwmgr)
 static int vega12_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
 		struct PP_TemperatureRange *range)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	int low = VEGA12_THERMAL_MINIMUM_ALERT_TEMP *
 			PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
 	int high = VEGA12_THERMAL_MAXIMUM_ALERT_TEMP *
 			PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
-	uint32_t val, reg;
+	uint32_t val;
 
 	if (low < range->min)
 		low = range->min;
@@ -189,18 +187,15 @@ static int vega12_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
 	if (low > high)
 		return -EINVAL;
 
-	reg = soc15_get_register_offset(THM_HWID, 0,
-			mmTHM_THERMAL_INT_CTRL_BASE_IDX, mmTHM_THERMAL_INT_CTRL);
-
-	val = cgs_read_register(hwmgr->device, reg);
+	val = RREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_CTRL);
 
-	val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, MAX_IH_CREDIT, 5);
-	val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, THERM_IH_HW_ENA, 1);
-	val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTH, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
-	val = CGS_REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTL, (low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, MAX_IH_CREDIT, 5);
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, THERM_IH_HW_ENA, 1);
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTH, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTL, (low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
 	val = val & (~THM_THERMAL_INT_CTRL__THERM_TRIGGER_MASK_MASK);
 
-	cgs_write_register(hwmgr->device, reg, val);
+	WREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_CTRL, val);
 
 	return 0;
 }
@@ -212,15 +207,14 @@ static int vega12_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
 */
 static int vega12_thermal_enable_alert(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	uint32_t val = 0;
-	uint32_t reg;
 
 	val |= (1 << THM_THERMAL_INT_ENA__THERM_INTH_CLR__SHIFT);
 	val |= (1 << THM_THERMAL_INT_ENA__THERM_INTL_CLR__SHIFT);
 	val |= (1 << THM_THERMAL_INT_ENA__THERM_TRIGGER_CLR__SHIFT);
 
-	reg = soc15_get_register_offset(THM_HWID, 0, mmTHM_THERMAL_INT_ENA_BASE_IDX, mmTHM_THERMAL_INT_ENA);
-	cgs_write_register(hwmgr->device, reg, val);
+	WREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_ENA, val);
 
 	return 0;
 }
@@ -231,10 +225,9 @@ static int vega12_thermal_enable_alert(struct pp_hwmgr *hwmgr)
 */
 int vega12_thermal_disable_alert(struct pp_hwmgr *hwmgr)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 
-	reg = soc15_get_register_offset(THM_HWID, 0, mmTHM_THERMAL_INT_ENA_BASE_IDX, mmTHM_THERMAL_INT_ENA);
-	cgs_write_register(hwmgr->device, reg, 0);
+	WREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_ENA, 0);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
index 8b78bbecd1bc..a202247c9894 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
@@ -377,11 +377,7 @@ struct phm_clocks {
 #define DPMTABLE_UPDATE_SCLK        0x00000004
 #define DPMTABLE_UPDATE_MCLK        0x00000008
 #define DPMTABLE_OD_UPDATE_VDDC     0x00000010
-
-/* To determine if sclk and mclk are in overdrive state */
-#define SCLK_OVERDRIVE_ENABLED           0x00000001
-#define MCLK_OVERDRIVE_ENABLED           0x00000002
-#define VDDC_OVERDRIVE_ENABLED           0x00000010
+#define DPMTABLE_UPDATE_SOCCLK      0x00000020
 
 struct phm_odn_performance_level {
 	uint32_t clock;
@@ -414,7 +410,10 @@ extern int phm_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 				   struct pp_power_state *adjusted_ps,
 			     const struct pp_power_state *current_ps);
 
+extern int phm_apply_clock_adjust_rules(struct pp_hwmgr *hwmgr);
+
 extern int phm_force_dpm_levels(struct pp_hwmgr *hwmgr, enum amd_dpm_forced_level level);
+extern int phm_pre_display_configuration_changed(struct pp_hwmgr *hwmgr);
 extern int phm_display_configuration_changed(struct pp_hwmgr *hwmgr);
 extern int phm_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr);
 extern int phm_register_irq_handlers(struct pp_hwmgr *hwmgr);
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
index 17f811d181c8..b99fb8ac822c 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
@@ -38,6 +38,8 @@ struct phm_fan_speed_info;
 struct pp_atomctrl_voltage_table;
 
 #define VOLTAGE_SCALE 4
+#define VOLTAGE_VID_OFFSET_SCALE1   625
+#define VOLTAGE_VID_OFFSET_SCALE2   100
 
 enum DISPLAY_GAP {
 	DISPLAY_GAP_VBLANK_OR_WM = 0,   /* Wait for vblank or MCHG watermark. */
@@ -64,24 +66,6 @@ struct vi_dpm_table {
 #define PCIE_PERF_REQ_GEN2         3
 #define PCIE_PERF_REQ_GEN3         4
 
-enum PP_FEATURE_MASK {
-	PP_SCLK_DPM_MASK = 0x1,
-	PP_MCLK_DPM_MASK = 0x2,
-	PP_PCIE_DPM_MASK = 0x4,
-	PP_SCLK_DEEP_SLEEP_MASK = 0x8,
-	PP_POWER_CONTAINMENT_MASK = 0x10,
-	PP_UVD_HANDSHAKE_MASK = 0x20,
-	PP_SMC_VOLTAGE_CONTROL_MASK = 0x40,
-	PP_VBI_TIME_SUPPORT_MASK = 0x80,
-	PP_ULV_MASK = 0x100,
-	PP_ENABLE_GFX_CG_THRU_SMU = 0x200,
-	PP_CLOCK_STRETCH_MASK = 0x400,
-	PP_OD_FUZZY_FAN_CONTROL_MASK = 0x800,
-	PP_SOCCLK_DPM_MASK = 0x1000,
-	PP_DCEFCLK_DPM_MASK = 0x2000,
-	PP_OVERDRIVE_MASK = 0x4000,
-};
-
 enum PHM_BackEnd_Magic {
 	PHM_Dummy_Magic       = 0xAA5555AA,
 	PHM_RV770_Magic       = 0xDCBAABCD,
@@ -245,6 +229,8 @@ struct pp_hwmgr_func {
 				struct pp_power_state  *prequest_ps,
 			const struct pp_power_state *pcurrent_ps);
 
+	int (*apply_clocks_adjust_rules)(struct pp_hwmgr *hwmgr);
+
 	int (*force_dpm_level)(struct pp_hwmgr *hw_mgr,
 					enum amd_dpm_forced_level level);
 
@@ -268,6 +254,7 @@ struct pp_hwmgr_func {
 						const void *state);
 	int (*enable_clock_power_gating)(struct pp_hwmgr *hwmgr);
 	int (*notify_smc_display_config_after_ps_adjustment)(struct pp_hwmgr *hwmgr);
+	int (*pre_display_config_changed)(struct pp_hwmgr *hwmgr);
 	int (*display_config_changed)(struct pp_hwmgr *hwmgr);
 	int (*disable_clock_power_gating)(struct pp_hwmgr *hwmgr);
 	int (*update_clock_gatings)(struct pp_hwmgr *hwmgr,
@@ -312,6 +299,7 @@ struct pp_hwmgr_func {
 	int (*display_clock_voltage_request)(struct pp_hwmgr *hwmgr,
 			struct pp_display_clock_request *clock);
 	int (*get_max_high_clocks)(struct pp_hwmgr *hwmgr, struct amd_pp_simple_clock_info *clocks);
+	int (*gfx_off_control)(struct pp_hwmgr *hwmgr, bool enable);
 	int (*power_off_asic)(struct pp_hwmgr *hwmgr);
 	int (*force_clock_level)(struct pp_hwmgr *hwmgr, enum pp_clock_type type, uint32_t mask);
 	int (*print_clock_levels)(struct pp_hwmgr *hwmgr, enum pp_clock_type type, char *buf);
@@ -341,6 +329,7 @@ struct pp_hwmgr_func {
 					long *input, uint32_t size);
 	int (*set_power_limit)(struct pp_hwmgr *hwmgr, uint32_t n);
 	int (*set_mmhub_powergating_by_smu)(struct pp_hwmgr *hwmgr);
+	int (*smus_notify_pwe)(struct pp_hwmgr *hwmgr);
 };
 
 struct pp_table_func {
@@ -718,6 +707,7 @@ struct pp_hwmgr {
 	uint32_t chip_family;
 	uint32_t chip_id;
 	uint32_t smu_version;
+	bool not_vf;
 	bool pm_en;
 	struct mutex smu_lock;
 
@@ -764,7 +754,7 @@ struct pp_hwmgr {
 	struct pp_power_state    *request_ps;
 	struct pp_power_state    *boot_ps;
 	struct pp_power_state    *uvd_ps;
-	struct amd_pp_display_configuration display_config;
+	const struct amd_pp_display_configuration *display_config;
 	uint32_t feature_mask;
 	bool avfs_supported;
 	/* UMD Pstate */
@@ -782,10 +772,13 @@ struct pp_hwmgr {
 };
 
 int hwmgr_early_init(struct pp_hwmgr *hwmgr);
+int hwmgr_sw_init(struct pp_hwmgr *hwmgr);
+int hwmgr_sw_fini(struct pp_hwmgr *hwmgr);
 int hwmgr_hw_init(struct pp_hwmgr *hwmgr);
 int hwmgr_hw_fini(struct pp_hwmgr *hwmgr);
-int hwmgr_hw_suspend(struct pp_hwmgr *hwmgr);
-int hwmgr_hw_resume(struct pp_hwmgr *hwmgr);
+int hwmgr_suspend(struct pp_hwmgr *hwmgr);
+int hwmgr_resume(struct pp_hwmgr *hwmgr);
+
 int hwmgr_handle_task(struct pp_hwmgr *hwmgr,
 				enum amd_pp_task task_id,
 				enum amd_pm_state_type *user_state);
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h b/drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h
deleted file mode 100644
index 214f370c5efd..000000000000
--- a/drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2016 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-#ifndef PP_SOC15_H
-#define PP_SOC15_H
-
-#include "soc15_hw_ip.h"
-#include "vega10_ip_offset.h"
-
-inline static uint32_t soc15_get_register_offset(
-		uint32_t hw_id,
-		uint32_t inst,
-		uint32_t segment,
-		uint32_t offset)
-{
-	uint32_t reg = 0;
-
-	if (hw_id == THM_HWID)
-		reg = THM_BASE.instance[inst].segment[segment] + offset;
-	else if (hw_id == NBIF_HWID)
-		reg = NBIF_BASE.instance[inst].segment[segment] + offset;
-	else if (hw_id == MP1_HWID)
-		reg = MP1_BASE.instance[inst].segment[segment] + offset;
-	else if (hw_id == DF_HWID)
-		reg = DF_BASE.instance[inst].segment[segment] + offset;
-	else if (hw_id == GC_HWID)
-		reg = GC_BASE.instance[inst].segment[segment] + offset;
-	else if (hw_id == SMUIO_HWID)
-		reg = SMUIO_BASE.instance[inst].segment[segment] + offset;
-	return reg;
-}
-
-#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
index 426bff2aad2b..a2991fa2e6f8 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
@@ -75,13 +75,15 @@
 #define PPSMC_MSG_GetMinGfxclkFrequency         0x2C
 #define PPSMC_MSG_GetMaxGfxclkFrequency         0x2D
 #define PPSMC_MSG_SoftReset                     0x2E
+#define PPSMC_MSG_SetGfxCGPG			0x2F
 #define PPSMC_MSG_SetSoftMaxGfxClk              0x30
 #define PPSMC_MSG_SetHardMinGfxClk              0x31
 #define PPSMC_MSG_SetSoftMaxSocclkByFreq        0x32
 #define PPSMC_MSG_SetSoftMaxFclkByFreq          0x33
 #define PPSMC_MSG_SetSoftMaxVcn                 0x34
 #define PPSMC_MSG_PowerGateMmHub                0x35
-#define PPSMC_Message_Count                     0x36
+#define PPSMC_MSG_SetRccPfcPmeRestoreRegister   0x36
+#define PPSMC_Message_Count                     0x37
 
 
 typedef uint16_t PPSMC_Result;
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu75.h b/drivers/gpu/drm/amd/powerplay/inc/smu75.h
new file mode 100644
index 000000000000..771523001533
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu75.h
@@ -0,0 +1,760 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef SMU75_H
+#define SMU75_H
+
+#pragma pack(push, 1)
+
+typedef struct {
+	uint32_t high;
+	uint32_t low;
+} data_64_t;
+
+typedef struct {
+	data_64_t high;
+	data_64_t low;
+} data_128_t;
+
+#define SMU__DGPU_ONLY
+
+#define SMU__NUM_SCLK_DPM_STATE  8
+#define SMU__NUM_MCLK_DPM_LEVELS 4
+#define SMU__NUM_LCLK_DPM_LEVELS 8
+#define SMU__NUM_PCIE_DPM_LEVELS 8
+
+#define SMU7_CONTEXT_ID_SMC        1
+#define SMU7_CONTEXT_ID_VBIOS      2
+
+#define SMU75_MAX_LEVELS_VDDC            16
+#define SMU75_MAX_LEVELS_VDDGFX          16
+#define SMU75_MAX_LEVELS_VDDCI           8
+#define SMU75_MAX_LEVELS_MVDD            4
+
+#define SMU_MAX_SMIO_LEVELS              4
+
+#define SMU75_MAX_LEVELS_GRAPHICS        SMU__NUM_SCLK_DPM_STATE
+#define SMU75_MAX_LEVELS_MEMORY          SMU__NUM_MCLK_DPM_LEVELS
+#define SMU75_MAX_LEVELS_GIO             SMU__NUM_LCLK_DPM_LEVELS
+#define SMU75_MAX_LEVELS_LINK            SMU__NUM_PCIE_DPM_LEVELS
+#define SMU75_MAX_LEVELS_UVD             8
+#define SMU75_MAX_LEVELS_VCE             8
+#define SMU75_MAX_LEVELS_ACP             8
+#define SMU75_MAX_LEVELS_SAMU            8
+#define SMU75_MAX_ENTRIES_SMIO           32
+
+#define DPM_NO_LIMIT 0
+#define DPM_NO_UP 1
+#define DPM_GO_DOWN 2
+#define DPM_GO_UP 3
+
+#define SMU7_FIRST_DPM_GRAPHICS_LEVEL    0
+#define SMU7_FIRST_DPM_MEMORY_LEVEL      0
+
+#define GPIO_CLAMP_MODE_VRHOT      1
+#define GPIO_CLAMP_MODE_THERM      2
+#define GPIO_CLAMP_MODE_DC         4
+
+#define SCRATCH_B_TARG_PCIE_INDEX_SHIFT 0
+#define SCRATCH_B_TARG_PCIE_INDEX_MASK  (0x7<<SCRATCH_B_TARG_PCIE_INDEX_SHIFT)
+#define SCRATCH_B_CURR_PCIE_INDEX_SHIFT 3
+#define SCRATCH_B_CURR_PCIE_INDEX_MASK  (0x7<<SCRATCH_B_CURR_PCIE_INDEX_SHIFT)
+#define SCRATCH_B_TARG_UVD_INDEX_SHIFT  6
+#define SCRATCH_B_TARG_UVD_INDEX_MASK   (0x7<<SCRATCH_B_TARG_UVD_INDEX_SHIFT)
+#define SCRATCH_B_CURR_UVD_INDEX_SHIFT  9
+#define SCRATCH_B_CURR_UVD_INDEX_MASK   (0x7<<SCRATCH_B_CURR_UVD_INDEX_SHIFT)
+#define SCRATCH_B_TARG_VCE_INDEX_SHIFT  12
+#define SCRATCH_B_TARG_VCE_INDEX_MASK   (0x7<<SCRATCH_B_TARG_VCE_INDEX_SHIFT)
+#define SCRATCH_B_CURR_VCE_INDEX_SHIFT  15
+#define SCRATCH_B_CURR_VCE_INDEX_MASK   (0x7<<SCRATCH_B_CURR_VCE_INDEX_SHIFT)
+#define SCRATCH_B_TARG_ACP_INDEX_SHIFT  18
+#define SCRATCH_B_TARG_ACP_INDEX_MASK   (0x7<<SCRATCH_B_TARG_ACP_INDEX_SHIFT)
+#define SCRATCH_B_CURR_ACP_INDEX_SHIFT  21
+#define SCRATCH_B_CURR_ACP_INDEX_MASK   (0x7<<SCRATCH_B_CURR_ACP_INDEX_SHIFT)
+#define SCRATCH_B_TARG_SAMU_INDEX_SHIFT 24
+#define SCRATCH_B_TARG_SAMU_INDEX_MASK  (0x7<<SCRATCH_B_TARG_SAMU_INDEX_SHIFT)
+#define SCRATCH_B_CURR_SAMU_INDEX_SHIFT 27
+#define SCRATCH_B_CURR_SAMU_INDEX_MASK  (0x7<<SCRATCH_B_CURR_SAMU_INDEX_SHIFT)
+
+/* Virtualization Defines */
+#define CG_XDMA_MASK  0x1
+#define CG_XDMA_SHIFT 0
+#define CG_UVD_MASK   0x2
+#define CG_UVD_SHIFT  1
+#define CG_VCE_MASK   0x4
+#define CG_VCE_SHIFT  2
+#define CG_SAMU_MASK  0x8
+#define CG_SAMU_SHIFT 3
+#define CG_GFX_MASK   0x10
+#define CG_GFX_SHIFT  4
+#define CG_SDMA_MASK  0x20
+#define CG_SDMA_SHIFT 5
+#define CG_HDP_MASK   0x40
+#define CG_HDP_SHIFT  6
+#define CG_MC_MASK    0x80
+#define CG_MC_SHIFT   7
+#define CG_DRM_MASK   0x100
+#define CG_DRM_SHIFT  8
+#define CG_ROM_MASK   0x200
+#define CG_ROM_SHIFT  9
+#define CG_BIF_MASK   0x400
+#define CG_BIF_SHIFT  10
+
+#if defined SMU__DGPU_ONLY
+#define SMU75_DTE_ITERATIONS 5
+#define SMU75_DTE_SOURCES 3
+#define SMU75_DTE_SINKS 1
+#define SMU75_NUM_CPU_TES 0
+#define SMU75_NUM_GPU_TES 1
+#define SMU75_NUM_NON_TES 2
+#define SMU75_DTE_FAN_SCALAR_MIN 0x100
+#define SMU75_DTE_FAN_SCALAR_MAX 0x166
+#define SMU75_DTE_FAN_TEMP_MAX 93
+#define SMU75_DTE_FAN_TEMP_MIN 83
+#endif
+#define SMU75_THERMAL_INPUT_LOOP_COUNT 2
+#define SMU75_THERMAL_CLAMP_MODE_COUNT 2
+
+#define EXP_M1_1  93
+#define EXP_M2_1  195759
+#define EXP_B_1   111176531
+
+#define EXP_M1_2  67
+#define EXP_M2_2  153720
+#define EXP_B_2   94415767
+
+#define EXP_M1_3  48
+#define EXP_M2_3  119796
+#define EXP_B_3   79195279
+
+#define EXP_M1_4  550
+#define EXP_M2_4  1484190
+#define EXP_B_4   1051432828
+
+#define EXP_M1_5  394
+#define EXP_M2_5  1143049
+#define EXP_B_5   864288432
+
+struct SMU7_HystController_Data {
+	uint16_t waterfall_up;
+	uint16_t waterfall_down;
+	uint16_t waterfall_limit;
+	uint16_t release_cnt;
+	uint16_t release_limit;
+	uint16_t spare;
+};
+
+typedef struct SMU7_HystController_Data SMU7_HystController_Data;
+
+struct SMU75_PIDController {
+	uint32_t Ki;
+	int32_t LFWindupUpperLim;
+	int32_t LFWindupLowerLim;
+	uint32_t StatePrecision;
+	uint32_t LfPrecision;
+	uint32_t LfOffset;
+	uint32_t MaxState;
+	uint32_t MaxLfFraction;
+	uint32_t StateShift;
+};
+
+typedef struct SMU75_PIDController SMU75_PIDController;
+
+struct SMU7_LocalDpmScoreboard {
+	uint32_t PercentageBusy;
+
+	int32_t  PIDError;
+	int32_t  PIDIntegral;
+	int32_t  PIDOutput;
+
+	uint32_t SigmaDeltaAccum;
+	uint32_t SigmaDeltaOutput;
+	uint32_t SigmaDeltaLevel;
+
+	uint32_t UtilizationSetpoint;
+
+	uint8_t  TdpClampMode;
+	uint8_t  TdcClampMode;
+	uint8_t  ThermClampMode;
+	uint8_t  VoltageBusy;
+
+	int8_t   CurrLevel;
+	int8_t   TargLevel;
+	uint8_t  LevelChangeInProgress;
+	uint8_t  UpHyst;
+
+	uint8_t  DownHyst;
+	uint8_t  VoltageDownHyst;
+	uint8_t  DpmEnable;
+	uint8_t  DpmRunning;
+
+	uint8_t  DpmForce;
+	uint8_t  DpmForceLevel;
+	uint8_t  DisplayWatermark;
+	uint8_t  McArbIndex;
+
+	uint32_t MinimumPerfSclk;
+
+	uint8_t  AcpiReq;
+	uint8_t  AcpiAck;
+	uint8_t  GfxClkSlow;
+	uint8_t  GpioClampMode;
+
+	uint8_t  EnableModeSwitchRLCNotification;
+	uint8_t  EnabledLevelsChange;
+	uint8_t  DteClampMode;
+	uint8_t  FpsClampMode;
+
+	uint16_t LevelResidencyCounters [SMU75_MAX_LEVELS_GRAPHICS];
+	uint16_t LevelSwitchCounters [SMU75_MAX_LEVELS_GRAPHICS];
+
+	void     (*TargetStateCalculator)(uint8_t);
+	void     (*SavedTargetStateCalculator)(uint8_t);
+
+	uint16_t AutoDpmInterval;
+	uint16_t AutoDpmRange;
+
+	uint8_t  FpsEnabled;
+	uint8_t  MaxPerfLevel;
+	uint8_t  AllowLowClkInterruptToHost;
+	uint8_t  FpsRunning;
+
+	uint32_t MaxAllowedFrequency;
+
+	uint32_t FilteredSclkFrequency;
+	uint32_t LastSclkFrequency;
+	uint32_t FilteredSclkFrequencyCnt;
+
+	uint8_t MinPerfLevel;
+#ifdef SMU__FIRMWARE_SCKS_PRESENT__1
+	uint8_t ScksClampMode;
+	uint8_t padding[2];
+#else
+	uint8_t padding[3];
+#endif
+
+	uint16_t FpsAlpha;
+	uint16_t DeltaTime;
+	uint32_t CurrentFps;
+	uint32_t FilteredFps;
+	uint32_t FrameCount;
+	uint32_t FrameCountLast;
+	uint16_t FpsTargetScalar;
+	uint16_t FpsWaterfallLimitScalar;
+	uint16_t FpsAlphaScalar;
+	uint16_t spare8;
+	SMU7_HystController_Data HystControllerData;
+};
+
+typedef struct SMU7_LocalDpmScoreboard SMU7_LocalDpmScoreboard;
+
+#define SMU7_MAX_VOLTAGE_CLIENTS 12
+
+typedef uint8_t (*VoltageChangeHandler_t)(uint16_t, uint8_t);
+
+#define VDDC_MASK    0x00007FFF
+#define VDDC_SHIFT   0
+#define VDDCI_MASK   0x3FFF8000
+#define VDDCI_SHIFT  15
+#define PHASES_MASK  0xC0000000
+#define PHASES_SHIFT 30
+
+typedef uint32_t SMU_VoltageLevel;
+
+struct SMU7_VoltageScoreboard {
+	SMU_VoltageLevel TargetVoltage;
+	uint16_t MaxVid;
+	uint8_t  HighestVidOffset;
+	uint8_t  CurrentVidOffset;
+
+	uint16_t CurrentVddc;
+	uint16_t CurrentVddci;
+
+	uint8_t  ControllerBusy;
+	uint8_t  CurrentVid;
+	uint8_t  CurrentVddciVid;
+	uint8_t  padding;
+
+	SMU_VoltageLevel RequestedVoltage[SMU7_MAX_VOLTAGE_CLIENTS];
+	SMU_VoltageLevel TargetVoltageState;
+	uint8_t  EnabledRequest[SMU7_MAX_VOLTAGE_CLIENTS];
+
+	uint8_t  padding2;
+	uint8_t  padding3;
+	uint8_t  ControllerEnable;
+	uint8_t  ControllerRunning;
+	uint16_t CurrentStdVoltageHiSidd;
+	uint16_t CurrentStdVoltageLoSidd;
+	uint8_t  OverrideVoltage;
+	uint8_t  padding4;
+	uint8_t  padding5;
+	uint8_t  CurrentPhases;
+
+	VoltageChangeHandler_t ChangeVddc;
+	VoltageChangeHandler_t ChangeVddci;
+	VoltageChangeHandler_t ChangePhase;
+	VoltageChangeHandler_t ChangeMvdd;
+
+	VoltageChangeHandler_t functionLinks[6];
+
+	uint16_t * VddcFollower1;
+	int16_t  Driver_OD_RequestedVidOffset1;
+	int16_t  Driver_OD_RequestedVidOffset2;
+};
+
+typedef struct SMU7_VoltageScoreboard SMU7_VoltageScoreboard;
+
+#define SMU7_MAX_PCIE_LINK_SPEEDS 3
+
+struct SMU7_PCIeLinkSpeedScoreboard {
+	uint8_t     DpmEnable;
+	uint8_t     DpmRunning;
+	uint8_t     DpmForce;
+	uint8_t     DpmForceLevel;
+
+	uint8_t     CurrentLinkSpeed;
+	uint8_t     EnabledLevelsChange;
+	uint16_t    AutoDpmInterval;
+
+	uint16_t    AutoDpmRange;
+	uint16_t    AutoDpmCount;
+
+	uint8_t     DpmMode;
+	uint8_t     AcpiReq;
+	uint8_t     AcpiAck;
+	uint8_t     CurrentLinkLevel;
+};
+
+typedef struct SMU7_PCIeLinkSpeedScoreboard SMU7_PCIeLinkSpeedScoreboard;
+
+#define SMU7_LKGE_LUT_NUM_OF_TEMP_ENTRIES 16
+#define SMU7_LKGE_LUT_NUM_OF_VOLT_ENTRIES 16
+
+#define SMU7_SCALE_I  7
+#define SMU7_SCALE_R 12
+
+struct SMU7_PowerScoreboard {
+	uint32_t GpuPower;
+
+	uint32_t VddcPower;
+	uint32_t VddcVoltage;
+	uint32_t VddcCurrent;
+
+	uint32_t VddciPower;
+	uint32_t VddciVoltage;
+	uint32_t VddciCurrent;
+
+	uint32_t RocPower;
+
+	uint16_t Telemetry_1_slope;
+	uint16_t Telemetry_2_slope;
+	int32_t  Telemetry_1_offset;
+	int32_t  Telemetry_2_offset;
+
+	uint8_t MCLK_patch_flag;
+	uint8_t reserved[3];
+};
+
+typedef struct SMU7_PowerScoreboard SMU7_PowerScoreboard;
+
+#define SMU7_SCLK_DPM_CONFIG_MASK                        0x01
+#define SMU7_VOLTAGE_CONTROLLER_CONFIG_MASK              0x02
+#define SMU7_THERMAL_CONTROLLER_CONFIG_MASK              0x04
+#define SMU7_MCLK_DPM_CONFIG_MASK                        0x08
+#define SMU7_UVD_DPM_CONFIG_MASK                         0x10
+#define SMU7_VCE_DPM_CONFIG_MASK                         0x20
+#define SMU7_ACP_DPM_CONFIG_MASK                         0x40
+#define SMU7_SAMU_DPM_CONFIG_MASK                        0x80
+#define SMU7_PCIEGEN_DPM_CONFIG_MASK                    0x100
+
+#define SMU7_ACP_MCLK_HANDSHAKE_DISABLE                  0x00000001
+#define SMU7_ACP_SCLK_HANDSHAKE_DISABLE                  0x00000002
+#define SMU7_UVD_MCLK_HANDSHAKE_DISABLE                  0x00000100
+#define SMU7_UVD_SCLK_HANDSHAKE_DISABLE                  0x00000200
+#define SMU7_VCE_MCLK_HANDSHAKE_DISABLE                  0x00010000
+#define SMU7_VCE_SCLK_HANDSHAKE_DISABLE                  0x00020000
+
+struct SMU75_SoftRegisters {
+	uint32_t        RefClockFrequency;
+	uint32_t        PmTimerPeriod;
+	uint32_t        FeatureEnables;
+#if defined (SMU__DGPU_ONLY)
+	uint32_t        PreVBlankGap;
+	uint32_t        VBlankTimeout;
+	uint32_t        TrainTimeGap;
+	uint32_t        MvddSwitchTime;
+	uint32_t        LongestAcpiTrainTime;
+	uint32_t        AcpiDelay;
+	uint32_t        G5TrainTime;
+	uint32_t        DelayMpllPwron;
+	uint32_t        VoltageChangeTimeout;
+#endif
+	uint32_t        HandshakeDisables;
+
+	uint8_t         DisplayPhy1Config;
+	uint8_t         DisplayPhy2Config;
+	uint8_t         DisplayPhy3Config;
+	uint8_t         DisplayPhy4Config;
+
+	uint8_t         DisplayPhy5Config;
+	uint8_t         DisplayPhy6Config;
+	uint8_t         DisplayPhy7Config;
+	uint8_t         DisplayPhy8Config;
+
+	uint32_t        AverageGraphicsActivity;
+	uint32_t        AverageMemoryActivity;
+	uint32_t        AverageGioActivity;
+
+	uint8_t         SClkDpmEnabledLevels;
+	uint8_t         MClkDpmEnabledLevels;
+	uint8_t         LClkDpmEnabledLevels;
+	uint8_t         PCIeDpmEnabledLevels;
+
+	uint8_t         UVDDpmEnabledLevels;
+	uint8_t         SAMUDpmEnabledLevels;
+	uint8_t         ACPDpmEnabledLevels;
+	uint8_t         VCEDpmEnabledLevels;
+
+	uint32_t        DRAM_LOG_ADDR_H;
+	uint32_t        DRAM_LOG_ADDR_L;
+	uint32_t        DRAM_LOG_PHY_ADDR_H;
+	uint32_t        DRAM_LOG_PHY_ADDR_L;
+	uint32_t        DRAM_LOG_BUFF_SIZE;
+	uint32_t        UlvEnterCount;
+	uint32_t        UlvTime;
+	uint32_t        UcodeLoadStatus;
+	uint32_t        AllowMvddSwitch;
+	uint8_t         Activity_Weight;
+	uint8_t         Reserved8[3];
+};
+
+typedef struct SMU75_SoftRegisters SMU75_SoftRegisters;
+
+struct SMU75_Firmware_Header {
+	uint32_t Digest[5];
+	uint32_t Version;
+	uint32_t HeaderSize;
+	uint32_t Flags;
+	uint32_t EntryPoint;
+	uint32_t CodeSize;
+	uint32_t ImageSize;
+
+	uint32_t Rtos;
+	uint32_t SoftRegisters;
+	uint32_t DpmTable;
+	uint32_t FanTable;
+	uint32_t CacConfigTable;
+	uint32_t CacStatusTable;
+	uint32_t mcRegisterTable;
+	uint32_t mcArbDramTimingTable;
+	uint32_t PmFuseTable;
+	uint32_t Globals;
+	uint32_t ClockStretcherTable;
+	uint32_t VftTable;
+	uint32_t Reserved1;
+	uint32_t AvfsCksOff_AvfsGbvTable;
+	uint32_t AvfsCksOff_BtcGbvTable;
+	uint32_t MM_AvfsTable;
+	uint32_t PowerSharingTable;
+	uint32_t AvfsTable;
+	uint32_t AvfsCksOffGbvTable;
+	uint32_t AvfsMeanNSigma;
+	uint32_t AvfsSclkOffsetTable;
+	uint32_t Reserved[12];
+	uint32_t Signature;
+};
+
+typedef struct SMU75_Firmware_Header SMU75_Firmware_Header;
+
+#define SMU7_FIRMWARE_HEADER_LOCATION 0x20000
+
+enum  DisplayConfig {
+	PowerDown = 1,
+	DP54x4,
+	DP54x2,
+	DP54x1,
+	DP27x4,
+	DP27x2,
+	DP27x1,
+	HDMI297,
+	HDMI162,
+	LVDS,
+	DP324x4,
+	DP324x2,
+	DP324x1
+};
+
+#define MC_BLOCK_COUNT 1
+#define CPL_BLOCK_COUNT 5
+#define SE_BLOCK_COUNT 15
+#define GC_BLOCK_COUNT 24
+
+struct SMU7_Local_Cac {
+	uint8_t BlockId;
+	uint8_t SignalId;
+	uint8_t Threshold;
+	uint8_t Padding;
+};
+
+typedef struct SMU7_Local_Cac SMU7_Local_Cac;
+
+struct SMU7_Local_Cac_Table {
+	SMU7_Local_Cac CplLocalCac[CPL_BLOCK_COUNT];
+	SMU7_Local_Cac McLocalCac[MC_BLOCK_COUNT];
+	SMU7_Local_Cac SeLocalCac[SE_BLOCK_COUNT];
+	SMU7_Local_Cac GcLocalCac[GC_BLOCK_COUNT];
+};
+
+typedef struct SMU7_Local_Cac_Table SMU7_Local_Cac_Table;
+
+#pragma pack(pop)
+
+#define CG_SYS_BITMASK_FIRST_BIT      0
+#define CG_SYS_BITMASK_LAST_BIT       10
+#define CG_SYS_BIF_MGLS_SHIFT         0
+#define CG_SYS_ROM_SHIFT              1
+#define CG_SYS_MC_MGCG_SHIFT          2
+#define CG_SYS_MC_MGLS_SHIFT          3
+#define CG_SYS_SDMA_MGCG_SHIFT        4
+#define CG_SYS_SDMA_MGLS_SHIFT        5
+#define CG_SYS_DRM_MGCG_SHIFT         6
+#define CG_SYS_HDP_MGCG_SHIFT         7
+#define CG_SYS_HDP_MGLS_SHIFT         8
+#define CG_SYS_DRM_MGLS_SHIFT         9
+#define CG_SYS_BIF_MGCG_SHIFT         10
+
+#define CG_SYS_BIF_MGLS_MASK          0x1
+#define CG_SYS_ROM_MASK               0x2
+#define CG_SYS_MC_MGCG_MASK           0x4
+#define CG_SYS_MC_MGLS_MASK           0x8
+#define CG_SYS_SDMA_MGCG_MASK         0x10
+#define CG_SYS_SDMA_MGLS_MASK         0x20
+#define CG_SYS_DRM_MGCG_MASK          0x40
+#define CG_SYS_HDP_MGCG_MASK          0x80
+#define CG_SYS_HDP_MGLS_MASK          0x100
+#define CG_SYS_DRM_MGLS_MASK          0x200
+#define CG_SYS_BIF_MGCG_MASK          0x400
+
+#define CG_GFX_BITMASK_FIRST_BIT      16
+#define CG_GFX_BITMASK_LAST_BIT       24
+
+#define CG_GFX_CGCG_SHIFT             16
+#define CG_GFX_CGLS_SHIFT             17
+#define CG_CPF_MGCG_SHIFT             18
+#define CG_RLC_MGCG_SHIFT             19
+#define CG_GFX_OTHERS_MGCG_SHIFT      20
+#define CG_GFX_3DCG_SHIFT             21
+#define CG_GFX_3DLS_SHIFT             22
+#define CG_GFX_RLC_LS_SHIFT           23
+#define CG_GFX_CP_LS_SHIFT            24
+
+#define CG_GFX_CGCG_MASK              0x00010000
+#define CG_GFX_CGLS_MASK              0x00020000
+#define CG_CPF_MGCG_MASK              0x00040000
+#define CG_RLC_MGCG_MASK              0x00080000
+#define CG_GFX_OTHERS_MGCG_MASK       0x00100000
+#define CG_GFX_3DCG_MASK              0x00200000
+#define CG_GFX_3DLS_MASK              0x00400000
+#define CG_GFX_RLC_LS_MASK            0x00800000
+#define CG_GFX_CP_LS_MASK             0x01000000
+
+
+#define VRCONF_VDDC_MASK         0x000000FF
+#define VRCONF_VDDC_SHIFT        0
+#define VRCONF_VDDGFX_MASK       0x0000FF00
+#define VRCONF_VDDGFX_SHIFT      8
+#define VRCONF_VDDCI_MASK        0x00FF0000
+#define VRCONF_VDDCI_SHIFT       16
+#define VRCONF_MVDD_MASK         0xFF000000
+#define VRCONF_MVDD_SHIFT        24
+
+#define VR_MERGED_WITH_VDDC      0
+#define VR_SVI2_PLANE_1          1
+#define VR_SVI2_PLANE_2          2
+#define VR_SMIO_PATTERN_1        3
+#define VR_SMIO_PATTERN_2        4
+#define VR_STATIC_VOLTAGE        5
+
+#define CLOCK_STRETCHER_MAX_ENTRIES 0x4
+#define CKS_LOOKUPTable_MAX_ENTRIES 0x4
+
+#define CLOCK_STRETCHER_SETTING_DDT_MASK             0x01
+#define CLOCK_STRETCHER_SETTING_DDT_SHIFT            0x0
+#define CLOCK_STRETCHER_SETTING_STRETCH_AMOUNT_MASK  0x1E
+#define CLOCK_STRETCHER_SETTING_STRETCH_AMOUNT_SHIFT 0x1
+#define CLOCK_STRETCHER_SETTING_ENABLE_MASK          0x80
+#define CLOCK_STRETCHER_SETTING_ENABLE_SHIFT         0x7
+
+struct SMU_ClockStretcherDataTableEntry {
+	uint8_t minVID;
+	uint8_t maxVID;
+
+	uint16_t setting;
+};
+typedef struct SMU_ClockStretcherDataTableEntry SMU_ClockStretcherDataTableEntry;
+
+struct SMU_ClockStretcherDataTable {
+	SMU_ClockStretcherDataTableEntry ClockStretcherDataTableEntry[CLOCK_STRETCHER_MAX_ENTRIES];
+};
+typedef struct SMU_ClockStretcherDataTable SMU_ClockStretcherDataTable;
+
+struct SMU_CKS_LOOKUPTableEntry {
+	uint16_t minFreq;
+	uint16_t maxFreq;
+
+	uint8_t setting;
+	uint8_t padding[3];
+};
+typedef struct SMU_CKS_LOOKUPTableEntry SMU_CKS_LOOKUPTableEntry;
+
+struct SMU_CKS_LOOKUPTable {
+	SMU_CKS_LOOKUPTableEntry CKS_LOOKUPTableEntry[CKS_LOOKUPTable_MAX_ENTRIES];
+};
+typedef struct SMU_CKS_LOOKUPTable SMU_CKS_LOOKUPTable;
+
+struct AgmAvfsData_t {
+	uint16_t avgPsmCount[28];
+	uint16_t minPsmCount[28];
+};
+typedef struct AgmAvfsData_t AgmAvfsData_t;
+
+enum VFT_COLUMNS {
+	SCLK0,
+	SCLK1,
+	SCLK2,
+	SCLK3,
+	SCLK4,
+	SCLK5,
+	SCLK6,
+	SCLK7,
+
+	NUM_VFT_COLUMNS
+};
+enum {
+  SCS_FUSE_T0,
+  SCS_FUSE_T1,
+  NUM_SCS_FUSE_TEMPERATURE
+};
+enum {
+  SCKS_ON,
+  SCKS_OFF,
+  NUM_SCKS_STATE_TYPES
+};
+
+#define VFT_TABLE_DEFINED
+
+#define TEMP_RANGE_MAXSTEPS 12
+struct VFT_CELL_t {
+	uint16_t Voltage;
+};
+
+typedef struct VFT_CELL_t VFT_CELL_t;
+#ifdef SMU__FIRMWARE_SCKS_PRESENT__1
+struct SCS_CELL_t {
+	uint16_t PsmCnt[NUM_SCKS_STATE_TYPES];
+};
+typedef struct SCS_CELL_t SCS_CELL_t;
+#endif
+
+struct VFT_TABLE_t {
+	VFT_CELL_t    Cell[TEMP_RANGE_MAXSTEPS][NUM_VFT_COLUMNS];
+	uint16_t      AvfsGbv [NUM_VFT_COLUMNS];
+	uint16_t      BtcGbv  [NUM_VFT_COLUMNS];
+	int16_t       Temperature [TEMP_RANGE_MAXSTEPS];
+
+#ifdef SMU__FIRMWARE_SCKS_PRESENT__1
+	SCS_CELL_t    ScksCell[TEMP_RANGE_MAXSTEPS][NUM_VFT_COLUMNS];
+#endif
+
+	uint8_t       NumTemperatureSteps;
+	uint8_t       padding[3];
+};
+typedef struct VFT_TABLE_t VFT_TABLE_t;
+
+#define BTCGB_VDROOP_TABLE_MAX_ENTRIES 2
+#define AVFSGB_VDROOP_TABLE_MAX_ENTRIES 2
+
+struct GB_VDROOP_TABLE_t {
+	int32_t a0;
+	int32_t a1;
+	int32_t a2;
+	uint32_t spare;
+};
+typedef struct GB_VDROOP_TABLE_t GB_VDROOP_TABLE_t;
+
+struct SMU_QuadraticCoeffs {
+	int32_t m1;
+	int32_t b;
+
+	int16_t m2;
+	uint8_t m1_shift;
+	uint8_t m2_shift;
+};
+typedef struct SMU_QuadraticCoeffs SMU_QuadraticCoeffs;
+
+struct AVFS_Margin_t {
+	VFT_CELL_t Cell[NUM_VFT_COLUMNS];
+};
+typedef struct AVFS_Margin_t AVFS_Margin_t;
+
+struct AVFS_CksOff_Gbv_t {
+	VFT_CELL_t Cell[NUM_VFT_COLUMNS];
+};
+typedef struct AVFS_CksOff_Gbv_t AVFS_CksOff_Gbv_t;
+
+struct AVFS_CksOff_AvfsGbv_t {
+	VFT_CELL_t Cell[NUM_VFT_COLUMNS];
+};
+typedef struct AVFS_CksOff_AvfsGbv_t AVFS_CksOff_AvfsGbv_t;
+
+struct AVFS_CksOff_BtcGbv_t {
+	VFT_CELL_t Cell[NUM_VFT_COLUMNS];
+};
+typedef struct AVFS_CksOff_BtcGbv_t AVFS_CksOff_BtcGbv_t;
+
+struct AVFS_meanNsigma_t {
+	uint32_t Aconstant[3];
+	uint16_t DC_tol_sigma;
+	uint16_t Platform_mean;
+	uint16_t Platform_sigma;
+	uint16_t PSM_Age_CompFactor;
+	uint8_t  Static_Voltage_Offset[NUM_VFT_COLUMNS];
+};
+typedef struct AVFS_meanNsigma_t AVFS_meanNsigma_t;
+
+struct AVFS_Sclk_Offset_t {
+	uint16_t Sclk_Offset[8];
+};
+typedef struct AVFS_Sclk_Offset_t AVFS_Sclk_Offset_t;
+
+struct Power_Sharing_t {
+	uint32_t EnergyCounter;
+	uint32_t EngeryThreshold;
+	uint64_t AM_SCLK_CNT;
+	uint64_t AM_0_BUSY_CNT;
+};
+typedef struct Power_Sharing_t  Power_Sharing_t;
+
+
+#endif
+
+
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu75_discrete.h b/drivers/gpu/drm/amd/powerplay/inc/smu75_discrete.h
new file mode 100644
index 000000000000..b64e58a22ddf
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu75_discrete.h
@@ -0,0 +1,886 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef SMU75_DISCRETE_H
+#define SMU75_DISCRETE_H
+
+#include "smu75.h"
+
+#pragma pack(push, 1)
+
+#define NUM_SCLK_RANGE 8
+
+#define VCO_3_6 1
+#define VCO_2_4 3
+
+#define POSTDIV_DIV_BY_1  0
+#define POSTDIV_DIV_BY_2  1
+#define POSTDIV_DIV_BY_4  2
+#define POSTDIV_DIV_BY_8  3
+#define POSTDIV_DIV_BY_16 4
+
+struct sclkFcwRange_t {
+	uint8_t  vco_setting; /* 1: 3-6GHz, 3: 2-4GHz */
+	uint8_t  postdiv;     /* divide by 2^n */
+	uint16_t fcw_pcc;
+	uint16_t fcw_trans_upper;
+	uint16_t fcw_trans_lower;
+};
+typedef struct sclkFcwRange_t sclkFcwRange_t;
+
+struct SMIO_Pattern {
+	uint16_t Voltage;
+	uint8_t  Smio;
+	uint8_t  padding;
+};
+
+typedef struct SMIO_Pattern SMIO_Pattern;
+
+struct SMIO_Table {
+	SMIO_Pattern Pattern[SMU_MAX_SMIO_LEVELS];
+};
+
+typedef struct SMIO_Table SMIO_Table;
+
+struct SMU_SclkSetting {
+	uint32_t    SclkFrequency;
+	uint16_t    Fcw_int;
+	uint16_t    Fcw_frac;
+	uint16_t    Pcc_fcw_int;
+	uint8_t     PllRange;
+	uint8_t     SSc_En;
+	uint16_t    Sclk_slew_rate;
+	uint16_t    Pcc_up_slew_rate;
+	uint16_t    Pcc_down_slew_rate;
+	uint16_t    Fcw1_int;
+	uint16_t    Fcw1_frac;
+	uint16_t    Sclk_ss_slew_rate;
+};
+typedef struct SMU_SclkSetting SMU_SclkSetting;
+
+struct SMU75_Discrete_GraphicsLevel {
+	SMU_VoltageLevel MinVoltage;
+
+	uint8_t     pcieDpmLevel;
+	uint8_t     DeepSleepDivId;
+	uint16_t    ActivityLevel;
+
+	uint32_t    CgSpllFuncCntl3;
+	uint32_t    CgSpllFuncCntl4;
+	uint32_t    CcPwrDynRm;
+	uint32_t    CcPwrDynRm1;
+
+	uint8_t     SclkDid;
+	uint8_t     padding;
+	uint8_t     EnabledForActivity;
+	uint8_t     EnabledForThrottle;
+	uint8_t     UpHyst;
+	uint8_t     DownHyst;
+	uint8_t     VoltageDownHyst;
+	uint8_t     PowerThrottle;
+
+	SMU_SclkSetting SclkSetting;
+
+	uint8_t  ScksStretchThreshVid[NUM_SCKS_STATE_TYPES];
+	uint16_t Padding;
+};
+
+typedef struct SMU75_Discrete_GraphicsLevel SMU75_Discrete_GraphicsLevel;
+
+struct SMU75_Discrete_ACPILevel {
+	uint32_t    Flags;
+	SMU_VoltageLevel MinVoltage;
+	uint32_t    SclkFrequency;
+	uint8_t     SclkDid;
+	uint8_t     DisplayWatermark;
+	uint8_t     DeepSleepDivId;
+	uint8_t     padding;
+	uint32_t    CcPwrDynRm;
+	uint32_t    CcPwrDynRm1;
+
+	SMU_SclkSetting SclkSetting;
+};
+
+typedef struct SMU75_Discrete_ACPILevel SMU75_Discrete_ACPILevel;
+
+struct SMU75_Discrete_Ulv {
+	uint32_t    CcPwrDynRm;
+	uint32_t    CcPwrDynRm1;
+	uint16_t    VddcOffset;
+	uint8_t     VddcOffsetVid;
+	uint8_t     VddcPhase;
+	uint16_t    BifSclkDfs;
+	uint16_t    Reserved;
+};
+
+typedef struct SMU75_Discrete_Ulv SMU75_Discrete_Ulv;
+
+struct SMU75_Discrete_MemoryLevel {
+	SMU_VoltageLevel MinVoltage;
+	uint32_t    MinMvdd;
+
+	uint32_t    MclkFrequency;
+
+	uint8_t     StutterEnable;
+	uint8_t     EnabledForThrottle;
+	uint8_t     EnabledForActivity;
+	uint8_t     padding_0;
+
+	uint8_t     UpHyst;
+	uint8_t     DownHyst;
+	uint8_t     VoltageDownHyst;
+	uint8_t     padding_1;
+
+	uint16_t    ActivityLevel;
+	uint8_t     DisplayWatermark;
+	uint8_t     padding_2;
+
+	uint16_t    Fcw_int;
+	uint16_t    Fcw_frac;
+	uint8_t     Postdiv;
+	uint8_t     padding_3[3];
+};
+
+typedef struct SMU75_Discrete_MemoryLevel SMU75_Discrete_MemoryLevel;
+
+struct SMU75_Discrete_LinkLevel {
+	uint8_t     PcieGenSpeed;
+	uint8_t     PcieLaneCount;
+	uint8_t     EnabledForActivity;
+	uint8_t     SPC;
+	uint32_t    DownThreshold;
+	uint32_t    UpThreshold;
+	uint16_t    BifSclkDfs;
+	uint16_t    Reserved;
+};
+
+typedef struct SMU75_Discrete_LinkLevel SMU75_Discrete_LinkLevel;
+
+
+/* MC ARB DRAM Timing registers. */
+struct SMU75_Discrete_MCArbDramTimingTableEntry {
+	uint32_t McArbDramTiming;
+	uint32_t McArbDramTiming2;
+	uint32_t McArbBurstTime;
+	uint32_t McArbRfshRate;
+	uint32_t McArbMisc3;
+};
+
+typedef struct SMU75_Discrete_MCArbDramTimingTableEntry SMU75_Discrete_MCArbDramTimingTableEntry;
+
+struct SMU75_Discrete_MCArbDramTimingTable {
+	SMU75_Discrete_MCArbDramTimingTableEntry entries[SMU__NUM_SCLK_DPM_STATE][SMU__NUM_MCLK_DPM_LEVELS];
+};
+
+typedef struct SMU75_Discrete_MCArbDramTimingTable SMU75_Discrete_MCArbDramTimingTable;
+
+/* UVD VCLK/DCLK state (level) definition. */
+struct SMU75_Discrete_UvdLevel {
+	uint32_t VclkFrequency;
+	uint32_t DclkFrequency;
+	SMU_VoltageLevel MinVoltage;
+	uint8_t  VclkDivider;
+	uint8_t  DclkDivider;
+	uint8_t  padding[2];
+};
+
+typedef struct SMU75_Discrete_UvdLevel SMU75_Discrete_UvdLevel;
+
+/* Clocks for other external blocks (VCE, ACP, SAMU). */
+struct SMU75_Discrete_ExtClkLevel {
+	uint32_t Frequency;
+	SMU_VoltageLevel MinVoltage;
+	uint8_t  Divider;
+	uint8_t  padding[3];
+};
+
+typedef struct SMU75_Discrete_ExtClkLevel SMU75_Discrete_ExtClkLevel;
+
+struct SMU75_Discrete_StateInfo {
+	uint32_t SclkFrequency;
+	uint32_t MclkFrequency;
+	uint32_t VclkFrequency;
+	uint32_t DclkFrequency;
+	uint32_t SamclkFrequency;
+	uint32_t AclkFrequency;
+	uint32_t EclkFrequency;
+	uint16_t MvddVoltage;
+	uint16_t padding16;
+	uint8_t  DisplayWatermark;
+	uint8_t  McArbIndex;
+	uint8_t  McRegIndex;
+	uint8_t  SeqIndex;
+	uint8_t  SclkDid;
+	int8_t   SclkIndex;
+	int8_t   MclkIndex;
+	uint8_t  PCIeGen;
+};
+
+typedef struct SMU75_Discrete_StateInfo SMU75_Discrete_StateInfo;
+
+struct SMU75_Discrete_DpmTable {
+	SMU75_PIDController                  GraphicsPIDController;
+	SMU75_PIDController                  MemoryPIDController;
+	SMU75_PIDController                  LinkPIDController;
+
+	uint32_t                            SystemFlags;
+
+	uint32_t                            VRConfig;
+	uint32_t                            SmioMask1;
+	uint32_t                            SmioMask2;
+	SMIO_Table                          SmioTable1;
+	SMIO_Table                          SmioTable2;
+
+	uint32_t                            MvddLevelCount;
+
+	uint8_t                             BapmVddcVidHiSidd        [SMU75_MAX_LEVELS_VDDC];
+	uint8_t                             BapmVddcVidLoSidd        [SMU75_MAX_LEVELS_VDDC];
+	uint8_t                             BapmVddcVidHiSidd2       [SMU75_MAX_LEVELS_VDDC];
+
+	uint8_t                             GraphicsDpmLevelCount;
+	uint8_t                             MemoryDpmLevelCount;
+	uint8_t                             LinkLevelCount;
+	uint8_t                             MasterDeepSleepControl;
+
+	uint8_t                             UvdLevelCount;
+	uint8_t                             VceLevelCount;
+	uint8_t                             AcpLevelCount;
+	uint8_t                             SamuLevelCount;
+
+	uint8_t                             ThermOutGpio;
+	uint8_t                             ThermOutPolarity;
+	uint8_t                             ThermOutMode;
+	uint8_t                             BootPhases;
+
+	uint8_t                             VRHotLevel;
+	uint8_t                             LdoRefSel;
+
+	uint8_t                             Reserved1[2];
+
+	uint16_t                            FanStartTemperature;
+	uint16_t                            FanStopTemperature;
+
+	uint16_t                            MaxVoltage;
+	uint16_t                            Reserved2;
+	uint32_t                            Reserved;
+
+	SMU75_Discrete_GraphicsLevel        GraphicsLevel           [SMU75_MAX_LEVELS_GRAPHICS];
+	SMU75_Discrete_MemoryLevel          MemoryACPILevel;
+	SMU75_Discrete_MemoryLevel          MemoryLevel             [SMU75_MAX_LEVELS_MEMORY];
+	SMU75_Discrete_LinkLevel            LinkLevel               [SMU75_MAX_LEVELS_LINK];
+	SMU75_Discrete_ACPILevel            ACPILevel;
+	SMU75_Discrete_UvdLevel             UvdLevel                [SMU75_MAX_LEVELS_UVD];
+	SMU75_Discrete_ExtClkLevel          VceLevel                [SMU75_MAX_LEVELS_VCE];
+	SMU75_Discrete_ExtClkLevel          AcpLevel                [SMU75_MAX_LEVELS_ACP];
+	SMU75_Discrete_ExtClkLevel          SamuLevel               [SMU75_MAX_LEVELS_SAMU];
+	SMU75_Discrete_Ulv                  Ulv;
+
+	uint8_t                             DisplayWatermark        [SMU75_MAX_LEVELS_MEMORY][SMU75_MAX_LEVELS_GRAPHICS];
+
+	uint32_t                            SclkStepSize;
+	uint32_t                            Smio                    [SMU75_MAX_ENTRIES_SMIO];
+
+	uint8_t                             UvdBootLevel;
+	uint8_t                             VceBootLevel;
+	uint8_t                             AcpBootLevel;
+	uint8_t                             SamuBootLevel;
+
+	uint8_t                             GraphicsBootLevel;
+	uint8_t                             GraphicsVoltageChangeEnable;
+	uint8_t                             GraphicsThermThrottleEnable;
+	uint8_t                             GraphicsInterval;
+
+	uint8_t                             VoltageInterval;
+	uint8_t                             ThermalInterval;
+	uint16_t                            TemperatureLimitHigh;
+
+	uint16_t                            TemperatureLimitLow;
+	uint8_t                             MemoryBootLevel;
+	uint8_t                             MemoryVoltageChangeEnable;
+
+	uint16_t                            BootMVdd;
+	uint8_t                             MemoryInterval;
+	uint8_t                             MemoryThermThrottleEnable;
+
+	uint16_t                            VoltageResponseTime;
+	uint16_t                            PhaseResponseTime;
+
+	uint8_t                             PCIeBootLinkLevel;
+	uint8_t                             PCIeGenInterval;
+	uint8_t                             DTEInterval;
+	uint8_t                             DTEMode;
+
+	uint8_t                             SVI2Enable;
+	uint8_t                             VRHotGpio;
+	uint8_t                             AcDcGpio;
+	uint8_t                             ThermGpio;
+
+	uint16_t                            PPM_PkgPwrLimit;
+	uint16_t                            PPM_TemperatureLimit;
+
+	uint16_t                            DefaultTdp;
+	uint16_t                            TargetTdp;
+
+	uint16_t                            FpsHighThreshold;
+	uint16_t                            FpsLowThreshold;
+
+	uint16_t                            BAPMTI_R  [SMU75_DTE_ITERATIONS][SMU75_DTE_SOURCES][SMU75_DTE_SINKS];
+	uint16_t                            BAPMTI_RC [SMU75_DTE_ITERATIONS][SMU75_DTE_SOURCES][SMU75_DTE_SINKS];
+
+	uint16_t                            TemperatureLimitEdge;
+	uint16_t                            TemperatureLimitHotspot;
+
+	uint16_t                            BootVddc;
+	uint16_t                            BootVddci;
+
+	uint16_t                            FanGainEdge;
+	uint16_t                            FanGainHotspot;
+
+	uint32_t                            LowSclkInterruptThreshold;
+	uint32_t                            VddGfxReChkWait;
+
+	uint8_t                             ClockStretcherAmount;
+	uint8_t                             Sclk_CKS_masterEn0_7;
+	uint8_t                             Sclk_CKS_masterEn8_15;
+	uint8_t                             DPMFreezeAndForced;
+
+	uint8_t                             Sclk_voltageOffset[8];
+
+	SMU_ClockStretcherDataTable         ClockStretcherDataTable;
+	SMU_CKS_LOOKUPTable                 CKS_LOOKUPTable;
+
+	uint32_t                            CurrSclkPllRange;
+	sclkFcwRange_t                      SclkFcwRangeTable[NUM_SCLK_RANGE];
+
+	GB_VDROOP_TABLE_t                   BTCGB_VDROOP_TABLE[BTCGB_VDROOP_TABLE_MAX_ENTRIES];
+	SMU_QuadraticCoeffs                 AVFSGB_FUSE_TABLE[AVFSGB_VDROOP_TABLE_MAX_ENTRIES];
+};
+
+typedef struct SMU75_Discrete_DpmTable SMU75_Discrete_DpmTable;
+
+struct SMU75_Discrete_FanTable {
+	uint16_t FdoMode;
+	int16_t  TempMin;
+	int16_t  TempMed;
+	int16_t  TempMax;
+	int16_t  Slope1;
+	int16_t  Slope2;
+	int16_t  FdoMin;
+	int16_t  HystUp;
+	int16_t  HystDown;
+	int16_t  HystSlope;
+	int16_t  TempRespLim;
+	int16_t  TempCurr;
+	int16_t  SlopeCurr;
+	int16_t  PwmCurr;
+	uint32_t RefreshPeriod;
+	int16_t  FdoMax;
+	uint8_t  TempSrc;
+	int8_t   Padding;
+};
+
+typedef struct SMU75_Discrete_FanTable SMU75_Discrete_FanTable;
+
+#define SMU7_DISCRETE_GPIO_SCLK_DEBUG             4
+#define SMU7_DISCRETE_GPIO_SCLK_DEBUG_BIT         (0x1 << SMU7_DISCRETE_GPIO_SCLK_DEBUG)
+
+
+
+struct SMU7_MclkDpmScoreboard {
+	uint32_t PercentageBusy;
+
+	int32_t  PIDError;
+	int32_t  PIDIntegral;
+	int32_t  PIDOutput;
+
+	uint32_t SigmaDeltaAccum;
+	uint32_t SigmaDeltaOutput;
+	uint32_t SigmaDeltaLevel;
+
+	uint32_t UtilizationSetpoint;
+
+	uint8_t  TdpClampMode;
+	uint8_t  TdcClampMode;
+	uint8_t  ThermClampMode;
+	uint8_t  VoltageBusy;
+
+	int8_t   CurrLevel;
+	int8_t   TargLevel;
+	uint8_t  LevelChangeInProgress;
+	uint8_t  UpHyst;
+
+	uint8_t  DownHyst;
+	uint8_t  VoltageDownHyst;
+	uint8_t  DpmEnable;
+	uint8_t  DpmRunning;
+
+	uint8_t  DpmForce;
+	uint8_t  DpmForceLevel;
+	uint8_t  padding2;
+	uint8_t  McArbIndex;
+
+	uint32_t MinimumPerfMclk;
+
+	uint8_t  AcpiReq;
+	uint8_t  AcpiAck;
+	uint8_t  MclkSwitchInProgress;
+	uint8_t  MclkSwitchCritical;
+
+	uint8_t  IgnoreVBlank;
+	uint8_t  TargetMclkIndex;
+	uint8_t  TargetMvddIndex;
+	uint8_t  MclkSwitchResult;
+
+	uint16_t VbiFailureCount;
+	uint8_t  VbiWaitCounter;
+	uint8_t  EnabledLevelsChange;
+
+	uint16_t LevelResidencyCounters [SMU75_MAX_LEVELS_MEMORY];
+	uint16_t LevelSwitchCounters [SMU75_MAX_LEVELS_MEMORY];
+
+	void     (*TargetStateCalculator)(uint8_t);
+	void     (*SavedTargetStateCalculator)(uint8_t);
+
+	uint16_t AutoDpmInterval;
+	uint16_t AutoDpmRange;
+
+	uint16_t VbiTimeoutCount;
+	uint16_t MclkSwitchingTime;
+
+	uint8_t  fastSwitch;
+	uint8_t  Save_PIC_VDDGFX_EXIT;
+	uint8_t  Save_PIC_VDDGFX_ENTER;
+	uint8_t  VbiTimeout;
+
+	uint32_t HbmTempRegBackup;
+};
+
+typedef struct SMU7_MclkDpmScoreboard SMU7_MclkDpmScoreboard;
+
+struct SMU7_UlvScoreboard {
+	uint8_t     EnterUlv;
+	uint8_t     ExitUlv;
+	uint8_t     UlvActive;
+	uint8_t     WaitingForUlv;
+	uint8_t     UlvEnable;
+	uint8_t     UlvRunning;
+	uint8_t     UlvMasterEnable;
+	uint8_t     padding;
+	uint32_t    UlvAbortedCount;
+	uint32_t    UlvTimeStamp;
+};
+
+typedef struct SMU7_UlvScoreboard SMU7_UlvScoreboard;
+
+struct VddgfxSavedRegisters {
+	uint32_t GPU_DBG[3];
+	uint32_t MEC_BaseAddress_Hi;
+	uint32_t MEC_BaseAddress_Lo;
+	uint32_t THM_TMON0_CTRL2__RDIR_PRESENT;
+	uint32_t THM_TMON1_CTRL2__RDIR_PRESENT;
+	uint32_t CP_INT_CNTL;
+};
+
+typedef struct VddgfxSavedRegisters VddgfxSavedRegisters;
+
+struct SMU7_VddGfxScoreboard {
+	uint8_t     VddGfxEnable;
+	uint8_t     VddGfxActive;
+	uint8_t     VPUResetOccured;
+	uint8_t     padding;
+
+	uint32_t    VddGfxEnteredCount;
+	uint32_t    VddGfxAbortedCount;
+
+	uint32_t    VddGfxVid;
+
+	VddgfxSavedRegisters SavedRegisters;
+};
+
+typedef struct SMU7_VddGfxScoreboard SMU7_VddGfxScoreboard;
+
+struct SMU7_TdcLimitScoreboard {
+	uint8_t  Enable;
+	uint8_t  Running;
+	uint16_t Alpha;
+	uint32_t FilteredIddc;
+	uint32_t IddcLimit;
+	uint32_t IddcHyst;
+	SMU7_HystController_Data HystControllerData;
+};
+
+typedef struct SMU7_TdcLimitScoreboard SMU7_TdcLimitScoreboard;
+
+struct SMU7_PkgPwrLimitScoreboard {
+	uint8_t  Enable;
+	uint8_t  Running;
+	uint16_t Alpha;
+	uint32_t FilteredPkgPwr;
+	uint32_t Limit;
+	uint32_t Hyst;
+	uint32_t LimitFromDriver;
+	uint8_t PowerSharingEnabled;
+	uint8_t PowerSharingCounter;
+	uint8_t PowerSharingINTEnabled;
+	uint8_t GFXActivityCounterEnabled;
+	uint32_t EnergyCount;
+	uint32_t PSACTCount;
+	uint8_t RollOverRequired;
+	uint8_t RollOverCount;
+	uint8_t padding[2];
+	SMU7_HystController_Data HystControllerData;
+};
+
+typedef struct SMU7_PkgPwrLimitScoreboard SMU7_PkgPwrLimitScoreboard;
+
+struct SMU7_BapmScoreboard {
+	uint32_t source_powers[SMU75_DTE_SOURCES];
+	uint32_t source_powers_last[SMU75_DTE_SOURCES];
+	int32_t entity_temperatures[SMU75_NUM_GPU_TES];
+	int32_t initial_entity_temperatures[SMU75_NUM_GPU_TES];
+	int32_t Limit;
+	int32_t Hyst;
+	int32_t therm_influence_coeff_table[SMU75_DTE_ITERATIONS * SMU75_DTE_SOURCES * SMU75_DTE_SINKS * 2];
+	int32_t therm_node_table[SMU75_DTE_ITERATIONS * SMU75_DTE_SOURCES * SMU75_DTE_SINKS];
+	uint16_t ConfigTDPPowerScalar;
+	uint16_t FanSpeedPowerScalar;
+	uint16_t OverDrivePowerScalar;
+	uint16_t OverDriveLimitScalar;
+	uint16_t FinalPowerScalar;
+	uint8_t VariantID;
+	uint8_t spare997;
+
+	SMU7_HystController_Data HystControllerData;
+
+	int32_t temperature_gradient_slope;
+	int32_t temperature_gradient;
+	uint32_t measured_temperature;
+};
+
+
+typedef struct SMU7_BapmScoreboard SMU7_BapmScoreboard;
+
+struct SMU7_AcpiScoreboard {
+	uint32_t SavedInterruptMask[2];
+	uint8_t LastACPIRequest;
+	uint8_t CgBifResp;
+	uint8_t RequestType;
+	uint8_t Padding;
+	SMU75_Discrete_ACPILevel D0Level;
+};
+
+typedef struct SMU7_AcpiScoreboard SMU7_AcpiScoreboard;
+
+struct SMU75_Discrete_PmFuses {
+	uint8_t BapmVddCVidHiSidd[8];
+
+	uint8_t BapmVddCVidLoSidd[8];
+
+	uint8_t VddCVid[8];
+
+	uint8_t SviLoadLineEn;
+	uint8_t SviLoadLineVddC;
+	uint8_t SviLoadLineTrimVddC;
+	uint8_t SviLoadLineOffsetVddC;
+
+	uint16_t TDC_VDDC_PkgLimit;
+	uint8_t TDC_VDDC_ThrottleReleaseLimitPerc;
+	uint8_t TDC_MAWt;
+
+	uint8_t TdcWaterfallCtl;
+	uint8_t LPMLTemperatureMin;
+	uint8_t LPMLTemperatureMax;
+	uint8_t Reserved;
+
+	uint8_t LPMLTemperatureScaler[16];
+
+	int16_t FuzzyFan_ErrorSetDelta;
+	int16_t FuzzyFan_ErrorRateSetDelta;
+	int16_t FuzzyFan_PwmSetDelta;
+	uint16_t Reserved6;
+
+	uint8_t GnbLPML[16];
+
+	uint8_t GnbLPMLMaxVid;
+	uint8_t GnbLPMLMinVid;
+	uint8_t Reserved1[2];
+
+	uint16_t BapmVddCBaseLeakageHiSidd;
+	uint16_t BapmVddCBaseLeakageLoSidd;
+
+	uint16_t  VFT_Temp[3];
+	uint8_t   Version;
+	uint8_t   padding;
+
+	SMU_QuadraticCoeffs VFT_ATE[3];
+
+	SMU_QuadraticCoeffs AVFS_GB;
+	SMU_QuadraticCoeffs ATE_ACBTC_GB;
+
+	SMU_QuadraticCoeffs P2V;
+
+	uint32_t PsmCharzFreq;
+
+	uint16_t InversionVoltage;
+	uint16_t PsmCharzTemp;
+
+	uint32_t EnabledAvfsModules;
+
+	SMU_QuadraticCoeffs BtcGbv_CksOff;
+};
+
+typedef struct SMU75_Discrete_PmFuses SMU75_Discrete_PmFuses;
+
+struct SMU7_Discrete_Log_Header_Table {
+	uint32_t    version;
+	uint32_t    asic_id;
+	uint16_t    flags;
+	uint16_t    entry_size;
+	uint32_t    total_size;
+	uint32_t    num_of_entries;
+	uint8_t     type;
+	uint8_t     mode;
+	uint8_t     filler_0[2];
+	uint32_t    filler_1[2];
+};
+
+typedef struct SMU7_Discrete_Log_Header_Table SMU7_Discrete_Log_Header_Table;
+
+struct SMU7_Discrete_Log_Cntl {
+	uint8_t             Enabled;
+	uint8_t             Type;
+	uint8_t             padding[2];
+	uint32_t            BufferSize;
+	uint32_t            SamplesLogged;
+	uint32_t            SampleSize;
+	uint32_t            AddrL;
+	uint32_t            AddrH;
+};
+
+typedef struct SMU7_Discrete_Log_Cntl SMU7_Discrete_Log_Cntl;
+
+#if defined SMU__DGPU_ONLY
+#define CAC_ACC_NW_NUM_OF_SIGNALS 87
+#endif
+
+
+struct SMU7_Discrete_Cac_Collection_Table {
+	uint32_t temperature;
+	uint32_t cac_acc_nw[CAC_ACC_NW_NUM_OF_SIGNALS];
+};
+
+typedef struct SMU7_Discrete_Cac_Collection_Table SMU7_Discrete_Cac_Collection_Table;
+
+struct SMU7_Discrete_Cac_Verification_Table {
+	uint32_t VddcTotalPower;
+	uint32_t VddcLeakagePower;
+	uint32_t VddcConstantPower;
+	uint32_t VddcGfxDynamicPower;
+	uint32_t VddcUvdDynamicPower;
+	uint32_t VddcVceDynamicPower;
+	uint32_t VddcAcpDynamicPower;
+	uint32_t VddcPcieDynamicPower;
+	uint32_t VddcDceDynamicPower;
+	uint32_t VddcCurrent;
+	uint32_t VddcVoltage;
+	uint32_t VddciTotalPower;
+	uint32_t VddciLeakagePower;
+	uint32_t VddciConstantPower;
+	uint32_t VddciDynamicPower;
+	uint32_t Vddr1TotalPower;
+	uint32_t Vddr1LeakagePower;
+	uint32_t Vddr1ConstantPower;
+	uint32_t Vddr1DynamicPower;
+	uint32_t spare[4];
+	uint32_t temperature;
+};
+
+typedef struct SMU7_Discrete_Cac_Verification_Table SMU7_Discrete_Cac_Verification_Table;
+
+struct SMU7_Discrete_Pm_Status_Table {
+	int32_t T_meas_max[SMU75_THERMAL_INPUT_LOOP_COUNT];
+	int32_t T_meas_acc[SMU75_THERMAL_INPUT_LOOP_COUNT];
+
+	uint32_t I_calc_max;
+	uint32_t I_calc_acc;
+	uint32_t P_meas_acc;
+	uint32_t V_meas_load_acc;
+	uint32_t I_meas_acc;
+	uint32_t P_meas_acc_vddci;
+	uint32_t V_meas_load_acc_vddci;
+	uint32_t I_meas_acc_vddci;
+
+	uint16_t Sclk_dpm_residency[8];
+	uint16_t Uvd_dpm_residency[8];
+	uint16_t Vce_dpm_residency[8];
+	uint16_t Mclk_dpm_residency[4];
+
+	uint32_t P_roc_acc;
+	uint32_t PkgPwr_max;
+	uint32_t PkgPwr_acc;
+	uint32_t MclkSwitchingTime_max;
+	uint32_t MclkSwitchingTime_acc;
+	uint32_t FanPwm_acc;
+	uint32_t FanRpm_acc;
+	uint32_t Gfx_busy_acc;
+	uint32_t Mc_busy_acc;
+	uint32_t Fps_acc;
+
+	uint32_t AccCnt;
+};
+
+typedef struct SMU7_Discrete_Pm_Status_Table SMU7_Discrete_Pm_Status_Table;
+
+struct SMU7_Discrete_AutoWattMan_Status_Table {
+	int32_t T_meas_acc[SMU75_THERMAL_INPUT_LOOP_COUNT];
+	uint16_t Sclk_dpm_residency[8];
+	uint16_t Mclk_dpm_residency[4];
+	uint32_t TgpPwr_acc;
+	uint32_t Gfx_busy_acc;
+	uint32_t Mc_busy_acc;
+	uint32_t AccCnt;
+};
+
+typedef struct SMU7_Discrete_AutoWattMan_Status_Table SMU7_Discrete_AutoWattMan_Status_Table;
+
+#define SMU7_MAX_GFX_CU_COUNT 24
+#define SMU7_MIN_GFX_CU_COUNT  8
+#define SMU7_GFX_CU_PG_ENABLE_DC_MAX_CU_SHIFT 0
+#define SMU7_GFX_CU_PG_ENABLE_DC_MAX_CU_MASK  (0xFFFF << SMU7_GFX_CU_PG_ENABLE_DC_MAX_CU_SHIFT)
+#define SMU7_GFX_CU_PG_ENABLE_AC_MAX_CU_SHIFT 16
+#define SMU7_GFX_CU_PG_ENABLE_AC_MAX_CU_MASK  (0xFFFF << SMU7_GFX_CU_PG_ENABLE_AC_MAX_CU_SHIFT)
+
+struct SMU7_GfxCuPgScoreboard {
+	uint8_t Enabled;
+	uint8_t WaterfallUp;
+	uint8_t WaterfallDown;
+	uint8_t WaterfallLimit;
+	uint8_t CurrMaxCu;
+	uint8_t TargMaxCu;
+	uint8_t ClampMode;
+	uint8_t Active;
+	uint8_t MaxSupportedCu;
+	uint8_t MinSupportedCu;
+	uint8_t PendingGfxCuHostInterrupt;
+	uint8_t LastFilteredMaxCuInteger;
+	uint16_t FilteredMaxCu;
+	uint16_t FilteredMaxCuAlpha;
+	uint16_t FilterResetCount;
+	uint16_t FilterResetCountLimit;
+	uint8_t ForceCu;
+	uint8_t ForceCuCount;
+	uint8_t AcModeMaxCu;
+	uint8_t DcModeMaxCu;
+};
+
+typedef struct SMU7_GfxCuPgScoreboard SMU7_GfxCuPgScoreboard;
+
+#define SMU7_SCLK_CAC 0x561
+#define SMU7_MCLK_CAC 0xF9
+#define SMU7_VCLK_CAC 0x2DE
+#define SMU7_DCLK_CAC 0x2DE
+#define SMU7_ECLK_CAC 0x25E
+#define SMU7_ACLK_CAC 0x25E
+#define SMU7_SAMCLK_CAC 0x25E
+#define SMU7_DISPCLK_CAC 0x100
+#define SMU7_CAC_CONSTANT 0x2EE3430
+#define SMU7_CAC_CONSTANT_SHIFT 18
+
+#define SMU7_VDDCI_MCLK_CONST        1765
+#define SMU7_VDDCI_MCLK_CONST_SHIFT  16
+#define SMU7_VDDCI_VDDCI_CONST       50958
+#define SMU7_VDDCI_VDDCI_CONST_SHIFT 14
+#define SMU7_VDDCI_CONST             11781
+#define SMU7_VDDCI_STROBE_PWR        1331
+
+#define SMU7_VDDR1_CONST            693
+#define SMU7_VDDR1_CAC_WEIGHT       20
+#define SMU7_VDDR1_CAC_WEIGHT_SHIFT 19
+#define SMU7_VDDR1_STROBE_PWR       512
+
+#define SMU7_AREA_COEFF_UVD 0xA78
+#define SMU7_AREA_COEFF_VCE 0x190A
+#define SMU7_AREA_COEFF_ACP 0x22D1
+#define SMU7_AREA_COEFF_SAMU 0x534
+
+#define SMU7_THERM_OUT_MODE_DISABLE       0x0
+#define SMU7_THERM_OUT_MODE_THERM_ONLY    0x1
+#define SMU7_THERM_OUT_MODE_THERM_VRHOT   0x2
+
+#define SQ_Enable_MASK 0x1
+#define SQ_IR_MASK 0x2
+#define SQ_PCC_MASK 0x4
+#define SQ_EDC_MASK 0x8
+
+#define TCP_Enable_MASK 0x100
+#define TCP_IR_MASK 0x200
+#define TCP_PCC_MASK 0x400
+#define TCP_EDC_MASK 0x800
+
+#define TD_Enable_MASK 0x10000
+#define TD_IR_MASK 0x20000
+#define TD_PCC_MASK 0x40000
+#define TD_EDC_MASK 0x80000
+
+#define DB_Enable_MASK 0x1000000
+#define DB_IR_MASK 0x2000000
+#define DB_PCC_MASK 0x4000000
+#define DB_EDC_MASK 0x8000000
+
+#define SQ_Enable_SHIFT 0
+#define SQ_IR_SHIFT 1
+#define SQ_PCC_SHIFT 2
+#define SQ_EDC_SHIFT 3
+
+#define TCP_Enable_SHIFT 8
+#define TCP_IR_SHIFT 9
+#define TCP_PCC_SHIFT 10
+#define TCP_EDC_SHIFT 11
+
+#define TD_Enable_SHIFT 16
+#define TD_IR_SHIFT 17
+#define TD_PCC_SHIFT 18
+#define TD_EDC_SHIFT 19
+
+#define DB_Enable_SHIFT 24
+#define DB_IR_SHIFT 25
+#define DB_PCC_SHIFT 26
+#define DB_EDC_SHIFT 27
+
+#define PMFUSES_AVFSSIZE 104
+
+#define BTCGB0_Vdroop_Enable_MASK  0x1
+#define BTCGB1_Vdroop_Enable_MASK  0x2
+#define AVFSGB0_Vdroop_Enable_MASK 0x4
+#define AVFSGB1_Vdroop_Enable_MASK 0x8
+
+#define BTCGB0_Vdroop_Enable_SHIFT  0
+#define BTCGB1_Vdroop_Enable_SHIFT  1
+#define AVFSGB0_Vdroop_Enable_SHIFT 2
+#define AVFSGB1_Vdroop_Enable_SHIFT 3
+
+#pragma pack(pop)
+
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h
index c3ed737ab951..715b5a168831 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h
@@ -131,6 +131,7 @@ typedef uint16_t PPSMC_Result;
 #define PPSMC_MSG_RunAcgInOpenLoop               0x5E
 #define PPSMC_MSG_InitializeAcg                  0x5F
 #define PPSMC_MSG_GetCurrPkgPwr                  0x61
+#define PPSMC_MSG_GetAverageGfxclkActualFrequency 0x63
 #define PPSMC_MSG_SetPccThrottleLevel            0x67
 #define PPSMC_MSG_UpdatePkgPwrPidAlpha           0x68
 #define PPSMC_Message_Count                      0x69
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/Makefile b/drivers/gpu/drm/amd/powerplay/smumgr/Makefile
index 958755075421..0a200406a1ec 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/Makefile
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/Makefile
@@ -26,7 +26,7 @@
 SMU_MGR = smumgr.o smu8_smumgr.o tonga_smumgr.o fiji_smumgr.o \
 	  polaris10_smumgr.o iceland_smumgr.o \
 	  smu7_smumgr.o vega10_smumgr.o smu10_smumgr.o ci_smumgr.o \
-	  vega12_smumgr.o
+	  vega12_smumgr.o vegam_smumgr.o
 
 AMD_PP_SMUMGR = $(addprefix $(AMD_PP_PATH)/smumgr/,$(SMU_MGR))
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c
index 08d000140eca..2d4ec8ac3a08 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c
@@ -61,9 +61,6 @@
 
 #define SMC_RAM_END 0x40000
 
-#define VOLTAGE_SCALE               4
-#define VOLTAGE_VID_OFFSET_SCALE1    625
-#define VOLTAGE_VID_OFFSET_SCALE2    100
 #define CISLAND_MINIMUM_ENGINE_CLOCK 800
 #define CISLAND_MAX_DEEPSLEEP_DIVIDER_ID 5
 
@@ -211,9 +208,7 @@ static int ci_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
 {
 	int ret;
 
-	if (!ci_is_smc_ram_running(hwmgr))
-		return -EINVAL;
-
+	cgs_write_register(hwmgr->device, mmSMC_RESP_0, 0);
 	cgs_write_register(hwmgr->device, mmSMC_MESSAGE_0, msg);
 
 	PHM_WAIT_FIELD_UNEQUAL(hwmgr, SMC_RESP_0, SMC_RESP, 0);
@@ -1182,7 +1177,6 @@ static int ci_populate_single_memory_level(
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	int result = 0;
 	bool dll_state_on;
-	struct cgs_display_info info = {0};
 	uint32_t mclk_edc_wr_enable_threshold = 40000;
 	uint32_t mclk_edc_enable_threshold = 40000;
 	uint32_t mclk_strobe_mode_threshold = 40000;
@@ -1236,8 +1230,7 @@ static int ci_populate_single_memory_level(
 	/* default set to low watermark. Highest level will be set to high later.*/
 	memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
 
-	cgs_get_active_displays_info(hwmgr->device, &info);
-	data->display_timing.num_existing_displays = info.display_count;
+	data->display_timing.num_existing_displays = hwmgr->display_config->num_display;
 
 	/* stutter mode not support on ci */
 
@@ -2784,7 +2777,6 @@ static int ci_smu_fini(struct pp_hwmgr *hwmgr)
 {
 	kfree(hwmgr->smu_backend);
 	hwmgr->smu_backend = NULL;
-	cgs_rel_firmware(hwmgr->device, CGS_UCODE_ID_SMU);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
index faef78321446..53df9405f43a 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
@@ -53,10 +53,7 @@
 
 #define FIJI_SMC_SIZE 0x20000
 
-#define VOLTAGE_SCALE 4
 #define POWERTUNE_DEFAULT_SET_MAX    1
-#define VOLTAGE_VID_OFFSET_SCALE1   625
-#define VOLTAGE_VID_OFFSET_SCALE2   100
 #define VDDC_VDDCI_DELTA            300
 #define MC_CG_ARB_FREQ_F1           0x0b
 
@@ -288,8 +285,7 @@ static int fiji_start_smu(struct pp_hwmgr *hwmgr)
 	struct fiji_smumgr *priv = (struct fiji_smumgr *)(hwmgr->smu_backend);
 
 	/* Only start SMC if SMC RAM is not running */
-	if (!(smu7_is_smc_ram_running(hwmgr)
-		|| cgs_is_virtualization_enabled(hwmgr->device))) {
+	if (!smu7_is_smc_ram_running(hwmgr) && hwmgr->not_vf) {
 		/* Check if SMU is running in protected mode */
 		if (0 == PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device,
 				CGS_IND_REG__SMC,
@@ -307,13 +303,13 @@ static int fiji_start_smu(struct pp_hwmgr *hwmgr)
 	}
 
 	/* To initialize all clock gating before RLC loaded and running.*/
-	cgs_set_clockgating_state(hwmgr->device,
+	amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 			AMD_IP_BLOCK_TYPE_GFX, AMD_CG_STATE_GATE);
-	cgs_set_clockgating_state(hwmgr->device,
+	amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 			AMD_IP_BLOCK_TYPE_GMC, AMD_CG_STATE_GATE);
-	cgs_set_clockgating_state(hwmgr->device,
+	amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 			AMD_IP_BLOCK_TYPE_SDMA, AMD_CG_STATE_GATE);
-	cgs_set_clockgating_state(hwmgr->device,
+	amdgpu_device_ip_set_clockgating_state(hwmgr->adev,
 			AMD_IP_BLOCK_TYPE_COMMON, AMD_CG_STATE_GATE);
 
 	/* Setup SoftRegsStart here for register lookup in case
@@ -335,10 +331,10 @@ static bool fiji_is_hw_avfs_present(struct pp_hwmgr *hwmgr)
 	uint32_t efuse = 0;
 	uint32_t mask = (1 << ((AVFS_EN_MSB - AVFS_EN_LSB) + 1)) - 1;
 
-	if (cgs_is_virtualization_enabled(hwmgr->device))
-		return 0;
+	if (!hwmgr->not_vf)
+		return false;
 
-	if (!atomctrl_read_efuse(hwmgr->device, AVFS_EN_LSB, AVFS_EN_MSB,
+	if (!atomctrl_read_efuse(hwmgr, AVFS_EN_LSB, AVFS_EN_MSB,
 			mask, &efuse)) {
 		if (efuse)
 			return true;
@@ -989,11 +985,11 @@ static int fiji_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
 
 	threshold = clock * data->fast_watermark_threshold / 100;
 
-	data->display_timing.min_clock_in_sr = hwmgr->display_config.min_core_set_clock_in_sr;
+	data->display_timing.min_clock_in_sr = hwmgr->display_config->min_core_set_clock_in_sr;
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep))
 		level->DeepSleepDivId = smu7_get_sleep_divider_id_from_clock(clock,
-								hwmgr->display_config.min_core_set_clock_in_sr);
+								hwmgr->display_config->min_core_set_clock_in_sr);
 
 
 	/* Default to slow, highest DPM level will be
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c
index d4bb934e7334..415f691c3fa9 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c
@@ -60,10 +60,7 @@
 
 #define ICELAND_SMC_SIZE               0x20000
 
-#define VOLTAGE_SCALE 4
 #define POWERTUNE_DEFAULT_SET_MAX    1
-#define VOLTAGE_VID_OFFSET_SCALE1   625
-#define VOLTAGE_VID_OFFSET_SCALE2   100
 #define MC_CG_ARB_FREQ_F1           0x0b
 #define VDDC_VDDCI_DELTA            200
 
@@ -932,7 +929,7 @@ static int iceland_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
 	graphic_level->PowerThrottle = 0;
 
 	data->display_timing.min_clock_in_sr =
-			hwmgr->display_config.min_core_set_clock_in_sr;
+			hwmgr->display_config->min_core_set_clock_in_sr;
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SclkDeepSleep))
@@ -1236,7 +1233,6 @@ static int iceland_populate_single_memory_level(
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	int result = 0;
 	bool dll_state_on;
-	struct cgs_display_info info = {0};
 	uint32_t mclk_edc_wr_enable_threshold = 40000;
 	uint32_t mclk_edc_enable_threshold = 40000;
 	uint32_t mclk_strobe_mode_threshold = 40000;
@@ -1283,8 +1279,7 @@ static int iceland_populate_single_memory_level(
 	/* default set to low watermark. Highest level will be set to high later.*/
 	memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
 
-	cgs_get_active_displays_info(hwmgr->device, &info);
-	data->display_timing.num_existing_displays = info.display_count;
+	data->display_timing.num_existing_displays = hwmgr->display_config->num_display;
 
 	/* stutter mode not support on iceland */
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
index 997a777dd35b..a8c6524f07e4 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
@@ -52,8 +52,6 @@
 #include "dce/dce_10_0_sh_mask.h"
 
 #define POLARIS10_SMC_SIZE 0x20000
-#define VOLTAGE_VID_OFFSET_SCALE1   625
-#define VOLTAGE_VID_OFFSET_SCALE2   100
 #define POWERTUNE_DEFAULT_SET_MAX    1
 #define VDDC_VDDCI_DELTA            200
 #define MC_CG_ARB_FREQ_F1           0x0b
@@ -295,25 +293,16 @@ static int polaris10_start_smu(struct pp_hwmgr *hwmgr)
 	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smu_backend);
 
 	/* Only start SMC if SMC RAM is not running */
-	if (!(smu7_is_smc_ram_running(hwmgr)
-		|| cgs_is_virtualization_enabled(hwmgr->device))) {
+	if (!smu7_is_smc_ram_running(hwmgr) && hwmgr->not_vf) {
 		smu_data->protected_mode = (uint8_t) (PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SMU_FIRMWARE, SMU_MODE));
 		smu_data->smu7_data.security_hard_key = (uint8_t) (PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SMU_FIRMWARE, SMU_SEL));
 
 		/* Check if SMU is running in protected mode */
-		if (smu_data->protected_mode == 0) {
+		if (smu_data->protected_mode == 0)
 			result = polaris10_start_smu_in_non_protection_mode(hwmgr);
-		} else {
+		else
 			result = polaris10_start_smu_in_protection_mode(hwmgr);
 
-			/* If failed, try with different security Key. */
-			if (result != 0) {
-				smu_data->smu7_data.security_hard_key ^= 1;
-				cgs_rel_firmware(hwmgr->device, CGS_UCODE_ID_SMU);
-				result = polaris10_start_smu_in_protection_mode(hwmgr);
-			}
-		}
-
 		if (result != 0)
 			PP_ASSERT_WITH_CODE(0, "Failed to load SMU ucode.", return result);
 
@@ -951,11 +940,11 @@ static int polaris10_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
 	level->DownHyst = data->current_profile_setting.sclk_down_hyst;
 	level->VoltageDownHyst = 0;
 	level->PowerThrottle = 0;
-	data->display_timing.min_clock_in_sr = hwmgr->display_config.min_core_set_clock_in_sr;
+	data->display_timing.min_clock_in_sr = hwmgr->display_config->min_core_set_clock_in_sr;
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep))
 		level->DeepSleepDivId = smu7_get_sleep_divider_id_from_clock(clock,
-								hwmgr->display_config.min_core_set_clock_in_sr);
+								hwmgr->display_config->min_core_set_clock_in_sr);
 
 	/* Default to slow, highest DPM level will be
 	 * set to PPSMC_DISPLAY_WATERMARK_LOW later.
@@ -1085,11 +1074,9 @@ static int polaris10_populate_single_memory_level(struct pp_hwmgr *hwmgr,
 	struct phm_ppt_v1_information *table_info =
 			(struct phm_ppt_v1_information *)(hwmgr->pptable);
 	int result = 0;
-	struct cgs_display_info info = {0, 0, NULL};
 	uint32_t mclk_stutter_mode_threshold = 40000;
 	phm_ppt_v1_clock_voltage_dependency_table *vdd_dep_table = NULL;
 
-	cgs_get_active_displays_info(hwmgr->device, &info);
 
 	if (hwmgr->od_enabled)
 		vdd_dep_table = (phm_ppt_v1_clock_voltage_dependency_table *)&data->odn_dpm_table.vdd_dependency_on_mclk;
@@ -1115,7 +1102,7 @@ static int polaris10_populate_single_memory_level(struct pp_hwmgr *hwmgr,
 	mem_level->StutterEnable = false;
 	mem_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
 
-	data->display_timing.num_existing_displays = info.display_count;
+	data->display_timing.num_existing_displays = hwmgr->display_config->num_display;
 
 	if (mclk_stutter_mode_threshold &&
 		(clock <= mclk_stutter_mode_threshold) &&
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c
index bc53f2beda30..0a563f6fe9ea 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c
@@ -23,7 +23,7 @@
 
 #include "smumgr.h"
 #include "smu10_inc.h"
-#include "pp_soc15.h"
+#include "soc15_common.h"
 #include "smu10_smumgr.h"
 #include "ppatomctrl.h"
 #include "rv_ppsmc.h"
@@ -33,8 +33,6 @@
 #include "pp_debug.h"
 
 
-#define VOLTAGE_SCALE 4
-
 #define BUFFER_SIZE                 80000
 #define MAX_STRING_SIZE             15
 #define BUFFER_SIZETWO              131072
@@ -49,48 +47,41 @@
 
 static uint32_t smu10_wait_for_response(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	uint32_t reg;
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_90_BASE_IDX, mmMP1_SMN_C2PMSG_90);
+	reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_90);
 
 	phm_wait_for_register_unequal(hwmgr, reg,
 			0, MP1_C2PMSG_90__CONTENT_MASK);
 
-	return cgs_read_register(hwmgr->device, reg);
+	return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90);
 }
 
 static int smu10_send_msg_to_smc_without_waiting(struct pp_hwmgr *hwmgr,
 		uint16_t msg)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_66_BASE_IDX, mmMP1_SMN_C2PMSG_66);
-	cgs_write_register(hwmgr->device, reg, msg);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_66, msg);
 
 	return 0;
 }
 
 static int smu10_read_arg_from_smc(struct pp_hwmgr *hwmgr)
 {
-	uint32_t reg;
-
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_82_BASE_IDX, mmMP1_SMN_C2PMSG_82);
+	struct amdgpu_device *adev = hwmgr->adev;
 
-	return cgs_read_register(hwmgr->device, reg);
+	return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82);
 }
 
 static int smu10_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 
 	smu10_wait_for_response(hwmgr);
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_90_BASE_IDX, mmMP1_SMN_C2PMSG_90);
-	cgs_write_register(hwmgr->device, reg, 0);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
 
 	smu10_send_msg_to_smc_without_waiting(hwmgr, msg);
 
@@ -104,17 +95,13 @@ static int smu10_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
 static int smu10_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr,
 		uint16_t msg, uint32_t parameter)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 
 	smu10_wait_for_response(hwmgr);
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_90_BASE_IDX, mmMP1_SMN_C2PMSG_90);
-	cgs_write_register(hwmgr->device, reg, 0);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_82_BASE_IDX, mmMP1_SMN_C2PMSG_82);
-	cgs_write_register(hwmgr->device, reg, parameter);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82, parameter);
 
 	smu10_send_msg_to_smc_without_waiting(hwmgr, msg);
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
index 0399c10d2be0..d644a9bb9078 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
@@ -167,24 +167,25 @@ int smu7_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
 {
 	int ret;
 
-	if (!smu7_is_smc_ram_running(hwmgr))
-		return -EINVAL;
-
-
 	PHM_WAIT_FIELD_UNEQUAL(hwmgr, SMC_RESP_0, SMC_RESP, 0);
 
 	ret = PHM_READ_FIELD(hwmgr->device, SMC_RESP_0, SMC_RESP);
 
-	if (ret != 1)
-		pr_info("\n failed to send pre message %x ret is %d \n",  msg, ret);
+	if (ret == 0xFE)
+		pr_debug("last message was not supported\n");
+	else if (ret != 1)
+		pr_info("\n last message was failed ret is %d\n", ret);
 
+	cgs_write_register(hwmgr->device, mmSMC_RESP_0, 0);
 	cgs_write_register(hwmgr->device, mmSMC_MESSAGE_0, msg);
 
 	PHM_WAIT_FIELD_UNEQUAL(hwmgr, SMC_RESP_0, SMC_RESP, 0);
 
 	ret = PHM_READ_FIELD(hwmgr->device, SMC_RESP_0, SMC_RESP);
 
-	if (ret != 1)
+	if (ret == 0xFE)
+		pr_debug("message %x was not supported\n", msg);
+	else if (ret != 1)
 		pr_info("\n failed to send message %x ret is %d \n",  msg, ret);
 
 	return 0;
@@ -199,10 +200,6 @@ int smu7_send_msg_to_smc_without_waiting(struct pp_hwmgr *hwmgr, uint16_t msg)
 
 int smu7_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr, uint16_t msg, uint32_t parameter)
 {
-	if (!smu7_is_smc_ram_running(hwmgr)) {
-		return -EINVAL;
-	}
-
 	PHM_WAIT_FIELD_UNEQUAL(hwmgr, SMC_RESP_0, SMC_RESP, 0);
 
 	cgs_write_register(hwmgr->device, mmSMC_MSG_ARG_0, parameter);
@@ -231,16 +228,6 @@ int smu7_send_msg_to_smc_offset(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_wait_for_smc_inactive(struct pp_hwmgr *hwmgr)
-{
-	if (!smu7_is_smc_ram_running(hwmgr))
-		return -EINVAL;
-
-	PHM_WAIT_VFPF_INDIRECT_FIELD(hwmgr, SMC_IND, SMC_SYSCON_CLOCK_CNTL_0, cken, 0);
-	return 0;
-}
-
-
 enum cgs_ucode_id smu7_convert_fw_type_to_cgs(uint32_t fw_type)
 {
 	enum cgs_ucode_id result = CGS_UCODE_ID_MAXIMUM;
@@ -296,11 +283,9 @@ int smu7_read_smc_sram_dword(struct pp_hwmgr *hwmgr, uint32_t smc_addr, uint32_t
 
 	result = smu7_set_smc_sram_address(hwmgr, smc_addr, limit);
 
-	if (result)
-		return result;
+	*value = result ? 0 : cgs_read_register(hwmgr->device, mmSMC_IND_DATA_11);
 
-	*value = cgs_read_register(hwmgr->device, mmSMC_IND_DATA_11);
-	return 0;
+	return result;
 }
 
 int smu7_write_smc_sram_dword(struct pp_hwmgr *hwmgr, uint32_t smc_addr, uint32_t value, uint32_t limit)
@@ -375,7 +360,7 @@ static int smu7_populate_single_firmware_entry(struct pp_hwmgr *hwmgr,
 		entry->meta_data_addr_low = 0;
 
 		/* digest need be excluded out */
-		if (cgs_is_virtualization_enabled(hwmgr->device))
+		if (!hwmgr->not_vf)
 			info.image_size -= 20;
 		entry->data_size_byte = info.image_size;
 		entry->num_register_entries = 0;
@@ -409,7 +394,7 @@ int smu7_request_smu_load_fw(struct pp_hwmgr *hwmgr)
 					0x0);
 
 	if (hwmgr->chip_id > CHIP_TOPAZ) { /* add support for Topaz */
-		if (!cgs_is_virtualization_enabled(hwmgr->device)) {
+		if (hwmgr->not_vf) {
 			smu7_send_msg_to_smc_with_parameter(hwmgr,
 						PPSMC_MSG_SMU_DRAM_ADDR_HI,
 						upper_32_bits(smu_data->smu_buffer.mc_addr));
@@ -467,7 +452,7 @@ int smu7_request_smu_load_fw(struct pp_hwmgr *hwmgr)
 	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
 				UCODE_ID_SDMA1, &toc->entry[toc->num_entries++]),
 				"Failed to Get Firmware Entry.", return -EINVAL);
-	if (cgs_is_virtualization_enabled(hwmgr->device))
+	if (!hwmgr->not_vf)
 		PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
 				UCODE_ID_MEC_STORAGE, &toc->entry[toc->num_entries++]),
 				"Failed to Get Firmware Entry.", return -EINVAL);
@@ -608,7 +593,7 @@ int smu7_init(struct pp_hwmgr *hwmgr)
 	smu_data->header = smu_data->header_buffer.kaddr;
 	smu_data->header_buffer.mc_addr = mc_addr;
 
-	if (cgs_is_virtualization_enabled(hwmgr->device))
+	if (!hwmgr->not_vf)
 		return 0;
 
 	smu_data->smu_buffer.data_size = 200*4096;
@@ -643,13 +628,12 @@ int smu7_smu_fini(struct pp_hwmgr *hwmgr)
 					&smu_data->header_buffer.mc_addr,
 					&smu_data->header_buffer.kaddr);
 
-	if (!cgs_is_virtualization_enabled(hwmgr->device))
+	if (hwmgr->not_vf)
 		amdgpu_bo_free_kernel(&smu_data->smu_buffer.handle,
 					&smu_data->smu_buffer.mc_addr,
 					&smu_data->smu_buffer.kaddr);
 
 	kfree(hwmgr->smu_backend);
 	hwmgr->smu_backend = NULL;
-	cgs_rel_firmware(hwmgr->device, CGS_UCODE_ID_SMU);
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
index 126d300259ba..39c9bfda0ab4 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
@@ -67,7 +67,6 @@ int smu7_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr, uint16_t msg,
 int smu7_send_msg_to_smc_with_parameter_without_waiting(struct pp_hwmgr *hwmgr,
 						uint16_t msg, uint32_t parameter);
 int smu7_send_msg_to_smc_offset(struct pp_hwmgr *hwmgr);
-int smu7_wait_for_smc_inactive(struct pp_hwmgr *hwmgr);
 
 enum cgs_ucode_id smu7_convert_fw_type_to_cgs(uint32_t fw_type);
 int smu7_read_smc_sram_dword(struct pp_hwmgr *hwmgr, uint32_t smc_addr,
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
index c28b60aae5f8..c9837935f0f5 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
@@ -41,9 +41,11 @@ MODULE_FIRMWARE("amdgpu/polaris11_smc.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_smc_sk.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_k_smc.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_smc.bin");
+MODULE_FIRMWARE("amdgpu/vegam_smc.bin");
 MODULE_FIRMWARE("amdgpu/vega10_smc.bin");
 MODULE_FIRMWARE("amdgpu/vega10_acg_smc.bin");
 MODULE_FIRMWARE("amdgpu/vega12_smc.bin");
+MODULE_FIRMWARE("amdgpu/vega20_smc.bin");
 
 int smum_thermal_avfs_enable(struct pp_hwmgr *hwmgr)
 {
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
index b51d7468c3e7..782b19fc2e70 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
@@ -55,11 +55,7 @@
 #include "dce/dce_10_0_d.h"
 #include "dce/dce_10_0_sh_mask.h"
 
-
-#define VOLTAGE_SCALE 4
 #define POWERTUNE_DEFAULT_SET_MAX    1
-#define VOLTAGE_VID_OFFSET_SCALE1   625
-#define VOLTAGE_VID_OFFSET_SCALE2   100
 #define MC_CG_ARB_FREQ_F1           0x0b
 #define VDDC_VDDCI_DELTA            200
 
@@ -199,8 +195,7 @@ static int tonga_start_smu(struct pp_hwmgr *hwmgr)
 	int result;
 
 	/* Only start SMC if SMC RAM is not running */
-	if (!(smu7_is_smc_ram_running(hwmgr) ||
-		cgs_is_virtualization_enabled(hwmgr->device))) {
+	if (!smu7_is_smc_ram_running(hwmgr) && hwmgr->not_vf) {
 		/*Check if SMU is running in protected mode*/
 		if (0 == PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
 					SMU_FIRMWARE, SMU_MODE)) {
@@ -651,7 +646,7 @@ static int tonga_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
 	graphic_level->PowerThrottle = 0;
 
 	data->display_timing.min_clock_in_sr =
-			hwmgr->display_config.min_core_set_clock_in_sr;
+			hwmgr->display_config->min_core_set_clock_in_sr;
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SclkDeepSleep))
@@ -957,18 +952,17 @@ static int tonga_populate_single_memory_level(
 		SMU72_Discrete_MemoryLevel *memory_level
 		)
 {
-	uint32_t mvdd = 0;
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	struct phm_ppt_v1_information *pptable_info =
 			  (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	int result = 0;
-	bool dll_state_on;
-	struct cgs_display_info info = {0};
 	uint32_t mclk_edc_wr_enable_threshold = 40000;
 	uint32_t mclk_stutter_mode_threshold = 30000;
 	uint32_t mclk_edc_enable_threshold = 40000;
 	uint32_t mclk_strobe_mode_threshold = 40000;
 	phm_ppt_v1_clock_voltage_dependency_table *vdd_dep_table = NULL;
+	int result = 0;
+	bool dll_state_on;
+	uint32_t mvdd = 0;
 
 	if (hwmgr->od_enabled)
 		vdd_dep_table = (phm_ppt_v1_clock_voltage_dependency_table *)&data->odn_dpm_table.vdd_dependency_on_mclk;
@@ -1009,8 +1003,7 @@ static int tonga_populate_single_memory_level(
 	/* default set to low watermark. Highest level will be set to high later.*/
 	memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
 
-	cgs_get_active_displays_info(hwmgr->device, &info);
-	data->display_timing.num_existing_displays = info.display_count;
+	data->display_timing.num_existing_displays = hwmgr->display_config->num_display;
 
 	if ((mclk_stutter_mode_threshold != 0) &&
 	    (memory_clock <= mclk_stutter_mode_threshold) &&
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
index 4aafb043bcb0..e84669c448a3 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
@@ -23,7 +23,7 @@
 
 #include "smumgr.h"
 #include "vega10_inc.h"
-#include "pp_soc15.h"
+#include "soc15_common.h"
 #include "vega10_smumgr.h"
 #include "vega10_hwmgr.h"
 #include "vega10_ppsmc.h"
@@ -35,8 +35,6 @@
 #define AVFS_EN_MSB		1568
 #define AVFS_EN_LSB		1568
 
-#define VOLTAGE_SCALE	4
-
 /* Microcode file is stored in this buffer */
 #define BUFFER_SIZE                 80000
 #define MAX_STRING_SIZE             15
@@ -54,18 +52,13 @@
 
 static bool vega10_is_smc_ram_running(struct pp_hwmgr *hwmgr)
 {
-	uint32_t mp1_fw_flags, reg;
-
-	reg = soc15_get_register_offset(NBIF_HWID, 0,
-			mmPCIE_INDEX2_BASE_IDX, mmPCIE_INDEX2);
+	struct amdgpu_device *adev = hwmgr->adev;
+	uint32_t mp1_fw_flags;
 
-	cgs_write_register(hwmgr->device, reg,
+	WREG32_SOC15(NBIF, 0, mmPCIE_INDEX2,
 			(MP1_Public | (smnMP1_FIRMWARE_FLAGS & 0xffffffff)));
 
-	reg = soc15_get_register_offset(NBIF_HWID, 0,
-			mmPCIE_DATA2_BASE_IDX, mmPCIE_DATA2);
-
-	mp1_fw_flags = cgs_read_register(hwmgr->device, reg);
+	mp1_fw_flags = RREG32_SOC15(NBIF, 0, mmPCIE_DATA2);
 
 	if (mp1_fw_flags & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK)
 		return true;
@@ -81,11 +74,11 @@ static bool vega10_is_smc_ram_running(struct pp_hwmgr *hwmgr)
  */
 static uint32_t vega10_wait_for_response(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	uint32_t reg;
 	uint32_t ret;
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_90_BASE_IDX, mmMP1_SMN_C2PMSG_90);
+	reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_90);
 
 	ret = phm_wait_for_register_unequal(hwmgr, reg,
 			0, MP1_C2PMSG_90__CONTENT_MASK);
@@ -93,7 +86,7 @@ static uint32_t vega10_wait_for_response(struct pp_hwmgr *hwmgr)
 	if (ret)
 		pr_err("No response from smu\n");
 
-	return cgs_read_register(hwmgr->device, reg);
+	return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90);
 }
 
 /*
@@ -105,11 +98,9 @@ static uint32_t vega10_wait_for_response(struct pp_hwmgr *hwmgr)
 static int vega10_send_msg_to_smc_without_waiting(struct pp_hwmgr *hwmgr,
 		uint16_t msg)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_66_BASE_IDX, mmMP1_SMN_C2PMSG_66);
-	cgs_write_register(hwmgr->device, reg, msg);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_66, msg);
 
 	return 0;
 }
@@ -122,14 +113,12 @@ static int vega10_send_msg_to_smc_without_waiting(struct pp_hwmgr *hwmgr,
  */
 static int vega10_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 	uint32_t ret;
 
 	vega10_wait_for_response(hwmgr);
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_90_BASE_IDX, mmMP1_SMN_C2PMSG_90);
-	cgs_write_register(hwmgr->device, reg, 0);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
 
 	vega10_send_msg_to_smc_without_waiting(hwmgr, msg);
 
@@ -150,18 +139,14 @@ static int vega10_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
 static int vega10_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr,
 		uint16_t msg, uint32_t parameter)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 	uint32_t ret;
 
 	vega10_wait_for_response(hwmgr);
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_90_BASE_IDX, mmMP1_SMN_C2PMSG_90);
-	cgs_write_register(hwmgr->device, reg, 0);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_82_BASE_IDX, mmMP1_SMN_C2PMSG_82);
-	cgs_write_register(hwmgr->device, reg, parameter);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82, parameter);
 
 	vega10_send_msg_to_smc_without_waiting(hwmgr, msg);
 
@@ -174,12 +159,9 @@ static int vega10_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr,
 
 static int vega10_get_argument(struct pp_hwmgr *hwmgr)
 {
-	uint32_t reg;
-
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_82_BASE_IDX, mmMP1_SMN_C2PMSG_82);
+	struct amdgpu_device *adev = hwmgr->adev;
 
-	return cgs_read_register(hwmgr->device, reg);
+	return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82);
 }
 
 static int vega10_copy_table_from_smc(struct pp_hwmgr *hwmgr,
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c
index 651a3f28734b..7d9b40e8b1bf 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c
@@ -23,7 +23,7 @@
 
 #include "smumgr.h"
 #include "vega12_inc.h"
-#include "pp_soc15.h"
+#include "soc15_common.h"
 #include "vega12_smumgr.h"
 #include "vega12_ppsmc.h"
 #include "vega12/smu9_driver_if.h"
@@ -44,18 +44,13 @@
 
 static bool vega12_is_smc_ram_running(struct pp_hwmgr *hwmgr)
 {
-	uint32_t mp1_fw_flags, reg;
+	struct amdgpu_device *adev = hwmgr->adev;
+	uint32_t mp1_fw_flags;
 
-	reg = soc15_get_register_offset(NBIF_HWID, 0,
-			mmPCIE_INDEX2_BASE_IDX, mmPCIE_INDEX2);
-
-	cgs_write_register(hwmgr->device, reg,
+	WREG32_SOC15(NBIF, 0, mmPCIE_INDEX2,
 			(MP1_Public | (smnMP1_FIRMWARE_FLAGS & 0xffffffff)));
 
-	reg = soc15_get_register_offset(NBIF_HWID, 0,
-			mmPCIE_DATA2_BASE_IDX, mmPCIE_DATA2);
-
-	mp1_fw_flags = cgs_read_register(hwmgr->device, reg);
+	mp1_fw_flags = RREG32_SOC15(NBIF, 0, mmPCIE_DATA2);
 
 	if ((mp1_fw_flags & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) >>
 				MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT)
@@ -72,15 +67,15 @@ static bool vega12_is_smc_ram_running(struct pp_hwmgr *hwmgr)
  */
 static uint32_t vega12_wait_for_response(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = hwmgr->adev;
 	uint32_t reg;
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_90_BASE_IDX, mmMP1_SMN_C2PMSG_90);
+	reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_90);
 
 	phm_wait_for_register_unequal(hwmgr, reg,
 			0, MP1_C2PMSG_90__CONTENT_MASK);
 
-	return cgs_read_register(hwmgr->device, reg);
+	return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90);
 }
 
 /*
@@ -92,11 +87,9 @@ static uint32_t vega12_wait_for_response(struct pp_hwmgr *hwmgr)
 int vega12_send_msg_to_smc_without_waiting(struct pp_hwmgr *hwmgr,
 		uint16_t msg)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_66_BASE_IDX, mmMP1_SMN_C2PMSG_66);
-	cgs_write_register(hwmgr->device, reg, msg);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_66, msg);
 
 	return 0;
 }
@@ -109,13 +102,11 @@ int vega12_send_msg_to_smc_without_waiting(struct pp_hwmgr *hwmgr,
  */
 int vega12_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 
 	vega12_wait_for_response(hwmgr);
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_90_BASE_IDX, mmMP1_SMN_C2PMSG_90);
-	cgs_write_register(hwmgr->device, reg, 0);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
 
 	vega12_send_msg_to_smc_without_waiting(hwmgr, msg);
 
@@ -135,17 +126,13 @@ int vega12_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
 int vega12_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr,
 		uint16_t msg, uint32_t parameter)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 
 	vega12_wait_for_response(hwmgr);
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_90_BASE_IDX, mmMP1_SMN_C2PMSG_90);
-	cgs_write_register(hwmgr->device, reg, 0);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_82_BASE_IDX, mmMP1_SMN_C2PMSG_82);
-	cgs_write_register(hwmgr->device, reg, parameter);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82, parameter);
 
 	vega12_send_msg_to_smc_without_waiting(hwmgr, msg);
 
@@ -166,11 +153,9 @@ int vega12_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr,
 int vega12_send_msg_to_smc_with_parameter_without_waiting(
 		struct pp_hwmgr *hwmgr, uint16_t msg, uint32_t parameter)
 {
-	uint32_t reg;
+	struct amdgpu_device *adev = hwmgr->adev;
 
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_66_BASE_IDX, mmMP1_SMN_C2PMSG_66);
-	cgs_write_register(hwmgr->device, reg, parameter);
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_66, parameter);
 
 	return vega12_send_msg_to_smc_without_waiting(hwmgr, msg);
 }
@@ -183,12 +168,9 @@ int vega12_send_msg_to_smc_with_parameter_without_waiting(
  */
 int vega12_read_arg_from_smc(struct pp_hwmgr *hwmgr, uint32_t *arg)
 {
-	uint32_t reg;
-
-	reg = soc15_get_register_offset(MP1_HWID, 0,
-			mmMP1_SMN_C2PMSG_82_BASE_IDX, mmMP1_SMN_C2PMSG_82);
+	struct amdgpu_device *adev = hwmgr->adev;
 
-	*arg = cgs_read_register(hwmgr->device, reg);
+	*arg = RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.c
new file mode 100644
index 000000000000..2de48959ac93
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.c
@@ -0,0 +1,2383 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "pp_debug.h"
+#include "smumgr.h"
+#include "smu_ucode_xfer_vi.h"
+#include "vegam_smumgr.h"
+#include "smu/smu_7_1_3_d.h"
+#include "smu/smu_7_1_3_sh_mask.h"
+#include "gmc/gmc_8_1_d.h"
+#include "gmc/gmc_8_1_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "gca/gfx_8_0_d.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "ppatomctrl.h"
+#include "cgs_common.h"
+#include "smu7_ppsmc.h"
+
+#include "smu7_dyn_defaults.h"
+
+#include "smu7_hwmgr.h"
+#include "hardwaremanager.h"
+#include "ppatomctrl.h"
+#include "atombios.h"
+#include "pppcielanes.h"
+
+#include "dce/dce_11_2_d.h"
+#include "dce/dce_11_2_sh_mask.h"
+
+#define PPVEGAM_TARGETACTIVITY_DFLT                     50
+
+#define VOLTAGE_VID_OFFSET_SCALE1   625
+#define VOLTAGE_VID_OFFSET_SCALE2   100
+#define POWERTUNE_DEFAULT_SET_MAX    1
+#define VDDC_VDDCI_DELTA            200
+#define MC_CG_ARB_FREQ_F1           0x0b
+
+#define STRAP_ASIC_RO_LSB    2168
+#define STRAP_ASIC_RO_MSB    2175
+
+#define PPSMC_MSG_ApplyAvfsCksOffVoltage      ((uint16_t) 0x415)
+#define PPSMC_MSG_EnableModeSwitchRLCNotification  ((uint16_t) 0x305)
+
+static const struct vegam_pt_defaults
+vegam_power_tune_data_set_array[POWERTUNE_DEFAULT_SET_MAX] = {
+	/* sviLoadLIneEn, SviLoadLineVddC, TDC_VDDC_ThrottleReleaseLimitPerc, TDC_MAWt,
+	 * TdcWaterfallCtl, DTEAmbientTempBase, DisplayCac, BAPM_TEMP_GRADIENT */
+	{ 1, 0xF, 0xFD, 0x19, 5, 45, 0, 0xB0000,
+	{ 0x79, 0x253, 0x25D, 0xAE, 0x72, 0x80, 0x83, 0x86, 0x6F, 0xC8, 0xC9, 0xC9, 0x2F, 0x4D, 0x61},
+	{ 0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203, 0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4 } },
+};
+
+static const sclkFcwRange_t Range_Table[NUM_SCLK_RANGE] = {
+			{VCO_2_4, POSTDIV_DIV_BY_16,  75, 160, 112},
+			{VCO_3_6, POSTDIV_DIV_BY_16, 112, 224, 160},
+			{VCO_2_4, POSTDIV_DIV_BY_8,   75, 160, 112},
+			{VCO_3_6, POSTDIV_DIV_BY_8,  112, 224, 160},
+			{VCO_2_4, POSTDIV_DIV_BY_4,   75, 160, 112},
+			{VCO_3_6, POSTDIV_DIV_BY_4,  112, 216, 160},
+			{VCO_2_4, POSTDIV_DIV_BY_2,   75, 160, 108},
+			{VCO_3_6, POSTDIV_DIV_BY_2,  112, 216, 160} };
+
+static int vegam_smu_init(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data;
+
+	smu_data = kzalloc(sizeof(struct vegam_smumgr), GFP_KERNEL);
+	if (smu_data == NULL)
+		return -ENOMEM;
+
+	hwmgr->smu_backend = smu_data;
+
+	if (smu7_init(hwmgr)) {
+		kfree(smu_data);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int vegam_start_smu_in_protection_mode(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+
+	/* Wait for smc boot up */
+	/* PHM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL(smumgr, SMC_IND, RCU_UC_EVENTS, boot_seq_done, 0) */
+
+	/* Assert reset */
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+					SMC_SYSCON_RESET_CNTL, rst_reg, 1);
+
+	result = smu7_upload_smu_firmware_image(hwmgr);
+	if (result != 0)
+		return result;
+
+	/* Clear status */
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixSMU_STATUS, 0);
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+					SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
+
+	/* De-assert reset */
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+					SMC_SYSCON_RESET_CNTL, rst_reg, 0);
+
+
+	PHM_WAIT_VFPF_INDIRECT_FIELD(hwmgr, SMC_IND, RCU_UC_EVENTS, INTERRUPTS_ENABLED, 1);
+
+
+	/* Call Test SMU message with 0x20000 offset to trigger SMU start */
+	smu7_send_msg_to_smc_offset(hwmgr);
+
+	/* Wait done bit to be set */
+	/* Check pass/failed indicator */
+
+	PHM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL(hwmgr, SMC_IND, SMU_STATUS, SMU_DONE, 0);
+
+	if (1 != PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+						SMU_STATUS, SMU_PASS))
+		PP_ASSERT_WITH_CODE(false, "SMU Firmware start failed!", return -1);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixFIRMWARE_FLAGS, 0);
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+					SMC_SYSCON_RESET_CNTL, rst_reg, 1);
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+					SMC_SYSCON_RESET_CNTL, rst_reg, 0);
+
+	/* Wait for firmware to initialize */
+	PHM_WAIT_VFPF_INDIRECT_FIELD(hwmgr, SMC_IND, FIRMWARE_FLAGS, INTERRUPTS_ENABLED, 1);
+
+	return result;
+}
+
+static int vegam_start_smu_in_non_protection_mode(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+
+	/* wait for smc boot up */
+	PHM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL(hwmgr, SMC_IND, RCU_UC_EVENTS, boot_seq_done, 0);
+
+	/* Clear firmware interrupt enable flag */
+	/* PHM_WRITE_VFPF_INDIRECT_FIELD(pSmuMgr, SMC_IND, SMC_SYSCON_MISC_CNTL, pre_fetcher_en, 1); */
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixFIRMWARE_FLAGS, 0);
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+					SMC_SYSCON_RESET_CNTL,
+					rst_reg, 1);
+
+	result = smu7_upload_smu_firmware_image(hwmgr);
+	if (result != 0)
+		return result;
+
+	/* Set smc instruct start point at 0x0 */
+	smu7_program_jump_on_start(hwmgr);
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+					SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
+
+	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+					SMC_SYSCON_RESET_CNTL, rst_reg, 0);
+
+	/* Wait for firmware to initialize */
+
+	PHM_WAIT_VFPF_INDIRECT_FIELD(hwmgr, SMC_IND,
+					FIRMWARE_FLAGS, INTERRUPTS_ENABLED, 1);
+
+	return result;
+}
+
+static int vegam_start_smu(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+
+	/* Only start SMC if SMC RAM is not running */
+	if (!smu7_is_smc_ram_running(hwmgr) && hwmgr->not_vf) {
+		smu_data->protected_mode = (uint8_t)(PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device,
+				CGS_IND_REG__SMC, SMU_FIRMWARE, SMU_MODE));
+		smu_data->smu7_data.security_hard_key = (uint8_t)(PHM_READ_VFPF_INDIRECT_FIELD(
+				hwmgr->device, CGS_IND_REG__SMC, SMU_FIRMWARE, SMU_SEL));
+
+		/* Check if SMU is running in protected mode */
+		if (smu_data->protected_mode == 0)
+			result = vegam_start_smu_in_non_protection_mode(hwmgr);
+		else
+			result = vegam_start_smu_in_protection_mode(hwmgr);
+
+		if (result != 0)
+			PP_ASSERT_WITH_CODE(0, "Failed to load SMU ucode.", return result);
+	}
+
+	/* Setup SoftRegsStart here for register lookup in case DummyBackEnd is used and ProcessFirmwareHeader is not executed */
+	smu7_read_smc_sram_dword(hwmgr,
+			SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU75_Firmware_Header, SoftRegisters),
+			&(smu_data->smu7_data.soft_regs_start),
+			0x40000);
+
+	result = smu7_request_smu_load_fw(hwmgr);
+
+	return result;
+}
+
+static int vegam_process_firmware_header(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t tmp;
+	int result;
+	bool error = false;
+
+	result = smu7_read_smc_sram_dword(hwmgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU75_Firmware_Header, DpmTable),
+			&tmp, SMC_RAM_END);
+
+	if (0 == result)
+		smu_data->smu7_data.dpm_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU75_Firmware_Header, SoftRegisters),
+			&tmp, SMC_RAM_END);
+
+	if (!result) {
+		data->soft_regs_start = tmp;
+		smu_data->smu7_data.soft_regs_start = tmp;
+	}
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU75_Firmware_Header, mcRegisterTable),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.mc_reg_table_start = tmp;
+
+	result = smu7_read_smc_sram_dword(hwmgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU75_Firmware_Header, FanTable),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.fan_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU75_Firmware_Header, mcArbDramTimingTable),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.arb_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU75_Firmware_Header, Version),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		hwmgr->microcode_version_info.SMC = tmp;
+
+	error |= (0 != result);
+
+	return error ? -1 : 0;
+}
+
+static bool vegam_is_dpm_running(struct pp_hwmgr *hwmgr)
+{
+	return (1 == PHM_READ_INDIRECT_FIELD(hwmgr->device,
+			CGS_IND_REG__SMC, FEATURE_STATUS, VOLTAGE_CONTROLLER_ON))
+			? true : false;
+}
+
+static uint32_t vegam_get_mac_definition(uint32_t value)
+{
+	switch (value) {
+	case SMU_MAX_LEVELS_GRAPHICS:
+		return SMU75_MAX_LEVELS_GRAPHICS;
+	case SMU_MAX_LEVELS_MEMORY:
+		return SMU75_MAX_LEVELS_MEMORY;
+	case SMU_MAX_LEVELS_LINK:
+		return SMU75_MAX_LEVELS_LINK;
+	case SMU_MAX_ENTRIES_SMIO:
+		return SMU75_MAX_ENTRIES_SMIO;
+	case SMU_MAX_LEVELS_VDDC:
+		return SMU75_MAX_LEVELS_VDDC;
+	case SMU_MAX_LEVELS_VDDGFX:
+		return SMU75_MAX_LEVELS_VDDGFX;
+	case SMU_MAX_LEVELS_VDDCI:
+		return SMU75_MAX_LEVELS_VDDCI;
+	case SMU_MAX_LEVELS_MVDD:
+		return SMU75_MAX_LEVELS_MVDD;
+	case SMU_UVD_MCLK_HANDSHAKE_DISABLE:
+		return SMU7_UVD_MCLK_HANDSHAKE_DISABLE |
+				SMU7_VCE_MCLK_HANDSHAKE_DISABLE;
+	}
+
+	pr_warn("can't get the mac of %x\n", value);
+	return 0;
+}
+
+static int vegam_update_uvd_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	smu_data->smc_state_table.UvdBootLevel = 0;
+	if (table_info->mm_dep_table->count > 0)
+		smu_data->smc_state_table.UvdBootLevel =
+				(uint8_t) (table_info->mm_dep_table->count - 1);
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start + offsetof(SMU75_Discrete_DpmTable,
+						UvdBootLevel);
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0x00FFFFFF;
+	mm_boot_level_value |= smu_data->smc_state_table.UvdBootLevel << 24;
+	cgs_write_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_UVDDPM) ||
+		phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+				PPSMC_MSG_UVDDPM_SetEnabledMask,
+				(uint32_t)(1 << smu_data->smc_state_table.UvdBootLevel));
+	return 0;
+}
+
+static int vegam_update_vce_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_StablePState))
+		smu_data->smc_state_table.VceBootLevel =
+			(uint8_t) (table_info->mm_dep_table->count - 1);
+	else
+		smu_data->smc_state_table.VceBootLevel = 0;
+
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start +
+					offsetof(SMU75_Discrete_DpmTable, VceBootLevel);
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0xFF00FFFF;
+	mm_boot_level_value |= smu_data->smc_state_table.VceBootLevel << 16;
+	cgs_write_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+				PPSMC_MSG_VCEDPM_SetEnabledMask,
+				(uint32_t)1 << smu_data->smc_state_table.VceBootLevel);
+	return 0;
+}
+
+static int vegam_update_samu_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+
+
+	smu_data->smc_state_table.SamuBootLevel = 0;
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU75_Discrete_DpmTable, SamuBootLevel);
+
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0xFFFFFF00;
+	mm_boot_level_value |= smu_data->smc_state_table.SamuBootLevel << 0;
+	cgs_write_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+				PPSMC_MSG_SAMUDPM_SetEnabledMask,
+				(uint32_t)(1 << smu_data->smc_state_table.SamuBootLevel));
+	return 0;
+}
+
+
+static int vegam_update_bif_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_pcie_table *pcie_table = table_info->pcie_table;
+	int max_entry, i;
+
+	max_entry = (SMU75_MAX_LEVELS_LINK < pcie_table->count) ?
+						SMU75_MAX_LEVELS_LINK :
+						pcie_table->count;
+	/* Setup BIF_SCLK levels */
+	for (i = 0; i < max_entry; i++)
+		smu_data->bif_sclk_table[i] = pcie_table->entries[i].pcie_sclk;
+	return 0;
+}
+
+static int vegam_update_smc_table(struct pp_hwmgr *hwmgr, uint32_t type)
+{
+	switch (type) {
+	case SMU_UVD_TABLE:
+		vegam_update_uvd_smc_table(hwmgr);
+		break;
+	case SMU_VCE_TABLE:
+		vegam_update_vce_smc_table(hwmgr);
+		break;
+	case SMU_SAMU_TABLE:
+		vegam_update_samu_smc_table(hwmgr);
+		break;
+	case SMU_BIF_TABLE:
+		vegam_update_bif_smc_table(hwmgr);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static void vegam_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	struct  phm_ppt_v1_information *table_info =
+			(struct  phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (table_info &&
+			table_info->cac_dtp_table->usPowerTuneDataSetID <= POWERTUNE_DEFAULT_SET_MAX &&
+			table_info->cac_dtp_table->usPowerTuneDataSetID)
+		smu_data->power_tune_defaults =
+				&vegam_power_tune_data_set_array
+				[table_info->cac_dtp_table->usPowerTuneDataSetID - 1];
+	else
+		smu_data->power_tune_defaults = &vegam_power_tune_data_set_array[0];
+
+}
+
+static int vegam_populate_smc_mvdd_table(struct pp_hwmgr *hwmgr,
+			SMU75_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t count, level;
+
+	if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		count = data->mvdd_voltage_table.count;
+		if (count > SMU_MAX_SMIO_LEVELS)
+			count = SMU_MAX_SMIO_LEVELS;
+		for (level = 0; level < count; level++) {
+			table->SmioTable2.Pattern[level].Voltage = PP_HOST_TO_SMC_US(
+					data->mvdd_voltage_table.entries[count].value * VOLTAGE_SCALE);
+			/* Index into DpmTable.Smio. Drive bits from Smio entry to get this voltage level.*/
+			table->SmioTable2.Pattern[level].Smio =
+				(uint8_t) level;
+			table->Smio[level] |=
+				data->mvdd_voltage_table.entries[level].smio_low;
+		}
+		table->SmioMask2 = data->mvdd_voltage_table.mask_low;
+
+		table->MvddLevelCount = (uint32_t) PP_HOST_TO_SMC_UL(count);
+	}
+
+	return 0;
+}
+
+static int vegam_populate_smc_vddci_table(struct pp_hwmgr *hwmgr,
+					struct SMU75_Discrete_DpmTable *table)
+{
+	uint32_t count, level;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	count = data->vddci_voltage_table.count;
+
+	if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
+		if (count > SMU_MAX_SMIO_LEVELS)
+			count = SMU_MAX_SMIO_LEVELS;
+		for (level = 0; level < count; ++level) {
+			table->SmioTable1.Pattern[level].Voltage = PP_HOST_TO_SMC_US(
+					data->vddci_voltage_table.entries[level].value * VOLTAGE_SCALE);
+			table->SmioTable1.Pattern[level].Smio = (uint8_t) level;
+
+			table->Smio[level] |= data->vddci_voltage_table.entries[level].smio_low;
+		}
+	}
+
+	table->SmioMask1 = data->vddci_voltage_table.mask_low;
+
+	return 0;
+}
+
+static int vegam_populate_cac_table(struct pp_hwmgr *hwmgr,
+		struct SMU75_Discrete_DpmTable *table)
+{
+	uint32_t count;
+	uint8_t index;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_voltage_lookup_table *lookup_table =
+			table_info->vddc_lookup_table;
+	/* tables is already swapped, so in order to use the value from it,
+	 * we need to swap it back.
+	 * We are populating vddc CAC data to BapmVddc table
+	 * in split and merged mode
+	 */
+	for (count = 0; count < lookup_table->count; count++) {
+		index = phm_get_voltage_index(lookup_table,
+				data->vddc_voltage_table.entries[count].value);
+		table->BapmVddcVidLoSidd[count] =
+				convert_to_vid(lookup_table->entries[index].us_cac_low);
+		table->BapmVddcVidHiSidd[count] =
+				convert_to_vid(lookup_table->entries[index].us_cac_mid);
+		table->BapmVddcVidHiSidd2[count] =
+				convert_to_vid(lookup_table->entries[index].us_cac_high);
+	}
+
+	return 0;
+}
+
+static int vegam_populate_smc_voltage_tables(struct pp_hwmgr *hwmgr,
+		struct SMU75_Discrete_DpmTable *table)
+{
+	vegam_populate_smc_vddci_table(hwmgr, table);
+	vegam_populate_smc_mvdd_table(hwmgr, table);
+	vegam_populate_cac_table(hwmgr, table);
+
+	return 0;
+}
+
+static int vegam_populate_ulv_level(struct pp_hwmgr *hwmgr,
+		struct SMU75_Discrete_Ulv *state)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	state->CcPwrDynRm = 0;
+	state->CcPwrDynRm1 = 0;
+
+	state->VddcOffset = (uint16_t) table_info->us_ulv_voltage_offset;
+	state->VddcOffsetVid = (uint8_t)(table_info->us_ulv_voltage_offset *
+			VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1);
+
+	state->VddcPhase = data->vddc_phase_shed_control ^ 0x3;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm1);
+	CONVERT_FROM_HOST_TO_SMC_US(state->VddcOffset);
+
+	return 0;
+}
+
+static int vegam_populate_ulv_state(struct pp_hwmgr *hwmgr,
+		struct SMU75_Discrete_DpmTable *table)
+{
+	return vegam_populate_ulv_level(hwmgr, &table->Ulv);
+}
+
+static int vegam_populate_smc_link_level(struct pp_hwmgr *hwmgr,
+		struct SMU75_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct vegam_smumgr *smu_data =
+			(struct vegam_smumgr *)(hwmgr->smu_backend);
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+	int i;
+
+	/* Index (dpm_table->pcie_speed_table.count)
+	 * is reserved for PCIE boot level. */
+	for (i = 0; i <= dpm_table->pcie_speed_table.count; i++) {
+		table->LinkLevel[i].PcieGenSpeed  =
+				(uint8_t)dpm_table->pcie_speed_table.dpm_levels[i].value;
+		table->LinkLevel[i].PcieLaneCount = (uint8_t)encode_pcie_lane_width(
+				dpm_table->pcie_speed_table.dpm_levels[i].param1);
+		table->LinkLevel[i].EnabledForActivity = 1;
+		table->LinkLevel[i].SPC = (uint8_t)(data->pcie_spc_cap & 0xff);
+		table->LinkLevel[i].DownThreshold = PP_HOST_TO_SMC_UL(5);
+		table->LinkLevel[i].UpThreshold = PP_HOST_TO_SMC_UL(30);
+	}
+
+	smu_data->smc_state_table.LinkLevelCount =
+			(uint8_t)dpm_table->pcie_speed_table.count;
+
+/* To Do move to hwmgr */
+	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
+
+	return 0;
+}
+
+static int vegam_get_dependency_volt_by_clk(struct pp_hwmgr *hwmgr,
+		struct phm_ppt_v1_clock_voltage_dependency_table *dep_table,
+		uint32_t clock, SMU_VoltageLevel *voltage, uint32_t *mvdd)
+{
+	uint32_t i;
+	uint16_t vddci;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	*voltage = *mvdd = 0;
+
+	/* clock - voltage dependency table is empty table */
+	if (dep_table->count == 0)
+		return -EINVAL;
+
+	for (i = 0; i < dep_table->count; i++) {
+		/* find first sclk bigger than request */
+		if (dep_table->entries[i].clk >= clock) {
+			*voltage |= (dep_table->entries[i].vddc *
+					VOLTAGE_SCALE) << VDDC_SHIFT;
+			if (SMU7_VOLTAGE_CONTROL_NONE == data->vddci_control)
+				*voltage |= (data->vbios_boot_state.vddci_bootup_value *
+						VOLTAGE_SCALE) << VDDCI_SHIFT;
+			else if (dep_table->entries[i].vddci)
+				*voltage |= (dep_table->entries[i].vddci *
+						VOLTAGE_SCALE) << VDDCI_SHIFT;
+			else {
+				vddci = phm_find_closest_vddci(&(data->vddci_voltage_table),
+						(dep_table->entries[i].vddc -
+								(uint16_t)VDDC_VDDCI_DELTA));
+				*voltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+			}
+
+			if (SMU7_VOLTAGE_CONTROL_NONE == data->mvdd_control)
+				*mvdd = data->vbios_boot_state.mvdd_bootup_value *
+					VOLTAGE_SCALE;
+			else if (dep_table->entries[i].mvdd)
+				*mvdd = (uint32_t) dep_table->entries[i].mvdd *
+					VOLTAGE_SCALE;
+
+			*voltage |= 1 << PHASES_SHIFT;
+			return 0;
+		}
+	}
+
+	/* sclk is bigger than max sclk in the dependence table */
+	*voltage |= (dep_table->entries[i - 1].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
+	vddci = phm_find_closest_vddci(&(data->vddci_voltage_table),
+			(dep_table->entries[i - 1].vddc -
+					(uint16_t)VDDC_VDDCI_DELTA));
+
+	if (SMU7_VOLTAGE_CONTROL_NONE == data->vddci_control)
+		*voltage |= (data->vbios_boot_state.vddci_bootup_value *
+				VOLTAGE_SCALE) << VDDCI_SHIFT;
+	else if (dep_table->entries[i - 1].vddci)
+		*voltage |= (dep_table->entries[i - 1].vddci *
+				VOLTAGE_SCALE) << VDDC_SHIFT;
+	else
+		*voltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+
+	if (SMU7_VOLTAGE_CONTROL_NONE == data->mvdd_control)
+		*mvdd = data->vbios_boot_state.mvdd_bootup_value * VOLTAGE_SCALE;
+	else if (dep_table->entries[i].mvdd)
+		*mvdd = (uint32_t) dep_table->entries[i - 1].mvdd * VOLTAGE_SCALE;
+
+	return 0;
+}
+
+static void vegam_get_sclk_range_table(struct pp_hwmgr *hwmgr,
+				   SMU75_Discrete_DpmTable  *table)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	uint32_t i, ref_clk;
+
+	struct pp_atom_ctrl_sclk_range_table range_table_from_vbios = { { {0} } };
+
+	ref_clk = amdgpu_asic_get_xclk((struct amdgpu_device *)hwmgr->adev);
+
+	if (0 == atomctrl_get_smc_sclk_range_table(hwmgr, &range_table_from_vbios)) {
+		for (i = 0; i < NUM_SCLK_RANGE; i++) {
+			table->SclkFcwRangeTable[i].vco_setting =
+					range_table_from_vbios.entry[i].ucVco_setting;
+			table->SclkFcwRangeTable[i].postdiv =
+					range_table_from_vbios.entry[i].ucPostdiv;
+			table->SclkFcwRangeTable[i].fcw_pcc =
+					range_table_from_vbios.entry[i].usFcw_pcc;
+
+			table->SclkFcwRangeTable[i].fcw_trans_upper =
+					range_table_from_vbios.entry[i].usFcw_trans_upper;
+			table->SclkFcwRangeTable[i].fcw_trans_lower =
+					range_table_from_vbios.entry[i].usRcw_trans_lower;
+
+			CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_pcc);
+			CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_upper);
+			CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_lower);
+		}
+		return;
+	}
+
+	for (i = 0; i < NUM_SCLK_RANGE; i++) {
+		smu_data->range_table[i].trans_lower_frequency =
+				(ref_clk * Range_Table[i].fcw_trans_lower) >> Range_Table[i].postdiv;
+		smu_data->range_table[i].trans_upper_frequency =
+				(ref_clk * Range_Table[i].fcw_trans_upper) >> Range_Table[i].postdiv;
+
+		table->SclkFcwRangeTable[i].vco_setting = Range_Table[i].vco_setting;
+		table->SclkFcwRangeTable[i].postdiv = Range_Table[i].postdiv;
+		table->SclkFcwRangeTable[i].fcw_pcc = Range_Table[i].fcw_pcc;
+
+		table->SclkFcwRangeTable[i].fcw_trans_upper = Range_Table[i].fcw_trans_upper;
+		table->SclkFcwRangeTable[i].fcw_trans_lower = Range_Table[i].fcw_trans_lower;
+
+		CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_pcc);
+		CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_upper);
+		CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_lower);
+	}
+}
+
+static int vegam_calculate_sclk_params(struct pp_hwmgr *hwmgr,
+		uint32_t clock, SMU_SclkSetting *sclk_setting)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	const SMU75_Discrete_DpmTable *table = &(smu_data->smc_state_table);
+	struct pp_atomctrl_clock_dividers_ai dividers;
+	uint32_t ref_clock;
+	uint32_t pcc_target_percent, pcc_target_freq, ss_target_percent, ss_target_freq;
+	uint8_t i;
+	int result;
+	uint64_t temp;
+
+	sclk_setting->SclkFrequency = clock;
+	/* get the engine clock dividers for this clock value */
+	result = atomctrl_get_engine_pll_dividers_ai(hwmgr, clock,  &dividers);
+	if (result == 0) {
+		sclk_setting->Fcw_int = dividers.usSclk_fcw_int;
+		sclk_setting->Fcw_frac = dividers.usSclk_fcw_frac;
+		sclk_setting->Pcc_fcw_int = dividers.usPcc_fcw_int;
+		sclk_setting->PllRange = dividers.ucSclkPllRange;
+		sclk_setting->Sclk_slew_rate = 0x400;
+		sclk_setting->Pcc_up_slew_rate = dividers.usPcc_fcw_slew_frac;
+		sclk_setting->Pcc_down_slew_rate = 0xffff;
+		sclk_setting->SSc_En = dividers.ucSscEnable;
+		sclk_setting->Fcw1_int = dividers.usSsc_fcw1_int;
+		sclk_setting->Fcw1_frac = dividers.usSsc_fcw1_frac;
+		sclk_setting->Sclk_ss_slew_rate = dividers.usSsc_fcw_slew_frac;
+		return result;
+	}
+
+	ref_clock = amdgpu_asic_get_xclk((struct amdgpu_device *)hwmgr->adev);
+
+	for (i = 0; i < NUM_SCLK_RANGE; i++) {
+		if (clock > smu_data->range_table[i].trans_lower_frequency
+		&& clock <= smu_data->range_table[i].trans_upper_frequency) {
+			sclk_setting->PllRange = i;
+			break;
+		}
+	}
+
+	sclk_setting->Fcw_int = (uint16_t)
+			((clock << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv) /
+					ref_clock);
+	temp = clock << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv;
+	temp <<= 0x10;
+	do_div(temp, ref_clock);
+	sclk_setting->Fcw_frac = temp & 0xffff;
+
+	pcc_target_percent = 10; /*  Hardcode 10% for now. */
+	pcc_target_freq = clock - (clock * pcc_target_percent / 100);
+	sclk_setting->Pcc_fcw_int = (uint16_t)
+			((pcc_target_freq << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv) /
+					ref_clock);
+
+	ss_target_percent = 2; /*  Hardcode 2% for now. */
+	sclk_setting->SSc_En = 0;
+	if (ss_target_percent) {
+		sclk_setting->SSc_En = 1;
+		ss_target_freq = clock - (clock * ss_target_percent / 100);
+		sclk_setting->Fcw1_int = (uint16_t)
+				((ss_target_freq << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv) /
+						ref_clock);
+		temp = ss_target_freq << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv;
+		temp <<= 0x10;
+		do_div(temp, ref_clock);
+		sclk_setting->Fcw1_frac = temp & 0xffff;
+	}
+
+	return 0;
+}
+
+static uint8_t vegam_get_sleep_divider_id_from_clock(uint32_t clock,
+		uint32_t clock_insr)
+{
+	uint8_t i;
+	uint32_t temp;
+	uint32_t min = max(clock_insr, (uint32_t)SMU7_MINIMUM_ENGINE_CLOCK);
+
+	PP_ASSERT_WITH_CODE((clock >= min),
+			"Engine clock can't satisfy stutter requirement!",
+			return 0);
+	for (i = 31;  ; i--) {
+		temp = clock / (i + 1);
+
+		if (temp >= min || i == 0)
+			break;
+	}
+	return i;
+}
+
+static int vegam_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
+		uint32_t clock, struct SMU75_Discrete_GraphicsLevel *level)
+{
+	int result;
+	/* PP_Clocks minClocks; */
+	uint32_t mvdd;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	SMU_SclkSetting curr_sclk_setting = { 0 };
+
+	result = vegam_calculate_sclk_params(hwmgr, clock, &curr_sclk_setting);
+
+	/* populate graphics levels */
+	result = vegam_get_dependency_volt_by_clk(hwmgr,
+			table_info->vdd_dep_on_sclk, clock,
+			&level->MinVoltage, &mvdd);
+
+	PP_ASSERT_WITH_CODE((0 == result),
+			"can not find VDDC voltage value for "
+			"VDDC engine clock dependency table",
+			return result);
+	level->ActivityLevel = (uint16_t)(SclkDPMTuning_VEGAM >> DPMTuning_Activity_Shift);
+
+	level->CcPwrDynRm = 0;
+	level->CcPwrDynRm1 = 0;
+	level->EnabledForActivity = 0;
+	level->EnabledForThrottle = 1;
+	level->VoltageDownHyst = 0;
+	level->PowerThrottle = 0;
+	data->display_timing.min_clock_in_sr = hwmgr->display_config->min_core_set_clock_in_sr;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep))
+		level->DeepSleepDivId = vegam_get_sleep_divider_id_from_clock(clock,
+								hwmgr->display_config->min_core_set_clock_in_sr);
+
+	level->SclkSetting = curr_sclk_setting;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(level->MinVoltage);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm1);
+	CONVERT_FROM_HOST_TO_SMC_US(level->ActivityLevel);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->SclkSetting.SclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw_int);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw_frac);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Pcc_fcw_int);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Sclk_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Pcc_up_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Pcc_down_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw1_int);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw1_frac);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Sclk_ss_slew_rate);
+	return 0;
+}
+
+static int vegam_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *hw_data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	struct smu7_dpm_table *dpm_table = &hw_data->dpm_table;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_pcie_table *pcie_table = table_info->pcie_table;
+	uint8_t pcie_entry_cnt = (uint8_t) hw_data->dpm_table.pcie_speed_table.count;
+	int result = 0;
+	uint32_t array = smu_data->smu7_data.dpm_table_start +
+			offsetof(SMU75_Discrete_DpmTable, GraphicsLevel);
+	uint32_t array_size = sizeof(struct SMU75_Discrete_GraphicsLevel) *
+			SMU75_MAX_LEVELS_GRAPHICS;
+	struct SMU75_Discrete_GraphicsLevel *levels =
+			smu_data->smc_state_table.GraphicsLevel;
+	uint32_t i, max_entry;
+	uint8_t hightest_pcie_level_enabled = 0,
+		lowest_pcie_level_enabled = 0,
+		mid_pcie_level_enabled = 0,
+		count = 0;
+
+	vegam_get_sclk_range_table(hwmgr, &(smu_data->smc_state_table));
+
+	for (i = 0; i < dpm_table->sclk_table.count; i++) {
+
+		result = vegam_populate_single_graphic_level(hwmgr,
+				dpm_table->sclk_table.dpm_levels[i].value,
+				&(smu_data->smc_state_table.GraphicsLevel[i]));
+		if (result)
+			return result;
+
+		levels[i].UpHyst = (uint8_t)
+				(SclkDPMTuning_VEGAM >> DPMTuning_Uphyst_Shift);
+		levels[i].DownHyst = (uint8_t)
+				(SclkDPMTuning_VEGAM >> DPMTuning_Downhyst_Shift);
+		/* Making sure only DPM level 0-1 have Deep Sleep Div ID populated. */
+		if (i > 1)
+			levels[i].DeepSleepDivId = 0;
+	}
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_SPLLShutdownSupport))
+		smu_data->smc_state_table.GraphicsLevel[0].SclkSetting.SSc_En = 0;
+
+	smu_data->smc_state_table.GraphicsDpmLevelCount =
+			(uint8_t)dpm_table->sclk_table.count;
+	hw_data->dpm_level_enable_mask.sclk_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
+
+	for (i = 0; i < dpm_table->sclk_table.count; i++)
+		levels[i].EnabledForActivity =
+				(hw_data->dpm_level_enable_mask.sclk_dpm_enable_mask >> i) & 0x1;
+
+	if (pcie_table != NULL) {
+		PP_ASSERT_WITH_CODE((1 <= pcie_entry_cnt),
+				"There must be 1 or more PCIE levels defined in PPTable.",
+				return -EINVAL);
+		max_entry = pcie_entry_cnt - 1;
+		for (i = 0; i < dpm_table->sclk_table.count; i++)
+			levels[i].pcieDpmLevel =
+					(uint8_t) ((i < max_entry) ? i : max_entry);
+	} else {
+		while (hw_data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((hw_data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << (hightest_pcie_level_enabled + 1))) != 0))
+			hightest_pcie_level_enabled++;
+
+		while (hw_data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((hw_data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << lowest_pcie_level_enabled)) == 0))
+			lowest_pcie_level_enabled++;
+
+		while ((count < hightest_pcie_level_enabled) &&
+				((hw_data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << (lowest_pcie_level_enabled + 1 + count))) == 0))
+			count++;
+
+		mid_pcie_level_enabled = (lowest_pcie_level_enabled + 1 + count) <
+				hightest_pcie_level_enabled ?
+						(lowest_pcie_level_enabled + 1 + count) :
+						hightest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to hightest_pcie_level_enabled */
+		for (i = 2; i < dpm_table->sclk_table.count; i++)
+			levels[i].pcieDpmLevel = hightest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to lowest_pcie_level_enabled */
+		levels[0].pcieDpmLevel = lowest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to mid_pcie_level_enabled */
+		levels[1].pcieDpmLevel = mid_pcie_level_enabled;
+	}
+	/* level count will send to smc once at init smc table and never change */
+	result = smu7_copy_bytes_to_smc(hwmgr, array, (uint8_t *)levels,
+			(uint32_t)array_size, SMC_RAM_END);
+
+	return result;
+}
+
+static int vegam_calculate_mclk_params(struct pp_hwmgr *hwmgr,
+		uint32_t clock, struct SMU75_Discrete_MemoryLevel *mem_level)
+{
+	struct pp_atomctrl_memory_clock_param_ai mpll_param;
+
+	PP_ASSERT_WITH_CODE(!atomctrl_get_memory_pll_dividers_ai(hwmgr,
+			clock, &mpll_param),
+			"Failed to retrieve memory pll parameter.",
+			return -EINVAL);
+
+	mem_level->MclkFrequency = (uint32_t)mpll_param.ulClock;
+	mem_level->Fcw_int = (uint16_t)mpll_param.ulMclk_fcw_int;
+	mem_level->Fcw_frac = (uint16_t)mpll_param.ulMclk_fcw_frac;
+	mem_level->Postdiv = (uint8_t)mpll_param.ulPostDiv;
+
+	return 0;
+}
+
+static int vegam_populate_single_memory_level(struct pp_hwmgr *hwmgr,
+		uint32_t clock, struct SMU75_Discrete_MemoryLevel *mem_level)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	int result = 0;
+	uint32_t mclk_stutter_mode_threshold = 60000;
+
+
+	if (table_info->vdd_dep_on_mclk) {
+		result = vegam_get_dependency_volt_by_clk(hwmgr,
+				table_info->vdd_dep_on_mclk, clock,
+				&mem_level->MinVoltage, &mem_level->MinMvdd);
+		PP_ASSERT_WITH_CODE(!result,
+				"can not find MinVddc voltage value from memory "
+				"VDDC voltage dependency table", return result);
+	}
+
+	result = vegam_calculate_mclk_params(hwmgr, clock, mem_level);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to calculate mclk params.",
+			return -EINVAL);
+
+	mem_level->EnabledForThrottle = 1;
+	mem_level->EnabledForActivity = 0;
+	mem_level->VoltageDownHyst = 0;
+	mem_level->ActivityLevel = (uint16_t)
+			(MemoryDPMTuning_VEGAM >> DPMTuning_Activity_Shift);
+	mem_level->StutterEnable = false;
+	mem_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	data->display_timing.num_existing_displays = hwmgr->display_config->num_display;
+
+	if (mclk_stutter_mode_threshold &&
+		(clock <= mclk_stutter_mode_threshold) &&
+		(PHM_READ_FIELD(hwmgr->device, DPG_PIPE_STUTTER_CONTROL,
+				STUTTER_ENABLE) & 0x1))
+		mem_level->StutterEnable = true;
+
+	if (!result) {
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinMvdd);
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_US(mem_level->Fcw_int);
+		CONVERT_FROM_HOST_TO_SMC_US(mem_level->Fcw_frac);
+		CONVERT_FROM_HOST_TO_SMC_US(mem_level->ActivityLevel);
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinVoltage);
+	}
+
+	return result;
+}
+
+static int vegam_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *hw_data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	struct smu7_dpm_table *dpm_table = &hw_data->dpm_table;
+	int result;
+	/* populate MCLK dpm table to SMU7 */
+	uint32_t array = smu_data->smu7_data.dpm_table_start +
+			offsetof(SMU75_Discrete_DpmTable, MemoryLevel);
+	uint32_t array_size = sizeof(SMU75_Discrete_MemoryLevel) *
+			SMU75_MAX_LEVELS_MEMORY;
+	struct SMU75_Discrete_MemoryLevel *levels =
+			smu_data->smc_state_table.MemoryLevel;
+	uint32_t i;
+
+	for (i = 0; i < dpm_table->mclk_table.count; i++) {
+		PP_ASSERT_WITH_CODE((0 != dpm_table->mclk_table.dpm_levels[i].value),
+				"can not populate memory level as memory clock is zero",
+				return -EINVAL);
+		result = vegam_populate_single_memory_level(hwmgr,
+				dpm_table->mclk_table.dpm_levels[i].value,
+				&levels[i]);
+
+		if (result)
+			return result;
+
+		levels[i].UpHyst = (uint8_t)
+				(MemoryDPMTuning_VEGAM >> DPMTuning_Uphyst_Shift);
+		levels[i].DownHyst = (uint8_t)
+				(MemoryDPMTuning_VEGAM >> DPMTuning_Downhyst_Shift);
+	}
+
+	smu_data->smc_state_table.MemoryDpmLevelCount =
+			(uint8_t)dpm_table->mclk_table.count;
+	hw_data->dpm_level_enable_mask.mclk_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
+
+	for (i = 0; i < dpm_table->mclk_table.count; i++)
+		levels[i].EnabledForActivity =
+				(hw_data->dpm_level_enable_mask.mclk_dpm_enable_mask >> i) & 0x1;
+
+	levels[dpm_table->mclk_table.count - 1].DisplayWatermark =
+			PPSMC_DISPLAY_WATERMARK_HIGH;
+
+	/* level count will send to smc once at init smc table and never change */
+	result = smu7_copy_bytes_to_smc(hwmgr, array, (uint8_t *)levels,
+			(uint32_t)array_size, SMC_RAM_END);
+
+	return result;
+}
+
+static int vegam_populate_mvdd_value(struct pp_hwmgr *hwmgr,
+		uint32_t mclk, SMIO_Pattern *smio_pat)
+{
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint32_t i = 0;
+
+	if (SMU7_VOLTAGE_CONTROL_NONE != data->mvdd_control) {
+		/* find mvdd value which clock is more than request */
+		for (i = 0; i < table_info->vdd_dep_on_mclk->count; i++) {
+			if (mclk <= table_info->vdd_dep_on_mclk->entries[i].clk) {
+				smio_pat->Voltage = data->mvdd_voltage_table.entries[i].value;
+				break;
+			}
+		}
+		PP_ASSERT_WITH_CODE(i < table_info->vdd_dep_on_mclk->count,
+				"MVDD Voltage is outside the supported range.",
+				return -EINVAL);
+	} else
+		return -EINVAL;
+
+	return 0;
+}
+
+static int vegam_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
+		SMU75_Discrete_DpmTable *table)
+{
+	int result = 0;
+	uint32_t sclk_frequency;
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	SMIO_Pattern vol_level;
+	uint32_t mvdd;
+	uint16_t us_mvdd;
+
+	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+	/* Get MinVoltage and Frequency from DPM0,
+	 * already converted to SMC_UL */
+	sclk_frequency = data->vbios_boot_state.sclk_bootup_value;
+	result = vegam_get_dependency_volt_by_clk(hwmgr,
+			table_info->vdd_dep_on_sclk,
+			sclk_frequency,
+			&table->ACPILevel.MinVoltage, &mvdd);
+	PP_ASSERT_WITH_CODE(!result,
+			"Cannot find ACPI VDDC voltage value "
+			"in Clock Dependency Table",
+			);
+
+	result = vegam_calculate_sclk_params(hwmgr, sclk_frequency,
+			&(table->ACPILevel.SclkSetting));
+	PP_ASSERT_WITH_CODE(!result,
+			"Error retrieving Engine Clock dividers from VBIOS.",
+			return result);
+
+	table->ACPILevel.DeepSleepDivId = 0;
+	table->ACPILevel.CcPwrDynRm = 0;
+	table->ACPILevel.CcPwrDynRm1 = 0;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.Flags);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.MinVoltage);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm1);
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SclkSetting.SclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw_int);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw_frac);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Pcc_fcw_int);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Sclk_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Pcc_up_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Pcc_down_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw1_int);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw1_frac);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Sclk_ss_slew_rate);
+
+
+	/* Get MinVoltage and Frequency from DPM0, already converted to SMC_UL */
+	table->MemoryACPILevel.MclkFrequency = data->vbios_boot_state.mclk_bootup_value;
+	result = vegam_get_dependency_volt_by_clk(hwmgr,
+			table_info->vdd_dep_on_mclk,
+			table->MemoryACPILevel.MclkFrequency,
+			&table->MemoryACPILevel.MinVoltage, &mvdd);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"Cannot find ACPI VDDCI voltage value "
+			"in Clock Dependency Table",
+			);
+
+	us_mvdd = 0;
+	if ((SMU7_VOLTAGE_CONTROL_NONE == data->mvdd_control) ||
+			(data->mclk_dpm_key_disabled))
+		us_mvdd = data->vbios_boot_state.mvdd_bootup_value;
+	else {
+		if (!vegam_populate_mvdd_value(hwmgr,
+				data->dpm_table.mclk_table.dpm_levels[0].value,
+				&vol_level))
+			us_mvdd = vol_level.Voltage;
+	}
+
+	if (!vegam_populate_mvdd_value(hwmgr, 0, &vol_level))
+		table->MemoryACPILevel.MinMvdd = PP_HOST_TO_SMC_UL(vol_level.Voltage);
+	else
+		table->MemoryACPILevel.MinMvdd = 0;
+
+	table->MemoryACPILevel.StutterEnable = false;
+
+	table->MemoryACPILevel.EnabledForThrottle = 0;
+	table->MemoryACPILevel.EnabledForActivity = 0;
+	table->MemoryACPILevel.UpHyst = 0;
+	table->MemoryACPILevel.DownHyst = 100;
+	table->MemoryACPILevel.VoltageDownHyst = 0;
+	table->MemoryACPILevel.ActivityLevel =
+		PP_HOST_TO_SMC_US(data->current_profile_setting.mclk_activity);
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MinVoltage);
+
+	return result;
+}
+
+static int vegam_populate_smc_vce_level(struct pp_hwmgr *hwmgr,
+		SMU75_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t vddci;
+
+	table->VceLevelCount = (uint8_t)(mm_table->count);
+	table->VceBootLevel = 0;
+
+	for (count = 0; count < table->VceLevelCount; count++) {
+		table->VceLevel[count].Frequency = mm_table->entries[count].eclk;
+		table->VceLevel[count].MinVoltage = 0;
+		table->VceLevel[count].MinVoltage |=
+				(mm_table->entries[count].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
+
+		if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control)
+			vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table),
+						mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
+		else if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control)
+			vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA;
+		else
+			vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT;
+
+
+		table->VceLevel[count].MinVoltage |=
+				(vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->VceLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/*retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->VceLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for VCE engine clock",
+				return result);
+
+		table->VceLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].Frequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].MinVoltage);
+	}
+	return result;
+}
+
+static int vegam_populate_smc_samu_level(struct pp_hwmgr *hwmgr,
+		SMU75_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t vddci;
+
+	table->SamuBootLevel = 0;
+	table->SamuLevelCount = (uint8_t)(mm_table->count);
+
+	for (count = 0; count < table->SamuLevelCount; count++) {
+		/* not sure whether we need evclk or not */
+		table->SamuLevel[count].MinVoltage = 0;
+		table->SamuLevel[count].Frequency = mm_table->entries[count].samclock;
+		table->SamuLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
+				VOLTAGE_SCALE) << VDDC_SHIFT;
+
+		if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control)
+			vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table),
+						mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
+		else if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control)
+			vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA;
+		else
+			vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT;
+
+		table->SamuLevel[count].MinVoltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->SamuLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->SamuLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for samu clock", return result);
+
+		table->SamuLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].Frequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].MinVoltage);
+	}
+	return result;
+}
+
+static int vegam_populate_memory_timing_parameters(struct pp_hwmgr *hwmgr,
+		int32_t eng_clock, int32_t mem_clock,
+		SMU75_Discrete_MCArbDramTimingTableEntry *arb_regs)
+{
+	uint32_t dram_timing;
+	uint32_t dram_timing2;
+	uint32_t burst_time;
+	uint32_t rfsh_rate;
+	uint32_t misc3;
+
+	int result;
+
+	result = atomctrl_set_engine_dram_timings_rv770(hwmgr,
+			eng_clock, mem_clock);
+	PP_ASSERT_WITH_CODE(result == 0,
+			"Error calling VBIOS to set DRAM_TIMING.",
+			return result);
+
+	dram_timing = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
+	dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
+	burst_time = cgs_read_register(hwmgr->device, mmMC_ARB_BURST_TIME);
+	rfsh_rate = cgs_read_register(hwmgr->device, mmMC_ARB_RFSH_RATE);
+	misc3 = cgs_read_register(hwmgr->device, mmMC_ARB_MISC3);
+
+	arb_regs->McArbDramTiming  = PP_HOST_TO_SMC_UL(dram_timing);
+	arb_regs->McArbDramTiming2 = PP_HOST_TO_SMC_UL(dram_timing2);
+	arb_regs->McArbBurstTime   = PP_HOST_TO_SMC_UL(burst_time);
+	arb_regs->McArbRfshRate = PP_HOST_TO_SMC_UL(rfsh_rate);
+	arb_regs->McArbMisc3 = PP_HOST_TO_SMC_UL(misc3);
+
+	return 0;
+}
+
+static int vegam_program_memory_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *hw_data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	struct SMU75_Discrete_MCArbDramTimingTable arb_regs;
+	uint32_t i, j;
+	int result = 0;
+
+	memset(&arb_regs, 0, sizeof(SMU75_Discrete_MCArbDramTimingTable));
+
+	for (i = 0; i < hw_data->dpm_table.sclk_table.count; i++) {
+		for (j = 0; j < hw_data->dpm_table.mclk_table.count; j++) {
+			result = vegam_populate_memory_timing_parameters(hwmgr,
+					hw_data->dpm_table.sclk_table.dpm_levels[i].value,
+					hw_data->dpm_table.mclk_table.dpm_levels[j].value,
+					&arb_regs.entries[i][j]);
+			if (result)
+				return result;
+		}
+	}
+
+	result = smu7_copy_bytes_to_smc(
+			hwmgr,
+			smu_data->smu7_data.arb_table_start,
+			(uint8_t *)&arb_regs,
+			sizeof(SMU75_Discrete_MCArbDramTimingTable),
+			SMC_RAM_END);
+	return result;
+}
+
+static int vegam_populate_smc_uvd_level(struct pp_hwmgr *hwmgr,
+		struct SMU75_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t vddci;
+
+	table->UvdLevelCount = (uint8_t)(mm_table->count);
+	table->UvdBootLevel = 0;
+
+	for (count = 0; count < table->UvdLevelCount; count++) {
+		table->UvdLevel[count].MinVoltage = 0;
+		table->UvdLevel[count].VclkFrequency = mm_table->entries[count].vclk;
+		table->UvdLevel[count].DclkFrequency = mm_table->entries[count].dclk;
+		table->UvdLevel[count].MinVoltage |=
+				(mm_table->entries[count].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
+
+		if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control)
+			vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table),
+						mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
+		else if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control)
+			vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA;
+		else
+			vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT;
+
+		table->UvdLevel[count].MinVoltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->UvdLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->UvdLevel[count].VclkFrequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for Vclk clock", return result);
+
+		table->UvdLevel[count].VclkDivider = (uint8_t)dividers.pll_post_divider;
+
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->UvdLevel[count].DclkFrequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for Dclk clock", return result);
+
+		table->UvdLevel[count].DclkDivider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].VclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].DclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].MinVoltage);
+	}
+
+	return result;
+}
+
+static int vegam_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
+		struct SMU75_Discrete_DpmTable *table)
+{
+	int result = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	table->GraphicsBootLevel = 0;
+	table->MemoryBootLevel = 0;
+
+	/* find boot level from dpm table */
+	result = phm_find_boot_level(&(data->dpm_table.sclk_table),
+			data->vbios_boot_state.sclk_bootup_value,
+			(uint32_t *)&(table->GraphicsBootLevel));
+
+	result = phm_find_boot_level(&(data->dpm_table.mclk_table),
+			data->vbios_boot_state.mclk_bootup_value,
+			(uint32_t *)&(table->MemoryBootLevel));
+
+	table->BootVddc  = data->vbios_boot_state.vddc_bootup_value *
+			VOLTAGE_SCALE;
+	table->BootVddci = data->vbios_boot_state.vddci_bootup_value *
+			VOLTAGE_SCALE;
+	table->BootMVdd  = data->vbios_boot_state.mvdd_bootup_value *
+			VOLTAGE_SCALE;
+
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddc);
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddci);
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootMVdd);
+
+	return 0;
+}
+
+static int vegam_populate_smc_initial_state(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *hw_data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint8_t count, level;
+
+	count = (uint8_t)(table_info->vdd_dep_on_sclk->count);
+
+	for (level = 0; level < count; level++) {
+		if (table_info->vdd_dep_on_sclk->entries[level].clk >=
+				hw_data->vbios_boot_state.sclk_bootup_value) {
+			smu_data->smc_state_table.GraphicsBootLevel = level;
+			break;
+		}
+	}
+
+	count = (uint8_t)(table_info->vdd_dep_on_mclk->count);
+	for (level = 0; level < count; level++) {
+		if (table_info->vdd_dep_on_mclk->entries[level].clk >=
+				hw_data->vbios_boot_state.mclk_bootup_value) {
+			smu_data->smc_state_table.MemoryBootLevel = level;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static uint16_t scale_fan_gain_settings(uint16_t raw_setting)
+{
+	uint32_t tmp;
+	tmp = raw_setting * 4096 / 100;
+	return (uint16_t)tmp;
+}
+
+static int vegam_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+
+	const struct vegam_pt_defaults *defaults = smu_data->power_tune_defaults;
+	SMU75_Discrete_DpmTable  *table = &(smu_data->smc_state_table);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_cac_tdp_table *cac_dtp_table = table_info->cac_dtp_table;
+	struct pp_advance_fan_control_parameters *fan_table =
+			&hwmgr->thermal_controller.advanceFanControlParameters;
+	int i, j, k;
+	const uint16_t *pdef1;
+	const uint16_t *pdef2;
+
+	table->DefaultTdp = PP_HOST_TO_SMC_US((uint16_t)(cac_dtp_table->usTDP * 128));
+	table->TargetTdp  = PP_HOST_TO_SMC_US((uint16_t)(cac_dtp_table->usTDP * 128));
+
+	PP_ASSERT_WITH_CODE(cac_dtp_table->usTargetOperatingTemp <= 255,
+				"Target Operating Temp is out of Range!",
+				);
+
+	table->TemperatureLimitEdge = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTargetOperatingTemp * 256);
+	table->TemperatureLimitHotspot = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitHotspot * 256);
+	table->FanGainEdge = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainEdge));
+	table->FanGainHotspot = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainHotspot));
+
+	pdef1 = defaults->BAPMTI_R;
+	pdef2 = defaults->BAPMTI_RC;
+
+	for (i = 0; i < SMU75_DTE_ITERATIONS; i++) {
+		for (j = 0; j < SMU75_DTE_SOURCES; j++) {
+			for (k = 0; k < SMU75_DTE_SINKS; k++) {
+				table->BAPMTI_R[i][j][k] = PP_HOST_TO_SMC_US(*pdef1);
+				table->BAPMTI_RC[i][j][k] = PP_HOST_TO_SMC_US(*pdef2);
+				pdef1++;
+				pdef2++;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int vegam_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
+{
+	uint32_t ro, efuse, volt_without_cks, volt_with_cks, value, max, min;
+	struct vegam_smumgr *smu_data =
+			(struct vegam_smumgr *)(hwmgr->smu_backend);
+
+	uint8_t i, stretch_amount, stretch_amount2, volt_offset = 0;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
+			table_info->vdd_dep_on_sclk;
+	uint32_t mask = (1 << ((STRAP_ASIC_RO_MSB - STRAP_ASIC_RO_LSB) + 1)) - 1;
+
+	stretch_amount = (uint8_t)table_info->cac_dtp_table->usClockStretchAmount;
+
+	atomctrl_read_efuse(hwmgr, STRAP_ASIC_RO_LSB, STRAP_ASIC_RO_MSB,
+			mask, &efuse);
+
+	min = 1200;
+	max = 2500;
+
+	ro = efuse * (max - min) / 255 + min;
+
+	/* Populate Sclk_CKS_masterEn0_7 and Sclk_voltageOffset */
+	for (i = 0; i < sclk_table->count; i++) {
+		smu_data->smc_state_table.Sclk_CKS_masterEn0_7 |=
+				sclk_table->entries[i].cks_enable << i;
+		volt_without_cks = (uint32_t)((2753594000U + (sclk_table->entries[i].clk/100) *
+				136418 - (ro - 70) * 1000000) /
+				(2424180 - (sclk_table->entries[i].clk/100) * 1132925/1000));
+		volt_with_cks = (uint32_t)((2797202000U + sclk_table->entries[i].clk/100 *
+				3232 - (ro - 65) * 1000000) /
+				(2522480 - sclk_table->entries[i].clk/100 * 115764/100));
+
+		if (volt_without_cks >= volt_with_cks)
+			volt_offset = (uint8_t)(((volt_without_cks - volt_with_cks +
+					sclk_table->entries[i].cks_voffset) * 100 + 624) / 625);
+
+		smu_data->smc_state_table.Sclk_voltageOffset[i] = volt_offset;
+	}
+
+	smu_data->smc_state_table.LdoRefSel =
+			(table_info->cac_dtp_table->ucCKS_LDO_REFSEL != 0) ?
+			table_info->cac_dtp_table->ucCKS_LDO_REFSEL : 5;
+	/* Populate CKS Lookup Table */
+	if (stretch_amount == 1 || stretch_amount == 2 || stretch_amount == 5)
+		stretch_amount2 = 0;
+	else if (stretch_amount == 3 || stretch_amount == 4)
+		stretch_amount2 = 1;
+	else {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ClockStretcher);
+		PP_ASSERT_WITH_CODE(false,
+				"Stretch Amount in PPTable not supported\n",
+				return -EINVAL);
+	}
+
+	value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL);
+	value &= 0xFFFFFFFE;
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL, value);
+
+	return 0;
+}
+
+static bool vegam_is_hw_avfs_present(struct pp_hwmgr *hwmgr)
+{
+	uint32_t efuse;
+
+	efuse = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixSMU_EFUSE_0 + (49 * 4));
+	efuse &= 0x00000001;
+
+	if (efuse)
+		return true;
+
+	return false;
+}
+
+static int vegam_populate_avfs_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+
+	SMU75_Discrete_DpmTable  *table = &(smu_data->smc_state_table);
+	int result = 0;
+	struct pp_atom_ctrl__avfs_parameters avfs_params = {0};
+	AVFS_meanNsigma_t AVFS_meanNsigma = { {0} };
+	AVFS_Sclk_Offset_t AVFS_SclkOffset = { {0} };
+	uint32_t tmp, i;
+
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)hwmgr->pptable;
+	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
+			table_info->vdd_dep_on_sclk;
+
+	if (!hwmgr->avfs_supported)
+		return 0;
+
+	result = atomctrl_get_avfs_information(hwmgr, &avfs_params);
+
+	if (0 == result) {
+		table->BTCGB_VDROOP_TABLE[0].a0 =
+				PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a0);
+		table->BTCGB_VDROOP_TABLE[0].a1 =
+				PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a1);
+		table->BTCGB_VDROOP_TABLE[0].a2 =
+				PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a2);
+		table->BTCGB_VDROOP_TABLE[1].a0 =
+				PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a0);
+		table->BTCGB_VDROOP_TABLE[1].a1 =
+				PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a1);
+		table->BTCGB_VDROOP_TABLE[1].a2 =
+				PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a2);
+		table->AVFSGB_FUSE_TABLE[0].m1 =
+				PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSON_m1);
+		table->AVFSGB_FUSE_TABLE[0].m2 =
+				PP_HOST_TO_SMC_US(avfs_params.usAVFSGB_FUSE_TABLE_CKSON_m2);
+		table->AVFSGB_FUSE_TABLE[0].b =
+				PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSON_b);
+		table->AVFSGB_FUSE_TABLE[0].m1_shift = 24;
+		table->AVFSGB_FUSE_TABLE[0].m2_shift = 12;
+		table->AVFSGB_FUSE_TABLE[1].m1 =
+				PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSOFF_m1);
+		table->AVFSGB_FUSE_TABLE[1].m2 =
+				PP_HOST_TO_SMC_US(avfs_params.usAVFSGB_FUSE_TABLE_CKSOFF_m2);
+		table->AVFSGB_FUSE_TABLE[1].b =
+				PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSOFF_b);
+		table->AVFSGB_FUSE_TABLE[1].m1_shift = 24;
+		table->AVFSGB_FUSE_TABLE[1].m2_shift = 12;
+		table->MaxVoltage = PP_HOST_TO_SMC_US(avfs_params.usMaxVoltage_0_25mv);
+		AVFS_meanNsigma.Aconstant[0] =
+				PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant0);
+		AVFS_meanNsigma.Aconstant[1] =
+				PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant1);
+		AVFS_meanNsigma.Aconstant[2] =
+				PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant2);
+		AVFS_meanNsigma.DC_tol_sigma =
+				PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_DC_tol_sigma);
+		AVFS_meanNsigma.Platform_mean =
+				PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_Platform_mean);
+		AVFS_meanNsigma.PSM_Age_CompFactor =
+				PP_HOST_TO_SMC_US(avfs_params.usPSM_Age_ComFactor);
+		AVFS_meanNsigma.Platform_sigma =
+				PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_Platform_sigma);
+
+		for (i = 0; i < sclk_table->count; i++) {
+			AVFS_meanNsigma.Static_Voltage_Offset[i] =
+					(uint8_t)(sclk_table->entries[i].cks_voffset * 100 / 625);
+			AVFS_SclkOffset.Sclk_Offset[i] =
+					PP_HOST_TO_SMC_US((uint16_t)
+							(sclk_table->entries[i].sclk_offset) / 100);
+		}
+
+		result = smu7_read_smc_sram_dword(hwmgr,
+				SMU7_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU75_Firmware_Header, AvfsMeanNSigma),
+				&tmp, SMC_RAM_END);
+		smu7_copy_bytes_to_smc(hwmgr,
+					tmp,
+					(uint8_t *)&AVFS_meanNsigma,
+					sizeof(AVFS_meanNsigma_t),
+					SMC_RAM_END);
+
+		result = smu7_read_smc_sram_dword(hwmgr,
+				SMU7_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU75_Firmware_Header, AvfsSclkOffsetTable),
+				&tmp, SMC_RAM_END);
+		smu7_copy_bytes_to_smc(hwmgr,
+					tmp,
+					(uint8_t *)&AVFS_SclkOffset,
+					sizeof(AVFS_Sclk_Offset_t),
+					SMC_RAM_END);
+
+		data->avfs_vdroop_override_setting =
+				(avfs_params.ucEnableGB_VDROOP_TABLE_CKSON << BTCGB0_Vdroop_Enable_SHIFT) |
+				(avfs_params.ucEnableGB_VDROOP_TABLE_CKSOFF << BTCGB1_Vdroop_Enable_SHIFT) |
+				(avfs_params.ucEnableGB_FUSE_TABLE_CKSON << AVFSGB0_Vdroop_Enable_SHIFT) |
+				(avfs_params.ucEnableGB_FUSE_TABLE_CKSOFF << AVFSGB1_Vdroop_Enable_SHIFT);
+		data->apply_avfs_cks_off_voltage =
+				(avfs_params.ucEnableApplyAVFS_CKS_OFF_Voltage == 1) ? true : false;
+	}
+	return result;
+}
+
+static int vegam_populate_vr_config(struct pp_hwmgr *hwmgr,
+		struct SMU75_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct vegam_smumgr *smu_data =
+			(struct vegam_smumgr *)(hwmgr->smu_backend);
+	uint16_t config;
+
+	config = VR_MERGED_WITH_VDDC;
+	table->VRConfig |= (config << VRCONF_VDDGFX_SHIFT);
+
+	/* Set Vddc Voltage Controller */
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+		config = VR_SVI2_PLANE_1;
+		table->VRConfig |= config;
+	} else {
+		PP_ASSERT_WITH_CODE(false,
+				"VDDC should be on SVI2 control in merged mode!",
+				);
+	}
+	/* Set Vddci Voltage Controller */
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
+		config = VR_SVI2_PLANE_2;  /* only in merged mode */
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	} else if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
+		config = VR_SMIO_PATTERN_1;
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	} else {
+		config = VR_STATIC_VOLTAGE;
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	}
+	/* Set Mvdd Voltage Controller */
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) {
+		if (config != VR_SVI2_PLANE_2) {
+			config = VR_SVI2_PLANE_2;
+			table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
+			cgs_write_ind_register(hwmgr->device,
+					CGS_IND_REG__SMC,
+					smu_data->smu7_data.soft_regs_start +
+					offsetof(SMU75_SoftRegisters, AllowMvddSwitch),
+					0x1);
+		} else {
+			PP_ASSERT_WITH_CODE(false,
+					"SVI2 Plane 2 is already taken, set MVDD as Static",);
+			config = VR_STATIC_VOLTAGE;
+			table->VRConfig = (config << VRCONF_MVDD_SHIFT);
+		}
+	} else if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		config = VR_SMIO_PATTERN_2;
+		table->VRConfig = (config << VRCONF_MVDD_SHIFT);
+		cgs_write_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC,
+				smu_data->smu7_data.soft_regs_start +
+				offsetof(SMU75_SoftRegisters, AllowMvddSwitch),
+				0x1);
+	} else {
+		config = VR_STATIC_VOLTAGE;
+		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
+	}
+
+	return 0;
+}
+
+static int vegam_populate_svi_load_line(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	const struct vegam_pt_defaults *defaults = smu_data->power_tune_defaults;
+
+	smu_data->power_tune_table.SviLoadLineEn = defaults->SviLoadLineEn;
+	smu_data->power_tune_table.SviLoadLineVddC = defaults->SviLoadLineVddC;
+	smu_data->power_tune_table.SviLoadLineTrimVddC = 3;
+	smu_data->power_tune_table.SviLoadLineOffsetVddC = 0;
+
+	return 0;
+}
+
+static int vegam_populate_tdc_limit(struct pp_hwmgr *hwmgr)
+{
+	uint16_t tdc_limit;
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	const struct vegam_pt_defaults *defaults = smu_data->power_tune_defaults;
+
+	tdc_limit = (uint16_t)(table_info->cac_dtp_table->usTDC * 128);
+	smu_data->power_tune_table.TDC_VDDC_PkgLimit =
+			CONVERT_FROM_HOST_TO_SMC_US(tdc_limit);
+	smu_data->power_tune_table.TDC_VDDC_ThrottleReleaseLimitPerc =
+			defaults->TDC_VDDC_ThrottleReleaseLimitPerc;
+	smu_data->power_tune_table.TDC_MAWt = defaults->TDC_MAWt;
+
+	return 0;
+}
+
+static int vegam_populate_dw8(struct pp_hwmgr *hwmgr, uint32_t fuse_table_offset)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	const struct vegam_pt_defaults *defaults = smu_data->power_tune_defaults;
+	uint32_t temp;
+
+	if (smu7_read_smc_sram_dword(hwmgr,
+			fuse_table_offset +
+			offsetof(SMU75_Discrete_PmFuses, TdcWaterfallCtl),
+			(uint32_t *)&temp, SMC_RAM_END))
+		PP_ASSERT_WITH_CODE(false,
+				"Attempt to read PmFuses.DW6 (SviLoadLineEn) from SMC Failed!",
+				return -EINVAL);
+	else {
+		smu_data->power_tune_table.TdcWaterfallCtl = defaults->TdcWaterfallCtl;
+		smu_data->power_tune_table.LPMLTemperatureMin =
+				(uint8_t)((temp >> 16) & 0xff);
+		smu_data->power_tune_table.LPMLTemperatureMax =
+				(uint8_t)((temp >> 8) & 0xff);
+		smu_data->power_tune_table.Reserved = (uint8_t)(temp & 0xff);
+	}
+	return 0;
+}
+
+static int vegam_populate_temperature_scaler(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+
+	/* Currently not used. Set all to zero. */
+	for (i = 0; i < 16; i++)
+		smu_data->power_tune_table.LPMLTemperatureScaler[i] = 0;
+
+	return 0;
+}
+
+static int vegam_populate_fuzzy_fan(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+
+/* TO DO move to hwmgr */
+	if ((hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity & (1 << 15))
+		|| 0 == hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity)
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity =
+			hwmgr->thermal_controller.advanceFanControlParameters.usDefaultFanOutputSensitivity;
+
+	smu_data->power_tune_table.FuzzyFan_PwmSetDelta = PP_HOST_TO_SMC_US(
+				hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity);
+	return 0;
+}
+
+static int vegam_populate_gnb_lpml(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+
+	/* Currently not used. Set all to zero. */
+	for (i = 0; i < 16; i++)
+		smu_data->power_tune_table.GnbLPML[i] = 0;
+
+	return 0;
+}
+
+static int vegam_populate_bapm_vddc_base_leakage_sidd(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint16_t hi_sidd = smu_data->power_tune_table.BapmVddCBaseLeakageHiSidd;
+	uint16_t lo_sidd = smu_data->power_tune_table.BapmVddCBaseLeakageLoSidd;
+	struct phm_cac_tdp_table *cac_table = table_info->cac_dtp_table;
+
+	hi_sidd = (uint16_t)(cac_table->usHighCACLeakage / 100 * 256);
+	lo_sidd = (uint16_t)(cac_table->usLowCACLeakage / 100 * 256);
+
+	smu_data->power_tune_table.BapmVddCBaseLeakageHiSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(hi_sidd);
+	smu_data->power_tune_table.BapmVddCBaseLeakageLoSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(lo_sidd);
+
+	return 0;
+}
+
+static int vegam_populate_pm_fuses(struct pp_hwmgr *hwmgr)
+{
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+	uint32_t pm_fuse_table_offset;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PowerContainment)) {
+		if (smu7_read_smc_sram_dword(hwmgr,
+				SMU7_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU75_Firmware_Header, PmFuseTable),
+				&pm_fuse_table_offset, SMC_RAM_END))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to get pm_fuse_table_offset Failed!",
+					return -EINVAL);
+
+		if (vegam_populate_svi_load_line(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate SviLoadLine Failed!",
+					return -EINVAL);
+
+		if (vegam_populate_tdc_limit(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate TDCLimit Failed!", return -EINVAL);
+
+		if (vegam_populate_dw8(hwmgr, pm_fuse_table_offset))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate TdcWaterfallCtl, "
+					"LPMLTemperature Min and Max Failed!",
+					return -EINVAL);
+
+		if (0 != vegam_populate_temperature_scaler(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate LPMLTemperatureScaler Failed!",
+					return -EINVAL);
+
+		if (vegam_populate_fuzzy_fan(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate Fuzzy Fan Control parameters Failed!",
+					return -EINVAL);
+
+		if (vegam_populate_gnb_lpml(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate GnbLPML Failed!",
+					return -EINVAL);
+
+		if (vegam_populate_bapm_vddc_base_leakage_sidd(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate BapmVddCBaseLeakage Hi and Lo "
+					"Sidd Failed!", return -EINVAL);
+
+		if (smu7_copy_bytes_to_smc(hwmgr, pm_fuse_table_offset,
+				(uint8_t *)&smu_data->power_tune_table,
+				(sizeof(struct SMU75_Discrete_PmFuses) - PMFUSES_AVFSSIZE),
+				SMC_RAM_END))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to download PmFuseTable Failed!",
+					return -EINVAL);
+	}
+	return 0;
+}
+
+static int vegam_enable_reconfig_cus(struct pp_hwmgr *hwmgr)
+{
+	struct amdgpu_device *adev = hwmgr->adev;
+
+	smum_send_msg_to_smc_with_parameter(hwmgr,
+					    PPSMC_MSG_EnableModeSwitchRLCNotification,
+					    adev->gfx.cu_info.number);
+
+	return 0;
+}
+
+static int vegam_init_smc_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct smu7_hwmgr *hw_data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct vegam_smumgr *smu_data = (struct vegam_smumgr *)(hwmgr->smu_backend);
+
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct SMU75_Discrete_DpmTable *table = &(smu_data->smc_state_table);
+	uint8_t i;
+	struct pp_atomctrl_gpio_pin_assignment gpio_pin;
+	struct phm_ppt_v1_gpio_table *gpio_table =
+			(struct phm_ppt_v1_gpio_table *)table_info->gpio_table;
+	pp_atomctrl_clock_dividers_vi dividers;
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition);
+
+	vegam_initialize_power_tune_defaults(hwmgr);
+
+	if (SMU7_VOLTAGE_CONTROL_NONE != hw_data->voltage_control)
+		vegam_populate_smc_voltage_tables(hwmgr, table);
+
+	table->SystemFlags = 0;
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StepVddc))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+	if (hw_data->is_memory_gddr5)
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+	if (hw_data->ulv_supported && table_info->us_ulv_voltage_offset) {
+		result = vegam_populate_ulv_state(hwmgr, table);
+		PP_ASSERT_WITH_CODE(!result,
+				"Failed to initialize ULV state!", return result);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixCG_ULV_PARAMETER, SMU7_CGULVPARAMETER_DFLT);
+	}
+
+	result = vegam_populate_smc_link_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to initialize Link Level!", return result);
+
+	result = vegam_populate_all_graphic_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to initialize Graphics Level!", return result);
+
+	result = vegam_populate_all_memory_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to initialize Memory Level!", return result);
+
+	result = vegam_populate_smc_acpi_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to initialize ACPI Level!", return result);
+
+	result = vegam_populate_smc_vce_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to initialize VCE Level!", return result);
+
+	result = vegam_populate_smc_samu_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to initialize SAMU Level!", return result);
+
+	/* Since only the initial state is completely set up at this point
+	 * (the other states are just copies of the boot state) we only
+	 * need to populate the  ARB settings for the initial state.
+	 */
+	result = vegam_program_memory_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to Write ARB settings for the initial state.", return result);
+
+	result = vegam_populate_smc_uvd_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to initialize UVD Level!", return result);
+
+	result = vegam_populate_smc_boot_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to initialize Boot Level!", return result);
+
+	result = vegam_populate_smc_initial_state(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to initialize Boot State!", return result);
+
+	result = vegam_populate_bapm_parameters_in_dpm_table(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to populate BAPM Parameters!", return result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ClockStretcher)) {
+		result = vegam_populate_clock_stretcher_data_table(hwmgr);
+		PP_ASSERT_WITH_CODE(!result,
+				"Failed to populate Clock Stretcher Data Table!",
+				return result);
+	}
+
+	result = vegam_populate_avfs_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to populate AVFS Parameters!", return result;);
+
+	table->CurrSclkPllRange = 0xff;
+	table->GraphicsVoltageChangeEnable  = 1;
+	table->GraphicsThermThrottleEnable  = 1;
+	table->GraphicsInterval = 1;
+	table->VoltageInterval  = 1;
+	table->ThermalInterval  = 1;
+	table->TemperatureLimitHigh =
+			table_info->cac_dtp_table->usTargetOperatingTemp *
+			SMU7_Q88_FORMAT_CONVERSION_UNIT;
+	table->TemperatureLimitLow  =
+			(table_info->cac_dtp_table->usTargetOperatingTemp - 1) *
+			SMU7_Q88_FORMAT_CONVERSION_UNIT;
+	table->MemoryVoltageChangeEnable = 1;
+	table->MemoryInterval = 1;
+	table->VoltageResponseTime = 0;
+	table->PhaseResponseTime = 0;
+	table->MemoryThermThrottleEnable = 1;
+
+	PP_ASSERT_WITH_CODE(hw_data->dpm_table.pcie_speed_table.count >= 1,
+			"There must be 1 or more PCIE levels defined in PPTable.",
+			return -EINVAL);
+	table->PCIeBootLinkLevel =
+			hw_data->dpm_table.pcie_speed_table.count;
+	table->PCIeGenInterval = 1;
+	table->VRConfig = 0;
+
+	result = vegam_populate_vr_config(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to populate VRConfig setting!", return result);
+
+	table->ThermGpio = 17;
+	table->SclkStepSize = 0x4000;
+
+	if (atomctrl_get_pp_assign_pin(hwmgr,
+			VDDC_VRHOT_GPIO_PINID, &gpio_pin)) {
+		table->VRHotGpio = gpio_pin.uc_gpio_pin_bit_shift;
+		if (gpio_table)
+			table->VRHotLevel =
+					table_info->gpio_table->vrhot_triggered_sclk_dpm_index;
+	} else {
+		table->VRHotGpio = SMU7_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_RegulatorHot);
+	}
+
+	if (atomctrl_get_pp_assign_pin(hwmgr,
+			PP_AC_DC_SWITCH_GPIO_PINID,	&gpio_pin)) {
+		table->AcDcGpio = gpio_pin.uc_gpio_pin_bit_shift;
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_AutomaticDCTransition) &&
+				!smum_send_msg_to_smc(hwmgr, PPSMC_MSG_UseNewGPIOScheme))
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_SMCtoPPLIBAcdcGpioScheme);
+	} else {
+		table->AcDcGpio = SMU7_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_AutomaticDCTransition);
+	}
+
+	/* Thermal Output GPIO */
+	if (atomctrl_get_pp_assign_pin(hwmgr,
+			THERMAL_INT_OUTPUT_GPIO_PINID, &gpio_pin)) {
+		table->ThermOutGpio = gpio_pin.uc_gpio_pin_bit_shift;
+
+		/* For porlarity read GPIOPAD_A with assigned Gpio pin
+		 * since VBIOS will program this register to set 'inactive state',
+		 * driver can then determine 'active state' from this and
+		 * program SMU with correct polarity
+		 */
+		table->ThermOutPolarity =
+				(0 == (cgs_read_register(hwmgr->device, mmGPIOPAD_A) &
+				(1 << gpio_pin.uc_gpio_pin_bit_shift))) ? 1:0;
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_ONLY;
+
+		/* if required, combine VRHot/PCC with thermal out GPIO */
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_RegulatorHot) &&
+			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_CombinePCCWithThermalSignal))
+			table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_VRHOT;
+	} else {
+		table->ThermOutGpio = 17;
+		table->ThermOutPolarity = 1;
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_DISABLE;
+	}
+
+	/* Populate BIF_SCLK levels into SMC DPM table */
+	for (i = 0; i <= hw_data->dpm_table.pcie_speed_table.count; i++) {
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				smu_data->bif_sclk_table[i], &dividers);
+		PP_ASSERT_WITH_CODE(!result,
+				"Can not find DFS divide id for Sclk",
+				return result);
+
+		if (i == 0)
+			table->Ulv.BifSclkDfs =
+					PP_HOST_TO_SMC_US((uint16_t)(dividers.pll_post_divider));
+		else
+			table->LinkLevel[i - 1].BifSclkDfs =
+					PP_HOST_TO_SMC_US((uint16_t)(dividers.pll_post_divider));
+	}
+
+	for (i = 0; i < SMU75_MAX_ENTRIES_SMIO; i++)
+		table->Smio[i] = PP_HOST_TO_SMC_UL(table->Smio[i]);
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SystemFlags);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->VRConfig);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask1);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SclkStepSize);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->CurrSclkPllRange);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitHigh);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitLow);
+	CONVERT_FROM_HOST_TO_SMC_US(table->VoltageResponseTime);
+	CONVERT_FROM_HOST_TO_SMC_US(table->PhaseResponseTime);
+
+	/* Upload all dpm data to SMC memory.(dpm level, dpm level count etc) */
+	result = smu7_copy_bytes_to_smc(hwmgr,
+			smu_data->smu7_data.dpm_table_start +
+			offsetof(SMU75_Discrete_DpmTable, SystemFlags),
+			(uint8_t *)&(table->SystemFlags),
+			sizeof(SMU75_Discrete_DpmTable) - 3 * sizeof(SMU75_PIDController),
+			SMC_RAM_END);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to upload dpm data to SMC memory!", return result);
+
+	result = vegam_populate_pm_fuses(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to  populate PM fuses to SMC memory!", return result);
+
+	result = vegam_enable_reconfig_cus(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to enable reconfigurable CUs!", return result);
+
+	return 0;
+}
+
+static uint32_t vegam_get_offsetof(uint32_t type, uint32_t member)
+{
+	switch (type) {
+	case SMU_SoftRegisters:
+		switch (member) {
+		case HandshakeDisables:
+			return offsetof(SMU75_SoftRegisters, HandshakeDisables);
+		case VoltageChangeTimeout:
+			return offsetof(SMU75_SoftRegisters, VoltageChangeTimeout);
+		case AverageGraphicsActivity:
+			return offsetof(SMU75_SoftRegisters, AverageGraphicsActivity);
+		case PreVBlankGap:
+			return offsetof(SMU75_SoftRegisters, PreVBlankGap);
+		case VBlankTimeout:
+			return offsetof(SMU75_SoftRegisters, VBlankTimeout);
+		case UcodeLoadStatus:
+			return offsetof(SMU75_SoftRegisters, UcodeLoadStatus);
+		case DRAM_LOG_ADDR_H:
+			return offsetof(SMU75_SoftRegisters, DRAM_LOG_ADDR_H);
+		case DRAM_LOG_ADDR_L:
+			return offsetof(SMU75_SoftRegisters, DRAM_LOG_ADDR_L);
+		case DRAM_LOG_PHY_ADDR_H:
+			return offsetof(SMU75_SoftRegisters, DRAM_LOG_PHY_ADDR_H);
+		case DRAM_LOG_PHY_ADDR_L:
+			return offsetof(SMU75_SoftRegisters, DRAM_LOG_PHY_ADDR_L);
+		case DRAM_LOG_BUFF_SIZE:
+			return offsetof(SMU75_SoftRegisters, DRAM_LOG_BUFF_SIZE);
+		}
+	case SMU_Discrete_DpmTable:
+		switch (member) {
+		case UvdBootLevel:
+			return offsetof(SMU75_Discrete_DpmTable, UvdBootLevel);
+		case VceBootLevel:
+			return offsetof(SMU75_Discrete_DpmTable, VceBootLevel);
+		case SamuBootLevel:
+			return offsetof(SMU75_Discrete_DpmTable, SamuBootLevel);
+		case LowSclkInterruptThreshold:
+			return offsetof(SMU75_Discrete_DpmTable, LowSclkInterruptThreshold);
+		}
+	}
+	pr_warn("can't get the offset of type %x member %x\n", type, member);
+	return 0;
+}
+
+static int vegam_program_mem_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (data->need_update_smu7_dpm_table &
+		(DPMTABLE_OD_UPDATE_SCLK +
+		DPMTABLE_UPDATE_SCLK +
+		DPMTABLE_UPDATE_MCLK))
+		return vegam_program_memory_timing_parameters(hwmgr);
+
+	return 0;
+}
+
+static int vegam_update_sclk_threshold(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct vegam_smumgr *smu_data =
+			(struct vegam_smumgr *)(hwmgr->smu_backend);
+	int result = 0;
+	uint32_t low_sclk_interrupt_threshold = 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkThrottleLowNotification)
+	    && (data->low_sclk_interrupt_threshold != 0)) {
+		low_sclk_interrupt_threshold =
+				data->low_sclk_interrupt_threshold;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(low_sclk_interrupt_threshold);
+
+		result = smu7_copy_bytes_to_smc(
+				hwmgr,
+				smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU75_Discrete_DpmTable,
+					LowSclkInterruptThreshold),
+				(uint8_t *)&low_sclk_interrupt_threshold,
+				sizeof(uint32_t),
+				SMC_RAM_END);
+	}
+	PP_ASSERT_WITH_CODE((result == 0),
+			"Failed to update SCLK threshold!", return result);
+
+	result = vegam_program_mem_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE((result == 0),
+			"Failed to program memory timing parameters!",
+			);
+
+	return result;
+}
+
+int vegam_thermal_avfs_enable(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	int ret;
+
+	if (!hwmgr->avfs_supported)
+		return 0;
+
+	ret = smum_send_msg_to_smc(hwmgr, PPSMC_MSG_EnableAvfs);
+	if (!ret) {
+		if (data->apply_avfs_cks_off_voltage)
+			ret = smum_send_msg_to_smc(hwmgr, PPSMC_MSG_ApplyAvfsCksOffVoltage);
+	}
+
+	return ret;
+}
+
+static int vegam_thermal_setup_fan_table(struct pp_hwmgr *hwmgr)
+{
+	PP_ASSERT_WITH_CODE(hwmgr->thermal_controller.fanInfo.bNoFan,
+			"VBIOS fan info is not correct!",
+			);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MicrocodeFanControl);
+	return 0;
+}
+
+const struct pp_smumgr_func vegam_smu_funcs = {
+	.smu_init = vegam_smu_init,
+	.smu_fini = smu7_smu_fini,
+	.start_smu = vegam_start_smu,
+	.check_fw_load_finish = smu7_check_fw_load_finish,
+	.request_smu_load_fw = smu7_reload_firmware,
+	.request_smu_load_specific_fw = NULL,
+	.send_msg_to_smc = smu7_send_msg_to_smc,
+	.send_msg_to_smc_with_parameter = smu7_send_msg_to_smc_with_parameter,
+	.process_firmware_header = vegam_process_firmware_header,
+	.is_dpm_running = vegam_is_dpm_running,
+	.get_mac_definition = vegam_get_mac_definition,
+	.update_smc_table = vegam_update_smc_table,
+	.init_smc_table = vegam_init_smc_table,
+	.get_offsetof = vegam_get_offsetof,
+	.populate_all_graphic_levels = vegam_populate_all_graphic_levels,
+	.populate_all_memory_levels = vegam_populate_all_memory_levels,
+	.update_sclk_threshold = vegam_update_sclk_threshold,
+	.is_hw_avfs_present = vegam_is_hw_avfs_present,
+	.thermal_avfs_enable = vegam_thermal_avfs_enable,
+	.thermal_setup_fan_table = vegam_thermal_setup_fan_table,
+};
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.h
new file mode 100644
index 000000000000..2b6558238500
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _VEGAM_SMUMANAGER_H
+#define _VEGAM_SMUMANAGER_H
+
+
+#include <pp_endian.h>
+#include "smu75_discrete.h"
+#include "smu7_smumgr.h"
+
+#define SMC_RAM_END 0x40000
+
+#define DPMTuning_Uphyst_Shift    0
+#define DPMTuning_Downhyst_Shift  8
+#define DPMTuning_Activity_Shift  16
+
+#define GraphicsDPMTuning_VEGAM    0x001e6400
+#define MemoryDPMTuning_VEGAM      0x000f3c0a
+#define SclkDPMTuning_VEGAM        0x002d000a
+#define MclkDPMTuning_VEGAM        0x001f100a
+
+
+struct vegam_pt_defaults {
+	uint8_t   SviLoadLineEn;
+	uint8_t   SviLoadLineVddC;
+	uint8_t   TDC_VDDC_ThrottleReleaseLimitPerc;
+	uint8_t   TDC_MAWt;
+	uint8_t   TdcWaterfallCtl;
+	uint8_t   DTEAmbientTempBase;
+
+	uint32_t  DisplayCac;
+	uint32_t  BAPM_TEMP_GRADIENT;
+	uint16_t  BAPMTI_R[SMU75_DTE_ITERATIONS * SMU75_DTE_SOURCES * SMU75_DTE_SINKS];
+	uint16_t  BAPMTI_RC[SMU75_DTE_ITERATIONS * SMU75_DTE_SOURCES * SMU75_DTE_SINKS];
+};
+
+struct vegam_range_table {
+	uint32_t trans_lower_frequency; /* in 10khz */
+	uint32_t trans_upper_frequency;
+};
+
+struct vegam_smumgr {
+	struct smu7_smumgr smu7_data;
+	uint8_t protected_mode;
+	SMU75_Discrete_DpmTable              smc_state_table;
+	struct SMU75_Discrete_Ulv            ulv_setting;
+	struct SMU75_Discrete_PmFuses  power_tune_table;
+	struct vegam_range_table                range_table[NUM_SCLK_RANGE];
+	const struct vegam_pt_defaults       *power_tune_defaults;
+	uint32_t               bif_sclk_table[SMU75_MAX_LEVELS_LINK];
+};
+
+
+#endif
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index 831b73392d82..036dff8a1f33 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -799,7 +799,7 @@ static int ast_get_modes(struct drm_connector *connector)
 	return 0;
 }
 
-static int ast_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status ast_mode_valid(struct drm_connector *connector,
 			  struct drm_display_mode *mode)
 {
 	struct ast_private *ast = connector->dev->dev_private;
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
index ab32d5b268d2..60c937f42114 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
@@ -299,7 +299,6 @@ struct atmel_hlcdc_layer {
 struct atmel_hlcdc_plane {
 	struct drm_plane base;
 	struct atmel_hlcdc_layer layer;
-	struct atmel_hlcdc_plane_properties *properties;
 };
 
 static inline struct atmel_hlcdc_plane *
@@ -345,18 +344,6 @@ struct atmel_hlcdc_dc_desc {
 	int nlayers;
 };
 
-/**
- * Atmel HLCDC Plane properties.
- *
- * This structure stores plane property definitions.
- *
- * @alpha: alpha blending (or transparency) property
- * @rotation: rotation property
- */
-struct atmel_hlcdc_plane_properties {
-	struct drm_property *alpha;
-};
-
 /**
  * Atmel HLCDC Display Controller.
  *
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
index e18800ed7cd1..73c875db45f4 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
@@ -31,7 +31,6 @@
  * @src_y: y buffer position
  * @src_w: buffer width
  * @src_h: buffer height
- * @alpha: alpha blending of the plane
  * @disc_x: x discard position
  * @disc_y: y discard position
  * @disc_w: discard width
@@ -54,8 +53,6 @@ struct atmel_hlcdc_plane_state {
 	uint32_t src_w;
 	uint32_t src_h;
 
-	u8 alpha;
-
 	int disc_x;
 	int disc_y;
 	int disc_w;
@@ -385,7 +382,7 @@ atmel_hlcdc_plane_update_general_settings(struct atmel_hlcdc_plane *plane,
 			cfg |= ATMEL_HLCDC_LAYER_LAEN;
 		else
 			cfg |= ATMEL_HLCDC_LAYER_GAEN |
-			       ATMEL_HLCDC_LAYER_GA(state->alpha);
+			       ATMEL_HLCDC_LAYER_GA(state->base.alpha >> 8);
 	}
 
 	if (state->disc_h && state->disc_w)
@@ -553,7 +550,7 @@ atmel_hlcdc_plane_prepare_disc_area(struct drm_crtc_state *c_state)
 
 		if (!ovl_s->fb ||
 		    ovl_s->fb->format->has_alpha ||
-		    ovl_state->alpha != 255)
+		    ovl_s->alpha != DRM_BLEND_ALPHA_OPAQUE)
 			continue;
 
 		/* TODO: implement a smarter hidden area detection */
@@ -829,51 +826,18 @@ static void atmel_hlcdc_plane_destroy(struct drm_plane *p)
 	drm_plane_cleanup(p);
 }
 
-static int atmel_hlcdc_plane_atomic_set_property(struct drm_plane *p,
-						 struct drm_plane_state *s,
-						 struct drm_property *property,
-						 uint64_t val)
-{
-	struct atmel_hlcdc_plane *plane = drm_plane_to_atmel_hlcdc_plane(p);
-	struct atmel_hlcdc_plane_properties *props = plane->properties;
-	struct atmel_hlcdc_plane_state *state =
-			drm_plane_state_to_atmel_hlcdc_plane_state(s);
-
-	if (property == props->alpha)
-		state->alpha = val;
-	else
-		return -EINVAL;
-
-	return 0;
-}
-
-static int atmel_hlcdc_plane_atomic_get_property(struct drm_plane *p,
-					const struct drm_plane_state *s,
-					struct drm_property *property,
-					uint64_t *val)
-{
-	struct atmel_hlcdc_plane *plane = drm_plane_to_atmel_hlcdc_plane(p);
-	struct atmel_hlcdc_plane_properties *props = plane->properties;
-	const struct atmel_hlcdc_plane_state *state =
-		container_of(s, const struct atmel_hlcdc_plane_state, base);
-
-	if (property == props->alpha)
-		*val = state->alpha;
-	else
-		return -EINVAL;
-
-	return 0;
-}
-
-static int atmel_hlcdc_plane_init_properties(struct atmel_hlcdc_plane *plane,
-				struct atmel_hlcdc_plane_properties *props)
+static int atmel_hlcdc_plane_init_properties(struct atmel_hlcdc_plane *plane)
 {
 	const struct atmel_hlcdc_layer_desc *desc = plane->layer.desc;
 
 	if (desc->type == ATMEL_HLCDC_OVERLAY_LAYER ||
-	    desc->type == ATMEL_HLCDC_CURSOR_LAYER)
-		drm_object_attach_property(&plane->base.base,
-					   props->alpha, 255);
+	    desc->type == ATMEL_HLCDC_CURSOR_LAYER) {
+		int ret;
+
+		ret = drm_plane_create_alpha_property(&plane->base);
+		if (ret)
+			return ret;
+	}
 
 	if (desc->layout.xstride && desc->layout.pstride) {
 		int ret;
@@ -988,8 +952,8 @@ static void atmel_hlcdc_plane_reset(struct drm_plane *p)
 			return;
 		}
 
-		state->alpha = 255;
 		p->state = &state->base;
+		p->state->alpha = DRM_BLEND_ALPHA_OPAQUE;
 		p->state->plane = p;
 	}
 }
@@ -1042,13 +1006,10 @@ static const struct drm_plane_funcs layer_plane_funcs = {
 	.reset = atmel_hlcdc_plane_reset,
 	.atomic_duplicate_state = atmel_hlcdc_plane_atomic_duplicate_state,
 	.atomic_destroy_state = atmel_hlcdc_plane_atomic_destroy_state,
-	.atomic_set_property = atmel_hlcdc_plane_atomic_set_property,
-	.atomic_get_property = atmel_hlcdc_plane_atomic_get_property,
 };
 
 static int atmel_hlcdc_plane_create(struct drm_device *dev,
-				    const struct atmel_hlcdc_layer_desc *desc,
-				    struct atmel_hlcdc_plane_properties *props)
+				    const struct atmel_hlcdc_layer_desc *desc)
 {
 	struct atmel_hlcdc_dc *dc = dev->dev_private;
 	struct atmel_hlcdc_plane *plane;
@@ -1060,7 +1021,6 @@ static int atmel_hlcdc_plane_create(struct drm_device *dev,
 		return -ENOMEM;
 
 	atmel_hlcdc_layer_init(&plane->layer, desc, dc->hlcdc->regmap);
-	plane->properties = props;
 
 	if (desc->type == ATMEL_HLCDC_BASE_LAYER)
 		type = DRM_PLANE_TYPE_PRIMARY;
@@ -1081,7 +1041,7 @@ static int atmel_hlcdc_plane_create(struct drm_device *dev,
 			     &atmel_hlcdc_layer_plane_helper_funcs);
 
 	/* Set default property values*/
-	ret = atmel_hlcdc_plane_init_properties(plane, props);
+	ret = atmel_hlcdc_plane_init_properties(plane);
 	if (ret)
 		return ret;
 
@@ -1090,34 +1050,13 @@ static int atmel_hlcdc_plane_create(struct drm_device *dev,
 	return 0;
 }
 
-static struct atmel_hlcdc_plane_properties *
-atmel_hlcdc_plane_create_properties(struct drm_device *dev)
-{
-	struct atmel_hlcdc_plane_properties *props;
-
-	props = devm_kzalloc(dev->dev, sizeof(*props), GFP_KERNEL);
-	if (!props)
-		return ERR_PTR(-ENOMEM);
-
-	props->alpha = drm_property_create_range(dev, 0, "alpha", 0, 255);
-	if (!props->alpha)
-		return ERR_PTR(-ENOMEM);
-
-	return props;
-}
-
 int atmel_hlcdc_create_planes(struct drm_device *dev)
 {
 	struct atmel_hlcdc_dc *dc = dev->dev_private;
-	struct atmel_hlcdc_plane_properties *props;
 	const struct atmel_hlcdc_layer_desc *descs = dc->desc->layers;
 	int nlayers = dc->desc->nlayers;
 	int i, ret;
 
-	props = atmel_hlcdc_plane_create_properties(dev);
-	if (IS_ERR(props))
-		return PTR_ERR(props);
-
 	dc->dscrpool = dmam_pool_create("atmel-hlcdc-dscr", dev->dev,
 				sizeof(struct atmel_hlcdc_dma_channel_dscr),
 				sizeof(u64), 0);
@@ -1130,7 +1069,7 @@ int atmel_hlcdc_create_planes(struct drm_device *dev)
 		    descs[i].type != ATMEL_HLCDC_CURSOR_LAYER)
 			continue;
 
-		ret = atmel_hlcdc_plane_create(dev, &descs[i], props);
+		ret = atmel_hlcdc_plane_create(dev, &descs[i]);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/gpu/drm/bochs/bochs_kms.c b/drivers/gpu/drm/bochs/bochs_kms.c
index a24a18fbd65a..233980a78591 100644
--- a/drivers/gpu/drm/bochs/bochs_kms.c
+++ b/drivers/gpu/drm/bochs/bochs_kms.c
@@ -188,7 +188,7 @@ static int bochs_connector_get_modes(struct drm_connector *connector)
 	return count;
 }
 
-static int bochs_connector_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status bochs_connector_mode_valid(struct drm_connector *connector,
 				      struct drm_display_mode *mode)
 {
 	struct bochs_device *bochs =
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
index 684ac626ac53..fa2c7997e2fd 100644
--- a/drivers/gpu/drm/bridge/Kconfig
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -25,6 +25,16 @@ config DRM_ANALOGIX_ANX78XX
 	  the HDMI output of an application processor to MyDP
 	  or DisplayPort.
 
+config DRM_CDNS_DSI
+	tristate "Cadence DPI/DSI bridge"
+	select DRM_KMS_HELPER
+	select DRM_MIPI_DSI
+	select DRM_PANEL_BRIDGE
+	depends on OF
+	help
+	  Support Cadence DPI to DSI bridge. This is an internal
+	  bridge and is meant to be directly embedded in a SoC.
+
 config DRM_DUMB_VGA_DAC
 	tristate "Dumb VGA DAC Bridge support"
 	depends on OF
@@ -94,6 +104,12 @@ config DRM_SII9234
 	  It is an I2C driver, that detects connection of MHL bridge
 	  and starts encapsulation of HDMI signal.
 
+config DRM_THINE_THC63LVD1024
+	tristate "Thine THC63LVD1024 LVDS decoder bridge"
+	depends on OF
+	---help---
+	  Thine THC63LVD1024 LVDS/parallel converter driver.
+
 config DRM_TOSHIBA_TC358767
 	tristate "Toshiba TC358767 eDP bridge"
 	depends on OF
diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile
index 373eb28f31ed..35f88d48ec20 100644
--- a/drivers/gpu/drm/bridge/Makefile
+++ b/drivers/gpu/drm/bridge/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_DRM_ANALOGIX_ANX78XX) += analogix-anx78xx.o
+obj-$(CONFIG_DRM_CDNS_DSI) += cdns-dsi.o
 obj-$(CONFIG_DRM_DUMB_VGA_DAC) += dumb-vga-dac.o
 obj-$(CONFIG_DRM_LVDS_ENCODER) += lvds-encoder.o
 obj-$(CONFIG_DRM_MEGACHIPS_STDPXXXX_GE_B850V3_FW) += megachips-stdpxxxx-ge-b850v3-fw.o
@@ -8,6 +9,7 @@ obj-$(CONFIG_DRM_PARADE_PS8622) += parade-ps8622.o
 obj-$(CONFIG_DRM_SIL_SII8620) += sil-sii8620.o
 obj-$(CONFIG_DRM_SII902X) += sii902x.o
 obj-$(CONFIG_DRM_SII9234) += sii9234.o
+obj-$(CONFIG_DRM_THINE_THC63LVD1024) += thc63lvd1024.o
 obj-$(CONFIG_DRM_TOSHIBA_TC358767) += tc358767.o
 obj-$(CONFIG_DRM_ANALOGIX_DP) += analogix/
 obj-$(CONFIG_DRM_I2C_ADV7511) += adv7511/
diff --git a/drivers/gpu/drm/bridge/adv7511/Kconfig b/drivers/gpu/drm/bridge/adv7511/Kconfig
index 592b9d2ec034..944e440c4fde 100644
--- a/drivers/gpu/drm/bridge/adv7511/Kconfig
+++ b/drivers/gpu/drm/bridge/adv7511/Kconfig
@@ -1,5 +1,5 @@
 config DRM_I2C_ADV7511
-	tristate "AV7511 encoder"
+	tristate "ADV7511 encoder"
 	depends on OF
 	select DRM_KMS_HELPER
 	select REGMAP_I2C
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511.h b/drivers/gpu/drm/bridge/adv7511/adv7511.h
index d034b2cb5eee..73d8ccb97742 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511.h
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511.h
@@ -93,6 +93,11 @@
 #define ADV7511_REG_CHIP_ID_HIGH		0xf5
 #define ADV7511_REG_CHIP_ID_LOW			0xf6
 
+/* Hardware defined default addresses for I2C register maps */
+#define ADV7511_CEC_I2C_ADDR_DEFAULT		0x3c
+#define ADV7511_EDID_I2C_ADDR_DEFAULT		0x3f
+#define ADV7511_PACKET_I2C_ADDR_DEFAULT		0x38
+
 #define ADV7511_CSC_ENABLE			BIT(7)
 #define ADV7511_CSC_UPDATE_MODE			BIT(5)
 
@@ -321,6 +326,7 @@ enum adv7511_type {
 struct adv7511 {
 	struct i2c_client *i2c_main;
 	struct i2c_client *i2c_edid;
+	struct i2c_client *i2c_packet;
 	struct i2c_client *i2c_cec;
 
 	struct regmap *regmap;
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
index efa29db5fc2b..73021b388e12 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
@@ -586,7 +586,7 @@ static int adv7511_get_modes(struct adv7511 *adv7511,
 	/* Reading the EDID only works if the device is powered */
 	if (!adv7511->powered) {
 		unsigned int edid_i2c_addr =
-					(adv7511->i2c_main->addr << 1) + 4;
+					(adv7511->i2c_edid->addr << 1);
 
 		__adv7511_power_on(adv7511);
 
@@ -654,7 +654,7 @@ adv7511_detect(struct adv7511 *adv7511, struct drm_connector *connector)
 	return status;
 }
 
-static int adv7511_mode_valid(struct adv7511 *adv7511,
+static enum drm_mode_status adv7511_mode_valid(struct adv7511 *adv7511,
 			      struct drm_display_mode *mode)
 {
 	if (mode->clock > 165000)
@@ -969,10 +969,10 @@ static int adv7511_init_cec_regmap(struct adv7511 *adv)
 {
 	int ret;
 
-	adv->i2c_cec = i2c_new_dummy(adv->i2c_main->adapter,
-				     adv->i2c_main->addr - 1);
+	adv->i2c_cec = i2c_new_secondary_device(adv->i2c_main, "cec",
+						ADV7511_CEC_I2C_ADDR_DEFAULT);
 	if (!adv->i2c_cec)
-		return -ENOMEM;
+		return -EINVAL;
 	i2c_set_clientdata(adv->i2c_cec, adv);
 
 	adv->regmap_cec = devm_regmap_init_i2c(adv->i2c_cec,
@@ -1082,8 +1082,6 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
 	struct adv7511_link_config link_config;
 	struct adv7511 *adv7511;
 	struct device *dev = &i2c->dev;
-	unsigned int main_i2c_addr = i2c->addr << 1;
-	unsigned int edid_i2c_addr = main_i2c_addr + 4;
 	unsigned int val;
 	int ret;
 
@@ -1129,7 +1127,7 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
 	}
 
 	if (adv7511->gpio_pd) {
-		mdelay(5);
+		usleep_range(5000, 6000);
 		gpiod_set_value_cansleep(adv7511->gpio_pd, 0);
 	}
 
@@ -1153,23 +1151,34 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
 	if (ret)
 		goto uninit_regulators;
 
-	regmap_write(adv7511->regmap, ADV7511_REG_EDID_I2C_ADDR, edid_i2c_addr);
-	regmap_write(adv7511->regmap, ADV7511_REG_PACKET_I2C_ADDR,
-		     main_i2c_addr - 0xa);
-	regmap_write(adv7511->regmap, ADV7511_REG_CEC_I2C_ADDR,
-		     main_i2c_addr - 2);
-
 	adv7511_packet_disable(adv7511, 0xffff);
 
-	adv7511->i2c_edid = i2c_new_dummy(i2c->adapter, edid_i2c_addr >> 1);
+	adv7511->i2c_edid = i2c_new_secondary_device(i2c, "edid",
+					ADV7511_EDID_I2C_ADDR_DEFAULT);
 	if (!adv7511->i2c_edid) {
-		ret = -ENOMEM;
+		ret = -EINVAL;
 		goto uninit_regulators;
 	}
 
+	regmap_write(adv7511->regmap, ADV7511_REG_EDID_I2C_ADDR,
+		     adv7511->i2c_edid->addr << 1);
+
+	adv7511->i2c_packet = i2c_new_secondary_device(i2c, "packet",
+					ADV7511_PACKET_I2C_ADDR_DEFAULT);
+	if (!adv7511->i2c_packet) {
+		ret = -EINVAL;
+		goto err_i2c_unregister_edid;
+	}
+
+	regmap_write(adv7511->regmap, ADV7511_REG_PACKET_I2C_ADDR,
+		     adv7511->i2c_packet->addr << 1);
+
 	ret = adv7511_init_cec_regmap(adv7511);
 	if (ret)
-		goto err_i2c_unregister_edid;
+		goto err_i2c_unregister_packet;
+
+	regmap_write(adv7511->regmap, ADV7511_REG_CEC_I2C_ADDR,
+		     adv7511->i2c_cec->addr << 1);
 
 	INIT_WORK(&adv7511->hpd_work, adv7511_hpd_work);
 
@@ -1207,6 +1216,8 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
 	i2c_unregister_device(adv7511->i2c_cec);
 	if (adv7511->cec_clk)
 		clk_disable_unprepare(adv7511->cec_clk);
+err_i2c_unregister_packet:
+	i2c_unregister_device(adv7511->i2c_packet);
 err_i2c_unregister_edid:
 	i2c_unregister_device(adv7511->i2c_edid);
 uninit_regulators:
@@ -1233,6 +1244,7 @@ static int adv7511_remove(struct i2c_client *i2c)
 
 	cec_unregister_adapter(adv7511->cec_adap);
 
+	i2c_unregister_device(adv7511->i2c_packet);
 	i2c_unregister_device(adv7511->i2c_edid);
 
 	return 0;
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
index 5c52307146c7..2bcbfadb6ac5 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
@@ -43,8 +43,10 @@ struct bridge_init {
 	struct device_node *node;
 };
 
-static void analogix_dp_init_dp(struct analogix_dp_device *dp)
+static int analogix_dp_init_dp(struct analogix_dp_device *dp)
 {
+	int ret;
+
 	analogix_dp_reset(dp);
 
 	analogix_dp_swreset(dp);
@@ -56,10 +58,13 @@ static void analogix_dp_init_dp(struct analogix_dp_device *dp)
 	analogix_dp_enable_sw_function(dp);
 
 	analogix_dp_config_interrupt(dp);
-	analogix_dp_init_analog_func(dp);
+	ret = analogix_dp_init_analog_func(dp);
+	if (ret)
+		return ret;
 
 	analogix_dp_init_hpd(dp);
 	analogix_dp_init_aux(dp);
+	return 0;
 }
 
 static int analogix_dp_detect_hpd(struct analogix_dp_device *dp)
@@ -71,7 +76,7 @@ static int analogix_dp_detect_hpd(struct analogix_dp_device *dp)
 			return 0;
 
 		timeout_loop++;
-		usleep_range(10, 11);
+		usleep_range(1000, 1100);
 	}
 
 	/*
@@ -148,87 +153,146 @@ int analogix_dp_disable_psr(struct analogix_dp_device *dp)
 	psr_vsc.DB1 = 0;
 
 	ret = drm_dp_dpcd_writeb(&dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
-	if (ret != 1)
+	if (ret != 1) {
 		dev_err(dp->dev, "Failed to set DP Power0 %d\n", ret);
+		return ret;
+	}
 
 	return analogix_dp_send_psr_spd(dp, &psr_vsc, false);
 }
 EXPORT_SYMBOL_GPL(analogix_dp_disable_psr);
 
-static bool analogix_dp_detect_sink_psr(struct analogix_dp_device *dp)
+static int analogix_dp_detect_sink_psr(struct analogix_dp_device *dp)
 {
 	unsigned char psr_version;
+	int ret;
+
+	ret = drm_dp_dpcd_readb(&dp->aux, DP_PSR_SUPPORT, &psr_version);
+	if (ret != 1) {
+		dev_err(dp->dev, "failed to get PSR version, disable it\n");
+		return ret;
+	}
 
-	drm_dp_dpcd_readb(&dp->aux, DP_PSR_SUPPORT, &psr_version);
 	dev_dbg(dp->dev, "Panel PSR version : %x\n", psr_version);
 
-	return (psr_version & DP_PSR_IS_SUPPORTED) ? true : false;
+	dp->psr_enable = (psr_version & DP_PSR_IS_SUPPORTED) ? true : false;
+
+	return 0;
 }
 
-static void analogix_dp_enable_sink_psr(struct analogix_dp_device *dp)
+static int analogix_dp_enable_sink_psr(struct analogix_dp_device *dp)
 {
 	unsigned char psr_en;
+	int ret;
 
 	/* Disable psr function */
-	drm_dp_dpcd_readb(&dp->aux, DP_PSR_EN_CFG, &psr_en);
+	ret = drm_dp_dpcd_readb(&dp->aux, DP_PSR_EN_CFG, &psr_en);
+	if (ret != 1) {
+		dev_err(dp->dev, "failed to get psr config\n");
+		goto end;
+	}
+
 	psr_en &= ~DP_PSR_ENABLE;
-	drm_dp_dpcd_writeb(&dp->aux, DP_PSR_EN_CFG, psr_en);
+	ret = drm_dp_dpcd_writeb(&dp->aux, DP_PSR_EN_CFG, psr_en);
+	if (ret != 1) {
+		dev_err(dp->dev, "failed to disable panel psr\n");
+		goto end;
+	}
 
 	/* Main-Link transmitter remains active during PSR active states */
 	psr_en = DP_PSR_MAIN_LINK_ACTIVE | DP_PSR_CRC_VERIFICATION;
-	drm_dp_dpcd_writeb(&dp->aux, DP_PSR_EN_CFG, psr_en);
+	ret = drm_dp_dpcd_writeb(&dp->aux, DP_PSR_EN_CFG, psr_en);
+	if (ret != 1) {
+		dev_err(dp->dev, "failed to set panel psr\n");
+		goto end;
+	}
 
 	/* Enable psr function */
 	psr_en = DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE |
 		 DP_PSR_CRC_VERIFICATION;
-	drm_dp_dpcd_writeb(&dp->aux, DP_PSR_EN_CFG, psr_en);
+	ret = drm_dp_dpcd_writeb(&dp->aux, DP_PSR_EN_CFG, psr_en);
+	if (ret != 1) {
+		dev_err(dp->dev, "failed to set panel psr\n");
+		goto end;
+	}
 
 	analogix_dp_enable_psr_crc(dp);
+
+	return 0;
+end:
+	dev_err(dp->dev, "enable psr fail, force to disable psr\n");
+	dp->psr_enable = false;
+
+	return ret;
 }
 
-static void
+static int
 analogix_dp_enable_rx_to_enhanced_mode(struct analogix_dp_device *dp,
 				       bool enable)
 {
 	u8 data;
+	int ret;
 
-	drm_dp_dpcd_readb(&dp->aux, DP_LANE_COUNT_SET, &data);
+	ret = drm_dp_dpcd_readb(&dp->aux, DP_LANE_COUNT_SET, &data);
+	if (ret != 1)
+		return ret;
 
 	if (enable)
-		drm_dp_dpcd_writeb(&dp->aux, DP_LANE_COUNT_SET,
-				   DP_LANE_COUNT_ENHANCED_FRAME_EN |
-					DPCD_LANE_COUNT_SET(data));
+		ret = drm_dp_dpcd_writeb(&dp->aux, DP_LANE_COUNT_SET,
+					 DP_LANE_COUNT_ENHANCED_FRAME_EN |
+					 DPCD_LANE_COUNT_SET(data));
 	else
-		drm_dp_dpcd_writeb(&dp->aux, DP_LANE_COUNT_SET,
-				   DPCD_LANE_COUNT_SET(data));
+		ret = drm_dp_dpcd_writeb(&dp->aux, DP_LANE_COUNT_SET,
+					 DPCD_LANE_COUNT_SET(data));
+
+	return ret < 0 ? ret : 0;
 }
 
-static int analogix_dp_is_enhanced_mode_available(struct analogix_dp_device *dp)
+static int analogix_dp_is_enhanced_mode_available(struct analogix_dp_device *dp,
+						  u8 *enhanced_mode_support)
 {
 	u8 data;
-	int retval;
+	int ret;
 
-	drm_dp_dpcd_readb(&dp->aux, DP_MAX_LANE_COUNT, &data);
-	retval = DPCD_ENHANCED_FRAME_CAP(data);
+	ret = drm_dp_dpcd_readb(&dp->aux, DP_MAX_LANE_COUNT, &data);
+	if (ret != 1) {
+		*enhanced_mode_support = 0;
+		return ret;
+	}
 
-	return retval;
+	*enhanced_mode_support = DPCD_ENHANCED_FRAME_CAP(data);
+
+	return 0;
 }
 
-static void analogix_dp_set_enhanced_mode(struct analogix_dp_device *dp)
+static int analogix_dp_set_enhanced_mode(struct analogix_dp_device *dp)
 {
 	u8 data;
+	int ret;
+
+	ret = analogix_dp_is_enhanced_mode_available(dp, &data);
+	if (ret < 0)
+		return ret;
+
+	ret = analogix_dp_enable_rx_to_enhanced_mode(dp, data);
+	if (ret < 0)
+		return ret;
 
-	data = analogix_dp_is_enhanced_mode_available(dp);
-	analogix_dp_enable_rx_to_enhanced_mode(dp, data);
 	analogix_dp_enable_enhanced_mode(dp, data);
+
+	return 0;
 }
 
-static void analogix_dp_training_pattern_dis(struct analogix_dp_device *dp)
+static int analogix_dp_training_pattern_dis(struct analogix_dp_device *dp)
 {
+	int ret;
+
 	analogix_dp_set_training_pattern(dp, DP_NONE);
 
-	drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET,
-			   DP_TRAINING_PATTERN_DISABLE);
+	ret = drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET,
+				 DP_TRAINING_PATTERN_DISABLE);
+
+	return ret < 0 ? ret : 0;
 }
 
 static void
@@ -276,6 +340,12 @@ static int analogix_dp_link_start(struct analogix_dp_device *dp)
 	retval = drm_dp_dpcd_write(&dp->aux, DP_LINK_BW_SET, buf, 2);
 	if (retval < 0)
 		return retval;
+	/* set enhanced mode if available */
+	retval = analogix_dp_set_enhanced_mode(dp);
+	if (retval < 0) {
+		dev_err(dp->dev, "failed to set enhance mode\n");
+		return retval;
+	}
 
 	/* Set TX pre-emphasis to minimum */
 	for (lane = 0; lane < lane_count; lane++)
@@ -531,7 +601,7 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp)
 {
 	int lane, lane_count, retval;
 	u32 reg;
-	u8 link_align, link_status[2], adjust_request[2], spread;
+	u8 link_align, link_status[2], adjust_request[2];
 
 	usleep_range(400, 401);
 
@@ -560,10 +630,11 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp)
 
 	if (!analogix_dp_channel_eq_ok(link_status, link_align, lane_count)) {
 		/* traing pattern Set to Normal */
-		analogix_dp_training_pattern_dis(dp);
+		retval = analogix_dp_training_pattern_dis(dp);
+		if (retval < 0)
+			return retval;
 
 		dev_info(dp->dev, "Link Training success!\n");
-
 		analogix_dp_get_link_bandwidth(dp, &reg);
 		dp->link_train.link_rate = reg;
 		dev_dbg(dp->dev, "final bandwidth = %.2x\n",
@@ -574,22 +645,6 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp)
 		dev_dbg(dp->dev, "final lane count = %.2x\n",
 			dp->link_train.lane_count);
 
-		retval = drm_dp_dpcd_readb(&dp->aux, DP_MAX_DOWNSPREAD,
-					   &spread);
-		if (retval != 1) {
-			dev_err(dp->dev, "failed to read downspread %d\n",
-				retval);
-			dp->fast_train_support = false;
-		} else {
-			dp->fast_train_support =
-				(spread & DP_NO_AUX_HANDSHAKE_LINK_TRAINING) ?
-					true : false;
-		}
-		dev_dbg(dp->dev, "fast link training %s\n",
-			dp->fast_train_support ? "supported" : "unsupported");
-
-		/* set enhanced mode if available */
-		analogix_dp_set_enhanced_mode(dp);
 		dp->link_train.lt_state = FINISHED;
 
 		return 0;
@@ -793,7 +848,7 @@ static int analogix_dp_fast_link_train(struct analogix_dp_device *dp)
 
 static int analogix_dp_train_link(struct analogix_dp_device *dp)
 {
-	if (dp->fast_train_support)
+	if (dp->fast_train_enable)
 		return analogix_dp_fast_link_train(dp);
 
 	return analogix_dp_full_link_train(dp, dp->video_info.max_lane_count,
@@ -819,11 +874,10 @@ static int analogix_dp_config_video(struct analogix_dp_device *dp)
 		if (analogix_dp_is_slave_video_stream_clock_on(dp) == 0)
 			break;
 		if (timeout_loop > DP_TIMEOUT_LOOP_COUNT) {
-			dev_err(dp->dev, "Timeout of video streamclk ok\n");
+			dev_err(dp->dev, "Timeout of slave video streamclk ok\n");
 			return -ETIMEDOUT;
 		}
-
-		usleep_range(1, 2);
+		usleep_range(1000, 1001);
 	}
 
 	/* Set to use the register calculated M/N video */
@@ -838,6 +892,9 @@ static int analogix_dp_config_video(struct analogix_dp_device *dp)
 	/* Configure video slave mode */
 	analogix_dp_enable_video_master(dp, 0);
 
+	/* Enable video */
+	analogix_dp_start_video(dp);
+
 	timeout_loop = 0;
 
 	for (;;) {
@@ -850,8 +907,9 @@ static int analogix_dp_config_video(struct analogix_dp_device *dp)
 			done_count = 0;
 		}
 		if (timeout_loop > DP_TIMEOUT_LOOP_COUNT) {
-			dev_err(dp->dev, "Timeout of video streamclk ok\n");
-			return -ETIMEDOUT;
+			dev_warn(dp->dev,
+				 "Ignoring timeout of video streamclk ok\n");
+			break;
 		}
 
 		usleep_range(1000, 1001);
@@ -860,24 +918,32 @@ static int analogix_dp_config_video(struct analogix_dp_device *dp)
 	return 0;
 }
 
-static void analogix_dp_enable_scramble(struct analogix_dp_device *dp,
-					bool enable)
+static int analogix_dp_enable_scramble(struct analogix_dp_device *dp,
+				       bool enable)
 {
 	u8 data;
+	int ret;
 
 	if (enable) {
 		analogix_dp_enable_scrambling(dp);
 
-		drm_dp_dpcd_readb(&dp->aux, DP_TRAINING_PATTERN_SET, &data);
-		drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET,
+		ret = drm_dp_dpcd_readb(&dp->aux, DP_TRAINING_PATTERN_SET,
+					&data);
+		if (ret != 1)
+			return ret;
+		ret = drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET,
 				   (u8)(data & ~DP_LINK_SCRAMBLING_DISABLE));
 	} else {
 		analogix_dp_disable_scrambling(dp);
 
-		drm_dp_dpcd_readb(&dp->aux, DP_TRAINING_PATTERN_SET, &data);
-		drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET,
+		ret = drm_dp_dpcd_readb(&dp->aux, DP_TRAINING_PATTERN_SET,
+					&data);
+		if (ret != 1)
+			return ret;
+		ret = drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET,
 				   (u8)(data | DP_LINK_SCRAMBLING_DISABLE));
 	}
+	return ret < 0 ? ret : 0;
 }
 
 static irqreturn_t analogix_dp_hardirq(int irq, void *arg)
@@ -916,7 +982,23 @@ static irqreturn_t analogix_dp_irq_thread(int irq, void *arg)
 	return IRQ_HANDLED;
 }
 
-static void analogix_dp_commit(struct analogix_dp_device *dp)
+static int analogix_dp_fast_link_train_detection(struct analogix_dp_device *dp)
+{
+	int ret;
+	u8 spread;
+
+	ret = drm_dp_dpcd_readb(&dp->aux, DP_MAX_DOWNSPREAD, &spread);
+	if (ret != 1) {
+		dev_err(dp->dev, "failed to read downspread %d\n", ret);
+		return ret;
+	}
+	dp->fast_train_enable = !!(spread & DP_NO_AUX_HANDSHAKE_LINK_TRAINING);
+	dev_dbg(dp->dev, "fast link training %s\n",
+		dp->fast_train_enable ? "supported" : "unsupported");
+	return 0;
+}
+
+static int analogix_dp_commit(struct analogix_dp_device *dp)
 {
 	int ret;
 
@@ -926,34 +1008,50 @@ static void analogix_dp_commit(struct analogix_dp_device *dp)
 			DRM_ERROR("failed to disable the panel\n");
 	}
 
-	ret = readx_poll_timeout(analogix_dp_train_link, dp, ret, !ret, 100,
-				 DP_TIMEOUT_TRAINING_US * 5);
+	ret = analogix_dp_train_link(dp);
 	if (ret) {
 		dev_err(dp->dev, "unable to do link train, ret=%d\n", ret);
-		return;
+		return ret;
 	}
 
-	analogix_dp_enable_scramble(dp, 1);
-	analogix_dp_enable_rx_to_enhanced_mode(dp, 1);
-	analogix_dp_enable_enhanced_mode(dp, 1);
+	ret = analogix_dp_enable_scramble(dp, 1);
+	if (ret < 0) {
+		dev_err(dp->dev, "can not enable scramble\n");
+		return ret;
+	}
 
 	analogix_dp_init_video(dp);
 	ret = analogix_dp_config_video(dp);
-	if (ret)
+	if (ret) {
 		dev_err(dp->dev, "unable to config video\n");
+		return ret;
+	}
 
 	/* Safe to enable the panel now */
 	if (dp->plat_data->panel) {
-		if (drm_panel_enable(dp->plat_data->panel))
+		ret = drm_panel_enable(dp->plat_data->panel);
+		if (ret) {
 			DRM_ERROR("failed to enable the panel\n");
+			return ret;
+		}
 	}
 
-	/* Enable video */
-	analogix_dp_start_video(dp);
+	ret = analogix_dp_detect_sink_psr(dp);
+	if (ret)
+		return ret;
 
-	dp->psr_enable = analogix_dp_detect_sink_psr(dp);
-	if (dp->psr_enable)
-		analogix_dp_enable_sink_psr(dp);
+	if (dp->psr_enable) {
+		ret = analogix_dp_enable_sink_psr(dp);
+		if (ret)
+			return ret;
+	}
+
+	/* Check whether panel supports fast training */
+	ret =  analogix_dp_fast_link_train_detection(dp);
+	if (ret)
+		dp->psr_enable = false;
+
+	return ret;
 }
 
 /*
@@ -1150,24 +1248,80 @@ static void analogix_dp_bridge_pre_enable(struct drm_bridge *bridge)
 		DRM_ERROR("failed to setup the panel ret = %d\n", ret);
 }
 
-static void analogix_dp_bridge_enable(struct drm_bridge *bridge)
+static int analogix_dp_set_bridge(struct analogix_dp_device *dp)
 {
-	struct analogix_dp_device *dp = bridge->driver_private;
-
-	if (dp->dpms_mode == DRM_MODE_DPMS_ON)
-		return;
+	int ret;
 
 	pm_runtime_get_sync(dp->dev);
 
-	if (dp->plat_data->power_on)
-		dp->plat_data->power_on(dp->plat_data);
+	ret = clk_prepare_enable(dp->clock);
+	if (ret < 0) {
+		DRM_ERROR("Failed to prepare_enable the clock clk [%d]\n", ret);
+		goto out_dp_clk_pre;
+	}
+
+	if (dp->plat_data->power_on_start)
+		dp->plat_data->power_on_start(dp->plat_data);
 
 	phy_power_on(dp->phy);
-	analogix_dp_init_dp(dp);
+
+	ret = analogix_dp_init_dp(dp);
+	if (ret)
+		goto out_dp_init;
+
+	/*
+	 * According to DP spec v1.3 chap 3.5.1.2 Link Training,
+	 * We should first make sure the HPD signal is asserted high by device
+	 * when we want to establish a link with it.
+	 */
+	ret = analogix_dp_detect_hpd(dp);
+	if (ret) {
+		DRM_ERROR("failed to get hpd single ret = %d\n", ret);
+		goto out_dp_init;
+	}
+
+	ret = analogix_dp_commit(dp);
+	if (ret) {
+		DRM_ERROR("dp commit error, ret = %d\n", ret);
+		goto out_dp_init;
+	}
+
+	if (dp->plat_data->power_on_end)
+		dp->plat_data->power_on_end(dp->plat_data);
+
 	enable_irq(dp->irq);
-	analogix_dp_commit(dp);
+	return 0;
 
-	dp->dpms_mode = DRM_MODE_DPMS_ON;
+out_dp_init:
+	phy_power_off(dp->phy);
+	if (dp->plat_data->power_off)
+		dp->plat_data->power_off(dp->plat_data);
+	clk_disable_unprepare(dp->clock);
+out_dp_clk_pre:
+	pm_runtime_put_sync(dp->dev);
+
+	return ret;
+}
+
+static void analogix_dp_bridge_enable(struct drm_bridge *bridge)
+{
+	struct analogix_dp_device *dp = bridge->driver_private;
+	int timeout_loop = 0;
+
+	if (dp->dpms_mode == DRM_MODE_DPMS_ON)
+		return;
+
+	while (timeout_loop < MAX_PLL_LOCK_LOOP) {
+		if (analogix_dp_set_bridge(dp) == 0) {
+			dp->dpms_mode = DRM_MODE_DPMS_ON;
+			return;
+		}
+		dev_err(dp->dev, "failed to set bridge, retry: %d\n",
+			timeout_loop);
+		timeout_loop++;
+		usleep_range(10, 11);
+	}
+	dev_err(dp->dev, "too many times retry set bridge, give it up\n");
 }
 
 static void analogix_dp_bridge_disable(struct drm_bridge *bridge)
@@ -1186,11 +1340,15 @@ static void analogix_dp_bridge_disable(struct drm_bridge *bridge)
 	}
 
 	disable_irq(dp->irq);
-	phy_power_off(dp->phy);
 
 	if (dp->plat_data->power_off)
 		dp->plat_data->power_off(dp->plat_data);
 
+	analogix_dp_set_analog_power_down(dp, POWER_ALL, 1);
+	phy_power_off(dp->phy);
+
+	clk_disable_unprepare(dp->clock);
+
 	pm_runtime_put_sync(dp->dev);
 
 	ret = analogix_dp_prepare_panel(dp, false, true);
@@ -1198,6 +1356,7 @@ static void analogix_dp_bridge_disable(struct drm_bridge *bridge)
 		DRM_ERROR("failed to setup the panel ret = %d\n", ret);
 
 	dp->psr_enable = false;
+	dp->fast_train_enable = false;
 	dp->dpms_mode = DRM_MODE_DPMS_OFF;
 }
 
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
index 6a96ef7e6934..769255dc6e99 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
@@ -19,6 +19,7 @@
 #define DP_TIMEOUT_LOOP_COUNT 100
 #define MAX_CR_LOOP 5
 #define MAX_EQ_LOOP 5
+#define MAX_PLL_LOCK_LOOP 5
 
 /* Training takes 22ms if AUX channel comm fails. Use this as retry interval */
 #define DP_TIMEOUT_TRAINING_US			22000
@@ -173,7 +174,7 @@ struct analogix_dp_device {
 	int			hpd_gpio;
 	bool                    force_hpd;
 	bool			psr_enable;
-	bool			fast_train_support;
+	bool			fast_train_enable;
 
 	struct mutex		panel_lock;
 	bool			panel_is_modeset;
@@ -197,7 +198,7 @@ void analogix_dp_set_pll_power_down(struct analogix_dp_device *dp, bool enable);
 void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp,
 				       enum analog_power_block block,
 				       bool enable);
-void analogix_dp_init_analog_func(struct analogix_dp_device *dp);
+int analogix_dp_init_analog_func(struct analogix_dp_device *dp);
 void analogix_dp_init_hpd(struct analogix_dp_device *dp);
 void analogix_dp_force_hpd(struct analogix_dp_device *dp);
 enum dp_irq_type analogix_dp_get_irq_type(struct analogix_dp_device *dp);
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
index 9df2f3ef000c..a5f2763d72e4 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
@@ -126,9 +126,14 @@ void analogix_dp_reset(struct analogix_dp_device *dp)
 	analogix_dp_stop_video(dp);
 	analogix_dp_enable_video_mute(dp, 0);
 
-	reg = MASTER_VID_FUNC_EN_N | SLAVE_VID_FUNC_EN_N |
-		AUD_FIFO_FUNC_EN_N | AUD_FUNC_EN_N |
-		HDCP_FUNC_EN_N | SW_FUNC_EN_N;
+	if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
+		reg = RK_VID_CAP_FUNC_EN_N | RK_VID_FIFO_FUNC_EN_N |
+			SW_FUNC_EN_N;
+	else
+		reg = MASTER_VID_FUNC_EN_N | SLAVE_VID_FUNC_EN_N |
+			AUD_FIFO_FUNC_EN_N | AUD_FUNC_EN_N |
+			HDCP_FUNC_EN_N | SW_FUNC_EN_N;
+
 	writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_1);
 
 	reg = SSC_FUNC_EN_N | AUX_FUNC_EN_N |
@@ -230,16 +235,20 @@ enum pll_status analogix_dp_get_pll_lock_status(struct analogix_dp_device *dp)
 void analogix_dp_set_pll_power_down(struct analogix_dp_device *dp, bool enable)
 {
 	u32 reg;
+	u32 mask = DP_PLL_PD;
+	u32 pd_addr = ANALOGIX_DP_PLL_CTL;
 
-	if (enable) {
-		reg = readl(dp->reg_base + ANALOGIX_DP_PLL_CTL);
-		reg |= DP_PLL_PD;
-		writel(reg, dp->reg_base + ANALOGIX_DP_PLL_CTL);
-	} else {
-		reg = readl(dp->reg_base + ANALOGIX_DP_PLL_CTL);
-		reg &= ~DP_PLL_PD;
-		writel(reg, dp->reg_base + ANALOGIX_DP_PLL_CTL);
+	if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) {
+		pd_addr = ANALOGIX_DP_PD;
+		mask = RK_PLL_PD;
 	}
+
+	reg = readl(dp->reg_base + pd_addr);
+	if (enable)
+		reg |= mask;
+	else
+		reg &= ~mask;
+	writel(reg, dp->reg_base + pd_addr);
 }
 
 void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp,
@@ -248,83 +257,98 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp,
 {
 	u32 reg;
 	u32 phy_pd_addr = ANALOGIX_DP_PHY_PD;
+	u32 mask;
 
 	if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
 		phy_pd_addr = ANALOGIX_DP_PD;
 
 	switch (block) {
 	case AUX_BLOCK:
-		if (enable) {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg |= AUX_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		} else {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg &= ~AUX_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		}
+		if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
+			mask = RK_AUX_PD;
+		else
+			mask = AUX_PD;
+
+		reg = readl(dp->reg_base + phy_pd_addr);
+		if (enable)
+			reg |= mask;
+		else
+			reg &= ~mask;
+		writel(reg, dp->reg_base + phy_pd_addr);
 		break;
 	case CH0_BLOCK:
-		if (enable) {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg |= CH0_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		} else {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg &= ~CH0_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		}
+		mask = CH0_PD;
+		reg = readl(dp->reg_base + phy_pd_addr);
+
+		if (enable)
+			reg |= mask;
+		else
+			reg &= ~mask;
+		writel(reg, dp->reg_base + phy_pd_addr);
 		break;
 	case CH1_BLOCK:
-		if (enable) {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg |= CH1_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		} else {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg &= ~CH1_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		}
+		mask = CH1_PD;
+		reg = readl(dp->reg_base + phy_pd_addr);
+
+		if (enable)
+			reg |= mask;
+		else
+			reg &= ~mask;
+		writel(reg, dp->reg_base + phy_pd_addr);
 		break;
 	case CH2_BLOCK:
-		if (enable) {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg |= CH2_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		} else {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg &= ~CH2_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		}
+		mask = CH2_PD;
+		reg = readl(dp->reg_base + phy_pd_addr);
+
+		if (enable)
+			reg |= mask;
+		else
+			reg &= ~mask;
+		writel(reg, dp->reg_base + phy_pd_addr);
 		break;
 	case CH3_BLOCK:
-		if (enable) {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg |= CH3_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		} else {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg &= ~CH3_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		}
+		mask = CH3_PD;
+		reg = readl(dp->reg_base + phy_pd_addr);
+
+		if (enable)
+			reg |= mask;
+		else
+			reg &= ~mask;
+		writel(reg, dp->reg_base + phy_pd_addr);
 		break;
 	case ANALOG_TOTAL:
-		if (enable) {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg |= DP_PHY_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		} else {
-			reg = readl(dp->reg_base + phy_pd_addr);
-			reg &= ~DP_PHY_PD;
-			writel(reg, dp->reg_base + phy_pd_addr);
-		}
+		/*
+		 * There is no bit named DP_PHY_PD, so We used DP_INC_BG
+		 * to power off everything instead of DP_PHY_PD in
+		 * Rockchip
+		 */
+		if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
+			mask = DP_INC_BG;
+		else
+			mask = DP_PHY_PD;
+
+		reg = readl(dp->reg_base + phy_pd_addr);
+		if (enable)
+			reg |= mask;
+		else
+			reg &= ~mask;
+
+		writel(reg, dp->reg_base + phy_pd_addr);
+		if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
+			usleep_range(10, 15);
 		break;
 	case POWER_ALL:
 		if (enable) {
-			reg = DP_PHY_PD | AUX_PD | CH3_PD | CH2_PD |
-				CH1_PD | CH0_PD;
+			reg = DP_ALL_PD;
 			writel(reg, dp->reg_base + phy_pd_addr);
 		} else {
+			reg = DP_ALL_PD;
+			writel(reg, dp->reg_base + phy_pd_addr);
+			usleep_range(10, 15);
+			reg &= ~DP_INC_BG;
+			writel(reg, dp->reg_base + phy_pd_addr);
+			usleep_range(10, 15);
+
 			writel(0x00, dp->reg_base + phy_pd_addr);
 		}
 		break;
@@ -333,7 +357,7 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp,
 	}
 }
 
-void analogix_dp_init_analog_func(struct analogix_dp_device *dp)
+int analogix_dp_init_analog_func(struct analogix_dp_device *dp)
 {
 	u32 reg;
 	int timeout_loop = 0;
@@ -355,7 +379,7 @@ void analogix_dp_init_analog_func(struct analogix_dp_device *dp)
 			timeout_loop++;
 			if (DP_TIMEOUT_LOOP_COUNT < timeout_loop) {
 				dev_err(dp->dev, "failed to get pll lock status\n");
-				return;
+				return -ETIMEDOUT;
 			}
 			usleep_range(10, 20);
 		}
@@ -366,6 +390,7 @@ void analogix_dp_init_analog_func(struct analogix_dp_device *dp)
 	reg &= ~(SERDES_FIFO_FUNC_EN_N | LS_CLK_DOMAIN_FUNC_EN_N
 		| AUX_FUNC_EN_N);
 	writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2);
+	return 0;
 }
 
 void analogix_dp_clear_hotplug_interrupts(struct analogix_dp_device *dp)
@@ -450,17 +475,22 @@ void analogix_dp_init_aux(struct analogix_dp_device *dp)
 	reg = RPLY_RECEIV | AUX_ERR;
 	writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA);
 
+	analogix_dp_set_analog_power_down(dp, AUX_BLOCK, true);
+	usleep_range(10, 11);
+	analogix_dp_set_analog_power_down(dp, AUX_BLOCK, false);
+
 	analogix_dp_reset_aux(dp);
 
-	/* Disable AUX transaction H/W retry */
+	/* AUX_BIT_PERIOD_EXPECTED_DELAY doesn't apply to Rockchip IP */
 	if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
-		reg = AUX_BIT_PERIOD_EXPECTED_DELAY(0) |
-		      AUX_HW_RETRY_COUNT_SEL(3) |
-		      AUX_HW_RETRY_INTERVAL_600_MICROSECONDS;
+		reg = 0;
 	else
-		reg = AUX_BIT_PERIOD_EXPECTED_DELAY(3) |
-		      AUX_HW_RETRY_COUNT_SEL(0) |
-		      AUX_HW_RETRY_INTERVAL_600_MICROSECONDS;
+		reg = AUX_BIT_PERIOD_EXPECTED_DELAY(3);
+
+	/* Disable AUX transaction H/W retry */
+	reg |= AUX_HW_RETRY_COUNT_SEL(0) |
+	       AUX_HW_RETRY_INTERVAL_600_MICROSECONDS;
+
 	writel(reg, dp->reg_base + ANALOGIX_DP_AUX_HW_RETRY_CTL);
 
 	/* Receive AUX Channel DEFER commands equal to DEFFER_COUNT*64 */
@@ -947,8 +977,12 @@ void analogix_dp_config_video_slave_mode(struct analogix_dp_device *dp)
 	u32 reg;
 
 	reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_1);
-	reg &= ~(MASTER_VID_FUNC_EN_N | SLAVE_VID_FUNC_EN_N);
-	reg |= MASTER_VID_FUNC_EN_N;
+	if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) {
+		reg &= ~(RK_VID_CAP_FUNC_EN_N | RK_VID_FIFO_FUNC_EN_N);
+	} else {
+		reg &= ~(MASTER_VID_FUNC_EN_N | SLAVE_VID_FUNC_EN_N);
+		reg |= MASTER_VID_FUNC_EN_N;
+	}
 	writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_1);
 
 	reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10);
@@ -1072,10 +1106,11 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp,
 			     struct drm_dp_aux_msg *msg)
 {
 	u32 reg;
+	u32 status_reg;
 	u8 *buffer = msg->buffer;
-	int timeout_loop = 0;
 	unsigned int i;
 	int num_transferred = 0;
+	int ret;
 
 	/* Buffer size of AUX CH is 16 bytes */
 	if (WARN_ON(msg->size > 16))
@@ -1139,17 +1174,20 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp,
 
 	writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2);
 
-	/* Is AUX CH command reply received? */
+	ret = readx_poll_timeout(readl, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2,
+				 reg, !(reg & AUX_EN), 25, 500 * 1000);
+	if (ret) {
+		dev_err(dp->dev, "AUX CH enable timeout!\n");
+		goto aux_error;
+	}
+
 	/* TODO: Wait for an interrupt instead of looping? */
-	reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA);
-	while (!(reg & RPLY_RECEIV)) {
-		timeout_loop++;
-		if (timeout_loop > DP_TIMEOUT_LOOP_COUNT) {
-			dev_err(dp->dev, "AUX CH command reply failed!\n");
-			return -ETIMEDOUT;
-		}
-		reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA);
-		usleep_range(10, 11);
+	/* Is AUX CH command reply received? */
+	ret = readx_poll_timeout(readl, dp->reg_base + ANALOGIX_DP_INT_STA,
+				 reg, reg & RPLY_RECEIV, 10, 20 * 1000);
+	if (ret) {
+		dev_err(dp->dev, "AUX CH cmd reply timeout!\n");
+		goto aux_error;
 	}
 
 	/* Clear interrupt source for AUX CH command reply */
@@ -1157,17 +1195,13 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp,
 
 	/* Clear interrupt source for AUX CH access error */
 	reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA);
-	if (reg & AUX_ERR) {
+	status_reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_STA);
+	if ((reg & AUX_ERR) || (status_reg & AUX_STATUS_MASK)) {
 		writel(AUX_ERR, dp->reg_base + ANALOGIX_DP_INT_STA);
-		return -EREMOTEIO;
-	}
 
-	/* Check AUX CH error access status */
-	reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_STA);
-	if ((reg & AUX_STATUS_MASK)) {
-		dev_err(dp->dev, "AUX CH error happened: %d\n\n",
-			reg & AUX_STATUS_MASK);
-		return -EREMOTEIO;
+		dev_warn(dp->dev, "AUX CH error happened: %#x (%d)\n",
+			 status_reg & AUX_STATUS_MASK, !!(reg & AUX_ERR));
+		goto aux_error;
 	}
 
 	if (msg->request & DP_AUX_I2C_READ) {
@@ -1193,4 +1227,10 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp,
 		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
 
 	return num_transferred > 0 ? num_transferred : -EBUSY;
+
+aux_error:
+	/* if aux err happen, reset aux */
+	analogix_dp_init_aux(dp);
+
+	return -EREMOTEIO;
 }
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
index 40200c652533..0cf27c731727 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
@@ -127,7 +127,9 @@
 
 /* ANALOGIX_DP_FUNC_EN_1 */
 #define MASTER_VID_FUNC_EN_N			(0x1 << 7)
+#define RK_VID_CAP_FUNC_EN_N			(0x1 << 6)
 #define SLAVE_VID_FUNC_EN_N			(0x1 << 5)
+#define RK_VID_FIFO_FUNC_EN_N			(0x1 << 5)
 #define AUD_FIFO_FUNC_EN_N			(0x1 << 4)
 #define AUD_FUNC_EN_N				(0x1 << 3)
 #define HDCP_FUNC_EN_N				(0x1 << 2)
@@ -342,12 +344,17 @@
 #define DP_PLL_REF_BIT_1_2500V			(0x7 << 0)
 
 /* ANALOGIX_DP_PHY_PD */
+#define DP_INC_BG				(0x1 << 7)
+#define DP_EXP_BG				(0x1 << 6)
 #define DP_PHY_PD				(0x1 << 5)
+#define RK_AUX_PD				(0x1 << 5)
 #define AUX_PD					(0x1 << 4)
+#define RK_PLL_PD				(0x1 << 4)
 #define CH3_PD					(0x1 << 3)
 #define CH2_PD					(0x1 << 2)
 #define CH1_PD					(0x1 << 1)
 #define CH0_PD					(0x1 << 0)
+#define DP_ALL_PD				(0xff)
 
 /* ANALOGIX_DP_PHY_TEST */
 #define MACRO_RST				(0x1 << 5)
diff --git a/drivers/gpu/drm/bridge/cdns-dsi.c b/drivers/gpu/drm/bridge/cdns-dsi.c
new file mode 100644
index 000000000000..c255fc3e1be5
--- /dev/null
+++ b/drivers/gpu/drm/bridge/cdns-dsi.c
@@ -0,0 +1,1623 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright: 2017 Cadence Design Systems, Inc.
+ *
+ * Author: Boris Brezillon <boris.brezillon@bootlin.com>
+ */
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+#include <video/mipi_display.h>
+
+#include <linux/clk.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+
+#define IP_CONF				0x0
+#define SP_HS_FIFO_DEPTH(x)		(((x) & GENMASK(30, 26)) >> 26)
+#define SP_LP_FIFO_DEPTH(x)		(((x) & GENMASK(25, 21)) >> 21)
+#define VRS_FIFO_DEPTH(x)		(((x) & GENMASK(20, 16)) >> 16)
+#define DIRCMD_FIFO_DEPTH(x)		(((x) & GENMASK(15, 13)) >> 13)
+#define SDI_IFACE_32			BIT(12)
+#define INTERNAL_DATAPATH_32		(0 << 10)
+#define INTERNAL_DATAPATH_16		(1 << 10)
+#define INTERNAL_DATAPATH_8		(3 << 10)
+#define INTERNAL_DATAPATH_SIZE		((x) & GENMASK(11, 10))
+#define NUM_IFACE(x)			((((x) & GENMASK(9, 8)) >> 8) + 1)
+#define MAX_LANE_NB(x)			(((x) & GENMASK(7, 6)) >> 6)
+#define RX_FIFO_DEPTH(x)		((x) & GENMASK(5, 0))
+
+#define MCTL_MAIN_DATA_CTL		0x4
+#define TE_MIPI_POLLING_EN		BIT(25)
+#define TE_HW_POLLING_EN		BIT(24)
+#define DISP_EOT_GEN			BIT(18)
+#define HOST_EOT_GEN			BIT(17)
+#define DISP_GEN_CHECKSUM		BIT(16)
+#define DISP_GEN_ECC			BIT(15)
+#define BTA_EN				BIT(14)
+#define READ_EN				BIT(13)
+#define REG_TE_EN			BIT(12)
+#define IF_TE_EN(x)			BIT(8 + (x))
+#define TVG_SEL				BIT(6)
+#define VID_EN				BIT(5)
+#define IF_VID_SELECT(x)		((x) << 2)
+#define IF_VID_SELECT_MASK		GENMASK(3, 2)
+#define IF_VID_MODE			BIT(1)
+#define LINK_EN				BIT(0)
+
+#define MCTL_MAIN_PHY_CTL		0x8
+#define HS_INVERT_DAT(x)		BIT(19 + ((x) * 2))
+#define SWAP_PINS_DAT(x)		BIT(18 + ((x) * 2))
+#define HS_INVERT_CLK			BIT(17)
+#define SWAP_PINS_CLK			BIT(16)
+#define HS_SKEWCAL_EN			BIT(15)
+#define WAIT_BURST_TIME(x)		((x) << 10)
+#define DATA_ULPM_EN(x)			BIT(6 + (x))
+#define CLK_ULPM_EN			BIT(5)
+#define CLK_CONTINUOUS			BIT(4)
+#define DATA_LANE_EN(x)			BIT((x) - 1)
+
+#define MCTL_MAIN_EN			0xc
+#define DATA_FORCE_STOP			BIT(17)
+#define CLK_FORCE_STOP			BIT(16)
+#define IF_EN(x)			BIT(13 + (x))
+#define DATA_LANE_ULPM_REQ(l)		BIT(9 + (l))
+#define CLK_LANE_ULPM_REQ		BIT(8)
+#define DATA_LANE_START(x)		BIT(4 + (x))
+#define CLK_LANE_EN			BIT(3)
+#define PLL_START			BIT(0)
+
+#define MCTL_DPHY_CFG0			0x10
+#define DPHY_C_RSTB			BIT(20)
+#define DPHY_D_RSTB(x)			GENMASK(15 + (x), 16)
+#define DPHY_PLL_PDN			BIT(10)
+#define DPHY_CMN_PDN			BIT(9)
+#define DPHY_C_PDN			BIT(8)
+#define DPHY_D_PDN(x)			GENMASK(3 + (x), 4)
+#define DPHY_ALL_D_PDN			GENMASK(7, 4)
+#define DPHY_PLL_PSO			BIT(1)
+#define DPHY_CMN_PSO			BIT(0)
+
+#define MCTL_DPHY_TIMEOUT1		0x14
+#define HSTX_TIMEOUT(x)			((x) << 4)
+#define HSTX_TIMEOUT_MAX		GENMASK(17, 0)
+#define CLK_DIV(x)			(x)
+#define CLK_DIV_MAX			GENMASK(3, 0)
+
+#define MCTL_DPHY_TIMEOUT2		0x18
+#define LPRX_TIMEOUT(x)			(x)
+
+#define MCTL_ULPOUT_TIME		0x1c
+#define DATA_LANE_ULPOUT_TIME(x)	((x) << 9)
+#define CLK_LANE_ULPOUT_TIME(x)		(x)
+
+#define MCTL_3DVIDEO_CTL		0x20
+#define VID_VSYNC_3D_EN			BIT(7)
+#define VID_VSYNC_3D_LR			BIT(5)
+#define VID_VSYNC_3D_SECOND_EN		BIT(4)
+#define VID_VSYNC_3DFORMAT_LINE		(0 << 2)
+#define VID_VSYNC_3DFORMAT_FRAME	(1 << 2)
+#define VID_VSYNC_3DFORMAT_PIXEL	(2 << 2)
+#define VID_VSYNC_3DMODE_OFF		0
+#define VID_VSYNC_3DMODE_PORTRAIT	1
+#define VID_VSYNC_3DMODE_LANDSCAPE	2
+
+#define MCTL_MAIN_STS			0x24
+#define MCTL_MAIN_STS_CTL		0x130
+#define MCTL_MAIN_STS_CLR		0x150
+#define MCTL_MAIN_STS_FLAG		0x170
+#define HS_SKEWCAL_DONE			BIT(11)
+#define IF_UNTERM_PKT_ERR(x)		BIT(8 + (x))
+#define LPRX_TIMEOUT_ERR		BIT(7)
+#define HSTX_TIMEOUT_ERR		BIT(6)
+#define DATA_LANE_RDY(l)		BIT(2 + (l))
+#define CLK_LANE_RDY			BIT(1)
+#define PLL_LOCKED			BIT(0)
+
+#define MCTL_DPHY_ERR			0x28
+#define MCTL_DPHY_ERR_CTL1		0x148
+#define MCTL_DPHY_ERR_CLR		0x168
+#define MCTL_DPHY_ERR_FLAG		0x188
+#define ERR_CONT_LP(x, l)		BIT(18 + ((x) * 4) + (l))
+#define ERR_CONTROL(l)			BIT(14 + (l))
+#define ERR_SYNESC(l)			BIT(10 + (l))
+#define ERR_ESC(l)			BIT(6 + (l))
+
+#define MCTL_DPHY_ERR_CTL2		0x14c
+#define ERR_CONT_LP_EDGE(x, l)		BIT(12 + ((x) * 4) + (l))
+#define ERR_CONTROL_EDGE(l)		BIT(8 + (l))
+#define ERR_SYN_ESC_EDGE(l)		BIT(4 + (l))
+#define ERR_ESC_EDGE(l)			BIT(0 + (l))
+
+#define MCTL_LANE_STS			0x2c
+#define PPI_C_TX_READY_HS		BIT(18)
+#define DPHY_PLL_LOCK			BIT(17)
+#define PPI_D_RX_ULPS_ESC(x)		(((x) & GENMASK(15, 12)) >> 12)
+#define LANE_STATE_START		0
+#define LANE_STATE_IDLE			1
+#define LANE_STATE_WRITE		2
+#define LANE_STATE_ULPM			3
+#define LANE_STATE_READ			4
+#define DATA_LANE_STATE(l, val)		\
+	(((val) >> (2 + 2 * (l) + ((l) ? 1 : 0))) & GENMASK((l) ? 1 : 2, 0))
+#define CLK_LANE_STATE_HS		2
+#define CLK_LANE_STATE(val)		((val) & GENMASK(1, 0))
+
+#define DSC_MODE_CTL			0x30
+#define DSC_MODE_EN			BIT(0)
+
+#define DSC_CMD_SEND			0x34
+#define DSC_SEND_PPS			BIT(0)
+#define DSC_EXECUTE_QUEUE		BIT(1)
+
+#define DSC_PPS_WRDAT			0x38
+
+#define DSC_MODE_STS			0x3c
+#define DSC_PPS_DONE			BIT(1)
+#define DSC_EXEC_DONE			BIT(2)
+
+#define CMD_MODE_CTL			0x70
+#define IF_LP_EN(x)			BIT(9 + (x))
+#define IF_VCHAN_ID(x, c)		((c) << ((x) * 2))
+
+#define CMD_MODE_CTL2			0x74
+#define TE_TIMEOUT(x)			((x) << 11)
+#define FILL_VALUE(x)			((x) << 3)
+#define ARB_IF_WITH_HIGHEST_PRIORITY(x)	((x) << 1)
+#define ARB_ROUND_ROBIN_MODE		BIT(0)
+
+#define CMD_MODE_STS			0x78
+#define CMD_MODE_STS_CTL		0x134
+#define CMD_MODE_STS_CLR		0x154
+#define CMD_MODE_STS_FLAG		0x174
+#define ERR_IF_UNDERRUN(x)		BIT(4 + (x))
+#define ERR_UNWANTED_READ		BIT(3)
+#define ERR_TE_MISS			BIT(2)
+#define ERR_NO_TE			BIT(1)
+#define CSM_RUNNING			BIT(0)
+
+#define DIRECT_CMD_SEND			0x80
+
+#define DIRECT_CMD_MAIN_SETTINGS	0x84
+#define TRIGGER_VAL(x)			((x) << 25)
+#define CMD_LP_EN			BIT(24)
+#define CMD_SIZE(x)			((x) << 16)
+#define CMD_VCHAN_ID(x)			((x) << 14)
+#define CMD_DATATYPE(x)			((x) << 8)
+#define CMD_LONG			BIT(3)
+#define WRITE_CMD			0
+#define READ_CMD			1
+#define TE_REQ				4
+#define TRIGGER_REQ			5
+#define BTA_REQ				6
+
+#define DIRECT_CMD_STS			0x88
+#define DIRECT_CMD_STS_CTL		0x138
+#define DIRECT_CMD_STS_CLR		0x158
+#define DIRECT_CMD_STS_FLAG		0x178
+#define RCVD_ACK_VAL(val)		((val) >> 16)
+#define RCVD_TRIGGER_VAL(val)		(((val) & GENMASK(14, 11)) >> 11)
+#define READ_COMPLETED_WITH_ERR		BIT(10)
+#define BTA_FINISHED			BIT(9)
+#define BTA_COMPLETED			BIT(8)
+#define TE_RCVD				BIT(7)
+#define TRIGGER_RCVD			BIT(6)
+#define ACK_WITH_ERR_RCVD		BIT(5)
+#define ACK_RCVD			BIT(4)
+#define READ_COMPLETED			BIT(3)
+#define TRIGGER_COMPLETED		BIT(2)
+#define WRITE_COMPLETED			BIT(1)
+#define SENDING_CMD			BIT(0)
+
+#define DIRECT_CMD_STOP_READ		0x8c
+
+#define DIRECT_CMD_WRDATA		0x90
+
+#define DIRECT_CMD_FIFO_RST		0x94
+
+#define DIRECT_CMD_RDDATA		0xa0
+
+#define DIRECT_CMD_RD_PROPS		0xa4
+#define RD_DCS				BIT(18)
+#define RD_VCHAN_ID(val)		(((val) >> 16) & GENMASK(1, 0))
+#define RD_SIZE(val)			((val) & GENMASK(15, 0))
+
+#define DIRECT_CMD_RD_STS		0xa8
+#define DIRECT_CMD_RD_STS_CTL		0x13c
+#define DIRECT_CMD_RD_STS_CLR		0x15c
+#define DIRECT_CMD_RD_STS_FLAG		0x17c
+#define ERR_EOT_WITH_ERR		BIT(8)
+#define ERR_MISSING_EOT			BIT(7)
+#define ERR_WRONG_LENGTH		BIT(6)
+#define ERR_OVERSIZE			BIT(5)
+#define ERR_RECEIVE			BIT(4)
+#define ERR_UNDECODABLE			BIT(3)
+#define ERR_CHECKSUM			BIT(2)
+#define ERR_UNCORRECTABLE		BIT(1)
+#define ERR_FIXED			BIT(0)
+
+#define VID_MAIN_CTL			0xb0
+#define VID_IGNORE_MISS_VSYNC		BIT(31)
+#define VID_FIELD_SW			BIT(28)
+#define VID_INTERLACED_EN		BIT(27)
+#define RECOVERY_MODE(x)		((x) << 25)
+#define RECOVERY_MODE_NEXT_HSYNC	0
+#define RECOVERY_MODE_NEXT_STOP_POINT	2
+#define RECOVERY_MODE_NEXT_VSYNC	3
+#define REG_BLKEOL_MODE(x)		((x) << 23)
+#define REG_BLKLINE_MODE(x)		((x) << 21)
+#define REG_BLK_MODE_NULL_PKT		0
+#define REG_BLK_MODE_BLANKING_PKT	1
+#define REG_BLK_MODE_LP			2
+#define SYNC_PULSE_HORIZONTAL		BIT(20)
+#define SYNC_PULSE_ACTIVE		BIT(19)
+#define BURST_MODE			BIT(18)
+#define VID_PIXEL_MODE_MASK		GENMASK(17, 14)
+#define VID_PIXEL_MODE_RGB565		(0 << 14)
+#define VID_PIXEL_MODE_RGB666_PACKED	(1 << 14)
+#define VID_PIXEL_MODE_RGB666		(2 << 14)
+#define VID_PIXEL_MODE_RGB888		(3 << 14)
+#define VID_PIXEL_MODE_RGB101010	(4 << 14)
+#define VID_PIXEL_MODE_RGB121212	(5 << 14)
+#define VID_PIXEL_MODE_YUV420		(8 << 14)
+#define VID_PIXEL_MODE_YUV422_PACKED	(9 << 14)
+#define VID_PIXEL_MODE_YUV422		(10 << 14)
+#define VID_PIXEL_MODE_YUV422_24B	(11 << 14)
+#define VID_PIXEL_MODE_DSC_COMP		(12 << 14)
+#define VID_DATATYPE(x)			((x) << 8)
+#define VID_VIRTCHAN_ID(iface, x)	((x) << (4 + (iface) * 2))
+#define STOP_MODE(x)			((x) << 2)
+#define START_MODE(x)			(x)
+
+#define VID_VSIZE1			0xb4
+#define VFP_LEN(x)			((x) << 12)
+#define VBP_LEN(x)			((x) << 6)
+#define VSA_LEN(x)			(x)
+
+#define VID_VSIZE2			0xb8
+#define VACT_LEN(x)			(x)
+
+#define VID_HSIZE1			0xc0
+#define HBP_LEN(x)			((x) << 16)
+#define HSA_LEN(x)			(x)
+
+#define VID_HSIZE2			0xc4
+#define HFP_LEN(x)			((x) << 16)
+#define HACT_LEN(x)			(x)
+
+#define VID_BLKSIZE1			0xcc
+#define BLK_EOL_PKT_LEN(x)		((x) << 15)
+#define BLK_LINE_EVENT_PKT_LEN(x)	(x)
+
+#define VID_BLKSIZE2			0xd0
+#define BLK_LINE_PULSE_PKT_LEN(x)	(x)
+
+#define VID_PKT_TIME			0xd8
+#define BLK_EOL_DURATION(x)		(x)
+
+#define VID_DPHY_TIME			0xdc
+#define REG_WAKEUP_TIME(x)		((x) << 17)
+#define REG_LINE_DURATION(x)		(x)
+
+#define VID_ERR_COLOR1			0xe0
+#define COL_GREEN(x)			((x) << 12)
+#define COL_RED(x)			(x)
+
+#define VID_ERR_COLOR2			0xe4
+#define PAD_VAL(x)			((x) << 12)
+#define COL_BLUE(x)			(x)
+
+#define VID_VPOS			0xe8
+#define LINE_VAL(val)			(((val) & GENMASK(14, 2)) >> 2)
+#define LINE_POS(val)			((val) & GENMASK(1, 0))
+
+#define VID_HPOS			0xec
+#define HORIZ_VAL(val)			(((val) & GENMASK(17, 3)) >> 3)
+#define HORIZ_POS(val)			((val) & GENMASK(2, 0))
+
+#define VID_MODE_STS			0xf0
+#define VID_MODE_STS_CTL		0x140
+#define VID_MODE_STS_CLR		0x160
+#define VID_MODE_STS_FLAG		0x180
+#define VSG_RECOVERY			BIT(10)
+#define ERR_VRS_WRONG_LEN		BIT(9)
+#define ERR_LONG_READ			BIT(8)
+#define ERR_LINE_WRITE			BIT(7)
+#define ERR_BURST_WRITE			BIT(6)
+#define ERR_SMALL_HEIGHT		BIT(5)
+#define ERR_SMALL_LEN			BIT(4)
+#define ERR_MISSING_VSYNC		BIT(3)
+#define ERR_MISSING_HSYNC		BIT(2)
+#define ERR_MISSING_DATA		BIT(1)
+#define VSG_RUNNING			BIT(0)
+
+#define VID_VCA_SETTING1		0xf4
+#define BURST_LP			BIT(16)
+#define MAX_BURST_LIMIT(x)		(x)
+
+#define VID_VCA_SETTING2		0xf8
+#define MAX_LINE_LIMIT(x)		((x) << 16)
+#define EXACT_BURST_LIMIT(x)		(x)
+
+#define TVG_CTL				0xfc
+#define TVG_STRIPE_SIZE(x)		((x) << 5)
+#define TVG_MODE_MASK			GENMASK(4, 3)
+#define TVG_MODE_SINGLE_COLOR		(0 << 3)
+#define TVG_MODE_VSTRIPES		(2 << 3)
+#define TVG_MODE_HSTRIPES		(3 << 3)
+#define TVG_STOPMODE_MASK		GENMASK(2, 1)
+#define TVG_STOPMODE_EOF		(0 << 1)
+#define TVG_STOPMODE_EOL		(1 << 1)
+#define TVG_STOPMODE_NOW		(2 << 1)
+#define TVG_RUN				BIT(0)
+
+#define TVG_IMG_SIZE			0x100
+#define TVG_NBLINES(x)			((x) << 16)
+#define TVG_LINE_SIZE(x)		(x)
+
+#define TVG_COLOR1			0x104
+#define TVG_COL1_GREEN(x)		((x) << 12)
+#define TVG_COL1_RED(x)			(x)
+
+#define TVG_COLOR1_BIS			0x108
+#define TVG_COL1_BLUE(x)		(x)
+
+#define TVG_COLOR2			0x10c
+#define TVG_COL2_GREEN(x)		((x) << 12)
+#define TVG_COL2_RED(x)			(x)
+
+#define TVG_COLOR2_BIS			0x110
+#define TVG_COL2_BLUE(x)		(x)
+
+#define TVG_STS				0x114
+#define TVG_STS_CTL			0x144
+#define TVG_STS_CLR			0x164
+#define TVG_STS_FLAG			0x184
+#define TVG_STS_RUNNING			BIT(0)
+
+#define STS_CTL_EDGE(e)			((e) << 16)
+
+#define DPHY_LANES_MAP			0x198
+#define DAT_REMAP_CFG(b, l)		((l) << ((b) * 8))
+
+#define DPI_IRQ_EN			0x1a0
+#define DPI_IRQ_CLR			0x1a4
+#define DPI_IRQ_STS			0x1a8
+#define PIXEL_BUF_OVERFLOW		BIT(0)
+
+#define DPI_CFG				0x1ac
+#define DPI_CFG_FIFO_DEPTH(x)		((x) >> 16)
+#define DPI_CFG_FIFO_LEVEL(x)		((x) & GENMASK(15, 0))
+
+#define TEST_GENERIC			0x1f0
+#define TEST_STATUS(x)			((x) >> 16)
+#define TEST_CTRL(x)			(x)
+
+#define ID_REG				0x1fc
+#define REV_VENDOR_ID(x)		(((x) & GENMASK(31, 20)) >> 20)
+#define REV_PRODUCT_ID(x)		(((x) & GENMASK(19, 12)) >> 12)
+#define REV_HW(x)			(((x) & GENMASK(11, 8)) >> 8)
+#define REV_MAJOR(x)			(((x) & GENMASK(7, 4)) >> 4)
+#define REV_MINOR(x)			((x) & GENMASK(3, 0))
+
+#define DSI_OUTPUT_PORT			0
+#define DSI_INPUT_PORT(inputid)		(1 + (inputid))
+
+#define DSI_HBP_FRAME_OVERHEAD		12
+#define DSI_HSA_FRAME_OVERHEAD		14
+#define DSI_HFP_FRAME_OVERHEAD		6
+#define DSI_HSS_VSS_VSE_FRAME_OVERHEAD	4
+#define DSI_BLANKING_FRAME_OVERHEAD	6
+#define DSI_NULL_FRAME_OVERHEAD		6
+#define DSI_EOT_PKT_SIZE		4
+
+#define REG_WAKEUP_TIME_NS		800
+#define DPHY_PLL_RATE_HZ		108000000
+
+/* DPHY registers */
+#define DPHY_PMA_CMN(reg)		(reg)
+#define DPHY_PMA_LCLK(reg)		(0x100 + (reg))
+#define DPHY_PMA_LDATA(lane, reg)	(0x200 + ((lane) * 0x100) + (reg))
+#define DPHY_PMA_RCLK(reg)		(0x600 + (reg))
+#define DPHY_PMA_RDATA(lane, reg)	(0x700 + ((lane) * 0x100) + (reg))
+#define DPHY_PCS(reg)			(0xb00 + (reg))
+
+#define DPHY_CMN_SSM			DPHY_PMA_CMN(0x20)
+#define DPHY_CMN_SSM_EN			BIT(0)
+#define DPHY_CMN_TX_MODE_EN		BIT(9)
+
+#define DPHY_CMN_PWM			DPHY_PMA_CMN(0x40)
+#define DPHY_CMN_PWM_DIV(x)		((x) << 20)
+#define DPHY_CMN_PWM_LOW(x)		((x) << 10)
+#define DPHY_CMN_PWM_HIGH(x)		(x)
+
+#define DPHY_CMN_FBDIV			DPHY_PMA_CMN(0x4c)
+#define DPHY_CMN_FBDIV_VAL(low, high)	(((high) << 11) | ((low) << 22))
+#define DPHY_CMN_FBDIV_FROM_REG		(BIT(10) | BIT(21))
+
+#define DPHY_CMN_OPIPDIV		DPHY_PMA_CMN(0x50)
+#define DPHY_CMN_IPDIV_FROM_REG		BIT(0)
+#define DPHY_CMN_IPDIV(x)		((x) << 1)
+#define DPHY_CMN_OPDIV_FROM_REG		BIT(6)
+#define DPHY_CMN_OPDIV(x)		((x) << 7)
+
+#define DPHY_PSM_CFG			DPHY_PCS(0x4)
+#define DPHY_PSM_CFG_FROM_REG		BIT(0)
+#define DPHY_PSM_CLK_DIV(x)		((x) << 1)
+
+struct cdns_dsi_output {
+	struct mipi_dsi_device *dev;
+	struct drm_panel *panel;
+	struct drm_bridge *bridge;
+};
+
+enum cdns_dsi_input_id {
+	CDNS_SDI_INPUT,
+	CDNS_DPI_INPUT,
+	CDNS_DSC_INPUT,
+};
+
+struct cdns_dphy_cfg {
+	u8 pll_ipdiv;
+	u8 pll_opdiv;
+	u16 pll_fbdiv;
+	unsigned long lane_bps;
+	unsigned int nlanes;
+};
+
+struct cdns_dsi_cfg {
+	unsigned int hfp;
+	unsigned int hsa;
+	unsigned int hbp;
+	unsigned int hact;
+	unsigned int htotal;
+};
+
+struct cdns_dphy;
+
+enum cdns_dphy_clk_lane_cfg {
+	DPHY_CLK_CFG_LEFT_DRIVES_ALL = 0,
+	DPHY_CLK_CFG_LEFT_DRIVES_RIGHT = 1,
+	DPHY_CLK_CFG_LEFT_DRIVES_LEFT = 2,
+	DPHY_CLK_CFG_RIGHT_DRIVES_ALL = 3,
+};
+
+struct cdns_dphy_ops {
+	int (*probe)(struct cdns_dphy *dphy);
+	void (*remove)(struct cdns_dphy *dphy);
+	void (*set_psm_div)(struct cdns_dphy *dphy, u8 div);
+	void (*set_clk_lane_cfg)(struct cdns_dphy *dphy,
+				 enum cdns_dphy_clk_lane_cfg cfg);
+	void (*set_pll_cfg)(struct cdns_dphy *dphy,
+			    const struct cdns_dphy_cfg *cfg);
+	unsigned long (*get_wakeup_time_ns)(struct cdns_dphy *dphy);
+};
+
+struct cdns_dphy {
+	struct cdns_dphy_cfg cfg;
+	void __iomem *regs;
+	struct clk *psm_clk;
+	struct clk *pll_ref_clk;
+	const struct cdns_dphy_ops *ops;
+};
+
+struct cdns_dsi_input {
+	enum cdns_dsi_input_id id;
+	struct drm_bridge bridge;
+};
+
+struct cdns_dsi {
+	struct mipi_dsi_host base;
+	void __iomem *regs;
+	struct cdns_dsi_input input;
+	struct cdns_dsi_output output;
+	unsigned int direct_cmd_fifo_depth;
+	unsigned int rx_fifo_depth;
+	struct completion direct_cmd_comp;
+	struct clk *dsi_p_clk;
+	struct reset_control *dsi_p_rst;
+	struct clk *dsi_sys_clk;
+	bool link_initialized;
+	struct cdns_dphy *dphy;
+};
+
+static inline struct cdns_dsi *input_to_dsi(struct cdns_dsi_input *input)
+{
+	return container_of(input, struct cdns_dsi, input);
+}
+
+static inline struct cdns_dsi *to_cdns_dsi(struct mipi_dsi_host *host)
+{
+	return container_of(host, struct cdns_dsi, base);
+}
+
+static inline struct cdns_dsi_input *
+bridge_to_cdns_dsi_input(struct drm_bridge *bridge)
+{
+	return container_of(bridge, struct cdns_dsi_input, bridge);
+}
+
+static int cdns_dsi_get_dphy_pll_cfg(struct cdns_dphy *dphy,
+				     struct cdns_dphy_cfg *cfg,
+				     unsigned int dpi_htotal,
+				     unsigned int dpi_bpp,
+				     unsigned int dpi_hz,
+				     unsigned int dsi_htotal,
+				     unsigned int dsi_nlanes,
+				     unsigned int *dsi_hfp_ext)
+{
+	u64 dlane_bps, dlane_bps_max, fbdiv, fbdiv_max, adj_dsi_htotal;
+	unsigned long pll_ref_hz = clk_get_rate(dphy->pll_ref_clk);
+
+	memset(cfg, 0, sizeof(*cfg));
+
+	cfg->nlanes = dsi_nlanes;
+
+	if (pll_ref_hz < 9600000 || pll_ref_hz >= 150000000)
+		return -EINVAL;
+	else if (pll_ref_hz < 19200000)
+		cfg->pll_ipdiv = 1;
+	else if (pll_ref_hz < 38400000)
+		cfg->pll_ipdiv = 2;
+	else if (pll_ref_hz < 76800000)
+		cfg->pll_ipdiv = 4;
+	else
+		cfg->pll_ipdiv = 8;
+
+	/*
+	 * Make sure DSI htotal is aligned on a lane boundary when calculating
+	 * the expected data rate. This is done by extending HFP in case of
+	 * misalignment.
+	 */
+	adj_dsi_htotal = dsi_htotal;
+	if (dsi_htotal % dsi_nlanes)
+		adj_dsi_htotal += dsi_nlanes - (dsi_htotal % dsi_nlanes);
+
+	dlane_bps = (u64)dpi_hz * adj_dsi_htotal;
+
+	/* data rate in bytes/sec is not an integer, refuse the mode. */
+	if (do_div(dlane_bps, dsi_nlanes * dpi_htotal))
+		return -EINVAL;
+
+	/* data rate was in bytes/sec, convert to bits/sec. */
+	dlane_bps *= 8;
+
+	if (dlane_bps > 2500000000UL || dlane_bps < 160000000UL)
+		return -EINVAL;
+	else if (dlane_bps >= 1250000000)
+		cfg->pll_opdiv = 1;
+	else if (dlane_bps >= 630000000)
+		cfg->pll_opdiv = 2;
+	else if (dlane_bps >= 320000000)
+		cfg->pll_opdiv = 4;
+	else if (dlane_bps >= 160000000)
+		cfg->pll_opdiv = 8;
+
+	/*
+	 * Allow a deviation of 0.2% on the per-lane data rate to try to
+	 * recover a potential mismatch between DPI and PPI clks.
+	 */
+	dlane_bps_max = dlane_bps + DIV_ROUND_DOWN_ULL(dlane_bps, 500);
+	fbdiv_max = DIV_ROUND_DOWN_ULL(dlane_bps_max * 2 *
+				       cfg->pll_opdiv * cfg->pll_ipdiv,
+				       pll_ref_hz);
+	fbdiv = DIV_ROUND_UP_ULL(dlane_bps * 2 * cfg->pll_opdiv *
+				 cfg->pll_ipdiv,
+				 pll_ref_hz);
+
+	/*
+	 * Iterate over all acceptable fbdiv and try to find an adjusted DSI
+	 * htotal length providing an exact match.
+	 *
+	 * Note that we could do something even trickier by relying on the fact
+	 * that a new line is not necessarily aligned on a lane boundary, so,
+	 * by making adj_dsi_htotal non aligned on a dsi_lanes we can improve a
+	 * bit the precision. With this, the step would be
+	 *
+	 *	pll_ref_hz / (2 * opdiv * ipdiv * nlanes)
+	 *
+	 * instead of
+	 *
+	 *	pll_ref_hz / (2 * opdiv * ipdiv)
+	 *
+	 * The drawback of this approach is that we would need to make sure the
+	 * number or lines is a multiple of the realignment periodicity which is
+	 * a function of the number of lanes and the original misalignment. For
+	 * example, for NLANES = 4 and HTOTAL % NLANES = 3, it takes 4 lines
+	 * to realign on a lane:
+	 * LINE 0: expected number of bytes, starts emitting first byte of
+	 *	   LINE 1 on LANE 3
+	 * LINE 1: expected number of bytes, starts emitting first 2 bytes of
+	 *	   LINE 2 on LANES 2 and 3
+	 * LINE 2: expected number of bytes, starts emitting first 3 bytes of
+	 *	   of LINE 3 on LANES 1, 2 and 3
+	 * LINE 3: one byte less, now things are realigned on LANE 0 for LINE 4
+	 *
+	 * I figured this extra complexity was not worth the benefit, but if
+	 * someone really has unfixable mismatch, that would be something to
+	 * investigate.
+	 */
+	for (; fbdiv <= fbdiv_max; fbdiv++) {
+		u32 rem;
+
+		adj_dsi_htotal = (u64)fbdiv * pll_ref_hz * dsi_nlanes *
+				 dpi_htotal;
+
+		/*
+		 * Do the division in 2 steps to avoid an overflow on the
+		 * divider.
+		 */
+		rem = do_div(adj_dsi_htotal, dpi_hz);
+		if (rem)
+			continue;
+
+		rem = do_div(adj_dsi_htotal,
+			     cfg->pll_opdiv * cfg->pll_ipdiv * 2 * 8);
+		if (rem)
+			continue;
+
+		cfg->pll_fbdiv = fbdiv;
+		*dsi_hfp_ext = adj_dsi_htotal - dsi_htotal;
+		break;
+	}
+
+	/* No match, let's just reject the display mode. */
+	if (!cfg->pll_fbdiv)
+		return -EINVAL;
+
+	dlane_bps = DIV_ROUND_DOWN_ULL((u64)dpi_hz * adj_dsi_htotal * 8,
+				       dsi_nlanes * dpi_htotal);
+	cfg->lane_bps = dlane_bps;
+
+	return 0;
+}
+
+static int cdns_dphy_setup_psm(struct cdns_dphy *dphy)
+{
+	unsigned long psm_clk_hz = clk_get_rate(dphy->psm_clk);
+	unsigned long psm_div;
+
+	if (!psm_clk_hz || psm_clk_hz > 100000000)
+		return -EINVAL;
+
+	psm_div = DIV_ROUND_CLOSEST(psm_clk_hz, 1000000);
+	if (dphy->ops->set_psm_div)
+		dphy->ops->set_psm_div(dphy, psm_div);
+
+	return 0;
+}
+
+static void cdns_dphy_set_clk_lane_cfg(struct cdns_dphy *dphy,
+				       enum cdns_dphy_clk_lane_cfg cfg)
+{
+	if (dphy->ops->set_clk_lane_cfg)
+		dphy->ops->set_clk_lane_cfg(dphy, cfg);
+}
+
+static void cdns_dphy_set_pll_cfg(struct cdns_dphy *dphy,
+				  const struct cdns_dphy_cfg *cfg)
+{
+	if (dphy->ops->set_pll_cfg)
+		dphy->ops->set_pll_cfg(dphy, cfg);
+}
+
+static unsigned long cdns_dphy_get_wakeup_time_ns(struct cdns_dphy *dphy)
+{
+	return dphy->ops->get_wakeup_time_ns(dphy);
+}
+
+static unsigned int dpi_to_dsi_timing(unsigned int dpi_timing,
+				      unsigned int dpi_bpp,
+				      unsigned int dsi_pkt_overhead)
+{
+	unsigned int dsi_timing = DIV_ROUND_UP(dpi_timing * dpi_bpp, 8);
+
+	if (dsi_timing < dsi_pkt_overhead)
+		dsi_timing = 0;
+	else
+		dsi_timing -= dsi_pkt_overhead;
+
+	return dsi_timing;
+}
+
+static int cdns_dsi_mode2cfg(struct cdns_dsi *dsi,
+			     const struct drm_display_mode *mode,
+			     struct cdns_dsi_cfg *dsi_cfg,
+			     struct cdns_dphy_cfg *dphy_cfg,
+			     bool mode_valid_check)
+{
+	unsigned long dsi_htotal = 0, dsi_hss_hsa_hse_hbp = 0;
+	struct cdns_dsi_output *output = &dsi->output;
+	unsigned int dsi_hfp_ext = 0, dpi_hfp, tmp;
+	bool sync_pulse = false;
+	int bpp, nlanes, ret;
+
+	memset(dsi_cfg, 0, sizeof(*dsi_cfg));
+
+	if (output->dev->mode_flags & MIPI_DSI_MODE_VIDEO_SYNC_PULSE)
+		sync_pulse = true;
+
+	bpp = mipi_dsi_pixel_format_to_bpp(output->dev->format);
+	nlanes = output->dev->lanes;
+
+	if (mode_valid_check)
+		tmp = mode->htotal -
+		      (sync_pulse ? mode->hsync_end : mode->hsync_start);
+	else
+		tmp = mode->crtc_htotal -
+		      (sync_pulse ?
+		       mode->crtc_hsync_end : mode->crtc_hsync_start);
+
+	dsi_cfg->hbp = dpi_to_dsi_timing(tmp, bpp, DSI_HBP_FRAME_OVERHEAD);
+	dsi_htotal += dsi_cfg->hbp + DSI_HBP_FRAME_OVERHEAD;
+	dsi_hss_hsa_hse_hbp += dsi_cfg->hbp + DSI_HBP_FRAME_OVERHEAD;
+
+	if (sync_pulse) {
+		if (mode_valid_check)
+			tmp = mode->hsync_end - mode->hsync_start;
+		else
+			tmp = mode->crtc_hsync_end - mode->crtc_hsync_start;
+
+		dsi_cfg->hsa = dpi_to_dsi_timing(tmp, bpp,
+						 DSI_HSA_FRAME_OVERHEAD);
+		dsi_htotal += dsi_cfg->hsa + DSI_HSA_FRAME_OVERHEAD;
+		dsi_hss_hsa_hse_hbp += dsi_cfg->hsa + DSI_HSA_FRAME_OVERHEAD;
+	}
+
+	dsi_cfg->hact = dpi_to_dsi_timing(mode_valid_check ?
+					  mode->hdisplay : mode->crtc_hdisplay,
+					  bpp, 0);
+	dsi_htotal += dsi_cfg->hact;
+
+	if (mode_valid_check)
+		dpi_hfp = mode->hsync_start - mode->hdisplay;
+	else
+		dpi_hfp = mode->crtc_hsync_start - mode->crtc_hdisplay;
+
+	dsi_cfg->hfp = dpi_to_dsi_timing(dpi_hfp, bpp, DSI_HFP_FRAME_OVERHEAD);
+	dsi_htotal += dsi_cfg->hfp + DSI_HFP_FRAME_OVERHEAD;
+
+	if (mode_valid_check)
+		ret = cdns_dsi_get_dphy_pll_cfg(dsi->dphy, dphy_cfg,
+						mode->htotal, bpp,
+						mode->clock * 1000,
+						dsi_htotal, nlanes,
+						&dsi_hfp_ext);
+	else
+		ret = cdns_dsi_get_dphy_pll_cfg(dsi->dphy, dphy_cfg,
+						mode->crtc_htotal, bpp,
+						mode->crtc_clock * 1000,
+						dsi_htotal, nlanes,
+						&dsi_hfp_ext);
+
+	if (ret)
+		return ret;
+
+	dsi_cfg->hfp += dsi_hfp_ext;
+	dsi_htotal += dsi_hfp_ext;
+	dsi_cfg->htotal = dsi_htotal;
+
+	/*
+	 * Make sure DPI(HFP) > DSI(HSS+HSA+HSE+HBP) to guarantee that the FIFO
+	 * is empty before we start a receiving a new line on the DPI
+	 * interface.
+	 */
+	if ((u64)dphy_cfg->lane_bps * dpi_hfp * nlanes <
+	    (u64)dsi_hss_hsa_hse_hbp *
+	    (mode_valid_check ? mode->clock : mode->crtc_clock) * 1000)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int cdns_dsi_bridge_attach(struct drm_bridge *bridge)
+{
+	struct cdns_dsi_input *input = bridge_to_cdns_dsi_input(bridge);
+	struct cdns_dsi *dsi = input_to_dsi(input);
+	struct cdns_dsi_output *output = &dsi->output;
+
+	if (!drm_core_check_feature(bridge->dev, DRIVER_ATOMIC)) {
+		dev_err(dsi->base.dev,
+			"cdns-dsi driver is only compatible with DRM devices supporting atomic updates");
+		return -ENOTSUPP;
+	}
+
+	return drm_bridge_attach(bridge->encoder, output->bridge, bridge);
+}
+
+static enum drm_mode_status
+cdns_dsi_bridge_mode_valid(struct drm_bridge *bridge,
+			   const struct drm_display_mode *mode)
+{
+	struct cdns_dsi_input *input = bridge_to_cdns_dsi_input(bridge);
+	struct cdns_dsi *dsi = input_to_dsi(input);
+	struct cdns_dsi_output *output = &dsi->output;
+	struct cdns_dphy_cfg dphy_cfg;
+	struct cdns_dsi_cfg dsi_cfg;
+	int bpp, nlanes, ret;
+
+	/*
+	 * VFP_DSI should be less than VFP_DPI and VFP_DSI should be at
+	 * least 1.
+	 */
+	if (mode->vtotal - mode->vsync_end < 2)
+		return MODE_V_ILLEGAL;
+
+	/* VSA_DSI = VSA_DPI and must be at least 2. */
+	if (mode->vsync_end - mode->vsync_start < 2)
+		return MODE_V_ILLEGAL;
+
+	/* HACT must be 32-bits aligned. */
+	bpp = mipi_dsi_pixel_format_to_bpp(output->dev->format);
+	if ((mode->hdisplay * bpp) % 32)
+		return MODE_H_ILLEGAL;
+
+	nlanes = output->dev->lanes;
+
+	ret = cdns_dsi_mode2cfg(dsi, mode, &dsi_cfg, &dphy_cfg, true);
+	if (ret)
+		return MODE_CLOCK_RANGE;
+
+	return MODE_OK;
+}
+
+static void cdns_dsi_bridge_disable(struct drm_bridge *bridge)
+{
+	struct cdns_dsi_input *input = bridge_to_cdns_dsi_input(bridge);
+	struct cdns_dsi *dsi = input_to_dsi(input);
+	u32 val;
+
+	val = readl(dsi->regs + MCTL_MAIN_DATA_CTL);
+	val &= ~(IF_VID_SELECT_MASK | IF_VID_MODE | VID_EN | HOST_EOT_GEN |
+		 DISP_EOT_GEN);
+	writel(val, dsi->regs + MCTL_MAIN_DATA_CTL);
+
+	val = readl(dsi->regs + MCTL_MAIN_EN) & ~IF_EN(input->id);
+	writel(val, dsi->regs + MCTL_MAIN_EN);
+	pm_runtime_put(dsi->base.dev);
+}
+
+static void cdns_dsi_hs_init(struct cdns_dsi *dsi,
+			     const struct cdns_dphy_cfg *dphy_cfg)
+{
+	u32 status;
+
+	/*
+	 * Power all internal DPHY blocks down and maintain their reset line
+	 * asserted before changing the DPHY config.
+	 */
+	writel(DPHY_CMN_PSO | DPHY_PLL_PSO | DPHY_ALL_D_PDN | DPHY_C_PDN |
+	       DPHY_CMN_PDN | DPHY_PLL_PDN,
+	       dsi->regs + MCTL_DPHY_CFG0);
+
+	/*
+	 * Configure the internal PSM clk divider so that the DPHY has a
+	 * 1MHz clk (or something close).
+	 */
+	WARN_ON_ONCE(cdns_dphy_setup_psm(dsi->dphy));
+
+	/*
+	 * Configure attach clk lanes to data lanes: the DPHY has 2 clk lanes
+	 * and 8 data lanes, each clk lane can be attache different set of
+	 * data lanes. The 2 groups are named 'left' and 'right', so here we
+	 * just say that we want the 'left' clk lane to drive the 'left' data
+	 * lanes.
+	 */
+	cdns_dphy_set_clk_lane_cfg(dsi->dphy, DPHY_CLK_CFG_LEFT_DRIVES_LEFT);
+
+	/*
+	 * Configure the DPHY PLL that will be used to generate the TX byte
+	 * clk.
+	 */
+	cdns_dphy_set_pll_cfg(dsi->dphy, dphy_cfg);
+
+	/* Start TX state machine. */
+	writel(DPHY_CMN_SSM_EN | DPHY_CMN_TX_MODE_EN,
+	       dsi->dphy->regs + DPHY_CMN_SSM);
+
+	/* Activate the PLL and wait until it's locked. */
+	writel(PLL_LOCKED, dsi->regs + MCTL_MAIN_STS_CLR);
+	writel(DPHY_CMN_PSO | DPHY_ALL_D_PDN | DPHY_C_PDN | DPHY_CMN_PDN,
+	       dsi->regs + MCTL_DPHY_CFG0);
+	WARN_ON_ONCE(readl_poll_timeout(dsi->regs + MCTL_MAIN_STS, status,
+					status & PLL_LOCKED, 100, 100));
+	/* De-assert data and clock reset lines. */
+	writel(DPHY_CMN_PSO | DPHY_ALL_D_PDN | DPHY_C_PDN | DPHY_CMN_PDN |
+	       DPHY_D_RSTB(dphy_cfg->nlanes) | DPHY_C_RSTB,
+	       dsi->regs + MCTL_DPHY_CFG0);
+}
+
+static void cdns_dsi_init_link(struct cdns_dsi *dsi)
+{
+	struct cdns_dsi_output *output = &dsi->output;
+	unsigned long sysclk_period, ulpout;
+	u32 val;
+	int i;
+
+	if (dsi->link_initialized)
+		return;
+
+	val = 0;
+	for (i = 1; i < output->dev->lanes; i++)
+		val |= DATA_LANE_EN(i);
+
+	if (!(output->dev->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS))
+		val |= CLK_CONTINUOUS;
+
+	writel(val, dsi->regs + MCTL_MAIN_PHY_CTL);
+
+	/* ULPOUT should be set to 1ms and is expressed in sysclk cycles. */
+	sysclk_period = NSEC_PER_SEC / clk_get_rate(dsi->dsi_sys_clk);
+	ulpout = DIV_ROUND_UP(NSEC_PER_MSEC, sysclk_period);
+	writel(CLK_LANE_ULPOUT_TIME(ulpout) | DATA_LANE_ULPOUT_TIME(ulpout),
+	       dsi->regs + MCTL_ULPOUT_TIME);
+
+	writel(LINK_EN, dsi->regs + MCTL_MAIN_DATA_CTL);
+
+	val = CLK_LANE_EN | PLL_START;
+	for (i = 0; i < output->dev->lanes; i++)
+		val |= DATA_LANE_START(i);
+
+	writel(val, dsi->regs + MCTL_MAIN_EN);
+
+	dsi->link_initialized = true;
+}
+
+static void cdns_dsi_bridge_enable(struct drm_bridge *bridge)
+{
+	struct cdns_dsi_input *input = bridge_to_cdns_dsi_input(bridge);
+	struct cdns_dsi *dsi = input_to_dsi(input);
+	struct cdns_dsi_output *output = &dsi->output;
+	struct drm_display_mode *mode;
+	struct cdns_dphy_cfg dphy_cfg;
+	unsigned long tx_byte_period;
+	struct cdns_dsi_cfg dsi_cfg;
+	u32 tmp, reg_wakeup, div;
+	int bpp, nlanes;
+
+	if (WARN_ON(pm_runtime_get_sync(dsi->base.dev) < 0))
+		return;
+
+	mode = &bridge->encoder->crtc->state->adjusted_mode;
+	bpp = mipi_dsi_pixel_format_to_bpp(output->dev->format);
+	nlanes = output->dev->lanes;
+
+	WARN_ON_ONCE(cdns_dsi_mode2cfg(dsi, mode, &dsi_cfg, &dphy_cfg, false));
+
+	cdns_dsi_hs_init(dsi, &dphy_cfg);
+	cdns_dsi_init_link(dsi);
+
+	writel(HBP_LEN(dsi_cfg.hbp) | HSA_LEN(dsi_cfg.hsa),
+	       dsi->regs + VID_HSIZE1);
+	writel(HFP_LEN(dsi_cfg.hfp) | HACT_LEN(dsi_cfg.hact),
+	       dsi->regs + VID_HSIZE2);
+
+	writel(VBP_LEN(mode->crtc_vtotal - mode->crtc_vsync_end - 1) |
+	       VFP_LEN(mode->crtc_vsync_start - mode->crtc_vdisplay) |
+	       VSA_LEN(mode->crtc_vsync_end - mode->crtc_vsync_start + 1),
+	       dsi->regs + VID_VSIZE1);
+	writel(mode->crtc_vdisplay, dsi->regs + VID_VSIZE2);
+
+	tmp = dsi_cfg.htotal -
+	      (dsi_cfg.hsa + DSI_BLANKING_FRAME_OVERHEAD +
+	       DSI_HSA_FRAME_OVERHEAD);
+	writel(BLK_LINE_PULSE_PKT_LEN(tmp), dsi->regs + VID_BLKSIZE2);
+	if (output->dev->mode_flags & MIPI_DSI_MODE_VIDEO_SYNC_PULSE)
+		writel(MAX_LINE_LIMIT(tmp - DSI_NULL_FRAME_OVERHEAD),
+		       dsi->regs + VID_VCA_SETTING2);
+
+	tmp = dsi_cfg.htotal -
+	      (DSI_HSS_VSS_VSE_FRAME_OVERHEAD + DSI_BLANKING_FRAME_OVERHEAD);
+	writel(BLK_LINE_EVENT_PKT_LEN(tmp), dsi->regs + VID_BLKSIZE1);
+	if (!(output->dev->mode_flags & MIPI_DSI_MODE_VIDEO_SYNC_PULSE))
+		writel(MAX_LINE_LIMIT(tmp - DSI_NULL_FRAME_OVERHEAD),
+		       dsi->regs + VID_VCA_SETTING2);
+
+	tmp = DIV_ROUND_UP(dsi_cfg.htotal, nlanes) -
+	      DIV_ROUND_UP(dsi_cfg.hsa, nlanes);
+
+	if (!(output->dev->mode_flags & MIPI_DSI_MODE_EOT_PACKET))
+		tmp -= DIV_ROUND_UP(DSI_EOT_PKT_SIZE, nlanes);
+
+	tx_byte_period = DIV_ROUND_DOWN_ULL((u64)NSEC_PER_SEC * 8,
+					    dphy_cfg.lane_bps);
+	reg_wakeup = cdns_dphy_get_wakeup_time_ns(dsi->dphy) /
+		     tx_byte_period;
+	writel(REG_WAKEUP_TIME(reg_wakeup) | REG_LINE_DURATION(tmp),
+	       dsi->regs + VID_DPHY_TIME);
+
+	/*
+	 * HSTX and LPRX timeouts are both expressed in TX byte clk cycles and
+	 * both should be set to at least the time it takes to transmit a
+	 * frame.
+	 */
+	tmp = NSEC_PER_SEC / drm_mode_vrefresh(mode);
+	tmp /= tx_byte_period;
+
+	for (div = 0; div <= CLK_DIV_MAX; div++) {
+		if (tmp <= HSTX_TIMEOUT_MAX)
+			break;
+
+		tmp >>= 1;
+	}
+
+	if (tmp > HSTX_TIMEOUT_MAX)
+		tmp = HSTX_TIMEOUT_MAX;
+
+	writel(CLK_DIV(div) | HSTX_TIMEOUT(tmp),
+	       dsi->regs + MCTL_DPHY_TIMEOUT1);
+
+	writel(LPRX_TIMEOUT(tmp), dsi->regs + MCTL_DPHY_TIMEOUT2);
+
+	if (output->dev->mode_flags & MIPI_DSI_MODE_VIDEO) {
+		switch (output->dev->format) {
+		case MIPI_DSI_FMT_RGB888:
+			tmp = VID_PIXEL_MODE_RGB888 |
+			      VID_DATATYPE(MIPI_DSI_PACKED_PIXEL_STREAM_24);
+			break;
+
+		case MIPI_DSI_FMT_RGB666:
+			tmp = VID_PIXEL_MODE_RGB666 |
+			      VID_DATATYPE(MIPI_DSI_PIXEL_STREAM_3BYTE_18);
+			break;
+
+		case MIPI_DSI_FMT_RGB666_PACKED:
+			tmp = VID_PIXEL_MODE_RGB666_PACKED |
+			      VID_DATATYPE(MIPI_DSI_PACKED_PIXEL_STREAM_18);
+			break;
+
+		case MIPI_DSI_FMT_RGB565:
+			tmp = VID_PIXEL_MODE_RGB565 |
+			      VID_DATATYPE(MIPI_DSI_PACKED_PIXEL_STREAM_16);
+			break;
+
+		default:
+			dev_err(dsi->base.dev, "Unsupported DSI format\n");
+			return;
+		}
+
+		if (output->dev->mode_flags & MIPI_DSI_MODE_VIDEO_SYNC_PULSE)
+			tmp |= SYNC_PULSE_ACTIVE | SYNC_PULSE_HORIZONTAL;
+
+		tmp |= REG_BLKLINE_MODE(REG_BLK_MODE_BLANKING_PKT) |
+		       REG_BLKEOL_MODE(REG_BLK_MODE_BLANKING_PKT) |
+		       RECOVERY_MODE(RECOVERY_MODE_NEXT_HSYNC) |
+		       VID_IGNORE_MISS_VSYNC;
+
+		writel(tmp, dsi->regs + VID_MAIN_CTL);
+	}
+
+	tmp = readl(dsi->regs + MCTL_MAIN_DATA_CTL);
+	tmp &= ~(IF_VID_SELECT_MASK | HOST_EOT_GEN | IF_VID_MODE);
+
+	if (!(output->dev->mode_flags & MIPI_DSI_MODE_EOT_PACKET))
+		tmp |= HOST_EOT_GEN;
+
+	if (output->dev->mode_flags & MIPI_DSI_MODE_VIDEO)
+		tmp |= IF_VID_MODE | IF_VID_SELECT(input->id) | VID_EN;
+
+	writel(tmp, dsi->regs + MCTL_MAIN_DATA_CTL);
+
+	tmp = readl(dsi->regs + MCTL_MAIN_EN) | IF_EN(input->id);
+	writel(tmp, dsi->regs + MCTL_MAIN_EN);
+}
+
+static const struct drm_bridge_funcs cdns_dsi_bridge_funcs = {
+	.attach = cdns_dsi_bridge_attach,
+	.mode_valid = cdns_dsi_bridge_mode_valid,
+	.disable = cdns_dsi_bridge_disable,
+	.enable = cdns_dsi_bridge_enable,
+};
+
+static int cdns_dsi_attach(struct mipi_dsi_host *host,
+			   struct mipi_dsi_device *dev)
+{
+	struct cdns_dsi *dsi = to_cdns_dsi(host);
+	struct cdns_dsi_output *output = &dsi->output;
+	struct cdns_dsi_input *input = &dsi->input;
+	struct drm_bridge *bridge;
+	struct drm_panel *panel;
+	struct device_node *np;
+	int ret;
+
+	/*
+	 * We currently do not support connecting several DSI devices to the
+	 * same host. In order to support that we'd need the DRM bridge
+	 * framework to allow dynamic reconfiguration of the bridge chain.
+	 */
+	if (output->dev)
+		return -EBUSY;
+
+	/* We do not support burst mode yet. */
+	if (dev->mode_flags & MIPI_DSI_MODE_VIDEO_BURST)
+		return -ENOTSUPP;
+
+	/*
+	 * The host <-> device link might be described using an OF-graph
+	 * representation, in this case we extract the device of_node from
+	 * this representation, otherwise we use dsidev->dev.of_node which
+	 * should have been filled by the core.
+	 */
+	np = of_graph_get_remote_node(dsi->base.dev->of_node, DSI_OUTPUT_PORT,
+				      dev->channel);
+	if (!np)
+		np = of_node_get(dev->dev.of_node);
+
+	panel = of_drm_find_panel(np);
+	if (panel) {
+		bridge = drm_panel_bridge_add(panel, DRM_MODE_CONNECTOR_DSI);
+	} else {
+		bridge = of_drm_find_bridge(dev->dev.of_node);
+		if (!bridge)
+			bridge = ERR_PTR(-EINVAL);
+	}
+
+	of_node_put(np);
+
+	if (IS_ERR(bridge)) {
+		ret = PTR_ERR(bridge);
+		dev_err(host->dev, "failed to add DSI device %s (err = %d)",
+			dev->name, ret);
+		return ret;
+	}
+
+	output->dev = dev;
+	output->bridge = bridge;
+	output->panel = panel;
+
+	/*
+	 * The DSI output has been properly configured, we can now safely
+	 * register the input to the bridge framework so that it can take place
+	 * in a display pipeline.
+	 */
+	drm_bridge_add(&input->bridge);
+
+	return 0;
+}
+
+static int cdns_dsi_detach(struct mipi_dsi_host *host,
+			   struct mipi_dsi_device *dev)
+{
+	struct cdns_dsi *dsi = to_cdns_dsi(host);
+	struct cdns_dsi_output *output = &dsi->output;
+	struct cdns_dsi_input *input = &dsi->input;
+
+	drm_bridge_remove(&input->bridge);
+	if (output->panel)
+		drm_panel_bridge_remove(output->bridge);
+
+	return 0;
+}
+
+static irqreturn_t cdns_dsi_interrupt(int irq, void *data)
+{
+	struct cdns_dsi *dsi = data;
+	irqreturn_t ret = IRQ_NONE;
+	u32 flag, ctl;
+
+	flag = readl(dsi->regs + DIRECT_CMD_STS_FLAG);
+	if (flag) {
+		ctl = readl(dsi->regs + DIRECT_CMD_STS_CTL);
+		ctl &= ~flag;
+		writel(ctl, dsi->regs + DIRECT_CMD_STS_CTL);
+		complete(&dsi->direct_cmd_comp);
+		ret = IRQ_HANDLED;
+	}
+
+	return ret;
+}
+
+static ssize_t cdns_dsi_transfer(struct mipi_dsi_host *host,
+				 const struct mipi_dsi_msg *msg)
+{
+	struct cdns_dsi *dsi = to_cdns_dsi(host);
+	u32 cmd, sts, val, wait = WRITE_COMPLETED, ctl = 0;
+	struct mipi_dsi_packet packet;
+	int ret, i, tx_len, rx_len;
+
+	ret = pm_runtime_get_sync(host->dev);
+	if (ret < 0)
+		return ret;
+
+	cdns_dsi_init_link(dsi);
+
+	ret = mipi_dsi_create_packet(&packet, msg);
+	if (ret)
+		goto out;
+
+	tx_len = msg->tx_buf ? msg->tx_len : 0;
+	rx_len = msg->rx_buf ? msg->rx_len : 0;
+
+	/* For read operations, the maximum TX len is 2. */
+	if (rx_len && tx_len > 2) {
+		ret = -ENOTSUPP;
+		goto out;
+	}
+
+	/* TX len is limited by the CMD FIFO depth. */
+	if (tx_len > dsi->direct_cmd_fifo_depth) {
+		ret = -ENOTSUPP;
+		goto out;
+	}
+
+	/* RX len is limited by the RX FIFO depth. */
+	if (rx_len > dsi->rx_fifo_depth) {
+		ret = -ENOTSUPP;
+		goto out;
+	}
+
+	cmd = CMD_SIZE(tx_len) | CMD_VCHAN_ID(msg->channel) |
+	      CMD_DATATYPE(msg->type);
+
+	if (msg->flags & MIPI_DSI_MSG_USE_LPM)
+		cmd |= CMD_LP_EN;
+
+	if (mipi_dsi_packet_format_is_long(msg->type))
+		cmd |= CMD_LONG;
+
+	if (rx_len) {
+		cmd |= READ_CMD;
+		wait = READ_COMPLETED_WITH_ERR | READ_COMPLETED;
+		ctl = READ_EN | BTA_EN;
+	} else if (msg->flags & MIPI_DSI_MSG_REQ_ACK) {
+		cmd |= BTA_REQ;
+		wait = ACK_WITH_ERR_RCVD | ACK_RCVD;
+		ctl = BTA_EN;
+	}
+
+	writel(readl(dsi->regs + MCTL_MAIN_DATA_CTL) | ctl,
+	       dsi->regs + MCTL_MAIN_DATA_CTL);
+
+	writel(cmd, dsi->regs + DIRECT_CMD_MAIN_SETTINGS);
+
+	for (i = 0; i < tx_len; i += 4) {
+		const u8 *buf = msg->tx_buf;
+		int j;
+
+		val = 0;
+		for (j = 0; j < 4 && j + i < tx_len; j++)
+			val |= (u32)buf[i + j] << (8 * j);
+
+		writel(val, dsi->regs + DIRECT_CMD_WRDATA);
+	}
+
+	/* Clear status flags before sending the command. */
+	writel(wait, dsi->regs + DIRECT_CMD_STS_CLR);
+	writel(wait, dsi->regs + DIRECT_CMD_STS_CTL);
+	reinit_completion(&dsi->direct_cmd_comp);
+	writel(0, dsi->regs + DIRECT_CMD_SEND);
+
+	wait_for_completion_timeout(&dsi->direct_cmd_comp,
+				    msecs_to_jiffies(1000));
+
+	sts = readl(dsi->regs + DIRECT_CMD_STS);
+	writel(wait, dsi->regs + DIRECT_CMD_STS_CLR);
+	writel(0, dsi->regs + DIRECT_CMD_STS_CTL);
+
+	writel(readl(dsi->regs + MCTL_MAIN_DATA_CTL) & ~ctl,
+	       dsi->regs + MCTL_MAIN_DATA_CTL);
+
+	/* We did not receive the events we were waiting for. */
+	if (!(sts & wait)) {
+		ret = -ETIMEDOUT;
+		goto out;
+	}
+
+	/* 'READ' or 'WRITE with ACK' failed. */
+	if (sts & (READ_COMPLETED_WITH_ERR | ACK_WITH_ERR_RCVD)) {
+		ret = -EIO;
+		goto out;
+	}
+
+	for (i = 0; i < rx_len; i += 4) {
+		u8 *buf = msg->rx_buf;
+		int j;
+
+		val = readl(dsi->regs + DIRECT_CMD_RDDATA);
+		for (j = 0; j < 4 && j + i < rx_len; j++)
+			buf[i + j] = val >> (8 * j);
+	}
+
+out:
+	pm_runtime_put(host->dev);
+	return ret;
+}
+
+static const struct mipi_dsi_host_ops cdns_dsi_ops = {
+	.attach = cdns_dsi_attach,
+	.detach = cdns_dsi_detach,
+	.transfer = cdns_dsi_transfer,
+};
+
+static int cdns_dsi_resume(struct device *dev)
+{
+	struct cdns_dsi *dsi = dev_get_drvdata(dev);
+
+	reset_control_deassert(dsi->dsi_p_rst);
+	clk_prepare_enable(dsi->dsi_p_clk);
+	clk_prepare_enable(dsi->dsi_sys_clk);
+	clk_prepare_enable(dsi->dphy->psm_clk);
+	clk_prepare_enable(dsi->dphy->pll_ref_clk);
+
+	return 0;
+}
+
+static int cdns_dsi_suspend(struct device *dev)
+{
+	struct cdns_dsi *dsi = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(dsi->dphy->pll_ref_clk);
+	clk_disable_unprepare(dsi->dphy->psm_clk);
+	clk_disable_unprepare(dsi->dsi_sys_clk);
+	clk_disable_unprepare(dsi->dsi_p_clk);
+	reset_control_assert(dsi->dsi_p_rst);
+	dsi->link_initialized = false;
+	return 0;
+}
+
+static UNIVERSAL_DEV_PM_OPS(cdns_dsi_pm_ops, cdns_dsi_suspend, cdns_dsi_resume,
+			    NULL);
+
+static unsigned long cdns_dphy_ref_get_wakeup_time_ns(struct cdns_dphy *dphy)
+{
+	/* Default wakeup time is 800 ns (in a simulated environment). */
+	return 800;
+}
+
+static void cdns_dphy_ref_set_pll_cfg(struct cdns_dphy *dphy,
+				      const struct cdns_dphy_cfg *cfg)
+{
+	u32 fbdiv_low, fbdiv_high;
+
+	fbdiv_low = (cfg->pll_fbdiv / 4) - 2;
+	fbdiv_high = cfg->pll_fbdiv - fbdiv_low - 2;
+
+	writel(DPHY_CMN_IPDIV_FROM_REG | DPHY_CMN_OPDIV_FROM_REG |
+	       DPHY_CMN_IPDIV(cfg->pll_ipdiv) |
+	       DPHY_CMN_OPDIV(cfg->pll_opdiv),
+	       dphy->regs + DPHY_CMN_OPIPDIV);
+	writel(DPHY_CMN_FBDIV_FROM_REG |
+	       DPHY_CMN_FBDIV_VAL(fbdiv_low, fbdiv_high),
+	       dphy->regs + DPHY_CMN_FBDIV);
+	writel(DPHY_CMN_PWM_HIGH(6) | DPHY_CMN_PWM_LOW(0x101) |
+	       DPHY_CMN_PWM_DIV(0x8),
+	       dphy->regs + DPHY_CMN_PWM);
+}
+
+static void cdns_dphy_ref_set_psm_div(struct cdns_dphy *dphy, u8 div)
+{
+	writel(DPHY_PSM_CFG_FROM_REG | DPHY_PSM_CLK_DIV(div),
+	       dphy->regs + DPHY_PSM_CFG);
+}
+
+/*
+ * This is the reference implementation of DPHY hooks. Specific integration of
+ * this IP may have to re-implement some of them depending on how they decided
+ * to wire things in the SoC.
+ */
+static const struct cdns_dphy_ops ref_dphy_ops = {
+	.get_wakeup_time_ns = cdns_dphy_ref_get_wakeup_time_ns,
+	.set_pll_cfg = cdns_dphy_ref_set_pll_cfg,
+	.set_psm_div = cdns_dphy_ref_set_psm_div,
+};
+
+static const struct of_device_id cdns_dphy_of_match[] = {
+	{ .compatible = "cdns,dphy", .data = &ref_dphy_ops },
+	{ /* sentinel */ },
+};
+
+static struct cdns_dphy *cdns_dphy_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *match;
+	struct cdns_dphy *dphy;
+	struct of_phandle_args args;
+	struct resource res;
+	int ret;
+
+	ret = of_parse_phandle_with_args(pdev->dev.of_node, "phys",
+					 "#phy-cells", 0, &args);
+	if (ret)
+		return ERR_PTR(-ENOENT);
+
+	match = of_match_node(cdns_dphy_of_match, args.np);
+	if (!match || !match->data)
+		return ERR_PTR(-EINVAL);
+
+	dphy = devm_kzalloc(&pdev->dev, sizeof(*dphy), GFP_KERNEL);
+	if (!dphy)
+		return ERR_PTR(-ENOMEM);
+
+	dphy->ops = match->data;
+
+	ret = of_address_to_resource(args.np, 0, &res);
+	if (ret)
+		return ERR_PTR(ret);
+
+	dphy->regs = devm_ioremap_resource(&pdev->dev, &res);
+	if (IS_ERR(dphy->regs))
+		return ERR_CAST(dphy->regs);
+
+	dphy->psm_clk = of_clk_get_by_name(args.np, "psm");
+	if (IS_ERR(dphy->psm_clk))
+		return ERR_CAST(dphy->psm_clk);
+
+	dphy->pll_ref_clk = of_clk_get_by_name(args.np, "pll_ref");
+	if (IS_ERR(dphy->pll_ref_clk)) {
+		ret = PTR_ERR(dphy->pll_ref_clk);
+		goto err_put_psm_clk;
+	}
+
+	if (dphy->ops->probe) {
+		ret = dphy->ops->probe(dphy);
+		if (ret)
+			goto err_put_pll_ref_clk;
+	}
+
+	return dphy;
+
+err_put_pll_ref_clk:
+	clk_put(dphy->pll_ref_clk);
+
+err_put_psm_clk:
+	clk_put(dphy->psm_clk);
+
+	return ERR_PTR(ret);
+}
+
+static void cdns_dphy_remove(struct cdns_dphy *dphy)
+{
+	if (dphy->ops->remove)
+		dphy->ops->remove(dphy);
+
+	clk_put(dphy->pll_ref_clk);
+	clk_put(dphy->psm_clk);
+}
+
+static int cdns_dsi_drm_probe(struct platform_device *pdev)
+{
+	struct cdns_dsi *dsi;
+	struct cdns_dsi_input *input;
+	struct resource *res;
+	int ret, irq;
+	u32 val;
+
+	dsi = devm_kzalloc(&pdev->dev, sizeof(*dsi), GFP_KERNEL);
+	if (!dsi)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, dsi);
+
+	input = &dsi->input;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	dsi->regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(dsi->regs))
+		return PTR_ERR(dsi->regs);
+
+	dsi->dsi_p_clk = devm_clk_get(&pdev->dev, "dsi_p_clk");
+	if (IS_ERR(dsi->dsi_p_clk))
+		return PTR_ERR(dsi->dsi_p_clk);
+
+	dsi->dsi_p_rst = devm_reset_control_get_optional_exclusive(&pdev->dev,
+								"dsi_p_rst");
+	if (IS_ERR(dsi->dsi_p_rst))
+		return PTR_ERR(dsi->dsi_p_rst);
+
+	dsi->dsi_sys_clk = devm_clk_get(&pdev->dev, "dsi_sys_clk");
+	if (IS_ERR(dsi->dsi_sys_clk))
+		return PTR_ERR(dsi->dsi_sys_clk);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	dsi->dphy = cdns_dphy_probe(pdev);
+	if (IS_ERR(dsi->dphy))
+		return PTR_ERR(dsi->dphy);
+
+	ret = clk_prepare_enable(dsi->dsi_p_clk);
+	if (ret)
+		goto err_remove_dphy;
+
+	val = readl(dsi->regs + ID_REG);
+	if (REV_VENDOR_ID(val) != 0xcad) {
+		dev_err(&pdev->dev, "invalid vendor id\n");
+		ret = -EINVAL;
+		goto err_disable_pclk;
+	}
+
+	val = readl(dsi->regs + IP_CONF);
+	dsi->direct_cmd_fifo_depth = 1 << (DIRCMD_FIFO_DEPTH(val) + 2);
+	dsi->rx_fifo_depth = RX_FIFO_DEPTH(val);
+	init_completion(&dsi->direct_cmd_comp);
+
+	writel(0, dsi->regs + MCTL_MAIN_DATA_CTL);
+	writel(0, dsi->regs + MCTL_MAIN_EN);
+	writel(0, dsi->regs + MCTL_MAIN_PHY_CTL);
+
+	/*
+	 * We only support the DPI input, so force input->id to
+	 * CDNS_DPI_INPUT.
+	 */
+	input->id = CDNS_DPI_INPUT;
+	input->bridge.funcs = &cdns_dsi_bridge_funcs;
+	input->bridge.of_node = pdev->dev.of_node;
+
+	/* Mask all interrupts before registering the IRQ handler. */
+	writel(0, dsi->regs + MCTL_MAIN_STS_CTL);
+	writel(0, dsi->regs + MCTL_DPHY_ERR_CTL1);
+	writel(0, dsi->regs + CMD_MODE_STS_CTL);
+	writel(0, dsi->regs + DIRECT_CMD_STS_CTL);
+	writel(0, dsi->regs + DIRECT_CMD_RD_STS_CTL);
+	writel(0, dsi->regs + VID_MODE_STS_CTL);
+	writel(0, dsi->regs + TVG_STS_CTL);
+	writel(0, dsi->regs + DPI_IRQ_EN);
+	ret = devm_request_irq(&pdev->dev, irq, cdns_dsi_interrupt, 0,
+			       dev_name(&pdev->dev), dsi);
+	if (ret)
+		goto err_disable_pclk;
+
+	pm_runtime_enable(&pdev->dev);
+	dsi->base.dev = &pdev->dev;
+	dsi->base.ops = &cdns_dsi_ops;
+
+	ret = mipi_dsi_host_register(&dsi->base);
+	if (ret)
+		goto err_disable_runtime_pm;
+
+	clk_disable_unprepare(dsi->dsi_p_clk);
+
+	return 0;
+
+err_disable_runtime_pm:
+	pm_runtime_disable(&pdev->dev);
+
+err_disable_pclk:
+	clk_disable_unprepare(dsi->dsi_p_clk);
+
+err_remove_dphy:
+	cdns_dphy_remove(dsi->dphy);
+
+	return ret;
+}
+
+static int cdns_dsi_drm_remove(struct platform_device *pdev)
+{
+	struct cdns_dsi *dsi = platform_get_drvdata(pdev);
+
+	mipi_dsi_host_unregister(&dsi->base);
+	pm_runtime_disable(&pdev->dev);
+	cdns_dphy_remove(dsi->dphy);
+
+	return 0;
+}
+
+static const struct of_device_id cdns_dsi_of_match[] = {
+	{ .compatible = "cdns,dsi" },
+	{ },
+};
+
+static struct platform_driver cdns_dsi_platform_driver = {
+	.probe  = cdns_dsi_drm_probe,
+	.remove = cdns_dsi_drm_remove,
+	.driver = {
+		.name   = "cdns-dsi",
+		.of_match_table = cdns_dsi_of_match,
+		.pm = &cdns_dsi_pm_ops,
+	},
+};
+module_platform_driver(cdns_dsi_platform_driver);
+
+MODULE_AUTHOR("Boris Brezillon <boris.brezillon@bootlin.com>");
+MODULE_DESCRIPTION("Cadence DSI driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:cdns-dsi");
+
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
index 3b7e5c59a5e9..8f9c8a6b46de 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
@@ -152,7 +152,6 @@ static struct platform_driver snd_dw_hdmi_driver = {
 	.remove	= snd_dw_hdmi_remove,
 	.driver	= {
 		.name = DRIVER_NAME,
-		.owner = THIS_MODULE,
 	},
 };
 module_platform_driver(snd_dw_hdmi_driver);
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
index ec8d0006ef7c..3c136f2b954f 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
@@ -2077,7 +2077,7 @@ static irqreturn_t dw_hdmi_hardirq(int irq, void *dev_id)
 	return ret;
 }
 
-void __dw_hdmi_setup_rx_sense(struct dw_hdmi *hdmi, bool hpd, bool rx_sense)
+void dw_hdmi_setup_rx_sense(struct dw_hdmi *hdmi, bool hpd, bool rx_sense)
 {
 	mutex_lock(&hdmi->mutex);
 
@@ -2103,13 +2103,6 @@ void __dw_hdmi_setup_rx_sense(struct dw_hdmi *hdmi, bool hpd, bool rx_sense)
 	}
 	mutex_unlock(&hdmi->mutex);
 }
-
-void dw_hdmi_setup_rx_sense(struct device *dev, bool hpd, bool rx_sense)
-{
-	struct dw_hdmi *hdmi = dev_get_drvdata(dev);
-
-	__dw_hdmi_setup_rx_sense(hdmi, hpd, rx_sense);
-}
 EXPORT_SYMBOL_GPL(dw_hdmi_setup_rx_sense);
 
 static irqreturn_t dw_hdmi_irq(int irq, void *dev_id)
@@ -2145,9 +2138,9 @@ static irqreturn_t dw_hdmi_irq(int irq, void *dev_id)
 	 */
 	if (intr_stat &
 	    (HDMI_IH_PHY_STAT0_RX_SENSE | HDMI_IH_PHY_STAT0_HPD)) {
-		__dw_hdmi_setup_rx_sense(hdmi,
-					 phy_stat & HDMI_PHY_HPD,
-					 phy_stat & HDMI_PHY_RX_SENSE);
+		dw_hdmi_setup_rx_sense(hdmi,
+				       phy_stat & HDMI_PHY_HPD,
+				       phy_stat & HDMI_PHY_RX_SENSE);
 
 		if ((phy_stat & (HDMI_PHY_RX_SENSE | HDMI_PHY_HPD)) == 0)
 			cec_notifier_set_phys_addr(hdmi->cec_notifier,
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
index 226171a3ece1..fd7999642cf8 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd
  * Copyright (C) STMicroelectronics SA 2017
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
  * Modified by Philippe Cornu <philippe.cornu@st.com>
  * This generic Synopsys DesignWare MIPI DSI host driver is based on the
  * Rockchip version from rockchip/dw-mipi-dsi.c with phy & bridge APIs.
@@ -775,20 +771,20 @@ static void dw_mipi_dsi_bridge_mode_set(struct drm_bridge *bridge,
 
 	clk_prepare_enable(dsi->pclk);
 
-	ret = phy_ops->get_lane_mbps(priv_data, mode, dsi->mode_flags,
+	ret = phy_ops->get_lane_mbps(priv_data, adjusted_mode, dsi->mode_flags,
 				     dsi->lanes, dsi->format, &dsi->lane_mbps);
 	if (ret)
 		DRM_DEBUG_DRIVER("Phy get_lane_mbps() failed\n");
 
 	pm_runtime_get_sync(dsi->dev);
 	dw_mipi_dsi_init(dsi);
-	dw_mipi_dsi_dpi_config(dsi, mode);
+	dw_mipi_dsi_dpi_config(dsi, adjusted_mode);
 	dw_mipi_dsi_packet_handler_config(dsi);
 	dw_mipi_dsi_video_mode_config(dsi);
-	dw_mipi_dsi_video_packet_config(dsi, mode);
+	dw_mipi_dsi_video_packet_config(dsi, adjusted_mode);
 	dw_mipi_dsi_command_mode_config(dsi);
-	dw_mipi_dsi_line_timer_config(dsi, mode);
-	dw_mipi_dsi_vertical_timing_config(dsi, mode);
+	dw_mipi_dsi_line_timer_config(dsi, adjusted_mode);
+	dw_mipi_dsi_vertical_timing_config(dsi, adjusted_mode);
 
 	dw_mipi_dsi_dphy_init(dsi);
 	dw_mipi_dsi_dphy_timing_config(dsi);
@@ -802,7 +798,7 @@ static void dw_mipi_dsi_bridge_mode_set(struct drm_bridge *bridge,
 
 	dw_mipi_dsi_dphy_enable(dsi);
 
-	dw_mipi_dsi_wait_for_two_frames(mode);
+	dw_mipi_dsi_wait_for_two_frames(adjusted_mode);
 
 	/* Switch to cmd mode for panel-bridge pre_enable & panel prepare */
 	dw_mipi_dsi_set_mode(dsi, 0);
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
index 08ab7d6aea65..0fd9cf27542c 100644
--- a/drivers/gpu/drm/bridge/tc358767.c
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -1102,7 +1102,7 @@ static bool tc_bridge_mode_fixup(struct drm_bridge *bridge,
 	return true;
 }
 
-static int tc_connector_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status tc_connector_mode_valid(struct drm_connector *connector,
 				   struct drm_display_mode *mode)
 {
 	/* DPI interface clock limitation: upto 154 MHz */
diff --git a/drivers/gpu/drm/bridge/thc63lvd1024.c b/drivers/gpu/drm/bridge/thc63lvd1024.c
new file mode 100644
index 000000000000..c8b9edd5a7f4
--- /dev/null
+++ b/drivers/gpu/drm/bridge/thc63lvd1024.c
@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * THC63LVD1024 LVDS to parallel data DRM bridge driver.
+ *
+ * Copyright (C) 2018 Jacopo Mondi <jacopo+renesas@jmondi.org>
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_panel.h>
+
+#include <linux/gpio/consumer.h>
+#include <linux/of_graph.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+
+enum thc63_ports {
+	THC63_LVDS_IN0,
+	THC63_LVDS_IN1,
+	THC63_RGB_OUT0,
+	THC63_RGB_OUT1,
+};
+
+struct thc63_dev {
+	struct device *dev;
+
+	struct regulator *vcc;
+
+	struct gpio_desc *pdwn;
+	struct gpio_desc *oe;
+
+	struct drm_bridge bridge;
+	struct drm_bridge *next;
+};
+
+static inline struct thc63_dev *to_thc63(struct drm_bridge *bridge)
+{
+	return container_of(bridge, struct thc63_dev, bridge);
+}
+
+static int thc63_attach(struct drm_bridge *bridge)
+{
+	struct thc63_dev *thc63 = to_thc63(bridge);
+
+	return drm_bridge_attach(bridge->encoder, thc63->next, bridge);
+}
+
+static void thc63_enable(struct drm_bridge *bridge)
+{
+	struct thc63_dev *thc63 = to_thc63(bridge);
+	int ret;
+
+	ret = regulator_enable(thc63->vcc);
+	if (ret) {
+		dev_err(thc63->dev,
+			"Failed to enable regulator \"vcc\": %d\n", ret);
+		return;
+	}
+
+	gpiod_set_value(thc63->pdwn, 0);
+	gpiod_set_value(thc63->oe, 1);
+}
+
+static void thc63_disable(struct drm_bridge *bridge)
+{
+	struct thc63_dev *thc63 = to_thc63(bridge);
+	int ret;
+
+	gpiod_set_value(thc63->oe, 0);
+	gpiod_set_value(thc63->pdwn, 1);
+
+	ret = regulator_disable(thc63->vcc);
+	if (ret)
+		dev_err(thc63->dev,
+			"Failed to disable regulator \"vcc\": %d\n", ret);
+}
+
+static const struct drm_bridge_funcs thc63_bridge_func = {
+	.attach	= thc63_attach,
+	.enable = thc63_enable,
+	.disable = thc63_disable,
+};
+
+static int thc63_parse_dt(struct thc63_dev *thc63)
+{
+	struct device_node *thc63_out;
+	struct device_node *remote;
+
+	thc63_out = of_graph_get_endpoint_by_regs(thc63->dev->of_node,
+						  THC63_RGB_OUT0, -1);
+	if (!thc63_out) {
+		dev_err(thc63->dev, "Missing endpoint in port@%u\n",
+			THC63_RGB_OUT0);
+		return -ENODEV;
+	}
+
+	remote = of_graph_get_remote_port_parent(thc63_out);
+	of_node_put(thc63_out);
+	if (!remote) {
+		dev_err(thc63->dev, "Endpoint in port@%u unconnected\n",
+			THC63_RGB_OUT0);
+		return -ENODEV;
+	}
+
+	if (!of_device_is_available(remote)) {
+		dev_err(thc63->dev, "port@%u remote endpoint is disabled\n",
+			THC63_RGB_OUT0);
+		of_node_put(remote);
+		return -ENODEV;
+	}
+
+	thc63->next = of_drm_find_bridge(remote);
+	of_node_put(remote);
+	if (!thc63->next)
+		return -EPROBE_DEFER;
+
+	return 0;
+}
+
+static int thc63_gpio_init(struct thc63_dev *thc63)
+{
+	thc63->oe = devm_gpiod_get_optional(thc63->dev, "oe", GPIOD_OUT_LOW);
+	if (IS_ERR(thc63->oe)) {
+		dev_err(thc63->dev, "Unable to get \"oe-gpios\": %ld\n",
+			PTR_ERR(thc63->oe));
+		return PTR_ERR(thc63->oe);
+	}
+
+	thc63->pdwn = devm_gpiod_get_optional(thc63->dev, "powerdown",
+					      GPIOD_OUT_HIGH);
+	if (IS_ERR(thc63->pdwn)) {
+		dev_err(thc63->dev, "Unable to get \"powerdown-gpios\": %ld\n",
+			PTR_ERR(thc63->pdwn));
+		return PTR_ERR(thc63->pdwn);
+	}
+
+	return 0;
+}
+
+static int thc63_probe(struct platform_device *pdev)
+{
+	struct thc63_dev *thc63;
+	int ret;
+
+	thc63 = devm_kzalloc(&pdev->dev, sizeof(*thc63), GFP_KERNEL);
+	if (!thc63)
+		return -ENOMEM;
+
+	thc63->dev = &pdev->dev;
+	platform_set_drvdata(pdev, thc63);
+
+	thc63->vcc = devm_regulator_get_optional(thc63->dev, "vcc");
+	if (IS_ERR(thc63->vcc)) {
+		if (PTR_ERR(thc63->vcc) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+
+		dev_err(thc63->dev, "Unable to get \"vcc\" supply: %ld\n",
+			PTR_ERR(thc63->vcc));
+		return PTR_ERR(thc63->vcc);
+	}
+
+	ret = thc63_gpio_init(thc63);
+	if (ret)
+		return ret;
+
+	ret = thc63_parse_dt(thc63);
+	if (ret)
+		return ret;
+
+	thc63->bridge.driver_private = thc63;
+	thc63->bridge.of_node = pdev->dev.of_node;
+	thc63->bridge.funcs = &thc63_bridge_func;
+
+	drm_bridge_add(&thc63->bridge);
+
+	return 0;
+}
+
+static int thc63_remove(struct platform_device *pdev)
+{
+	struct thc63_dev *thc63 = platform_get_drvdata(pdev);
+
+	drm_bridge_remove(&thc63->bridge);
+
+	return 0;
+}
+
+static const struct of_device_id thc63_match[] = {
+	{ .compatible = "thine,thc63lvd1024", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, thc63_match);
+
+static struct platform_driver thc63_driver = {
+	.probe	= thc63_probe,
+	.remove	= thc63_remove,
+	.driver	= {
+		.name		= "thc63lvd1024",
+		.of_match_table	= thc63_match,
+	},
+};
+module_platform_driver(thc63_driver);
+
+MODULE_AUTHOR("Jacopo Mondi <jacopo@jmondi.org>");
+MODULE_DESCRIPTION("Thine THC63LVD1024 LVDS decoder DRM bridge driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index c825c76edc1d..895741e9cd7d 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -791,6 +791,8 @@ static int drm_atomic_plane_set_property(struct drm_plane *plane,
 		state->src_w = val;
 	} else if (property == config->prop_src_h) {
 		state->src_h = val;
+	} else if (property == plane->alpha_property) {
+		state->alpha = val;
 	} else if (property == plane->rotation_property) {
 		if (!is_power_of_2(val & DRM_MODE_ROTATE_MASK))
 			return -EINVAL;
@@ -856,6 +858,8 @@ drm_atomic_plane_get_property(struct drm_plane *plane,
 		*val = state->src_w;
 	} else if (property == config->prop_src_h) {
 		*val = state->src_h;
+	} else if (property == plane->alpha_property) {
+		*val = state->alpha;
 	} else if (property == plane->rotation_property) {
 		*val = state->rotation;
 	} else if (property == plane->zpos_property) {
@@ -1429,7 +1433,9 @@ drm_atomic_set_crtc_for_plane(struct drm_plane_state *plane_state,
 {
 	struct drm_plane *plane = plane_state->plane;
 	struct drm_crtc_state *crtc_state;
-
+	/* Nothing to do for same crtc*/
+	if (plane_state->crtc == crtc)
+		return 0;
 	if (plane_state->crtc) {
 		crtc_state = drm_atomic_get_crtc_state(plane_state->state,
 						       plane_state->crtc);
@@ -1500,6 +1506,14 @@ EXPORT_SYMBOL(drm_atomic_set_fb_for_plane);
  * Otherwise, if &drm_plane_state.fence is not set this function we just set it
  * with the received implicit fence. In both cases this function consumes a
  * reference for @fence.
+ *
+ * This way explicit fencing can be used to overrule implicit fencing, which is
+ * important to make explicit fencing use-cases work: One example is using one
+ * buffer for 2 screens with different refresh rates. Implicit fencing will
+ * clamp rendering to the refresh rate of the slower screen, whereas explicit
+ * fence allows 2 independent render and display loops on a single buffer. If a
+ * driver allows obeys both implicit and explicit fences for plane updates, then
+ * it will break all the benefits of explicit fencing.
  */
 void
 drm_atomic_set_fence_for_plane(struct drm_plane_state *plane_state,
@@ -1710,11 +1724,15 @@ int drm_atomic_check_only(struct drm_atomic_state *state)
 		}
 	}
 
-	if (config->funcs->atomic_check)
+	if (config->funcs->atomic_check) {
 		ret = config->funcs->atomic_check(state->dev, state);
 
-	if (ret)
-		return ret;
+		if (ret) {
+			DRM_DEBUG_ATOMIC("atomic driver check for %p failed: %d\n",
+					 state, ret);
+			return ret;
+		}
+	}
 
 	if (!state->allow_modeset) {
 		for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index c35654591c12..130da5195f3b 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -766,7 +766,7 @@ int drm_atomic_helper_check_plane_state(struct drm_plane_state *plane_state,
 	if (crtc_state->enable)
 		drm_mode_get_hv_timing(&crtc_state->mode, &clip.x2, &clip.y2);
 
-	plane_state->visible = drm_rect_clip_scaled(src, dst, &clip, hscale, vscale);
+	plane_state->visible = drm_rect_clip_scaled(src, dst, &clip);
 
 	drm_rect_rotate_inv(src, fb->width << 16, fb->height << 16, rotation);
 
@@ -875,6 +875,11 @@ EXPORT_SYMBOL(drm_atomic_helper_check_planes);
  * functions depend upon an updated adjusted_mode.clock to e.g. properly compute
  * watermarks.
  *
+ * Note that zpos normalization will add all enable planes to the state which
+ * might not desired for some drivers.
+ * For example enable/disable of a cursor plane which have fixed zpos value
+ * would trigger all other enabled planes to be forced to the state change.
+ *
  * RETURNS:
  * Zero for success or -errno
  */
@@ -887,6 +892,12 @@ int drm_atomic_helper_check(struct drm_device *dev,
 	if (ret)
 		return ret;
 
+	if (dev->mode_config.normalize_zpos) {
+		ret = drm_atomic_normalize_zpos(dev, state);
+		if (ret)
+			return ret;
+	}
+
 	ret = drm_atomic_helper_check_planes(dev, state);
 	if (ret)
 		return ret;
@@ -1561,6 +1572,17 @@ void drm_atomic_helper_async_commit(struct drm_device *dev,
 	for_each_new_plane_in_state(state, plane, plane_state, i) {
 		funcs = plane->helper_private;
 		funcs->atomic_async_update(plane, plane_state);
+
+		/*
+		 * ->atomic_async_update() is supposed to update the
+		 * plane->state in-place, make sure at least common
+		 * properties have been properly updated.
+		 */
+		WARN_ON_ONCE(plane->state->fb != plane_state->fb);
+		WARN_ON_ONCE(plane->state->crtc_x != plane_state->crtc_x);
+		WARN_ON_ONCE(plane->state->crtc_y != plane_state->crtc_y);
+		WARN_ON_ONCE(plane->state->src_x != plane_state->src_x);
+		WARN_ON_ONCE(plane->state->src_y != plane_state->src_y);
 	}
 }
 EXPORT_SYMBOL(drm_atomic_helper_async_commit);
@@ -2659,7 +2681,7 @@ int drm_atomic_helper_disable_plane(struct drm_plane *plane,
 		goto fail;
 	}
 
-	if (plane_state->crtc && (plane == plane->crtc->cursor))
+	if (plane_state->crtc && plane_state->crtc->cursor == plane)
 		plane_state->state->legacy_cursor_update = true;
 
 	ret = __drm_atomic_helper_disable_plane(plane, plane_state);
@@ -2881,31 +2903,9 @@ int __drm_atomic_helper_set_config(struct drm_mode_set *set,
 	return 0;
 }
 
-/**
- * drm_atomic_helper_disable_all - disable all currently active outputs
- * @dev: DRM device
- * @ctx: lock acquisition context
- *
- * Loops through all connectors, finding those that aren't turned off and then
- * turns them off by setting their DPMS mode to OFF and deactivating the CRTC
- * that they are connected to.
- *
- * This is used for example in suspend/resume to disable all currently active
- * functions when suspending. If you just want to shut down everything at e.g.
- * driver unload, look at drm_atomic_helper_shutdown().
- *
- * Note that if callers haven't already acquired all modeset locks this might
- * return -EDEADLK, which must be handled by calling drm_modeset_backoff().
- *
- * Returns:
- * 0 on success or a negative error code on failure.
- *
- * See also:
- * drm_atomic_helper_suspend(), drm_atomic_helper_resume() and
- * drm_atomic_helper_shutdown().
- */
-int drm_atomic_helper_disable_all(struct drm_device *dev,
-				  struct drm_modeset_acquire_ctx *ctx)
+static int __drm_atomic_helper_disable_all(struct drm_device *dev,
+					   struct drm_modeset_acquire_ctx *ctx,
+					   bool clean_old_fbs)
 {
 	struct drm_atomic_state *state;
 	struct drm_connector_state *conn_state;
@@ -2957,8 +2957,11 @@ int drm_atomic_helper_disable_all(struct drm_device *dev,
 			goto free;
 
 		drm_atomic_set_fb_for_plane(plane_state, NULL);
-		plane_mask |= BIT(drm_plane_index(plane));
-		plane->old_fb = plane->fb;
+
+		if (clean_old_fbs) {
+			plane->old_fb = plane->fb;
+			plane_mask |= BIT(drm_plane_index(plane));
+		}
 	}
 
 	ret = drm_atomic_commit(state);
@@ -2969,6 +2972,34 @@ int drm_atomic_helper_disable_all(struct drm_device *dev,
 	return ret;
 }
 
+/**
+ * drm_atomic_helper_disable_all - disable all currently active outputs
+ * @dev: DRM device
+ * @ctx: lock acquisition context
+ *
+ * Loops through all connectors, finding those that aren't turned off and then
+ * turns them off by setting their DPMS mode to OFF and deactivating the CRTC
+ * that they are connected to.
+ *
+ * This is used for example in suspend/resume to disable all currently active
+ * functions when suspending. If you just want to shut down everything at e.g.
+ * driver unload, look at drm_atomic_helper_shutdown().
+ *
+ * Note that if callers haven't already acquired all modeset locks this might
+ * return -EDEADLK, which must be handled by calling drm_modeset_backoff().
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ *
+ * See also:
+ * drm_atomic_helper_suspend(), drm_atomic_helper_resume() and
+ * drm_atomic_helper_shutdown().
+ */
+int drm_atomic_helper_disable_all(struct drm_device *dev,
+				  struct drm_modeset_acquire_ctx *ctx)
+{
+	return __drm_atomic_helper_disable_all(dev, ctx, false);
+}
 EXPORT_SYMBOL(drm_atomic_helper_disable_all);
 
 /**
@@ -2991,7 +3022,7 @@ void drm_atomic_helper_shutdown(struct drm_device *dev)
 	while (1) {
 		ret = drm_modeset_lock_all_ctx(dev, &ctx);
 		if (!ret)
-			ret = drm_atomic_helper_disable_all(dev, &ctx);
+			ret = __drm_atomic_helper_disable_all(dev, &ctx, true);
 
 		if (ret != -EDEADLK)
 			break;
@@ -3095,14 +3126,14 @@ int drm_atomic_helper_commit_duplicated_state(struct drm_atomic_state *state,
 	struct drm_connector_state *new_conn_state;
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_state;
-	unsigned plane_mask = 0;
-	struct drm_device *dev = state->dev;
-	int ret;
 
 	state->acquire_ctx = ctx;
 
 	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
-		plane_mask |= BIT(drm_plane_index(plane));
+		WARN_ON(plane->crtc != new_plane_state->crtc);
+		WARN_ON(plane->fb != new_plane_state->fb);
+		WARN_ON(plane->old_fb);
+
 		state->planes[i].old_state = plane->state;
 	}
 
@@ -3112,11 +3143,7 @@ int drm_atomic_helper_commit_duplicated_state(struct drm_atomic_state *state,
 	for_each_new_connector_in_state(state, connector, new_conn_state, i)
 		state->connectors[i].old_state = connector->state;
 
-	ret = drm_atomic_commit(state);
-	if (plane_mask)
-		drm_atomic_clean_old_fb(dev, plane_mask, ret);
-
-	return ret;
+	return drm_atomic_commit(state);
 }
 EXPORT_SYMBOL(drm_atomic_helper_commit_duplicated_state);
 
@@ -3484,6 +3511,10 @@ void drm_atomic_helper_plane_reset(struct drm_plane *plane)
 	if (plane->state) {
 		plane->state->plane = plane;
 		plane->state->rotation = DRM_MODE_ROTATE_0;
+
+		/* Reset the alpha value to fully opaque if it matters */
+		if (plane->alpha_property)
+			plane->state->alpha = plane->alpha_property->values[1];
 	}
 }
 EXPORT_SYMBOL(drm_atomic_helper_plane_reset);
diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c
index 5a81e1b4c076..a16a74d7e15e 100644
--- a/drivers/gpu/drm/drm_blend.c
+++ b/drivers/gpu/drm/drm_blend.c
@@ -88,6 +88,13 @@
  * On top of this basic transformation additional properties can be exposed by
  * the driver:
  *
+ * alpha:
+ * 	Alpha is setup with drm_plane_create_alpha_property(). It controls the
+ * 	plane-wide opacity, from transparent (0) to opaque (0xffff). It can be
+ * 	combined with pixel alpha.
+ *	The pixel values in the framebuffers are expected to not be
+ *	pre-multiplied by the global alpha associated to the plane.
+ *
  * rotation:
  *	Rotation is set up with drm_plane_create_rotation_property(). It adds a
  *	rotation and reflection step between the source and destination rectangles.
@@ -105,6 +112,38 @@
  * exposed and assumed to be black).
  */
 
+/**
+ * drm_plane_create_alpha_property - create a new alpha property
+ * @plane: drm plane
+ *
+ * This function creates a generic, mutable, alpha property and enables support
+ * for it in the DRM core. It is attached to @plane.
+ *
+ * The alpha property will be allowed to be within the bounds of 0
+ * (transparent) to 0xffff (opaque).
+ *
+ * Returns:
+ * 0 on success, negative error code on failure.
+ */
+int drm_plane_create_alpha_property(struct drm_plane *plane)
+{
+	struct drm_property *prop;
+
+	prop = drm_property_create_range(plane->dev, 0, "alpha",
+					 0, DRM_BLEND_ALPHA_OPAQUE);
+	if (!prop)
+		return -ENOMEM;
+
+	drm_object_attach_property(&plane->base, prop, DRM_BLEND_ALPHA_OPAQUE);
+	plane->alpha_property = prop;
+
+	if (plane->state)
+		plane->state->alpha = DRM_BLEND_ALPHA_OPAQUE;
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_plane_create_alpha_property);
+
 /**
  * drm_plane_create_rotation_property - create a new rotation property
  * @plane: drm plane
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index b3cde897cd80..9b9ba5d5ec0c 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -1069,7 +1069,7 @@ int drm_mode_create_tv_properties(struct drm_device *dev,
 		goto nomem;
 
 	for (i = 0; i < num_modes; i++)
-		drm_property_add_enum(dev->mode_config.tv_mode_property, i,
+		drm_property_add_enum(dev->mode_config.tv_mode_property,
 				      i, modes[i]);
 
 	dev->mode_config.tv_brightness_property =
@@ -1156,7 +1156,7 @@ int drm_connector_attach_scaling_mode_property(struct drm_connector *connector,
 {
 	struct drm_device *dev = connector->dev;
 	struct drm_property *scaling_mode_property;
-	int i, j = 0;
+	int i;
 	const unsigned valid_scaling_mode_mask =
 		(1U << ARRAY_SIZE(drm_scaling_mode_enum_list)) - 1;
 
@@ -1177,7 +1177,7 @@ int drm_connector_attach_scaling_mode_property(struct drm_connector *connector,
 		if (!(BIT(i) & scaling_mode_mask))
 			continue;
 
-		ret = drm_property_add_enum(scaling_mode_property, j++,
+		ret = drm_property_add_enum(scaling_mode_property,
 					    drm_scaling_mode_enum_list[i].type,
 					    drm_scaling_mode_enum_list[i].name);
 
@@ -1531,8 +1531,10 @@ static struct drm_encoder *drm_connector_get_encoder(struct drm_connector *conne
 	return connector->encoder;
 }
 
-static bool drm_mode_expose_to_userspace(const struct drm_display_mode *mode,
-					 const struct drm_file *file_priv)
+static bool
+drm_mode_expose_to_userspace(const struct drm_display_mode *mode,
+			     const struct list_head *export_list,
+			     const struct drm_file *file_priv)
 {
 	/*
 	 * If user-space hasn't configured the driver to expose the stereo 3D
@@ -1540,6 +1542,23 @@ static bool drm_mode_expose_to_userspace(const struct drm_display_mode *mode,
 	 */
 	if (!file_priv->stereo_allowed && drm_mode_is_stereo(mode))
 		return false;
+	/*
+	 * If user-space hasn't configured the driver to expose the modes
+	 * with aspect-ratio, don't expose them. However if such a mode
+	 * is unique, let it be exposed, but reset the aspect-ratio flags
+	 * while preparing the list of user-modes.
+	 */
+	if (!file_priv->aspect_ratio_allowed) {
+		struct drm_display_mode *mode_itr;
+
+		list_for_each_entry(mode_itr, export_list, export_head)
+			if (drm_mode_match(mode_itr, mode,
+					   DRM_MODE_MATCH_TIMINGS |
+					   DRM_MODE_MATCH_CLOCK |
+					   DRM_MODE_MATCH_FLAGS |
+					   DRM_MODE_MATCH_3D_FLAGS))
+				return false;
+	}
 
 	return true;
 }
@@ -1559,6 +1578,7 @@ int drm_mode_getconnector(struct drm_device *dev, void *data,
 	struct drm_mode_modeinfo u_mode;
 	struct drm_mode_modeinfo __user *mode_ptr;
 	uint32_t __user *encoder_ptr;
+	LIST_HEAD(export_list);
 
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 		return -EINVAL;
@@ -1607,21 +1627,31 @@ int drm_mode_getconnector(struct drm_device *dev, void *data,
 
 	/* delayed so we get modes regardless of pre-fill_modes state */
 	list_for_each_entry(mode, &connector->modes, head)
-		if (drm_mode_expose_to_userspace(mode, file_priv))
+		if (drm_mode_expose_to_userspace(mode, &export_list,
+						 file_priv)) {
+			list_add_tail(&mode->export_head, &export_list);
 			mode_count++;
+		}
 
 	/*
 	 * This ioctl is called twice, once to determine how much space is
 	 * needed, and the 2nd time to fill it.
+	 * The modes that need to be exposed to the user are maintained in the
+	 * 'export_list'. When the ioctl is called first time to determine the,
+	 * space, the export_list gets filled, to find the no.of modes. In the
+	 * 2nd time, the user modes are filled, one by one from the export_list.
 	 */
 	if ((out_resp->count_modes >= mode_count) && mode_count) {
 		copied = 0;
 		mode_ptr = (struct drm_mode_modeinfo __user *)(unsigned long)out_resp->modes_ptr;
-		list_for_each_entry(mode, &connector->modes, head) {
-			if (!drm_mode_expose_to_userspace(mode, file_priv))
-				continue;
-
+		list_for_each_entry(mode, &export_list, export_head) {
 			drm_mode_convert_to_umode(&u_mode, mode);
+			/*
+			 * Reset aspect ratio flags of user-mode, if modes with
+			 * aspect-ratio are not supported.
+			 */
+			if (!file_priv->aspect_ratio_allowed)
+				u_mode.flags &= ~DRM_MODE_FLAG_PIC_AR_MASK;
 			if (copy_to_user(mode_ptr + copied,
 					 &u_mode, sizeof(u_mode))) {
 				ret = -EFAULT;
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 03583887cfec..98a36e6c69ad 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -402,6 +402,7 @@ int drm_mode_getcrtc(struct drm_device *dev,
 {
 	struct drm_mode_crtc *crtc_resp = data;
 	struct drm_crtc *crtc;
+	struct drm_plane *plane;
 
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 		return -EINVAL;
@@ -410,34 +411,36 @@ int drm_mode_getcrtc(struct drm_device *dev,
 	if (!crtc)
 		return -ENOENT;
 
+	plane = crtc->primary;
+
 	crtc_resp->gamma_size = crtc->gamma_size;
 
-	drm_modeset_lock(&crtc->primary->mutex, NULL);
-	if (crtc->primary->state && crtc->primary->state->fb)
-		crtc_resp->fb_id = crtc->primary->state->fb->base.id;
-	else if (!crtc->primary->state && crtc->primary->fb)
-		crtc_resp->fb_id = crtc->primary->fb->base.id;
+	drm_modeset_lock(&plane->mutex, NULL);
+	if (plane->state && plane->state->fb)
+		crtc_resp->fb_id = plane->state->fb->base.id;
+	else if (!plane->state && plane->fb)
+		crtc_resp->fb_id = plane->fb->base.id;
 	else
 		crtc_resp->fb_id = 0;
 
-	if (crtc->primary->state) {
-		crtc_resp->x = crtc->primary->state->src_x >> 16;
-		crtc_resp->y = crtc->primary->state->src_y >> 16;
+	if (plane->state) {
+		crtc_resp->x = plane->state->src_x >> 16;
+		crtc_resp->y = plane->state->src_y >> 16;
 	}
-	drm_modeset_unlock(&crtc->primary->mutex);
+	drm_modeset_unlock(&plane->mutex);
 
 	drm_modeset_lock(&crtc->mutex, NULL);
 	if (crtc->state) {
 		if (crtc->state->enable) {
 			drm_mode_convert_to_umode(&crtc_resp->mode, &crtc->state->mode);
 			crtc_resp->mode_valid = 1;
-
 		} else {
 			crtc_resp->mode_valid = 0;
 		}
 	} else {
 		crtc_resp->x = crtc->x;
 		crtc_resp->y = crtc->y;
+
 		if (crtc->enabled) {
 			drm_mode_convert_to_umode(&crtc_resp->mode, &crtc->mode);
 			crtc_resp->mode_valid = 1;
@@ -446,6 +449,8 @@ int drm_mode_getcrtc(struct drm_device *dev,
 			crtc_resp->mode_valid = 0;
 		}
 	}
+	if (!file_priv->aspect_ratio_allowed)
+		crtc_resp->mode.flags &= ~DRM_MODE_FLAG_PIC_AR_MASK;
 	drm_modeset_unlock(&crtc->mutex);
 
 	return 0;
@@ -471,7 +476,7 @@ static int __drm_mode_set_config_internal(struct drm_mode_set *set,
 
 	ret = crtc->funcs->set_config(set, ctx);
 	if (ret == 0) {
-		crtc->primary->crtc = crtc;
+		crtc->primary->crtc = fb ? crtc : NULL;
 		crtc->primary->fb = fb;
 	}
 
@@ -554,6 +559,7 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 	struct drm_mode_config *config = &dev->mode_config;
 	struct drm_mode_crtc *crtc_req = data;
 	struct drm_crtc *crtc;
+	struct drm_plane *plane;
 	struct drm_connector **connector_set = NULL, *connector;
 	struct drm_framebuffer *fb = NULL;
 	struct drm_display_mode *mode = NULL;
@@ -580,22 +586,33 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 	}
 	DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
 
+	plane = crtc->primary;
+
 	mutex_lock(&crtc->dev->mode_config.mutex);
 	drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
 retry:
 	ret = drm_modeset_lock_all_ctx(crtc->dev, &ctx);
 	if (ret)
 		goto out;
+
 	if (crtc_req->mode_valid) {
 		/* If we have a mode we need a framebuffer. */
 		/* If we pass -1, set the mode with the currently bound fb */
 		if (crtc_req->fb_id == -1) {
-			if (!crtc->primary->fb) {
+			struct drm_framebuffer *old_fb;
+
+			if (plane->state)
+				old_fb = plane->state->fb;
+			else
+				old_fb = plane->fb;
+
+			if (!old_fb) {
 				DRM_DEBUG_KMS("CRTC doesn't have current FB\n");
 				ret = -EINVAL;
 				goto out;
 			}
-			fb = crtc->primary->fb;
+
+			fb = old_fb;
 			/* Make refcounting symmetric with the lookup path. */
 			drm_framebuffer_get(fb);
 		} else {
@@ -613,6 +630,13 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 			ret = -ENOMEM;
 			goto out;
 		}
+		if (!file_priv->aspect_ratio_allowed &&
+		    (crtc_req->mode.flags & DRM_MODE_FLAG_PIC_AR_MASK) != DRM_MODE_FLAG_PIC_AR_NONE) {
+			DRM_DEBUG_KMS("Unexpected aspect-ratio flag bits\n");
+			ret = -EINVAL;
+			goto out;
+		}
+
 
 		ret = drm_mode_convert_umode(dev, mode, &crtc_req->mode);
 		if (ret) {
@@ -627,8 +651,8 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 		 * match real hardware capabilities. Skip the check in that
 		 * case.
 		 */
-		if (!crtc->primary->format_default) {
-			ret = drm_plane_check_pixel_format(crtc->primary,
+		if (!plane->format_default) {
+			ret = drm_plane_check_pixel_format(plane,
 							   fb->format->format,
 							   fb->modifier);
 			if (ret) {
diff --git a/drivers/gpu/drm/drm_crtc_internal.h b/drivers/gpu/drm/drm_crtc_internal.h
index 3c2b82865ad2..5d307b23a4e6 100644
--- a/drivers/gpu/drm/drm_crtc_internal.h
+++ b/drivers/gpu/drm/drm_crtc_internal.h
@@ -220,3 +220,5 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 
 /* drm_edid.c */
 void drm_mode_fixup_1366x768(struct drm_display_mode *mode);
+void drm_reset_display_info(struct drm_connector *connector);
+u32 drm_add_display_info(struct drm_connector *connector, const struct edid *edid);
diff --git a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c
index ffe14ec3e7f2..a7ba602a43a8 100644
--- a/drivers/gpu/drm/drm_dp_helper.c
+++ b/drivers/gpu/drm/drm_dp_helper.c
@@ -119,18 +119,32 @@ u8 drm_dp_get_adjust_request_pre_emphasis(const u8 link_status[DP_LINK_STATUS_SI
 EXPORT_SYMBOL(drm_dp_get_adjust_request_pre_emphasis);
 
 void drm_dp_link_train_clock_recovery_delay(const u8 dpcd[DP_RECEIVER_CAP_SIZE]) {
-	if (dpcd[DP_TRAINING_AUX_RD_INTERVAL] == 0)
+	int rd_interval = dpcd[DP_TRAINING_AUX_RD_INTERVAL] &
+			  DP_TRAINING_AUX_RD_MASK;
+
+	if (rd_interval > 4)
+		DRM_DEBUG_KMS("AUX interval %d, out of range (max 4)\n",
+			      rd_interval);
+
+	if (rd_interval == 0 || dpcd[DP_DPCD_REV] >= DP_DPCD_REV_14)
 		udelay(100);
 	else
-		mdelay(dpcd[DP_TRAINING_AUX_RD_INTERVAL] * 4);
+		mdelay(rd_interval * 4);
 }
 EXPORT_SYMBOL(drm_dp_link_train_clock_recovery_delay);
 
 void drm_dp_link_train_channel_eq_delay(const u8 dpcd[DP_RECEIVER_CAP_SIZE]) {
-	if (dpcd[DP_TRAINING_AUX_RD_INTERVAL] == 0)
+	int rd_interval = dpcd[DP_TRAINING_AUX_RD_INTERVAL] &
+			  DP_TRAINING_AUX_RD_MASK;
+
+	if (rd_interval > 4)
+		DRM_DEBUG_KMS("AUX interval %d, out of range (max 4)\n",
+			      rd_interval);
+
+	if (rd_interval == 0)
 		udelay(400);
 	else
-		mdelay(dpcd[DP_TRAINING_AUX_RD_INTERVAL] * 4);
+		mdelay(rd_interval * 4);
 }
 EXPORT_SYMBOL(drm_dp_link_train_channel_eq_delay);
 
@@ -1145,6 +1159,7 @@ int drm_dp_psr_setup_time(const u8 psr_cap[EDP_PSR_RECEIVER_CAP_SIZE])
 	static const u16 psr_setup_time_us[] = {
 		PSR_SETUP_TIME(330),
 		PSR_SETUP_TIME(275),
+		PSR_SETUP_TIME(220),
 		PSR_SETUP_TIME(165),
 		PSR_SETUP_TIME(110),
 		PSR_SETUP_TIME(55),
diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c
index 6fac4129e6a2..658830620ca3 100644
--- a/drivers/gpu/drm/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/drm_dp_mst_topology.c
@@ -2941,12 +2941,14 @@ static void drm_dp_mst_dump_mstb(struct seq_file *m,
 	}
 }
 
+#define DP_PAYLOAD_TABLE_SIZE		64
+
 static bool dump_dp_payload_table(struct drm_dp_mst_topology_mgr *mgr,
 				  char *buf)
 {
 	int i;
 
-	for (i = 0; i < 64; i += 16) {
+	for (i = 0; i < DP_PAYLOAD_TABLE_SIZE; i += 16) {
 		if (drm_dp_dpcd_read(mgr->aux,
 				     DP_PAYLOAD_TABLE_UPDATE_STATUS + i,
 				     &buf[i], 16) != 16)
@@ -3015,7 +3017,7 @@ void drm_dp_mst_dump_topology(struct seq_file *m,
 
 	mutex_lock(&mgr->lock);
 	if (mgr->mst_primary) {
-		u8 buf[64];
+		u8 buf[DP_PAYLOAD_TABLE_SIZE];
 		int ret;
 
 		ret = drm_dp_dpcd_read(mgr->aux, DP_DPCD_REV, buf, DP_RECEIVER_CAP_SIZE);
@@ -3033,8 +3035,7 @@ void drm_dp_mst_dump_topology(struct seq_file *m,
 		seq_printf(m, " revision: hw: %x.%x sw: %x.%x\n",
 			   buf[0x9] >> 4, buf[0x9] & 0xf, buf[0xa], buf[0xb]);
 		if (dump_dp_payload_table(mgr, buf))
-			seq_printf(m, "payload table: %*ph\n", 63, buf);
-
+			seq_printf(m, "payload table: %*ph\n", DP_PAYLOAD_TABLE_SIZE, buf);
 	}
 
 	mutex_unlock(&mgr->lock);
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index c2c21d839727..b553a6f2ff0e 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -32,6 +32,7 @@
 #include <linux/moduleparam.h>
 #include <linux/mount.h>
 #include <linux/slab.h>
+#include <linux/srcu.h>
 
 #include <drm/drm_drv.h>
 #include <drm/drmP.h>
@@ -75,6 +76,8 @@ static bool drm_core_init_complete = false;
 
 static struct dentry *drm_debugfs_root;
 
+DEFINE_STATIC_SRCU(drm_unplug_srcu);
+
 /*
  * DRM Minors
  * A DRM device can provide several char-dev interfaces on the DRM-Major. Each
@@ -96,8 +99,6 @@ static struct drm_minor **drm_minor_get_slot(struct drm_device *dev,
 		return &dev->primary;
 	case DRM_MINOR_RENDER:
 		return &dev->render;
-	case DRM_MINOR_CONTROL:
-		return &dev->control;
 	default:
 		BUG();
 	}
@@ -318,18 +319,51 @@ void drm_put_dev(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_put_dev);
 
-static void drm_device_set_unplugged(struct drm_device *dev)
+/**
+ * drm_dev_enter - Enter device critical section
+ * @dev: DRM device
+ * @idx: Pointer to index that will be passed to the matching drm_dev_exit()
+ *
+ * This function marks and protects the beginning of a section that should not
+ * be entered after the device has been unplugged. The section end is marked
+ * with drm_dev_exit(). Calls to this function can be nested.
+ *
+ * Returns:
+ * True if it is OK to enter the section, false otherwise.
+ */
+bool drm_dev_enter(struct drm_device *dev, int *idx)
 {
-	smp_wmb();
-	atomic_set(&dev->unplugged, 1);
+	*idx = srcu_read_lock(&drm_unplug_srcu);
+
+	if (dev->unplugged) {
+		srcu_read_unlock(&drm_unplug_srcu, *idx);
+		return false;
+	}
+
+	return true;
 }
+EXPORT_SYMBOL(drm_dev_enter);
+
+/**
+ * drm_dev_exit - Exit device critical section
+ * @idx: index returned from drm_dev_enter()
+ *
+ * This function marks the end of a section that should not be entered after
+ * the device has been unplugged.
+ */
+void drm_dev_exit(int idx)
+{
+	srcu_read_unlock(&drm_unplug_srcu, idx);
+}
+EXPORT_SYMBOL(drm_dev_exit);
 
 /**
  * drm_dev_unplug - unplug a DRM device
  * @dev: DRM device
  *
  * This unplugs a hotpluggable DRM device, which makes it inaccessible to
- * userspace operations. Entry-points can use drm_dev_is_unplugged(). This
+ * userspace operations. Entry-points can use drm_dev_enter() and
+ * drm_dev_exit() to protect device resources in a race free manner. This
  * essentially unregisters the device like drm_dev_unregister(), but can be
  * called while there are still open users of @dev.
  */
@@ -338,10 +372,18 @@ void drm_dev_unplug(struct drm_device *dev)
 	drm_dev_unregister(dev);
 
 	mutex_lock(&drm_global_mutex);
-	drm_device_set_unplugged(dev);
 	if (dev->open_count == 0)
 		drm_dev_put(dev);
 	mutex_unlock(&drm_global_mutex);
+
+	/*
+	 * After synchronizing any critical read section is guaranteed to see
+	 * the new value of ->unplugged, and any critical section which might
+	 * still have seen the old value of ->unplugged is guaranteed to have
+	 * finished.
+	 */
+	dev->unplugged = true;
+	synchronize_srcu(&drm_unplug_srcu);
 }
 EXPORT_SYMBOL(drm_dev_unplug);
 
@@ -523,7 +565,6 @@ int drm_dev_init(struct drm_device *dev,
 err_minors:
 	drm_minor_free(dev, DRM_MINOR_PRIMARY);
 	drm_minor_free(dev, DRM_MINOR_RENDER);
-	drm_minor_free(dev, DRM_MINOR_CONTROL);
 	drm_fs_inode_free(dev->anon_inode);
 err_free:
 	mutex_destroy(&dev->master_mutex);
@@ -559,7 +600,6 @@ void drm_dev_fini(struct drm_device *dev)
 
 	drm_minor_free(dev, DRM_MINOR_PRIMARY);
 	drm_minor_free(dev, DRM_MINOR_RENDER);
-	drm_minor_free(dev, DRM_MINOR_CONTROL);
 
 	mutex_destroy(&dev->master_mutex);
 	mutex_destroy(&dev->ctxlist_mutex);
@@ -752,10 +792,6 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
 
 	mutex_lock(&drm_global_mutex);
 
-	ret = drm_minor_register(dev, DRM_MINOR_CONTROL);
-	if (ret)
-		goto err_minors;
-
 	ret = drm_minor_register(dev, DRM_MINOR_RENDER);
 	if (ret)
 		goto err_minors;
@@ -793,7 +829,6 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
 	remove_compat_control_link(dev);
 	drm_minor_unregister(dev, DRM_MINOR_PRIMARY);
 	drm_minor_unregister(dev, DRM_MINOR_RENDER);
-	drm_minor_unregister(dev, DRM_MINOR_CONTROL);
 out_unlock:
 	mutex_unlock(&drm_global_mutex);
 	return ret;
@@ -838,7 +873,6 @@ void drm_dev_unregister(struct drm_device *dev)
 	remove_compat_control_link(dev);
 	drm_minor_unregister(dev, DRM_MINOR_PRIMARY);
 	drm_minor_unregister(dev, DRM_MINOR_RENDER);
-	drm_minor_unregister(dev, DRM_MINOR_CONTROL);
 }
 EXPORT_SYMBOL(drm_dev_unregister);
 
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 39f1db4acda4..40e1e24f2ff0 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -2930,11 +2930,15 @@ cea_mode_alternate_timings(u8 vic, struct drm_display_mode *mode)
 static u8 drm_match_cea_mode_clock_tolerance(const struct drm_display_mode *to_match,
 					     unsigned int clock_tolerance)
 {
+	unsigned int match_flags = DRM_MODE_MATCH_TIMINGS | DRM_MODE_MATCH_FLAGS;
 	u8 vic;
 
 	if (!to_match->clock)
 		return 0;
 
+	if (to_match->picture_aspect_ratio)
+		match_flags |= DRM_MODE_MATCH_ASPECT_RATIO;
+
 	for (vic = 1; vic < ARRAY_SIZE(edid_cea_modes); vic++) {
 		struct drm_display_mode cea_mode = edid_cea_modes[vic];
 		unsigned int clock1, clock2;
@@ -2948,7 +2952,7 @@ static u8 drm_match_cea_mode_clock_tolerance(const struct drm_display_mode *to_m
 			continue;
 
 		do {
-			if (drm_mode_equal_no_clocks_no_stereo(to_match, &cea_mode))
+			if (drm_mode_match(to_match, &cea_mode, match_flags))
 				return vic;
 		} while (cea_mode_alternate_timings(vic, &cea_mode));
 	}
@@ -2965,11 +2969,15 @@ static u8 drm_match_cea_mode_clock_tolerance(const struct drm_display_mode *to_m
  */
 u8 drm_match_cea_mode(const struct drm_display_mode *to_match)
 {
+	unsigned int match_flags = DRM_MODE_MATCH_TIMINGS | DRM_MODE_MATCH_FLAGS;
 	u8 vic;
 
 	if (!to_match->clock)
 		return 0;
 
+	if (to_match->picture_aspect_ratio)
+		match_flags |= DRM_MODE_MATCH_ASPECT_RATIO;
+
 	for (vic = 1; vic < ARRAY_SIZE(edid_cea_modes); vic++) {
 		struct drm_display_mode cea_mode = edid_cea_modes[vic];
 		unsigned int clock1, clock2;
@@ -2983,7 +2991,7 @@ u8 drm_match_cea_mode(const struct drm_display_mode *to_match)
 			continue;
 
 		do {
-			if (drm_mode_equal_no_clocks_no_stereo(to_match, &cea_mode))
+			if (drm_mode_match(to_match, &cea_mode, match_flags))
 				return vic;
 		} while (cea_mode_alternate_timings(vic, &cea_mode));
 	}
@@ -3030,6 +3038,7 @@ hdmi_mode_alternate_clock(const struct drm_display_mode *hdmi_mode)
 static u8 drm_match_hdmi_mode_clock_tolerance(const struct drm_display_mode *to_match,
 					      unsigned int clock_tolerance)
 {
+	unsigned int match_flags = DRM_MODE_MATCH_TIMINGS | DRM_MODE_MATCH_FLAGS;
 	u8 vic;
 
 	if (!to_match->clock)
@@ -3047,7 +3056,7 @@ static u8 drm_match_hdmi_mode_clock_tolerance(const struct drm_display_mode *to_
 		    abs(to_match->clock - clock2) > clock_tolerance)
 			continue;
 
-		if (drm_mode_equal_no_clocks(to_match, hdmi_mode))
+		if (drm_mode_match(to_match, hdmi_mode, match_flags))
 			return vic;
 	}
 
@@ -3064,6 +3073,7 @@ static u8 drm_match_hdmi_mode_clock_tolerance(const struct drm_display_mode *to_
  */
 static u8 drm_match_hdmi_mode(const struct drm_display_mode *to_match)
 {
+	unsigned int match_flags = DRM_MODE_MATCH_TIMINGS | DRM_MODE_MATCH_FLAGS;
 	u8 vic;
 
 	if (!to_match->clock)
@@ -3079,7 +3089,7 @@ static u8 drm_match_hdmi_mode(const struct drm_display_mode *to_match)
 
 		if ((KHZ2PICOS(to_match->clock) == KHZ2PICOS(clock1) ||
 		     KHZ2PICOS(to_match->clock) == KHZ2PICOS(clock2)) &&
-		    drm_mode_equal_no_clocks_no_stereo(to_match, hdmi_mode))
+		    drm_mode_match(to_match, hdmi_mode, match_flags))
 			return vic;
 	}
 	return 0;
@@ -4455,7 +4465,6 @@ drm_reset_display_info(struct drm_connector *connector)
 
 	info->non_desktop = 0;
 }
-EXPORT_SYMBOL_GPL(drm_reset_display_info);
 
 u32 drm_add_display_info(struct drm_connector *connector, const struct edid *edid)
 {
@@ -4533,7 +4542,6 @@ u32 drm_add_display_info(struct drm_connector *connector, const struct edid *edi
 		info->color_formats |= DRM_COLOR_FORMAT_YCRCB422;
 	return quirks;
 }
-EXPORT_SYMBOL_GPL(drm_add_display_info);
 
 static int validate_displayid(u8 *displayid, int length, int idx)
 {
@@ -4825,6 +4833,7 @@ drm_hdmi_avi_infoframe_from_display_mode(struct hdmi_avi_infoframe *frame,
 					 const struct drm_display_mode *mode,
 					 bool is_hdmi2_sink)
 {
+	enum hdmi_picture_aspect picture_aspect;
 	int err;
 
 	if (!frame || !mode)
@@ -4867,13 +4876,23 @@ drm_hdmi_avi_infoframe_from_display_mode(struct hdmi_avi_infoframe *frame,
 	 * Populate picture aspect ratio from either
 	 * user input (if specified) or from the CEA mode list.
 	 */
-	if (mode->picture_aspect_ratio == HDMI_PICTURE_ASPECT_4_3 ||
-		mode->picture_aspect_ratio == HDMI_PICTURE_ASPECT_16_9)
-		frame->picture_aspect = mode->picture_aspect_ratio;
-	else if (frame->video_code > 0)
-		frame->picture_aspect = drm_get_cea_aspect_ratio(
-						frame->video_code);
+	picture_aspect = mode->picture_aspect_ratio;
+	if (picture_aspect == HDMI_PICTURE_ASPECT_NONE)
+		picture_aspect = drm_get_cea_aspect_ratio(frame->video_code);
+
+	/*
+	 * The infoframe can't convey anything but none, 4:3
+	 * and 16:9, so if the user has asked for anything else
+	 * we can only satisfy it by specifying the right VIC.
+	 */
+	if (picture_aspect > HDMI_PICTURE_ASPECT_16_9) {
+		if (picture_aspect !=
+		    drm_get_cea_aspect_ratio(frame->video_code))
+			return -EINVAL;
+		picture_aspect = HDMI_PICTURE_ASPECT_NONE;
+	}
 
+	frame->picture_aspect = picture_aspect;
 	frame->active_aspect = HDMI_ACTIVE_ASPECT_PICTURE;
 	frame->scan_mode = HDMI_SCAN_MODE_UNDERSCAN;
 
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 0646b108030b..2ee1eaa66188 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -2183,7 +2183,11 @@ static bool drm_target_cloned(struct drm_fb_helper *fb_helper,
 		for (j = 0; j < i; j++) {
 			if (!enabled[j])
 				continue;
-			if (!drm_mode_equal(modes[j], modes[i]))
+			if (!drm_mode_match(modes[j], modes[i],
+					    DRM_MODE_MATCH_TIMINGS |
+					    DRM_MODE_MATCH_CLOCK |
+					    DRM_MODE_MATCH_FLAGS |
+					    DRM_MODE_MATCH_3D_FLAGS))
 				can_clone = false;
 		}
 	}
@@ -2203,7 +2207,11 @@ static bool drm_target_cloned(struct drm_fb_helper *fb_helper,
 
 		fb_helper_conn = fb_helper->connector_info[i];
 		list_for_each_entry(mode, &fb_helper_conn->connector->modes, head) {
-			if (drm_mode_equal(mode, dmt_mode))
+			if (drm_mode_match(mode, dmt_mode,
+					   DRM_MODE_MATCH_TIMINGS |
+					   DRM_MODE_MATCH_CLOCK |
+					   DRM_MODE_MATCH_FLAGS |
+					   DRM_MODE_MATCH_3D_FLAGS))
 				modes[i] = mode;
 		}
 		if (!modes[i])
diff --git a/drivers/gpu/drm/drm_framebuffer.c b/drivers/gpu/drm/drm_framebuffer.c
index ad67203de715..bfedceff87bb 100644
--- a/drivers/gpu/drm/drm_framebuffer.c
+++ b/drivers/gpu/drm/drm_framebuffer.c
@@ -468,29 +468,30 @@ int drm_mode_getfb(struct drm_device *dev,
 		goto out;
 	}
 
+	if (!fb->funcs->create_handle) {
+		ret = -ENODEV;
+		goto out;
+	}
+
 	r->height = fb->height;
 	r->width = fb->width;
 	r->depth = fb->format->depth;
 	r->bpp = fb->format->cpp[0] * 8;
 	r->pitch = fb->pitches[0];
-	if (fb->funcs->create_handle) {
-		if (drm_is_current_master(file_priv) || capable(CAP_SYS_ADMIN) ||
-		    drm_is_control_client(file_priv)) {
-			ret = fb->funcs->create_handle(fb, file_priv,
-						       &r->handle);
-		} else {
-			/* GET_FB() is an unprivileged ioctl so we must not
-			 * return a buffer-handle to non-master processes! For
-			 * backwards-compatibility reasons, we cannot make
-			 * GET_FB() privileged, so just return an invalid handle
-			 * for non-masters. */
-			r->handle = 0;
-			ret = 0;
-		}
-	} else {
-		ret = -ENODEV;
+
+	/* GET_FB() is an unprivileged ioctl so we must not return a
+	 * buffer-handle to non-master processes! For
+	 * backwards-compatibility reasons, we cannot make GET_FB() privileged,
+	 * so just return an invalid handle for non-masters.
+	 */
+	if (!drm_is_current_master(file_priv) && !capable(CAP_SYS_ADMIN)) {
+		r->handle = 0;
+		ret = 0;
+		goto out;
 	}
 
+	ret = fb->funcs->create_handle(fb, file_priv, &r->handle);
+
 out:
 	drm_framebuffer_put(fb);
 
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 4975ba9a7bc8..4a16d7b26c89 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -436,9 +436,12 @@ drm_gem_handle_create_tail(struct drm_file *file_priv,
  * @obj: object to register
  * @handlep: pionter to return the created handle to the caller
  *
- * Create a handle for this object. This adds a handle reference
- * to the object, which includes a regular reference count. Callers
- * will likely want to dereference the object afterwards.
+ * Create a handle for this object. This adds a handle reference to the object,
+ * which includes a regular reference count. Callers will likely want to
+ * dereference the object afterwards.
+ *
+ * Since this publishes @obj to userspace it must be fully set up by this point,
+ * drivers must call this last in their buffer object creation callbacks.
  */
 int drm_gem_handle_create(struct drm_file *file_priv,
 			  struct drm_gem_object *obj,
diff --git a/drivers/gpu/drm/drm_gem_framebuffer_helper.c b/drivers/gpu/drm/drm_gem_framebuffer_helper.c
index 4d682a6e8bcb..acfbc0641a06 100644
--- a/drivers/gpu/drm/drm_gem_framebuffer_helper.c
+++ b/drivers/gpu/drm/drm_gem_framebuffer_helper.c
@@ -22,6 +22,7 @@
 #include <drm/drm_gem.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_modeset_helper.h>
+#include <drm/drm_simple_kms_helper.h>
 
 /**
  * DOC: overview
@@ -265,6 +266,24 @@ int drm_gem_fb_prepare_fb(struct drm_plane *plane,
 }
 EXPORT_SYMBOL_GPL(drm_gem_fb_prepare_fb);
 
+/**
+ * drm_gem_fb_simple_display_pipe_prepare_fb - prepare_fb helper for
+ *     &drm_simple_display_pipe
+ * @pipe: Simple display pipe
+ * @plane_state: Plane state
+ *
+ * This function uses drm_gem_fb_prepare_fb() to check if the plane FB has a
+ * &dma_buf attached, extracts the exclusive fence and attaches it to plane
+ * state for the atomic helper to wait on. Drivers can use this as their
+ * &drm_simple_display_pipe_funcs.prepare_fb callback.
+ */
+int drm_gem_fb_simple_display_pipe_prepare_fb(struct drm_simple_display_pipe *pipe,
+					      struct drm_plane_state *plane_state)
+{
+	return drm_gem_fb_prepare_fb(&pipe->plane, plane_state);
+}
+EXPORT_SYMBOL(drm_gem_fb_simple_display_pipe_prepare_fb);
+
 /**
  * drm_gem_fbdev_fb_create - Create a GEM backed &drm_framebuffer for fbdev
  *                           emulation
diff --git a/drivers/gpu/drm/drm_ioc32.c b/drivers/gpu/drm/drm_ioc32.c
index f8e96e648acf..67b1fca39aa6 100644
--- a/drivers/gpu/drm/drm_ioc32.c
+++ b/drivers/gpu/drm/drm_ioc32.c
@@ -105,7 +105,7 @@ static int compat_drm_version(struct file *file, unsigned int cmd,
 		.desc = compat_ptr(v32.desc),
 	};
 	err = drm_ioctl_kernel(file, drm_version, &v,
-			DRM_UNLOCKED|DRM_RENDER_ALLOW|DRM_CONTROL_ALLOW);
+			       DRM_UNLOCKED|DRM_RENDER_ALLOW);
 	if (err)
 		return err;
 
@@ -885,7 +885,7 @@ static int compat_drm_mode_addfb2(struct file *file, unsigned int cmd,
 		return -EFAULT;
 
 	err = drm_ioctl_kernel(file, drm_mode_addfb2, &req64,
-				DRM_CONTROL_ALLOW|DRM_UNLOCKED);
+			       DRM_UNLOCKED);
 	if (err)
 		return err;
 
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index af782911c505..0d4cfb232576 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -324,6 +324,15 @@ drm_setclientcap(struct drm_device *dev, void *data, struct drm_file *file_priv)
 			return -EINVAL;
 		file_priv->atomic = req->value;
 		file_priv->universal_planes = req->value;
+		/*
+		 * No atomic user-space blows up on aspect ratio mode bits.
+		 */
+		file_priv->aspect_ratio_allowed = req->value;
+		break;
+	case DRM_CLIENT_CAP_ASPECT_RATIO:
+		if (req->value > 1)
+			return -EINVAL;
+		file_priv->aspect_ratio_allowed = req->value;
 		break;
 	default:
 		return -EINVAL;
@@ -510,13 +519,7 @@ int drm_ioctl_permit(u32 flags, struct drm_file *file_priv)
 
 	/* MASTER is only for master or control clients */
 	if (unlikely((flags & DRM_MASTER) &&
-		     !drm_is_current_master(file_priv) &&
-		     !drm_is_control_client(file_priv)))
-		return -EACCES;
-
-	/* Control clients must be explicitly allowed */
-	if (unlikely(!(flags & DRM_CONTROL_ALLOW) &&
-		     drm_is_control_client(file_priv)))
+		     !drm_is_current_master(file_priv)))
 		return -EACCES;
 
 	/* Render clients must be explicitly allowed */
@@ -539,7 +542,7 @@ EXPORT_SYMBOL(drm_ioctl_permit);
 /* Ioctl table */
 static const struct drm_ioctl_desc drm_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_IOCTL_VERSION, drm_version,
-		      DRM_UNLOCKED|DRM_RENDER_ALLOW|DRM_CONTROL_ALLOW),
+		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF(DRM_IOCTL_GET_UNIQUE, drm_getunique, DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_GET_MAGIC, drm_getmagic, DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_IRQ_BUSID, drm_irq_by_busid, DRM_MASTER|DRM_ROOT_ONLY),
@@ -613,41 +616,41 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_IOCTL_GEM_FLINK, drm_gem_flink_ioctl, DRM_AUTH|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_GEM_OPEN, drm_gem_open_ioctl, DRM_AUTH|DRM_UNLOCKED),
 
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETRESOURCES, drm_mode_getresources, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETRESOURCES, drm_mode_getresources, DRM_UNLOCKED),
 
 	DRM_IOCTL_DEF(DRM_IOCTL_PRIME_HANDLE_TO_FD, drm_prime_handle_to_fd_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF(DRM_IOCTL_PRIME_FD_TO_HANDLE, drm_prime_fd_to_handle_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
 
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPLANERESOURCES, drm_mode_getplane_res, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETCRTC, drm_mode_getcrtc, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETCRTC, drm_mode_setcrtc, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPLANE, drm_mode_getplane, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETPLANE, drm_mode_setplane, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CURSOR, drm_mode_cursor_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPLANERESOURCES, drm_mode_getplane_res, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETCRTC, drm_mode_getcrtc, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETCRTC, drm_mode_setcrtc, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPLANE, drm_mode_getplane, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETPLANE, drm_mode_setplane, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CURSOR, drm_mode_cursor_ioctl, DRM_MASTER|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETGAMMA, drm_mode_gamma_get_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETGAMMA, drm_mode_gamma_set_ioctl, DRM_MASTER|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETENCODER, drm_mode_getencoder, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETCONNECTOR, drm_mode_getconnector, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ATTACHMODE, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DETACHMODE, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPROPERTY, drm_mode_getproperty_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETPROPERTY, drm_mode_connector_property_set_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPROPBLOB, drm_mode_getblob_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETFB, drm_mode_getfb, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB, drm_mode_addfb, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB2, drm_mode_addfb2, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_RMFB, drm_mode_rmfb, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_PAGE_FLIP, drm_mode_page_flip_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_GETPROPERTIES, drm_mode_obj_get_properties_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_SETPROPERTY, drm_mode_obj_set_property_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CURSOR2, drm_mode_cursor2_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ATOMIC, drm_mode_atomic_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATEPROPBLOB, drm_mode_createblob_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROYPROPBLOB, drm_mode_destroyblob_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETENCODER, drm_mode_getencoder, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETCONNECTOR, drm_mode_getconnector, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ATTACHMODE, drm_noop, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DETACHMODE, drm_noop, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPROPERTY, drm_mode_getproperty_ioctl, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETPROPERTY, drm_mode_connector_property_set_ioctl, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPROPBLOB, drm_mode_getblob_ioctl, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETFB, drm_mode_getfb, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB, drm_mode_addfb, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB2, drm_mode_addfb2, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_RMFB, drm_mode_rmfb, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_PAGE_FLIP, drm_mode_page_flip_ioctl, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_GETPROPERTIES, drm_mode_obj_get_properties_ioctl, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_SETPROPERTY, drm_mode_obj_set_property_ioctl, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CURSOR2, drm_mode_cursor2_ioctl, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ATOMIC, drm_mode_atomic_ioctl, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATEPROPBLOB, drm_mode_createblob_ioctl, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROYPROPBLOB, drm_mode_destroyblob_ioctl, DRM_UNLOCKED),
 
 	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_CREATE, drm_syncobj_create_ioctl,
 		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
@@ -665,10 +668,10 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
 		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF(DRM_IOCTL_CRTC_GET_SEQUENCE, drm_crtc_get_sequence_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_CRTC_QUEUE_SEQUENCE, drm_crtc_queue_sequence_ioctl, DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_LEASE, drm_mode_create_lease_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_LIST_LESSEES, drm_mode_list_lessees_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GET_LEASE, drm_mode_get_lease_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_REVOKE_LEASE, drm_mode_revoke_lease_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_LEASE, drm_mode_create_lease_ioctl, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_LIST_LESSEES, drm_mode_list_lessees_ioctl, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GET_LEASE, drm_mode_get_lease_ioctl, DRM_MASTER|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_REVOKE_LEASE, drm_mode_revoke_lease_ioctl, DRM_MASTER|DRM_UNLOCKED),
 };
 
 #define DRM_CORE_IOCTL_COUNT	ARRAY_SIZE( drm_ioctls )
diff --git a/drivers/gpu/drm/drm_lease.c b/drivers/gpu/drm/drm_lease.c
index d345563fdff3..50c73c0a20b9 100644
--- a/drivers/gpu/drm/drm_lease.c
+++ b/drivers/gpu/drm/drm_lease.c
@@ -340,7 +340,7 @@ static void _drm_lease_revoke(struct drm_master *top)
 				break;
 
 			/* Over */
-			master = list_entry(master->lessee_list.next, struct drm_master, lessee_list);
+			master = list_next_entry(master, lessee_list);
 		}
 	}
 }
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index e82b61e08f8c..c78ca0e84ffd 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -939,17 +939,68 @@ struct drm_display_mode *drm_mode_duplicate(struct drm_device *dev,
 }
 EXPORT_SYMBOL(drm_mode_duplicate);
 
+static bool drm_mode_match_timings(const struct drm_display_mode *mode1,
+				   const struct drm_display_mode *mode2)
+{
+	return mode1->hdisplay == mode2->hdisplay &&
+		mode1->hsync_start == mode2->hsync_start &&
+		mode1->hsync_end == mode2->hsync_end &&
+		mode1->htotal == mode2->htotal &&
+		mode1->hskew == mode2->hskew &&
+		mode1->vdisplay == mode2->vdisplay &&
+		mode1->vsync_start == mode2->vsync_start &&
+		mode1->vsync_end == mode2->vsync_end &&
+		mode1->vtotal == mode2->vtotal &&
+		mode1->vscan == mode2->vscan;
+}
+
+static bool drm_mode_match_clock(const struct drm_display_mode *mode1,
+				  const struct drm_display_mode *mode2)
+{
+	/*
+	 * do clock check convert to PICOS
+	 * so fb modes get matched the same
+	 */
+	if (mode1->clock && mode2->clock)
+		return KHZ2PICOS(mode1->clock) == KHZ2PICOS(mode2->clock);
+	else
+		return mode1->clock == mode2->clock;
+}
+
+static bool drm_mode_match_flags(const struct drm_display_mode *mode1,
+				 const struct drm_display_mode *mode2)
+{
+	return (mode1->flags & ~DRM_MODE_FLAG_3D_MASK) ==
+		(mode2->flags & ~DRM_MODE_FLAG_3D_MASK);
+}
+
+static bool drm_mode_match_3d_flags(const struct drm_display_mode *mode1,
+				    const struct drm_display_mode *mode2)
+{
+	return (mode1->flags & DRM_MODE_FLAG_3D_MASK) ==
+		(mode2->flags & DRM_MODE_FLAG_3D_MASK);
+}
+
+static bool drm_mode_match_aspect_ratio(const struct drm_display_mode *mode1,
+					const struct drm_display_mode *mode2)
+{
+	return mode1->picture_aspect_ratio == mode2->picture_aspect_ratio;
+}
+
 /**
- * drm_mode_equal - test modes for equality
+ * drm_mode_match - test modes for (partial) equality
  * @mode1: first mode
  * @mode2: second mode
+ * @match_flags: which parts need to match (DRM_MODE_MATCH_*)
  *
  * Check to see if @mode1 and @mode2 are equivalent.
  *
  * Returns:
- * True if the modes are equal, false otherwise.
+ * True if the modes are (partially) equal, false otherwise.
  */
-bool drm_mode_equal(const struct drm_display_mode *mode1, const struct drm_display_mode *mode2)
+bool drm_mode_match(const struct drm_display_mode *mode1,
+		    const struct drm_display_mode *mode2,
+		    unsigned int match_flags)
 {
 	if (!mode1 && !mode2)
 		return true;
@@ -957,15 +1008,49 @@ bool drm_mode_equal(const struct drm_display_mode *mode1, const struct drm_displ
 	if (!mode1 || !mode2)
 		return false;
 
-	/* do clock check convert to PICOS so fb modes get matched
-	 * the same */
-	if (mode1->clock && mode2->clock) {
-		if (KHZ2PICOS(mode1->clock) != KHZ2PICOS(mode2->clock))
-			return false;
-	} else if (mode1->clock != mode2->clock)
+	if (match_flags & DRM_MODE_MATCH_TIMINGS &&
+	    !drm_mode_match_timings(mode1, mode2))
+		return false;
+
+	if (match_flags & DRM_MODE_MATCH_CLOCK &&
+	    !drm_mode_match_clock(mode1, mode2))
+		return false;
+
+	if (match_flags & DRM_MODE_MATCH_FLAGS &&
+	    !drm_mode_match_flags(mode1, mode2))
+		return false;
+
+	if (match_flags & DRM_MODE_MATCH_3D_FLAGS &&
+	    !drm_mode_match_3d_flags(mode1, mode2))
 		return false;
 
-	return drm_mode_equal_no_clocks(mode1, mode2);
+	if (match_flags & DRM_MODE_MATCH_ASPECT_RATIO &&
+	    !drm_mode_match_aspect_ratio(mode1, mode2))
+		return false;
+
+	return true;
+}
+EXPORT_SYMBOL(drm_mode_match);
+
+/**
+ * drm_mode_equal - test modes for equality
+ * @mode1: first mode
+ * @mode2: second mode
+ *
+ * Check to see if @mode1 and @mode2 are equivalent.
+ *
+ * Returns:
+ * True if the modes are equal, false otherwise.
+ */
+bool drm_mode_equal(const struct drm_display_mode *mode1,
+		    const struct drm_display_mode *mode2)
+{
+	return drm_mode_match(mode1, mode2,
+			      DRM_MODE_MATCH_TIMINGS |
+			      DRM_MODE_MATCH_CLOCK |
+			      DRM_MODE_MATCH_FLAGS |
+			      DRM_MODE_MATCH_3D_FLAGS|
+			      DRM_MODE_MATCH_ASPECT_RATIO);
 }
 EXPORT_SYMBOL(drm_mode_equal);
 
@@ -980,13 +1065,13 @@ EXPORT_SYMBOL(drm_mode_equal);
  * Returns:
  * True if the modes are equal, false otherwise.
  */
-bool drm_mode_equal_no_clocks(const struct drm_display_mode *mode1, const struct drm_display_mode *mode2)
+bool drm_mode_equal_no_clocks(const struct drm_display_mode *mode1,
+			      const struct drm_display_mode *mode2)
 {
-	if ((mode1->flags & DRM_MODE_FLAG_3D_MASK) !=
-	    (mode2->flags & DRM_MODE_FLAG_3D_MASK))
-		return false;
-
-	return drm_mode_equal_no_clocks_no_stereo(mode1, mode2);
+	return drm_mode_match(mode1, mode2,
+			      DRM_MODE_MATCH_TIMINGS |
+			      DRM_MODE_MATCH_FLAGS |
+			      DRM_MODE_MATCH_3D_FLAGS);
 }
 EXPORT_SYMBOL(drm_mode_equal_no_clocks);
 
@@ -1004,21 +1089,9 @@ EXPORT_SYMBOL(drm_mode_equal_no_clocks);
 bool drm_mode_equal_no_clocks_no_stereo(const struct drm_display_mode *mode1,
 					const struct drm_display_mode *mode2)
 {
-	if (mode1->hdisplay == mode2->hdisplay &&
-	    mode1->hsync_start == mode2->hsync_start &&
-	    mode1->hsync_end == mode2->hsync_end &&
-	    mode1->htotal == mode2->htotal &&
-	    mode1->hskew == mode2->hskew &&
-	    mode1->vdisplay == mode2->vdisplay &&
-	    mode1->vsync_start == mode2->vsync_start &&
-	    mode1->vsync_end == mode2->vsync_end &&
-	    mode1->vtotal == mode2->vtotal &&
-	    mode1->vscan == mode2->vscan &&
-	    (mode1->flags & ~DRM_MODE_FLAG_3D_MASK) ==
-	     (mode2->flags & ~DRM_MODE_FLAG_3D_MASK))
-		return true;
-
-	return false;
+	return drm_mode_match(mode1, mode2,
+			      DRM_MODE_MATCH_TIMINGS |
+			      DRM_MODE_MATCH_FLAGS);
 }
 EXPORT_SYMBOL(drm_mode_equal_no_clocks_no_stereo);
 
@@ -1575,6 +1648,26 @@ void drm_mode_convert_to_umode(struct drm_mode_modeinfo *out,
 	out->vrefresh = in->vrefresh;
 	out->flags = in->flags;
 	out->type = in->type;
+
+	switch (in->picture_aspect_ratio) {
+	case HDMI_PICTURE_ASPECT_4_3:
+		out->flags |= DRM_MODE_FLAG_PIC_AR_4_3;
+		break;
+	case HDMI_PICTURE_ASPECT_16_9:
+		out->flags |= DRM_MODE_FLAG_PIC_AR_16_9;
+		break;
+	case HDMI_PICTURE_ASPECT_64_27:
+		out->flags |= DRM_MODE_FLAG_PIC_AR_64_27;
+		break;
+	case HDMI_PICTURE_ASPECT_256_135:
+		out->flags |= DRM_MODE_FLAG_PIC_AR_256_135;
+		break;
+	case HDMI_PICTURE_ASPECT_RESERVED:
+	default:
+		out->flags |= DRM_MODE_FLAG_PIC_AR_NONE;
+		break;
+	}
+
 	strncpy(out->name, in->name, DRM_DISPLAY_MODE_LEN);
 	out->name[DRM_DISPLAY_MODE_LEN-1] = 0;
 }
@@ -1621,6 +1714,30 @@ int drm_mode_convert_umode(struct drm_device *dev,
 	strncpy(out->name, in->name, DRM_DISPLAY_MODE_LEN);
 	out->name[DRM_DISPLAY_MODE_LEN-1] = 0;
 
+	/* Clearing picture aspect ratio bits from out flags,
+	 * as the aspect-ratio information is not stored in
+	 * flags for kernel-mode, but in picture_aspect_ratio.
+	 */
+	out->flags &= ~DRM_MODE_FLAG_PIC_AR_MASK;
+
+	switch (in->flags & DRM_MODE_FLAG_PIC_AR_MASK) {
+	case DRM_MODE_FLAG_PIC_AR_4_3:
+		out->picture_aspect_ratio |= HDMI_PICTURE_ASPECT_4_3;
+		break;
+	case DRM_MODE_FLAG_PIC_AR_16_9:
+		out->picture_aspect_ratio |= HDMI_PICTURE_ASPECT_16_9;
+		break;
+	case DRM_MODE_FLAG_PIC_AR_64_27:
+		out->picture_aspect_ratio |= HDMI_PICTURE_ASPECT_64_27;
+		break;
+	case DRM_MODE_FLAG_PIC_AR_256_135:
+		out->picture_aspect_ratio |= HDMI_PICTURE_ASPECT_256_135;
+		break;
+	default:
+		out->picture_aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
+		break;
+	}
+
 	out->status = drm_mode_validate_driver(dev, out);
 	if (out->status != MODE_OK)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/drm_panel_orientation_quirks.c b/drivers/gpu/drm/drm_panel_orientation_quirks.c
index 902cc1a71e45..fe9c6c731e87 100644
--- a/drivers/gpu/drm/drm_panel_orientation_quirks.c
+++ b/drivers/gpu/drm/drm_panel_orientation_quirks.c
@@ -60,7 +60,7 @@ static const struct drm_dmi_panel_orientation_data itworks_tw891 = {
 	.orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
 };
 
-static const struct drm_dmi_panel_orientation_data vios_lth17 = {
+static const struct drm_dmi_panel_orientation_data lcd800x1280_rightside_up = {
 	.width = 800,
 	.height = 1280,
 	.orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
@@ -102,12 +102,30 @@ static const struct dmi_system_id orientation_data[] = {
 		  DMI_EXACT_MATCH(DMI_BOARD_NAME, "TW891"),
 		},
 		.driver_data = (void *)&itworks_tw891,
+	}, {	/*
+		 * Lenovo Ideapad Miix 310 laptop, only some production batches
+		 * have a portrait screen, the resolution checks makes the quirk
+		 * apply only to those batches.
+		 */
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "80SG"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "MIIX 310-10ICR"),
+		},
+		.driver_data = (void *)&lcd800x1280_rightside_up,
+	}, {	/* Lenovo Ideapad Miix 320 */
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "80XF"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "Lenovo MIIX 320-10ICR"),
+		},
+		.driver_data = (void *)&lcd800x1280_rightside_up,
 	}, {	/* VIOS LTH17 */
 		.matches = {
 		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "VIOS"),
 		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "LTH17"),
 		},
-		.driver_data = (void *)&vios_lth17,
+		.driver_data = (void *)&lcd800x1280_rightside_up,
 	},
 	{}
 };
@@ -154,10 +172,9 @@ int drm_get_panel_orientation_quirk(int width, int height)
 		if (!bios_date)
 			continue;
 
-		for (i = 0; data->bios_dates[i]; i++) {
-			if (!strcmp(data->bios_dates[i], bios_date))
-				return data->orientation;
-		}
+		i = match_string(data->bios_dates, -1, bios_date);
+		if (i >= 0)
+			return data->orientation;
 	}
 
 	return DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c
index 6d2a6e428a3e..035054455301 100644
--- a/drivers/gpu/drm/drm_plane.c
+++ b/drivers/gpu/drm/drm_plane.c
@@ -756,6 +756,7 @@ static int drm_mode_cursor_universal(struct drm_crtc *crtc,
 				     struct drm_modeset_acquire_ctx *ctx)
 {
 	struct drm_device *dev = crtc->dev;
+	struct drm_plane *plane = crtc->cursor;
 	struct drm_framebuffer *fb = NULL;
 	struct drm_mode_fb_cmd2 fbreq = {
 		.width = req->width,
@@ -769,8 +770,8 @@ static int drm_mode_cursor_universal(struct drm_crtc *crtc,
 	uint32_t src_w = 0, src_h = 0;
 	int ret = 0;
 
-	BUG_ON(!crtc->cursor);
-	WARN_ON(crtc->cursor->crtc != crtc && crtc->cursor->crtc != NULL);
+	BUG_ON(!plane);
+	WARN_ON(plane->crtc != crtc && plane->crtc != NULL);
 
 	/*
 	 * Obtain fb we'll be using (either new or existing) and take an extra
@@ -784,13 +785,18 @@ static int drm_mode_cursor_universal(struct drm_crtc *crtc,
 				DRM_DEBUG_KMS("failed to wrap cursor buffer in drm framebuffer\n");
 				return PTR_ERR(fb);
 			}
+
 			fb->hot_x = req->hot_x;
 			fb->hot_y = req->hot_y;
 		} else {
 			fb = NULL;
 		}
 	} else {
-		fb = crtc->cursor->fb;
+		if (plane->state)
+			fb = plane->state->fb;
+		else
+			fb = plane->fb;
+
 		if (fb)
 			drm_framebuffer_get(fb);
 	}
@@ -810,7 +816,7 @@ static int drm_mode_cursor_universal(struct drm_crtc *crtc,
 		src_h = fb->height << 16;
 	}
 
-	ret = __setplane_internal(crtc->cursor, crtc, fb,
+	ret = __setplane_internal(plane, crtc, fb,
 				  crtc_x, crtc_y, crtc_w, crtc_h,
 				  0, 0, src_w, src_h, ctx);
 
@@ -931,7 +937,8 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 {
 	struct drm_mode_crtc_page_flip_target *page_flip = data;
 	struct drm_crtc *crtc;
-	struct drm_framebuffer *fb = NULL;
+	struct drm_plane *plane;
+	struct drm_framebuffer *fb = NULL, *old_fb;
 	struct drm_pending_vblank_event *e = NULL;
 	u32 target_vblank = page_flip->sequence;
 	struct drm_modeset_acquire_ctx ctx;
@@ -959,6 +966,8 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 	if (!crtc)
 		return -ENOENT;
 
+	plane = crtc->primary;
+
 	if (crtc->funcs->page_flip_target) {
 		u32 current_vblank;
 		int r;
@@ -1003,11 +1012,16 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 	ret = drm_modeset_lock(&crtc->mutex, &ctx);
 	if (ret)
 		goto out;
-	ret = drm_modeset_lock(&crtc->primary->mutex, &ctx);
+	ret = drm_modeset_lock(&plane->mutex, &ctx);
 	if (ret)
 		goto out;
 
-	if (crtc->primary->fb == NULL) {
+	if (plane->state)
+		old_fb = plane->state->fb;
+	else
+		old_fb = plane->fb;
+
+	if (old_fb == NULL) {
 		/* The framebuffer is currently unbound, presumably
 		 * due to a hotplug event, that userspace has not
 		 * yet discovered.
@@ -1022,8 +1036,8 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 		goto out;
 	}
 
-	if (crtc->state) {
-		const struct drm_plane_state *state = crtc->primary->state;
+	if (plane->state) {
+		const struct drm_plane_state *state = plane->state;
 
 		ret = drm_framebuffer_check_src_coords(state->src_x,
 						       state->src_y,
@@ -1031,12 +1045,13 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 						       state->src_h,
 						       fb);
 	} else {
-		ret = drm_crtc_check_viewport(crtc, crtc->x, crtc->y, &crtc->mode, fb);
+		ret = drm_crtc_check_viewport(crtc, crtc->x, crtc->y,
+					      &crtc->mode, fb);
 	}
 	if (ret)
 		goto out;
 
-	if (crtc->primary->fb->format != fb->format) {
+	if (old_fb->format != fb->format) {
 		DRM_DEBUG_KMS("Page flip is not allowed to change frame buffer format.\n");
 		ret = -EINVAL;
 		goto out;
@@ -1048,10 +1063,12 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 			ret = -ENOMEM;
 			goto out;
 		}
+
 		e->event.base.type = DRM_EVENT_FLIP_COMPLETE;
 		e->event.base.length = sizeof(e->event);
 		e->event.vbl.user_data = page_flip->user_data;
 		e->event.vbl.crtc_id = crtc->base.id;
+
 		ret = drm_event_reserve_init(dev, file_priv, &e->base, &e->event.base);
 		if (ret) {
 			kfree(e);
@@ -1060,7 +1077,7 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 		}
 	}
 
-	crtc->primary->old_fb = crtc->primary->fb;
+	plane->old_fb = plane->fb;
 	if (crtc->funcs->page_flip_target)
 		ret = crtc->funcs->page_flip_target(crtc, fb, e,
 						    page_flip->flags,
@@ -1073,19 +1090,18 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 		if (page_flip->flags & DRM_MODE_PAGE_FLIP_EVENT)
 			drm_event_cancel_free(dev, &e->base);
 		/* Keep the old fb, don't unref it. */
-		crtc->primary->old_fb = NULL;
+		plane->old_fb = NULL;
 	} else {
-		crtc->primary->fb = fb;
-		/* Unref only the old framebuffer. */
-		fb = NULL;
+		plane->fb = fb;
+		drm_framebuffer_get(fb);
 	}
 
 out:
 	if (fb)
 		drm_framebuffer_put(fb);
-	if (crtc->primary->old_fb)
-		drm_framebuffer_put(crtc->primary->old_fb);
-	crtc->primary->old_fb = NULL;
+	if (plane->old_fb)
+		drm_framebuffer_put(plane->old_fb);
+	plane->old_fb = NULL;
 
 	if (ret == -EDEADLK) {
 		ret = drm_modeset_backoff(&ctx);
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index 7856a9b3f8a8..397b46b33739 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -331,6 +331,9 @@ EXPORT_SYMBOL(drm_gem_map_dma_buf);
 
 /**
  * drm_gem_unmap_dma_buf - unmap_dma_buf implementation for GEM
+ * @attach: attachment to unmap buffer from
+ * @sgt: scatterlist info of the buffer to unmap
+ * @dir: direction of DMA transfer
  *
  * Not implemented. The unmap is done at drm_gem_map_detach().  This can be
  * used as the &dma_buf_ops.unmap_dma_buf callback.
@@ -406,7 +409,10 @@ void *drm_gem_dmabuf_vmap(struct dma_buf *dma_buf)
 	struct drm_gem_object *obj = dma_buf->priv;
 	struct drm_device *dev = obj->dev;
 
-	return dev->driver->gem_prime_vmap(obj);
+	if (dev->driver->gem_prime_vmap)
+		return dev->driver->gem_prime_vmap(obj);
+	else
+		return NULL;
 }
 EXPORT_SYMBOL(drm_gem_dmabuf_vmap);
 
@@ -423,12 +429,15 @@ void drm_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr)
 	struct drm_gem_object *obj = dma_buf->priv;
 	struct drm_device *dev = obj->dev;
 
-	dev->driver->gem_prime_vunmap(obj, vaddr);
+	if (dev->driver->gem_prime_vunmap)
+		dev->driver->gem_prime_vunmap(obj, vaddr);
 }
 EXPORT_SYMBOL(drm_gem_dmabuf_vunmap);
 
 /**
  * drm_gem_dmabuf_kmap_atomic - map_atomic implementation for GEM
+ * @dma_buf: buffer to be mapped
+ * @page_num: page number within the buffer
  *
  * Not implemented. This can be used as the &dma_buf_ops.map_atomic callback.
  */
@@ -441,6 +450,9 @@ EXPORT_SYMBOL(drm_gem_dmabuf_kmap_atomic);
 
 /**
  * drm_gem_dmabuf_kunmap_atomic - unmap_atomic implementation for GEM
+ * @dma_buf: buffer to be unmapped
+ * @page_num: page number within the buffer
+ * @addr: virtual address of the buffer
  *
  * Not implemented. This can be used as the &dma_buf_ops.unmap_atomic callback.
  */
@@ -453,6 +465,8 @@ EXPORT_SYMBOL(drm_gem_dmabuf_kunmap_atomic);
 
 /**
  * drm_gem_dmabuf_kmap - map implementation for GEM
+ * @dma_buf: buffer to be mapped
+ * @page_num: page number within the buffer
  *
  * Not implemented. This can be used as the &dma_buf_ops.map callback.
  */
@@ -464,6 +478,9 @@ EXPORT_SYMBOL(drm_gem_dmabuf_kmap);
 
 /**
  * drm_gem_dmabuf_kunmap - unmap implementation for GEM
+ * @dma_buf: buffer to be unmapped
+ * @page_num: page number within the buffer
+ * @addr: virtual address of the buffer
  *
  * Not implemented. This can be used as the &dma_buf_ops.unmap callback.
  */
diff --git a/drivers/gpu/drm/drm_property.c b/drivers/gpu/drm/drm_property.c
index 8f4672daac7f..1f8031e30f53 100644
--- a/drivers/gpu/drm/drm_property.c
+++ b/drivers/gpu/drm/drm_property.c
@@ -169,9 +169,9 @@ struct drm_property *drm_property_create_enum(struct drm_device *dev,
 		return NULL;
 
 	for (i = 0; i < num_values; i++) {
-		ret = drm_property_add_enum(property, i,
-				      props[i].type,
-				      props[i].name);
+		ret = drm_property_add_enum(property,
+					    props[i].type,
+					    props[i].name);
 		if (ret) {
 			drm_property_destroy(dev, property);
 			return NULL;
@@ -209,7 +209,7 @@ struct drm_property *drm_property_create_bitmask(struct drm_device *dev,
 						 uint64_t supported_bits)
 {
 	struct drm_property *property;
-	int i, ret, index = 0;
+	int i, ret;
 	int num_values = hweight64(supported_bits);
 
 	flags |= DRM_MODE_PROP_BITMASK;
@@ -221,14 +221,9 @@ struct drm_property *drm_property_create_bitmask(struct drm_device *dev,
 		if (!(supported_bits & (1ULL << props[i].type)))
 			continue;
 
-		if (WARN_ON(index >= num_values)) {
-			drm_property_destroy(dev, property);
-			return NULL;
-		}
-
-		ret = drm_property_add_enum(property, index++,
-				      props[i].type,
-				      props[i].name);
+		ret = drm_property_add_enum(property,
+					    props[i].type,
+					    props[i].name);
 		if (ret) {
 			drm_property_destroy(dev, property);
 			return NULL;
@@ -376,7 +371,6 @@ EXPORT_SYMBOL(drm_property_create_bool);
 /**
  * drm_property_add_enum - add a possible value to an enumeration property
  * @property: enumeration property to change
- * @index: index of the new enumeration
  * @value: value of the new enumeration
  * @name: symbolic name of the new enumeration
  *
@@ -388,10 +382,11 @@ EXPORT_SYMBOL(drm_property_create_bool);
  * Returns:
  * Zero on success, error code on failure.
  */
-int drm_property_add_enum(struct drm_property *property, int index,
+int drm_property_add_enum(struct drm_property *property,
 			  uint64_t value, const char *name)
 {
 	struct drm_property_enum *prop_enum;
+	int index = 0;
 
 	if (WARN_ON(strlen(name) >= DRM_PROP_NAME_LEN))
 		return -EINVAL;
@@ -411,8 +406,12 @@ int drm_property_add_enum(struct drm_property *property, int index,
 	list_for_each_entry(prop_enum, &property->enum_list, head) {
 		if (WARN_ON(prop_enum->value == value))
 			return -EINVAL;
+		index++;
 	}
 
+	if (WARN_ON(index >= property->num_values))
+		return -EINVAL;
+
 	prop_enum = kzalloc(sizeof(struct drm_property_enum), GFP_KERNEL);
 	if (!prop_enum)
 		return -ENOMEM;
diff --git a/drivers/gpu/drm/drm_rect.c b/drivers/gpu/drm/drm_rect.c
index 9817c1445ba9..8c057829b804 100644
--- a/drivers/gpu/drm/drm_rect.c
+++ b/drivers/gpu/drm/drm_rect.c
@@ -50,13 +50,25 @@ bool drm_rect_intersect(struct drm_rect *r1, const struct drm_rect *r2)
 }
 EXPORT_SYMBOL(drm_rect_intersect);
 
+static u32 clip_scaled(u32 src, u32 dst, u32 clip)
+{
+	u64 tmp = mul_u32_u32(src, dst - clip);
+
+	/*
+	 * Round toward 1.0 when clipping so that we don't accidentally
+	 * change upscaling to downscaling or vice versa.
+	 */
+	if (src < (dst << 16))
+		return DIV_ROUND_UP_ULL(tmp, dst);
+	else
+		return DIV_ROUND_DOWN_ULL(tmp, dst);
+}
+
 /**
  * drm_rect_clip_scaled - perform a scaled clip operation
  * @src: source window rectangle
  * @dst: destination window rectangle
  * @clip: clip rectangle
- * @hscale: horizontal scaling factor
- * @vscale: vertical scaling factor
  *
  * Clip rectangle @dst by rectangle @clip. Clip rectangle @src by the
  * same amounts multiplied by @hscale and @vscale.
@@ -66,33 +78,44 @@ EXPORT_SYMBOL(drm_rect_intersect);
  * %false otherwise
  */
 bool drm_rect_clip_scaled(struct drm_rect *src, struct drm_rect *dst,
-			  const struct drm_rect *clip,
-			  int hscale, int vscale)
+			  const struct drm_rect *clip)
 {
 	int diff;
 
 	diff = clip->x1 - dst->x1;
 	if (diff > 0) {
-		int64_t tmp = src->x1 + (int64_t) diff * hscale;
-		src->x1 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+		u32 new_src_w = clip_scaled(drm_rect_width(src),
+					    drm_rect_width(dst), diff);
+
+		src->x1 = clamp_t(int64_t, src->x2 - new_src_w, INT_MIN, INT_MAX);
+		dst->x1 = clip->x1;
 	}
 	diff = clip->y1 - dst->y1;
 	if (diff > 0) {
-		int64_t tmp = src->y1 + (int64_t) diff * vscale;
-		src->y1 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+		u32 new_src_h = clip_scaled(drm_rect_height(src),
+					    drm_rect_height(dst), diff);
+
+		src->y1 = clamp_t(int64_t, src->y2 - new_src_h, INT_MIN, INT_MAX);
+		dst->y1 = clip->y1;
 	}
 	diff = dst->x2 - clip->x2;
 	if (diff > 0) {
-		int64_t tmp = src->x2 - (int64_t) diff * hscale;
-		src->x2 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+		u32 new_src_w = clip_scaled(drm_rect_width(src),
+					    drm_rect_width(dst), diff);
+
+		src->x2 = clamp_t(int64_t, src->x1 + new_src_w, INT_MIN, INT_MAX);
+		dst->x2 = clip->x2;
 	}
 	diff = dst->y2 - clip->y2;
 	if (diff > 0) {
-		int64_t tmp = src->y2 - (int64_t) diff * vscale;
-		src->y2 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+		u32 new_src_h = clip_scaled(drm_rect_height(src),
+					    drm_rect_height(dst), diff);
+
+		src->y2 = clamp_t(int64_t, src->y1 + new_src_h, INT_MIN, INT_MAX);
+		dst->y2 = clip->y2;
 	}
 
-	return drm_rect_intersect(dst, clip);
+	return drm_rect_visible(dst);
 }
 EXPORT_SYMBOL(drm_rect_clip_scaled);
 
@@ -106,7 +129,10 @@ static int drm_calc_scale(int src, int dst)
 	if (dst == 0)
 		return 0;
 
-	scale = src / dst;
+	if (src > (dst << 16))
+		return DIV_ROUND_UP(src, dst);
+	else
+		scale = src / dst;
 
 	return scale;
 }
@@ -121,6 +147,10 @@ static int drm_calc_scale(int src, int dst)
  * Calculate the horizontal scaling factor as
  * (@src width) / (@dst width).
  *
+ * If the scale is below 1 << 16, round down. If the scale is above
+ * 1 << 16, round up. This will calculate the scale with the most
+ * pessimistic limit calculation.
+ *
  * RETURNS:
  * The horizontal scaling factor, or errno of out of limits.
  */
@@ -152,6 +182,10 @@ EXPORT_SYMBOL(drm_rect_calc_hscale);
  * Calculate the vertical scaling factor as
  * (@src height) / (@dst height).
  *
+ * If the scale is below 1 << 16, round down. If the scale is above
+ * 1 << 16, round up. This will calculate the scale with the most
+ * pessimistic limit calculation.
+ *
  * RETURNS:
  * The vertical scaling factor, or errno of out of limits.
  */
@@ -189,6 +223,10 @@ EXPORT_SYMBOL(drm_rect_calc_vscale);
  * If the calculated scaling factor is above @max_vscale,
  * decrease the height of rectangle @src to compensate.
  *
+ * If the scale is below 1 << 16, round down. If the scale is above
+ * 1 << 16, round up. This will calculate the scale with the most
+ * pessimistic limit calculation.
+ *
  * RETURNS:
  * The horizontal scaling factor.
  */
@@ -239,6 +277,10 @@ EXPORT_SYMBOL(drm_rect_calc_hscale_relaxed);
  * If the calculated scaling factor is above @max_vscale,
  * decrease the height of rectangle @src to compensate.
  *
+ * If the scale is below 1 << 16, round down. If the scale is above
+ * 1 << 16, round up. This will calculate the scale with the most
+ * pessimistic limit calculation.
+ *
  * RETURNS:
  * The vertical scaling factor.
  */
@@ -373,8 +415,8 @@ EXPORT_SYMBOL(drm_rect_rotate);
  * them when doing a rotatation and its inverse.
  * That is, if you do ::
  *
- *     DRM_MODE_PROP_ROTATE(&r, width, height, rotation);
- *     DRM_MODE_ROTATE_inv(&r, width, height, rotation);
+ *     drm_rect_rotate(&r, width, height, rotation);
+ *     drm_rect_rotate_inv(&r, width, height, rotation);
  *
  * you will always get back the original rectangle.
  */
diff --git a/drivers/gpu/drm/drm_scdc_helper.c b/drivers/gpu/drm/drm_scdc_helper.c
index 657ea5ab6c3f..870e25f1f788 100644
--- a/drivers/gpu/drm/drm_scdc_helper.c
+++ b/drivers/gpu/drm/drm_scdc_helper.c
@@ -141,7 +141,7 @@ bool drm_scdc_get_scrambling_status(struct i2c_adapter *adapter)
 
 	ret = drm_scdc_readb(adapter, SCDC_SCRAMBLER_STATUS, &status);
 	if (ret < 0) {
-		DRM_ERROR("Failed to read scrambling status: %d\n", ret);
+		DRM_DEBUG_KMS("Failed to read scrambling status: %d\n", ret);
 		return false;
 	}
 
@@ -168,7 +168,7 @@ bool drm_scdc_set_scrambling(struct i2c_adapter *adapter, bool enable)
 
 	ret = drm_scdc_readb(adapter, SCDC_TMDS_CONFIG, &config);
 	if (ret < 0) {
-		DRM_ERROR("Failed to read TMDS config: %d\n", ret);
+		DRM_DEBUG_KMS("Failed to read TMDS config: %d\n", ret);
 		return false;
 	}
 
@@ -179,7 +179,7 @@ bool drm_scdc_set_scrambling(struct i2c_adapter *adapter, bool enable)
 
 	ret = drm_scdc_writeb(adapter, SCDC_TMDS_CONFIG, config);
 	if (ret < 0) {
-		DRM_ERROR("Failed to enable scrambling: %d\n", ret);
+		DRM_DEBUG_KMS("Failed to enable scrambling: %d\n", ret);
 		return false;
 	}
 
@@ -223,7 +223,7 @@ bool drm_scdc_set_high_tmds_clock_ratio(struct i2c_adapter *adapter, bool set)
 
 	ret = drm_scdc_readb(adapter, SCDC_TMDS_CONFIG, &config);
 	if (ret < 0) {
-		DRM_ERROR("Failed to read TMDS config: %d\n", ret);
+		DRM_DEBUG_KMS("Failed to read TMDS config: %d\n", ret);
 		return false;
 	}
 
@@ -234,7 +234,7 @@ bool drm_scdc_set_high_tmds_clock_ratio(struct i2c_adapter *adapter, bool set)
 
 	ret = drm_scdc_writeb(adapter, SCDC_TMDS_CONFIG, config);
 	if (ret < 0) {
-		DRM_ERROR("Failed to set TMDS clock ratio: %d\n", ret);
+		DRM_DEBUG_KMS("Failed to set TMDS clock ratio: %d\n", ret);
 		return false;
 	}
 
diff --git a/drivers/gpu/drm/drm_simple_kms_helper.c b/drivers/gpu/drm/drm_simple_kms_helper.c
index 987a353c7f72..7a00455ca568 100644
--- a/drivers/gpu/drm/drm_simple_kms_helper.c
+++ b/drivers/gpu/drm/drm_simple_kms_helper.c
@@ -64,13 +64,15 @@ static int drm_simple_kms_crtc_check(struct drm_crtc *crtc,
 static void drm_simple_kms_crtc_enable(struct drm_crtc *crtc,
 				       struct drm_crtc_state *old_state)
 {
+	struct drm_plane *plane;
 	struct drm_simple_display_pipe *pipe;
 
 	pipe = container_of(crtc, struct drm_simple_display_pipe, crtc);
 	if (!pipe->funcs || !pipe->funcs->enable)
 		return;
 
-	pipe->funcs->enable(pipe, crtc->state);
+	plane = &pipe->plane;
+	pipe->funcs->enable(pipe, crtc->state, plane->state);
 }
 
 static void drm_simple_kms_crtc_disable(struct drm_crtc *crtc,
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index 1c5b5ce1fd7f..b3c1daad1169 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -331,9 +331,7 @@ struct device *drm_sysfs_minor_alloc(struct drm_minor *minor)
 	struct device *kdev;
 	int r;
 
-	if (minor->type == DRM_MINOR_CONTROL)
-		minor_str = "controlD%d";
-	else if (minor->type == DRM_MINOR_RENDER)
+	if (minor->type == DRM_MINOR_RENDER)
 		minor_str = "renderD%d";
 	else
 		minor_str = "card%d";
diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig
index e5bfeca361bd..041a77e400d4 100644
--- a/drivers/gpu/drm/etnaviv/Kconfig
+++ b/drivers/gpu/drm/etnaviv/Kconfig
@@ -22,11 +22,3 @@ config DRM_ETNAVIV_THERMAL
 	help
 	  Compile in support for thermal throttling.
 	  Say Y unless you want to risk burning your SoC.
-
-config DRM_ETNAVIV_REGISTER_LOGGING
-	bool "enable ETNAVIV register logging"
-	depends on DRM_ETNAVIV
-	help
-	  Compile in support for logging register reads/writes in a format
-	  that can be parsed by envytools demsm tool.  If enabled, register
-	  logging can be switched on via etnaviv.reglog=y module param.
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
index bfc6d4aa3b7c..7fea74861a87 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
@@ -1,18 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) 2014 Etnaviv Project
- * Author: Christian Gmeiner <christian.gmeiner@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2014-2018 Etnaviv Project
  */
 
 #include "etnaviv_cmdbuf.h"
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c b/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c
index 68e6d3772ad8..b106e8b288ad 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c
@@ -1,17 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) 2015 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2015-2018 Etnaviv Project
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c
index 3746827f45eb..a3c44f145c1d 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c
@@ -1,17 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) 2017 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2017-2018 Etnaviv Project
  */
 
 #include <drm/drm_mm.h>
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h
index ddc3f7ea169c..acb68c698363 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h
@@ -1,17 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2017 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #ifndef __ETNAVIV_CMDBUF_H__
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index ab50090d066c..e5013a999147 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -1,17 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) 2015 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2015-2018 Etnaviv Project
  */
 
 #include <linux/component.h>
@@ -25,57 +14,6 @@
 #include "etnaviv_mmu.h"
 #include "etnaviv_perfmon.h"
 
-#ifdef CONFIG_DRM_ETNAVIV_REGISTER_LOGGING
-static bool reglog;
-MODULE_PARM_DESC(reglog, "Enable register read/write logging");
-module_param(reglog, bool, 0600);
-#else
-#define reglog 0
-#endif
-
-void __iomem *etnaviv_ioremap(struct platform_device *pdev, const char *name,
-		const char *dbgname)
-{
-	struct resource *res;
-	void __iomem *ptr;
-
-	if (name)
-		res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
-	else
-		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-
-	ptr = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(ptr)) {
-		dev_err(&pdev->dev, "failed to ioremap %s: %ld\n", name,
-			PTR_ERR(ptr));
-		return ptr;
-	}
-
-	if (reglog)
-		dev_printk(KERN_DEBUG, &pdev->dev, "IO:region %s 0x%p %08zx\n",
-			   dbgname, ptr, (size_t)resource_size(res));
-
-	return ptr;
-}
-
-void etnaviv_writel(u32 data, void __iomem *addr)
-{
-	if (reglog)
-		printk(KERN_DEBUG "IO:W %p %08x\n", addr, data);
-
-	writel(data, addr);
-}
-
-u32 etnaviv_readl(const void __iomem *addr)
-{
-	u32 val = readl(addr);
-
-	if (reglog)
-		printk(KERN_DEBUG "IO:R %p %08x\n", addr, val);
-
-	return val;
-}
-
 /*
  * DRM operations:
  */
@@ -116,7 +54,7 @@ static int etnaviv_open(struct drm_device *dev, struct drm_file *file)
 			drm_sched_entity_init(&gpu->sched,
 				&ctx->sched_entity[i],
 				&gpu->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL],
-				32, NULL);
+				NULL);
 			}
 	}
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
index ddb17ee565e9..d36c7bbe66db 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -1,17 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (C) 2015 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2015-2018 Etnaviv Project
  */
 
 #ifndef __ETNAVIV_DRV_H__
@@ -26,6 +15,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/list.h>
+#include <linux/time64.h>
 #include <linux/types.h>
 #include <linux/sizes.h>
 
@@ -101,11 +91,6 @@ void etnaviv_gem_describe_objects(struct etnaviv_drm_private *priv,
 	struct seq_file *m);
 #endif
 
-void __iomem *etnaviv_ioremap(struct platform_device *pdev, const char *name,
-		const char *dbgname);
-void etnaviv_writel(u32 data, void __iomem *addr);
-u32 etnaviv_readl(const void __iomem *addr);
-
 #define DBG(fmt, ...) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
 #define VERB(fmt, ...) if (0) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
 
@@ -132,19 +117,27 @@ static inline bool fence_after_eq(u32 a, u32 b)
 	return (s32)(a - b) >= 0;
 }
 
+/*
+ * Etnaviv timeouts are specified wrt CLOCK_MONOTONIC, not jiffies.
+ * We need to calculate the timeout in terms of number of jiffies
+ * between the specified timeout and the current CLOCK_MONOTONIC time.
+ */
 static inline unsigned long etnaviv_timeout_to_jiffies(
 	const struct timespec *timeout)
 {
-	unsigned long timeout_jiffies = timespec_to_jiffies(timeout);
-	unsigned long start_jiffies = jiffies;
-	unsigned long remaining_jiffies;
+	struct timespec64 ts, to;
+
+	to = timespec_to_timespec64(*timeout);
+
+	ktime_get_ts64(&ts);
+
+	/* timeouts before "now" have already expired */
+	if (timespec64_compare(&to, &ts) <= 0)
+		return 0;
 
-	if (time_after(start_jiffies, timeout_jiffies))
-		remaining_jiffies = 0;
-	else
-		remaining_jiffies = timeout_jiffies - start_jiffies;
+	ts = timespec64_sub(to, ts);
 
-	return remaining_jiffies;
+	return timespec64_to_jiffies(&ts);
 }
 
 #endif /* __ETNAVIV_DRV_H__ */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c
index 48aef6cf6a42..9146e30e24a6 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c
@@ -1,17 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) 2015 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2015-2018 Etnaviv Project
  */
 
 #include <linux/devcoredump.h>
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.h b/drivers/gpu/drm/etnaviv/etnaviv_dump.h
index 97f2f8db9133..2d916c2667ee 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_dump.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.h
@@ -1,20 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2015 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * Etnaviv devcoredump file definitions
  */
+
 #ifndef ETNAVIV_DUMP_H
 #define ETNAVIV_DUMP_H
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index fcc969fa0e69..209ef1274b80 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -1,17 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) 2015 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2015-2018 Etnaviv Project
  */
 
 #include <linux/spinlock.h>
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.h b/drivers/gpu/drm/etnaviv/etnaviv_gem.h
index 93e696fcc14f..76079c2291f8 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.h
@@ -1,17 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (C) 2015 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2015-2018 Etnaviv Project
  */
 
 #ifndef __ETNAVIV_GEM_H__
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
index 5704305d41e6..0566171f8df2 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
@@ -1,18 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) 2013 Red Hat
- * Author: Rob Clark <robdclark@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2014-2018 Etnaviv Project
  */
 
 #include <linux/dma-buf.h>
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index 8a88799bf79b..686f6552db48 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -1,17 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) 2015 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2015-2018 Etnaviv Project
  */
 
 #include <linux/component.h>
@@ -1735,6 +1724,7 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct etnaviv_gpu *gpu;
+	struct resource *res;
 	int err;
 
 	gpu = devm_kzalloc(dev, sizeof(*gpu), GFP_KERNEL);
@@ -1746,7 +1736,8 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 	mutex_init(&gpu->fence_idr_lock);
 
 	/* Map registers: */
-	gpu->mmio = etnaviv_ioremap(pdev, NULL, dev_name(gpu->dev));
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	gpu->mmio = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(gpu->mmio))
 		return PTR_ERR(gpu->mmio);
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
index 3c3005501846..dd430f0f8ff5 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
@@ -1,17 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (C) 2015 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2015-2018 Etnaviv Project
  */
 
 #ifndef __ETNAVIV_GPU_H__
@@ -161,12 +150,12 @@ struct etnaviv_gpu {
 
 static inline void gpu_write(struct etnaviv_gpu *gpu, u32 reg, u32 data)
 {
-	etnaviv_writel(data, gpu->mmio + reg);
+	writel(data, gpu->mmio + reg);
 }
 
 static inline u32 gpu_read(struct etnaviv_gpu *gpu, u32 reg)
 {
-	return etnaviv_readl(gpu->mmio + reg);
+	return readl(gpu->mmio + reg);
 }
 
 static inline bool fence_completed(struct etnaviv_gpu *gpu, u32 fence)
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c b/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c
index ea08bb38caaf..39b463db76c9 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c
@@ -1,17 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2018 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include "etnaviv_gpu.h"
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c
index 4b9b11ca6f03..b163bdbcb880 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c
@@ -1,17 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) 2014 Christian Gmeiner <christian.gmeiner@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2014-2018 Etnaviv Project
  */
 
 #include <linux/platform_device.h>
@@ -47,11 +36,10 @@ static int __etnaviv_iommu_init(struct etnaviv_iommuv1_domain *etnaviv_domain)
 	u32 *p;
 	int i;
 
-	etnaviv_domain->base.bad_page_cpu = dma_alloc_coherent(
-						etnaviv_domain->base.dev,
-						SZ_4K,
-						&etnaviv_domain->base.bad_page_dma,
-						GFP_KERNEL);
+	etnaviv_domain->base.bad_page_cpu =
+			dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K,
+				     &etnaviv_domain->base.bad_page_dma,
+				     GFP_KERNEL);
 	if (!etnaviv_domain->base.bad_page_cpu)
 		return -ENOMEM;
 
@@ -59,14 +47,14 @@ static int __etnaviv_iommu_init(struct etnaviv_iommuv1_domain *etnaviv_domain)
 	for (i = 0; i < SZ_4K / 4; i++)
 		*p++ = 0xdead55aa;
 
-	etnaviv_domain->pgtable_cpu =
-			dma_alloc_coherent(etnaviv_domain->base.dev, PT_SIZE,
-					   &etnaviv_domain->pgtable_dma,
-					   GFP_KERNEL);
+	etnaviv_domain->pgtable_cpu = dma_alloc_wc(etnaviv_domain->base.dev,
+						   PT_SIZE,
+						   &etnaviv_domain->pgtable_dma,
+						   GFP_KERNEL);
 	if (!etnaviv_domain->pgtable_cpu) {
-		dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
-				  etnaviv_domain->base.bad_page_cpu,
-				  etnaviv_domain->base.bad_page_dma);
+		dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
+			    etnaviv_domain->base.bad_page_cpu,
+			    etnaviv_domain->base.bad_page_dma);
 		return -ENOMEM;
 	}
 
@@ -81,13 +69,12 @@ static void etnaviv_iommuv1_domain_free(struct etnaviv_iommu_domain *domain)
 	struct etnaviv_iommuv1_domain *etnaviv_domain =
 			to_etnaviv_domain(domain);
 
-	dma_free_coherent(etnaviv_domain->base.dev, PT_SIZE,
-			  etnaviv_domain->pgtable_cpu,
-			  etnaviv_domain->pgtable_dma);
+	dma_free_wc(etnaviv_domain->base.dev, PT_SIZE,
+		    etnaviv_domain->pgtable_cpu, etnaviv_domain->pgtable_dma);
 
-	dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
-			  etnaviv_domain->base.bad_page_cpu,
-			  etnaviv_domain->base.bad_page_dma);
+	dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
+		    etnaviv_domain->base.bad_page_cpu,
+		    etnaviv_domain->base.bad_page_dma);
 
 	kfree(etnaviv_domain);
 }
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.h b/drivers/gpu/drm/etnaviv/etnaviv_iommu.h
index 01d59bf70d78..b279404ce91a 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu.h
@@ -1,17 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (C) 2014 Christian Gmeiner <christian.gmeiner@gmail.com>
-  *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2014-2018 Etnaviv Project
  */
 
 #ifndef __ETNAVIV_IOMMU_H__
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
index 9752dbd5d28b..71fbc1f96cb6 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
@@ -1,17 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) 2016 Etnaviv Project
-  *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2016-2018 Etnaviv Project
  */
 
 #include <linux/platform_device.h>
@@ -47,8 +36,8 @@ struct etnaviv_iommuv2_domain {
 	u32 *mtlb_cpu;
 	dma_addr_t mtlb_dma;
 	/* S(lave) TLB aka second level pagetable */
-	u32 *stlb_cpu[1024];
-	dma_addr_t stlb_dma[1024];
+	u32 *stlb_cpu[MMUv2_MAX_STLB_ENTRIES];
+	dma_addr_t stlb_dma[MMUv2_MAX_STLB_ENTRIES];
 };
 
 static struct etnaviv_iommuv2_domain *
@@ -57,24 +46,54 @@ to_etnaviv_domain(struct etnaviv_iommu_domain *domain)
 	return container_of(domain, struct etnaviv_iommuv2_domain, base);
 }
 
+static int
+etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_domain *etnaviv_domain,
+			    int stlb)
+{
+	if (etnaviv_domain->stlb_cpu[stlb])
+		return 0;
+
+	etnaviv_domain->stlb_cpu[stlb] =
+			dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K,
+				     &etnaviv_domain->stlb_dma[stlb],
+				     GFP_KERNEL);
+
+	if (!etnaviv_domain->stlb_cpu[stlb])
+		return -ENOMEM;
+
+	memset32(etnaviv_domain->stlb_cpu[stlb], MMUv2_PTE_EXCEPTION,
+		 SZ_4K / sizeof(u32));
+
+	etnaviv_domain->mtlb_cpu[stlb] = etnaviv_domain->stlb_dma[stlb] |
+						      MMUv2_PTE_PRESENT;
+	return 0;
+}
+
 static int etnaviv_iommuv2_map(struct etnaviv_iommu_domain *domain,
 			       unsigned long iova, phys_addr_t paddr,
 			       size_t size, int prot)
 {
 	struct etnaviv_iommuv2_domain *etnaviv_domain =
 			to_etnaviv_domain(domain);
-	int mtlb_entry, stlb_entry;
-	u32 entry = (u32)paddr | MMUv2_PTE_PRESENT;
+	int mtlb_entry, stlb_entry, ret;
+	u32 entry = lower_32_bits(paddr) | MMUv2_PTE_PRESENT;
 
 	if (size != SZ_4K)
 		return -EINVAL;
 
+	if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
+		entry |= (upper_32_bits(paddr) & 0xff) << 4;
+
 	if (prot & ETNAVIV_PROT_WRITE)
 		entry |= MMUv2_PTE_WRITEABLE;
 
 	mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
 	stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
 
+	ret = etnaviv_iommuv2_ensure_stlb(etnaviv_domain, mtlb_entry);
+	if (ret)
+		return ret;
+
 	etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry;
 
 	return 0;
@@ -101,14 +120,13 @@ static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_domain *domain,
 static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
 {
 	u32 *p;
-	int ret, i, j;
+	int ret, i;
 
 	/* allocate scratch page */
-	etnaviv_domain->base.bad_page_cpu = dma_alloc_coherent(
-						etnaviv_domain->base.dev,
-						SZ_4K,
-						&etnaviv_domain->base.bad_page_dma,
-						GFP_KERNEL);
+	etnaviv_domain->base.bad_page_cpu =
+			dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K,
+				     &etnaviv_domain->base.bad_page_dma,
+				     GFP_KERNEL);
 	if (!etnaviv_domain->base.bad_page_cpu) {
 		ret = -ENOMEM;
 		goto fail_mem;
@@ -117,67 +135,40 @@ static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
 	for (i = 0; i < SZ_4K / 4; i++)
 		*p++ = 0xdead55aa;
 
-	etnaviv_domain->pta_cpu = dma_alloc_coherent(etnaviv_domain->base.dev,
-						     SZ_4K,
-						     &etnaviv_domain->pta_dma,
-						     GFP_KERNEL);
+	etnaviv_domain->pta_cpu = dma_alloc_wc(etnaviv_domain->base.dev,
+					       SZ_4K, &etnaviv_domain->pta_dma,
+					       GFP_KERNEL);
 	if (!etnaviv_domain->pta_cpu) {
 		ret = -ENOMEM;
 		goto fail_mem;
 	}
 
-	etnaviv_domain->mtlb_cpu = dma_alloc_coherent(etnaviv_domain->base.dev,
-						  SZ_4K,
-						  &etnaviv_domain->mtlb_dma,
-						  GFP_KERNEL);
+	etnaviv_domain->mtlb_cpu = dma_alloc_wc(etnaviv_domain->base.dev,
+						SZ_4K, &etnaviv_domain->mtlb_dma,
+						GFP_KERNEL);
 	if (!etnaviv_domain->mtlb_cpu) {
 		ret = -ENOMEM;
 		goto fail_mem;
 	}
 
-	/* pre-populate STLB pages (may want to switch to on-demand later) */
-	for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
-		etnaviv_domain->stlb_cpu[i] =
-				dma_alloc_coherent(etnaviv_domain->base.dev,
-						   SZ_4K,
-						   &etnaviv_domain->stlb_dma[i],
-						   GFP_KERNEL);
-		if (!etnaviv_domain->stlb_cpu[i]) {
-			ret = -ENOMEM;
-			goto fail_mem;
-		}
-		p = etnaviv_domain->stlb_cpu[i];
-		for (j = 0; j < SZ_4K / 4; j++)
-			*p++ = MMUv2_PTE_EXCEPTION;
-
-		etnaviv_domain->mtlb_cpu[i] = etnaviv_domain->stlb_dma[i] |
-					      MMUv2_PTE_PRESENT;
-	}
+	memset32(etnaviv_domain->mtlb_cpu, MMUv2_PTE_EXCEPTION,
+		 MMUv2_MAX_STLB_ENTRIES);
 
 	return 0;
 
 fail_mem:
 	if (etnaviv_domain->base.bad_page_cpu)
-		dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
-				  etnaviv_domain->base.bad_page_cpu,
-				  etnaviv_domain->base.bad_page_dma);
+		dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
+			    etnaviv_domain->base.bad_page_cpu,
+			    etnaviv_domain->base.bad_page_dma);
 
 	if (etnaviv_domain->pta_cpu)
-		dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
-				  etnaviv_domain->pta_cpu,
-				  etnaviv_domain->pta_dma);
+		dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
+			    etnaviv_domain->pta_cpu, etnaviv_domain->pta_dma);
 
 	if (etnaviv_domain->mtlb_cpu)
-		dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
-				  etnaviv_domain->mtlb_cpu,
-				  etnaviv_domain->mtlb_dma);
-
-	for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
-		if (etnaviv_domain->stlb_cpu[i])
-			dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
-					  etnaviv_domain->stlb_cpu[i],
-					  etnaviv_domain->stlb_dma[i]);
-	}
+		dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
+			    etnaviv_domain->mtlb_cpu, etnaviv_domain->mtlb_dma);
 
 	return ret;
 }
@@ -188,23 +179,21 @@ static void etnaviv_iommuv2_domain_free(struct etnaviv_iommu_domain *domain)
 			to_etnaviv_domain(domain);
 	int i;
 
-	dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
-			  etnaviv_domain->base.bad_page_cpu,
-			  etnaviv_domain->base.bad_page_dma);
+	dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
+		    etnaviv_domain->base.bad_page_cpu,
+		    etnaviv_domain->base.bad_page_dma);
 
-	dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
-			  etnaviv_domain->pta_cpu,
-			  etnaviv_domain->pta_dma);
+	dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
+		    etnaviv_domain->pta_cpu, etnaviv_domain->pta_dma);
 
-	dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
-			  etnaviv_domain->mtlb_cpu,
-			  etnaviv_domain->mtlb_dma);
+	dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
+		    etnaviv_domain->mtlb_cpu, etnaviv_domain->mtlb_dma);
 
 	for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
 		if (etnaviv_domain->stlb_cpu[i])
-			dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
-					  etnaviv_domain->stlb_cpu[i],
-					  etnaviv_domain->stlb_dma[i]);
+			dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
+				    etnaviv_domain->stlb_cpu[i],
+				    etnaviv_domain->stlb_dma[i]);
 	}
 
 	vfree(etnaviv_domain);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
index 49e049713a52..8069f9f36a2e 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
@@ -1,17 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) 2015 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2015-2018 Etnaviv Project
  */
 
 #include "common.xml.h"
@@ -162,22 +151,10 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu,
 		bool found;
 
 		ret = drm_mm_insert_node_in_range(&mmu->mm, node,
-						  size, 0, 0,
-						  mmu->last_iova, U64_MAX,
-						  mode);
+						  size, 0, 0, 0, U64_MAX, mode);
 		if (ret != -ENOSPC)
 			break;
 
-		/*
-		 * If we did not search from the start of the MMU region,
-		 * try again in case there are free slots.
-		 */
-		if (mmu->last_iova) {
-			mmu->last_iova = 0;
-			mmu->need_flush = true;
-			continue;
-		}
-
 		/* Try to retire some entries */
 		drm_mm_scan_init(&scan, &mmu->mm, size, 0, 0, mode);
 
@@ -274,7 +251,6 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu,
 	if (ret < 0)
 		goto unlock;
 
-	mmu->last_iova = node->start + etnaviv_obj->base.size;
 	mapping->iova = node->start;
 	ret = etnaviv_iommu_map(mmu, node->start, sgt, etnaviv_obj->base.size,
 				ETNAVIV_PROT_READ | ETNAVIV_PROT_WRITE);
@@ -381,7 +357,6 @@ int etnaviv_iommu_get_suballoc_va(struct etnaviv_gpu *gpu, dma_addr_t paddr,
 			mutex_unlock(&mmu->lock);
 			return ret;
 		}
-		mmu->last_iova = vram_node->start + size;
 		gpu->mmu->need_flush = true;
 		mutex_unlock(&mmu->lock);
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h
index ab603f5166b1..a0db17ffb686 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h
@@ -1,17 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (C) 2015 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) 2015-2018 Etnaviv Project
  */
 
 #ifndef __ETNAVIV_MMU_H__
@@ -59,7 +48,6 @@ struct etnaviv_iommu {
 	struct mutex lock;
 	struct list_head mappings;
 	struct drm_mm mm;
-	u32 last_iova;
 	bool need_flush;
 };
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c
index 26dddfc41aac..9980d81a26e3 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c
@@ -1,18 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2017 Etnaviv Project
  * Copyright (C) 2017 Zodiac Inflight Innovations
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include "etnaviv_gpu.h"
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.h b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.h
index c1653c64ab6b..4a9d508f6e10 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.h
@@ -1,18 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2017 Etnaviv Project
  * Copyright (C) 2017 Zodiac Inflight Innovations
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #ifndef __ETNAVIV_PERFMON_H__
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
index 6cf0775dbcd7..a74eb57af15b 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
@@ -1,17 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2017 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <linux/kthread.h>
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.h b/drivers/gpu/drm/etnaviv/etnaviv_sched.h
index 097635fa78ae..c0a6796e22c9 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.h
@@ -1,17 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2017 Etnaviv Project
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #ifndef __ETNAVIV_SCHED_H__
diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig
index 735ce47688f9..208bc27be3cc 100644
--- a/drivers/gpu/drm/exynos/Kconfig
+++ b/drivers/gpu/drm/exynos/Kconfig
@@ -1,6 +1,6 @@
 config DRM_EXYNOS
 	tristate "DRM Support for Samsung SoC EXYNOS Series"
-	depends on OF && DRM && (ARCH_S3C64XX || ARCH_EXYNOS || ARCH_MULTIPLATFORM)
+	depends on OF && DRM && (ARCH_S3C64XX || ARCH_S5PV210 || ARCH_EXYNOS || ARCH_MULTIPLATFORM)
 	select DRM_KMS_HELPER
 	select VIDEOMODE_HELPERS
 	select SND_SOC_HDMI_CODEC if SND_SOC
@@ -95,21 +95,31 @@ config DRM_EXYNOS_G2D
 	help
 	  Choose this option if you want to use Exynos G2D for DRM.
 
+config DRM_EXYNOS_IPP
+	bool
+
 config DRM_EXYNOS_FIMC
 	bool "FIMC"
-	depends on BROKEN && MFD_SYSCON
+	select DRM_EXYNOS_IPP
 	help
 	  Choose this option if you want to use Exynos FIMC for DRM.
 
 config DRM_EXYNOS_ROTATOR
 	bool "Rotator"
-	depends on BROKEN
+	select DRM_EXYNOS_IPP
 	help
 	  Choose this option if you want to use Exynos Rotator for DRM.
 
+config DRM_EXYNOS_SCALER
+	bool "Scaler"
+	select DRM_EXYNOS_IPP
+	help
+	  Choose this option if you want to use Exynos Scaler for DRM.
+
 config DRM_EXYNOS_GSC
 	bool "GScaler"
-	depends on BROKEN && ARCH_EXYNOS5 && VIDEO_SAMSUNG_EXYNOS_GSC=n
+	depends on VIDEO_SAMSUNG_EXYNOS_GSC=n
+	select DRM_EXYNOS_IPP
 	help
 	  Choose this option if you want to use Exynos GSC for DRM.
 
diff --git a/drivers/gpu/drm/exynos/Makefile b/drivers/gpu/drm/exynos/Makefile
index a51c5459bb13..3b323f1e0475 100644
--- a/drivers/gpu/drm/exynos/Makefile
+++ b/drivers/gpu/drm/exynos/Makefile
@@ -18,8 +18,10 @@ exynosdrm-$(CONFIG_DRM_EXYNOS_MIXER)	+= exynos_mixer.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_HDMI)	+= exynos_hdmi.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_VIDI)	+= exynos_drm_vidi.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_G2D)	+= exynos_drm_g2d.o
+exynosdrm-$(CONFIG_DRM_EXYNOS_IPP)	+= exynos_drm_ipp.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_FIMC)	+= exynos_drm_fimc.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_ROTATOR)	+= exynos_drm_rotator.o
+exynosdrm-$(CONFIG_DRM_EXYNOS_SCALER)	+= exynos_drm_scaler.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_GSC)	+= exynos_drm_gsc.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_MIC)     += exynos_drm_mic.o
 
diff --git a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
index 1c330f2a7a5d..82c95c34447f 100644
--- a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
@@ -31,7 +31,10 @@
 #define DSD_CFG_MUX 0x1004
 #define DSD_CFG_MUX_TE_UNMASK_GLOBAL BIT(13)
 
-#define WINDOWS_NR	3
+#define WINDOWS_NR	5
+#define PRIMARY_WIN	2
+#define CURSON_WIN	4
+
 #define MIN_FB_WIDTH_FOR_16WORD_BURST	128
 
 #define I80_HW_TRG	(1 << 0)
@@ -43,6 +46,9 @@ static const char * const decon_clks_name[] = {
 	"aclk_smmu_decon0x",
 	"aclk_xiu_decon0x",
 	"pclk_smmu_decon0x",
+	"aclk_smmu_decon1x",
+	"aclk_xiu_decon1x",
+	"pclk_smmu_decon1x",
 	"sclk_decon_vclk",
 	"sclk_decon_eclk",
 };
@@ -74,9 +80,8 @@ static const uint32_t decon_formats[] = {
 };
 
 static const enum drm_plane_type decon_win_types[WINDOWS_NR] = {
-	DRM_PLANE_TYPE_PRIMARY,
-	DRM_PLANE_TYPE_OVERLAY,
-	DRM_PLANE_TYPE_CURSOR,
+	[PRIMARY_WIN] = DRM_PLANE_TYPE_PRIMARY,
+	[CURSON_WIN] = DRM_PLANE_TYPE_CURSOR,
 };
 
 static inline void decon_set_bits(struct decon_context *ctx, u32 reg, u32 mask,
@@ -552,12 +557,10 @@ static int decon_bind(struct device *dev, struct device *master, void *data)
 	drm_dev->max_vblank_count = 0xffffffff;
 
 	for (win = ctx->first_win; win < WINDOWS_NR; win++) {
-		int tmp = (win == ctx->first_win) ? 0 : win;
-
 		ctx->configs[win].pixel_formats = decon_formats;
 		ctx->configs[win].num_pixel_formats = ARRAY_SIZE(decon_formats);
-		ctx->configs[win].zpos = win;
-		ctx->configs[win].type = decon_win_types[tmp];
+		ctx->configs[win].zpos = win - ctx->first_win;
+		ctx->configs[win].type = decon_win_types[win];
 
 		ret = exynos_plane_init(drm_dev, &ctx->planes[win], win,
 					&ctx->configs[win]);
@@ -565,7 +568,7 @@ static int decon_bind(struct device *dev, struct device *master, void *data)
 			return ret;
 	}
 
-	exynos_plane = &ctx->planes[ctx->first_win];
+	exynos_plane = &ctx->planes[PRIMARY_WIN];
 	out_type = (ctx->out_type & IFTYPE_HDMI) ? EXYNOS_DISPLAY_TYPE_HDMI
 						  : EXYNOS_DISPLAY_TYPE_LCD;
 	ctx->crtc = exynos_drm_crtc_create(drm_dev, &exynos_plane->base,
diff --git a/drivers/gpu/drm/exynos/exynos_dp.c b/drivers/gpu/drm/exynos/exynos_dp.c
index 964831dab102..86330f396784 100644
--- a/drivers/gpu/drm/exynos/exynos_dp.c
+++ b/drivers/gpu/drm/exynos/exynos_dp.c
@@ -162,7 +162,7 @@ static int exynos_dp_bind(struct device *dev, struct device *master, void *data)
 	dp->drm_dev = drm_dev;
 
 	dp->plat_data.dev_type = EXYNOS_DP;
-	dp->plat_data.power_on = exynos_dp_poweron;
+	dp->plat_data.power_on_start = exynos_dp_poweron;
 	dp->plat_data.power_off = exynos_dp_poweroff;
 	dp->plat_data.attach = exynos_dp_bridge_attach;
 	dp->plat_data.get_modes = exynos_dp_get_modes;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.c b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
index dc01342e759a..eea90251808f 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
@@ -228,7 +228,7 @@ struct exynos_drm_crtc *exynos_drm_crtc_get_by_type(struct drm_device *drm_dev,
 		if (to_exynos_crtc(crtc)->type == out_type)
 			return to_exynos_crtc(crtc);
 
-	return ERR_PTR(-EPERM);
+	return ERR_PTR(-ENODEV);
 }
 
 int exynos_drm_set_possible_crtcs(struct drm_encoder *encoder,
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index a518e9c6d6cc..a81b4a5e24a7 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -27,35 +27,23 @@
 #include "exynos_drm_fb.h"
 #include "exynos_drm_gem.h"
 #include "exynos_drm_plane.h"
+#include "exynos_drm_ipp.h"
 #include "exynos_drm_vidi.h"
 #include "exynos_drm_g2d.h"
 #include "exynos_drm_iommu.h"
 
 #define DRIVER_NAME	"exynos"
 #define DRIVER_DESC	"Samsung SoC DRM"
-#define DRIVER_DATE	"20110530"
-#define DRIVER_MAJOR	1
-#define DRIVER_MINOR	0
-
-int exynos_atomic_check(struct drm_device *dev,
-			struct drm_atomic_state *state)
-{
-	int ret;
-
-	ret = drm_atomic_helper_check_modeset(dev, state);
-	if (ret)
-		return ret;
+#define DRIVER_DATE	"20180330"
 
-	ret = drm_atomic_normalize_zpos(dev, state);
-	if (ret)
-		return ret;
-
-	ret = drm_atomic_helper_check_planes(dev, state);
-	if (ret)
-		return ret;
-
-	return ret;
-}
+/*
+ * Interface history:
+ *
+ * 1.0 - Original version
+ * 1.1 - Upgrade IPP driver to version 2.0
+ */
+#define DRIVER_MAJOR	1
+#define DRIVER_MINOR	1
 
 static int exynos_drm_open(struct drm_device *dev, struct drm_file *file)
 {
@@ -108,6 +96,16 @@ static const struct drm_ioctl_desc exynos_ioctls[] = {
 			DRM_AUTH | DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(EXYNOS_G2D_EXEC, exynos_g2d_exec_ioctl,
 			DRM_AUTH | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(EXYNOS_IPP_GET_RESOURCES,
+			exynos_drm_ipp_get_res_ioctl,
+			DRM_AUTH | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(EXYNOS_IPP_GET_CAPS, exynos_drm_ipp_get_caps_ioctl,
+			DRM_AUTH | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(EXYNOS_IPP_GET_LIMITS,
+			exynos_drm_ipp_get_limits_ioctl,
+			DRM_AUTH | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(EXYNOS_IPP_COMMIT, exynos_drm_ipp_commit_ioctl,
+			DRM_AUTH | DRM_RENDER_ALLOW),
 };
 
 static const struct file_operations exynos_drm_driver_fops = {
@@ -204,6 +202,7 @@ struct exynos_drm_driver_info {
 #define DRM_COMPONENT_DRIVER	BIT(0)	/* supports component framework */
 #define DRM_VIRTUAL_DEVICE	BIT(1)	/* create virtual platform device */
 #define DRM_DMA_DEVICE		BIT(2)	/* can be used for dma allocations */
+#define DRM_FIMC_DEVICE		BIT(3)	/* devices shared with V4L2 subsystem */
 
 #define DRV_PTR(drv, cond) (IS_ENABLED(cond) ? &drv : NULL)
 
@@ -243,10 +242,16 @@ static struct exynos_drm_driver_info exynos_drm_drivers[] = {
 		DRV_PTR(g2d_driver, CONFIG_DRM_EXYNOS_G2D),
 	}, {
 		DRV_PTR(fimc_driver, CONFIG_DRM_EXYNOS_FIMC),
+		DRM_COMPONENT_DRIVER | DRM_FIMC_DEVICE,
 	}, {
 		DRV_PTR(rotator_driver, CONFIG_DRM_EXYNOS_ROTATOR),
+		DRM_COMPONENT_DRIVER
+	}, {
+		DRV_PTR(scaler_driver, CONFIG_DRM_EXYNOS_SCALER),
+		DRM_COMPONENT_DRIVER
 	}, {
 		DRV_PTR(gsc_driver, CONFIG_DRM_EXYNOS_GSC),
+		DRM_COMPONENT_DRIVER
 	}, {
 		&exynos_drm_platform_driver,
 		DRM_VIRTUAL_DEVICE
@@ -274,7 +279,11 @@ static struct component_match *exynos_drm_match_add(struct device *dev)
 					    &info->driver->driver,
 					    (void *)platform_bus_type.match))) {
 			put_device(p);
-			component_match_add(dev, &match, compare_dev, d);
+
+			if (!(info->flags & DRM_FIMC_DEVICE) ||
+			    exynos_drm_check_fimc_device(d) == 0)
+				component_match_add(dev, &match,
+						    compare_dev, d);
 			p = d;
 		}
 		put_device(p);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h
index df2262f70d91..0f6d079a55c9 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h
@@ -273,9 +273,17 @@ static inline int exynos_dpi_bind(struct drm_device *dev,
 }
 #endif
 
+#ifdef CONFIG_DRM_EXYNOS_FIMC
+int exynos_drm_check_fimc_device(struct device *dev);
+#else
+static inline int exynos_drm_check_fimc_device(struct device *dev)
+{
+	return 0;
+}
+#endif
+
 int exynos_atomic_commit(struct drm_device *dev, struct drm_atomic_state *state,
 			 bool nonblock);
-int exynos_atomic_check(struct drm_device *dev, struct drm_atomic_state *state);
 
 
 extern struct platform_driver fimd_driver;
@@ -289,6 +297,7 @@ extern struct platform_driver vidi_driver;
 extern struct platform_driver g2d_driver;
 extern struct platform_driver fimc_driver;
 extern struct platform_driver rotator_driver;
+extern struct platform_driver scaler_driver;
 extern struct platform_driver gsc_driver;
 extern struct platform_driver mic_driver;
 #endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dsi.c b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
index 7904ffa9abfb..7c3030b7e586 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dsi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
@@ -270,7 +270,6 @@ struct exynos_dsi {
 	u32 lanes;
 	u32 mode_flags;
 	u32 format;
-	struct videomode vm;
 
 	int state;
 	struct drm_property *brightness;
@@ -881,30 +880,30 @@ static int exynos_dsi_init_link(struct exynos_dsi *dsi)
 
 static void exynos_dsi_set_display_mode(struct exynos_dsi *dsi)
 {
-	struct videomode *vm = &dsi->vm;
+	struct drm_display_mode *m = &dsi->encoder.crtc->state->adjusted_mode;
 	unsigned int num_bits_resol = dsi->driver_data->num_bits_resol;
 	u32 reg;
 
 	if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO) {
 		reg = DSIM_CMD_ALLOW(0xf)
-			| DSIM_STABLE_VFP(vm->vfront_porch)
-			| DSIM_MAIN_VBP(vm->vback_porch);
+			| DSIM_STABLE_VFP(m->vsync_start - m->vdisplay)
+			| DSIM_MAIN_VBP(m->vtotal - m->vsync_end);
 		exynos_dsi_write(dsi, DSIM_MVPORCH_REG, reg);
 
-		reg = DSIM_MAIN_HFP(vm->hfront_porch)
-			| DSIM_MAIN_HBP(vm->hback_porch);
+		reg = DSIM_MAIN_HFP(m->hsync_start - m->hdisplay)
+			| DSIM_MAIN_HBP(m->htotal - m->hsync_end);
 		exynos_dsi_write(dsi, DSIM_MHPORCH_REG, reg);
 
-		reg = DSIM_MAIN_VSA(vm->vsync_len)
-			| DSIM_MAIN_HSA(vm->hsync_len);
+		reg = DSIM_MAIN_VSA(m->vsync_end - m->vsync_start)
+			| DSIM_MAIN_HSA(m->hsync_end - m->hsync_start);
 		exynos_dsi_write(dsi, DSIM_MSYNC_REG, reg);
 	}
-	reg =  DSIM_MAIN_HRESOL(vm->hactive, num_bits_resol) |
-		DSIM_MAIN_VRESOL(vm->vactive, num_bits_resol);
+	reg =  DSIM_MAIN_HRESOL(m->hdisplay, num_bits_resol) |
+		DSIM_MAIN_VRESOL(m->vdisplay, num_bits_resol);
 
 	exynos_dsi_write(dsi, DSIM_MDRESOL_REG, reg);
 
-	dev_dbg(dsi->dev, "LCD size = %dx%d\n", vm->hactive, vm->vactive);
+	dev_dbg(dsi->dev, "LCD size = %dx%d\n", m->hdisplay, m->vdisplay);
 }
 
 static void exynos_dsi_set_display_enable(struct exynos_dsi *dsi, bool enable)
@@ -1265,15 +1264,15 @@ static irqreturn_t exynos_dsi_irq(int irq, void *dev_id)
 
 	if (status & DSIM_INT_SW_RST_RELEASE) {
 		u32 mask = ~(DSIM_INT_RX_DONE | DSIM_INT_SFR_FIFO_EMPTY |
-			DSIM_INT_SFR_HDR_FIFO_EMPTY | DSIM_INT_FRAME_DONE |
-			DSIM_INT_RX_ECC_ERR | DSIM_INT_SW_RST_RELEASE);
+			DSIM_INT_SFR_HDR_FIFO_EMPTY | DSIM_INT_RX_ECC_ERR |
+			DSIM_INT_SW_RST_RELEASE);
 		exynos_dsi_write(dsi, DSIM_INTMSK_REG, mask);
 		complete(&dsi->completed);
 		return IRQ_HANDLED;
 	}
 
 	if (!(status & (DSIM_INT_RX_DONE | DSIM_INT_SFR_FIFO_EMPTY |
-			DSIM_INT_FRAME_DONE | DSIM_INT_PLL_STABLE)))
+			DSIM_INT_PLL_STABLE)))
 		return IRQ_HANDLED;
 
 	if (exynos_dsi_transfer_finish(dsi))
@@ -1485,26 +1484,7 @@ static int exynos_dsi_create_connector(struct drm_encoder *encoder)
 	return 0;
 }
 
-static void exynos_dsi_mode_set(struct drm_encoder *encoder,
-				struct drm_display_mode *mode,
-				struct drm_display_mode *adjusted_mode)
-{
-	struct exynos_dsi *dsi = encoder_to_dsi(encoder);
-	struct videomode *vm = &dsi->vm;
-	struct drm_display_mode *m = adjusted_mode;
-
-	vm->hactive = m->hdisplay;
-	vm->vactive = m->vdisplay;
-	vm->vfront_porch = m->vsync_start - m->vdisplay;
-	vm->vback_porch = m->vtotal - m->vsync_end;
-	vm->vsync_len = m->vsync_end - m->vsync_start;
-	vm->hfront_porch = m->hsync_start - m->hdisplay;
-	vm->hback_porch = m->htotal - m->hsync_end;
-	vm->hsync_len = m->hsync_end - m->hsync_start;
-}
-
 static const struct drm_encoder_helper_funcs exynos_dsi_encoder_helper_funcs = {
-	.mode_set = exynos_dsi_mode_set,
 	.enable = exynos_dsi_enable,
 	.disable = exynos_dsi_disable,
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c
index f0e79178bde6..7fcc1a7ab1a0 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c
@@ -161,7 +161,7 @@ static struct drm_mode_config_helper_funcs exynos_drm_mode_config_helpers = {
 static const struct drm_mode_config_funcs exynos_drm_mode_config_funcs = {
 	.fb_create = exynos_user_fb_create,
 	.output_poll_changed = drm_fb_helper_output_poll_changed,
-	.atomic_check = exynos_atomic_check,
+	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
@@ -182,4 +182,6 @@ void exynos_drm_mode_config_init(struct drm_device *dev)
 	dev->mode_config.helper_private = &exynos_drm_mode_config_helpers;
 
 	dev->mode_config.allow_fb_modifiers = true;
+
+	dev->mode_config.normalize_zpos = true;
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimc.c b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
index 5b18b5c5fdf2..5ce84025d1cb 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
@@ -12,6 +12,7 @@
  *
  */
 #include <linux/kernel.h>
+#include <linux/component.h>
 #include <linux/platform_device.h>
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
@@ -24,8 +25,8 @@
 #include <drm/exynos_drm.h>
 #include "regs-fimc.h"
 #include "exynos_drm_drv.h"
+#include "exynos_drm_iommu.h"
 #include "exynos_drm_ipp.h"
-#include "exynos_drm_fimc.h"
 
 /*
  * FIMC stands for Fully Interactive Mobile Camera and
@@ -33,23 +34,6 @@
  * input DMA reads image data from the memory.
  * output DMA writes image data to memory.
  * FIMC supports image rotation and image effect functions.
- *
- * M2M operation : supports crop/scale/rotation/csc so on.
- * Memory ----> FIMC H/W ----> Memory.
- * Writeback operation : supports cloned screen with FIMD.
- * FIMD ----> FIMC H/W ----> Memory.
- * Output operation : supports direct display using local path.
- * Memory ----> FIMC H/W ----> FIMD.
- */
-
-/*
- * TODO
- * 1. check suspend/resume api if needed.
- * 2. need to check use case platform_device_id.
- * 3. check src/dst size with, height.
- * 4. added check_prepare api for right register.
- * 5. need to add supported list in prop_list.
- * 6. check prescaler/scaler optimization.
  */
 
 #define FIMC_MAX_DEVS	4
@@ -59,29 +43,19 @@
 #define FIMC_BUF_STOP	1
 #define FIMC_BUF_START	2
 #define FIMC_WIDTH_ITU_709	1280
-#define FIMC_REFRESH_MAX	60
-#define FIMC_REFRESH_MIN	12
-#define FIMC_CROP_MAX	8192
-#define FIMC_CROP_MIN	32
-#define FIMC_SCALE_MAX	4224
-#define FIMC_SCALE_MIN	32
+#define FIMC_AUTOSUSPEND_DELAY	2000
+
+static unsigned int fimc_mask = 0xc;
+module_param_named(fimc_devs, fimc_mask, uint, 0644);
+MODULE_PARM_DESC(fimc_devs, "Alias mask for assigning FIMC devices to Exynos DRM");
 
 #define get_fimc_context(dev)	platform_get_drvdata(to_platform_device(dev))
-#define get_ctx_from_ippdrv(ippdrv)	container_of(ippdrv,\
-					struct fimc_context, ippdrv);
-enum fimc_wb {
-	FIMC_WB_NONE,
-	FIMC_WB_A,
-	FIMC_WB_B,
-};
 
 enum {
 	FIMC_CLK_LCLK,
 	FIMC_CLK_GATE,
 	FIMC_CLK_WB_A,
 	FIMC_CLK_WB_B,
-	FIMC_CLK_MUX,
-	FIMC_CLK_PARENT,
 	FIMC_CLKS_MAX
 };
 
@@ -90,12 +64,8 @@ static const char * const fimc_clock_names[] = {
 	[FIMC_CLK_GATE]   = "fimc",
 	[FIMC_CLK_WB_A]   = "pxl_async0",
 	[FIMC_CLK_WB_B]   = "pxl_async1",
-	[FIMC_CLK_MUX]    = "mux",
-	[FIMC_CLK_PARENT] = "parent",
 };
 
-#define FIMC_DEFAULT_LCLK_FREQUENCY 133000000UL
-
 /*
  * A structure of scaler.
  *
@@ -107,7 +77,7 @@ static const char * const fimc_clock_names[] = {
  * @vratio: vertical ratio.
  */
 struct fimc_scaler {
-	bool	range;
+	bool range;
 	bool bypass;
 	bool up_h;
 	bool up_v;
@@ -115,57 +85,33 @@ struct fimc_scaler {
 	u32 vratio;
 };
 
-/*
- * A structure of scaler capability.
- *
- * find user manual table 43-1.
- * @in_hori: scaler input horizontal size.
- * @bypass: scaler bypass mode.
- * @dst_h_wo_rot: target horizontal size without output rotation.
- * @dst_h_rot: target horizontal size with output rotation.
- * @rl_w_wo_rot: real width without input rotation.
- * @rl_h_rot: real height without output rotation.
- */
-struct fimc_capability {
-	/* scaler */
-	u32	in_hori;
-	u32	bypass;
-	/* output rotator */
-	u32	dst_h_wo_rot;
-	u32	dst_h_rot;
-	/* input rotator */
-	u32	rl_w_wo_rot;
-	u32	rl_h_rot;
-};
-
 /*
  * A structure of fimc context.
  *
- * @ippdrv: prepare initialization using ippdrv.
  * @regs_res: register resources.
  * @regs: memory mapped io registers.
  * @lock: locking of operations.
  * @clocks: fimc clocks.
- * @clk_frequency: LCLK clock frequency.
- * @sysreg: handle to SYSREG block regmap.
  * @sc: scaler infomations.
  * @pol: porarity of writeback.
  * @id: fimc id.
  * @irq: irq number.
- * @suspended: qos operations.
  */
 struct fimc_context {
-	struct exynos_drm_ippdrv	ippdrv;
+	struct exynos_drm_ipp ipp;
+	struct drm_device *drm_dev;
+	struct device	*dev;
+	struct exynos_drm_ipp_task	*task;
+	struct exynos_drm_ipp_formats	*formats;
+	unsigned int			num_formats;
+
 	struct resource	*regs_res;
 	void __iomem	*regs;
 	spinlock_t	lock;
 	struct clk	*clocks[FIMC_CLKS_MAX];
-	u32		clk_frequency;
-	struct regmap	*sysreg;
 	struct fimc_scaler	sc;
 	int	id;
 	int	irq;
-	bool	suspended;
 };
 
 static u32 fimc_read(struct fimc_context *ctx, u32 reg)
@@ -217,19 +163,10 @@ static void fimc_sw_reset(struct fimc_context *ctx)
 	fimc_write(ctx, 0x0, EXYNOS_CIFCNTSEQ);
 }
 
-static int fimc_set_camblk_fimd0_wb(struct fimc_context *ctx)
-{
-	return regmap_update_bits(ctx->sysreg, SYSREG_CAMERA_BLK,
-				  SYSREG_FIMD0WB_DEST_MASK,
-				  ctx->id << SYSREG_FIMD0WB_DEST_SHIFT);
-}
-
-static void fimc_set_type_ctrl(struct fimc_context *ctx, enum fimc_wb wb)
+static void fimc_set_type_ctrl(struct fimc_context *ctx)
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("wb[%d]\n", wb);
-
 	cfg = fimc_read(ctx, EXYNOS_CIGCTRL);
 	cfg &= ~(EXYNOS_CIGCTRL_TESTPATTERN_MASK |
 		EXYNOS_CIGCTRL_SELCAM_ITU_MASK |
@@ -238,23 +175,10 @@ static void fimc_set_type_ctrl(struct fimc_context *ctx, enum fimc_wb wb)
 		EXYNOS_CIGCTRL_SELWB_CAMIF_MASK |
 		EXYNOS_CIGCTRL_SELWRITEBACK_MASK);
 
-	switch (wb) {
-	case FIMC_WB_A:
-		cfg |= (EXYNOS_CIGCTRL_SELWRITEBACK_A |
-			EXYNOS_CIGCTRL_SELWB_CAMIF_WRITEBACK);
-		break;
-	case FIMC_WB_B:
-		cfg |= (EXYNOS_CIGCTRL_SELWRITEBACK_B |
-			EXYNOS_CIGCTRL_SELWB_CAMIF_WRITEBACK);
-		break;
-	case FIMC_WB_NONE:
-	default:
-		cfg |= (EXYNOS_CIGCTRL_SELCAM_ITU_A |
-			EXYNOS_CIGCTRL_SELWRITEBACK_A |
-			EXYNOS_CIGCTRL_SELCAM_MIPI_A |
-			EXYNOS_CIGCTRL_SELCAM_FIMC_ITU);
-		break;
-	}
+	cfg |= (EXYNOS_CIGCTRL_SELCAM_ITU_A |
+		EXYNOS_CIGCTRL_SELWRITEBACK_A |
+		EXYNOS_CIGCTRL_SELCAM_MIPI_A |
+		EXYNOS_CIGCTRL_SELCAM_FIMC_ITU);
 
 	fimc_write(ctx, cfg, EXYNOS_CIGCTRL);
 }
@@ -296,7 +220,6 @@ static void fimc_clear_irq(struct fimc_context *ctx)
 
 static bool fimc_check_ovf(struct fimc_context *ctx)
 {
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
 	u32 status, flag;
 
 	status = fimc_read(ctx, EXYNOS_CISTATUS);
@@ -310,7 +233,7 @@ static bool fimc_check_ovf(struct fimc_context *ctx)
 			EXYNOS_CIWDOFST_CLROVFIY | EXYNOS_CIWDOFST_CLROVFICB |
 			EXYNOS_CIWDOFST_CLROVFICR);
 
-		dev_err(ippdrv->dev, "occurred overflow at %d, status 0x%x.\n",
+		dev_err(ctx->dev, "occurred overflow at %d, status 0x%x.\n",
 			ctx->id, status);
 		return true;
 	}
@@ -376,10 +299,8 @@ static void fimc_handle_lastend(struct fimc_context *ctx, bool enable)
 	fimc_write(ctx, cfg, EXYNOS_CIOCTRL);
 }
 
-
-static int fimc_src_set_fmt_order(struct fimc_context *ctx, u32 fmt)
+static void fimc_src_set_fmt_order(struct fimc_context *ctx, u32 fmt)
 {
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
 	u32 cfg;
 
 	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
@@ -392,12 +313,12 @@ static int fimc_src_set_fmt_order(struct fimc_context *ctx, u32 fmt)
 	case DRM_FORMAT_RGB565:
 		cfg |= EXYNOS_CISCCTRL_INRGB_FMT_RGB565;
 		fimc_write(ctx, cfg, EXYNOS_CISCCTRL);
-		return 0;
+		return;
 	case DRM_FORMAT_RGB888:
 	case DRM_FORMAT_XRGB8888:
 		cfg |= EXYNOS_CISCCTRL_INRGB_FMT_RGB888;
 		fimc_write(ctx, cfg, EXYNOS_CISCCTRL);
-		return 0;
+		return;
 	default:
 		/* bypass */
 		break;
@@ -438,20 +359,13 @@ static int fimc_src_set_fmt_order(struct fimc_context *ctx, u32 fmt)
 		cfg |= (EXYNOS_MSCTRL_ORDER2P_LSB_CBCR |
 			EXYNOS_MSCTRL_C_INT_IN_2PLANE);
 		break;
-	default:
-		dev_err(ippdrv->dev, "invalid source yuv order 0x%x.\n", fmt);
-		return -EINVAL;
 	}
 
 	fimc_write(ctx, cfg, EXYNOS_MSCTRL);
-
-	return 0;
 }
 
-static int fimc_src_set_fmt(struct device *dev, u32 fmt)
+static void fimc_src_set_fmt(struct fimc_context *ctx, u32 fmt, bool tiled)
 {
-	struct fimc_context *ctx = get_fimc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
 	u32 cfg;
 
 	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
@@ -485,9 +399,6 @@ static int fimc_src_set_fmt(struct device *dev, u32 fmt)
 	case DRM_FORMAT_NV21:
 		cfg |= EXYNOS_MSCTRL_INFORMAT_YCBCR420;
 		break;
-	default:
-		dev_err(ippdrv->dev, "invalid source format 0x%x.\n", fmt);
-		return -EINVAL;
 	}
 
 	fimc_write(ctx, cfg, EXYNOS_MSCTRL);
@@ -495,22 +406,22 @@ static int fimc_src_set_fmt(struct device *dev, u32 fmt)
 	cfg = fimc_read(ctx, EXYNOS_CIDMAPARAM);
 	cfg &= ~EXYNOS_CIDMAPARAM_R_MODE_MASK;
 
-	cfg |= EXYNOS_CIDMAPARAM_R_MODE_LINEAR;
+	if (tiled)
+		cfg |= EXYNOS_CIDMAPARAM_R_MODE_64X32;
+	else
+		cfg |= EXYNOS_CIDMAPARAM_R_MODE_LINEAR;
 
 	fimc_write(ctx, cfg, EXYNOS_CIDMAPARAM);
 
-	return fimc_src_set_fmt_order(ctx, fmt);
+	fimc_src_set_fmt_order(ctx, fmt);
 }
 
-static int fimc_src_set_transf(struct device *dev,
-		enum drm_exynos_degree degree,
-		enum drm_exynos_flip flip, bool *swap)
+static void fimc_src_set_transf(struct fimc_context *ctx, unsigned int rotation)
 {
-	struct fimc_context *ctx = get_fimc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
+	unsigned int degree = rotation & DRM_MODE_ROTATE_MASK;
 	u32 cfg1, cfg2;
 
-	DRM_DEBUG_KMS("degree[%d]flip[0x%x]\n", degree, flip);
+	DRM_DEBUG_KMS("rotation[%x]\n", rotation);
 
 	cfg1 = fimc_read(ctx, EXYNOS_MSCTRL);
 	cfg1 &= ~(EXYNOS_MSCTRL_FLIP_X_MIRROR |
@@ -520,61 +431,56 @@ static int fimc_src_set_transf(struct device *dev,
 	cfg2 &= ~EXYNOS_CITRGFMT_INROT90_CLOCKWISE;
 
 	switch (degree) {
-	case EXYNOS_DRM_DEGREE_0:
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
+	case DRM_MODE_ROTATE_0:
+		if (rotation & DRM_MODE_REFLECT_X)
 			cfg1 |= EXYNOS_MSCTRL_FLIP_X_MIRROR;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg1 |= EXYNOS_MSCTRL_FLIP_Y_MIRROR;
 		break;
-	case EXYNOS_DRM_DEGREE_90:
+	case DRM_MODE_ROTATE_90:
 		cfg2 |= EXYNOS_CITRGFMT_INROT90_CLOCKWISE;
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
+		if (rotation & DRM_MODE_REFLECT_X)
 			cfg1 |= EXYNOS_MSCTRL_FLIP_X_MIRROR;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg1 |= EXYNOS_MSCTRL_FLIP_Y_MIRROR;
 		break;
-	case EXYNOS_DRM_DEGREE_180:
+	case DRM_MODE_ROTATE_180:
 		cfg1 |= (EXYNOS_MSCTRL_FLIP_X_MIRROR |
 			EXYNOS_MSCTRL_FLIP_Y_MIRROR);
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
+		if (rotation & DRM_MODE_REFLECT_X)
 			cfg1 &= ~EXYNOS_MSCTRL_FLIP_X_MIRROR;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg1 &= ~EXYNOS_MSCTRL_FLIP_Y_MIRROR;
 		break;
-	case EXYNOS_DRM_DEGREE_270:
+	case DRM_MODE_ROTATE_270:
 		cfg1 |= (EXYNOS_MSCTRL_FLIP_X_MIRROR |
 			EXYNOS_MSCTRL_FLIP_Y_MIRROR);
 		cfg2 |= EXYNOS_CITRGFMT_INROT90_CLOCKWISE;
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
+		if (rotation & DRM_MODE_REFLECT_X)
 			cfg1 &= ~EXYNOS_MSCTRL_FLIP_X_MIRROR;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg1 &= ~EXYNOS_MSCTRL_FLIP_Y_MIRROR;
 		break;
-	default:
-		dev_err(ippdrv->dev, "invalid degree value %d.\n", degree);
-		return -EINVAL;
 	}
 
 	fimc_write(ctx, cfg1, EXYNOS_MSCTRL);
 	fimc_write(ctx, cfg2, EXYNOS_CITRGFMT);
-	*swap = (cfg2 & EXYNOS_CITRGFMT_INROT90_CLOCKWISE) ? 1 : 0;
-
-	return 0;
 }
 
-static int fimc_set_window(struct fimc_context *ctx,
-		struct drm_exynos_pos *pos, struct drm_exynos_sz *sz)
+static void fimc_set_window(struct fimc_context *ctx,
+			    struct exynos_drm_ipp_buffer *buf)
 {
 	u32 cfg, h1, h2, v1, v2;
 
 	/* cropped image */
-	h1 = pos->x;
-	h2 = sz->hsize - pos->w - pos->x;
-	v1 = pos->y;
-	v2 = sz->vsize - pos->h - pos->y;
+	h1 = buf->rect.x;
+	h2 = buf->buf.width - buf->rect.w - buf->rect.x;
+	v1 = buf->rect.y;
+	v2 = buf->buf.height - buf->rect.h - buf->rect.y;
 
 	DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]hsize[%d]vsize[%d]\n",
-		pos->x, pos->y, pos->w, pos->h, sz->hsize, sz->vsize);
+		buf->rect.x, buf->rect.y, buf->rect.w, buf->rect.h,
+		buf->buf.width, buf->buf.height);
 	DRM_DEBUG_KMS("h1[%d]h2[%d]v1[%d]v2[%d]\n", h1, h2, v1, v2);
 
 	/*
@@ -592,42 +498,30 @@ static int fimc_set_window(struct fimc_context *ctx,
 	cfg = (EXYNOS_CIWDOFST2_WINHOROFST2(h2) |
 		EXYNOS_CIWDOFST2_WINVEROFST2(v2));
 	fimc_write(ctx, cfg, EXYNOS_CIWDOFST2);
-
-	return 0;
 }
 
-static int fimc_src_set_size(struct device *dev, int swap,
-		struct drm_exynos_pos *pos, struct drm_exynos_sz *sz)
+static void fimc_src_set_size(struct fimc_context *ctx,
+			      struct exynos_drm_ipp_buffer *buf)
 {
-	struct fimc_context *ctx = get_fimc_context(dev);
-	struct drm_exynos_pos img_pos = *pos;
-	struct drm_exynos_sz img_sz = *sz;
 	u32 cfg;
 
-	DRM_DEBUG_KMS("swap[%d]hsize[%d]vsize[%d]\n",
-		swap, sz->hsize, sz->vsize);
+	DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", buf->buf.width, buf->buf.height);
 
 	/* original size */
-	cfg = (EXYNOS_ORGISIZE_HORIZONTAL(img_sz.hsize) |
-		EXYNOS_ORGISIZE_VERTICAL(img_sz.vsize));
+	cfg = (EXYNOS_ORGISIZE_HORIZONTAL(buf->buf.width) |
+		EXYNOS_ORGISIZE_VERTICAL(buf->buf.height));
 
 	fimc_write(ctx, cfg, EXYNOS_ORGISIZE);
 
-	DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]\n", pos->x, pos->y, pos->w, pos->h);
-
-	if (swap) {
-		img_pos.w = pos->h;
-		img_pos.h = pos->w;
-		img_sz.hsize = sz->vsize;
-		img_sz.vsize = sz->hsize;
-	}
+	DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]\n", buf->rect.x, buf->rect.y,
+		buf->rect.w, buf->rect.h);
 
 	/* set input DMA image size */
 	cfg = fimc_read(ctx, EXYNOS_CIREAL_ISIZE);
 	cfg &= ~(EXYNOS_CIREAL_ISIZE_HEIGHT_MASK |
 		EXYNOS_CIREAL_ISIZE_WIDTH_MASK);
-	cfg |= (EXYNOS_CIREAL_ISIZE_WIDTH(img_pos.w) |
-		EXYNOS_CIREAL_ISIZE_HEIGHT(img_pos.h));
+	cfg |= (EXYNOS_CIREAL_ISIZE_WIDTH(buf->rect.w) |
+		EXYNOS_CIREAL_ISIZE_HEIGHT(buf->rect.h));
 	fimc_write(ctx, cfg, EXYNOS_CIREAL_ISIZE);
 
 	/*
@@ -635,91 +529,34 @@ static int fimc_src_set_size(struct device *dev, int swap,
 	 * for now, we support only ITU601 8 bit mode
 	 */
 	cfg = (EXYNOS_CISRCFMT_ITU601_8BIT |
-		EXYNOS_CISRCFMT_SOURCEHSIZE(img_sz.hsize) |
-		EXYNOS_CISRCFMT_SOURCEVSIZE(img_sz.vsize));
+		EXYNOS_CISRCFMT_SOURCEHSIZE(buf->buf.width) |
+		EXYNOS_CISRCFMT_SOURCEVSIZE(buf->buf.height));
 	fimc_write(ctx, cfg, EXYNOS_CISRCFMT);
 
 	/* offset Y(RGB), Cb, Cr */
-	cfg = (EXYNOS_CIIYOFF_HORIZONTAL(img_pos.x) |
-		EXYNOS_CIIYOFF_VERTICAL(img_pos.y));
+	cfg = (EXYNOS_CIIYOFF_HORIZONTAL(buf->rect.x) |
+		EXYNOS_CIIYOFF_VERTICAL(buf->rect.y));
 	fimc_write(ctx, cfg, EXYNOS_CIIYOFF);
-	cfg = (EXYNOS_CIICBOFF_HORIZONTAL(img_pos.x) |
-		EXYNOS_CIICBOFF_VERTICAL(img_pos.y));
+	cfg = (EXYNOS_CIICBOFF_HORIZONTAL(buf->rect.x) |
+		EXYNOS_CIICBOFF_VERTICAL(buf->rect.y));
 	fimc_write(ctx, cfg, EXYNOS_CIICBOFF);
-	cfg = (EXYNOS_CIICROFF_HORIZONTAL(img_pos.x) |
-		EXYNOS_CIICROFF_VERTICAL(img_pos.y));
+	cfg = (EXYNOS_CIICROFF_HORIZONTAL(buf->rect.x) |
+		EXYNOS_CIICROFF_VERTICAL(buf->rect.y));
 	fimc_write(ctx, cfg, EXYNOS_CIICROFF);
 
-	return fimc_set_window(ctx, &img_pos, &img_sz);
+	fimc_set_window(ctx, buf);
 }
 
-static int fimc_src_set_addr(struct device *dev,
-		struct drm_exynos_ipp_buf_info *buf_info, u32 buf_id,
-		enum drm_exynos_ipp_buf_type buf_type)
+static void fimc_src_set_addr(struct fimc_context *ctx,
+			      struct exynos_drm_ipp_buffer *buf)
 {
-	struct fimc_context *ctx = get_fimc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node = ippdrv->c_node;
-	struct drm_exynos_ipp_property *property;
-	struct drm_exynos_ipp_config *config;
-
-	if (!c_node) {
-		DRM_ERROR("failed to get c_node.\n");
-		return -EINVAL;
-	}
-
-	property = &c_node->property;
-
-	DRM_DEBUG_KMS("prop_id[%d]buf_id[%d]buf_type[%d]\n",
-		property->prop_id, buf_id, buf_type);
-
-	if (buf_id > FIMC_MAX_SRC) {
-		dev_info(ippdrv->dev, "invalid buf_id %d.\n", buf_id);
-		return -ENOMEM;
-	}
-
-	/* address register set */
-	switch (buf_type) {
-	case IPP_BUF_ENQUEUE:
-		config = &property->config[EXYNOS_DRM_OPS_SRC];
-		fimc_write(ctx, buf_info->base[EXYNOS_DRM_PLANAR_Y],
-			EXYNOS_CIIYSA0);
-
-		if (config->fmt == DRM_FORMAT_YVU420) {
-			fimc_write(ctx, buf_info->base[EXYNOS_DRM_PLANAR_CR],
-				EXYNOS_CIICBSA0);
-			fimc_write(ctx, buf_info->base[EXYNOS_DRM_PLANAR_CB],
-				EXYNOS_CIICRSA0);
-		} else {
-			fimc_write(ctx, buf_info->base[EXYNOS_DRM_PLANAR_CB],
-				EXYNOS_CIICBSA0);
-			fimc_write(ctx, buf_info->base[EXYNOS_DRM_PLANAR_CR],
-				EXYNOS_CIICRSA0);
-		}
-		break;
-	case IPP_BUF_DEQUEUE:
-		fimc_write(ctx, 0x0, EXYNOS_CIIYSA0);
-		fimc_write(ctx, 0x0, EXYNOS_CIICBSA0);
-		fimc_write(ctx, 0x0, EXYNOS_CIICRSA0);
-		break;
-	default:
-		/* bypass */
-		break;
-	}
-
-	return 0;
+	fimc_write(ctx, buf->dma_addr[0], EXYNOS_CIIYSA(0));
+	fimc_write(ctx, buf->dma_addr[1], EXYNOS_CIICBSA(0));
+	fimc_write(ctx, buf->dma_addr[2], EXYNOS_CIICRSA(0));
 }
 
-static struct exynos_drm_ipp_ops fimc_src_ops = {
-	.set_fmt = fimc_src_set_fmt,
-	.set_transf = fimc_src_set_transf,
-	.set_size = fimc_src_set_size,
-	.set_addr = fimc_src_set_addr,
-};
-
-static int fimc_dst_set_fmt_order(struct fimc_context *ctx, u32 fmt)
+static void fimc_dst_set_fmt_order(struct fimc_context *ctx, u32 fmt)
 {
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
 	u32 cfg;
 
 	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
@@ -732,11 +569,11 @@ static int fimc_dst_set_fmt_order(struct fimc_context *ctx, u32 fmt)
 	case DRM_FORMAT_RGB565:
 		cfg |= EXYNOS_CISCCTRL_OUTRGB_FMT_RGB565;
 		fimc_write(ctx, cfg, EXYNOS_CISCCTRL);
-		return 0;
+		return;
 	case DRM_FORMAT_RGB888:
 		cfg |= EXYNOS_CISCCTRL_OUTRGB_FMT_RGB888;
 		fimc_write(ctx, cfg, EXYNOS_CISCCTRL);
-		return 0;
+		return;
 	case DRM_FORMAT_XRGB8888:
 		cfg |= (EXYNOS_CISCCTRL_OUTRGB_FMT_RGB888 |
 			EXYNOS_CISCCTRL_EXTRGB_EXTENSION);
@@ -784,20 +621,13 @@ static int fimc_dst_set_fmt_order(struct fimc_context *ctx, u32 fmt)
 		cfg |= EXYNOS_CIOCTRL_ORDER2P_LSB_CBCR;
 		cfg |= EXYNOS_CIOCTRL_YCBCR_2PLANE;
 		break;
-	default:
-		dev_err(ippdrv->dev, "invalid target yuv order 0x%x.\n", fmt);
-		return -EINVAL;
 	}
 
 	fimc_write(ctx, cfg, EXYNOS_CIOCTRL);
-
-	return 0;
 }
 
-static int fimc_dst_set_fmt(struct device *dev, u32 fmt)
+static void fimc_dst_set_fmt(struct fimc_context *ctx, u32 fmt, bool tiled)
 {
-	struct fimc_context *ctx = get_fimc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
 	u32 cfg;
 
 	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
@@ -837,10 +667,6 @@ static int fimc_dst_set_fmt(struct device *dev, u32 fmt)
 		case DRM_FORMAT_NV21:
 			cfg |= EXYNOS_CITRGFMT_OUTFORMAT_YCBCR420;
 			break;
-		default:
-			dev_err(ippdrv->dev, "invalid target format 0x%x.\n",
-				fmt);
-			return -EINVAL;
 		}
 
 		fimc_write(ctx, cfg, EXYNOS_CITRGFMT);
@@ -849,73 +675,67 @@ static int fimc_dst_set_fmt(struct device *dev, u32 fmt)
 	cfg = fimc_read(ctx, EXYNOS_CIDMAPARAM);
 	cfg &= ~EXYNOS_CIDMAPARAM_W_MODE_MASK;
 
-	cfg |= EXYNOS_CIDMAPARAM_W_MODE_LINEAR;
+	if (tiled)
+		cfg |= EXYNOS_CIDMAPARAM_W_MODE_64X32;
+	else
+		cfg |= EXYNOS_CIDMAPARAM_W_MODE_LINEAR;
 
 	fimc_write(ctx, cfg, EXYNOS_CIDMAPARAM);
 
-	return fimc_dst_set_fmt_order(ctx, fmt);
+	fimc_dst_set_fmt_order(ctx, fmt);
 }
 
-static int fimc_dst_set_transf(struct device *dev,
-		enum drm_exynos_degree degree,
-		enum drm_exynos_flip flip, bool *swap)
+static void fimc_dst_set_transf(struct fimc_context *ctx, unsigned int rotation)
 {
-	struct fimc_context *ctx = get_fimc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
+	unsigned int degree = rotation & DRM_MODE_ROTATE_MASK;
 	u32 cfg;
 
-	DRM_DEBUG_KMS("degree[%d]flip[0x%x]\n", degree, flip);
+	DRM_DEBUG_KMS("rotation[0x%x]\n", rotation);
 
 	cfg = fimc_read(ctx, EXYNOS_CITRGFMT);
 	cfg &= ~EXYNOS_CITRGFMT_FLIP_MASK;
 	cfg &= ~EXYNOS_CITRGFMT_OUTROT90_CLOCKWISE;
 
 	switch (degree) {
-	case EXYNOS_DRM_DEGREE_0:
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
+	case DRM_MODE_ROTATE_0:
+		if (rotation & DRM_MODE_REFLECT_X)
 			cfg |= EXYNOS_CITRGFMT_FLIP_X_MIRROR;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg |= EXYNOS_CITRGFMT_FLIP_Y_MIRROR;
 		break;
-	case EXYNOS_DRM_DEGREE_90:
+	case DRM_MODE_ROTATE_90:
 		cfg |= EXYNOS_CITRGFMT_OUTROT90_CLOCKWISE;
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
+		if (rotation & DRM_MODE_REFLECT_X)
 			cfg |= EXYNOS_CITRGFMT_FLIP_X_MIRROR;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg |= EXYNOS_CITRGFMT_FLIP_Y_MIRROR;
 		break;
-	case EXYNOS_DRM_DEGREE_180:
+	case DRM_MODE_ROTATE_180:
 		cfg |= (EXYNOS_CITRGFMT_FLIP_X_MIRROR |
 			EXYNOS_CITRGFMT_FLIP_Y_MIRROR);
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
+		if (rotation & DRM_MODE_REFLECT_X)
 			cfg &= ~EXYNOS_CITRGFMT_FLIP_X_MIRROR;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg &= ~EXYNOS_CITRGFMT_FLIP_Y_MIRROR;
 		break;
-	case EXYNOS_DRM_DEGREE_270:
+	case DRM_MODE_ROTATE_270:
 		cfg |= (EXYNOS_CITRGFMT_OUTROT90_CLOCKWISE |
 			EXYNOS_CITRGFMT_FLIP_X_MIRROR |
 			EXYNOS_CITRGFMT_FLIP_Y_MIRROR);
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
+		if (rotation & DRM_MODE_REFLECT_X)
 			cfg &= ~EXYNOS_CITRGFMT_FLIP_X_MIRROR;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg &= ~EXYNOS_CITRGFMT_FLIP_Y_MIRROR;
 		break;
-	default:
-		dev_err(ippdrv->dev, "invalid degree value %d.\n", degree);
-		return -EINVAL;
 	}
 
 	fimc_write(ctx, cfg, EXYNOS_CITRGFMT);
-	*swap = (cfg & EXYNOS_CITRGFMT_OUTROT90_CLOCKWISE) ? 1 : 0;
-
-	return 0;
 }
 
 static int fimc_set_prescaler(struct fimc_context *ctx, struct fimc_scaler *sc,
-		struct drm_exynos_pos *src, struct drm_exynos_pos *dst)
+			      struct drm_exynos_ipp_task_rect *src,
+			      struct drm_exynos_ipp_task_rect *dst)
 {
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
 	u32 cfg, cfg_ext, shfactor;
 	u32 pre_dst_width, pre_dst_height;
 	u32 hfactor, vfactor;
@@ -942,13 +762,13 @@ static int fimc_set_prescaler(struct fimc_context *ctx, struct fimc_scaler *sc,
 	/* fimc_ippdrv_check_property assures that dividers are not null */
 	hfactor = fls(src_w / dst_w / 2);
 	if (hfactor > FIMC_SHFACTOR / 2) {
-		dev_err(ippdrv->dev, "failed to get ratio horizontal.\n");
+		dev_err(ctx->dev, "failed to get ratio horizontal.\n");
 		return -EINVAL;
 	}
 
 	vfactor = fls(src_h / dst_h / 2);
 	if (vfactor > FIMC_SHFACTOR / 2) {
-		dev_err(ippdrv->dev, "failed to get ratio vertical.\n");
+		dev_err(ctx->dev, "failed to get ratio vertical.\n");
 		return -EINVAL;
 	}
 
@@ -1019,83 +839,77 @@ static void fimc_set_scaler(struct fimc_context *ctx, struct fimc_scaler *sc)
 	fimc_write(ctx, cfg_ext, EXYNOS_CIEXTEN);
 }
 
-static int fimc_dst_set_size(struct device *dev, int swap,
-		struct drm_exynos_pos *pos, struct drm_exynos_sz *sz)
+static void fimc_dst_set_size(struct fimc_context *ctx,
+			     struct exynos_drm_ipp_buffer *buf)
 {
-	struct fimc_context *ctx = get_fimc_context(dev);
-	struct drm_exynos_pos img_pos = *pos;
-	struct drm_exynos_sz img_sz = *sz;
-	u32 cfg;
+	u32 cfg, cfg_ext;
 
-	DRM_DEBUG_KMS("swap[%d]hsize[%d]vsize[%d]\n",
-		swap, sz->hsize, sz->vsize);
+	DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", buf->buf.width, buf->buf.height);
 
 	/* original size */
-	cfg = (EXYNOS_ORGOSIZE_HORIZONTAL(img_sz.hsize) |
-		EXYNOS_ORGOSIZE_VERTICAL(img_sz.vsize));
+	cfg = (EXYNOS_ORGOSIZE_HORIZONTAL(buf->buf.width) |
+		EXYNOS_ORGOSIZE_VERTICAL(buf->buf.height));
 
 	fimc_write(ctx, cfg, EXYNOS_ORGOSIZE);
 
-	DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]\n", pos->x, pos->y, pos->w, pos->h);
+	DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]\n", buf->rect.x, buf->rect.y,
+		buf->rect.w, buf->rect.h);
 
 	/* CSC ITU */
 	cfg = fimc_read(ctx, EXYNOS_CIGCTRL);
 	cfg &= ~EXYNOS_CIGCTRL_CSC_MASK;
 
-	if (sz->hsize >= FIMC_WIDTH_ITU_709)
+	if (buf->buf.width >= FIMC_WIDTH_ITU_709)
 		cfg |= EXYNOS_CIGCTRL_CSC_ITU709;
 	else
 		cfg |= EXYNOS_CIGCTRL_CSC_ITU601;
 
 	fimc_write(ctx, cfg, EXYNOS_CIGCTRL);
 
-	if (swap) {
-		img_pos.w = pos->h;
-		img_pos.h = pos->w;
-		img_sz.hsize = sz->vsize;
-		img_sz.vsize = sz->hsize;
-	}
+	cfg_ext = fimc_read(ctx, EXYNOS_CITRGFMT);
 
 	/* target image size */
 	cfg = fimc_read(ctx, EXYNOS_CITRGFMT);
 	cfg &= ~(EXYNOS_CITRGFMT_TARGETH_MASK |
 		EXYNOS_CITRGFMT_TARGETV_MASK);
-	cfg |= (EXYNOS_CITRGFMT_TARGETHSIZE(img_pos.w) |
-		EXYNOS_CITRGFMT_TARGETVSIZE(img_pos.h));
+	if (cfg_ext & EXYNOS_CITRGFMT_OUTROT90_CLOCKWISE)
+		cfg |= (EXYNOS_CITRGFMT_TARGETHSIZE(buf->rect.h) |
+			EXYNOS_CITRGFMT_TARGETVSIZE(buf->rect.w));
+	else
+		cfg |= (EXYNOS_CITRGFMT_TARGETHSIZE(buf->rect.w) |
+			EXYNOS_CITRGFMT_TARGETVSIZE(buf->rect.h));
 	fimc_write(ctx, cfg, EXYNOS_CITRGFMT);
 
 	/* target area */
-	cfg = EXYNOS_CITAREA_TARGET_AREA(img_pos.w * img_pos.h);
+	cfg = EXYNOS_CITAREA_TARGET_AREA(buf->rect.w * buf->rect.h);
 	fimc_write(ctx, cfg, EXYNOS_CITAREA);
 
 	/* offset Y(RGB), Cb, Cr */
-	cfg = (EXYNOS_CIOYOFF_HORIZONTAL(img_pos.x) |
-		EXYNOS_CIOYOFF_VERTICAL(img_pos.y));
+	cfg = (EXYNOS_CIOYOFF_HORIZONTAL(buf->rect.x) |
+		EXYNOS_CIOYOFF_VERTICAL(buf->rect.y));
 	fimc_write(ctx, cfg, EXYNOS_CIOYOFF);
-	cfg = (EXYNOS_CIOCBOFF_HORIZONTAL(img_pos.x) |
-		EXYNOS_CIOCBOFF_VERTICAL(img_pos.y));
+	cfg = (EXYNOS_CIOCBOFF_HORIZONTAL(buf->rect.x) |
+		EXYNOS_CIOCBOFF_VERTICAL(buf->rect.y));
 	fimc_write(ctx, cfg, EXYNOS_CIOCBOFF);
-	cfg = (EXYNOS_CIOCROFF_HORIZONTAL(img_pos.x) |
-		EXYNOS_CIOCROFF_VERTICAL(img_pos.y));
+	cfg = (EXYNOS_CIOCROFF_HORIZONTAL(buf->rect.x) |
+		EXYNOS_CIOCROFF_VERTICAL(buf->rect.y));
 	fimc_write(ctx, cfg, EXYNOS_CIOCROFF);
-
-	return 0;
 }
 
 static void fimc_dst_set_buf_seq(struct fimc_context *ctx, u32 buf_id,
-		enum drm_exynos_ipp_buf_type buf_type)
+		bool enqueue)
 {
 	unsigned long flags;
 	u32 buf_num;
 	u32 cfg;
 
-	DRM_DEBUG_KMS("buf_id[%d]buf_type[%d]\n", buf_id, buf_type);
+	DRM_DEBUG_KMS("buf_id[%d]enqueu[%d]\n", buf_id, enqueue);
 
 	spin_lock_irqsave(&ctx->lock, flags);
 
 	cfg = fimc_read(ctx, EXYNOS_CIFCNTSEQ);
 
-	if (buf_type == IPP_BUF_ENQUEUE)
+	if (enqueue)
 		cfg |= (1 << buf_id);
 	else
 		cfg &= ~(1 << buf_id);
@@ -1104,88 +918,29 @@ static void fimc_dst_set_buf_seq(struct fimc_context *ctx, u32 buf_id,
 
 	buf_num = hweight32(cfg);
 
-	if (buf_type == IPP_BUF_ENQUEUE && buf_num >= FIMC_BUF_START)
+	if (enqueue && buf_num >= FIMC_BUF_START)
 		fimc_mask_irq(ctx, true);
-	else if (buf_type == IPP_BUF_DEQUEUE && buf_num <= FIMC_BUF_STOP)
+	else if (!enqueue && buf_num <= FIMC_BUF_STOP)
 		fimc_mask_irq(ctx, false);
 
 	spin_unlock_irqrestore(&ctx->lock, flags);
 }
 
-static int fimc_dst_set_addr(struct device *dev,
-		struct drm_exynos_ipp_buf_info *buf_info, u32 buf_id,
-		enum drm_exynos_ipp_buf_type buf_type)
+static void fimc_dst_set_addr(struct fimc_context *ctx,
+			     struct exynos_drm_ipp_buffer *buf)
 {
-	struct fimc_context *ctx = get_fimc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node = ippdrv->c_node;
-	struct drm_exynos_ipp_property *property;
-	struct drm_exynos_ipp_config *config;
-
-	if (!c_node) {
-		DRM_ERROR("failed to get c_node.\n");
-		return -EINVAL;
-	}
-
-	property = &c_node->property;
-
-	DRM_DEBUG_KMS("prop_id[%d]buf_id[%d]buf_type[%d]\n",
-		property->prop_id, buf_id, buf_type);
+	fimc_write(ctx, buf->dma_addr[0], EXYNOS_CIOYSA(0));
+	fimc_write(ctx, buf->dma_addr[1], EXYNOS_CIOCBSA(0));
+	fimc_write(ctx, buf->dma_addr[2], EXYNOS_CIOCRSA(0));
 
-	if (buf_id > FIMC_MAX_DST) {
-		dev_info(ippdrv->dev, "invalid buf_id %d.\n", buf_id);
-		return -ENOMEM;
-	}
-
-	/* address register set */
-	switch (buf_type) {
-	case IPP_BUF_ENQUEUE:
-		config = &property->config[EXYNOS_DRM_OPS_DST];
-
-		fimc_write(ctx, buf_info->base[EXYNOS_DRM_PLANAR_Y],
-			EXYNOS_CIOYSA(buf_id));
-
-		if (config->fmt == DRM_FORMAT_YVU420) {
-			fimc_write(ctx, buf_info->base[EXYNOS_DRM_PLANAR_CR],
-				EXYNOS_CIOCBSA(buf_id));
-			fimc_write(ctx, buf_info->base[EXYNOS_DRM_PLANAR_CB],
-				EXYNOS_CIOCRSA(buf_id));
-		} else {
-			fimc_write(ctx, buf_info->base[EXYNOS_DRM_PLANAR_CB],
-				EXYNOS_CIOCBSA(buf_id));
-			fimc_write(ctx, buf_info->base[EXYNOS_DRM_PLANAR_CR],
-				EXYNOS_CIOCRSA(buf_id));
-		}
-		break;
-	case IPP_BUF_DEQUEUE:
-		fimc_write(ctx, 0x0, EXYNOS_CIOYSA(buf_id));
-		fimc_write(ctx, 0x0, EXYNOS_CIOCBSA(buf_id));
-		fimc_write(ctx, 0x0, EXYNOS_CIOCRSA(buf_id));
-		break;
-	default:
-		/* bypass */
-		break;
-	}
-
-	fimc_dst_set_buf_seq(ctx, buf_id, buf_type);
-
-	return 0;
+	fimc_dst_set_buf_seq(ctx, 0, true);
 }
 
-static struct exynos_drm_ipp_ops fimc_dst_ops = {
-	.set_fmt = fimc_dst_set_fmt,
-	.set_transf = fimc_dst_set_transf,
-	.set_size = fimc_dst_set_size,
-	.set_addr = fimc_dst_set_addr,
-};
+static void fimc_stop(struct fimc_context *ctx);
 
 static irqreturn_t fimc_irq_handler(int irq, void *dev_id)
 {
 	struct fimc_context *ctx = dev_id;
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node = ippdrv->c_node;
-	struct drm_exynos_ipp_event_work *event_work =
-		c_node->event_work;
 	int buf_id;
 
 	DRM_DEBUG_KMS("fimc id[%d]\n", ctx->id);
@@ -1203,170 +958,19 @@ static irqreturn_t fimc_irq_handler(int irq, void *dev_id)
 
 	DRM_DEBUG_KMS("buf_id[%d]\n", buf_id);
 
-	fimc_dst_set_buf_seq(ctx, buf_id, IPP_BUF_DEQUEUE);
-
-	event_work->ippdrv = ippdrv;
-	event_work->buf_id[EXYNOS_DRM_OPS_DST] = buf_id;
-	queue_work(ippdrv->event_workq, &event_work->work);
-
-	return IRQ_HANDLED;
-}
-
-static int fimc_init_prop_list(struct exynos_drm_ippdrv *ippdrv)
-{
-	struct drm_exynos_ipp_prop_list *prop_list = &ippdrv->prop_list;
-
-	prop_list->version = 1;
-	prop_list->writeback = 1;
-	prop_list->refresh_min = FIMC_REFRESH_MIN;
-	prop_list->refresh_max = FIMC_REFRESH_MAX;
-	prop_list->flip = (1 << EXYNOS_DRM_FLIP_NONE) |
-				(1 << EXYNOS_DRM_FLIP_VERTICAL) |
-				(1 << EXYNOS_DRM_FLIP_HORIZONTAL);
-	prop_list->degree = (1 << EXYNOS_DRM_DEGREE_0) |
-				(1 << EXYNOS_DRM_DEGREE_90) |
-				(1 << EXYNOS_DRM_DEGREE_180) |
-				(1 << EXYNOS_DRM_DEGREE_270);
-	prop_list->csc = 1;
-	prop_list->crop = 1;
-	prop_list->crop_max.hsize = FIMC_CROP_MAX;
-	prop_list->crop_max.vsize = FIMC_CROP_MAX;
-	prop_list->crop_min.hsize = FIMC_CROP_MIN;
-	prop_list->crop_min.vsize = FIMC_CROP_MIN;
-	prop_list->scale = 1;
-	prop_list->scale_max.hsize = FIMC_SCALE_MAX;
-	prop_list->scale_max.vsize = FIMC_SCALE_MAX;
-	prop_list->scale_min.hsize = FIMC_SCALE_MIN;
-	prop_list->scale_min.vsize = FIMC_SCALE_MIN;
-
-	return 0;
-}
-
-static inline bool fimc_check_drm_flip(enum drm_exynos_flip flip)
-{
-	switch (flip) {
-	case EXYNOS_DRM_FLIP_NONE:
-	case EXYNOS_DRM_FLIP_VERTICAL:
-	case EXYNOS_DRM_FLIP_HORIZONTAL:
-	case EXYNOS_DRM_FLIP_BOTH:
-		return true;
-	default:
-		DRM_DEBUG_KMS("invalid flip\n");
-		return false;
-	}
-}
-
-static int fimc_ippdrv_check_property(struct device *dev,
-		struct drm_exynos_ipp_property *property)
-{
-	struct fimc_context *ctx = get_fimc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	struct drm_exynos_ipp_prop_list *pp = &ippdrv->prop_list;
-	struct drm_exynos_ipp_config *config;
-	struct drm_exynos_pos *pos;
-	struct drm_exynos_sz *sz;
-	bool swap;
-	int i;
-
-	for_each_ipp_ops(i) {
-		if ((i == EXYNOS_DRM_OPS_SRC) &&
-			(property->cmd == IPP_CMD_WB))
-			continue;
-
-		config = &property->config[i];
-		pos = &config->pos;
-		sz = &config->sz;
-
-		/* check for flip */
-		if (!fimc_check_drm_flip(config->flip)) {
-			DRM_ERROR("invalid flip.\n");
-			goto err_property;
-		}
-
-		/* check for degree */
-		switch (config->degree) {
-		case EXYNOS_DRM_DEGREE_90:
-		case EXYNOS_DRM_DEGREE_270:
-			swap = true;
-			break;
-		case EXYNOS_DRM_DEGREE_0:
-		case EXYNOS_DRM_DEGREE_180:
-			swap = false;
-			break;
-		default:
-			DRM_ERROR("invalid degree.\n");
-			goto err_property;
-		}
-
-		/* check for buffer bound */
-		if ((pos->x + pos->w > sz->hsize) ||
-			(pos->y + pos->h > sz->vsize)) {
-			DRM_ERROR("out of buf bound.\n");
-			goto err_property;
-		}
+	if (ctx->task) {
+		struct exynos_drm_ipp_task *task = ctx->task;
 
-		/* check for crop */
-		if ((i == EXYNOS_DRM_OPS_SRC) && (pp->crop)) {
-			if (swap) {
-				if ((pos->h < pp->crop_min.hsize) ||
-					(sz->vsize > pp->crop_max.hsize) ||
-					(pos->w < pp->crop_min.vsize) ||
-					(sz->hsize > pp->crop_max.vsize)) {
-					DRM_ERROR("out of crop size.\n");
-					goto err_property;
-				}
-			} else {
-				if ((pos->w < pp->crop_min.hsize) ||
-					(sz->hsize > pp->crop_max.hsize) ||
-					(pos->h < pp->crop_min.vsize) ||
-					(sz->vsize > pp->crop_max.vsize)) {
-					DRM_ERROR("out of crop size.\n");
-					goto err_property;
-				}
-			}
-		}
-
-		/* check for scale */
-		if ((i == EXYNOS_DRM_OPS_DST) && (pp->scale)) {
-			if (swap) {
-				if ((pos->h < pp->scale_min.hsize) ||
-					(sz->vsize > pp->scale_max.hsize) ||
-					(pos->w < pp->scale_min.vsize) ||
-					(sz->hsize > pp->scale_max.vsize)) {
-					DRM_ERROR("out of scale size.\n");
-					goto err_property;
-				}
-			} else {
-				if ((pos->w < pp->scale_min.hsize) ||
-					(sz->hsize > pp->scale_max.hsize) ||
-					(pos->h < pp->scale_min.vsize) ||
-					(sz->vsize > pp->scale_max.vsize)) {
-					DRM_ERROR("out of scale size.\n");
-					goto err_property;
-				}
-			}
-		}
+		ctx->task = NULL;
+		pm_runtime_mark_last_busy(ctx->dev);
+		pm_runtime_put_autosuspend(ctx->dev);
+		exynos_drm_ipp_task_done(task, 0);
 	}
 
-	return 0;
+	fimc_dst_set_buf_seq(ctx, buf_id, false);
+	fimc_stop(ctx);
 
-err_property:
-	for_each_ipp_ops(i) {
-		if ((i == EXYNOS_DRM_OPS_SRC) &&
-			(property->cmd == IPP_CMD_WB))
-			continue;
-
-		config = &property->config[i];
-		pos = &config->pos;
-		sz = &config->sz;
-
-		DRM_ERROR("[%s]f[%d]r[%d]pos[%d %d %d %d]sz[%d %d]\n",
-			i ? "dst" : "src", config->flip, config->degree,
-			pos->x, pos->y, pos->w, pos->h,
-			sz->hsize, sz->vsize);
-	}
-
-	return -EINVAL;
+	return IRQ_HANDLED;
 }
 
 static void fimc_clear_addr(struct fimc_context *ctx)
@@ -1386,10 +990,8 @@ static void fimc_clear_addr(struct fimc_context *ctx)
 	}
 }
 
-static int fimc_ippdrv_reset(struct device *dev)
+static void fimc_reset(struct fimc_context *ctx)
 {
-	struct fimc_context *ctx = get_fimc_context(dev);
-
 	/* reset h/w block */
 	fimc_sw_reset(ctx);
 
@@ -1397,82 +999,26 @@ static int fimc_ippdrv_reset(struct device *dev)
 	memset(&ctx->sc, 0x0, sizeof(ctx->sc));
 
 	fimc_clear_addr(ctx);
-
-	return 0;
 }
 
-static int fimc_ippdrv_start(struct device *dev, enum drm_exynos_ipp_cmd cmd)
+static void fimc_start(struct fimc_context *ctx)
 {
-	struct fimc_context *ctx = get_fimc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node = ippdrv->c_node;
-	struct drm_exynos_ipp_property *property;
-	struct drm_exynos_ipp_config *config;
-	struct drm_exynos_pos	img_pos[EXYNOS_DRM_OPS_MAX];
-	struct drm_exynos_ipp_set_wb set_wb;
-	int ret, i;
 	u32 cfg0, cfg1;
 
-	DRM_DEBUG_KMS("cmd[%d]\n", cmd);
-
-	if (!c_node) {
-		DRM_ERROR("failed to get c_node.\n");
-		return -EINVAL;
-	}
-
-	property = &c_node->property;
-
 	fimc_mask_irq(ctx, true);
 
-	for_each_ipp_ops(i) {
-		config = &property->config[i];
-		img_pos[i] = config->pos;
-	}
-
-	ret = fimc_set_prescaler(ctx, &ctx->sc,
-		&img_pos[EXYNOS_DRM_OPS_SRC],
-		&img_pos[EXYNOS_DRM_OPS_DST]);
-	if (ret) {
-		dev_err(dev, "failed to set prescaler.\n");
-		return ret;
-	}
-
-	/* If set ture, we can save jpeg about screen */
+	/* If set true, we can save jpeg about screen */
 	fimc_handle_jpeg(ctx, false);
 	fimc_set_scaler(ctx, &ctx->sc);
 
-	switch (cmd) {
-	case IPP_CMD_M2M:
-		fimc_set_type_ctrl(ctx, FIMC_WB_NONE);
-		fimc_handle_lastend(ctx, false);
-
-		/* setup dma */
-		cfg0 = fimc_read(ctx, EXYNOS_MSCTRL);
-		cfg0 &= ~EXYNOS_MSCTRL_INPUT_MASK;
-		cfg0 |= EXYNOS_MSCTRL_INPUT_MEMORY;
-		fimc_write(ctx, cfg0, EXYNOS_MSCTRL);
-		break;
-	case IPP_CMD_WB:
-		fimc_set_type_ctrl(ctx, FIMC_WB_A);
-		fimc_handle_lastend(ctx, true);
-
-		/* setup FIMD */
-		ret = fimc_set_camblk_fimd0_wb(ctx);
-		if (ret < 0) {
-			dev_err(dev, "camblk setup failed.\n");
-			return ret;
-		}
+	fimc_set_type_ctrl(ctx);
+	fimc_handle_lastend(ctx, false);
 
-		set_wb.enable = 1;
-		set_wb.refresh = property->refresh_rate;
-		exynos_drm_ippnb_send_event(IPP_SET_WRITEBACK, (void *)&set_wb);
-		break;
-	case IPP_CMD_OUTPUT:
-	default:
-		ret = -EINVAL;
-		dev_err(dev, "invalid operations.\n");
-		return ret;
-	}
+	/* setup dma */
+	cfg0 = fimc_read(ctx, EXYNOS_MSCTRL);
+	cfg0 &= ~EXYNOS_MSCTRL_INPUT_MASK;
+	cfg0 |= EXYNOS_MSCTRL_INPUT_MEMORY;
+	fimc_write(ctx, cfg0, EXYNOS_MSCTRL);
 
 	/* Reset status */
 	fimc_write(ctx, 0x0, EXYNOS_CISTATUS);
@@ -1498,36 +1044,18 @@ static int fimc_ippdrv_start(struct device *dev, enum drm_exynos_ipp_cmd cmd)
 
 	fimc_clear_bits(ctx, EXYNOS_CIOCTRL, EXYNOS_CIOCTRL_WEAVE_MASK);
 
-	if (cmd == IPP_CMD_M2M)
-		fimc_set_bits(ctx, EXYNOS_MSCTRL, EXYNOS_MSCTRL_ENVID);
-
-	return 0;
+	fimc_set_bits(ctx, EXYNOS_MSCTRL, EXYNOS_MSCTRL_ENVID);
 }
 
-static void fimc_ippdrv_stop(struct device *dev, enum drm_exynos_ipp_cmd cmd)
+static void fimc_stop(struct fimc_context *ctx)
 {
-	struct fimc_context *ctx = get_fimc_context(dev);
-	struct drm_exynos_ipp_set_wb set_wb = {0, 0};
 	u32 cfg;
 
-	DRM_DEBUG_KMS("cmd[%d]\n", cmd);
-
-	switch (cmd) {
-	case IPP_CMD_M2M:
-		/* Source clear */
-		cfg = fimc_read(ctx, EXYNOS_MSCTRL);
-		cfg &= ~EXYNOS_MSCTRL_INPUT_MASK;
-		cfg &= ~EXYNOS_MSCTRL_ENVID;
-		fimc_write(ctx, cfg, EXYNOS_MSCTRL);
-		break;
-	case IPP_CMD_WB:
-		exynos_drm_ippnb_send_event(IPP_SET_WRITEBACK, (void *)&set_wb);
-		break;
-	case IPP_CMD_OUTPUT:
-	default:
-		dev_err(dev, "invalid operations.\n");
-		break;
-	}
+	/* Source clear */
+	cfg = fimc_read(ctx, EXYNOS_MSCTRL);
+	cfg &= ~EXYNOS_MSCTRL_INPUT_MASK;
+	cfg &= ~EXYNOS_MSCTRL_ENVID;
+	fimc_write(ctx, cfg, EXYNOS_MSCTRL);
 
 	fimc_mask_irq(ctx, false);
 
@@ -1545,6 +1073,87 @@ static void fimc_ippdrv_stop(struct device *dev, enum drm_exynos_ipp_cmd cmd)
 	fimc_set_bits(ctx, EXYNOS_CIGCTRL, EXYNOS_CIGCTRL_IRQ_END_DISABLE);
 }
 
+static int fimc_commit(struct exynos_drm_ipp *ipp,
+			  struct exynos_drm_ipp_task *task)
+{
+	struct fimc_context *ctx =
+			container_of(ipp, struct fimc_context, ipp);
+
+	pm_runtime_get_sync(ctx->dev);
+	ctx->task = task;
+
+	fimc_src_set_fmt(ctx, task->src.buf.fourcc, task->src.buf.modifier);
+	fimc_src_set_size(ctx, &task->src);
+	fimc_src_set_transf(ctx, DRM_MODE_ROTATE_0);
+	fimc_src_set_addr(ctx, &task->src);
+	fimc_dst_set_fmt(ctx, task->dst.buf.fourcc, task->dst.buf.modifier);
+	fimc_dst_set_transf(ctx, task->transform.rotation);
+	fimc_dst_set_size(ctx, &task->dst);
+	fimc_dst_set_addr(ctx, &task->dst);
+	fimc_set_prescaler(ctx, &ctx->sc, &task->src.rect, &task->dst.rect);
+	fimc_start(ctx);
+
+	return 0;
+}
+
+static void fimc_abort(struct exynos_drm_ipp *ipp,
+			  struct exynos_drm_ipp_task *task)
+{
+	struct fimc_context *ctx =
+			container_of(ipp, struct fimc_context, ipp);
+
+	fimc_reset(ctx);
+
+	if (ctx->task) {
+		struct exynos_drm_ipp_task *task = ctx->task;
+
+		ctx->task = NULL;
+		pm_runtime_mark_last_busy(ctx->dev);
+		pm_runtime_put_autosuspend(ctx->dev);
+		exynos_drm_ipp_task_done(task, -EIO);
+	}
+}
+
+static struct exynos_drm_ipp_funcs ipp_funcs = {
+	.commit = fimc_commit,
+	.abort = fimc_abort,
+};
+
+static int fimc_bind(struct device *dev, struct device *master, void *data)
+{
+	struct fimc_context *ctx = dev_get_drvdata(dev);
+	struct drm_device *drm_dev = data;
+	struct exynos_drm_ipp *ipp = &ctx->ipp;
+
+	ctx->drm_dev = drm_dev;
+	drm_iommu_attach_device(drm_dev, dev);
+
+	exynos_drm_ipp_register(drm_dev, ipp, &ipp_funcs,
+			DRM_EXYNOS_IPP_CAP_CROP | DRM_EXYNOS_IPP_CAP_ROTATE |
+			DRM_EXYNOS_IPP_CAP_SCALE | DRM_EXYNOS_IPP_CAP_CONVERT,
+			ctx->formats, ctx->num_formats, "fimc");
+
+	dev_info(dev, "The exynos fimc has been probed successfully\n");
+
+	return 0;
+}
+
+static void fimc_unbind(struct device *dev, struct device *master,
+			void *data)
+{
+	struct fimc_context *ctx = dev_get_drvdata(dev);
+	struct drm_device *drm_dev = data;
+	struct exynos_drm_ipp *ipp = &ctx->ipp;
+
+	exynos_drm_ipp_unregister(drm_dev, ipp);
+	drm_iommu_detach_device(drm_dev, dev);
+}
+
+static const struct component_ops fimc_component_ops = {
+	.bind	= fimc_bind,
+	.unbind = fimc_unbind,
+};
+
 static void fimc_put_clocks(struct fimc_context *ctx)
 {
 	int i;
@@ -1559,7 +1168,7 @@ static void fimc_put_clocks(struct fimc_context *ctx)
 
 static int fimc_setup_clocks(struct fimc_context *ctx)
 {
-	struct device *fimc_dev = ctx->ippdrv.dev;
+	struct device *fimc_dev = ctx->dev;
 	struct device *dev;
 	int ret, i;
 
@@ -1574,8 +1183,6 @@ static int fimc_setup_clocks(struct fimc_context *ctx)
 
 		ctx->clocks[i] = clk_get(dev, fimc_clock_names[i]);
 		if (IS_ERR(ctx->clocks[i])) {
-			if (i >= FIMC_CLK_MUX)
-				break;
 			ret = PTR_ERR(ctx->clocks[i]);
 			dev_err(fimc_dev, "failed to get clock: %s\n",
 						fimc_clock_names[i]);
@@ -1583,20 +1190,6 @@ static int fimc_setup_clocks(struct fimc_context *ctx)
 		}
 	}
 
-	/* Optional FIMC LCLK parent clock setting */
-	if (!IS_ERR(ctx->clocks[FIMC_CLK_PARENT])) {
-		ret = clk_set_parent(ctx->clocks[FIMC_CLK_MUX],
-				     ctx->clocks[FIMC_CLK_PARENT]);
-		if (ret < 0) {
-			dev_err(fimc_dev, "failed to set parent.\n");
-			goto e_clk_free;
-		}
-	}
-
-	ret = clk_set_rate(ctx->clocks[FIMC_CLK_LCLK], ctx->clk_frequency);
-	if (ret < 0)
-		goto e_clk_free;
-
 	ret = clk_prepare_enable(ctx->clocks[FIMC_CLK_LCLK]);
 	if (!ret)
 		return ret;
@@ -1605,57 +1198,118 @@ static int fimc_setup_clocks(struct fimc_context *ctx)
 	return ret;
 }
 
-static int fimc_parse_dt(struct fimc_context *ctx)
+int exynos_drm_check_fimc_device(struct device *dev)
 {
-	struct device_node *node = ctx->ippdrv.dev->of_node;
+	int id = of_alias_get_id(dev->of_node, "fimc");
 
-	/* Handle only devices that support the LCD Writeback data path */
-	if (!of_property_read_bool(node, "samsung,lcd-wb"))
-		return -ENODEV;
+	if (id >= 0 && (BIT(id) & fimc_mask))
+		return 0;
+	return -ENODEV;
+}
 
-	if (of_property_read_u32(node, "clock-frequency",
-					&ctx->clk_frequency))
-		ctx->clk_frequency = FIMC_DEFAULT_LCLK_FREQUENCY;
+static const unsigned int fimc_formats[] = {
+	DRM_FORMAT_XRGB8888, DRM_FORMAT_RGB565,
+	DRM_FORMAT_NV12, DRM_FORMAT_NV16, DRM_FORMAT_NV21, DRM_FORMAT_NV61,
+	DRM_FORMAT_UYVY, DRM_FORMAT_VYUY, DRM_FORMAT_YUYV, DRM_FORMAT_YVYU,
+	DRM_FORMAT_YUV420, DRM_FORMAT_YVU420, DRM_FORMAT_YUV422,
+	DRM_FORMAT_YUV444,
+};
 
-	ctx->id = of_alias_get_id(node, "fimc");
+static const unsigned int fimc_tiled_formats[] = {
+	DRM_FORMAT_NV12, DRM_FORMAT_NV21,
+};
 
-	if (ctx->id < 0) {
-		dev_err(ctx->ippdrv.dev, "failed to get node alias id.\n");
-		return -EINVAL;
-	}
+static const struct drm_exynos_ipp_limit fimc_4210_limits_v1[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 16, 8192, 8 }, .v = { 16, 8192, 2 }) },
+	{ IPP_SIZE_LIMIT(AREA, .h = { 16, 4224, 2 }, .v = { 16, 0, 2 }) },
+	{ IPP_SIZE_LIMIT(ROTATED, .h = { 128, 1920 }, .v = { 128, 0 }) },
+	{ IPP_SCALE_LIMIT(.h = { (1 << 16) / 64, (1 << 16) * 64 },
+			  .v = { (1 << 16) / 64, (1 << 16) * 64 }) },
+};
 
-	return 0;
-}
+static const struct drm_exynos_ipp_limit fimc_4210_limits_v2[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 16, 8192, 8 }, .v = { 16, 8192, 2 }) },
+	{ IPP_SIZE_LIMIT(AREA, .h = { 16, 1920, 2 }, .v = { 16, 0, 2 }) },
+	{ IPP_SIZE_LIMIT(ROTATED, .h = { 128, 1366 }, .v = { 128, 0 }) },
+	{ IPP_SCALE_LIMIT(.h = { (1 << 16) / 64, (1 << 16) * 64 },
+			  .v = { (1 << 16) / 64, (1 << 16) * 64 }) },
+};
+
+static const struct drm_exynos_ipp_limit fimc_4210_limits_tiled_v1[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 128, 1920, 128 }, .v = { 32, 1920, 32 }) },
+	{ IPP_SIZE_LIMIT(AREA, .h = { 128, 1920, 2 }, .v = { 128, 0, 2 }) },
+	{ IPP_SCALE_LIMIT(.h = { (1 << 16) / 64, (1 << 16) * 64 },
+			  .v = { (1 << 16) / 64, (1 << 16) * 64 }) },
+};
+
+static const struct drm_exynos_ipp_limit fimc_4210_limits_tiled_v2[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 128, 1920, 128 }, .v = { 32, 1920, 32 }) },
+	{ IPP_SIZE_LIMIT(AREA, .h = { 128, 1366, 2 }, .v = { 128, 0, 2 }) },
+	{ IPP_SCALE_LIMIT(.h = { (1 << 16) / 64, (1 << 16) * 64 },
+			  .v = { (1 << 16) / 64, (1 << 16) * 64 }) },
+};
 
 static int fimc_probe(struct platform_device *pdev)
 {
+	const struct drm_exynos_ipp_limit *limits;
+	struct exynos_drm_ipp_formats *formats;
 	struct device *dev = &pdev->dev;
 	struct fimc_context *ctx;
 	struct resource *res;
-	struct exynos_drm_ippdrv *ippdrv;
 	int ret;
+	int i, j, num_limits, num_formats;
 
-	if (!dev->of_node) {
-		dev_err(dev, "device tree node not found.\n");
+	if (exynos_drm_check_fimc_device(dev) != 0)
 		return -ENODEV;
-	}
 
 	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
 		return -ENOMEM;
 
-	ctx->ippdrv.dev = dev;
+	ctx->dev = dev;
+	ctx->id = of_alias_get_id(dev->of_node, "fimc");
 
-	ret = fimc_parse_dt(ctx);
-	if (ret < 0)
-		return ret;
+	/* construct formats/limits array */
+	num_formats = ARRAY_SIZE(fimc_formats) + ARRAY_SIZE(fimc_tiled_formats);
+	formats = devm_kzalloc(dev, sizeof(*formats) * num_formats, GFP_KERNEL);
+	if (!formats)
+		return -ENOMEM;
+
+	/* linear formats */
+	if (ctx->id < 3) {
+		limits = fimc_4210_limits_v1;
+		num_limits = ARRAY_SIZE(fimc_4210_limits_v1);
+	} else {
+		limits = fimc_4210_limits_v2;
+		num_limits = ARRAY_SIZE(fimc_4210_limits_v2);
+	}
+	for (i = 0; i < ARRAY_SIZE(fimc_formats); i++) {
+		formats[i].fourcc = fimc_formats[i];
+		formats[i].type = DRM_EXYNOS_IPP_FORMAT_SOURCE |
+				  DRM_EXYNOS_IPP_FORMAT_DESTINATION;
+		formats[i].limits = limits;
+		formats[i].num_limits = num_limits;
+	}
 
-	ctx->sysreg = syscon_regmap_lookup_by_phandle(dev->of_node,
-						"samsung,sysreg");
-	if (IS_ERR(ctx->sysreg)) {
-		dev_err(dev, "syscon regmap lookup failed.\n");
-		return PTR_ERR(ctx->sysreg);
+	/* tiled formats */
+	if (ctx->id < 3) {
+		limits = fimc_4210_limits_tiled_v1;
+		num_limits = ARRAY_SIZE(fimc_4210_limits_tiled_v1);
+	} else {
+		limits = fimc_4210_limits_tiled_v2;
+		num_limits = ARRAY_SIZE(fimc_4210_limits_tiled_v2);
 	}
+	for (j = i, i = 0; i < ARRAY_SIZE(fimc_tiled_formats); j++, i++) {
+		formats[j].fourcc = fimc_tiled_formats[i];
+		formats[j].modifier = DRM_FORMAT_MOD_SAMSUNG_64_32_TILE;
+		formats[j].type = DRM_EXYNOS_IPP_FORMAT_SOURCE |
+				  DRM_EXYNOS_IPP_FORMAT_DESTINATION;
+		formats[j].limits = limits;
+		formats[j].num_limits = num_limits;
+	}
+
+	ctx->formats = formats;
+	ctx->num_formats = num_formats;
 
 	/* resource memory */
 	ctx->regs_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -1670,9 +1324,8 @@ static int fimc_probe(struct platform_device *pdev)
 		return -ENOENT;
 	}
 
-	ctx->irq = res->start;
-	ret = devm_request_threaded_irq(dev, ctx->irq, NULL, fimc_irq_handler,
-		IRQF_ONESHOT, "drm_fimc", ctx);
+	ret = devm_request_irq(dev, res->start, fimc_irq_handler,
+		0, dev_name(dev), ctx);
 	if (ret < 0) {
 		dev_err(dev, "failed to request irq.\n");
 		return ret;
@@ -1682,39 +1335,24 @@ static int fimc_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
-	ippdrv = &ctx->ippdrv;
-	ippdrv->ops[EXYNOS_DRM_OPS_SRC] = &fimc_src_ops;
-	ippdrv->ops[EXYNOS_DRM_OPS_DST] = &fimc_dst_ops;
-	ippdrv->check_property = fimc_ippdrv_check_property;
-	ippdrv->reset = fimc_ippdrv_reset;
-	ippdrv->start = fimc_ippdrv_start;
-	ippdrv->stop = fimc_ippdrv_stop;
-	ret = fimc_init_prop_list(ippdrv);
-	if (ret < 0) {
-		dev_err(dev, "failed to init property list.\n");
-		goto err_put_clk;
-	}
-
-	DRM_DEBUG_KMS("id[%d]ippdrv[%pK]\n", ctx->id, ippdrv);
-
 	spin_lock_init(&ctx->lock);
 	platform_set_drvdata(pdev, ctx);
 
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_autosuspend_delay(dev, FIMC_AUTOSUSPEND_DELAY);
 	pm_runtime_enable(dev);
 
-	ret = exynos_drm_ippdrv_register(ippdrv);
-	if (ret < 0) {
-		dev_err(dev, "failed to register drm fimc device.\n");
+	ret = component_add(dev, &fimc_component_ops);
+	if (ret)
 		goto err_pm_dis;
-	}
 
 	dev_info(dev, "drm fimc registered successfully.\n");
 
 	return 0;
 
 err_pm_dis:
+	pm_runtime_dont_use_autosuspend(dev);
 	pm_runtime_disable(dev);
-err_put_clk:
 	fimc_put_clocks(ctx);
 
 	return ret;
@@ -1724,42 +1362,24 @@ static int fimc_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct fimc_context *ctx = get_fimc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
 
-	exynos_drm_ippdrv_unregister(ippdrv);
+	component_del(dev, &fimc_component_ops);
+	pm_runtime_dont_use_autosuspend(dev);
+	pm_runtime_disable(dev);
 
 	fimc_put_clocks(ctx);
-	pm_runtime_set_suspended(dev);
-	pm_runtime_disable(dev);
 
 	return 0;
 }
 
 #ifdef CONFIG_PM
-static int fimc_clk_ctrl(struct fimc_context *ctx, bool enable)
-{
-	DRM_DEBUG_KMS("enable[%d]\n", enable);
-
-	if (enable) {
-		clk_prepare_enable(ctx->clocks[FIMC_CLK_GATE]);
-		clk_prepare_enable(ctx->clocks[FIMC_CLK_WB_A]);
-		ctx->suspended = false;
-	} else {
-		clk_disable_unprepare(ctx->clocks[FIMC_CLK_GATE]);
-		clk_disable_unprepare(ctx->clocks[FIMC_CLK_WB_A]);
-		ctx->suspended = true;
-	}
-
-	return 0;
-}
-
 static int fimc_runtime_suspend(struct device *dev)
 {
 	struct fimc_context *ctx = get_fimc_context(dev);
 
 	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
-
-	return  fimc_clk_ctrl(ctx, false);
+	clk_disable_unprepare(ctx->clocks[FIMC_CLK_GATE]);
+	return 0;
 }
 
 static int fimc_runtime_resume(struct device *dev)
@@ -1767,8 +1387,7 @@ static int fimc_runtime_resume(struct device *dev)
 	struct fimc_context *ctx = get_fimc_context(dev);
 
 	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
-
-	return  fimc_clk_ctrl(ctx, true);
+	return clk_prepare_enable(ctx->clocks[FIMC_CLK_GATE]);
 }
 #endif
 
@@ -1795,4 +1414,3 @@ struct platform_driver fimc_driver = {
 		.pm	= &fimc_pm_ops,
 	},
 };
-
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimc.h b/drivers/gpu/drm/exynos/exynos_drm_fimc.h
deleted file mode 100644
index 127a424c5fdf..000000000000
--- a/drivers/gpu/drm/exynos/exynos_drm_fimc.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright (c) 2012 Samsung Electronics Co., Ltd.
- *
- * Authors:
- *	Eunchul Kim <chulspro.kim@samsung.com>
- *	Jinyoung Jeon <jy0.jeon@samsung.com>
- *	Sangmin Lee <lsmin.lee@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-#ifndef _EXYNOS_DRM_FIMC_H_
-#define _EXYNOS_DRM_FIMC_H_
-
-/*
- * TODO
- * FIMD output interface notifier callback.
- */
-
-#endif /* _EXYNOS_DRM_FIMC_H_ */
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
index d42ae2bc3e56..01b1570d0c3a 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
@@ -121,6 +121,12 @@ static struct fimd_driver_data s3c64xx_fimd_driver_data = {
 	.has_limited_fmt = 1,
 };
 
+static struct fimd_driver_data s5pv210_fimd_driver_data = {
+	.timing_base = 0x0,
+	.has_shadowcon = 1,
+	.has_clksel = 1,
+};
+
 static struct fimd_driver_data exynos3_fimd_driver_data = {
 	.timing_base = 0x20000,
 	.lcdblk_offset = 0x210,
@@ -193,6 +199,8 @@ struct fimd_context {
 static const struct of_device_id fimd_driver_dt_match[] = {
 	{ .compatible = "samsung,s3c6400-fimd",
 	  .data = &s3c64xx_fimd_driver_data },
+	{ .compatible = "samsung,s5pv210-fimd",
+	  .data = &s5pv210_fimd_driver_data },
 	{ .compatible = "samsung,exynos3250-fimd",
 	  .data = &exynos3_fimd_driver_data },
 	{ .compatible = "samsung,exynos4210-fimd",
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c
index 11cc01b47bc0..6e1494fa71b4 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c
@@ -431,37 +431,24 @@ int exynos_drm_gem_dumb_create(struct drm_file *file_priv,
 	return 0;
 }
 
-int exynos_drm_gem_fault(struct vm_fault *vmf)
+vm_fault_t exynos_drm_gem_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct drm_gem_object *obj = vma->vm_private_data;
 	struct exynos_drm_gem *exynos_gem = to_exynos_gem(obj);
 	unsigned long pfn;
 	pgoff_t page_offset;
-	int ret;
 
 	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 
 	if (page_offset >= (exynos_gem->size >> PAGE_SHIFT)) {
 		DRM_ERROR("invalid page offset\n");
-		ret = -EINVAL;
-		goto out;
+		return VM_FAULT_SIGBUS;
 	}
 
 	pfn = page_to_pfn(exynos_gem->pages[page_offset]);
-	ret = vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
-
-out:
-	switch (ret) {
-	case 0:
-	case -ERESTARTSYS:
-	case -EINTR:
-		return VM_FAULT_NOPAGE;
-	case -ENOMEM:
-		return VM_FAULT_OOM;
-	default:
-		return VM_FAULT_SIGBUS;
-	}
+	return vmf_insert_mixed(vma, vmf->address,
+			__pfn_to_pfn_t(pfn, PFN_DEV));
 }
 
 static int exynos_drm_gem_mmap_obj(struct drm_gem_object *obj,
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.h b/drivers/gpu/drm/exynos/exynos_drm_gem.h
index 5a4c7de80f65..9057d7f1d6ed 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.h
@@ -13,6 +13,7 @@
 #define _EXYNOS_DRM_GEM_H_
 
 #include <drm/drm_gem.h>
+#include <linux/mm_types.h>
 
 #define to_exynos_gem(x)	container_of(x, struct exynos_drm_gem, base)
 
@@ -111,7 +112,7 @@ int exynos_drm_gem_dumb_create(struct drm_file *file_priv,
 			       struct drm_mode_create_dumb *args);
 
 /* page fault handler and mmap fault address(virtual) to physical memory. */
-int exynos_drm_gem_fault(struct vm_fault *vmf);
+vm_fault_t exynos_drm_gem_fault(struct vm_fault *vmf);
 
 /* set vm_flags and we can change the vm attribute to other one at here. */
 int exynos_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gsc.c b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
index 0506b2b17ac1..e99dd1e4ba65 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gsc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
@@ -12,18 +12,20 @@
  *
  */
 #include <linux/kernel.h>
+#include <linux/component.h>
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/pm_runtime.h>
 #include <linux/mfd/syscon.h>
+#include <linux/of_device.h>
 #include <linux/regmap.h>
 
 #include <drm/drmP.h>
 #include <drm/exynos_drm.h>
 #include "regs-gsc.h"
 #include "exynos_drm_drv.h"
+#include "exynos_drm_iommu.h"
 #include "exynos_drm_ipp.h"
-#include "exynos_drm_gsc.h"
 
 /*
  * GSC stands for General SCaler and
@@ -31,26 +33,10 @@
  * input DMA reads image data from the memory.
  * output DMA writes image data to memory.
  * GSC supports image rotation and image effect functions.
- *
- * M2M operation : supports crop/scale/rotation/csc so on.
- * Memory ----> GSC H/W ----> Memory.
- * Writeback operation : supports cloned screen with FIMD.
- * FIMD ----> GSC H/W ----> Memory.
- * Output operation : supports direct display using local path.
- * Memory ----> GSC H/W ----> FIMD, Mixer.
  */
 
-/*
- * TODO
- * 1. check suspend/resume api if needed.
- * 2. need to check use case platform_device_id.
- * 3. check src/dst size with, height.
- * 4. added check_prepare api for right register.
- * 5. need to add supported list in prop_list.
- * 6. check prescaler/scaler optimization.
- */
 
-#define GSC_MAX_DEVS	4
+#define GSC_MAX_CLOCKS	8
 #define GSC_MAX_SRC		4
 #define GSC_MAX_DST		16
 #define GSC_RESET_TIMEOUT	50
@@ -65,8 +51,6 @@
 #define GSC_SC_DOWN_RATIO_4_8		131072
 #define GSC_SC_DOWN_RATIO_3_8		174762
 #define GSC_SC_DOWN_RATIO_2_8		262144
-#define GSC_REFRESH_MIN	12
-#define GSC_REFRESH_MAX	60
 #define GSC_CROP_MAX	8192
 #define GSC_CROP_MIN	32
 #define GSC_SCALE_MAX	4224
@@ -77,10 +61,9 @@
 #define GSC_COEF_H_8T	8
 #define GSC_COEF_V_4T	4
 #define GSC_COEF_DEPTH	3
+#define GSC_AUTOSUSPEND_DELAY		2000
 
 #define get_gsc_context(dev)	platform_get_drvdata(to_platform_device(dev))
-#define get_ctx_from_ippdrv(ippdrv)	container_of(ippdrv,\
-					struct gsc_context, ippdrv);
 #define gsc_read(offset)		readl(ctx->regs + (offset))
 #define gsc_write(cfg, offset)	writel(cfg, ctx->regs + (offset))
 
@@ -103,51 +86,48 @@ struct gsc_scaler {
 	unsigned long main_vratio;
 };
 
-/*
- * A structure of scaler capability.
- *
- * find user manual 49.2 features.
- * @tile_w: tile mode or rotation width.
- * @tile_h: tile mode or rotation height.
- * @w: other cases width.
- * @h: other cases height.
- */
-struct gsc_capability {
-	/* tile or rotation */
-	u32	tile_w;
-	u32	tile_h;
-	/* other cases */
-	u32	w;
-	u32	h;
-};
-
 /*
  * A structure of gsc context.
  *
- * @ippdrv: prepare initialization using ippdrv.
  * @regs_res: register resources.
  * @regs: memory mapped io registers.
- * @sysreg: handle to SYSREG block regmap.
- * @lock: locking of operations.
  * @gsc_clk: gsc gate clock.
  * @sc: scaler infomations.
  * @id: gsc id.
  * @irq: irq number.
  * @rotation: supports rotation of src.
- * @suspended: qos operations.
  */
 struct gsc_context {
-	struct exynos_drm_ippdrv	ippdrv;
+	struct exynos_drm_ipp ipp;
+	struct drm_device *drm_dev;
+	struct device	*dev;
+	struct exynos_drm_ipp_task	*task;
+	struct exynos_drm_ipp_formats	*formats;
+	unsigned int			num_formats;
+
 	struct resource	*regs_res;
 	void __iomem	*regs;
-	struct regmap	*sysreg;
-	struct mutex	lock;
-	struct clk	*gsc_clk;
+	const char	**clk_names;
+	struct clk	*clocks[GSC_MAX_CLOCKS];
+	int		num_clocks;
 	struct gsc_scaler	sc;
 	int	id;
 	int	irq;
 	bool	rotation;
-	bool	suspended;
+};
+
+/**
+ * struct gsc_driverdata - per device type driver data for init time.
+ *
+ * @limits: picture size limits array
+ * @clk_names: names of clocks needed by this variant
+ * @num_clocks: the number of clocks needed by this variant
+ */
+struct gsc_driverdata {
+	const struct drm_exynos_ipp_limit *limits;
+	int		num_limits;
+	const char	*clk_names[GSC_MAX_CLOCKS];
+	int		num_clocks;
 };
 
 /* 8-tap Filter Coefficient */
@@ -438,25 +418,6 @@ static int gsc_sw_reset(struct gsc_context *ctx)
 	return 0;
 }
 
-static void gsc_set_gscblk_fimd_wb(struct gsc_context *ctx, bool enable)
-{
-	unsigned int gscblk_cfg;
-
-	if (!ctx->sysreg)
-		return;
-
-	regmap_read(ctx->sysreg, SYSREG_GSCBLK_CFG1, &gscblk_cfg);
-
-	if (enable)
-		gscblk_cfg |= GSC_BLK_DISP1WB_DEST(ctx->id) |
-				GSC_BLK_GSCL_WB_IN_SRC_SEL(ctx->id) |
-				GSC_BLK_SW_RESET_WB_DEST(ctx->id);
-	else
-		gscblk_cfg |= GSC_BLK_PXLASYNC_LO_MASK_WB(ctx->id);
-
-	regmap_write(ctx->sysreg, SYSREG_GSCBLK_CFG1, gscblk_cfg);
-}
-
 static void gsc_handle_irq(struct gsc_context *ctx, bool enable,
 		bool overflow, bool done)
 {
@@ -487,10 +448,8 @@ static void gsc_handle_irq(struct gsc_context *ctx, bool enable,
 }
 
 
-static int gsc_src_set_fmt(struct device *dev, u32 fmt)
+static void gsc_src_set_fmt(struct gsc_context *ctx, u32 fmt)
 {
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
 	u32 cfg;
 
 	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
@@ -506,6 +465,7 @@ static int gsc_src_set_fmt(struct device *dev, u32 fmt)
 		cfg |= GSC_IN_RGB565;
 		break;
 	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_ARGB8888:
 		cfg |= GSC_IN_XRGB8888;
 		break;
 	case DRM_FORMAT_BGRX8888:
@@ -548,115 +508,84 @@ static int gsc_src_set_fmt(struct device *dev, u32 fmt)
 		cfg |= (GSC_IN_CHROMA_ORDER_CBCR |
 			GSC_IN_YUV420_2P);
 		break;
-	default:
-		dev_err(ippdrv->dev, "invalid target yuv order 0x%x.\n", fmt);
-		return -EINVAL;
 	}
 
 	gsc_write(cfg, GSC_IN_CON);
-
-	return 0;
 }
 
-static int gsc_src_set_transf(struct device *dev,
-		enum drm_exynos_degree degree,
-		enum drm_exynos_flip flip, bool *swap)
+static void gsc_src_set_transf(struct gsc_context *ctx, unsigned int rotation)
 {
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
+	unsigned int degree = rotation & DRM_MODE_ROTATE_MASK;
 	u32 cfg;
 
-	DRM_DEBUG_KMS("degree[%d]flip[0x%x]\n", degree, flip);
-
 	cfg = gsc_read(GSC_IN_CON);
 	cfg &= ~GSC_IN_ROT_MASK;
 
 	switch (degree) {
-	case EXYNOS_DRM_DEGREE_0:
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
+	case DRM_MODE_ROTATE_0:
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg |= GSC_IN_ROT_XFLIP;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
+		if (rotation & DRM_MODE_REFLECT_X)
 			cfg |= GSC_IN_ROT_YFLIP;
 		break;
-	case EXYNOS_DRM_DEGREE_90:
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
-			cfg |= GSC_IN_ROT_90_XFLIP;
-		else if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
-			cfg |= GSC_IN_ROT_90_YFLIP;
-		else
-			cfg |= GSC_IN_ROT_90;
+	case DRM_MODE_ROTATE_90:
+		cfg |= GSC_IN_ROT_90;
+		if (rotation & DRM_MODE_REFLECT_Y)
+			cfg |= GSC_IN_ROT_XFLIP;
+		if (rotation & DRM_MODE_REFLECT_X)
+			cfg |= GSC_IN_ROT_YFLIP;
 		break;
-	case EXYNOS_DRM_DEGREE_180:
+	case DRM_MODE_ROTATE_180:
 		cfg |= GSC_IN_ROT_180;
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg &= ~GSC_IN_ROT_XFLIP;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
+		if (rotation & DRM_MODE_REFLECT_X)
 			cfg &= ~GSC_IN_ROT_YFLIP;
 		break;
-	case EXYNOS_DRM_DEGREE_270:
+	case DRM_MODE_ROTATE_270:
 		cfg |= GSC_IN_ROT_270;
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg &= ~GSC_IN_ROT_XFLIP;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
+		if (rotation & DRM_MODE_REFLECT_X)
 			cfg &= ~GSC_IN_ROT_YFLIP;
 		break;
-	default:
-		dev_err(ippdrv->dev, "invalid degree value %d.\n", degree);
-		return -EINVAL;
 	}
 
 	gsc_write(cfg, GSC_IN_CON);
 
 	ctx->rotation = (cfg & GSC_IN_ROT_90) ? 1 : 0;
-	*swap = ctx->rotation;
-
-	return 0;
 }
 
-static int gsc_src_set_size(struct device *dev, int swap,
-		struct drm_exynos_pos *pos, struct drm_exynos_sz *sz)
+static void gsc_src_set_size(struct gsc_context *ctx,
+			     struct exynos_drm_ipp_buffer *buf)
 {
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct drm_exynos_pos img_pos = *pos;
 	struct gsc_scaler *sc = &ctx->sc;
 	u32 cfg;
 
-	DRM_DEBUG_KMS("swap[%d]x[%d]y[%d]w[%d]h[%d]\n",
-		swap, pos->x, pos->y, pos->w, pos->h);
-
-	if (swap) {
-		img_pos.w = pos->h;
-		img_pos.h = pos->w;
-	}
-
 	/* pixel offset */
-	cfg = (GSC_SRCIMG_OFFSET_X(img_pos.x) |
-		GSC_SRCIMG_OFFSET_Y(img_pos.y));
+	cfg = (GSC_SRCIMG_OFFSET_X(buf->rect.x) |
+		GSC_SRCIMG_OFFSET_Y(buf->rect.y));
 	gsc_write(cfg, GSC_SRCIMG_OFFSET);
 
 	/* cropped size */
-	cfg = (GSC_CROPPED_WIDTH(img_pos.w) |
-		GSC_CROPPED_HEIGHT(img_pos.h));
+	cfg = (GSC_CROPPED_WIDTH(buf->rect.w) |
+		GSC_CROPPED_HEIGHT(buf->rect.h));
 	gsc_write(cfg, GSC_CROPPED_SIZE);
 
-	DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", sz->hsize, sz->vsize);
-
 	/* original size */
 	cfg = gsc_read(GSC_SRCIMG_SIZE);
 	cfg &= ~(GSC_SRCIMG_HEIGHT_MASK |
 		GSC_SRCIMG_WIDTH_MASK);
 
-	cfg |= (GSC_SRCIMG_WIDTH(sz->hsize) |
-		GSC_SRCIMG_HEIGHT(sz->vsize));
+	cfg |= (GSC_SRCIMG_WIDTH(buf->buf.width) |
+		GSC_SRCIMG_HEIGHT(buf->buf.height));
 
 	gsc_write(cfg, GSC_SRCIMG_SIZE);
 
 	cfg = gsc_read(GSC_IN_CON);
 	cfg &= ~GSC_IN_RGB_TYPE_MASK;
 
-	DRM_DEBUG_KMS("width[%d]range[%d]\n", pos->w, sc->range);
-
-	if (pos->w >= GSC_WIDTH_ITU_709)
+	if (buf->rect.w >= GSC_WIDTH_ITU_709)
 		if (sc->range)
 			cfg |= GSC_IN_RGB_HD_WIDE;
 		else
@@ -668,103 +597,39 @@ static int gsc_src_set_size(struct device *dev, int swap,
 			cfg |= GSC_IN_RGB_SD_NARROW;
 
 	gsc_write(cfg, GSC_IN_CON);
-
-	return 0;
 }
 
-static int gsc_src_set_buf_seq(struct gsc_context *ctx, u32 buf_id,
-		enum drm_exynos_ipp_buf_type buf_type)
+static void gsc_src_set_buf_seq(struct gsc_context *ctx, u32 buf_id,
+			       bool enqueue)
 {
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	bool masked;
+	bool masked = !enqueue;
 	u32 cfg;
 	u32 mask = 0x00000001 << buf_id;
 
-	DRM_DEBUG_KMS("buf_id[%d]buf_type[%d]\n", buf_id, buf_type);
-
 	/* mask register set */
 	cfg = gsc_read(GSC_IN_BASE_ADDR_Y_MASK);
 
-	switch (buf_type) {
-	case IPP_BUF_ENQUEUE:
-		masked = false;
-		break;
-	case IPP_BUF_DEQUEUE:
-		masked = true;
-		break;
-	default:
-		dev_err(ippdrv->dev, "invalid buf ctrl parameter.\n");
-		return -EINVAL;
-	}
-
 	/* sequence id */
 	cfg &= ~mask;
 	cfg |= masked << buf_id;
 	gsc_write(cfg, GSC_IN_BASE_ADDR_Y_MASK);
 	gsc_write(cfg, GSC_IN_BASE_ADDR_CB_MASK);
 	gsc_write(cfg, GSC_IN_BASE_ADDR_CR_MASK);
-
-	return 0;
 }
 
-static int gsc_src_set_addr(struct device *dev,
-		struct drm_exynos_ipp_buf_info *buf_info, u32 buf_id,
-		enum drm_exynos_ipp_buf_type buf_type)
+static void gsc_src_set_addr(struct gsc_context *ctx, u32 buf_id,
+			    struct exynos_drm_ipp_buffer *buf)
 {
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node = ippdrv->c_node;
-	struct drm_exynos_ipp_property *property;
-
-	if (!c_node) {
-		DRM_ERROR("failed to get c_node.\n");
-		return -EFAULT;
-	}
-
-	property = &c_node->property;
-
-	DRM_DEBUG_KMS("prop_id[%d]buf_id[%d]buf_type[%d]\n",
-		property->prop_id, buf_id, buf_type);
-
-	if (buf_id > GSC_MAX_SRC) {
-		dev_info(ippdrv->dev, "invalid buf_id %d.\n", buf_id);
-		return -EINVAL;
-	}
-
 	/* address register set */
-	switch (buf_type) {
-	case IPP_BUF_ENQUEUE:
-		gsc_write(buf_info->base[EXYNOS_DRM_PLANAR_Y],
-			GSC_IN_BASE_ADDR_Y(buf_id));
-		gsc_write(buf_info->base[EXYNOS_DRM_PLANAR_CB],
-			GSC_IN_BASE_ADDR_CB(buf_id));
-		gsc_write(buf_info->base[EXYNOS_DRM_PLANAR_CR],
-			GSC_IN_BASE_ADDR_CR(buf_id));
-		break;
-	case IPP_BUF_DEQUEUE:
-		gsc_write(0x0, GSC_IN_BASE_ADDR_Y(buf_id));
-		gsc_write(0x0, GSC_IN_BASE_ADDR_CB(buf_id));
-		gsc_write(0x0, GSC_IN_BASE_ADDR_CR(buf_id));
-		break;
-	default:
-		/* bypass */
-		break;
-	}
+	gsc_write(buf->dma_addr[0], GSC_IN_BASE_ADDR_Y(buf_id));
+	gsc_write(buf->dma_addr[1], GSC_IN_BASE_ADDR_CB(buf_id));
+	gsc_write(buf->dma_addr[2], GSC_IN_BASE_ADDR_CR(buf_id));
 
-	return gsc_src_set_buf_seq(ctx, buf_id, buf_type);
+	gsc_src_set_buf_seq(ctx, buf_id, true);
 }
 
-static struct exynos_drm_ipp_ops gsc_src_ops = {
-	.set_fmt = gsc_src_set_fmt,
-	.set_transf = gsc_src_set_transf,
-	.set_size = gsc_src_set_size,
-	.set_addr = gsc_src_set_addr,
-};
-
-static int gsc_dst_set_fmt(struct device *dev, u32 fmt)
+static void gsc_dst_set_fmt(struct gsc_context *ctx, u32 fmt)
 {
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
 	u32 cfg;
 
 	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
@@ -779,8 +644,9 @@ static int gsc_dst_set_fmt(struct device *dev, u32 fmt)
 	case DRM_FORMAT_RGB565:
 		cfg |= GSC_OUT_RGB565;
 		break;
+	case DRM_FORMAT_ARGB8888:
 	case DRM_FORMAT_XRGB8888:
-		cfg |= GSC_OUT_XRGB8888;
+		cfg |= (GSC_OUT_XRGB8888 | GSC_OUT_GLOBAL_ALPHA(0xff));
 		break;
 	case DRM_FORMAT_BGRX8888:
 		cfg |= (GSC_OUT_XRGB8888 | GSC_OUT_RB_SWAP);
@@ -819,69 +685,9 @@ static int gsc_dst_set_fmt(struct device *dev, u32 fmt)
 		cfg |= (GSC_OUT_CHROMA_ORDER_CBCR |
 			GSC_OUT_YUV420_2P);
 		break;
-	default:
-		dev_err(ippdrv->dev, "invalid target yuv order 0x%x.\n", fmt);
-		return -EINVAL;
 	}
 
 	gsc_write(cfg, GSC_OUT_CON);
-
-	return 0;
-}
-
-static int gsc_dst_set_transf(struct device *dev,
-		enum drm_exynos_degree degree,
-		enum drm_exynos_flip flip, bool *swap)
-{
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	u32 cfg;
-
-	DRM_DEBUG_KMS("degree[%d]flip[0x%x]\n", degree, flip);
-
-	cfg = gsc_read(GSC_IN_CON);
-	cfg &= ~GSC_IN_ROT_MASK;
-
-	switch (degree) {
-	case EXYNOS_DRM_DEGREE_0:
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
-			cfg |= GSC_IN_ROT_XFLIP;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
-			cfg |= GSC_IN_ROT_YFLIP;
-		break;
-	case EXYNOS_DRM_DEGREE_90:
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
-			cfg |= GSC_IN_ROT_90_XFLIP;
-		else if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
-			cfg |= GSC_IN_ROT_90_YFLIP;
-		else
-			cfg |= GSC_IN_ROT_90;
-		break;
-	case EXYNOS_DRM_DEGREE_180:
-		cfg |= GSC_IN_ROT_180;
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
-			cfg &= ~GSC_IN_ROT_XFLIP;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
-			cfg &= ~GSC_IN_ROT_YFLIP;
-		break;
-	case EXYNOS_DRM_DEGREE_270:
-		cfg |= GSC_IN_ROT_270;
-		if (flip & EXYNOS_DRM_FLIP_VERTICAL)
-			cfg &= ~GSC_IN_ROT_XFLIP;
-		if (flip & EXYNOS_DRM_FLIP_HORIZONTAL)
-			cfg &= ~GSC_IN_ROT_YFLIP;
-		break;
-	default:
-		dev_err(ippdrv->dev, "invalid degree value %d.\n", degree);
-		return -EINVAL;
-	}
-
-	gsc_write(cfg, GSC_IN_CON);
-
-	ctx->rotation = (cfg & GSC_IN_ROT_90) ? 1 : 0;
-	*swap = ctx->rotation;
-
-	return 0;
 }
 
 static int gsc_get_ratio_shift(u32 src, u32 dst, u32 *ratio)
@@ -919,9 +725,9 @@ static void gsc_get_prescaler_shfactor(u32 hratio, u32 vratio, u32 *shfactor)
 }
 
 static int gsc_set_prescaler(struct gsc_context *ctx, struct gsc_scaler *sc,
-		struct drm_exynos_pos *src, struct drm_exynos_pos *dst)
+			     struct drm_exynos_ipp_task_rect *src,
+			     struct drm_exynos_ipp_task_rect *dst)
 {
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
 	u32 cfg;
 	u32 src_w, src_h, dst_w, dst_h;
 	int ret = 0;
@@ -939,13 +745,13 @@ static int gsc_set_prescaler(struct gsc_context *ctx, struct gsc_scaler *sc,
 
 	ret = gsc_get_ratio_shift(src_w, dst_w, &sc->pre_hratio);
 	if (ret) {
-		dev_err(ippdrv->dev, "failed to get ratio horizontal.\n");
+		dev_err(ctx->dev, "failed to get ratio horizontal.\n");
 		return ret;
 	}
 
 	ret = gsc_get_ratio_shift(src_h, dst_h, &sc->pre_vratio);
 	if (ret) {
-		dev_err(ippdrv->dev, "failed to get ratio vertical.\n");
+		dev_err(ctx->dev, "failed to get ratio vertical.\n");
 		return ret;
 	}
 
@@ -1039,47 +845,37 @@ static void gsc_set_scaler(struct gsc_context *ctx, struct gsc_scaler *sc)
 	gsc_write(cfg, GSC_MAIN_V_RATIO);
 }
 
-static int gsc_dst_set_size(struct device *dev, int swap,
-		struct drm_exynos_pos *pos, struct drm_exynos_sz *sz)
+static void gsc_dst_set_size(struct gsc_context *ctx,
+			     struct exynos_drm_ipp_buffer *buf)
 {
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct drm_exynos_pos img_pos = *pos;
 	struct gsc_scaler *sc = &ctx->sc;
 	u32 cfg;
 
-	DRM_DEBUG_KMS("swap[%d]x[%d]y[%d]w[%d]h[%d]\n",
-		swap, pos->x, pos->y, pos->w, pos->h);
-
-	if (swap) {
-		img_pos.w = pos->h;
-		img_pos.h = pos->w;
-	}
-
 	/* pixel offset */
-	cfg = (GSC_DSTIMG_OFFSET_X(pos->x) |
-		GSC_DSTIMG_OFFSET_Y(pos->y));
+	cfg = (GSC_DSTIMG_OFFSET_X(buf->rect.x) |
+		GSC_DSTIMG_OFFSET_Y(buf->rect.y));
 	gsc_write(cfg, GSC_DSTIMG_OFFSET);
 
 	/* scaled size */
-	cfg = (GSC_SCALED_WIDTH(img_pos.w) | GSC_SCALED_HEIGHT(img_pos.h));
+	if (ctx->rotation)
+		cfg = (GSC_SCALED_WIDTH(buf->rect.h) |
+		       GSC_SCALED_HEIGHT(buf->rect.w));
+	else
+		cfg = (GSC_SCALED_WIDTH(buf->rect.w) |
+		       GSC_SCALED_HEIGHT(buf->rect.h));
 	gsc_write(cfg, GSC_SCALED_SIZE);
 
-	DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", sz->hsize, sz->vsize);
-
 	/* original size */
 	cfg = gsc_read(GSC_DSTIMG_SIZE);
-	cfg &= ~(GSC_DSTIMG_HEIGHT_MASK |
-		GSC_DSTIMG_WIDTH_MASK);
-	cfg |= (GSC_DSTIMG_WIDTH(sz->hsize) |
-		GSC_DSTIMG_HEIGHT(sz->vsize));
+	cfg &= ~(GSC_DSTIMG_HEIGHT_MASK | GSC_DSTIMG_WIDTH_MASK);
+	cfg |= GSC_DSTIMG_WIDTH(buf->buf.width) |
+	       GSC_DSTIMG_HEIGHT(buf->buf.height);
 	gsc_write(cfg, GSC_DSTIMG_SIZE);
 
 	cfg = gsc_read(GSC_OUT_CON);
 	cfg &= ~GSC_OUT_RGB_TYPE_MASK;
 
-	DRM_DEBUG_KMS("width[%d]range[%d]\n", pos->w, sc->range);
-
-	if (pos->w >= GSC_WIDTH_ITU_709)
+	if (buf->rect.w >= GSC_WIDTH_ITU_709)
 		if (sc->range)
 			cfg |= GSC_OUT_RGB_HD_WIDE;
 		else
@@ -1091,8 +887,6 @@ static int gsc_dst_set_size(struct device *dev, int swap,
 			cfg |= GSC_OUT_RGB_SD_NARROW;
 
 	gsc_write(cfg, GSC_OUT_CON);
-
-	return 0;
 }
 
 static int gsc_dst_get_buf_seq(struct gsc_context *ctx)
@@ -1111,35 +905,16 @@ static int gsc_dst_get_buf_seq(struct gsc_context *ctx)
 	return buf_num;
 }
 
-static int gsc_dst_set_buf_seq(struct gsc_context *ctx, u32 buf_id,
-		enum drm_exynos_ipp_buf_type buf_type)
+static void gsc_dst_set_buf_seq(struct gsc_context *ctx, u32 buf_id,
+				bool enqueue)
 {
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	bool masked;
+	bool masked = !enqueue;
 	u32 cfg;
 	u32 mask = 0x00000001 << buf_id;
-	int ret = 0;
-
-	DRM_DEBUG_KMS("buf_id[%d]buf_type[%d]\n", buf_id, buf_type);
-
-	mutex_lock(&ctx->lock);
 
 	/* mask register set */
 	cfg = gsc_read(GSC_OUT_BASE_ADDR_Y_MASK);
 
-	switch (buf_type) {
-	case IPP_BUF_ENQUEUE:
-		masked = false;
-		break;
-	case IPP_BUF_DEQUEUE:
-		masked = true;
-		break;
-	default:
-		dev_err(ippdrv->dev, "invalid buf ctrl parameter.\n");
-		ret =  -EINVAL;
-		goto err_unlock;
-	}
-
 	/* sequence id */
 	cfg &= ~mask;
 	cfg |= masked << buf_id;
@@ -1148,94 +923,29 @@ static int gsc_dst_set_buf_seq(struct gsc_context *ctx, u32 buf_id,
 	gsc_write(cfg, GSC_OUT_BASE_ADDR_CR_MASK);
 
 	/* interrupt enable */
-	if (buf_type == IPP_BUF_ENQUEUE &&
-	    gsc_dst_get_buf_seq(ctx) >= GSC_BUF_START)
+	if (enqueue && gsc_dst_get_buf_seq(ctx) >= GSC_BUF_START)
 		gsc_handle_irq(ctx, true, false, true);
 
 	/* interrupt disable */
-	if (buf_type == IPP_BUF_DEQUEUE &&
-	    gsc_dst_get_buf_seq(ctx) <= GSC_BUF_STOP)
+	if (!enqueue && gsc_dst_get_buf_seq(ctx) <= GSC_BUF_STOP)
 		gsc_handle_irq(ctx, false, false, true);
-
-err_unlock:
-	mutex_unlock(&ctx->lock);
-	return ret;
 }
 
-static int gsc_dst_set_addr(struct device *dev,
-		struct drm_exynos_ipp_buf_info *buf_info, u32 buf_id,
-		enum drm_exynos_ipp_buf_type buf_type)
+static void gsc_dst_set_addr(struct gsc_context *ctx,
+			     u32 buf_id, struct exynos_drm_ipp_buffer *buf)
 {
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node = ippdrv->c_node;
-	struct drm_exynos_ipp_property *property;
-
-	if (!c_node) {
-		DRM_ERROR("failed to get c_node.\n");
-		return -EFAULT;
-	}
-
-	property = &c_node->property;
-
-	DRM_DEBUG_KMS("prop_id[%d]buf_id[%d]buf_type[%d]\n",
-		property->prop_id, buf_id, buf_type);
-
-	if (buf_id > GSC_MAX_DST) {
-		dev_info(ippdrv->dev, "invalid buf_id %d.\n", buf_id);
-		return -EINVAL;
-	}
-
 	/* address register set */
-	switch (buf_type) {
-	case IPP_BUF_ENQUEUE:
-		gsc_write(buf_info->base[EXYNOS_DRM_PLANAR_Y],
-			GSC_OUT_BASE_ADDR_Y(buf_id));
-		gsc_write(buf_info->base[EXYNOS_DRM_PLANAR_CB],
-			GSC_OUT_BASE_ADDR_CB(buf_id));
-		gsc_write(buf_info->base[EXYNOS_DRM_PLANAR_CR],
-			GSC_OUT_BASE_ADDR_CR(buf_id));
-		break;
-	case IPP_BUF_DEQUEUE:
-		gsc_write(0x0, GSC_OUT_BASE_ADDR_Y(buf_id));
-		gsc_write(0x0, GSC_OUT_BASE_ADDR_CB(buf_id));
-		gsc_write(0x0, GSC_OUT_BASE_ADDR_CR(buf_id));
-		break;
-	default:
-		/* bypass */
-		break;
-	}
+	gsc_write(buf->dma_addr[0], GSC_OUT_BASE_ADDR_Y(buf_id));
+	gsc_write(buf->dma_addr[1], GSC_OUT_BASE_ADDR_CB(buf_id));
+	gsc_write(buf->dma_addr[2], GSC_OUT_BASE_ADDR_CR(buf_id));
 
-	return gsc_dst_set_buf_seq(ctx, buf_id, buf_type);
-}
-
-static struct exynos_drm_ipp_ops gsc_dst_ops = {
-	.set_fmt = gsc_dst_set_fmt,
-	.set_transf = gsc_dst_set_transf,
-	.set_size = gsc_dst_set_size,
-	.set_addr = gsc_dst_set_addr,
-};
-
-static int gsc_clk_ctrl(struct gsc_context *ctx, bool enable)
-{
-	DRM_DEBUG_KMS("enable[%d]\n", enable);
-
-	if (enable) {
-		clk_prepare_enable(ctx->gsc_clk);
-		ctx->suspended = false;
-	} else {
-		clk_disable_unprepare(ctx->gsc_clk);
-		ctx->suspended = true;
-	}
-
-	return 0;
+	gsc_dst_set_buf_seq(ctx, buf_id, true);
 }
 
 static int gsc_get_src_buf_index(struct gsc_context *ctx)
 {
 	u32 cfg, curr_index, i;
 	u32 buf_id = GSC_MAX_SRC;
-	int ret;
 
 	DRM_DEBUG_KMS("gsc id[%d]\n", ctx->id);
 
@@ -1249,19 +959,15 @@ static int gsc_get_src_buf_index(struct gsc_context *ctx)
 		}
 	}
 
+	DRM_DEBUG_KMS("cfg[0x%x]curr_index[%d]buf_id[%d]\n", cfg,
+		curr_index, buf_id);
+
 	if (buf_id == GSC_MAX_SRC) {
 		DRM_ERROR("failed to get in buffer index.\n");
 		return -EINVAL;
 	}
 
-	ret = gsc_src_set_buf_seq(ctx, buf_id, IPP_BUF_DEQUEUE);
-	if (ret < 0) {
-		DRM_ERROR("failed to dequeue.\n");
-		return ret;
-	}
-
-	DRM_DEBUG_KMS("cfg[0x%x]curr_index[%d]buf_id[%d]\n", cfg,
-		curr_index, buf_id);
+	gsc_src_set_buf_seq(ctx, buf_id, false);
 
 	return buf_id;
 }
@@ -1270,7 +976,6 @@ static int gsc_get_dst_buf_index(struct gsc_context *ctx)
 {
 	u32 cfg, curr_index, i;
 	u32 buf_id = GSC_MAX_DST;
-	int ret;
 
 	DRM_DEBUG_KMS("gsc id[%d]\n", ctx->id);
 
@@ -1289,11 +994,7 @@ static int gsc_get_dst_buf_index(struct gsc_context *ctx)
 		return -EINVAL;
 	}
 
-	ret = gsc_dst_set_buf_seq(ctx, buf_id, IPP_BUF_DEQUEUE);
-	if (ret < 0) {
-		DRM_ERROR("failed to dequeue.\n");
-		return ret;
-	}
+	gsc_dst_set_buf_seq(ctx, buf_id, false);
 
 	DRM_DEBUG_KMS("cfg[0x%x]curr_index[%d]buf_id[%d]\n", cfg,
 		curr_index, buf_id);
@@ -1304,215 +1005,55 @@ static int gsc_get_dst_buf_index(struct gsc_context *ctx)
 static irqreturn_t gsc_irq_handler(int irq, void *dev_id)
 {
 	struct gsc_context *ctx = dev_id;
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node = ippdrv->c_node;
-	struct drm_exynos_ipp_event_work *event_work =
-		c_node->event_work;
 	u32 status;
-	int buf_id[EXYNOS_DRM_OPS_MAX];
+	int err = 0;
 
 	DRM_DEBUG_KMS("gsc id[%d]\n", ctx->id);
 
 	status = gsc_read(GSC_IRQ);
 	if (status & GSC_IRQ_STATUS_OR_IRQ) {
-		dev_err(ippdrv->dev, "occurred overflow at %d, status 0x%x.\n",
+		dev_err(ctx->dev, "occurred overflow at %d, status 0x%x.\n",
 			ctx->id, status);
-		return IRQ_NONE;
+		err = -EINVAL;
 	}
 
 	if (status & GSC_IRQ_STATUS_OR_FRM_DONE) {
-		dev_dbg(ippdrv->dev, "occurred frame done at %d, status 0x%x.\n",
-			ctx->id, status);
-
-		buf_id[EXYNOS_DRM_OPS_SRC] = gsc_get_src_buf_index(ctx);
-		if (buf_id[EXYNOS_DRM_OPS_SRC] < 0)
-			return IRQ_HANDLED;
-
-		buf_id[EXYNOS_DRM_OPS_DST] = gsc_get_dst_buf_index(ctx);
-		if (buf_id[EXYNOS_DRM_OPS_DST] < 0)
-			return IRQ_HANDLED;
-
-		DRM_DEBUG_KMS("buf_id_src[%d]buf_id_dst[%d]\n",
-			buf_id[EXYNOS_DRM_OPS_SRC], buf_id[EXYNOS_DRM_OPS_DST]);
-
-		event_work->ippdrv = ippdrv;
-		event_work->buf_id[EXYNOS_DRM_OPS_SRC] =
-			buf_id[EXYNOS_DRM_OPS_SRC];
-		event_work->buf_id[EXYNOS_DRM_OPS_DST] =
-			buf_id[EXYNOS_DRM_OPS_DST];
-		queue_work(ippdrv->event_workq, &event_work->work);
-	}
-
-	return IRQ_HANDLED;
-}
-
-static int gsc_init_prop_list(struct exynos_drm_ippdrv *ippdrv)
-{
-	struct drm_exynos_ipp_prop_list *prop_list = &ippdrv->prop_list;
-
-	prop_list->version = 1;
-	prop_list->writeback = 1;
-	prop_list->refresh_min = GSC_REFRESH_MIN;
-	prop_list->refresh_max = GSC_REFRESH_MAX;
-	prop_list->flip = (1 << EXYNOS_DRM_FLIP_VERTICAL) |
-				(1 << EXYNOS_DRM_FLIP_HORIZONTAL);
-	prop_list->degree = (1 << EXYNOS_DRM_DEGREE_0) |
-				(1 << EXYNOS_DRM_DEGREE_90) |
-				(1 << EXYNOS_DRM_DEGREE_180) |
-				(1 << EXYNOS_DRM_DEGREE_270);
-	prop_list->csc = 1;
-	prop_list->crop = 1;
-	prop_list->crop_max.hsize = GSC_CROP_MAX;
-	prop_list->crop_max.vsize = GSC_CROP_MAX;
-	prop_list->crop_min.hsize = GSC_CROP_MIN;
-	prop_list->crop_min.vsize = GSC_CROP_MIN;
-	prop_list->scale = 1;
-	prop_list->scale_max.hsize = GSC_SCALE_MAX;
-	prop_list->scale_max.vsize = GSC_SCALE_MAX;
-	prop_list->scale_min.hsize = GSC_SCALE_MIN;
-	prop_list->scale_min.vsize = GSC_SCALE_MIN;
-
-	return 0;
-}
-
-static inline bool gsc_check_drm_flip(enum drm_exynos_flip flip)
-{
-	switch (flip) {
-	case EXYNOS_DRM_FLIP_NONE:
-	case EXYNOS_DRM_FLIP_VERTICAL:
-	case EXYNOS_DRM_FLIP_HORIZONTAL:
-	case EXYNOS_DRM_FLIP_BOTH:
-		return true;
-	default:
-		DRM_DEBUG_KMS("invalid flip\n");
-		return false;
-	}
-}
-
-static int gsc_ippdrv_check_property(struct device *dev,
-		struct drm_exynos_ipp_property *property)
-{
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	struct drm_exynos_ipp_prop_list *pp = &ippdrv->prop_list;
-	struct drm_exynos_ipp_config *config;
-	struct drm_exynos_pos *pos;
-	struct drm_exynos_sz *sz;
-	bool swap;
-	int i;
-
-	for_each_ipp_ops(i) {
-		if ((i == EXYNOS_DRM_OPS_SRC) &&
-			(property->cmd == IPP_CMD_WB))
-			continue;
+		int src_buf_id, dst_buf_id;
 
-		config = &property->config[i];
-		pos = &config->pos;
-		sz = &config->sz;
-
-		/* check for flip */
-		if (!gsc_check_drm_flip(config->flip)) {
-			DRM_ERROR("invalid flip.\n");
-			goto err_property;
-		}
-
-		/* check for degree */
-		switch (config->degree) {
-		case EXYNOS_DRM_DEGREE_90:
-		case EXYNOS_DRM_DEGREE_270:
-			swap = true;
-			break;
-		case EXYNOS_DRM_DEGREE_0:
-		case EXYNOS_DRM_DEGREE_180:
-			swap = false;
-			break;
-		default:
-			DRM_ERROR("invalid degree.\n");
-			goto err_property;
-		}
+		dev_dbg(ctx->dev, "occurred frame done at %d, status 0x%x.\n",
+			ctx->id, status);
 
-		/* check for buffer bound */
-		if ((pos->x + pos->w > sz->hsize) ||
-			(pos->y + pos->h > sz->vsize)) {
-			DRM_ERROR("out of buf bound.\n");
-			goto err_property;
-		}
+		src_buf_id = gsc_get_src_buf_index(ctx);
+		dst_buf_id = gsc_get_dst_buf_index(ctx);
 
-		/* check for crop */
-		if ((i == EXYNOS_DRM_OPS_SRC) && (pp->crop)) {
-			if (swap) {
-				if ((pos->h < pp->crop_min.hsize) ||
-					(sz->vsize > pp->crop_max.hsize) ||
-					(pos->w < pp->crop_min.vsize) ||
-					(sz->hsize > pp->crop_max.vsize)) {
-					DRM_ERROR("out of crop size.\n");
-					goto err_property;
-				}
-			} else {
-				if ((pos->w < pp->crop_min.hsize) ||
-					(sz->hsize > pp->crop_max.hsize) ||
-					(pos->h < pp->crop_min.vsize) ||
-					(sz->vsize > pp->crop_max.vsize)) {
-					DRM_ERROR("out of crop size.\n");
-					goto err_property;
-				}
-			}
-		}
+		DRM_DEBUG_KMS("buf_id_src[%d]buf_id_dst[%d]\n",	src_buf_id,
+			      dst_buf_id);
 
-		/* check for scale */
-		if ((i == EXYNOS_DRM_OPS_DST) && (pp->scale)) {
-			if (swap) {
-				if ((pos->h < pp->scale_min.hsize) ||
-					(sz->vsize > pp->scale_max.hsize) ||
-					(pos->w < pp->scale_min.vsize) ||
-					(sz->hsize > pp->scale_max.vsize)) {
-					DRM_ERROR("out of scale size.\n");
-					goto err_property;
-				}
-			} else {
-				if ((pos->w < pp->scale_min.hsize) ||
-					(sz->hsize > pp->scale_max.hsize) ||
-					(pos->h < pp->scale_min.vsize) ||
-					(sz->vsize > pp->scale_max.vsize)) {
-					DRM_ERROR("out of scale size.\n");
-					goto err_property;
-				}
-			}
-		}
+		if (src_buf_id < 0 || dst_buf_id < 0)
+			err = -EINVAL;
 	}
 
-	return 0;
-
-err_property:
-	for_each_ipp_ops(i) {
-		if ((i == EXYNOS_DRM_OPS_SRC) &&
-			(property->cmd == IPP_CMD_WB))
-			continue;
+	if (ctx->task) {
+		struct exynos_drm_ipp_task *task = ctx->task;
 
-		config = &property->config[i];
-		pos = &config->pos;
-		sz = &config->sz;
-
-		DRM_ERROR("[%s]f[%d]r[%d]pos[%d %d %d %d]sz[%d %d]\n",
-			i ? "dst" : "src", config->flip, config->degree,
-			pos->x, pos->y, pos->w, pos->h,
-			sz->hsize, sz->vsize);
+		ctx->task = NULL;
+		pm_runtime_mark_last_busy(ctx->dev);
+		pm_runtime_put_autosuspend(ctx->dev);
+		exynos_drm_ipp_task_done(task, err);
 	}
 
-	return -EINVAL;
+	return IRQ_HANDLED;
 }
 
-
-static int gsc_ippdrv_reset(struct device *dev)
+static int gsc_reset(struct gsc_context *ctx)
 {
-	struct gsc_context *ctx = get_gsc_context(dev);
 	struct gsc_scaler *sc = &ctx->sc;
 	int ret;
 
 	/* reset h/w block */
 	ret = gsc_sw_reset(ctx);
 	if (ret < 0) {
-		dev_err(dev, "failed to reset hardware.\n");
+		dev_err(ctx->dev, "failed to reset hardware.\n");
 		return ret;
 	}
 
@@ -1523,166 +1064,172 @@ static int gsc_ippdrv_reset(struct device *dev)
 	return 0;
 }
 
-static int gsc_ippdrv_start(struct device *dev, enum drm_exynos_ipp_cmd cmd)
+static void gsc_start(struct gsc_context *ctx)
 {
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node = ippdrv->c_node;
-	struct drm_exynos_ipp_property *property;
-	struct drm_exynos_ipp_config *config;
-	struct drm_exynos_pos	img_pos[EXYNOS_DRM_OPS_MAX];
-	struct drm_exynos_ipp_set_wb set_wb;
 	u32 cfg;
-	int ret, i;
-
-	DRM_DEBUG_KMS("cmd[%d]\n", cmd);
-
-	if (!c_node) {
-		DRM_ERROR("failed to get c_node.\n");
-		return -EINVAL;
-	}
-
-	property = &c_node->property;
 
 	gsc_handle_irq(ctx, true, false, true);
 
-	for_each_ipp_ops(i) {
-		config = &property->config[i];
-		img_pos[i] = config->pos;
-	}
+	/* enable one shot */
+	cfg = gsc_read(GSC_ENABLE);
+	cfg &= ~(GSC_ENABLE_ON_CLEAR_MASK |
+		GSC_ENABLE_CLK_GATE_MODE_MASK);
+	cfg |= GSC_ENABLE_ON_CLEAR_ONESHOT;
+	gsc_write(cfg, GSC_ENABLE);
 
-	switch (cmd) {
-	case IPP_CMD_M2M:
-		/* enable one shot */
-		cfg = gsc_read(GSC_ENABLE);
-		cfg &= ~(GSC_ENABLE_ON_CLEAR_MASK |
-			GSC_ENABLE_CLK_GATE_MODE_MASK);
-		cfg |= GSC_ENABLE_ON_CLEAR_ONESHOT;
-		gsc_write(cfg, GSC_ENABLE);
-
-		/* src dma memory */
-		cfg = gsc_read(GSC_IN_CON);
-		cfg &= ~(GSC_IN_PATH_MASK | GSC_IN_LOCAL_SEL_MASK);
-		cfg |= GSC_IN_PATH_MEMORY;
-		gsc_write(cfg, GSC_IN_CON);
-
-		/* dst dma memory */
-		cfg = gsc_read(GSC_OUT_CON);
-		cfg |= GSC_OUT_PATH_MEMORY;
-		gsc_write(cfg, GSC_OUT_CON);
-		break;
-	case IPP_CMD_WB:
-		set_wb.enable = 1;
-		set_wb.refresh = property->refresh_rate;
-		gsc_set_gscblk_fimd_wb(ctx, set_wb.enable);
-		exynos_drm_ippnb_send_event(IPP_SET_WRITEBACK, (void *)&set_wb);
-
-		/* src local path */
-		cfg = gsc_read(GSC_IN_CON);
-		cfg &= ~(GSC_IN_PATH_MASK | GSC_IN_LOCAL_SEL_MASK);
-		cfg |= (GSC_IN_PATH_LOCAL | GSC_IN_LOCAL_FIMD_WB);
-		gsc_write(cfg, GSC_IN_CON);
-
-		/* dst dma memory */
-		cfg = gsc_read(GSC_OUT_CON);
-		cfg |= GSC_OUT_PATH_MEMORY;
-		gsc_write(cfg, GSC_OUT_CON);
-		break;
-	case IPP_CMD_OUTPUT:
-		/* src dma memory */
-		cfg = gsc_read(GSC_IN_CON);
-		cfg &= ~(GSC_IN_PATH_MASK | GSC_IN_LOCAL_SEL_MASK);
-		cfg |= GSC_IN_PATH_MEMORY;
-		gsc_write(cfg, GSC_IN_CON);
-
-		/* dst local path */
-		cfg = gsc_read(GSC_OUT_CON);
-		cfg |= GSC_OUT_PATH_MEMORY;
-		gsc_write(cfg, GSC_OUT_CON);
-		break;
-	default:
-		ret = -EINVAL;
-		dev_err(dev, "invalid operations.\n");
-		return ret;
-	}
+	/* src dma memory */
+	cfg = gsc_read(GSC_IN_CON);
+	cfg &= ~(GSC_IN_PATH_MASK | GSC_IN_LOCAL_SEL_MASK);
+	cfg |= GSC_IN_PATH_MEMORY;
+	gsc_write(cfg, GSC_IN_CON);
 
-	ret = gsc_set_prescaler(ctx, &ctx->sc,
-		&img_pos[EXYNOS_DRM_OPS_SRC],
-		&img_pos[EXYNOS_DRM_OPS_DST]);
-	if (ret) {
-		dev_err(dev, "failed to set prescaler.\n");
-		return ret;
-	}
+	/* dst dma memory */
+	cfg = gsc_read(GSC_OUT_CON);
+	cfg |= GSC_OUT_PATH_MEMORY;
+	gsc_write(cfg, GSC_OUT_CON);
 
 	gsc_set_scaler(ctx, &ctx->sc);
 
 	cfg = gsc_read(GSC_ENABLE);
 	cfg |= GSC_ENABLE_ON;
 	gsc_write(cfg, GSC_ENABLE);
+}
+
+static int gsc_commit(struct exynos_drm_ipp *ipp,
+			  struct exynos_drm_ipp_task *task)
+{
+	struct gsc_context *ctx = container_of(ipp, struct gsc_context, ipp);
+	int ret;
+
+	pm_runtime_get_sync(ctx->dev);
+	ctx->task = task;
+
+	ret = gsc_reset(ctx);
+	if (ret) {
+		pm_runtime_put_autosuspend(ctx->dev);
+		ctx->task = NULL;
+		return ret;
+	}
+
+	gsc_src_set_fmt(ctx, task->src.buf.fourcc);
+	gsc_src_set_transf(ctx, task->transform.rotation);
+	gsc_src_set_size(ctx, &task->src);
+	gsc_src_set_addr(ctx, 0, &task->src);
+	gsc_dst_set_fmt(ctx, task->dst.buf.fourcc);
+	gsc_dst_set_size(ctx, &task->dst);
+	gsc_dst_set_addr(ctx, 0, &task->dst);
+	gsc_set_prescaler(ctx, &ctx->sc, &task->src.rect, &task->dst.rect);
+	gsc_start(ctx);
 
 	return 0;
 }
 
-static void gsc_ippdrv_stop(struct device *dev, enum drm_exynos_ipp_cmd cmd)
+static void gsc_abort(struct exynos_drm_ipp *ipp,
+			  struct exynos_drm_ipp_task *task)
 {
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct drm_exynos_ipp_set_wb set_wb = {0, 0};
-	u32 cfg;
+	struct gsc_context *ctx =
+			container_of(ipp, struct gsc_context, ipp);
 
-	DRM_DEBUG_KMS("cmd[%d]\n", cmd);
+	gsc_reset(ctx);
+	if (ctx->task) {
+		struct exynos_drm_ipp_task *task = ctx->task;
 
-	switch (cmd) {
-	case IPP_CMD_M2M:
-		/* bypass */
-		break;
-	case IPP_CMD_WB:
-		gsc_set_gscblk_fimd_wb(ctx, set_wb.enable);
-		exynos_drm_ippnb_send_event(IPP_SET_WRITEBACK, (void *)&set_wb);
-		break;
-	case IPP_CMD_OUTPUT:
-	default:
-		dev_err(dev, "invalid operations.\n");
-		break;
+		ctx->task = NULL;
+		pm_runtime_mark_last_busy(ctx->dev);
+		pm_runtime_put_autosuspend(ctx->dev);
+		exynos_drm_ipp_task_done(task, -EIO);
 	}
+}
 
-	gsc_handle_irq(ctx, false, false, true);
+static struct exynos_drm_ipp_funcs ipp_funcs = {
+	.commit = gsc_commit,
+	.abort = gsc_abort,
+};
 
-	/* reset sequence */
-	gsc_write(0xff, GSC_OUT_BASE_ADDR_Y_MASK);
-	gsc_write(0xff, GSC_OUT_BASE_ADDR_CB_MASK);
-	gsc_write(0xff, GSC_OUT_BASE_ADDR_CR_MASK);
+static int gsc_bind(struct device *dev, struct device *master, void *data)
+{
+	struct gsc_context *ctx = dev_get_drvdata(dev);
+	struct drm_device *drm_dev = data;
+	struct exynos_drm_ipp *ipp = &ctx->ipp;
 
-	cfg = gsc_read(GSC_ENABLE);
-	cfg &= ~GSC_ENABLE_ON;
-	gsc_write(cfg, GSC_ENABLE);
+	ctx->drm_dev = drm_dev;
+	drm_iommu_attach_device(drm_dev, dev);
+
+	exynos_drm_ipp_register(drm_dev, ipp, &ipp_funcs,
+			DRM_EXYNOS_IPP_CAP_CROP | DRM_EXYNOS_IPP_CAP_ROTATE |
+			DRM_EXYNOS_IPP_CAP_SCALE | DRM_EXYNOS_IPP_CAP_CONVERT,
+			ctx->formats, ctx->num_formats, "gsc");
+
+	dev_info(dev, "The exynos gscaler has been probed successfully\n");
+
+	return 0;
+}
+
+static void gsc_unbind(struct device *dev, struct device *master,
+			void *data)
+{
+	struct gsc_context *ctx = dev_get_drvdata(dev);
+	struct drm_device *drm_dev = data;
+	struct exynos_drm_ipp *ipp = &ctx->ipp;
+
+	exynos_drm_ipp_unregister(drm_dev, ipp);
+	drm_iommu_detach_device(drm_dev, dev);
 }
 
+static const struct component_ops gsc_component_ops = {
+	.bind	= gsc_bind,
+	.unbind = gsc_unbind,
+};
+
+static const unsigned int gsc_formats[] = {
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XRGB8888, DRM_FORMAT_RGB565, DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_NV12, DRM_FORMAT_NV16, DRM_FORMAT_NV21, DRM_FORMAT_NV61,
+	DRM_FORMAT_UYVY, DRM_FORMAT_VYUY, DRM_FORMAT_YUYV, DRM_FORMAT_YVYU,
+	DRM_FORMAT_YUV420, DRM_FORMAT_YVU420, DRM_FORMAT_YUV422,
+};
+
 static int gsc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	struct gsc_driverdata *driver_data;
+	struct exynos_drm_ipp_formats *formats;
 	struct gsc_context *ctx;
 	struct resource *res;
-	struct exynos_drm_ippdrv *ippdrv;
-	int ret;
+	int ret, i;
 
 	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
 		return -ENOMEM;
 
-	if (dev->of_node) {
-		ctx->sysreg = syscon_regmap_lookup_by_phandle(dev->of_node,
-							"samsung,sysreg");
-		if (IS_ERR(ctx->sysreg)) {
-			dev_warn(dev, "failed to get system register.\n");
-			ctx->sysreg = NULL;
-		}
+	formats = devm_kzalloc(dev, sizeof(*formats) *
+			       (ARRAY_SIZE(gsc_formats)), GFP_KERNEL);
+	if (!formats)
+		return -ENOMEM;
+
+	driver_data = (struct gsc_driverdata *)of_device_get_match_data(dev);
+	ctx->dev = dev;
+	ctx->num_clocks = driver_data->num_clocks;
+	ctx->clk_names = driver_data->clk_names;
+
+	for (i = 0; i < ARRAY_SIZE(gsc_formats); i++) {
+		formats[i].fourcc = gsc_formats[i];
+		formats[i].type = DRM_EXYNOS_IPP_FORMAT_SOURCE |
+				  DRM_EXYNOS_IPP_FORMAT_DESTINATION;
+		formats[i].limits = driver_data->limits;
+		formats[i].num_limits = driver_data->num_limits;
 	}
+	ctx->formats = formats;
+	ctx->num_formats = ARRAY_SIZE(gsc_formats);
 
 	/* clock control */
-	ctx->gsc_clk = devm_clk_get(dev, "gscl");
-	if (IS_ERR(ctx->gsc_clk)) {
-		dev_err(dev, "failed to get gsc clock.\n");
-		return PTR_ERR(ctx->gsc_clk);
+	for (i = 0; i < ctx->num_clocks; i++) {
+		ctx->clocks[i] = devm_clk_get(dev, ctx->clk_names[i]);
+		if (IS_ERR(ctx->clocks[i])) {
+			dev_err(dev, "failed to get clock: %s\n",
+				ctx->clk_names[i]);
+			return PTR_ERR(ctx->clocks[i]);
+		}
 	}
 
 	/* resource memory */
@@ -1699,8 +1246,8 @@ static int gsc_probe(struct platform_device *pdev)
 	}
 
 	ctx->irq = res->start;
-	ret = devm_request_threaded_irq(dev, ctx->irq, NULL, gsc_irq_handler,
-		IRQF_ONESHOT, "drm_gsc", ctx);
+	ret = devm_request_irq(dev, ctx->irq, gsc_irq_handler, 0,
+			       dev_name(dev), ctx);
 	if (ret < 0) {
 		dev_err(dev, "failed to request irq.\n");
 		return ret;
@@ -1709,38 +1256,22 @@ static int gsc_probe(struct platform_device *pdev)
 	/* context initailization */
 	ctx->id = pdev->id;
 
-	ippdrv = &ctx->ippdrv;
-	ippdrv->dev = dev;
-	ippdrv->ops[EXYNOS_DRM_OPS_SRC] = &gsc_src_ops;
-	ippdrv->ops[EXYNOS_DRM_OPS_DST] = &gsc_dst_ops;
-	ippdrv->check_property = gsc_ippdrv_check_property;
-	ippdrv->reset = gsc_ippdrv_reset;
-	ippdrv->start = gsc_ippdrv_start;
-	ippdrv->stop = gsc_ippdrv_stop;
-	ret = gsc_init_prop_list(ippdrv);
-	if (ret < 0) {
-		dev_err(dev, "failed to init property list.\n");
-		return ret;
-	}
-
-	DRM_DEBUG_KMS("id[%d]ippdrv[%pK]\n", ctx->id, ippdrv);
-
-	mutex_init(&ctx->lock);
 	platform_set_drvdata(pdev, ctx);
 
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_autosuspend_delay(dev, GSC_AUTOSUSPEND_DELAY);
 	pm_runtime_enable(dev);
 
-	ret = exynos_drm_ippdrv_register(ippdrv);
-	if (ret < 0) {
-		dev_err(dev, "failed to register drm gsc device.\n");
-		goto err_ippdrv_register;
-	}
+	ret = component_add(dev, &gsc_component_ops);
+	if (ret)
+		goto err_pm_dis;
 
 	dev_info(dev, "drm gsc registered successfully.\n");
 
 	return 0;
 
-err_ippdrv_register:
+err_pm_dis:
+	pm_runtime_dont_use_autosuspend(dev);
 	pm_runtime_disable(dev);
 	return ret;
 }
@@ -1748,13 +1279,8 @@ static int gsc_probe(struct platform_device *pdev)
 static int gsc_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct gsc_context *ctx = get_gsc_context(dev);
-	struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
 
-	exynos_drm_ippdrv_unregister(ippdrv);
-	mutex_destroy(&ctx->lock);
-
-	pm_runtime_set_suspended(dev);
+	pm_runtime_dont_use_autosuspend(dev);
 	pm_runtime_disable(dev);
 
 	return 0;
@@ -1763,19 +1289,32 @@ static int gsc_remove(struct platform_device *pdev)
 static int __maybe_unused gsc_runtime_suspend(struct device *dev)
 {
 	struct gsc_context *ctx = get_gsc_context(dev);
+	int i;
 
 	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
 
-	return  gsc_clk_ctrl(ctx, false);
+	for (i = ctx->num_clocks - 1; i >= 0; i--)
+		clk_disable_unprepare(ctx->clocks[i]);
+
+	return 0;
 }
 
 static int __maybe_unused gsc_runtime_resume(struct device *dev)
 {
 	struct gsc_context *ctx = get_gsc_context(dev);
+	int i, ret;
 
 	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
 
-	return  gsc_clk_ctrl(ctx, true);
+	for (i = 0; i < ctx->num_clocks; i++) {
+		ret = clk_prepare_enable(ctx->clocks[i]);
+		if (ret) {
+			while (--i > 0)
+				clk_disable_unprepare(ctx->clocks[i]);
+			return ret;
+		}
+	}
+	return 0;
 }
 
 static const struct dev_pm_ops gsc_pm_ops = {
@@ -1784,9 +1323,66 @@ static const struct dev_pm_ops gsc_pm_ops = {
 	SET_RUNTIME_PM_OPS(gsc_runtime_suspend, gsc_runtime_resume, NULL)
 };
 
+static const struct drm_exynos_ipp_limit gsc_5250_limits[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 32, 4800, 8 }, .v = { 16, 3344, 8 }) },
+	{ IPP_SIZE_LIMIT(AREA, .h = { 16, 4800, 2 }, .v = { 8, 3344, 2 }) },
+	{ IPP_SIZE_LIMIT(ROTATED, .h = { 32, 2048 }, .v = { 16, 2048 }) },
+	{ IPP_SCALE_LIMIT(.h = { (1 << 16) / 16, (1 << 16) * 8 },
+			  .v = { (1 << 16) / 16, (1 << 16) * 8 }) },
+};
+
+static const struct drm_exynos_ipp_limit gsc_5420_limits[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 32, 4800, 8 }, .v = { 16, 3344, 8 }) },
+	{ IPP_SIZE_LIMIT(AREA, .h = { 16, 4800, 2 }, .v = { 8, 3344, 2 }) },
+	{ IPP_SIZE_LIMIT(ROTATED, .h = { 16, 2016 }, .v = { 8, 2016 }) },
+	{ IPP_SCALE_LIMIT(.h = { (1 << 16) / 16, (1 << 16) * 8 },
+			  .v = { (1 << 16) / 16, (1 << 16) * 8 }) },
+};
+
+static const struct drm_exynos_ipp_limit gsc_5433_limits[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 32, 8191, 2 }, .v = { 16, 8191, 2 }) },
+	{ IPP_SIZE_LIMIT(AREA, .h = { 16, 4800, 1 }, .v = { 8, 3344, 1 }) },
+	{ IPP_SIZE_LIMIT(ROTATED, .h = { 32, 2047 }, .v = { 8, 8191 }) },
+	{ IPP_SCALE_LIMIT(.h = { (1 << 16) / 16, (1 << 16) * 8 },
+			  .v = { (1 << 16) / 16, (1 << 16) * 8 }) },
+};
+
+static struct gsc_driverdata gsc_exynos5250_drvdata = {
+	.clk_names = {"gscl"},
+	.num_clocks = 1,
+	.limits = gsc_5250_limits,
+	.num_limits = ARRAY_SIZE(gsc_5250_limits),
+};
+
+static struct gsc_driverdata gsc_exynos5420_drvdata = {
+	.clk_names = {"gscl"},
+	.num_clocks = 1,
+	.limits = gsc_5420_limits,
+	.num_limits = ARRAY_SIZE(gsc_5420_limits),
+};
+
+static struct gsc_driverdata gsc_exynos5433_drvdata = {
+	.clk_names = {"pclk", "aclk", "aclk_xiu", "aclk_gsclbend"},
+	.num_clocks = 4,
+	.limits = gsc_5433_limits,
+	.num_limits = ARRAY_SIZE(gsc_5433_limits),
+};
+
 static const struct of_device_id exynos_drm_gsc_of_match[] = {
-	{ .compatible = "samsung,exynos5-gsc" },
-	{ },
+	{
+		.compatible = "samsung,exynos5-gsc",
+		.data = &gsc_exynos5250_drvdata,
+	}, {
+		.compatible = "samsung,exynos5250-gsc",
+		.data = &gsc_exynos5250_drvdata,
+	}, {
+		.compatible = "samsung,exynos5420-gsc",
+		.data = &gsc_exynos5420_drvdata,
+	}, {
+		.compatible = "samsung,exynos5433-gsc",
+		.data = &gsc_exynos5433_drvdata,
+	}, {
+	},
 };
 MODULE_DEVICE_TABLE(of, exynos_drm_gsc_of_match);
 
@@ -1800,4 +1396,3 @@ struct platform_driver gsc_driver = {
 		.of_match_table = of_match_ptr(exynos_drm_gsc_of_match),
 	},
 };
-
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gsc.h b/drivers/gpu/drm/exynos/exynos_drm_gsc.h
deleted file mode 100644
index 29ec1c5efcf2..000000000000
--- a/drivers/gpu/drm/exynos/exynos_drm_gsc.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright (c) 2012 Samsung Electronics Co., Ltd.
- *
- * Authors:
- *	Eunchul Kim <chulspro.kim@samsung.com>
- *	Jinyoung Jeon <jy0.jeon@samsung.com>
- *	Sangmin Lee <lsmin.lee@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-#ifndef _EXYNOS_DRM_GSC_H_
-#define _EXYNOS_DRM_GSC_H_
-
-/*
- * TODO
- * FIMD output interface notifier callback.
- * Mixer output interface notifier callback.
- */
-
-#endif /* _EXYNOS_DRM_GSC_H_ */
diff --git a/drivers/gpu/drm/exynos/exynos_drm_ipp.c b/drivers/gpu/drm/exynos/exynos_drm_ipp.c
new file mode 100644
index 000000000000..26374e58c557
--- /dev/null
+++ b/drivers/gpu/drm/exynos/exynos_drm_ipp.c
@@ -0,0 +1,916 @@
+/*
+ * Copyright (C) 2017 Samsung Electronics Co.Ltd
+ * Authors:
+ *	Marek Szyprowski <m.szyprowski@samsung.com>
+ *
+ * Exynos DRM Image Post Processing (IPP) related functions
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ */
+
+
+#include <drm/drmP.h>
+#include <drm/drm_mode.h>
+#include <uapi/drm/exynos_drm.h>
+
+#include "exynos_drm_drv.h"
+#include "exynos_drm_gem.h"
+#include "exynos_drm_ipp.h"
+
+static int num_ipp;
+static LIST_HEAD(ipp_list);
+
+/**
+ * exynos_drm_ipp_register - Register a new picture processor hardware module
+ * @dev: DRM device
+ * @ipp: ipp module to init
+ * @funcs: callbacks for the new ipp object
+ * @caps: bitmask of ipp capabilities (%DRM_EXYNOS_IPP_CAP_*)
+ * @formats: array of supported formats
+ * @num_formats: size of the supported formats array
+ * @name: name (for debugging purposes)
+ *
+ * Initializes a ipp module.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int exynos_drm_ipp_register(struct drm_device *dev, struct exynos_drm_ipp *ipp,
+		const struct exynos_drm_ipp_funcs *funcs, unsigned int caps,
+		const struct exynos_drm_ipp_formats *formats,
+		unsigned int num_formats, const char *name)
+{
+	WARN_ON(!ipp);
+	WARN_ON(!funcs);
+	WARN_ON(!formats);
+	WARN_ON(!num_formats);
+
+	spin_lock_init(&ipp->lock);
+	INIT_LIST_HEAD(&ipp->todo_list);
+	init_waitqueue_head(&ipp->done_wq);
+	ipp->dev = dev;
+	ipp->funcs = funcs;
+	ipp->capabilities = caps;
+	ipp->name = name;
+	ipp->formats = formats;
+	ipp->num_formats = num_formats;
+
+	/* ipp_list modification is serialized by component framework */
+	list_add_tail(&ipp->head, &ipp_list);
+	ipp->id = num_ipp++;
+
+	DRM_DEBUG_DRIVER("Registered ipp %d\n", ipp->id);
+
+	return 0;
+}
+
+/**
+ * exynos_drm_ipp_unregister - Unregister the picture processor module
+ * @dev: DRM device
+ * @ipp: ipp module
+ */
+void exynos_drm_ipp_unregister(struct drm_device *dev,
+			       struct exynos_drm_ipp *ipp)
+{
+	WARN_ON(ipp->task);
+	WARN_ON(!list_empty(&ipp->todo_list));
+	list_del(&ipp->head);
+}
+
+/**
+ * exynos_drm_ipp_ioctl_get_res_ioctl - enumerate all ipp modules
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * Construct a list of ipp ids.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int exynos_drm_ipp_get_res_ioctl(struct drm_device *dev, void *data,
+				 struct drm_file *file_priv)
+{
+	struct drm_exynos_ioctl_ipp_get_res *resp = data;
+	struct exynos_drm_ipp *ipp;
+	uint32_t __user *ipp_ptr = (uint32_t __user *)
+						(unsigned long)resp->ipp_id_ptr;
+	unsigned int count = num_ipp, copied = 0;
+
+	/*
+	 * This ioctl is called twice, once to determine how much space is
+	 * needed, and the 2nd time to fill it.
+	 */
+	if (count && resp->count_ipps >= count) {
+		list_for_each_entry(ipp, &ipp_list, head) {
+			if (put_user(ipp->id, ipp_ptr + copied))
+				return -EFAULT;
+			copied++;
+		}
+	}
+	resp->count_ipps = count;
+
+	return 0;
+}
+
+static inline struct exynos_drm_ipp *__ipp_get(uint32_t id)
+{
+	struct exynos_drm_ipp *ipp;
+
+	list_for_each_entry(ipp, &ipp_list, head)
+		if (ipp->id == id)
+			return ipp;
+	return NULL;
+}
+
+/**
+ * exynos_drm_ipp_ioctl_get_caps - get ipp module capabilities and formats
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * Construct a structure describing ipp module capabilities.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int exynos_drm_ipp_get_caps_ioctl(struct drm_device *dev, void *data,
+				  struct drm_file *file_priv)
+{
+	struct drm_exynos_ioctl_ipp_get_caps *resp = data;
+	void __user *ptr = (void __user *)(unsigned long)resp->formats_ptr;
+	struct exynos_drm_ipp *ipp;
+	int i;
+
+	ipp = __ipp_get(resp->ipp_id);
+	if (!ipp)
+		return -ENOENT;
+
+	resp->ipp_id = ipp->id;
+	resp->capabilities = ipp->capabilities;
+
+	/*
+	 * This ioctl is called twice, once to determine how much space is
+	 * needed, and the 2nd time to fill it.
+	 */
+	if (resp->formats_count >= ipp->num_formats) {
+		for (i = 0; i < ipp->num_formats; i++) {
+			struct drm_exynos_ipp_format tmp = {
+				.fourcc = ipp->formats[i].fourcc,
+				.type = ipp->formats[i].type,
+				.modifier = ipp->formats[i].modifier,
+			};
+
+			if (copy_to_user(ptr, &tmp, sizeof(tmp)))
+				return -EFAULT;
+			ptr += sizeof(tmp);
+		}
+	}
+	resp->formats_count = ipp->num_formats;
+
+	return 0;
+}
+
+static inline const struct exynos_drm_ipp_formats *__ipp_format_get(
+				struct exynos_drm_ipp *ipp, uint32_t fourcc,
+				uint64_t mod, unsigned int type)
+{
+	int i;
+
+	for (i = 0; i < ipp->num_formats; i++) {
+		if ((ipp->formats[i].type & type) &&
+		    ipp->formats[i].fourcc == fourcc &&
+		    ipp->formats[i].modifier == mod)
+			return &ipp->formats[i];
+	}
+	return NULL;
+}
+
+/**
+ * exynos_drm_ipp_get_limits_ioctl - get ipp module limits
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * Construct a structure describing ipp module limitations for provided
+ * picture format.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int exynos_drm_ipp_get_limits_ioctl(struct drm_device *dev, void *data,
+				    struct drm_file *file_priv)
+{
+	struct drm_exynos_ioctl_ipp_get_limits *resp = data;
+	void __user *ptr = (void __user *)(unsigned long)resp->limits_ptr;
+	const struct exynos_drm_ipp_formats *format;
+	struct exynos_drm_ipp *ipp;
+
+	if (resp->type != DRM_EXYNOS_IPP_FORMAT_SOURCE &&
+	    resp->type != DRM_EXYNOS_IPP_FORMAT_DESTINATION)
+		return -EINVAL;
+
+	ipp = __ipp_get(resp->ipp_id);
+	if (!ipp)
+		return -ENOENT;
+
+	format = __ipp_format_get(ipp, resp->fourcc, resp->modifier,
+				  resp->type);
+	if (!format)
+		return -EINVAL;
+
+	/*
+	 * This ioctl is called twice, once to determine how much space is
+	 * needed, and the 2nd time to fill it.
+	 */
+	if (format->num_limits && resp->limits_count >= format->num_limits)
+		if (copy_to_user((void __user *)ptr, format->limits,
+				 sizeof(*format->limits) * format->num_limits))
+			return -EFAULT;
+	resp->limits_count = format->num_limits;
+
+	return 0;
+}
+
+struct drm_pending_exynos_ipp_event {
+	struct drm_pending_event base;
+	struct drm_exynos_ipp_event event;
+};
+
+static inline struct exynos_drm_ipp_task *
+			exynos_drm_ipp_task_alloc(struct exynos_drm_ipp *ipp)
+{
+	struct exynos_drm_ipp_task *task;
+
+	task = kzalloc(sizeof(*task), GFP_KERNEL);
+	if (!task)
+		return NULL;
+
+	task->dev = ipp->dev;
+	task->ipp = ipp;
+
+	/* some defaults */
+	task->src.rect.w = task->dst.rect.w = UINT_MAX;
+	task->src.rect.h = task->dst.rect.h = UINT_MAX;
+	task->transform.rotation = DRM_MODE_ROTATE_0;
+
+	DRM_DEBUG_DRIVER("Allocated task %pK\n", task);
+
+	return task;
+}
+
+static const struct exynos_drm_param_map {
+	unsigned int id;
+	unsigned int size;
+	unsigned int offset;
+} exynos_drm_ipp_params_maps[] = {
+	{
+		DRM_EXYNOS_IPP_TASK_BUFFER | DRM_EXYNOS_IPP_TASK_TYPE_SOURCE,
+		sizeof(struct drm_exynos_ipp_task_buffer),
+		offsetof(struct exynos_drm_ipp_task, src.buf),
+	}, {
+		DRM_EXYNOS_IPP_TASK_BUFFER |
+			DRM_EXYNOS_IPP_TASK_TYPE_DESTINATION,
+		sizeof(struct drm_exynos_ipp_task_buffer),
+		offsetof(struct exynos_drm_ipp_task, dst.buf),
+	}, {
+		DRM_EXYNOS_IPP_TASK_RECTANGLE | DRM_EXYNOS_IPP_TASK_TYPE_SOURCE,
+		sizeof(struct drm_exynos_ipp_task_rect),
+		offsetof(struct exynos_drm_ipp_task, src.rect),
+	}, {
+		DRM_EXYNOS_IPP_TASK_RECTANGLE |
+			DRM_EXYNOS_IPP_TASK_TYPE_DESTINATION,
+		sizeof(struct drm_exynos_ipp_task_rect),
+		offsetof(struct exynos_drm_ipp_task, dst.rect),
+	}, {
+		DRM_EXYNOS_IPP_TASK_TRANSFORM,
+		sizeof(struct drm_exynos_ipp_task_transform),
+		offsetof(struct exynos_drm_ipp_task, transform),
+	}, {
+		DRM_EXYNOS_IPP_TASK_ALPHA,
+		sizeof(struct drm_exynos_ipp_task_alpha),
+		offsetof(struct exynos_drm_ipp_task, alpha),
+	},
+};
+
+static int exynos_drm_ipp_task_set(struct exynos_drm_ipp_task *task,
+				   struct drm_exynos_ioctl_ipp_commit *arg)
+{
+	const struct exynos_drm_param_map *map = exynos_drm_ipp_params_maps;
+	void __user *params = (void __user *)(unsigned long)arg->params_ptr;
+	unsigned int size = arg->params_size;
+	uint32_t id;
+	int i;
+
+	while (size) {
+		if (get_user(id, (uint32_t __user *)params))
+			return -EFAULT;
+
+		for (i = 0; i < ARRAY_SIZE(exynos_drm_ipp_params_maps); i++)
+			if (map[i].id == id)
+				break;
+		if (i == ARRAY_SIZE(exynos_drm_ipp_params_maps) ||
+		    map[i].size > size)
+			return -EINVAL;
+
+		if (copy_from_user((void *)task + map[i].offset, params,
+				   map[i].size))
+			return -EFAULT;
+
+		params += map[i].size;
+		size -= map[i].size;
+	}
+
+	DRM_DEBUG_DRIVER("Got task %pK configuration from userspace\n", task);
+	return 0;
+}
+
+static int exynos_drm_ipp_task_setup_buffer(struct exynos_drm_ipp_buffer *buf,
+					    struct drm_file *filp)
+{
+	int ret = 0;
+	int i;
+
+	/* basic checks */
+	if (buf->buf.width == 0 || buf->buf.height == 0)
+		return -EINVAL;
+	buf->format = drm_format_info(buf->buf.fourcc);
+	for (i = 0; i < buf->format->num_planes; i++) {
+		unsigned int width = (i == 0) ? buf->buf.width :
+			     DIV_ROUND_UP(buf->buf.width, buf->format->hsub);
+
+		if (buf->buf.pitch[i] == 0)
+			buf->buf.pitch[i] = width * buf->format->cpp[i];
+		if (buf->buf.pitch[i] < width * buf->format->cpp[i])
+			return -EINVAL;
+		if (!buf->buf.gem_id[i])
+			return -ENOENT;
+	}
+
+	/* pitch for additional planes must match */
+	if (buf->format->num_planes > 2 &&
+	    buf->buf.pitch[1] != buf->buf.pitch[2])
+		return -EINVAL;
+
+	/* get GEM buffers and check their size */
+	for (i = 0; i < buf->format->num_planes; i++) {
+		unsigned int height = (i == 0) ? buf->buf.height :
+			     DIV_ROUND_UP(buf->buf.height, buf->format->vsub);
+		unsigned long size = height * buf->buf.pitch[i];
+		struct drm_gem_object *obj = drm_gem_object_lookup(filp,
+							    buf->buf.gem_id[i]);
+		if (!obj) {
+			ret = -ENOENT;
+			goto gem_free;
+		}
+		buf->exynos_gem[i] = to_exynos_gem(obj);
+
+		if (size + buf->buf.offset[i] > buf->exynos_gem[i]->size) {
+			i++;
+			ret = -EINVAL;
+			goto gem_free;
+		}
+		buf->dma_addr[i] = buf->exynos_gem[i]->dma_addr +
+				   buf->buf.offset[i];
+	}
+
+	return 0;
+gem_free:
+	while (i--) {
+		drm_gem_object_put_unlocked(&buf->exynos_gem[i]->base);
+		buf->exynos_gem[i] = NULL;
+	}
+	return ret;
+}
+
+static void exynos_drm_ipp_task_release_buf(struct exynos_drm_ipp_buffer *buf)
+{
+	int i;
+
+	if (!buf->exynos_gem[0])
+		return;
+	for (i = 0; i < buf->format->num_planes; i++)
+		drm_gem_object_put_unlocked(&buf->exynos_gem[i]->base);
+}
+
+static void exynos_drm_ipp_task_free(struct exynos_drm_ipp *ipp,
+				 struct exynos_drm_ipp_task *task)
+{
+	DRM_DEBUG_DRIVER("Freeing task %pK\n", task);
+
+	exynos_drm_ipp_task_release_buf(&task->src);
+	exynos_drm_ipp_task_release_buf(&task->dst);
+	if (task->event)
+		drm_event_cancel_free(ipp->dev, &task->event->base);
+	kfree(task);
+}
+
+struct drm_ipp_limit {
+	struct drm_exynos_ipp_limit_val h;
+	struct drm_exynos_ipp_limit_val v;
+};
+
+enum drm_ipp_size_id {
+	IPP_LIMIT_BUFFER, IPP_LIMIT_AREA, IPP_LIMIT_ROTATED, IPP_LIMIT_MAX
+};
+
+static const enum drm_ipp_size_id limit_id_fallback[IPP_LIMIT_MAX][4] = {
+	[IPP_LIMIT_BUFFER]  = { DRM_EXYNOS_IPP_LIMIT_SIZE_BUFFER },
+	[IPP_LIMIT_AREA]    = { DRM_EXYNOS_IPP_LIMIT_SIZE_AREA,
+				DRM_EXYNOS_IPP_LIMIT_SIZE_BUFFER },
+	[IPP_LIMIT_ROTATED] = { DRM_EXYNOS_IPP_LIMIT_SIZE_ROTATED,
+				DRM_EXYNOS_IPP_LIMIT_SIZE_AREA,
+				DRM_EXYNOS_IPP_LIMIT_SIZE_BUFFER },
+};
+
+static inline void __limit_set_val(unsigned int *ptr, unsigned int val)
+{
+	if (!*ptr)
+		*ptr = val;
+}
+
+static void __get_size_limit(const struct drm_exynos_ipp_limit *limits,
+			     unsigned int num_limits, enum drm_ipp_size_id id,
+			     struct drm_ipp_limit *res)
+{
+	const struct drm_exynos_ipp_limit *l = limits;
+	int i = 0;
+
+	memset(res, 0, sizeof(*res));
+	for (i = 0; limit_id_fallback[id][i]; i++)
+		for (l = limits; l - limits < num_limits; l++) {
+			if (((l->type & DRM_EXYNOS_IPP_LIMIT_TYPE_MASK) !=
+			      DRM_EXYNOS_IPP_LIMIT_TYPE_SIZE) ||
+			    ((l->type & DRM_EXYNOS_IPP_LIMIT_SIZE_MASK) !=
+						     limit_id_fallback[id][i]))
+				continue;
+			__limit_set_val(&res->h.min, l->h.min);
+			__limit_set_val(&res->h.max, l->h.max);
+			__limit_set_val(&res->h.align, l->h.align);
+			__limit_set_val(&res->v.min, l->v.min);
+			__limit_set_val(&res->v.max, l->v.max);
+			__limit_set_val(&res->v.align, l->v.align);
+		}
+}
+
+static inline bool __align_check(unsigned int val, unsigned int align)
+{
+	if (align && (val & (align - 1))) {
+		DRM_DEBUG_DRIVER("Value %d exceeds HW limits (align %d)\n",
+				 val, align);
+		return false;
+	}
+	return true;
+}
+
+static inline bool __size_limit_check(unsigned int val,
+				 struct drm_exynos_ipp_limit_val *l)
+{
+	if ((l->min && val < l->min) || (l->max && val > l->max)) {
+		DRM_DEBUG_DRIVER("Value %d exceeds HW limits (min %d, max %d)\n",
+				 val, l->min, l->max);
+		return false;
+	}
+	return __align_check(val, l->align);
+}
+
+static int exynos_drm_ipp_check_size_limits(struct exynos_drm_ipp_buffer *buf,
+	const struct drm_exynos_ipp_limit *limits, unsigned int num_limits,
+	bool rotate, bool swap)
+{
+	enum drm_ipp_size_id id = rotate ? IPP_LIMIT_ROTATED : IPP_LIMIT_AREA;
+	struct drm_ipp_limit l;
+	struct drm_exynos_ipp_limit_val *lh = &l.h, *lv = &l.v;
+
+	if (!limits)
+		return 0;
+
+	__get_size_limit(limits, num_limits, IPP_LIMIT_BUFFER, &l);
+	if (!__size_limit_check(buf->buf.width, &l.h) ||
+	    !__size_limit_check(buf->buf.height, &l.v))
+		return -EINVAL;
+
+	if (swap) {
+		lv = &l.h;
+		lh = &l.v;
+	}
+	__get_size_limit(limits, num_limits, id, &l);
+	if (!__size_limit_check(buf->rect.w, lh) ||
+	    !__align_check(buf->rect.x, lh->align) ||
+	    !__size_limit_check(buf->rect.h, lv) ||
+	    !__align_check(buf->rect.y, lv->align))
+		return -EINVAL;
+
+	return 0;
+}
+
+static inline bool __scale_limit_check(unsigned int src, unsigned int dst,
+				       unsigned int min, unsigned int max)
+{
+	if ((max && (dst << 16) > src * max) ||
+	    (min && (dst << 16) < src * min)) {
+		DRM_DEBUG_DRIVER("Scale from %d to %d exceeds HW limits (ratio min %d.%05d, max %d.%05d)\n",
+			 src, dst,
+			 min >> 16, 100000 * (min & 0xffff) / (1 << 16),
+			 max >> 16, 100000 * (max & 0xffff) / (1 << 16));
+		return false;
+	}
+	return true;
+}
+
+static int exynos_drm_ipp_check_scale_limits(
+				struct drm_exynos_ipp_task_rect *src,
+				struct drm_exynos_ipp_task_rect *dst,
+				const struct drm_exynos_ipp_limit *limits,
+				unsigned int num_limits, bool swap)
+{
+	const struct drm_exynos_ipp_limit_val *lh, *lv;
+	int dw, dh;
+
+	for (; num_limits; limits++, num_limits--)
+		if ((limits->type & DRM_EXYNOS_IPP_LIMIT_TYPE_MASK) ==
+		    DRM_EXYNOS_IPP_LIMIT_TYPE_SCALE)
+			break;
+	if (!num_limits)
+		return 0;
+
+	lh = (!swap) ? &limits->h : &limits->v;
+	lv = (!swap) ? &limits->v : &limits->h;
+	dw = (!swap) ? dst->w : dst->h;
+	dh = (!swap) ? dst->h : dst->w;
+
+	if (!__scale_limit_check(src->w, dw, lh->min, lh->max) ||
+	    !__scale_limit_check(src->h, dh, lv->min, lv->max))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int exynos_drm_ipp_task_check(struct exynos_drm_ipp_task *task)
+{
+	struct exynos_drm_ipp *ipp = task->ipp;
+	const struct exynos_drm_ipp_formats *src_fmt, *dst_fmt;
+	struct exynos_drm_ipp_buffer *src = &task->src, *dst = &task->dst;
+	unsigned int rotation = task->transform.rotation;
+	int ret = 0;
+	bool swap = drm_rotation_90_or_270(rotation);
+	bool rotate = (rotation != DRM_MODE_ROTATE_0);
+	bool scale = false;
+
+	DRM_DEBUG_DRIVER("Checking task %pK\n", task);
+
+	if (src->rect.w == UINT_MAX)
+		src->rect.w = src->buf.width;
+	if (src->rect.h == UINT_MAX)
+		src->rect.h = src->buf.height;
+	if (dst->rect.w == UINT_MAX)
+		dst->rect.w = dst->buf.width;
+	if (dst->rect.h == UINT_MAX)
+		dst->rect.h = dst->buf.height;
+
+	if (src->rect.x + src->rect.w > (src->buf.width) ||
+	    src->rect.y + src->rect.h > (src->buf.height) ||
+	    dst->rect.x + dst->rect.w > (dst->buf.width) ||
+	    dst->rect.y + dst->rect.h > (dst->buf.height)) {
+		DRM_DEBUG_DRIVER("Task %pK: defined area is outside provided buffers\n",
+				 task);
+		return -EINVAL;
+	}
+
+	if ((!swap && (src->rect.w != dst->rect.w ||
+		       src->rect.h != dst->rect.h)) ||
+	    (swap && (src->rect.w != dst->rect.h ||
+		      src->rect.h != dst->rect.w)))
+		scale = true;
+
+	if ((!(ipp->capabilities & DRM_EXYNOS_IPP_CAP_CROP) &&
+	     (src->rect.x || src->rect.y || dst->rect.x || dst->rect.y)) ||
+	    (!(ipp->capabilities & DRM_EXYNOS_IPP_CAP_ROTATE) && rotate) ||
+	    (!(ipp->capabilities & DRM_EXYNOS_IPP_CAP_SCALE) && scale) ||
+	    (!(ipp->capabilities & DRM_EXYNOS_IPP_CAP_CONVERT) &&
+	     src->buf.fourcc != dst->buf.fourcc)) {
+		DRM_DEBUG_DRIVER("Task %pK: hw capabilities exceeded\n", task);
+		return -EINVAL;
+	}
+
+	src_fmt = __ipp_format_get(ipp, src->buf.fourcc, src->buf.modifier,
+				   DRM_EXYNOS_IPP_FORMAT_SOURCE);
+	if (!src_fmt) {
+		DRM_DEBUG_DRIVER("Task %pK: src format not supported\n", task);
+		return -EINVAL;
+	}
+	ret = exynos_drm_ipp_check_size_limits(src, src_fmt->limits,
+					       src_fmt->num_limits,
+					       rotate, false);
+	if (ret)
+		return ret;
+	ret = exynos_drm_ipp_check_scale_limits(&src->rect, &dst->rect,
+						src_fmt->limits,
+						src_fmt->num_limits, swap);
+	if (ret)
+		return ret;
+
+	dst_fmt = __ipp_format_get(ipp, dst->buf.fourcc, dst->buf.modifier,
+				   DRM_EXYNOS_IPP_FORMAT_DESTINATION);
+	if (!dst_fmt) {
+		DRM_DEBUG_DRIVER("Task %pK: dst format not supported\n", task);
+		return -EINVAL;
+	}
+	ret = exynos_drm_ipp_check_size_limits(dst, dst_fmt->limits,
+					       dst_fmt->num_limits,
+					       false, swap);
+	if (ret)
+		return ret;
+	ret = exynos_drm_ipp_check_scale_limits(&src->rect, &dst->rect,
+						dst_fmt->limits,
+						dst_fmt->num_limits, swap);
+	if (ret)
+		return ret;
+
+	DRM_DEBUG_DRIVER("Task %pK: all checks done.\n", task);
+
+	return ret;
+}
+
+static int exynos_drm_ipp_task_setup_buffers(struct exynos_drm_ipp_task *task,
+				     struct drm_file *filp)
+{
+	struct exynos_drm_ipp_buffer *src = &task->src, *dst = &task->dst;
+	int ret = 0;
+
+	DRM_DEBUG_DRIVER("Setting buffer for task %pK\n", task);
+
+	ret = exynos_drm_ipp_task_setup_buffer(src, filp);
+	if (ret) {
+		DRM_DEBUG_DRIVER("Task %pK: src buffer setup failed\n", task);
+		return ret;
+	}
+	ret = exynos_drm_ipp_task_setup_buffer(dst, filp);
+	if (ret) {
+		DRM_DEBUG_DRIVER("Task %pK: dst buffer setup failed\n", task);
+		return ret;
+	}
+
+	DRM_DEBUG_DRIVER("Task %pK: buffers prepared.\n", task);
+
+	return ret;
+}
+
+
+static int exynos_drm_ipp_event_create(struct exynos_drm_ipp_task *task,
+				 struct drm_file *file_priv, uint64_t user_data)
+{
+	struct drm_pending_exynos_ipp_event *e = NULL;
+	int ret;
+
+	e = kzalloc(sizeof(*e), GFP_KERNEL);
+	if (!e)
+		return -ENOMEM;
+
+	e->event.base.type = DRM_EXYNOS_IPP_EVENT;
+	e->event.base.length = sizeof(e->event);
+	e->event.user_data = user_data;
+
+	ret = drm_event_reserve_init(task->dev, file_priv, &e->base,
+				     &e->event.base);
+	if (ret)
+		goto free;
+
+	task->event = e;
+	return 0;
+free:
+	kfree(e);
+	return ret;
+}
+
+static void exynos_drm_ipp_event_send(struct exynos_drm_ipp_task *task)
+{
+	struct timespec64 now;
+
+	ktime_get_ts64(&now);
+	task->event->event.tv_sec = now.tv_sec;
+	task->event->event.tv_usec = now.tv_nsec / NSEC_PER_USEC;
+	task->event->event.sequence = atomic_inc_return(&task->ipp->sequence);
+
+	drm_send_event(task->dev, &task->event->base);
+}
+
+static int exynos_drm_ipp_task_cleanup(struct exynos_drm_ipp_task *task)
+{
+	int ret = task->ret;
+
+	if (ret == 0 && task->event) {
+		exynos_drm_ipp_event_send(task);
+		/* ensure event won't be canceled on task free */
+		task->event = NULL;
+	}
+
+	exynos_drm_ipp_task_free(task->ipp, task);
+	return ret;
+}
+
+static void exynos_drm_ipp_cleanup_work(struct work_struct *work)
+{
+	struct exynos_drm_ipp_task *task = container_of(work,
+				      struct exynos_drm_ipp_task, cleanup_work);
+
+	exynos_drm_ipp_task_cleanup(task);
+}
+
+static void exynos_drm_ipp_next_task(struct exynos_drm_ipp *ipp);
+
+/**
+ * exynos_drm_ipp_task_done - finish given task and set return code
+ * @task: ipp task to finish
+ * @ret: error code or 0 if operation has been performed successfully
+ */
+void exynos_drm_ipp_task_done(struct exynos_drm_ipp_task *task, int ret)
+{
+	struct exynos_drm_ipp *ipp = task->ipp;
+	unsigned long flags;
+
+	DRM_DEBUG_DRIVER("ipp: %d, task %pK done: %d\n", ipp->id, task, ret);
+
+	spin_lock_irqsave(&ipp->lock, flags);
+	if (ipp->task == task)
+		ipp->task = NULL;
+	task->flags |= DRM_EXYNOS_IPP_TASK_DONE;
+	task->ret = ret;
+	spin_unlock_irqrestore(&ipp->lock, flags);
+
+	exynos_drm_ipp_next_task(ipp);
+	wake_up(&ipp->done_wq);
+
+	if (task->flags & DRM_EXYNOS_IPP_TASK_ASYNC) {
+		INIT_WORK(&task->cleanup_work, exynos_drm_ipp_cleanup_work);
+		schedule_work(&task->cleanup_work);
+	}
+}
+
+static void exynos_drm_ipp_next_task(struct exynos_drm_ipp *ipp)
+{
+	struct exynos_drm_ipp_task *task;
+	unsigned long flags;
+	int ret;
+
+	DRM_DEBUG_DRIVER("ipp: %d, try to run new task\n", ipp->id);
+
+	spin_lock_irqsave(&ipp->lock, flags);
+
+	if (ipp->task || list_empty(&ipp->todo_list)) {
+		spin_unlock_irqrestore(&ipp->lock, flags);
+		return;
+	}
+
+	task = list_first_entry(&ipp->todo_list, struct exynos_drm_ipp_task,
+				head);
+	list_del_init(&task->head);
+	ipp->task = task;
+
+	spin_unlock_irqrestore(&ipp->lock, flags);
+
+	DRM_DEBUG_DRIVER("ipp: %d, selected task %pK to run\n", ipp->id, task);
+
+	ret = ipp->funcs->commit(ipp, task);
+	if (ret)
+		exynos_drm_ipp_task_done(task, ret);
+}
+
+static void exynos_drm_ipp_schedule_task(struct exynos_drm_ipp *ipp,
+					 struct exynos_drm_ipp_task *task)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&ipp->lock, flags);
+	list_add(&task->head, &ipp->todo_list);
+	spin_unlock_irqrestore(&ipp->lock, flags);
+
+	exynos_drm_ipp_next_task(ipp);
+}
+
+static void exynos_drm_ipp_task_abort(struct exynos_drm_ipp *ipp,
+				      struct exynos_drm_ipp_task *task)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&ipp->lock, flags);
+	if (task->flags & DRM_EXYNOS_IPP_TASK_DONE) {
+		/* already completed task */
+		exynos_drm_ipp_task_cleanup(task);
+	} else if (ipp->task != task) {
+		/* task has not been scheduled for execution yet */
+		list_del_init(&task->head);
+		exynos_drm_ipp_task_cleanup(task);
+	} else {
+		/*
+		 * currently processed task, call abort() and perform
+		 * cleanup with async worker
+		 */
+		task->flags |= DRM_EXYNOS_IPP_TASK_ASYNC;
+		spin_unlock_irqrestore(&ipp->lock, flags);
+		if (ipp->funcs->abort)
+			ipp->funcs->abort(ipp, task);
+		return;
+	}
+	spin_unlock_irqrestore(&ipp->lock, flags);
+}
+
+/**
+ * exynos_drm_ipp_commit_ioctl - perform image processing operation
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * Construct a ipp task from the set of properties provided from the user
+ * and try to schedule it to framebuffer processor hardware.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int exynos_drm_ipp_commit_ioctl(struct drm_device *dev, void *data,
+				struct drm_file *file_priv)
+{
+	struct drm_exynos_ioctl_ipp_commit *arg = data;
+	struct exynos_drm_ipp *ipp;
+	struct exynos_drm_ipp_task *task;
+	int ret = 0;
+
+	if ((arg->flags & ~DRM_EXYNOS_IPP_FLAGS) || arg->reserved)
+		return -EINVAL;
+
+	/* can't test and expect an event at the same time */
+	if ((arg->flags & DRM_EXYNOS_IPP_FLAG_TEST_ONLY) &&
+			(arg->flags & DRM_EXYNOS_IPP_FLAG_EVENT))
+		return -EINVAL;
+
+	ipp = __ipp_get(arg->ipp_id);
+	if (!ipp)
+		return -ENOENT;
+
+	task = exynos_drm_ipp_task_alloc(ipp);
+	if (!task)
+		return -ENOMEM;
+
+	ret = exynos_drm_ipp_task_set(task, arg);
+	if (ret)
+		goto free;
+
+	ret = exynos_drm_ipp_task_check(task);
+	if (ret)
+		goto free;
+
+	ret = exynos_drm_ipp_task_setup_buffers(task, file_priv);
+	if (ret || arg->flags & DRM_EXYNOS_IPP_FLAG_TEST_ONLY)
+		goto free;
+
+	if (arg->flags & DRM_EXYNOS_IPP_FLAG_EVENT) {
+		ret = exynos_drm_ipp_event_create(task, file_priv,
+						 arg->user_data);
+		if (ret)
+			goto free;
+	}
+
+	/*
+	 * Queue task for processing on the hardware. task object will be
+	 * then freed after exynos_drm_ipp_task_done()
+	 */
+	if (arg->flags & DRM_EXYNOS_IPP_FLAG_NONBLOCK) {
+		DRM_DEBUG_DRIVER("ipp: %d, nonblocking processing task %pK\n",
+				 ipp->id, task);
+
+		task->flags |= DRM_EXYNOS_IPP_TASK_ASYNC;
+		exynos_drm_ipp_schedule_task(task->ipp, task);
+		ret = 0;
+	} else {
+		DRM_DEBUG_DRIVER("ipp: %d, processing task %pK\n", ipp->id,
+				 task);
+		exynos_drm_ipp_schedule_task(ipp, task);
+		ret = wait_event_interruptible(ipp->done_wq,
+					task->flags & DRM_EXYNOS_IPP_TASK_DONE);
+		if (ret)
+			exynos_drm_ipp_task_abort(ipp, task);
+		else
+			ret = exynos_drm_ipp_task_cleanup(task);
+	}
+	return ret;
+free:
+	exynos_drm_ipp_task_free(ipp, task);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/exynos/exynos_drm_ipp.h b/drivers/gpu/drm/exynos/exynos_drm_ipp.h
new file mode 100644
index 000000000000..0b27d4a9bf94
--- /dev/null
+++ b/drivers/gpu/drm/exynos/exynos_drm_ipp.h
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2017 Samsung Electronics Co., Ltd.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#ifndef _EXYNOS_DRM_IPP_H_
+#define _EXYNOS_DRM_IPP_H_
+
+#include <drm/drmP.h>
+
+struct exynos_drm_ipp;
+struct exynos_drm_ipp_task;
+
+/**
+ * struct exynos_drm_ipp_funcs - exynos_drm_ipp control functions
+ */
+struct exynos_drm_ipp_funcs {
+	/**
+	 * @commit:
+	 *
+	 * This is the main entry point to start framebuffer processing
+	 * in the hardware. The exynos_drm_ipp_task has been already validated.
+	 * This function must not wait until the device finishes processing.
+	 * When the driver finishes processing, it has to call
+	 * exynos_exynos_drm_ipp_task_done() function.
+	 *
+	 * RETURNS:
+	 *
+	 * 0 on success or negative error codes in case of failure.
+	 */
+	int (*commit)(struct exynos_drm_ipp *ipp,
+		      struct exynos_drm_ipp_task *task);
+
+	/**
+	 * @abort:
+	 *
+	 * Informs the driver that it has to abort the currently running
+	 * task as soon as possible (i.e. as soon as it can stop the device
+	 * safely), even if the task would not have been finished by then.
+	 * After the driver performs the necessary steps, it has to call
+	 * exynos_drm_ipp_task_done() (as if the task ended normally).
+	 * This function does not have to (and will usually not) wait
+	 * until the device enters a state when it can be stopped.
+	 */
+	void (*abort)(struct exynos_drm_ipp *ipp,
+		      struct exynos_drm_ipp_task *task);
+};
+
+/**
+ * struct exynos_drm_ipp - central picture processor module structure
+ */
+struct exynos_drm_ipp {
+	struct drm_device *dev;
+	struct list_head head;
+	unsigned int id;
+
+	const char *name;
+	const struct exynos_drm_ipp_funcs *funcs;
+	unsigned int capabilities;
+	const struct exynos_drm_ipp_formats *formats;
+	unsigned int num_formats;
+	atomic_t sequence;
+
+	spinlock_t lock;
+	struct exynos_drm_ipp_task *task;
+	struct list_head todo_list;
+	wait_queue_head_t done_wq;
+};
+
+struct exynos_drm_ipp_buffer {
+	struct drm_exynos_ipp_task_buffer buf;
+	struct drm_exynos_ipp_task_rect rect;
+
+	struct exynos_drm_gem *exynos_gem[MAX_FB_BUFFER];
+	const struct drm_format_info *format;
+	dma_addr_t dma_addr[MAX_FB_BUFFER];
+};
+
+/**
+ * struct exynos_drm_ipp_task - a structure describing transformation that
+ * has to be performed by the picture processor hardware module
+ */
+struct exynos_drm_ipp_task {
+	struct drm_device *dev;
+	struct exynos_drm_ipp *ipp;
+	struct list_head head;
+
+	struct exynos_drm_ipp_buffer src;
+	struct exynos_drm_ipp_buffer dst;
+
+	struct drm_exynos_ipp_task_transform transform;
+	struct drm_exynos_ipp_task_alpha alpha;
+
+	struct work_struct cleanup_work;
+	unsigned int flags;
+	int ret;
+
+	struct drm_pending_exynos_ipp_event *event;
+};
+
+#define DRM_EXYNOS_IPP_TASK_DONE	(1 << 0)
+#define DRM_EXYNOS_IPP_TASK_ASYNC	(1 << 1)
+
+struct exynos_drm_ipp_formats {
+	uint32_t fourcc;
+	uint32_t type;
+	uint64_t modifier;
+	const struct drm_exynos_ipp_limit *limits;
+	unsigned int num_limits;
+};
+
+/* helper macros to set exynos_drm_ipp_formats structure and limits*/
+#define IPP_SRCDST_MFORMAT(f, m, l) \
+	.fourcc = DRM_FORMAT_##f, .modifier = m, .limits = l, \
+	.num_limits = ARRAY_SIZE(l), \
+	.type = (DRM_EXYNOS_IPP_FORMAT_SOURCE | \
+		 DRM_EXYNOS_IPP_FORMAT_DESTINATION)
+
+#define IPP_SRCDST_FORMAT(f, l) IPP_SRCDST_MFORMAT(f, 0, l)
+
+#define IPP_SIZE_LIMIT(l, val...)	\
+	.type = (DRM_EXYNOS_IPP_LIMIT_TYPE_SIZE | \
+		 DRM_EXYNOS_IPP_LIMIT_SIZE_##l), val
+
+#define IPP_SCALE_LIMIT(val...)		\
+	.type = (DRM_EXYNOS_IPP_LIMIT_TYPE_SCALE), val
+
+int exynos_drm_ipp_register(struct drm_device *dev, struct exynos_drm_ipp *ipp,
+		const struct exynos_drm_ipp_funcs *funcs, unsigned int caps,
+		const struct exynos_drm_ipp_formats *formats,
+		unsigned int num_formats, const char *name);
+void exynos_drm_ipp_unregister(struct drm_device *dev,
+			       struct exynos_drm_ipp *ipp);
+
+void exynos_drm_ipp_task_done(struct exynos_drm_ipp_task *task, int ret);
+
+#ifdef CONFIG_DRM_EXYNOS_IPP
+int exynos_drm_ipp_get_res_ioctl(struct drm_device *dev, void *data,
+				 struct drm_file *file_priv);
+int exynos_drm_ipp_get_caps_ioctl(struct drm_device *dev, void *data,
+				  struct drm_file *file_priv);
+int exynos_drm_ipp_get_limits_ioctl(struct drm_device *dev, void *data,
+				    struct drm_file *file_priv);
+int exynos_drm_ipp_commit_ioctl(struct drm_device *dev,
+				void *data, struct drm_file *file_priv);
+#else
+static inline int exynos_drm_ipp_get_res_ioctl(struct drm_device *dev,
+	 void *data, struct drm_file *file_priv)
+{
+	struct drm_exynos_ioctl_ipp_get_res *resp = data;
+
+	resp->count_ipps = 0;
+	return 0;
+}
+static inline int exynos_drm_ipp_get_caps_ioctl(struct drm_device *dev,
+	 void *data, struct drm_file *file_priv)
+{
+	return -ENODEV;
+}
+static inline int exynos_drm_ipp_get_limits_ioctl(struct drm_device *dev,
+	 void *data, struct drm_file *file_priv)
+{
+	return -ENODEV;
+}
+static inline int exynos_drm_ipp_commit_ioctl(struct drm_device *dev,
+	 void *data, struct drm_file *file_priv)
+{
+	return -ENODEV;
+}
+#endif
+#endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index d2a90dae5c71..38a2a7f1204b 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -289,13 +289,12 @@ static const struct drm_plane_helper_funcs plane_helper_funcs = {
 };
 
 static void exynos_plane_attach_zpos_property(struct drm_plane *plane,
-					      bool immutable)
+					      int zpos, bool immutable)
 {
-	/* FIXME */
 	if (immutable)
-		drm_plane_create_zpos_immutable_property(plane, 0);
+		drm_plane_create_zpos_immutable_property(plane, zpos);
 	else
-		drm_plane_create_zpos_property(plane, 0, 0, MAX_PLANE - 1);
+		drm_plane_create_zpos_property(plane, zpos, 0, MAX_PLANE - 1);
 }
 
 int exynos_plane_init(struct drm_device *dev,
@@ -320,7 +319,7 @@ int exynos_plane_init(struct drm_device *dev,
 	exynos_plane->index = index;
 	exynos_plane->config = config;
 
-	exynos_plane_attach_zpos_property(&exynos_plane->base,
+	exynos_plane_attach_zpos_property(&exynos_plane->base, config->zpos,
 			   !(config->capabilities & EXYNOS_DRM_PLANE_CAP_ZPOS));
 
 	return 0;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_rotator.c b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
index 79282a820ecc..1a76dd3d52e1 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_rotator.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
@@ -10,6 +10,7 @@
  */
 
 #include <linux/kernel.h>
+#include <linux/component.h>
 #include <linux/err.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
@@ -22,29 +23,18 @@
 #include <drm/exynos_drm.h>
 #include "regs-rotator.h"
 #include "exynos_drm_drv.h"
+#include "exynos_drm_iommu.h"
 #include "exynos_drm_ipp.h"
 
 /*
  * Rotator supports image crop/rotator and input/output DMA operations.
  * input DMA reads image data from the memory.
  * output DMA writes image data to memory.
- *
- * M2M operation : supports crop/scale/rotation/csc so on.
- * Memory ----> Rotator H/W ----> Memory.
  */
 
-/*
- * TODO
- * 1. check suspend/resume api if needed.
- * 2. need to check use case platform_device_id.
- * 3. check src/dst size with, height.
- * 4. need to add supported list in prop_list.
- */
+#define ROTATOR_AUTOSUSPEND_DELAY	2000
 
-#define get_rot_context(dev)	platform_get_drvdata(to_platform_device(dev))
-#define get_ctx_from_ippdrv(ippdrv)	container_of(ippdrv,\
-					struct rot_context, ippdrv);
-#define rot_read(offset)		readl(rot->regs + (offset))
+#define rot_read(offset)	readl(rot->regs + (offset))
 #define rot_write(cfg, offset)	writel(cfg, rot->regs + (offset))
 
 enum rot_irq_status {
@@ -52,54 +42,28 @@ enum rot_irq_status {
 	ROT_IRQ_STATUS_ILLEGAL	= 9,
 };
 
-/*
- * A structure of limitation.
- *
- * @min_w: minimum width.
- * @min_h: minimum height.
- * @max_w: maximum width.
- * @max_h: maximum height.
- * @align: align size.
- */
-struct rot_limit {
-	u32	min_w;
-	u32	min_h;
-	u32	max_w;
-	u32	max_h;
-	u32	align;
-};
-
-/*
- * A structure of limitation table.
- *
- * @ycbcr420_2p: case of YUV.
- * @rgb888: case of RGB.
- */
-struct rot_limit_table {
-	struct rot_limit	ycbcr420_2p;
-	struct rot_limit	rgb888;
+struct rot_variant {
+	const struct exynos_drm_ipp_formats *formats;
+	unsigned int	num_formats;
 };
 
 /*
  * A structure of rotator context.
  * @ippdrv: prepare initialization using ippdrv.
- * @regs_res: register resources.
  * @regs: memory mapped io registers.
  * @clock: rotator gate clock.
  * @limit_tbl: limitation of rotator.
  * @irq: irq number.
- * @cur_buf_id: current operation buffer id.
- * @suspended: suspended state.
  */
 struct rot_context {
-	struct exynos_drm_ippdrv	ippdrv;
-	struct resource	*regs_res;
+	struct exynos_drm_ipp ipp;
+	struct drm_device *drm_dev;
+	struct device	*dev;
 	void __iomem	*regs;
 	struct clk	*clock;
-	struct rot_limit_table	*limit_tbl;
-	int	irq;
-	int	cur_buf_id[EXYNOS_DRM_OPS_MAX];
-	bool	suspended;
+	const struct exynos_drm_ipp_formats *formats;
+	unsigned int	num_formats;
+	struct exynos_drm_ipp_task	*task;
 };
 
 static void rotator_reg_set_irq(struct rot_context *rot, bool enable)
@@ -114,15 +78,6 @@ static void rotator_reg_set_irq(struct rot_context *rot, bool enable)
 	rot_write(val, ROT_CONFIG);
 }
 
-static u32 rotator_reg_get_fmt(struct rot_context *rot)
-{
-	u32 val = rot_read(ROT_CONTROL);
-
-	val &= ROT_CONTROL_FMT_MASK;
-
-	return val;
-}
-
 static enum rot_irq_status rotator_reg_get_irq_status(struct rot_context *rot)
 {
 	u32 val = rot_read(ROT_STATUS);
@@ -138,9 +93,6 @@ static enum rot_irq_status rotator_reg_get_irq_status(struct rot_context *rot)
 static irqreturn_t rotator_irq_handler(int irq, void *arg)
 {
 	struct rot_context *rot = arg;
-	struct exynos_drm_ippdrv *ippdrv = &rot->ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node = ippdrv->c_node;
-	struct drm_exynos_ipp_event_work *event_work = c_node->event_work;
 	enum rot_irq_status irq_status;
 	u32 val;
 
@@ -152,56 +104,21 @@ static irqreturn_t rotator_irq_handler(int irq, void *arg)
 	val |= ROT_STATUS_IRQ_PENDING((u32)irq_status);
 	rot_write(val, ROT_STATUS);
 
-	if (irq_status == ROT_IRQ_STATUS_COMPLETE) {
-		event_work->ippdrv = ippdrv;
-		event_work->buf_id[EXYNOS_DRM_OPS_DST] =
-			rot->cur_buf_id[EXYNOS_DRM_OPS_DST];
-		queue_work(ippdrv->event_workq, &event_work->work);
-	} else {
-		DRM_ERROR("the SFR is set illegally\n");
+	if (rot->task) {
+		struct exynos_drm_ipp_task *task = rot->task;
+
+		rot->task = NULL;
+		pm_runtime_mark_last_busy(rot->dev);
+		pm_runtime_put_autosuspend(rot->dev);
+		exynos_drm_ipp_task_done(task,
+			irq_status == ROT_IRQ_STATUS_COMPLETE ? 0 : -EINVAL);
 	}
 
 	return IRQ_HANDLED;
 }
 
-static void rotator_align_size(struct rot_context *rot, u32 fmt, u32 *hsize,
-		u32 *vsize)
+static void rotator_src_set_fmt(struct rot_context *rot, u32 fmt)
 {
-	struct rot_limit_table *limit_tbl = rot->limit_tbl;
-	struct rot_limit *limit;
-	u32 mask, val;
-
-	/* Get size limit */
-	if (fmt == ROT_CONTROL_FMT_RGB888)
-		limit = &limit_tbl->rgb888;
-	else
-		limit = &limit_tbl->ycbcr420_2p;
-
-	/* Get mask for rounding to nearest aligned val */
-	mask = ~((1 << limit->align) - 1);
-
-	/* Set aligned width */
-	val = ROT_ALIGN(*hsize, limit->align, mask);
-	if (val < limit->min_w)
-		*hsize = ROT_MIN(limit->min_w, mask);
-	else if (val > limit->max_w)
-		*hsize = ROT_MAX(limit->max_w, mask);
-	else
-		*hsize = val;
-
-	/* Set aligned height */
-	val = ROT_ALIGN(*vsize, limit->align, mask);
-	if (val < limit->min_h)
-		*vsize = ROT_MIN(limit->min_h, mask);
-	else if (val > limit->max_h)
-		*vsize = ROT_MAX(limit->max_h, mask);
-	else
-		*vsize = val;
-}
-
-static int rotator_src_set_fmt(struct device *dev, u32 fmt)
-{
-	struct rot_context *rot = dev_get_drvdata(dev);
 	u32 val;
 
 	val = rot_read(ROT_CONTROL);
@@ -214,515 +131,176 @@ static int rotator_src_set_fmt(struct device *dev, u32 fmt)
 	case DRM_FORMAT_XRGB8888:
 		val |= ROT_CONTROL_FMT_RGB888;
 		break;
-	default:
-		DRM_ERROR("invalid image format\n");
-		return -EINVAL;
 	}
 
 	rot_write(val, ROT_CONTROL);
-
-	return 0;
 }
 
-static inline bool rotator_check_reg_fmt(u32 fmt)
+static void rotator_src_set_buf(struct rot_context *rot,
+				struct exynos_drm_ipp_buffer *buf)
 {
-	if ((fmt == ROT_CONTROL_FMT_YCBCR420_2P) ||
-	    (fmt == ROT_CONTROL_FMT_RGB888))
-		return true;
-
-	return false;
-}
-
-static int rotator_src_set_size(struct device *dev, int swap,
-		struct drm_exynos_pos *pos,
-		struct drm_exynos_sz *sz)
-{
-	struct rot_context *rot = dev_get_drvdata(dev);
-	u32 fmt, hsize, vsize;
 	u32 val;
 
-	/* Get format */
-	fmt = rotator_reg_get_fmt(rot);
-	if (!rotator_check_reg_fmt(fmt)) {
-		DRM_ERROR("invalid format.\n");
-		return -EINVAL;
-	}
-
-	/* Align buffer size */
-	hsize = sz->hsize;
-	vsize = sz->vsize;
-	rotator_align_size(rot, fmt, &hsize, &vsize);
-
 	/* Set buffer size configuration */
-	val = ROT_SET_BUF_SIZE_H(vsize) | ROT_SET_BUF_SIZE_W(hsize);
+	val = ROT_SET_BUF_SIZE_H(buf->buf.height) |
+	      ROT_SET_BUF_SIZE_W(buf->buf.pitch[0] / buf->format->cpp[0]);
 	rot_write(val, ROT_SRC_BUF_SIZE);
 
 	/* Set crop image position configuration */
-	val = ROT_CROP_POS_Y(pos->y) | ROT_CROP_POS_X(pos->x);
+	val = ROT_CROP_POS_Y(buf->rect.y) | ROT_CROP_POS_X(buf->rect.x);
 	rot_write(val, ROT_SRC_CROP_POS);
-	val = ROT_SRC_CROP_SIZE_H(pos->h) | ROT_SRC_CROP_SIZE_W(pos->w);
+	val = ROT_SRC_CROP_SIZE_H(buf->rect.h) |
+	      ROT_SRC_CROP_SIZE_W(buf->rect.w);
 	rot_write(val, ROT_SRC_CROP_SIZE);
 
-	return 0;
+	/* Set buffer DMA address */
+	rot_write(buf->dma_addr[0], ROT_SRC_BUF_ADDR(0));
+	rot_write(buf->dma_addr[1], ROT_SRC_BUF_ADDR(1));
 }
 
-static int rotator_src_set_addr(struct device *dev,
-		struct drm_exynos_ipp_buf_info *buf_info,
-		u32 buf_id, enum drm_exynos_ipp_buf_type buf_type)
+static void rotator_dst_set_transf(struct rot_context *rot,
+				   unsigned int rotation)
 {
-	struct rot_context *rot = dev_get_drvdata(dev);
-	dma_addr_t addr[EXYNOS_DRM_PLANAR_MAX];
-	u32 val, fmt, hsize, vsize;
-	int i;
-
-	/* Set current buf_id */
-	rot->cur_buf_id[EXYNOS_DRM_OPS_SRC] = buf_id;
-
-	switch (buf_type) {
-	case IPP_BUF_ENQUEUE:
-		/* Set address configuration */
-		for_each_ipp_planar(i)
-			addr[i] = buf_info->base[i];
-
-		/* Get format */
-		fmt = rotator_reg_get_fmt(rot);
-		if (!rotator_check_reg_fmt(fmt)) {
-			DRM_ERROR("invalid format.\n");
-			return -EINVAL;
-		}
-
-		/* Re-set cb planar for NV12 format */
-		if ((fmt == ROT_CONTROL_FMT_YCBCR420_2P) &&
-		    !addr[EXYNOS_DRM_PLANAR_CB]) {
-
-			val = rot_read(ROT_SRC_BUF_SIZE);
-			hsize = ROT_GET_BUF_SIZE_W(val);
-			vsize = ROT_GET_BUF_SIZE_H(val);
-
-			/* Set cb planar */
-			addr[EXYNOS_DRM_PLANAR_CB] =
-				addr[EXYNOS_DRM_PLANAR_Y] + hsize * vsize;
-		}
-
-		for_each_ipp_planar(i)
-			rot_write(addr[i], ROT_SRC_BUF_ADDR(i));
-		break;
-	case IPP_BUF_DEQUEUE:
-		for_each_ipp_planar(i)
-			rot_write(0x0, ROT_SRC_BUF_ADDR(i));
-		break;
-	default:
-		/* Nothing to do */
-		break;
-	}
-
-	return 0;
-}
-
-static int rotator_dst_set_transf(struct device *dev,
-		enum drm_exynos_degree degree,
-		enum drm_exynos_flip flip, bool *swap)
-{
-	struct rot_context *rot = dev_get_drvdata(dev);
 	u32 val;
 
 	/* Set transform configuration */
 	val = rot_read(ROT_CONTROL);
 	val &= ~ROT_CONTROL_FLIP_MASK;
 
-	switch (flip) {
-	case EXYNOS_DRM_FLIP_VERTICAL:
-		val |= ROT_CONTROL_FLIP_VERTICAL;
-		break;
-	case EXYNOS_DRM_FLIP_HORIZONTAL:
+	if (rotation & DRM_MODE_REFLECT_X)
 		val |= ROT_CONTROL_FLIP_HORIZONTAL;
-		break;
-	default:
-		/* Flip None */
-		break;
-	}
+	if (rotation & DRM_MODE_REFLECT_Y)
+		val |= ROT_CONTROL_FLIP_VERTICAL;
 
 	val &= ~ROT_CONTROL_ROT_MASK;
 
-	switch (degree) {
-	case EXYNOS_DRM_DEGREE_90:
+	if (rotation & DRM_MODE_ROTATE_90)
 		val |= ROT_CONTROL_ROT_90;
-		break;
-	case EXYNOS_DRM_DEGREE_180:
+	else if (rotation & DRM_MODE_ROTATE_180)
 		val |= ROT_CONTROL_ROT_180;
-		break;
-	case EXYNOS_DRM_DEGREE_270:
+	else if (rotation & DRM_MODE_ROTATE_270)
 		val |= ROT_CONTROL_ROT_270;
-		break;
-	default:
-		/* Rotation 0 Degree */
-		break;
-	}
 
 	rot_write(val, ROT_CONTROL);
-
-	/* Check degree for setting buffer size swap */
-	if ((degree == EXYNOS_DRM_DEGREE_90) ||
-	    (degree == EXYNOS_DRM_DEGREE_270))
-		*swap = true;
-	else
-		*swap = false;
-
-	return 0;
 }
 
-static int rotator_dst_set_size(struct device *dev, int swap,
-		struct drm_exynos_pos *pos,
-		struct drm_exynos_sz *sz)
+static void rotator_dst_set_buf(struct rot_context *rot,
+				struct exynos_drm_ipp_buffer *buf)
 {
-	struct rot_context *rot = dev_get_drvdata(dev);
-	u32 val, fmt, hsize, vsize;
-
-	/* Get format */
-	fmt = rotator_reg_get_fmt(rot);
-	if (!rotator_check_reg_fmt(fmt)) {
-		DRM_ERROR("invalid format.\n");
-		return -EINVAL;
-	}
-
-	/* Align buffer size */
-	hsize = sz->hsize;
-	vsize = sz->vsize;
-	rotator_align_size(rot, fmt, &hsize, &vsize);
+	u32 val;
 
 	/* Set buffer size configuration */
-	val = ROT_SET_BUF_SIZE_H(vsize) | ROT_SET_BUF_SIZE_W(hsize);
+	val = ROT_SET_BUF_SIZE_H(buf->buf.height) |
+	      ROT_SET_BUF_SIZE_W(buf->buf.pitch[0] / buf->format->cpp[0]);
 	rot_write(val, ROT_DST_BUF_SIZE);
 
 	/* Set crop image position configuration */
-	val = ROT_CROP_POS_Y(pos->y) | ROT_CROP_POS_X(pos->x);
+	val = ROT_CROP_POS_Y(buf->rect.y) | ROT_CROP_POS_X(buf->rect.x);
 	rot_write(val, ROT_DST_CROP_POS);
 
-	return 0;
+	/* Set buffer DMA address */
+	rot_write(buf->dma_addr[0], ROT_DST_BUF_ADDR(0));
+	rot_write(buf->dma_addr[1], ROT_DST_BUF_ADDR(1));
 }
 
-static int rotator_dst_set_addr(struct device *dev,
-		struct drm_exynos_ipp_buf_info *buf_info,
-		u32 buf_id, enum drm_exynos_ipp_buf_type buf_type)
+static void rotator_start(struct rot_context *rot)
 {
-	struct rot_context *rot = dev_get_drvdata(dev);
-	dma_addr_t addr[EXYNOS_DRM_PLANAR_MAX];
-	u32 val, fmt, hsize, vsize;
-	int i;
-
-	/* Set current buf_id */
-	rot->cur_buf_id[EXYNOS_DRM_OPS_DST] = buf_id;
-
-	switch (buf_type) {
-	case IPP_BUF_ENQUEUE:
-		/* Set address configuration */
-		for_each_ipp_planar(i)
-			addr[i] = buf_info->base[i];
-
-		/* Get format */
-		fmt = rotator_reg_get_fmt(rot);
-		if (!rotator_check_reg_fmt(fmt)) {
-			DRM_ERROR("invalid format.\n");
-			return -EINVAL;
-		}
-
-		/* Re-set cb planar for NV12 format */
-		if ((fmt == ROT_CONTROL_FMT_YCBCR420_2P) &&
-		    !addr[EXYNOS_DRM_PLANAR_CB]) {
-			/* Get buf size */
-			val = rot_read(ROT_DST_BUF_SIZE);
-
-			hsize = ROT_GET_BUF_SIZE_W(val);
-			vsize = ROT_GET_BUF_SIZE_H(val);
-
-			/* Set cb planar */
-			addr[EXYNOS_DRM_PLANAR_CB] =
-				addr[EXYNOS_DRM_PLANAR_Y] + hsize * vsize;
-		}
-
-		for_each_ipp_planar(i)
-			rot_write(addr[i], ROT_DST_BUF_ADDR(i));
-		break;
-	case IPP_BUF_DEQUEUE:
-		for_each_ipp_planar(i)
-			rot_write(0x0, ROT_DST_BUF_ADDR(i));
-		break;
-	default:
-		/* Nothing to do */
-		break;
-	}
+	u32 val;
 
-	return 0;
+	/* Set interrupt enable */
+	rotator_reg_set_irq(rot, true);
+
+	val = rot_read(ROT_CONTROL);
+	val |= ROT_CONTROL_START;
+	rot_write(val, ROT_CONTROL);
 }
 
-static struct exynos_drm_ipp_ops rot_src_ops = {
-	.set_fmt	=	rotator_src_set_fmt,
-	.set_size	=	rotator_src_set_size,
-	.set_addr	=	rotator_src_set_addr,
-};
+static int rotator_commit(struct exynos_drm_ipp *ipp,
+			  struct exynos_drm_ipp_task *task)
+{
+	struct rot_context *rot =
+			container_of(ipp, struct rot_context, ipp);
 
-static struct exynos_drm_ipp_ops rot_dst_ops = {
-	.set_transf	=	rotator_dst_set_transf,
-	.set_size	=	rotator_dst_set_size,
-	.set_addr	=	rotator_dst_set_addr,
-};
+	pm_runtime_get_sync(rot->dev);
+	rot->task = task;
 
-static int rotator_init_prop_list(struct exynos_drm_ippdrv *ippdrv)
-{
-	struct drm_exynos_ipp_prop_list *prop_list = &ippdrv->prop_list;
-
-	prop_list->version = 1;
-	prop_list->flip = (1 << EXYNOS_DRM_FLIP_VERTICAL) |
-				(1 << EXYNOS_DRM_FLIP_HORIZONTAL);
-	prop_list->degree = (1 << EXYNOS_DRM_DEGREE_0) |
-				(1 << EXYNOS_DRM_DEGREE_90) |
-				(1 << EXYNOS_DRM_DEGREE_180) |
-				(1 << EXYNOS_DRM_DEGREE_270);
-	prop_list->csc = 0;
-	prop_list->crop = 0;
-	prop_list->scale = 0;
+	rotator_src_set_fmt(rot, task->src.buf.fourcc);
+	rotator_src_set_buf(rot, &task->src);
+	rotator_dst_set_transf(rot, task->transform.rotation);
+	rotator_dst_set_buf(rot, &task->dst);
+	rotator_start(rot);
 
 	return 0;
 }
 
-static inline bool rotator_check_drm_fmt(u32 fmt)
-{
-	switch (fmt) {
-	case DRM_FORMAT_XRGB8888:
-	case DRM_FORMAT_NV12:
-		return true;
-	default:
-		DRM_DEBUG_KMS("not support format\n");
-		return false;
-	}
-}
-
-static inline bool rotator_check_drm_flip(enum drm_exynos_flip flip)
-{
-	switch (flip) {
-	case EXYNOS_DRM_FLIP_NONE:
-	case EXYNOS_DRM_FLIP_VERTICAL:
-	case EXYNOS_DRM_FLIP_HORIZONTAL:
-	case EXYNOS_DRM_FLIP_BOTH:
-		return true;
-	default:
-		DRM_DEBUG_KMS("invalid flip\n");
-		return false;
-	}
-}
+static const struct exynos_drm_ipp_funcs ipp_funcs = {
+	.commit = rotator_commit,
+};
 
-static int rotator_ippdrv_check_property(struct device *dev,
-		struct drm_exynos_ipp_property *property)
+static int rotator_bind(struct device *dev, struct device *master, void *data)
 {
-	struct drm_exynos_ipp_config *src_config =
-					&property->config[EXYNOS_DRM_OPS_SRC];
-	struct drm_exynos_ipp_config *dst_config =
-					&property->config[EXYNOS_DRM_OPS_DST];
-	struct drm_exynos_pos *src_pos = &src_config->pos;
-	struct drm_exynos_pos *dst_pos = &dst_config->pos;
-	struct drm_exynos_sz *src_sz = &src_config->sz;
-	struct drm_exynos_sz *dst_sz = &dst_config->sz;
-	bool swap = false;
-
-	/* Check format configuration */
-	if (src_config->fmt != dst_config->fmt) {
-		DRM_DEBUG_KMS("not support csc feature\n");
-		return -EINVAL;
-	}
-
-	if (!rotator_check_drm_fmt(dst_config->fmt)) {
-		DRM_DEBUG_KMS("invalid format\n");
-		return -EINVAL;
-	}
-
-	/* Check transform configuration */
-	if (src_config->degree != EXYNOS_DRM_DEGREE_0) {
-		DRM_DEBUG_KMS("not support source-side rotation\n");
-		return -EINVAL;
-	}
-
-	switch (dst_config->degree) {
-	case EXYNOS_DRM_DEGREE_90:
-	case EXYNOS_DRM_DEGREE_270:
-		swap = true;
-	case EXYNOS_DRM_DEGREE_0:
-	case EXYNOS_DRM_DEGREE_180:
-		/* No problem */
-		break;
-	default:
-		DRM_DEBUG_KMS("invalid degree\n");
-		return -EINVAL;
-	}
-
-	if (src_config->flip != EXYNOS_DRM_FLIP_NONE) {
-		DRM_DEBUG_KMS("not support source-side flip\n");
-		return -EINVAL;
-	}
+	struct rot_context *rot = dev_get_drvdata(dev);
+	struct drm_device *drm_dev = data;
+	struct exynos_drm_ipp *ipp = &rot->ipp;
 
-	if (!rotator_check_drm_flip(dst_config->flip)) {
-		DRM_DEBUG_KMS("invalid flip\n");
-		return -EINVAL;
-	}
+	rot->drm_dev = drm_dev;
+	drm_iommu_attach_device(drm_dev, dev);
 
-	/* Check size configuration */
-	if ((src_pos->x + src_pos->w > src_sz->hsize) ||
-		(src_pos->y + src_pos->h > src_sz->vsize)) {
-		DRM_DEBUG_KMS("out of source buffer bound\n");
-		return -EINVAL;
-	}
+	exynos_drm_ipp_register(drm_dev, ipp, &ipp_funcs,
+			   DRM_EXYNOS_IPP_CAP_CROP | DRM_EXYNOS_IPP_CAP_ROTATE,
+			   rot->formats, rot->num_formats, "rotator");
 
-	if (swap) {
-		if ((dst_pos->x + dst_pos->h > dst_sz->vsize) ||
-			(dst_pos->y + dst_pos->w > dst_sz->hsize)) {
-			DRM_DEBUG_KMS("out of destination buffer bound\n");
-			return -EINVAL;
-		}
-
-		if ((src_pos->w != dst_pos->h) || (src_pos->h != dst_pos->w)) {
-			DRM_DEBUG_KMS("not support scale feature\n");
-			return -EINVAL;
-		}
-	} else {
-		if ((dst_pos->x + dst_pos->w > dst_sz->hsize) ||
-			(dst_pos->y + dst_pos->h > dst_sz->vsize)) {
-			DRM_DEBUG_KMS("out of destination buffer bound\n");
-			return -EINVAL;
-		}
-
-		if ((src_pos->w != dst_pos->w) || (src_pos->h != dst_pos->h)) {
-			DRM_DEBUG_KMS("not support scale feature\n");
-			return -EINVAL;
-		}
-	}
+	dev_info(dev, "The exynos rotator has been probed successfully\n");
 
 	return 0;
 }
 
-static int rotator_ippdrv_start(struct device *dev, enum drm_exynos_ipp_cmd cmd)
+static void rotator_unbind(struct device *dev, struct device *master,
+			void *data)
 {
 	struct rot_context *rot = dev_get_drvdata(dev);
-	u32 val;
-
-	if (rot->suspended) {
-		DRM_ERROR("suspended state\n");
-		return -EPERM;
-	}
-
-	if (cmd != IPP_CMD_M2M) {
-		DRM_ERROR("not support cmd: %d\n", cmd);
-		return -EINVAL;
-	}
-
-	/* Set interrupt enable */
-	rotator_reg_set_irq(rot, true);
-
-	val = rot_read(ROT_CONTROL);
-	val |= ROT_CONTROL_START;
-
-	rot_write(val, ROT_CONTROL);
+	struct drm_device *drm_dev = data;
+	struct exynos_drm_ipp *ipp = &rot->ipp;
 
-	return 0;
+	exynos_drm_ipp_unregister(drm_dev, ipp);
+	drm_iommu_detach_device(rot->drm_dev, rot->dev);
 }
 
-static struct rot_limit_table rot_limit_tbl_4210 = {
-	.ycbcr420_2p = {
-		.min_w = 32,
-		.min_h = 32,
-		.max_w = SZ_64K,
-		.max_h = SZ_64K,
-		.align = 3,
-	},
-	.rgb888 = {
-		.min_w = 8,
-		.min_h = 8,
-		.max_w = SZ_16K,
-		.max_h = SZ_16K,
-		.align = 2,
-	},
-};
-
-static struct rot_limit_table rot_limit_tbl_4x12 = {
-	.ycbcr420_2p = {
-		.min_w = 32,
-		.min_h = 32,
-		.max_w = SZ_32K,
-		.max_h = SZ_32K,
-		.align = 3,
-	},
-	.rgb888 = {
-		.min_w = 8,
-		.min_h = 8,
-		.max_w = SZ_8K,
-		.max_h = SZ_8K,
-		.align = 2,
-	},
+static const struct component_ops rotator_component_ops = {
+	.bind	= rotator_bind,
+	.unbind = rotator_unbind,
 };
 
-static struct rot_limit_table rot_limit_tbl_5250 = {
-	.ycbcr420_2p = {
-		.min_w = 32,
-		.min_h = 32,
-		.max_w = SZ_32K,
-		.max_h = SZ_32K,
-		.align = 3,
-	},
-	.rgb888 = {
-		.min_w = 8,
-		.min_h = 8,
-		.max_w = SZ_8K,
-		.max_h = SZ_8K,
-		.align = 1,
-	},
-};
-
-static const struct of_device_id exynos_rotator_match[] = {
-	{
-		.compatible = "samsung,exynos4210-rotator",
-		.data = &rot_limit_tbl_4210,
-	},
-	{
-		.compatible = "samsung,exynos4212-rotator",
-		.data = &rot_limit_tbl_4x12,
-	},
-	{
-		.compatible = "samsung,exynos5250-rotator",
-		.data = &rot_limit_tbl_5250,
-	},
-	{},
-};
-MODULE_DEVICE_TABLE(of, exynos_rotator_match);
-
 static int rotator_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	struct resource	*regs_res;
 	struct rot_context *rot;
-	struct exynos_drm_ippdrv *ippdrv;
+	const struct rot_variant *variant;
+	int irq;
 	int ret;
 
-	if (!dev->of_node) {
-		dev_err(dev, "cannot find of_node.\n");
-		return -ENODEV;
-	}
-
 	rot = devm_kzalloc(dev, sizeof(*rot), GFP_KERNEL);
 	if (!rot)
 		return -ENOMEM;
 
-	rot->limit_tbl = (struct rot_limit_table *)
-				of_device_get_match_data(dev);
-	rot->regs_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	rot->regs = devm_ioremap_resource(dev, rot->regs_res);
+	variant = of_device_get_match_data(dev);
+	rot->formats = variant->formats;
+	rot->num_formats = variant->num_formats;
+	rot->dev = dev;
+	regs_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	rot->regs = devm_ioremap_resource(dev, regs_res);
 	if (IS_ERR(rot->regs))
 		return PTR_ERR(rot->regs);
 
-	rot->irq = platform_get_irq(pdev, 0);
-	if (rot->irq < 0) {
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
 		dev_err(dev, "failed to get irq\n");
-		return rot->irq;
+		return irq;
 	}
 
-	ret = devm_request_threaded_irq(dev, rot->irq, NULL,
-			rotator_irq_handler, IRQF_ONESHOT, "drm_rotator", rot);
+	ret = devm_request_irq(dev, irq, rotator_irq_handler, 0, dev_name(dev),
+			       rot);
 	if (ret < 0) {
 		dev_err(dev, "failed to request irq\n");
 		return ret;
@@ -734,35 +312,19 @@ static int rotator_probe(struct platform_device *pdev)
 		return PTR_ERR(rot->clock);
 	}
 
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_autosuspend_delay(dev, ROTATOR_AUTOSUSPEND_DELAY);
 	pm_runtime_enable(dev);
-
-	ippdrv = &rot->ippdrv;
-	ippdrv->dev = dev;
-	ippdrv->ops[EXYNOS_DRM_OPS_SRC] = &rot_src_ops;
-	ippdrv->ops[EXYNOS_DRM_OPS_DST] = &rot_dst_ops;
-	ippdrv->check_property = rotator_ippdrv_check_property;
-	ippdrv->start = rotator_ippdrv_start;
-	ret = rotator_init_prop_list(ippdrv);
-	if (ret < 0) {
-		dev_err(dev, "failed to init property list.\n");
-		goto err_ippdrv_register;
-	}
-
-	DRM_DEBUG_KMS("ippdrv[%pK]\n", ippdrv);
-
 	platform_set_drvdata(pdev, rot);
 
-	ret = exynos_drm_ippdrv_register(ippdrv);
-	if (ret < 0) {
-		dev_err(dev, "failed to register drm rotator device\n");
-		goto err_ippdrv_register;
-	}
-
-	dev_info(dev, "The exynos rotator is probed successfully\n");
+	ret = component_add(dev, &rotator_component_ops);
+	if (ret)
+		goto err_component;
 
 	return 0;
 
-err_ippdrv_register:
+err_component:
+	pm_runtime_dont_use_autosuspend(dev);
 	pm_runtime_disable(dev);
 	return ret;
 }
@@ -770,45 +332,101 @@ static int rotator_probe(struct platform_device *pdev)
 static int rotator_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct rot_context *rot = dev_get_drvdata(dev);
-	struct exynos_drm_ippdrv *ippdrv = &rot->ippdrv;
-
-	exynos_drm_ippdrv_unregister(ippdrv);
 
+	component_del(dev, &rotator_component_ops);
+	pm_runtime_dont_use_autosuspend(dev);
 	pm_runtime_disable(dev);
 
 	return 0;
 }
 
 #ifdef CONFIG_PM
-static int rotator_clk_crtl(struct rot_context *rot, bool enable)
-{
-	if (enable) {
-		clk_prepare_enable(rot->clock);
-		rot->suspended = false;
-	} else {
-		clk_disable_unprepare(rot->clock);
-		rot->suspended = true;
-	}
-
-	return 0;
-}
-
 static int rotator_runtime_suspend(struct device *dev)
 {
 	struct rot_context *rot = dev_get_drvdata(dev);
 
-	return  rotator_clk_crtl(rot, false);
+	clk_disable_unprepare(rot->clock);
+	return 0;
 }
 
 static int rotator_runtime_resume(struct device *dev)
 {
 	struct rot_context *rot = dev_get_drvdata(dev);
 
-	return  rotator_clk_crtl(rot, true);
+	return clk_prepare_enable(rot->clock);
 }
 #endif
 
+static const struct drm_exynos_ipp_limit rotator_4210_rbg888_limits[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 8, SZ_16K }, .v = { 8, SZ_16K }) },
+	{ IPP_SIZE_LIMIT(AREA, .h.align = 4, .v.align = 4) },
+};
+
+static const struct drm_exynos_ipp_limit rotator_4412_rbg888_limits[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 8, SZ_8K }, .v = { 8, SZ_8K }) },
+	{ IPP_SIZE_LIMIT(AREA, .h.align = 4, .v.align = 4) },
+};
+
+static const struct drm_exynos_ipp_limit rotator_5250_rbg888_limits[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 8, SZ_8K }, .v = { 8, SZ_8K }) },
+	{ IPP_SIZE_LIMIT(AREA, .h.align = 2, .v.align = 2) },
+};
+
+static const struct drm_exynos_ipp_limit rotator_4210_yuv_limits[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 32, SZ_64K }, .v = { 32, SZ_64K }) },
+	{ IPP_SIZE_LIMIT(AREA, .h.align = 8, .v.align = 8) },
+};
+
+static const struct drm_exynos_ipp_limit rotator_4412_yuv_limits[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 32, SZ_32K }, .v = { 32, SZ_32K }) },
+	{ IPP_SIZE_LIMIT(AREA, .h.align = 8, .v.align = 8) },
+};
+
+static const struct exynos_drm_ipp_formats rotator_4210_formats[] = {
+	{ IPP_SRCDST_FORMAT(XRGB8888, rotator_4210_rbg888_limits) },
+	{ IPP_SRCDST_FORMAT(NV12, rotator_4210_yuv_limits) },
+};
+
+static const struct exynos_drm_ipp_formats rotator_4412_formats[] = {
+	{ IPP_SRCDST_FORMAT(XRGB8888, rotator_4412_rbg888_limits) },
+	{ IPP_SRCDST_FORMAT(NV12, rotator_4412_yuv_limits) },
+};
+
+static const struct exynos_drm_ipp_formats rotator_5250_formats[] = {
+	{ IPP_SRCDST_FORMAT(XRGB8888, rotator_5250_rbg888_limits) },
+	{ IPP_SRCDST_FORMAT(NV12, rotator_4412_yuv_limits) },
+};
+
+static const struct rot_variant rotator_4210_data = {
+	.formats = rotator_4210_formats,
+	.num_formats = ARRAY_SIZE(rotator_4210_formats),
+};
+
+static const struct rot_variant rotator_4412_data = {
+	.formats = rotator_4412_formats,
+	.num_formats = ARRAY_SIZE(rotator_4412_formats),
+};
+
+static const struct rot_variant rotator_5250_data = {
+	.formats = rotator_5250_formats,
+	.num_formats = ARRAY_SIZE(rotator_5250_formats),
+};
+
+static const struct of_device_id exynos_rotator_match[] = {
+	{
+		.compatible = "samsung,exynos4210-rotator",
+		.data = &rotator_4210_data,
+	}, {
+		.compatible = "samsung,exynos4212-rotator",
+		.data = &rotator_4412_data,
+	}, {
+		.compatible = "samsung,exynos5250-rotator",
+		.data = &rotator_5250_data,
+	}, {
+	},
+};
+MODULE_DEVICE_TABLE(of, exynos_rotator_match);
+
 static const struct dev_pm_ops rotator_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
 				pm_runtime_force_resume)
@@ -820,7 +438,7 @@ struct platform_driver rotator_driver = {
 	.probe		= rotator_probe,
 	.remove		= rotator_remove,
 	.driver		= {
-		.name	= "exynos-rot",
+		.name	= "exynos-rotator",
 		.owner	= THIS_MODULE,
 		.pm	= &rotator_pm_ops,
 		.of_match_table = exynos_rotator_match,
diff --git a/drivers/gpu/drm/exynos/exynos_drm_scaler.c b/drivers/gpu/drm/exynos/exynos_drm_scaler.c
new file mode 100644
index 000000000000..91d4382343d0
--- /dev/null
+++ b/drivers/gpu/drm/exynos/exynos_drm_scaler.c
@@ -0,0 +1,694 @@
+/*
+ * Copyright (C) 2017 Samsung Electronics Co.Ltd
+ * Author:
+ *	Andrzej Pietrasiewicz <andrzej.p@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundationr
+ */
+
+#include <linux/kernel.h>
+#include <linux/component.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/clk.h>
+#include <linux/of_device.h>
+#include <linux/pm_runtime.h>
+
+#include <drm/drmP.h>
+#include <drm/exynos_drm.h>
+#include "regs-scaler.h"
+#include "exynos_drm_fb.h"
+#include "exynos_drm_drv.h"
+#include "exynos_drm_iommu.h"
+#include "exynos_drm_ipp.h"
+
+#define scaler_read(offset)		readl(scaler->regs + (offset))
+#define scaler_write(cfg, offset)	writel(cfg, scaler->regs + (offset))
+#define SCALER_MAX_CLK			4
+#define SCALER_AUTOSUSPEND_DELAY	2000
+
+struct scaler_data {
+	const char	*clk_name[SCALER_MAX_CLK];
+	unsigned int	num_clk;
+	const struct exynos_drm_ipp_formats *formats;
+	unsigned int	num_formats;
+};
+
+struct scaler_context {
+	struct exynos_drm_ipp		ipp;
+	struct drm_device		*drm_dev;
+	struct device			*dev;
+	void __iomem			*regs;
+	struct clk			*clock[SCALER_MAX_CLK];
+	struct exynos_drm_ipp_task	*task;
+	const struct scaler_data	*scaler_data;
+};
+
+static u32 scaler_get_format(u32 drm_fmt)
+{
+	switch (drm_fmt) {
+	case DRM_FORMAT_NV21:
+		return SCALER_YUV420_2P_UV;
+	case DRM_FORMAT_NV12:
+		return SCALER_YUV420_2P_VU;
+	case DRM_FORMAT_YUV420:
+		return SCALER_YUV420_3P;
+	case DRM_FORMAT_YUYV:
+		return SCALER_YUV422_1P_YUYV;
+	case DRM_FORMAT_UYVY:
+		return SCALER_YUV422_1P_UYVY;
+	case DRM_FORMAT_YVYU:
+		return SCALER_YUV422_1P_YVYU;
+	case DRM_FORMAT_NV61:
+		return SCALER_YUV422_2P_UV;
+	case DRM_FORMAT_NV16:
+		return SCALER_YUV422_2P_VU;
+	case DRM_FORMAT_YUV422:
+		return SCALER_YUV422_3P;
+	case DRM_FORMAT_NV42:
+		return SCALER_YUV444_2P_UV;
+	case DRM_FORMAT_NV24:
+		return SCALER_YUV444_2P_VU;
+	case DRM_FORMAT_YUV444:
+		return SCALER_YUV444_3P;
+	case DRM_FORMAT_RGB565:
+		return SCALER_RGB_565;
+	case DRM_FORMAT_XRGB1555:
+		return SCALER_ARGB1555;
+	case DRM_FORMAT_ARGB1555:
+		return SCALER_ARGB1555;
+	case DRM_FORMAT_XRGB4444:
+		return SCALER_ARGB4444;
+	case DRM_FORMAT_ARGB4444:
+		return SCALER_ARGB4444;
+	case DRM_FORMAT_XRGB8888:
+		return SCALER_ARGB8888;
+	case DRM_FORMAT_ARGB8888:
+		return SCALER_ARGB8888;
+	case DRM_FORMAT_RGBX8888:
+		return SCALER_RGBA8888;
+	case DRM_FORMAT_RGBA8888:
+		return SCALER_RGBA8888;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static inline void scaler_enable_int(struct scaler_context *scaler)
+{
+	u32 val;
+
+	val = SCALER_INT_EN_TIMEOUT |
+		SCALER_INT_EN_ILLEGAL_BLEND |
+		SCALER_INT_EN_ILLEGAL_RATIO |
+		SCALER_INT_EN_ILLEGAL_DST_HEIGHT |
+		SCALER_INT_EN_ILLEGAL_DST_WIDTH |
+		SCALER_INT_EN_ILLEGAL_DST_V_POS |
+		SCALER_INT_EN_ILLEGAL_DST_H_POS |
+		SCALER_INT_EN_ILLEGAL_DST_C_SPAN |
+		SCALER_INT_EN_ILLEGAL_DST_Y_SPAN |
+		SCALER_INT_EN_ILLEGAL_DST_CR_BASE |
+		SCALER_INT_EN_ILLEGAL_DST_CB_BASE |
+		SCALER_INT_EN_ILLEGAL_DST_Y_BASE |
+		SCALER_INT_EN_ILLEGAL_DST_COLOR |
+		SCALER_INT_EN_ILLEGAL_SRC_HEIGHT |
+		SCALER_INT_EN_ILLEGAL_SRC_WIDTH |
+		SCALER_INT_EN_ILLEGAL_SRC_CV_POS |
+		SCALER_INT_EN_ILLEGAL_SRC_CH_POS |
+		SCALER_INT_EN_ILLEGAL_SRC_YV_POS |
+		SCALER_INT_EN_ILLEGAL_SRC_YH_POS |
+		SCALER_INT_EN_ILLEGAL_DST_SPAN |
+		SCALER_INT_EN_ILLEGAL_SRC_Y_SPAN |
+		SCALER_INT_EN_ILLEGAL_SRC_CR_BASE |
+		SCALER_INT_EN_ILLEGAL_SRC_CB_BASE |
+		SCALER_INT_EN_ILLEGAL_SRC_Y_BASE |
+		SCALER_INT_EN_ILLEGAL_SRC_COLOR |
+		SCALER_INT_EN_FRAME_END;
+	scaler_write(val, SCALER_INT_EN);
+}
+
+static inline void scaler_set_src_fmt(struct scaler_context *scaler,
+	u32 src_fmt)
+{
+	u32 val;
+
+	val = SCALER_SRC_CFG_SET_COLOR_FORMAT(src_fmt);
+	scaler_write(val, SCALER_SRC_CFG);
+}
+
+static inline void scaler_set_src_base(struct scaler_context *scaler,
+	struct exynos_drm_ipp_buffer *src_buf)
+{
+	static unsigned int bases[] = {
+		SCALER_SRC_Y_BASE,
+		SCALER_SRC_CB_BASE,
+		SCALER_SRC_CR_BASE,
+	};
+	int i;
+
+	for (i = 0; i < src_buf->format->num_planes; ++i)
+		scaler_write(src_buf->dma_addr[i], bases[i]);
+}
+
+static inline void scaler_set_src_span(struct scaler_context *scaler,
+	struct exynos_drm_ipp_buffer *src_buf)
+{
+	u32 val;
+
+	val = SCALER_SRC_SPAN_SET_Y_SPAN(src_buf->buf.pitch[0] /
+		src_buf->format->cpp[0]);
+
+	if (src_buf->format->num_planes > 1)
+		val |= SCALER_SRC_SPAN_SET_C_SPAN(src_buf->buf.pitch[1]);
+
+	scaler_write(val, SCALER_SRC_SPAN);
+}
+
+static inline void scaler_set_src_luma_pos(struct scaler_context *scaler,
+	struct drm_exynos_ipp_task_rect *src_pos)
+{
+	u32 val;
+
+	val = SCALER_SRC_Y_POS_SET_YH_POS(src_pos->x << 2);
+	val |=  SCALER_SRC_Y_POS_SET_YV_POS(src_pos->y << 2);
+	scaler_write(val, SCALER_SRC_Y_POS);
+	scaler_write(val, SCALER_SRC_C_POS); /* ATTENTION! */
+}
+
+static inline void scaler_set_src_wh(struct scaler_context *scaler,
+	struct drm_exynos_ipp_task_rect *src_pos)
+{
+	u32 val;
+
+	val = SCALER_SRC_WH_SET_WIDTH(src_pos->w);
+	val |= SCALER_SRC_WH_SET_HEIGHT(src_pos->h);
+	scaler_write(val, SCALER_SRC_WH);
+}
+
+static inline void scaler_set_dst_fmt(struct scaler_context *scaler,
+	u32 dst_fmt)
+{
+	u32 val;
+
+	val = SCALER_DST_CFG_SET_COLOR_FORMAT(dst_fmt);
+	scaler_write(val, SCALER_DST_CFG);
+}
+
+static inline void scaler_set_dst_base(struct scaler_context *scaler,
+	struct exynos_drm_ipp_buffer *dst_buf)
+{
+	static unsigned int bases[] = {
+		SCALER_DST_Y_BASE,
+		SCALER_DST_CB_BASE,
+		SCALER_DST_CR_BASE,
+	};
+	int i;
+
+	for (i = 0; i < dst_buf->format->num_planes; ++i)
+		scaler_write(dst_buf->dma_addr[i], bases[i]);
+}
+
+static inline void scaler_set_dst_span(struct scaler_context *scaler,
+	struct exynos_drm_ipp_buffer *dst_buf)
+{
+	u32 val;
+
+	val = SCALER_DST_SPAN_SET_Y_SPAN(dst_buf->buf.pitch[0] /
+		dst_buf->format->cpp[0]);
+
+	if (dst_buf->format->num_planes > 1)
+		val |= SCALER_DST_SPAN_SET_C_SPAN(dst_buf->buf.pitch[1]);
+
+	scaler_write(val, SCALER_DST_SPAN);
+}
+
+static inline void scaler_set_dst_luma_pos(struct scaler_context *scaler,
+	struct drm_exynos_ipp_task_rect *dst_pos)
+{
+	u32 val;
+
+	val = SCALER_DST_WH_SET_WIDTH(dst_pos->w);
+	val |= SCALER_DST_WH_SET_HEIGHT(dst_pos->h);
+	scaler_write(val, SCALER_DST_WH);
+}
+
+static inline void scaler_set_dst_wh(struct scaler_context *scaler,
+	struct drm_exynos_ipp_task_rect *dst_pos)
+{
+	u32 val;
+
+	val = SCALER_DST_POS_SET_H_POS(dst_pos->x);
+	val |= SCALER_DST_POS_SET_V_POS(dst_pos->y);
+	scaler_write(val, SCALER_DST_POS);
+}
+
+static inline void scaler_set_hv_ratio(struct scaler_context *scaler,
+	unsigned int rotation,
+	struct drm_exynos_ipp_task_rect *src_pos,
+	struct drm_exynos_ipp_task_rect *dst_pos)
+{
+	u32 val, h_ratio, v_ratio;
+
+	if (drm_rotation_90_or_270(rotation)) {
+		h_ratio = (src_pos->h << 16) / dst_pos->w;
+		v_ratio = (src_pos->w << 16) / dst_pos->h;
+	} else {
+		h_ratio = (src_pos->w << 16) / dst_pos->w;
+		v_ratio = (src_pos->h << 16) / dst_pos->h;
+	}
+
+	val = SCALER_H_RATIO_SET(h_ratio);
+	scaler_write(val, SCALER_H_RATIO);
+
+	val = SCALER_V_RATIO_SET(v_ratio);
+	scaler_write(val, SCALER_V_RATIO);
+}
+
+static inline void scaler_set_rotation(struct scaler_context *scaler,
+	unsigned int rotation)
+{
+	u32 val = 0;
+
+	if (rotation & DRM_MODE_ROTATE_90)
+		val |= SCALER_ROT_CFG_SET_ROTMODE(SCALER_ROT_MODE_90);
+	else if (rotation & DRM_MODE_ROTATE_180)
+		val |= SCALER_ROT_CFG_SET_ROTMODE(SCALER_ROT_MODE_180);
+	else if (rotation & DRM_MODE_ROTATE_270)
+		val |= SCALER_ROT_CFG_SET_ROTMODE(SCALER_ROT_MODE_270);
+	if (rotation & DRM_MODE_REFLECT_X)
+		val |= SCALER_ROT_CFG_FLIP_X_EN;
+	if (rotation & DRM_MODE_REFLECT_Y)
+		val |= SCALER_ROT_CFG_FLIP_Y_EN;
+	scaler_write(val, SCALER_ROT_CFG);
+}
+
+static inline void scaler_set_csc(struct scaler_context *scaler,
+	const struct drm_format_info *fmt)
+{
+	static const u32 csc_mtx[2][3][3] = {
+		{ /* YCbCr to RGB */
+			{0x254, 0x000, 0x331},
+			{0x254, 0xf38, 0xe60},
+			{0x254, 0x409, 0x000},
+		},
+		{ /* RGB to YCbCr */
+			{0x084, 0x102, 0x032},
+			{0xfb4, 0xf6b, 0x0e1},
+			{0x0e1, 0xf44, 0xfdc},
+		},
+	};
+	int i, j, dir;
+
+	switch (fmt->format) {
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_XRGB1555:
+	case DRM_FORMAT_ARGB1555:
+	case DRM_FORMAT_XRGB4444:
+	case DRM_FORMAT_ARGB4444:
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_RGBA8888:
+		dir = 1;
+		break;
+	default:
+		dir = 0;
+	}
+
+	for (i = 0; i < 3; i++)
+		for (j = 0; j < 3; j++)
+			scaler_write(csc_mtx[dir][i][j], SCALER_CSC_COEF(j, i));
+}
+
+static inline void scaler_set_timer(struct scaler_context *scaler,
+	unsigned int timer, unsigned int divider)
+{
+	u32 val;
+
+	val = SCALER_TIMEOUT_CTRL_TIMER_ENABLE;
+	val |= SCALER_TIMEOUT_CTRL_SET_TIMER_VALUE(timer);
+	val |= SCALER_TIMEOUT_CTRL_SET_TIMER_DIV(divider);
+	scaler_write(val, SCALER_TIMEOUT_CTRL);
+}
+
+static inline void scaler_start_hw(struct scaler_context *scaler)
+{
+	scaler_write(SCALER_CFG_START_CMD, SCALER_CFG);
+}
+
+static int scaler_commit(struct exynos_drm_ipp *ipp,
+			  struct exynos_drm_ipp_task *task)
+{
+	struct scaler_context *scaler =
+			container_of(ipp, struct scaler_context, ipp);
+
+	u32 src_fmt = scaler_get_format(task->src.buf.fourcc);
+	struct drm_exynos_ipp_task_rect *src_pos = &task->src.rect;
+
+	u32 dst_fmt = scaler_get_format(task->dst.buf.fourcc);
+	struct drm_exynos_ipp_task_rect *dst_pos = &task->dst.rect;
+
+	scaler->task = task;
+
+	pm_runtime_get_sync(scaler->dev);
+
+	scaler_set_src_fmt(scaler, src_fmt);
+	scaler_set_src_base(scaler, &task->src);
+	scaler_set_src_span(scaler, &task->src);
+	scaler_set_src_luma_pos(scaler, src_pos);
+	scaler_set_src_wh(scaler, src_pos);
+
+	scaler_set_dst_fmt(scaler, dst_fmt);
+	scaler_set_dst_base(scaler, &task->dst);
+	scaler_set_dst_span(scaler, &task->dst);
+	scaler_set_dst_luma_pos(scaler, dst_pos);
+	scaler_set_dst_wh(scaler, dst_pos);
+
+	scaler_set_hv_ratio(scaler, task->transform.rotation, src_pos, dst_pos);
+	scaler_set_rotation(scaler, task->transform.rotation);
+
+	scaler_set_csc(scaler, task->src.format);
+
+	scaler_set_timer(scaler, 0xffff, 0xf);
+
+	scaler_enable_int(scaler);
+	scaler_start_hw(scaler);
+
+	return 0;
+}
+
+static struct exynos_drm_ipp_funcs ipp_funcs = {
+	.commit = scaler_commit,
+};
+
+static inline void scaler_disable_int(struct scaler_context *scaler)
+{
+	scaler_write(0, SCALER_INT_EN);
+}
+
+static inline u32 scaler_get_int_status(struct scaler_context *scaler)
+{
+	return scaler_read(SCALER_INT_STATUS);
+}
+
+static inline int scaler_task_done(u32 val)
+{
+	return val & SCALER_INT_STATUS_FRAME_END ? 0 : -EINVAL;
+}
+
+static irqreturn_t scaler_irq_handler(int irq, void *arg)
+{
+	struct scaler_context *scaler = arg;
+
+	u32 val = scaler_get_int_status(scaler);
+
+	scaler_disable_int(scaler);
+
+	if (scaler->task) {
+		struct exynos_drm_ipp_task *task = scaler->task;
+
+		scaler->task = NULL;
+		pm_runtime_mark_last_busy(scaler->dev);
+		pm_runtime_put_autosuspend(scaler->dev);
+		exynos_drm_ipp_task_done(task, scaler_task_done(val));
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int scaler_bind(struct device *dev, struct device *master, void *data)
+{
+	struct scaler_context *scaler = dev_get_drvdata(dev);
+	struct drm_device *drm_dev = data;
+	struct exynos_drm_ipp *ipp = &scaler->ipp;
+
+	scaler->drm_dev = drm_dev;
+	drm_iommu_attach_device(drm_dev, dev);
+
+	exynos_drm_ipp_register(drm_dev, ipp, &ipp_funcs,
+			DRM_EXYNOS_IPP_CAP_CROP | DRM_EXYNOS_IPP_CAP_ROTATE |
+			DRM_EXYNOS_IPP_CAP_SCALE | DRM_EXYNOS_IPP_CAP_CONVERT,
+			scaler->scaler_data->formats,
+			scaler->scaler_data->num_formats, "scaler");
+
+	dev_info(dev, "The exynos scaler has been probed successfully\n");
+
+	return 0;
+}
+
+static void scaler_unbind(struct device *dev, struct device *master,
+			void *data)
+{
+	struct scaler_context *scaler = dev_get_drvdata(dev);
+	struct drm_device *drm_dev = data;
+	struct exynos_drm_ipp *ipp = &scaler->ipp;
+
+	exynos_drm_ipp_unregister(drm_dev, ipp);
+	drm_iommu_detach_device(scaler->drm_dev, scaler->dev);
+}
+
+static const struct component_ops scaler_component_ops = {
+	.bind	= scaler_bind,
+	.unbind = scaler_unbind,
+};
+
+static int scaler_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource	*regs_res;
+	struct scaler_context *scaler;
+	int irq;
+	int ret, i;
+
+	scaler = devm_kzalloc(dev, sizeof(*scaler), GFP_KERNEL);
+	if (!scaler)
+		return -ENOMEM;
+
+	scaler->scaler_data =
+		(struct scaler_data *)of_device_get_match_data(dev);
+
+	scaler->dev = dev;
+	regs_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	scaler->regs = devm_ioremap_resource(dev, regs_res);
+	if (IS_ERR(scaler->regs))
+		return PTR_ERR(scaler->regs);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(dev, "failed to get irq\n");
+		return irq;
+	}
+
+	ret = devm_request_threaded_irq(dev, irq, NULL,	scaler_irq_handler,
+					IRQF_ONESHOT, "drm_scaler", scaler);
+	if (ret < 0) {
+		dev_err(dev, "failed to request irq\n");
+		return ret;
+	}
+
+	for (i = 0; i < scaler->scaler_data->num_clk; ++i) {
+		scaler->clock[i] = devm_clk_get(dev,
+					      scaler->scaler_data->clk_name[i]);
+		if (IS_ERR(scaler->clock[i])) {
+			dev_err(dev, "failed to get clock\n");
+			return PTR_ERR(scaler->clock[i]);
+		}
+	}
+
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_autosuspend_delay(dev, SCALER_AUTOSUSPEND_DELAY);
+	pm_runtime_enable(dev);
+	platform_set_drvdata(pdev, scaler);
+
+	ret = component_add(dev, &scaler_component_ops);
+	if (ret)
+		goto err_ippdrv_register;
+
+	return 0;
+
+err_ippdrv_register:
+	pm_runtime_dont_use_autosuspend(dev);
+	pm_runtime_disable(dev);
+	return ret;
+}
+
+static int scaler_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	component_del(dev, &scaler_component_ops);
+	pm_runtime_dont_use_autosuspend(dev);
+	pm_runtime_disable(dev);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+
+static int clk_disable_unprepare_wrapper(struct clk *clk)
+{
+	clk_disable_unprepare(clk);
+
+	return 0;
+}
+
+static int scaler_clk_ctrl(struct scaler_context *scaler, bool enable)
+{
+	int (*clk_fun)(struct clk *clk), i;
+
+	clk_fun = enable ? clk_prepare_enable : clk_disable_unprepare_wrapper;
+
+	for (i = 0; i < scaler->scaler_data->num_clk; ++i)
+		clk_fun(scaler->clock[i]);
+
+	return 0;
+}
+
+static int scaler_runtime_suspend(struct device *dev)
+{
+	struct scaler_context *scaler = dev_get_drvdata(dev);
+
+	return  scaler_clk_ctrl(scaler, false);
+}
+
+static int scaler_runtime_resume(struct device *dev)
+{
+	struct scaler_context *scaler = dev_get_drvdata(dev);
+
+	return  scaler_clk_ctrl(scaler, true);
+}
+#endif
+
+static const struct dev_pm_ops scaler_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(scaler_runtime_suspend, scaler_runtime_resume, NULL)
+};
+
+static const struct drm_exynos_ipp_limit scaler_5420_two_pixel_hv_limits[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 16, SZ_8K }, .v = { 16, SZ_8K }) },
+	{ IPP_SIZE_LIMIT(AREA, .h.align = 2, .v.align = 2) },
+	{ IPP_SCALE_LIMIT(.h = { 65536 * 1 / 4, 65536 * 16 },
+			  .v = { 65536 * 1 / 4, 65536 * 16 }) },
+};
+
+static const struct drm_exynos_ipp_limit scaler_5420_two_pixel_h_limits[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 16, SZ_8K }, .v = { 16, SZ_8K }) },
+	{ IPP_SIZE_LIMIT(AREA, .h.align = 2, .v.align = 1) },
+	{ IPP_SCALE_LIMIT(.h = { 65536 * 1 / 4, 65536 * 16 },
+			  .v = { 65536 * 1 / 4, 65536 * 16 }) },
+};
+
+static const struct drm_exynos_ipp_limit scaler_5420_one_pixel_limits[] = {
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 16, SZ_8K }, .v = { 16, SZ_8K }) },
+	{ IPP_SCALE_LIMIT(.h = { 65536 * 1 / 4, 65536 * 16 },
+			  .v = { 65536 * 1 / 4, 65536 * 16 }) },
+};
+
+static const struct exynos_drm_ipp_formats exynos5420_formats[] = {
+	/* SCALER_YUV420_2P_UV */
+	{ IPP_SRCDST_FORMAT(NV21, scaler_5420_two_pixel_hv_limits) },
+
+	/* SCALER_YUV420_2P_VU */
+	{ IPP_SRCDST_FORMAT(NV12, scaler_5420_two_pixel_hv_limits) },
+
+	/* SCALER_YUV420_3P */
+	{ IPP_SRCDST_FORMAT(YUV420, scaler_5420_two_pixel_hv_limits) },
+
+	/* SCALER_YUV422_1P_YUYV */
+	{ IPP_SRCDST_FORMAT(YUYV, scaler_5420_two_pixel_h_limits) },
+
+	/* SCALER_YUV422_1P_UYVY */
+	{ IPP_SRCDST_FORMAT(UYVY, scaler_5420_two_pixel_h_limits) },
+
+	/* SCALER_YUV422_1P_YVYU */
+	{ IPP_SRCDST_FORMAT(YVYU, scaler_5420_two_pixel_h_limits) },
+
+	/* SCALER_YUV422_2P_UV */
+	{ IPP_SRCDST_FORMAT(NV61, scaler_5420_two_pixel_h_limits) },
+
+	/* SCALER_YUV422_2P_VU */
+	{ IPP_SRCDST_FORMAT(NV16, scaler_5420_two_pixel_h_limits) },
+
+	/* SCALER_YUV422_3P */
+	{ IPP_SRCDST_FORMAT(YUV422, scaler_5420_two_pixel_h_limits) },
+
+	/* SCALER_YUV444_2P_UV */
+	{ IPP_SRCDST_FORMAT(NV42, scaler_5420_one_pixel_limits) },
+
+	/* SCALER_YUV444_2P_VU */
+	{ IPP_SRCDST_FORMAT(NV24, scaler_5420_one_pixel_limits) },
+
+	/* SCALER_YUV444_3P */
+	{ IPP_SRCDST_FORMAT(YUV444, scaler_5420_one_pixel_limits) },
+
+	/* SCALER_RGB_565 */
+	{ IPP_SRCDST_FORMAT(RGB565, scaler_5420_one_pixel_limits) },
+
+	/* SCALER_ARGB1555 */
+	{ IPP_SRCDST_FORMAT(XRGB1555, scaler_5420_one_pixel_limits) },
+
+	/* SCALER_ARGB1555 */
+	{ IPP_SRCDST_FORMAT(ARGB1555, scaler_5420_one_pixel_limits) },
+
+	/* SCALER_ARGB4444 */
+	{ IPP_SRCDST_FORMAT(XRGB4444, scaler_5420_one_pixel_limits) },
+
+	/* SCALER_ARGB4444 */
+	{ IPP_SRCDST_FORMAT(ARGB4444, scaler_5420_one_pixel_limits) },
+
+	/* SCALER_ARGB8888 */
+	{ IPP_SRCDST_FORMAT(XRGB8888, scaler_5420_one_pixel_limits) },
+
+	/* SCALER_ARGB8888 */
+	{ IPP_SRCDST_FORMAT(ARGB8888, scaler_5420_one_pixel_limits) },
+
+	/* SCALER_RGBA8888 */
+	{ IPP_SRCDST_FORMAT(RGBX8888, scaler_5420_one_pixel_limits) },
+
+	/* SCALER_RGBA8888 */
+	{ IPP_SRCDST_FORMAT(RGBA8888, scaler_5420_one_pixel_limits) },
+};
+
+static const struct scaler_data exynos5420_data = {
+	.clk_name	= {"mscl"},
+	.num_clk	= 1,
+	.formats	= exynos5420_formats,
+	.num_formats	= ARRAY_SIZE(exynos5420_formats),
+};
+
+static const struct scaler_data exynos5433_data = {
+	.clk_name	= {"pclk", "aclk", "aclk_xiu"},
+	.num_clk	= 3,
+	.formats	= exynos5420_formats, /* intentional */
+	.num_formats	= ARRAY_SIZE(exynos5420_formats),
+};
+
+static const struct of_device_id exynos_scaler_match[] = {
+	{
+		.compatible = "samsung,exynos5420-scaler",
+		.data = &exynos5420_data,
+	}, {
+		.compatible = "samsung,exynos5433-scaler",
+		.data = &exynos5433_data,
+	}, {
+	},
+};
+MODULE_DEVICE_TABLE(of, exynos_scaler_match);
+
+struct platform_driver scaler_driver = {
+	.probe		= scaler_probe,
+	.remove		= scaler_remove,
+	.driver		= {
+		.name	= "exynos-scaler",
+		.owner	= THIS_MODULE,
+		.pm	= &scaler_pm_ops,
+		.of_match_table = exynos_scaler_match,
+	},
+};
diff --git a/drivers/gpu/drm/exynos/regs-scaler.h b/drivers/gpu/drm/exynos/regs-scaler.h
new file mode 100644
index 000000000000..fc7ccad75e74
--- /dev/null
+++ b/drivers/gpu/drm/exynos/regs-scaler.h
@@ -0,0 +1,426 @@
+/* drivers/gpu/drm/exynos/regs-scaler.h
+ *
+ * Copyright (c) 2017 Samsung Electronics Co., Ltd.
+ *		http://www.samsung.com/
+ * Author: Andrzej Pietrasiewicz <andrzej.p@samsung.com>
+ *
+ * Register definition file for Samsung scaler driver
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef EXYNOS_REGS_SCALER_H
+#define EXYNOS_REGS_SCALER_H
+
+/* Register part */
+
+/* Global setting */
+#define SCALER_STATUS			0x0	/* no shadow */
+#define SCALER_CFG			0x4
+
+/* Interrupt */
+#define SCALER_INT_EN			0x8	/* no shadow */
+#define SCALER_INT_STATUS		0xc	/* no shadow */
+
+/* SRC */
+#define SCALER_SRC_CFG			0x10
+#define SCALER_SRC_Y_BASE		0x14
+#define SCALER_SRC_CB_BASE		0x18
+#define SCALER_SRC_CR_BASE		0x294
+#define SCALER_SRC_SPAN			0x1c
+#define SCALER_SRC_Y_POS		0x20
+#define SCALER_SRC_WH			0x24
+#define SCALER_SRC_C_POS		0x28
+
+/* DST */
+#define SCALER_DST_CFG			0x30
+#define SCALER_DST_Y_BASE		0x34
+#define SCALER_DST_CB_BASE		0x38
+#define SCALER_DST_CR_BASE		0x298
+#define SCALER_DST_SPAN			0x3c
+#define SCALER_DST_WH			0x40
+#define SCALER_DST_POS			0x44
+
+/* Ratio */
+#define SCALER_H_RATIO			0x50
+#define SCALER_V_RATIO			0x54
+
+/* Rotation */
+#define SCALER_ROT_CFG			0x58
+
+/* Coefficient */
+/*
+ * YHCOEF_{x}{A|B|C|D}			CHCOEF_{x}{A|B|C|D}
+ *
+ *	A	B	C	D	A	B	C	D
+ * 0	60	64	68	6c	140	144	148	14c
+ * 1	70	74	78	7c	150	154	158	15c
+ * 2	80	84	88	8c	160	164	168	16c
+ * 3	90	94	98	9c	170	174	178	17c
+ * 4	a0	a4	a8	ac	180	184	188	18c
+ * 5	b0	b4	b8	bc	190	194	198	19c
+ * 6	c0	c4	c8	cc	1a0	1a4	1a8	1ac
+ * 7	d0	d4	d8	dc	1b0	1b4	1b8	1bc
+ * 8	e0	e4	e8	ec	1c0	1c4	1c8	1cc
+ *
+ *
+ * YVCOEF_{x}{A|B}			CVCOEF_{x}{A|B}
+ *
+ *	A	B			A	B
+ * 0	f0	f4			1d0	1d4
+ * 1	f8	fc			1d8	1dc
+ * 2	100	104			1e0	1e4
+ * 3	108	10c			1e8	1ec
+ * 4	110	114			1f0	1f4
+ * 5	118	11c			1f8	1fc
+ * 6	120	124			200	204
+ * 7	128	12c			208	20c
+ * 8	130	134			210	214
+ */
+#define _SCALER_HCOEF_DELTA(r, c)	((r) * 0x10 + (c) * 0x4)
+#define _SCALER_VCOEF_DELTA(r, c)	((r) * 0x8 + (c) * 0x4)
+
+#define SCALER_YHCOEF(r, c)		(0x60 + _SCALER_HCOEF_DELTA((r), (c)))
+#define SCALER_YVCOEF(r, c)		(0xf0 + _SCALER_VCOEF_DELTA((r), (c)))
+#define SCALER_CHCOEF(r, c)		(0x140 + _SCALER_HCOEF_DELTA((r), (c)))
+#define SCALER_CVCOEF(r, c)		(0x1d0 + _SCALER_VCOEF_DELTA((r), (c)))
+
+
+/* Color Space Conversion */
+#define SCALER_CSC_COEF(x, y)		(0x220 + (y) * 0xc + (x) * 0x4)
+
+/* Dithering */
+#define SCALER_DITH_CFG			0x250
+
+/* Version Number */
+#define SCALER_VER			0x260	/* no shadow */
+
+/* Cycle count and Timeout */
+#define SCALER_CYCLE_COUNT		0x278	/* no shadow */
+#define SCALER_TIMEOUT_CTRL		0x2c0	/* no shadow */
+#define SCALER_TIMEOUT_CNT		0x2c4	/* no shadow */
+
+/* Blending */
+#define SCALER_SRC_BLEND_COLOR		0x280
+#define SCALER_SRC_BLEND_ALPHA		0x284
+#define SCALER_DST_BLEND_COLOR		0x288
+#define SCALER_DST_BLEND_ALPHA		0x28c
+
+/* Color Fill */
+#define SCALER_FILL_COLOR		0x290
+
+/* Multiple Command Queue */
+#define SCALER_ADDR_Q_CONFIG		0x2a0	/* no shadow */
+#define SCALER_SRC_ADDR_Q_STATUS	0x2a4	/* no shadow */
+#define SCALER_SRC_ADDR_Q		0x2a8	/* no shadow */
+
+/* CRC */
+#define SCALER_CRC_COLOR00_10		0x2b0	/* no shadow */
+#define SCALER_CRC_COLOR20_30		0x2b4	/* no shadow */
+#define SCALER_CRC_COLOR01_11		0x2b8	/* no shadow */
+#define SCALER_CRC_COLOR21_31		0x2bc	/* no shadow */
+
+/* Shadow Registers */
+#define SCALER_SHADOW_OFFSET		0x1000
+
+
+/* Bit definition part */
+#define SCALER_MASK(hi_b, lo_b)		((1 << ((hi_b) - (lo_b) + 1)) - 1)
+#define SCALER_GET(reg, hi_b, lo_b)	\
+	(((reg) >> (lo_b)) & SCALER_MASK(hi_b, lo_b))
+#define SCALER_SET(val, hi_b, lo_b) \
+	(((val) & SCALER_MASK(hi_b, lo_b)) << lo_b)
+
+/* SCALER_STATUS */
+#define SCALER_STATUS_SCALER_RUNNING		(1 << 1)
+#define SCALER_STATUS_SCALER_READY_CLK_DOWN	(1 << 0)
+
+/* SCALER_CFG */
+#define SCALER_CFG_FILL_EN			(1 << 24)
+#define SCALER_CFG_BLEND_COLOR_DIVIDE_ALPHA_EN	(1 << 17)
+#define SCALER_CFG_BLEND_EN			(1 << 16)
+#define SCALER_CFG_CSC_Y_OFFSET_SRC_EN		(1 << 10)
+#define SCALER_CFG_CSC_Y_OFFSET_DST_EN		(1 << 9)
+#define SCALER_CFG_16_BURST_MODE		(1 << 8)
+#define SCALER_CFG_SOFT_RESET			(1 << 1)
+#define SCALER_CFG_START_CMD			(1 << 0)
+
+/* SCALER_INT_EN */
+#define SCALER_INT_EN_TIMEOUT			(1 << 31)
+#define SCALER_INT_EN_ILLEGAL_BLEND		(1 << 24)
+#define SCALER_INT_EN_ILLEGAL_RATIO		(1 << 23)
+#define SCALER_INT_EN_ILLEGAL_DST_HEIGHT	(1 << 22)
+#define SCALER_INT_EN_ILLEGAL_DST_WIDTH		(1 << 21)
+#define SCALER_INT_EN_ILLEGAL_DST_V_POS		(1 << 20)
+#define SCALER_INT_EN_ILLEGAL_DST_H_POS		(1 << 19)
+#define SCALER_INT_EN_ILLEGAL_DST_C_SPAN	(1 << 18)
+#define SCALER_INT_EN_ILLEGAL_DST_Y_SPAN	(1 << 17)
+#define SCALER_INT_EN_ILLEGAL_DST_CR_BASE	(1 << 16)
+#define SCALER_INT_EN_ILLEGAL_DST_CB_BASE	(1 << 15)
+#define SCALER_INT_EN_ILLEGAL_DST_Y_BASE	(1 << 14)
+#define SCALER_INT_EN_ILLEGAL_DST_COLOR		(1 << 13)
+#define SCALER_INT_EN_ILLEGAL_SRC_HEIGHT	(1 << 12)
+#define SCALER_INT_EN_ILLEGAL_SRC_WIDTH		(1 << 11)
+#define SCALER_INT_EN_ILLEGAL_SRC_CV_POS	(1 << 10)
+#define SCALER_INT_EN_ILLEGAL_SRC_CH_POS	(1 << 9)
+#define SCALER_INT_EN_ILLEGAL_SRC_YV_POS	(1 << 8)
+#define SCALER_INT_EN_ILLEGAL_SRC_YH_POS	(1 << 7)
+#define SCALER_INT_EN_ILLEGAL_DST_SPAN		(1 << 6)
+#define SCALER_INT_EN_ILLEGAL_SRC_Y_SPAN	(1 << 5)
+#define SCALER_INT_EN_ILLEGAL_SRC_CR_BASE	(1 << 4)
+#define SCALER_INT_EN_ILLEGAL_SRC_CB_BASE	(1 << 3)
+#define SCALER_INT_EN_ILLEGAL_SRC_Y_BASE	(1 << 2)
+#define SCALER_INT_EN_ILLEGAL_SRC_COLOR		(1 << 1)
+#define SCALER_INT_EN_FRAME_END			(1 << 0)
+
+/* SCALER_INT_STATUS */
+#define SCALER_INT_STATUS_TIMEOUT		(1 << 31)
+#define SCALER_INT_STATUS_ILLEGAL_BLEND		(1 << 24)
+#define SCALER_INT_STATUS_ILLEGAL_RATIO		(1 << 23)
+#define SCALER_INT_STATUS_ILLEGAL_DST_HEIGHT	(1 << 22)
+#define SCALER_INT_STATUS_ILLEGAL_DST_WIDTH	(1 << 21)
+#define SCALER_INT_STATUS_ILLEGAL_DST_V_POS	(1 << 20)
+#define SCALER_INT_STATUS_ILLEGAL_DST_H_POS	(1 << 19)
+#define SCALER_INT_STATUS_ILLEGAL_DST_C_SPAN	(1 << 18)
+#define SCALER_INT_STATUS_ILLEGAL_DST_Y_SPAN	(1 << 17)
+#define SCALER_INT_STATUS_ILLEGAL_DST_CR_BASE	(1 << 16)
+#define SCALER_INT_STATUS_ILLEGAL_DST_CB_BASE	(1 << 15)
+#define SCALER_INT_STATUS_ILLEGAL_DST_Y_BASE	(1 << 14)
+#define SCALER_INT_STATUS_ILLEGAL_DST_COLOR	(1 << 13)
+#define SCALER_INT_STATUS_ILLEGAL_SRC_HEIGHT	(1 << 12)
+#define SCALER_INT_STATUS_ILLEGAL_SRC_WIDTH	(1 << 11)
+#define SCALER_INT_STATUS_ILLEGAL_SRC_CV_POS	(1 << 10)
+#define SCALER_INT_STATUS_ILLEGAL_SRC_CH_POS	(1 << 9)
+#define SCALER_INT_STATUS_ILLEGAL_SRC_YV_POS	(1 << 8)
+#define SCALER_INT_STATUS_ILLEGAL_SRC_YH_POS	(1 << 7)
+#define SCALER_INT_STATUS_ILLEGAL_DST_SPAN	(1 << 6)
+#define SCALER_INT_STATUS_ILLEGAL_SRC_Y_SPAN	(1 << 5)
+#define SCALER_INT_STATUS_ILLEGAL_SRC_CR_BASE	(1 << 4)
+#define SCALER_INT_STATUS_ILLEGAL_SRC_CB_BASE	(1 << 3)
+#define SCALER_INT_STATUS_ILLEGAL_SRC_Y_BASE	(1 << 2)
+#define SCALER_INT_STATUS_ILLEGAL_SRC_COLOR	(1 << 1)
+#define SCALER_INT_STATUS_FRAME_END		(1 << 0)
+
+/* SCALER_SRC_CFG */
+#define SCALER_SRC_CFG_TILE_EN			(1 << 10)
+#define SCALER_SRC_CFG_GET_BYTE_SWAP(r)		SCALER_GET(r, 6, 5)
+#define SCALER_SRC_CFG_SET_BYTE_SWAP(v)		SCALER_SET(v, 6, 5)
+#define SCALER_SRC_CFG_GET_COLOR_FORMAT(r)	SCALER_GET(r, 4, 0)
+#define SCALER_SRC_CFG_SET_COLOR_FORMAT(v)	SCALER_SET(v, 4, 0)
+#define SCALER_YUV420_2P_UV			0
+#define SCALER_YUV422_2P_UV			2
+#define SCALER_YUV444_2P_UV			3
+#define SCALER_RGB_565				4
+#define SCALER_ARGB1555				5
+#define SCALER_ARGB8888				6
+#define SCALER_ARGB8888_PRE			7
+#define SCALER_YUV422_1P_YVYU			9
+#define SCALER_YUV422_1P_YUYV			10
+#define SCALER_YUV422_1P_UYVY			11
+#define SCALER_ARGB4444				12
+#define SCALER_L8A8				13
+#define SCALER_RGBA8888				14
+#define SCALER_L8				15
+#define SCALER_YUV420_2P_VU			16
+#define SCALER_YUV422_2P_VU			18
+#define SCALER_YUV444_2P_VU			19
+#define SCALER_YUV420_3P			20
+#define SCALER_YUV422_3P			22
+#define SCALER_YUV444_3P			23
+
+/* SCALER_SRC_SPAN */
+#define SCALER_SRC_SPAN_GET_C_SPAN(r)		SCALER_GET(r, 29, 16)
+#define SCALER_SRC_SPAN_SET_C_SPAN(v)		SCALER_SET(v, 29, 16)
+#define SCALER_SRC_SPAN_GET_Y_SPAN(r)		SCALER_GET(r, 13, 0)
+#define SCALER_SRC_SPAN_SET_Y_SPAN(v)		SCALER_SET(v, 13, 0)
+
+/* SCALER_SRC_Y_POS */
+#define SCALER_SRC_Y_POS_GET_YH_POS(r)		SCALER_GET(r, 31, 16)
+#define SCALER_SRC_Y_POS_SET_YH_POS(v)		SCALER_SET(v, 31, 16)
+#define SCALER_SRC_Y_POS_GET_YV_POS(r)		SCALER_GET(r, 15, 0)
+#define SCALER_SRC_Y_POS_SET_YV_POS(v)		SCALER_SET(v, 15, 0)
+
+/* SCALER_SRC_WH */
+#define SCALER_SRC_WH_GET_WIDTH(r)		SCALER_GET(r, 29, 16)
+#define SCALER_SRC_WH_SET_WIDTH(v)		SCALER_SET(v, 29, 16)
+#define SCALER_SRC_WH_GET_HEIGHT(r)		SCALER_GET(r, 13, 0)
+#define SCALER_SRC_WH_SET_HEIGHT(v)		SCALER_SET(v, 13, 0)
+
+/* SCALER_SRC_C_POS */
+#define SCALER_SRC_C_POS_GET_CH_POS(r)		SCALER_GET(r, 31, 16)
+#define SCALER_SRC_C_POS_SET_CH_POS(v)		SCALER_SET(v, 31, 16)
+#define SCALER_SRC_C_POS_GET_CV_POS(r)		SCALER_GET(r, 15, 0)
+#define SCALER_SRC_C_POS_SET_CV_POS(v)		SCALER_SET(v, 15, 0)
+
+/* SCALER_DST_CFG */
+#define SCALER_DST_CFG_GET_BYTE_SWAP(r)		SCALER_GET(r, 6, 5)
+#define SCALER_DST_CFG_SET_BYTE_SWAP(v)		SCALER_SET(v, 6, 5)
+#define SCALER_DST_CFG_GET_COLOR_FORMAT(r)	SCALER_GET(r, 4, 0)
+#define SCALER_DST_CFG_SET_COLOR_FORMAT(v)	SCALER_SET(v, 4, 0)
+
+/* SCALER_DST_SPAN */
+#define SCALER_DST_SPAN_GET_C_SPAN(r)		SCALER_GET(r, 29, 16)
+#define SCALER_DST_SPAN_SET_C_SPAN(v)		SCALER_SET(v, 29, 16)
+#define SCALER_DST_SPAN_GET_Y_SPAN(r)		SCALER_GET(r, 13, 0)
+#define SCALER_DST_SPAN_SET_Y_SPAN(v)		SCALER_SET(v, 13, 0)
+
+/* SCALER_DST_WH */
+#define SCALER_DST_WH_GET_WIDTH(r)		SCALER_GET(r, 29, 16)
+#define SCALER_DST_WH_SET_WIDTH(v)		SCALER_SET(v, 29, 16)
+#define SCALER_DST_WH_GET_HEIGHT(r)		SCALER_GET(r, 13, 0)
+#define SCALER_DST_WH_SET_HEIGHT(v)		SCALER_SET(v, 13, 0)
+
+/* SCALER_DST_POS */
+#define SCALER_DST_POS_GET_H_POS(r)		SCALER_GET(r, 29, 16)
+#define SCALER_DST_POS_SET_H_POS(v)		SCALER_SET(v, 29, 16)
+#define SCALER_DST_POS_GET_V_POS(r)		SCALER_GET(r, 13, 0)
+#define SCALER_DST_POS_SET_V_POS(v)		SCALER_SET(v, 13, 0)
+
+/* SCALER_H_RATIO */
+#define SCALER_H_RATIO_GET(r)			SCALER_GET(r, 18, 0)
+#define SCALER_H_RATIO_SET(v)			SCALER_SET(v, 18, 0)
+
+/* SCALER_V_RATIO */
+#define SCALER_V_RATIO_GET(r)			SCALER_GET(r, 18, 0)
+#define SCALER_V_RATIO_SET(v)			SCALER_SET(v, 18, 0)
+
+/* SCALER_ROT_CFG */
+#define SCALER_ROT_CFG_FLIP_X_EN		(1 << 3)
+#define SCALER_ROT_CFG_FLIP_Y_EN		(1 << 2)
+#define SCALER_ROT_CFG_GET_ROTMODE(r)		SCALER_GET(r, 1, 0)
+#define SCALER_ROT_CFG_SET_ROTMODE(v)		SCALER_SET(v, 1, 0)
+#define SCALER_ROT_MODE_90			1
+#define SCALER_ROT_MODE_180			2
+#define SCALER_ROT_MODE_270			3
+
+/* SCALER_HCOEF, SCALER_VCOEF */
+#define SCALER_COEF_SHIFT(i)			(16 * (1 - (i) % 2))
+#define SCALER_COEF_GET(r, i)			\
+				(((r) >> SCALER_COEF_SHIFT(i)) & 0x1ff)
+#define SCALER_COEF_SET(v, i)			\
+				(((v) & 0x1ff) << SCALER_COEF_SHIFT(i))
+
+/* SCALER_CSC_COEFxy */
+#define SCALER_CSC_COEF_GET(r)			SCALER_GET(r, 11, 0)
+#define SCALER_CSC_COEF_SET(v)			SCALER_SET(v, 11, 0)
+
+/* SCALER_DITH_CFG */
+#define SCALER_DITH_CFG_GET_R_TYPE(r)		SCALER_GET(r, 8, 6)
+#define SCALER_DITH_CFG_SET_R_TYPE(v)		SCALER_SET(v, 8, 6)
+#define SCALER_DITH_CFG_GET_G_TYPE(r)		SCALER_GET(r, 5, 3)
+#define SCALER_DITH_CFG_SET_G_TYPE(v)		SCALER_SET(v, 5, 3)
+#define SCALER_DITH_CFG_GET_B_TYPE(r)		SCALER_GET(r, 2, 0)
+#define SCALER_DITH_CFG_SET_B_TYPE(v)		SCALER_SET(v, 2, 0)
+
+/* SCALER_TIMEOUT_CTRL */
+#define SCALER_TIMEOUT_CTRL_GET_TIMER_VALUE(r)	SCALER_GET(r, 31, 16)
+#define SCALER_TIMEOUT_CTRL_SET_TIMER_VALUE(v)	SCALER_SET(v, 31, 16)
+#define SCALER_TIMEOUT_CTRL_GET_TIMER_DIV(r)	SCALER_GET(r, 7, 4)
+#define SCALER_TIMEOUT_CTRL_SET_TIMER_DIV(v)	SCALER_SET(v, 7, 4)
+#define SCALER_TIMEOUT_CTRL_TIMER_ENABLE	(1 << 0)
+
+/* SCALER_TIMEOUT_CNT */
+#define SCALER_TIMEOUT_CTRL_GET_TIMER_COUNT(r)	SCALER_GET(r, 31, 16)
+
+/* SCALER_SRC_BLEND_COLOR */
+#define SCALER_SRC_BLEND_COLOR_SEL_INV		(1 << 31)
+#define SCALER_SRC_BLEND_COLOR_GET_SEL(r)	SCALER_GET(r, 30, 29)
+#define SCALER_SRC_BLEND_COLOR_SET_SEL(v)	SCALER_SET(v, 30, 29)
+#define SCALER_SRC_BLEND_COLOR_OP_SEL_INV	(1 << 28)
+#define SCALER_SRC_BLEND_COLOR_GET_OP_SEL(r)	SCALER_GET(r, 27, 24)
+#define SCALER_SRC_BLEND_COLOR_SET_OP_SEL(v)	SCALER_SET(v, 27, 24)
+#define SCALER_SRC_BLEND_COLOR_GET_COLOR0(r)	SCALER_GET(r, 23, 16)
+#define SCALER_SRC_BLEND_COLOR_SET_COLOR0(v)	SCALER_SET(v, 23, 16)
+#define SCALER_SRC_BLEND_COLOR_GET_COLOR1(r)	SCALER_GET(r, 15, 8)
+#define SCALER_SRC_BLEND_COLOR_SET_COLOR1(v)	SCALER_SET(v, 15, 8)
+#define SCALER_SRC_BLEND_COLOR_GET_COLOR2(r)	SCALER_GET(r, 7, 0)
+#define SCALER_SRC_BLEND_COLOR_SET_COLOR2(v)	SCALER_SET(v, 7, 0)
+
+/* SCALER_SRC_BLEND_ALPHA */
+#define SCALER_SRC_BLEND_ALPHA_SEL_INV		(1 << 31)
+#define SCALER_SRC_BLEND_ALPHA_GET_SEL(r)	SCALER_GET(r, 30, 29)
+#define SCALER_SRC_BLEND_ALPHA_SET_SEL(v)	SCALER_SET(v, 30, 29)
+#define SCALER_SRC_BLEND_ALPHA_OP_SEL_INV	(1 << 28)
+#define SCALER_SRC_BLEND_ALPHA_GET_OP_SEL(r)	SCALER_GET(r, 27, 24)
+#define SCALER_SRC_BLEND_ALPHA_SET_OP_SEL(v)	SCALER_SET(v, 27, 24)
+#define SCALER_SRC_BLEND_ALPHA_GET_ALPHA(r)	SCALER_GET(r, 7, 0)
+#define SCALER_SRC_BLEND_ALPHA_SET_ALPHA(v)	SCALER_SET(v, 7, 0)
+
+/* SCALER_DST_BLEND_COLOR */
+#define SCALER_DST_BLEND_COLOR_SEL_INV		(1 << 31)
+#define SCALER_DST_BLEND_COLOR_GET_SEL(r)	SCALER_GET(r, 30, 29)
+#define SCALER_DST_BLEND_COLOR_SET_SEL(v)	SCALER_SET(v, 30, 29)
+#define SCALER_DST_BLEND_COLOR_OP_SEL_INV	(1 << 28)
+#define SCALER_DST_BLEND_COLOR_GET_OP_SEL(r)	SCALER_GET(r, 27, 24)
+#define SCALER_DST_BLEND_COLOR_SET_OP_SEL(v)	SCALER_SET(v, 27, 24)
+#define SCALER_DST_BLEND_COLOR_GET_COLOR0(r)	SCALER_GET(r, 23, 16)
+#define SCALER_DST_BLEND_COLOR_SET_COLOR0(v)	SCALER_SET(v, 23, 16)
+#define SCALER_DST_BLEND_COLOR_GET_COLOR1(r)	SCALER_GET(r, 15, 8)
+#define SCALER_DST_BLEND_COLOR_SET_COLOR1(v)	SCALER_SET(v, 15, 8)
+#define SCALER_DST_BLEND_COLOR_GET_COLOR2(r)	SCALER_GET(r, 7, 0)
+#define SCALER_DST_BLEND_COLOR_SET_COLOR2(v)	SCALER_SET(v, 7, 0)
+
+/* SCALER_DST_BLEND_ALPHA */
+#define SCALER_DST_BLEND_ALPHA_SEL_INV		(1 << 31)
+#define SCALER_DST_BLEND_ALPHA_GET_SEL(r)	SCALER_GET(r, 30, 29)
+#define SCALER_DST_BLEND_ALPHA_SET_SEL(v)	SCALER_SET(v, 30, 29)
+#define SCALER_DST_BLEND_ALPHA_OP_SEL_INV	(1 << 28)
+#define SCALER_DST_BLEND_ALPHA_GET_OP_SEL(r)	SCALER_GET(r, 27, 24)
+#define SCALER_DST_BLEND_ALPHA_SET_OP_SEL(v)	SCALER_SET(v, 27, 24)
+#define SCALER_DST_BLEND_ALPHA_GET_ALPHA(r)	SCALER_GET(r, 7, 0)
+#define SCALER_DST_BLEND_ALPHA_SET_ALPHA(v)	SCALER_SET(v, 7, 0)
+
+/* SCALER_FILL_COLOR */
+#define SCALER_FILL_COLOR_GET_ALPHA(r)		SCALER_GET(r, 31, 24)
+#define SCALER_FILL_COLOR_SET_ALPHA(v)		SCALER_SET(v, 31, 24)
+#define SCALER_FILL_COLOR_GET_FILL_COLOR0(r)	SCALER_GET(r, 23, 16)
+#define SCALER_FILL_COLOR_SET_FILL_COLOR0(v)	SCALER_SET(v, 23, 16)
+#define SCALER_FILL_COLOR_GET_FILL_COLOR1(r)	SCALER_GET(r, 15, 8)
+#define SCALER_FILL_COLOR_SET_FILL_COLOR1(v)	SCALER_SET(v, 15, 8)
+#define SCALER_FILL_COLOR_GET_FILL_COLOR2(r)	SCALER_GET(r, 7, 0)
+#define SCALER_FILL_COLOR_SET_FILL_COLOR2(v)	SCALER_SET(v, 7, 0)
+
+/* SCALER_ADDR_Q_CONFIG */
+#define SCALER_ADDR_Q_CONFIG_RST		(1 << 0)
+
+/* SCALER_SRC_ADDR_Q_STATUS */
+#define SCALER_SRC_ADDR_Q_STATUS_Y_FULL		(1 << 23)
+#define SCALER_SRC_ADDR_Q_STATUS_Y_EMPTY	(1 << 22)
+#define SCALER_SRC_ADDR_Q_STATUS_GET_Y_WR_IDX(r)	SCALER_GET(r, 21, 16)
+#define SCALER_SRC_ADDR_Q_STATUS_CB_FULL	(1 << 15)
+#define SCALER_SRC_ADDR_Q_STATUS_CB_EMPTY	(1 << 14)
+#define SCALER_SRC_ADDR_Q_STATUS_GET_CB_WR_IDX(r)	SCALER_GET(r, 13, 8)
+#define SCALER_SRC_ADDR_Q_STATUS_CR_FULL	(1 << 7)
+#define SCALER_SRC_ADDR_Q_STATUS_CR_EMPTY	(1 << 6)
+#define SCALER_SRC_ADDR_Q_STATUS_GET_CR_WR_IDX(r)	SCALER_GET(r, 5, 0)
+
+/* SCALER_DST_ADDR_Q_STATUS */
+#define SCALER_DST_ADDR_Q_STATUS_Y_FULL		(1 << 23)
+#define SCALER_DST_ADDR_Q_STATUS_Y_EMPTY	(1 << 22)
+#define SCALER_DST_ADDR_Q_STATUS_GET_Y_WR_IDX(r)	SCALER_GET(r, 21, 16)
+#define SCALER_DST_ADDR_Q_STATUS_CB_FULL	(1 << 15)
+#define SCALER_DST_ADDR_Q_STATUS_CB_EMPTY	(1 << 14)
+#define SCALER_DST_ADDR_Q_STATUS_GET_CB_WR_IDX(r)	SCALER_GET(r, 13, 8)
+#define SCALER_DST_ADDR_Q_STATUS_CR_FULL	(1 << 7)
+#define SCALER_DST_ADDR_Q_STATUS_CR_EMPTY	(1 << 6)
+#define SCALER_DST_ADDR_Q_STATUS_GET_CR_WR_IDX(r)	SCALER_GET(r, 5, 0)
+
+/* SCALER_CRC_COLOR00_10 */
+#define SCALER_CRC_COLOR00_10_GET_00(r)		SCALER_GET(r, 31, 16)
+#define SCALER_CRC_COLOR00_10_GET_10(r)		SCALER_GET(r, 15, 0)
+
+/* SCALER_CRC_COLOR20_30 */
+#define SCALER_CRC_COLOR20_30_GET_20(r)		SCALER_GET(r, 31, 16)
+#define SCALER_CRC_COLOR20_30_GET_30(r)		SCALER_GET(r, 15, 0)
+
+/* SCALER_CRC_COLOR01_11 */
+#define SCALER_CRC_COLOR01_11_GET_01(r)		SCALER_GET(r, 31, 16)
+#define SCALER_CRC_COLOR01_11_GET_11(r)		SCALER_GET(r, 15, 0)
+
+/* SCALER_CRC_COLOR21_31 */
+#define SCALER_CRC_COLOR21_31_GET_21(r)		SCALER_GET(r, 31, 16)
+#define SCALER_CRC_COLOR21_31_GET_31(r)		SCALER_GET(r, 15, 0)
+
+#endif /* EXYNOS_REGS_SCALER_H */
diff --git a/drivers/gpu/drm/gma500/cdv_device.c b/drivers/gpu/drm/gma500/cdv_device.c
index 3a3bf752e03a..34b85767e4da 100644
--- a/drivers/gpu/drm/gma500/cdv_device.c
+++ b/drivers/gpu/drm/gma500/cdv_device.c
@@ -485,7 +485,7 @@ void cdv_intel_attach_force_audio_property(struct drm_connector *connector)
 			return;
 
 		for (i = 0; i < ARRAY_SIZE(force_audio_names); i++)
-			drm_property_add_enum(prop, i, i-1, force_audio_names[i]);
+			drm_property_add_enum(prop, i-1, force_audio_names[i]);
 
 		dev_priv->force_audio_property = prop;
 	}
@@ -514,7 +514,7 @@ void cdv_intel_attach_broadcast_rgb_property(struct drm_connector *connector)
 			return;
 
 		for (i = 0; i < ARRAY_SIZE(broadcast_rgb_names); i++)
-			drm_property_add_enum(prop, i, i, broadcast_rgb_names[i]);
+			drm_property_add_enum(prop, i, broadcast_rgb_names[i]);
 
 		dev_priv->broadcast_rgb_property = prop;
 	}
diff --git a/drivers/gpu/drm/gma500/cdv_intel_crt.c b/drivers/gpu/drm/gma500/cdv_intel_crt.c
index b837e7a92196..cb5a14b7ec7f 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_crt.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_crt.c
@@ -64,7 +64,7 @@ static void cdv_intel_crt_dpms(struct drm_encoder *encoder, int mode)
 	REG_WRITE(reg, temp);
 }
 
-static int cdv_intel_crt_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status cdv_intel_crt_mode_valid(struct drm_connector *connector,
 				struct drm_display_mode *mode)
 {
 	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
diff --git a/drivers/gpu/drm/gma500/cdv_intel_dp.c b/drivers/gpu/drm/gma500/cdv_intel_dp.c
index a4bb89b7878f..5ea785f07ba8 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_dp.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_dp.c
@@ -505,7 +505,7 @@ static void cdv_intel_edp_backlight_off (struct gma_encoder *intel_encoder)
 	msleep(intel_dp->backlight_off_delay);
 }
 
-static int
+static enum drm_mode_status
 cdv_intel_dp_mode_valid(struct drm_connector *connector,
 		    struct drm_display_mode *mode)
 {
diff --git a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
index 563f193fcfac..f0878998526a 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
@@ -223,7 +223,7 @@ static int cdv_hdmi_get_modes(struct drm_connector *connector)
 	return ret;
 }
 
-static int cdv_hdmi_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status cdv_hdmi_mode_valid(struct drm_connector *connector,
 				 struct drm_display_mode *mode)
 {
 	if (mode->clock > 165000)
diff --git a/drivers/gpu/drm/gma500/cdv_intel_lvds.c b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
index e64960db3224..de9531caaca0 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
@@ -244,7 +244,7 @@ static void cdv_intel_lvds_restore(struct drm_connector *connector)
 {
 }
 
-static int cdv_intel_lvds_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status cdv_intel_lvds_mode_valid(struct drm_connector *connector,
 			      struct drm_display_mode *mode)
 {
 	struct drm_device *dev = connector->dev;
diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_output.c b/drivers/gpu/drm/gma500/mdfld_dsi_output.c
index acb3848ef1c9..fe020926ea4f 100644
--- a/drivers/gpu/drm/gma500/mdfld_dsi_output.c
+++ b/drivers/gpu/drm/gma500/mdfld_dsi_output.c
@@ -346,7 +346,7 @@ static int mdfld_dsi_connector_get_modes(struct drm_connector *connector)
 	return 0;
 }
 
-static int mdfld_dsi_connector_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status mdfld_dsi_connector_mode_valid(struct drm_connector *connector,
 						struct drm_display_mode *mode)
 {
 	struct mdfld_dsi_connector *dsi_connector =
diff --git a/drivers/gpu/drm/gma500/oaktrail_hdmi.c b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
index 8b2eb32ee988..78566a80ad25 100644
--- a/drivers/gpu/drm/gma500/oaktrail_hdmi.c
+++ b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
@@ -509,7 +509,7 @@ static void oaktrail_hdmi_dpms(struct drm_encoder *encoder, int mode)
 	HDMI_WRITE(HDMI_VIDEO_REG, temp);
 }
 
-static int oaktrail_hdmi_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status oaktrail_hdmi_mode_valid(struct drm_connector *connector,
 				struct drm_display_mode *mode)
 {
 	if (mode->clock > 165000)
diff --git a/drivers/gpu/drm/gma500/psb_intel_drv.h b/drivers/gpu/drm/gma500/psb_intel_drv.h
index e8e4ea14b12b..e05e5399af2d 100644
--- a/drivers/gpu/drm/gma500/psb_intel_drv.h
+++ b/drivers/gpu/drm/gma500/psb_intel_drv.h
@@ -255,7 +255,7 @@ extern int intelfb_remove(struct drm_device *dev,
 extern bool psb_intel_lvds_mode_fixup(struct drm_encoder *encoder,
 				      const struct drm_display_mode *mode,
 				      struct drm_display_mode *adjusted_mode);
-extern int psb_intel_lvds_mode_valid(struct drm_connector *connector,
+extern enum drm_mode_status psb_intel_lvds_mode_valid(struct drm_connector *connector,
 				     struct drm_display_mode *mode);
 extern int psb_intel_lvds_set_property(struct drm_connector *connector,
 					struct drm_property *property,
diff --git a/drivers/gpu/drm/gma500/psb_intel_lvds.c b/drivers/gpu/drm/gma500/psb_intel_lvds.c
index be3eefec5152..8baf6325c6e4 100644
--- a/drivers/gpu/drm/gma500/psb_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/psb_intel_lvds.c
@@ -343,7 +343,7 @@ static void psb_intel_lvds_restore(struct drm_connector *connector)
 	}
 }
 
-int psb_intel_lvds_mode_valid(struct drm_connector *connector,
+enum drm_mode_status psb_intel_lvds_mode_valid(struct drm_connector *connector,
 				 struct drm_display_mode *mode)
 {
 	struct drm_psb_private *dev_priv = connector->dev->dev_private;
diff --git a/drivers/gpu/drm/gma500/psb_intel_sdvo.c b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
index 84507912be84..f2ee6aa10afa 100644
--- a/drivers/gpu/drm/gma500/psb_intel_sdvo.c
+++ b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
@@ -1157,7 +1157,7 @@ static void psb_intel_sdvo_dpms(struct drm_encoder *encoder, int mode)
 	return;
 }
 
-static int psb_intel_sdvo_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status psb_intel_sdvo_mode_valid(struct drm_connector *connector,
 				 struct drm_display_mode *mode)
 {
 	struct psb_intel_sdvo *psb_intel_sdvo = intel_attached_sdvo(connector);
@@ -2281,7 +2281,7 @@ static bool psb_intel_sdvo_tv_create_property(struct psb_intel_sdvo *psb_intel_s
 
 	for (i = 0; i < psb_intel_sdvo_connector->format_supported_num; i++)
 		drm_property_add_enum(
-				psb_intel_sdvo_connector->tv_format, i,
+				psb_intel_sdvo_connector->tv_format,
 				i, tv_format_names[psb_intel_sdvo_connector->tv_format_supported[i]]);
 
 	psb_intel_sdvo->tv_format_index = psb_intel_sdvo_connector->tv_format_supported[0];
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
index f4eba87c96f3..d2f4749ebf8d 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
@@ -27,7 +27,7 @@ static int hibmc_connector_get_modes(struct drm_connector *connector)
 	return drm_add_modes_noedid(connector, 800, 600);
 }
 
-static int hibmc_connector_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status hibmc_connector_mode_valid(struct drm_connector *connector,
 				      struct drm_display_mode *mode)
 {
 	return MODE_OK;
diff --git a/drivers/gpu/drm/i2c/Kconfig b/drivers/gpu/drm/i2c/Kconfig
index a6c92beb410a..65d3acb61c03 100644
--- a/drivers/gpu/drm/i2c/Kconfig
+++ b/drivers/gpu/drm/i2c/Kconfig
@@ -22,8 +22,14 @@ config DRM_I2C_SIL164
 config DRM_I2C_NXP_TDA998X
 	tristate "NXP Semiconductors TDA998X HDMI encoder"
 	default m if DRM_TILCDC
+	select CEC_CORE if CEC_NOTIFIER
 	select SND_SOC_HDMI_CODEC if SND_SOC
 	help
 	  Support for NXP Semiconductors TDA998X HDMI encoders.
 
+config DRM_I2C_NXP_TDA9950
+	tristate "NXP Semiconductors TDA9950/TDA998X HDMI CEC"
+	select CEC_NOTIFIER
+	select CEC_CORE
+
 endmenu
diff --git a/drivers/gpu/drm/i2c/Makefile b/drivers/gpu/drm/i2c/Makefile
index b20100c18ffb..a962f6f08568 100644
--- a/drivers/gpu/drm/i2c/Makefile
+++ b/drivers/gpu/drm/i2c/Makefile
@@ -7,3 +7,4 @@ obj-$(CONFIG_DRM_I2C_SIL164) += sil164.o
 
 tda998x-y := tda998x_drv.o
 obj-$(CONFIG_DRM_I2C_NXP_TDA998X) += tda998x.o
+obj-$(CONFIG_DRM_I2C_NXP_TDA9950) += tda9950.o
diff --git a/drivers/gpu/drm/i2c/tda9950.c b/drivers/gpu/drm/i2c/tda9950.c
new file mode 100644
index 000000000000..3f7396caad48
--- /dev/null
+++ b/drivers/gpu/drm/i2c/tda9950.c
@@ -0,0 +1,509 @@
+/*
+ *  TDA9950 Consumer Electronics Control driver
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * The NXP TDA9950 implements the HDMI Consumer Electronics Control
+ * interface.  The host interface is similar to a mailbox: the data
+ * registers starting at REG_CDR0 are written to send a command to the
+ * internal CPU, and replies are read from these registers.
+ *
+ * As the data registers represent a mailbox, they must be accessed
+ * as a single I2C transaction.  See the TDA9950 data sheet for details.
+ */
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/platform_data/tda9950.h>
+#include <linux/slab.h>
+#include <drm/drm_edid.h>
+#include <media/cec.h>
+#include <media/cec-notifier.h>
+
+enum {
+	REG_CSR = 0x00,
+	CSR_BUSY = BIT(7),
+	CSR_INT  = BIT(6),
+	CSR_ERR  = BIT(5),
+
+	REG_CER = 0x01,
+
+	REG_CVR = 0x02,
+
+	REG_CCR = 0x03,
+	CCR_RESET = BIT(7),
+	CCR_ON    = BIT(6),
+
+	REG_ACKH = 0x04,
+	REG_ACKL = 0x05,
+
+	REG_CCONR = 0x06,
+	CCONR_ENABLE_ERROR = BIT(4),
+	CCONR_RETRY_MASK = 7,
+
+	REG_CDR0 = 0x07,
+
+	CDR1_REQ = 0x00,
+	CDR1_CNF = 0x01,
+	CDR1_IND = 0x81,
+	CDR1_ERR = 0x82,
+	CDR1_IER = 0x83,
+
+	CDR2_CNF_SUCCESS    = 0x00,
+	CDR2_CNF_OFF_STATE  = 0x80,
+	CDR2_CNF_BAD_REQ    = 0x81,
+	CDR2_CNF_CEC_ACCESS = 0x82,
+	CDR2_CNF_ARB_ERROR  = 0x83,
+	CDR2_CNF_BAD_TIMING = 0x84,
+	CDR2_CNF_NACK_ADDR  = 0x85,
+	CDR2_CNF_NACK_DATA  = 0x86,
+};
+
+struct tda9950_priv {
+	struct i2c_client *client;
+	struct device *hdmi;
+	struct cec_adapter *adap;
+	struct tda9950_glue *glue;
+	u16 addresses;
+	struct cec_msg rx_msg;
+	struct cec_notifier *notify;
+	bool open;
+};
+
+static int tda9950_write_range(struct i2c_client *client, u8 addr, u8 *p, int cnt)
+{
+	struct i2c_msg msg;
+	u8 buf[cnt + 1];
+	int ret;
+
+	buf[0] = addr;
+	memcpy(buf + 1, p, cnt);
+
+	msg.addr = client->addr;
+	msg.flags = 0;
+	msg.len = cnt + 1;
+	msg.buf = buf;
+
+	dev_dbg(&client->dev, "wr 0x%02x: %*ph\n", addr, cnt, p);
+
+	ret = i2c_transfer(client->adapter, &msg, 1);
+	if (ret < 0)
+		dev_err(&client->dev, "Error %d writing to cec:0x%x\n", ret, addr);
+	return ret < 0 ? ret : 0;
+}
+
+static void tda9950_write(struct i2c_client *client, u8 addr, u8 val)
+{
+	tda9950_write_range(client, addr, &val, 1);
+}
+
+static int tda9950_read_range(struct i2c_client *client, u8 addr, u8 *p, int cnt)
+{
+	struct i2c_msg msg[2];
+	int ret;
+
+	msg[0].addr = client->addr;
+	msg[0].flags = 0;
+	msg[0].len = 1;
+	msg[0].buf = &addr;
+	msg[1].addr = client->addr;
+	msg[1].flags = I2C_M_RD;
+	msg[1].len = cnt;
+	msg[1].buf = p;
+
+	ret = i2c_transfer(client->adapter, msg, 2);
+	if (ret < 0)
+		dev_err(&client->dev, "Error %d reading from cec:0x%x\n", ret, addr);
+
+	dev_dbg(&client->dev, "rd 0x%02x: %*ph\n", addr, cnt, p);
+
+	return ret;
+}
+
+static u8 tda9950_read(struct i2c_client *client, u8 addr)
+{
+	int ret;
+	u8 val;
+
+	ret = tda9950_read_range(client, addr, &val, 1);
+	if (ret < 0)
+		val = 0;
+
+	return val;
+}
+
+static irqreturn_t tda9950_irq(int irq, void *data)
+{
+	struct tda9950_priv *priv = data;
+	unsigned int tx_status;
+	u8 csr, cconr, buf[19];
+	u8 arb_lost_cnt, nack_cnt, err_cnt;
+
+	if (!priv->open)
+		return IRQ_NONE;
+
+	csr = tda9950_read(priv->client, REG_CSR);
+	if (!(csr & CSR_INT))
+		return IRQ_NONE;
+
+	cconr = tda9950_read(priv->client, REG_CCONR) & CCONR_RETRY_MASK;
+
+	tda9950_read_range(priv->client, REG_CDR0, buf, sizeof(buf));
+
+	/*
+	 * This should never happen: the data sheet says that there will
+	 * always be a valid message if the interrupt line is asserted.
+	 */
+	if (buf[0] == 0) {
+		dev_warn(&priv->client->dev, "interrupt pending, but no message?\n");
+		return IRQ_NONE;
+	}
+
+	switch (buf[1]) {
+	case CDR1_CNF: /* transmit result */
+		arb_lost_cnt = nack_cnt = err_cnt = 0;
+		switch (buf[2]) {
+		case CDR2_CNF_SUCCESS:
+			tx_status = CEC_TX_STATUS_OK;
+			break;
+
+		case CDR2_CNF_ARB_ERROR:
+			tx_status = CEC_TX_STATUS_ARB_LOST;
+			arb_lost_cnt = cconr;
+			break;
+
+		case CDR2_CNF_NACK_ADDR:
+			tx_status = CEC_TX_STATUS_NACK;
+			nack_cnt = cconr;
+			break;
+
+		default: /* some other error, refer to TDA9950 docs */
+			dev_err(&priv->client->dev, "CNF reply error 0x%02x\n",
+				buf[2]);
+			tx_status = CEC_TX_STATUS_ERROR;
+			err_cnt = cconr;
+			break;
+		}
+		/* TDA9950 executes all retries for us */
+		tx_status |= CEC_TX_STATUS_MAX_RETRIES;
+		cec_transmit_done(priv->adap, tx_status, arb_lost_cnt,
+				  nack_cnt, 0, err_cnt);
+		break;
+
+	case CDR1_IND:
+		priv->rx_msg.len = buf[0] - 2;
+		if (priv->rx_msg.len > CEC_MAX_MSG_SIZE)
+			priv->rx_msg.len = CEC_MAX_MSG_SIZE;
+
+		memcpy(priv->rx_msg.msg, buf + 2, priv->rx_msg.len);
+		cec_received_msg(priv->adap, &priv->rx_msg);
+		break;
+
+	default: /* unknown */
+		dev_err(&priv->client->dev, "unknown service id 0x%02x\n",
+			buf[1]);
+		break;
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int tda9950_cec_transmit(struct cec_adapter *adap, u8 attempts,
+				u32 signal_free_time, struct cec_msg *msg)
+{
+	struct tda9950_priv *priv = adap->priv;
+	u8 buf[CEC_MAX_MSG_SIZE + 2];
+
+	buf[0] = 2 + msg->len;
+	buf[1] = CDR1_REQ;
+	memcpy(buf + 2, msg->msg, msg->len);
+
+	if (attempts > 5)
+		attempts = 5;
+
+	tda9950_write(priv->client, REG_CCONR, attempts);
+
+	return tda9950_write_range(priv->client, REG_CDR0, buf, 2 + msg->len);
+}
+
+static int tda9950_cec_adap_log_addr(struct cec_adapter *adap, u8 addr)
+{
+	struct tda9950_priv *priv = adap->priv;
+	u16 addresses;
+	u8 buf[2];
+
+	if (addr == CEC_LOG_ADDR_INVALID)
+		addresses = priv->addresses = 0;
+	else
+		addresses = priv->addresses |= BIT(addr);
+
+	/* TDA9950 doesn't want address 15 set */
+	addresses &= 0x7fff;
+	buf[0] = addresses >> 8;
+	buf[1] = addresses;
+
+	return tda9950_write_range(priv->client, REG_ACKH, buf, 2);
+}
+
+/*
+ * When operating as part of the TDA998x, we need additional handling
+ * to initialise and shut down the TDA9950 part of the device.  These
+ * two hooks are provided to allow the TDA998x code to perform those
+ * activities.
+ */
+static int tda9950_glue_open(struct tda9950_priv *priv)
+{
+	int ret = 0;
+
+	if (priv->glue && priv->glue->open)
+		ret = priv->glue->open(priv->glue->data);
+
+	priv->open = true;
+
+	return ret;
+}
+
+static void tda9950_glue_release(struct tda9950_priv *priv)
+{
+	priv->open = false;
+
+	if (priv->glue && priv->glue->release)
+		priv->glue->release(priv->glue->data);
+}
+
+static int tda9950_open(struct tda9950_priv *priv)
+{
+	struct i2c_client *client = priv->client;
+	int ret;
+
+	ret = tda9950_glue_open(priv);
+	if (ret)
+		return ret;
+
+	/* Reset the TDA9950, and wait 250ms for it to recover */
+	tda9950_write(client, REG_CCR, CCR_RESET);
+	msleep(250);
+
+	tda9950_cec_adap_log_addr(priv->adap, CEC_LOG_ADDR_INVALID);
+
+	/* Start the command processor */
+	tda9950_write(client, REG_CCR, CCR_ON);
+
+	return 0;
+}
+
+static void tda9950_release(struct tda9950_priv *priv)
+{
+	struct i2c_client *client = priv->client;
+	int timeout = 50;
+	u8 csr;
+
+	/* Stop the command processor */
+	tda9950_write(client, REG_CCR, 0);
+
+	/* Wait up to .5s for it to signal non-busy */
+	do {
+		csr = tda9950_read(client, REG_CSR);
+		if (!(csr & CSR_BUSY) || --timeout)
+			break;
+		msleep(10);
+	} while (1);
+
+	/* Warn the user that their IRQ may die if it's shared. */
+	if (csr & CSR_BUSY)
+		dev_warn(&client->dev, "command processor failed to stop, irq%d may die (csr=0x%02x)\n",
+			 client->irq, csr);
+
+	tda9950_glue_release(priv);
+}
+
+static int tda9950_cec_adap_enable(struct cec_adapter *adap, bool enable)
+{
+	struct tda9950_priv *priv = adap->priv;
+
+	if (!enable) {
+		tda9950_release(priv);
+		return 0;
+	} else {
+		return tda9950_open(priv);
+	}
+}
+
+static const struct cec_adap_ops tda9950_cec_ops = {
+	.adap_enable = tda9950_cec_adap_enable,
+	.adap_log_addr = tda9950_cec_adap_log_addr,
+	.adap_transmit = tda9950_cec_transmit,
+};
+
+/*
+ * When operating as part of the TDA998x, we need to claim additional
+ * resources.  These two hooks permit the management of those resources.
+ */
+static void tda9950_devm_glue_exit(void *data)
+{
+	struct tda9950_glue *glue = data;
+
+	if (glue && glue->exit)
+		glue->exit(glue->data);
+}
+
+static int tda9950_devm_glue_init(struct device *dev, struct tda9950_glue *glue)
+{
+	int ret;
+
+	if (glue && glue->init) {
+		ret = glue->init(glue->data);
+		if (ret)
+			return ret;
+	}
+
+	ret = devm_add_action(dev, tda9950_devm_glue_exit, glue);
+	if (ret)
+		tda9950_devm_glue_exit(glue);
+
+	return ret;
+}
+
+static void tda9950_cec_del(void *data)
+{
+	struct tda9950_priv *priv = data;
+
+	cec_delete_adapter(priv->adap);
+}
+
+static int tda9950_probe(struct i2c_client *client,
+			 const struct i2c_device_id *id)
+{
+	struct tda9950_glue *glue = client->dev.platform_data;
+	struct device *dev = &client->dev;
+	struct tda9950_priv *priv;
+	unsigned long irqflags;
+	int ret;
+	u8 cvr;
+
+	/*
+	 * We must have I2C functionality: our multi-byte accesses
+	 * must be performed as a single contiguous transaction.
+	 */
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
+		dev_err(&client->dev,
+			"adapter does not support I2C functionality\n");
+		return -ENXIO;
+	}
+
+	/* We must have an interrupt to be functional. */
+	if (client->irq <= 0) {
+		dev_err(&client->dev, "driver requires an interrupt\n");
+		return -ENXIO;
+	}
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->client = client;
+	priv->glue = glue;
+
+	i2c_set_clientdata(client, priv);
+
+	/*
+	 * If we're part of a TDA998x, we want the class devices to be
+	 * associated with the HDMI Tx so we have a tight relationship
+	 * between the HDMI interface and the CEC interface.
+	 */
+	priv->hdmi = dev;
+	if (glue && glue->parent)
+		priv->hdmi = glue->parent;
+
+	priv->adap = cec_allocate_adapter(&tda9950_cec_ops, priv, "tda9950",
+					  CEC_CAP_DEFAULTS,
+					  CEC_MAX_LOG_ADDRS);
+	if (IS_ERR(priv->adap))
+		return PTR_ERR(priv->adap);
+
+	ret = devm_add_action(dev, tda9950_cec_del, priv);
+	if (ret) {
+		cec_delete_adapter(priv->adap);
+		return ret;
+	}
+
+	ret = tda9950_devm_glue_init(dev, glue);
+	if (ret)
+		return ret;
+
+	ret = tda9950_glue_open(priv);
+	if (ret)
+		return ret;
+
+	cvr = tda9950_read(client, REG_CVR);
+
+	dev_info(&client->dev,
+		 "TDA9950 CEC interface, hardware version %u.%u\n",
+		 cvr >> 4, cvr & 15);
+
+	tda9950_glue_release(priv);
+
+	irqflags = IRQF_TRIGGER_FALLING;
+	if (glue)
+		irqflags = glue->irq_flags;
+
+	ret = devm_request_threaded_irq(dev, client->irq, NULL, tda9950_irq,
+					irqflags | IRQF_SHARED | IRQF_ONESHOT,
+					dev_name(&client->dev), priv);
+	if (ret < 0)
+		return ret;
+
+	priv->notify = cec_notifier_get(priv->hdmi);
+	if (!priv->notify)
+		return -ENOMEM;
+
+	ret = cec_register_adapter(priv->adap, priv->hdmi);
+	if (ret < 0) {
+		cec_notifier_put(priv->notify);
+		return ret;
+	}
+
+	/*
+	 * CEC documentation says we must not call cec_delete_adapter
+	 * after a successful call to cec_register_adapter().
+	 */
+	devm_remove_action(dev, tda9950_cec_del, priv);
+
+	cec_register_cec_notifier(priv->adap, priv->notify);
+
+	return 0;
+}
+
+static int tda9950_remove(struct i2c_client *client)
+{
+	struct tda9950_priv *priv = i2c_get_clientdata(client);
+
+	cec_unregister_adapter(priv->adap);
+	cec_notifier_put(priv->notify);
+
+	return 0;
+}
+
+static struct i2c_device_id tda9950_ids[] = {
+	{ "tda9950", 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(i2c, tda9950_ids);
+
+static struct i2c_driver tda9950_driver = {
+	.probe = tda9950_probe,
+	.remove = tda9950_remove,
+	.driver = {
+		.name = "tda9950",
+	},
+	.id_table = tda9950_ids,
+};
+
+module_i2c_driver(tda9950_driver);
+
+MODULE_AUTHOR("Russell King <rmk+kernel@armlinux.org.uk>");
+MODULE_DESCRIPTION("TDA9950/TDA998x Consumer Electronics Control Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/i2c/tda998x_drv.c b/drivers/gpu/drm/i2c/tda998x_drv.c
index 9e67a7b4e3a4..6ebd8842dbcc 100644
--- a/drivers/gpu/drm/i2c/tda998x_drv.c
+++ b/drivers/gpu/drm/i2c/tda998x_drv.c
@@ -16,8 +16,10 @@
  */
 
 #include <linux/component.h>
+#include <linux/gpio/consumer.h>
 #include <linux/hdmi.h>
 #include <linux/module.h>
+#include <linux/platform_data/tda9950.h>
 #include <linux/irq.h>
 #include <sound/asoundef.h>
 #include <sound/hdmi-codec.h>
@@ -29,6 +31,8 @@
 #include <drm/drm_of.h>
 #include <drm/i2c/tda998x.h>
 
+#include <media/cec-notifier.h>
+
 #define DBG(fmt, ...) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
 
 struct tda998x_audio_port {
@@ -55,6 +59,7 @@ struct tda998x_priv {
 	struct platform_device *audio_pdev;
 	struct mutex audio_mutex;
 
+	struct mutex edid_mutex;
 	wait_queue_head_t wq_edid;
 	volatile int wq_edid_wait;
 
@@ -67,6 +72,9 @@ struct tda998x_priv {
 	struct drm_connector connector;
 
 	struct tda998x_audio_port audio_port[2];
+	struct tda9950_glue cec_glue;
+	struct gpio_desc *calib;
+	struct cec_notifier *cec_notify;
 };
 
 #define conn_to_tda998x_priv(x) \
@@ -345,6 +353,12 @@ struct tda998x_priv {
 #define REG_CEC_INTSTATUS	  0xee		      /* read */
 # define CEC_INTSTATUS_CEC	  (1 << 0)
 # define CEC_INTSTATUS_HDMI	  (1 << 1)
+#define REG_CEC_CAL_XOSC_CTRL1    0xf2
+# define CEC_CAL_XOSC_CTRL1_ENA_CAL	BIT(0)
+#define REG_CEC_DES_FREQ2         0xf5
+# define CEC_DES_FREQ2_DIS_AUTOCAL BIT(7)
+#define REG_CEC_CLK               0xf6
+# define CEC_CLK_FRO              0x11
 #define REG_CEC_FRO_IM_CLK_CTRL   0xfb                /* read/write */
 # define CEC_FRO_IM_CLK_CTRL_GHOST_DIS (1 << 7)
 # define CEC_FRO_IM_CLK_CTRL_ENA_OTP   (1 << 6)
@@ -359,6 +373,7 @@ struct tda998x_priv {
 # define CEC_RXSHPDLEV_HPD        (1 << 1)
 
 #define REG_CEC_ENAMODS           0xff                /* read/write */
+# define CEC_ENAMODS_EN_CEC_CLK   (1 << 7)
 # define CEC_ENAMODS_DIS_FRO      (1 << 6)
 # define CEC_ENAMODS_DIS_CCLK     (1 << 5)
 # define CEC_ENAMODS_EN_RXSENS    (1 << 2)
@@ -417,6 +432,114 @@ cec_read(struct tda998x_priv *priv, u8 addr)
 	return val;
 }
 
+static void cec_enamods(struct tda998x_priv *priv, u8 mods, bool enable)
+{
+	int val = cec_read(priv, REG_CEC_ENAMODS);
+
+	if (val < 0)
+		return;
+
+	if (enable)
+		val |= mods;
+	else
+		val &= ~mods;
+
+	cec_write(priv, REG_CEC_ENAMODS, val);
+}
+
+static void tda998x_cec_set_calibration(struct tda998x_priv *priv, bool enable)
+{
+	if (enable) {
+		u8 val;
+
+		cec_write(priv, 0xf3, 0xc0);
+		cec_write(priv, 0xf4, 0xd4);
+
+		/* Enable automatic calibration mode */
+		val = cec_read(priv, REG_CEC_DES_FREQ2);
+		val &= ~CEC_DES_FREQ2_DIS_AUTOCAL;
+		cec_write(priv, REG_CEC_DES_FREQ2, val);
+
+		/* Enable free running oscillator */
+		cec_write(priv, REG_CEC_CLK, CEC_CLK_FRO);
+		cec_enamods(priv, CEC_ENAMODS_DIS_FRO, false);
+
+		cec_write(priv, REG_CEC_CAL_XOSC_CTRL1,
+			  CEC_CAL_XOSC_CTRL1_ENA_CAL);
+	} else {
+		cec_write(priv, REG_CEC_CAL_XOSC_CTRL1, 0);
+	}
+}
+
+/*
+ * Calibration for the internal oscillator: we need to set calibration mode,
+ * and then pulse the IRQ line low for a 10ms ± 1% period.
+ */
+static void tda998x_cec_calibration(struct tda998x_priv *priv)
+{
+	struct gpio_desc *calib = priv->calib;
+
+	mutex_lock(&priv->edid_mutex);
+	if (priv->hdmi->irq > 0)
+		disable_irq(priv->hdmi->irq);
+	gpiod_direction_output(calib, 1);
+	tda998x_cec_set_calibration(priv, true);
+
+	local_irq_disable();
+	gpiod_set_value(calib, 0);
+	mdelay(10);
+	gpiod_set_value(calib, 1);
+	local_irq_enable();
+
+	tda998x_cec_set_calibration(priv, false);
+	gpiod_direction_input(calib);
+	if (priv->hdmi->irq > 0)
+		enable_irq(priv->hdmi->irq);
+	mutex_unlock(&priv->edid_mutex);
+}
+
+static int tda998x_cec_hook_init(void *data)
+{
+	struct tda998x_priv *priv = data;
+	struct gpio_desc *calib;
+
+	calib = gpiod_get(&priv->hdmi->dev, "nxp,calib", GPIOD_ASIS);
+	if (IS_ERR(calib)) {
+		dev_warn(&priv->hdmi->dev, "failed to get calibration gpio: %ld\n",
+			 PTR_ERR(calib));
+		return PTR_ERR(calib);
+	}
+
+	priv->calib = calib;
+
+	return 0;
+}
+
+static void tda998x_cec_hook_exit(void *data)
+{
+	struct tda998x_priv *priv = data;
+
+	gpiod_put(priv->calib);
+	priv->calib = NULL;
+}
+
+static int tda998x_cec_hook_open(void *data)
+{
+	struct tda998x_priv *priv = data;
+
+	cec_enamods(priv, CEC_ENAMODS_EN_CEC_CLK | CEC_ENAMODS_EN_CEC, true);
+	tda998x_cec_calibration(priv);
+
+	return 0;
+}
+
+static void tda998x_cec_hook_release(void *data)
+{
+	struct tda998x_priv *priv = data;
+
+	cec_enamods(priv, CEC_ENAMODS_EN_CEC_CLK | CEC_ENAMODS_EN_CEC, false);
+}
+
 static int
 set_page(struct tda998x_priv *priv, u16 reg)
 {
@@ -657,10 +780,13 @@ static irqreturn_t tda998x_irq_thread(int irq, void *data)
 			sta, cec, lvl, flag0, flag1, flag2);
 
 		if (cec & CEC_RXSHPDINT_HPD) {
-			if (lvl & CEC_RXSHPDLEV_HPD)
+			if (lvl & CEC_RXSHPDLEV_HPD) {
 				tda998x_edid_delay_start(priv);
-			else
+			} else {
 				schedule_work(&priv->detect_work);
+				cec_notifier_set_phys_addr(priv->cec_notify,
+						   CEC_PHYS_ADDR_INVALID);
+			}
 
 			handled = true;
 		}
@@ -981,6 +1107,8 @@ static int tda998x_connector_fill_modes(struct drm_connector *connector,
 	if (connector->edid_blob_ptr) {
 		struct edid *edid = (void *)connector->edid_blob_ptr->data;
 
+		cec_notifier_set_phys_addr_from_edid(priv->cec_notify, edid);
+
 		priv->sink_has_audio = drm_detect_monitor_audio(edid);
 	} else {
 		priv->sink_has_audio = false;
@@ -1024,6 +1152,8 @@ static int read_edid_block(void *data, u8 *buf, unsigned int blk, size_t length)
 	offset = (blk & 1) ? 128 : 0;
 	segptr = blk / 2;
 
+	mutex_lock(&priv->edid_mutex);
+
 	reg_write(priv, REG_DDC_ADDR, 0xa0);
 	reg_write(priv, REG_DDC_OFFS, offset);
 	reg_write(priv, REG_DDC_SEGM_ADDR, 0x60);
@@ -1043,14 +1173,15 @@ static int read_edid_block(void *data, u8 *buf, unsigned int blk, size_t length)
 					msecs_to_jiffies(100));
 		if (i < 0) {
 			dev_err(&priv->hdmi->dev, "read edid wait err %d\n", i);
-			return i;
+			ret = i;
+			goto failed;
 		}
 	} else {
 		for (i = 100; i > 0; i--) {
 			msleep(1);
 			ret = reg_read(priv, REG_INT_FLAGS_2);
 			if (ret < 0)
-				return ret;
+				goto failed;
 			if (ret & INT_FLAGS_2_EDID_BLK_RD)
 				break;
 		}
@@ -1058,17 +1189,22 @@ static int read_edid_block(void *data, u8 *buf, unsigned int blk, size_t length)
 
 	if (i == 0) {
 		dev_err(&priv->hdmi->dev, "read edid timeout\n");
-		return -ETIMEDOUT;
+		ret = -ETIMEDOUT;
+		goto failed;
 	}
 
 	ret = reg_read_range(priv, REG_EDID_DATA_0, buf, length);
 	if (ret != length) {
 		dev_err(&priv->hdmi->dev, "failed to read edid block %d: %d\n",
 			blk, ret);
-		return ret;
+		goto failed;
 	}
 
-	return 0;
+	ret = 0;
+
+ failed:
+	mutex_unlock(&priv->edid_mutex);
+	return ret;
 }
 
 static int tda998x_connector_get_modes(struct drm_connector *connector)
@@ -1106,7 +1242,7 @@ static int tda998x_connector_get_modes(struct drm_connector *connector)
 	return n;
 }
 
-static int tda998x_connector_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status tda998x_connector_mode_valid(struct drm_connector *connector,
 					struct drm_display_mode *mode)
 {
 	/* TDA19988 dotclock can go up to 165MHz */
@@ -1423,6 +1559,9 @@ static void tda998x_destroy(struct tda998x_priv *priv)
 	cancel_work_sync(&priv->detect_work);
 
 	i2c_unregister_device(priv->cec);
+
+	if (priv->cec_notify)
+		cec_notifier_put(priv->cec_notify);
 }
 
 /* I2C driver functions */
@@ -1472,10 +1611,16 @@ static int tda998x_get_audio_ports(struct tda998x_priv *priv,
 static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 {
 	struct device_node *np = client->dev.of_node;
+	struct i2c_board_info cec_info;
 	u32 video;
 	int rev_lo, rev_hi, ret;
 
-	mutex_init(&priv->audio_mutex); /* Protect access from audio thread */
+	mutex_init(&priv->mutex);	/* protect the page access */
+	mutex_init(&priv->audio_mutex); /* protect access from audio thread */
+	mutex_init(&priv->edid_mutex);
+	init_waitqueue_head(&priv->edid_delay_waitq);
+	timer_setup(&priv->edid_delay_timer, tda998x_edid_delay_done, 0);
+	INIT_WORK(&priv->detect_work, tda998x_detect_work);
 
 	priv->vip_cntrl_0 = VIP_CNTRL_0_SWAP_A(2) | VIP_CNTRL_0_SWAP_B(3);
 	priv->vip_cntrl_1 = VIP_CNTRL_1_SWAP_C(0) | VIP_CNTRL_1_SWAP_D(1);
@@ -1485,14 +1630,6 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 	priv->cec_addr = 0x34 + (client->addr & 0x03);
 	priv->current_page = 0xff;
 	priv->hdmi = client;
-	priv->cec = i2c_new_dummy(client->adapter, priv->cec_addr);
-	if (!priv->cec)
-		return -ENODEV;
-
-	mutex_init(&priv->mutex);	/* protect the page access */
-	init_waitqueue_head(&priv->edid_delay_waitq);
-	timer_setup(&priv->edid_delay_timer, tda998x_edid_delay_done, 0);
-	INIT_WORK(&priv->detect_work, tda998x_detect_work);
 
 	/* wake up the device: */
 	cec_write(priv, REG_CEC_ENAMODS,
@@ -1502,10 +1639,15 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 
 	/* read version: */
 	rev_lo = reg_read(priv, REG_VERSION_LSB);
+	if (rev_lo < 0) {
+		dev_err(&client->dev, "failed to read version: %d\n", rev_lo);
+		return rev_lo;
+	}
+
 	rev_hi = reg_read(priv, REG_VERSION_MSB);
-	if (rev_lo < 0 || rev_hi < 0) {
-		ret = rev_lo < 0 ? rev_lo : rev_hi;
-		goto fail;
+	if (rev_hi < 0) {
+		dev_err(&client->dev, "failed to read version: %d\n", rev_hi);
+		return rev_hi;
 	}
 
 	priv->rev = rev_lo | rev_hi << 8;
@@ -1529,7 +1671,7 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 	default:
 		dev_err(&client->dev, "found unsupported device: %04x\n",
 			priv->rev);
-		goto fail;
+		return -ENXIO;
 	}
 
 	/* after reset, enable DDC: */
@@ -1545,6 +1687,15 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 	cec_write(priv, REG_CEC_FRO_IM_CLK_CTRL,
 			CEC_FRO_IM_CLK_CTRL_GHOST_DIS | CEC_FRO_IM_CLK_CTRL_IMCLK_SEL);
 
+	/* ensure interrupts are disabled */
+	cec_write(priv, REG_CEC_RXSHPDINTENA, 0);
+
+	/* clear pending interrupts */
+	cec_read(priv, REG_CEC_RXSHPDINT);
+	reg_read(priv, REG_INT_FLAGS_0);
+	reg_read(priv, REG_INT_FLAGS_1);
+	reg_read(priv, REG_INT_FLAGS_2);
+
 	/* initialize the optional IRQ */
 	if (client->irq) {
 		unsigned long irq_flags;
@@ -1552,13 +1703,11 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 		/* init read EDID waitqueue and HDP work */
 		init_waitqueue_head(&priv->wq_edid);
 
-		/* clear pending interrupts */
-		reg_read(priv, REG_INT_FLAGS_0);
-		reg_read(priv, REG_INT_FLAGS_1);
-		reg_read(priv, REG_INT_FLAGS_2);
-
 		irq_flags =
 			irqd_get_trigger_type(irq_get_irq_data(client->irq));
+
+		priv->cec_glue.irq_flags = irq_flags;
+
 		irq_flags |= IRQF_SHARED | IRQF_ONESHOT;
 		ret = request_threaded_irq(client->irq, NULL,
 					   tda998x_irq_thread, irq_flags,
@@ -1567,13 +1716,46 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 			dev_err(&client->dev,
 				"failed to request IRQ#%u: %d\n",
 				client->irq, ret);
-			goto fail;
+			goto err_irq;
 		}
 
 		/* enable HPD irq */
 		cec_write(priv, REG_CEC_RXSHPDINTENA, CEC_RXSHPDLEV_HPD);
 	}
 
+	priv->cec_notify = cec_notifier_get(&client->dev);
+	if (!priv->cec_notify) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	priv->cec_glue.parent = &client->dev;
+	priv->cec_glue.data = priv;
+	priv->cec_glue.init = tda998x_cec_hook_init;
+	priv->cec_glue.exit = tda998x_cec_hook_exit;
+	priv->cec_glue.open = tda998x_cec_hook_open;
+	priv->cec_glue.release = tda998x_cec_hook_release;
+
+	/*
+	 * Some TDA998x are actually two I2C devices merged onto one piece
+	 * of silicon: TDA9989 and TDA19989 combine the HDMI transmitter
+	 * with a slightly modified TDA9950 CEC device.  The CEC device
+	 * is at the TDA9950 address, with the address pins strapped across
+	 * to the TDA998x address pins.  Hence, it always has the same
+	 * offset.
+	 */
+	memset(&cec_info, 0, sizeof(cec_info));
+	strlcpy(cec_info.type, "tda9950", sizeof(cec_info.type));
+	cec_info.addr = priv->cec_addr;
+	cec_info.platform_data = &priv->cec_glue;
+	cec_info.irq = client->irq;
+
+	priv->cec = i2c_new_device(client->adapter, &cec_info);
+	if (!priv->cec) {
+		ret = -ENODEV;
+		goto fail;
+	}
+
 	/* enable EDID read irq: */
 	reg_set(priv, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
 
@@ -1596,12 +1778,18 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 		tda998x_audio_codec_init(priv, &client->dev);
 
 	return 0;
+
 fail:
 	/* if encoder_init fails, the encoder slave is never registered,
 	 * so cleanup here:
 	 */
 	i2c_unregister_device(priv->cec);
-	return -ENXIO;
+	if (priv->cec_notify)
+		cec_notifier_put(priv->cec_notify);
+	if (client->irq)
+		free_irq(client->irq, priv);
+err_irq:
+	return ret;
 }
 
 static void tda998x_encoder_prepare(struct drm_encoder *encoder)
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index 108d21f34777..9de8b1c51a5c 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -25,7 +25,8 @@ config DRM_I915_DEBUG
         select X86_MSR # used by igt/pm_rpm
         select DRM_VGEM # used by igt/prime_vgem (dmabuf interop checks)
         select DRM_DEBUG_MM if DRM=y
-	select DRM_DEBUG_MM_SELFTEST
+        select STACKDEPOT if DRM=y # for DRM_DEBUG_MM
+	select DRM_DEBUG_SELFTEST
 	select SW_SYNC # signaling validation framework (igt/syncobj*)
 	select DRM_I915_SW_FENCE_DEBUG_OBJECTS
 	select DRM_I915_SELFTEST
@@ -89,6 +90,18 @@ config DRM_I915_SW_FENCE_CHECK_DAG
 
           If in doubt, say "N".
 
+config DRM_I915_DEBUG_GUC
+        bool "Enable additional driver debugging for GuC"
+        depends on DRM_I915
+        default n
+        help
+          Choose this option to turn on extra driver debugging that may affect
+          performance but will help resolve GuC related issues.
+
+          Recommended for driver developers only.
+
+          If in doubt, say "N".
+
 config DRM_I915_SELFTEST
 	bool "Enable selftests upon driver load"
 	depends on DRM_I915
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 4eee91a3a236..4c6adae23e18 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -12,12 +12,16 @@
 # Note the danger in using -Wall -Wextra is that when CI updates gcc we
 # will most likely get a sudden build breakage... Hopefully we will fix
 # new warnings before CI updates!
-subdir-ccflags-y := -Wall -Wextra
+subdir-ccflags-y := -Wall -Wextra -Wvla
 subdir-ccflags-y += $(call cc-disable-warning, unused-parameter)
 subdir-ccflags-y += $(call cc-disable-warning, type-limits)
 subdir-ccflags-y += $(call cc-disable-warning, missing-field-initializers)
 subdir-ccflags-y += $(call cc-disable-warning, implicit-fallthrough)
 subdir-ccflags-y += $(call cc-disable-warning, unused-but-set-variable)
+# clang warnings
+subdir-ccflags-y += $(call cc-disable-warning, sign-compare)
+subdir-ccflags-y += $(call cc-disable-warning, sometimes-uninitialized)
+subdir-ccflags-y += $(call cc-disable-warning, initializer-overrides)
 subdir-ccflags-$(CONFIG_DRM_I915_WERROR) += -Werror
 
 # Fine grained warnings disable
@@ -43,7 +47,8 @@ i915-y := i915_drv.o \
 	  intel_csr.o \
 	  intel_device_info.o \
 	  intel_pm.o \
-	  intel_runtime_pm.o
+	  intel_runtime_pm.o \
+	  intel_workarounds.o
 
 i915-$(CONFIG_COMPAT)   += i915_ioc32.o
 i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o intel_pipe_crc.o
@@ -66,11 +71,11 @@ i915-y += i915_cmd_parser.o \
 	  i915_gem_shrinker.o \
 	  i915_gem_stolen.o \
 	  i915_gem_tiling.o \
-	  i915_gem_timeline.o \
 	  i915_gem_userptr.o \
 	  i915_gemfs.o \
 	  i915_query.o \
 	  i915_request.o \
+	  i915_timeline.o \
 	  i915_trace_points.o \
 	  i915_vma.o \
 	  intel_breadcrumbs.o \
@@ -79,7 +84,8 @@ i915-y += i915_cmd_parser.o \
 	  intel_lrc.o \
 	  intel_mocs.o \
 	  intel_ringbuffer.o \
-	  intel_uncore.o
+	  intel_uncore.o \
+	  intel_wopcm.o
 
 # general-purpose microcontroller (GuC) support
 i915-y += intel_uc.o \
@@ -152,7 +158,8 @@ i915-y += dvo_ch7017.o \
 i915-$(CONFIG_DRM_I915_CAPTURE_ERROR) += i915_gpu_error.o
 i915-$(CONFIG_DRM_I915_SELFTEST) += \
 	selftests/i915_random.o \
-	selftests/i915_selftest.o
+	selftests/i915_selftest.o \
+	selftests/igt_flush_test.o
 
 # virtual gpu code
 i915-y += i915_vgpu.o
@@ -171,7 +178,8 @@ i915-y += i915_perf.o \
 	  i915_oa_glk.o \
 	  i915_oa_cflgt2.o \
 	  i915_oa_cflgt3.o \
-	  i915_oa_cnl.o
+	  i915_oa_cnl.o \
+	  i915_oa_icl.o
 
 ifeq ($(CONFIG_DRM_I915_GVT),y)
 i915-y += intel_gvt.o
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
index d85939bd7b47..718ca08f9575 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.c
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -813,15 +813,31 @@ static inline bool is_force_nonpriv_mmio(unsigned int offset)
 }
 
 static int force_nonpriv_reg_handler(struct parser_exec_state *s,
-				     unsigned int offset, unsigned int index)
+		unsigned int offset, unsigned int index, char *cmd)
 {
 	struct intel_gvt *gvt = s->vgpu->gvt;
-	unsigned int data = cmd_val(s, index + 1);
+	unsigned int data;
+	u32 ring_base;
+	u32 nopid;
+	struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+
+	if (!strcmp(cmd, "lri"))
+		data = cmd_val(s, index + 1);
+	else {
+		gvt_err("Unexpected forcenonpriv 0x%x write from cmd %s\n",
+			offset, cmd);
+		return -EINVAL;
+	}
+
+	ring_base = dev_priv->engine[s->ring_id]->mmio_base;
+	nopid = i915_mmio_reg_offset(RING_NOPID(ring_base));
 
-	if (!intel_gvt_in_force_nonpriv_whitelist(gvt, data)) {
+	if (!intel_gvt_in_force_nonpriv_whitelist(gvt, data) &&
+			data != nopid) {
 		gvt_err("Unexpected forcenonpriv 0x%x LRI write, value=0x%x\n",
 			offset, data);
-		return -EPERM;
+		patch_value(s, cmd_ptr(s, index), nopid);
+		return 0;
 	}
 	return 0;
 }
@@ -869,7 +885,7 @@ static int cmd_reg_handler(struct parser_exec_state *s,
 		return -EINVAL;
 
 	if (is_force_nonpriv_mmio(offset) &&
-		force_nonpriv_reg_handler(s, offset, index))
+		force_nonpriv_reg_handler(s, offset, index, cmd))
 		return -EPERM;
 
 	if (offset == i915_mmio_reg_offset(DERRMR) ||
@@ -1604,7 +1620,8 @@ static int batch_buffer_needs_scan(struct parser_exec_state *s)
 	if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)
 		|| IS_KABYLAKE(gvt->dev_priv)) {
 		/* BDW decides privilege based on address space */
-		if (cmd_val(s, 0) & (1 << 8))
+		if (cmd_val(s, 0) & (1 << 8) &&
+			!(s->vgpu->scan_nonprivbb & (1 << s->ring_id)))
 			return 0;
 	}
 	return 1;
@@ -1618,6 +1635,8 @@ static int find_bb_size(struct parser_exec_state *s, unsigned long *bb_size)
 	bool bb_end = false;
 	struct intel_vgpu *vgpu = s->vgpu;
 	u32 cmd;
+	struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ?
+		s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm;
 
 	*bb_size = 0;
 
@@ -1629,18 +1648,22 @@ static int find_bb_size(struct parser_exec_state *s, unsigned long *bb_size)
 	cmd = cmd_val(s, 0);
 	info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
 	if (info == NULL) {
-		gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
-				cmd, get_opcode(cmd, s->ring_id));
+		gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n",
+				cmd, get_opcode(cmd, s->ring_id),
+				(s->buf_addr_type == PPGTT_BUFFER) ?
+				"ppgtt" : "ggtt", s->ring_id, s->workload);
 		return -EBADRQC;
 	}
 	do {
-		if (copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
+		if (copy_gma_to_hva(s->vgpu, mm,
 				gma, gma + 4, &cmd) < 0)
 			return -EFAULT;
 		info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
 		if (info == NULL) {
-			gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
-				cmd, get_opcode(cmd, s->ring_id));
+			gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n",
+				cmd, get_opcode(cmd, s->ring_id),
+				(s->buf_addr_type == PPGTT_BUFFER) ?
+				"ppgtt" : "ggtt", s->ring_id, s->workload);
 			return -EBADRQC;
 		}
 
@@ -1666,6 +1689,9 @@ static int perform_bb_shadow(struct parser_exec_state *s)
 	unsigned long gma = 0;
 	unsigned long bb_size;
 	int ret = 0;
+	struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ?
+		s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm;
+	unsigned long gma_start_offset = 0;
 
 	/* get the start gm address of the batch buffer */
 	gma = get_gma_bb_from_cmd(s, 1);
@@ -1680,8 +1706,24 @@ static int perform_bb_shadow(struct parser_exec_state *s)
 	if (!bb)
 		return -ENOMEM;
 
+	bb->ppgtt = (s->buf_addr_type == GTT_BUFFER) ? false : true;
+
+	/* the gma_start_offset stores the batch buffer's start gma's
+	 * offset relative to page boundary. so for non-privileged batch
+	 * buffer, the shadowed gem object holds exactly the same page
+	 * layout as original gem object. This is for the convience of
+	 * replacing the whole non-privilged batch buffer page to this
+	 * shadowed one in PPGTT at the same gma address. (this replacing
+	 * action is not implemented yet now, but may be necessary in
+	 * future).
+	 * for prileged batch buffer, we just change start gma address to
+	 * that of shadowed page.
+	 */
+	if (bb->ppgtt)
+		gma_start_offset = gma & ~I915_GTT_PAGE_MASK;
+
 	bb->obj = i915_gem_object_create(s->vgpu->gvt->dev_priv,
-					 roundup(bb_size, PAGE_SIZE));
+			 roundup(bb_size + gma_start_offset, PAGE_SIZE));
 	if (IS_ERR(bb->obj)) {
 		ret = PTR_ERR(bb->obj);
 		goto err_free_bb;
@@ -1702,9 +1744,9 @@ static int perform_bb_shadow(struct parser_exec_state *s)
 		bb->clflush &= ~CLFLUSH_BEFORE;
 	}
 
-	ret = copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
+	ret = copy_gma_to_hva(s->vgpu, mm,
 			      gma, gma + bb_size,
-			      bb->va);
+			      bb->va + gma_start_offset);
 	if (ret < 0) {
 		gvt_vgpu_err("fail to copy guest ring buffer\n");
 		ret = -EFAULT;
@@ -1730,7 +1772,7 @@ static int perform_bb_shadow(struct parser_exec_state *s)
 	 * buffer's gma in pair. After all, we don't want to pin the shadow
 	 * buffer here (too early).
 	 */
-	s->ip_va = bb->va;
+	s->ip_va = bb->va + gma_start_offset;
 	s->ip_gma = gma;
 	return 0;
 err_unmap:
@@ -2469,15 +2511,18 @@ static int cmd_parser_exec(struct parser_exec_state *s)
 
 	info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
 	if (info == NULL) {
-		gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
-				cmd, get_opcode(cmd, s->ring_id));
+		gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n",
+				cmd, get_opcode(cmd, s->ring_id),
+				(s->buf_addr_type == PPGTT_BUFFER) ?
+				"ppgtt" : "ggtt", s->ring_id, s->workload);
 		return -EBADRQC;
 	}
 
 	s->info = info;
 
 	trace_gvt_command(vgpu->id, s->ring_id, s->ip_gma, s->ip_va,
-			  cmd_length(s), s->buf_type);
+			  cmd_length(s), s->buf_type, s->buf_addr_type,
+			  s->workload, info->name);
 
 	if (info->handler) {
 		ret = info->handler(s);
diff --git a/drivers/gpu/drm/i915/gvt/debugfs.c b/drivers/gpu/drm/i915/gvt/debugfs.c
index 32a66dfdf112..2ec89bcb59f1 100644
--- a/drivers/gpu/drm/i915/gvt/debugfs.c
+++ b/drivers/gpu/drm/i915/gvt/debugfs.c
@@ -122,18 +122,69 @@ static int vgpu_mmio_diff_show(struct seq_file *s, void *unused)
 	seq_printf(s, "Total: %d, Diff: %d\n", param.total, param.diff);
 	return 0;
 }
+DEFINE_SHOW_ATTRIBUTE(vgpu_mmio_diff);
 
-static int vgpu_mmio_diff_open(struct inode *inode, struct file *file)
+static int
+vgpu_scan_nonprivbb_get(void *data, u64 *val)
 {
-	return single_open(file, vgpu_mmio_diff_show, inode->i_private);
+	struct intel_vgpu *vgpu = (struct intel_vgpu *)data;
+	*val = vgpu->scan_nonprivbb;
+	return 0;
 }
 
-static const struct file_operations vgpu_mmio_diff_fops = {
-	.open		= vgpu_mmio_diff_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
+/*
+ * set/unset bit engine_id of vgpu->scan_nonprivbb to turn on/off scanning
+ * of non-privileged batch buffer. e.g.
+ * if vgpu->scan_nonprivbb=3, then it will scan non-privileged batch buffer
+ * on engine 0 and 1.
+ */
+static int
+vgpu_scan_nonprivbb_set(void *data, u64 val)
+{
+	struct intel_vgpu *vgpu = (struct intel_vgpu *)data;
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	enum intel_engine_id id;
+	char buf[128], *s;
+	int len;
+
+	val &= (1 << I915_NUM_ENGINES) - 1;
+
+	if (vgpu->scan_nonprivbb == val)
+		return 0;
+
+	if (!val)
+		goto done;
+
+	len = sprintf(buf,
+		"gvt: vgpu %d turns on non-privileged batch buffers scanning on Engines:",
+		vgpu->id);
+
+	s = buf + len;
+
+	for (id = 0; id < I915_NUM_ENGINES; id++) {
+		struct intel_engine_cs *engine;
+
+		engine = dev_priv->engine[id];
+		if (engine && (val & (1 << id))) {
+			len = snprintf(s, 4, "%d, ", engine->id);
+			s += len;
+		} else
+			val &=  ~(1 << id);
+	}
+
+	if (val)
+		sprintf(s, "low performance expected.");
+
+	pr_warn("%s\n", buf);
+
+done:
+	vgpu->scan_nonprivbb = val;
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(vgpu_scan_nonprivbb_fops,
+			vgpu_scan_nonprivbb_get, vgpu_scan_nonprivbb_set,
+			"0x%llx\n");
 
 /**
  * intel_gvt_debugfs_add_vgpu - register debugfs entries for a vGPU
@@ -162,6 +213,11 @@ int intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu)
 	if (!ent)
 		return -ENOMEM;
 
+	ent = debugfs_create_file("scan_nonprivbb", 0644, vgpu->debugfs,
+				 vgpu, &vgpu_scan_nonprivbb_fops);
+	if (!ent)
+		return -ENOMEM;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index efacd8abbedc..05d15a095310 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -99,7 +99,6 @@ struct intel_vgpu_fence {
 struct intel_vgpu_mmio {
 	void *vreg;
 	void *sreg;
-	bool disable_warn_untrack;
 };
 
 #define INTEL_GVT_MAX_BAR_NUM 4
@@ -226,6 +225,7 @@ struct intel_vgpu {
 
 	struct completion vblank_done;
 
+	u32 scan_nonprivbb;
 };
 
 /* validating GM healthy status*/
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index a33c1c3e4a21..4b6532fb789a 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -191,6 +191,8 @@ static int sanitize_fence_mmio_access(struct intel_vgpu *vgpu,
 	unsigned int max_fence = vgpu_fence_sz(vgpu);
 
 	if (fence_num >= max_fence) {
+		gvt_vgpu_err("access oob fence reg %d/%d\n",
+			     fence_num, max_fence);
 
 		/* When guest access oob fence regs without access
 		 * pv_info first, we treat guest not supporting GVT,
@@ -200,11 +202,6 @@ static int sanitize_fence_mmio_access(struct intel_vgpu *vgpu,
 			enter_failsafe_mode(vgpu,
 					GVT_FAILSAFE_UNSUPPORTED_GUEST);
 
-		if (!vgpu->mmio.disable_warn_untrack) {
-			gvt_vgpu_err("found oob fence register access\n");
-			gvt_vgpu_err("total fence %d, access fence %d\n",
-				     max_fence, fence_num);
-		}
 		memset(p_data, 0, bytes);
 		return -EINVAL;
 	}
@@ -477,22 +474,28 @@ static int force_nonpriv_write(struct intel_vgpu *vgpu,
 	unsigned int offset, void *p_data, unsigned int bytes)
 {
 	u32 reg_nonpriv = *(u32 *)p_data;
+	int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset);
+	u32 ring_base;
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	int ret = -EINVAL;
 
-	if ((bytes != 4) || ((offset & (bytes - 1)) != 0)) {
-		gvt_err("vgpu(%d) Invalid FORCE_NONPRIV offset %x(%dB)\n",
-			vgpu->id, offset, bytes);
+	if ((bytes != 4) || ((offset & (bytes - 1)) != 0) || ring_id < 0) {
+		gvt_err("vgpu(%d) ring %d Invalid FORCE_NONPRIV offset %x(%dB)\n",
+			vgpu->id, ring_id, offset, bytes);
 		return ret;
 	}
 
-	if (in_whitelist(reg_nonpriv)) {
+	ring_base = dev_priv->engine[ring_id]->mmio_base;
+
+	if (in_whitelist(reg_nonpriv) ||
+		reg_nonpriv == i915_mmio_reg_offset(RING_NOPID(ring_base))) {
 		ret = intel_vgpu_default_mmio_write(vgpu, offset, p_data,
 			bytes);
-	} else {
-		gvt_err("vgpu(%d) Invalid FORCE_NONPRIV write %x\n",
-			vgpu->id, reg_nonpriv);
-	}
-	return ret;
+	} else
+		gvt_err("vgpu(%d) Invalid FORCE_NONPRIV write %x at offset %x\n",
+			vgpu->id, reg_nonpriv, offset);
+
+	return 0;
 }
 
 static int ddi_buf_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
@@ -3092,9 +3095,7 @@ int intel_vgpu_mmio_reg_rw(struct intel_vgpu *vgpu, unsigned int offset,
 	 */
 	mmio_info = find_mmio_info(gvt, offset);
 	if (!mmio_info) {
-		if (!vgpu->mmio.disable_warn_untrack)
-			gvt_vgpu_err("untracked MMIO %08x len %d\n",
-				     offset, bytes);
+		gvt_dbg_mmio("untracked MMIO %08x len %d\n", offset, bytes);
 		goto default_rw;
 	}
 
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
index 11b71b33f1c0..e4960aff68bd 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.c
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -244,8 +244,6 @@ void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr)
 
 		/* set the bit 0:2(Core C-State ) to C0 */
 		vgpu_vreg_t(vgpu, GEN6_GT_CORE_STATUS) = 0;
-
-		vgpu->mmio.disable_warn_untrack = false;
 	} else {
 #define GVT_GEN8_MMIO_RESET_OFFSET		(0x44200)
 		/* only reset the engine related, so starting with 0x44200
diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c
index a5bac83d53a9..0f949554d118 100644
--- a/drivers/gpu/drm/i915/gvt/mmio_context.c
+++ b/drivers/gpu/drm/i915/gvt/mmio_context.c
@@ -448,7 +448,7 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
 
 bool is_inhibit_context(struct i915_gem_context *ctx, int ring_id)
 {
-	u32 *reg_state = ctx->engine[ring_id].lrc_reg_state;
+	u32 *reg_state = ctx->__engine[ring_id].lrc_reg_state;
 	u32 inhibit_mask =
 		_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
 
diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.c b/drivers/gpu/drm/i915/gvt/sched_policy.c
index 75b7bc7b344c..d053cbe1dc94 100644
--- a/drivers/gpu/drm/i915/gvt/sched_policy.c
+++ b/drivers/gpu/drm/i915/gvt/sched_policy.c
@@ -53,7 +53,6 @@ struct vgpu_sched_data {
 	bool active;
 
 	ktime_t sched_in_time;
-	ktime_t sched_out_time;
 	ktime_t sched_time;
 	ktime_t left_ts;
 	ktime_t allocated_ts;
@@ -66,17 +65,22 @@ struct gvt_sched_data {
 	struct hrtimer timer;
 	unsigned long period;
 	struct list_head lru_runq_head;
+	ktime_t expire_time;
 };
 
-static void vgpu_update_timeslice(struct intel_vgpu *pre_vgpu)
+static void vgpu_update_timeslice(struct intel_vgpu *vgpu, ktime_t cur_time)
 {
 	ktime_t delta_ts;
-	struct vgpu_sched_data *vgpu_data = pre_vgpu->sched_data;
+	struct vgpu_sched_data *vgpu_data;
 
-	delta_ts = vgpu_data->sched_out_time - vgpu_data->sched_in_time;
+	if (!vgpu || vgpu == vgpu->gvt->idle_vgpu)
+		return;
 
-	vgpu_data->sched_time += delta_ts;
-	vgpu_data->left_ts -= delta_ts;
+	vgpu_data = vgpu->sched_data;
+	delta_ts = ktime_sub(cur_time, vgpu_data->sched_in_time);
+	vgpu_data->sched_time = ktime_add(vgpu_data->sched_time, delta_ts);
+	vgpu_data->left_ts = ktime_sub(vgpu_data->left_ts, delta_ts);
+	vgpu_data->sched_in_time = cur_time;
 }
 
 #define GVT_TS_BALANCE_PERIOD_MS 100
@@ -150,11 +154,7 @@ static void try_to_schedule_next_vgpu(struct intel_gvt *gvt)
 	}
 
 	cur_time = ktime_get();
-	if (scheduler->current_vgpu) {
-		vgpu_data = scheduler->current_vgpu->sched_data;
-		vgpu_data->sched_out_time = cur_time;
-		vgpu_update_timeslice(scheduler->current_vgpu);
-	}
+	vgpu_update_timeslice(scheduler->current_vgpu, cur_time);
 	vgpu_data = scheduler->next_vgpu->sched_data;
 	vgpu_data->sched_in_time = cur_time;
 
@@ -226,17 +226,22 @@ static void tbs_sched_func(struct gvt_sched_data *sched_data)
 void intel_gvt_schedule(struct intel_gvt *gvt)
 {
 	struct gvt_sched_data *sched_data = gvt->scheduler.sched_data;
-	static uint64_t timer_check;
+	ktime_t cur_time;
 
 	mutex_lock(&gvt->lock);
+	cur_time = ktime_get();
 
 	if (test_and_clear_bit(INTEL_GVT_REQUEST_SCHED,
 				(void *)&gvt->service_request)) {
-		if (!(timer_check++ % GVT_TS_BALANCE_PERIOD_MS))
+		if (cur_time >= sched_data->expire_time) {
 			gvt_balance_timeslice(sched_data);
+			sched_data->expire_time = ktime_add_ms(
+				cur_time, GVT_TS_BALANCE_PERIOD_MS);
+		}
 	}
 	clear_bit(INTEL_GVT_REQUEST_EVENT_SCHED, (void *)&gvt->service_request);
 
+	vgpu_update_timeslice(gvt->scheduler.current_vgpu, cur_time);
 	tbs_sched_func(sched_data);
 
 	mutex_unlock(&gvt->lock);
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index 638abe84857c..c2d183b91500 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -58,7 +58,7 @@ static void update_shadow_pdps(struct intel_vgpu_workload *workload)
 	int ring_id = workload->ring_id;
 	struct i915_gem_context *shadow_ctx = vgpu->submission.shadow_ctx;
 	struct drm_i915_gem_object *ctx_obj =
-		shadow_ctx->engine[ring_id].state->obj;
+		shadow_ctx->__engine[ring_id].state->obj;
 	struct execlist_ring_context *shadow_ring_context;
 	struct page *page;
 
@@ -97,7 +97,7 @@ static void sr_oa_regs(struct intel_vgpu_workload *workload,
 		i915_mmio_reg_offset(EU_PERF_CNTL6),
 	};
 
-	if (!workload || !reg_state || workload->ring_id != RCS)
+	if (workload->ring_id != RCS)
 		return;
 
 	if (save) {
@@ -130,7 +130,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
 	int ring_id = workload->ring_id;
 	struct i915_gem_context *shadow_ctx = vgpu->submission.shadow_ctx;
 	struct drm_i915_gem_object *ctx_obj =
-		shadow_ctx->engine[ring_id].state->obj;
+		shadow_ctx->__engine[ring_id].state->obj;
 	struct execlist_ring_context *shadow_ring_context;
 	struct page *page;
 	void *dst;
@@ -283,7 +283,7 @@ static int shadow_context_status_change(struct notifier_block *nb,
 static void shadow_context_descriptor_update(struct i915_gem_context *ctx,
 		struct intel_engine_cs *engine)
 {
-	struct intel_context *ce = &ctx->engine[engine->id];
+	struct intel_context *ce = to_intel_context(ctx, engine);
 	u64 desc = 0;
 
 	desc = ce->lrc_desc;
@@ -389,7 +389,7 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
 	 * shadow_ctx pages invalid. So gvt need to pin itself. After update
 	 * the guest context, gvt can unpin the shadow_ctx safely.
 	 */
-	ring = engine->context_pin(engine, shadow_ctx);
+	ring = intel_context_pin(shadow_ctx, engine);
 	if (IS_ERR(ring)) {
 		ret = PTR_ERR(ring);
 		gvt_vgpu_err("fail to pin shadow context\n");
@@ -403,7 +403,7 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
 	return 0;
 
 err_unpin:
-	engine->context_unpin(engine, shadow_ctx);
+	intel_context_unpin(shadow_ctx, engine);
 err_shadow:
 	release_shadow_wa_ctx(&workload->wa_ctx);
 err_scan:
@@ -437,7 +437,7 @@ static int intel_gvt_generate_request(struct intel_vgpu_workload *workload)
 	return 0;
 
 err_unpin:
-	engine->context_unpin(engine, shadow_ctx);
+	intel_context_unpin(shadow_ctx, engine);
 	release_shadow_wa_ctx(&workload->wa_ctx);
 	return ret;
 }
@@ -452,12 +452,6 @@ static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
 	int ret;
 
 	list_for_each_entry(bb, &workload->shadow_bb, list) {
-		bb->vma = i915_gem_object_ggtt_pin(bb->obj, NULL, 0, 0, 0);
-		if (IS_ERR(bb->vma)) {
-			ret = PTR_ERR(bb->vma);
-			goto err;
-		}
-
 		/* For privilge batch buffer and not wa_ctx, the bb_start_cmd_va
 		 * is only updated into ring_scan_buffer, not real ring address
 		 * allocated in later copy_workload_to_ring_buffer. pls be noted
@@ -469,25 +463,53 @@ static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
 			bb->bb_start_cmd_va = workload->shadow_ring_buffer_va
 				+ bb->bb_offset;
 
-		/* relocate shadow batch buffer */
-		bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma);
-		if (gmadr_bytes == 8)
-			bb->bb_start_cmd_va[2] = 0;
+		if (bb->ppgtt) {
+			/* for non-priv bb, scan&shadow is only for
+			 * debugging purpose, so the content of shadow bb
+			 * is the same as original bb. Therefore,
+			 * here, rather than switch to shadow bb's gma
+			 * address, we directly use original batch buffer's
+			 * gma address, and send original bb to hardware
+			 * directly
+			 */
+			if (bb->clflush & CLFLUSH_AFTER) {
+				drm_clflush_virt_range(bb->va,
+						bb->obj->base.size);
+				bb->clflush &= ~CLFLUSH_AFTER;
+			}
+			i915_gem_obj_finish_shmem_access(bb->obj);
+			bb->accessing = false;
+
+		} else {
+			bb->vma = i915_gem_object_ggtt_pin(bb->obj,
+					NULL, 0, 0, 0);
+			if (IS_ERR(bb->vma)) {
+				ret = PTR_ERR(bb->vma);
+				goto err;
+			}
 
-		/* No one is going to touch shadow bb from now on. */
-		if (bb->clflush & CLFLUSH_AFTER) {
-			drm_clflush_virt_range(bb->va, bb->obj->base.size);
-			bb->clflush &= ~CLFLUSH_AFTER;
-		}
+			/* relocate shadow batch buffer */
+			bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma);
+			if (gmadr_bytes == 8)
+				bb->bb_start_cmd_va[2] = 0;
 
-		ret = i915_gem_object_set_to_gtt_domain(bb->obj, false);
-		if (ret)
-			goto err;
+			/* No one is going to touch shadow bb from now on. */
+			if (bb->clflush & CLFLUSH_AFTER) {
+				drm_clflush_virt_range(bb->va,
+						bb->obj->base.size);
+				bb->clflush &= ~CLFLUSH_AFTER;
+			}
 
-		i915_gem_obj_finish_shmem_access(bb->obj);
-		bb->accessing = false;
+			ret = i915_gem_object_set_to_gtt_domain(bb->obj,
+					false);
+			if (ret)
+				goto err;
 
-		i915_vma_move_to_active(bb->vma, workload->req, 0);
+			i915_gem_obj_finish_shmem_access(bb->obj);
+			bb->accessing = false;
+
+			i915_vma_move_to_active(bb->vma, workload->req, 0);
+		}
 	}
 	return 0;
 err:
@@ -504,7 +526,7 @@ static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 	struct intel_vgpu_submission *s = &workload->vgpu->submission;
 	struct i915_gem_context *shadow_ctx = s->shadow_ctx;
 	struct drm_i915_gem_object *ctx_obj =
-		shadow_ctx->engine[ring_id].state->obj;
+		shadow_ctx->__engine[ring_id].state->obj;
 	struct execlist_ring_context *shadow_ring_context;
 	struct page *page;
 
@@ -666,7 +688,7 @@ static int dispatch_workload(struct intel_vgpu_workload *workload)
 
 	ret = prepare_workload(workload);
 	if (ret) {
-		engine->context_unpin(engine, shadow_ctx);
+		intel_context_unpin(shadow_ctx, engine);
 		goto out;
 	}
 
@@ -749,7 +771,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
 	struct i915_gem_context *shadow_ctx = s->shadow_ctx;
 	int ring_id = workload->ring_id;
 	struct drm_i915_gem_object *ctx_obj =
-		shadow_ctx->engine[ring_id].state->obj;
+		shadow_ctx->__engine[ring_id].state->obj;
 	struct execlist_ring_context *shadow_ring_context;
 	struct page *page;
 	void *src;
@@ -876,7 +898,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 		}
 		mutex_lock(&dev_priv->drm.struct_mutex);
 		/* unpin shadow ctx as the shadow_ctx update is done */
-		engine->context_unpin(engine, s->shadow_ctx);
+		intel_context_unpin(s->shadow_ctx, engine);
 		mutex_unlock(&dev_priv->drm.struct_mutex);
 	}
 
@@ -1134,9 +1156,6 @@ int intel_vgpu_setup_submission(struct intel_vgpu *vgpu)
 	if (IS_ERR(s->shadow_ctx))
 		return PTR_ERR(s->shadow_ctx);
 
-	if (HAS_LOGICAL_RING_PREEMPTION(vgpu->gvt->dev_priv))
-		s->shadow_ctx->priority = INT_MAX;
-
 	bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES);
 
 	s->workloads = kmem_cache_create_usercopy("gvt-g_vgpu_workload",
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h
index 486ed57a4ad1..6c644782193e 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.h
+++ b/drivers/gpu/drm/i915/gvt/scheduler.h
@@ -125,6 +125,7 @@ struct intel_vgpu_shadow_bb {
 	unsigned int clflush;
 	bool accessing;
 	unsigned long bb_offset;
+	bool ppgtt;
 };
 
 #define workload_q_head(vgpu, ring_id) \
diff --git a/drivers/gpu/drm/i915/gvt/trace.h b/drivers/gpu/drm/i915/gvt/trace.h
index 82093f1e8612..1fd64202d74e 100644
--- a/drivers/gpu/drm/i915/gvt/trace.h
+++ b/drivers/gpu/drm/i915/gvt/trace.h
@@ -224,19 +224,25 @@ TRACE_EVENT(oos_sync,
 	TP_printk("%s", __entry->buf)
 );
 
+#define GVT_CMD_STR_LEN 40
 TRACE_EVENT(gvt_command,
-	TP_PROTO(u8 vgpu_id, u8 ring_id, u32 ip_gma, u32 *cmd_va, u32 cmd_len,
-		 u32 buf_type),
+	TP_PROTO(u8 vgpu_id, u8 ring_id, u32 ip_gma, u32 *cmd_va,
+		u32 cmd_len,  u32 buf_type, u32 buf_addr_type,
+		void *workload, char *cmd_name),
 
-	TP_ARGS(vgpu_id, ring_id, ip_gma, cmd_va, cmd_len, buf_type),
+	TP_ARGS(vgpu_id, ring_id, ip_gma, cmd_va, cmd_len, buf_type,
+		buf_addr_type, workload, cmd_name),
 
 	TP_STRUCT__entry(
 		__field(u8, vgpu_id)
 		__field(u8, ring_id)
 		__field(u32, ip_gma)
 		__field(u32, buf_type)
+		__field(u32, buf_addr_type)
 		__field(u32, cmd_len)
+		__field(void*, workload)
 		__dynamic_array(u32, raw_cmd, cmd_len)
+		__array(char, cmd_name, GVT_CMD_STR_LEN)
 	),
 
 	TP_fast_assign(
@@ -244,17 +250,25 @@ TRACE_EVENT(gvt_command,
 		__entry->ring_id = ring_id;
 		__entry->ip_gma = ip_gma;
 		__entry->buf_type = buf_type;
+		__entry->buf_addr_type = buf_addr_type;
 		__entry->cmd_len = cmd_len;
+		__entry->workload = workload;
+		snprintf(__entry->cmd_name, GVT_CMD_STR_LEN, "%s", cmd_name);
 		memcpy(__get_dynamic_array(raw_cmd), cmd_va, cmd_len * sizeof(*cmd_va));
 	),
 
 
-	TP_printk("vgpu%d ring %d: buf_type %u, ip_gma %08x, raw cmd %s",
+	TP_printk("vgpu%d ring %d: address_type %u, buf_type %u, ip_gma %08x,cmd (name=%s,len=%u,raw cmd=%s), workload=%p\n",
 		__entry->vgpu_id,
 		__entry->ring_id,
+		__entry->buf_addr_type,
 		__entry->buf_type,
 		__entry->ip_gma,
-		__print_array(__get_dynamic_array(raw_cmd), __entry->cmd_len, 4))
+		__entry->cmd_name,
+		__entry->cmd_len,
+		__print_array(__get_dynamic_array(raw_cmd),
+			__entry->cmd_len, 4),
+		__entry->workload)
 );
 
 #define GVT_TEMP_STR_LEN 10
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 89f7ff2c652e..13e7b9e4a6e6 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -377,16 +377,19 @@ static void print_batch_pool_stats(struct seq_file *m,
 	print_file_stats(m, "[k]batch pool", stats);
 }
 
-static int per_file_ctx_stats(int id, void *ptr, void *data)
+static int per_file_ctx_stats(int idx, void *ptr, void *data)
 {
 	struct i915_gem_context *ctx = ptr;
-	int n;
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	for_each_engine(engine, ctx->i915, id) {
+		struct intel_context *ce = to_intel_context(ctx, engine);
 
-	for (n = 0; n < ARRAY_SIZE(ctx->engine); n++) {
-		if (ctx->engine[n].state)
-			per_file_stats(0, ctx->engine[n].state->obj, data);
-		if (ctx->engine[n].ring)
-			per_file_stats(0, ctx->engine[n].ring->vma->obj, data);
+		if (ce->state)
+			per_file_stats(0, ce->state->obj, data);
+		if (ce->ring)
+			per_file_stats(0, ce->ring->vma->obj, data);
 	}
 
 	return 0;
@@ -1215,20 +1218,20 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 0 :
 			    rp_state_cap >> 16) & 0xff;
 		max_freq *= (IS_GEN9_BC(dev_priv) ||
-			     IS_CANNONLAKE(dev_priv) ? GEN9_FREQ_SCALER : 1);
+			     INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
 
 		max_freq = (rp_state_cap & 0xff00) >> 8;
 		max_freq *= (IS_GEN9_BC(dev_priv) ||
-			     IS_CANNONLAKE(dev_priv) ? GEN9_FREQ_SCALER : 1);
+			     INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
 
 		max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 16 :
 			    rp_state_cap >> 0) & 0xff;
 		max_freq *= (IS_GEN9_BC(dev_priv) ||
-			     IS_CANNONLAKE(dev_priv) ? GEN9_FREQ_SCALER : 1);
+			     INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
 		seq_printf(m, "Max overclocked frequency: %dMHz\n",
@@ -1340,10 +1343,9 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 		struct rb_node *rb;
 
 		seq_printf(m, "%s:\n", engine->name);
-		seq_printf(m, "\tseqno = %x [current %x, last %x], inflight %d\n",
+		seq_printf(m, "\tseqno = %x [current %x, last %x]\n",
 			   engine->hangcheck.seqno, seqno[id],
-			   intel_engine_last_submit(engine),
-			   engine->timeline->inflight_seqnos);
+			   intel_engine_last_submit(engine));
 		seq_printf(m, "\twaiters? %s, fake irq active? %s, stalled? %s\n",
 			   yesno(intel_engine_has_waiter(engine)),
 			   yesno(test_bit(engine->id,
@@ -1796,9 +1798,9 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
-	int ret = 0;
-	int gpu_freq, ia_freq;
 	unsigned int max_gpu_freq, min_gpu_freq;
+	int gpu_freq, ia_freq;
+	int ret;
 
 	if (!HAS_LLC(dev_priv))
 		return -ENODEV;
@@ -1809,13 +1811,12 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 	if (ret)
 		goto out;
 
-	if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+	min_gpu_freq = rps->min_freq;
+	max_gpu_freq = rps->max_freq;
+	if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
 		/* Convert GT frequency to 50 HZ units */
-		min_gpu_freq = rps->min_freq_softlimit / GEN9_FREQ_SCALER;
-		max_gpu_freq = rps->max_freq_softlimit / GEN9_FREQ_SCALER;
-	} else {
-		min_gpu_freq = rps->min_freq_softlimit;
-		max_gpu_freq = rps->max_freq_softlimit;
+		min_gpu_freq /= GEN9_FREQ_SCALER;
+		max_gpu_freq /= GEN9_FREQ_SCALER;
 	}
 
 	seq_puts(m, "GPU freq (MHz)\tEffective CPU freq (MHz)\tEffective Ring freq (MHz)\n");
@@ -1828,7 +1829,7 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 		seq_printf(m, "%d\t\t%d\t\t\t\t%d\n",
 			   intel_gpu_freq(dev_priv, (gpu_freq *
 						     (IS_GEN9_BC(dev_priv) ||
-						      IS_CANNONLAKE(dev_priv) ?
+						      INTEL_GEN(dev_priv) >= 10 ?
 						      GEN9_FREQ_SCALER : 1))),
 			   ((ia_freq >> 0) & 0xff) * 100,
 			   ((ia_freq >> 8) & 0xff) * 100);
@@ -1923,8 +1924,8 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 
 static void describe_ctx_ring(struct seq_file *m, struct intel_ring *ring)
 {
-	seq_printf(m, " (ringbuffer, space: %d, head: %u, tail: %u)",
-		   ring->space, ring->head, ring->tail);
+	seq_printf(m, " (ringbuffer, space: %d, head: %u, tail: %u, emit: %u)",
+		   ring->space, ring->head, ring->tail, ring->emit);
 }
 
 static int i915_context_status(struct seq_file *m, void *unused)
@@ -1961,7 +1962,8 @@ static int i915_context_status(struct seq_file *m, void *unused)
 		seq_putc(m, '\n');
 
 		for_each_engine(engine, dev_priv, id) {
-			struct intel_context *ce = &ctx->engine[engine->id];
+			struct intel_context *ce =
+				to_intel_context(ctx, engine);
 
 			seq_printf(m, "%s: ", engine->name);
 			if (ce->state)
@@ -2326,30 +2328,45 @@ static int i915_guc_load_status_info(struct seq_file *m, void *data)
 	return 0;
 }
 
-static void i915_guc_log_info(struct seq_file *m,
-			      struct drm_i915_private *dev_priv)
+static const char *
+stringify_guc_log_type(enum guc_log_buffer_type type)
 {
-	struct intel_guc *guc = &dev_priv->guc;
+	switch (type) {
+	case GUC_ISR_LOG_BUFFER:
+		return "ISR";
+	case GUC_DPC_LOG_BUFFER:
+		return "DPC";
+	case GUC_CRASH_DUMP_LOG_BUFFER:
+		return "CRASH";
+	default:
+		MISSING_CASE(type);
+	}
 
-	seq_puts(m, "\nGuC logging stats:\n");
+	return "";
+}
 
-	seq_printf(m, "\tISR:   flush count %10u, overflow count %10u\n",
-		   guc->log.flush_count[GUC_ISR_LOG_BUFFER],
-		   guc->log.total_overflow_count[GUC_ISR_LOG_BUFFER]);
+static void i915_guc_log_info(struct seq_file *m,
+			      struct drm_i915_private *dev_priv)
+{
+	struct intel_guc_log *log = &dev_priv->guc.log;
+	enum guc_log_buffer_type type;
 
-	seq_printf(m, "\tDPC:   flush count %10u, overflow count %10u\n",
-		   guc->log.flush_count[GUC_DPC_LOG_BUFFER],
-		   guc->log.total_overflow_count[GUC_DPC_LOG_BUFFER]);
+	if (!intel_guc_log_relay_enabled(log)) {
+		seq_puts(m, "GuC log relay disabled\n");
+		return;
+	}
 
-	seq_printf(m, "\tCRASH: flush count %10u, overflow count %10u\n",
-		   guc->log.flush_count[GUC_CRASH_DUMP_LOG_BUFFER],
-		   guc->log.total_overflow_count[GUC_CRASH_DUMP_LOG_BUFFER]);
+	seq_puts(m, "GuC logging stats:\n");
 
-	seq_printf(m, "\tTotal flush interrupt count: %u\n",
-		   guc->log.flush_interrupt_count);
+	seq_printf(m, "\tRelay full count: %u\n",
+		   log->relay.full_count);
 
-	seq_printf(m, "\tCapture miss count: %u\n",
-		   guc->log.capture_miss_count);
+	for (type = GUC_ISR_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) {
+		seq_printf(m, "\t%s:\tflush count %10u, overflow count %10u\n",
+			   stringify_guc_log_type(type),
+			   log->stats[type].flush,
+			   log->stats[type].sampled_overflow);
+	}
 }
 
 static void i915_guc_client_info(struct seq_file *m,
@@ -2379,14 +2396,19 @@ static int i915_guc_info(struct seq_file *m, void *data)
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	const struct intel_guc *guc = &dev_priv->guc;
 
-	if (!USES_GUC_SUBMISSION(dev_priv))
+	if (!USES_GUC(dev_priv))
 		return -ENODEV;
 
+	i915_guc_log_info(m, dev_priv);
+
+	if (!USES_GUC_SUBMISSION(dev_priv))
+		return 0;
+
 	GEM_BUG_ON(!guc->execbuf_client);
 
-	seq_printf(m, "Doorbell map:\n");
+	seq_printf(m, "\nDoorbell map:\n");
 	seq_printf(m, "\t%*pb\n", GUC_NUM_DOORBELLS, guc->doorbell_bitmap);
-	seq_printf(m, "Doorbell next cacheline: 0x%x\n\n", guc->db_cacheline);
+	seq_printf(m, "Doorbell next cacheline: 0x%x\n", guc->db_cacheline);
 
 	seq_printf(m, "\nGuC execbuf client @ %p:\n", guc->execbuf_client);
 	i915_guc_client_info(m, dev_priv, guc->execbuf_client);
@@ -2396,8 +2418,6 @@ static int i915_guc_info(struct seq_file *m, void *data)
 		i915_guc_client_info(m, dev_priv, guc->preempt_client);
 	}
 
-	i915_guc_log_info(m, dev_priv);
-
 	/* Add more as required ... */
 
 	return 0;
@@ -2496,35 +2516,73 @@ static int i915_guc_log_dump(struct seq_file *m, void *data)
 	return 0;
 }
 
-static int i915_guc_log_control_get(void *data, u64 *val)
+static int i915_guc_log_level_get(void *data, u64 *val)
 {
 	struct drm_i915_private *dev_priv = data;
 
-	if (!HAS_GUC(dev_priv))
+	if (!USES_GUC(dev_priv))
 		return -ENODEV;
 
-	if (!dev_priv->guc.log.vma)
-		return -EINVAL;
-
-	*val = i915_modparams.guc_log_level;
+	*val = intel_guc_log_level_get(&dev_priv->guc.log);
 
 	return 0;
 }
 
-static int i915_guc_log_control_set(void *data, u64 val)
+static int i915_guc_log_level_set(void *data, u64 val)
 {
 	struct drm_i915_private *dev_priv = data;
 
-	if (!HAS_GUC(dev_priv))
+	if (!USES_GUC(dev_priv))
 		return -ENODEV;
 
-	return intel_guc_log_control(&dev_priv->guc, val);
+	return intel_guc_log_level_set(&dev_priv->guc.log, val);
 }
 
-DEFINE_SIMPLE_ATTRIBUTE(i915_guc_log_control_fops,
-			i915_guc_log_control_get, i915_guc_log_control_set,
+DEFINE_SIMPLE_ATTRIBUTE(i915_guc_log_level_fops,
+			i915_guc_log_level_get, i915_guc_log_level_set,
 			"%lld\n");
 
+static int i915_guc_log_relay_open(struct inode *inode, struct file *file)
+{
+	struct drm_i915_private *dev_priv = inode->i_private;
+
+	if (!USES_GUC(dev_priv))
+		return -ENODEV;
+
+	file->private_data = &dev_priv->guc.log;
+
+	return intel_guc_log_relay_open(&dev_priv->guc.log);
+}
+
+static ssize_t
+i915_guc_log_relay_write(struct file *filp,
+			 const char __user *ubuf,
+			 size_t cnt,
+			 loff_t *ppos)
+{
+	struct intel_guc_log *log = filp->private_data;
+
+	intel_guc_log_relay_flush(log);
+
+	return cnt;
+}
+
+static int i915_guc_log_relay_release(struct inode *inode, struct file *file)
+{
+	struct drm_i915_private *dev_priv = inode->i_private;
+
+	intel_guc_log_relay_close(&dev_priv->guc.log);
+
+	return 0;
+}
+
+static const struct file_operations i915_guc_log_relay_fops = {
+	.owner = THIS_MODULE,
+	.open = i915_guc_log_relay_open,
+	.write = i915_guc_log_relay_write,
+	.release = i915_guc_log_relay_release,
+};
+
 static const char *psr2_live_status(u32 val)
 {
 	static const char * const live_status[] = {
@@ -2548,6 +2606,26 @@ static const char *psr2_live_status(u32 val)
 	return "unknown";
 }
 
+static const char *psr_sink_status(u8 val)
+{
+	static const char * const sink_status[] = {
+		"inactive",
+		"transition to active, capture and display",
+		"active, display from RFB",
+		"active, capture and display on sink device timings",
+		"transition to inactive, capture and display, timing re-sync",
+		"reserved",
+		"reserved",
+		"sink internal error"
+	};
+
+	val &= DP_PSR_SINK_STATE_MASK;
+	if (val < ARRAY_SIZE(sink_status))
+		return sink_status[val];
+
+	return "unknown";
+}
+
 static int i915_edp_psr_status(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -2569,14 +2647,13 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 
 	mutex_lock(&dev_priv->psr.lock);
 	seq_printf(m, "Enabled: %s\n", yesno((bool)dev_priv->psr.enabled));
-	seq_printf(m, "Active: %s\n", yesno(dev_priv->psr.active));
 	seq_printf(m, "Busy frontbuffer bits: 0x%03x\n",
 		   dev_priv->psr.busy_frontbuffer_bits);
 	seq_printf(m, "Re-enable work scheduled: %s\n",
 		   yesno(work_busy(&dev_priv->psr.work.work)));
 
 	if (HAS_DDI(dev_priv)) {
-		if (dev_priv->psr.psr2_support)
+		if (dev_priv->psr.psr2_enabled)
 			enabled = I915_READ(EDP_PSR2_CTL) & EDP_PSR2_ENABLE;
 		else
 			enabled = I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE;
@@ -2624,18 +2701,67 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 
 		seq_printf(m, "Performance_Counter: %u\n", psrperf);
 	}
-	if (dev_priv->psr.psr2_support) {
+	if (dev_priv->psr.psr2_enabled) {
 		u32 psr2 = I915_READ(EDP_PSR2_STATUS);
 
 		seq_printf(m, "EDP_PSR2_STATUS: %x [%s]\n",
 			   psr2, psr2_live_status(psr2));
 	}
+
+	if (dev_priv->psr.enabled) {
+		struct drm_dp_aux *aux = &dev_priv->psr.enabled->aux;
+		u8 val;
+
+		if (drm_dp_dpcd_readb(aux, DP_PSR_STATUS, &val) == 1)
+			seq_printf(m, "Sink PSR status: 0x%x [%s]\n", val,
+				   psr_sink_status(val));
+	}
 	mutex_unlock(&dev_priv->psr.lock);
 
+	if (READ_ONCE(dev_priv->psr.debug)) {
+		seq_printf(m, "Last attempted entry at: %lld\n",
+			   dev_priv->psr.last_entry_attempt);
+		seq_printf(m, "Last exit at: %lld\n",
+			   dev_priv->psr.last_exit);
+	}
+
 	intel_runtime_pm_put(dev_priv);
 	return 0;
 }
 
+static int
+i915_edp_psr_debug_set(void *data, u64 val)
+{
+	struct drm_i915_private *dev_priv = data;
+
+	if (!CAN_PSR(dev_priv))
+		return -ENODEV;
+
+	DRM_DEBUG_KMS("PSR debug %s\n", enableddisabled(val));
+
+	intel_runtime_pm_get(dev_priv);
+	intel_psr_irq_control(dev_priv, !!val);
+	intel_runtime_pm_put(dev_priv);
+
+	return 0;
+}
+
+static int
+i915_edp_psr_debug_get(void *data, u64 *val)
+{
+	struct drm_i915_private *dev_priv = data;
+
+	if (!CAN_PSR(dev_priv))
+		return -ENODEV;
+
+	*val = READ_ONCE(dev_priv->psr.debug);
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(i915_edp_psr_debug_fops,
+			i915_edp_psr_debug_get, i915_edp_psr_debug_set,
+			"%llu\n");
+
 static int i915_sink_crc(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -3231,7 +3357,8 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 	for (i = 0; i < dev_priv->num_shared_dpll; i++) {
 		struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
 
-		seq_printf(m, "DPLL%i: %s, id: %i\n", i, pll->name, pll->id);
+		seq_printf(m, "DPLL%i: %s, id: %i\n", i, pll->info->name,
+			   pll->info->id);
 		seq_printf(m, " crtc_mask: 0x%08x, active: 0x%x, on: %s\n",
 			   pll->state.crtc_mask, pll->active_mask, yesno(pll->on));
 		seq_printf(m, " tracked hardware state:\n");
@@ -3241,6 +3368,28 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 		seq_printf(m, " fp0:     0x%08x\n", pll->state.hw_state.fp0);
 		seq_printf(m, " fp1:     0x%08x\n", pll->state.hw_state.fp1);
 		seq_printf(m, " wrpll:   0x%08x\n", pll->state.hw_state.wrpll);
+		seq_printf(m, " cfgcr0:  0x%08x\n", pll->state.hw_state.cfgcr0);
+		seq_printf(m, " cfgcr1:  0x%08x\n", pll->state.hw_state.cfgcr1);
+		seq_printf(m, " mg_refclkin_ctl:        0x%08x\n",
+			   pll->state.hw_state.mg_refclkin_ctl);
+		seq_printf(m, " mg_clktop2_coreclkctl1: 0x%08x\n",
+			   pll->state.hw_state.mg_clktop2_coreclkctl1);
+		seq_printf(m, " mg_clktop2_hsclkctl:    0x%08x\n",
+			   pll->state.hw_state.mg_clktop2_hsclkctl);
+		seq_printf(m, " mg_pll_div0:  0x%08x\n",
+			   pll->state.hw_state.mg_pll_div0);
+		seq_printf(m, " mg_pll_div1:  0x%08x\n",
+			   pll->state.hw_state.mg_pll_div1);
+		seq_printf(m, " mg_pll_lf:    0x%08x\n",
+			   pll->state.hw_state.mg_pll_lf);
+		seq_printf(m, " mg_pll_frac_lock: 0x%08x\n",
+			   pll->state.hw_state.mg_pll_frac_lock);
+		seq_printf(m, " mg_pll_ssc:   0x%08x\n",
+			   pll->state.hw_state.mg_pll_ssc);
+		seq_printf(m, " mg_pll_bias:  0x%08x\n",
+			   pll->state.hw_state.mg_pll_bias);
+		seq_printf(m, " mg_pll_tdc_coldst_bias: 0x%08x\n",
+			   pll->state.hw_state.mg_pll_tdc_coldst_bias);
 	}
 	drm_modeset_unlock_all(dev);
 
@@ -3249,24 +3398,13 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 
 static int i915_wa_registers(struct seq_file *m, void *unused)
 {
-	int i;
-	int ret;
-	struct intel_engine_cs *engine;
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct drm_device *dev = &dev_priv->drm;
 	struct i915_workarounds *workarounds = &dev_priv->workarounds;
-	enum intel_engine_id id;
-
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
+	int i;
 
 	intel_runtime_pm_get(dev_priv);
 
 	seq_printf(m, "Workarounds applied: %d\n", workarounds->count);
-	for_each_engine(engine, dev_priv, id)
-		seq_printf(m, "HW whitelist count for %s: %d\n",
-			   engine->name, workarounds->hw_whitelist_count[id]);
 	for (i = 0; i < workarounds->count; ++i) {
 		i915_reg_t addr;
 		u32 mask, value, read;
@@ -3282,7 +3420,6 @@ static int i915_wa_registers(struct seq_file *m, void *unused)
 	}
 
 	intel_runtime_pm_put(dev_priv);
-	mutex_unlock(&dev->struct_mutex);
 
 	return 0;
 }
@@ -3567,7 +3704,8 @@ static ssize_t i915_displayport_test_active_write(struct file *file,
 
 static int i915_displayport_test_active_show(struct seq_file *m, void *data)
 {
-	struct drm_device *dev = m->private;
+	struct drm_i915_private *dev_priv = m->private;
+	struct drm_device *dev = &dev_priv->drm;
 	struct drm_connector *connector;
 	struct drm_connector_list_iter conn_iter;
 	struct intel_dp *intel_dp;
@@ -3601,10 +3739,8 @@ static int i915_displayport_test_active_show(struct seq_file *m, void *data)
 static int i915_displayport_test_active_open(struct inode *inode,
 					     struct file *file)
 {
-	struct drm_i915_private *dev_priv = inode->i_private;
-
 	return single_open(file, i915_displayport_test_active_show,
-			   &dev_priv->drm);
+			   inode->i_private);
 }
 
 static const struct file_operations i915_displayport_test_active_fops = {
@@ -3618,7 +3754,8 @@ static const struct file_operations i915_displayport_test_active_fops = {
 
 static int i915_displayport_test_data_show(struct seq_file *m, void *data)
 {
-	struct drm_device *dev = m->private;
+	struct drm_i915_private *dev_priv = m->private;
+	struct drm_device *dev = &dev_priv->drm;
 	struct drm_connector *connector;
 	struct drm_connector_list_iter conn_iter;
 	struct intel_dp *intel_dp;
@@ -3657,26 +3794,12 @@ static int i915_displayport_test_data_show(struct seq_file *m, void *data)
 
 	return 0;
 }
-static int i915_displayport_test_data_open(struct inode *inode,
-					   struct file *file)
-{
-	struct drm_i915_private *dev_priv = inode->i_private;
-
-	return single_open(file, i915_displayport_test_data_show,
-			   &dev_priv->drm);
-}
-
-static const struct file_operations i915_displayport_test_data_fops = {
-	.owner = THIS_MODULE,
-	.open = i915_displayport_test_data_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release
-};
+DEFINE_SHOW_ATTRIBUTE(i915_displayport_test_data);
 
 static int i915_displayport_test_type_show(struct seq_file *m, void *data)
 {
-	struct drm_device *dev = m->private;
+	struct drm_i915_private *dev_priv = m->private;
+	struct drm_device *dev = &dev_priv->drm;
 	struct drm_connector *connector;
 	struct drm_connector_list_iter conn_iter;
 	struct intel_dp *intel_dp;
@@ -3703,23 +3826,7 @@ static int i915_displayport_test_type_show(struct seq_file *m, void *data)
 
 	return 0;
 }
-
-static int i915_displayport_test_type_open(struct inode *inode,
-				       struct file *file)
-{
-	struct drm_i915_private *dev_priv = inode->i_private;
-
-	return single_open(file, i915_displayport_test_type_show,
-			   &dev_priv->drm);
-}
-
-static const struct file_operations i915_displayport_test_type_fops = {
-	.owner = THIS_MODULE,
-	.open = i915_displayport_test_type_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release
-};
+DEFINE_SHOW_ATTRIBUTE(i915_displayport_test_type);
 
 static void wm_latency_show(struct seq_file *m, const uint16_t wm[8])
 {
@@ -3987,8 +4094,8 @@ i915_wedged_set(void *data, u64 val)
 		engine->hangcheck.stalled = true;
 	}
 
-	i915_handle_error(i915, val, "Manually set wedged engine mask = %llx",
-			  val);
+	i915_handle_error(i915, val, I915_ERROR_CAPTURE,
+			  "Manually set wedged engine mask = %llx", val);
 
 	wait_on_bit(&i915->gpu_error.flags,
 		    I915_RESET_HANDOFF,
@@ -4151,119 +4258,6 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_drop_caches_fops,
 			i915_drop_caches_get, i915_drop_caches_set,
 			"0x%08llx\n");
 
-static int
-i915_max_freq_get(void *data, u64 *val)
-{
-	struct drm_i915_private *dev_priv = data;
-
-	if (INTEL_GEN(dev_priv) < 6)
-		return -ENODEV;
-
-	*val = intel_gpu_freq(dev_priv, dev_priv->gt_pm.rps.max_freq_softlimit);
-	return 0;
-}
-
-static int
-i915_max_freq_set(void *data, u64 val)
-{
-	struct drm_i915_private *dev_priv = data;
-	struct intel_rps *rps = &dev_priv->gt_pm.rps;
-	u32 hw_max, hw_min;
-	int ret;
-
-	if (INTEL_GEN(dev_priv) < 6)
-		return -ENODEV;
-
-	DRM_DEBUG_DRIVER("Manually setting max freq to %llu\n", val);
-
-	ret = mutex_lock_interruptible(&dev_priv->pcu_lock);
-	if (ret)
-		return ret;
-
-	/*
-	 * Turbo will still be enabled, but won't go above the set value.
-	 */
-	val = intel_freq_opcode(dev_priv, val);
-
-	hw_max = rps->max_freq;
-	hw_min = rps->min_freq;
-
-	if (val < hw_min || val > hw_max || val < rps->min_freq_softlimit) {
-		mutex_unlock(&dev_priv->pcu_lock);
-		return -EINVAL;
-	}
-
-	rps->max_freq_softlimit = val;
-
-	if (intel_set_rps(dev_priv, val))
-		DRM_DEBUG_DRIVER("failed to update RPS to new softlimit\n");
-
-	mutex_unlock(&dev_priv->pcu_lock);
-
-	return 0;
-}
-
-DEFINE_SIMPLE_ATTRIBUTE(i915_max_freq_fops,
-			i915_max_freq_get, i915_max_freq_set,
-			"%llu\n");
-
-static int
-i915_min_freq_get(void *data, u64 *val)
-{
-	struct drm_i915_private *dev_priv = data;
-
-	if (INTEL_GEN(dev_priv) < 6)
-		return -ENODEV;
-
-	*val = intel_gpu_freq(dev_priv, dev_priv->gt_pm.rps.min_freq_softlimit);
-	return 0;
-}
-
-static int
-i915_min_freq_set(void *data, u64 val)
-{
-	struct drm_i915_private *dev_priv = data;
-	struct intel_rps *rps = &dev_priv->gt_pm.rps;
-	u32 hw_max, hw_min;
-	int ret;
-
-	if (INTEL_GEN(dev_priv) < 6)
-		return -ENODEV;
-
-	DRM_DEBUG_DRIVER("Manually setting min freq to %llu\n", val);
-
-	ret = mutex_lock_interruptible(&dev_priv->pcu_lock);
-	if (ret)
-		return ret;
-
-	/*
-	 * Turbo will still be enabled, but won't go below the set value.
-	 */
-	val = intel_freq_opcode(dev_priv, val);
-
-	hw_max = rps->max_freq;
-	hw_min = rps->min_freq;
-
-	if (val < hw_min ||
-	    val > hw_max || val > rps->max_freq_softlimit) {
-		mutex_unlock(&dev_priv->pcu_lock);
-		return -EINVAL;
-	}
-
-	rps->min_freq_softlimit = val;
-
-	if (intel_set_rps(dev_priv, val))
-		DRM_DEBUG_DRIVER("failed to update RPS to new softlimit\n");
-
-	mutex_unlock(&dev_priv->pcu_lock);
-
-	return 0;
-}
-
-DEFINE_SIMPLE_ATTRIBUTE(i915_min_freq_fops,
-			i915_min_freq_get, i915_min_freq_set,
-			"%llu\n");
-
 static int
 i915_cache_sharing_get(void *data, u64 *val)
 {
@@ -4316,9 +4310,10 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_cache_sharing_fops,
 static void cherryview_sseu_device_status(struct drm_i915_private *dev_priv,
 					  struct sseu_dev_info *sseu)
 {
-	int ss_max = 2;
+#define SS_MAX 2
+	const int ss_max = SS_MAX;
+	u32 sig1[SS_MAX], sig2[SS_MAX];
 	int ss;
-	u32 sig1[ss_max], sig2[ss_max];
 
 	sig1[0] = I915_READ(CHV_POWER_SS0_SIG1);
 	sig1[1] = I915_READ(CHV_POWER_SS1_SIG1);
@@ -4342,15 +4337,16 @@ static void cherryview_sseu_device_status(struct drm_i915_private *dev_priv,
 		sseu->eu_per_subslice = max_t(unsigned int,
 					      sseu->eu_per_subslice, eu_cnt);
 	}
+#undef SS_MAX
 }
 
 static void gen10_sseu_device_status(struct drm_i915_private *dev_priv,
 				     struct sseu_dev_info *sseu)
 {
+#define SS_MAX 6
 	const struct intel_device_info *info = INTEL_INFO(dev_priv);
+	u32 s_reg[SS_MAX], eu_reg[2 * SS_MAX], eu_mask[2];
 	int s, ss;
-	u32 s_reg[info->sseu.max_slices];
-	u32 eu_reg[2 * info->sseu.max_subslices], eu_mask[2];
 
 	for (s = 0; s < info->sseu.max_slices; s++) {
 		/*
@@ -4397,15 +4393,16 @@ static void gen10_sseu_device_status(struct drm_i915_private *dev_priv,
 						      eu_cnt);
 		}
 	}
+#undef SS_MAX
 }
 
 static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
 				    struct sseu_dev_info *sseu)
 {
+#define SS_MAX 3
 	const struct intel_device_info *info = INTEL_INFO(dev_priv);
+	u32 s_reg[SS_MAX], eu_reg[2 * SS_MAX], eu_mask[2];
 	int s, ss;
-	u32 s_reg[info->sseu.max_slices];
-	u32 eu_reg[2 * info->sseu.max_subslices], eu_mask[2];
 
 	for (s = 0; s < info->sseu.max_slices; s++) {
 		s_reg[s] = I915_READ(GEN9_SLICE_PGCTL_ACK(s));
@@ -4452,6 +4449,7 @@ static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
 						      eu_cnt);
 		}
 	}
+#undef SS_MAX
 }
 
 static void broadwell_sseu_device_status(struct drm_i915_private *dev_priv,
@@ -4703,6 +4701,67 @@ static int i915_drrs_ctl_set(void *data, u64 val)
 
 DEFINE_SIMPLE_ATTRIBUTE(i915_drrs_ctl_fops, NULL, i915_drrs_ctl_set, "%llu\n");
 
+static ssize_t
+i915_fifo_underrun_reset_write(struct file *filp,
+			       const char __user *ubuf,
+			       size_t cnt, loff_t *ppos)
+{
+	struct drm_i915_private *dev_priv = filp->private_data;
+	struct intel_crtc *intel_crtc;
+	struct drm_device *dev = &dev_priv->drm;
+	int ret;
+	bool reset;
+
+	ret = kstrtobool_from_user(ubuf, cnt, &reset);
+	if (ret)
+		return ret;
+
+	if (!reset)
+		return cnt;
+
+	for_each_intel_crtc(dev, intel_crtc) {
+		struct drm_crtc_commit *commit;
+		struct intel_crtc_state *crtc_state;
+
+		ret = drm_modeset_lock_single_interruptible(&intel_crtc->base.mutex);
+		if (ret)
+			return ret;
+
+		crtc_state = to_intel_crtc_state(intel_crtc->base.state);
+		commit = crtc_state->base.commit;
+		if (commit) {
+			ret = wait_for_completion_interruptible(&commit->hw_done);
+			if (!ret)
+				ret = wait_for_completion_interruptible(&commit->flip_done);
+		}
+
+		if (!ret && crtc_state->base.active) {
+			DRM_DEBUG_KMS("Re-arming FIFO underruns on pipe %c\n",
+				      pipe_name(intel_crtc->pipe));
+
+			intel_crtc_arm_fifo_underrun(intel_crtc, crtc_state);
+		}
+
+		drm_modeset_unlock(&intel_crtc->base.mutex);
+
+		if (ret)
+			return ret;
+	}
+
+	ret = intel_fbc_reset_underrun(dev_priv);
+	if (ret)
+		return ret;
+
+	return cnt;
+}
+
+static const struct file_operations i915_fifo_underrun_reset_ops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.write = i915_fifo_underrun_reset_write,
+	.llseek = default_llseek,
+};
+
 static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_capabilities", i915_capabilities, 0},
 	{"i915_gem_objects", i915_gem_object_info, 0},
@@ -4760,8 +4819,6 @@ static const struct i915_debugfs_files {
 	const struct file_operations *fops;
 } i915_debugfs_files[] = {
 	{"i915_wedged", &i915_wedged_fops},
-	{"i915_max_freq", &i915_max_freq_fops},
-	{"i915_min_freq", &i915_min_freq_fops},
 	{"i915_cache_sharing", &i915_cache_sharing_fops},
 	{"i915_ring_missed_irq", &i915_ring_missed_irq_fops},
 	{"i915_ring_test_irq", &i915_ring_test_irq_fops},
@@ -4770,6 +4827,7 @@ static const struct i915_debugfs_files {
 	{"i915_error_state", &i915_error_state_fops},
 	{"i915_gpu_info", &i915_gpu_info_fops},
 #endif
+	{"i915_fifo_underrun_reset", &i915_fifo_underrun_reset_ops},
 	{"i915_next_seqno", &i915_next_seqno_fops},
 	{"i915_display_crc_ctl", &i915_display_crc_ctl_fops},
 	{"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
@@ -4779,10 +4837,12 @@ static const struct i915_debugfs_files {
 	{"i915_dp_test_data", &i915_displayport_test_data_fops},
 	{"i915_dp_test_type", &i915_displayport_test_type_fops},
 	{"i915_dp_test_active", &i915_displayport_test_active_fops},
-	{"i915_guc_log_control", &i915_guc_log_control_fops},
+	{"i915_guc_log_level", &i915_guc_log_level_fops},
+	{"i915_guc_log_relay", &i915_guc_log_relay_fops},
 	{"i915_hpd_storm_ctl", &i915_hpd_storm_ctl_fops},
 	{"i915_ipc_status", &i915_ipc_status_fops},
-	{"i915_drrs_ctl", &i915_drrs_ctl_fops}
+	{"i915_drrs_ctl", &i915_drrs_ctl_fops},
+	{"i915_edp_psr_debug", &i915_edp_psr_debug_fops}
 };
 
 int i915_debugfs_register(struct drm_i915_private *dev_priv)
@@ -4876,19 +4936,7 @@ static int i915_dpcd_show(struct seq_file *m, void *data)
 
 	return 0;
 }
-
-static int i915_dpcd_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, i915_dpcd_show, inode->i_private);
-}
-
-static const struct file_operations i915_dpcd_fops = {
-	.owner = THIS_MODULE,
-	.open = i915_dpcd_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(i915_dpcd);
 
 static int i915_panel_show(struct seq_file *m, void *data)
 {
@@ -4910,19 +4958,7 @@ static int i915_panel_show(struct seq_file *m, void *data)
 
 	return 0;
 }
-
-static int i915_panel_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, i915_panel_show, inode->i_private);
-}
-
-static const struct file_operations i915_panel_fops = {
-	.owner = THIS_MODULE,
-	.open = i915_panel_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(i915_panel);
 
 /**
  * i915_debugfs_connector_add - add i915 specific connector debugfs files
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 3b4daafebdcb..9c449b8d8eab 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -101,7 +101,13 @@ __i915_printk(struct drm_i915_private *dev_priv, const char *level,
 		   __builtin_return_address(0), &vaf);
 
 	if (is_error && !shown_bug_once) {
-		dev_notice(kdev, "%s", FDO_BUG_MSG);
+		/*
+		 * Ask the user to file a bug report for the error, except
+		 * if they may have caused the bug by fiddling with unsafe
+		 * module parameters.
+		 */
+		if (!test_taint(TAINT_USER))
+			dev_notice(kdev, "%s", FDO_BUG_MSG);
 		shown_bug_once = true;
 	}
 
@@ -377,9 +383,9 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data,
 		value = INTEL_INFO(dev_priv)->sseu.min_eu_in_pool;
 		break;
 	case I915_PARAM_HUC_STATUS:
-		intel_runtime_pm_get(dev_priv);
-		value = I915_READ(HUC_STATUS2) & HUC_FW_VERIFIED;
-		intel_runtime_pm_put(dev_priv);
+		value = intel_huc_check_status(&dev_priv->huc);
+		if (value < 0)
+			return value;
 		break;
 	case I915_PARAM_MMAP_GTT_VERSION:
 		/* Though we've started our numbering from 1, and so class all
@@ -695,11 +701,9 @@ static int i915_load_modeset_init(struct drm_device *dev)
 	if (ret)
 		goto cleanup_irq;
 
-	intel_uc_init_fw(dev_priv);
-
 	ret = i915_gem_init(dev_priv);
 	if (ret)
-		goto cleanup_uc;
+		goto cleanup_irq;
 
 	intel_setup_overlay(dev_priv);
 
@@ -719,8 +723,6 @@ static int i915_load_modeset_init(struct drm_device *dev)
 	if (i915_gem_suspend(dev_priv))
 		DRM_ERROR("failed to idle hardware; continuing to unload!\n");
 	i915_gem_fini(dev_priv);
-cleanup_uc:
-	intel_uc_fini_fw(dev_priv);
 cleanup_irq:
 	drm_irq_uninstall(dev);
 	intel_teardown_gmbus(dev_priv);
@@ -922,16 +924,21 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 	mutex_init(&dev_priv->wm.wm_mutex);
 	mutex_init(&dev_priv->pps_mutex);
 
-	intel_uc_init_early(dev_priv);
 	i915_memcpy_init_early(dev_priv);
 
 	ret = i915_workqueues_init(dev_priv);
 	if (ret < 0)
 		goto err_engines;
 
+	ret = i915_gem_init_early(dev_priv);
+	if (ret < 0)
+		goto err_workqueues;
+
 	/* This must be called before any calls to HAS_PCH_* */
 	intel_detect_pch(dev_priv);
 
+	intel_wopcm_init_early(&dev_priv->wopcm);
+	intel_uc_init_early(dev_priv);
 	intel_pm_setup(dev_priv);
 	intel_init_dpio(dev_priv);
 	intel_power_domains_init(dev_priv);
@@ -940,18 +947,13 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 	intel_init_display_hooks(dev_priv);
 	intel_init_clock_gating_hooks(dev_priv);
 	intel_init_audio_hooks(dev_priv);
-	ret = i915_gem_load_init(dev_priv);
-	if (ret < 0)
-		goto err_irq;
-
 	intel_display_crc_init(dev_priv);
 
 	intel_detect_preproduction_hw(dev_priv);
 
 	return 0;
 
-err_irq:
-	intel_irq_fini(dev_priv);
+err_workqueues:
 	i915_workqueues_cleanup(dev_priv);
 err_engines:
 	i915_engines_cleanup(dev_priv);
@@ -964,8 +966,9 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
  */
 static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
 {
-	i915_gem_load_cleanup(dev_priv);
 	intel_irq_fini(dev_priv);
+	intel_uc_cleanup_early(dev_priv);
+	i915_gem_cleanup_early(dev_priv);
 	i915_workqueues_cleanup(dev_priv);
 	i915_engines_cleanup(dev_priv);
 }
@@ -1035,6 +1038,10 @@ static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
 
 	intel_uncore_init(dev_priv);
 
+	intel_device_info_init_mmio(dev_priv);
+
+	intel_uncore_prune(dev_priv);
+
 	intel_uc_init_mmio(dev_priv);
 
 	ret = intel_engines_init_mmio(dev_priv);
@@ -1077,8 +1084,6 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv)
 					    i915_modparams.enable_ppgtt);
 	DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915_modparams.enable_ppgtt);
 
-	intel_uc_sanitize_options(dev_priv);
-
 	intel_gvt_sanitize_options(dev_priv);
 }
 
@@ -1244,7 +1249,6 @@ static void i915_driver_register(struct drm_i915_private *dev_priv)
 	/* Reveal our presence to userspace */
 	if (drm_dev_register(dev, 0) == 0) {
 		i915_debugfs_register(dev_priv);
-		i915_guc_log_register(dev_priv);
 		i915_setup_sysfs(dev_priv);
 
 		/* Depends on sysfs having been initialized */
@@ -1304,7 +1308,6 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv)
 	i915_pmu_unregister(dev_priv);
 
 	i915_teardown_sysfs(dev_priv);
-	i915_guc_log_unregister(dev_priv);
 	drm_dev_unregister(&dev_priv->drm);
 
 	i915_gem_shrinker_unregister(dev_priv);
@@ -1463,7 +1466,6 @@ void i915_driver_unload(struct drm_device *dev)
 	i915_reset_error_state(dev_priv);
 
 	i915_gem_fini(dev_priv);
-	intel_uc_fini_fw(dev_priv);
 	intel_fbc_cleanup_cfb(dev_priv);
 
 	intel_power_domains_fini(dev_priv);
@@ -1876,7 +1878,8 @@ static int i915_resume_switcheroo(struct drm_device *dev)
 /**
  * i915_reset - reset chip after a hang
  * @i915: #drm_i915_private to reset
- * @flags: Instructions
+ * @stalled_mask: mask of the stalled engines with the guilty requests
+ * @reason: user error message for why we are resetting
  *
  * Reset the chip.  Useful if a hang is detected. Marks the device as wedged
  * on failure.
@@ -1891,12 +1894,16 @@ static int i915_resume_switcheroo(struct drm_device *dev)
  *   - re-init interrupt state
  *   - re-init display
  */
-void i915_reset(struct drm_i915_private *i915, unsigned int flags)
+void i915_reset(struct drm_i915_private *i915,
+		unsigned int stalled_mask,
+		const char *reason)
 {
 	struct i915_gpu_error *error = &i915->gpu_error;
 	int ret;
 	int i;
 
+	GEM_TRACE("flags=%lx\n", error->flags);
+
 	might_sleep();
 	lockdep_assert_held(&i915->drm.struct_mutex);
 	GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &error->flags));
@@ -1908,8 +1915,8 @@ void i915_reset(struct drm_i915_private *i915, unsigned int flags)
 	if (!i915_gem_unset_wedged(i915))
 		goto wakeup;
 
-	if (!(flags & I915_RESET_QUIET))
-		dev_notice(i915->drm.dev, "Resetting chip after gpu hang\n");
+	if (reason)
+		dev_notice(i915->drm.dev, "Resetting chip for %s\n", reason);
 	error->reset_count++;
 
 	disable_irq(i915->drm.irq);
@@ -1952,7 +1959,7 @@ void i915_reset(struct drm_i915_private *i915, unsigned int flags)
 		goto error;
 	}
 
-	i915_gem_reset(i915);
+	i915_gem_reset(i915, stalled_mask);
 	intel_overlay_reset(i915);
 
 	/*
@@ -1998,7 +2005,6 @@ void i915_reset(struct drm_i915_private *i915, unsigned int flags)
 error:
 	i915_gem_set_wedged(i915);
 	i915_retire_requests(i915);
-	intel_gpu_reset(i915, ALL_ENGINES);
 	goto finish;
 }
 
@@ -2011,7 +2017,7 @@ static inline int intel_gt_reset_engine(struct drm_i915_private *dev_priv,
 /**
  * i915_reset_engine - reset GPU engine to recover from a hang
  * @engine: engine to reset
- * @flags: options
+ * @msg: reason for GPU reset; or NULL for no dev_notice()
  *
  * Reset a specific GPU engine. Useful if a hang is detected.
  * Returns zero on successful reset or otherwise an error code.
@@ -2021,12 +2027,13 @@ static inline int intel_gt_reset_engine(struct drm_i915_private *dev_priv,
  *  - reset engine (which will force the engine to idle)
  *  - re-init/configure engine
  */
-int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags)
+int i915_reset_engine(struct intel_engine_cs *engine, const char *msg)
 {
 	struct i915_gpu_error *error = &engine->i915->gpu_error;
 	struct i915_request *active_request;
 	int ret;
 
+	GEM_TRACE("%s flags=%lx\n", engine->name, error->flags);
 	GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &error->flags));
 
 	active_request = i915_gem_reset_prepare_engine(engine);
@@ -2036,10 +2043,9 @@ int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags)
 		goto out;
 	}
 
-	if (!(flags & I915_RESET_QUIET)) {
+	if (msg)
 		dev_notice(engine->i915->drm.dev,
-			   "Resetting %s after gpu hang\n", engine->name);
-	}
+			   "Resetting %s for %s\n", engine->name, msg);
 	error->reset_engine_count[engine->id]++;
 
 	if (!engine->i915->guc.execbuf_client)
@@ -2059,7 +2065,7 @@ int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags)
 	 * active request and can drop it, adjust head to skip the offending
 	 * request to resume executing remaining requests in the queue.
 	 */
-	i915_gem_reset_engine(engine, active_request);
+	i915_gem_reset_engine(engine, active_request, true);
 
 	/*
 	 * The engine and its registers (and workarounds in case of render)
@@ -2468,10 +2474,13 @@ static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
 	/*
 	 * RC6 transitioning can be delayed up to 2 msec (see
 	 * valleyview_enable_rps), use 3 msec for safety.
+	 *
+	 * This can fail to turn off the rc6 if the GPU is stuck after a failed
+	 * reset and we are trying to force the machine to sleep.
 	 */
 	if (vlv_wait_for_pw_status(dev_priv, mask, val))
-		DRM_ERROR("timeout waiting for GT wells to go %s\n",
-			  onoff(wait_for_on));
+		DRM_DEBUG_DRIVER("timeout waiting for GT wells to go %s\n",
+				 onoff(wait_for_on));
 }
 
 static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
@@ -2822,10 +2831,10 @@ static const struct drm_ioctl_desc i915_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id_ioctl, 0),
 	DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
-	DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
-	DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
-	DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
-	DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW),
+	DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER),
+	DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER),
+	DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey_ioctl, DRM_MASTER),
+	DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER),
 	DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index ce18b6cf6e68..34c125e2d90c 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -64,6 +64,7 @@
 #include "intel_opregion.h"
 #include "intel_ringbuffer.h"
 #include "intel_uncore.h"
+#include "intel_wopcm.h"
 #include "intel_uc.h"
 
 #include "i915_gem.h"
@@ -71,9 +72,10 @@
 #include "i915_gem_fence_reg.h"
 #include "i915_gem_object.h"
 #include "i915_gem_gtt.h"
-#include "i915_gem_timeline.h"
-
+#include "i915_gpu_error.h"
 #include "i915_request.h"
+#include "i915_scheduler.h"
+#include "i915_timeline.h"
 #include "i915_vma.h"
 
 #include "intel_gvt.h"
@@ -83,8 +85,8 @@
 
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
-#define DRIVER_DATE		"20180308"
-#define DRIVER_TIMESTAMP	1520513379
+#define DRIVER_DATE		"20180514"
+#define DRIVER_TIMESTAMP	1526300884
 
 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
  * WARN_ON()) for hw state sanity checks to check for unexpected conditions
@@ -261,6 +263,7 @@ enum hpd_pin {
 	HPD_PORT_C,
 	HPD_PORT_D,
 	HPD_PORT_E,
+	HPD_PORT_F,
 	HPD_NUM_PINS
 };
 
@@ -453,172 +456,6 @@ struct intel_csr {
 	uint32_t allowed_dc_mask;
 };
 
-struct intel_display_error_state;
-
-struct i915_gpu_state {
-	struct kref ref;
-	ktime_t time;
-	ktime_t boottime;
-	ktime_t uptime;
-
-	struct drm_i915_private *i915;
-
-	char error_msg[128];
-	bool simulated;
-	bool awake;
-	bool wakelock;
-	bool suspended;
-	int iommu;
-	u32 reset_count;
-	u32 suspend_count;
-	struct intel_device_info device_info;
-	struct intel_driver_caps driver_caps;
-	struct i915_params params;
-
-	struct i915_error_uc {
-		struct intel_uc_fw guc_fw;
-		struct intel_uc_fw huc_fw;
-		struct drm_i915_error_object *guc_log;
-	} uc;
-
-	/* Generic register state */
-	u32 eir;
-	u32 pgtbl_er;
-	u32 ier;
-	u32 gtier[4], ngtier;
-	u32 ccid;
-	u32 derrmr;
-	u32 forcewake;
-	u32 error; /* gen6+ */
-	u32 err_int; /* gen7 */
-	u32 fault_data0; /* gen8, gen9 */
-	u32 fault_data1; /* gen8, gen9 */
-	u32 done_reg;
-	u32 gac_eco;
-	u32 gam_ecochk;
-	u32 gab_ctl;
-	u32 gfx_mode;
-
-	u32 nfence;
-	u64 fence[I915_MAX_NUM_FENCES];
-	struct intel_overlay_error_state *overlay;
-	struct intel_display_error_state *display;
-
-	struct drm_i915_error_engine {
-		int engine_id;
-		/* Software tracked state */
-		bool idle;
-		bool waiting;
-		int num_waiters;
-		unsigned long hangcheck_timestamp;
-		bool hangcheck_stalled;
-		enum intel_engine_hangcheck_action hangcheck_action;
-		struct i915_address_space *vm;
-		int num_requests;
-		u32 reset_count;
-
-		/* position of active request inside the ring */
-		u32 rq_head, rq_post, rq_tail;
-
-		/* our own tracking of ring head and tail */
-		u32 cpu_ring_head;
-		u32 cpu_ring_tail;
-
-		u32 last_seqno;
-
-		/* Register state */
-		u32 start;
-		u32 tail;
-		u32 head;
-		u32 ctl;
-		u32 mode;
-		u32 hws;
-		u32 ipeir;
-		u32 ipehr;
-		u32 bbstate;
-		u32 instpm;
-		u32 instps;
-		u32 seqno;
-		u64 bbaddr;
-		u64 acthd;
-		u32 fault_reg;
-		u64 faddr;
-		u32 rc_psmi; /* sleep state */
-		u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
-		struct intel_instdone instdone;
-
-		struct drm_i915_error_context {
-			char comm[TASK_COMM_LEN];
-			pid_t pid;
-			u32 handle;
-			u32 hw_id;
-			int priority;
-			int ban_score;
-			int active;
-			int guilty;
-			bool bannable;
-		} context;
-
-		struct drm_i915_error_object {
-			u64 gtt_offset;
-			u64 gtt_size;
-			int page_count;
-			int unused;
-			u32 *pages[0];
-		} *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
-
-		struct drm_i915_error_object **user_bo;
-		long user_bo_count;
-
-		struct drm_i915_error_object *wa_ctx;
-		struct drm_i915_error_object *default_state;
-
-		struct drm_i915_error_request {
-			long jiffies;
-			pid_t pid;
-			u32 context;
-			int priority;
-			int ban_score;
-			u32 seqno;
-			u32 head;
-			u32 tail;
-		} *requests, execlist[EXECLIST_MAX_PORTS];
-		unsigned int num_ports;
-
-		struct drm_i915_error_waiter {
-			char comm[TASK_COMM_LEN];
-			pid_t pid;
-			u32 seqno;
-		} *waiters;
-
-		struct {
-			u32 gfx_mode;
-			union {
-				u64 pdp[4];
-				u32 pp_dir_base;
-			};
-		} vm_info;
-	} engine[I915_NUM_ENGINES];
-
-	struct drm_i915_error_buffer {
-		u32 size;
-		u32 name;
-		u32 rseqno[I915_NUM_ENGINES], wseqno;
-		u64 gtt_offset;
-		u32 read_domains;
-		u32 write_domain;
-		s32 fence_reg:I915_MAX_NUM_FENCE_BITS;
-		u32 tiling:2;
-		u32 dirty:1;
-		u32 purgeable:1;
-		u32 userptr:1;
-		s32 engine:4;
-		u32 cache_level:3;
-	} *active_bo[I915_NUM_ENGINES], *pinned_bo;
-	u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count;
-	struct i915_address_space *active_vm[I915_NUM_ENGINES];
-};
-
 enum i915_cache_level {
 	I915_CACHE_NONE = 0,
 	I915_CACHE_LLC, /* also used for snoopable memory on non-LLC */
@@ -766,12 +603,16 @@ struct i915_psr {
 	bool active;
 	struct delayed_work work;
 	unsigned busy_frontbuffer_bits;
-	bool psr2_support;
-	bool aux_frame_sync;
+	bool sink_psr2_support;
 	bool link_standby;
-	bool y_cord_support;
 	bool colorimetry_support;
 	bool alpm;
+	bool has_hw_tracking;
+	bool psr2_enabled;
+	u8 sink_sync_latency;
+	bool debug;
+	ktime_t last_entry_attempt;
+	ktime_t last_exit;
 
 	void (*enable_source)(struct intel_dp *,
 			      const struct intel_crtc_state *);
@@ -1146,16 +987,6 @@ struct i915_gem_mm {
 	u32 object_count;
 };
 
-struct drm_i915_error_state_buf {
-	struct drm_i915_private *i915;
-	unsigned bytes;
-	unsigned size;
-	int err;
-	u8 *buf;
-	loff_t start;
-	loff_t pos;
-};
-
 #define I915_IDLE_ENGINES_TIMEOUT (200) /* in ms */
 
 #define I915_RESET_TIMEOUT (10 * HZ) /* 10s */
@@ -1164,102 +995,6 @@ struct drm_i915_error_state_buf {
 #define I915_ENGINE_DEAD_TIMEOUT  (4 * HZ)  /* Seqno, head and subunits dead */
 #define I915_SEQNO_DEAD_TIMEOUT   (12 * HZ) /* Seqno dead with active head */
 
-struct i915_gpu_error {
-	/* For hangcheck timer */
-#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
-#define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)
-
-	struct delayed_work hangcheck_work;
-
-	/* For reset and error_state handling. */
-	spinlock_t lock;
-	/* Protected by the above dev->gpu_error.lock. */
-	struct i915_gpu_state *first_error;
-
-	atomic_t pending_fb_pin;
-
-	unsigned long missed_irq_rings;
-
-	/**
-	 * State variable controlling the reset flow and count
-	 *
-	 * This is a counter which gets incremented when reset is triggered,
-	 *
-	 * Before the reset commences, the I915_RESET_BACKOFF bit is set
-	 * meaning that any waiters holding onto the struct_mutex should
-	 * relinquish the lock immediately in order for the reset to start.
-	 *
-	 * If reset is not completed succesfully, the I915_WEDGE bit is
-	 * set meaning that hardware is terminally sour and there is no
-	 * recovery. All waiters on the reset_queue will be woken when
-	 * that happens.
-	 *
-	 * This counter is used by the wait_seqno code to notice that reset
-	 * event happened and it needs to restart the entire ioctl (since most
-	 * likely the seqno it waited for won't ever signal anytime soon).
-	 *
-	 * This is important for lock-free wait paths, where no contended lock
-	 * naturally enforces the correct ordering between the bail-out of the
-	 * waiter and the gpu reset work code.
-	 */
-	unsigned long reset_count;
-
-	/**
-	 * flags: Control various stages of the GPU reset
-	 *
-	 * #I915_RESET_BACKOFF - When we start a reset, we want to stop any
-	 * other users acquiring the struct_mutex. To do this we set the
-	 * #I915_RESET_BACKOFF bit in the error flags when we detect a reset
-	 * and then check for that bit before acquiring the struct_mutex (in
-	 * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a
-	 * secondary role in preventing two concurrent global reset attempts.
-	 *
-	 * #I915_RESET_HANDOFF - To perform the actual GPU reset, we need the
-	 * struct_mutex. We try to acquire the struct_mutex in the reset worker,
-	 * but it may be held by some long running waiter (that we cannot
-	 * interrupt without causing trouble). Once we are ready to do the GPU
-	 * reset, we set the I915_RESET_HANDOFF bit and wakeup any waiters. If
-	 * they already hold the struct_mutex and want to participate they can
-	 * inspect the bit and do the reset directly, otherwise the worker
-	 * waits for the struct_mutex.
-	 *
-	 * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to
-	 * acquire the struct_mutex to reset an engine, we need an explicit
-	 * flag to prevent two concurrent reset attempts in the same engine.
-	 * As the number of engines continues to grow, allocate the flags from
-	 * the most significant bits.
-	 *
-	 * #I915_WEDGED - If reset fails and we can no longer use the GPU,
-	 * we set the #I915_WEDGED bit. Prior to command submission, e.g.
-	 * i915_request_alloc(), this bit is checked and the sequence
-	 * aborted (with -EIO reported to userspace) if set.
-	 */
-	unsigned long flags;
-#define I915_RESET_BACKOFF	0
-#define I915_RESET_HANDOFF	1
-#define I915_RESET_MODESET	2
-#define I915_WEDGED		(BITS_PER_LONG - 1)
-#define I915_RESET_ENGINE	(I915_WEDGED - I915_NUM_ENGINES)
-
-	/** Number of times an engine has been reset */
-	u32 reset_engine_count[I915_NUM_ENGINES];
-
-	/**
-	 * Waitqueue to signal when a hang is detected. Used to for waiters
-	 * to release the struct_mutex for the reset to procede.
-	 */
-	wait_queue_head_t wait_queue;
-
-	/**
-	 * Waitqueue to signal when the reset has completed. Used by clients
-	 * that wait for dev_priv->mm.wedged to settle.
-	 */
-	wait_queue_head_t reset_queue;
-
-	/* For missed irq/seqno simulation. */
-	unsigned long test_irq_rings;
-};
-
 enum modeset_restore {
 	MODESET_ON_LID_OPEN,
 	MODESET_DONE,
@@ -1338,6 +1073,7 @@ struct intel_vbt_data {
 	} edp;
 
 	struct {
+		bool enable;
 		bool full_link;
 		bool require_aux_wakeup;
 		int idle_frames;
@@ -1451,11 +1187,13 @@ static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1,
 }
 
 struct skl_ddb_allocation {
-	struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* packed/uv */
-	struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES];
+	/* packed/y */
+	struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES];
+	struct skl_ddb_entry uv_plane[I915_MAX_PIPES][I915_MAX_PLANES];
+	u8 enabled_slices; /* GEN11 has configurable 2 slices */
 };
 
-struct skl_wm_values {
+struct skl_ddb_values {
 	unsigned dirty_pipes;
 	struct skl_ddb_allocation ddb;
 };
@@ -1470,6 +1208,7 @@ struct skl_wm_level {
 struct skl_wm_params {
 	bool x_tiled, y_tiled;
 	bool rc_surface;
+	bool is_planar;
 	uint32_t width;
 	uint8_t cpp;
 	uint32_t plane_pixel_rate;
@@ -1564,7 +1303,6 @@ struct i915_wa_reg {
 struct i915_workarounds {
 	struct i915_wa_reg reg[I915_MAX_WA_REGS];
 	u32 count;
-	u32 hw_whitelist_count[I915_NUM_ENGINES];
 };
 
 struct i915_virtual_gpu {
@@ -1860,6 +1598,8 @@ struct drm_i915_private {
 
 	struct intel_gvt *gvt;
 
+	struct intel_wopcm wopcm;
+
 	struct intel_huc huc;
 	struct intel_guc guc;
 
@@ -2152,7 +1892,7 @@ struct drm_i915_private {
 		/* current hardware state */
 		union {
 			struct ilk_wm_values hw;
-			struct skl_wm_values skl_hw;
+			struct skl_ddb_values skl_hw;
 			struct vlv_wm_values vlv;
 			struct g4x_wm_values g4x;
 		};
@@ -2321,8 +2061,11 @@ struct drm_i915_private {
 		void (*cleanup_engine)(struct intel_engine_cs *engine);
 
 		struct list_head timelines;
-		struct i915_gem_timeline global_timeline;
+
+		struct list_head active_rings;
+		struct list_head closed_vma;
 		u32 active_requests;
+		u32 request_serial;
 
 		/**
 		 * Is the GPU currently considered idle, or busy executing
@@ -2392,6 +2135,11 @@ static inline struct drm_i915_private *kdev_to_i915(struct device *kdev)
 	return to_i915(dev_get_drvdata(kdev));
 }
 
+static inline struct drm_i915_private *wopcm_to_i915(struct intel_wopcm *wopcm)
+{
+	return container_of(wopcm, struct drm_i915_private, wopcm);
+}
+
 static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
 {
 	return container_of(guc, struct drm_i915_private, guc);
@@ -2411,8 +2159,10 @@ static inline struct drm_i915_private *huc_to_i915(struct intel_huc *huc)
 
 /* Iterator over subset of engines selected by mask */
 #define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \
-	for (tmp__ = mask__ & INTEL_INFO(dev_priv__)->ring_mask;	\
-	     tmp__ ? (engine__ = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; )
+	for ((tmp__) = (mask__) & INTEL_INFO(dev_priv__)->ring_mask; \
+	     (tmp__) ? \
+	     ((engine__) = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : \
+	     0;)
 
 enum hdmi_force_audio {
 	HDMI_AUDIO_OFF_DVI = -2,	/* no aux data for HDMI-DVI converter */
@@ -2720,6 +2470,15 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define IS_CNL_REVID(p, since, until) \
 	(IS_CANNONLAKE(p) && IS_REVID(p, since, until))
 
+#define ICL_REVID_A0		0x0
+#define ICL_REVID_A2		0x1
+#define ICL_REVID_B0		0x3
+#define ICL_REVID_B2		0x4
+#define ICL_REVID_C0		0x5
+
+#define IS_ICL_REVID(p, since, until) \
+	(IS_ICELAKE(p) && IS_REVID(p, since, until))
+
 /*
  * The genX designation typically refers to the render engine, so render
  * capability related checks should use IS_GEN, while display and other checks
@@ -2963,10 +2722,11 @@ extern void i915_driver_unload(struct drm_device *dev);
 extern int intel_gpu_reset(struct drm_i915_private *dev_priv, u32 engine_mask);
 extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv);
 
-#define I915_RESET_QUIET BIT(0)
-extern void i915_reset(struct drm_i915_private *i915, unsigned int flags);
+extern void i915_reset(struct drm_i915_private *i915,
+		       unsigned int stalled_mask,
+		       const char *reason);
 extern int i915_reset_engine(struct intel_engine_cs *engine,
-			     unsigned int flags);
+			     const char *reason);
 
 extern bool intel_has_reset_engine(struct drm_i915_private *dev_priv);
 extern int intel_reset_guc(struct drm_i915_private *dev_priv);
@@ -3014,10 +2774,12 @@ static inline void i915_queue_hangcheck(struct drm_i915_private *dev_priv)
 			   &dev_priv->gpu_error.hangcheck_work, delay);
 }
 
-__printf(3, 4)
+__printf(4, 5)
 void i915_handle_error(struct drm_i915_private *dev_priv,
 		       u32 engine_mask,
+		       unsigned long flags,
 		       const char *fmt, ...);
+#define I915_ERROR_CAPTURE BIT(0)
 
 extern void intel_irq_init(struct drm_i915_private *dev_priv);
 extern void intel_irq_fini(struct drm_i915_private *dev_priv);
@@ -3132,8 +2894,8 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 void i915_gem_sanitize(struct drm_i915_private *i915);
-int i915_gem_load_init(struct drm_i915_private *dev_priv);
-void i915_gem_load_cleanup(struct drm_i915_private *dev_priv);
+int i915_gem_init_early(struct drm_i915_private *dev_priv);
+void i915_gem_cleanup_early(struct drm_i915_private *dev_priv);
 void i915_gem_load_init_fences(struct drm_i915_private *dev_priv);
 int i915_gem_freeze(struct drm_i915_private *dev_priv);
 int i915_gem_freeze_late(struct drm_i915_private *dev_priv);
@@ -3388,13 +3150,15 @@ static inline u32 i915_reset_engine_count(struct i915_gpu_error *error,
 struct i915_request *
 i915_gem_reset_prepare_engine(struct intel_engine_cs *engine);
 int i915_gem_reset_prepare(struct drm_i915_private *dev_priv);
-void i915_gem_reset(struct drm_i915_private *dev_priv);
+void i915_gem_reset(struct drm_i915_private *dev_priv,
+		    unsigned int stalled_mask);
 void i915_gem_reset_finish_engine(struct intel_engine_cs *engine);
 void i915_gem_reset_finish(struct drm_i915_private *dev_priv);
 void i915_gem_set_wedged(struct drm_i915_private *dev_priv);
 bool i915_gem_unset_wedged(struct drm_i915_private *dev_priv);
 void i915_gem_reset_engine(struct intel_engine_cs *engine,
-			   struct i915_request *request);
+			   struct i915_request *request,
+			   bool stalled);
 
 void i915_gem_init_mmio(struct drm_i915_private *i915);
 int __must_check i915_gem_init(struct drm_i915_private *dev_priv);
@@ -3412,7 +3176,7 @@ int i915_gem_object_wait(struct drm_i915_gem_object *obj,
 			 struct intel_rps_client *rps);
 int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 				  unsigned int flags,
-				  int priority);
+				  const struct i915_sched_attr *attr);
 #define I915_PRIORITY_DISPLAY I915_PRIORITY_MAX
 
 int __must_check
@@ -3481,16 +3245,6 @@ i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id)
 	return ctx;
 }
 
-static inline struct intel_timeline *
-i915_gem_context_lookup_timeline(struct i915_gem_context *ctx,
-				 struct intel_engine_cs *engine)
-{
-	struct i915_address_space *vm;
-
-	vm = ctx->ppgtt ? &ctx->ppgtt->base : &ctx->i915->ggtt.base;
-	return &vm->timeline.engine[engine->id];
-}
-
 int i915_perf_open_ioctl(struct drm_device *dev, void *data,
 			 struct drm_file *file);
 int i915_perf_add_config_ioctl(struct drm_device *dev, void *data,
@@ -3589,64 +3343,6 @@ static inline int i915_debugfs_connector_add(struct drm_connector *connector)
 static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {}
 #endif
 
-/* i915_gpu_error.c */
-#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
-
-__printf(2, 3)
-void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
-int i915_error_state_to_str(struct drm_i915_error_state_buf *estr,
-			    const struct i915_gpu_state *gpu);
-int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb,
-			      struct drm_i915_private *i915,
-			      size_t count, loff_t pos);
-static inline void i915_error_state_buf_release(
-	struct drm_i915_error_state_buf *eb)
-{
-	kfree(eb->buf);
-}
-
-struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915);
-void i915_capture_error_state(struct drm_i915_private *dev_priv,
-			      u32 engine_mask,
-			      const char *error_msg);
-
-static inline struct i915_gpu_state *
-i915_gpu_state_get(struct i915_gpu_state *gpu)
-{
-	kref_get(&gpu->ref);
-	return gpu;
-}
-
-void __i915_gpu_state_free(struct kref *kref);
-static inline void i915_gpu_state_put(struct i915_gpu_state *gpu)
-{
-	if (gpu)
-		kref_put(&gpu->ref, __i915_gpu_state_free);
-}
-
-struct i915_gpu_state *i915_first_error_state(struct drm_i915_private *i915);
-void i915_reset_error_state(struct drm_i915_private *i915);
-
-#else
-
-static inline void i915_capture_error_state(struct drm_i915_private *dev_priv,
-					    u32 engine_mask,
-					    const char *error_msg)
-{
-}
-
-static inline struct i915_gpu_state *
-i915_first_error_state(struct drm_i915_private *i915)
-{
-	return NULL;
-}
-
-static inline void i915_reset_error_state(struct drm_i915_private *i915)
-{
-}
-
-#endif
-
 const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
 
 /* i915_cmd_parser.c */
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 7b5a9d7c9593..0a2070112b66 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -35,6 +35,7 @@
 #include "intel_drv.h"
 #include "intel_frontbuffer.h"
 #include "intel_mocs.h"
+#include "intel_workarounds.h"
 #include "i915_gemfs.h"
 #include <linux/dma-fence-array.h>
 #include <linux/kthread.h>
@@ -136,6 +137,102 @@ int i915_mutex_lock_interruptible(struct drm_device *dev)
 	return 0;
 }
 
+static u32 __i915_gem_park(struct drm_i915_private *i915)
+{
+	lockdep_assert_held(&i915->drm.struct_mutex);
+	GEM_BUG_ON(i915->gt.active_requests);
+	GEM_BUG_ON(!list_empty(&i915->gt.active_rings));
+
+	if (!i915->gt.awake)
+		return I915_EPOCH_INVALID;
+
+	GEM_BUG_ON(i915->gt.epoch == I915_EPOCH_INVALID);
+
+	/*
+	 * Be paranoid and flush a concurrent interrupt to make sure
+	 * we don't reactivate any irq tasklets after parking.
+	 *
+	 * FIXME: Note that even though we have waited for execlists to be idle,
+	 * there may still be an in-flight interrupt even though the CSB
+	 * is now empty. synchronize_irq() makes sure that a residual interrupt
+	 * is completed before we continue, but it doesn't prevent the HW from
+	 * raising a spurious interrupt later. To complete the shield we should
+	 * coordinate disabling the CS irq with flushing the interrupts.
+	 */
+	synchronize_irq(i915->drm.irq);
+
+	intel_engines_park(i915);
+	i915_timelines_park(i915);
+
+	i915_pmu_gt_parked(i915);
+	i915_vma_parked(i915);
+
+	i915->gt.awake = false;
+
+	if (INTEL_GEN(i915) >= 6)
+		gen6_rps_idle(i915);
+
+	intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ);
+
+	intel_runtime_pm_put(i915);
+
+	return i915->gt.epoch;
+}
+
+void i915_gem_park(struct drm_i915_private *i915)
+{
+	lockdep_assert_held(&i915->drm.struct_mutex);
+	GEM_BUG_ON(i915->gt.active_requests);
+
+	if (!i915->gt.awake)
+		return;
+
+	/* Defer the actual call to __i915_gem_park() to prevent ping-pongs */
+	mod_delayed_work(i915->wq, &i915->gt.idle_work, msecs_to_jiffies(100));
+}
+
+void i915_gem_unpark(struct drm_i915_private *i915)
+{
+	lockdep_assert_held(&i915->drm.struct_mutex);
+	GEM_BUG_ON(!i915->gt.active_requests);
+
+	if (i915->gt.awake)
+		return;
+
+	intel_runtime_pm_get_noresume(i915);
+
+	/*
+	 * It seems that the DMC likes to transition between the DC states a lot
+	 * when there are no connected displays (no active power domains) during
+	 * command submission.
+	 *
+	 * This activity has negative impact on the performance of the chip with
+	 * huge latencies observed in the interrupt handler and elsewhere.
+	 *
+	 * Work around it by grabbing a GT IRQ power domain whilst there is any
+	 * GT activity, preventing any DC state transitions.
+	 */
+	intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
+
+	i915->gt.awake = true;
+	if (unlikely(++i915->gt.epoch == 0)) /* keep 0 as invalid */
+		i915->gt.epoch = 1;
+
+	intel_enable_gt_powersave(i915);
+	i915_update_gfx_val(i915);
+	if (INTEL_GEN(i915) >= 6)
+		gen6_rps_busy(i915);
+	i915_pmu_gt_unparked(i915);
+
+	intel_engines_unpark(i915);
+
+	i915_queue_hangcheck(i915);
+
+	queue_delayed_work(i915->wq,
+			   &i915->gt.retire_work,
+			   round_jiffies_up_relative(HZ));
+}
+
 int
 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file)
@@ -469,7 +566,8 @@ i915_gem_object_wait_reservation(struct reservation_object *resv,
 	return timeout;
 }
 
-static void __fence_set_priority(struct dma_fence *fence, int prio)
+static void __fence_set_priority(struct dma_fence *fence,
+				 const struct i915_sched_attr *attr)
 {
 	struct i915_request *rq;
 	struct intel_engine_cs *engine;
@@ -480,13 +578,16 @@ static void __fence_set_priority(struct dma_fence *fence, int prio)
 	rq = to_request(fence);
 	engine = rq->engine;
 
-	rcu_read_lock();
+	local_bh_disable();
+	rcu_read_lock(); /* RCU serialisation for set-wedged protection */
 	if (engine->schedule)
-		engine->schedule(rq, prio);
+		engine->schedule(rq, attr);
 	rcu_read_unlock();
+	local_bh_enable(); /* kick the tasklets if queues were reprioritised */
 }
 
-static void fence_set_priority(struct dma_fence *fence, int prio)
+static void fence_set_priority(struct dma_fence *fence,
+			       const struct i915_sched_attr *attr)
 {
 	/* Recurse once into a fence-array */
 	if (dma_fence_is_array(fence)) {
@@ -494,16 +595,16 @@ static void fence_set_priority(struct dma_fence *fence, int prio)
 		int i;
 
 		for (i = 0; i < array->num_fences; i++)
-			__fence_set_priority(array->fences[i], prio);
+			__fence_set_priority(array->fences[i], attr);
 	} else {
-		__fence_set_priority(fence, prio);
+		__fence_set_priority(fence, attr);
 	}
 }
 
 int
 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 			      unsigned int flags,
-			      int prio)
+			      const struct i915_sched_attr *attr)
 {
 	struct dma_fence *excl;
 
@@ -518,7 +619,7 @@ i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 			return ret;
 
 		for (i = 0; i < count; i++) {
-			fence_set_priority(shared[i], prio);
+			fence_set_priority(shared[i], attr);
 			dma_fence_put(shared[i]);
 		}
 
@@ -528,7 +629,7 @@ i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 	}
 
 	if (excl) {
-		fence_set_priority(excl, prio);
+		fence_set_priority(excl, attr);
 		dma_fence_put(excl);
 	}
 	return 0;
@@ -2879,8 +2980,8 @@ i915_gem_find_active_request(struct intel_engine_cs *engine)
 	 * extra delay for a recent interrupt is pointless. Hence, we do
 	 * not need an engine->irq_seqno_barrier() before the seqno reads.
 	 */
-	spin_lock_irqsave(&engine->timeline->lock, flags);
-	list_for_each_entry(request, &engine->timeline->requests, link) {
+	spin_lock_irqsave(&engine->timeline.lock, flags);
+	list_for_each_entry(request, &engine->timeline.requests, link) {
 		if (__i915_request_completed(request, request->global_seqno))
 			continue;
 
@@ -2891,25 +2992,11 @@ i915_gem_find_active_request(struct intel_engine_cs *engine)
 		active = request;
 		break;
 	}
-	spin_unlock_irqrestore(&engine->timeline->lock, flags);
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 
 	return active;
 }
 
-static bool engine_stalled(struct intel_engine_cs *engine)
-{
-	if (!engine->hangcheck.stalled)
-		return false;
-
-	/* Check for possible seqno movement after hang declaration */
-	if (engine->hangcheck.seqno != intel_engine_get_seqno(engine)) {
-		DRM_DEBUG_DRIVER("%s pardoned\n", engine->name);
-		return false;
-	}
-
-	return true;
-}
-
 /*
  * Ensure irq handler finishes, and not run again.
  * Also return the active request so that we only search for it once.
@@ -2998,6 +3085,7 @@ int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
 	}
 
 	i915_gem_revoke_fences(dev_priv);
+	intel_uc_sanitize(dev_priv);
 
 	return err;
 }
@@ -3025,15 +3113,15 @@ static void engine_skip_context(struct i915_request *request)
 {
 	struct intel_engine_cs *engine = request->engine;
 	struct i915_gem_context *hung_ctx = request->ctx;
-	struct intel_timeline *timeline;
+	struct i915_timeline *timeline = request->timeline;
 	unsigned long flags;
 
-	timeline = i915_gem_context_lookup_timeline(hung_ctx, engine);
+	GEM_BUG_ON(timeline == &engine->timeline);
 
-	spin_lock_irqsave(&engine->timeline->lock, flags);
-	spin_lock(&timeline->lock);
+	spin_lock_irqsave(&engine->timeline.lock, flags);
+	spin_lock_nested(&timeline->lock, SINGLE_DEPTH_NESTING);
 
-	list_for_each_entry_continue(request, &engine->timeline->requests, link)
+	list_for_each_entry_continue(request, &engine->timeline.requests, link)
 		if (request->ctx == hung_ctx)
 			skip_request(request);
 
@@ -3041,13 +3129,14 @@ static void engine_skip_context(struct i915_request *request)
 		skip_request(request);
 
 	spin_unlock(&timeline->lock);
-	spin_unlock_irqrestore(&engine->timeline->lock, flags);
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
 /* Returns the request if it was guilty of the hang */
 static struct i915_request *
 i915_gem_reset_request(struct intel_engine_cs *engine,
-		       struct i915_request *request)
+		       struct i915_request *request,
+		       bool stalled)
 {
 	/* The guilty request will get skipped on a hung engine.
 	 *
@@ -3070,7 +3159,15 @@ i915_gem_reset_request(struct intel_engine_cs *engine,
 	 * subsequent hangs.
 	 */
 
-	if (engine_stalled(engine)) {
+	if (i915_request_completed(request)) {
+		GEM_TRACE("%s pardoned global=%d (fence %llx:%d), current %d\n",
+			  engine->name, request->global_seqno,
+			  request->fence.context, request->fence.seqno,
+			  intel_engine_get_seqno(engine));
+		stalled = false;
+	}
+
+	if (stalled) {
 		i915_gem_context_mark_guilty(request->ctx);
 		skip_request(request);
 
@@ -3089,11 +3186,11 @@ i915_gem_reset_request(struct intel_engine_cs *engine,
 			dma_fence_set_error(&request->fence, -EAGAIN);
 
 			/* Rewind the engine to replay the incomplete rq */
-			spin_lock_irq(&engine->timeline->lock);
+			spin_lock_irq(&engine->timeline.lock);
 			request = list_prev_entry(request, link);
-			if (&request->link == &engine->timeline->requests)
+			if (&request->link == &engine->timeline.requests)
 				request = NULL;
-			spin_unlock_irq(&engine->timeline->lock);
+			spin_unlock_irq(&engine->timeline.lock);
 		}
 	}
 
@@ -3101,7 +3198,8 @@ i915_gem_reset_request(struct intel_engine_cs *engine,
 }
 
 void i915_gem_reset_engine(struct intel_engine_cs *engine,
-			   struct i915_request *request)
+			   struct i915_request *request,
+			   bool stalled)
 {
 	/*
 	 * Make sure this write is visible before we re-enable the interrupt
@@ -3111,7 +3209,7 @@ void i915_gem_reset_engine(struct intel_engine_cs *engine,
 	smp_store_mb(engine->irq_posted, 0);
 
 	if (request)
-		request = i915_gem_reset_request(engine, request);
+		request = i915_gem_reset_request(engine, request, stalled);
 
 	if (request) {
 		DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n",
@@ -3122,7 +3220,8 @@ void i915_gem_reset_engine(struct intel_engine_cs *engine,
 	engine->reset_hw(engine, request);
 }
 
-void i915_gem_reset(struct drm_i915_private *dev_priv)
+void i915_gem_reset(struct drm_i915_private *dev_priv,
+		    unsigned int stalled_mask)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
@@ -3134,10 +3233,12 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
 	for_each_engine(engine, dev_priv, id) {
 		struct i915_gem_context *ctx;
 
-		i915_gem_reset_engine(engine, engine->hangcheck.active_request);
+		i915_gem_reset_engine(engine,
+				      engine->hangcheck.active_request,
+				      stalled_mask & ENGINE_MASK(id));
 		ctx = fetch_and_zero(&engine->last_retired_context);
 		if (ctx)
-			engine->context_unpin(engine, ctx);
+			intel_context_unpin(ctx, engine);
 
 		/*
 		 * Ostensibily, we always want a context loaded for powersaving,
@@ -3160,13 +3261,6 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
 	}
 
 	i915_gem_restore_fences(dev_priv);
-
-	if (dev_priv->gt.awake) {
-		intel_sanitize_gt_powersave(dev_priv);
-		intel_enable_gt_powersave(dev_priv);
-		if (INTEL_GEN(dev_priv) >= 6)
-			gen6_rps_busy(dev_priv);
-	}
 }
 
 void i915_gem_reset_finish_engine(struct intel_engine_cs *engine)
@@ -3192,6 +3286,9 @@ void i915_gem_reset_finish(struct drm_i915_private *dev_priv)
 
 static void nop_submit_request(struct i915_request *request)
 {
+	GEM_TRACE("%s fence %llx:%d -> -EIO\n",
+		  request->engine->name,
+		  request->fence.context, request->fence.seqno);
 	dma_fence_set_error(&request->fence, -EIO);
 
 	i915_request_submit(request);
@@ -3201,12 +3298,15 @@ static void nop_complete_submit_request(struct i915_request *request)
 {
 	unsigned long flags;
 
+	GEM_TRACE("%s fence %llx:%d -> -EIO\n",
+		  request->engine->name,
+		  request->fence.context, request->fence.seqno);
 	dma_fence_set_error(&request->fence, -EIO);
 
-	spin_lock_irqsave(&request->engine->timeline->lock, flags);
+	spin_lock_irqsave(&request->engine->timeline.lock, flags);
 	__i915_request_submit(request);
 	intel_engine_init_global_seqno(request->engine, request->global_seqno);
-	spin_unlock_irqrestore(&request->engine->timeline->lock, flags);
+	spin_unlock_irqrestore(&request->engine->timeline.lock, flags);
 }
 
 void i915_gem_set_wedged(struct drm_i915_private *i915)
@@ -3214,7 +3314,9 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 
-	if (drm_debug & DRM_UT_DRIVER) {
+	GEM_TRACE("start\n");
+
+	if (GEM_SHOW_DEBUG()) {
 		struct drm_printer p = drm_debug_printer(__func__);
 
 		for_each_engine(engine, i915, id)
@@ -3237,6 +3339,9 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
 	}
 	i915->caps.scheduler = 0;
 
+	/* Even if the GPU reset fails, it should still stop the engines */
+	intel_gpu_reset(i915, ALL_ENGINES);
+
 	/*
 	 * Make sure no one is running the old callback before we proceed with
 	 * cancelling requests and resetting the completion tracking. Otherwise
@@ -3270,27 +3375,31 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
 		 * (lockless) lookup doesn't try and wait upon the request as we
 		 * reset it.
 		 */
-		spin_lock_irqsave(&engine->timeline->lock, flags);
+		spin_lock_irqsave(&engine->timeline.lock, flags);
 		intel_engine_init_global_seqno(engine,
 					       intel_engine_last_submit(engine));
-		spin_unlock_irqrestore(&engine->timeline->lock, flags);
+		spin_unlock_irqrestore(&engine->timeline.lock, flags);
 
 		i915_gem_reset_finish_engine(engine);
 	}
 
+	GEM_TRACE("end\n");
+
 	wake_up_all(&i915->gpu_error.reset_queue);
 }
 
 bool i915_gem_unset_wedged(struct drm_i915_private *i915)
 {
-	struct i915_gem_timeline *tl;
-	int i;
+	struct i915_timeline *tl;
 
 	lockdep_assert_held(&i915->drm.struct_mutex);
 	if (!test_bit(I915_WEDGED, &i915->gpu_error.flags))
 		return true;
 
-	/* Before unwedging, make sure that all pending operations
+	GEM_TRACE("start\n");
+
+	/*
+	 * Before unwedging, make sure that all pending operations
 	 * are flushed and errored out - we may have requests waiting upon
 	 * third party fences. We marked all inflight requests as EIO, and
 	 * every execbuf since returned EIO, for consistency we want all
@@ -3300,31 +3409,33 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
 	 * No more can be submitted until we reset the wedged bit.
 	 */
 	list_for_each_entry(tl, &i915->gt.timelines, link) {
-		for (i = 0; i < ARRAY_SIZE(tl->engine); i++) {
-			struct i915_request *rq;
+		struct i915_request *rq;
 
-			rq = i915_gem_active_peek(&tl->engine[i].last_request,
-						  &i915->drm.struct_mutex);
-			if (!rq)
-				continue;
+		rq = i915_gem_active_peek(&tl->last_request,
+					  &i915->drm.struct_mutex);
+		if (!rq)
+			continue;
 
-			/* We can't use our normal waiter as we want to
-			 * avoid recursively trying to handle the current
-			 * reset. The basic dma_fence_default_wait() installs
-			 * a callback for dma_fence_signal(), which is
-			 * triggered by our nop handler (indirectly, the
-			 * callback enables the signaler thread which is
-			 * woken by the nop_submit_request() advancing the seqno
-			 * and when the seqno passes the fence, the signaler
-			 * then signals the fence waking us up).
-			 */
-			if (dma_fence_default_wait(&rq->fence, true,
-						   MAX_SCHEDULE_TIMEOUT) < 0)
-				return false;
-		}
+		/*
+		 * We can't use our normal waiter as we want to
+		 * avoid recursively trying to handle the current
+		 * reset. The basic dma_fence_default_wait() installs
+		 * a callback for dma_fence_signal(), which is
+		 * triggered by our nop handler (indirectly, the
+		 * callback enables the signaler thread which is
+		 * woken by the nop_submit_request() advancing the seqno
+		 * and when the seqno passes the fence, the signaler
+		 * then signals the fence waking us up).
+		 */
+		if (dma_fence_default_wait(&rq->fence, true,
+					   MAX_SCHEDULE_TIMEOUT) < 0)
+			return false;
 	}
+	i915_retire_requests(i915);
+	GEM_BUG_ON(i915->gt.active_requests);
 
-	/* Undo nop_submit_request. We prevent all new i915 requests from
+	/*
+	 * Undo nop_submit_request. We prevent all new i915 requests from
 	 * being queued (by disallowing execbuf whilst wedged) so having
 	 * waited for all active requests above, we know the system is idle
 	 * and do not have to worry about a thread being inside
@@ -3335,6 +3446,8 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
 	intel_engines_reset_default_submission(i915);
 	i915_gem_contexts_lost(i915);
 
+	GEM_TRACE("end\n");
+
 	smp_mb__before_atomic(); /* complete takeover before enabling execbuf */
 	clear_bit(I915_WEDGED, &i915->gpu_error.flags);
 
@@ -3473,36 +3586,9 @@ i915_gem_idle_work_handler(struct work_struct *work)
 	if (new_requests_since_last_retire(dev_priv))
 		goto out_unlock;
 
-	/*
-	 * Be paranoid and flush a concurrent interrupt to make sure
-	 * we don't reactivate any irq tasklets after parking.
-	 *
-	 * FIXME: Note that even though we have waited for execlists to be idle,
-	 * there may still be an in-flight interrupt even though the CSB
-	 * is now empty. synchronize_irq() makes sure that a residual interrupt
-	 * is completed before we continue, but it doesn't prevent the HW from
-	 * raising a spurious interrupt later. To complete the shield we should
-	 * coordinate disabling the CS irq with flushing the interrupts.
-	 */
-	synchronize_irq(dev_priv->drm.irq);
-
-	intel_engines_park(dev_priv);
-	i915_gem_timelines_park(dev_priv);
+	epoch = __i915_gem_park(dev_priv);
 
-	i915_pmu_gt_parked(dev_priv);
-
-	GEM_BUG_ON(!dev_priv->gt.awake);
-	dev_priv->gt.awake = false;
-	epoch = dev_priv->gt.epoch;
-	GEM_BUG_ON(epoch == I915_EPOCH_INVALID);
 	rearm_hangcheck = false;
-
-	if (INTEL_GEN(dev_priv) >= 6)
-		gen6_rps_idle(dev_priv);
-
-	intel_display_power_put(dev_priv, POWER_DOMAIN_GT_IRQ);
-
-	intel_runtime_pm_put(dev_priv);
 out_unlock:
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
@@ -3648,17 +3734,9 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 	return ret;
 }
 
-static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags)
+static int wait_for_timeline(struct i915_timeline *tl, unsigned int flags)
 {
-	int ret, i;
-
-	for (i = 0; i < ARRAY_SIZE(tl->engine); i++) {
-		ret = i915_gem_active_wait(&tl->engine[i].last_request, flags);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
+	return i915_gem_active_wait(&tl->last_request, flags);
 }
 
 static int wait_for_engines(struct drm_i915_private *i915)
@@ -3666,16 +3744,7 @@ static int wait_for_engines(struct drm_i915_private *i915)
 	if (wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) {
 		dev_err(i915->drm.dev,
 			"Failed to idle engines, declaring wedged!\n");
-		if (drm_debug & DRM_UT_DRIVER) {
-			struct drm_printer p = drm_debug_printer(__func__);
-			struct intel_engine_cs *engine;
-			enum intel_engine_id id;
-
-			for_each_engine(engine, i915, id)
-				intel_engine_dump(engine, &p,
-						  "%s\n", engine->name);
-		}
-
+		GEM_TRACE_DUMP();
 		i915_gem_set_wedged(i915);
 		return -EIO;
 	}
@@ -3685,30 +3754,37 @@ static int wait_for_engines(struct drm_i915_private *i915)
 
 int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
 {
-	int ret;
-
 	/* If the device is asleep, we have no requests outstanding */
 	if (!READ_ONCE(i915->gt.awake))
 		return 0;
 
 	if (flags & I915_WAIT_LOCKED) {
-		struct i915_gem_timeline *tl;
+		struct i915_timeline *tl;
+		int err;
 
 		lockdep_assert_held(&i915->drm.struct_mutex);
 
 		list_for_each_entry(tl, &i915->gt.timelines, link) {
-			ret = wait_for_timeline(tl, flags);
-			if (ret)
-				return ret;
+			err = wait_for_timeline(tl, flags);
+			if (err)
+				return err;
 		}
 		i915_retire_requests(i915);
 
-		ret = wait_for_engines(i915);
+		return wait_for_engines(i915);
 	} else {
-		ret = wait_for_timeline(&i915->gt.global_timeline, flags);
-	}
+		struct intel_engine_cs *engine;
+		enum intel_engine_id id;
+		int err;
 
-	return ret;
+		for_each_engine(engine, i915, id) {
+			err = wait_for_timeline(&engine->timeline, flags);
+			if (err)
+				return err;
+		}
+
+		return 0;
+	}
 }
 
 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
@@ -4088,9 +4164,10 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 }
 
 /*
- * Prepare buffer for display plane (scanout, cursors, etc).
- * Can be called from an uninterruptible phase (modesetting) and allows
- * any flushes to be pipelined (for pageflips).
+ * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
+ * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
+ * (for pageflips). We only flush the caches while preparing the buffer for
+ * display, the callers are responsible for frontbuffer flush.
  */
 struct i915_vma *
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
@@ -4146,9 +4223,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 
 	vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
 
-	/* Treat this as an end-of-frame, like intel_user_framebuffer_dirty() */
 	__i915_gem_object_flush_for_display(obj);
-	intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
 
 	/* It should now be out of any other write domains, and we can update
 	 * the domain values for our changes.
@@ -4723,7 +4798,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
 					 &obj->vma_list, obj_link) {
 			GEM_BUG_ON(i915_vma_is_active(vma));
 			vma->flags &= ~I915_VMA_PIN_MASK;
-			i915_vma_close(vma);
+			i915_vma_destroy(vma);
 		}
 		GEM_BUG_ON(!list_empty(&obj->vma_list));
 		GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree));
@@ -4878,7 +4953,7 @@ static void assert_kernel_context_is_current(struct drm_i915_private *i915)
 	enum intel_engine_id id;
 
 	for_each_engine(engine, i915, id) {
-		GEM_BUG_ON(__i915_gem_active_peek(&engine->timeline->last_request));
+		GEM_BUG_ON(__i915_gem_active_peek(&engine->timeline.last_request));
 		GEM_BUG_ON(engine->last_retired_context != kernel_context);
 	}
 }
@@ -4973,6 +5048,7 @@ int i915_gem_suspend(struct drm_i915_private *dev_priv)
 	 * machines is a good idea, we don't - just in case it leaves the
 	 * machine in an unusable condition.
 	 */
+	intel_uc_sanitize(dev_priv);
 	i915_gem_sanitize(dev_priv);
 
 	intel_runtime_pm_put(dev_priv);
@@ -5118,6 +5194,8 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv)
 		}
 	}
 
+	intel_gt_workarounds_apply(dev_priv);
+
 	i915_gem_init_swizzling(dev_priv);
 
 	/*
@@ -5140,6 +5218,12 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv)
 		goto out;
 	}
 
+	ret = intel_wopcm_init_hw(&dev_priv->wopcm);
+	if (ret) {
+		DRM_ERROR("Enabling WOPCM failed (%d)\n", ret);
+		goto out;
+	}
+
 	/* We can't enable contexts until all firmware is loaded */
 	ret = intel_uc_init_hw(dev_priv);
 	if (ret) {
@@ -5207,7 +5291,7 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
 	for_each_engine(engine, i915, id) {
 		struct i915_vma *state;
 
-		state = ctx->engine[id].state;
+		state = to_intel_context(ctx, engine)->state;
 		if (!state)
 			continue;
 
@@ -5297,6 +5381,10 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 	if (ret)
 		return ret;
 
+	ret = intel_wopcm_init(&dev_priv->wopcm);
+	if (ret)
+		return ret;
+
 	ret = intel_uc_init_misc(dev_priv);
 	if (ret)
 		return ret;
@@ -5478,8 +5566,7 @@ static void i915_gem_init__mm(struct drm_i915_private *i915)
 	INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);
 }
 
-int
-i915_gem_load_init(struct drm_i915_private *dev_priv)
+int i915_gem_init_early(struct drm_i915_private *dev_priv)
 {
 	int err = -ENOMEM;
 
@@ -5512,12 +5599,9 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
 	if (!dev_priv->priorities)
 		goto err_dependencies;
 
-	mutex_lock(&dev_priv->drm.struct_mutex);
 	INIT_LIST_HEAD(&dev_priv->gt.timelines);
-	err = i915_gem_timeline_init__global(dev_priv);
-	mutex_unlock(&dev_priv->drm.struct_mutex);
-	if (err)
-		goto err_priorities;
+	INIT_LIST_HEAD(&dev_priv->gt.active_rings);
+	INIT_LIST_HEAD(&dev_priv->gt.closed_vma);
 
 	i915_gem_init__mm(dev_priv);
 
@@ -5538,8 +5622,6 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
 
 	return 0;
 
-err_priorities:
-	kmem_cache_destroy(dev_priv->priorities);
 err_dependencies:
 	kmem_cache_destroy(dev_priv->dependencies);
 err_requests:
@@ -5554,17 +5636,13 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
 	return err;
 }
 
-void i915_gem_load_cleanup(struct drm_i915_private *dev_priv)
+void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
 {
 	i915_gem_drain_freed_objects(dev_priv);
 	GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
 	GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
 	WARN_ON(dev_priv->mm.object_count);
-
-	mutex_lock(&dev_priv->drm.struct_mutex);
-	i915_gem_timeline_fini(&dev_priv->gt.global_timeline);
 	WARN_ON(!list_empty(&dev_priv->gt.timelines));
-	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	kmem_cache_destroy(dev_priv->priorities);
 	kmem_cache_destroy(dev_priv->dependencies);
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index f54c4ff74ded..525920404ede 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -27,7 +27,12 @@
 
 #include <linux/bug.h>
 
+struct drm_i915_private;
+
 #ifdef CONFIG_DRM_I915_DEBUG_GEM
+
+#define GEM_SHOW_DEBUG() (drm_debug & DRM_UT_DRIVER)
+
 #define GEM_BUG_ON(condition) do { if (unlikely((condition))) {	\
 		pr_err("%s:%d GEM_BUG_ON(%s)\n", \
 		       __func__, __LINE__, __stringify(condition)); \
@@ -43,6 +48,9 @@
 #define GEM_DEBUG_BUG_ON(expr) GEM_BUG_ON(expr)
 
 #else
+
+#define GEM_SHOW_DEBUG() (0)
+
 #define GEM_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr)
 #define GEM_WARN_ON(expr) (BUILD_BUG_ON_INVALID(expr), 0)
 
@@ -53,10 +61,15 @@
 
 #if IS_ENABLED(CONFIG_DRM_I915_TRACE_GEM)
 #define GEM_TRACE(...) trace_printk(__VA_ARGS__)
+#define GEM_TRACE_DUMP() ftrace_dump(DUMP_ALL)
 #else
 #define GEM_TRACE(...) do { } while (0)
+#define GEM_TRACE_DUMP() do { } while (0)
 #endif
 
 #define I915_NUM_ENGINES 8
 
+void i915_gem_park(struct drm_i915_private *i915);
+void i915_gem_unpark(struct drm_i915_private *i915);
+
 #endif /* __I915_GEM_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
index d3cbe8432f48..f3890b664e3f 100644
--- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c
+++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
@@ -1,29 +1,11 @@
 /*
- * Copyright © 2014 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  *
+ * Copyright © 2014-2018 Intel Corporation
  */
 
-#include "i915_drv.h"
 #include "i915_gem_batch_pool.h"
+#include "i915_drv.h"
 
 /**
  * DOC: batch pool
@@ -41,11 +23,11 @@
 
 /**
  * i915_gem_batch_pool_init() - initialize a batch buffer pool
- * @engine: the associated request submission engine
  * @pool: the batch buffer pool
+ * @engine: the associated request submission engine
  */
-void i915_gem_batch_pool_init(struct intel_engine_cs *engine,
-			      struct i915_gem_batch_pool *pool)
+void i915_gem_batch_pool_init(struct i915_gem_batch_pool *pool,
+			      struct intel_engine_cs *engine)
 {
 	int n;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.h b/drivers/gpu/drm/i915/i915_gem_batch_pool.h
index 10d5ac4c00d3..56947daaaf65 100644
--- a/drivers/gpu/drm/i915/i915_gem_batch_pool.h
+++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.h
@@ -1,31 +1,13 @@
 /*
- * Copyright © 2014 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  *
+ * Copyright © 2014-2018 Intel Corporation
  */
 
 #ifndef I915_GEM_BATCH_POOL_H
 #define I915_GEM_BATCH_POOL_H
 
-#include "i915_drv.h"
+#include <linux/types.h>
 
 struct intel_engine_cs;
 
@@ -34,9 +16,8 @@ struct i915_gem_batch_pool {
 	struct list_head cache_list[4];
 };
 
-/* i915_gem_batch_pool.c */
-void i915_gem_batch_pool_init(struct intel_engine_cs *engine,
-			      struct i915_gem_batch_pool *pool);
+void i915_gem_batch_pool_init(struct i915_gem_batch_pool *pool,
+			      struct intel_engine_cs *engine);
 void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool);
 struct drm_i915_gem_object*
 i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size);
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index f2cbea7cf940..33f8a4b3c981 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -90,6 +90,7 @@
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_trace.h"
+#include "intel_workarounds.h"
 
 #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
 
@@ -116,15 +117,15 @@ static void lut_close(struct i915_gem_context *ctx)
 
 static void i915_gem_context_free(struct i915_gem_context *ctx)
 {
-	int i;
+	unsigned int n;
 
 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
 	GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
 
 	i915_ppgtt_put(ctx->ppgtt);
 
-	for (i = 0; i < I915_NUM_ENGINES; i++) {
-		struct intel_context *ce = &ctx->engine[i];
+	for (n = 0; n < ARRAY_SIZE(ctx->__engine); n++) {
+		struct intel_context *ce = &ctx->__engine[n];
 
 		if (!ce->state)
 			continue;
@@ -280,7 +281,7 @@ __create_hw_context(struct drm_i915_private *dev_priv,
 	kref_init(&ctx->ref);
 	list_add_tail(&ctx->link, &dev_priv->contexts.list);
 	ctx->i915 = dev_priv;
-	ctx->priority = I915_PRIORITY_NORMAL;
+	ctx->sched.priority = I915_PRIORITY_NORMAL;
 
 	INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL);
 	INIT_LIST_HEAD(&ctx->handles_list);
@@ -318,12 +319,13 @@ __create_hw_context(struct drm_i915_private *dev_priv,
 	ctx->desc_template =
 		default_desc_template(dev_priv, dev_priv->mm.aliasing_ppgtt);
 
-	/* GuC requires the ring to be placed above GUC_WOPCM_TOP. If GuC is not
+	/*
+	 * GuC requires the ring to be placed in Non-WOPCM memory. If GuC is not
 	 * present or not in use we still need a small bias as ring wraparound
 	 * at offset 0 sometimes hangs. No idea why.
 	 */
 	if (USES_GUC(dev_priv))
-		ctx->ggtt_offset_bias = GUC_WOPCM_TOP;
+		ctx->ggtt_offset_bias = dev_priv->guc.ggtt_pin_bias;
 	else
 		ctx->ggtt_offset_bias = I915_GTT_PAGE_SIZE;
 
@@ -429,7 +431,7 @@ i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio)
 		return ctx;
 
 	i915_gem_context_clear_bannable(ctx);
-	ctx->priority = prio;
+	ctx->sched.priority = prio;
 	ctx->ring_size = PAGE_SIZE;
 
 	GEM_BUG_ON(!i915_gem_context_is_kernel(ctx));
@@ -458,11 +460,16 @@ static bool needs_preempt_context(struct drm_i915_private *i915)
 int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
 {
 	struct i915_gem_context *ctx;
+	int ret;
 
 	/* Reassure ourselves we are only called once */
 	GEM_BUG_ON(dev_priv->kernel_context);
 	GEM_BUG_ON(dev_priv->preempt_context);
 
+	ret = intel_ctx_workarounds_init(dev_priv);
+	if (ret)
+		return ret;
+
 	INIT_LIST_HEAD(&dev_priv->contexts.list);
 	INIT_WORK(&dev_priv->contexts.free_work, contexts_free_worker);
 	init_llist_head(&dev_priv->contexts.free_list);
@@ -514,7 +521,7 @@ void i915_gem_contexts_lost(struct drm_i915_private *dev_priv)
 		if (!engine->last_retired_context)
 			continue;
 
-		engine->context_unpin(engine, engine->last_retired_context);
+		intel_context_unpin(engine->last_retired_context, engine);
 		engine->last_retired_context = NULL;
 	}
 }
@@ -570,19 +577,29 @@ void i915_gem_context_close(struct drm_file *file)
 	idr_destroy(&file_priv->context_idr);
 }
 
-static bool engine_has_idle_kernel_context(struct intel_engine_cs *engine)
+static struct i915_request *
+last_request_on_engine(struct i915_timeline *timeline,
+		       struct intel_engine_cs *engine)
 {
-	struct i915_gem_timeline *timeline;
+	struct i915_request *rq;
 
-	list_for_each_entry(timeline, &engine->i915->gt.timelines, link) {
-		struct intel_timeline *tl;
+	if (timeline == &engine->timeline)
+		return NULL;
 
-		if (timeline == &engine->i915->gt.global_timeline)
-			continue;
+	rq = i915_gem_active_raw(&timeline->last_request,
+				 &engine->i915->drm.struct_mutex);
+	if (rq && rq->engine == engine)
+		return rq;
+
+	return NULL;
+}
 
-		tl = &timeline->engine[engine->id];
-		if (i915_gem_active_peek(&tl->last_request,
-					 &engine->i915->drm.struct_mutex))
+static bool engine_has_idle_kernel_context(struct intel_engine_cs *engine)
+{
+	struct i915_timeline *timeline;
+
+	list_for_each_entry(timeline, &engine->i915->gt.timelines, link) {
+		if (last_request_on_engine(timeline, engine))
 			return false;
 	}
 
@@ -592,7 +609,7 @@ static bool engine_has_idle_kernel_context(struct intel_engine_cs *engine)
 int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
-	struct i915_gem_timeline *timeline;
+	struct i915_timeline *timeline;
 	enum intel_engine_id id;
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
@@ -612,11 +629,8 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
 		/* Queue this switch after all other activity */
 		list_for_each_entry(timeline, &dev_priv->gt.timelines, link) {
 			struct i915_request *prev;
-			struct intel_timeline *tl;
 
-			tl = &timeline->engine[engine->id];
-			prev = i915_gem_active_raw(&tl->last_request,
-						   &dev_priv->drm.struct_mutex);
+			prev = last_request_on_engine(timeline, engine);
 			if (prev)
 				i915_sw_fence_await_sw_fence_gfp(&rq->submit,
 								 &prev->submit,
@@ -746,7 +760,7 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
 		args->value = i915_gem_context_is_bannable(ctx);
 		break;
 	case I915_CONTEXT_PARAM_PRIORITY:
-		args->value = ctx->priority;
+		args->value = ctx->sched.priority;
 		break;
 	default:
 		ret = -EINVAL;
@@ -819,7 +833,7 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 				 !capable(CAP_SYS_NICE))
 				ret = -EPERM;
 			else
-				ctx->priority = priority;
+				ctx->sched.priority = priority;
 		}
 		break;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_context.h b/drivers/gpu/drm/i915/i915_gem_context.h
index 7854262ddfd9..ace3b129c189 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.h
+++ b/drivers/gpu/drm/i915/i915_gem_context.h
@@ -137,18 +137,7 @@ struct i915_gem_context {
 	 */
 	u32 user_handle;
 
-	/**
-	 * @priority: execution and service priority
-	 *
-	 * All clients are equal, but some are more equal than others!
-	 *
-	 * Requests from a context with a greater (more positive) value of
-	 * @priority will be executed before those with a lower @priority
-	 * value, forming a simple QoS.
-	 *
-	 * The &drm_i915_private.kernel_context is assigned the lowest priority.
-	 */
-	int priority;
+	struct i915_sched_attr sched;
 
 	/** ggtt_offset_bias: placement restriction for context objects */
 	u32 ggtt_offset_bias;
@@ -160,7 +149,7 @@ struct i915_gem_context {
 		u32 *lrc_reg_state;
 		u64 lrc_desc;
 		int pin_count;
-	} engine[I915_NUM_ENGINES];
+	} __engine[I915_NUM_ENGINES];
 
 	/** ring_size: size for allocating the per-engine ring buffer */
 	u32 ring_size;
@@ -267,6 +256,34 @@ static inline bool i915_gem_context_is_kernel(struct i915_gem_context *ctx)
 	return !ctx->file_priv;
 }
 
+static inline struct intel_context *
+to_intel_context(struct i915_gem_context *ctx,
+		 const struct intel_engine_cs *engine)
+{
+	return &ctx->__engine[engine->id];
+}
+
+static inline struct intel_ring *
+intel_context_pin(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
+{
+	return engine->context_pin(engine, ctx);
+}
+
+static inline void __intel_context_pin(struct i915_gem_context *ctx,
+				       const struct intel_engine_cs *engine)
+{
+	struct intel_context *ce = to_intel_context(ctx, engine);
+
+	GEM_BUG_ON(!ce->pin_count);
+	ce->pin_count++;
+}
+
+static inline void intel_context_unpin(struct i915_gem_context *ctx,
+				       struct intel_engine_cs *engine)
+{
+	engine->context_unpin(engine, ctx);
+}
+
 /* i915_gem_context.c */
 int __must_check i915_gem_contexts_init(struct drm_i915_private *dev_priv);
 void i915_gem_contexts_lost(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 0414228cd2b5..f627a8c47c58 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -81,6 +81,35 @@ enum {
  * but this remains just a hint as the kernel may choose a new location for
  * any object in the future.
  *
+ * At the level of talking to the hardware, submitting a batchbuffer for the
+ * GPU to execute is to add content to a buffer from which the HW
+ * command streamer is reading.
+ *
+ * 1. Add a command to load the HW context. For Logical Ring Contexts, i.e.
+ *    Execlists, this command is not placed on the same buffer as the
+ *    remaining items.
+ *
+ * 2. Add a command to invalidate caches to the buffer.
+ *
+ * 3. Add a batchbuffer start command to the buffer; the start command is
+ *    essentially a token together with the GPU address of the batchbuffer
+ *    to be executed.
+ *
+ * 4. Add a pipeline flush to the buffer.
+ *
+ * 5. Add a memory write command to the buffer to record when the GPU
+ *    is done executing the batchbuffer. The memory write writes the
+ *    global sequence number of the request, ``i915_request::global_seqno``;
+ *    the i915 driver uses the current value in the register to determine
+ *    if the GPU has completed the batchbuffer.
+ *
+ * 6. Add a user interrupt command to the buffer. This command instructs
+ *    the GPU to issue an interrupt when the command, pipeline flush and
+ *    memory write are completed.
+ *
+ * 7. Inform the hardware of the additional commands added to the buffer
+ *    (by updating the tail pointer).
+ *
  * Processing an execbuf ioctl is conceptually split up into a few phases.
  *
  * 1. Validation - Ensure all the pointers, handles and flags are valid.
@@ -733,7 +762,8 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
 		}
 
 		/* transfer ref to ctx */
-		vma->open_count++;
+		if (!vma->open_count++)
+			i915_vma_reopen(vma);
 		list_add(&lut->obj_link, &obj->lut_list);
 		list_add(&lut->ctx_link, &eb->ctx->handles_list);
 		lut->ctx = eb->ctx;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 21d72f695adb..996ab2ad6c45 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -110,7 +110,8 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma);
 
 static void gen6_ggtt_invalidate(struct drm_i915_private *dev_priv)
 {
-	/* Note that as an uncached mmio write, this should flush the
+	/*
+	 * Note that as an uncached mmio write, this will flush the
 	 * WCB of the writes into the GGTT before it triggers the invalidate.
 	 */
 	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
@@ -1161,6 +1162,27 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
 			vaddr[idx.pde] |= GEN8_PDE_IPS_64K;
 			kunmap_atomic(vaddr);
 			page_size = I915_GTT_PAGE_SIZE_64K;
+
+			/*
+			 * We write all 4K page entries, even when using 64K
+			 * pages. In order to verify that the HW isn't cheating
+			 * by using the 4K PTE instead of the 64K PTE, we want
+			 * to remove all the surplus entries. If the HW skipped
+			 * the 64K PTE, it will read/write into the scratch page
+			 * instead - which we detect as missing results during
+			 * selftests.
+			 */
+			if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) {
+				u16 i;
+
+				encode = pte_encode | vma->vm->scratch_page.daddr;
+				vaddr = kmap_atomic_px(pd->page_table[idx.pde]);
+
+				for (i = 1; i < index; i += 16)
+					memset64(vaddr + i, encode, 15);
+
+				kunmap_atomic(vaddr);
+			}
 		}
 
 		vma->page_sizes.gtt |= page_size;
@@ -2111,8 +2133,6 @@ static void i915_address_space_init(struct i915_address_space *vm,
 				    struct drm_i915_private *dev_priv,
 				    const char *name)
 {
-	i915_gem_timeline_init(dev_priv, &vm->timeline, name);
-
 	drm_mm_init(&vm->mm, 0, vm->total);
 	vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
 
@@ -2129,7 +2149,6 @@ static void i915_address_space_fini(struct i915_address_space *vm)
 	if (pagevec_count(&vm->free_pages))
 		vm_free_pages_release(vm, true);
 
-	i915_gem_timeline_fini(&vm->timeline);
 	drm_mm_takedown(&vm->mm);
 	list_del(&vm->global_link);
 }
@@ -2140,15 +2159,15 @@ static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
 	 * called on driver load and after a GPU reset, so you can place
 	 * workarounds here even if they get overwritten by GPU reset.
 	 */
-	/* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl,cnl */
+	/* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl,cnl,icl */
 	if (IS_BROADWELL(dev_priv))
 		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
 	else if (IS_CHERRYVIEW(dev_priv))
 		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
-	else if (IS_GEN9_BC(dev_priv) || IS_GEN10(dev_priv))
-		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
 	else if (IS_GEN9_LP(dev_priv))
 		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
+	else if (INTEL_GEN(dev_priv) >= 9)
+		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
 
 	/*
 	 * To support 64K PTEs we need to first enable the use of the
@@ -2221,6 +2240,12 @@ i915_ppgtt_create(struct drm_i915_private *dev_priv,
 }
 
 void i915_ppgtt_close(struct i915_address_space *vm)
+{
+	GEM_BUG_ON(vm->closed);
+	vm->closed = true;
+}
+
+static void ppgtt_destroy_vma(struct i915_address_space *vm)
 {
 	struct list_head *phases[] = {
 		&vm->active_list,
@@ -2229,15 +2254,12 @@ void i915_ppgtt_close(struct i915_address_space *vm)
 		NULL,
 	}, **phase;
 
-	GEM_BUG_ON(vm->closed);
 	vm->closed = true;
-
 	for (phase = phases; *phase; phase++) {
 		struct i915_vma *vma, *vn;
 
 		list_for_each_entry_safe(vma, vn, *phase, vm_link)
-			if (!i915_vma_is_closed(vma))
-				i915_vma_close(vma);
+			i915_vma_destroy(vma);
 	}
 }
 
@@ -2248,7 +2270,8 @@ void i915_ppgtt_release(struct kref *kref)
 
 	trace_i915_ppgtt_release(&ppgtt->base);
 
-	/* vmas should already be unbound and destroyed */
+	ppgtt_destroy_vma(&ppgtt->base);
+
 	GEM_BUG_ON(!list_empty(&ppgtt->base.active_list));
 	GEM_BUG_ON(!list_empty(&ppgtt->base.inactive_list));
 	GEM_BUG_ON(!list_empty(&ppgtt->base.unbound_list));
@@ -2417,11 +2440,9 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 	for_each_sgt_dma(addr, sgt_iter, vma->pages)
 		gen8_set_pte(gtt_entries++, pte_encode | addr);
 
-	wmb();
-
-	/* This next bit makes the above posting read even more important. We
-	 * want to flush the TLBs only after we're certain all the PTE updates
-	 * have finished.
+	/*
+	 * We want to flush the TLBs only after we're certain all the PTE
+	 * updates have finished.
 	 */
 	ggtt->invalidate(vm->i915);
 }
@@ -2459,11 +2480,10 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 	dma_addr_t addr;
 	for_each_sgt_dma(addr, iter, vma->pages)
 		iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]);
-	wmb();
 
-	/* This next bit makes the above posting read even more important. We
-	 * want to flush the TLBs only after we're certain all the PTE updates
-	 * have finished.
+	/*
+	 * We want to flush the TLBs only after we're certain all the PTE
+	 * updates have finished.
 	 */
 	ggtt->invalidate(vm->i915);
 }
@@ -3325,14 +3345,10 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 		DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
 
 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
-
-	if (INTEL_GEN(dev_priv) >= 9) {
-		size = gen8_get_total_gtt_size(snb_gmch_ctl);
-	} else if (IS_CHERRYVIEW(dev_priv)) {
+	if (IS_CHERRYVIEW(dev_priv))
 		size = chv_get_total_gtt_size(snb_gmch_ctl);
-	} else {
+	else
 		size = gen8_get_total_gtt_size(snb_gmch_ctl);
-	}
 
 	ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
 	ggtt->base.cleanup = gen6_gmch_remove;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 6efc017e8bb3..aec4f73574f4 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -38,10 +38,9 @@
 #include <linux/mm.h>
 #include <linux/pagevec.h>
 
-#include "i915_gem_timeline.h"
-
 #include "i915_request.h"
 #include "i915_selftest.h"
+#include "i915_timeline.h"
 
 #define I915_GTT_PAGE_SIZE_4K BIT(12)
 #define I915_GTT_PAGE_SIZE_64K BIT(16)
@@ -257,7 +256,6 @@ struct i915_pml4 {
 
 struct i915_address_space {
 	struct drm_mm mm;
-	struct i915_gem_timeline timeline;
 	struct drm_i915_private *i915;
 	struct device *dma;
 	/* Every address space belongs to a struct file - except for the global
@@ -344,6 +342,7 @@ struct i915_address_space {
 	void (*clear_pages)(struct i915_vma *vma);
 
 	I915_SELFTEST_DECLARE(struct fault_attr fault_attr);
+	I915_SELFTEST_DECLARE(bool scrub_64K);
 };
 
 #define i915_is_ggtt(V) (!(V)->file)
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 62aa67960bf4..ad949cc30928 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -51,6 +51,10 @@ int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv,
 	if (!drm_mm_initialized(&dev_priv->mm.stolen))
 		return -ENODEV;
 
+	/* WaSkipStolenMemoryFirstPage:bdw+ */
+	if (INTEL_GEN(dev_priv) >= 8 && start < 4096)
+		start = 4096;
+
 	mutex_lock(&dev_priv->mm.stolen_lock);
 	ret = drm_mm_insert_node_in_range(&dev_priv->mm.stolen, node,
 					  size, alignment, 0,
@@ -121,8 +125,8 @@ static int i915_adjust_stolen(struct drm_i915_private *dev_priv,
 
 		if (stolen[0].start != stolen[1].start ||
 		    stolen[0].end != stolen[1].end) {
-			DRM_DEBUG_KMS("GTT within stolen memory at %pR\n", &ggtt_res);
-			DRM_DEBUG_KMS("Stolen memory adjusted to %pR\n", dsm);
+			DRM_DEBUG_DRIVER("GTT within stolen memory at %pR\n", &ggtt_res);
+			DRM_DEBUG_DRIVER("Stolen memory adjusted to %pR\n", dsm);
 		}
 	}
 
@@ -174,18 +178,19 @@ void i915_gem_cleanup_stolen(struct drm_device *dev)
 }
 
 static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
-				    resource_size_t *base, resource_size_t *size)
+				    resource_size_t *base,
+				    resource_size_t *size)
 {
-	uint32_t reg_val = I915_READ(IS_GM45(dev_priv) ?
-				     CTG_STOLEN_RESERVED :
-				     ELK_STOLEN_RESERVED);
+	u32 reg_val = I915_READ(IS_GM45(dev_priv) ?
+				CTG_STOLEN_RESERVED :
+				ELK_STOLEN_RESERVED);
 	resource_size_t stolen_top = dev_priv->dsm.end + 1;
 
-	if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0) {
-		*base = 0;
-		*size = 0;
+	DRM_DEBUG_DRIVER("%s_STOLEN_RESERVED = %08x\n",
+			 IS_GM45(dev_priv) ? "CTG" : "ELK", reg_val);
+
+	if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0)
 		return;
-	}
 
 	/*
 	 * Whether ILK really reuses the ELK register for this is unclear.
@@ -193,30 +198,25 @@ static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
 	 */
 	WARN(IS_GEN5(dev_priv), "ILK stolen reserved found? 0x%08x\n", reg_val);
 
-	*base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
+	if (!(reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK))
+		return;
 
+	*base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
 	WARN_ON((reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base);
 
-	/* On these platforms, the register doesn't have a size field, so the
-	 * size is the distance between the base and the top of the stolen
-	 * memory. We also have the genuine case where base is zero and there's
-	 * nothing reserved. */
-	if (*base == 0)
-		*size = 0;
-	else
-		*size = stolen_top - *base;
+	*size = stolen_top - *base;
 }
 
 static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
-				     resource_size_t *base, resource_size_t *size)
+				     resource_size_t *base,
+				     resource_size_t *size)
 {
-	uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+	u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+
+	DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 
-	if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
-		*base = 0;
-		*size = 0;
+	if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 		return;
-	}
 
 	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 
@@ -239,17 +239,44 @@ static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
 	}
 }
 
-static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
-				     resource_size_t *base, resource_size_t *size)
+static void vlv_get_stolen_reserved(struct drm_i915_private *dev_priv,
+				    resource_size_t *base,
+				    resource_size_t *size)
 {
-	uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+	u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+	resource_size_t stolen_top = dev_priv->dsm.end + 1;
+
+	DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 
-	if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
-		*base = 0;
-		*size = 0;
+	if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 		return;
+
+	switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
+	default:
+		MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
+	case GEN7_STOLEN_RESERVED_1M:
+		*size = 1024 * 1024;
+		break;
 	}
 
+	/*
+	 * On vlv, the ADDR_MASK portion is left as 0 and HW deduces the
+	 * reserved location as (top - size).
+	 */
+	*base = stolen_top - *size;
+}
+
+static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
+				     resource_size_t *base,
+				     resource_size_t *size)
+{
+	u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+
+	DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
+
+	if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
+		return;
+
 	*base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK;
 
 	switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
@@ -266,15 +293,15 @@ static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
 }
 
 static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv,
-				    resource_size_t *base, resource_size_t *size)
+				    resource_size_t *base,
+				    resource_size_t *size)
 {
-	uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+	u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 
-	if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
-		*base = 0;
-		*size = 0;
+	DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
+
+	if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 		return;
-	}
 
 	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 
@@ -298,36 +325,28 @@ static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv,
 }
 
 static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
-				    resource_size_t *base, resource_size_t *size)
+				    resource_size_t *base,
+				    resource_size_t *size)
 {
-	uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
-	resource_size_t stolen_top;
+	u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+	resource_size_t stolen_top = dev_priv->dsm.end + 1;
 
-	if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
-		*base = 0;
-		*size = 0;
+	DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
+
+	if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 		return;
-	}
 
-	stolen_top = dev_priv->dsm.end + 1;
+	if (!(reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK))
+		return;
 
 	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
-
-	/* On these platforms, the register doesn't have a size field, so the
-	 * size is the distance between the base and the top of the stolen
-	 * memory. We also have the genuine case where base is zero and there's
-	 * nothing reserved. */
-	if (*base == 0)
-		*size = 0;
-	else
-		*size = stolen_top - *base;
+	*size = stolen_top - *base;
 }
 
 int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 {
 	resource_size_t reserved_base, stolen_top;
 	resource_size_t reserved_total, reserved_size;
-	resource_size_t stolen_usable_start;
 
 	mutex_init(&dev_priv->mm.stolen_lock);
 
@@ -353,7 +372,7 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 	GEM_BUG_ON(dev_priv->dsm.end <= dev_priv->dsm.start);
 
 	stolen_top = dev_priv->dsm.end + 1;
-	reserved_base = 0;
+	reserved_base = stolen_top;
 	reserved_size = 0;
 
 	switch (INTEL_GEN(dev_priv)) {
@@ -373,8 +392,12 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 					 &reserved_base, &reserved_size);
 		break;
 	case 7:
-		gen7_get_stolen_reserved(dev_priv,
-					 &reserved_base, &reserved_size);
+		if (IS_VALLEYVIEW(dev_priv))
+			vlv_get_stolen_reserved(dev_priv,
+						&reserved_base, &reserved_size);
+		else
+			gen7_get_stolen_reserved(dev_priv,
+						 &reserved_base, &reserved_size);
 		break;
 	default:
 		if (IS_LP(dev_priv))
@@ -386,11 +409,16 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 		break;
 	}
 
-	/* It is possible for the reserved base to be zero, but the register
-	 * field for size doesn't have a zero option. */
-	if (reserved_base == 0) {
-		reserved_size = 0;
+	/*
+	 * Our expectation is that the reserved space is at the top of the
+	 * stolen region and *never* at the bottom. If we see !reserved_base,
+	 * it likely means we failed to read the registers correctly.
+	 */
+	if (!reserved_base) {
+		DRM_ERROR("inconsistent reservation %pa + %pa; ignoring\n",
+			  &reserved_base, &reserved_size);
 		reserved_base = stolen_top;
+		reserved_size = 0;
 	}
 
 	dev_priv->dsm_reserved =
@@ -406,21 +434,15 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 	 * memory, so just consider the start. */
 	reserved_total = stolen_top - reserved_base;
 
-	DRM_DEBUG_KMS("Memory reserved for graphics device: %lluK, usable: %lluK\n",
-		      (u64)resource_size(&dev_priv->dsm) >> 10,
-		      ((u64)resource_size(&dev_priv->dsm) - reserved_total) >> 10);
-
-	stolen_usable_start = 0;
-	/* WaSkipStolenMemoryFirstPage:bdw+ */
-	if (INTEL_GEN(dev_priv) >= 8)
-		stolen_usable_start = 4096;
+	DRM_DEBUG_DRIVER("Memory reserved for graphics device: %lluK, usable: %lluK\n",
+			 (u64)resource_size(&dev_priv->dsm) >> 10,
+			 ((u64)resource_size(&dev_priv->dsm) - reserved_total) >> 10);
 
 	dev_priv->stolen_usable_size =
-		resource_size(&dev_priv->dsm) - reserved_total - stolen_usable_start;
+		resource_size(&dev_priv->dsm) - reserved_total;
 
 	/* Basic memrange allocator for stolen space. */
-	drm_mm_init(&dev_priv->mm.stolen, stolen_usable_start,
-		    dev_priv->stolen_usable_size);
+	drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->stolen_usable_size);
 
 	return 0;
 }
@@ -580,8 +602,8 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
-	DRM_DEBUG_KMS("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
-			&stolen_offset, &gtt_offset, &size);
+	DRM_DEBUG_DRIVER("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
+			 &stolen_offset, &gtt_offset, &size);
 
 	/* KISS and expect everything to be page-aligned */
 	if (WARN_ON(size == 0) ||
@@ -599,14 +621,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
 	ret = drm_mm_reserve_node(&dev_priv->mm.stolen, stolen);
 	mutex_unlock(&dev_priv->mm.stolen_lock);
 	if (ret) {
-		DRM_DEBUG_KMS("failed to allocate stolen space\n");
+		DRM_DEBUG_DRIVER("failed to allocate stolen space\n");
 		kfree(stolen);
 		return NULL;
 	}
 
 	obj = _i915_gem_object_create_stolen(dev_priv, stolen);
 	if (obj == NULL) {
-		DRM_DEBUG_KMS("failed to allocate stolen object\n");
+		DRM_DEBUG_DRIVER("failed to allocate stolen object\n");
 		i915_gem_stolen_remove_node(dev_priv, stolen);
 		kfree(stolen);
 		return NULL;
@@ -635,7 +657,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
 				   size, gtt_offset, obj->cache_level,
 				   0);
 	if (ret) {
-		DRM_DEBUG_KMS("failed to allocate stolen GTT space\n");
+		DRM_DEBUG_DRIVER("failed to allocate stolen GTT space\n");
 		goto err_pages;
 	}
 
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.c b/drivers/gpu/drm/i915/i915_gem_timeline.c
deleted file mode 100644
index e9fd87604067..000000000000
--- a/drivers/gpu/drm/i915/i915_gem_timeline.c
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Copyright © 2016 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- *
- */
-
-#include "i915_drv.h"
-#include "i915_syncmap.h"
-
-static void __intel_timeline_init(struct intel_timeline *tl,
-				  struct i915_gem_timeline *parent,
-				  u64 context,
-				  struct lock_class_key *lockclass,
-				  const char *lockname)
-{
-	tl->fence_context = context;
-	tl->common = parent;
-	spin_lock_init(&tl->lock);
-	lockdep_set_class_and_name(&tl->lock, lockclass, lockname);
-	init_request_active(&tl->last_request, NULL);
-	INIT_LIST_HEAD(&tl->requests);
-	i915_syncmap_init(&tl->sync);
-}
-
-static void __intel_timeline_fini(struct intel_timeline *tl)
-{
-	GEM_BUG_ON(!list_empty(&tl->requests));
-
-	i915_syncmap_free(&tl->sync);
-}
-
-static int __i915_gem_timeline_init(struct drm_i915_private *i915,
-				    struct i915_gem_timeline *timeline,
-				    const char *name,
-				    struct lock_class_key *lockclass,
-				    const char *lockname)
-{
-	unsigned int i;
-	u64 fences;
-
-	lockdep_assert_held(&i915->drm.struct_mutex);
-
-	/*
-	 * Ideally we want a set of engines on a single leaf as we expect
-	 * to mostly be tracking synchronisation between engines. It is not
-	 * a huge issue if this is not the case, but we may want to mitigate
-	 * any page crossing penalties if they become an issue.
-	 */
-	BUILD_BUG_ON(KSYNCMAP < I915_NUM_ENGINES);
-
-	timeline->i915 = i915;
-	timeline->name = kstrdup(name ?: "[kernel]", GFP_KERNEL);
-	if (!timeline->name)
-		return -ENOMEM;
-
-	list_add(&timeline->link, &i915->gt.timelines);
-
-	/* Called during early_init before we know how many engines there are */
-	fences = dma_fence_context_alloc(ARRAY_SIZE(timeline->engine));
-	for (i = 0; i < ARRAY_SIZE(timeline->engine); i++)
-		__intel_timeline_init(&timeline->engine[i],
-				      timeline, fences++,
-				      lockclass, lockname);
-
-	return 0;
-}
-
-int i915_gem_timeline_init(struct drm_i915_private *i915,
-			   struct i915_gem_timeline *timeline,
-			   const char *name)
-{
-	static struct lock_class_key class;
-
-	return __i915_gem_timeline_init(i915, timeline, name,
-					&class, "&timeline->lock");
-}
-
-int i915_gem_timeline_init__global(struct drm_i915_private *i915)
-{
-	static struct lock_class_key class;
-
-	return __i915_gem_timeline_init(i915,
-					&i915->gt.global_timeline,
-					"[execution]",
-					&class, "&global_timeline->lock");
-}
-
-/**
- * i915_gem_timelines_park - called when the driver idles
- * @i915: the drm_i915_private device
- *
- * When the driver is completely idle, we know that all of our sync points
- * have been signaled and our tracking is then entirely redundant. Any request
- * to wait upon an older sync point will be completed instantly as we know
- * the fence is signaled and therefore we will not even look them up in the
- * sync point map.
- */
-void i915_gem_timelines_park(struct drm_i915_private *i915)
-{
-	struct i915_gem_timeline *timeline;
-	int i;
-
-	lockdep_assert_held(&i915->drm.struct_mutex);
-
-	list_for_each_entry(timeline, &i915->gt.timelines, link) {
-		for (i = 0; i < ARRAY_SIZE(timeline->engine); i++) {
-			struct intel_timeline *tl = &timeline->engine[i];
-
-			/*
-			 * All known fences are completed so we can scrap
-			 * the current sync point tracking and start afresh,
-			 * any attempt to wait upon a previous sync point
-			 * will be skipped as the fence was signaled.
-			 */
-			i915_syncmap_free(&tl->sync);
-		}
-	}
-}
-
-void i915_gem_timeline_fini(struct i915_gem_timeline *timeline)
-{
-	int i;
-
-	lockdep_assert_held(&timeline->i915->drm.struct_mutex);
-
-	for (i = 0; i < ARRAY_SIZE(timeline->engine); i++)
-		__intel_timeline_fini(&timeline->engine[i]);
-
-	list_del(&timeline->link);
-	kfree(timeline->name);
-}
-
-#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
-#include "selftests/mock_timeline.c"
-#include "selftests/i915_gem_timeline.c"
-#endif
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h
deleted file mode 100644
index 33e01bf6aa36..000000000000
--- a/drivers/gpu/drm/i915/i915_gem_timeline.h
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright © 2016 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- *
- */
-
-#ifndef I915_GEM_TIMELINE_H
-#define I915_GEM_TIMELINE_H
-
-#include <linux/list.h>
-
-#include "i915_request.h"
-#include "i915_syncmap.h"
-#include "i915_utils.h"
-
-struct i915_gem_timeline;
-
-struct intel_timeline {
-	u64 fence_context;
-	u32 seqno;
-
-	/**
-	 * Count of outstanding requests, from the time they are constructed
-	 * to the moment they are retired. Loosely coupled to hardware.
-	 */
-	u32 inflight_seqnos;
-
-	spinlock_t lock;
-
-	/**
-	 * List of breadcrumbs associated with GPU requests currently
-	 * outstanding.
-	 */
-	struct list_head requests;
-
-	/* Contains an RCU guarded pointer to the last request. No reference is
-	 * held to the request, users must carefully acquire a reference to
-	 * the request using i915_gem_active_get_request_rcu(), or hold the
-	 * struct_mutex.
-	 */
-	struct i915_gem_active last_request;
-
-	/**
-	 * We track the most recent seqno that we wait on in every context so
-	 * that we only have to emit a new await and dependency on a more
-	 * recent sync point. As the contexts may be executed out-of-order, we
-	 * have to track each individually and can not rely on an absolute
-	 * global_seqno. When we know that all tracked fences are completed
-	 * (i.e. when the driver is idle), we know that the syncmap is
-	 * redundant and we can discard it without loss of generality.
-	 */
-	struct i915_syncmap *sync;
-	/**
-	 * Separately to the inter-context seqno map above, we track the last
-	 * barrier (e.g. semaphore wait) to the global engine timelines. Note
-	 * that this tracks global_seqno rather than the context.seqno, and
-	 * so it is subject to the limitations of hw wraparound and that we
-	 * may need to revoke global_seqno (on pre-emption).
-	 */
-	u32 global_sync[I915_NUM_ENGINES];
-
-	struct i915_gem_timeline *common;
-};
-
-struct i915_gem_timeline {
-	struct list_head link;
-
-	struct drm_i915_private *i915;
-	const char *name;
-
-	struct intel_timeline engine[I915_NUM_ENGINES];
-};
-
-int i915_gem_timeline_init(struct drm_i915_private *i915,
-			   struct i915_gem_timeline *tl,
-			   const char *name);
-int i915_gem_timeline_init__global(struct drm_i915_private *i915);
-void i915_gem_timelines_park(struct drm_i915_private *i915);
-void i915_gem_timeline_fini(struct i915_gem_timeline *tl);
-
-static inline int __intel_timeline_sync_set(struct intel_timeline *tl,
-					    u64 context, u32 seqno)
-{
-	return i915_syncmap_set(&tl->sync, context, seqno);
-}
-
-static inline int intel_timeline_sync_set(struct intel_timeline *tl,
-					  const struct dma_fence *fence)
-{
-	return __intel_timeline_sync_set(tl, fence->context, fence->seqno);
-}
-
-static inline bool __intel_timeline_sync_is_later(struct intel_timeline *tl,
-						  u64 context, u32 seqno)
-{
-	return i915_syncmap_is_later(&tl->sync, context, seqno);
-}
-
-static inline bool intel_timeline_sync_is_later(struct intel_timeline *tl,
-						const struct dma_fence *fence)
-{
-	return __intel_timeline_sync_is_later(tl, fence->context, fence->seqno);
-}
-
-#endif
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index f89ac7a8f95f..df234dc23274 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -32,6 +32,7 @@
 #include <linux/zlib.h>
 #include <drm/drm_print.h>
 
+#include "i915_gpu_error.h"
 #include "i915_drv.h"
 
 static inline const struct intel_engine_cs *
@@ -403,16 +404,17 @@ static const char *bannable(const struct drm_i915_error_context *ctx)
 
 static void error_print_request(struct drm_i915_error_state_buf *m,
 				const char *prefix,
-				const struct drm_i915_error_request *erq)
+				const struct drm_i915_error_request *erq,
+				const unsigned long epoch)
 {
 	if (!erq->seqno)
 		return;
 
-	err_printf(m, "%s pid %d, ban score %d, seqno %8x:%08x, prio %d, emitted %dms ago, head %08x, tail %08x\n",
+	err_printf(m, "%s pid %d, ban score %d, seqno %8x:%08x, prio %d, emitted %dms, start %08x, head %08x, tail %08x\n",
 		   prefix, erq->pid, erq->ban_score,
-		   erq->context, erq->seqno, erq->priority,
-		   jiffies_to_msecs(jiffies - erq->jiffies),
-		   erq->head, erq->tail);
+		   erq->context, erq->seqno, erq->sched_attr.priority,
+		   jiffies_to_msecs(erq->jiffies - epoch),
+		   erq->start, erq->head, erq->tail);
 }
 
 static void error_print_context(struct drm_i915_error_state_buf *m,
@@ -421,12 +423,13 @@ static void error_print_context(struct drm_i915_error_state_buf *m,
 {
 	err_printf(m, "%s%s[%d] user_handle %d hw_id %d, prio %d, ban score %d%s guilty %d active %d\n",
 		   header, ctx->comm, ctx->pid, ctx->handle, ctx->hw_id,
-		   ctx->priority, ctx->ban_score, bannable(ctx),
+		   ctx->sched_attr.priority, ctx->ban_score, bannable(ctx),
 		   ctx->guilty, ctx->active);
 }
 
 static void error_print_engine(struct drm_i915_error_state_buf *m,
-			       const struct drm_i915_error_engine *ee)
+			       const struct drm_i915_error_engine *ee,
+			       const unsigned long epoch)
 {
 	int n;
 
@@ -496,14 +499,15 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
 	err_printf(m, "  hangcheck stall: %s\n", yesno(ee->hangcheck_stalled));
 	err_printf(m, "  hangcheck action: %s\n",
 		   hangcheck_action_to_str(ee->hangcheck_action));
-	err_printf(m, "  hangcheck action timestamp: %lu, %u ms ago\n",
+	err_printf(m, "  hangcheck action timestamp: %dms (%lu%s)\n",
+		   jiffies_to_msecs(ee->hangcheck_timestamp - epoch),
 		   ee->hangcheck_timestamp,
-		   jiffies_to_msecs(jiffies - ee->hangcheck_timestamp));
+		   ee->hangcheck_timestamp == epoch ? "; epoch" : "");
 	err_printf(m, "  engine reset count: %u\n", ee->reset_count);
 
 	for (n = 0; n < ee->num_ports; n++) {
 		err_printf(m, "  ELSP[%d]:", n);
-		error_print_request(m, " ", &ee->execlist[n]);
+		error_print_request(m, " ", &ee->execlist[n], epoch);
 	}
 
 	error_print_context(m, "  Active context: ", &ee->context);
@@ -649,6 +653,11 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 	ts = ktime_to_timespec64(error->uptime);
 	err_printf(m, "Uptime: %lld s %ld us\n",
 		   (s64)ts.tv_sec, ts.tv_nsec / NSEC_PER_USEC);
+	err_printf(m, "Epoch: %lu jiffies (%u HZ)\n", error->epoch, HZ);
+	err_printf(m, "Capture: %lu jiffies; %d ms ago, %d ms after epoch\n",
+		   error->capture,
+		   jiffies_to_msecs(jiffies - error->capture),
+		   jiffies_to_msecs(error->capture - error->epoch));
 
 	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
 		if (error->engine[i].hangcheck_stalled &&
@@ -709,7 +718,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 
 	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
 		if (error->engine[i].engine_id != -1)
-			error_print_engine(m, &error->engine[i]);
+			error_print_engine(m, &error->engine[i], error->epoch);
 	}
 
 	for (i = 0; i < ARRAY_SIZE(error->active_vm); i++) {
@@ -768,7 +777,9 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 				   dev_priv->engine[i]->name,
 				   ee->num_requests);
 			for (j = 0; j < ee->num_requests; j++)
-				error_print_request(m, " ", &ee->requests[j]);
+				error_print_request(m, " ",
+						    &ee->requests[j],
+						    error->epoch);
 		}
 
 		if (IS_ERR(ee->waiters)) {
@@ -1277,10 +1288,11 @@ static void record_request(struct i915_request *request,
 			   struct drm_i915_error_request *erq)
 {
 	erq->context = request->ctx->hw_id;
-	erq->priority = request->priotree.priority;
+	erq->sched_attr = request->sched.attr;
 	erq->ban_score = atomic_read(&request->ctx->ban_score);
 	erq->seqno = request->global_seqno;
 	erq->jiffies = request->emitted_jiffies;
+	erq->start = i915_ggtt_offset(request->ring->vma);
 	erq->head = request->head;
 	erq->tail = request->tail;
 
@@ -1298,7 +1310,7 @@ static void engine_record_requests(struct intel_engine_cs *engine,
 
 	count = 0;
 	request = first;
-	list_for_each_entry_from(request, &engine->timeline->requests, link)
+	list_for_each_entry_from(request, &engine->timeline.requests, link)
 		count++;
 	if (!count)
 		return;
@@ -1311,7 +1323,7 @@ static void engine_record_requests(struct intel_engine_cs *engine,
 
 	count = 0;
 	request = first;
-	list_for_each_entry_from(request, &engine->timeline->requests, link) {
+	list_for_each_entry_from(request, &engine->timeline.requests, link) {
 		if (count >= ee->num_requests) {
 			/*
 			 * If the ring request list was changed in
@@ -1371,7 +1383,7 @@ static void record_context(struct drm_i915_error_context *e,
 
 	e->handle = ctx->user_handle;
 	e->hw_id = ctx->hw_id;
-	e->priority = ctx->priority;
+	e->sched_attr = ctx->sched;
 	e->ban_score = atomic_read(&ctx->ban_score);
 	e->bannable = i915_gem_context_is_bannable(ctx);
 	e->guilty = atomic_read(&ctx->guilty_count);
@@ -1471,7 +1483,8 @@ static void gem_record_rings(struct i915_gpu_state *error)
 
 			ee->ctx =
 				i915_error_object_create(i915,
-							 request->ctx->engine[i].state);
+							 to_intel_context(request->ctx,
+									  engine)->state);
 
 			error->simulated |=
 				i915_gem_context_no_error_capture(request->ctx);
@@ -1734,6 +1747,22 @@ static void capture_params(struct i915_gpu_state *error)
 #undef DUP
 }
 
+static unsigned long capture_find_epoch(const struct i915_gpu_state *error)
+{
+	unsigned long epoch = error->capture;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
+		const struct drm_i915_error_engine *ee = &error->engine[i];
+
+		if (ee->hangcheck_stalled &&
+		    time_before(ee->hangcheck_timestamp, epoch))
+			epoch = ee->hangcheck_timestamp;
+	}
+
+	return epoch;
+}
+
 static int capture(void *data)
 {
 	struct i915_gpu_state *error = data;
@@ -1742,6 +1771,7 @@ static int capture(void *data)
 	error->boottime = ktime_get_boottime();
 	error->uptime = ktime_sub(ktime_get(),
 				  error->i915->gt.last_init_time);
+	error->capture = jiffies;
 
 	capture_params(error);
 	capture_gen_state(error);
@@ -1755,6 +1785,8 @@ static int capture(void *data)
 	error->overlay = intel_overlay_capture_error_state(error->i915);
 	error->display = intel_display_capture_error_state(error->i915);
 
+	error->epoch = capture_find_epoch(error);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h
new file mode 100644
index 000000000000..dac0f8c4c1cf
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gpu_error.h
@@ -0,0 +1,366 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright � 2008-2018 Intel Corporation
+ */
+
+#ifndef _I915_GPU_ERROR_H_
+#define _I915_GPU_ERROR_H_
+
+#include <linux/kref.h>
+#include <linux/ktime.h>
+#include <linux/sched.h>
+
+#include <drm/drm_mm.h>
+
+#include "intel_device_info.h"
+#include "intel_ringbuffer.h"
+#include "intel_uc_fw.h"
+
+#include "i915_gem.h"
+#include "i915_gem_gtt.h"
+#include "i915_params.h"
+#include "i915_scheduler.h"
+
+struct drm_i915_private;
+struct intel_overlay_error_state;
+struct intel_display_error_state;
+
+struct i915_gpu_state {
+	struct kref ref;
+	ktime_t time;
+	ktime_t boottime;
+	ktime_t uptime;
+	unsigned long capture;
+	unsigned long epoch;
+
+	struct drm_i915_private *i915;
+
+	char error_msg[128];
+	bool simulated;
+	bool awake;
+	bool wakelock;
+	bool suspended;
+	int iommu;
+	u32 reset_count;
+	u32 suspend_count;
+	struct intel_device_info device_info;
+	struct intel_driver_caps driver_caps;
+	struct i915_params params;
+
+	struct i915_error_uc {
+		struct intel_uc_fw guc_fw;
+		struct intel_uc_fw huc_fw;
+		struct drm_i915_error_object *guc_log;
+	} uc;
+
+	/* Generic register state */
+	u32 eir;
+	u32 pgtbl_er;
+	u32 ier;
+	u32 gtier[4], ngtier;
+	u32 ccid;
+	u32 derrmr;
+	u32 forcewake;
+	u32 error; /* gen6+ */
+	u32 err_int; /* gen7 */
+	u32 fault_data0; /* gen8, gen9 */
+	u32 fault_data1; /* gen8, gen9 */
+	u32 done_reg;
+	u32 gac_eco;
+	u32 gam_ecochk;
+	u32 gab_ctl;
+	u32 gfx_mode;
+
+	u32 nfence;
+	u64 fence[I915_MAX_NUM_FENCES];
+	struct intel_overlay_error_state *overlay;
+	struct intel_display_error_state *display;
+
+	struct drm_i915_error_engine {
+		int engine_id;
+		/* Software tracked state */
+		bool idle;
+		bool waiting;
+		int num_waiters;
+		unsigned long hangcheck_timestamp;
+		bool hangcheck_stalled;
+		enum intel_engine_hangcheck_action hangcheck_action;
+		struct i915_address_space *vm;
+		int num_requests;
+		u32 reset_count;
+
+		/* position of active request inside the ring */
+		u32 rq_head, rq_post, rq_tail;
+
+		/* our own tracking of ring head and tail */
+		u32 cpu_ring_head;
+		u32 cpu_ring_tail;
+
+		u32 last_seqno;
+
+		/* Register state */
+		u32 start;
+		u32 tail;
+		u32 head;
+		u32 ctl;
+		u32 mode;
+		u32 hws;
+		u32 ipeir;
+		u32 ipehr;
+		u32 bbstate;
+		u32 instpm;
+		u32 instps;
+		u32 seqno;
+		u64 bbaddr;
+		u64 acthd;
+		u32 fault_reg;
+		u64 faddr;
+		u32 rc_psmi; /* sleep state */
+		u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
+		struct intel_instdone instdone;
+
+		struct drm_i915_error_context {
+			char comm[TASK_COMM_LEN];
+			pid_t pid;
+			u32 handle;
+			u32 hw_id;
+			int ban_score;
+			int active;
+			int guilty;
+			bool bannable;
+			struct i915_sched_attr sched_attr;
+		} context;
+
+		struct drm_i915_error_object {
+			u64 gtt_offset;
+			u64 gtt_size;
+			int page_count;
+			int unused;
+			u32 *pages[0];
+		} *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
+
+		struct drm_i915_error_object **user_bo;
+		long user_bo_count;
+
+		struct drm_i915_error_object *wa_ctx;
+		struct drm_i915_error_object *default_state;
+
+		struct drm_i915_error_request {
+			long jiffies;
+			pid_t pid;
+			u32 context;
+			int ban_score;
+			u32 seqno;
+			u32 start;
+			u32 head;
+			u32 tail;
+			struct i915_sched_attr sched_attr;
+		} *requests, execlist[EXECLIST_MAX_PORTS];
+		unsigned int num_ports;
+
+		struct drm_i915_error_waiter {
+			char comm[TASK_COMM_LEN];
+			pid_t pid;
+			u32 seqno;
+		} *waiters;
+
+		struct {
+			u32 gfx_mode;
+			union {
+				u64 pdp[4];
+				u32 pp_dir_base;
+			};
+		} vm_info;
+	} engine[I915_NUM_ENGINES];
+
+	struct drm_i915_error_buffer {
+		u32 size;
+		u32 name;
+		u32 rseqno[I915_NUM_ENGINES], wseqno;
+		u64 gtt_offset;
+		u32 read_domains;
+		u32 write_domain;
+		s32 fence_reg:I915_MAX_NUM_FENCE_BITS;
+		u32 tiling:2;
+		u32 dirty:1;
+		u32 purgeable:1;
+		u32 userptr:1;
+		s32 engine:4;
+		u32 cache_level:3;
+	} *active_bo[I915_NUM_ENGINES], *pinned_bo;
+	u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count;
+	struct i915_address_space *active_vm[I915_NUM_ENGINES];
+};
+
+struct i915_gpu_error {
+	/* For hangcheck timer */
+#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
+#define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)
+
+	struct delayed_work hangcheck_work;
+
+	/* For reset and error_state handling. */
+	spinlock_t lock;
+	/* Protected by the above dev->gpu_error.lock. */
+	struct i915_gpu_state *first_error;
+
+	atomic_t pending_fb_pin;
+
+	unsigned long missed_irq_rings;
+
+	/**
+	 * State variable controlling the reset flow and count
+	 *
+	 * This is a counter which gets incremented when reset is triggered,
+	 *
+	 * Before the reset commences, the I915_RESET_BACKOFF bit is set
+	 * meaning that any waiters holding onto the struct_mutex should
+	 * relinquish the lock immediately in order for the reset to start.
+	 *
+	 * If reset is not completed successfully, the I915_WEDGE bit is
+	 * set meaning that hardware is terminally sour and there is no
+	 * recovery. All waiters on the reset_queue will be woken when
+	 * that happens.
+	 *
+	 * This counter is used by the wait_seqno code to notice that reset
+	 * event happened and it needs to restart the entire ioctl (since most
+	 * likely the seqno it waited for won't ever signal anytime soon).
+	 *
+	 * This is important for lock-free wait paths, where no contended lock
+	 * naturally enforces the correct ordering between the bail-out of the
+	 * waiter and the gpu reset work code.
+	 */
+	unsigned long reset_count;
+
+	/**
+	 * flags: Control various stages of the GPU reset
+	 *
+	 * #I915_RESET_BACKOFF - When we start a reset, we want to stop any
+	 * other users acquiring the struct_mutex. To do this we set the
+	 * #I915_RESET_BACKOFF bit in the error flags when we detect a reset
+	 * and then check for that bit before acquiring the struct_mutex (in
+	 * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a
+	 * secondary role in preventing two concurrent global reset attempts.
+	 *
+	 * #I915_RESET_HANDOFF - To perform the actual GPU reset, we need the
+	 * struct_mutex. We try to acquire the struct_mutex in the reset worker,
+	 * but it may be held by some long running waiter (that we cannot
+	 * interrupt without causing trouble). Once we are ready to do the GPU
+	 * reset, we set the I915_RESET_HANDOFF bit and wakeup any waiters. If
+	 * they already hold the struct_mutex and want to participate they can
+	 * inspect the bit and do the reset directly, otherwise the worker
+	 * waits for the struct_mutex.
+	 *
+	 * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to
+	 * acquire the struct_mutex to reset an engine, we need an explicit
+	 * flag to prevent two concurrent reset attempts in the same engine.
+	 * As the number of engines continues to grow, allocate the flags from
+	 * the most significant bits.
+	 *
+	 * #I915_WEDGED - If reset fails and we can no longer use the GPU,
+	 * we set the #I915_WEDGED bit. Prior to command submission, e.g.
+	 * i915_request_alloc(), this bit is checked and the sequence
+	 * aborted (with -EIO reported to userspace) if set.
+	 */
+	unsigned long flags;
+#define I915_RESET_BACKOFF	0
+#define I915_RESET_HANDOFF	1
+#define I915_RESET_MODESET	2
+#define I915_WEDGED		(BITS_PER_LONG - 1)
+#define I915_RESET_ENGINE	(I915_WEDGED - I915_NUM_ENGINES)
+
+	/** Number of times an engine has been reset */
+	u32 reset_engine_count[I915_NUM_ENGINES];
+
+	/** Set of stalled engines with guilty requests, in the current reset */
+	u32 stalled_mask;
+
+	/** Reason for the current *global* reset */
+	const char *reason;
+
+	/**
+	 * Waitqueue to signal when a hang is detected. Used to for waiters
+	 * to release the struct_mutex for the reset to procede.
+	 */
+	wait_queue_head_t wait_queue;
+
+	/**
+	 * Waitqueue to signal when the reset has completed. Used by clients
+	 * that wait for dev_priv->mm.wedged to settle.
+	 */
+	wait_queue_head_t reset_queue;
+
+	/* For missed irq/seqno simulation. */
+	unsigned long test_irq_rings;
+};
+
+struct drm_i915_error_state_buf {
+	struct drm_i915_private *i915;
+	unsigned int bytes;
+	unsigned int size;
+	int err;
+	u8 *buf;
+	loff_t start;
+	loff_t pos;
+};
+
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+
+__printf(2, 3)
+void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
+int i915_error_state_to_str(struct drm_i915_error_state_buf *estr,
+			    const struct i915_gpu_state *gpu);
+int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb,
+			      struct drm_i915_private *i915,
+			      size_t count, loff_t pos);
+
+static inline void
+i915_error_state_buf_release(struct drm_i915_error_state_buf *eb)
+{
+	kfree(eb->buf);
+}
+
+struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915);
+void i915_capture_error_state(struct drm_i915_private *dev_priv,
+			      u32 engine_mask,
+			      const char *error_msg);
+
+static inline struct i915_gpu_state *
+i915_gpu_state_get(struct i915_gpu_state *gpu)
+{
+	kref_get(&gpu->ref);
+	return gpu;
+}
+
+void __i915_gpu_state_free(struct kref *kref);
+static inline void i915_gpu_state_put(struct i915_gpu_state *gpu)
+{
+	if (gpu)
+		kref_put(&gpu->ref, __i915_gpu_state_free);
+}
+
+struct i915_gpu_state *i915_first_error_state(struct drm_i915_private *i915);
+void i915_reset_error_state(struct drm_i915_private *i915);
+
+#else
+
+static inline void i915_capture_error_state(struct drm_i915_private *dev_priv,
+					    u32 engine_mask,
+					    const char *error_msg)
+{
+}
+
+static inline struct i915_gpu_state *
+i915_first_error_state(struct drm_i915_private *i915)
+{
+	return NULL;
+}
+
+static inline void i915_reset_error_state(struct drm_i915_private *i915)
+{
+}
+
+#endif /* IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) */
+
+#endif /* _I915_GPU_ERROR_H_ */
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 633c18785c1e..f9bc3aaa90d0 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -243,6 +243,41 @@ void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
 	spin_unlock_irq(&dev_priv->irq_lock);
 }
 
+static u32
+gen11_gt_engine_identity(struct drm_i915_private * const i915,
+			 const unsigned int bank, const unsigned int bit);
+
+bool gen11_reset_one_iir(struct drm_i915_private * const i915,
+			 const unsigned int bank,
+			 const unsigned int bit)
+{
+	void __iomem * const regs = i915->regs;
+	u32 dw;
+
+	lockdep_assert_held(&i915->irq_lock);
+
+	dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
+	if (dw & BIT(bit)) {
+		/*
+		 * According to the BSpec, DW_IIR bits cannot be cleared without
+		 * first servicing the Selector & Shared IIR registers.
+		 */
+		gen11_gt_engine_identity(i915, bank, bit);
+
+		/*
+		 * We locked GT INT DW by reading it. If we want to (try
+		 * to) recover from this succesfully, we need to clear
+		 * our bit, otherwise we are locking the register for
+		 * everybody.
+		 */
+		raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
+
+		return true;
+	}
+
+	return false;
+}
+
 /**
  * ilk_update_display_irq - update DEIMR
  * @dev_priv: driver private
@@ -308,17 +343,29 @@ void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 
 static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
 {
+	WARN_ON_ONCE(INTEL_GEN(dev_priv) >= 11);
+
 	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
 }
 
 static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
 {
-	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
+	if (INTEL_GEN(dev_priv) >= 11)
+		return GEN11_GPM_WGBOXPERF_INTR_MASK;
+	else if (INTEL_GEN(dev_priv) >= 8)
+		return GEN8_GT_IMR(2);
+	else
+		return GEN6_PMIMR;
 }
 
 static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
 {
-	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
+	if (INTEL_GEN(dev_priv) >= 11)
+		return GEN11_GPM_WGBOXPERF_INTR_ENABLE;
+	else if (INTEL_GEN(dev_priv) >= 8)
+		return GEN8_GT_IER(2);
+	else
+		return GEN6_PMIER;
 }
 
 /**
@@ -400,6 +447,18 @@ static void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_m
 	/* though a barrier is missing here, but don't really need a one */
 }
 
+void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv)
+{
+	spin_lock_irq(&dev_priv->irq_lock);
+
+	while (gen11_reset_one_iir(dev_priv, 0, GEN11_GTPM))
+		;
+
+	dev_priv->gt_pm.rps.pm_iir = 0;
+
+	spin_unlock_irq(&dev_priv->irq_lock);
+}
+
 void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
 {
 	spin_lock_irq(&dev_priv->irq_lock);
@@ -415,12 +474,14 @@ void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
 	if (READ_ONCE(rps->interrupts_enabled))
 		return;
 
-	if (WARN_ON_ONCE(IS_GEN11(dev_priv)))
-		return;
-
 	spin_lock_irq(&dev_priv->irq_lock);
 	WARN_ON_ONCE(rps->pm_iir);
-	WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
+
+	if (INTEL_GEN(dev_priv) >= 11)
+		WARN_ON_ONCE(gen11_reset_one_iir(dev_priv, 0, GEN11_GTPM));
+	else
+		WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
+
 	rps->interrupts_enabled = true;
 	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 
@@ -434,9 +495,6 @@ void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
 	if (!READ_ONCE(rps->interrupts_enabled))
 		return;
 
-	if (WARN_ON_ONCE(IS_GEN11(dev_priv)))
-		return;
-
 	spin_lock_irq(&dev_priv->irq_lock);
 	rps->interrupts_enabled = false;
 
@@ -453,7 +511,10 @@ void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
 	 * state of the worker can be discarded.
 	 */
 	cancel_work_sync(&rps->work);
-	gen6_reset_rps_interrupts(dev_priv);
+	if (INTEL_GEN(dev_priv) >= 11)
+		gen11_reset_rps_interrupts(dev_priv);
+	else
+		gen6_reset_rps_interrupts(dev_priv);
 }
 
 void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv)
@@ -1399,19 +1460,18 @@ static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
 }
 
 static void
-gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
+gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
 {
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	bool tasklet = false;
 
-	if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) {
-		if (READ_ONCE(engine->execlists.active)) {
-			__set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
-			tasklet = true;
-		}
+	if (iir & GT_CONTEXT_SWITCH_INTERRUPT) {
+		if (READ_ONCE(engine->execlists.active))
+			tasklet = !test_and_set_bit(ENGINE_IRQ_EXECLIST,
+						    &engine->irq_posted);
 	}
 
-	if (iir & (GT_RENDER_USER_INTERRUPT << test_shift)) {
+	if (iir & GT_RENDER_USER_INTERRUPT) {
 		notify_ring(engine);
 		tasklet |= USES_GUC_SUBMISSION(engine->i915);
 	}
@@ -1466,21 +1526,21 @@ static void gen8_gt_irq_handler(struct drm_i915_private *i915,
 {
 	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
 		gen8_cs_irq_handler(i915->engine[RCS],
-				    gt_iir[0], GEN8_RCS_IRQ_SHIFT);
+				    gt_iir[0] >> GEN8_RCS_IRQ_SHIFT);
 		gen8_cs_irq_handler(i915->engine[BCS],
-				    gt_iir[0], GEN8_BCS_IRQ_SHIFT);
+				    gt_iir[0] >> GEN8_BCS_IRQ_SHIFT);
 	}
 
 	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
 		gen8_cs_irq_handler(i915->engine[VCS],
-				    gt_iir[1], GEN8_VCS1_IRQ_SHIFT);
+				    gt_iir[1] >> GEN8_VCS1_IRQ_SHIFT);
 		gen8_cs_irq_handler(i915->engine[VCS2],
-				    gt_iir[1], GEN8_VCS2_IRQ_SHIFT);
+				    gt_iir[1] >> GEN8_VCS2_IRQ_SHIFT);
 	}
 
 	if (master_ctl & GEN8_GT_VECS_IRQ) {
 		gen8_cs_irq_handler(i915->engine[VECS],
-				    gt_iir[3], GEN8_VECS_IRQ_SHIFT);
+				    gt_iir[3] >> GEN8_VECS_IRQ_SHIFT);
 	}
 
 	if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
@@ -1627,7 +1687,7 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 	int head, tail;
 
 	spin_lock(&pipe_crc->lock);
-	if (pipe_crc->source) {
+	if (pipe_crc->source && !crtc->base.crc.opened) {
 		if (!pipe_crc->entries) {
 			spin_unlock(&pipe_crc->lock);
 			DRM_DEBUG_KMS("spurious interrupt\n");
@@ -1667,7 +1727,7 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 		 * On GEN8+ sometimes the second CRC is bonkers as well, so
 		 * don't trust that one either.
 		 */
-		if (pipe_crc->skipped == 0 ||
+		if (pipe_crc->skipped <= 0 ||
 		    (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
 			pipe_crc->skipped++;
 			spin_unlock(&pipe_crc->lock);
@@ -1766,37 +1826,8 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
 
 static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
 {
-	if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT) {
-		/* Sample the log buffer flush related bits & clear them out now
-		 * itself from the message identity register to minimize the
-		 * probability of losing a flush interrupt, when there are back
-		 * to back flush interrupts.
-		 * There can be a new flush interrupt, for different log buffer
-		 * type (like for ISR), whilst Host is handling one (for DPC).
-		 * Since same bit is used in message register for ISR & DPC, it
-		 * could happen that GuC sets the bit for 2nd interrupt but Host
-		 * clears out the bit on handling the 1st interrupt.
-		 */
-		u32 msg, flush;
-
-		msg = I915_READ(SOFT_SCRATCH(15));
-		flush = msg & (INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED |
-			       INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER);
-		if (flush) {
-			/* Clear the message bits that are handled */
-			I915_WRITE(SOFT_SCRATCH(15), msg & ~flush);
-
-			/* Handle flush interrupt in bottom half */
-			queue_work(dev_priv->guc.log.runtime.flush_wq,
-				   &dev_priv->guc.log.runtime.flush_work);
-
-			dev_priv->guc.log.flush_interrupt_count++;
-		} else {
-			/* Not clearing of unhandled event bits won't result in
-			 * re-triggering of the interrupt.
-			 */
-		}
-	}
+	if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT)
+		intel_guc_to_host_event_handler(&dev_priv->guc);
 }
 
 static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
@@ -2433,6 +2464,13 @@ static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
 	if (de_iir & DE_ERR_INT_IVB)
 		ivb_err_int_handler(dev_priv);
 
+	if (de_iir & DE_EDP_PSR_INT_HSW) {
+		u32 psr_iir = I915_READ(EDP_PSR_IIR);
+
+		intel_psr_irq_handler(dev_priv, psr_iir);
+		I915_WRITE(EDP_PSR_IIR, psr_iir);
+	}
+
 	if (de_iir & DE_AUX_CHANNEL_A_IVB)
 		dp_aux_irq_handler(dev_priv);
 
@@ -2562,11 +2600,25 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 	if (master_ctl & GEN8_DE_MISC_IRQ) {
 		iir = I915_READ(GEN8_DE_MISC_IIR);
 		if (iir) {
+			bool found = false;
+
 			I915_WRITE(GEN8_DE_MISC_IIR, iir);
 			ret = IRQ_HANDLED;
-			if (iir & GEN8_DE_MISC_GSE)
+
+			if (iir & GEN8_DE_MISC_GSE) {
 				intel_opregion_asle_intr(dev_priv);
-			else
+				found = true;
+			}
+
+			if (iir & GEN8_DE_EDP_PSR) {
+				u32 psr_iir = I915_READ(EDP_PSR_IIR);
+
+				intel_psr_irq_handler(dev_priv, psr_iir);
+				I915_WRITE(EDP_PSR_IIR, psr_iir);
+				found = true;
+			}
+
+			if (!found)
 				DRM_ERROR("Unexpected DE Misc interrupt\n");
 		}
 		else
@@ -2762,58 +2814,16 @@ static void __fini_wedge(struct wedge_me *w)
 	     (W)->i915;							\
 	     __fini_wedge((W)))
 
-static __always_inline void
-gen11_cs_irq_handler(struct intel_engine_cs * const engine, const u32 iir)
-{
-	gen8_cs_irq_handler(engine, iir, 0);
-}
-
-static void
-gen11_gt_engine_irq_handler(struct drm_i915_private * const i915,
-			    const unsigned int bank,
-			    const unsigned int engine_n,
-			    const u16 iir)
-{
-	struct intel_engine_cs ** const engine = i915->engine;
-
-	switch (bank) {
-	case 0:
-		switch (engine_n) {
-
-		case GEN11_RCS0:
-			return gen11_cs_irq_handler(engine[RCS], iir);
-
-		case GEN11_BCS:
-			return gen11_cs_irq_handler(engine[BCS], iir);
-		}
-	case 1:
-		switch (engine_n) {
-
-		case GEN11_VCS(0):
-			return gen11_cs_irq_handler(engine[_VCS(0)], iir);
-		case GEN11_VCS(1):
-			return gen11_cs_irq_handler(engine[_VCS(1)], iir);
-		case GEN11_VCS(2):
-			return gen11_cs_irq_handler(engine[_VCS(2)], iir);
-		case GEN11_VCS(3):
-			return gen11_cs_irq_handler(engine[_VCS(3)], iir);
-
-		case GEN11_VECS(0):
-			return gen11_cs_irq_handler(engine[_VECS(0)], iir);
-		case GEN11_VECS(1):
-			return gen11_cs_irq_handler(engine[_VECS(1)], iir);
-		}
-	}
-}
-
 static u32
-gen11_gt_engine_intr(struct drm_i915_private * const i915,
-		     const unsigned int bank, const unsigned int bit)
+gen11_gt_engine_identity(struct drm_i915_private * const i915,
+			 const unsigned int bank, const unsigned int bit)
 {
 	void __iomem * const regs = i915->regs;
 	u32 timeout_ts;
 	u32 ident;
 
+	lockdep_assert_held(&i915->irq_lock);
+
 	raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
 
 	/*
@@ -2835,42 +2845,101 @@ gen11_gt_engine_intr(struct drm_i915_private * const i915,
 	raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
 		      GEN11_INTR_DATA_VALID);
 
-	return ident & GEN11_INTR_ENGINE_MASK;
+	return ident;
 }
 
 static void
-gen11_gt_irq_handler(struct drm_i915_private * const i915,
-		     const u32 master_ctl)
+gen11_other_irq_handler(struct drm_i915_private * const i915,
+			const u8 instance, const u16 iir)
+{
+	if (instance == OTHER_GTPM_INSTANCE)
+		return gen6_rps_irq_handler(i915, iir);
+
+	WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
+		  instance, iir);
+}
+
+static void
+gen11_engine_irq_handler(struct drm_i915_private * const i915,
+			 const u8 class, const u8 instance, const u16 iir)
+{
+	struct intel_engine_cs *engine;
+
+	if (instance <= MAX_ENGINE_INSTANCE)
+		engine = i915->engine_class[class][instance];
+	else
+		engine = NULL;
+
+	if (likely(engine))
+		return gen8_cs_irq_handler(engine, iir);
+
+	WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
+		  class, instance);
+}
+
+static void
+gen11_gt_identity_handler(struct drm_i915_private * const i915,
+			  const u32 identity)
+{
+	const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
+	const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
+	const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
+
+	if (unlikely(!intr))
+		return;
+
+	if (class <= COPY_ENGINE_CLASS)
+		return gen11_engine_irq_handler(i915, class, instance, intr);
+
+	if (class == OTHER_CLASS)
+		return gen11_other_irq_handler(i915, instance, intr);
+
+	WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
+		  class, instance, intr);
+}
+
+static void
+gen11_gt_bank_handler(struct drm_i915_private * const i915,
+		      const unsigned int bank)
 {
 	void __iomem * const regs = i915->regs;
-	unsigned int bank;
+	unsigned long intr_dw;
+	unsigned int bit;
 
-	for (bank = 0; bank < 2; bank++) {
-		unsigned long intr_dw;
-		unsigned int bit;
+	lockdep_assert_held(&i915->irq_lock);
 
-		if (!(master_ctl & GEN11_GT_DW_IRQ(bank)))
-			continue;
+	intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 
-		intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
+	if (unlikely(!intr_dw)) {
+		DRM_ERROR("GT_INTR_DW%u blank!\n", bank);
+		return;
+	}
 
-		if (unlikely(!intr_dw)) {
-			DRM_ERROR("GT_INTR_DW%u blank!\n", bank);
-			continue;
-		}
+	for_each_set_bit(bit, &intr_dw, 32) {
+		const u32 ident = gen11_gt_engine_identity(i915,
+							   bank, bit);
 
-		for_each_set_bit(bit, &intr_dw, 32) {
-			const u16 iir = gen11_gt_engine_intr(i915, bank, bit);
+		gen11_gt_identity_handler(i915, ident);
+	}
 
-			if (unlikely(!iir))
-				continue;
+	/* Clear must be after shared has been served for engine */
+	raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
+}
 
-			gen11_gt_engine_irq_handler(i915, bank, bit, iir);
-		}
+static void
+gen11_gt_irq_handler(struct drm_i915_private * const i915,
+		     const u32 master_ctl)
+{
+	unsigned int bank;
 
-		/* Clear must be after shared has been served for engine */
-		raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
+	spin_lock(&i915->irq_lock);
+
+	for (bank = 0; bank < 2; bank++) {
+		if (master_ctl & GEN11_GT_DW_IRQ(bank))
+			gen11_gt_bank_handler(i915, bank);
 	}
+
+	spin_unlock(&i915->irq_lock);
 }
 
 static irqreturn_t gen11_irq_handler(int irq, void *arg)
@@ -2912,15 +2981,11 @@ static irqreturn_t gen11_irq_handler(int irq, void *arg)
 	return IRQ_HANDLED;
 }
 
-/**
- * i915_reset_device - do process context error handling work
- * @dev_priv: i915 device private
- *
- * Fire an error uevent so userspace can see that a hang or error
- * was detected.
- */
-static void i915_reset_device(struct drm_i915_private *dev_priv)
+static void i915_reset_device(struct drm_i915_private *dev_priv,
+			      u32 engine_mask,
+			      const char *reason)
 {
+	struct i915_gpu_error *error = &dev_priv->gpu_error;
 	struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
 	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
 	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
@@ -2936,29 +3001,35 @@ static void i915_reset_device(struct drm_i915_private *dev_priv)
 	i915_wedge_on_timeout(&w, dev_priv, 5*HZ) {
 		intel_prepare_reset(dev_priv);
 
+		error->reason = reason;
+		error->stalled_mask = engine_mask;
+
 		/* Signal that locked waiters should reset the GPU */
-		set_bit(I915_RESET_HANDOFF, &dev_priv->gpu_error.flags);
-		wake_up_all(&dev_priv->gpu_error.wait_queue);
+		smp_mb__before_atomic();
+		set_bit(I915_RESET_HANDOFF, &error->flags);
+		wake_up_all(&error->wait_queue);
 
 		/* Wait for anyone holding the lock to wakeup, without
 		 * blocking indefinitely on struct_mutex.
 		 */
 		do {
 			if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
-				i915_reset(dev_priv, 0);
+				i915_reset(dev_priv, engine_mask, reason);
 				mutex_unlock(&dev_priv->drm.struct_mutex);
 			}
-		} while (wait_on_bit_timeout(&dev_priv->gpu_error.flags,
+		} while (wait_on_bit_timeout(&error->flags,
 					     I915_RESET_HANDOFF,
 					     TASK_UNINTERRUPTIBLE,
 					     1));
 
+		error->stalled_mask = 0;
+		error->reason = NULL;
+
 		intel_finish_reset(dev_priv);
 	}
 
-	if (!test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
-		kobject_uevent_env(kobj,
-				   KOBJ_CHANGE, reset_done_event);
+	if (!test_bit(I915_WEDGED, &error->flags))
+		kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event);
 }
 
 static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
@@ -2990,6 +3061,7 @@ static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
  * i915_handle_error - handle a gpu error
  * @dev_priv: i915 device private
  * @engine_mask: mask representing engines that are hung
+ * @flags: control flags
  * @fmt: Error message format string
  *
  * Do some basic checking of register state at error time and
@@ -3000,16 +3072,23 @@ static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
  */
 void i915_handle_error(struct drm_i915_private *dev_priv,
 		       u32 engine_mask,
+		       unsigned long flags,
 		       const char *fmt, ...)
 {
 	struct intel_engine_cs *engine;
 	unsigned int tmp;
-	va_list args;
 	char error_msg[80];
+	char *msg = NULL;
 
-	va_start(args, fmt);
-	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
-	va_end(args);
+	if (fmt) {
+		va_list args;
+
+		va_start(args, fmt);
+		vscnprintf(error_msg, sizeof(error_msg), fmt, args);
+		va_end(args);
+
+		msg = error_msg;
+	}
 
 	/*
 	 * In most cases it's guaranteed that we get here with an RPM
@@ -3020,8 +3099,12 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
 	 */
 	intel_runtime_pm_get(dev_priv);
 
-	i915_capture_error_state(dev_priv, engine_mask, error_msg);
-	i915_clear_error_registers(dev_priv);
+	engine_mask &= INTEL_INFO(dev_priv)->ring_mask;
+
+	if (flags & I915_ERROR_CAPTURE) {
+		i915_capture_error_state(dev_priv, engine_mask, msg);
+		i915_clear_error_registers(dev_priv);
+	}
 
 	/*
 	 * Try engine reset when available. We fall back to full reset if
@@ -3034,7 +3117,7 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
 					     &dev_priv->gpu_error.flags))
 				continue;
 
-			if (i915_reset_engine(engine, 0) == 0)
+			if (i915_reset_engine(engine, msg) == 0)
 				engine_mask &= ~intel_engine_flag(engine);
 
 			clear_bit(I915_RESET_ENGINE + engine->id,
@@ -3064,7 +3147,7 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
 				    TASK_UNINTERRUPTIBLE);
 	}
 
-	i915_reset_device(dev_priv);
+	i915_reset_device(dev_priv, engine_mask, msg);
 
 	for_each_engine(engine, dev_priv, tmp) {
 		clear_bit(I915_RESET_ENGINE + engine->id,
@@ -3286,6 +3369,11 @@ static void ironlake_irq_reset(struct drm_device *dev)
 	if (IS_GEN7(dev_priv))
 		I915_WRITE(GEN7_ERR_INT, 0xffffffff);
 
+	if (IS_HASWELL(dev_priv)) {
+		I915_WRITE(EDP_PSR_IMR, 0xffffffff);
+		I915_WRITE(EDP_PSR_IIR, 0xffffffff);
+	}
+
 	gen5_gt_irq_reset(dev_priv);
 
 	ibx_irq_reset(dev_priv);
@@ -3324,6 +3412,9 @@ static void gen8_irq_reset(struct drm_device *dev)
 
 	gen8_gt_irq_reset(dev_priv);
 
+	I915_WRITE(EDP_PSR_IMR, 0xffffffff);
+	I915_WRITE(EDP_PSR_IIR, 0xffffffff);
+
 	for_each_pipe(dev_priv, pipe)
 		if (intel_display_power_is_enabled(dev_priv,
 						   POWER_DOMAIN_PIPE(pipe)))
@@ -3349,6 +3440,9 @@ static void gen11_gt_irq_reset(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN11_VCS0_VCS1_INTR_MASK,	~0);
 	I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK,	~0);
 	I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK,	~0);
+
+	I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
+	I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 }
 
 static void gen11_irq_reset(struct drm_device *dev)
@@ -3697,6 +3791,12 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
 			      DE_DP_A_HOTPLUG);
 	}
 
+	if (IS_HASWELL(dev_priv)) {
+		gen3_assert_iir_is_zero(dev_priv, EDP_PSR_IIR);
+		intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
+		display_mask |= DE_EDP_PSR_INT_HSW;
+	}
+
 	dev_priv->irq_mask = ~display_mask;
 
 	ibx_irq_pre_postinstall(dev);
@@ -3807,7 +3907,7 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
 	uint32_t de_pipe_enables;
 	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
 	u32 de_port_enables;
-	u32 de_misc_masked = GEN8_DE_MISC_GSE;
+	u32 de_misc_masked = GEN8_DE_MISC_GSE | GEN8_DE_EDP_PSR;
 	enum pipe pipe;
 
 	if (INTEL_GEN(dev_priv) >= 9) {
@@ -3832,6 +3932,9 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
 	else if (IS_BROADWELL(dev_priv))
 		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
 
+	gen3_assert_iir_is_zero(dev_priv, EDP_PSR_IIR);
+	intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
+
 	for_each_pipe(dev_priv, pipe) {
 		dev_priv->de_irq_mask[pipe] = ~de_pipe_masked;
 
@@ -3887,7 +3990,14 @@ static void gen11_gt_irq_postinstall(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK,	~(irqs | irqs << 16));
 	I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK,	~(irqs | irqs << 16));
 
-	dev_priv->pm_imr = 0xffffffff; /* TODO */
+	/*
+	 * RPS interrupts will get enabled/disabled on demand when RPS itself
+	 * is enabled/disabled.
+	 */
+	dev_priv->pm_ier = 0x0;
+	dev_priv->pm_imr = ~dev_priv->pm_ier;
+	I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
+	I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 }
 
 static int gen11_irq_postinstall(struct drm_device *dev)
diff --git a/drivers/gpu/drm/i915/i915_oa_icl.c b/drivers/gpu/drm/i915/i915_oa_icl.c
new file mode 100644
index 000000000000..a5667926e3de
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_icl.c
@@ -0,0 +1,118 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_icl.h"
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x0000ffff },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x0000ffff },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x0000ffff },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0xd04), 0x00000200 },
+	{ _MMIO(0x9840), 0x00000000 },
+	{ _MMIO(0x9884), 0x00000000 },
+	{ _MMIO(0x9888), 0x10060000 },
+	{ _MMIO(0x9888), 0x22060000 },
+	{ _MMIO(0x9888), 0x16060000 },
+	{ _MMIO(0x9888), 0x24060000 },
+	{ _MMIO(0x9888), 0x18060000 },
+	{ _MMIO(0x9888), 0x1a060000 },
+	{ _MMIO(0x9888), 0x12060000 },
+	{ _MMIO(0x9888), 0x14060000 },
+	{ _MMIO(0x9888), 0x10060000 },
+	{ _MMIO(0x9888), 0x22060000 },
+	{ _MMIO(0x9884), 0x00000003 },
+	{ _MMIO(0x9888), 0x16130000 },
+	{ _MMIO(0x9888), 0x24000001 },
+	{ _MMIO(0x9888), 0x0e130056 },
+	{ _MMIO(0x9888), 0x10130000 },
+	{ _MMIO(0x9888), 0x1a130000 },
+	{ _MMIO(0x9888), 0x541f0001 },
+	{ _MMIO(0x9888), 0x181f0000 },
+	{ _MMIO(0x9888), 0x4c1f0000 },
+	{ _MMIO(0x9888), 0x301f0000 },
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "1\n");
+}
+
+void
+i915_perf_load_test_config_icl(struct drm_i915_private *dev_priv)
+{
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
+		"a291665e-244b-4b76-9b9a-01de9d3c8068",
+		sizeof(dev_priv->perf.oa.test_config.uuid));
+	dev_priv->perf.oa.test_config.id = 1;
+
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
+
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
+
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
+
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "a291665e-244b-4b76-9b9a-01de9d3c8068";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
+
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_icl.h b/drivers/gpu/drm/i915/i915_oa_icl.h
new file mode 100644
index 000000000000..ae1c24aafe4f
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_icl.h
@@ -0,0 +1,34 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_ICL_H__
+#define __I915_OA_ICL_H__
+
+extern void i915_perf_load_test_config_icl(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index 08108ce5be21..66ea3552c63e 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -164,6 +164,9 @@ i915_param_named_unsafe(guc_firmware_path, charp, 0400,
 i915_param_named_unsafe(huc_firmware_path, charp, 0400,
 	"HuC firmware path to use instead of the default one");
 
+i915_param_named_unsafe(dmc_firmware_path, charp, 0400,
+	"DMC firmware path to use instead of the default one");
+
 i915_param_named_unsafe(enable_dp_mst, bool, 0600,
 	"Enable multi-stream transport (MST) for new DisplayPort sinks. (default: true)");
 
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 430f5f9d0ff4..6684025b7af8 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -48,9 +48,10 @@ struct drm_printer;
 	param(int, enable_ips, 1) \
 	param(int, invert_brightness, 0) \
 	param(int, enable_guc, 0) \
-	param(int, guc_log_level, 0) \
+	param(int, guc_log_level, -1) \
 	param(char *, guc_firmware_path, NULL) \
 	param(char *, huc_firmware_path, NULL) \
+	param(char *, dmc_firmware_path, NULL) \
 	param(int, mmio_debug, 0) \
 	param(int, edp_vswing, 0) \
 	param(int, reset, 2) \
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 062e91b39085..4364922e935d 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -602,6 +602,7 @@ static const struct intel_device_info intel_icelake_11_info = {
 	PLATFORM(INTEL_ICELAKE),
 	.is_alpha_support = 1,
 	.has_resource_streamer = 0,
+	.ring_mask = RENDER_RING | BLT_RING | VEBOX_RING | BSD_RING | BSD3_RING,
 };
 
 #undef GEN
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index abaca6edeb71..019bd2d073ad 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -209,6 +209,7 @@
 #include "i915_oa_cflgt2.h"
 #include "i915_oa_cflgt3.h"
 #include "i915_oa_cnl.h"
+#include "i915_oa_icl.h"
 
 /* HW requires this to be a power of two, between 128k and 16M, though driver
  * is currently generally designed assuming the largest 16M size is used such
@@ -1042,7 +1043,7 @@ static int gen7_append_oa_reports(struct i915_perf_stream *stream,
 
 		I915_WRITE(GEN7_OASTATUS2,
 			   ((head & GEN7_OASTATUS2_HEAD_MASK) |
-			    OA_MEM_SELECT_GGTT));
+			    GEN7_OASTATUS2_MEM_SELECT_GGTT));
 		dev_priv->perf.oa.oa_buffer.head = head;
 
 		spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
@@ -1233,7 +1234,7 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
 		 *
 		 * NB: implied RCS engine...
 		 */
-		ring = engine->context_pin(engine, stream->ctx);
+		ring = intel_context_pin(stream->ctx, engine);
 		mutex_unlock(&dev_priv->drm.struct_mutex);
 		if (IS_ERR(ring))
 			return PTR_ERR(ring);
@@ -1245,7 +1246,7 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
 		 * with gen8+ and execlists
 		 */
 		dev_priv->perf.oa.specific_ctx_id =
-			i915_ggtt_offset(stream->ctx->engine[engine->id].state);
+			i915_ggtt_offset(to_intel_context(stream->ctx, engine)->state);
 	}
 
 	return 0;
@@ -1270,7 +1271,7 @@ static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
 		mutex_lock(&dev_priv->drm.struct_mutex);
 
 		dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID;
-		engine->context_unpin(engine, stream->ctx);
+		intel_context_unpin(stream->ctx, engine);
 
 		mutex_unlock(&dev_priv->drm.struct_mutex);
 	}
@@ -1332,7 +1333,8 @@ static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv)
 	/* Pre-DevBDW: OABUFFER must be set with counters off,
 	 * before OASTATUS1, but after OASTATUS2
 	 */
-	I915_WRITE(GEN7_OASTATUS2, gtt_offset | OA_MEM_SELECT_GGTT); /* head */
+	I915_WRITE(GEN7_OASTATUS2,
+		   gtt_offset | GEN7_OASTATUS2_MEM_SELECT_GGTT); /* head */
 	dev_priv->perf.oa.oa_buffer.head = gtt_offset;
 
 	I915_WRITE(GEN7_OABUFFER, gtt_offset);
@@ -1392,7 +1394,7 @@ static void gen8_init_oa_buffer(struct drm_i915_private *dev_priv)
 	 *  bit."
 	 */
 	I915_WRITE(GEN8_OABUFFER, gtt_offset |
-		   OABUFFER_SIZE_16M | OA_MEM_SELECT_GGTT);
+		   OABUFFER_SIZE_16M | GEN8_OABUFFER_MEM_SELECT_GGTT);
 	I915_WRITE(GEN8_OATAILPTR, gtt_offset & GEN8_OATAILPTR_MASK);
 
 	/* Mark that we need updated tail pointers to read from... */
@@ -1693,7 +1695,7 @@ static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_pr
 						 const struct i915_oa_config *oa_config)
 {
 	struct intel_engine_cs *engine = dev_priv->engine[RCS];
-	struct i915_gem_timeline *timeline;
+	struct i915_timeline *timeline;
 	struct i915_request *rq;
 	int ret;
 
@@ -1714,15 +1716,11 @@ static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_pr
 	/* Queue this switch after all other activity */
 	list_for_each_entry(timeline, &dev_priv->gt.timelines, link) {
 		struct i915_request *prev;
-		struct intel_timeline *tl;
 
-		tl = &timeline->engine[engine->id];
-		prev = i915_gem_active_raw(&tl->last_request,
+		prev = i915_gem_active_raw(&timeline->last_request,
 					   &dev_priv->drm.struct_mutex);
 		if (prev)
-			i915_sw_fence_await_sw_fence_gfp(&rq->submit,
-							 &prev->submit,
-							 GFP_KERNEL);
+			i915_request_await_dma_fence(rq, &prev->fence);
 	}
 
 	i915_request_add(rq);
@@ -1757,6 +1755,7 @@ static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_pr
 static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 				       const struct i915_oa_config *oa_config)
 {
+	struct intel_engine_cs *engine = dev_priv->engine[RCS];
 	struct i915_gem_context *ctx;
 	int ret;
 	unsigned int wait_flags = I915_WAIT_LOCKED;
@@ -1787,7 +1786,7 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 
 	/* Update all contexts now that we've stalled the submission. */
 	list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
-		struct intel_context *ce = &ctx->engine[RCS];
+		struct intel_context *ce = to_intel_context(ctx, engine);
 		u32 *regs;
 
 		/* OA settings will be set upon first use */
@@ -1840,7 +1839,7 @@ static int gen8_enable_metric_set(struct drm_i915_private *dev_priv,
 	 * be read back from automatically triggered reports, as part of the
 	 * RPT_ID field.
 	 */
-	if (IS_GEN9(dev_priv) || IS_GEN10(dev_priv)) {
+	if (IS_GEN(dev_priv, 9, 11)) {
 		I915_WRITE(GEN8_OA_DEBUG,
 			   _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
 					      GEN9_OA_DEBUG_INCLUDE_CLK_RATIO));
@@ -1870,7 +1869,6 @@ static void gen8_disable_metric_set(struct drm_i915_private *dev_priv)
 
 	I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) &
 				      ~GT_NOA_ENABLE));
-
 }
 
 static void gen10_disable_metric_set(struct drm_i915_private *dev_priv)
@@ -1885,6 +1883,13 @@ static void gen10_disable_metric_set(struct drm_i915_private *dev_priv)
 
 static void gen7_oa_enable(struct drm_i915_private *dev_priv)
 {
+	struct i915_gem_context *ctx =
+			dev_priv->perf.oa.exclusive_stream->ctx;
+	u32 ctx_id = dev_priv->perf.oa.specific_ctx_id;
+	bool periodic = dev_priv->perf.oa.periodic;
+	u32 period_exponent = dev_priv->perf.oa.period_exponent;
+	u32 report_format = dev_priv->perf.oa.oa_buffer.format;
+
 	/*
 	 * Reset buf pointers so we don't forward reports from before now.
 	 *
@@ -1896,25 +1901,14 @@ static void gen7_oa_enable(struct drm_i915_private *dev_priv)
 	 */
 	gen7_init_oa_buffer(dev_priv);
 
-	if (dev_priv->perf.oa.exclusive_stream->enabled) {
-		struct i915_gem_context *ctx =
-			dev_priv->perf.oa.exclusive_stream->ctx;
-		u32 ctx_id = dev_priv->perf.oa.specific_ctx_id;
-
-		bool periodic = dev_priv->perf.oa.periodic;
-		u32 period_exponent = dev_priv->perf.oa.period_exponent;
-		u32 report_format = dev_priv->perf.oa.oa_buffer.format;
-
-		I915_WRITE(GEN7_OACONTROL,
-			   (ctx_id & GEN7_OACONTROL_CTX_MASK) |
-			   (period_exponent <<
-			    GEN7_OACONTROL_TIMER_PERIOD_SHIFT) |
-			   (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) |
-			   (report_format << GEN7_OACONTROL_FORMAT_SHIFT) |
-			   (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) |
-			   GEN7_OACONTROL_ENABLE);
-	} else
-		I915_WRITE(GEN7_OACONTROL, 0);
+	I915_WRITE(GEN7_OACONTROL,
+		   (ctx_id & GEN7_OACONTROL_CTX_MASK) |
+		   (period_exponent <<
+		    GEN7_OACONTROL_TIMER_PERIOD_SHIFT) |
+		   (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) |
+		   (report_format << GEN7_OACONTROL_FORMAT_SHIFT) |
+		   (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) |
+		   GEN7_OACONTROL_ENABLE);
 }
 
 static void gen8_oa_enable(struct drm_i915_private *dev_priv)
@@ -1966,11 +1960,19 @@ static void i915_oa_stream_enable(struct i915_perf_stream *stream)
 static void gen7_oa_disable(struct drm_i915_private *dev_priv)
 {
 	I915_WRITE(GEN7_OACONTROL, 0);
+	if (intel_wait_for_register(dev_priv,
+				    GEN7_OACONTROL, GEN7_OACONTROL_ENABLE, 0,
+				    50))
+		DRM_ERROR("wait for OA to be disabled timed out\n");
 }
 
 static void gen8_oa_disable(struct drm_i915_private *dev_priv)
 {
 	I915_WRITE(GEN8_OACONTROL, 0);
+	if (intel_wait_for_register(dev_priv,
+				    GEN8_OACONTROL, GEN8_OA_COUNTER_ENABLE, 0,
+				    50))
+		DRM_ERROR("wait for OA to be disabled timed out\n");
 }
 
 /**
@@ -2099,13 +2101,17 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 
 	if (stream->ctx) {
 		ret = oa_get_render_ctx_id(stream);
-		if (ret)
+		if (ret) {
+			DRM_DEBUG("Invalid context id to filter with\n");
 			return ret;
+		}
 	}
 
 	ret = get_oa_config(dev_priv, props->metrics_set, &stream->oa_config);
-	if (ret)
+	if (ret) {
+		DRM_DEBUG("Invalid OA config id=%i\n", props->metrics_set);
 		goto err_config;
+	}
 
 	/* PRM - observability performance counters:
 	 *
@@ -2132,8 +2138,10 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 
 	ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv,
 						      stream->oa_config);
-	if (ret)
+	if (ret) {
+		DRM_DEBUG("Unable to enable metric set\n");
 		goto err_enable;
+	}
 
 	stream->ops = &i915_oa_stream_ops;
 
@@ -2745,7 +2753,8 @@ static int read_properties_unlocked(struct drm_i915_private *dev_priv,
 			props->ctx_handle = value;
 			break;
 		case DRM_I915_PERF_PROP_SAMPLE_OA:
-			props->sample_flags |= SAMPLE_OA_REPORT;
+			if (value)
+				props->sample_flags |= SAMPLE_OA_REPORT;
 			break;
 		case DRM_I915_PERF_PROP_OA_METRICS_SET:
 			if (value == 0) {
@@ -2935,6 +2944,8 @@ void i915_perf_register(struct drm_i915_private *dev_priv)
 			i915_perf_load_test_config_cflgt3(dev_priv);
 	} else if (IS_CANNONLAKE(dev_priv)) {
 		i915_perf_load_test_config_cnl(dev_priv);
+	} else if (IS_ICELAKE(dev_priv)) {
+		i915_perf_load_test_config_icl(dev_priv);
 	}
 
 	if (dev_priv->perf.oa.test_config.id == 0)
@@ -3292,6 +3303,8 @@ int i915_perf_add_config_ioctl(struct drm_device *dev, void *data,
 
 	mutex_unlock(&dev_priv->perf.metrics_lock);
 
+	DRM_DEBUG("Added config %s id=%i\n", oa_config->uuid, oa_config->id);
+
 	return oa_config->id;
 
 sysfs_err:
@@ -3348,6 +3361,9 @@ int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data,
 			   &oa_config->sysfs_metric);
 
 	idr_remove(&dev_priv->perf.metrics_idr, *arg);
+
+	DRM_DEBUG("Removed config %s id=%i\n", oa_config->uuid, oa_config->id);
+
 	put_oa_config(dev_priv, oa_config);
 
 config_err:
@@ -3467,7 +3483,7 @@ void i915_perf_init(struct drm_i915_private *dev_priv)
 
 				dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16);
 			}
-		} else if (IS_GEN10(dev_priv)) {
+		} else if (IS_GEN(dev_priv, 10, 11)) {
 			dev_priv->perf.oa.ops.is_valid_b_counter_reg =
 				gen7_is_valid_b_counter_addr;
 			dev_priv->perf.oa.ops.is_valid_mux_reg =
diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c
index f0519e31543a..dc87797db500 100644
--- a/drivers/gpu/drm/i915/i915_pmu.c
+++ b/drivers/gpu/drm/i915/i915_pmu.c
@@ -1,33 +1,12 @@
 /*
- * Copyright © 2017 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  *
+ * Copyright © 2017-2018 Intel Corporation
  */
 
-#include <linux/perf_event.h>
-#include <linux/pm_runtime.h>
-
-#include "i915_drv.h"
 #include "i915_pmu.h"
 #include "intel_ringbuffer.h"
+#include "i915_drv.h"
 
 /* Frequency for the sampling timer for events which need it. */
 #define FREQUENCY 200
diff --git a/drivers/gpu/drm/i915/i915_pmu.h b/drivers/gpu/drm/i915/i915_pmu.h
index aa1b1a987ea1..2ba735299f7c 100644
--- a/drivers/gpu/drm/i915/i915_pmu.h
+++ b/drivers/gpu/drm/i915/i915_pmu.h
@@ -1,29 +1,19 @@
 /*
- * Copyright © 2017 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  *
+ * Copyright © 2017-2018 Intel Corporation
  */
+
 #ifndef __I915_PMU_H__
 #define __I915_PMU_H__
 
+#include <linux/hrtimer.h>
+#include <linux/perf_event.h>
+#include <linux/spinlock_types.h>
+#include <drm/i915_drm.h>
+
+struct drm_i915_private;
+
 enum {
 	__I915_SAMPLE_FREQ_ACT = 0,
 	__I915_SAMPLE_FREQ_REQ,
diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c
index 3ace929dd90f..3f502eef2431 100644
--- a/drivers/gpu/drm/i915/i915_query.c
+++ b/drivers/gpu/drm/i915/i915_query.c
@@ -4,6 +4,8 @@
  * Copyright © 2018 Intel Corporation
  */
 
+#include <linux/nospec.h>
+
 #include "i915_drv.h"
 #include "i915_query.h"
 #include <uapi/drm/i915_drm.h>
@@ -100,7 +102,7 @@ int i915_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 
 	for (i = 0; i < args->num_items; i++, user_item_ptr++) {
 		struct drm_i915_query_item item;
-		u64 func_idx;
+		unsigned long func_idx;
 		int ret;
 
 		if (copy_from_user(&item, user_item_ptr, sizeof(item)))
@@ -109,12 +111,17 @@ int i915_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 		if (item.query_id == 0)
 			return -EINVAL;
 
+		if (overflows_type(item.query_id - 1, unsigned long))
+			return -EINVAL;
+
 		func_idx = item.query_id - 1;
 
-		if (func_idx < ARRAY_SIZE(i915_query_funcs))
+		ret = -EINVAL;
+		if (func_idx < ARRAY_SIZE(i915_query_funcs)) {
+			func_idx = array_index_nospec(func_idx,
+						      ARRAY_SIZE(i915_query_funcs));
 			ret = i915_query_funcs[func_idx](dev_priv, &item);
-		else
-			ret = -EINVAL;
+		}
 
 		/* Only write the length back to userspace if they differ. */
 		if (ret != item.length && put_user(ret, &user_item_ptr->length))
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 8a69a9275e28..f11bb213ec07 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -153,9 +153,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c))
 #define _PLL(pll, a, b) ((a) + (pll)*((b)-(a)))
 #define _MMIO_PLL(pll, a, b) _MMIO(_PLL(pll, a, b))
-#define _MMIO_PORT6(port, a, b, c, d, e, f) _MMIO(_PICK(port, a, b, c, d, e, f))
-#define _MMIO_PORT6_LN(port, ln, a0, a1, b, c, d, e, f)			\
-	_MMIO(_PICK(port, a0, b, c, d, e, f) + (ln * (a1 - a0)))
 #define _PHY3(phy, ...) _PICK(phy, __VA_ARGS__)
 #define _MMIO_PHY3(phy, a, b, c) _MMIO(_PHY3(phy, a, b, c))
 
@@ -191,6 +188,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define OTHER_CLASS		4
 #define MAX_ENGINE_CLASS	4
 
+#define OTHER_GTPM_INSTANCE	1
 #define MAX_ENGINE_INSTANCE    3
 
 /* PCI config space */
@@ -304,6 +302,17 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define  GEN6_GRDOM_VECS		(1 << 4)
 #define  GEN9_GRDOM_GUC			(1 << 5)
 #define  GEN8_GRDOM_MEDIA2		(1 << 7)
+/* GEN11 changed all bit defs except for FULL & RENDER */
+#define  GEN11_GRDOM_FULL		GEN6_GRDOM_FULL
+#define  GEN11_GRDOM_RENDER		GEN6_GRDOM_RENDER
+#define  GEN11_GRDOM_BLT		(1 << 2)
+#define  GEN11_GRDOM_GUC		(1 << 3)
+#define  GEN11_GRDOM_MEDIA		(1 << 5)
+#define  GEN11_GRDOM_MEDIA2		(1 << 6)
+#define  GEN11_GRDOM_MEDIA3		(1 << 7)
+#define  GEN11_GRDOM_MEDIA4		(1 << 8)
+#define  GEN11_GRDOM_VECS		(1 << 13)
+#define  GEN11_GRDOM_VECS2		(1 << 14)
 
 #define RING_PP_DIR_BASE(engine)	_MMIO((engine)->mmio_base+0x228)
 #define RING_PP_DIR_BASE_READ(engine)	_MMIO((engine)->mmio_base+0x518)
@@ -430,145 +439,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define VGA_CR_INDEX_CGA 0x3d4
 #define VGA_CR_DATA_CGA 0x3d5
 
-/*
- * Instruction field definitions used by the command parser
- */
-#define INSTR_CLIENT_SHIFT      29
-#define   INSTR_MI_CLIENT       0x0
-#define   INSTR_BC_CLIENT       0x2
-#define   INSTR_RC_CLIENT       0x3
-#define INSTR_SUBCLIENT_SHIFT   27
-#define INSTR_SUBCLIENT_MASK    0x18000000
-#define   INSTR_MEDIA_SUBCLIENT 0x2
-#define INSTR_26_TO_24_MASK	0x7000000
-#define   INSTR_26_TO_24_SHIFT	24
-
-/*
- * Memory interface instructions used by the kernel
- */
-#define MI_INSTR(opcode, flags) (((opcode) << 23) | (flags))
-/* Many MI commands use bit 22 of the header dword for GGTT vs PPGTT */
-#define  MI_GLOBAL_GTT    (1<<22)
-
-#define MI_NOOP			MI_INSTR(0, 0)
-#define MI_USER_INTERRUPT	MI_INSTR(0x02, 0)
-#define MI_WAIT_FOR_EVENT       MI_INSTR(0x03, 0)
-#define   MI_WAIT_FOR_OVERLAY_FLIP	(1<<16)
-#define   MI_WAIT_FOR_PLANE_B_FLIP      (1<<6)
-#define   MI_WAIT_FOR_PLANE_A_FLIP      (1<<2)
-#define   MI_WAIT_FOR_PLANE_A_SCANLINES (1<<1)
-#define MI_FLUSH		MI_INSTR(0x04, 0)
-#define   MI_READ_FLUSH		(1 << 0)
-#define   MI_EXE_FLUSH		(1 << 1)
-#define   MI_NO_WRITE_FLUSH	(1 << 2)
-#define   MI_SCENE_COUNT	(1 << 3) /* just increment scene count */
-#define   MI_END_SCENE		(1 << 4) /* flush binner and incr scene count */
-#define   MI_INVALIDATE_ISP	(1 << 5) /* invalidate indirect state pointers */
-#define MI_REPORT_HEAD		MI_INSTR(0x07, 0)
-#define MI_ARB_ON_OFF		MI_INSTR(0x08, 0)
-#define   MI_ARB_ENABLE			(1<<0)
-#define   MI_ARB_DISABLE		(0<<0)
-#define MI_BATCH_BUFFER_END	MI_INSTR(0x0a, 0)
-#define MI_SUSPEND_FLUSH	MI_INSTR(0x0b, 0)
-#define   MI_SUSPEND_FLUSH_EN	(1<<0)
-#define MI_SET_APPID		MI_INSTR(0x0e, 0)
-#define MI_OVERLAY_FLIP		MI_INSTR(0x11, 0)
-#define   MI_OVERLAY_CONTINUE	(0x0<<21)
-#define   MI_OVERLAY_ON		(0x1<<21)
-#define   MI_OVERLAY_OFF	(0x2<<21)
-#define MI_LOAD_SCAN_LINES_INCL MI_INSTR(0x12, 0)
-#define MI_DISPLAY_FLIP		MI_INSTR(0x14, 2)
-#define MI_DISPLAY_FLIP_I915	MI_INSTR(0x14, 1)
-#define   MI_DISPLAY_FLIP_PLANE(n) ((n) << 20)
-/* IVB has funny definitions for which plane to flip. */
-#define   MI_DISPLAY_FLIP_IVB_PLANE_A  (0 << 19)
-#define   MI_DISPLAY_FLIP_IVB_PLANE_B  (1 << 19)
-#define   MI_DISPLAY_FLIP_IVB_SPRITE_A (2 << 19)
-#define   MI_DISPLAY_FLIP_IVB_SPRITE_B (3 << 19)
-#define   MI_DISPLAY_FLIP_IVB_PLANE_C  (4 << 19)
-#define   MI_DISPLAY_FLIP_IVB_SPRITE_C (5 << 19)
-/* SKL ones */
-#define   MI_DISPLAY_FLIP_SKL_PLANE_1_A	(0 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_1_B	(1 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_1_C	(2 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_2_A	(4 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_2_B	(5 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_2_C	(6 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_3_A	(7 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_3_B	(8 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_3_C	(9 << 8)
-#define MI_SEMAPHORE_MBOX	MI_INSTR(0x16, 1) /* gen6, gen7 */
-#define   MI_SEMAPHORE_GLOBAL_GTT    (1<<22)
-#define   MI_SEMAPHORE_UPDATE	    (1<<21)
-#define   MI_SEMAPHORE_COMPARE	    (1<<20)
-#define   MI_SEMAPHORE_REGISTER	    (1<<18)
-#define   MI_SEMAPHORE_SYNC_VR	    (0<<16) /* RCS  wait for VCS  (RVSYNC) */
-#define   MI_SEMAPHORE_SYNC_VER	    (1<<16) /* RCS  wait for VECS (RVESYNC) */
-#define   MI_SEMAPHORE_SYNC_BR	    (2<<16) /* RCS  wait for BCS  (RBSYNC) */
-#define   MI_SEMAPHORE_SYNC_BV	    (0<<16) /* VCS  wait for BCS  (VBSYNC) */
-#define   MI_SEMAPHORE_SYNC_VEV	    (1<<16) /* VCS  wait for VECS (VVESYNC) */
-#define   MI_SEMAPHORE_SYNC_RV	    (2<<16) /* VCS  wait for RCS  (VRSYNC) */
-#define   MI_SEMAPHORE_SYNC_RB	    (0<<16) /* BCS  wait for RCS  (BRSYNC) */
-#define   MI_SEMAPHORE_SYNC_VEB	    (1<<16) /* BCS  wait for VECS (BVESYNC) */
-#define   MI_SEMAPHORE_SYNC_VB	    (2<<16) /* BCS  wait for VCS  (BVSYNC) */
-#define   MI_SEMAPHORE_SYNC_BVE	    (0<<16) /* VECS wait for BCS  (VEBSYNC) */
-#define   MI_SEMAPHORE_SYNC_VVE	    (1<<16) /* VECS wait for VCS  (VEVSYNC) */
-#define   MI_SEMAPHORE_SYNC_RVE	    (2<<16) /* VECS wait for RCS  (VERSYNC) */
-#define   MI_SEMAPHORE_SYNC_INVALID (3<<16)
-#define   MI_SEMAPHORE_SYNC_MASK    (3<<16)
-#define MI_SET_CONTEXT		MI_INSTR(0x18, 0)
-#define   MI_MM_SPACE_GTT		(1<<8)
-#define   MI_MM_SPACE_PHYSICAL		(0<<8)
-#define   MI_SAVE_EXT_STATE_EN		(1<<3)
-#define   MI_RESTORE_EXT_STATE_EN	(1<<2)
-#define   MI_FORCE_RESTORE		(1<<1)
-#define   MI_RESTORE_INHIBIT		(1<<0)
-#define   HSW_MI_RS_SAVE_STATE_EN       (1<<3)
-#define   HSW_MI_RS_RESTORE_STATE_EN    (1<<2)
-#define MI_SEMAPHORE_SIGNAL	MI_INSTR(0x1b, 0) /* GEN8+ */
-#define   MI_SEMAPHORE_TARGET(engine)	((engine)<<15)
-#define MI_SEMAPHORE_WAIT	MI_INSTR(0x1c, 2) /* GEN8+ */
-#define   MI_SEMAPHORE_POLL		(1<<15)
-#define   MI_SEMAPHORE_SAD_GTE_SDD	(1<<12)
-#define MI_STORE_DWORD_IMM	MI_INSTR(0x20, 1)
-#define MI_STORE_DWORD_IMM_GEN4	MI_INSTR(0x20, 2)
-#define   MI_MEM_VIRTUAL	(1 << 22) /* 945,g33,965 */
-#define   MI_USE_GGTT		(1 << 22) /* g4x+ */
-#define MI_STORE_DWORD_INDEX	MI_INSTR(0x21, 1)
-#define   MI_STORE_DWORD_INDEX_SHIFT 2
-/* Official intel docs are somewhat sloppy concerning MI_LOAD_REGISTER_IMM:
- * - Always issue a MI_NOOP _before_ the MI_LOAD_REGISTER_IMM - otherwise hw
- *   simply ignores the register load under certain conditions.
- * - One can actually load arbitrary many arbitrary registers: Simply issue x
- *   address/value pairs. Don't overdue it, though, x <= 2^4 must hold!
- */
-#define MI_LOAD_REGISTER_IMM(x)	MI_INSTR(0x22, 2*(x)-1)
-#define   MI_LRI_FORCE_POSTED		(1<<12)
-#define MI_STORE_REGISTER_MEM        MI_INSTR(0x24, 1)
-#define MI_STORE_REGISTER_MEM_GEN8   MI_INSTR(0x24, 2)
-#define   MI_SRM_LRM_GLOBAL_GTT		(1<<22)
-#define MI_FLUSH_DW		MI_INSTR(0x26, 1) /* for GEN6 */
-#define   MI_FLUSH_DW_STORE_INDEX	(1<<21)
-#define   MI_INVALIDATE_TLB		(1<<18)
-#define   MI_FLUSH_DW_OP_STOREDW	(1<<14)
-#define   MI_FLUSH_DW_OP_MASK		(3<<14)
-#define   MI_FLUSH_DW_NOTIFY		(1<<8)
-#define   MI_INVALIDATE_BSD		(1<<7)
-#define   MI_FLUSH_DW_USE_GTT		(1<<2)
-#define   MI_FLUSH_DW_USE_PPGTT		(0<<2)
-#define MI_LOAD_REGISTER_MEM	   MI_INSTR(0x29, 1)
-#define MI_LOAD_REGISTER_MEM_GEN8  MI_INSTR(0x29, 2)
-#define MI_BATCH_BUFFER		MI_INSTR(0x30, 1)
-#define   MI_BATCH_NON_SECURE		(1)
-/* for snb/ivb/vlv this also means "batch in ppgtt" when ppgtt is enabled. */
-#define   MI_BATCH_NON_SECURE_I965	(1<<8)
-#define   MI_BATCH_PPGTT_HSW		(1<<8)
-#define   MI_BATCH_NON_SECURE_HSW	(1<<13)
-#define MI_BATCH_BUFFER_START	MI_INSTR(0x31, 0)
-#define   MI_BATCH_GTT		    (2<<6) /* aliased with (1<<7) on gen4 */
-#define MI_BATCH_BUFFER_START_GEN8	MI_INSTR(0x31, 1)
-#define   MI_BATCH_RESOURCE_STREAMER (1<<10)
-
 #define MI_PREDICATE_SRC0	_MMIO(0x2400)
 #define MI_PREDICATE_SRC0_UDW	_MMIO(0x2400 + 4)
 #define MI_PREDICATE_SRC1	_MMIO(0x2408)
@@ -578,130 +448,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define  LOWER_SLICE_ENABLED	(1<<0)
 #define  LOWER_SLICE_DISABLED	(0<<0)
 
-/*
- * 3D instructions used by the kernel
- */
-#define GFX_INSTR(opcode, flags) ((0x3 << 29) | ((opcode) << 24) | (flags))
-
-#define GEN9_MEDIA_POOL_STATE     ((0x3 << 29) | (0x2 << 27) | (0x5 << 16) | 4)
-#define   GEN9_MEDIA_POOL_ENABLE  (1 << 31)
-#define GFX_OP_RASTER_RULES    ((0x3<<29)|(0x7<<24))
-#define GFX_OP_SCISSOR         ((0x3<<29)|(0x1c<<24)|(0x10<<19))
-#define   SC_UPDATE_SCISSOR       (0x1<<1)
-#define   SC_ENABLE_MASK          (0x1<<0)
-#define   SC_ENABLE               (0x1<<0)
-#define GFX_OP_LOAD_INDIRECT   ((0x3<<29)|(0x1d<<24)|(0x7<<16))
-#define GFX_OP_SCISSOR_INFO    ((0x3<<29)|(0x1d<<24)|(0x81<<16)|(0x1))
-#define   SCI_YMIN_MASK      (0xffff<<16)
-#define   SCI_XMIN_MASK      (0xffff<<0)
-#define   SCI_YMAX_MASK      (0xffff<<16)
-#define   SCI_XMAX_MASK      (0xffff<<0)
-#define GFX_OP_SCISSOR_ENABLE	 ((0x3<<29)|(0x1c<<24)|(0x10<<19))
-#define GFX_OP_SCISSOR_RECT	 ((0x3<<29)|(0x1d<<24)|(0x81<<16)|1)
-#define GFX_OP_COLOR_FACTOR      ((0x3<<29)|(0x1d<<24)|(0x1<<16)|0x0)
-#define GFX_OP_STIPPLE           ((0x3<<29)|(0x1d<<24)|(0x83<<16))
-#define GFX_OP_MAP_INFO          ((0x3<<29)|(0x1d<<24)|0x4)
-#define GFX_OP_DESTBUFFER_VARS   ((0x3<<29)|(0x1d<<24)|(0x85<<16)|0x0)
-#define GFX_OP_DESTBUFFER_INFO	 ((0x3<<29)|(0x1d<<24)|(0x8e<<16)|1)
-#define GFX_OP_DRAWRECT_INFO     ((0x3<<29)|(0x1d<<24)|(0x80<<16)|(0x3))
-#define GFX_OP_DRAWRECT_INFO_I965  ((0x7900<<16)|0x2)
-
-#define COLOR_BLT_CMD			(2<<29 | 0x40<<22 | (5-2))
-#define SRC_COPY_BLT_CMD		((2<<29)|(0x43<<22)|4)
-#define XY_SRC_COPY_BLT_CMD		((2<<29)|(0x53<<22)|6)
-#define XY_MONO_SRC_COPY_IMM_BLT	((2<<29)|(0x71<<22)|5)
-#define   BLT_WRITE_A			(2<<20)
-#define   BLT_WRITE_RGB			(1<<20)
-#define   BLT_WRITE_RGBA		(BLT_WRITE_RGB | BLT_WRITE_A)
-#define   BLT_DEPTH_8			(0<<24)
-#define   BLT_DEPTH_16_565		(1<<24)
-#define   BLT_DEPTH_16_1555		(2<<24)
-#define   BLT_DEPTH_32			(3<<24)
-#define   BLT_ROP_SRC_COPY		(0xcc<<16)
-#define   BLT_ROP_COLOR_COPY		(0xf0<<16)
-#define XY_SRC_COPY_BLT_SRC_TILED	(1<<15) /* 965+ only */
-#define XY_SRC_COPY_BLT_DST_TILED	(1<<11) /* 965+ only */
-#define CMD_OP_DISPLAYBUFFER_INFO ((0x0<<29)|(0x14<<23)|2)
-#define   ASYNC_FLIP                (1<<22)
-#define   DISPLAY_PLANE_A           (0<<20)
-#define   DISPLAY_PLANE_B           (1<<20)
-#define GFX_OP_PIPE_CONTROL(len)	((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2))
-#define   PIPE_CONTROL_FLUSH_L3				(1<<27)
-#define   PIPE_CONTROL_GLOBAL_GTT_IVB			(1<<24) /* gen7+ */
-#define   PIPE_CONTROL_MMIO_WRITE			(1<<23)
-#define   PIPE_CONTROL_STORE_DATA_INDEX			(1<<21)
-#define   PIPE_CONTROL_CS_STALL				(1<<20)
-#define   PIPE_CONTROL_TLB_INVALIDATE			(1<<18)
-#define   PIPE_CONTROL_MEDIA_STATE_CLEAR		(1<<16)
-#define   PIPE_CONTROL_QW_WRITE				(1<<14)
-#define   PIPE_CONTROL_POST_SYNC_OP_MASK                (3<<14)
-#define   PIPE_CONTROL_DEPTH_STALL			(1<<13)
-#define   PIPE_CONTROL_WRITE_FLUSH			(1<<12)
-#define   PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH	(1<<12) /* gen6+ */
-#define   PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE	(1<<11) /* MBZ on Ironlake */
-#define   PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE		(1<<10) /* GM45+ only */
-#define   PIPE_CONTROL_INDIRECT_STATE_DISABLE		(1<<9)
-#define   PIPE_CONTROL_NOTIFY				(1<<8)
-#define   PIPE_CONTROL_FLUSH_ENABLE			(1<<7) /* gen7+ */
-#define   PIPE_CONTROL_DC_FLUSH_ENABLE			(1<<5)
-#define   PIPE_CONTROL_VF_CACHE_INVALIDATE		(1<<4)
-#define   PIPE_CONTROL_CONST_CACHE_INVALIDATE		(1<<3)
-#define   PIPE_CONTROL_STATE_CACHE_INVALIDATE		(1<<2)
-#define   PIPE_CONTROL_STALL_AT_SCOREBOARD		(1<<1)
-#define   PIPE_CONTROL_DEPTH_CACHE_FLUSH		(1<<0)
-#define   PIPE_CONTROL_GLOBAL_GTT (1<<2) /* in addr dword */
-
-/*
- * Commands used only by the command parser
- */
-#define MI_SET_PREDICATE        MI_INSTR(0x01, 0)
-#define MI_ARB_CHECK            MI_INSTR(0x05, 0)
-#define MI_RS_CONTROL           MI_INSTR(0x06, 0)
-#define MI_URB_ATOMIC_ALLOC     MI_INSTR(0x09, 0)
-#define MI_PREDICATE            MI_INSTR(0x0C, 0)
-#define MI_RS_CONTEXT           MI_INSTR(0x0F, 0)
-#define MI_TOPOLOGY_FILTER      MI_INSTR(0x0D, 0)
-#define MI_LOAD_SCAN_LINES_EXCL MI_INSTR(0x13, 0)
-#define MI_URB_CLEAR            MI_INSTR(0x19, 0)
-#define MI_UPDATE_GTT           MI_INSTR(0x23, 0)
-#define MI_CLFLUSH              MI_INSTR(0x27, 0)
-#define MI_REPORT_PERF_COUNT    MI_INSTR(0x28, 0)
-#define   MI_REPORT_PERF_COUNT_GGTT (1<<0)
-#define MI_LOAD_REGISTER_REG    MI_INSTR(0x2A, 0)
-#define MI_RS_STORE_DATA_IMM    MI_INSTR(0x2B, 0)
-#define MI_LOAD_URB_MEM         MI_INSTR(0x2C, 0)
-#define MI_STORE_URB_MEM        MI_INSTR(0x2D, 0)
-#define MI_CONDITIONAL_BATCH_BUFFER_END MI_INSTR(0x36, 0)
-
-#define PIPELINE_SELECT                ((0x3<<29)|(0x1<<27)|(0x1<<24)|(0x4<<16))
-#define GFX_OP_3DSTATE_VF_STATISTICS   ((0x3<<29)|(0x1<<27)|(0x0<<24)|(0xB<<16))
-#define MEDIA_VFE_STATE                ((0x3<<29)|(0x2<<27)|(0x0<<24)|(0x0<<16))
-#define  MEDIA_VFE_STATE_MMIO_ACCESS_MASK (0x18)
-#define GPGPU_OBJECT                   ((0x3<<29)|(0x2<<27)|(0x1<<24)|(0x4<<16))
-#define GPGPU_WALKER                   ((0x3<<29)|(0x2<<27)|(0x1<<24)|(0x5<<16))
-#define GFX_OP_3DSTATE_DX9_CONSTANTF_VS \
-	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x39<<16))
-#define GFX_OP_3DSTATE_DX9_CONSTANTF_PS \
-	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x3A<<16))
-#define GFX_OP_3DSTATE_SO_DECL_LIST \
-	((0x3<<29)|(0x3<<27)|(0x1<<24)|(0x17<<16))
-
-#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_VS \
-	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x43<<16))
-#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_GS \
-	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x44<<16))
-#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_HS \
-	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x45<<16))
-#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_DS \
-	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x46<<16))
-#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS \
-	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x47<<16))
-
-#define MFX_WAIT  ((0x3<<29)|(0x1<<27)|(0x0<<16))
-
-#define COLOR_BLT     ((0x2<<29)|(0x40<<22))
-#define SRC_COPY_BLT  ((0x2<<29)|(0x43<<22))
-
 /*
  * Registers used only by the command parser
  */
@@ -802,6 +548,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 
 #define GEN8_OABUFFER_UDW _MMIO(0x23b4)
 #define GEN8_OABUFFER _MMIO(0x2b14)
+#define  GEN8_OABUFFER_MEM_SELECT_GGTT      (1 << 0)  /* 0: PPGTT, 1: GGTT */
 
 #define GEN7_OASTATUS1 _MMIO(0x2364)
 #define  GEN7_OASTATUS1_TAIL_MASK	    0xffffffc0
@@ -810,7 +557,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define  GEN7_OASTATUS1_REPORT_LOST	    (1<<0)
 
 #define GEN7_OASTATUS2 _MMIO(0x2368)
-#define GEN7_OASTATUS2_HEAD_MASK    0xffffffc0
+#define  GEN7_OASTATUS2_HEAD_MASK           0xffffffc0
+#define  GEN7_OASTATUS2_MEM_SELECT_GGTT     (1 << 0) /* 0: PPGTT, 1: GGTT */
 
 #define GEN8_OASTATUS _MMIO(0x2b08)
 #define  GEN8_OASTATUS_OVERRUN_STATUS	    (1<<3)
@@ -832,8 +580,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define OABUFFER_SIZE_8M    (6<<3)
 #define OABUFFER_SIZE_16M   (7<<3)
 
-#define OA_MEM_SELECT_GGTT  (1<<0)
-
 /*
  * Flexible, Aggregate EU Counter Registers.
  * Note: these aren't contiguous
@@ -1127,6 +873,12 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK	(1 << GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT)
 #define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ	0
 #define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ	1
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT	3
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK	(0x7 << GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT)
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ	0
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ	1
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_38_4_MHZ	2
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_25_MHZ	3
 #define  GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT	1
 #define  GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK	(0x3 << GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT)
 
@@ -1948,79 +1700,100 @@ enum i915_power_well_id {
 #define _CNL_PORT_PCS_DW1_LN0_C		0x162C04
 #define _CNL_PORT_PCS_DW1_LN0_D		0x162E04
 #define _CNL_PORT_PCS_DW1_LN0_F		0x162804
-#define CNL_PORT_PCS_DW1_GRP(port)	_MMIO_PORT6(port, \
+#define CNL_PORT_PCS_DW1_GRP(port)	_MMIO(_PICK(port, \
 						    _CNL_PORT_PCS_DW1_GRP_AE, \
 						    _CNL_PORT_PCS_DW1_GRP_B, \
 						    _CNL_PORT_PCS_DW1_GRP_C, \
 						    _CNL_PORT_PCS_DW1_GRP_D, \
 						    _CNL_PORT_PCS_DW1_GRP_AE, \
-						    _CNL_PORT_PCS_DW1_GRP_F)
-#define CNL_PORT_PCS_DW1_LN0(port)	_MMIO_PORT6(port, \
+						    _CNL_PORT_PCS_DW1_GRP_F))
+
+#define CNL_PORT_PCS_DW1_LN0(port)	_MMIO(_PICK(port, \
 						    _CNL_PORT_PCS_DW1_LN0_AE, \
 						    _CNL_PORT_PCS_DW1_LN0_B, \
 						    _CNL_PORT_PCS_DW1_LN0_C, \
 						    _CNL_PORT_PCS_DW1_LN0_D, \
 						    _CNL_PORT_PCS_DW1_LN0_AE, \
-						    _CNL_PORT_PCS_DW1_LN0_F)
+						    _CNL_PORT_PCS_DW1_LN0_F))
+#define _ICL_PORT_PCS_DW1_GRP_A		0x162604
+#define _ICL_PORT_PCS_DW1_GRP_B		0x6C604
+#define _ICL_PORT_PCS_DW1_LN0_A		0x162804
+#define _ICL_PORT_PCS_DW1_LN0_B		0x6C804
+#define ICL_PORT_PCS_DW1_GRP(port)	_MMIO_PORT(port,\
+						   _ICL_PORT_PCS_DW1_GRP_A, \
+						   _ICL_PORT_PCS_DW1_GRP_B)
+#define ICL_PORT_PCS_DW1_LN0(port)	_MMIO_PORT(port, \
+						   _ICL_PORT_PCS_DW1_LN0_A, \
+						   _ICL_PORT_PCS_DW1_LN0_B)
 #define   COMMON_KEEPER_EN		(1 << 26)
 
-#define _CNL_PORT_TX_DW2_GRP_AE		0x162348
-#define _CNL_PORT_TX_DW2_GRP_B		0x1623C8
-#define _CNL_PORT_TX_DW2_GRP_C		0x162B48
-#define _CNL_PORT_TX_DW2_GRP_D		0x162BC8
-#define _CNL_PORT_TX_DW2_GRP_F		0x162A48
-#define _CNL_PORT_TX_DW2_LN0_AE		0x162448
-#define _CNL_PORT_TX_DW2_LN0_B		0x162648
-#define _CNL_PORT_TX_DW2_LN0_C		0x162C48
-#define _CNL_PORT_TX_DW2_LN0_D		0x162E48
-#define _CNL_PORT_TX_DW2_LN0_F		0x162848
-#define CNL_PORT_TX_DW2_GRP(port)	_MMIO_PORT6(port, \
-						    _CNL_PORT_TX_DW2_GRP_AE, \
-						    _CNL_PORT_TX_DW2_GRP_B, \
-						    _CNL_PORT_TX_DW2_GRP_C, \
-						    _CNL_PORT_TX_DW2_GRP_D, \
-						    _CNL_PORT_TX_DW2_GRP_AE, \
-						    _CNL_PORT_TX_DW2_GRP_F)
-#define CNL_PORT_TX_DW2_LN0(port)	_MMIO_PORT6(port, \
-						    _CNL_PORT_TX_DW2_LN0_AE, \
-						    _CNL_PORT_TX_DW2_LN0_B, \
-						    _CNL_PORT_TX_DW2_LN0_C, \
-						    _CNL_PORT_TX_DW2_LN0_D, \
-						    _CNL_PORT_TX_DW2_LN0_AE, \
-						    _CNL_PORT_TX_DW2_LN0_F)
-#define   SWING_SEL_UPPER(x)		((x >> 3) << 15)
+/* CNL Port TX registers */
+#define _CNL_PORT_TX_AE_GRP_OFFSET		0x162340
+#define _CNL_PORT_TX_B_GRP_OFFSET		0x1623C0
+#define _CNL_PORT_TX_C_GRP_OFFSET		0x162B40
+#define _CNL_PORT_TX_D_GRP_OFFSET		0x162BC0
+#define _CNL_PORT_TX_F_GRP_OFFSET		0x162A40
+#define _CNL_PORT_TX_AE_LN0_OFFSET		0x162440
+#define _CNL_PORT_TX_B_LN0_OFFSET		0x162640
+#define _CNL_PORT_TX_C_LN0_OFFSET		0x162C40
+#define _CNL_PORT_TX_D_LN0_OFFSET		0x162E40
+#define _CNL_PORT_TX_F_LN0_OFFSET		0x162840
+#define _CNL_PORT_TX_DW_GRP(port, dw)	(_PICK((port), \
+					       _CNL_PORT_TX_AE_GRP_OFFSET, \
+					       _CNL_PORT_TX_B_GRP_OFFSET, \
+					       _CNL_PORT_TX_B_GRP_OFFSET, \
+					       _CNL_PORT_TX_D_GRP_OFFSET, \
+					       _CNL_PORT_TX_AE_GRP_OFFSET, \
+					       _CNL_PORT_TX_F_GRP_OFFSET) + \
+					       4*(dw))
+#define _CNL_PORT_TX_DW_LN0(port, dw)	(_PICK((port), \
+					       _CNL_PORT_TX_AE_LN0_OFFSET, \
+					       _CNL_PORT_TX_B_LN0_OFFSET, \
+					       _CNL_PORT_TX_B_LN0_OFFSET, \
+					       _CNL_PORT_TX_D_LN0_OFFSET, \
+					       _CNL_PORT_TX_AE_LN0_OFFSET, \
+					       _CNL_PORT_TX_F_LN0_OFFSET) + \
+					       4*(dw))
+
+#define CNL_PORT_TX_DW2_GRP(port)	_MMIO(_CNL_PORT_TX_DW_GRP((port), 2))
+#define CNL_PORT_TX_DW2_LN0(port)	_MMIO(_CNL_PORT_TX_DW_LN0((port), 2))
+#define _ICL_PORT_TX_DW2_GRP_A		0x162688
+#define _ICL_PORT_TX_DW2_GRP_B		0x6C688
+#define _ICL_PORT_TX_DW2_LN0_A		0x162888
+#define _ICL_PORT_TX_DW2_LN0_B		0x6C888
+#define ICL_PORT_TX_DW2_GRP(port)	_MMIO_PORT(port, \
+						   _ICL_PORT_TX_DW2_GRP_A, \
+						   _ICL_PORT_TX_DW2_GRP_B)
+#define ICL_PORT_TX_DW2_LN0(port)	_MMIO_PORT(port, \
+						   _ICL_PORT_TX_DW2_LN0_A, \
+						   _ICL_PORT_TX_DW2_LN0_B)
+#define   SWING_SEL_UPPER(x)		(((x) >> 3) << 15)
 #define   SWING_SEL_UPPER_MASK		(1 << 15)
-#define   SWING_SEL_LOWER(x)		((x & 0x7) << 11)
+#define   SWING_SEL_LOWER(x)		(((x) & 0x7) << 11)
 #define   SWING_SEL_LOWER_MASK		(0x7 << 11)
 #define   RCOMP_SCALAR(x)		((x) << 0)
 #define   RCOMP_SCALAR_MASK		(0xFF << 0)
 
-#define _CNL_PORT_TX_DW4_GRP_AE		0x162350
-#define _CNL_PORT_TX_DW4_GRP_B		0x1623D0
-#define _CNL_PORT_TX_DW4_GRP_C		0x162B50
-#define _CNL_PORT_TX_DW4_GRP_D		0x162BD0
-#define _CNL_PORT_TX_DW4_GRP_F		0x162A50
 #define _CNL_PORT_TX_DW4_LN0_AE		0x162450
 #define _CNL_PORT_TX_DW4_LN1_AE		0x1624D0
-#define _CNL_PORT_TX_DW4_LN0_B		0x162650
-#define _CNL_PORT_TX_DW4_LN0_C		0x162C50
-#define _CNL_PORT_TX_DW4_LN0_D		0x162E50
-#define _CNL_PORT_TX_DW4_LN0_F		0x162850
-#define CNL_PORT_TX_DW4_GRP(port)       _MMIO_PORT6(port, \
-						    _CNL_PORT_TX_DW4_GRP_AE, \
-						    _CNL_PORT_TX_DW4_GRP_B, \
-						    _CNL_PORT_TX_DW4_GRP_C, \
-						    _CNL_PORT_TX_DW4_GRP_D, \
-						    _CNL_PORT_TX_DW4_GRP_AE, \
-						    _CNL_PORT_TX_DW4_GRP_F)
-#define CNL_PORT_TX_DW4_LN(port, ln)       _MMIO_PORT6_LN(port, ln,	\
-						    _CNL_PORT_TX_DW4_LN0_AE, \
-						    _CNL_PORT_TX_DW4_LN1_AE, \
-						    _CNL_PORT_TX_DW4_LN0_B, \
-						    _CNL_PORT_TX_DW4_LN0_C, \
-						    _CNL_PORT_TX_DW4_LN0_D, \
-						    _CNL_PORT_TX_DW4_LN0_AE, \
-						    _CNL_PORT_TX_DW4_LN0_F)
+#define CNL_PORT_TX_DW4_GRP(port)	_MMIO(_CNL_PORT_TX_DW_GRP((port), 4))
+#define CNL_PORT_TX_DW4_LN0(port)	_MMIO(_CNL_PORT_TX_DW_LN0((port), 4))
+#define CNL_PORT_TX_DW4_LN(port, ln)   _MMIO(_CNL_PORT_TX_DW_LN0((port), 4) + \
+					     (ln * (_CNL_PORT_TX_DW4_LN1_AE - \
+						    _CNL_PORT_TX_DW4_LN0_AE)))
+#define _ICL_PORT_TX_DW4_GRP_A		0x162690
+#define _ICL_PORT_TX_DW4_GRP_B		0x6C690
+#define _ICL_PORT_TX_DW4_LN0_A		0x162890
+#define _ICL_PORT_TX_DW4_LN1_A		0x162990
+#define _ICL_PORT_TX_DW4_LN0_B		0x6C890
+#define ICL_PORT_TX_DW4_GRP(port)	_MMIO_PORT(port, \
+						   _ICL_PORT_TX_DW4_GRP_A, \
+						   _ICL_PORT_TX_DW4_GRP_B)
+#define ICL_PORT_TX_DW4_LN(port, ln)	_MMIO(_PORT(port, \
+						   _ICL_PORT_TX_DW4_LN0_A, \
+						   _ICL_PORT_TX_DW4_LN0_B) + \
+					      (ln * (_ICL_PORT_TX_DW4_LN1_A - \
+						     _ICL_PORT_TX_DW4_LN0_A)))
 #define   LOADGEN_SELECT		(1 << 31)
 #define   POST_CURSOR_1(x)		((x) << 12)
 #define   POST_CURSOR_1_MASK		(0x3F << 12)
@@ -2029,64 +1802,147 @@ enum i915_power_well_id {
 #define   CURSOR_COEFF(x)		((x) << 0)
 #define   CURSOR_COEFF_MASK		(0x3F << 0)
 
-#define _CNL_PORT_TX_DW5_GRP_AE		0x162354
-#define _CNL_PORT_TX_DW5_GRP_B		0x1623D4
-#define _CNL_PORT_TX_DW5_GRP_C		0x162B54
-#define _CNL_PORT_TX_DW5_GRP_D		0x162BD4
-#define _CNL_PORT_TX_DW5_GRP_F		0x162A54
-#define _CNL_PORT_TX_DW5_LN0_AE		0x162454
-#define _CNL_PORT_TX_DW5_LN0_B		0x162654
-#define _CNL_PORT_TX_DW5_LN0_C		0x162C54
-#define _CNL_PORT_TX_DW5_LN0_D		0x162E54
-#define _CNL_PORT_TX_DW5_LN0_F		0x162854
-#define CNL_PORT_TX_DW5_GRP(port)	_MMIO_PORT6(port, \
-						    _CNL_PORT_TX_DW5_GRP_AE, \
-						    _CNL_PORT_TX_DW5_GRP_B, \
-						    _CNL_PORT_TX_DW5_GRP_C, \
-						    _CNL_PORT_TX_DW5_GRP_D, \
-						    _CNL_PORT_TX_DW5_GRP_AE, \
-						    _CNL_PORT_TX_DW5_GRP_F)
-#define CNL_PORT_TX_DW5_LN0(port)	_MMIO_PORT6(port, \
-						    _CNL_PORT_TX_DW5_LN0_AE, \
-						    _CNL_PORT_TX_DW5_LN0_B, \
-						    _CNL_PORT_TX_DW5_LN0_C, \
-						    _CNL_PORT_TX_DW5_LN0_D, \
-						    _CNL_PORT_TX_DW5_LN0_AE, \
-						    _CNL_PORT_TX_DW5_LN0_F)
+#define CNL_PORT_TX_DW5_GRP(port)	_MMIO(_CNL_PORT_TX_DW_GRP((port), 5))
+#define CNL_PORT_TX_DW5_LN0(port)	_MMIO(_CNL_PORT_TX_DW_LN0((port), 5))
+#define _ICL_PORT_TX_DW5_GRP_A		0x162694
+#define _ICL_PORT_TX_DW5_GRP_B		0x6C694
+#define _ICL_PORT_TX_DW5_LN0_A		0x162894
+#define _ICL_PORT_TX_DW5_LN0_B		0x6C894
+#define ICL_PORT_TX_DW5_GRP(port)	_MMIO_PORT(port, \
+						   _ICL_PORT_TX_DW5_GRP_A, \
+						   _ICL_PORT_TX_DW5_GRP_B)
+#define ICL_PORT_TX_DW5_LN0(port)	_MMIO_PORT(port, \
+						   _ICL_PORT_TX_DW5_LN0_A, \
+						   _ICL_PORT_TX_DW5_LN0_B)
 #define   TX_TRAINING_EN		(1 << 31)
+#define   TAP2_DISABLE			(1 << 30)
 #define   TAP3_DISABLE			(1 << 29)
 #define   SCALING_MODE_SEL(x)		((x) << 18)
 #define   SCALING_MODE_SEL_MASK		(0x7 << 18)
 #define   RTERM_SELECT(x)		((x) << 3)
 #define   RTERM_SELECT_MASK		(0x7 << 3)
 
-#define _CNL_PORT_TX_DW7_GRP_AE		0x16235C
-#define _CNL_PORT_TX_DW7_GRP_B		0x1623DC
-#define _CNL_PORT_TX_DW7_GRP_C		0x162B5C
-#define _CNL_PORT_TX_DW7_GRP_D		0x162BDC
-#define _CNL_PORT_TX_DW7_GRP_F		0x162A5C
-#define _CNL_PORT_TX_DW7_LN0_AE		0x16245C
-#define _CNL_PORT_TX_DW7_LN0_B		0x16265C
-#define _CNL_PORT_TX_DW7_LN0_C		0x162C5C
-#define _CNL_PORT_TX_DW7_LN0_D		0x162E5C
-#define _CNL_PORT_TX_DW7_LN0_F		0x16285C
-#define CNL_PORT_TX_DW7_GRP(port)	_MMIO_PORT6(port, \
-						    _CNL_PORT_TX_DW7_GRP_AE, \
-						    _CNL_PORT_TX_DW7_GRP_B, \
-						    _CNL_PORT_TX_DW7_GRP_C, \
-						    _CNL_PORT_TX_DW7_GRP_D, \
-						    _CNL_PORT_TX_DW7_GRP_AE, \
-						    _CNL_PORT_TX_DW7_GRP_F)
-#define CNL_PORT_TX_DW7_LN0(port)	_MMIO_PORT6(port, \
-						    _CNL_PORT_TX_DW7_LN0_AE, \
-						    _CNL_PORT_TX_DW7_LN0_B, \
-						    _CNL_PORT_TX_DW7_LN0_C, \
-						    _CNL_PORT_TX_DW7_LN0_D, \
-						    _CNL_PORT_TX_DW7_LN0_AE, \
-						    _CNL_PORT_TX_DW7_LN0_F)
+#define CNL_PORT_TX_DW7_GRP(port)	_MMIO(_CNL_PORT_TX_DW_GRP((port), 7))
+#define CNL_PORT_TX_DW7_LN0(port)	_MMIO(_CNL_PORT_TX_DW_LN0((port), 7))
 #define   N_SCALAR(x)			((x) << 24)
 #define   N_SCALAR_MASK			(0x7F << 24)
 
+#define _ICL_MG_PHY_PORT_LN(port, ln, ln0p1, ln0p2, ln1p1) \
+	_MMIO(_PORT((port) - PORT_C, ln0p1, ln0p2) + (ln) * ((ln1p1) - (ln0p1)))
+
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT1		0x16812C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN1_PORT1		0x16852C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT2		0x16912C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN1_PORT2		0x16952C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT3		0x16A12C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN1_PORT3		0x16A52C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT4		0x16B12C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN1_PORT4		0x16B52C
+#define ICL_PORT_MG_TX1_LINK_PARAMS(port, ln) \
+	_ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT1, \
+				      _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT2, \
+				      _ICL_MG_TX_LINK_PARAMS_TX1LN1_PORT1)
+
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT1		0x1680AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN1_PORT1		0x1684AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT2		0x1690AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN1_PORT2		0x1694AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT3		0x16A0AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN1_PORT3		0x16A4AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT4		0x16B0AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN1_PORT4		0x16B4AC
+#define ICL_PORT_MG_TX2_LINK_PARAMS(port, ln) \
+	_ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT1, \
+				      _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT2, \
+				      _ICL_MG_TX_LINK_PARAMS_TX2LN1_PORT1)
+#define CRI_USE_FS32			(1 << 5)
+
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT1		0x16814C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN1_PORT1		0x16854C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT2		0x16914C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN1_PORT2		0x16954C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT3		0x16A14C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN1_PORT3		0x16A54C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT4		0x16B14C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN1_PORT4		0x16B54C
+#define ICL_PORT_MG_TX1_PISO_READLOAD(port, ln) \
+	_ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT1, \
+				      _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT2, \
+				      _ICL_MG_TX_PISO_READLOAD_TX1LN1_PORT1)
+
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT1		0x1680CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN1_PORT1		0x1684CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT2		0x1690CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN1_PORT2		0x1694CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT3		0x16A0CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN1_PORT3		0x16A4CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT4		0x16B0CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN1_PORT4		0x16B4CC
+#define ICL_PORT_MG_TX2_PISO_READLOAD(port, ln) \
+	_ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT1, \
+				      _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT2, \
+				      _ICL_MG_TX_PISO_READLOAD_TX2LN1_PORT1)
+#define CRI_CALCINIT					(1 << 1)
+
+#define _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT1		0x168148
+#define _ICL_MG_TX_SWINGCTRL_TX1LN1_PORT1		0x168548
+#define _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT2		0x169148
+#define _ICL_MG_TX_SWINGCTRL_TX1LN1_PORT2		0x169548
+#define _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT3		0x16A148
+#define _ICL_MG_TX_SWINGCTRL_TX1LN1_PORT3		0x16A548
+#define _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT4		0x16B148
+#define _ICL_MG_TX_SWINGCTRL_TX1LN1_PORT4		0x16B548
+#define ICL_PORT_MG_TX1_SWINGCTRL(port, ln) \
+	_ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT1, \
+				      _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT2, \
+				      _ICL_MG_TX_SWINGCTRL_TX1LN1_PORT1)
+
+#define _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT1		0x1680C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN1_PORT1		0x1684C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT2		0x1690C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN1_PORT2		0x1694C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT3		0x16A0C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN1_PORT3		0x16A4C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT4		0x16B0C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN1_PORT4		0x16B4C8
+#define ICL_PORT_MG_TX2_SWINGCTRL(port, ln) \
+	_ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT1, \
+				      _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT2, \
+				      _ICL_MG_TX_SWINGCTRL_TX2LN1_PORT1)
+#define CRI_TXDEEMPH_OVERRIDE_17_12(x)			((x) << 0)
+#define CRI_TXDEEMPH_OVERRIDE_17_12_MASK		(0x3F << 0)
+
+#define _ICL_MG_TX_DRVCTRL_TX1LN0_PORT1			0x168144
+#define _ICL_MG_TX_DRVCTRL_TX1LN1_PORT1			0x168544
+#define _ICL_MG_TX_DRVCTRL_TX1LN0_PORT2			0x169144
+#define _ICL_MG_TX_DRVCTRL_TX1LN1_PORT2			0x169544
+#define _ICL_MG_TX_DRVCTRL_TX1LN0_PORT3			0x16A144
+#define _ICL_MG_TX_DRVCTRL_TX1LN1_PORT3			0x16A544
+#define _ICL_MG_TX_DRVCTRL_TX1LN0_PORT4			0x16B144
+#define _ICL_MG_TX_DRVCTRL_TX1LN1_PORT4			0x16B544
+#define ICL_PORT_MG_TX1_DRVCTRL(port, ln) \
+	_ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_DRVCTRL_TX1LN0_PORT1, \
+				      _ICL_MG_TX_DRVCTRL_TX1LN0_PORT2, \
+				      _ICL_MG_TX_DRVCTRL_TX1LN1_PORT1)
+
+#define _ICL_MG_TX_DRVCTRL_TX2LN0_PORT1			0x1680C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN1_PORT1			0x1684C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN0_PORT2			0x1690C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN1_PORT2			0x1694C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN0_PORT3			0x16A0C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN1_PORT3			0x16A4C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN0_PORT4			0x16B0C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN1_PORT4			0x16B4C4
+#define ICL_PORT_MG_TX2_DRVCTRL(port, ln) \
+	_ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_DRVCTRL_TX2LN0_PORT1, \
+				      _ICL_MG_TX_DRVCTRL_TX2LN0_PORT2, \
+				      _ICL_MG_TX_DRVCTRL_TX2LN1_PORT1)
+#define CRI_TXDEEMPH_OVERRIDE_11_6(x)			((x) << 24)
+#define CRI_TXDEEMPH_OVERRIDE_11_6_MASK			(0x3F << 24)
+#define CRI_TXDEEMPH_OVERRIDE_EN			(1 << 22)
+#define CRI_TXDEEMPH_OVERRIDE_5_0(x)			((x) << 16)
+#define CRI_TXDEEMPH_OVERRIDE_5_0_MASK			(0x3F << 16)
+
 /* The spec defines this only for BXT PHY0, but lets assume that this
  * would exist for PHY1 too if it had a second channel.
  */
@@ -2473,6 +2329,10 @@ enum i915_power_well_id {
 #define   GEN8_MCR_SLICE_MASK		GEN8_MCR_SLICE(3)
 #define   GEN8_MCR_SUBSLICE(subslice)	(((subslice) & 3) << 24)
 #define   GEN8_MCR_SUBSLICE_MASK	GEN8_MCR_SUBSLICE(3)
+#define   GEN11_MCR_SLICE(slice)	(((slice) & 0xf) << 27)
+#define   GEN11_MCR_SLICE_MASK		GEN11_MCR_SLICE(0xf)
+#define   GEN11_MCR_SUBSLICE(subslice)	(((subslice) & 0x7) << 24)
+#define   GEN11_MCR_SUBSLICE_MASK	GEN11_MCR_SUBSLICE(0x7)
 #define RING_IPEIR(base)	_MMIO((base)+0x64)
 #define RING_IPEHR(base)	_MMIO((base)+0x68)
 /*
@@ -2867,6 +2727,19 @@ enum i915_power_well_id {
 #define GEN10_EU_DISABLE3		_MMIO(0x9140)
 #define   GEN10_EU_DIS_SS_MASK		0xff
 
+#define GEN11_GT_VEBOX_VDBOX_DISABLE	_MMIO(0x9140)
+#define   GEN11_GT_VDBOX_DISABLE_MASK	0xff
+#define   GEN11_GT_VEBOX_DISABLE_SHIFT	16
+#define   GEN11_GT_VEBOX_DISABLE_MASK	(0xff << GEN11_GT_VEBOX_DISABLE_SHIFT)
+
+#define GEN11_EU_DISABLE _MMIO(0x9134)
+#define GEN11_EU_DIS_MASK 0xFF
+
+#define GEN11_GT_SLICE_ENABLE _MMIO(0x9138)
+#define GEN11_GT_S_ENA_MASK 0xFF
+
+#define GEN11_GT_SUBSLICE_DISABLE _MMIO(0x913C)
+
 #define GEN6_BSD_SLEEP_PSMI_CONTROL	_MMIO(0x12050)
 #define   GEN6_BSD_SLEEP_MSG_DISABLE	(1 << 0)
 #define   GEN6_BSD_SLEEP_FLUSH_DISABLE	(1 << 2)
@@ -3951,6 +3824,9 @@ enum {
 #define _CLKGATE_DIS_PSL_A		0x46520
 #define _CLKGATE_DIS_PSL_B		0x46524
 #define _CLKGATE_DIS_PSL_C		0x46528
+#define   DUPS1_GATING_DIS		(1 << 15)
+#define   DUPS2_GATING_DIS		(1 << 19)
+#define   DUPS3_GATING_DIS		(1 << 23)
 #define   DPF_GATING_DIS		(1 << 10)
 #define   DPF_RAM_GATING_DIS		(1 << 9)
 #define   DPFR_GATING_DIS		(1 << 8)
@@ -3964,6 +3840,7 @@ enum {
 #define SLICE_UNIT_LEVEL_CLKGATE	_MMIO(0x94d4)
 #define  SARBUNIT_CLKGATE_DIS		(1 << 5)
 #define  RCCUNIT_CLKGATE_DIS		(1 << 7)
+#define  MSCUNIT_CLKGATE_DIS		(1 << 10)
 
 #define SUBSLICE_UNIT_LEVEL_CLKGATE	_MMIO(0x9524)
 #define  GWUNIT_CLKGATE_DIS		(1 << 16)
@@ -3971,6 +3848,9 @@ enum {
 #define UNSLICE_UNIT_LEVEL_CLKGATE	_MMIO(0x9434)
 #define  VFUNIT_CLKGATE_DIS		(1 << 20)
 
+#define INF_UNIT_LEVEL_CLKGATE		_MMIO(0x9560)
+#define   CGPSF_CLKGATE_DIS		(1 << 3)
+
 /*
  * Display engine regs
  */
@@ -4150,7 +4030,20 @@ enum {
 #define   EDP_PSR_TP1_TIME_0us			(3<<4)
 #define   EDP_PSR_IDLE_FRAME_SHIFT		0
 
+/* Bspec claims those aren't shifted but stay at 0x64800 */
+#define EDP_PSR_IMR				_MMIO(0x64834)
+#define EDP_PSR_IIR				_MMIO(0x64838)
+#define   EDP_PSR_ERROR(trans)			(1 << (((trans) * 8 + 10) & 31))
+#define   EDP_PSR_POST_EXIT(trans)		(1 << (((trans) * 8 + 9) & 31))
+#define   EDP_PSR_PRE_ENTRY(trans)		(1 << (((trans) * 8 + 8) & 31))
+
 #define EDP_PSR_AUX_CTL				_MMIO(dev_priv->psr_mmio_base + 0x10)
+#define   EDP_PSR_AUX_CTL_TIME_OUT_MASK		(3 << 26)
+#define   EDP_PSR_AUX_CTL_MESSAGE_SIZE_MASK	(0x1f << 20)
+#define   EDP_PSR_AUX_CTL_PRECHARGE_2US_MASK	(0xf << 16)
+#define   EDP_PSR_AUX_CTL_ERROR_INTERRUPT	(1 << 11)
+#define   EDP_PSR_AUX_CTL_BIT_CLOCK_2X_MASK	(0x7ff)
+
 #define EDP_PSR_AUX_DATA(i)			_MMIO(dev_priv->psr_mmio_base + 0x14 + (i) * 4) /* 5 registers */
 
 #define EDP_PSR_STATUS				_MMIO(dev_priv->psr_mmio_base + 0x40)
@@ -4180,17 +4073,19 @@ enum {
 #define EDP_PSR_PERF_CNT		_MMIO(dev_priv->psr_mmio_base + 0x44)
 #define   EDP_PSR_PERF_CNT_MASK		0xffffff
 
-#define EDP_PSR_DEBUG				_MMIO(dev_priv->psr_mmio_base + 0x60)
+#define EDP_PSR_DEBUG				_MMIO(dev_priv->psr_mmio_base + 0x60) /* PSR_MASK on SKL+ */
 #define   EDP_PSR_DEBUG_MASK_MAX_SLEEP         (1<<28)
 #define   EDP_PSR_DEBUG_MASK_LPSP              (1<<27)
 #define   EDP_PSR_DEBUG_MASK_MEMUP             (1<<26)
 #define   EDP_PSR_DEBUG_MASK_HPD               (1<<25)
 #define   EDP_PSR_DEBUG_MASK_DISP_REG_WRITE    (1<<16)
-#define   EDP_PSR_DEBUG_EXIT_ON_PIXEL_UNDERRUN (1<<15)
+#define   EDP_PSR_DEBUG_EXIT_ON_PIXEL_UNDERRUN (1<<15) /* SKL+ */
 
 #define EDP_PSR2_CTL			_MMIO(0x6f900)
 #define   EDP_PSR2_ENABLE		(1<<31)
 #define   EDP_SU_TRACK_ENABLE		(1<<30)
+#define   EDP_Y_COORDINATE_VALID	(1<<26) /* GLK and CNL+ */
+#define   EDP_Y_COORDINATE_ENABLE	(1<<25) /* GLK and CNL+ */
 #define   EDP_MAX_SU_DISABLE_TIME(t)	((t)<<20)
 #define   EDP_MAX_SU_DISABLE_TIME_MASK	(0x1f<<20)
 #define   EDP_PSR2_TP2_TIME_500		(0<<8)
@@ -4200,8 +4095,32 @@ enum {
 #define   EDP_PSR2_TP2_TIME_MASK	(3<<8)
 #define   EDP_PSR2_FRAME_BEFORE_SU_SHIFT 4
 #define   EDP_PSR2_FRAME_BEFORE_SU_MASK	(0xf<<4)
-#define   EDP_PSR2_IDLE_MASK		0xf
 #define   EDP_PSR2_FRAME_BEFORE_SU(a)	((a)<<4)
+#define   EDP_PSR2_IDLE_FRAME_MASK	0xf
+#define   EDP_PSR2_IDLE_FRAME_SHIFT	0
+
+#define _PSR_EVENT_TRANS_A			0x60848
+#define _PSR_EVENT_TRANS_B			0x61848
+#define _PSR_EVENT_TRANS_C			0x62848
+#define _PSR_EVENT_TRANS_D			0x63848
+#define _PSR_EVENT_TRANS_EDP			0x6F848
+#define PSR_EVENT(trans)			_MMIO_TRANS2(trans, _PSR_EVENT_TRANS_A)
+#define  PSR_EVENT_PSR2_WD_TIMER_EXPIRE		(1 << 17)
+#define  PSR_EVENT_PSR2_DISABLED		(1 << 16)
+#define  PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN	(1 << 15)
+#define  PSR_EVENT_SU_CRC_FIFO_UNDERRUN		(1 << 14)
+#define  PSR_EVENT_GRAPHICS_RESET		(1 << 12)
+#define  PSR_EVENT_PCH_INTERRUPT		(1 << 11)
+#define  PSR_EVENT_MEMORY_UP			(1 << 10)
+#define  PSR_EVENT_FRONT_BUFFER_MODIFY		(1 << 9)
+#define  PSR_EVENT_WD_TIMER_EXPIRE		(1 << 8)
+#define  PSR_EVENT_PIPE_REGISTERS_UPDATE	(1 << 6)
+#define  PSR_EVENT_REGISTER_UPDATE		(1 << 5)
+#define  PSR_EVENT_HDCP_ENABLE			(1 << 4)
+#define  PSR_EVENT_KVMR_SESSION_ENABLE		(1 << 3)
+#define  PSR_EVENT_VBI_ENABLE			(1 << 2)
+#define  PSR_EVENT_LPSP_MODE_EXIT		(1 << 1)
+#define  PSR_EVENT_PSR_DISABLE			(1 << 0)
 
 #define EDP_PSR2_STATUS			_MMIO(0x6f940)
 #define EDP_PSR2_STATUS_STATE_MASK     (0xf<<28)
@@ -5265,8 +5184,6 @@ enum {
 #define   DP_LINK_TRAIN_OFF		(3 << 28)
 #define   DP_LINK_TRAIN_MASK		(3 << 28)
 #define   DP_LINK_TRAIN_SHIFT		28
-#define   DP_LINK_TRAIN_PAT_3_CHV	(1 << 14)
-#define   DP_LINK_TRAIN_MASK_CHV	((3 << 28)|(1<<14))
 
 /* CPT Link training mode */
 #define   DP_LINK_TRAIN_PAT_1_CPT	(0 << 8)
@@ -6009,6 +5926,7 @@ enum {
 #define CURSIZE			_MMIO(0x700a0) /* 845/865 */
 #define _CUR_FBC_CTL_A		0x700a0 /* ivb+ */
 #define   CUR_FBC_CTL_EN	(1 << 31)
+#define _CURASURFLIVE		0x700ac /* g4x+ */
 #define _CURBCNTR		0x700c0
 #define _CURBBASE		0x700c4
 #define _CURBPOS		0x700c8
@@ -6025,6 +5943,7 @@ enum {
 #define CURBASE(pipe) _CURSOR2(pipe, _CURABASE)
 #define CURPOS(pipe) _CURSOR2(pipe, _CURAPOS)
 #define CUR_FBC_CTL(pipe) _CURSOR2(pipe, _CUR_FBC_CTL_A)
+#define CURSURFLIVE(pipe) _CURSOR2(pipe, _CURASURFLIVE)
 
 #define CURSOR_A_OFFSET 0x70080
 #define CURSOR_B_OFFSET 0x700c0
@@ -6492,9 +6411,9 @@ enum {
 #define _PLANE_COLOR_CTL_1_A			0x701CC /* GLK+ */
 #define _PLANE_COLOR_CTL_2_A			0x702CC /* GLK+ */
 #define _PLANE_COLOR_CTL_3_A			0x703CC /* GLK+ */
-#define   PLANE_COLOR_PIPE_GAMMA_ENABLE		(1 << 30)
+#define   PLANE_COLOR_PIPE_GAMMA_ENABLE		(1 << 30) /* Pre-ICL */
 #define   PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE	(1 << 28)
-#define   PLANE_COLOR_PIPE_CSC_ENABLE		(1 << 23)
+#define   PLANE_COLOR_PIPE_CSC_ENABLE		(1 << 23) /* Pre-ICL */
 #define   PLANE_COLOR_CSC_MODE_BYPASS			(0 << 17)
 #define   PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709		(1 << 17)
 #define   PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709		(2 << 17)
@@ -6589,6 +6508,9 @@ enum {
 
 #define _PLANE_BUF_CFG_1_B			0x7127c
 #define _PLANE_BUF_CFG_2_B			0x7137c
+#define  SKL_DDB_ENTRY_MASK			0x3FF
+#define  ICL_DDB_ENTRY_MASK			0x7FF
+#define  DDB_ENTRY_END_SHIFT			16
 #define _PLANE_BUF_CFG_1(pipe)	\
 	_PIPE(pipe, _PLANE_BUF_CFG_1_A, _PLANE_BUF_CFG_1_B)
 #define _PLANE_BUF_CFG_2(pipe)	\
@@ -6779,6 +6701,8 @@ enum {
 #define PS_SCALER_MODE_MASK (3 << 28)
 #define PS_SCALER_MODE_DYN  (0 << 28)
 #define PS_SCALER_MODE_HQ  (1 << 28)
+#define SKL_PS_SCALER_MODE_NV12 (2 << 28)
+#define PS_SCALER_MODE_PLANAR (1 << 29)
 #define PS_PLANE_SEL_MASK  (7 << 25)
 #define PS_PLANE_SEL(plane) (((plane) + 1) << 25)
 #define PS_FILTER_MASK         (3 << 23)
@@ -6950,6 +6874,7 @@ enum {
 #define DE_PCH_EVENT_IVB		(1<<28)
 #define DE_DP_A_HOTPLUG_IVB		(1<<27)
 #define DE_AUX_CHANNEL_A_IVB		(1<<26)
+#define DE_EDP_PSR_INT_HSW		(1<<19)
 #define DE_SPRITEC_FLIP_DONE_IVB	(1<<14)
 #define DE_PLANEC_FLIP_DONE_IVB		(1<<13)
 #define DE_PIPEC_VBLANK_IVB		(1<<10)
@@ -7074,6 +6999,7 @@ enum {
 #define GEN8_DE_MISC_IIR _MMIO(0x44468)
 #define GEN8_DE_MISC_IER _MMIO(0x4446c)
 #define  GEN8_DE_MISC_GSE		(1 << 27)
+#define  GEN8_DE_EDP_PSR		(1 << 19)
 
 #define GEN8_PCU_ISR _MMIO(0x444e0)
 #define GEN8_PCU_IMR _MMIO(0x444e4)
@@ -7117,7 +7043,9 @@ enum {
 #define GEN11_INTR_IDENTITY_REG0	_MMIO(0x190060)
 #define GEN11_INTR_IDENTITY_REG1	_MMIO(0x190064)
 #define  GEN11_INTR_DATA_VALID		(1 << 31)
-#define  GEN11_INTR_ENGINE_MASK		(0xffff)
+#define  GEN11_INTR_ENGINE_CLASS(x)	(((x) & GENMASK(18, 16)) >> 16)
+#define  GEN11_INTR_ENGINE_INSTANCE(x)	(((x) & GENMASK(25, 20)) >> 20)
+#define  GEN11_INTR_ENGINE_INTR(x)	((x) & 0xffff)
 
 #define GEN11_INTR_IDENTITY_REG(x)	_MMIO(0x190060 + (x * 4))
 
@@ -7197,6 +7125,7 @@ enum {
 #define CHICKEN_TRANS_A         0x420c0
 #define CHICKEN_TRANS_B         0x420c4
 #define CHICKEN_TRANS(trans) _MMIO_TRANS(trans, CHICKEN_TRANS_A, CHICKEN_TRANS_B)
+#define  VSC_DATA_SEL_SOFTWARE_CONTROL	(1<<25) /* GLK and CNL+ */
 #define  DDI_TRAINING_OVERRIDE_ENABLE	(1<<19)
 #define  DDI_TRAINING_OVERRIDE_VALUE	(1<<18)
 #define  DDIE_TRAINING_OVERRIDE_ENABLE	(1<<17) /* CHICKEN_TRANS_A only */
@@ -7301,18 +7230,22 @@ enum {
 #define GEN7_L3CNTLREG3				_MMIO(0xB024)
 
 #define GEN7_L3_CHICKEN_MODE_REGISTER		_MMIO(0xB030)
-#define  GEN7_WA_L3_CHICKEN_MODE				0x20000000
+#define   GEN7_WA_L3_CHICKEN_MODE		0x20000000
+#define GEN10_L3_CHICKEN_MODE_REGISTER		_MMIO(0xB114)
+#define   GEN11_I2M_WRITE_DISABLE		(1 << 28)
 
 #define GEN7_L3SQCREG4				_MMIO(0xb034)
 #define  L3SQ_URB_READ_CAM_MATCH_DISABLE	(1<<27)
 
 #define GEN8_L3SQCREG4				_MMIO(0xb118)
-#define  GEN8_LQSC_RO_PERF_DIS			(1<<27)
-#define  GEN8_LQSC_FLUSH_COHERENT_LINES		(1<<21)
+#define  GEN11_LQSC_CLEAN_EVICT_DISABLE		(1 << 6)
+#define  GEN8_LQSC_RO_PERF_DIS			(1 << 27)
+#define  GEN8_LQSC_FLUSH_COHERENT_LINES		(1 << 21)
 
 /* GEN8 chicken */
 #define HDC_CHICKEN0				_MMIO(0x7300)
 #define CNL_HDC_CHICKEN0			_MMIO(0xE5F0)
+#define ICL_HDC_MODE				_MMIO(0xE5F4)
 #define  HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE	(1<<15)
 #define  HDC_FENCE_DEST_SLM_DISABLE		(1<<14)
 #define  HDC_DONOT_FETCH_MEM_WHEN_MASKED	(1<<11)
@@ -8327,8 +8260,30 @@ enum {
 #define   GEN8_DOP_CLOCK_GATE_GUC_ENABLE	(1<<4)
 #define   GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE     (1<<6)
 
-#define GEN8_GARBCNTL                   _MMIO(0xB004)
-#define   GEN9_GAPS_TSV_CREDIT_DISABLE  (1<<7)
+#define GEN8_GARBCNTL				_MMIO(0xB004)
+#define   GEN9_GAPS_TSV_CREDIT_DISABLE		(1 << 7)
+#define   GEN11_ARBITRATION_PRIO_ORDER_MASK	(0x3f << 22)
+#define   GEN11_HASH_CTRL_EXCL_MASK		(0x7f << 0)
+#define   GEN11_HASH_CTRL_EXCL_BIT0		(1 << 0)
+
+#define GEN11_GLBLINVL				_MMIO(0xB404)
+#define   GEN11_BANK_HASH_ADDR_EXCL_MASK	(0x7f << 5)
+#define   GEN11_BANK_HASH_ADDR_EXCL_BIT0	(1 << 5)
+
+#define GEN10_DFR_RATIO_EN_AND_CHICKEN	_MMIO(0x9550)
+#define   DFR_DISABLE			(1 << 9)
+
+#define GEN11_GACB_PERF_CTRL			_MMIO(0x4B80)
+#define   GEN11_HASH_CTRL_MASK			(0x3 << 12 | 0xf << 0)
+#define   GEN11_HASH_CTRL_BIT0			(1 << 0)
+#define   GEN11_HASH_CTRL_BIT4			(1 << 12)
+
+#define GEN11_LSN_UNSLCVC				_MMIO(0xB43C)
+#define   GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC	(1 << 9)
+#define   GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC	(1 << 7)
+
+#define GAMW_ECO_DEV_RW_IA_REG			_MMIO(0x4080)
+#define   GAMW_ECO_DEV_CTX_RELOAD_DISABLE	(1 << 7)
 
 /* IVYBRIDGE DPF */
 #define GEN7_L3CDERRST1(slice)		_MMIO(0xB008 + (slice) * 0x200) /* L3CD Error Status 1 */
@@ -8837,6 +8792,12 @@ enum skl_power_gate {
 #define  PORT_CLK_SEL_NONE		(7<<29)
 #define  PORT_CLK_SEL_MASK		(7<<29)
 
+/* On ICL+ this is the same as PORT_CLK_SEL, but all bits change. */
+#define DDI_CLK_SEL(port)		PORT_CLK_SEL(port)
+#define  DDI_CLK_SEL_NONE		(0x0 << 28)
+#define  DDI_CLK_SEL_MG			(0x8 << 28)
+#define  DDI_CLK_SEL_MASK		(0xF << 28)
+
 /* Transcoder clock selection */
 #define _TRANS_CLK_SEL_A		0x46140
 #define _TRANS_CLK_SEL_B		0x46144
@@ -8967,6 +8928,7 @@ enum skl_power_gate {
  * CNL Clocks
  */
 #define DPCLKA_CFGCR0				_MMIO(0x6C200)
+#define DPCLKA_CFGCR0_ICL			_MMIO(0x164280)
 #define  DPCLKA_CFGCR0_DDI_CLK_OFF(port)	(1 << ((port) ==  PORT_F ? 23 : \
 						      (port)+10))
 #define  DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port)	((port) == PORT_F ? 21 : \
@@ -8983,10 +8945,141 @@ enum skl_power_gate {
 #define  PLL_POWER_STATE	(1 << 26)
 #define CNL_DPLL_ENABLE(pll)	_MMIO_PLL(pll, DPLL0_ENABLE, DPLL1_ENABLE)
 
+#define _MG_PLL1_ENABLE		0x46030
+#define _MG_PLL2_ENABLE		0x46034
+#define _MG_PLL3_ENABLE		0x46038
+#define _MG_PLL4_ENABLE		0x4603C
+/* Bits are the same as DPLL0_ENABLE */
+#define MG_PLL_ENABLE(port)	_MMIO_PORT((port) - PORT_C, _MG_PLL1_ENABLE, \
+					   _MG_PLL2_ENABLE)
+
+#define _MG_REFCLKIN_CTL_PORT1				0x16892C
+#define _MG_REFCLKIN_CTL_PORT2				0x16992C
+#define _MG_REFCLKIN_CTL_PORT3				0x16A92C
+#define _MG_REFCLKIN_CTL_PORT4				0x16B92C
+#define   MG_REFCLKIN_CTL_OD_2_MUX(x)			((x) << 8)
+#define MG_REFCLKIN_CTL(port) _MMIO_PORT((port) - PORT_C, \
+					 _MG_REFCLKIN_CTL_PORT1, \
+					 _MG_REFCLKIN_CTL_PORT2)
+
+#define _MG_CLKTOP2_CORECLKCTL1_PORT1			0x1688D8
+#define _MG_CLKTOP2_CORECLKCTL1_PORT2			0x1698D8
+#define _MG_CLKTOP2_CORECLKCTL1_PORT3			0x16A8D8
+#define _MG_CLKTOP2_CORECLKCTL1_PORT4			0x16B8D8
+#define   MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO(x)		((x) << 16)
+#define   MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO(x)		((x) << 8)
+#define MG_CLKTOP2_CORECLKCTL1(port) _MMIO_PORT((port) - PORT_C, \
+						_MG_CLKTOP2_CORECLKCTL1_PORT1, \
+						_MG_CLKTOP2_CORECLKCTL1_PORT2)
+
+#define _MG_CLKTOP2_HSCLKCTL_PORT1			0x1688D4
+#define _MG_CLKTOP2_HSCLKCTL_PORT2			0x1698D4
+#define _MG_CLKTOP2_HSCLKCTL_PORT3			0x16A8D4
+#define _MG_CLKTOP2_HSCLKCTL_PORT4			0x16B8D4
+#define   MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL(x)		((x) << 16)
+#define   MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL(x)	((x) << 14)
+#define   MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO(x)		((x) << 12)
+#define   MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO(x)		((x) << 8)
+#define MG_CLKTOP2_HSCLKCTL(port) _MMIO_PORT((port) - PORT_C, \
+					     _MG_CLKTOP2_HSCLKCTL_PORT1, \
+					     _MG_CLKTOP2_HSCLKCTL_PORT2)
+
+#define _MG_PLL_DIV0_PORT1				0x168A00
+#define _MG_PLL_DIV0_PORT2				0x169A00
+#define _MG_PLL_DIV0_PORT3				0x16AA00
+#define _MG_PLL_DIV0_PORT4				0x16BA00
+#define   MG_PLL_DIV0_FRACNEN_H				(1 << 30)
+#define   MG_PLL_DIV0_FBDIV_FRAC(x)			((x) << 8)
+#define   MG_PLL_DIV0_FBDIV_INT(x)			((x) << 0)
+#define MG_PLL_DIV0(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_DIV0_PORT1, \
+				     _MG_PLL_DIV0_PORT2)
+
+#define _MG_PLL_DIV1_PORT1				0x168A04
+#define _MG_PLL_DIV1_PORT2				0x169A04
+#define _MG_PLL_DIV1_PORT3				0x16AA04
+#define _MG_PLL_DIV1_PORT4				0x16BA04
+#define   MG_PLL_DIV1_IREF_NDIVRATIO(x)			((x) << 16)
+#define   MG_PLL_DIV1_DITHER_DIV_1			(0 << 12)
+#define   MG_PLL_DIV1_DITHER_DIV_2			(1 << 12)
+#define   MG_PLL_DIV1_DITHER_DIV_4			(2 << 12)
+#define   MG_PLL_DIV1_DITHER_DIV_8			(3 << 12)
+#define   MG_PLL_DIV1_NDIVRATIO(x)			((x) << 4)
+#define   MG_PLL_DIV1_FBPREDIV(x)			((x) << 0)
+#define MG_PLL_DIV1(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_DIV1_PORT1, \
+				     _MG_PLL_DIV1_PORT2)
+
+#define _MG_PLL_LF_PORT1				0x168A08
+#define _MG_PLL_LF_PORT2				0x169A08
+#define _MG_PLL_LF_PORT3				0x16AA08
+#define _MG_PLL_LF_PORT4				0x16BA08
+#define   MG_PLL_LF_TDCTARGETCNT(x)			((x) << 24)
+#define   MG_PLL_LF_AFCCNTSEL_256			(0 << 20)
+#define   MG_PLL_LF_AFCCNTSEL_512			(1 << 20)
+#define   MG_PLL_LF_GAINCTRL(x)				((x) << 16)
+#define   MG_PLL_LF_INT_COEFF(x)			((x) << 8)
+#define   MG_PLL_LF_PROP_COEFF(x)			((x) << 0)
+#define MG_PLL_LF(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_LF_PORT1, \
+				   _MG_PLL_LF_PORT2)
+
+#define _MG_PLL_FRAC_LOCK_PORT1				0x168A0C
+#define _MG_PLL_FRAC_LOCK_PORT2				0x169A0C
+#define _MG_PLL_FRAC_LOCK_PORT3				0x16AA0C
+#define _MG_PLL_FRAC_LOCK_PORT4				0x16BA0C
+#define   MG_PLL_FRAC_LOCK_TRUELOCK_CRIT_32		(1 << 18)
+#define   MG_PLL_FRAC_LOCK_EARLYLOCK_CRIT_32		(1 << 16)
+#define   MG_PLL_FRAC_LOCK_LOCKTHRESH(x)		((x) << 11)
+#define   MG_PLL_FRAC_LOCK_DCODITHEREN			(1 << 10)
+#define   MG_PLL_FRAC_LOCK_FEEDFWRDCAL_EN		(1 << 8)
+#define   MG_PLL_FRAC_LOCK_FEEDFWRDGAIN(x)		((x) << 0)
+#define MG_PLL_FRAC_LOCK(port) _MMIO_PORT((port) - PORT_C, \
+					  _MG_PLL_FRAC_LOCK_PORT1, \
+					  _MG_PLL_FRAC_LOCK_PORT2)
+
+#define _MG_PLL_SSC_PORT1				0x168A10
+#define _MG_PLL_SSC_PORT2				0x169A10
+#define _MG_PLL_SSC_PORT3				0x16AA10
+#define _MG_PLL_SSC_PORT4				0x16BA10
+#define   MG_PLL_SSC_EN					(1 << 28)
+#define   MG_PLL_SSC_TYPE(x)				((x) << 26)
+#define   MG_PLL_SSC_STEPLENGTH(x)			((x) << 16)
+#define   MG_PLL_SSC_STEPNUM(x)				((x) << 10)
+#define   MG_PLL_SSC_FLLEN				(1 << 9)
+#define   MG_PLL_SSC_STEPSIZE(x)			((x) << 0)
+#define MG_PLL_SSC(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_SSC_PORT1, \
+				    _MG_PLL_SSC_PORT2)
+
+#define _MG_PLL_BIAS_PORT1				0x168A14
+#define _MG_PLL_BIAS_PORT2				0x169A14
+#define _MG_PLL_BIAS_PORT3				0x16AA14
+#define _MG_PLL_BIAS_PORT4				0x16BA14
+#define   MG_PLL_BIAS_BIAS_GB_SEL(x)			((x) << 30)
+#define   MG_PLL_BIAS_INIT_DCOAMP(x)			((x) << 24)
+#define   MG_PLL_BIAS_BIAS_BONUS(x)			((x) << 16)
+#define   MG_PLL_BIAS_BIASCAL_EN			(1 << 15)
+#define   MG_PLL_BIAS_CTRIM(x)				((x) << 8)
+#define   MG_PLL_BIAS_VREF_RDAC(x)			((x) << 5)
+#define   MG_PLL_BIAS_IREFTRIM(x)			((x) << 0)
+#define MG_PLL_BIAS(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_BIAS_PORT1, \
+				     _MG_PLL_BIAS_PORT2)
+
+#define _MG_PLL_TDC_COLDST_BIAS_PORT1			0x168A18
+#define _MG_PLL_TDC_COLDST_BIAS_PORT2			0x169A18
+#define _MG_PLL_TDC_COLDST_BIAS_PORT3			0x16AA18
+#define _MG_PLL_TDC_COLDST_BIAS_PORT4			0x16BA18
+#define   MG_PLL_TDC_COLDST_IREFINT_EN			(1 << 27)
+#define   MG_PLL_TDC_COLDST_REFBIAS_START_PULSE_W(x)	((x) << 17)
+#define   MG_PLL_TDC_COLDST_COLDSTART			(1 << 16)
+#define   MG_PLL_TDC_TDCOVCCORR_EN			(1 << 2)
+#define   MG_PLL_TDC_TDCSEL(x)				((x) << 0)
+#define MG_PLL_TDC_COLDST_BIAS(port) _MMIO_PORT((port) - PORT_C, \
+						_MG_PLL_TDC_COLDST_BIAS_PORT1, \
+						_MG_PLL_TDC_COLDST_BIAS_PORT2)
+
 #define _CNL_DPLL0_CFGCR0		0x6C000
 #define _CNL_DPLL1_CFGCR0		0x6C080
 #define  DPLL_CFGCR0_HDMI_MODE		(1 << 30)
 #define  DPLL_CFGCR0_SSC_ENABLE		(1 << 29)
+#define  DPLL_CFGCR0_SSC_ENABLE_ICL	(1 << 25)
 #define  DPLL_CFGCR0_LINK_RATE_MASK	(0xf << 25)
 #define  DPLL_CFGCR0_LINK_RATE_2700	(0 << 25)
 #define  DPLL_CFGCR0_LINK_RATE_1350	(1 << 25)
@@ -9020,8 +9113,19 @@ enum skl_power_gate {
 #define  DPLL_CFGCR1_PDIV_5		(4 << 2)
 #define  DPLL_CFGCR1_PDIV_7		(8 << 2)
 #define  DPLL_CFGCR1_CENTRAL_FREQ	(3 << 0)
+#define  DPLL_CFGCR1_CENTRAL_FREQ_8400	(3 << 0)
 #define CNL_DPLL_CFGCR1(pll)		_MMIO_PLL(pll, _CNL_DPLL0_CFGCR1, _CNL_DPLL1_CFGCR1)
 
+#define _ICL_DPLL0_CFGCR0		0x164000
+#define _ICL_DPLL1_CFGCR0		0x164080
+#define ICL_DPLL_CFGCR0(pll)		_MMIO_PLL(pll, _ICL_DPLL0_CFGCR0, \
+						  _ICL_DPLL1_CFGCR0)
+
+#define _ICL_DPLL0_CFGCR1		0x164004
+#define _ICL_DPLL1_CFGCR1		0x164084
+#define ICL_DPLL_CFGCR1(pll)		_MMIO_PLL(pll, _ICL_DPLL0_CFGCR1, \
+						  _ICL_DPLL1_CFGCR1)
+
 /* BXT display engine PLL */
 #define BXT_DE_PLL_CTL			_MMIO(0x6d000)
 #define   BXT_DE_PLL_RATIO(x)		(x)	/* {60,65,100} * 19.2MHz */
@@ -9793,6 +9897,13 @@ enum skl_power_gate {
 #define GEN9_MFX1_MOCS(i)	_MMIO(0xca00 + (i) * 4)	/* Media 1 MOCS registers */
 #define GEN9_VEBOX_MOCS(i)	_MMIO(0xcb00 + (i) * 4)	/* Video MOCS registers */
 #define GEN9_BLT_MOCS(i)	_MMIO(0xcc00 + (i) * 4)	/* Blitter MOCS registers */
+/* Media decoder 2 MOCS registers */
+#define GEN11_MFX2_MOCS(i)	_MMIO(0x10000 + (i) * 4)
+
+#define GEN10_SCRATCH_LNCF2		_MMIO(0xb0a0)
+#define   PMFLUSHDONE_LNICRSDROP	(1 << 20)
+#define   PMFLUSH_GAPL3UNBLOCK		(1 << 21)
+#define   PMFLUSHDONE_LNEBLK		(1 << 22)
 
 /* gamt regs */
 #define GEN8_L3_LRA_1_GPGPU _MMIO(0x4dd4)
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index 282f57630cc1..8928894dd9c7 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -49,7 +49,7 @@ static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 		return "signaled";
 
-	return to_request(fence)->timeline->common->name;
+	return to_request(fence)->timeline->name;
 }
 
 static bool i915_fence_signaled(struct dma_fence *fence)
@@ -59,11 +59,7 @@ static bool i915_fence_signaled(struct dma_fence *fence)
 
 static bool i915_fence_enable_signaling(struct dma_fence *fence)
 {
-	if (i915_fence_signaled(fence))
-		return false;
-
-	intel_engine_enable_signaling(to_request(fence), true);
-	return !i915_fence_signaled(fence);
+	return intel_engine_enable_signaling(to_request(fence), true);
 }
 
 static signed long i915_fence_wait(struct dma_fence *fence,
@@ -129,22 +125,22 @@ i915_dependency_free(struct drm_i915_private *i915,
 }
 
 static void
-__i915_priotree_add_dependency(struct i915_priotree *pt,
-			       struct i915_priotree *signal,
-			       struct i915_dependency *dep,
-			       unsigned long flags)
+__i915_sched_node_add_dependency(struct i915_sched_node *node,
+				 struct i915_sched_node *signal,
+				 struct i915_dependency *dep,
+				 unsigned long flags)
 {
 	INIT_LIST_HEAD(&dep->dfs_link);
 	list_add(&dep->wait_link, &signal->waiters_list);
-	list_add(&dep->signal_link, &pt->signalers_list);
+	list_add(&dep->signal_link, &node->signalers_list);
 	dep->signaler = signal;
 	dep->flags = flags;
 }
 
 static int
-i915_priotree_add_dependency(struct drm_i915_private *i915,
-			     struct i915_priotree *pt,
-			     struct i915_priotree *signal)
+i915_sched_node_add_dependency(struct drm_i915_private *i915,
+			       struct i915_sched_node *node,
+			       struct i915_sched_node *signal)
 {
 	struct i915_dependency *dep;
 
@@ -152,16 +148,18 @@ i915_priotree_add_dependency(struct drm_i915_private *i915,
 	if (!dep)
 		return -ENOMEM;
 
-	__i915_priotree_add_dependency(pt, signal, dep, I915_DEPENDENCY_ALLOC);
+	__i915_sched_node_add_dependency(node, signal, dep,
+					 I915_DEPENDENCY_ALLOC);
 	return 0;
 }
 
 static void
-i915_priotree_fini(struct drm_i915_private *i915, struct i915_priotree *pt)
+i915_sched_node_fini(struct drm_i915_private *i915,
+		     struct i915_sched_node *node)
 {
-	struct i915_dependency *dep, *next;
+	struct i915_dependency *dep, *tmp;
 
-	GEM_BUG_ON(!list_empty(&pt->link));
+	GEM_BUG_ON(!list_empty(&node->link));
 
 	/*
 	 * Everyone we depended upon (the fences we wait to be signaled)
@@ -169,8 +167,8 @@ i915_priotree_fini(struct drm_i915_private *i915, struct i915_priotree *pt)
 	 * However, retirement is run independently on each timeline and
 	 * so we may be called out-of-order.
 	 */
-	list_for_each_entry_safe(dep, next, &pt->signalers_list, signal_link) {
-		GEM_BUG_ON(!i915_priotree_signaled(dep->signaler));
+	list_for_each_entry_safe(dep, tmp, &node->signalers_list, signal_link) {
+		GEM_BUG_ON(!i915_sched_node_signaled(dep->signaler));
 		GEM_BUG_ON(!list_empty(&dep->dfs_link));
 
 		list_del(&dep->wait_link);
@@ -179,8 +177,8 @@ i915_priotree_fini(struct drm_i915_private *i915, struct i915_priotree *pt)
 	}
 
 	/* Remove ourselves from everyone who depends upon us */
-	list_for_each_entry_safe(dep, next, &pt->waiters_list, wait_link) {
-		GEM_BUG_ON(dep->signaler != pt);
+	list_for_each_entry_safe(dep, tmp, &node->waiters_list, wait_link) {
+		GEM_BUG_ON(dep->signaler != node);
 		GEM_BUG_ON(!list_empty(&dep->dfs_link));
 
 		list_del(&dep->signal_link);
@@ -190,17 +188,18 @@ i915_priotree_fini(struct drm_i915_private *i915, struct i915_priotree *pt)
 }
 
 static void
-i915_priotree_init(struct i915_priotree *pt)
+i915_sched_node_init(struct i915_sched_node *node)
 {
-	INIT_LIST_HEAD(&pt->signalers_list);
-	INIT_LIST_HEAD(&pt->waiters_list);
-	INIT_LIST_HEAD(&pt->link);
-	pt->priority = I915_PRIORITY_INVALID;
+	INIT_LIST_HEAD(&node->signalers_list);
+	INIT_LIST_HEAD(&node->waiters_list);
+	INIT_LIST_HEAD(&node->link);
+	node->attr.priority = I915_PRIORITY_INVALID;
 }
 
 static int reset_all_global_seqno(struct drm_i915_private *i915, u32 seqno)
 {
 	struct intel_engine_cs *engine;
+	struct i915_timeline *timeline;
 	enum intel_engine_id id;
 	int ret;
 
@@ -211,30 +210,37 @@ static int reset_all_global_seqno(struct drm_i915_private *i915, u32 seqno)
 	if (ret)
 		return ret;
 
+	GEM_BUG_ON(i915->gt.active_requests);
+
 	/* If the seqno wraps around, we need to clear the breadcrumb rbtree */
 	for_each_engine(engine, i915, id) {
-		struct i915_gem_timeline *timeline;
-		struct intel_timeline *tl = engine->timeline;
+		GEM_TRACE("%s seqno %d (current %d) -> %d\n",
+			  engine->name,
+			  engine->timeline.seqno,
+			  intel_engine_get_seqno(engine),
+			  seqno);
 
-		if (!i915_seqno_passed(seqno, tl->seqno)) {
+		if (!i915_seqno_passed(seqno, engine->timeline.seqno)) {
 			/* Flush any waiters before we reuse the seqno */
 			intel_engine_disarm_breadcrumbs(engine);
+			intel_engine_init_hangcheck(engine);
 			GEM_BUG_ON(!list_empty(&engine->breadcrumbs.signals));
 		}
 
 		/* Check we are idle before we fiddle with hw state! */
 		GEM_BUG_ON(!intel_engine_is_idle(engine));
-		GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request));
+		GEM_BUG_ON(i915_gem_active_isset(&engine->timeline.last_request));
 
 		/* Finally reset hw state */
 		intel_engine_init_global_seqno(engine, seqno);
-		tl->seqno = seqno;
-
-		list_for_each_entry(timeline, &i915->gt.timelines, link)
-			memset(timeline->engine[id].global_sync, 0,
-			       sizeof(timeline->engine[id].global_sync));
+		engine->timeline.seqno = seqno;
 	}
 
+	list_for_each_entry(timeline, &i915->gt.timelines, link)
+		memset(timeline->global_sync, 0, sizeof(timeline->global_sync));
+
+	i915->gt.request_serial = seqno;
+
 	return 0;
 }
 
@@ -251,83 +257,37 @@ int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno)
 	return reset_all_global_seqno(i915, seqno - 1);
 }
 
-static void mark_busy(struct drm_i915_private *i915)
+static int reserve_gt(struct drm_i915_private *i915)
 {
-	if (i915->gt.awake)
-		return;
-
-	GEM_BUG_ON(!i915->gt.active_requests);
-
-	intel_runtime_pm_get_noresume(i915);
+	int ret;
 
 	/*
-	 * It seems that the DMC likes to transition between the DC states a lot
-	 * when there are no connected displays (no active power domains) during
-	 * command submission.
-	 *
-	 * This activity has negative impact on the performance of the chip with
-	 * huge latencies observed in the interrupt handler and elsewhere.
+	 * Reservation is fine until we may need to wrap around
 	 *
-	 * Work around it by grabbing a GT IRQ power domain whilst there is any
-	 * GT activity, preventing any DC state transitions.
+	 * By incrementing the serial for every request, we know that no
+	 * individual engine may exceed that serial (as each is reset to 0
+	 * on any wrap). This protects even the most pessimistic of migrations
+	 * of every request from all engines onto just one.
 	 */
-	intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
-
-	i915->gt.awake = true;
-	if (unlikely(++i915->gt.epoch == 0)) /* keep 0 as invalid */
-		i915->gt.epoch = 1;
-
-	intel_enable_gt_powersave(i915);
-	i915_update_gfx_val(i915);
-	if (INTEL_GEN(i915) >= 6)
-		gen6_rps_busy(i915);
-	i915_pmu_gt_unparked(i915);
-
-	intel_engines_unpark(i915);
-
-	i915_queue_hangcheck(i915);
-
-	queue_delayed_work(i915->wq,
-			   &i915->gt.retire_work,
-			   round_jiffies_up_relative(HZ));
-}
-
-static int reserve_engine(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *i915 = engine->i915;
-	u32 active = ++engine->timeline->inflight_seqnos;
-	u32 seqno = engine->timeline->seqno;
-	int ret;
-
-	/* Reservation is fine until we need to wrap around */
-	if (unlikely(add_overflows(seqno, active))) {
+	while (unlikely(++i915->gt.request_serial == 0)) {
 		ret = reset_all_global_seqno(i915, 0);
 		if (ret) {
-			engine->timeline->inflight_seqnos--;
+			i915->gt.request_serial--;
 			return ret;
 		}
 	}
 
 	if (!i915->gt.active_requests++)
-		mark_busy(i915);
+		i915_gem_unpark(i915);
 
 	return 0;
 }
 
-static void unreserve_engine(struct intel_engine_cs *engine)
+static void unreserve_gt(struct drm_i915_private *i915)
 {
-	struct drm_i915_private *i915 = engine->i915;
-
-	if (!--i915->gt.active_requests) {
-		/* Cancel the mark_busy() from our reserve_engine() */
-		GEM_BUG_ON(!i915->gt.awake);
-		mod_delayed_work(i915->wq,
-				 &i915->gt.idle_work,
-				 msecs_to_jiffies(100));
-	}
-
-	GEM_BUG_ON(!engine->timeline->inflight_seqnos);
-	engine->timeline->inflight_seqnos--;
+	GEM_BUG_ON(!i915->gt.active_requests);
+	if (!--i915->gt.active_requests)
+		i915_gem_park(i915);
 }
 
 void i915_gem_retire_noop(struct i915_gem_active *active,
@@ -338,6 +298,7 @@ void i915_gem_retire_noop(struct i915_gem_active *active,
 
 static void advance_ring(struct i915_request *request)
 {
+	struct intel_ring *ring = request->ring;
 	unsigned int tail;
 
 	/*
@@ -349,7 +310,8 @@ static void advance_ring(struct i915_request *request)
 	 * Note this requires that we are always called in request
 	 * completion order.
 	 */
-	if (list_is_last(&request->ring_link, &request->ring->request_list)) {
+	GEM_BUG_ON(!list_is_first(&request->ring_link, &ring->request_list));
+	if (list_is_last(&request->ring_link, &ring->request_list)) {
 		/*
 		 * We may race here with execlists resubmitting this request
 		 * as we retire it. The resubmission will move the ring->tail
@@ -358,13 +320,14 @@ static void advance_ring(struct i915_request *request)
 		 * is just about to be. Either works, if we miss the last two
 		 * noops - they are safe to be replayed on a reset.
 		 */
-		tail = READ_ONCE(request->ring->tail);
+		tail = READ_ONCE(request->tail);
+		list_del(&ring->active_link);
 	} else {
 		tail = request->postfix;
 	}
-	list_del(&request->ring_link);
+	list_del_init(&request->ring_link);
 
-	request->ring->head = tail;
+	ring->head = tail;
 }
 
 static void free_capture_list(struct i915_request *request)
@@ -380,25 +343,84 @@ static void free_capture_list(struct i915_request *request)
 	}
 }
 
+static void __retire_engine_request(struct intel_engine_cs *engine,
+				    struct i915_request *rq)
+{
+	GEM_TRACE("%s(%s) fence %llx:%d, global=%d, current %d\n",
+		  __func__, engine->name,
+		  rq->fence.context, rq->fence.seqno,
+		  rq->global_seqno,
+		  intel_engine_get_seqno(engine));
+
+	GEM_BUG_ON(!i915_request_completed(rq));
+
+	local_irq_disable();
+
+	spin_lock(&engine->timeline.lock);
+	GEM_BUG_ON(!list_is_first(&rq->link, &engine->timeline.requests));
+	list_del_init(&rq->link);
+	spin_unlock(&engine->timeline.lock);
+
+	spin_lock(&rq->lock);
+	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
+		dma_fence_signal_locked(&rq->fence);
+	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags))
+		intel_engine_cancel_signaling(rq);
+	if (rq->waitboost) {
+		GEM_BUG_ON(!atomic_read(&rq->i915->gt_pm.rps.num_waiters));
+		atomic_dec(&rq->i915->gt_pm.rps.num_waiters);
+	}
+	spin_unlock(&rq->lock);
+
+	local_irq_enable();
+
+	/*
+	 * The backing object for the context is done after switching to the
+	 * *next* context. Therefore we cannot retire the previous context until
+	 * the next context has already started running. However, since we
+	 * cannot take the required locks at i915_request_submit() we
+	 * defer the unpinning of the active context to now, retirement of
+	 * the subsequent request.
+	 */
+	if (engine->last_retired_context)
+		intel_context_unpin(engine->last_retired_context, engine);
+	engine->last_retired_context = rq->ctx;
+}
+
+static void __retire_engine_upto(struct intel_engine_cs *engine,
+				 struct i915_request *rq)
+{
+	struct i915_request *tmp;
+
+	if (list_empty(&rq->link))
+		return;
+
+	do {
+		tmp = list_first_entry(&engine->timeline.requests,
+				       typeof(*tmp), link);
+
+		GEM_BUG_ON(tmp->engine != engine);
+		__retire_engine_request(engine, tmp);
+	} while (tmp != rq);
+}
+
 static void i915_request_retire(struct i915_request *request)
 {
-	struct intel_engine_cs *engine = request->engine;
 	struct i915_gem_active *active, *next;
 
+	GEM_TRACE("%s fence %llx:%d, global=%d, current %d\n",
+		  request->engine->name,
+		  request->fence.context, request->fence.seqno,
+		  request->global_seqno,
+		  intel_engine_get_seqno(request->engine));
+
 	lockdep_assert_held(&request->i915->drm.struct_mutex);
 	GEM_BUG_ON(!i915_sw_fence_signaled(&request->submit));
 	GEM_BUG_ON(!i915_request_completed(request));
-	GEM_BUG_ON(!request->i915->gt.active_requests);
 
 	trace_i915_request_retire(request);
 
-	spin_lock_irq(&engine->timeline->lock);
-	list_del_init(&request->link);
-	spin_unlock_irq(&engine->timeline->lock);
-
-	unreserve_engine(request->engine);
 	advance_ring(request);
-
 	free_capture_list(request);
 
 	/*
@@ -434,73 +456,74 @@ static void i915_request_retire(struct i915_request *request)
 
 	/* Retirement decays the ban score as it is a sign of ctx progress */
 	atomic_dec_if_positive(&request->ctx->ban_score);
+	intel_context_unpin(request->ctx, request->engine);
 
-	/*
-	 * The backing object for the context is done after switching to the
-	 * *next* context. Therefore we cannot retire the previous context until
-	 * the next context has already started running. However, since we
-	 * cannot take the required locks at i915_request_submit() we
-	 * defer the unpinning of the active context to now, retirement of
-	 * the subsequent request.
-	 */
-	if (engine->last_retired_context)
-		engine->context_unpin(engine, engine->last_retired_context);
-	engine->last_retired_context = request->ctx;
+	__retire_engine_upto(request->engine, request);
 
-	spin_lock_irq(&request->lock);
-	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &request->fence.flags))
-		dma_fence_signal_locked(&request->fence);
-	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
-		intel_engine_cancel_signaling(request);
-	if (request->waitboost) {
-		GEM_BUG_ON(!atomic_read(&request->i915->gt_pm.rps.num_waiters));
-		atomic_dec(&request->i915->gt_pm.rps.num_waiters);
-	}
-	spin_unlock_irq(&request->lock);
+	unreserve_gt(request->i915);
 
-	i915_priotree_fini(request->i915, &request->priotree);
+	i915_sched_node_fini(request->i915, &request->sched);
 	i915_request_put(request);
 }
 
 void i915_request_retire_upto(struct i915_request *rq)
 {
-	struct intel_engine_cs *engine = rq->engine;
+	struct intel_ring *ring = rq->ring;
 	struct i915_request *tmp;
 
+	GEM_TRACE("%s fence %llx:%d, global=%d, current %d\n",
+		  rq->engine->name,
+		  rq->fence.context, rq->fence.seqno,
+		  rq->global_seqno,
+		  intel_engine_get_seqno(rq->engine));
+
 	lockdep_assert_held(&rq->i915->drm.struct_mutex);
 	GEM_BUG_ON(!i915_request_completed(rq));
 
-	if (list_empty(&rq->link))
+	if (list_empty(&rq->ring_link))
 		return;
 
 	do {
-		tmp = list_first_entry(&engine->timeline->requests,
-				       typeof(*tmp), link);
+		tmp = list_first_entry(&ring->request_list,
+				       typeof(*tmp), ring_link);
 
 		i915_request_retire(tmp);
 	} while (tmp != rq);
 }
 
-static u32 timeline_get_seqno(struct intel_timeline *tl)
+static u32 timeline_get_seqno(struct i915_timeline *tl)
 {
 	return ++tl->seqno;
 }
 
+static void move_to_timeline(struct i915_request *request,
+			     struct i915_timeline *timeline)
+{
+	GEM_BUG_ON(request->timeline == &request->engine->timeline);
+	lockdep_assert_held(&request->engine->timeline.lock);
+
+	spin_lock_nested(&request->timeline->lock, SINGLE_DEPTH_NESTING);
+	list_move_tail(&request->link, &timeline->requests);
+	spin_unlock(&request->timeline->lock);
+}
+
 void __i915_request_submit(struct i915_request *request)
 {
 	struct intel_engine_cs *engine = request->engine;
-	struct intel_timeline *timeline;
 	u32 seqno;
 
+	GEM_TRACE("%s fence %llx:%d -> global=%d, current %d\n",
+		  engine->name,
+		  request->fence.context, request->fence.seqno,
+		  engine->timeline.seqno + 1,
+		  intel_engine_get_seqno(engine));
+
 	GEM_BUG_ON(!irqs_disabled());
-	lockdep_assert_held(&engine->timeline->lock);
+	lockdep_assert_held(&engine->timeline.lock);
 
-	/* Transfer from per-context onto the global per-engine timeline */
-	timeline = engine->timeline;
-	GEM_BUG_ON(timeline == request->timeline);
 	GEM_BUG_ON(request->global_seqno);
 
-	seqno = timeline_get_seqno(timeline);
+	seqno = timeline_get_seqno(&engine->timeline);
 	GEM_BUG_ON(!seqno);
 	GEM_BUG_ON(i915_seqno_passed(intel_engine_get_seqno(engine), seqno));
 
@@ -514,9 +537,8 @@ void __i915_request_submit(struct i915_request *request)
 	engine->emit_breadcrumb(request,
 				request->ring->vaddr + request->postfix);
 
-	spin_lock(&request->timeline->lock);
-	list_move_tail(&request->link, &timeline->requests);
-	spin_unlock(&request->timeline->lock);
+	/* Transfer from per-context onto the global per-engine timeline */
+	move_to_timeline(request, &engine->timeline);
 
 	trace_i915_request_execute(request);
 
@@ -529,30 +551,35 @@ void i915_request_submit(struct i915_request *request)
 	unsigned long flags;
 
 	/* Will be called from irq-context when using foreign fences. */
-	spin_lock_irqsave(&engine->timeline->lock, flags);
+	spin_lock_irqsave(&engine->timeline.lock, flags);
 
 	__i915_request_submit(request);
 
-	spin_unlock_irqrestore(&engine->timeline->lock, flags);
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
 void __i915_request_unsubmit(struct i915_request *request)
 {
 	struct intel_engine_cs *engine = request->engine;
-	struct intel_timeline *timeline;
+
+	GEM_TRACE("%s fence %llx:%d <- global=%d, current %d\n",
+		  engine->name,
+		  request->fence.context, request->fence.seqno,
+		  request->global_seqno,
+		  intel_engine_get_seqno(engine));
 
 	GEM_BUG_ON(!irqs_disabled());
-	lockdep_assert_held(&engine->timeline->lock);
+	lockdep_assert_held(&engine->timeline.lock);
 
 	/*
 	 * Only unwind in reverse order, required so that the per-context list
 	 * is kept in seqno/ring order.
 	 */
 	GEM_BUG_ON(!request->global_seqno);
-	GEM_BUG_ON(request->global_seqno != engine->timeline->seqno);
+	GEM_BUG_ON(request->global_seqno != engine->timeline.seqno);
 	GEM_BUG_ON(i915_seqno_passed(intel_engine_get_seqno(engine),
 				     request->global_seqno));
-	engine->timeline->seqno--;
+	engine->timeline.seqno--;
 
 	/* We may be recursing from the signal callback of another i915 fence */
 	spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
@@ -562,12 +589,7 @@ void __i915_request_unsubmit(struct i915_request *request)
 	spin_unlock(&request->lock);
 
 	/* Transfer back from the global per-engine timeline to per-context */
-	timeline = request->timeline;
-	GEM_BUG_ON(timeline == engine->timeline);
-
-	spin_lock(&timeline->lock);
-	list_move(&request->link, &timeline->requests);
-	spin_unlock(&timeline->lock);
+	move_to_timeline(request, request->timeline);
 
 	/*
 	 * We don't need to wake_up any waiters on request->execute, they
@@ -584,11 +606,11 @@ void i915_request_unsubmit(struct i915_request *request)
 	unsigned long flags;
 
 	/* Will be called from irq-context when using foreign fences. */
-	spin_lock_irqsave(&engine->timeline->lock, flags);
+	spin_lock_irqsave(&engine->timeline.lock, flags);
 
 	__i915_request_unsubmit(request);
 
-	spin_unlock_irqrestore(&engine->timeline->lock, flags);
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
 static int __i915_sw_fence_call
@@ -659,12 +681,12 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 	 * GGTT space, so do this first before we reserve a seqno for
 	 * ourselves.
 	 */
-	ring = engine->context_pin(engine, ctx);
+	ring = intel_context_pin(ctx, engine);
 	if (IS_ERR(ring))
 		return ERR_CAST(ring);
 	GEM_BUG_ON(!ring);
 
-	ret = reserve_engine(engine);
+	ret = reserve_gt(i915);
 	if (ret)
 		goto err_unpin;
 
@@ -672,10 +694,10 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 	if (ret)
 		goto err_unreserve;
 
-	/* Move the oldest request to the slab-cache (if not in use!) */
-	rq = list_first_entry_or_null(&engine->timeline->requests,
-				      typeof(*rq), link);
-	if (rq && i915_request_completed(rq))
+	/* Move our oldest request to the slab-cache (if not in use!) */
+	rq = list_first_entry(&ring->request_list, typeof(*rq), ring_link);
+	if (!list_is_last(&rq->ring_link, &ring->request_list) &&
+	    i915_request_completed(rq))
 		i915_request_retire(rq);
 
 	/*
@@ -735,8 +757,13 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 		}
 	}
 
-	rq->timeline = i915_gem_context_lookup_timeline(ctx, engine);
-	GEM_BUG_ON(rq->timeline == engine->timeline);
+	INIT_LIST_HEAD(&rq->active_list);
+	rq->i915 = i915;
+	rq->engine = engine;
+	rq->ctx = ctx;
+	rq->ring = ring;
+	rq->timeline = ring->timeline;
+	GEM_BUG_ON(rq->timeline == &engine->timeline);
 
 	spin_lock_init(&rq->lock);
 	dma_fence_init(&rq->fence,
@@ -749,13 +776,7 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 	i915_sw_fence_init(&i915_request_get(rq)->submit, submit_notify);
 	init_waitqueue_head(&rq->execute);
 
-	i915_priotree_init(&rq->priotree);
-
-	INIT_LIST_HEAD(&rq->active_list);
-	rq->i915 = i915;
-	rq->engine = engine;
-	rq->ctx = ctx;
-	rq->ring = ring;
+	i915_sched_node_init(&rq->sched);
 
 	/* No zalloc, must clear what we need by hand */
 	rq->global_seqno = 0;
@@ -792,6 +813,9 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 	if (ret)
 		goto err_unwind;
 
+	/* Keep a second pin for the dual retirement along engine and ring */
+	__intel_context_pin(rq->ctx, engine);
+
 	/* Check that we didn't interrupt ourselves with a new request */
 	GEM_BUG_ON(rq->timeline->seqno != rq->fence.seqno);
 	return rq;
@@ -801,14 +825,14 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 
 	/* Make sure we didn't add ourselves to external state before freeing */
 	GEM_BUG_ON(!list_empty(&rq->active_list));
-	GEM_BUG_ON(!list_empty(&rq->priotree.signalers_list));
-	GEM_BUG_ON(!list_empty(&rq->priotree.waiters_list));
+	GEM_BUG_ON(!list_empty(&rq->sched.signalers_list));
+	GEM_BUG_ON(!list_empty(&rq->sched.waiters_list));
 
 	kmem_cache_free(i915->requests, rq);
 err_unreserve:
-	unreserve_engine(engine);
+	unreserve_gt(i915);
 err_unpin:
-	engine->context_unpin(engine, ctx);
+	intel_context_unpin(ctx, engine);
 	return ERR_PTR(ret);
 }
 
@@ -824,9 +848,9 @@ i915_request_await_request(struct i915_request *to, struct i915_request *from)
 		return 0;
 
 	if (to->engine->schedule) {
-		ret = i915_priotree_add_dependency(to->i915,
-						   &to->priotree,
-						   &from->priotree);
+		ret = i915_sched_node_add_dependency(to->i915,
+						     &to->sched,
+						     &from->sched);
 		if (ret < 0)
 			return ret;
 	}
@@ -904,7 +928,7 @@ i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
 
 		/* Squash repeated waits to the same timelines */
 		if (fence->context != rq->i915->mm.unordered_timeline &&
-		    intel_timeline_sync_is_later(rq->timeline, fence))
+		    i915_timeline_sync_is_later(rq->timeline, fence))
 			continue;
 
 		if (dma_fence_is_i915(fence))
@@ -918,7 +942,7 @@ i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
 
 		/* Record the latest fence used against each timeline */
 		if (fence->context != rq->i915->mm.unordered_timeline)
-			intel_timeline_sync_set(rq->timeline, fence);
+			i915_timeline_sync_set(rq->timeline, fence);
 	} while (--nchild);
 
 	return 0;
@@ -995,11 +1019,14 @@ void __i915_request_add(struct i915_request *request, bool flush_caches)
 {
 	struct intel_engine_cs *engine = request->engine;
 	struct intel_ring *ring = request->ring;
-	struct intel_timeline *timeline = request->timeline;
+	struct i915_timeline *timeline = request->timeline;
 	struct i915_request *prev;
 	u32 *cs;
 	int err;
 
+	GEM_TRACE("%s fence %llx:%d\n",
+		  engine->name, request->fence.context, request->fence.seqno);
+
 	lockdep_assert_held(&request->i915->drm.struct_mutex);
 	trace_i915_request_add(request);
 
@@ -1054,10 +1081,10 @@ void __i915_request_add(struct i915_request *request, bool flush_caches)
 		i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,
 					     &request->submitq);
 		if (engine->schedule)
-			__i915_priotree_add_dependency(&request->priotree,
-						       &prev->priotree,
-						       &request->dep,
-						       0);
+			__i915_sched_node_add_dependency(&request->sched,
+							 &prev->sched,
+							 &request->dep,
+							 0);
 	}
 
 	spin_lock_irq(&timeline->lock);
@@ -1068,6 +1095,8 @@ void __i915_request_add(struct i915_request *request, bool flush_caches)
 	i915_gem_active_set(&timeline->last_request, request);
 
 	list_add_tail(&request->ring_link, &ring->request_list);
+	if (list_is_first(&request->ring_link, &ring->request_list))
+		list_add(&ring->active_link, &request->i915->gt.active_rings);
 	request->emitted_jiffies = jiffies;
 
 	/*
@@ -1081,12 +1110,11 @@ void __i915_request_add(struct i915_request *request, bool flush_caches)
 	 * decide whether to preempt the entire chain so that it is ready to
 	 * run at the earliest possible convenience.
 	 */
-	rcu_read_lock();
+	local_bh_disable();
+	rcu_read_lock(); /* RCU serialisation for set-wedged protection */
 	if (engine->schedule)
-		engine->schedule(request, request->ctx->priority);
+		engine->schedule(request, &request->ctx->sched);
 	rcu_read_unlock();
-
-	local_bh_disable();
 	i915_sw_fence_commit(&request->submit);
 	local_bh_enable(); /* Kick the execlists tasklet if just scheduled */
 
@@ -1206,11 +1234,13 @@ static bool __i915_spin_request(const struct i915_request *rq,
 
 static bool __i915_wait_request_check_and_reset(struct i915_request *request)
 {
-	if (likely(!i915_reset_handoff(&request->i915->gpu_error)))
+	struct i915_gpu_error *error = &request->i915->gpu_error;
+
+	if (likely(!i915_reset_handoff(error)))
 		return false;
 
 	__set_current_state(TASK_RUNNING);
-	i915_reset(request->i915, 0);
+	i915_reset(request->i915, error->stalled_mask, error->reason);
 	return true;
 }
 
@@ -1373,38 +1403,30 @@ long i915_request_wait(struct i915_request *rq,
 	return timeout;
 }
 
-static void engine_retire_requests(struct intel_engine_cs *engine)
+static void ring_retire_requests(struct intel_ring *ring)
 {
 	struct i915_request *request, *next;
-	u32 seqno = intel_engine_get_seqno(engine);
-	LIST_HEAD(retire);
 
-	spin_lock_irq(&engine->timeline->lock);
 	list_for_each_entry_safe(request, next,
-				 &engine->timeline->requests, link) {
-		if (!i915_seqno_passed(seqno, request->global_seqno))
+				 &ring->request_list, ring_link) {
+		if (!i915_request_completed(request))
 			break;
 
-		list_move_tail(&request->link, &retire);
-	}
-	spin_unlock_irq(&engine->timeline->lock);
-
-	list_for_each_entry_safe(request, next, &retire, link)
 		i915_request_retire(request);
+	}
 }
 
 void i915_retire_requests(struct drm_i915_private *i915)
 {
-	struct intel_engine_cs *engine;
-	enum intel_engine_id id;
+	struct intel_ring *ring, *tmp;
 
 	lockdep_assert_held(&i915->drm.struct_mutex);
 
 	if (!i915->gt.active_requests)
 		return;
 
-	for_each_engine(engine, i915, id)
-		engine_retire_requests(engine);
+	list_for_each_entry_safe(ring, tmp, &i915->gt.active_rings, active_link)
+		ring_retire_requests(ring);
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index 7d6eb82eeb91..eddbd4245cb3 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -28,13 +28,16 @@
 #include <linux/dma-fence.h>
 
 #include "i915_gem.h"
+#include "i915_scheduler.h"
 #include "i915_sw_fence.h"
+#include "i915_scheduler.h"
 
 #include <uapi/drm/i915_drm.h>
 
 struct drm_file;
 struct drm_i915_gem_object;
 struct i915_request;
+struct i915_timeline;
 
 struct intel_wait {
 	struct rb_node node;
@@ -48,44 +51,6 @@ struct intel_signal_node {
 	struct list_head link;
 };
 
-struct i915_dependency {
-	struct i915_priotree *signaler;
-	struct list_head signal_link;
-	struct list_head wait_link;
-	struct list_head dfs_link;
-	unsigned long flags;
-#define I915_DEPENDENCY_ALLOC BIT(0)
-};
-
-/*
- * "People assume that time is a strict progression of cause to effect, but
- * actually, from a nonlinear, non-subjective viewpoint, it's more like a big
- * ball of wibbly-wobbly, timey-wimey ... stuff." -The Doctor, 2015
- *
- * Requests exist in a complex web of interdependencies. Each request
- * has to wait for some other request to complete before it is ready to be run
- * (e.g. we have to wait until the pixels have been rendering into a texture
- * before we can copy from it). We track the readiness of a request in terms
- * of fences, but we also need to keep the dependency tree for the lifetime
- * of the request (beyond the life of an individual fence). We use the tree
- * at various points to reorder the requests whilst keeping the requests
- * in order with respect to their various dependencies.
- */
-struct i915_priotree {
-	struct list_head signalers_list; /* those before us, we depend upon */
-	struct list_head waiters_list; /* those after us, they depend upon us */
-	struct list_head link;
-	int priority;
-};
-
-enum {
-	I915_PRIORITY_MIN = I915_CONTEXT_MIN_USER_PRIORITY - 1,
-	I915_PRIORITY_NORMAL = I915_CONTEXT_DEFAULT_PRIORITY,
-	I915_PRIORITY_MAX = I915_CONTEXT_MAX_USER_PRIORITY + 1,
-
-	I915_PRIORITY_INVALID = INT_MIN
-};
-
 struct i915_capture_list {
 	struct i915_capture_list *next;
 	struct i915_vma *vma;
@@ -131,7 +96,7 @@ struct i915_request {
 	struct i915_gem_context *ctx;
 	struct intel_engine_cs *engine;
 	struct intel_ring *ring;
-	struct intel_timeline *timeline;
+	struct i915_timeline *timeline;
 	struct intel_signal_node signaling;
 
 	/*
@@ -154,7 +119,7 @@ struct i915_request {
 	 * to retirement), i.e. bidirectional dependency information for the
 	 * request not tied to individual fences.
 	 */
-	struct i915_priotree priotree;
+	struct i915_sched_node sched;
 	struct i915_dependency dep;
 
 	/**
@@ -343,10 +308,10 @@ static inline bool i915_request_started(const struct i915_request *rq)
 				 seqno - 1);
 }
 
-static inline bool i915_priotree_signaled(const struct i915_priotree *pt)
+static inline bool i915_sched_node_signaled(const struct i915_sched_node *node)
 {
 	const struct i915_request *rq =
-		container_of(pt, const struct i915_request, priotree);
+		container_of(node, const struct i915_request, sched);
 
 	return i915_request_completed(rq);
 }
diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h
new file mode 100644
index 000000000000..70a42220358d
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_scheduler.h
@@ -0,0 +1,72 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2018 Intel Corporation
+ */
+
+#ifndef _I915_SCHEDULER_H_
+#define _I915_SCHEDULER_H_
+
+#include <linux/bitops.h>
+
+#include <uapi/drm/i915_drm.h>
+
+enum {
+	I915_PRIORITY_MIN = I915_CONTEXT_MIN_USER_PRIORITY - 1,
+	I915_PRIORITY_NORMAL = I915_CONTEXT_DEFAULT_PRIORITY,
+	I915_PRIORITY_MAX = I915_CONTEXT_MAX_USER_PRIORITY + 1,
+
+	I915_PRIORITY_INVALID = INT_MIN
+};
+
+struct i915_sched_attr {
+	/**
+	 * @priority: execution and service priority
+	 *
+	 * All clients are equal, but some are more equal than others!
+	 *
+	 * Requests from a context with a greater (more positive) value of
+	 * @priority will be executed before those with a lower @priority
+	 * value, forming a simple QoS.
+	 *
+	 * The &drm_i915_private.kernel_context is assigned the lowest priority.
+	 */
+	int priority;
+};
+
+/*
+ * "People assume that time is a strict progression of cause to effect, but
+ * actually, from a nonlinear, non-subjective viewpoint, it's more like a big
+ * ball of wibbly-wobbly, timey-wimey ... stuff." -The Doctor, 2015
+ *
+ * Requests exist in a complex web of interdependencies. Each request
+ * has to wait for some other request to complete before it is ready to be run
+ * (e.g. we have to wait until the pixels have been rendering into a texture
+ * before we can copy from it). We track the readiness of a request in terms
+ * of fences, but we also need to keep the dependency tree for the lifetime
+ * of the request (beyond the life of an individual fence). We use the tree
+ * at various points to reorder the requests whilst keeping the requests
+ * in order with respect to their various dependencies.
+ *
+ * There is no active component to the "scheduler". As we know the dependency
+ * DAG of each request, we are able to insert it into a sorted queue when it
+ * is ready, and are able to reorder its portion of the graph to accommodate
+ * dynamic priority changes.
+ */
+struct i915_sched_node {
+	struct list_head signalers_list; /* those before us, we depend upon */
+	struct list_head waiters_list; /* those after us, they depend upon us */
+	struct list_head link;
+	struct i915_sched_attr attr;
+};
+
+struct i915_dependency {
+	struct i915_sched_node *signaler;
+	struct list_head signal_link;
+	struct list_head wait_link;
+	struct list_head dfs_link;
+	unsigned long flags;
+#define I915_DEPENDENCY_ALLOC BIT(0)
+};
+
+#endif /* _I915_SCHEDULER_H_ */
diff --git a/drivers/gpu/drm/i915/i915_timeline.c b/drivers/gpu/drm/i915/i915_timeline.c
new file mode 100644
index 000000000000..4667cc08c416
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_timeline.c
@@ -0,0 +1,105 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2016-2018 Intel Corporation
+ */
+
+#include "i915_drv.h"
+
+#include "i915_timeline.h"
+#include "i915_syncmap.h"
+
+void i915_timeline_init(struct drm_i915_private *i915,
+			struct i915_timeline *timeline,
+			const char *name)
+{
+	lockdep_assert_held(&i915->drm.struct_mutex);
+
+	/*
+	 * Ideally we want a set of engines on a single leaf as we expect
+	 * to mostly be tracking synchronisation between engines. It is not
+	 * a huge issue if this is not the case, but we may want to mitigate
+	 * any page crossing penalties if they become an issue.
+	 */
+	BUILD_BUG_ON(KSYNCMAP < I915_NUM_ENGINES);
+
+	timeline->name = name;
+
+	list_add(&timeline->link, &i915->gt.timelines);
+
+	/* Called during early_init before we know how many engines there are */
+
+	timeline->fence_context = dma_fence_context_alloc(1);
+
+	spin_lock_init(&timeline->lock);
+
+	init_request_active(&timeline->last_request, NULL);
+	INIT_LIST_HEAD(&timeline->requests);
+
+	i915_syncmap_init(&timeline->sync);
+}
+
+/**
+ * i915_timelines_park - called when the driver idles
+ * @i915: the drm_i915_private device
+ *
+ * When the driver is completely idle, we know that all of our sync points
+ * have been signaled and our tracking is then entirely redundant. Any request
+ * to wait upon an older sync point will be completed instantly as we know
+ * the fence is signaled and therefore we will not even look them up in the
+ * sync point map.
+ */
+void i915_timelines_park(struct drm_i915_private *i915)
+{
+	struct i915_timeline *timeline;
+
+	lockdep_assert_held(&i915->drm.struct_mutex);
+
+	list_for_each_entry(timeline, &i915->gt.timelines, link) {
+		/*
+		 * All known fences are completed so we can scrap
+		 * the current sync point tracking and start afresh,
+		 * any attempt to wait upon a previous sync point
+		 * will be skipped as the fence was signaled.
+		 */
+		i915_syncmap_free(&timeline->sync);
+	}
+}
+
+void i915_timeline_fini(struct i915_timeline *timeline)
+{
+	GEM_BUG_ON(!list_empty(&timeline->requests));
+
+	i915_syncmap_free(&timeline->sync);
+
+	list_del(&timeline->link);
+}
+
+struct i915_timeline *
+i915_timeline_create(struct drm_i915_private *i915, const char *name)
+{
+	struct i915_timeline *timeline;
+
+	timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
+	if (!timeline)
+		return ERR_PTR(-ENOMEM);
+
+	i915_timeline_init(i915, timeline, name);
+	kref_init(&timeline->kref);
+
+	return timeline;
+}
+
+void __i915_timeline_free(struct kref *kref)
+{
+	struct i915_timeline *timeline =
+		container_of(kref, typeof(*timeline), kref);
+
+	i915_timeline_fini(timeline);
+	kfree(timeline);
+}
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftests/mock_timeline.c"
+#include "selftests/i915_timeline.c"
+#endif
diff --git a/drivers/gpu/drm/i915/i915_timeline.h b/drivers/gpu/drm/i915/i915_timeline.h
new file mode 100644
index 000000000000..dc2a4632faa7
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_timeline.h
@@ -0,0 +1,126 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef I915_TIMELINE_H
+#define I915_TIMELINE_H
+
+#include <linux/list.h>
+#include <linux/kref.h>
+
+#include "i915_request.h"
+#include "i915_syncmap.h"
+#include "i915_utils.h"
+
+struct i915_timeline {
+	u64 fence_context;
+	u32 seqno;
+
+	spinlock_t lock;
+
+	/**
+	 * List of breadcrumbs associated with GPU requests currently
+	 * outstanding.
+	 */
+	struct list_head requests;
+
+	/* Contains an RCU guarded pointer to the last request. No reference is
+	 * held to the request, users must carefully acquire a reference to
+	 * the request using i915_gem_active_get_request_rcu(), or hold the
+	 * struct_mutex.
+	 */
+	struct i915_gem_active last_request;
+
+	/**
+	 * We track the most recent seqno that we wait on in every context so
+	 * that we only have to emit a new await and dependency on a more
+	 * recent sync point. As the contexts may be executed out-of-order, we
+	 * have to track each individually and can not rely on an absolute
+	 * global_seqno. When we know that all tracked fences are completed
+	 * (i.e. when the driver is idle), we know that the syncmap is
+	 * redundant and we can discard it without loss of generality.
+	 */
+	struct i915_syncmap *sync;
+	/**
+	 * Separately to the inter-context seqno map above, we track the last
+	 * barrier (e.g. semaphore wait) to the global engine timelines. Note
+	 * that this tracks global_seqno rather than the context.seqno, and
+	 * so it is subject to the limitations of hw wraparound and that we
+	 * may need to revoke global_seqno (on pre-emption).
+	 */
+	u32 global_sync[I915_NUM_ENGINES];
+
+	struct list_head link;
+	const char *name;
+
+	struct kref kref;
+};
+
+void i915_timeline_init(struct drm_i915_private *i915,
+			struct i915_timeline *tl,
+			const char *name);
+void i915_timeline_fini(struct i915_timeline *tl);
+
+struct i915_timeline *
+i915_timeline_create(struct drm_i915_private *i915, const char *name);
+
+static inline struct i915_timeline *
+i915_timeline_get(struct i915_timeline *timeline)
+{
+	kref_get(&timeline->kref);
+	return timeline;
+}
+
+void __i915_timeline_free(struct kref *kref);
+static inline void i915_timeline_put(struct i915_timeline *timeline)
+{
+	kref_put(&timeline->kref, __i915_timeline_free);
+}
+
+static inline int __i915_timeline_sync_set(struct i915_timeline *tl,
+					   u64 context, u32 seqno)
+{
+	return i915_syncmap_set(&tl->sync, context, seqno);
+}
+
+static inline int i915_timeline_sync_set(struct i915_timeline *tl,
+					 const struct dma_fence *fence)
+{
+	return __i915_timeline_sync_set(tl, fence->context, fence->seqno);
+}
+
+static inline bool __i915_timeline_sync_is_later(struct i915_timeline *tl,
+						 u64 context, u32 seqno)
+{
+	return i915_syncmap_is_later(&tl->sync, context, seqno);
+}
+
+static inline bool i915_timeline_sync_is_later(struct i915_timeline *tl,
+					       const struct dma_fence *fence)
+{
+	return __i915_timeline_sync_is_later(tl, fence->context, fence->seqno);
+}
+
+void i915_timelines_park(struct drm_i915_private *i915);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index 408827bf5d96..8cc3a256f29d 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -679,45 +679,68 @@ DEFINE_EVENT(i915_request, i915_request_execute,
 	     TP_ARGS(rq)
 );
 
-DECLARE_EVENT_CLASS(i915_request_hw,
-		    TP_PROTO(struct i915_request *rq, unsigned int port),
-		    TP_ARGS(rq, port),
-
-		    TP_STRUCT__entry(
-				     __field(u32, dev)
-				     __field(u32, hw_id)
-				     __field(u32, ring)
-				     __field(u32, ctx)
-				     __field(u32, seqno)
-				     __field(u32, global_seqno)
-				     __field(u32, port)
-				    ),
-
-		    TP_fast_assign(
-				   __entry->dev = rq->i915->drm.primary->index;
-				   __entry->hw_id = rq->ctx->hw_id;
-				   __entry->ring = rq->engine->id;
-				   __entry->ctx = rq->fence.context;
-				   __entry->seqno = rq->fence.seqno;
-				   __entry->global_seqno = rq->global_seqno;
-				   __entry->port = port;
-				  ),
-
-		    TP_printk("dev=%u, hw_id=%u, ring=%u, ctx=%u, seqno=%u, global=%u, port=%u",
-			      __entry->dev, __entry->hw_id, __entry->ring,
-			      __entry->ctx, __entry->seqno,
-			      __entry->global_seqno, __entry->port)
-);
+TRACE_EVENT(i915_request_in,
+	    TP_PROTO(struct i915_request *rq, unsigned int port),
+	    TP_ARGS(rq, port),
+
+	    TP_STRUCT__entry(
+			     __field(u32, dev)
+			     __field(u32, hw_id)
+			     __field(u32, ring)
+			     __field(u32, ctx)
+			     __field(u32, seqno)
+			     __field(u32, global_seqno)
+			     __field(u32, port)
+			     __field(u32, prio)
+			    ),
+
+	    TP_fast_assign(
+			   __entry->dev = rq->i915->drm.primary->index;
+			   __entry->hw_id = rq->ctx->hw_id;
+			   __entry->ring = rq->engine->id;
+			   __entry->ctx = rq->fence.context;
+			   __entry->seqno = rq->fence.seqno;
+			   __entry->global_seqno = rq->global_seqno;
+			   __entry->prio = rq->sched.attr.priority;
+			   __entry->port = port;
+			   ),
 
-DEFINE_EVENT(i915_request_hw, i915_request_in,
-	     TP_PROTO(struct i915_request *rq, unsigned int port),
-	     TP_ARGS(rq, port)
+	    TP_printk("dev=%u, hw_id=%u, ring=%u, ctx=%u, seqno=%u, prio=%u, global=%u, port=%u",
+		      __entry->dev, __entry->hw_id, __entry->ring, __entry->ctx,
+		      __entry->seqno, __entry->prio, __entry->global_seqno,
+		      __entry->port)
 );
 
-DEFINE_EVENT(i915_request, i915_request_out,
-	     TP_PROTO(struct i915_request *rq),
-	     TP_ARGS(rq)
+TRACE_EVENT(i915_request_out,
+	    TP_PROTO(struct i915_request *rq),
+	    TP_ARGS(rq),
+
+	    TP_STRUCT__entry(
+			     __field(u32, dev)
+			     __field(u32, hw_id)
+			     __field(u32, ring)
+			     __field(u32, ctx)
+			     __field(u32, seqno)
+			     __field(u32, global_seqno)
+			     __field(u32, completed)
+			    ),
+
+	    TP_fast_assign(
+			   __entry->dev = rq->i915->drm.primary->index;
+			   __entry->hw_id = rq->ctx->hw_id;
+			   __entry->ring = rq->engine->id;
+			   __entry->ctx = rq->fence.context;
+			   __entry->seqno = rq->fence.seqno;
+			   __entry->global_seqno = rq->global_seqno;
+			   __entry->completed = i915_request_completed(rq);
+			   ),
+
+		    TP_printk("dev=%u, hw_id=%u, ring=%u, ctx=%u, seqno=%u, global=%u, completed?=%u",
+			      __entry->dev, __entry->hw_id, __entry->ring,
+			      __entry->ctx, __entry->seqno,
+			      __entry->global_seqno, __entry->completed)
 );
+
 #else
 #if !defined(TRACE_HEADER_MULTI_READ)
 static inline void
@@ -811,42 +834,6 @@ DEFINE_EVENT(i915_request, i915_request_wait_end,
 	    TP_ARGS(rq)
 );
 
-TRACE_EVENT(i915_flip_request,
-	    TP_PROTO(int plane, struct drm_i915_gem_object *obj),
-
-	    TP_ARGS(plane, obj),
-
-	    TP_STRUCT__entry(
-		    __field(int, plane)
-		    __field(struct drm_i915_gem_object *, obj)
-		    ),
-
-	    TP_fast_assign(
-		    __entry->plane = plane;
-		    __entry->obj = obj;
-		    ),
-
-	    TP_printk("plane=%d, obj=%p", __entry->plane, __entry->obj)
-);
-
-TRACE_EVENT(i915_flip_complete,
-	    TP_PROTO(int plane, struct drm_i915_gem_object *obj),
-
-	    TP_ARGS(plane, obj),
-
-	    TP_STRUCT__entry(
-		    __field(int, plane)
-		    __field(struct drm_i915_gem_object *, obj)
-		    ),
-
-	    TP_fast_assign(
-		    __entry->plane = plane;
-		    __entry->obj = obj;
-		    ),
-
-	    TP_printk("plane=%d, obj=%p", __entry->plane, __entry->obj)
-);
-
 TRACE_EVENT_CONDITION(i915_reg_rw,
 	TP_PROTO(bool write, i915_reg_t reg, u64 val, int len, bool trace),
 
diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h
index 51dbfe5bb418..00165ad55fb3 100644
--- a/drivers/gpu/drm/i915/i915_utils.h
+++ b/drivers/gpu/drm/i915/i915_utils.h
@@ -40,8 +40,8 @@
 #undef WARN_ON_ONCE
 #define WARN_ON_ONCE(x) WARN_ONCE((x), "%s", "WARN_ON_ONCE(" __stringify(x) ")")
 
-#define MISSING_CASE(x) WARN(1, "Missing switch case (%lu) in %s\n", \
-			     (long)(x), __func__)
+#define MISSING_CASE(x) WARN(1, "Missing case (%s == %ld)\n", \
+			     __stringify(x), (long)(x))
 
 #if GCC_VERSION >= 70000
 #define add_overflows(A, B) \
@@ -120,6 +120,12 @@ static inline u64 ptr_to_u64(const void *ptr)
 
 #include <linux/list.h>
 
+static inline int list_is_first(const struct list_head *list,
+				const struct list_head *head)
+{
+	return head->next == list;
+}
+
 static inline void __list_del_many(struct list_head *head,
 				   struct list_head *first)
 {
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 4bda3bd29bf5..9324d476e0a7 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -46,8 +46,6 @@ i915_vma_retire(struct i915_gem_active *active, struct i915_request *rq)
 
 	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 	list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
-	if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma)))
-		WARN_ON(i915_vma_unbind(vma));
 
 	GEM_BUG_ON(!i915_gem_object_is_active(obj));
 	if (--obj->active_count)
@@ -232,7 +230,6 @@ i915_vma_instance(struct drm_i915_gem_object *obj,
 	if (!vma)
 		vma = vma_create(obj, vm, view);
 
-	GEM_BUG_ON(!IS_ERR(vma) && i915_vma_is_closed(vma));
 	GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view));
 	GEM_BUG_ON(!IS_ERR(vma) && vma_lookup(obj, vm, view) != vma);
 	return vma;
@@ -684,13 +681,43 @@ int __i915_vma_do_pin(struct i915_vma *vma,
 	return ret;
 }
 
-static void i915_vma_destroy(struct i915_vma *vma)
+void i915_vma_close(struct i915_vma *vma)
+{
+	lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+
+	GEM_BUG_ON(i915_vma_is_closed(vma));
+	vma->flags |= I915_VMA_CLOSED;
+
+	/*
+	 * We defer actually closing, unbinding and destroying the VMA until
+	 * the next idle point, or if the object is freed in the meantime. By
+	 * postponing the unbind, we allow for it to be resurrected by the
+	 * client, avoiding the work required to rebind the VMA. This is
+	 * advantageous for DRI, where the client/server pass objects
+	 * between themselves, temporarily opening a local VMA to the
+	 * object, and then closing it again. The same object is then reused
+	 * on the next frame (or two, depending on the depth of the swap queue)
+	 * causing us to rebind the VMA once more. This ends up being a lot
+	 * of wasted work for the steady state.
+	 */
+	list_add_tail(&vma->closed_link, &vma->vm->i915->gt.closed_vma);
+}
+
+void i915_vma_reopen(struct i915_vma *vma)
+{
+	lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+
+	if (vma->flags & I915_VMA_CLOSED) {
+		vma->flags &= ~I915_VMA_CLOSED;
+		list_del(&vma->closed_link);
+	}
+}
+
+static void __i915_vma_destroy(struct i915_vma *vma)
 {
 	int i;
 
 	GEM_BUG_ON(vma->node.allocated);
-	GEM_BUG_ON(i915_vma_is_active(vma));
-	GEM_BUG_ON(!i915_vma_is_closed(vma));
 	GEM_BUG_ON(vma->fence);
 
 	for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
@@ -699,6 +726,7 @@ static void i915_vma_destroy(struct i915_vma *vma)
 
 	list_del(&vma->obj_link);
 	list_del(&vma->vm_link);
+	rb_erase(&vma->obj_node, &vma->obj->vma_tree);
 
 	if (!i915_vma_is_ggtt(vma))
 		i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
@@ -706,15 +734,30 @@ static void i915_vma_destroy(struct i915_vma *vma)
 	kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
 }
 
-void i915_vma_close(struct i915_vma *vma)
+void i915_vma_destroy(struct i915_vma *vma)
 {
-	GEM_BUG_ON(i915_vma_is_closed(vma));
-	vma->flags |= I915_VMA_CLOSED;
+	lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
 
-	rb_erase(&vma->obj_node, &vma->obj->vma_tree);
+	GEM_BUG_ON(i915_vma_is_active(vma));
+	GEM_BUG_ON(i915_vma_is_pinned(vma));
+
+	if (i915_vma_is_closed(vma))
+		list_del(&vma->closed_link);
+
+	WARN_ON(i915_vma_unbind(vma));
+	__i915_vma_destroy(vma);
+}
+
+void i915_vma_parked(struct drm_i915_private *i915)
+{
+	struct i915_vma *vma, *next;
 
-	if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma))
-		WARN_ON(i915_vma_unbind(vma));
+	list_for_each_entry_safe(vma, next, &i915->gt.closed_vma, closed_link) {
+		GEM_BUG_ON(!i915_vma_is_closed(vma));
+		i915_vma_destroy(vma);
+	}
+
+	GEM_BUG_ON(!list_empty(&i915->gt.closed_vma));
 }
 
 static void __i915_vma_iounmap(struct i915_vma *vma)
@@ -804,7 +847,7 @@ int i915_vma_unbind(struct i915_vma *vma)
 		return -EBUSY;
 
 	if (!drm_mm_node_allocated(&vma->node))
-		goto destroy;
+		return 0;
 
 	GEM_BUG_ON(obj->bind_count == 0);
 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
@@ -841,10 +884,6 @@ int i915_vma_unbind(struct i915_vma *vma)
 
 	i915_vma_remove(vma);
 
-destroy:
-	if (unlikely(i915_vma_is_closed(vma)))
-		i915_vma_destroy(vma);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 8c5022095418..fc4294cfaa91 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -119,6 +119,8 @@ struct i915_vma {
 	/** This vma's place in the eviction list */
 	struct list_head evict_link;
 
+	struct list_head closed_link;
+
 	/**
 	 * Used for performing relocations during execbuffer insertion.
 	 */
@@ -285,6 +287,8 @@ void i915_vma_revoke_mmap(struct i915_vma *vma);
 int __must_check i915_vma_unbind(struct i915_vma *vma);
 void i915_vma_unlink_ctx(struct i915_vma *vma);
 void i915_vma_close(struct i915_vma *vma);
+void i915_vma_reopen(struct i915_vma *vma);
+void i915_vma_destroy(struct i915_vma *vma);
 
 int __i915_vma_do_pin(struct i915_vma *vma,
 		      u64 size, u64 alignment, u64 flags);
@@ -408,6 +412,8 @@ i915_vma_unpin_fence(struct i915_vma *vma)
 		__i915_vma_unpin_fence(vma);
 }
 
+void i915_vma_parked(struct drm_i915_private *i915);
+
 #define for_each_until(cond) if (cond) break; else
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_atomic.c b/drivers/gpu/drm/i915/intel_atomic.c
index e9fb692076d7..40285d1b91b7 100644
--- a/drivers/gpu/drm/i915/intel_atomic.c
+++ b/drivers/gpu/drm/i915/intel_atomic.c
@@ -227,6 +227,7 @@ int intel_atomic_setup_scalers(struct drm_i915_private *dev_priv,
 	struct intel_crtc_scaler_state *scaler_state =
 		&crtc_state->scaler_state;
 	struct drm_atomic_state *drm_state = crtc_state->base.state;
+	struct intel_atomic_state *intel_state = to_intel_atomic_state(drm_state);
 	int num_scalers_need;
 	int i, j;
 
@@ -304,8 +305,8 @@ int intel_atomic_setup_scalers(struct drm_i915_private *dev_priv,
 				continue;
 			}
 
-			plane_state = intel_atomic_get_existing_plane_state(drm_state,
-									    intel_plane);
+			plane_state = intel_atomic_get_new_plane_state(intel_state,
+								       intel_plane);
 			scaler_id = &plane_state->scaler_id;
 		}
 
@@ -328,8 +329,18 @@ int intel_atomic_setup_scalers(struct drm_i915_private *dev_priv,
 		}
 
 		/* set scaler mode */
-		if (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) {
-			scaler_state->scalers[*scaler_id].mode = 0;
+		if ((INTEL_GEN(dev_priv) >= 9) &&
+		    plane_state && plane_state->base.fb &&
+		    plane_state->base.fb->format->format ==
+		    DRM_FORMAT_NV12) {
+			if (INTEL_GEN(dev_priv) == 9 &&
+			    !IS_GEMINILAKE(dev_priv) &&
+			    !IS_SKYLAKE(dev_priv))
+				scaler_state->scalers[*scaler_id].mode =
+					SKL_PS_SCALER_MODE_NV12;
+			else
+				scaler_state->scalers[*scaler_id].mode =
+					PS_SCALER_MODE_PLANAR;
 		} else if (num_scalers_need == 1 && intel_crtc->pipe != PIPE_C) {
 			/*
 			 * when only 1 scaler is in use on either pipe A or B,
diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c
index 7481ce85746b..6d068786eb41 100644
--- a/drivers/gpu/drm/i915/intel_atomic_plane.c
+++ b/drivers/gpu/drm/i915/intel_atomic_plane.c
@@ -183,11 +183,16 @@ int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_
 	}
 
 	/* FIXME pre-g4x don't work like this */
-	if (intel_state->base.visible)
+	if (state->visible)
 		crtc_state->active_planes |= BIT(intel_plane->id);
 	else
 		crtc_state->active_planes &= ~BIT(intel_plane->id);
 
+	if (state->visible && state->fb->format->format == DRM_FORMAT_NV12)
+		crtc_state->nv12_planes |= BIT(intel_plane->id);
+	else
+		crtc_state->nv12_planes &= ~BIT(intel_plane->id);
+
 	return intel_plane_atomic_calc_changes(old_crtc_state,
 					       &crtc_state->base,
 					       old_plane_state,
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index 447b721c3be9..54270bdde100 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -530,6 +530,7 @@ parse_driver_features(struct drm_i915_private *dev_priv,
 	 */
 	if (!driver->drrs_enabled)
 		dev_priv->vbt.drrs_type = DRRS_NOT_SUPPORTED;
+	dev_priv->vbt.psr.enable = driver->psr_enabled;
 }
 
 static void
@@ -1215,10 +1216,8 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 {
 	struct child_device_config *it, *child = NULL;
 	struct ddi_vbt_port_info *info = &dev_priv->vbt.ddi_port_info[port];
-	uint8_t hdmi_level_shift;
 	int i, j;
 	bool is_dvi, is_hdmi, is_dp, is_edp, is_crt;
-	uint8_t aux_channel, ddc_pin;
 	/* Each DDI port can have more than one value on the "DVO Port" field,
 	 * so look for all the possible values for each port.
 	 */
@@ -1255,8 +1254,6 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 	if (!child)
 		return;
 
-	aux_channel = child->aux_channel;
-
 	is_dvi = child->device_type & DEVICE_TYPE_TMDS_DVI_SIGNALING;
 	is_dp = child->device_type & DEVICE_TYPE_DISPLAYPORT_OUTPUT;
 	is_crt = child->device_type & DEVICE_TYPE_ANALOG_OUTPUT;
@@ -1270,13 +1267,6 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 		is_hdmi = false;
 	}
 
-	if (port == PORT_A && is_dvi) {
-		DRM_DEBUG_KMS("VBT claims port A supports DVI%s, ignoring\n",
-			      is_hdmi ? "/HDMI" : "");
-		is_dvi = false;
-		is_hdmi = false;
-	}
-
 	info->supports_dvi = is_dvi;
 	info->supports_hdmi = is_hdmi;
 	info->supports_dp = is_dp;
@@ -1302,6 +1292,8 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 		DRM_DEBUG_KMS("Port %c is internal DP\n", port_name(port));
 
 	if (is_dvi) {
+		u8 ddc_pin;
+
 		ddc_pin = map_ddc_pin(dev_priv, child->ddc_pin);
 		if (intel_gmbus_is_valid_pin(dev_priv, ddc_pin)) {
 			info->alternate_ddc_pin = ddc_pin;
@@ -1314,14 +1306,14 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 	}
 
 	if (is_dp) {
-		info->alternate_aux_channel = aux_channel;
+		info->alternate_aux_channel = child->aux_channel;
 
 		sanitize_aux_ch(dev_priv, port);
 	}
 
 	if (bdb_version >= 158) {
 		/* The VBT HDMI level shift values match the table we have. */
-		hdmi_level_shift = child->hdmi_level_shifter_value;
+		u8 hdmi_level_shift = child->hdmi_level_shifter_value;
 		DRM_DEBUG_KMS("VBT HDMI level shift for port %c: %d\n",
 			      port_name(port),
 			      hdmi_level_shift);
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index 1f79e7a47433..18e643df523e 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -82,7 +82,7 @@ static unsigned long wait_timeout(void)
 
 static noinline void missed_breadcrumb(struct intel_engine_cs *engine)
 {
-	if (drm_debug & DRM_UT_DRIVER) {
+	if (GEM_SHOW_DEBUG()) {
 		struct drm_printer p = drm_debug_printer(__func__);
 
 		intel_engine_dump(engine, &p,
@@ -130,11 +130,12 @@ static void intel_breadcrumbs_hangcheck(struct timer_list *t)
 
 static void intel_breadcrumbs_fake_irq(struct timer_list *t)
 {
-	struct intel_engine_cs *engine = from_timer(engine, t,
-						    breadcrumbs.fake_irq);
+	struct intel_engine_cs *engine =
+		from_timer(engine, t, breadcrumbs.fake_irq);
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 
-	/* The timer persists in case we cannot enable interrupts,
+	/*
+	 * The timer persists in case we cannot enable interrupts,
 	 * or if we have previously seen seqno/interrupt incoherency
 	 * ("missed interrupt" syndrome, better known as a "missed breadcrumb").
 	 * Here the worker will wake up every jiffie in order to kick the
@@ -148,6 +149,12 @@ static void intel_breadcrumbs_fake_irq(struct timer_list *t)
 	if (!b->irq_armed)
 		return;
 
+	/* If the user has disabled the fake-irq, restore the hangchecking */
+	if (!test_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings)) {
+		mod_timer(&b->hangcheck, wait_timeout());
+		return;
+	}
+
 	mod_timer(&b->fake_irq, jiffies + 1);
 }
 
@@ -730,10 +737,11 @@ static void insert_signal(struct intel_breadcrumbs *b,
 	list_add(&request->signaling.link, &iter->signaling.link);
 }
 
-void intel_engine_enable_signaling(struct i915_request *request, bool wakeup)
+bool intel_engine_enable_signaling(struct i915_request *request, bool wakeup)
 {
 	struct intel_engine_cs *engine = request->engine;
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
+	struct intel_wait *wait = &request->signaling.wait;
 	u32 seqno;
 
 	/*
@@ -750,12 +758,12 @@ void intel_engine_enable_signaling(struct i915_request *request, bool wakeup)
 
 	seqno = i915_request_global_seqno(request);
 	if (!seqno) /* will be enabled later upon execution */
-		return;
+		return true;
 
-	GEM_BUG_ON(request->signaling.wait.seqno);
-	request->signaling.wait.tsk = b->signaler;
-	request->signaling.wait.request = request;
-	request->signaling.wait.seqno = seqno;
+	GEM_BUG_ON(wait->seqno);
+	wait->tsk = b->signaler;
+	wait->request = request;
+	wait->seqno = seqno;
 
 	/*
 	 * Add ourselves into the list of waiters, but registering our
@@ -768,11 +776,15 @@ void intel_engine_enable_signaling(struct i915_request *request, bool wakeup)
 	 */
 	spin_lock(&b->rb_lock);
 	insert_signal(b, request, seqno);
-	wakeup &= __intel_engine_add_wait(engine, &request->signaling.wait);
+	wakeup &= __intel_engine_add_wait(engine, wait);
 	spin_unlock(&b->rb_lock);
 
-	if (wakeup)
+	if (wakeup) {
 		wake_up_process(b->signaler);
+		return !intel_wait_complete(wait);
+	}
+
+	return true;
 }
 
 void intel_engine_cancel_signaling(struct i915_request *request)
@@ -826,8 +838,8 @@ static void cancel_fake_irq(struct intel_engine_cs *engine)
 {
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 
+	del_timer_sync(&b->fake_irq); /* may queue b->hangcheck */
 	del_timer_sync(&b->hangcheck);
-	del_timer_sync(&b->fake_irq);
 	clear_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
 }
 
@@ -835,15 +847,22 @@ void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
 {
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 
-	cancel_fake_irq(engine);
 	spin_lock_irq(&b->irq_lock);
 
+	/*
+	 * Leave the fake_irq timer enabled (if it is running), but clear the
+	 * bit so that it turns itself off on its next wake up and goes back
+	 * to the long hangcheck interval if still required.
+	 */
+	clear_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
+
 	if (b->irq_enabled)
 		irq_enable(engine);
 	else
 		irq_disable(engine);
 
-	/* We set the IRQ_BREADCRUMB bit when we enable the irq presuming the
+	/*
+	 * We set the IRQ_BREADCRUMB bit when we enable the irq presuming the
 	 * GPU is active and may have already executed the MI_USER_INTERRUPT
 	 * before the CPU is ready to receive. However, the engine is currently
 	 * idle (we haven't started it yet), there is no possibility for a
@@ -852,9 +871,6 @@ void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
 	 */
 	clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
 
-	if (b->irq_armed)
-		enable_fake_irq(b);
-
 	spin_unlock_irq(&b->irq_lock);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index c0a8805b277f..de0e22322c76 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -748,6 +748,11 @@ intel_crt_detect(struct drm_connector *connector,
 		      connector->base.id, connector->name,
 		      force);
 
+	if (i915_modparams.load_detect_test) {
+		intel_display_power_get(dev_priv, intel_encoder->power_domain);
+		goto load_detect;
+	}
+
 	/* Skip machines without VGA that falsely report hotplug events */
 	if (dmi_check_system(intel_spurious_crt_detect))
 		return connector_status_disconnected;
@@ -776,11 +781,12 @@ intel_crt_detect(struct drm_connector *connector,
 	 * broken monitor (without edid) to work behind a broken kvm (that fails
 	 * to have the right resistors for HP detection) needs to fix this up.
 	 * For now just bail out. */
-	if (I915_HAS_HOTPLUG(dev_priv) && !i915_modparams.load_detect_test) {
+	if (I915_HAS_HOTPLUG(dev_priv)) {
 		status = connector_status_disconnected;
 		goto out;
 	}
 
+load_detect:
 	if (!force) {
 		status = connector->status;
 		goto out;
diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c
index f9550ea46c26..cf9b600cca79 100644
--- a/drivers/gpu/drm/i915/intel_csr.c
+++ b/drivers/gpu/drm/i915/intel_csr.c
@@ -298,7 +298,10 @@ static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv,
 
 	csr->version = css_header->version;
 
-	if (IS_CANNONLAKE(dev_priv)) {
+	if (csr->fw_path == i915_modparams.dmc_firmware_path) {
+		/* Bypass version check for firmware override. */
+		required_version = csr->version;
+	} else if (IS_CANNONLAKE(dev_priv)) {
 		required_version = CNL_CSR_VERSION_REQUIRED;
 	} else if (IS_GEMINILAKE(dev_priv)) {
 		required_version = GLK_CSR_VERSION_REQUIRED;
@@ -453,7 +456,9 @@ void intel_csr_ucode_init(struct drm_i915_private *dev_priv)
 	if (!HAS_CSR(dev_priv))
 		return;
 
-	if (IS_CANNONLAKE(dev_priv))
+	if (i915_modparams.dmc_firmware_path)
+		csr->fw_path = i915_modparams.dmc_firmware_path;
+	else if (IS_CANNONLAKE(dev_priv))
 		csr->fw_path = I915_CSR_CNL;
 	else if (IS_GEMINILAKE(dev_priv))
 		csr->fw_path = I915_CSR_GLK;
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 8c2d778560f0..b98ac0541f19 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -493,6 +493,125 @@ static const struct cnl_ddi_buf_trans cnl_ddi_translations_edp_1_05V[] = {
 	{ 0x2, 0x7F, 0x3F, 0x00, 0x00 },	/* 400   400      0.0   */
 };
 
+struct icl_combo_phy_ddi_buf_trans {
+	u32 dw2_swing_select;
+	u32 dw2_swing_scalar;
+	u32 dw4_scaling;
+};
+
+/* Voltage Swing Programming for VccIO 0.85V for DP */
+static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_dp_hdmi_0_85V[] = {
+				/* Voltage mV  db    */
+	{ 0x2, 0x98, 0x0018 },	/* 400         0.0   */
+	{ 0x2, 0x98, 0x3015 },	/* 400         3.5   */
+	{ 0x2, 0x98, 0x6012 },	/* 400         6.0   */
+	{ 0x2, 0x98, 0x900F },	/* 400         9.5   */
+	{ 0xB, 0x70, 0x0018 },	/* 600         0.0   */
+	{ 0xB, 0x70, 0x3015 },	/* 600         3.5   */
+	{ 0xB, 0x70, 0x6012 },	/* 600         6.0   */
+	{ 0x5, 0x00, 0x0018 },	/* 800         0.0   */
+	{ 0x5, 0x00, 0x3015 },	/* 800         3.5   */
+	{ 0x6, 0x98, 0x0018 },	/* 1200        0.0   */
+};
+
+/* FIXME - After table is updated in Bspec */
+/* Voltage Swing Programming for VccIO 0.85V for eDP */
+static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_edp_0_85V[] = {
+				/* Voltage mV  db    */
+	{ 0x0, 0x00, 0x00 },	/* 200         0.0   */
+	{ 0x0, 0x00, 0x00 },	/* 200         1.5   */
+	{ 0x0, 0x00, 0x00 },	/* 200         4.0   */
+	{ 0x0, 0x00, 0x00 },	/* 200         6.0   */
+	{ 0x0, 0x00, 0x00 },	/* 250         0.0   */
+	{ 0x0, 0x00, 0x00 },	/* 250         1.5   */
+	{ 0x0, 0x00, 0x00 },	/* 250         4.0   */
+	{ 0x0, 0x00, 0x00 },	/* 300         0.0   */
+	{ 0x0, 0x00, 0x00 },	/* 300         1.5   */
+	{ 0x0, 0x00, 0x00 },	/* 350         0.0   */
+};
+
+/* Voltage Swing Programming for VccIO 0.95V for DP */
+static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_dp_hdmi_0_95V[] = {
+				/* Voltage mV  db    */
+	{ 0x2, 0x98, 0x0018 },	/* 400         0.0   */
+	{ 0x2, 0x98, 0x3015 },	/* 400         3.5   */
+	{ 0x2, 0x98, 0x6012 },	/* 400         6.0   */
+	{ 0x2, 0x98, 0x900F },	/* 400         9.5   */
+	{ 0x4, 0x98, 0x0018 },	/* 600         0.0   */
+	{ 0x4, 0x98, 0x3015 },	/* 600         3.5   */
+	{ 0x4, 0x98, 0x6012 },	/* 600         6.0   */
+	{ 0x5, 0x76, 0x0018 },	/* 800         0.0   */
+	{ 0x5, 0x76, 0x3015 },	/* 800         3.5   */
+	{ 0x6, 0x98, 0x0018 },	/* 1200        0.0   */
+};
+
+/* FIXME - After table is updated in Bspec */
+/* Voltage Swing Programming for VccIO 0.95V for eDP */
+static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_edp_0_95V[] = {
+				/* Voltage mV  db    */
+	{ 0x0, 0x00, 0x00 },	/* 200         0.0   */
+	{ 0x0, 0x00, 0x00 },	/* 200         1.5   */
+	{ 0x0, 0x00, 0x00 },	/* 200         4.0   */
+	{ 0x0, 0x00, 0x00 },	/* 200         6.0   */
+	{ 0x0, 0x00, 0x00 },	/* 250         0.0   */
+	{ 0x0, 0x00, 0x00 },	/* 250         1.5   */
+	{ 0x0, 0x00, 0x00 },	/* 250         4.0   */
+	{ 0x0, 0x00, 0x00 },	/* 300         0.0   */
+	{ 0x0, 0x00, 0x00 },	/* 300         1.5   */
+	{ 0x0, 0x00, 0x00 },	/* 350         0.0   */
+};
+
+/* Voltage Swing Programming for VccIO 1.05V for DP */
+static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_dp_hdmi_1_05V[] = {
+				/* Voltage mV  db    */
+	{ 0x2, 0x98, 0x0018 },	/* 400         0.0   */
+	{ 0x2, 0x98, 0x3015 },	/* 400         3.5   */
+	{ 0x2, 0x98, 0x6012 },	/* 400         6.0   */
+	{ 0x2, 0x98, 0x900F },	/* 400         9.5   */
+	{ 0x4, 0x98, 0x0018 },	/* 600         0.0   */
+	{ 0x4, 0x98, 0x3015 },	/* 600         3.5   */
+	{ 0x4, 0x98, 0x6012 },	/* 600         6.0   */
+	{ 0x5, 0x71, 0x0018 },	/* 800         0.0   */
+	{ 0x5, 0x71, 0x3015 },	/* 800         3.5   */
+	{ 0x6, 0x98, 0x0018 },	/* 1200        0.0   */
+};
+
+/* FIXME - After table is updated in Bspec */
+/* Voltage Swing Programming for VccIO 1.05V for eDP */
+static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_edp_1_05V[] = {
+				/* Voltage mV  db    */
+	{ 0x0, 0x00, 0x00 },	/* 200         0.0   */
+	{ 0x0, 0x00, 0x00 },	/* 200         1.5   */
+	{ 0x0, 0x00, 0x00 },	/* 200         4.0   */
+	{ 0x0, 0x00, 0x00 },	/* 200         6.0   */
+	{ 0x0, 0x00, 0x00 },	/* 250         0.0   */
+	{ 0x0, 0x00, 0x00 },	/* 250         1.5   */
+	{ 0x0, 0x00, 0x00 },	/* 250         4.0   */
+	{ 0x0, 0x00, 0x00 },	/* 300         0.0   */
+	{ 0x0, 0x00, 0x00 },	/* 300         1.5   */
+	{ 0x0, 0x00, 0x00 },	/* 350         0.0   */
+};
+
+struct icl_mg_phy_ddi_buf_trans {
+	u32 cri_txdeemph_override_5_0;
+	u32 cri_txdeemph_override_11_6;
+	u32 cri_txdeemph_override_17_12;
+};
+
+static const struct icl_mg_phy_ddi_buf_trans icl_mg_phy_ddi_translations[] = {
+				/* Voltage swing  pre-emphasis */
+	{ 0x0, 0x1B, 0x00 },	/* 0              0   */
+	{ 0x0, 0x23, 0x08 },	/* 0              1   */
+	{ 0x0, 0x2D, 0x12 },	/* 0              2   */
+	{ 0x0, 0x00, 0x00 },	/* 0              3   */
+	{ 0x0, 0x23, 0x00 },	/* 1              0   */
+	{ 0x0, 0x2B, 0x09 },	/* 1              1   */
+	{ 0x0, 0x2E, 0x11 },	/* 1              2   */
+	{ 0x0, 0x2F, 0x00 },	/* 2              0   */
+	{ 0x0, 0x33, 0x0C },	/* 2              1   */
+	{ 0x0, 0x00, 0x00 },	/* 3              0   */
+};
+
 static const struct ddi_buf_trans *
 bdw_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 {
@@ -751,6 +870,45 @@ cnl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 	}
 }
 
+static const struct icl_combo_phy_ddi_buf_trans *
+icl_get_combo_buf_trans(struct drm_i915_private *dev_priv, enum port port,
+			int type, int *n_entries)
+{
+	u32 voltage = I915_READ(ICL_PORT_COMP_DW3(port)) & VOLTAGE_INFO_MASK;
+
+	if (type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.low_vswing) {
+		switch (voltage) {
+		case VOLTAGE_INFO_0_85V:
+			*n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_edp_0_85V);
+			return icl_combo_phy_ddi_translations_edp_0_85V;
+		case VOLTAGE_INFO_0_95V:
+			*n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_edp_0_95V);
+			return icl_combo_phy_ddi_translations_edp_0_95V;
+		case VOLTAGE_INFO_1_05V:
+			*n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_edp_1_05V);
+			return icl_combo_phy_ddi_translations_edp_1_05V;
+		default:
+			MISSING_CASE(voltage);
+			return NULL;
+		}
+	} else {
+		switch (voltage) {
+		case VOLTAGE_INFO_0_85V:
+			*n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_dp_hdmi_0_85V);
+			return icl_combo_phy_ddi_translations_dp_hdmi_0_85V;
+		case VOLTAGE_INFO_0_95V:
+			*n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_dp_hdmi_0_95V);
+			return icl_combo_phy_ddi_translations_dp_hdmi_0_95V;
+		case VOLTAGE_INFO_1_05V:
+			*n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_dp_hdmi_1_05V);
+			return icl_combo_phy_ddi_translations_dp_hdmi_1_05V;
+		default:
+			MISSING_CASE(voltage);
+			return NULL;
+		}
+	}
+}
+
 static int intel_ddi_hdmi_level(struct drm_i915_private *dev_priv, enum port port)
 {
 	int n_entries, level, default_entry;
@@ -875,7 +1033,7 @@ static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv,
 
 static uint32_t hsw_pll_to_ddi_pll_sel(const struct intel_shared_dpll *pll)
 {
-	switch (pll->id) {
+	switch (pll->info->id) {
 	case DPLL_ID_WRPLL1:
 		return PORT_CLK_SEL_WRPLL1;
 	case DPLL_ID_WRPLL2:
@@ -889,11 +1047,30 @@ static uint32_t hsw_pll_to_ddi_pll_sel(const struct intel_shared_dpll *pll)
 	case DPLL_ID_LCPLL_2700:
 		return PORT_CLK_SEL_LCPLL_2700;
 	default:
-		MISSING_CASE(pll->id);
+		MISSING_CASE(pll->info->id);
 		return PORT_CLK_SEL_NONE;
 	}
 }
 
+static uint32_t icl_pll_to_ddi_pll_sel(struct intel_encoder *encoder,
+				       const struct intel_shared_dpll *pll)
+{
+	const enum intel_dpll_id id = pll->info->id;
+
+	switch (id) {
+	default:
+		MISSING_CASE(id);
+	case DPLL_ID_ICL_DPLL0:
+	case DPLL_ID_ICL_DPLL1:
+		return DDI_CLK_SEL_NONE;
+	case DPLL_ID_ICL_MGPLL1:
+	case DPLL_ID_ICL_MGPLL2:
+	case DPLL_ID_ICL_MGPLL3:
+	case DPLL_ID_ICL_MGPLL4:
+		return DDI_CLK_SEL_MG;
+	}
+}
+
 /* Starting with Haswell, different DDI ports can work in FDI mode for
  * connection to the PCH-located connectors. For this, it is necessary to train
  * both the DDI port and PCH receiver for the desired DDI buffer settings.
@@ -1906,7 +2083,13 @@ u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder)
 	enum port port = encoder->port;
 	int n_entries;
 
-	if (IS_CANNONLAKE(dev_priv)) {
+	if (IS_ICELAKE(dev_priv)) {
+		if (port == PORT_A || port == PORT_B)
+			icl_get_combo_buf_trans(dev_priv, port, encoder->type,
+						&n_entries);
+		else
+			n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations);
+	} else if (IS_CANNONLAKE(dev_priv)) {
 		if (encoder->type == INTEL_OUTPUT_EDP)
 			cnl_get_buf_trans_edp(dev_priv, &n_entries);
 		else
@@ -2063,6 +2246,146 @@ static void cnl_ddi_vswing_sequence(struct intel_encoder *encoder,
 	I915_WRITE(CNL_PORT_TX_DW5_GRP(port), val);
 }
 
+static void icl_ddi_combo_vswing_program(struct drm_i915_private *dev_priv,
+					 u32 level, enum port port, int type)
+{
+	const struct icl_combo_phy_ddi_buf_trans *ddi_translations = NULL;
+	u32 n_entries, val;
+	int ln;
+
+	ddi_translations = icl_get_combo_buf_trans(dev_priv, port, type,
+						   &n_entries);
+	if (!ddi_translations)
+		return;
+
+	if (level >= n_entries) {
+		DRM_DEBUG_KMS("DDI translation not found for level %d. Using %d instead.", level, n_entries - 1);
+		level = n_entries - 1;
+	}
+
+	/* Set PORT_TX_DW5 Rterm Sel to 110b. */
+	val = I915_READ(ICL_PORT_TX_DW5_LN0(port));
+	val &= ~RTERM_SELECT_MASK;
+	val |= RTERM_SELECT(0x6);
+	I915_WRITE(ICL_PORT_TX_DW5_GRP(port), val);
+
+	/* Program PORT_TX_DW5 */
+	val = I915_READ(ICL_PORT_TX_DW5_LN0(port));
+	/* Set DisableTap2 and DisableTap3 if MIPI DSI
+	 * Clear DisableTap2 and DisableTap3 for all other Ports
+	 */
+	if (type == INTEL_OUTPUT_DSI) {
+		val |= TAP2_DISABLE;
+		val |= TAP3_DISABLE;
+	} else {
+		val &= ~TAP2_DISABLE;
+		val &= ~TAP3_DISABLE;
+	}
+	I915_WRITE(ICL_PORT_TX_DW5_GRP(port), val);
+
+	/* Program PORT_TX_DW2 */
+	val = I915_READ(ICL_PORT_TX_DW2_LN0(port));
+	val &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK |
+		 RCOMP_SCALAR_MASK);
+	val |= SWING_SEL_UPPER(ddi_translations[level].dw2_swing_select);
+	val |= SWING_SEL_LOWER(ddi_translations[level].dw2_swing_select);
+	/* Program Rcomp scalar for every table entry */
+	val |= RCOMP_SCALAR(ddi_translations[level].dw2_swing_scalar);
+	I915_WRITE(ICL_PORT_TX_DW2_GRP(port), val);
+
+	/* Program PORT_TX_DW4 */
+	/* We cannot write to GRP. It would overwrite individual loadgen. */
+	for (ln = 0; ln <= 3; ln++) {
+		val = I915_READ(ICL_PORT_TX_DW4_LN(port, ln));
+		val &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK |
+			 CURSOR_COEFF_MASK);
+		val |= ddi_translations[level].dw4_scaling;
+		I915_WRITE(ICL_PORT_TX_DW4_LN(port, ln), val);
+	}
+}
+
+static void icl_combo_phy_ddi_vswing_sequence(struct intel_encoder *encoder,
+					      u32 level,
+					      enum intel_output_type type)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
+	int width = 0;
+	int rate = 0;
+	u32 val;
+	int ln = 0;
+
+	if (type == INTEL_OUTPUT_HDMI) {
+		width = 4;
+		/* Rate is always < than 6GHz for HDMI */
+	} else {
+		struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+
+		width = intel_dp->lane_count;
+		rate = intel_dp->link_rate;
+	}
+
+	/*
+	 * 1. If port type is eDP or DP,
+	 * set PORT_PCS_DW1 cmnkeeper_enable to 1b,
+	 * else clear to 0b.
+	 */
+	val = I915_READ(ICL_PORT_PCS_DW1_LN0(port));
+	if (type == INTEL_OUTPUT_HDMI)
+		val &= ~COMMON_KEEPER_EN;
+	else
+		val |= COMMON_KEEPER_EN;
+	I915_WRITE(ICL_PORT_PCS_DW1_GRP(port), val);
+
+	/* 2. Program loadgen select */
+	/*
+	 * Program PORT_TX_DW4_LN depending on Bit rate and used lanes
+	 * <= 6 GHz and 4 lanes (LN0=0, LN1=1, LN2=1, LN3=1)
+	 * <= 6 GHz and 1,2 lanes (LN0=0, LN1=1, LN2=1, LN3=0)
+	 * > 6 GHz (LN0=0, LN1=0, LN2=0, LN3=0)
+	 */
+	for (ln = 0; ln <= 3; ln++) {
+		val = I915_READ(ICL_PORT_TX_DW4_LN(port, ln));
+		val &= ~LOADGEN_SELECT;
+
+		if ((rate <= 600000 && width == 4 && ln >= 1) ||
+		    (rate <= 600000 && width < 4 && (ln == 1 || ln == 2))) {
+			val |= LOADGEN_SELECT;
+		}
+		I915_WRITE(ICL_PORT_TX_DW4_LN(port, ln), val);
+	}
+
+	/* 3. Set PORT_CL_DW5 SUS Clock Config to 11b */
+	val = I915_READ(ICL_PORT_CL_DW5(port));
+	val |= SUS_CLOCK_CONFIG;
+	I915_WRITE(ICL_PORT_CL_DW5(port), val);
+
+	/* 4. Clear training enable to change swing values */
+	val = I915_READ(ICL_PORT_TX_DW5_LN0(port));
+	val &= ~TX_TRAINING_EN;
+	I915_WRITE(ICL_PORT_TX_DW5_GRP(port), val);
+
+	/* 5. Program swing and de-emphasis */
+	icl_ddi_combo_vswing_program(dev_priv, level, port, type);
+
+	/* 6. Set training enable to trigger update */
+	val = I915_READ(ICL_PORT_TX_DW5_LN0(port));
+	val |= TX_TRAINING_EN;
+	I915_WRITE(ICL_PORT_TX_DW5_GRP(port), val);
+}
+
+static void icl_ddi_vswing_sequence(struct intel_encoder *encoder, u32 level,
+				    enum intel_output_type type)
+{
+	enum port port = encoder->port;
+
+	if (port == PORT_A || port == PORT_B)
+		icl_combo_phy_ddi_vswing_sequence(encoder, level, type);
+	else
+		/* Not Implemented Yet */
+		WARN_ON(1);
+}
+
 static uint32_t translate_signal_level(int signal_levels)
 {
 	int i;
@@ -2094,7 +2417,9 @@ u32 bxt_signal_levels(struct intel_dp *intel_dp)
 	struct intel_encoder *encoder = &dport->base;
 	int level = intel_ddi_dp_level(intel_dp);
 
-	if (IS_CANNONLAKE(dev_priv))
+	if (IS_ICELAKE(dev_priv))
+		icl_ddi_vswing_sequence(encoder, level, encoder->type);
+	else if (IS_CANNONLAKE(dev_priv))
 		cnl_ddi_vswing_sequence(encoder, level, encoder->type);
 	else
 		bxt_ddi_vswing_sequence(encoder, level, encoder->type);
@@ -2115,6 +2440,69 @@ uint32_t ddi_signal_levels(struct intel_dp *intel_dp)
 	return DDI_BUF_TRANS_SELECT(level);
 }
 
+void icl_map_plls_to_ports(struct drm_crtc *crtc,
+			   struct intel_crtc_state *crtc_state,
+			   struct drm_atomic_state *old_state)
+{
+	struct intel_shared_dpll *pll = crtc_state->shared_dpll;
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
+	struct drm_connector_state *conn_state;
+	struct drm_connector *conn;
+	int i;
+
+	for_each_new_connector_in_state(old_state, conn, conn_state, i) {
+		struct intel_encoder *encoder =
+			to_intel_encoder(conn_state->best_encoder);
+		enum port port = encoder->port;
+		uint32_t val;
+
+		if (conn_state->crtc != crtc)
+			continue;
+
+		mutex_lock(&dev_priv->dpll_lock);
+
+		val = I915_READ(DPCLKA_CFGCR0_ICL);
+		WARN_ON((val & DPCLKA_CFGCR0_DDI_CLK_OFF(port)) == 0);
+
+		if (port == PORT_A || port == PORT_B) {
+			val &= ~DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
+			val |= DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, port);
+			I915_WRITE(DPCLKA_CFGCR0_ICL, val);
+			POSTING_READ(DPCLKA_CFGCR0_ICL);
+		}
+
+		val &= ~DPCLKA_CFGCR0_DDI_CLK_OFF(port);
+		I915_WRITE(DPCLKA_CFGCR0_ICL, val);
+
+		mutex_unlock(&dev_priv->dpll_lock);
+	}
+}
+
+void icl_unmap_plls_to_ports(struct drm_crtc *crtc,
+			     struct intel_crtc_state *crtc_state,
+			     struct drm_atomic_state *old_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
+	struct drm_connector_state *old_conn_state;
+	struct drm_connector *conn;
+	int i;
+
+	for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
+		struct intel_encoder *encoder =
+			to_intel_encoder(old_conn_state->best_encoder);
+		enum port port = encoder->port;
+
+		if (old_conn_state->crtc != crtc)
+			continue;
+
+		mutex_lock(&dev_priv->dpll_lock);
+		I915_WRITE(DPCLKA_CFGCR0_ICL,
+			   I915_READ(DPCLKA_CFGCR0_ICL) |
+			   DPCLKA_CFGCR0_DDI_CLK_OFF(port));
+		mutex_unlock(&dev_priv->dpll_lock);
+	}
+}
+
 static void intel_ddi_clk_select(struct intel_encoder *encoder,
 				 const struct intel_shared_dpll *pll)
 {
@@ -2127,11 +2515,15 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder,
 
 	mutex_lock(&dev_priv->dpll_lock);
 
-	if (IS_CANNONLAKE(dev_priv)) {
+	if (IS_ICELAKE(dev_priv)) {
+		if (port >= PORT_C)
+			I915_WRITE(DDI_CLK_SEL(port),
+				   icl_pll_to_ddi_pll_sel(encoder, pll));
+	} else if (IS_CANNONLAKE(dev_priv)) {
 		/* Configure DPCLKA_CFGCR0 to map the DPLL to the DDI. */
 		val = I915_READ(DPCLKA_CFGCR0);
 		val &= ~DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
-		val |= DPCLKA_CFGCR0_DDI_CLK_SEL(pll->id, port);
+		val |= DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, port);
 		I915_WRITE(DPCLKA_CFGCR0, val);
 
 		/*
@@ -2148,7 +2540,7 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder,
 
 		val &= ~(DPLL_CTRL2_DDI_CLK_OFF(port) |
 			 DPLL_CTRL2_DDI_CLK_SEL_MASK(port));
-		val |= (DPLL_CTRL2_DDI_CLK_SEL(pll->id, port) |
+		val |= (DPLL_CTRL2_DDI_CLK_SEL(pll->info->id, port) |
 			DPLL_CTRL2_DDI_SEL_OVERRIDE(port));
 
 		I915_WRITE(DPLL_CTRL2, val);
@@ -2165,14 +2557,18 @@ static void intel_ddi_clk_disable(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum port port = encoder->port;
 
-	if (IS_CANNONLAKE(dev_priv))
+	if (IS_ICELAKE(dev_priv)) {
+		if (port >= PORT_C)
+			I915_WRITE(DDI_CLK_SEL(port), DDI_CLK_SEL_NONE);
+	} else if (IS_CANNONLAKE(dev_priv)) {
 		I915_WRITE(DPCLKA_CFGCR0, I915_READ(DPCLKA_CFGCR0) |
 			   DPCLKA_CFGCR0_DDI_CLK_OFF(port));
-	else if (IS_GEN9_BC(dev_priv))
+	} else if (IS_GEN9_BC(dev_priv)) {
 		I915_WRITE(DPLL_CTRL2, I915_READ(DPLL_CTRL2) |
 			   DPLL_CTRL2_DDI_CLK_OFF(port));
-	else if (INTEL_GEN(dev_priv) < 9)
+	} else if (INTEL_GEN(dev_priv) < 9) {
 		I915_WRITE(PORT_CLK_SEL(port), PORT_CLK_SEL_NONE);
+	}
 }
 
 static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder,
@@ -2197,7 +2593,9 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder,
 
 	intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain);
 
-	if (IS_CANNONLAKE(dev_priv))
+	if (IS_ICELAKE(dev_priv))
+		icl_ddi_vswing_sequence(encoder, level, encoder->type);
+	else if (IS_CANNONLAKE(dev_priv))
 		cnl_ddi_vswing_sequence(encoder, level, encoder->type);
 	else if (IS_GEN9_LP(dev_priv))
 		bxt_ddi_vswing_sequence(encoder, level, encoder->type);
@@ -2205,7 +2603,8 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder,
 		intel_prepare_dp_ddi_buffers(encoder, crtc_state);
 
 	intel_ddi_init_dp_buf_reg(encoder);
-	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
+	if (!is_mst)
+		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
 	intel_dp_start_link_train(intel_dp);
 	if (port != PORT_A || INTEL_GEN(dev_priv) >= 9)
 		intel_dp_stop_link_train(intel_dp);
@@ -2227,7 +2626,9 @@ static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder,
 
 	intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain);
 
-	if (IS_CANNONLAKE(dev_priv))
+	if (IS_ICELAKE(dev_priv))
+		icl_ddi_vswing_sequence(encoder, level, INTEL_OUTPUT_HDMI);
+	else if (IS_CANNONLAKE(dev_priv))
 		cnl_ddi_vswing_sequence(encoder, level, INTEL_OUTPUT_HDMI);
 	else if (IS_GEN9_LP(dev_priv))
 		bxt_ddi_vswing_sequence(encoder, level, INTEL_OUTPUT_HDMI);
@@ -2303,12 +2704,15 @@ static void intel_ddi_post_disable_dp(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
 	struct intel_dp *intel_dp = &dig_port->dp;
+	bool is_mst = intel_crtc_has_type(old_crtc_state,
+					  INTEL_OUTPUT_DP_MST);
 
 	/*
 	 * Power down sink before disabling the port, otherwise we end
 	 * up getting interrupts from the sink on detecting link loss.
 	 */
-	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
+	if (!is_mst)
+		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
 
 	intel_disable_ddi_buf(encoder);
 
@@ -2424,12 +2828,14 @@ static void intel_enable_ddi_hdmi(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
+	struct drm_connector *connector = conn_state->connector;
 	enum port port = encoder->port;
 
-	intel_hdmi_handle_sink_scrambling(encoder,
-					  conn_state->connector,
-					  crtc_state->hdmi_high_tmds_clock_ratio,
-					  crtc_state->hdmi_scrambling);
+	if (!intel_hdmi_handle_sink_scrambling(encoder, connector,
+					       crtc_state->hdmi_high_tmds_clock_ratio,
+					       crtc_state->hdmi_scrambling))
+		DRM_ERROR("[CONNECTOR:%d:%s] Failed to configure sink scrambling/TMDS bit clock ratio\n",
+			  connector->base.id, connector->name);
 
 	/* Display WA #1143: skl,kbl,cfl */
 	if (IS_GEN9_BC(dev_priv)) {
@@ -2520,13 +2926,16 @@ static void intel_disable_ddi_hdmi(struct intel_encoder *encoder,
 				   const struct intel_crtc_state *old_crtc_state,
 				   const struct drm_connector_state *old_conn_state)
 {
+	struct drm_connector *connector = old_conn_state->connector;
+
 	if (old_crtc_state->has_audio)
 		intel_audio_codec_disable(encoder,
 					  old_crtc_state, old_conn_state);
 
-	intel_hdmi_handle_sink_scrambling(encoder,
-					  old_conn_state->connector,
-					  false, false);
+	if (!intel_hdmi_handle_sink_scrambling(encoder, connector,
+					       false, false))
+		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Failed to reset sink scrambling/TMDS bit clock ratio\n",
+			      connector->base.id, connector->name);
 }
 
 static void intel_disable_ddi(struct intel_encoder *encoder,
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 3dd350f7b8e6..0fd13df424cf 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -83,11 +83,11 @@ static void sseu_dump(const struct sseu_dev_info *sseu, struct drm_printer *p)
 {
 	int s;
 
-	drm_printf(p, "slice mask: %04x\n", sseu->slice_mask);
-	drm_printf(p, "slice total: %u\n", hweight8(sseu->slice_mask));
+	drm_printf(p, "slice total: %u, mask=%04x\n",
+		   hweight8(sseu->slice_mask), sseu->slice_mask);
 	drm_printf(p, "subslice total: %u\n", sseu_subslice_total(sseu));
-	for (s = 0; s < ARRAY_SIZE(sseu->subslice_mask); s++) {
-		drm_printf(p, "slice%d %u subslices mask=%04x\n",
+	for (s = 0; s < sseu->max_slices; s++) {
+		drm_printf(p, "slice%d: %u subslices, mask=%04x\n",
 			   s, hweight8(sseu->subslice_mask[s]),
 			   sseu->subslice_mask[s]);
 	}
@@ -158,6 +158,45 @@ static u16 compute_eu_total(const struct sseu_dev_info *sseu)
 	return total;
 }
 
+static void gen11_sseu_info_init(struct drm_i915_private *dev_priv)
+{
+	struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu;
+	u8 s_en;
+	u32 ss_en, ss_en_mask;
+	u8 eu_en;
+	int s;
+
+	sseu->max_slices = 1;
+	sseu->max_subslices = 8;
+	sseu->max_eus_per_subslice = 8;
+
+	s_en = I915_READ(GEN11_GT_SLICE_ENABLE) & GEN11_GT_S_ENA_MASK;
+	ss_en = ~I915_READ(GEN11_GT_SUBSLICE_DISABLE);
+	ss_en_mask = BIT(sseu->max_subslices) - 1;
+	eu_en = ~(I915_READ(GEN11_EU_DISABLE) & GEN11_EU_DIS_MASK);
+
+	for (s = 0; s < sseu->max_slices; s++) {
+		if (s_en & BIT(s)) {
+			int ss_idx = sseu->max_subslices * s;
+			int ss;
+
+			sseu->slice_mask |= BIT(s);
+			sseu->subslice_mask[s] = (ss_en >> ss_idx) & ss_en_mask;
+			for (ss = 0; ss < sseu->max_subslices; ss++) {
+				if (sseu->subslice_mask[s] & BIT(ss))
+					sseu_set_eus(sseu, s, ss, eu_en);
+			}
+		}
+	}
+	sseu->eu_per_subslice = hweight8(eu_en);
+	sseu->eu_total = compute_eu_total(sseu);
+
+	/* ICL has no power gating restrictions. */
+	sseu->has_slice_pg = 1;
+	sseu->has_subslice_pg = 1;
+	sseu->has_eu_pg = 1;
+}
+
 static void gen10_sseu_info_init(struct drm_i915_private *dev_priv)
 {
 	struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu;
@@ -557,6 +596,52 @@ static u32 read_reference_ts_freq(struct drm_i915_private *dev_priv)
 	return base_freq + frac_freq;
 }
 
+static u32 gen10_get_crystal_clock_freq(struct drm_i915_private *dev_priv,
+					u32 rpm_config_reg)
+{
+	u32 f19_2_mhz = 19200;
+	u32 f24_mhz = 24000;
+	u32 crystal_clock = (rpm_config_reg &
+			     GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK) >>
+			    GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT;
+
+	switch (crystal_clock) {
+	case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ:
+		return f19_2_mhz;
+	case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ:
+		return f24_mhz;
+	default:
+		MISSING_CASE(crystal_clock);
+		return 0;
+	}
+}
+
+static u32 gen11_get_crystal_clock_freq(struct drm_i915_private *dev_priv,
+					u32 rpm_config_reg)
+{
+	u32 f19_2_mhz = 19200;
+	u32 f24_mhz = 24000;
+	u32 f25_mhz = 25000;
+	u32 f38_4_mhz = 38400;
+	u32 crystal_clock = (rpm_config_reg &
+			     GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK) >>
+			    GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT;
+
+	switch (crystal_clock) {
+	case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ:
+		return f24_mhz;
+	case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ:
+		return f19_2_mhz;
+	case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_38_4_MHZ:
+		return f38_4_mhz;
+	case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_25_MHZ:
+		return f25_mhz;
+	default:
+		MISSING_CASE(crystal_clock);
+		return 0;
+	}
+}
+
 static u32 read_timestamp_frequency(struct drm_i915_private *dev_priv)
 {
 	u32 f12_5_mhz = 12500;
@@ -597,10 +682,9 @@ static u32 read_timestamp_frequency(struct drm_i915_private *dev_priv)
 		}
 
 		return freq;
-	} else if (INTEL_GEN(dev_priv) <= 10) {
+	} else if (INTEL_GEN(dev_priv) <= 11) {
 		u32 ctc_reg = I915_READ(CTC_MODE);
 		u32 freq = 0;
-		u32 rpm_config_reg = 0;
 
 		/* First figure out the reference frequency. There are 2 ways
 		 * we can compute the frequency, either through the
@@ -610,20 +694,14 @@ static u32 read_timestamp_frequency(struct drm_i915_private *dev_priv)
 		if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) {
 			freq = read_reference_ts_freq(dev_priv);
 		} else {
-			u32 crystal_clock;
-
-			rpm_config_reg = I915_READ(RPM_CONFIG0);
-			crystal_clock = (rpm_config_reg &
-					 GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK) >>
-				GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT;
-			switch (crystal_clock) {
-			case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ:
-				freq = f19_2_mhz;
-				break;
-			case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ:
-				freq = f24_mhz;
-				break;
-			}
+			u32 rpm_config_reg = I915_READ(RPM_CONFIG0);
+
+			if (INTEL_GEN(dev_priv) <= 10)
+				freq = gen10_get_crystal_clock_freq(dev_priv,
+								rpm_config_reg);
+			else
+				freq = gen11_get_crystal_clock_freq(dev_priv,
+								rpm_config_reg);
 
 			/* Now figure out how the command stream's timestamp
 			 * register increments from this frequency (it might
@@ -768,8 +846,10 @@ void intel_device_info_runtime_init(struct intel_device_info *info)
 		broadwell_sseu_info_init(dev_priv);
 	else if (INTEL_GEN(dev_priv) == 9)
 		gen9_sseu_info_init(dev_priv);
-	else if (INTEL_GEN(dev_priv) >= 10)
+	else if (INTEL_GEN(dev_priv) == 10)
 		gen10_sseu_info_init(dev_priv);
+	else if (INTEL_GEN(dev_priv) >= 11)
+		gen11_sseu_info_init(dev_priv);
 
 	/* Initialize command stream timestamp frequency */
 	info->cs_timestamp_frequency_khz = read_timestamp_frequency(dev_priv);
@@ -780,3 +860,50 @@ void intel_driver_caps_print(const struct intel_driver_caps *caps,
 {
 	drm_printf(p, "scheduler: %x\n", caps->scheduler);
 }
+
+/*
+ * Determine which engines are fused off in our particular hardware. Since the
+ * fuse register is in the blitter powerwell, we need forcewake to be ready at
+ * this point (but later we need to prune the forcewake domains for engines that
+ * are indeed fused off).
+ */
+void intel_device_info_init_mmio(struct drm_i915_private *dev_priv)
+{
+	struct intel_device_info *info = mkwrite_device_info(dev_priv);
+	u8 vdbox_disable, vebox_disable;
+	u32 media_fuse;
+	int i;
+
+	if (INTEL_GEN(dev_priv) < 11)
+		return;
+
+	media_fuse = I915_READ(GEN11_GT_VEBOX_VDBOX_DISABLE);
+
+	vdbox_disable = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK;
+	vebox_disable = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >>
+			GEN11_GT_VEBOX_DISABLE_SHIFT;
+
+	DRM_DEBUG_DRIVER("vdbox disable: %04x\n", vdbox_disable);
+	for (i = 0; i < I915_MAX_VCS; i++) {
+		if (!HAS_ENGINE(dev_priv, _VCS(i)))
+			continue;
+
+		if (!(BIT(i) & vdbox_disable))
+			continue;
+
+		info->ring_mask &= ~ENGINE_MASK(_VCS(i));
+		DRM_DEBUG_DRIVER("vcs%u fused off\n", i);
+	}
+
+	DRM_DEBUG_DRIVER("vebox disable: %04x\n", vebox_disable);
+	for (i = 0; i < I915_MAX_VECS; i++) {
+		if (!HAS_ENGINE(dev_priv, _VECS(i)))
+			continue;
+
+		if (!(BIT(i) & vebox_disable))
+			continue;
+
+		info->ring_mask &= ~ENGINE_MASK(_VECS(i));
+		DRM_DEBUG_DRIVER("vecs%u fused off\n", i);
+	}
+}
diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h
index 0835752c8b22..933e31669557 100644
--- a/drivers/gpu/drm/i915/intel_device_info.h
+++ b/drivers/gpu/drm/i915/intel_device_info.h
@@ -114,7 +114,7 @@ enum intel_platform {
 	func(has_ipc);
 
 #define GEN_MAX_SLICES		(6) /* CNL upper bound */
-#define GEN_MAX_SUBSLICES	(7)
+#define GEN_MAX_SUBSLICES	(8) /* ICL upper bound */
 
 struct sseu_dev_info {
 	u8 slice_mask;
@@ -247,6 +247,8 @@ void intel_device_info_dump_runtime(const struct intel_device_info *info,
 void intel_device_info_dump_topology(const struct sseu_dev_info *sseu,
 				     struct drm_printer *p);
 
+void intel_device_info_init_mmio(struct drm_i915_private *dev_priv);
+
 void intel_driver_caps_print(const struct intel_driver_caps *caps,
 			     struct drm_printer *p);
 
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 56004ffbd8bb..ad588d564198 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -88,6 +88,22 @@ static const uint32_t skl_primary_formats[] = {
 	DRM_FORMAT_VYUY,
 };
 
+static const uint32_t skl_pri_planar_formats[] = {
+	DRM_FORMAT_C8,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_NV12,
+};
+
 static const uint64_t skl_format_modifiers_noccs[] = {
 	I915_FORMAT_MOD_Yf_TILED,
 	I915_FORMAT_MOD_Y_TILED,
@@ -488,6 +504,33 @@ static const struct intel_limit intel_limits_bxt = {
 	.p2 = { .p2_slow = 1, .p2_fast = 20 },
 };
 
+static void
+skl_wa_528(struct drm_i915_private *dev_priv, int pipe, bool enable)
+{
+	if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
+		return;
+
+	if (enable)
+		I915_WRITE(CHICKEN_PIPESL_1(pipe), HSW_FBCQ_DIS);
+	else
+		I915_WRITE(CHICKEN_PIPESL_1(pipe), 0);
+}
+
+static void
+skl_wa_clkgate(struct drm_i915_private *dev_priv, int pipe, bool enable)
+{
+	if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
+		return;
+
+	if (enable)
+		I915_WRITE(CLKGATE_DIS_PSL(pipe),
+			   DUPS1_GATING_DIS | DUPS2_GATING_DIS);
+	else
+		I915_WRITE(CLKGATE_DIS_PSL(pipe),
+			   I915_READ(CLKGATE_DIS_PSL(pipe)) &
+			   ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
+}
+
 static bool
 needs_modeset(const struct drm_crtc_state *state)
 {
@@ -2657,11 +2700,13 @@ static int i9xx_format_to_fourcc(int format)
 	}
 }
 
-static int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
+int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
 {
 	switch (format) {
 	case PLANE_CTL_FORMAT_RGB_565:
 		return DRM_FORMAT_RGB565;
+	case PLANE_CTL_FORMAT_NV12:
+		return DRM_FORMAT_NV12;
 	default:
 	case PLANE_CTL_FORMAT_XRGB_8888:
 		if (rgb_order) {
@@ -2824,7 +2869,7 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
 			continue;
 
 		if (intel_plane_ggtt_offset(state) == plane_config->base) {
-			fb = c->primary->fb;
+			fb = state->base.fb;
 			drm_framebuffer_get(fb);
 			goto valid_fb;
 		}
@@ -2858,6 +2903,9 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
 		return;
 	}
 
+	obj = intel_fb_obj(fb);
+	intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
+
 	plane_state->src_x = 0;
 	plane_state->src_y = 0;
 	plane_state->src_w = fb->width << 16;
@@ -2871,7 +2919,6 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
 	intel_state->base.src = drm_plane_state_src(plane_state);
 	intel_state->base.dst = drm_plane_state_dest(plane_state);
 
-	obj = intel_fb_obj(fb);
 	if (i915_gem_object_is_tiled(obj))
 		dev_priv->preserve_bios_swizzle = true;
 
@@ -3071,6 +3118,29 @@ static int skl_check_main_surface(const struct intel_crtc_state *crtc_state,
 	return 0;
 }
 
+static int
+skl_check_nv12_surface(const struct intel_crtc_state *crtc_state,
+		       struct intel_plane_state *plane_state)
+{
+	/* Display WA #1106 */
+	if (plane_state->base.rotation !=
+	    (DRM_MODE_REFLECT_X | DRM_MODE_ROTATE_90) &&
+	    plane_state->base.rotation != DRM_MODE_ROTATE_270)
+		return 0;
+
+	/*
+	 * src coordinates are rotated here.
+	 * We check height but report it as width
+	 */
+	if (((drm_rect_height(&plane_state->base.src) >> 16) % 4) != 0) {
+		DRM_DEBUG_KMS("src width must be multiple "
+			      "of 4 for rotated NV12\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
 {
 	const struct drm_framebuffer *fb = plane_state->base.fb;
@@ -3154,6 +3224,9 @@ int skl_check_plane_surface(const struct intel_crtc_state *crtc_state,
 	 * the main surface setup depends on it.
 	 */
 	if (fb->format->format == DRM_FORMAT_NV12) {
+		ret = skl_check_nv12_surface(crtc_state, plane_state);
+		if (ret)
+			return ret;
 		ret = skl_check_nv12_aux_surface(plane_state);
 		if (ret)
 			return ret;
@@ -3464,6 +3537,8 @@ static u32 skl_plane_ctl_format(uint32_t pixel_format)
 		return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
 	case DRM_FORMAT_VYUY:
 		return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
+	case DRM_FORMAT_NV12:
+		return PLANE_CTL_FORMAT_NV12;
 	default:
 		MISSING_CASE(pixel_format);
 	}
@@ -3602,15 +3677,24 @@ u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
 			const struct intel_plane_state *plane_state)
 {
+	struct drm_i915_private *dev_priv =
+		to_i915(plane_state->base.plane->dev);
 	const struct drm_framebuffer *fb = plane_state->base.fb;
 	u32 plane_color_ctl = 0;
 
-	plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
-	plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
+	if (INTEL_GEN(dev_priv) < 11) {
+		plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
+		plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
+	}
 	plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
 	plane_color_ctl |= glk_plane_color_ctl_alpha(fb->format->format);
 
 	if (intel_format_is_yuv(fb->format->format)) {
+		if (fb->format->format == DRM_FORMAT_NV12) {
+			plane_color_ctl |=
+				PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
+			goto out;
+		}
 		if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
 			plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
 		else
@@ -3619,7 +3703,7 @@ u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
 		if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
 			plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
 	}
-
+out:
 	return plane_color_ctl;
 }
 
@@ -3675,7 +3759,6 @@ void intel_prepare_reset(struct drm_i915_private *dev_priv)
 	struct drm_atomic_state *state;
 	int ret;
 
-
 	/* reset doesn't touch the display */
 	if (!i915_modparams.force_reset_modeset_test &&
 	    !gpu_reset_clobbers_display(dev_priv))
@@ -3729,19 +3812,17 @@ void intel_finish_reset(struct drm_i915_private *dev_priv)
 {
 	struct drm_device *dev = &dev_priv->drm;
 	struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
-	struct drm_atomic_state *state = dev_priv->modeset_restore_state;
+	struct drm_atomic_state *state;
 	int ret;
 
 	/* reset doesn't touch the display */
-	if (!i915_modparams.force_reset_modeset_test &&
-	    !gpu_reset_clobbers_display(dev_priv))
+	if (!test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
 		return;
 
+	state = fetch_and_zero(&dev_priv->modeset_restore_state);
 	if (!state)
 		goto unlock;
 
-	dev_priv->modeset_restore_state = NULL;
-
 	/* reset doesn't touch the display */
 	if (!gpu_reset_clobbers_display(dev_priv)) {
 		/* for testing only restore the display */
@@ -4703,7 +4784,9 @@ static void cpt_verify_modeset(struct drm_device *dev, int pipe)
 static int
 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
 		  unsigned int scaler_user, int *scaler_id,
-		  int src_w, int src_h, int dst_w, int dst_h)
+		  int src_w, int src_h, int dst_w, int dst_h,
+		  bool plane_scaler_check,
+		  uint32_t pixel_format)
 {
 	struct intel_crtc_scaler_state *scaler_state =
 		&crtc_state->scaler_state;
@@ -4721,6 +4804,10 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
 	 */
 	need_scaling = src_w != dst_w || src_h != dst_h;
 
+	if (plane_scaler_check)
+		if (pixel_format == DRM_FORMAT_NV12)
+			need_scaling = true;
+
 	if (crtc_state->ycbcr420 && scaler_user == SKL_CRTC_INDEX)
 		need_scaling = true;
 
@@ -4760,12 +4847,21 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
 		return 0;
 	}
 
+	if (plane_scaler_check && pixel_format == DRM_FORMAT_NV12 &&
+	    (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
+		DRM_DEBUG_KMS("NV12: src dimensions not met\n");
+		return -EINVAL;
+	}
+
 	/* range checks */
 	if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
-		dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
-
-		src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
-		dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H) {
+	    dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
+	    (IS_GEN11(dev_priv) &&
+	     (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
+	      dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
+	    (!IS_GEN11(dev_priv) &&
+	     (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
+	      dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H)))	{
 		DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
 			"size is out of scaler range\n",
 			intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
@@ -4796,9 +4892,10 @@ int skl_update_scaler_crtc(struct intel_crtc_state *state)
 	const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
 
 	return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
-		&state->scaler_state.scaler_id,
-		state->pipe_src_w, state->pipe_src_h,
-		adjusted_mode->crtc_hdisplay, adjusted_mode->crtc_vdisplay);
+				 &state->scaler_state.scaler_id,
+				 state->pipe_src_w, state->pipe_src_h,
+				 adjusted_mode->crtc_hdisplay,
+				 adjusted_mode->crtc_vdisplay, false, 0);
 }
 
 /**
@@ -4827,7 +4924,8 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
 				drm_rect_width(&plane_state->base.src) >> 16,
 				drm_rect_height(&plane_state->base.src) >> 16,
 				drm_rect_width(&plane_state->base.dst),
-				drm_rect_height(&plane_state->base.dst));
+				drm_rect_height(&plane_state->base.dst),
+				fb ? true : false, fb ? fb->format->format : 0);
 
 	if (ret || plane_state->scaler_id < 0)
 		return ret;
@@ -4853,6 +4951,7 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
 	case DRM_FORMAT_YVYU:
 	case DRM_FORMAT_UYVY:
 	case DRM_FORMAT_VYUY:
+	case DRM_FORMAT_NV12:
 		break;
 	default:
 		DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
@@ -5096,16 +5195,34 @@ static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_s
 	return !old_crtc_state->ips_enabled;
 }
 
+static bool needs_nv12_wa(struct drm_i915_private *dev_priv,
+			  const struct intel_crtc_state *crtc_state)
+{
+	if (!crtc_state->nv12_planes)
+		return false;
+
+	if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
+		return false;
+
+	if ((INTEL_GEN(dev_priv) == 9 && !IS_GEMINILAKE(dev_priv)) ||
+	    IS_CANNONLAKE(dev_priv))
+		return true;
+
+	return false;
+}
+
 static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
+	struct drm_device *dev = crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_atomic_state *old_state = old_crtc_state->base.state;
 	struct intel_crtc_state *pipe_config =
 		intel_atomic_get_new_crtc_state(to_intel_atomic_state(old_state),
 						crtc);
 	struct drm_plane *primary = crtc->base.primary;
-	struct drm_plane_state *old_pri_state =
-		drm_atomic_get_existing_plane_state(old_state, primary);
+	struct drm_plane_state *old_primary_state =
+		drm_atomic_get_old_plane_state(old_state, primary);
 
 	intel_frontbuffer_flip(to_i915(crtc->base.dev), pipe_config->fb_bits);
 
@@ -5115,20 +5232,24 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
 	if (hsw_post_update_enable_ips(old_crtc_state, pipe_config))
 		hsw_enable_ips(pipe_config);
 
-	if (old_pri_state) {
-		struct intel_plane_state *primary_state =
-			intel_atomic_get_new_plane_state(to_intel_atomic_state(old_state),
-							 to_intel_plane(primary));
-		struct intel_plane_state *old_primary_state =
-			to_intel_plane_state(old_pri_state);
+	if (old_primary_state) {
+		struct drm_plane_state *new_primary_state =
+			drm_atomic_get_new_plane_state(old_state, primary);
 
 		intel_fbc_post_update(crtc);
 
-		if (primary_state->base.visible &&
+		if (new_primary_state->visible &&
 		    (needs_modeset(&pipe_config->base) ||
-		     !old_primary_state->base.visible))
+		     !old_primary_state->visible))
 			intel_post_enable_primary(&crtc->base, pipe_config);
 	}
+
+	/* Display WA 827 */
+	if (needs_nv12_wa(dev_priv, old_crtc_state) &&
+	    !needs_nv12_wa(dev_priv, pipe_config)) {
+		skl_wa_clkgate(dev_priv, crtc->pipe, false);
+		skl_wa_528(dev_priv, crtc->pipe, false);
+	}
 }
 
 static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
@@ -5139,8 +5260,8 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_atomic_state *old_state = old_crtc_state->base.state;
 	struct drm_plane *primary = crtc->base.primary;
-	struct drm_plane_state *old_pri_state =
-		drm_atomic_get_existing_plane_state(old_state, primary);
+	struct drm_plane_state *old_primary_state =
+		drm_atomic_get_old_plane_state(old_state, primary);
 	bool modeset = needs_modeset(&pipe_config->base);
 	struct intel_atomic_state *old_intel_state =
 		to_intel_atomic_state(old_state);
@@ -5148,23 +5269,28 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
 	if (hsw_pre_update_disable_ips(old_crtc_state, pipe_config))
 		hsw_disable_ips(old_crtc_state);
 
-	if (old_pri_state) {
-		struct intel_plane_state *primary_state =
+	if (old_primary_state) {
+		struct intel_plane_state *new_primary_state =
 			intel_atomic_get_new_plane_state(old_intel_state,
 							 to_intel_plane(primary));
-		struct intel_plane_state *old_primary_state =
-			to_intel_plane_state(old_pri_state);
 
-		intel_fbc_pre_update(crtc, pipe_config, primary_state);
+		intel_fbc_pre_update(crtc, pipe_config, new_primary_state);
 		/*
 		 * Gen2 reports pipe underruns whenever all planes are disabled.
 		 * So disable underrun reporting before all the planes get disabled.
 		 */
-		if (IS_GEN2(dev_priv) && old_primary_state->base.visible &&
-		    (modeset || !primary_state->base.visible))
+		if (IS_GEN2(dev_priv) && old_primary_state->visible &&
+		    (modeset || !new_primary_state->base.visible))
 			intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
 	}
 
+	/* Display WA 827 */
+	if (!needs_nv12_wa(dev_priv, old_crtc_state) &&
+	    needs_nv12_wa(dev_priv, pipe_config)) {
+		skl_wa_clkgate(dev_priv, crtc->pipe, true);
+		skl_wa_528(dev_priv, crtc->pipe, true);
+	}
+
 	/*
 	 * Vblank time updates from the shadow to live plane control register
 	 * are blocked if the memory self-refresh mode is active at that
@@ -5499,6 +5625,9 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 	if (intel_crtc->config->shared_dpll)
 		intel_enable_shared_dpll(intel_crtc);
 
+	if (INTEL_GEN(dev_priv) >= 11)
+		icl_map_plls_to_ports(crtc, pipe_config, old_state);
+
 	if (intel_crtc_has_dp_encoder(intel_crtc->config))
 		intel_dp_set_m_n(intel_crtc, M1_N1);
 
@@ -5696,6 +5825,9 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
 		intel_ddi_disable_pipe_clock(intel_crtc->config);
 
 	intel_encoders_post_disable(crtc, old_crtc_state, old_state);
+
+	if (INTEL_GEN(dev_priv) >= 11)
+		icl_unmap_plls_to_ports(crtc, old_crtc_state, old_state);
 }
 
 static void i9xx_pfit_enable(struct intel_crtc *crtc)
@@ -8766,8 +8898,8 @@ static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
 			intel_get_shared_dpll_by_id(dev_priv, pll_id);
 		pll = pipe_config->shared_dpll;
 
-		WARN_ON(!pll->funcs.get_hw_state(dev_priv, pll,
-						 &pipe_config->dpll_hw_state));
+		WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
+						&pipe_config->dpll_hw_state));
 
 		tmp = pipe_config->dpll_hw_state.dpll;
 		pipe_config->pixel_multiplier =
@@ -9243,8 +9375,8 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
 
 	pll = pipe_config->shared_dpll;
 	if (pll) {
-		WARN_ON(!pll->funcs.get_hw_state(dev_priv, pll,
-						 &pipe_config->dpll_hw_state));
+		WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
+						&pipe_config->dpll_hw_state));
 	}
 
 	/*
@@ -9974,6 +10106,8 @@ int intel_get_load_detect_pipe(struct drm_connector *connector,
 	ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
 	if (!ret)
 		ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
+	if (!ret)
+		ret = drm_atomic_add_affected_planes(restore_state, crtc);
 	if (ret) {
 		DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
 		goto fail;
@@ -10773,7 +10907,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 		struct drm_connector_state *connector_state;
 		struct intel_encoder *encoder;
 
-		connector_state = drm_atomic_get_existing_connector_state(state, connector);
+		connector_state = drm_atomic_get_new_connector_state(state, connector);
 		if (!connector_state)
 			connector_state = connector->state;
 
@@ -11085,39 +11219,42 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 		(current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
 		!(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED);
 
-#define PIPE_CONF_CHECK_X(name)	\
+#define PIPE_CONF_CHECK_X(name) do { \
 	if (current_config->name != pipe_config->name) { \
 		pipe_config_err(adjust, __stringify(name), \
 			  "(expected 0x%08x, found 0x%08x)\n", \
 			  current_config->name, \
 			  pipe_config->name); \
 		ret = false; \
-	}
+	} \
+} while (0)
 
-#define PIPE_CONF_CHECK_I(name)	\
+#define PIPE_CONF_CHECK_I(name) do { \
 	if (current_config->name != pipe_config->name) { \
 		pipe_config_err(adjust, __stringify(name), \
 			  "(expected %i, found %i)\n", \
 			  current_config->name, \
 			  pipe_config->name); \
 		ret = false; \
-	}
+	} \
+} while (0)
 
-#define PIPE_CONF_CHECK_BOOL(name)	\
+#define PIPE_CONF_CHECK_BOOL(name) do { \
 	if (current_config->name != pipe_config->name) { \
 		pipe_config_err(adjust, __stringify(name), \
 			  "(expected %s, found %s)\n", \
 			  yesno(current_config->name), \
 			  yesno(pipe_config->name)); \
 		ret = false; \
-	}
+	} \
+} while (0)
 
 /*
  * Checks state where we only read out the enabling, but not the entire
  * state itself (like full infoframes or ELD for audio). These states
  * require a full modeset on bootup to fix up.
  */
-#define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) \
+#define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
 	if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
 		PIPE_CONF_CHECK_BOOL(name); \
 	} else { \
@@ -11126,18 +11263,20 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 			  yesno(current_config->name), \
 			  yesno(pipe_config->name)); \
 		ret = false; \
-	}
+	} \
+} while (0)
 
-#define PIPE_CONF_CHECK_P(name)	\
+#define PIPE_CONF_CHECK_P(name) do { \
 	if (current_config->name != pipe_config->name) { \
 		pipe_config_err(adjust, __stringify(name), \
 			  "(expected %p, found %p)\n", \
 			  current_config->name, \
 			  pipe_config->name); \
 		ret = false; \
-	}
+	} \
+} while (0)
 
-#define PIPE_CONF_CHECK_M_N(name) \
+#define PIPE_CONF_CHECK_M_N(name) do { \
 	if (!intel_compare_link_m_n(&current_config->name, \
 				    &pipe_config->name,\
 				    adjust)) { \
@@ -11155,14 +11294,15 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 			  pipe_config->name.link_m, \
 			  pipe_config->name.link_n); \
 		ret = false; \
-	}
+	} \
+} while (0)
 
 /* This is required for BDW+ where there is only one set of registers for
  * switching between high and low RR.
  * This macro can be used whenever a comparison has to be made between one
  * hw state and multiple sw state variables.
  */
-#define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) \
+#define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
 	if (!intel_compare_link_m_n(&current_config->name, \
 				    &pipe_config->name, adjust) && \
 	    !intel_compare_link_m_n(&current_config->alt_name, \
@@ -11187,9 +11327,10 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 			  pipe_config->name.link_m, \
 			  pipe_config->name.link_n); \
 		ret = false; \
-	}
+	} \
+} while (0)
 
-#define PIPE_CONF_CHECK_FLAGS(name, mask)	\
+#define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
 	if ((current_config->name ^ pipe_config->name) & (mask)) { \
 		pipe_config_err(adjust, __stringify(name), \
 			  "(%x) (expected %i, found %i)\n", \
@@ -11197,16 +11338,18 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 			  current_config->name & (mask), \
 			  pipe_config->name & (mask)); \
 		ret = false; \
-	}
+	} \
+} while (0)
 
-#define PIPE_CONF_CHECK_CLOCK_FUZZY(name) \
+#define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
 	if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
 		pipe_config_err(adjust, __stringify(name), \
 			  "(expected %i, found %i)\n", \
 			  current_config->name, \
 			  pipe_config->name); \
 		ret = false; \
-	}
+	} \
+} while (0)
 
 #define PIPE_CONF_QUIRK(quirk)	\
 	((current_config->quirks | pipe_config->quirks) & (quirk))
@@ -11315,6 +11458,16 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 	PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
 	PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
 	PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
+	PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
+	PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
+	PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
+	PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
+	PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
+	PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
+	PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
+	PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
+	PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
+	PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
 
 	PIPE_CONF_CHECK_X(dsi_pll.ctrl);
 	PIPE_CONF_CHECK_X(dsi_pll.div);
@@ -11378,6 +11531,11 @@ static void verify_wm_state(struct drm_crtc *crtc,
 	skl_ddb_get_hw_state(dev_priv, &hw_ddb);
 	sw_ddb = &dev_priv->wm.skl_hw.ddb;
 
+	if (INTEL_GEN(dev_priv) >= 11)
+		if (hw_ddb.enabled_slices != sw_ddb->enabled_slices)
+			DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
+				  sw_ddb->enabled_slices,
+				  hw_ddb.enabled_slices);
 	/* planes */
 	for_each_universal_plane(dev_priv, pipe, plane) {
 		hw_plane_wm = &hw_wm.planes[plane];
@@ -11643,11 +11801,11 @@ verify_single_dpll_state(struct drm_i915_private *dev_priv,
 
 	memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
 
-	DRM_DEBUG_KMS("%s\n", pll->name);
+	DRM_DEBUG_KMS("%s\n", pll->info->name);
 
-	active = pll->funcs.get_hw_state(dev_priv, pll, &dpll_hw_state);
+	active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
 
-	if (!(pll->flags & INTEL_DPLL_ALWAYS_ON)) {
+	if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
 		I915_STATE_WARN(!pll->on && pll->active_mask,
 		     "pll in active use but not on in sw tracking\n");
 		I915_STATE_WARN(pll->on && !pll->active_mask,
@@ -12136,20 +12294,23 @@ static void intel_update_crtc(struct drm_crtc *crtc,
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct intel_crtc_state *pipe_config = to_intel_crtc_state(new_crtc_state);
 	bool modeset = needs_modeset(new_crtc_state);
+	struct intel_plane_state *new_plane_state =
+		intel_atomic_get_new_plane_state(to_intel_atomic_state(state),
+						 to_intel_plane(crtc->primary));
 
 	if (modeset) {
 		update_scanline_offset(intel_crtc);
 		dev_priv->display.crtc_enable(pipe_config, state);
+
+		/* vblanks work again, re-enable pipe CRC. */
+		intel_crtc_enable_pipe_crc(intel_crtc);
 	} else {
 		intel_pre_plane_update(to_intel_crtc_state(old_crtc_state),
 				       pipe_config);
 	}
 
-	if (drm_atomic_get_existing_plane_state(state, crtc->primary)) {
-		intel_fbc_enable(
-		    intel_crtc, pipe_config,
-		    to_intel_plane_state(crtc->primary->state));
-	}
+	if (new_plane_state)
+		intel_fbc_enable(intel_crtc, pipe_config, new_plane_state);
 
 	drm_atomic_helper_commit_planes_on_crtc(old_crtc_state);
 }
@@ -12181,6 +12342,8 @@ static void skl_update_crtcs(struct drm_atomic_state *state)
 	bool progress;
 	enum pipe pipe;
 	int i;
+	u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
+	u8 required_slices = intel_state->wm_results.ddb.enabled_slices;
 
 	const struct skl_ddb_entry *entries[I915_MAX_PIPES] = {};
 
@@ -12189,6 +12352,10 @@ static void skl_update_crtcs(struct drm_atomic_state *state)
 		if (new_crtc_state->active)
 			entries[i] = &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb;
 
+	/* If 2nd DBuf slice required, enable it here */
+	if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
+		icl_dbuf_slices_update(dev_priv, required_slices);
+
 	/*
 	 * Whenever the number of active pipes changes, we need to make sure we
 	 * update the pipes in the right order so that their ddb allocations
@@ -12239,6 +12406,10 @@ static void skl_update_crtcs(struct drm_atomic_state *state)
 			progress = true;
 		}
 	} while (progress);
+
+	/* If 2nd DBuf slice is no more required disable it */
+	if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
+		icl_dbuf_slices_update(dev_priv, required_slices);
 }
 
 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
@@ -12320,6 +12491,13 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 
 		if (old_crtc_state->active) {
 			intel_crtc_disable_planes(crtc, old_crtc_state->plane_mask);
+
+			/*
+			 * We need to disable pipe CRC before disabling the pipe,
+			 * or we race against vblank off.
+			 */
+			intel_crtc_disable_pipe_crc(intel_crtc);
+
 			dev_priv->display.crtc_disable(to_intel_crtc_state(old_crtc_state), state);
 			intel_crtc->active = false;
 			intel_fbc_disable(intel_crtc);
@@ -12695,6 +12873,15 @@ static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
 		intel_unpin_fb_vma(vma, old_plane_state->flags);
 }
 
+static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
+{
+	struct i915_sched_attr attr = {
+		.priority = I915_PRIORITY_DISPLAY,
+	};
+
+	i915_gem_object_wait_priority(obj, 0, &attr);
+}
+
 /**
  * intel_prepare_plane_fb - Prepare fb for usage on plane
  * @plane: drm plane to prepare for
@@ -12723,8 +12910,8 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 
 	if (old_obj) {
 		struct drm_crtc_state *crtc_state =
-			drm_atomic_get_existing_crtc_state(new_state->state,
-							   plane->state->crtc);
+			drm_atomic_get_new_crtc_state(new_state->state,
+						      plane->state->crtc);
 
 		/* Big Hammer, we also need to ensure that any pending
 		 * MI_WAIT_FOR_EVENT inside a user batch buffer on the
@@ -12771,13 +12958,15 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 
 	ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
 
-	i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY);
+	fb_obj_bump_render_priority(obj);
 
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 	i915_gem_object_unpin_pages(obj);
 	if (ret)
 		return ret;
 
+	intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
+
 	if (!new_state->fence) { /* implicit fencing */
 		struct dma_fence *fence;
 
@@ -12822,11 +13011,13 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
 }
 
 int
-skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state)
+skl_max_scale(struct intel_crtc *intel_crtc,
+	      struct intel_crtc_state *crtc_state,
+	      uint32_t pixel_format)
 {
 	struct drm_i915_private *dev_priv;
-	int max_scale;
-	int crtc_clock, max_dotclk;
+	int max_scale, mult;
+	int crtc_clock, max_dotclk, tmpclk1, tmpclk2;
 
 	if (!intel_crtc || !crtc_state->base.enable)
 		return DRM_PLANE_HELPER_NO_SCALING;
@@ -12848,8 +13039,10 @@ skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state
 	 *            or
 	 *    cdclk/crtc_clock
 	 */
-	max_scale = min((1 << 16) * 3 - 1,
-			(1 << 8) * ((max_dotclk << 8) / crtc_clock));
+	mult = pixel_format == DRM_FORMAT_NV12 ? 2 : 3;
+	tmpclk1 = (1 << 16) * mult - 1;
+	tmpclk2 = (1 << 8) * ((max_dotclk << 8) / crtc_clock);
+	max_scale = min(tmpclk1, tmpclk2);
 
 	return max_scale;
 }
@@ -12865,12 +13058,16 @@ intel_check_primary_plane(struct intel_plane *plane,
 	int max_scale = DRM_PLANE_HELPER_NO_SCALING;
 	bool can_position = false;
 	int ret;
+	uint32_t pixel_format = 0;
 
 	if (INTEL_GEN(dev_priv) >= 9) {
 		/* use scaler when colorkey is not required */
 		if (!state->ckey.flags) {
 			min_scale = 1;
-			max_scale = skl_max_scale(to_intel_crtc(crtc), crtc_state);
+			if (state->base.fb)
+				pixel_format = state->base.fb->format->format;
+			max_scale = skl_max_scale(to_intel_crtc(crtc),
+						  crtc_state, pixel_format);
 		}
 		can_position = true;
 	}
@@ -12943,10 +13140,25 @@ static void intel_begin_crtc_commit(struct drm_crtc *crtc,
 							   intel_cstate);
 }
 
+void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
+				  struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+
+	if (!IS_GEN2(dev_priv))
+		intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
+
+	if (crtc_state->has_pch_encoder) {
+		enum pipe pch_transcoder =
+			intel_crtc_pch_transcoder(crtc);
+
+		intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
+	}
+}
+
 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
 				     struct drm_crtc_state *old_crtc_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct intel_atomic_state *old_intel_state =
 		to_intel_atomic_state(old_crtc_state->state);
@@ -12957,17 +13169,8 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc,
 
 	if (new_crtc_state->update_pipe &&
 	    !needs_modeset(&new_crtc_state->base) &&
-	    old_crtc_state->mode.private_flags & I915_MODE_FLAG_INHERITED) {
-		if (!IS_GEN2(dev_priv))
-			intel_set_cpu_fifo_underrun_reporting(dev_priv, intel_crtc->pipe, true);
-
-		if (new_crtc_state->has_pch_encoder) {
-			enum pipe pch_transcoder =
-				intel_crtc_pch_transcoder(intel_crtc);
-
-			intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
-		}
-	}
+	    old_crtc_state->mode.private_flags & I915_MODE_FLAG_INHERITED)
+		intel_crtc_arm_fifo_underrun(intel_crtc, new_crtc_state);
 }
 
 /**
@@ -13031,6 +13234,7 @@ static bool skl_mod_supported(uint32_t format, uint64_t modifier)
 	case DRM_FORMAT_YVYU:
 	case DRM_FORMAT_UYVY:
 	case DRM_FORMAT_VYUY:
+	case DRM_FORMAT_NV12:
 		if (modifier == I915_FORMAT_MOD_Yf_TILED)
 			return true;
 		/* fall through */
@@ -13165,8 +13369,9 @@ intel_legacy_cursor_update(struct drm_plane *plane,
 	if (ret)
 		goto out_unlock;
 
-	old_fb = old_plane_state->fb;
+	intel_fb_obj_flush(intel_fb_obj(fb), ORIGIN_FLIP);
 
+	old_fb = old_plane_state->fb;
 	i915_gem_track_fb(intel_fb_obj(old_fb), intel_fb_obj(fb),
 			  intel_plane->frontbuffer_bit);
 
@@ -13237,6 +13442,30 @@ static bool skl_plane_has_fbc(struct drm_i915_private *dev_priv,
 	return pipe == PIPE_A && plane_id == PLANE_PRIMARY;
 }
 
+bool skl_plane_has_planar(struct drm_i915_private *dev_priv,
+			  enum pipe pipe, enum plane_id plane_id)
+{
+	if (plane_id == PLANE_PRIMARY) {
+		if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
+			return false;
+		else if ((INTEL_GEN(dev_priv) == 9 && pipe == PIPE_C) &&
+			 !IS_GEMINILAKE(dev_priv))
+			return false;
+	} else if (plane_id >= PLANE_SPRITE0) {
+		if (plane_id == PLANE_CURSOR)
+			return false;
+		if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) == 10) {
+			if (plane_id != PLANE_SPRITE0)
+				return false;
+		} else {
+			if (plane_id != PLANE_SPRITE0 || pipe == PIPE_C ||
+			    IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
+				return false;
+		}
+	}
+	return true;
+}
+
 static struct intel_plane *
 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
@@ -13297,8 +13526,13 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 	primary->check_plane = intel_check_primary_plane;
 
 	if (INTEL_GEN(dev_priv) >= 9) {
-		intel_primary_formats = skl_primary_formats;
-		num_formats = ARRAY_SIZE(skl_primary_formats);
+		if (skl_plane_has_planar(dev_priv, pipe, PLANE_PRIMARY)) {
+			intel_primary_formats = skl_pri_planar_formats;
+			num_formats = ARRAY_SIZE(skl_pri_planar_formats);
+		} else {
+			intel_primary_formats = skl_primary_formats;
+			num_formats = ARRAY_SIZE(skl_primary_formats);
+		}
 
 		if (skl_plane_has_ccs(dev_priv, pipe, PLANE_PRIMARY))
 			modifiers = skl_format_modifiers_ccs;
@@ -13553,10 +13787,17 @@ static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
 	/* initialize shared scalers */
 	intel_crtc_init_scalers(intel_crtc, crtc_state);
 
-	BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
-	       dev_priv->plane_to_crtc_mapping[primary->i9xx_plane] != NULL);
-	dev_priv->plane_to_crtc_mapping[primary->i9xx_plane] = intel_crtc;
-	dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = intel_crtc;
+	BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
+	       dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
+	dev_priv->pipe_to_crtc_mapping[pipe] = intel_crtc;
+
+	if (INTEL_GEN(dev_priv) < 9) {
+		enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
+
+		BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
+		       dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
+		dev_priv->plane_to_crtc_mapping[i9xx_plane] = intel_crtc;
+	}
 
 	drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
 
@@ -14112,6 +14353,20 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 			goto err;
 		}
 		break;
+	case DRM_FORMAT_NV12:
+		if (mode_cmd->modifier[0] == I915_FORMAT_MOD_Y_TILED_CCS ||
+		    mode_cmd->modifier[0] == I915_FORMAT_MOD_Yf_TILED_CCS) {
+			DRM_DEBUG_KMS("RC not to be enabled with NV12\n");
+			goto err;
+		}
+		if (INTEL_GEN(dev_priv) < 9 || IS_SKYLAKE(dev_priv) ||
+		    IS_BROXTON(dev_priv)) {
+			DRM_DEBUG_KMS("unsupported pixel format: %s\n",
+				      drm_get_format_name(mode_cmd->pixel_format,
+							  &format_name));
+			goto err;
+		}
+		break;
 	default:
 		DRM_DEBUG_KMS("unsupported pixel format: %s\n",
 			      drm_get_format_name(mode_cmd->pixel_format, &format_name));
@@ -14124,6 +14379,14 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 
 	drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
 
+	if (fb->format->format == DRM_FORMAT_NV12 &&
+	    (fb->width < SKL_MIN_YUV_420_SRC_W ||
+	     fb->height < SKL_MIN_YUV_420_SRC_H ||
+	     (fb->width % 4) != 0 || (fb->height % 4) != 0)) {
+		DRM_DEBUG_KMS("src dimensions not correct for NV12\n");
+		return -EINVAL;
+	}
+
 	for (i = 0; i < fb->format->num_planes; i++) {
 		u32 stride_alignment;
 
@@ -15101,8 +15364,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 	for (i = 0; i < dev_priv->num_shared_dpll; i++) {
 		struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
 
-		pll->on = pll->funcs.get_hw_state(dev_priv, pll,
-						  &pll->state.hw_state);
+		pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
+							&pll->state.hw_state);
 		pll->state.crtc_mask = 0;
 		for_each_intel_crtc(dev, crtc) {
 			struct intel_crtc_state *crtc_state =
@@ -15115,7 +15378,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 		pll->active_mask = pll->state.crtc_mask;
 
 		DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
-			      pll->name, pll->state.crtc_mask, pll->on);
+			      pll->info->name, pll->state.crtc_mask, pll->on);
 	}
 
 	for_each_intel_encoder(dev, encoder) {
@@ -15291,9 +15554,10 @@ intel_modeset_setup_hw_state(struct drm_device *dev,
 		if (!pll->on || pll->active_mask)
 			continue;
 
-		DRM_DEBUG_KMS("%s enabled but not in use, disabling\n", pll->name);
+		DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
+			      pll->info->name);
 
-		pll->funcs.disable(dev_priv, pll);
+		pll->info->funcs->disable(dev_priv, pll);
 		pll->on = false;
 	}
 
diff --git a/drivers/gpu/drm/i915/intel_display.h b/drivers/gpu/drm/i915/intel_display.h
index 4e7418b345bc..2ef31617614a 100644
--- a/drivers/gpu/drm/i915/intel_display.h
+++ b/drivers/gpu/drm/i915/intel_display.h
@@ -218,6 +218,10 @@ struct intel_link_m_n {
 	for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \
 		for_each_if((__mask) & BIT(__p))
 
+#define for_each_cpu_transcoder_masked(__dev_priv, __t, __mask) \
+	for ((__t) = 0; (__t) < I915_MAX_TRANSCODERS; (__t)++)	\
+		for_each_if ((__mask) & (1 << (__t)))
+
 #define for_each_universal_plane(__dev_priv, __pipe, __p)		\
 	for ((__p) = 0;							\
 	     (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1;	\
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index b7b4cfdeb974..dde92e4af5d3 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -43,7 +43,6 @@
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 
-#define DP_LINK_CHECK_TIMEOUT	(10 * 1000)
 #define DP_DPRX_ESI_LEN 14
 
 /* Compliance test status bits  */
@@ -92,8 +91,6 @@ static const struct dp_link_dpll chv_dpll[] = {
 		{ .p1 = 4, .p2 = 2, .n = 1, .m1 = 2, .m2 = 0x819999a } },
 	{ 270000,	/* m2_int = 27, m2_fraction = 0 */
 		{ .p1 = 4, .p2 = 1, .n = 1, .m1 = 2, .m2 = 0x6c00000 } },
-	{ 540000,	/* m2_int = 27, m2_fraction = 0 */
-		{ .p1 = 2, .p2 = 1, .n = 1, .m1 = 2, .m2 = 0x6c00000 } }
 };
 
 /**
@@ -1650,9 +1647,17 @@ void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock,
 	}
 }
 
+struct link_config_limits {
+	int min_clock, max_clock;
+	int min_lane_count, max_lane_count;
+	int min_bpp, max_bpp;
+};
+
 static int intel_dp_compute_bpp(struct intel_dp *intel_dp,
 				struct intel_crtc_state *pipe_config)
 {
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
+	struct intel_connector *intel_connector = intel_dp->attached_connector;
 	int bpp, bpc;
 
 	bpp = pipe_config->pipe_bpp;
@@ -1661,13 +1666,16 @@ static int intel_dp_compute_bpp(struct intel_dp *intel_dp,
 	if (bpc > 0)
 		bpp = min(bpp, 3*bpc);
 
-	/* For DP Compliance we override the computed bpp for the pipe */
-	if (intel_dp->compliance.test_data.bpc != 0) {
-		pipe_config->pipe_bpp =	3*intel_dp->compliance.test_data.bpc;
-		pipe_config->dither_force_disable = pipe_config->pipe_bpp == 6*3;
-		DRM_DEBUG_KMS("Setting pipe_bpp to %d\n",
-			      pipe_config->pipe_bpp);
+	if (intel_dp_is_edp(intel_dp)) {
+		/* Get bpp from vbt only for panels that dont have bpp in edid */
+		if (intel_connector->base.display_info.bpc == 0 &&
+		    dev_priv->vbt.edp.bpp && dev_priv->vbt.edp.bpp < bpp) {
+			DRM_DEBUG_KMS("clamping bpp for eDP panel to BIOS-provided %i\n",
+				      dev_priv->vbt.edp.bpp);
+			bpp = dev_priv->vbt.edp.bpp;
+		}
 	}
+
 	return bpp;
 }
 
@@ -1688,6 +1696,142 @@ static bool intel_edp_compare_alt_mode(struct drm_display_mode *m1,
 	return bres;
 }
 
+/* Adjust link config limits based on compliance test requests. */
+static void
+intel_dp_adjust_compliance_config(struct intel_dp *intel_dp,
+				  struct intel_crtc_state *pipe_config,
+				  struct link_config_limits *limits)
+{
+	/* For DP Compliance we override the computed bpp for the pipe */
+	if (intel_dp->compliance.test_data.bpc != 0) {
+		int bpp = 3 * intel_dp->compliance.test_data.bpc;
+
+		limits->min_bpp = limits->max_bpp = bpp;
+		pipe_config->dither_force_disable = bpp == 6 * 3;
+
+		DRM_DEBUG_KMS("Setting pipe_bpp to %d\n", bpp);
+	}
+
+	/* Use values requested by Compliance Test Request */
+	if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) {
+		int index;
+
+		/* Validate the compliance test data since max values
+		 * might have changed due to link train fallback.
+		 */
+		if (intel_dp_link_params_valid(intel_dp, intel_dp->compliance.test_link_rate,
+					       intel_dp->compliance.test_lane_count)) {
+			index = intel_dp_rate_index(intel_dp->common_rates,
+						    intel_dp->num_common_rates,
+						    intel_dp->compliance.test_link_rate);
+			if (index >= 0)
+				limits->min_clock = limits->max_clock = index;
+			limits->min_lane_count = limits->max_lane_count =
+				intel_dp->compliance.test_lane_count;
+		}
+	}
+}
+
+/* Optimize link config in order: max bpp, min clock, min lanes */
+static bool
+intel_dp_compute_link_config_wide(struct intel_dp *intel_dp,
+				  struct intel_crtc_state *pipe_config,
+				  const struct link_config_limits *limits)
+{
+	struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
+	int bpp, clock, lane_count;
+	int mode_rate, link_clock, link_avail;
+
+	for (bpp = limits->max_bpp; bpp >= limits->min_bpp; bpp -= 2 * 3) {
+		mode_rate = intel_dp_link_required(adjusted_mode->crtc_clock,
+						   bpp);
+
+		for (clock = limits->min_clock; clock <= limits->max_clock; clock++) {
+			for (lane_count = limits->min_lane_count;
+			     lane_count <= limits->max_lane_count;
+			     lane_count <<= 1) {
+				link_clock = intel_dp->common_rates[clock];
+				link_avail = intel_dp_max_data_rate(link_clock,
+								    lane_count);
+
+				if (mode_rate <= link_avail) {
+					pipe_config->lane_count = lane_count;
+					pipe_config->pipe_bpp = bpp;
+					pipe_config->port_clock = link_clock;
+
+					return true;
+				}
+			}
+		}
+	}
+
+	return false;
+}
+
+static bool
+intel_dp_compute_link_config(struct intel_encoder *encoder,
+			     struct intel_crtc_state *pipe_config)
+{
+	struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
+	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+	struct link_config_limits limits;
+	int common_len;
+
+	common_len = intel_dp_common_len_rate_limit(intel_dp,
+						    intel_dp->max_link_rate);
+
+	/* No common link rates between source and sink */
+	WARN_ON(common_len <= 0);
+
+	limits.min_clock = 0;
+	limits.max_clock = common_len - 1;
+
+	limits.min_lane_count = 1;
+	limits.max_lane_count = intel_dp_max_lane_count(intel_dp);
+
+	limits.min_bpp = 6 * 3;
+	limits.max_bpp = intel_dp_compute_bpp(intel_dp, pipe_config);
+
+	if (intel_dp_is_edp(intel_dp)) {
+		/*
+		 * Use the maximum clock and number of lanes the eDP panel
+		 * advertizes being capable of. The panels are generally
+		 * designed to support only a single clock and lane
+		 * configuration, and typically these values correspond to the
+		 * native resolution of the panel.
+		 */
+		limits.min_lane_count = limits.max_lane_count;
+		limits.min_clock = limits.max_clock;
+	}
+
+	intel_dp_adjust_compliance_config(intel_dp, pipe_config, &limits);
+
+	DRM_DEBUG_KMS("DP link computation with max lane count %i "
+		      "max rate %d max bpp %d pixel clock %iKHz\n",
+		      limits.max_lane_count,
+		      intel_dp->common_rates[limits.max_clock],
+		      limits.max_bpp, adjusted_mode->crtc_clock);
+
+	/*
+	 * Optimize for slow and wide. This is the place to add alternative
+	 * optimization policy.
+	 */
+	if (!intel_dp_compute_link_config_wide(intel_dp, pipe_config, &limits))
+		return false;
+
+	DRM_DEBUG_KMS("DP lane count %d clock %d bpp %d\n",
+		      pipe_config->lane_count, pipe_config->port_clock,
+		      pipe_config->pipe_bpp);
+
+	DRM_DEBUG_KMS("DP link rate required %i available %i\n",
+		      intel_dp_link_required(adjusted_mode->crtc_clock,
+					     pipe_config->pipe_bpp),
+		      intel_dp_max_data_rate(pipe_config->port_clock,
+					     pipe_config->lane_count));
+
+	return true;
+}
+
 bool
 intel_dp_compute_config(struct intel_encoder *encoder,
 			struct intel_crtc_state *pipe_config,
@@ -1701,27 +1845,9 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	struct intel_connector *intel_connector = intel_dp->attached_connector;
 	struct intel_digital_connector_state *intel_conn_state =
 		to_intel_digital_connector_state(conn_state);
-	int lane_count, clock;
-	int min_lane_count = 1;
-	int max_lane_count = intel_dp_max_lane_count(intel_dp);
-	/* Conveniently, the link BW constants become indices with a shift...*/
-	int min_clock = 0;
-	int max_clock;
-	int bpp, mode_rate;
-	int link_avail, link_clock;
-	int common_len;
-	uint8_t link_bw, rate_select;
 	bool reduce_m_n = drm_dp_has_quirk(&intel_dp->desc,
 					   DP_DPCD_QUIRK_LIMITED_M_N);
 
-	common_len = intel_dp_common_len_rate_limit(intel_dp,
-						    intel_dp->max_link_rate);
-
-	/* No common link rates between source and sink */
-	WARN_ON(common_len <= 0);
-
-	max_clock = common_len - 1;
-
 	if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv) && port != PORT_A)
 		pipe_config->has_pch_encoder = true;
 
@@ -1747,6 +1873,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 
 		if (INTEL_GEN(dev_priv) >= 9) {
 			int ret;
+
 			ret = skl_update_scaler_crtc(pipe_config);
 			if (ret)
 				return ret;
@@ -1767,75 +1894,9 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK)
 		return false;
 
-	/* Use values requested by Compliance Test Request */
-	if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) {
-		int index;
-
-		/* Validate the compliance test data since max values
-		 * might have changed due to link train fallback.
-		 */
-		if (intel_dp_link_params_valid(intel_dp, intel_dp->compliance.test_link_rate,
-					       intel_dp->compliance.test_lane_count)) {
-			index = intel_dp_rate_index(intel_dp->common_rates,
-						    intel_dp->num_common_rates,
-						    intel_dp->compliance.test_link_rate);
-			if (index >= 0)
-				min_clock = max_clock = index;
-			min_lane_count = max_lane_count = intel_dp->compliance.test_lane_count;
-		}
-	}
-	DRM_DEBUG_KMS("DP link computation with max lane count %i "
-		      "max bw %d pixel clock %iKHz\n",
-		      max_lane_count, intel_dp->common_rates[max_clock],
-		      adjusted_mode->crtc_clock);
-
-	/* Walk through all bpp values. Luckily they're all nicely spaced with 2
-	 * bpc in between. */
-	bpp = intel_dp_compute_bpp(intel_dp, pipe_config);
-	if (intel_dp_is_edp(intel_dp)) {
-
-		/* Get bpp from vbt only for panels that dont have bpp in edid */
-		if (intel_connector->base.display_info.bpc == 0 &&
-			(dev_priv->vbt.edp.bpp && dev_priv->vbt.edp.bpp < bpp)) {
-			DRM_DEBUG_KMS("clamping bpp for eDP panel to BIOS-provided %i\n",
-				      dev_priv->vbt.edp.bpp);
-			bpp = dev_priv->vbt.edp.bpp;
-		}
-
-		/*
-		 * Use the maximum clock and number of lanes the eDP panel
-		 * advertizes being capable of. The panels are generally
-		 * designed to support only a single clock and lane
-		 * configuration, and typically these values correspond to the
-		 * native resolution of the panel.
-		 */
-		min_lane_count = max_lane_count;
-		min_clock = max_clock;
-	}
-
-	for (; bpp >= 6*3; bpp -= 2*3) {
-		mode_rate = intel_dp_link_required(adjusted_mode->crtc_clock,
-						   bpp);
-
-		for (clock = min_clock; clock <= max_clock; clock++) {
-			for (lane_count = min_lane_count;
-				lane_count <= max_lane_count;
-				lane_count <<= 1) {
-
-				link_clock = intel_dp->common_rates[clock];
-				link_avail = intel_dp_max_data_rate(link_clock,
-								    lane_count);
-
-				if (mode_rate <= link_avail) {
-					goto found;
-				}
-			}
-		}
-	}
-
-	return false;
+	if (!intel_dp_compute_link_config(encoder, pipe_config))
+		return false;
 
-found:
 	if (intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_AUTO) {
 		/*
 		 * See:
@@ -1843,7 +1904,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 		 * VESA DisplayPort Ver.1.2a - 5.1.1.1 Video Colorimetry
 		 */
 		pipe_config->limited_color_range =
-			bpp != 18 &&
+			pipe_config->pipe_bpp != 18 &&
 			drm_default_rgb_quant_range(adjusted_mode) ==
 			HDMI_QUANTIZATION_RANGE_LIMITED;
 	} else {
@@ -1851,21 +1912,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 			intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_LIMITED;
 	}
 
-	pipe_config->lane_count = lane_count;
-
-	pipe_config->pipe_bpp = bpp;
-	pipe_config->port_clock = intel_dp->common_rates[clock];
-
-	intel_dp_compute_rate(intel_dp, pipe_config->port_clock,
-			      &link_bw, &rate_select);
-
-	DRM_DEBUG_KMS("DP link bw %02x rate select %02x lane count %d clock %d bpp %d\n",
-		      link_bw, rate_select, pipe_config->lane_count,
-		      pipe_config->port_clock, bpp);
-	DRM_DEBUG_KMS("DP link bw required %i available %i\n",
-		      mode_rate, link_avail);
-
-	intel_link_compute_m_n(bpp, lane_count,
+	intel_link_compute_m_n(pipe_config->pipe_bpp, pipe_config->lane_count,
 			       adjusted_mode->crtc_clock,
 			       pipe_config->port_clock,
 			       &pipe_config->dp_m_n,
@@ -1874,11 +1921,12 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	if (intel_connector->panel.downclock_mode != NULL &&
 		dev_priv->drrs.type == SEAMLESS_DRRS_SUPPORT) {
 			pipe_config->has_drrs = true;
-			intel_link_compute_m_n(bpp, lane_count,
-				intel_connector->panel.downclock_mode->clock,
-				pipe_config->port_clock,
-				&pipe_config->dp_m2_n2,
-				reduce_m_n);
+			intel_link_compute_m_n(pipe_config->pipe_bpp,
+					       pipe_config->lane_count,
+					       intel_connector->panel.downclock_mode->clock,
+					       pipe_config->port_clock,
+					       &pipe_config->dp_m2_n2,
+					       reduce_m_n);
 	}
 
 	if (!HAS_DDI(dev_priv))
@@ -2881,10 +2929,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 		}
 
 	} else {
-		if (IS_CHERRYVIEW(dev_priv))
-			*DP &= ~DP_LINK_TRAIN_MASK_CHV;
-		else
-			*DP &= ~DP_LINK_TRAIN_MASK;
+		*DP &= ~DP_LINK_TRAIN_MASK;
 
 		switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
 		case DP_TRAINING_PATTERN_DISABLE:
@@ -2897,12 +2942,8 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 			*DP |= DP_LINK_TRAIN_PAT_2;
 			break;
 		case DP_TRAINING_PATTERN_3:
-			if (IS_CHERRYVIEW(dev_priv)) {
-				*DP |= DP_LINK_TRAIN_PAT_3_CHV;
-			} else {
-				DRM_DEBUG_KMS("TPS3 not supported, using TPS2 instead\n");
-				*DP |= DP_LINK_TRAIN_PAT_2;
-			}
+			DRM_DEBUG_KMS("TPS3 not supported, using TPS2 instead\n");
+			*DP |= DP_LINK_TRAIN_PAT_2;
 			break;
 		}
 	}
@@ -3641,10 +3682,7 @@ intel_dp_link_down(struct intel_encoder *encoder,
 		DP &= ~DP_LINK_TRAIN_MASK_CPT;
 		DP |= DP_LINK_TRAIN_PAT_IDLE_CPT;
 	} else {
-		if (IS_CHERRYVIEW(dev_priv))
-			DP &= ~DP_LINK_TRAIN_MASK_CHV;
-		else
-			DP &= ~DP_LINK_TRAIN_MASK;
+		DP &= ~DP_LINK_TRAIN_MASK;
 		DP |= DP_LINK_TRAIN_PAT_IDLE;
 	}
 	I915_WRITE(intel_dp->output_reg, DP);
diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c
index f59b59bb0a21..3fcaa98b9055 100644
--- a/drivers/gpu/drm/i915/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/intel_dp_link_training.c
@@ -139,6 +139,11 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp)
 	intel_dp_compute_rate(intel_dp, intel_dp->link_rate,
 			      &link_bw, &rate_select);
 
+	if (link_bw)
+		DRM_DEBUG_KMS("Using LINK_BW_SET value %02x\n", link_bw);
+	else
+		DRM_DEBUG_KMS("Using LINK_RATE_SET value %02x\n", rate_select);
+
 	/* Write the link configuration data */
 	link_config[0] = link_bw;
 	link_config[1] = intel_dp->lane_count;
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index c3de0918ee13..9e6956c08688 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -180,9 +180,11 @@ static void intel_mst_post_disable_dp(struct intel_encoder *encoder,
 	intel_dp->active_mst_links--;
 
 	intel_mst->connector = NULL;
-	if (intel_dp->active_mst_links == 0)
+	if (intel_dp->active_mst_links == 0) {
+		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
 		intel_dig_port->base.post_disable(&intel_dig_port->base,
 						  old_crtc_state, NULL);
+	}
 
 	DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links);
 }
@@ -223,7 +225,11 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder,
 
 	DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links);
 
+	if (intel_dp->active_mst_links == 0)
+		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
+
 	drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, true);
+
 	if (intel_dp->active_mst_links == 0)
 		intel_dig_port->base.pre_enable(&intel_dig_port->base,
 						pipe_config, NULL);
diff --git a/drivers/gpu/drm/i915/intel_dpio_phy.c b/drivers/gpu/drm/i915/intel_dpio_phy.c
index c8e9e44e5981..00b3ab656b06 100644
--- a/drivers/gpu/drm/i915/intel_dpio_phy.c
+++ b/drivers/gpu/drm/i915/intel_dpio_phy.c
@@ -380,13 +380,14 @@ static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv,
 	 * all 1s.  Eventually they become accessible as they power up, then
 	 * the reserved bit will give the default 0.  Poll on the reserved bit
 	 * becoming 0 to find when the PHY is accessible.
-	 * HW team confirmed that the time to reach phypowergood status is
-	 * anywhere between 50 us and 100us.
+	 * The flag should get set in 100us according to the HW team, but
+	 * use 1ms due to occasional timeouts observed with that.
 	 */
-	if (wait_for_us(((I915_READ(BXT_PORT_CL1CM_DW0(phy)) &
-		(PHY_RESERVED | PHY_POWER_GOOD)) == PHY_POWER_GOOD), 100)) {
+	if (intel_wait_for_register_fw(dev_priv, BXT_PORT_CL1CM_DW0(phy),
+				       PHY_RESERVED | PHY_POWER_GOOD,
+				       PHY_POWER_GOOD,
+				       1))
 		DRM_ERROR("timeout during PHY%d power on\n", phy);
-	}
 
 	/* Program PLL Rcomp code offset */
 	val = I915_READ(BXT_PORT_CL1CM_DW9(phy));
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index 51c5ae4e9116..383fbc15113d 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -118,10 +118,10 @@ void assert_shared_dpll(struct drm_i915_private *dev_priv,
 	if (WARN(!pll, "asserting DPLL %s with no DPLL\n", onoff(state)))
 		return;
 
-	cur_state = pll->funcs.get_hw_state(dev_priv, pll, &hw_state);
+	cur_state = pll->info->funcs->get_hw_state(dev_priv, pll, &hw_state);
 	I915_STATE_WARN(cur_state != state,
 	     "%s assertion failure (expected %s, current %s)\n",
-			pll->name, onoff(state), onoff(cur_state));
+			pll->info->name, onoff(state), onoff(cur_state));
 }
 
 /**
@@ -143,11 +143,11 @@ void intel_prepare_shared_dpll(struct intel_crtc *crtc)
 	mutex_lock(&dev_priv->dpll_lock);
 	WARN_ON(!pll->state.crtc_mask);
 	if (!pll->active_mask) {
-		DRM_DEBUG_DRIVER("setting up %s\n", pll->name);
+		DRM_DEBUG_DRIVER("setting up %s\n", pll->info->name);
 		WARN_ON(pll->on);
 		assert_shared_dpll_disabled(dev_priv, pll);
 
-		pll->funcs.prepare(dev_priv, pll);
+		pll->info->funcs->prepare(dev_priv, pll);
 	}
 	mutex_unlock(&dev_priv->dpll_lock);
 }
@@ -179,7 +179,7 @@ void intel_enable_shared_dpll(struct intel_crtc *crtc)
 	pll->active_mask |= crtc_mask;
 
 	DRM_DEBUG_KMS("enable %s (active %x, on? %d) for crtc %d\n",
-		      pll->name, pll->active_mask, pll->on,
+		      pll->info->name, pll->active_mask, pll->on,
 		      crtc->base.base.id);
 
 	if (old_mask) {
@@ -189,8 +189,8 @@ void intel_enable_shared_dpll(struct intel_crtc *crtc)
 	}
 	WARN_ON(pll->on);
 
-	DRM_DEBUG_KMS("enabling %s\n", pll->name);
-	pll->funcs.enable(dev_priv, pll);
+	DRM_DEBUG_KMS("enabling %s\n", pll->info->name);
+	pll->info->funcs->enable(dev_priv, pll);
 	pll->on = true;
 
 out:
@@ -221,7 +221,7 @@ void intel_disable_shared_dpll(struct intel_crtc *crtc)
 		goto out;
 
 	DRM_DEBUG_KMS("disable %s (active %x, on? %d) for crtc %d\n",
-		      pll->name, pll->active_mask, pll->on,
+		      pll->info->name, pll->active_mask, pll->on,
 		      crtc->base.base.id);
 
 	assert_shared_dpll_enabled(dev_priv, pll);
@@ -231,8 +231,8 @@ void intel_disable_shared_dpll(struct intel_crtc *crtc)
 	if (pll->active_mask)
 		goto out;
 
-	DRM_DEBUG_KMS("disabling %s\n", pll->name);
-	pll->funcs.disable(dev_priv, pll);
+	DRM_DEBUG_KMS("disabling %s\n", pll->info->name);
+	pll->info->funcs->disable(dev_priv, pll);
 	pll->on = false;
 
 out:
@@ -263,7 +263,8 @@ intel_find_shared_dpll(struct intel_crtc *crtc,
 			   &shared_dpll[i].hw_state,
 			   sizeof(crtc_state->dpll_hw_state)) == 0) {
 			DRM_DEBUG_KMS("[CRTC:%d:%s] sharing existing %s (crtc mask 0x%08x, active %x)\n",
-				      crtc->base.base.id, crtc->base.name, pll->name,
+				      crtc->base.base.id, crtc->base.name,
+				      pll->info->name,
 				      shared_dpll[i].crtc_mask,
 				      pll->active_mask);
 			return pll;
@@ -275,7 +276,8 @@ intel_find_shared_dpll(struct intel_crtc *crtc,
 		pll = &dev_priv->shared_dplls[i];
 		if (shared_dpll[i].crtc_mask == 0) {
 			DRM_DEBUG_KMS("[CRTC:%d:%s] allocated %s\n",
-				      crtc->base.base.id, crtc->base.name, pll->name);
+				      crtc->base.base.id, crtc->base.name,
+				      pll->info->name);
 			return pll;
 		}
 	}
@@ -289,19 +291,19 @@ intel_reference_shared_dpll(struct intel_shared_dpll *pll,
 {
 	struct intel_shared_dpll_state *shared_dpll;
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
-	enum intel_dpll_id i = pll->id;
+	const enum intel_dpll_id id = pll->info->id;
 
 	shared_dpll = intel_atomic_get_shared_dpll_state(crtc_state->base.state);
 
-	if (shared_dpll[i].crtc_mask == 0)
-		shared_dpll[i].hw_state =
+	if (shared_dpll[id].crtc_mask == 0)
+		shared_dpll[id].hw_state =
 			crtc_state->dpll_hw_state;
 
 	crtc_state->shared_dpll = pll;
-	DRM_DEBUG_DRIVER("using %s for pipe %c\n", pll->name,
+	DRM_DEBUG_DRIVER("using %s for pipe %c\n", pll->info->name,
 			 pipe_name(crtc->pipe));
 
-	shared_dpll[pll->id].crtc_mask |= 1 << crtc->pipe;
+	shared_dpll[id].crtc_mask |= 1 << crtc->pipe;
 }
 
 /**
@@ -341,15 +343,16 @@ static bool ibx_pch_dpll_get_hw_state(struct drm_i915_private *dev_priv,
 				      struct intel_shared_dpll *pll,
 				      struct intel_dpll_hw_state *hw_state)
 {
+	const enum intel_dpll_id id = pll->info->id;
 	uint32_t val;
 
 	if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS))
 		return false;
 
-	val = I915_READ(PCH_DPLL(pll->id));
+	val = I915_READ(PCH_DPLL(id));
 	hw_state->dpll = val;
-	hw_state->fp0 = I915_READ(PCH_FP0(pll->id));
-	hw_state->fp1 = I915_READ(PCH_FP1(pll->id));
+	hw_state->fp0 = I915_READ(PCH_FP0(id));
+	hw_state->fp1 = I915_READ(PCH_FP1(id));
 
 	intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
 
@@ -359,8 +362,10 @@ static bool ibx_pch_dpll_get_hw_state(struct drm_i915_private *dev_priv,
 static void ibx_pch_dpll_prepare(struct drm_i915_private *dev_priv,
 				 struct intel_shared_dpll *pll)
 {
-	I915_WRITE(PCH_FP0(pll->id), pll->state.hw_state.fp0);
-	I915_WRITE(PCH_FP1(pll->id), pll->state.hw_state.fp1);
+	const enum intel_dpll_id id = pll->info->id;
+
+	I915_WRITE(PCH_FP0(id), pll->state.hw_state.fp0);
+	I915_WRITE(PCH_FP1(id), pll->state.hw_state.fp1);
 }
 
 static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
@@ -379,13 +384,15 @@ static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
 static void ibx_pch_dpll_enable(struct drm_i915_private *dev_priv,
 				struct intel_shared_dpll *pll)
 {
+	const enum intel_dpll_id id = pll->info->id;
+
 	/* PCH refclock must be enabled first */
 	ibx_assert_pch_refclk_enabled(dev_priv);
 
-	I915_WRITE(PCH_DPLL(pll->id), pll->state.hw_state.dpll);
+	I915_WRITE(PCH_DPLL(id), pll->state.hw_state.dpll);
 
 	/* Wait for the clocks to stabilize. */
-	POSTING_READ(PCH_DPLL(pll->id));
+	POSTING_READ(PCH_DPLL(id));
 	udelay(150);
 
 	/* The pixel multiplier can only be updated once the
@@ -393,14 +400,15 @@ static void ibx_pch_dpll_enable(struct drm_i915_private *dev_priv,
 	 *
 	 * So write it again.
 	 */
-	I915_WRITE(PCH_DPLL(pll->id), pll->state.hw_state.dpll);
-	POSTING_READ(PCH_DPLL(pll->id));
+	I915_WRITE(PCH_DPLL(id), pll->state.hw_state.dpll);
+	POSTING_READ(PCH_DPLL(id));
 	udelay(200);
 }
 
 static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv,
 				 struct intel_shared_dpll *pll)
 {
+	const enum intel_dpll_id id = pll->info->id;
 	struct drm_device *dev = &dev_priv->drm;
 	struct intel_crtc *crtc;
 
@@ -410,8 +418,8 @@ static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv,
 			assert_pch_transcoder_disabled(dev_priv, crtc->pipe);
 	}
 
-	I915_WRITE(PCH_DPLL(pll->id), 0);
-	POSTING_READ(PCH_DPLL(pll->id));
+	I915_WRITE(PCH_DPLL(id), 0);
+	POSTING_READ(PCH_DPLL(id));
 	udelay(200);
 }
 
@@ -429,7 +437,8 @@ ibx_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 		pll = &dev_priv->shared_dplls[i];
 
 		DRM_DEBUG_KMS("[CRTC:%d:%s] using pre-allocated %s\n",
-			      crtc->base.base.id, crtc->base.name, pll->name);
+			      crtc->base.base.id, crtc->base.name,
+			      pll->info->name);
 	} else {
 		pll = intel_find_shared_dpll(crtc, crtc_state,
 					     DPLL_ID_PCH_PLL_A,
@@ -466,8 +475,10 @@ static const struct intel_shared_dpll_funcs ibx_pch_dpll_funcs = {
 static void hsw_ddi_wrpll_enable(struct drm_i915_private *dev_priv,
 			       struct intel_shared_dpll *pll)
 {
-	I915_WRITE(WRPLL_CTL(pll->id), pll->state.hw_state.wrpll);
-	POSTING_READ(WRPLL_CTL(pll->id));
+	const enum intel_dpll_id id = pll->info->id;
+
+	I915_WRITE(WRPLL_CTL(id), pll->state.hw_state.wrpll);
+	POSTING_READ(WRPLL_CTL(id));
 	udelay(20);
 }
 
@@ -482,11 +493,12 @@ static void hsw_ddi_spll_enable(struct drm_i915_private *dev_priv,
 static void hsw_ddi_wrpll_disable(struct drm_i915_private *dev_priv,
 				  struct intel_shared_dpll *pll)
 {
+	const enum intel_dpll_id id = pll->info->id;
 	uint32_t val;
 
-	val = I915_READ(WRPLL_CTL(pll->id));
-	I915_WRITE(WRPLL_CTL(pll->id), val & ~WRPLL_PLL_ENABLE);
-	POSTING_READ(WRPLL_CTL(pll->id));
+	val = I915_READ(WRPLL_CTL(id));
+	I915_WRITE(WRPLL_CTL(id), val & ~WRPLL_PLL_ENABLE);
+	POSTING_READ(WRPLL_CTL(id));
 }
 
 static void hsw_ddi_spll_disable(struct drm_i915_private *dev_priv,
@@ -503,12 +515,13 @@ static bool hsw_ddi_wrpll_get_hw_state(struct drm_i915_private *dev_priv,
 				       struct intel_shared_dpll *pll,
 				       struct intel_dpll_hw_state *hw_state)
 {
+	const enum intel_dpll_id id = pll->info->id;
 	uint32_t val;
 
 	if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS))
 		return false;
 
-	val = I915_READ(WRPLL_CTL(pll->id));
+	val = I915_READ(WRPLL_CTL(id));
 	hw_state->wrpll = val;
 
 	intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
@@ -914,13 +927,15 @@ static const struct skl_dpll_regs skl_dpll_regs[4] = {
 static void skl_ddi_pll_write_ctrl1(struct drm_i915_private *dev_priv,
 				    struct intel_shared_dpll *pll)
 {
+	const enum intel_dpll_id id = pll->info->id;
 	uint32_t val;
 
 	val = I915_READ(DPLL_CTRL1);
 
-	val &= ~(DPLL_CTRL1_HDMI_MODE(pll->id) | DPLL_CTRL1_SSC(pll->id) |
-		 DPLL_CTRL1_LINK_RATE_MASK(pll->id));
-	val |= pll->state.hw_state.ctrl1 << (pll->id * 6);
+	val &= ~(DPLL_CTRL1_HDMI_MODE(id) |
+		 DPLL_CTRL1_SSC(id) |
+		 DPLL_CTRL1_LINK_RATE_MASK(id));
+	val |= pll->state.hw_state.ctrl1 << (id * 6);
 
 	I915_WRITE(DPLL_CTRL1, val);
 	POSTING_READ(DPLL_CTRL1);
@@ -930,24 +945,25 @@ static void skl_ddi_pll_enable(struct drm_i915_private *dev_priv,
 			       struct intel_shared_dpll *pll)
 {
 	const struct skl_dpll_regs *regs = skl_dpll_regs;
+	const enum intel_dpll_id id = pll->info->id;
 
 	skl_ddi_pll_write_ctrl1(dev_priv, pll);
 
-	I915_WRITE(regs[pll->id].cfgcr1, pll->state.hw_state.cfgcr1);
-	I915_WRITE(regs[pll->id].cfgcr2, pll->state.hw_state.cfgcr2);
-	POSTING_READ(regs[pll->id].cfgcr1);
-	POSTING_READ(regs[pll->id].cfgcr2);
+	I915_WRITE(regs[id].cfgcr1, pll->state.hw_state.cfgcr1);
+	I915_WRITE(regs[id].cfgcr2, pll->state.hw_state.cfgcr2);
+	POSTING_READ(regs[id].cfgcr1);
+	POSTING_READ(regs[id].cfgcr2);
 
 	/* the enable bit is always bit 31 */
-	I915_WRITE(regs[pll->id].ctl,
-		   I915_READ(regs[pll->id].ctl) | LCPLL_PLL_ENABLE);
+	I915_WRITE(regs[id].ctl,
+		   I915_READ(regs[id].ctl) | LCPLL_PLL_ENABLE);
 
 	if (intel_wait_for_register(dev_priv,
 				    DPLL_STATUS,
-				    DPLL_LOCK(pll->id),
-				    DPLL_LOCK(pll->id),
+				    DPLL_LOCK(id),
+				    DPLL_LOCK(id),
 				    5))
-		DRM_ERROR("DPLL %d not locked\n", pll->id);
+		DRM_ERROR("DPLL %d not locked\n", id);
 }
 
 static void skl_ddi_dpll0_enable(struct drm_i915_private *dev_priv,
@@ -960,11 +976,12 @@ static void skl_ddi_pll_disable(struct drm_i915_private *dev_priv,
 				struct intel_shared_dpll *pll)
 {
 	const struct skl_dpll_regs *regs = skl_dpll_regs;
+	const enum intel_dpll_id id = pll->info->id;
 
 	/* the enable bit is always bit 31 */
-	I915_WRITE(regs[pll->id].ctl,
-		   I915_READ(regs[pll->id].ctl) & ~LCPLL_PLL_ENABLE);
-	POSTING_READ(regs[pll->id].ctl);
+	I915_WRITE(regs[id].ctl,
+		   I915_READ(regs[id].ctl) & ~LCPLL_PLL_ENABLE);
+	POSTING_READ(regs[id].ctl);
 }
 
 static void skl_ddi_dpll0_disable(struct drm_i915_private *dev_priv,
@@ -978,6 +995,7 @@ static bool skl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv,
 {
 	uint32_t val;
 	const struct skl_dpll_regs *regs = skl_dpll_regs;
+	const enum intel_dpll_id id = pll->info->id;
 	bool ret;
 
 	if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS))
@@ -985,17 +1003,17 @@ static bool skl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv,
 
 	ret = false;
 
-	val = I915_READ(regs[pll->id].ctl);
+	val = I915_READ(regs[id].ctl);
 	if (!(val & LCPLL_PLL_ENABLE))
 		goto out;
 
 	val = I915_READ(DPLL_CTRL1);
-	hw_state->ctrl1 = (val >> (pll->id * 6)) & 0x3f;
+	hw_state->ctrl1 = (val >> (id * 6)) & 0x3f;
 
 	/* avoid reading back stale values if HDMI mode is not enabled */
-	if (val & DPLL_CTRL1_HDMI_MODE(pll->id)) {
-		hw_state->cfgcr1 = I915_READ(regs[pll->id].cfgcr1);
-		hw_state->cfgcr2 = I915_READ(regs[pll->id].cfgcr2);
+	if (val & DPLL_CTRL1_HDMI_MODE(id)) {
+		hw_state->cfgcr1 = I915_READ(regs[id].cfgcr1);
+		hw_state->cfgcr2 = I915_READ(regs[id].cfgcr2);
 	}
 	ret = true;
 
@@ -1011,6 +1029,7 @@ static bool skl_ddi_dpll0_get_hw_state(struct drm_i915_private *dev_priv,
 {
 	uint32_t val;
 	const struct skl_dpll_regs *regs = skl_dpll_regs;
+	const enum intel_dpll_id id = pll->info->id;
 	bool ret;
 
 	if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS))
@@ -1019,12 +1038,12 @@ static bool skl_ddi_dpll0_get_hw_state(struct drm_i915_private *dev_priv,
 	ret = false;
 
 	/* DPLL0 is always enabled since it drives CDCLK */
-	val = I915_READ(regs[pll->id].ctl);
+	val = I915_READ(regs[id].ctl);
 	if (WARN_ON(!(val & LCPLL_PLL_ENABLE)))
 		goto out;
 
 	val = I915_READ(DPLL_CTRL1);
-	hw_state->ctrl1 = (val >> (pll->id * 6)) & 0x3f;
+	hw_state->ctrl1 = (val >> (id * 6)) & 0x3f;
 
 	ret = true;
 
@@ -1424,7 +1443,7 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv,
 				struct intel_shared_dpll *pll)
 {
 	uint32_t temp;
-	enum port port = (enum port)pll->id;	/* 1:1 port->PLL mapping */
+	enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */
 	enum dpio_phy phy;
 	enum dpio_channel ch;
 
@@ -1543,7 +1562,7 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv,
 static void bxt_ddi_pll_disable(struct drm_i915_private *dev_priv,
 					struct intel_shared_dpll *pll)
 {
-	enum port port = (enum port)pll->id;	/* 1:1 port->PLL mapping */
+	enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */
 	uint32_t temp;
 
 	temp = I915_READ(BXT_PORT_PLL_ENABLE(port));
@@ -1566,7 +1585,7 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv,
 					struct intel_shared_dpll *pll,
 					struct intel_dpll_hw_state *hw_state)
 {
-	enum port port = (enum port)pll->id;	/* 1:1 port->PLL mapping */
+	enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */
 	uint32_t val;
 	bool ret;
 	enum dpio_phy phy;
@@ -1824,7 +1843,7 @@ bxt_get_dpll(struct intel_crtc *crtc,
 	pll = intel_get_shared_dpll_by_id(dev_priv, i);
 
 	DRM_DEBUG_KMS("[CRTC:%d:%s] using pre-allocated %s\n",
-		      crtc->base.base.id, crtc->base.name, pll->name);
+		      crtc->base.base.id, crtc->base.name, pll->info->name);
 
 	intel_reference_shared_dpll(pll, crtc_state);
 
@@ -1877,13 +1896,6 @@ static void intel_ddi_pll_init(struct drm_device *dev)
 	}
 }
 
-struct dpll_info {
-	const char *name;
-	const int id;
-	const struct intel_shared_dpll_funcs *funcs;
-	uint32_t flags;
-};
-
 struct intel_dpll_mgr {
 	const struct dpll_info *dpll_info;
 
@@ -1896,9 +1908,9 @@ struct intel_dpll_mgr {
 };
 
 static const struct dpll_info pch_plls[] = {
-	{ "PCH DPLL A", DPLL_ID_PCH_PLL_A, &ibx_pch_dpll_funcs, 0 },
-	{ "PCH DPLL B", DPLL_ID_PCH_PLL_B, &ibx_pch_dpll_funcs, 0 },
-	{ NULL, -1, NULL, 0 },
+	{ "PCH DPLL A", &ibx_pch_dpll_funcs, DPLL_ID_PCH_PLL_A, 0 },
+	{ "PCH DPLL B", &ibx_pch_dpll_funcs, DPLL_ID_PCH_PLL_B, 0 },
+	{ },
 };
 
 static const struct intel_dpll_mgr pch_pll_mgr = {
@@ -1908,13 +1920,13 @@ static const struct intel_dpll_mgr pch_pll_mgr = {
 };
 
 static const struct dpll_info hsw_plls[] = {
-	{ "WRPLL 1",    DPLL_ID_WRPLL1,     &hsw_ddi_wrpll_funcs, 0 },
-	{ "WRPLL 2",    DPLL_ID_WRPLL2,     &hsw_ddi_wrpll_funcs, 0 },
-	{ "SPLL",       DPLL_ID_SPLL,       &hsw_ddi_spll_funcs,  0 },
-	{ "LCPLL 810",  DPLL_ID_LCPLL_810,  &hsw_ddi_lcpll_funcs, INTEL_DPLL_ALWAYS_ON },
-	{ "LCPLL 1350", DPLL_ID_LCPLL_1350, &hsw_ddi_lcpll_funcs, INTEL_DPLL_ALWAYS_ON },
-	{ "LCPLL 2700", DPLL_ID_LCPLL_2700, &hsw_ddi_lcpll_funcs, INTEL_DPLL_ALWAYS_ON },
-	{ NULL, -1, NULL, },
+	{ "WRPLL 1",    &hsw_ddi_wrpll_funcs, DPLL_ID_WRPLL1,     0 },
+	{ "WRPLL 2",    &hsw_ddi_wrpll_funcs, DPLL_ID_WRPLL2,     0 },
+	{ "SPLL",       &hsw_ddi_spll_funcs,  DPLL_ID_SPLL,       0 },
+	{ "LCPLL 810",  &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_810,  INTEL_DPLL_ALWAYS_ON },
+	{ "LCPLL 1350", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_1350, INTEL_DPLL_ALWAYS_ON },
+	{ "LCPLL 2700", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_2700, INTEL_DPLL_ALWAYS_ON },
+	{ },
 };
 
 static const struct intel_dpll_mgr hsw_pll_mgr = {
@@ -1924,11 +1936,11 @@ static const struct intel_dpll_mgr hsw_pll_mgr = {
 };
 
 static const struct dpll_info skl_plls[] = {
-	{ "DPLL 0", DPLL_ID_SKL_DPLL0, &skl_ddi_dpll0_funcs, INTEL_DPLL_ALWAYS_ON },
-	{ "DPLL 1", DPLL_ID_SKL_DPLL1, &skl_ddi_pll_funcs,   0 },
-	{ "DPLL 2", DPLL_ID_SKL_DPLL2, &skl_ddi_pll_funcs,   0 },
-	{ "DPLL 3", DPLL_ID_SKL_DPLL3, &skl_ddi_pll_funcs,   0 },
-	{ NULL, -1, NULL, },
+	{ "DPLL 0", &skl_ddi_dpll0_funcs, DPLL_ID_SKL_DPLL0, INTEL_DPLL_ALWAYS_ON },
+	{ "DPLL 1", &skl_ddi_pll_funcs,   DPLL_ID_SKL_DPLL1, 0 },
+	{ "DPLL 2", &skl_ddi_pll_funcs,   DPLL_ID_SKL_DPLL2, 0 },
+	{ "DPLL 3", &skl_ddi_pll_funcs,   DPLL_ID_SKL_DPLL3, 0 },
+	{ },
 };
 
 static const struct intel_dpll_mgr skl_pll_mgr = {
@@ -1938,10 +1950,10 @@ static const struct intel_dpll_mgr skl_pll_mgr = {
 };
 
 static const struct dpll_info bxt_plls[] = {
-	{ "PORT PLL A", DPLL_ID_SKL_DPLL0, &bxt_ddi_pll_funcs, 0 },
-	{ "PORT PLL B", DPLL_ID_SKL_DPLL1, &bxt_ddi_pll_funcs, 0 },
-	{ "PORT PLL C", DPLL_ID_SKL_DPLL2, &bxt_ddi_pll_funcs, 0 },
-	{ NULL, -1, NULL, },
+	{ "PORT PLL A", &bxt_ddi_pll_funcs, DPLL_ID_SKL_DPLL0, 0 },
+	{ "PORT PLL B", &bxt_ddi_pll_funcs, DPLL_ID_SKL_DPLL1, 0 },
+	{ "PORT PLL C", &bxt_ddi_pll_funcs, DPLL_ID_SKL_DPLL2, 0 },
+	{ },
 };
 
 static const struct intel_dpll_mgr bxt_pll_mgr = {
@@ -1953,38 +1965,39 @@ static const struct intel_dpll_mgr bxt_pll_mgr = {
 static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv,
 			       struct intel_shared_dpll *pll)
 {
+	const enum intel_dpll_id id = pll->info->id;
 	uint32_t val;
 
 	/* 1. Enable DPLL power in DPLL_ENABLE. */
-	val = I915_READ(CNL_DPLL_ENABLE(pll->id));
+	val = I915_READ(CNL_DPLL_ENABLE(id));
 	val |= PLL_POWER_ENABLE;
-	I915_WRITE(CNL_DPLL_ENABLE(pll->id), val);
+	I915_WRITE(CNL_DPLL_ENABLE(id), val);
 
 	/* 2. Wait for DPLL power state enabled in DPLL_ENABLE. */
 	if (intel_wait_for_register(dev_priv,
-				    CNL_DPLL_ENABLE(pll->id),
+				    CNL_DPLL_ENABLE(id),
 				    PLL_POWER_STATE,
 				    PLL_POWER_STATE,
 				    5))
-		DRM_ERROR("PLL %d Power not enabled\n", pll->id);
+		DRM_ERROR("PLL %d Power not enabled\n", id);
 
 	/*
 	 * 3. Configure DPLL_CFGCR0 to set SSC enable/disable,
 	 * select DP mode, and set DP link rate.
 	 */
 	val = pll->state.hw_state.cfgcr0;
-	I915_WRITE(CNL_DPLL_CFGCR0(pll->id), val);
+	I915_WRITE(CNL_DPLL_CFGCR0(id), val);
 
 	/* 4. Reab back to ensure writes completed */
-	POSTING_READ(CNL_DPLL_CFGCR0(pll->id));
+	POSTING_READ(CNL_DPLL_CFGCR0(id));
 
 	/* 3. Configure DPLL_CFGCR0 */
 	/* Avoid touch CFGCR1 if HDMI mode is not enabled */
 	if (pll->state.hw_state.cfgcr0 & DPLL_CFGCR0_HDMI_MODE) {
 		val = pll->state.hw_state.cfgcr1;
-		I915_WRITE(CNL_DPLL_CFGCR1(pll->id), val);
+		I915_WRITE(CNL_DPLL_CFGCR1(id), val);
 		/* 4. Reab back to ensure writes completed */
-		POSTING_READ(CNL_DPLL_CFGCR1(pll->id));
+		POSTING_READ(CNL_DPLL_CFGCR1(id));
 	}
 
 	/*
@@ -1997,17 +2010,17 @@ static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	 */
 
 	/* 6. Enable DPLL in DPLL_ENABLE. */
-	val = I915_READ(CNL_DPLL_ENABLE(pll->id));
+	val = I915_READ(CNL_DPLL_ENABLE(id));
 	val |= PLL_ENABLE;
-	I915_WRITE(CNL_DPLL_ENABLE(pll->id), val);
+	I915_WRITE(CNL_DPLL_ENABLE(id), val);
 
 	/* 7. Wait for PLL lock status in DPLL_ENABLE. */
 	if (intel_wait_for_register(dev_priv,
-				    CNL_DPLL_ENABLE(pll->id),
+				    CNL_DPLL_ENABLE(id),
 				    PLL_LOCK,
 				    PLL_LOCK,
 				    5))
-		DRM_ERROR("PLL %d not locked\n", pll->id);
+		DRM_ERROR("PLL %d not locked\n", id);
 
 	/*
 	 * 8. If the frequency will result in a change to the voltage
@@ -2027,6 +2040,7 @@ static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv,
 static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv,
 				struct intel_shared_dpll *pll)
 {
+	const enum intel_dpll_id id = pll->info->id;
 	uint32_t val;
 
 	/*
@@ -2044,17 +2058,17 @@ static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv,
 	 */
 
 	/* 3. Disable DPLL through DPLL_ENABLE. */
-	val = I915_READ(CNL_DPLL_ENABLE(pll->id));
+	val = I915_READ(CNL_DPLL_ENABLE(id));
 	val &= ~PLL_ENABLE;
-	I915_WRITE(CNL_DPLL_ENABLE(pll->id), val);
+	I915_WRITE(CNL_DPLL_ENABLE(id), val);
 
 	/* 4. Wait for PLL not locked status in DPLL_ENABLE. */
 	if (intel_wait_for_register(dev_priv,
-				    CNL_DPLL_ENABLE(pll->id),
+				    CNL_DPLL_ENABLE(id),
 				    PLL_LOCK,
 				    0,
 				    5))
-		DRM_ERROR("PLL %d locked\n", pll->id);
+		DRM_ERROR("PLL %d locked\n", id);
 
 	/*
 	 * 5. If the frequency will result in a change to the voltage
@@ -2066,23 +2080,24 @@ static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv,
 	 */
 
 	/* 6. Disable DPLL power in DPLL_ENABLE. */
-	val = I915_READ(CNL_DPLL_ENABLE(pll->id));
+	val = I915_READ(CNL_DPLL_ENABLE(id));
 	val &= ~PLL_POWER_ENABLE;
-	I915_WRITE(CNL_DPLL_ENABLE(pll->id), val);
+	I915_WRITE(CNL_DPLL_ENABLE(id), val);
 
 	/* 7. Wait for DPLL power state disabled in DPLL_ENABLE. */
 	if (intel_wait_for_register(dev_priv,
-				    CNL_DPLL_ENABLE(pll->id),
+				    CNL_DPLL_ENABLE(id),
 				    PLL_POWER_STATE,
 				    0,
 				    5))
-		DRM_ERROR("PLL %d Power not disabled\n", pll->id);
+		DRM_ERROR("PLL %d Power not disabled\n", id);
 }
 
 static bool cnl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv,
 				     struct intel_shared_dpll *pll,
 				     struct intel_dpll_hw_state *hw_state)
 {
+	const enum intel_dpll_id id = pll->info->id;
 	uint32_t val;
 	bool ret;
 
@@ -2091,16 +2106,16 @@ static bool cnl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv,
 
 	ret = false;
 
-	val = I915_READ(CNL_DPLL_ENABLE(pll->id));
+	val = I915_READ(CNL_DPLL_ENABLE(id));
 	if (!(val & PLL_ENABLE))
 		goto out;
 
-	val = I915_READ(CNL_DPLL_CFGCR0(pll->id));
+	val = I915_READ(CNL_DPLL_CFGCR0(id));
 	hw_state->cfgcr0 = val;
 
 	/* avoid reading back stale values if HDMI mode is not enabled */
 	if (val & DPLL_CFGCR0_HDMI_MODE) {
-		hw_state->cfgcr1 = I915_READ(CNL_DPLL_CFGCR1(pll->id));
+		hw_state->cfgcr1 = I915_READ(CNL_DPLL_CFGCR1(id));
 	}
 	ret = true;
 
@@ -2203,6 +2218,7 @@ cnl_ddi_calculate_wrpll(int clock,
 			struct skl_wrpll_params *wrpll_params)
 {
 	u32 afe_clock = clock * 5;
+	uint32_t ref_clock;
 	u32 dco_min = 7998000;
 	u32 dco_max = 10000000;
 	u32 dco_mid = (dco_min + dco_max) / 2;
@@ -2235,8 +2251,17 @@ cnl_ddi_calculate_wrpll(int clock,
 
 	cnl_wrpll_get_multipliers(best_div, &pdiv, &qdiv, &kdiv);
 
-	cnl_wrpll_params_populate(wrpll_params, best_dco,
-				  dev_priv->cdclk.hw.ref, pdiv, qdiv, kdiv);
+	ref_clock = dev_priv->cdclk.hw.ref;
+
+	/*
+	 * For ICL, the spec states: if reference frequency is 38.4, use 19.2
+	 * because the DPLL automatically divides that by 2.
+	 */
+	if (IS_ICELAKE(dev_priv) && ref_clock == 38400)
+		ref_clock = 19200;
+
+	cnl_wrpll_params_populate(wrpll_params, best_dco, ref_clock, pdiv, qdiv,
+				  kdiv);
 
 	return true;
 }
@@ -2372,10 +2397,10 @@ static const struct intel_shared_dpll_funcs cnl_ddi_pll_funcs = {
 };
 
 static const struct dpll_info cnl_plls[] = {
-	{ "DPLL 0", DPLL_ID_SKL_DPLL0, &cnl_ddi_pll_funcs, 0 },
-	{ "DPLL 1", DPLL_ID_SKL_DPLL1, &cnl_ddi_pll_funcs, 0 },
-	{ "DPLL 2", DPLL_ID_SKL_DPLL2, &cnl_ddi_pll_funcs, 0 },
-	{ NULL, -1, NULL, },
+	{ "DPLL 0", &cnl_ddi_pll_funcs, DPLL_ID_SKL_DPLL0, 0 },
+	{ "DPLL 1", &cnl_ddi_pll_funcs, DPLL_ID_SKL_DPLL1, 0 },
+	{ "DPLL 2", &cnl_ddi_pll_funcs, DPLL_ID_SKL_DPLL2, 0 },
+	{ },
 };
 
 static const struct intel_dpll_mgr cnl_pll_mgr = {
@@ -2384,6 +2409,644 @@ static const struct intel_dpll_mgr cnl_pll_mgr = {
 	.dump_hw_state = cnl_dump_hw_state,
 };
 
+/*
+ * These values alrea already adjusted: they're the bits we write to the
+ * registers, not the logical values.
+ */
+static const struct skl_wrpll_params icl_dp_combo_pll_24MHz_values[] = {
+	{ .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [0]: 5.4 */
+	  .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [1]: 2.7 */
+	  .pdiv = 0x2 /* 3 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [2]: 1.62 */
+	  .pdiv = 0x4 /* 5 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [3]: 3.24 */
+	  .pdiv = 0x4 /* 5 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x168, .dco_fraction = 0x0000,		/* [4]: 2.16 */
+	  .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 1, .qdiv_ratio = 2},
+	{ .dco_integer = 0x168, .dco_fraction = 0x0000,		/* [5]: 4.32 */
+	  .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x195, .dco_fraction = 0x0000,		/* [6]: 6.48 */
+	  .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [7]: 8.1 */
+	  .pdiv = 0x1 /* 2 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
+};
+
+/* Also used for 38.4 MHz values. */
+static const struct skl_wrpll_params icl_dp_combo_pll_19_2MHz_values[] = {
+	{ .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [0]: 5.4 */
+	  .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [1]: 2.7 */
+	  .pdiv = 0x2 /* 3 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [2]: 1.62 */
+	  .pdiv = 0x4 /* 5 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [3]: 3.24 */
+	  .pdiv = 0x4 /* 5 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x1C2, .dco_fraction = 0x0000,		/* [4]: 2.16 */
+	  .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 1, .qdiv_ratio = 2},
+	{ .dco_integer = 0x1C2, .dco_fraction = 0x0000,		/* [5]: 4.32 */
+	  .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x1FA, .dco_fraction = 0x2000,		/* [6]: 6.48 */
+	  .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
+	{ .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [7]: 8.1 */
+	  .pdiv = 0x1 /* 2 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
+};
+
+static bool icl_calc_dp_combo_pll(struct drm_i915_private *dev_priv, int clock,
+				  struct skl_wrpll_params *pll_params)
+{
+	const struct skl_wrpll_params *params;
+
+	params = dev_priv->cdclk.hw.ref == 24000 ?
+			icl_dp_combo_pll_24MHz_values :
+			icl_dp_combo_pll_19_2MHz_values;
+
+	switch (clock) {
+	case 540000:
+		*pll_params = params[0];
+		break;
+	case 270000:
+		*pll_params = params[1];
+		break;
+	case 162000:
+		*pll_params = params[2];
+		break;
+	case 324000:
+		*pll_params = params[3];
+		break;
+	case 216000:
+		*pll_params = params[4];
+		break;
+	case 432000:
+		*pll_params = params[5];
+		break;
+	case 648000:
+		*pll_params = params[6];
+		break;
+	case 810000:
+		*pll_params = params[7];
+		break;
+	default:
+		MISSING_CASE(clock);
+		return false;
+	}
+
+	return true;
+}
+
+static bool icl_calc_dpll_state(struct intel_crtc_state *crtc_state,
+				struct intel_encoder *encoder, int clock,
+				struct intel_dpll_hw_state *pll_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	uint32_t cfgcr0, cfgcr1;
+	struct skl_wrpll_params pll_params = { 0 };
+	bool ret;
+
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
+		ret = cnl_ddi_calculate_wrpll(clock, dev_priv, &pll_params);
+	else
+		ret = icl_calc_dp_combo_pll(dev_priv, clock, &pll_params);
+
+	if (!ret)
+		return false;
+
+	cfgcr0 = DPLL_CFGCR0_DCO_FRACTION(pll_params.dco_fraction) |
+		 pll_params.dco_integer;
+
+	cfgcr1 = DPLL_CFGCR1_QDIV_RATIO(pll_params.qdiv_ratio) |
+		 DPLL_CFGCR1_QDIV_MODE(pll_params.qdiv_mode) |
+		 DPLL_CFGCR1_KDIV(pll_params.kdiv) |
+		 DPLL_CFGCR1_PDIV(pll_params.pdiv) |
+		 DPLL_CFGCR1_CENTRAL_FREQ_8400;
+
+	pll_state->cfgcr0 = cfgcr0;
+	pll_state->cfgcr1 = cfgcr1;
+	return true;
+}
+
+static enum port icl_mg_pll_id_to_port(enum intel_dpll_id id)
+{
+	return id - DPLL_ID_ICL_MGPLL1 + PORT_C;
+}
+
+static enum intel_dpll_id icl_port_to_mg_pll_id(enum port port)
+{
+	return port - PORT_C + DPLL_ID_ICL_MGPLL1;
+}
+
+static bool icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc,
+				     uint32_t *target_dco_khz,
+				     struct intel_dpll_hw_state *state)
+{
+	uint32_t dco_min_freq, dco_max_freq;
+	int div1_vals[] = {7, 5, 3, 2};
+	unsigned int i;
+	int div2;
+
+	dco_min_freq = is_dp ? 8100000 : use_ssc ? 8000000 : 7992000;
+	dco_max_freq = is_dp ? 8100000 : 10000000;
+
+	for (i = 0; i < ARRAY_SIZE(div1_vals); i++) {
+		int div1 = div1_vals[i];
+
+		for (div2 = 10; div2 > 0; div2--) {
+			int dco = div1 * div2 * clock_khz * 5;
+			int a_divratio, tlinedrv, inputsel, hsdiv;
+
+			if (dco < dco_min_freq || dco > dco_max_freq)
+				continue;
+
+			if (div2 >= 2) {
+				a_divratio = is_dp ? 10 : 5;
+				tlinedrv = 2;
+			} else {
+				a_divratio = 5;
+				tlinedrv = 0;
+			}
+			inputsel = is_dp ? 0 : 1;
+
+			switch (div1) {
+			default:
+				MISSING_CASE(div1);
+			case 2:
+				hsdiv = 0;
+				break;
+			case 3:
+				hsdiv = 1;
+				break;
+			case 5:
+				hsdiv = 2;
+				break;
+			case 7:
+				hsdiv = 3;
+				break;
+			}
+
+			*target_dco_khz = dco;
+
+			state->mg_refclkin_ctl = MG_REFCLKIN_CTL_OD_2_MUX(1);
+
+			state->mg_clktop2_coreclkctl1 =
+				MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO(a_divratio);
+
+			state->mg_clktop2_hsclkctl =
+				MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL(tlinedrv) |
+				MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL(inputsel) |
+				MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO(hsdiv) |
+				MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO(div2);
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+/*
+ * The specification for this function uses real numbers, so the math had to be
+ * adapted to integer-only calculation, that's why it looks so different.
+ */
+static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
+				  struct intel_encoder *encoder, int clock,
+				  struct intel_dpll_hw_state *pll_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	int refclk_khz = dev_priv->cdclk.hw.ref;
+	uint32_t dco_khz, m1div, m2div_int, m2div_rem, m2div_frac;
+	uint32_t iref_ndiv, iref_trim, iref_pulse_w;
+	uint32_t prop_coeff, int_coeff;
+	uint32_t tdc_targetcnt, feedfwgain;
+	uint64_t ssc_stepsize, ssc_steplen, ssc_steplog;
+	uint64_t tmp;
+	bool use_ssc = false;
+	bool is_dp = !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI);
+
+	if (!icl_mg_pll_find_divisors(clock, is_dp, use_ssc, &dco_khz,
+				      pll_state)) {
+		DRM_DEBUG_KMS("Failed to find divisors for clock %d\n", clock);
+		return false;
+	}
+
+	m1div = 2;
+	m2div_int = dco_khz / (refclk_khz * m1div);
+	if (m2div_int > 255) {
+		m1div = 4;
+		m2div_int = dco_khz / (refclk_khz * m1div);
+		if (m2div_int > 255) {
+			DRM_DEBUG_KMS("Failed to find mdiv for clock %d\n",
+				      clock);
+			return false;
+		}
+	}
+	m2div_rem = dco_khz % (refclk_khz * m1div);
+
+	tmp = (uint64_t)m2div_rem * (1 << 22);
+	do_div(tmp, refclk_khz * m1div);
+	m2div_frac = tmp;
+
+	switch (refclk_khz) {
+	case 19200:
+		iref_ndiv = 1;
+		iref_trim = 28;
+		iref_pulse_w = 1;
+		break;
+	case 24000:
+		iref_ndiv = 1;
+		iref_trim = 25;
+		iref_pulse_w = 2;
+		break;
+	case 38400:
+		iref_ndiv = 2;
+		iref_trim = 28;
+		iref_pulse_w = 1;
+		break;
+	default:
+		MISSING_CASE(refclk_khz);
+		return false;
+	}
+
+	/*
+	 * tdc_res = 0.000003
+	 * tdc_targetcnt = int(2 / (tdc_res * 8 * 50 * 1.1) / refclk_mhz + 0.5)
+	 *
+	 * The multiplication by 1000 is due to refclk MHz to KHz conversion. It
+	 * was supposed to be a division, but we rearranged the operations of
+	 * the formula to avoid early divisions so we don't multiply the
+	 * rounding errors.
+	 *
+	 * 0.000003 * 8 * 50 * 1.1 = 0.00132, also known as 132 / 100000, which
+	 * we also rearrange to work with integers.
+	 *
+	 * The 0.5 transformed to 5 results in a multiplication by 10 and the
+	 * last division by 10.
+	 */
+	tdc_targetcnt = (2 * 1000 * 100000 * 10 / (132 * refclk_khz) + 5) / 10;
+
+	/*
+	 * Here we divide dco_khz by 10 in order to allow the dividend to fit in
+	 * 32 bits. That's not a problem since we round the division down
+	 * anyway.
+	 */
+	feedfwgain = (use_ssc || m2div_rem > 0) ?
+		m1div * 1000000 * 100 / (dco_khz * 3 / 10) : 0;
+
+	if (dco_khz >= 9000000) {
+		prop_coeff = 5;
+		int_coeff = 10;
+	} else {
+		prop_coeff = 4;
+		int_coeff = 8;
+	}
+
+	if (use_ssc) {
+		tmp = (uint64_t)dco_khz * 47 * 32;
+		do_div(tmp, refclk_khz * m1div * 10000);
+		ssc_stepsize = tmp;
+
+		tmp = (uint64_t)dco_khz * 1000;
+		ssc_steplen = DIV_ROUND_UP_ULL(tmp, 32 * 2 * 32);
+	} else {
+		ssc_stepsize = 0;
+		ssc_steplen = 0;
+	}
+	ssc_steplog = 4;
+
+	pll_state->mg_pll_div0 = (m2div_rem > 0 ? MG_PLL_DIV0_FRACNEN_H : 0) |
+				  MG_PLL_DIV0_FBDIV_FRAC(m2div_frac) |
+				  MG_PLL_DIV0_FBDIV_INT(m2div_int);
+
+	pll_state->mg_pll_div1 = MG_PLL_DIV1_IREF_NDIVRATIO(iref_ndiv) |
+				 MG_PLL_DIV1_DITHER_DIV_2 |
+				 MG_PLL_DIV1_NDIVRATIO(1) |
+				 MG_PLL_DIV1_FBPREDIV(m1div);
+
+	pll_state->mg_pll_lf = MG_PLL_LF_TDCTARGETCNT(tdc_targetcnt) |
+			       MG_PLL_LF_AFCCNTSEL_512 |
+			       MG_PLL_LF_GAINCTRL(1) |
+			       MG_PLL_LF_INT_COEFF(int_coeff) |
+			       MG_PLL_LF_PROP_COEFF(prop_coeff);
+
+	pll_state->mg_pll_frac_lock = MG_PLL_FRAC_LOCK_TRUELOCK_CRIT_32 |
+				      MG_PLL_FRAC_LOCK_EARLYLOCK_CRIT_32 |
+				      MG_PLL_FRAC_LOCK_LOCKTHRESH(10) |
+				      MG_PLL_FRAC_LOCK_DCODITHEREN |
+				      MG_PLL_FRAC_LOCK_FEEDFWRDGAIN(feedfwgain);
+	if (use_ssc || m2div_rem > 0)
+		pll_state->mg_pll_frac_lock |= MG_PLL_FRAC_LOCK_FEEDFWRDCAL_EN;
+
+	pll_state->mg_pll_ssc = (use_ssc ? MG_PLL_SSC_EN : 0) |
+				MG_PLL_SSC_TYPE(2) |
+				MG_PLL_SSC_STEPLENGTH(ssc_steplen) |
+				MG_PLL_SSC_STEPNUM(ssc_steplog) |
+				MG_PLL_SSC_FLLEN |
+				MG_PLL_SSC_STEPSIZE(ssc_stepsize);
+
+	pll_state->mg_pll_tdc_coldst_bias = MG_PLL_TDC_COLDST_COLDSTART;
+
+	if (refclk_khz != 38400) {
+		pll_state->mg_pll_tdc_coldst_bias |=
+			MG_PLL_TDC_COLDST_IREFINT_EN |
+			MG_PLL_TDC_COLDST_REFBIAS_START_PULSE_W(iref_pulse_w) |
+			MG_PLL_TDC_COLDST_COLDSTART |
+			MG_PLL_TDC_TDCOVCCORR_EN |
+			MG_PLL_TDC_TDCSEL(3);
+
+		pll_state->mg_pll_bias = MG_PLL_BIAS_BIAS_GB_SEL(3) |
+					 MG_PLL_BIAS_INIT_DCOAMP(0x3F) |
+					 MG_PLL_BIAS_BIAS_BONUS(10) |
+					 MG_PLL_BIAS_BIASCAL_EN |
+					 MG_PLL_BIAS_CTRIM(12) |
+					 MG_PLL_BIAS_VREF_RDAC(4) |
+					 MG_PLL_BIAS_IREFTRIM(iref_trim);
+	}
+
+	return true;
+}
+
+static struct intel_shared_dpll *
+icl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
+	     struct intel_encoder *encoder)
+{
+	struct intel_shared_dpll *pll;
+	struct intel_dpll_hw_state pll_state = {};
+	enum port port = encoder->port;
+	enum intel_dpll_id min, max;
+	int clock = crtc_state->port_clock;
+	bool ret;
+
+	switch (port) {
+	case PORT_A:
+	case PORT_B:
+		min = DPLL_ID_ICL_DPLL0;
+		max = DPLL_ID_ICL_DPLL1;
+		ret = icl_calc_dpll_state(crtc_state, encoder, clock,
+					  &pll_state);
+		break;
+	case PORT_C:
+	case PORT_D:
+	case PORT_E:
+	case PORT_F:
+		min = icl_port_to_mg_pll_id(port);
+		max = min;
+		ret = icl_calc_mg_pll_state(crtc_state, encoder, clock,
+					    &pll_state);
+		break;
+	default:
+		MISSING_CASE(port);
+		return NULL;
+	}
+
+	if (!ret) {
+		DRM_DEBUG_KMS("Could not calculate PLL state.\n");
+		return NULL;
+	}
+
+	crtc_state->dpll_hw_state = pll_state;
+
+	pll = intel_find_shared_dpll(crtc, crtc_state, min, max);
+	if (!pll) {
+		DRM_DEBUG_KMS("No PLL selected\n");
+		return NULL;
+	}
+
+	intel_reference_shared_dpll(pll, crtc_state);
+
+	return pll;
+}
+
+static i915_reg_t icl_pll_id_to_enable_reg(enum intel_dpll_id id)
+{
+	switch (id) {
+	default:
+		MISSING_CASE(id);
+	case DPLL_ID_ICL_DPLL0:
+	case DPLL_ID_ICL_DPLL1:
+		return CNL_DPLL_ENABLE(id);
+	case DPLL_ID_ICL_MGPLL1:
+	case DPLL_ID_ICL_MGPLL2:
+	case DPLL_ID_ICL_MGPLL3:
+	case DPLL_ID_ICL_MGPLL4:
+		return MG_PLL_ENABLE(icl_mg_pll_id_to_port(id));
+	}
+}
+
+static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv,
+				 struct intel_shared_dpll *pll,
+				 struct intel_dpll_hw_state *hw_state)
+{
+	const enum intel_dpll_id id = pll->info->id;
+	uint32_t val;
+	enum port port;
+	bool ret = false;
+
+	if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS))
+		return false;
+
+	val = I915_READ(icl_pll_id_to_enable_reg(id));
+	if (!(val & PLL_ENABLE))
+		goto out;
+
+	switch (id) {
+	case DPLL_ID_ICL_DPLL0:
+	case DPLL_ID_ICL_DPLL1:
+		hw_state->cfgcr0 = I915_READ(ICL_DPLL_CFGCR0(id));
+		hw_state->cfgcr1 = I915_READ(ICL_DPLL_CFGCR1(id));
+		break;
+	case DPLL_ID_ICL_MGPLL1:
+	case DPLL_ID_ICL_MGPLL2:
+	case DPLL_ID_ICL_MGPLL3:
+	case DPLL_ID_ICL_MGPLL4:
+		port = icl_mg_pll_id_to_port(id);
+		hw_state->mg_refclkin_ctl = I915_READ(MG_REFCLKIN_CTL(port));
+		hw_state->mg_clktop2_coreclkctl1 =
+			I915_READ(MG_CLKTOP2_CORECLKCTL1(port));
+		hw_state->mg_clktop2_hsclkctl =
+			I915_READ(MG_CLKTOP2_HSCLKCTL(port));
+		hw_state->mg_pll_div0 = I915_READ(MG_PLL_DIV0(port));
+		hw_state->mg_pll_div1 = I915_READ(MG_PLL_DIV1(port));
+		hw_state->mg_pll_lf = I915_READ(MG_PLL_LF(port));
+		hw_state->mg_pll_frac_lock = I915_READ(MG_PLL_FRAC_LOCK(port));
+		hw_state->mg_pll_ssc = I915_READ(MG_PLL_SSC(port));
+		hw_state->mg_pll_bias = I915_READ(MG_PLL_BIAS(port));
+		hw_state->mg_pll_tdc_coldst_bias =
+			I915_READ(MG_PLL_TDC_COLDST_BIAS(port));
+		break;
+	default:
+		MISSING_CASE(id);
+	}
+
+	ret = true;
+out:
+	intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
+	return ret;
+}
+
+static void icl_dpll_write(struct drm_i915_private *dev_priv,
+			   struct intel_shared_dpll *pll)
+{
+	struct intel_dpll_hw_state *hw_state = &pll->state.hw_state;
+	const enum intel_dpll_id id = pll->info->id;
+
+	I915_WRITE(ICL_DPLL_CFGCR0(id), hw_state->cfgcr0);
+	I915_WRITE(ICL_DPLL_CFGCR1(id), hw_state->cfgcr1);
+	POSTING_READ(ICL_DPLL_CFGCR1(id));
+}
+
+static void icl_mg_pll_write(struct drm_i915_private *dev_priv,
+			     struct intel_shared_dpll *pll)
+{
+	struct intel_dpll_hw_state *hw_state = &pll->state.hw_state;
+	enum port port = icl_mg_pll_id_to_port(pll->info->id);
+
+	I915_WRITE(MG_REFCLKIN_CTL(port), hw_state->mg_refclkin_ctl);
+	I915_WRITE(MG_CLKTOP2_CORECLKCTL1(port),
+		   hw_state->mg_clktop2_coreclkctl1);
+	I915_WRITE(MG_CLKTOP2_HSCLKCTL(port), hw_state->mg_clktop2_hsclkctl);
+	I915_WRITE(MG_PLL_DIV0(port), hw_state->mg_pll_div0);
+	I915_WRITE(MG_PLL_DIV1(port), hw_state->mg_pll_div1);
+	I915_WRITE(MG_PLL_LF(port), hw_state->mg_pll_lf);
+	I915_WRITE(MG_PLL_FRAC_LOCK(port), hw_state->mg_pll_frac_lock);
+	I915_WRITE(MG_PLL_SSC(port), hw_state->mg_pll_ssc);
+	I915_WRITE(MG_PLL_BIAS(port), hw_state->mg_pll_bias);
+	I915_WRITE(MG_PLL_TDC_COLDST_BIAS(port),
+		   hw_state->mg_pll_tdc_coldst_bias);
+	POSTING_READ(MG_PLL_TDC_COLDST_BIAS(port));
+}
+
+static void icl_pll_enable(struct drm_i915_private *dev_priv,
+			   struct intel_shared_dpll *pll)
+{
+	const enum intel_dpll_id id = pll->info->id;
+	i915_reg_t enable_reg = icl_pll_id_to_enable_reg(id);
+	uint32_t val;
+
+	val = I915_READ(enable_reg);
+	val |= PLL_POWER_ENABLE;
+	I915_WRITE(enable_reg, val);
+
+	/*
+	 * The spec says we need to "wait" but it also says it should be
+	 * immediate.
+	 */
+	if (intel_wait_for_register(dev_priv, enable_reg, PLL_POWER_STATE,
+				    PLL_POWER_STATE, 1))
+		DRM_ERROR("PLL %d Power not enabled\n", id);
+
+	switch (id) {
+	case DPLL_ID_ICL_DPLL0:
+	case DPLL_ID_ICL_DPLL1:
+		icl_dpll_write(dev_priv, pll);
+		break;
+	case DPLL_ID_ICL_MGPLL1:
+	case DPLL_ID_ICL_MGPLL2:
+	case DPLL_ID_ICL_MGPLL3:
+	case DPLL_ID_ICL_MGPLL4:
+		icl_mg_pll_write(dev_priv, pll);
+		break;
+	default:
+		MISSING_CASE(id);
+	}
+
+	/*
+	 * DVFS pre sequence would be here, but in our driver the cdclk code
+	 * paths should already be setting the appropriate voltage, hence we do
+	 * nothign here.
+	 */
+
+	val = I915_READ(enable_reg);
+	val |= PLL_ENABLE;
+	I915_WRITE(enable_reg, val);
+
+	if (intel_wait_for_register(dev_priv, enable_reg, PLL_LOCK, PLL_LOCK,
+				    1)) /* 600us actually. */
+		DRM_ERROR("PLL %d not locked\n", id);
+
+	/* DVFS post sequence would be here. See the comment above. */
+}
+
+static void icl_pll_disable(struct drm_i915_private *dev_priv,
+			    struct intel_shared_dpll *pll)
+{
+	const enum intel_dpll_id id = pll->info->id;
+	i915_reg_t enable_reg = icl_pll_id_to_enable_reg(id);
+	uint32_t val;
+
+	/* The first steps are done by intel_ddi_post_disable(). */
+
+	/*
+	 * DVFS pre sequence would be here, but in our driver the cdclk code
+	 * paths should already be setting the appropriate voltage, hence we do
+	 * nothign here.
+	 */
+
+	val = I915_READ(enable_reg);
+	val &= ~PLL_ENABLE;
+	I915_WRITE(enable_reg, val);
+
+	/* Timeout is actually 1us. */
+	if (intel_wait_for_register(dev_priv, enable_reg, PLL_LOCK, 0, 1))
+		DRM_ERROR("PLL %d locked\n", id);
+
+	/* DVFS post sequence would be here. See the comment above. */
+
+	val = I915_READ(enable_reg);
+	val &= ~PLL_POWER_ENABLE;
+	I915_WRITE(enable_reg, val);
+
+	/*
+	 * The spec says we need to "wait" but it also says it should be
+	 * immediate.
+	 */
+	if (intel_wait_for_register(dev_priv, enable_reg, PLL_POWER_STATE, 0,
+				    1))
+		DRM_ERROR("PLL %d Power not disabled\n", id);
+}
+
+static void icl_dump_hw_state(struct drm_i915_private *dev_priv,
+			      struct intel_dpll_hw_state *hw_state)
+{
+	DRM_DEBUG_KMS("dpll_hw_state: cfgcr0: 0x%x, cfgcr1: 0x%x, "
+		      "mg_refclkin_ctl: 0x%x, hg_clktop2_coreclkctl1: 0x%x, "
+		      "mg_clktop2_hsclkctl: 0x%x, mg_pll_div0: 0x%x, "
+		      "mg_pll_div2: 0x%x, mg_pll_lf: 0x%x, "
+		      "mg_pll_frac_lock: 0x%x, mg_pll_ssc: 0x%x, "
+		      "mg_pll_bias: 0x%x, mg_pll_tdc_coldst_bias: 0x%x\n",
+		      hw_state->cfgcr0, hw_state->cfgcr1,
+		      hw_state->mg_refclkin_ctl,
+		      hw_state->mg_clktop2_coreclkctl1,
+		      hw_state->mg_clktop2_hsclkctl,
+		      hw_state->mg_pll_div0,
+		      hw_state->mg_pll_div1,
+		      hw_state->mg_pll_lf,
+		      hw_state->mg_pll_frac_lock,
+		      hw_state->mg_pll_ssc,
+		      hw_state->mg_pll_bias,
+		      hw_state->mg_pll_tdc_coldst_bias);
+}
+
+static const struct intel_shared_dpll_funcs icl_pll_funcs = {
+	.enable = icl_pll_enable,
+	.disable = icl_pll_disable,
+	.get_hw_state = icl_pll_get_hw_state,
+};
+
+static const struct dpll_info icl_plls[] = {
+	{ "DPLL 0",   &icl_pll_funcs, DPLL_ID_ICL_DPLL0,  0 },
+	{ "DPLL 1",   &icl_pll_funcs, DPLL_ID_ICL_DPLL1,  0 },
+	{ "MG PLL 1", &icl_pll_funcs, DPLL_ID_ICL_MGPLL1, 0 },
+	{ "MG PLL 2", &icl_pll_funcs, DPLL_ID_ICL_MGPLL2, 0 },
+	{ "MG PLL 3", &icl_pll_funcs, DPLL_ID_ICL_MGPLL3, 0 },
+	{ "MG PLL 4", &icl_pll_funcs, DPLL_ID_ICL_MGPLL4, 0 },
+	{ },
+};
+
+static const struct intel_dpll_mgr icl_pll_mgr = {
+	.dpll_info = icl_plls,
+	.get_dpll = icl_get_dpll,
+	.dump_hw_state = icl_dump_hw_state,
+};
+
 /**
  * intel_shared_dpll_init - Initialize shared DPLLs
  * @dev: drm device
@@ -2397,7 +3060,9 @@ void intel_shared_dpll_init(struct drm_device *dev)
 	const struct dpll_info *dpll_info;
 	int i;
 
-	if (IS_CANNONLAKE(dev_priv))
+	if (IS_ICELAKE(dev_priv))
+		dpll_mgr = &icl_pll_mgr;
+	else if (IS_CANNONLAKE(dev_priv))
 		dpll_mgr = &cnl_pll_mgr;
 	else if (IS_GEN9_BC(dev_priv))
 		dpll_mgr = &skl_pll_mgr;
@@ -2415,13 +3080,9 @@ void intel_shared_dpll_init(struct drm_device *dev)
 
 	dpll_info = dpll_mgr->dpll_info;
 
-	for (i = 0; dpll_info[i].id >= 0; i++) {
+	for (i = 0; dpll_info[i].name; i++) {
 		WARN_ON(i != dpll_info[i].id);
-
-		dev_priv->shared_dplls[i].id = dpll_info[i].id;
-		dev_priv->shared_dplls[i].name = dpll_info[i].name;
-		dev_priv->shared_dplls[i].funcs = *dpll_info[i].funcs;
-		dev_priv->shared_dplls[i].flags = dpll_info[i].flags;
+		dev_priv->shared_dplls[i].info = &dpll_info[i];
 	}
 
 	dev_priv->dpll_mgr = dpll_mgr;
@@ -2481,7 +3142,7 @@ void intel_release_shared_dpll(struct intel_shared_dpll *dpll,
 	struct intel_shared_dpll_state *shared_dpll_state;
 
 	shared_dpll_state = intel_atomic_get_shared_dpll_state(state);
-	shared_dpll_state[dpll->id].crtc_mask &= ~(1 << crtc->pipe);
+	shared_dpll_state[dpll->info->id].crtc_mask &= ~(1 << crtc->pipe);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.h b/drivers/gpu/drm/i915/intel_dpll_mgr.h
index f24ccf443d25..7a0cd564a9ee 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.h
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.h
@@ -103,6 +103,32 @@ enum intel_dpll_id {
 	 * @DPLL_ID_SKL_DPLL3: SKL and later DPLL3
 	 */
 	DPLL_ID_SKL_DPLL3 = 3,
+
+
+	/**
+	 * @DPLL_ID_ICL_DPLL0: ICL combo PHY DPLL0
+	 */
+	DPLL_ID_ICL_DPLL0 = 0,
+	/**
+	 * @DPLL_ID_ICL_DPLL1: ICL combo PHY DPLL1
+	 */
+	DPLL_ID_ICL_DPLL1 = 1,
+	/**
+	 * @DPLL_ID_ICL_MGPLL1: ICL MG PLL 1 port 1 (C)
+	 */
+	DPLL_ID_ICL_MGPLL1 = 2,
+	/**
+	 * @DPLL_ID_ICL_MGPLL2: ICL MG PLL 1 port 2 (D)
+	 */
+	DPLL_ID_ICL_MGPLL2 = 3,
+	/**
+	 * @DPLL_ID_ICL_MGPLL3: ICL MG PLL 1 port 3 (E)
+	 */
+	DPLL_ID_ICL_MGPLL3 = 4,
+	/**
+	 * @DPLL_ID_ICL_MGPLL4: ICL MG PLL 1 port 4 (F)
+	 */
+	DPLL_ID_ICL_MGPLL4 = 5,
 };
 #define I915_NUM_PLLS 6
 
@@ -135,6 +161,21 @@ struct intel_dpll_hw_state {
 	/* bxt */
 	uint32_t ebb0, ebb4, pll0, pll1, pll2, pll3, pll6, pll8, pll9, pll10,
 		 pcsdw12;
+
+	/*
+	 * ICL uses the following, already defined:
+	 * uint32_t cfgcr0, cfgcr1;
+	 */
+	uint32_t mg_refclkin_ctl;
+	uint32_t mg_clktop2_coreclkctl1;
+	uint32_t mg_clktop2_hsclkctl;
+	uint32_t mg_pll_div0;
+	uint32_t mg_pll_div1;
+	uint32_t mg_pll_lf;
+	uint32_t mg_pll_frac_lock;
+	uint32_t mg_pll_ssc;
+	uint32_t mg_pll_bias;
+	uint32_t mg_pll_tdc_coldst_bias;
 };
 
 /**
@@ -205,6 +246,37 @@ struct intel_shared_dpll_funcs {
 			     struct intel_dpll_hw_state *hw_state);
 };
 
+/**
+ * struct dpll_info - display PLL platform specific info
+ */
+struct dpll_info {
+	/**
+	 * @name: DPLL name; used for logging
+	 */
+	const char *name;
+
+	/**
+	 * @funcs: platform specific hooks
+	 */
+	const struct intel_shared_dpll_funcs *funcs;
+
+	/**
+	 * @id: unique indentifier for this DPLL; should match the index in the
+	 * dev_priv->shared_dplls array
+	 */
+	enum intel_dpll_id id;
+
+#define INTEL_DPLL_ALWAYS_ON	(1 << 0)
+	/**
+	 * @flags:
+	 *
+	 * INTEL_DPLL_ALWAYS_ON
+	 *     Inform the state checker that the DPLL is kept enabled even if
+	 *     not in use by any CRTC.
+	 */
+	uint32_t flags;
+};
+
 /**
  * struct intel_shared_dpll - display PLL with tracked state and users
  */
@@ -228,30 +300,9 @@ struct intel_shared_dpll {
 	bool on;
 
 	/**
-	 * @name: DPLL name; used for logging
+	 * @info: platform specific info
 	 */
-	const char *name;
-
-	/**
-	 * @id: unique indentifier for this DPLL; should match the index in the
-	 * dev_priv->shared_dplls array
-	 */
-	enum intel_dpll_id id;
-
-	/**
-	 * @funcs: platform specific hooks
-	 */
-	struct intel_shared_dpll_funcs funcs;
-
-#define INTEL_DPLL_ALWAYS_ON	(1 << 0)
-	/**
-	 * @flags:
-	 *
-	 * INTEL_DPLL_ALWAYS_ON
-	 *     Inform the state checker that the DPLL is kept enabled even if
-	 *     not in use by any CRTC.
-	 */
-	uint32_t flags;
+	const struct dpll_info *info;
 };
 
 #define SKL_DPLL0 0
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index a80fbad9be0f..d7dbca1aabff 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -56,6 +56,8 @@
 	for (;;) {							\
 		const bool expired__ = ktime_after(ktime_get_raw(), end__); \
 		OP;							\
+		/* Guarantee COND check prior to timeout */		\
+		barrier();						\
 		if (COND) {						\
 			ret__ = 0;					\
 			break;						\
@@ -96,6 +98,8 @@
 		u64 now = local_clock(); \
 		if (!(ATOMIC)) \
 			preempt_enable(); \
+		/* Guarantee COND check prior to timeout */ \
+		barrier(); \
 		if (COND) { \
 			ret = 0; \
 			break; \
@@ -140,6 +144,10 @@
 #define KHz(x) (1000 * (x))
 #define MHz(x) KHz(1000 * (x))
 
+#define KBps(x) (1000 * (x))
+#define MBps(x) KBps(1000 * (x))
+#define GBps(x) ((u64)1000 * MBps((x)))
+
 /*
  * Display related stuff
  */
@@ -482,7 +490,7 @@ struct intel_atomic_state {
 	bool skip_intermediate_wm;
 
 	/* Gen9+ only */
-	struct skl_wm_values wm_results;
+	struct skl_ddb_values wm_results;
 
 	struct i915_sw_fence commit_ready;
 
@@ -548,6 +556,12 @@ struct intel_initial_plane_config {
 #define SKL_MAX_DST_W 4096
 #define SKL_MIN_DST_H 8
 #define SKL_MAX_DST_H 4096
+#define ICL_MAX_SRC_W 5120
+#define ICL_MAX_SRC_H 4096
+#define ICL_MAX_DST_W 5120
+#define ICL_MAX_DST_H 4096
+#define SKL_MIN_YUV_420_SRC_W 16
+#define SKL_MIN_YUV_420_SRC_H 16
 
 struct intel_scaler {
 	int in_use;
@@ -598,7 +612,9 @@ struct intel_pipe_wm {
 
 struct skl_plane_wm {
 	struct skl_wm_level wm[8];
+	struct skl_wm_level uv_wm[8];
 	struct skl_wm_level trans_wm;
+	bool is_planar;
 };
 
 struct skl_pipe_wm {
@@ -874,6 +890,7 @@ struct intel_crtc_state {
 
 	/* bitmask of visible planes (enum plane_id) */
 	u8 active_planes;
+	u8 nv12_planes;
 
 	/* HDMI scrambling status */
 	bool hdmi_scrambling;
@@ -1321,10 +1338,14 @@ void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv);
 void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv);
 
 /* i915_irq.c */
+bool gen11_reset_one_iir(struct drm_i915_private * const i915,
+			 const unsigned int bank,
+			 const unsigned int bit);
 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask);
 void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask);
+void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv);
 void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv);
 void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv);
 void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv);
@@ -1389,6 +1410,12 @@ uint32_t ddi_signal_levels(struct intel_dp *intel_dp);
 u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder);
 int intel_ddi_toggle_hdcp_signalling(struct intel_encoder *intel_encoder,
 				     bool enable);
+void icl_map_plls_to_ports(struct drm_crtc *crtc,
+			   struct intel_crtc_state *crtc_state,
+			   struct drm_atomic_state *old_state);
+void icl_unmap_plls_to_ports(struct drm_crtc *crtc,
+			     struct intel_crtc_state *crtc_state,
+			     struct drm_atomic_state *old_state);
 
 unsigned int intel_fb_align_height(const struct drm_framebuffer *fb,
 				   int plane, unsigned int height);
@@ -1571,8 +1598,6 @@ void bxt_enable_dc9(struct drm_i915_private *dev_priv);
 void bxt_disable_dc9(struct drm_i915_private *dev_priv);
 void gen9_enable_dc5(struct drm_i915_private *dev_priv);
 unsigned int skl_cdclk_get_vco(unsigned int freq);
-void skl_enable_dc6(struct drm_i915_private *dev_priv);
-void skl_disable_dc6(struct drm_i915_private *dev_priv);
 void intel_dp_get_m_n(struct intel_crtc *crtc,
 		      struct intel_crtc_state *pipe_config);
 void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n);
@@ -1588,9 +1613,12 @@ void hsw_disable_ips(const struct intel_crtc_state *crtc_state);
 enum intel_display_power_domain intel_port_to_power_domain(enum port port);
 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
 				 struct intel_crtc_state *pipe_config);
+void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
+				  struct intel_crtc_state *crtc_state);
 
 int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state);
-int skl_max_scale(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state);
+int skl_max_scale(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
+		  uint32_t pixel_format);
 
 static inline u32 intel_plane_ggtt_offset(const struct intel_plane_state *state)
 {
@@ -1607,6 +1635,7 @@ u32 skl_plane_stride(const struct drm_framebuffer *fb, int plane,
 int skl_check_plane_surface(const struct intel_crtc_state *crtc_state,
 			    struct intel_plane_state *plane_state);
 int i9xx_check_plane_surface(struct intel_plane_state *plane_state);
+int skl_format_to_fourcc(int format, bool rgb_order, bool alpha);
 
 /* intel_csr.c */
 void intel_csr_ucode_init(struct drm_i915_private *);
@@ -1773,6 +1802,7 @@ void intel_fbc_flush(struct drm_i915_private *dev_priv,
 		     unsigned int frontbuffer_bits, enum fb_op_origin origin);
 void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv);
 void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *dev_priv);
+int intel_fbc_reset_underrun(struct drm_i915_private *dev_priv);
 
 /* intel_hdmi.c */
 void intel_hdmi_init(struct drm_i915_private *dev_priv, i915_reg_t hdmi_reg,
@@ -1783,7 +1813,7 @@ struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder);
 bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 			       struct intel_crtc_state *pipe_config,
 			       struct drm_connector_state *conn_state);
-void intel_hdmi_handle_sink_scrambling(struct intel_encoder *intel_encoder,
+bool intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder,
 				       struct drm_connector *connector,
 				       bool high_tmds_clock_ratio,
 				       bool scrambling);
@@ -1877,7 +1907,8 @@ void intel_psr_enable(struct intel_dp *intel_dp,
 void intel_psr_disable(struct intel_dp *intel_dp,
 		      const struct intel_crtc_state *old_crtc_state);
 void intel_psr_invalidate(struct drm_i915_private *dev_priv,
-			  unsigned frontbuffer_bits);
+			  unsigned frontbuffer_bits,
+			  enum fb_op_origin origin);
 void intel_psr_flush(struct drm_i915_private *dev_priv,
 		     unsigned frontbuffer_bits,
 		     enum fb_op_origin origin);
@@ -1886,6 +1917,8 @@ void intel_psr_single_frame_update(struct drm_i915_private *dev_priv,
 				   unsigned frontbuffer_bits);
 void intel_psr_compute_config(struct intel_dp *intel_dp,
 			      struct intel_crtc_state *crtc_state);
+void intel_psr_irq_control(struct drm_i915_private *dev_priv, bool debug);
+void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir);
 
 /* intel_runtime_pm.c */
 int intel_power_domains_init(struct drm_i915_private *);
@@ -1909,6 +1942,8 @@ bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
 					enum intel_display_power_domain domain);
 void intel_display_power_put(struct drm_i915_private *dev_priv,
 			     enum intel_display_power_domain domain);
+void icl_dbuf_slices_update(struct drm_i915_private *dev_priv,
+			    u8 req_slices);
 
 static inline void
 assert_rpm_device_not_suspended(struct drm_i915_private *dev_priv)
@@ -2046,6 +2081,9 @@ void skl_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc);
 bool skl_plane_get_hw_state(struct intel_plane *plane);
 bool skl_plane_has_ccs(struct drm_i915_private *dev_priv,
 		       enum pipe pipe, enum plane_id plane_id);
+bool intel_format_is_yuv(uint32_t format);
+bool skl_plane_has_planar(struct drm_i915_private *dev_priv,
+			  enum pipe pipe, enum plane_id plane_id);
 
 /* intel_tv.c */
 void intel_tv_init(struct drm_i915_private *dev_priv);
@@ -2082,31 +2120,6 @@ intel_atomic_get_crtc_state(struct drm_atomic_state *state,
 	return to_intel_crtc_state(crtc_state);
 }
 
-static inline struct intel_crtc_state *
-intel_atomic_get_existing_crtc_state(struct drm_atomic_state *state,
-				     struct intel_crtc *crtc)
-{
-	struct drm_crtc_state *crtc_state;
-
-	crtc_state = drm_atomic_get_existing_crtc_state(state, &crtc->base);
-
-	if (crtc_state)
-		return to_intel_crtc_state(crtc_state);
-	else
-		return NULL;
-}
-
-static inline struct intel_plane_state *
-intel_atomic_get_existing_plane_state(struct drm_atomic_state *state,
-				      struct intel_plane *plane)
-{
-	struct drm_plane_state *plane_state;
-
-	plane_state = drm_atomic_get_existing_plane_state(state, &plane->base);
-
-	return to_intel_plane_state(plane_state);
-}
-
 int intel_atomic_setup_scalers(struct drm_i915_private *dev_priv,
 			       struct intel_crtc *intel_crtc,
 			       struct intel_crtc_state *crtc_state);
@@ -2138,8 +2151,17 @@ int intel_pipe_crc_create(struct drm_minor *minor);
 #ifdef CONFIG_DEBUG_FS
 int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name,
 			      size_t *values_cnt);
+void intel_crtc_disable_pipe_crc(struct intel_crtc *crtc);
+void intel_crtc_enable_pipe_crc(struct intel_crtc *crtc);
 #else
 #define intel_crtc_set_crc_source NULL
+static inline void intel_crtc_disable_pipe_crc(struct intel_crtc *crtc)
+{
+}
+
+static inline void intel_crtc_enable_pipe_crc(struct intel_crtc *crtc)
+{
+}
 #endif
 extern const struct file_operations i915_display_crc_ctl_fops;
 #endif /* __INTEL_DRV_H__ */
diff --git a/drivers/gpu/drm/i915/intel_dsi_vbt.c b/drivers/gpu/drm/i915/intel_dsi_vbt.c
index 91c07b0c8db9..4d6ffa7b3e7b 100644
--- a/drivers/gpu/drm/i915/intel_dsi_vbt.c
+++ b/drivers/gpu/drm/i915/intel_dsi_vbt.c
@@ -647,6 +647,11 @@ bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id)
 	/* prepare count */
 	prepare_cnt = DIV_ROUND_UP(ths_prepare_ns * ui_den, ui_num * mul);
 
+	if (prepare_cnt > PREPARE_CNT_MAX) {
+		DRM_DEBUG_KMS("prepare count too high %u\n", prepare_cnt);
+		prepare_cnt = PREPARE_CNT_MAX;
+	}
+
 	/* exit zero count */
 	exit_zero_cnt = DIV_ROUND_UP(
 				(ths_prepare_hszero - ths_prepare_ns) * ui_den,
@@ -662,32 +667,29 @@ bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id)
 	if (exit_zero_cnt < (55 * ui_den / ui_num) && (55 * ui_den) % ui_num)
 		exit_zero_cnt += 1;
 
+	if (exit_zero_cnt > EXIT_ZERO_CNT_MAX) {
+		DRM_DEBUG_KMS("exit zero count too high %u\n", exit_zero_cnt);
+		exit_zero_cnt = EXIT_ZERO_CNT_MAX;
+	}
+
 	/* clk zero count */
 	clk_zero_cnt = DIV_ROUND_UP(
 				(tclk_prepare_clkzero -	ths_prepare_ns)
 				* ui_den, ui_num * mul);
 
+	if (clk_zero_cnt > CLK_ZERO_CNT_MAX) {
+		DRM_DEBUG_KMS("clock zero count too high %u\n", clk_zero_cnt);
+		clk_zero_cnt = CLK_ZERO_CNT_MAX;
+	}
+
 	/* trail count */
 	tclk_trail_ns = max(mipi_config->tclk_trail, mipi_config->ths_trail);
 	trail_cnt = DIV_ROUND_UP(tclk_trail_ns * ui_den, ui_num * mul);
 
-	if (prepare_cnt > PREPARE_CNT_MAX ||
-		exit_zero_cnt > EXIT_ZERO_CNT_MAX ||
-		clk_zero_cnt > CLK_ZERO_CNT_MAX ||
-		trail_cnt > TRAIL_CNT_MAX)
-		DRM_DEBUG_DRIVER("Values crossing maximum limits, restricting to max values\n");
-
-	if (prepare_cnt > PREPARE_CNT_MAX)
-		prepare_cnt = PREPARE_CNT_MAX;
-
-	if (exit_zero_cnt > EXIT_ZERO_CNT_MAX)
-		exit_zero_cnt = EXIT_ZERO_CNT_MAX;
-
-	if (clk_zero_cnt > CLK_ZERO_CNT_MAX)
-		clk_zero_cnt = CLK_ZERO_CNT_MAX;
-
-	if (trail_cnt > TRAIL_CNT_MAX)
+	if (trail_cnt > TRAIL_CNT_MAX) {
+		DRM_DEBUG_KMS("trail count too high %u\n", trail_cnt);
 		trail_cnt = TRAIL_CNT_MAX;
+	}
 
 	/* B080 */
 	intel_dsi->dphy_reg = exit_zero_cnt << 24 | trail_cnt << 16 |
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index f7c25828d3bb..6bfd7e3ed152 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -81,13 +81,17 @@ static const struct engine_class_info intel_engine_classes[] = {
 	},
 };
 
+#define MAX_MMIO_BASES 3
 struct engine_info {
 	unsigned int hw_id;
 	unsigned int uabi_id;
 	u8 class;
 	u8 instance;
-	u32 mmio_base;
-	unsigned irq_shift;
+	/* mmio bases table *must* be sorted in reverse gen order */
+	struct engine_mmio_base {
+		u32 gen : 8;
+		u32 base : 24;
+	} mmio_bases[MAX_MMIO_BASES];
 };
 
 static const struct engine_info intel_engines[] = {
@@ -96,64 +100,76 @@ static const struct engine_info intel_engines[] = {
 		.uabi_id = I915_EXEC_RENDER,
 		.class = RENDER_CLASS,
 		.instance = 0,
-		.mmio_base = RENDER_RING_BASE,
-		.irq_shift = GEN8_RCS_IRQ_SHIFT,
+		.mmio_bases = {
+			{ .gen = 1, .base = RENDER_RING_BASE }
+		},
 	},
 	[BCS] = {
 		.hw_id = BCS_HW,
 		.uabi_id = I915_EXEC_BLT,
 		.class = COPY_ENGINE_CLASS,
 		.instance = 0,
-		.mmio_base = BLT_RING_BASE,
-		.irq_shift = GEN8_BCS_IRQ_SHIFT,
+		.mmio_bases = {
+			{ .gen = 6, .base = BLT_RING_BASE }
+		},
 	},
 	[VCS] = {
 		.hw_id = VCS_HW,
 		.uabi_id = I915_EXEC_BSD,
 		.class = VIDEO_DECODE_CLASS,
 		.instance = 0,
-		.mmio_base = GEN6_BSD_RING_BASE,
-		.irq_shift = GEN8_VCS1_IRQ_SHIFT,
+		.mmio_bases = {
+			{ .gen = 11, .base = GEN11_BSD_RING_BASE },
+			{ .gen = 6, .base = GEN6_BSD_RING_BASE },
+			{ .gen = 4, .base = BSD_RING_BASE }
+		},
 	},
 	[VCS2] = {
 		.hw_id = VCS2_HW,
 		.uabi_id = I915_EXEC_BSD,
 		.class = VIDEO_DECODE_CLASS,
 		.instance = 1,
-		.mmio_base = GEN8_BSD2_RING_BASE,
-		.irq_shift = GEN8_VCS2_IRQ_SHIFT,
+		.mmio_bases = {
+			{ .gen = 11, .base = GEN11_BSD2_RING_BASE },
+			{ .gen = 8, .base = GEN8_BSD2_RING_BASE }
+		},
 	},
 	[VCS3] = {
 		.hw_id = VCS3_HW,
 		.uabi_id = I915_EXEC_BSD,
 		.class = VIDEO_DECODE_CLASS,
 		.instance = 2,
-		.mmio_base = GEN11_BSD3_RING_BASE,
-		.irq_shift = 0, /* not used */
+		.mmio_bases = {
+			{ .gen = 11, .base = GEN11_BSD3_RING_BASE }
+		},
 	},
 	[VCS4] = {
 		.hw_id = VCS4_HW,
 		.uabi_id = I915_EXEC_BSD,
 		.class = VIDEO_DECODE_CLASS,
 		.instance = 3,
-		.mmio_base = GEN11_BSD4_RING_BASE,
-		.irq_shift = 0, /* not used */
+		.mmio_bases = {
+			{ .gen = 11, .base = GEN11_BSD4_RING_BASE }
+		},
 	},
 	[VECS] = {
 		.hw_id = VECS_HW,
 		.uabi_id = I915_EXEC_VEBOX,
 		.class = VIDEO_ENHANCEMENT_CLASS,
 		.instance = 0,
-		.mmio_base = VEBOX_RING_BASE,
-		.irq_shift = GEN8_VECS_IRQ_SHIFT,
+		.mmio_bases = {
+			{ .gen = 11, .base = GEN11_VEBOX_RING_BASE },
+			{ .gen = 7, .base = VEBOX_RING_BASE }
+		},
 	},
 	[VECS2] = {
 		.hw_id = VECS2_HW,
 		.uabi_id = I915_EXEC_VEBOX,
 		.class = VIDEO_ENHANCEMENT_CLASS,
 		.instance = 1,
-		.mmio_base = GEN11_VEBOX2_RING_BASE,
-		.irq_shift = 0, /* not used */
+		.mmio_bases = {
+			{ .gen = 11, .base = GEN11_VEBOX2_RING_BASE }
+		},
 	},
 };
 
@@ -223,16 +239,36 @@ __intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
 	}
 }
 
+static u32 __engine_mmio_base(struct drm_i915_private *i915,
+			      const struct engine_mmio_base *bases)
+{
+	int i;
+
+	for (i = 0; i < MAX_MMIO_BASES; i++)
+		if (INTEL_GEN(i915) >= bases[i].gen)
+			break;
+
+	GEM_BUG_ON(i == MAX_MMIO_BASES);
+	GEM_BUG_ON(!bases[i].base);
+
+	return bases[i].base;
+}
+
+static void __sprint_engine_name(char *name, const struct engine_info *info)
+{
+	WARN_ON(snprintf(name, INTEL_ENGINE_CS_MAX_NAME, "%s%u",
+			 intel_engine_classes[info->class].name,
+			 info->instance) >= INTEL_ENGINE_CS_MAX_NAME);
+}
+
 static int
 intel_engine_setup(struct drm_i915_private *dev_priv,
 		   enum intel_engine_id id)
 {
 	const struct engine_info *info = &intel_engines[id];
-	const struct engine_class_info *class_info;
 	struct intel_engine_cs *engine;
 
 	GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
-	class_info = &intel_engine_classes[info->class];
 
 	BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH));
 	BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH));
@@ -253,35 +289,14 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
 
 	engine->id = id;
 	engine->i915 = dev_priv;
-	WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s%u",
-			 class_info->name, info->instance) >=
-		sizeof(engine->name));
+	__sprint_engine_name(engine->name, info);
 	engine->hw_id = engine->guc_id = info->hw_id;
-	if (INTEL_GEN(dev_priv) >= 11) {
-		switch (engine->id) {
-		case VCS:
-			engine->mmio_base = GEN11_BSD_RING_BASE;
-			break;
-		case VCS2:
-			engine->mmio_base = GEN11_BSD2_RING_BASE;
-			break;
-		case VECS:
-			engine->mmio_base = GEN11_VEBOX_RING_BASE;
-			break;
-		default:
-			/* take the original value for all other engines  */
-			engine->mmio_base = info->mmio_base;
-			break;
-		}
-	} else {
-		engine->mmio_base = info->mmio_base;
-	}
-	engine->irq_shift = info->irq_shift;
+	engine->mmio_base = __engine_mmio_base(dev_priv, info->mmio_bases);
 	engine->class = info->class;
 	engine->instance = info->instance;
 
 	engine->uabi_id = info->uabi_id;
-	engine->uabi_class = class_info->uabi_class;
+	engine->uabi_class = intel_engine_classes[info->class].uabi_class;
 
 	engine->context_size = __intel_engine_context_size(dev_priv,
 							   engine->class);
@@ -291,7 +306,7 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
 	/* Nothing to do here, execute in order of dependencies */
 	engine->schedule = NULL;
 
-	spin_lock_init(&engine->stats.lock);
+	seqlock_init(&engine->stats.lock);
 
 	ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
 
@@ -436,21 +451,13 @@ void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
 	GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno);
 }
 
-static void intel_engine_init_timeline(struct intel_engine_cs *engine)
+static void intel_engine_init_batch_pool(struct intel_engine_cs *engine)
 {
-	engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id];
+	i915_gem_batch_pool_init(&engine->batch_pool, engine);
 }
 
 static bool csb_force_mmio(struct drm_i915_private *i915)
 {
-	/*
-	 * IOMMU adds unpredictable latency causing the CSB write (from the
-	 * GPU into the HWSP) to only be visible some time after the interrupt
-	 * (missed breadcrumb syndrome).
-	 */
-	if (intel_vtd_active())
-		return true;
-
 	/* Older GVT emulation depends upon intercepting CSB mmio */
 	if (intel_vgpu_active(i915) && !intel_vgpu_has_hwsp_emulation(i915))
 		return true;
@@ -484,12 +491,11 @@ static void intel_engine_init_execlist(struct intel_engine_cs *engine)
  */
 void intel_engine_setup_common(struct intel_engine_cs *engine)
 {
-	intel_engine_init_execlist(engine);
+	i915_timeline_init(engine->i915, &engine->timeline, engine->name);
 
-	intel_engine_init_timeline(engine);
+	intel_engine_init_execlist(engine);
 	intel_engine_init_hangcheck(engine);
-	i915_gem_batch_pool_init(engine, &engine->batch_pool);
-
+	intel_engine_init_batch_pool(engine);
 	intel_engine_init_cmd_parser(engine);
 }
 
@@ -520,8 +526,6 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
 		goto err_unref;
 
 	engine->scratch = vma;
-	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
-			 engine->name, i915_ggtt_offset(vma));
 	return 0;
 
 err_unref:
@@ -615,9 +619,6 @@ static int init_status_page(struct intel_engine_cs *engine)
 	engine->status_page.vma = vma;
 	engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
 	engine->status_page.page_addr = memset(vaddr, 0, PAGE_SIZE);
-
-	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
-			 engine->name, i915_ggtt_offset(vma));
 	return 0;
 
 err_unpin:
@@ -669,7 +670,7 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
 	 * be available. To avoid this we always pin the default
 	 * context.
 	 */
-	ring = engine->context_pin(engine, engine->i915->kernel_context);
+	ring = intel_context_pin(engine->i915->kernel_context, engine);
 	if (IS_ERR(ring))
 		return PTR_ERR(ring);
 
@@ -678,8 +679,7 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
 	 * we can interrupt the engine at any time.
 	 */
 	if (engine->i915->preempt_context) {
-		ring = engine->context_pin(engine,
-					   engine->i915->preempt_context);
+		ring = intel_context_pin(engine->i915->preempt_context, engine);
 		if (IS_ERR(ring)) {
 			ret = PTR_ERR(ring);
 			goto err_unpin_kernel;
@@ -703,9 +703,9 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
 	intel_engine_fini_breadcrumbs(engine);
 err_unpin_preempt:
 	if (engine->i915->preempt_context)
-		engine->context_unpin(engine, engine->i915->preempt_context);
+		intel_context_unpin(engine->i915->preempt_context, engine);
 err_unpin_kernel:
-	engine->context_unpin(engine, engine->i915->kernel_context);
+	intel_context_unpin(engine->i915->kernel_context, engine);
 	return ret;
 }
 
@@ -733,8 +733,10 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 		i915_gem_object_put(engine->default_state);
 
 	if (engine->i915->preempt_context)
-		engine->context_unpin(engine, engine->i915->preempt_context);
-	engine->context_unpin(engine, engine->i915->kernel_context);
+		intel_context_unpin(engine->i915->preempt_context, engine);
+	intel_context_unpin(engine->i915->kernel_context, engine);
+
+	i915_timeline_fini(&engine->timeline);
 }
 
 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
@@ -782,10 +784,24 @@ static inline uint32_t
 read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
 		  int subslice, i915_reg_t reg)
 {
+	uint32_t mcr_slice_subslice_mask;
+	uint32_t mcr_slice_subslice_select;
 	uint32_t mcr;
 	uint32_t ret;
 	enum forcewake_domains fw_domains;
 
+	if (INTEL_GEN(dev_priv) >= 11) {
+		mcr_slice_subslice_mask = GEN11_MCR_SLICE_MASK |
+					  GEN11_MCR_SUBSLICE_MASK;
+		mcr_slice_subslice_select = GEN11_MCR_SLICE(slice) |
+					    GEN11_MCR_SUBSLICE(subslice);
+	} else {
+		mcr_slice_subslice_mask = GEN8_MCR_SLICE_MASK |
+					  GEN8_MCR_SUBSLICE_MASK;
+		mcr_slice_subslice_select = GEN8_MCR_SLICE(slice) |
+					    GEN8_MCR_SUBSLICE(subslice);
+	}
+
 	fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
 						    FW_REG_READ);
 	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
@@ -800,14 +816,14 @@ read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
 	 * The HW expects the slice and sublice selectors to be reset to 0
 	 * after reading out the registers.
 	 */
-	WARN_ON_ONCE(mcr & (GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK));
-	mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
-	mcr |= GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
+	WARN_ON_ONCE(mcr & mcr_slice_subslice_mask);
+	mcr &= ~mcr_slice_subslice_mask;
+	mcr |= mcr_slice_subslice_select;
 	I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
 
 	ret = I915_READ_FW(reg);
 
-	mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
+	mcr &= ~mcr_slice_subslice_mask;
 	I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
 
 	intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
@@ -871,644 +887,6 @@ void intel_engine_get_instdone(struct intel_engine_cs *engine,
 	}
 }
 
-static int wa_add(struct drm_i915_private *dev_priv,
-		  i915_reg_t addr,
-		  const u32 mask, const u32 val)
-{
-	const u32 idx = dev_priv->workarounds.count;
-
-	if (WARN_ON(idx >= I915_MAX_WA_REGS))
-		return -ENOSPC;
-
-	dev_priv->workarounds.reg[idx].addr = addr;
-	dev_priv->workarounds.reg[idx].value = val;
-	dev_priv->workarounds.reg[idx].mask = mask;
-
-	dev_priv->workarounds.count++;
-
-	return 0;
-}
-
-#define WA_REG(addr, mask, val) do { \
-		const int r = wa_add(dev_priv, (addr), (mask), (val)); \
-		if (r) \
-			return r; \
-	} while (0)
-
-#define WA_SET_BIT_MASKED(addr, mask) \
-	WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
-
-#define WA_CLR_BIT_MASKED(addr, mask) \
-	WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
-
-#define WA_SET_FIELD_MASKED(addr, mask, value) \
-	WA_REG(addr, mask, _MASKED_FIELD(mask, value))
-
-static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
-				 i915_reg_t reg)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	struct i915_workarounds *wa = &dev_priv->workarounds;
-	const uint32_t index = wa->hw_whitelist_count[engine->id];
-
-	if (WARN_ON(index >= RING_MAX_NONPRIV_SLOTS))
-		return -EINVAL;
-
-	I915_WRITE(RING_FORCE_TO_NONPRIV(engine->mmio_base, index),
-		   i915_mmio_reg_offset(reg));
-	wa->hw_whitelist_count[engine->id]++;
-
-	return 0;
-}
-
-static int gen8_init_workarounds(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
-
-	/* WaDisableAsyncFlipPerfMode:bdw,chv */
-	WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
-
-	/* WaDisablePartialInstShootdown:bdw,chv */
-	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
-			  PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
-
-	/* Use Force Non-Coherent whenever executing a 3D context. This is a
-	 * workaround for for a possible hang in the unlikely event a TLB
-	 * invalidation occurs during a PSD flush.
-	 */
-	/* WaForceEnableNonCoherent:bdw,chv */
-	/* WaHdcDisableFetchWhenMasked:bdw,chv */
-	WA_SET_BIT_MASKED(HDC_CHICKEN0,
-			  HDC_DONOT_FETCH_MEM_WHEN_MASKED |
-			  HDC_FORCE_NON_COHERENT);
-
-	/* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
-	 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
-	 *  polygons in the same 8x4 pixel/sample area to be processed without
-	 *  stalling waiting for the earlier ones to write to Hierarchical Z
-	 *  buffer."
-	 *
-	 * This optimization is off by default for BDW and CHV; turn it on.
-	 */
-	WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
-
-	/* Wa4x4STCOptimizationDisable:bdw,chv */
-	WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
-
-	/*
-	 * BSpec recommends 8x4 when MSAA is used,
-	 * however in practice 16x4 seems fastest.
-	 *
-	 * Note that PS/WM thread counts depend on the WIZ hashing
-	 * disable bit, which we don't touch here, but it's good
-	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
-	 */
-	WA_SET_FIELD_MASKED(GEN7_GT_MODE,
-			    GEN6_WIZ_HASHING_MASK,
-			    GEN6_WIZ_HASHING_16x4);
-
-	return 0;
-}
-
-static int bdw_init_workarounds(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int ret;
-
-	ret = gen8_init_workarounds(engine);
-	if (ret)
-		return ret;
-
-	/* WaDisableThreadStallDopClockGating:bdw (pre-production) */
-	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
-
-	/* WaDisableDopClockGating:bdw
-	 *
-	 * Also see the related UCGTCL1 write in broadwell_init_clock_gating()
-	 * to disable EUTC clock gating.
-	 */
-	WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
-			  DOP_CLOCK_GATING_DISABLE);
-
-	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
-			  GEN8_SAMPLER_POWER_BYPASS_DIS);
-
-	WA_SET_BIT_MASKED(HDC_CHICKEN0,
-			  /* WaForceContextSaveRestoreNonCoherent:bdw */
-			  HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
-			  /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
-			  (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
-
-	return 0;
-}
-
-static int chv_init_workarounds(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int ret;
-
-	ret = gen8_init_workarounds(engine);
-	if (ret)
-		return ret;
-
-	/* WaDisableThreadStallDopClockGating:chv */
-	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
-
-	/* Improve HiZ throughput on CHV. */
-	WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
-
-	return 0;
-}
-
-static int gen9_init_workarounds(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int ret;
-
-	/* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
-	I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
-
-	/* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
-	I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
-		   GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
-
-	/* WaDisableKillLogic:bxt,skl,kbl */
-	if (!IS_COFFEELAKE(dev_priv))
-		I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
-			   ECOCHK_DIS_TLB);
-
-	if (HAS_LLC(dev_priv)) {
-		/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
-		 *
-		 * Must match Display Engine. See
-		 * WaCompressedResourceDisplayNewHashMode.
-		 */
-		WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
-				  GEN9_PBE_COMPRESSED_HASH_SELECTION);
-		WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
-				  GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
-
-		I915_WRITE(MMCD_MISC_CTRL,
-			   I915_READ(MMCD_MISC_CTRL) |
-			   MMCD_PCLA |
-			   MMCD_HOTSPOT_EN);
-	}
-
-	/* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
-	/* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
-	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
-			  FLOW_CONTROL_ENABLE |
-			  PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
-
-	/* Syncing dependencies between camera and graphics:skl,bxt,kbl */
-	if (!IS_COFFEELAKE(dev_priv))
-		WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
-				  GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
-
-	/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
-	/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
-	WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
-			  GEN9_ENABLE_YV12_BUGFIX |
-			  GEN9_ENABLE_GPGPU_PREEMPTION);
-
-	/* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
-	/* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
-	WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
-					 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
-
-	/* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
-	WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
-			  GEN9_CCS_TLB_PREFETCH_ENABLE);
-
-	/* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
-	WA_SET_BIT_MASKED(HDC_CHICKEN0,
-			  HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
-			  HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
-
-	/* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
-	 * both tied to WaForceContextSaveRestoreNonCoherent
-	 * in some hsds for skl. We keep the tie for all gen9. The
-	 * documentation is a bit hazy and so we want to get common behaviour,
-	 * even though there is no clear evidence we would need both on kbl/bxt.
-	 * This area has been source of system hangs so we play it safe
-	 * and mimic the skl regardless of what bspec says.
-	 *
-	 * Use Force Non-Coherent whenever executing a 3D context. This
-	 * is a workaround for a possible hang in the unlikely event
-	 * a TLB invalidation occurs during a PSD flush.
-	 */
-
-	/* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
-	WA_SET_BIT_MASKED(HDC_CHICKEN0,
-			  HDC_FORCE_NON_COHERENT);
-
-	/* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
-	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
-		   BDW_DISABLE_HDC_INVALIDATION);
-
-	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
-	if (IS_SKYLAKE(dev_priv) ||
-	    IS_KABYLAKE(dev_priv) ||
-	    IS_COFFEELAKE(dev_priv))
-		WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
-				  GEN8_SAMPLER_POWER_BYPASS_DIS);
-
-	/* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
-	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
-
-	/* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
-	if (IS_GEN9_LP(dev_priv)) {
-		u32 val = I915_READ(GEN8_L3SQCREG1);
-
-		val &= ~L3_PRIO_CREDITS_MASK;
-		val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
-		I915_WRITE(GEN8_L3SQCREG1, val);
-	}
-
-	/* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
-	I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
-				    GEN8_LQSC_FLUSH_COHERENT_LINES));
-
-	/*
-	 * Supporting preemption with fine-granularity requires changes in the
-	 * batch buffer programming. Since we can't break old userspace, we
-	 * need to set our default preemption level to safe value. Userspace is
-	 * still able to use more fine-grained preemption levels, since in
-	 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
-	 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
-	 * not real HW workarounds, but merely a way to start using preemption
-	 * while maintaining old contract with userspace.
-	 */
-
-	/* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
-	WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
-
-	/* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
-	WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK,
-			    GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
-
-	/* WaClearHIZ_WM_CHICKEN3:bxt,glk */
-	if (IS_GEN9_LP(dev_priv))
-		WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
-
-	/* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
-	ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG);
-	if (ret)
-		return ret;
-
-	/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
-	I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
-		   _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
-	ret = wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
-	if (ret)
-		return ret;
-
-	/* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
-	ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	u8 vals[3] = { 0, 0, 0 };
-	unsigned int i;
-
-	for (i = 0; i < 3; i++) {
-		u8 ss;
-
-		/*
-		 * Only consider slices where one, and only one, subslice has 7
-		 * EUs
-		 */
-		if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]))
-			continue;
-
-		/*
-		 * subslice_7eu[i] != 0 (because of the check above) and
-		 * ss_max == 4 (maximum number of subslices possible per slice)
-		 *
-		 * ->    0 <= ss <= 3;
-		 */
-		ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1;
-		vals[i] = 3 - ss;
-	}
-
-	if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
-		return 0;
-
-	/* Tune IZ hashing. See intel_device_info_runtime_init() */
-	WA_SET_FIELD_MASKED(GEN7_GT_MODE,
-			    GEN9_IZ_HASHING_MASK(2) |
-			    GEN9_IZ_HASHING_MASK(1) |
-			    GEN9_IZ_HASHING_MASK(0),
-			    GEN9_IZ_HASHING(2, vals[2]) |
-			    GEN9_IZ_HASHING(1, vals[1]) |
-			    GEN9_IZ_HASHING(0, vals[0]));
-
-	return 0;
-}
-
-static int skl_init_workarounds(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int ret;
-
-	ret = gen9_init_workarounds(engine);
-	if (ret)
-		return ret;
-
-	/* WaEnableGapsTsvCreditFix:skl */
-	I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
-				   GEN9_GAPS_TSV_CREDIT_DISABLE));
-
-	/* WaDisableGafsUnitClkGating:skl */
-	I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
-				  GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
-
-	/* WaInPlaceDecompressionHang:skl */
-	if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
-		I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
-			   (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
-			    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
-
-	/* WaDisableLSQCROPERFforOCL:skl */
-	ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
-	if (ret)
-		return ret;
-
-	return skl_tune_iz_hashing(engine);
-}
-
-static int bxt_init_workarounds(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int ret;
-
-	ret = gen9_init_workarounds(engine);
-	if (ret)
-		return ret;
-
-	/* WaDisableThreadStallDopClockGating:bxt */
-	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
-			  STALL_DOP_GATING_DISABLE);
-
-	/* WaDisablePooledEuLoadBalancingFix:bxt */
-	I915_WRITE(FF_SLICE_CS_CHICKEN2,
-		   _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
-
-	/* WaToEnableHwFixForPushConstHWBug:bxt */
-	WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
-			  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
-
-	/* WaInPlaceDecompressionHang:bxt */
-	I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
-		   (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
-		    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
-
-	return 0;
-}
-
-static int cnl_init_workarounds(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int ret;
-
-	/* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
-	if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
-		I915_WRITE(GAMT_CHKN_BIT_REG,
-			   (I915_READ(GAMT_CHKN_BIT_REG) |
-			    GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT));
-
-	/* WaForceContextSaveRestoreNonCoherent:cnl */
-	WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
-			  HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
-
-	/* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
-	if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
-		WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
-
-	/* WaDisableReplayBufferBankArbitrationOptimization:cnl */
-	WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
-			  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
-
-	/* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
-	if (IS_CNL_REVID(dev_priv, 0, CNL_REVID_B0))
-		WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
-				  GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
-
-	/* WaInPlaceDecompressionHang:cnl */
-	I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
-		   (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
-		    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
-
-	/* WaPushConstantDereferenceHoldDisable:cnl */
-	WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
-
-	/* FtrEnableFastAnisoL1BankingFix: cnl */
-	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
-
-	/* WaDisable3DMidCmdPreemption:cnl */
-	WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
-
-	/* WaDisableGPGPUMidCmdPreemption:cnl */
-	WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK,
-			    GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
-
-	/* WaEnablePreemptionGranularityControlByUMD:cnl */
-	I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
-		   _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
-	ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
-	if (ret)
-		return ret;
-
-	/* WaDisableEarlyEOT:cnl */
-	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
-
-	return 0;
-}
-
-static int kbl_init_workarounds(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int ret;
-
-	ret = gen9_init_workarounds(engine);
-	if (ret)
-		return ret;
-
-	/* WaEnableGapsTsvCreditFix:kbl */
-	I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
-				   GEN9_GAPS_TSV_CREDIT_DISABLE));
-
-	/* WaDisableDynamicCreditSharing:kbl */
-	if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
-		I915_WRITE(GAMT_CHKN_BIT_REG,
-			   (I915_READ(GAMT_CHKN_BIT_REG) |
-			    GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING));
-
-	/* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
-	if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
-		WA_SET_BIT_MASKED(HDC_CHICKEN0,
-				  HDC_FENCE_DEST_SLM_DISABLE);
-
-	/* WaToEnableHwFixForPushConstHWBug:kbl */
-	if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER))
-		WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
-				  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
-
-	/* WaDisableGafsUnitClkGating:kbl */
-	I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
-				  GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
-
-	/* WaDisableSbeCacheDispatchPortSharing:kbl */
-	WA_SET_BIT_MASKED(
-		GEN7_HALF_SLICE_CHICKEN1,
-		GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
-
-	/* WaInPlaceDecompressionHang:kbl */
-	I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
-		   (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
-		    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
-
-	/* WaDisableLSQCROPERFforOCL:kbl */
-	ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int glk_init_workarounds(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int ret;
-
-	ret = gen9_init_workarounds(engine);
-	if (ret)
-		return ret;
-
-	/* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
-	ret = wa_ring_whitelist_reg(engine, GEN9_SLICE_COMMON_ECO_CHICKEN1);
-	if (ret)
-		return ret;
-
-	/* WaToEnableHwFixForPushConstHWBug:glk */
-	WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
-			  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
-
-	return 0;
-}
-
-static int cfl_init_workarounds(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int ret;
-
-	ret = gen9_init_workarounds(engine);
-	if (ret)
-		return ret;
-
-	/* WaEnableGapsTsvCreditFix:cfl */
-	I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
-				   GEN9_GAPS_TSV_CREDIT_DISABLE));
-
-	/* WaToEnableHwFixForPushConstHWBug:cfl */
-	WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
-			  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
-
-	/* WaDisableGafsUnitClkGating:cfl */
-	I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
-				  GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
-
-	/* WaDisableSbeCacheDispatchPortSharing:cfl */
-	WA_SET_BIT_MASKED(
-		GEN7_HALF_SLICE_CHICKEN1,
-		GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
-
-	/* WaInPlaceDecompressionHang:cfl */
-	I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
-		   (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
-		    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
-
-	return 0;
-}
-
-int init_workarounds_ring(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int err;
-
-	if (GEM_WARN_ON(engine->id != RCS))
-		return -EINVAL;
-
-	dev_priv->workarounds.count = 0;
-	dev_priv->workarounds.hw_whitelist_count[engine->id] = 0;
-
-	if (IS_BROADWELL(dev_priv))
-		err = bdw_init_workarounds(engine);
-	else if (IS_CHERRYVIEW(dev_priv))
-		err = chv_init_workarounds(engine);
-	else if (IS_SKYLAKE(dev_priv))
-		err =  skl_init_workarounds(engine);
-	else if (IS_BROXTON(dev_priv))
-		err = bxt_init_workarounds(engine);
-	else if (IS_KABYLAKE(dev_priv))
-		err = kbl_init_workarounds(engine);
-	else if (IS_GEMINILAKE(dev_priv))
-		err =  glk_init_workarounds(engine);
-	else if (IS_COFFEELAKE(dev_priv))
-		err = cfl_init_workarounds(engine);
-	else if (IS_CANNONLAKE(dev_priv))
-		err = cnl_init_workarounds(engine);
-	else
-		err = 0;
-	if (err)
-		return err;
-
-	DRM_DEBUG_DRIVER("%s: Number of context specific w/a: %d\n",
-			 engine->name, dev_priv->workarounds.count);
-	return 0;
-}
-
-int intel_ring_workarounds_emit(struct i915_request *rq)
-{
-	struct i915_workarounds *w = &rq->i915->workarounds;
-	u32 *cs;
-	int ret, i;
-
-	if (w->count == 0)
-		return 0;
-
-	ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
-	if (ret)
-		return ret;
-
-	cs = intel_ring_begin(rq, w->count * 2 + 2);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	*cs++ = MI_LOAD_REGISTER_IMM(w->count);
-	for (i = 0; i < w->count; i++) {
-		*cs++ = i915_mmio_reg_offset(w->reg[i].addr);
-		*cs++ = w->reg[i].value;
-	}
-	*cs++ = MI_NOOP;
-
-	intel_ring_advance(rq, cs);
-
-	ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
 static bool ring_is_idle(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
@@ -1611,7 +989,7 @@ bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine)
 	 * the last request that remains in the timeline. When idle, it is
 	 * the last executed context as tracked by retirement.
 	 */
-	rq = __i915_gem_active_peek(&engine->timeline->last_request);
+	rq = __i915_gem_active_peek(&engine->timeline.last_request);
 	if (rq)
 		return rq->ctx == kernel_context;
 	else
@@ -1659,6 +1037,9 @@ void intel_engines_park(struct drm_i915_private *i915)
 			intel_engine_dump(engine, &p, NULL);
 		}
 
+		/* Must be reset upon idling, or we may miss the busy wakeup. */
+		GEM_BUG_ON(engine->execlists.queue_priority != INT_MIN);
+
 		if (engine->park)
 			engine->park(engine);
 
@@ -1681,6 +1062,8 @@ void intel_engines_unpark(struct drm_i915_private *i915)
 	for_each_engine(engine, i915, id) {
 		if (engine->unpark)
 			engine->unpark(engine);
+
+		intel_engine_init_hangcheck(engine);
 	}
 }
 
@@ -1713,17 +1096,37 @@ unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915)
 	return which;
 }
 
+static int print_sched_attr(struct drm_i915_private *i915,
+			    const struct i915_sched_attr *attr,
+			    char *buf, int x, int len)
+{
+	if (attr->priority == I915_PRIORITY_INVALID)
+		return x;
+
+	x += snprintf(buf + x, len - x,
+		      " prio=%d", attr->priority);
+
+	return x;
+}
+
 static void print_request(struct drm_printer *m,
 			  struct i915_request *rq,
 			  const char *prefix)
 {
-	drm_printf(m, "%s%x%s [%llx:%x] prio=%d @ %dms: %s\n", prefix,
+	const char *name = rq->fence.ops->get_timeline_name(&rq->fence);
+	char buf[80];
+	int x = 0;
+
+	x = print_sched_attr(rq->i915, &rq->sched.attr, buf, x, sizeof(buf));
+
+	drm_printf(m, "%s%x%s [%llx:%x]%s @ %dms: %s\n",
+		   prefix,
 		   rq->global_seqno,
 		   i915_request_completed(rq) ? "!" : "",
 		   rq->fence.context, rq->fence.seqno,
-		   rq->priotree.priority,
+		   buf,
 		   jiffies_to_msecs(jiffies - rq->emitted_jiffies),
-		   rq->timeline->common->name);
+		   name);
 }
 
 static void hexdump(struct drm_printer *m, const void *buf, size_t len)
@@ -1829,12 +1232,15 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
 		ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
 		read = GEN8_CSB_READ_PTR(ptr);
 		write = GEN8_CSB_WRITE_PTR(ptr);
-		drm_printf(m, "\tExeclist CSB read %d [%d cached], write %d [%d from hws], interrupt posted? %s\n",
+		drm_printf(m, "\tExeclist CSB read %d [%d cached], write %d [%d from hws], interrupt posted? %s, tasklet queued? %s (%s)\n",
 			   read, execlists->csb_head,
 			   write,
 			   intel_read_status_page(engine, intel_hws_csb_write_index(engine->i915)),
 			   yesno(test_bit(ENGINE_IRQ_EXECLIST,
-					  &engine->irq_posted)));
+					  &engine->irq_posted)),
+			   yesno(test_bit(TASKLET_STATE_SCHED,
+					  &engine->execlists.tasklet.state)),
+			   enableddisabled(!atomic_read(&engine->execlists.tasklet.count)));
 		if (read >= GEN8_CSB_ENTRIES)
 			read = 0;
 		if (write >= GEN8_CSB_ENTRIES)
@@ -1861,8 +1267,9 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
 				char hdr[80];
 
 				snprintf(hdr, sizeof(hdr),
-					 "\t\tELSP[%d] count=%d, rq: ",
-					 idx, count);
+					 "\t\tELSP[%d] count=%d, ring->start=%08x, rq: ",
+					 idx, count,
+					 i915_ggtt_offset(rq->ring->vma));
 				print_request(m, rq, hdr);
 			} else {
 				drm_printf(m, "\t\tELSP[%d] idle\n", idx);
@@ -1884,11 +1291,13 @@ void intel_engine_dump(struct intel_engine_cs *engine,
 		       struct drm_printer *m,
 		       const char *header, ...)
 {
+	const int MAX_REQUESTS_TO_SHOW = 8;
 	struct intel_breadcrumbs * const b = &engine->breadcrumbs;
 	const struct intel_engine_execlists * const execlists = &engine->execlists;
 	struct i915_gpu_error * const error = &engine->i915->gpu_error;
-	struct i915_request *rq;
+	struct i915_request *rq, *last;
 	struct rb_node *rb;
+	int count;
 
 	if (header) {
 		va_list ap;
@@ -1901,12 +1310,11 @@ void intel_engine_dump(struct intel_engine_cs *engine,
 	if (i915_terminally_wedged(&engine->i915->gpu_error))
 		drm_printf(m, "*** WEDGED ***\n");
 
-	drm_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [%d ms], inflight %d\n",
+	drm_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [%d ms]\n",
 		   intel_engine_get_seqno(engine),
 		   intel_engine_last_submit(engine),
 		   engine->hangcheck.seqno,
-		   jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp),
-		   engine->timeline->inflight_seqnos);
+		   jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp));
 	drm_printf(m, "\tReset count: %d (global %d)\n",
 		   i915_reset_engine_count(error, engine),
 		   i915_reset_count(error));
@@ -1915,14 +1323,14 @@ void intel_engine_dump(struct intel_engine_cs *engine,
 
 	drm_printf(m, "\tRequests:\n");
 
-	rq = list_first_entry(&engine->timeline->requests,
+	rq = list_first_entry(&engine->timeline.requests,
 			      struct i915_request, link);
-	if (&rq->link != &engine->timeline->requests)
+	if (&rq->link != &engine->timeline.requests)
 		print_request(m, rq, "\t\tfirst  ");
 
-	rq = list_last_entry(&engine->timeline->requests,
+	rq = list_last_entry(&engine->timeline.requests,
 			     struct i915_request, link);
-	if (&rq->link != &engine->timeline->requests)
+	if (&rq->link != &engine->timeline.requests)
 		print_request(m, rq, "\t\tlast   ");
 
 	rq = i915_gem_find_active_request(engine);
@@ -1933,12 +1341,16 @@ void intel_engine_dump(struct intel_engine_cs *engine,
 			   rq->head, rq->postfix, rq->tail,
 			   rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
 			   rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);
-		drm_printf(m, "\t\tring->start: 0x%08x\n",
+		drm_printf(m, "\t\tring->start:  0x%08x\n",
 			   i915_ggtt_offset(rq->ring->vma));
-		drm_printf(m, "\t\tring->head:  0x%08x\n",
+		drm_printf(m, "\t\tring->head:   0x%08x\n",
 			   rq->ring->head);
-		drm_printf(m, "\t\tring->tail:  0x%08x\n",
+		drm_printf(m, "\t\tring->tail:   0x%08x\n",
 			   rq->ring->tail);
+		drm_printf(m, "\t\tring->emit:   0x%08x\n",
+			   rq->ring->emit);
+		drm_printf(m, "\t\tring->space:  0x%08x\n",
+			   rq->ring->space);
 	}
 
 	rcu_read_unlock();
@@ -1950,18 +1362,49 @@ void intel_engine_dump(struct intel_engine_cs *engine,
 		drm_printf(m, "\tDevice is asleep; skipping register dump\n");
 	}
 
-	spin_lock_irq(&engine->timeline->lock);
-	list_for_each_entry(rq, &engine->timeline->requests, link)
-		print_request(m, rq, "\t\tE ");
+	spin_lock_irq(&engine->timeline.lock);
+
+	last = NULL;
+	count = 0;
+	list_for_each_entry(rq, &engine->timeline.requests, link) {
+		if (count++ < MAX_REQUESTS_TO_SHOW - 1)
+			print_request(m, rq, "\t\tE ");
+		else
+			last = rq;
+	}
+	if (last) {
+		if (count > MAX_REQUESTS_TO_SHOW) {
+			drm_printf(m,
+				   "\t\t...skipping %d executing requests...\n",
+				   count - MAX_REQUESTS_TO_SHOW);
+		}
+		print_request(m, last, "\t\tE ");
+	}
+
+	last = NULL;
+	count = 0;
 	drm_printf(m, "\t\tQueue priority: %d\n", execlists->queue_priority);
 	for (rb = execlists->first; rb; rb = rb_next(rb)) {
 		struct i915_priolist *p =
 			rb_entry(rb, typeof(*p), node);
 
-		list_for_each_entry(rq, &p->requests, priotree.link)
-			print_request(m, rq, "\t\tQ ");
+		list_for_each_entry(rq, &p->requests, sched.link) {
+			if (count++ < MAX_REQUESTS_TO_SHOW - 1)
+				print_request(m, rq, "\t\tQ ");
+			else
+				last = rq;
+		}
+	}
+	if (last) {
+		if (count > MAX_REQUESTS_TO_SHOW) {
+			drm_printf(m,
+				   "\t\t...skipping %d queued requests...\n",
+				   count - MAX_REQUESTS_TO_SHOW);
+		}
+		print_request(m, last, "\t\tQ ");
 	}
-	spin_unlock_irq(&engine->timeline->lock);
+
+	spin_unlock_irq(&engine->timeline.lock);
 
 	spin_lock_irq(&b->rb_lock);
 	for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
@@ -2026,7 +1469,7 @@ int intel_enable_engine_stats(struct intel_engine_cs *engine)
 		return -ENODEV;
 
 	tasklet_disable(&execlists->tasklet);
-	spin_lock_irqsave(&engine->stats.lock, flags);
+	write_seqlock_irqsave(&engine->stats.lock, flags);
 
 	if (unlikely(engine->stats.enabled == ~0)) {
 		err = -EBUSY;
@@ -2050,7 +1493,7 @@ int intel_enable_engine_stats(struct intel_engine_cs *engine)
 	}
 
 unlock:
-	spin_unlock_irqrestore(&engine->stats.lock, flags);
+	write_sequnlock_irqrestore(&engine->stats.lock, flags);
 	tasklet_enable(&execlists->tasklet);
 
 	return err;
@@ -2079,12 +1522,13 @@ static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine)
  */
 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine)
 {
+	unsigned int seq;
 	ktime_t total;
-	unsigned long flags;
 
-	spin_lock_irqsave(&engine->stats.lock, flags);
-	total = __intel_engine_get_busy_time(engine);
-	spin_unlock_irqrestore(&engine->stats.lock, flags);
+	do {
+		seq = read_seqbegin(&engine->stats.lock);
+		total = __intel_engine_get_busy_time(engine);
+	} while (read_seqretry(&engine->stats.lock, seq));
 
 	return total;
 }
@@ -2102,15 +1546,16 @@ void intel_disable_engine_stats(struct intel_engine_cs *engine)
 	if (!intel_engine_supports_stats(engine))
 		return;
 
-	spin_lock_irqsave(&engine->stats.lock, flags);
+	write_seqlock_irqsave(&engine->stats.lock, flags);
 	WARN_ON_ONCE(engine->stats.enabled == 0);
 	if (--engine->stats.enabled == 0) {
 		engine->stats.total = __intel_engine_get_busy_time(engine);
 		engine->stats.active = 0;
 	}
-	spin_unlock_irqrestore(&engine->stats.lock, flags);
+	write_sequnlock_irqrestore(&engine->stats.lock, flags);
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftests/mock_engine.c"
+#include "selftests/intel_engine_cs.c"
 #endif
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index 707d49c12638..b431b6733cc1 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -1272,6 +1272,34 @@ static void intel_fbc_underrun_work_fn(struct work_struct *work)
 	mutex_unlock(&fbc->lock);
 }
 
+/*
+ * intel_fbc_reset_underrun - reset FBC fifo underrun status.
+ * @dev_priv: i915 device instance
+ *
+ * See intel_fbc_handle_fifo_underrun_irq(). For automated testing we
+ * want to re-enable FBC after an underrun to increase test coverage.
+ */
+int intel_fbc_reset_underrun(struct drm_i915_private *dev_priv)
+{
+	int ret;
+
+	cancel_work_sync(&dev_priv->fbc.underrun_work);
+
+	ret = mutex_lock_interruptible(&dev_priv->fbc.lock);
+	if (ret)
+		return ret;
+
+	if (dev_priv->fbc.underrun_detected) {
+		DRM_DEBUG_KMS("Re-allowing FBC after fifo underrun\n");
+		dev_priv->fbc.no_fbc_reason = "FIFO underrun cleared";
+	}
+
+	dev_priv->fbc.underrun_detected = false;
+	mutex_unlock(&dev_priv->fbc.lock);
+
+	return 0;
+}
+
 /**
  * intel_fbc_handle_fifo_underrun_irq - disable FBC when we get a FIFO underrun
  * @dev_priv: i915 device instance
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index 6467a5cc2ca3..e9e02b58b7be 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -221,6 +221,9 @@ static int intelfb_create(struct drm_fb_helper *helper,
 		goto out_unlock;
 	}
 
+	fb = &ifbdev->fb->base;
+	intel_fb_obj_flush(intel_fb_obj(fb), ORIGIN_DIRTYFB);
+
 	info = drm_fb_helper_alloc_fbi(helper);
 	if (IS_ERR(info)) {
 		DRM_ERROR("Failed to allocate fb_info\n");
@@ -230,8 +233,6 @@ static int intelfb_create(struct drm_fb_helper *helper,
 
 	info->par = helper;
 
-	fb = &ifbdev->fb->base;
-
 	ifbdev->helper.fb = fb;
 
 	strcpy(info->fix.id, "inteldrmfb");
@@ -640,7 +641,7 @@ static bool intel_fbdev_init_bios(struct drm_device *dev,
 		if (!crtc->state->active)
 			continue;
 
-		WARN(!crtc->primary->fb,
+		WARN(!crtc->primary->state->fb,
 		     "re-used BIOS config but lost an fb on crtc %d\n",
 		     crtc->base.id);
 	}
diff --git a/drivers/gpu/drm/i915/intel_frontbuffer.c b/drivers/gpu/drm/i915/intel_frontbuffer.c
index 3a8d3d06c26a..7fff0a0eceb4 100644
--- a/drivers/gpu/drm/i915/intel_frontbuffer.c
+++ b/drivers/gpu/drm/i915/intel_frontbuffer.c
@@ -80,7 +80,7 @@ void __intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
 	}
 
 	might_sleep();
-	intel_psr_invalidate(dev_priv, frontbuffer_bits);
+	intel_psr_invalidate(dev_priv, frontbuffer_bits, origin);
 	intel_edp_drrs_invalidate(dev_priv, frontbuffer_bits);
 	intel_fbc_invalidate(dev_priv, frontbuffer_bits, origin);
 }
diff --git a/drivers/gpu/drm/i915/intel_gpu_commands.h b/drivers/gpu/drm/i915/intel_gpu_commands.h
new file mode 100644
index 000000000000..105e2a9e874a
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_gpu_commands.h
@@ -0,0 +1,274 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright � 2003-2018 Intel Corporation
+ */
+
+#ifndef _INTEL_GPU_COMMANDS_H_
+#define _INTEL_GPU_COMMANDS_H_
+
+/*
+ * Instruction field definitions used by the command parser
+ */
+#define INSTR_CLIENT_SHIFT      29
+#define   INSTR_MI_CLIENT       0x0
+#define   INSTR_BC_CLIENT       0x2
+#define   INSTR_RC_CLIENT       0x3
+#define INSTR_SUBCLIENT_SHIFT   27
+#define INSTR_SUBCLIENT_MASK    0x18000000
+#define   INSTR_MEDIA_SUBCLIENT 0x2
+#define INSTR_26_TO_24_MASK	0x7000000
+#define   INSTR_26_TO_24_SHIFT	24
+
+/*
+ * Memory interface instructions used by the kernel
+ */
+#define MI_INSTR(opcode, flags) (((opcode) << 23) | (flags))
+/* Many MI commands use bit 22 of the header dword for GGTT vs PPGTT */
+#define  MI_GLOBAL_GTT    (1<<22)
+
+#define MI_NOOP			MI_INSTR(0, 0)
+#define MI_USER_INTERRUPT	MI_INSTR(0x02, 0)
+#define MI_WAIT_FOR_EVENT       MI_INSTR(0x03, 0)
+#define   MI_WAIT_FOR_OVERLAY_FLIP	(1<<16)
+#define   MI_WAIT_FOR_PLANE_B_FLIP      (1<<6)
+#define   MI_WAIT_FOR_PLANE_A_FLIP      (1<<2)
+#define   MI_WAIT_FOR_PLANE_A_SCANLINES (1<<1)
+#define MI_FLUSH		MI_INSTR(0x04, 0)
+#define   MI_READ_FLUSH		(1 << 0)
+#define   MI_EXE_FLUSH		(1 << 1)
+#define   MI_NO_WRITE_FLUSH	(1 << 2)
+#define   MI_SCENE_COUNT	(1 << 3) /* just increment scene count */
+#define   MI_END_SCENE		(1 << 4) /* flush binner and incr scene count */
+#define   MI_INVALIDATE_ISP	(1 << 5) /* invalidate indirect state pointers */
+#define MI_REPORT_HEAD		MI_INSTR(0x07, 0)
+#define MI_ARB_ON_OFF		MI_INSTR(0x08, 0)
+#define   MI_ARB_ENABLE			(1<<0)
+#define   MI_ARB_DISABLE		(0<<0)
+#define MI_BATCH_BUFFER_END	MI_INSTR(0x0a, 0)
+#define MI_SUSPEND_FLUSH	MI_INSTR(0x0b, 0)
+#define   MI_SUSPEND_FLUSH_EN	(1<<0)
+#define MI_SET_APPID		MI_INSTR(0x0e, 0)
+#define MI_OVERLAY_FLIP		MI_INSTR(0x11, 0)
+#define   MI_OVERLAY_CONTINUE	(0x0<<21)
+#define   MI_OVERLAY_ON		(0x1<<21)
+#define   MI_OVERLAY_OFF	(0x2<<21)
+#define MI_LOAD_SCAN_LINES_INCL MI_INSTR(0x12, 0)
+#define MI_DISPLAY_FLIP		MI_INSTR(0x14, 2)
+#define MI_DISPLAY_FLIP_I915	MI_INSTR(0x14, 1)
+#define   MI_DISPLAY_FLIP_PLANE(n) ((n) << 20)
+/* IVB has funny definitions for which plane to flip. */
+#define   MI_DISPLAY_FLIP_IVB_PLANE_A  (0 << 19)
+#define   MI_DISPLAY_FLIP_IVB_PLANE_B  (1 << 19)
+#define   MI_DISPLAY_FLIP_IVB_SPRITE_A (2 << 19)
+#define   MI_DISPLAY_FLIP_IVB_SPRITE_B (3 << 19)
+#define   MI_DISPLAY_FLIP_IVB_PLANE_C  (4 << 19)
+#define   MI_DISPLAY_FLIP_IVB_SPRITE_C (5 << 19)
+/* SKL ones */
+#define   MI_DISPLAY_FLIP_SKL_PLANE_1_A	(0 << 8)
+#define   MI_DISPLAY_FLIP_SKL_PLANE_1_B	(1 << 8)
+#define   MI_DISPLAY_FLIP_SKL_PLANE_1_C	(2 << 8)
+#define   MI_DISPLAY_FLIP_SKL_PLANE_2_A	(4 << 8)
+#define   MI_DISPLAY_FLIP_SKL_PLANE_2_B	(5 << 8)
+#define   MI_DISPLAY_FLIP_SKL_PLANE_2_C	(6 << 8)
+#define   MI_DISPLAY_FLIP_SKL_PLANE_3_A	(7 << 8)
+#define   MI_DISPLAY_FLIP_SKL_PLANE_3_B	(8 << 8)
+#define   MI_DISPLAY_FLIP_SKL_PLANE_3_C	(9 << 8)
+#define MI_SEMAPHORE_MBOX	MI_INSTR(0x16, 1) /* gen6, gen7 */
+#define   MI_SEMAPHORE_GLOBAL_GTT    (1<<22)
+#define   MI_SEMAPHORE_UPDATE	    (1<<21)
+#define   MI_SEMAPHORE_COMPARE	    (1<<20)
+#define   MI_SEMAPHORE_REGISTER	    (1<<18)
+#define   MI_SEMAPHORE_SYNC_VR	    (0<<16) /* RCS  wait for VCS  (RVSYNC) */
+#define   MI_SEMAPHORE_SYNC_VER	    (1<<16) /* RCS  wait for VECS (RVESYNC) */
+#define   MI_SEMAPHORE_SYNC_BR	    (2<<16) /* RCS  wait for BCS  (RBSYNC) */
+#define   MI_SEMAPHORE_SYNC_BV	    (0<<16) /* VCS  wait for BCS  (VBSYNC) */
+#define   MI_SEMAPHORE_SYNC_VEV	    (1<<16) /* VCS  wait for VECS (VVESYNC) */
+#define   MI_SEMAPHORE_SYNC_RV	    (2<<16) /* VCS  wait for RCS  (VRSYNC) */
+#define   MI_SEMAPHORE_SYNC_RB	    (0<<16) /* BCS  wait for RCS  (BRSYNC) */
+#define   MI_SEMAPHORE_SYNC_VEB	    (1<<16) /* BCS  wait for VECS (BVESYNC) */
+#define   MI_SEMAPHORE_SYNC_VB	    (2<<16) /* BCS  wait for VCS  (BVSYNC) */
+#define   MI_SEMAPHORE_SYNC_BVE	    (0<<16) /* VECS wait for BCS  (VEBSYNC) */
+#define   MI_SEMAPHORE_SYNC_VVE	    (1<<16) /* VECS wait for VCS  (VEVSYNC) */
+#define   MI_SEMAPHORE_SYNC_RVE	    (2<<16) /* VECS wait for RCS  (VERSYNC) */
+#define   MI_SEMAPHORE_SYNC_INVALID (3<<16)
+#define   MI_SEMAPHORE_SYNC_MASK    (3<<16)
+#define MI_SET_CONTEXT		MI_INSTR(0x18, 0)
+#define   MI_MM_SPACE_GTT		(1<<8)
+#define   MI_MM_SPACE_PHYSICAL		(0<<8)
+#define   MI_SAVE_EXT_STATE_EN		(1<<3)
+#define   MI_RESTORE_EXT_STATE_EN	(1<<2)
+#define   MI_FORCE_RESTORE		(1<<1)
+#define   MI_RESTORE_INHIBIT		(1<<0)
+#define   HSW_MI_RS_SAVE_STATE_EN       (1<<3)
+#define   HSW_MI_RS_RESTORE_STATE_EN    (1<<2)
+#define MI_SEMAPHORE_SIGNAL	MI_INSTR(0x1b, 0) /* GEN8+ */
+#define   MI_SEMAPHORE_TARGET(engine)	((engine)<<15)
+#define MI_SEMAPHORE_WAIT	MI_INSTR(0x1c, 2) /* GEN8+ */
+#define   MI_SEMAPHORE_POLL		(1<<15)
+#define   MI_SEMAPHORE_SAD_GTE_SDD	(1<<12)
+#define MI_STORE_DWORD_IMM	MI_INSTR(0x20, 1)
+#define MI_STORE_DWORD_IMM_GEN4	MI_INSTR(0x20, 2)
+#define   MI_MEM_VIRTUAL	(1 << 22) /* 945,g33,965 */
+#define   MI_USE_GGTT		(1 << 22) /* g4x+ */
+#define MI_STORE_DWORD_INDEX	MI_INSTR(0x21, 1)
+#define   MI_STORE_DWORD_INDEX_SHIFT 2
+/*
+ * Official intel docs are somewhat sloppy concerning MI_LOAD_REGISTER_IMM:
+ * - Always issue a MI_NOOP _before_ the MI_LOAD_REGISTER_IMM - otherwise hw
+ *   simply ignores the register load under certain conditions.
+ * - One can actually load arbitrary many arbitrary registers: Simply issue x
+ *   address/value pairs. Don't overdue it, though, x <= 2^4 must hold!
+ */
+#define MI_LOAD_REGISTER_IMM(x)	MI_INSTR(0x22, 2*(x)-1)
+#define   MI_LRI_FORCE_POSTED		(1<<12)
+#define MI_STORE_REGISTER_MEM        MI_INSTR(0x24, 1)
+#define MI_STORE_REGISTER_MEM_GEN8   MI_INSTR(0x24, 2)
+#define   MI_SRM_LRM_GLOBAL_GTT		(1<<22)
+#define MI_FLUSH_DW		MI_INSTR(0x26, 1) /* for GEN6 */
+#define   MI_FLUSH_DW_STORE_INDEX	(1<<21)
+#define   MI_INVALIDATE_TLB		(1<<18)
+#define   MI_FLUSH_DW_OP_STOREDW	(1<<14)
+#define   MI_FLUSH_DW_OP_MASK		(3<<14)
+#define   MI_FLUSH_DW_NOTIFY		(1<<8)
+#define   MI_INVALIDATE_BSD		(1<<7)
+#define   MI_FLUSH_DW_USE_GTT		(1<<2)
+#define   MI_FLUSH_DW_USE_PPGTT		(0<<2)
+#define MI_LOAD_REGISTER_MEM	   MI_INSTR(0x29, 1)
+#define MI_LOAD_REGISTER_MEM_GEN8  MI_INSTR(0x29, 2)
+#define MI_BATCH_BUFFER		MI_INSTR(0x30, 1)
+#define   MI_BATCH_NON_SECURE		(1)
+/* for snb/ivb/vlv this also means "batch in ppgtt" when ppgtt is enabled. */
+#define   MI_BATCH_NON_SECURE_I965	(1<<8)
+#define   MI_BATCH_PPGTT_HSW		(1<<8)
+#define   MI_BATCH_NON_SECURE_HSW	(1<<13)
+#define MI_BATCH_BUFFER_START	MI_INSTR(0x31, 0)
+#define   MI_BATCH_GTT		    (2<<6) /* aliased with (1<<7) on gen4 */
+#define MI_BATCH_BUFFER_START_GEN8	MI_INSTR(0x31, 1)
+#define   MI_BATCH_RESOURCE_STREAMER (1<<10)
+
+/*
+ * 3D instructions used by the kernel
+ */
+#define GFX_INSTR(opcode, flags) ((0x3 << 29) | ((opcode) << 24) | (flags))
+
+#define GEN9_MEDIA_POOL_STATE     ((0x3 << 29) | (0x2 << 27) | (0x5 << 16) | 4)
+#define   GEN9_MEDIA_POOL_ENABLE  (1 << 31)
+#define GFX_OP_RASTER_RULES    ((0x3<<29)|(0x7<<24))
+#define GFX_OP_SCISSOR         ((0x3<<29)|(0x1c<<24)|(0x10<<19))
+#define   SC_UPDATE_SCISSOR       (0x1<<1)
+#define   SC_ENABLE_MASK          (0x1<<0)
+#define   SC_ENABLE               (0x1<<0)
+#define GFX_OP_LOAD_INDIRECT   ((0x3<<29)|(0x1d<<24)|(0x7<<16))
+#define GFX_OP_SCISSOR_INFO    ((0x3<<29)|(0x1d<<24)|(0x81<<16)|(0x1))
+#define   SCI_YMIN_MASK      (0xffff<<16)
+#define   SCI_XMIN_MASK      (0xffff<<0)
+#define   SCI_YMAX_MASK      (0xffff<<16)
+#define   SCI_XMAX_MASK      (0xffff<<0)
+#define GFX_OP_SCISSOR_ENABLE	 ((0x3<<29)|(0x1c<<24)|(0x10<<19))
+#define GFX_OP_SCISSOR_RECT	 ((0x3<<29)|(0x1d<<24)|(0x81<<16)|1)
+#define GFX_OP_COLOR_FACTOR      ((0x3<<29)|(0x1d<<24)|(0x1<<16)|0x0)
+#define GFX_OP_STIPPLE           ((0x3<<29)|(0x1d<<24)|(0x83<<16))
+#define GFX_OP_MAP_INFO          ((0x3<<29)|(0x1d<<24)|0x4)
+#define GFX_OP_DESTBUFFER_VARS   ((0x3<<29)|(0x1d<<24)|(0x85<<16)|0x0)
+#define GFX_OP_DESTBUFFER_INFO	 ((0x3<<29)|(0x1d<<24)|(0x8e<<16)|1)
+#define GFX_OP_DRAWRECT_INFO     ((0x3<<29)|(0x1d<<24)|(0x80<<16)|(0x3))
+#define GFX_OP_DRAWRECT_INFO_I965  ((0x7900<<16)|0x2)
+
+#define COLOR_BLT_CMD			(2<<29 | 0x40<<22 | (5-2))
+#define SRC_COPY_BLT_CMD		((2<<29)|(0x43<<22)|4)
+#define XY_SRC_COPY_BLT_CMD		((2<<29)|(0x53<<22)|6)
+#define XY_MONO_SRC_COPY_IMM_BLT	((2<<29)|(0x71<<22)|5)
+#define   BLT_WRITE_A			(2<<20)
+#define   BLT_WRITE_RGB			(1<<20)
+#define   BLT_WRITE_RGBA		(BLT_WRITE_RGB | BLT_WRITE_A)
+#define   BLT_DEPTH_8			(0<<24)
+#define   BLT_DEPTH_16_565		(1<<24)
+#define   BLT_DEPTH_16_1555		(2<<24)
+#define   BLT_DEPTH_32			(3<<24)
+#define   BLT_ROP_SRC_COPY		(0xcc<<16)
+#define   BLT_ROP_COLOR_COPY		(0xf0<<16)
+#define XY_SRC_COPY_BLT_SRC_TILED	(1<<15) /* 965+ only */
+#define XY_SRC_COPY_BLT_DST_TILED	(1<<11) /* 965+ only */
+#define CMD_OP_DISPLAYBUFFER_INFO ((0x0<<29)|(0x14<<23)|2)
+#define   ASYNC_FLIP                (1<<22)
+#define   DISPLAY_PLANE_A           (0<<20)
+#define   DISPLAY_PLANE_B           (1<<20)
+#define GFX_OP_PIPE_CONTROL(len)	((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2))
+#define   PIPE_CONTROL_FLUSH_L3				(1<<27)
+#define   PIPE_CONTROL_GLOBAL_GTT_IVB			(1<<24) /* gen7+ */
+#define   PIPE_CONTROL_MMIO_WRITE			(1<<23)
+#define   PIPE_CONTROL_STORE_DATA_INDEX			(1<<21)
+#define   PIPE_CONTROL_CS_STALL				(1<<20)
+#define   PIPE_CONTROL_TLB_INVALIDATE			(1<<18)
+#define   PIPE_CONTROL_MEDIA_STATE_CLEAR		(1<<16)
+#define   PIPE_CONTROL_QW_WRITE				(1<<14)
+#define   PIPE_CONTROL_POST_SYNC_OP_MASK                (3<<14)
+#define   PIPE_CONTROL_DEPTH_STALL			(1<<13)
+#define   PIPE_CONTROL_WRITE_FLUSH			(1<<12)
+#define   PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH	(1<<12) /* gen6+ */
+#define   PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE	(1<<11) /* MBZ on ILK */
+#define   PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE		(1<<10) /* GM45+ only */
+#define   PIPE_CONTROL_INDIRECT_STATE_DISABLE		(1<<9)
+#define   PIPE_CONTROL_NOTIFY				(1<<8)
+#define   PIPE_CONTROL_FLUSH_ENABLE			(1<<7) /* gen7+ */
+#define   PIPE_CONTROL_DC_FLUSH_ENABLE			(1<<5)
+#define   PIPE_CONTROL_VF_CACHE_INVALIDATE		(1<<4)
+#define   PIPE_CONTROL_CONST_CACHE_INVALIDATE		(1<<3)
+#define   PIPE_CONTROL_STATE_CACHE_INVALIDATE		(1<<2)
+#define   PIPE_CONTROL_STALL_AT_SCOREBOARD		(1<<1)
+#define   PIPE_CONTROL_DEPTH_CACHE_FLUSH		(1<<0)
+#define   PIPE_CONTROL_GLOBAL_GTT (1<<2) /* in addr dword */
+
+/*
+ * Commands used only by the command parser
+ */
+#define MI_SET_PREDICATE        MI_INSTR(0x01, 0)
+#define MI_ARB_CHECK            MI_INSTR(0x05, 0)
+#define MI_RS_CONTROL           MI_INSTR(0x06, 0)
+#define MI_URB_ATOMIC_ALLOC     MI_INSTR(0x09, 0)
+#define MI_PREDICATE            MI_INSTR(0x0C, 0)
+#define MI_RS_CONTEXT           MI_INSTR(0x0F, 0)
+#define MI_TOPOLOGY_FILTER      MI_INSTR(0x0D, 0)
+#define MI_LOAD_SCAN_LINES_EXCL MI_INSTR(0x13, 0)
+#define MI_URB_CLEAR            MI_INSTR(0x19, 0)
+#define MI_UPDATE_GTT           MI_INSTR(0x23, 0)
+#define MI_CLFLUSH              MI_INSTR(0x27, 0)
+#define MI_REPORT_PERF_COUNT    MI_INSTR(0x28, 0)
+#define   MI_REPORT_PERF_COUNT_GGTT (1<<0)
+#define MI_LOAD_REGISTER_REG    MI_INSTR(0x2A, 0)
+#define MI_RS_STORE_DATA_IMM    MI_INSTR(0x2B, 0)
+#define MI_LOAD_URB_MEM         MI_INSTR(0x2C, 0)
+#define MI_STORE_URB_MEM        MI_INSTR(0x2D, 0)
+#define MI_CONDITIONAL_BATCH_BUFFER_END MI_INSTR(0x36, 0)
+
+#define PIPELINE_SELECT                ((0x3<<29)|(0x1<<27)|(0x1<<24)|(0x4<<16))
+#define GFX_OP_3DSTATE_VF_STATISTICS   ((0x3<<29)|(0x1<<27)|(0x0<<24)|(0xB<<16))
+#define MEDIA_VFE_STATE                ((0x3<<29)|(0x2<<27)|(0x0<<24)|(0x0<<16))
+#define  MEDIA_VFE_STATE_MMIO_ACCESS_MASK (0x18)
+#define GPGPU_OBJECT                   ((0x3<<29)|(0x2<<27)|(0x1<<24)|(0x4<<16))
+#define GPGPU_WALKER                   ((0x3<<29)|(0x2<<27)|(0x1<<24)|(0x5<<16))
+#define GFX_OP_3DSTATE_DX9_CONSTANTF_VS \
+	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x39<<16))
+#define GFX_OP_3DSTATE_DX9_CONSTANTF_PS \
+	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x3A<<16))
+#define GFX_OP_3DSTATE_SO_DECL_LIST \
+	((0x3<<29)|(0x3<<27)|(0x1<<24)|(0x17<<16))
+
+#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_VS \
+	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x43<<16))
+#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_GS \
+	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x44<<16))
+#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_HS \
+	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x45<<16))
+#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_DS \
+	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x46<<16))
+#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS \
+	((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x47<<16))
+
+#define MFX_WAIT  ((0x3<<29)|(0x1<<27)|(0x0<<16))
+
+#define COLOR_BLT     ((0x2<<29)|(0x40<<22))
+#define SRC_COPY_BLT  ((0x2<<29)|(0x43<<22))
+
+#endif /* _INTEL_GPU_COMMANDS_H_ */
diff --git a/drivers/gpu/drm/i915/intel_guc.c b/drivers/gpu/drm/i915/intel_guc.c
index ff08ea0ebf49..116f4ccf1bbd 100644
--- a/drivers/gpu/drm/i915/intel_guc.c
+++ b/drivers/gpu/drm/i915/intel_guc.c
@@ -64,10 +64,12 @@ void intel_guc_init_early(struct intel_guc *guc)
 {
 	intel_guc_fw_init_early(guc);
 	intel_guc_ct_init_early(&guc->ct);
-	intel_guc_log_init_early(guc);
+	intel_guc_log_init_early(&guc->log);
 
 	mutex_init(&guc->send_mutex);
+	spin_lock_init(&guc->irq_lock);
 	guc->send = intel_guc_send_nop;
+	guc->handler = intel_guc_to_host_event_handler_nop;
 	guc->notify = gen8_guc_raise_irq;
 }
 
@@ -86,9 +88,10 @@ int intel_guc_init_wq(struct intel_guc *guc)
 	 * or scheduled later on resume. This way the handling of work
 	 * item can be kept same between system suspend & rpm suspend.
 	 */
-	guc->log.runtime.flush_wq = alloc_ordered_workqueue("i915-guc_log",
-						WQ_HIGHPRI | WQ_FREEZABLE);
-	if (!guc->log.runtime.flush_wq) {
+	guc->log.relay.flush_wq =
+		alloc_ordered_workqueue("i915-guc_log",
+					WQ_HIGHPRI | WQ_FREEZABLE);
+	if (!guc->log.relay.flush_wq) {
 		DRM_ERROR("Couldn't allocate workqueue for GuC log\n");
 		return -ENOMEM;
 	}
@@ -111,7 +114,7 @@ int intel_guc_init_wq(struct intel_guc *guc)
 		guc->preempt_wq = alloc_ordered_workqueue("i915-guc_preempt",
 							  WQ_HIGHPRI);
 		if (!guc->preempt_wq) {
-			destroy_workqueue(guc->log.runtime.flush_wq);
+			destroy_workqueue(guc->log.relay.flush_wq);
 			DRM_ERROR("Couldn't allocate workqueue for GuC "
 				  "preemption\n");
 			return -ENOMEM;
@@ -129,7 +132,7 @@ void intel_guc_fini_wq(struct intel_guc *guc)
 	    USES_GUC_SUBMISSION(dev_priv))
 		destroy_workqueue(guc->preempt_wq);
 
-	destroy_workqueue(guc->log.runtime.flush_wq);
+	destroy_workqueue(guc->log.relay.flush_wq);
 }
 
 static int guc_shared_data_create(struct intel_guc *guc)
@@ -169,7 +172,7 @@ int intel_guc_init(struct intel_guc *guc)
 		return ret;
 	GEM_BUG_ON(!guc->shared_data);
 
-	ret = intel_guc_log_create(guc);
+	ret = intel_guc_log_create(&guc->log);
 	if (ret)
 		goto err_shared;
 
@@ -184,7 +187,7 @@ int intel_guc_init(struct intel_guc *guc)
 	return 0;
 
 err_log:
-	intel_guc_log_destroy(guc);
+	intel_guc_log_destroy(&guc->log);
 err_shared:
 	guc_shared_data_destroy(guc);
 	return ret;
@@ -196,41 +199,27 @@ void intel_guc_fini(struct intel_guc *guc)
 
 	i915_ggtt_disable_guc(dev_priv);
 	intel_guc_ads_destroy(guc);
-	intel_guc_log_destroy(guc);
+	intel_guc_log_destroy(&guc->log);
 	guc_shared_data_destroy(guc);
 }
 
-static u32 get_gt_type(struct drm_i915_private *dev_priv)
+static u32 get_log_control_flags(void)
 {
-	/* XXX: GT type based on PCI device ID? field seems unused by fw */
-	return 0;
-}
-
-static u32 get_core_family(struct drm_i915_private *dev_priv)
-{
-	u32 gen = INTEL_GEN(dev_priv);
+	u32 level = i915_modparams.guc_log_level;
+	u32 flags = 0;
 
-	switch (gen) {
-	case 9:
-		return GUC_CORE_FAMILY_GEN9;
-
-	default:
-		MISSING_CASE(gen);
-		return GUC_CORE_FAMILY_UNKNOWN;
-	}
-}
+	GEM_BUG_ON(level < 0);
 
-static u32 get_log_verbosity_flags(void)
-{
-	if (i915_modparams.guc_log_level > 0) {
-		u32 verbosity = i915_modparams.guc_log_level - 1;
+	if (!GUC_LOG_LEVEL_IS_ENABLED(level))
+		flags |= GUC_LOG_DEFAULT_DISABLED;
 
-		GEM_BUG_ON(verbosity > GUC_LOG_VERBOSITY_MAX);
-		return verbosity << GUC_LOG_VERBOSITY_SHIFT;
-	}
+	if (!GUC_LOG_LEVEL_IS_VERBOSE(level))
+		flags |= GUC_LOG_DISABLED;
+	else
+		flags |= GUC_LOG_LEVEL_TO_VERBOSITY(level) <<
+			 GUC_LOG_VERBOSITY_SHIFT;
 
-	GEM_BUG_ON(i915_modparams.enable_guc < 0);
-	return GUC_LOG_DISABLED;
+	return flags;
 }
 
 /*
@@ -246,10 +235,6 @@ void intel_guc_init_params(struct intel_guc *guc)
 
 	memset(params, 0, sizeof(params));
 
-	params[GUC_CTL_DEVICE_INFO] |=
-		(get_gt_type(dev_priv) << GUC_CTL_GT_TYPE_SHIFT) |
-		(get_core_family(dev_priv) << GUC_CTL_CORE_FAMILY_SHIFT);
-
 	/*
 	 * GuC ARAT increment is 10 ns. GuC default scheduler quantum is one
 	 * second. This ARAR is calculated by:
@@ -265,12 +250,13 @@ void intel_guc_init_params(struct intel_guc *guc)
 
 	params[GUC_CTL_LOG_PARAMS] = guc->log.flags;
 
-	params[GUC_CTL_DEBUG] = get_log_verbosity_flags();
+	params[GUC_CTL_DEBUG] = get_log_control_flags();
 
 	/* If GuC submission is enabled, set up additional parameters here */
 	if (USES_GUC_SUBMISSION(dev_priv)) {
-		u32 ads = guc_ggtt_offset(guc->ads_vma) >> PAGE_SHIFT;
-		u32 pgs = guc_ggtt_offset(dev_priv->guc.stage_desc_pool);
+		u32 ads = intel_guc_ggtt_offset(guc,
+						guc->ads_vma) >> PAGE_SHIFT;
+		u32 pgs = intel_guc_ggtt_offset(guc, guc->stage_desc_pool);
 		u32 ctx_in_16 = GUC_MAX_STAGE_DESCRIPTORS / 16;
 
 		params[GUC_CTL_DEBUG] |= ads << GUC_ADS_ADDR_SHIFT;
@@ -301,16 +287,23 @@ void intel_guc_init_params(struct intel_guc *guc)
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_BLITTER);
 }
 
-int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len)
+int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len,
+		       u32 *response_buf, u32 response_buf_size)
 {
 	WARN(1, "Unexpected send: action=%#x\n", *action);
 	return -ENODEV;
 }
 
+void intel_guc_to_host_event_handler_nop(struct intel_guc *guc)
+{
+	WARN(1, "Unexpected event: no suitable handler\n");
+}
+
 /*
  * This function implements the MMIO based host to GuC interface.
  */
-int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len)
+int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
+			u32 *response_buf, u32 response_buf_size)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 	u32 status;
@@ -320,6 +313,9 @@ int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len)
 	GEM_BUG_ON(!len);
 	GEM_BUG_ON(len > guc->send_regs.count);
 
+	/* We expect only action code */
+	GEM_BUG_ON(*action & ~INTEL_GUC_MSG_CODE_MASK);
+
 	/* If CT is available, we expect to use MMIO only during init/fini */
 	GEM_BUG_ON(HAS_GUC_CT(dev_priv) &&
 		*action != INTEL_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER &&
@@ -341,29 +337,74 @@ int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len)
 	 */
 	ret = __intel_wait_for_register_fw(dev_priv,
 					   guc_send_reg(guc, 0),
-					   INTEL_GUC_RECV_MASK,
-					   INTEL_GUC_RECV_MASK,
+					   INTEL_GUC_MSG_TYPE_MASK,
+					   INTEL_GUC_MSG_TYPE_RESPONSE <<
+					   INTEL_GUC_MSG_TYPE_SHIFT,
 					   10, 10, &status);
-	if (status != INTEL_GUC_STATUS_SUCCESS) {
-		/*
-		 * Either the GuC explicitly returned an error (which
-		 * we convert to -EIO here) or no response at all was
-		 * received within the timeout limit (-ETIMEDOUT)
-		 */
-		if (ret != -ETIMEDOUT)
-			ret = -EIO;
-
-		DRM_WARN("INTEL_GUC_SEND: Action 0x%X failed;"
-			 " ret=%d status=0x%08X response=0x%08X\n",
-			 action[0], ret, status, I915_READ(SOFT_SCRATCH(15)));
+	/* If GuC explicitly returned an error, convert it to -EIO */
+	if (!ret && !INTEL_GUC_MSG_IS_RESPONSE_SUCCESS(status))
+		ret = -EIO;
+
+	if (ret) {
+		DRM_DEBUG_DRIVER("INTEL_GUC_SEND: Action 0x%X failed;"
+				 " ret=%d status=0x%08X response=0x%08X\n",
+				 action[0], ret, status,
+				 I915_READ(SOFT_SCRATCH(15)));
+		goto out;
 	}
 
+	if (response_buf) {
+		int count = min(response_buf_size, guc->send_regs.count - 1);
+
+		for (i = 0; i < count; i++)
+			response_buf[i] = I915_READ(guc_send_reg(guc, i + 1));
+	}
+
+	/* Use data from the GuC response as our return value */
+	ret = INTEL_GUC_MSG_TO_DATA(status);
+
+out:
 	intel_uncore_forcewake_put(dev_priv, guc->send_regs.fw_domains);
 	mutex_unlock(&guc->send_mutex);
 
 	return ret;
 }
 
+void intel_guc_to_host_event_handler_mmio(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	u32 msg, val;
+
+	/*
+	 * Sample the log buffer flush related bits & clear them out now
+	 * itself from the message identity register to minimize the
+	 * probability of losing a flush interrupt, when there are back
+	 * to back flush interrupts.
+	 * There can be a new flush interrupt, for different log buffer
+	 * type (like for ISR), whilst Host is handling one (for DPC).
+	 * Since same bit is used in message register for ISR & DPC, it
+	 * could happen that GuC sets the bit for 2nd interrupt but Host
+	 * clears out the bit on handling the 1st interrupt.
+	 */
+	spin_lock(&guc->irq_lock);
+	val = I915_READ(SOFT_SCRATCH(15));
+	msg = val & guc->msg_enabled_mask;
+	I915_WRITE(SOFT_SCRATCH(15), val & ~msg);
+	spin_unlock(&guc->irq_lock);
+
+	intel_guc_to_host_process_recv_msg(guc, msg);
+}
+
+void intel_guc_to_host_process_recv_msg(struct intel_guc *guc, u32 msg)
+{
+	/* Make sure to handle only enabled messages */
+	msg &= guc->msg_enabled_mask;
+
+	if (msg & (INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER |
+		   INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED))
+		intel_guc_log_handle_flush_event(&guc->log);
+}
+
 int intel_guc_sample_forcewake(struct intel_guc *guc)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
@@ -410,7 +451,7 @@ int intel_guc_suspend(struct intel_guc *guc)
 	u32 data[] = {
 		INTEL_GUC_ACTION_ENTER_S_STATE,
 		GUC_POWER_D1, /* any value greater than GUC_POWER_D0 */
-		guc_ggtt_offset(guc->shared_data)
+		intel_guc_ggtt_offset(guc, guc->shared_data)
 	};
 
 	return intel_guc_send(guc, data, ARRAY_SIZE(data));
@@ -434,7 +475,7 @@ int intel_guc_reset_engine(struct intel_guc *guc,
 	data[3] = 0;
 	data[4] = 0;
 	data[5] = guc->execbuf_client->stage_id;
-	data[6] = guc_ggtt_offset(guc->shared_data);
+	data[6] = intel_guc_ggtt_offset(guc, guc->shared_data);
 
 	return intel_guc_send(guc, data, ARRAY_SIZE(data));
 }
@@ -448,12 +489,65 @@ int intel_guc_resume(struct intel_guc *guc)
 	u32 data[] = {
 		INTEL_GUC_ACTION_EXIT_S_STATE,
 		GUC_POWER_D0,
-		guc_ggtt_offset(guc->shared_data)
+		intel_guc_ggtt_offset(guc, guc->shared_data)
 	};
 
 	return intel_guc_send(guc, data, ARRAY_SIZE(data));
 }
 
+/**
+ * DOC: GuC Address Space
+ *
+ * The layout of GuC address space is shown below:
+ *
+ * ::
+ *
+ *     +==============> +====================+ <== GUC_GGTT_TOP
+ *     ^                |                    |
+ *     |                |                    |
+ *     |                |        DRAM        |
+ *     |                |       Memory       |
+ *     |                |                    |
+ *    GuC               |                    |
+ *  Address  +========> +====================+ <== WOPCM Top
+ *   Space   ^          |   HW contexts RSVD |
+ *     |     |          |        WOPCM       |
+ *     |     |     +==> +--------------------+ <== GuC WOPCM Top
+ *     |    GuC    ^    |                    |
+ *     |    GGTT   |    |                    |
+ *     |    Pin   GuC   |        GuC         |
+ *     |    Bias WOPCM  |       WOPCM        |
+ *     |     |    Size  |                    |
+ *     |     |     |    |                    |
+ *     v     v     v    |                    |
+ *     +=====+=====+==> +====================+ <== GuC WOPCM Base
+ *                      |   Non-GuC WOPCM    |
+ *                      |   (HuC/Reserved)   |
+ *                      +====================+ <== WOPCM Base
+ *
+ * The lower part of GuC Address Space [0, ggtt_pin_bias) is mapped to WOPCM
+ * while upper part of GuC Address Space [ggtt_pin_bias, GUC_GGTT_TOP) is mapped
+ * to DRAM. The value of the GuC ggtt_pin_bias is determined by WOPCM size and
+ * actual GuC WOPCM size.
+ */
+
+/**
+ * intel_guc_init_ggtt_pin_bias() - Initialize the GuC ggtt_pin_bias value.
+ * @guc: intel_guc structure.
+ *
+ * This function will calculate and initialize the ggtt_pin_bias value based on
+ * overall WOPCM size and GuC WOPCM size.
+ */
+void intel_guc_init_ggtt_pin_bias(struct intel_guc *guc)
+{
+	struct drm_i915_private *i915 = guc_to_i915(guc);
+
+	GEM_BUG_ON(!i915->wopcm.size);
+	GEM_BUG_ON(i915->wopcm.size < i915->wopcm.guc.base);
+
+	guc->ggtt_pin_bias = i915->wopcm.size - i915->wopcm.guc.base;
+}
+
 /**
  * intel_guc_allocate_vma() - Allocate a GGTT VMA for GuC usage
  * @guc:	the guc
@@ -462,7 +556,7 @@ int intel_guc_resume(struct intel_guc *guc)
  * This is a wrapper to create an object for use with the GuC. In order to
  * use it inside the GuC, an object needs to be pinned lifetime, so we allocate
  * both some backing storage and a range inside the Global GTT. We must pin
- * it in the GGTT somewhere other than than [0, GUC_WOPCM_TOP) because that
+ * it in the GGTT somewhere other than than [0, GUC ggtt_pin_bias) because that
  * range is reserved inside GuC.
  *
  * Return:	A i915_vma if successful, otherwise an ERR_PTR.
@@ -483,7 +577,7 @@ struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size)
 		goto err;
 
 	ret = i915_vma_pin(vma, 0, PAGE_SIZE,
-			   PIN_GLOBAL | PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
+			   PIN_GLOBAL | PIN_OFFSET_BIAS | guc->ggtt_pin_bias);
 	if (ret) {
 		vma = ERR_PTR(ret);
 		goto err;
@@ -495,14 +589,3 @@ struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size)
 	i915_gem_object_put(obj);
 	return vma;
 }
-
-u32 intel_guc_wopcm_size(struct drm_i915_private *dev_priv)
-{
-	u32 wopcm_size = GUC_WOPCM_TOP;
-
-	/* On BXT, the top of WOPCM is reserved for RC6 context */
-	if (IS_GEN9_LP(dev_priv))
-		wopcm_size -= BXT_GUC_WOPCM_RC6_RESERVED;
-
-	return wopcm_size;
-}
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index b9424ac644ac..f1265e122d30 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -49,11 +49,16 @@ struct intel_guc {
 	struct intel_guc_log log;
 	struct intel_guc_ct ct;
 
+	/* Offset where Non-WOPCM memory starts. */
+	u32 ggtt_pin_bias;
+
 	/* Log snapshot if GuC errors during load */
 	struct drm_i915_gem_object *load_err_log;
 
 	/* intel_guc_recv interrupt related state */
+	spinlock_t irq_lock;
 	bool interrupts_enabled;
+	unsigned int msg_enabled_mask;
 
 	struct i915_vma *ads_vma;
 	struct i915_vma *stage_desc_pool;
@@ -83,7 +88,11 @@ struct intel_guc {
 	struct mutex send_mutex;
 
 	/* GuC's FW specific send function */
-	int (*send)(struct intel_guc *guc, const u32 *data, u32 len);
+	int (*send)(struct intel_guc *guc, const u32 *data, u32 len,
+		    u32 *response_buf, u32 response_buf_size);
+
+	/* GuC's FW specific event handler function */
+	void (*handler)(struct intel_guc *guc);
 
 	/* GuC's FW specific notify function */
 	void (*notify)(struct intel_guc *guc);
@@ -92,7 +101,14 @@ struct intel_guc {
 static
 inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len)
 {
-	return guc->send(guc, action, len);
+	return guc->send(guc, action, len, NULL, 0);
+}
+
+static inline int
+intel_guc_send_and_receive(struct intel_guc *guc, const u32 *action, u32 len,
+			   u32 *response_buf, u32 response_buf_size)
+{
+	return guc->send(guc, action, len, response_buf, response_buf_size);
 }
 
 static inline void intel_guc_notify(struct intel_guc *guc)
@@ -100,17 +116,33 @@ static inline void intel_guc_notify(struct intel_guc *guc)
 	guc->notify(guc);
 }
 
-/*
- * GuC does not allow any gfx GGTT address that falls into range [0, WOPCM_TOP),
- * which is reserved for Boot ROM, SRAM and WOPCM. Currently this top address is
- * 512K. In order to exclude 0-512K address space from GGTT, all gfx objects
- * used by GuC is pinned with PIN_OFFSET_BIAS along with size of WOPCM.
+static inline void intel_guc_to_host_event_handler(struct intel_guc *guc)
+{
+	guc->handler(guc);
+}
+
+/* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */
+#define GUC_GGTT_TOP	0xFEE00000
+
+/**
+ * intel_guc_ggtt_offset() - Get and validate the GGTT offset of @vma
+ * @guc: intel_guc structure.
+ * @vma: i915 graphics virtual memory area.
+ *
+ * GuC does not allow any gfx GGTT address that falls into range
+ * [0, GuC ggtt_pin_bias), which is reserved for Boot ROM, SRAM and WOPCM.
+ * Currently, in order to exclude [0, GuC ggtt_pin_bias) address space from
+ * GGTT, all gfx objects used by GuC are allocated with intel_guc_allocate_vma()
+ * and pinned with PIN_OFFSET_BIAS along with the value of GuC ggtt_pin_bias.
+ *
+ * Return: GGTT offset of the @vma.
  */
-static inline u32 guc_ggtt_offset(struct i915_vma *vma)
+static inline u32 intel_guc_ggtt_offset(struct intel_guc *guc,
+					struct i915_vma *vma)
 {
 	u32 offset = i915_ggtt_offset(vma);
 
-	GEM_BUG_ON(offset < GUC_WOPCM_TOP);
+	GEM_BUG_ON(offset < guc->ggtt_pin_bias);
 	GEM_BUG_ON(range_overflows_t(u64, offset, vma->size, GUC_GGTT_TOP));
 
 	return offset;
@@ -119,17 +151,43 @@ static inline u32 guc_ggtt_offset(struct i915_vma *vma)
 void intel_guc_init_early(struct intel_guc *guc);
 void intel_guc_init_send_regs(struct intel_guc *guc);
 void intel_guc_init_params(struct intel_guc *guc);
+void intel_guc_init_ggtt_pin_bias(struct intel_guc *guc);
 int intel_guc_init_wq(struct intel_guc *guc);
 void intel_guc_fini_wq(struct intel_guc *guc);
 int intel_guc_init(struct intel_guc *guc);
 void intel_guc_fini(struct intel_guc *guc);
-int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len);
-int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len);
+int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len,
+		       u32 *response_buf, u32 response_buf_size);
+int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
+			u32 *response_buf, u32 response_buf_size);
+void intel_guc_to_host_event_handler(struct intel_guc *guc);
+void intel_guc_to_host_event_handler_nop(struct intel_guc *guc);
+void intel_guc_to_host_event_handler_mmio(struct intel_guc *guc);
+void intel_guc_to_host_process_recv_msg(struct intel_guc *guc, u32 msg);
 int intel_guc_sample_forcewake(struct intel_guc *guc);
 int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset);
 int intel_guc_suspend(struct intel_guc *guc);
 int intel_guc_resume(struct intel_guc *guc);
 struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size);
-u32 intel_guc_wopcm_size(struct drm_i915_private *dev_priv);
+
+static inline int intel_guc_sanitize(struct intel_guc *guc)
+{
+	intel_uc_fw_sanitize(&guc->fw);
+	return 0;
+}
+
+static inline void intel_guc_enable_msg(struct intel_guc *guc, u32 mask)
+{
+	spin_lock_irq(&guc->irq_lock);
+	guc->msg_enabled_mask |= mask;
+	spin_unlock_irq(&guc->irq_lock);
+}
+
+static inline void intel_guc_disable_msg(struct intel_guc *guc, u32 mask)
+{
+	spin_lock_irq(&guc->irq_lock);
+	guc->msg_enabled_mask &= ~mask;
+	spin_unlock_irq(&guc->irq_lock);
+}
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_guc_ads.c b/drivers/gpu/drm/i915/intel_guc_ads.c
index ac627534667d..dcaa3fb71765 100644
--- a/drivers/gpu/drm/i915/intel_guc_ads.c
+++ b/drivers/gpu/drm/i915/intel_guc_ads.c
@@ -75,7 +75,7 @@ static void guc_policies_init(struct guc_policies *policies)
 int intel_guc_ads_create(struct intel_guc *guc)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
-	struct i915_vma *vma;
+	struct i915_vma *vma, *kernel_ctx_vma;
 	struct page *page;
 	/* The ads obj includes the struct itself and buffers passed to GuC */
 	struct {
@@ -121,9 +121,10 @@ int intel_guc_ads_create(struct intel_guc *guc)
 	 * to find it. Note that we have to skip our header (1 page),
 	 * because our GuC shared data is there.
 	 */
+	kernel_ctx_vma = to_intel_context(dev_priv->kernel_context,
+					  dev_priv->engine[RCS])->state;
 	blob->ads.golden_context_lrca =
-		guc_ggtt_offset(dev_priv->kernel_context->engine[RCS].state) +
-		skipped_offset;
+		intel_guc_ggtt_offset(guc, kernel_ctx_vma) + skipped_offset;
 
 	/*
 	 * The GuC expects us to exclude the portion of the context image that
@@ -135,7 +136,7 @@ int intel_guc_ads_create(struct intel_guc *guc)
 		blob->ads.eng_state_size[engine->guc_id] =
 			engine->context_size - skipped_size;
 
-	base = guc_ggtt_offset(vma);
+	base = intel_guc_ggtt_offset(guc, vma);
 	blob->ads.scheduler_policies = base + ptr_offset(blob, policies);
 	blob->ads.reg_state_buffer = base + ptr_offset(blob, reg_state_buffer);
 	blob->ads.reg_state_addr = base + ptr_offset(blob, reg_state);
diff --git a/drivers/gpu/drm/i915/intel_guc_ct.c b/drivers/gpu/drm/i915/intel_guc_ct.c
index 24ad55752396..371b6005954a 100644
--- a/drivers/gpu/drm/i915/intel_guc_ct.c
+++ b/drivers/gpu/drm/i915/intel_guc_ct.c
@@ -24,14 +24,49 @@
 #include "i915_drv.h"
 #include "intel_guc_ct.h"
 
+#ifdef CONFIG_DRM_I915_DEBUG_GUC
+#define CT_DEBUG_DRIVER(...)	DRM_DEBUG_DRIVER(__VA_ARGS__)
+#else
+#define CT_DEBUG_DRIVER(...)	do { } while (0)
+#endif
+
+struct ct_request {
+	struct list_head link;
+	u32 fence;
+	u32 status;
+	u32 response_len;
+	u32 *response_buf;
+};
+
+struct ct_incoming_request {
+	struct list_head link;
+	u32 msg[];
+};
+
 enum { CTB_SEND = 0, CTB_RECV = 1 };
 
 enum { CTB_OWNER_HOST = 0 };
 
+static void ct_incoming_request_worker_func(struct work_struct *w);
+
+/**
+ * intel_guc_ct_init_early - Initialize CT state without requiring device access
+ * @ct: pointer to CT struct
+ */
 void intel_guc_ct_init_early(struct intel_guc_ct *ct)
 {
 	/* we're using static channel owners */
 	ct->host_channel.owner = CTB_OWNER_HOST;
+
+	spin_lock_init(&ct->lock);
+	INIT_LIST_HEAD(&ct->pending_requests);
+	INIT_LIST_HEAD(&ct->incoming_requests);
+	INIT_WORK(&ct->worker, ct_incoming_request_worker_func);
+}
+
+static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct)
+{
+	return container_of(ct, struct intel_guc, ct);
 }
 
 static inline const char *guc_ct_buffer_type_to_str(u32 type)
@@ -49,8 +84,8 @@ static inline const char *guc_ct_buffer_type_to_str(u32 type)
 static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc,
 				    u32 cmds_addr, u32 size, u32 owner)
 {
-	DRM_DEBUG_DRIVER("CT: desc %p init addr=%#x size=%u owner=%u\n",
-			 desc, cmds_addr, size, owner);
+	CT_DEBUG_DRIVER("CT: desc %p init addr=%#x size=%u owner=%u\n",
+			desc, cmds_addr, size, owner);
 	memset(desc, 0, sizeof(*desc));
 	desc->addr = cmds_addr;
 	desc->size = size;
@@ -59,8 +94,8 @@ static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc,
 
 static void guc_ct_buffer_desc_reset(struct guc_ct_buffer_desc *desc)
 {
-	DRM_DEBUG_DRIVER("CT: desc %p reset head=%u tail=%u\n",
-			 desc, desc->head, desc->tail);
+	CT_DEBUG_DRIVER("CT: desc %p reset head=%u tail=%u\n",
+			desc, desc->head, desc->tail);
 	desc->head = 0;
 	desc->tail = 0;
 	desc->is_in_error = 0;
@@ -79,7 +114,7 @@ static int guc_action_register_ct_buffer(struct intel_guc *guc,
 	int err;
 
 	/* Can't use generic send(), CT registration must go over MMIO */
-	err = intel_guc_send_mmio(guc, action, ARRAY_SIZE(action));
+	err = intel_guc_send_mmio(guc, action, ARRAY_SIZE(action), NULL, 0);
 	if (err)
 		DRM_ERROR("CT: register %s buffer failed; err=%d\n",
 			  guc_ct_buffer_type_to_str(type), err);
@@ -98,7 +133,7 @@ static int guc_action_deregister_ct_buffer(struct intel_guc *guc,
 	int err;
 
 	/* Can't use generic send(), CT deregistration must go over MMIO */
-	err = intel_guc_send_mmio(guc, action, ARRAY_SIZE(action));
+	err = intel_guc_send_mmio(guc, action, ARRAY_SIZE(action), NULL, 0);
 	if (err)
 		DRM_ERROR("CT: deregister %s buffer failed; owner=%d err=%d\n",
 			  guc_ct_buffer_type_to_str(type), owner, err);
@@ -156,7 +191,8 @@ static int ctch_init(struct intel_guc *guc,
 		err = PTR_ERR(blob);
 		goto err_vma;
 	}
-	DRM_DEBUG_DRIVER("CT: vma base=%#x\n", guc_ggtt_offset(ctch->vma));
+	CT_DEBUG_DRIVER("CT: vma base=%#x\n",
+			intel_guc_ggtt_offset(guc, ctch->vma));
 
 	/* store pointers to desc and cmds */
 	for (i = 0; i < ARRAY_SIZE(ctch->ctbs); i++) {
@@ -170,8 +206,8 @@ static int ctch_init(struct intel_guc *guc,
 err_vma:
 	i915_vma_unpin_and_release(&ctch->vma);
 err_out:
-	DRM_DEBUG_DRIVER("CT: channel %d initialization failed; err=%d\n",
-			 ctch->owner, err);
+	CT_DEBUG_DRIVER("CT: channel %d initialization failed; err=%d\n",
+			ctch->owner, err);
 	return err;
 }
 
@@ -191,8 +227,8 @@ static int ctch_open(struct intel_guc *guc,
 	int err;
 	int i;
 
-	DRM_DEBUG_DRIVER("CT: channel %d reopen=%s\n",
-			 ctch->owner, yesno(ctch_is_open(ctch)));
+	CT_DEBUG_DRIVER("CT: channel %d reopen=%s\n",
+			ctch->owner, yesno(ctch_is_open(ctch)));
 
 	if (!ctch->vma) {
 		err = ctch_init(guc, ctch);
@@ -202,7 +238,7 @@ static int ctch_open(struct intel_guc *guc,
 	}
 
 	/* vma should be already allocated and map'ed */
-	base = guc_ggtt_offset(ctch->vma);
+	base = intel_guc_ggtt_offset(guc, ctch->vma);
 
 	/* (re)initialize descriptors
 	 * cmds buffers are in the second half of the blob page
@@ -263,10 +299,29 @@ static u32 ctch_get_next_fence(struct intel_guc_ct_channel *ctch)
 	return ++ctch->next_fence;
 }
 
+/**
+ * DOC: CTB Host to GuC request
+ *
+ * Format of the CTB Host to GuC request message is as follows::
+ *
+ *      +------------+---------+---------+---------+---------+
+ *      |   msg[0]   |   [1]   |   [2]   |   ...   |  [n-1]  |
+ *      +------------+---------+---------+---------+---------+
+ *      |   MESSAGE  |       MESSAGE PAYLOAD                 |
+ *      +   HEADER   +---------+---------+---------+---------+
+ *      |            |    0    |    1    |   ...   |    n    |
+ *      +============+=========+=========+=========+=========+
+ *      |  len >= 1  |  FENCE  |     request specific data   |
+ *      +------+-----+---------+---------+---------+---------+
+ *
+ *                   ^-----------------len-------------------^
+ */
+
 static int ctb_write(struct intel_guc_ct_buffer *ctb,
 		     const u32 *action,
 		     u32 len /* in dwords */,
-		     u32 fence)
+		     u32 fence,
+		     bool want_response)
 {
 	struct guc_ct_buffer_desc *desc = ctb->desc;
 	u32 head = desc->head / 4;	/* in dwords */
@@ -295,15 +350,21 @@ static int ctb_write(struct intel_guc_ct_buffer *ctb,
 	if (unlikely(used + len + 1 >= size))
 		return -ENOSPC;
 
-	/* Write the message. The format is the following:
+	/*
+	 * Write the message. The format is the following:
 	 * DW0: header (including action code)
 	 * DW1: fence
 	 * DW2+: action data
 	 */
 	header = (len << GUC_CT_MSG_LEN_SHIFT) |
 		 (GUC_CT_MSG_WRITE_FENCE_TO_DESC) |
+		 (want_response ? GUC_CT_MSG_SEND_STATUS : 0) |
 		 (action[0] << GUC_CT_MSG_ACTION_SHIFT);
 
+	CT_DEBUG_DRIVER("CT: writing %*ph %*ph %*ph\n",
+			4, &header, 4, &fence,
+			4 * (len - 1), &action[1]);
+
 	cmds[tail] = header;
 	tail = (tail + 1) % size;
 
@@ -322,16 +383,25 @@ static int ctb_write(struct intel_guc_ct_buffer *ctb,
 	return 0;
 }
 
-/* Wait for the response from the GuC.
+/**
+ * wait_for_ctb_desc_update - Wait for the CT buffer descriptor update.
+ * @desc:	buffer descriptor
  * @fence:	response fence
  * @status:	placeholder for status
- * return:	0 response received (status is valid)
- *		-ETIMEDOUT no response within hardcoded timeout
- *		-EPROTO no response, ct buffer was in error
+ *
+ * Guc will update CT buffer descriptor with new fence and status
+ * after processing the command identified by the fence. Wait for
+ * specified fence and then read from the descriptor status of the
+ * command.
+ *
+ * Return:
+ * *	0 response received (status is valid)
+ * *	-ETIMEDOUT no response within hardcoded timeout
+ * *	-EPROTO no response, CT buffer is in error
  */
-static int wait_for_response(struct guc_ct_buffer_desc *desc,
-			     u32 fence,
-			     u32 *status)
+static int wait_for_ctb_desc_update(struct guc_ct_buffer_desc *desc,
+				    u32 fence,
+				    u32 *status)
 {
 	int err;
 
@@ -363,71 +433,440 @@ static int wait_for_response(struct guc_ct_buffer_desc *desc,
 	return err;
 }
 
-static int ctch_send(struct intel_guc *guc,
+/**
+ * wait_for_ct_request_update - Wait for CT request state update.
+ * @req:	pointer to pending request
+ * @status:	placeholder for status
+ *
+ * For each sent request, Guc shall send bac CT response message.
+ * Our message handler will update status of tracked request once
+ * response message with given fence is received. Wait here and
+ * check for valid response status value.
+ *
+ * Return:
+ * *	0 response received (status is valid)
+ * *	-ETIMEDOUT no response within hardcoded timeout
+ */
+static int wait_for_ct_request_update(struct ct_request *req, u32 *status)
+{
+	int err;
+
+	/*
+	 * Fast commands should complete in less than 10us, so sample quickly
+	 * up to that length of time, then switch to a slower sleep-wait loop.
+	 * No GuC command should ever take longer than 10ms.
+	 */
+#define done INTEL_GUC_MSG_IS_RESPONSE(READ_ONCE(req->status))
+	err = wait_for_us(done, 10);
+	if (err)
+		err = wait_for(done, 10);
+#undef done
+
+	if (unlikely(err))
+		DRM_ERROR("CT: fence %u err %d\n", req->fence, err);
+
+	*status = req->status;
+	return err;
+}
+
+static int ctch_send(struct intel_guc_ct *ct,
 		     struct intel_guc_ct_channel *ctch,
 		     const u32 *action,
 		     u32 len,
+		     u32 *response_buf,
+		     u32 response_buf_size,
 		     u32 *status)
 {
 	struct intel_guc_ct_buffer *ctb = &ctch->ctbs[CTB_SEND];
 	struct guc_ct_buffer_desc *desc = ctb->desc;
+	struct ct_request request;
+	unsigned long flags;
 	u32 fence;
 	int err;
 
 	GEM_BUG_ON(!ctch_is_open(ctch));
 	GEM_BUG_ON(!len);
 	GEM_BUG_ON(len & ~GUC_CT_MSG_LEN_MASK);
+	GEM_BUG_ON(!response_buf && response_buf_size);
 
 	fence = ctch_get_next_fence(ctch);
-	err = ctb_write(ctb, action, len, fence);
+	request.fence = fence;
+	request.status = 0;
+	request.response_len = response_buf_size;
+	request.response_buf = response_buf;
+
+	spin_lock_irqsave(&ct->lock, flags);
+	list_add_tail(&request.link, &ct->pending_requests);
+	spin_unlock_irqrestore(&ct->lock, flags);
+
+	err = ctb_write(ctb, action, len, fence, !!response_buf);
 	if (unlikely(err))
-		return err;
+		goto unlink;
 
-	intel_guc_notify(guc);
+	intel_guc_notify(ct_to_guc(ct));
 
-	err = wait_for_response(desc, fence, status);
+	if (response_buf)
+		err = wait_for_ct_request_update(&request, status);
+	else
+		err = wait_for_ctb_desc_update(desc, fence, status);
 	if (unlikely(err))
-		return err;
-	if (*status != INTEL_GUC_STATUS_SUCCESS)
-		return -EIO;
-	return 0;
+		goto unlink;
+
+	if (!INTEL_GUC_MSG_IS_RESPONSE_SUCCESS(*status)) {
+		err = -EIO;
+		goto unlink;
+	}
+
+	if (response_buf) {
+		/* There shall be no data in the status */
+		WARN_ON(INTEL_GUC_MSG_TO_DATA(request.status));
+		/* Return actual response len */
+		err = request.response_len;
+	} else {
+		/* There shall be no response payload */
+		WARN_ON(request.response_len);
+		/* Return data decoded from the status dword */
+		err = INTEL_GUC_MSG_TO_DATA(*status);
+	}
+
+unlink:
+	spin_lock_irqsave(&ct->lock, flags);
+	list_del(&request.link);
+	spin_unlock_irqrestore(&ct->lock, flags);
+
+	return err;
 }
 
 /*
  * Command Transport (CT) buffer based GuC send function.
  */
-static int intel_guc_send_ct(struct intel_guc *guc, const u32 *action, u32 len)
+static int intel_guc_send_ct(struct intel_guc *guc, const u32 *action, u32 len,
+			     u32 *response_buf, u32 response_buf_size)
 {
-	struct intel_guc_ct_channel *ctch = &guc->ct.host_channel;
+	struct intel_guc_ct *ct = &guc->ct;
+	struct intel_guc_ct_channel *ctch = &ct->host_channel;
 	u32 status = ~0; /* undefined */
-	int err;
+	int ret;
 
 	mutex_lock(&guc->send_mutex);
 
-	err = ctch_send(guc, ctch, action, len, &status);
-	if (unlikely(err)) {
+	ret = ctch_send(ct, ctch, action, len, response_buf, response_buf_size,
+			&status);
+	if (unlikely(ret < 0)) {
 		DRM_ERROR("CT: send action %#X failed; err=%d status=%#X\n",
-			  action[0], err, status);
+			  action[0], ret, status);
+	} else if (unlikely(ret)) {
+		CT_DEBUG_DRIVER("CT: send action %#x returned %d (%#x)\n",
+				action[0], ret, ret);
 	}
 
 	mutex_unlock(&guc->send_mutex);
-	return err;
+	return ret;
+}
+
+static inline unsigned int ct_header_get_len(u32 header)
+{
+	return (header >> GUC_CT_MSG_LEN_SHIFT) & GUC_CT_MSG_LEN_MASK;
+}
+
+static inline unsigned int ct_header_get_action(u32 header)
+{
+	return (header >> GUC_CT_MSG_ACTION_SHIFT) & GUC_CT_MSG_ACTION_MASK;
+}
+
+static inline bool ct_header_is_response(u32 header)
+{
+	return ct_header_get_action(header) == INTEL_GUC_ACTION_DEFAULT;
+}
+
+static int ctb_read(struct intel_guc_ct_buffer *ctb, u32 *data)
+{
+	struct guc_ct_buffer_desc *desc = ctb->desc;
+	u32 head = desc->head / 4;	/* in dwords */
+	u32 tail = desc->tail / 4;	/* in dwords */
+	u32 size = desc->size / 4;	/* in dwords */
+	u32 *cmds = ctb->cmds;
+	s32 available;			/* in dwords */
+	unsigned int len;
+	unsigned int i;
+
+	GEM_BUG_ON(desc->size % 4);
+	GEM_BUG_ON(desc->head % 4);
+	GEM_BUG_ON(desc->tail % 4);
+	GEM_BUG_ON(tail >= size);
+	GEM_BUG_ON(head >= size);
+
+	/* tail == head condition indicates empty */
+	available = tail - head;
+	if (unlikely(available == 0))
+		return -ENODATA;
+
+	/* beware of buffer wrap case */
+	if (unlikely(available < 0))
+		available += size;
+	CT_DEBUG_DRIVER("CT: available %d (%u:%u)\n", available, head, tail);
+	GEM_BUG_ON(available < 0);
+
+	data[0] = cmds[head];
+	head = (head + 1) % size;
+
+	/* message len with header */
+	len = ct_header_get_len(data[0]) + 1;
+	if (unlikely(len > (u32)available)) {
+		DRM_ERROR("CT: incomplete message %*ph %*ph %*ph\n",
+			  4, data,
+			  4 * (head + available - 1 > size ?
+			       size - head : available - 1), &cmds[head],
+			  4 * (head + available - 1 > size ?
+			       available - 1 - size + head : 0), &cmds[0]);
+		return -EPROTO;
+	}
+
+	for (i = 1; i < len; i++) {
+		data[i] = cmds[head];
+		head = (head + 1) % size;
+	}
+	CT_DEBUG_DRIVER("CT: received %*ph\n", 4 * len, data);
+
+	desc->head = head * 4;
+	return 0;
 }
 
 /**
- * Enable buffer based command transport
+ * DOC: CTB GuC to Host response
+ *
+ * Format of the CTB GuC to Host response message is as follows::
+ *
+ *      +------------+---------+---------+---------+---------+---------+
+ *      |   msg[0]   |   [1]   |   [2]   |   [3]   |   ...   |  [n-1]  |
+ *      +------------+---------+---------+---------+---------+---------+
+ *      |   MESSAGE  |       MESSAGE PAYLOAD                           |
+ *      +   HEADER   +---------+---------+---------+---------+---------+
+ *      |            |    0    |    1    |    2    |   ...   |    n    |
+ *      +============+=========+=========+=========+=========+=========+
+ *      |  len >= 2  |  FENCE  |  STATUS |   response specific data    |
+ *      +------+-----+---------+---------+---------+---------+---------+
+ *
+ *                   ^-----------------------len-----------------------^
+ */
+
+static int ct_handle_response(struct intel_guc_ct *ct, const u32 *msg)
+{
+	u32 header = msg[0];
+	u32 len = ct_header_get_len(header);
+	u32 msglen = len + 1; /* total message length including header */
+	u32 fence;
+	u32 status;
+	u32 datalen;
+	struct ct_request *req;
+	bool found = false;
+
+	GEM_BUG_ON(!ct_header_is_response(header));
+	GEM_BUG_ON(!in_irq());
+
+	/* Response payload shall at least include fence and status */
+	if (unlikely(len < 2)) {
+		DRM_ERROR("CT: corrupted response %*ph\n", 4 * msglen, msg);
+		return -EPROTO;
+	}
+
+	fence = msg[1];
+	status = msg[2];
+	datalen = len - 2;
+
+	/* Format of the status follows RESPONSE message */
+	if (unlikely(!INTEL_GUC_MSG_IS_RESPONSE(status))) {
+		DRM_ERROR("CT: corrupted response %*ph\n", 4 * msglen, msg);
+		return -EPROTO;
+	}
+
+	CT_DEBUG_DRIVER("CT: response fence %u status %#x\n", fence, status);
+
+	spin_lock(&ct->lock);
+	list_for_each_entry(req, &ct->pending_requests, link) {
+		if (unlikely(fence != req->fence)) {
+			CT_DEBUG_DRIVER("CT: request %u awaits response\n",
+					req->fence);
+			continue;
+		}
+		if (unlikely(datalen > req->response_len)) {
+			DRM_ERROR("CT: response %u too long %*ph\n",
+				  req->fence, 4 * msglen, msg);
+			datalen = 0;
+		}
+		if (datalen)
+			memcpy(req->response_buf, msg + 3, 4 * datalen);
+		req->response_len = datalen;
+		WRITE_ONCE(req->status, status);
+		found = true;
+		break;
+	}
+	spin_unlock(&ct->lock);
+
+	if (!found)
+		DRM_ERROR("CT: unsolicited response %*ph\n", 4 * msglen, msg);
+	return 0;
+}
+
+static void ct_process_request(struct intel_guc_ct *ct,
+			       u32 action, u32 len, const u32 *payload)
+{
+	struct intel_guc *guc = ct_to_guc(ct);
+
+	CT_DEBUG_DRIVER("CT: request %x %*ph\n", action, 4 * len, payload);
+
+	switch (action) {
+	case INTEL_GUC_ACTION_DEFAULT:
+		if (unlikely(len < 1))
+			goto fail_unexpected;
+		intel_guc_to_host_process_recv_msg(guc, *payload);
+		break;
+
+	default:
+fail_unexpected:
+		DRM_ERROR("CT: unexpected request %x %*ph\n",
+			  action, 4 * len, payload);
+		break;
+	}
+}
+
+static bool ct_process_incoming_requests(struct intel_guc_ct *ct)
+{
+	unsigned long flags;
+	struct ct_incoming_request *request;
+	u32 header;
+	u32 *payload;
+	bool done;
+
+	spin_lock_irqsave(&ct->lock, flags);
+	request = list_first_entry_or_null(&ct->incoming_requests,
+					   struct ct_incoming_request, link);
+	if (request)
+		list_del(&request->link);
+	done = !!list_empty(&ct->incoming_requests);
+	spin_unlock_irqrestore(&ct->lock, flags);
+
+	if (!request)
+		return true;
+
+	header = request->msg[0];
+	payload = &request->msg[1];
+	ct_process_request(ct,
+			   ct_header_get_action(header),
+			   ct_header_get_len(header),
+			   payload);
+
+	kfree(request);
+	return done;
+}
+
+static void ct_incoming_request_worker_func(struct work_struct *w)
+{
+	struct intel_guc_ct *ct = container_of(w, struct intel_guc_ct, worker);
+	bool done;
+
+	done = ct_process_incoming_requests(ct);
+	if (!done)
+		queue_work(system_unbound_wq, &ct->worker);
+}
+
+/**
+ * DOC: CTB GuC to Host request
+ *
+ * Format of the CTB GuC to Host request message is as follows::
+ *
+ *      +------------+---------+---------+---------+---------+---------+
+ *      |   msg[0]   |   [1]   |   [2]   |   [3]   |   ...   |  [n-1]  |
+ *      +------------+---------+---------+---------+---------+---------+
+ *      |   MESSAGE  |       MESSAGE PAYLOAD                           |
+ *      +   HEADER   +---------+---------+---------+---------+---------+
+ *      |            |    0    |    1    |    2    |   ...   |    n    |
+ *      +============+=========+=========+=========+=========+=========+
+ *      |     len    |            request specific data                |
+ *      +------+-----+---------+---------+---------+---------+---------+
+ *
+ *                   ^-----------------------len-----------------------^
+ */
+
+static int ct_handle_request(struct intel_guc_ct *ct, const u32 *msg)
+{
+	u32 header = msg[0];
+	u32 len = ct_header_get_len(header);
+	u32 msglen = len + 1; /* total message length including header */
+	struct ct_incoming_request *request;
+	unsigned long flags;
+
+	GEM_BUG_ON(ct_header_is_response(header));
+
+	request = kmalloc(sizeof(*request) + 4 * msglen, GFP_ATOMIC);
+	if (unlikely(!request)) {
+		DRM_ERROR("CT: dropping request %*ph\n", 4 * msglen, msg);
+		return 0; /* XXX: -ENOMEM ? */
+	}
+	memcpy(request->msg, msg, 4 * msglen);
+
+	spin_lock_irqsave(&ct->lock, flags);
+	list_add_tail(&request->link, &ct->incoming_requests);
+	spin_unlock_irqrestore(&ct->lock, flags);
+
+	queue_work(system_unbound_wq, &ct->worker);
+	return 0;
+}
+
+static void ct_process_host_channel(struct intel_guc_ct *ct)
+{
+	struct intel_guc_ct_channel *ctch = &ct->host_channel;
+	struct intel_guc_ct_buffer *ctb = &ctch->ctbs[CTB_RECV];
+	u32 msg[GUC_CT_MSG_LEN_MASK + 1]; /* one extra dw for the header */
+	int err = 0;
+
+	if (!ctch_is_open(ctch))
+		return;
+
+	do {
+		err = ctb_read(ctb, msg);
+		if (err)
+			break;
+
+		if (ct_header_is_response(msg[0]))
+			err = ct_handle_response(ct, msg);
+		else
+			err = ct_handle_request(ct, msg);
+	} while (!err);
+
+	if (GEM_WARN_ON(err == -EPROTO)) {
+		DRM_ERROR("CT: corrupted message detected!\n");
+		ctb->desc->is_in_error = 1;
+	}
+}
+
+/*
+ * When we're communicating with the GuC over CT, GuC uses events
+ * to notify us about new messages being posted on the RECV buffer.
+ */
+static void intel_guc_to_host_event_handler_ct(struct intel_guc *guc)
+{
+	struct intel_guc_ct *ct = &guc->ct;
+
+	ct_process_host_channel(ct);
+}
+
+/**
+ * intel_guc_ct_enable - Enable buffer based command transport.
+ * @ct: pointer to CT struct
+ *
  * Shall only be called for platforms with HAS_GUC_CT.
- * @guc:	the guc
- * return:	0 on success
- *		non-zero on failure
+ *
+ * Return: 0 on success, a negative errno code on failure.
  */
-int intel_guc_enable_ct(struct intel_guc *guc)
+int intel_guc_ct_enable(struct intel_guc_ct *ct)
 {
-	struct drm_i915_private *dev_priv = guc_to_i915(guc);
-	struct intel_guc_ct_channel *ctch = &guc->ct.host_channel;
+	struct intel_guc *guc = ct_to_guc(ct);
+	struct drm_i915_private *i915 = guc_to_i915(guc);
+	struct intel_guc_ct_channel *ctch = &ct->host_channel;
 	int err;
 
-	GEM_BUG_ON(!HAS_GUC_CT(dev_priv));
+	GEM_BUG_ON(!HAS_GUC_CT(i915));
 
 	err = ctch_open(guc, ctch);
 	if (unlikely(err))
@@ -435,21 +874,24 @@ int intel_guc_enable_ct(struct intel_guc *guc)
 
 	/* Switch into cmd transport buffer based send() */
 	guc->send = intel_guc_send_ct;
+	guc->handler = intel_guc_to_host_event_handler_ct;
 	DRM_INFO("CT: %s\n", enableddisabled(true));
 	return 0;
 }
 
 /**
- * Disable buffer based command transport.
+ * intel_guc_ct_disable - Disable buffer based command transport.
+ * @ct: pointer to CT struct
+ *
  * Shall only be called for platforms with HAS_GUC_CT.
- * @guc: the guc
  */
-void intel_guc_disable_ct(struct intel_guc *guc)
+void intel_guc_ct_disable(struct intel_guc_ct *ct)
 {
-	struct drm_i915_private *dev_priv = guc_to_i915(guc);
-	struct intel_guc_ct_channel *ctch = &guc->ct.host_channel;
+	struct intel_guc *guc = ct_to_guc(ct);
+	struct drm_i915_private *i915 = guc_to_i915(guc);
+	struct intel_guc_ct_channel *ctch = &ct->host_channel;
 
-	GEM_BUG_ON(!HAS_GUC_CT(dev_priv));
+	GEM_BUG_ON(!HAS_GUC_CT(i915));
 
 	if (!ctch_is_open(ctch))
 		return;
@@ -458,5 +900,6 @@ void intel_guc_disable_ct(struct intel_guc *guc)
 
 	/* Disable send */
 	guc->send = intel_guc_send_nop;
+	guc->handler = intel_guc_to_host_event_handler_nop;
 	DRM_INFO("CT: %s\n", enableddisabled(false));
 }
diff --git a/drivers/gpu/drm/i915/intel_guc_ct.h b/drivers/gpu/drm/i915/intel_guc_ct.h
index 6d97f36fcc62..d774895ab143 100644
--- a/drivers/gpu/drm/i915/intel_guc_ct.h
+++ b/drivers/gpu/drm/i915/intel_guc_ct.h
@@ -75,12 +75,22 @@ struct intel_guc_ct_channel {
 struct intel_guc_ct {
 	struct intel_guc_ct_channel host_channel;
 	/* other channels are tbd */
+
+	/** @lock: protects pending requests list */
+	spinlock_t lock;
+
+	/** @pending_requests: list of requests waiting for response */
+	struct list_head pending_requests;
+
+	/** @incoming_requests: list of incoming requests */
+	struct list_head incoming_requests;
+
+	/** @worker: worker for handling incoming requests */
+	struct work_struct worker;
 };
 
 void intel_guc_ct_init_early(struct intel_guc_ct *ct);
-
-/* XXX: move to intel_uc.h ? don't fit there either */
-int intel_guc_enable_ct(struct intel_guc *guc);
-void intel_guc_disable_ct(struct intel_guc *guc);
+int intel_guc_ct_enable(struct intel_guc_ct *ct);
+void intel_guc_ct_disable(struct intel_guc_ct *ct);
 
 #endif /* _INTEL_GUC_CT_H_ */
diff --git a/drivers/gpu/drm/i915/intel_guc_fw.c b/drivers/gpu/drm/i915/intel_guc_fw.c
index d07f2b985f1c..a9e6fcce467c 100644
--- a/drivers/gpu/drm/i915/intel_guc_fw.c
+++ b/drivers/gpu/drm/i915/intel_guc_fw.c
@@ -165,7 +165,7 @@ static int guc_xfer_ucode(struct intel_guc *guc, struct i915_vma *vma)
 	I915_WRITE(DMA_COPY_SIZE, guc_fw->header_size + guc_fw->ucode_size);
 
 	/* Set the source address for the new blob */
-	offset = guc_ggtt_offset(vma) + guc_fw->header_offset;
+	offset = intel_guc_ggtt_offset(guc, vma) + guc_fw->header_offset;
 	I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset));
 	I915_WRITE(DMA_ADDR_0_HIGH, upper_32_bits(offset) & 0xFFFF);
 
@@ -275,9 +275,8 @@ static int guc_fw_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
  * Called from intel_uc_init_hw() during driver load, resume from sleep and
  * after a GPU reset.
  *
- * The firmware image should have already been fetched into memory by the
- * earlier call to intel_uc_init_fw(), so here we need to only check that
- * fetch succeeded, and then transfer the image to the h/w.
+ * The firmware image should have already been fetched into memory, so only
+ * check that fetch succeeded, and then transfer the image to the h/w.
  *
  * Return:	non-zero code on error
  */
diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h
index 6a10aa6f04d3..0867ba76d445 100644
--- a/drivers/gpu/drm/i915/intel_guc_fwif.h
+++ b/drivers/gpu/drm/i915/intel_guc_fwif.h
@@ -23,9 +23,6 @@
 #ifndef _INTEL_GUC_FWIF_H
 #define _INTEL_GUC_FWIF_H
 
-#define GUC_CORE_FAMILY_GEN9		12
-#define GUC_CORE_FAMILY_UNKNOWN		0x7fffffff
-
 #define GUC_CLIENT_PRIORITY_KMD_HIGH	0
 #define GUC_CLIENT_PRIORITY_HIGH	1
 #define GUC_CLIENT_PRIORITY_KMD_NORMAL	2
@@ -82,8 +79,6 @@
 #define GUC_CTL_ARAT_LOW		2
 
 #define GUC_CTL_DEVICE_INFO		3
-#define   GUC_CTL_GT_TYPE_SHIFT		0
-#define   GUC_CTL_CORE_FAMILY_SHIFT	7
 
 #define GUC_CTL_LOG_PARAMS		4
 #define   GUC_LOG_VALID			(1 << 0)
@@ -127,7 +122,7 @@
 #define   GUC_PROFILE_ENABLED		(1 << 7)
 #define   GUC_WQ_TRACK_ENABLED		(1 << 8)
 #define   GUC_ADS_ENABLED		(1 << 9)
-#define   GUC_DEBUG_RESERVED		(1 << 10)
+#define   GUC_LOG_DEFAULT_DISABLED	(1 << 10)
 #define   GUC_ADS_ADDR_SHIFT		11
 #define   GUC_ADS_ADDR_MASK		0xfffff800
 
@@ -327,6 +322,58 @@ struct guc_stage_desc {
 	u64 desc_private;
 } __packed;
 
+/**
+ * DOC: CTB based communication
+ *
+ * The CTB (command transport buffer) communication between Host and GuC
+ * is based on u32 data stream written to the shared buffer. One buffer can
+ * be used to transmit data only in one direction (one-directional channel).
+ *
+ * Current status of the each buffer is stored in the buffer descriptor.
+ * Buffer descriptor holds tail and head fields that represents active data
+ * stream. The tail field is updated by the data producer (sender), and head
+ * field is updated by the data consumer (receiver)::
+ *
+ *      +------------+
+ *      | DESCRIPTOR |          +=================+============+========+
+ *      +============+          |                 | MESSAGE(s) |        |
+ *      | address    |--------->+=================+============+========+
+ *      +------------+
+ *      | head       |          ^-----head--------^
+ *      +------------+
+ *      | tail       |          ^---------tail-----------------^
+ *      +------------+
+ *      | size       |          ^---------------size--------------------^
+ *      +------------+
+ *
+ * Each message in data stream starts with the single u32 treated as a header,
+ * followed by optional set of u32 data that makes message specific payload::
+ *
+ *      +------------+---------+---------+---------+
+ *      |         MESSAGE                          |
+ *      +------------+---------+---------+---------+
+ *      |   msg[0]   |   [1]   |   ...   |  [n-1]  |
+ *      +------------+---------+---------+---------+
+ *      |   MESSAGE  |       MESSAGE PAYLOAD       |
+ *      +   HEADER   +---------+---------+---------+
+ *      |            |    0    |   ...   |    n    |
+ *      +======+=====+=========+=========+=========+
+ *      | 31:16| code|         |         |         |
+ *      +------+-----+         |         |         |
+ *      |  15:5|flags|         |         |         |
+ *      +------+-----+         |         |         |
+ *      |   4:0|  len|         |         |         |
+ *      +------+-----+---------+---------+---------+
+ *
+ *                   ^-------------len-------------^
+ *
+ * The message header consists of:
+ *
+ * - **len**, indicates length of the message payload (in u32)
+ * - **code**, indicates message code
+ * - **flags**, holds various bits to control message handling
+ */
+
 /*
  * Describes single command transport buffer.
  * Used by both guc-master and clients.
@@ -534,16 +581,6 @@ struct guc_log_buffer_state {
 	u32 version;
 } __packed;
 
-union guc_log_control {
-	struct {
-		u32 logging_enabled:1;
-		u32 reserved1:3;
-		u32 verbosity:4;
-		u32 reserved2:24;
-	};
-	u32 value;
-} __packed;
-
 struct guc_ctx_report {
 	u32 report_return_status;
 	u32 reserved1[64];
@@ -570,7 +607,68 @@ struct guc_shared_ctx_data {
 	struct guc_ctx_report preempt_ctx_report[GUC_MAX_ENGINES_NUM];
 } __packed;
 
-/* This Action will be programmed in C180 - SOFT_SCRATCH_O_REG */
+/**
+ * DOC: MMIO based communication
+ *
+ * The MMIO based communication between Host and GuC uses software scratch
+ * registers, where first register holds data treated as message header,
+ * and other registers are used to hold message payload.
+ *
+ * For Gen9+, GuC uses software scratch registers 0xC180-0xC1B8
+ *
+ *      +-----------+---------+---------+---------+
+ *      |  MMIO[0]  | MMIO[1] |   ...   | MMIO[n] |
+ *      +-----------+---------+---------+---------+
+ *      | header    |      optional payload       |
+ *      +======+====+=========+=========+=========+
+ *      | 31:28|type|         |         |         |
+ *      +------+----+         |         |         |
+ *      | 27:16|data|         |         |         |
+ *      +------+----+         |         |         |
+ *      |  15:0|code|         |         |         |
+ *      +------+----+---------+---------+---------+
+ *
+ * The message header consists of:
+ *
+ * - **type**, indicates message type
+ * - **code**, indicates message code, is specific for **type**
+ * - **data**, indicates message data, optional, depends on **code**
+ *
+ * The following message **types** are supported:
+ *
+ * - **REQUEST**, indicates Host-to-GuC request, requested GuC action code
+ *   must be priovided in **code** field. Optional action specific parameters
+ *   can be provided in remaining payload registers or **data** field.
+ *
+ * - **RESPONSE**, indicates GuC-to-Host response from earlier GuC request,
+ *   action response status will be provided in **code** field. Optional
+ *   response data can be returned in remaining payload registers or **data**
+ *   field.
+ */
+
+#define INTEL_GUC_MSG_TYPE_SHIFT	28
+#define INTEL_GUC_MSG_TYPE_MASK		(0xF << INTEL_GUC_MSG_TYPE_SHIFT)
+#define INTEL_GUC_MSG_DATA_SHIFT	16
+#define INTEL_GUC_MSG_DATA_MASK		(0xFFF << INTEL_GUC_MSG_DATA_SHIFT)
+#define INTEL_GUC_MSG_CODE_SHIFT	0
+#define INTEL_GUC_MSG_CODE_MASK		(0xFFFF << INTEL_GUC_MSG_CODE_SHIFT)
+
+#define __INTEL_GUC_MSG_GET(T, m) \
+	(((m) & INTEL_GUC_MSG_ ## T ## _MASK) >> INTEL_GUC_MSG_ ## T ## _SHIFT)
+#define INTEL_GUC_MSG_TO_TYPE(m)	__INTEL_GUC_MSG_GET(TYPE, m)
+#define INTEL_GUC_MSG_TO_DATA(m)	__INTEL_GUC_MSG_GET(DATA, m)
+#define INTEL_GUC_MSG_TO_CODE(m)	__INTEL_GUC_MSG_GET(CODE, m)
+
+enum intel_guc_msg_type {
+	INTEL_GUC_MSG_TYPE_REQUEST = 0x0,
+	INTEL_GUC_MSG_TYPE_RESPONSE = 0xF,
+};
+
+#define __INTEL_GUC_MSG_TYPE_IS(T, m) \
+	(INTEL_GUC_MSG_TO_TYPE(m) == INTEL_GUC_MSG_TYPE_ ## T)
+#define INTEL_GUC_MSG_IS_REQUEST(m)	__INTEL_GUC_MSG_TYPE_IS(REQUEST, m)
+#define INTEL_GUC_MSG_IS_RESPONSE(m)	__INTEL_GUC_MSG_TYPE_IS(RESPONSE, m)
+
 enum intel_guc_action {
 	INTEL_GUC_ACTION_DEFAULT = 0x0,
 	INTEL_GUC_ACTION_REQUEST_PREEMPTION = 0x2,
@@ -602,24 +700,22 @@ enum intel_guc_report_status {
 	INTEL_GUC_REPORT_STATUS_COMPLETE = 0x4,
 };
 
-/*
- * The GuC sends its response to a command by overwriting the
- * command in SS0. The response is distinguishable from a command
- * by the fact that all the MASK bits are set. The remaining bits
- * give more detail.
- */
-#define	INTEL_GUC_RECV_MASK	((u32)0xF0000000)
-#define	INTEL_GUC_RECV_IS_RESPONSE(x)	((u32)(x) >= INTEL_GUC_RECV_MASK)
-#define	INTEL_GUC_RECV_STATUS(x)	(INTEL_GUC_RECV_MASK | (x))
-
-/* GUC will return status back to SOFT_SCRATCH_O_REG */
-enum intel_guc_status {
-	INTEL_GUC_STATUS_SUCCESS = INTEL_GUC_RECV_STATUS(0x0),
-	INTEL_GUC_STATUS_ALLOCATE_DOORBELL_FAIL = INTEL_GUC_RECV_STATUS(0x10),
-	INTEL_GUC_STATUS_DEALLOCATE_DOORBELL_FAIL = INTEL_GUC_RECV_STATUS(0x20),
-	INTEL_GUC_STATUS_GENERIC_FAIL = INTEL_GUC_RECV_STATUS(0x0000F000)
+#define GUC_LOG_CONTROL_LOGGING_ENABLED	(1 << 0)
+#define GUC_LOG_CONTROL_VERBOSITY_SHIFT	4
+#define GUC_LOG_CONTROL_VERBOSITY_MASK	(0xF << GUC_LOG_CONTROL_VERBOSITY_SHIFT)
+#define GUC_LOG_CONTROL_DEFAULT_LOGGING	(1 << 8)
+
+enum intel_guc_response_status {
+	INTEL_GUC_RESPONSE_STATUS_SUCCESS = 0x0,
+	INTEL_GUC_RESPONSE_STATUS_GENERIC_FAIL = 0xF000,
 };
 
+#define INTEL_GUC_MSG_IS_RESPONSE_SUCCESS(m) \
+	 (typecheck(u32, (m)) && \
+	  ((m) & (INTEL_GUC_MSG_TYPE_MASK | INTEL_GUC_MSG_CODE_MASK)) == \
+	  ((INTEL_GUC_MSG_TYPE_RESPONSE << INTEL_GUC_MSG_TYPE_SHIFT) | \
+	   (INTEL_GUC_RESPONSE_STATUS_SUCCESS << INTEL_GUC_MSG_CODE_SHIFT)))
+
 /* This action will be programmed in C1BC - SOFT_SCRATCH_15_REG */
 enum intel_guc_recv_message {
 	INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED = BIT(1),
diff --git a/drivers/gpu/drm/i915/intel_guc_log.c b/drivers/gpu/drm/i915/intel_guc_log.c
index c0c2e7d1c7d7..401e1704d61e 100644
--- a/drivers/gpu/drm/i915/intel_guc_log.c
+++ b/drivers/gpu/drm/i915/intel_guc_log.c
@@ -23,12 +23,11 @@
  */
 
 #include <linux/debugfs.h>
-#include <linux/relay.h>
 
 #include "intel_guc_log.h"
 #include "i915_drv.h"
 
-static void guc_log_capture_logs(struct intel_guc *guc);
+static void guc_log_capture_logs(struct intel_guc_log *log);
 
 /**
  * DOC: GuC firmware log
@@ -39,7 +38,7 @@ static void guc_log_capture_logs(struct intel_guc *guc);
  * registers value.
  */
 
-static int guc_log_flush_complete(struct intel_guc *guc)
+static int guc_action_flush_log_complete(struct intel_guc *guc)
 {
 	u32 action[] = {
 		INTEL_GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE
@@ -48,7 +47,7 @@ static int guc_log_flush_complete(struct intel_guc *guc)
 	return intel_guc_send(guc, action, ARRAY_SIZE(action));
 }
 
-static int guc_log_flush(struct intel_guc *guc)
+static int guc_action_flush_log(struct intel_guc *guc)
 {
 	u32 action[] = {
 		INTEL_GUC_ACTION_FORCE_LOG_BUFFER_FLUSH,
@@ -58,22 +57,40 @@ static int guc_log_flush(struct intel_guc *guc)
 	return intel_guc_send(guc, action, ARRAY_SIZE(action));
 }
 
-static int guc_log_control(struct intel_guc *guc, bool enable, u32 verbosity)
+static int guc_action_control_log(struct intel_guc *guc, bool enable,
+				  bool default_logging, u32 verbosity)
 {
-	union guc_log_control control_val = {
-		{
-			.logging_enabled = enable,
-			.verbosity = verbosity,
-		},
-	};
 	u32 action[] = {
 		INTEL_GUC_ACTION_UK_LOG_ENABLE_LOGGING,
-		control_val.value
+		(enable ? GUC_LOG_CONTROL_LOGGING_ENABLED : 0) |
+		(verbosity << GUC_LOG_CONTROL_VERBOSITY_SHIFT) |
+		(default_logging ? GUC_LOG_CONTROL_DEFAULT_LOGGING : 0)
 	};
 
+	GEM_BUG_ON(verbosity > GUC_LOG_VERBOSITY_MAX);
+
 	return intel_guc_send(guc, action, ARRAY_SIZE(action));
 }
 
+static inline struct intel_guc *log_to_guc(struct intel_guc_log *log)
+{
+	return container_of(log, struct intel_guc, log);
+}
+
+static void guc_log_enable_flush_events(struct intel_guc_log *log)
+{
+	intel_guc_enable_msg(log_to_guc(log),
+			     INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER |
+			     INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED);
+}
+
+static void guc_log_disable_flush_events(struct intel_guc_log *log)
+{
+	intel_guc_disable_msg(log_to_guc(log),
+			      INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER |
+			      INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED);
+}
+
 /*
  * Sub buffer switch callback. Called whenever relay has to switch to a new
  * sub buffer, relay stays on the same sub buffer if 0 is returned.
@@ -121,14 +138,7 @@ static struct dentry *create_buf_file_callback(const char *filename,
 	if (!parent)
 		return NULL;
 
-	/*
-	 * Not using the channel filename passed as an argument, since for each
-	 * channel relay appends the corresponding CPU number to the filename
-	 * passed in relay_open(). This should be fine as relay just needs a
-	 * dentry of the file associated with the channel buffer and that file's
-	 * name need not be same as the filename passed as an argument.
-	 */
-	buf_file = debugfs_create_file("guc_log", mode,
+	buf_file = debugfs_create_file(filename, mode,
 				       parent, buf, &relay_file_operations);
 	return buf_file;
 }
@@ -149,59 +159,7 @@ static struct rchan_callbacks relay_callbacks = {
 	.remove_buf_file = remove_buf_file_callback,
 };
 
-static int guc_log_relay_file_create(struct intel_guc *guc)
-{
-	struct drm_i915_private *dev_priv = guc_to_i915(guc);
-	struct dentry *log_dir;
-	int ret;
-
-	if (!i915_modparams.guc_log_level)
-		return 0;
-
-	mutex_lock(&guc->log.runtime.relay_lock);
-
-	/* For now create the log file in /sys/kernel/debug/dri/0 dir */
-	log_dir = dev_priv->drm.primary->debugfs_root;
-
-	/*
-	 * If /sys/kernel/debug/dri/0 location do not exist, then debugfs is
-	 * not mounted and so can't create the relay file.
-	 * The relay API seems to fit well with debugfs only, for availing relay
-	 * there are 3 requirements which can be met for debugfs file only in a
-	 * straightforward/clean manner :-
-	 * i)   Need the associated dentry pointer of the file, while opening the
-	 *      relay channel.
-	 * ii)  Should be able to use 'relay_file_operations' fops for the file.
-	 * iii) Set the 'i_private' field of file's inode to the pointer of
-	 *	relay channel buffer.
-	 */
-	if (!log_dir) {
-		DRM_ERROR("Debugfs dir not available yet for GuC log file\n");
-		ret = -ENODEV;
-		goto out_unlock;
-	}
-
-	ret = relay_late_setup_files(guc->log.runtime.relay_chan, "guc_log", log_dir);
-	if (ret < 0 && ret != -EEXIST) {
-		DRM_ERROR("Couldn't associate relay chan with file %d\n", ret);
-		goto out_unlock;
-	}
-
-	ret = 0;
-
-out_unlock:
-	mutex_unlock(&guc->log.runtime.relay_lock);
-	return ret;
-}
-
-static bool guc_log_has_relay(struct intel_guc *guc)
-{
-	lockdep_assert_held(&guc->log.runtime.relay_lock);
-
-	return guc->log.runtime.relay_chan != NULL;
-}
-
-static void guc_move_to_next_buf(struct intel_guc *guc)
+static void guc_move_to_next_buf(struct intel_guc_log *log)
 {
 	/*
 	 * Make sure the updates made in the sub buffer are visible when
@@ -209,21 +167,15 @@ static void guc_move_to_next_buf(struct intel_guc *guc)
 	 */
 	smp_wmb();
 
-	if (!guc_log_has_relay(guc))
-		return;
-
 	/* All data has been written, so now move the offset of sub buffer. */
-	relay_reserve(guc->log.runtime.relay_chan, guc->log.vma->obj->base.size);
+	relay_reserve(log->relay.channel, log->vma->obj->base.size);
 
 	/* Switch to the next sub buffer */
-	relay_flush(guc->log.runtime.relay_chan);
+	relay_flush(log->relay.channel);
 }
 
-static void *guc_get_write_buffer(struct intel_guc *guc)
+static void *guc_get_write_buffer(struct intel_guc_log *log)
 {
-	if (!guc_log_has_relay(guc))
-		return NULL;
-
 	/*
 	 * Just get the base address of a new sub buffer and copy data into it
 	 * ourselves. NULL will be returned in no-overwrite mode, if all sub
@@ -233,25 +185,25 @@ static void *guc_get_write_buffer(struct intel_guc *guc)
 	 * done without using relay_reserve() along with relay_write(). So its
 	 * better to use relay_reserve() alone.
 	 */
-	return relay_reserve(guc->log.runtime.relay_chan, 0);
+	return relay_reserve(log->relay.channel, 0);
 }
 
-static bool guc_check_log_buf_overflow(struct intel_guc *guc,
+static bool guc_check_log_buf_overflow(struct intel_guc_log *log,
 				       enum guc_log_buffer_type type,
 				       unsigned int full_cnt)
 {
-	unsigned int prev_full_cnt = guc->log.prev_overflow_count[type];
+	unsigned int prev_full_cnt = log->stats[type].sampled_overflow;
 	bool overflow = false;
 
 	if (full_cnt != prev_full_cnt) {
 		overflow = true;
 
-		guc->log.prev_overflow_count[type] = full_cnt;
-		guc->log.total_overflow_count[type] += full_cnt - prev_full_cnt;
+		log->stats[type].overflow = full_cnt;
+		log->stats[type].sampled_overflow += full_cnt - prev_full_cnt;
 
 		if (full_cnt < prev_full_cnt) {
 			/* buffer_full_cnt is a 4 bit counter */
-			guc->log.total_overflow_count[type] += 16;
+			log->stats[type].sampled_overflow += 16;
 		}
 		DRM_ERROR_RATELIMITED("GuC log buffer overflow\n");
 	}
@@ -275,7 +227,7 @@ static unsigned int guc_get_log_buffer_size(enum guc_log_buffer_type type)
 	return 0;
 }
 
-static void guc_read_update_log_buffer(struct intel_guc *guc)
+static void guc_read_update_log_buffer(struct intel_guc_log *log)
 {
 	unsigned int buffer_size, read_offset, write_offset, bytes_to_copy, full_cnt;
 	struct guc_log_buffer_state *log_buf_state, *log_buf_snapshot_state;
@@ -284,16 +236,16 @@ static void guc_read_update_log_buffer(struct intel_guc *guc)
 	void *src_data, *dst_data;
 	bool new_overflow;
 
-	if (WARN_ON(!guc->log.runtime.buf_addr))
-		return;
+	mutex_lock(&log->relay.lock);
 
-	/* Get the pointer to shared GuC log buffer */
-	log_buf_state = src_data = guc->log.runtime.buf_addr;
+	if (WARN_ON(!intel_guc_log_relay_enabled(log)))
+		goto out_unlock;
 
-	mutex_lock(&guc->log.runtime.relay_lock);
+	/* Get the pointer to shared GuC log buffer */
+	log_buf_state = src_data = log->relay.buf_addr;
 
 	/* Get the pointer to local buffer to store the logs */
-	log_buf_snapshot_state = dst_data = guc_get_write_buffer(guc);
+	log_buf_snapshot_state = dst_data = guc_get_write_buffer(log);
 
 	if (unlikely(!log_buf_snapshot_state)) {
 		/*
@@ -301,10 +253,9 @@ static void guc_read_update_log_buffer(struct intel_guc *guc)
 		 * getting consumed by User at a slow rate.
 		 */
 		DRM_ERROR_RATELIMITED("no sub-buffer to capture logs\n");
-		guc->log.capture_miss_count++;
-		mutex_unlock(&guc->log.runtime.relay_lock);
+		log->relay.full_count++;
 
-		return;
+		goto out_unlock;
 	}
 
 	/* Actual logs are present from the 2nd page */
@@ -325,8 +276,8 @@ static void guc_read_update_log_buffer(struct intel_guc *guc)
 		full_cnt = log_buf_state_local.buffer_full_cnt;
 
 		/* Bookkeeping stuff */
-		guc->log.flush_count[type] += log_buf_state_local.flush_to_file;
-		new_overflow = guc_check_log_buf_overflow(guc, type, full_cnt);
+		log->stats[type].flush += log_buf_state_local.flush_to_file;
+		new_overflow = guc_check_log_buf_overflow(log, type, full_cnt);
 
 		/* Update the state of shared log buffer */
 		log_buf_state->read_ptr = write_offset;
@@ -373,38 +324,35 @@ static void guc_read_update_log_buffer(struct intel_guc *guc)
 		dst_data += buffer_size;
 	}
 
-	guc_move_to_next_buf(guc);
+	guc_move_to_next_buf(log);
 
-	mutex_unlock(&guc->log.runtime.relay_lock);
+out_unlock:
+	mutex_unlock(&log->relay.lock);
 }
 
 static void capture_logs_work(struct work_struct *work)
 {
-	struct intel_guc *guc =
-		container_of(work, struct intel_guc, log.runtime.flush_work);
-
-	guc_log_capture_logs(guc);
-}
+	struct intel_guc_log *log =
+		container_of(work, struct intel_guc_log, relay.flush_work);
 
-static bool guc_log_has_runtime(struct intel_guc *guc)
-{
-	return guc->log.runtime.buf_addr != NULL;
+	guc_log_capture_logs(log);
 }
 
-static int guc_log_runtime_create(struct intel_guc *guc)
+static int guc_log_map(struct intel_guc_log *log)
 {
+	struct intel_guc *guc = log_to_guc(log);
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 	void *vaddr;
 	int ret;
 
-	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+	lockdep_assert_held(&log->relay.lock);
 
-	if (!guc->log.vma)
+	if (!log->vma)
 		return -ENODEV;
 
-	GEM_BUG_ON(guc_log_has_runtime(guc));
-
-	ret = i915_gem_object_set_to_wc_domain(guc->log.vma->obj, true);
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	ret = i915_gem_object_set_to_wc_domain(log->vma->obj, true);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
 	if (ret)
 		return ret;
 
@@ -413,49 +361,40 @@ static int guc_log_runtime_create(struct intel_guc *guc)
 	 * buffer pages, so that we can directly get the data
 	 * (up-to-date) from memory.
 	 */
-	vaddr = i915_gem_object_pin_map(guc->log.vma->obj, I915_MAP_WC);
+	vaddr = i915_gem_object_pin_map(log->vma->obj, I915_MAP_WC);
 	if (IS_ERR(vaddr)) {
 		DRM_ERROR("Couldn't map log buffer pages %d\n", ret);
 		return PTR_ERR(vaddr);
 	}
 
-	guc->log.runtime.buf_addr = vaddr;
+	log->relay.buf_addr = vaddr;
 
 	return 0;
 }
 
-static void guc_log_runtime_destroy(struct intel_guc *guc)
+static void guc_log_unmap(struct intel_guc_log *log)
 {
-	/*
-	 * It's possible that the runtime stuff was never allocated because
-	 * GuC log was disabled at the boot time.
-	 */
-	if (!guc_log_has_runtime(guc))
-		return;
+	lockdep_assert_held(&log->relay.lock);
 
-	i915_gem_object_unpin_map(guc->log.vma->obj);
-	guc->log.runtime.buf_addr = NULL;
+	i915_gem_object_unpin_map(log->vma->obj);
+	log->relay.buf_addr = NULL;
 }
 
-void intel_guc_log_init_early(struct intel_guc *guc)
+void intel_guc_log_init_early(struct intel_guc_log *log)
 {
-	mutex_init(&guc->log.runtime.relay_lock);
-	INIT_WORK(&guc->log.runtime.flush_work, capture_logs_work);
+	mutex_init(&log->relay.lock);
+	INIT_WORK(&log->relay.flush_work, capture_logs_work);
 }
 
-int intel_guc_log_relay_create(struct intel_guc *guc)
+static int guc_log_relay_create(struct intel_guc_log *log)
 {
+	struct intel_guc *guc = log_to_guc(log);
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 	struct rchan *guc_log_relay_chan;
 	size_t n_subbufs, subbuf_size;
 	int ret;
 
-	if (!i915_modparams.guc_log_level)
-		return 0;
-
-	mutex_lock(&guc->log.runtime.relay_lock);
-
-	GEM_BUG_ON(guc_log_has_relay(guc));
+	lockdep_assert_held(&log->relay.lock);
 
 	 /* Keep the size of sub buffers same as shared log buffer */
 	subbuf_size = GUC_LOG_SIZE;
@@ -468,157 +407,56 @@ int intel_guc_log_relay_create(struct intel_guc *guc)
 	 */
 	n_subbufs = 8;
 
-	/*
-	 * Create a relay channel, so that we have buffers for storing
-	 * the GuC firmware logs, the channel will be linked with a file
-	 * later on when debugfs is registered.
-	 */
-	guc_log_relay_chan = relay_open(NULL, NULL, subbuf_size,
-					n_subbufs, &relay_callbacks, dev_priv);
+	guc_log_relay_chan = relay_open("guc_log",
+					dev_priv->drm.primary->debugfs_root,
+					subbuf_size, n_subbufs,
+					&relay_callbacks, dev_priv);
 	if (!guc_log_relay_chan) {
 		DRM_ERROR("Couldn't create relay chan for GuC logging\n");
 
 		ret = -ENOMEM;
-		goto err;
+		return ret;
 	}
 
 	GEM_BUG_ON(guc_log_relay_chan->subbuf_size < subbuf_size);
-	guc->log.runtime.relay_chan = guc_log_relay_chan;
-
-	mutex_unlock(&guc->log.runtime.relay_lock);
+	log->relay.channel = guc_log_relay_chan;
 
 	return 0;
-
-err:
-	mutex_unlock(&guc->log.runtime.relay_lock);
-	/* logging will be off */
-	i915_modparams.guc_log_level = 0;
-	return ret;
-}
-
-void intel_guc_log_relay_destroy(struct intel_guc *guc)
-{
-	mutex_lock(&guc->log.runtime.relay_lock);
-
-	/*
-	 * It's possible that the relay was never allocated because
-	 * GuC log was disabled at the boot time.
-	 */
-	if (!guc_log_has_relay(guc))
-		goto out_unlock;
-
-	relay_close(guc->log.runtime.relay_chan);
-	guc->log.runtime.relay_chan = NULL;
-
-out_unlock:
-	mutex_unlock(&guc->log.runtime.relay_lock);
 }
 
-static int guc_log_late_setup(struct intel_guc *guc)
+static void guc_log_relay_destroy(struct intel_guc_log *log)
 {
-	struct drm_i915_private *dev_priv = guc_to_i915(guc);
-	int ret;
-
-	if (!guc_log_has_runtime(guc)) {
-		/*
-		 * If log was disabled at boot time, then setup needed to handle
-		 * log buffer flush interrupts would not have been done yet, so
-		 * do that now.
-		 */
-		ret = intel_guc_log_relay_create(guc);
-		if (ret)
-			goto err;
-
-		mutex_lock(&dev_priv->drm.struct_mutex);
-		intel_runtime_pm_get(dev_priv);
-		ret = guc_log_runtime_create(guc);
-		intel_runtime_pm_put(dev_priv);
-		mutex_unlock(&dev_priv->drm.struct_mutex);
-
-		if (ret)
-			goto err_relay;
-	}
-
-	ret = guc_log_relay_file_create(guc);
-	if (ret)
-		goto err_runtime;
-
-	return 0;
+	lockdep_assert_held(&log->relay.lock);
 
-err_runtime:
-	mutex_lock(&dev_priv->drm.struct_mutex);
-	guc_log_runtime_destroy(guc);
-	mutex_unlock(&dev_priv->drm.struct_mutex);
-err_relay:
-	intel_guc_log_relay_destroy(guc);
-err:
-	/* logging will remain off */
-	i915_modparams.guc_log_level = 0;
-	return ret;
+	relay_close(log->relay.channel);
+	log->relay.channel = NULL;
 }
 
-static void guc_log_capture_logs(struct intel_guc *guc)
+static void guc_log_capture_logs(struct intel_guc_log *log)
 {
+	struct intel_guc *guc = log_to_guc(log);
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 
-	guc_read_update_log_buffer(guc);
+	guc_read_update_log_buffer(log);
 
 	/*
 	 * Generally device is expected to be active only at this
 	 * time, so get/put should be really quick.
 	 */
 	intel_runtime_pm_get(dev_priv);
-	guc_log_flush_complete(guc);
-	intel_runtime_pm_put(dev_priv);
-}
-
-static void guc_flush_logs(struct intel_guc *guc)
-{
-	struct drm_i915_private *dev_priv = guc_to_i915(guc);
-
-	if (!USES_GUC_SUBMISSION(dev_priv) || !i915_modparams.guc_log_level)
-		return;
-
-	/* First disable the interrupts, will be renabled afterwards */
-	mutex_lock(&dev_priv->drm.struct_mutex);
-	intel_runtime_pm_get(dev_priv);
-	gen9_disable_guc_interrupts(dev_priv);
-	intel_runtime_pm_put(dev_priv);
-	mutex_unlock(&dev_priv->drm.struct_mutex);
-
-	/*
-	 * Before initiating the forceful flush, wait for any pending/ongoing
-	 * flush to complete otherwise forceful flush may not actually happen.
-	 */
-	flush_work(&guc->log.runtime.flush_work);
-
-	/* Ask GuC to update the log buffer state */
-	intel_runtime_pm_get(dev_priv);
-	guc_log_flush(guc);
+	guc_action_flush_log_complete(guc);
 	intel_runtime_pm_put(dev_priv);
-
-	/* GuC would have updated log buffer by now, so capture it */
-	guc_log_capture_logs(guc);
 }
 
-int intel_guc_log_create(struct intel_guc *guc)
+int intel_guc_log_create(struct intel_guc_log *log)
 {
+	struct intel_guc *guc = log_to_guc(log);
 	struct i915_vma *vma;
 	unsigned long offset;
 	u32 flags;
 	int ret;
 
-	GEM_BUG_ON(guc->log.vma);
-
-	/*
-	 * We require SSE 4.1 for fast reads from the GuC log buffer and
-	 * it should be present on the chipsets supporting GuC based
-	 * submisssions.
-	 */
-	if (WARN_ON(!i915_has_memcpy_from_wc())) {
-		ret = -EINVAL;
-		goto err;
-	}
+	GEM_BUG_ON(log->vma);
 
 	vma = intel_guc_allocate_vma(guc, GUC_LOG_SIZE);
 	if (IS_ERR(vma)) {
@@ -626,13 +464,7 @@ int intel_guc_log_create(struct intel_guc *guc)
 		goto err;
 	}
 
-	guc->log.vma = vma;
-
-	if (i915_modparams.guc_log_level) {
-		ret = guc_log_runtime_create(guc);
-		if (ret < 0)
-			goto err_vma;
-	}
+	log->vma = vma;
 
 	/* each allocated unit is a page */
 	flags = GUC_LOG_VALID | GUC_LOG_NOTIFY_ON_HALF_FULL |
@@ -640,117 +472,159 @@ int intel_guc_log_create(struct intel_guc *guc)
 		(GUC_LOG_ISR_PAGES << GUC_LOG_ISR_SHIFT) |
 		(GUC_LOG_CRASH_PAGES << GUC_LOG_CRASH_SHIFT);
 
-	offset = guc_ggtt_offset(vma) >> PAGE_SHIFT; /* in pages */
-	guc->log.flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags;
+	offset = intel_guc_ggtt_offset(guc, vma) >> PAGE_SHIFT;
+	log->flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags;
 
 	return 0;
 
-err_vma:
-	i915_vma_unpin_and_release(&guc->log.vma);
 err:
 	/* logging will be off */
 	i915_modparams.guc_log_level = 0;
 	return ret;
 }
 
-void intel_guc_log_destroy(struct intel_guc *guc)
+void intel_guc_log_destroy(struct intel_guc_log *log)
+{
+	i915_vma_unpin_and_release(&log->vma);
+}
+
+int intel_guc_log_level_get(struct intel_guc_log *log)
 {
-	guc_log_runtime_destroy(guc);
-	i915_vma_unpin_and_release(&guc->log.vma);
+	GEM_BUG_ON(!log->vma);
+	GEM_BUG_ON(i915_modparams.guc_log_level < 0);
+
+	return i915_modparams.guc_log_level;
 }
 
-int intel_guc_log_control(struct intel_guc *guc, u64 control_val)
+int intel_guc_log_level_set(struct intel_guc_log *log, u64 val)
 {
+	struct intel_guc *guc = log_to_guc(log);
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
-	bool enable_logging = control_val > 0;
-	u32 verbosity;
 	int ret;
 
-	if (!guc->log.vma)
-		return -ENODEV;
+	BUILD_BUG_ON(GUC_LOG_VERBOSITY_MIN != 0);
+	GEM_BUG_ON(!log->vma);
+	GEM_BUG_ON(i915_modparams.guc_log_level < 0);
 
-	BUILD_BUG_ON(GUC_LOG_VERBOSITY_MIN);
-	if (control_val > 1 + GUC_LOG_VERBOSITY_MAX)
+	/*
+	 * GuC is recognizing log levels starting from 0 to max, we're using 0
+	 * as indication that logging should be disabled.
+	 */
+	if (val < GUC_LOG_LEVEL_DISABLED || val > GUC_LOG_LEVEL_MAX)
 		return -EINVAL;
 
-	/* This combination doesn't make sense & won't have any effect */
-	if (!enable_logging && !i915_modparams.guc_log_level)
-		return 0;
+	mutex_lock(&dev_priv->drm.struct_mutex);
 
-	verbosity = enable_logging ? control_val - 1 : 0;
+	if (i915_modparams.guc_log_level == val) {
+		ret = 0;
+		goto out_unlock;
+	}
 
-	ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
-	if (ret)
-		return ret;
 	intel_runtime_pm_get(dev_priv);
-	ret = guc_log_control(guc, enable_logging, verbosity);
+	ret = guc_action_control_log(guc, GUC_LOG_LEVEL_IS_VERBOSE(val),
+				     GUC_LOG_LEVEL_IS_ENABLED(val),
+				     GUC_LOG_LEVEL_TO_VERBOSITY(val));
 	intel_runtime_pm_put(dev_priv);
+	if (ret) {
+		DRM_DEBUG_DRIVER("guc_log_control action failed %d\n", ret);
+		goto out_unlock;
+	}
+
+	i915_modparams.guc_log_level = val;
+
+out_unlock:
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
-	if (ret < 0) {
-		DRM_DEBUG_DRIVER("guc_logging_control action failed %d\n", ret);
-		return ret;
-	}
+	return ret;
+}
 
-	if (enable_logging) {
-		i915_modparams.guc_log_level = 1 + verbosity;
+bool intel_guc_log_relay_enabled(const struct intel_guc_log *log)
+{
+	return log->relay.buf_addr;
+}
 
-		/*
-		 * If log was disabled at boot time, then the relay channel file
-		 * wouldn't have been created by now and interrupts also would
-		 * not have been enabled. Try again now, just in case.
-		 */
-		ret = guc_log_late_setup(guc);
-		if (ret < 0) {
-			DRM_DEBUG_DRIVER("GuC log late setup failed %d\n", ret);
-			return ret;
-		}
+int intel_guc_log_relay_open(struct intel_guc_log *log)
+{
+	int ret;
 
-		/* GuC logging is currently the only user of Guc2Host interrupts */
-		mutex_lock(&dev_priv->drm.struct_mutex);
-		intel_runtime_pm_get(dev_priv);
-		gen9_enable_guc_interrupts(dev_priv);
-		intel_runtime_pm_put(dev_priv);
-		mutex_unlock(&dev_priv->drm.struct_mutex);
-	} else {
-		/*
-		 * Once logging is disabled, GuC won't generate logs & send an
-		 * interrupt. But there could be some data in the log buffer
-		 * which is yet to be captured. So request GuC to update the log
-		 * buffer state and then collect the left over logs.
-		 */
-		guc_flush_logs(guc);
+	mutex_lock(&log->relay.lock);
 
-		/* As logging is disabled, update log level to reflect that */
-		i915_modparams.guc_log_level = 0;
+	if (intel_guc_log_relay_enabled(log)) {
+		ret = -EEXIST;
+		goto out_unlock;
 	}
 
-	return ret;
-}
+	/*
+	 * We require SSE 4.1 for fast reads from the GuC log buffer and
+	 * it should be present on the chipsets supporting GuC based
+	 * submisssions.
+	 */
+	if (!i915_has_memcpy_from_wc()) {
+		ret = -ENXIO;
+		goto out_unlock;
+	}
 
-void i915_guc_log_register(struct drm_i915_private *dev_priv)
-{
-	if (!USES_GUC_SUBMISSION(dev_priv) || !i915_modparams.guc_log_level)
-		return;
+	ret = guc_log_relay_create(log);
+	if (ret)
+		goto out_unlock;
+
+	ret = guc_log_map(log);
+	if (ret)
+		goto out_relay;
 
-	guc_log_late_setup(&dev_priv->guc);
+	mutex_unlock(&log->relay.lock);
+
+	guc_log_enable_flush_events(log);
+
+	/*
+	 * When GuC is logging without us relaying to userspace, we're ignoring
+	 * the flush notification. This means that we need to unconditionally
+	 * flush on relay enabling, since GuC only notifies us once.
+	 */
+	queue_work(log->relay.flush_wq, &log->relay.flush_work);
+
+	return 0;
+
+out_relay:
+	guc_log_relay_destroy(log);
+out_unlock:
+	mutex_unlock(&log->relay.lock);
+
+	return ret;
 }
 
-void i915_guc_log_unregister(struct drm_i915_private *dev_priv)
+void intel_guc_log_relay_flush(struct intel_guc_log *log)
 {
-	struct intel_guc *guc = &dev_priv->guc;
+	struct intel_guc *guc = log_to_guc(log);
+	struct drm_i915_private *i915 = guc_to_i915(guc);
+
+	/*
+	 * Before initiating the forceful flush, wait for any pending/ongoing
+	 * flush to complete otherwise forceful flush may not actually happen.
+	 */
+	flush_work(&log->relay.flush_work);
 
-	if (!USES_GUC_SUBMISSION(dev_priv))
-		return;
+	intel_runtime_pm_get(i915);
+	guc_action_flush_log(guc);
+	intel_runtime_pm_put(i915);
 
-	mutex_lock(&dev_priv->drm.struct_mutex);
-	/* GuC logging is currently the only user of Guc2Host interrupts */
-	intel_runtime_pm_get(dev_priv);
-	gen9_disable_guc_interrupts(dev_priv);
-	intel_runtime_pm_put(dev_priv);
+	/* GuC would have updated log buffer by now, so capture it */
+	guc_log_capture_logs(log);
+}
 
-	guc_log_runtime_destroy(guc);
-	mutex_unlock(&dev_priv->drm.struct_mutex);
+void intel_guc_log_relay_close(struct intel_guc_log *log)
+{
+	guc_log_disable_flush_events(log);
+	flush_work(&log->relay.flush_work);
+
+	mutex_lock(&log->relay.lock);
+	GEM_BUG_ON(!intel_guc_log_relay_enabled(log));
+	guc_log_unmap(log);
+	guc_log_relay_destroy(log);
+	mutex_unlock(&log->relay.lock);
+}
 
-	intel_guc_log_relay_destroy(guc);
+void intel_guc_log_handle_flush_event(struct intel_guc_log *log)
+{
+	queue_work(log->relay.flush_wq, &log->relay.flush_work);
 }
diff --git a/drivers/gpu/drm/i915/intel_guc_log.h b/drivers/gpu/drm/i915/intel_guc_log.h
index dab0e949567a..fa80535a6f9d 100644
--- a/drivers/gpu/drm/i915/intel_guc_log.h
+++ b/drivers/gpu/drm/i915/intel_guc_log.h
@@ -25,11 +25,12 @@
 #ifndef _INTEL_GUC_LOG_H_
 #define _INTEL_GUC_LOG_H_
 
+#include <linux/mutex.h>
+#include <linux/relay.h>
 #include <linux/workqueue.h>
 
 #include "intel_guc_fwif.h"
 
-struct drm_i915_private;
 struct intel_guc;
 
 /*
@@ -39,33 +40,53 @@ struct intel_guc;
 #define GUC_LOG_SIZE	((1 + GUC_LOG_DPC_PAGES + 1 + GUC_LOG_ISR_PAGES + \
 			  1 + GUC_LOG_CRASH_PAGES + 1) << PAGE_SHIFT)
 
+/*
+ * While we're using plain log level in i915, GuC controls are much more...
+ * "elaborate"? We have a couple of bits for verbosity, separate bit for actual
+ * log enabling, and separate bit for default logging - which "conveniently"
+ * ignores the enable bit.
+ */
+#define GUC_LOG_LEVEL_DISABLED		0
+#define GUC_LOG_LEVEL_NON_VERBOSE	1
+#define GUC_LOG_LEVEL_IS_ENABLED(x)	((x) > GUC_LOG_LEVEL_DISABLED)
+#define GUC_LOG_LEVEL_IS_VERBOSE(x)	((x) > GUC_LOG_LEVEL_NON_VERBOSE)
+#define GUC_LOG_LEVEL_TO_VERBOSITY(x) ({		\
+	typeof(x) _x = (x);				\
+	GUC_LOG_LEVEL_IS_VERBOSE(_x) ? _x - 2 : 0;	\
+})
+#define GUC_VERBOSITY_TO_LOG_LEVEL(x)	((x) + 2)
+#define GUC_LOG_LEVEL_MAX GUC_VERBOSITY_TO_LOG_LEVEL(GUC_LOG_VERBOSITY_MAX)
+
 struct intel_guc_log {
 	u32 flags;
 	struct i915_vma *vma;
-	/* The runtime stuff gets created only when GuC logging gets enabled */
 	struct {
 		void *buf_addr;
 		struct workqueue_struct *flush_wq;
 		struct work_struct flush_work;
-		struct rchan *relay_chan;
-		/* To serialize the access to relay_chan */
-		struct mutex relay_lock;
-	} runtime;
+		struct rchan *channel;
+		struct mutex lock;
+		u32 full_count;
+	} relay;
 	/* logging related stats */
-	u32 capture_miss_count;
-	u32 flush_interrupt_count;
-	u32 prev_overflow_count[GUC_MAX_LOG_BUFFER];
-	u32 total_overflow_count[GUC_MAX_LOG_BUFFER];
-	u32 flush_count[GUC_MAX_LOG_BUFFER];
+	struct {
+		u32 sampled_overflow;
+		u32 overflow;
+		u32 flush;
+	} stats[GUC_MAX_LOG_BUFFER];
 };
 
-int intel_guc_log_create(struct intel_guc *guc);
-void intel_guc_log_destroy(struct intel_guc *guc);
-void intel_guc_log_init_early(struct intel_guc *guc);
-int intel_guc_log_relay_create(struct intel_guc *guc);
-void intel_guc_log_relay_destroy(struct intel_guc *guc);
-int intel_guc_log_control(struct intel_guc *guc, u64 control_val);
-void i915_guc_log_register(struct drm_i915_private *dev_priv);
-void i915_guc_log_unregister(struct drm_i915_private *dev_priv);
+void intel_guc_log_init_early(struct intel_guc_log *log);
+int intel_guc_log_create(struct intel_guc_log *log);
+void intel_guc_log_destroy(struct intel_guc_log *log);
+
+int intel_guc_log_level_get(struct intel_guc_log *log);
+int intel_guc_log_level_set(struct intel_guc_log *log, u64 control_val);
+bool intel_guc_log_relay_enabled(const struct intel_guc_log *log);
+int intel_guc_log_relay_open(struct intel_guc_log *log);
+void intel_guc_log_relay_flush(struct intel_guc_log *log);
+void intel_guc_log_relay_close(struct intel_guc_log *log);
+
+void intel_guc_log_handle_flush_event(struct intel_guc_log *log);
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_guc_reg.h b/drivers/gpu/drm/i915/intel_guc_reg.h
index 19a9247c5664..d86084742a4a 100644
--- a/drivers/gpu/drm/i915/intel_guc_reg.h
+++ b/drivers/gpu/drm/i915/intel_guc_reg.h
@@ -66,22 +66,20 @@
 #define   UOS_MOVE			  (1<<4)
 #define   START_DMA			  (1<<0)
 #define DMA_GUC_WOPCM_OFFSET		_MMIO(0xc340)
+#define   GUC_WOPCM_OFFSET_VALID	  (1<<0)
 #define   HUC_LOADING_AGENT_VCR		  (0<<1)
 #define   HUC_LOADING_AGENT_GUC		  (1<<1)
-#define   GUC_WOPCM_OFFSET_VALUE	  0x80000	/* 512KB */
+#define   GUC_WOPCM_OFFSET_SHIFT	14
+#define   GUC_WOPCM_OFFSET_MASK		  (0x3ffff << GUC_WOPCM_OFFSET_SHIFT)
 #define GUC_MAX_IDLE_COUNT		_MMIO(0xC3E4)
 
 #define HUC_STATUS2             _MMIO(0xD3B0)
 #define   HUC_FW_VERIFIED       (1<<7)
 
-/* Defines WOPCM space available to GuC firmware */
 #define GUC_WOPCM_SIZE			_MMIO(0xc050)
-/* GuC addresses below GUC_WOPCM_TOP don't map through the GTT */
-#define   GUC_WOPCM_TOP			  (0x80 << 12)	/* 512KB */
-#define   BXT_GUC_WOPCM_RC6_RESERVED	  (0x10 << 12)	/* 64KB  */
-
-/* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */
-#define GUC_GGTT_TOP			0xFEE00000
+#define   GUC_WOPCM_SIZE_LOCKED		  (1<<0)
+#define   GUC_WOPCM_SIZE_SHIFT		12
+#define   GUC_WOPCM_SIZE_MASK		  (0xfffff << GUC_WOPCM_SIZE_SHIFT)
 
 #define GEN8_GT_PM_CONFIG		_MMIO(0x138140)
 #define GEN9LP_GT_PM_CONFIG		_MMIO(0x138140)
diff --git a/drivers/gpu/drm/i915/intel_guc_submission.c b/drivers/gpu/drm/i915/intel_guc_submission.c
index 8a8ad2fe158d..2feb65096966 100644
--- a/drivers/gpu/drm/i915/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/intel_guc_submission.c
@@ -124,9 +124,17 @@ static int reserve_doorbell(struct intel_guc_client *client)
 	return 0;
 }
 
+static bool has_doorbell(struct intel_guc_client *client)
+{
+	if (client->doorbell_id == GUC_DOORBELL_INVALID)
+		return false;
+
+	return test_bit(client->doorbell_id, client->guc->doorbell_bitmap);
+}
+
 static void unreserve_doorbell(struct intel_guc_client *client)
 {
-	GEM_BUG_ON(client->doorbell_id == GUC_DOORBELL_INVALID);
+	GEM_BUG_ON(!has_doorbell(client));
 
 	__clear_bit(client->doorbell_id, client->guc->doorbell_bitmap);
 	client->doorbell_id = GUC_DOORBELL_INVALID;
@@ -184,14 +192,6 @@ static struct guc_doorbell_info *__get_doorbell(struct intel_guc_client *client)
 	return client->vaddr + client->doorbell_offset;
 }
 
-static bool has_doorbell(struct intel_guc_client *client)
-{
-	if (client->doorbell_id == GUC_DOORBELL_INVALID)
-		return false;
-
-	return test_bit(client->doorbell_id, client->guc->doorbell_bitmap);
-}
-
 static void __create_doorbell(struct intel_guc_client *client)
 {
 	struct guc_doorbell_info *doorbell;
@@ -207,7 +207,6 @@ static void __destroy_doorbell(struct intel_guc_client *client)
 	struct guc_doorbell_info *doorbell;
 	u16 db_id = client->doorbell_id;
 
-
 	doorbell = __get_doorbell(client);
 	doorbell->db_status = GUC_DOORBELL_DISABLED;
 	doorbell->cookie = 0;
@@ -224,6 +223,9 @@ static int create_doorbell(struct intel_guc_client *client)
 {
 	int ret;
 
+	if (WARN_ON(!has_doorbell(client)))
+		return -ENODEV; /* internal setup error, should never happen */
+
 	__update_doorbell_desc(client, client->doorbell_id);
 	__create_doorbell(client);
 
@@ -231,8 +233,8 @@ static int create_doorbell(struct intel_guc_client *client)
 	if (ret) {
 		__destroy_doorbell(client);
 		__update_doorbell_desc(client, GUC_DOORBELL_INVALID);
-		DRM_ERROR("Couldn't create client %u doorbell: %d\n",
-			  client->stage_id, ret);
+		DRM_DEBUG_DRIVER("Couldn't create client %u doorbell: %d\n",
+				 client->stage_id, ret);
 		return ret;
 	}
 
@@ -362,7 +364,7 @@ static void guc_stage_desc_init(struct intel_guc *guc,
 	desc->db_id = client->doorbell_id;
 
 	for_each_engine_masked(engine, dev_priv, client->engines, tmp) {
-		struct intel_context *ce = &ctx->engine[engine->id];
+		struct intel_context *ce = to_intel_context(ctx, engine);
 		u32 guc_engine_id = engine->guc_id;
 		struct guc_execlist_context *lrc = &desc->lrc[guc_engine_id];
 
@@ -386,8 +388,8 @@ static void guc_stage_desc_init(struct intel_guc *guc,
 		lrc->context_desc = lower_32_bits(ce->lrc_desc);
 
 		/* The state page is after PPHWSP */
-		lrc->ring_lrca =
-			guc_ggtt_offset(ce->state) + LRC_STATE_PN * PAGE_SIZE;
+		lrc->ring_lrca = intel_guc_ggtt_offset(guc, ce->state) +
+				 LRC_STATE_PN * PAGE_SIZE;
 
 		/* XXX: In direct submission, the GuC wants the HW context id
 		 * here. In proxy submission, it wants the stage id
@@ -395,7 +397,7 @@ static void guc_stage_desc_init(struct intel_guc *guc,
 		lrc->context_id = (client->stage_id << GUC_ELC_CTXID_OFFSET) |
 				(guc_engine_id << GUC_ELC_ENGINE_OFFSET);
 
-		lrc->ring_begin = guc_ggtt_offset(ce->ring->vma);
+		lrc->ring_begin = intel_guc_ggtt_offset(guc, ce->ring->vma);
 		lrc->ring_end = lrc->ring_begin + ce->ring->size - 1;
 		lrc->ring_next_free_location = lrc->ring_begin;
 		lrc->ring_current_tail_pointer_value = 0;
@@ -411,7 +413,7 @@ static void guc_stage_desc_init(struct intel_guc *guc,
 	 * The doorbell, process descriptor, and workqueue are all parts
 	 * of the client object, which the GuC will reference via the GGTT
 	 */
-	gfx_addr = guc_ggtt_offset(client->vma);
+	gfx_addr = intel_guc_ggtt_offset(guc, client->vma);
 	desc->db_trigger_phy = sg_dma_address(client->vma->pages->sgl) +
 				client->doorbell_offset;
 	desc->db_trigger_cpu = ptr_to_u64(__get_doorbell(client));
@@ -584,7 +586,7 @@ static void inject_preempt_context(struct work_struct *work)
 	data[3] = engine->guc_id;
 	data[4] = guc->execbuf_client->priority;
 	data[5] = guc->execbuf_client->stage_id;
-	data[6] = guc_ggtt_offset(guc->shared_data);
+	data[6] = intel_guc_ggtt_offset(guc, guc->shared_data);
 
 	if (WARN_ON(intel_guc_send(guc, data, ARRAY_SIZE(data)))) {
 		execlists_clear_active(&engine->execlists,
@@ -657,7 +659,17 @@ static void port_assign(struct execlist_port *port, struct i915_request *rq)
 	port_set(port, i915_request_get(rq));
 }
 
-static void guc_dequeue(struct intel_engine_cs *engine)
+static inline int rq_prio(const struct i915_request *rq)
+{
+	return rq->sched.attr.priority;
+}
+
+static inline int port_prio(const struct execlist_port *port)
+{
+	return rq_prio(port_request(port));
+}
+
+static bool __guc_dequeue(struct intel_engine_cs *engine)
 {
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	struct execlist_port *port = execlists->port;
@@ -667,28 +679,29 @@ static void guc_dequeue(struct intel_engine_cs *engine)
 	bool submit = false;
 	struct rb_node *rb;
 
-	spin_lock_irq(&engine->timeline->lock);
+	lockdep_assert_held(&engine->timeline.lock);
+
 	rb = execlists->first;
 	GEM_BUG_ON(rb_first(&execlists->queue) != rb);
 
 	if (port_isset(port)) {
-		if (engine->i915->preempt_context) {
+		if (intel_engine_has_preemption(engine)) {
 			struct guc_preempt_work *preempt_work =
 				&engine->i915->guc.preempt_work[engine->id];
+			int prio = execlists->queue_priority;
 
-			if (execlists->queue_priority >
-			    max(port_request(port)->priotree.priority, 0)) {
+			if (__execlists_need_preempt(prio, port_prio(port))) {
 				execlists_set_active(execlists,
 						     EXECLISTS_ACTIVE_PREEMPT);
 				queue_work(engine->i915->guc.preempt_wq,
 					   &preempt_work->work);
-				goto unlock;
+				return false;
 			}
 		}
 
 		port++;
 		if (port_isset(port))
-			goto unlock;
+			return false;
 	}
 	GEM_BUG_ON(port_isset(port));
 
@@ -696,11 +709,11 @@ static void guc_dequeue(struct intel_engine_cs *engine)
 		struct i915_priolist *p = to_priolist(rb);
 		struct i915_request *rq, *rn;
 
-		list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
+		list_for_each_entry_safe(rq, rn, &p->requests, sched.link) {
 			if (last && rq->ctx != last->ctx) {
 				if (port == last_port) {
 					__list_del_many(&p->requests,
-							&rq->priotree.link);
+							&rq->sched.link);
 					goto done;
 				}
 
@@ -709,7 +722,7 @@ static void guc_dequeue(struct intel_engine_cs *engine)
 				port++;
 			}
 
-			INIT_LIST_HEAD(&rq->priotree.link);
+			INIT_LIST_HEAD(&rq->sched.link);
 
 			__i915_request_submit(rq);
 			trace_i915_request_in(rq, port_index(port, execlists));
@@ -726,19 +739,34 @@ static void guc_dequeue(struct intel_engine_cs *engine)
 done:
 	execlists->queue_priority = rb ? to_priolist(rb)->priority : INT_MIN;
 	execlists->first = rb;
-	if (submit) {
+	if (submit)
 		port_assign(port, last);
-		execlists_set_active(execlists, EXECLISTS_ACTIVE_USER);
-		guc_submit(engine);
-	}
+	if (last)
+		execlists_user_begin(execlists, execlists->port);
 
 	/* We must always keep the beast fed if we have work piled up */
 	GEM_BUG_ON(port_isset(execlists->port) &&
 		   !execlists_is_active(execlists, EXECLISTS_ACTIVE_USER));
 	GEM_BUG_ON(execlists->first && !port_isset(execlists->port));
 
-unlock:
-	spin_unlock_irq(&engine->timeline->lock);
+	return submit;
+}
+
+static void guc_dequeue(struct intel_engine_cs *engine)
+{
+	unsigned long flags;
+	bool submit;
+
+	local_irq_save(flags);
+
+	spin_lock(&engine->timeline.lock);
+	submit = __guc_dequeue(engine);
+	spin_unlock(&engine->timeline.lock);
+
+	if (submit)
+		guc_submit(engine);
+
+	local_irq_restore(flags);
 }
 
 static void guc_submission_tasklet(unsigned long data)
@@ -748,17 +776,20 @@ static void guc_submission_tasklet(unsigned long data)
 	struct execlist_port *port = execlists->port;
 	struct i915_request *rq;
 
-	rq = port_request(&port[0]);
+	rq = port_request(port);
 	while (rq && i915_request_completed(rq)) {
 		trace_i915_request_out(rq);
 		i915_request_put(rq);
 
-		execlists_port_complete(execlists, port);
-
-		rq = port_request(&port[0]);
+		port = execlists_port_complete(execlists, port);
+		if (port_isset(port)) {
+			execlists_user_begin(execlists, port);
+			rq = port_request(port);
+		} else {
+			execlists_user_end(execlists);
+			rq = NULL;
+		}
 	}
-	if (!rq)
-		execlists_clear_active(execlists, EXECLISTS_ACTIVE_USER);
 
 	if (execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT) &&
 	    intel_read_status_page(engine, I915_GEM_HWS_PREEMPT_INDEX) ==
@@ -977,7 +1008,8 @@ static void guc_fill_preempt_context(struct intel_guc *guc)
 	enum intel_engine_id id;
 
 	for_each_engine(engine, dev_priv, id) {
-		struct intel_context *ce = &client->owner->engine[id];
+		struct intel_context *ce =
+			to_intel_context(client->owner, engine);
 		u32 addr = intel_hws_preempt_done_address(engine);
 		u32 *cs;
 
diff --git a/drivers/gpu/drm/i915/intel_hangcheck.c b/drivers/gpu/drm/i915/intel_hangcheck.c
index c8ea510629fa..d47e346bd49e 100644
--- a/drivers/gpu/drm/i915/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/intel_hangcheck.c
@@ -246,9 +246,8 @@ engine_stuck(struct intel_engine_cs *engine, u64 acthd)
 	 */
 	tmp = I915_READ_CTL(engine);
 	if (tmp & RING_WAIT) {
-		i915_handle_error(dev_priv, BIT(engine->id),
-				  "Kicking stuck wait on %s",
-				  engine->name);
+		i915_handle_error(dev_priv, BIT(engine->id), 0,
+				  "stuck wait on %s", engine->name);
 		I915_WRITE_CTL(engine, tmp);
 		return ENGINE_WAIT_KICK;
 	}
@@ -258,8 +257,8 @@ engine_stuck(struct intel_engine_cs *engine, u64 acthd)
 		default:
 			return ENGINE_DEAD;
 		case 1:
-			i915_handle_error(dev_priv, ALL_ENGINES,
-					  "Kicking stuck semaphore on %s",
+			i915_handle_error(dev_priv, ALL_ENGINES, 0,
+					  "stuck semaphore on %s",
 					  engine->name);
 			I915_WRITE_CTL(engine, tmp);
 			return ENGINE_WAIT_KICK;
@@ -357,7 +356,7 @@ static void hangcheck_accumulate_sample(struct intel_engine_cs *engine,
 		break;
 
 	case ENGINE_DEAD:
-		if (drm_debug & DRM_UT_DRIVER) {
+		if (GEM_SHOW_DEBUG()) {
 			struct drm_printer p = drm_debug_printer("hangcheck");
 			intel_engine_dump(engine, &p, "%s\n", engine->name);
 		}
@@ -386,13 +385,13 @@ static void hangcheck_declare_hang(struct drm_i915_private *i915,
 	if (stuck != hung)
 		hung &= ~stuck;
 	len = scnprintf(msg, sizeof(msg),
-			"%s on ", stuck == hung ? "No progress" : "Hang");
+			"%s on ", stuck == hung ? "no progress" : "hang");
 	for_each_engine_masked(engine, i915, hung, tmp)
 		len += scnprintf(msg + len, sizeof(msg) - len,
 				 "%s, ", engine->name);
 	msg[len-2] = '\0';
 
-	return i915_handle_error(i915, hung, "%s", msg);
+	return i915_handle_error(i915, hung, I915_ERROR_CAPTURE, "%s", msg);
 }
 
 /*
@@ -453,6 +452,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 void intel_engine_init_hangcheck(struct intel_engine_cs *engine)
 {
 	memset(&engine->hangcheck, 0, sizeof(engine->hangcheck));
+	engine->hangcheck.action_timestamp = jiffies;
 }
 
 void intel_hangcheck_init(struct drm_i915_private *i915)
diff --git a/drivers/gpu/drm/i915/intel_hdcp.c b/drivers/gpu/drm/i915/intel_hdcp.c
index 14ca5d3057a7..2db5da550a1c 100644
--- a/drivers/gpu/drm/i915/intel_hdcp.c
+++ b/drivers/gpu/drm/i915/intel_hdcp.c
@@ -37,6 +37,43 @@ static int intel_hdcp_poll_ksv_fifo(struct intel_digital_port *intel_dig_port,
 	return 0;
 }
 
+static bool hdcp_key_loadable(struct drm_i915_private *dev_priv)
+{
+	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_well *power_well;
+	enum i915_power_well_id id;
+	bool enabled = false;
+
+	/*
+	 * On HSW and BDW, Display HW loads the Key as soon as Display resumes.
+	 * On all BXT+, SW can load the keys only when the PW#1 is turned on.
+	 */
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
+		id = HSW_DISP_PW_GLOBAL;
+	else
+		id = SKL_DISP_PW_1;
+
+	mutex_lock(&power_domains->lock);
+
+	/* PG1 (power well #1) needs to be enabled */
+	for_each_power_well(dev_priv, power_well) {
+		if (power_well->id == id) {
+			enabled = power_well->ops->is_enabled(dev_priv,
+							      power_well);
+			break;
+		}
+	}
+	mutex_unlock(&power_domains->lock);
+
+	/*
+	 * Another req for hdcp key loadability is enabled state of pll for
+	 * cdclk. Without active crtc we wont land here. So we are assuming that
+	 * cdclk is already on.
+	 */
+
+	return enabled;
+}
+
 static void intel_hdcp_clear_keys(struct drm_i915_private *dev_priv)
 {
 	I915_WRITE(HDCP_KEY_CONF, HDCP_CLEAR_KEYS_TRIGGER);
@@ -142,53 +179,17 @@ bool intel_hdcp_is_ksv_valid(u8 *ksv)
 	return true;
 }
 
-/* Implements Part 2 of the HDCP authorization procedure */
 static
-int intel_hdcp_auth_downstream(struct intel_digital_port *intel_dig_port,
-			       const struct intel_hdcp_shim *shim)
+int intel_hdcp_validate_v_prime(struct intel_digital_port *intel_dig_port,
+				const struct intel_hdcp_shim *shim,
+				u8 *ksv_fifo, u8 num_downstream, u8 *bstatus)
 {
 	struct drm_i915_private *dev_priv;
 	u32 vprime, sha_text, sha_leftovers, rep_ctl;
-	u8 bstatus[2], num_downstream, *ksv_fifo;
 	int ret, i, j, sha_idx;
 
 	dev_priv = intel_dig_port->base.base.dev->dev_private;
 
-	ret = intel_hdcp_poll_ksv_fifo(intel_dig_port, shim);
-	if (ret) {
-		DRM_ERROR("KSV list failed to become ready (%d)\n", ret);
-		return ret;
-	}
-
-	ret = shim->read_bstatus(intel_dig_port, bstatus);
-	if (ret)
-		return ret;
-
-	if (DRM_HDCP_MAX_DEVICE_EXCEEDED(bstatus[0]) ||
-	    DRM_HDCP_MAX_CASCADE_EXCEEDED(bstatus[1])) {
-		DRM_ERROR("Max Topology Limit Exceeded\n");
-		return -EPERM;
-	}
-
-	/*
-	 * When repeater reports 0 device count, HDCP1.4 spec allows disabling
-	 * the HDCP encryption. That implies that repeater can't have its own
-	 * display. As there is no consumption of encrypted content in the
-	 * repeater with 0 downstream devices, we are failing the
-	 * authentication.
-	 */
-	num_downstream = DRM_HDCP_NUM_DOWNSTREAM(bstatus[0]);
-	if (num_downstream == 0)
-		return -EINVAL;
-
-	ksv_fifo = kzalloc(num_downstream * DRM_HDCP_KSV_LEN, GFP_KERNEL);
-	if (!ksv_fifo)
-		return -ENOMEM;
-
-	ret = shim->read_ksv_fifo(intel_dig_port, num_downstream, ksv_fifo);
-	if (ret)
-		return ret;
-
 	/* Process V' values from the receiver */
 	for (i = 0; i < DRM_HDCP_V_PRIME_NUM_PARTS; i++) {
 		ret = shim->read_v_prime_part(intel_dig_port, i, &vprime);
@@ -353,7 +354,8 @@ int intel_hdcp_auth_downstream(struct intel_digital_port *intel_dig_port,
 			return ret;
 		sha_idx += sizeof(sha_text);
 	} else {
-		DRM_ERROR("Invalid number of leftovers %d\n", sha_leftovers);
+		DRM_DEBUG_KMS("Invalid number of leftovers %d\n",
+			      sha_leftovers);
 		return -EINVAL;
 	}
 
@@ -381,17 +383,83 @@ int intel_hdcp_auth_downstream(struct intel_digital_port *intel_dig_port,
 	if (intel_wait_for_register(dev_priv, HDCP_REP_CTL,
 				    HDCP_SHA1_COMPLETE,
 				    HDCP_SHA1_COMPLETE, 1)) {
-		DRM_ERROR("Timed out waiting for SHA1 complete\n");
+		DRM_DEBUG_KMS("Timed out waiting for SHA1 complete\n");
 		return -ETIMEDOUT;
 	}
 	if (!(I915_READ(HDCP_REP_CTL) & HDCP_SHA1_V_MATCH)) {
-		DRM_ERROR("SHA-1 mismatch, HDCP failed\n");
+		DRM_DEBUG_KMS("SHA-1 mismatch, HDCP failed\n");
 		return -ENXIO;
 	}
 
+	return 0;
+}
+
+/* Implements Part 2 of the HDCP authorization procedure */
+static
+int intel_hdcp_auth_downstream(struct intel_digital_port *intel_dig_port,
+			       const struct intel_hdcp_shim *shim)
+{
+	u8 bstatus[2], num_downstream, *ksv_fifo;
+	int ret, i, tries = 3;
+
+	ret = intel_hdcp_poll_ksv_fifo(intel_dig_port, shim);
+	if (ret) {
+		DRM_ERROR("KSV list failed to become ready (%d)\n", ret);
+		return ret;
+	}
+
+	ret = shim->read_bstatus(intel_dig_port, bstatus);
+	if (ret)
+		return ret;
+
+	if (DRM_HDCP_MAX_DEVICE_EXCEEDED(bstatus[0]) ||
+	    DRM_HDCP_MAX_CASCADE_EXCEEDED(bstatus[1])) {
+		DRM_ERROR("Max Topology Limit Exceeded\n");
+		return -EPERM;
+	}
+
+	/*
+	 * When repeater reports 0 device count, HDCP1.4 spec allows disabling
+	 * the HDCP encryption. That implies that repeater can't have its own
+	 * display. As there is no consumption of encrypted content in the
+	 * repeater with 0 downstream devices, we are failing the
+	 * authentication.
+	 */
+	num_downstream = DRM_HDCP_NUM_DOWNSTREAM(bstatus[0]);
+	if (num_downstream == 0)
+		return -EINVAL;
+
+	ksv_fifo = kzalloc(num_downstream * DRM_HDCP_KSV_LEN, GFP_KERNEL);
+	if (!ksv_fifo)
+		return -ENOMEM;
+
+	ret = shim->read_ksv_fifo(intel_dig_port, num_downstream, ksv_fifo);
+	if (ret)
+		goto err;
+
+	/*
+	 * When V prime mismatches, DP Spec mandates re-read of
+	 * V prime atleast twice.
+	 */
+	for (i = 0; i < tries; i++) {
+		ret = intel_hdcp_validate_v_prime(intel_dig_port, shim,
+						  ksv_fifo, num_downstream,
+						  bstatus);
+		if (!ret)
+			break;
+	}
+
+	if (i == tries) {
+		DRM_ERROR("V Prime validation failed.(%d)\n", ret);
+		goto err;
+	}
+
 	DRM_DEBUG_KMS("HDCP is enabled (%d downstream devices)\n",
 		      num_downstream);
-	return 0;
+	ret = 0;
+err:
+	kfree(ksv_fifo);
+	return ret;
 }
 
 /* Implements Part 1 of the HDCP authorization procedure */
@@ -506,15 +574,26 @@ static int intel_hdcp_auth(struct intel_digital_port *intel_dig_port,
 	 */
 	wait_remaining_ms_from_jiffies(r0_prime_gen_start, 300);
 
-	ri.reg = 0;
-	ret = shim->read_ri_prime(intel_dig_port, ri.shim);
-	if (ret)
-		return ret;
-	I915_WRITE(PORT_HDCP_RPRIME(port), ri.reg);
+	tries = 3;
 
-	/* Wait for Ri prime match */
-	if (wait_for(I915_READ(PORT_HDCP_STATUS(port)) &
-		     (HDCP_STATUS_RI_MATCH | HDCP_STATUS_ENC), 1)) {
+	/*
+	 * DP HDCP Spec mandates the two more reattempt to read R0, incase
+	 * of R0 mismatch.
+	 */
+	for (i = 0; i < tries; i++) {
+		ri.reg = 0;
+		ret = shim->read_ri_prime(intel_dig_port, ri.shim);
+		if (ret)
+			return ret;
+		I915_WRITE(PORT_HDCP_RPRIME(port), ri.reg);
+
+		/* Wait for Ri prime match */
+		if (!wait_for(I915_READ(PORT_HDCP_STATUS(port)) &
+		    (HDCP_STATUS_RI_MATCH | HDCP_STATUS_ENC), 1))
+			break;
+	}
+
+	if (i == tries) {
 		DRM_ERROR("Timed out waiting for Ri prime match (%x)\n",
 			  I915_READ(PORT_HDCP_STATUS(port)));
 		return -ETIMEDOUT;
@@ -580,8 +659,8 @@ static int _intel_hdcp_enable(struct intel_connector *connector)
 	DRM_DEBUG_KMS("[%s:%d] HDCP is being enabled...\n",
 		      connector->base.name, connector->base.base.id);
 
-	if (!(I915_READ(SKL_FUSE_STATUS) & SKL_FUSE_PG_DIST_STATUS(1))) {
-		DRM_ERROR("PG1 is disabled, cannot load keys\n");
+	if (!hdcp_key_loadable(dev_priv)) {
+		DRM_ERROR("HDCP key Load is not possible\n");
 		return -ENXIO;
 	}
 
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index 1baef4ac7ecb..ee929f31f7db 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -2082,41 +2082,33 @@ intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *c
  * it enables scrambling. This should be called before enabling the HDMI
  * 2.0 port, as the sink can choose to disable the scrambling if it doesn't
  * detect a scrambled clock within 100 ms.
+ *
+ * Returns:
+ * True on success, false on failure.
  */
-void intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder,
+bool intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder,
 				       struct drm_connector *connector,
 				       bool high_tmds_clock_ratio,
 				       bool scrambling)
 {
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
-	struct drm_i915_private *dev_priv = connector->dev->dev_private;
 	struct drm_scrambling *sink_scrambling =
-				&connector->display_info.hdmi.scdc.scrambling;
-	struct i2c_adapter *adptr = intel_gmbus_get_adapter(dev_priv,
-							   intel_hdmi->ddc_bus);
-	bool ret;
+		&connector->display_info.hdmi.scdc.scrambling;
+	struct i2c_adapter *adapter =
+		intel_gmbus_get_adapter(dev_priv, intel_hdmi->ddc_bus);
 
 	if (!sink_scrambling->supported)
-		return;
-
-	DRM_DEBUG_KMS("Setting sink scrambling for enc:%s connector:%s\n",
-		      encoder->base.name, connector->name);
+		return true;
 
-	/* Set TMDS bit clock ratio to 1/40 or 1/10 */
-	ret = drm_scdc_set_high_tmds_clock_ratio(adptr, high_tmds_clock_ratio);
-	if (!ret) {
-		DRM_ERROR("Set TMDS ratio failed\n");
-		return;
-	}
-
-	/* Enable/disable sink scrambling */
-	ret = drm_scdc_set_scrambling(adptr, scrambling);
-	if (!ret) {
-		DRM_ERROR("Set sink scrambling failed\n");
-		return;
-	}
+	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] scrambling=%s, TMDS bit clock ratio=1/%d\n",
+		      connector->base.id, connector->name,
+		      yesno(scrambling), high_tmds_clock_ratio ? 40 : 10);
 
-	DRM_DEBUG_KMS("sink scrambling handled\n");
+	/* Set TMDS bit clock ratio to 1/40 or 1/10, and enable/disable scrambling */
+	return drm_scdc_set_high_tmds_clock_ratio(adapter,
+						  high_tmds_clock_ratio) &&
+		drm_scdc_set_scrambling(adapter, scrambling);
 }
 
 static u8 chv_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c
index 0e3d3e89d66a..43aa92beff2a 100644
--- a/drivers/gpu/drm/i915/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/intel_hotplug.c
@@ -100,6 +100,8 @@ enum port intel_hpd_pin_to_port(struct drm_i915_private *dev_priv,
 		if (IS_CNL_WITH_PORT_F(dev_priv))
 			return PORT_F;
 		return PORT_E;
+	case HPD_PORT_F:
+		return PORT_F;
 	default:
 		return PORT_NONE; /* no port for this pin */
 	}
@@ -132,6 +134,7 @@ enum hpd_pin intel_hpd_pin_default(struct drm_i915_private *dev_priv,
 	case PORT_F:
 		if (IS_CNL_WITH_PORT_F(dev_priv))
 			return HPD_PORT_E;
+		return HPD_PORT_F;
 	default:
 		MISSING_CASE(port);
 		return HPD_NONE;
diff --git a/drivers/gpu/drm/i915/intel_huc.c b/drivers/gpu/drm/i915/intel_huc.c
index 65e2afb9b955..291285277403 100644
--- a/drivers/gpu/drm/i915/intel_huc.c
+++ b/drivers/gpu/drm/i915/intel_huc.c
@@ -55,7 +55,7 @@ int intel_huc_auth(struct intel_huc *huc)
 		return -ENOEXEC;
 
 	vma = i915_gem_object_ggtt_pin(huc->fw.obj, NULL, 0, 0,
-				PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
+				       PIN_OFFSET_BIAS | guc->ggtt_pin_bias);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		DRM_ERROR("HuC: Failed to pin huc fw object %d\n", ret);
@@ -63,7 +63,8 @@ int intel_huc_auth(struct intel_huc *huc)
 	}
 
 	ret = intel_guc_auth_huc(guc,
-				 guc_ggtt_offset(vma) + huc->fw.rsa_offset);
+				 intel_guc_ggtt_offset(guc, vma) +
+				 huc->fw.rsa_offset);
 	if (ret) {
 		DRM_ERROR("HuC: GuC did not ack Auth request %d\n", ret);
 		goto fail_unpin;
@@ -91,3 +92,28 @@ int intel_huc_auth(struct intel_huc *huc)
 	DRM_ERROR("HuC: Authentication failed %d\n", ret);
 	return ret;
 }
+
+/**
+ * intel_huc_check_status() - check HuC status
+ * @huc: intel_huc structure
+ *
+ * This function reads status register to verify if HuC
+ * firmware was successfully loaded.
+ *
+ * Returns positive value if HuC firmware is loaded and verified
+ * and -ENODEV if HuC is not present.
+ */
+int intel_huc_check_status(struct intel_huc *huc)
+{
+	struct drm_i915_private *dev_priv = huc_to_i915(huc);
+	u32 status;
+
+	if (!HAS_HUC(dev_priv))
+		return -ENODEV;
+
+	intel_runtime_pm_get(dev_priv);
+	status = I915_READ(HUC_STATUS2) & HUC_FW_VERIFIED;
+	intel_runtime_pm_put(dev_priv);
+
+	return status;
+}
diff --git a/drivers/gpu/drm/i915/intel_huc.h b/drivers/gpu/drm/i915/intel_huc.h
index 5d6e804f9771..aa854907abac 100644
--- a/drivers/gpu/drm/i915/intel_huc.h
+++ b/drivers/gpu/drm/i915/intel_huc.h
@@ -37,5 +37,12 @@ struct intel_huc {
 
 void intel_huc_init_early(struct intel_huc *huc);
 int intel_huc_auth(struct intel_huc *huc);
+int intel_huc_check_status(struct intel_huc *huc);
+
+static inline int intel_huc_sanitize(struct intel_huc *huc)
+{
+	intel_uc_fw_sanitize(&huc->fw);
+	return 0;
+}
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_huc_fw.c b/drivers/gpu/drm/i915/intel_huc_fw.c
index c66afa9b989a..f93d2384d482 100644
--- a/drivers/gpu/drm/i915/intel_huc_fw.c
+++ b/drivers/gpu/drm/i915/intel_huc_fw.c
@@ -118,7 +118,8 @@ static int huc_fw_xfer(struct intel_uc_fw *huc_fw, struct i915_vma *vma)
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
 	/* Set the source address for the uCode */
-	offset = guc_ggtt_offset(vma) + huc_fw->header_offset;
+	offset = intel_guc_ggtt_offset(&dev_priv->guc, vma) +
+		 huc_fw->header_offset;
 	I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset));
 	I915_WRITE(DMA_ADDR_0_HIGH, upper_32_bits(offset) & 0xFFFF);
 
@@ -154,9 +155,8 @@ static int huc_fw_xfer(struct intel_uc_fw *huc_fw, struct i915_vma *vma)
  * Called from intel_uc_init_hw() during driver load, resume from sleep and
  * after a GPU reset. Note that HuC must be loaded before GuC.
  *
- * The firmware image should have already been fetched into memory by the
- * earlier call to intel_uc_init_fw(), so here we need to only check that
- * fetch succeeded, and then transfer the image to the h/w.
+ * The firmware image should have already been fetched into memory, so only
+ * check that fetch succeeded, and then transfer the image to the h/w.
  *
  * Return:	non-zero code on error
  */
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 8704f7f8d072..15434cad5430 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -139,6 +139,7 @@
 #include "i915_gem_render_state.h"
 #include "intel_lrc_reg.h"
 #include "intel_mocs.h"
+#include "intel_workarounds.h"
 
 #define RING_EXECLIST_QFULL		(1 << 0x2)
 #define RING_EXECLIST1_VALID		(1 << 0x3)
@@ -176,14 +177,16 @@ static inline struct i915_priolist *to_priolist(struct rb_node *rb)
 
 static inline int rq_prio(const struct i915_request *rq)
 {
-	return rq->priotree.priority;
+	return rq->sched.attr.priority;
 }
 
 static inline bool need_preempt(const struct intel_engine_cs *engine,
 				const struct i915_request *last,
 				int prio)
 {
-	return engine->i915->preempt_context && prio > max(rq_prio(last), 0);
+	return (intel_engine_has_preemption(engine) &&
+		__execlists_need_preempt(prio, rq_prio(last)) &&
+		!i915_request_completed(last));
 }
 
 /**
@@ -221,7 +224,7 @@ static void
 intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
 				   struct intel_engine_cs *engine)
 {
-	struct intel_context *ce = &ctx->engine[engine->id];
+	struct intel_context *ce = to_intel_context(ctx, engine);
 	u64 desc;
 
 	BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (BIT(GEN8_CTX_ID_WIDTH)));
@@ -255,9 +258,7 @@ intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
 }
 
 static struct i915_priolist *
-lookup_priolist(struct intel_engine_cs *engine,
-		struct i915_priotree *pt,
-		int prio)
+lookup_priolist(struct intel_engine_cs *engine, int prio)
 {
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	struct i915_priolist *p;
@@ -328,10 +329,10 @@ static void __unwind_incomplete_requests(struct intel_engine_cs *engine)
 	struct i915_priolist *uninitialized_var(p);
 	int last_prio = I915_PRIORITY_INVALID;
 
-	lockdep_assert_held(&engine->timeline->lock);
+	lockdep_assert_held(&engine->timeline.lock);
 
 	list_for_each_entry_safe_reverse(rq, rn,
-					 &engine->timeline->requests,
+					 &engine->timeline.requests,
 					 link) {
 		if (i915_request_completed(rq))
 			return;
@@ -342,10 +343,11 @@ static void __unwind_incomplete_requests(struct intel_engine_cs *engine)
 		GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
 		if (rq_prio(rq) != last_prio) {
 			last_prio = rq_prio(rq);
-			p = lookup_priolist(engine, &rq->priotree, last_prio);
+			p = lookup_priolist(engine, last_prio);
 		}
 
-		list_add(&rq->priotree.link, &p->requests);
+		GEM_BUG_ON(p->priority != rq_prio(rq));
+		list_add(&rq->sched.link, &p->requests);
 	}
 }
 
@@ -354,10 +356,13 @@ execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
 {
 	struct intel_engine_cs *engine =
 		container_of(execlists, typeof(*engine), execlists);
+	unsigned long flags;
+
+	spin_lock_irqsave(&engine->timeline.lock, flags);
 
-	spin_lock_irq(&engine->timeline->lock);
 	__unwind_incomplete_requests(engine);
-	spin_unlock_irq(&engine->timeline->lock);
+
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
 static inline void
@@ -374,6 +379,19 @@ execlists_context_status_change(struct i915_request *rq, unsigned long status)
 				   status, rq);
 }
 
+inline void
+execlists_user_begin(struct intel_engine_execlists *execlists,
+		     const struct execlist_port *port)
+{
+	execlists_set_active_once(execlists, EXECLISTS_ACTIVE_USER);
+}
+
+inline void
+execlists_user_end(struct intel_engine_execlists *execlists)
+{
+	execlists_clear_active(execlists, EXECLISTS_ACTIVE_USER);
+}
+
 static inline void
 execlists_context_schedule_in(struct i915_request *rq)
 {
@@ -382,10 +400,11 @@ execlists_context_schedule_in(struct i915_request *rq)
 }
 
 static inline void
-execlists_context_schedule_out(struct i915_request *rq)
+execlists_context_schedule_out(struct i915_request *rq, unsigned long status)
 {
 	intel_engine_context_out(rq->engine);
-	execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
+	execlists_context_status_change(rq, status);
+	trace_i915_request_out(rq);
 }
 
 static void
@@ -399,7 +418,7 @@ execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state)
 
 static u64 execlists_update_context(struct i915_request *rq)
 {
-	struct intel_context *ce = &rq->ctx->engine[rq->engine->id];
+	struct intel_context *ce = to_intel_context(rq->ctx, rq->engine);
 	struct i915_hw_ppgtt *ppgtt =
 		rq->ctx->ppgtt ?: rq->i915->mm.aliasing_ppgtt;
 	u32 *reg_state = ce->lrc_reg_state;
@@ -454,10 +473,12 @@ static void execlists_submit_ports(struct intel_engine_cs *engine)
 			desc = execlists_update_context(rq);
 			GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc));
 
-			GEM_TRACE("%s in[%d]:  ctx=%d.%d, seqno=%x, prio=%d\n",
+			GEM_TRACE("%s in[%d]:  ctx=%d.%d, global=%d (fence %llx:%d) (current %d), prio=%d\n",
 				  engine->name, n,
 				  port[n].context_id, count,
 				  rq->global_seqno,
+				  rq->fence.context, rq->fence.seqno,
+				  intel_engine_get_seqno(engine),
 				  rq_prio(rq));
 		} else {
 			GEM_BUG_ON(!n);
@@ -506,7 +527,7 @@ static void inject_preempt_context(struct intel_engine_cs *engine)
 {
 	struct intel_engine_execlists *execlists = &engine->execlists;
 	struct intel_context *ce =
-		&engine->i915->preempt_context->engine[engine->id];
+		to_intel_context(engine->i915->preempt_context, engine);
 	unsigned int n;
 
 	GEM_BUG_ON(execlists->preempt_complete_status !=
@@ -535,7 +556,7 @@ static void inject_preempt_context(struct intel_engine_cs *engine)
 	execlists_set_active(&engine->execlists, EXECLISTS_ACTIVE_PREEMPT);
 }
 
-static void execlists_dequeue(struct intel_engine_cs *engine)
+static bool __execlists_dequeue(struct intel_engine_cs *engine)
 {
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	struct execlist_port *port = execlists->port;
@@ -545,6 +566,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 	struct rb_node *rb;
 	bool submit = false;
 
+	lockdep_assert_held(&engine->timeline.lock);
+
 	/* Hardware submission is through 2 ports. Conceptually each port
 	 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
 	 * static for a context, and unique to each, so we only execute
@@ -566,7 +589,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 	 * and context switches) submission.
 	 */
 
-	spin_lock_irq(&engine->timeline->lock);
 	rb = execlists->first;
 	GEM_BUG_ON(rb_first(&execlists->queue) != rb);
 
@@ -581,7 +603,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 						EXECLISTS_ACTIVE_USER));
 		GEM_BUG_ON(!port_count(&port[0]));
 		if (port_count(&port[0]) > 1)
-			goto unlock;
+			return false;
 
 		/*
 		 * If we write to ELSP a second time before the HW has had
@@ -591,11 +613,11 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 		 * the HW to indicate that it has had a chance to respond.
 		 */
 		if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK))
-			goto unlock;
+			return false;
 
 		if (need_preempt(engine, last, execlists->queue_priority)) {
 			inject_preempt_context(engine);
-			goto unlock;
+			return false;
 		}
 
 		/*
@@ -620,7 +642,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 		 * priorities of the ports haven't been switch.
 		 */
 		if (port_count(&port[1]))
-			goto unlock;
+			return false;
 
 		/*
 		 * WaIdleLiteRestore:bdw,skl
@@ -637,7 +659,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 		struct i915_priolist *p = to_priolist(rb);
 		struct i915_request *rq, *rn;
 
-		list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
+		list_for_each_entry_safe(rq, rn, &p->requests, sched.link) {
 			/*
 			 * Can we combine this request with the current port?
 			 * It has to be the same context/ringbuffer and not
@@ -657,7 +679,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 				 */
 				if (port == last_port) {
 					__list_del_many(&p->requests,
-							&rq->priotree.link);
+							&rq->sched.link);
 					goto done;
 				}
 
@@ -671,7 +693,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 				if (ctx_single_port_submission(last->ctx) ||
 				    ctx_single_port_submission(rq->ctx)) {
 					__list_del_many(&p->requests,
-							&rq->priotree.link);
+							&rq->sched.link);
 					goto done;
 				}
 
@@ -684,7 +706,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 				GEM_BUG_ON(port_isset(port));
 			}
 
-			INIT_LIST_HEAD(&rq->priotree.link);
+			INIT_LIST_HEAD(&rq->sched.link);
 			__i915_request_submit(rq);
 			trace_i915_request_in(rq, port_index(port, execlists));
 			last = rq;
@@ -697,8 +719,27 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 		if (p->priority != I915_PRIORITY_NORMAL)
 			kmem_cache_free(engine->i915->priorities, p);
 	}
+
 done:
-	execlists->queue_priority = rb ? to_priolist(rb)->priority : INT_MIN;
+	/*
+	 * Here be a bit of magic! Or sleight-of-hand, whichever you prefer.
+	 *
+	 * We choose queue_priority such that if we add a request of greater
+	 * priority than this, we kick the submission tasklet to decide on
+	 * the right order of submitting the requests to hardware. We must
+	 * also be prepared to reorder requests as they are in-flight on the
+	 * HW. We derive the queue_priority then as the first "hole" in
+	 * the HW submission ports and if there are no available slots,
+	 * the priority of the lowest executing request, i.e. last.
+	 *
+	 * When we do receive a higher priority request ready to run from the
+	 * user, see queue_request(), the queue_priority is bumped to that
+	 * request triggering preemption on the next dequeue (or subsequent
+	 * interrupt for secondary ports).
+	 */
+	execlists->queue_priority =
+		port != execlists->port ? rq_prio(last) : INT_MIN;
+
 	execlists->first = rb;
 	if (submit)
 		port_assign(port, last);
@@ -706,13 +747,25 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 	/* We must always keep the beast fed if we have work piled up */
 	GEM_BUG_ON(execlists->first && !port_isset(execlists->port));
 
-unlock:
-	spin_unlock_irq(&engine->timeline->lock);
+	/* Re-evaluate the executing context setup after each preemptive kick */
+	if (last)
+		execlists_user_begin(execlists, execlists->port);
 
-	if (submit) {
-		execlists_set_active(execlists, EXECLISTS_ACTIVE_USER);
+	return submit;
+}
+
+static void execlists_dequeue(struct intel_engine_cs *engine)
+{
+	struct intel_engine_execlists * const execlists = &engine->execlists;
+	unsigned long flags;
+	bool submit;
+
+	spin_lock_irqsave(&engine->timeline.lock, flags);
+	submit = __execlists_dequeue(engine);
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
+
+	if (submit)
 		execlists_submit_ports(engine);
-	}
 
 	GEM_BUG_ON(port_isset(execlists->port) &&
 		   !execlists_is_active(execlists, EXECLISTS_ACTIVE_USER));
@@ -727,13 +780,18 @@ execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
 	while (num_ports-- && port_isset(port)) {
 		struct i915_request *rq = port_request(port);
 
-		GEM_BUG_ON(!execlists->active);
-		intel_engine_context_out(rq->engine);
+		GEM_TRACE("%s:port%u global=%d (fence %llx:%d), (current %d)\n",
+			  rq->engine->name,
+			  (unsigned int)(port - execlists->port),
+			  rq->global_seqno,
+			  rq->fence.context, rq->fence.seqno,
+			  intel_engine_get_seqno(rq->engine));
 
-		execlists_context_status_change(rq,
-						i915_request_completed(rq) ?
-						INTEL_CONTEXT_SCHEDULE_OUT :
-						INTEL_CONTEXT_SCHEDULE_PREEMPTED);
+		GEM_BUG_ON(!execlists->active);
+		execlists_context_schedule_out(rq,
+					       i915_request_completed(rq) ?
+					       INTEL_CONTEXT_SCHEDULE_OUT :
+					       INTEL_CONTEXT_SCHEDULE_PREEMPTED);
 
 		i915_request_put(rq);
 
@@ -741,7 +799,82 @@ execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
 		port++;
 	}
 
-	execlists_clear_active(execlists, EXECLISTS_ACTIVE_USER);
+	execlists_user_end(execlists);
+}
+
+static void clear_gtiir(struct intel_engine_cs *engine)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	int i;
+
+	/*
+	 * Clear any pending interrupt state.
+	 *
+	 * We do it twice out of paranoia that some of the IIR are
+	 * double buffered, and so if we only reset it once there may
+	 * still be an interrupt pending.
+	 */
+	if (INTEL_GEN(dev_priv) >= 11) {
+		static const struct {
+			u8 bank;
+			u8 bit;
+		} gen11_gtiir[] = {
+			[RCS] = {0, GEN11_RCS0},
+			[BCS] = {0, GEN11_BCS},
+			[_VCS(0)] = {1, GEN11_VCS(0)},
+			[_VCS(1)] = {1, GEN11_VCS(1)},
+			[_VCS(2)] = {1, GEN11_VCS(2)},
+			[_VCS(3)] = {1, GEN11_VCS(3)},
+			[_VECS(0)] = {1, GEN11_VECS(0)},
+			[_VECS(1)] = {1, GEN11_VECS(1)},
+		};
+		unsigned long irqflags;
+
+		GEM_BUG_ON(engine->id >= ARRAY_SIZE(gen11_gtiir));
+
+		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+		for (i = 0; i < 2; i++) {
+			gen11_reset_one_iir(dev_priv,
+					    gen11_gtiir[engine->id].bank,
+					    gen11_gtiir[engine->id].bit);
+		}
+		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+	} else {
+		static const u8 gtiir[] = {
+			[RCS]  = 0,
+			[BCS]  = 0,
+			[VCS]  = 1,
+			[VCS2] = 1,
+			[VECS] = 3,
+		};
+
+		GEM_BUG_ON(engine->id >= ARRAY_SIZE(gtiir));
+
+		for (i = 0; i < 2; i++) {
+			I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]),
+				   engine->irq_keep_mask);
+			POSTING_READ(GEN8_GT_IIR(gtiir[engine->id]));
+		}
+		GEM_BUG_ON(I915_READ(GEN8_GT_IIR(gtiir[engine->id])) &
+			   engine->irq_keep_mask);
+	}
+}
+
+static void reset_irq(struct intel_engine_cs *engine)
+{
+	/* Mark all CS interrupts as complete */
+	smp_store_mb(engine->execlists.active, 0);
+	synchronize_hardirq(engine->i915->drm.irq);
+
+	clear_gtiir(engine);
+
+	/*
+	 * The port is checked prior to scheduling a tasklet, but
+	 * just in case we have suspended the tasklet to do the
+	 * wedging make sure that when it wakes, it decides there
+	 * is no work to do by clearing the irq_posted bit.
+	 */
+	clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
 }
 
 static void execlists_cancel_requests(struct intel_engine_cs *engine)
@@ -751,7 +884,8 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
 	struct rb_node *rb;
 	unsigned long flags;
 
-	GEM_TRACE("%s\n", engine->name);
+	GEM_TRACE("%s current %d\n",
+		  engine->name, intel_engine_get_seqno(engine));
 
 	/*
 	 * Before we call engine->cancel_requests(), we should have exclusive
@@ -771,11 +905,12 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
 
 	/* Cancel the requests on the HW and clear the ELSP tracker. */
 	execlists_cancel_port_requests(execlists);
+	reset_irq(engine);
 
-	spin_lock(&engine->timeline->lock);
+	spin_lock(&engine->timeline.lock);
 
 	/* Mark all executing requests as skipped. */
-	list_for_each_entry(rq, &engine->timeline->requests, link) {
+	list_for_each_entry(rq, &engine->timeline.requests, link) {
 		GEM_BUG_ON(!rq->global_seqno);
 		if (!i915_request_completed(rq))
 			dma_fence_set_error(&rq->fence, -EIO);
@@ -786,8 +921,8 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
 	while (rb) {
 		struct i915_priolist *p = to_priolist(rb);
 
-		list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
-			INIT_LIST_HEAD(&rq->priotree.link);
+		list_for_each_entry_safe(rq, rn, &p->requests, sched.link) {
+			INIT_LIST_HEAD(&rq->sched.link);
 
 			dma_fence_set_error(&rq->fence, -EIO);
 			__i915_request_submit(rq);
@@ -807,18 +942,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
 	execlists->first = NULL;
 	GEM_BUG_ON(port_isset(execlists->port));
 
-	spin_unlock(&engine->timeline->lock);
-
-	/*
-	 * The port is checked prior to scheduling a tasklet, but
-	 * just in case we have suspended the tasklet to do the
-	 * wedging make sure that when it wakes, it decides there
-	 * is no work to do by clearing the irq_posted bit.
-	 */
-	clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
-
-	/* Mark all CS interrupts as complete */
-	execlists->active = 0;
+	spin_unlock(&engine->timeline.lock);
 
 	local_irq_restore(flags);
 }
@@ -831,7 +955,7 @@ static void execlists_submission_tasklet(unsigned long data)
 {
 	struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
 	struct intel_engine_execlists * const execlists = &engine->execlists;
-	struct execlist_port * const port = execlists->port;
+	struct execlist_port *port = execlists->port;
 	struct drm_i915_private *dev_priv = engine->i915;
 	bool fw = false;
 
@@ -959,10 +1083,13 @@ static void execlists_submission_tasklet(unsigned long data)
 							EXECLISTS_ACTIVE_USER));
 
 			rq = port_unpack(port, &count);
-			GEM_TRACE("%s out[0]: ctx=%d.%d, seqno=%x, prio=%d\n",
+			GEM_TRACE("%s out[0]: ctx=%d.%d, global=%d (fence %llx:%d) (current %d), prio=%d\n",
 				  engine->name,
 				  port->context_id, count,
 				  rq ? rq->global_seqno : 0,
+				  rq ? rq->fence.context : 0,
+				  rq ? rq->fence.seqno : 0,
+				  intel_engine_get_seqno(engine),
 				  rq ? rq_prio(rq) : 0);
 
 			/* Check the context/desc id for this event matches */
@@ -970,28 +1097,43 @@ static void execlists_submission_tasklet(unsigned long data)
 
 			GEM_BUG_ON(count == 0);
 			if (--count == 0) {
+				/*
+				 * On the final event corresponding to the
+				 * submission of this context, we expect either
+				 * an element-switch event or a completion
+				 * event (and on completion, the active-idle
+				 * marker). No more preemptions, lite-restore
+				 * or otherwise.
+				 */
 				GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
 				GEM_BUG_ON(port_isset(&port[1]) &&
 					   !(status & GEN8_CTX_STATUS_ELEMENT_SWITCH));
+				GEM_BUG_ON(!port_isset(&port[1]) &&
+					   !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
+
+				/*
+				 * We rely on the hardware being strongly
+				 * ordered, that the breadcrumb write is
+				 * coherent (visible from the CPU) before the
+				 * user interrupt and CSB is processed.
+				 */
 				GEM_BUG_ON(!i915_request_completed(rq));
-				execlists_context_schedule_out(rq);
-				trace_i915_request_out(rq);
+
+				execlists_context_schedule_out(rq,
+							       INTEL_CONTEXT_SCHEDULE_OUT);
 				i915_request_put(rq);
 
 				GEM_TRACE("%s completed ctx=%d\n",
 					  engine->name, port->context_id);
 
-				execlists_port_complete(execlists, port);
+				port = execlists_port_complete(execlists, port);
+				if (port_isset(port))
+					execlists_user_begin(execlists, port);
+				else
+					execlists_user_end(execlists);
 			} else {
 				port_set(port, port_pack(rq, count));
 			}
-
-			/* After the final element, the hw should be idle */
-			GEM_BUG_ON(port_count(port) == 0 &&
-				   !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
-			if (port_count(port) == 0)
-				execlists_clear_active(execlists,
-						       EXECLISTS_ACTIVE_USER);
 		}
 
 		if (head != execlists->csb_head) {
@@ -1014,18 +1156,23 @@ static void execlists_submission_tasklet(unsigned long data)
 }
 
 static void queue_request(struct intel_engine_cs *engine,
-			  struct i915_priotree *pt,
+			  struct i915_sched_node *node,
 			  int prio)
 {
-	list_add_tail(&pt->link, &lookup_priolist(engine, pt, prio)->requests);
+	list_add_tail(&node->link,
+		      &lookup_priolist(engine, prio)->requests);
+}
+
+static void __submit_queue(struct intel_engine_cs *engine, int prio)
+{
+	engine->execlists.queue_priority = prio;
+	tasklet_hi_schedule(&engine->execlists.tasklet);
 }
 
 static void submit_queue(struct intel_engine_cs *engine, int prio)
 {
-	if (prio > engine->execlists.queue_priority) {
-		engine->execlists.queue_priority = prio;
-		tasklet_hi_schedule(&engine->execlists.tasklet);
-	}
+	if (prio > engine->execlists.queue_priority)
+		__submit_queue(engine, prio);
 }
 
 static void execlists_submit_request(struct i915_request *request)
@@ -1034,42 +1181,45 @@ static void execlists_submit_request(struct i915_request *request)
 	unsigned long flags;
 
 	/* Will be called from irq-context when using foreign fences. */
-	spin_lock_irqsave(&engine->timeline->lock, flags);
+	spin_lock_irqsave(&engine->timeline.lock, flags);
 
-	queue_request(engine, &request->priotree, rq_prio(request));
+	queue_request(engine, &request->sched, rq_prio(request));
 	submit_queue(engine, rq_prio(request));
 
 	GEM_BUG_ON(!engine->execlists.first);
-	GEM_BUG_ON(list_empty(&request->priotree.link));
+	GEM_BUG_ON(list_empty(&request->sched.link));
 
-	spin_unlock_irqrestore(&engine->timeline->lock, flags);
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
-static struct i915_request *pt_to_request(struct i915_priotree *pt)
+static struct i915_request *sched_to_request(struct i915_sched_node *node)
 {
-	return container_of(pt, struct i915_request, priotree);
+	return container_of(node, struct i915_request, sched);
 }
 
 static struct intel_engine_cs *
-pt_lock_engine(struct i915_priotree *pt, struct intel_engine_cs *locked)
+sched_lock_engine(struct i915_sched_node *node, struct intel_engine_cs *locked)
 {
-	struct intel_engine_cs *engine = pt_to_request(pt)->engine;
+	struct intel_engine_cs *engine = sched_to_request(node)->engine;
 
 	GEM_BUG_ON(!locked);
 
 	if (engine != locked) {
-		spin_unlock(&locked->timeline->lock);
-		spin_lock(&engine->timeline->lock);
+		spin_unlock(&locked->timeline.lock);
+		spin_lock(&engine->timeline.lock);
 	}
 
 	return engine;
 }
 
-static void execlists_schedule(struct i915_request *request, int prio)
+static void execlists_schedule(struct i915_request *request,
+			       const struct i915_sched_attr *attr)
 {
-	struct intel_engine_cs *engine;
+	struct i915_priolist *uninitialized_var(pl);
+	struct intel_engine_cs *engine, *last;
 	struct i915_dependency *dep, *p;
 	struct i915_dependency stack;
+	const int prio = attr->priority;
 	LIST_HEAD(dfs);
 
 	GEM_BUG_ON(prio == I915_PRIORITY_INVALID);
@@ -1077,23 +1227,23 @@ static void execlists_schedule(struct i915_request *request, int prio)
 	if (i915_request_completed(request))
 		return;
 
-	if (prio <= READ_ONCE(request->priotree.priority))
+	if (prio <= READ_ONCE(request->sched.attr.priority))
 		return;
 
 	/* Need BKL in order to use the temporary link inside i915_dependency */
 	lockdep_assert_held(&request->i915->drm.struct_mutex);
 
-	stack.signaler = &request->priotree;
+	stack.signaler = &request->sched;
 	list_add(&stack.dfs_link, &dfs);
 
 	/*
 	 * Recursively bump all dependent priorities to match the new request.
 	 *
 	 * A naive approach would be to use recursion:
-	 * static void update_priorities(struct i915_priotree *pt, prio) {
-	 *	list_for_each_entry(dep, &pt->signalers_list, signal_link)
+	 * static void update_priorities(struct i915_sched_node *node, prio) {
+	 *	list_for_each_entry(dep, &node->signalers_list, signal_link)
 	 *		update_priorities(dep->signal, prio)
-	 *	queue_request(pt);
+	 *	queue_request(node);
 	 * }
 	 * but that may have unlimited recursion depth and so runs a very
 	 * real risk of overunning the kernel stack. Instead, we build
@@ -1105,7 +1255,7 @@ static void execlists_schedule(struct i915_request *request, int prio)
 	 * last element in the list is the request we must execute first.
 	 */
 	list_for_each_entry(dep, &dfs, dfs_link) {
-		struct i915_priotree *pt = dep->signaler;
+		struct i915_sched_node *node = dep->signaler;
 
 		/*
 		 * Within an engine, there can be no cycle, but we may
@@ -1113,14 +1263,14 @@ static void execlists_schedule(struct i915_request *request, int prio)
 		 * (redundant dependencies are not eliminated) and across
 		 * engines.
 		 */
-		list_for_each_entry(p, &pt->signalers_list, signal_link) {
+		list_for_each_entry(p, &node->signalers_list, signal_link) {
 			GEM_BUG_ON(p == dep); /* no cycles! */
 
-			if (i915_priotree_signaled(p->signaler))
+			if (i915_sched_node_signaled(p->signaler))
 				continue;
 
-			GEM_BUG_ON(p->signaler->priority < pt->priority);
-			if (prio > READ_ONCE(p->signaler->priority))
+			GEM_BUG_ON(p->signaler->attr.priority < node->attr.priority);
+			if (prio > READ_ONCE(p->signaler->attr.priority))
 				list_move_tail(&p->dfs_link, &dfs);
 		}
 	}
@@ -1131,37 +1281,45 @@ static void execlists_schedule(struct i915_request *request, int prio)
 	 * execlists_submit_request()), we can set our own priority and skip
 	 * acquiring the engine locks.
 	 */
-	if (request->priotree.priority == I915_PRIORITY_INVALID) {
-		GEM_BUG_ON(!list_empty(&request->priotree.link));
-		request->priotree.priority = prio;
+	if (request->sched.attr.priority == I915_PRIORITY_INVALID) {
+		GEM_BUG_ON(!list_empty(&request->sched.link));
+		request->sched.attr = *attr;
 		if (stack.dfs_link.next == stack.dfs_link.prev)
 			return;
 		__list_del_entry(&stack.dfs_link);
 	}
 
+	last = NULL;
 	engine = request->engine;
-	spin_lock_irq(&engine->timeline->lock);
+	spin_lock_irq(&engine->timeline.lock);
 
 	/* Fifo and depth-first replacement ensure our deps execute before us */
 	list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) {
-		struct i915_priotree *pt = dep->signaler;
+		struct i915_sched_node *node = dep->signaler;
 
 		INIT_LIST_HEAD(&dep->dfs_link);
 
-		engine = pt_lock_engine(pt, engine);
+		engine = sched_lock_engine(node, engine);
 
-		if (prio <= pt->priority)
+		if (prio <= node->attr.priority)
 			continue;
 
-		pt->priority = prio;
-		if (!list_empty(&pt->link)) {
-			__list_del_entry(&pt->link);
-			queue_request(engine, pt, prio);
+		node->attr.priority = prio;
+		if (!list_empty(&node->link)) {
+			if (last != engine) {
+				pl = lookup_priolist(engine, prio);
+				last = engine;
+			}
+			GEM_BUG_ON(pl->priority != prio);
+			list_move_tail(&node->link, &pl->requests);
 		}
-		submit_queue(engine, prio);
+
+		if (prio > engine->execlists.queue_priority &&
+		    i915_sw_fence_done(&sched_to_request(node)->submit))
+			__submit_queue(engine, prio);
 	}
 
-	spin_unlock_irq(&engine->timeline->lock);
+	spin_unlock_irq(&engine->timeline.lock);
 }
 
 static int __context_pin(struct i915_gem_context *ctx, struct i915_vma *vma)
@@ -1191,7 +1349,7 @@ static struct intel_ring *
 execlists_context_pin(struct intel_engine_cs *engine,
 		      struct i915_gem_context *ctx)
 {
-	struct intel_context *ce = &ctx->engine[engine->id];
+	struct intel_context *ce = to_intel_context(ctx, engine);
 	void *vaddr;
 	int ret;
 
@@ -1225,6 +1383,7 @@ execlists_context_pin(struct intel_engine_cs *engine,
 	ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
 	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
 		i915_ggtt_offset(ce->ring->vma);
+	ce->lrc_reg_state[CTX_RING_HEAD+1] = ce->ring->head;
 
 	ce->state->obj->pin_global++;
 	i915_gem_context_get(ctx);
@@ -1243,7 +1402,7 @@ execlists_context_pin(struct intel_engine_cs *engine,
 static void execlists_context_unpin(struct intel_engine_cs *engine,
 				    struct i915_gem_context *ctx)
 {
-	struct intel_context *ce = &ctx->engine[engine->id];
+	struct intel_context *ce = to_intel_context(ctx, engine);
 
 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
 	GEM_BUG_ON(ce->pin_count == 0);
@@ -1262,8 +1421,8 @@ static void execlists_context_unpin(struct intel_engine_cs *engine,
 
 static int execlists_request_alloc(struct i915_request *request)
 {
-	struct intel_engine_cs *engine = request->engine;
-	struct intel_context *ce = &request->ctx->engine[engine->id];
+	struct intel_context *ce =
+		to_intel_context(request->ctx, request->engine);
 	int ret;
 
 	GEM_BUG_ON(!ce->pin_count);
@@ -1523,6 +1682,8 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
 		return -EINVAL;
 
 	switch (INTEL_GEN(engine->i915)) {
+	case 11:
+		return 0;
 	case 10:
 		wa_bb_fn[0] = gen10_init_indirectctx_bb;
 		wa_bb_fn[1] = NULL;
@@ -1575,14 +1736,6 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
 	return ret;
 }
 
-static u8 gtiir[] = {
-	[RCS] = 0,
-	[BCS] = 0,
-	[VCS] = 1,
-	[VCS2] = 1,
-	[VECS] = 3,
-};
-
 static void enable_execlists(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
@@ -1642,6 +1795,8 @@ static int gen8_init_render_ring(struct intel_engine_cs *engine)
 	if (ret)
 		return ret;
 
+	intel_whitelist_workarounds_apply(engine);
+
 	/* We need to disable the AsyncFlip performance optimisations in order
 	 * to use MI_WAIT_FOR_EVENT within the CS. It should already be
 	 * programmed to '1' on all products.
@@ -1652,7 +1807,7 @@ static int gen8_init_render_ring(struct intel_engine_cs *engine)
 
 	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 
-	return init_workarounds_ring(engine);
+	return 0;
 }
 
 static int gen9_init_render_ring(struct intel_engine_cs *engine)
@@ -1663,49 +1818,25 @@ static int gen9_init_render_ring(struct intel_engine_cs *engine)
 	if (ret)
 		return ret;
 
-	return init_workarounds_ring(engine);
-}
-
-static void reset_irq(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int i;
-
-	GEM_BUG_ON(engine->id >= ARRAY_SIZE(gtiir));
-
-	/*
-	 * Clear any pending interrupt state.
-	 *
-	 * We do it twice out of paranoia that some of the IIR are double
-	 * buffered, and if we only reset it once there may still be
-	 * an interrupt pending.
-	 */
-	for (i = 0; i < 2; i++) {
-		I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]),
-			   GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift);
-		POSTING_READ(GEN8_GT_IIR(gtiir[engine->id]));
-	}
-	GEM_BUG_ON(I915_READ(GEN8_GT_IIR(gtiir[engine->id])) &
-		   (GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift));
+	intel_whitelist_workarounds_apply(engine);
 
-	clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
+	return 0;
 }
 
 static void reset_common_ring(struct intel_engine_cs *engine,
 			      struct i915_request *request)
 {
 	struct intel_engine_execlists * const execlists = &engine->execlists;
-	struct intel_context *ce;
 	unsigned long flags;
+	u32 *regs;
 
-	GEM_TRACE("%s seqno=%x\n",
-		  engine->name, request ? request->global_seqno : 0);
+	GEM_TRACE("%s request global=%x, current=%d\n",
+		  engine->name, request ? request->global_seqno : 0,
+		  intel_engine_get_seqno(engine));
 
 	/* See execlists_cancel_requests() for the irq/spinlock split. */
 	local_irq_save(flags);
 
-	reset_irq(engine);
-
 	/*
 	 * Catch up with any missed context-switch interrupts.
 	 *
@@ -1716,14 +1847,12 @@ static void reset_common_ring(struct intel_engine_cs *engine,
 	 * requests were completed.
 	 */
 	execlists_cancel_port_requests(execlists);
+	reset_irq(engine);
 
 	/* Push back any incomplete requests for replay after the reset. */
-	spin_lock(&engine->timeline->lock);
+	spin_lock(&engine->timeline.lock);
 	__unwind_incomplete_requests(engine);
-	spin_unlock(&engine->timeline->lock);
-
-	/* Mark all CS interrupts as complete */
-	execlists->active = 0;
+	spin_unlock(&engine->timeline.lock);
 
 	local_irq_restore(flags);
 
@@ -1749,14 +1878,24 @@ static void reset_common_ring(struct intel_engine_cs *engine,
 	 * future request will be after userspace has had the opportunity
 	 * to recreate its own state.
 	 */
-	ce = &request->ctx->engine[engine->id];
-	execlists_init_reg_state(ce->lrc_reg_state,
-				 request->ctx, engine, ce->ring);
+	regs = to_intel_context(request->ctx, engine)->lrc_reg_state;
+	if (engine->default_state) {
+		void *defaults;
+
+		defaults = i915_gem_object_pin_map(engine->default_state,
+						   I915_MAP_WB);
+		if (!IS_ERR(defaults)) {
+			memcpy(regs, /* skip restoring the vanilla PPHWSP */
+			       defaults + LRC_STATE_PN * PAGE_SIZE,
+			       engine->context_size - PAGE_SIZE);
+			i915_gem_object_unpin_map(engine->default_state);
+		}
+	}
+	execlists_init_reg_state(regs, request->ctx, engine, request->ring);
 
 	/* Move the RING_HEAD onto the breadcrumb, past the hanging batch */
-	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
-		i915_ggtt_offset(ce->ring->vma);
-	ce->lrc_reg_state[CTX_RING_HEAD+1] = request->postfix;
+	regs[CTX_RING_BUFFER_START + 1] = i915_ggtt_offset(request->ring->vma);
+	regs[CTX_RING_HEAD + 1] = request->postfix;
 
 	request->ring->head = request->postfix;
 	intel_ring_update_space(request->ring);
@@ -1817,7 +1956,7 @@ static int gen8_emit_bb_start(struct i915_request *rq,
 		rq->ctx->ppgtt->pd_dirty_rings &= ~intel_engine_flag(rq->engine);
 	}
 
-	cs = intel_ring_begin(rq, 4);
+	cs = intel_ring_begin(rq, 6);
 	if (IS_ERR(cs))
 		return PTR_ERR(cs);
 
@@ -1846,6 +1985,9 @@ static int gen8_emit_bb_start(struct i915_request *rq,
 		(flags & I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0);
 	*cs++ = lower_32_bits(offset);
 	*cs++ = upper_32_bits(offset);
+
+	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
+	*cs++ = MI_NOOP;
 	intel_ring_advance(rq, cs);
 
 	return 0;
@@ -1988,7 +2130,7 @@ static void gen8_emit_breadcrumb(struct i915_request *request, u32 *cs)
 	cs = gen8_emit_ggtt_write(cs, request->global_seqno,
 				  intel_hws_seqno_address(request->engine));
 	*cs++ = MI_USER_INTERRUPT;
-	*cs++ = MI_NOOP;
+	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 	request->tail = intel_ring_offset(request, cs);
 	assert_ring_tail_valid(request->ring, request->tail);
 
@@ -2004,7 +2146,7 @@ static void gen8_emit_breadcrumb_rcs(struct i915_request *request, u32 *cs)
 	cs = gen8_emit_ggtt_write_rcs(cs, request->global_seqno,
 				      intel_hws_seqno_address(request->engine));
 	*cs++ = MI_USER_INTERRUPT;
-	*cs++ = MI_NOOP;
+	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 	request->tail = intel_ring_offset(request, cs);
 	assert_ring_tail_valid(request->ring, request->tail);
 
@@ -2016,7 +2158,7 @@ static int gen8_init_rcs_context(struct i915_request *rq)
 {
 	int ret;
 
-	ret = intel_ring_workarounds_emit(rq);
+	ret = intel_ctx_workarounds_emit(rq);
 	if (ret)
 		return ret;
 
@@ -2076,11 +2218,13 @@ static void execlists_set_default_submission(struct intel_engine_cs *engine)
 	engine->unpark = NULL;
 
 	engine->flags |= I915_ENGINE_SUPPORTS_STATS;
+	if (engine->i915->preempt_context)
+		engine->flags |= I915_ENGINE_HAS_PREEMPTION;
 
 	engine->i915->caps.scheduler =
 		I915_SCHEDULER_CAP_ENABLED |
 		I915_SCHEDULER_CAP_PRIORITY;
-	if (engine->i915->preempt_context)
+	if (intel_engine_has_preemption(engine))
 		engine->i915->caps.scheduler |= I915_SCHEDULER_CAP_PREEMPTION;
 }
 
@@ -2119,7 +2263,20 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine)
 static inline void
 logical_ring_default_irqs(struct intel_engine_cs *engine)
 {
-	unsigned shift = engine->irq_shift;
+	unsigned int shift = 0;
+
+	if (INTEL_GEN(engine->i915) < 11) {
+		const u8 irq_shifts[] = {
+			[RCS]  = GEN8_RCS_IRQ_SHIFT,
+			[BCS]  = GEN8_BCS_IRQ_SHIFT,
+			[VCS]  = GEN8_VCS1_IRQ_SHIFT,
+			[VCS2] = GEN8_VCS2_IRQ_SHIFT,
+			[VECS] = GEN8_VECS_IRQ_SHIFT,
+		};
+
+		shift = irq_shifts[engine->id];
+	}
+
 	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
 	engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
 }
@@ -2175,9 +2332,13 @@ static int logical_ring_init(struct intel_engine_cs *engine)
 	}
 
 	engine->execlists.preempt_complete_status = ~0u;
-	if (engine->i915->preempt_context)
+	if (engine->i915->preempt_context) {
+		struct intel_context *ce =
+			to_intel_context(engine->i915->preempt_context, engine);
+
 		engine->execlists.preempt_complete_status =
-			upper_32_bits(engine->i915->preempt_context->engine[engine->id].lrc_desc);
+			upper_32_bits(ce->lrc_desc);
+	}
 
 	return 0;
 
@@ -2431,8 +2592,10 @@ populate_lr_context(struct i915_gem_context *ctx,
 
 		defaults = i915_gem_object_pin_map(engine->default_state,
 						   I915_MAP_WB);
-		if (IS_ERR(defaults))
-			return PTR_ERR(defaults);
+		if (IS_ERR(defaults)) {
+			ret = PTR_ERR(defaults);
+			goto err_unpin_ctx;
+		}
 
 		memcpy(vaddr + start, defaults + start, engine->context_size);
 		i915_gem_object_unpin_map(engine->default_state);
@@ -2450,19 +2613,20 @@ populate_lr_context(struct i915_gem_context *ctx,
 			_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
 					   CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT);
 
+err_unpin_ctx:
 	i915_gem_object_unpin_map(ctx_obj);
-
-	return 0;
+	return ret;
 }
 
 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 					    struct intel_engine_cs *engine)
 {
 	struct drm_i915_gem_object *ctx_obj;
-	struct intel_context *ce = &ctx->engine[engine->id];
+	struct intel_context *ce = to_intel_context(ctx, engine);
 	struct i915_vma *vma;
 	uint32_t context_size;
 	struct intel_ring *ring;
+	struct i915_timeline *timeline;
 	int ret;
 
 	if (ce->state)
@@ -2478,8 +2642,8 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 
 	ctx_obj = i915_gem_object_create(ctx->i915, context_size);
 	if (IS_ERR(ctx_obj)) {
-		DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n");
-		return PTR_ERR(ctx_obj);
+		ret = PTR_ERR(ctx_obj);
+		goto error_deref_obj;
 	}
 
 	vma = i915_vma_instance(ctx_obj, &ctx->i915->ggtt.base, NULL);
@@ -2488,7 +2652,14 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 		goto error_deref_obj;
 	}
 
-	ring = intel_engine_create_ring(engine, ctx->ring_size);
+	timeline = i915_timeline_create(ctx->i915, ctx->name);
+	if (IS_ERR(timeline)) {
+		ret = PTR_ERR(timeline);
+		goto error_deref_obj;
+	}
+
+	ring = intel_engine_create_ring(engine, timeline, ctx->ring_size);
+	i915_timeline_put(timeline);
 	if (IS_ERR(ring)) {
 		ret = PTR_ERR(ring);
 		goto error_deref_obj;
@@ -2530,7 +2701,8 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv)
 	 */
 	list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
 		for_each_engine(engine, dev_priv, id) {
-			struct intel_context *ce = &ctx->engine[engine->id];
+			struct intel_context *ce =
+				to_intel_context(ctx, engine);
 			u32 *reg;
 
 			if (!ce->state)
@@ -2552,3 +2724,7 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv)
 		}
 	}
 }
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftests/intel_lrc.c"
+#endif
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index 59d7b86012e9..4ec7d8dd13c8 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -108,7 +108,7 @@ static inline uint64_t
 intel_lr_context_descriptor(struct i915_gem_context *ctx,
 			    struct intel_engine_cs *engine)
 {
-	return ctx->engine[engine->id].lrc_desc;
+	return to_intel_context(ctx, engine)->lrc_desc;
 }
 
 #endif /* _INTEL_LRC_H_ */
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 8691c86f579c..e125d16a1aa7 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -574,6 +574,36 @@ static int intel_lid_notify(struct notifier_block *nb, unsigned long val,
 	return NOTIFY_OK;
 }
 
+static int
+intel_lvds_connector_register(struct drm_connector *connector)
+{
+	struct intel_lvds_connector *lvds = to_lvds_connector(connector);
+	int ret;
+
+	ret = intel_connector_register(connector);
+	if (ret)
+		return ret;
+
+	lvds->lid_notifier.notifier_call = intel_lid_notify;
+	if (acpi_lid_notifier_register(&lvds->lid_notifier)) {
+		DRM_DEBUG_KMS("lid notifier registration failed\n");
+		lvds->lid_notifier.notifier_call = NULL;
+	}
+
+	return 0;
+}
+
+static void
+intel_lvds_connector_unregister(struct drm_connector *connector)
+{
+	struct intel_lvds_connector *lvds = to_lvds_connector(connector);
+
+	if (lvds->lid_notifier.notifier_call)
+		acpi_lid_notifier_unregister(&lvds->lid_notifier);
+
+	intel_connector_unregister(connector);
+}
+
 /**
  * intel_lvds_destroy - unregister and free LVDS structures
  * @connector: connector to free
@@ -586,9 +616,6 @@ static void intel_lvds_destroy(struct drm_connector *connector)
 	struct intel_lvds_connector *lvds_connector =
 		to_lvds_connector(connector);
 
-	if (lvds_connector->lid_notifier.notifier_call)
-		acpi_lid_notifier_unregister(&lvds_connector->lid_notifier);
-
 	if (!IS_ERR_OR_NULL(lvds_connector->base.edid))
 		kfree(lvds_connector->base.edid);
 
@@ -609,8 +636,8 @@ static const struct drm_connector_funcs intel_lvds_connector_funcs = {
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.atomic_get_property = intel_digital_connector_atomic_get_property,
 	.atomic_set_property = intel_digital_connector_atomic_set_property,
-	.late_register = intel_connector_register,
-	.early_unregister = intel_connector_unregister,
+	.late_register = intel_lvds_connector_register,
+	.early_unregister = intel_lvds_connector_unregister,
 	.destroy = intel_lvds_destroy,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 	.atomic_duplicate_state = intel_digital_connector_duplicate_state,
@@ -827,6 +854,14 @@ static const struct dmi_system_id intel_no_lvds[] = {
 			DMI_EXACT_MATCH(DMI_BOARD_NAME, "D525MW"),
 		},
 	},
+	{
+		.callback = intel_no_lvds_dmi_callback,
+		.ident = "Radiant P845",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Radiant Systems Inc"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "P845"),
+		},
+	},
 
 	{ }	/* terminating entry */
 };
@@ -1150,12 +1185,6 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 
 	lvds_encoder->a3_power = lvds & LVDS_A3_POWER_MASK;
 
-	lvds_connector->lid_notifier.notifier_call = intel_lid_notify;
-	if (acpi_lid_notifier_register(&lvds_connector->lid_notifier)) {
-		DRM_DEBUG_KMS("lid notifier registration failed\n");
-		lvds_connector->lid_notifier.notifier_call = NULL;
-	}
-
 	return;
 
 failed:
diff --git a/drivers/gpu/drm/i915/intel_mocs.c b/drivers/gpu/drm/i915/intel_mocs.c
index c0b34b7943b9..9f0bd6a4cb79 100644
--- a/drivers/gpu/drm/i915/intel_mocs.c
+++ b/drivers/gpu/drm/i915/intel_mocs.c
@@ -178,7 +178,8 @@ static bool get_mocs_settings(struct drm_i915_private *dev_priv,
 {
 	bool result = false;
 
-	if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+	if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv) ||
+	    IS_ICELAKE(dev_priv)) {
 		table->size  = ARRAY_SIZE(skylake_mocs_table);
 		table->table = skylake_mocs_table;
 		result = true;
@@ -217,6 +218,8 @@ static i915_reg_t mocs_register(enum intel_engine_id engine_id, int index)
 		return GEN9_VEBOX_MOCS(index);
 	case VCS2:
 		return GEN9_MFX1_MOCS(index);
+	case VCS3:
+		return GEN11_MFX2_MOCS(index);
 	default:
 		MISSING_CASE(engine_id);
 		return INVALID_MMIO_REG;
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 36671a937fa4..c2f10d899329 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -807,6 +807,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 		ret = PTR_ERR(vma);
 		goto out_pin_section;
 	}
+	intel_fb_obj_flush(new_bo, ORIGIN_DIRTYFB);
 
 	ret = i915_vma_put_fence(vma);
 	if (ret)
diff --git a/drivers/gpu/drm/i915/intel_pipe_crc.c b/drivers/gpu/drm/i915/intel_pipe_crc.c
index 1f5cd572a7ff..39a4e4edda07 100644
--- a/drivers/gpu/drm/i915/intel_pipe_crc.c
+++ b/drivers/gpu/drm/i915/intel_pipe_crc.c
@@ -569,7 +569,8 @@ static void hsw_pipe_A_crc_wa(struct drm_i915_private *dev_priv,
 static int ivb_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
 				enum pipe pipe,
 				enum intel_pipe_crc_source *source,
-				uint32_t *val)
+				uint32_t *val,
+				bool set_wa)
 {
 	if (*source == INTEL_PIPE_CRC_SOURCE_AUTO)
 		*source = INTEL_PIPE_CRC_SOURCE_PF;
@@ -582,7 +583,7 @@ static int ivb_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_SPRITE_IVB;
 		break;
 	case INTEL_PIPE_CRC_SOURCE_PF:
-		if ((IS_HASWELL(dev_priv) ||
+		if (set_wa && (IS_HASWELL(dev_priv) ||
 		     IS_BROADWELL(dev_priv)) && pipe == PIPE_A)
 			hsw_pipe_A_crc_wa(dev_priv, true);
 
@@ -600,7 +601,8 @@ static int ivb_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
 
 static int get_new_crc_ctl_reg(struct drm_i915_private *dev_priv,
 			       enum pipe pipe,
-			       enum intel_pipe_crc_source *source, u32 *val)
+			       enum intel_pipe_crc_source *source, u32 *val,
+			       bool set_wa)
 {
 	if (IS_GEN2(dev_priv))
 		return i8xx_pipe_crc_ctl_reg(source, val);
@@ -611,7 +613,7 @@ static int get_new_crc_ctl_reg(struct drm_i915_private *dev_priv,
 	else if (IS_GEN5(dev_priv) || IS_GEN6(dev_priv))
 		return ilk_pipe_crc_ctl_reg(source, val);
 	else
-		return ivb_pipe_crc_ctl_reg(dev_priv, pipe, source, val);
+		return ivb_pipe_crc_ctl_reg(dev_priv, pipe, source, val, set_wa);
 }
 
 static int pipe_crc_set_source(struct drm_i915_private *dev_priv,
@@ -636,7 +638,7 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv,
 		return -EIO;
 	}
 
-	ret = get_new_crc_ctl_reg(dev_priv, pipe, &source, &val);
+	ret = get_new_crc_ctl_reg(dev_priv, pipe, &source, &val, true);
 	if (ret != 0)
 		goto out;
 
@@ -764,13 +766,12 @@ display_crc_ctl_parse_object(const char *buf, enum intel_pipe_crc_object *o)
 {
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(pipe_crc_objects); i++)
-		if (!strcmp(buf, pipe_crc_objects[i])) {
-			*o = i;
-			return 0;
-		}
+	i = match_string(pipe_crc_objects, ARRAY_SIZE(pipe_crc_objects), buf);
+	if (i < 0)
+		return i;
 
-	return -EINVAL;
+	*o = i;
+	return 0;
 }
 
 static int display_crc_ctl_parse_pipe(struct drm_i915_private *dev_priv,
@@ -796,13 +797,12 @@ display_crc_ctl_parse_source(const char *buf, enum intel_pipe_crc_source *s)
 		return 0;
 	}
 
-	for (i = 0; i < ARRAY_SIZE(pipe_crc_sources); i++)
-		if (!strcmp(buf, pipe_crc_sources[i])) {
-			*s = i;
-			return 0;
-		}
+	i = match_string(pipe_crc_sources, ARRAY_SIZE(pipe_crc_sources), buf);
+	if (i < 0)
+		return i;
 
-	return -EINVAL;
+	*s = i;
+	return 0;
 }
 
 static int display_crc_ctl_parse(struct drm_i915_private *dev_priv,
@@ -916,7 +916,7 @@ int intel_pipe_crc_create(struct drm_minor *minor)
 int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name,
 			      size_t *values_cnt)
 {
-	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[crtc->index];
 	enum intel_display_power_domain power_domain;
 	enum intel_pipe_crc_source source;
@@ -934,10 +934,11 @@ int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name,
 		return -EIO;
 	}
 
-	ret = get_new_crc_ctl_reg(dev_priv, crtc->index, &source, &val);
+	ret = get_new_crc_ctl_reg(dev_priv, crtc->index, &source, &val, true);
 	if (ret != 0)
 		goto out;
 
+	pipe_crc->source = source;
 	I915_WRITE(PIPE_CRC_CTL(crtc->index), val);
 	POSTING_READ(PIPE_CRC_CTL(crtc->index));
 
@@ -959,3 +960,39 @@ int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name,
 
 	return ret;
 }
+
+void intel_crtc_enable_pipe_crc(struct intel_crtc *intel_crtc)
+{
+	struct drm_crtc *crtc = &intel_crtc->base;
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
+	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[crtc->index];
+	u32 val = 0;
+
+	if (!crtc->crc.opened)
+		return;
+
+	if (get_new_crc_ctl_reg(dev_priv, crtc->index, &pipe_crc->source, &val, false) < 0)
+		return;
+
+	/* Don't need pipe_crc->lock here, IRQs are not generated. */
+	pipe_crc->skipped = 0;
+
+	I915_WRITE(PIPE_CRC_CTL(crtc->index), val);
+	POSTING_READ(PIPE_CRC_CTL(crtc->index));
+}
+
+void intel_crtc_disable_pipe_crc(struct intel_crtc *intel_crtc)
+{
+	struct drm_crtc *crtc = &intel_crtc->base;
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
+	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[crtc->index];
+
+	/* Swallow crc's until we stop generating them. */
+	spin_lock_irq(&pipe_crc->lock);
+	pipe_crc->skipped = INT_MIN;
+	spin_unlock_irq(&pipe_crc->lock);
+
+	I915_WRITE(PIPE_CRC_CTL(crtc->index), 0);
+	POSTING_READ(PIPE_CRC_CTL(crtc->index));
+	synchronize_irq(dev_priv->drm.irq);
+}
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index b8da4dcdd584..b85229e153c4 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -3567,6 +3567,23 @@ bool ilk_disable_lp_wm(struct drm_device *dev)
 	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
 }
 
+static u8 intel_enabled_dbuf_slices_num(struct drm_i915_private *dev_priv)
+{
+	u8 enabled_slices;
+
+	/* Slice 1 will always be enabled */
+	enabled_slices = 1;
+
+	/* Gen prior to GEN11 have only one DBuf slice */
+	if (INTEL_GEN(dev_priv) < 11)
+		return enabled_slices;
+
+	if (I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE)
+		enabled_slices++;
+
+	return enabled_slices;
+}
+
 /*
  * FIXME: We still don't have the proper code detect if we need to apply the WA,
  * so assume we'll always need it in order to avoid underruns.
@@ -3754,9 +3771,42 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state)
 	return true;
 }
 
+static unsigned int intel_get_ddb_size(struct drm_i915_private *dev_priv,
+				       const struct intel_crtc_state *cstate,
+				       const unsigned int total_data_rate,
+				       const int num_active,
+				       struct skl_ddb_allocation *ddb)
+{
+	const struct drm_display_mode *adjusted_mode;
+	u64 total_data_bw;
+	u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
+
+	WARN_ON(ddb_size == 0);
+
+	if (INTEL_GEN(dev_priv) < 11)
+		return ddb_size - 4; /* 4 blocks for bypass path allocation */
+
+	adjusted_mode = &cstate->base.adjusted_mode;
+	total_data_bw = (u64)total_data_rate * drm_mode_vrefresh(adjusted_mode);
+
+	/*
+	 * 12GB/s is maximum BW supported by single DBuf slice.
+	 */
+	if (total_data_bw >= GBps(12) || num_active > 1) {
+		ddb->enabled_slices = 2;
+	} else {
+		ddb->enabled_slices = 1;
+		ddb_size /= 2;
+	}
+
+	return ddb_size;
+}
+
 static void
 skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
 				   const struct intel_crtc_state *cstate,
+				   const unsigned int total_data_rate,
+				   struct skl_ddb_allocation *ddb,
 				   struct skl_ddb_entry *alloc, /* out */
 				   int *num_active /* out */)
 {
@@ -3779,11 +3829,8 @@ skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
 	else
 		*num_active = hweight32(dev_priv->active_crtcs);
 
-	ddb_size = INTEL_INFO(dev_priv)->ddb_size;
-	WARN_ON(ddb_size == 0);
-
-	if (INTEL_GEN(dev_priv) < 11)
-		ddb_size -= 4; /* 4 blocks for bypass path allocation */
+	ddb_size = intel_get_ddb_size(dev_priv, cstate, total_data_rate,
+				      *num_active, ddb);
 
 	/*
 	 * If the state doesn't change the active CRTC's, then there's
@@ -3817,14 +3864,64 @@ static unsigned int skl_cursor_allocation(int num_active)
 	return 8;
 }
 
-static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
+static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
+				       struct skl_ddb_entry *entry, u32 reg)
 {
-	entry->start = reg & 0x3ff;
-	entry->end = (reg >> 16) & 0x3ff;
+	u16 mask;
+
+	if (INTEL_GEN(dev_priv) >= 11)
+		mask = ICL_DDB_ENTRY_MASK;
+	else
+		mask = SKL_DDB_ENTRY_MASK;
+	entry->start = reg & mask;
+	entry->end = (reg >> DDB_ENTRY_END_SHIFT) & mask;
+
 	if (entry->end)
 		entry->end += 1;
 }
 
+static void
+skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
+			   const enum pipe pipe,
+			   const enum plane_id plane_id,
+			   struct skl_ddb_allocation *ddb /* out */)
+{
+	u32 val, val2 = 0;
+	int fourcc, pixel_format;
+
+	/* Cursor doesn't support NV12/planar, so no extra calculation needed */
+	if (plane_id == PLANE_CURSOR) {
+		val = I915_READ(CUR_BUF_CFG(pipe));
+		skl_ddb_entry_init_from_hw(dev_priv,
+					   &ddb->plane[pipe][plane_id], val);
+		return;
+	}
+
+	val = I915_READ(PLANE_CTL(pipe, plane_id));
+
+	/* No DDB allocated for disabled planes */
+	if (!(val & PLANE_CTL_ENABLE))
+		return;
+
+	pixel_format = val & PLANE_CTL_FORMAT_MASK;
+	fourcc = skl_format_to_fourcc(pixel_format,
+				      val & PLANE_CTL_ORDER_RGBX,
+				      val & PLANE_CTL_ALPHA_MASK);
+
+	val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
+	val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
+
+	if (fourcc == DRM_FORMAT_NV12) {
+		skl_ddb_entry_init_from_hw(dev_priv,
+					   &ddb->plane[pipe][plane_id], val2);
+		skl_ddb_entry_init_from_hw(dev_priv,
+					   &ddb->uv_plane[pipe][plane_id], val);
+	} else {
+		skl_ddb_entry_init_from_hw(dev_priv,
+					   &ddb->plane[pipe][plane_id], val);
+	}
+}
+
 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
 			  struct skl_ddb_allocation *ddb /* out */)
 {
@@ -3832,6 +3929,8 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
 
 	memset(ddb, 0, sizeof(*ddb));
 
+	ddb->enabled_slices = intel_enabled_dbuf_slices_num(dev_priv);
+
 	for_each_intel_crtc(&dev_priv->drm, crtc) {
 		enum intel_display_power_domain power_domain;
 		enum plane_id plane_id;
@@ -3841,16 +3940,9 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
 		if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
 			continue;
 
-		for_each_plane_id_on_crtc(crtc, plane_id) {
-			u32 val;
-
-			if (plane_id != PLANE_CURSOR)
-				val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
-			else
-				val = I915_READ(CUR_BUF_CFG(pipe));
-
-			skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane_id], val);
-		}
+		for_each_plane_id_on_crtc(crtc, plane_id)
+			skl_ddb_get_hw_plane_state(dev_priv, pipe,
+						   plane_id, ddb);
 
 		intel_display_power_put(dev_priv, power_domain);
 	}
@@ -4009,9 +4101,9 @@ int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
 static unsigned int
 skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
 			     const struct drm_plane_state *pstate,
-			     int y)
+			     const int plane)
 {
-	struct intel_plane *plane = to_intel_plane(pstate->plane);
+	struct intel_plane *intel_plane = to_intel_plane(pstate->plane);
 	struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
 	uint32_t data_rate;
 	uint32_t width = 0, height = 0;
@@ -4025,9 +4117,9 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
 	fb = pstate->fb;
 	format = fb->format->format;
 
-	if (plane->id == PLANE_CURSOR)
+	if (intel_plane->id == PLANE_CURSOR)
 		return 0;
-	if (y && format != DRM_FORMAT_NV12)
+	if (plane == 1 && format != DRM_FORMAT_NV12)
 		return 0;
 
 	/*
@@ -4038,19 +4130,14 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
 	width = drm_rect_width(&intel_pstate->base.src) >> 16;
 	height = drm_rect_height(&intel_pstate->base.src) >> 16;
 
-	/* for planar format */
-	if (format == DRM_FORMAT_NV12) {
-		if (y)  /* y-plane data rate */
-			data_rate = width * height *
-				fb->format->cpp[0];
-		else    /* uv-plane data rate */
-			data_rate = (width / 2) * (height / 2) *
-				fb->format->cpp[1];
-	} else {
-		/* for packed formats */
-		data_rate = width * height * fb->format->cpp[0];
+	/* UV plane does 1/2 pixel sub-sampling */
+	if (plane == 1 && format == DRM_FORMAT_NV12) {
+		width /= 2;
+		height /= 2;
 	}
 
+	data_rate = width * height * fb->format->cpp[plane];
+
 	down_scale_amount = skl_plane_downscale_amount(cstate, intel_pstate);
 
 	return mul_round_up_u32_fixed16(data_rate, down_scale_amount);
@@ -4063,8 +4150,8 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
  */
 static unsigned int
 skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
-				 unsigned *plane_data_rate,
-				 unsigned *plane_y_data_rate)
+				 unsigned int *plane_data_rate,
+				 unsigned int *uv_plane_data_rate)
 {
 	struct drm_crtc_state *cstate = &intel_cstate->base;
 	struct drm_atomic_state *state = cstate->state;
@@ -4080,17 +4167,17 @@ skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
 		enum plane_id plane_id = to_intel_plane(plane)->id;
 		unsigned int rate;
 
-		/* packed/uv */
+		/* packed/y */
 		rate = skl_plane_relative_data_rate(intel_cstate,
 						    pstate, 0);
 		plane_data_rate[plane_id] = rate;
 
 		total_data_rate += rate;
 
-		/* y-plane */
+		/* uv-plane */
 		rate = skl_plane_relative_data_rate(intel_cstate,
 						    pstate, 1);
-		plane_y_data_rate[plane_id] = rate;
+		uv_plane_data_rate[plane_id] = rate;
 
 		total_data_rate += rate;
 	}
@@ -4099,8 +4186,7 @@ skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
 }
 
 static uint16_t
-skl_ddb_min_alloc(const struct drm_plane_state *pstate,
-		  const int y)
+skl_ddb_min_alloc(const struct drm_plane_state *pstate, const int plane)
 {
 	struct drm_framebuffer *fb = pstate->fb;
 	struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
@@ -4111,8 +4197,8 @@ skl_ddb_min_alloc(const struct drm_plane_state *pstate,
 	if (WARN_ON(!fb))
 		return 0;
 
-	/* For packed formats, no y-plane, return 0 */
-	if (y && fb->format->format != DRM_FORMAT_NV12)
+	/* For packed formats, and uv-plane, return 0 */
+	if (plane == 1 && fb->format->format != DRM_FORMAT_NV12)
 		return 0;
 
 	/* For Non Y-tile return 8-blocks */
@@ -4131,15 +4217,12 @@ skl_ddb_min_alloc(const struct drm_plane_state *pstate,
 	src_h = drm_rect_height(&intel_pstate->base.src) >> 16;
 
 	/* Halve UV plane width and height for NV12 */
-	if (fb->format->format == DRM_FORMAT_NV12 && !y) {
+	if (plane == 1) {
 		src_w /= 2;
 		src_h /= 2;
 	}
 
-	if (fb->format->format == DRM_FORMAT_NV12 && !y)
-		plane_bpp = fb->format->cpp[1];
-	else
-		plane_bpp = fb->format->cpp[0];
+	plane_bpp = fb->format->cpp[plane];
 
 	if (drm_rotation_90_or_270(pstate->rotation)) {
 		switch (plane_bpp) {
@@ -4167,7 +4250,7 @@ skl_ddb_min_alloc(const struct drm_plane_state *pstate,
 
 static void
 skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active,
-		 uint16_t *minimum, uint16_t *y_minimum)
+		 uint16_t *minimum, uint16_t *uv_minimum)
 {
 	const struct drm_plane_state *pstate;
 	struct drm_plane *plane;
@@ -4182,7 +4265,7 @@ skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active,
 			continue;
 
 		minimum[plane_id] = skl_ddb_min_alloc(pstate, 0);
-		y_minimum[plane_id] = skl_ddb_min_alloc(pstate, 1);
+		uv_minimum[plane_id] = skl_ddb_min_alloc(pstate, 1);
 	}
 
 	minimum[PLANE_CURSOR] = skl_cursor_allocation(num_active);
@@ -4200,17 +4283,17 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 	struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb;
 	uint16_t alloc_size, start;
 	uint16_t minimum[I915_MAX_PLANES] = {};
-	uint16_t y_minimum[I915_MAX_PLANES] = {};
+	uint16_t uv_minimum[I915_MAX_PLANES] = {};
 	unsigned int total_data_rate;
 	enum plane_id plane_id;
 	int num_active;
-	unsigned plane_data_rate[I915_MAX_PLANES] = {};
-	unsigned plane_y_data_rate[I915_MAX_PLANES] = {};
+	unsigned int plane_data_rate[I915_MAX_PLANES] = {};
+	unsigned int uv_plane_data_rate[I915_MAX_PLANES] = {};
 	uint16_t total_min_blocks = 0;
 
 	/* Clear the partitioning for disabled planes. */
 	memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
-	memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe]));
+	memset(ddb->uv_plane[pipe], 0, sizeof(ddb->uv_plane[pipe]));
 
 	if (WARN_ON(!state))
 		return 0;
@@ -4220,12 +4303,16 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 		return 0;
 	}
 
-	skl_ddb_get_pipe_allocation_limits(dev, cstate, alloc, &num_active);
+	total_data_rate = skl_get_total_relative_data_rate(cstate,
+							   plane_data_rate,
+							   uv_plane_data_rate);
+	skl_ddb_get_pipe_allocation_limits(dev, cstate, total_data_rate, ddb,
+					   alloc, &num_active);
 	alloc_size = skl_ddb_entry_size(alloc);
 	if (alloc_size == 0)
 		return 0;
 
-	skl_ddb_calc_min(cstate, num_active, minimum, y_minimum);
+	skl_ddb_calc_min(cstate, num_active, minimum, uv_minimum);
 
 	/*
 	 * 1. Allocate the mininum required blocks for each active plane
@@ -4235,7 +4322,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 
 	for_each_plane_id_on_crtc(intel_crtc, plane_id) {
 		total_min_blocks += minimum[plane_id];
-		total_min_blocks += y_minimum[plane_id];
+		total_min_blocks += uv_minimum[plane_id];
 	}
 
 	if (total_min_blocks > alloc_size) {
@@ -4255,16 +4342,13 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 	 *
 	 * FIXME: we may not allocate every single block here.
 	 */
-	total_data_rate = skl_get_total_relative_data_rate(cstate,
-							   plane_data_rate,
-							   plane_y_data_rate);
 	if (total_data_rate == 0)
 		return 0;
 
 	start = alloc->start;
 	for_each_plane_id_on_crtc(intel_crtc, plane_id) {
-		unsigned int data_rate, y_data_rate;
-		uint16_t plane_blocks, y_plane_blocks = 0;
+		unsigned int data_rate, uv_data_rate;
+		uint16_t plane_blocks, uv_plane_blocks;
 
 		if (plane_id == PLANE_CURSOR)
 			continue;
@@ -4288,21 +4372,20 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 
 		start += plane_blocks;
 
-		/*
-		 * allocation for y_plane part of planar format:
-		 */
-		y_data_rate = plane_y_data_rate[plane_id];
+		/* Allocate DDB for UV plane for planar format/NV12 */
+		uv_data_rate = uv_plane_data_rate[plane_id];
 
-		y_plane_blocks = y_minimum[plane_id];
-		y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
-					total_data_rate);
+		uv_plane_blocks = uv_minimum[plane_id];
+		uv_plane_blocks += div_u64((uint64_t)alloc_size * uv_data_rate,
+					   total_data_rate);
 
-		if (y_data_rate) {
-			ddb->y_plane[pipe][plane_id].start = start;
-			ddb->y_plane[pipe][plane_id].end = start + y_plane_blocks;
+		if (uv_data_rate) {
+			ddb->uv_plane[pipe][plane_id].start = start;
+			ddb->uv_plane[pipe][plane_id].end =
+				start + uv_plane_blocks;
 		}
 
-		start += y_plane_blocks;
+		start += uv_plane_blocks;
 	}
 
 	return 0;
@@ -4398,7 +4481,7 @@ static int
 skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv,
 			    struct intel_crtc_state *cstate,
 			    const struct intel_plane_state *intel_pstate,
-			    struct skl_wm_params *wp)
+			    struct skl_wm_params *wp, int plane_id)
 {
 	struct intel_plane *plane = to_intel_plane(intel_pstate->base.plane);
 	const struct drm_plane_state *pstate = &intel_pstate->base;
@@ -4411,6 +4494,12 @@ skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv,
 	if (!intel_wm_plane_visible(cstate, intel_pstate))
 		return 0;
 
+	/* only NV12 format has two planes */
+	if (plane_id == 1 && fb->format->format != DRM_FORMAT_NV12) {
+		DRM_DEBUG_KMS("Non NV12 format have single plane\n");
+		return -EINVAL;
+	}
+
 	wp->y_tiled = fb->modifier == I915_FORMAT_MOD_Y_TILED ||
 		      fb->modifier == I915_FORMAT_MOD_Yf_TILED ||
 		      fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
@@ -4418,6 +4507,7 @@ skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv,
 	wp->x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
 	wp->rc_surface = fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
 			 fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
+	wp->is_planar = fb->format->format == DRM_FORMAT_NV12;
 
 	if (plane->id == PLANE_CURSOR) {
 		wp->width = intel_pstate->base.crtc_w;
@@ -4430,8 +4520,10 @@ skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv,
 		wp->width = drm_rect_width(&intel_pstate->base.src) >> 16;
 	}
 
-	wp->cpp = (fb->format->format == DRM_FORMAT_NV12) ? fb->format->cpp[1] :
-							    fb->format->cpp[0];
+	if (plane_id == 1 && wp->is_planar)
+		wp->width /= 2;
+
+	wp->cpp = fb->format->cpp[plane_id];
 	wp->plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate,
 							     intel_pstate);
 
@@ -4499,9 +4591,8 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 				uint16_t ddb_allocation,
 				int level,
 				const struct skl_wm_params *wp,
-				uint16_t *out_blocks, /* out */
-				uint8_t *out_lines, /* out */
-				bool *enabled /* out */)
+				const struct skl_wm_level *result_prev,
+				struct skl_wm_level *result /* out */)
 {
 	const struct drm_plane_state *pstate = &intel_pstate->base;
 	uint32_t latency = dev_priv->wm.skl_latency[level];
@@ -4515,7 +4606,7 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 
 	if (latency == 0 ||
 	    !intel_wm_plane_visible(cstate, intel_pstate)) {
-		*enabled = false;
+		result->plane_en = false;
 		return 0;
 	}
 
@@ -4568,6 +4659,15 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 		} else {
 			res_blocks++;
 		}
+
+		/*
+		 * Make sure result blocks for higher latency levels are atleast
+		 * as high as level below the current level.
+		 * Assumption in DDB algorithm optimization for special cases.
+		 * Also covers Display WA #1125 for RC.
+		 */
+		if (result_prev->plane_res_b > res_blocks)
+			res_blocks = result_prev->plane_res_b;
 	}
 
 	if (INTEL_GEN(dev_priv) >= 11) {
@@ -4596,7 +4696,7 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	if ((level > 0 && res_lines > 31) ||
 	    res_blocks >= ddb_allocation ||
 	    min_disp_buf_needed >= ddb_allocation) {
-		*enabled = false;
+		result->plane_en = false;
 
 		/*
 		 * If there are no valid level 0 watermarks, then we can't
@@ -4615,10 +4715,21 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 		}
 	}
 
+	/*
+	 * Display WA #826 (SKL:ALL, BXT:ALL) & #1059 (CNL:A)
+	 * disable wm level 1-7 on NV12 planes
+	 */
+	if (wp->is_planar && level >= 1 &&
+	    (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv) ||
+	     IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))) {
+		result->plane_en = false;
+		return 0;
+	}
+
 	/* The number of lines are ignored for the level 0 watermark. */
-	*out_lines = level ? res_lines : 0;
-	*out_blocks = res_blocks;
-	*enabled = true;
+	result->plane_res_b = res_blocks;
+	result->plane_res_l = res_lines;
+	result->plane_en = true;
 
 	return 0;
 }
@@ -4629,7 +4740,8 @@ skl_compute_wm_levels(const struct drm_i915_private *dev_priv,
 		      struct intel_crtc_state *cstate,
 		      const struct intel_plane_state *intel_pstate,
 		      const struct skl_wm_params *wm_params,
-		      struct skl_plane_wm *wm)
+		      struct skl_plane_wm *wm,
+		      int plane_id)
 {
 	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
 	struct drm_plane *plane = intel_pstate->base.plane;
@@ -4637,15 +4749,26 @@ skl_compute_wm_levels(const struct drm_i915_private *dev_priv,
 	uint16_t ddb_blocks;
 	enum pipe pipe = intel_crtc->pipe;
 	int level, max_level = ilk_wm_max_level(dev_priv);
+	enum plane_id intel_plane_id = intel_plane->id;
 	int ret;
 
 	if (WARN_ON(!intel_pstate->base.fb))
 		return -EINVAL;
 
-	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][intel_plane->id]);
+	ddb_blocks = plane_id ?
+		     skl_ddb_entry_size(&ddb->uv_plane[pipe][intel_plane_id]) :
+		     skl_ddb_entry_size(&ddb->plane[pipe][intel_plane_id]);
 
 	for (level = 0; level <= max_level; level++) {
-		struct skl_wm_level *result = &wm->wm[level];
+		struct skl_wm_level *result = plane_id ? &wm->uv_wm[level] :
+							  &wm->wm[level];
+		struct skl_wm_level *result_prev;
+
+		if (level)
+			result_prev = plane_id ? &wm->uv_wm[level - 1] :
+						  &wm->wm[level - 1];
+		else
+			result_prev = plane_id ? &wm->uv_wm[0] : &wm->wm[0];
 
 		ret = skl_compute_plane_wm(dev_priv,
 					   cstate,
@@ -4653,13 +4776,15 @@ skl_compute_wm_levels(const struct drm_i915_private *dev_priv,
 					   ddb_blocks,
 					   level,
 					   wm_params,
-					   &result->plane_res_b,
-					   &result->plane_res_l,
-					   &result->plane_en);
+					   result_prev,
+					   result);
 		if (ret)
 			return ret;
 	}
 
+	if (intel_pstate->base.fb->format->format == DRM_FORMAT_NV12)
+		wm->is_planar = true;
+
 	return 0;
 }
 
@@ -4769,20 +4894,39 @@ static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
 
 		wm = &pipe_wm->planes[plane_id];
 		ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][plane_id]);
-		memset(&wm_params, 0, sizeof(struct skl_wm_params));
 
 		ret = skl_compute_plane_wm_params(dev_priv, cstate,
-						  intel_pstate, &wm_params);
+						  intel_pstate, &wm_params, 0);
 		if (ret)
 			return ret;
 
 		ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
-					    intel_pstate, &wm_params, wm);
+					    intel_pstate, &wm_params, wm, 0);
 		if (ret)
 			return ret;
+
 		skl_compute_transition_wm(cstate, &wm_params, &wm->wm[0],
 					  ddb_blocks, &wm->trans_wm);
+
+		/* uv plane watermarks must also be validated for NV12/Planar */
+		if (wm_params.is_planar) {
+			memset(&wm_params, 0, sizeof(struct skl_wm_params));
+			wm->is_planar = true;
+
+			ret = skl_compute_plane_wm_params(dev_priv, cstate,
+							  intel_pstate,
+							  &wm_params, 1);
+			if (ret)
+				return ret;
+
+			ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
+						    intel_pstate, &wm_params,
+						    wm, 1);
+			if (ret)
+				return ret;
+		}
 	}
+
 	pipe_wm->linetime = skl_compute_linetime_wm(cstate);
 
 	return 0;
@@ -4833,10 +4977,21 @@ static void skl_write_plane_wm(struct intel_crtc *intel_crtc,
 
 	skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
 			    &ddb->plane[pipe][plane_id]);
-	if (INTEL_GEN(dev_priv) < 11)
+	if (INTEL_GEN(dev_priv) >= 11)
+		return skl_ddb_entry_write(dev_priv,
+					   PLANE_BUF_CFG(pipe, plane_id),
+					   &ddb->plane[pipe][plane_id]);
+	if (wm->is_planar) {
+		skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
+				    &ddb->uv_plane[pipe][plane_id]);
 		skl_ddb_entry_write(dev_priv,
 				    PLANE_NV12_BUF_CFG(pipe, plane_id),
-				    &ddb->y_plane[pipe][plane_id]);
+				    &ddb->plane[pipe][plane_id]);
+	} else {
+		skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
+				    &ddb->plane[pipe][plane_id]);
+		I915_WRITE(PLANE_NV12_BUF_CFG(pipe, plane_id), 0x0);
+	}
 }
 
 static void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
@@ -4944,15 +5099,13 @@ skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
 	struct drm_plane *plane;
 	enum pipe pipe = intel_crtc->pipe;
 
-	WARN_ON(!drm_atomic_get_existing_crtc_state(state, crtc));
-
 	drm_for_each_plane_mask(plane, dev, cstate->base.plane_mask) {
 		enum plane_id plane_id = to_intel_plane(plane)->id;
 
 		if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][plane_id],
 					&new_ddb->plane[pipe][plane_id]) &&
-		    skl_ddb_entry_equal(&cur_ddb->y_plane[pipe][plane_id],
-					&new_ddb->y_plane[pipe][plane_id]))
+		    skl_ddb_entry_equal(&cur_ddb->uv_plane[pipe][plane_id],
+					&new_ddb->uv_plane[pipe][plane_id]))
 			continue;
 
 		plane_state = drm_atomic_get_plane_state(state, plane);
@@ -4966,69 +5119,16 @@ skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
 static int
 skl_compute_ddb(struct drm_atomic_state *state)
 {
-	struct drm_device *dev = state->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	const struct drm_i915_private *dev_priv = to_i915(state->dev);
 	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
-	struct intel_crtc *intel_crtc;
 	struct skl_ddb_allocation *ddb = &intel_state->wm_results.ddb;
-	uint32_t realloc_pipes = pipes_modified(state);
-	int ret;
-
-	/*
-	 * If this is our first atomic update following hardware readout,
-	 * we can't trust the DDB that the BIOS programmed for us.  Let's
-	 * pretend that all pipes switched active status so that we'll
-	 * ensure a full DDB recompute.
-	 */
-	if (dev_priv->wm.distrust_bios_wm) {
-		ret = drm_modeset_lock(&dev->mode_config.connection_mutex,
-				       state->acquire_ctx);
-		if (ret)
-			return ret;
-
-		intel_state->active_pipe_changes = ~0;
-
-		/*
-		 * We usually only initialize intel_state->active_crtcs if we
-		 * we're doing a modeset; make sure this field is always
-		 * initialized during the sanitization process that happens
-		 * on the first commit too.
-		 */
-		if (!intel_state->modeset)
-			intel_state->active_crtcs = dev_priv->active_crtcs;
-	}
-
-	/*
-	 * If the modeset changes which CRTC's are active, we need to
-	 * recompute the DDB allocation for *all* active pipes, even
-	 * those that weren't otherwise being modified in any way by this
-	 * atomic commit.  Due to the shrinking of the per-pipe allocations
-	 * when new active CRTC's are added, it's possible for a pipe that
-	 * we were already using and aren't changing at all here to suddenly
-	 * become invalid if its DDB needs exceeds its new allocation.
-	 *
-	 * Note that if we wind up doing a full DDB recompute, we can't let
-	 * any other display updates race with this transaction, so we need
-	 * to grab the lock on *all* CRTC's.
-	 */
-	if (intel_state->active_pipe_changes) {
-		realloc_pipes = ~0;
-		intel_state->wm_results.dirty_pipes = ~0;
-	}
+	struct intel_crtc *crtc;
+	struct intel_crtc_state *cstate;
+	int ret, i;
 
-	/*
-	 * We're not recomputing for the pipes not included in the commit, so
-	 * make sure we start with the current state.
-	 */
 	memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb));
 
-	for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) {
-		struct intel_crtc_state *cstate;
-
-		cstate = intel_atomic_get_crtc_state(state, intel_crtc);
-		if (IS_ERR(cstate))
-			return PTR_ERR(cstate);
-
+	for_each_new_intel_crtc_in_state(intel_state, crtc, cstate, i) {
 		ret = skl_allocate_pipe_ddb(cstate, ddb);
 		if (ret)
 			return ret;
@@ -5042,14 +5142,15 @@ skl_compute_ddb(struct drm_atomic_state *state)
 }
 
 static void
-skl_copy_wm_for_pipe(struct skl_wm_values *dst,
-		     struct skl_wm_values *src,
-		     enum pipe pipe)
+skl_copy_ddb_for_pipe(struct skl_ddb_values *dst,
+		      struct skl_ddb_values *src,
+		      enum pipe pipe)
 {
-	memcpy(dst->ddb.y_plane[pipe], src->ddb.y_plane[pipe],
-	       sizeof(dst->ddb.y_plane[pipe]));
+	memcpy(dst->ddb.uv_plane[pipe], src->ddb.uv_plane[pipe],
+	       sizeof(dst->ddb.uv_plane[pipe]));
 	memcpy(dst->ddb.plane[pipe], src->ddb.plane[pipe],
 	       sizeof(dst->ddb.plane[pipe]));
+	dst->ddb.enabled_slices = src->ddb.enabled_slices;
 }
 
 static void
@@ -5090,23 +5191,23 @@ skl_print_wm_changes(const struct drm_atomic_state *state)
 }
 
 static int
-skl_compute_wm(struct drm_atomic_state *state)
+skl_ddb_add_affected_pipes(struct drm_atomic_state *state, bool *changed)
 {
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *cstate;
-	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
-	struct skl_wm_values *results = &intel_state->wm_results;
 	struct drm_device *dev = state->dev;
-	struct skl_pipe_wm *pipe_wm;
-	bool changed = false;
+	const struct drm_i915_private *dev_priv = to_i915(dev);
+	const struct drm_crtc *crtc;
+	const struct drm_crtc_state *cstate;
+	struct intel_crtc *intel_crtc;
+	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+	uint32_t realloc_pipes = pipes_modified(state);
 	int ret, i;
 
 	/*
 	 * When we distrust bios wm we always need to recompute to set the
 	 * expected DDB allocations for each CRTC.
 	 */
-	if (to_i915(dev)->wm.distrust_bios_wm)
-		changed = true;
+	if (dev_priv->wm.distrust_bios_wm)
+		(*changed) = true;
 
 	/*
 	 * If this transaction isn't actually touching any CRTC's, don't
@@ -5117,14 +5218,86 @@ skl_compute_wm(struct drm_atomic_state *state)
 	 * hold _all_ CRTC state mutexes.
 	 */
 	for_each_new_crtc_in_state(state, crtc, cstate, i)
-		changed = true;
+		(*changed) = true;
 
-	if (!changed)
+	if (!*changed)
 		return 0;
 
+	/*
+	 * If this is our first atomic update following hardware readout,
+	 * we can't trust the DDB that the BIOS programmed for us.  Let's
+	 * pretend that all pipes switched active status so that we'll
+	 * ensure a full DDB recompute.
+	 */
+	if (dev_priv->wm.distrust_bios_wm) {
+		ret = drm_modeset_lock(&dev->mode_config.connection_mutex,
+				       state->acquire_ctx);
+		if (ret)
+			return ret;
+
+		intel_state->active_pipe_changes = ~0;
+
+		/*
+		 * We usually only initialize intel_state->active_crtcs if we
+		 * we're doing a modeset; make sure this field is always
+		 * initialized during the sanitization process that happens
+		 * on the first commit too.
+		 */
+		if (!intel_state->modeset)
+			intel_state->active_crtcs = dev_priv->active_crtcs;
+	}
+
+	/*
+	 * If the modeset changes which CRTC's are active, we need to
+	 * recompute the DDB allocation for *all* active pipes, even
+	 * those that weren't otherwise being modified in any way by this
+	 * atomic commit.  Due to the shrinking of the per-pipe allocations
+	 * when new active CRTC's are added, it's possible for a pipe that
+	 * we were already using and aren't changing at all here to suddenly
+	 * become invalid if its DDB needs exceeds its new allocation.
+	 *
+	 * Note that if we wind up doing a full DDB recompute, we can't let
+	 * any other display updates race with this transaction, so we need
+	 * to grab the lock on *all* CRTC's.
+	 */
+	if (intel_state->active_pipe_changes) {
+		realloc_pipes = ~0;
+		intel_state->wm_results.dirty_pipes = ~0;
+	}
+
+	/*
+	 * We're not recomputing for the pipes not included in the commit, so
+	 * make sure we start with the current state.
+	 */
+	for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) {
+		struct intel_crtc_state *cstate;
+
+		cstate = intel_atomic_get_crtc_state(state, intel_crtc);
+		if (IS_ERR(cstate))
+			return PTR_ERR(cstate);
+	}
+
+	return 0;
+}
+
+static int
+skl_compute_wm(struct drm_atomic_state *state)
+{
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *cstate;
+	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+	struct skl_ddb_values *results = &intel_state->wm_results;
+	struct skl_pipe_wm *pipe_wm;
+	bool changed = false;
+	int ret, i;
+
 	/* Clear all dirty flags */
 	results->dirty_pipes = 0;
 
+	ret = skl_ddb_add_affected_pipes(state, &changed);
+	if (ret || !changed)
+		return ret;
+
 	ret = skl_compute_ddb(state);
 	if (ret)
 		return ret;
@@ -5197,8 +5370,8 @@ static void skl_initial_wm(struct intel_atomic_state *state,
 	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
 	struct drm_device *dev = intel_crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct skl_wm_values *results = &state->wm_results;
-	struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw;
+	struct skl_ddb_values *results = &state->wm_results;
+	struct skl_ddb_values *hw_vals = &dev_priv->wm.skl_hw;
 	enum pipe pipe = intel_crtc->pipe;
 
 	if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0)
@@ -5209,7 +5382,7 @@ static void skl_initial_wm(struct intel_atomic_state *state,
 	if (cstate->base.active_changed)
 		skl_atomic_update_crtc_wm(state, cstate);
 
-	skl_copy_wm_for_pipe(hw_vals, results, pipe);
+	skl_copy_ddb_for_pipe(hw_vals, results, pipe);
 
 	mutex_unlock(&dev_priv->wm.wm_mutex);
 }
@@ -5341,7 +5514,7 @@ void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
 void skl_wm_get_hw_state(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
+	struct skl_ddb_values *hw = &dev_priv->wm.skl_hw;
 	struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
 	struct drm_crtc *crtc;
 	struct intel_crtc *intel_crtc;
@@ -5362,8 +5535,12 @@ void skl_wm_get_hw_state(struct drm_device *dev)
 		/* Fully recompute DDB on first atomic commit */
 		dev_priv->wm.distrust_bios_wm = true;
 	} else {
-		/* Easy/common case; just sanitize DDB now if everything off */
-		memset(ddb, 0, sizeof(*ddb));
+		/*
+		 * Easy/common case; just sanitize DDB now if everything off
+		 * Keep dbuf slice info intact
+		 */
+		memset(ddb->plane, 0, sizeof(ddb->plane));
+		memset(ddb->uv_plane, 0, sizeof(ddb->uv_plane));
 	}
 }
 
@@ -6572,7 +6749,7 @@ static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
 
 	rps->efficient_freq = rps->rp1_freq;
 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
-	    IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+	    IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
 		u32 ddcc_status = 0;
 
 		if (sandybridge_pcode_read(dev_priv,
@@ -6585,7 +6762,7 @@ static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
 					rps->max_freq);
 	}
 
-	if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+	if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
 		/* Store the frequency values in 16.66 MHZ units, which is
 		 * the natural hardware unit for SKL
 		 */
@@ -6890,15 +7067,18 @@ static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
 {
 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
-	int min_freq = 15;
+	const int min_freq = 15;
+	const int scaling_factor = 180;
 	unsigned int gpu_freq;
 	unsigned int max_ia_freq, min_ring_freq;
 	unsigned int max_gpu_freq, min_gpu_freq;
-	int scaling_factor = 180;
 	struct cpufreq_policy *policy;
 
 	WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
 
+	if (rps->max_freq <= rps->min_freq)
+		return;
+
 	policy = cpufreq_cpu_get(0);
 	if (policy) {
 		max_ia_freq = policy->cpuinfo.max_freq;
@@ -6918,13 +7098,12 @@ static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
 	/* convert DDR frequency from units of 266.6MHz to bandwidth */
 	min_ring_freq = mult_frac(min_ring_freq, 8, 3);
 
-	if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+	min_gpu_freq = rps->min_freq;
+	max_gpu_freq = rps->max_freq;
+	if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
 		/* Convert GT frequency to 50 HZ units */
-		min_gpu_freq = rps->min_freq / GEN9_FREQ_SCALER;
-		max_gpu_freq = rps->max_freq / GEN9_FREQ_SCALER;
-	} else {
-		min_gpu_freq = rps->min_freq;
-		max_gpu_freq = rps->max_freq;
+		min_gpu_freq /= GEN9_FREQ_SCALER;
+		max_gpu_freq /= GEN9_FREQ_SCALER;
 	}
 
 	/*
@@ -6933,10 +7112,10 @@ static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
 	 * the PCU should use as a reference to determine the ring frequency.
 	 */
 	for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
-		int diff = max_gpu_freq - gpu_freq;
+		const int diff = max_gpu_freq - gpu_freq;
 		unsigned int ia_freq = 0, ring_freq = 0;
 
-		if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+		if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
 			/*
 			 * ring_freq = 2 * GT. ring_freq is in 100MHz units
 			 * No floor required for ring frequency on SKL.
@@ -8026,10 +8205,10 @@ void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
 	dev_priv->gt_pm.rc6.enabled = true; /* force RC6 disabling */
 	intel_disable_gt_powersave(dev_priv);
 
-	if (INTEL_GEN(dev_priv) < 11)
-		gen6_reset_rps_interrupts(dev_priv);
+	if (INTEL_GEN(dev_priv) >= 11)
+		gen11_reset_rps_interrupts(dev_priv);
 	else
-		WARN_ON_ONCE(1);
+		gen6_reset_rps_interrupts(dev_priv);
 }
 
 static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
@@ -8142,8 +8321,6 @@ static void intel_enable_rps(struct drm_i915_private *dev_priv)
 		cherryview_enable_rps(dev_priv);
 	} else if (IS_VALLEYVIEW(dev_priv)) {
 		valleyview_enable_rps(dev_priv);
-	} else if (WARN_ON_ONCE(INTEL_GEN(dev_priv) >= 11)) {
-		/* TODO */
 	} else if (INTEL_GEN(dev_priv) >= 9) {
 		gen9_enable_rps(dev_priv);
 	} else if (IS_BROADWELL(dev_priv)) {
@@ -8487,6 +8664,13 @@ static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
 	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
 }
 
+static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
+{
+	/* This is not an Wa. Enable to reduce Sampler power */
+	I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
+		   I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
+}
+
 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
 {
 	if (!HAS_PCH_CNP(dev_priv))
@@ -9013,7 +9197,9 @@ static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
  */
 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
 {
-	if (IS_CANNONLAKE(dev_priv))
+	if (IS_ICELAKE(dev_priv))
+		dev_priv->display.init_clock_gating = icl_init_clock_gating;
+	else if (IS_CANNONLAKE(dev_priv))
 		dev_priv->display.init_clock_gating = cnl_init_clock_gating;
 	else if (IS_COFFEELAKE(dev_priv))
 		dev_priv->display.init_clock_gating = cfl_init_clock_gating;
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index 23175c5c4a50..db27f2faa1de 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -93,7 +93,115 @@ static void psr_aux_io_power_put(struct intel_dp *intel_dp)
 	intel_display_power_put(dev_priv, psr_aux_domain(intel_dp));
 }
 
-static bool intel_dp_get_y_cord_status(struct intel_dp *intel_dp)
+void intel_psr_irq_control(struct drm_i915_private *dev_priv, bool debug)
+{
+	u32 debug_mask, mask;
+
+	/* No PSR interrupts on VLV/CHV */
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		return;
+
+	mask = EDP_PSR_ERROR(TRANSCODER_EDP);
+	debug_mask = EDP_PSR_POST_EXIT(TRANSCODER_EDP) |
+		     EDP_PSR_PRE_ENTRY(TRANSCODER_EDP);
+
+	if (INTEL_GEN(dev_priv) >= 8) {
+		mask |= EDP_PSR_ERROR(TRANSCODER_A) |
+			EDP_PSR_ERROR(TRANSCODER_B) |
+			EDP_PSR_ERROR(TRANSCODER_C);
+
+		debug_mask |= EDP_PSR_POST_EXIT(TRANSCODER_A) |
+			      EDP_PSR_PRE_ENTRY(TRANSCODER_A) |
+			      EDP_PSR_POST_EXIT(TRANSCODER_B) |
+			      EDP_PSR_PRE_ENTRY(TRANSCODER_B) |
+			      EDP_PSR_POST_EXIT(TRANSCODER_C) |
+			      EDP_PSR_PRE_ENTRY(TRANSCODER_C);
+	}
+
+	if (debug)
+		mask |= debug_mask;
+
+	WRITE_ONCE(dev_priv->psr.debug, debug);
+	I915_WRITE(EDP_PSR_IMR, ~mask);
+}
+
+static void psr_event_print(u32 val, bool psr2_enabled)
+{
+	DRM_DEBUG_KMS("PSR exit events: 0x%x\n", val);
+	if (val & PSR_EVENT_PSR2_WD_TIMER_EXPIRE)
+		DRM_DEBUG_KMS("\tPSR2 watchdog timer expired\n");
+	if ((val & PSR_EVENT_PSR2_DISABLED) && psr2_enabled)
+		DRM_DEBUG_KMS("\tPSR2 disabled\n");
+	if (val & PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN)
+		DRM_DEBUG_KMS("\tSU dirty FIFO underrun\n");
+	if (val & PSR_EVENT_SU_CRC_FIFO_UNDERRUN)
+		DRM_DEBUG_KMS("\tSU CRC FIFO underrun\n");
+	if (val & PSR_EVENT_GRAPHICS_RESET)
+		DRM_DEBUG_KMS("\tGraphics reset\n");
+	if (val & PSR_EVENT_PCH_INTERRUPT)
+		DRM_DEBUG_KMS("\tPCH interrupt\n");
+	if (val & PSR_EVENT_MEMORY_UP)
+		DRM_DEBUG_KMS("\tMemory up\n");
+	if (val & PSR_EVENT_FRONT_BUFFER_MODIFY)
+		DRM_DEBUG_KMS("\tFront buffer modification\n");
+	if (val & PSR_EVENT_WD_TIMER_EXPIRE)
+		DRM_DEBUG_KMS("\tPSR watchdog timer expired\n");
+	if (val & PSR_EVENT_PIPE_REGISTERS_UPDATE)
+		DRM_DEBUG_KMS("\tPIPE registers updated\n");
+	if (val & PSR_EVENT_REGISTER_UPDATE)
+		DRM_DEBUG_KMS("\tRegister updated\n");
+	if (val & PSR_EVENT_HDCP_ENABLE)
+		DRM_DEBUG_KMS("\tHDCP enabled\n");
+	if (val & PSR_EVENT_KVMR_SESSION_ENABLE)
+		DRM_DEBUG_KMS("\tKVMR session enabled\n");
+	if (val & PSR_EVENT_VBI_ENABLE)
+		DRM_DEBUG_KMS("\tVBI enabled\n");
+	if (val & PSR_EVENT_LPSP_MODE_EXIT)
+		DRM_DEBUG_KMS("\tLPSP mode exited\n");
+	if ((val & PSR_EVENT_PSR_DISABLE) && !psr2_enabled)
+		DRM_DEBUG_KMS("\tPSR disabled\n");
+}
+
+void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir)
+{
+	u32 transcoders = BIT(TRANSCODER_EDP);
+	enum transcoder cpu_transcoder;
+	ktime_t time_ns =  ktime_get();
+
+	if (INTEL_GEN(dev_priv) >= 8)
+		transcoders |= BIT(TRANSCODER_A) |
+			       BIT(TRANSCODER_B) |
+			       BIT(TRANSCODER_C);
+
+	for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, transcoders) {
+		/* FIXME: Exit PSR and link train manually when this happens. */
+		if (psr_iir & EDP_PSR_ERROR(cpu_transcoder))
+			DRM_DEBUG_KMS("[transcoder %s] PSR aux error\n",
+				      transcoder_name(cpu_transcoder));
+
+		if (psr_iir & EDP_PSR_PRE_ENTRY(cpu_transcoder)) {
+			dev_priv->psr.last_entry_attempt = time_ns;
+			DRM_DEBUG_KMS("[transcoder %s] PSR entry attempt in 2 vblanks\n",
+				      transcoder_name(cpu_transcoder));
+		}
+
+		if (psr_iir & EDP_PSR_POST_EXIT(cpu_transcoder)) {
+			dev_priv->psr.last_exit = time_ns;
+			DRM_DEBUG_KMS("[transcoder %s] PSR exit completed\n",
+				      transcoder_name(cpu_transcoder));
+
+			if (INTEL_GEN(dev_priv) >= 9) {
+				u32 val = I915_READ(PSR_EVENT(cpu_transcoder));
+				bool psr2_enabled = dev_priv->psr.psr2_enabled;
+
+				I915_WRITE(PSR_EVENT(cpu_transcoder), val);
+				psr_event_print(val, psr2_enabled);
+			}
+		}
+	}
+}
+
+static bool intel_dp_get_y_coord_required(struct intel_dp *intel_dp)
 {
 	uint8_t psr_caps = 0;
 
@@ -122,6 +230,18 @@ static bool intel_dp_get_alpm_status(struct intel_dp *intel_dp)
 	return alpm_caps & DP_ALPM_CAP;
 }
 
+static u8 intel_dp_get_sink_sync_latency(struct intel_dp *intel_dp)
+{
+	u8 val = 0;
+
+	if (drm_dp_dpcd_readb(&intel_dp->aux,
+			      DP_SYNCHRONIZATION_LATENCY_IN_SINK, &val) == 1)
+		val &= DP_MAX_RESYNC_FRAME_COUNT_MASK;
+	else
+		DRM_ERROR("Unable to get sink synchronization latency\n");
+	return val;
+}
+
 void intel_psr_init_dpcd(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv =
@@ -130,33 +250,36 @@ void intel_psr_init_dpcd(struct intel_dp *intel_dp)
 	drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT, intel_dp->psr_dpcd,
 			 sizeof(intel_dp->psr_dpcd));
 
-	if (intel_dp->psr_dpcd[0] & DP_PSR_IS_SUPPORTED) {
+	if (intel_dp->psr_dpcd[0]) {
 		dev_priv->psr.sink_support = true;
 		DRM_DEBUG_KMS("Detected EDP PSR Panel.\n");
 	}
 
 	if (INTEL_GEN(dev_priv) >= 9 &&
-	    (intel_dp->psr_dpcd[0] & DP_PSR2_IS_SUPPORTED)) {
-		uint8_t frame_sync_cap;
+	    (intel_dp->psr_dpcd[0] == DP_PSR2_WITH_Y_COORD_IS_SUPPORTED)) {
+		/*
+		 * All panels that supports PSR version 03h (PSR2 +
+		 * Y-coordinate) can handle Y-coordinates in VSC but we are
+		 * only sure that it is going to be used when required by the
+		 * panel. This way panel is capable to do selective update
+		 * without a aux frame sync.
+		 *
+		 * To support PSR version 02h and PSR version 03h without
+		 * Y-coordinate requirement panels we would need to enable
+		 * GTC first.
+		 */
+		dev_priv->psr.sink_psr2_support =
+				intel_dp_get_y_coord_required(intel_dp);
+		DRM_DEBUG_KMS("PSR2 %s on sink", dev_priv->psr.sink_psr2_support
+			      ? "supported" : "not supported");
 
-		dev_priv->psr.sink_support = true;
-		if (drm_dp_dpcd_readb(&intel_dp->aux,
-				      DP_SINK_DEVICE_AUX_FRAME_SYNC_CAP,
-				      &frame_sync_cap) != 1)
-			frame_sync_cap = 0;
-		dev_priv->psr.aux_frame_sync = frame_sync_cap & DP_AUX_FRAME_SYNC_CAP;
-		/* PSR2 needs frame sync as well */
-		dev_priv->psr.psr2_support = dev_priv->psr.aux_frame_sync;
-		DRM_DEBUG_KMS("PSR2 %s on sink",
-			      dev_priv->psr.psr2_support ? "supported" : "not supported");
-
-		if (dev_priv->psr.psr2_support) {
-			dev_priv->psr.y_cord_support =
-				intel_dp_get_y_cord_status(intel_dp);
+		if (dev_priv->psr.sink_psr2_support) {
 			dev_priv->psr.colorimetry_support =
 				intel_dp_get_colorimetry_status(intel_dp);
 			dev_priv->psr.alpm =
 				intel_dp_get_alpm_status(intel_dp);
+			dev_priv->psr.sink_sync_latency =
+				intel_dp_get_sink_sync_latency(intel_dp);
 		}
 	}
 }
@@ -193,21 +316,17 @@ static void hsw_psr_setup_vsc(struct intel_dp *intel_dp,
 	struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
 	struct edp_vsc_psr psr_vsc;
 
-	if (dev_priv->psr.psr2_support) {
+	if (dev_priv->psr.psr2_enabled) {
 		/* Prepare VSC Header for SU as per EDP 1.4 spec, Table 6.11 */
 		memset(&psr_vsc, 0, sizeof(psr_vsc));
 		psr_vsc.sdp_header.HB0 = 0;
 		psr_vsc.sdp_header.HB1 = 0x7;
-		if (dev_priv->psr.colorimetry_support &&
-		    dev_priv->psr.y_cord_support) {
+		if (dev_priv->psr.colorimetry_support) {
 			psr_vsc.sdp_header.HB2 = 0x5;
 			psr_vsc.sdp_header.HB3 = 0x13;
-		} else if (dev_priv->psr.y_cord_support) {
+		} else {
 			psr_vsc.sdp_header.HB2 = 0x4;
 			psr_vsc.sdp_header.HB3 = 0xe;
-		} else {
-			psr_vsc.sdp_header.HB2 = 0x3;
-			psr_vsc.sdp_header.HB3 = 0xc;
 		}
 	} else {
 		/* Prepare VSC packet as per EDP 1.3 spec, Table 3.10 */
@@ -228,31 +347,12 @@ static void vlv_psr_enable_sink(struct intel_dp *intel_dp)
 			   DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE);
 }
 
-static i915_reg_t psr_aux_ctl_reg(struct drm_i915_private *dev_priv,
-				       enum port port)
-{
-	if (INTEL_GEN(dev_priv) >= 9)
-		return DP_AUX_CH_CTL(port);
-	else
-		return EDP_PSR_AUX_CTL;
-}
-
-static i915_reg_t psr_aux_data_reg(struct drm_i915_private *dev_priv,
-					enum port port, int index)
-{
-	if (INTEL_GEN(dev_priv) >= 9)
-		return DP_AUX_CH_DATA(port, index);
-	else
-		return EDP_PSR_AUX_DATA(index);
-}
-
-static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
+static void hsw_psr_setup_aux(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	uint32_t aux_clock_divider;
-	i915_reg_t aux_ctl_reg;
+	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
+	u32 aux_clock_divider, aux_ctl;
+	int i;
 	static const uint8_t aux_msg[] = {
 		[0] = DP_AUX_NATIVE_WRITE << 4,
 		[1] = DP_SET_POWER >> 8,
@@ -260,41 +360,47 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
 		[3] = 1 - 1,
 		[4] = DP_SET_POWER_D0,
 	};
-	enum port port = dig_port->base.port;
-	u32 aux_ctl;
-	int i;
+	u32 psr_aux_mask = EDP_PSR_AUX_CTL_TIME_OUT_MASK |
+			   EDP_PSR_AUX_CTL_MESSAGE_SIZE_MASK |
+			   EDP_PSR_AUX_CTL_PRECHARGE_2US_MASK |
+			   EDP_PSR_AUX_CTL_BIT_CLOCK_2X_MASK;
 
 	BUILD_BUG_ON(sizeof(aux_msg) > 20);
+	for (i = 0; i < sizeof(aux_msg); i += 4)
+		I915_WRITE(EDP_PSR_AUX_DATA(i >> 2),
+			   intel_dp_pack_aux(&aux_msg[i], sizeof(aux_msg) - i));
 
 	aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0);
 
-	/* Enable AUX frame sync at sink */
-	if (dev_priv->psr.aux_frame_sync)
-		drm_dp_dpcd_writeb(&intel_dp->aux,
-				DP_SINK_DEVICE_AUX_FRAME_SYNC_CONF,
-				DP_AUX_FRAME_SYNC_ENABLE);
+	/* Start with bits set for DDI_AUX_CTL register */
+	aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, 0, sizeof(aux_msg),
+					     aux_clock_divider);
+
+	/* Select only valid bits for SRD_AUX_CTL */
+	aux_ctl &= psr_aux_mask;
+	I915_WRITE(EDP_PSR_AUX_CTL, aux_ctl);
+}
+
+static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
+{
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	struct drm_device *dev = dig_port->base.base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	u8 dpcd_val = DP_PSR_ENABLE;
+
 	/* Enable ALPM at sink for psr2 */
-	if (dev_priv->psr.psr2_support && dev_priv->psr.alpm)
+	if (dev_priv->psr.psr2_enabled && dev_priv->psr.alpm)
 		drm_dp_dpcd_writeb(&intel_dp->aux,
 				DP_RECEIVER_ALPM_CONFIG,
 				DP_ALPM_ENABLE);
-	if (dev_priv->psr.link_standby)
-		drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG,
-				   DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE);
-	else
-		drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG,
-				   DP_PSR_ENABLE);
-
-	aux_ctl_reg = psr_aux_ctl_reg(dev_priv, port);
 
-	/* Setup AUX registers */
-	for (i = 0; i < sizeof(aux_msg); i += 4)
-		I915_WRITE(psr_aux_data_reg(dev_priv, port, i >> 2),
-			   intel_dp_pack_aux(&aux_msg[i], sizeof(aux_msg) - i));
+	if (dev_priv->psr.psr2_enabled)
+		dpcd_val |= DP_PSR_ENABLE_PSR2;
+	if (dev_priv->psr.link_standby)
+		dpcd_val |= DP_PSR_MAIN_LINK_ACTIVE;
+	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, dpcd_val);
 
-	aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, 0, sizeof(aux_msg),
-					     aux_clock_divider);
-	I915_WRITE(aux_ctl_reg, aux_ctl);
+	drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
 }
 
 static void vlv_psr_enable_source(struct intel_dp *intel_dp,
@@ -396,25 +502,16 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp)
 	 * with the 5 or 6 idle patterns.
 	 */
 	uint32_t idle_frames = max(6, dev_priv->vbt.psr.idle_frames);
-	uint32_t val;
-	uint8_t sink_latency;
-
-	val = idle_frames << EDP_PSR_IDLE_FRAME_SHIFT;
+	u32 val = idle_frames << EDP_PSR2_IDLE_FRAME_SHIFT;
 
 	/* FIXME: selective update is probably totally broken because it doesn't
 	 * mesh at all with our frontbuffer tracking. And the hw alone isn't
 	 * good enough. */
-	val |= EDP_PSR2_ENABLE |
-		EDP_SU_TRACK_ENABLE;
+	val |= EDP_PSR2_ENABLE | EDP_SU_TRACK_ENABLE;
+	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
+		val |= EDP_Y_COORDINATE_ENABLE;
 
-	if (drm_dp_dpcd_readb(&intel_dp->aux,
-				DP_SYNCHRONIZATION_LATENCY_IN_SINK,
-				&sink_latency) == 1) {
-		sink_latency &= DP_MAX_RESYNC_FRAME_COUNT_MASK;
-	} else {
-		sink_latency = 0;
-	}
-	val |= EDP_PSR2_FRAME_BEFORE_SU(sink_latency + 1);
+	val |= EDP_PSR2_FRAME_BEFORE_SU(dev_priv->psr.sink_sync_latency + 1);
 
 	if (dev_priv->vbt.psr.tp2_tp3_wakeup_time > 5)
 		val |= EDP_PSR2_TP2_TIME_2500;
@@ -440,7 +537,7 @@ static void hsw_psr_activate(struct intel_dp *intel_dp)
 	 */
 
 	/* psr1 and psr2 are mutually exclusive.*/
-	if (dev_priv->psr.psr2_support)
+	if (dev_priv->psr.psr2_enabled)
 		hsw_activate_psr2(intel_dp);
 	else
 		hsw_activate_psr1(intel_dp);
@@ -460,7 +557,7 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 	 * dynamically during PSR enable, and extracted from sink
 	 * caps during eDP detection.
 	 */
-	if (!dev_priv->psr.psr2_support)
+	if (!dev_priv->psr.sink_psr2_support)
 		return false;
 
 	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
@@ -478,15 +575,6 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 		return false;
 	}
 
-	/*
-	 * FIXME:enable psr2 only for y-cordinate psr2 panels
-	 * After gtc implementation , remove this restriction.
-	 */
-	if (!dev_priv->psr.y_cord_support) {
-		DRM_DEBUG_KMS("PSR2 not enabled, panel does not support Y coordinate\n");
-		return false;
-	}
-
 	return true;
 }
 
@@ -568,7 +656,7 @@ static void intel_psr_activate(struct intel_dp *intel_dp)
 	struct drm_device *dev = intel_dig_port->base.base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
-	if (dev_priv->psr.psr2_support)
+	if (dev_priv->psr.psr2_enabled)
 		WARN_ON(I915_READ(EDP_PSR2_CTL) & EDP_PSR2_ENABLE);
 	else
 		WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
@@ -586,14 +674,24 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp,
 	struct drm_device *dev = dig_port->base.base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
-	u32 chicken;
 
 	psr_aux_io_power_get(intel_dp);
 
-	if (dev_priv->psr.psr2_support) {
-		chicken = PSR2_VSC_ENABLE_PROG_HEADER;
-		if (dev_priv->psr.y_cord_support)
-			chicken |= PSR2_ADD_VERTICAL_LINE_COUNT;
+	/* Only HSW and BDW have PSR AUX registers that need to be setup. SKL+
+	 * use hardcoded values PSR AUX transactions
+	 */
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
+		hsw_psr_setup_aux(intel_dp);
+
+	if (dev_priv->psr.psr2_enabled) {
+		u32 chicken = I915_READ(CHICKEN_TRANS(cpu_transcoder));
+
+		if (INTEL_GEN(dev_priv) == 9 && !IS_GEMINILAKE(dev_priv))
+			chicken |= (PSR2_VSC_ENABLE_PROG_HEADER
+				   | PSR2_ADD_VERTICAL_LINE_COUNT);
+
+		else
+			chicken &= ~VSC_DATA_SEL_SOFTWARE_CONTROL;
 		I915_WRITE(CHICKEN_TRANS(cpu_transcoder), chicken);
 
 		I915_WRITE(EDP_PSR_DEBUG,
@@ -613,7 +711,8 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp,
 		I915_WRITE(EDP_PSR_DEBUG,
 			   EDP_PSR_DEBUG_MASK_MEMUP |
 			   EDP_PSR_DEBUG_MASK_HPD |
-			   EDP_PSR_DEBUG_MASK_LPSP);
+			   EDP_PSR_DEBUG_MASK_LPSP |
+			   EDP_PSR_DEBUG_MASK_DISP_REG_WRITE);
 	}
 }
 
@@ -644,7 +743,7 @@ void intel_psr_enable(struct intel_dp *intel_dp,
 		goto unlock;
 	}
 
-	dev_priv->psr.psr2_support = crtc_state->has_psr2;
+	dev_priv->psr.psr2_enabled = crtc_state->has_psr2;
 	dev_priv->psr.busy_frontbuffer_bits = 0;
 
 	dev_priv->psr.setup_vsc(intel_dp, crtc_state);
@@ -714,12 +813,7 @@ static void hsw_psr_disable(struct intel_dp *intel_dp,
 		i915_reg_t psr_status;
 		u32 psr_status_mask;
 
-		if (dev_priv->psr.aux_frame_sync)
-			drm_dp_dpcd_writeb(&intel_dp->aux,
-					DP_SINK_DEVICE_AUX_FRAME_SYNC_CONF,
-					0);
-
-		if (dev_priv->psr.psr2_support) {
+		if (dev_priv->psr.psr2_enabled) {
 			psr_status = EDP_PSR2_STATUS;
 			psr_status_mask = EDP_PSR2_STATUS_STATE_MASK;
 
@@ -743,7 +837,7 @@ static void hsw_psr_disable(struct intel_dp *intel_dp,
 
 		dev_priv->psr.active = false;
 	} else {
-		if (dev_priv->psr.psr2_support)
+		if (dev_priv->psr.psr2_enabled)
 			WARN_ON(I915_READ(EDP_PSR2_CTL) & EDP_PSR2_ENABLE);
 		else
 			WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
@@ -789,53 +883,59 @@ void intel_psr_disable(struct intel_dp *intel_dp,
 	cancel_delayed_work_sync(&dev_priv->psr.work);
 }
 
-static void intel_psr_work(struct work_struct *work)
+static bool psr_wait_for_idle(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv =
-		container_of(work, typeof(*dev_priv), psr.work.work);
-	struct intel_dp *intel_dp = dev_priv->psr.enabled;
-	struct drm_crtc *crtc = dp_to_dig_port(intel_dp)->base.base.crtc;
-	enum pipe pipe = to_intel_crtc(crtc)->pipe;
+	struct intel_dp *intel_dp;
+	i915_reg_t reg;
+	u32 mask;
+	int err;
+
+	intel_dp = dev_priv->psr.enabled;
+	if (!intel_dp)
+		return false;
 
-	/* We have to make sure PSR is ready for re-enable
-	 * otherwise it keeps disabled until next full enable/disable cycle.
-	 * PSR might take some time to get fully disabled
-	 * and be ready for re-enable.
-	 */
 	if (HAS_DDI(dev_priv)) {
-		if (dev_priv->psr.psr2_support) {
-			if (intel_wait_for_register(dev_priv,
-						    EDP_PSR2_STATUS,
-						    EDP_PSR2_STATUS_STATE_MASK,
-						    0,
-						    50)) {
-				DRM_ERROR("Timed out waiting for PSR2 Idle for re-enable\n");
-				return;
-			}
+		if (dev_priv->psr.psr2_enabled) {
+			reg = EDP_PSR2_STATUS;
+			mask = EDP_PSR2_STATUS_STATE_MASK;
 		} else {
-			if (intel_wait_for_register(dev_priv,
-						    EDP_PSR_STATUS,
-						    EDP_PSR_STATUS_STATE_MASK,
-						    0,
-						    50)) {
-				DRM_ERROR("Timed out waiting for PSR Idle for re-enable\n");
-				return;
-			}
+			reg = EDP_PSR_STATUS;
+			mask = EDP_PSR_STATUS_STATE_MASK;
 		}
 	} else {
-		if (intel_wait_for_register(dev_priv,
-					    VLV_PSRSTAT(pipe),
-					    VLV_EDP_PSR_IN_TRANS,
-					    0,
-					    1)) {
-			DRM_ERROR("Timed out waiting for PSR Idle for re-enable\n");
-			return;
-		}
+		struct drm_crtc *crtc =
+			dp_to_dig_port(intel_dp)->base.base.crtc;
+		enum pipe pipe = to_intel_crtc(crtc)->pipe;
+
+		reg = VLV_PSRSTAT(pipe);
+		mask = VLV_EDP_PSR_IN_TRANS;
 	}
+
+	mutex_unlock(&dev_priv->psr.lock);
+
+	err = intel_wait_for_register(dev_priv, reg, mask, 0, 50);
+	if (err)
+		DRM_ERROR("Timed out waiting for PSR Idle for re-enable\n");
+
+	/* After the unlocked wait, verify that PSR is still wanted! */
 	mutex_lock(&dev_priv->psr.lock);
-	intel_dp = dev_priv->psr.enabled;
+	return err == 0 && dev_priv->psr.enabled;
+}
 
-	if (!intel_dp)
+static void intel_psr_work(struct work_struct *work)
+{
+	struct drm_i915_private *dev_priv =
+		container_of(work, typeof(*dev_priv), psr.work.work);
+
+	mutex_lock(&dev_priv->psr.lock);
+
+	/*
+	 * We have to make sure PSR is ready for re-enable
+	 * otherwise it keeps disabled until next full enable/disable cycle.
+	 * PSR might take some time to get fully disabled
+	 * and be ready for re-enable.
+	 */
+	if (!psr_wait_for_idle(dev_priv))
 		goto unlock;
 
 	/*
@@ -846,7 +946,7 @@ static void intel_psr_work(struct work_struct *work)
 	if (dev_priv->psr.busy_frontbuffer_bits)
 		goto unlock;
 
-	intel_psr_activate(intel_dp);
+	intel_psr_activate(dev_priv->psr.enabled);
 unlock:
 	mutex_unlock(&dev_priv->psr.lock);
 }
@@ -862,11 +962,7 @@ static void intel_psr_exit(struct drm_i915_private *dev_priv)
 		return;
 
 	if (HAS_DDI(dev_priv)) {
-		if (dev_priv->psr.aux_frame_sync)
-			drm_dp_dpcd_writeb(&intel_dp->aux,
-					DP_SINK_DEVICE_AUX_FRAME_SYNC_CONF,
-					0);
-		if (dev_priv->psr.psr2_support) {
+		if (dev_priv->psr.psr2_enabled) {
 			val = I915_READ(EDP_PSR2_CTL);
 			WARN_ON(!(val & EDP_PSR2_ENABLE));
 			I915_WRITE(EDP_PSR2_CTL, val & ~EDP_PSR2_ENABLE);
@@ -957,6 +1053,7 @@ void intel_psr_single_frame_update(struct drm_i915_private *dev_priv,
  * intel_psr_invalidate - Invalidade PSR
  * @dev_priv: i915 device
  * @frontbuffer_bits: frontbuffer plane tracking bits
+ * @origin: which operation caused the invalidate
  *
  * Since the hardware frontbuffer tracking has gaps we need to integrate
  * with the software frontbuffer tracking. This function gets called every
@@ -966,7 +1063,7 @@ void intel_psr_single_frame_update(struct drm_i915_private *dev_priv,
  * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits."
  */
 void intel_psr_invalidate(struct drm_i915_private *dev_priv,
-			  unsigned frontbuffer_bits)
+			  unsigned frontbuffer_bits, enum fb_op_origin origin)
 {
 	struct drm_crtc *crtc;
 	enum pipe pipe;
@@ -974,6 +1071,9 @@ void intel_psr_invalidate(struct drm_i915_private *dev_priv,
 	if (!CAN_PSR(dev_priv))
 		return;
 
+	if (dev_priv->psr.has_hw_tracking && origin == ORIGIN_FLIP)
+		return;
+
 	mutex_lock(&dev_priv->psr.lock);
 	if (!dev_priv->psr.enabled) {
 		mutex_unlock(&dev_priv->psr.lock);
@@ -1014,6 +1114,9 @@ void intel_psr_flush(struct drm_i915_private *dev_priv,
 	if (!CAN_PSR(dev_priv))
 		return;
 
+	if (dev_priv->psr.has_hw_tracking && origin == ORIGIN_FLIP)
+		return;
+
 	mutex_lock(&dev_priv->psr.lock);
 	if (!dev_priv->psr.enabled) {
 		mutex_unlock(&dev_priv->psr.lock);
@@ -1027,8 +1130,23 @@ void intel_psr_flush(struct drm_i915_private *dev_priv,
 	dev_priv->psr.busy_frontbuffer_bits &= ~frontbuffer_bits;
 
 	/* By definition flush = invalidate + flush */
-	if (frontbuffer_bits)
-		intel_psr_exit(dev_priv);
+	if (frontbuffer_bits) {
+		if (dev_priv->psr.psr2_enabled ||
+		    IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
+			intel_psr_exit(dev_priv);
+		} else {
+			/*
+			 * Display WA #0884: all
+			 * This documented WA for bxt can be safely applied
+			 * broadly so we can force HW tracking to exit PSR
+			 * instead of disabling and re-enabling.
+			 * Workaround tells us to write 0 to CUR_SURFLIVE_A,
+			 * but it makes more sense write to the current active
+			 * pipe.
+			 */
+			I915_WRITE(CURSURFLIVE(pipe), 0);
+		}
+	}
 
 	if (!dev_priv->psr.active && !dev_priv->psr.busy_frontbuffer_bits)
 		if (!work_busy(&dev_priv->psr.work.work))
@@ -1055,9 +1173,12 @@ void intel_psr_init(struct drm_i915_private *dev_priv)
 	if (!dev_priv->psr.sink_support)
 		return;
 
-	/* Per platform default: all disabled. */
-	if (i915_modparams.enable_psr == -1)
+	if (i915_modparams.enable_psr == -1) {
+		i915_modparams.enable_psr = dev_priv->vbt.psr.enable;
+
+		/* Per platform default: all disabled. */
 		i915_modparams.enable_psr = 0;
+	}
 
 	/* Set link_standby x link_off defaults */
 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
@@ -1090,6 +1211,7 @@ void intel_psr_init(struct drm_i915_private *dev_priv)
 		dev_priv->psr.activate = vlv_psr_activate;
 		dev_priv->psr.setup_vsc = vlv_psr_setup_vsc;
 	} else {
+		dev_priv->psr.has_hw_tracking = true;
 		dev_priv->psr.enable_source = hsw_psr_enable_source;
 		dev_priv->psr.disable_source = hsw_psr_disable;
 		dev_priv->psr.enable_sink = hsw_psr_enable_sink;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 1d599524a759..8f19349a6055 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -36,6 +36,7 @@
 #include "i915_gem_render_state.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
+#include "intel_workarounds.h"
 
 /* Rough estimate of the typical request size, performing a flush,
  * set-context and then emitting the batch.
@@ -557,7 +558,8 @@ static void reset_ring_common(struct intel_engine_cs *engine,
 	 */
 	if (request) {
 		struct drm_i915_private *dev_priv = request->i915;
-		struct intel_context *ce = &request->ctx->engine[engine->id];
+		struct intel_context *ce = to_intel_context(request->ctx,
+							    engine);
 		struct i915_hw_ppgtt *ppgtt;
 
 		if (ce->state) {
@@ -599,7 +601,7 @@ static int intel_rcs_ctx_init(struct i915_request *rq)
 {
 	int ret;
 
-	ret = intel_ring_workarounds_emit(rq);
+	ret = intel_ctx_workarounds_emit(rq);
 	if (ret != 0)
 		return ret;
 
@@ -617,6 +619,8 @@ static int init_render_ring(struct intel_engine_cs *engine)
 	if (ret)
 		return ret;
 
+	intel_whitelist_workarounds_apply(engine);
+
 	/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
 	if (IS_GEN(dev_priv, 4, 6))
 		I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
@@ -658,7 +662,7 @@ static int init_render_ring(struct intel_engine_cs *engine)
 	if (INTEL_GEN(dev_priv) >= 6)
 		I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
 
-	return init_workarounds_ring(engine);
+	return 0;
 }
 
 static u32 *gen6_signal(struct i915_request *rq, u32 *cs)
@@ -693,17 +697,17 @@ static void cancel_requests(struct intel_engine_cs *engine)
 	struct i915_request *request;
 	unsigned long flags;
 
-	spin_lock_irqsave(&engine->timeline->lock, flags);
+	spin_lock_irqsave(&engine->timeline.lock, flags);
 
 	/* Mark all submitted requests as skipped. */
-	list_for_each_entry(request, &engine->timeline->requests, link) {
+	list_for_each_entry(request, &engine->timeline.requests, link) {
 		GEM_BUG_ON(!request->global_seqno);
 		if (!i915_request_completed(request))
 			dma_fence_set_error(&request->fence, -EIO);
 	}
 	/* Remaining _unready_ requests will be nop'ed when submitted */
 
-	spin_unlock_irqrestore(&engine->timeline->lock, flags);
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
 static void i9xx_submit_request(struct i915_request *request)
@@ -1062,7 +1066,6 @@ int intel_ring_pin(struct intel_ring *ring,
 
 void intel_ring_reset(struct intel_ring *ring, u32 tail)
 {
-	GEM_BUG_ON(!list_empty(&ring->request_list));
 	ring->tail = tail;
 	ring->head = tail;
 	ring->emit = tail;
@@ -1114,19 +1117,24 @@ intel_ring_create_vma(struct drm_i915_private *dev_priv, int size)
 }
 
 struct intel_ring *
-intel_engine_create_ring(struct intel_engine_cs *engine, int size)
+intel_engine_create_ring(struct intel_engine_cs *engine,
+			 struct i915_timeline *timeline,
+			 int size)
 {
 	struct intel_ring *ring;
 	struct i915_vma *vma;
 
 	GEM_BUG_ON(!is_power_of_2(size));
 	GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
+	GEM_BUG_ON(timeline == &engine->timeline);
+	lockdep_assert_held(&engine->i915->drm.struct_mutex);
 
 	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
 	if (!ring)
 		return ERR_PTR(-ENOMEM);
 
 	INIT_LIST_HEAD(&ring->request_list);
+	ring->timeline = i915_timeline_get(timeline);
 
 	ring->size = size;
 	/* Workaround an erratum on the i830 which causes a hang if
@@ -1157,12 +1165,13 @@ intel_ring_free(struct intel_ring *ring)
 	i915_vma_close(ring->vma);
 	__i915_gem_object_release_unless_active(obj);
 
+	i915_timeline_put(ring->timeline);
 	kfree(ring);
 }
 
-static int context_pin(struct i915_gem_context *ctx)
+static int context_pin(struct intel_context *ce)
 {
-	struct i915_vma *vma = ctx->engine[RCS].state;
+	struct i915_vma *vma = ce->state;
 	int ret;
 
 	/*
@@ -1253,7 +1262,7 @@ static struct intel_ring *
 intel_ring_context_pin(struct intel_engine_cs *engine,
 		       struct i915_gem_context *ctx)
 {
-	struct intel_context *ce = &ctx->engine[engine->id];
+	struct intel_context *ce = to_intel_context(ctx, engine);
 	int ret;
 
 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
@@ -1275,7 +1284,7 @@ intel_ring_context_pin(struct intel_engine_cs *engine,
 	}
 
 	if (ce->state) {
-		ret = context_pin(ctx);
+		ret = context_pin(ce);
 		if (ret)
 			goto err;
 
@@ -1296,7 +1305,7 @@ intel_ring_context_pin(struct intel_engine_cs *engine,
 static void intel_ring_context_unpin(struct intel_engine_cs *engine,
 				     struct i915_gem_context *ctx)
 {
-	struct intel_context *ce = &ctx->engine[engine->id];
+	struct intel_context *ce = to_intel_context(ctx, engine);
 
 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
 	GEM_BUG_ON(ce->pin_count == 0);
@@ -1315,6 +1324,7 @@ static void intel_ring_context_unpin(struct intel_engine_cs *engine,
 static int intel_init_ring_buffer(struct intel_engine_cs *engine)
 {
 	struct intel_ring *ring;
+	struct i915_timeline *timeline;
 	int err;
 
 	intel_engine_setup_common(engine);
@@ -1323,7 +1333,14 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine)
 	if (err)
 		goto err;
 
-	ring = intel_engine_create_ring(engine, 32 * PAGE_SIZE);
+	timeline = i915_timeline_create(engine->i915, engine->name);
+	if (IS_ERR(timeline)) {
+		err = PTR_ERR(timeline);
+		goto err;
+	}
+
+	ring = intel_engine_create_ring(engine, timeline, 32 * PAGE_SIZE);
+	i915_timeline_put(timeline);
 	if (IS_ERR(ring)) {
 		err = PTR_ERR(ring);
 		goto err;
@@ -1424,7 +1441,7 @@ static inline int mi_set_context(struct i915_request *rq, u32 flags)
 
 	*cs++ = MI_NOOP;
 	*cs++ = MI_SET_CONTEXT;
-	*cs++ = i915_ggtt_offset(rq->ctx->engine[RCS].state) | flags;
+	*cs++ = i915_ggtt_offset(to_intel_context(rq->ctx, engine)->state) | flags;
 	/*
 	 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
 	 * WaMiSetContext_Hang:snb,ivb,vlv
@@ -1515,7 +1532,7 @@ static int switch_context(struct i915_request *rq)
 		hw_flags = MI_FORCE_RESTORE;
 	}
 
-	if (to_ctx->engine[engine->id].state &&
+	if (to_intel_context(to_ctx, engine)->state &&
 	    (to_ctx != from_ctx || hw_flags & MI_FORCE_RESTORE)) {
 		GEM_BUG_ON(engine->id != RCS);
 
@@ -1563,7 +1580,7 @@ static int ring_request_alloc(struct i915_request *request)
 {
 	int ret;
 
-	GEM_BUG_ON(!request->ctx->engine[request->engine->id].pin_count);
+	GEM_BUG_ON(!to_intel_context(request->ctx, request->engine)->pin_count);
 
 	/* Flush enough space to reduce the likelihood of waiting after
 	 * we start building the request - in which case we will just
@@ -1593,6 +1610,7 @@ static noinline int wait_for_space(struct intel_ring *ring, unsigned int bytes)
 	if (intel_ring_update_space(ring) >= bytes)
 		return 0;
 
+	GEM_BUG_ON(list_empty(&ring->request_list));
 	list_for_each_entry(target, &ring->request_list, ring_link) {
 		/* Would completion of this request free enough space? */
 		if (bytes <= __intel_ring_space(target->postfix,
@@ -1692,17 +1710,18 @@ u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
 		need_wrap &= ~1;
 		GEM_BUG_ON(need_wrap > ring->space);
 		GEM_BUG_ON(ring->emit + need_wrap > ring->size);
+		GEM_BUG_ON(!IS_ALIGNED(need_wrap, sizeof(u64)));
 
 		/* Fill the tail with MI_NOOP */
-		memset(ring->vaddr + ring->emit, 0, need_wrap);
-		ring->emit = 0;
+		memset64(ring->vaddr + ring->emit, 0, need_wrap / sizeof(u64));
 		ring->space -= need_wrap;
+		ring->emit = 0;
 	}
 
 	GEM_BUG_ON(ring->emit > ring->size - bytes);
 	GEM_BUG_ON(ring->space < bytes);
 	cs = ring->vaddr + ring->emit;
-	GEM_DEBUG_EXEC(memset(cs, POISON_INUSE, bytes));
+	GEM_DEBUG_EXEC(memset32(cs, POISON_INUSE, bytes / sizeof(*cs)));
 	ring->emit += bytes;
 	ring->space -= bytes;
 
@@ -1712,22 +1731,24 @@ u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
 /* Align the ring tail to a cacheline boundary */
 int intel_ring_cacheline_align(struct i915_request *rq)
 {
-	int num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);
-	u32 *cs;
+	int num_dwords;
+	void *cs;
 
+	num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);
 	if (num_dwords == 0)
 		return 0;
 
-	num_dwords = CACHELINE_BYTES / sizeof(u32) - num_dwords;
+	num_dwords = CACHELINE_DWORDS - num_dwords;
+	GEM_BUG_ON(num_dwords & 1);
+
 	cs = intel_ring_begin(rq, num_dwords);
 	if (IS_ERR(cs))
 		return PTR_ERR(cs);
 
-	while (num_dwords--)
-		*cs++ = MI_NOOP;
-
+	memset64(cs, (u64)MI_NOOP << 32 | MI_NOOP, num_dwords / 2);
 	intel_ring_advance(rq, cs);
 
+	GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1));
 	return 0;
 }
 
@@ -1943,8 +1964,6 @@ static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
 static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
 				struct intel_engine_cs *engine)
 {
-	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << engine->irq_shift;
-
 	if (INTEL_GEN(dev_priv) >= 6) {
 		engine->irq_enable = gen6_irq_enable;
 		engine->irq_disable = gen6_irq_disable;
@@ -2029,6 +2048,8 @@ int intel_init_render_ring_buffer(struct intel_engine_cs *engine)
 	if (HAS_L3_DPF(dev_priv))
 		engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
 
+	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
+
 	if (INTEL_GEN(dev_priv) >= 6) {
 		engine->init_context = intel_rcs_ctx_init;
 		engine->emit_flush = gen7_render_ring_flush;
@@ -2079,7 +2100,6 @@ int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine)
 		engine->emit_flush = gen6_bsd_ring_flush;
 		engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
 	} else {
-		engine->mmio_base = BSD_RING_BASE;
 		engine->emit_flush = bsd_ring_flush;
 		if (IS_GEN5(dev_priv))
 			engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 0320c2c4cfba..010750e8ee44 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -3,15 +3,19 @@
 #define _INTEL_RINGBUFFER_H_
 
 #include <linux/hashtable.h>
+#include <linux/seqlock.h>
 
 #include "i915_gem_batch_pool.h"
-#include "i915_gem_timeline.h"
 
+#include "i915_reg.h"
 #include "i915_pmu.h"
 #include "i915_request.h"
 #include "i915_selftest.h"
+#include "i915_timeline.h"
+#include "intel_gpu_commands.h"
 
 struct drm_printer;
+struct i915_sched_attr;
 
 #define I915_CMD_HASH_ORDER 9
 
@@ -84,7 +88,7 @@ hangcheck_action_to_str(const enum intel_engine_hangcheck_action a)
 }
 
 #define I915_MAX_SLICES	3
-#define I915_MAX_SUBSLICES 3
+#define I915_MAX_SUBSLICES 8
 
 #define instdone_slice_mask(dev_priv__) \
 	(INTEL_GEN(dev_priv__) == 7 ? \
@@ -125,7 +129,9 @@ struct intel_ring {
 	struct i915_vma *vma;
 	void *vaddr;
 
+	struct i915_timeline *timeline;
 	struct list_head request_list;
+	struct list_head active_link;
 
 	u32 head;
 	u32 tail;
@@ -330,10 +336,10 @@ struct intel_engine_cs {
 	u8 instance;
 	u32 context_size;
 	u32 mmio_base;
-	unsigned int irq_shift;
 
 	struct intel_ring *buffer;
-	struct intel_timeline *timeline;
+
+	struct i915_timeline timeline;
 
 	struct drm_i915_gem_object *default_state;
 
@@ -459,7 +465,8 @@ struct intel_engine_cs {
 	 *
 	 * Called under the struct_mutex.
 	 */
-	void		(*schedule)(struct i915_request *request, int priority);
+	void		(*schedule)(struct i915_request *request,
+				    const struct i915_sched_attr *attr);
 
 	/*
 	 * Cancel all requests on the hardware, or queued for execution.
@@ -561,6 +568,7 @@ struct intel_engine_cs {
 
 #define I915_ENGINE_NEEDS_CMD_PARSER BIT(0)
 #define I915_ENGINE_SUPPORTS_STATS   BIT(1)
+#define I915_ENGINE_HAS_PREEMPTION   BIT(2)
 	unsigned int flags;
 
 	/*
@@ -591,7 +599,7 @@ struct intel_engine_cs {
 		/**
 		 * @lock: Lock protecting the below fields.
 		 */
-		spinlock_t lock;
+		seqlock_t lock;
 		/**
 		 * @enabled: Reference count indicating number of listeners.
 		 */
@@ -620,16 +628,29 @@ struct intel_engine_cs {
 	} stats;
 };
 
-static inline bool intel_engine_needs_cmd_parser(struct intel_engine_cs *engine)
+static inline bool
+intel_engine_needs_cmd_parser(const struct intel_engine_cs *engine)
 {
 	return engine->flags & I915_ENGINE_NEEDS_CMD_PARSER;
 }
 
-static inline bool intel_engine_supports_stats(struct intel_engine_cs *engine)
+static inline bool
+intel_engine_supports_stats(const struct intel_engine_cs *engine)
 {
 	return engine->flags & I915_ENGINE_SUPPORTS_STATS;
 }
 
+static inline bool
+intel_engine_has_preemption(const struct intel_engine_cs *engine)
+{
+	return engine->flags & I915_ENGINE_HAS_PREEMPTION;
+}
+
+static inline bool __execlists_need_preempt(int prio, int last)
+{
+	return prio > max(0, last);
+}
+
 static inline void
 execlists_set_active(struct intel_engine_execlists *execlists,
 		     unsigned int bit)
@@ -637,6 +658,13 @@ execlists_set_active(struct intel_engine_execlists *execlists,
 	__set_bit(bit, (unsigned long *)&execlists->active);
 }
 
+static inline bool
+execlists_set_active_once(struct intel_engine_execlists *execlists,
+			  unsigned int bit)
+{
+	return !__test_and_set_bit(bit, (unsigned long *)&execlists->active);
+}
+
 static inline void
 execlists_clear_active(struct intel_engine_execlists *execlists,
 		       unsigned int bit)
@@ -651,6 +679,10 @@ execlists_is_active(const struct intel_engine_execlists *execlists,
 	return test_bit(bit, (unsigned long *)&execlists->active);
 }
 
+void execlists_user_begin(struct intel_engine_execlists *execlists,
+			  const struct execlist_port *port);
+void execlists_user_end(struct intel_engine_execlists *execlists);
+
 void
 execlists_cancel_port_requests(struct intel_engine_execlists * const execlists);
 
@@ -663,7 +695,7 @@ execlists_num_ports(const struct intel_engine_execlists * const execlists)
 	return execlists->port_mask + 1;
 }
 
-static inline void
+static inline struct execlist_port *
 execlists_port_complete(struct intel_engine_execlists * const execlists,
 			struct execlist_port * const port)
 {
@@ -674,6 +706,8 @@ execlists_port_complete(struct intel_engine_execlists * const execlists,
 
 	memmove(port, port + 1, m * sizeof(struct execlist_port));
 	memset(port + m, 0, sizeof(struct execlist_port));
+
+	return port;
 }
 
 static inline unsigned int
@@ -736,7 +770,9 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
 #define CNL_HWS_CSB_WRITE_INDEX		0x2f
 
 struct intel_ring *
-intel_engine_create_ring(struct intel_engine_cs *engine, int size);
+intel_engine_create_ring(struct intel_engine_cs *engine,
+			 struct i915_timeline *timeline,
+			 int size);
 int intel_ring_pin(struct intel_ring *ring,
 		   struct drm_i915_private *i915,
 		   unsigned int offset_bias);
@@ -854,12 +890,9 @@ static inline u32 intel_engine_last_submit(struct intel_engine_cs *engine)
 	 * wtih serialising this hint with anything, so document it as
 	 * a hint and nothing more.
 	 */
-	return READ_ONCE(engine->timeline->seqno);
+	return READ_ONCE(engine->timeline.seqno);
 }
 
-int init_workarounds_ring(struct intel_engine_cs *engine);
-int intel_ring_workarounds_emit(struct i915_request *rq);
-
 void intel_engine_get_instdone(struct intel_engine_cs *engine,
 			       struct intel_instdone *instdone);
 
@@ -939,7 +972,7 @@ bool intel_engine_add_wait(struct intel_engine_cs *engine,
 			   struct intel_wait *wait);
 void intel_engine_remove_wait(struct intel_engine_cs *engine,
 			      struct intel_wait *wait);
-void intel_engine_enable_signaling(struct i915_request *request, bool wakeup);
+bool intel_engine_enable_signaling(struct i915_request *request, bool wakeup);
 void intel_engine_cancel_signaling(struct i915_request *request);
 
 static inline bool intel_engine_has_waiter(const struct intel_engine_cs *engine)
@@ -1037,7 +1070,7 @@ static inline void intel_engine_context_in(struct intel_engine_cs *engine)
 	if (READ_ONCE(engine->stats.enabled) == 0)
 		return;
 
-	spin_lock_irqsave(&engine->stats.lock, flags);
+	write_seqlock_irqsave(&engine->stats.lock, flags);
 
 	if (engine->stats.enabled > 0) {
 		if (engine->stats.active++ == 0)
@@ -1045,7 +1078,7 @@ static inline void intel_engine_context_in(struct intel_engine_cs *engine)
 		GEM_BUG_ON(engine->stats.active == 0);
 	}
 
-	spin_unlock_irqrestore(&engine->stats.lock, flags);
+	write_sequnlock_irqrestore(&engine->stats.lock, flags);
 }
 
 static inline void intel_engine_context_out(struct intel_engine_cs *engine)
@@ -1055,7 +1088,7 @@ static inline void intel_engine_context_out(struct intel_engine_cs *engine)
 	if (READ_ONCE(engine->stats.enabled) == 0)
 		return;
 
-	spin_lock_irqsave(&engine->stats.lock, flags);
+	write_seqlock_irqsave(&engine->stats.lock, flags);
 
 	if (engine->stats.enabled > 0) {
 		ktime_t last;
@@ -1082,7 +1115,7 @@ static inline void intel_engine_context_out(struct intel_engine_cs *engine)
 		}
 	}
 
-	spin_unlock_irqrestore(&engine->stats.lock, flags);
+	write_sequnlock_irqrestore(&engine->stats.lock, flags);
 }
 
 int intel_enable_engine_stats(struct intel_engine_cs *engine);
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index 66de4b2dc8b7..53a6eaa9671a 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -542,6 +542,29 @@ void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 	dev_priv->csr.dc_state = val;
 }
 
+/**
+ * gen9_set_dc_state - set target display C power state
+ * @dev_priv: i915 device instance
+ * @state: target DC power state
+ * - DC_STATE_DISABLE
+ * - DC_STATE_EN_UPTO_DC5
+ * - DC_STATE_EN_UPTO_DC6
+ * - DC_STATE_EN_DC9
+ *
+ * Signal to DMC firmware/HW the target DC power state passed in @state.
+ * DMC/HW can turn off individual display clocks and power rails when entering
+ * a deeper DC power state (higher in number) and turns these back when exiting
+ * that state to a shallower power state (lower in number). The HW will decide
+ * when to actually enter a given state on an on-demand basis, for instance
+ * depending on the active state of display pipes. The state of display
+ * registers backed by affected power rails are saved/restored as needed.
+ *
+ * Based on the above enabling a deeper DC power state is asynchronous wrt.
+ * enabling it. Disabling a deeper power state is synchronous: for instance
+ * setting %DC_STATE_DISABLE won't complete until all HW resources are turned
+ * back on and register state is restored. This is guaranteed by the MMIO write
+ * to DC_STATE_EN blocking until the state is restored.
+ */
 static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
 {
 	uint32_t val;
@@ -635,7 +658,7 @@ static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
 	assert_csr_loaded(dev_priv);
 }
 
-void skl_enable_dc6(struct drm_i915_private *dev_priv)
+static void skl_enable_dc6(struct drm_i915_private *dev_priv)
 {
 	assert_can_enable_dc6(dev_priv);
 
@@ -649,13 +672,6 @@ void skl_enable_dc6(struct drm_i915_private *dev_priv)
 	gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 }
 
-void skl_disable_dc6(struct drm_i915_private *dev_priv)
-{
-	DRM_DEBUG_KMS("Disabling DC6\n");
-
-	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
-}
-
 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 				   struct i915_power_well *power_well)
 {
@@ -2626,32 +2642,69 @@ static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
 	mutex_unlock(&power_domains->lock);
 }
 
-static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
+static inline
+bool intel_dbuf_slice_set(struct drm_i915_private *dev_priv,
+			  i915_reg_t reg, bool enable)
 {
-	I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
-	POSTING_READ(DBUF_CTL);
+	u32 val, status;
 
+	val = I915_READ(reg);
+	val = enable ? (val | DBUF_POWER_REQUEST) : (val & ~DBUF_POWER_REQUEST);
+	I915_WRITE(reg, val);
+	POSTING_READ(reg);
 	udelay(10);
 
-	if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
-		DRM_ERROR("DBuf power enable timeout\n");
+	status = I915_READ(reg) & DBUF_POWER_STATE;
+	if ((enable && !status) || (!enable && status)) {
+		DRM_ERROR("DBus power %s timeout!\n",
+			  enable ? "enable" : "disable");
+		return false;
+	}
+	return true;
+}
+
+static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
+{
+	intel_dbuf_slice_set(dev_priv, DBUF_CTL, true);
 }
 
 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
 {
-	I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
-	POSTING_READ(DBUF_CTL);
+	intel_dbuf_slice_set(dev_priv, DBUF_CTL, false);
+}
 
-	udelay(10);
+static u8 intel_dbuf_max_slices(struct drm_i915_private *dev_priv)
+{
+	if (INTEL_GEN(dev_priv) < 11)
+		return 1;
+	return 2;
+}
 
-	if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
-		DRM_ERROR("DBuf power disable timeout!\n");
+void icl_dbuf_slices_update(struct drm_i915_private *dev_priv,
+			    u8 req_slices)
+{
+	u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
+	u32 val;
+	bool ret;
+
+	if (req_slices > intel_dbuf_max_slices(dev_priv)) {
+		DRM_ERROR("Invalid number of dbuf slices requested\n");
+		return;
+	}
+
+	if (req_slices == hw_enabled_slices || req_slices == 0)
+		return;
+
+	val = I915_READ(DBUF_CTL_S2);
+	if (req_slices > hw_enabled_slices)
+		ret = intel_dbuf_slice_set(dev_priv, DBUF_CTL_S2, true);
+	else
+		ret = intel_dbuf_slice_set(dev_priv, DBUF_CTL_S2, false);
+
+	if (ret)
+		dev_priv->wm.skl_hw.ddb.enabled_slices = req_slices;
 }
 
-/*
- * TODO: we shouldn't always enable DBUF_CTL_S2, we should only enable it when
- * needed and keep it disabled as much as possible.
- */
 static void icl_dbuf_enable(struct drm_i915_private *dev_priv)
 {
 	I915_WRITE(DBUF_CTL_S1, I915_READ(DBUF_CTL_S1) | DBUF_POWER_REQUEST);
@@ -2663,6 +2716,8 @@ static void icl_dbuf_enable(struct drm_i915_private *dev_priv)
 	if (!(I915_READ(DBUF_CTL_S1) & DBUF_POWER_STATE) ||
 	    !(I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE))
 		DRM_ERROR("DBuf power enable timeout\n");
+	else
+		dev_priv->wm.skl_hw.ddb.enabled_slices = 2;
 }
 
 static void icl_dbuf_disable(struct drm_i915_private *dev_priv)
@@ -2676,6 +2731,8 @@ static void icl_dbuf_disable(struct drm_i915_private *dev_priv)
 	if ((I915_READ(DBUF_CTL_S1) & DBUF_POWER_STATE) ||
 	    (I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE))
 		DRM_ERROR("DBuf power disable timeout!\n");
+	else
+		dev_priv->wm.skl_hw.ddb.enabled_slices = 0;
 }
 
 static void icl_mbus_init(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 96e213ec202d..25005023c243 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -2779,9 +2779,8 @@ static bool intel_sdvo_tv_create_property(struct intel_sdvo *intel_sdvo,
 		return false;
 
 	for (i = 0; i < intel_sdvo_connector->format_supported_num; i++)
-		drm_property_add_enum(
-				intel_sdvo_connector->tv_format, i,
-				i, tv_format_names[intel_sdvo_connector->tv_format_supported[i]]);
+		drm_property_add_enum(intel_sdvo_connector->tv_format, i,
+				      tv_format_names[intel_sdvo_connector->tv_format_supported[i]]);
 
 	intel_sdvo_connector->base.base.state->tv.mode = intel_sdvo_connector->tv_format_supported[0];
 	drm_object_attach_property(&intel_sdvo_connector->base.base.base,
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index dbdcf85032df..ee23613f9fd4 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -48,6 +48,7 @@ bool intel_format_is_yuv(u32 format)
 	case DRM_FORMAT_UYVY:
 	case DRM_FORMAT_VYUY:
 	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_NV12:
 		return true;
 	default:
 		return false;
@@ -130,7 +131,7 @@ void intel_pipe_update_start(const struct intel_crtc_state *new_crtc_state)
 		if (scanline < min || scanline > max)
 			break;
 
-		if (timeout <= 0) {
+		if (!timeout) {
 			DRM_ERROR("Potential atomic update failure on pipe %c\n",
 				  pipe_name(crtc->pipe));
 			break;
@@ -935,20 +936,11 @@ intel_check_sprite_plane(struct intel_plane *plane,
 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_framebuffer *fb = state->base.fb;
-	int crtc_x, crtc_y;
-	unsigned int crtc_w, crtc_h;
-	uint32_t src_x, src_y, src_w, src_h;
-	struct drm_rect *src = &state->base.src;
-	struct drm_rect *dst = &state->base.dst;
-	struct drm_rect clip = {};
 	int max_stride = INTEL_GEN(dev_priv) >= 9 ? 32768 : 16384;
-	int hscale, vscale;
 	int max_scale, min_scale;
 	bool can_scale;
 	int ret;
-
-	*src = drm_plane_state_src(&state->base);
-	*dst = drm_plane_state_dest(&state->base);
+	uint32_t pixel_format = 0;
 
 	if (!fb) {
 		state->base.visible = false;
@@ -969,11 +961,14 @@ intel_check_sprite_plane(struct intel_plane *plane,
 
 	/* setup can_scale, min_scale, max_scale */
 	if (INTEL_GEN(dev_priv) >= 9) {
+		if (state->base.fb)
+			pixel_format = state->base.fb->format->format;
 		/* use scaler when colorkey is not required */
 		if (!state->ckey.flags) {
 			can_scale = 1;
 			min_scale = 1;
-			max_scale = skl_max_scale(crtc, crtc_state);
+			max_scale =
+				skl_max_scale(crtc, crtc_state, pixel_format);
 		} else {
 			can_scale = 0;
 			min_scale = DRM_PLANE_HELPER_NO_SCALING;
@@ -985,64 +980,19 @@ intel_check_sprite_plane(struct intel_plane *plane,
 		min_scale = plane->can_scale ? 1 : (1 << 16);
 	}
 
-	/*
-	 * FIXME the following code does a bunch of fuzzy adjustments to the
-	 * coordinates and sizes. We probably need some way to decide whether
-	 * more strict checking should be done instead.
-	 */
-	drm_rect_rotate(src, fb->width << 16, fb->height << 16,
-			state->base.rotation);
-
-	hscale = drm_rect_calc_hscale_relaxed(src, dst, min_scale, max_scale);
-	BUG_ON(hscale < 0);
-
-	vscale = drm_rect_calc_vscale_relaxed(src, dst, min_scale, max_scale);
-	BUG_ON(vscale < 0);
-
-	if (crtc_state->base.enable)
-		drm_mode_get_hv_timing(&crtc_state->base.mode,
-				       &clip.x2, &clip.y2);
-
-	state->base.visible = drm_rect_clip_scaled(src, dst, &clip, hscale, vscale);
-
-	crtc_x = dst->x1;
-	crtc_y = dst->y1;
-	crtc_w = drm_rect_width(dst);
-	crtc_h = drm_rect_height(dst);
+	ret = drm_atomic_helper_check_plane_state(&state->base,
+						  &crtc_state->base,
+						  min_scale, max_scale,
+						  true, true);
+	if (ret)
+		return ret;
 
 	if (state->base.visible) {
-		/* check again in case clipping clamped the results */
-		hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale);
-		if (hscale < 0) {
-			DRM_DEBUG_KMS("Horizontal scaling factor out of limits\n");
-			drm_rect_debug_print("src: ", src, true);
-			drm_rect_debug_print("dst: ", dst, false);
-
-			return hscale;
-		}
-
-		vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale);
-		if (vscale < 0) {
-			DRM_DEBUG_KMS("Vertical scaling factor out of limits\n");
-			drm_rect_debug_print("src: ", src, true);
-			drm_rect_debug_print("dst: ", dst, false);
-
-			return vscale;
-		}
-
-		/* Make the source viewport size an exact multiple of the scaling factors. */
-		drm_rect_adjust_size(src,
-				     drm_rect_width(dst) * hscale - drm_rect_width(src),
-				     drm_rect_height(dst) * vscale - drm_rect_height(src));
-
-		drm_rect_rotate_inv(src, fb->width << 16, fb->height << 16,
-				    state->base.rotation);
-
-		/* sanity check to make sure the src viewport wasn't enlarged */
-		WARN_ON(src->x1 < (int) state->base.src_x ||
-			src->y1 < (int) state->base.src_y ||
-			src->x2 > (int) state->base.src_x + state->base.src_w ||
-			src->y2 > (int) state->base.src_y + state->base.src_h);
+		struct drm_rect *src = &state->base.src;
+		struct drm_rect *dst = &state->base.dst;
+		unsigned int crtc_w = drm_rect_width(dst);
+		unsigned int crtc_h = drm_rect_height(dst);
+		uint32_t src_x, src_y, src_w, src_h;
 
 		/*
 		 * Hardware doesn't handle subpixel coordinates.
@@ -1055,58 +1005,40 @@ intel_check_sprite_plane(struct intel_plane *plane,
 		src_y = src->y1 >> 16;
 		src_h = drm_rect_height(src) >> 16;
 
-		if (intel_format_is_yuv(fb->format->format)) {
-			src_x &= ~1;
-			src_w &= ~1;
-
-			/*
-			 * Must keep src and dst the
-			 * same if we can't scale.
-			 */
-			if (!can_scale)
-				crtc_w &= ~1;
+		src->x1 = src_x << 16;
+		src->x2 = (src_x + src_w) << 16;
+		src->y1 = src_y << 16;
+		src->y2 = (src_y + src_h) << 16;
 
-			if (crtc_w == 0)
-				state->base.visible = false;
+		if (intel_format_is_yuv(fb->format->format) &&
+    		    fb->format->format != DRM_FORMAT_NV12 &&
+		    (src_x % 2 || src_w % 2)) {
+			DRM_DEBUG_KMS("src x/w (%u, %u) must be a multiple of 2 for YUV planes\n",
+				      src_x, src_w);
+			return -EINVAL;
 		}
-	}
-
-	/* Check size restrictions when scaling */
-	if (state->base.visible && (src_w != crtc_w || src_h != crtc_h)) {
-		unsigned int width_bytes;
-		int cpp = fb->format->cpp[0];
 
-		WARN_ON(!can_scale);
+		/* Check size restrictions when scaling */
+		if (src_w != crtc_w || src_h != crtc_h) {
+			unsigned int width_bytes;
+			int cpp = fb->format->cpp[0];
 
-		/* FIXME interlacing min height is 6 */
+			WARN_ON(!can_scale);
 
-		if (crtc_w < 3 || crtc_h < 3)
-			state->base.visible = false;
+			width_bytes = ((src_x * cpp) & 63) + src_w * cpp;
 
-		if (src_w < 3 || src_h < 3)
-			state->base.visible = false;
-
-		width_bytes = ((src_x * cpp) & 63) + src_w * cpp;
-
-		if (INTEL_GEN(dev_priv) < 9 && (src_w > 2048 || src_h > 2048 ||
-		    width_bytes > 4096 || fb->pitches[0] > 4096)) {
-			DRM_DEBUG_KMS("Source dimensions exceed hardware limits\n");
-			return -EINVAL;
+			/* FIXME interlacing min height is 6 */
+			if (INTEL_GEN(dev_priv) < 9 && (
+			     src_w < 3 || src_h < 3 ||
+			     src_w > 2048 || src_h > 2048 ||
+			     crtc_w < 3 || crtc_h < 3 ||
+			     width_bytes > 4096 || fb->pitches[0] > 4096)) {
+				DRM_DEBUG_KMS("Source dimensions exceed hardware limits\n");
+				return -EINVAL;
+			}
 		}
 	}
 
-	if (state->base.visible) {
-		src->x1 = src_x << 16;
-		src->x2 = (src_x + src_w) << 16;
-		src->y1 = src_y << 16;
-		src->y2 = (src_y + src_h) << 16;
-	}
-
-	dst->x1 = crtc_x;
-	dst->x2 = crtc_x + crtc_w;
-	dst->y1 = crtc_y;
-	dst->y2 = crtc_y + crtc_h;
-
 	if (INTEL_GEN(dev_priv) >= 9) {
 		ret = skl_check_plane_surface(crtc_state, state);
 		if (ret)
@@ -1248,6 +1180,19 @@ static uint32_t skl_plane_formats[] = {
 	DRM_FORMAT_VYUY,
 };
 
+static uint32_t skl_planar_formats[] = {
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_NV12,
+};
+
 static const uint64_t skl_plane_format_modifiers_noccs[] = {
 	I915_FORMAT_MOD_Yf_TILED,
 	I915_FORMAT_MOD_Y_TILED,
@@ -1342,6 +1287,7 @@ static bool skl_mod_supported(uint32_t format, uint64_t modifier)
 	case DRM_FORMAT_YVYU:
 	case DRM_FORMAT_UYVY:
 	case DRM_FORMAT_VYUY:
+	case DRM_FORMAT_NV12:
 		if (modifier == I915_FORMAT_MOD_Yf_TILED)
 			return true;
 		/* fall through */
@@ -1441,8 +1387,14 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 		intel_plane->disable_plane = skl_disable_plane;
 		intel_plane->get_hw_state = skl_plane_get_hw_state;
 
-		plane_formats = skl_plane_formats;
-		num_plane_formats = ARRAY_SIZE(skl_plane_formats);
+		if (skl_plane_has_planar(dev_priv, pipe,
+					 PLANE_SPRITE0 + plane)) {
+			plane_formats = skl_planar_formats;
+			num_plane_formats = ARRAY_SIZE(skl_planar_formats);
+		} else {
+			plane_formats = skl_plane_formats;
+			num_plane_formats = ARRAY_SIZE(skl_plane_formats);
+		}
 
 		if (skl_plane_has_ccs(dev_priv, pipe, PLANE_SPRITE0 + plane))
 			modifiers = skl_plane_format_modifiers_ccs;
diff --git a/drivers/gpu/drm/i915/intel_uc.c b/drivers/gpu/drm/i915/intel_uc.c
index e5bf0d37bf43..1cffaf7b5dbe 100644
--- a/drivers/gpu/drm/i915/intel_uc.c
+++ b/drivers/gpu/drm/i915/intel_uc.c
@@ -69,13 +69,15 @@ static int __get_platform_enable_guc(struct drm_i915_private *dev_priv)
 
 static int __get_default_guc_log_level(struct drm_i915_private *dev_priv)
 {
-	int guc_log_level = 0; /* disabled */
+	int guc_log_level;
 
-	/* Enable if we're running on platform with GuC and debug config */
-	if (HAS_GUC(dev_priv) && intel_uc_is_using_guc() &&
-	    (IS_ENABLED(CONFIG_DRM_I915_DEBUG) ||
-	     IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)))
-		guc_log_level = 1 + GUC_LOG_VERBOSITY_MAX;
+	if (!HAS_GUC(dev_priv) || !intel_uc_is_using_guc())
+		guc_log_level = GUC_LOG_LEVEL_DISABLED;
+	else if (IS_ENABLED(CONFIG_DRM_I915_DEBUG) ||
+		 IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
+		guc_log_level = GUC_LOG_LEVEL_MAX;
+	else
+		guc_log_level = GUC_LOG_LEVEL_NON_VERBOSE;
 
 	/* Any platform specific fine-tuning can be done here */
 
@@ -83,7 +85,7 @@ static int __get_default_guc_log_level(struct drm_i915_private *dev_priv)
 }
 
 /**
- * intel_uc_sanitize_options - sanitize uC related modparam options
+ * sanitize_options_early - sanitize uC related modparam options
  * @dev_priv: device private
  *
  * In case of "enable_guc" option this function will attempt to modify
@@ -99,7 +101,7 @@ static int __get_default_guc_log_level(struct drm_i915_private *dev_priv)
  * unless GuC is enabled on given platform and the driver is compiled with
  * debug config when this modparam will default to "enable(1..4)".
  */
-void intel_uc_sanitize_options(struct drm_i915_private *dev_priv)
+static void sanitize_options_early(struct drm_i915_private *dev_priv)
 {
 	struct intel_uc_fw *guc_fw = &dev_priv->guc.fw;
 	struct intel_uc_fw *huc_fw = &dev_priv->huc.fw;
@@ -142,51 +144,53 @@ void intel_uc_sanitize_options(struct drm_i915_private *dev_priv)
 		i915_modparams.guc_log_level = 0;
 	}
 
-	if (i915_modparams.guc_log_level > 1 + GUC_LOG_VERBOSITY_MAX) {
+	if (i915_modparams.guc_log_level > GUC_LOG_LEVEL_MAX) {
 		DRM_WARN("Incompatible option detected: %s=%d, %s!\n",
 			 "guc_log_level", i915_modparams.guc_log_level,
 			 "verbosity too high");
-		i915_modparams.guc_log_level = 1 + GUC_LOG_VERBOSITY_MAX;
+		i915_modparams.guc_log_level = GUC_LOG_LEVEL_MAX;
 	}
 
-	DRM_DEBUG_DRIVER("guc_log_level=%d (enabled:%s verbosity:%d)\n",
+	DRM_DEBUG_DRIVER("guc_log_level=%d (enabled:%s, verbose:%s, verbosity:%d)\n",
 			 i915_modparams.guc_log_level,
 			 yesno(i915_modparams.guc_log_level),
-			 i915_modparams.guc_log_level - 1);
+			 yesno(GUC_LOG_LEVEL_IS_VERBOSE(i915_modparams.guc_log_level)),
+			 GUC_LOG_LEVEL_TO_VERBOSITY(i915_modparams.guc_log_level));
 
 	/* Make sure that sanitization was done */
 	GEM_BUG_ON(i915_modparams.enable_guc < 0);
 	GEM_BUG_ON(i915_modparams.guc_log_level < 0);
 }
 
-void intel_uc_init_early(struct drm_i915_private *dev_priv)
+void intel_uc_init_early(struct drm_i915_private *i915)
 {
-	intel_guc_init_early(&dev_priv->guc);
-	intel_huc_init_early(&dev_priv->huc);
-}
+	struct intel_guc *guc = &i915->guc;
+	struct intel_huc *huc = &i915->huc;
 
-void intel_uc_init_fw(struct drm_i915_private *dev_priv)
-{
-	if (!USES_GUC(dev_priv))
-		return;
+	intel_guc_init_early(guc);
+	intel_huc_init_early(huc);
 
-	if (USES_HUC(dev_priv))
-		intel_uc_fw_fetch(dev_priv, &dev_priv->huc.fw);
+	sanitize_options_early(i915);
 
-	intel_uc_fw_fetch(dev_priv, &dev_priv->guc.fw);
+	if (USES_GUC(i915))
+		intel_uc_fw_fetch(i915, &guc->fw);
+
+	if (USES_HUC(i915))
+		intel_uc_fw_fetch(i915, &huc->fw);
 }
 
-void intel_uc_fini_fw(struct drm_i915_private *dev_priv)
+void intel_uc_cleanup_early(struct drm_i915_private *i915)
 {
-	if (!USES_GUC(dev_priv))
-		return;
+	struct intel_guc *guc = &i915->guc;
+	struct intel_huc *huc = &i915->huc;
 
-	intel_uc_fw_fini(&dev_priv->guc.fw);
+	if (USES_HUC(i915))
+		intel_uc_fw_fini(&huc->fw);
 
-	if (USES_HUC(dev_priv))
-		intel_uc_fw_fini(&dev_priv->huc.fw);
+	if (USES_GUC(i915))
+		intel_uc_fw_fini(&guc->fw);
 
-	guc_free_load_err_log(&dev_priv->guc);
+	guc_free_load_err_log(guc);
 }
 
 /**
@@ -223,10 +227,13 @@ static int guc_enable_communication(struct intel_guc *guc)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 
+	gen9_enable_guc_interrupts(dev_priv);
+
 	if (HAS_GUC_CT(dev_priv))
-		return intel_guc_enable_ct(guc);
+		return intel_guc_ct_enable(&guc->ct);
 
 	guc->send = intel_guc_send_mmio;
+	guc->handler = intel_guc_to_host_event_handler_mmio;
 	return 0;
 }
 
@@ -235,9 +242,12 @@ static void guc_disable_communication(struct intel_guc *guc)
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 
 	if (HAS_GUC_CT(dev_priv))
-		intel_guc_disable_ct(guc);
+		intel_guc_ct_disable(&guc->ct);
+
+	gen9_disable_guc_interrupts(dev_priv);
 
 	guc->send = intel_guc_send_nop;
+	guc->handler = intel_guc_to_host_event_handler_nop;
 }
 
 int intel_uc_init_misc(struct drm_i915_private *dev_priv)
@@ -248,24 +258,13 @@ int intel_uc_init_misc(struct drm_i915_private *dev_priv)
 	if (!USES_GUC(dev_priv))
 		return 0;
 
-	ret = intel_guc_init_wq(guc);
-	if (ret) {
-		DRM_ERROR("Couldn't allocate workqueues for GuC\n");
-		goto err;
-	}
+	intel_guc_init_ggtt_pin_bias(guc);
 
-	ret = intel_guc_log_relay_create(guc);
-	if (ret) {
-		DRM_ERROR("Couldn't allocate relay for GuC log\n");
-		goto err_relay;
-	}
+	ret = intel_guc_init_wq(guc);
+	if (ret)
+		return ret;
 
 	return 0;
-
-err_relay:
-	intel_guc_fini_wq(guc);
-err:
-	return ret;
 }
 
 void intel_uc_fini_misc(struct drm_i915_private *dev_priv)
@@ -276,8 +275,6 @@ void intel_uc_fini_misc(struct drm_i915_private *dev_priv)
 		return;
 
 	intel_guc_fini_wq(guc);
-
-	intel_guc_log_relay_destroy(guc);
 }
 
 int intel_uc_init(struct drm_i915_private *dev_priv)
@@ -325,6 +322,24 @@ void intel_uc_fini(struct drm_i915_private *dev_priv)
 	intel_guc_fini(guc);
 }
 
+void intel_uc_sanitize(struct drm_i915_private *i915)
+{
+	struct intel_guc *guc = &i915->guc;
+	struct intel_huc *huc = &i915->huc;
+
+	if (!USES_GUC(i915))
+		return;
+
+	GEM_BUG_ON(!HAS_GUC(i915));
+
+	guc_disable_communication(guc);
+
+	intel_huc_sanitize(huc);
+	intel_guc_sanitize(guc);
+
+	__intel_uc_reset_hw(i915);
+}
+
 int intel_uc_init_hw(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc *guc = &dev_priv->guc;
@@ -336,14 +351,8 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv)
 
 	GEM_BUG_ON(!HAS_GUC(dev_priv));
 
-	guc_disable_communication(guc);
 	gen9_reset_guc_interrupts(dev_priv);
 
-	/* init WOPCM */
-	I915_WRITE(GUC_WOPCM_SIZE, intel_guc_wopcm_size(dev_priv));
-	I915_WRITE(DMA_GUC_WOPCM_OFFSET,
-		   GUC_WOPCM_OFFSET_VALUE | HUC_LOADING_AGENT_GUC);
-
 	/* WaEnableuKernelHeaderValidFix:skl */
 	/* WaEnableGuCBootHashCheckNotSet:skl,bxt,kbl */
 	if (IS_GEN9(dev_priv))
@@ -390,12 +399,9 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv)
 	}
 
 	if (USES_GUC_SUBMISSION(dev_priv)) {
-		if (i915_modparams.guc_log_level)
-			gen9_enable_guc_interrupts(dev_priv);
-
 		ret = intel_guc_submission_enable(guc);
 		if (ret)
-			goto err_interrupts;
+			goto err_communication;
 	}
 
 	dev_info(dev_priv->drm.dev, "GuC firmware version %u.%u\n",
@@ -410,8 +416,6 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv)
 	/*
 	 * We've failed to load the firmware :(
 	 */
-err_interrupts:
-	gen9_disable_guc_interrupts(dev_priv);
 err_communication:
 	guc_disable_communication(guc);
 err_log_capture:
@@ -441,9 +445,6 @@ void intel_uc_fini_hw(struct drm_i915_private *dev_priv)
 		intel_guc_submission_disable(guc);
 
 	guc_disable_communication(guc);
-
-	if (USES_GUC_SUBMISSION(dev_priv))
-		gen9_disable_guc_interrupts(dev_priv);
 }
 
 int intel_uc_suspend(struct drm_i915_private *i915)
@@ -479,8 +480,7 @@ int intel_uc_resume(struct drm_i915_private *i915)
 	if (guc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
 		return 0;
 
-	if (i915_modparams.guc_log_level)
-		gen9_enable_guc_interrupts(i915);
+	gen9_enable_guc_interrupts(i915);
 
 	err = intel_guc_resume(guc);
 	if (err) {
diff --git a/drivers/gpu/drm/i915/intel_uc.h b/drivers/gpu/drm/i915/intel_uc.h
index f76d51d1ce70..25d73ada74ae 100644
--- a/drivers/gpu/drm/i915/intel_uc.h
+++ b/drivers/gpu/drm/i915/intel_uc.h
@@ -28,13 +28,12 @@
 #include "intel_huc.h"
 #include "i915_params.h"
 
-void intel_uc_sanitize_options(struct drm_i915_private *dev_priv);
 void intel_uc_init_early(struct drm_i915_private *dev_priv);
+void intel_uc_cleanup_early(struct drm_i915_private *dev_priv);
 void intel_uc_init_mmio(struct drm_i915_private *dev_priv);
-void intel_uc_init_fw(struct drm_i915_private *dev_priv);
-void intel_uc_fini_fw(struct drm_i915_private *dev_priv);
 int intel_uc_init_misc(struct drm_i915_private *dev_priv);
 void intel_uc_fini_misc(struct drm_i915_private *dev_priv);
+void intel_uc_sanitize(struct drm_i915_private *dev_priv);
 int intel_uc_init_hw(struct drm_i915_private *dev_priv);
 void intel_uc_fini_hw(struct drm_i915_private *dev_priv);
 int intel_uc_init(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/intel_uc_fw.c b/drivers/gpu/drm/i915/intel_uc_fw.c
index 3ec0ce505b76..6e8e0b546743 100644
--- a/drivers/gpu/drm/i915/intel_uc_fw.c
+++ b/drivers/gpu/drm/i915/intel_uc_fw.c
@@ -95,15 +95,6 @@ void intel_uc_fw_fetch(struct drm_i915_private *dev_priv,
 	uc_fw->ucode_offset = uc_fw->header_offset + uc_fw->header_size;
 	uc_fw->ucode_size = (css->size_dw - css->header_size_dw) * sizeof(u32);
 
-	/* Header and uCode will be loaded to WOPCM */
-	size = uc_fw->header_size + uc_fw->ucode_size;
-	if (size > intel_guc_wopcm_size(dev_priv)) {
-		DRM_WARN("%s: Firmware is too large to fit in WOPCM\n",
-			 intel_uc_fw_type_repr(uc_fw->type));
-		err = -E2BIG;
-		goto fail;
-	}
-
 	/* now RSA */
 	if (css->key_size_dw != UOS_RSA_SCRATCH_COUNT) {
 		DRM_WARN("%s: Mismatched firmware RSA key size (%u)\n",
@@ -209,6 +200,7 @@ int intel_uc_fw_upload(struct intel_uc_fw *uc_fw,
 				   struct i915_vma *vma))
 {
 	struct i915_vma *vma;
+	u32 ggtt_pin_bias;
 	int err;
 
 	DRM_DEBUG_DRIVER("%s fw load %s\n",
@@ -230,8 +222,9 @@ int intel_uc_fw_upload(struct intel_uc_fw *uc_fw,
 		goto fail;
 	}
 
+	ggtt_pin_bias = to_i915(uc_fw->obj->base.dev)->guc.ggtt_pin_bias;
 	vma = i915_gem_object_ggtt_pin(uc_fw->obj, NULL, 0, 0,
-				       PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
+				       PIN_OFFSET_BIAS | ggtt_pin_bias);
 	if (IS_ERR(vma)) {
 		err = PTR_ERR(vma);
 		DRM_DEBUG_DRIVER("%s fw ggtt-pin err=%d\n",
diff --git a/drivers/gpu/drm/i915/intel_uc_fw.h b/drivers/gpu/drm/i915/intel_uc_fw.h
index d5fd4609c785..87910aa83267 100644
--- a/drivers/gpu/drm/i915/intel_uc_fw.h
+++ b/drivers/gpu/drm/i915/intel_uc_fw.h
@@ -30,7 +30,7 @@ struct drm_i915_private;
 struct i915_vma;
 
 /* Home of GuC, HuC and DMC firmwares */
-#define INTEL_UC_FIRMWARE_URL "https://01.org/linuxgraphics/downloads/firmware"
+#define INTEL_UC_FIRMWARE_URL "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/i915"
 
 enum intel_uc_fw_status {
 	INTEL_UC_FIRMWARE_FAIL = -1,
@@ -115,6 +115,28 @@ static inline bool intel_uc_fw_is_selected(struct intel_uc_fw *uc_fw)
 	return uc_fw->path != NULL;
 }
 
+static inline void intel_uc_fw_sanitize(struct intel_uc_fw *uc_fw)
+{
+	if (uc_fw->load_status == INTEL_UC_FIRMWARE_SUCCESS)
+		uc_fw->load_status = INTEL_UC_FIRMWARE_PENDING;
+}
+
+/**
+ * intel_uc_fw_get_upload_size() - Get size of firmware needed to be uploaded.
+ * @uc_fw: uC firmware.
+ *
+ * Get the size of the firmware and header that will be uploaded to WOPCM.
+ *
+ * Return: Upload firmware size, or zero on firmware fetch failure.
+ */
+static inline u32 intel_uc_fw_get_upload_size(struct intel_uc_fw *uc_fw)
+{
+	if (uc_fw->fetch_status != INTEL_UC_FIRMWARE_SUCCESS)
+		return 0;
+
+	return uc_fw->header_size + uc_fw->ucode_size;
+}
+
 void intel_uc_fw_fetch(struct drm_i915_private *dev_priv,
 		       struct intel_uc_fw *uc_fw);
 int intel_uc_fw_upload(struct intel_uc_fw *uc_fw,
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 4df7c2ef8576..448293eb638d 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -62,6 +62,11 @@ static inline void
 fw_domain_reset(struct drm_i915_private *i915,
 		const struct intel_uncore_forcewake_domain *d)
 {
+	/*
+	 * We don't really know if the powerwell for the forcewake domain we are
+	 * trying to reset here does exist at this point (engines could be fused
+	 * off in ICL+), so no waiting for acks
+	 */
 	__raw_i915_write32(i915, d->reg_set, i915->uncore.fw_reset);
 }
 
@@ -134,7 +139,9 @@ fw_domain_wait_ack_with_fallback(const struct drm_i915_private *i915,
 	 * in the hope that the original ack will be delivered along with
 	 * the fallback ack.
 	 *
-	 * This workaround is described in HSDES #1604254524
+	 * This workaround is described in HSDES #1604254524 and it's known as:
+	 * WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl
+	 * although the name is a bit misleading.
 	 */
 
 	pass = 1;
@@ -1353,6 +1360,23 @@ static void fw_domain_init(struct drm_i915_private *dev_priv,
 	fw_domain_reset(dev_priv, d);
 }
 
+static void fw_domain_fini(struct drm_i915_private *dev_priv,
+			   enum forcewake_domain_id domain_id)
+{
+	struct intel_uncore_forcewake_domain *d;
+
+	if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT))
+		return;
+
+	d = &dev_priv->uncore.fw_domain[domain_id];
+
+	WARN_ON(d->wake_count);
+	WARN_ON(hrtimer_cancel(&d->timer));
+	memset(d, 0, sizeof(*d));
+
+	dev_priv->uncore.fw_domains &= ~BIT(domain_id);
+}
+
 static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 {
 	if (INTEL_GEN(dev_priv) <= 5 || intel_vgpu_active(dev_priv))
@@ -1372,7 +1396,8 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 	if (INTEL_GEN(dev_priv) >= 11) {
 		int i;
 
-		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
+		dev_priv->uncore.funcs.force_wake_get =
+			fw_domains_get_with_fallback;
 		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
 		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
 			       FORCEWAKE_RENDER_GEN9,
@@ -1565,6 +1590,40 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
 		&dev_priv->uncore.pmic_bus_access_nb);
 }
 
+/*
+ * We might have detected that some engines are fused off after we initialized
+ * the forcewake domains. Prune them, to make sure they only reference existing
+ * engines.
+ */
+void intel_uncore_prune(struct drm_i915_private *dev_priv)
+{
+	if (INTEL_GEN(dev_priv) >= 11) {
+		enum forcewake_domains fw_domains = dev_priv->uncore.fw_domains;
+		enum forcewake_domain_id domain_id;
+		int i;
+
+		for (i = 0; i < I915_MAX_VCS; i++) {
+			domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i;
+
+			if (HAS_ENGINE(dev_priv, _VCS(i)))
+				continue;
+
+			if (fw_domains & BIT(domain_id))
+				fw_domain_fini(dev_priv, domain_id);
+		}
+
+		for (i = 0; i < I915_MAX_VECS; i++) {
+			domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i;
+
+			if (HAS_ENGINE(dev_priv, _VECS(i)))
+				continue;
+
+			if (fw_domains & BIT(domain_id))
+				fw_domain_fini(dev_priv, domain_id);
+		}
+	}
+}
+
 void intel_uncore_fini(struct drm_i915_private *dev_priv)
 {
 	/* Paranoia: make sure we have disabled everything before we exit. */
@@ -1646,11 +1705,10 @@ static void gen3_stop_engine(struct intel_engine_cs *engine)
 	const i915_reg_t mode = RING_MI_MODE(base);
 
 	I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING));
-	if (intel_wait_for_register_fw(dev_priv,
-				       mode,
-				       MODE_IDLE,
-				       MODE_IDLE,
-				       500))
+	if (__intel_wait_for_register_fw(dev_priv,
+					 mode, MODE_IDLE, MODE_IDLE,
+					 500, 0,
+					 NULL))
 		DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n",
 				 engine->name);
 
@@ -1804,9 +1862,10 @@ static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
 	__raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask);
 
 	/* Wait for the device to ack the reset requests */
-	err = intel_wait_for_register_fw(dev_priv,
-					  GEN6_GDRST, hw_domain_mask, 0,
-					  500);
+	err = __intel_wait_for_register_fw(dev_priv,
+					   GEN6_GDRST, hw_domain_mask, 0,
+					   500, 0,
+					   NULL);
 	if (err)
 		DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n",
 				 hw_domain_mask);
@@ -1853,6 +1912,50 @@ static int gen6_reset_engines(struct drm_i915_private *dev_priv,
 	return gen6_hw_domain_reset(dev_priv, hw_mask);
 }
 
+/**
+ * gen11_reset_engines - reset individual engines
+ * @dev_priv: i915 device
+ * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset
+ *
+ * This function will reset the individual engines that are set in engine_mask.
+ * If you provide ALL_ENGINES as mask, full global domain reset will be issued.
+ *
+ * Note: It is responsibility of the caller to handle the difference between
+ * asking full domain reset versus reset for all available individual engines.
+ *
+ * Returns 0 on success, nonzero on error.
+ */
+static int gen11_reset_engines(struct drm_i915_private *dev_priv,
+			       unsigned engine_mask)
+{
+	struct intel_engine_cs *engine;
+	const u32 hw_engine_mask[I915_NUM_ENGINES] = {
+		[RCS] = GEN11_GRDOM_RENDER,
+		[BCS] = GEN11_GRDOM_BLT,
+		[VCS] = GEN11_GRDOM_MEDIA,
+		[VCS2] = GEN11_GRDOM_MEDIA2,
+		[VCS3] = GEN11_GRDOM_MEDIA3,
+		[VCS4] = GEN11_GRDOM_MEDIA4,
+		[VECS] = GEN11_GRDOM_VECS,
+		[VECS2] = GEN11_GRDOM_VECS2,
+	};
+	u32 hw_mask;
+
+	BUILD_BUG_ON(VECS2 + 1 != I915_NUM_ENGINES);
+
+	if (engine_mask == ALL_ENGINES) {
+		hw_mask = GEN11_GRDOM_FULL;
+	} else {
+		unsigned int tmp;
+
+		hw_mask = 0;
+		for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
+			hw_mask |= hw_engine_mask[engine->id];
+	}
+
+	return gen6_hw_domain_reset(dev_priv, hw_mask);
+}
+
 /**
  * __intel_wait_for_register_fw - wait until register matches expected state
  * @dev_priv: the i915 device
@@ -1940,7 +2043,7 @@ int __intel_wait_for_register(struct drm_i915_private *dev_priv,
 	u32 reg_value;
 	int ret;
 
-	might_sleep();
+	might_sleep_if(slow_timeout_ms);
 
 	spin_lock_irq(&dev_priv->uncore.lock);
 	intel_uncore_forcewake_get__locked(dev_priv, fw);
@@ -1952,7 +2055,7 @@ int __intel_wait_for_register(struct drm_i915_private *dev_priv,
 	intel_uncore_forcewake_put__locked(dev_priv, fw);
 	spin_unlock_irq(&dev_priv->uncore.lock);
 
-	if (ret)
+	if (ret && slow_timeout_ms)
 		ret = __wait_for(reg_value = I915_READ_NOTRACE(reg),
 				 (reg_value & mask) == value,
 				 slow_timeout_ms * 1000, 10, 1000);
@@ -1971,11 +2074,12 @@ static int gen8_reset_engine_start(struct intel_engine_cs *engine)
 	I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
 		      _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET));
 
-	ret = intel_wait_for_register_fw(dev_priv,
-					 RING_RESET_CTL(engine->mmio_base),
-					 RESET_CTL_READY_TO_RESET,
-					 RESET_CTL_READY_TO_RESET,
-					 700);
+	ret = __intel_wait_for_register_fw(dev_priv,
+					   RING_RESET_CTL(engine->mmio_base),
+					   RESET_CTL_READY_TO_RESET,
+					   RESET_CTL_READY_TO_RESET,
+					   700, 0,
+					   NULL);
 	if (ret)
 		DRM_ERROR("%s: reset request timeout\n", engine->name);
 
@@ -2000,7 +2104,10 @@ static int gen8_reset_engines(struct drm_i915_private *dev_priv,
 		if (gen8_reset_engine_start(engine))
 			goto not_ready;
 
-	return gen6_reset_engines(dev_priv, engine_mask);
+	if (INTEL_GEN(dev_priv) >= 11)
+		return gen11_reset_engines(dev_priv, engine_mask);
+	else
+		return gen6_reset_engines(dev_priv, engine_mask);
 
 not_ready:
 	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
@@ -2038,15 +2145,31 @@ int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
 	int retry;
 	int ret;
 
-	might_sleep();
+	/*
+	 * We want to perform per-engine reset from atomic context (e.g.
+	 * softirq), which imposes the constraint that we cannot sleep.
+	 * However, experience suggests that spending a bit of time waiting
+	 * for a reset helps in various cases, so for a full-device reset
+	 * we apply the opposite rule and wait if we want to. As we should
+	 * always follow up a failed per-engine reset with a full device reset,
+	 * being a little faster, stricter and more error prone for the
+	 * atomic case seems an acceptable compromise.
+	 *
+	 * Unfortunately this leads to a bimodal routine, when the goal was
+	 * to have a single reset function that worked for resetting any
+	 * number of engines simultaneously.
+	 */
+	might_sleep_if(engine_mask == ALL_ENGINES);
 
-	/* If the power well sleeps during the reset, the reset
+	/*
+	 * If the power well sleeps during the reset, the reset
 	 * request may be dropped and never completes (causing -EIO).
 	 */
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 	for (retry = 0; retry < 3; retry++) {
 
-		/* We stop engines, otherwise we might get failed reset and a
+		/*
+		 * We stop engines, otherwise we might get failed reset and a
 		 * dead gpu (on elk). Also as modern gpu as kbl can suffer
 		 * from system hang if batchbuffer is progressing when
 		 * the reset is issued, regardless of READY_TO_RESET ack.
@@ -2060,9 +2183,11 @@ int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
 		i915_stop_engines(dev_priv, engine_mask);
 
 		ret = -ENODEV;
-		if (reset)
+		if (reset) {
+			GEM_TRACE("engine_mask=%x\n", engine_mask);
 			ret = reset(dev_priv, engine_mask);
-		if (ret != -ETIMEDOUT)
+		}
+		if (ret != -ETIMEDOUT || engine_mask != ALL_ENGINES)
 			break;
 
 		cond_resched();
@@ -2085,12 +2210,14 @@ bool intel_has_reset_engine(struct drm_i915_private *dev_priv)
 
 int intel_reset_guc(struct drm_i915_private *dev_priv)
 {
+	u32 guc_domain = INTEL_GEN(dev_priv) >= 11 ? GEN11_GRDOM_GUC :
+						     GEN9_GRDOM_GUC;
 	int ret;
 
 	GEM_BUG_ON(!HAS_GUC(dev_priv));
 
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
-	ret = gen6_hw_domain_reset(dev_priv, GEN9_GRDOM_GUC);
+	ret = gen6_hw_domain_reset(dev_priv, guc_domain);
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 
 	return ret;
diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h
index dfdf444e4bcc..47478d609630 100644
--- a/drivers/gpu/drm/i915/intel_uncore.h
+++ b/drivers/gpu/drm/i915/intel_uncore.h
@@ -140,6 +140,7 @@ struct intel_uncore {
 
 void intel_uncore_sanitize(struct drm_i915_private *dev_priv);
 void intel_uncore_init(struct drm_i915_private *dev_priv);
+void intel_uncore_prune(struct drm_i915_private *dev_priv);
 bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv);
 bool intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv);
 void intel_uncore_fini(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/intel_wopcm.c b/drivers/gpu/drm/i915/intel_wopcm.c
new file mode 100644
index 000000000000..74bf76f3fddc
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_wopcm.c
@@ -0,0 +1,275 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2017-2018 Intel Corporation
+ */
+
+#include "intel_wopcm.h"
+#include "i915_drv.h"
+
+/**
+ * DOC: WOPCM Layout
+ *
+ * The layout of the WOPCM will be fixed after writing to GuC WOPCM size and
+ * offset registers whose values are calculated and determined by HuC/GuC
+ * firmware size and set of hardware requirements/restrictions as shown below:
+ *
+ * ::
+ *
+ *    +=========> +====================+ <== WOPCM Top
+ *    ^           |  HW contexts RSVD  |
+ *    |     +===> +====================+ <== GuC WOPCM Top
+ *    |     ^     |                    |
+ *    |     |     |                    |
+ *    |     |     |                    |
+ *    |    GuC    |                    |
+ *    |   WOPCM   |                    |
+ *    |    Size   +--------------------+
+ *  WOPCM   |     |    GuC FW RSVD     |
+ *    |     |     +--------------------+
+ *    |     |     |   GuC Stack RSVD   |
+ *    |     |     +------------------- +
+ *    |     v     |   GuC WOPCM RSVD   |
+ *    |     +===> +====================+ <== GuC WOPCM base
+ *    |           |     WOPCM RSVD     |
+ *    |           +------------------- + <== HuC Firmware Top
+ *    v           |      HuC FW        |
+ *    +=========> +====================+ <== WOPCM Base
+ *
+ * GuC accessible WOPCM starts at GuC WOPCM base and ends at GuC WOPCM top.
+ * The top part of the WOPCM is reserved for hardware contexts (e.g. RC6
+ * context).
+ */
+
+/* Default WOPCM size 1MB. */
+#define GEN9_WOPCM_SIZE			(1024 * 1024)
+/* 16KB WOPCM (RSVD WOPCM) is reserved from HuC firmware top. */
+#define WOPCM_RESERVED_SIZE		(16 * 1024)
+
+/* 16KB reserved at the beginning of GuC WOPCM. */
+#define GUC_WOPCM_RESERVED		(16 * 1024)
+/* 8KB from GUC_WOPCM_RESERVED is reserved for GuC stack. */
+#define GUC_WOPCM_STACK_RESERVED	(8 * 1024)
+
+/* GuC WOPCM Offset value needs to be aligned to 16KB. */
+#define GUC_WOPCM_OFFSET_ALIGNMENT	(1UL << GUC_WOPCM_OFFSET_SHIFT)
+
+/* 24KB at the end of WOPCM is reserved for RC6 CTX on BXT. */
+#define BXT_WOPCM_RC6_CTX_RESERVED	(24 * 1024)
+/* 36KB WOPCM reserved at the end of WOPCM on CNL. */
+#define CNL_WOPCM_HW_CTX_RESERVED	(36 * 1024)
+
+/* 128KB from GUC_WOPCM_RESERVED is reserved for FW on Gen9. */
+#define GEN9_GUC_FW_RESERVED	(128 * 1024)
+#define GEN9_GUC_WOPCM_OFFSET	(GUC_WOPCM_RESERVED + GEN9_GUC_FW_RESERVED)
+
+/**
+ * intel_wopcm_init_early() - Early initialization of the WOPCM.
+ * @wopcm: pointer to intel_wopcm.
+ *
+ * Setup the size of WOPCM which will be used by later on WOPCM partitioning.
+ */
+void intel_wopcm_init_early(struct intel_wopcm *wopcm)
+{
+	wopcm->size = GEN9_WOPCM_SIZE;
+
+	DRM_DEBUG_DRIVER("WOPCM size: %uKiB\n", wopcm->size / 1024);
+}
+
+static inline u32 context_reserved_size(struct drm_i915_private *i915)
+{
+	if (IS_GEN9_LP(i915))
+		return BXT_WOPCM_RC6_CTX_RESERVED;
+	else if (INTEL_GEN(i915) >= 10)
+		return CNL_WOPCM_HW_CTX_RESERVED;
+	else
+		return 0;
+}
+
+static inline int gen9_check_dword_gap(u32 guc_wopcm_base, u32 guc_wopcm_size)
+{
+	u32 offset;
+
+	/*
+	 * GuC WOPCM size shall be at least a dword larger than the offset from
+	 * WOPCM base (GuC WOPCM offset from WOPCM base + GEN9_GUC_WOPCM_OFFSET)
+	 * due to hardware limitation on Gen9.
+	 */
+	offset = guc_wopcm_base + GEN9_GUC_WOPCM_OFFSET;
+	if (offset > guc_wopcm_size ||
+	    (guc_wopcm_size - offset) < sizeof(u32)) {
+		DRM_ERROR("GuC WOPCM size %uKiB is too small. %uKiB needed.\n",
+			  guc_wopcm_size / 1024,
+			  (u32)(offset + sizeof(u32)) / 1024);
+		return -E2BIG;
+	}
+
+	return 0;
+}
+
+static inline int gen9_check_huc_fw_fits(u32 guc_wopcm_size, u32 huc_fw_size)
+{
+	/*
+	 * On Gen9 & CNL A0, hardware requires the total available GuC WOPCM
+	 * size to be larger than or equal to HuC firmware size. Otherwise,
+	 * firmware uploading would fail.
+	 */
+	if (huc_fw_size > guc_wopcm_size - GUC_WOPCM_RESERVED) {
+		DRM_ERROR("HuC FW (%uKiB) won't fit in GuC WOPCM (%uKiB).\n",
+			  huc_fw_size / 1024,
+			  (guc_wopcm_size - GUC_WOPCM_RESERVED) / 1024);
+		return -E2BIG;
+	}
+
+	return 0;
+}
+
+static inline int check_hw_restriction(struct drm_i915_private *i915,
+				       u32 guc_wopcm_base, u32 guc_wopcm_size,
+				       u32 huc_fw_size)
+{
+	int err = 0;
+
+	if (IS_GEN9(i915))
+		err = gen9_check_dword_gap(guc_wopcm_base, guc_wopcm_size);
+
+	if (!err &&
+	    (IS_GEN9(i915) || IS_CNL_REVID(i915, CNL_REVID_A0, CNL_REVID_A0)))
+		err = gen9_check_huc_fw_fits(guc_wopcm_size, huc_fw_size);
+
+	return err;
+}
+
+/**
+ * intel_wopcm_init() - Initialize the WOPCM structure.
+ * @wopcm: pointer to intel_wopcm.
+ *
+ * This function will partition WOPCM space based on GuC and HuC firmware sizes
+ * and will allocate max remaining for use by GuC. This function will also
+ * enforce platform dependent hardware restrictions on GuC WOPCM offset and
+ * size. It will fail the WOPCM init if any of these checks were failed, so that
+ * the following GuC firmware uploading would be aborted.
+ *
+ * Return: 0 on success, non-zero error code on failure.
+ */
+int intel_wopcm_init(struct intel_wopcm *wopcm)
+{
+	struct drm_i915_private *i915 = wopcm_to_i915(wopcm);
+	u32 guc_fw_size = intel_uc_fw_get_upload_size(&i915->guc.fw);
+	u32 huc_fw_size = intel_uc_fw_get_upload_size(&i915->huc.fw);
+	u32 ctx_rsvd = context_reserved_size(i915);
+	u32 guc_wopcm_base;
+	u32 guc_wopcm_size;
+	u32 guc_wopcm_rsvd;
+	int err;
+
+	GEM_BUG_ON(!wopcm->size);
+
+	if (guc_fw_size >= wopcm->size) {
+		DRM_ERROR("GuC FW (%uKiB) is too big to fit in WOPCM.",
+			  guc_fw_size / 1024);
+		return -E2BIG;
+	}
+
+	if (huc_fw_size >= wopcm->size) {
+		DRM_ERROR("HuC FW (%uKiB) is too big to fit in WOPCM.",
+			  huc_fw_size / 1024);
+		return -E2BIG;
+	}
+
+	guc_wopcm_base = ALIGN(huc_fw_size + WOPCM_RESERVED_SIZE,
+			       GUC_WOPCM_OFFSET_ALIGNMENT);
+	if ((guc_wopcm_base + ctx_rsvd) >= wopcm->size) {
+		DRM_ERROR("GuC WOPCM base (%uKiB) is too big.\n",
+			  guc_wopcm_base / 1024);
+		return -E2BIG;
+	}
+
+	guc_wopcm_size = wopcm->size - guc_wopcm_base - ctx_rsvd;
+	guc_wopcm_size &= GUC_WOPCM_SIZE_MASK;
+
+	DRM_DEBUG_DRIVER("Calculated GuC WOPCM Region: [%uKiB, %uKiB)\n",
+			 guc_wopcm_base / 1024, guc_wopcm_size / 1024);
+
+	guc_wopcm_rsvd = GUC_WOPCM_RESERVED + GUC_WOPCM_STACK_RESERVED;
+	if ((guc_fw_size + guc_wopcm_rsvd) > guc_wopcm_size) {
+		DRM_ERROR("Need %uKiB WOPCM for GuC, %uKiB available.\n",
+			  (guc_fw_size + guc_wopcm_rsvd) / 1024,
+			  guc_wopcm_size / 1024);
+		return -E2BIG;
+	}
+
+	err = check_hw_restriction(i915, guc_wopcm_base, guc_wopcm_size,
+				   huc_fw_size);
+	if (err)
+		return err;
+
+	wopcm->guc.base = guc_wopcm_base;
+	wopcm->guc.size = guc_wopcm_size;
+
+	return 0;
+}
+
+static inline int write_and_verify(struct drm_i915_private *dev_priv,
+				   i915_reg_t reg, u32 val, u32 mask,
+				   u32 locked_bit)
+{
+	u32 reg_val;
+
+	GEM_BUG_ON(val & ~mask);
+
+	I915_WRITE(reg, val);
+
+	reg_val = I915_READ(reg);
+
+	return (reg_val & mask) != (val | locked_bit) ? -EIO : 0;
+}
+
+/**
+ * intel_wopcm_init_hw() - Setup GuC WOPCM registers.
+ * @wopcm: pointer to intel_wopcm.
+ *
+ * Setup the GuC WOPCM size and offset registers with the calculated values. It
+ * will verify the register values to make sure the registers are locked with
+ * correct values.
+ *
+ * Return: 0 on success. -EIO if registers were locked with incorrect values.
+ */
+int intel_wopcm_init_hw(struct intel_wopcm *wopcm)
+{
+	struct drm_i915_private *dev_priv = wopcm_to_i915(wopcm);
+	u32 huc_agent;
+	u32 mask;
+	int err;
+
+	if (!USES_GUC(dev_priv))
+		return 0;
+
+	GEM_BUG_ON(!HAS_GUC(dev_priv));
+	GEM_BUG_ON(!wopcm->guc.size);
+	GEM_BUG_ON(!wopcm->guc.base);
+
+	err = write_and_verify(dev_priv, GUC_WOPCM_SIZE, wopcm->guc.size,
+			       GUC_WOPCM_SIZE_MASK | GUC_WOPCM_SIZE_LOCKED,
+			       GUC_WOPCM_SIZE_LOCKED);
+	if (err)
+		goto err_out;
+
+	huc_agent = USES_HUC(dev_priv) ? HUC_LOADING_AGENT_GUC : 0;
+	mask = GUC_WOPCM_OFFSET_MASK | GUC_WOPCM_OFFSET_VALID | huc_agent;
+	err = write_and_verify(dev_priv, DMA_GUC_WOPCM_OFFSET,
+			       wopcm->guc.base | huc_agent, mask,
+			       GUC_WOPCM_OFFSET_VALID);
+	if (err)
+		goto err_out;
+
+	return 0;
+
+err_out:
+	DRM_ERROR("Failed to init WOPCM registers:\n");
+	DRM_ERROR("DMA_GUC_WOPCM_OFFSET=%#x\n",
+		  I915_READ(DMA_GUC_WOPCM_OFFSET));
+	DRM_ERROR("GUC_WOPCM_SIZE=%#x\n", I915_READ(GUC_WOPCM_SIZE));
+
+	return err;
+}
diff --git a/drivers/gpu/drm/i915/intel_wopcm.h b/drivers/gpu/drm/i915/intel_wopcm.h
new file mode 100644
index 000000000000..6298910a384c
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_wopcm.h
@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2017-2018 Intel Corporation
+ */
+
+#ifndef _INTEL_WOPCM_H_
+#define _INTEL_WOPCM_H_
+
+#include <linux/types.h>
+
+/**
+ * struct intel_wopcm - Overall WOPCM info and WOPCM regions.
+ * @size: Size of overall WOPCM.
+ * @guc: GuC WOPCM Region info.
+ * @guc.base: GuC WOPCM base which is offset from WOPCM base.
+ * @guc.size: Size of the GuC WOPCM region.
+ */
+struct intel_wopcm {
+	u32 size;
+	struct {
+		u32 base;
+		u32 size;
+	} guc;
+};
+
+void intel_wopcm_init_early(struct intel_wopcm *wopcm);
+int intel_wopcm_init(struct intel_wopcm *wopcm);
+int intel_wopcm_init_hw(struct intel_wopcm *wopcm);
+
+#endif
diff --git a/drivers/gpu/drm/i915/intel_workarounds.c b/drivers/gpu/drm/i915/intel_workarounds.c
new file mode 100644
index 000000000000..2df3538ceba5
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_workarounds.c
@@ -0,0 +1,949 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2014-2018 Intel Corporation
+ */
+
+#include "i915_drv.h"
+#include "intel_workarounds.h"
+
+/**
+ * DOC: Hardware workarounds
+ *
+ * This file is intended as a central place to implement most [1]_ of the
+ * required workarounds for hardware to work as originally intended. They fall
+ * in five basic categories depending on how/when they are applied:
+ *
+ * - Workarounds that touch registers that are saved/restored to/from the HW
+ *   context image. The list is emitted (via Load Register Immediate commands)
+ *   everytime a new context is created.
+ * - GT workarounds. The list of these WAs is applied whenever these registers
+ *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
+ * - Display workarounds. The list is applied during display clock-gating
+ *   initialization.
+ * - Workarounds that whitelist a privileged register, so that UMDs can manage
+ *   them directly. This is just a special case of a MMMIO workaround (as we
+ *   write the list of these to/be-whitelisted registers to some special HW
+ *   registers).
+ * - Workaround batchbuffers, that get executed automatically by the hardware
+ *   on every HW context restore.
+ *
+ * .. [1] Please notice that there are other WAs that, due to their nature,
+ *    cannot be applied from a central place. Those are peppered around the rest
+ *    of the code, as needed.
+ *
+ * .. [2] Technically, some registers are powercontext saved & restored, so they
+ *    survive a suspend/resume. In practice, writing them again is not too
+ *    costly and simplifies things. We can revisit this in the future.
+ *
+ * Layout
+ * ''''''
+ *
+ * Keep things in this file ordered by WA type, as per the above (context, GT,
+ * display, register whitelist, batchbuffer). Then, inside each type, keep the
+ * following order:
+ *
+ * - Infrastructure functions and macros
+ * - WAs per platform in standard gen/chrono order
+ * - Public functions to init or apply the given workaround type.
+ */
+
+static int wa_add(struct drm_i915_private *dev_priv,
+		  i915_reg_t addr,
+		  const u32 mask, const u32 val)
+{
+	const unsigned int idx = dev_priv->workarounds.count;
+
+	if (WARN_ON(idx >= I915_MAX_WA_REGS))
+		return -ENOSPC;
+
+	dev_priv->workarounds.reg[idx].addr = addr;
+	dev_priv->workarounds.reg[idx].value = val;
+	dev_priv->workarounds.reg[idx].mask = mask;
+
+	dev_priv->workarounds.count++;
+
+	return 0;
+}
+
+#define WA_REG(addr, mask, val) do { \
+		const int r = wa_add(dev_priv, (addr), (mask), (val)); \
+		if (r) \
+			return r; \
+	} while (0)
+
+#define WA_SET_BIT_MASKED(addr, mask) \
+	WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
+
+#define WA_CLR_BIT_MASKED(addr, mask) \
+	WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
+
+#define WA_SET_FIELD_MASKED(addr, mask, value) \
+	WA_REG(addr, (mask), _MASKED_FIELD(mask, value))
+
+static int gen8_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
+
+	/* WaDisableAsyncFlipPerfMode:bdw,chv */
+	WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
+
+	/* WaDisablePartialInstShootdown:bdw,chv */
+	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
+			  PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
+
+	/* Use Force Non-Coherent whenever executing a 3D context. This is a
+	 * workaround for for a possible hang in the unlikely event a TLB
+	 * invalidation occurs during a PSD flush.
+	 */
+	/* WaForceEnableNonCoherent:bdw,chv */
+	/* WaHdcDisableFetchWhenMasked:bdw,chv */
+	WA_SET_BIT_MASKED(HDC_CHICKEN0,
+			  HDC_DONOT_FETCH_MEM_WHEN_MASKED |
+			  HDC_FORCE_NON_COHERENT);
+
+	/* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
+	 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
+	 *  polygons in the same 8x4 pixel/sample area to be processed without
+	 *  stalling waiting for the earlier ones to write to Hierarchical Z
+	 *  buffer."
+	 *
+	 * This optimization is off by default for BDW and CHV; turn it on.
+	 */
+	WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
+
+	/* Wa4x4STCOptimizationDisable:bdw,chv */
+	WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
+
+	/*
+	 * BSpec recommends 8x4 when MSAA is used,
+	 * however in practice 16x4 seems fastest.
+	 *
+	 * Note that PS/WM thread counts depend on the WIZ hashing
+	 * disable bit, which we don't touch here, but it's good
+	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
+	 */
+	WA_SET_FIELD_MASKED(GEN7_GT_MODE,
+			    GEN6_WIZ_HASHING_MASK,
+			    GEN6_WIZ_HASHING_16x4);
+
+	return 0;
+}
+
+static int bdw_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	int ret;
+
+	ret = gen8_ctx_workarounds_init(dev_priv);
+	if (ret)
+		return ret;
+
+	/* WaDisableThreadStallDopClockGating:bdw (pre-production) */
+	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
+
+	/* WaDisableDopClockGating:bdw
+	 *
+	 * Also see the related UCGTCL1 write in broadwell_init_clock_gating()
+	 * to disable EUTC clock gating.
+	 */
+	WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
+			  DOP_CLOCK_GATING_DISABLE);
+
+	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
+			  GEN8_SAMPLER_POWER_BYPASS_DIS);
+
+	WA_SET_BIT_MASKED(HDC_CHICKEN0,
+			  /* WaForceContextSaveRestoreNonCoherent:bdw */
+			  HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
+			  /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
+			  (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
+
+	return 0;
+}
+
+static int chv_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	int ret;
+
+	ret = gen8_ctx_workarounds_init(dev_priv);
+	if (ret)
+		return ret;
+
+	/* WaDisableThreadStallDopClockGating:chv */
+	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
+
+	/* Improve HiZ throughput on CHV. */
+	WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
+
+	return 0;
+}
+
+static int gen9_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	if (HAS_LLC(dev_priv)) {
+		/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
+		 *
+		 * Must match Display Engine. See
+		 * WaCompressedResourceDisplayNewHashMode.
+		 */
+		WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
+				  GEN9_PBE_COMPRESSED_HASH_SELECTION);
+		WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
+				  GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
+	}
+
+	/* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
+	/* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
+	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
+			  FLOW_CONTROL_ENABLE |
+			  PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
+
+	/* Syncing dependencies between camera and graphics:skl,bxt,kbl */
+	if (!IS_COFFEELAKE(dev_priv))
+		WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
+				  GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
+
+	/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
+	/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
+	WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
+			  GEN9_ENABLE_YV12_BUGFIX |
+			  GEN9_ENABLE_GPGPU_PREEMPTION);
+
+	/* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
+	/* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
+	WA_SET_BIT_MASKED(CACHE_MODE_1,
+			  GEN8_4x4_STC_OPTIMIZATION_DISABLE |
+			  GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
+
+	/* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
+	WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
+			  GEN9_CCS_TLB_PREFETCH_ENABLE);
+
+	/* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
+	WA_SET_BIT_MASKED(HDC_CHICKEN0,
+			  HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
+			  HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
+
+	/* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
+	 * both tied to WaForceContextSaveRestoreNonCoherent
+	 * in some hsds for skl. We keep the tie for all gen9. The
+	 * documentation is a bit hazy and so we want to get common behaviour,
+	 * even though there is no clear evidence we would need both on kbl/bxt.
+	 * This area has been source of system hangs so we play it safe
+	 * and mimic the skl regardless of what bspec says.
+	 *
+	 * Use Force Non-Coherent whenever executing a 3D context. This
+	 * is a workaround for a possible hang in the unlikely event
+	 * a TLB invalidation occurs during a PSD flush.
+	 */
+
+	/* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
+	WA_SET_BIT_MASKED(HDC_CHICKEN0,
+			  HDC_FORCE_NON_COHERENT);
+
+	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
+	if (IS_SKYLAKE(dev_priv) ||
+	    IS_KABYLAKE(dev_priv) ||
+	    IS_COFFEELAKE(dev_priv))
+		WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
+				  GEN8_SAMPLER_POWER_BYPASS_DIS);
+
+	/* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
+	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
+
+	/*
+	 * Supporting preemption with fine-granularity requires changes in the
+	 * batch buffer programming. Since we can't break old userspace, we
+	 * need to set our default preemption level to safe value. Userspace is
+	 * still able to use more fine-grained preemption levels, since in
+	 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
+	 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
+	 * not real HW workarounds, but merely a way to start using preemption
+	 * while maintaining old contract with userspace.
+	 */
+
+	/* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
+	WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
+
+	/* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
+	WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
+			    GEN9_PREEMPT_GPGPU_LEVEL_MASK,
+			    GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
+
+	/* WaClearHIZ_WM_CHICKEN3:bxt,glk */
+	if (IS_GEN9_LP(dev_priv))
+		WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
+
+	return 0;
+}
+
+static int skl_tune_iz_hashing(struct drm_i915_private *dev_priv)
+{
+	u8 vals[3] = { 0, 0, 0 };
+	unsigned int i;
+
+	for (i = 0; i < 3; i++) {
+		u8 ss;
+
+		/*
+		 * Only consider slices where one, and only one, subslice has 7
+		 * EUs
+		 */
+		if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]))
+			continue;
+
+		/*
+		 * subslice_7eu[i] != 0 (because of the check above) and
+		 * ss_max == 4 (maximum number of subslices possible per slice)
+		 *
+		 * ->    0 <= ss <= 3;
+		 */
+		ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1;
+		vals[i] = 3 - ss;
+	}
+
+	if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
+		return 0;
+
+	/* Tune IZ hashing. See intel_device_info_runtime_init() */
+	WA_SET_FIELD_MASKED(GEN7_GT_MODE,
+			    GEN9_IZ_HASHING_MASK(2) |
+			    GEN9_IZ_HASHING_MASK(1) |
+			    GEN9_IZ_HASHING_MASK(0),
+			    GEN9_IZ_HASHING(2, vals[2]) |
+			    GEN9_IZ_HASHING(1, vals[1]) |
+			    GEN9_IZ_HASHING(0, vals[0]));
+
+	return 0;
+}
+
+static int skl_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	int ret;
+
+	ret = gen9_ctx_workarounds_init(dev_priv);
+	if (ret)
+		return ret;
+
+	return skl_tune_iz_hashing(dev_priv);
+}
+
+static int bxt_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	int ret;
+
+	ret = gen9_ctx_workarounds_init(dev_priv);
+	if (ret)
+		return ret;
+
+	/* WaDisableThreadStallDopClockGating:bxt */
+	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
+			  STALL_DOP_GATING_DISABLE);
+
+	/* WaToEnableHwFixForPushConstHWBug:bxt */
+	WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
+			  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
+
+	return 0;
+}
+
+static int kbl_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	int ret;
+
+	ret = gen9_ctx_workarounds_init(dev_priv);
+	if (ret)
+		return ret;
+
+	/* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
+	if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
+		WA_SET_BIT_MASKED(HDC_CHICKEN0,
+				  HDC_FENCE_DEST_SLM_DISABLE);
+
+	/* WaToEnableHwFixForPushConstHWBug:kbl */
+	if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER))
+		WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
+				  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
+
+	/* WaDisableSbeCacheDispatchPortSharing:kbl */
+	WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
+			  GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
+
+	return 0;
+}
+
+static int glk_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	int ret;
+
+	ret = gen9_ctx_workarounds_init(dev_priv);
+	if (ret)
+		return ret;
+
+	/* WaToEnableHwFixForPushConstHWBug:glk */
+	WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
+			  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
+
+	return 0;
+}
+
+static int cfl_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	int ret;
+
+	ret = gen9_ctx_workarounds_init(dev_priv);
+	if (ret)
+		return ret;
+
+	/* WaToEnableHwFixForPushConstHWBug:cfl */
+	WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
+			  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
+
+	/* WaDisableSbeCacheDispatchPortSharing:cfl */
+	WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
+			  GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
+
+	return 0;
+}
+
+static int cnl_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	/* WaForceContextSaveRestoreNonCoherent:cnl */
+	WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
+			  HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
+
+	/* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
+	if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
+		WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
+
+	/* WaDisableReplayBufferBankArbitrationOptimization:cnl */
+	WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
+			  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
+
+	/* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
+	if (IS_CNL_REVID(dev_priv, 0, CNL_REVID_B0))
+		WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
+				  GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
+
+	/* WaPushConstantDereferenceHoldDisable:cnl */
+	WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
+
+	/* FtrEnableFastAnisoL1BankingFix:cnl */
+	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
+
+	/* WaDisable3DMidCmdPreemption:cnl */
+	WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
+
+	/* WaDisableGPGPUMidCmdPreemption:cnl */
+	WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
+			    GEN9_PREEMPT_GPGPU_LEVEL_MASK,
+			    GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
+
+	/* WaDisableEarlyEOT:cnl */
+	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
+
+	return 0;
+}
+
+static int icl_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	/* Wa_1604370585:icl (pre-prod)
+	 * Formerly known as WaPushConstantDereferenceHoldDisable
+	 */
+	if (IS_ICL_REVID(dev_priv, ICL_REVID_A0, ICL_REVID_B0))
+		WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
+				  PUSH_CONSTANT_DEREF_DISABLE);
+
+	/* WaForceEnableNonCoherent:icl
+	 * This is not the same workaround as in early Gen9 platforms, where
+	 * lacking this could cause system hangs, but coherency performance
+	 * overhead is high and only a few compute workloads really need it
+	 * (the register is whitelisted in hardware now, so UMDs can opt in
+	 * for coherency if they have a good reason).
+	 */
+	WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
+
+	return 0;
+}
+
+int intel_ctx_workarounds_init(struct drm_i915_private *dev_priv)
+{
+	int err = 0;
+
+	dev_priv->workarounds.count = 0;
+
+	if (INTEL_GEN(dev_priv) < 8)
+		err = 0;
+	else if (IS_BROADWELL(dev_priv))
+		err = bdw_ctx_workarounds_init(dev_priv);
+	else if (IS_CHERRYVIEW(dev_priv))
+		err = chv_ctx_workarounds_init(dev_priv);
+	else if (IS_SKYLAKE(dev_priv))
+		err = skl_ctx_workarounds_init(dev_priv);
+	else if (IS_BROXTON(dev_priv))
+		err = bxt_ctx_workarounds_init(dev_priv);
+	else if (IS_KABYLAKE(dev_priv))
+		err = kbl_ctx_workarounds_init(dev_priv);
+	else if (IS_GEMINILAKE(dev_priv))
+		err = glk_ctx_workarounds_init(dev_priv);
+	else if (IS_COFFEELAKE(dev_priv))
+		err = cfl_ctx_workarounds_init(dev_priv);
+	else if (IS_CANNONLAKE(dev_priv))
+		err = cnl_ctx_workarounds_init(dev_priv);
+	else if (IS_ICELAKE(dev_priv))
+		err = icl_ctx_workarounds_init(dev_priv);
+	else
+		MISSING_CASE(INTEL_GEN(dev_priv));
+	if (err)
+		return err;
+
+	DRM_DEBUG_DRIVER("Number of context specific w/a: %d\n",
+			 dev_priv->workarounds.count);
+	return 0;
+}
+
+int intel_ctx_workarounds_emit(struct i915_request *rq)
+{
+	struct i915_workarounds *w = &rq->i915->workarounds;
+	u32 *cs;
+	int ret, i;
+
+	if (w->count == 0)
+		return 0;
+
+	ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
+	if (ret)
+		return ret;
+
+	cs = intel_ring_begin(rq, (w->count * 2 + 2));
+	if (IS_ERR(cs))
+		return PTR_ERR(cs);
+
+	*cs++ = MI_LOAD_REGISTER_IMM(w->count);
+	for (i = 0; i < w->count; i++) {
+		*cs++ = i915_mmio_reg_offset(w->reg[i].addr);
+		*cs++ = w->reg[i].value;
+	}
+	*cs++ = MI_NOOP;
+
+	intel_ring_advance(rq, cs);
+
+	ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void bdw_gt_workarounds_apply(struct drm_i915_private *dev_priv)
+{
+}
+
+static void chv_gt_workarounds_apply(struct drm_i915_private *dev_priv)
+{
+}
+
+static void gen9_gt_workarounds_apply(struct drm_i915_private *dev_priv)
+{
+	/* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
+	I915_WRITE(GEN9_CSFE_CHICKEN1_RCS,
+		   _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
+
+	/* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
+	I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
+		   GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
+
+	/* WaDisableKillLogic:bxt,skl,kbl */
+	if (!IS_COFFEELAKE(dev_priv))
+		I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
+			   ECOCHK_DIS_TLB);
+
+	if (HAS_LLC(dev_priv)) {
+		/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
+		 *
+		 * Must match Display Engine. See
+		 * WaCompressedResourceDisplayNewHashMode.
+		 */
+		I915_WRITE(MMCD_MISC_CTRL,
+			   I915_READ(MMCD_MISC_CTRL) |
+			   MMCD_PCLA |
+			   MMCD_HOTSPOT_EN);
+	}
+
+	/* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
+	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
+		   BDW_DISABLE_HDC_INVALIDATION);
+
+	/* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
+	if (IS_GEN9_LP(dev_priv)) {
+		u32 val = I915_READ(GEN8_L3SQCREG1);
+
+		val &= ~L3_PRIO_CREDITS_MASK;
+		val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
+		I915_WRITE(GEN8_L3SQCREG1, val);
+	}
+
+	/* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
+	I915_WRITE(GEN8_L3SQCREG4,
+		   I915_READ(GEN8_L3SQCREG4) | GEN8_LQSC_FLUSH_COHERENT_LINES);
+
+	/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
+	I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
+		   _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
+}
+
+static void skl_gt_workarounds_apply(struct drm_i915_private *dev_priv)
+{
+	gen9_gt_workarounds_apply(dev_priv);
+
+	/* WaEnableGapsTsvCreditFix:skl */
+	I915_WRITE(GEN8_GARBCNTL,
+		   I915_READ(GEN8_GARBCNTL) | GEN9_GAPS_TSV_CREDIT_DISABLE);
+
+	/* WaDisableGafsUnitClkGating:skl */
+	I915_WRITE(GEN7_UCGCTL4,
+		   I915_READ(GEN7_UCGCTL4) | GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
+
+	/* WaInPlaceDecompressionHang:skl */
+	if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
+		I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+			   I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+			   GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+}
+
+static void bxt_gt_workarounds_apply(struct drm_i915_private *dev_priv)
+{
+	gen9_gt_workarounds_apply(dev_priv);
+
+	/* WaDisablePooledEuLoadBalancingFix:bxt */
+	I915_WRITE(FF_SLICE_CS_CHICKEN2,
+		   _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
+
+	/* WaInPlaceDecompressionHang:bxt */
+	I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+		   I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+		   GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+}
+
+static void kbl_gt_workarounds_apply(struct drm_i915_private *dev_priv)
+{
+	gen9_gt_workarounds_apply(dev_priv);
+
+	/* WaEnableGapsTsvCreditFix:kbl */
+	I915_WRITE(GEN8_GARBCNTL,
+		   I915_READ(GEN8_GARBCNTL) | GEN9_GAPS_TSV_CREDIT_DISABLE);
+
+	/* WaDisableDynamicCreditSharing:kbl */
+	if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
+		I915_WRITE(GAMT_CHKN_BIT_REG,
+			   I915_READ(GAMT_CHKN_BIT_REG) |
+			   GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
+
+	/* WaDisableGafsUnitClkGating:kbl */
+	I915_WRITE(GEN7_UCGCTL4,
+		   I915_READ(GEN7_UCGCTL4) | GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
+
+	/* WaInPlaceDecompressionHang:kbl */
+	I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+		   I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+		   GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+}
+
+static void glk_gt_workarounds_apply(struct drm_i915_private *dev_priv)
+{
+	gen9_gt_workarounds_apply(dev_priv);
+}
+
+static void cfl_gt_workarounds_apply(struct drm_i915_private *dev_priv)
+{
+	gen9_gt_workarounds_apply(dev_priv);
+
+	/* WaEnableGapsTsvCreditFix:cfl */
+	I915_WRITE(GEN8_GARBCNTL,
+		   I915_READ(GEN8_GARBCNTL) | GEN9_GAPS_TSV_CREDIT_DISABLE);
+
+	/* WaDisableGafsUnitClkGating:cfl */
+	I915_WRITE(GEN7_UCGCTL4,
+		   I915_READ(GEN7_UCGCTL4) | GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
+
+	/* WaInPlaceDecompressionHang:cfl */
+	I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+		   I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+		   GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+}
+
+static void cnl_gt_workarounds_apply(struct drm_i915_private *dev_priv)
+{
+	/* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
+	if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
+		I915_WRITE(GAMT_CHKN_BIT_REG,
+			   I915_READ(GAMT_CHKN_BIT_REG) |
+			   GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT);
+
+	/* WaInPlaceDecompressionHang:cnl */
+	I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+		   I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+		   GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+
+	/* WaEnablePreemptionGranularityControlByUMD:cnl */
+	I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
+		   _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
+}
+
+static void icl_gt_workarounds_apply(struct drm_i915_private *dev_priv)
+{
+	/* This is not an Wa. Enable for better image quality */
+	I915_WRITE(_3D_CHICKEN3,
+		   _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
+
+	/* WaInPlaceDecompressionHang:icl */
+	I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA, I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+					    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+
+	/* WaPipelineFlushCoherentLines:icl */
+	I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
+				   GEN8_LQSC_FLUSH_COHERENT_LINES);
+
+	/* Wa_1405543622:icl
+	 * Formerly known as WaGAPZPriorityScheme
+	 */
+	I915_WRITE(GEN8_GARBCNTL, I915_READ(GEN8_GARBCNTL) |
+				  GEN11_ARBITRATION_PRIO_ORDER_MASK);
+
+	/* Wa_1604223664:icl
+	 * Formerly known as WaL3BankAddressHashing
+	 */
+	I915_WRITE(GEN8_GARBCNTL,
+		   (I915_READ(GEN8_GARBCNTL) & ~GEN11_HASH_CTRL_EXCL_MASK) |
+		   GEN11_HASH_CTRL_EXCL_BIT0);
+	I915_WRITE(GEN11_GLBLINVL,
+		   (I915_READ(GEN11_GLBLINVL) & ~GEN11_BANK_HASH_ADDR_EXCL_MASK) |
+		   GEN11_BANK_HASH_ADDR_EXCL_BIT0);
+
+	/* WaModifyGamTlbPartitioning:icl */
+	I915_WRITE(GEN11_GACB_PERF_CTRL,
+		   (I915_READ(GEN11_GACB_PERF_CTRL) & ~GEN11_HASH_CTRL_MASK) |
+		   GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
+
+	/* Wa_1405733216:icl
+	 * Formerly known as WaDisableCleanEvicts
+	 */
+	I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
+				   GEN11_LQSC_CLEAN_EVICT_DISABLE);
+
+	/* Wa_1405766107:icl
+	 * Formerly known as WaCL2SFHalfMaxAlloc
+	 */
+	I915_WRITE(GEN11_LSN_UNSLCVC, I915_READ(GEN11_LSN_UNSLCVC) |
+				      GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
+				      GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
+
+	/* Wa_220166154:icl
+	 * Formerly known as WaDisCtxReload
+	 */
+	I915_WRITE(GAMW_ECO_DEV_RW_IA_REG, I915_READ(GAMW_ECO_DEV_RW_IA_REG) |
+					   GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
+
+	/* Wa_1405779004:icl (pre-prod) */
+	if (IS_ICL_REVID(dev_priv, ICL_REVID_A0, ICL_REVID_A0))
+		I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE,
+			   I915_READ(SLICE_UNIT_LEVEL_CLKGATE) |
+			   MSCUNIT_CLKGATE_DIS);
+
+	/* Wa_1406680159:icl */
+	I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE,
+		   I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE) |
+		   GWUNIT_CLKGATE_DIS);
+
+	/* Wa_1604302699:icl */
+	I915_WRITE(GEN10_L3_CHICKEN_MODE_REGISTER,
+		   I915_READ(GEN10_L3_CHICKEN_MODE_REGISTER) |
+		   GEN11_I2M_WRITE_DISABLE);
+
+	/* Wa_1406838659:icl (pre-prod) */
+	if (IS_ICL_REVID(dev_priv, ICL_REVID_A0, ICL_REVID_B0))
+		I915_WRITE(INF_UNIT_LEVEL_CLKGATE,
+			   I915_READ(INF_UNIT_LEVEL_CLKGATE) |
+			   CGPSF_CLKGATE_DIS);
+
+	/* WaForwardProgressSoftReset:icl */
+	I915_WRITE(GEN10_SCRATCH_LNCF2,
+		   I915_READ(GEN10_SCRATCH_LNCF2) |
+		   PMFLUSHDONE_LNICRSDROP |
+		   PMFLUSH_GAPL3UNBLOCK |
+		   PMFLUSHDONE_LNEBLK);
+}
+
+void intel_gt_workarounds_apply(struct drm_i915_private *dev_priv)
+{
+	if (INTEL_GEN(dev_priv) < 8)
+		return;
+	else if (IS_BROADWELL(dev_priv))
+		bdw_gt_workarounds_apply(dev_priv);
+	else if (IS_CHERRYVIEW(dev_priv))
+		chv_gt_workarounds_apply(dev_priv);
+	else if (IS_SKYLAKE(dev_priv))
+		skl_gt_workarounds_apply(dev_priv);
+	else if (IS_BROXTON(dev_priv))
+		bxt_gt_workarounds_apply(dev_priv);
+	else if (IS_KABYLAKE(dev_priv))
+		kbl_gt_workarounds_apply(dev_priv);
+	else if (IS_GEMINILAKE(dev_priv))
+		glk_gt_workarounds_apply(dev_priv);
+	else if (IS_COFFEELAKE(dev_priv))
+		cfl_gt_workarounds_apply(dev_priv);
+	else if (IS_CANNONLAKE(dev_priv))
+		cnl_gt_workarounds_apply(dev_priv);
+	else if (IS_ICELAKE(dev_priv))
+		icl_gt_workarounds_apply(dev_priv);
+	else
+		MISSING_CASE(INTEL_GEN(dev_priv));
+}
+
+struct whitelist {
+	i915_reg_t reg[RING_MAX_NONPRIV_SLOTS];
+	unsigned int count;
+	u32 nopid;
+};
+
+static void whitelist_reg(struct whitelist *w, i915_reg_t reg)
+{
+	if (GEM_WARN_ON(w->count >= RING_MAX_NONPRIV_SLOTS))
+		return;
+
+	w->reg[w->count++] = reg;
+}
+
+static void bdw_whitelist_build(struct whitelist *w)
+{
+}
+
+static void chv_whitelist_build(struct whitelist *w)
+{
+}
+
+static void gen9_whitelist_build(struct whitelist *w)
+{
+	/* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
+	whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
+
+	/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
+	whitelist_reg(w, GEN8_CS_CHICKEN1);
+
+	/* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
+	whitelist_reg(w, GEN8_HDC_CHICKEN1);
+}
+
+static void skl_whitelist_build(struct whitelist *w)
+{
+	gen9_whitelist_build(w);
+
+	/* WaDisableLSQCROPERFforOCL:skl */
+	whitelist_reg(w, GEN8_L3SQCREG4);
+}
+
+static void bxt_whitelist_build(struct whitelist *w)
+{
+	gen9_whitelist_build(w);
+}
+
+static void kbl_whitelist_build(struct whitelist *w)
+{
+	gen9_whitelist_build(w);
+
+	/* WaDisableLSQCROPERFforOCL:kbl */
+	whitelist_reg(w, GEN8_L3SQCREG4);
+}
+
+static void glk_whitelist_build(struct whitelist *w)
+{
+	gen9_whitelist_build(w);
+
+	/* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
+	whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
+}
+
+static void cfl_whitelist_build(struct whitelist *w)
+{
+	gen9_whitelist_build(w);
+}
+
+static void cnl_whitelist_build(struct whitelist *w)
+{
+	/* WaEnablePreemptionGranularityControlByUMD:cnl */
+	whitelist_reg(w, GEN8_CS_CHICKEN1);
+}
+
+static void icl_whitelist_build(struct whitelist *w)
+{
+}
+
+static struct whitelist *whitelist_build(struct intel_engine_cs *engine,
+					 struct whitelist *w)
+{
+	struct drm_i915_private *i915 = engine->i915;
+
+	GEM_BUG_ON(engine->id != RCS);
+
+	w->count = 0;
+	w->nopid = i915_mmio_reg_offset(RING_NOPID(engine->mmio_base));
+
+	if (INTEL_GEN(i915) < 8)
+		return NULL;
+	else if (IS_BROADWELL(i915))
+		bdw_whitelist_build(w);
+	else if (IS_CHERRYVIEW(i915))
+		chv_whitelist_build(w);
+	else if (IS_SKYLAKE(i915))
+		skl_whitelist_build(w);
+	else if (IS_BROXTON(i915))
+		bxt_whitelist_build(w);
+	else if (IS_KABYLAKE(i915))
+		kbl_whitelist_build(w);
+	else if (IS_GEMINILAKE(i915))
+		glk_whitelist_build(w);
+	else if (IS_COFFEELAKE(i915))
+		cfl_whitelist_build(w);
+	else if (IS_CANNONLAKE(i915))
+		cnl_whitelist_build(w);
+	else if (IS_ICELAKE(i915))
+		icl_whitelist_build(w);
+	else
+		MISSING_CASE(INTEL_GEN(i915));
+
+	return w;
+}
+
+static void whitelist_apply(struct intel_engine_cs *engine,
+			    const struct whitelist *w)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	const u32 base = engine->mmio_base;
+	unsigned int i;
+
+	if (!w)
+		return;
+
+	intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL);
+
+	for (i = 0; i < w->count; i++)
+		I915_WRITE_FW(RING_FORCE_TO_NONPRIV(base, i),
+			      i915_mmio_reg_offset(w->reg[i]));
+
+	/* And clear the rest just in case of garbage */
+	for (; i < RING_MAX_NONPRIV_SLOTS; i++)
+		I915_WRITE_FW(RING_FORCE_TO_NONPRIV(base, i), w->nopid);
+
+	intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL);
+}
+
+void intel_whitelist_workarounds_apply(struct intel_engine_cs *engine)
+{
+	struct whitelist w;
+
+	whitelist_apply(engine, whitelist_build(engine, &w));
+}
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftests/intel_workarounds.c"
+#endif
diff --git a/drivers/gpu/drm/i915/intel_workarounds.h b/drivers/gpu/drm/i915/intel_workarounds.h
new file mode 100644
index 000000000000..b11d0623e626
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_workarounds.h
@@ -0,0 +1,17 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2014-2018 Intel Corporation
+ */
+
+#ifndef _I915_WORKAROUNDS_H_
+#define _I915_WORKAROUNDS_H_
+
+int intel_ctx_workarounds_init(struct drm_i915_private *dev_priv);
+int intel_ctx_workarounds_emit(struct i915_request *rq);
+
+void intel_gt_workarounds_apply(struct drm_i915_private *dev_priv);
+
+void intel_whitelist_workarounds_apply(struct intel_engine_cs *engine);
+
+#endif
diff --git a/drivers/gpu/drm/i915/selftests/huge_pages.c b/drivers/gpu/drm/i915/selftests/huge_pages.c
index 05bbef363fff..91c72911be3c 100644
--- a/drivers/gpu/drm/i915/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/selftests/huge_pages.c
@@ -1091,7 +1091,7 @@ static int __igt_write_huge(struct i915_gem_context *ctx,
 out_vma_unpin:
 	i915_vma_unpin(vma);
 out_vma_close:
-	i915_vma_close(vma);
+	i915_vma_destroy(vma);
 
 	return err;
 }
@@ -1757,6 +1757,9 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv)
 		goto out_unlock;
 	}
 
+	if (ctx->ppgtt)
+		ctx->ppgtt->base.scrub_64K = true;
+
 	err = i915_subtests(tests, ctx);
 
 out_unlock:
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
index 7ecaed50d0b9..ddb03f009232 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
@@ -23,6 +23,7 @@
  */
 
 #include "../i915_selftest.h"
+#include "igt_flush_test.h"
 
 #include "mock_drm.h"
 #include "huge_gem_object.h"
@@ -411,6 +412,8 @@ static int igt_ctx_exec(void *arg)
 	}
 
 out_unlock:
+	if (igt_flush_test(i915, I915_WAIT_LOCKED))
+		err = -EIO;
 	mutex_unlock(&i915->drm.struct_mutex);
 
 	mock_file_free(i915, file);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c b/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c
deleted file mode 100644
index 3000e6a7d82d..000000000000
--- a/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Copyright © 2017 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- *
- */
-
-#include "../i915_selftest.h"
-#include "i915_random.h"
-
-#include "mock_gem_device.h"
-#include "mock_timeline.h"
-
-struct __igt_sync {
-	const char *name;
-	u32 seqno;
-	bool expected;
-	bool set;
-};
-
-static int __igt_sync(struct intel_timeline *tl,
-		      u64 ctx,
-		      const struct __igt_sync *p,
-		      const char *name)
-{
-	int ret;
-
-	if (__intel_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) {
-		pr_err("%s: %s(ctx=%llu, seqno=%u) expected passed %s but failed\n",
-		       name, p->name, ctx, p->seqno, yesno(p->expected));
-		return -EINVAL;
-	}
-
-	if (p->set) {
-		ret = __intel_timeline_sync_set(tl, ctx, p->seqno);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int igt_sync(void *arg)
-{
-	const struct __igt_sync pass[] = {
-		{ "unset", 0, false, false },
-		{ "new", 0, false, true },
-		{ "0a", 0, true, true },
-		{ "1a", 1, false, true },
-		{ "1b", 1, true, true },
-		{ "0b", 0, true, false },
-		{ "2a", 2, false, true },
-		{ "4", 4, false, true },
-		{ "INT_MAX", INT_MAX, false, true },
-		{ "INT_MAX-1", INT_MAX-1, true, false },
-		{ "INT_MAX+1", (u32)INT_MAX+1, false, true },
-		{ "INT_MAX", INT_MAX, true, false },
-		{ "UINT_MAX", UINT_MAX, false, true },
-		{ "wrap", 0, false, true },
-		{ "unwrap", UINT_MAX, true, false },
-		{},
-	}, *p;
-	struct intel_timeline *tl;
-	int order, offset;
-	int ret = -ENODEV;
-
-	tl = mock_timeline(0);
-	if (!tl)
-		return -ENOMEM;
-
-	for (p = pass; p->name; p++) {
-		for (order = 1; order < 64; order++) {
-			for (offset = -1; offset <= (order > 1); offset++) {
-				u64 ctx = BIT_ULL(order) + offset;
-
-				ret = __igt_sync(tl, ctx, p, "1");
-				if (ret)
-					goto out;
-			}
-		}
-	}
-	mock_timeline_destroy(tl);
-
-	tl = mock_timeline(0);
-	if (!tl)
-		return -ENOMEM;
-
-	for (order = 1; order < 64; order++) {
-		for (offset = -1; offset <= (order > 1); offset++) {
-			u64 ctx = BIT_ULL(order) + offset;
-
-			for (p = pass; p->name; p++) {
-				ret = __igt_sync(tl, ctx, p, "2");
-				if (ret)
-					goto out;
-			}
-		}
-	}
-
-out:
-	mock_timeline_destroy(tl);
-	return ret;
-}
-
-static unsigned int random_engine(struct rnd_state *rnd)
-{
-	return i915_prandom_u32_max_state(I915_NUM_ENGINES, rnd);
-}
-
-static int bench_sync(void *arg)
-{
-	struct rnd_state prng;
-	struct intel_timeline *tl;
-	unsigned long end_time, count;
-	u64 prng32_1M;
-	ktime_t kt;
-	int order, last_order;
-
-	tl = mock_timeline(0);
-	if (!tl)
-		return -ENOMEM;
-
-	/* Lookups from cache are very fast and so the random number generation
-	 * and the loop itself becomes a significant factor in the per-iteration
-	 * timings. We try to compensate the results by measuring the overhead
-	 * of the prng and subtract it from the reported results.
-	 */
-	prandom_seed_state(&prng, i915_selftest.random_seed);
-	count = 0;
-	kt = ktime_get();
-	end_time = jiffies + HZ/10;
-	do {
-		u32 x;
-
-		/* Make sure the compiler doesn't optimise away the prng call */
-		WRITE_ONCE(x, prandom_u32_state(&prng));
-
-		count++;
-	} while (!time_after(jiffies, end_time));
-	kt = ktime_sub(ktime_get(), kt);
-	pr_debug("%s: %lu random evaluations, %lluns/prng\n",
-		 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
-	prng32_1M = div64_ul(ktime_to_ns(kt) << 20, count);
-
-	/* Benchmark (only) setting random context ids */
-	prandom_seed_state(&prng, i915_selftest.random_seed);
-	count = 0;
-	kt = ktime_get();
-	end_time = jiffies + HZ/10;
-	do {
-		u64 id = i915_prandom_u64_state(&prng);
-
-		__intel_timeline_sync_set(tl, id, 0);
-		count++;
-	} while (!time_after(jiffies, end_time));
-	kt = ktime_sub(ktime_get(), kt);
-	kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
-	pr_info("%s: %lu random insertions, %lluns/insert\n",
-		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
-
-	/* Benchmark looking up the exact same context ids as we just set */
-	prandom_seed_state(&prng, i915_selftest.random_seed);
-	end_time = count;
-	kt = ktime_get();
-	while (end_time--) {
-		u64 id = i915_prandom_u64_state(&prng);
-
-		if (!__intel_timeline_sync_is_later(tl, id, 0)) {
-			mock_timeline_destroy(tl);
-			pr_err("Lookup of %llu failed\n", id);
-			return -EINVAL;
-		}
-	}
-	kt = ktime_sub(ktime_get(), kt);
-	kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
-	pr_info("%s: %lu random lookups, %lluns/lookup\n",
-		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
-
-	mock_timeline_destroy(tl);
-	cond_resched();
-
-	tl = mock_timeline(0);
-	if (!tl)
-		return -ENOMEM;
-
-	/* Benchmark setting the first N (in order) contexts */
-	count = 0;
-	kt = ktime_get();
-	end_time = jiffies + HZ/10;
-	do {
-		__intel_timeline_sync_set(tl, count++, 0);
-	} while (!time_after(jiffies, end_time));
-	kt = ktime_sub(ktime_get(), kt);
-	pr_info("%s: %lu in-order insertions, %lluns/insert\n",
-		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
-
-	/* Benchmark looking up the exact same context ids as we just set */
-	end_time = count;
-	kt = ktime_get();
-	while (end_time--) {
-		if (!__intel_timeline_sync_is_later(tl, end_time, 0)) {
-			pr_err("Lookup of %lu failed\n", end_time);
-			mock_timeline_destroy(tl);
-			return -EINVAL;
-		}
-	}
-	kt = ktime_sub(ktime_get(), kt);
-	pr_info("%s: %lu in-order lookups, %lluns/lookup\n",
-		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
-
-	mock_timeline_destroy(tl);
-	cond_resched();
-
-	tl = mock_timeline(0);
-	if (!tl)
-		return -ENOMEM;
-
-	/* Benchmark searching for a random context id and maybe changing it */
-	prandom_seed_state(&prng, i915_selftest.random_seed);
-	count = 0;
-	kt = ktime_get();
-	end_time = jiffies + HZ/10;
-	do {
-		u32 id = random_engine(&prng);
-		u32 seqno = prandom_u32_state(&prng);
-
-		if (!__intel_timeline_sync_is_later(tl, id, seqno))
-			__intel_timeline_sync_set(tl, id, seqno);
-
-		count++;
-	} while (!time_after(jiffies, end_time));
-	kt = ktime_sub(ktime_get(), kt);
-	kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
-	pr_info("%s: %lu repeated insert/lookups, %lluns/op\n",
-		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
-	mock_timeline_destroy(tl);
-	cond_resched();
-
-	/* Benchmark searching for a known context id and changing the seqno */
-	for (last_order = 1, order = 1; order < 32;
-	     ({ int tmp = last_order; last_order = order; order += tmp; })) {
-		unsigned int mask = BIT(order) - 1;
-
-		tl = mock_timeline(0);
-		if (!tl)
-			return -ENOMEM;
-
-		count = 0;
-		kt = ktime_get();
-		end_time = jiffies + HZ/10;
-		do {
-			/* Without assuming too many details of the underlying
-			 * implementation, try to identify its phase-changes
-			 * (if any)!
-			 */
-			u64 id = (u64)(count & mask) << order;
-
-			__intel_timeline_sync_is_later(tl, id, 0);
-			__intel_timeline_sync_set(tl, id, 0);
-
-			count++;
-		} while (!time_after(jiffies, end_time));
-		kt = ktime_sub(ktime_get(), kt);
-		pr_info("%s: %lu cyclic/%d insert/lookups, %lluns/op\n",
-			__func__, count, order,
-			(long long)div64_ul(ktime_to_ns(kt), count));
-		mock_timeline_destroy(tl);
-		cond_resched();
-	}
-
-	return 0;
-}
-
-int i915_gem_timeline_mock_selftests(void)
-{
-	static const struct i915_subtest tests[] = {
-		SUBTEST(igt_sync),
-		SUBTEST(bench_sync),
-	};
-
-	return i915_subtests(tests, NULL);
-}
diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
index 9c76f0305b6a..a00e2bd08bce 100644
--- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
@@ -11,6 +11,7 @@
  */
 selftest(sanitycheck, i915_live_sanitycheck) /* keep first (igt selfcheck) */
 selftest(uncore, intel_uncore_live_selftests)
+selftest(workarounds, intel_workarounds_live_selftests)
 selftest(requests, i915_request_live_selftests)
 selftest(objects, i915_gem_object_live_selftests)
 selftest(dmabuf, i915_gem_dmabuf_live_selftests)
@@ -20,4 +21,5 @@ selftest(evict, i915_gem_evict_live_selftests)
 selftest(hugepages, i915_gem_huge_page_live_selftests)
 selftest(contexts, i915_gem_context_live_selftests)
 selftest(hangcheck, intel_hangcheck_live_selftests)
+selftest(execlists, intel_execlists_live_selftests)
 selftest(guc, intel_guc_live_selftest)
diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
index 9a48aa441743..d16d74178e9d 100644
--- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
@@ -14,6 +14,7 @@ selftest(fence, i915_sw_fence_mock_selftests)
 selftest(scatterlist, scatterlist_mock_selftests)
 selftest(syncmap, i915_syncmap_mock_selftests)
 selftest(uncore, intel_uncore_mock_selftests)
+selftest(engine, intel_engine_cs_mock_selftests)
 selftest(breadcrumbs, intel_breadcrumbs_mock_selftests)
 selftest(timelines, i915_gem_timeline_mock_selftests)
 selftest(requests, i915_request_mock_selftests)
diff --git a/drivers/gpu/drm/i915/selftests/i915_timeline.c b/drivers/gpu/drm/i915/selftests/i915_timeline.c
new file mode 100644
index 000000000000..19f1c6a5c8fb
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/i915_timeline.c
@@ -0,0 +1,267 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2017-2018 Intel Corporation
+ */
+
+#include "../i915_selftest.h"
+#include "i915_random.h"
+
+#include "mock_gem_device.h"
+#include "mock_timeline.h"
+
+struct __igt_sync {
+	const char *name;
+	u32 seqno;
+	bool expected;
+	bool set;
+};
+
+static int __igt_sync(struct i915_timeline *tl,
+		      u64 ctx,
+		      const struct __igt_sync *p,
+		      const char *name)
+{
+	int ret;
+
+	if (__i915_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) {
+		pr_err("%s: %s(ctx=%llu, seqno=%u) expected passed %s but failed\n",
+		       name, p->name, ctx, p->seqno, yesno(p->expected));
+		return -EINVAL;
+	}
+
+	if (p->set) {
+		ret = __i915_timeline_sync_set(tl, ctx, p->seqno);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int igt_sync(void *arg)
+{
+	const struct __igt_sync pass[] = {
+		{ "unset", 0, false, false },
+		{ "new", 0, false, true },
+		{ "0a", 0, true, true },
+		{ "1a", 1, false, true },
+		{ "1b", 1, true, true },
+		{ "0b", 0, true, false },
+		{ "2a", 2, false, true },
+		{ "4", 4, false, true },
+		{ "INT_MAX", INT_MAX, false, true },
+		{ "INT_MAX-1", INT_MAX-1, true, false },
+		{ "INT_MAX+1", (u32)INT_MAX+1, false, true },
+		{ "INT_MAX", INT_MAX, true, false },
+		{ "UINT_MAX", UINT_MAX, false, true },
+		{ "wrap", 0, false, true },
+		{ "unwrap", UINT_MAX, true, false },
+		{},
+	}, *p;
+	struct i915_timeline tl;
+	int order, offset;
+	int ret = -ENODEV;
+
+	mock_timeline_init(&tl, 0);
+	for (p = pass; p->name; p++) {
+		for (order = 1; order < 64; order++) {
+			for (offset = -1; offset <= (order > 1); offset++) {
+				u64 ctx = BIT_ULL(order) + offset;
+
+				ret = __igt_sync(&tl, ctx, p, "1");
+				if (ret)
+					goto out;
+			}
+		}
+	}
+	mock_timeline_fini(&tl);
+
+	mock_timeline_init(&tl, 0);
+	for (order = 1; order < 64; order++) {
+		for (offset = -1; offset <= (order > 1); offset++) {
+			u64 ctx = BIT_ULL(order) + offset;
+
+			for (p = pass; p->name; p++) {
+				ret = __igt_sync(&tl, ctx, p, "2");
+				if (ret)
+					goto out;
+			}
+		}
+	}
+
+out:
+	mock_timeline_fini(&tl);
+	return ret;
+}
+
+static unsigned int random_engine(struct rnd_state *rnd)
+{
+	return i915_prandom_u32_max_state(I915_NUM_ENGINES, rnd);
+}
+
+static int bench_sync(void *arg)
+{
+	struct rnd_state prng;
+	struct i915_timeline tl;
+	unsigned long end_time, count;
+	u64 prng32_1M;
+	ktime_t kt;
+	int order, last_order;
+
+	mock_timeline_init(&tl, 0);
+
+	/* Lookups from cache are very fast and so the random number generation
+	 * and the loop itself becomes a significant factor in the per-iteration
+	 * timings. We try to compensate the results by measuring the overhead
+	 * of the prng and subtract it from the reported results.
+	 */
+	prandom_seed_state(&prng, i915_selftest.random_seed);
+	count = 0;
+	kt = ktime_get();
+	end_time = jiffies + HZ/10;
+	do {
+		u32 x;
+
+		/* Make sure the compiler doesn't optimise away the prng call */
+		WRITE_ONCE(x, prandom_u32_state(&prng));
+
+		count++;
+	} while (!time_after(jiffies, end_time));
+	kt = ktime_sub(ktime_get(), kt);
+	pr_debug("%s: %lu random evaluations, %lluns/prng\n",
+		 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+	prng32_1M = div64_ul(ktime_to_ns(kt) << 20, count);
+
+	/* Benchmark (only) setting random context ids */
+	prandom_seed_state(&prng, i915_selftest.random_seed);
+	count = 0;
+	kt = ktime_get();
+	end_time = jiffies + HZ/10;
+	do {
+		u64 id = i915_prandom_u64_state(&prng);
+
+		__i915_timeline_sync_set(&tl, id, 0);
+		count++;
+	} while (!time_after(jiffies, end_time));
+	kt = ktime_sub(ktime_get(), kt);
+	kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
+	pr_info("%s: %lu random insertions, %lluns/insert\n",
+		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+
+	/* Benchmark looking up the exact same context ids as we just set */
+	prandom_seed_state(&prng, i915_selftest.random_seed);
+	end_time = count;
+	kt = ktime_get();
+	while (end_time--) {
+		u64 id = i915_prandom_u64_state(&prng);
+
+		if (!__i915_timeline_sync_is_later(&tl, id, 0)) {
+			mock_timeline_fini(&tl);
+			pr_err("Lookup of %llu failed\n", id);
+			return -EINVAL;
+		}
+	}
+	kt = ktime_sub(ktime_get(), kt);
+	kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
+	pr_info("%s: %lu random lookups, %lluns/lookup\n",
+		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+
+	mock_timeline_fini(&tl);
+	cond_resched();
+
+	mock_timeline_init(&tl, 0);
+
+	/* Benchmark setting the first N (in order) contexts */
+	count = 0;
+	kt = ktime_get();
+	end_time = jiffies + HZ/10;
+	do {
+		__i915_timeline_sync_set(&tl, count++, 0);
+	} while (!time_after(jiffies, end_time));
+	kt = ktime_sub(ktime_get(), kt);
+	pr_info("%s: %lu in-order insertions, %lluns/insert\n",
+		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+
+	/* Benchmark looking up the exact same context ids as we just set */
+	end_time = count;
+	kt = ktime_get();
+	while (end_time--) {
+		if (!__i915_timeline_sync_is_later(&tl, end_time, 0)) {
+			pr_err("Lookup of %lu failed\n", end_time);
+			mock_timeline_fini(&tl);
+			return -EINVAL;
+		}
+	}
+	kt = ktime_sub(ktime_get(), kt);
+	pr_info("%s: %lu in-order lookups, %lluns/lookup\n",
+		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+
+	mock_timeline_fini(&tl);
+	cond_resched();
+
+	mock_timeline_init(&tl, 0);
+
+	/* Benchmark searching for a random context id and maybe changing it */
+	prandom_seed_state(&prng, i915_selftest.random_seed);
+	count = 0;
+	kt = ktime_get();
+	end_time = jiffies + HZ/10;
+	do {
+		u32 id = random_engine(&prng);
+		u32 seqno = prandom_u32_state(&prng);
+
+		if (!__i915_timeline_sync_is_later(&tl, id, seqno))
+			__i915_timeline_sync_set(&tl, id, seqno);
+
+		count++;
+	} while (!time_after(jiffies, end_time));
+	kt = ktime_sub(ktime_get(), kt);
+	kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
+	pr_info("%s: %lu repeated insert/lookups, %lluns/op\n",
+		__func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
+	mock_timeline_fini(&tl);
+	cond_resched();
+
+	/* Benchmark searching for a known context id and changing the seqno */
+	for (last_order = 1, order = 1; order < 32;
+	     ({ int tmp = last_order; last_order = order; order += tmp; })) {
+		unsigned int mask = BIT(order) - 1;
+
+		mock_timeline_init(&tl, 0);
+
+		count = 0;
+		kt = ktime_get();
+		end_time = jiffies + HZ/10;
+		do {
+			/* Without assuming too many details of the underlying
+			 * implementation, try to identify its phase-changes
+			 * (if any)!
+			 */
+			u64 id = (u64)(count & mask) << order;
+
+			__i915_timeline_sync_is_later(&tl, id, 0);
+			__i915_timeline_sync_set(&tl, id, 0);
+
+			count++;
+		} while (!time_after(jiffies, end_time));
+		kt = ktime_sub(ktime_get(), kt);
+		pr_info("%s: %lu cyclic/%d insert/lookups, %lluns/op\n",
+			__func__, count, order,
+			(long long)div64_ul(ktime_to_ns(kt), count));
+		mock_timeline_fini(&tl);
+		cond_resched();
+	}
+
+	return 0;
+}
+
+int i915_gem_timeline_mock_selftests(void)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(igt_sync),
+		SUBTEST(bench_sync),
+	};
+
+	return i915_subtests(tests, NULL);
+}
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index eb89e301b602..e90f97236e50 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -81,7 +81,7 @@ checked_vma_instance(struct drm_i915_gem_object *obj,
 	}
 
 	if (i915_vma_compare(vma, vm, view)) {
-		pr_err("i915_vma_compare failed with create parmaters!\n");
+		pr_err("i915_vma_compare failed with create parameters!\n");
 		return ERR_PTR(-EINVAL);
 	}
 
diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.c b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
new file mode 100644
index 000000000000..0d06f559243f
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
@@ -0,0 +1,70 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2018 Intel Corporation
+ */
+
+#include "../i915_drv.h"
+
+#include "../i915_selftest.h"
+#include "igt_flush_test.h"
+
+struct wedge_me {
+	struct delayed_work work;
+	struct drm_i915_private *i915;
+	const void *symbol;
+};
+
+static void wedge_me(struct work_struct *work)
+{
+	struct wedge_me *w = container_of(work, typeof(*w), work.work);
+
+	pr_err("%pS timed out, cancelling all further testing.\n", w->symbol);
+
+	GEM_TRACE("%pS timed out.\n", w->symbol);
+	GEM_TRACE_DUMP();
+
+	i915_gem_set_wedged(w->i915);
+}
+
+static void __init_wedge(struct wedge_me *w,
+			 struct drm_i915_private *i915,
+			 long timeout,
+			 const void *symbol)
+{
+	w->i915 = i915;
+	w->symbol = symbol;
+
+	INIT_DELAYED_WORK_ONSTACK(&w->work, wedge_me);
+	schedule_delayed_work(&w->work, timeout);
+}
+
+static void __fini_wedge(struct wedge_me *w)
+{
+	cancel_delayed_work_sync(&w->work);
+	destroy_delayed_work_on_stack(&w->work);
+	w->i915 = NULL;
+}
+
+#define wedge_on_timeout(W, DEV, TIMEOUT)				\
+	for (__init_wedge((W), (DEV), (TIMEOUT), __builtin_return_address(0)); \
+	     (W)->i915;							\
+	     __fini_wedge((W)))
+
+int igt_flush_test(struct drm_i915_private *i915, unsigned int flags)
+{
+	struct wedge_me w;
+
+	cond_resched();
+
+	if (flags & I915_WAIT_LOCKED &&
+	    i915_gem_switch_to_kernel_context(i915)) {
+		pr_err("Failed to switch back to kernel context; declaring wedged\n");
+		i915_gem_set_wedged(i915);
+	}
+
+	wedge_on_timeout(&w, i915, HZ)
+		i915_gem_wait_for_idle(i915, flags);
+
+	return i915_terminally_wedged(&i915->gpu_error) ? -EIO : 0;
+}
diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.h b/drivers/gpu/drm/i915/selftests/igt_flush_test.h
new file mode 100644
index 000000000000..63e009927c43
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.h
@@ -0,0 +1,14 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2018 Intel Corporation
+ */
+
+#ifndef IGT_FLUSH_TEST_H
+#define IGT_FLUSH_TEST_H
+
+struct drm_i915_private;
+
+int igt_flush_test(struct drm_i915_private *i915, unsigned int flags);
+
+#endif /* IGT_FLUSH_TEST_H */
diff --git a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c b/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
index 46580026c7fc..d6926e7820e5 100644
--- a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
@@ -412,10 +412,11 @@ static int igt_wakeup(void *arg)
 		 * that they are ready for the next test. We wait until all
 		 * threads are complete and waiting for us (i.e. not a seqno).
 		 */
-		err = wait_var_event_timeout(&done, !atomic_read(&done), 10 * HZ);
-		if (err) {
+		if (!wait_var_event_timeout(&done,
+					    !atomic_read(&done), 10 * HZ)) {
 			pr_err("Timed out waiting for %d remaining waiters\n",
 			       atomic_read(&done));
+			err = -ETIMEDOUT;
 			break;
 		}
 
diff --git a/drivers/gpu/drm/i915/selftests/intel_engine_cs.c b/drivers/gpu/drm/i915/selftests/intel_engine_cs.c
new file mode 100644
index 000000000000..cfaa6b296835
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/intel_engine_cs.c
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright © 2018 Intel Corporation
+ */
+
+#include "../i915_selftest.h"
+
+static int intel_mmio_bases_check(void *arg)
+{
+	int i, j;
+
+	for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
+		const struct engine_info *info = &intel_engines[i];
+		char name[INTEL_ENGINE_CS_MAX_NAME];
+		u8 prev = U8_MAX;
+
+		__sprint_engine_name(name, info);
+
+		for (j = 0; j < MAX_MMIO_BASES; j++) {
+			u8 gen = info->mmio_bases[j].gen;
+			u32 base = info->mmio_bases[j].base;
+
+			if (gen >= prev) {
+				pr_err("%s: %s: mmio base for gen %x "
+					"is before the one for gen %x\n",
+				       __func__, name, prev, gen);
+				return -EINVAL;
+			}
+
+			if (gen == 0)
+				break;
+
+			if (!base) {
+				pr_err("%s: %s: invalid mmio base (%x) "
+					"for gen %x at entry %u\n",
+				       __func__, name, base, gen, j);
+				return -EINVAL;
+			}
+
+			prev = gen;
+		}
+
+		pr_info("%s: min gen supported for %s = %d\n",
+			__func__, name, prev);
+	}
+
+	return 0;
+}
+
+int intel_engine_cs_mock_selftests(void)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(intel_mmio_bases_check),
+	};
+
+	return i915_subtests(tests, NULL);
+}
diff --git a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
index df7898c8edcb..438e0b045a2c 100644
--- a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
@@ -25,10 +25,14 @@
 #include <linux/kthread.h>
 
 #include "../i915_selftest.h"
+#include "i915_random.h"
+#include "igt_flush_test.h"
 
 #include "mock_context.h"
 #include "mock_drm.h"
 
+#define IGT_IDLE_TIMEOUT 50 /* ms; time to wait after flushing between tests */
+
 struct hang {
 	struct drm_i915_private *i915;
 	struct drm_i915_gem_object *hws;
@@ -250,58 +254,6 @@ static u32 hws_seqno(const struct hang *h, const struct i915_request *rq)
 	return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]);
 }
 
-struct wedge_me {
-	struct delayed_work work;
-	struct drm_i915_private *i915;
-	const void *symbol;
-};
-
-static void wedge_me(struct work_struct *work)
-{
-	struct wedge_me *w = container_of(work, typeof(*w), work.work);
-
-	pr_err("%pS timed out, cancelling all further testing.\n",
-	       w->symbol);
-	i915_gem_set_wedged(w->i915);
-}
-
-static void __init_wedge(struct wedge_me *w,
-			 struct drm_i915_private *i915,
-			 long timeout,
-			 const void *symbol)
-{
-	w->i915 = i915;
-	w->symbol = symbol;
-
-	INIT_DELAYED_WORK_ONSTACK(&w->work, wedge_me);
-	schedule_delayed_work(&w->work, timeout);
-}
-
-static void __fini_wedge(struct wedge_me *w)
-{
-	cancel_delayed_work_sync(&w->work);
-	destroy_delayed_work_on_stack(&w->work);
-	w->i915 = NULL;
-}
-
-#define wedge_on_timeout(W, DEV, TIMEOUT)				\
-	for (__init_wedge((W), (DEV), (TIMEOUT), __builtin_return_address(0)); \
-	     (W)->i915;							\
-	     __fini_wedge((W)))
-
-static noinline int
-flush_test(struct drm_i915_private *i915, unsigned int flags)
-{
-	struct wedge_me w;
-
-	cond_resched();
-
-	wedge_on_timeout(&w, i915, HZ)
-		i915_gem_wait_for_idle(i915, flags);
-
-	return i915_terminally_wedged(&i915->gpu_error) ? -EIO : 0;
-}
-
 static void hang_fini(struct hang *h)
 {
 	*h->batch = MI_BATCH_BUFFER_END;
@@ -315,10 +267,10 @@ static void hang_fini(struct hang *h)
 
 	kernel_context_close(h->ctx);
 
-	flush_test(h->i915, I915_WAIT_LOCKED);
+	igt_flush_test(h->i915, I915_WAIT_LOCKED);
 }
 
-static bool wait_for_hang(struct hang *h, struct i915_request *rq)
+static bool wait_until_running(struct hang *h, struct i915_request *rq)
 {
 	return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq),
 					       rq->fence.seqno),
@@ -433,7 +385,7 @@ static int igt_global_reset(void *arg)
 	mutex_lock(&i915->drm.struct_mutex);
 	reset_count = i915_reset_count(&i915->gpu_error);
 
-	i915_reset(i915, I915_RESET_QUIET);
+	i915_reset(i915, ALL_ENGINES, NULL);
 
 	if (i915_reset_count(&i915->gpu_error) == reset_count) {
 		pr_err("No GPU reset recorded!\n");
@@ -450,6 +402,11 @@ static int igt_global_reset(void *arg)
 	return err;
 }
 
+static bool wait_for_idle(struct intel_engine_cs *engine)
+{
+	return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0;
+}
+
 static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
 {
 	struct intel_engine_cs *engine;
@@ -477,12 +434,21 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
 		if (active && !intel_engine_can_store_dword(engine))
 			continue;
 
+		if (!wait_for_idle(engine)) {
+			pr_err("%s failed to idle before reset\n",
+			       engine->name);
+			err = -EIO;
+			break;
+		}
+
 		reset_count = i915_reset_count(&i915->gpu_error);
 		reset_engine_count = i915_reset_engine_count(&i915->gpu_error,
 							     engine);
 
 		set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
 		do {
+			u32 seqno = intel_engine_get_seqno(engine);
+
 			if (active) {
 				struct i915_request *rq;
 
@@ -498,7 +464,7 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
 				__i915_request_add(rq, true);
 				mutex_unlock(&i915->drm.struct_mutex);
 
-				if (!wait_for_hang(&h, rq)) {
+				if (!wait_until_running(&h, rq)) {
 					struct drm_printer p = drm_info_printer(i915->drm.dev);
 
 					pr_err("%s: Failed to start request %x, at %x\n",
@@ -511,14 +477,12 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
 					break;
 				}
 
+				GEM_BUG_ON(!rq->global_seqno);
+				seqno = rq->global_seqno - 1;
 				i915_request_put(rq);
 			}
 
-			engine->hangcheck.stalled = true;
-			engine->hangcheck.seqno =
-				intel_engine_get_seqno(engine);
-
-			err = i915_reset_engine(engine, I915_RESET_QUIET);
+			err = i915_reset_engine(engine, NULL);
 			if (err) {
 				pr_err("i915_reset_engine failed\n");
 				break;
@@ -539,14 +503,25 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
 				break;
 			}
 
-			engine->hangcheck.stalled = false;
+			if (!wait_for_idle(engine)) {
+				struct drm_printer p =
+					drm_info_printer(i915->drm.dev);
+
+				pr_err("%s failed to idle after reset\n",
+				       engine->name);
+				intel_engine_dump(engine, &p,
+						  "%s\n", engine->name);
+
+				err = -EIO;
+				break;
+			}
 		} while (time_before(jiffies, end_time));
 		clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
 
 		if (err)
 			break;
 
-		err = flush_test(i915, 0);
+		err = igt_flush_test(i915, 0);
 		if (err)
 			break;
 	}
@@ -573,11 +548,25 @@ static int igt_reset_active_engine(void *arg)
 	return __igt_reset_engine(arg, true);
 }
 
+struct active_engine {
+	struct task_struct *task;
+	struct intel_engine_cs *engine;
+	unsigned long resets;
+	unsigned int flags;
+};
+
+#define TEST_ACTIVE	BIT(0)
+#define TEST_OTHERS	BIT(1)
+#define TEST_SELF	BIT(2)
+#define TEST_PRIORITY	BIT(3)
+
 static int active_engine(void *data)
 {
-	struct intel_engine_cs *engine = data;
-	struct i915_request *rq[2] = {};
-	struct i915_gem_context *ctx[2];
+	I915_RND_STATE(prng);
+	struct active_engine *arg = data;
+	struct intel_engine_cs *engine = arg->engine;
+	struct i915_request *rq[8] = {};
+	struct i915_gem_context *ctx[ARRAY_SIZE(rq)];
 	struct drm_file *file;
 	unsigned long count = 0;
 	int err = 0;
@@ -586,25 +575,20 @@ static int active_engine(void *data)
 	if (IS_ERR(file))
 		return PTR_ERR(file);
 
-	mutex_lock(&engine->i915->drm.struct_mutex);
-	ctx[0] = live_context(engine->i915, file);
-	mutex_unlock(&engine->i915->drm.struct_mutex);
-	if (IS_ERR(ctx[0])) {
-		err = PTR_ERR(ctx[0]);
-		goto err_file;
-	}
-
-	mutex_lock(&engine->i915->drm.struct_mutex);
-	ctx[1] = live_context(engine->i915, file);
-	mutex_unlock(&engine->i915->drm.struct_mutex);
-	if (IS_ERR(ctx[1])) {
-		err = PTR_ERR(ctx[1]);
-		i915_gem_context_put(ctx[0]);
-		goto err_file;
+	for (count = 0; count < ARRAY_SIZE(ctx); count++) {
+		mutex_lock(&engine->i915->drm.struct_mutex);
+		ctx[count] = live_context(engine->i915, file);
+		mutex_unlock(&engine->i915->drm.struct_mutex);
+		if (IS_ERR(ctx[count])) {
+			err = PTR_ERR(ctx[count]);
+			while (--count)
+				i915_gem_context_put(ctx[count]);
+			goto err_file;
+		}
 	}
 
 	while (!kthread_should_stop()) {
-		unsigned int idx = count++ & 1;
+		unsigned int idx = count++ & (ARRAY_SIZE(rq) - 1);
 		struct i915_request *old = rq[idx];
 		struct i915_request *new;
 
@@ -616,14 +600,28 @@ static int active_engine(void *data)
 			break;
 		}
 
+		if (arg->flags & TEST_PRIORITY)
+			ctx[idx]->sched.priority =
+				i915_prandom_u32_max_state(512, &prng);
+
 		rq[idx] = i915_request_get(new);
 		i915_request_add(new);
 		mutex_unlock(&engine->i915->drm.struct_mutex);
 
 		if (old) {
-			i915_request_wait(old, 0, MAX_SCHEDULE_TIMEOUT);
+			if (i915_request_wait(old, 0, HZ) < 0) {
+				GEM_TRACE("%s timed out.\n", engine->name);
+				GEM_TRACE_DUMP();
+
+				i915_gem_set_wedged(engine->i915);
+				i915_request_put(old);
+				err = -EIO;
+				break;
+			}
 			i915_request_put(old);
 		}
+
+		cond_resched();
 	}
 
 	for (count = 0; count < ARRAY_SIZE(rq); count++)
@@ -634,8 +632,9 @@ static int active_engine(void *data)
 	return err;
 }
 
-static int __igt_reset_engine_others(struct drm_i915_private *i915,
-				     bool active)
+static int __igt_reset_engines(struct drm_i915_private *i915,
+			       const char *test_name,
+			       unsigned int flags)
 {
 	struct intel_engine_cs *engine, *other;
 	enum intel_engine_id id, tmp;
@@ -649,50 +648,68 @@ static int __igt_reset_engine_others(struct drm_i915_private *i915,
 	if (!intel_has_reset_engine(i915))
 		return 0;
 
-	if (active) {
+	if (flags & TEST_ACTIVE) {
 		mutex_lock(&i915->drm.struct_mutex);
 		err = hang_init(&h, i915);
 		mutex_unlock(&i915->drm.struct_mutex);
 		if (err)
 			return err;
+
+		if (flags & TEST_PRIORITY)
+			h.ctx->sched.priority = 1024;
 	}
 
 	for_each_engine(engine, i915, id) {
-		struct task_struct *threads[I915_NUM_ENGINES] = {};
-		unsigned long resets[I915_NUM_ENGINES];
+		struct active_engine threads[I915_NUM_ENGINES] = {};
 		unsigned long global = i915_reset_count(&i915->gpu_error);
-		unsigned long count = 0;
+		unsigned long count = 0, reported;
 		IGT_TIMEOUT(end_time);
 
-		if (active && !intel_engine_can_store_dword(engine))
+		if (flags & TEST_ACTIVE &&
+		    !intel_engine_can_store_dword(engine))
 			continue;
 
+		if (!wait_for_idle(engine)) {
+			pr_err("i915_reset_engine(%s:%s): failed to idle before reset\n",
+			       engine->name, test_name);
+			err = -EIO;
+			break;
+		}
+
 		memset(threads, 0, sizeof(threads));
 		for_each_engine(other, i915, tmp) {
 			struct task_struct *tsk;
 
-			resets[tmp] = i915_reset_engine_count(&i915->gpu_error,
-							      other);
+			threads[tmp].resets =
+				i915_reset_engine_count(&i915->gpu_error,
+							other);
+
+			if (!(flags & TEST_OTHERS))
+				continue;
 
-			if (other == engine)
+			if (other == engine && !(flags & TEST_SELF))
 				continue;
 
-			tsk = kthread_run(active_engine, other,
+			threads[tmp].engine = other;
+			threads[tmp].flags = flags;
+
+			tsk = kthread_run(active_engine, &threads[tmp],
 					  "igt/%s", other->name);
 			if (IS_ERR(tsk)) {
 				err = PTR_ERR(tsk);
 				goto unwind;
 			}
 
-			threads[tmp] = tsk;
+			threads[tmp].task = tsk;
 			get_task_struct(tsk);
 		}
 
 		set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
 		do {
-			if (active) {
-				struct i915_request *rq;
+			u32 seqno = intel_engine_get_seqno(engine);
+			struct i915_request *rq = NULL;
 
+			if (flags & TEST_ACTIVE) {
 				mutex_lock(&i915->drm.struct_mutex);
 				rq = hang_create_request(&h, engine);
 				if (IS_ERR(rq)) {
@@ -705,7 +722,7 @@ static int __igt_reset_engine_others(struct drm_i915_private *i915,
 				__i915_request_add(rq, true);
 				mutex_unlock(&i915->drm.struct_mutex);
 
-				if (!wait_for_hang(&h, rq)) {
+				if (!wait_until_running(&h, rq)) {
 					struct drm_printer p = drm_info_printer(i915->drm.dev);
 
 					pr_err("%s: Failed to start request %x, at %x\n",
@@ -718,33 +735,48 @@ static int __igt_reset_engine_others(struct drm_i915_private *i915,
 					break;
 				}
 
-				i915_request_put(rq);
+				GEM_BUG_ON(!rq->global_seqno);
+				seqno = rq->global_seqno - 1;
 			}
 
-			engine->hangcheck.stalled = true;
-			engine->hangcheck.seqno =
-				intel_engine_get_seqno(engine);
-
-			err = i915_reset_engine(engine, I915_RESET_QUIET);
+			err = i915_reset_engine(engine, NULL);
 			if (err) {
-				pr_err("i915_reset_engine(%s:%s) failed, err=%d\n",
-				       engine->name, active ? "active" : "idle", err);
+				pr_err("i915_reset_engine(%s:%s): failed, err=%d\n",
+				       engine->name, test_name, err);
 				break;
 			}
 
-			engine->hangcheck.stalled = false;
 			count++;
+
+			if (rq) {
+				i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
+				i915_request_put(rq);
+			}
+
+			if (!(flags & TEST_SELF) && !wait_for_idle(engine)) {
+				struct drm_printer p =
+					drm_info_printer(i915->drm.dev);
+
+				pr_err("i915_reset_engine(%s:%s):"
+				       " failed to idle after reset\n",
+				       engine->name, test_name);
+				intel_engine_dump(engine, &p,
+						  "%s\n", engine->name);
+
+				err = -EIO;
+				break;
+			}
 		} while (time_before(jiffies, end_time));
 		clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
 		pr_info("i915_reset_engine(%s:%s): %lu resets\n",
-			engine->name, active ? "active" : "idle", count);
-
-		if (i915_reset_engine_count(&i915->gpu_error, engine) -
-		    resets[engine->id] != (active ? count : 0)) {
-			pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n",
-			       engine->name, active ? "active" : "idle", count,
-			       i915_reset_engine_count(&i915->gpu_error,
-						       engine) - resets[engine->id]);
+			engine->name, test_name, count);
+
+		reported = i915_reset_engine_count(&i915->gpu_error, engine);
+		reported -= threads[engine->id].resets;
+		if (reported != (flags & TEST_ACTIVE ? count : 0)) {
+			pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu, expected %lu reported\n",
+			       engine->name, test_name, count, reported,
+			       (flags & TEST_ACTIVE ? count : 0));
 			if (!err)
 				err = -EINVAL;
 		}
@@ -753,24 +785,26 @@ static int __igt_reset_engine_others(struct drm_i915_private *i915,
 		for_each_engine(other, i915, tmp) {
 			int ret;
 
-			if (!threads[tmp])
+			if (!threads[tmp].task)
 				continue;
 
-			ret = kthread_stop(threads[tmp]);
+			ret = kthread_stop(threads[tmp].task);
 			if (ret) {
 				pr_err("kthread for other engine %s failed, err=%d\n",
 				       other->name, ret);
 				if (!err)
 					err = ret;
 			}
-			put_task_struct(threads[tmp]);
+			put_task_struct(threads[tmp].task);
 
-			if (resets[tmp] != i915_reset_engine_count(&i915->gpu_error,
-								   other)) {
+			if (other != engine &&
+			    threads[tmp].resets !=
+			    i915_reset_engine_count(&i915->gpu_error, other)) {
 				pr_err("Innocent engine %s was reset (count=%ld)\n",
 				       other->name,
 				       i915_reset_engine_count(&i915->gpu_error,
-							       other) - resets[tmp]);
+							       other) -
+				       threads[tmp].resets);
 				if (!err)
 					err = -EINVAL;
 			}
@@ -786,7 +820,7 @@ static int __igt_reset_engine_others(struct drm_i915_private *i915,
 		if (err)
 			break;
 
-		err = flush_test(i915, 0);
+		err = igt_flush_test(i915, 0);
 		if (err)
 			break;
 	}
@@ -794,7 +828,7 @@ static int __igt_reset_engine_others(struct drm_i915_private *i915,
 	if (i915_terminally_wedged(&i915->gpu_error))
 		err = -EIO;
 
-	if (active) {
+	if (flags & TEST_ACTIVE) {
 		mutex_lock(&i915->drm.struct_mutex);
 		hang_fini(&h);
 		mutex_unlock(&i915->drm.struct_mutex);
@@ -803,27 +837,56 @@ static int __igt_reset_engine_others(struct drm_i915_private *i915,
 	return err;
 }
 
-static int igt_reset_idle_engine_others(void *arg)
+static int igt_reset_engines(void *arg)
 {
-	return __igt_reset_engine_others(arg, false);
-}
+	static const struct {
+		const char *name;
+		unsigned int flags;
+	} phases[] = {
+		{ "idle", 0 },
+		{ "active", TEST_ACTIVE },
+		{ "others-idle", TEST_OTHERS },
+		{ "others-active", TEST_OTHERS | TEST_ACTIVE },
+		{
+			"others-priority",
+			TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY
+		},
+		{
+			"self-priority",
+			TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY | TEST_SELF,
+		},
+		{ }
+	};
+	struct drm_i915_private *i915 = arg;
+	typeof(*phases) *p;
+	int err;
 
-static int igt_reset_active_engine_others(void *arg)
-{
-	return __igt_reset_engine_others(arg, true);
+	for (p = phases; p->name; p++) {
+		if (p->flags & TEST_PRIORITY) {
+			if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY))
+				continue;
+		}
+
+		err = __igt_reset_engines(arg, p->name, p->flags);
+		if (err)
+			return err;
+	}
+
+	return 0;
 }
 
-static u32 fake_hangcheck(struct i915_request *rq)
+static u32 fake_hangcheck(struct i915_request *rq, u32 mask)
 {
-	u32 reset_count;
+	struct i915_gpu_error *error = &rq->i915->gpu_error;
+	u32 reset_count = i915_reset_count(error);
 
-	rq->engine->hangcheck.stalled = true;
-	rq->engine->hangcheck.seqno = intel_engine_get_seqno(rq->engine);
+	error->stalled_mask = mask;
 
-	reset_count = i915_reset_count(&rq->i915->gpu_error);
+	/* set_bit() must be after we have setup the backchannel (mask) */
+	smp_mb__before_atomic();
+	set_bit(I915_RESET_HANDOFF, &error->flags);
 
-	set_bit(I915_RESET_HANDOFF, &rq->i915->gpu_error.flags);
-	wake_up_all(&rq->i915->gpu_error.wait_queue);
+	wake_up_all(&error->wait_queue);
 
 	return reset_count;
 }
@@ -858,21 +921,20 @@ static int igt_wait_reset(void *arg)
 	i915_request_get(rq);
 	__i915_request_add(rq, true);
 
-	if (!wait_for_hang(&h, rq)) {
+	if (!wait_until_running(&h, rq)) {
 		struct drm_printer p = drm_info_printer(i915->drm.dev);
 
 		pr_err("%s: Failed to start request %x, at %x\n",
 		       __func__, rq->fence.seqno, hws_seqno(&h, rq));
 		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
 
-		i915_reset(i915, 0);
 		i915_gem_set_wedged(i915);
 
 		err = -EIO;
 		goto out_rq;
 	}
 
-	reset_count = fake_hangcheck(rq);
+	reset_count = fake_hangcheck(rq, ALL_ENGINES);
 
 	timeout = i915_request_wait(rq, I915_WAIT_LOCKED, 10);
 	if (timeout < 0) {
@@ -903,6 +965,23 @@ static int igt_wait_reset(void *arg)
 	return err;
 }
 
+static int wait_for_others(struct drm_i915_private *i915,
+			   struct intel_engine_cs *exclude)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	for_each_engine(engine, i915, id) {
+		if (engine == exclude)
+			continue;
+
+		if (!wait_for_idle(engine))
+			return -EIO;
+	}
+
+	return 0;
+}
+
 static int igt_reset_queue(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
@@ -951,27 +1030,49 @@ static int igt_reset_queue(void *arg)
 			i915_request_get(rq);
 			__i915_request_add(rq, true);
 
-			if (!wait_for_hang(&h, prev)) {
+			/*
+			 * XXX We don't handle resetting the kernel context
+			 * very well. If we trigger a device reset twice in
+			 * quick succession while the kernel context is
+			 * executing, we may end up skipping the breadcrumb.
+			 * This is really only a problem for the selftest as
+			 * normally there is a large interlude between resets
+			 * (hangcheck), or we focus on resetting just one
+			 * engine and so avoid repeatedly resetting innocents.
+			 */
+			err = wait_for_others(i915, engine);
+			if (err) {
+				pr_err("%s(%s): Failed to idle other inactive engines after device reset\n",
+				       __func__, engine->name);
+				i915_request_put(rq);
+				i915_request_put(prev);
+
+				GEM_TRACE_DUMP();
+				i915_gem_set_wedged(i915);
+				goto fini;
+			}
+
+			if (!wait_until_running(&h, prev)) {
 				struct drm_printer p = drm_info_printer(i915->drm.dev);
 
-				pr_err("%s: Failed to start request %x, at %x\n",
-				       __func__, prev->fence.seqno, hws_seqno(&h, prev));
-				intel_engine_dump(prev->engine, &p,
-						  "%s\n", prev->engine->name);
+				pr_err("%s(%s): Failed to start request %x, at %x\n",
+				       __func__, engine->name,
+				       prev->fence.seqno, hws_seqno(&h, prev));
+				intel_engine_dump(engine, &p,
+						  "%s\n", engine->name);
 
 				i915_request_put(rq);
 				i915_request_put(prev);
 
-				i915_reset(i915, 0);
 				i915_gem_set_wedged(i915);
 
 				err = -EIO;
 				goto fini;
 			}
 
-			reset_count = fake_hangcheck(prev);
+			reset_count = fake_hangcheck(prev, ENGINE_MASK(id));
 
-			i915_reset(i915, I915_RESET_QUIET);
+			i915_reset(i915, ENGINE_MASK(id), NULL);
 
 			GEM_BUG_ON(test_bit(I915_RESET_HANDOFF,
 					    &i915->gpu_error.flags));
@@ -1013,7 +1114,7 @@ static int igt_reset_queue(void *arg)
 
 		i915_request_put(prev);
 
-		err = flush_test(i915, I915_WAIT_LOCKED);
+		err = igt_flush_test(i915, I915_WAIT_LOCKED);
 		if (err)
 			break;
 	}
@@ -1044,7 +1145,7 @@ static int igt_handle_error(void *arg)
 	if (!intel_has_reset_engine(i915))
 		return 0;
 
-	if (!intel_engine_can_store_dword(i915->engine[RCS]))
+	if (!engine || !intel_engine_can_store_dword(engine))
 		return 0;
 
 	mutex_lock(&i915->drm.struct_mutex);
@@ -1062,14 +1163,13 @@ static int igt_handle_error(void *arg)
 	i915_request_get(rq);
 	__i915_request_add(rq, true);
 
-	if (!wait_for_hang(&h, rq)) {
+	if (!wait_until_running(&h, rq)) {
 		struct drm_printer p = drm_info_printer(i915->drm.dev);
 
 		pr_err("%s: Failed to start request %x, at %x\n",
 		       __func__, rq->fence.seqno, hws_seqno(&h, rq));
 		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
 
-		i915_reset(i915, 0);
 		i915_gem_set_wedged(i915);
 
 		err = -EIO;
@@ -1081,10 +1181,7 @@ static int igt_handle_error(void *arg)
 	/* Temporarily disable error capture */
 	error = xchg(&i915->gpu_error.first_error, (void *)-1);
 
-	engine->hangcheck.stalled = true;
-	engine->hangcheck.seqno = intel_engine_get_seqno(engine);
-
-	i915_handle_error(i915, intel_engine_flag(engine), "%s", __func__);
+	i915_handle_error(i915, ENGINE_MASK(engine->id), 0, NULL);
 
 	xchg(&i915->gpu_error.first_error, error);
 
@@ -1112,8 +1209,7 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_hang_sanitycheck),
 		SUBTEST(igt_reset_idle_engine),
 		SUBTEST(igt_reset_active_engine),
-		SUBTEST(igt_reset_idle_engine_others),
-		SUBTEST(igt_reset_active_engine_others),
+		SUBTEST(igt_reset_engines),
 		SUBTEST(igt_wait_reset),
 		SUBTEST(igt_reset_queue),
 		SUBTEST(igt_handle_error),
@@ -1129,6 +1225,10 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
 
 	err = i915_subtests(tests, i915);
 
+	mutex_lock(&i915->drm.struct_mutex);
+	igt_flush_test(i915, I915_WAIT_LOCKED);
+	mutex_unlock(&i915->drm.struct_mutex);
+
 	i915_modparams.enable_hangcheck = saved_hangcheck;
 	intel_runtime_pm_put(i915);
 
diff --git a/drivers/gpu/drm/i915/selftests/intel_lrc.c b/drivers/gpu/drm/i915/selftests/intel_lrc.c
new file mode 100644
index 000000000000..1b8a07125150
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/intel_lrc.c
@@ -0,0 +1,459 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2018 Intel Corporation
+ */
+
+#include "../i915_selftest.h"
+#include "igt_flush_test.h"
+
+#include "mock_context.h"
+
+struct spinner {
+	struct drm_i915_private *i915;
+	struct drm_i915_gem_object *hws;
+	struct drm_i915_gem_object *obj;
+	u32 *batch;
+	void *seqno;
+};
+
+static int spinner_init(struct spinner *spin, struct drm_i915_private *i915)
+{
+	unsigned int mode;
+	void *vaddr;
+	int err;
+
+	GEM_BUG_ON(INTEL_GEN(i915) < 8);
+
+	memset(spin, 0, sizeof(*spin));
+	spin->i915 = i915;
+
+	spin->hws = i915_gem_object_create_internal(i915, PAGE_SIZE);
+	if (IS_ERR(spin->hws)) {
+		err = PTR_ERR(spin->hws);
+		goto err;
+	}
+
+	spin->obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
+	if (IS_ERR(spin->obj)) {
+		err = PTR_ERR(spin->obj);
+		goto err_hws;
+	}
+
+	i915_gem_object_set_cache_level(spin->hws, I915_CACHE_LLC);
+	vaddr = i915_gem_object_pin_map(spin->hws, I915_MAP_WB);
+	if (IS_ERR(vaddr)) {
+		err = PTR_ERR(vaddr);
+		goto err_obj;
+	}
+	spin->seqno = memset(vaddr, 0xff, PAGE_SIZE);
+
+	mode = HAS_LLC(i915) ? I915_MAP_WB : I915_MAP_WC;
+	vaddr = i915_gem_object_pin_map(spin->obj, mode);
+	if (IS_ERR(vaddr)) {
+		err = PTR_ERR(vaddr);
+		goto err_unpin_hws;
+	}
+	spin->batch = vaddr;
+
+	return 0;
+
+err_unpin_hws:
+	i915_gem_object_unpin_map(spin->hws);
+err_obj:
+	i915_gem_object_put(spin->obj);
+err_hws:
+	i915_gem_object_put(spin->hws);
+err:
+	return err;
+}
+
+static unsigned int seqno_offset(u64 fence)
+{
+	return offset_in_page(sizeof(u32) * fence);
+}
+
+static u64 hws_address(const struct i915_vma *hws,
+		       const struct i915_request *rq)
+{
+	return hws->node.start + seqno_offset(rq->fence.context);
+}
+
+static int emit_recurse_batch(struct spinner *spin,
+			      struct i915_request *rq,
+			      u32 arbitration_command)
+{
+	struct i915_address_space *vm = &rq->ctx->ppgtt->base;
+	struct i915_vma *hws, *vma;
+	u32 *batch;
+	int err;
+
+	vma = i915_vma_instance(spin->obj, vm, NULL);
+	if (IS_ERR(vma))
+		return PTR_ERR(vma);
+
+	hws = i915_vma_instance(spin->hws, vm, NULL);
+	if (IS_ERR(hws))
+		return PTR_ERR(hws);
+
+	err = i915_vma_pin(vma, 0, 0, PIN_USER);
+	if (err)
+		return err;
+
+	err = i915_vma_pin(hws, 0, 0, PIN_USER);
+	if (err)
+		goto unpin_vma;
+
+	i915_vma_move_to_active(vma, rq, 0);
+	if (!i915_gem_object_has_active_reference(vma->obj)) {
+		i915_gem_object_get(vma->obj);
+		i915_gem_object_set_active_reference(vma->obj);
+	}
+
+	i915_vma_move_to_active(hws, rq, 0);
+	if (!i915_gem_object_has_active_reference(hws->obj)) {
+		i915_gem_object_get(hws->obj);
+		i915_gem_object_set_active_reference(hws->obj);
+	}
+
+	batch = spin->batch;
+
+	*batch++ = MI_STORE_DWORD_IMM_GEN4;
+	*batch++ = lower_32_bits(hws_address(hws, rq));
+	*batch++ = upper_32_bits(hws_address(hws, rq));
+	*batch++ = rq->fence.seqno;
+
+	*batch++ = arbitration_command;
+
+	*batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
+	*batch++ = lower_32_bits(vma->node.start);
+	*batch++ = upper_32_bits(vma->node.start);
+	*batch++ = MI_BATCH_BUFFER_END; /* not reached */
+
+	i915_gem_chipset_flush(spin->i915);
+
+	err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, 0);
+
+	i915_vma_unpin(hws);
+unpin_vma:
+	i915_vma_unpin(vma);
+	return err;
+}
+
+static struct i915_request *
+spinner_create_request(struct spinner *spin,
+		       struct i915_gem_context *ctx,
+		       struct intel_engine_cs *engine,
+		       u32 arbitration_command)
+{
+	struct i915_request *rq;
+	int err;
+
+	rq = i915_request_alloc(engine, ctx);
+	if (IS_ERR(rq))
+		return rq;
+
+	err = emit_recurse_batch(spin, rq, arbitration_command);
+	if (err) {
+		__i915_request_add(rq, false);
+		return ERR_PTR(err);
+	}
+
+	return rq;
+}
+
+static u32 hws_seqno(const struct spinner *spin, const struct i915_request *rq)
+{
+	u32 *seqno = spin->seqno + seqno_offset(rq->fence.context);
+
+	return READ_ONCE(*seqno);
+}
+
+static void spinner_end(struct spinner *spin)
+{
+	*spin->batch = MI_BATCH_BUFFER_END;
+	i915_gem_chipset_flush(spin->i915);
+}
+
+static void spinner_fini(struct spinner *spin)
+{
+	spinner_end(spin);
+
+	i915_gem_object_unpin_map(spin->obj);
+	i915_gem_object_put(spin->obj);
+
+	i915_gem_object_unpin_map(spin->hws);
+	i915_gem_object_put(spin->hws);
+}
+
+static bool wait_for_spinner(struct spinner *spin, struct i915_request *rq)
+{
+	if (!wait_event_timeout(rq->execute,
+				READ_ONCE(rq->global_seqno),
+				msecs_to_jiffies(10)))
+		return false;
+
+	return !(wait_for_us(i915_seqno_passed(hws_seqno(spin, rq),
+					       rq->fence.seqno),
+			     10) &&
+		 wait_for(i915_seqno_passed(hws_seqno(spin, rq),
+					    rq->fence.seqno),
+			  1000));
+}
+
+static int live_sanitycheck(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct intel_engine_cs *engine;
+	struct i915_gem_context *ctx;
+	enum intel_engine_id id;
+	struct spinner spin;
+	int err = -ENOMEM;
+
+	if (!HAS_LOGICAL_RING_CONTEXTS(i915))
+		return 0;
+
+	mutex_lock(&i915->drm.struct_mutex);
+
+	if (spinner_init(&spin, i915))
+		goto err_unlock;
+
+	ctx = kernel_context(i915);
+	if (!ctx)
+		goto err_spin;
+
+	for_each_engine(engine, i915, id) {
+		struct i915_request *rq;
+
+		rq = spinner_create_request(&spin, ctx, engine, MI_NOOP);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_ctx;
+		}
+
+		i915_request_add(rq);
+		if (!wait_for_spinner(&spin, rq)) {
+			GEM_TRACE("spinner failed to start\n");
+			GEM_TRACE_DUMP();
+			i915_gem_set_wedged(i915);
+			err = -EIO;
+			goto err_ctx;
+		}
+
+		spinner_end(&spin);
+		if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
+			err = -EIO;
+			goto err_ctx;
+		}
+	}
+
+	err = 0;
+err_ctx:
+	kernel_context_close(ctx);
+err_spin:
+	spinner_fini(&spin);
+err_unlock:
+	igt_flush_test(i915, I915_WAIT_LOCKED);
+	mutex_unlock(&i915->drm.struct_mutex);
+	return err;
+}
+
+static int live_preempt(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct i915_gem_context *ctx_hi, *ctx_lo;
+	struct spinner spin_hi, spin_lo;
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	int err = -ENOMEM;
+
+	if (!HAS_LOGICAL_RING_PREEMPTION(i915))
+		return 0;
+
+	mutex_lock(&i915->drm.struct_mutex);
+
+	if (spinner_init(&spin_hi, i915))
+		goto err_unlock;
+
+	if (spinner_init(&spin_lo, i915))
+		goto err_spin_hi;
+
+	ctx_hi = kernel_context(i915);
+	if (!ctx_hi)
+		goto err_spin_lo;
+	ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
+
+	ctx_lo = kernel_context(i915);
+	if (!ctx_lo)
+		goto err_ctx_hi;
+	ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
+
+	for_each_engine(engine, i915, id) {
+		struct i915_request *rq;
+
+		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
+					    MI_ARB_CHECK);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_ctx_lo;
+		}
+
+		i915_request_add(rq);
+		if (!wait_for_spinner(&spin_lo, rq)) {
+			GEM_TRACE("lo spinner failed to start\n");
+			GEM_TRACE_DUMP();
+			i915_gem_set_wedged(i915);
+			err = -EIO;
+			goto err_ctx_lo;
+		}
+
+		rq = spinner_create_request(&spin_hi, ctx_hi, engine,
+					    MI_ARB_CHECK);
+		if (IS_ERR(rq)) {
+			spinner_end(&spin_lo);
+			err = PTR_ERR(rq);
+			goto err_ctx_lo;
+		}
+
+		i915_request_add(rq);
+		if (!wait_for_spinner(&spin_hi, rq)) {
+			GEM_TRACE("hi spinner failed to start\n");
+			GEM_TRACE_DUMP();
+			i915_gem_set_wedged(i915);
+			err = -EIO;
+			goto err_ctx_lo;
+		}
+
+		spinner_end(&spin_hi);
+		spinner_end(&spin_lo);
+		if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
+			err = -EIO;
+			goto err_ctx_lo;
+		}
+	}
+
+	err = 0;
+err_ctx_lo:
+	kernel_context_close(ctx_lo);
+err_ctx_hi:
+	kernel_context_close(ctx_hi);
+err_spin_lo:
+	spinner_fini(&spin_lo);
+err_spin_hi:
+	spinner_fini(&spin_hi);
+err_unlock:
+	igt_flush_test(i915, I915_WAIT_LOCKED);
+	mutex_unlock(&i915->drm.struct_mutex);
+	return err;
+}
+
+static int live_late_preempt(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct i915_gem_context *ctx_hi, *ctx_lo;
+	struct spinner spin_hi, spin_lo;
+	struct intel_engine_cs *engine;
+	struct i915_sched_attr attr = {};
+	enum intel_engine_id id;
+	int err = -ENOMEM;
+
+	if (!HAS_LOGICAL_RING_PREEMPTION(i915))
+		return 0;
+
+	mutex_lock(&i915->drm.struct_mutex);
+
+	if (spinner_init(&spin_hi, i915))
+		goto err_unlock;
+
+	if (spinner_init(&spin_lo, i915))
+		goto err_spin_hi;
+
+	ctx_hi = kernel_context(i915);
+	if (!ctx_hi)
+		goto err_spin_lo;
+
+	ctx_lo = kernel_context(i915);
+	if (!ctx_lo)
+		goto err_ctx_hi;
+
+	for_each_engine(engine, i915, id) {
+		struct i915_request *rq;
+
+		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
+					    MI_ARB_CHECK);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_ctx_lo;
+		}
+
+		i915_request_add(rq);
+		if (!wait_for_spinner(&spin_lo, rq)) {
+			pr_err("First context failed to start\n");
+			goto err_wedged;
+		}
+
+		rq = spinner_create_request(&spin_hi, ctx_hi, engine, MI_NOOP);
+		if (IS_ERR(rq)) {
+			spinner_end(&spin_lo);
+			err = PTR_ERR(rq);
+			goto err_ctx_lo;
+		}
+
+		i915_request_add(rq);
+		if (wait_for_spinner(&spin_hi, rq)) {
+			pr_err("Second context overtook first?\n");
+			goto err_wedged;
+		}
+
+		attr.priority = I915_PRIORITY_MAX;
+		engine->schedule(rq, &attr);
+
+		if (!wait_for_spinner(&spin_hi, rq)) {
+			pr_err("High priority context failed to preempt the low priority context\n");
+			GEM_TRACE_DUMP();
+			goto err_wedged;
+		}
+
+		spinner_end(&spin_hi);
+		spinner_end(&spin_lo);
+		if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
+			err = -EIO;
+			goto err_ctx_lo;
+		}
+	}
+
+	err = 0;
+err_ctx_lo:
+	kernel_context_close(ctx_lo);
+err_ctx_hi:
+	kernel_context_close(ctx_hi);
+err_spin_lo:
+	spinner_fini(&spin_lo);
+err_spin_hi:
+	spinner_fini(&spin_hi);
+err_unlock:
+	igt_flush_test(i915, I915_WAIT_LOCKED);
+	mutex_unlock(&i915->drm.struct_mutex);
+	return err;
+
+err_wedged:
+	spinner_end(&spin_hi);
+	spinner_end(&spin_lo);
+	i915_gem_set_wedged(i915);
+	err = -EIO;
+	goto err_ctx_lo;
+}
+
+int intel_execlists_live_selftests(struct drm_i915_private *i915)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(live_sanitycheck),
+		SUBTEST(live_preempt),
+		SUBTEST(live_late_preempt),
+	};
+
+	if (!HAS_EXECLISTS(i915))
+		return 0;
+
+	return i915_subtests(tests, i915);
+}
diff --git a/drivers/gpu/drm/i915/selftests/intel_workarounds.c b/drivers/gpu/drm/i915/selftests/intel_workarounds.c
new file mode 100644
index 000000000000..17444a3abbb9
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/intel_workarounds.c
@@ -0,0 +1,291 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2018 Intel Corporation
+ */
+
+#include "../i915_selftest.h"
+
+#include "mock_context.h"
+
+static struct drm_i915_gem_object *
+read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
+{
+	struct drm_i915_gem_object *result;
+	struct i915_request *rq;
+	struct i915_vma *vma;
+	const u32 base = engine->mmio_base;
+	u32 srm, *cs;
+	int err;
+	int i;
+
+	result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
+	if (IS_ERR(result))
+		return result;
+
+	i915_gem_object_set_cache_level(result, I915_CACHE_LLC);
+
+	cs = i915_gem_object_pin_map(result, I915_MAP_WB);
+	if (IS_ERR(cs)) {
+		err = PTR_ERR(cs);
+		goto err_obj;
+	}
+	memset(cs, 0xc5, PAGE_SIZE);
+	i915_gem_object_unpin_map(result);
+
+	vma = i915_vma_instance(result, &engine->i915->ggtt.base, NULL);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto err_obj;
+	}
+
+	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
+	if (err)
+		goto err_obj;
+
+	rq = i915_request_alloc(engine, ctx);
+	if (IS_ERR(rq)) {
+		err = PTR_ERR(rq);
+		goto err_pin;
+	}
+
+	srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
+	if (INTEL_GEN(ctx->i915) >= 8)
+		srm++;
+
+	cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
+	if (IS_ERR(cs)) {
+		err = PTR_ERR(cs);
+		goto err_req;
+	}
+
+	for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
+		*cs++ = srm;
+		*cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
+		*cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
+		*cs++ = 0;
+	}
+	intel_ring_advance(rq, cs);
+
+	i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+	reservation_object_lock(vma->resv, NULL);
+	reservation_object_add_excl_fence(vma->resv, &rq->fence);
+	reservation_object_unlock(vma->resv);
+
+	i915_gem_object_get(result);
+	i915_gem_object_set_active_reference(result);
+
+	__i915_request_add(rq, true);
+	i915_vma_unpin(vma);
+
+	return result;
+
+err_req:
+	i915_request_add(rq);
+err_pin:
+	i915_vma_unpin(vma);
+err_obj:
+	i915_gem_object_put(result);
+	return ERR_PTR(err);
+}
+
+static u32 get_whitelist_reg(const struct whitelist *w, unsigned int i)
+{
+	return i < w->count ? i915_mmio_reg_offset(w->reg[i]) : w->nopid;
+}
+
+static void print_results(const struct whitelist *w, const u32 *results)
+{
+	unsigned int i;
+
+	for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
+		u32 expected = get_whitelist_reg(w, i);
+		u32 actual = results[i];
+
+		pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
+			i, expected, actual);
+	}
+}
+
+static int check_whitelist(const struct whitelist *w,
+			   struct i915_gem_context *ctx,
+			   struct intel_engine_cs *engine)
+{
+	struct drm_i915_gem_object *results;
+	u32 *vaddr;
+	int err;
+	int i;
+
+	results = read_nonprivs(ctx, engine);
+	if (IS_ERR(results))
+		return PTR_ERR(results);
+
+	err = i915_gem_object_set_to_cpu_domain(results, false);
+	if (err)
+		goto out_put;
+
+	vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
+	if (IS_ERR(vaddr)) {
+		err = PTR_ERR(vaddr);
+		goto out_put;
+	}
+
+	for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
+		u32 expected = get_whitelist_reg(w, i);
+		u32 actual = vaddr[i];
+
+		if (expected != actual) {
+			print_results(w, vaddr);
+			pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
+			       i, expected, actual);
+
+			err = -EINVAL;
+			break;
+		}
+	}
+
+	i915_gem_object_unpin_map(results);
+out_put:
+	i915_gem_object_put(results);
+	return err;
+}
+
+static int do_device_reset(struct intel_engine_cs *engine)
+{
+	i915_reset(engine->i915, ENGINE_MASK(engine->id), NULL);
+	return 0;
+}
+
+static int do_engine_reset(struct intel_engine_cs *engine)
+{
+	return i915_reset_engine(engine, NULL);
+}
+
+static int switch_to_scratch_context(struct intel_engine_cs *engine)
+{
+	struct i915_gem_context *ctx;
+	struct i915_request *rq;
+
+	ctx = kernel_context(engine->i915);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	rq = i915_request_alloc(engine, ctx);
+	kernel_context_close(ctx);
+	if (IS_ERR(rq))
+		return PTR_ERR(rq);
+
+	i915_request_add(rq);
+
+	return 0;
+}
+
+static int check_whitelist_across_reset(struct intel_engine_cs *engine,
+					int (*reset)(struct intel_engine_cs *),
+					const struct whitelist *w,
+					const char *name)
+{
+	struct i915_gem_context *ctx;
+	int err;
+
+	ctx = kernel_context(engine->i915);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	err = check_whitelist(w, ctx, engine);
+	if (err) {
+		pr_err("Invalid whitelist *before* %s reset!\n", name);
+		goto out;
+	}
+
+	err = switch_to_scratch_context(engine);
+	if (err)
+		goto out;
+
+	err = reset(engine);
+	if (err) {
+		pr_err("%s reset failed\n", name);
+		goto out;
+	}
+
+	err = check_whitelist(w, ctx, engine);
+	if (err) {
+		pr_err("Whitelist not preserved in context across %s reset!\n",
+		       name);
+		goto out;
+	}
+
+	kernel_context_close(ctx);
+
+	ctx = kernel_context(engine->i915);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	err = check_whitelist(w, ctx, engine);
+	if (err) {
+		pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
+		       name);
+		goto out;
+	}
+
+out:
+	kernel_context_close(ctx);
+	return err;
+}
+
+static int live_reset_whitelist(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct intel_engine_cs *engine = i915->engine[RCS];
+	struct i915_gpu_error *error = &i915->gpu_error;
+	struct whitelist w;
+	int err = 0;
+
+	/* If we reset the gpu, we should not lose the RING_NONPRIV */
+
+	if (!engine)
+		return 0;
+
+	if (!whitelist_build(engine, &w))
+		return 0;
+
+	pr_info("Checking %d whitelisted registers (RING_NONPRIV)\n", w.count);
+
+	set_bit(I915_RESET_BACKOFF, &error->flags);
+	set_bit(I915_RESET_ENGINE + engine->id, &error->flags);
+
+	if (intel_has_reset_engine(i915)) {
+		err = check_whitelist_across_reset(engine,
+						   do_engine_reset, &w,
+						   "engine");
+		if (err)
+			goto out;
+	}
+
+	if (intel_has_gpu_reset(i915)) {
+		err = check_whitelist_across_reset(engine,
+						   do_device_reset, &w,
+						   "device");
+		if (err)
+			goto out;
+	}
+
+out:
+	clear_bit(I915_RESET_ENGINE + engine->id, &error->flags);
+	clear_bit(I915_RESET_BACKOFF, &error->flags);
+	return err;
+}
+
+int intel_workarounds_live_selftests(struct drm_i915_private *i915)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(live_reset_whitelist),
+	};
+	int err;
+
+	mutex_lock(&i915->drm.struct_mutex);
+	err = i915_subtests(tests, i915);
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	return err;
+}
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c
index 78a89efa1119..26bf29d97007 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.c
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.c
@@ -25,6 +25,11 @@
 #include "mock_engine.h"
 #include "mock_request.h"
 
+struct mock_ring {
+	struct intel_ring base;
+	struct i915_timeline timeline;
+};
+
 static struct mock_request *first_request(struct mock_engine *engine)
 {
 	return list_first_entry_or_null(&engine->hw_queue,
@@ -71,14 +76,21 @@ static struct intel_ring *
 mock_context_pin(struct intel_engine_cs *engine,
 		 struct i915_gem_context *ctx)
 {
-	i915_gem_context_get(ctx);
+	struct intel_context *ce = to_intel_context(ctx, engine);
+
+	if (!ce->pin_count++)
+		i915_gem_context_get(ctx);
+
 	return engine->buffer;
 }
 
 static void mock_context_unpin(struct intel_engine_cs *engine,
 			       struct i915_gem_context *ctx)
 {
-	i915_gem_context_put(ctx);
+	struct intel_context *ce = to_intel_context(ctx, engine);
+
+	if (!--ce->pin_count)
+		i915_gem_context_put(ctx);
 }
 
 static int mock_request_alloc(struct i915_request *request)
@@ -125,7 +137,7 @@ static void mock_submit_request(struct i915_request *request)
 static struct intel_ring *mock_ring(struct intel_engine_cs *engine)
 {
 	const unsigned long sz = PAGE_SIZE / 2;
-	struct intel_ring *ring;
+	struct mock_ring *ring;
 
 	BUILD_BUG_ON(MIN_SPACE_FOR_ADD_REQUEST > sz);
 
@@ -133,14 +145,25 @@ static struct intel_ring *mock_ring(struct intel_engine_cs *engine)
 	if (!ring)
 		return NULL;
 
-	ring->size = sz;
-	ring->effective_size = sz;
-	ring->vaddr = (void *)(ring + 1);
+	i915_timeline_init(engine->i915, &ring->timeline, engine->name);
+
+	ring->base.size = sz;
+	ring->base.effective_size = sz;
+	ring->base.vaddr = (void *)(ring + 1);
+	ring->base.timeline = &ring->timeline;
 
-	INIT_LIST_HEAD(&ring->request_list);
-	intel_ring_update_space(ring);
+	INIT_LIST_HEAD(&ring->base.request_list);
+	intel_ring_update_space(&ring->base);
 
-	return ring;
+	return &ring->base;
+}
+
+static void mock_ring_free(struct intel_ring *base)
+{
+	struct mock_ring *ring = container_of(base, typeof(*ring), base);
+
+	i915_timeline_fini(&ring->timeline);
+	kfree(ring);
 }
 
 struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
@@ -155,12 +178,6 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
 	if (!engine)
 		return NULL;
 
-	engine->base.buffer = mock_ring(&engine->base);
-	if (!engine->base.buffer) {
-		kfree(engine);
-		return NULL;
-	}
-
 	/* minimal engine setup for requests */
 	engine->base.i915 = i915;
 	snprintf(engine->base.name, sizeof(engine->base.name), "%s", name);
@@ -174,9 +191,7 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
 	engine->base.emit_breadcrumb = mock_emit_breadcrumb;
 	engine->base.submit_request = mock_submit_request;
 
-	engine->base.timeline =
-		&i915->gt.global_timeline.engine[engine->base.id];
-
+	i915_timeline_init(i915, &engine->base.timeline, engine->base.name);
 	intel_engine_init_breadcrumbs(&engine->base);
 	engine->base.breadcrumbs.mock = true; /* prevent touching HW for irqs */
 
@@ -185,7 +200,17 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
 	timer_setup(&engine->hw_delay, hw_delay_complete, 0);
 	INIT_LIST_HEAD(&engine->hw_queue);
 
+	engine->base.buffer = mock_ring(&engine->base);
+	if (!engine->base.buffer)
+		goto err_breadcrumbs;
+
 	return &engine->base;
+
+err_breadcrumbs:
+	intel_engine_fini_breadcrumbs(&engine->base);
+	i915_timeline_fini(&engine->base.timeline);
+	kfree(engine);
+	return NULL;
 }
 
 void mock_engine_flush(struct intel_engine_cs *engine)
@@ -217,10 +242,12 @@ void mock_engine_free(struct intel_engine_cs *engine)
 	GEM_BUG_ON(timer_pending(&mock->hw_delay));
 
 	if (engine->last_retired_context)
-		engine->context_unpin(engine, engine->last_retired_context);
+		intel_context_unpin(engine->last_retired_context, engine);
+
+	mock_ring_free(engine->buffer);
 
 	intel_engine_fini_breadcrumbs(engine);
+	i915_timeline_fini(&engine->timeline);
 
-	kfree(engine->buffer);
 	kfree(engine);
 }
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index e6d4b882599a..94baedfa0f74 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -44,6 +44,7 @@ void mock_device_flush(struct drm_i915_private *i915)
 		mock_engine_flush(engine);
 
 	i915_retire_requests(i915);
+	GEM_BUG_ON(i915->gt.active_requests);
 }
 
 static void mock_device_release(struct drm_device *dev)
@@ -72,8 +73,8 @@ static void mock_device_release(struct drm_device *dev)
 
 	mutex_lock(&i915->drm.struct_mutex);
 	mock_fini_ggtt(i915);
-	i915_gem_timeline_fini(&i915->gt.global_timeline);
 	mutex_unlock(&i915->drm.struct_mutex);
+	WARN_ON(!list_empty(&i915->gt.timelines));
 
 	destroy_workqueue(i915->wq);
 
@@ -223,26 +224,25 @@ struct drm_i915_private *mock_gem_device(void)
 	if (!i915->priorities)
 		goto err_dependencies;
 
-	mutex_lock(&i915->drm.struct_mutex);
 	INIT_LIST_HEAD(&i915->gt.timelines);
-	err = i915_gem_timeline_init__global(i915);
-	if (err) {
-		mutex_unlock(&i915->drm.struct_mutex);
-		goto err_priorities;
-	}
+	INIT_LIST_HEAD(&i915->gt.active_rings);
+	INIT_LIST_HEAD(&i915->gt.closed_vma);
+
+	mutex_lock(&i915->drm.struct_mutex);
 
 	mock_init_ggtt(i915);
-	mutex_unlock(&i915->drm.struct_mutex);
 
 	mkwrite_device_info(i915)->ring_mask = BIT(0);
 	i915->engine[RCS] = mock_engine(i915, "mock", RCS);
 	if (!i915->engine[RCS])
-		goto err_priorities;
+		goto err_unlock;
 
 	i915->kernel_context = mock_context(i915, NULL);
 	if (!i915->kernel_context)
 		goto err_engine;
 
+	mutex_unlock(&i915->drm.struct_mutex);
+
 	WARN_ON(i915_gemfs_init(i915));
 
 	return i915;
@@ -250,7 +250,8 @@ struct drm_i915_private *mock_gem_device(void)
 err_engine:
 	for_each_engine(engine, i915, id)
 		mock_engine_free(engine);
-err_priorities:
+err_unlock:
+	mutex_unlock(&i915->drm.struct_mutex);
 	kmem_cache_destroy(i915->priorities);
 err_dependencies:
 	kmem_cache_destroy(i915->dependencies);
diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c
index e96873f96116..36c112088940 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c
@@ -76,7 +76,6 @@ mock_ppgtt(struct drm_i915_private *i915,
 
 	INIT_LIST_HEAD(&ppgtt->base.global_link);
 	drm_mm_init(&ppgtt->base.mm, 0, ppgtt->base.total);
-	i915_gem_timeline_init(i915, &ppgtt->base.timeline, name);
 
 	ppgtt->base.clear_range = nop_clear_range;
 	ppgtt->base.insert_page = mock_insert_page;
diff --git a/drivers/gpu/drm/i915/selftests/mock_timeline.c b/drivers/gpu/drm/i915/selftests/mock_timeline.c
index 47b1f47c5812..dcf3b16f5a07 100644
--- a/drivers/gpu/drm/i915/selftests/mock_timeline.c
+++ b/drivers/gpu/drm/i915/selftests/mock_timeline.c
@@ -1,45 +1,28 @@
 /*
- * Copyright © 2017 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  *
+ * Copyright © 2017-2018 Intel Corporation
  */
 
+#include "../i915_timeline.h"
+
 #include "mock_timeline.h"
 
-struct intel_timeline *mock_timeline(u64 context)
+void mock_timeline_init(struct i915_timeline *timeline, u64 context)
 {
-	static struct lock_class_key class;
-	struct intel_timeline *tl;
+	timeline->fence_context = context;
+
+	spin_lock_init(&timeline->lock);
 
-	tl = kzalloc(sizeof(*tl), GFP_KERNEL);
-	if (!tl)
-		return NULL;
+	init_request_active(&timeline->last_request, NULL);
+	INIT_LIST_HEAD(&timeline->requests);
 
-	__intel_timeline_init(tl, NULL, context, &class, "mock");
+	i915_syncmap_init(&timeline->sync);
 
-	return tl;
+	INIT_LIST_HEAD(&timeline->link);
 }
 
-void mock_timeline_destroy(struct intel_timeline *tl)
+void mock_timeline_fini(struct i915_timeline *timeline)
 {
-	__intel_timeline_fini(tl);
-	kfree(tl);
+	i915_timeline_fini(timeline);
 }
diff --git a/drivers/gpu/drm/i915/selftests/mock_timeline.h b/drivers/gpu/drm/i915/selftests/mock_timeline.h
index c27ff4639b8b..b6deaa61110d 100644
--- a/drivers/gpu/drm/i915/selftests/mock_timeline.h
+++ b/drivers/gpu/drm/i915/selftests/mock_timeline.h
@@ -1,33 +1,15 @@
 /*
- * Copyright © 2017 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  *
+ * Copyright © 2017-2018 Intel Corporation
  */
 
 #ifndef __MOCK_TIMELINE__
 #define __MOCK_TIMELINE__
 
-#include "../i915_gem_timeline.h"
+struct i915_timeline;
 
-struct intel_timeline *mock_timeline(u64 context);
-void mock_timeline_destroy(struct intel_timeline *tl);
+void mock_timeline_init(struct i915_timeline *timeline, u64 context);
+void mock_timeline_fini(struct i915_timeline *timeline);
 
 #endif /* !__MOCK_TIMELINE__ */
diff --git a/drivers/gpu/drm/imx/imx-drm.h b/drivers/gpu/drm/imx/imx-drm.h
index 15c2bec47a04..ab9c6f706eb3 100644
--- a/drivers/gpu/drm/imx/imx-drm.h
+++ b/drivers/gpu/drm/imx/imx-drm.h
@@ -10,7 +10,6 @@ struct drm_display_mode;
 struct drm_encoder;
 struct drm_framebuffer;
 struct drm_plane;
-struct imx_drm_crtc;
 struct platform_device;
 
 struct imx_crtc_state {
diff --git a/drivers/gpu/drm/imx/imx-ldb.c b/drivers/gpu/drm/imx/imx-ldb.c
index 56dd7a9a8e25..dd5312b02a8d 100644
--- a/drivers/gpu/drm/imx/imx-ldb.c
+++ b/drivers/gpu/drm/imx/imx-ldb.c
@@ -612,6 +612,9 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data)
 		return PTR_ERR(imx_ldb->regmap);
 	}
 
+	/* disable LDB by resetting the control register to POR default */
+	regmap_write(imx_ldb->regmap, IOMUXC_GPR2, 0);
+
 	imx_ldb->dev = dev;
 
 	if (of_id)
@@ -652,14 +655,14 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data)
 		if (ret || i < 0 || i > 1)
 			return -EINVAL;
 
+		if (!of_device_is_available(child))
+			continue;
+
 		if (dual && i > 0) {
 			dev_warn(dev, "dual-channel mode, ignoring second output\n");
 			continue;
 		}
 
-		if (!of_device_is_available(child))
-			continue;
-
 		channel = &imx_ldb->channel[i];
 		channel->ldb = imx_ldb;
 		channel->chno = i;
diff --git a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c
index e83af0f2be86..50d9f2bbe348 100644
--- a/drivers/gpu/drm/imx/ipuv3-crtc.c
+++ b/drivers/gpu/drm/imx/ipuv3-crtc.c
@@ -35,7 +35,6 @@
 struct ipu_crtc {
 	struct device		*dev;
 	struct drm_crtc		base;
-	struct imx_drm_crtc	*imx_crtc;
 
 	/* plane[0] is the full plane, plane[1] is the partial plane */
 	struct ipu_plane	*plane[2];
diff --git a/drivers/gpu/drm/mediatek/Kconfig b/drivers/gpu/drm/mediatek/Kconfig
index 294de4549922..119ec0a21de2 100644
--- a/drivers/gpu/drm/mediatek/Kconfig
+++ b/drivers/gpu/drm/mediatek/Kconfig
@@ -11,6 +11,7 @@ config DRM_MEDIATEK
 	select DRM_PANEL
 	select MEMORY
 	select MTK_SMI
+	select VIDEOMODE_HELPERS
 	help
 	  Choose this option if you have a Mediatek SoCs.
 	  The module will be called mediatek-drm
diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c
index e80a603e5fb0..6c0ea39d5739 100644
--- a/drivers/gpu/drm/mediatek/mtk_dpi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dpi.c
@@ -22,6 +22,7 @@
 #include <linux/interrupt.h>
 #include <linux/types.h>
 #include <linux/clk.h>
+#include <video/videomode.h>
 
 #include "mtk_dpi_regs.h"
 #include "mtk_drm_ddp_comp.h"
@@ -429,34 +430,35 @@ static int mtk_dpi_set_display_mode(struct mtk_dpi *dpi,
 	struct mtk_dpi_sync_param vsync_leven = { 0 };
 	struct mtk_dpi_sync_param vsync_rodd = { 0 };
 	struct mtk_dpi_sync_param vsync_reven = { 0 };
-	unsigned long pix_rate;
+	struct videomode vm = { 0 };
 	unsigned long pll_rate;
 	unsigned int factor;
 
 	/* let pll_rate can fix the valid range of tvdpll (1G~2GHz) */
-	pix_rate = 1000UL * mode->clock;
+
 	if (mode->clock <= 27000)
-		factor = 16 * 3;
+		factor = 3 << 4;
 	else if (mode->clock <= 84000)
-		factor = 8 * 3;
+		factor = 3 << 3;
 	else if (mode->clock <= 167000)
-		factor = 4 * 3;
+		factor = 3 << 2;
 	else
-		factor = 2 * 3;
-	pll_rate = pix_rate * factor;
+		factor = 3 << 1;
+	drm_display_mode_to_videomode(mode, &vm);
+	pll_rate = vm.pixelclock * factor;
 
 	dev_dbg(dpi->dev, "Want PLL %lu Hz, pixel clock %lu Hz\n",
-		pll_rate, pix_rate);
+		pll_rate, vm.pixelclock);
 
 	clk_set_rate(dpi->tvd_clk, pll_rate);
 	pll_rate = clk_get_rate(dpi->tvd_clk);
 
-	pix_rate = pll_rate / factor;
-	clk_set_rate(dpi->pixel_clk, pix_rate);
-	pix_rate = clk_get_rate(dpi->pixel_clk);
+	vm.pixelclock = pll_rate / factor;
+	clk_set_rate(dpi->pixel_clk, vm.pixelclock);
+	vm.pixelclock = clk_get_rate(dpi->pixel_clk);
 
 	dev_dbg(dpi->dev, "Got  PLL %lu Hz, pixel clock %lu Hz\n",
-		pll_rate, pix_rate);
+		pll_rate, vm.pixelclock);
 
 	limit.c_bottom = 0x0010;
 	limit.c_top = 0x0FE0;
@@ -465,33 +467,31 @@ static int mtk_dpi_set_display_mode(struct mtk_dpi *dpi,
 
 	dpi_pol.ck_pol = MTK_DPI_POLARITY_FALLING;
 	dpi_pol.de_pol = MTK_DPI_POLARITY_RISING;
-	dpi_pol.hsync_pol = mode->flags & DRM_MODE_FLAG_PHSYNC ?
+	dpi_pol.hsync_pol = vm.flags & DISPLAY_FLAGS_HSYNC_HIGH ?
 			    MTK_DPI_POLARITY_FALLING : MTK_DPI_POLARITY_RISING;
-	dpi_pol.vsync_pol = mode->flags & DRM_MODE_FLAG_PVSYNC ?
+	dpi_pol.vsync_pol = vm.flags & DISPLAY_FLAGS_VSYNC_HIGH ?
 			    MTK_DPI_POLARITY_FALLING : MTK_DPI_POLARITY_RISING;
-
-	hsync.sync_width = mode->hsync_end - mode->hsync_start;
-	hsync.back_porch = mode->htotal - mode->hsync_end;
-	hsync.front_porch = mode->hsync_start - mode->hdisplay;
+	hsync.sync_width = vm.hsync_len;
+	hsync.back_porch = vm.hback_porch;
+	hsync.front_porch = vm.hfront_porch;
 	hsync.shift_half_line = false;
-
-	vsync_lodd.sync_width = mode->vsync_end - mode->vsync_start;
-	vsync_lodd.back_porch = mode->vtotal - mode->vsync_end;
-	vsync_lodd.front_porch = mode->vsync_start - mode->vdisplay;
+	vsync_lodd.sync_width = vm.vsync_len;
+	vsync_lodd.back_porch = vm.vback_porch;
+	vsync_lodd.front_porch = vm.vfront_porch;
 	vsync_lodd.shift_half_line = false;
 
-	if (mode->flags & DRM_MODE_FLAG_INTERLACE &&
+	if (vm.flags & DISPLAY_FLAGS_INTERLACED &&
 	    mode->flags & DRM_MODE_FLAG_3D_MASK) {
 		vsync_leven = vsync_lodd;
 		vsync_rodd = vsync_lodd;
 		vsync_reven = vsync_lodd;
 		vsync_leven.shift_half_line = true;
 		vsync_reven.shift_half_line = true;
-	} else if (mode->flags & DRM_MODE_FLAG_INTERLACE &&
+	} else if (vm.flags & DISPLAY_FLAGS_INTERLACED &&
 		   !(mode->flags & DRM_MODE_FLAG_3D_MASK)) {
 		vsync_leven = vsync_lodd;
 		vsync_leven.shift_half_line = true;
-	} else if (!(mode->flags & DRM_MODE_FLAG_INTERLACE) &&
+	} else if (!(vm.flags & DISPLAY_FLAGS_INTERLACED) &&
 		   mode->flags & DRM_MODE_FLAG_3D_MASK) {
 		vsync_rodd = vsync_lodd;
 	}
@@ -505,12 +505,12 @@ static int mtk_dpi_set_display_mode(struct mtk_dpi *dpi,
 	mtk_dpi_config_vsync_reven(dpi, &vsync_reven);
 
 	mtk_dpi_config_3d(dpi, !!(mode->flags & DRM_MODE_FLAG_3D_MASK));
-	mtk_dpi_config_interface(dpi, !!(mode->flags &
-					 DRM_MODE_FLAG_INTERLACE));
-	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
-		mtk_dpi_config_fb_size(dpi, mode->hdisplay, mode->vdisplay / 2);
+	mtk_dpi_config_interface(dpi, !!(vm.flags &
+					 DISPLAY_FLAGS_INTERLACED));
+	if (vm.flags & DISPLAY_FLAGS_INTERLACED)
+		mtk_dpi_config_fb_size(dpi, vm.hactive, vm.vactive >> 1);
 	else
-		mtk_dpi_config_fb_size(dpi, mode->hdisplay, mode->vdisplay);
+		mtk_dpi_config_fb_size(dpi, vm.hactive, vm.vactive);
 
 	mtk_dpi_config_channel_limit(dpi, &limit);
 	mtk_dpi_config_bit_num(dpi, dpi->bit_num);
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_gem.c b/drivers/gpu/drm/mediatek/mtk_drm_gem.c
index f595ac816b55..259b7b0de1d2 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_gem.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_gem.c
@@ -220,7 +220,7 @@ struct drm_gem_object *mtk_gem_prime_import_sg_table(struct drm_device *dev,
 	mtk_gem = mtk_drm_gem_init(dev, attach->dmabuf->size);
 
 	if (IS_ERR(mtk_gem))
-		return ERR_PTR(PTR_ERR(mtk_gem));
+		return ERR_CAST(mtk_gem);
 
 	expected = sg_dma_address(sg->sgl);
 	for_each_sg(sg->sgl, s, sg->nents, i) {
diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c
index 7e5e24c2152a..aa0943ec32b0 100644
--- a/drivers/gpu/drm/mediatek/mtk_dsi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dsi.c
@@ -551,13 +551,12 @@ static int mtk_dsi_poweron(struct mtk_dsi *dsi)
 	}
 
 	/**
-	 * vm.pixelclock is in kHz, pixel_clock unit is Hz, so multiply by 1000
 	 * htotal_time = htotal * byte_per_pixel / num_lanes
 	 * overhead_time = lpx + hs_prepare + hs_zero + hs_trail + hs_exit
 	 * mipi_ratio = (htotal_time + overhead_time) / htotal_time
 	 * data_rate = pixel_clock * bit_per_pixel * mipi_ratio / num_lanes;
 	 */
-	pixel_clock = dsi->vm.pixelclock * 1000;
+	pixel_clock = dsi->vm.pixelclock;
 	htotal = dsi->vm.hactive + dsi->vm.hback_porch + dsi->vm.hfront_porch +
 			dsi->vm.hsync_len;
 	htotal_bits = htotal * bit_per_pixel;
@@ -725,16 +724,7 @@ static void mtk_dsi_encoder_mode_set(struct drm_encoder *encoder,
 {
 	struct mtk_dsi *dsi = encoder_to_dsi(encoder);
 
-	dsi->vm.pixelclock = adjusted->clock;
-	dsi->vm.hactive = adjusted->hdisplay;
-	dsi->vm.hback_porch = adjusted->htotal - adjusted->hsync_end;
-	dsi->vm.hfront_porch = adjusted->hsync_start - adjusted->hdisplay;
-	dsi->vm.hsync_len = adjusted->hsync_end - adjusted->hsync_start;
-
-	dsi->vm.vactive = adjusted->vdisplay;
-	dsi->vm.vback_porch = adjusted->vtotal - adjusted->vsync_end;
-	dsi->vm.vfront_porch = adjusted->vsync_start - adjusted->vdisplay;
-	dsi->vm.vsync_len = adjusted->vsync_end - adjusted->vsync_start;
+	drm_display_mode_to_videomode(adjusted, &dsi->vm);
 }
 
 static void mtk_dsi_encoder_disable(struct drm_encoder *encoder)
diff --git a/drivers/gpu/drm/meson/meson_dw_hdmi.c b/drivers/gpu/drm/meson/meson_dw_hdmi.c
index a393095aac1a..c9ad45686e7a 100644
--- a/drivers/gpu/drm/meson/meson_dw_hdmi.c
+++ b/drivers/gpu/drm/meson/meson_dw_hdmi.c
@@ -529,7 +529,7 @@ static irqreturn_t dw_hdmi_top_thread_irq(int irq, void *dev_id)
 		if (stat & HDMITX_TOP_INTR_HPD_RISE)
 			hpd_connected = true;
 
-		dw_hdmi_setup_rx_sense(dw_hdmi->dev, hpd_connected,
+		dw_hdmi_setup_rx_sense(dw_hdmi->hdmi, hpd_connected,
 				       hpd_connected);
 
 		drm_helper_hpd_irq_event(dw_hdmi->encoder.dev);
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index fb50a9ddaae8..8918539a19aa 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -1586,7 +1586,7 @@ static uint32_t mga_vga_calculate_mode_bandwidth(struct drm_display_mode *mode,
 
 #define MODE_BANDWIDTH	MODE_BAD
 
-static int mga_vga_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status mga_vga_mode_valid(struct drm_connector *connector,
 				 struct drm_display_mode *mode)
 {
 	struct drm_device *dev = connector->dev;
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index 8e0cb161754b..0ae5ace65462 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -168,7 +168,6 @@ struct msm_gpu *adreno_load_gpu(struct drm_device *dev)
 	if (gpu->funcs->debugfs_init) {
 		gpu->funcs->debugfs_init(gpu, dev->primary);
 		gpu->funcs->debugfs_init(gpu, dev->render);
-		gpu->funcs->debugfs_init(gpu, dev->control);
 	}
 #endif
 
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
index 7a1ad3af08e3..20e956e14c21 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
@@ -98,21 +98,6 @@ static const struct drm_plane_funcs mdp4_plane_funcs = {
 		.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
 
-static int mdp4_plane_prepare_fb(struct drm_plane *plane,
-				 struct drm_plane_state *new_state)
-{
-	struct mdp4_plane *mdp4_plane = to_mdp4_plane(plane);
-	struct mdp4_kms *mdp4_kms = get_kms(plane);
-	struct msm_kms *kms = &mdp4_kms->base.base;
-	struct drm_framebuffer *fb = new_state->fb;
-
-	if (!fb)
-		return 0;
-
-	DBG("%s: prepare: FB[%u]", mdp4_plane->name, fb->base.id);
-	return msm_framebuffer_prepare(fb, kms->aspace);
-}
-
 static void mdp4_plane_cleanup_fb(struct drm_plane *plane,
 				  struct drm_plane_state *old_state)
 {
@@ -152,7 +137,7 @@ static void mdp4_plane_atomic_update(struct drm_plane *plane,
 }
 
 static const struct drm_plane_helper_funcs mdp4_plane_helper_funcs = {
-		.prepare_fb = mdp4_plane_prepare_fb,
+		.prepare_fb = msm_atomic_prepare_fb,
 		.cleanup_fb = mdp4_plane_cleanup_fb,
 		.atomic_check = mdp4_plane_atomic_check,
 		.atomic_update = mdp4_plane_atomic_update,
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
index 76b96081916f..10271359789e 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
@@ -430,6 +430,7 @@ static void mdp5_crtc_atomic_disable(struct drm_crtc *crtc,
 	struct mdp5_crtc_state *mdp5_cstate = to_mdp5_crtc_state(crtc->state);
 	struct mdp5_kms *mdp5_kms = get_kms(crtc);
 	struct device *dev = &mdp5_kms->pdev->dev;
+	unsigned long flags;
 
 	DBG("%s", crtc->name);
 
@@ -445,6 +446,14 @@ static void mdp5_crtc_atomic_disable(struct drm_crtc *crtc,
 	mdp_irq_unregister(&mdp5_kms->base, &mdp5_crtc->err);
 	pm_runtime_put_sync(dev);
 
+	if (crtc->state->event && !crtc->state->active) {
+		WARN_ON(mdp5_crtc->event);
+		spin_lock_irqsave(&mdp5_kms->dev->event_lock, flags);
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		crtc->state->event = NULL;
+		spin_unlock_irqrestore(&mdp5_kms->dev->event_lock, flags);
+	}
+
 	mdp5_crtc->enabled = false;
 }
 
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
index 6d8e3a9a6fc0..6e12e275deba 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
@@ -70,60 +70,110 @@ static int mdp5_hw_init(struct msm_kms *kms)
 	return 0;
 }
 
-struct mdp5_state *mdp5_get_state(struct drm_atomic_state *s)
+/* Global/shared object state funcs */
+
+/*
+ * This is a helper that returns the private state currently in operation.
+ * Note that this would return the "old_state" if called in the atomic check
+ * path, and the "new_state" after the atomic swap has been done.
+ */
+struct mdp5_global_state *
+mdp5_get_existing_global_state(struct mdp5_kms *mdp5_kms)
+{
+	return to_mdp5_global_state(mdp5_kms->glob_state.state);
+}
+
+/*
+ * This acquires the modeset lock set aside for global state, creates
+ * a new duplicated private object state.
+ */
+struct mdp5_global_state *mdp5_get_global_state(struct drm_atomic_state *s)
 {
 	struct msm_drm_private *priv = s->dev->dev_private;
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms));
-	struct msm_kms_state *state = to_kms_state(s);
-	struct mdp5_state *new_state;
+	struct drm_private_state *priv_state;
 	int ret;
 
-	if (state->state)
-		return state->state;
-
-	ret = drm_modeset_lock(&mdp5_kms->state_lock, s->acquire_ctx);
+	ret = drm_modeset_lock(&mdp5_kms->glob_state_lock, s->acquire_ctx);
 	if (ret)
 		return ERR_PTR(ret);
 
-	new_state = kmalloc(sizeof(*mdp5_kms->state), GFP_KERNEL);
-	if (!new_state)
-		return ERR_PTR(-ENOMEM);
+	priv_state = drm_atomic_get_private_obj_state(s, &mdp5_kms->glob_state);
+	if (IS_ERR(priv_state))
+		return ERR_CAST(priv_state);
 
-	/* Copy state: */
-	new_state->hwpipe = mdp5_kms->state->hwpipe;
-	new_state->hwmixer = mdp5_kms->state->hwmixer;
-	if (mdp5_kms->smp)
-		new_state->smp = mdp5_kms->state->smp;
+	return to_mdp5_global_state(priv_state);
+}
+
+static struct drm_private_state *
+mdp5_global_duplicate_state(struct drm_private_obj *obj)
+{
+	struct mdp5_global_state *state;
+
+	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
 
-	state->state = new_state;
+	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
 
-	return new_state;
+	return &state->base;
 }
 
-static void mdp5_swap_state(struct msm_kms *kms, struct drm_atomic_state *state)
+static void mdp5_global_destroy_state(struct drm_private_obj *obj,
+				      struct drm_private_state *state)
 {
-	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
-	swap(to_kms_state(state)->state, mdp5_kms->state);
+	struct mdp5_global_state *mdp5_state = to_mdp5_global_state(state);
+
+	kfree(mdp5_state);
+}
+
+static const struct drm_private_state_funcs mdp5_global_state_funcs = {
+	.atomic_duplicate_state = mdp5_global_duplicate_state,
+	.atomic_destroy_state = mdp5_global_destroy_state,
+};
+
+static int mdp5_global_obj_init(struct mdp5_kms *mdp5_kms)
+{
+	struct mdp5_global_state *state;
+
+	drm_modeset_lock_init(&mdp5_kms->glob_state_lock);
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return -ENOMEM;
+
+	state->mdp5_kms = mdp5_kms;
+
+	drm_atomic_private_obj_init(&mdp5_kms->glob_state,
+				    &state->base,
+				    &mdp5_global_state_funcs);
+	return 0;
 }
 
 static void mdp5_prepare_commit(struct msm_kms *kms, struct drm_atomic_state *state)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
 	struct device *dev = &mdp5_kms->pdev->dev;
+	struct mdp5_global_state *global_state;
+
+	global_state = mdp5_get_existing_global_state(mdp5_kms);
 
 	pm_runtime_get_sync(dev);
 
 	if (mdp5_kms->smp)
-		mdp5_smp_prepare_commit(mdp5_kms->smp, &mdp5_kms->state->smp);
+		mdp5_smp_prepare_commit(mdp5_kms->smp, &global_state->smp);
 }
 
 static void mdp5_complete_commit(struct msm_kms *kms, struct drm_atomic_state *state)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
 	struct device *dev = &mdp5_kms->pdev->dev;
+	struct mdp5_global_state *global_state;
+
+	global_state = mdp5_get_existing_global_state(mdp5_kms);
 
 	if (mdp5_kms->smp)
-		mdp5_smp_complete_commit(mdp5_kms->smp, &mdp5_kms->state->smp);
+		mdp5_smp_complete_commit(mdp5_kms->smp, &global_state->smp);
 
 	pm_runtime_put_sync(dev);
 }
@@ -229,7 +279,6 @@ static const struct mdp_kms_funcs kms_funcs = {
 		.irq             = mdp5_irq,
 		.enable_vblank   = mdp5_enable_vblank,
 		.disable_vblank  = mdp5_disable_vblank,
-		.swap_state      = mdp5_swap_state,
 		.prepare_commit  = mdp5_prepare_commit,
 		.complete_commit = mdp5_complete_commit,
 		.wait_for_crtc_commit_done = mdp5_wait_for_crtc_commit_done,
@@ -727,7 +776,8 @@ static void mdp5_destroy(struct platform_device *pdev)
 	if (mdp5_kms->rpm_enabled)
 		pm_runtime_disable(&pdev->dev);
 
-	kfree(mdp5_kms->state);
+	drm_atomic_private_obj_fini(&mdp5_kms->glob_state);
+	drm_modeset_lock_fini(&mdp5_kms->glob_state_lock);
 }
 
 static int construct_pipes(struct mdp5_kms *mdp5_kms, int cnt,
@@ -880,12 +930,9 @@ static int mdp5_init(struct platform_device *pdev, struct drm_device *dev)
 	mdp5_kms->dev = dev;
 	mdp5_kms->pdev = pdev;
 
-	drm_modeset_lock_init(&mdp5_kms->state_lock);
-	mdp5_kms->state = kzalloc(sizeof(*mdp5_kms->state), GFP_KERNEL);
-	if (!mdp5_kms->state) {
-		ret = -ENOMEM;
+	ret = mdp5_global_obj_init(mdp5_kms);
+	if (ret)
 		goto fail;
-	}
 
 	mdp5_kms->mmio = msm_ioremap(pdev, "mdp_phys", "MDP5");
 	if (IS_ERR(mdp5_kms->mmio)) {
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h
index 425a03d213e5..854dfd30e829 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h
@@ -28,8 +28,6 @@
 #include "mdp5_ctl.h"
 #include "mdp5_smp.h"
 
-struct mdp5_state;
-
 struct mdp5_kms {
 	struct mdp_kms base;
 
@@ -49,11 +47,12 @@ struct mdp5_kms {
 	struct mdp5_cfg_handler *cfg;
 	uint32_t caps;	/* MDP capabilities (MDP_CAP_XXX bits) */
 
-	/**
-	 * Global atomic state.  Do not access directly, use mdp5_get_state()
+	/*
+	 * Global private object state, Do not access directly, use
+	 * mdp5_global_get_state()
 	 */
-	struct mdp5_state *state;
-	struct drm_modeset_lock state_lock;
+	struct drm_modeset_lock glob_state_lock;
+	struct drm_private_obj glob_state;
 
 	struct mdp5_smp *smp;
 	struct mdp5_ctl_manager *ctlm;
@@ -81,19 +80,23 @@ struct mdp5_kms {
 };
 #define to_mdp5_kms(x) container_of(x, struct mdp5_kms, base)
 
-/* Global atomic state for tracking resources that are shared across
+/* Global private object state for tracking resources that are shared across
  * multiple kms objects (planes/crtcs/etc).
- *
- * For atomic updates which require modifying global state,
  */
-struct mdp5_state {
+#define to_mdp5_global_state(x) container_of(x, struct mdp5_global_state, base)
+struct mdp5_global_state {
+	struct drm_private_state base;
+
+	struct drm_atomic_state *state;
+	struct mdp5_kms *mdp5_kms;
+
 	struct mdp5_hw_pipe_state hwpipe;
 	struct mdp5_hw_mixer_state hwmixer;
 	struct mdp5_smp_state smp;
 };
 
-struct mdp5_state *__must_check
-mdp5_get_state(struct drm_atomic_state *s);
+struct mdp5_global_state * mdp5_get_existing_global_state(struct mdp5_kms *mdp5_kms);
+struct mdp5_global_state *__must_check mdp5_get_global_state(struct drm_atomic_state *s);
 
 /* Atomic plane state.  Subclasses the base drm_plane_state in order to
  * track assigned hwpipe and hw specific state.
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c
index 8a00991f03c7..113e6b569562 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c
@@ -52,14 +52,14 @@ int mdp5_mixer_assign(struct drm_atomic_state *s, struct drm_crtc *crtc,
 {
 	struct msm_drm_private *priv = s->dev->dev_private;
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms));
-	struct mdp5_state *state = mdp5_get_state(s);
+	struct mdp5_global_state *global_state = mdp5_get_global_state(s);
 	struct mdp5_hw_mixer_state *new_state;
 	int i;
 
-	if (IS_ERR(state))
-		return PTR_ERR(state);
+	if (IS_ERR(global_state))
+		return PTR_ERR(global_state);
 
-	new_state = &state->hwmixer;
+	new_state = &global_state->hwmixer;
 
 	for (i = 0; i < mdp5_kms->num_hwmixers; i++) {
 		struct mdp5_hw_mixer *cur = mdp5_kms->hwmixers[i];
@@ -129,8 +129,8 @@ int mdp5_mixer_assign(struct drm_atomic_state *s, struct drm_crtc *crtc,
 
 void mdp5_mixer_release(struct drm_atomic_state *s, struct mdp5_hw_mixer *mixer)
 {
-	struct mdp5_state *state = mdp5_get_state(s);
-	struct mdp5_hw_mixer_state *new_state = &state->hwmixer;
+	struct mdp5_global_state *global_state = mdp5_get_global_state(s);
+	struct mdp5_hw_mixer_state *new_state = &global_state->hwmixer;
 
 	if (!mixer)
 		return;
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c
index ff52c49095f9..1ef26bc63163 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c
@@ -24,17 +24,19 @@ int mdp5_pipe_assign(struct drm_atomic_state *s, struct drm_plane *plane,
 {
 	struct msm_drm_private *priv = s->dev->dev_private;
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms));
-	struct mdp5_state *state;
+	struct mdp5_global_state *new_global_state, *old_global_state;
 	struct mdp5_hw_pipe_state *old_state, *new_state;
 	int i, j;
 
-	state = mdp5_get_state(s);
-	if (IS_ERR(state))
-		return PTR_ERR(state);
+	new_global_state = mdp5_get_global_state(s);
+	if (IS_ERR(new_global_state))
+		return PTR_ERR(new_global_state);
 
-	/* grab old_state after mdp5_get_state(), since now we hold lock: */
-	old_state = &mdp5_kms->state->hwpipe;
-	new_state = &state->hwpipe;
+	/* grab old_state after mdp5_get_global_state(), since now we hold lock: */
+	old_global_state = mdp5_get_existing_global_state(mdp5_kms);
+
+	old_state = &old_global_state->hwpipe;
+	new_state = &new_global_state->hwpipe;
 
 	for (i = 0; i < mdp5_kms->num_hwpipes; i++) {
 		struct mdp5_hw_pipe *cur = mdp5_kms->hwpipes[i];
@@ -107,7 +109,7 @@ int mdp5_pipe_assign(struct drm_atomic_state *s, struct drm_plane *plane,
 		WARN_ON(r_hwpipe);
 
 		DBG("%s: alloc SMP blocks", (*hwpipe)->name);
-		ret = mdp5_smp_assign(mdp5_kms->smp, &state->smp,
+		ret = mdp5_smp_assign(mdp5_kms->smp, &new_global_state->smp,
 				(*hwpipe)->pipe, blkcfg);
 		if (ret)
 			return -ENOMEM;
@@ -132,7 +134,7 @@ void mdp5_pipe_release(struct drm_atomic_state *s, struct mdp5_hw_pipe *hwpipe)
 {
 	struct msm_drm_private *priv = s->dev->dev_private;
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms));
-	struct mdp5_state *state = mdp5_get_state(s);
+	struct mdp5_global_state *state = mdp5_get_global_state(s);
 	struct mdp5_hw_pipe_state *new_state = &state->hwpipe;
 
 	if (!hwpipe)
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
index a9f31da7d45a..e09bc53a0e65 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
@@ -245,20 +245,6 @@ static const struct drm_plane_funcs mdp5_plane_funcs = {
 		.atomic_print_state = mdp5_plane_atomic_print_state,
 };
 
-static int mdp5_plane_prepare_fb(struct drm_plane *plane,
-				 struct drm_plane_state *new_state)
-{
-	struct mdp5_kms *mdp5_kms = get_kms(plane);
-	struct msm_kms *kms = &mdp5_kms->base.base;
-	struct drm_framebuffer *fb = new_state->fb;
-
-	if (!new_state->fb)
-		return 0;
-
-	DBG("%s: prepare: FB[%u]", plane->name, fb->base.id);
-	return msm_framebuffer_prepare(fb, kms->aspace);
-}
-
 static void mdp5_plane_cleanup_fb(struct drm_plane *plane,
 				  struct drm_plane_state *old_state)
 {
@@ -543,7 +529,7 @@ static void mdp5_plane_atomic_async_update(struct drm_plane *plane,
 }
 
 static const struct drm_plane_helper_funcs mdp5_plane_helper_funcs = {
-		.prepare_fb = mdp5_plane_prepare_fb,
+		.prepare_fb = msm_atomic_prepare_fb,
 		.cleanup_fb = mdp5_plane_cleanup_fb,
 		.atomic_check = mdp5_plane_atomic_check,
 		.atomic_update = mdp5_plane_atomic_update,
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c
index ae4983d9d0a5..96c2b828dba4 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c
@@ -340,17 +340,20 @@ void mdp5_smp_dump(struct mdp5_smp *smp, struct drm_printer *p)
 	struct mdp5_kms *mdp5_kms = get_kms(smp);
 	struct mdp5_hw_pipe_state *hwpstate;
 	struct mdp5_smp_state *state;
+	struct mdp5_global_state *global_state;
 	int total = 0, i, j;
 
 	drm_printf(p, "name\tinuse\tplane\n");
 	drm_printf(p, "----\t-----\t-----\n");
 
 	if (drm_can_sleep())
-		drm_modeset_lock(&mdp5_kms->state_lock, NULL);
+		drm_modeset_lock(&mdp5_kms->glob_state_lock, NULL);
+
+	global_state = mdp5_get_existing_global_state(mdp5_kms);
 
 	/* grab these *after* we hold the state_lock */
-	hwpstate = &mdp5_kms->state->hwpipe;
-	state = &mdp5_kms->state->smp;
+	hwpstate = &global_state->hwpipe;
+	state = &global_state->smp;
 
 	for (i = 0; i < mdp5_kms->num_hwpipes; i++) {
 		struct mdp5_hw_pipe *hwpipe = mdp5_kms->hwpipes[i];
@@ -374,7 +377,7 @@ void mdp5_smp_dump(struct mdp5_smp *smp, struct drm_printer *p)
 			bitmap_weight(state->state, smp->blk_cnt));
 
 	if (drm_can_sleep())
-		drm_modeset_unlock(&mdp5_kms->state_lock);
+		drm_modeset_unlock(&mdp5_kms->glob_state_lock);
 }
 
 void mdp5_smp_destroy(struct mdp5_smp *smp)
@@ -384,7 +387,8 @@ void mdp5_smp_destroy(struct mdp5_smp *smp)
 
 struct mdp5_smp *mdp5_smp_init(struct mdp5_kms *mdp5_kms, const struct mdp5_smp_block *cfg)
 {
-	struct mdp5_smp_state *state = &mdp5_kms->state->smp;
+	struct mdp5_smp_state *state;
+	struct mdp5_global_state *global_state;
 	struct mdp5_smp *smp = NULL;
 	int ret;
 
@@ -398,6 +402,9 @@ struct mdp5_smp *mdp5_smp_init(struct mdp5_kms *mdp5_kms, const struct mdp5_smp_
 	smp->blk_cnt = cfg->mmb_count;
 	smp->blk_size = cfg->mmb_size;
 
+	global_state = mdp5_get_existing_global_state(mdp5_kms);
+	state = &global_state->smp;
+
 	/* statically tied MMBs cannot be re-allocated: */
 	bitmap_copy(state->state, cfg->reserved_state, smp->blk_cnt);
 	memcpy(smp->reserved, cfg->reserved, sizeof(smp->reserved));
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index 8baba30d6c65..b916f464f4ec 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -1036,7 +1036,6 @@ static int dsi_tx_buf_alloc(struct msm_dsi_host *msm_host, int size)
 
 		ret = msm_gem_get_iova(msm_host->tx_gem_obj,
 				priv->kms->aspace, &iova);
-		mutex_unlock(&dev->struct_mutex);
 		if (ret) {
 			pr_err("%s: failed to get iova, %d\n", __func__, ret);
 			return ret;
@@ -1067,9 +1066,10 @@ static int dsi_tx_buf_alloc(struct msm_dsi_host *msm_host, int size)
 static void dsi_tx_buf_free(struct msm_dsi_host *msm_host)
 {
 	struct drm_device *dev = msm_host->dev;
+	struct msm_drm_private *priv = dev->dev_private;
 
 	if (msm_host->tx_gem_obj) {
-		msm_gem_put_iova(msm_host->tx_gem_obj, 0);
+		msm_gem_put_iova(msm_host->tx_gem_obj, priv->kms->aspace);
 		drm_gem_object_put_unlocked(msm_host->tx_gem_obj);
 		msm_host->tx_gem_obj = NULL;
 	}
diff --git a/drivers/gpu/drm/msm/msm_atomic.c b/drivers/gpu/drm/msm/msm_atomic.c
index bf5f8c39f34d..f0635c3da7f4 100644
--- a/drivers/gpu/drm/msm/msm_atomic.c
+++ b/drivers/gpu/drm/msm/msm_atomic.c
@@ -16,69 +16,8 @@
  */
 
 #include "msm_drv.h"
-#include "msm_kms.h"
 #include "msm_gem.h"
-#include "msm_fence.h"
-
-struct msm_commit {
-	struct drm_device *dev;
-	struct drm_atomic_state *state;
-	struct work_struct work;
-	uint32_t crtc_mask;
-};
-
-static void commit_worker(struct work_struct *work);
-
-/* block until specified crtcs are no longer pending update, and
- * atomically mark them as pending update
- */
-static int start_atomic(struct msm_drm_private *priv, uint32_t crtc_mask)
-{
-	int ret;
-
-	spin_lock(&priv->pending_crtcs_event.lock);
-	ret = wait_event_interruptible_locked(priv->pending_crtcs_event,
-			!(priv->pending_crtcs & crtc_mask));
-	if (ret == 0) {
-		DBG("start: %08x", crtc_mask);
-		priv->pending_crtcs |= crtc_mask;
-	}
-	spin_unlock(&priv->pending_crtcs_event.lock);
-
-	return ret;
-}
-
-/* clear specified crtcs (no longer pending update)
- */
-static void end_atomic(struct msm_drm_private *priv, uint32_t crtc_mask)
-{
-	spin_lock(&priv->pending_crtcs_event.lock);
-	DBG("end: %08x", crtc_mask);
-	priv->pending_crtcs &= ~crtc_mask;
-	wake_up_all_locked(&priv->pending_crtcs_event);
-	spin_unlock(&priv->pending_crtcs_event.lock);
-}
-
-static struct msm_commit *commit_init(struct drm_atomic_state *state)
-{
-	struct msm_commit *c = kzalloc(sizeof(*c), GFP_KERNEL);
-
-	if (!c)
-		return NULL;
-
-	c->dev = state->dev;
-	c->state = state;
-
-	INIT_WORK(&c->work, commit_worker);
-
-	return c;
-}
-
-static void commit_destroy(struct msm_commit *c)
-{
-	end_atomic(c->dev->dev_private, c->crtc_mask);
-	kfree(c);
-}
+#include "msm_kms.h"
 
 static void msm_atomic_wait_for_commit_done(struct drm_device *dev,
 		struct drm_atomic_state *old_state)
@@ -97,195 +36,48 @@ static void msm_atomic_wait_for_commit_done(struct drm_device *dev,
 	}
 }
 
-/* The (potentially) asynchronous part of the commit.  At this point
- * nothing can fail short of armageddon.
- */
-static void complete_commit(struct msm_commit *c, bool async)
+int msm_atomic_prepare_fb(struct drm_plane *plane,
+			  struct drm_plane_state *new_state)
 {
-	struct drm_atomic_state *state = c->state;
-	struct drm_device *dev = state->dev;
-	struct msm_drm_private *priv = dev->dev_private;
+	struct msm_drm_private *priv = plane->dev->dev_private;
 	struct msm_kms *kms = priv->kms;
+	struct drm_gem_object *obj;
+	struct msm_gem_object *msm_obj;
+	struct dma_fence *fence;
 
-	drm_atomic_helper_wait_for_fences(dev, state, false);
-
-	kms->funcs->prepare_commit(kms, state);
-
-	drm_atomic_helper_commit_modeset_disables(dev, state);
-
-	drm_atomic_helper_commit_planes(dev, state, 0);
-
-	drm_atomic_helper_commit_modeset_enables(dev, state);
-
-	/* NOTE: _wait_for_vblanks() only waits for vblank on
-	 * enabled CRTCs.  So we end up faulting when disabling
-	 * due to (potentially) unref'ing the outgoing fb's
-	 * before the vblank when the disable has latched.
-	 *
-	 * But if it did wait on disabled (or newly disabled)
-	 * CRTCs, that would be racy (ie. we could have missed
-	 * the irq.  We need some way to poll for pipe shut
-	 * down.  Or just live with occasionally hitting the
-	 * timeout in the CRTC disable path (which really should
-	 * not be critical path)
-	 */
-
-	msm_atomic_wait_for_commit_done(dev, state);
-
-	drm_atomic_helper_cleanup_planes(dev, state);
+	if (!new_state->fb)
+		return 0;
 
-	kms->funcs->complete_commit(kms, state);
+	obj = msm_framebuffer_bo(new_state->fb, 0);
+	msm_obj = to_msm_bo(obj);
+	fence = reservation_object_get_excl_rcu(msm_obj->resv);
 
-	drm_atomic_state_put(state);
+	drm_atomic_set_fence_for_plane(new_state, fence);
 
-	commit_destroy(c);
-}
-
-static void commit_worker(struct work_struct *work)
-{
-	complete_commit(container_of(work, struct msm_commit, work), true);
+	return msm_framebuffer_prepare(new_state->fb, kms->aspace);
 }
 
-/**
- * drm_atomic_helper_commit - commit validated state object
- * @dev: DRM device
- * @state: the driver state object
- * @nonblock: nonblocking commit
- *
- * This function commits a with drm_atomic_helper_check() pre-validated state
- * object. This can still fail when e.g. the framebuffer reservation fails.
- *
- * RETURNS
- * Zero for success or -errno.
- */
-int msm_atomic_commit(struct drm_device *dev,
-		struct drm_atomic_state *state, bool nonblock)
+void msm_atomic_commit_tail(struct drm_atomic_state *state)
 {
+	struct drm_device *dev = state->dev;
 	struct msm_drm_private *priv = dev->dev_private;
-	struct msm_commit *c;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *crtc_state;
-	struct drm_plane *plane;
-	struct drm_plane_state *old_plane_state, *new_plane_state;
-	int i, ret;
-
-	ret = drm_atomic_helper_prepare_planes(dev, state);
-	if (ret)
-		return ret;
-
-	/*
-	 * Note that plane->atomic_async_check() should fail if we need
-	 * to re-assign hwpipe or anything that touches global atomic
-	 * state, so we'll never go down the async update path in those
-	 * cases.
-	 */
-	if (state->async_update) {
-		drm_atomic_helper_async_commit(dev, state);
-		drm_atomic_helper_cleanup_planes(dev, state);
-		return 0;
-	}
-
-	c = commit_init(state);
-	if (!c) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	/*
-	 * Figure out what crtcs we have:
-	 */
-	for_each_new_crtc_in_state(state, crtc, crtc_state, i)
-		c->crtc_mask |= drm_crtc_mask(crtc);
-
-	/*
-	 * Figure out what fence to wait for:
-	 */
-	for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) {
-		if ((new_plane_state->fb != old_plane_state->fb) && new_plane_state->fb) {
-			struct drm_gem_object *obj = msm_framebuffer_bo(new_plane_state->fb, 0);
-			struct msm_gem_object *msm_obj = to_msm_bo(obj);
-			struct dma_fence *fence = reservation_object_get_excl_rcu(msm_obj->resv);
+	struct msm_kms *kms = priv->kms;
 
-			drm_atomic_set_fence_for_plane(new_plane_state, fence);
-		}
-	}
+	kms->funcs->prepare_commit(kms, state);
 
-	/*
-	 * Wait for pending updates on any of the same crtc's and then
-	 * mark our set of crtc's as busy:
-	 */
-	ret = start_atomic(dev->dev_private, c->crtc_mask);
-	if (ret)
-		goto err_free;
+	drm_atomic_helper_commit_modeset_disables(dev, state);
 
-	BUG_ON(drm_atomic_helper_swap_state(state, false) < 0);
+	drm_atomic_helper_commit_planes(dev, state, 0);
 
-	/*
-	 * This is the point of no return - everything below never fails except
-	 * when the hw goes bonghits. Which means we can commit the new state on
-	 * the software side now.
-	 *
-	 * swap driver private state while still holding state_lock
-	 */
-	if (to_kms_state(state)->state)
-		priv->kms->funcs->swap_state(priv->kms, state);
+	drm_atomic_helper_commit_modeset_enables(dev, state);
 
-	/*
-	 * Everything below can be run asynchronously without the need to grab
-	 * any modeset locks at all under one conditions: It must be guaranteed
-	 * that the asynchronous work has either been cancelled (if the driver
-	 * supports it, which at least requires that the framebuffers get
-	 * cleaned up with drm_atomic_helper_cleanup_planes()) or completed
-	 * before the new state gets committed on the software side with
-	 * drm_atomic_helper_swap_state().
-	 *
-	 * This scheme allows new atomic state updates to be prepared and
-	 * checked in parallel to the asynchronous completion of the previous
-	 * update. Which is important since compositors need to figure out the
-	 * composition of the next frame right after having submitted the
-	 * current layout.
-	 */
+	msm_atomic_wait_for_commit_done(dev, state);
 
-	drm_atomic_state_get(state);
-	if (nonblock) {
-		queue_work(priv->atomic_wq, &c->work);
-		return 0;
-	}
+	kms->funcs->complete_commit(kms, state);
 
-	complete_commit(c, false);
+	drm_atomic_helper_wait_for_vblanks(dev, state);
 
-	return 0;
+	drm_atomic_helper_commit_hw_done(state);
 
-err_free:
-	kfree(c);
-error:
 	drm_atomic_helper_cleanup_planes(dev, state);
-	return ret;
-}
-
-struct drm_atomic_state *msm_atomic_state_alloc(struct drm_device *dev)
-{
-	struct msm_kms_state *state = kzalloc(sizeof(*state), GFP_KERNEL);
-
-	if (!state || drm_atomic_state_init(dev, &state->base) < 0) {
-		kfree(state);
-		return NULL;
-	}
-
-	return &state->base;
-}
-
-void msm_atomic_state_clear(struct drm_atomic_state *s)
-{
-	struct msm_kms_state *state = to_kms_state(s);
-	drm_atomic_state_default_clear(&state->base);
-	kfree(state->state);
-	state->state = NULL;
-}
-
-void msm_atomic_state_free(struct drm_atomic_state *state)
-{
-	kfree(to_kms_state(state)->state);
-	drm_atomic_state_default_release(state);
-	kfree(state);
 }
diff --git a/drivers/gpu/drm/msm/msm_debugfs.c b/drivers/gpu/drm/msm/msm_debugfs.c
index ba74cb4f94df..1ff3fda245d1 100644
--- a/drivers/gpu/drm/msm/msm_debugfs.c
+++ b/drivers/gpu/drm/msm/msm_debugfs.c
@@ -140,9 +140,6 @@ int msm_debugfs_late_init(struct drm_device *dev)
 	if (ret)
 		return ret;
 	ret = late_init_minor(dev->render);
-	if (ret)
-		return ret;
-	ret = late_init_minor(dev->control);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 30cd514d8f7c..021a0b6f9a59 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -41,10 +41,11 @@ static const struct drm_mode_config_funcs mode_config_funcs = {
 	.fb_create = msm_framebuffer_create,
 	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
-	.atomic_commit = msm_atomic_commit,
-	.atomic_state_alloc = msm_atomic_state_alloc,
-	.atomic_state_clear = msm_atomic_state_clear,
-	.atomic_state_free = msm_atomic_state_free,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static const struct drm_mode_config_helper_funcs mode_config_helper_funcs = {
+	.atomic_commit_tail = msm_atomic_commit_tail,
 };
 
 #ifdef CONFIG_DRM_MSM_REGISTER_LOGGING
@@ -384,7 +385,6 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
 
 	priv->wq = alloc_ordered_workqueue("msm", 0);
 	priv->atomic_wq = alloc_ordered_workqueue("msm:atomic", 0);
-	init_waitqueue_head(&priv->pending_crtcs_event);
 
 	INIT_LIST_HEAD(&priv->inactive_list);
 	INIT_LIST_HEAD(&priv->vblank_ctrl.event_list);
@@ -442,6 +442,7 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
 	}
 
 	ddev->mode_config.funcs = &mode_config_funcs;
+	ddev->mode_config.helper_private = &mode_config_helper_funcs;
 
 	ret = drm_vblank_init(ddev, priv->num_crtcs);
 	if (ret < 0) {
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index 48ed5b9a8580..b2da1fbf81e0 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -117,10 +117,6 @@ struct msm_drm_private {
 	struct workqueue_struct *wq;
 	struct workqueue_struct *atomic_wq;
 
-	/* crtcs pending async atomic updates: */
-	uint32_t pending_crtcs;
-	wait_queue_head_t pending_crtcs_event;
-
 	unsigned int num_planes;
 	struct drm_plane *planes[16];
 
@@ -160,8 +156,9 @@ struct msm_format {
 	uint32_t pixel_format;
 };
 
-int msm_atomic_commit(struct drm_device *dev,
-		struct drm_atomic_state *state, bool nonblock);
+int msm_atomic_prepare_fb(struct drm_plane *plane,
+			  struct drm_plane_state *new_state);
+void msm_atomic_commit_tail(struct drm_atomic_state *state);
 struct drm_atomic_state *msm_atomic_state_alloc(struct drm_device *dev);
 void msm_atomic_state_clear(struct drm_atomic_state *state);
 void msm_atomic_state_free(struct drm_atomic_state *state);
diff --git a/drivers/gpu/drm/msm/msm_kms.h b/drivers/gpu/drm/msm/msm_kms.h
index aaa329dc020e..dfd92947de2c 100644
--- a/drivers/gpu/drm/msm/msm_kms.h
+++ b/drivers/gpu/drm/msm/msm_kms.h
@@ -40,8 +40,6 @@ struct msm_kms_funcs {
 	irqreturn_t (*irq)(struct msm_kms *kms);
 	int (*enable_vblank)(struct msm_kms *kms, struct drm_crtc *crtc);
 	void (*disable_vblank)(struct msm_kms *kms, struct drm_crtc *crtc);
-	/* swap global atomic state: */
-	void (*swap_state)(struct msm_kms *kms, struct drm_atomic_state *state);
 	/* modeset, bracketing atomic_commit(): */
 	void (*prepare_commit)(struct msm_kms *kms, struct drm_atomic_state *state);
 	void (*complete_commit)(struct msm_kms *kms, struct drm_atomic_state *state);
@@ -80,18 +78,6 @@ struct msm_kms {
 	struct msm_gem_address_space *aspace;
 };
 
-/**
- * Subclass of drm_atomic_state, to allow kms backend to have driver
- * private global state.  The kms backend can do whatever it wants
- * with the ->state ptr.  On ->atomic_state_clear() the ->state ptr
- * is kfree'd and set back to NULL.
- */
-struct msm_kms_state {
-	struct drm_atomic_state base;
-	void *state;
-};
-#define to_kms_state(x) container_of(x, struct msm_kms_state, base)
-
 static inline void msm_kms_init(struct msm_kms *kms,
 		const struct msm_kms_funcs *funcs)
 {
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_drv.c b/drivers/gpu/drm/mxsfb/mxsfb_drv.c
index 5cae8db9dcd4..ffe5137ccaf8 100644
--- a/drivers/gpu/drm/mxsfb/mxsfb_drv.c
+++ b/drivers/gpu/drm/mxsfb/mxsfb_drv.c
@@ -99,7 +99,8 @@ static const struct drm_mode_config_funcs mxsfb_mode_config_funcs = {
 };
 
 static void mxsfb_pipe_enable(struct drm_simple_display_pipe *pipe,
-			      struct drm_crtc_state *crtc_state)
+			      struct drm_crtc_state *crtc_state,
+			      struct drm_plane_state *plane_state)
 {
 	struct mxsfb_drm_private *mxsfb = drm_pipe_to_mxsfb_drm_private(pipe);
 
@@ -125,12 +126,6 @@ static void mxsfb_pipe_update(struct drm_simple_display_pipe *pipe,
 	mxsfb_plane_atomic_update(mxsfb, plane_state);
 }
 
-static int mxsfb_pipe_prepare_fb(struct drm_simple_display_pipe *pipe,
-				 struct drm_plane_state *plane_state)
-{
-	return drm_gem_fb_prepare_fb(&pipe->plane, plane_state);
-}
-
 static int mxsfb_pipe_enable_vblank(struct drm_simple_display_pipe *pipe)
 {
 	struct mxsfb_drm_private *mxsfb = drm_pipe_to_mxsfb_drm_private(pipe);
@@ -159,7 +154,7 @@ static struct drm_simple_display_pipe_funcs mxsfb_funcs = {
 	.enable		= mxsfb_pipe_enable,
 	.disable	= mxsfb_pipe_disable,
 	.update		= mxsfb_pipe_update,
-	.prepare_fb	= mxsfb_pipe_prepare_fb,
+	.prepare_fb	= drm_gem_fb_simple_display_pipe_prepare_fb,
 	.enable_vblank	= mxsfb_pipe_enable_vblank,
 	.disable_vblank	= mxsfb_pipe_disable_vblank,
 };
diff --git a/drivers/gpu/drm/nouveau/Kbuild b/drivers/gpu/drm/nouveau/Kbuild
index 9c0c650655e9..b17843dd050d 100644
--- a/drivers/gpu/drm/nouveau/Kbuild
+++ b/drivers/gpu/drm/nouveau/Kbuild
@@ -38,14 +38,16 @@ nouveau-y += nouveau_vmm.o
 
 # DRM - modesetting
 nouveau-$(CONFIG_DRM_NOUVEAU_BACKLIGHT) += nouveau_backlight.o
+nouveau-y += nouveau_bios.o
 nouveau-y += nouveau_connector.o
 nouveau-y += nouveau_display.o
-nouveau-y += nv50_display.o
 nouveau-y += nouveau_dp.o
 nouveau-y += nouveau_fbcon.o
 nouveau-y += nv04_fbcon.o
 nouveau-y += nv50_fbcon.o
 nouveau-y += nvc0_fbcon.o
+include $(src)/dispnv04/Kbuild
+include $(src)/dispnv50/Kbuild
 
 # DRM - command submission
 nouveau-y += nouveau_abi16.o
@@ -59,8 +61,4 @@ nouveau-y += nv50_fence.o
 nouveau-y += nv84_fence.o
 nouveau-y += nvc0_fence.o
 
-# DRM - prehistoric modesetting (NV04-G7x)
-nouveau-y += nouveau_bios.o
-include $(src)/dispnv04/Kbuild
-
 obj-$(CONFIG_DRM_NOUVEAU) += nouveau.o
diff --git a/drivers/gpu/drm/nouveau/dispnv50/Kbuild b/drivers/gpu/drm/nouveau/dispnv50/Kbuild
new file mode 100644
index 000000000000..849b0f45afb8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/Kbuild
@@ -0,0 +1,51 @@
+nouveau-y += dispnv50/disp.o
+nouveau-y += dispnv50/lut.o
+
+nouveau-y += dispnv50/core.o
+nouveau-y += dispnv50/core507d.o
+nouveau-y += dispnv50/core827d.o
+nouveau-y += dispnv50/core907d.o
+nouveau-y += dispnv50/core917d.o
+nouveau-y += dispnv50/corec37d.o
+
+nouveau-y += dispnv50/dac507d.o
+nouveau-y += dispnv50/dac907d.o
+
+nouveau-y += dispnv50/pior507d.o
+
+nouveau-y += dispnv50/sor507d.o
+nouveau-y += dispnv50/sor907d.o
+nouveau-y += dispnv50/sorc37d.o
+
+nouveau-y += dispnv50/head.o
+nouveau-y += dispnv50/head507d.o
+nouveau-y += dispnv50/head827d.o
+nouveau-y += dispnv50/head907d.o
+nouveau-y += dispnv50/head917d.o
+nouveau-y += dispnv50/headc37d.o
+
+nouveau-y += dispnv50/wimm.o
+nouveau-y += dispnv50/wimmc37b.o
+
+nouveau-y += dispnv50/wndw.o
+nouveau-y += dispnv50/wndwc37e.o
+
+nouveau-y += dispnv50/base.o
+nouveau-y += dispnv50/base507c.o
+nouveau-y += dispnv50/base827c.o
+nouveau-y += dispnv50/base907c.o
+nouveau-y += dispnv50/base917c.o
+
+nouveau-y += dispnv50/curs.o
+nouveau-y += dispnv50/curs507a.o
+nouveau-y += dispnv50/curs907a.o
+nouveau-y += dispnv50/cursc37a.o
+
+nouveau-y += dispnv50/oimm.o
+nouveau-y += dispnv50/oimm507b.o
+
+nouveau-y += dispnv50/ovly.o
+nouveau-y += dispnv50/ovly507e.o
+nouveau-y += dispnv50/ovly827e.o
+nouveau-y += dispnv50/ovly907e.o
+nouveau-y += dispnv50/ovly917e.o
diff --git a/drivers/gpu/drm/nouveau/dispnv50/atom.h b/drivers/gpu/drm/nouveau/dispnv50/atom.h
new file mode 100644
index 000000000000..908feb1fc60f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/atom.h
@@ -0,0 +1,222 @@
+#ifndef __NV50_KMS_ATOM_H__
+#define __NV50_KMS_ATOM_H__
+#define nv50_atom(p) container_of((p), struct nv50_atom, state)
+#include <drm/drm_atomic.h>
+
+struct nv50_atom {
+	struct drm_atomic_state state;
+
+	struct list_head outp;
+	bool lock_core;
+	bool flush_disable;
+};
+
+#define nv50_head_atom(p) container_of((p), struct nv50_head_atom, state)
+
+struct nv50_head_atom {
+	struct drm_crtc_state state;
+
+	struct {
+		u32 mask;
+		u32 olut;
+	} wndw;
+
+	struct {
+		u16 iW;
+		u16 iH;
+		u16 oW;
+		u16 oH;
+	} view;
+
+	struct nv50_head_mode {
+		bool interlace;
+		u32 clock;
+		struct {
+			u16 active;
+			u16 synce;
+			u16 blanke;
+			u16 blanks;
+		} h;
+		struct {
+			u32 active;
+			u16 synce;
+			u16 blanke;
+			u16 blanks;
+			u16 blank2s;
+			u16 blank2e;
+			u16 blankus;
+		} v;
+	} mode;
+
+	struct {
+		bool visible;
+		u32 handle;
+		u64 offset:40;
+		u8 buffer:1;
+		u8 mode:4;
+		u8 size:2;
+		u8 range:2;
+		u8 output_mode:2;
+	} olut;
+
+	struct {
+		bool visible;
+		u32 handle;
+		u64 offset:40;
+		u8  format;
+		u8  kind:7;
+		u8  layout:1;
+		u8  blockh:4;
+		u16 blocks:12;
+		u32 pitch:20;
+		u16 x;
+		u16 y;
+		u16 w;
+		u16 h;
+	} core;
+
+	struct {
+		bool visible;
+		u32 handle;
+		u64 offset:40;
+		u8  layout:2;
+		u8  format:8;
+	} curs;
+
+	struct {
+		u8  depth;
+		u8  cpp;
+		u16 x;
+		u16 y;
+		u16 w;
+		u16 h;
+	} base;
+
+	struct {
+		u8 cpp;
+	} ovly;
+
+	struct {
+		bool enable:1;
+		u8 bits:2;
+		u8 mode:4;
+	} dither;
+
+	struct {
+		struct {
+			u16 cos:12;
+			u16 sin:12;
+		} sat;
+	} procamp;
+
+	struct {
+		u8 nhsync:1;
+		u8 nvsync:1;
+		u8 depth:4;
+	} or;
+
+	union nv50_head_atom_mask {
+		struct {
+			bool olut:1;
+			bool core:1;
+			bool curs:1;
+			bool view:1;
+			bool mode:1;
+			bool base:1;
+			bool ovly:1;
+			bool dither:1;
+			bool procamp:1;
+			bool or:1;
+		};
+		u16 mask;
+	} set, clr;
+};
+
+static inline struct nv50_head_atom *
+nv50_head_atom_get(struct drm_atomic_state *state, struct drm_crtc *crtc)
+{
+	struct drm_crtc_state *statec = drm_atomic_get_crtc_state(state, crtc);
+	if (IS_ERR(statec))
+		return (void *)statec;
+	return nv50_head_atom(statec);
+}
+
+#define nv50_wndw_atom(p) container_of((p), struct nv50_wndw_atom, state)
+
+struct nv50_wndw_atom {
+	struct drm_plane_state state;
+
+	struct drm_property_blob *ilut;
+	bool visible;
+
+	struct {
+		u32  handle;
+		u16  offset:12;
+		bool awaken:1;
+	} ntfy;
+
+	struct {
+		u32 handle;
+		u16 offset:12;
+		u32 acquire;
+		u32 release;
+	} sema;
+
+	struct {
+		u32 handle;
+		struct {
+			u64 offset:40;
+			u8  buffer:1;
+			u8  enable:2;
+			u8  mode:4;
+			u8  size:2;
+			u8  range:2;
+			u8  output_mode:2;
+		} i;
+	} xlut;
+
+	struct {
+		u8  mode:2;
+		u8  interval:4;
+
+		u8  colorspace:2;
+		u8  format;
+		u8  kind:7;
+		u8  layout:1;
+		u8  blockh:4;
+		u16 blocks[3];
+		u32 pitch[3];
+		u16 w;
+		u16 h;
+
+		u32 handle[6];
+		u64 offset[6];
+	} image;
+
+	struct {
+		u16 sx;
+		u16 sy;
+		u16 sw;
+		u16 sh;
+		u16 dw;
+		u16 dh;
+	} scale;
+
+	struct {
+		u16 x;
+		u16 y;
+	} point;
+
+	union nv50_wndw_atom_mask {
+		struct {
+			bool ntfy:1;
+			bool sema:1;
+			bool xlut:1;
+			bool image:1;
+			bool scale:1;
+			bool point:1;
+		};
+		u8 mask;
+	} set, clr;
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv50/base.c b/drivers/gpu/drm/nouveau/dispnv50/base.c
new file mode 100644
index 000000000000..7c752acf2b48
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/base.c
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "base.h"
+
+#include <nvif/class.h>
+
+int
+nv50_base_new(struct nouveau_drm *drm, int head, struct nv50_wndw **pwndw)
+{
+	struct {
+		s32 oclass;
+		int version;
+		int (*new)(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+	} bases[] = {
+		{ GK110_DISP_BASE_CHANNEL_DMA, 0, base917c_new },
+		{ GK104_DISP_BASE_CHANNEL_DMA, 0, base917c_new },
+		{ GF110_DISP_BASE_CHANNEL_DMA, 0, base907c_new },
+		{ GT214_DISP_BASE_CHANNEL_DMA, 0, base827c_new },
+		{ GT200_DISP_BASE_CHANNEL_DMA, 0, base827c_new },
+		{   G82_DISP_BASE_CHANNEL_DMA, 0, base827c_new },
+		{  NV50_DISP_BASE_CHANNEL_DMA, 0, base507c_new },
+		{}
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	int cid;
+
+	cid = nvif_mclass(&disp->disp->object, bases);
+	if (cid < 0) {
+		NV_ERROR(drm, "No supported base class\n");
+		return cid;
+	}
+
+	return bases[cid].new(drm, head, bases[cid].oclass, pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/base.h b/drivers/gpu/drm/nouveau/dispnv50/base.h
new file mode 100644
index 000000000000..e7f14f230f35
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/base.h
@@ -0,0 +1,31 @@
+#ifndef __NV50_KMS_BASE_H__
+#define __NV50_KMS_BASE_H__
+#include "wndw.h"
+
+int base507c_new(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+int base507c_new_(const struct nv50_wndw_func *, const u32 *format,
+		  struct nouveau_drm *, int head, s32 oclass,
+		  u32 interlock_data, struct nv50_wndw **);
+extern const u32 base507c_format[];
+int base507c_acquire(struct nv50_wndw *, struct nv50_wndw_atom *,
+		     struct nv50_head_atom *);
+void base507c_release(struct nv50_wndw *, struct nv50_wndw_atom *,
+		      struct nv50_head_atom *);
+void base507c_sema_set(struct nv50_wndw *, struct nv50_wndw_atom *);
+void base507c_sema_clr(struct nv50_wndw *);
+void base507c_ntfy_set(struct nv50_wndw *, struct nv50_wndw_atom *);
+void base507c_ntfy_clr(struct nv50_wndw *);
+void base507c_xlut_set(struct nv50_wndw *, struct nv50_wndw_atom *);
+void base507c_xlut_clr(struct nv50_wndw *);
+void base507c_image_clr(struct nv50_wndw *);
+void base507c_update(struct nv50_wndw *, u32 *);
+
+int base827c_new(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+
+int base907c_new(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+extern const struct nv50_wndw_func base907c;
+
+int base917c_new(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+
+int nv50_base_new(struct nouveau_drm *, int head, struct nv50_wndw **);
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv50/base507c.c b/drivers/gpu/drm/nouveau/dispnv50/base507c.c
new file mode 100644
index 000000000000..d5e295ca2caa
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/base507c.c
@@ -0,0 +1,286 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "base.h"
+
+#include <nvif/cl507c.h>
+#include <nvif/event.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+#include "nouveau_bo.h"
+
+void
+base507c_update(struct nv50_wndw *wndw, u32 *interlock)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 2))) {
+		evo_mthd(push, 0x0080, 1);
+		evo_data(push, interlock[NV50_DISP_INTERLOCK_CORE]);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+base507c_image_clr(struct nv50_wndw *wndw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 4))) {
+		evo_mthd(push, 0x0084, 1);
+		evo_data(push, 0x00000000);
+		evo_mthd(push, 0x00c0, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+base507c_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 10))) {
+		evo_mthd(push, 0x0084, 1);
+		evo_data(push, asyw->image.mode << 8 |
+			       asyw->image.interval << 4);
+		evo_mthd(push, 0x00c0, 1);
+		evo_data(push, asyw->image.handle[0]);
+		evo_mthd(push, 0x0800, 5);
+		evo_data(push, asyw->image.offset[0] >> 8);
+		evo_data(push, 0x00000000);
+		evo_data(push, asyw->image.h << 16 | asyw->image.w);
+		evo_data(push, asyw->image.layout << 20 |
+			       (asyw->image.pitch[0] >> 8) << 8 |
+			       asyw->image.blocks[0] << 8 |
+			       asyw->image.blockh);
+		evo_data(push, asyw->image.kind << 16 |
+			       asyw->image.format << 8);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+base507c_xlut_clr(struct nv50_wndw *wndw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 2))) {
+		evo_mthd(push, 0x00e0, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+base507c_xlut_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 2))) {
+		evo_mthd(push, 0x00e0, 1);
+		evo_data(push, 0x40000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+int
+base507c_ntfy_wait_begun(struct nouveau_bo *bo, u32 offset,
+			 struct nvif_device *device)
+{
+	s64 time = nvif_msec(device, 2000ULL,
+		u32 data = nouveau_bo_rd32(bo, offset / 4);
+		if ((data & 0xc0000000) == 0x40000000)
+			break;
+		usleep_range(1, 2);
+	);
+	return time < 0 ? time : 0;
+}
+
+void
+base507c_ntfy_clr(struct nv50_wndw *wndw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 2))) {
+		evo_mthd(push, 0x00a4, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+base507c_ntfy_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 3))) {
+		evo_mthd(push, 0x00a0, 2);
+		evo_data(push, asyw->ntfy.awaken << 30 | asyw->ntfy.offset);
+		evo_data(push, asyw->ntfy.handle);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+base507c_ntfy_reset(struct nouveau_bo *bo, u32 offset)
+{
+	nouveau_bo_wr32(bo, offset / 4, 0x00000000);
+}
+
+void
+base507c_sema_clr(struct nv50_wndw *wndw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 2))) {
+		evo_mthd(push, 0x0094, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+base507c_sema_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 5))) {
+		evo_mthd(push, 0x0088, 4);
+		evo_data(push, asyw->sema.offset);
+		evo_data(push, asyw->sema.acquire);
+		evo_data(push, asyw->sema.release);
+		evo_data(push, asyw->sema.handle);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+base507c_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		 struct nv50_head_atom *asyh)
+{
+	asyh->base.cpp = 0;
+}
+
+int
+base507c_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		 struct nv50_head_atom *asyh)
+{
+	const struct drm_framebuffer *fb = asyw->state.fb;
+	int ret;
+
+	if (!fb->format->depth)
+		return -EINVAL;
+
+	ret = drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  false, true);
+	if (ret)
+		return ret;
+
+	if (!wndw->func->ilut) {
+		if ((asyh->base.cpp != 1) ^ (fb->format->cpp[0] != 1))
+			asyh->state.color_mgmt_changed = true;
+	}
+
+	asyh->base.depth = fb->format->depth;
+	asyh->base.cpp = fb->format->cpp[0];
+	asyh->base.x = asyw->state.src.x1 >> 16;
+	asyh->base.y = asyw->state.src.y1 >> 16;
+	asyh->base.w = asyw->state.fb->width;
+	asyh->base.h = asyw->state.fb->height;
+	return 0;
+}
+
+const u32
+base507c_format[] = {
+	DRM_FORMAT_C8,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_ABGR2101010,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ABGR8888,
+	0
+};
+
+static const struct nv50_wndw_func
+base507c = {
+	.acquire = base507c_acquire,
+	.release = base507c_release,
+	.sema_set = base507c_sema_set,
+	.sema_clr = base507c_sema_clr,
+	.ntfy_reset = base507c_ntfy_reset,
+	.ntfy_set = base507c_ntfy_set,
+	.ntfy_clr = base507c_ntfy_clr,
+	.ntfy_wait_begun = base507c_ntfy_wait_begun,
+	.olut_core = 1,
+	.xlut_set = base507c_xlut_set,
+	.xlut_clr = base507c_xlut_clr,
+	.image_set = base507c_image_set,
+	.image_clr = base507c_image_clr,
+	.update = base507c_update,
+};
+
+int
+base507c_new_(const struct nv50_wndw_func *func, const u32 *format,
+	      struct nouveau_drm *drm, int head, s32 oclass, u32 interlock_data,
+	      struct nv50_wndw **pwndw)
+{
+	struct nv50_disp_base_channel_dma_v0 args = {
+		.head = head,
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	struct nv50_wndw *wndw;
+	int ret;
+
+	ret = nv50_wndw_new_(func, drm->dev, DRM_PLANE_TYPE_PRIMARY,
+			     "base", head, format, BIT(head),
+			     NV50_DISP_INTERLOCK_BASE, interlock_data, &wndw);
+	if (*pwndw = wndw, ret)
+		return ret;
+
+	ret = nv50_dmac_create(&drm->client.device, &disp->disp->object,
+			       &oclass, head, &args, sizeof(args),
+			       disp->sync->bo.offset, &wndw->wndw);
+	if (ret) {
+		NV_ERROR(drm, "base%04x allocation failed: %d\n", oclass, ret);
+		return ret;
+	}
+
+	ret = nvif_notify_init(&wndw->wndw.base.user, wndw->notify.func,
+			       false, NV50_DISP_BASE_CHANNEL_DMA_V0_NTFY_UEVENT,
+			       &(struct nvif_notify_uevent_req) {},
+			       sizeof(struct nvif_notify_uevent_req),
+			       sizeof(struct nvif_notify_uevent_rep),
+			       &wndw->notify);
+	if (ret)
+		return ret;
+
+	wndw->ntfy = NV50_DISP_BASE_NTFY(wndw->id);
+	wndw->sema = NV50_DISP_BASE_SEM0(wndw->id);
+	wndw->data = 0x00000000;
+	return 0;
+}
+
+int
+base507c_new(struct nouveau_drm *drm, int head, s32 oclass,
+	     struct nv50_wndw **pwndw)
+{
+	return base507c_new_(&base507c, base507c_format, drm, head, oclass,
+			     0x00000002 << (head * 8), pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/base827c.c b/drivers/gpu/drm/nouveau/dispnv50/base827c.c
new file mode 100644
index 000000000000..73646819a0d6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/base827c.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "base.h"
+
+static void
+base827c_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 10))) {
+		evo_mthd(push, 0x0084, 1);
+		evo_data(push, asyw->image.mode << 8 |
+			       asyw->image.interval << 4);
+		evo_mthd(push, 0x00c0, 1);
+		evo_data(push, asyw->image.handle[0]);
+		evo_mthd(push, 0x0800, 5);
+		evo_data(push, asyw->image.offset[0] >> 8);
+		evo_data(push, 0x00000000);
+		evo_data(push, asyw->image.h << 16 | asyw->image.w);
+		evo_data(push, asyw->image.layout << 20 |
+			       (asyw->image.pitch[0] >> 8) << 8 |
+			       asyw->image.blocks[0] << 8 |
+			       asyw->image.blockh);
+		evo_data(push, asyw->image.format << 8);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static const struct nv50_wndw_func
+base827c = {
+	.acquire = base507c_acquire,
+	.release = base507c_release,
+	.sema_set = base507c_sema_set,
+	.sema_clr = base507c_sema_clr,
+	.ntfy_reset = base507c_ntfy_reset,
+	.ntfy_set = base507c_ntfy_set,
+	.ntfy_clr = base507c_ntfy_clr,
+	.ntfy_wait_begun = base507c_ntfy_wait_begun,
+	.olut_core = 1,
+	.xlut_set = base507c_xlut_set,
+	.xlut_clr = base507c_xlut_clr,
+	.image_set = base827c_image_set,
+	.image_clr = base507c_image_clr,
+	.update = base507c_update,
+};
+
+int
+base827c_new(struct nouveau_drm *drm, int head, s32 oclass,
+	     struct nv50_wndw **pwndw)
+{
+	return base507c_new_(&base827c, base507c_format, drm, head, oclass,
+			     0x00000002 << (head * 8), pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/base907c.c b/drivers/gpu/drm/nouveau/dispnv50/base907c.c
new file mode 100644
index 000000000000..a562fc94ce59
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/base907c.c
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "base.h"
+
+static void
+base907c_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 10))) {
+		evo_mthd(push, 0x0084, 1);
+		evo_data(push, asyw->image.mode << 8 |
+			       asyw->image.interval << 4);
+		evo_mthd(push, 0x00c0, 1);
+		evo_data(push, asyw->image.handle[0]);
+		evo_mthd(push, 0x0400, 5);
+		evo_data(push, asyw->image.offset[0] >> 8);
+		evo_data(push, 0x00000000);
+		evo_data(push, asyw->image.h << 16 | asyw->image.w);
+		evo_data(push, asyw->image.layout << 24 |
+			       (asyw->image.pitch[0] >> 8) << 8 |
+			       asyw->image.blocks[0] << 8 |
+			       asyw->image.blockh);
+		evo_data(push, asyw->image.format << 8);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+base907c_xlut_clr(struct nv50_wndw *wndw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 6))) {
+		evo_mthd(push, 0x00e0, 1);
+		evo_data(push, 0x00000000);
+		evo_mthd(push, 0x00e8, 1);
+		evo_data(push, 0x00000000);
+		evo_mthd(push, 0x00fc, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+base907c_xlut_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 6))) {
+		evo_mthd(push, 0x00e0, 3);
+		evo_data(push, asyw->xlut.i.enable << 30 |
+			       asyw->xlut.i.mode << 24);
+		evo_data(push, asyw->xlut.i.offset >> 8);
+		evo_data(push, 0x40000000);
+		evo_mthd(push, 0x00fc, 1);
+		evo_data(push, asyw->xlut.handle);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+base907c_ilut(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	asyw->xlut.i.mode = 7;
+	asyw->xlut.i.enable = 2;
+}
+
+const struct nv50_wndw_func
+base907c = {
+	.acquire = base507c_acquire,
+	.release = base507c_release,
+	.sema_set = base507c_sema_set,
+	.sema_clr = base507c_sema_clr,
+	.ntfy_reset = base507c_ntfy_reset,
+	.ntfy_set = base507c_ntfy_set,
+	.ntfy_clr = base507c_ntfy_clr,
+	.ntfy_wait_begun = base507c_ntfy_wait_begun,
+	.ilut = base907c_ilut,
+	.olut_core = true,
+	.xlut_set = base907c_xlut_set,
+	.xlut_clr = base907c_xlut_clr,
+	.image_set = base907c_image_set,
+	.image_clr = base507c_image_clr,
+	.update = base507c_update,
+};
+
+int
+base907c_new(struct nouveau_drm *drm, int head, s32 oclass,
+	     struct nv50_wndw **pwndw)
+{
+	return base507c_new_(&base907c, base507c_format, drm, head, oclass,
+			     0x00000002 << (head * 4), pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/base917c.c b/drivers/gpu/drm/nouveau/dispnv50/base917c.c
new file mode 100644
index 000000000000..54d705bb81a5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/base917c.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "base.h"
+#include "atom.h"
+
+const u32
+base917c_format[] = {
+	DRM_FORMAT_C8,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_ABGR2101010,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_ARGB2101010,
+	0
+};
+
+int
+base917c_new(struct nouveau_drm *drm, int head, s32 oclass,
+	     struct nv50_wndw **pwndw)
+{
+	return base507c_new_(&base907c, base917c_format, drm, head, oclass,
+			     0x00000002 << (head * 4), pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/core.c b/drivers/gpu/drm/nouveau/dispnv50/core.c
new file mode 100644
index 000000000000..f3c49adb1bdb
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/core.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+
+#include <nvif/class.h>
+
+void
+nv50_core_del(struct nv50_core **pcore)
+{
+	struct nv50_core *core = *pcore;
+	if (core) {
+		nv50_dmac_destroy(&core->chan);
+		kfree(*pcore);
+		*pcore = NULL;
+	}
+}
+
+int
+nv50_core_new(struct nouveau_drm *drm, struct nv50_core **pcore)
+{
+	struct {
+		s32 oclass;
+		int version;
+		int (*new)(struct nouveau_drm *, s32, struct nv50_core **);
+	} cores[] = {
+		{ GV100_DISP_CORE_CHANNEL_DMA, 0, corec37d_new },
+		{ GP102_DISP_CORE_CHANNEL_DMA, 0, core917d_new },
+		{ GP100_DISP_CORE_CHANNEL_DMA, 0, core917d_new },
+		{ GM200_DISP_CORE_CHANNEL_DMA, 0, core917d_new },
+		{ GM107_DISP_CORE_CHANNEL_DMA, 0, core917d_new },
+		{ GK110_DISP_CORE_CHANNEL_DMA, 0, core917d_new },
+		{ GK104_DISP_CORE_CHANNEL_DMA, 0, core917d_new },
+		{ GF110_DISP_CORE_CHANNEL_DMA, 0, core907d_new },
+		{ GT214_DISP_CORE_CHANNEL_DMA, 0, core827d_new },
+		{ GT206_DISP_CORE_CHANNEL_DMA, 0, core827d_new },
+		{ GT200_DISP_CORE_CHANNEL_DMA, 0, core827d_new },
+		{   G82_DISP_CORE_CHANNEL_DMA, 0, core827d_new },
+		{  NV50_DISP_CORE_CHANNEL_DMA, 0, core507d_new },
+		{}
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	int cid;
+
+	cid = nvif_mclass(&disp->disp->object, cores);
+	if (cid < 0) {
+		NV_ERROR(drm, "No supported core channel class\n");
+		return cid;
+	}
+
+	return cores[cid].new(drm, cores[cid].oclass, pcore);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/core.h b/drivers/gpu/drm/nouveau/dispnv50/core.h
new file mode 100644
index 000000000000..8470df9dd13d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/core.h
@@ -0,0 +1,50 @@
+#ifndef __NV50_KMS_CORE_H__
+#define __NV50_KMS_CORE_H__
+#include "disp.h"
+#include "atom.h"
+
+struct nv50_core {
+	const struct nv50_core_func *func;
+	struct nv50_dmac chan;
+};
+
+int nv50_core_new(struct nouveau_drm *, struct nv50_core **);
+void nv50_core_del(struct nv50_core **);
+
+struct nv50_core_func {
+	void (*init)(struct nv50_core *);
+	void (*ntfy_init)(struct nouveau_bo *, u32 offset);
+	int (*ntfy_wait_done)(struct nouveau_bo *, u32 offset,
+			      struct nvif_device *);
+	void (*update)(struct nv50_core *, u32 *interlock, bool ntfy);
+
+	const struct nv50_head_func *head;
+	const struct nv50_outp_func {
+		void (*ctrl)(struct nv50_core *, int or, u32 ctrl,
+			     struct nv50_head_atom *);
+	} *dac, *pior, *sor;
+};
+
+int core507d_new(struct nouveau_drm *, s32, struct nv50_core **);
+int core507d_new_(const struct nv50_core_func *, struct nouveau_drm *, s32,
+		  struct nv50_core **);
+void core507d_init(struct nv50_core *);
+void core507d_ntfy_init(struct nouveau_bo *, u32);
+int core507d_ntfy_wait_done(struct nouveau_bo *, u32, struct nvif_device *);
+void core507d_update(struct nv50_core *, u32 *, bool);
+
+extern const struct nv50_outp_func dac507d;
+extern const struct nv50_outp_func sor507d;
+extern const struct nv50_outp_func pior507d;
+
+int core827d_new(struct nouveau_drm *, s32, struct nv50_core **);
+
+int core907d_new(struct nouveau_drm *, s32, struct nv50_core **);
+extern const struct nv50_outp_func dac907d;
+extern const struct nv50_outp_func sor907d;
+
+int core917d_new(struct nouveau_drm *, s32, struct nv50_core **);
+
+int corec37d_new(struct nouveau_drm *, s32, struct nv50_core **);
+extern const struct nv50_outp_func sorc37d;
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv50/core507d.c b/drivers/gpu/drm/nouveau/dispnv50/core507d.c
new file mode 100644
index 000000000000..e7fcfa6e6467
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/core507d.c
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+#include "head.h"
+
+#include <nvif/cl507d.h>
+
+#include "nouveau_bo.h"
+
+void
+core507d_update(struct nv50_core *core, u32 *interlock, bool ntfy)
+{
+	u32 *push;
+	if ((push = evo_wait(&core->chan, 5))) {
+		if (ntfy) {
+			evo_mthd(push, 0x0084, 1);
+			evo_data(push, 0x80000000 | NV50_DISP_CORE_NTFY);
+		}
+		evo_mthd(push, 0x0080, 2);
+		evo_data(push, interlock[NV50_DISP_INTERLOCK_BASE] |
+			       interlock[NV50_DISP_INTERLOCK_OVLY]);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &core->chan);
+	}
+}
+
+int
+core507d_ntfy_wait_done(struct nouveau_bo *bo, u32 offset,
+			struct nvif_device *device)
+{
+	s64 time = nvif_msec(device, 2000ULL,
+		if (nouveau_bo_rd32(bo, offset / 4))
+			break;
+		usleep_range(1, 2);
+	);
+	return time < 0 ? time : 0;
+}
+
+void
+core507d_ntfy_init(struct nouveau_bo *bo, u32 offset)
+{
+	nouveau_bo_wr32(bo, offset / 4, 0x00000000);
+}
+
+void
+core507d_init(struct nv50_core *core)
+{
+	u32 *push;
+	if ((push = evo_wait(&core->chan, 2))) {
+		evo_mthd(push, 0x0088, 1);
+		evo_data(push, core->chan.sync.handle);
+		evo_kick(push, &core->chan);
+	}
+}
+
+static const struct nv50_core_func
+core507d = {
+	.init = core507d_init,
+	.ntfy_init = core507d_ntfy_init,
+	.ntfy_wait_done = core507d_ntfy_wait_done,
+	.update = core507d_update,
+	.head = &head507d,
+	.dac = &dac507d,
+	.sor = &sor507d,
+	.pior = &pior507d,
+};
+
+int
+core507d_new_(const struct nv50_core_func *func, struct nouveau_drm *drm,
+	      s32 oclass, struct nv50_core **pcore)
+{
+	struct nv50_disp_core_channel_dma_v0 args = {};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	struct nv50_core *core;
+	int ret;
+
+	if (!(core = *pcore = kzalloc(sizeof(*core), GFP_KERNEL)))
+		return -ENOMEM;
+	core->func = func;
+
+	ret = nv50_dmac_create(&drm->client.device, &disp->disp->object,
+			       &oclass, 0, &args, sizeof(args),
+			       disp->sync->bo.offset, &core->chan);
+	if (ret) {
+		NV_ERROR(drm, "core%04x allocation failed: %d\n", oclass, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int
+core507d_new(struct nouveau_drm *drm, s32 oclass, struct nv50_core **pcore)
+{
+	return core507d_new_(&core507d, drm, oclass, pcore);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/core827d.c b/drivers/gpu/drm/nouveau/dispnv50/core827d.c
new file mode 100644
index 000000000000..6123a068f836
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/core827d.c
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+#include "head.h"
+
+static const struct nv50_core_func
+core827d = {
+	.init = core507d_init,
+	.ntfy_init = core507d_ntfy_init,
+	.ntfy_wait_done = core507d_ntfy_wait_done,
+	.update = core507d_update,
+	.head = &head827d,
+	.dac = &dac507d,
+	.sor = &sor507d,
+	.pior = &pior507d,
+};
+
+int
+core827d_new(struct nouveau_drm *drm, s32 oclass, struct nv50_core **pcore)
+{
+	return core507d_new_(&core827d, drm, oclass, pcore);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/core907d.c b/drivers/gpu/drm/nouveau/dispnv50/core907d.c
new file mode 100644
index 000000000000..ef822f813435
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/core907d.c
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+#include "head.h"
+
+static const struct nv50_core_func
+core907d = {
+	.init = core507d_init,
+	.ntfy_init = core507d_ntfy_init,
+	.ntfy_wait_done = core507d_ntfy_wait_done,
+	.update = core507d_update,
+	.head = &head907d,
+	.dac = &dac907d,
+	.sor = &sor907d,
+};
+
+int
+core907d_new(struct nouveau_drm *drm, s32 oclass, struct nv50_core **pcore)
+{
+	return core507d_new_(&core907d, drm, oclass, pcore);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/core917d.c b/drivers/gpu/drm/nouveau/dispnv50/core917d.c
new file mode 100644
index 000000000000..392338df5bfd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/core917d.c
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+#include "head.h"
+
+static const struct nv50_core_func
+core917d = {
+	.init = core507d_init,
+	.ntfy_init = core507d_ntfy_init,
+	.ntfy_wait_done = core507d_ntfy_wait_done,
+	.update = core507d_update,
+	.head = &head917d,
+	.dac = &dac907d,
+	.sor = &sor907d,
+};
+
+int
+core917d_new(struct nouveau_drm *drm, s32 oclass, struct nv50_core **pcore)
+{
+	return core507d_new_(&core917d, drm, oclass, pcore);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/corec37d.c b/drivers/gpu/drm/nouveau/dispnv50/corec37d.c
new file mode 100644
index 000000000000..b5c17c948918
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/corec37d.c
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+#include "head.h"
+
+#include <nouveau_bo.h>
+
+static void
+corec37d_update(struct nv50_core *core, u32 *interlock, bool ntfy)
+{
+	u32 *push;
+	if ((push = evo_wait(&core->chan, 9))) {
+		if (ntfy) {
+			evo_mthd(push, 0x020c, 1);
+			evo_data(push, 0x00001000 | NV50_DISP_CORE_NTFY);
+		}
+
+		evo_mthd(push, 0x0218, 2);
+		evo_data(push, interlock[NV50_DISP_INTERLOCK_CURS]);
+		evo_data(push, interlock[NV50_DISP_INTERLOCK_WNDW]);
+		evo_mthd(push, 0x0200, 1);
+		evo_data(push, 0x00000001);
+
+		if (ntfy) {
+			evo_mthd(push, 0x020c, 1);
+			evo_data(push, 0x00000000);
+		}
+		evo_kick(push, &core->chan);
+	}
+}
+
+int
+corec37d_ntfy_wait_done(struct nouveau_bo *bo, u32 offset,
+			struct nvif_device *device)
+{
+	u32 data;
+	s64 time = nvif_msec(device, 2000ULL,
+		data = nouveau_bo_rd32(bo, offset / 4 + 0);
+		if ((data & 0xc0000000) == 0x80000000)
+			break;
+		usleep_range(1, 2);
+	);
+	return time < 0 ? time : 0;
+}
+
+void
+corec37d_ntfy_init(struct nouveau_bo *bo, u32 offset)
+{
+	nouveau_bo_wr32(bo, offset / 4 + 0, 0x00000000);
+	nouveau_bo_wr32(bo, offset / 4 + 1, 0x00000000);
+	nouveau_bo_wr32(bo, offset / 4 + 2, 0x00000000);
+	nouveau_bo_wr32(bo, offset / 4 + 3, 0x00000000);
+}
+
+void
+corec37d_init(struct nv50_core *core)
+{
+	const u32 windows = 8; /*XXX*/
+	u32 *push, i;
+	if ((push = evo_wait(&core->chan, 2 + 6 * windows + 2))) {
+		evo_mthd(push, 0x0208, 1);
+		evo_data(push, core->chan.sync.handle);
+		for (i = 0; i < windows; i++) {
+			evo_mthd(push, 0x1000 + (i * 0x080), 3);
+			evo_data(push, i >> 1);
+			evo_data(push, 0x00000017);
+			evo_data(push, 0x00000000);
+			evo_mthd(push, 0x1010 + (i * 0x080), 1);
+			evo_data(push, 0x00127fff);
+		}
+		evo_mthd(push, 0x0200, 1);
+		evo_data(push, 0x00000001);
+		evo_kick(push, &core->chan);
+	}
+}
+
+static const struct nv50_core_func
+corec37d = {
+	.init = corec37d_init,
+	.ntfy_init = corec37d_ntfy_init,
+	.ntfy_wait_done = corec37d_ntfy_wait_done,
+	.update = corec37d_update,
+	.head = &headc37d,
+	.sor = &sorc37d,
+};
+
+int
+corec37d_new(struct nouveau_drm *drm, s32 oclass, struct nv50_core **pcore)
+{
+	return core507d_new_(&corec37d, drm, oclass, pcore);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/curs.c b/drivers/gpu/drm/nouveau/dispnv50/curs.c
new file mode 100644
index 000000000000..f592087338c4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/curs.c
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "curs.h"
+
+#include <nvif/class.h>
+
+int
+nv50_curs_new(struct nouveau_drm *drm, int head, struct nv50_wndw **pwndw)
+{
+	struct {
+		s32 oclass;
+		int version;
+		int (*new)(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+	} curses[] = {
+		{ GV100_DISP_CURSOR, 0, cursc37a_new },
+		{ GK104_DISP_CURSOR, 0, curs907a_new },
+		{ GF110_DISP_CURSOR, 0, curs907a_new },
+		{ GT214_DISP_CURSOR, 0, curs507a_new },
+		{   G82_DISP_CURSOR, 0, curs507a_new },
+		{  NV50_DISP_CURSOR, 0, curs507a_new },
+		{}
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	int cid;
+
+	cid = nvif_mclass(&disp->disp->object, curses);
+	if (cid < 0) {
+		NV_ERROR(drm, "No supported cursor immediate class\n");
+		return cid;
+	}
+
+	return curses[cid].new(drm, head, curses[cid].oclass, pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/curs.h b/drivers/gpu/drm/nouveau/dispnv50/curs.h
new file mode 100644
index 000000000000..23aff5fd6747
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/curs.h
@@ -0,0 +1,14 @@
+#ifndef __NV50_KMS_CURS_H__
+#define __NV50_KMS_CURS_H__
+#include "wndw.h"
+
+int curs507a_new(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+int curs507a_new_(const struct nv50_wimm_func *, struct nouveau_drm *,
+		  int head, s32 oclass, u32 interlock_data,
+		  struct nv50_wndw **);
+
+int curs907a_new(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+int cursc37a_new(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+
+int nv50_curs_new(struct nouveau_drm *, int head, struct nv50_wndw **);
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv50/curs507a.c b/drivers/gpu/drm/nouveau/dispnv50/curs507a.c
new file mode 100644
index 000000000000..291c08117ab6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/curs507a.c
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "curs.h"
+#include "core.h"
+#include "head.h"
+
+#include <nvif/cl507a.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+
+static void
+curs507a_update(struct nv50_wndw *wndw, u32 *interlock)
+{
+	nvif_wr32(&wndw->wimm.base.user, 0x0080, 0x00000000);
+}
+
+static void
+curs507a_point(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	nvif_wr32(&wndw->wimm.base.user, 0x0084, asyw->point.y << 16 |
+						 asyw->point.x);
+}
+
+const struct nv50_wimm_func
+curs507a = {
+	.point = curs507a_point,
+	.update = curs507a_update,
+};
+
+static void
+curs507a_prepare(struct nv50_wndw *wndw, struct nv50_head_atom *asyh,
+		 struct nv50_wndw_atom *asyw)
+{
+	u32 handle = nv50_disp(wndw->plane.dev)->core->chan.vram.handle;
+	u32 offset = asyw->image.offset[0];
+	if (asyh->curs.handle != handle || asyh->curs.offset != offset) {
+		asyh->curs.handle = handle;
+		asyh->curs.offset = offset;
+		asyh->set.curs = asyh->curs.visible;
+	}
+}
+
+static void
+curs507a_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		 struct nv50_head_atom *asyh)
+{
+	asyh->curs.visible = false;
+}
+
+static int
+curs507a_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		 struct nv50_head_atom *asyh)
+{
+	struct nv50_head *head = nv50_head(asyw->state.crtc);
+	int ret;
+
+	ret = drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  true, true);
+	asyh->curs.visible = asyw->state.visible;
+	if (ret || !asyh->curs.visible)
+		return ret;
+
+	if (asyw->image.w != asyw->image.h)
+		return -EINVAL;
+
+	ret = head->func->curs_layout(head, asyw, asyh);
+	if (ret)
+		return ret;
+
+	return head->func->curs_format(head, asyw, asyh);
+}
+
+static const u32
+curs507a_format[] = {
+	DRM_FORMAT_ARGB8888,
+	0
+};
+
+static const struct nv50_wndw_func
+curs507a_wndw = {
+	.acquire = curs507a_acquire,
+	.release = curs507a_release,
+	.prepare = curs507a_prepare,
+};
+
+int
+curs507a_new_(const struct nv50_wimm_func *func, struct nouveau_drm *drm,
+	      int head, s32 oclass, u32 interlock_data,
+	      struct nv50_wndw **pwndw)
+{
+	struct nv50_disp_cursor_v0 args = {
+		.head = head,
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	struct nv50_wndw *wndw;
+	int ret;
+
+	ret = nv50_wndw_new_(&curs507a_wndw, drm->dev, DRM_PLANE_TYPE_CURSOR,
+			     "curs", head, curs507a_format, BIT(head),
+			     NV50_DISP_INTERLOCK_CURS, interlock_data, &wndw);
+	if (*pwndw = wndw, ret)
+		return ret;
+
+	ret = nvif_object_init(&disp->disp->object, 0, oclass, &args,
+			       sizeof(args), &wndw->wimm.base.user);
+	if (ret) {
+		NV_ERROR(drm, "curs%04x allocation failed: %d\n", oclass, ret);
+		return ret;
+	}
+
+	nvif_object_map(&wndw->wimm.base.user, NULL, 0);
+	wndw->immd = func;
+	wndw->ctxdma.parent = &disp->core->chan.base.user;
+	return 0;
+}
+
+int
+curs507a_new(struct nouveau_drm *drm, int head, s32 oclass,
+	     struct nv50_wndw **pwndw)
+{
+	return curs507a_new_(&curs507a, drm, head, oclass,
+			     0x00000001 << (head * 8), pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/curs907a.c b/drivers/gpu/drm/nouveau/dispnv50/curs907a.c
new file mode 100644
index 000000000000..d742362de03e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/curs907a.c
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "curs.h"
+
+int
+curs907a_new(struct nouveau_drm *drm, int head, s32 oclass,
+	     struct nv50_wndw **pwndw)
+{
+	return curs507a_new_(&curs507a, drm, head, oclass,
+			     0x00000001 << (head * 4), pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/cursc37a.c b/drivers/gpu/drm/nouveau/dispnv50/cursc37a.c
new file mode 100644
index 000000000000..23fb29d41efe
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/cursc37a.c
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "curs.h"
+#include "atom.h"
+
+static void
+cursc37a_update(struct nv50_wndw *wndw, u32 *interlock)
+{
+	nvif_wr32(&wndw->wimm.base.user, 0x0200, 0x00000001);
+}
+
+static void
+cursc37a_point(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	nvif_wr32(&wndw->wimm.base.user, 0x0208, asyw->point.y << 16 |
+						 asyw->point.x);
+}
+
+static const struct nv50_wimm_func
+cursc37a = {
+	.point = cursc37a_point,
+	.update = cursc37a_update,
+};
+
+int
+cursc37a_new(struct nouveau_drm *drm, int head, s32 oclass,
+	     struct nv50_wndw **pwndw)
+{
+	return curs507a_new_(&cursc37a, drm, head, oclass,
+			     0x00000001 << head, pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/dac507d.c b/drivers/gpu/drm/nouveau/dispnv50/dac507d.c
new file mode 100644
index 000000000000..2a10ef7d30a8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/dac507d.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+
+static void
+dac507d_ctrl(struct nv50_core *core, int or, u32 ctrl,
+	     struct nv50_head_atom *asyh)
+{
+	u32 *push, sync = 0;
+	if ((push = evo_wait(&core->chan, 3))) {
+		if (asyh) {
+			sync |= asyh->or.nvsync << 1;
+			sync |= asyh->or.nhsync;
+		}
+		evo_mthd(push, 0x0400 + (or * 0x080), 2);
+		evo_data(push, ctrl);
+		evo_data(push, sync);
+		evo_kick(push, &core->chan);
+	}
+}
+
+const struct nv50_outp_func
+dac507d = {
+	.ctrl = dac507d_ctrl,
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv50/dac907d.c b/drivers/gpu/drm/nouveau/dispnv50/dac907d.c
new file mode 100644
index 000000000000..11e87fa53fac
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/dac907d.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+
+static void
+dac907d_ctrl(struct nv50_core *core, int or, u32 ctrl,
+	     struct nv50_head_atom *asyh)
+{
+	u32 *push;
+	if ((push = evo_wait(&core->chan, 2))) {
+		evo_mthd(push, 0x0180 + (or * 0x020), 1);
+		evo_data(push, ctrl);
+		evo_kick(push, &core->chan);
+	}
+}
+
+const struct nv50_outp_func
+dac907d = {
+	.ctrl = dac907d_ctrl,
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c
new file mode 100644
index 000000000000..b83465ae7c1b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c
@@ -0,0 +1,2238 @@
+/*
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include "disp.h"
+#include "atom.h"
+#include "core.h"
+#include "head.h"
+#include "wndw.h"
+
+#include <linux/dma-mapping.h>
+#include <linux/hdmi.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_dp_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_edid.h>
+
+#include <nvif/class.h>
+#include <nvif/cl0002.h>
+#include <nvif/cl5070.h>
+#include <nvif/cl507d.h>
+#include <nvif/event.h>
+
+#include "nouveau_drv.h"
+#include "nouveau_dma.h"
+#include "nouveau_gem.h"
+#include "nouveau_connector.h"
+#include "nouveau_encoder.h"
+#include "nouveau_fence.h"
+#include "nouveau_fbcon.h"
+
+#include <subdev/bios/dp.h>
+
+/******************************************************************************
+ * Atomic state
+ *****************************************************************************/
+
+struct nv50_outp_atom {
+	struct list_head head;
+
+	struct drm_encoder *encoder;
+	bool flush_disable;
+
+	union nv50_outp_atom_mask {
+		struct {
+			bool ctrl:1;
+		};
+		u8 mask;
+	} set, clr;
+};
+
+/******************************************************************************
+ * EVO channel
+ *****************************************************************************/
+
+static int
+nv50_chan_create(struct nvif_device *device, struct nvif_object *disp,
+		 const s32 *oclass, u8 head, void *data, u32 size,
+		 struct nv50_chan *chan)
+{
+	struct nvif_sclass *sclass;
+	int ret, i, n;
+
+	chan->device = device;
+
+	ret = n = nvif_object_sclass_get(disp, &sclass);
+	if (ret < 0)
+		return ret;
+
+	while (oclass[0]) {
+		for (i = 0; i < n; i++) {
+			if (sclass[i].oclass == oclass[0]) {
+				ret = nvif_object_init(disp, 0, oclass[0],
+						       data, size, &chan->user);
+				if (ret == 0)
+					nvif_object_map(&chan->user, NULL, 0);
+				nvif_object_sclass_put(&sclass);
+				return ret;
+			}
+		}
+		oclass++;
+	}
+
+	nvif_object_sclass_put(&sclass);
+	return -ENOSYS;
+}
+
+static void
+nv50_chan_destroy(struct nv50_chan *chan)
+{
+	nvif_object_fini(&chan->user);
+}
+
+/******************************************************************************
+ * DMA EVO channel
+ *****************************************************************************/
+
+void
+nv50_dmac_destroy(struct nv50_dmac *dmac)
+{
+	nvif_object_fini(&dmac->vram);
+	nvif_object_fini(&dmac->sync);
+
+	nv50_chan_destroy(&dmac->base);
+
+	nvif_mem_fini(&dmac->push);
+}
+
+int
+nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
+		 const s32 *oclass, u8 head, void *data, u32 size, u64 syncbuf,
+		 struct nv50_dmac *dmac)
+{
+	struct nouveau_cli *cli = (void *)device->object.client;
+	struct nv50_disp_core_channel_dma_v0 *args = data;
+	int ret;
+
+	mutex_init(&dmac->lock);
+
+	ret = nvif_mem_init_map(&cli->mmu, NVIF_MEM_COHERENT, 0x1000,
+				&dmac->push);
+	if (ret)
+		return ret;
+
+	dmac->ptr = dmac->push.object.map.ptr;
+
+	args->pushbuf = nvif_handle(&dmac->push.object);
+
+	ret = nv50_chan_create(device, disp, oclass, head, data, size,
+			       &dmac->base);
+	if (ret)
+		return ret;
+
+	if (!syncbuf)
+		return 0;
+
+	ret = nvif_object_init(&dmac->base.user, 0xf0000000, NV_DMA_IN_MEMORY,
+			       &(struct nv_dma_v0) {
+					.target = NV_DMA_V0_TARGET_VRAM,
+					.access = NV_DMA_V0_ACCESS_RDWR,
+					.start = syncbuf + 0x0000,
+					.limit = syncbuf + 0x0fff,
+			       }, sizeof(struct nv_dma_v0),
+			       &dmac->sync);
+	if (ret)
+		return ret;
+
+	ret = nvif_object_init(&dmac->base.user, 0xf0000001, NV_DMA_IN_MEMORY,
+			       &(struct nv_dma_v0) {
+					.target = NV_DMA_V0_TARGET_VRAM,
+					.access = NV_DMA_V0_ACCESS_RDWR,
+					.start = 0,
+					.limit = device->info.ram_user - 1,
+			       }, sizeof(struct nv_dma_v0),
+			       &dmac->vram);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+/******************************************************************************
+ * EVO channel helpers
+ *****************************************************************************/
+u32 *
+evo_wait(struct nv50_dmac *evoc, int nr)
+{
+	struct nv50_dmac *dmac = evoc;
+	struct nvif_device *device = dmac->base.device;
+	u32 put = nvif_rd32(&dmac->base.user, 0x0000) / 4;
+
+	mutex_lock(&dmac->lock);
+	if (put + nr >= (PAGE_SIZE / 4) - 8) {
+		dmac->ptr[put] = 0x20000000;
+
+		nvif_wr32(&dmac->base.user, 0x0000, 0x00000000);
+		if (nvif_msec(device, 2000,
+			if (!nvif_rd32(&dmac->base.user, 0x0004))
+				break;
+		) < 0) {
+			mutex_unlock(&dmac->lock);
+			pr_err("nouveau: evo channel stalled\n");
+			return NULL;
+		}
+
+		put = 0;
+	}
+
+	return dmac->ptr + put;
+}
+
+void
+evo_kick(u32 *push, struct nv50_dmac *evoc)
+{
+	struct nv50_dmac *dmac = evoc;
+	nvif_wr32(&dmac->base.user, 0x0000, (push - dmac->ptr) << 2);
+	mutex_unlock(&dmac->lock);
+}
+
+/******************************************************************************
+ * Output path helpers
+ *****************************************************************************/
+static void
+nv50_outp_release(struct nouveau_encoder *nv_encoder)
+{
+	struct nv50_disp *disp = nv50_disp(nv_encoder->base.base.dev);
+	struct {
+		struct nv50_disp_mthd_v1 base;
+	} args = {
+		.base.version = 1,
+		.base.method = NV50_DISP_MTHD_V1_RELEASE,
+		.base.hasht  = nv_encoder->dcb->hasht,
+		.base.hashm  = nv_encoder->dcb->hashm,
+	};
+
+	nvif_mthd(&disp->disp->object, 0, &args, sizeof(args));
+	nv_encoder->or = -1;
+	nv_encoder->link = 0;
+}
+
+static int
+nv50_outp_acquire(struct nouveau_encoder *nv_encoder)
+{
+	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	struct {
+		struct nv50_disp_mthd_v1 base;
+		struct nv50_disp_acquire_v0 info;
+	} args = {
+		.base.version = 1,
+		.base.method = NV50_DISP_MTHD_V1_ACQUIRE,
+		.base.hasht  = nv_encoder->dcb->hasht,
+		.base.hashm  = nv_encoder->dcb->hashm,
+	};
+	int ret;
+
+	ret = nvif_mthd(&disp->disp->object, 0, &args, sizeof(args));
+	if (ret) {
+		NV_ERROR(drm, "error acquiring output path: %d\n", ret);
+		return ret;
+	}
+
+	nv_encoder->or = args.info.or;
+	nv_encoder->link = args.info.link;
+	return 0;
+}
+
+static int
+nv50_outp_atomic_check_view(struct drm_encoder *encoder,
+			    struct drm_crtc_state *crtc_state,
+			    struct drm_connector_state *conn_state,
+			    struct drm_display_mode *native_mode)
+{
+	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
+	struct drm_display_mode *mode = &crtc_state->mode;
+	struct drm_connector *connector = conn_state->connector;
+	struct nouveau_conn_atom *asyc = nouveau_conn_atom(conn_state);
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
+
+	NV_ATOMIC(drm, "%s atomic_check\n", encoder->name);
+	asyc->scaler.full = false;
+	if (!native_mode)
+		return 0;
+
+	if (asyc->scaler.mode == DRM_MODE_SCALE_NONE) {
+		switch (connector->connector_type) {
+		case DRM_MODE_CONNECTOR_LVDS:
+		case DRM_MODE_CONNECTOR_eDP:
+			/* Force use of scaler for non-EDID modes. */
+			if (adjusted_mode->type & DRM_MODE_TYPE_DRIVER)
+				break;
+			mode = native_mode;
+			asyc->scaler.full = true;
+			break;
+		default:
+			break;
+		}
+	} else {
+		mode = native_mode;
+	}
+
+	if (!drm_mode_equal(adjusted_mode, mode)) {
+		drm_mode_copy(adjusted_mode, mode);
+		crtc_state->mode_changed = true;
+	}
+
+	return 0;
+}
+
+static int
+nv50_outp_atomic_check(struct drm_encoder *encoder,
+		       struct drm_crtc_state *crtc_state,
+		       struct drm_connector_state *conn_state)
+{
+	struct nouveau_connector *nv_connector =
+		nouveau_connector(conn_state->connector);
+	return nv50_outp_atomic_check_view(encoder, crtc_state, conn_state,
+					   nv_connector->native_mode);
+}
+
+/******************************************************************************
+ * DAC
+ *****************************************************************************/
+static void
+nv50_dac_disable(struct drm_encoder *encoder)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nv50_core *core = nv50_disp(encoder->dev)->core;
+	if (nv_encoder->crtc)
+		core->func->dac->ctrl(core, nv_encoder->or, 0x00000000, NULL);
+	nv_encoder->crtc = NULL;
+	nv50_outp_release(nv_encoder);
+}
+
+static void
+nv50_dac_enable(struct drm_encoder *encoder)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
+	struct nv50_head_atom *asyh = nv50_head_atom(nv_crtc->base.state);
+	struct nv50_core *core = nv50_disp(encoder->dev)->core;
+
+	nv50_outp_acquire(nv_encoder);
+
+	core->func->dac->ctrl(core, nv_encoder->or, 1 << nv_crtc->index, asyh);
+	asyh->or.depth = 0;
+
+	nv_encoder->crtc = encoder->crtc;
+}
+
+static enum drm_connector_status
+nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nv50_disp *disp = nv50_disp(encoder->dev);
+	struct {
+		struct nv50_disp_mthd_v1 base;
+		struct nv50_disp_dac_load_v0 load;
+	} args = {
+		.base.version = 1,
+		.base.method = NV50_DISP_MTHD_V1_DAC_LOAD,
+		.base.hasht  = nv_encoder->dcb->hasht,
+		.base.hashm  = nv_encoder->dcb->hashm,
+	};
+	int ret;
+
+	args.load.data = nouveau_drm(encoder->dev)->vbios.dactestval;
+	if (args.load.data == 0)
+		args.load.data = 340;
+
+	ret = nvif_mthd(&disp->disp->object, 0, &args, sizeof(args));
+	if (ret || !args.load.load)
+		return connector_status_disconnected;
+
+	return connector_status_connected;
+}
+
+static const struct drm_encoder_helper_funcs
+nv50_dac_help = {
+	.atomic_check = nv50_outp_atomic_check,
+	.enable = nv50_dac_enable,
+	.disable = nv50_dac_disable,
+	.detect = nv50_dac_detect
+};
+
+static void
+nv50_dac_destroy(struct drm_encoder *encoder)
+{
+	drm_encoder_cleanup(encoder);
+	kfree(encoder);
+}
+
+static const struct drm_encoder_funcs
+nv50_dac_func = {
+	.destroy = nv50_dac_destroy,
+};
+
+static int
+nv50_dac_create(struct drm_connector *connector, struct dcb_output *dcbe)
+{
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
+	struct nvkm_i2c *i2c = nvxx_i2c(&drm->client.device);
+	struct nvkm_i2c_bus *bus;
+	struct nouveau_encoder *nv_encoder;
+	struct drm_encoder *encoder;
+	int type = DRM_MODE_ENCODER_DAC;
+
+	nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
+	if (!nv_encoder)
+		return -ENOMEM;
+	nv_encoder->dcb = dcbe;
+
+	bus = nvkm_i2c_bus_find(i2c, dcbe->i2c_index);
+	if (bus)
+		nv_encoder->i2c = &bus->i2c;
+
+	encoder = to_drm_encoder(nv_encoder);
+	encoder->possible_crtcs = dcbe->heads;
+	encoder->possible_clones = 0;
+	drm_encoder_init(connector->dev, encoder, &nv50_dac_func, type,
+			 "dac-%04x-%04x", dcbe->hasht, dcbe->hashm);
+	drm_encoder_helper_add(encoder, &nv50_dac_help);
+
+	drm_mode_connector_attach_encoder(connector, encoder);
+	return 0;
+}
+
+/******************************************************************************
+ * Audio
+ *****************************************************************************/
+static void
+nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nv50_disp *disp = nv50_disp(encoder->dev);
+	struct {
+		struct nv50_disp_mthd_v1 base;
+		struct nv50_disp_sor_hda_eld_v0 eld;
+	} args = {
+		.base.version = 1,
+		.base.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
+		.base.hasht   = nv_encoder->dcb->hasht,
+		.base.hashm   = (0xf0ff & nv_encoder->dcb->hashm) |
+				(0x0100 << nv_crtc->index),
+	};
+
+	nvif_mthd(&disp->disp->object, 0, &args, sizeof(args));
+}
+
+static void
+nv50_audio_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
+	struct nouveau_connector *nv_connector;
+	struct nv50_disp *disp = nv50_disp(encoder->dev);
+	struct __packed {
+		struct {
+			struct nv50_disp_mthd_v1 mthd;
+			struct nv50_disp_sor_hda_eld_v0 eld;
+		} base;
+		u8 data[sizeof(nv_connector->base.eld)];
+	} args = {
+		.base.mthd.version = 1,
+		.base.mthd.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
+		.base.mthd.hasht   = nv_encoder->dcb->hasht,
+		.base.mthd.hashm   = (0xf0ff & nv_encoder->dcb->hashm) |
+				     (0x0100 << nv_crtc->index),
+	};
+
+	nv_connector = nouveau_encoder_connector_get(nv_encoder);
+	if (!drm_detect_monitor_audio(nv_connector->edid))
+		return;
+
+	memcpy(args.data, nv_connector->base.eld, sizeof(args.data));
+
+	nvif_mthd(&disp->disp->object, 0, &args,
+		  sizeof(args.base) + drm_eld_size(args.data));
+}
+
+/******************************************************************************
+ * HDMI
+ *****************************************************************************/
+static void
+nv50_hdmi_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nv50_disp *disp = nv50_disp(encoder->dev);
+	struct {
+		struct nv50_disp_mthd_v1 base;
+		struct nv50_disp_sor_hdmi_pwr_v0 pwr;
+	} args = {
+		.base.version = 1,
+		.base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
+		.base.hasht  = nv_encoder->dcb->hasht,
+		.base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
+			       (0x0100 << nv_crtc->index),
+	};
+
+	nvif_mthd(&disp->disp->object, 0, &args, sizeof(args));
+}
+
+static void
+nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
+	struct nv50_disp *disp = nv50_disp(encoder->dev);
+	struct {
+		struct nv50_disp_mthd_v1 base;
+		struct nv50_disp_sor_hdmi_pwr_v0 pwr;
+		u8 infoframes[2 * 17]; /* two frames, up to 17 bytes each */
+	} args = {
+		.base.version = 1,
+		.base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
+		.base.hasht  = nv_encoder->dcb->hasht,
+		.base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
+			       (0x0100 << nv_crtc->index),
+		.pwr.state = 1,
+		.pwr.rekey = 56, /* binary driver, and tegra, constant */
+	};
+	struct nouveau_connector *nv_connector;
+	u32 max_ac_packet;
+	union hdmi_infoframe avi_frame;
+	union hdmi_infoframe vendor_frame;
+	int ret;
+	int size;
+
+	nv_connector = nouveau_encoder_connector_get(nv_encoder);
+	if (!drm_detect_hdmi_monitor(nv_connector->edid))
+		return;
+
+	ret = drm_hdmi_avi_infoframe_from_display_mode(&avi_frame.avi, mode,
+						       false);
+	if (!ret) {
+		/* We have an AVI InfoFrame, populate it to the display */
+		args.pwr.avi_infoframe_length
+			= hdmi_infoframe_pack(&avi_frame, args.infoframes, 17);
+	}
+
+	ret = drm_hdmi_vendor_infoframe_from_display_mode(&vendor_frame.vendor.hdmi,
+							  &nv_connector->base, mode);
+	if (!ret) {
+		/* We have a Vendor InfoFrame, populate it to the display */
+		args.pwr.vendor_infoframe_length
+			= hdmi_infoframe_pack(&vendor_frame,
+					      args.infoframes
+					      + args.pwr.avi_infoframe_length,
+					      17);
+	}
+
+	max_ac_packet  = mode->htotal - mode->hdisplay;
+	max_ac_packet -= args.pwr.rekey;
+	max_ac_packet -= 18; /* constant from tegra */
+	args.pwr.max_ac_packet = max_ac_packet / 32;
+
+	size = sizeof(args.base)
+		+ sizeof(args.pwr)
+		+ args.pwr.avi_infoframe_length
+		+ args.pwr.vendor_infoframe_length;
+	nvif_mthd(&disp->disp->object, 0, &args, size);
+	nv50_audio_enable(encoder, mode);
+}
+
+/******************************************************************************
+ * MST
+ *****************************************************************************/
+#define nv50_mstm(p) container_of((p), struct nv50_mstm, mgr)
+#define nv50_mstc(p) container_of((p), struct nv50_mstc, connector)
+#define nv50_msto(p) container_of((p), struct nv50_msto, encoder)
+
+struct nv50_mstm {
+	struct nouveau_encoder *outp;
+
+	struct drm_dp_mst_topology_mgr mgr;
+	struct nv50_msto *msto[4];
+
+	bool modified;
+	bool disabled;
+	int links;
+};
+
+struct nv50_mstc {
+	struct nv50_mstm *mstm;
+	struct drm_dp_mst_port *port;
+	struct drm_connector connector;
+
+	struct drm_display_mode *native;
+	struct edid *edid;
+
+	int pbn;
+};
+
+struct nv50_msto {
+	struct drm_encoder encoder;
+
+	struct nv50_head *head;
+	struct nv50_mstc *mstc;
+	bool disabled;
+};
+
+static struct drm_dp_payload *
+nv50_msto_payload(struct nv50_msto *msto)
+{
+	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
+	struct nv50_mstc *mstc = msto->mstc;
+	struct nv50_mstm *mstm = mstc->mstm;
+	int vcpi = mstc->port->vcpi.vcpi, i;
+
+	NV_ATOMIC(drm, "%s: vcpi %d\n", msto->encoder.name, vcpi);
+	for (i = 0; i < mstm->mgr.max_payloads; i++) {
+		struct drm_dp_payload *payload = &mstm->mgr.payloads[i];
+		NV_ATOMIC(drm, "%s: %d: vcpi %d start 0x%02x slots 0x%02x\n",
+			  mstm->outp->base.base.name, i, payload->vcpi,
+			  payload->start_slot, payload->num_slots);
+	}
+
+	for (i = 0; i < mstm->mgr.max_payloads; i++) {
+		struct drm_dp_payload *payload = &mstm->mgr.payloads[i];
+		if (payload->vcpi == vcpi)
+			return payload;
+	}
+
+	return NULL;
+}
+
+static void
+nv50_msto_cleanup(struct nv50_msto *msto)
+{
+	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
+	struct nv50_mstc *mstc = msto->mstc;
+	struct nv50_mstm *mstm = mstc->mstm;
+
+	NV_ATOMIC(drm, "%s: msto cleanup\n", msto->encoder.name);
+	if (mstc->port && mstc->port->vcpi.vcpi > 0 && !nv50_msto_payload(msto))
+		drm_dp_mst_deallocate_vcpi(&mstm->mgr, mstc->port);
+	if (msto->disabled) {
+		msto->mstc = NULL;
+		msto->head = NULL;
+		msto->disabled = false;
+	}
+}
+
+static void
+nv50_msto_prepare(struct nv50_msto *msto)
+{
+	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
+	struct nv50_mstc *mstc = msto->mstc;
+	struct nv50_mstm *mstm = mstc->mstm;
+	struct {
+		struct nv50_disp_mthd_v1 base;
+		struct nv50_disp_sor_dp_mst_vcpi_v0 vcpi;
+	} args = {
+		.base.version = 1,
+		.base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI,
+		.base.hasht  = mstm->outp->dcb->hasht,
+		.base.hashm  = (0xf0ff & mstm->outp->dcb->hashm) |
+			       (0x0100 << msto->head->base.index),
+	};
+
+	NV_ATOMIC(drm, "%s: msto prepare\n", msto->encoder.name);
+	if (mstc->port && mstc->port->vcpi.vcpi > 0) {
+		struct drm_dp_payload *payload = nv50_msto_payload(msto);
+		if (payload) {
+			args.vcpi.start_slot = payload->start_slot;
+			args.vcpi.num_slots = payload->num_slots;
+			args.vcpi.pbn = mstc->port->vcpi.pbn;
+			args.vcpi.aligned_pbn = mstc->port->vcpi.aligned_pbn;
+		}
+	}
+
+	NV_ATOMIC(drm, "%s: %s: %02x %02x %04x %04x\n",
+		  msto->encoder.name, msto->head->base.base.name,
+		  args.vcpi.start_slot, args.vcpi.num_slots,
+		  args.vcpi.pbn, args.vcpi.aligned_pbn);
+	nvif_mthd(&drm->display->disp.object, 0, &args, sizeof(args));
+}
+
+static int
+nv50_msto_atomic_check(struct drm_encoder *encoder,
+		       struct drm_crtc_state *crtc_state,
+		       struct drm_connector_state *conn_state)
+{
+	struct nv50_mstc *mstc = nv50_mstc(conn_state->connector);
+	struct nv50_mstm *mstm = mstc->mstm;
+	int bpp = conn_state->connector->display_info.bpc * 3;
+	int slots;
+
+	mstc->pbn = drm_dp_calc_pbn_mode(crtc_state->adjusted_mode.clock, bpp);
+
+	slots = drm_dp_find_vcpi_slots(&mstm->mgr, mstc->pbn);
+	if (slots < 0)
+		return slots;
+
+	return nv50_outp_atomic_check_view(encoder, crtc_state, conn_state,
+					   mstc->native);
+}
+
+static void
+nv50_msto_enable(struct drm_encoder *encoder)
+{
+	struct nv50_head *head = nv50_head(encoder->crtc);
+	struct nv50_msto *msto = nv50_msto(encoder);
+	struct nv50_mstc *mstc = NULL;
+	struct nv50_mstm *mstm = NULL;
+	struct drm_connector *connector;
+	struct drm_connector_list_iter conn_iter;
+	u8 proto, depth;
+	int slots;
+	bool r;
+
+	drm_connector_list_iter_begin(encoder->dev, &conn_iter);
+	drm_for_each_connector_iter(connector, &conn_iter) {
+		if (connector->state->best_encoder == &msto->encoder) {
+			mstc = nv50_mstc(connector);
+			mstm = mstc->mstm;
+			break;
+		}
+	}
+	drm_connector_list_iter_end(&conn_iter);
+
+	if (WARN_ON(!mstc))
+		return;
+
+	slots = drm_dp_find_vcpi_slots(&mstm->mgr, mstc->pbn);
+	r = drm_dp_mst_allocate_vcpi(&mstm->mgr, mstc->port, mstc->pbn, slots);
+	WARN_ON(!r);
+
+	if (!mstm->links++)
+		nv50_outp_acquire(mstm->outp);
+
+	if (mstm->outp->link & 1)
+		proto = 0x8;
+	else
+		proto = 0x9;
+
+	switch (mstc->connector.display_info.bpc) {
+	case  6: depth = 0x2; break;
+	case  8: depth = 0x5; break;
+	case 10:
+	default: depth = 0x6; break;
+	}
+
+	mstm->outp->update(mstm->outp, head->base.index,
+			   nv50_head_atom(head->base.base.state), proto, depth);
+
+	msto->head = head;
+	msto->mstc = mstc;
+	mstm->modified = true;
+}
+
+static void
+nv50_msto_disable(struct drm_encoder *encoder)
+{
+	struct nv50_msto *msto = nv50_msto(encoder);
+	struct nv50_mstc *mstc = msto->mstc;
+	struct nv50_mstm *mstm = mstc->mstm;
+
+	if (mstc->port)
+		drm_dp_mst_reset_vcpi_slots(&mstm->mgr, mstc->port);
+
+	mstm->outp->update(mstm->outp, msto->head->base.index, NULL, 0, 0);
+	mstm->modified = true;
+	if (!--mstm->links)
+		mstm->disabled = true;
+	msto->disabled = true;
+}
+
+static const struct drm_encoder_helper_funcs
+nv50_msto_help = {
+	.disable = nv50_msto_disable,
+	.enable = nv50_msto_enable,
+	.atomic_check = nv50_msto_atomic_check,
+};
+
+static void
+nv50_msto_destroy(struct drm_encoder *encoder)
+{
+	struct nv50_msto *msto = nv50_msto(encoder);
+	drm_encoder_cleanup(&msto->encoder);
+	kfree(msto);
+}
+
+static const struct drm_encoder_funcs
+nv50_msto = {
+	.destroy = nv50_msto_destroy,
+};
+
+static int
+nv50_msto_new(struct drm_device *dev, u32 heads, const char *name, int id,
+	      struct nv50_msto **pmsto)
+{
+	struct nv50_msto *msto;
+	int ret;
+
+	if (!(msto = *pmsto = kzalloc(sizeof(*msto), GFP_KERNEL)))
+		return -ENOMEM;
+
+	ret = drm_encoder_init(dev, &msto->encoder, &nv50_msto,
+			       DRM_MODE_ENCODER_DPMST, "%s-mst-%d", name, id);
+	if (ret) {
+		kfree(*pmsto);
+		*pmsto = NULL;
+		return ret;
+	}
+
+	drm_encoder_helper_add(&msto->encoder, &nv50_msto_help);
+	msto->encoder.possible_crtcs = heads;
+	return 0;
+}
+
+static struct drm_encoder *
+nv50_mstc_atomic_best_encoder(struct drm_connector *connector,
+			      struct drm_connector_state *connector_state)
+{
+	struct nv50_head *head = nv50_head(connector_state->crtc);
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+	if (mstc->port) {
+		struct nv50_mstm *mstm = mstc->mstm;
+		return &mstm->msto[head->base.index]->encoder;
+	}
+	return NULL;
+}
+
+static struct drm_encoder *
+nv50_mstc_best_encoder(struct drm_connector *connector)
+{
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+	if (mstc->port) {
+		struct nv50_mstm *mstm = mstc->mstm;
+		return &mstm->msto[0]->encoder;
+	}
+	return NULL;
+}
+
+static enum drm_mode_status
+nv50_mstc_mode_valid(struct drm_connector *connector,
+		     struct drm_display_mode *mode)
+{
+	return MODE_OK;
+}
+
+static int
+nv50_mstc_get_modes(struct drm_connector *connector)
+{
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+	int ret = 0;
+
+	mstc->edid = drm_dp_mst_get_edid(&mstc->connector, mstc->port->mgr, mstc->port);
+	drm_mode_connector_update_edid_property(&mstc->connector, mstc->edid);
+	if (mstc->edid)
+		ret = drm_add_edid_modes(&mstc->connector, mstc->edid);
+
+	if (!mstc->connector.display_info.bpc)
+		mstc->connector.display_info.bpc = 8;
+
+	if (mstc->native)
+		drm_mode_destroy(mstc->connector.dev, mstc->native);
+	mstc->native = nouveau_conn_native_mode(&mstc->connector);
+	return ret;
+}
+
+static const struct drm_connector_helper_funcs
+nv50_mstc_help = {
+	.get_modes = nv50_mstc_get_modes,
+	.mode_valid = nv50_mstc_mode_valid,
+	.best_encoder = nv50_mstc_best_encoder,
+	.atomic_best_encoder = nv50_mstc_atomic_best_encoder,
+};
+
+static enum drm_connector_status
+nv50_mstc_detect(struct drm_connector *connector, bool force)
+{
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+	if (!mstc->port)
+		return connector_status_disconnected;
+	return drm_dp_mst_detect_port(connector, mstc->port->mgr, mstc->port);
+}
+
+static void
+nv50_mstc_destroy(struct drm_connector *connector)
+{
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+	drm_connector_cleanup(&mstc->connector);
+	kfree(mstc);
+}
+
+static const struct drm_connector_funcs
+nv50_mstc = {
+	.reset = nouveau_conn_reset,
+	.detect = nv50_mstc_detect,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = nv50_mstc_destroy,
+	.atomic_duplicate_state = nouveau_conn_atomic_duplicate_state,
+	.atomic_destroy_state = nouveau_conn_atomic_destroy_state,
+	.atomic_set_property = nouveau_conn_atomic_set_property,
+	.atomic_get_property = nouveau_conn_atomic_get_property,
+};
+
+static int
+nv50_mstc_new(struct nv50_mstm *mstm, struct drm_dp_mst_port *port,
+	      const char *path, struct nv50_mstc **pmstc)
+{
+	struct drm_device *dev = mstm->outp->base.base.dev;
+	struct nv50_mstc *mstc;
+	int ret, i;
+
+	if (!(mstc = *pmstc = kzalloc(sizeof(*mstc), GFP_KERNEL)))
+		return -ENOMEM;
+	mstc->mstm = mstm;
+	mstc->port = port;
+
+	ret = drm_connector_init(dev, &mstc->connector, &nv50_mstc,
+				 DRM_MODE_CONNECTOR_DisplayPort);
+	if (ret) {
+		kfree(*pmstc);
+		*pmstc = NULL;
+		return ret;
+	}
+
+	drm_connector_helper_add(&mstc->connector, &nv50_mstc_help);
+
+	mstc->connector.funcs->reset(&mstc->connector);
+	nouveau_conn_attach_properties(&mstc->connector);
+
+	for (i = 0; i < ARRAY_SIZE(mstm->msto) && mstm->msto[i]; i++)
+		drm_mode_connector_attach_encoder(&mstc->connector, &mstm->msto[i]->encoder);
+
+	drm_object_attach_property(&mstc->connector.base, dev->mode_config.path_property, 0);
+	drm_object_attach_property(&mstc->connector.base, dev->mode_config.tile_property, 0);
+	drm_mode_connector_set_path_property(&mstc->connector, path);
+	return 0;
+}
+
+static void
+nv50_mstm_cleanup(struct nv50_mstm *mstm)
+{
+	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
+	struct drm_encoder *encoder;
+	int ret;
+
+	NV_ATOMIC(drm, "%s: mstm cleanup\n", mstm->outp->base.base.name);
+	ret = drm_dp_check_act_status(&mstm->mgr);
+
+	ret = drm_dp_update_payload_part2(&mstm->mgr);
+
+	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
+		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
+			struct nv50_msto *msto = nv50_msto(encoder);
+			struct nv50_mstc *mstc = msto->mstc;
+			if (mstc && mstc->mstm == mstm)
+				nv50_msto_cleanup(msto);
+		}
+	}
+
+	mstm->modified = false;
+}
+
+static void
+nv50_mstm_prepare(struct nv50_mstm *mstm)
+{
+	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
+	struct drm_encoder *encoder;
+	int ret;
+
+	NV_ATOMIC(drm, "%s: mstm prepare\n", mstm->outp->base.base.name);
+	ret = drm_dp_update_payload_part1(&mstm->mgr);
+
+	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
+		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
+			struct nv50_msto *msto = nv50_msto(encoder);
+			struct nv50_mstc *mstc = msto->mstc;
+			if (mstc && mstc->mstm == mstm)
+				nv50_msto_prepare(msto);
+		}
+	}
+
+	if (mstm->disabled) {
+		if (!mstm->links)
+			nv50_outp_release(mstm->outp);
+		mstm->disabled = false;
+	}
+}
+
+static void
+nv50_mstm_hotplug(struct drm_dp_mst_topology_mgr *mgr)
+{
+	struct nv50_mstm *mstm = nv50_mstm(mgr);
+	drm_kms_helper_hotplug_event(mstm->outp->base.base.dev);
+}
+
+static void
+nv50_mstm_destroy_connector(struct drm_dp_mst_topology_mgr *mgr,
+			    struct drm_connector *connector)
+{
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+
+	drm_connector_unregister(&mstc->connector);
+
+	drm_fb_helper_remove_one_connector(&drm->fbcon->helper, &mstc->connector);
+
+	drm_modeset_lock(&drm->dev->mode_config.connection_mutex, NULL);
+	mstc->port = NULL;
+	drm_modeset_unlock(&drm->dev->mode_config.connection_mutex);
+
+	drm_connector_unreference(&mstc->connector);
+}
+
+static void
+nv50_mstm_register_connector(struct drm_connector *connector)
+{
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
+
+	drm_fb_helper_add_one_connector(&drm->fbcon->helper, connector);
+
+	drm_connector_register(connector);
+}
+
+static struct drm_connector *
+nv50_mstm_add_connector(struct drm_dp_mst_topology_mgr *mgr,
+			struct drm_dp_mst_port *port, const char *path)
+{
+	struct nv50_mstm *mstm = nv50_mstm(mgr);
+	struct nv50_mstc *mstc;
+	int ret;
+
+	ret = nv50_mstc_new(mstm, port, path, &mstc);
+	if (ret) {
+		if (mstc)
+			mstc->connector.funcs->destroy(&mstc->connector);
+		return NULL;
+	}
+
+	return &mstc->connector;
+}
+
+static const struct drm_dp_mst_topology_cbs
+nv50_mstm = {
+	.add_connector = nv50_mstm_add_connector,
+	.register_connector = nv50_mstm_register_connector,
+	.destroy_connector = nv50_mstm_destroy_connector,
+	.hotplug = nv50_mstm_hotplug,
+};
+
+void
+nv50_mstm_service(struct nv50_mstm *mstm)
+{
+	struct drm_dp_aux *aux = mstm ? mstm->mgr.aux : NULL;
+	bool handled = true;
+	int ret;
+	u8 esi[8] = {};
+
+	if (!aux)
+		return;
+
+	while (handled) {
+		ret = drm_dp_dpcd_read(aux, DP_SINK_COUNT_ESI, esi, 8);
+		if (ret != 8) {
+			drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false);
+			return;
+		}
+
+		drm_dp_mst_hpd_irq(&mstm->mgr, esi, &handled);
+		if (!handled)
+			break;
+
+		drm_dp_dpcd_write(aux, DP_SINK_COUNT_ESI + 1, &esi[1], 3);
+	}
+}
+
+void
+nv50_mstm_remove(struct nv50_mstm *mstm)
+{
+	if (mstm)
+		drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false);
+}
+
+static int
+nv50_mstm_enable(struct nv50_mstm *mstm, u8 dpcd, int state)
+{
+	struct nouveau_encoder *outp = mstm->outp;
+	struct {
+		struct nv50_disp_mthd_v1 base;
+		struct nv50_disp_sor_dp_mst_link_v0 mst;
+	} args = {
+		.base.version = 1,
+		.base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_LINK,
+		.base.hasht = outp->dcb->hasht,
+		.base.hashm = outp->dcb->hashm,
+		.mst.state = state,
+	};
+	struct nouveau_drm *drm = nouveau_drm(outp->base.base.dev);
+	struct nvif_object *disp = &drm->display->disp.object;
+	int ret;
+
+	if (dpcd >= 0x12) {
+		ret = drm_dp_dpcd_readb(mstm->mgr.aux, DP_MSTM_CTRL, &dpcd);
+		if (ret < 0)
+			return ret;
+
+		dpcd &= ~DP_MST_EN;
+		if (state)
+			dpcd |= DP_MST_EN;
+
+		ret = drm_dp_dpcd_writeb(mstm->mgr.aux, DP_MSTM_CTRL, dpcd);
+		if (ret < 0)
+			return ret;
+	}
+
+	return nvif_mthd(disp, 0, &args, sizeof(args));
+}
+
+int
+nv50_mstm_detect(struct nv50_mstm *mstm, u8 dpcd[8], int allow)
+{
+	int ret, state = 0;
+
+	if (!mstm)
+		return 0;
+
+	if (dpcd[0] >= 0x12) {
+		ret = drm_dp_dpcd_readb(mstm->mgr.aux, DP_MSTM_CAP, &dpcd[1]);
+		if (ret < 0)
+			return ret;
+
+		if (!(dpcd[1] & DP_MST_CAP))
+			dpcd[0] = 0x11;
+		else
+			state = allow;
+	}
+
+	ret = nv50_mstm_enable(mstm, dpcd[0], state);
+	if (ret)
+		return ret;
+
+	ret = drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, state);
+	if (ret)
+		return nv50_mstm_enable(mstm, dpcd[0], 0);
+
+	return mstm->mgr.mst_state;
+}
+
+static void
+nv50_mstm_fini(struct nv50_mstm *mstm)
+{
+	if (mstm && mstm->mgr.mst_state)
+		drm_dp_mst_topology_mgr_suspend(&mstm->mgr);
+}
+
+static void
+nv50_mstm_init(struct nv50_mstm *mstm)
+{
+	if (mstm && mstm->mgr.mst_state)
+		drm_dp_mst_topology_mgr_resume(&mstm->mgr);
+}
+
+static void
+nv50_mstm_del(struct nv50_mstm **pmstm)
+{
+	struct nv50_mstm *mstm = *pmstm;
+	if (mstm) {
+		kfree(*pmstm);
+		*pmstm = NULL;
+	}
+}
+
+static int
+nv50_mstm_new(struct nouveau_encoder *outp, struct drm_dp_aux *aux, int aux_max,
+	      int conn_base_id, struct nv50_mstm **pmstm)
+{
+	const int max_payloads = hweight8(outp->dcb->heads);
+	struct drm_device *dev = outp->base.base.dev;
+	struct nv50_mstm *mstm;
+	int ret, i;
+	u8 dpcd;
+
+	/* This is a workaround for some monitors not functioning
+	 * correctly in MST mode on initial module load.  I think
+	 * some bad interaction with the VBIOS may be responsible.
+	 *
+	 * A good ol' off and on again seems to work here ;)
+	 */
+	ret = drm_dp_dpcd_readb(aux, DP_DPCD_REV, &dpcd);
+	if (ret >= 0 && dpcd >= 0x12)
+		drm_dp_dpcd_writeb(aux, DP_MSTM_CTRL, 0);
+
+	if (!(mstm = *pmstm = kzalloc(sizeof(*mstm), GFP_KERNEL)))
+		return -ENOMEM;
+	mstm->outp = outp;
+	mstm->mgr.cbs = &nv50_mstm;
+
+	ret = drm_dp_mst_topology_mgr_init(&mstm->mgr, dev, aux, aux_max,
+					   max_payloads, conn_base_id);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < max_payloads; i++) {
+		ret = nv50_msto_new(dev, outp->dcb->heads, outp->base.base.name,
+				    i, &mstm->msto[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ * SOR
+ *****************************************************************************/
+static void
+nv50_sor_update(struct nouveau_encoder *nv_encoder, u8 head,
+		struct nv50_head_atom *asyh, u8 proto, u8 depth)
+{
+	struct nv50_disp *disp = nv50_disp(nv_encoder->base.base.dev);
+	struct nv50_core *core = disp->core;
+
+	if (!asyh) {
+		nv_encoder->ctrl &= ~BIT(head);
+		if (!(nv_encoder->ctrl & 0x0000000f))
+			nv_encoder->ctrl = 0;
+	} else {
+		nv_encoder->ctrl |= proto << 8;
+		nv_encoder->ctrl |= BIT(head);
+		asyh->or.depth = depth;
+	}
+
+	core->func->sor->ctrl(core, nv_encoder->or, nv_encoder->ctrl, asyh);
+}
+
+static void
+nv50_sor_disable(struct drm_encoder *encoder)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_crtc *nv_crtc = nouveau_crtc(nv_encoder->crtc);
+
+	nv_encoder->crtc = NULL;
+
+	if (nv_crtc) {
+		struct nvkm_i2c_aux *aux = nv_encoder->aux;
+		u8 pwr;
+
+		if (aux) {
+			int ret = nvkm_rdaux(aux, DP_SET_POWER, &pwr, 1);
+			if (ret == 0) {
+				pwr &= ~DP_SET_POWER_MASK;
+				pwr |=  DP_SET_POWER_D3;
+				nvkm_wraux(aux, DP_SET_POWER, &pwr, 1);
+			}
+		}
+
+		nv_encoder->update(nv_encoder, nv_crtc->index, NULL, 0, 0);
+		nv50_audio_disable(encoder, nv_crtc);
+		nv50_hdmi_disable(&nv_encoder->base.base, nv_crtc);
+		nv50_outp_release(nv_encoder);
+	}
+}
+
+static void
+nv50_sor_enable(struct drm_encoder *encoder)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
+	struct nv50_head_atom *asyh = nv50_head_atom(nv_crtc->base.state);
+	struct drm_display_mode *mode = &asyh->state.adjusted_mode;
+	struct {
+		struct nv50_disp_mthd_v1 base;
+		struct nv50_disp_sor_lvds_script_v0 lvds;
+	} lvds = {
+		.base.version = 1,
+		.base.method  = NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT,
+		.base.hasht   = nv_encoder->dcb->hasht,
+		.base.hashm   = nv_encoder->dcb->hashm,
+	};
+	struct nv50_disp *disp = nv50_disp(encoder->dev);
+	struct drm_device *dev = encoder->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_connector *nv_connector;
+	struct nvbios *bios = &drm->vbios;
+	u8 proto = 0xf;
+	u8 depth = 0x0;
+
+	nv_connector = nouveau_encoder_connector_get(nv_encoder);
+	nv_encoder->crtc = encoder->crtc;
+	nv50_outp_acquire(nv_encoder);
+
+	switch (nv_encoder->dcb->type) {
+	case DCB_OUTPUT_TMDS:
+		if (nv_encoder->link & 1) {
+			proto = 0x1;
+			/* Only enable dual-link if:
+			 *  - Need to (i.e. rate > 165MHz)
+			 *  - DCB says we can
+			 *  - Not an HDMI monitor, since there's no dual-link
+			 *    on HDMI.
+			 */
+			if (mode->clock >= 165000 &&
+			    nv_encoder->dcb->duallink_possible &&
+			    !drm_detect_hdmi_monitor(nv_connector->edid))
+				proto |= 0x4;
+		} else {
+			proto = 0x2;
+		}
+
+		nv50_hdmi_enable(&nv_encoder->base.base, mode);
+		break;
+	case DCB_OUTPUT_LVDS:
+		proto = 0x0;
+
+		if (bios->fp_no_ddc) {
+			if (bios->fp.dual_link)
+				lvds.lvds.script |= 0x0100;
+			if (bios->fp.if_is_24bit)
+				lvds.lvds.script |= 0x0200;
+		} else {
+			if (nv_connector->type == DCB_CONNECTOR_LVDS_SPWG) {
+				if (((u8 *)nv_connector->edid)[121] == 2)
+					lvds.lvds.script |= 0x0100;
+			} else
+			if (mode->clock >= bios->fp.duallink_transition_clk) {
+				lvds.lvds.script |= 0x0100;
+			}
+
+			if (lvds.lvds.script & 0x0100) {
+				if (bios->fp.strapless_is_24bit & 2)
+					lvds.lvds.script |= 0x0200;
+			} else {
+				if (bios->fp.strapless_is_24bit & 1)
+					lvds.lvds.script |= 0x0200;
+			}
+
+			if (nv_connector->base.display_info.bpc == 8)
+				lvds.lvds.script |= 0x0200;
+		}
+
+		nvif_mthd(&disp->disp->object, 0, &lvds, sizeof(lvds));
+		break;
+	case DCB_OUTPUT_DP:
+		if (nv_connector->base.display_info.bpc == 6)
+			depth = 0x2;
+		else
+		if (nv_connector->base.display_info.bpc == 8)
+			depth = 0x5;
+		else
+			depth = 0x6;
+
+		if (nv_encoder->link & 1)
+			proto = 0x8;
+		else
+			proto = 0x9;
+
+		nv50_audio_enable(encoder, mode);
+		break;
+	default:
+		BUG();
+		break;
+	}
+
+	nv_encoder->update(nv_encoder, nv_crtc->index, asyh, proto, depth);
+}
+
+static const struct drm_encoder_helper_funcs
+nv50_sor_help = {
+	.atomic_check = nv50_outp_atomic_check,
+	.enable = nv50_sor_enable,
+	.disable = nv50_sor_disable,
+};
+
+static void
+nv50_sor_destroy(struct drm_encoder *encoder)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	nv50_mstm_del(&nv_encoder->dp.mstm);
+	drm_encoder_cleanup(encoder);
+	kfree(encoder);
+}
+
+static const struct drm_encoder_funcs
+nv50_sor_func = {
+	.destroy = nv50_sor_destroy,
+};
+
+static int
+nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe)
+{
+	struct nouveau_connector *nv_connector = nouveau_connector(connector);
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
+	struct nvkm_bios *bios = nvxx_bios(&drm->client.device);
+	struct nvkm_i2c *i2c = nvxx_i2c(&drm->client.device);
+	struct nouveau_encoder *nv_encoder;
+	struct drm_encoder *encoder;
+	u8 ver, hdr, cnt, len;
+	u32 data;
+	int type, ret;
+
+	switch (dcbe->type) {
+	case DCB_OUTPUT_LVDS: type = DRM_MODE_ENCODER_LVDS; break;
+	case DCB_OUTPUT_TMDS:
+	case DCB_OUTPUT_DP:
+	default:
+		type = DRM_MODE_ENCODER_TMDS;
+		break;
+	}
+
+	nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
+	if (!nv_encoder)
+		return -ENOMEM;
+	nv_encoder->dcb = dcbe;
+	nv_encoder->update = nv50_sor_update;
+
+	encoder = to_drm_encoder(nv_encoder);
+	encoder->possible_crtcs = dcbe->heads;
+	encoder->possible_clones = 0;
+	drm_encoder_init(connector->dev, encoder, &nv50_sor_func, type,
+			 "sor-%04x-%04x", dcbe->hasht, dcbe->hashm);
+	drm_encoder_helper_add(encoder, &nv50_sor_help);
+
+	drm_mode_connector_attach_encoder(connector, encoder);
+
+	if (dcbe->type == DCB_OUTPUT_DP) {
+		struct nv50_disp *disp = nv50_disp(encoder->dev);
+		struct nvkm_i2c_aux *aux =
+			nvkm_i2c_aux_find(i2c, dcbe->i2c_index);
+		if (aux) {
+			if (disp->disp->object.oclass < GF110_DISP) {
+				/* HW has no support for address-only
+				 * transactions, so we're required to
+				 * use custom I2C-over-AUX code.
+				 */
+				nv_encoder->i2c = &aux->i2c;
+			} else {
+				nv_encoder->i2c = &nv_connector->aux.ddc;
+			}
+			nv_encoder->aux = aux;
+		}
+
+		if ((data = nvbios_dp_table(bios, &ver, &hdr, &cnt, &len)) &&
+		    ver >= 0x40 && (nvbios_rd08(bios, data + 0x08) & 0x04)) {
+			ret = nv50_mstm_new(nv_encoder, &nv_connector->aux, 16,
+					    nv_connector->base.base.id,
+					    &nv_encoder->dp.mstm);
+			if (ret)
+				return ret;
+		}
+	} else {
+		struct nvkm_i2c_bus *bus =
+			nvkm_i2c_bus_find(i2c, dcbe->i2c_index);
+		if (bus)
+			nv_encoder->i2c = &bus->i2c;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ * PIOR
+ *****************************************************************************/
+static int
+nv50_pior_atomic_check(struct drm_encoder *encoder,
+		       struct drm_crtc_state *crtc_state,
+		       struct drm_connector_state *conn_state)
+{
+	int ret = nv50_outp_atomic_check(encoder, crtc_state, conn_state);
+	if (ret)
+		return ret;
+	crtc_state->adjusted_mode.clock *= 2;
+	return 0;
+}
+
+static void
+nv50_pior_disable(struct drm_encoder *encoder)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nv50_core *core = nv50_disp(encoder->dev)->core;
+	if (nv_encoder->crtc)
+		core->func->pior->ctrl(core, nv_encoder->or, 0x00000000, NULL);
+	nv_encoder->crtc = NULL;
+	nv50_outp_release(nv_encoder);
+}
+
+static void
+nv50_pior_enable(struct drm_encoder *encoder)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
+	struct nouveau_connector *nv_connector;
+	struct nv50_head_atom *asyh = nv50_head_atom(nv_crtc->base.state);
+	struct nv50_core *core = nv50_disp(encoder->dev)->core;
+	u8 owner = 1 << nv_crtc->index;
+	u8 proto;
+
+	nv50_outp_acquire(nv_encoder);
+
+	nv_connector = nouveau_encoder_connector_get(nv_encoder);
+	switch (nv_connector->base.display_info.bpc) {
+	case 10: asyh->or.depth = 0x6; break;
+	case  8: asyh->or.depth = 0x5; break;
+	case  6: asyh->or.depth = 0x2; break;
+	default: asyh->or.depth = 0x0; break;
+	}
+
+	switch (nv_encoder->dcb->type) {
+	case DCB_OUTPUT_TMDS:
+	case DCB_OUTPUT_DP:
+		proto = 0x0;
+		break;
+	default:
+		BUG();
+		break;
+	}
+
+	core->func->pior->ctrl(core, nv_encoder->or, (proto << 8) | owner, asyh);
+	nv_encoder->crtc = encoder->crtc;
+}
+
+static const struct drm_encoder_helper_funcs
+nv50_pior_help = {
+	.atomic_check = nv50_pior_atomic_check,
+	.enable = nv50_pior_enable,
+	.disable = nv50_pior_disable,
+};
+
+static void
+nv50_pior_destroy(struct drm_encoder *encoder)
+{
+	drm_encoder_cleanup(encoder);
+	kfree(encoder);
+}
+
+static const struct drm_encoder_funcs
+nv50_pior_func = {
+	.destroy = nv50_pior_destroy,
+};
+
+static int
+nv50_pior_create(struct drm_connector *connector, struct dcb_output *dcbe)
+{
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
+	struct nvkm_i2c *i2c = nvxx_i2c(&drm->client.device);
+	struct nvkm_i2c_bus *bus = NULL;
+	struct nvkm_i2c_aux *aux = NULL;
+	struct i2c_adapter *ddc;
+	struct nouveau_encoder *nv_encoder;
+	struct drm_encoder *encoder;
+	int type;
+
+	switch (dcbe->type) {
+	case DCB_OUTPUT_TMDS:
+		bus  = nvkm_i2c_bus_find(i2c, NVKM_I2C_BUS_EXT(dcbe->extdev));
+		ddc  = bus ? &bus->i2c : NULL;
+		type = DRM_MODE_ENCODER_TMDS;
+		break;
+	case DCB_OUTPUT_DP:
+		aux  = nvkm_i2c_aux_find(i2c, NVKM_I2C_AUX_EXT(dcbe->extdev));
+		ddc  = aux ? &aux->i2c : NULL;
+		type = DRM_MODE_ENCODER_TMDS;
+		break;
+	default:
+		return -ENODEV;
+	}
+
+	nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
+	if (!nv_encoder)
+		return -ENOMEM;
+	nv_encoder->dcb = dcbe;
+	nv_encoder->i2c = ddc;
+	nv_encoder->aux = aux;
+
+	encoder = to_drm_encoder(nv_encoder);
+	encoder->possible_crtcs = dcbe->heads;
+	encoder->possible_clones = 0;
+	drm_encoder_init(connector->dev, encoder, &nv50_pior_func, type,
+			 "pior-%04x-%04x", dcbe->hasht, dcbe->hashm);
+	drm_encoder_helper_add(encoder, &nv50_pior_help);
+
+	drm_mode_connector_attach_encoder(connector, encoder);
+	return 0;
+}
+
+/******************************************************************************
+ * Atomic
+ *****************************************************************************/
+
+static void
+nv50_disp_atomic_commit_core(struct nouveau_drm *drm, u32 *interlock)
+{
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	struct nv50_core *core = disp->core;
+	struct nv50_mstm *mstm;
+	struct drm_encoder *encoder;
+
+	NV_ATOMIC(drm, "commit core %08x\n", interlock[NV50_DISP_INTERLOCK_BASE]);
+
+	drm_for_each_encoder(encoder, drm->dev) {
+		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
+			mstm = nouveau_encoder(encoder)->dp.mstm;
+			if (mstm && mstm->modified)
+				nv50_mstm_prepare(mstm);
+		}
+	}
+
+	core->func->ntfy_init(disp->sync, NV50_DISP_CORE_NTFY);
+	core->func->update(core, interlock, true);
+	if (core->func->ntfy_wait_done(disp->sync, NV50_DISP_CORE_NTFY,
+				       disp->core->chan.base.device))
+		NV_ERROR(drm, "core notifier timeout\n");
+
+	drm_for_each_encoder(encoder, drm->dev) {
+		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
+			mstm = nouveau_encoder(encoder)->dp.mstm;
+			if (mstm && mstm->modified)
+				nv50_mstm_cleanup(mstm);
+		}
+	}
+}
+
+static void
+nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
+{
+	struct drm_device *dev = state->dev;
+	struct drm_crtc_state *new_crtc_state, *old_crtc_state;
+	struct drm_crtc *crtc;
+	struct drm_plane_state *new_plane_state;
+	struct drm_plane *plane;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nv50_disp *disp = nv50_disp(dev);
+	struct nv50_atom *atom = nv50_atom(state);
+	struct nv50_outp_atom *outp, *outt;
+	u32 interlock[NV50_DISP_INTERLOCK__SIZE] = {};
+	int i;
+
+	NV_ATOMIC(drm, "commit %d %d\n", atom->lock_core, atom->flush_disable);
+	drm_atomic_helper_wait_for_fences(dev, state, false);
+	drm_atomic_helper_wait_for_dependencies(state);
+	drm_atomic_helper_update_legacy_modeset_state(dev, state);
+
+	if (atom->lock_core)
+		mutex_lock(&disp->mutex);
+
+	/* Disable head(s). */
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
+		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
+		struct nv50_head *head = nv50_head(crtc);
+
+		NV_ATOMIC(drm, "%s: clr %04x (set %04x)\n", crtc->name,
+			  asyh->clr.mask, asyh->set.mask);
+		if (old_crtc_state->active && !new_crtc_state->active)
+			drm_crtc_vblank_off(crtc);
+
+		if (asyh->clr.mask) {
+			nv50_head_flush_clr(head, asyh, atom->flush_disable);
+			interlock[NV50_DISP_INTERLOCK_CORE] |= 1;
+		}
+	}
+
+	/* Disable plane(s). */
+	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+
+		NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", plane->name,
+			  asyw->clr.mask, asyw->set.mask);
+		if (!asyw->clr.mask)
+			continue;
+
+		nv50_wndw_flush_clr(wndw, interlock, atom->flush_disable, asyw);
+	}
+
+	/* Disable output path(s). */
+	list_for_each_entry(outp, &atom->outp, head) {
+		const struct drm_encoder_helper_funcs *help;
+		struct drm_encoder *encoder;
+
+		encoder = outp->encoder;
+		help = encoder->helper_private;
+
+		NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", encoder->name,
+			  outp->clr.mask, outp->set.mask);
+
+		if (outp->clr.mask) {
+			help->disable(encoder);
+			interlock[NV50_DISP_INTERLOCK_CORE] |= 1;
+			if (outp->flush_disable) {
+				nv50_disp_atomic_commit_core(drm, interlock);
+				memset(interlock, 0x00, sizeof(interlock));
+			}
+		}
+	}
+
+	/* Flush disable. */
+	if (interlock[NV50_DISP_INTERLOCK_CORE]) {
+		if (atom->flush_disable) {
+			for_each_new_plane_in_state(state, plane, new_plane_state, i) {
+				struct nv50_wndw *wndw = nv50_wndw(plane);
+				if (interlock[wndw->interlock.type] & wndw->interlock.data) {
+					if (wndw->func->update)
+						wndw->func->update(wndw, interlock);
+				}
+			}
+
+			nv50_disp_atomic_commit_core(drm, interlock);
+			memset(interlock, 0x00, sizeof(interlock));
+		}
+	}
+
+	/* Update output path(s). */
+	list_for_each_entry_safe(outp, outt, &atom->outp, head) {
+		const struct drm_encoder_helper_funcs *help;
+		struct drm_encoder *encoder;
+
+		encoder = outp->encoder;
+		help = encoder->helper_private;
+
+		NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", encoder->name,
+			  outp->set.mask, outp->clr.mask);
+
+		if (outp->set.mask) {
+			help->enable(encoder);
+			interlock[NV50_DISP_INTERLOCK_CORE] = 1;
+		}
+
+		list_del(&outp->head);
+		kfree(outp);
+	}
+
+	/* Update head(s). */
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
+		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
+		struct nv50_head *head = nv50_head(crtc);
+
+		NV_ATOMIC(drm, "%s: set %04x (clr %04x)\n", crtc->name,
+			  asyh->set.mask, asyh->clr.mask);
+
+		if (asyh->set.mask) {
+			nv50_head_flush_set(head, asyh);
+			interlock[NV50_DISP_INTERLOCK_CORE] = 1;
+		}
+
+		if (new_crtc_state->active) {
+			if (!old_crtc_state->active)
+				drm_crtc_vblank_on(crtc);
+			if (new_crtc_state->event)
+				drm_crtc_vblank_get(crtc);
+		}
+	}
+
+	/* Update plane(s). */
+	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+
+		NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", plane->name,
+			  asyw->set.mask, asyw->clr.mask);
+		if ( !asyw->set.mask &&
+		    (!asyw->clr.mask || atom->flush_disable))
+			continue;
+
+		nv50_wndw_flush_set(wndw, interlock, asyw);
+	}
+
+	/* Flush update. */
+	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+		if (interlock[wndw->interlock.type] & wndw->interlock.data) {
+			if (wndw->func->update)
+				wndw->func->update(wndw, interlock);
+		}
+	}
+
+	if (interlock[NV50_DISP_INTERLOCK_CORE]) {
+		if (interlock[NV50_DISP_INTERLOCK_BASE] ||
+		    !atom->state.legacy_cursor_update)
+			nv50_disp_atomic_commit_core(drm, interlock);
+		else
+			disp->core->func->update(disp->core, interlock, false);
+	}
+
+	if (atom->lock_core)
+		mutex_unlock(&disp->mutex);
+
+	/* Wait for HW to signal completion. */
+	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+		int ret = nv50_wndw_wait_armed(wndw, asyw);
+		if (ret)
+			NV_ERROR(drm, "%s: timeout\n", plane->name);
+	}
+
+	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
+		if (new_crtc_state->event) {
+			unsigned long flags;
+			/* Get correct count/ts if racing with vblank irq */
+			if (new_crtc_state->active)
+				drm_crtc_accurate_vblank_count(crtc);
+			spin_lock_irqsave(&crtc->dev->event_lock, flags);
+			drm_crtc_send_vblank_event(crtc, new_crtc_state->event);
+			spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+
+			new_crtc_state->event = NULL;
+			if (new_crtc_state->active)
+				drm_crtc_vblank_put(crtc);
+		}
+	}
+
+	drm_atomic_helper_commit_hw_done(state);
+	drm_atomic_helper_cleanup_planes(dev, state);
+	drm_atomic_helper_commit_cleanup_done(state);
+	drm_atomic_state_put(state);
+}
+
+static void
+nv50_disp_atomic_commit_work(struct work_struct *work)
+{
+	struct drm_atomic_state *state =
+		container_of(work, typeof(*state), commit_work);
+	nv50_disp_atomic_commit_tail(state);
+}
+
+static int
+nv50_disp_atomic_commit(struct drm_device *dev,
+			struct drm_atomic_state *state, bool nonblock)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct drm_plane_state *new_plane_state;
+	struct drm_plane *plane;
+	struct drm_crtc *crtc;
+	bool active = false;
+	int ret, i;
+
+	ret = pm_runtime_get_sync(dev->dev);
+	if (ret < 0 && ret != -EACCES)
+		return ret;
+
+	ret = drm_atomic_helper_setup_commit(state, nonblock);
+	if (ret)
+		goto done;
+
+	INIT_WORK(&state->commit_work, nv50_disp_atomic_commit_work);
+
+	ret = drm_atomic_helper_prepare_planes(dev, state);
+	if (ret)
+		goto done;
+
+	if (!nonblock) {
+		ret = drm_atomic_helper_wait_for_fences(dev, state, true);
+		if (ret)
+			goto err_cleanup;
+	}
+
+	ret = drm_atomic_helper_swap_state(state, true);
+	if (ret)
+		goto err_cleanup;
+
+	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+
+		if (asyw->set.image)
+			nv50_wndw_ntfy_enable(wndw, asyw);
+	}
+
+	drm_atomic_state_get(state);
+
+	if (nonblock)
+		queue_work(system_unbound_wq, &state->commit_work);
+	else
+		nv50_disp_atomic_commit_tail(state);
+
+	drm_for_each_crtc(crtc, dev) {
+		if (crtc->state->enable) {
+			if (!drm->have_disp_power_ref) {
+				drm->have_disp_power_ref = true;
+				return 0;
+			}
+			active = true;
+			break;
+		}
+	}
+
+	if (!active && drm->have_disp_power_ref) {
+		pm_runtime_put_autosuspend(dev->dev);
+		drm->have_disp_power_ref = false;
+	}
+
+err_cleanup:
+	if (ret)
+		drm_atomic_helper_cleanup_planes(dev, state);
+done:
+	pm_runtime_put_autosuspend(dev->dev);
+	return ret;
+}
+
+static struct nv50_outp_atom *
+nv50_disp_outp_atomic_add(struct nv50_atom *atom, struct drm_encoder *encoder)
+{
+	struct nv50_outp_atom *outp;
+
+	list_for_each_entry(outp, &atom->outp, head) {
+		if (outp->encoder == encoder)
+			return outp;
+	}
+
+	outp = kzalloc(sizeof(*outp), GFP_KERNEL);
+	if (!outp)
+		return ERR_PTR(-ENOMEM);
+
+	list_add(&outp->head, &atom->outp);
+	outp->encoder = encoder;
+	return outp;
+}
+
+static int
+nv50_disp_outp_atomic_check_clr(struct nv50_atom *atom,
+				struct drm_connector_state *old_connector_state)
+{
+	struct drm_encoder *encoder = old_connector_state->best_encoder;
+	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
+	struct drm_crtc *crtc;
+	struct nv50_outp_atom *outp;
+
+	if (!(crtc = old_connector_state->crtc))
+		return 0;
+
+	old_crtc_state = drm_atomic_get_old_crtc_state(&atom->state, crtc);
+	new_crtc_state = drm_atomic_get_new_crtc_state(&atom->state, crtc);
+	if (old_crtc_state->active && drm_atomic_crtc_needs_modeset(new_crtc_state)) {
+		outp = nv50_disp_outp_atomic_add(atom, encoder);
+		if (IS_ERR(outp))
+			return PTR_ERR(outp);
+
+		if (outp->encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
+			outp->flush_disable = true;
+			atom->flush_disable = true;
+		}
+		outp->clr.ctrl = true;
+		atom->lock_core = true;
+	}
+
+	return 0;
+}
+
+static int
+nv50_disp_outp_atomic_check_set(struct nv50_atom *atom,
+				struct drm_connector_state *connector_state)
+{
+	struct drm_encoder *encoder = connector_state->best_encoder;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_crtc *crtc;
+	struct nv50_outp_atom *outp;
+
+	if (!(crtc = connector_state->crtc))
+		return 0;
+
+	new_crtc_state = drm_atomic_get_new_crtc_state(&atom->state, crtc);
+	if (new_crtc_state->active && drm_atomic_crtc_needs_modeset(new_crtc_state)) {
+		outp = nv50_disp_outp_atomic_add(atom, encoder);
+		if (IS_ERR(outp))
+			return PTR_ERR(outp);
+
+		outp->set.ctrl = true;
+		atom->lock_core = true;
+	}
+
+	return 0;
+}
+
+static int
+nv50_disp_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
+{
+	struct nv50_atom *atom = nv50_atom(state);
+	struct drm_connector_state *old_connector_state, *new_connector_state;
+	struct drm_connector *connector;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_crtc *crtc;
+	int ret, i;
+
+	/* We need to handle colour management on a per-plane basis. */
+	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
+		if (new_crtc_state->color_mgmt_changed) {
+			ret = drm_atomic_add_affected_planes(state, crtc);
+			if (ret)
+				return ret;
+		}
+	}
+
+	ret = drm_atomic_helper_check(dev, state);
+	if (ret)
+		return ret;
+
+	for_each_oldnew_connector_in_state(state, connector, old_connector_state, new_connector_state, i) {
+		ret = nv50_disp_outp_atomic_check_clr(atom, old_connector_state);
+		if (ret)
+			return ret;
+
+		ret = nv50_disp_outp_atomic_check_set(atom, new_connector_state);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void
+nv50_disp_atomic_state_clear(struct drm_atomic_state *state)
+{
+	struct nv50_atom *atom = nv50_atom(state);
+	struct nv50_outp_atom *outp, *outt;
+
+	list_for_each_entry_safe(outp, outt, &atom->outp, head) {
+		list_del(&outp->head);
+		kfree(outp);
+	}
+
+	drm_atomic_state_default_clear(state);
+}
+
+static void
+nv50_disp_atomic_state_free(struct drm_atomic_state *state)
+{
+	struct nv50_atom *atom = nv50_atom(state);
+	drm_atomic_state_default_release(&atom->state);
+	kfree(atom);
+}
+
+static struct drm_atomic_state *
+nv50_disp_atomic_state_alloc(struct drm_device *dev)
+{
+	struct nv50_atom *atom;
+	if (!(atom = kzalloc(sizeof(*atom), GFP_KERNEL)) ||
+	    drm_atomic_state_init(dev, &atom->state) < 0) {
+		kfree(atom);
+		return NULL;
+	}
+	INIT_LIST_HEAD(&atom->outp);
+	return &atom->state;
+}
+
+static const struct drm_mode_config_funcs
+nv50_disp_func = {
+	.fb_create = nouveau_user_framebuffer_create,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
+	.atomic_check = nv50_disp_atomic_check,
+	.atomic_commit = nv50_disp_atomic_commit,
+	.atomic_state_alloc = nv50_disp_atomic_state_alloc,
+	.atomic_state_clear = nv50_disp_atomic_state_clear,
+	.atomic_state_free = nv50_disp_atomic_state_free,
+};
+
+/******************************************************************************
+ * Init
+ *****************************************************************************/
+
+void
+nv50_display_fini(struct drm_device *dev)
+{
+	struct nouveau_encoder *nv_encoder;
+	struct drm_encoder *encoder;
+	struct drm_plane *plane;
+
+	drm_for_each_plane(plane, dev) {
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+		if (plane->funcs != &nv50_wndw)
+			continue;
+		nv50_wndw_fini(wndw);
+	}
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
+			nv_encoder = nouveau_encoder(encoder);
+			nv50_mstm_fini(nv_encoder->dp.mstm);
+		}
+	}
+}
+
+int
+nv50_display_init(struct drm_device *dev)
+{
+	struct nv50_core *core = nv50_disp(dev)->core;
+	struct drm_encoder *encoder;
+	struct drm_plane *plane;
+
+	core->func->init(core);
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
+			struct nouveau_encoder *nv_encoder =
+				nouveau_encoder(encoder);
+			nv50_mstm_init(nv_encoder->dp.mstm);
+		}
+	}
+
+	drm_for_each_plane(plane, dev) {
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+		if (plane->funcs != &nv50_wndw)
+			continue;
+		nv50_wndw_init(wndw);
+	}
+
+	return 0;
+}
+
+void
+nv50_display_destroy(struct drm_device *dev)
+{
+	struct nv50_disp *disp = nv50_disp(dev);
+
+	nv50_core_del(&disp->core);
+
+	nouveau_bo_unmap(disp->sync);
+	if (disp->sync)
+		nouveau_bo_unpin(disp->sync);
+	nouveau_bo_ref(NULL, &disp->sync);
+
+	nouveau_display(dev)->priv = NULL;
+	kfree(disp);
+}
+
+MODULE_PARM_DESC(atomic, "Expose atomic ioctl (default: disabled)");
+static int nouveau_atomic = 0;
+module_param_named(atomic, nouveau_atomic, int, 0400);
+
+int
+nv50_display_create(struct drm_device *dev)
+{
+	struct nvif_device *device = &nouveau_drm(dev)->client.device;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct dcb_table *dcb = &drm->vbios.dcb;
+	struct drm_connector *connector, *tmp;
+	struct nv50_disp *disp;
+	struct dcb_output *dcbe;
+	int crtcs, ret, i;
+
+	disp = kzalloc(sizeof(*disp), GFP_KERNEL);
+	if (!disp)
+		return -ENOMEM;
+
+	mutex_init(&disp->mutex);
+
+	nouveau_display(dev)->priv = disp;
+	nouveau_display(dev)->dtor = nv50_display_destroy;
+	nouveau_display(dev)->init = nv50_display_init;
+	nouveau_display(dev)->fini = nv50_display_fini;
+	disp->disp = &nouveau_display(dev)->disp;
+	dev->mode_config.funcs = &nv50_disp_func;
+	dev->driver->driver_features |= DRIVER_PREFER_XBGR_30BPP;
+	if (nouveau_atomic)
+		dev->driver->driver_features |= DRIVER_ATOMIC;
+
+	/* small shared memory area we use for notifiers and semaphores */
+	ret = nouveau_bo_new(&drm->client, 4096, 0x1000, TTM_PL_FLAG_VRAM,
+			     0, 0x0000, NULL, NULL, &disp->sync);
+	if (!ret) {
+		ret = nouveau_bo_pin(disp->sync, TTM_PL_FLAG_VRAM, true);
+		if (!ret) {
+			ret = nouveau_bo_map(disp->sync);
+			if (ret)
+				nouveau_bo_unpin(disp->sync);
+		}
+		if (ret)
+			nouveau_bo_ref(NULL, &disp->sync);
+	}
+
+	if (ret)
+		goto out;
+
+	/* allocate master evo channel */
+	ret = nv50_core_new(drm, &disp->core);
+	if (ret)
+		goto out;
+
+	/* create crtc objects to represent the hw heads */
+	if (disp->disp->object.oclass >= GV100_DISP)
+		crtcs = nvif_rd32(&device->object, 0x610060) & 0xff;
+	else
+	if (disp->disp->object.oclass >= GF110_DISP)
+		crtcs = nvif_rd32(&device->object, 0x612004) & 0xf;
+	else
+		crtcs = 0x3;
+
+	for (i = 0; i < fls(crtcs); i++) {
+		if (!(crtcs & (1 << i)))
+			continue;
+		ret = nv50_head_create(dev, i);
+		if (ret)
+			goto out;
+	}
+
+	/* create encoder/connector objects based on VBIOS DCB table */
+	for (i = 0, dcbe = &dcb->entry[0]; i < dcb->entries; i++, dcbe++) {
+		connector = nouveau_connector_create(dev, dcbe->connector);
+		if (IS_ERR(connector))
+			continue;
+
+		if (dcbe->location == DCB_LOC_ON_CHIP) {
+			switch (dcbe->type) {
+			case DCB_OUTPUT_TMDS:
+			case DCB_OUTPUT_LVDS:
+			case DCB_OUTPUT_DP:
+				ret = nv50_sor_create(connector, dcbe);
+				break;
+			case DCB_OUTPUT_ANALOG:
+				ret = nv50_dac_create(connector, dcbe);
+				break;
+			default:
+				ret = -ENODEV;
+				break;
+			}
+		} else {
+			ret = nv50_pior_create(connector, dcbe);
+		}
+
+		if (ret) {
+			NV_WARN(drm, "failed to create encoder %d/%d/%d: %d\n",
+				     dcbe->location, dcbe->type,
+				     ffs(dcbe->or) - 1, ret);
+			ret = 0;
+		}
+	}
+
+	/* cull any connectors we created that don't have an encoder */
+	list_for_each_entry_safe(connector, tmp, &dev->mode_config.connector_list, head) {
+		if (connector->encoder_ids[0])
+			continue;
+
+		NV_WARN(drm, "%s has no encoders, removing\n",
+			connector->name);
+		connector->funcs->destroy(connector);
+	}
+
+out:
+	if (ret)
+		nv50_display_destroy(dev);
+	return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.h b/drivers/gpu/drm/nouveau/dispnv50/disp.h
new file mode 100644
index 000000000000..e48c5eb35b49
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.h
@@ -0,0 +1,89 @@
+#ifndef __NV50_KMS_H__
+#define __NV50_KMS_H__
+#include <nvif/mem.h>
+
+#include "nouveau_display.h"
+
+struct nv50_disp {
+	struct nvif_disp *disp;
+	struct nv50_core *core;
+
+#define NV50_DISP_SYNC(c, o)                                ((c) * 0x040 + (o))
+#define NV50_DISP_CORE_NTFY                       NV50_DISP_SYNC(0      , 0x00)
+#define NV50_DISP_WNDW_SEM0(c)                    NV50_DISP_SYNC(1 + (c), 0x00)
+#define NV50_DISP_WNDW_SEM1(c)                    NV50_DISP_SYNC(1 + (c), 0x10)
+#define NV50_DISP_WNDW_NTFY(c)                    NV50_DISP_SYNC(1 + (c), 0x20)
+#define NV50_DISP_BASE_SEM0(c)                    NV50_DISP_WNDW_SEM0(0 + (c))
+#define NV50_DISP_BASE_SEM1(c)                    NV50_DISP_WNDW_SEM1(0 + (c))
+#define NV50_DISP_BASE_NTFY(c)                    NV50_DISP_WNDW_NTFY(0 + (c))
+#define NV50_DISP_OVLY_SEM0(c)                    NV50_DISP_WNDW_SEM0(4 + (c))
+#define NV50_DISP_OVLY_SEM1(c)                    NV50_DISP_WNDW_SEM1(4 + (c))
+#define NV50_DISP_OVLY_NTFY(c)                    NV50_DISP_WNDW_NTFY(4 + (c))
+	struct nouveau_bo *sync;
+
+	struct mutex mutex;
+};
+
+static inline struct nv50_disp *
+nv50_disp(struct drm_device *dev)
+{
+	return nouveau_display(dev)->priv;
+}
+
+struct nv50_disp_interlock {
+	enum nv50_disp_interlock_type {
+		NV50_DISP_INTERLOCK_CORE = 0,
+		NV50_DISP_INTERLOCK_CURS,
+		NV50_DISP_INTERLOCK_BASE,
+		NV50_DISP_INTERLOCK_OVLY,
+		NV50_DISP_INTERLOCK_WNDW,
+		NV50_DISP_INTERLOCK_WIMM,
+		NV50_DISP_INTERLOCK__SIZE
+	} type;
+	u32 data;
+};
+
+void corec37d_ntfy_init(struct nouveau_bo *, u32);
+
+struct nv50_chan {
+	struct nvif_object user;
+	struct nvif_device *device;
+};
+
+struct nv50_dmac {
+	struct nv50_chan base;
+
+	struct nvif_mem push;
+	u32 *ptr;
+
+	struct nvif_object sync;
+	struct nvif_object vram;
+
+	/* Protects against concurrent pushbuf access to this channel, lock is
+	 * grabbed by evo_wait (if the pushbuf reservation is successful) and
+	 * dropped again by evo_kick. */
+	struct mutex lock;
+};
+
+int nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
+		     const s32 *oclass, u8 head, void *data, u32 size,
+		     u64 syncbuf, struct nv50_dmac *dmac);
+void nv50_dmac_destroy(struct nv50_dmac *);
+
+u32 *evo_wait(struct nv50_dmac *, int nr);
+void evo_kick(u32 *, struct nv50_dmac *);
+
+#define evo_mthd(p, m, s) do {						\
+	const u32 _m = (m), _s = (s);					\
+	if (drm_debug & DRM_UT_KMS)					\
+		pr_err("%04x %d %s\n", _m, _s, __func__);		\
+	*((p)++) = ((_s << 18) | _m);					\
+} while(0)
+
+#define evo_data(p, d) do {						\
+	const u32 _d = (d);						\
+	if (drm_debug & DRM_UT_KMS)					\
+		pr_err("\t%08x\n", _d);					\
+	*((p)++) = _d;							\
+} while(0)
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv50/head.c b/drivers/gpu/drm/nouveau/dispnv50/head.c
new file mode 100644
index 000000000000..4f57e5379796
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/head.c
@@ -0,0 +1,511 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "head.h"
+#include "base.h"
+#include "core.h"
+#include "curs.h"
+#include "ovly.h"
+
+#include <nvif/class.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include "nouveau_connector.h"
+void
+nv50_head_flush_clr(struct nv50_head *head,
+		    struct nv50_head_atom *asyh, bool flush)
+{
+	union nv50_head_atom_mask clr = {
+		.mask = asyh->clr.mask & ~(flush ? 0 : asyh->set.mask),
+	};
+	if (clr.olut) head->func->olut_clr(head);
+	if (clr.core) head->func->core_clr(head);
+	if (clr.curs) head->func->curs_clr(head);
+}
+
+void
+nv50_head_flush_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	if (asyh->set.view   ) head->func->view    (head, asyh);
+	if (asyh->set.mode   ) head->func->mode    (head, asyh);
+	if (asyh->set.core   ) head->func->core_set(head, asyh);
+	if (asyh->set.olut   ) {
+		asyh->olut.offset = nv50_lut_load(&head->olut,
+						  asyh->olut.mode <= 1,
+						  asyh->olut.buffer,
+						  asyh->state.gamma_lut);
+		head->func->olut_set(head, asyh);
+	}
+	if (asyh->set.curs   ) head->func->curs_set(head, asyh);
+	if (asyh->set.base   ) head->func->base    (head, asyh);
+	if (asyh->set.ovly   ) head->func->ovly    (head, asyh);
+	if (asyh->set.dither ) head->func->dither  (head, asyh);
+	if (asyh->set.procamp) head->func->procamp (head, asyh);
+	if (asyh->set.or     ) head->func->or      (head, asyh);
+}
+
+static void
+nv50_head_atomic_check_procamp(struct nv50_head_atom *armh,
+			       struct nv50_head_atom *asyh,
+			       struct nouveau_conn_atom *asyc)
+{
+	const int vib = asyc->procamp.color_vibrance - 100;
+	const int hue = asyc->procamp.vibrant_hue - 90;
+	const int adj = (vib > 0) ? 50 : 0;
+	asyh->procamp.sat.cos = ((vib * 2047 + adj) / 100) & 0xfff;
+	asyh->procamp.sat.sin = ((hue * 2047) / 100) & 0xfff;
+	asyh->set.procamp = true;
+}
+
+static void
+nv50_head_atomic_check_dither(struct nv50_head_atom *armh,
+			      struct nv50_head_atom *asyh,
+			      struct nouveau_conn_atom *asyc)
+{
+	struct drm_connector *connector = asyc->state.connector;
+	u32 mode = 0x00;
+
+	if (asyc->dither.mode == DITHERING_MODE_AUTO) {
+		if (asyh->base.depth > connector->display_info.bpc * 3)
+			mode = DITHERING_MODE_DYNAMIC2X2;
+	} else {
+		mode = asyc->dither.mode;
+	}
+
+	if (asyc->dither.depth == DITHERING_DEPTH_AUTO) {
+		if (connector->display_info.bpc >= 8)
+			mode |= DITHERING_DEPTH_8BPC;
+	} else {
+		mode |= asyc->dither.depth;
+	}
+
+	asyh->dither.enable = mode;
+	asyh->dither.bits = mode >> 1;
+	asyh->dither.mode = mode >> 3;
+	asyh->set.dither = true;
+}
+
+static void
+nv50_head_atomic_check_view(struct nv50_head_atom *armh,
+			    struct nv50_head_atom *asyh,
+			    struct nouveau_conn_atom *asyc)
+{
+	struct drm_connector *connector = asyc->state.connector;
+	struct drm_display_mode *omode = &asyh->state.adjusted_mode;
+	struct drm_display_mode *umode = &asyh->state.mode;
+	int mode = asyc->scaler.mode;
+	struct edid *edid;
+	int umode_vdisplay, omode_hdisplay, omode_vdisplay;
+
+	if (connector->edid_blob_ptr)
+		edid = (struct edid *)connector->edid_blob_ptr->data;
+	else
+		edid = NULL;
+
+	if (!asyc->scaler.full) {
+		if (mode == DRM_MODE_SCALE_NONE)
+			omode = umode;
+	} else {
+		/* Non-EDID LVDS/eDP mode. */
+		mode = DRM_MODE_SCALE_FULLSCREEN;
+	}
+
+	/* For the user-specified mode, we must ignore doublescan and
+	 * the like, but honor frame packing.
+	 */
+	umode_vdisplay = umode->vdisplay;
+	if ((umode->flags & DRM_MODE_FLAG_3D_MASK) == DRM_MODE_FLAG_3D_FRAME_PACKING)
+		umode_vdisplay += umode->vtotal;
+	asyh->view.iW = umode->hdisplay;
+	asyh->view.iH = umode_vdisplay;
+	/* For the output mode, we can just use the stock helper. */
+	drm_mode_get_hv_timing(omode, &omode_hdisplay, &omode_vdisplay);
+	asyh->view.oW = omode_hdisplay;
+	asyh->view.oH = omode_vdisplay;
+
+	/* Add overscan compensation if necessary, will keep the aspect
+	 * ratio the same as the backend mode unless overridden by the
+	 * user setting both hborder and vborder properties.
+	 */
+	if ((asyc->scaler.underscan.mode == UNDERSCAN_ON ||
+	    (asyc->scaler.underscan.mode == UNDERSCAN_AUTO &&
+	     drm_detect_hdmi_monitor(edid)))) {
+		u32 bX = asyc->scaler.underscan.hborder;
+		u32 bY = asyc->scaler.underscan.vborder;
+		u32 r = (asyh->view.oH << 19) / asyh->view.oW;
+
+		if (bX) {
+			asyh->view.oW -= (bX * 2);
+			if (bY) asyh->view.oH -= (bY * 2);
+			else    asyh->view.oH  = ((asyh->view.oW * r) + (r / 2)) >> 19;
+		} else {
+			asyh->view.oW -= (asyh->view.oW >> 4) + 32;
+			if (bY) asyh->view.oH -= (bY * 2);
+			else    asyh->view.oH  = ((asyh->view.oW * r) + (r / 2)) >> 19;
+		}
+	}
+
+	/* Handle CENTER/ASPECT scaling, taking into account the areas
+	 * removed already for overscan compensation.
+	 */
+	switch (mode) {
+	case DRM_MODE_SCALE_CENTER:
+		asyh->view.oW = min((u16)umode->hdisplay, asyh->view.oW);
+		asyh->view.oH = min((u16)umode_vdisplay, asyh->view.oH);
+		/* fall-through */
+	case DRM_MODE_SCALE_ASPECT:
+		if (asyh->view.oH < asyh->view.oW) {
+			u32 r = (asyh->view.iW << 19) / asyh->view.iH;
+			asyh->view.oW = ((asyh->view.oH * r) + (r / 2)) >> 19;
+		} else {
+			u32 r = (asyh->view.iH << 19) / asyh->view.iW;
+			asyh->view.oH = ((asyh->view.oW * r) + (r / 2)) >> 19;
+		}
+		break;
+	default:
+		break;
+	}
+
+	asyh->set.view = true;
+}
+
+static int
+nv50_head_atomic_check_lut(struct nv50_head *head,
+			   struct nv50_head_atom *asyh)
+{
+	struct nv50_disp *disp = nv50_disp(head->base.base.dev);
+	struct drm_property_blob *olut = asyh->state.gamma_lut;
+
+	/* Determine whether core output LUT should be enabled. */
+	if (olut) {
+		/* Check if any window(s) have stolen the core output LUT
+		 * to as an input LUT for legacy gamma + I8 colour format.
+		 */
+		if (asyh->wndw.olut) {
+			/* If any window has stolen the core output LUT,
+			 * all of them must.
+			 */
+			if (asyh->wndw.olut != asyh->wndw.mask)
+				return -EINVAL;
+			olut = NULL;
+		}
+	}
+
+	if (!olut) {
+		asyh->olut.handle = 0;
+		return 0;
+	}
+
+	asyh->olut.handle = disp->core->chan.vram.handle;
+	asyh->olut.buffer = !asyh->olut.buffer;
+	head->func->olut(head, asyh);
+	return 0;
+}
+
+static void
+nv50_head_atomic_check_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct drm_display_mode *mode = &asyh->state.adjusted_mode;
+	struct nv50_head_mode *m = &asyh->mode;
+	u32 blankus;
+
+	drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V | CRTC_STEREO_DOUBLE);
+
+	/*
+	 * DRM modes are defined in terms of a repeating interval
+	 * starting with the active display area.  The hardware modes
+	 * are defined in terms of a repeating interval starting one
+	 * unit (pixel or line) into the sync pulse.  So, add bias.
+	 */
+
+	m->h.active = mode->crtc_htotal;
+	m->h.synce  = mode->crtc_hsync_end - mode->crtc_hsync_start - 1;
+	m->h.blanke = mode->crtc_hblank_end - mode->crtc_hsync_start - 1;
+	m->h.blanks = m->h.blanke + mode->crtc_hdisplay;
+
+	m->v.active = mode->crtc_vtotal;
+	m->v.synce  = mode->crtc_vsync_end - mode->crtc_vsync_start - 1;
+	m->v.blanke = mode->crtc_vblank_end - mode->crtc_vsync_start - 1;
+	m->v.blanks = m->v.blanke + mode->crtc_vdisplay;
+
+	/*XXX: Safe underestimate, even "0" works */
+	blankus = (m->v.active - mode->crtc_vdisplay - 2) * m->h.active;
+	blankus *= 1000;
+	blankus /= mode->crtc_clock;
+	m->v.blankus = blankus;
+
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
+		m->v.blank2e =  m->v.active + m->v.blanke;
+		m->v.blank2s =  m->v.blank2e + mode->crtc_vdisplay;
+		m->v.active  = (m->v.active * 2) + 1;
+		m->interlace = true;
+	} else {
+		m->v.blank2e = 0;
+		m->v.blank2s = 1;
+		m->interlace = false;
+	}
+	m->clock = mode->crtc_clock;
+
+	asyh->or.nhsync = !!(mode->flags & DRM_MODE_FLAG_NHSYNC);
+	asyh->or.nvsync = !!(mode->flags & DRM_MODE_FLAG_NVSYNC);
+	asyh->set.or = head->func->or != NULL;
+	asyh->set.mode = true;
+}
+
+static int
+nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state)
+{
+	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
+	struct nv50_head *head = nv50_head(crtc);
+	struct nv50_head_atom *armh = nv50_head_atom(crtc->state);
+	struct nv50_head_atom *asyh = nv50_head_atom(state);
+	struct nouveau_conn_atom *asyc = NULL;
+	struct drm_connector_state *conns;
+	struct drm_connector *conn;
+	int i;
+
+	NV_ATOMIC(drm, "%s atomic_check %d\n", crtc->name, asyh->state.active);
+	if (asyh->state.active) {
+		for_each_new_connector_in_state(asyh->state.state, conn, conns, i) {
+			if (conns->crtc == crtc) {
+				asyc = nouveau_conn_atom(conns);
+				break;
+			}
+		}
+
+		if (armh->state.active) {
+			if (asyc) {
+				if (asyh->state.mode_changed)
+					asyc->set.scaler = true;
+				if (armh->base.depth != asyh->base.depth)
+					asyc->set.dither = true;
+			}
+		} else {
+			if (asyc)
+				asyc->set.mask = ~0;
+			asyh->set.mask = ~0;
+			asyh->set.or = head->func->or != NULL;
+		}
+
+		if (asyh->state.mode_changed)
+			nv50_head_atomic_check_mode(head, asyh);
+
+		if (asyh->state.color_mgmt_changed ||
+		    memcmp(&armh->wndw, &asyh->wndw, sizeof(asyh->wndw))) {
+			int ret = nv50_head_atomic_check_lut(head, asyh);
+			if (ret)
+				return ret;
+
+			asyh->olut.visible = asyh->olut.handle != 0;
+		}
+
+		if (asyc) {
+			if (asyc->set.scaler)
+				nv50_head_atomic_check_view(armh, asyh, asyc);
+			if (asyc->set.dither)
+				nv50_head_atomic_check_dither(armh, asyh, asyc);
+			if (asyc->set.procamp)
+				nv50_head_atomic_check_procamp(armh, asyh, asyc);
+		}
+
+		if (head->func->core_calc) {
+			head->func->core_calc(head, asyh);
+			if (!asyh->core.visible)
+				asyh->olut.visible = false;
+		}
+
+		asyh->set.base = armh->base.cpp != asyh->base.cpp;
+		asyh->set.ovly = armh->ovly.cpp != asyh->ovly.cpp;
+	} else {
+		asyh->olut.visible = false;
+		asyh->core.visible = false;
+		asyh->curs.visible = false;
+		asyh->base.cpp = 0;
+		asyh->ovly.cpp = 0;
+	}
+
+	if (!drm_atomic_crtc_needs_modeset(&asyh->state)) {
+		if (asyh->core.visible) {
+			if (memcmp(&armh->core, &asyh->core, sizeof(asyh->core)))
+				asyh->set.core = true;
+		} else
+		if (armh->core.visible) {
+			asyh->clr.core = true;
+		}
+
+		if (asyh->curs.visible) {
+			if (memcmp(&armh->curs, &asyh->curs, sizeof(asyh->curs)))
+				asyh->set.curs = true;
+		} else
+		if (armh->curs.visible) {
+			asyh->clr.curs = true;
+		}
+
+		if (asyh->olut.visible) {
+			if (memcmp(&armh->olut, &asyh->olut, sizeof(asyh->olut)))
+				asyh->set.olut = true;
+		} else
+		if (armh->olut.visible) {
+			asyh->clr.olut = true;
+		}
+	} else {
+		asyh->clr.olut = armh->olut.visible;
+		asyh->clr.core = armh->core.visible;
+		asyh->clr.curs = armh->curs.visible;
+		asyh->set.olut = asyh->olut.visible;
+		asyh->set.core = asyh->core.visible;
+		asyh->set.curs = asyh->curs.visible;
+	}
+
+	if (asyh->clr.mask || asyh->set.mask)
+		nv50_atom(asyh->state.state)->lock_core = true;
+	return 0;
+}
+
+static const struct drm_crtc_helper_funcs
+nv50_head_help = {
+	.atomic_check = nv50_head_atomic_check,
+};
+
+static void
+nv50_head_atomic_destroy_state(struct drm_crtc *crtc,
+			       struct drm_crtc_state *state)
+{
+	struct nv50_head_atom *asyh = nv50_head_atom(state);
+	__drm_atomic_helper_crtc_destroy_state(&asyh->state);
+	kfree(asyh);
+}
+
+static struct drm_crtc_state *
+nv50_head_atomic_duplicate_state(struct drm_crtc *crtc)
+{
+	struct nv50_head_atom *armh = nv50_head_atom(crtc->state);
+	struct nv50_head_atom *asyh;
+	if (!(asyh = kmalloc(sizeof(*asyh), GFP_KERNEL)))
+		return NULL;
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &asyh->state);
+	asyh->wndw = armh->wndw;
+	asyh->view = armh->view;
+	asyh->mode = armh->mode;
+	asyh->olut = armh->olut;
+	asyh->core = armh->core;
+	asyh->curs = armh->curs;
+	asyh->base = armh->base;
+	asyh->ovly = armh->ovly;
+	asyh->dither = armh->dither;
+	asyh->procamp = armh->procamp;
+	asyh->clr.mask = 0;
+	asyh->set.mask = 0;
+	return &asyh->state;
+}
+
+static void
+__drm_atomic_helper_crtc_reset(struct drm_crtc *crtc,
+			       struct drm_crtc_state *state)
+{
+	if (crtc->state)
+		crtc->funcs->atomic_destroy_state(crtc, crtc->state);
+	crtc->state = state;
+	crtc->state->crtc = crtc;
+}
+
+static void
+nv50_head_reset(struct drm_crtc *crtc)
+{
+	struct nv50_head_atom *asyh;
+
+	if (WARN_ON(!(asyh = kzalloc(sizeof(*asyh), GFP_KERNEL))))
+		return;
+
+	__drm_atomic_helper_crtc_reset(crtc, &asyh->state);
+}
+
+static void
+nv50_head_destroy(struct drm_crtc *crtc)
+{
+	struct nv50_head *head = nv50_head(crtc);
+	nv50_lut_fini(&head->olut);
+	drm_crtc_cleanup(crtc);
+	kfree(head);
+}
+
+static const struct drm_crtc_funcs
+nv50_head_func = {
+	.reset = nv50_head_reset,
+	.gamma_set = drm_atomic_helper_legacy_gamma_set,
+	.destroy = nv50_head_destroy,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = nv50_head_atomic_duplicate_state,
+	.atomic_destroy_state = nv50_head_atomic_destroy_state,
+};
+
+int
+nv50_head_create(struct drm_device *dev, int index)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nv50_disp *disp = nv50_disp(dev);
+	struct nv50_head *head;
+	struct nv50_wndw *curs, *wndw;
+	struct drm_crtc *crtc;
+	int ret;
+
+	head = kzalloc(sizeof(*head), GFP_KERNEL);
+	if (!head)
+		return -ENOMEM;
+
+	head->func = disp->core->func->head;
+	head->base.index = index;
+
+	if (disp->disp->object.oclass < GV100_DISP) {
+		ret = nv50_ovly_new(drm, head->base.index, &wndw);
+		ret = nv50_base_new(drm, head->base.index, &wndw);
+	} else {
+		ret = nv50_wndw_new(drm, DRM_PLANE_TYPE_OVERLAY,
+				    head->base.index * 2 + 1, &wndw);
+		ret = nv50_wndw_new(drm, DRM_PLANE_TYPE_PRIMARY,
+				    head->base.index * 2 + 0, &wndw);
+	}
+	if (ret == 0)
+		ret = nv50_curs_new(drm, head->base.index, &curs);
+	if (ret) {
+		kfree(head);
+		return ret;
+	}
+
+	crtc = &head->base.base;
+	drm_crtc_init_with_planes(dev, crtc, &wndw->plane, &curs->plane,
+				  &nv50_head_func, "head-%d", head->base.index);
+	drm_crtc_helper_add(crtc, &nv50_head_help);
+	drm_mode_crtc_set_gamma_size(crtc, 256);
+
+	if (head->func->olut_set) {
+		ret = nv50_lut_init(disp, &drm->client.mmu, &head->olut);
+		if (ret)
+			goto out;
+	}
+
+out:
+	if (ret)
+		nv50_head_destroy(crtc);
+	return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/head.h b/drivers/gpu/drm/nouveau/dispnv50/head.h
new file mode 100644
index 000000000000..37b3248c6dae
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/head.h
@@ -0,0 +1,78 @@
+#ifndef __NV50_KMS_HEAD_H__
+#define __NV50_KMS_HEAD_H__
+#define nv50_head(c) container_of((c), struct nv50_head, base.base)
+#include "disp.h"
+#include "atom.h"
+#include "lut.h"
+
+#include "nouveau_crtc.h"
+
+struct nv50_head {
+	const struct nv50_head_func *func;
+	struct nouveau_crtc base;
+	struct nv50_lut olut;
+};
+
+int nv50_head_create(struct drm_device *, int index);
+void nv50_head_flush_set(struct nv50_head *, struct nv50_head_atom *);
+void nv50_head_flush_clr(struct nv50_head *, struct nv50_head_atom *, bool y);
+
+struct nv50_head_func {
+	void (*view)(struct nv50_head *, struct nv50_head_atom *);
+	void (*mode)(struct nv50_head *, struct nv50_head_atom *);
+	void (*olut)(struct nv50_head *, struct nv50_head_atom *);
+	void (*olut_set)(struct nv50_head *, struct nv50_head_atom *);
+	void (*olut_clr)(struct nv50_head *);
+	void (*core_calc)(struct nv50_head *, struct nv50_head_atom *);
+	void (*core_set)(struct nv50_head *, struct nv50_head_atom *);
+	void (*core_clr)(struct nv50_head *);
+	int (*curs_layout)(struct nv50_head *, struct nv50_wndw_atom *,
+			   struct nv50_head_atom *);
+	int (*curs_format)(struct nv50_head *, struct nv50_wndw_atom *,
+			   struct nv50_head_atom *);
+	void (*curs_set)(struct nv50_head *, struct nv50_head_atom *);
+	void (*curs_clr)(struct nv50_head *);
+	void (*base)(struct nv50_head *, struct nv50_head_atom *);
+	void (*ovly)(struct nv50_head *, struct nv50_head_atom *);
+	void (*dither)(struct nv50_head *, struct nv50_head_atom *);
+	void (*procamp)(struct nv50_head *, struct nv50_head_atom *);
+	void (*or)(struct nv50_head *, struct nv50_head_atom *);
+};
+
+extern const struct nv50_head_func head507d;
+void head507d_view(struct nv50_head *, struct nv50_head_atom *);
+void head507d_mode(struct nv50_head *, struct nv50_head_atom *);
+void head507d_olut(struct nv50_head *, struct nv50_head_atom *);
+void head507d_core_calc(struct nv50_head *, struct nv50_head_atom *);
+void head507d_core_clr(struct nv50_head *);
+int head507d_curs_layout(struct nv50_head *, struct nv50_wndw_atom *,
+			 struct nv50_head_atom *);
+int head507d_curs_format(struct nv50_head *, struct nv50_wndw_atom *,
+			 struct nv50_head_atom *);
+void head507d_base(struct nv50_head *, struct nv50_head_atom *);
+void head507d_ovly(struct nv50_head *, struct nv50_head_atom *);
+void head507d_dither(struct nv50_head *, struct nv50_head_atom *);
+void head507d_procamp(struct nv50_head *, struct nv50_head_atom *);
+
+extern const struct nv50_head_func head827d;
+
+extern const struct nv50_head_func head907d;
+void head907d_view(struct nv50_head *, struct nv50_head_atom *);
+void head907d_mode(struct nv50_head *, struct nv50_head_atom *);
+void head907d_olut(struct nv50_head *, struct nv50_head_atom *);
+void head907d_olut_set(struct nv50_head *, struct nv50_head_atom *);
+void head907d_olut_clr(struct nv50_head *);
+void head907d_core_set(struct nv50_head *, struct nv50_head_atom *);
+void head907d_core_clr(struct nv50_head *);
+void head907d_curs_set(struct nv50_head *, struct nv50_head_atom *);
+void head907d_curs_clr(struct nv50_head *);
+void head907d_ovly(struct nv50_head *, struct nv50_head_atom *);
+void head907d_procamp(struct nv50_head *, struct nv50_head_atom *);
+void head907d_or(struct nv50_head *, struct nv50_head_atom *);
+
+extern const struct nv50_head_func head917d;
+int head917d_curs_layout(struct nv50_head *, struct nv50_wndw_atom *,
+			 struct nv50_head_atom *);
+
+extern const struct nv50_head_func headc37d;
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv50/head507d.c b/drivers/gpu/drm/nouveau/dispnv50/head507d.c
new file mode 100644
index 000000000000..51bc5996fd37
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/head507d.c
@@ -0,0 +1,325 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "head.h"
+#include "core.h"
+
+void
+head507d_procamp(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x08a8 + (head->base.index * 0x400), 1);
+		evo_data(push, asyh->procamp.sat.sin << 20 |
+			       asyh->procamp.sat.cos << 8);
+		evo_kick(push, core);
+	}
+}
+
+void
+head507d_dither(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x08a0 + (head->base.index * 0x0400), 1);
+		evo_data(push, asyh->dither.mode << 3 |
+			       asyh->dither.bits << 1 |
+			       asyh->dither.enable);
+		evo_kick(push, core);
+	}
+}
+
+void
+head507d_ovly(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 bounds = 0;
+	u32 *push;
+
+	if (asyh->ovly.cpp) {
+		switch (asyh->ovly.cpp) {
+		case 4: bounds |= 0x00000300; break;
+		case 2: bounds |= 0x00000100; break;
+		default:
+			WARN_ON(1);
+			break;
+		}
+		bounds |= 0x00000001;
+	} else {
+		bounds |= 0x00000100;
+	}
+
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x0904 + head->base.index * 0x400, 1);
+		evo_data(push, bounds);
+		evo_kick(push, core);
+	}
+}
+
+void
+head507d_base(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 bounds = 0;
+	u32 *push;
+
+	if (asyh->base.cpp) {
+		switch (asyh->base.cpp) {
+		case 8: bounds |= 0x00000500; break;
+		case 4: bounds |= 0x00000300; break;
+		case 2: bounds |= 0x00000100; break;
+		case 1: bounds |= 0x00000000; break;
+		default:
+			WARN_ON(1);
+			break;
+		}
+		bounds |= 0x00000001;
+	}
+
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x0900 + head->base.index * 0x400, 1);
+		evo_data(push, bounds);
+		evo_kick(push, core);
+	}
+}
+
+static void
+head507d_curs_clr(struct nv50_head *head)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x0880 + head->base.index * 0x400, 1);
+		evo_data(push, 0x05000000);
+		evo_kick(push, core);
+	}
+}
+
+static void
+head507d_curs_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 3))) {
+		evo_mthd(push, 0x0880 + head->base.index * 0x400, 2);
+		evo_data(push, 0x80000000 | asyh->curs.layout << 26 |
+					    asyh->curs.format << 24);
+		evo_data(push, asyh->curs.offset >> 8);
+		evo_kick(push, core);
+	}
+}
+
+int
+head507d_curs_format(struct nv50_head *head, struct nv50_wndw_atom *asyw,
+		     struct nv50_head_atom *asyh)
+{
+	switch (asyw->image.format) {
+	case 0xcf: asyh->curs.format = 1; break;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+int
+head507d_curs_layout(struct nv50_head *head, struct nv50_wndw_atom *asyw,
+		     struct nv50_head_atom *asyh)
+{
+	switch (asyw->image.w) {
+	case 32: asyh->curs.layout = 0; break;
+	case 64: asyh->curs.layout = 1; break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+void
+head507d_core_clr(struct nv50_head *head)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x0874 + head->base.index * 0x400, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, core);
+	}
+}
+
+static void
+head507d_core_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 9))) {
+		evo_mthd(push, 0x0860 + head->base.index * 0x400, 1);
+		evo_data(push, asyh->core.offset >> 8);
+		evo_mthd(push, 0x0868 + head->base.index * 0x400, 4);
+		evo_data(push, asyh->core.h << 16 | asyh->core.w);
+		evo_data(push, asyh->core.layout << 20 |
+			       (asyh->core.pitch >> 8) << 8 |
+			       asyh->core.blocks << 8 |
+			       asyh->core.blockh);
+		evo_data(push, asyh->core.kind << 16 |
+			       asyh->core.format << 8);
+		evo_data(push, asyh->core.handle);
+		evo_mthd(push, 0x08c0 + head->base.index * 0x400, 1);
+		evo_data(push, asyh->core.y << 16 | asyh->core.x);
+		evo_kick(push, core);
+
+		/* EVO will complain with INVALID_STATE if we have an
+		 * active cursor and (re)specify HeadSetContextDmaIso
+		 * without also updating HeadSetOffsetCursor.
+		 */
+		asyh->set.curs = asyh->curs.visible;
+		asyh->set.olut = asyh->olut.handle != 0;
+	}
+}
+
+void
+head507d_core_calc(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_disp *disp = nv50_disp(head->base.base.dev);
+	if ((asyh->core.visible = (asyh->base.cpp != 0))) {
+		asyh->core.x = asyh->base.x;
+		asyh->core.y = asyh->base.y;
+		asyh->core.w = asyh->base.w;
+		asyh->core.h = asyh->base.h;
+	} else
+	if ((asyh->core.visible = (asyh->ovly.cpp != 0)) ||
+	    (asyh->core.visible = asyh->curs.visible)) {
+		/*XXX: We need to either find some way of having the
+		 *     primary base layer appear black, while still
+		 *     being able to display the other layers, or we
+		 *     need to allocate a dummy black surface here.
+		 */
+		asyh->core.x = 0;
+		asyh->core.y = 0;
+		asyh->core.w = asyh->state.mode.hdisplay;
+		asyh->core.h = asyh->state.mode.vdisplay;
+	}
+	asyh->core.handle = disp->core->chan.vram.handle;
+	asyh->core.offset = 0;
+	asyh->core.format = 0xcf;
+	asyh->core.kind = 0;
+	asyh->core.layout = 1;
+	asyh->core.blockh = 0;
+	asyh->core.blocks = 0;
+	asyh->core.pitch = ALIGN(asyh->core.w, 64) * 4;
+}
+
+static void
+head507d_olut_clr(struct nv50_head *head)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x0840 + (head->base.index * 0x400), 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, core);
+	}
+}
+
+static void
+head507d_olut_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 3))) {
+		evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2);
+		evo_data(push, 0x80000000 | asyh->olut.mode << 30);
+		evo_data(push, asyh->olut.offset >> 8);
+		evo_kick(push, core);
+	}
+}
+
+void
+head507d_olut(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	if (asyh->base.cpp == 1)
+		asyh->olut.mode = 0;
+	else
+		asyh->olut.mode = 1;
+}
+
+void
+head507d_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	struct nv50_head_mode *m = &asyh->mode;
+	u32 *push;
+	if ((push = evo_wait(core, 13))) {
+		evo_mthd(push, 0x0804 + (head->base.index * 0x400), 2);
+		evo_data(push, 0x00800000 | m->clock);
+		evo_data(push, m->interlace ? 0x00000002 : 0x00000000);
+		evo_mthd(push, 0x0810 + (head->base.index * 0x400), 7);
+		evo_data(push, 0x00000000);
+		evo_data(push, m->v.active  << 16 | m->h.active );
+		evo_data(push, m->v.synce   << 16 | m->h.synce  );
+		evo_data(push, m->v.blanke  << 16 | m->h.blanke );
+		evo_data(push, m->v.blanks  << 16 | m->h.blanks );
+		evo_data(push, m->v.blank2e << 16 | m->v.blank2s);
+		evo_data(push, asyh->mode.v.blankus);
+		evo_mthd(push, 0x082c + (head->base.index * 0x400), 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, core);
+	}
+}
+
+void
+head507d_view(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 7))) {
+		evo_mthd(push, 0x08a4 + (head->base.index * 0x400), 1);
+		evo_data(push, 0x00000000);
+		evo_mthd(push, 0x08c8 + (head->base.index * 0x400), 1);
+		evo_data(push, asyh->view.iH << 16 | asyh->view.iW);
+		evo_mthd(push, 0x08d8 + (head->base.index * 0x400), 2);
+		evo_data(push, asyh->view.oH << 16 | asyh->view.oW);
+		evo_data(push, asyh->view.oH << 16 | asyh->view.oW);
+		evo_kick(push, core);
+	}
+}
+
+const struct nv50_head_func
+head507d = {
+	.view = head507d_view,
+	.mode = head507d_mode,
+	.olut = head507d_olut,
+	.olut_set = head507d_olut_set,
+	.olut_clr = head507d_olut_clr,
+	.core_calc = head507d_core_calc,
+	.core_set = head507d_core_set,
+	.core_clr = head507d_core_clr,
+	.curs_layout = head507d_curs_layout,
+	.curs_format = head507d_curs_format,
+	.curs_set = head507d_curs_set,
+	.curs_clr = head507d_curs_clr,
+	.base = head507d_base,
+	.ovly = head507d_ovly,
+	.dither = head507d_dither,
+	.procamp = head507d_procamp,
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv50/head827d.c b/drivers/gpu/drm/nouveau/dispnv50/head827d.c
new file mode 100644
index 000000000000..af5e7bd5978b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/head827d.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "head.h"
+#include "core.h"
+
+static void
+head827d_curs_clr(struct nv50_head *head)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 4))) {
+		evo_mthd(push, 0x0880 + head->base.index * 0x400, 1);
+		evo_data(push, 0x05000000);
+		evo_mthd(push, 0x089c + head->base.index * 0x400, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, core);
+	}
+}
+
+static void
+head827d_curs_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 5))) {
+		evo_mthd(push, 0x0880 + head->base.index * 0x400, 2);
+		evo_data(push, 0x80000000 | asyh->curs.layout << 26 |
+					    asyh->curs.format << 24);
+		evo_data(push, asyh->curs.offset >> 8);
+		evo_mthd(push, 0x089c + head->base.index * 0x400, 1);
+		evo_data(push, asyh->curs.handle);
+		evo_kick(push, core);
+	}
+}
+
+static void
+head827d_core_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 9))) {
+		evo_mthd(push, 0x0860 + head->base.index * 0x400, 1);
+		evo_data(push, asyh->core.offset >> 8);
+		evo_mthd(push, 0x0868 + head->base.index * 0x400, 4);
+		evo_data(push, asyh->core.h << 16 | asyh->core.w);
+		evo_data(push, asyh->core.layout << 20 |
+			       (asyh->core.pitch >> 8) << 8 |
+			       asyh->core.blocks << 8 |
+			       asyh->core.blockh);
+		evo_data(push, asyh->core.format << 8);
+		evo_data(push, asyh->core.handle);
+		evo_mthd(push, 0x08c0 + head->base.index * 0x400, 1);
+		evo_data(push, asyh->core.y << 16 | asyh->core.x);
+		evo_kick(push, core);
+	}
+}
+
+static void
+head827d_olut_clr(struct nv50_head *head)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 4))) {
+		evo_mthd(push, 0x0840 + (head->base.index * 0x400), 1);
+		evo_data(push, 0x00000000);
+		evo_mthd(push, 0x085c + (head->base.index * 0x400), 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, core);
+	}
+}
+
+static void
+head827d_olut_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 5))) {
+		evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2);
+		evo_data(push, 0x80000000 | asyh->olut.mode << 30);
+		evo_data(push, asyh->olut.offset >> 8);
+		evo_mthd(push, 0x085c + (head->base.index * 0x400), 1);
+		evo_data(push, asyh->olut.handle);
+		evo_kick(push, core);
+	}
+}
+
+const struct nv50_head_func
+head827d = {
+	.view = head507d_view,
+	.mode = head507d_mode,
+	.olut = head507d_olut,
+	.olut_set = head827d_olut_set,
+	.olut_clr = head827d_olut_clr,
+	.core_calc = head507d_core_calc,
+	.core_set = head827d_core_set,
+	.core_clr = head507d_core_clr,
+	.curs_layout = head507d_curs_layout,
+	.curs_format = head507d_curs_format,
+	.curs_set = head827d_curs_set,
+	.curs_clr = head827d_curs_clr,
+	.base = head507d_base,
+	.ovly = head507d_ovly,
+	.dither = head507d_dither,
+	.procamp = head507d_procamp,
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv50/head907d.c b/drivers/gpu/drm/nouveau/dispnv50/head907d.c
new file mode 100644
index 000000000000..633907163eb1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/head907d.c
@@ -0,0 +1,284 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "head.h"
+#include "core.h"
+
+void
+head907d_or(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 3))) {
+		evo_mthd(push, 0x0404 + (head->base.index * 0x300), 2);
+		evo_data(push, 0x00000001 | asyh->or.depth  << 6 |
+					    asyh->or.nvsync << 4 |
+					    asyh->or.nhsync << 3);
+		evo_data(push, 0x31ec6000 | head->base.index << 25 |
+					    asyh->mode.interlace);
+		evo_kick(push, core);
+	}
+}
+
+void
+head907d_procamp(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x0498 + (head->base.index * 0x300), 1);
+		evo_data(push, asyh->procamp.sat.sin << 20 |
+			       asyh->procamp.sat.cos << 8);
+		evo_kick(push, core);
+	}
+}
+
+static void
+head907d_dither(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x0490 + (head->base.index * 0x0300), 1);
+		evo_data(push, asyh->dither.mode << 3 |
+			       asyh->dither.bits << 1 |
+			       asyh->dither.enable);
+		evo_kick(push, core);
+	}
+}
+
+void
+head907d_ovly(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 bounds = 0;
+	u32 *push;
+
+	if (asyh->ovly.cpp) {
+		switch (asyh->ovly.cpp) {
+		case 8: bounds |= 0x00000500; break;
+		case 4: bounds |= 0x00000300; break;
+		case 2: bounds |= 0x00000100; break;
+		default:
+			WARN_ON(1);
+			break;
+		}
+		bounds |= 0x00000001;
+	} else {
+		bounds |= 0x00000100;
+	}
+
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x04d4 + head->base.index * 0x300, 1);
+		evo_data(push, bounds);
+		evo_kick(push, core);
+	}
+}
+
+static void
+head907d_base(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 bounds = 0;
+	u32 *push;
+
+	if (asyh->base.cpp) {
+		switch (asyh->base.cpp) {
+		case 8: bounds |= 0x00000500; break;
+		case 4: bounds |= 0x00000300; break;
+		case 2: bounds |= 0x00000100; break;
+		case 1: bounds |= 0x00000000; break;
+		default:
+			WARN_ON(1);
+			break;
+		}
+		bounds |= 0x00000001;
+	}
+
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x04d0 + head->base.index * 0x300, 1);
+		evo_data(push, bounds);
+		evo_kick(push, core);
+	}
+}
+
+void
+head907d_curs_clr(struct nv50_head *head)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 4))) {
+		evo_mthd(push, 0x0480 + head->base.index * 0x300, 1);
+		evo_data(push, 0x05000000);
+		evo_mthd(push, 0x048c + head->base.index * 0x300, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, core);
+	}
+}
+
+void
+head907d_curs_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 5))) {
+		evo_mthd(push, 0x0480 + head->base.index * 0x300, 2);
+		evo_data(push, 0x80000000 | asyh->curs.layout << 26 |
+					    asyh->curs.format << 24);
+		evo_data(push, asyh->curs.offset >> 8);
+		evo_mthd(push, 0x048c + head->base.index * 0x300, 1);
+		evo_data(push, asyh->curs.handle);
+		evo_kick(push, core);
+	}
+}
+
+void
+head907d_core_clr(struct nv50_head *head)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x0474 + head->base.index * 0x300, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, core);
+	}
+}
+
+void
+head907d_core_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 9))) {
+		evo_mthd(push, 0x0460 + head->base.index * 0x300, 1);
+		evo_data(push, asyh->core.offset >> 8);
+		evo_mthd(push, 0x0468 + head->base.index * 0x300, 4);
+		evo_data(push, asyh->core.h << 16 | asyh->core.w);
+		evo_data(push, asyh->core.layout << 24 |
+			       (asyh->core.pitch >> 8) << 8 |
+			       asyh->core.blocks << 8 |
+			       asyh->core.blockh);
+		evo_data(push, asyh->core.format << 8);
+		evo_data(push, asyh->core.handle);
+		evo_mthd(push, 0x04b0 + head->base.index * 0x300, 1);
+		evo_data(push, asyh->core.y << 16 | asyh->core.x);
+		evo_kick(push, core);
+	}
+}
+
+void
+head907d_olut_clr(struct nv50_head *head)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 4))) {
+		evo_mthd(push, 0x0448 + (head->base.index * 0x300), 1);
+		evo_data(push, 0x00000000);
+		evo_mthd(push, 0x045c + (head->base.index * 0x300), 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, core);
+	}
+}
+
+void
+head907d_olut_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 5))) {
+		evo_mthd(push, 0x0448 + (head->base.index * 0x300), 2);
+		evo_data(push, 0x80000000 | asyh->olut.mode << 24);
+		evo_data(push, asyh->olut.offset >> 8);
+		evo_mthd(push, 0x045c + (head->base.index * 0x300), 1);
+		evo_data(push, asyh->olut.handle);
+		evo_kick(push, core);
+	}
+}
+
+void
+head907d_olut(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	asyh->olut.mode = 7;
+}
+
+void
+head907d_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	struct nv50_head_mode *m = &asyh->mode;
+	u32 *push;
+	if ((push = evo_wait(core, 14))) {
+		evo_mthd(push, 0x0410 + (head->base.index * 0x300), 6);
+		evo_data(push, 0x00000000);
+		evo_data(push, m->v.active  << 16 | m->h.active );
+		evo_data(push, m->v.synce   << 16 | m->h.synce  );
+		evo_data(push, m->v.blanke  << 16 | m->h.blanke );
+		evo_data(push, m->v.blanks  << 16 | m->h.blanks );
+		evo_data(push, m->v.blank2e << 16 | m->v.blank2s);
+		evo_mthd(push, 0x042c + (head->base.index * 0x300), 2);
+		evo_data(push, 0x00000000); /* ??? */
+		evo_data(push, 0xffffff00);
+		evo_mthd(push, 0x0450 + (head->base.index * 0x300), 3);
+		evo_data(push, m->clock * 1000);
+		evo_data(push, 0x00200000); /* ??? */
+		evo_data(push, m->clock * 1000);
+		evo_kick(push, core);
+	}
+}
+
+void
+head907d_view(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 8))) {
+		evo_mthd(push, 0x0494 + (head->base.index * 0x300), 1);
+		evo_data(push, 0x00000000);
+		evo_mthd(push, 0x04b8 + (head->base.index * 0x300), 1);
+		evo_data(push, asyh->view.iH << 16 | asyh->view.iW);
+		evo_mthd(push, 0x04c0 + (head->base.index * 0x300), 3);
+		evo_data(push, asyh->view.oH << 16 | asyh->view.oW);
+		evo_data(push, asyh->view.oH << 16 | asyh->view.oW);
+		evo_data(push, asyh->view.oH << 16 | asyh->view.oW);
+		evo_kick(push, core);
+	}
+}
+
+const struct nv50_head_func
+head907d = {
+	.view = head907d_view,
+	.mode = head907d_mode,
+	.olut = head907d_olut,
+	.olut_set = head907d_olut_set,
+	.olut_clr = head907d_olut_clr,
+	.core_calc = head507d_core_calc,
+	.core_set = head907d_core_set,
+	.core_clr = head907d_core_clr,
+	.curs_layout = head507d_curs_layout,
+	.curs_format = head507d_curs_format,
+	.curs_set = head907d_curs_set,
+	.curs_clr = head907d_curs_clr,
+	.base = head907d_base,
+	.ovly = head907d_ovly,
+	.dither = head907d_dither,
+	.procamp = head907d_procamp,
+	.or = head907d_or,
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv50/head917d.c b/drivers/gpu/drm/nouveau/dispnv50/head917d.c
new file mode 100644
index 000000000000..303df8459ca8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/head917d.c
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "head.h"
+#include "core.h"
+
+static void
+head917d_dither(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x04a0 + (head->base.index * 0x0300), 1);
+		evo_data(push, asyh->dither.mode << 3 |
+			       asyh->dither.bits << 1 |
+			       asyh->dither.enable);
+		evo_kick(push, core);
+	}
+}
+
+static void
+head917d_base(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 bounds = 0;
+	u32 *push;
+
+	if (asyh->base.cpp) {
+		switch (asyh->base.cpp) {
+		case 8: bounds |= 0x00000500; break;
+		case 4: bounds |= 0x00000300; break;
+		case 2: bounds |= 0x00000100; break;
+		case 1: bounds |= 0x00000000; break;
+		default:
+			WARN_ON(1);
+			break;
+		}
+		bounds |= 0x00020001;
+	}
+
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x04d0 + head->base.index * 0x300, 1);
+		evo_data(push, bounds);
+		evo_kick(push, core);
+	}
+}
+
+int
+head917d_curs_layout(struct nv50_head *head, struct nv50_wndw_atom *asyw,
+		     struct nv50_head_atom *asyh)
+{
+	switch (asyw->state.fb->width) {
+	case  32: asyh->curs.layout = 0; break;
+	case  64: asyh->curs.layout = 1; break;
+	case 128: asyh->curs.layout = 2; break;
+	case 256: asyh->curs.layout = 3; break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+const struct nv50_head_func
+head917d = {
+	.view = head907d_view,
+	.mode = head907d_mode,
+	.olut = head907d_olut,
+	.olut_set = head907d_olut_set,
+	.olut_clr = head907d_olut_clr,
+	.core_calc = head507d_core_calc,
+	.core_set = head907d_core_set,
+	.core_clr = head907d_core_clr,
+	.curs_layout = head917d_curs_layout,
+	.curs_format = head507d_curs_format,
+	.curs_set = head907d_curs_set,
+	.curs_clr = head907d_curs_clr,
+	.base = head917d_base,
+	.ovly = head907d_ovly,
+	.dither = head917d_dither,
+	.procamp = head907d_procamp,
+	.or = head907d_or,
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv50/headc37d.c b/drivers/gpu/drm/nouveau/dispnv50/headc37d.c
new file mode 100644
index 000000000000..989c14083066
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/headc37d.c
@@ -0,0 +1,212 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "head.h"
+#include "atom.h"
+#include "core.h"
+
+static void
+headc37d_or(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		/*XXX: This is a dirty hack until OR depth handling is
+		 *     improved later for deep colour etc.
+		 */
+		switch (asyh->or.depth) {
+		case 6: asyh->or.depth = 5; break;
+		case 5: asyh->or.depth = 4; break;
+		case 2: asyh->or.depth = 1; break;
+		case 0:	asyh->or.depth = 4; break;
+		default:
+			WARN_ON(1);
+			break;
+		}
+
+		evo_mthd(push, 0x2004 + (head->base.index * 0x400), 1);
+		evo_data(push, 0x00000001 |
+			       asyh->or.depth << 4 |
+			       asyh->or.nvsync << 3 |
+			       asyh->or.nhsync << 2);
+		evo_kick(push, core);
+	}
+}
+
+static void
+headc37d_procamp(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x2000 + (head->base.index * 0x400), 1);
+		evo_data(push, 0x80000000 |
+			       asyh->procamp.sat.sin << 16 |
+			       asyh->procamp.sat.cos << 4);
+		evo_kick(push, core);
+	}
+}
+
+static void
+headc37d_dither(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x2018 + (head->base.index * 0x0400), 1);
+		evo_data(push, asyh->dither.mode << 8 |
+			       asyh->dither.bits << 4 |
+			       asyh->dither.enable);
+		evo_kick(push, core);
+	}
+}
+
+static void
+headc37d_curs_clr(struct nv50_head *head)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 4))) {
+		evo_mthd(push, 0x209c + head->base.index * 0x400, 1);
+		evo_data(push, 0x000000cf);
+		evo_mthd(push, 0x2088 + head->base.index * 0x400, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, core);
+	}
+}
+
+static void
+headc37d_curs_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 7))) {
+		evo_mthd(push, 0x209c + head->base.index * 0x400, 2);
+		evo_data(push, 0x80000000 |
+			       asyh->curs.layout << 8 |
+			       asyh->curs.format << 0);
+		evo_data(push, 0x000072ff);
+		evo_mthd(push, 0x2088 + head->base.index * 0x400, 1);
+		evo_data(push, asyh->curs.handle);
+		evo_mthd(push, 0x2090 + head->base.index * 0x400, 1);
+		evo_data(push, asyh->curs.offset >> 8);
+		evo_kick(push, core);
+	}
+}
+
+static int
+headc37d_curs_format(struct nv50_head *head, struct nv50_wndw_atom *asyw,
+		     struct nv50_head_atom *asyh)
+{
+	asyh->curs.format = asyw->image.format;
+	return 0;
+}
+
+static void
+headc37d_olut_clr(struct nv50_head *head)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		evo_mthd(push, 0x20ac + (head->base.index * 0x400), 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, core);
+	}
+}
+
+static void
+headc37d_olut_set(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 4))) {
+		evo_mthd(push, 0x20a4 + (head->base.index * 0x400), 3);
+		evo_data(push, asyh->olut.output_mode << 8 |
+			       asyh->olut.range << 4 |
+			       asyh->olut.size);
+		evo_data(push, asyh->olut.offset >> 8);
+		evo_data(push, asyh->olut.handle);
+		evo_kick(push, core);
+	}
+}
+
+static void
+headc37d_olut(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	asyh->olut.mode = 2;
+	asyh->olut.size = 0;
+	asyh->olut.range = 0;
+	asyh->olut.output_mode = 1;
+}
+
+static void
+headc37d_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	struct nv50_head_mode *m = &asyh->mode;
+	u32 *push;
+	if ((push = evo_wait(core, 12))) {
+		evo_mthd(push, 0x2064 + (head->base.index * 0x400), 5);
+		evo_data(push, (m->v.active  << 16) | m->h.active );
+		evo_data(push, (m->v.synce   << 16) | m->h.synce  );
+		evo_data(push, (m->v.blanke  << 16) | m->h.blanke );
+		evo_data(push, (m->v.blanks  << 16) | m->h.blanks );
+		evo_data(push, (m->v.blank2e << 16) | m->v.blank2s);
+		evo_mthd(push, 0x200c + (head->base.index * 0x400), 1);
+		evo_data(push, m->clock * 1000);
+		evo_mthd(push, 0x2028 + (head->base.index * 0x400), 1);
+		evo_data(push, m->clock * 1000);
+		/*XXX: HEAD_USAGE_BOUNDS, doesn't belong here. */
+		evo_mthd(push, 0x2030 + (head->base.index * 0x400), 1);
+		evo_data(push, 0x00000124);
+		evo_kick(push, core);
+	}
+}
+
+static void
+headc37d_view(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->core->chan;
+	u32 *push;
+	if ((push = evo_wait(core, 4))) {
+		evo_mthd(push, 0x204c + (head->base.index * 0x400), 1);
+		evo_data(push, (asyh->view.iH << 16) | asyh->view.iW);
+		evo_mthd(push, 0x2058 + (head->base.index * 0x400), 1);
+		evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
+		evo_kick(push, core);
+	}
+}
+
+const struct nv50_head_func
+headc37d = {
+	.view = headc37d_view,
+	.mode = headc37d_mode,
+	.olut = headc37d_olut,
+	.olut_set = headc37d_olut_set,
+	.olut_clr = headc37d_olut_clr,
+	.curs_layout = head917d_curs_layout,
+	.curs_format = headc37d_curs_format,
+	.curs_set = headc37d_curs_set,
+	.curs_clr = headc37d_curs_clr,
+	.dither = headc37d_dither,
+	.procamp = headc37d_procamp,
+	.or = headc37d_or,
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv50/lut.c b/drivers/gpu/drm/nouveau/dispnv50/lut.c
new file mode 100644
index 000000000000..a6b96ae2a22f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/lut.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "lut.h"
+#include "disp.h"
+
+#include <drm/drm_color_mgmt.h>
+#include <drm/drm_mode.h>
+#include <drm/drm_property.h>
+
+#include <nvif/class.h>
+
+u32
+nv50_lut_load(struct nv50_lut *lut, bool legacy, int buffer,
+	      struct drm_property_blob *blob)
+{
+	struct drm_color_lut *in = (struct drm_color_lut *)blob->data;
+	void __iomem *mem = lut->mem[buffer].object.map.ptr;
+	const int size = blob->length / sizeof(*in);
+	int bits, shift, i;
+	u16 zero, r, g, b;
+	u32 addr = lut->mem[buffer].addr;
+
+	/* This can't happen.. But it shuts the compiler up. */
+	if (WARN_ON(size != 256))
+		return 0;
+
+	if (legacy) {
+		bits = 11;
+		shift = 3;
+		zero = 0x0000;
+	} else {
+		bits = 14;
+		shift = 0;
+		zero = 0x6000;
+	}
+
+	for (i = 0; i < size; i++) {
+		r = (drm_color_lut_extract(in[i].  red, bits) + zero) << shift;
+		g = (drm_color_lut_extract(in[i].green, bits) + zero) << shift;
+		b = (drm_color_lut_extract(in[i]. blue, bits) + zero) << shift;
+		writew(r, mem + (i * 0x08) + 0);
+		writew(g, mem + (i * 0x08) + 2);
+		writew(b, mem + (i * 0x08) + 4);
+	}
+
+	/* INTERPOLATE modes require a "next" entry to interpolate with,
+	 * so we replicate the last entry to deal with this for now.
+	 */
+	writew(r, mem + (i * 0x08) + 0);
+	writew(g, mem + (i * 0x08) + 2);
+	writew(b, mem + (i * 0x08) + 4);
+	return addr;
+}
+
+void
+nv50_lut_fini(struct nv50_lut *lut)
+{
+	int i;
+	for (i = 0; i < ARRAY_SIZE(lut->mem); i++)
+		nvif_mem_fini(&lut->mem[i]);
+}
+
+int
+nv50_lut_init(struct nv50_disp *disp, struct nvif_mmu *mmu,
+	      struct nv50_lut *lut)
+{
+	const u32 size = disp->disp->object.oclass < GF110_DISP ? 257 : 1025;
+	int i;
+	for (i = 0; i < ARRAY_SIZE(lut->mem); i++) {
+		int ret = nvif_mem_init_map(mmu, NVIF_MEM_VRAM, size * 8,
+					    &lut->mem[i]);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/lut.h b/drivers/gpu/drm/nouveau/dispnv50/lut.h
new file mode 100644
index 000000000000..6d7b8352e4cb
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/lut.h
@@ -0,0 +1,15 @@
+#ifndef __NV50_KMS_LUT_H__
+#define __NV50_KMS_LUT_H__
+#include <nvif/mem.h>
+struct drm_property_blob;
+struct nv50_disp;
+
+struct nv50_lut {
+	struct nvif_mem mem[2];
+};
+
+int nv50_lut_init(struct nv50_disp *, struct nvif_mmu *, struct nv50_lut *);
+void nv50_lut_fini(struct nv50_lut *);
+u32 nv50_lut_load(struct nv50_lut *, bool legacy, int buffer,
+		  struct drm_property_blob *);
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv50/oimm.c b/drivers/gpu/drm/nouveau/dispnv50/oimm.c
new file mode 100644
index 000000000000..2a2841d344c8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/oimm.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "oimm.h"
+
+#include <nvif/class.h>
+
+int
+nv50_oimm_init(struct nouveau_drm *drm, struct nv50_wndw *wndw)
+{
+	static const struct {
+		s32 oclass;
+		int version;
+		int (*init)(struct nouveau_drm *, s32, struct nv50_wndw *);
+	} oimms[] = {
+		{ GK104_DISP_OVERLAY, 0, oimm507b_init },
+		{ GF110_DISP_OVERLAY, 0, oimm507b_init },
+		{ GT214_DISP_OVERLAY, 0, oimm507b_init },
+		{   G82_DISP_OVERLAY, 0, oimm507b_init },
+		{  NV50_DISP_OVERLAY, 0, oimm507b_init },
+		{}
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	int cid;
+
+	cid = nvif_mclass(&disp->disp->object, oimms);
+	if (cid < 0) {
+		NV_ERROR(drm, "No supported overlay immediate class\n");
+		return cid;
+	}
+
+	return oimms[cid].init(drm, oimms[cid].oclass, wndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/oimm.h b/drivers/gpu/drm/nouveau/dispnv50/oimm.h
new file mode 100644
index 000000000000..6fa51f101e94
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/oimm.h
@@ -0,0 +1,8 @@
+#ifndef __NV50_KMS_OIMM_H__
+#define __NV50_KMS_OIMM_H__
+#include "wndw.h"
+
+int oimm507b_init(struct nouveau_drm *, s32, struct nv50_wndw *);
+
+int nv50_oimm_init(struct nouveau_drm *, struct nv50_wndw *);
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv50/oimm507b.c b/drivers/gpu/drm/nouveau/dispnv50/oimm507b.c
new file mode 100644
index 000000000000..2ee404b3e19f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/oimm507b.c
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "oimm.h"
+
+#include <nvif/cl507b.h>
+
+static int
+oimm507b_init_(const struct nv50_wimm_func *func, struct nouveau_drm *drm,
+	       s32 oclass, struct nv50_wndw *wndw)
+{
+	struct nv50_disp_overlay_v0 args = {
+		.head = wndw->id,
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	int ret;
+
+	ret = nvif_object_init(&disp->disp->object, 0, oclass, &args,
+			       sizeof(args), &wndw->wimm.base.user);
+	if (ret) {
+		NV_ERROR(drm, "oimm%04x allocation failed: %d\n", oclass, ret);
+		return ret;
+	}
+
+	nvif_object_map(&wndw->wimm.base.user, NULL, 0);
+	wndw->immd = func;
+	return 0;
+}
+
+int
+oimm507b_init(struct nouveau_drm *drm, s32 oclass, struct nv50_wndw *wndw)
+{
+	return oimm507b_init_(&curs507a, drm, oclass, wndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/ovly.c b/drivers/gpu/drm/nouveau/dispnv50/ovly.c
new file mode 100644
index 000000000000..90c246d47604
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/ovly.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "ovly.h"
+#include "oimm.h"
+
+#include <nvif/class.h>
+
+int
+nv50_ovly_new(struct nouveau_drm *drm, int head, struct nv50_wndw **pwndw)
+{
+	static const struct {
+		s32 oclass;
+		int version;
+		int (*new)(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+	} ovlys[] = {
+		{ GK104_DISP_OVERLAY_CONTROL_DMA, 0, ovly917e_new },
+		{ GF110_DISP_OVERLAY_CONTROL_DMA, 0, ovly907e_new },
+		{ GT214_DISP_OVERLAY_CHANNEL_DMA, 0, ovly827e_new },
+		{ GT200_DISP_OVERLAY_CHANNEL_DMA, 0, ovly827e_new },
+		{   G82_DISP_OVERLAY_CHANNEL_DMA, 0, ovly827e_new },
+		{  NV50_DISP_OVERLAY_CHANNEL_DMA, 0, ovly507e_new },
+		{}
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	int cid, ret;
+
+	cid = nvif_mclass(&disp->disp->object, ovlys);
+	if (cid < 0) {
+		NV_ERROR(drm, "No supported overlay class\n");
+		return cid;
+	}
+
+	ret = ovlys[cid].new(drm, head, ovlys[cid].oclass, pwndw);
+	if (ret)
+		return ret;
+
+	return nv50_oimm_init(drm, *pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/ovly.h b/drivers/gpu/drm/nouveau/dispnv50/ovly.h
new file mode 100644
index 000000000000..4869d52d1786
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/ovly.h
@@ -0,0 +1,30 @@
+#ifndef __NV50_KMS_OVLY_H__
+#define __NV50_KMS_OVLY_H__
+#include "wndw.h"
+
+int ovly507e_new(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+int ovly507e_new_(const struct nv50_wndw_func *, const u32 *format,
+		  struct nouveau_drm *, int head, s32 oclass,
+		  u32 interlock_data, struct nv50_wndw **);
+int ovly507e_acquire(struct nv50_wndw *, struct nv50_wndw_atom *,
+		     struct nv50_head_atom *);
+void ovly507e_release(struct nv50_wndw *, struct nv50_wndw_atom *,
+		      struct nv50_head_atom *);
+void ovly507e_ntfy_set(struct nv50_wndw *, struct nv50_wndw_atom *);
+void ovly507e_ntfy_clr(struct nv50_wndw *);
+void ovly507e_image_clr(struct nv50_wndw *);
+void ovly507e_scale_set(struct nv50_wndw *, struct nv50_wndw_atom *);
+void ovly507e_update(struct nv50_wndw *, u32 *);
+
+extern const u32 ovly827e_format[];
+void ovly827e_ntfy_reset(struct nouveau_bo *, u32);
+int ovly827e_ntfy_wait_begun(struct nouveau_bo *, u32, struct nvif_device *);
+
+extern const struct nv50_wndw_func ovly907e;
+
+int ovly827e_new(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+int ovly907e_new(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+int ovly917e_new(struct nouveau_drm *, int, s32, struct nv50_wndw **);
+
+int nv50_ovly_new(struct nouveau_drm *, int head, struct nv50_wndw **);
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv50/ovly507e.c b/drivers/gpu/drm/nouveau/dispnv50/ovly507e.c
new file mode 100644
index 000000000000..cc417664f823
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/ovly507e.c
@@ -0,0 +1,217 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "ovly.h"
+#include "atom.h"
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+
+#include <nvif/cl507e.h>
+#include <nvif/event.h>
+
+void
+ovly507e_update(struct nv50_wndw *wndw, u32 *interlock)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 2))) {
+		evo_mthd(push, 0x0080, 1);
+		evo_data(push, interlock[NV50_DISP_INTERLOCK_CORE]);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+ovly507e_scale_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 4))) {
+		evo_mthd(push, 0x00e0, 3);
+		evo_data(push, asyw->scale.sy << 16 | asyw->scale.sx);
+		evo_data(push, asyw->scale.sh << 16 | asyw->scale.sw);
+		evo_data(push, asyw->scale.dw);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+ovly507e_image_clr(struct nv50_wndw *wndw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 4))) {
+		evo_mthd(push, 0x0084, 1);
+		evo_data(push, 0x00000000);
+		evo_mthd(push, 0x00c0, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+ovly507e_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 12))) {
+		evo_mthd(push, 0x0084, 1);
+		evo_data(push, asyw->image.interval << 4);
+		evo_mthd(push, 0x00c0, 1);
+		evo_data(push, asyw->image.handle[0]);
+		evo_mthd(push, 0x0100, 1);
+		evo_data(push, 0x00000002);
+		evo_mthd(push, 0x0800, 1);
+		evo_data(push, asyw->image.offset[0] >> 8);
+		evo_mthd(push, 0x0808, 3);
+		evo_data(push, asyw->image.h << 16 | asyw->image.w);
+		evo_data(push, asyw->image.layout << 20 |
+			       (asyw->image.pitch[0] >> 8) << 8 |
+			       asyw->image.blocks[0] << 8 |
+			       asyw->image.blockh);
+		evo_data(push, asyw->image.kind << 16 |
+			       asyw->image.format << 8 |
+			       asyw->image.colorspace);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+ovly507e_ntfy_clr(struct nv50_wndw *wndw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 2))) {
+		evo_mthd(push, 0x00a4, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+ovly507e_ntfy_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 3))) {
+		evo_mthd(push, 0x00a0, 2);
+		evo_data(push, asyw->ntfy.awaken << 30 | asyw->ntfy.offset);
+		evo_data(push, asyw->ntfy.handle);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+void
+ovly507e_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		 struct nv50_head_atom *asyh)
+{
+	asyh->ovly.cpp = 0;
+}
+
+int
+ovly507e_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		 struct nv50_head_atom *asyh)
+{
+	const struct drm_framebuffer *fb = asyw->state.fb;
+	int ret;
+
+	ret = drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  true, true);
+	if (ret)
+		return ret;
+
+	asyh->ovly.cpp = fb->format->cpp[0];
+	return 0;
+}
+
+#include "nouveau_bo.h"
+
+static const struct nv50_wndw_func
+ovly507e = {
+	.acquire = ovly507e_acquire,
+	.release = ovly507e_release,
+	.ntfy_set = ovly507e_ntfy_set,
+	.ntfy_clr = ovly507e_ntfy_clr,
+	.ntfy_reset = base507c_ntfy_reset,
+	.ntfy_wait_begun = base507c_ntfy_wait_begun,
+	.image_set = ovly507e_image_set,
+	.image_clr = ovly507e_image_clr,
+	.scale_set = ovly507e_scale_set,
+	.update = ovly507e_update,
+};
+
+static const u32
+ovly507e_format[] = {
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB1555,
+	0
+};
+
+int
+ovly507e_new_(const struct nv50_wndw_func *func, const u32 *format,
+	      struct nouveau_drm *drm, int head, s32 oclass, u32 interlock_data,
+	      struct nv50_wndw **pwndw)
+{
+	struct nv50_disp_overlay_channel_dma_v0 args = {
+		.head = head,
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	struct nv50_wndw *wndw;
+	int ret;
+
+	ret = nv50_wndw_new_(func, drm->dev, DRM_PLANE_TYPE_OVERLAY,
+			     "ovly", head, format, BIT(head),
+			     NV50_DISP_INTERLOCK_OVLY, interlock_data,
+			     &wndw);
+	if (*pwndw = wndw, ret)
+		return ret;
+
+	ret = nv50_dmac_create(&drm->client.device, &disp->disp->object,
+			       &oclass, 0, &args, sizeof(args),
+			       disp->sync->bo.offset, &wndw->wndw);
+	if (ret) {
+		NV_ERROR(drm, "ovly%04x allocation failed: %d\n", oclass, ret);
+		return ret;
+	}
+
+	ret = nvif_notify_init(&wndw->wndw.base.user, wndw->notify.func, false,
+			       NV50_DISP_OVERLAY_CHANNEL_DMA_V0_NTFY_UEVENT,
+			       &(struct nvif_notify_uevent_req) {},
+			       sizeof(struct nvif_notify_uevent_req),
+			       sizeof(struct nvif_notify_uevent_rep),
+			       &wndw->notify);
+	if (ret)
+		return ret;
+
+	wndw->ntfy = NV50_DISP_OVLY_NTFY(wndw->id);
+	wndw->sema = NV50_DISP_OVLY_SEM0(wndw->id);
+	wndw->data = 0x00000000;
+	return 0;
+}
+
+int
+ovly507e_new(struct nouveau_drm *drm, int head, s32 oclass,
+	     struct nv50_wndw **pwndw)
+{
+	return ovly507e_new_(&ovly507e, ovly507e_format, drm, head, oclass,
+			     0x00000004 << (head * 8), pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/ovly827e.c b/drivers/gpu/drm/nouveau/dispnv50/ovly827e.c
new file mode 100644
index 000000000000..aaa9fe5a4fc8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/ovly827e.c
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "ovly.h"
+#include "atom.h"
+
+#include <nouveau_bo.h>
+
+static void
+ovly827e_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 12))) {
+		evo_mthd(push, 0x0084, 1);
+		evo_data(push, asyw->image.interval << 4);
+		evo_mthd(push, 0x00c0, 1);
+		evo_data(push, asyw->image.handle[0]);
+		evo_mthd(push, 0x0100, 1);
+		evo_data(push, 0x00000002);
+		evo_mthd(push, 0x0800, 1);
+		evo_data(push, asyw->image.offset[0] >> 8);
+		evo_mthd(push, 0x0808, 3);
+		evo_data(push, asyw->image.h << 16 | asyw->image.w);
+		evo_data(push, asyw->image.layout << 20 |
+			       (asyw->image.pitch[0] >> 8) << 8 |
+			       asyw->image.blocks[0] << 8 |
+			       asyw->image.blockh);
+		evo_data(push, asyw->image.format << 8 |
+			       asyw->image.colorspace);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+int
+ovly827e_ntfy_wait_begun(struct nouveau_bo *bo, u32 offset,
+			 struct nvif_device *device)
+{
+	s64 time = nvif_msec(device, 2000ULL,
+		u32 data = nouveau_bo_rd32(bo, offset / 4 + 3);
+		if ((data & 0xffff0000) == 0xffff0000)
+			break;
+		usleep_range(1, 2);
+	);
+	return time < 0 ? time : 0;
+}
+
+void
+ovly827e_ntfy_reset(struct nouveau_bo *bo, u32 offset)
+{
+	nouveau_bo_wr32(bo, offset / 4 + 0, 0x00000000);
+	nouveau_bo_wr32(bo, offset / 4 + 1, 0x00000000);
+	nouveau_bo_wr32(bo, offset / 4 + 2, 0x00000000);
+	nouveau_bo_wr32(bo, offset / 4 + 3, 0x80000000);
+}
+
+static const struct nv50_wndw_func
+ovly827e = {
+	.acquire = ovly507e_acquire,
+	.release = ovly507e_release,
+	.ntfy_set = ovly507e_ntfy_set,
+	.ntfy_clr = ovly507e_ntfy_clr,
+	.ntfy_reset = ovly827e_ntfy_reset,
+	.ntfy_wait_begun = ovly827e_ntfy_wait_begun,
+	.image_set = ovly827e_image_set,
+	.image_clr = ovly507e_image_clr,
+	.scale_set = ovly507e_scale_set,
+	.update = ovly507e_update,
+};
+
+const u32
+ovly827e_format[] = {
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_ABGR2101010,
+	0
+};
+
+int
+ovly827e_new(struct nouveau_drm *drm, int head, s32 oclass,
+	     struct nv50_wndw **pwndw)
+{
+	return ovly507e_new_(&ovly827e, ovly827e_format, drm, head, oclass,
+			     0x00000004 << (head * 8), pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/ovly907e.c b/drivers/gpu/drm/nouveau/dispnv50/ovly907e.c
new file mode 100644
index 000000000000..a3ce53046015
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/ovly907e.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "ovly.h"
+#include "atom.h"
+
+static void
+ovly907e_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 12))) {
+		evo_mthd(push, 0x0084, 1);
+		evo_data(push, asyw->image.interval << 4);
+		evo_mthd(push, 0x00c0, 1);
+		evo_data(push, asyw->image.handle[0]);
+		evo_mthd(push, 0x0100, 1);
+		evo_data(push, 0x00000002);
+		evo_mthd(push, 0x0400, 1);
+		evo_data(push, asyw->image.offset[0] >> 8);
+		evo_mthd(push, 0x0408, 3);
+		evo_data(push, asyw->image.h << 16 | asyw->image.w);
+		evo_data(push, asyw->image.layout << 24 |
+			       (asyw->image.pitch[0] >> 8) << 8 |
+			       asyw->image.blocks[0] << 8 |
+			       asyw->image.blockh);
+		evo_data(push, asyw->image.format << 8 |
+			       asyw->image.colorspace);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+const struct nv50_wndw_func
+ovly907e = {
+	.acquire = ovly507e_acquire,
+	.release = ovly507e_release,
+	.ntfy_set = ovly507e_ntfy_set,
+	.ntfy_clr = ovly507e_ntfy_clr,
+	.ntfy_reset = ovly827e_ntfy_reset,
+	.ntfy_wait_begun = ovly827e_ntfy_wait_begun,
+	.image_set = ovly907e_image_set,
+	.image_clr = ovly507e_image_clr,
+	.scale_set = ovly507e_scale_set,
+	.update = ovly507e_update,
+};
+
+int
+ovly907e_new(struct nouveau_drm *drm, int head, s32 oclass,
+	     struct nv50_wndw **pwndw)
+{
+	return ovly507e_new_(&ovly907e, ovly827e_format, drm, head, oclass,
+			     0x00000004 << (head * 4), pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/ovly917e.c b/drivers/gpu/drm/nouveau/dispnv50/ovly917e.c
new file mode 100644
index 000000000000..505fa7e78523
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/ovly917e.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "ovly.h"
+
+static const u32
+ovly917e_format[] = {
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_ABGR2101010,
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_ARGB2101010,
+	0
+};
+
+int
+ovly917e_new(struct nouveau_drm *drm, int head, s32 oclass,
+	     struct nv50_wndw **pwndw)
+{
+	return ovly507e_new_(&ovly907e, ovly917e_format, drm, head, oclass,
+			     0x00000004 << (head * 4), pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/pior507d.c b/drivers/gpu/drm/nouveau/dispnv50/pior507d.c
new file mode 100644
index 000000000000..d2bac6a341dc
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/pior507d.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+
+static void
+pior507d_ctrl(struct nv50_core *core, int or, u32 ctrl,
+	      struct nv50_head_atom *asyh)
+{
+	u32 *push;
+	if ((push = evo_wait(&core->chan, 2))) {
+		if (asyh) {
+			ctrl |= asyh->or.depth  << 16;
+			ctrl |= asyh->or.nvsync << 13;
+			ctrl |= asyh->or.nhsync << 12;
+		}
+		evo_mthd(push, 0x0700 + (or * 0x040), 1);
+		evo_data(push, ctrl);
+		evo_kick(push, &core->chan);
+	}
+}
+
+const struct nv50_outp_func
+pior507d = {
+	.ctrl = pior507d_ctrl,
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv50/sor507d.c b/drivers/gpu/drm/nouveau/dispnv50/sor507d.c
new file mode 100644
index 000000000000..5222fe6a9b21
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/sor507d.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+
+static void
+sor507d_ctrl(struct nv50_core *core, int or, u32 ctrl,
+	     struct nv50_head_atom *asyh)
+{
+	u32 *push;
+	if ((push = evo_wait(&core->chan, 2))) {
+		if (asyh) {
+			ctrl |= asyh->or.depth  << 16;
+			ctrl |= asyh->or.nvsync << 13;
+			ctrl |= asyh->or.nhsync << 12;
+		}
+		evo_mthd(push, 0x0600 + (or * 0x40), 1);
+		evo_data(push, ctrl);
+		evo_kick(push, &core->chan);
+	}
+}
+
+const struct nv50_outp_func
+sor507d = {
+	.ctrl = sor507d_ctrl,
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv50/sor907d.c b/drivers/gpu/drm/nouveau/dispnv50/sor907d.c
new file mode 100644
index 000000000000..b0314ec11fb3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/sor907d.c
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+
+#include <nvif/class.h>
+
+static void
+sor907d_ctrl(struct nv50_core *core, int or, u32 ctrl,
+	     struct nv50_head_atom *asyh)
+{
+	u32 *push;
+	if ((push = evo_wait(&core->chan, 2))) {
+		evo_mthd(push, 0x0200 + (or * 0x20), 1);
+		evo_data(push, ctrl);
+		evo_kick(push, &core->chan);
+	}
+}
+
+const struct nv50_outp_func
+sor907d = {
+	.ctrl = sor907d_ctrl,
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv50/sorc37d.c b/drivers/gpu/drm/nouveau/dispnv50/sorc37d.c
new file mode 100644
index 000000000000..dff059241c5d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/sorc37d.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "core.h"
+
+static void
+sorc37d_ctrl(struct nv50_core *core, int or, u32 ctrl,
+	     struct nv50_head_atom *asyh)
+{
+	u32 *push;
+	if ((push = evo_wait(&core->chan, 2))) {
+		evo_mthd(push, 0x0300 + (or * 0x20), 1);
+		evo_data(push, ctrl);
+		evo_kick(push, &core->chan);
+	}
+}
+
+const struct nv50_outp_func
+sorc37d = {
+	.ctrl = sorc37d_ctrl,
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wimm.c b/drivers/gpu/drm/nouveau/dispnv50/wimm.c
new file mode 100644
index 000000000000..fc36e0696407
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/wimm.c
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "wimm.h"
+
+#include <nvif/class.h>
+
+int
+nv50_wimm_init(struct nouveau_drm *drm, struct nv50_wndw *wndw)
+{
+	struct {
+		s32 oclass;
+		int version;
+		int (*init)(struct nouveau_drm *, s32, struct nv50_wndw *);
+	} wimms[] = {
+		{ GV100_DISP_WINDOW_IMM_CHANNEL_DMA, 0, wimmc37b_init },
+		{}
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	int cid;
+
+	cid = nvif_mclass(&disp->disp->object, wimms);
+	if (cid < 0) {
+		NV_ERROR(drm, "No supported window immediate class\n");
+		return cid;
+	}
+
+	return wimms[cid].init(drm, wimms[cid].oclass, wndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wimm.h b/drivers/gpu/drm/nouveau/dispnv50/wimm.h
new file mode 100644
index 000000000000..363052309be9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/wimm.h
@@ -0,0 +1,8 @@
+#ifndef __NV50_KMS_WIMM_H__
+#define __NV50_KMS_WIMM_H__
+#include "wndw.h"
+
+int nv50_wimm_init(struct nouveau_drm *drm, struct nv50_wndw *);
+
+int wimmc37b_init(struct nouveau_drm *, s32, struct nv50_wndw *);
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wimmc37b.c b/drivers/gpu/drm/nouveau/dispnv50/wimmc37b.c
new file mode 100644
index 000000000000..9103b8494279
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/wimmc37b.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "wimm.h"
+#include "atom.h"
+#include "wndw.h"
+
+#include <nvif/clc37b.h>
+
+static void
+wimmc37b_update(struct nv50_wndw *wndw, u32 *interlock)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wimm, 2))) {
+		evo_mthd(push, 0x0200, 1);
+		if (interlock[NV50_DISP_INTERLOCK_WNDW] & wndw->interlock.data)
+			evo_data(push, 0x00000003);
+		else
+			evo_data(push, 0x00000001);
+		evo_kick(push, &wndw->wimm);
+	}
+}
+
+static void
+wimmc37b_point(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wimm, 2))) {
+		evo_mthd(push, 0x0208, 1);
+		evo_data(push, asyw->point.y << 16 | asyw->point.x);
+		evo_kick(push, &wndw->wimm);
+	}
+}
+
+static const struct nv50_wimm_func
+wimmc37b = {
+	.point = wimmc37b_point,
+	.update = wimmc37b_update,
+};
+
+static int
+wimmc37b_init_(const struct nv50_wimm_func *func, struct nouveau_drm *drm,
+	       s32 oclass, struct nv50_wndw *wndw)
+{
+	struct nvc37b_window_imm_channel_dma_v0 args = {
+		.pushbuf = 0xb0007b00 | wndw->id,
+		.index = wndw->id,
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	int ret;
+
+	ret = nv50_dmac_create(&drm->client.device, &disp->disp->object,
+			       &oclass, 0, &args, sizeof(args), 0,
+			       &wndw->wimm);
+	if (ret) {
+		NV_ERROR(drm, "wimm%04x allocation failed: %d\n", oclass, ret);
+		return ret;
+	}
+
+	wndw->immd = func;
+	return 0;
+}
+
+int
+wimmc37b_init(struct nouveau_drm *drm, s32 oclass, struct nv50_wndw *wndw)
+{
+	return wimmc37b_init_(&wimmc37b, drm, oclass, wndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndw.c b/drivers/gpu/drm/nouveau/dispnv50/wndw.c
new file mode 100644
index 000000000000..224963b533a6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/wndw.c
@@ -0,0 +1,641 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "wndw.h"
+#include "wimm.h"
+
+#include <nvif/class.h>
+#include <nvif/cl0002.h>
+
+#include <drm/drm_atomic_helper.h>
+#include "nouveau_bo.h"
+
+static void
+nv50_wndw_ctxdma_del(struct nv50_wndw_ctxdma *ctxdma)
+{
+	nvif_object_fini(&ctxdma->object);
+	list_del(&ctxdma->head);
+	kfree(ctxdma);
+}
+
+static struct nv50_wndw_ctxdma *
+nv50_wndw_ctxdma_new(struct nv50_wndw *wndw, struct nouveau_framebuffer *fb)
+{
+	struct nouveau_drm *drm = nouveau_drm(fb->base.dev);
+	struct nv50_wndw_ctxdma *ctxdma;
+	const u8    kind = fb->nvbo->kind;
+	const u32 handle = 0xfb000000 | kind;
+	struct {
+		struct nv_dma_v0 base;
+		union {
+			struct nv50_dma_v0 nv50;
+			struct gf100_dma_v0 gf100;
+			struct gf119_dma_v0 gf119;
+		};
+	} args = {};
+	u32 argc = sizeof(args.base);
+	int ret;
+
+	list_for_each_entry(ctxdma, &wndw->ctxdma.list, head) {
+		if (ctxdma->object.handle == handle)
+			return ctxdma;
+	}
+
+	if (!(ctxdma = kzalloc(sizeof(*ctxdma), GFP_KERNEL)))
+		return ERR_PTR(-ENOMEM);
+	list_add(&ctxdma->head, &wndw->ctxdma.list);
+
+	args.base.target = NV_DMA_V0_TARGET_VRAM;
+	args.base.access = NV_DMA_V0_ACCESS_RDWR;
+	args.base.start  = 0;
+	args.base.limit  = drm->client.device.info.ram_user - 1;
+
+	if (drm->client.device.info.chipset < 0x80) {
+		args.nv50.part = NV50_DMA_V0_PART_256;
+		argc += sizeof(args.nv50);
+	} else
+	if (drm->client.device.info.chipset < 0xc0) {
+		args.nv50.part = NV50_DMA_V0_PART_256;
+		args.nv50.kind = kind;
+		argc += sizeof(args.nv50);
+	} else
+	if (drm->client.device.info.chipset < 0xd0) {
+		args.gf100.kind = kind;
+		argc += sizeof(args.gf100);
+	} else {
+		args.gf119.page = GF119_DMA_V0_PAGE_LP;
+		args.gf119.kind = kind;
+		argc += sizeof(args.gf119);
+	}
+
+	ret = nvif_object_init(wndw->ctxdma.parent, handle, NV_DMA_IN_MEMORY,
+			       &args, argc, &ctxdma->object);
+	if (ret) {
+		nv50_wndw_ctxdma_del(ctxdma);
+		return ERR_PTR(ret);
+	}
+
+	return ctxdma;
+}
+
+int
+nv50_wndw_wait_armed(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	struct nv50_disp *disp = nv50_disp(wndw->plane.dev);
+	if (asyw->set.ntfy) {
+		return wndw->func->ntfy_wait_begun(disp->sync,
+						   asyw->ntfy.offset,
+						   wndw->wndw.base.device);
+	}
+	return 0;
+}
+
+void
+nv50_wndw_flush_clr(struct nv50_wndw *wndw, u32 *interlock, bool flush,
+		    struct nv50_wndw_atom *asyw)
+{
+	union nv50_wndw_atom_mask clr = {
+		.mask = asyw->clr.mask & ~(flush ? 0 : asyw->set.mask),
+	};
+	if (clr.sema ) wndw->func-> sema_clr(wndw);
+	if (clr.ntfy ) wndw->func-> ntfy_clr(wndw);
+	if (clr.xlut ) wndw->func-> xlut_clr(wndw);
+	if (clr.image) wndw->func->image_clr(wndw);
+
+	interlock[wndw->interlock.type] |= wndw->interlock.data;
+}
+
+void
+nv50_wndw_flush_set(struct nv50_wndw *wndw, u32 *interlock,
+		    struct nv50_wndw_atom *asyw)
+{
+	if (interlock) {
+		asyw->image.mode = 0;
+		asyw->image.interval = 1;
+	}
+
+	if (asyw->set.sema ) wndw->func->sema_set (wndw, asyw);
+	if (asyw->set.ntfy ) wndw->func->ntfy_set (wndw, asyw);
+	if (asyw->set.image) wndw->func->image_set(wndw, asyw);
+
+	if (asyw->set.xlut ) {
+		if (asyw->ilut) {
+			asyw->xlut.i.offset =
+				nv50_lut_load(&wndw->ilut,
+					      asyw->xlut.i.mode <= 1,
+					      asyw->xlut.i.buffer,
+					      asyw->ilut);
+		}
+		wndw->func->xlut_set(wndw, asyw);
+	}
+
+	if (asyw->set.scale) wndw->func->scale_set(wndw, asyw);
+	if (asyw->set.point) {
+		if (asyw->set.point = false, asyw->set.mask)
+			interlock[wndw->interlock.type] |= wndw->interlock.data;
+		interlock[NV50_DISP_INTERLOCK_WIMM] |= wndw->interlock.data;
+
+		wndw->immd->point(wndw, asyw);
+		wndw->immd->update(wndw, interlock);
+	} else {
+		interlock[wndw->interlock.type] |= wndw->interlock.data;
+	}
+}
+
+void
+nv50_wndw_ntfy_enable(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	struct nv50_disp *disp = nv50_disp(wndw->plane.dev);
+
+	asyw->ntfy.handle = wndw->wndw.sync.handle;
+	asyw->ntfy.offset = wndw->ntfy;
+	asyw->ntfy.awaken = false;
+	asyw->set.ntfy = true;
+
+	wndw->func->ntfy_reset(disp->sync, wndw->ntfy);
+	wndw->ntfy ^= 0x10;
+}
+
+static void
+nv50_wndw_atomic_check_release(struct nv50_wndw *wndw,
+			       struct nv50_wndw_atom *asyw,
+			       struct nv50_head_atom *asyh)
+{
+	struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev);
+	NV_ATOMIC(drm, "%s release\n", wndw->plane.name);
+	wndw->func->release(wndw, asyw, asyh);
+	asyw->ntfy.handle = 0;
+	asyw->sema.handle = 0;
+}
+
+static int
+nv50_wndw_atomic_check_acquire_yuv(struct nv50_wndw_atom *asyw)
+{
+	switch (asyw->state.fb->format->format) {
+	case DRM_FORMAT_YUYV: asyw->image.format = 0x28; break;
+	case DRM_FORMAT_UYVY: asyw->image.format = 0x29; break;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
+	}
+	asyw->image.colorspace = 1;
+	return 0;
+}
+
+static int
+nv50_wndw_atomic_check_acquire_rgb(struct nv50_wndw_atom *asyw)
+{
+	switch (asyw->state.fb->format->format) {
+	case DRM_FORMAT_C8         : asyw->image.format = 0x1e; break;
+	case DRM_FORMAT_XRGB8888   :
+	case DRM_FORMAT_ARGB8888   : asyw->image.format = 0xcf; break;
+	case DRM_FORMAT_RGB565     : asyw->image.format = 0xe8; break;
+	case DRM_FORMAT_XRGB1555   :
+	case DRM_FORMAT_ARGB1555   : asyw->image.format = 0xe9; break;
+	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_ABGR2101010: asyw->image.format = 0xd1; break;
+	case DRM_FORMAT_XBGR8888   :
+	case DRM_FORMAT_ABGR8888   : asyw->image.format = 0xd5; break;
+	case DRM_FORMAT_XRGB2101010:
+	case DRM_FORMAT_ARGB2101010: asyw->image.format = 0xdf; break;
+	default:
+		return -EINVAL;
+	}
+	asyw->image.colorspace = 0;
+	return 0;
+}
+
+static int
+nv50_wndw_atomic_check_acquire(struct nv50_wndw *wndw, bool modeset,
+			       struct nv50_wndw_atom *armw,
+			       struct nv50_wndw_atom *asyw,
+			       struct nv50_head_atom *asyh)
+{
+	struct nouveau_framebuffer *fb = nouveau_framebuffer(asyw->state.fb);
+	struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev);
+	int ret;
+
+	NV_ATOMIC(drm, "%s acquire\n", wndw->plane.name);
+
+	if (asyw->state.fb != armw->state.fb || !armw->visible || modeset) {
+		asyw->image.w = fb->base.width;
+		asyw->image.h = fb->base.height;
+		asyw->image.kind = fb->nvbo->kind;
+
+		ret = nv50_wndw_atomic_check_acquire_rgb(asyw);
+		if (ret) {
+			ret = nv50_wndw_atomic_check_acquire_yuv(asyw);
+			if (ret)
+				return ret;
+		}
+
+		if (asyw->image.kind) {
+			asyw->image.layout = 0;
+			if (drm->client.device.info.chipset >= 0xc0)
+				asyw->image.blockh = fb->nvbo->mode >> 4;
+			else
+				asyw->image.blockh = fb->nvbo->mode;
+			asyw->image.blocks[0] = fb->base.pitches[0] / 64;
+			asyw->image.pitch[0] = 0;
+		} else {
+			asyw->image.layout = 1;
+			asyw->image.blockh = 0;
+			asyw->image.blocks[0] = 0;
+			asyw->image.pitch[0] = fb->base.pitches[0];
+		}
+
+		if (!(asyh->state.pageflip_flags & DRM_MODE_PAGE_FLIP_ASYNC))
+			asyw->image.interval = 1;
+		else
+			asyw->image.interval = 0;
+		asyw->image.mode = asyw->image.interval ? 0 : 1;
+		asyw->set.image = wndw->func->image_set != NULL;
+	}
+
+	if (wndw->func->scale_set) {
+		asyw->scale.sx = asyw->state.src_x >> 16;
+		asyw->scale.sy = asyw->state.src_y >> 16;
+		asyw->scale.sw = asyw->state.src_w >> 16;
+		asyw->scale.sh = asyw->state.src_h >> 16;
+		asyw->scale.dw = asyw->state.crtc_w;
+		asyw->scale.dh = asyw->state.crtc_h;
+		if (memcmp(&armw->scale, &asyw->scale, sizeof(asyw->scale)))
+			asyw->set.scale = true;
+	}
+
+	if (wndw->immd) {
+		asyw->point.x = asyw->state.crtc_x;
+		asyw->point.y = asyw->state.crtc_y;
+		if (memcmp(&armw->point, &asyw->point, sizeof(asyw->point)))
+			asyw->set.point = true;
+	}
+
+	return wndw->func->acquire(wndw, asyw, asyh);
+}
+
+static void
+nv50_wndw_atomic_check_lut(struct nv50_wndw *wndw,
+			   struct nv50_wndw_atom *armw,
+			   struct nv50_wndw_atom *asyw,
+			   struct nv50_head_atom *asyh)
+{
+	struct drm_property_blob *ilut = asyh->state.degamma_lut;
+
+	/* I8 format without an input LUT makes no sense, and the
+	 * HW error-checks for this.
+	 *
+	 * In order to handle legacy gamma, when there's no input
+	 * LUT we need to steal the output LUT and use it instead.
+	 */
+	if (!ilut && asyw->state.fb->format->format == DRM_FORMAT_C8) {
+		/* This should be an error, but there's legacy clients
+		 * that do a modeset before providing a gamma table.
+		 *
+		 * We keep the window disabled to avoid angering HW.
+		 */
+		if (!(ilut = asyh->state.gamma_lut)) {
+			asyw->visible = false;
+			return;
+		}
+
+		if (wndw->func->ilut)
+			asyh->wndw.olut |= BIT(wndw->id);
+	} else {
+		asyh->wndw.olut &= ~BIT(wndw->id);
+	}
+
+	/* Recalculate LUT state. */
+	memset(&asyw->xlut, 0x00, sizeof(asyw->xlut));
+	if ((asyw->ilut = wndw->func->ilut ? ilut : NULL)) {
+		wndw->func->ilut(wndw, asyw);
+		asyw->xlut.handle = wndw->wndw.vram.handle;
+		asyw->xlut.i.buffer = !asyw->xlut.i.buffer;
+		asyw->set.xlut = true;
+	}
+
+	/* Handle setting base SET_OUTPUT_LUT_LO_ENABLE_USE_CORE_LUT. */
+	if (wndw->func->olut_core &&
+	    (!armw->visible || (armw->xlut.handle && !asyw->xlut.handle)))
+		asyw->set.xlut = true;
+
+	/* Can't do an immediate flip while changing the LUT. */
+	asyh->state.pageflip_flags &= ~DRM_MODE_PAGE_FLIP_ASYNC;
+}
+
+static int
+nv50_wndw_atomic_check(struct drm_plane *plane, struct drm_plane_state *state)
+{
+	struct nouveau_drm *drm = nouveau_drm(plane->dev);
+	struct nv50_wndw *wndw = nv50_wndw(plane);
+	struct nv50_wndw_atom *armw = nv50_wndw_atom(wndw->plane.state);
+	struct nv50_wndw_atom *asyw = nv50_wndw_atom(state);
+	struct nv50_head_atom *harm = NULL, *asyh = NULL;
+	bool modeset = false;
+	int ret;
+
+	NV_ATOMIC(drm, "%s atomic_check\n", plane->name);
+
+	/* Fetch the assembly state for the head the window will belong to,
+	 * and determine whether the window will be visible.
+	 */
+	if (asyw->state.crtc) {
+		asyh = nv50_head_atom_get(asyw->state.state, asyw->state.crtc);
+		if (IS_ERR(asyh))
+			return PTR_ERR(asyh);
+		modeset = drm_atomic_crtc_needs_modeset(&asyh->state);
+		asyw->visible = asyh->state.active;
+	} else {
+		asyw->visible = false;
+	}
+
+	/* Fetch assembly state for the head the window used to belong to. */
+	if (armw->state.crtc) {
+		harm = nv50_head_atom_get(asyw->state.state, armw->state.crtc);
+		if (IS_ERR(harm))
+			return PTR_ERR(harm);
+	}
+
+	/* LUT configuration can potentially cause the window to be disabled. */
+	if (asyw->visible && wndw->func->xlut_set &&
+	    (!armw->visible ||
+	     asyh->state.color_mgmt_changed ||
+	     asyw->state.fb->format->format !=
+	     armw->state.fb->format->format))
+		nv50_wndw_atomic_check_lut(wndw, armw, asyw, asyh);
+
+	/* Calculate new window state. */
+	if (asyw->visible) {
+		ret = nv50_wndw_atomic_check_acquire(wndw, modeset,
+						     armw, asyw, asyh);
+		if (ret)
+			return ret;
+
+		asyh->wndw.mask |= BIT(wndw->id);
+	} else
+	if (armw->visible) {
+		nv50_wndw_atomic_check_release(wndw, asyw, harm);
+		harm->wndw.mask &= ~BIT(wndw->id);
+	} else {
+		return 0;
+	}
+
+	/* Aside from the obvious case where the window is actively being
+	 * disabled, we might also need to temporarily disable the window
+	 * when performing certain modeset operations.
+	 */
+	if (!asyw->visible || modeset) {
+		asyw->clr.ntfy = armw->ntfy.handle != 0;
+		asyw->clr.sema = armw->sema.handle != 0;
+		asyw->clr.xlut = armw->xlut.handle != 0;
+		if (wndw->func->image_clr)
+			asyw->clr.image = armw->image.handle[0] != 0;
+	}
+
+	return 0;
+}
+
+static void
+nv50_wndw_cleanup_fb(struct drm_plane *plane, struct drm_plane_state *old_state)
+{
+	struct nouveau_framebuffer *fb = nouveau_framebuffer(old_state->fb);
+	struct nouveau_drm *drm = nouveau_drm(plane->dev);
+
+	NV_ATOMIC(drm, "%s cleanup: %p\n", plane->name, old_state->fb);
+	if (!old_state->fb)
+		return;
+
+	nouveau_bo_unpin(fb->nvbo);
+}
+
+static int
+nv50_wndw_prepare_fb(struct drm_plane *plane, struct drm_plane_state *state)
+{
+	struct nouveau_framebuffer *fb = nouveau_framebuffer(state->fb);
+	struct nouveau_drm *drm = nouveau_drm(plane->dev);
+	struct nv50_wndw *wndw = nv50_wndw(plane);
+	struct nv50_wndw_atom *asyw = nv50_wndw_atom(state);
+	struct nv50_head_atom *asyh;
+	struct nv50_wndw_ctxdma *ctxdma;
+	int ret;
+
+	NV_ATOMIC(drm, "%s prepare: %p\n", plane->name, state->fb);
+	if (!asyw->state.fb)
+		return 0;
+
+	ret = nouveau_bo_pin(fb->nvbo, TTM_PL_FLAG_VRAM, true);
+	if (ret)
+		return ret;
+
+	ctxdma = nv50_wndw_ctxdma_new(wndw, fb);
+	if (IS_ERR(ctxdma)) {
+		nouveau_bo_unpin(fb->nvbo);
+		return PTR_ERR(ctxdma);
+	}
+
+	asyw->state.fence = reservation_object_get_excl_rcu(fb->nvbo->bo.resv);
+	asyw->image.handle[0] = ctxdma->object.handle;
+	asyw->image.offset[0] = fb->nvbo->bo.offset;
+
+	if (wndw->func->prepare) {
+		asyh = nv50_head_atom_get(asyw->state.state, asyw->state.crtc);
+		if (IS_ERR(asyh))
+			return PTR_ERR(asyh);
+
+		wndw->func->prepare(wndw, asyh, asyw);
+	}
+
+	return 0;
+}
+
+static const struct drm_plane_helper_funcs
+nv50_wndw_helper = {
+	.prepare_fb = nv50_wndw_prepare_fb,
+	.cleanup_fb = nv50_wndw_cleanup_fb,
+	.atomic_check = nv50_wndw_atomic_check,
+};
+
+static void
+nv50_wndw_atomic_destroy_state(struct drm_plane *plane,
+			       struct drm_plane_state *state)
+{
+	struct nv50_wndw_atom *asyw = nv50_wndw_atom(state);
+	__drm_atomic_helper_plane_destroy_state(&asyw->state);
+	kfree(asyw);
+}
+
+static struct drm_plane_state *
+nv50_wndw_atomic_duplicate_state(struct drm_plane *plane)
+{
+	struct nv50_wndw_atom *armw = nv50_wndw_atom(plane->state);
+	struct nv50_wndw_atom *asyw;
+	if (!(asyw = kmalloc(sizeof(*asyw), GFP_KERNEL)))
+		return NULL;
+	__drm_atomic_helper_plane_duplicate_state(plane, &asyw->state);
+	asyw->sema = armw->sema;
+	asyw->ntfy = armw->ntfy;
+	asyw->ilut = NULL;
+	asyw->xlut = armw->xlut;
+	asyw->image = armw->image;
+	asyw->point = armw->point;
+	asyw->clr.mask = 0;
+	asyw->set.mask = 0;
+	return &asyw->state;
+}
+
+static void
+nv50_wndw_reset(struct drm_plane *plane)
+{
+	struct nv50_wndw_atom *asyw;
+
+	if (WARN_ON(!(asyw = kzalloc(sizeof(*asyw), GFP_KERNEL))))
+		return;
+
+	if (plane->state)
+		plane->funcs->atomic_destroy_state(plane, plane->state);
+	plane->state = &asyw->state;
+	plane->state->plane = plane;
+	plane->state->rotation = DRM_MODE_ROTATE_0;
+}
+
+static void
+nv50_wndw_destroy(struct drm_plane *plane)
+{
+	struct nv50_wndw *wndw = nv50_wndw(plane);
+	struct nv50_wndw_ctxdma *ctxdma, *ctxtmp;
+
+	list_for_each_entry_safe(ctxdma, ctxtmp, &wndw->ctxdma.list, head) {
+		nv50_wndw_ctxdma_del(ctxdma);
+	}
+
+	nvif_notify_fini(&wndw->notify);
+	nv50_dmac_destroy(&wndw->wimm);
+	nv50_dmac_destroy(&wndw->wndw);
+
+	nv50_lut_fini(&wndw->ilut);
+
+	drm_plane_cleanup(&wndw->plane);
+	kfree(wndw);
+}
+
+const struct drm_plane_funcs
+nv50_wndw = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = nv50_wndw_destroy,
+	.reset = nv50_wndw_reset,
+	.atomic_duplicate_state = nv50_wndw_atomic_duplicate_state,
+	.atomic_destroy_state = nv50_wndw_atomic_destroy_state,
+};
+
+static int
+nv50_wndw_notify(struct nvif_notify *notify)
+{
+	return NVIF_NOTIFY_KEEP;
+}
+
+void
+nv50_wndw_fini(struct nv50_wndw *wndw)
+{
+	nvif_notify_put(&wndw->notify);
+}
+
+void
+nv50_wndw_init(struct nv50_wndw *wndw)
+{
+	nvif_notify_get(&wndw->notify);
+}
+
+int
+nv50_wndw_new_(const struct nv50_wndw_func *func, struct drm_device *dev,
+	       enum drm_plane_type type, const char *name, int index,
+	       const u32 *format, u32 heads,
+	       enum nv50_disp_interlock_type interlock_type, u32 interlock_data,
+	       struct nv50_wndw **pwndw)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvif_mmu *mmu = &drm->client.mmu;
+	struct nv50_disp *disp = nv50_disp(dev);
+	struct nv50_wndw *wndw;
+	int nformat;
+	int ret;
+
+	if (!(wndw = *pwndw = kzalloc(sizeof(*wndw), GFP_KERNEL)))
+		return -ENOMEM;
+	wndw->func = func;
+	wndw->id = index;
+	wndw->interlock.type = interlock_type;
+	wndw->interlock.data = interlock_data;
+	wndw->ctxdma.parent = &wndw->wndw.base.user;
+
+	wndw->ctxdma.parent = &wndw->wndw.base.user;
+	INIT_LIST_HEAD(&wndw->ctxdma.list);
+
+	for (nformat = 0; format[nformat]; nformat++);
+
+	ret = drm_universal_plane_init(dev, &wndw->plane, heads, &nv50_wndw,
+				       format, nformat, NULL,
+				       type, "%s-%d", name, index);
+	if (ret) {
+		kfree(*pwndw);
+		*pwndw = NULL;
+		return ret;
+	}
+
+	drm_plane_helper_add(&wndw->plane, &nv50_wndw_helper);
+
+	if (wndw->func->ilut) {
+		ret = nv50_lut_init(disp, mmu, &wndw->ilut);
+		if (ret)
+			return ret;
+	}
+
+	wndw->notify.func = nv50_wndw_notify;
+	return 0;
+}
+
+int
+nv50_wndw_new(struct nouveau_drm *drm, enum drm_plane_type type, int index,
+	      struct nv50_wndw **pwndw)
+{
+	struct {
+		s32 oclass;
+		int version;
+		int (*new)(struct nouveau_drm *, enum drm_plane_type,
+			   int, s32, struct nv50_wndw **);
+	} wndws[] = {
+		{ GV100_DISP_WINDOW_CHANNEL_DMA, 0, wndwc37e_new },
+		{}
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	int cid, ret;
+
+	cid = nvif_mclass(&disp->disp->object, wndws);
+	if (cid < 0) {
+		NV_ERROR(drm, "No supported window class\n");
+		return cid;
+	}
+
+	ret = wndws[cid].new(drm, type, index, wndws[cid].oclass, pwndw);
+	if (ret)
+		return ret;
+
+	return nv50_wimm_init(drm, *pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndw.h b/drivers/gpu/drm/nouveau/dispnv50/wndw.h
new file mode 100644
index 000000000000..b0b6428034b0
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/wndw.h
@@ -0,0 +1,96 @@
+#ifndef __NV50_KMS_WNDW_H__
+#define __NV50_KMS_WNDW_H__
+#define nv50_wndw(p) container_of((p), struct nv50_wndw, plane)
+#include "disp.h"
+#include "atom.h"
+#include "lut.h"
+
+#include <nvif/notify.h>
+
+struct nv50_wndw_ctxdma {
+	struct list_head head;
+	struct nvif_object object;
+};
+
+struct nv50_wndw {
+	const struct nv50_wndw_func *func;
+	const struct nv50_wimm_func *immd;
+	int id;
+	struct nv50_disp_interlock interlock;
+
+	struct {
+		struct nvif_object *parent;
+		struct list_head list;
+	} ctxdma;
+
+	struct drm_plane plane;
+
+	struct nv50_lut ilut;
+
+	struct nv50_dmac wndw;
+	struct nv50_dmac wimm;
+
+	struct nvif_notify notify;
+	u16 ntfy;
+	u16 sema;
+	u32 data;
+};
+
+int nv50_wndw_new_(const struct nv50_wndw_func *, struct drm_device *,
+		   enum drm_plane_type, const char *name, int index,
+		   const u32 *format, enum nv50_disp_interlock_type,
+		   u32 interlock_data, u32 heads, struct nv50_wndw **);
+void nv50_wndw_init(struct nv50_wndw *);
+void nv50_wndw_fini(struct nv50_wndw *);
+void nv50_wndw_flush_set(struct nv50_wndw *, u32 *interlock,
+			 struct nv50_wndw_atom *);
+void nv50_wndw_flush_clr(struct nv50_wndw *, u32 *interlock, bool flush,
+			 struct nv50_wndw_atom *);
+void nv50_wndw_ntfy_enable(struct nv50_wndw *, struct nv50_wndw_atom *);
+int nv50_wndw_wait_armed(struct nv50_wndw *, struct nv50_wndw_atom *);
+
+struct nv50_wndw_func {
+	int (*acquire)(struct nv50_wndw *, struct nv50_wndw_atom *asyw,
+		       struct nv50_head_atom *asyh);
+	void (*release)(struct nv50_wndw *, struct nv50_wndw_atom *asyw,
+			struct nv50_head_atom *asyh);
+	void (*prepare)(struct nv50_wndw *, struct nv50_head_atom *asyh,
+			struct nv50_wndw_atom *asyw);
+
+	void (*sema_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
+	void (*sema_clr)(struct nv50_wndw *);
+	void (*ntfy_reset)(struct nouveau_bo *, u32 offset);
+	void (*ntfy_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
+	void (*ntfy_clr)(struct nv50_wndw *);
+	int (*ntfy_wait_begun)(struct nouveau_bo *, u32 offset,
+			       struct nvif_device *);
+	void (*ilut)(struct nv50_wndw *, struct nv50_wndw_atom *);
+	bool olut_core;
+	void (*xlut_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
+	void (*xlut_clr)(struct nv50_wndw *);
+	void (*image_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
+	void (*image_clr)(struct nv50_wndw *);
+	void (*scale_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
+
+	void (*update)(struct nv50_wndw *, u32 *interlock);
+};
+
+extern const struct drm_plane_funcs nv50_wndw;
+
+void base507c_ntfy_reset(struct nouveau_bo *, u32);
+int base507c_ntfy_wait_begun(struct nouveau_bo *, u32, struct nvif_device *);
+
+struct nv50_wimm_func {
+	void (*point)(struct nv50_wndw *, struct nv50_wndw_atom *);
+
+	void (*update)(struct nv50_wndw *, u32 *interlock);
+};
+
+extern const struct nv50_wimm_func curs507a;
+
+int wndwc37e_new(struct nouveau_drm *, enum drm_plane_type, int, s32,
+		 struct nv50_wndw **);
+
+int nv50_wndw_new(struct nouveau_drm *, enum drm_plane_type, int index,
+		  struct nv50_wndw **);
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndwc37e.c b/drivers/gpu/drm/nouveau/dispnv50/wndwc37e.c
new file mode 100644
index 000000000000..44afb0f069a5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv50/wndwc37e.c
@@ -0,0 +1,278 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "wndw.h"
+#include "atom.h"
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <nouveau_bo.h>
+
+#include <nvif/clc37e.h>
+
+static void
+wndwc37e_ilut_clr(struct nv50_wndw *wndw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 2))) {
+		evo_mthd(push, 0x02b8, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+wndwc37e_ilut_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 4))) {
+		evo_mthd(push, 0x02b0, 3);
+		evo_data(push, asyw->xlut.i.output_mode << 8 |
+			       asyw->xlut.i.range << 4 |
+			       asyw->xlut.i.size);
+		evo_data(push, asyw->xlut.i.offset >> 8);
+		evo_data(push, asyw->xlut.handle);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+wndwc37e_ilut(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	asyw->xlut.i.mode = 2;
+	asyw->xlut.i.size = 0;
+	asyw->xlut.i.range = 0;
+	asyw->xlut.i.output_mode = 1;
+}
+
+static void
+wndwc37e_image_clr(struct nv50_wndw *wndw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 4))) {
+		evo_mthd(push, 0x0308, 1);
+		evo_data(push, 0x00000000);
+		evo_mthd(push, 0x0240, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+wndwc37e_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+
+	if (!(push = evo_wait(&wndw->wndw, 25)))
+		return;
+
+	evo_mthd(push, 0x0308, 1);
+	evo_data(push, asyw->image.mode << 4 | asyw->image.interval);
+	evo_mthd(push, 0x0224, 4);
+	evo_data(push, asyw->image.h << 16 | asyw->image.w);
+	evo_data(push, asyw->image.layout << 4 | asyw->image.blockh);
+	evo_data(push, asyw->image.colorspace << 8 | asyw->image.format);
+	evo_data(push, asyw->image.blocks[0] | (asyw->image.pitch[0] >> 6));
+	evo_mthd(push, 0x0240, 1);
+	evo_data(push, asyw->image.handle[0]);
+	evo_mthd(push, 0x0260, 1);
+	evo_data(push, asyw->image.offset[0] >> 8);
+	evo_mthd(push, 0x0290, 1);
+	evo_data(push, (asyw->state.src_y >> 16) << 16 |
+		       (asyw->state.src_x >> 16));
+	evo_mthd(push, 0x0298, 1);
+	evo_data(push, (asyw->state.src_h >> 16) << 16 |
+		       (asyw->state.src_w >> 16));
+	evo_mthd(push, 0x02a4, 1);
+	evo_data(push, asyw->state.crtc_h << 16 |
+		       asyw->state.crtc_w);
+
+	/*XXX: Composition-related stuff.  Need to implement properly. */
+	evo_mthd(push, 0x02ec, 1);
+	evo_data(push, (2 - (wndw->id & 1)) << 4);
+	evo_mthd(push, 0x02f4, 5);
+	evo_data(push, 0x00000011);
+	evo_data(push, 0xffff0000);
+	evo_data(push, 0xffff0000);
+	evo_data(push, 0xffff0000);
+	evo_data(push, 0xffff0000);
+	evo_kick(push, &wndw->wndw);
+}
+
+static void
+wndwc37e_ntfy_clr(struct nv50_wndw *wndw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 2))) {
+		evo_mthd(push, 0x021c, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+wndwc37e_ntfy_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 3))) {
+		evo_mthd(push, 0x021c, 2);
+		evo_data(push, asyw->ntfy.handle);
+		evo_data(push, asyw->ntfy.offset | asyw->ntfy.awaken);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+wndwc37e_sema_clr(struct nv50_wndw *wndw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 2))) {
+		evo_mthd(push, 0x0218, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+wndwc37e_sema_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 5))) {
+		evo_mthd(push, 0x020c, 4);
+		evo_data(push, asyw->sema.offset);
+		evo_data(push, asyw->sema.acquire);
+		evo_data(push, asyw->sema.release);
+		evo_data(push, asyw->sema.handle);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+wndwc37e_update(struct nv50_wndw *wndw, u32 *interlock)
+{
+	u32 *push;
+	if ((push = evo_wait(&wndw->wndw, 5))) {
+		evo_mthd(push, 0x0370, 2);
+		evo_data(push, interlock[NV50_DISP_INTERLOCK_CURS] << 1 |
+			       interlock[NV50_DISP_INTERLOCK_CORE]);
+		evo_data(push, interlock[NV50_DISP_INTERLOCK_WNDW]);
+		evo_mthd(push, 0x0200, 1);
+		if (interlock[NV50_DISP_INTERLOCK_WIMM] & wndw->interlock.data)
+			evo_data(push, 0x00001001);
+		else
+			evo_data(push, 0x00000001);
+		evo_kick(push, &wndw->wndw);
+	}
+}
+
+static void
+wndwc37e_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		 struct nv50_head_atom *asyh)
+{
+}
+
+static int
+wndwc37e_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		 struct nv50_head_atom *asyh)
+{
+	return drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
+						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_HELPER_NO_SCALING,
+						   true, true);
+}
+
+static const u32
+wndwc37e_format[] = {
+	DRM_FORMAT_C8,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_ABGR2101010,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_ARGB2101010,
+	0
+};
+
+static const struct nv50_wndw_func
+wndwc37e = {
+	.acquire = wndwc37e_acquire,
+	.release = wndwc37e_release,
+	.sema_set = wndwc37e_sema_set,
+	.sema_clr = wndwc37e_sema_clr,
+	.ntfy_set = wndwc37e_ntfy_set,
+	.ntfy_clr = wndwc37e_ntfy_clr,
+	.ntfy_reset = corec37d_ntfy_init,
+	.ntfy_wait_begun = base507c_ntfy_wait_begun,
+	.ilut = wndwc37e_ilut,
+	.xlut_set = wndwc37e_ilut_set,
+	.xlut_clr = wndwc37e_ilut_clr,
+	.image_set = wndwc37e_image_set,
+	.image_clr = wndwc37e_image_clr,
+	.update = wndwc37e_update,
+};
+
+static int
+wndwc37e_new_(const struct nv50_wndw_func *func, struct nouveau_drm *drm,
+	      enum drm_plane_type type, int index, s32 oclass, u32 heads,
+	      struct nv50_wndw **pwndw)
+{
+	struct nvc37e_window_channel_dma_v0 args = {
+		.pushbuf = 0xb0007e00 | index,
+		.index = index,
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	struct nv50_wndw *wndw;
+	int ret;
+
+	ret = nv50_wndw_new_(func, drm->dev, type, "wndw", index,
+			     wndwc37e_format, heads, NV50_DISP_INTERLOCK_WNDW,
+			     BIT(index), &wndw);
+	if (*pwndw = wndw, ret)
+		return ret;
+
+	ret = nv50_dmac_create(&drm->client.device, &disp->disp->object,
+			       &oclass, 0, &args, sizeof(args),
+			       disp->sync->bo.offset, &wndw->wndw);
+	if (ret) {
+		NV_ERROR(drm, "qndw%04x allocation failed: %d\n", oclass, ret);
+		return ret;
+	}
+
+	wndw->ntfy = NV50_DISP_WNDW_NTFY(wndw->id);
+	wndw->sema = NV50_DISP_WNDW_SEM0(wndw->id);
+	wndw->data = 0x00000000;
+	return 0;
+}
+
+int
+wndwc37e_new(struct nouveau_drm *drm, enum drm_plane_type type, int index,
+	     s32 oclass, struct nv50_wndw **pwndw)
+{
+	return wndwc37e_new_(&wndwc37e, drm, type, index, oclass,
+			     BIT(index >> 1), pwndw);
+}
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h
index 2740278d226b..4f5233107f5f 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h
@@ -31,6 +31,7 @@ struct nv_device_info_v0 {
 #define NV_DEVICE_INFO_V0_KEPLER                                           0x08
 #define NV_DEVICE_INFO_V0_MAXWELL                                          0x09
 #define NV_DEVICE_INFO_V0_PASCAL                                           0x0a
+#define NV_DEVICE_INFO_V0_VOLTA                                            0x0b
 	__u8  family;
 	__u8  pad06[2];
 	__u64 ram_size;
@@ -39,9 +40,55 @@ struct nv_device_info_v0 {
 	char  name[64];
 };
 
+struct nv_device_info_v1 {
+	__u8  version;
+	__u8  count;
+	__u8  pad02[6];
+	struct nv_device_info_v1_data {
+		__u64 mthd; /* NV_DEVICE_INFO_* (see below). */
+		__u64 data;
+	} data[];
+};
+
 struct nv_device_time_v0 {
 	__u8  version;
 	__u8  pad01[7];
 	__u64 time;
 };
+
+#define NV_DEVICE_INFO_UNIT                               (0xffffffffULL << 32)
+#define NV_DEVICE_INFO(n)                          ((n) | (0x00000000ULL << 32))
+#define NV_DEVICE_FIFO(n)                          ((n) | (0x00000001ULL << 32))
+
+/* This will be returned for unsupported queries. */
+#define NV_DEVICE_INFO_INVALID                                           ~0ULL
+
+/* These return a mask of available engines of particular type. */
+#define NV_DEVICE_INFO_ENGINE_SW                     NV_DEVICE_INFO(0x00000000)
+#define NV_DEVICE_INFO_ENGINE_GR                     NV_DEVICE_INFO(0x00000001)
+#define NV_DEVICE_INFO_ENGINE_MPEG                   NV_DEVICE_INFO(0x00000002)
+#define NV_DEVICE_INFO_ENGINE_ME                     NV_DEVICE_INFO(0x00000003)
+#define NV_DEVICE_INFO_ENGINE_CIPHER                 NV_DEVICE_INFO(0x00000004)
+#define NV_DEVICE_INFO_ENGINE_BSP                    NV_DEVICE_INFO(0x00000005)
+#define NV_DEVICE_INFO_ENGINE_VP                     NV_DEVICE_INFO(0x00000006)
+#define NV_DEVICE_INFO_ENGINE_CE                     NV_DEVICE_INFO(0x00000007)
+#define NV_DEVICE_INFO_ENGINE_SEC                    NV_DEVICE_INFO(0x00000008)
+#define NV_DEVICE_INFO_ENGINE_MSVLD                  NV_DEVICE_INFO(0x00000009)
+#define NV_DEVICE_INFO_ENGINE_MSPDEC                 NV_DEVICE_INFO(0x0000000a)
+#define NV_DEVICE_INFO_ENGINE_MSPPP                  NV_DEVICE_INFO(0x0000000b)
+#define NV_DEVICE_INFO_ENGINE_MSENC                  NV_DEVICE_INFO(0x0000000c)
+#define NV_DEVICE_INFO_ENGINE_VIC                    NV_DEVICE_INFO(0x0000000d)
+#define NV_DEVICE_INFO_ENGINE_SEC2                   NV_DEVICE_INFO(0x0000000e)
+#define NV_DEVICE_INFO_ENGINE_NVDEC                  NV_DEVICE_INFO(0x0000000f)
+#define NV_DEVICE_INFO_ENGINE_NVENC                  NV_DEVICE_INFO(0x00000010)
+
+/* Returns the number of available channels. */
+#define NV_DEVICE_FIFO_CHANNELS                      NV_DEVICE_FIFO(0x00000000)
+
+/* Returns a mask of available runlists. */
+#define NV_DEVICE_FIFO_RUNLISTS                      NV_DEVICE_FIFO(0x00000001)
+
+/* These return a mask of engines available on a particular runlist. */
+#define NV_DEVICE_FIFO_RUNLIST_ENGINES(n)     ((n) + NV_DEVICE_FIFO(0x00000010))
+#define NV_DEVICE_FIFO_RUNLIST_ENGINES__SIZE                                64
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cla06f.h b/drivers/gpu/drm/nouveau/include/nvif/cla06f.h
index 56f5bd81e480..fbfcffc5feb2 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/cla06f.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/cla06f.h
@@ -4,25 +4,11 @@
 
 struct kepler_channel_gpfifo_a_v0 {
 	__u8  version;
-	__u8  pad01[5];
+	__u8  pad01[1];
 	__u16 chid;
-#define NVA06F_V0_ENGINE_SW                                          0x00000001
-#define NVA06F_V0_ENGINE_GR                                          0x00000002
-#define NVA06F_V0_ENGINE_SEC                                         0x00000004
-#define NVA06F_V0_ENGINE_MSVLD                                       0x00000010
-#define NVA06F_V0_ENGINE_MSPDEC                                      0x00000020
-#define NVA06F_V0_ENGINE_MSPPP                                       0x00000040
-#define NVA06F_V0_ENGINE_MSENC                                       0x00000080
-#define NVA06F_V0_ENGINE_VIC                                         0x00000100
-#define NVA06F_V0_ENGINE_NVDEC                                       0x00000200
-#define NVA06F_V0_ENGINE_NVENC0                                      0x00000400
-#define NVA06F_V0_ENGINE_NVENC1                                      0x00000800
-#define NVA06F_V0_ENGINE_CE0                                         0x00010000
-#define NVA06F_V0_ENGINE_CE1                                         0x00020000
-#define NVA06F_V0_ENGINE_CE2                                         0x00040000
-	__u32 engines;
 	__u32 ilength;
 	__u64 ioffset;
+	__u64 runlist;
 	__u64 vmm;
 };
 
diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h
index a7c5bf572788..6db56bd7d67e 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/class.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/class.h
@@ -52,6 +52,8 @@
 
 #define NV04_DISP                                     /* cl0046.h */ 0x00000046
 
+#define VOLTA_USERMODE_A                                             0x0000c361
+
 #define NV03_CHANNEL_DMA                              /* cl506b.h */ 0x0000006b
 #define NV10_CHANNEL_DMA                              /* cl506b.h */ 0x0000006e
 #define NV17_CHANNEL_DMA                              /* cl506b.h */ 0x0000176e
@@ -66,6 +68,7 @@
 #define KEPLER_CHANNEL_GPFIFO_B                       /* cla06f.h */ 0x0000a16f
 #define MAXWELL_CHANNEL_GPFIFO_A                      /* cla06f.h */ 0x0000b06f
 #define PASCAL_CHANNEL_GPFIFO_A                       /* cla06f.h */ 0x0000c06f
+#define VOLTA_CHANNEL_GPFIFO_A                        /* cla06f.h */ 0x0000c36f
 
 #define NV50_DISP                                     /* cl5070.h */ 0x00005070
 #define G82_DISP                                      /* cl5070.h */ 0x00008270
@@ -79,6 +82,7 @@
 #define GM200_DISP                                    /* cl5070.h */ 0x00009570
 #define GP100_DISP                                    /* cl5070.h */ 0x00009770
 #define GP102_DISP                                    /* cl5070.h */ 0x00009870
+#define GV100_DISP                                    /* cl5070.h */ 0x0000c370
 
 #define NV31_MPEG                                                    0x00003174
 #define G82_MPEG                                                     0x00008274
@@ -90,6 +94,7 @@
 #define GT214_DISP_CURSOR                             /* cl507a.h */ 0x0000857a
 #define GF110_DISP_CURSOR                             /* cl507a.h */ 0x0000907a
 #define GK104_DISP_CURSOR                             /* cl507a.h */ 0x0000917a
+#define GV100_DISP_CURSOR                             /* cl507a.h */ 0x0000c37a
 
 #define NV50_DISP_OVERLAY                             /* cl507b.h */ 0x0000507b
 #define G82_DISP_OVERLAY                              /* cl507b.h */ 0x0000827b
@@ -97,6 +102,8 @@
 #define GF110_DISP_OVERLAY                            /* cl507b.h */ 0x0000907b
 #define GK104_DISP_OVERLAY                            /* cl507b.h */ 0x0000917b
 
+#define GV100_DISP_WINDOW_IMM_CHANNEL_DMA             /* clc37b.h */ 0x0000c37b
+
 #define NV50_DISP_BASE_CHANNEL_DMA                    /* cl507c.h */ 0x0000507c
 #define G82_DISP_BASE_CHANNEL_DMA                     /* cl507c.h */ 0x0000827c
 #define GT200_DISP_BASE_CHANNEL_DMA                   /* cl507c.h */ 0x0000837c
@@ -117,6 +124,7 @@
 #define GM200_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000957d
 #define GP100_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000977d
 #define GP102_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000987d
+#define GV100_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000c37d
 
 #define NV50_DISP_OVERLAY_CHANNEL_DMA                 /* cl507e.h */ 0x0000507e
 #define G82_DISP_OVERLAY_CHANNEL_DMA                  /* cl507e.h */ 0x0000827e
@@ -125,6 +133,8 @@
 #define GF110_DISP_OVERLAY_CONTROL_DMA                /* cl507e.h */ 0x0000907e
 #define GK104_DISP_OVERLAY_CONTROL_DMA                /* cl507e.h */ 0x0000917e
 
+#define GV100_DISP_WINDOW_CHANNEL_DMA                 /* clc37e.h */ 0x0000c37e
+
 #define NV50_TESLA                                                   0x00005097
 #define G82_TESLA                                                    0x00008297
 #define GT200_TESLA                                                  0x00008397
@@ -145,6 +155,8 @@
 #define PASCAL_A                                      /* cl9097.h */ 0x0000c097
 #define PASCAL_B                                      /* cl9097.h */ 0x0000c197
 
+#define VOLTA_A                                       /* cl9097.h */ 0x0000c397
+
 #define NV74_BSP                                                     0x000074b0
 
 #define GT212_MSVLD                                                  0x000085b1
@@ -170,6 +182,7 @@
 #define MAXWELL_DMA_COPY_A                                           0x0000b0b5
 #define PASCAL_DMA_COPY_A                                            0x0000c0b5
 #define PASCAL_DMA_COPY_B                                            0x0000c1b5
+#define VOLTA_DMA_COPY_A                                             0x0000c3b5
 
 #define FERMI_DECOMPRESS                                             0x000090b8
 
@@ -183,6 +196,7 @@
 #define MAXWELL_COMPUTE_B                                            0x0000b1c0
 #define PASCAL_COMPUTE_A                                             0x0000c0c0
 #define PASCAL_COMPUTE_B                                             0x0000c1c0
+#define VOLTA_COMPUTE_A                                              0x0000c3c0
 
 #define NV74_CIPHER                                                  0x000074c1
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/clc37b.h b/drivers/gpu/drm/nouveau/include/nvif/clc37b.h
new file mode 100644
index 000000000000..89b18189d43b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/clc37b.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NVIF_CLC37B_H__
+#define __NVIF_CLC37B_H__
+
+struct nvc37b_window_imm_channel_dma_v0 {
+	__u8  version;
+	__u8  index;
+	__u8  pad02[6];
+	__u64 pushbuf;
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/clc37e.h b/drivers/gpu/drm/nouveau/include/nvif/clc37e.h
new file mode 100644
index 000000000000..899db9e915ef
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/clc37e.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NVIF_CLC37E_H__
+#define __NVIF_CLC37E_H__
+
+struct nvc37e_window_channel_dma_v0 {
+	__u8  version;
+	__u8  index;
+	__u8  pad02[6];
+	__u64 pushbuf;
+};
+
+#define NVC37E_WINDOW_CHANNEL_DMA_V0_NTFY_UEVENT                           0x00
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/device.h b/drivers/gpu/drm/nouveau/include/nvif/device.h
index 6edb6266857e..ef839bd1d37e 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/device.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/device.h
@@ -4,10 +4,18 @@
 
 #include <nvif/object.h>
 #include <nvif/cl0080.h>
+#include <nvif/user.h>
 
 struct nvif_device {
 	struct nvif_object object;
 	struct nv_device_info_v0 info;
+
+	struct nvif_fifo_runlist {
+		u64 engines;
+	} *runlist;
+	int runlists;
+
+	struct nvif_user user;
 };
 
 int  nvif_device_init(struct nvif_object *, u32 handle, s32 oclass, void *, u32,
@@ -67,6 +75,5 @@ u64  nvif_device_time(struct nvif_device *);
 #include <engine/fifo.h>
 #include <engine/gr.h>
 
-#define nvxx_fifo(a) nvxx_device(a)->fifo
 #define nvxx_gr(a) nvxx_device(a)->gr
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/disp.h b/drivers/gpu/drm/nouveau/include/nvif/disp.h
new file mode 100644
index 000000000000..7c0eda375c01
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/disp.h
@@ -0,0 +1,12 @@
+#ifndef __NVIF_DISP_H__
+#define __NVIF_DISP_H__
+#include <nvif/object.h>
+struct nvif_device;
+
+struct nvif_disp {
+	struct nvif_object object;
+};
+
+int nvif_disp_ctor(struct nvif_device *, s32 oclass, struct nvif_disp *);
+void nvif_disp_dtor(struct nvif_disp *);
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/fifo.h b/drivers/gpu/drm/nouveau/include/nvif/fifo.h
new file mode 100644
index 000000000000..e9468c9f9abf
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/fifo.h
@@ -0,0 +1,18 @@
+#ifndef __NVIF_FIFO_H__
+#define __NVIF_FIFO_H__
+#include <nvif/device.h>
+
+/* Returns mask of runlists that support a NV_DEVICE_INFO_ENGINE_* type. */
+u64 nvif_fifo_runlist(struct nvif_device *, u64 engine);
+
+/* CE-supporting runlists (excluding GRCE, if others exist). */
+static inline u64
+nvif_fifo_runlist_ce(struct nvif_device *device)
+{
+	u64 runmgr = nvif_fifo_runlist(device, NV_DEVICE_INFO_ENGINE_GR);
+	u64 runmce = nvif_fifo_runlist(device, NV_DEVICE_INFO_ENGINE_CE);
+	if (runmce && !(runmce &= ~runmgr))
+		runmce = runmgr;
+	return runmce;
+}
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/mem.h b/drivers/gpu/drm/nouveau/include/nvif/mem.h
index b542fe38398e..80ee4ab0f016 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/mem.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/mem.h
@@ -15,4 +15,6 @@ int nvif_mem_init_type(struct nvif_mmu *mmu, s32 oclass, int type, u8 page,
 int nvif_mem_init(struct nvif_mmu *mmu, s32 oclass, u8 type, u8 page,
 		  u64 size, void *argv, u32 argc, struct nvif_mem *);
 void nvif_mem_fini(struct nvif_mem *);
+
+int nvif_mem_init_map(struct nvif_mmu *, u8 type, u64 size, struct nvif_mem *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvif/mmu.h b/drivers/gpu/drm/nouveau/include/nvif/mmu.h
index c8cd5b5b0688..747ecf67e403 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/mmu.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/mmu.h
@@ -8,6 +8,7 @@ struct nvif_mmu {
 	u8  heap_nr;
 	u8  type_nr;
 	u16 kind_nr;
+	s32 mem;
 
 	struct {
 		u64 size;
diff --git a/drivers/gpu/drm/nouveau/include/nvif/object.h b/drivers/gpu/drm/nouveau/include/nvif/object.h
index a2d5244ff2b7..20754d9e6883 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/object.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/object.h
@@ -99,6 +99,22 @@ struct nvif_mclass {
 	ret;                                                                   \
 })
 
+#define nvif_sclass(o,m,u) ({                                                  \
+	const typeof(m[0]) *_mclass = (m);                                     \
+	s32 _oclass = (u);                                                     \
+	int _cid;                                                              \
+	if (_oclass) {                                                         \
+		for (_cid = 0; _mclass[_cid].oclass; _cid++) {                 \
+			if (_mclass[_cid].oclass == _oclass)                   \
+				break;                                         \
+		}                                                              \
+		_cid = _mclass[_cid].oclass ? _cid : -ENOSYS;                  \
+	} else {                                                               \
+		_cid = nvif_mclass((o), _mclass);                              \
+	}                                                                      \
+	_cid;                                                                  \
+})
+
 /*XXX*/
 #include <core/object.h>
 #define nvxx_object(a) ({                                                      \
diff --git a/drivers/gpu/drm/nouveau/include/nvif/user.h b/drivers/gpu/drm/nouveau/include/nvif/user.h
new file mode 100644
index 000000000000..03c11826b693
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvif/user.h
@@ -0,0 +1,19 @@
+#ifndef __NVIF_USER_H__
+#define __NVIF_USER_H__
+#include <nvif/object.h>
+struct nvif_device;
+
+struct nvif_user {
+	const struct nvif_user_func *func;
+	struct nvif_object object;
+};
+
+struct nvif_user_func {
+	void (*doorbell)(struct nvif_user *, u32 token);
+};
+
+int nvif_user_init(struct nvif_device *);
+void nvif_user_fini(struct nvif_device *);
+
+extern const struct nvif_user_func nvif_userc361;
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
index 560265b15ec2..d83d834b7452 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
@@ -22,6 +22,7 @@ enum nvkm_devidx {
 	NVKM_SUBDEV_LTC,
 	NVKM_SUBDEV_MMU,
 	NVKM_SUBDEV_BAR,
+	NVKM_SUBDEV_FAULT,
 	NVKM_SUBDEV_PMU,
 	NVKM_SUBDEV_VOLT,
 	NVKM_SUBDEV_ICCSENSE,
@@ -37,7 +38,10 @@ enum nvkm_devidx {
 	NVKM_ENGINE_CE3,
 	NVKM_ENGINE_CE4,
 	NVKM_ENGINE_CE5,
-	NVKM_ENGINE_CE_LAST = NVKM_ENGINE_CE5,
+	NVKM_ENGINE_CE6,
+	NVKM_ENGINE_CE7,
+	NVKM_ENGINE_CE8,
+	NVKM_ENGINE_CE_LAST = NVKM_ENGINE_CE8,
 
 	NVKM_ENGINE_CIPHER,
 	NVKM_ENGINE_DISP,
@@ -109,6 +113,7 @@ struct nvkm_device {
 		NV_E0    = 0xe0,
 		GM100    = 0x110,
 		GP100    = 0x130,
+		GV100    = 0x140,
 	} card_type;
 	u32 chipset;
 	u8  chiprev;
@@ -123,6 +128,7 @@ struct nvkm_device {
 	struct nvkm_bus *bus;
 	struct nvkm_clk *clk;
 	struct nvkm_devinit *devinit;
+	struct nvkm_fault *fault;
 	struct nvkm_fb *fb;
 	struct nvkm_fuse *fuse;
 	struct nvkm_gpio *gpio;
@@ -143,7 +149,7 @@ struct nvkm_device {
 	struct nvkm_volt *volt;
 
 	struct nvkm_engine *bsp;
-	struct nvkm_engine *ce[6];
+	struct nvkm_engine *ce[9];
 	struct nvkm_engine *cipher;
 	struct nvkm_disp *disp;
 	struct nvkm_dma *dma;
@@ -194,6 +200,7 @@ struct nvkm_device_chip {
 	int (*bus     )(struct nvkm_device *, int idx, struct nvkm_bus **);
 	int (*clk     )(struct nvkm_device *, int idx, struct nvkm_clk **);
 	int (*devinit )(struct nvkm_device *, int idx, struct nvkm_devinit **);
+	int (*fault   )(struct nvkm_device *, int idx, struct nvkm_fault **);
 	int (*fb      )(struct nvkm_device *, int idx, struct nvkm_fb **);
 	int (*fuse    )(struct nvkm_device *, int idx, struct nvkm_fuse **);
 	int (*gpio    )(struct nvkm_device *, int idx, struct nvkm_gpio **);
@@ -214,7 +221,7 @@ struct nvkm_device_chip {
 	int (*volt    )(struct nvkm_device *, int idx, struct nvkm_volt **);
 
 	int (*bsp     )(struct nvkm_device *, int idx, struct nvkm_engine **);
-	int (*ce[6]   )(struct nvkm_device *, int idx, struct nvkm_engine **);
+	int (*ce[9]   )(struct nvkm_device *, int idx, struct nvkm_engine **);
 	int (*cipher  )(struct nvkm_device *, int idx, struct nvkm_engine **);
 	int (*disp    )(struct nvkm_device *, int idx, struct nvkm_disp **);
 	int (*dma     )(struct nvkm_device *, int idx, struct nvkm_dma **);
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h b/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h
index ebf8473a39fe..8a2be5b635e2 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h
@@ -18,6 +18,7 @@ struct nvkm_engine_func {
 	void *(*dtor)(struct nvkm_engine *);
 	void (*preinit)(struct nvkm_engine *);
 	int (*oneinit)(struct nvkm_engine *);
+	int (*info)(struct nvkm_engine *, u64 mthd, u64 *data);
 	int (*init)(struct nvkm_engine *);
 	int (*fini)(struct nvkm_engine *, bool suspend);
 	void (*intr)(struct nvkm_engine *);
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h
index 63df2290177f..85a0777c2ce4 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h
@@ -17,6 +17,7 @@ struct nvkm_subdev_func {
 	void *(*dtor)(struct nvkm_subdev *);
 	int (*preinit)(struct nvkm_subdev *);
 	int (*oneinit)(struct nvkm_subdev *);
+	int (*info)(struct nvkm_subdev *, u64 mthd, u64 *data);
 	int (*init)(struct nvkm_subdev *);
 	int (*fini)(struct nvkm_subdev *, bool suspend);
 	void (*intr)(struct nvkm_subdev *);
@@ -29,6 +30,7 @@ void nvkm_subdev_del(struct nvkm_subdev **);
 int  nvkm_subdev_preinit(struct nvkm_subdev *);
 int  nvkm_subdev_init(struct nvkm_subdev *);
 int  nvkm_subdev_fini(struct nvkm_subdev *, bool suspend);
+int  nvkm_subdev_info(struct nvkm_subdev *, u64, u64 *);
 void nvkm_subdev_intr(struct nvkm_subdev *);
 
 /* subdev logging */
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h
index 553245994450..fc295e1faa19 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h
@@ -10,4 +10,5 @@ int gm107_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 int gm200_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 int gp100_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 int gp102_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
+int gv100_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h
index e83193d3ccab..ef7dc0844d26 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h
@@ -35,4 +35,5 @@ int gm107_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 int gm200_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 int gp100_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 int gp102_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
+int gv100_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h
index 0f9c1c702ed6..f0c1b2c8c78c 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h
@@ -27,4 +27,5 @@ int nv04_dma_new(struct nvkm_device *, int, struct nvkm_dma **);
 int nv50_dma_new(struct nvkm_device *, int, struct nvkm_dma **);
 int gf100_dma_new(struct nvkm_device *, int, struct nvkm_dma **);
 int gf119_dma_new(struct nvkm_device *, int, struct nvkm_dma **);
+int gv100_dma_new(struct nvkm_device *, int, struct nvkm_dma **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
index c17b3a9bf8fb..7e39fbed2519 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
@@ -4,6 +4,7 @@
 #include <core/engine.h>
 #include <core/object.h>
 #include <core/event.h>
+struct nvkm_fault_data;
 
 #define NVKM_FIFO_CHID_NR 4096
 
@@ -45,6 +46,7 @@ struct nvkm_fifo {
 	struct nvkm_event kevent; /* channel killed */
 };
 
+void nvkm_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *);
 void nvkm_fifo_pause(struct nvkm_fifo *, unsigned long *);
 void nvkm_fifo_start(struct nvkm_fifo *, unsigned long *);
 
@@ -71,4 +73,5 @@ int gm200_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **);
 int gm20b_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **);
 int gp100_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **);
 int gp10b_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **);
+int gv100_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h
index fb18f105fc43..ba1518ff8b66 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h
@@ -45,6 +45,8 @@ int gm200_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
 int gm20b_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
 int gp100_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
 int gp102_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
+int gp104_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
 int gp107_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
 int gp10b_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
+int gv100_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/dp.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/dp.h
index df34b41838d6..512e25a41803 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/dp.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/dp.h
@@ -1,6 +1,10 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef __NVBIOS_DP_H__
 #define __NVBIOS_DP_H__
+
+u16
+nvbios_dp_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+
 struct nvbios_dpout {
 	u16 type;
 	u16 mask;
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/devinit.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/devinit.h
index 40558064d589..486e7635c29d 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/devinit.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/devinit.h
@@ -30,4 +30,5 @@ int mcp89_devinit_new(struct nvkm_device *, int, struct nvkm_devinit **);
 int gf100_devinit_new(struct nvkm_device *, int, struct nvkm_devinit **);
 int gm107_devinit_new(struct nvkm_device *, int, struct nvkm_devinit **);
 int gm200_devinit_new(struct nvkm_device *, int, struct nvkm_devinit **);
+int gv100_devinit_new(struct nvkm_device *, int, struct nvkm_devinit **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h
new file mode 100644
index 000000000000..5a77498fe6a0
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h
@@ -0,0 +1,33 @@
+#ifndef __NVKM_FAULT_H__
+#define __NVKM_FAULT_H__
+#include <core/subdev.h>
+#include <core/notify.h>
+
+struct nvkm_fault {
+	const struct nvkm_fault_func *func;
+	struct nvkm_subdev subdev;
+
+	struct nvkm_fault_buffer *buffer[2];
+	int buffer_nr;
+
+	struct nvkm_event event;
+
+	struct nvkm_notify nrpfb;
+};
+
+struct nvkm_fault_data {
+	u64  addr;
+	u64  inst;
+	u64  time;
+	u8 engine;
+	u8  valid;
+	u8    gpc;
+	u8    hub;
+	u8 access;
+	u8 client;
+	u8 reason;
+};
+
+int gp100_fault_new(struct nvkm_device *, int, struct nvkm_fault **);
+int gv100_fault_new(struct nvkm_device *, int, struct nvkm_fault **);
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
index 92be0e5269c6..96ccc624ee81 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
@@ -83,6 +83,7 @@ int gm20b_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gp100_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gp102_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gp10b_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
+int gv100_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 
 #include <subdev/bios.h>
 #include <subdev/bios/ramcfg.h>
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h
index 95b611554d53..9db5f8293198 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h
@@ -21,12 +21,14 @@ struct nvkm_ltc {
 	int zbc_max;
 	u32 zbc_color[NVKM_LTC_MAX_ZBC_CNT][4];
 	u32 zbc_depth[NVKM_LTC_MAX_ZBC_CNT];
+	u32 zbc_stencil[NVKM_LTC_MAX_ZBC_CNT];
 };
 
 void nvkm_ltc_tags_clear(struct nvkm_device *, u32 first, u32 count);
 
 int nvkm_ltc_zbc_color_get(struct nvkm_ltc *, int index, const u32[4]);
 int nvkm_ltc_zbc_depth_get(struct nvkm_ltc *, int index, const u32);
+int nvkm_ltc_zbc_stencil_get(struct nvkm_ltc *, int index, const u32);
 
 void nvkm_ltc_invalidate(struct nvkm_ltc *);
 void nvkm_ltc_flush(struct nvkm_ltc *);
@@ -37,4 +39,5 @@ int gk20a_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 int gm107_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 int gm200_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 int gp100_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
+int gp102_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
index baab93398e54..688595545e21 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
@@ -129,4 +129,5 @@ int gm200_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **);
 int gm20b_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **);
 int gp100_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **);
 int gp10b_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **);
+int gv100_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index ece650a0c5f9..e2211bb2cf79 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -23,6 +23,7 @@
 
 #include <nvif/client.h>
 #include <nvif/driver.h>
+#include <nvif/fifo.h>
 #include <nvif/ioctl.h>
 #include <nvif/class.h>
 #include <nvif/cl0002.h>
@@ -102,6 +103,7 @@ nouveau_abi16_swclass(struct nouveau_drm *drm)
 	case NV_DEVICE_INFO_V0_KEPLER:
 	case NV_DEVICE_INFO_V0_MAXWELL:
 	case NV_DEVICE_INFO_V0_PASCAL:
+	case NV_DEVICE_INFO_V0_VOLTA:
 		return NVIF_CLASS_SW_GF100;
 	}
 
@@ -256,6 +258,7 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
 	struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
 	struct nouveau_abi16_chan *chan;
 	struct nvif_device *device;
+	u64 engine;
 	int ret;
 
 	if (unlikely(!abi16))
@@ -268,25 +271,26 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
 
 	/* hack to allow channel engine type specification on kepler */
 	if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
-		if (init->fb_ctxdma_handle != ~0)
-			init->fb_ctxdma_handle = NVA06F_V0_ENGINE_GR;
-		else {
-			init->fb_ctxdma_handle = 0;
-#define _(A,B) if (init->tt_ctxdma_handle & (A)) init->fb_ctxdma_handle |= (B)
-			_(0x01, NVA06F_V0_ENGINE_GR);
-			_(0x02, NVA06F_V0_ENGINE_MSPDEC);
-			_(0x04, NVA06F_V0_ENGINE_MSPPP);
-			_(0x08, NVA06F_V0_ENGINE_MSVLD);
-			_(0x10, NVA06F_V0_ENGINE_CE0);
-			_(0x20, NVA06F_V0_ENGINE_CE1);
-			_(0x40, NVA06F_V0_ENGINE_MSENC);
-#undef _
+		if (init->fb_ctxdma_handle == ~0) {
+			switch (init->tt_ctxdma_handle) {
+			case 0x01: engine = NV_DEVICE_INFO_ENGINE_GR    ; break;
+			case 0x02: engine = NV_DEVICE_INFO_ENGINE_MSPDEC; break;
+			case 0x04: engine = NV_DEVICE_INFO_ENGINE_MSPPP ; break;
+			case 0x08: engine = NV_DEVICE_INFO_ENGINE_MSVLD ; break;
+			case 0x30: engine = NV_DEVICE_INFO_ENGINE_CE    ; break;
+			default:
+				return nouveau_abi16_put(abi16, -ENOSYS);
+			}
+		} else {
+			engine = NV_DEVICE_INFO_ENGINE_GR;
 		}
 
-		/* allow flips to be executed if this is a graphics channel */
+		if (engine != NV_DEVICE_INFO_ENGINE_CE)
+			engine = nvif_fifo_runlist(device, engine);
+		else
+			engine = nvif_fifo_runlist_ce(device);
+		init->fb_ctxdma_handle = engine;
 		init->tt_ctxdma_handle = 0;
-		if (init->fb_ctxdma_handle == NVA06F_V0_ENGINE_GR)
-			init->tt_ctxdma_handle = 1;
 	}
 
 	if (init->fb_ctxdma_handle == ~0 || init->tt_ctxdma_handle == ~0)
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.c b/drivers/gpu/drm/nouveau/nouveau_acpi.c
index 5ffcb6683776..ffb195850314 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.c
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.c
@@ -193,7 +193,7 @@ static int nouveau_dsm_power_state(enum vga_switcheroo_client_id id,
 	return nouveau_dsm_set_discrete_state(nouveau_dsm_priv.dhandle, state);
 }
 
-static int nouveau_dsm_get_client_id(struct pci_dev *pdev)
+static enum vga_switcheroo_client_id nouveau_dsm_get_client_id(struct pci_dev *pdev)
 {
 	/* easy option one - intel vendor ID means Integrated */
 	if (pdev->vendor == PCI_VENDOR_ID_INTEL)
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index ab61c038f42c..7214022dfb91 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -1141,6 +1141,8 @@ nouveau_bo_move_init(struct nouveau_drm *drm)
 			    struct ttm_mem_reg *, struct ttm_mem_reg *);
 		int (*init)(struct nouveau_channel *, u32 handle);
 	} _methods[] = {
+		{  "COPY", 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init },
+		{  "GRCE", 0, 0xc3b5, nve0_bo_move_copy, nvc0_bo_move_init },
 		{  "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init },
 		{  "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init },
 		{  "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init },
diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c
index af1116655910..92d3115f96b5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_chan.c
+++ b/drivers/gpu/drm/nouveau/nouveau_chan.c
@@ -163,12 +163,15 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device,
 			return ret;
 		}
 
+		chan->push.addr = chan->push.vma->addr;
+
+		if (device->info.family >= NV_DEVICE_INFO_V0_FERMI)
+			return 0;
+
 		args.target = NV_DMA_V0_TARGET_VM;
 		args.access = NV_DMA_V0_ACCESS_VM;
 		args.start = 0;
 		args.limit = cli->vmm.vmm.limit - 1;
-
-		chan->push.addr = chan->push.vma->addr;
 	} else
 	if (chan->push.buffer->bo.mem.mem_type == TTM_PL_VRAM) {
 		if (device->info.family == NV_DEVICE_INFO_V0_TNT) {
@@ -214,10 +217,11 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device,
 
 static int
 nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device,
-		    u32 engine, struct nouveau_channel **pchan)
+		    u64 runlist, struct nouveau_channel **pchan)
 {
 	struct nouveau_cli *cli = (void *)device->object.client;
-	static const u16 oclasses[] = { PASCAL_CHANNEL_GPFIFO_A,
+	static const u16 oclasses[] = { VOLTA_CHANNEL_GPFIFO_A,
+					PASCAL_CHANNEL_GPFIFO_A,
 					MAXWELL_CHANNEL_GPFIFO_A,
 					KEPLER_CHANNEL_GPFIFO_B,
 					KEPLER_CHANNEL_GPFIFO_A,
@@ -245,9 +249,9 @@ nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device,
 	do {
 		if (oclass[0] >= KEPLER_CHANNEL_GPFIFO_A) {
 			args.kepler.version = 0;
-			args.kepler.engines = engine;
 			args.kepler.ilength = 0x02000;
 			args.kepler.ioffset = 0x10000 + chan->push.addr;
+			args.kepler.runlist = runlist;
 			args.kepler.vmm = nvif_handle(&cli->vmm.vmm.object);
 			size = sizeof(args.kepler);
 		} else
@@ -474,3 +478,28 @@ nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device,
 	cli->base.super = super;
 	return ret;
 }
+
+int
+nouveau_channels_init(struct nouveau_drm *drm)
+{
+	struct {
+		struct nv_device_info_v1 m;
+		struct {
+			struct nv_device_info_v1_data channels;
+		} v;
+	} args = {
+		.m.version = 1,
+		.m.count = sizeof(args.v) / sizeof(args.v.channels),
+		.v.channels.mthd = NV_DEVICE_FIFO_CHANNELS,
+	};
+	struct nvif_object *device = &drm->client.device.object;
+	int ret;
+
+	ret = nvif_object_mthd(device, NV_DEVICE_V0_INFO, &args, sizeof(args));
+	if (ret || args.v.channels.mthd == NV_DEVICE_INFO_INVALID)
+		return -ENODEV;
+
+	drm->chan.nr = args.v.channels.data;
+	drm->chan.context_base = dma_fence_context_alloc(drm->chan.nr);
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.h b/drivers/gpu/drm/nouveau/nouveau_chan.h
index 14607c16a2bd..64454c2ebd90 100644
--- a/drivers/gpu/drm/nouveau/nouveau_chan.h
+++ b/drivers/gpu/drm/nouveau/nouveau_chan.h
@@ -45,6 +45,7 @@ struct nouveau_channel {
 	atomic_t killed;
 };
 
+int nouveau_channels_init(struct nouveau_drm *);
 
 int  nouveau_channel_new(struct nouveau_drm *, struct nvif_device *,
 			 u32 arg0, u32 arg1, struct nouveau_channel **);
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index 6ed9cb053dfa..7b557c354307 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -151,7 +151,7 @@ nouveau_conn_atomic_set_property(struct drm_connector *connector,
 				/* ... except prior to G80, where the code
 				 * doesn't support such things.
 				 */
-				if (disp->disp.oclass < NV50_DISP)
+				if (disp->disp.object.oclass < NV50_DISP)
 					return -EINVAL;
 				break;
 			default:
@@ -260,7 +260,7 @@ nouveau_conn_reset(struct drm_connector *connector)
 	asyc->procamp.color_vibrance = 150;
 	asyc->procamp.vibrant_hue = 90;
 
-	if (nouveau_display(connector->dev)->disp.oclass < NV50_DISP) {
+	if (nouveau_display(connector->dev)->disp.object.oclass < NV50_DISP) {
 		switch (connector->connector_type) {
 		case DRM_MODE_CONNECTOR_LVDS:
 			/* See note in nouveau_conn_atomic_set_property(). */
@@ -314,7 +314,7 @@ nouveau_conn_attach_properties(struct drm_connector *connector)
 	case DRM_MODE_CONNECTOR_TV:
 		break;
 	case DRM_MODE_CONNECTOR_VGA:
-		if (disp->disp.oclass < NV50_DISP)
+		if (disp->disp.object.oclass < NV50_DISP)
 			break; /* Can only scale on DFPs. */
 		/* Fall-through. */
 	default:
@@ -1005,7 +1005,7 @@ get_tmds_link_bandwidth(struct drm_connector *connector, bool hdmi)
 		return 112000;
 }
 
-static int
+static enum drm_mode_status
 nouveau_connector_mode_valid(struct drm_connector *connector,
 			     struct drm_display_mode *mode)
 {
@@ -1321,7 +1321,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
 	}
 
 	/* HDMI 3D support */
-	if ((disp->disp.oclass >= G82_DISP)
+	if ((disp->disp.object.oclass >= G82_DISP)
 	    && ((type == DRM_MODE_CONNECTOR_DisplayPort)
 		|| (type == DRM_MODE_CONNECTOR_eDP)
 		|| (type == DRM_MODE_CONNECTOR_HDMIA)))
@@ -1343,7 +1343,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
 	case DCB_CONNECTOR_LVDS_SPWG:
 	case DCB_CONNECTOR_eDP:
 		/* see note in nouveau_connector_set_property() */
-		if (disp->disp.oclass < NV50_DISP) {
+		if (disp->disp.object.oclass < NV50_DISP) {
 			nv_connector->scaling_mode = DRM_MODE_SCALE_FULLSCREEN;
 			break;
 		}
@@ -1366,8 +1366,8 @@ nouveau_connector_create(struct drm_device *dev, int index)
 		break;
 	}
 
-	ret = nvif_notify_init(&disp->disp, nouveau_connector_hotplug, true,
-			       NV04_DISP_NTFY_CONN,
+	ret = nvif_notify_init(&disp->disp.object, nouveau_connector_hotplug,
+			       true, NV04_DISP_NTFY_CONN,
 			       &(struct nvif_notify_conn_req_v0) {
 				.mask = NVIF_NOTIFY_CONN_V0_ANY,
 				.conn = index,
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 009713404cc4..774b429142bc 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -116,7 +116,7 @@ nouveau_display_scanoutpos_head(struct drm_crtc *crtc, int *vpos, int *hpos,
 	bool ret = false;
 
 	do {
-		ret = nvif_mthd(&disp->disp, 0, &args, sizeof(args));
+		ret = nvif_mthd(&disp->disp.object, 0, &args, sizeof(args));
 		if (ret != 0)
 			return false;
 
@@ -175,7 +175,7 @@ nouveau_display_vblank_init(struct drm_device *dev)
 
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-		ret = nvif_notify_init(&disp->disp,
+		ret = nvif_notify_init(&disp->disp.object,
 				       nouveau_display_vblank_handler, false,
 				       NV04_DISP_NTFY_VBLANK,
 				       &(struct nvif_notify_head_req_v0) {
@@ -338,11 +338,9 @@ static struct nouveau_drm_prop_enum_list dither_depth[] = {
 	if (c) {                                                               \
 		p = drm_property_create(dev, DRM_MODE_PROP_ENUM, n, c);        \
 		l = (list);                                                    \
-		c = 0;                                                         \
 		while (p && l->gen_mask) {                                     \
 			if (l->gen_mask & (1 << (gen))) {                      \
-				drm_property_add_enum(p, c, l->type, l->name); \
-				c++;                                           \
+				drm_property_add_enum(p, l->type, l->name);    \
 			}                                                      \
 			l++;                                                   \
 		}                                                              \
@@ -456,10 +454,10 @@ nouveau_display_create_properties(struct drm_device *dev)
 	struct nouveau_display *disp = nouveau_display(dev);
 	int gen;
 
-	if (disp->disp.oclass < NV50_DISP)
+	if (disp->disp.object.oclass < NV50_DISP)
 		gen = 0;
 	else
-	if (disp->disp.oclass < GF110_DISP)
+	if (disp->disp.object.oclass < GF110_DISP)
 		gen = 1;
 	else
 		gen = 2;
@@ -535,31 +533,10 @@ nouveau_display_create(struct drm_device *dev)
 	drm_kms_helper_poll_disable(dev);
 
 	if (nouveau_modeset != 2 && drm->vbios.dcb.entries) {
-		static const u16 oclass[] = {
-			GP102_DISP,
-			GP100_DISP,
-			GM200_DISP,
-			GM107_DISP,
-			GK110_DISP,
-			GK104_DISP,
-			GF110_DISP,
-			GT214_DISP,
-			GT206_DISP,
-			GT200_DISP,
-			G82_DISP,
-			NV50_DISP,
-			NV04_DISP,
-		};
-		int i;
-
-		for (i = 0, ret = -ENODEV; ret && i < ARRAY_SIZE(oclass); i++) {
-			ret = nvif_object_init(&drm->client.device.object, 0,
-					       oclass[i], NULL, 0, &disp->disp);
-		}
-
+		ret = nvif_disp_ctor(&drm->client.device, 0, &disp->disp);
 		if (ret == 0) {
 			nouveau_display_create_properties(dev);
-			if (disp->disp.oclass < NV50_DISP)
+			if (disp->disp.object.oclass < NV50_DISP)
 				ret = nv04_display_create(dev);
 			else
 				ret = nv50_display_create(dev);
@@ -613,7 +590,7 @@ nouveau_display_destroy(struct drm_device *dev)
 	if (disp->dtor)
 		disp->dtor(dev);
 
-	nvif_object_fini(&disp->disp);
+	nvif_disp_dtor(&disp->disp);
 
 	nouveau_drm(dev)->display = NULL;
 	kfree(disp);
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h
index 270ba56f2756..54aa7c3fa42d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.h
+++ b/drivers/gpu/drm/nouveau/nouveau_display.h
@@ -2,6 +2,7 @@
 #ifndef __NOUVEAU_DISPLAY_H__
 #define __NOUVEAU_DISPLAY_H__
 #include "nouveau_drv.h"
+#include <nvif/disp.h>
 
 struct nouveau_framebuffer {
 	struct drm_framebuffer base;
@@ -38,7 +39,7 @@ struct nouveau_display {
 	int  (*init)(struct drm_device *);
 	void (*fini)(struct drm_device *);
 
-	struct nvif_object disp;
+	struct nvif_disp disp;
 
 	struct drm_property *dithering_mode;
 	struct drm_property *dithering_depth;
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.c b/drivers/gpu/drm/nouveau/nouveau_dma.c
index 10e84f6ca2b7..945afd34138e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.c
@@ -28,6 +28,8 @@
 #include "nouveau_dma.h"
 #include "nouveau_vmm.h"
 
+#include <nvif/user.h>
+
 void
 OUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords)
 {
@@ -80,18 +82,11 @@ READ_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout)
 }
 
 void
-nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo,
-	      int delta, int length)
+nv50_dma_push(struct nouveau_channel *chan, u64 offset, int length)
 {
-	struct nouveau_cli *cli = (void *)chan->user.client;
+	struct nvif_user *user = &chan->drm->client.device.user;
 	struct nouveau_bo *pb = chan->push.buffer;
-	struct nouveau_vma *vma;
 	int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base;
-	u64 offset;
-
-	vma = nouveau_vma_find(bo, &cli->vmm);
-	BUG_ON(!vma);
-	offset = vma->addr + delta;
 
 	BUG_ON(chan->dma.ib_free < 1);
 
@@ -105,6 +100,8 @@ nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo,
 	nouveau_bo_rd32(pb, 0);
 
 	nvif_wr32(&chan->user, 0x8c, chan->dma.ib_put);
+	if (user->func && user->func->doorbell)
+		user->func->doorbell(user, chan->chid);
 	chan->dma.ib_free--;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.h b/drivers/gpu/drm/nouveau/nouveau_dma.h
index 74e10b14a7da..fc5e3f41282d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.h
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.h
@@ -31,8 +31,7 @@
 #include "nouveau_chan.h"
 
 int nouveau_dma_wait(struct nouveau_channel *, int slots, int size);
-void nv50_dma_push(struct nouveau_channel *, struct nouveau_bo *,
-		   int delta, int length);
+void nv50_dma_push(struct nouveau_channel *, u64 addr, int length);
 
 /*
  * There's a hw race condition where you can't jump to your PUT offset,
@@ -55,7 +54,6 @@ enum {
 
 	NvSub2D		= 3, /* DO NOT CHANGE - hardcoded for kepler gr fifo */
 	NvSubCopy	= 4, /* DO NOT CHANGE - hardcoded for kepler gr fifo */
-	FermiSw		= 5, /* DO NOT CHANGE (well.. 6/7 will work...) */
 };
 
 /* Object handles - for stuff that's doesn't use handle == oclass. */
@@ -151,7 +149,7 @@ FIRE_RING(struct nouveau_channel *chan)
 	chan->accel_done = true;
 
 	if (chan->dma.ib_max) {
-		nv50_dma_push(chan, chan->push.buffer, chan->dma.put << 2,
+		nv50_dma_push(chan, chan->push.addr + (chan->dma.put << 2),
 			      (chan->dma.cur - chan->dma.put) << 2);
 	} else {
 		WRITE_PUT(chan->dma.cur);
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index bbbf353682e1..775443c9af94 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -38,6 +38,8 @@
 #include <core/tegra.h>
 
 #include <nvif/driver.h>
+#include <nvif/fifo.h>
+#include <nvif/user.h>
 
 #include <nvif/class.h>
 #include <nvif/cl0002.h>
@@ -112,24 +114,22 @@ nouveau_name(struct drm_device *dev)
 }
 
 static inline bool
-nouveau_cli_work_ready(struct dma_fence *fence, bool wait)
+nouveau_cli_work_ready(struct dma_fence *fence)
 {
-	if (!dma_fence_is_signaled(fence)) {
-		if (!wait)
-			return false;
-		WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0);
-	}
+	if (!dma_fence_is_signaled(fence))
+		return false;
 	dma_fence_put(fence);
 	return true;
 }
 
 static void
-nouveau_cli_work_flush(struct nouveau_cli *cli, bool wait)
+nouveau_cli_work(struct work_struct *w)
 {
+	struct nouveau_cli *cli = container_of(w, typeof(*cli), work);
 	struct nouveau_cli_work *work, *wtmp;
 	mutex_lock(&cli->lock);
 	list_for_each_entry_safe(work, wtmp, &cli->worker, head) {
-		if (!work->fence || nouveau_cli_work_ready(work->fence, wait)) {
+		if (!work->fence || nouveau_cli_work_ready(work->fence)) {
 			list_del(&work->head);
 			work->func(work);
 		}
@@ -157,17 +157,17 @@ nouveau_cli_work_queue(struct nouveau_cli *cli, struct dma_fence *fence,
 	mutex_unlock(&cli->lock);
 }
 
-static void
-nouveau_cli_work(struct work_struct *w)
-{
-	struct nouveau_cli *cli = container_of(w, typeof(*cli), work);
-	nouveau_cli_work_flush(cli, false);
-}
-
 static void
 nouveau_cli_fini(struct nouveau_cli *cli)
 {
-	nouveau_cli_work_flush(cli, true);
+	/* All our channels are dead now, which means all the fences they
+	 * own are signalled, and all callback functions have been called.
+	 *
+	 * So, after flushing the workqueue, there should be nothing left.
+	 */
+	flush_work(&cli->work);
+	WARN_ON(!list_empty(&cli->worker));
+
 	usif_client_fini(cli);
 	nouveau_vmm_fini(&cli->vmm);
 	nvif_mmu_fini(&cli->mmu);
@@ -307,6 +307,16 @@ nouveau_accel_init(struct nouveau_drm *drm)
 	if (nouveau_noaccel)
 		return;
 
+	ret = nouveau_channels_init(drm);
+	if (ret)
+		return;
+
+	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_VOLTA) {
+		ret = nvif_user_init(device);
+		if (ret)
+			return;
+	}
+
 	/* initialise synchronisation routines */
 	/*XXX: this is crap, but the fence/channel stuff is a little
 	 *     backwards in some places.  this will be fixed.
@@ -338,6 +348,7 @@ nouveau_accel_init(struct nouveau_drm *drm)
 		case KEPLER_CHANNEL_GPFIFO_B:
 		case MAXWELL_CHANNEL_GPFIFO_A:
 		case PASCAL_CHANNEL_GPFIFO_A:
+		case VOLTA_CHANNEL_GPFIFO_A:
 			ret = nvc0_fence_create(drm);
 			break;
 		default:
@@ -354,13 +365,12 @@ nouveau_accel_init(struct nouveau_drm *drm)
 
 	if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
 		ret = nouveau_channel_new(drm, &drm->client.device,
-					  NVA06F_V0_ENGINE_CE0 |
-					  NVA06F_V0_ENGINE_CE1,
-					  0, &drm->cechan);
+					  nvif_fifo_runlist_ce(device), 0,
+					  &drm->cechan);
 		if (ret)
 			NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
 
-		arg0 = NVA06F_V0_ENGINE_GR;
+		arg0 = nvif_fifo_runlist(device, NV_DEVICE_INFO_ENGINE_GR);
 		arg1 = 1;
 	} else
 	if (device->info.chipset >= 0xa3 &&
@@ -386,38 +396,36 @@ nouveau_accel_init(struct nouveau_drm *drm)
 		return;
 	}
 
-	ret = nvif_object_init(&drm->channel->user, NVDRM_NVSW,
-			       nouveau_abi16_swclass(drm), NULL, 0, &drm->nvsw);
-	if (ret == 0) {
-		ret = RING_SPACE(drm->channel, 2);
+	if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
+		ret = nvif_object_init(&drm->channel->user, NVDRM_NVSW,
+				       nouveau_abi16_swclass(drm), NULL, 0,
+				       &drm->nvsw);
 		if (ret == 0) {
-			if (device->info.family < NV_DEVICE_INFO_V0_FERMI) {
+			ret = RING_SPACE(drm->channel, 2);
+			if (ret == 0) {
 				BEGIN_NV04(drm->channel, NvSubSw, 0, 1);
-				OUT_RING  (drm->channel, NVDRM_NVSW);
-			} else
-			if (device->info.family < NV_DEVICE_INFO_V0_KEPLER) {
-				BEGIN_NVC0(drm->channel, FermiSw, 0, 1);
-				OUT_RING  (drm->channel, 0x001f0000);
+				OUT_RING  (drm->channel, drm->nvsw.handle);
+			}
+
+			ret = nvif_notify_init(&drm->nvsw,
+					       nouveau_flip_complete,
+					       false, NV04_NVSW_NTFY_UEVENT,
+					       NULL, 0, 0, &drm->flip);
+			if (ret == 0)
+				ret = nvif_notify_get(&drm->flip);
+			if (ret) {
+				nouveau_accel_fini(drm);
+				return;
 			}
 		}
 
-		ret = nvif_notify_init(&drm->nvsw, nouveau_flip_complete,
-				       false, NV04_NVSW_NTFY_UEVENT,
-				       NULL, 0, 0, &drm->flip);
-		if (ret == 0)
-			ret = nvif_notify_get(&drm->flip);
 		if (ret) {
+			NV_ERROR(drm, "failed to allocate sw class, %d\n", ret);
 			nouveau_accel_fini(drm);
 			return;
 		}
 	}
 
-	if (ret) {
-		NV_ERROR(drm, "failed to allocate software object, %d\n", ret);
-		nouveau_accel_fini(drm);
-		return;
-	}
-
 	if (device->info.family < NV_DEVICE_INFO_V0_FERMI) {
 		ret = nvkm_gpuobj_new(nvxx_device(&drm->client.device), 32, 0,
 				      false, NULL, &drm->notify);
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h
index 881b44b89a01..6e1acaec3400 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.h
@@ -170,6 +170,12 @@ struct nouveau_drm {
 	/* synchronisation */
 	void *fence;
 
+	/* Global channel management. */
+	struct {
+		int nr;
+		u64 context_base;
+	} chan;
+
 	/* context for accelerated drm-internal operations */
 	struct nouveau_channel *cechan;
 	struct nouveau_channel *channel;
diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h
index e28d966946a1..3517f920bf89 100644
--- a/drivers/gpu/drm/nouveau/nouveau_encoder.h
+++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h
@@ -32,6 +32,7 @@
 #include <drm/drm_encoder_slave.h>
 #include <drm/drm_dp_mst_helper.h>
 #include "dispnv04/disp.h"
+struct nv50_head_atom;
 
 #define NV_DPMS_CLEARED 0x80
 
@@ -68,7 +69,7 @@ struct nouveau_encoder {
 	void (*enc_save)(struct drm_encoder *encoder);
 	void (*enc_restore)(struct drm_encoder *encoder);
 	void (*update)(struct nouveau_encoder *, u8 head,
-		       struct drm_display_mode *, u8 proto, u8 depth);
+		       struct nv50_head_atom *, u8 proto, u8 depth);
 };
 
 struct nouveau_encoder *
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index 503fa94dc06d..412d49bc6e56 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -74,15 +74,14 @@ nouveau_fence_signal(struct nouveau_fence *fence)
 }
 
 static struct nouveau_fence *
-nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm) {
-	struct nouveau_fence_priv *priv = (void*)drm->fence;
-
+nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm)
+{
 	if (fence->ops != &nouveau_fence_ops_legacy &&
 	    fence->ops != &nouveau_fence_ops_uevent)
 		return NULL;
 
-	if (fence->context < priv->context_base ||
-	    fence->context >= priv->context_base + priv->contexts)
+	if (fence->context < drm->chan.context_base ||
+	    fence->context >= drm->chan.context_base + drm->chan.nr)
 		return NULL;
 
 	return from_fence(fence);
@@ -176,7 +175,7 @@ nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_cha
 	INIT_LIST_HEAD(&fctx->flip);
 	INIT_LIST_HEAD(&fctx->pending);
 	spin_lock_init(&fctx->lock);
-	fctx->context = priv->context_base + chan->chid;
+	fctx->context = chan->drm->chan.context_base + chan->chid;
 
 	if (chan == chan->drm->cechan)
 		strcpy(fctx->name, "copy engine channel");
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h
index 5bd8d30d1657..b999e6058046 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.h
@@ -55,8 +55,6 @@ struct nouveau_fence_priv {
 	int  (*context_new)(struct nouveau_channel *);
 	void (*context_del)(struct nouveau_channel *);
 
-	u32 contexts;
-	u64 context_base;
 	bool uevent;
 };
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index e72a7e37eb0a..300daee74209 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -99,6 +99,7 @@ struct nouveau_gem_object_unmap {
 static void
 nouveau_gem_object_delete(struct nouveau_vma *vma)
 {
+	nouveau_fence_unref(&vma->fence);
 	nouveau_vma_del(&vma);
 }
 
@@ -114,25 +115,12 @@ nouveau_gem_object_delete_work(struct nouveau_cli_work *w)
 static void
 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
 {
-	const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM;
-	struct reservation_object *resv = nvbo->bo.resv;
-	struct reservation_object_list *fobj;
+	struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL;
 	struct nouveau_gem_object_unmap *work;
-	struct dma_fence *fence = NULL;
-
-	fobj = reservation_object_get_list(resv);
 
 	list_del_init(&vma->head);
 
-	if (fobj && fobj->shared_count > 1)
-		ttm_bo_wait(&nvbo->bo, false, false);
-	else if (fobj && fobj->shared_count == 1)
-		fence = rcu_dereference_protected(fobj->shared[0],
-						reservation_object_held(resv));
-	else
-		fence = reservation_object_get_excl(nvbo->bo.resv);
-
-	if (!fence || !mapped) {
+	if (!fence) {
 		nouveau_gem_object_delete(vma);
 		return;
 	}
@@ -344,9 +332,20 @@ validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence,
 		nvbo = list_entry(op->list.next, struct nouveau_bo, entry);
 		b = &pbbo[nvbo->pbbo_index];
 
-		if (likely(fence))
+		if (likely(fence)) {
+			struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
+			struct nouveau_vma *vma;
+
 			nouveau_bo_fence(nvbo, fence, !!b->write_domains);
 
+			if (drm->client.vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
+				vma = (void *)(unsigned long)b->user_priv;
+				nouveau_fence_unref(&vma->fence);
+				dma_fence_get(&fence->base);
+				vma->fence = fence;
+			}
+		}
+
 		if (unlikely(nvbo->validate_mapped)) {
 			ttm_bo_kunmap(&nvbo->kmap);
 			nvbo->validate_mapped = false;
@@ -432,7 +431,20 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
 			}
 		}
 
-		b->user_priv = (uint64_t)(unsigned long)nvbo;
+		if (cli->vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
+			struct nouveau_vmm *vmm = &cli->vmm;
+			struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm);
+			if (!vma) {
+				NV_PRINTK(err, cli, "vma not found!\n");
+				ret = -EINVAL;
+				break;
+			}
+
+			b->user_priv = (uint64_t)(unsigned long)vma;
+		} else {
+			b->user_priv = (uint64_t)(unsigned long)nvbo;
+		}
+
 		nvbo->reserved_by = file_priv;
 		nvbo->pbbo_index = i;
 		if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
@@ -763,10 +775,10 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
 		}
 
 		for (i = 0; i < req->nr_push; i++) {
-			struct nouveau_bo *nvbo = (void *)(unsigned long)
+			struct nouveau_vma *vma = (void *)(unsigned long)
 				bo[push[i].bo_index].user_priv;
 
-			nv50_dma_push(chan, nvbo, push[i].offset,
+			nv50_dma_push(chan, vma->addr + push[i].offset,
 				      push[i].length);
 		}
 	} else
diff --git a/drivers/gpu/drm/nouveau/nouveau_hwmon.c b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
index 7c965648df80..44178b4c3599 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hwmon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
@@ -327,7 +327,7 @@ nouveau_temp_is_visible(const void *data, u32 attr, int channel)
 	struct nouveau_drm *drm = nouveau_drm((struct drm_device *)data);
 	struct nvkm_therm *therm = nvxx_therm(&drm->client.device);
 
-	if (therm && therm->attr_get && nvkm_therm_temp_get(therm) < 0)
+	if (!therm || !therm->attr_get || nvkm_therm_temp_get(therm) < 0)
 		return 0;
 
 	switch (attr) {
@@ -351,8 +351,8 @@ nouveau_pwm_is_visible(const void *data, u32 attr, int channel)
 	struct nouveau_drm *drm = nouveau_drm((struct drm_device *)data);
 	struct nvkm_therm *therm = nvxx_therm(&drm->client.device);
 
-	if (therm && therm->attr_get && therm->fan_get &&
-				therm->fan_get(therm) < 0)
+	if (!therm || !therm->attr_get || !therm->fan_get ||
+	    therm->fan_get(therm) < 0)
 		return 0;
 
 	switch (attr) {
@@ -707,13 +707,20 @@ nouveau_hwmon_init(struct drm_device *dev)
 {
 #if defined(CONFIG_HWMON) || (defined(MODULE) && defined(CONFIG_HWMON_MODULE))
 	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvkm_iccsense *iccsense = nvxx_iccsense(&drm->client.device);
 	struct nvkm_therm *therm = nvxx_therm(&drm->client.device);
+	struct nvkm_volt *volt = nvxx_volt(&drm->client.device);
 	const struct attribute_group *special_groups[N_ATTR_GROUPS];
 	struct nouveau_hwmon *hwmon;
 	struct device *hwmon_dev;
 	int ret = 0;
 	int i = 0;
 
+	if (!iccsense && !therm && !volt) {
+		NV_DEBUG(drm, "Skipping hwmon registration\n");
+		return 0;
+	}
+
 	hwmon = drm->hwmon = kzalloc(sizeof(*hwmon), GFP_KERNEL);
 	if (!hwmon)
 		return -ENOMEM;
@@ -749,6 +756,9 @@ nouveau_hwmon_fini(struct drm_device *dev)
 #if defined(CONFIG_HWMON) || (defined(MODULE) && defined(CONFIG_HWMON_MODULE))
 	struct nouveau_hwmon *hwmon = nouveau_hwmon(dev);
 
+	if (!hwmon)
+		return;
+
 	if (hwmon->hwmon)
 		hwmon_device_unregister(hwmon->hwmon);
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.c b/drivers/gpu/drm/nouveau/nouveau_vmm.c
index f5371d96b003..2032c3e4f6e5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vmm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_vmm.c
@@ -92,6 +92,7 @@ nouveau_vma_new(struct nouveau_bo *nvbo, struct nouveau_vmm *vmm,
 	vma->refs = 1;
 	vma->addr = ~0ULL;
 	vma->mem = NULL;
+	vma->fence = NULL;
 	list_add_tail(&vma->head, &nvbo->vma_list);
 
 	if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM &&
diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.h b/drivers/gpu/drm/nouveau/nouveau_vmm.h
index 5c31f43678d3..7e3b118cf7c4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vmm.h
+++ b/drivers/gpu/drm/nouveau/nouveau_vmm.h
@@ -11,6 +11,8 @@ struct nouveau_vma {
 	u64 addr;
 
 	struct nouveau_mem *mem;
+
+	struct nouveau_fence *fence;
 };
 
 struct nouveau_vma *nouveau_vma_find(struct nouveau_bo *, struct nouveau_vmm *);
diff --git a/drivers/gpu/drm/nouveau/nv04_fence.c b/drivers/gpu/drm/nouveau/nv04_fence.c
index fa8f2375c398..c41e82be4893 100644
--- a/drivers/gpu/drm/nouveau/nv04_fence.c
+++ b/drivers/gpu/drm/nouveau/nv04_fence.c
@@ -109,7 +109,5 @@ nv04_fence_create(struct nouveau_drm *drm)
 	priv->base.dtor = nv04_fence_destroy;
 	priv->base.context_new = nv04_fence_context_new;
 	priv->base.context_del = nv04_fence_context_del;
-	priv->base.contexts = 15;
-	priv->base.context_base = dma_fence_context_alloc(priv->base.contexts);
 	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nv10_fence.c b/drivers/gpu/drm/nouveau/nv10_fence.c
index 2998bde29211..4476b712dc84 100644
--- a/drivers/gpu/drm/nouveau/nv10_fence.c
+++ b/drivers/gpu/drm/nouveau/nv10_fence.c
@@ -103,8 +103,6 @@ nv10_fence_create(struct nouveau_drm *drm)
 	priv->base.dtor = nv10_fence_destroy;
 	priv->base.context_new = nv10_fence_context_new;
 	priv->base.context_del = nv10_fence_context_del;
-	priv->base.contexts = 31;
-	priv->base.context_base = dma_fence_context_alloc(priv->base.contexts);
 	spin_lock_init(&priv->lock);
 	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nv17_fence.c b/drivers/gpu/drm/nouveau/nv17_fence.c
index 6477b7069e14..5d613d43b84d 100644
--- a/drivers/gpu/drm/nouveau/nv17_fence.c
+++ b/drivers/gpu/drm/nouveau/nv17_fence.c
@@ -125,8 +125,6 @@ nv17_fence_create(struct nouveau_drm *drm)
 	priv->base.resume = nv17_fence_resume;
 	priv->base.context_new = nv17_fence_context_new;
 	priv->base.context_del = nv10_fence_context_del;
-	priv->base.contexts = 31;
-	priv->base.context_base = dma_fence_context_alloc(priv->base.contexts);
 	spin_lock_init(&priv->lock);
 
 	ret = nouveau_bo_new(&drm->client, 4096, 0x1000, TTM_PL_FLAG_VRAM,
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
deleted file mode 100644
index 2b3ccd850750..000000000000
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ /dev/null
@@ -1,4558 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <linux/dma-mapping.h>
-#include <linux/hdmi.h>
-
-#include <drm/drmP.h>
-#include <drm/drm_atomic.h>
-#include <drm/drm_atomic_helper.h>
-#include <drm/drm_crtc_helper.h>
-#include <drm/drm_dp_helper.h>
-#include <drm/drm_fb_helper.h>
-#include <drm/drm_plane_helper.h>
-#include <drm/drm_edid.h>
-
-#include <nvif/class.h>
-#include <nvif/cl0002.h>
-#include <nvif/cl5070.h>
-#include <nvif/cl507a.h>
-#include <nvif/cl507b.h>
-#include <nvif/cl507c.h>
-#include <nvif/cl507d.h>
-#include <nvif/cl507e.h>
-#include <nvif/event.h>
-
-#include "nouveau_drv.h"
-#include "nouveau_dma.h"
-#include "nouveau_gem.h"
-#include "nouveau_connector.h"
-#include "nouveau_encoder.h"
-#include "nouveau_crtc.h"
-#include "nouveau_fence.h"
-#include "nouveau_fbcon.h"
-#include "nv50_display.h"
-
-#define EVO_DMA_NR 9
-
-#define EVO_MASTER  (0x00)
-#define EVO_FLIP(c) (0x01 + (c))
-#define EVO_OVLY(c) (0x05 + (c))
-#define EVO_OIMM(c) (0x09 + (c))
-#define EVO_CURS(c) (0x0d + (c))
-
-/* offsets in shared sync bo of various structures */
-#define EVO_SYNC(c, o) ((c) * 0x0100 + (o))
-#define EVO_MAST_NTFY     EVO_SYNC(      0, 0x00)
-#define EVO_FLIP_SEM0(c)  EVO_SYNC((c) + 1, 0x00)
-#define EVO_FLIP_SEM1(c)  EVO_SYNC((c) + 1, 0x10)
-#define EVO_FLIP_NTFY0(c) EVO_SYNC((c) + 1, 0x20)
-#define EVO_FLIP_NTFY1(c) EVO_SYNC((c) + 1, 0x30)
-
-/******************************************************************************
- * Atomic state
- *****************************************************************************/
-#define nv50_atom(p) container_of((p), struct nv50_atom, state)
-
-struct nv50_atom {
-	struct drm_atomic_state state;
-
-	struct list_head outp;
-	bool lock_core;
-	bool flush_disable;
-};
-
-struct nv50_outp_atom {
-	struct list_head head;
-
-	struct drm_encoder *encoder;
-	bool flush_disable;
-
-	union {
-		struct {
-			bool ctrl:1;
-		};
-		u8 mask;
-	} clr;
-
-	union {
-		struct {
-			bool ctrl:1;
-		};
-		u8 mask;
-	} set;
-};
-
-#define nv50_head_atom(p) container_of((p), struct nv50_head_atom, state)
-
-struct nv50_head_atom {
-	struct drm_crtc_state state;
-
-	struct {
-		u16 iW;
-		u16 iH;
-		u16 oW;
-		u16 oH;
-	} view;
-
-	struct nv50_head_mode {
-		bool interlace;
-		u32 clock;
-		struct {
-			u16 active;
-			u16 synce;
-			u16 blanke;
-			u16 blanks;
-		} h;
-		struct {
-			u32 active;
-			u16 synce;
-			u16 blanke;
-			u16 blanks;
-			u16 blank2s;
-			u16 blank2e;
-			u16 blankus;
-		} v;
-	} mode;
-
-	struct {
-		bool visible;
-		u32 handle;
-		u64 offset:40;
-		u8  mode:4;
-	} lut;
-
-	struct {
-		bool visible;
-		u32 handle;
-		u64 offset:40;
-		u8  format;
-		u8  kind:7;
-		u8  layout:1;
-		u8  block:4;
-		u32 pitch:20;
-		u16 x;
-		u16 y;
-		u16 w;
-		u16 h;
-	} core;
-
-	struct {
-		bool visible;
-		u32 handle;
-		u64 offset:40;
-		u8  layout:1;
-		u8  format:1;
-	} curs;
-
-	struct {
-		u8  depth;
-		u8  cpp;
-		u16 x;
-		u16 y;
-		u16 w;
-		u16 h;
-	} base;
-
-	struct {
-		u8 cpp;
-	} ovly;
-
-	struct {
-		bool enable:1;
-		u8 bits:2;
-		u8 mode:4;
-	} dither;
-
-	struct {
-		struct {
-			u16 cos:12;
-			u16 sin:12;
-		} sat;
-	} procamp;
-
-	union {
-		struct {
-			bool ilut:1;
-			bool core:1;
-			bool curs:1;
-		};
-		u8 mask;
-	} clr;
-
-	union {
-		struct {
-			bool ilut:1;
-			bool core:1;
-			bool curs:1;
-			bool view:1;
-			bool mode:1;
-			bool base:1;
-			bool ovly:1;
-			bool dither:1;
-			bool procamp:1;
-		};
-		u16 mask;
-	} set;
-};
-
-static inline struct nv50_head_atom *
-nv50_head_atom_get(struct drm_atomic_state *state, struct drm_crtc *crtc)
-{
-	struct drm_crtc_state *statec = drm_atomic_get_crtc_state(state, crtc);
-	if (IS_ERR(statec))
-		return (void *)statec;
-	return nv50_head_atom(statec);
-}
-
-#define nv50_wndw_atom(p) container_of((p), struct nv50_wndw_atom, state)
-
-struct nv50_wndw_atom {
-	struct drm_plane_state state;
-	u8 interval;
-
-	struct {
-		u32  handle;
-		u16  offset:12;
-		bool awaken:1;
-	} ntfy;
-
-	struct {
-		u32 handle;
-		u16 offset:12;
-		u32 acquire;
-		u32 release;
-	} sema;
-
-	struct {
-		u8 enable:2;
-	} lut;
-
-	struct {
-		u8  mode:2;
-		u8  interval:4;
-
-		u8  format;
-		u8  kind:7;
-		u8  layout:1;
-		u8  block:4;
-		u32 pitch:20;
-		u16 w;
-		u16 h;
-
-		u32 handle;
-		u64 offset;
-	} image;
-
-	struct {
-		u16 x;
-		u16 y;
-	} point;
-
-	union {
-		struct {
-			bool ntfy:1;
-			bool sema:1;
-			bool image:1;
-		};
-		u8 mask;
-	} clr;
-
-	union {
-		struct {
-			bool ntfy:1;
-			bool sema:1;
-			bool image:1;
-			bool lut:1;
-			bool point:1;
-		};
-		u8 mask;
-	} set;
-};
-
-/******************************************************************************
- * EVO channel
- *****************************************************************************/
-
-struct nv50_chan {
-	struct nvif_object user;
-	struct nvif_device *device;
-};
-
-static int
-nv50_chan_create(struct nvif_device *device, struct nvif_object *disp,
-		 const s32 *oclass, u8 head, void *data, u32 size,
-		 struct nv50_chan *chan)
-{
-	struct nvif_sclass *sclass;
-	int ret, i, n;
-
-	chan->device = device;
-
-	ret = n = nvif_object_sclass_get(disp, &sclass);
-	if (ret < 0)
-		return ret;
-
-	while (oclass[0]) {
-		for (i = 0; i < n; i++) {
-			if (sclass[i].oclass == oclass[0]) {
-				ret = nvif_object_init(disp, 0, oclass[0],
-						       data, size, &chan->user);
-				if (ret == 0)
-					nvif_object_map(&chan->user, NULL, 0);
-				nvif_object_sclass_put(&sclass);
-				return ret;
-			}
-		}
-		oclass++;
-	}
-
-	nvif_object_sclass_put(&sclass);
-	return -ENOSYS;
-}
-
-static void
-nv50_chan_destroy(struct nv50_chan *chan)
-{
-	nvif_object_fini(&chan->user);
-}
-
-/******************************************************************************
- * PIO EVO channel
- *****************************************************************************/
-
-struct nv50_pioc {
-	struct nv50_chan base;
-};
-
-static void
-nv50_pioc_destroy(struct nv50_pioc *pioc)
-{
-	nv50_chan_destroy(&pioc->base);
-}
-
-static int
-nv50_pioc_create(struct nvif_device *device, struct nvif_object *disp,
-		 const s32 *oclass, u8 head, void *data, u32 size,
-		 struct nv50_pioc *pioc)
-{
-	return nv50_chan_create(device, disp, oclass, head, data, size,
-				&pioc->base);
-}
-
-/******************************************************************************
- * Overlay Immediate
- *****************************************************************************/
-
-struct nv50_oimm {
-	struct nv50_pioc base;
-};
-
-static int
-nv50_oimm_create(struct nvif_device *device, struct nvif_object *disp,
-		 int head, struct nv50_oimm *oimm)
-{
-	struct nv50_disp_cursor_v0 args = {
-		.head = head,
-	};
-	static const s32 oclass[] = {
-		GK104_DISP_OVERLAY,
-		GF110_DISP_OVERLAY,
-		GT214_DISP_OVERLAY,
-		G82_DISP_OVERLAY,
-		NV50_DISP_OVERLAY,
-		0
-	};
-
-	return nv50_pioc_create(device, disp, oclass, head, &args, sizeof(args),
-				&oimm->base);
-}
-
-/******************************************************************************
- * DMA EVO channel
- *****************************************************************************/
-
-struct nv50_dmac_ctxdma {
-	struct list_head head;
-	struct nvif_object object;
-};
-
-struct nv50_dmac {
-	struct nv50_chan base;
-	dma_addr_t handle;
-	u32 *ptr;
-
-	struct nvif_object sync;
-	struct nvif_object vram;
-	struct list_head ctxdma;
-
-	/* Protects against concurrent pushbuf access to this channel, lock is
-	 * grabbed by evo_wait (if the pushbuf reservation is successful) and
-	 * dropped again by evo_kick. */
-	struct mutex lock;
-};
-
-static void
-nv50_dmac_ctxdma_del(struct nv50_dmac_ctxdma *ctxdma)
-{
-	nvif_object_fini(&ctxdma->object);
-	list_del(&ctxdma->head);
-	kfree(ctxdma);
-}
-
-static struct nv50_dmac_ctxdma *
-nv50_dmac_ctxdma_new(struct nv50_dmac *dmac, struct nouveau_framebuffer *fb)
-{
-	struct nouveau_drm *drm = nouveau_drm(fb->base.dev);
-	struct nv50_dmac_ctxdma *ctxdma;
-	const u8    kind = fb->nvbo->kind;
-	const u32 handle = 0xfb000000 | kind;
-	struct {
-		struct nv_dma_v0 base;
-		union {
-			struct nv50_dma_v0 nv50;
-			struct gf100_dma_v0 gf100;
-			struct gf119_dma_v0 gf119;
-		};
-	} args = {};
-	u32 argc = sizeof(args.base);
-	int ret;
-
-	list_for_each_entry(ctxdma, &dmac->ctxdma, head) {
-		if (ctxdma->object.handle == handle)
-			return ctxdma;
-	}
-
-	if (!(ctxdma = kzalloc(sizeof(*ctxdma), GFP_KERNEL)))
-		return ERR_PTR(-ENOMEM);
-	list_add(&ctxdma->head, &dmac->ctxdma);
-
-	args.base.target = NV_DMA_V0_TARGET_VRAM;
-	args.base.access = NV_DMA_V0_ACCESS_RDWR;
-	args.base.start  = 0;
-	args.base.limit  = drm->client.device.info.ram_user - 1;
-
-	if (drm->client.device.info.chipset < 0x80) {
-		args.nv50.part = NV50_DMA_V0_PART_256;
-		argc += sizeof(args.nv50);
-	} else
-	if (drm->client.device.info.chipset < 0xc0) {
-		args.nv50.part = NV50_DMA_V0_PART_256;
-		args.nv50.kind = kind;
-		argc += sizeof(args.nv50);
-	} else
-	if (drm->client.device.info.chipset < 0xd0) {
-		args.gf100.kind = kind;
-		argc += sizeof(args.gf100);
-	} else {
-		args.gf119.page = GF119_DMA_V0_PAGE_LP;
-		args.gf119.kind = kind;
-		argc += sizeof(args.gf119);
-	}
-
-	ret = nvif_object_init(&dmac->base.user, handle, NV_DMA_IN_MEMORY,
-			       &args, argc, &ctxdma->object);
-	if (ret) {
-		nv50_dmac_ctxdma_del(ctxdma);
-		return ERR_PTR(ret);
-	}
-
-	return ctxdma;
-}
-
-static void
-nv50_dmac_destroy(struct nv50_dmac *dmac, struct nvif_object *disp)
-{
-	struct nvif_device *device = dmac->base.device;
-	struct nv50_dmac_ctxdma *ctxdma, *ctxtmp;
-
-	list_for_each_entry_safe(ctxdma, ctxtmp, &dmac->ctxdma, head) {
-		nv50_dmac_ctxdma_del(ctxdma);
-	}
-
-	nvif_object_fini(&dmac->vram);
-	nvif_object_fini(&dmac->sync);
-
-	nv50_chan_destroy(&dmac->base);
-
-	if (dmac->ptr) {
-		struct device *dev = nvxx_device(device)->dev;
-		dma_free_coherent(dev, PAGE_SIZE, dmac->ptr, dmac->handle);
-	}
-}
-
-static int
-nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
-		 const s32 *oclass, u8 head, void *data, u32 size, u64 syncbuf,
-		 struct nv50_dmac *dmac)
-{
-	struct nv50_disp_core_channel_dma_v0 *args = data;
-	struct nvif_object pushbuf;
-	int ret;
-
-	mutex_init(&dmac->lock);
-	INIT_LIST_HEAD(&dmac->ctxdma);
-
-	dmac->ptr = dma_alloc_coherent(nvxx_device(device)->dev, PAGE_SIZE,
-				       &dmac->handle, GFP_KERNEL);
-	if (!dmac->ptr)
-		return -ENOMEM;
-
-	ret = nvif_object_init(&device->object, 0, NV_DMA_FROM_MEMORY,
-			       &(struct nv_dma_v0) {
-					.target = NV_DMA_V0_TARGET_PCI_US,
-					.access = NV_DMA_V0_ACCESS_RD,
-					.start = dmac->handle + 0x0000,
-					.limit = dmac->handle + 0x0fff,
-			       }, sizeof(struct nv_dma_v0), &pushbuf);
-	if (ret)
-		return ret;
-
-	args->pushbuf = nvif_handle(&pushbuf);
-
-	ret = nv50_chan_create(device, disp, oclass, head, data, size,
-			       &dmac->base);
-	nvif_object_fini(&pushbuf);
-	if (ret)
-		return ret;
-
-	ret = nvif_object_init(&dmac->base.user, 0xf0000000, NV_DMA_IN_MEMORY,
-			       &(struct nv_dma_v0) {
-					.target = NV_DMA_V0_TARGET_VRAM,
-					.access = NV_DMA_V0_ACCESS_RDWR,
-					.start = syncbuf + 0x0000,
-					.limit = syncbuf + 0x0fff,
-			       }, sizeof(struct nv_dma_v0),
-			       &dmac->sync);
-	if (ret)
-		return ret;
-
-	ret = nvif_object_init(&dmac->base.user, 0xf0000001, NV_DMA_IN_MEMORY,
-			       &(struct nv_dma_v0) {
-					.target = NV_DMA_V0_TARGET_VRAM,
-					.access = NV_DMA_V0_ACCESS_RDWR,
-					.start = 0,
-					.limit = device->info.ram_user - 1,
-			       }, sizeof(struct nv_dma_v0),
-			       &dmac->vram);
-	if (ret)
-		return ret;
-
-	return ret;
-}
-
-/******************************************************************************
- * Core
- *****************************************************************************/
-
-struct nv50_mast {
-	struct nv50_dmac base;
-};
-
-static int
-nv50_core_create(struct nvif_device *device, struct nvif_object *disp,
-		 u64 syncbuf, struct nv50_mast *core)
-{
-	struct nv50_disp_core_channel_dma_v0 args = {
-		.pushbuf = 0xb0007d00,
-	};
-	static const s32 oclass[] = {
-		GP102_DISP_CORE_CHANNEL_DMA,
-		GP100_DISP_CORE_CHANNEL_DMA,
-		GM200_DISP_CORE_CHANNEL_DMA,
-		GM107_DISP_CORE_CHANNEL_DMA,
-		GK110_DISP_CORE_CHANNEL_DMA,
-		GK104_DISP_CORE_CHANNEL_DMA,
-		GF110_DISP_CORE_CHANNEL_DMA,
-		GT214_DISP_CORE_CHANNEL_DMA,
-		GT206_DISP_CORE_CHANNEL_DMA,
-		GT200_DISP_CORE_CHANNEL_DMA,
-		G82_DISP_CORE_CHANNEL_DMA,
-		NV50_DISP_CORE_CHANNEL_DMA,
-		0
-	};
-
-	return nv50_dmac_create(device, disp, oclass, 0, &args, sizeof(args),
-				syncbuf, &core->base);
-}
-
-/******************************************************************************
- * Base
- *****************************************************************************/
-
-struct nv50_sync {
-	struct nv50_dmac base;
-	u32 addr;
-	u32 data;
-};
-
-static int
-nv50_base_create(struct nvif_device *device, struct nvif_object *disp,
-		 int head, u64 syncbuf, struct nv50_sync *base)
-{
-	struct nv50_disp_base_channel_dma_v0 args = {
-		.pushbuf = 0xb0007c00 | head,
-		.head = head,
-	};
-	static const s32 oclass[] = {
-		GK110_DISP_BASE_CHANNEL_DMA,
-		GK104_DISP_BASE_CHANNEL_DMA,
-		GF110_DISP_BASE_CHANNEL_DMA,
-		GT214_DISP_BASE_CHANNEL_DMA,
-		GT200_DISP_BASE_CHANNEL_DMA,
-		G82_DISP_BASE_CHANNEL_DMA,
-		NV50_DISP_BASE_CHANNEL_DMA,
-		0
-	};
-
-	return nv50_dmac_create(device, disp, oclass, head, &args, sizeof(args),
-				syncbuf, &base->base);
-}
-
-/******************************************************************************
- * Overlay
- *****************************************************************************/
-
-struct nv50_ovly {
-	struct nv50_dmac base;
-};
-
-static int
-nv50_ovly_create(struct nvif_device *device, struct nvif_object *disp,
-		 int head, u64 syncbuf, struct nv50_ovly *ovly)
-{
-	struct nv50_disp_overlay_channel_dma_v0 args = {
-		.pushbuf = 0xb0007e00 | head,
-		.head = head,
-	};
-	static const s32 oclass[] = {
-		GK104_DISP_OVERLAY_CONTROL_DMA,
-		GF110_DISP_OVERLAY_CONTROL_DMA,
-		GT214_DISP_OVERLAY_CHANNEL_DMA,
-		GT200_DISP_OVERLAY_CHANNEL_DMA,
-		G82_DISP_OVERLAY_CHANNEL_DMA,
-		NV50_DISP_OVERLAY_CHANNEL_DMA,
-		0
-	};
-
-	return nv50_dmac_create(device, disp, oclass, head, &args, sizeof(args),
-				syncbuf, &ovly->base);
-}
-
-struct nv50_head {
-	struct nouveau_crtc base;
-	struct {
-		struct nouveau_bo *nvbo[2];
-		int next;
-	} lut;
-	struct nv50_ovly ovly;
-	struct nv50_oimm oimm;
-};
-
-#define nv50_head(c) ((struct nv50_head *)nouveau_crtc(c))
-#define nv50_ovly(c) (&nv50_head(c)->ovly)
-#define nv50_oimm(c) (&nv50_head(c)->oimm)
-#define nv50_chan(c) (&(c)->base.base)
-#define nv50_vers(c) nv50_chan(c)->user.oclass
-
-struct nv50_disp {
-	struct nvif_object *disp;
-	struct nv50_mast mast;
-
-	struct nouveau_bo *sync;
-
-	struct mutex mutex;
-};
-
-static struct nv50_disp *
-nv50_disp(struct drm_device *dev)
-{
-	return nouveau_display(dev)->priv;
-}
-
-#define nv50_mast(d) (&nv50_disp(d)->mast)
-
-/******************************************************************************
- * EVO channel helpers
- *****************************************************************************/
-static u32 *
-evo_wait(void *evoc, int nr)
-{
-	struct nv50_dmac *dmac = evoc;
-	struct nvif_device *device = dmac->base.device;
-	u32 put = nvif_rd32(&dmac->base.user, 0x0000) / 4;
-
-	mutex_lock(&dmac->lock);
-	if (put + nr >= (PAGE_SIZE / 4) - 8) {
-		dmac->ptr[put] = 0x20000000;
-
-		nvif_wr32(&dmac->base.user, 0x0000, 0x00000000);
-		if (nvif_msec(device, 2000,
-			if (!nvif_rd32(&dmac->base.user, 0x0004))
-				break;
-		) < 0) {
-			mutex_unlock(&dmac->lock);
-			pr_err("nouveau: evo channel stalled\n");
-			return NULL;
-		}
-
-		put = 0;
-	}
-
-	return dmac->ptr + put;
-}
-
-static void
-evo_kick(u32 *push, void *evoc)
-{
-	struct nv50_dmac *dmac = evoc;
-	nvif_wr32(&dmac->base.user, 0x0000, (push - dmac->ptr) << 2);
-	mutex_unlock(&dmac->lock);
-}
-
-#define evo_mthd(p, m, s) do {						\
-	const u32 _m = (m), _s = (s);					\
-	if (drm_debug & DRM_UT_KMS)					\
-		pr_err("%04x %d %s\n", _m, _s, __func__);		\
-	*((p)++) = ((_s << 18) | _m);					\
-} while(0)
-
-#define evo_data(p, d) do {						\
-	const u32 _d = (d);						\
-	if (drm_debug & DRM_UT_KMS)					\
-		pr_err("\t%08x\n", _d);					\
-	*((p)++) = _d;							\
-} while(0)
-
-/******************************************************************************
- * Plane
- *****************************************************************************/
-#define nv50_wndw(p) container_of((p), struct nv50_wndw, plane)
-
-struct nv50_wndw {
-	const struct nv50_wndw_func *func;
-	struct nv50_dmac *dmac;
-
-	struct drm_plane plane;
-
-	struct nvif_notify notify;
-	u16 ntfy;
-	u16 sema;
-	u32 data;
-};
-
-struct nv50_wndw_func {
-	void *(*dtor)(struct nv50_wndw *);
-	int (*acquire)(struct nv50_wndw *, struct nv50_wndw_atom *asyw,
-		       struct nv50_head_atom *asyh);
-	void (*release)(struct nv50_wndw *, struct nv50_wndw_atom *asyw,
-			struct nv50_head_atom *asyh);
-	void (*prepare)(struct nv50_wndw *, struct nv50_head_atom *asyh,
-			struct nv50_wndw_atom *asyw);
-
-	void (*sema_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
-	void (*sema_clr)(struct nv50_wndw *);
-	void (*ntfy_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
-	void (*ntfy_clr)(struct nv50_wndw *);
-	int (*ntfy_wait_begun)(struct nv50_wndw *, struct nv50_wndw_atom *);
-	void (*image_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
-	void (*image_clr)(struct nv50_wndw *);
-	void (*lut)(struct nv50_wndw *, struct nv50_wndw_atom *);
-	void (*point)(struct nv50_wndw *, struct nv50_wndw_atom *);
-
-	u32 (*update)(struct nv50_wndw *, u32 interlock);
-};
-
-static int
-nv50_wndw_wait_armed(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
-{
-	if (asyw->set.ntfy)
-		return wndw->func->ntfy_wait_begun(wndw, asyw);
-	return 0;
-}
-
-static u32
-nv50_wndw_flush_clr(struct nv50_wndw *wndw, u32 interlock, bool flush,
-		    struct nv50_wndw_atom *asyw)
-{
-	if (asyw->clr.sema && (!asyw->set.sema || flush))
-		wndw->func->sema_clr(wndw);
-	if (asyw->clr.ntfy && (!asyw->set.ntfy || flush))
-		wndw->func->ntfy_clr(wndw);
-	if (asyw->clr.image && (!asyw->set.image || flush))
-		wndw->func->image_clr(wndw);
-
-	return flush ? wndw->func->update(wndw, interlock) : 0;
-}
-
-static u32
-nv50_wndw_flush_set(struct nv50_wndw *wndw, u32 interlock,
-		    struct nv50_wndw_atom *asyw)
-{
-	if (interlock) {
-		asyw->image.mode = 0;
-		asyw->image.interval = 1;
-	}
-
-	if (asyw->set.sema ) wndw->func->sema_set (wndw, asyw);
-	if (asyw->set.ntfy ) wndw->func->ntfy_set (wndw, asyw);
-	if (asyw->set.image) wndw->func->image_set(wndw, asyw);
-	if (asyw->set.lut  ) wndw->func->lut      (wndw, asyw);
-	if (asyw->set.point) wndw->func->point    (wndw, asyw);
-
-	return wndw->func->update(wndw, interlock);
-}
-
-static void
-nv50_wndw_atomic_check_release(struct nv50_wndw *wndw,
-			       struct nv50_wndw_atom *asyw,
-			       struct nv50_head_atom *asyh)
-{
-	struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev);
-	NV_ATOMIC(drm, "%s release\n", wndw->plane.name);
-	wndw->func->release(wndw, asyw, asyh);
-	asyw->ntfy.handle = 0;
-	asyw->sema.handle = 0;
-}
-
-static int
-nv50_wndw_atomic_check_acquire(struct nv50_wndw *wndw,
-			       struct nv50_wndw_atom *asyw,
-			       struct nv50_head_atom *asyh)
-{
-	struct nouveau_framebuffer *fb = nouveau_framebuffer(asyw->state.fb);
-	struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev);
-	int ret;
-
-	NV_ATOMIC(drm, "%s acquire\n", wndw->plane.name);
-
-	asyw->image.w = fb->base.width;
-	asyw->image.h = fb->base.height;
-	asyw->image.kind = fb->nvbo->kind;
-
-	if (asyh->state.pageflip_flags & DRM_MODE_PAGE_FLIP_ASYNC)
-		asyw->interval = 0;
-	else
-		asyw->interval = 1;
-
-	if (asyw->image.kind) {
-		asyw->image.layout = 0;
-		if (drm->client.device.info.chipset >= 0xc0)
-			asyw->image.block = fb->nvbo->mode >> 4;
-		else
-			asyw->image.block = fb->nvbo->mode;
-		asyw->image.pitch = (fb->base.pitches[0] / 4) << 4;
-	} else {
-		asyw->image.layout = 1;
-		asyw->image.block  = 0;
-		asyw->image.pitch  = fb->base.pitches[0];
-	}
-
-	ret = wndw->func->acquire(wndw, asyw, asyh);
-	if (ret)
-		return ret;
-
-	if (asyw->set.image) {
-		if (!(asyw->image.mode = asyw->interval ? 0 : 1))
-			asyw->image.interval = asyw->interval;
-		else
-			asyw->image.interval = 0;
-	}
-
-	return 0;
-}
-
-static int
-nv50_wndw_atomic_check(struct drm_plane *plane, struct drm_plane_state *state)
-{
-	struct nouveau_drm *drm = nouveau_drm(plane->dev);
-	struct nv50_wndw *wndw = nv50_wndw(plane);
-	struct nv50_wndw_atom *armw = nv50_wndw_atom(wndw->plane.state);
-	struct nv50_wndw_atom *asyw = nv50_wndw_atom(state);
-	struct nv50_head_atom *harm = NULL, *asyh = NULL;
-	bool varm = false, asyv = false, asym = false;
-	int ret;
-
-	NV_ATOMIC(drm, "%s atomic_check\n", plane->name);
-	if (asyw->state.crtc) {
-		asyh = nv50_head_atom_get(asyw->state.state, asyw->state.crtc);
-		if (IS_ERR(asyh))
-			return PTR_ERR(asyh);
-		asym = drm_atomic_crtc_needs_modeset(&asyh->state);
-		asyv = asyh->state.active;
-	}
-
-	if (armw->state.crtc) {
-		harm = nv50_head_atom_get(asyw->state.state, armw->state.crtc);
-		if (IS_ERR(harm))
-			return PTR_ERR(harm);
-		varm = harm->state.crtc->state->active;
-	}
-
-	if (asyv) {
-		asyw->point.x = asyw->state.crtc_x;
-		asyw->point.y = asyw->state.crtc_y;
-		if (memcmp(&armw->point, &asyw->point, sizeof(asyw->point)))
-			asyw->set.point = true;
-
-		ret = nv50_wndw_atomic_check_acquire(wndw, asyw, asyh);
-		if (ret)
-			return ret;
-	} else
-	if (varm) {
-		nv50_wndw_atomic_check_release(wndw, asyw, harm);
-	} else {
-		return 0;
-	}
-
-	if (!asyv || asym) {
-		asyw->clr.ntfy = armw->ntfy.handle != 0;
-		asyw->clr.sema = armw->sema.handle != 0;
-		if (wndw->func->image_clr)
-			asyw->clr.image = armw->image.handle != 0;
-		asyw->set.lut = wndw->func->lut && asyv;
-	}
-
-	return 0;
-}
-
-static void
-nv50_wndw_cleanup_fb(struct drm_plane *plane, struct drm_plane_state *old_state)
-{
-	struct nouveau_framebuffer *fb = nouveau_framebuffer(old_state->fb);
-	struct nouveau_drm *drm = nouveau_drm(plane->dev);
-
-	NV_ATOMIC(drm, "%s cleanup: %p\n", plane->name, old_state->fb);
-	if (!old_state->fb)
-		return;
-
-	nouveau_bo_unpin(fb->nvbo);
-}
-
-static int
-nv50_wndw_prepare_fb(struct drm_plane *plane, struct drm_plane_state *state)
-{
-	struct nouveau_framebuffer *fb = nouveau_framebuffer(state->fb);
-	struct nouveau_drm *drm = nouveau_drm(plane->dev);
-	struct nv50_wndw *wndw = nv50_wndw(plane);
-	struct nv50_wndw_atom *asyw = nv50_wndw_atom(state);
-	struct nv50_head_atom *asyh;
-	struct nv50_dmac_ctxdma *ctxdma;
-	int ret;
-
-	NV_ATOMIC(drm, "%s prepare: %p\n", plane->name, state->fb);
-	if (!asyw->state.fb)
-		return 0;
-
-	ret = nouveau_bo_pin(fb->nvbo, TTM_PL_FLAG_VRAM, true);
-	if (ret)
-		return ret;
-
-	ctxdma = nv50_dmac_ctxdma_new(wndw->dmac, fb);
-	if (IS_ERR(ctxdma)) {
-		nouveau_bo_unpin(fb->nvbo);
-		return PTR_ERR(ctxdma);
-	}
-
-	asyw->state.fence = reservation_object_get_excl_rcu(fb->nvbo->bo.resv);
-	asyw->image.handle = ctxdma->object.handle;
-	asyw->image.offset = fb->nvbo->bo.offset;
-
-	if (wndw->func->prepare) {
-		asyh = nv50_head_atom_get(asyw->state.state, asyw->state.crtc);
-		if (IS_ERR(asyh))
-			return PTR_ERR(asyh);
-
-		wndw->func->prepare(wndw, asyh, asyw);
-	}
-
-	return 0;
-}
-
-static const struct drm_plane_helper_funcs
-nv50_wndw_helper = {
-	.prepare_fb = nv50_wndw_prepare_fb,
-	.cleanup_fb = nv50_wndw_cleanup_fb,
-	.atomic_check = nv50_wndw_atomic_check,
-};
-
-static void
-nv50_wndw_atomic_destroy_state(struct drm_plane *plane,
-			       struct drm_plane_state *state)
-{
-	struct nv50_wndw_atom *asyw = nv50_wndw_atom(state);
-	__drm_atomic_helper_plane_destroy_state(&asyw->state);
-	kfree(asyw);
-}
-
-static struct drm_plane_state *
-nv50_wndw_atomic_duplicate_state(struct drm_plane *plane)
-{
-	struct nv50_wndw_atom *armw = nv50_wndw_atom(plane->state);
-	struct nv50_wndw_atom *asyw;
-	if (!(asyw = kmalloc(sizeof(*asyw), GFP_KERNEL)))
-		return NULL;
-	__drm_atomic_helper_plane_duplicate_state(plane, &asyw->state);
-	asyw->interval = 1;
-	asyw->sema = armw->sema;
-	asyw->ntfy = armw->ntfy;
-	asyw->image = armw->image;
-	asyw->point = armw->point;
-	asyw->lut = armw->lut;
-	asyw->clr.mask = 0;
-	asyw->set.mask = 0;
-	return &asyw->state;
-}
-
-static void
-nv50_wndw_reset(struct drm_plane *plane)
-{
-	struct nv50_wndw_atom *asyw;
-
-	if (WARN_ON(!(asyw = kzalloc(sizeof(*asyw), GFP_KERNEL))))
-		return;
-
-	if (plane->state)
-		plane->funcs->atomic_destroy_state(plane, plane->state);
-	plane->state = &asyw->state;
-	plane->state->plane = plane;
-	plane->state->rotation = DRM_MODE_ROTATE_0;
-}
-
-static void
-nv50_wndw_destroy(struct drm_plane *plane)
-{
-	struct nv50_wndw *wndw = nv50_wndw(plane);
-	void *data;
-	nvif_notify_fini(&wndw->notify);
-	data = wndw->func->dtor(wndw);
-	drm_plane_cleanup(&wndw->plane);
-	kfree(data);
-}
-
-static const struct drm_plane_funcs
-nv50_wndw = {
-	.update_plane = drm_atomic_helper_update_plane,
-	.disable_plane = drm_atomic_helper_disable_plane,
-	.destroy = nv50_wndw_destroy,
-	.reset = nv50_wndw_reset,
-	.atomic_duplicate_state = nv50_wndw_atomic_duplicate_state,
-	.atomic_destroy_state = nv50_wndw_atomic_destroy_state,
-};
-
-static void
-nv50_wndw_fini(struct nv50_wndw *wndw)
-{
-	nvif_notify_put(&wndw->notify);
-}
-
-static void
-nv50_wndw_init(struct nv50_wndw *wndw)
-{
-	nvif_notify_get(&wndw->notify);
-}
-
-static int
-nv50_wndw_ctor(const struct nv50_wndw_func *func, struct drm_device *dev,
-	       enum drm_plane_type type, const char *name, int index,
-	       struct nv50_dmac *dmac, const u32 *format, int nformat,
-	       struct nv50_wndw *wndw)
-{
-	int ret;
-
-	wndw->func = func;
-	wndw->dmac = dmac;
-
-	ret = drm_universal_plane_init(dev, &wndw->plane, 0, &nv50_wndw,
-				       format, nformat, NULL,
-				       type, "%s-%d", name, index);
-	if (ret)
-		return ret;
-
-	drm_plane_helper_add(&wndw->plane, &nv50_wndw_helper);
-	return 0;
-}
-
-/******************************************************************************
- * Cursor plane
- *****************************************************************************/
-#define nv50_curs(p) container_of((p), struct nv50_curs, wndw)
-
-struct nv50_curs {
-	struct nv50_wndw wndw;
-	struct nvif_object chan;
-};
-
-static u32
-nv50_curs_update(struct nv50_wndw *wndw, u32 interlock)
-{
-	struct nv50_curs *curs = nv50_curs(wndw);
-	nvif_wr32(&curs->chan, 0x0080, 0x00000000);
-	return 0;
-}
-
-static void
-nv50_curs_point(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
-{
-	struct nv50_curs *curs = nv50_curs(wndw);
-	nvif_wr32(&curs->chan, 0x0084, (asyw->point.y << 16) | asyw->point.x);
-}
-
-static void
-nv50_curs_prepare(struct nv50_wndw *wndw, struct nv50_head_atom *asyh,
-		  struct nv50_wndw_atom *asyw)
-{
-	u32 handle = nv50_disp(wndw->plane.dev)->mast.base.vram.handle;
-	u32 offset = asyw->image.offset;
-	if (asyh->curs.handle != handle || asyh->curs.offset != offset) {
-		asyh->curs.handle = handle;
-		asyh->curs.offset = offset;
-		asyh->set.curs = asyh->curs.visible;
-	}
-}
-
-static void
-nv50_curs_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
-		  struct nv50_head_atom *asyh)
-{
-	asyh->curs.visible = false;
-}
-
-static int
-nv50_curs_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
-		  struct nv50_head_atom *asyh)
-{
-	int ret;
-
-	ret = drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  true, true);
-	asyh->curs.visible = asyw->state.visible;
-	if (ret || !asyh->curs.visible)
-		return ret;
-
-	switch (asyw->state.fb->width) {
-	case 32: asyh->curs.layout = 0; break;
-	case 64: asyh->curs.layout = 1; break;
-	default:
-		return -EINVAL;
-	}
-
-	if (asyw->state.fb->width != asyw->state.fb->height)
-		return -EINVAL;
-
-	switch (asyw->state.fb->format->format) {
-	case DRM_FORMAT_ARGB8888: asyh->curs.format = 1; break;
-	default:
-		WARN_ON(1);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static void *
-nv50_curs_dtor(struct nv50_wndw *wndw)
-{
-	struct nv50_curs *curs = nv50_curs(wndw);
-	nvif_object_fini(&curs->chan);
-	return curs;
-}
-
-static const u32
-nv50_curs_format[] = {
-	DRM_FORMAT_ARGB8888,
-};
-
-static const struct nv50_wndw_func
-nv50_curs = {
-	.dtor = nv50_curs_dtor,
-	.acquire = nv50_curs_acquire,
-	.release = nv50_curs_release,
-	.prepare = nv50_curs_prepare,
-	.point = nv50_curs_point,
-	.update = nv50_curs_update,
-};
-
-static int
-nv50_curs_new(struct nouveau_drm *drm, struct nv50_head *head,
-	      struct nv50_curs **pcurs)
-{
-	static const struct nvif_mclass curses[] = {
-		{ GK104_DISP_CURSOR, 0 },
-		{ GF110_DISP_CURSOR, 0 },
-		{ GT214_DISP_CURSOR, 0 },
-		{   G82_DISP_CURSOR, 0 },
-		{  NV50_DISP_CURSOR, 0 },
-		{}
-	};
-	struct nv50_disp_cursor_v0 args = {
-		.head = head->base.index,
-	};
-	struct nv50_disp *disp = nv50_disp(drm->dev);
-	struct nv50_curs *curs;
-	int cid, ret;
-
-	cid = nvif_mclass(disp->disp, curses);
-	if (cid < 0) {
-		NV_ERROR(drm, "No supported cursor immediate class\n");
-		return cid;
-	}
-
-	if (!(curs = *pcurs = kzalloc(sizeof(*curs), GFP_KERNEL)))
-		return -ENOMEM;
-
-	ret = nv50_wndw_ctor(&nv50_curs, drm->dev, DRM_PLANE_TYPE_CURSOR,
-			     "curs", head->base.index, &disp->mast.base,
-			     nv50_curs_format, ARRAY_SIZE(nv50_curs_format),
-			     &curs->wndw);
-	if (ret) {
-		kfree(curs);
-		return ret;
-	}
-
-	ret = nvif_object_init(disp->disp, 0, curses[cid].oclass, &args,
-			       sizeof(args), &curs->chan);
-	if (ret) {
-		NV_ERROR(drm, "curs%04x allocation failed: %d\n",
-			 curses[cid].oclass, ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-/******************************************************************************
- * Primary plane
- *****************************************************************************/
-#define nv50_base(p) container_of((p), struct nv50_base, wndw)
-
-struct nv50_base {
-	struct nv50_wndw wndw;
-	struct nv50_sync chan;
-	int id;
-};
-
-static int
-nv50_base_notify(struct nvif_notify *notify)
-{
-	return NVIF_NOTIFY_KEEP;
-}
-
-static void
-nv50_base_lut(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
-{
-	struct nv50_base *base = nv50_base(wndw);
-	u32 *push;
-	if ((push = evo_wait(&base->chan, 2))) {
-		evo_mthd(push, 0x00e0, 1);
-		evo_data(push, asyw->lut.enable << 30);
-		evo_kick(push, &base->chan);
-	}
-}
-
-static void
-nv50_base_image_clr(struct nv50_wndw *wndw)
-{
-	struct nv50_base *base = nv50_base(wndw);
-	u32 *push;
-	if ((push = evo_wait(&base->chan, 4))) {
-		evo_mthd(push, 0x0084, 1);
-		evo_data(push, 0x00000000);
-		evo_mthd(push, 0x00c0, 1);
-		evo_data(push, 0x00000000);
-		evo_kick(push, &base->chan);
-	}
-}
-
-static void
-nv50_base_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
-{
-	struct nv50_base *base = nv50_base(wndw);
-	const s32 oclass = base->chan.base.base.user.oclass;
-	u32 *push;
-	if ((push = evo_wait(&base->chan, 10))) {
-		evo_mthd(push, 0x0084, 1);
-		evo_data(push, (asyw->image.mode << 8) |
-			       (asyw->image.interval << 4));
-		evo_mthd(push, 0x00c0, 1);
-		evo_data(push, asyw->image.handle);
-		if (oclass < G82_DISP_BASE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0800, 5);
-			evo_data(push, asyw->image.offset >> 8);
-			evo_data(push, 0x00000000);
-			evo_data(push, (asyw->image.h << 16) | asyw->image.w);
-			evo_data(push, (asyw->image.layout << 20) |
-					asyw->image.pitch |
-					asyw->image.block);
-			evo_data(push, (asyw->image.kind << 16) |
-				       (asyw->image.format << 8));
-		} else
-		if (oclass < GF110_DISP_BASE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0800, 5);
-			evo_data(push, asyw->image.offset >> 8);
-			evo_data(push, 0x00000000);
-			evo_data(push, (asyw->image.h << 16) | asyw->image.w);
-			evo_data(push, (asyw->image.layout << 20) |
-					asyw->image.pitch |
-					asyw->image.block);
-			evo_data(push, asyw->image.format << 8);
-		} else {
-			evo_mthd(push, 0x0400, 5);
-			evo_data(push, asyw->image.offset >> 8);
-			evo_data(push, 0x00000000);
-			evo_data(push, (asyw->image.h << 16) | asyw->image.w);
-			evo_data(push, (asyw->image.layout << 24) |
-					asyw->image.pitch |
-					asyw->image.block);
-			evo_data(push, asyw->image.format << 8);
-		}
-		evo_kick(push, &base->chan);
-	}
-}
-
-static void
-nv50_base_ntfy_clr(struct nv50_wndw *wndw)
-{
-	struct nv50_base *base = nv50_base(wndw);
-	u32 *push;
-	if ((push = evo_wait(&base->chan, 2))) {
-		evo_mthd(push, 0x00a4, 1);
-		evo_data(push, 0x00000000);
-		evo_kick(push, &base->chan);
-	}
-}
-
-static void
-nv50_base_ntfy_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
-{
-	struct nv50_base *base = nv50_base(wndw);
-	u32 *push;
-	if ((push = evo_wait(&base->chan, 3))) {
-		evo_mthd(push, 0x00a0, 2);
-		evo_data(push, (asyw->ntfy.awaken << 30) | asyw->ntfy.offset);
-		evo_data(push, asyw->ntfy.handle);
-		evo_kick(push, &base->chan);
-	}
-}
-
-static void
-nv50_base_sema_clr(struct nv50_wndw *wndw)
-{
-	struct nv50_base *base = nv50_base(wndw);
-	u32 *push;
-	if ((push = evo_wait(&base->chan, 2))) {
-		evo_mthd(push, 0x0094, 1);
-		evo_data(push, 0x00000000);
-		evo_kick(push, &base->chan);
-	}
-}
-
-static void
-nv50_base_sema_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
-{
-	struct nv50_base *base = nv50_base(wndw);
-	u32 *push;
-	if ((push = evo_wait(&base->chan, 5))) {
-		evo_mthd(push, 0x0088, 4);
-		evo_data(push, asyw->sema.offset);
-		evo_data(push, asyw->sema.acquire);
-		evo_data(push, asyw->sema.release);
-		evo_data(push, asyw->sema.handle);
-		evo_kick(push, &base->chan);
-	}
-}
-
-static u32
-nv50_base_update(struct nv50_wndw *wndw, u32 interlock)
-{
-	struct nv50_base *base = nv50_base(wndw);
-	u32 *push;
-
-	if (!(push = evo_wait(&base->chan, 2)))
-		return 0;
-	evo_mthd(push, 0x0080, 1);
-	evo_data(push, interlock);
-	evo_kick(push, &base->chan);
-
-	if (base->chan.base.base.user.oclass < GF110_DISP_BASE_CHANNEL_DMA)
-		return interlock ? 2 << (base->id * 8) : 0;
-	return interlock ? 2 << (base->id * 4) : 0;
-}
-
-static int
-nv50_base_ntfy_wait_begun(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
-{
-	struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev);
-	struct nv50_disp *disp = nv50_disp(wndw->plane.dev);
-	if (nvif_msec(&drm->client.device, 2000ULL,
-		u32 data = nouveau_bo_rd32(disp->sync, asyw->ntfy.offset / 4);
-		if ((data & 0xc0000000) == 0x40000000)
-			break;
-		usleep_range(1, 2);
-	) < 0)
-		return -ETIMEDOUT;
-	return 0;
-}
-
-static void
-nv50_base_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
-		  struct nv50_head_atom *asyh)
-{
-	asyh->base.cpp = 0;
-}
-
-static int
-nv50_base_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
-		  struct nv50_head_atom *asyh)
-{
-	const struct drm_framebuffer *fb = asyw->state.fb;
-	int ret;
-
-	if (!fb->format->depth)
-		return -EINVAL;
-
-	ret = drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  false, true);
-	if (ret)
-		return ret;
-
-	asyh->base.depth = fb->format->depth;
-	asyh->base.cpp = fb->format->cpp[0];
-	asyh->base.x = asyw->state.src.x1 >> 16;
-	asyh->base.y = asyw->state.src.y1 >> 16;
-	asyh->base.w = asyw->state.fb->width;
-	asyh->base.h = asyw->state.fb->height;
-
-	switch (fb->format->format) {
-	case DRM_FORMAT_C8         : asyw->image.format = 0x1e; break;
-	case DRM_FORMAT_RGB565     : asyw->image.format = 0xe8; break;
-	case DRM_FORMAT_XRGB1555   :
-	case DRM_FORMAT_ARGB1555   : asyw->image.format = 0xe9; break;
-	case DRM_FORMAT_XRGB8888   :
-	case DRM_FORMAT_ARGB8888   : asyw->image.format = 0xcf; break;
-	case DRM_FORMAT_XBGR2101010:
-	case DRM_FORMAT_ABGR2101010: asyw->image.format = 0xd1; break;
-	case DRM_FORMAT_XBGR8888   :
-	case DRM_FORMAT_ABGR8888   : asyw->image.format = 0xd5; break;
-	default:
-		WARN_ON(1);
-		return -EINVAL;
-	}
-
-	asyw->lut.enable = 1;
-	asyw->set.image = true;
-	return 0;
-}
-
-static void *
-nv50_base_dtor(struct nv50_wndw *wndw)
-{
-	struct nv50_disp *disp = nv50_disp(wndw->plane.dev);
-	struct nv50_base *base = nv50_base(wndw);
-	nv50_dmac_destroy(&base->chan.base, disp->disp);
-	return base;
-}
-
-static const u32
-nv50_base_format[] = {
-	DRM_FORMAT_C8,
-	DRM_FORMAT_RGB565,
-	DRM_FORMAT_XRGB1555,
-	DRM_FORMAT_ARGB1555,
-	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_XBGR2101010,
-	DRM_FORMAT_ABGR2101010,
-	DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_ABGR8888,
-};
-
-static const struct nv50_wndw_func
-nv50_base = {
-	.dtor = nv50_base_dtor,
-	.acquire = nv50_base_acquire,
-	.release = nv50_base_release,
-	.sema_set = nv50_base_sema_set,
-	.sema_clr = nv50_base_sema_clr,
-	.ntfy_set = nv50_base_ntfy_set,
-	.ntfy_clr = nv50_base_ntfy_clr,
-	.ntfy_wait_begun = nv50_base_ntfy_wait_begun,
-	.image_set = nv50_base_image_set,
-	.image_clr = nv50_base_image_clr,
-	.lut = nv50_base_lut,
-	.update = nv50_base_update,
-};
-
-static int
-nv50_base_new(struct nouveau_drm *drm, struct nv50_head *head,
-	      struct nv50_base **pbase)
-{
-	struct nv50_disp *disp = nv50_disp(drm->dev);
-	struct nv50_base *base;
-	int ret;
-
-	if (!(base = *pbase = kzalloc(sizeof(*base), GFP_KERNEL)))
-		return -ENOMEM;
-	base->id = head->base.index;
-	base->wndw.ntfy = EVO_FLIP_NTFY0(base->id);
-	base->wndw.sema = EVO_FLIP_SEM0(base->id);
-	base->wndw.data = 0x00000000;
-
-	ret = nv50_wndw_ctor(&nv50_base, drm->dev, DRM_PLANE_TYPE_PRIMARY,
-			     "base", base->id, &base->chan.base,
-			     nv50_base_format, ARRAY_SIZE(nv50_base_format),
-			     &base->wndw);
-	if (ret) {
-		kfree(base);
-		return ret;
-	}
-
-	ret = nv50_base_create(&drm->client.device, disp->disp, base->id,
-			       disp->sync->bo.offset, &base->chan);
-	if (ret)
-		return ret;
-
-	return nvif_notify_init(&base->chan.base.base.user, nv50_base_notify,
-				false,
-				NV50_DISP_BASE_CHANNEL_DMA_V0_NTFY_UEVENT,
-				&(struct nvif_notify_uevent_req) {},
-				sizeof(struct nvif_notify_uevent_req),
-				sizeof(struct nvif_notify_uevent_rep),
-				&base->wndw.notify);
-}
-
-/******************************************************************************
- * Head
- *****************************************************************************/
-static void
-nv50_head_procamp(struct nv50_head *head, struct nv50_head_atom *asyh)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	u32 *push;
-	if ((push = evo_wait(core, 2))) {
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
-			evo_mthd(push, 0x08a8 + (head->base.index * 0x400), 1);
-		else
-			evo_mthd(push, 0x0498 + (head->base.index * 0x300), 1);
-		evo_data(push, (asyh->procamp.sat.sin << 20) |
-			       (asyh->procamp.sat.cos << 8));
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_dither(struct nv50_head *head, struct nv50_head_atom *asyh)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	u32 *push;
-	if ((push = evo_wait(core, 2))) {
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
-			evo_mthd(push, 0x08a0 + (head->base.index * 0x0400), 1);
-		else
-		if (core->base.user.oclass < GK104_DISP_CORE_CHANNEL_DMA)
-			evo_mthd(push, 0x0490 + (head->base.index * 0x0300), 1);
-		else
-			evo_mthd(push, 0x04a0 + (head->base.index * 0x0300), 1);
-		evo_data(push, (asyh->dither.mode << 3) |
-			       (asyh->dither.bits << 1) |
-			        asyh->dither.enable);
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_ovly(struct nv50_head *head, struct nv50_head_atom *asyh)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	u32 bounds = 0;
-	u32 *push;
-
-	if (asyh->base.cpp) {
-		switch (asyh->base.cpp) {
-		case 8: bounds |= 0x00000500; break;
-		case 4: bounds |= 0x00000300; break;
-		case 2: bounds |= 0x00000100; break;
-		default:
-			WARN_ON(1);
-			break;
-		}
-		bounds |= 0x00000001;
-	}
-
-	if ((push = evo_wait(core, 2))) {
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
-			evo_mthd(push, 0x0904 + head->base.index * 0x400, 1);
-		else
-			evo_mthd(push, 0x04d4 + head->base.index * 0x300, 1);
-		evo_data(push, bounds);
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_base(struct nv50_head *head, struct nv50_head_atom *asyh)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	u32 bounds = 0;
-	u32 *push;
-
-	if (asyh->base.cpp) {
-		switch (asyh->base.cpp) {
-		case 8: bounds |= 0x00000500; break;
-		case 4: bounds |= 0x00000300; break;
-		case 2: bounds |= 0x00000100; break;
-		case 1: bounds |= 0x00000000; break;
-		default:
-			WARN_ON(1);
-			break;
-		}
-		bounds |= 0x00000001;
-	}
-
-	if ((push = evo_wait(core, 2))) {
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
-			evo_mthd(push, 0x0900 + head->base.index * 0x400, 1);
-		else
-			evo_mthd(push, 0x04d0 + head->base.index * 0x300, 1);
-		evo_data(push, bounds);
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_curs_clr(struct nv50_head *head)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	u32 *push;
-	if ((push = evo_wait(core, 4))) {
-		if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0880 + head->base.index * 0x400, 1);
-			evo_data(push, 0x05000000);
-		} else
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0880 + head->base.index * 0x400, 1);
-			evo_data(push, 0x05000000);
-			evo_mthd(push, 0x089c + head->base.index * 0x400, 1);
-			evo_data(push, 0x00000000);
-		} else {
-			evo_mthd(push, 0x0480 + head->base.index * 0x300, 1);
-			evo_data(push, 0x05000000);
-			evo_mthd(push, 0x048c + head->base.index * 0x300, 1);
-			evo_data(push, 0x00000000);
-		}
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_curs_set(struct nv50_head *head, struct nv50_head_atom *asyh)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	u32 *push;
-	if ((push = evo_wait(core, 5))) {
-		if (core->base.user.oclass < G82_DISP_BASE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0880 + head->base.index * 0x400, 2);
-			evo_data(push, 0x80000000 | (asyh->curs.layout << 26) |
-						    (asyh->curs.format << 24));
-			evo_data(push, asyh->curs.offset >> 8);
-		} else
-		if (core->base.user.oclass < GF110_DISP_BASE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0880 + head->base.index * 0x400, 2);
-			evo_data(push, 0x80000000 | (asyh->curs.layout << 26) |
-						    (asyh->curs.format << 24));
-			evo_data(push, asyh->curs.offset >> 8);
-			evo_mthd(push, 0x089c + head->base.index * 0x400, 1);
-			evo_data(push, asyh->curs.handle);
-		} else {
-			evo_mthd(push, 0x0480 + head->base.index * 0x300, 2);
-			evo_data(push, 0x80000000 | (asyh->curs.layout << 26) |
-						    (asyh->curs.format << 24));
-			evo_data(push, asyh->curs.offset >> 8);
-			evo_mthd(push, 0x048c + head->base.index * 0x300, 1);
-			evo_data(push, asyh->curs.handle);
-		}
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_core_clr(struct nv50_head *head)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	u32 *push;
-	if ((push = evo_wait(core, 2))) {
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
-			evo_mthd(push, 0x0874 + head->base.index * 0x400, 1);
-		else
-			evo_mthd(push, 0x0474 + head->base.index * 0x300, 1);
-		evo_data(push, 0x00000000);
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_core_set(struct nv50_head *head, struct nv50_head_atom *asyh)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	u32 *push;
-	if ((push = evo_wait(core, 9))) {
-		if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0860 + head->base.index * 0x400, 1);
-			evo_data(push, asyh->core.offset >> 8);
-			evo_mthd(push, 0x0868 + head->base.index * 0x400, 4);
-			evo_data(push, (asyh->core.h << 16) | asyh->core.w);
-			evo_data(push, asyh->core.layout << 20 |
-				       (asyh->core.pitch >> 8) << 8 |
-				       asyh->core.block);
-			evo_data(push, asyh->core.kind << 16 |
-				       asyh->core.format << 8);
-			evo_data(push, asyh->core.handle);
-			evo_mthd(push, 0x08c0 + head->base.index * 0x400, 1);
-			evo_data(push, (asyh->core.y << 16) | asyh->core.x);
-			/* EVO will complain with INVALID_STATE if we have an
-			 * active cursor and (re)specify HeadSetContextDmaIso
-			 * without also updating HeadSetOffsetCursor.
-			 */
-			asyh->set.curs = asyh->curs.visible;
-		} else
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0860 + head->base.index * 0x400, 1);
-			evo_data(push, asyh->core.offset >> 8);
-			evo_mthd(push, 0x0868 + head->base.index * 0x400, 4);
-			evo_data(push, (asyh->core.h << 16) | asyh->core.w);
-			evo_data(push, asyh->core.layout << 20 |
-				       (asyh->core.pitch >> 8) << 8 |
-				       asyh->core.block);
-			evo_data(push, asyh->core.format << 8);
-			evo_data(push, asyh->core.handle);
-			evo_mthd(push, 0x08c0 + head->base.index * 0x400, 1);
-			evo_data(push, (asyh->core.y << 16) | asyh->core.x);
-		} else {
-			evo_mthd(push, 0x0460 + head->base.index * 0x300, 1);
-			evo_data(push, asyh->core.offset >> 8);
-			evo_mthd(push, 0x0468 + head->base.index * 0x300, 4);
-			evo_data(push, (asyh->core.h << 16) | asyh->core.w);
-			evo_data(push, asyh->core.layout << 24 |
-				       (asyh->core.pitch >> 8) << 8 |
-				       asyh->core.block);
-			evo_data(push, asyh->core.format << 8);
-			evo_data(push, asyh->core.handle);
-			evo_mthd(push, 0x04b0 + head->base.index * 0x300, 1);
-			evo_data(push, (asyh->core.y << 16) | asyh->core.x);
-		}
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_lut_clr(struct nv50_head *head)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	u32 *push;
-	if ((push = evo_wait(core, 4))) {
-		if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0840 + (head->base.index * 0x400), 1);
-			evo_data(push, 0x40000000);
-		} else
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0840 + (head->base.index * 0x400), 1);
-			evo_data(push, 0x40000000);
-			evo_mthd(push, 0x085c + (head->base.index * 0x400), 1);
-			evo_data(push, 0x00000000);
-		} else {
-			evo_mthd(push, 0x0440 + (head->base.index * 0x300), 1);
-			evo_data(push, 0x03000000);
-			evo_mthd(push, 0x045c + (head->base.index * 0x300), 1);
-			evo_data(push, 0x00000000);
-		}
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_lut_load(struct drm_property_blob *blob, int mode,
-		   struct nouveau_bo *nvbo)
-{
-	struct drm_color_lut *in = (struct drm_color_lut *)blob->data;
-	void __iomem *lut = (u8 *)nvbo_kmap_obj_iovirtual(nvbo);
-	const int size = blob->length / sizeof(*in);
-	int bits, shift, i;
-	u16 zero, r, g, b;
-
-	/* This can't happen.. But it shuts the compiler up. */
-	if (WARN_ON(size != 256))
-		return;
-
-	switch (mode) {
-	case 0: /* LORES. */
-	case 1: /* HIRES. */
-		bits = 11;
-		shift = 3;
-		zero = 0x0000;
-		break;
-	case 7: /* INTERPOLATE_257_UNITY_RANGE. */
-		bits = 14;
-		shift = 0;
-		zero = 0x6000;
-		break;
-	default:
-		WARN_ON(1);
-		return;
-	}
-
-	for (i = 0; i < size; i++) {
-		r = (drm_color_lut_extract(in[i].  red, bits) + zero) << shift;
-		g = (drm_color_lut_extract(in[i].green, bits) + zero) << shift;
-		b = (drm_color_lut_extract(in[i]. blue, bits) + zero) << shift;
-		writew(r, lut + (i * 0x08) + 0);
-		writew(g, lut + (i * 0x08) + 2);
-		writew(b, lut + (i * 0x08) + 4);
-	}
-
-	/* INTERPOLATE modes require a "next" entry to interpolate with,
-	 * so we replicate the last entry to deal with this for now.
-	 */
-	writew(r, lut + (i * 0x08) + 0);
-	writew(g, lut + (i * 0x08) + 2);
-	writew(b, lut + (i * 0x08) + 4);
-}
-
-static void
-nv50_head_lut_set(struct nv50_head *head, struct nv50_head_atom *asyh)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	u32 *push;
-	if ((push = evo_wait(core, 7))) {
-		if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2);
-			evo_data(push, 0x80000000 | asyh->lut.mode << 30);
-			evo_data(push, asyh->lut.offset >> 8);
-		} else
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2);
-			evo_data(push, 0x80000000 | asyh->lut.mode << 30);
-			evo_data(push, asyh->lut.offset >> 8);
-			evo_mthd(push, 0x085c + (head->base.index * 0x400), 1);
-			evo_data(push, asyh->lut.handle);
-		} else {
-			evo_mthd(push, 0x0440 + (head->base.index * 0x300), 4);
-			evo_data(push, 0x80000000 | asyh->lut.mode << 24);
-			evo_data(push, asyh->lut.offset >> 8);
-			evo_data(push, 0x00000000);
-			evo_data(push, 0x00000000);
-			evo_mthd(push, 0x045c + (head->base.index * 0x300), 1);
-			evo_data(push, asyh->lut.handle);
-		}
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	struct nv50_head_mode *m = &asyh->mode;
-	u32 *push;
-	if ((push = evo_wait(core, 14))) {
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0804 + (head->base.index * 0x400), 2);
-			evo_data(push, 0x00800000 | m->clock);
-			evo_data(push, m->interlace ? 0x00000002 : 0x00000000);
-			evo_mthd(push, 0x0810 + (head->base.index * 0x400), 7);
-			evo_data(push, 0x00000000);
-			evo_data(push, (m->v.active  << 16) | m->h.active );
-			evo_data(push, (m->v.synce   << 16) | m->h.synce  );
-			evo_data(push, (m->v.blanke  << 16) | m->h.blanke );
-			evo_data(push, (m->v.blanks  << 16) | m->h.blanks );
-			evo_data(push, (m->v.blank2e << 16) | m->v.blank2s);
-			evo_data(push, asyh->mode.v.blankus);
-			evo_mthd(push, 0x082c + (head->base.index * 0x400), 1);
-			evo_data(push, 0x00000000);
-		} else {
-			evo_mthd(push, 0x0410 + (head->base.index * 0x300), 6);
-			evo_data(push, 0x00000000);
-			evo_data(push, (m->v.active  << 16) | m->h.active );
-			evo_data(push, (m->v.synce   << 16) | m->h.synce  );
-			evo_data(push, (m->v.blanke  << 16) | m->h.blanke );
-			evo_data(push, (m->v.blanks  << 16) | m->h.blanks );
-			evo_data(push, (m->v.blank2e << 16) | m->v.blank2s);
-			evo_mthd(push, 0x042c + (head->base.index * 0x300), 2);
-			evo_data(push, 0x00000000); /* ??? */
-			evo_data(push, 0xffffff00);
-			evo_mthd(push, 0x0450 + (head->base.index * 0x300), 3);
-			evo_data(push, m->clock * 1000);
-			evo_data(push, 0x00200000); /* ??? */
-			evo_data(push, m->clock * 1000);
-		}
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_view(struct nv50_head *head, struct nv50_head_atom *asyh)
-{
-	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
-	u32 *push;
-	if ((push = evo_wait(core, 10))) {
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x08a4 + (head->base.index * 0x400), 1);
-			evo_data(push, 0x00000000);
-			evo_mthd(push, 0x08c8 + (head->base.index * 0x400), 1);
-			evo_data(push, (asyh->view.iH << 16) | asyh->view.iW);
-			evo_mthd(push, 0x08d8 + (head->base.index * 0x400), 2);
-			evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
-			evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
-		} else {
-			evo_mthd(push, 0x0494 + (head->base.index * 0x300), 1);
-			evo_data(push, 0x00000000);
-			evo_mthd(push, 0x04b8 + (head->base.index * 0x300), 1);
-			evo_data(push, (asyh->view.iH << 16) | asyh->view.iW);
-			evo_mthd(push, 0x04c0 + (head->base.index * 0x300), 3);
-			evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
-			evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
-			evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
-		}
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_head_flush_clr(struct nv50_head *head, struct nv50_head_atom *asyh, bool y)
-{
-	if (asyh->clr.ilut && (!asyh->set.ilut || y))
-		nv50_head_lut_clr(head);
-	if (asyh->clr.core && (!asyh->set.core || y))
-		nv50_head_core_clr(head);
-	if (asyh->clr.curs && (!asyh->set.curs || y))
-		nv50_head_curs_clr(head);
-}
-
-static void
-nv50_head_flush_set(struct nv50_head *head, struct nv50_head_atom *asyh)
-{
-	if (asyh->set.view   ) nv50_head_view    (head, asyh);
-	if (asyh->set.mode   ) nv50_head_mode    (head, asyh);
-	if (asyh->set.ilut   ) {
-		struct nouveau_bo *nvbo = head->lut.nvbo[head->lut.next];
-		struct drm_property_blob *blob = asyh->state.gamma_lut;
-		if (blob)
-			nv50_head_lut_load(blob, asyh->lut.mode, nvbo);
-		asyh->lut.offset = nvbo->bo.offset;
-		head->lut.next ^= 1;
-		nv50_head_lut_set(head, asyh);
-	}
-	if (asyh->set.core   ) nv50_head_core_set(head, asyh);
-	if (asyh->set.curs   ) nv50_head_curs_set(head, asyh);
-	if (asyh->set.base   ) nv50_head_base    (head, asyh);
-	if (asyh->set.ovly   ) nv50_head_ovly    (head, asyh);
-	if (asyh->set.dither ) nv50_head_dither  (head, asyh);
-	if (asyh->set.procamp) nv50_head_procamp (head, asyh);
-}
-
-static void
-nv50_head_atomic_check_procamp(struct nv50_head_atom *armh,
-			       struct nv50_head_atom *asyh,
-			       struct nouveau_conn_atom *asyc)
-{
-	const int vib = asyc->procamp.color_vibrance - 100;
-	const int hue = asyc->procamp.vibrant_hue - 90;
-	const int adj = (vib > 0) ? 50 : 0;
-	asyh->procamp.sat.cos = ((vib * 2047 + adj) / 100) & 0xfff;
-	asyh->procamp.sat.sin = ((hue * 2047) / 100) & 0xfff;
-	asyh->set.procamp = true;
-}
-
-static void
-nv50_head_atomic_check_dither(struct nv50_head_atom *armh,
-			      struct nv50_head_atom *asyh,
-			      struct nouveau_conn_atom *asyc)
-{
-	struct drm_connector *connector = asyc->state.connector;
-	u32 mode = 0x00;
-
-	if (asyc->dither.mode == DITHERING_MODE_AUTO) {
-		if (asyh->base.depth > connector->display_info.bpc * 3)
-			mode = DITHERING_MODE_DYNAMIC2X2;
-	} else {
-		mode = asyc->dither.mode;
-	}
-
-	if (asyc->dither.depth == DITHERING_DEPTH_AUTO) {
-		if (connector->display_info.bpc >= 8)
-			mode |= DITHERING_DEPTH_8BPC;
-	} else {
-		mode |= asyc->dither.depth;
-	}
-
-	asyh->dither.enable = mode;
-	asyh->dither.bits = mode >> 1;
-	asyh->dither.mode = mode >> 3;
-	asyh->set.dither = true;
-}
-
-static void
-nv50_head_atomic_check_view(struct nv50_head_atom *armh,
-			    struct nv50_head_atom *asyh,
-			    struct nouveau_conn_atom *asyc)
-{
-	struct drm_connector *connector = asyc->state.connector;
-	struct drm_display_mode *omode = &asyh->state.adjusted_mode;
-	struct drm_display_mode *umode = &asyh->state.mode;
-	int mode = asyc->scaler.mode;
-	struct edid *edid;
-	int umode_vdisplay, omode_hdisplay, omode_vdisplay;
-
-	if (connector->edid_blob_ptr)
-		edid = (struct edid *)connector->edid_blob_ptr->data;
-	else
-		edid = NULL;
-
-	if (!asyc->scaler.full) {
-		if (mode == DRM_MODE_SCALE_NONE)
-			omode = umode;
-	} else {
-		/* Non-EDID LVDS/eDP mode. */
-		mode = DRM_MODE_SCALE_FULLSCREEN;
-	}
-
-	/* For the user-specified mode, we must ignore doublescan and
-	 * the like, but honor frame packing.
-	 */
-	umode_vdisplay = umode->vdisplay;
-	if ((umode->flags & DRM_MODE_FLAG_3D_MASK) == DRM_MODE_FLAG_3D_FRAME_PACKING)
-		umode_vdisplay += umode->vtotal;
-	asyh->view.iW = umode->hdisplay;
-	asyh->view.iH = umode_vdisplay;
-	/* For the output mode, we can just use the stock helper. */
-	drm_mode_get_hv_timing(omode, &omode_hdisplay, &omode_vdisplay);
-	asyh->view.oW = omode_hdisplay;
-	asyh->view.oH = omode_vdisplay;
-
-	/* Add overscan compensation if necessary, will keep the aspect
-	 * ratio the same as the backend mode unless overridden by the
-	 * user setting both hborder and vborder properties.
-	 */
-	if ((asyc->scaler.underscan.mode == UNDERSCAN_ON ||
-	    (asyc->scaler.underscan.mode == UNDERSCAN_AUTO &&
-	     drm_detect_hdmi_monitor(edid)))) {
-		u32 bX = asyc->scaler.underscan.hborder;
-		u32 bY = asyc->scaler.underscan.vborder;
-		u32 r = (asyh->view.oH << 19) / asyh->view.oW;
-
-		if (bX) {
-			asyh->view.oW -= (bX * 2);
-			if (bY) asyh->view.oH -= (bY * 2);
-			else    asyh->view.oH  = ((asyh->view.oW * r) + (r / 2)) >> 19;
-		} else {
-			asyh->view.oW -= (asyh->view.oW >> 4) + 32;
-			if (bY) asyh->view.oH -= (bY * 2);
-			else    asyh->view.oH  = ((asyh->view.oW * r) + (r / 2)) >> 19;
-		}
-	}
-
-	/* Handle CENTER/ASPECT scaling, taking into account the areas
-	 * removed already for overscan compensation.
-	 */
-	switch (mode) {
-	case DRM_MODE_SCALE_CENTER:
-		asyh->view.oW = min((u16)umode->hdisplay, asyh->view.oW);
-		asyh->view.oH = min((u16)umode_vdisplay, asyh->view.oH);
-		/* fall-through */
-	case DRM_MODE_SCALE_ASPECT:
-		if (asyh->view.oH < asyh->view.oW) {
-			u32 r = (asyh->view.iW << 19) / asyh->view.iH;
-			asyh->view.oW = ((asyh->view.oH * r) + (r / 2)) >> 19;
-		} else {
-			u32 r = (asyh->view.iH << 19) / asyh->view.iW;
-			asyh->view.oH = ((asyh->view.oW * r) + (r / 2)) >> 19;
-		}
-		break;
-	default:
-		break;
-	}
-
-	asyh->set.view = true;
-}
-
-static void
-nv50_head_atomic_check_lut(struct nv50_head *head,
-			   struct nv50_head_atom *armh,
-			   struct nv50_head_atom *asyh)
-{
-	struct nv50_disp *disp = nv50_disp(head->base.base.dev);
-
-	/* An I8 surface without an input LUT makes no sense, and
-	 * EVO will throw an error if you try.
-	 *
-	 * Legacy clients actually cause this due to the order in
-	 * which they call ioctls, so we will enable the LUT with
-	 * whatever contents the buffer already contains to avoid
-	 * triggering the error check.
-	 */
-	if (!asyh->state.gamma_lut && asyh->base.cpp != 1) {
-		asyh->lut.handle = 0;
-		asyh->clr.ilut = armh->lut.visible;
-		return;
-	}
-
-	if (disp->disp->oclass < GF110_DISP) {
-		asyh->lut.mode = (asyh->base.cpp == 1) ? 0 : 1;
-		asyh->set.ilut = true;
-	} else {
-		asyh->lut.mode = 7;
-		asyh->set.ilut = asyh->state.color_mgmt_changed;
-	}
-	asyh->lut.handle = disp->mast.base.vram.handle;
-}
-
-static void
-nv50_head_atomic_check_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
-{
-	struct drm_display_mode *mode = &asyh->state.adjusted_mode;
-	struct nv50_head_mode *m = &asyh->mode;
-	u32 blankus;
-
-	drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V | CRTC_STEREO_DOUBLE);
-
-	/*
-	 * DRM modes are defined in terms of a repeating interval
-	 * starting with the active display area.  The hardware modes
-	 * are defined in terms of a repeating interval starting one
-	 * unit (pixel or line) into the sync pulse.  So, add bias.
-	 */
-
-	m->h.active = mode->crtc_htotal;
-	m->h.synce  = mode->crtc_hsync_end - mode->crtc_hsync_start - 1;
-	m->h.blanke = mode->crtc_hblank_end - mode->crtc_hsync_start - 1;
-	m->h.blanks = m->h.blanke + mode->crtc_hdisplay;
-
-	m->v.active = mode->crtc_vtotal;
-	m->v.synce  = mode->crtc_vsync_end - mode->crtc_vsync_start - 1;
-	m->v.blanke = mode->crtc_vblank_end - mode->crtc_vsync_start - 1;
-	m->v.blanks = m->v.blanke + mode->crtc_vdisplay;
-
-	/*XXX: Safe underestimate, even "0" works */
-	blankus = (m->v.active - mode->crtc_vdisplay - 2) * m->h.active;
-	blankus *= 1000;
-	blankus /= mode->crtc_clock;
-	m->v.blankus = blankus;
-
-	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
-		m->v.blank2e =  m->v.active + m->v.blanke;
-		m->v.blank2s =  m->v.blank2e + mode->crtc_vdisplay;
-		m->v.active  = (m->v.active * 2) + 1;
-		m->interlace = true;
-	} else {
-		m->v.blank2e = 0;
-		m->v.blank2s = 1;
-		m->interlace = false;
-	}
-	m->clock = mode->crtc_clock;
-
-	asyh->set.mode = true;
-}
-
-static int
-nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state)
-{
-	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
-	struct nv50_disp *disp = nv50_disp(crtc->dev);
-	struct nv50_head *head = nv50_head(crtc);
-	struct nv50_head_atom *armh = nv50_head_atom(crtc->state);
-	struct nv50_head_atom *asyh = nv50_head_atom(state);
-	struct nouveau_conn_atom *asyc = NULL;
-	struct drm_connector_state *conns;
-	struct drm_connector *conn;
-	int i;
-
-	NV_ATOMIC(drm, "%s atomic_check %d\n", crtc->name, asyh->state.active);
-	if (asyh->state.active) {
-		for_each_new_connector_in_state(asyh->state.state, conn, conns, i) {
-			if (conns->crtc == crtc) {
-				asyc = nouveau_conn_atom(conns);
-				break;
-			}
-		}
-
-		if (armh->state.active) {
-			if (asyc) {
-				if (asyh->state.mode_changed)
-					asyc->set.scaler = true;
-				if (armh->base.depth != asyh->base.depth)
-					asyc->set.dither = true;
-			}
-		} else {
-			if (asyc)
-				asyc->set.mask = ~0;
-			asyh->set.mask = ~0;
-		}
-
-		if (asyh->state.mode_changed)
-			nv50_head_atomic_check_mode(head, asyh);
-
-		if (asyh->state.color_mgmt_changed ||
-		    asyh->base.cpp != armh->base.cpp)
-			nv50_head_atomic_check_lut(head, armh, asyh);
-		asyh->lut.visible = asyh->lut.handle != 0;
-
-		if (asyc) {
-			if (asyc->set.scaler)
-				nv50_head_atomic_check_view(armh, asyh, asyc);
-			if (asyc->set.dither)
-				nv50_head_atomic_check_dither(armh, asyh, asyc);
-			if (asyc->set.procamp)
-				nv50_head_atomic_check_procamp(armh, asyh, asyc);
-		}
-
-		if ((asyh->core.visible = (asyh->base.cpp != 0))) {
-			asyh->core.x = asyh->base.x;
-			asyh->core.y = asyh->base.y;
-			asyh->core.w = asyh->base.w;
-			asyh->core.h = asyh->base.h;
-		} else
-		if ((asyh->core.visible = asyh->curs.visible) ||
-		    (asyh->core.visible = asyh->lut.visible)) {
-			/*XXX: We need to either find some way of having the
-			 *     primary base layer appear black, while still
-			 *     being able to display the other layers, or we
-			 *     need to allocate a dummy black surface here.
-			 */
-			asyh->core.x = 0;
-			asyh->core.y = 0;
-			asyh->core.w = asyh->state.mode.hdisplay;
-			asyh->core.h = asyh->state.mode.vdisplay;
-		}
-		asyh->core.handle = disp->mast.base.vram.handle;
-		asyh->core.offset = 0;
-		asyh->core.format = 0xcf;
-		asyh->core.kind = 0;
-		asyh->core.layout = 1;
-		asyh->core.block = 0;
-		asyh->core.pitch = ALIGN(asyh->core.w, 64) * 4;
-		asyh->set.base = armh->base.cpp != asyh->base.cpp;
-		asyh->set.ovly = armh->ovly.cpp != asyh->ovly.cpp;
-	} else {
-		asyh->lut.visible = false;
-		asyh->core.visible = false;
-		asyh->curs.visible = false;
-		asyh->base.cpp = 0;
-		asyh->ovly.cpp = 0;
-	}
-
-	if (!drm_atomic_crtc_needs_modeset(&asyh->state)) {
-		if (asyh->core.visible) {
-			if (memcmp(&armh->core, &asyh->core, sizeof(asyh->core)))
-				asyh->set.core = true;
-		} else
-		if (armh->core.visible) {
-			asyh->clr.core = true;
-		}
-
-		if (asyh->curs.visible) {
-			if (memcmp(&armh->curs, &asyh->curs, sizeof(asyh->curs)))
-				asyh->set.curs = true;
-		} else
-		if (armh->curs.visible) {
-			asyh->clr.curs = true;
-		}
-	} else {
-		asyh->clr.ilut = armh->lut.visible;
-		asyh->clr.core = armh->core.visible;
-		asyh->clr.curs = armh->curs.visible;
-		asyh->set.ilut = asyh->lut.visible;
-		asyh->set.core = asyh->core.visible;
-		asyh->set.curs = asyh->curs.visible;
-	}
-
-	if (asyh->clr.mask || asyh->set.mask)
-		nv50_atom(asyh->state.state)->lock_core = true;
-	return 0;
-}
-
-static const struct drm_crtc_helper_funcs
-nv50_head_help = {
-	.atomic_check = nv50_head_atomic_check,
-};
-
-static void
-nv50_head_atomic_destroy_state(struct drm_crtc *crtc,
-			       struct drm_crtc_state *state)
-{
-	struct nv50_head_atom *asyh = nv50_head_atom(state);
-	__drm_atomic_helper_crtc_destroy_state(&asyh->state);
-	kfree(asyh);
-}
-
-static struct drm_crtc_state *
-nv50_head_atomic_duplicate_state(struct drm_crtc *crtc)
-{
-	struct nv50_head_atom *armh = nv50_head_atom(crtc->state);
-	struct nv50_head_atom *asyh;
-	if (!(asyh = kmalloc(sizeof(*asyh), GFP_KERNEL)))
-		return NULL;
-	__drm_atomic_helper_crtc_duplicate_state(crtc, &asyh->state);
-	asyh->view = armh->view;
-	asyh->mode = armh->mode;
-	asyh->lut  = armh->lut;
-	asyh->core = armh->core;
-	asyh->curs = armh->curs;
-	asyh->base = armh->base;
-	asyh->ovly = armh->ovly;
-	asyh->dither = armh->dither;
-	asyh->procamp = armh->procamp;
-	asyh->clr.mask = 0;
-	asyh->set.mask = 0;
-	return &asyh->state;
-}
-
-static void
-__drm_atomic_helper_crtc_reset(struct drm_crtc *crtc,
-			       struct drm_crtc_state *state)
-{
-	if (crtc->state)
-		crtc->funcs->atomic_destroy_state(crtc, crtc->state);
-	crtc->state = state;
-	crtc->state->crtc = crtc;
-}
-
-static void
-nv50_head_reset(struct drm_crtc *crtc)
-{
-	struct nv50_head_atom *asyh;
-
-	if (WARN_ON(!(asyh = kzalloc(sizeof(*asyh), GFP_KERNEL))))
-		return;
-
-	__drm_atomic_helper_crtc_reset(crtc, &asyh->state);
-}
-
-static void
-nv50_head_destroy(struct drm_crtc *crtc)
-{
-	struct nv50_disp *disp = nv50_disp(crtc->dev);
-	struct nv50_head *head = nv50_head(crtc);
-	int i;
-
-	nv50_dmac_destroy(&head->ovly.base, disp->disp);
-	nv50_pioc_destroy(&head->oimm.base);
-
-	for (i = 0; i < ARRAY_SIZE(head->lut.nvbo); i++)
-		nouveau_bo_unmap_unpin_unref(&head->lut.nvbo[i]);
-
-	drm_crtc_cleanup(crtc);
-	kfree(crtc);
-}
-
-static const struct drm_crtc_funcs
-nv50_head_func = {
-	.reset = nv50_head_reset,
-	.gamma_set = drm_atomic_helper_legacy_gamma_set,
-	.destroy = nv50_head_destroy,
-	.set_config = drm_atomic_helper_set_config,
-	.page_flip = drm_atomic_helper_page_flip,
-	.atomic_duplicate_state = nv50_head_atomic_duplicate_state,
-	.atomic_destroy_state = nv50_head_atomic_destroy_state,
-};
-
-static int
-nv50_head_create(struct drm_device *dev, int index)
-{
-	struct nouveau_drm *drm = nouveau_drm(dev);
-	struct nvif_device *device = &drm->client.device;
-	struct nv50_disp *disp = nv50_disp(dev);
-	struct nv50_head *head;
-	struct nv50_base *base;
-	struct nv50_curs *curs;
-	struct drm_crtc *crtc;
-	int ret, i;
-
-	head = kzalloc(sizeof(*head), GFP_KERNEL);
-	if (!head)
-		return -ENOMEM;
-
-	head->base.index = index;
-	ret = nv50_base_new(drm, head, &base);
-	if (ret == 0)
-		ret = nv50_curs_new(drm, head, &curs);
-	if (ret) {
-		kfree(head);
-		return ret;
-	}
-
-	crtc = &head->base.base;
-	drm_crtc_init_with_planes(dev, crtc, &base->wndw.plane,
-				  &curs->wndw.plane, &nv50_head_func,
-				  "head-%d", head->base.index);
-	drm_crtc_helper_add(crtc, &nv50_head_help);
-	drm_mode_crtc_set_gamma_size(crtc, 256);
-
-	for (i = 0; i < ARRAY_SIZE(head->lut.nvbo); i++) {
-		ret = nouveau_bo_new_pin_map(&drm->client, 1025 * 8, 0x100,
-					     TTM_PL_FLAG_VRAM,
-					     &head->lut.nvbo[i]);
-		if (ret)
-			goto out;
-	}
-
-	/* allocate overlay resources */
-	ret = nv50_oimm_create(device, disp->disp, index, &head->oimm);
-	if (ret)
-		goto out;
-
-	ret = nv50_ovly_create(device, disp->disp, index, disp->sync->bo.offset,
-			       &head->ovly);
-	if (ret)
-		goto out;
-
-out:
-	if (ret)
-		nv50_head_destroy(crtc);
-	return ret;
-}
-
-/******************************************************************************
- * Output path helpers
- *****************************************************************************/
-static void
-nv50_outp_release(struct nouveau_encoder *nv_encoder)
-{
-	struct nv50_disp *disp = nv50_disp(nv_encoder->base.base.dev);
-	struct {
-		struct nv50_disp_mthd_v1 base;
-	} args = {
-		.base.version = 1,
-		.base.method = NV50_DISP_MTHD_V1_RELEASE,
-		.base.hasht  = nv_encoder->dcb->hasht,
-		.base.hashm  = nv_encoder->dcb->hashm,
-	};
-
-	nvif_mthd(disp->disp, 0, &args, sizeof(args));
-	nv_encoder->or = -1;
-	nv_encoder->link = 0;
-}
-
-static int
-nv50_outp_acquire(struct nouveau_encoder *nv_encoder)
-{
-	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
-	struct nv50_disp *disp = nv50_disp(drm->dev);
-	struct {
-		struct nv50_disp_mthd_v1 base;
-		struct nv50_disp_acquire_v0 info;
-	} args = {
-		.base.version = 1,
-		.base.method = NV50_DISP_MTHD_V1_ACQUIRE,
-		.base.hasht  = nv_encoder->dcb->hasht,
-		.base.hashm  = nv_encoder->dcb->hashm,
-	};
-	int ret;
-
-	ret = nvif_mthd(disp->disp, 0, &args, sizeof(args));
-	if (ret) {
-		NV_ERROR(drm, "error acquiring output path: %d\n", ret);
-		return ret;
-	}
-
-	nv_encoder->or = args.info.or;
-	nv_encoder->link = args.info.link;
-	return 0;
-}
-
-static int
-nv50_outp_atomic_check_view(struct drm_encoder *encoder,
-			    struct drm_crtc_state *crtc_state,
-			    struct drm_connector_state *conn_state,
-			    struct drm_display_mode *native_mode)
-{
-	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
-	struct drm_display_mode *mode = &crtc_state->mode;
-	struct drm_connector *connector = conn_state->connector;
-	struct nouveau_conn_atom *asyc = nouveau_conn_atom(conn_state);
-	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
-
-	NV_ATOMIC(drm, "%s atomic_check\n", encoder->name);
-	asyc->scaler.full = false;
-	if (!native_mode)
-		return 0;
-
-	if (asyc->scaler.mode == DRM_MODE_SCALE_NONE) {
-		switch (connector->connector_type) {
-		case DRM_MODE_CONNECTOR_LVDS:
-		case DRM_MODE_CONNECTOR_eDP:
-			/* Force use of scaler for non-EDID modes. */
-			if (adjusted_mode->type & DRM_MODE_TYPE_DRIVER)
-				break;
-			mode = native_mode;
-			asyc->scaler.full = true;
-			break;
-		default:
-			break;
-		}
-	} else {
-		mode = native_mode;
-	}
-
-	if (!drm_mode_equal(adjusted_mode, mode)) {
-		drm_mode_copy(adjusted_mode, mode);
-		crtc_state->mode_changed = true;
-	}
-
-	return 0;
-}
-
-static int
-nv50_outp_atomic_check(struct drm_encoder *encoder,
-		       struct drm_crtc_state *crtc_state,
-		       struct drm_connector_state *conn_state)
-{
-	struct nouveau_connector *nv_connector =
-		nouveau_connector(conn_state->connector);
-	return nv50_outp_atomic_check_view(encoder, crtc_state, conn_state,
-					   nv_connector->native_mode);
-}
-
-/******************************************************************************
- * DAC
- *****************************************************************************/
-static void
-nv50_dac_disable(struct drm_encoder *encoder)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nv50_mast *mast = nv50_mast(encoder->dev);
-	const int or = nv_encoder->or;
-	u32 *push;
-
-	if (nv_encoder->crtc) {
-		push = evo_wait(mast, 4);
-		if (push) {
-			if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-				evo_mthd(push, 0x0400 + (or * 0x080), 1);
-				evo_data(push, 0x00000000);
-			} else {
-				evo_mthd(push, 0x0180 + (or * 0x020), 1);
-				evo_data(push, 0x00000000);
-			}
-			evo_kick(push, mast);
-		}
-	}
-
-	nv_encoder->crtc = NULL;
-	nv50_outp_release(nv_encoder);
-}
-
-static void
-nv50_dac_enable(struct drm_encoder *encoder)
-{
-	struct nv50_mast *mast = nv50_mast(encoder->dev);
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
-	struct drm_display_mode *mode = &nv_crtc->base.state->adjusted_mode;
-	u32 *push;
-
-	nv50_outp_acquire(nv_encoder);
-
-	push = evo_wait(mast, 8);
-	if (push) {
-		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-			u32 syncs = 0x00000000;
-
-			if (mode->flags & DRM_MODE_FLAG_NHSYNC)
-				syncs |= 0x00000001;
-			if (mode->flags & DRM_MODE_FLAG_NVSYNC)
-				syncs |= 0x00000002;
-
-			evo_mthd(push, 0x0400 + (nv_encoder->or * 0x080), 2);
-			evo_data(push, 1 << nv_crtc->index);
-			evo_data(push, syncs);
-		} else {
-			u32 magic = 0x31ec6000 | (nv_crtc->index << 25);
-			u32 syncs = 0x00000001;
-
-			if (mode->flags & DRM_MODE_FLAG_NHSYNC)
-				syncs |= 0x00000008;
-			if (mode->flags & DRM_MODE_FLAG_NVSYNC)
-				syncs |= 0x00000010;
-
-			if (mode->flags & DRM_MODE_FLAG_INTERLACE)
-				magic |= 0x00000001;
-
-			evo_mthd(push, 0x0404 + (nv_crtc->index * 0x300), 2);
-			evo_data(push, syncs);
-			evo_data(push, magic);
-			evo_mthd(push, 0x0180 + (nv_encoder->or * 0x020), 1);
-			evo_data(push, 1 << nv_crtc->index);
-		}
-
-		evo_kick(push, mast);
-	}
-
-	nv_encoder->crtc = encoder->crtc;
-}
-
-static enum drm_connector_status
-nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nv50_disp *disp = nv50_disp(encoder->dev);
-	struct {
-		struct nv50_disp_mthd_v1 base;
-		struct nv50_disp_dac_load_v0 load;
-	} args = {
-		.base.version = 1,
-		.base.method = NV50_DISP_MTHD_V1_DAC_LOAD,
-		.base.hasht  = nv_encoder->dcb->hasht,
-		.base.hashm  = nv_encoder->dcb->hashm,
-	};
-	int ret;
-
-	args.load.data = nouveau_drm(encoder->dev)->vbios.dactestval;
-	if (args.load.data == 0)
-		args.load.data = 340;
-
-	ret = nvif_mthd(disp->disp, 0, &args, sizeof(args));
-	if (ret || !args.load.load)
-		return connector_status_disconnected;
-
-	return connector_status_connected;
-}
-
-static const struct drm_encoder_helper_funcs
-nv50_dac_help = {
-	.atomic_check = nv50_outp_atomic_check,
-	.enable = nv50_dac_enable,
-	.disable = nv50_dac_disable,
-	.detect = nv50_dac_detect
-};
-
-static void
-nv50_dac_destroy(struct drm_encoder *encoder)
-{
-	drm_encoder_cleanup(encoder);
-	kfree(encoder);
-}
-
-static const struct drm_encoder_funcs
-nv50_dac_func = {
-	.destroy = nv50_dac_destroy,
-};
-
-static int
-nv50_dac_create(struct drm_connector *connector, struct dcb_output *dcbe)
-{
-	struct nouveau_drm *drm = nouveau_drm(connector->dev);
-	struct nvkm_i2c *i2c = nvxx_i2c(&drm->client.device);
-	struct nvkm_i2c_bus *bus;
-	struct nouveau_encoder *nv_encoder;
-	struct drm_encoder *encoder;
-	int type = DRM_MODE_ENCODER_DAC;
-
-	nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
-	if (!nv_encoder)
-		return -ENOMEM;
-	nv_encoder->dcb = dcbe;
-
-	bus = nvkm_i2c_bus_find(i2c, dcbe->i2c_index);
-	if (bus)
-		nv_encoder->i2c = &bus->i2c;
-
-	encoder = to_drm_encoder(nv_encoder);
-	encoder->possible_crtcs = dcbe->heads;
-	encoder->possible_clones = 0;
-	drm_encoder_init(connector->dev, encoder, &nv50_dac_func, type,
-			 "dac-%04x-%04x", dcbe->hasht, dcbe->hashm);
-	drm_encoder_helper_add(encoder, &nv50_dac_help);
-
-	drm_mode_connector_attach_encoder(connector, encoder);
-	return 0;
-}
-
-/******************************************************************************
- * Audio
- *****************************************************************************/
-static void
-nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nv50_disp *disp = nv50_disp(encoder->dev);
-	struct {
-		struct nv50_disp_mthd_v1 base;
-		struct nv50_disp_sor_hda_eld_v0 eld;
-	} args = {
-		.base.version = 1,
-		.base.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
-		.base.hasht   = nv_encoder->dcb->hasht,
-		.base.hashm   = (0xf0ff & nv_encoder->dcb->hashm) |
-				(0x0100 << nv_crtc->index),
-	};
-
-	nvif_mthd(disp->disp, 0, &args, sizeof(args));
-}
-
-static void
-nv50_audio_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
-	struct nouveau_connector *nv_connector;
-	struct nv50_disp *disp = nv50_disp(encoder->dev);
-	struct __packed {
-		struct {
-			struct nv50_disp_mthd_v1 mthd;
-			struct nv50_disp_sor_hda_eld_v0 eld;
-		} base;
-		u8 data[sizeof(nv_connector->base.eld)];
-	} args = {
-		.base.mthd.version = 1,
-		.base.mthd.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
-		.base.mthd.hasht   = nv_encoder->dcb->hasht,
-		.base.mthd.hashm   = (0xf0ff & nv_encoder->dcb->hashm) |
-				     (0x0100 << nv_crtc->index),
-	};
-
-	nv_connector = nouveau_encoder_connector_get(nv_encoder);
-	if (!drm_detect_monitor_audio(nv_connector->edid))
-		return;
-
-	memcpy(args.data, nv_connector->base.eld, sizeof(args.data));
-
-	nvif_mthd(disp->disp, 0, &args,
-		  sizeof(args.base) + drm_eld_size(args.data));
-}
-
-/******************************************************************************
- * HDMI
- *****************************************************************************/
-static void
-nv50_hdmi_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nv50_disp *disp = nv50_disp(encoder->dev);
-	struct {
-		struct nv50_disp_mthd_v1 base;
-		struct nv50_disp_sor_hdmi_pwr_v0 pwr;
-	} args = {
-		.base.version = 1,
-		.base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
-		.base.hasht  = nv_encoder->dcb->hasht,
-		.base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
-			       (0x0100 << nv_crtc->index),
-	};
-
-	nvif_mthd(disp->disp, 0, &args, sizeof(args));
-}
-
-static void
-nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
-	struct nv50_disp *disp = nv50_disp(encoder->dev);
-	struct {
-		struct nv50_disp_mthd_v1 base;
-		struct nv50_disp_sor_hdmi_pwr_v0 pwr;
-		u8 infoframes[2 * 17]; /* two frames, up to 17 bytes each */
-	} args = {
-		.base.version = 1,
-		.base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
-		.base.hasht  = nv_encoder->dcb->hasht,
-		.base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
-			       (0x0100 << nv_crtc->index),
-		.pwr.state = 1,
-		.pwr.rekey = 56, /* binary driver, and tegra, constant */
-	};
-	struct nouveau_connector *nv_connector;
-	u32 max_ac_packet;
-	union hdmi_infoframe avi_frame;
-	union hdmi_infoframe vendor_frame;
-	int ret;
-	int size;
-
-	nv_connector = nouveau_encoder_connector_get(nv_encoder);
-	if (!drm_detect_hdmi_monitor(nv_connector->edid))
-		return;
-
-	ret = drm_hdmi_avi_infoframe_from_display_mode(&avi_frame.avi, mode,
-						       false);
-	if (!ret) {
-		/* We have an AVI InfoFrame, populate it to the display */
-		args.pwr.avi_infoframe_length
-			= hdmi_infoframe_pack(&avi_frame, args.infoframes, 17);
-	}
-
-	ret = drm_hdmi_vendor_infoframe_from_display_mode(&vendor_frame.vendor.hdmi,
-							  &nv_connector->base, mode);
-	if (!ret) {
-		/* We have a Vendor InfoFrame, populate it to the display */
-		args.pwr.vendor_infoframe_length
-			= hdmi_infoframe_pack(&vendor_frame,
-					      args.infoframes
-					      + args.pwr.avi_infoframe_length,
-					      17);
-	}
-
-	max_ac_packet  = mode->htotal - mode->hdisplay;
-	max_ac_packet -= args.pwr.rekey;
-	max_ac_packet -= 18; /* constant from tegra */
-	args.pwr.max_ac_packet = max_ac_packet / 32;
-
-	size = sizeof(args.base)
-		+ sizeof(args.pwr)
-		+ args.pwr.avi_infoframe_length
-		+ args.pwr.vendor_infoframe_length;
-	nvif_mthd(disp->disp, 0, &args, size);
-	nv50_audio_enable(encoder, mode);
-}
-
-/******************************************************************************
- * MST
- *****************************************************************************/
-#define nv50_mstm(p) container_of((p), struct nv50_mstm, mgr)
-#define nv50_mstc(p) container_of((p), struct nv50_mstc, connector)
-#define nv50_msto(p) container_of((p), struct nv50_msto, encoder)
-
-struct nv50_mstm {
-	struct nouveau_encoder *outp;
-
-	struct drm_dp_mst_topology_mgr mgr;
-	struct nv50_msto *msto[4];
-
-	bool modified;
-	bool disabled;
-	int links;
-};
-
-struct nv50_mstc {
-	struct nv50_mstm *mstm;
-	struct drm_dp_mst_port *port;
-	struct drm_connector connector;
-
-	struct drm_display_mode *native;
-	struct edid *edid;
-
-	int pbn;
-};
-
-struct nv50_msto {
-	struct drm_encoder encoder;
-
-	struct nv50_head *head;
-	struct nv50_mstc *mstc;
-	bool disabled;
-};
-
-static struct drm_dp_payload *
-nv50_msto_payload(struct nv50_msto *msto)
-{
-	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
-	struct nv50_mstc *mstc = msto->mstc;
-	struct nv50_mstm *mstm = mstc->mstm;
-	int vcpi = mstc->port->vcpi.vcpi, i;
-
-	NV_ATOMIC(drm, "%s: vcpi %d\n", msto->encoder.name, vcpi);
-	for (i = 0; i < mstm->mgr.max_payloads; i++) {
-		struct drm_dp_payload *payload = &mstm->mgr.payloads[i];
-		NV_ATOMIC(drm, "%s: %d: vcpi %d start 0x%02x slots 0x%02x\n",
-			  mstm->outp->base.base.name, i, payload->vcpi,
-			  payload->start_slot, payload->num_slots);
-	}
-
-	for (i = 0; i < mstm->mgr.max_payloads; i++) {
-		struct drm_dp_payload *payload = &mstm->mgr.payloads[i];
-		if (payload->vcpi == vcpi)
-			return payload;
-	}
-
-	return NULL;
-}
-
-static void
-nv50_msto_cleanup(struct nv50_msto *msto)
-{
-	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
-	struct nv50_mstc *mstc = msto->mstc;
-	struct nv50_mstm *mstm = mstc->mstm;
-
-	NV_ATOMIC(drm, "%s: msto cleanup\n", msto->encoder.name);
-	if (mstc->port && mstc->port->vcpi.vcpi > 0 && !nv50_msto_payload(msto))
-		drm_dp_mst_deallocate_vcpi(&mstm->mgr, mstc->port);
-	if (msto->disabled) {
-		msto->mstc = NULL;
-		msto->head = NULL;
-		msto->disabled = false;
-	}
-}
-
-static void
-nv50_msto_prepare(struct nv50_msto *msto)
-{
-	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
-	struct nv50_mstc *mstc = msto->mstc;
-	struct nv50_mstm *mstm = mstc->mstm;
-	struct {
-		struct nv50_disp_mthd_v1 base;
-		struct nv50_disp_sor_dp_mst_vcpi_v0 vcpi;
-	} args = {
-		.base.version = 1,
-		.base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI,
-		.base.hasht  = mstm->outp->dcb->hasht,
-		.base.hashm  = (0xf0ff & mstm->outp->dcb->hashm) |
-			       (0x0100 << msto->head->base.index),
-	};
-
-	NV_ATOMIC(drm, "%s: msto prepare\n", msto->encoder.name);
-	if (mstc->port && mstc->port->vcpi.vcpi > 0) {
-		struct drm_dp_payload *payload = nv50_msto_payload(msto);
-		if (payload) {
-			args.vcpi.start_slot = payload->start_slot;
-			args.vcpi.num_slots = payload->num_slots;
-			args.vcpi.pbn = mstc->port->vcpi.pbn;
-			args.vcpi.aligned_pbn = mstc->port->vcpi.aligned_pbn;
-		}
-	}
-
-	NV_ATOMIC(drm, "%s: %s: %02x %02x %04x %04x\n",
-		  msto->encoder.name, msto->head->base.base.name,
-		  args.vcpi.start_slot, args.vcpi.num_slots,
-		  args.vcpi.pbn, args.vcpi.aligned_pbn);
-	nvif_mthd(&drm->display->disp, 0, &args, sizeof(args));
-}
-
-static int
-nv50_msto_atomic_check(struct drm_encoder *encoder,
-		       struct drm_crtc_state *crtc_state,
-		       struct drm_connector_state *conn_state)
-{
-	struct nv50_mstc *mstc = nv50_mstc(conn_state->connector);
-	struct nv50_mstm *mstm = mstc->mstm;
-	int bpp = conn_state->connector->display_info.bpc * 3;
-	int slots;
-
-	mstc->pbn = drm_dp_calc_pbn_mode(crtc_state->adjusted_mode.clock, bpp);
-
-	slots = drm_dp_find_vcpi_slots(&mstm->mgr, mstc->pbn);
-	if (slots < 0)
-		return slots;
-
-	return nv50_outp_atomic_check_view(encoder, crtc_state, conn_state,
-					   mstc->native);
-}
-
-static void
-nv50_msto_enable(struct drm_encoder *encoder)
-{
-	struct nv50_head *head = nv50_head(encoder->crtc);
-	struct nv50_msto *msto = nv50_msto(encoder);
-	struct nv50_mstc *mstc = NULL;
-	struct nv50_mstm *mstm = NULL;
-	struct drm_connector *connector;
-	struct drm_connector_list_iter conn_iter;
-	u8 proto, depth;
-	int slots;
-	bool r;
-
-	drm_connector_list_iter_begin(encoder->dev, &conn_iter);
-	drm_for_each_connector_iter(connector, &conn_iter) {
-		if (connector->state->best_encoder == &msto->encoder) {
-			mstc = nv50_mstc(connector);
-			mstm = mstc->mstm;
-			break;
-		}
-	}
-	drm_connector_list_iter_end(&conn_iter);
-
-	if (WARN_ON(!mstc))
-		return;
-
-	slots = drm_dp_find_vcpi_slots(&mstm->mgr, mstc->pbn);
-	r = drm_dp_mst_allocate_vcpi(&mstm->mgr, mstc->port, mstc->pbn, slots);
-	WARN_ON(!r);
-
-	if (!mstm->links++)
-		nv50_outp_acquire(mstm->outp);
-
-	if (mstm->outp->link & 1)
-		proto = 0x8;
-	else
-		proto = 0x9;
-
-	switch (mstc->connector.display_info.bpc) {
-	case  6: depth = 0x2; break;
-	case  8: depth = 0x5; break;
-	case 10:
-	default: depth = 0x6; break;
-	}
-
-	mstm->outp->update(mstm->outp, head->base.index,
-			   &head->base.base.state->adjusted_mode, proto, depth);
-
-	msto->head = head;
-	msto->mstc = mstc;
-	mstm->modified = true;
-}
-
-static void
-nv50_msto_disable(struct drm_encoder *encoder)
-{
-	struct nv50_msto *msto = nv50_msto(encoder);
-	struct nv50_mstc *mstc = msto->mstc;
-	struct nv50_mstm *mstm = mstc->mstm;
-
-	if (mstc->port)
-		drm_dp_mst_reset_vcpi_slots(&mstm->mgr, mstc->port);
-
-	mstm->outp->update(mstm->outp, msto->head->base.index, NULL, 0, 0);
-	mstm->modified = true;
-	if (!--mstm->links)
-		mstm->disabled = true;
-	msto->disabled = true;
-}
-
-static const struct drm_encoder_helper_funcs
-nv50_msto_help = {
-	.disable = nv50_msto_disable,
-	.enable = nv50_msto_enable,
-	.atomic_check = nv50_msto_atomic_check,
-};
-
-static void
-nv50_msto_destroy(struct drm_encoder *encoder)
-{
-	struct nv50_msto *msto = nv50_msto(encoder);
-	drm_encoder_cleanup(&msto->encoder);
-	kfree(msto);
-}
-
-static const struct drm_encoder_funcs
-nv50_msto = {
-	.destroy = nv50_msto_destroy,
-};
-
-static int
-nv50_msto_new(struct drm_device *dev, u32 heads, const char *name, int id,
-	      struct nv50_msto **pmsto)
-{
-	struct nv50_msto *msto;
-	int ret;
-
-	if (!(msto = *pmsto = kzalloc(sizeof(*msto), GFP_KERNEL)))
-		return -ENOMEM;
-
-	ret = drm_encoder_init(dev, &msto->encoder, &nv50_msto,
-			       DRM_MODE_ENCODER_DPMST, "%s-mst-%d", name, id);
-	if (ret) {
-		kfree(*pmsto);
-		*pmsto = NULL;
-		return ret;
-	}
-
-	drm_encoder_helper_add(&msto->encoder, &nv50_msto_help);
-	msto->encoder.possible_crtcs = heads;
-	return 0;
-}
-
-static struct drm_encoder *
-nv50_mstc_atomic_best_encoder(struct drm_connector *connector,
-			      struct drm_connector_state *connector_state)
-{
-	struct nv50_head *head = nv50_head(connector_state->crtc);
-	struct nv50_mstc *mstc = nv50_mstc(connector);
-	if (mstc->port) {
-		struct nv50_mstm *mstm = mstc->mstm;
-		return &mstm->msto[head->base.index]->encoder;
-	}
-	return NULL;
-}
-
-static struct drm_encoder *
-nv50_mstc_best_encoder(struct drm_connector *connector)
-{
-	struct nv50_mstc *mstc = nv50_mstc(connector);
-	if (mstc->port) {
-		struct nv50_mstm *mstm = mstc->mstm;
-		return &mstm->msto[0]->encoder;
-	}
-	return NULL;
-}
-
-static enum drm_mode_status
-nv50_mstc_mode_valid(struct drm_connector *connector,
-		     struct drm_display_mode *mode)
-{
-	return MODE_OK;
-}
-
-static int
-nv50_mstc_get_modes(struct drm_connector *connector)
-{
-	struct nv50_mstc *mstc = nv50_mstc(connector);
-	int ret = 0;
-
-	mstc->edid = drm_dp_mst_get_edid(&mstc->connector, mstc->port->mgr, mstc->port);
-	drm_mode_connector_update_edid_property(&mstc->connector, mstc->edid);
-	if (mstc->edid)
-		ret = drm_add_edid_modes(&mstc->connector, mstc->edid);
-
-	if (!mstc->connector.display_info.bpc)
-		mstc->connector.display_info.bpc = 8;
-
-	if (mstc->native)
-		drm_mode_destroy(mstc->connector.dev, mstc->native);
-	mstc->native = nouveau_conn_native_mode(&mstc->connector);
-	return ret;
-}
-
-static const struct drm_connector_helper_funcs
-nv50_mstc_help = {
-	.get_modes = nv50_mstc_get_modes,
-	.mode_valid = nv50_mstc_mode_valid,
-	.best_encoder = nv50_mstc_best_encoder,
-	.atomic_best_encoder = nv50_mstc_atomic_best_encoder,
-};
-
-static enum drm_connector_status
-nv50_mstc_detect(struct drm_connector *connector, bool force)
-{
-	struct nv50_mstc *mstc = nv50_mstc(connector);
-	if (!mstc->port)
-		return connector_status_disconnected;
-	return drm_dp_mst_detect_port(connector, mstc->port->mgr, mstc->port);
-}
-
-static void
-nv50_mstc_destroy(struct drm_connector *connector)
-{
-	struct nv50_mstc *mstc = nv50_mstc(connector);
-	drm_connector_cleanup(&mstc->connector);
-	kfree(mstc);
-}
-
-static const struct drm_connector_funcs
-nv50_mstc = {
-	.reset = nouveau_conn_reset,
-	.detect = nv50_mstc_detect,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = nv50_mstc_destroy,
-	.atomic_duplicate_state = nouveau_conn_atomic_duplicate_state,
-	.atomic_destroy_state = nouveau_conn_atomic_destroy_state,
-	.atomic_set_property = nouveau_conn_atomic_set_property,
-	.atomic_get_property = nouveau_conn_atomic_get_property,
-};
-
-static int
-nv50_mstc_new(struct nv50_mstm *mstm, struct drm_dp_mst_port *port,
-	      const char *path, struct nv50_mstc **pmstc)
-{
-	struct drm_device *dev = mstm->outp->base.base.dev;
-	struct nv50_mstc *mstc;
-	int ret, i;
-
-	if (!(mstc = *pmstc = kzalloc(sizeof(*mstc), GFP_KERNEL)))
-		return -ENOMEM;
-	mstc->mstm = mstm;
-	mstc->port = port;
-
-	ret = drm_connector_init(dev, &mstc->connector, &nv50_mstc,
-				 DRM_MODE_CONNECTOR_DisplayPort);
-	if (ret) {
-		kfree(*pmstc);
-		*pmstc = NULL;
-		return ret;
-	}
-
-	drm_connector_helper_add(&mstc->connector, &nv50_mstc_help);
-
-	mstc->connector.funcs->reset(&mstc->connector);
-	nouveau_conn_attach_properties(&mstc->connector);
-
-	for (i = 0; i < ARRAY_SIZE(mstm->msto) && mstm->msto[i]; i++)
-		drm_mode_connector_attach_encoder(&mstc->connector, &mstm->msto[i]->encoder);
-
-	drm_object_attach_property(&mstc->connector.base, dev->mode_config.path_property, 0);
-	drm_object_attach_property(&mstc->connector.base, dev->mode_config.tile_property, 0);
-	drm_mode_connector_set_path_property(&mstc->connector, path);
-	return 0;
-}
-
-static void
-nv50_mstm_cleanup(struct nv50_mstm *mstm)
-{
-	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
-	struct drm_encoder *encoder;
-	int ret;
-
-	NV_ATOMIC(drm, "%s: mstm cleanup\n", mstm->outp->base.base.name);
-	ret = drm_dp_check_act_status(&mstm->mgr);
-
-	ret = drm_dp_update_payload_part2(&mstm->mgr);
-
-	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
-		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
-			struct nv50_msto *msto = nv50_msto(encoder);
-			struct nv50_mstc *mstc = msto->mstc;
-			if (mstc && mstc->mstm == mstm)
-				nv50_msto_cleanup(msto);
-		}
-	}
-
-	mstm->modified = false;
-}
-
-static void
-nv50_mstm_prepare(struct nv50_mstm *mstm)
-{
-	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
-	struct drm_encoder *encoder;
-	int ret;
-
-	NV_ATOMIC(drm, "%s: mstm prepare\n", mstm->outp->base.base.name);
-	ret = drm_dp_update_payload_part1(&mstm->mgr);
-
-	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
-		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
-			struct nv50_msto *msto = nv50_msto(encoder);
-			struct nv50_mstc *mstc = msto->mstc;
-			if (mstc && mstc->mstm == mstm)
-				nv50_msto_prepare(msto);
-		}
-	}
-
-	if (mstm->disabled) {
-		if (!mstm->links)
-			nv50_outp_release(mstm->outp);
-		mstm->disabled = false;
-	}
-}
-
-static void
-nv50_mstm_hotplug(struct drm_dp_mst_topology_mgr *mgr)
-{
-	struct nv50_mstm *mstm = nv50_mstm(mgr);
-	drm_kms_helper_hotplug_event(mstm->outp->base.base.dev);
-}
-
-static void
-nv50_mstm_destroy_connector(struct drm_dp_mst_topology_mgr *mgr,
-			    struct drm_connector *connector)
-{
-	struct nouveau_drm *drm = nouveau_drm(connector->dev);
-	struct nv50_mstc *mstc = nv50_mstc(connector);
-
-	drm_connector_unregister(&mstc->connector);
-
-	drm_fb_helper_remove_one_connector(&drm->fbcon->helper, &mstc->connector);
-
-	drm_modeset_lock(&drm->dev->mode_config.connection_mutex, NULL);
-	mstc->port = NULL;
-	drm_modeset_unlock(&drm->dev->mode_config.connection_mutex);
-
-	drm_connector_unreference(&mstc->connector);
-}
-
-static void
-nv50_mstm_register_connector(struct drm_connector *connector)
-{
-	struct nouveau_drm *drm = nouveau_drm(connector->dev);
-
-	drm_fb_helper_add_one_connector(&drm->fbcon->helper, connector);
-
-	drm_connector_register(connector);
-}
-
-static struct drm_connector *
-nv50_mstm_add_connector(struct drm_dp_mst_topology_mgr *mgr,
-			struct drm_dp_mst_port *port, const char *path)
-{
-	struct nv50_mstm *mstm = nv50_mstm(mgr);
-	struct nv50_mstc *mstc;
-	int ret;
-
-	ret = nv50_mstc_new(mstm, port, path, &mstc);
-	if (ret) {
-		if (mstc)
-			mstc->connector.funcs->destroy(&mstc->connector);
-		return NULL;
-	}
-
-	return &mstc->connector;
-}
-
-static const struct drm_dp_mst_topology_cbs
-nv50_mstm = {
-	.add_connector = nv50_mstm_add_connector,
-	.register_connector = nv50_mstm_register_connector,
-	.destroy_connector = nv50_mstm_destroy_connector,
-	.hotplug = nv50_mstm_hotplug,
-};
-
-void
-nv50_mstm_service(struct nv50_mstm *mstm)
-{
-	struct drm_dp_aux *aux = mstm ? mstm->mgr.aux : NULL;
-	bool handled = true;
-	int ret;
-	u8 esi[8] = {};
-
-	if (!aux)
-		return;
-
-	while (handled) {
-		ret = drm_dp_dpcd_read(aux, DP_SINK_COUNT_ESI, esi, 8);
-		if (ret != 8) {
-			drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false);
-			return;
-		}
-
-		drm_dp_mst_hpd_irq(&mstm->mgr, esi, &handled);
-		if (!handled)
-			break;
-
-		drm_dp_dpcd_write(aux, DP_SINK_COUNT_ESI + 1, &esi[1], 3);
-	}
-}
-
-void
-nv50_mstm_remove(struct nv50_mstm *mstm)
-{
-	if (mstm)
-		drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false);
-}
-
-static int
-nv50_mstm_enable(struct nv50_mstm *mstm, u8 dpcd, int state)
-{
-	struct nouveau_encoder *outp = mstm->outp;
-	struct {
-		struct nv50_disp_mthd_v1 base;
-		struct nv50_disp_sor_dp_mst_link_v0 mst;
-	} args = {
-		.base.version = 1,
-		.base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_LINK,
-		.base.hasht = outp->dcb->hasht,
-		.base.hashm = outp->dcb->hashm,
-		.mst.state = state,
-	};
-	struct nouveau_drm *drm = nouveau_drm(outp->base.base.dev);
-	struct nvif_object *disp = &drm->display->disp;
-	int ret;
-
-	if (dpcd >= 0x12) {
-		ret = drm_dp_dpcd_readb(mstm->mgr.aux, DP_MSTM_CTRL, &dpcd);
-		if (ret < 0)
-			return ret;
-
-		dpcd &= ~DP_MST_EN;
-		if (state)
-			dpcd |= DP_MST_EN;
-
-		ret = drm_dp_dpcd_writeb(mstm->mgr.aux, DP_MSTM_CTRL, dpcd);
-		if (ret < 0)
-			return ret;
-	}
-
-	return nvif_mthd(disp, 0, &args, sizeof(args));
-}
-
-int
-nv50_mstm_detect(struct nv50_mstm *mstm, u8 dpcd[8], int allow)
-{
-	int ret, state = 0;
-
-	if (!mstm)
-		return 0;
-
-	if (dpcd[0] >= 0x12) {
-		ret = drm_dp_dpcd_readb(mstm->mgr.aux, DP_MSTM_CAP, &dpcd[1]);
-		if (ret < 0)
-			return ret;
-
-		if (!(dpcd[1] & DP_MST_CAP))
-			dpcd[0] = 0x11;
-		else
-			state = allow;
-	}
-
-	ret = nv50_mstm_enable(mstm, dpcd[0], state);
-	if (ret)
-		return ret;
-
-	ret = drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, state);
-	if (ret)
-		return nv50_mstm_enable(mstm, dpcd[0], 0);
-
-	return mstm->mgr.mst_state;
-}
-
-static void
-nv50_mstm_fini(struct nv50_mstm *mstm)
-{
-	if (mstm && mstm->mgr.mst_state)
-		drm_dp_mst_topology_mgr_suspend(&mstm->mgr);
-}
-
-static void
-nv50_mstm_init(struct nv50_mstm *mstm)
-{
-	if (mstm && mstm->mgr.mst_state)
-		drm_dp_mst_topology_mgr_resume(&mstm->mgr);
-}
-
-static void
-nv50_mstm_del(struct nv50_mstm **pmstm)
-{
-	struct nv50_mstm *mstm = *pmstm;
-	if (mstm) {
-		kfree(*pmstm);
-		*pmstm = NULL;
-	}
-}
-
-static int
-nv50_mstm_new(struct nouveau_encoder *outp, struct drm_dp_aux *aux, int aux_max,
-	      int conn_base_id, struct nv50_mstm **pmstm)
-{
-	const int max_payloads = hweight8(outp->dcb->heads);
-	struct drm_device *dev = outp->base.base.dev;
-	struct nv50_mstm *mstm;
-	int ret, i;
-	u8 dpcd;
-
-	/* This is a workaround for some monitors not functioning
-	 * correctly in MST mode on initial module load.  I think
-	 * some bad interaction with the VBIOS may be responsible.
-	 *
-	 * A good ol' off and on again seems to work here ;)
-	 */
-	ret = drm_dp_dpcd_readb(aux, DP_DPCD_REV, &dpcd);
-	if (ret >= 0 && dpcd >= 0x12)
-		drm_dp_dpcd_writeb(aux, DP_MSTM_CTRL, 0);
-
-	if (!(mstm = *pmstm = kzalloc(sizeof(*mstm), GFP_KERNEL)))
-		return -ENOMEM;
-	mstm->outp = outp;
-	mstm->mgr.cbs = &nv50_mstm;
-
-	ret = drm_dp_mst_topology_mgr_init(&mstm->mgr, dev, aux, aux_max,
-					   max_payloads, conn_base_id);
-	if (ret)
-		return ret;
-
-	for (i = 0; i < max_payloads; i++) {
-		ret = nv50_msto_new(dev, outp->dcb->heads, outp->base.base.name,
-				    i, &mstm->msto[i]);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-/******************************************************************************
- * SOR
- *****************************************************************************/
-static void
-nv50_sor_update(struct nouveau_encoder *nv_encoder, u8 head,
-		struct drm_display_mode *mode, u8 proto, u8 depth)
-{
-	struct nv50_dmac *core = &nv50_mast(nv_encoder->base.base.dev)->base;
-	u32 *push;
-
-	if (!mode) {
-		nv_encoder->ctrl &= ~BIT(head);
-		if (!(nv_encoder->ctrl & 0x0000000f))
-			nv_encoder->ctrl = 0;
-	} else {
-		nv_encoder->ctrl |= proto << 8;
-		nv_encoder->ctrl |= BIT(head);
-	}
-
-	if ((push = evo_wait(core, 6))) {
-		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
-			if (mode) {
-				if (mode->flags & DRM_MODE_FLAG_NHSYNC)
-					nv_encoder->ctrl |= 0x00001000;
-				if (mode->flags & DRM_MODE_FLAG_NVSYNC)
-					nv_encoder->ctrl |= 0x00002000;
-				nv_encoder->ctrl |= depth << 16;
-			}
-			evo_mthd(push, 0x0600 + (nv_encoder->or * 0x40), 1);
-		} else {
-			if (mode) {
-				u32 magic = 0x31ec6000 | (head << 25);
-				u32 syncs = 0x00000001;
-				if (mode->flags & DRM_MODE_FLAG_NHSYNC)
-					syncs |= 0x00000008;
-				if (mode->flags & DRM_MODE_FLAG_NVSYNC)
-					syncs |= 0x00000010;
-				if (mode->flags & DRM_MODE_FLAG_INTERLACE)
-					magic |= 0x00000001;
-
-				evo_mthd(push, 0x0404 + (head * 0x300), 2);
-				evo_data(push, syncs | (depth << 6));
-				evo_data(push, magic);
-			}
-			evo_mthd(push, 0x0200 + (nv_encoder->or * 0x20), 1);
-		}
-		evo_data(push, nv_encoder->ctrl);
-		evo_kick(push, core);
-	}
-}
-
-static void
-nv50_sor_disable(struct drm_encoder *encoder)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(nv_encoder->crtc);
-
-	nv_encoder->crtc = NULL;
-
-	if (nv_crtc) {
-		struct nvkm_i2c_aux *aux = nv_encoder->aux;
-		u8 pwr;
-
-		if (aux) {
-			int ret = nvkm_rdaux(aux, DP_SET_POWER, &pwr, 1);
-			if (ret == 0) {
-				pwr &= ~DP_SET_POWER_MASK;
-				pwr |=  DP_SET_POWER_D3;
-				nvkm_wraux(aux, DP_SET_POWER, &pwr, 1);
-			}
-		}
-
-		nv_encoder->update(nv_encoder, nv_crtc->index, NULL, 0, 0);
-		nv50_audio_disable(encoder, nv_crtc);
-		nv50_hdmi_disable(&nv_encoder->base.base, nv_crtc);
-		nv50_outp_release(nv_encoder);
-	}
-}
-
-static void
-nv50_sor_enable(struct drm_encoder *encoder)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
-	struct drm_display_mode *mode = &nv_crtc->base.state->adjusted_mode;
-	struct {
-		struct nv50_disp_mthd_v1 base;
-		struct nv50_disp_sor_lvds_script_v0 lvds;
-	} lvds = {
-		.base.version = 1,
-		.base.method  = NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT,
-		.base.hasht   = nv_encoder->dcb->hasht,
-		.base.hashm   = nv_encoder->dcb->hashm,
-	};
-	struct nv50_disp *disp = nv50_disp(encoder->dev);
-	struct drm_device *dev = encoder->dev;
-	struct nouveau_drm *drm = nouveau_drm(dev);
-	struct nouveau_connector *nv_connector;
-	struct nvbios *bios = &drm->vbios;
-	u8 proto = 0xf;
-	u8 depth = 0x0;
-
-	nv_connector = nouveau_encoder_connector_get(nv_encoder);
-	nv_encoder->crtc = encoder->crtc;
-	nv50_outp_acquire(nv_encoder);
-
-	switch (nv_encoder->dcb->type) {
-	case DCB_OUTPUT_TMDS:
-		if (nv_encoder->link & 1) {
-			proto = 0x1;
-			/* Only enable dual-link if:
-			 *  - Need to (i.e. rate > 165MHz)
-			 *  - DCB says we can
-			 *  - Not an HDMI monitor, since there's no dual-link
-			 *    on HDMI.
-			 */
-			if (mode->clock >= 165000 &&
-			    nv_encoder->dcb->duallink_possible &&
-			    !drm_detect_hdmi_monitor(nv_connector->edid))
-				proto |= 0x4;
-		} else {
-			proto = 0x2;
-		}
-
-		nv50_hdmi_enable(&nv_encoder->base.base, mode);
-		break;
-	case DCB_OUTPUT_LVDS:
-		proto = 0x0;
-
-		if (bios->fp_no_ddc) {
-			if (bios->fp.dual_link)
-				lvds.lvds.script |= 0x0100;
-			if (bios->fp.if_is_24bit)
-				lvds.lvds.script |= 0x0200;
-		} else {
-			if (nv_connector->type == DCB_CONNECTOR_LVDS_SPWG) {
-				if (((u8 *)nv_connector->edid)[121] == 2)
-					lvds.lvds.script |= 0x0100;
-			} else
-			if (mode->clock >= bios->fp.duallink_transition_clk) {
-				lvds.lvds.script |= 0x0100;
-			}
-
-			if (lvds.lvds.script & 0x0100) {
-				if (bios->fp.strapless_is_24bit & 2)
-					lvds.lvds.script |= 0x0200;
-			} else {
-				if (bios->fp.strapless_is_24bit & 1)
-					lvds.lvds.script |= 0x0200;
-			}
-
-			if (nv_connector->base.display_info.bpc == 8)
-				lvds.lvds.script |= 0x0200;
-		}
-
-		nvif_mthd(disp->disp, 0, &lvds, sizeof(lvds));
-		break;
-	case DCB_OUTPUT_DP:
-		if (nv_connector->base.display_info.bpc == 6)
-			depth = 0x2;
-		else
-		if (nv_connector->base.display_info.bpc == 8)
-			depth = 0x5;
-		else
-			depth = 0x6;
-
-		if (nv_encoder->link & 1)
-			proto = 0x8;
-		else
-			proto = 0x9;
-
-		nv50_audio_enable(encoder, mode);
-		break;
-	default:
-		BUG();
-		break;
-	}
-
-	nv_encoder->update(nv_encoder, nv_crtc->index, mode, proto, depth);
-}
-
-static const struct drm_encoder_helper_funcs
-nv50_sor_help = {
-	.atomic_check = nv50_outp_atomic_check,
-	.enable = nv50_sor_enable,
-	.disable = nv50_sor_disable,
-};
-
-static void
-nv50_sor_destroy(struct drm_encoder *encoder)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	nv50_mstm_del(&nv_encoder->dp.mstm);
-	drm_encoder_cleanup(encoder);
-	kfree(encoder);
-}
-
-static const struct drm_encoder_funcs
-nv50_sor_func = {
-	.destroy = nv50_sor_destroy,
-};
-
-static int
-nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe)
-{
-	struct nouveau_connector *nv_connector = nouveau_connector(connector);
-	struct nouveau_drm *drm = nouveau_drm(connector->dev);
-	struct nvkm_i2c *i2c = nvxx_i2c(&drm->client.device);
-	struct nouveau_encoder *nv_encoder;
-	struct drm_encoder *encoder;
-	int type, ret;
-
-	switch (dcbe->type) {
-	case DCB_OUTPUT_LVDS: type = DRM_MODE_ENCODER_LVDS; break;
-	case DCB_OUTPUT_TMDS:
-	case DCB_OUTPUT_DP:
-	default:
-		type = DRM_MODE_ENCODER_TMDS;
-		break;
-	}
-
-	nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
-	if (!nv_encoder)
-		return -ENOMEM;
-	nv_encoder->dcb = dcbe;
-	nv_encoder->update = nv50_sor_update;
-
-	encoder = to_drm_encoder(nv_encoder);
-	encoder->possible_crtcs = dcbe->heads;
-	encoder->possible_clones = 0;
-	drm_encoder_init(connector->dev, encoder, &nv50_sor_func, type,
-			 "sor-%04x-%04x", dcbe->hasht, dcbe->hashm);
-	drm_encoder_helper_add(encoder, &nv50_sor_help);
-
-	drm_mode_connector_attach_encoder(connector, encoder);
-
-	if (dcbe->type == DCB_OUTPUT_DP) {
-		struct nv50_disp *disp = nv50_disp(encoder->dev);
-		struct nvkm_i2c_aux *aux =
-			nvkm_i2c_aux_find(i2c, dcbe->i2c_index);
-		if (aux) {
-			if (disp->disp->oclass < GF110_DISP) {
-				/* HW has no support for address-only
-				 * transactions, so we're required to
-				 * use custom I2C-over-AUX code.
-				 */
-				nv_encoder->i2c = &aux->i2c;
-			} else {
-				nv_encoder->i2c = &nv_connector->aux.ddc;
-			}
-			nv_encoder->aux = aux;
-		}
-
-		/*TODO: Use DP Info Table to check for support. */
-		if (disp->disp->oclass >= GF110_DISP) {
-			ret = nv50_mstm_new(nv_encoder, &nv_connector->aux, 16,
-					    nv_connector->base.base.id,
-					    &nv_encoder->dp.mstm);
-			if (ret)
-				return ret;
-		}
-	} else {
-		struct nvkm_i2c_bus *bus =
-			nvkm_i2c_bus_find(i2c, dcbe->i2c_index);
-		if (bus)
-			nv_encoder->i2c = &bus->i2c;
-	}
-
-	return 0;
-}
-
-/******************************************************************************
- * PIOR
- *****************************************************************************/
-static int
-nv50_pior_atomic_check(struct drm_encoder *encoder,
-		       struct drm_crtc_state *crtc_state,
-		       struct drm_connector_state *conn_state)
-{
-	int ret = nv50_outp_atomic_check(encoder, crtc_state, conn_state);
-	if (ret)
-		return ret;
-	crtc_state->adjusted_mode.clock *= 2;
-	return 0;
-}
-
-static void
-nv50_pior_disable(struct drm_encoder *encoder)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nv50_mast *mast = nv50_mast(encoder->dev);
-	const int or = nv_encoder->or;
-	u32 *push;
-
-	if (nv_encoder->crtc) {
-		push = evo_wait(mast, 4);
-		if (push) {
-			if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-				evo_mthd(push, 0x0700 + (or * 0x040), 1);
-				evo_data(push, 0x00000000);
-			}
-			evo_kick(push, mast);
-		}
-	}
-
-	nv_encoder->crtc = NULL;
-	nv50_outp_release(nv_encoder);
-}
-
-static void
-nv50_pior_enable(struct drm_encoder *encoder)
-{
-	struct nv50_mast *mast = nv50_mast(encoder->dev);
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
-	struct nouveau_connector *nv_connector;
-	struct drm_display_mode *mode = &nv_crtc->base.state->adjusted_mode;
-	u8 owner = 1 << nv_crtc->index;
-	u8 proto, depth;
-	u32 *push;
-
-	nv50_outp_acquire(nv_encoder);
-
-	nv_connector = nouveau_encoder_connector_get(nv_encoder);
-	switch (nv_connector->base.display_info.bpc) {
-	case 10: depth = 0x6; break;
-	case  8: depth = 0x5; break;
-	case  6: depth = 0x2; break;
-	default: depth = 0x0; break;
-	}
-
-	switch (nv_encoder->dcb->type) {
-	case DCB_OUTPUT_TMDS:
-	case DCB_OUTPUT_DP:
-		proto = 0x0;
-		break;
-	default:
-		BUG();
-		break;
-	}
-
-	push = evo_wait(mast, 8);
-	if (push) {
-		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-			u32 ctrl = (depth << 16) | (proto << 8) | owner;
-			if (mode->flags & DRM_MODE_FLAG_NHSYNC)
-				ctrl |= 0x00001000;
-			if (mode->flags & DRM_MODE_FLAG_NVSYNC)
-				ctrl |= 0x00002000;
-			evo_mthd(push, 0x0700 + (nv_encoder->or * 0x040), 1);
-			evo_data(push, ctrl);
-		}
-
-		evo_kick(push, mast);
-	}
-
-	nv_encoder->crtc = encoder->crtc;
-}
-
-static const struct drm_encoder_helper_funcs
-nv50_pior_help = {
-	.atomic_check = nv50_pior_atomic_check,
-	.enable = nv50_pior_enable,
-	.disable = nv50_pior_disable,
-};
-
-static void
-nv50_pior_destroy(struct drm_encoder *encoder)
-{
-	drm_encoder_cleanup(encoder);
-	kfree(encoder);
-}
-
-static const struct drm_encoder_funcs
-nv50_pior_func = {
-	.destroy = nv50_pior_destroy,
-};
-
-static int
-nv50_pior_create(struct drm_connector *connector, struct dcb_output *dcbe)
-{
-	struct nouveau_connector *nv_connector = nouveau_connector(connector);
-	struct nouveau_drm *drm = nouveau_drm(connector->dev);
-	struct nvkm_i2c *i2c = nvxx_i2c(&drm->client.device);
-	struct nvkm_i2c_bus *bus = NULL;
-	struct nvkm_i2c_aux *aux = NULL;
-	struct i2c_adapter *ddc;
-	struct nouveau_encoder *nv_encoder;
-	struct drm_encoder *encoder;
-	int type;
-
-	switch (dcbe->type) {
-	case DCB_OUTPUT_TMDS:
-		bus  = nvkm_i2c_bus_find(i2c, NVKM_I2C_BUS_EXT(dcbe->extdev));
-		ddc  = bus ? &bus->i2c : NULL;
-		type = DRM_MODE_ENCODER_TMDS;
-		break;
-	case DCB_OUTPUT_DP:
-		aux  = nvkm_i2c_aux_find(i2c, NVKM_I2C_AUX_EXT(dcbe->extdev));
-		ddc  = aux ? &nv_connector->aux.ddc : NULL;
-		type = DRM_MODE_ENCODER_TMDS;
-		break;
-	default:
-		return -ENODEV;
-	}
-
-	nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
-	if (!nv_encoder)
-		return -ENOMEM;
-	nv_encoder->dcb = dcbe;
-	nv_encoder->i2c = ddc;
-	nv_encoder->aux = aux;
-
-	encoder = to_drm_encoder(nv_encoder);
-	encoder->possible_crtcs = dcbe->heads;
-	encoder->possible_clones = 0;
-	drm_encoder_init(connector->dev, encoder, &nv50_pior_func, type,
-			 "pior-%04x-%04x", dcbe->hasht, dcbe->hashm);
-	drm_encoder_helper_add(encoder, &nv50_pior_help);
-
-	drm_mode_connector_attach_encoder(connector, encoder);
-	return 0;
-}
-
-/******************************************************************************
- * Atomic
- *****************************************************************************/
-
-static void
-nv50_disp_atomic_commit_core(struct nouveau_drm *drm, u32 interlock)
-{
-	struct nv50_disp *disp = nv50_disp(drm->dev);
-	struct nv50_dmac *core = &disp->mast.base;
-	struct nv50_mstm *mstm;
-	struct drm_encoder *encoder;
-	u32 *push;
-
-	NV_ATOMIC(drm, "commit core %08x\n", interlock);
-
-	drm_for_each_encoder(encoder, drm->dev) {
-		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
-			mstm = nouveau_encoder(encoder)->dp.mstm;
-			if (mstm && mstm->modified)
-				nv50_mstm_prepare(mstm);
-		}
-	}
-
-	if ((push = evo_wait(core, 5))) {
-		evo_mthd(push, 0x0084, 1);
-		evo_data(push, 0x80000000);
-		evo_mthd(push, 0x0080, 2);
-		evo_data(push, interlock);
-		evo_data(push, 0x00000000);
-		nouveau_bo_wr32(disp->sync, 0, 0x00000000);
-		evo_kick(push, core);
-		if (nvif_msec(&drm->client.device, 2000ULL,
-			if (nouveau_bo_rd32(disp->sync, 0))
-				break;
-			usleep_range(1, 2);
-		) < 0)
-			NV_ERROR(drm, "EVO timeout\n");
-	}
-
-	drm_for_each_encoder(encoder, drm->dev) {
-		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
-			mstm = nouveau_encoder(encoder)->dp.mstm;
-			if (mstm && mstm->modified)
-				nv50_mstm_cleanup(mstm);
-		}
-	}
-}
-
-static void
-nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
-{
-	struct drm_device *dev = state->dev;
-	struct drm_crtc_state *new_crtc_state, *old_crtc_state;
-	struct drm_crtc *crtc;
-	struct drm_plane_state *new_plane_state;
-	struct drm_plane *plane;
-	struct nouveau_drm *drm = nouveau_drm(dev);
-	struct nv50_disp *disp = nv50_disp(dev);
-	struct nv50_atom *atom = nv50_atom(state);
-	struct nv50_outp_atom *outp, *outt;
-	u32 interlock_core = 0;
-	u32 interlock_chan = 0;
-	int i;
-
-	NV_ATOMIC(drm, "commit %d %d\n", atom->lock_core, atom->flush_disable);
-	drm_atomic_helper_wait_for_fences(dev, state, false);
-	drm_atomic_helper_wait_for_dependencies(state);
-	drm_atomic_helper_update_legacy_modeset_state(dev, state);
-
-	if (atom->lock_core)
-		mutex_lock(&disp->mutex);
-
-	/* Disable head(s). */
-	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
-		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
-		struct nv50_head *head = nv50_head(crtc);
-
-		NV_ATOMIC(drm, "%s: clr %04x (set %04x)\n", crtc->name,
-			  asyh->clr.mask, asyh->set.mask);
-		if (old_crtc_state->active && !new_crtc_state->active)
-			drm_crtc_vblank_off(crtc);
-
-		if (asyh->clr.mask) {
-			nv50_head_flush_clr(head, asyh, atom->flush_disable);
-			interlock_core |= 1;
-		}
-	}
-
-	/* Disable plane(s). */
-	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
-		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
-		struct nv50_wndw *wndw = nv50_wndw(plane);
-
-		NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", plane->name,
-			  asyw->clr.mask, asyw->set.mask);
-		if (!asyw->clr.mask)
-			continue;
-
-		interlock_chan |= nv50_wndw_flush_clr(wndw, interlock_core,
-						      atom->flush_disable,
-						      asyw);
-	}
-
-	/* Disable output path(s). */
-	list_for_each_entry(outp, &atom->outp, head) {
-		const struct drm_encoder_helper_funcs *help;
-		struct drm_encoder *encoder;
-
-		encoder = outp->encoder;
-		help = encoder->helper_private;
-
-		NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", encoder->name,
-			  outp->clr.mask, outp->set.mask);
-
-		if (outp->clr.mask) {
-			help->disable(encoder);
-			interlock_core |= 1;
-			if (outp->flush_disable) {
-				nv50_disp_atomic_commit_core(drm, interlock_chan);
-				interlock_core = 0;
-				interlock_chan = 0;
-			}
-		}
-	}
-
-	/* Flush disable. */
-	if (interlock_core) {
-		if (atom->flush_disable) {
-			nv50_disp_atomic_commit_core(drm, interlock_chan);
-			interlock_core = 0;
-			interlock_chan = 0;
-		}
-	}
-
-	/* Update output path(s). */
-	list_for_each_entry_safe(outp, outt, &atom->outp, head) {
-		const struct drm_encoder_helper_funcs *help;
-		struct drm_encoder *encoder;
-
-		encoder = outp->encoder;
-		help = encoder->helper_private;
-
-		NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", encoder->name,
-			  outp->set.mask, outp->clr.mask);
-
-		if (outp->set.mask) {
-			help->enable(encoder);
-			interlock_core = 1;
-		}
-
-		list_del(&outp->head);
-		kfree(outp);
-	}
-
-	/* Update head(s). */
-	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
-		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
-		struct nv50_head *head = nv50_head(crtc);
-
-		NV_ATOMIC(drm, "%s: set %04x (clr %04x)\n", crtc->name,
-			  asyh->set.mask, asyh->clr.mask);
-
-		if (asyh->set.mask) {
-			nv50_head_flush_set(head, asyh);
-			interlock_core = 1;
-		}
-
-		if (new_crtc_state->active) {
-			if (!old_crtc_state->active)
-				drm_crtc_vblank_on(crtc);
-			if (new_crtc_state->event)
-				drm_crtc_vblank_get(crtc);
-		}
-	}
-
-	/* Update plane(s). */
-	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
-		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
-		struct nv50_wndw *wndw = nv50_wndw(plane);
-
-		NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", plane->name,
-			  asyw->set.mask, asyw->clr.mask);
-		if ( !asyw->set.mask &&
-		    (!asyw->clr.mask || atom->flush_disable))
-			continue;
-
-		interlock_chan |= nv50_wndw_flush_set(wndw, interlock_core, asyw);
-	}
-
-	/* Flush update. */
-	if (interlock_core) {
-		if (!interlock_chan && atom->state.legacy_cursor_update) {
-			u32 *push = evo_wait(&disp->mast, 2);
-			if (push) {
-				evo_mthd(push, 0x0080, 1);
-				evo_data(push, 0x00000000);
-				evo_kick(push, &disp->mast);
-			}
-		} else {
-			nv50_disp_atomic_commit_core(drm, interlock_chan);
-		}
-	}
-
-	if (atom->lock_core)
-		mutex_unlock(&disp->mutex);
-
-	/* Wait for HW to signal completion. */
-	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
-		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
-		struct nv50_wndw *wndw = nv50_wndw(plane);
-		int ret = nv50_wndw_wait_armed(wndw, asyw);
-		if (ret)
-			NV_ERROR(drm, "%s: timeout\n", plane->name);
-	}
-
-	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
-		if (new_crtc_state->event) {
-			unsigned long flags;
-			/* Get correct count/ts if racing with vblank irq */
-			if (new_crtc_state->active)
-				drm_crtc_accurate_vblank_count(crtc);
-			spin_lock_irqsave(&crtc->dev->event_lock, flags);
-			drm_crtc_send_vblank_event(crtc, new_crtc_state->event);
-			spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
-
-			new_crtc_state->event = NULL;
-			if (new_crtc_state->active)
-				drm_crtc_vblank_put(crtc);
-		}
-	}
-
-	drm_atomic_helper_commit_hw_done(state);
-	drm_atomic_helper_cleanup_planes(dev, state);
-	drm_atomic_helper_commit_cleanup_done(state);
-	drm_atomic_state_put(state);
-}
-
-static void
-nv50_disp_atomic_commit_work(struct work_struct *work)
-{
-	struct drm_atomic_state *state =
-		container_of(work, typeof(*state), commit_work);
-	nv50_disp_atomic_commit_tail(state);
-}
-
-static int
-nv50_disp_atomic_commit(struct drm_device *dev,
-			struct drm_atomic_state *state, bool nonblock)
-{
-	struct nouveau_drm *drm = nouveau_drm(dev);
-	struct nv50_disp *disp = nv50_disp(dev);
-	struct drm_plane_state *new_plane_state;
-	struct drm_plane *plane;
-	struct drm_crtc *crtc;
-	bool active = false;
-	int ret, i;
-
-	ret = pm_runtime_get_sync(dev->dev);
-	if (ret < 0 && ret != -EACCES)
-		return ret;
-
-	ret = drm_atomic_helper_setup_commit(state, nonblock);
-	if (ret)
-		goto done;
-
-	INIT_WORK(&state->commit_work, nv50_disp_atomic_commit_work);
-
-	ret = drm_atomic_helper_prepare_planes(dev, state);
-	if (ret)
-		goto done;
-
-	if (!nonblock) {
-		ret = drm_atomic_helper_wait_for_fences(dev, state, true);
-		if (ret)
-			goto err_cleanup;
-	}
-
-	ret = drm_atomic_helper_swap_state(state, true);
-	if (ret)
-		goto err_cleanup;
-
-	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
-		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
-		struct nv50_wndw *wndw = nv50_wndw(plane);
-
-		if (asyw->set.image) {
-			asyw->ntfy.handle = wndw->dmac->sync.handle;
-			asyw->ntfy.offset = wndw->ntfy;
-			asyw->ntfy.awaken = false;
-			asyw->set.ntfy = true;
-			nouveau_bo_wr32(disp->sync, wndw->ntfy / 4, 0x00000000);
-			wndw->ntfy ^= 0x10;
-		}
-	}
-
-	drm_atomic_state_get(state);
-
-	if (nonblock)
-		queue_work(system_unbound_wq, &state->commit_work);
-	else
-		nv50_disp_atomic_commit_tail(state);
-
-	drm_for_each_crtc(crtc, dev) {
-		if (crtc->state->enable) {
-			if (!drm->have_disp_power_ref) {
-				drm->have_disp_power_ref = true;
-				return 0;
-			}
-			active = true;
-			break;
-		}
-	}
-
-	if (!active && drm->have_disp_power_ref) {
-		pm_runtime_put_autosuspend(dev->dev);
-		drm->have_disp_power_ref = false;
-	}
-
-err_cleanup:
-	if (ret)
-		drm_atomic_helper_cleanup_planes(dev, state);
-done:
-	pm_runtime_put_autosuspend(dev->dev);
-	return ret;
-}
-
-static struct nv50_outp_atom *
-nv50_disp_outp_atomic_add(struct nv50_atom *atom, struct drm_encoder *encoder)
-{
-	struct nv50_outp_atom *outp;
-
-	list_for_each_entry(outp, &atom->outp, head) {
-		if (outp->encoder == encoder)
-			return outp;
-	}
-
-	outp = kzalloc(sizeof(*outp), GFP_KERNEL);
-	if (!outp)
-		return ERR_PTR(-ENOMEM);
-
-	list_add(&outp->head, &atom->outp);
-	outp->encoder = encoder;
-	return outp;
-}
-
-static int
-nv50_disp_outp_atomic_check_clr(struct nv50_atom *atom,
-				struct drm_connector_state *old_connector_state)
-{
-	struct drm_encoder *encoder = old_connector_state->best_encoder;
-	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
-	struct drm_crtc *crtc;
-	struct nv50_outp_atom *outp;
-
-	if (!(crtc = old_connector_state->crtc))
-		return 0;
-
-	old_crtc_state = drm_atomic_get_old_crtc_state(&atom->state, crtc);
-	new_crtc_state = drm_atomic_get_new_crtc_state(&atom->state, crtc);
-	if (old_crtc_state->active && drm_atomic_crtc_needs_modeset(new_crtc_state)) {
-		outp = nv50_disp_outp_atomic_add(atom, encoder);
-		if (IS_ERR(outp))
-			return PTR_ERR(outp);
-
-		if (outp->encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
-			outp->flush_disable = true;
-			atom->flush_disable = true;
-		}
-		outp->clr.ctrl = true;
-		atom->lock_core = true;
-	}
-
-	return 0;
-}
-
-static int
-nv50_disp_outp_atomic_check_set(struct nv50_atom *atom,
-				struct drm_connector_state *connector_state)
-{
-	struct drm_encoder *encoder = connector_state->best_encoder;
-	struct drm_crtc_state *new_crtc_state;
-	struct drm_crtc *crtc;
-	struct nv50_outp_atom *outp;
-
-	if (!(crtc = connector_state->crtc))
-		return 0;
-
-	new_crtc_state = drm_atomic_get_new_crtc_state(&atom->state, crtc);
-	if (new_crtc_state->active && drm_atomic_crtc_needs_modeset(new_crtc_state)) {
-		outp = nv50_disp_outp_atomic_add(atom, encoder);
-		if (IS_ERR(outp))
-			return PTR_ERR(outp);
-
-		outp->set.ctrl = true;
-		atom->lock_core = true;
-	}
-
-	return 0;
-}
-
-static int
-nv50_disp_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
-{
-	struct nv50_atom *atom = nv50_atom(state);
-	struct drm_connector_state *old_connector_state, *new_connector_state;
-	struct drm_connector *connector;
-	int ret, i;
-
-	ret = drm_atomic_helper_check(dev, state);
-	if (ret)
-		return ret;
-
-	for_each_oldnew_connector_in_state(state, connector, old_connector_state, new_connector_state, i) {
-		ret = nv50_disp_outp_atomic_check_clr(atom, old_connector_state);
-		if (ret)
-			return ret;
-
-		ret = nv50_disp_outp_atomic_check_set(atom, new_connector_state);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static void
-nv50_disp_atomic_state_clear(struct drm_atomic_state *state)
-{
-	struct nv50_atom *atom = nv50_atom(state);
-	struct nv50_outp_atom *outp, *outt;
-
-	list_for_each_entry_safe(outp, outt, &atom->outp, head) {
-		list_del(&outp->head);
-		kfree(outp);
-	}
-
-	drm_atomic_state_default_clear(state);
-}
-
-static void
-nv50_disp_atomic_state_free(struct drm_atomic_state *state)
-{
-	struct nv50_atom *atom = nv50_atom(state);
-	drm_atomic_state_default_release(&atom->state);
-	kfree(atom);
-}
-
-static struct drm_atomic_state *
-nv50_disp_atomic_state_alloc(struct drm_device *dev)
-{
-	struct nv50_atom *atom;
-	if (!(atom = kzalloc(sizeof(*atom), GFP_KERNEL)) ||
-	    drm_atomic_state_init(dev, &atom->state) < 0) {
-		kfree(atom);
-		return NULL;
-	}
-	INIT_LIST_HEAD(&atom->outp);
-	return &atom->state;
-}
-
-static const struct drm_mode_config_funcs
-nv50_disp_func = {
-	.fb_create = nouveau_user_framebuffer_create,
-	.output_poll_changed = drm_fb_helper_output_poll_changed,
-	.atomic_check = nv50_disp_atomic_check,
-	.atomic_commit = nv50_disp_atomic_commit,
-	.atomic_state_alloc = nv50_disp_atomic_state_alloc,
-	.atomic_state_clear = nv50_disp_atomic_state_clear,
-	.atomic_state_free = nv50_disp_atomic_state_free,
-};
-
-/******************************************************************************
- * Init
- *****************************************************************************/
-
-void
-nv50_display_fini(struct drm_device *dev)
-{
-	struct nouveau_encoder *nv_encoder;
-	struct drm_encoder *encoder;
-	struct drm_plane *plane;
-
-	drm_for_each_plane(plane, dev) {
-		struct nv50_wndw *wndw = nv50_wndw(plane);
-		if (plane->funcs != &nv50_wndw)
-			continue;
-		nv50_wndw_fini(wndw);
-	}
-
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
-			nv_encoder = nouveau_encoder(encoder);
-			nv50_mstm_fini(nv_encoder->dp.mstm);
-		}
-	}
-}
-
-int
-nv50_display_init(struct drm_device *dev)
-{
-	struct drm_encoder *encoder;
-	struct drm_plane *plane;
-	u32 *push;
-
-	push = evo_wait(nv50_mast(dev), 32);
-	if (!push)
-		return -EBUSY;
-
-	evo_mthd(push, 0x0088, 1);
-	evo_data(push, nv50_mast(dev)->base.sync.handle);
-	evo_kick(push, nv50_mast(dev));
-
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
-			struct nouveau_encoder *nv_encoder =
-				nouveau_encoder(encoder);
-			nv50_mstm_init(nv_encoder->dp.mstm);
-		}
-	}
-
-	drm_for_each_plane(plane, dev) {
-		struct nv50_wndw *wndw = nv50_wndw(plane);
-		if (plane->funcs != &nv50_wndw)
-			continue;
-		nv50_wndw_init(wndw);
-	}
-
-	return 0;
-}
-
-void
-nv50_display_destroy(struct drm_device *dev)
-{
-	struct nv50_disp *disp = nv50_disp(dev);
-
-	nv50_dmac_destroy(&disp->mast.base, disp->disp);
-
-	nouveau_bo_unmap(disp->sync);
-	if (disp->sync)
-		nouveau_bo_unpin(disp->sync);
-	nouveau_bo_ref(NULL, &disp->sync);
-
-	nouveau_display(dev)->priv = NULL;
-	kfree(disp);
-}
-
-MODULE_PARM_DESC(atomic, "Expose atomic ioctl (default: disabled)");
-static int nouveau_atomic = 0;
-module_param_named(atomic, nouveau_atomic, int, 0400);
-
-int
-nv50_display_create(struct drm_device *dev)
-{
-	struct nvif_device *device = &nouveau_drm(dev)->client.device;
-	struct nouveau_drm *drm = nouveau_drm(dev);
-	struct dcb_table *dcb = &drm->vbios.dcb;
-	struct drm_connector *connector, *tmp;
-	struct nv50_disp *disp;
-	struct dcb_output *dcbe;
-	int crtcs, ret, i;
-
-	disp = kzalloc(sizeof(*disp), GFP_KERNEL);
-	if (!disp)
-		return -ENOMEM;
-
-	mutex_init(&disp->mutex);
-
-	nouveau_display(dev)->priv = disp;
-	nouveau_display(dev)->dtor = nv50_display_destroy;
-	nouveau_display(dev)->init = nv50_display_init;
-	nouveau_display(dev)->fini = nv50_display_fini;
-	disp->disp = &nouveau_display(dev)->disp;
-	dev->mode_config.funcs = &nv50_disp_func;
-	dev->driver->driver_features |= DRIVER_PREFER_XBGR_30BPP;
-	if (nouveau_atomic)
-		dev->driver->driver_features |= DRIVER_ATOMIC;
-
-	/* small shared memory area we use for notifiers and semaphores */
-	ret = nouveau_bo_new(&drm->client, 4096, 0x1000, TTM_PL_FLAG_VRAM,
-			     0, 0x0000, NULL, NULL, &disp->sync);
-	if (!ret) {
-		ret = nouveau_bo_pin(disp->sync, TTM_PL_FLAG_VRAM, true);
-		if (!ret) {
-			ret = nouveau_bo_map(disp->sync);
-			if (ret)
-				nouveau_bo_unpin(disp->sync);
-		}
-		if (ret)
-			nouveau_bo_ref(NULL, &disp->sync);
-	}
-
-	if (ret)
-		goto out;
-
-	/* allocate master evo channel */
-	ret = nv50_core_create(device, disp->disp, disp->sync->bo.offset,
-			      &disp->mast);
-	if (ret)
-		goto out;
-
-	/* create crtc objects to represent the hw heads */
-	if (disp->disp->oclass >= GF110_DISP)
-		crtcs = nvif_rd32(&device->object, 0x612004) & 0xf;
-	else
-		crtcs = 0x3;
-
-	for (i = 0; i < fls(crtcs); i++) {
-		if (!(crtcs & (1 << i)))
-			continue;
-		ret = nv50_head_create(dev, i);
-		if (ret)
-			goto out;
-	}
-
-	/* create encoder/connector objects based on VBIOS DCB table */
-	for (i = 0, dcbe = &dcb->entry[0]; i < dcb->entries; i++, dcbe++) {
-		connector = nouveau_connector_create(dev, dcbe->connector);
-		if (IS_ERR(connector))
-			continue;
-
-		if (dcbe->location == DCB_LOC_ON_CHIP) {
-			switch (dcbe->type) {
-			case DCB_OUTPUT_TMDS:
-			case DCB_OUTPUT_LVDS:
-			case DCB_OUTPUT_DP:
-				ret = nv50_sor_create(connector, dcbe);
-				break;
-			case DCB_OUTPUT_ANALOG:
-				ret = nv50_dac_create(connector, dcbe);
-				break;
-			default:
-				ret = -ENODEV;
-				break;
-			}
-		} else {
-			ret = nv50_pior_create(connector, dcbe);
-		}
-
-		if (ret) {
-			NV_WARN(drm, "failed to create encoder %d/%d/%d: %d\n",
-				     dcbe->location, dcbe->type,
-				     ffs(dcbe->or) - 1, ret);
-			ret = 0;
-		}
-	}
-
-	/* cull any connectors we created that don't have an encoder */
-	list_for_each_entry_safe(connector, tmp, &dev->mode_config.connector_list, head) {
-		if (connector->encoder_ids[0])
-			continue;
-
-		NV_WARN(drm, "%s has no encoders, removing\n",
-			connector->name);
-		connector->funcs->destroy(connector);
-	}
-
-out:
-	if (ret)
-		nv50_display_destroy(dev);
-	return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/nv50_display.h b/drivers/gpu/drm/nouveau/nv50_display.h
index 918187cee84b..fbd3b15583bc 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.h
+++ b/drivers/gpu/drm/nouveau/nv50_display.h
@@ -28,7 +28,6 @@
 #define __NV50_DISPLAY_H__
 
 #include "nouveau_display.h"
-#include "nouveau_crtc.h"
 #include "nouveau_reg.h"
 
 int  nv50_display_create(struct drm_device *);
diff --git a/drivers/gpu/drm/nouveau/nv50_fence.c b/drivers/gpu/drm/nouveau/nv50_fence.c
index a369d978e267..a00ecc3de053 100644
--- a/drivers/gpu/drm/nouveau/nv50_fence.c
+++ b/drivers/gpu/drm/nouveau/nv50_fence.c
@@ -78,8 +78,6 @@ nv50_fence_create(struct nouveau_drm *drm)
 	priv->base.resume = nv17_fence_resume;
 	priv->base.context_new = nv50_fence_context_new;
 	priv->base.context_del = nv10_fence_context_del;
-	priv->base.contexts = 127;
-	priv->base.context_base = dma_fence_context_alloc(priv->base.contexts);
 	spin_lock_init(&priv->lock);
 
 	ret = nouveau_bo_new(&drm->client, 4096, 0x1000, TTM_PL_FLAG_VRAM,
diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c
index 5f0c0c27d5dc..090664899247 100644
--- a/drivers/gpu/drm/nouveau/nv84_fence.c
+++ b/drivers/gpu/drm/nouveau/nv84_fence.c
@@ -141,9 +141,9 @@ nv84_fence_suspend(struct nouveau_drm *drm)
 	struct nv84_fence_priv *priv = drm->fence;
 	int i;
 
-	priv->suspend = vmalloc(priv->base.contexts * sizeof(u32));
+	priv->suspend = vmalloc(drm->chan.nr * sizeof(u32));
 	if (priv->suspend) {
-		for (i = 0; i < priv->base.contexts; i++)
+		for (i = 0; i < drm->chan.nr; i++)
 			priv->suspend[i] = nouveau_bo_rd32(priv->bo, i*4);
 	}
 
@@ -157,7 +157,7 @@ nv84_fence_resume(struct nouveau_drm *drm)
 	int i;
 
 	if (priv->suspend) {
-		for (i = 0; i < priv->base.contexts; i++)
+		for (i = 0; i < drm->chan.nr; i++)
 			nouveau_bo_wr32(priv->bo, i*4, priv->suspend[i]);
 		vfree(priv->suspend);
 		priv->suspend = NULL;
@@ -179,7 +179,6 @@ nv84_fence_destroy(struct nouveau_drm *drm)
 int
 nv84_fence_create(struct nouveau_drm *drm)
 {
-	struct nvkm_fifo *fifo = nvxx_fifo(&drm->client.device);
 	struct nv84_fence_priv *priv;
 	u32 domain;
 	int ret;
@@ -194,8 +193,6 @@ nv84_fence_create(struct nouveau_drm *drm)
 	priv->base.context_new = nv84_fence_context_new;
 	priv->base.context_del = nv84_fence_context_del;
 
-	priv->base.contexts = fifo->nr;
-	priv->base.context_base = dma_fence_context_alloc(priv->base.contexts);
 	priv->base.uevent = true;
 
 	mutex_init(&priv->mutex);
@@ -207,7 +204,7 @@ nv84_fence_create(struct nouveau_drm *drm)
 			  * will lose CPU/GPU coherency!
 			  */
 			 TTM_PL_FLAG_TT | TTM_PL_FLAG_UNCACHED;
-	ret = nouveau_bo_new(&drm->client, 16 * priv->base.contexts, 0,
+	ret = nouveau_bo_new(&drm->client, 16 * drm->chan.nr, 0,
 			     domain, 0, 0, NULL, NULL, &priv->bo);
 	if (ret == 0) {
 		ret = nouveau_bo_pin(priv->bo, domain, false);
diff --git a/drivers/gpu/drm/nouveau/nvif/Kbuild b/drivers/gpu/drm/nouveau/nvif/Kbuild
index f1675a4ab6fa..42e8c85caa33 100644
--- a/drivers/gpu/drm/nouveau/nvif/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvif/Kbuild
@@ -1,8 +1,14 @@
 nvif-y := nvif/object.o
 nvif-y += nvif/client.o
 nvif-y += nvif/device.o
+nvif-y += nvif/disp.o
 nvif-y += nvif/driver.o
+nvif-y += nvif/fifo.o
 nvif-y += nvif/mem.o
 nvif-y += nvif/mmu.o
 nvif-y += nvif/notify.o
 nvif-y += nvif/vmm.o
+
+# Usermode classes
+nvif-y += nvif/user.o
+nvif-y += nvif/userc361.o
diff --git a/drivers/gpu/drm/nouveau/nvif/device.c b/drivers/gpu/drm/nouveau/nvif/device.c
index 252d8c33215b..1ec101ba3b42 100644
--- a/drivers/gpu/drm/nouveau/nvif/device.c
+++ b/drivers/gpu/drm/nouveau/nvif/device.c
@@ -37,6 +37,9 @@ nvif_device_time(struct nvif_device *device)
 void
 nvif_device_fini(struct nvif_device *device)
 {
+	nvif_user_fini(device);
+	kfree(device->runlist);
+	device->runlist = NULL;
 	nvif_object_fini(&device->object);
 }
 
@@ -46,6 +49,8 @@ nvif_device_init(struct nvif_object *parent, u32 handle, s32 oclass,
 {
 	int ret = nvif_object_init(parent, handle, oclass, data, size,
 				   &device->object);
+	device->runlist = NULL;
+	device->user.func = NULL;
 	if (ret == 0) {
 		device->info.version = 0;
 		ret = nvif_object_mthd(&device->object, NV_DEVICE_V0_INFO,
diff --git a/drivers/gpu/drm/nouveau/nvif/disp.c b/drivers/gpu/drm/nouveau/nvif/disp.c
new file mode 100644
index 000000000000..18c7d064f75c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/disp.c
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <nvif/disp.h>
+#include <nvif/device.h>
+
+#include <nvif/class.h>
+
+void
+nvif_disp_dtor(struct nvif_disp *disp)
+{
+	nvif_object_fini(&disp->object);
+}
+
+int
+nvif_disp_ctor(struct nvif_device *device, s32 oclass, struct nvif_disp *disp)
+{
+	static const struct nvif_mclass disps[] = {
+		{ GV100_DISP, -1 },
+		{ GP102_DISP, -1 },
+		{ GP100_DISP, -1 },
+		{ GM200_DISP, -1 },
+		{ GM107_DISP, -1 },
+		{ GK110_DISP, -1 },
+		{ GK104_DISP, -1 },
+		{ GF110_DISP, -1 },
+		{ GT214_DISP, -1 },
+		{ GT206_DISP, -1 },
+		{ GT200_DISP, -1 },
+		{   G82_DISP, -1 },
+		{  NV50_DISP, -1 },
+		{  NV04_DISP, -1 },
+		{}
+	};
+	int cid = nvif_sclass(&device->object, disps, oclass);
+	disp->object.client = NULL;
+	if (cid < 0)
+		return cid;
+
+	return nvif_object_init(&device->object, 0, disps[cid].oclass,
+				NULL, 0, &disp->object);
+}
diff --git a/drivers/gpu/drm/nouveau/nvif/fifo.c b/drivers/gpu/drm/nouveau/nvif/fifo.c
new file mode 100644
index 000000000000..99d4fd17543c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/fifo.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <nvif/fifo.h>
+
+static int
+nvif_fifo_runlists(struct nvif_device *device)
+{
+	struct nvif_object *object = &device->object;
+	struct {
+		struct nv_device_info_v1 m;
+		struct {
+			struct nv_device_info_v1_data runlists;
+			struct nv_device_info_v1_data runlist[64];
+		} v;
+	} *a;
+	int ret, i;
+
+	if (device->runlist)
+		return 0;
+
+	if (!(a = kmalloc(sizeof(*a), GFP_KERNEL)))
+		return -ENOMEM;
+	a->m.version = 1;
+	a->m.count = sizeof(a->v) / sizeof(a->v.runlists);
+	a->v.runlists.mthd = NV_DEVICE_FIFO_RUNLISTS;
+	for (i = 0; i < ARRAY_SIZE(a->v.runlist); i++)
+		a->v.runlist[i].mthd = NV_DEVICE_FIFO_RUNLIST_ENGINES(i);
+
+	ret = nvif_object_mthd(object, NV_DEVICE_V0_INFO, a, sizeof(*a));
+	if (ret)
+		goto done;
+
+	device->runlists = fls64(a->v.runlists.data);
+	device->runlist = kzalloc(sizeof(*device->runlist) *
+				  device->runlists, GFP_KERNEL);
+	if (!device->runlist) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	for (i = 0; i < device->runlists; i++) {
+		if (a->v.runlists.data & BIT_ULL(i))
+			device->runlist[i].engines = a->v.runlist[i].data;
+	}
+
+done:
+	kfree(a);
+	return ret;
+}
+
+u64
+nvif_fifo_runlist(struct nvif_device *device, u64 engine)
+{
+	struct nvif_object *object = &device->object;
+	struct {
+		struct nv_device_info_v1 m;
+		struct {
+			struct nv_device_info_v1_data engine;
+		} v;
+	} a = {
+		.m.version = 1,
+		.m.count = sizeof(a.v) / sizeof(a.v.engine),
+		.v.engine.mthd = engine,
+	};
+	u64 runm = 0;
+	int ret, i;
+
+	if ((ret = nvif_fifo_runlists(device)))
+		return runm;
+
+	ret = nvif_object_mthd(object, NV_DEVICE_V0_INFO, &a, sizeof(a));
+	if (ret == 0) {
+		for (i = 0; i < device->runlists; i++) {
+			if (device->runlist[i].engines & a.v.engine.data)
+				runm |= BIT_ULL(i);
+		}
+	}
+
+	return runm;
+}
diff --git a/drivers/gpu/drm/nouveau/nvif/mem.c b/drivers/gpu/drm/nouveau/nvif/mem.c
index 0f9382c60145..b6ebb3b58673 100644
--- a/drivers/gpu/drm/nouveau/nvif/mem.c
+++ b/drivers/gpu/drm/nouveau/nvif/mem.c
@@ -24,6 +24,19 @@
 
 #include <nvif/if000a.h>
 
+int
+nvif_mem_init_map(struct nvif_mmu *mmu, u8 type, u64 size, struct nvif_mem *mem)
+{
+	int ret = nvif_mem_init(mmu, mmu->mem, NVIF_MEM_MAPPABLE | type, 0,
+				size, NULL, 0, mem);
+	if (ret == 0) {
+		ret = nvif_object_map(&mem->object, NULL, 0);
+		if (ret)
+			nvif_mem_fini(mem);
+	}
+	return ret;
+}
+
 void
 nvif_mem_fini(struct nvif_mem *mem)
 {
diff --git a/drivers/gpu/drm/nouveau/nvif/mmu.c b/drivers/gpu/drm/nouveau/nvif/mmu.c
index 15d0dcbf7ab4..358ac4f3cf91 100644
--- a/drivers/gpu/drm/nouveau/nvif/mmu.c
+++ b/drivers/gpu/drm/nouveau/nvif/mmu.c
@@ -36,6 +36,12 @@ nvif_mmu_fini(struct nvif_mmu *mmu)
 int
 nvif_mmu_init(struct nvif_object *parent, s32 oclass, struct nvif_mmu *mmu)
 {
+	static const struct nvif_mclass mems[] = {
+		{ NVIF_CLASS_MEM_GF100, -1 },
+		{ NVIF_CLASS_MEM_NV50 , -1 },
+		{ NVIF_CLASS_MEM_NV04 , -1 },
+		{}
+	};
 	struct nvif_mmu_v0 args;
 	int ret, i;
 
@@ -54,6 +60,11 @@ nvif_mmu_init(struct nvif_object *parent, s32 oclass, struct nvif_mmu *mmu)
 	mmu->type_nr = args.type_nr;
 	mmu->kind_nr = args.kind_nr;
 
+	ret = nvif_mclass(&mmu->object, mems);
+	if (ret < 0)
+		goto done;
+	mmu->mem = mems[ret].oclass;
+
 	mmu->heap = kmalloc(sizeof(*mmu->heap) * mmu->heap_nr, GFP_KERNEL);
 	mmu->type = kmalloc(sizeof(*mmu->type) * mmu->type_nr, GFP_KERNEL);
 	if (ret = -ENOMEM, !mmu->heap || !mmu->type)
diff --git a/drivers/gpu/drm/nouveau/nvif/user.c b/drivers/gpu/drm/nouveau/nvif/user.c
new file mode 100644
index 000000000000..10da3cdca647
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/user.c
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <nvif/user.h>
+#include <nvif/device.h>
+
+#include <nvif/class.h>
+
+void
+nvif_user_fini(struct nvif_device *device)
+{
+	if (device->user.func) {
+		nvif_object_fini(&device->user.object);
+		device->user.func = NULL;
+	}
+}
+
+int
+nvif_user_init(struct nvif_device *device)
+{
+	struct {
+		s32 oclass;
+		int version;
+		const struct nvif_user_func *func;
+	} users[] = {
+		{ VOLTA_USERMODE_A, -1, &nvif_userc361 },
+		{}
+	};
+	int cid, ret;
+
+	if (device->user.func)
+		return 0;
+
+	cid = nvif_mclass(&device->object, users);
+	if (cid < 0)
+		return cid;
+
+	ret = nvif_object_init(&device->object, 0, users[cid].oclass, NULL, 0,
+			       &device->user.object);
+	if (ret)
+		return ret;
+
+	nvif_object_map(&device->user.object, NULL, 0);
+	device->user.func = users[cid].func;
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvif/userc361.c b/drivers/gpu/drm/nouveau/nvif/userc361.c
new file mode 100644
index 000000000000..19f9958e7e01
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvif/userc361.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <nvif/user.h>
+
+static void
+nvif_userc361_doorbell(struct nvif_user *user, u32 token)
+{
+	nvif_wr32(&user->object, 0x90, token);
+}
+
+const struct nvif_user_func
+nvif_userc361 = {
+	.doorbell = nvif_userc361_doorbell,
+};
diff --git a/drivers/gpu/drm/nouveau/nvif/vmm.c b/drivers/gpu/drm/nouveau/nvif/vmm.c
index 31cdb2d2e1ff..191832be6c65 100644
--- a/drivers/gpu/drm/nouveau/nvif/vmm.c
+++ b/drivers/gpu/drm/nouveau/nvif/vmm.c
@@ -37,7 +37,7 @@ nvif_vmm_map(struct nvif_vmm *vmm, u64 addr, u64 size, void *argv, u32 argc,
 	     struct nvif_mem *mem, u64 offset)
 {
 	struct nvif_vmm_map_v0 *args;
-	u8 stack[16];
+	u8 stack[48];
 	int ret;
 
 	if (sizeof(*args) + argc > sizeof(stack)) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/engine.c b/drivers/gpu/drm/nouveau/nvkm/core/engine.c
index 657231c3c098..d0322ce85172 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/engine.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/engine.c
@@ -82,6 +82,20 @@ nvkm_engine_intr(struct nvkm_subdev *subdev)
 		engine->func->intr(engine);
 }
 
+static int
+nvkm_engine_info(struct nvkm_subdev *subdev, u64 mthd, u64 *data)
+{
+	struct nvkm_engine *engine = nvkm_engine(subdev);
+	if (engine->func->info) {
+		if ((engine = nvkm_engine_ref(engine))) {
+			int ret = engine->func->info(engine, mthd, data);
+			nvkm_engine_unref(&engine);
+			return ret;
+		}
+	}
+	return -ENOSYS;
+}
+
 static int
 nvkm_engine_fini(struct nvkm_subdev *subdev, bool suspend)
 {
@@ -150,6 +164,7 @@ nvkm_engine_func = {
 	.preinit = nvkm_engine_preinit,
 	.init = nvkm_engine_init,
 	.fini = nvkm_engine_fini,
+	.info = nvkm_engine_info,
 	.intr = nvkm_engine_intr,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c
index a134d225f958..03f676c18aad 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c
@@ -35,6 +35,7 @@ nvkm_subdev_name[NVKM_SUBDEV_NR] = {
 	[NVKM_SUBDEV_BUS     ] = "bus",
 	[NVKM_SUBDEV_CLK     ] = "clk",
 	[NVKM_SUBDEV_DEVINIT ] = "devinit",
+	[NVKM_SUBDEV_FAULT   ] = "fault",
 	[NVKM_SUBDEV_FB      ] = "fb",
 	[NVKM_SUBDEV_FUSE    ] = "fuse",
 	[NVKM_SUBDEV_GPIO    ] = "gpio",
@@ -60,6 +61,9 @@ nvkm_subdev_name[NVKM_SUBDEV_NR] = {
 	[NVKM_ENGINE_CE3     ] = "ce3",
 	[NVKM_ENGINE_CE4     ] = "ce4",
 	[NVKM_ENGINE_CE5     ] = "ce5",
+	[NVKM_ENGINE_CE6     ] = "ce6",
+	[NVKM_ENGINE_CE7     ] = "ce7",
+	[NVKM_ENGINE_CE8     ] = "ce8",
 	[NVKM_ENGINE_CIPHER  ] = "cipher",
 	[NVKM_ENGINE_DISP    ] = "disp",
 	[NVKM_ENGINE_DMAOBJ  ] = "dma",
@@ -91,6 +95,14 @@ nvkm_subdev_intr(struct nvkm_subdev *subdev)
 		subdev->func->intr(subdev);
 }
 
+int
+nvkm_subdev_info(struct nvkm_subdev *subdev, u64 mthd, u64 *data)
+{
+	if (subdev->func->info)
+		return subdev->func->info(subdev, mthd, data);
+	return -ENOSYS;
+}
+
 int
 nvkm_subdev_fini(struct nvkm_subdev *subdev, bool suspend)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild
index 255d81ccf916..80d784441904 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild
@@ -5,3 +5,4 @@ nvkm-y += nvkm/engine/ce/gm107.o
 nvkm-y += nvkm/engine/ce/gm200.o
 nvkm-y += nvkm/engine/ce/gp100.o
 nvkm-y += nvkm/engine/ce/gp102.o
+nvkm-y += nvkm/engine/ce/gv100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gv100.c
new file mode 100644
index 000000000000..fcda3de45857
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gv100.c
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "priv.h"
+
+#include <nvif/class.h>
+
+static const struct nvkm_engine_func
+gv100_ce = {
+	.intr = gp100_ce_intr,
+	.sclass = {
+		{ -1, -1, VOLTA_DMA_COPY_A },
+		{}
+	}
+};
+
+int
+gv100_ce_new(struct nvkm_device *device, int index,
+	     struct nvkm_engine **pengine)
+{
+	return nvkm_engine_new_(&gv100_ce, device, index, true, pengine);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index 05cd674326a6..e294013426ce 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -2161,6 +2161,7 @@ nv130_chipset = {
 	.bios = nvkm_bios_new,
 	.bus = gf100_bus_new,
 	.devinit = gm200_devinit_new,
+	.fault = gp100_fault_new,
 	.fb = gp100_fb_new,
 	.fuse = gm107_fuse_new,
 	.gpio = gk104_gpio_new,
@@ -2196,13 +2197,14 @@ nv132_chipset = {
 	.bios = nvkm_bios_new,
 	.bus = gf100_bus_new,
 	.devinit = gm200_devinit_new,
+	.fault = gp100_fault_new,
 	.fb = gp102_fb_new,
 	.fuse = gm107_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gm200_i2c_new,
 	.ibus = gm200_ibus_new,
 	.imem = nv50_instmem_new,
-	.ltc = gp100_ltc_new,
+	.ltc = gp102_ltc_new,
 	.mc = gp100_mc_new,
 	.mmu = gp100_mmu_new,
 	.therm = gp100_therm_new,
@@ -2231,13 +2233,14 @@ nv134_chipset = {
 	.bios = nvkm_bios_new,
 	.bus = gf100_bus_new,
 	.devinit = gm200_devinit_new,
+	.fault = gp100_fault_new,
 	.fb = gp102_fb_new,
 	.fuse = gm107_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gm200_i2c_new,
 	.ibus = gm200_ibus_new,
 	.imem = nv50_instmem_new,
-	.ltc = gp100_ltc_new,
+	.ltc = gp102_ltc_new,
 	.mc = gp100_mc_new,
 	.mmu = gp100_mmu_new,
 	.therm = gp100_therm_new,
@@ -2253,7 +2256,7 @@ nv134_chipset = {
 	.disp = gp102_disp_new,
 	.dma = gf119_dma_new,
 	.fifo = gp100_fifo_new,
-	.gr = gp102_gr_new,
+	.gr = gp104_gr_new,
 	.nvdec = gp102_nvdec_new,
 	.sec2 = gp102_sec2_new,
 	.sw = gf100_sw_new,
@@ -2266,13 +2269,14 @@ nv136_chipset = {
 	.bios = nvkm_bios_new,
 	.bus = gf100_bus_new,
 	.devinit = gm200_devinit_new,
+	.fault = gp100_fault_new,
 	.fb = gp102_fb_new,
 	.fuse = gm107_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gm200_i2c_new,
 	.ibus = gm200_ibus_new,
 	.imem = nv50_instmem_new,
-	.ltc = gp100_ltc_new,
+	.ltc = gp102_ltc_new,
 	.mc = gp100_mc_new,
 	.mmu = gp100_mmu_new,
 	.therm = gp100_therm_new,
@@ -2288,7 +2292,7 @@ nv136_chipset = {
 	.disp = gp102_disp_new,
 	.dma = gf119_dma_new,
 	.fifo = gp100_fifo_new,
-	.gr = gp102_gr_new,
+	.gr = gp104_gr_new,
 	.nvdec = gp102_nvdec_new,
 	.sec2 = gp102_sec2_new,
 	.sw = gf100_sw_new,
@@ -2301,13 +2305,14 @@ nv137_chipset = {
 	.bios = nvkm_bios_new,
 	.bus = gf100_bus_new,
 	.devinit = gm200_devinit_new,
+	.fault = gp100_fault_new,
 	.fb = gp102_fb_new,
 	.fuse = gm107_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gm200_i2c_new,
 	.ibus = gm200_ibus_new,
 	.imem = nv50_instmem_new,
-	.ltc = gp100_ltc_new,
+	.ltc = gp102_ltc_new,
 	.mc = gp100_mc_new,
 	.mmu = gp100_mmu_new,
 	.therm = gp100_therm_new,
@@ -2336,13 +2341,14 @@ nv138_chipset = {
 	.bios = nvkm_bios_new,
 	.bus = gf100_bus_new,
 	.devinit = gm200_devinit_new,
+	.fault = gp100_fault_new,
 	.fb = gp102_fb_new,
 	.fuse = gm107_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gm200_i2c_new,
 	.ibus = gm200_ibus_new,
 	.imem = nv50_instmem_new,
-	.ltc = gp100_ltc_new,
+	.ltc = gp102_ltc_new,
 	.mc = gp100_mc_new,
 	.mmu = gp100_mmu_new,
 	.therm = gp100_therm_new,
@@ -2369,11 +2375,12 @@ nv13b_chipset = {
 	.name = "GP10B",
 	.bar = gm20b_bar_new,
 	.bus = gf100_bus_new,
+	.fault = gp100_fault_new,
 	.fb = gp10b_fb_new,
 	.fuse = gm107_fuse_new,
 	.ibus = gp10b_ibus_new,
 	.imem = gk20a_instmem_new,
-	.ltc = gp100_ltc_new,
+	.ltc = gp102_ltc_new,
 	.mc = gp10b_mc_new,
 	.mmu = gp10b_mmu_new,
 	.secboot = gp10b_secboot_new,
@@ -2387,6 +2394,46 @@ nv13b_chipset = {
 	.sw = gf100_sw_new,
 };
 
+static const struct nvkm_device_chip
+nv140_chipset = {
+	.name = "GV100",
+	.bar = gm107_bar_new,
+	.bios = nvkm_bios_new,
+	.bus = gf100_bus_new,
+	.devinit = gv100_devinit_new,
+	.fault = gv100_fault_new,
+	.fb = gv100_fb_new,
+	.fuse = gm107_fuse_new,
+	.gpio = gk104_gpio_new,
+	.i2c = gm200_i2c_new,
+	.ibus = gm200_ibus_new,
+	.imem = nv50_instmem_new,
+	.ltc = gp102_ltc_new,
+	.mc = gp100_mc_new,
+	.mmu = gv100_mmu_new,
+	.pci = gp100_pci_new,
+	.pmu = gp102_pmu_new,
+	.secboot = gp108_secboot_new,
+	.therm = gp100_therm_new,
+	.timer = gk20a_timer_new,
+	.top = gk104_top_new,
+	.disp = gv100_disp_new,
+	.ce[0] = gv100_ce_new,
+	.ce[1] = gv100_ce_new,
+	.ce[2] = gv100_ce_new,
+	.ce[3] = gv100_ce_new,
+	.ce[4] = gv100_ce_new,
+	.ce[5] = gv100_ce_new,
+	.ce[6] = gv100_ce_new,
+	.ce[7] = gv100_ce_new,
+	.ce[8] = gv100_ce_new,
+	.dma = gv100_dma_new,
+	.fifo = gv100_fifo_new,
+	.gr = gv100_gr_new,
+	.nvdec = gp102_nvdec_new,
+	.sec2 = gp102_sec2_new,
+};
+
 static int
 nvkm_device_event_ctor(struct nvkm_object *object, void *data, u32 size,
 		       struct nvkm_notify *notify)
@@ -2420,6 +2467,7 @@ nvkm_device_subdev(struct nvkm_device *device, int index)
 	_(BUS     , device->bus     , &device->bus->subdev);
 	_(CLK     , device->clk     , &device->clk->subdev);
 	_(DEVINIT , device->devinit , &device->devinit->subdev);
+	_(FAULT   , device->fault   , &device->fault->subdev);
 	_(FB      , device->fb      , &device->fb->subdev);
 	_(FUSE    , device->fuse    , &device->fuse->subdev);
 	_(GPIO    , device->gpio    , &device->gpio->subdev);
@@ -2463,6 +2511,9 @@ nvkm_device_engine(struct nvkm_device *device, int index)
 	_(CE3    , device->ce[3]   ,  device->ce[3]);
 	_(CE4    , device->ce[4]   ,  device->ce[4]);
 	_(CE5    , device->ce[5]   ,  device->ce[5]);
+	_(CE6    , device->ce[6]   ,  device->ce[6]);
+	_(CE7    , device->ce[7]   ,  device->ce[7]);
+	_(CE8    , device->ce[8]   ,  device->ce[8]);
 	_(CIPHER , device->cipher  ,  device->cipher);
 	_(DISP   , device->disp    , &device->disp->engine);
 	_(DMAOBJ , device->dma     , &device->dma->engine);
@@ -2739,6 +2790,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
 			case 0x110:
 			case 0x120: device->card_type = GM100; break;
 			case 0x130: device->card_type = GP100; break;
+			case 0x140: device->card_type = GV100; break;
 			default:
 				break;
 			}
@@ -2830,6 +2882,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
 		case 0x137: device->chip = &nv137_chipset; break;
 		case 0x138: device->chip = &nv138_chipset; break;
 		case 0x13b: device->chip = &nv13b_chipset; break;
+		case 0x140: device->chip = &nv140_chipset; break;
 		default:
 			nvdev_error(device, "unknown chipset (%08x)\n", boot0);
 			goto done;
@@ -2891,6 +2944,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
 		_(NVKM_SUBDEV_BUS     ,      bus);
 		_(NVKM_SUBDEV_CLK     ,      clk);
 		_(NVKM_SUBDEV_DEVINIT ,  devinit);
+		_(NVKM_SUBDEV_FAULT   ,    fault);
 		_(NVKM_SUBDEV_FB      ,       fb);
 		_(NVKM_SUBDEV_FUSE    ,     fuse);
 		_(NVKM_SUBDEV_GPIO    ,     gpio);
@@ -2916,6 +2970,9 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
 		_(NVKM_ENGINE_CE3     ,    ce[3]);
 		_(NVKM_ENGINE_CE4     ,    ce[4]);
 		_(NVKM_ENGINE_CE5     ,    ce[5]);
+		_(NVKM_ENGINE_CE6     ,    ce[6]);
+		_(NVKM_ENGINE_CE7     ,    ce[7]);
+		_(NVKM_ENGINE_CE8     ,    ce[8]);
 		_(NVKM_ENGINE_CIPHER  ,   cipher);
 		_(NVKM_ENGINE_DISP    ,     disp);
 		_(NVKM_ENGINE_DMAOBJ  ,      dma);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h
index 08d0bf605722..253ab914a8ef 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h
@@ -8,6 +8,7 @@
 #include <subdev/bus.h>
 #include <subdev/clk.h>
 #include <subdev/devinit.h>
+#include <subdev/fault.h>
 #include <subdev/fb.h>
 #include <subdev/fuse.h>
 #include <subdev/gpio.h>
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
index 17adcb4e8854..dde6bbafa709 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
@@ -39,6 +39,66 @@ struct nvkm_udevice {
 	struct nvkm_device *device;
 };
 
+static int
+nvkm_udevice_info_subdev(struct nvkm_device *device, u64 mthd, u64 *data)
+{
+	struct nvkm_subdev *subdev;
+	enum nvkm_devidx subidx;
+
+	switch (mthd & NV_DEVICE_INFO_UNIT) {
+	case NV_DEVICE_FIFO(0): subidx = NVKM_ENGINE_FIFO; break;
+	default:
+		return -EINVAL;
+	}
+
+	subdev = nvkm_device_subdev(device, subidx);
+	if (subdev)
+		return nvkm_subdev_info(subdev, mthd, data);
+	return -ENODEV;
+}
+
+static void
+nvkm_udevice_info_v1(struct nvkm_device *device,
+		     struct nv_device_info_v1_data *args)
+{
+	if (args->mthd & NV_DEVICE_INFO_UNIT) {
+		if (nvkm_udevice_info_subdev(device, args->mthd, &args->data))
+			args->mthd = NV_DEVICE_INFO_INVALID;
+		return;
+	}
+
+	switch (args->mthd) {
+#define ENGINE__(A,B,C) NV_DEVICE_INFO_ENGINE_##A: { int _i;                   \
+	for (_i = (B), args->data = 0ULL; _i <= (C); _i++) {                   \
+		if (nvkm_device_engine(device, _i))                            \
+			args->data |= BIT_ULL(_i);                             \
+	}                                                                      \
+}
+#define ENGINE_A(A) ENGINE__(A, NVKM_ENGINE_##A   , NVKM_ENGINE_##A)
+#define ENGINE_B(A) ENGINE__(A, NVKM_ENGINE_##A##0, NVKM_ENGINE_##A##_LAST)
+	case ENGINE_A(SW    ); break;
+	case ENGINE_A(GR    ); break;
+	case ENGINE_A(MPEG  ); break;
+	case ENGINE_A(ME    ); break;
+	case ENGINE_A(CIPHER); break;
+	case ENGINE_A(BSP   ); break;
+	case ENGINE_A(VP    ); break;
+	case ENGINE_B(CE    ); break;
+	case ENGINE_A(SEC   ); break;
+	case ENGINE_A(MSVLD ); break;
+	case ENGINE_A(MSPDEC); break;
+	case ENGINE_A(MSPPP ); break;
+	case ENGINE_A(MSENC ); break;
+	case ENGINE_A(VIC   ); break;
+	case ENGINE_A(SEC2  ); break;
+	case ENGINE_A(NVDEC ); break;
+	case ENGINE_B(NVENC ); break;
+	default:
+		args->mthd = NV_DEVICE_INFO_INVALID;
+		break;
+	}
+}
+
 static int
 nvkm_udevice_info(struct nvkm_udevice *udev, void *data, u32 size)
 {
@@ -48,10 +108,21 @@ nvkm_udevice_info(struct nvkm_udevice *udev, void *data, u32 size)
 	struct nvkm_instmem *imem = device->imem;
 	union {
 		struct nv_device_info_v0 v0;
+		struct nv_device_info_v1 v1;
 	} *args = data;
-	int ret = -ENOSYS;
+	int ret = -ENOSYS, i;
 
 	nvif_ioctl(object, "device info size %d\n", size);
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v1, 1, 1, true))) {
+		nvif_ioctl(object, "device info vers %d count %d\n",
+			   args->v1.version, args->v1.count);
+		if (args->v1.count * sizeof(args->v1.data[0]) == size) {
+			for (i = 0; i < args->v1.count; i++)
+				nvkm_udevice_info_v1(device, &args->v1.data[i]);
+			return 0;
+		}
+		return -EINVAL;
+	} else
 	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(object, "device info vers %d\n", args->v0.version);
 	} else
@@ -103,6 +174,7 @@ nvkm_udevice_info(struct nvkm_udevice *udev, void *data, u32 size)
 	case NV_E0: args->v0.family = NV_DEVICE_INFO_V0_KEPLER; break;
 	case GM100: args->v0.family = NV_DEVICE_INFO_V0_MAXWELL; break;
 	case GP100: args->v0.family = NV_DEVICE_INFO_V0_PASCAL; break;
+	case GV100: args->v0.family = NV_DEVICE_INFO_V0_VOLTA; break;
 	default:
 		args->v0.family = 0;
 		break;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
index 48ce6699183e..3d485dbf310a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
@@ -14,12 +14,14 @@ nvkm-y += nvkm/engine/disp/gm107.o
 nvkm-y += nvkm/engine/disp/gm200.o
 nvkm-y += nvkm/engine/disp/gp100.o
 nvkm-y += nvkm/engine/disp/gp102.o
+nvkm-y += nvkm/engine/disp/gv100.o
 nvkm-y += nvkm/engine/disp/vga.o
 
 nvkm-y += nvkm/engine/disp/head.o
 nvkm-y += nvkm/engine/disp/headnv04.o
 nvkm-y += nvkm/engine/disp/headnv50.o
 nvkm-y += nvkm/engine/disp/headgf119.o
+nvkm-y += nvkm/engine/disp/headgv100.o
 
 nvkm-y += nvkm/engine/disp/ior.o
 nvkm-y += nvkm/engine/disp/dacnv50.o
@@ -35,6 +37,7 @@ nvkm-y += nvkm/engine/disp/sorgf119.o
 nvkm-y += nvkm/engine/disp/sorgk104.o
 nvkm-y += nvkm/engine/disp/sorgm107.o
 nvkm-y += nvkm/engine/disp/sorgm200.o
+nvkm-y += nvkm/engine/disp/sorgv100.o
 
 nvkm-y += nvkm/engine/disp/outp.o
 nvkm-y += nvkm/engine/disp/dp.o
@@ -47,6 +50,7 @@ nvkm-y += nvkm/engine/disp/hdmig84.o
 nvkm-y += nvkm/engine/disp/hdmigt215.o
 nvkm-y += nvkm/engine/disp/hdmigf119.o
 nvkm-y += nvkm/engine/disp/hdmigk104.o
+nvkm-y += nvkm/engine/disp/hdmigv100.o
 
 nvkm-y += nvkm/engine/disp/conn.o
 
@@ -63,57 +67,49 @@ nvkm-y += nvkm/engine/disp/rootgm107.o
 nvkm-y += nvkm/engine/disp/rootgm200.o
 nvkm-y += nvkm/engine/disp/rootgp100.o
 nvkm-y += nvkm/engine/disp/rootgp102.o
+nvkm-y += nvkm/engine/disp/rootgv100.o
 
 nvkm-y += nvkm/engine/disp/channv50.o
 nvkm-y += nvkm/engine/disp/changf119.o
+nvkm-y += nvkm/engine/disp/changv100.o
 
 nvkm-y += nvkm/engine/disp/dmacnv50.o
 nvkm-y += nvkm/engine/disp/dmacgf119.o
 nvkm-y += nvkm/engine/disp/dmacgp102.o
+nvkm-y += nvkm/engine/disp/dmacgv100.o
 
 nvkm-y += nvkm/engine/disp/basenv50.o
 nvkm-y += nvkm/engine/disp/baseg84.o
-nvkm-y += nvkm/engine/disp/basegt200.o
-nvkm-y += nvkm/engine/disp/basegt215.o
 nvkm-y += nvkm/engine/disp/basegf119.o
-nvkm-y += nvkm/engine/disp/basegk104.o
-nvkm-y += nvkm/engine/disp/basegk110.o
 nvkm-y += nvkm/engine/disp/basegp102.o
 
 nvkm-y += nvkm/engine/disp/corenv50.o
 nvkm-y += nvkm/engine/disp/coreg84.o
 nvkm-y += nvkm/engine/disp/coreg94.o
-nvkm-y += nvkm/engine/disp/coregt200.o
-nvkm-y += nvkm/engine/disp/coregt215.o
 nvkm-y += nvkm/engine/disp/coregf119.o
 nvkm-y += nvkm/engine/disp/coregk104.o
-nvkm-y += nvkm/engine/disp/coregk110.o
-nvkm-y += nvkm/engine/disp/coregm107.o
-nvkm-y += nvkm/engine/disp/coregm200.o
-nvkm-y += nvkm/engine/disp/coregp100.o
 nvkm-y += nvkm/engine/disp/coregp102.o
+nvkm-y += nvkm/engine/disp/coregv100.o
 
 nvkm-y += nvkm/engine/disp/ovlynv50.o
 nvkm-y += nvkm/engine/disp/ovlyg84.o
 nvkm-y += nvkm/engine/disp/ovlygt200.o
-nvkm-y += nvkm/engine/disp/ovlygt215.o
 nvkm-y += nvkm/engine/disp/ovlygf119.o
 nvkm-y += nvkm/engine/disp/ovlygk104.o
 nvkm-y += nvkm/engine/disp/ovlygp102.o
 
+nvkm-y += nvkm/engine/disp/wimmgv100.o
+
+nvkm-y += nvkm/engine/disp/wndwgv100.o
+
 nvkm-y += nvkm/engine/disp/piocnv50.o
 nvkm-y += nvkm/engine/disp/piocgf119.o
 
 nvkm-y += nvkm/engine/disp/cursnv50.o
-nvkm-y += nvkm/engine/disp/cursg84.o
-nvkm-y += nvkm/engine/disp/cursgt215.o
 nvkm-y += nvkm/engine/disp/cursgf119.o
-nvkm-y += nvkm/engine/disp/cursgk104.o
 nvkm-y += nvkm/engine/disp/cursgp102.o
+nvkm-y += nvkm/engine/disp/cursgv100.o
 
 nvkm-y += nvkm/engine/disp/oimmnv50.o
-nvkm-y += nvkm/engine/disp/oimmg84.o
-nvkm-y += nvkm/engine/disp/oimmgt215.o
 nvkm-y += nvkm/engine/disp/oimmgf119.o
-nvkm-y += nvkm/engine/disp/oimmgk104.o
 nvkm-y += nvkm/engine/disp/oimmgp102.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c
index 93a75e5b2791..32fa94a9773f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c
@@ -220,6 +220,9 @@ nvkm_disp_fini(struct nvkm_engine *engine, bool suspend)
 	struct nvkm_conn *conn;
 	struct nvkm_outp *outp;
 
+	if (disp->func->fini)
+		disp->func->fini(disp);
+
 	list_for_each_entry(outp, &disp->outp, head) {
 		nvkm_outp_fini(outp);
 	}
@@ -237,6 +240,7 @@ nvkm_disp_init(struct nvkm_engine *engine)
 	struct nvkm_disp *disp = nvkm_disp(engine);
 	struct nvkm_conn *conn;
 	struct nvkm_outp *outp;
+	struct nvkm_ior *ior;
 
 	list_for_each_entry(conn, &disp->conn, head) {
 		nvkm_conn_init(conn);
@@ -246,6 +250,19 @@ nvkm_disp_init(struct nvkm_engine *engine)
 		nvkm_outp_init(outp);
 	}
 
+	if (disp->func->init) {
+		int ret = disp->func->init(disp);
+		if (ret)
+			return ret;
+	}
+
+	/* Set 'normal' (ie. when it's attached to a head) state for
+	 * each output resource to 'fully enabled'.
+	 */
+	list_for_each_entry(ior, &disp->ior, head) {
+		ior->func->power(ior, true, true, true, true, true);
+	}
+
 	return 0;
 }
 
@@ -376,6 +393,12 @@ nvkm_disp_oneinit(struct nvkm_engine *engine)
 	if (ret)
 		return ret;
 
+	if (disp->func->oneinit) {
+		ret = disp->func->oneinit(disp);
+		if (ret)
+			return ret;
+	}
+
 	i = 0;
 	list_for_each_entry(head, &disp->head, head)
 		i = max(i, head->id + 1);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/baseg84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/baseg84.c
index 6d17630a3dee..01253f4a9946 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/baseg84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/baseg84.c
@@ -21,10 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
+#include "channv50.h"
 
 static const struct nv50_disp_mthd_list
 g84_disp_base_mthd_base = {
@@ -56,8 +53,8 @@ g84_disp_base_mthd_base = {
 	}
 };
 
-const struct nv50_disp_chan_mthd
-g84_disp_base_chan_mthd = {
+static const struct nv50_disp_chan_mthd
+g84_disp_base_mthd = {
 	.name = "Base",
 	.addr = 0x000540,
 	.prev = 0x000004,
@@ -68,13 +65,10 @@ g84_disp_base_chan_mthd = {
 	}
 };
 
-const struct nv50_disp_dmac_oclass
-g84_disp_base_oclass = {
-	.base.oclass = G82_DISP_BASE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_base_new,
-	.func = &nv50_disp_dmac_func,
-	.mthd = &g84_disp_base_chan_mthd,
-	.chid = 1,
-};
+int
+g84_disp_base_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		  struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_base_new_(&nv50_disp_dmac_func, &g84_disp_base_mthd,
+				   disp, 1, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegf119.c
index ebcb925e9d90..389e19dfc514 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegf119.c
@@ -21,10 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
+#include "channv50.h"
 
 static const struct nv50_disp_mthd_list
 gf119_disp_base_mthd_base = {
@@ -91,7 +88,7 @@ gf119_disp_base_mthd_image = {
 };
 
 const struct nv50_disp_chan_mthd
-gf119_disp_base_chan_mthd = {
+gf119_disp_base_mthd = {
 	.name = "Base",
 	.addr = 0x001000,
 	.prev = -0x020000,
@@ -102,13 +99,10 @@ gf119_disp_base_chan_mthd = {
 	}
 };
 
-const struct nv50_disp_dmac_oclass
-gf119_disp_base_oclass = {
-	.base.oclass = GF110_DISP_BASE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_base_new,
-	.func = &gf119_disp_dmac_func,
-	.mthd = &gf119_disp_base_chan_mthd,
-	.chid = 1,
-};
+int
+gf119_disp_base_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_base_new_(&gf119_disp_dmac_func, &gf119_disp_base_mthd,
+				   disp, 1, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegk104.c
deleted file mode 100644
index 780a1d973634..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegk104.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gk104_disp_base_oclass = {
-	.base.oclass = GK104_DISP_BASE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_base_new,
-	.func = &gf119_disp_dmac_func,
-	.mthd = &gf119_disp_base_chan_mthd,
-	.chid = 1,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegk110.c
deleted file mode 100644
index d8bdd246c8ed..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegk110.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gk110_disp_base_oclass = {
-	.base.oclass = GK110_DISP_BASE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_base_new,
-	.func = &gf119_disp_dmac_func,
-	.mthd = &gf119_disp_base_chan_mthd,
-	.chid = 1,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp102.c
index 8a3cdeef8d2c..0cb23d673aa0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp102.c
@@ -21,18 +21,12 @@
  *
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
+#include "channv50.h"
 
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gp102_disp_base_oclass = {
-	.base.oclass = GK110_DISP_BASE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_base_new,
-	.func = &gp102_disp_dmac_func,
-	.mthd = &gf119_disp_base_chan_mthd,
-	.chid = 1,
-};
+int
+gp102_disp_base_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_base_new_(&gp102_disp_dmac_func, &gf119_disp_base_mthd,
+				   disp, 1, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegt200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegt200.c
deleted file mode 100644
index 93451e46570c..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegt200.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gt200_disp_base_oclass = {
-	.base.oclass = GT200_DISP_BASE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_base_new,
-	.func = &nv50_disp_dmac_func,
-	.mthd = &g84_disp_base_chan_mthd,
-	.chid = 1,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegt215.c
deleted file mode 100644
index 08e2b1fa3806..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegt215.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gt215_disp_base_oclass = {
-	.base.oclass = GT214_DISP_BASE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_base_new,
-	.func = &nv50_disp_dmac_func,
-	.mthd = &g84_disp_base_chan_mthd,
-	.chid = 1,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basenv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basenv50.c
index f1d6b820d482..19eb7dde01f2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basenv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basenv50.c
@@ -21,33 +21,30 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
+#include "channv50.h"
 #include "head.h"
-#include "rootnv50.h"
 
 #include <core/client.h>
 
-#include <nvif/class.h>
 #include <nvif/cl507c.h>
 #include <nvif/unpack.h>
 
 int
-nv50_disp_base_new(const struct nv50_disp_dmac_func *func,
-		   const struct nv50_disp_chan_mthd *mthd,
-		   struct nv50_disp_root *root, int chid,
-		   const struct nvkm_oclass *oclass, void *data, u32 size,
-		   struct nvkm_object **pobject)
+nv50_disp_base_new_(const struct nv50_disp_chan_func *func,
+		    const struct nv50_disp_chan_mthd *mthd,
+		    struct nv50_disp *disp, int chid,
+		    const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nvkm_object **pobject)
 {
 	union {
 		struct nv50_disp_base_channel_dma_v0 v0;
-	} *args = data;
+	} *args = argv;
 	struct nvkm_object *parent = oclass->parent;
-	struct nv50_disp *disp = root->disp;
 	int head, ret = -ENOSYS;
 	u64 push;
 
-	nvif_ioctl(parent, "create disp base channel dma size %d\n", size);
-	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
+	nvif_ioctl(parent, "create disp base channel dma size %d\n", argc);
+	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create disp base channel dma vers %d "
 				   "pushbuf %016llx head %d\n",
 			   args->v0.version, args->v0.pushbuf, args->v0.head);
@@ -58,7 +55,7 @@ nv50_disp_base_new(const struct nv50_disp_dmac_func *func,
 	} else
 		return ret;
 
-	return nv50_disp_dmac_new_(func, mthd, root, chid + head,
+	return nv50_disp_dmac_new_(func, mthd, disp, chid + head,
 				   head, push, oclass, pobject);
 }
 
@@ -102,7 +99,7 @@ nv50_disp_base_mthd_image = {
 };
 
 static const struct nv50_disp_chan_mthd
-nv50_disp_base_chan_mthd = {
+nv50_disp_base_mthd = {
 	.name = "Base",
 	.addr = 0x000540,
 	.prev = 0x000004,
@@ -113,13 +110,10 @@ nv50_disp_base_chan_mthd = {
 	}
 };
 
-const struct nv50_disp_dmac_oclass
-nv50_disp_base_oclass = {
-	.base.oclass = NV50_DISP_BASE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_base_new,
-	.func = &nv50_disp_dmac_func,
-	.mthd = &nv50_disp_base_chan_mthd,
-	.chid = 1,
-};
+int
+nv50_disp_base_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		   struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_base_new_(&nv50_disp_dmac_func, &nv50_disp_base_mthd,
+				   disp, 1, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/changf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/changf119.c
index 17a3d835cb42..29e6dd58ac48 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/changf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/changf119.c
@@ -47,3 +47,16 @@ gf119_disp_chan_uevent = {
 	.init = gf119_disp_chan_uevent_init,
 	.fini = gf119_disp_chan_uevent_fini,
 };
+
+void
+gf119_disp_chan_intr(struct nv50_disp_chan *chan, bool en)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const u64 mask = 0x00000001 << chan->chid.user;
+	if (!en) {
+		nvkm_mask(device, 0x610090, mask, 0x00000000);
+		nvkm_mask(device, 0x6100a0, mask, 0x00000000);
+	} else {
+		nvkm_mask(device, 0x6100a0, mask, mask);
+	}
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/changv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/changv100.c
new file mode 100644
index 000000000000..75247c9c7e10
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/changv100.c
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "channv50.h"
+
+const struct nvkm_event_func
+gv100_disp_chan_uevent = {
+	.ctor = nv50_disp_chan_uevent_ctor,
+};
+
+u64
+gv100_disp_chan_user(struct nv50_disp_chan *chan, u64 *psize)
+{
+	*psize = 0x1000;
+	return 0x690000 + ((chan->chid.user - 1) * 0x1000);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
index 723dcbde2ac2..57719f675eec 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
@@ -26,6 +26,7 @@
 
 #include <core/client.h>
 #include <core/notify.h>
+#include <core/oproxy.h>
 #include <core/ramht.h>
 #include <engine/dma.h>
 
@@ -65,7 +66,7 @@ nv50_disp_mthd_list(struct nv50_disp *disp, int debug, u32 base, int c,
 void
 nv50_disp_chan_mthd(struct nv50_disp_chan *chan, int debug)
 {
-	struct nv50_disp *disp = chan->root->disp;
+	struct nv50_disp *disp = chan->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	const struct nv50_disp_chan_mthd *mthd = chan->mthd;
 	const struct nv50_disp_mthd_list *list;
@@ -154,13 +155,29 @@ nv50_disp_chan_uevent = {
 	.fini = nv50_disp_chan_uevent_fini,
 };
 
+u64
+nv50_disp_chan_user(struct nv50_disp_chan *chan, u64 *psize)
+{
+	*psize = 0x1000;
+	return 0x640000 + (chan->chid.user * 0x1000);
+}
+
+void
+nv50_disp_chan_intr(struct nv50_disp_chan *chan, bool en)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const u64 mask = 0x00010001 << chan->chid.user;
+	const u64 data = en ? 0x00010000 : 0x00000000;
+	nvkm_mask(device, 0x610028, mask, data);
+}
+
 static int
 nv50_disp_chan_rd32(struct nvkm_object *object, u64 addr, u32 *data)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
-	struct nv50_disp *disp = chan->root->disp;
-	struct nvkm_device *device = disp->base.engine.subdev.device;
-	*data = nvkm_rd32(device, 0x640000 + (chan->chid.user * 0x1000) + addr);
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	u64 size, base = chan->func->user(chan, &size);
+	*data = nvkm_rd32(device, base + addr);
 	return 0;
 }
 
@@ -168,9 +185,9 @@ static int
 nv50_disp_chan_wr32(struct nvkm_object *object, u64 addr, u32 data)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
-	struct nv50_disp *disp = chan->root->disp;
-	struct nvkm_device *device = disp->base.engine.subdev.device;
-	nvkm_wr32(device, 0x640000 + (chan->chid.user * 0x1000) + addr, data);
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	u64 size, base = chan->func->user(chan, &size);
+	nvkm_wr32(device, base + addr, data);
 	return 0;
 }
 
@@ -179,7 +196,7 @@ nv50_disp_chan_ntfy(struct nvkm_object *object, u32 type,
 		    struct nvkm_event **pevent)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
-	struct nv50_disp *disp = chan->root->disp;
+	struct nv50_disp *disp = chan->disp;
 	switch (type) {
 	case NV50_DISP_CORE_CHANNEL_DMA_V0_NTFY_UEVENT:
 		*pevent = &disp->uevent;
@@ -195,34 +212,83 @@ nv50_disp_chan_map(struct nvkm_object *object, void *argv, u32 argc,
 		   enum nvkm_object_map *type, u64 *addr, u64 *size)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
-	struct nv50_disp *disp = chan->root->disp;
-	struct nvkm_device *device = disp->base.engine.subdev.device;
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const u64 base = device->func->resource_addr(device, 0);
 	*type = NVKM_OBJECT_MAP_IO;
-	*addr = device->func->resource_addr(device, 0) +
-		0x640000 + (chan->chid.user * 0x1000);
-	*size = 0x001000;
+	*addr = base + chan->func->user(chan, size);
 	return 0;
 }
 
+struct nv50_disp_chan_object {
+	struct nvkm_oproxy oproxy;
+	struct nv50_disp *disp;
+	int hash;
+};
+
+static void
+nv50_disp_chan_child_del_(struct nvkm_oproxy *base)
+{
+	struct nv50_disp_chan_object *object =
+		container_of(base, typeof(*object), oproxy);
+	nvkm_ramht_remove(object->disp->ramht, object->hash);
+}
+
+static const struct nvkm_oproxy_func
+nv50_disp_chan_child_func_ = {
+	.dtor[0] = nv50_disp_chan_child_del_,
+};
+
 static int
 nv50_disp_chan_child_new(const struct nvkm_oclass *oclass,
-			 void *data, u32 size, struct nvkm_object **pobject)
+			 void *argv, u32 argc, struct nvkm_object **pobject)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(oclass->parent);
-	return chan->func->child_new(chan, oclass, data, size, pobject);
+	struct nv50_disp *disp = chan->disp;
+	struct nvkm_device *device = disp->base.engine.subdev.device;
+	const struct nvkm_device_oclass *sclass = oclass->priv;
+	struct nv50_disp_chan_object *object;
+	int ret;
+
+	if (!(object = kzalloc(sizeof(*object), GFP_KERNEL)))
+		return -ENOMEM;
+	nvkm_oproxy_ctor(&nv50_disp_chan_child_func_, oclass, &object->oproxy);
+	object->disp = disp;
+	*pobject = &object->oproxy.base;
+
+	ret = sclass->ctor(device, oclass, argv, argc, &object->oproxy.object);
+	if (ret)
+		return ret;
+
+	object->hash = chan->func->bind(chan, object->oproxy.object,
+					      oclass->handle);
+	if (object->hash < 0)
+		return object->hash;
+
+	return 0;
 }
 
 static int
 nv50_disp_chan_child_get(struct nvkm_object *object, int index,
-			 struct nvkm_oclass *oclass)
+			 struct nvkm_oclass *sclass)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
-	if (chan->func->child_get) {
-		int ret = chan->func->child_get(chan, index, oclass);
-		if (ret == 0)
-			oclass->ctor = nv50_disp_chan_child_new;
-		return ret;
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const struct nvkm_device_oclass *oclass = NULL;
+
+	if (chan->func->bind)
+		sclass->engine = nvkm_device_engine(device, NVKM_ENGINE_DMAOBJ);
+	else
+		sclass->engine = NULL;
+
+	if (sclass->engine && sclass->engine->func->base.sclass) {
+		sclass->engine->func->base.sclass(sclass, index, &oclass);
+		if (oclass) {
+			sclass->ctor = nv50_disp_chan_child_new,
+			sclass->priv = oclass;
+			return 0;
+		}
 	}
+
 	return -EINVAL;
 }
 
@@ -231,6 +297,7 @@ nv50_disp_chan_fini(struct nvkm_object *object, bool suspend)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
 	chan->func->fini(chan);
+	chan->func->intr(chan, false);
 	return 0;
 }
 
@@ -238,6 +305,7 @@ static int
 nv50_disp_chan_init(struct nvkm_object *object)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
+	chan->func->intr(chan, true);
 	return chan->func->init(chan);
 }
 
@@ -245,10 +313,11 @@ static void *
 nv50_disp_chan_dtor(struct nvkm_object *object)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
-	struct nv50_disp *disp = chan->root->disp;
+	struct nv50_disp *disp = chan->disp;
 	if (chan->chid.user >= 0)
 		disp->chan[chan->chid.user] = NULL;
-	return chan->func->dtor ? chan->func->dtor(chan) : chan;
+	nvkm_memory_unref(&chan->memory);
+	return chan;
 }
 
 static const struct nvkm_object_func
@@ -264,18 +333,22 @@ nv50_disp_chan = {
 };
 
 int
-nv50_disp_chan_ctor(const struct nv50_disp_chan_func *func,
+nv50_disp_chan_new_(const struct nv50_disp_chan_func *func,
 		    const struct nv50_disp_chan_mthd *mthd,
-		    struct nv50_disp_root *root, int ctrl, int user, int head,
+		    struct nv50_disp *disp, int ctrl, int user, int head,
 		    const struct nvkm_oclass *oclass,
-		    struct nv50_disp_chan *chan)
+		    struct nvkm_object **pobject)
 {
-	struct nv50_disp *disp = root->disp;
+	struct nv50_disp_chan *chan;
+
+	if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL)))
+		return -ENOMEM;
+	*pobject = &chan->object;
 
 	nvkm_object_ctor(&nv50_disp_chan, oclass, &chan->object);
 	chan->func = func;
 	chan->mthd = mthd;
-	chan->root = root;
+	chan->disp = disp;
 	chan->chid.ctrl = ctrl;
 	chan->chid.user = user;
 	chan->head = head;
@@ -287,20 +360,3 @@ nv50_disp_chan_ctor(const struct nv50_disp_chan_func *func,
 	disp->chan[chan->chid.user] = chan;
 	return 0;
 }
-
-int
-nv50_disp_chan_new_(const struct nv50_disp_chan_func *func,
-		    const struct nv50_disp_chan_mthd *mthd,
-		    struct nv50_disp_root *root, int ctrl, int user, int head,
-		    const struct nvkm_oclass *oclass,
-		    struct nvkm_object **pobject)
-{
-	struct nv50_disp_chan *chan;
-
-	if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL)))
-		return -ENOMEM;
-	*pobject = &chan->object;
-
-	return nv50_disp_chan_ctor(func, mthd, root, ctrl, user,
-				   head, oclass, chan);
-}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
index 40681db91a02..adc9d76d09cc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
@@ -4,11 +4,12 @@
 #define nv50_disp_chan(p) container_of((p), struct nv50_disp_chan, object)
 #include <core/object.h>
 #include "nv50.h"
+struct nv50_disp_root;
 
 struct nv50_disp_chan {
 	const struct nv50_disp_chan_func *func;
 	const struct nv50_disp_chan_mthd *mthd;
-	struct nv50_disp_root *root;
+	struct nv50_disp *disp;
 
 	struct {
 		int ctrl;
@@ -17,36 +18,133 @@ struct nv50_disp_chan {
 	int head;
 
 	struct nvkm_object object;
+
+	struct nvkm_memory *memory;
+	u64 push;
 };
 
 struct nv50_disp_chan_func {
-	void *(*dtor)(struct nv50_disp_chan *);
 	int (*init)(struct nv50_disp_chan *);
 	void (*fini)(struct nv50_disp_chan *);
-	int (*child_get)(struct nv50_disp_chan *, int index,
-			 struct nvkm_oclass *);
-	int (*child_new)(struct nv50_disp_chan *, const struct nvkm_oclass *,
-			 void *data, u32 size, struct nvkm_object **);
+	void (*intr)(struct nv50_disp_chan *, bool en);
+	u64 (*user)(struct nv50_disp_chan *, u64 *size);
+	int (*bind)(struct nv50_disp_chan *, struct nvkm_object *, u32 handle);
 };
 
-int nv50_disp_chan_ctor(const struct nv50_disp_chan_func *,
-			const struct nv50_disp_chan_mthd *,
-			struct nv50_disp_root *, int ctrl, int user, int head,
-			const struct nvkm_oclass *, struct nv50_disp_chan *);
 int nv50_disp_chan_new_(const struct nv50_disp_chan_func *,
 			const struct nv50_disp_chan_mthd *,
-			struct nv50_disp_root *, int ctrl, int user, int head,
+			struct nv50_disp *, int ctrl, int user, int head,
+			const struct nvkm_oclass *, struct nvkm_object **);
+int nv50_disp_dmac_new_(const struct nv50_disp_chan_func *,
+			const struct nv50_disp_chan_mthd *,
+			struct nv50_disp *, int chid, int head, u64 push,
 			const struct nvkm_oclass *, struct nvkm_object **);
 
+void nv50_disp_chan_intr(struct nv50_disp_chan *, bool);
+u64 nv50_disp_chan_user(struct nv50_disp_chan *, u64 *);
 extern const struct nv50_disp_chan_func nv50_disp_pioc_func;
-extern const struct nv50_disp_chan_func gf119_disp_pioc_func;
-
-extern const struct nvkm_event_func nv50_disp_chan_uevent;
-int  nv50_disp_chan_uevent_ctor(struct nvkm_object *, void *, u32,
-				struct nvkm_notify *);
-void nv50_disp_chan_uevent_send(struct nv50_disp *, int);
+extern const struct nv50_disp_chan_func nv50_disp_dmac_func;
+int nv50_disp_dmac_bind(struct nv50_disp_chan *, struct nvkm_object *, u32);
+extern const struct nv50_disp_chan_func nv50_disp_core_func;
 
-extern const struct nvkm_event_func gf119_disp_chan_uevent;
+void gf119_disp_chan_intr(struct nv50_disp_chan *, bool);
+extern const struct nv50_disp_chan_func gf119_disp_pioc_func;
+extern const struct nv50_disp_chan_func gf119_disp_dmac_func;
+void gf119_disp_dmac_fini(struct nv50_disp_chan *);
+int gf119_disp_dmac_bind(struct nv50_disp_chan *, struct nvkm_object *, u32);
+extern const struct nv50_disp_chan_func gf119_disp_core_func;
+void gf119_disp_core_fini(struct nv50_disp_chan *);
+
+extern const struct nv50_disp_chan_func gp102_disp_dmac_func;
+
+u64 gv100_disp_chan_user(struct nv50_disp_chan *, u64 *);
+int gv100_disp_dmac_init(struct nv50_disp_chan *);
+void gv100_disp_dmac_fini(struct nv50_disp_chan *);
+int gv100_disp_dmac_bind(struct nv50_disp_chan *, struct nvkm_object *, u32);
+
+int nv50_disp_curs_new_(const struct nv50_disp_chan_func *,
+			struct nv50_disp *, int ctrl, int user,
+			const struct nvkm_oclass *, void *argv, u32 argc,
+			struct nvkm_object **);
+int nv50_disp_oimm_new_(const struct nv50_disp_chan_func *,
+			struct nv50_disp *, int ctrl, int user,
+			const struct nvkm_oclass *, void *argv, u32 argc,
+			struct nvkm_object **);
+int nv50_disp_base_new_(const struct nv50_disp_chan_func *,
+			const struct nv50_disp_chan_mthd *,
+			struct nv50_disp *, int chid,
+			const struct nvkm_oclass *, void *argv, u32 argc,
+			struct nvkm_object **);
+int nv50_disp_core_new_(const struct nv50_disp_chan_func *,
+			const struct nv50_disp_chan_mthd *,
+			struct nv50_disp *, int chid,
+			const struct nvkm_oclass *oclass, void *argv, u32 argc,
+			struct nvkm_object **);
+int nv50_disp_ovly_new_(const struct nv50_disp_chan_func *,
+			const struct nv50_disp_chan_mthd *,
+			struct nv50_disp *, int chid,
+			const struct nvkm_oclass *, void *argv, u32 argc,
+			struct nvkm_object **);
+
+int nv50_disp_curs_new(const struct nvkm_oclass *, void *, u32,
+		       struct nv50_disp *, struct nvkm_object **);
+int nv50_disp_oimm_new(const struct nvkm_oclass *, void *, u32,
+		       struct nv50_disp *, struct nvkm_object **);
+int nv50_disp_base_new(const struct nvkm_oclass *, void *, u32,
+		       struct nv50_disp *, struct nvkm_object **);
+int nv50_disp_core_new(const struct nvkm_oclass *, void *, u32,
+		       struct nv50_disp *, struct nvkm_object **);
+int nv50_disp_ovly_new(const struct nvkm_oclass *, void *, u32,
+		       struct nv50_disp *, struct nvkm_object **);
+
+int g84_disp_base_new(const struct nvkm_oclass *, void *, u32,
+		      struct nv50_disp *, struct nvkm_object **);
+int g84_disp_core_new(const struct nvkm_oclass *, void *, u32,
+		      struct nv50_disp *, struct nvkm_object **);
+int g84_disp_ovly_new(const struct nvkm_oclass *, void *, u32,
+		      struct nv50_disp *, struct nvkm_object **);
+
+int g94_disp_core_new(const struct nvkm_oclass *, void *, u32,
+		      struct nv50_disp *, struct nvkm_object **);
+
+int gt200_disp_ovly_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+
+int gf119_disp_curs_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gf119_disp_oimm_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gf119_disp_base_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gf119_disp_core_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gf119_disp_ovly_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+
+int gk104_disp_core_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gk104_disp_ovly_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+
+int gp102_disp_curs_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gp102_disp_oimm_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gp102_disp_base_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gp102_disp_core_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gp102_disp_ovly_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+
+int gv100_disp_curs_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gv100_disp_wimm_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gv100_disp_core_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
+int gv100_disp_wndw_new(const struct nvkm_oclass *, void *, u32,
+			struct nv50_disp *, struct nvkm_object **);
 
 struct nv50_disp_mthd_list {
 	u32 mthd;
@@ -76,64 +174,18 @@ extern const struct nv50_disp_mthd_list nv50_disp_core_mthd_sor;
 extern const struct nv50_disp_mthd_list nv50_disp_core_mthd_pior;
 extern const struct nv50_disp_mthd_list nv50_disp_base_mthd_image;
 
-extern const struct nv50_disp_chan_mthd g84_disp_core_chan_mthd;
+extern const struct nv50_disp_chan_mthd g84_disp_core_mthd;
 extern const struct nv50_disp_mthd_list g84_disp_core_mthd_dac;
 extern const struct nv50_disp_mthd_list g84_disp_core_mthd_head;
-extern const struct nv50_disp_chan_mthd g84_disp_base_chan_mthd;
-extern const struct nv50_disp_chan_mthd g84_disp_ovly_chan_mthd;
 
-extern const struct nv50_disp_chan_mthd g94_disp_core_chan_mthd;
+extern const struct nv50_disp_chan_mthd g94_disp_core_mthd;
 
 extern const struct nv50_disp_mthd_list gf119_disp_core_mthd_base;
 extern const struct nv50_disp_mthd_list gf119_disp_core_mthd_dac;
 extern const struct nv50_disp_mthd_list gf119_disp_core_mthd_sor;
 extern const struct nv50_disp_mthd_list gf119_disp_core_mthd_pior;
-extern const struct nv50_disp_chan_mthd gf119_disp_base_chan_mthd;
-
-extern const struct nv50_disp_chan_mthd gk104_disp_core_chan_mthd;
-extern const struct nv50_disp_chan_mthd gk104_disp_ovly_chan_mthd;
-
-struct nv50_disp_pioc_oclass {
-	int (*ctor)(const struct nv50_disp_chan_func *,
-		    const struct nv50_disp_chan_mthd *,
-		    struct nv50_disp_root *, int ctrl, int user,
-		    const struct nvkm_oclass *, void *data, u32 size,
-		    struct nvkm_object **);
-	struct nvkm_sclass base;
-	const struct nv50_disp_chan_func *func;
-	const struct nv50_disp_chan_mthd *mthd;
-	struct {
-		int ctrl;
-		int user;
-	} chid;
-};
-
-extern const struct nv50_disp_pioc_oclass nv50_disp_oimm_oclass;
-extern const struct nv50_disp_pioc_oclass nv50_disp_curs_oclass;
-
-extern const struct nv50_disp_pioc_oclass g84_disp_oimm_oclass;
-extern const struct nv50_disp_pioc_oclass g84_disp_curs_oclass;
-
-extern const struct nv50_disp_pioc_oclass gt215_disp_oimm_oclass;
-extern const struct nv50_disp_pioc_oclass gt215_disp_curs_oclass;
-
-extern const struct nv50_disp_pioc_oclass gf119_disp_oimm_oclass;
-extern const struct nv50_disp_pioc_oclass gf119_disp_curs_oclass;
-
-extern const struct nv50_disp_pioc_oclass gk104_disp_oimm_oclass;
-extern const struct nv50_disp_pioc_oclass gk104_disp_curs_oclass;
-
-extern const struct nv50_disp_pioc_oclass gp102_disp_oimm_oclass;
-extern const struct nv50_disp_pioc_oclass gp102_disp_curs_oclass;
+extern const struct nv50_disp_chan_mthd gf119_disp_base_mthd;
 
-int nv50_disp_curs_new(const struct nv50_disp_chan_func *,
-		       const struct nv50_disp_chan_mthd *,
-		       struct nv50_disp_root *, int ctrl, int user,
-		       const struct nvkm_oclass *, void *data, u32 size,
-		       struct nvkm_object **);
-int nv50_disp_oimm_new(const struct nv50_disp_chan_func *,
-		       const struct nv50_disp_chan_mthd *,
-		       struct nv50_disp_root *, int ctrl, int user,
-		       const struct nvkm_oclass *, void *data, u32 size,
-		       struct nvkm_object **);
+extern const struct nv50_disp_chan_mthd gk104_disp_core_mthd;
+extern const struct nv50_disp_chan_mthd gk104_disp_ovly_mthd;
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg84.c
index 1baa5c34b327..cfc54aad3e7c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg84.c
@@ -21,10 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
+#include "channv50.h"
 
 const struct nv50_disp_mthd_list
 g84_disp_core_mthd_dac = {
@@ -91,7 +88,7 @@ g84_disp_core_mthd_head = {
 };
 
 const struct nv50_disp_chan_mthd
-g84_disp_core_chan_mthd = {
+g84_disp_core_mthd = {
 	.name = "Core",
 	.addr = 0x000000,
 	.prev = 0x000004,
@@ -105,13 +102,10 @@ g84_disp_core_chan_mthd = {
 	}
 };
 
-const struct nv50_disp_dmac_oclass
-g84_disp_core_oclass = {
-	.base.oclass = G82_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &nv50_disp_core_func,
-	.mthd = &g84_disp_core_chan_mthd,
-	.chid = 0,
-};
+int
+g84_disp_core_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		  struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_core_new_(&nv50_disp_core_func, &g84_disp_core_mthd,
+				   disp, 0, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c
index c65c9f3ff69f..e911925f1182 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c
@@ -21,10 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
+#include "channv50.h"
 
 static const struct nv50_disp_mthd_list
 g94_disp_core_mthd_sor = {
@@ -37,7 +34,7 @@ g94_disp_core_mthd_sor = {
 };
 
 const struct nv50_disp_chan_mthd
-g94_disp_core_chan_mthd = {
+g94_disp_core_mthd = {
 	.name = "Core",
 	.addr = 0x000000,
 	.prev = 0x000004,
@@ -51,13 +48,10 @@ g94_disp_core_chan_mthd = {
 	}
 };
 
-const struct nv50_disp_dmac_oclass
-g94_disp_core_oclass = {
-	.base.oclass = GT206_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &nv50_disp_core_func,
-	.mthd = &g94_disp_core_chan_mthd,
-	.chid = 0,
-};
+int
+g94_disp_core_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		  struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_core_new_(&nv50_disp_core_func, &g94_disp_core_mthd,
+				   disp, 0, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c
index 21fbf89b6319..d162b9cf4eac 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c
@@ -21,15 +21,10 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
+#include "channv50.h"
 
-#include <core/client.h>
 #include <subdev/timer.h>
 
-#include <nvif/class.h>
-#include <nvif/unpack.h>
-
 const struct nv50_disp_mthd_list
 gf119_disp_core_mthd_base = {
 	.mthd = 0x0000,
@@ -157,7 +152,7 @@ gf119_disp_core_mthd_head = {
 };
 
 static const struct nv50_disp_chan_mthd
-gf119_disp_core_chan_mthd = {
+gf119_disp_core_mthd = {
 	.name = "Core",
 	.addr = 0x000000,
 	.prev = -0x020000,
@@ -172,10 +167,9 @@ gf119_disp_core_chan_mthd = {
 };
 
 void
-gf119_disp_core_fini(struct nv50_disp_dmac *chan)
+gf119_disp_core_fini(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->base.root->disp;
-	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 
 	/* deactivate channel */
@@ -188,22 +182,14 @@ gf119_disp_core_fini(struct nv50_disp_dmac *chan)
 		nvkm_error(subdev, "core fini: %08x\n",
 			   nvkm_rd32(device, 0x610490));
 	}
-
-	/* disable error reporting and completion notification */
-	nvkm_mask(device, 0x610090, 0x00000001, 0x00000000);
-	nvkm_mask(device, 0x6100a0, 0x00000001, 0x00000000);
 }
 
 static int
-gf119_disp_core_init(struct nv50_disp_dmac *chan)
+gf119_disp_core_init(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->base.root->disp;
-	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 
-	/* enable error reporting */
-	nvkm_mask(device, 0x6100a0, 0x00000001, 0x00000001);
-
 	/* initialise channel for dma command submission */
 	nvkm_wr32(device, 0x610494, chan->push);
 	nvkm_wr32(device, 0x610498, 0x00010000);
@@ -225,20 +211,19 @@ gf119_disp_core_init(struct nv50_disp_dmac *chan)
 	return 0;
 }
 
-const struct nv50_disp_dmac_func
+const struct nv50_disp_chan_func
 gf119_disp_core_func = {
 	.init = gf119_disp_core_init,
 	.fini = gf119_disp_core_fini,
+	.intr = gf119_disp_chan_intr,
+	.user = nv50_disp_chan_user,
 	.bind = gf119_disp_dmac_bind,
 };
 
-const struct nv50_disp_dmac_oclass
-gf119_disp_core_oclass = {
-	.base.oclass = GF110_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &gf119_disp_core_func,
-	.mthd = &gf119_disp_core_chan_mthd,
-	.chid = 0,
-};
+int
+gf119_disp_core_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_core_new_(&gf119_disp_core_func, &gf119_disp_core_mthd,
+				   disp, 0, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregk104.c
index 088ab222e823..5c800174e079 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregk104.c
@@ -21,10 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
+#include "channv50.h"
 
 static const struct nv50_disp_mthd_list
 gk104_disp_core_mthd_head = {
@@ -106,7 +103,7 @@ gk104_disp_core_mthd_head = {
 };
 
 const struct nv50_disp_chan_mthd
-gk104_disp_core_chan_mthd = {
+gk104_disp_core_mthd = {
 	.name = "Core",
 	.addr = 0x000000,
 	.prev = -0x020000,
@@ -120,13 +117,10 @@ gk104_disp_core_chan_mthd = {
 	}
 };
 
-const struct nv50_disp_dmac_oclass
-gk104_disp_core_oclass = {
-	.base.oclass = GK104_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &gf119_disp_core_func,
-	.mthd = &gk104_disp_core_chan_mthd,
-	.chid = 0,
-};
+int
+gk104_disp_core_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_core_new_(&gf119_disp_core_func, &gk104_disp_core_mthd,
+				   disp, 0, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregk110.c
deleted file mode 100644
index df0f45c20108..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregk110.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gk110_disp_core_oclass = {
-	.base.oclass = GK110_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &gf119_disp_core_func,
-	.mthd = &gk104_disp_core_chan_mthd,
-	.chid = 0,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregm107.c
deleted file mode 100644
index 9e27f8fd98b6..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregm107.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gm107_disp_core_oclass = {
-	.base.oclass = GM107_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &gf119_disp_core_func,
-	.mthd = &gk104_disp_core_chan_mthd,
-	.chid = 0,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregm200.c
deleted file mode 100644
index bb23a8658ac0..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregm200.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gm200_disp_core_oclass = {
-	.base.oclass = GM200_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &gf119_disp_core_func,
-	.mthd = &gk104_disp_core_chan_mthd,
-	.chid = 0,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp100.c
deleted file mode 100644
index d5dff6619d4d..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp100.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gp100_disp_core_oclass = {
-	.base.oclass = GP100_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &gf119_disp_core_func,
-	.mthd = &gk104_disp_core_chan_mthd,
-	.chid = 0,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c
index b0df4b752b8c..5b7f993c73c7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c
@@ -21,23 +21,16 @@
  *
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
+#include "channv50.h"
 
 #include <subdev/timer.h>
 
-#include <nvif/class.h>
-
 static int
-gp102_disp_core_init(struct nv50_disp_dmac *chan)
+gp102_disp_core_init(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->base.root->disp;
-	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 
-	/* enable error reporting */
-	nvkm_mask(device, 0x6100a0, 0x00000001, 0x00000001);
-
 	/* initialise channel for dma command submission */
 	nvkm_wr32(device, 0x611494, chan->push);
 	nvkm_wr32(device, 0x611498, 0x00010000);
@@ -59,20 +52,19 @@ gp102_disp_core_init(struct nv50_disp_dmac *chan)
 	return 0;
 }
 
-static const struct nv50_disp_dmac_func
+static const struct nv50_disp_chan_func
 gp102_disp_core_func = {
 	.init = gp102_disp_core_init,
 	.fini = gf119_disp_core_fini,
+	.intr = gf119_disp_chan_intr,
+	.user = nv50_disp_chan_user,
 	.bind = gf119_disp_dmac_bind,
 };
 
-const struct nv50_disp_dmac_oclass
-gp102_disp_core_oclass = {
-	.base.oclass = GP102_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &gp102_disp_core_func,
-	.mthd = &gk104_disp_core_chan_mthd,
-	.chid = 0,
-};
+int
+gp102_disp_core_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_core_new_(&gp102_disp_core_func, &gk104_disp_core_mthd,
+				   disp, 0, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregt200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregt200.c
deleted file mode 100644
index b234547708fc..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregt200.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gt200_disp_core_oclass = {
-	.base.oclass = GT200_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &nv50_disp_core_func,
-	.mthd = &g84_disp_core_chan_mthd,
-	.chid = 0,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregt215.c
deleted file mode 100644
index 8f5ba2018975..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregt215.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gt215_disp_core_oclass = {
-	.base.oclass = GT214_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &nv50_disp_core_func,
-	.mthd = &g94_disp_core_chan_mthd,
-	.chid = 0,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c
new file mode 100644
index 000000000000..4592d0e69fec
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c
@@ -0,0 +1,204 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "channv50.h"
+
+#include <subdev/timer.h>
+
+const struct nv50_disp_mthd_list
+gv100_disp_core_mthd_base = {
+	.mthd = 0x0000,
+	.addr = 0x000000,
+	.data = {
+		{ 0x0200, 0x680200 },
+		{ 0x0208, 0x680208 },
+		{ 0x020c, 0x68020c },
+		{ 0x0210, 0x680210 },
+		{ 0x0214, 0x680214 },
+		{ 0x0218, 0x680218 },
+		{ 0x021c, 0x68021c },
+		{}
+	}
+};
+
+const struct nv50_disp_mthd_list
+gv100_disp_core_mthd_sor = {
+	.mthd = 0x0020,
+	.addr = 0x000020,
+	.data = {
+		{ 0x0300, 0x680300 },
+		{ 0x0304, 0x680304 },
+		{ 0x0308, 0x680308 },
+		{ 0x030c, 0x68030c },
+		{}
+	}
+};
+
+static const struct nv50_disp_mthd_list
+gv100_disp_core_mthd_wndw = {
+	.mthd = 0x0080,
+	.addr = 0x000080,
+	.data = {
+		{ 0x1000, 0x681000 },
+		{ 0x1004, 0x681004 },
+		{ 0x1008, 0x681008 },
+		{ 0x100c, 0x68100c },
+		{ 0x1010, 0x681010 },
+		{}
+	}
+};
+
+static const struct nv50_disp_mthd_list
+gv100_disp_core_mthd_head = {
+	.mthd = 0x0400,
+	.addr = 0x000400,
+	.data = {
+		{ 0x2000, 0x682000 },
+		{ 0x2004, 0x682004 },
+		{ 0x2008, 0x682008 },
+		{ 0x200c, 0x68200c },
+		{ 0x2014, 0x682014 },
+		{ 0x2018, 0x682018 },
+		{ 0x201c, 0x68201c },
+		{ 0x2020, 0x682020 },
+		{ 0x2028, 0x682028 },
+		{ 0x202c, 0x68202c },
+		{ 0x2030, 0x682030 },
+		{ 0x2038, 0x682038 },
+		{ 0x203c, 0x68203c },
+		{ 0x2048, 0x682048 },
+		{ 0x204c, 0x68204c },
+		{ 0x2050, 0x682050 },
+		{ 0x2054, 0x682054 },
+		{ 0x2058, 0x682058 },
+		{ 0x205c, 0x68205c },
+		{ 0x2060, 0x682060 },
+		{ 0x2064, 0x682064 },
+		{ 0x2068, 0x682068 },
+		{ 0x206c, 0x68206c },
+		{ 0x2070, 0x682070 },
+		{ 0x2074, 0x682074 },
+		{ 0x2078, 0x682078 },
+		{ 0x207c, 0x68207c },
+		{ 0x2080, 0x682080 },
+		{ 0x2088, 0x682088 },
+		{ 0x2090, 0x682090 },
+		{ 0x209c, 0x68209c },
+		{ 0x20a0, 0x6820a0 },
+		{ 0x20a4, 0x6820a4 },
+		{ 0x20a8, 0x6820a8 },
+		{ 0x20ac, 0x6820ac },
+		{ 0x218c, 0x68218c },
+		{ 0x2194, 0x682194 },
+		{ 0x2198, 0x682198 },
+		{ 0x219c, 0x68219c },
+		{ 0x21a0, 0x6821a0 },
+		{ 0x21a4, 0x6821a4 },
+		{ 0x2214, 0x682214 },
+		{ 0x2218, 0x682218 },
+		{}
+	}
+};
+
+static const struct nv50_disp_chan_mthd
+gv100_disp_core_mthd = {
+	.name = "Core",
+	.addr = 0x000000,
+	.prev = 0x008000,
+	.data = {
+		{ "Global", 1, &gv100_disp_core_mthd_base },
+		{    "SOR", 4, &gv100_disp_core_mthd_sor  },
+		{ "WINDOW", 8, &gv100_disp_core_mthd_wndw },
+		{   "HEAD", 4, &gv100_disp_core_mthd_head },
+		{}
+	}
+};
+
+static int
+gv100_disp_core_idle(struct nv50_disp_chan *chan)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	nvkm_msec(device, 2000,
+		u32 stat = nvkm_rd32(device, 0x610630);
+		if ((stat & 0x001f0000) == 0x000b0000)
+			return 0;
+	);
+	return -EBUSY;
+}
+
+static u64
+gv100_disp_core_user(struct nv50_disp_chan *chan, u64 *psize)
+{
+	*psize = 0x10000;
+	return 0x680000;
+}
+
+static void
+gv100_disp_core_intr(struct nv50_disp_chan *chan, bool en)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const u32 mask = 0x00000001;
+	const u32 data = en ? mask : 0;
+	nvkm_mask(device, 0x611dac, mask, data);
+}
+
+static void
+gv100_disp_core_fini(struct nv50_disp_chan *chan)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	nvkm_mask(device, 0x6104e0, 0x00000010, 0x00000000);
+	gv100_disp_core_idle(chan);
+	nvkm_mask(device, 0x6104e0, 0x00000002, 0x00000000);
+}
+
+static int
+gv100_disp_core_init(struct nv50_disp_chan *chan)
+{
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+
+	nvkm_wr32(device, 0x610b24, lower_32_bits(chan->push));
+	nvkm_wr32(device, 0x610b20, upper_32_bits(chan->push));
+	nvkm_wr32(device, 0x610b28, 0x00000001);
+	nvkm_wr32(device, 0x610b2c, 0x00000040);
+
+	nvkm_mask(device, 0x6104e0, 0x00000010, 0x00000010);
+	nvkm_wr32(device, 0x680000, 0x00000000);
+	nvkm_wr32(device, 0x6104e0, 0x00000013);
+	return gv100_disp_core_idle(chan);
+}
+
+static const struct nv50_disp_chan_func
+gv100_disp_core = {
+	.init = gv100_disp_core_init,
+	.fini = gv100_disp_core_fini,
+	.intr = gv100_disp_core_intr,
+	.user = gv100_disp_core_user,
+	.bind = gv100_disp_dmac_bind,
+};
+
+int
+gv100_disp_core_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_core_new_(&gv100_disp_core, &gv100_disp_core_mthd,
+				   disp, 0, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c
index b547c8b833ca..55db9a22b4be 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c
@@ -21,32 +21,30 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
+#include "channv50.h"
 
 #include <core/client.h>
 #include <subdev/timer.h>
 
-#include <nvif/class.h>
 #include <nvif/cl507d.h>
 #include <nvif/unpack.h>
 
 int
-nv50_disp_core_new(const struct nv50_disp_dmac_func *func,
-		   const struct nv50_disp_chan_mthd *mthd,
-		   struct nv50_disp_root *root, int chid,
-		   const struct nvkm_oclass *oclass, void *data, u32 size,
-		   struct nvkm_object **pobject)
+nv50_disp_core_new_(const struct nv50_disp_chan_func *func,
+		    const struct nv50_disp_chan_mthd *mthd,
+		    struct nv50_disp *disp, int chid,
+		    const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nvkm_object **pobject)
 {
 	union {
 		struct nv50_disp_core_channel_dma_v0 v0;
-	} *args = data;
+	} *args = argv;
 	struct nvkm_object *parent = oclass->parent;
 	u64 push;
 	int ret = -ENOSYS;
 
-	nvif_ioctl(parent, "create disp core channel dma size %d\n", size);
-	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
+	nvif_ioctl(parent, "create disp core channel dma size %d\n", argc);
+	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create disp core channel dma vers %d "
 				   "pushbuf %016llx\n",
 			   args->v0.version, args->v0.pushbuf);
@@ -54,7 +52,7 @@ nv50_disp_core_new(const struct nv50_disp_dmac_func *func,
 	} else
 		return ret;
 
-	return nv50_disp_dmac_new_(func, mthd, root, chid, 0,
+	return nv50_disp_dmac_new_(func, mthd, disp, chid, 0,
 				   push, oclass, pobject);
 }
 
@@ -151,7 +149,7 @@ nv50_disp_core_mthd_head = {
 };
 
 static const struct nv50_disp_chan_mthd
-nv50_disp_core_chan_mthd = {
+nv50_disp_core_mthd = {
 	.name = "Core",
 	.addr = 0x000000,
 	.prev = 0x000004,
@@ -166,10 +164,9 @@ nv50_disp_core_chan_mthd = {
 };
 
 static void
-nv50_disp_core_fini(struct nv50_disp_dmac *chan)
+nv50_disp_core_fini(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->base.root->disp;
-	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 
 	/* deactivate channel */
@@ -182,21 +179,14 @@ nv50_disp_core_fini(struct nv50_disp_dmac *chan)
 		nvkm_error(subdev, "core fini: %08x\n",
 			   nvkm_rd32(device, 0x610200));
 	}
-
-	/* disable error reporting and completion notifications */
-	nvkm_mask(device, 0x610028, 0x00010001, 0x00000000);
 }
 
 static int
-nv50_disp_core_init(struct nv50_disp_dmac *chan)
+nv50_disp_core_init(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->base.root->disp;
-	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 
-	/* enable error reporting */
-	nvkm_mask(device, 0x610028, 0x00010000, 0x00010000);
-
 	/* attempt to unstick channel from some unknown state */
 	if ((nvkm_rd32(device, 0x610200) & 0x009f0000) == 0x00020000)
 		nvkm_mask(device, 0x610200, 0x00800000, 0x00800000);
@@ -224,20 +214,19 @@ nv50_disp_core_init(struct nv50_disp_dmac *chan)
 	return 0;
 }
 
-const struct nv50_disp_dmac_func
+const struct nv50_disp_chan_func
 nv50_disp_core_func = {
 	.init = nv50_disp_core_init,
 	.fini = nv50_disp_core_fini,
+	.intr = nv50_disp_chan_intr,
+	.user = nv50_disp_chan_user,
 	.bind = nv50_disp_dmac_bind,
 };
 
-const struct nv50_disp_dmac_oclass
-nv50_disp_core_oclass = {
-	.base.oclass = NV50_DISP_CORE_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_core_new,
-	.func = &nv50_disp_core_func,
-	.mthd = &nv50_disp_core_chan_mthd,
-	.chid = 0,
-};
+int
+nv50_disp_core_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		   struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_core_new_(&nv50_disp_core_func, &nv50_disp_core_mthd,
+				   disp, 0, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursg84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursg84.c
deleted file mode 100644
index fa781b5a7e07..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursg84.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "channv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_pioc_oclass
-g84_disp_curs_oclass = {
-	.base.oclass = G82_DISP_CURSOR,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_curs_new,
-	.func = &nv50_disp_pioc_func,
-	.chid = { 7, 7 },
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgf119.c
index 2be6fb052c65..cdda3658dcb3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgf119.c
@@ -22,16 +22,11 @@
  * Authors: Ben Skeggs
  */
 #include "channv50.h"
-#include "rootnv50.h"
 
-#include <nvif/class.h>
-
-const struct nv50_disp_pioc_oclass
-gf119_disp_curs_oclass = {
-	.base.oclass = GF110_DISP_CURSOR,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_curs_new,
-	.func = &gf119_disp_pioc_func,
-	.chid = { 13, 13 },
-};
+int
+gf119_disp_curs_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_curs_new_(&gf119_disp_pioc_func, disp, 13, 13,
+				   oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgk104.c
deleted file mode 100644
index 2a99db4bf8f8..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgk104.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2012 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-#include "channv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_pioc_oclass
-gk104_disp_curs_oclass = {
-	.base.oclass = GK104_DISP_CURSOR,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_curs_new,
-	.func = &gf119_disp_pioc_func,
-	.chid = { 13, 13 },
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgp102.c
index e958210d8105..1a4601f975e6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgp102.c
@@ -22,16 +22,11 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 #include "channv50.h"
-#include "rootnv50.h"
 
-#include <nvif/class.h>
-
-const struct nv50_disp_pioc_oclass
-gp102_disp_curs_oclass = {
-	.base.oclass = GK104_DISP_CURSOR,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_curs_new,
-	.func = &gf119_disp_pioc_func,
-	.chid = { 13, 17 },
-};
+int
+gp102_disp_curs_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_curs_new_(&gf119_disp_pioc_func, disp, 13, 17,
+				   oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgt215.c
deleted file mode 100644
index 00a7f3564450..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgt215.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "channv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_pioc_oclass
-gt215_disp_curs_oclass = {
-	.base.oclass = GT214_DISP_CURSOR,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_curs_new,
-	.func = &nv50_disp_pioc_func,
-	.chid = { 7, 7 },
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgv100.c
new file mode 100644
index 000000000000..a3e4f6900245
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgv100.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "channv50.h"
+
+#include <subdev/timer.h>
+
+static int
+gv100_disp_curs_idle(struct nv50_disp_chan *chan)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const u32 soff = (chan->chid.ctrl - 1) * 0x04;
+	nvkm_msec(device, 2000,
+		u32 stat = nvkm_rd32(device, 0x610664 + soff);
+		if ((stat & 0x00070000) == 0x00040000)
+			return 0;
+	);
+	return -EBUSY;
+}
+
+static void
+gv100_disp_curs_intr(struct nv50_disp_chan *chan, bool en)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const u32 mask = 0x00010000 << chan->head;
+	const u32 data = en ? mask : 0;
+	nvkm_mask(device, 0x611dac, mask, data);
+}
+
+static void
+gv100_disp_curs_fini(struct nv50_disp_chan *chan)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const u32 hoff = chan->chid.ctrl * 4;
+	nvkm_mask(device, 0x6104e0 + hoff, 0x00000010, 0x00000010);
+	gv100_disp_curs_idle(chan);
+	nvkm_mask(device, 0x6104e0 + hoff, 0x00000001, 0x00000000);
+}
+
+static int
+gv100_disp_curs_init(struct nv50_disp_chan *chan)
+{
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	nvkm_wr32(device, 0x6104e0 + chan->chid.ctrl * 4, 0x00000001);
+	return gv100_disp_curs_idle(chan);
+}
+
+static const struct nv50_disp_chan_func
+gv100_disp_curs = {
+	.init = gv100_disp_curs_init,
+	.fini = gv100_disp_curs_fini,
+	.intr = gv100_disp_curs_intr,
+	.user = gv100_disp_chan_user,
+};
+
+int
+gv100_disp_curs_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_curs_new_(&gv100_disp_curs, disp, 73, 73,
+				   oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c
index ab51121b7982..d29758504a5f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c
@@ -23,30 +23,26 @@
  */
 #include "channv50.h"
 #include "head.h"
-#include "rootnv50.h"
 
 #include <core/client.h>
 
-#include <nvif/class.h>
 #include <nvif/cl507a.h>
 #include <nvif/unpack.h>
 
 int
-nv50_disp_curs_new(const struct nv50_disp_chan_func *func,
-		   const struct nv50_disp_chan_mthd *mthd,
-		   struct nv50_disp_root *root, int ctrl, int user,
-		   const struct nvkm_oclass *oclass, void *data, u32 size,
-		   struct nvkm_object **pobject)
+nv50_disp_curs_new_(const struct nv50_disp_chan_func *func,
+		    struct nv50_disp *disp, int ctrl, int user,
+		    const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nvkm_object **pobject)
 {
 	union {
 		struct nv50_disp_cursor_v0 v0;
-	} *args = data;
+	} *args = argv;
 	struct nvkm_object *parent = oclass->parent;
-	struct nv50_disp *disp = root->disp;
 	int head, ret = -ENOSYS;
 
-	nvif_ioctl(parent, "create disp cursor size %d\n", size);
-	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
+	nvif_ioctl(parent, "create disp cursor size %d\n", argc);
+	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create disp cursor vers %d head %d\n",
 			   args->v0.version, args->v0.head);
 		if (!nvkm_head_find(&disp->base, args->v0.head))
@@ -55,16 +51,14 @@ nv50_disp_curs_new(const struct nv50_disp_chan_func *func,
 	} else
 		return ret;
 
-	return nv50_disp_chan_new_(func, mthd, root, ctrl + head, user + head,
+	return nv50_disp_chan_new_(func, NULL, disp, ctrl + head, user + head,
 				   head, oclass, pobject);
 }
 
-const struct nv50_disp_pioc_oclass
-nv50_disp_curs_oclass = {
-	.base.oclass = NV50_DISP_CURSOR,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_curs_new,
-	.func = &nv50_disp_pioc_func,
-	.chid = { 7, 7 },
-};
+int
+nv50_disp_curs_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		   struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_curs_new_(&nv50_disp_pioc_func, disp, 7, 7,
+				   oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacgf119.c
index dbd032ef352a..71a94777ea2e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacgf119.c
@@ -58,8 +58,13 @@ gf119_dac = {
 int
 gf119_dac_new(struct nvkm_disp *disp, int id)
 {
-	struct nvkm_device *device = disp->engine.subdev.device;
-	if (!(nvkm_rd32(device, 0x612004) & (0x00000010 << id)))
-		return 0;
 	return nvkm_ior_new_(&gf119_dac, disp, DAC, id);
 }
+
+int
+gf119_dac_cnt(struct nvkm_disp *disp, unsigned long *pmask)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	*pmask = (nvkm_rd32(device, 0x612004) & 0x000000f0) >> 4;
+	return 4;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacnv50.c
index 85e692b12260..558012db35f8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dacnv50.c
@@ -109,8 +109,13 @@ nv50_dac = {
 int
 nv50_dac_new(struct nvkm_disp *disp, int id)
 {
-	struct nvkm_device *device = disp->engine.subdev.device;
-	if (!(nvkm_rd32(device, 0x610184) & (0x00100000 << id)))
-		return 0;
 	return nvkm_ior_new_(&nv50_dac, disp, DAC, id);
 }
+
+int
+nv50_dac_cnt(struct nvkm_disp *disp, unsigned long *pmask)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	*pmask = (nvkm_rd32(device, 0x610184) & 0x00700000) >> 20;
+	return 3;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
index ce7cd74fbd5d..edf7dd0d931d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
@@ -21,29 +21,27 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
+#include "channv50.h"
 
 #include <core/ramht.h>
 #include <subdev/timer.h>
 
 int
-gf119_disp_dmac_bind(struct nv50_disp_dmac *chan,
+gf119_disp_dmac_bind(struct nv50_disp_chan *chan,
 		     struct nvkm_object *object, u32 handle)
 {
-	return nvkm_ramht_insert(chan->base.root->ramht, object,
-				 chan->base.chid.user, -9, handle,
-				 chan->base.chid.user << 27 | 0x00000001);
+	return nvkm_ramht_insert(chan->disp->ramht, object,
+				 chan->chid.user, -9, handle,
+				 chan->chid.user << 27 | 0x00000001);
 }
 
 void
-gf119_disp_dmac_fini(struct nv50_disp_dmac *chan)
+gf119_disp_dmac_fini(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->base.root->disp;
-	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int ctrl = chan->base.chid.ctrl;
-	int user = chan->base.chid.user;
+	int ctrl = chan->chid.ctrl;
+	int user = chan->chid.user;
 
 	/* deactivate channel */
 	nvkm_mask(device, 0x610490 + (ctrl * 0x0010), 0x00001010, 0x00001000);
@@ -55,23 +53,15 @@ gf119_disp_dmac_fini(struct nv50_disp_dmac *chan)
 		nvkm_error(subdev, "ch %d fini: %08x\n", user,
 			   nvkm_rd32(device, 0x610490 + (ctrl * 0x10)));
 	}
-
-	/* disable error reporting and completion notification */
-	nvkm_mask(device, 0x610090, 0x00000001 << user, 0x00000000);
-	nvkm_mask(device, 0x6100a0, 0x00000001 << user, 0x00000000);
 }
 
 static int
-gf119_disp_dmac_init(struct nv50_disp_dmac *chan)
+gf119_disp_dmac_init(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->base.root->disp;
-	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int ctrl = chan->base.chid.ctrl;
-	int user = chan->base.chid.user;
-
-	/* enable error reporting */
-	nvkm_mask(device, 0x6100a0, 0x00000001 << user, 0x00000001 << user);
+	int ctrl = chan->chid.ctrl;
+	int user = chan->chid.user;
 
 	/* initialise channel for dma command submission */
 	nvkm_wr32(device, 0x610494 + (ctrl * 0x0010), chan->push);
@@ -94,9 +84,11 @@ gf119_disp_dmac_init(struct nv50_disp_dmac *chan)
 	return 0;
 }
 
-const struct nv50_disp_dmac_func
+const struct nv50_disp_chan_func
 gf119_disp_dmac_func = {
 	.init = gf119_disp_dmac_init,
 	.fini = gf119_disp_dmac_fini,
+	.intr = gf119_disp_chan_intr,
+	.user = nv50_disp_chan_user,
 	.bind = gf119_disp_dmac_bind,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c
index cdead9500343..f21a433199aa 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c
@@ -21,22 +21,17 @@
  *
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
+#include "channv50.h"
 
 #include <subdev/timer.h>
 
 static int
-gp102_disp_dmac_init(struct nv50_disp_dmac *chan)
+gp102_disp_dmac_init(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->base.root->disp;
-	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int ctrl = chan->base.chid.ctrl;
-	int user = chan->base.chid.user;
-
-	/* enable error reporting */
-	nvkm_mask(device, 0x6100a0, 0x00000001 << user, 0x00000001 << user);
+	int ctrl = chan->chid.ctrl;
+	int user = chan->chid.user;
 
 	/* initialise channel for dma command submission */
 	nvkm_wr32(device, 0x611494 + (ctrl * 0x0010), chan->push);
@@ -59,9 +54,11 @@ gp102_disp_dmac_init(struct nv50_disp_dmac *chan)
 	return 0;
 }
 
-const struct nv50_disp_dmac_func
+const struct nv50_disp_chan_func
 gp102_disp_dmac_func = {
 	.init = gp102_disp_dmac_init,
 	.fini = gf119_disp_dmac_fini,
+	.intr = gf119_disp_chan_intr,
+	.user = nv50_disp_chan_user,
 	.bind = gf119_disp_dmac_bind,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c
new file mode 100644
index 000000000000..eac0e42da354
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "channv50.h"
+
+#include <core/ramht.h>
+#include <subdev/timer.h>
+
+static int
+gv100_disp_dmac_idle(struct nv50_disp_chan *chan)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const u32 soff = (chan->chid.ctrl - 1) * 0x04;
+	nvkm_msec(device, 2000,
+		u32 stat = nvkm_rd32(device, 0x610664 + soff);
+		if ((stat & 0x000f0000) == 0x00040000)
+			return 0;
+	);
+	return -EBUSY;
+}
+
+int
+gv100_disp_dmac_bind(struct nv50_disp_chan *chan,
+		     struct nvkm_object *object, u32 handle)
+{
+	return nvkm_ramht_insert(chan->disp->ramht, object,
+				 chan->chid.user, -9, handle,
+				 chan->chid.user << 25 | 0x00000040);
+}
+
+void
+gv100_disp_dmac_fini(struct nv50_disp_chan *chan)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const u32 coff = chan->chid.ctrl * 0x04;
+	nvkm_mask(device, 0x6104e0 + coff, 0x00000010, 0x00000000);
+	gv100_disp_dmac_idle(chan);
+	nvkm_mask(device, 0x6104e0 + coff, 0x00000002, 0x00000000);
+}
+
+int
+gv100_disp_dmac_init(struct nv50_disp_chan *chan)
+{
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	const u32 uoff = (chan->chid.ctrl - 1) * 0x1000;
+	const u32 poff = chan->chid.ctrl * 0x10;
+	const u32 coff = chan->chid.ctrl * 0x04;
+
+	nvkm_wr32(device, 0x610b24 + poff, lower_32_bits(chan->push));
+	nvkm_wr32(device, 0x610b20 + poff, upper_32_bits(chan->push));
+	nvkm_wr32(device, 0x610b28 + poff, 0x00000001);
+	nvkm_wr32(device, 0x610b2c + poff, 0x00000040);
+
+	nvkm_mask(device, 0x6104e0 + coff, 0x00000010, 0x00000010);
+	nvkm_wr32(device, 0x690000 + uoff, 0x00000000);
+	nvkm_wr32(device, 0x6104e0 + coff, 0x00000013);
+	return gv100_disp_dmac_idle(chan);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c
index 070ec5e18fdb..9e8a9d7a9b68 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c
@@ -21,176 +21,68 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
+#include "channv50.h"
 
 #include <core/client.h>
-#include <core/oproxy.h>
 #include <core/ramht.h>
 #include <subdev/fb.h>
+#include <subdev/mmu.h>
 #include <subdev/timer.h>
 #include <engine/dma.h>
 
-struct nv50_disp_dmac_object {
-	struct nvkm_oproxy oproxy;
-	struct nv50_disp_root *root;
-	int hash;
-};
-
-static void
-nv50_disp_dmac_child_del_(struct nvkm_oproxy *base)
-{
-	struct nv50_disp_dmac_object *object =
-		container_of(base, typeof(*object), oproxy);
-	nvkm_ramht_remove(object->root->ramht, object->hash);
-}
-
-static const struct nvkm_oproxy_func
-nv50_disp_dmac_child_func_ = {
-	.dtor[0] = nv50_disp_dmac_child_del_,
-};
-
-static int
-nv50_disp_dmac_child_new_(struct nv50_disp_chan *base,
-			  const struct nvkm_oclass *oclass,
-			  void *data, u32 size, struct nvkm_object **pobject)
-{
-	struct nv50_disp_dmac *chan = nv50_disp_dmac(base);
-	struct nv50_disp_root *root = chan->base.root;
-	struct nvkm_device *device = root->disp->base.engine.subdev.device;
-	const struct nvkm_device_oclass *sclass = oclass->priv;
-	struct nv50_disp_dmac_object *object;
-	int ret;
-
-	if (!(object = kzalloc(sizeof(*object), GFP_KERNEL)))
-		return -ENOMEM;
-	nvkm_oproxy_ctor(&nv50_disp_dmac_child_func_, oclass, &object->oproxy);
-	object->root = root;
-	*pobject = &object->oproxy.base;
-
-	ret = sclass->ctor(device, oclass, data, size, &object->oproxy.object);
-	if (ret)
-		return ret;
-
-	object->hash = chan->func->bind(chan, object->oproxy.object,
-					      oclass->handle);
-	if (object->hash < 0)
-		return object->hash;
-
-	return 0;
-}
-
-static int
-nv50_disp_dmac_child_get_(struct nv50_disp_chan *base, int index,
-			  struct nvkm_oclass *sclass)
-{
-	struct nv50_disp_dmac *chan = nv50_disp_dmac(base);
-	struct nv50_disp *disp = chan->base.root->disp;
-	struct nvkm_device *device = disp->base.engine.subdev.device;
-	const struct nvkm_device_oclass *oclass = NULL;
-
-	sclass->engine = nvkm_device_engine(device, NVKM_ENGINE_DMAOBJ);
-	if (sclass->engine && sclass->engine->func->base.sclass) {
-		sclass->engine->func->base.sclass(sclass, index, &oclass);
-		if (oclass) {
-			sclass->priv = oclass;
-			return 0;
-		}
-	}
-
-	return -EINVAL;
-}
-
-static void
-nv50_disp_dmac_fini_(struct nv50_disp_chan *base)
-{
-	struct nv50_disp_dmac *chan = nv50_disp_dmac(base);
-	chan->func->fini(chan);
-}
-
-static int
-nv50_disp_dmac_init_(struct nv50_disp_chan *base)
-{
-	struct nv50_disp_dmac *chan = nv50_disp_dmac(base);
-	return chan->func->init(chan);
-}
-
-static void *
-nv50_disp_dmac_dtor_(struct nv50_disp_chan *base)
-{
-	return nv50_disp_dmac(base);
-}
-
-static const struct nv50_disp_chan_func
-nv50_disp_dmac_func_ = {
-	.dtor = nv50_disp_dmac_dtor_,
-	.init = nv50_disp_dmac_init_,
-	.fini = nv50_disp_dmac_fini_,
-	.child_get = nv50_disp_dmac_child_get_,
-	.child_new = nv50_disp_dmac_child_new_,
-};
-
 int
-nv50_disp_dmac_new_(const struct nv50_disp_dmac_func *func,
+nv50_disp_dmac_new_(const struct nv50_disp_chan_func *func,
 		    const struct nv50_disp_chan_mthd *mthd,
-		    struct nv50_disp_root *root, int chid, int head, u64 push,
+		    struct nv50_disp *disp, int chid, int head, u64 push,
 		    const struct nvkm_oclass *oclass,
 		    struct nvkm_object **pobject)
 {
 	struct nvkm_client *client = oclass->client;
-	struct nvkm_dmaobj *dmaobj;
-	struct nv50_disp_dmac *chan;
+	struct nv50_disp_chan *chan;
 	int ret;
 
-	if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL)))
-		return -ENOMEM;
-	*pobject = &chan->base.object;
-	chan->func = func;
-
-	ret = nv50_disp_chan_ctor(&nv50_disp_dmac_func_, mthd, root,
-				  chid, chid, head, oclass, &chan->base);
+	ret = nv50_disp_chan_new_(func, mthd, disp, chid, chid, head, oclass,
+				  pobject);
+	chan = nv50_disp_chan(*pobject);
 	if (ret)
 		return ret;
 
-	dmaobj = nvkm_dmaobj_search(client, push);
-	if (IS_ERR(dmaobj))
-		return PTR_ERR(dmaobj);
+	chan->memory = nvkm_umem_search(client, push);
+	if (IS_ERR(chan->memory))
+		return PTR_ERR(chan->memory);
 
-	if (dmaobj->limit - dmaobj->start != 0xfff)
+	if (nvkm_memory_size(chan->memory) < 0x1000)
 		return -EINVAL;
 
-	switch (dmaobj->target) {
-	case NV_MEM_TARGET_VRAM:
-		chan->push = 0x00000001 | dmaobj->start >> 8;
-		break;
-	case NV_MEM_TARGET_PCI_NOSNOOP:
-		chan->push = 0x00000003 | dmaobj->start >> 8;
-		break;
+	switch (nvkm_memory_target(chan->memory)) {
+	case NVKM_MEM_TARGET_VRAM: chan->push = 0x00000001; break;
+	case NVKM_MEM_TARGET_NCOH: chan->push = 0x00000002; break;
+	case NVKM_MEM_TARGET_HOST: chan->push = 0x00000003; break;
 	default:
 		return -EINVAL;
 	}
 
+	chan->push |= nvkm_memory_addr(chan->memory) >> 8;
 	return 0;
 }
 
 int
-nv50_disp_dmac_bind(struct nv50_disp_dmac *chan,
+nv50_disp_dmac_bind(struct nv50_disp_chan *chan,
 		    struct nvkm_object *object, u32 handle)
 {
-	return nvkm_ramht_insert(chan->base.root->ramht, object,
-				 chan->base.chid.user, -10, handle,
-				 chan->base.chid.user << 28 |
-				 chan->base.chid.user);
+	return nvkm_ramht_insert(chan->disp->ramht, object,
+				 chan->chid.user, -10, handle,
+				 chan->chid.user << 28 |
+				 chan->chid.user);
 }
 
 static void
-nv50_disp_dmac_fini(struct nv50_disp_dmac *chan)
+nv50_disp_dmac_fini(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->base.root->disp;
-	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int ctrl = chan->base.chid.ctrl;
-	int user = chan->base.chid.user;
+	int ctrl = chan->chid.ctrl;
+	int user = chan->chid.user;
 
 	/* deactivate channel */
 	nvkm_mask(device, 0x610200 + (ctrl * 0x0010), 0x00001010, 0x00001000);
@@ -202,22 +94,15 @@ nv50_disp_dmac_fini(struct nv50_disp_dmac *chan)
 		nvkm_error(subdev, "ch %d fini timeout, %08x\n", user,
 			   nvkm_rd32(device, 0x610200 + (ctrl * 0x10)));
 	}
-
-	/* disable error reporting and completion notifications */
-	nvkm_mask(device, 0x610028, 0x00010001 << user, 0x00000000 << user);
 }
 
 static int
-nv50_disp_dmac_init(struct nv50_disp_dmac *chan)
+nv50_disp_dmac_init(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->base.root->disp;
-	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_subdev *subdev = &chan->disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int ctrl = chan->base.chid.ctrl;
-	int user = chan->base.chid.user;
-
-	/* enable error reporting */
-	nvkm_mask(device, 0x610028, 0x00010000 << user, 0x00010000 << user);
+	int ctrl = chan->chid.ctrl;
+	int user = chan->chid.user;
 
 	/* initialise channel for dma command submission */
 	nvkm_wr32(device, 0x610204 + (ctrl * 0x0010), chan->push);
@@ -240,9 +125,11 @@ nv50_disp_dmac_init(struct nv50_disp_dmac *chan)
 	return 0;
 }
 
-const struct nv50_disp_dmac_func
+const struct nv50_disp_chan_func
 nv50_disp_dmac_func = {
 	.init = nv50_disp_dmac_init,
 	.fini = nv50_disp_dmac_fini,
+	.intr = nv50_disp_chan_intr,
+	.user = nv50_disp_chan_user,
 	.bind = nv50_disp_dmac_bind,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h
deleted file mode 100644
index f9b98211da6a..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h
+++ /dev/null
@@ -1,102 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __NV50_DISP_DMAC_H__
-#define __NV50_DISP_DMAC_H__
-#define nv50_disp_dmac(p) container_of((p), struct nv50_disp_dmac, base)
-#include "channv50.h"
-
-struct nv50_disp_dmac {
-	const struct nv50_disp_dmac_func *func;
-	struct nv50_disp_chan base;
-	u32 push;
-};
-
-struct nv50_disp_dmac_func {
-	int  (*init)(struct nv50_disp_dmac *);
-	void (*fini)(struct nv50_disp_dmac *);
-	int  (*bind)(struct nv50_disp_dmac *, struct nvkm_object *, u32 handle);
-};
-
-int nv50_disp_dmac_new_(const struct nv50_disp_dmac_func *,
-			const struct nv50_disp_chan_mthd *,
-			struct nv50_disp_root *, int chid, int head, u64 push,
-			const struct nvkm_oclass *, struct nvkm_object **);
-
-extern const struct nv50_disp_dmac_func nv50_disp_dmac_func;
-int nv50_disp_dmac_bind(struct nv50_disp_dmac *, struct nvkm_object *, u32);
-extern const struct nv50_disp_dmac_func nv50_disp_core_func;
-
-extern const struct nv50_disp_dmac_func gf119_disp_dmac_func;
-void gf119_disp_dmac_fini(struct nv50_disp_dmac *);
-int gf119_disp_dmac_bind(struct nv50_disp_dmac *, struct nvkm_object *, u32);
-extern const struct nv50_disp_dmac_func gf119_disp_core_func;
-void gf119_disp_core_fini(struct nv50_disp_dmac *);
-
-extern const struct nv50_disp_dmac_func gp102_disp_dmac_func;
-
-struct nv50_disp_dmac_oclass {
-	int (*ctor)(const struct nv50_disp_dmac_func *,
-		    const struct nv50_disp_chan_mthd *,
-		    struct nv50_disp_root *, int chid,
-		    const struct nvkm_oclass *, void *data, u32 size,
-		    struct nvkm_object **);
-	struct nvkm_sclass base;
-	const struct nv50_disp_dmac_func *func;
-	const struct nv50_disp_chan_mthd *mthd;
-	int chid;
-};
-
-int nv50_disp_core_new(const struct nv50_disp_dmac_func *,
-		       const struct nv50_disp_chan_mthd *,
-		       struct nv50_disp_root *, int chid,
-		       const struct nvkm_oclass *oclass, void *data, u32 size,
-		       struct nvkm_object **);
-int nv50_disp_base_new(const struct nv50_disp_dmac_func *,
-		       const struct nv50_disp_chan_mthd *,
-		       struct nv50_disp_root *, int chid,
-		       const struct nvkm_oclass *oclass, void *data, u32 size,
-		       struct nvkm_object **);
-int nv50_disp_ovly_new(const struct nv50_disp_dmac_func *,
-		       const struct nv50_disp_chan_mthd *,
-		       struct nv50_disp_root *, int chid,
-		       const struct nvkm_oclass *oclass, void *data, u32 size,
-		       struct nvkm_object **);
-
-extern const struct nv50_disp_dmac_oclass nv50_disp_core_oclass;
-extern const struct nv50_disp_dmac_oclass nv50_disp_base_oclass;
-extern const struct nv50_disp_dmac_oclass nv50_disp_ovly_oclass;
-
-extern const struct nv50_disp_dmac_oclass g84_disp_core_oclass;
-extern const struct nv50_disp_dmac_oclass g84_disp_base_oclass;
-extern const struct nv50_disp_dmac_oclass g84_disp_ovly_oclass;
-
-extern const struct nv50_disp_dmac_oclass g94_disp_core_oclass;
-
-extern const struct nv50_disp_dmac_oclass gt200_disp_core_oclass;
-extern const struct nv50_disp_dmac_oclass gt200_disp_base_oclass;
-extern const struct nv50_disp_dmac_oclass gt200_disp_ovly_oclass;
-
-extern const struct nv50_disp_dmac_oclass gt215_disp_core_oclass;
-extern const struct nv50_disp_dmac_oclass gt215_disp_base_oclass;
-extern const struct nv50_disp_dmac_oclass gt215_disp_ovly_oclass;
-
-extern const struct nv50_disp_dmac_oclass gf119_disp_core_oclass;
-extern const struct nv50_disp_dmac_oclass gf119_disp_base_oclass;
-extern const struct nv50_disp_dmac_oclass gf119_disp_ovly_oclass;
-
-extern const struct nv50_disp_dmac_oclass gk104_disp_core_oclass;
-extern const struct nv50_disp_dmac_oclass gk104_disp_base_oclass;
-extern const struct nv50_disp_dmac_oclass gk104_disp_ovly_oclass;
-
-extern const struct nv50_disp_dmac_oclass gk110_disp_core_oclass;
-extern const struct nv50_disp_dmac_oclass gk110_disp_base_oclass;
-
-extern const struct nv50_disp_dmac_oclass gm107_disp_core_oclass;
-
-extern const struct nv50_disp_dmac_oclass gm200_disp_core_oclass;
-
-extern const struct nv50_disp_dmac_oclass gp100_disp_core_oclass;
-
-extern const struct nv50_disp_dmac_oclass gp102_disp_core_oclass;
-extern const struct nv50_disp_dmac_oclass gp102_disp_base_oclass;
-extern const struct nv50_disp_dmac_oclass gp102_disp_ovly_oclass;
-#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/g84.c
index 842e1b72ee42..731f188fc1ee 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/g84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/g84.c
@@ -28,18 +28,20 @@
 
 static const struct nv50_disp_func
 g84_disp = {
+	.init = nv50_disp_init,
+	.fini = nv50_disp_fini,
 	.intr = nv50_disp_intr,
 	.uevent = &nv50_disp_chan_uevent,
 	.super = nv50_disp_super,
 	.root = &g84_disp_root_oclass,
-	.head.new = nv50_head_new,
-	.dac = { .nr = 3, .new = nv50_dac_new },
-	.sor = { .nr = 2, .new = g84_sor_new },
-	.pior = { .nr = 3, .new = nv50_pior_new },
+	.head = { .cnt = nv50_head_cnt, .new = nv50_head_new },
+	.dac = { .cnt = nv50_dac_cnt, .new = nv50_dac_new },
+	.sor = { .cnt = nv50_sor_cnt, .new = g84_sor_new },
+	.pior = { .cnt = nv50_pior_cnt, .new = nv50_pior_new },
 };
 
 int
 g84_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return nv50_disp_new_(&g84_disp, device, index, 2, pdisp);
+	return nv50_disp_new_(&g84_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/g94.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/g94.c
index d184e6ab8918..def54fe1951e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/g94.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/g94.c
@@ -28,18 +28,20 @@
 
 static const struct nv50_disp_func
 g94_disp = {
+	.init = nv50_disp_init,
+	.fini = nv50_disp_fini,
 	.intr = nv50_disp_intr,
 	.uevent = &nv50_disp_chan_uevent,
 	.super = nv50_disp_super,
 	.root = &g94_disp_root_oclass,
-	.head.new = nv50_head_new,
-	.dac = { .nr = 3, .new = nv50_dac_new },
-	.sor = { .nr = 4, .new = g94_sor_new },
-	.pior = { .nr = 3, .new = nv50_pior_new },
+	.head = { .cnt = nv50_head_cnt, .new = nv50_head_new },
+	.dac = { .cnt = nv50_dac_cnt, .new = nv50_dac_new },
+	.sor = { .cnt = g94_sor_cnt, .new = g94_sor_new },
+	.pior = { .cnt = nv50_pior_cnt, .new = nv50_pior_new },
 };
 
 int
 g94_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return nv50_disp_new_(&g94_disp, device, index, 2, pdisp);
+	return nv50_disp_new_(&g94_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
index d8765b57180b..794e90982641 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
@@ -24,8 +24,12 @@
 #include "nv50.h"
 #include "head.h"
 #include "ior.h"
+#include "channv50.h"
 #include "rootnv50.h"
 
+#include <core/ramht.h>
+#include <subdev/timer.h>
+
 void
 gf119_disp_super(struct work_struct *work)
 {
@@ -164,28 +168,99 @@ gf119_disp_intr(struct nv50_disp *disp)
 	}
 }
 
+void
+gf119_disp_fini(struct nv50_disp *disp)
+{
+	struct nvkm_device *device = disp->base.engine.subdev.device;
+	/* disable all interrupts */
+	nvkm_wr32(device, 0x6100b0, 0x00000000);
+}
+
 int
-gf119_disp_new_(const struct nv50_disp_func *func, struct nvkm_device *device,
-		int index, struct nvkm_disp **pdisp)
+gf119_disp_init(struct nv50_disp *disp)
 {
-	u32 heads = nvkm_rd32(device, 0x022448);
-	return nv50_disp_new_(func, device, index, heads, pdisp);
+	struct nvkm_device *device = disp->base.engine.subdev.device;
+	struct nvkm_head *head;
+	u32 tmp;
+	int i;
+
+	/* The below segments of code copying values from one register to
+	 * another appear to inform EVO of the display capabilities or
+	 * something similar.
+	 */
+
+	/* ... CRTC caps */
+	list_for_each_entry(head, &disp->base.head, head) {
+		const u32 hoff = head->id * 0x800;
+		tmp = nvkm_rd32(device, 0x616104 + hoff);
+		nvkm_wr32(device, 0x6101b4 + hoff, tmp);
+		tmp = nvkm_rd32(device, 0x616108 + hoff);
+		nvkm_wr32(device, 0x6101b8 + hoff, tmp);
+		tmp = nvkm_rd32(device, 0x61610c + hoff);
+		nvkm_wr32(device, 0x6101bc + hoff, tmp);
+	}
+
+	/* ... DAC caps */
+	for (i = 0; i < disp->dac.nr; i++) {
+		tmp = nvkm_rd32(device, 0x61a000 + (i * 0x800));
+		nvkm_wr32(device, 0x6101c0 + (i * 0x800), tmp);
+	}
+
+	/* ... SOR caps */
+	for (i = 0; i < disp->sor.nr; i++) {
+		tmp = nvkm_rd32(device, 0x61c000 + (i * 0x800));
+		nvkm_wr32(device, 0x6301c4 + (i * 0x800), tmp);
+	}
+
+	/* steal display away from vbios, or something like that */
+	if (nvkm_rd32(device, 0x6100ac) & 0x00000100) {
+		nvkm_wr32(device, 0x6100ac, 0x00000100);
+		nvkm_mask(device, 0x6194e8, 0x00000001, 0x00000000);
+		if (nvkm_msec(device, 2000,
+			if (!(nvkm_rd32(device, 0x6194e8) & 0x00000002))
+				break;
+		) < 0)
+			return -EBUSY;
+	}
+
+	/* point at display engine memory area (hash table, objects) */
+	nvkm_wr32(device, 0x610010, (disp->inst->addr >> 8) | 9);
+
+	/* enable supervisor interrupts, disable everything else */
+	nvkm_wr32(device, 0x610090, 0x00000000);
+	nvkm_wr32(device, 0x6100a0, 0x00000000);
+	nvkm_wr32(device, 0x6100b0, 0x00000307);
+
+	/* disable underflow reporting, preventing an intermittent issue
+	 * on some gk104 boards where the production vbios left this
+	 * setting enabled by default.
+	 *
+	 * ftp://download.nvidia.com/open-gpu-doc/gk104-disable-underflow-reporting/1/gk104-disable-underflow-reporting.txt
+	 */
+	list_for_each_entry(head, &disp->base.head, head) {
+		const u32 hoff = head->id * 0x800;
+		nvkm_mask(device, 0x616308 + hoff, 0x00000111, 0x00000010);
+	}
+
+	return 0;
 }
 
 static const struct nv50_disp_func
 gf119_disp = {
+	.init = gf119_disp_init,
+	.fini = gf119_disp_fini,
 	.intr = gf119_disp_intr,
 	.intr_error = gf119_disp_intr_error,
 	.uevent = &gf119_disp_chan_uevent,
 	.super = gf119_disp_super,
 	.root = &gf119_disp_root_oclass,
-	.head.new = gf119_head_new,
-	.dac = { .nr = 3, .new = gf119_dac_new },
-	.sor = { .nr = 4, .new = gf119_sor_new },
+	.head = { .cnt = gf119_head_cnt, .new = gf119_head_new },
+	.dac = { .cnt = gf119_dac_cnt, .new = gf119_dac_new },
+	.sor = { .cnt = gf119_sor_cnt, .new = gf119_sor_new },
 };
 
 int
 gf119_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return gf119_disp_new_(&gf119_disp, device, index, pdisp);
+	return nv50_disp_new_(&gf119_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c
index e8fe9f315d64..4c3439b1a62d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c
@@ -28,18 +28,20 @@
 
 static const struct nv50_disp_func
 gk104_disp = {
+	.init = gf119_disp_init,
+	.fini = gf119_disp_fini,
 	.intr = gf119_disp_intr,
 	.intr_error = gf119_disp_intr_error,
 	.uevent = &gf119_disp_chan_uevent,
 	.super = gf119_disp_super,
 	.root = &gk104_disp_root_oclass,
-	.head.new = gf119_head_new,
-	.dac = { .nr = 3, .new = gf119_dac_new },
-	.sor = { .nr = 4, .new = gk104_sor_new },
+	.head = { .cnt = gf119_head_cnt, .new = gf119_head_new },
+	.dac = { .cnt = gf119_dac_cnt, .new = gf119_dac_new },
+	.sor = { .cnt = gf119_sor_cnt, .new = gk104_sor_new },
 };
 
 int
 gk104_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return gf119_disp_new_(&gk104_disp, device, index, pdisp);
+	return nv50_disp_new_(&gk104_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk110.c
index 769687502e7a..bc6f4750c942 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk110.c
@@ -28,18 +28,20 @@
 
 static const struct nv50_disp_func
 gk110_disp = {
+	.init = gf119_disp_init,
+	.fini = gf119_disp_fini,
 	.intr = gf119_disp_intr,
 	.intr_error = gf119_disp_intr_error,
 	.uevent = &gf119_disp_chan_uevent,
 	.super = gf119_disp_super,
 	.root = &gk110_disp_root_oclass,
-	.head.new = gf119_head_new,
-	.dac = { .nr = 3, .new = gf119_dac_new },
-	.sor = { .nr = 4, .new = gk104_sor_new },
+	.head = { .cnt = gf119_head_cnt, .new = gf119_head_new },
+	.dac = { .cnt = gf119_dac_cnt, .new = gf119_dac_new },
+	.sor = { .cnt = gf119_sor_cnt, .new = gk104_sor_new },
 };
 
 int
 gk110_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return gf119_disp_new_(&gk110_disp, device, index, pdisp);
+	return nv50_disp_new_(&gk110_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c
index ede70e5d188e..031cf6b03a76 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c
@@ -28,18 +28,20 @@
 
 static const struct nv50_disp_func
 gm107_disp = {
+	.init = gf119_disp_init,
+	.fini = gf119_disp_fini,
 	.intr = gf119_disp_intr,
 	.intr_error = gf119_disp_intr_error,
 	.uevent = &gf119_disp_chan_uevent,
 	.super = gf119_disp_super,
 	.root = &gm107_disp_root_oclass,
-	.head.new = gf119_head_new,
-	.dac = { .nr = 3, .new = gf119_dac_new },
-	.sor = { .nr = 4, .new = gm107_sor_new },
+	.head = { .cnt = gf119_head_cnt, .new = gf119_head_new },
+	.dac = { .cnt = gf119_dac_cnt, .new = gf119_dac_new },
+	.sor = { .cnt = gf119_sor_cnt, .new = gm107_sor_new },
 };
 
 int
 gm107_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return gf119_disp_new_(&gm107_disp, device, index, pdisp);
+	return nv50_disp_new_(&gm107_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c
index 292d3b5f9704..ec9c33a5162d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c
@@ -28,18 +28,20 @@
 
 static const struct nv50_disp_func
 gm200_disp = {
+	.init = gf119_disp_init,
+	.fini = gf119_disp_fini,
 	.intr = gf119_disp_intr,
 	.intr_error = gf119_disp_intr_error,
 	.uevent = &gf119_disp_chan_uevent,
 	.super = gf119_disp_super,
 	.root = &gm200_disp_root_oclass,
-	.head.new = gf119_head_new,
-	.dac = { .nr = 3, .new = gf119_dac_new },
-	.sor = { .nr = 4, .new = gm200_sor_new },
+	.head = { .cnt = gf119_head_cnt, .new = gf119_head_new },
+	.dac = { .cnt = gf119_dac_cnt, .new = gf119_dac_new },
+	.sor = { .cnt = gf119_sor_cnt, .new = gm200_sor_new },
 };
 
 int
 gm200_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return gf119_disp_new_(&gm200_disp, device, index, pdisp);
+	return nv50_disp_new_(&gm200_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c
index 39eb98b2c3a2..fd6216684f6d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c
@@ -28,17 +28,19 @@
 
 static const struct nv50_disp_func
 gp100_disp = {
+	.init = gf119_disp_init,
+	.fini = gf119_disp_fini,
 	.intr = gf119_disp_intr,
 	.intr_error = gf119_disp_intr_error,
 	.uevent = &gf119_disp_chan_uevent,
 	.super = gf119_disp_super,
 	.root = &gp100_disp_root_oclass,
-	.head.new = gf119_head_new,
-	.sor = { .nr = 4, .new = gm200_sor_new },
+	.head = { .cnt = gf119_head_cnt, .new = gf119_head_new },
+	.sor = { .cnt = gf119_sor_cnt, .new = gm200_sor_new },
 };
 
 int
 gp100_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return gf119_disp_new_(&gp100_disp, device, index, pdisp);
+	return nv50_disp_new_(&gp100_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c
index 91d70fe18275..3468ddec1270 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c
@@ -24,6 +24,7 @@
 #include "nv50.h"
 #include "head.h"
 #include "ior.h"
+#include "channv50.h"
 #include "rootnv50.h"
 
 static void
@@ -54,17 +55,19 @@ gp102_disp_intr_error(struct nv50_disp *disp, int chid)
 
 static const struct nv50_disp_func
 gp102_disp = {
+	.init = gf119_disp_init,
+	.fini = gf119_disp_fini,
 	.intr = gf119_disp_intr,
 	.intr_error = gp102_disp_intr_error,
 	.uevent = &gf119_disp_chan_uevent,
 	.super = gf119_disp_super,
 	.root = &gp102_disp_root_oclass,
-	.head.new = gf119_head_new,
-	.sor = { .nr = 4, .new = gm200_sor_new },
+	.head = { .cnt = gf119_head_cnt, .new = gf119_head_new },
+	.sor = { .cnt = gf119_sor_cnt, .new = gm200_sor_new },
 };
 
 int
 gp102_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return gf119_disp_new_(&gp102_disp, device, index, pdisp);
+	return nv50_disp_new_(&gp102_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt200.c
index bf00c4e3be3a..f80183701f44 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt200.c
@@ -28,18 +28,20 @@
 
 static const struct nv50_disp_func
 gt200_disp = {
+	.init = nv50_disp_init,
+	.fini = nv50_disp_fini,
 	.intr = nv50_disp_intr,
 	.uevent = &nv50_disp_chan_uevent,
 	.super = nv50_disp_super,
 	.root = &gt200_disp_root_oclass,
-	.head.new = nv50_head_new,
-	.dac = { .nr = 3, .new = nv50_dac_new },
-	.sor = { .nr = 2, .new = g84_sor_new },
-	.pior = { .nr = 3, .new = nv50_pior_new },
+	.head = { .cnt = nv50_head_cnt, .new = nv50_head_new },
+	.dac = { .cnt = nv50_dac_cnt, .new = nv50_dac_new },
+	.sor = { .cnt = nv50_sor_cnt, .new = g84_sor_new },
+	.pior = { .cnt = nv50_pior_cnt, .new = nv50_pior_new },
 };
 
 int
 gt200_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return nv50_disp_new_(&gt200_disp, device, index, 2, pdisp);
+	return nv50_disp_new_(&gt200_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt215.c
index 2cdd4d7a98d3..7581efc1357e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt215.c
@@ -28,18 +28,20 @@
 
 static const struct nv50_disp_func
 gt215_disp = {
+	.init = nv50_disp_init,
+	.fini = nv50_disp_fini,
 	.intr = nv50_disp_intr,
 	.uevent = &nv50_disp_chan_uevent,
 	.super = nv50_disp_super,
 	.root = &gt215_disp_root_oclass,
-	.head.new = nv50_head_new,
-	.dac = { .nr = 3, .new = nv50_dac_new },
-	.sor = { .nr = 4, .new = gt215_sor_new },
-	.pior = { .nr = 3, .new = nv50_pior_new },
+	.head = { .cnt = nv50_head_cnt, .new = nv50_head_new },
+	.dac = { .cnt = nv50_dac_cnt, .new = nv50_dac_new },
+	.sor = { .cnt = g94_sor_cnt, .new = gt215_sor_new },
+	.pior = { .cnt = nv50_pior_cnt, .new = nv50_pior_new },
 };
 
 int
 gt215_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return nv50_disp_new_(&gt215_disp, device, index, 2, pdisp);
+	return nv50_disp_new_(&gt215_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c
new file mode 100644
index 000000000000..d0a7e3456da1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c
@@ -0,0 +1,427 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "nv50.h"
+#include "head.h"
+#include "ior.h"
+#include "channv50.h"
+#include "rootnv50.h"
+
+#include <core/gpuobj.h>
+#include <subdev/timer.h>
+
+static int
+gv100_disp_wndw_cnt(struct nvkm_disp *disp, unsigned long *pmask)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	*pmask = nvkm_rd32(device, 0x610064);
+	return (nvkm_rd32(device, 0x610074) & 0x03f00000) >> 20;
+}
+
+static void
+gv100_disp_super(struct work_struct *work)
+{
+	struct nv50_disp *disp =
+		container_of(work, struct nv50_disp, supervisor);
+	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	struct nvkm_head *head;
+	u32 stat = nvkm_rd32(device, 0x6107a8);
+	u32 mask[4];
+
+	nvkm_debug(subdev, "supervisor %d: %08x\n", ffs(disp->super), stat);
+	list_for_each_entry(head, &disp->base.head, head) {
+		mask[head->id] = nvkm_rd32(device, 0x6107ac + (head->id * 4));
+		HEAD_DBG(head, "%08x", mask[head->id]);
+	}
+
+	if (disp->super & 0x00000001) {
+		nv50_disp_chan_mthd(disp->chan[0], NV_DBG_DEBUG);
+		nv50_disp_super_1(disp);
+		list_for_each_entry(head, &disp->base.head, head) {
+			if (!(mask[head->id] & 0x00001000))
+				continue;
+			nv50_disp_super_1_0(disp, head);
+		}
+	} else
+	if (disp->super & 0x00000002) {
+		list_for_each_entry(head, &disp->base.head, head) {
+			if (!(mask[head->id] & 0x00001000))
+				continue;
+			nv50_disp_super_2_0(disp, head);
+		}
+		nvkm_outp_route(&disp->base);
+		list_for_each_entry(head, &disp->base.head, head) {
+			if (!(mask[head->id] & 0x00010000))
+				continue;
+			nv50_disp_super_2_1(disp, head);
+		}
+		list_for_each_entry(head, &disp->base.head, head) {
+			if (!(mask[head->id] & 0x00001000))
+				continue;
+			nv50_disp_super_2_2(disp, head);
+		}
+	} else
+	if (disp->super & 0x00000004) {
+		list_for_each_entry(head, &disp->base.head, head) {
+			if (!(mask[head->id] & 0x00001000))
+				continue;
+			nv50_disp_super_3_0(disp, head);
+		}
+	}
+
+	list_for_each_entry(head, &disp->base.head, head)
+		nvkm_wr32(device, 0x6107ac + (head->id * 4), 0x00000000);
+	nvkm_wr32(device, 0x6107a8, 0x80000000);
+}
+
+static void
+gv100_disp_exception(struct nv50_disp *disp, int chid)
+{
+	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 stat = nvkm_rd32(device, 0x611020 + (chid * 12));
+	u32 type = (stat & 0x00007000) >> 12;
+	u32 mthd = (stat & 0x00000fff) << 2;
+	u32 data = nvkm_rd32(device, 0x611024 + (chid * 12));
+	u32 code = nvkm_rd32(device, 0x611028 + (chid * 12));
+
+	nvkm_error(subdev, "chid %d %08x [type %d mthd %04x] "
+			   "data %08x code %08x\n",
+		   chid, stat, type, mthd, data, code);
+
+	if (chid < ARRAY_SIZE(disp->chan) && disp->chan[chid]) {
+		switch (mthd) {
+		case 0x0200:
+			nv50_disp_chan_mthd(disp->chan[chid], NV_DBG_ERROR);
+			break;
+		default:
+			break;
+		}
+	}
+
+	nvkm_wr32(device, 0x611020 + (chid * 12), 0x90000000);
+}
+
+static void
+gv100_disp_intr_ctrl_disp(struct nv50_disp *disp)
+{
+	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 stat = nvkm_rd32(device, 0x611c30);
+
+	if (stat & 0x00000007) {
+		disp->super = (stat & 0x00000007);
+		queue_work(disp->wq, &disp->supervisor);
+		nvkm_wr32(device, 0x611860, disp->super);
+		stat &= ~0x00000007;
+	}
+
+	/*TODO: I would guess this is VBIOS_RELEASE, however, NFI how to
+	 *      ACK it, nor does RM appear to bother.
+	 */
+	if (stat & 0x00000008)
+		stat &= ~0x00000008;
+
+	if (stat & 0x00000100) {
+		unsigned long wndws = nvkm_rd32(device, 0x611858);
+		unsigned long other = nvkm_rd32(device, 0x61185c);
+		int wndw;
+
+		nvkm_wr32(device, 0x611858, wndws);
+		nvkm_wr32(device, 0x61185c, other);
+
+		/* AWAKEN_OTHER_CORE. */
+		if (other & 0x00000001)
+			nv50_disp_chan_uevent_send(disp, 0);
+
+		/* AWAKEN_WIN_CH(n). */
+		for_each_set_bit(wndw, &wndws, disp->wndw.nr) {
+			nv50_disp_chan_uevent_send(disp, 1 + wndw);
+		}
+	}
+
+	if (stat)
+		nvkm_warn(subdev, "ctrl %08x\n", stat);
+}
+
+static void
+gv100_disp_intr_exc_other(struct nv50_disp *disp)
+{
+	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 stat = nvkm_rd32(device, 0x611854);
+	unsigned long mask;
+	int head;
+
+	if (stat & 0x00000001) {
+		nvkm_wr32(device, 0x611854, 0x00000001);
+		gv100_disp_exception(disp, 0);
+		stat &= ~0x00000001;
+	}
+
+	if ((mask = (stat & 0x00ff0000) >> 16)) {
+		for_each_set_bit(head, &mask, disp->wndw.nr) {
+			nvkm_wr32(device, 0x611854, 0x00010000 << head);
+			gv100_disp_exception(disp, 73 + head);
+			stat &= ~(0x00010000 << head);
+		}
+	}
+
+	if (stat) {
+		nvkm_warn(subdev, "exception %08x\n", stat);
+		nvkm_wr32(device, 0x611854, stat);
+	}
+}
+
+static void
+gv100_disp_intr_exc_winim(struct nv50_disp *disp)
+{
+	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	unsigned long stat = nvkm_rd32(device, 0x611850);
+	int wndw;
+
+	for_each_set_bit(wndw, &stat, disp->wndw.nr) {
+		nvkm_wr32(device, 0x611850, BIT(wndw));
+		gv100_disp_exception(disp, 33 + wndw);
+		stat &= ~BIT(wndw);
+	}
+
+	if (stat) {
+		nvkm_warn(subdev, "wimm %08x\n", (u32)stat);
+		nvkm_wr32(device, 0x611850, stat);
+	}
+}
+
+static void
+gv100_disp_intr_exc_win(struct nv50_disp *disp)
+{
+	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	unsigned long stat = nvkm_rd32(device, 0x61184c);
+	int wndw;
+
+	for_each_set_bit(wndw, &stat, disp->wndw.nr) {
+		nvkm_wr32(device, 0x61184c, BIT(wndw));
+		gv100_disp_exception(disp, 1 + wndw);
+		stat &= ~BIT(wndw);
+	}
+
+	if (stat) {
+		nvkm_warn(subdev, "wndw %08x\n", (u32)stat);
+		nvkm_wr32(device, 0x61184c, stat);
+	}
+}
+
+static void
+gv100_disp_intr_head_timing(struct nv50_disp *disp, int head)
+{
+	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 stat = nvkm_rd32(device, 0x611800 + (head * 0x04));
+
+	/* LAST_DATA, LOADV. */
+	if (stat & 0x00000003) {
+		nvkm_wr32(device, 0x611800 + (head * 0x04), stat & 0x00000003);
+		stat &= ~0x00000003;
+	}
+
+	if (stat & 0x00000004) {
+		nvkm_disp_vblank(&disp->base, head);
+		nvkm_wr32(device, 0x611800 + (head * 0x04), 0x00000004);
+		stat &= ~0x00000004;
+	}
+
+	if (stat) {
+		nvkm_warn(subdev, "head %08x\n", stat);
+		nvkm_wr32(device, 0x611800 + (head * 0x04), stat);
+	}
+}
+
+static void
+gv100_disp_intr(struct nv50_disp *disp)
+{
+	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 stat = nvkm_rd32(device, 0x611ec0);
+	unsigned long mask;
+	int head;
+
+	if ((mask = (stat & 0x000000ff))) {
+		for_each_set_bit(head, &mask, 8) {
+			gv100_disp_intr_head_timing(disp, head);
+			stat &= ~BIT(head);
+		}
+	}
+
+	if (stat & 0x00000200) {
+		gv100_disp_intr_exc_win(disp);
+		stat &= ~0x00000200;
+	}
+
+	if (stat & 0x00000400) {
+		gv100_disp_intr_exc_winim(disp);
+		stat &= ~0x00000400;
+	}
+
+	if (stat & 0x00000800) {
+		gv100_disp_intr_exc_other(disp);
+		stat &= ~0x00000800;
+	}
+
+	if (stat & 0x00001000) {
+		gv100_disp_intr_ctrl_disp(disp);
+		stat &= ~0x00001000;
+	}
+
+	if (stat)
+		nvkm_warn(subdev, "intr %08x\n", stat);
+}
+
+static void
+gv100_disp_fini(struct nv50_disp *disp)
+{
+	struct nvkm_device *device = disp->base.engine.subdev.device;
+	nvkm_wr32(device, 0x611db0, 0x00000000);
+}
+
+static int
+gv100_disp_init(struct nv50_disp *disp)
+{
+	struct nvkm_device *device = disp->base.engine.subdev.device;
+	struct nvkm_head *head;
+	int i, j;
+	u32 tmp;
+
+	/* Claim ownership of display. */
+	if (nvkm_rd32(device, 0x6254e8) & 0x00000002) {
+		nvkm_mask(device, 0x6254e8, 0x00000001, 0x00000000);
+		if (nvkm_msec(device, 2000,
+			if (!(nvkm_rd32(device, 0x6254e8) & 0x00000002))
+				break;
+		) < 0)
+			return -EBUSY;
+	}
+
+	/* Lock pin capabilities. */
+	tmp = nvkm_rd32(device, 0x610068);
+	nvkm_wr32(device, 0x640008, tmp);
+
+	/* SOR capabilities. */
+	for (i = 0; i < disp->sor.nr; i++) {
+		tmp = nvkm_rd32(device, 0x61c000 + (i * 0x800));
+		nvkm_mask(device, 0x640000, 0x00000100 << i, 0x00000100 << i);
+		nvkm_wr32(device, 0x640144 + (i * 0x08), tmp);
+	}
+
+	/* Head capabilities. */
+	list_for_each_entry(head, &disp->base.head, head) {
+		const int id = head->id;
+
+		/* RG. */
+		tmp = nvkm_rd32(device, 0x616300 + (id * 0x800));
+		nvkm_wr32(device, 0x640048 + (id * 0x020), tmp);
+
+		/* POSTCOMP. */
+		for (j = 0; j < 6 * 4; j += 4) {
+			tmp = nvkm_rd32(device, 0x616100 + (id * 0x800) + j);
+			nvkm_wr32(device, 0x640030 + (id * 0x20) + j, tmp);
+		}
+	}
+
+	/* Window capabilities. */
+	for (i = 0; i < disp->wndw.nr; i++) {
+		nvkm_mask(device, 0x640004, 1 << i, 1 << i);
+		for (j = 0; j < 6 * 4; j += 4) {
+			tmp = nvkm_rd32(device, 0x630050 + (i * 0x800) + j);
+			nvkm_wr32(device, 0x6401e4 + (i * 0x20) + j, tmp);
+		}
+	}
+
+	/* IHUB capabilities. */
+	for (i = 0; i < 4; i++) {
+		tmp = nvkm_rd32(device, 0x62e000 + (i * 0x04));
+		nvkm_wr32(device, 0x640010 + (i * 0x04), tmp);
+	}
+
+	nvkm_mask(device, 0x610078, 0x00000001, 0x00000001);
+
+	/* Setup instance memory. */
+	switch (nvkm_memory_target(disp->inst->memory)) {
+	case NVKM_MEM_TARGET_VRAM: tmp = 0x00000001; break;
+	case NVKM_MEM_TARGET_NCOH: tmp = 0x00000002; break;
+	case NVKM_MEM_TARGET_HOST: tmp = 0x00000003; break;
+	default:
+		break;
+	}
+	nvkm_wr32(device, 0x610010, 0x00000008 | tmp);
+	nvkm_wr32(device, 0x610014, disp->inst->addr >> 16);
+
+	/* CTRL_DISP: AWAKEN, ERROR, SUPERVISOR[1-3]. */
+	nvkm_wr32(device, 0x611cf0, 0x00000187); /* MSK. */
+	nvkm_wr32(device, 0x611db0, 0x00000187); /* EN. */
+
+	/* EXC_OTHER: CURSn, CORE. */
+	nvkm_wr32(device, 0x611cec, disp->head.mask << 16 |
+				    0x00000001); /* MSK. */
+	nvkm_wr32(device, 0x611dac, 0x00000000); /* EN. */
+
+	/* EXC_WINIM. */
+	nvkm_wr32(device, 0x611ce8, disp->wndw.mask); /* MSK. */
+	nvkm_wr32(device, 0x611da8, 0x00000000); /* EN. */
+
+	/* EXC_WIN. */
+	nvkm_wr32(device, 0x611ce4, disp->wndw.mask); /* MSK. */
+	nvkm_wr32(device, 0x611da4, 0x00000000); /* EN. */
+
+	/* HEAD_TIMING(n): VBLANK. */
+	list_for_each_entry(head, &disp->base.head, head) {
+		const u32 hoff = head->id * 4;
+		nvkm_wr32(device, 0x611cc0 + hoff, 0x00000004); /* MSK. */
+		nvkm_wr32(device, 0x611d80 + hoff, 0x00000000); /* EN. */
+	}
+
+	/* OR. */
+	nvkm_wr32(device, 0x611cf4, 0x00000000); /* MSK. */
+	nvkm_wr32(device, 0x611db4, 0x00000000); /* EN. */
+	return 0;
+}
+
+static const struct nv50_disp_func
+gv100_disp = {
+	.init = gv100_disp_init,
+	.fini = gv100_disp_fini,
+	.intr = gv100_disp_intr,
+	.uevent = &gv100_disp_chan_uevent,
+	.super = gv100_disp_super,
+	.root = &gv100_disp_root_oclass,
+	.wndw = { .cnt = gv100_disp_wndw_cnt },
+	.head = { .cnt = gv100_head_cnt, .new = gv100_head_new },
+	.sor = { .cnt = gv100_sor_cnt, .new = gv100_sor_new },
+	.ramht_size = 0x2000,
+};
+
+int
+gv100_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
+{
+	return nv50_disp_new_(&gv100_disp, device, index, pdisp);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigv100.c
new file mode 100644
index 000000000000..6e3c450eaace
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigv100.c
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "hdmi.h"
+
+void
+gv100_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet,
+		u8 rekey, u8 *avi, u8 avi_size, u8 *vendor, u8 vendor_size)
+{
+	struct nvkm_device *device = ior->disp->engine.subdev.device;
+	const u32 ctrl = 0x40000000 * enable |
+			 max_ac_packet << 16 |
+			 rekey;
+	const u32 hoff = head * 0x800;
+	const u32 hdmi = head * 0x400;
+	struct packed_hdmi_infoframe avi_infoframe;
+	struct packed_hdmi_infoframe vendor_infoframe;
+
+	pack_hdmi_infoframe(&avi_infoframe, avi, avi_size);
+	pack_hdmi_infoframe(&vendor_infoframe, vendor, vendor_size);
+
+	if (!(ctrl & 0x40000000)) {
+		nvkm_mask(device, 0x6165c0 + hoff, 0x40000000, 0x00000000);
+		nvkm_mask(device, 0x6f0100 + hdmi, 0x00000001, 0x00000000);
+		nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000000);
+		nvkm_mask(device, 0x6f0000 + hdmi, 0x00000001, 0x00000000);
+		return;
+	}
+
+	/* AVI InfoFrame (AVI). */
+	nvkm_mask(device, 0x6f0000 + hdmi, 0x00000001, 0x00000000);
+	if (avi_size) {
+		nvkm_wr32(device, 0x6f0008 + hdmi, avi_infoframe.header);
+		nvkm_wr32(device, 0x6f000c + hdmi, avi_infoframe.subpack0_low);
+		nvkm_wr32(device, 0x6f0010 + hdmi, avi_infoframe.subpack0_high);
+		nvkm_wr32(device, 0x6f0014 + hdmi, avi_infoframe.subpack1_low);
+		nvkm_wr32(device, 0x6f0018 + hdmi, avi_infoframe.subpack1_high);
+		nvkm_mask(device, 0x6f0000 + hdmi, 0x00000001, 0x00000001);
+	}
+
+	/* Vendor-specific InfoFrame (VSI). */
+	nvkm_mask(device, 0x6f0100 + hdmi, 0x00010001, 0x00000000);
+	if (vendor_size) {
+		nvkm_wr32(device, 0x6f0108 + hdmi, vendor_infoframe.header);
+		nvkm_wr32(device, 0x6f010c + hdmi, vendor_infoframe.subpack0_low);
+		nvkm_wr32(device, 0x6f0110 + hdmi, vendor_infoframe.subpack0_high);
+		nvkm_wr32(device, 0x6f0110 + hdmi, 0x00000000);
+		nvkm_wr32(device, 0x6f0114 + hdmi, 0x00000000);
+		nvkm_wr32(device, 0x6f0118 + hdmi, 0x00000000);
+		nvkm_wr32(device, 0x6f011c + hdmi, 0x00000000);
+		nvkm_wr32(device, 0x6f0120 + hdmi, 0x00000000);
+		nvkm_wr32(device, 0x6f0124 + hdmi, 0x00000000);
+		nvkm_mask(device, 0x6f0100 + hdmi, 0x00000001, 0x00000001);
+	}
+
+
+	/* General Control (GCP). */
+	nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000000);
+	nvkm_wr32(device, 0x6f00cc + hdmi, 0x00000010);
+	nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000001);
+
+	/* Audio Clock Regeneration (ACR). */
+	nvkm_wr32(device, 0x6f0080 + hdmi, 0x82000000);
+
+	/* NV_PDISP_SF_HDMI_CTRL. */
+	nvkm_mask(device, 0x6165c0 + hoff, 0x401f007f, ctrl);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h
index 57030b3a4a75..7d55faf52fcb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h
@@ -52,6 +52,14 @@ void nv50_head_rgpos(struct nvkm_head *, u16 *, u16 *);
 #define HEAD_DBG(h,f,a...) HEAD_MSG((h), debug, f, ##a)
 
 int nv04_head_new(struct nvkm_disp *, int id);
+
+int nv50_head_cnt(struct nvkm_disp *, unsigned long *);
 int nv50_head_new(struct nvkm_disp *, int id);
+
+int gf119_head_cnt(struct nvkm_disp *, unsigned long *);
 int gf119_head_new(struct nvkm_disp *, int id);
+void gf119_head_rgclk(struct nvkm_head *, int);
+
+int gv100_head_cnt(struct nvkm_disp *, unsigned long *);
+int gv100_head_new(struct nvkm_disp *, int id);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/headgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/headgf119.c
index 9fd7ae331308..e86298b35902 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/headgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/headgf119.c
@@ -39,7 +39,7 @@ gf119_head_vblank_get(struct nvkm_head *head)
 	nvkm_mask(device, 0x6100c0 + hoff, 0x00000001, 0x00000001);
 }
 
-static void
+void
 gf119_head_rgclk(struct nvkm_head *head, int div)
 {
 	struct nvkm_device *device = head->disp->engine.subdev.device;
@@ -92,8 +92,13 @@ gf119_head = {
 int
 gf119_head_new(struct nvkm_disp *disp, int id)
 {
-	struct nvkm_device *device = disp->engine.subdev.device;
-	if (!(nvkm_rd32(device, 0x612004) & (0x00000001 << id)))
-		return 0;
 	return nvkm_head_new_(&gf119_head, disp, id);
 }
+
+int
+gf119_head_cnt(struct nvkm_disp *disp, unsigned long *pmask)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	*pmask = nvkm_rd32(device, 0x612004) & 0x0000000f;
+	return nvkm_rd32(device, 0x022448);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/headgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/headgv100.c
new file mode 100644
index 000000000000..1a061b42ae5c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/headgv100.c
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "head.h"
+
+static void
+gv100_head_vblank_put(struct nvkm_head *head)
+{
+	struct nvkm_device *device = head->disp->engine.subdev.device;
+	nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000004, 0x00000000);
+}
+
+static void
+gv100_head_vblank_get(struct nvkm_head *head)
+{
+	struct nvkm_device *device = head->disp->engine.subdev.device;
+	nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000004, 0x00000004);
+}
+
+static void
+gv100_head_rgpos(struct nvkm_head *head, u16 *hline, u16 *vline)
+{
+	struct nvkm_device *device = head->disp->engine.subdev.device;
+	const u32 hoff = head->id * 0x800;
+	/* vline read locks hline. */
+	*vline = nvkm_rd32(device, 0x616330 + hoff) & 0x0000ffff;
+	*hline = nvkm_rd32(device, 0x616334 + hoff) & 0x0000ffff;
+}
+
+static void
+gv100_head_state(struct nvkm_head *head, struct nvkm_head_state *state)
+{
+	struct nvkm_device *device = head->disp->engine.subdev.device;
+	const u32 hoff = (state == &head->arm) * 0x8000 + head->id * 0x400;
+	u32 data;
+
+	data = nvkm_rd32(device, 0x682064 + hoff);
+	state->vtotal = (data & 0xffff0000) >> 16;
+	state->htotal = (data & 0x0000ffff);
+	data = nvkm_rd32(device, 0x682068 + hoff);
+	state->vsynce = (data & 0xffff0000) >> 16;
+	state->hsynce = (data & 0x0000ffff);
+	data = nvkm_rd32(device, 0x68206c + hoff);
+	state->vblanke = (data & 0xffff0000) >> 16;
+	state->hblanke = (data & 0x0000ffff);
+	data = nvkm_rd32(device, 0x682070 + hoff);
+	state->vblanks = (data & 0xffff0000) >> 16;
+	state->hblanks = (data & 0x0000ffff);
+	state->hz = nvkm_rd32(device, 0x68200c + hoff);
+
+	data = nvkm_rd32(device, 0x682004 + hoff);
+	switch ((data & 0x000000f0) >> 4) {
+	case 5: state->or.depth = 30; break;
+	case 4: state->or.depth = 24; break;
+	case 1: state->or.depth = 18; break;
+	default:
+		state->or.depth = 18;
+		WARN_ON(1);
+		break;
+	}
+}
+
+static const struct nvkm_head_func
+gv100_head = {
+	.state = gv100_head_state,
+	.rgpos = gv100_head_rgpos,
+	.rgclk = gf119_head_rgclk,
+	.vblank_get = gv100_head_vblank_get,
+	.vblank_put = gv100_head_vblank_put,
+};
+
+int
+gv100_head_new(struct nvkm_disp *disp, int id)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	if (!(nvkm_rd32(device, 0x610060) & (0x00000001 << id)))
+		return 0;
+	return nvkm_head_new_(&gv100_head, disp, id);
+}
+
+int
+gv100_head_cnt(struct nvkm_disp *disp, unsigned long *pmask)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	*pmask = nvkm_rd32(device, 0x610060) & 0x000000ff;
+	return nvkm_rd32(device, 0x610074) & 0x0000000f;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/headnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/headnv50.c
index c80d06d5168f..e7d5c397cd29 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/headnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/headnv50.c
@@ -90,3 +90,10 @@ nv50_head_new(struct nvkm_disp *disp, int id)
 {
 	return nvkm_head_new_(&nv50_head, disp, id);
 }
+
+int
+nv50_head_cnt(struct nvkm_disp *disp, unsigned long *pmask)
+{
+	*pmask = 3;
+	return 2;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h
index 4548c031b937..e0b4e0c5704e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h
@@ -30,7 +30,7 @@ struct nvkm_ior {
 			UNKNOWN
 		} proto:3;
 		unsigned link:2;
-		unsigned head:4;
+		unsigned head:8;
 	} arm, asy;
 
 	/* Armed DP state. */
@@ -106,7 +106,6 @@ nv50_sor_link(struct nvkm_ior *ior)
 	return nv50_ior_base(ior) + ((ior->asy.link == 2) * 0x80);
 }
 
-int nv50_sor_new_(const struct nvkm_ior_func *, struct nvkm_disp *, int id);
 void nv50_sor_state(struct nvkm_ior *, struct nvkm_ior_state *);
 void nv50_sor_power(struct nvkm_ior *, bool, bool, bool, bool, bool);
 void nv50_sor_clock(struct nvkm_ior *);
@@ -122,7 +121,6 @@ void g94_sor_dp_watermark(struct nvkm_ior *, int, u8);
 
 void gt215_sor_dp_audio(struct nvkm_ior *, int, bool);
 
-int gf119_sor_new_(const struct nvkm_ior_func *, struct nvkm_disp *, int id);
 void gf119_sor_state(struct nvkm_ior *, struct nvkm_ior_state *);
 void gf119_sor_clock(struct nvkm_ior *);
 int gf119_sor_dp_links(struct nvkm_ior *, struct nvkm_i2c_aux *);
@@ -135,10 +133,15 @@ void gf119_sor_dp_watermark(struct nvkm_ior *, int, u8);
 
 void gm107_sor_dp_pattern(struct nvkm_ior *, int);
 
+void gm200_sor_route_set(struct nvkm_outp *, struct nvkm_ior *);
+int gm200_sor_route_get(struct nvkm_outp *, int *);
+void gm200_sor_dp_drive(struct nvkm_ior *, int, int, int, int, int);
+
 void g84_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8);
 void gt215_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8);
 void gf119_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8);
 void gk104_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8);
+void gv100_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8);
 
 void gt215_hda_hpd(struct nvkm_ior *, int, bool);
 void gt215_hda_eld(struct nvkm_ior *, u8 *, u8);
@@ -153,19 +156,34 @@ void gf119_hda_eld(struct nvkm_ior *, u8 *, u8);
 #define IOR_WARN(i,f,a...) IOR_MSG((i), warn, f, ##a)
 #define IOR_DBG(i,f,a...) IOR_MSG((i), debug, f, ##a)
 
+int nv50_dac_cnt(struct nvkm_disp *, unsigned long *);
 int nv50_dac_new(struct nvkm_disp *, int);
+
+int gf119_dac_cnt(struct nvkm_disp *, unsigned long *);
 int gf119_dac_new(struct nvkm_disp *, int);
 
+int nv50_pior_cnt(struct nvkm_disp *, unsigned long *);
 int nv50_pior_new(struct nvkm_disp *, int);
 
+int nv50_sor_cnt(struct nvkm_disp *, unsigned long *);
 int nv50_sor_new(struct nvkm_disp *, int);
+
 int g84_sor_new(struct nvkm_disp *, int);
+
+int g94_sor_cnt(struct nvkm_disp *, unsigned long *);
 int g94_sor_new(struct nvkm_disp *, int);
+
 int mcp77_sor_new(struct nvkm_disp *, int);
 int gt215_sor_new(struct nvkm_disp *, int);
 int mcp89_sor_new(struct nvkm_disp *, int);
+
+int gf119_sor_cnt(struct nvkm_disp *, unsigned long *);
 int gf119_sor_new(struct nvkm_disp *, int);
+
 int gk104_sor_new(struct nvkm_disp *, int);
 int gm107_sor_new(struct nvkm_disp *, int);
 int gm200_sor_new(struct nvkm_disp *, int);
+
+int gv100_sor_cnt(struct nvkm_disp *, unsigned long *);
+int gv100_sor_new(struct nvkm_disp *, int);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp77.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp77.c
index d7e0fbb12bf1..cfdce23ab83a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp77.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp77.c
@@ -26,18 +26,20 @@
 
 static const struct nv50_disp_func
 mcp77_disp = {
+	.init = nv50_disp_init,
+	.fini = nv50_disp_fini,
 	.intr = nv50_disp_intr,
 	.uevent = &nv50_disp_chan_uevent,
 	.super = nv50_disp_super,
 	.root = &g94_disp_root_oclass,
-	.head.new = nv50_head_new,
-	.dac = { .nr = 3, .new = nv50_dac_new },
-	.sor = { .nr = 4, .new = mcp77_sor_new },
-	.pior = { .nr = 3, .new = nv50_pior_new },
+	.head = { .cnt = nv50_head_cnt, .new = nv50_head_new },
+	.dac = { .cnt = nv50_dac_cnt, .new = nv50_dac_new },
+	.sor = { .cnt = g94_sor_cnt, .new = mcp77_sor_new },
+	.pior = { .cnt = nv50_pior_cnt, .new = nv50_pior_new },
 };
 
 int
 mcp77_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return nv50_disp_new_(&mcp77_disp, device, index, 2, pdisp);
+	return nv50_disp_new_(&mcp77_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp89.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp89.c
index 7b75c57c12ed..85d9329cfa0e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp89.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp89.c
@@ -26,18 +26,20 @@
 
 static const struct nv50_disp_func
 mcp89_disp = {
+	.init = nv50_disp_init,
+	.fini = nv50_disp_fini,
 	.intr = nv50_disp_intr,
 	.uevent = &nv50_disp_chan_uevent,
 	.super = nv50_disp_super,
 	.root = &gt215_disp_root_oclass,
-	.head.new = nv50_head_new,
-	.dac = { .nr = 3, .new = nv50_dac_new },
-	.sor = { .nr = 4, .new = mcp89_sor_new },
-	.pior = { .nr = 3, .new = nv50_pior_new },
+	.head = { .cnt = nv50_head_cnt, .new = nv50_head_new },
+	.dac = { .cnt = nv50_dac_cnt, .new = nv50_dac_new },
+	.sor = { .cnt = g94_sor_cnt, .new = mcp89_sor_new },
+	.pior = { .cnt = nv50_pior_cnt, .new = nv50_pior_new },
 };
 
 int
 mcp89_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return nv50_disp_new_(&mcp89_disp, device, index, 2, pdisp);
+	return nv50_disp_new_(&mcp89_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
index 0c570dbd3021..f89c7b977aa5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
@@ -24,11 +24,12 @@
 #include "nv50.h"
 #include "head.h"
 #include "ior.h"
+#include "channv50.h"
 #include "rootnv50.h"
 
 #include <core/client.h>
 #include <core/enum.h>
-#include <core/gpuobj.h>
+#include <core/ramht.h>
 #include <subdev/bios.h>
 #include <subdev/bios/disp.h>
 #include <subdev/bios/init.h>
@@ -49,29 +50,115 @@ nv50_disp_intr_(struct nvkm_disp *base)
 	disp->func->intr(disp);
 }
 
+static void
+nv50_disp_fini_(struct nvkm_disp *base)
+{
+	struct nv50_disp *disp = nv50_disp(base);
+	disp->func->fini(disp);
+}
+
+static int
+nv50_disp_init_(struct nvkm_disp *base)
+{
+	struct nv50_disp *disp = nv50_disp(base);
+	return disp->func->init(disp);
+}
+
 static void *
 nv50_disp_dtor_(struct nvkm_disp *base)
 {
 	struct nv50_disp *disp = nv50_disp(base);
+
+	nvkm_ramht_del(&disp->ramht);
+	nvkm_gpuobj_del(&disp->inst);
+
 	nvkm_event_fini(&disp->uevent);
 	if (disp->wq)
 		destroy_workqueue(disp->wq);
+
 	return disp;
 }
 
+static int
+nv50_disp_oneinit_(struct nvkm_disp *base)
+{
+	struct nv50_disp *disp = nv50_disp(base);
+	const struct nv50_disp_func *func = disp->func;
+	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	int ret, i;
+
+	if (func->wndw.cnt) {
+		disp->wndw.nr = func->wndw.cnt(&disp->base, &disp->wndw.mask);
+		nvkm_debug(subdev, "Window(s): %d (%08lx)\n",
+			   disp->wndw.nr, disp->wndw.mask);
+	}
+
+	disp->head.nr = func->head.cnt(&disp->base, &disp->head.mask);
+	nvkm_debug(subdev, "  Head(s): %d (%02lx)\n",
+		   disp->head.nr, disp->head.mask);
+	for_each_set_bit(i, &disp->head.mask, disp->head.nr) {
+		ret = func->head.new(&disp->base, i);
+		if (ret)
+			return ret;
+	}
+
+	if (func->dac.cnt) {
+		disp->dac.nr = func->dac.cnt(&disp->base, &disp->dac.mask);
+		nvkm_debug(subdev, "   DAC(s): %d (%02lx)\n",
+			   disp->dac.nr, disp->dac.mask);
+		for_each_set_bit(i, &disp->dac.mask, disp->dac.nr) {
+			ret = func->dac.new(&disp->base, i);
+			if (ret)
+				return ret;
+		}
+	}
+
+	if (func->pior.cnt) {
+		disp->pior.nr = func->pior.cnt(&disp->base, &disp->pior.mask);
+		nvkm_debug(subdev, "  PIOR(s): %d (%02lx)\n",
+			   disp->pior.nr, disp->pior.mask);
+		for_each_set_bit(i, &disp->pior.mask, disp->pior.nr) {
+			ret = func->pior.new(&disp->base, i);
+			if (ret)
+				return ret;
+		}
+	}
+
+	disp->sor.nr = func->sor.cnt(&disp->base, &disp->sor.mask);
+	nvkm_debug(subdev, "   SOR(s): %d (%02lx)\n",
+		   disp->sor.nr, disp->sor.mask);
+	for_each_set_bit(i, &disp->sor.mask, disp->sor.nr) {
+		ret = func->sor.new(&disp->base, i);
+		if (ret)
+			return ret;
+	}
+
+	ret = nvkm_gpuobj_new(device, 0x10000, 0x10000, false, NULL,
+			      &disp->inst);
+	if (ret)
+		return ret;
+
+	return nvkm_ramht_new(device, func->ramht_size ? func->ramht_size :
+			      0x1000, 0, disp->inst, &disp->ramht);
+}
+
 static const struct nvkm_disp_func
 nv50_disp_ = {
 	.dtor = nv50_disp_dtor_,
+	.oneinit = nv50_disp_oneinit_,
+	.init = nv50_disp_init_,
+	.fini = nv50_disp_fini_,
 	.intr = nv50_disp_intr_,
 	.root = nv50_disp_root_,
 };
 
 int
 nv50_disp_new_(const struct nv50_disp_func *func, struct nvkm_device *device,
-	       int index, int heads, struct nvkm_disp **pdisp)
+	       int index, struct nvkm_disp **pdisp)
 {
 	struct nv50_disp *disp;
-	int ret, i;
+	int ret;
 
 	if (!(disp = kzalloc(sizeof(*disp), GFP_KERNEL)))
 		return -ENOMEM;
@@ -85,33 +172,11 @@ nv50_disp_new_(const struct nv50_disp_func *func, struct nvkm_device *device,
 	disp->wq = create_singlethread_workqueue("nvkm-disp");
 	if (!disp->wq)
 		return -ENOMEM;
-	INIT_WORK(&disp->supervisor, func->super);
-
-	for (i = 0; func->head.new && i < heads; i++) {
-		ret = func->head.new(&disp->base, i);
-		if (ret)
-			return ret;
-	}
-
-	for (i = 0; func->dac.new && i < func->dac.nr; i++) {
-		ret = func->dac.new(&disp->base, i);
-		if (ret)
-			return ret;
-	}
-
-	for (i = 0; func->pior.new && i < func->pior.nr; i++) {
-		ret = func->pior.new(&disp->base, i);
-		if (ret)
-			return ret;
-	}
 
-	for (i = 0; func->sor.new && i < func->sor.nr; i++) {
-		ret = func->sor.new(&disp->base, i);
-		if (ret)
-			return ret;
-	}
+	INIT_WORK(&disp->supervisor, func->super);
 
-	return nvkm_event_init(func->uevent, 1, 1 + (heads * 4), &disp->uevent);
+	return nvkm_event_init(func->uevent, 1, ARRAY_SIZE(disp->chan),
+			       &disp->uevent);
 }
 
 static u32
@@ -613,20 +678,96 @@ nv50_disp_intr(struct nv50_disp *disp)
 	}
 }
 
+void
+nv50_disp_fini(struct nv50_disp *disp)
+{
+	struct nvkm_device *device = disp->base.engine.subdev.device;
+	/* disable all interrupts */
+	nvkm_wr32(device, 0x610024, 0x00000000);
+	nvkm_wr32(device, 0x610020, 0x00000000);
+}
+
+int
+nv50_disp_init(struct nv50_disp *disp)
+{
+	struct nvkm_device *device = disp->base.engine.subdev.device;
+	struct nvkm_head *head;
+	u32 tmp;
+	int i;
+
+	/* The below segments of code copying values from one register to
+	 * another appear to inform EVO of the display capabilities or
+	 * something similar.  NFI what the 0x614004 caps are for..
+	 */
+	tmp = nvkm_rd32(device, 0x614004);
+	nvkm_wr32(device, 0x610184, tmp);
+
+	/* ... CRTC caps */
+	list_for_each_entry(head, &disp->base.head, head) {
+		tmp = nvkm_rd32(device, 0x616100 + (head->id * 0x800));
+		nvkm_wr32(device, 0x610190 + (head->id * 0x10), tmp);
+		tmp = nvkm_rd32(device, 0x616104 + (head->id * 0x800));
+		nvkm_wr32(device, 0x610194 + (head->id * 0x10), tmp);
+		tmp = nvkm_rd32(device, 0x616108 + (head->id * 0x800));
+		nvkm_wr32(device, 0x610198 + (head->id * 0x10), tmp);
+		tmp = nvkm_rd32(device, 0x61610c + (head->id * 0x800));
+		nvkm_wr32(device, 0x61019c + (head->id * 0x10), tmp);
+	}
+
+	/* ... DAC caps */
+	for (i = 0; i < disp->dac.nr; i++) {
+		tmp = nvkm_rd32(device, 0x61a000 + (i * 0x800));
+		nvkm_wr32(device, 0x6101d0 + (i * 0x04), tmp);
+	}
+
+	/* ... SOR caps */
+	for (i = 0; i < disp->sor.nr; i++) {
+		tmp = nvkm_rd32(device, 0x61c000 + (i * 0x800));
+		nvkm_wr32(device, 0x6101e0 + (i * 0x04), tmp);
+	}
+
+	/* ... PIOR caps */
+	for (i = 0; i < disp->pior.nr; i++) {
+		tmp = nvkm_rd32(device, 0x61e000 + (i * 0x800));
+		nvkm_wr32(device, 0x6101f0 + (i * 0x04), tmp);
+	}
+
+	/* steal display away from vbios, or something like that */
+	if (nvkm_rd32(device, 0x610024) & 0x00000100) {
+		nvkm_wr32(device, 0x610024, 0x00000100);
+		nvkm_mask(device, 0x6194e8, 0x00000001, 0x00000000);
+		if (nvkm_msec(device, 2000,
+			if (!(nvkm_rd32(device, 0x6194e8) & 0x00000002))
+				break;
+		) < 0)
+			return -EBUSY;
+	}
+
+	/* point at display engine memory area (hash table, objects) */
+	nvkm_wr32(device, 0x610010, (disp->inst->addr >> 8) | 9);
+
+	/* enable supervisor interrupts, disable everything else */
+	nvkm_wr32(device, 0x61002c, 0x00000370);
+	nvkm_wr32(device, 0x610028, 0x00000000);
+	return 0;
+}
+
 static const struct nv50_disp_func
 nv50_disp = {
+	.init = nv50_disp_init,
+	.fini = nv50_disp_fini,
 	.intr = nv50_disp_intr,
 	.uevent = &nv50_disp_chan_uevent,
 	.super = nv50_disp_super,
 	.root = &nv50_disp_root_oclass,
-	.head.new = nv50_head_new,
-	.dac = { .nr = 3, .new = nv50_dac_new },
-	.sor = { .nr = 2, .new = nv50_sor_new },
-	.pior = { .nr = 3, .new = nv50_pior_new },
+	.head = { .cnt = nv50_head_cnt, .new = nv50_head_new },
+	.dac = { .cnt = nv50_dac_cnt, .new = nv50_dac_new },
+	.sor = { .cnt = nv50_sor_cnt, .new = nv50_sor_new },
+	.pior = { .cnt = nv50_pior_cnt, .new = nv50_pior_new },
 };
 
 int
 nv50_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return nv50_disp_new_(&nv50_disp, device, index, 2, pdisp);
+	return nv50_disp_new_(&nv50_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h
index eb0b8acb1c5b..8580382ab248 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h
@@ -16,14 +16,26 @@ struct nv50_disp {
 	struct nvkm_event uevent;
 
 	struct {
+		unsigned long mask;
+		int nr;
+	} wndw, head, dac;
+
+	struct {
+		unsigned long mask;
+		int nr;
 		u32 lvdsconf;
 	} sor;
 
 	struct {
+		unsigned long mask;
+		int nr;
 		u8 type[3];
 	} pior;
 
-	struct nv50_disp_chan *chan[21];
+	struct nvkm_gpuobj *inst;
+	struct nvkm_ramht *ramht;
+
+	struct nv50_disp_chan *chan[81];
 };
 
 void nv50_disp_super_1(struct nv50_disp *);
@@ -34,11 +46,11 @@ void nv50_disp_super_2_2(struct nv50_disp *, struct nvkm_head *);
 void nv50_disp_super_3_0(struct nv50_disp *, struct nvkm_head *);
 
 int nv50_disp_new_(const struct nv50_disp_func *, struct nvkm_device *,
-		   int index, int heads, struct nvkm_disp **);
-int gf119_disp_new_(const struct nv50_disp_func *, struct nvkm_device *,
-		    int index, struct nvkm_disp **);
+		   int index, struct nvkm_disp **);
 
 struct nv50_disp_func {
+	int (*init)(struct nv50_disp *);
+	void (*fini)(struct nv50_disp *);
 	void (*intr)(struct nv50_disp *);
 	void (*intr_error)(struct nv50_disp *, int chid);
 
@@ -48,28 +60,20 @@ struct nv50_disp_func {
 	const struct nvkm_disp_oclass *root;
 
 	struct {
+		int (*cnt)(struct nvkm_disp *, unsigned long *mask);
 		int (*new)(struct nvkm_disp *, int id);
-	} head;
+	} wndw, head, dac, sor, pior;
 
-	struct {
-		int nr;
-		int (*new)(struct nvkm_disp *, int id);
-	} dac;
-
-	struct {
-		int nr;
-		int (*new)(struct nvkm_disp *, int id);
-	} sor;
-
-	struct {
-		int nr;
-		int (*new)(struct nvkm_disp *, int id);
-	} pior;
+	u16 ramht_size;
 };
 
+int nv50_disp_init(struct nv50_disp *);
+void nv50_disp_fini(struct nv50_disp *);
 void nv50_disp_intr(struct nv50_disp *);
 void nv50_disp_super(struct work_struct *);
 
+int gf119_disp_init(struct nv50_disp *);
+void gf119_disp_fini(struct nv50_disp *);
 void gf119_disp_intr(struct nv50_disp *);
 void gf119_disp_super(struct work_struct *);
 void gf119_disp_intr_error(struct nv50_disp *, int);
@@ -77,4 +81,12 @@ void gf119_disp_intr_error(struct nv50_disp *, int);
 void nv50_disp_dptmds_war_2(struct nv50_disp *, struct dcb_output *);
 void nv50_disp_dptmds_war_3(struct nv50_disp *, struct dcb_output *);
 void nv50_disp_update_sppll1(struct nv50_disp *);
+
+extern const struct nvkm_event_func nv50_disp_chan_uevent;
+int  nv50_disp_chan_uevent_ctor(struct nvkm_object *, void *, u32,
+				struct nvkm_notify *);
+void nv50_disp_chan_uevent_send(struct nv50_disp *, int);
+
+extern const struct nvkm_event_func gf119_disp_chan_uevent;
+extern const struct nvkm_event_func gv100_disp_chan_uevent;
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmg84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmg84.c
deleted file mode 100644
index 5ad5d0f5db05..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmg84.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2012 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-#include "channv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_pioc_oclass
-g84_disp_oimm_oclass = {
-	.base.oclass = G82_DISP_OVERLAY,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_oimm_new,
-	.func = &nv50_disp_pioc_func,
-	.chid = { 5, 5 },
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgf119.c
index 1f9fd3403f07..1ae0bcfc89b9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgf119.c
@@ -22,16 +22,11 @@
  * Authors: Ben Skeggs
  */
 #include "channv50.h"
-#include "rootnv50.h"
 
-#include <nvif/class.h>
-
-const struct nv50_disp_pioc_oclass
-gf119_disp_oimm_oclass = {
-	.base.oclass = GF110_DISP_OVERLAY,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_oimm_new,
-	.func = &gf119_disp_pioc_func,
-	.chid = { 9, 9 },
-};
+int
+gf119_disp_oimm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_oimm_new_(&gf119_disp_pioc_func, disp, 9, 9,
+				   oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgk104.c
deleted file mode 100644
index 0c09fe85e952..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgk104.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2012 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-#include "channv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_pioc_oclass
-gk104_disp_oimm_oclass = {
-	.base.oclass = GK104_DISP_OVERLAY,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_oimm_new,
-	.func = &gf119_disp_pioc_func,
-	.chid = { 9, 9 },
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgp102.c
index abf82365c671..30ffb1008505 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgp102.c
@@ -22,16 +22,11 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 #include "channv50.h"
-#include "rootnv50.h"
 
-#include <nvif/class.h>
-
-const struct nv50_disp_pioc_oclass
-gp102_disp_oimm_oclass = {
-	.base.oclass = GK104_DISP_OVERLAY,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_oimm_new,
-	.func = &gf119_disp_pioc_func,
-	.chid = { 9, 13 },
-};
+int
+gp102_disp_oimm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_oimm_new_(&gf119_disp_pioc_func, disp, 9, 13,
+				   oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgt215.c
deleted file mode 100644
index 1281db28aebd..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgt215.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2012 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-#include "channv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_pioc_oclass
-gt215_disp_oimm_oclass = {
-	.base.oclass = GT214_DISP_OVERLAY,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_oimm_new,
-	.func = &nv50_disp_pioc_func,
-	.chid = { 5, 5 },
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c
index f3b0fa2c5924..0db99bfe9db9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c
@@ -23,30 +23,26 @@
  */
 #include "channv50.h"
 #include "head.h"
-#include "rootnv50.h"
 
 #include <core/client.h>
 
-#include <nvif/class.h>
 #include <nvif/cl507b.h>
 #include <nvif/unpack.h>
 
 int
-nv50_disp_oimm_new(const struct nv50_disp_chan_func *func,
-		   const struct nv50_disp_chan_mthd *mthd,
-		   struct nv50_disp_root *root, int ctrl, int user,
-		   const struct nvkm_oclass *oclass, void *data, u32 size,
-		   struct nvkm_object **pobject)
+nv50_disp_oimm_new_(const struct nv50_disp_chan_func *func,
+		    struct nv50_disp *disp, int ctrl, int user,
+		    const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nvkm_object **pobject)
 {
 	union {
 		struct nv50_disp_overlay_v0 v0;
-	} *args = data;
+	} *args = argv;
 	struct nvkm_object *parent = oclass->parent;
-	struct nv50_disp *disp = root->disp;
 	int head, ret = -ENOSYS;
 
-	nvif_ioctl(parent, "create disp overlay size %d\n", size);
-	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
+	nvif_ioctl(parent, "create disp overlay size %d\n", argc);
+	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create disp overlay vers %d head %d\n",
 			   args->v0.version, args->v0.head);
 		if (!nvkm_head_find(&disp->base, args->v0.head))
@@ -55,16 +51,14 @@ nv50_disp_oimm_new(const struct nv50_disp_chan_func *func,
 	} else
 		return ret;
 
-	return nv50_disp_chan_new_(func, mthd, root, ctrl + head, user + head,
+	return nv50_disp_chan_new_(func, NULL, disp, ctrl + head, user + head,
 				   head, oclass, pobject);
 }
 
-const struct nv50_disp_pioc_oclass
-nv50_disp_oimm_oclass = {
-	.base.oclass = NV50_DISP_OVERLAY,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_oimm_new,
-	.func = &nv50_disp_pioc_func,
-	.chid = { 5, 5 },
-};
+int
+nv50_disp_oimm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		   struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_oimm_new_(&nv50_disp_pioc_func, disp, 5, 5,
+				   oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlyg84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlyg84.c
index db6234eebc61..31b915d48699 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlyg84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlyg84.c
@@ -21,10 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
+#include "channv50.h"
 
 static const struct nv50_disp_mthd_list
 g84_disp_ovly_mthd_base = {
@@ -54,8 +51,8 @@ g84_disp_ovly_mthd_base = {
 	}
 };
 
-const struct nv50_disp_chan_mthd
-g84_disp_ovly_chan_mthd = {
+static const struct nv50_disp_chan_mthd
+g84_disp_ovly_mthd = {
 	.name = "Overlay",
 	.addr = 0x000540,
 	.prev = 0x000004,
@@ -65,13 +62,10 @@ g84_disp_ovly_chan_mthd = {
 	}
 };
 
-const struct nv50_disp_dmac_oclass
-g84_disp_ovly_oclass = {
-	.base.oclass = G82_DISP_OVERLAY_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_ovly_new,
-	.func = &nv50_disp_dmac_func,
-	.mthd = &g84_disp_ovly_chan_mthd,
-	.chid = 3,
-};
+int
+g84_disp_ovly_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		  struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_ovly_new_(&nv50_disp_dmac_func, &g84_disp_ovly_mthd,
+				   disp, 3, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygf119.c
index 5985879abd23..83fd534c44da 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygf119.c
@@ -21,10 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
+#include "channv50.h"
 
 static const struct nv50_disp_mthd_list
 gf119_disp_ovly_mthd_base = {
@@ -79,7 +76,7 @@ gf119_disp_ovly_mthd_base = {
 };
 
 static const struct nv50_disp_chan_mthd
-gf119_disp_ovly_chan_mthd = {
+gf119_disp_ovly_mthd = {
 	.name = "Overlay",
 	.addr = 0x001000,
 	.prev = -0x020000,
@@ -89,13 +86,10 @@ gf119_disp_ovly_chan_mthd = {
 	}
 };
 
-const struct nv50_disp_dmac_oclass
-gf119_disp_ovly_oclass = {
-	.base.oclass = GF110_DISP_OVERLAY_CONTROL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_ovly_new,
-	.func = &gf119_disp_dmac_func,
-	.mthd = &gf119_disp_ovly_chan_mthd,
-	.chid = 5,
-};
+int
+gf119_disp_ovly_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_ovly_new_(&gf119_disp_dmac_func, &gf119_disp_ovly_mthd,
+				   disp, 5, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygk104.c
index 2f0220b39f34..a7acacbc92c1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygk104.c
@@ -21,10 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
+#include "channv50.h"
 
 static const struct nv50_disp_mthd_list
 gk104_disp_ovly_mthd_base = {
@@ -81,7 +78,7 @@ gk104_disp_ovly_mthd_base = {
 };
 
 const struct nv50_disp_chan_mthd
-gk104_disp_ovly_chan_mthd = {
+gk104_disp_ovly_mthd = {
 	.name = "Overlay",
 	.addr = 0x001000,
 	.prev = -0x020000,
@@ -91,13 +88,10 @@ gk104_disp_ovly_chan_mthd = {
 	}
 };
 
-const struct nv50_disp_dmac_oclass
-gk104_disp_ovly_oclass = {
-	.base.oclass = GK104_DISP_OVERLAY_CONTROL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_ovly_new,
-	.func = &gf119_disp_dmac_func,
-	.mthd = &gk104_disp_ovly_chan_mthd,
-	.chid = 5,
-};
+int
+gk104_disp_ovly_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_ovly_new_(&gf119_disp_dmac_func, &gk104_disp_ovly_mthd,
+				   disp, 5, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp102.c
index 589bd2f12b41..e0eca6ea914c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp102.c
@@ -21,18 +21,12 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
+#include "channv50.h"
 
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gp102_disp_ovly_oclass = {
-	.base.oclass = GK104_DISP_OVERLAY_CONTROL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_ovly_new,
-	.func = &gp102_disp_dmac_func,
-	.mthd = &gk104_disp_ovly_chan_mthd,
-	.chid = 5,
-};
+int
+gp102_disp_ovly_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_ovly_new_(&gp102_disp_dmac_func, &gk104_disp_ovly_mthd,
+				   disp, 5, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygt200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygt200.c
index f858053db83d..dc60cd00dc16 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygt200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygt200.c
@@ -21,10 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
+#include "channv50.h"
 
 static const struct nv50_disp_mthd_list
 gt200_disp_ovly_mthd_base = {
@@ -58,7 +55,7 @@ gt200_disp_ovly_mthd_base = {
 };
 
 static const struct nv50_disp_chan_mthd
-gt200_disp_ovly_chan_mthd = {
+gt200_disp_ovly_mthd = {
 	.name = "Overlay",
 	.addr = 0x000540,
 	.prev = 0x000004,
@@ -68,13 +65,10 @@ gt200_disp_ovly_chan_mthd = {
 	}
 };
 
-const struct nv50_disp_dmac_oclass
-gt200_disp_ovly_oclass = {
-	.base.oclass = GT200_DISP_OVERLAY_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_ovly_new,
-	.func = &nv50_disp_dmac_func,
-	.mthd = &gt200_disp_ovly_chan_mthd,
-	.chid = 3,
-};
+int
+gt200_disp_ovly_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_ovly_new_(&nv50_disp_dmac_func, &gt200_disp_ovly_mthd,
+				   disp, 3, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygt215.c
deleted file mode 100644
index c947e1e16a37..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygt215.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs <bskeggs@redhat.com>
- */
-#include "dmacnv50.h"
-#include "rootnv50.h"
-
-#include <nvif/class.h>
-
-const struct nv50_disp_dmac_oclass
-gt215_disp_ovly_oclass = {
-	.base.oclass = GT214_DISP_OVERLAY_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_ovly_new,
-	.func = &nv50_disp_dmac_func,
-	.mthd = &g84_disp_ovly_chan_mthd,
-	.chid = 3,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlynv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlynv50.c
index 9ebaaa6e9e33..6974c12c4518 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlynv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlynv50.c
@@ -21,33 +21,30 @@
  *
  * Authors: Ben Skeggs
  */
-#include "dmacnv50.h"
+#include "channv50.h"
 #include "head.h"
-#include "rootnv50.h"
 
 #include <core/client.h>
 
-#include <nvif/class.h>
 #include <nvif/cl507e.h>
 #include <nvif/unpack.h>
 
 int
-nv50_disp_ovly_new(const struct nv50_disp_dmac_func *func,
-		   const struct nv50_disp_chan_mthd *mthd,
-		   struct nv50_disp_root *root, int chid,
-		   const struct nvkm_oclass *oclass, void *data, u32 size,
-		   struct nvkm_object **pobject)
+nv50_disp_ovly_new_(const struct nv50_disp_chan_func *func,
+		    const struct nv50_disp_chan_mthd *mthd,
+		    struct nv50_disp *disp, int chid,
+		    const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nvkm_object **pobject)
 {
 	union {
 		struct nv50_disp_overlay_channel_dma_v0 v0;
-	} *args = data;
+	} *args = argv;
 	struct nvkm_object *parent = oclass->parent;
-	struct nv50_disp *disp = root->disp;
 	int head, ret = -ENOSYS;
 	u64 push;
 
-	nvif_ioctl(parent, "create disp overlay channel dma size %d\n", size);
-	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
+	nvif_ioctl(parent, "create disp overlay channel dma size %d\n", argc);
+	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create disp overlay channel dma vers %d "
 				   "pushbuf %016llx head %d\n",
 			   args->v0.version, args->v0.pushbuf, args->v0.head);
@@ -58,7 +55,7 @@ nv50_disp_ovly_new(const struct nv50_disp_dmac_func *func,
 	} else
 		return ret;
 
-	return nv50_disp_dmac_new_(func, mthd, root, chid + head,
+	return nv50_disp_dmac_new_(func, mthd, disp, chid + head,
 				   head, push, oclass, pobject);
 }
 
@@ -91,7 +88,7 @@ nv50_disp_ovly_mthd_base = {
 };
 
 static const struct nv50_disp_chan_mthd
-nv50_disp_ovly_chan_mthd = {
+nv50_disp_ovly_mthd = {
 	.name = "Overlay",
 	.addr = 0x000540,
 	.prev = 0x000004,
@@ -101,13 +98,10 @@ nv50_disp_ovly_chan_mthd = {
 	}
 };
 
-const struct nv50_disp_dmac_oclass
-nv50_disp_ovly_oclass = {
-	.base.oclass = NV50_DISP_OVERLAY_CHANNEL_DMA,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = nv50_disp_ovly_new,
-	.func = &nv50_disp_dmac_func,
-	.mthd = &nv50_disp_ovly_chan_mthd,
-	.chid = 3,
-};
+int
+nv50_disp_ovly_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		   struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return nv50_disp_ovly_new_(&nv50_disp_dmac_func, &nv50_disp_ovly_mthd,
+				   disp, 3, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocgf119.c
index 0abaa6431943..5296e7bee813 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocgf119.c
@@ -29,7 +29,7 @@
 static void
 gf119_disp_pioc_fini(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->root->disp;
+	struct nv50_disp *disp = chan->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 	int ctrl = chan->chid.ctrl;
@@ -43,24 +43,17 @@ gf119_disp_pioc_fini(struct nv50_disp_chan *chan)
 		nvkm_error(subdev, "ch %d fini: %08x\n", user,
 			   nvkm_rd32(device, 0x610490 + (ctrl * 0x10)));
 	}
-
-	/* disable error reporting and completion notification */
-	nvkm_mask(device, 0x610090, 0x00000001 << user, 0x00000000);
-	nvkm_mask(device, 0x6100a0, 0x00000001 << user, 0x00000000);
 }
 
 static int
 gf119_disp_pioc_init(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->root->disp;
+	struct nv50_disp *disp = chan->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 	int ctrl = chan->chid.ctrl;
 	int user = chan->chid.user;
 
-	/* enable error reporting */
-	nvkm_mask(device, 0x6100a0, 0x00000001 << user, 0x00000001 << user);
-
 	/* activate channel */
 	nvkm_wr32(device, 0x610490 + (ctrl * 0x10), 0x00000001);
 	if (nvkm_msec(device, 2000,
@@ -80,4 +73,6 @@ const struct nv50_disp_chan_func
 gf119_disp_pioc_func = {
 	.init = gf119_disp_pioc_init,
 	.fini = gf119_disp_pioc_fini,
+	.intr = gf119_disp_chan_intr,
+	.user = nv50_disp_chan_user,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocnv50.c
index 0211e0e8a35f..4faed6fce682 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocnv50.c
@@ -29,7 +29,7 @@
 static void
 nv50_disp_pioc_fini(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->root->disp;
+	struct nv50_disp *disp = chan->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 	int ctrl = chan->chid.ctrl;
@@ -48,7 +48,7 @@ nv50_disp_pioc_fini(struct nv50_disp_chan *chan)
 static int
 nv50_disp_pioc_init(struct nv50_disp_chan *chan)
 {
-	struct nv50_disp *disp = chan->root->disp;
+	struct nv50_disp *disp = chan->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 	int ctrl = chan->chid.ctrl;
@@ -82,4 +82,6 @@ const struct nv50_disp_chan_func
 nv50_disp_pioc_func = {
 	.init = nv50_disp_pioc_init,
 	.fini = nv50_disp_pioc_fini,
+	.intr = nv50_disp_chan_intr,
+	.user = nv50_disp_chan_user,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piornv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piornv50.c
index 99b3b9050635..e997a207f546 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piornv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piornv50.c
@@ -127,8 +127,13 @@ nv50_pior = {
 int
 nv50_pior_new(struct nvkm_disp *disp, int id)
 {
-	struct nvkm_device *device = disp->engine.subdev.device;
-	if (!(nvkm_rd32(device, 0x610184) & (0x10000000 << id)))
-		return 0;
 	return nvkm_ior_new_(&nv50_pior, disp, PIOR, id);
 }
+
+int
+nv50_pior_cnt(struct nvkm_disp *disp, unsigned long *pmask)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	*pmask = (nvkm_rd32(device, 0x610184) & 0x70000000) >> 28;
+	return 3;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h
index 6c9bfff6d043..ef66c5f38ad5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h
@@ -12,6 +12,9 @@ void nvkm_disp_vblank(struct nvkm_disp *, int head);
 
 struct nvkm_disp_func {
 	void *(*dtor)(struct nvkm_disp *);
+	int (*oneinit)(struct nvkm_disp *);
+	int (*init)(struct nvkm_disp *);
+	void (*fini)(struct nvkm_disp *);
 	void (*intr)(struct nvkm_disp *);
 
 	const struct nvkm_disp_oclass *(*root)(struct nvkm_disp *);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootg84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootg84.c
index 721e4f74d1fc..1ed371fd7ddf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootg84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootg84.c
@@ -22,22 +22,19 @@
  * Authors: Ben Skeggs
  */
 #include "rootnv50.h"
-#include "dmacnv50.h"
+#include "channv50.h"
 
 #include <nvif/class.h>
 
 static const struct nv50_disp_root_func
 g84_disp_root = {
-	.init = nv50_disp_root_init,
-	.fini = nv50_disp_root_fini,
-	.dmac = {
-		&g84_disp_core_oclass,
-		&g84_disp_base_oclass,
-		&g84_disp_ovly_oclass,
-	},
-	.pioc = {
-		&g84_disp_oimm_oclass,
-		&g84_disp_curs_oclass,
+	.user = {
+		{{0,0,G82_DISP_CURSOR             }, nv50_disp_curs_new },
+		{{0,0,G82_DISP_OVERLAY            }, nv50_disp_oimm_new },
+		{{0,0,G82_DISP_BASE_CHANNEL_DMA   },  g84_disp_base_new },
+		{{0,0,G82_DISP_CORE_CHANNEL_DMA   },  g84_disp_core_new },
+		{{0,0,G82_DISP_OVERLAY_CHANNEL_DMA},  g84_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootg94.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootg94.c
index 9493f6edf62b..ef579eb00238 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootg94.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootg94.c
@@ -22,22 +22,19 @@
  * Authors: Ben Skeggs
  */
 #include "rootnv50.h"
-#include "dmacnv50.h"
+#include "channv50.h"
 
 #include <nvif/class.h>
 
 static const struct nv50_disp_root_func
 g94_disp_root = {
-	.init = nv50_disp_root_init,
-	.fini = nv50_disp_root_fini,
-	.dmac = {
-		&g94_disp_core_oclass,
-		&gt200_disp_base_oclass,
-		&gt200_disp_ovly_oclass,
-	},
-	.pioc = {
-		&g84_disp_oimm_oclass,
-		&g84_disp_curs_oclass,
+	.user = {
+		{{0,0,  G82_DISP_CURSOR             },  nv50_disp_curs_new },
+		{{0,0,  G82_DISP_OVERLAY            },  nv50_disp_oimm_new },
+		{{0,0,GT200_DISP_BASE_CHANNEL_DMA   },   g84_disp_base_new },
+		{{0,0,GT206_DISP_CORE_CHANNEL_DMA   },   g94_disp_core_new },
+		{{0,0,GT200_DISP_OVERLAY_CHANNEL_DMA}, gt200_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgf119.c
index 333c8424b413..fe011165dc02 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgf119.c
@@ -22,104 +22,19 @@
  * Authors: Ben Skeggs
  */
 #include "rootnv50.h"
-#include "head.h"
-#include "dmacnv50.h"
-
-#include <core/ramht.h>
-#include <subdev/timer.h>
+#include "channv50.h"
 
 #include <nvif/class.h>
 
-void
-gf119_disp_root_fini(struct nv50_disp_root *root)
-{
-	struct nvkm_device *device = root->disp->base.engine.subdev.device;
-	/* disable all interrupts */
-	nvkm_wr32(device, 0x6100b0, 0x00000000);
-}
-
-int
-gf119_disp_root_init(struct nv50_disp_root *root)
-{
-	struct nv50_disp *disp = root->disp;
-	struct nvkm_head *head;
-	struct nvkm_device *device = disp->base.engine.subdev.device;
-	u32 tmp;
-	int i;
-
-	/* The below segments of code copying values from one register to
-	 * another appear to inform EVO of the display capabilities or
-	 * something similar.
-	 */
-
-	/* ... CRTC caps */
-	list_for_each_entry(head, &disp->base.head, head) {
-		const u32 hoff = head->id * 0x800;
-		tmp = nvkm_rd32(device, 0x616104 + hoff);
-		nvkm_wr32(device, 0x6101b4 + hoff, tmp);
-		tmp = nvkm_rd32(device, 0x616108 + hoff);
-		nvkm_wr32(device, 0x6101b8 + hoff, tmp);
-		tmp = nvkm_rd32(device, 0x61610c + hoff);
-		nvkm_wr32(device, 0x6101bc + hoff, tmp);
-	}
-
-	/* ... DAC caps */
-	for (i = 0; i < disp->func->dac.nr; i++) {
-		tmp = nvkm_rd32(device, 0x61a000 + (i * 0x800));
-		nvkm_wr32(device, 0x6101c0 + (i * 0x800), tmp);
-	}
-
-	/* ... SOR caps */
-	for (i = 0; i < disp->func->sor.nr; i++) {
-		tmp = nvkm_rd32(device, 0x61c000 + (i * 0x800));
-		nvkm_wr32(device, 0x6301c4 + (i * 0x800), tmp);
-	}
-
-	/* steal display away from vbios, or something like that */
-	if (nvkm_rd32(device, 0x6100ac) & 0x00000100) {
-		nvkm_wr32(device, 0x6100ac, 0x00000100);
-		nvkm_mask(device, 0x6194e8, 0x00000001, 0x00000000);
-		if (nvkm_msec(device, 2000,
-			if (!(nvkm_rd32(device, 0x6194e8) & 0x00000002))
-				break;
-		) < 0)
-			return -EBUSY;
-	}
-
-	/* point at display engine memory area (hash table, objects) */
-	nvkm_wr32(device, 0x610010, (root->instmem->addr >> 8) | 9);
-
-	/* enable supervisor interrupts, disable everything else */
-	nvkm_wr32(device, 0x610090, 0x00000000);
-	nvkm_wr32(device, 0x6100a0, 0x00000000);
-	nvkm_wr32(device, 0x6100b0, 0x00000307);
-
-	/* disable underflow reporting, preventing an intermittent issue
-	 * on some gk104 boards where the production vbios left this
-	 * setting enabled by default.
-	 *
-	 * ftp://download.nvidia.com/open-gpu-doc/gk104-disable-underflow-reporting/1/gk104-disable-underflow-reporting.txt
-	 */
-	list_for_each_entry(head, &disp->base.head, head) {
-		const u32 hoff = head->id * 0x800;
-		nvkm_mask(device, 0x616308 + hoff, 0x00000111, 0x00000010);
-	}
-
-	return 0;
-}
-
 static const struct nv50_disp_root_func
 gf119_disp_root = {
-	.init = gf119_disp_root_init,
-	.fini = gf119_disp_root_fini,
-	.dmac = {
-		&gf119_disp_core_oclass,
-		&gf119_disp_base_oclass,
-		&gf119_disp_ovly_oclass,
-	},
-	.pioc = {
-		&gf119_disp_oimm_oclass,
-		&gf119_disp_curs_oclass,
+	.user = {
+		{{0,0,GF110_DISP_CURSOR             }, gf119_disp_curs_new },
+		{{0,0,GF110_DISP_OVERLAY            }, gf119_disp_oimm_new },
+		{{0,0,GF110_DISP_BASE_CHANNEL_DMA   }, gf119_disp_base_new },
+		{{0,0,GF110_DISP_CORE_CHANNEL_DMA   }, gf119_disp_core_new },
+		{{0,0,GF110_DISP_OVERLAY_CONTROL_DMA}, gf119_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgk104.c
index 0bfdb1d1c6ab..9e8ffd348b50 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgk104.c
@@ -22,22 +22,19 @@
  * Authors: Ben Skeggs
  */
 #include "rootnv50.h"
-#include "dmacnv50.h"
+#include "channv50.h"
 
 #include <nvif/class.h>
 
 static const struct nv50_disp_root_func
 gk104_disp_root = {
-	.init = gf119_disp_root_init,
-	.fini = gf119_disp_root_fini,
-	.dmac = {
-		&gk104_disp_core_oclass,
-		&gk104_disp_base_oclass,
-		&gk104_disp_ovly_oclass,
-	},
-	.pioc = {
-		&gk104_disp_oimm_oclass,
-		&gk104_disp_curs_oclass,
+	.user = {
+		{{0,0,GK104_DISP_CURSOR             }, gf119_disp_curs_new },
+		{{0,0,GK104_DISP_OVERLAY            }, gf119_disp_oimm_new },
+		{{0,0,GK104_DISP_BASE_CHANNEL_DMA   }, gf119_disp_base_new },
+		{{0,0,GK104_DISP_CORE_CHANNEL_DMA   }, gk104_disp_core_new },
+		{{0,0,GK104_DISP_OVERLAY_CONTROL_DMA}, gk104_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgk110.c
index 1e8dbed8a67c..dc85cc1c9490 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgk110.c
@@ -22,22 +22,19 @@
  * Authors: Ben Skeggs
  */
 #include "rootnv50.h"
-#include "dmacnv50.h"
+#include "channv50.h"
 
 #include <nvif/class.h>
 
 static const struct nv50_disp_root_func
 gk110_disp_root = {
-	.init = gf119_disp_root_init,
-	.fini = gf119_disp_root_fini,
-	.dmac = {
-		&gk110_disp_core_oclass,
-		&gk110_disp_base_oclass,
-		&gk104_disp_ovly_oclass,
-	},
-	.pioc = {
-		&gk104_disp_oimm_oclass,
-		&gk104_disp_curs_oclass,
+	.user = {
+		{{0,0,GK104_DISP_CURSOR             }, gf119_disp_curs_new },
+		{{0,0,GK104_DISP_OVERLAY            }, gf119_disp_oimm_new },
+		{{0,0,GK110_DISP_BASE_CHANNEL_DMA   }, gf119_disp_base_new },
+		{{0,0,GK110_DISP_CORE_CHANNEL_DMA   }, gk104_disp_core_new },
+		{{0,0,GK104_DISP_OVERLAY_CONTROL_DMA}, gk104_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgm107.c
index 44c55be69e99..e0181ca08840 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgm107.c
@@ -22,22 +22,19 @@
  * Authors: Ben Skeggs
  */
 #include "rootnv50.h"
-#include "dmacnv50.h"
+#include "channv50.h"
 
 #include <nvif/class.h>
 
 static const struct nv50_disp_root_func
 gm107_disp_root = {
-	.init = gf119_disp_root_init,
-	.fini = gf119_disp_root_fini,
-	.dmac = {
-		&gm107_disp_core_oclass,
-		&gk110_disp_base_oclass,
-		&gk104_disp_ovly_oclass,
-	},
-	.pioc = {
-		&gk104_disp_oimm_oclass,
-		&gk104_disp_curs_oclass,
+	.user = {
+		{{0,0,GK104_DISP_CURSOR             }, gf119_disp_curs_new },
+		{{0,0,GK104_DISP_OVERLAY            }, gf119_disp_oimm_new },
+		{{0,0,GK110_DISP_BASE_CHANNEL_DMA   }, gf119_disp_base_new },
+		{{0,0,GM107_DISP_CORE_CHANNEL_DMA   }, gk104_disp_core_new },
+		{{0,0,GK104_DISP_OVERLAY_CONTROL_DMA}, gk104_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgm200.c
index 38f5ee1dfc58..e5e590e19f62 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgm200.c
@@ -22,22 +22,19 @@
  * Authors: Ben Skeggs
  */
 #include "rootnv50.h"
-#include "dmacnv50.h"
+#include "channv50.h"
 
 #include <nvif/class.h>
 
 static const struct nv50_disp_root_func
 gm200_disp_root = {
-	.init = gf119_disp_root_init,
-	.fini = gf119_disp_root_fini,
-	.dmac = {
-		&gm200_disp_core_oclass,
-		&gk110_disp_base_oclass,
-		&gk104_disp_ovly_oclass,
-	},
-	.pioc = {
-		&gk104_disp_oimm_oclass,
-		&gk104_disp_curs_oclass,
+	.user = {
+		{{0,0,GK104_DISP_CURSOR             }, gf119_disp_curs_new },
+		{{0,0,GK104_DISP_OVERLAY            }, gf119_disp_oimm_new },
+		{{0,0,GK110_DISP_BASE_CHANNEL_DMA   }, gf119_disp_base_new },
+		{{0,0,GM200_DISP_CORE_CHANNEL_DMA   }, gk104_disp_core_new },
+		{{0,0,GK104_DISP_OVERLAY_CONTROL_DMA}, gk104_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp100.c
index ac8fdd728ec6..762a1a922e05 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp100.c
@@ -22,22 +22,19 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 #include "rootnv50.h"
-#include "dmacnv50.h"
+#include "channv50.h"
 
 #include <nvif/class.h>
 
 static const struct nv50_disp_root_func
 gp100_disp_root = {
-	.init = gf119_disp_root_init,
-	.fini = gf119_disp_root_fini,
-	.dmac = {
-		&gp100_disp_core_oclass,
-		&gk110_disp_base_oclass,
-		&gk104_disp_ovly_oclass,
-	},
-	.pioc = {
-		&gk104_disp_oimm_oclass,
-		&gk104_disp_curs_oclass,
+	.user = {
+		{{0,0,GK104_DISP_CURSOR             }, gf119_disp_curs_new },
+		{{0,0,GK104_DISP_OVERLAY            }, gf119_disp_oimm_new },
+		{{0,0,GK110_DISP_BASE_CHANNEL_DMA   }, gf119_disp_base_new },
+		{{0,0,GP100_DISP_CORE_CHANNEL_DMA   }, gk104_disp_core_new },
+		{{0,0,GK104_DISP_OVERLAY_CONTROL_DMA}, gk104_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp102.c
index 37122ca579ad..c7f00946c9af 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp102.c
@@ -22,22 +22,19 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 #include "rootnv50.h"
-#include "dmacnv50.h"
+#include "channv50.h"
 
 #include <nvif/class.h>
 
 static const struct nv50_disp_root_func
 gp102_disp_root = {
-	.init = gf119_disp_root_init,
-	.fini = gf119_disp_root_fini,
-	.dmac = {
-		&gp102_disp_core_oclass,
-		&gp102_disp_base_oclass,
-		&gp102_disp_ovly_oclass,
-	},
-	.pioc = {
-		&gp102_disp_oimm_oclass,
-		&gp102_disp_curs_oclass,
+	.user = {
+		{{0,0,GK104_DISP_CURSOR             }, gp102_disp_curs_new },
+		{{0,0,GK104_DISP_OVERLAY            }, gp102_disp_oimm_new },
+		{{0,0,GK110_DISP_BASE_CHANNEL_DMA   }, gp102_disp_base_new },
+		{{0,0,GP102_DISP_CORE_CHANNEL_DMA   }, gp102_disp_core_new },
+		{{0,0,GK104_DISP_OVERLAY_CONTROL_DMA}, gp102_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgt200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgt200.c
index 124a0c24f92c..a6963654087c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgt200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgt200.c
@@ -22,22 +22,19 @@
  * Authors: Ben Skeggs
  */
 #include "rootnv50.h"
-#include "dmacnv50.h"
+#include "channv50.h"
 
 #include <nvif/class.h>
 
 static const struct nv50_disp_root_func
 gt200_disp_root = {
-	.init = nv50_disp_root_init,
-	.fini = nv50_disp_root_fini,
-	.dmac = {
-		&gt200_disp_core_oclass,
-		&gt200_disp_base_oclass,
-		&gt200_disp_ovly_oclass,
-	},
-	.pioc = {
-		&g84_disp_oimm_oclass,
-		&g84_disp_curs_oclass,
+	.user = {
+		{{0,0,  G82_DISP_CURSOR             },  nv50_disp_curs_new },
+		{{0,0,  G82_DISP_OVERLAY            },  nv50_disp_oimm_new },
+		{{0,0,GT200_DISP_BASE_CHANNEL_DMA   },   g84_disp_base_new },
+		{{0,0,GT200_DISP_CORE_CHANNEL_DMA   },   g84_disp_core_new },
+		{{0,0,GT200_DISP_OVERLAY_CHANNEL_DMA}, gt200_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgt215.c
index dff52f30668b..4fe0a3ae8891 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgt215.c
@@ -22,22 +22,19 @@
  * Authors: Ben Skeggs
  */
 #include "rootnv50.h"
-#include "dmacnv50.h"
+#include "channv50.h"
 
 #include <nvif/class.h>
 
 static const struct nv50_disp_root_func
 gt215_disp_root = {
-	.init = nv50_disp_root_init,
-	.fini = nv50_disp_root_fini,
-	.dmac = {
-		&gt215_disp_core_oclass,
-		&gt215_disp_base_oclass,
-		&gt215_disp_ovly_oclass,
-	},
-	.pioc = {
-		&gt215_disp_oimm_oclass,
-		&gt215_disp_curs_oclass,
+	.user = {
+		{{0,0,GT214_DISP_CURSOR             },  nv50_disp_curs_new },
+		{{0,0,GT214_DISP_OVERLAY            },  nv50_disp_oimm_new },
+		{{0,0,GT214_DISP_BASE_CHANNEL_DMA   },   g84_disp_base_new },
+		{{0,0,GT214_DISP_CORE_CHANNEL_DMA   },   g94_disp_core_new },
+		{{0,0,GT214_DISP_OVERLAY_CHANNEL_DMA},   g84_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgv100.c
new file mode 100644
index 000000000000..9c658d632d37
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgv100.c
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "rootnv50.h"
+#include "channv50.h"
+
+#include <nvif/class.h>
+
+static const struct nv50_disp_root_func
+gv100_disp_root = {
+	.user = {
+		{{0,0,GV100_DISP_CURSOR                }, gv100_disp_curs_new },
+		{{0,0,GV100_DISP_WINDOW_IMM_CHANNEL_DMA}, gv100_disp_wimm_new },
+		{{0,0,GV100_DISP_CORE_CHANNEL_DMA      }, gv100_disp_core_new },
+		{{0,0,GV100_DISP_WINDOW_CHANNEL_DMA    }, gv100_disp_wndw_new },
+		{}
+	},
+};
+
+static int
+gv100_disp_root_new(struct nvkm_disp *disp, const struct nvkm_oclass *oclass,
+		    void *data, u32 size, struct nvkm_object **pobject)
+{
+	return nv50_disp_root_new_(&gv100_disp_root, disp, oclass,
+				   data, size, pobject);
+}
+
+const struct nvkm_disp_oclass
+gv100_disp_root_oclass = {
+	.base.oclass = GV100_DISP,
+	.base.minver = -1,
+	.base.maxver = -1,
+	.ctor = gv100_disp_root_new,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
index 1208524aae14..3aa5a2879239 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
@@ -22,14 +22,12 @@
  * Authors: Ben Skeggs
  */
 #include "rootnv50.h"
-#include "dmacnv50.h"
+#include "channv50.h"
 #include "dp.h"
 #include "head.h"
 #include "ior.h"
 
 #include <core/client.h>
-#include <core/ramht.h>
-#include <subdev/timer.h>
 
 #include <nvif/class.h>
 #include <nvif/cl5070.h>
@@ -271,23 +269,12 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 }
 
 static int
-nv50_disp_root_dmac_new_(const struct nvkm_oclass *oclass,
-			 void *data, u32 size, struct nvkm_object **pobject)
+nv50_disp_root_child_new_(const struct nvkm_oclass *oclass,
+			  void *argv, u32 argc, struct nvkm_object **pobject)
 {
-	const struct nv50_disp_dmac_oclass *sclass = oclass->priv;
-	struct nv50_disp_root *root = nv50_disp_root(oclass->parent);
-	return sclass->ctor(sclass->func, sclass->mthd, root, sclass->chid,
-			    oclass, data, size, pobject);
-}
-
-static int
-nv50_disp_root_pioc_new_(const struct nvkm_oclass *oclass,
-			 void *data, u32 size, struct nvkm_object **pobject)
-{
-	const struct nv50_disp_pioc_oclass *sclass = oclass->priv;
-	struct nv50_disp_root *root = nv50_disp_root(oclass->parent);
-	return sclass->ctor(sclass->func, sclass->mthd, root, sclass->chid.ctrl,
-			    sclass->chid.user, oclass, data, size, pobject);
+	struct nv50_disp *disp = nv50_disp_root(oclass->parent)->disp;
+	const struct nv50_disp_user *user = oclass->priv;
+	return user->ctor(oclass, argv, argc, disp, pobject);
 }
 
 static int
@@ -296,68 +283,26 @@ nv50_disp_root_child_get_(struct nvkm_object *object, int index,
 {
 	struct nv50_disp_root *root = nv50_disp_root(object);
 
-	if (index < ARRAY_SIZE(root->func->dmac)) {
-		sclass->base = root->func->dmac[index]->base;
-		sclass->priv = root->func->dmac[index];
-		sclass->ctor = nv50_disp_root_dmac_new_;
-		return 0;
-	}
-
-	index -= ARRAY_SIZE(root->func->dmac);
-
-	if (index < ARRAY_SIZE(root->func->pioc)) {
-		sclass->base = root->func->pioc[index]->base;
-		sclass->priv = root->func->pioc[index];
-		sclass->ctor = nv50_disp_root_pioc_new_;
+	if (root->func->user[index].ctor) {
+		sclass->base = root->func->user[index].base;
+		sclass->priv = root->func->user + index;
+		sclass->ctor = nv50_disp_root_child_new_;
 		return 0;
 	}
 
 	return -EINVAL;
 }
 
-static int
-nv50_disp_root_fini_(struct nvkm_object *object, bool suspend)
-{
-	struct nv50_disp_root *root = nv50_disp_root(object);
-	root->func->fini(root);
-	return 0;
-}
-
-static int
-nv50_disp_root_init_(struct nvkm_object *object)
-{
-	struct nv50_disp_root *root = nv50_disp_root(object);
-	struct nvkm_ior *ior;
-	int ret;
-
-	ret = root->func->init(root);
-	if (ret)
-		return ret;
-
-	/* Set 'normal' (ie. when it's attached to a head) state for
-	 * each output resource to 'fully enabled'.
-	 */
-	list_for_each_entry(ior, &root->disp->base.ior, head) {
-		ior->func->power(ior, true, true, true, true, true);
-	}
-
-	return 0;
-}
-
 static void *
 nv50_disp_root_dtor_(struct nvkm_object *object)
 {
 	struct nv50_disp_root *root = nv50_disp_root(object);
-	nvkm_ramht_del(&root->ramht);
-	nvkm_gpuobj_del(&root->instmem);
 	return root;
 }
 
 static const struct nvkm_object_func
 nv50_disp_root_ = {
 	.dtor = nv50_disp_root_dtor_,
-	.init = nv50_disp_root_init_,
-	.fini = nv50_disp_root_fini_,
 	.mthd = nv50_disp_root_mthd_,
 	.ntfy = nvkm_disp_ntfy,
 	.sclass = nv50_disp_root_child_get_,
@@ -370,8 +315,6 @@ nv50_disp_root_new_(const struct nv50_disp_root_func *func,
 {
 	struct nv50_disp *disp = nv50_disp(base);
 	struct nv50_disp_root *root;
-	struct nvkm_device *device = disp->base.engine.subdev.device;
-	int ret;
 
 	if (!(root = kzalloc(sizeof(*root), GFP_KERNEL)))
 		return -ENOMEM;
@@ -380,102 +323,18 @@ nv50_disp_root_new_(const struct nv50_disp_root_func *func,
 	nvkm_object_ctor(&nv50_disp_root_, oclass, &root->object);
 	root->func = func;
 	root->disp = disp;
-
-	ret = nvkm_gpuobj_new(disp->base.engine.subdev.device, 0x10000, 0x10000,
-			      false, NULL, &root->instmem);
-	if (ret)
-		return ret;
-
-	return nvkm_ramht_new(device, 0x1000, 0, root->instmem, &root->ramht);
-}
-
-void
-nv50_disp_root_fini(struct nv50_disp_root *root)
-{
-	struct nvkm_device *device = root->disp->base.engine.subdev.device;
-	/* disable all interrupts */
-	nvkm_wr32(device, 0x610024, 0x00000000);
-	nvkm_wr32(device, 0x610020, 0x00000000);
-}
-
-int
-nv50_disp_root_init(struct nv50_disp_root *root)
-{
-	struct nv50_disp *disp = root->disp;
-	struct nvkm_head *head;
-	struct nvkm_device *device = disp->base.engine.subdev.device;
-	u32 tmp;
-	int i;
-
-	/* The below segments of code copying values from one register to
-	 * another appear to inform EVO of the display capabilities or
-	 * something similar.  NFI what the 0x614004 caps are for..
-	 */
-	tmp = nvkm_rd32(device, 0x614004);
-	nvkm_wr32(device, 0x610184, tmp);
-
-	/* ... CRTC caps */
-	list_for_each_entry(head, &disp->base.head, head) {
-		tmp = nvkm_rd32(device, 0x616100 + (head->id * 0x800));
-		nvkm_wr32(device, 0x610190 + (head->id * 0x10), tmp);
-		tmp = nvkm_rd32(device, 0x616104 + (head->id * 0x800));
-		nvkm_wr32(device, 0x610194 + (head->id * 0x10), tmp);
-		tmp = nvkm_rd32(device, 0x616108 + (head->id * 0x800));
-		nvkm_wr32(device, 0x610198 + (head->id * 0x10), tmp);
-		tmp = nvkm_rd32(device, 0x61610c + (head->id * 0x800));
-		nvkm_wr32(device, 0x61019c + (head->id * 0x10), tmp);
-	}
-
-	/* ... DAC caps */
-	for (i = 0; i < disp->func->dac.nr; i++) {
-		tmp = nvkm_rd32(device, 0x61a000 + (i * 0x800));
-		nvkm_wr32(device, 0x6101d0 + (i * 0x04), tmp);
-	}
-
-	/* ... SOR caps */
-	for (i = 0; i < disp->func->sor.nr; i++) {
-		tmp = nvkm_rd32(device, 0x61c000 + (i * 0x800));
-		nvkm_wr32(device, 0x6101e0 + (i * 0x04), tmp);
-	}
-
-	/* ... PIOR caps */
-	for (i = 0; i < disp->func->pior.nr; i++) {
-		tmp = nvkm_rd32(device, 0x61e000 + (i * 0x800));
-		nvkm_wr32(device, 0x6101f0 + (i * 0x04), tmp);
-	}
-
-	/* steal display away from vbios, or something like that */
-	if (nvkm_rd32(device, 0x610024) & 0x00000100) {
-		nvkm_wr32(device, 0x610024, 0x00000100);
-		nvkm_mask(device, 0x6194e8, 0x00000001, 0x00000000);
-		if (nvkm_msec(device, 2000,
-			if (!(nvkm_rd32(device, 0x6194e8) & 0x00000002))
-				break;
-		) < 0)
-			return -EBUSY;
-	}
-
-	/* point at display engine memory area (hash table, objects) */
-	nvkm_wr32(device, 0x610010, (root->instmem->addr >> 8) | 9);
-
-	/* enable supervisor interrupts, disable everything else */
-	nvkm_wr32(device, 0x61002c, 0x00000370);
-	nvkm_wr32(device, 0x610028, 0x00000000);
 	return 0;
 }
 
 static const struct nv50_disp_root_func
 nv50_disp_root = {
-	.init = nv50_disp_root_init,
-	.fini = nv50_disp_root_fini,
-	.dmac = {
-		&nv50_disp_core_oclass,
-		&nv50_disp_base_oclass,
-		&nv50_disp_ovly_oclass,
-	},
-	.pioc = {
-		&nv50_disp_oimm_oclass,
-		&nv50_disp_curs_oclass,
+	.user = {
+		{{0,0,NV50_DISP_CURSOR             }, nv50_disp_curs_new },
+		{{0,0,NV50_DISP_OVERLAY            }, nv50_disp_oimm_new },
+		{{0,0,NV50_DISP_BASE_CHANNEL_DMA   }, nv50_disp_base_new },
+		{{0,0,NV50_DISP_CORE_CHANNEL_DMA   }, nv50_disp_core_new },
+		{{0,0,NV50_DISP_OVERLAY_CHANNEL_DMA}, nv50_disp_ovly_new },
+		{}
 	},
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h
index 4818fa69ae6c..6ca4f9184b51 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h
@@ -2,34 +2,27 @@
 #ifndef __NV50_DISP_ROOT_H__
 #define __NV50_DISP_ROOT_H__
 #define nv50_disp_root(p) container_of((p), struct nv50_disp_root, object)
+#include <core/object.h>
 #include "nv50.h"
-#include "channv50.h"
-#include "dmacnv50.h"
 
 struct nv50_disp_root {
 	const struct nv50_disp_root_func *func;
 	struct nv50_disp *disp;
 	struct nvkm_object object;
-
-	struct nvkm_gpuobj *instmem;
-	struct nvkm_ramht *ramht;
 };
 
 struct nv50_disp_root_func {
-	int (*init)(struct nv50_disp_root *);
-	void (*fini)(struct nv50_disp_root *);
-	const struct nv50_disp_dmac_oclass *dmac[3];
-	const struct nv50_disp_pioc_oclass *pioc[2];
+	int blah;
+	struct nv50_disp_user {
+		struct nvkm_sclass base;
+		int (*ctor)(const struct nvkm_oclass *, void *argv, u32 argc,
+			    struct nv50_disp *, struct nvkm_object **);
+	} user[];
 };
 
 int  nv50_disp_root_new_(const struct nv50_disp_root_func *, struct nvkm_disp *,
 			 const struct nvkm_oclass *, void *data, u32 size,
 			 struct nvkm_object **);
-int  nv50_disp_root_init(struct nv50_disp_root *);
-void nv50_disp_root_fini(struct nv50_disp_root *);
-
-int  gf119_disp_root_init(struct nv50_disp_root *);
-void gf119_disp_root_fini(struct nv50_disp_root *);
 
 extern const struct nvkm_disp_oclass nv50_disp_root_oclass;
 extern const struct nvkm_disp_oclass g84_disp_root_oclass;
@@ -43,4 +36,5 @@ extern const struct nvkm_disp_oclass gm107_disp_root_oclass;
 extern const struct nvkm_disp_oclass gm200_disp_root_oclass;
 extern const struct nvkm_disp_oclass gp100_disp_root_oclass;
 extern const struct nvkm_disp_oclass gp102_disp_root_oclass;
+extern const struct nvkm_disp_oclass gv100_disp_root_oclass;
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg84.c
index f40b909b4ca2..ec3a7db08118 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg84.c
@@ -34,5 +34,5 @@ g84_sor = {
 int
 g84_sor_new(struct nvkm_disp *disp, int id)
 {
-	return nv50_sor_new_(&g84_sor, disp, id);
+	return nvkm_ior_new_(&g84_sor, disp, SOR, id);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg94.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg94.c
index 49aeafde0031..4d59d02525d9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg94.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg94.c
@@ -279,5 +279,13 @@ g94_sor = {
 int
 g94_sor_new(struct nvkm_disp *disp, int id)
 {
-	return nv50_sor_new_(&g94_sor, disp, id);
+	return nvkm_ior_new_(&g94_sor, disp, SOR, id);
+}
+
+int
+g94_sor_cnt(struct nvkm_disp *disp, unsigned long *pmask)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	*pmask = (nvkm_rd32(device, 0x610184) & 0x0f000000) >> 24;
+	return 4;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c
index 700fc754f28a..e6e6dfbb1283 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c
@@ -152,15 +152,6 @@ gf119_sor_state(struct nvkm_ior *sor, struct nvkm_ior_state *state)
 	state->head = ctrl & 0x0000000f;
 }
 
-int
-gf119_sor_new_(const struct nvkm_ior_func *func, struct nvkm_disp *disp, int id)
-{
-	struct nvkm_device *device = disp->engine.subdev.device;
-	if (!(nvkm_rd32(device, 0x612004) & (0x00000100 << id)))
-		return 0;
-	return nvkm_ior_new_(func, disp, SOR, id);
-}
-
 static const struct nvkm_ior_func
 gf119_sor = {
 	.state = gf119_sor_state,
@@ -189,5 +180,13 @@ gf119_sor = {
 int
 gf119_sor_new(struct nvkm_disp *disp, int id)
 {
-	return gf119_sor_new_(&gf119_sor, disp, id);
+	return nvkm_ior_new_(&gf119_sor, disp, SOR, id);
+}
+
+int
+gf119_sor_cnt(struct nvkm_disp *disp, unsigned long *pmask)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	*pmask = (nvkm_rd32(device, 0x612004) & 0x0000ff00) >> 8;
+	return 8;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgk104.c
index a1547bdf490b..b94090edaebf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgk104.c
@@ -49,5 +49,5 @@ gk104_sor = {
 int
 gk104_sor_new(struct nvkm_disp *disp, int id)
 {
-	return gf119_sor_new_(&gk104_sor, disp, id);
+	return nvkm_ior_new_(&gk104_sor, disp, SOR, id);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c
index 60230957d82b..e6965dec09c9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c
@@ -63,5 +63,5 @@ gm107_sor = {
 int
 gm107_sor_new(struct nvkm_disp *disp, int id)
 {
-	return gf119_sor_new_(&gm107_sor, disp, id);
+	return nvkm_ior_new_(&gm107_sor, disp, SOR, id);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c
index f9b8107aa2a2..d892bdf04034 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c
@@ -23,7 +23,7 @@
  */
 #include "ior.h"
 
-static void
+void
 gm200_sor_dp_drive(struct nvkm_ior *sor, int ln, int pc, int dc, int pe, int pu)
 {
 	struct nvkm_device *device = sor->disp->engine.subdev.device;
@@ -45,7 +45,7 @@ gm200_sor_dp_drive(struct nvkm_ior *sor, int ln, int pc, int dc, int pe, int pu)
 	nvkm_wr32(device, 0x61c13c + loff, data[3] | (pc << shift));
 }
 
-static void
+void
 gm200_sor_route_set(struct nvkm_outp *outp, struct nvkm_ior *ior)
 {
 	struct nvkm_device *device = outp->disp->engine.subdev.device;
@@ -62,7 +62,7 @@ gm200_sor_route_set(struct nvkm_outp *outp, struct nvkm_ior *ior)
 		nvkm_mask(device, 0x612388 + moff, 0x0000001f, link << 4 | sor);
 }
 
-static int
+int
 gm200_sor_route_get(struct nvkm_outp *outp, int *link)
 {
 	struct nvkm_device *device = outp->disp->engine.subdev.device;
@@ -120,5 +120,5 @@ gm200_sor = {
 int
 gm200_sor_new(struct nvkm_disp *disp, int id)
 {
-	return gf119_sor_new_(&gm200_sor, disp, id);
+	return nvkm_ior_new_(&gm200_sor, disp, SOR, id);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgt215.c
index da228b54b43e..54d134d4ca1d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgt215.c
@@ -65,5 +65,5 @@ gt215_sor = {
 int
 gt215_sor_new(struct nvkm_disp *disp, int id)
 {
-	return nv50_sor_new_(&gt215_sor, disp, id);
+	return nvkm_ior_new_(&gt215_sor, disp, SOR, id);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c
new file mode 100644
index 000000000000..040db8a338de
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "ior.h"
+
+#include <subdev/timer.h>
+
+static void
+gv100_sor_dp_watermark(struct nvkm_ior *sor, int head, u8 watermark)
+{
+	struct nvkm_device *device = sor->disp->engine.subdev.device;
+	const u32 hoff = head * 0x800;
+	nvkm_mask(device, 0x616550 + hoff, 0x0c00003f, 0x08000000 | watermark);
+}
+
+static void
+gv100_sor_dp_audio_sym(struct nvkm_ior *sor, int head, u16 h, u32 v)
+{
+	struct nvkm_device *device = sor->disp->engine.subdev.device;
+	const u32 hoff = head * 0x800;
+	nvkm_mask(device, 0x616568 + hoff, 0x0000ffff, h);
+	nvkm_mask(device, 0x61656c + hoff, 0x00ffffff, v);
+}
+
+static void
+gv100_sor_dp_audio(struct nvkm_ior *sor, int head, bool enable)
+{
+	struct nvkm_device *device = sor->disp->engine.subdev.device;
+	const u32 hoff = 0x800 * head;
+	const u32 data = 0x80000000 | (0x00000001 * enable);
+	const u32 mask = 0x8000000d;
+	nvkm_mask(device, 0x616560 + hoff, mask, data);
+	nvkm_msec(device, 2000,
+		if (!(nvkm_rd32(device, 0x616560 + hoff) & 0x80000000))
+			break;
+	);
+}
+
+static void
+gv100_sor_state(struct nvkm_ior *sor, struct nvkm_ior_state *state)
+{
+	struct nvkm_device *device = sor->disp->engine.subdev.device;
+	const u32 coff = (state == &sor->arm) * 0x8000 + sor->id * 0x20;
+	u32 ctrl = nvkm_rd32(device, 0x680300 + coff);
+
+	state->proto_evo = (ctrl & 0x00000f00) >> 8;
+	switch (state->proto_evo) {
+	case 0: state->proto = LVDS; state->link = 1; break;
+	case 1: state->proto = TMDS; state->link = 1; break;
+	case 2: state->proto = TMDS; state->link = 2; break;
+	case 5: state->proto = TMDS; state->link = 3; break;
+	case 8: state->proto =   DP; state->link = 1; break;
+	case 9: state->proto =   DP; state->link = 2; break;
+	default:
+		state->proto = UNKNOWN;
+		break;
+	}
+
+	state->head = ctrl & 0x000000ff;
+}
+
+static const struct nvkm_ior_func
+gv100_sor = {
+	.route = {
+		.get = gm200_sor_route_get,
+		.set = gm200_sor_route_set,
+	},
+	.state = gv100_sor_state,
+	.power = nv50_sor_power,
+	.clock = gf119_sor_clock,
+	.hdmi = {
+		.ctrl = gv100_hdmi_ctrl,
+	},
+	.dp = {
+		.lanes = { 0, 1, 2, 3 },
+		.links = gf119_sor_dp_links,
+		.power = g94_sor_dp_power,
+		.pattern = gm107_sor_dp_pattern,
+		.drive = gm200_sor_dp_drive,
+		.audio = gv100_sor_dp_audio,
+		.audio_sym = gv100_sor_dp_audio_sym,
+		.watermark = gv100_sor_dp_watermark,
+	},
+	.hda = {
+		.hpd = gf119_hda_hpd,
+		.eld = gf119_hda_eld,
+	},
+};
+
+int
+gv100_sor_new(struct nvkm_disp *disp, int id)
+{
+	return nvkm_ior_new_(&gv100_sor, disp, SOR, id);
+}
+
+int
+gv100_sor_cnt(struct nvkm_disp *disp, unsigned long *pmask)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	*pmask = (nvkm_rd32(device, 0x610060) & 0x0000ff00) >> 8;
+	return (nvkm_rd32(device, 0x610074) & 0x00000f00) >> 8;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sormcp77.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sormcp77.c
index c0179ccb956d..8a70dd25b13a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sormcp77.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sormcp77.c
@@ -44,5 +44,5 @@ mcp77_sor = {
 int
 mcp77_sor_new(struct nvkm_disp *disp, int id)
 {
-	return nv50_sor_new_(&mcp77_sor, disp, id);
+	return nvkm_ior_new_(&mcp77_sor, disp, SOR, id);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sormcp89.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sormcp89.c
index 9bb01cd96697..eac9c5be9166 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sormcp89.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sormcp89.c
@@ -49,5 +49,5 @@ mcp89_sor = {
 int
 mcp89_sor_new(struct nvkm_disp *disp, int id)
 {
-	return nv50_sor_new_(&mcp89_sor, disp, id);
+	return nvkm_ior_new_(&mcp89_sor, disp, SOR, id);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sornv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sornv50.c
index f3ebd0c22e7d..b4729f8798af 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sornv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sornv50.c
@@ -84,15 +84,6 @@ nv50_sor_state(struct nvkm_ior *sor, struct nvkm_ior_state *state)
 	state->head = ctrl & 0x00000003;
 }
 
-int
-nv50_sor_new_(const struct nvkm_ior_func *func, struct nvkm_disp *disp, int id)
-{
-	struct nvkm_device *device = disp->engine.subdev.device;
-	if (!(nvkm_rd32(device, 0x610184) & (0x01000000 << id)))
-		return 0;
-	return nvkm_ior_new_(func, disp, SOR, id);
-}
-
 static const struct nvkm_ior_func
 nv50_sor = {
 	.state = nv50_sor_state,
@@ -103,5 +94,13 @@ nv50_sor = {
 int
 nv50_sor_new(struct nvkm_disp *disp, int id)
 {
-	return nv50_sor_new_(&nv50_sor, disp, id);
+	return nvkm_ior_new_(&nv50_sor, disp, SOR, id);
+}
+
+int
+nv50_sor_cnt(struct nvkm_disp *disp, unsigned long *pmask)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	*pmask = (nvkm_rd32(device, 0x610184) & 0x03000000) >> 24;
+	return 2;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/wimmgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/wimmgv100.c
new file mode 100644
index 000000000000..89d783368b4f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/wimmgv100.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "channv50.h"
+
+#include <core/client.h>
+
+#include <nvif/clc37b.h>
+#include <nvif/unpack.h>
+
+static void
+gv100_disp_wimm_intr(struct nv50_disp_chan *chan, bool en)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const u32 mask = 0x00000001 << chan->head;
+	const u32 data = en ? mask : 0;
+	nvkm_mask(device, 0x611da8, mask, data);
+}
+
+const struct nv50_disp_chan_func
+gv100_disp_wimm = {
+	.init = gv100_disp_dmac_init,
+	.fini = gv100_disp_dmac_fini,
+	.intr = gv100_disp_wimm_intr,
+	.user = gv100_disp_chan_user,
+};
+
+static int
+gv100_disp_wimm_new_(const struct nv50_disp_chan_func *func,
+		     const struct nv50_disp_chan_mthd *mthd,
+		     struct nv50_disp *disp, int chid,
+		     const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		     struct nvkm_object **pobject)
+{
+	union {
+		struct nvc37b_window_imm_channel_dma_v0 v0;
+	} *args = argv;
+	struct nvkm_object *parent = oclass->parent;
+	int wndw, ret = -ENOSYS;
+	u64 push;
+
+	nvif_ioctl(parent, "create window imm channel dma size %d\n", argc);
+	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+		nvif_ioctl(parent, "create window imm channel dma vers %d "
+				   "pushbuf %016llx index %d\n",
+			   args->v0.version, args->v0.pushbuf, args->v0.index);
+		if (!(disp->wndw.mask & BIT(args->v0.index)))
+			return -EINVAL;
+		push = args->v0.pushbuf;
+		wndw = args->v0.index;
+	} else
+		return ret;
+
+	return nv50_disp_dmac_new_(func, mthd, disp, chid + wndw,
+				   wndw, push, oclass, pobject);
+}
+
+int
+gv100_disp_wimm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return gv100_disp_wimm_new_(&gv100_disp_wimm, NULL, disp, 33,
+				    oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/wndwgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/wndwgv100.c
new file mode 100644
index 000000000000..98911805aabf
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/wndwgv100.c
@@ -0,0 +1,184 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "channv50.h"
+
+#include <core/client.h>
+
+#include <nvif/clc37e.h>
+#include <nvif/unpack.h>
+
+static const struct nv50_disp_mthd_list
+gv100_disp_wndw_mthd_base = {
+	.mthd = 0x0000,
+	.addr = 0x000000,
+	.data = {
+		{ 0x0200, 0x690200 },
+		{ 0x020c, 0x69020c },
+		{ 0x0210, 0x690210 },
+		{ 0x0214, 0x690214 },
+		{ 0x0218, 0x690218 },
+		{ 0x021c, 0x69021c },
+		{ 0x0220, 0x690220 },
+		{ 0x0224, 0x690224 },
+		{ 0x0228, 0x690228 },
+		{ 0x022c, 0x69022c },
+		{ 0x0230, 0x690230 },
+		{ 0x0234, 0x690234 },
+		{ 0x0238, 0x690238 },
+		{ 0x0240, 0x690240 },
+		{ 0x0244, 0x690244 },
+		{ 0x0248, 0x690248 },
+		{ 0x024c, 0x69024c },
+		{ 0x0250, 0x690250 },
+		{ 0x0254, 0x690254 },
+		{ 0x0260, 0x690260 },
+		{ 0x0264, 0x690264 },
+		{ 0x0268, 0x690268 },
+		{ 0x026c, 0x69026c },
+		{ 0x0270, 0x690270 },
+		{ 0x0274, 0x690274 },
+		{ 0x0280, 0x690280 },
+		{ 0x0284, 0x690284 },
+		{ 0x0288, 0x690288 },
+		{ 0x028c, 0x69028c },
+		{ 0x0290, 0x690290 },
+		{ 0x0298, 0x690298 },
+		{ 0x029c, 0x69029c },
+		{ 0x02a0, 0x6902a0 },
+		{ 0x02a4, 0x6902a4 },
+		{ 0x02a8, 0x6902a8 },
+		{ 0x02ac, 0x6902ac },
+		{ 0x02b0, 0x6902b0 },
+		{ 0x02b4, 0x6902b4 },
+		{ 0x02b8, 0x6902b8 },
+		{ 0x02bc, 0x6902bc },
+		{ 0x02c0, 0x6902c0 },
+		{ 0x02c4, 0x6902c4 },
+		{ 0x02c8, 0x6902c8 },
+		{ 0x02cc, 0x6902cc },
+		{ 0x02d0, 0x6902d0 },
+		{ 0x02d4, 0x6902d4 },
+		{ 0x02d8, 0x6902d8 },
+		{ 0x02dc, 0x6902dc },
+		{ 0x02e0, 0x6902e0 },
+		{ 0x02e4, 0x6902e4 },
+		{ 0x02e8, 0x6902e8 },
+		{ 0x02ec, 0x6902ec },
+		{ 0x02f0, 0x6902f0 },
+		{ 0x02f4, 0x6902f4 },
+		{ 0x02f8, 0x6902f8 },
+		{ 0x02fc, 0x6902fc },
+		{ 0x0300, 0x690300 },
+		{ 0x0304, 0x690304 },
+		{ 0x0308, 0x690308 },
+		{ 0x0310, 0x690310 },
+		{ 0x0314, 0x690314 },
+		{ 0x0318, 0x690318 },
+		{ 0x031c, 0x69031c },
+		{ 0x0320, 0x690320 },
+		{ 0x0324, 0x690324 },
+		{ 0x0328, 0x690328 },
+		{ 0x032c, 0x69032c },
+		{ 0x033c, 0x69033c },
+		{ 0x0340, 0x690340 },
+		{ 0x0344, 0x690344 },
+		{ 0x0348, 0x690348 },
+		{ 0x034c, 0x69034c },
+		{ 0x0350, 0x690350 },
+		{ 0x0354, 0x690354 },
+		{ 0x0358, 0x690358 },
+		{ 0x0364, 0x690364 },
+		{ 0x0368, 0x690368 },
+		{ 0x036c, 0x69036c },
+		{ 0x0370, 0x690370 },
+		{ 0x0374, 0x690374 },
+		{ 0x0380, 0x690380 },
+		{}
+	}
+};
+
+const struct nv50_disp_chan_mthd
+gv100_disp_wndw_mthd = {
+	.name = "Base",
+	.addr = 0x001000,
+	.prev = 0x000800,
+	.data = {
+		{ "Global", 1, &gv100_disp_wndw_mthd_base },
+		{}
+	}
+};
+
+static void
+gv100_disp_wndw_intr(struct nv50_disp_chan *chan, bool en)
+{
+	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+	const u32 mask = 0x00000001 << chan->head;
+	const u32 data = en ? mask : 0;
+	nvkm_mask(device, 0x611da4, mask, data);
+}
+
+const struct nv50_disp_chan_func
+gv100_disp_wndw = {
+	.init = gv100_disp_dmac_init,
+	.fini = gv100_disp_dmac_fini,
+	.intr = gv100_disp_wndw_intr,
+	.user = gv100_disp_chan_user,
+	.bind = gv100_disp_dmac_bind,
+};
+
+static int
+gv100_disp_wndw_new_(const struct nv50_disp_chan_func *func,
+		     const struct nv50_disp_chan_mthd *mthd,
+		     struct nv50_disp *disp, int chid,
+		     const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		     struct nvkm_object **pobject)
+{
+	union {
+		struct nvc37e_window_channel_dma_v0 v0;
+	} *args = argv;
+	struct nvkm_object *parent = oclass->parent;
+	int wndw, ret = -ENOSYS;
+	u64 push;
+
+	nvif_ioctl(parent, "create window channel dma size %d\n", argc);
+	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+		nvif_ioctl(parent, "create window channel dma vers %d "
+				   "pushbuf %016llx index %d\n",
+			   args->v0.version, args->v0.pushbuf, args->v0.index);
+		if (!(disp->wndw.mask & BIT(args->v0.index)))
+			return -EINVAL;
+		push = args->v0.pushbuf;
+		wndw = args->v0.index;
+	} else
+		return ret;
+
+	return nv50_disp_dmac_new_(func, mthd, disp, chid + wndw,
+				   wndw, push, oclass, pobject);
+}
+
+int
+gv100_disp_wndw_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nv50_disp *disp, struct nvkm_object **pobject)
+{
+	return gv100_disp_wndw_new_(&gv100_disp_wndw, &gv100_disp_wndw_mthd,
+				    disp, 1, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/dma/Kbuild
index c4a2ce9b0d71..e96d1f57f9f9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/dma/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/Kbuild
@@ -3,9 +3,11 @@ nvkm-y += nvkm/engine/dma/nv04.o
 nvkm-y += nvkm/engine/dma/nv50.o
 nvkm-y += nvkm/engine/dma/gf100.o
 nvkm-y += nvkm/engine/dma/gf119.o
+nvkm-y += nvkm/engine/dma/gv100.o
 
 nvkm-y += nvkm/engine/dma/user.o
 nvkm-y += nvkm/engine/dma/usernv04.o
 nvkm-y += nvkm/engine/dma/usernv50.o
 nvkm-y += nvkm/engine/dma/usergf100.o
 nvkm-y += nvkm/engine/dma/usergf119.o
+nvkm-y += nvkm/engine/dma/usergv100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/dma/gv100.c
new file mode 100644
index 000000000000..c65a4c2ea93d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/gv100.c
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "priv.h"
+#include "user.h"
+
+static const struct nvkm_dma_func
+gv100_dma = {
+	.class_new = gv100_dmaobj_new,
+};
+
+int
+gv100_dma_new(struct nvkm_device *device, int index, struct nvkm_dma **pdma)
+{
+	return nvkm_dma_new_(&gv100_dma, device, index, pdma);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.h b/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.h
index 4bbac8a21c71..9fe01fd75474 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/user.h
@@ -16,4 +16,6 @@ int gf100_dmaobj_new(struct nvkm_dma *, const struct nvkm_oclass *, void *, u32,
 		     struct nvkm_dmaobj **);
 int gf119_dmaobj_new(struct nvkm_dma *, const struct nvkm_oclass *, void *, u32,
 		     struct nvkm_dmaobj **);
+int gv100_dmaobj_new(struct nvkm_dma *, const struct nvkm_oclass *, void *, u32,
+		     struct nvkm_dmaobj **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usergv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usergv100.c
new file mode 100644
index 000000000000..39eba9fc82be
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usergv100.c
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#define gv100_dmaobj(p) container_of((p), struct gv100_dmaobj, base)
+#include "user.h"
+
+#include <core/client.h>
+#include <core/gpuobj.h>
+#include <subdev/fb.h>
+
+#include <nvif/cl0002.h>
+#include <nvif/unpack.h>
+
+struct gv100_dmaobj {
+	struct nvkm_dmaobj base;
+	u32 flags0;
+};
+
+static int
+gv100_dmaobj_bind(struct nvkm_dmaobj *base, struct nvkm_gpuobj *parent,
+		  int align, struct nvkm_gpuobj **pgpuobj)
+{
+	struct gv100_dmaobj *dmaobj = gv100_dmaobj(base);
+	struct nvkm_device *device = dmaobj->base.dma->engine.subdev.device;
+	u64 start = dmaobj->base.start >> 8;
+	u64 limit = dmaobj->base.limit >> 8;
+	int ret;
+
+	ret = nvkm_gpuobj_new(device, 24, align, false, parent, pgpuobj);
+	if (ret == 0) {
+		nvkm_kmap(*pgpuobj);
+		nvkm_wo32(*pgpuobj, 0x00, dmaobj->flags0);
+		nvkm_wo32(*pgpuobj, 0x04, lower_32_bits(start));
+		nvkm_wo32(*pgpuobj, 0x08, upper_32_bits(start));
+		nvkm_wo32(*pgpuobj, 0x0c, lower_32_bits(limit));
+		nvkm_wo32(*pgpuobj, 0x10, upper_32_bits(limit));
+		nvkm_done(*pgpuobj);
+	}
+
+	return ret;
+}
+
+static const struct nvkm_dmaobj_func
+gv100_dmaobj_func = {
+	.bind = gv100_dmaobj_bind,
+};
+
+int
+gv100_dmaobj_new(struct nvkm_dma *dma, const struct nvkm_oclass *oclass,
+		 void *data, u32 size, struct nvkm_dmaobj **pdmaobj)
+{
+	union {
+		struct gf119_dma_v0 v0;
+	} *args;
+	struct nvkm_object *parent = oclass->parent;
+	struct gv100_dmaobj *dmaobj;
+	u32 kind, page;
+	int ret;
+
+	if (!(dmaobj = kzalloc(sizeof(*dmaobj), GFP_KERNEL)))
+		return -ENOMEM;
+	*pdmaobj = &dmaobj->base;
+
+	ret = nvkm_dmaobj_ctor(&gv100_dmaobj_func, dma, oclass,
+			       &data, &size, &dmaobj->base);
+	if (ret)
+		return ret;
+
+	ret  = -ENOSYS;
+	args = data;
+
+	nvif_ioctl(parent, "create gv100 dma size %d\n", size);
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
+		nvif_ioctl(parent,
+			   "create gv100 dma vers %d page %d kind %02x\n",
+			   args->v0.version, args->v0.page, args->v0.kind);
+		kind = args->v0.kind != 0;
+		page = args->v0.page != 0;
+	} else
+	if (size == 0) {
+		kind = 0;
+		page = GF119_DMA_V0_PAGE_SP;
+	} else
+		return ret;
+
+	if (kind)
+		dmaobj->flags0 |= 0x00100000;
+	if (page)
+		dmaobj->flags0 |= 0x00000040;
+	dmaobj->flags0 |= 0x00000004; /* rw */
+
+	switch (dmaobj->base.target) {
+	case NV_MEM_TARGET_VRAM       : dmaobj->flags0 |= 0x00000001; break;
+	case NV_MEM_TARGET_PCI        : dmaobj->flags0 |= 0x00000002; break;
+	case NV_MEM_TARGET_PCI_NOSNOOP: dmaobj->flags0 |= 0x00000003; break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
index 64e51838edf8..f00408577a6a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
@@ -15,6 +15,7 @@ nvkm-y += nvkm/engine/fifo/gm200.o
 nvkm-y += nvkm/engine/fifo/gm20b.o
 nvkm-y += nvkm/engine/fifo/gp100.o
 nvkm-y += nvkm/engine/fifo/gp10b.o
+nvkm-y += nvkm/engine/fifo/gv100.o
 
 nvkm-y += nvkm/engine/fifo/chan.o
 nvkm-y += nvkm/engine/fifo/channv50.o
@@ -31,6 +32,6 @@ nvkm-y += nvkm/engine/fifo/gpfifonv50.o
 nvkm-y += nvkm/engine/fifo/gpfifog84.o
 nvkm-y += nvkm/engine/fifo/gpfifogf100.o
 nvkm-y += nvkm/engine/fifo/gpfifogk104.o
-nvkm-y += nvkm/engine/fifo/gpfifogk110.o
-nvkm-y += nvkm/engine/fifo/gpfifogm200.o
-nvkm-y += nvkm/engine/fifo/gpfifogp100.o
+nvkm-y += nvkm/engine/fifo/gpfifogv100.o
+
+nvkm-y += nvkm/engine/fifo/usergv100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
index 64f6b7654a08..c773caf21f6b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
@@ -30,6 +30,7 @@
 #include <subdev/mc.h>
 
 #include <nvif/event.h>
+#include <nvif/cl0080.h>
 #include <nvif/unpack.h>
 
 void
@@ -55,6 +56,12 @@ nvkm_fifo_start(struct nvkm_fifo *fifo, unsigned long *flags)
 	return fifo->func->start(fifo, flags);
 }
 
+void
+nvkm_fifo_fault(struct nvkm_fifo *fifo, struct nvkm_fault_data *info)
+{
+	return fifo->func->fault(fifo, info);
+}
+
 void
 nvkm_fifo_chan_put(struct nvkm_fifo *fifo, unsigned long flags,
 		   struct nvkm_fifo_chan **pchan)
@@ -208,6 +215,20 @@ nvkm_fifo_uevent(struct nvkm_fifo *fifo)
 	nvkm_event_send(&fifo->uevent, 1, 0, &rep, sizeof(rep));
 }
 
+static int
+nvkm_fifo_class_new_(struct nvkm_device *device,
+		     const struct nvkm_oclass *oclass, void *data, u32 size,
+		     struct nvkm_object **pobject)
+{
+	struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine);
+	return fifo->func->class_new(fifo, oclass, data, size, pobject);
+}
+
+static const struct nvkm_device_oclass
+nvkm_fifo_class_ = {
+	.ctor = nvkm_fifo_class_new_,
+};
+
 static int
 nvkm_fifo_class_new(struct nvkm_device *device,
 		    const struct nvkm_oclass *oclass, void *data, u32 size,
@@ -232,13 +253,9 @@ nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index,
 	int c = 0;
 
 	if (fifo->func->class_get) {
-		int ret = fifo->func->class_get(fifo, index, &sclass);
-		if (ret == 0) {
-			oclass->base = sclass->base;
-			oclass->engn = sclass;
-			*class = &nvkm_fifo_class;
-			return 0;
-		}
+		int ret = fifo->func->class_get(fifo, index, oclass);
+		if (ret == 0)
+			*class = &nvkm_fifo_class_;
 		return ret;
 	}
 
@@ -270,6 +287,20 @@ nvkm_fifo_fini(struct nvkm_engine *engine, bool suspend)
 	return 0;
 }
 
+static int
+nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data)
+{
+	struct nvkm_fifo *fifo = nvkm_fifo(engine);
+	switch (mthd) {
+	case NV_DEVICE_FIFO_CHANNELS: *data = fifo->nr; return 0;
+	default:
+		if (fifo->func->info)
+			return fifo->func->info(fifo, mthd, data);
+		break;
+	}
+	return -ENOSYS;
+}
+
 static int
 nvkm_fifo_oneinit(struct nvkm_engine *engine)
 {
@@ -311,6 +342,7 @@ nvkm_fifo = {
 	.dtor = nvkm_fifo_dtor,
 	.preinit = nvkm_fifo_preinit,
 	.oneinit = nvkm_fifo_oneinit,
+	.info = nvkm_fifo_info,
 	.init = nvkm_fifo_init,
 	.fini = nvkm_fifo_fini,
 	.intr = nvkm_fifo_intr,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h
new file mode 100644
index 000000000000..d0ac60b06720
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h
@@ -0,0 +1,11 @@
+#ifndef __NVKM_FIFO_CGRP_H__
+#define __NVKM_FIFO_CGRP_H__
+#include "priv.h"
+
+struct nvkm_fifo_cgrp {
+	int id;
+	struct list_head head;
+	struct list_head chan;
+	int chan_nr;
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
index 1208e3d9dbe2..8e28ba6b2307 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
@@ -10,6 +10,7 @@ struct gk104_fifo_chan {
 	struct gk104_fifo *fifo;
 	int runl;
 
+	struct nvkm_fifo_cgrp *cgrp;
 	struct list_head head;
 	bool killed;
 
@@ -19,11 +20,20 @@ struct gk104_fifo_chan {
 	} engn[NVKM_SUBDEV_NR];
 };
 
-int gk104_fifo_gpfifo_new(struct nvkm_fifo *, const struct nvkm_oclass *,
+extern const struct nvkm_fifo_chan_func gk104_fifo_gpfifo_func;
+
+int gk104_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *,
 			  void *data, u32 size, struct nvkm_object **);
+void *gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *);
+void gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *);
+void gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *);
+int gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *, struct nvkm_engine *,
+				  struct nvkm_object *);
+void gk104_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *,
+				   struct nvkm_engine *);
+int gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *);
+int gk104_fifo_gpfifo_kick_locked(struct gk104_fifo_chan *);
 
-extern const struct nvkm_fifo_chan_oclass gk104_fifo_gpfifo_oclass;
-extern const struct nvkm_fifo_chan_oclass gk110_fifo_gpfifo_oclass;
-extern const struct nvkm_fifo_chan_oclass gm200_fifo_gpfifo_oclass;
-extern const struct nvkm_fifo_chan_oclass gp100_fifo_gpfifo_oclass;
+int gv100_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *,
+			  void *data, u32 size, struct nvkm_object **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
index 84bd703dd897..a99046414a18 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
@@ -22,16 +22,19 @@
  * Authors: Ben Skeggs
  */
 #include "gk104.h"
+#include "cgrp.h"
 #include "changk104.h"
 
 #include <core/client.h>
 #include <core/gpuobj.h>
 #include <subdev/bar.h>
+#include <subdev/fault.h>
 #include <subdev/timer.h>
 #include <subdev/top.h>
 #include <engine/sw.h>
 
 #include <nvif/class.h>
+#include <nvif/cl0080.h>
 
 struct gk104_fifo_engine_status {
 	bool busy;
@@ -92,16 +95,40 @@ gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn,
 		   status->chan == &status->next ? "*" : " ");
 }
 
+static int
+gk104_fifo_class_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
+		     void *argv, u32 argc, struct nvkm_object **pobject)
+{
+	struct gk104_fifo *fifo = gk104_fifo(base);
+	if (oclass->engn == &fifo->func->chan) {
+		const struct gk104_fifo_chan_user *user = oclass->engn;
+		return user->ctor(fifo, oclass, argv, argc, pobject);
+	} else
+	if (oclass->engn == &fifo->func->user) {
+		const struct gk104_fifo_user_user *user = oclass->engn;
+		return user->ctor(oclass, argv, argc, pobject);
+	}
+	WARN_ON(1);
+	return -EINVAL;
+}
+
 static int
 gk104_fifo_class_get(struct nvkm_fifo *base, int index,
-		     const struct nvkm_fifo_chan_oclass **psclass)
+		     struct nvkm_oclass *oclass)
 {
 	struct gk104_fifo *fifo = gk104_fifo(base);
 	int c = 0;
 
-	while ((*psclass = fifo->func->chan[c])) {
-		if (c++ == index)
-			return 0;
+	if (fifo->func->user.ctor && c++ == index) {
+		oclass->base =  fifo->func->user.user;
+		oclass->engn = &fifo->func->user;
+		return 0;
+	}
+
+	if (fifo->func->chan.ctor && c++ == index) {
+		oclass->base =  fifo->func->chan.user;
+		oclass->engn = &fifo->func->chan;
+		return 0;
 	}
 
 	return c;
@@ -124,10 +151,12 @@ gk104_fifo_uevent_init(struct nvkm_fifo *fifo)
 void
 gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl)
 {
+	const struct gk104_fifo_runlist_func *func = fifo->func->runlist;
 	struct gk104_fifo_chan *chan;
 	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 	struct nvkm_memory *mem;
+	struct nvkm_fifo_cgrp *cgrp;
 	int nr = 0;
 	int target;
 
@@ -137,9 +166,14 @@ gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl)
 
 	nvkm_kmap(mem);
 	list_for_each_entry(chan, &fifo->runlist[runl].chan, head) {
-		nvkm_wo32(mem, (nr * 8) + 0, chan->base.chid);
-		nvkm_wo32(mem, (nr * 8) + 4, 0x00000000);
-		nr++;
+		func->chan(chan, mem, nr++ * func->size);
+	}
+
+	list_for_each_entry(cgrp, &fifo->runlist[runl].cgrp, head) {
+		func->cgrp(cgrp, mem, nr++ * func->size);
+		list_for_each_entry(chan, &cgrp->chan, head) {
+			func->chan(chan, mem, nr++ * func->size);
+		}
 	}
 	nvkm_done(mem);
 
@@ -155,10 +189,10 @@ gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl)
 				    (target << 28));
 	nvkm_wr32(device, 0x002274, (runl << 20) | nr);
 
-	if (wait_event_timeout(fifo->runlist[runl].wait,
-			       !(nvkm_rd32(device, 0x002284 + (runl * 0x08))
-				       & 0x00100000),
-			       msecs_to_jiffies(2000)) == 0)
+	if (nvkm_msec(device, 2000,
+		if (!(nvkm_rd32(device, 0x002284 + (runl * 0x08)) & 0x00100000))
+			break;
+	) < 0)
 		nvkm_error(subdev, "runlist %d update timeout\n", runl);
 unlock:
 	mutex_unlock(&subdev->mutex);
@@ -167,19 +201,45 @@ gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl)
 void
 gk104_fifo_runlist_remove(struct gk104_fifo *fifo, struct gk104_fifo_chan *chan)
 {
+	struct nvkm_fifo_cgrp *cgrp = chan->cgrp;
 	mutex_lock(&fifo->base.engine.subdev.mutex);
-	list_del_init(&chan->head);
+	if (!list_empty(&chan->head)) {
+		list_del_init(&chan->head);
+		if (cgrp && !--cgrp->chan_nr)
+			list_del_init(&cgrp->head);
+	}
 	mutex_unlock(&fifo->base.engine.subdev.mutex);
 }
 
 void
 gk104_fifo_runlist_insert(struct gk104_fifo *fifo, struct gk104_fifo_chan *chan)
 {
+	struct nvkm_fifo_cgrp *cgrp = chan->cgrp;
 	mutex_lock(&fifo->base.engine.subdev.mutex);
-	list_add_tail(&chan->head, &fifo->runlist[chan->runl].chan);
+	if (cgrp) {
+		if (!cgrp->chan_nr++)
+			list_add_tail(&cgrp->head, &fifo->runlist[chan->runl].cgrp);
+		list_add_tail(&chan->head, &cgrp->chan);
+	} else {
+		list_add_tail(&chan->head, &fifo->runlist[chan->runl].chan);
+	}
 	mutex_unlock(&fifo->base.engine.subdev.mutex);
 }
 
+void
+gk104_fifo_runlist_chan(struct gk104_fifo_chan *chan,
+			struct nvkm_memory *memory, u32 offset)
+{
+	nvkm_wo32(memory, offset + 0, chan->base.chid);
+	nvkm_wo32(memory, offset + 4, 0x00000000);
+}
+
+const struct gk104_fifo_runlist_func
+gk104_fifo_runlist = {
+	.size = 8,
+	.chan = gk104_fifo_runlist_chan,
+};
+
 static void
 gk104_fifo_recover_work(struct work_struct *w)
 {
@@ -235,6 +295,32 @@ gk104_fifo_recover_runl(struct gk104_fifo *fifo, int runl)
 	schedule_work(&fifo->recover.work);
 }
 
+static struct gk104_fifo_chan *
+gk104_fifo_recover_chid(struct gk104_fifo *fifo, int runl, int chid)
+{
+	struct gk104_fifo_chan *chan;
+	struct nvkm_fifo_cgrp *cgrp;
+
+	list_for_each_entry(chan, &fifo->runlist[runl].chan, head) {
+		if (chan->base.chid == chid) {
+			list_del_init(&chan->head);
+			return chan;
+		}
+	}
+
+	list_for_each_entry(cgrp, &fifo->runlist[runl].cgrp, head) {
+		if (cgrp->id == chid) {
+			chan = list_first_entry(&cgrp->chan, typeof(*chan), head);
+			list_del_init(&chan->head);
+			if (!--cgrp->chan_nr)
+				list_del_init(&cgrp->head);
+			return chan;
+		}
+	}
+
+	return NULL;
+}
+
 static void
 gk104_fifo_recover_chan(struct nvkm_fifo *base, int chid)
 {
@@ -252,13 +338,10 @@ gk104_fifo_recover_chan(struct nvkm_fifo *base, int chid)
 		return;
 
 	/* Lookup SW state for channel, and mark it as dead. */
-	list_for_each_entry(chan, &fifo->runlist[runl].chan, head) {
-		if (chan->base.chid == chid) {
-			list_del_init(&chan->head);
-			chan->killed = true;
-			nvkm_fifo_kevent(&fifo->base, chid);
-			break;
-		}
+	chan = gk104_fifo_recover_chid(fifo, runl, chid);
+	if (chan) {
+		chan->killed = true;
+		nvkm_fifo_kevent(&fifo->base, chid);
 	}
 
 	/* Disable channel. */
@@ -347,6 +430,90 @@ gk104_fifo_recover_engn(struct gk104_fifo *fifo, int engn)
 	schedule_work(&fifo->recover.work);
 }
 
+static void
+gk104_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info)
+{
+	struct gk104_fifo *fifo = gk104_fifo(base);
+	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	const struct nvkm_enum *er, *ee, *ec, *ea;
+	struct nvkm_engine *engine = NULL;
+	struct nvkm_fifo_chan *chan;
+	unsigned long flags;
+	char ct[8] = "HUB/", en[16] = "";
+	int engn;
+
+	er = nvkm_enum_find(fifo->func->fault.reason, info->reason);
+	ee = nvkm_enum_find(fifo->func->fault.engine, info->engine);
+	if (info->hub) {
+		ec = nvkm_enum_find(fifo->func->fault.hubclient, info->client);
+	} else {
+		ec = nvkm_enum_find(fifo->func->fault.gpcclient, info->client);
+		snprintf(ct, sizeof(ct), "GPC%d/", info->gpc);
+	}
+	ea = nvkm_enum_find(fifo->func->fault.access, info->access);
+
+	if (ee && ee->data2) {
+		switch (ee->data2) {
+		case NVKM_SUBDEV_BAR:
+			nvkm_mask(device, 0x001704, 0x00000000, 0x00000000);
+			break;
+		case NVKM_SUBDEV_INSTMEM:
+			nvkm_mask(device, 0x001714, 0x00000000, 0x00000000);
+			break;
+		case NVKM_ENGINE_IFB:
+			nvkm_mask(device, 0x001718, 0x00000000, 0x00000000);
+			break;
+		default:
+			engine = nvkm_device_engine(device, ee->data2);
+			break;
+		}
+	}
+
+	if (ee == NULL) {
+		enum nvkm_devidx engidx = nvkm_top_fault(device, info->engine);
+		if (engidx < NVKM_SUBDEV_NR) {
+			const char *src = nvkm_subdev_name[engidx];
+			char *dst = en;
+			do {
+				*dst++ = toupper(*src++);
+			} while(*src);
+			engine = nvkm_device_engine(device, engidx);
+		}
+	} else {
+		snprintf(en, sizeof(en), "%s", ee->name);
+	}
+
+	spin_lock_irqsave(&fifo->base.lock, flags);
+	chan = nvkm_fifo_chan_inst_locked(&fifo->base, info->inst);
+
+	nvkm_error(subdev,
+		   "fault %02x [%s] at %016llx engine %02x [%s] client %02x "
+		   "[%s%s] reason %02x [%s] on channel %d [%010llx %s]\n",
+		   info->access, ea ? ea->name : "", info->addr,
+		   info->engine, ee ? ee->name : en,
+		   info->client, ct, ec ? ec->name : "",
+		   info->reason, er ? er->name : "", chan ? chan->chid : -1,
+		   info->inst, chan ? chan->object.client->name : "unknown");
+
+	/* Kill the channel that caused the fault. */
+	if (chan)
+		gk104_fifo_recover_chan(&fifo->base, chan->chid);
+
+	/* Channel recovery will probably have already done this for the
+	 * correct engine(s), but just in case we can't find the channel
+	 * information...
+	 */
+	for (engn = 0; engn < fifo->engine_nr && engine; engn++) {
+		if (fifo->engine[engn].engine == engine) {
+			gk104_fifo_recover_engn(fifo, engn);
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&fifo->base.lock, flags);
+}
+
 static const struct nvkm_enum
 gk104_fifo_bind_reason[] = {
 	{ 0x01, "BIND_NOT_UNBOUND" },
@@ -456,88 +623,21 @@ gk104_fifo_intr_fault(struct gk104_fifo *fifo, int unit)
 	u32 inst = nvkm_rd32(device, 0x002800 + (unit * 0x10));
 	u32 valo = nvkm_rd32(device, 0x002804 + (unit * 0x10));
 	u32 vahi = nvkm_rd32(device, 0x002808 + (unit * 0x10));
-	u32 stat = nvkm_rd32(device, 0x00280c + (unit * 0x10));
-	u32 gpc    = (stat & 0x1f000000) >> 24;
-	u32 client = (stat & 0x00001f00) >> 8;
-	u32 write  = (stat & 0x00000080);
-	u32 hub    = (stat & 0x00000040);
-	u32 reason = (stat & 0x0000000f);
-	const struct nvkm_enum *er, *eu, *ec;
-	struct nvkm_engine *engine = NULL;
-	struct nvkm_fifo_chan *chan;
-	unsigned long flags;
-	char gpcid[8] = "", en[16] = "";
-	int engn;
-
-	er = nvkm_enum_find(fifo->func->fault.reason, reason);
-	eu = nvkm_enum_find(fifo->func->fault.engine, unit);
-	if (hub) {
-		ec = nvkm_enum_find(fifo->func->fault.hubclient, client);
-	} else {
-		ec = nvkm_enum_find(fifo->func->fault.gpcclient, client);
-		snprintf(gpcid, sizeof(gpcid), "GPC%d/", gpc);
-	}
-
-	if (eu && eu->data2) {
-		switch (eu->data2) {
-		case NVKM_SUBDEV_BAR:
-			nvkm_mask(device, 0x001704, 0x00000000, 0x00000000);
-			break;
-		case NVKM_SUBDEV_INSTMEM:
-			nvkm_mask(device, 0x001714, 0x00000000, 0x00000000);
-			break;
-		case NVKM_ENGINE_IFB:
-			nvkm_mask(device, 0x001718, 0x00000000, 0x00000000);
-			break;
-		default:
-			engine = nvkm_device_engine(device, eu->data2);
-			break;
-		}
-	}
-
-	if (eu == NULL) {
-		enum nvkm_devidx engidx = nvkm_top_fault(device, unit);
-		if (engidx < NVKM_SUBDEV_NR) {
-			const char *src = nvkm_subdev_name[engidx];
-			char *dst = en;
-			do {
-				*dst++ = toupper(*src++);
-			} while(*src);
-			engine = nvkm_device_engine(device, engidx);
-		}
-	} else {
-		snprintf(en, sizeof(en), "%s", eu->name);
-	}
-
-	spin_lock_irqsave(&fifo->base.lock, flags);
-	chan = nvkm_fifo_chan_inst_locked(&fifo->base, (u64)inst << 12);
-
-	nvkm_error(subdev,
-		   "%s fault at %010llx engine %02x [%s] client %02x [%s%s] "
-		   "reason %02x [%s] on channel %d [%010llx %s]\n",
-		   write ? "write" : "read", (u64)vahi << 32 | valo,
-		   unit, en, client, gpcid, ec ? ec->name : "",
-		   reason, er ? er->name : "", chan ? chan->chid : -1,
-		   (u64)inst << 12,
-		   chan ? chan->object.client->name : "unknown");
-
-
-	/* Kill the channel that caused the fault. */
-	if (chan)
-		gk104_fifo_recover_chan(&fifo->base, chan->chid);
-
-	/* Channel recovery will probably have already done this for the
-	 * correct engine(s), but just in case we can't find the channel
-	 * information...
-	 */
-	for (engn = 0; engn < fifo->engine_nr && engine; engn++) {
-		if (fifo->engine[engn].engine == engine) {
-			gk104_fifo_recover_engn(fifo, engn);
-			break;
-		}
-	}
-
-	spin_unlock_irqrestore(&fifo->base.lock, flags);
+	u32 type = nvkm_rd32(device, 0x00280c + (unit * 0x10));
+	struct nvkm_fault_data info;
+
+	info.inst   =  (u64)inst << 12;
+	info.addr   = ((u64)vahi << 32) | valo;
+	info.time   = 0;
+	info.engine = unit;
+	info.valid  = 1;
+	info.gpc    = (type & 0x1f000000) >> 24;
+	info.client = (type & 0x00001f00) >> 8;
+	info.access = (type & 0x00000080) >> 7;
+	info.hub    = (type & 0x00000040) >> 6;
+	info.reason = (type & 0x000000ff);
+
+	nvkm_fifo_fault(&fifo->base, &info);
 }
 
 static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = {
@@ -765,6 +865,34 @@ gk104_fifo_fini(struct nvkm_fifo *base)
 	nvkm_mask(device, 0x002140, 0x10000000, 0x10000000);
 }
 
+static int
+gk104_fifo_info(struct nvkm_fifo *base, u64 mthd, u64 *data)
+{
+	struct gk104_fifo *fifo = gk104_fifo(base);
+	switch (mthd) {
+	case NV_DEVICE_FIFO_RUNLISTS:
+		*data = (1ULL << fifo->runlist_nr) - 1;
+		return 0;
+	case NV_DEVICE_FIFO_RUNLIST_ENGINES(0)...
+	     NV_DEVICE_FIFO_RUNLIST_ENGINES(63): {
+		int runl = mthd - NV_DEVICE_FIFO_RUNLIST_ENGINES(0), engn;
+		if (runl < fifo->runlist_nr) {
+			unsigned long engm = fifo->runlist[runl].engm;
+			struct nvkm_engine *engine;
+			*data = 0;
+			for_each_set_bit(engn, &engm, fifo->engine_nr) {
+				if ((engine = fifo->engine[engn].engine))
+					*data |= BIT_ULL(engine->subdev.index);
+			}
+			return 0;
+		}
+	}
+		return -EINVAL;
+	default:
+		return -EINVAL;
+	}
+}
+
 static int
 gk104_fifo_oneinit(struct nvkm_fifo *base)
 {
@@ -813,19 +941,18 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
 	kfree(map);
 
 	for (i = 0; i < fifo->runlist_nr; i++) {
-		ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST,
-				      0x8000, 0x1000, false,
-				      &fifo->runlist[i].mem[0]);
-		if (ret)
-			return ret;
-
-		ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST,
-				      0x8000, 0x1000, false,
-				      &fifo->runlist[i].mem[1]);
-		if (ret)
-			return ret;
+		for (j = 0; j < ARRAY_SIZE(fifo->runlist[i].mem); j++) {
+			ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST,
+					      fifo->base.nr * 2/* TSG+chan */ *
+					      fifo->func->runlist->size,
+					      0x1000, false,
+					      &fifo->runlist[i].mem[j]);
+			if (ret)
+				return ret;
+		}
 
 		init_waitqueue_head(&fifo->runlist[i].wait);
+		INIT_LIST_HEAD(&fifo->runlist[i].cgrp);
 		INIT_LIST_HEAD(&fifo->runlist[i].chan);
 	}
 
@@ -868,6 +995,9 @@ gk104_fifo_init(struct nvkm_fifo *base)
 
 	nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12);
 
+	if (fifo->func->init_pbdma_timeout)
+		fifo->func->init_pbdma_timeout(fifo);
+
 	nvkm_wr32(device, 0x002100, 0xffffffff);
 	nvkm_wr32(device, 0x002140, 0x7fffffff);
 }
@@ -894,13 +1024,16 @@ static const struct nvkm_fifo_func
 gk104_fifo_ = {
 	.dtor = gk104_fifo_dtor,
 	.oneinit = gk104_fifo_oneinit,
+	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
 	.intr = gk104_fifo_intr,
+	.fault = gk104_fifo_fault,
 	.uevent_init = gk104_fifo_uevent_init,
 	.uevent_fini = gk104_fifo_uevent_fini,
 	.recover_chan = gk104_fifo_recover_chan,
 	.class_get = gk104_fifo_class_get,
+	.class_new = gk104_fifo_class_new,
 };
 
 int
@@ -918,6 +1051,13 @@ gk104_fifo_new_(const struct gk104_fifo_func *func, struct nvkm_device *device,
 	return nvkm_fifo_ctor(&gk104_fifo_, device, index, nr, &fifo->base);
 }
 
+const struct nvkm_enum
+gk104_fifo_fault_access[] = {
+	{ 0x0, "READ" },
+	{ 0x1, "WRITE" },
+	{}
+};
+
 const struct nvkm_enum
 gk104_fifo_fault_engine[] = {
 	{ 0x00, "GR", NULL, NVKM_ENGINE_GR },
@@ -1035,14 +1175,13 @@ gk104_fifo_fault_gpcclient[] = {
 
 static const struct gk104_fifo_func
 gk104_fifo = {
+	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gk104_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
-	.chan = {
-		&gk104_fifo_gpfifo_oclass,
-		NULL
-	},
+	.runlist = &gk104_fifo_runlist,
+	.chan = {{0,0,KEPLER_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new },
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
index 1579785cf941..d295b81e18d6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
@@ -3,6 +3,7 @@
 #define __GK104_FIFO_H__
 #define gk104_fifo(p) container_of((p), struct gk104_fifo, base)
 #include "priv.h"
+struct nvkm_fifo_cgrp;
 
 #include <core/enum.h>
 #include <subdev/mmu.h>
@@ -31,6 +32,7 @@ struct gk104_fifo {
 		struct nvkm_memory *mem[2];
 		int next;
 		wait_queue_head_t wait;
+		struct list_head cgrp;
 		struct list_head chan;
 		u32 engm;
 	} runlist[16];
@@ -43,14 +45,36 @@ struct gk104_fifo {
 };
 
 struct gk104_fifo_func {
+	void (*init_pbdma_timeout)(struct gk104_fifo *);
+
 	struct {
+		const struct nvkm_enum *access;
 		const struct nvkm_enum *engine;
 		const struct nvkm_enum *reason;
 		const struct nvkm_enum *hubclient;
 		const struct nvkm_enum *gpcclient;
 	} fault;
 
-	const struct nvkm_fifo_chan_oclass *chan[];
+	const struct gk104_fifo_runlist_func {
+		u8 size;
+		void (*cgrp)(struct nvkm_fifo_cgrp *,
+			     struct nvkm_memory *, u32 offset);
+		void (*chan)(struct gk104_fifo_chan *,
+			     struct nvkm_memory *, u32 offset);
+	} *runlist;
+
+	struct gk104_fifo_user_user {
+		struct nvkm_sclass user;
+		int (*ctor)(const struct nvkm_oclass *, void *, u32,
+			    struct nvkm_object **);
+	} user;
+
+	struct gk104_fifo_chan_user {
+		struct nvkm_sclass user;
+		int (*ctor)(struct gk104_fifo *, const struct nvkm_oclass *,
+			    void *, u32, struct nvkm_object **);
+	} chan;
+	bool cgrp_force;
 };
 
 int gk104_fifo_new_(const struct gk104_fifo_func *, struct nvkm_device *,
@@ -59,30 +83,23 @@ void gk104_fifo_runlist_insert(struct gk104_fifo *, struct gk104_fifo_chan *);
 void gk104_fifo_runlist_remove(struct gk104_fifo *, struct gk104_fifo_chan *);
 void gk104_fifo_runlist_commit(struct gk104_fifo *, int runl);
 
-static inline u64
-gk104_fifo_engine_subdev(int engine)
-{
-	switch (engine) {
-	case 0: return (1ULL << NVKM_ENGINE_GR) |
-		       (1ULL << NVKM_ENGINE_SW) |
-		       (1ULL << NVKM_ENGINE_CE2);
-	case 1: return (1ULL << NVKM_ENGINE_MSPDEC);
-	case 2: return (1ULL << NVKM_ENGINE_MSPPP);
-	case 3: return (1ULL << NVKM_ENGINE_MSVLD);
-	case 4: return (1ULL << NVKM_ENGINE_CE0);
-	case 5: return (1ULL << NVKM_ENGINE_CE1);
-	case 6: return (1ULL << NVKM_ENGINE_MSENC);
-	default:
-		WARN_ON(1);
-		return 0;
-	}
-}
-
+extern const struct nvkm_enum gk104_fifo_fault_access[];
 extern const struct nvkm_enum gk104_fifo_fault_engine[];
 extern const struct nvkm_enum gk104_fifo_fault_reason[];
 extern const struct nvkm_enum gk104_fifo_fault_hubclient[];
 extern const struct nvkm_enum gk104_fifo_fault_gpcclient[];
+extern const struct gk104_fifo_runlist_func gk104_fifo_runlist;
+void gk104_fifo_runlist_chan(struct gk104_fifo_chan *,
+			     struct nvkm_memory *, u32);
+
+extern const struct gk104_fifo_runlist_func gk110_fifo_runlist;
+void gk110_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *,
+			     struct nvkm_memory *, u32);
+
+void gk208_fifo_init_pbdma_timeout(struct gk104_fifo *);
 
 extern const struct nvkm_enum gm107_fifo_fault_engine[];
+extern const struct gk104_fifo_runlist_func gm107_fifo_runlist;
+
 extern const struct nvkm_enum gp100_fifo_fault_engine[];
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
index b2f8ab7bf847..ac7655a130fb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
@@ -22,18 +22,38 @@
  * Authors: Ben Skeggs
  */
 #include "gk104.h"
+#include "cgrp.h"
 #include "changk104.h"
 
+#include <core/memory.h>
+
+#include <nvif/class.h>
+
+void
+gk110_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *cgrp,
+			struct nvkm_memory *memory, u32 offset)
+{
+	nvkm_wo32(memory, offset + 0, (cgrp->chan_nr << 26) | (128 << 18) |
+				      (3 << 14) | 0x00002000 | cgrp->id);
+	nvkm_wo32(memory, offset + 4, 0x00000000);
+}
+
+const struct gk104_fifo_runlist_func
+gk110_fifo_runlist = {
+	.size = 8,
+	.cgrp = gk110_fifo_runlist_cgrp,
+	.chan = gk104_fifo_runlist_chan,
+};
+
 static const struct gk104_fifo_func
 gk110_fifo = {
+	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gk104_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
-	.chan = {
-		&gk110_fifo_gpfifo_oclass,
-		NULL
-	},
+	.runlist = &gk110_fifo_runlist,
+	.chan = {{0,0,KEPLER_CHANNEL_GPFIFO_B}, gk104_fifo_gpfifo_new },
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
index 160617d376e4..5ea7e452cc66 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
@@ -24,16 +24,28 @@
 #include "gk104.h"
 #include "changk104.h"
 
+#include <nvif/class.h>
+
+void
+gk208_fifo_init_pbdma_timeout(struct gk104_fifo *fifo)
+{
+	struct nvkm_device *device = fifo->base.engine.subdev.device;
+	int i;
+
+	for (i = 0; i < fifo->pbdma_nr; i++)
+		nvkm_wr32(device, 0x04012c + (i * 0x2000), 0x0000ffff);
+}
+
 static const struct gk104_fifo_func
 gk208_fifo = {
+	.init_pbdma_timeout = gk208_fifo_init_pbdma_timeout,
+	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gk104_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
-	.chan = {
-		&gk104_fifo_gpfifo_oclass,
-		NULL
-	},
+	.runlist = &gk110_fifo_runlist,
+	.chan = {{0,0,KEPLER_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new },
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
index be9f5c16ed7d..535a0eb67a5f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
@@ -22,16 +22,18 @@
 #include "gk104.h"
 #include "changk104.h"
 
+#include <nvif/class.h>
+
 static const struct gk104_fifo_func
 gk20a_fifo = {
+	.init_pbdma_timeout = gk208_fifo_init_pbdma_timeout,
+	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gk104_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
-	.chan = {
-		&gk104_fifo_gpfifo_oclass,
-		NULL
-	},
+	.runlist = &gk110_fifo_runlist,
+	.chan = {{0,0,KEPLER_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new },
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
index 29c080683b32..79ae19b1db67 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
@@ -24,6 +24,25 @@
 #include "gk104.h"
 #include "changk104.h"
 
+#include <core/gpuobj.h>
+
+#include <nvif/class.h>
+
+static void
+gm107_fifo_runlist_chan(struct gk104_fifo_chan *chan,
+			struct nvkm_memory *memory, u32 offset)
+{
+	nvkm_wo32(memory, offset + 0, chan->base.chid);
+	nvkm_wo32(memory, offset + 4, chan->base.inst->addr >> 12);
+}
+
+const struct gk104_fifo_runlist_func
+gm107_fifo_runlist = {
+	.size = 8,
+	.cgrp = gk110_fifo_runlist_cgrp,
+	.chan = gm107_fifo_runlist_chan,
+};
+
 const struct nvkm_enum
 gm107_fifo_fault_engine[] = {
 	{ 0x01, "DISPLAY" },
@@ -49,14 +68,14 @@ gm107_fifo_fault_engine[] = {
 
 static const struct gk104_fifo_func
 gm107_fifo = {
+	.init_pbdma_timeout = gk208_fifo_init_pbdma_timeout,
+	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gm107_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
-	.chan = {
-		&gk110_fifo_gpfifo_oclass,
-		NULL
-	},
+	.runlist = &gm107_fifo_runlist,
+	.chan = {{0,0,KEPLER_CHANNEL_GPFIFO_B}, gk104_fifo_gpfifo_new },
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
index b069f785c5d8..49565faa854d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
@@ -24,16 +24,18 @@
 #include "gk104.h"
 #include "changk104.h"
 
+#include <nvif/class.h>
+
 static const struct gk104_fifo_func
 gm200_fifo = {
+	.init_pbdma_timeout = gk208_fifo_init_pbdma_timeout,
+	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gm107_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
-	.chan = {
-		&gm200_fifo_gpfifo_oclass,
-		NULL
-	},
+	.runlist = &gm107_fifo_runlist,
+	.chan = {{0,0,MAXWELL_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new },
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c
index 2ed87c2e8299..46736513bd11 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c
@@ -22,16 +22,18 @@
 #include "gk104.h"
 #include "changk104.h"
 
+#include <nvif/class.h>
+
 static const struct gk104_fifo_func
 gm20b_fifo = {
+	.init_pbdma_timeout = gk208_fifo_init_pbdma_timeout,
+	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gm107_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
-	.chan = {
-		&gm200_fifo_gpfifo_oclass,
-		NULL
-	},
+	.runlist = &gm107_fifo_runlist,
+	.chan = {{0,0,MAXWELL_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new },
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
index 41f16cf5a918..e2f8f9087d7c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
@@ -24,6 +24,8 @@
 #include "gk104.h"
 #include "changk104.h"
 
+#include <nvif/class.h>
+
 const struct nvkm_enum
 gp100_fifo_fault_engine[] = {
 	{ 0x01, "DISPLAY" },
@@ -50,14 +52,15 @@ gp100_fifo_fault_engine[] = {
 
 static const struct gk104_fifo_func
 gp100_fifo = {
+	.init_pbdma_timeout = gk208_fifo_init_pbdma_timeout,
+	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gp100_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
-	.chan = {
-		&gp100_fifo_gpfifo_oclass,
-		NULL
-	},
+	.runlist = &gm107_fifo_runlist,
+	.chan = {{0,0,PASCAL_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new },
+	.cgrp_force = true,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c
index 4af96c3e69ff..7733bf7c6545 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c
@@ -22,16 +22,19 @@
 #include "gk104.h"
 #include "changk104.h"
 
+#include <nvif/class.h>
+
 static const struct gk104_fifo_func
 gp10b_fifo = {
+	.init_pbdma_timeout = gk208_fifo_init_pbdma_timeout,
+	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gp100_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
-	.chan = {
-		&gp100_fifo_gpfifo_oclass,
-		NULL
-	},
+	.runlist = &gm107_fifo_runlist,
+	.chan = {{0,0,PASCAL_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new },
+	.cgrp_force = true,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
index 80c87521bebe..118b37aea318 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
@@ -22,6 +22,7 @@
  * Authors: Ben Skeggs
  */
 #include "changk104.h"
+#include "cgrp.h"
 
 #include <core/client.h>
 #include <core/gpuobj.h>
@@ -33,27 +34,40 @@
 #include <nvif/cla06f.h>
 #include <nvif/unpack.h>
 
-static int
-gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *chan)
+int
+gk104_fifo_gpfifo_kick_locked(struct gk104_fifo_chan *chan)
 {
 	struct gk104_fifo *fifo = chan->fifo;
 	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 	struct nvkm_client *client = chan->base.object.client;
+	struct nvkm_fifo_cgrp *cgrp = chan->cgrp;
 	int ret = 0;
 
-	mutex_lock(&subdev->mutex);
-	nvkm_wr32(device, 0x002634, chan->base.chid);
+	if (cgrp)
+		nvkm_wr32(device, 0x002634, cgrp->id | 0x01000000);
+	else
+		nvkm_wr32(device, 0x002634, chan->base.chid);
 	if (nvkm_msec(device, 2000,
 		if (!(nvkm_rd32(device, 0x002634) & 0x00100000))
 			break;
 	) < 0) {
-		nvkm_error(subdev, "channel %d [%s] kick timeout\n",
-			   chan->base.chid, client->name);
+		nvkm_error(subdev, "%s %d [%s] kick timeout\n",
+			   cgrp ? "tsg" : "channel",
+			   cgrp ? cgrp->id : chan->base.chid, client->name);
 		nvkm_fifo_recover_chan(&fifo->base, chan->base.chid);
 		ret = -ETIMEDOUT;
 	}
-	mutex_unlock(&subdev->mutex);
+	return ret;
+}
+
+int
+gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *chan)
+{
+	int ret;
+	mutex_lock(&chan->base.fifo->engine.subdev.mutex);
+	ret = gk104_fifo_gpfifo_kick_locked(chan);
+	mutex_unlock(&chan->base.fifo->engine.subdev.mutex);
 	return ret;
 }
 
@@ -62,9 +76,8 @@ gk104_fifo_gpfifo_engine_addr(struct nvkm_engine *engine)
 {
 	switch (engine->subdev.index) {
 	case NVKM_ENGINE_SW    :
-	case NVKM_ENGINE_CE0   :
-	case NVKM_ENGINE_CE1   :
-	case NVKM_ENGINE_CE2   : return 0x0000;
+	case NVKM_ENGINE_CE0...NVKM_ENGINE_CE_LAST:
+		return 0;
 	case NVKM_ENGINE_GR    : return 0x0210;
 	case NVKM_ENGINE_SEC   : return 0x0220;
 	case NVKM_ENGINE_MSPDEC: return 0x0250;
@@ -133,7 +146,7 @@ gk104_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base,
 	return 0;
 }
 
-static void
+void
 gk104_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *base,
 			      struct nvkm_engine *engine)
 {
@@ -142,7 +155,7 @@ gk104_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *base,
 	nvkm_gpuobj_del(&chan->engn[engine->subdev.index].inst);
 }
 
-static int
+int
 gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base,
 			      struct nvkm_engine *engine,
 			      struct nvkm_object *object)
@@ -167,7 +180,7 @@ gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base,
 			       chan->engn[engn].vma, NULL, 0);
 }
 
-static void
+void
 gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *base)
 {
 	struct gk104_fifo_chan *chan = gk104_fifo_chan(base);
@@ -185,7 +198,7 @@ gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *base)
 	nvkm_wr32(device, 0x800000 + coff, 0x00000000);
 }
 
-static void
+void
 gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base)
 {
 	struct gk104_fifo_chan *chan = gk104_fifo_chan(base);
@@ -205,13 +218,15 @@ gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base)
 	}
 }
 
-static void *
+void *
 gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base)
 {
-	return gk104_fifo_chan(base);
+	struct gk104_fifo_chan *chan = gk104_fifo_chan(base);
+	kfree(chan->cgrp);
+	return chan;
 }
 
-static const struct nvkm_fifo_chan_func
+const struct nvkm_fifo_chan_func
 gk104_fifo_gpfifo_func = {
 	.dtor = gk104_fifo_gpfifo_dtor,
 	.init = gk104_fifo_gpfifo_init,
@@ -223,62 +238,30 @@ gk104_fifo_gpfifo_func = {
 	.engine_fini = gk104_fifo_gpfifo_engine_fini,
 };
 
-struct gk104_fifo_chan_func {
-	u32 engine;
-	u64 subdev;
-};
-
 static int
-gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func,
-		       struct gk104_fifo *fifo, u32 *engmask, u16 *chid,
+gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid,
 		       u64 vmm, u64 ioffset, u64 ilength,
 		       const struct nvkm_oclass *oclass,
 		       struct nvkm_object **pobject)
 {
 	struct gk104_fifo_chan *chan;
-	int runlist = -1, ret = -ENOSYS, i, j;
-	u32 engines = 0, present = 0;
+	int runlist = ffs(*runlists) -1, ret, i;
+	unsigned long engm;
 	u64 subdevs = 0;
 	u64 usermem;
 
-	if (!vmm)
+	if (!vmm || runlist < 0 || runlist >= fifo->runlist_nr)
 		return -EINVAL;
+	*runlists = BIT_ULL(runlist);
 
-	/* Determine which downstream engines are present */
-	for (i = 0; i < fifo->engine_nr; i++) {
-		struct nvkm_engine *engine = fifo->engine[i].engine;
-		if (engine) {
-			u64 submask = BIT_ULL(engine->subdev.index);
-			for (j = 0; func[j].subdev; j++) {
-				if (func[j].subdev & submask) {
-					present |= func[j].engine;
-					break;
-				}
-			}
-
-			if (!func[j].subdev)
-				continue;
-
-			if (runlist < 0 && (*engmask & present))
-				runlist = fifo->engine[i].runl;
-			if (runlist == fifo->engine[i].runl) {
-				engines |= func[j].engine;
-				subdevs |= func[j].subdev;
-			}
-		}
-	}
-
-	/* Just an engine mask query?  All done here! */
-	if (!*engmask) {
-		*engmask = present;
-		return nvkm_object_new(oclass, NULL, 0, pobject);
+	engm = fifo->runlist[runlist].engm;
+	for_each_set_bit(i, &engm, fifo->engine_nr) {
+		if (fifo->engine[i].engine)
+			subdevs |= BIT_ULL(fifo->engine[i].engine->subdev.index);
 	}
 
-	/* No runlist?  No supported engines. */
-	*engmask = present;
-	if (runlist < 0)
-		return -ENODEV;
-	*engmask = engines;
+	if (subdevs & BIT_ULL(NVKM_ENGINE_GR))
+		subdevs |= BIT_ULL(NVKM_ENGINE_SW);
 
 	/* Allocate the channel. */
 	if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL)))
@@ -297,6 +280,18 @@ gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func,
 
 	*chid = chan->base.chid;
 
+	/* Hack to support GPUs where even individual channels should be
+	 * part of a channel group.
+	 */
+	if (fifo->func->cgrp_force) {
+		if (!(chan->cgrp = kmalloc(sizeof(*chan->cgrp), GFP_KERNEL)))
+			return -ENOMEM;
+		chan->cgrp->id = chan->base.chid;
+		INIT_LIST_HEAD(&chan->cgrp->head);
+		INIT_LIST_HEAD(&chan->cgrp->chan);
+		chan->cgrp->chan_nr = 0;
+	}
+
 	/* Clear channel control registers. */
 	usermem = chan->base.chid * 0x200;
 	ilength = order_base_2(ilength / 8);
@@ -328,45 +323,25 @@ gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func,
 	return 0;
 }
 
-static const struct gk104_fifo_chan_func
-gk104_fifo_gpfifo[] = {
-	{ NVA06F_V0_ENGINE_SW | NVA06F_V0_ENGINE_GR,
-		BIT_ULL(NVKM_ENGINE_SW) | BIT_ULL(NVKM_ENGINE_GR)
-	},
-	{ NVA06F_V0_ENGINE_SEC   , BIT_ULL(NVKM_ENGINE_SEC   ) },
-	{ NVA06F_V0_ENGINE_MSVLD , BIT_ULL(NVKM_ENGINE_MSVLD ) },
-	{ NVA06F_V0_ENGINE_MSPDEC, BIT_ULL(NVKM_ENGINE_MSPDEC) },
-	{ NVA06F_V0_ENGINE_MSPPP , BIT_ULL(NVKM_ENGINE_MSPPP ) },
-	{ NVA06F_V0_ENGINE_MSENC , BIT_ULL(NVKM_ENGINE_MSENC ) },
-	{ NVA06F_V0_ENGINE_VIC   , BIT_ULL(NVKM_ENGINE_VIC   ) },
-	{ NVA06F_V0_ENGINE_NVDEC , BIT_ULL(NVKM_ENGINE_NVDEC ) },
-	{ NVA06F_V0_ENGINE_NVENC0, BIT_ULL(NVKM_ENGINE_NVENC0) },
-	{ NVA06F_V0_ENGINE_NVENC1, BIT_ULL(NVKM_ENGINE_NVENC1) },
-	{ NVA06F_V0_ENGINE_CE0   , BIT_ULL(NVKM_ENGINE_CE0   ) },
-	{ NVA06F_V0_ENGINE_CE1   , BIT_ULL(NVKM_ENGINE_CE1   ) },
-	{ NVA06F_V0_ENGINE_CE2   , BIT_ULL(NVKM_ENGINE_CE2   ) },
-	{}
-};
-
 int
-gk104_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
+gk104_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass,
 		      void *data, u32 size, struct nvkm_object **pobject)
 {
 	struct nvkm_object *parent = oclass->parent;
 	union {
 		struct kepler_channel_gpfifo_a_v0 v0;
 	} *args = data;
-	struct gk104_fifo *fifo = gk104_fifo(base);
 	int ret = -ENOSYS;
 
 	nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
 	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 		nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx "
-				   "ioffset %016llx ilength %08x engine %08x\n",
+				   "ioffset %016llx ilength %08x "
+				   "runlist %016llx\n",
 			   args->v0.version, args->v0.vmm, args->v0.ioffset,
-			   args->v0.ilength, args->v0.engines);
-		return gk104_fifo_gpfifo_new_(gk104_fifo_gpfifo, fifo,
-					      &args->v0.engines,
+			   args->v0.ilength, args->v0.runlist);
+		return gk104_fifo_gpfifo_new_(fifo,
+					      &args->v0.runlist,
 					      &args->v0.chid,
 					       args->v0.vmm,
 					       args->v0.ioffset,
@@ -376,11 +351,3 @@ gk104_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 
 	return ret;
 }
-
-const struct nvkm_fifo_chan_oclass
-gk104_fifo_gpfifo_oclass = {
-	.base.oclass = KEPLER_CHANNEL_GPFIFO_A,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = gk104_fifo_gpfifo_new,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk110.c
deleted file mode 100644
index a9aa69c82e8e..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk110.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2016 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-#include "changk104.h"
-
-#include <nvif/class.h>
-
-const struct nvkm_fifo_chan_oclass
-gk110_fifo_gpfifo_oclass = {
-	.base.oclass = KEPLER_CHANNEL_GPFIFO_B,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = gk104_fifo_gpfifo_new,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogm200.c
deleted file mode 100644
index a13315147391..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogm200.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2015 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-#include "changk104.h"
-
-#include <nvif/class.h>
-
-const struct nvkm_fifo_chan_oclass
-gm200_fifo_gpfifo_oclass = {
-	.base.oclass = MAXWELL_CHANNEL_GPFIFO_A,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = gk104_fifo_gpfifo_new,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogp100.c
deleted file mode 100644
index 1530a9217aea..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogp100.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2016 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-#include "changk104.h"
-
-#include <nvif/class.h>
-
-const struct nvkm_fifo_chan_oclass
-gp100_fifo_gpfifo_oclass = {
-	.base.oclass = PASCAL_CHANNEL_GPFIFO_A,
-	.base.minver = 0,
-	.base.maxver = 0,
-	.ctor = gk104_fifo_gpfifo_new,
-};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c
new file mode 100644
index 000000000000..9598853ced56
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c
@@ -0,0 +1,225 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "changk104.h"
+#include "cgrp.h"
+
+#include <core/client.h>
+#include <core/gpuobj.h>
+
+#include <nvif/cla06f.h>
+#include <nvif/unpack.h>
+
+static int
+gv100_fifo_gpfifo_engine_valid(struct gk104_fifo_chan *chan, bool ce, bool valid)
+{
+	struct nvkm_subdev *subdev = &chan->base.fifo->engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	const u32 mask = ce ? 0x00020000 : 0x00010000;
+	const u32 data = valid ? mask : 0x00000000;
+	int ret;
+
+	/* Block runlist to prevent the channel from being rescheduled. */
+	mutex_lock(&subdev->mutex);
+	nvkm_mask(device, 0x002630, BIT(chan->runl), BIT(chan->runl));
+
+	/* Preempt the channel. */
+	ret = gk104_fifo_gpfifo_kick_locked(chan);
+	if (ret == 0) {
+		/* Update engine context validity. */
+		nvkm_kmap(chan->base.inst);
+		nvkm_mo32(chan->base.inst, 0x0ac, mask, data);
+		nvkm_done(chan->base.inst);
+	}
+
+	/* Resume runlist. */
+	nvkm_mask(device, 0x002630, BIT(chan->runl), 0);
+	mutex_unlock(&subdev->mutex);
+	return ret;
+}
+
+static int
+gv100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base,
+			      struct nvkm_engine *engine, bool suspend)
+{
+	struct gk104_fifo_chan *chan = gk104_fifo_chan(base);
+	struct nvkm_gpuobj *inst = chan->base.inst;
+	int ret;
+
+	if (engine->subdev.index >= NVKM_ENGINE_CE0 &&
+	    engine->subdev.index <= NVKM_ENGINE_CE_LAST)
+		return gk104_fifo_gpfifo_kick(chan);
+
+	ret = gv100_fifo_gpfifo_engine_valid(chan, false, false);
+	if (ret && suspend)
+		return ret;
+
+	nvkm_kmap(inst);
+	nvkm_wo32(inst, 0x0210, 0x00000000);
+	nvkm_wo32(inst, 0x0214, 0x00000000);
+	nvkm_done(inst);
+	return ret;
+}
+
+static int
+gv100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base,
+			      struct nvkm_engine *engine)
+{
+	struct gk104_fifo_chan *chan = gk104_fifo_chan(base);
+	struct nvkm_gpuobj *inst = chan->base.inst;
+	u64 addr;
+
+	if (engine->subdev.index >= NVKM_ENGINE_CE0 &&
+	    engine->subdev.index <= NVKM_ENGINE_CE_LAST)
+		return 0;
+
+	addr = chan->engn[engine->subdev.index].vma->addr;
+	nvkm_kmap(inst);
+	nvkm_wo32(inst, 0x210, lower_32_bits(addr) | 0x00000004);
+	nvkm_wo32(inst, 0x214, upper_32_bits(addr));
+	nvkm_done(inst);
+
+	return gv100_fifo_gpfifo_engine_valid(chan, false, true);
+}
+
+const struct nvkm_fifo_chan_func
+gv100_fifo_gpfifo_func = {
+	.dtor = gk104_fifo_gpfifo_dtor,
+	.init = gk104_fifo_gpfifo_init,
+	.fini = gk104_fifo_gpfifo_fini,
+	.ntfy = gf100_fifo_chan_ntfy,
+	.engine_ctor = gk104_fifo_gpfifo_engine_ctor,
+	.engine_dtor = gk104_fifo_gpfifo_engine_dtor,
+	.engine_init = gv100_fifo_gpfifo_engine_init,
+	.engine_fini = gv100_fifo_gpfifo_engine_fini,
+};
+
+static int
+gv100_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid,
+		       u64 vmm, u64 ioffset, u64 ilength,
+		       const struct nvkm_oclass *oclass,
+		       struct nvkm_object **pobject)
+{
+	struct gk104_fifo_chan *chan;
+	int runlist = ffs(*runlists) -1, ret, i;
+	unsigned long engm;
+	u64 subdevs = 0;
+	u64 usermem;
+
+	if (!vmm || runlist < 0 || runlist >= fifo->runlist_nr)
+		return -EINVAL;
+	*runlists = BIT_ULL(runlist);
+
+	engm = fifo->runlist[runlist].engm;
+	for_each_set_bit(i, &engm, fifo->engine_nr) {
+		if (fifo->engine[i].engine)
+			subdevs |= BIT_ULL(fifo->engine[i].engine->subdev.index);
+	}
+
+	/* Allocate the channel. */
+	if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL)))
+		return -ENOMEM;
+	*pobject = &chan->base.object;
+	chan->fifo = fifo;
+	chan->runl = runlist;
+	INIT_LIST_HEAD(&chan->head);
+
+	ret = nvkm_fifo_chan_ctor(&gv100_fifo_gpfifo_func, &fifo->base,
+				  0x1000, 0x1000, true, vmm, 0, subdevs,
+				  1, fifo->user.bar->addr, 0x200,
+				  oclass, &chan->base);
+	if (ret)
+		return ret;
+
+	*chid = chan->base.chid;
+
+	/* Hack to support GPUs where even individual channels should be
+	 * part of a channel group.
+	 */
+	if (fifo->func->cgrp_force) {
+		if (!(chan->cgrp = kmalloc(sizeof(*chan->cgrp), GFP_KERNEL)))
+			return -ENOMEM;
+		chan->cgrp->id = chan->base.chid;
+		INIT_LIST_HEAD(&chan->cgrp->head);
+		INIT_LIST_HEAD(&chan->cgrp->chan);
+		chan->cgrp->chan_nr = 0;
+	}
+
+	/* Clear channel control registers. */
+	usermem = chan->base.chid * 0x200;
+	ilength = order_base_2(ilength / 8);
+
+	nvkm_kmap(fifo->user.mem);
+	for (i = 0; i < 0x200; i += 4)
+		nvkm_wo32(fifo->user.mem, usermem + i, 0x00000000);
+	nvkm_done(fifo->user.mem);
+	usermem = nvkm_memory_addr(fifo->user.mem) + usermem;
+
+	/* RAMFC */
+	nvkm_kmap(chan->base.inst);
+	nvkm_wo32(chan->base.inst, 0x008, lower_32_bits(usermem));
+	nvkm_wo32(chan->base.inst, 0x00c, upper_32_bits(usermem));
+	nvkm_wo32(chan->base.inst, 0x010, 0x0000face);
+	nvkm_wo32(chan->base.inst, 0x030, 0x7ffff902);
+	nvkm_wo32(chan->base.inst, 0x048, lower_32_bits(ioffset));
+	nvkm_wo32(chan->base.inst, 0x04c, upper_32_bits(ioffset) |
+					  (ilength << 16));
+	nvkm_wo32(chan->base.inst, 0x084, 0x20400000);
+	nvkm_wo32(chan->base.inst, 0x094, 0x30000001);
+	nvkm_wo32(chan->base.inst, 0x0e4, 0x00000020);
+	nvkm_wo32(chan->base.inst, 0x0e8, chan->base.chid);
+	nvkm_wo32(chan->base.inst, 0x0f4, 0x00001100);
+	nvkm_wo32(chan->base.inst, 0x0f8, 0x10003080);
+	nvkm_mo32(chan->base.inst, 0x218, 0x00000000, 0x00000000);
+	nvkm_wo32(chan->base.inst, 0x220, 0x020a1000);
+	nvkm_wo32(chan->base.inst, 0x224, 0x00000000);
+	nvkm_done(chan->base.inst);
+	return gv100_fifo_gpfifo_engine_valid(chan, true, true);
+}
+
+int
+gv100_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass,
+		      void *data, u32 size, struct nvkm_object **pobject)
+{
+	struct nvkm_object *parent = oclass->parent;
+	union {
+		struct kepler_channel_gpfifo_a_v0 v0;
+	} *args = data;
+	int ret = -ENOSYS;
+
+	nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
+	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
+		nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx "
+				   "ioffset %016llx ilength %08x "
+				   "runlist %016llx\n",
+			   args->v0.version, args->v0.vmm, args->v0.ioffset,
+			   args->v0.ilength, args->v0.runlist);
+		return gv100_fifo_gpfifo_new_(fifo,
+					      &args->v0.runlist,
+					      &args->v0.chid,
+					       args->v0.vmm,
+					       args->v0.ioffset,
+					       args->v0.ilength,
+					      oclass, pobject);
+	}
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
new file mode 100644
index 000000000000..4e1d159c0ae7
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
@@ -0,0 +1,306 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "gk104.h"
+#include "cgrp.h"
+#include "changk104.h"
+#include "user.h"
+
+#include <core/gpuobj.h>
+
+#include <nvif/class.h>
+
+static void
+gv100_fifo_runlist_chan(struct gk104_fifo_chan *chan,
+			struct nvkm_memory *memory, u32 offset)
+{
+	struct nvkm_memory *usermem = chan->fifo->user.mem;
+	const u64 user = nvkm_memory_addr(usermem) + (chan->base.chid * 0x200);
+	const u64 inst = chan->base.inst->addr;
+
+	nvkm_wo32(memory, offset + 0x0, lower_32_bits(user));
+	nvkm_wo32(memory, offset + 0x4, upper_32_bits(user));
+	nvkm_wo32(memory, offset + 0x8, lower_32_bits(inst) | chan->base.chid);
+	nvkm_wo32(memory, offset + 0xc, upper_32_bits(inst));
+}
+
+static void
+gv100_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *cgrp,
+			struct nvkm_memory *memory, u32 offset)
+{
+	nvkm_wo32(memory, offset + 0x0, (128 << 24) | (3 << 16) | 0x00000001);
+	nvkm_wo32(memory, offset + 0x4, cgrp->chan_nr);
+	nvkm_wo32(memory, offset + 0x8, cgrp->id);
+	nvkm_wo32(memory, offset + 0xc, 0x00000000);
+}
+
+const struct gk104_fifo_runlist_func
+gv100_fifo_runlist = {
+	.size = 16,
+	.cgrp = gv100_fifo_runlist_cgrp,
+	.chan = gv100_fifo_runlist_chan,
+};
+
+static const struct nvkm_enum
+gv100_fifo_fault_gpcclient[] = {
+	{ 0x00, "T1_0" },
+	{ 0x01, "T1_1" },
+	{ 0x02, "T1_2" },
+	{ 0x03, "T1_3" },
+	{ 0x04, "T1_4" },
+	{ 0x05, "T1_5" },
+	{ 0x06, "T1_6" },
+	{ 0x07, "T1_7" },
+	{ 0x08, "PE_0" },
+	{ 0x09, "PE_1" },
+	{ 0x0a, "PE_2" },
+	{ 0x0b, "PE_3" },
+	{ 0x0c, "PE_4" },
+	{ 0x0d, "PE_5" },
+	{ 0x0e, "PE_6" },
+	{ 0x0f, "PE_7" },
+	{ 0x10, "RAST" },
+	{ 0x11, "GCC" },
+	{ 0x12, "GPCCS" },
+	{ 0x13, "PROP_0" },
+	{ 0x14, "PROP_1" },
+	{ 0x15, "PROP_2" },
+	{ 0x16, "PROP_3" },
+	{ 0x17, "GPM" },
+	{ 0x18, "LTP_UTLB_0" },
+	{ 0x19, "LTP_UTLB_1" },
+	{ 0x1a, "LTP_UTLB_2" },
+	{ 0x1b, "LTP_UTLB_3" },
+	{ 0x1c, "LTP_UTLB_4" },
+	{ 0x1d, "LTP_UTLB_5" },
+	{ 0x1e, "LTP_UTLB_6" },
+	{ 0x1f, "LTP_UTLB_7" },
+	{ 0x20, "RGG_UTLB" },
+	{ 0x21, "T1_8" },
+	{ 0x22, "T1_9" },
+	{ 0x23, "T1_10" },
+	{ 0x24, "T1_11" },
+	{ 0x25, "T1_12" },
+	{ 0x26, "T1_13" },
+	{ 0x27, "T1_14" },
+	{ 0x28, "T1_15" },
+	{ 0x29, "TPCCS_0" },
+	{ 0x2a, "TPCCS_1" },
+	{ 0x2b, "TPCCS_2" },
+	{ 0x2c, "TPCCS_3" },
+	{ 0x2d, "TPCCS_4" },
+	{ 0x2e, "TPCCS_5" },
+	{ 0x2f, "TPCCS_6" },
+	{ 0x30, "TPCCS_7" },
+	{ 0x31, "PE_8" },
+	{ 0x32, "PE_9" },
+	{ 0x33, "TPCCS_8" },
+	{ 0x34, "TPCCS_9" },
+	{ 0x35, "T1_16" },
+	{ 0x36, "T1_17" },
+	{ 0x37, "T1_18" },
+	{ 0x38, "T1_19" },
+	{ 0x39, "PE_10" },
+	{ 0x3a, "PE_11" },
+	{ 0x3b, "TPCCS_10" },
+	{ 0x3c, "TPCCS_11" },
+	{ 0x3d, "T1_20" },
+	{ 0x3e, "T1_21" },
+	{ 0x3f, "T1_22" },
+	{ 0x40, "T1_23" },
+	{ 0x41, "PE_12" },
+	{ 0x42, "PE_13" },
+	{ 0x43, "TPCCS_12" },
+	{ 0x44, "TPCCS_13" },
+	{ 0x45, "T1_24" },
+	{ 0x46, "T1_25" },
+	{ 0x47, "T1_26" },
+	{ 0x48, "T1_27" },
+	{ 0x49, "PE_14" },
+	{ 0x4a, "PE_15" },
+	{ 0x4b, "TPCCS_14" },
+	{ 0x4c, "TPCCS_15" },
+	{ 0x4d, "T1_28" },
+	{ 0x4e, "T1_29" },
+	{ 0x4f, "T1_30" },
+	{ 0x50, "T1_31" },
+	{ 0x51, "PE_16" },
+	{ 0x52, "PE_17" },
+	{ 0x53, "TPCCS_16" },
+	{ 0x54, "TPCCS_17" },
+	{ 0x55, "T1_32" },
+	{ 0x56, "T1_33" },
+	{ 0x57, "T1_34" },
+	{ 0x58, "T1_35" },
+	{ 0x59, "PE_18" },
+	{ 0x5a, "PE_19" },
+	{ 0x5b, "TPCCS_18" },
+	{ 0x5c, "TPCCS_19" },
+	{ 0x5d, "T1_36" },
+	{ 0x5e, "T1_37" },
+	{ 0x5f, "T1_38" },
+	{ 0x60, "T1_39" },
+	{}
+};
+
+static const struct nvkm_enum
+gv100_fifo_fault_hubclient[] = {
+	{ 0x00, "VIP" },
+	{ 0x01, "CE0" },
+	{ 0x02, "CE1" },
+	{ 0x03, "DNISO" },
+	{ 0x04, "FE" },
+	{ 0x05, "FECS" },
+	{ 0x06, "HOST" },
+	{ 0x07, "HOST_CPU" },
+	{ 0x08, "HOST_CPU_NB" },
+	{ 0x09, "ISO" },
+	{ 0x0a, "MMU" },
+	{ 0x0b, "NVDEC" },
+	{ 0x0d, "NVENC1" },
+	{ 0x0e, "NISO" },
+	{ 0x0f, "P2P" },
+	{ 0x10, "PD" },
+	{ 0x11, "PERF" },
+	{ 0x12, "PMU" },
+	{ 0x13, "RASTERTWOD" },
+	{ 0x14, "SCC" },
+	{ 0x15, "SCC_NB" },
+	{ 0x16, "SEC" },
+	{ 0x17, "SSYNC" },
+	{ 0x18, "CE2" },
+	{ 0x19, "XV" },
+	{ 0x1a, "MMU_NB" },
+	{ 0x1b, "NVENC0" },
+	{ 0x1c, "DFALCON" },
+	{ 0x1d, "SKED" },
+	{ 0x1e, "AFALCON" },
+	{ 0x1f, "DONT_CARE" },
+	{ 0x20, "HSCE0" },
+	{ 0x21, "HSCE1" },
+	{ 0x22, "HSCE2" },
+	{ 0x23, "HSCE3" },
+	{ 0x24, "HSCE4" },
+	{ 0x25, "HSCE5" },
+	{ 0x26, "HSCE6" },
+	{ 0x27, "HSCE7" },
+	{ 0x28, "HSCE8" },
+	{ 0x29, "HSCE9" },
+	{ 0x2a, "HSHUB" },
+	{ 0x2b, "PTP_X0" },
+	{ 0x2c, "PTP_X1" },
+	{ 0x2d, "PTP_X2" },
+	{ 0x2e, "PTP_X3" },
+	{ 0x2f, "PTP_X4" },
+	{ 0x30, "PTP_X5" },
+	{ 0x31, "PTP_X6" },
+	{ 0x32, "PTP_X7" },
+	{ 0x33, "NVENC2" },
+	{ 0x34, "VPR_SCRUBBER0" },
+	{ 0x35, "VPR_SCRUBBER1" },
+	{ 0x36, "DWBIF" },
+	{ 0x37, "FBFALCON" },
+	{ 0x38, "CE_SHIM" },
+	{ 0x39, "GSP" },
+	{}
+};
+
+static const struct nvkm_enum
+gv100_fifo_fault_reason[] = {
+	{ 0x00, "PDE" },
+	{ 0x01, "PDE_SIZE" },
+	{ 0x02, "PTE" },
+	{ 0x03, "VA_LIMIT_VIOLATION" },
+	{ 0x04, "UNBOUND_INST_BLOCK" },
+	{ 0x05, "PRIV_VIOLATION" },
+	{ 0x06, "RO_VIOLATION" },
+	{ 0x07, "WO_VIOLATION" },
+	{ 0x08, "PITCH_MASK_VIOLATION" },
+	{ 0x09, "WORK_CREATION" },
+	{ 0x0a, "UNSUPPORTED_APERTURE" },
+	{ 0x0b, "COMPRESSION_FAILURE" },
+	{ 0x0c, "UNSUPPORTED_KIND" },
+	{ 0x0d, "REGION_VIOLATION" },
+	{ 0x0e, "POISONED" },
+	{ 0x0f, "ATOMIC_VIOLATION" },
+	{}
+};
+
+static const struct nvkm_enum
+gv100_fifo_fault_engine[] = {
+	{ 0x01, "DISPLAY" },
+	{ 0x03, "PTP" },
+	{ 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR },
+	{ 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM },
+	{ 0x06, "PWR_PMU" },
+	{ 0x08, "IFB", NULL, NVKM_ENGINE_IFB },
+	{ 0x09, "PERF" },
+	{ 0x1f, "PHYSICAL" },
+	{ 0x20, "HOST0" },
+	{ 0x21, "HOST1" },
+	{ 0x22, "HOST2" },
+	{ 0x23, "HOST3" },
+	{ 0x24, "HOST4" },
+	{ 0x25, "HOST5" },
+	{ 0x26, "HOST6" },
+	{ 0x27, "HOST7" },
+	{ 0x28, "HOST8" },
+	{ 0x29, "HOST9" },
+	{ 0x2a, "HOST10" },
+	{ 0x2b, "HOST11" },
+	{ 0x2c, "HOST12" },
+	{ 0x2d, "HOST13" },
+	{}
+};
+
+static const struct nvkm_enum
+gv100_fifo_fault_access[] = {
+	{ 0x0, "VIRT_READ" },
+	{ 0x1, "VIRT_WRITE" },
+	{ 0x2, "VIRT_ATOMIC" },
+	{ 0x3, "VIRT_PREFETCH" },
+	{ 0x4, "VIRT_ATOMIC_WEAK" },
+	{ 0x8, "PHYS_READ" },
+	{ 0x9, "PHYS_WRITE" },
+	{ 0xa, "PHYS_ATOMIC" },
+	{ 0xb, "PHYS_PREFETCH" },
+	{}
+};
+
+static const struct gk104_fifo_func
+gv100_fifo = {
+	.init_pbdma_timeout = gk208_fifo_init_pbdma_timeout,
+	.fault.access = gv100_fifo_fault_access,
+	.fault.engine = gv100_fifo_fault_engine,
+	.fault.reason = gv100_fifo_fault_reason,
+	.fault.hubclient = gv100_fifo_fault_hubclient,
+	.fault.gpcclient = gv100_fifo_fault_gpcclient,
+	.runlist = &gv100_fifo_runlist,
+	.user = {{-1,-1,VOLTA_USERMODE_A      }, gv100_fifo_user_new   },
+	.chan = {{ 0, 0,VOLTA_CHANNEL_GPFIFO_A}, gv100_fifo_gpfifo_new },
+	.cgrp_force = true,
+};
+
+int
+gv100_fifo_new(struct nvkm_device *device, int index, struct nvkm_fifo **pfifo)
+{
+	return gk104_fifo_new_(&gv100_fifo, device, index, 4096, pfifo);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
index ae76b1aaccd4..d5acbba293f4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
@@ -18,16 +18,19 @@ struct nvkm_fifo_chan_oclass;
 struct nvkm_fifo_func {
 	void *(*dtor)(struct nvkm_fifo *);
 	int (*oneinit)(struct nvkm_fifo *);
+	int (*info)(struct nvkm_fifo *, u64 mthd, u64 *data);
 	void (*init)(struct nvkm_fifo *);
 	void (*fini)(struct nvkm_fifo *);
 	void (*intr)(struct nvkm_fifo *);
+	void (*fault)(struct nvkm_fifo *, struct nvkm_fault_data *);
 	void (*pause)(struct nvkm_fifo *, unsigned long *);
 	void (*start)(struct nvkm_fifo *, unsigned long *);
 	void (*uevent_init)(struct nvkm_fifo *);
 	void (*uevent_fini)(struct nvkm_fifo *);
 	void (*recover_chan)(struct nvkm_fifo *, int chid);
-	int (*class_get)(struct nvkm_fifo *, int index,
-			 const struct nvkm_fifo_chan_oclass **);
+	int (*class_get)(struct nvkm_fifo *, int index, struct nvkm_oclass *);
+	int (*class_new)(struct nvkm_fifo *, const struct nvkm_oclass *,
+			 void *, u32, struct nvkm_object **);
 	const struct nvkm_fifo_chan_oclass *chan[];
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h
new file mode 100644
index 000000000000..ed840921ebe8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h
@@ -0,0 +1,6 @@
+#ifndef __NVKM_FIFO_USER_H__
+#define __NVKM_FIFO_USER_H__
+#include "priv.h"
+int gv100_fifo_user_new(const struct nvkm_oclass *, void *, u32,
+			struct nvkm_object **);
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usergv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usergv100.c
new file mode 100644
index 000000000000..3dc3b8b312de
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usergv100.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "user.h"
+
+static int
+gv100_fifo_user_map(struct nvkm_object *object, void *argv, u32 argc,
+		    enum nvkm_object_map *type, u64 *addr, u64 *size)
+{
+	struct nvkm_device *device = object->engine->subdev.device;
+	*addr = 0x810000 + device->func->resource_addr(device, 0);
+	*size = 0x010000;
+	*type = NVKM_OBJECT_MAP_IO;
+	return 0;
+}
+
+static const struct nvkm_object_func
+gv100_fifo_user = {
+	.map = gv100_fifo_user_map,
+};
+
+int
+gv100_fifo_user_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+		    struct nvkm_object **pobject)
+{
+	return nvkm_object_new_(&gv100_fifo_user, oclass, argv, argc, pobject);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild
index 8a22558b7b52..93e3733f54e2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild
@@ -33,8 +33,10 @@ nvkm-y += nvkm/engine/gr/gm200.o
 nvkm-y += nvkm/engine/gr/gm20b.o
 nvkm-y += nvkm/engine/gr/gp100.o
 nvkm-y += nvkm/engine/gr/gp102.o
+nvkm-y += nvkm/engine/gr/gp104.o
 nvkm-y += nvkm/engine/gr/gp107.o
 nvkm-y += nvkm/engine/gr/gp10b.o
+nvkm-y += nvkm/engine/gr/gv100.o
 
 nvkm-y += nvkm/engine/gr/ctxnv40.o
 nvkm-y += nvkm/engine/gr/ctxnv50.o
@@ -54,4 +56,6 @@ nvkm-y += nvkm/engine/gr/ctxgm200.o
 nvkm-y += nvkm/engine/gr/ctxgm20b.o
 nvkm-y += nvkm/engine/gr/ctxgp100.o
 nvkm-y += nvkm/engine/gr/ctxgp102.o
+nvkm-y += nvkm/engine/gr/ctxgp104.o
 nvkm-y += nvkm/engine/gr/ctxgp107.o
+nvkm-y += nvkm/engine/gr/ctxgv100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c
index 881015080d83..e813a3f8ea93 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c
@@ -850,12 +850,17 @@ gf100_grctx_init_gcc_0[] = {
 };
 
 const struct gf100_gr_pack
-gf100_grctx_pack_gpc[] = {
+gf100_grctx_pack_gpc_0[] = {
 	{ gf100_grctx_init_gpc_unk_0 },
 	{ gf100_grctx_init_prop_0 },
 	{ gf100_grctx_init_gpc_unk_1 },
 	{ gf100_grctx_init_setup_0 },
 	{ gf100_grctx_init_zcull_0 },
+	{}
+};
+
+const struct gf100_gr_pack
+gf100_grctx_pack_gpc_1[] = {
 	{ gf100_grctx_init_crstr_0 },
 	{ gf100_grctx_init_gpm_0 },
 	{ gf100_grctx_init_gcc_0 },
@@ -1024,6 +1029,13 @@ gf100_grctx_mmio_item(struct gf100_grctx *info, u32 addr, u32 data,
 	nvkm_wr32(device, addr, data);
 }
 
+void
+gf100_grctx_generate_r419cb8(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, 0x419cb8, 0x00007c00, 0x00000000);
+}
+
 void
 gf100_grctx_generate_bundle(struct gf100_grctx *info)
 {
@@ -1079,90 +1091,39 @@ gf100_grctx_generate_unkn(struct gf100_gr *gr)
 {
 }
 
-void
-gf100_grctx_generate_tpcid(struct gf100_gr *gr)
-{
-	struct nvkm_device *device = gr->base.engine.subdev.device;
-	int gpc, tpc, id;
-
-	for (tpc = 0, id = 0; tpc < 4; tpc++) {
-		for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-			if (tpc < gr->tpc_nr[gpc]) {
-				nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x698), id);
-				nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x4e8), id);
-				nvkm_wr32(device, GPC_UNIT(gpc, 0x0c10 + tpc * 4), id);
-				nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x088), id);
-				id++;
-			}
-
-			nvkm_wr32(device, GPC_UNIT(gpc, 0x0c08), gr->tpc_nr[gpc]);
-			nvkm_wr32(device, GPC_UNIT(gpc, 0x0c8c), gr->tpc_nr[gpc]);
-		}
-	}
-}
-
-void
-gf100_grctx_generate_r406028(struct gf100_gr *gr)
-{
-	struct nvkm_device *device = gr->base.engine.subdev.device;
-	u32 tmp[GPC_MAX / 8] = {}, i = 0;
-	for (i = 0; i < gr->gpc_nr; i++)
-		tmp[i / 8] |= gr->tpc_nr[i] << ((i % 8) * 4);
-	for (i = 0; i < 4; i++) {
-		nvkm_wr32(device, 0x406028 + (i * 4), tmp[i]);
-		nvkm_wr32(device, 0x405870 + (i * 4), tmp[i]);
-	}
-}
-
 void
 gf100_grctx_generate_r4060a8(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-	u8  tpcnr[GPC_MAX], data[TPC_MAX];
-	int gpc, tpc, i;
-
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-	memset(data, 0x1f, sizeof(data));
-
-	gpc = -1;
-	for (tpc = 0; tpc < gr->tpc_total; tpc++) {
-		do {
-			gpc = (gpc + 1) % gr->gpc_nr;
-		} while (!tpcnr[gpc]);
-		tpcnr[gpc]--;
-		data[tpc] = gpc;
+	const u8 gpcmax = nvkm_rd32(device, 0x022430);
+	const u8 tpcmax = nvkm_rd32(device, 0x022434) * gpcmax;
+	int i, j, sm = 0;
+	u32 data;
+
+	for (i = 0; i < DIV_ROUND_UP(tpcmax, 4); i++) {
+		for (data = 0, j = 0; j < 4; j++) {
+			if (sm < gr->sm_nr)
+				data |= gr->sm[sm++].gpc << (j * 8);
+			else
+				data |= 0x1f << (j * 8);
+		}
+		nvkm_wr32(device, 0x4060a8 + (i * 4), data);
 	}
-
-	for (i = 0; i < 4; i++)
-		nvkm_wr32(device, 0x4060a8 + (i * 4), ((u32 *)data)[i]);
 }
 
 void
-gf100_grctx_generate_r418bb8(struct gf100_gr *gr)
+gf100_grctx_generate_rop_mapping(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
 	u32 data[6] = {}, data2[2] = {};
-	u8  tpcnr[GPC_MAX];
 	u8  shift, ntpcv;
-	int gpc, tpc, i;
-
-	/* calculate first set of magics */
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-
-	gpc = -1;
-	for (tpc = 0; tpc < gr->tpc_total; tpc++) {
-		do {
-			gpc = (gpc + 1) % gr->gpc_nr;
-		} while (!tpcnr[gpc]);
-		tpcnr[gpc]--;
-
-		data[tpc / 6] |= gpc << ((tpc % 6) * 5);
-	}
+	int i;
 
-	for (; tpc < 32; tpc++)
-		data[tpc / 6] |= 7 << ((tpc % 6) * 5);
+	/* Pack tile map into register format. */
+	for (i = 0; i < 32; i++)
+		data[i / 6] |= (gr->tile[i] & 0x07) << ((i % 6) * 5);
 
-	/* and the second... */
+	/* Magic. */
 	shift = 0;
 	ntpcv = gr->tpc_total;
 	while (!(ntpcv & (1 << 4))) {
@@ -1197,39 +1158,213 @@ gf100_grctx_generate_r418bb8(struct gf100_gr *gr)
 }
 
 void
-gf100_grctx_generate_r406800(struct gf100_gr *gr)
+gf100_grctx_generate_max_ways_evict(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-	u64 tpc_mask = 0, tpc_set = 0;
-	u8  tpcnr[GPC_MAX];
-	int gpc, tpc;
-	int i, a, b;
-
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-	for (gpc = 0; gpc < gr->gpc_nr; gpc++)
-		tpc_mask |= ((1ULL << gr->tpc_nr[gpc]) - 1) << (gpc * 8);
-
-	for (i = 0, gpc = -1, b = -1; i < 32; i++) {
-		a = (i * (gr->tpc_total - 1)) / 32;
-		if (a != b) {
-			b = a;
-			do {
-				gpc = (gpc + 1) % gr->gpc_nr;
-			} while (!tpcnr[gpc]);
-			tpc = gr->tpc_nr[gpc] - tpcnr[gpc]--;
-
-			tpc_set |= 1ULL << ((gpc * 8) + tpc);
+	u32 fbps = nvkm_rd32(device, 0x121c74);
+	if (fbps == 1)
+		nvkm_mask(device, 0x17e91c, 0x001f0000, 0x00090000);
+}
+
+static const u32
+gf100_grctx_alpha_beta_map[17][32] = {
+	[1] = {
+		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	},
+	[2] = {
+		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	},
+	//XXX: 3
+	[4] = {
+		1, 1, 1, 1, 1, 1, 1, 1,
+		2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+		3, 3, 3, 3, 3, 3, 3, 3,
+	},
+	//XXX: 5
+	//XXX: 6
+	[7] = {
+		1, 1, 1, 1,
+		2, 2, 2, 2, 2, 2,
+		3, 3, 3, 3, 3, 3,
+		4, 4, 4, 4, 4, 4,
+		5, 5, 5, 5, 5, 5,
+		6, 6, 6, 6,
+	},
+	[8] = {
+		1, 1, 1,
+		2, 2, 2, 2, 2,
+		3, 3, 3, 3, 3,
+		4, 4, 4, 4, 4, 4,
+		5, 5, 5, 5, 5,
+		6, 6, 6, 6, 6,
+		7, 7, 7,
+	},
+	//XXX: 9
+	//XXX: 10
+	[11] = {
+		1, 1,
+		2, 2, 2, 2,
+		3, 3, 3,
+		4, 4, 4, 4,
+		5, 5, 5,
+		6, 6, 6,
+		7, 7, 7, 7,
+		8, 8, 8,
+		9, 9, 9, 9,
+		10, 10,
+	},
+	//XXX: 12
+	//XXX: 13
+	[14] = {
+		1, 1,
+		2, 2,
+		3, 3, 3,
+		4, 4, 4,
+		5, 5,
+		6, 6, 6,
+		7, 7,
+		8, 8, 8,
+		9, 9,
+		10, 10, 10,
+		11, 11, 11,
+		12, 12,
+		13, 13,
+	},
+	[15] = {
+		1, 1,
+		2, 2,
+		3, 3,
+		4, 4, 4,
+		5, 5,
+		6, 6, 6,
+		7, 7,
+		8, 8,
+		9, 9, 9,
+		10, 10,
+		11, 11, 11,
+		12, 12,
+		13, 13,
+		14, 14,
+	},
+	[16] = {
+		1, 1,
+		2, 2,
+		3, 3,
+		4, 4,
+		5, 5,
+		6, 6, 6,
+		7, 7,
+		8, 8,
+		9, 9,
+		10, 10, 10,
+		11, 11,
+		12, 12,
+		13, 13,
+		14, 14,
+		15, 15,
+	},
+};
+
+void
+gf100_grctx_generate_alpha_beta_tables(struct gf100_gr *gr)
+{
+	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	int i, gpc;
+
+	for (i = 0; i < 32; i++) {
+		u32 atarget = gf100_grctx_alpha_beta_map[gr->tpc_total][i];
+		u32 abits[GPC_MAX] = {}, amask = 0, bmask = 0;
+
+		if (!atarget) {
+			nvkm_warn(subdev, "missing alpha/beta mapping table\n");
+			atarget = max_t(u32, gr->tpc_total * i / 32, 1);
+		}
+
+		while (atarget) {
+			for (gpc = 0; atarget && gpc < gr->gpc_nr; gpc++) {
+				if (abits[gpc] < gr->tpc_nr[gpc]) {
+					abits[gpc]++;
+					atarget--;
+				}
+			}
 		}
 
-		nvkm_wr32(device, 0x406800 + (i * 0x20), lower_32_bits(tpc_set));
-		nvkm_wr32(device, 0x406c00 + (i * 0x20), lower_32_bits(tpc_set ^ tpc_mask));
-		if (gr->gpc_nr > 4) {
-			nvkm_wr32(device, 0x406804 + (i * 0x20), upper_32_bits(tpc_set));
-			nvkm_wr32(device, 0x406c04 + (i * 0x20), upper_32_bits(tpc_set ^ tpc_mask));
+		for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
+			u32 bbits = gr->tpc_nr[gpc] - abits[gpc];
+			amask |= ((1 << abits[gpc]) - 1) << (gpc * 8);
+			bmask |= ((1 << bbits) - 1) << abits[gpc] << (gpc * 8);
 		}
+
+		nvkm_wr32(device, 0x406800 + (i * 0x20), amask);
+		nvkm_wr32(device, 0x406c00 + (i * 0x20), bmask);
 	}
 }
 
+void
+gf100_grctx_generate_tpc_nr(struct gf100_gr *gr, int gpc)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_wr32(device, GPC_UNIT(gpc, 0x0c08), gr->tpc_nr[gpc]);
+	nvkm_wr32(device, GPC_UNIT(gpc, 0x0c8c), gr->tpc_nr[gpc]);
+}
+
+void
+gf100_grctx_generate_sm_id(struct gf100_gr *gr, int gpc, int tpc, int sm)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x698), sm);
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x4e8), sm);
+	nvkm_wr32(device, GPC_UNIT(gpc, 0x0c10 + tpc * 4), sm);
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x088), sm);
+}
+
+void
+gf100_grctx_generate_floorsweep(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	const struct gf100_grctx_func *func = gr->func->grctx;
+	int gpc, sm, i, j;
+	u32 data;
+
+	for (sm = 0; sm < gr->sm_nr; sm++) {
+		func->sm_id(gr, gr->sm[sm].gpc, gr->sm[sm].tpc, sm);
+		if (func->tpc_nr)
+			func->tpc_nr(gr, gr->sm[sm].gpc);
+	}
+
+	for (gpc = 0, i = 0; i < 4; i++) {
+		for (data = 0, j = 0; j < 8 && gpc < gr->gpc_nr; j++, gpc++)
+			data |= gr->tpc_nr[gpc] << (j * 4);
+		nvkm_wr32(device, 0x406028 + (i * 4), data);
+		nvkm_wr32(device, 0x405870 + (i * 4), data);
+	}
+
+	if (func->r4060a8)
+		func->r4060a8(gr);
+
+	func->rop_mapping(gr);
+
+	if (func->alpha_beta_tables)
+		func->alpha_beta_tables(gr);
+	if (func->max_ways_evict)
+		func->max_ways_evict(gr);
+	if (func->dist_skip_table)
+		func->dist_skip_table(gr);
+	if (func->r406500)
+		func->r406500(gr);
+	if (func->gpc_tpc_nr)
+		func->gpc_tpc_nr(gr);
+	if (func->r419f78)
+		func->r419f78(gr);
+	if (func->tpc_mask)
+		func->tpc_mask(gr);
+	if (func->smid_config)
+		func->smid_config(gr);
+}
+
 void
 gf100_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
 {
@@ -1239,29 +1374,63 @@ gf100_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
 
 	nvkm_mc_unk260(device, 0);
 
-	gf100_gr_mmio(gr, grctx->hub);
-	gf100_gr_mmio(gr, grctx->gpc);
-	gf100_gr_mmio(gr, grctx->zcull);
-	gf100_gr_mmio(gr, grctx->tpc);
-	gf100_gr_mmio(gr, grctx->ppc);
+	if (!gr->fuc_sw_ctx) {
+		gf100_gr_mmio(gr, grctx->hub);
+		gf100_gr_mmio(gr, grctx->gpc_0);
+		gf100_gr_mmio(gr, grctx->zcull);
+		gf100_gr_mmio(gr, grctx->gpc_1);
+		gf100_gr_mmio(gr, grctx->tpc);
+		gf100_gr_mmio(gr, grctx->ppc);
+	} else {
+		gf100_gr_mmio(gr, gr->fuc_sw_ctx);
+	}
+
+	gf100_gr_wait_idle(gr);
 
 	idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000);
 
-	grctx->bundle(info);
 	grctx->pagepool(info);
+	grctx->bundle(info);
 	grctx->attrib(info);
+	if (grctx->patch_ltc)
+		grctx->patch_ltc(info);
 	grctx->unkn(gr);
 
-	gf100_grctx_generate_tpcid(gr);
-	gf100_grctx_generate_r406028(gr);
-	gf100_grctx_generate_r4060a8(gr);
-	gf100_grctx_generate_r418bb8(gr);
-	gf100_grctx_generate_r406800(gr);
+	gf100_grctx_generate_floorsweep(gr);
+
+	gf100_gr_wait_idle(gr);
+
+	if (grctx->r400088) grctx->r400088(gr, false);
+	if (gr->fuc_bundle)
+		gf100_gr_icmd(gr, gr->fuc_bundle);
+	else
+		gf100_gr_icmd(gr, grctx->icmd);
+	if (grctx->sw_veid_bundle_init)
+		gf100_gr_icmd(gr, grctx->sw_veid_bundle_init);
+	if (grctx->r400088) grctx->r400088(gr, true);
 
-	gf100_gr_icmd(gr, grctx->icmd);
 	nvkm_wr32(device, 0x404154, idle_timeout);
-	gf100_gr_mthd(gr, grctx->mthd);
+
+	if (gr->fuc_method)
+		gf100_gr_mthd(gr, gr->fuc_method);
+	else
+		gf100_gr_mthd(gr, grctx->mthd);
 	nvkm_mc_unk260(device, 1);
+
+	if (grctx->r419cb8)
+		grctx->r419cb8(gr);
+	if (grctx->r418800)
+		grctx->r418800(gr);
+	if (grctx->r419eb0)
+		grctx->r419eb0(gr);
+	if (grctx->r419e00)
+		grctx->r419e00(gr);
+	if (grctx->r418e94)
+		grctx->r418e94(gr);
+	if (grctx->r419a3c)
+		grctx->r419a3c(gr);
+	if (grctx->r408840)
+		grctx->r408840(gr);
 }
 
 #define CB_RESERVED 0x80000
@@ -1280,6 +1449,32 @@ gf100_grctx_generate(struct gf100_gr *gr)
 	int ret, i;
 	u64 addr;
 
+	/* NV_PGRAPH_FE_PWR_MODE_FORCE_ON. */
+	nvkm_wr32(device, 0x404170, 0x00000012);
+	nvkm_msec(device, 2000,
+		if (!(nvkm_rd32(device, 0x404170) & 0x00000010))
+			break;
+	);
+
+	if (grctx->unkn88c)
+		grctx->unkn88c(gr, true);
+
+	/* Reset FECS. */
+	nvkm_wr32(device, 0x409614, 0x00000070);
+	nvkm_usec(device, 10, NVKM_DELAY);
+	nvkm_mask(device, 0x409614, 0x00000700, 0x00000700);
+	nvkm_usec(device, 10, NVKM_DELAY);
+	nvkm_rd32(device, 0x409614);
+
+	if (grctx->unkn88c)
+		grctx->unkn88c(gr, false);
+
+	/* NV_PGRAPH_FE_PWR_MODE_AUTO. */
+	nvkm_wr32(device, 0x404170, 0x00000010);
+
+	/* Init SCC RAM. */
+	nvkm_wr32(device, 0x40802c, 0x00000001);
+
 	/* Allocate memory to for a "channel", which we'll use to generate
 	 * the default context values.
 	 */
@@ -1392,7 +1587,8 @@ gf100_grctx = {
 	.main  = gf100_grctx_generate_main,
 	.unkn  = gf100_grctx_generate_unkn,
 	.hub   = gf100_grctx_pack_hub,
-	.gpc   = gf100_grctx_pack_gpc,
+	.gpc_0 = gf100_grctx_pack_gpc_0,
+	.gpc_1 = gf100_grctx_pack_gpc_1,
 	.zcull = gf100_grctx_pack_zcull,
 	.tpc   = gf100_grctx_pack_tpc,
 	.icmd  = gf100_grctx_pack_icmd,
@@ -1404,4 +1600,11 @@ gf100_grctx = {
 	.attrib = gf100_grctx_generate_attrib,
 	.attrib_nr_max = 0x324,
 	.attrib_nr = 0x218,
+	.sm_id = gf100_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.r4060a8 = gf100_grctx_generate_r4060a8,
+	.rop_mapping = gf100_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gf100_grctx_generate_alpha_beta_tables,
+	.max_ways_evict = gf100_grctx_generate_max_ways_evict,
+	.r419cb8 = gf100_grctx_generate_r419cb8,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h
index 5199e5aa0cb7..33e932bd73b1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h
@@ -21,19 +21,22 @@ void gf100_grctx_mmio_item(struct gf100_grctx *, u32 addr, u32 data, int s, int)
 #define mmio_wr32(a,b,c) mmio_refn((a), (b), (c),  0, -1)
 
 struct gf100_grctx_func {
+	void (*unkn88c)(struct gf100_gr *, bool on);
 	/* main context generation function */
 	void  (*main)(struct gf100_gr *, struct gf100_grctx *);
 	/* context-specific modify-on-first-load list generation function */
 	void  (*unkn)(struct gf100_gr *);
 	/* mmio context data */
 	const struct gf100_gr_pack *hub;
-	const struct gf100_gr_pack *gpc;
+	const struct gf100_gr_pack *gpc_0;
+	const struct gf100_gr_pack *gpc_1;
 	const struct gf100_gr_pack *zcull;
 	const struct gf100_gr_pack *tpc;
 	const struct gf100_gr_pack *ppc;
 	/* indirect context data, generated with icmds/mthds */
 	const struct gf100_gr_pack *icmd;
 	const struct gf100_gr_pack *mthd;
+	const struct gf100_gr_pack *sw_veid_bundle_init;
 	/* bundle circular buffer */
 	void (*bundle)(struct gf100_grctx *);
 	u32 bundle_size;
@@ -48,6 +51,31 @@ struct gf100_grctx_func {
 	u32 attrib_nr;
 	u32 alpha_nr_max;
 	u32 alpha_nr;
+	u32 gfxp_nr;
+	/* other patch buffer stuff */
+	void (*patch_ltc)(struct gf100_grctx *);
+	/* floorsweeping */
+	void (*sm_id)(struct gf100_gr *, int gpc, int tpc, int sm);
+	void (*tpc_nr)(struct gf100_gr *, int gpc);
+	void (*r4060a8)(struct gf100_gr *);
+	void (*rop_mapping)(struct gf100_gr *);
+	void (*alpha_beta_tables)(struct gf100_gr *);
+	void (*max_ways_evict)(struct gf100_gr *);
+	void (*dist_skip_table)(struct gf100_gr *);
+	void (*r406500)(struct gf100_gr *);
+	void (*gpc_tpc_nr)(struct gf100_gr *);
+	void (*r419f78)(struct gf100_gr *);
+	void (*tpc_mask)(struct gf100_gr *);
+	void (*smid_config)(struct gf100_gr *);
+	/* misc other things */
+	void (*r400088)(struct gf100_gr *, bool);
+	void (*r419cb8)(struct gf100_gr *);
+	void (*r418800)(struct gf100_gr *);
+	void (*r419eb0)(struct gf100_gr *);
+	void (*r419e00)(struct gf100_gr *);
+	void (*r418e94)(struct gf100_gr *);
+	void (*r419a3c)(struct gf100_gr *);
+	void (*r408840)(struct gf100_gr *);
 };
 
 extern const struct gf100_grctx_func gf100_grctx;
@@ -57,11 +85,14 @@ void gf100_grctx_generate_bundle(struct gf100_grctx *);
 void gf100_grctx_generate_pagepool(struct gf100_grctx *);
 void gf100_grctx_generate_attrib(struct gf100_grctx *);
 void gf100_grctx_generate_unkn(struct gf100_gr *);
-void gf100_grctx_generate_tpcid(struct gf100_gr *);
-void gf100_grctx_generate_r406028(struct gf100_gr *);
+void gf100_grctx_generate_floorsweep(struct gf100_gr *);
+void gf100_grctx_generate_sm_id(struct gf100_gr *, int, int, int);
+void gf100_grctx_generate_tpc_nr(struct gf100_gr *, int);
 void gf100_grctx_generate_r4060a8(struct gf100_gr *);
-void gf100_grctx_generate_r418bb8(struct gf100_gr *);
-void gf100_grctx_generate_r406800(struct gf100_gr *);
+void gf100_grctx_generate_rop_mapping(struct gf100_gr *);
+void gf100_grctx_generate_alpha_beta_tables(struct gf100_gr *);
+void gf100_grctx_generate_max_ways_evict(struct gf100_gr *);
+void gf100_grctx_generate_r419cb8(struct gf100_gr *);
 
 extern const struct gf100_grctx_func gf108_grctx;
 void gf108_grctx_generate_attrib(struct gf100_grctx *);
@@ -72,22 +103,25 @@ extern const struct gf100_grctx_func gf110_grctx;
 
 extern const struct gf100_grctx_func gf117_grctx;
 void gf117_grctx_generate_attrib(struct gf100_grctx *);
+void gf117_grctx_generate_rop_mapping(struct gf100_gr *);
+void gf117_grctx_generate_dist_skip_table(struct gf100_gr *);
 
 extern const struct gf100_grctx_func gf119_grctx;
 
 extern const struct gf100_grctx_func gk104_grctx;
+void gk104_grctx_generate_alpha_beta_tables(struct gf100_gr *);
+void gk104_grctx_generate_gpc_tpc_nr(struct gf100_gr *);
+
 extern const struct gf100_grctx_func gk20a_grctx;
-void gk104_grctx_generate_main(struct gf100_gr *, struct gf100_grctx *);
 void gk104_grctx_generate_bundle(struct gf100_grctx *);
 void gk104_grctx_generate_pagepool(struct gf100_grctx *);
+void gk104_grctx_generate_patch_ltc(struct gf100_grctx *);
 void gk104_grctx_generate_unkn(struct gf100_gr *);
-void gk104_grctx_generate_r418bb8(struct gf100_gr *);
-
-void gm107_grctx_generate_bundle(struct gf100_grctx *);
-void gm107_grctx_generate_pagepool(struct gf100_grctx *);
-void gm107_grctx_generate_attrib(struct gf100_grctx *);
+void gk104_grctx_generate_r418800(struct gf100_gr *);
 
 extern const struct gf100_grctx_func gk110_grctx;
+void gk110_grctx_generate_r419eb0(struct gf100_gr *);
+
 extern const struct gf100_grctx_func gk110b_grctx;
 extern const struct gf100_grctx_func gk208_grctx;
 
@@ -95,22 +129,30 @@ extern const struct gf100_grctx_func gm107_grctx;
 void gm107_grctx_generate_bundle(struct gf100_grctx *);
 void gm107_grctx_generate_pagepool(struct gf100_grctx *);
 void gm107_grctx_generate_attrib(struct gf100_grctx *);
+void gm107_grctx_generate_sm_id(struct gf100_gr *, int, int, int);
 
 extern const struct gf100_grctx_func gm200_grctx;
-void gm200_grctx_generate_tpcid(struct gf100_gr *);
-void gm200_grctx_generate_405b60(struct gf100_gr *);
+void gm200_grctx_generate_dist_skip_table(struct gf100_gr *);
+void gm200_grctx_generate_r406500(struct gf100_gr *);
+void gm200_grctx_generate_tpc_mask(struct gf100_gr *);
+void gm200_grctx_generate_smid_config(struct gf100_gr *);
+void gm200_grctx_generate_r419a3c(struct gf100_gr *);
 
 extern const struct gf100_grctx_func gm20b_grctx;
 
 extern const struct gf100_grctx_func gp100_grctx;
-void gp100_grctx_generate_main(struct gf100_gr *, struct gf100_grctx *);
 void gp100_grctx_generate_pagepool(struct gf100_grctx *);
+void gp100_grctx_generate_smid_config(struct gf100_gr *);
 
 extern const struct gf100_grctx_func gp102_grctx;
 void gp102_grctx_generate_attrib(struct gf100_grctx *);
 
+extern const struct gf100_grctx_func gp104_grctx;
+
 extern const struct gf100_grctx_func gp107_grctx;
 
+extern const struct gf100_grctx_func gv100_grctx;
+
 /* context init value lists */
 
 extern const struct gf100_gr_pack gf100_grctx_pack_icmd[];
@@ -128,7 +170,8 @@ extern const struct gf100_gr_init gf100_grctx_init_memfmt_0[];
 extern const struct gf100_gr_init gf100_grctx_init_rstr2d_0[];
 extern const struct gf100_gr_init gf100_grctx_init_scc_0[];
 
-extern const struct gf100_gr_pack gf100_grctx_pack_gpc[];
+extern const struct gf100_gr_pack gf100_grctx_pack_gpc_0[];
+extern const struct gf100_gr_pack gf100_grctx_pack_gpc_1[];
 extern const struct gf100_gr_init gf100_grctx_init_gpc_unk_0[];
 extern const struct gf100_gr_init gf100_grctx_init_prop_0[];
 extern const struct gf100_gr_init gf100_grctx_init_gpc_unk_1[];
@@ -177,6 +220,8 @@ extern const struct gf100_gr_init gf117_grctx_init_pe_0[];
 
 extern const struct gf100_gr_init gf117_grctx_init_wwdx_0[];
 
+extern const struct gf100_gr_pack gf117_grctx_pack_gpc_1[];
+
 extern const struct gf100_gr_init gk104_grctx_init_memfmt_0[];
 extern const struct gf100_gr_init gk104_grctx_init_ds_0[];
 extern const struct gf100_gr_init gk104_grctx_init_scc_0[];
@@ -186,7 +231,6 @@ extern const struct gf100_gr_init gk104_grctx_init_gpm_0[];
 extern const struct gf100_gr_init gk104_grctx_init_pes_0[];
 
 extern const struct gf100_gr_pack gk104_grctx_pack_hub[];
-extern const struct gf100_gr_pack gk104_grctx_pack_gpc[];
 extern const struct gf100_gr_pack gk104_grctx_pack_tpc[];
 extern const struct gf100_gr_pack gk104_grctx_pack_ppc[];
 extern const struct gf100_gr_pack gk104_grctx_pack_icmd[];
@@ -200,7 +244,8 @@ extern const struct gf100_gr_pack gk110_grctx_pack_hub[];
 extern const struct gf100_gr_init gk110_grctx_init_pri_0[];
 extern const struct gf100_gr_init gk110_grctx_init_cwd_0[];
 
-extern const struct gf100_gr_pack gk110_grctx_pack_gpc[];
+extern const struct gf100_gr_pack gk110_grctx_pack_gpc_0[];
+extern const struct gf100_gr_pack gk110_grctx_pack_gpc_1[];
 extern const struct gf100_gr_init gk110_grctx_init_gpc_unk_2[];
 
 extern const struct gf100_gr_init gk110_grctx_init_tex_0[];
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf104.c
index 54fd74e9cca0..7a0564b6e3c7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf104.c
@@ -84,7 +84,8 @@ gf104_grctx = {
 	.main  = gf100_grctx_generate_main,
 	.unkn  = gf100_grctx_generate_unkn,
 	.hub   = gf100_grctx_pack_hub,
-	.gpc   = gf100_grctx_pack_gpc,
+	.gpc_0 = gf100_grctx_pack_gpc_0,
+	.gpc_1 = gf100_grctx_pack_gpc_1,
 	.zcull = gf100_grctx_pack_zcull,
 	.tpc   = gf104_grctx_pack_tpc,
 	.icmd  = gf100_grctx_pack_icmd,
@@ -96,4 +97,11 @@ gf104_grctx = {
 	.attrib = gf100_grctx_generate_attrib,
 	.attrib_nr_max = 0x324,
 	.attrib_nr = 0x218,
+	.sm_id = gf100_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.r4060a8 = gf100_grctx_generate_r4060a8,
+	.rop_mapping = gf100_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gf100_grctx_generate_alpha_beta_tables,
+	.max_ways_evict = gf100_grctx_generate_max_ways_evict,
+	.r419cb8 = gf100_grctx_generate_r419cb8,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c
index 82f71b10c06e..dda2c32e6232 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c
@@ -667,12 +667,17 @@ gf108_grctx_init_gpm_0[] = {
 };
 
 static const struct gf100_gr_pack
-gf108_grctx_pack_gpc[] = {
+gf108_grctx_pack_gpc_0[] = {
 	{ gf100_grctx_init_gpc_unk_0 },
 	{ gf100_grctx_init_prop_0 },
 	{ gf100_grctx_init_gpc_unk_1 },
 	{ gf108_grctx_init_setup_0 },
 	{ gf100_grctx_init_zcull_0 },
+	{}
+};
+
+static const struct gf100_gr_pack
+gf108_grctx_pack_gpc_1[] = {
 	{ gf100_grctx_init_crstr_0 },
 	{ gf108_grctx_init_gpm_0 },
 	{ gf100_grctx_init_gcc_0 },
@@ -780,7 +785,8 @@ gf108_grctx = {
 	.main  = gf100_grctx_generate_main,
 	.unkn  = gf108_grctx_generate_unkn,
 	.hub   = gf108_grctx_pack_hub,
-	.gpc   = gf108_grctx_pack_gpc,
+	.gpc_0 = gf108_grctx_pack_gpc_0,
+	.gpc_1 = gf108_grctx_pack_gpc_1,
 	.zcull = gf100_grctx_pack_zcull,
 	.tpc   = gf108_grctx_pack_tpc,
 	.icmd  = gf108_grctx_pack_icmd,
@@ -794,4 +800,11 @@ gf108_grctx = {
 	.attrib_nr = 0x218,
 	.alpha_nr_max = 0x324,
 	.alpha_nr = 0x218,
+	.sm_id = gf100_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.r4060a8 = gf100_grctx_generate_r4060a8,
+	.rop_mapping = gf100_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gf100_grctx_generate_alpha_beta_tables,
+	.max_ways_evict = gf100_grctx_generate_max_ways_evict,
+	.r419cb8 = gf100_grctx_generate_r419cb8,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf110.c
index 7df398b53f8f..f5cca5e6a4f2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf110.c
@@ -314,15 +314,12 @@ gf110_grctx_init_setup_0[] = {
 };
 
 static const struct gf100_gr_pack
-gf110_grctx_pack_gpc[] = {
+gf110_grctx_pack_gpc_0[] = {
 	{ gf100_grctx_init_gpc_unk_0 },
 	{ gf100_grctx_init_prop_0 },
 	{ gf100_grctx_init_gpc_unk_1 },
 	{ gf110_grctx_init_setup_0 },
 	{ gf100_grctx_init_zcull_0 },
-	{ gf100_grctx_init_crstr_0 },
-	{ gf100_grctx_init_gpm_0 },
-	{ gf100_grctx_init_gcc_0 },
 	{}
 };
 
@@ -335,7 +332,8 @@ gf110_grctx = {
 	.main  = gf100_grctx_generate_main,
 	.unkn  = gf100_grctx_generate_unkn,
 	.hub   = gf100_grctx_pack_hub,
-	.gpc   = gf110_grctx_pack_gpc,
+	.gpc_0 = gf110_grctx_pack_gpc_0,
+	.gpc_1 = gf100_grctx_pack_gpc_1,
 	.zcull = gf100_grctx_pack_zcull,
 	.tpc   = gf100_grctx_pack_tpc,
 	.icmd  = gf110_grctx_pack_icmd,
@@ -347,4 +345,11 @@ gf110_grctx = {
 	.attrib = gf100_grctx_generate_attrib,
 	.attrib_nr_max = 0x324,
 	.attrib_nr = 0x218,
+	.sm_id = gf100_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.r4060a8 = gf100_grctx_generate_r4060a8,
+	.rop_mapping = gf100_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gf100_grctx_generate_alpha_beta_tables,
+	.max_ways_evict = gf100_grctx_generate_max_ways_evict,
+	.r419cb8 = gf100_grctx_generate_r419cb8,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
index 19301d88577d..276c282d19aa 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
@@ -84,12 +84,17 @@ gf117_grctx_init_setup_0[] = {
 };
 
 static const struct gf100_gr_pack
-gf117_grctx_pack_gpc[] = {
+gf117_grctx_pack_gpc_0[] = {
 	{ gf100_grctx_init_gpc_unk_0 },
 	{ gf119_grctx_init_prop_0 },
 	{ gf119_grctx_init_gpc_unk_1 },
 	{ gf117_grctx_init_setup_0 },
 	{ gf100_grctx_init_zcull_0 },
+	{}
+};
+
+const struct gf100_gr_pack
+gf117_grctx_pack_gpc_1[] = {
 	{ gf119_grctx_init_crstr_0 },
 	{ gf108_grctx_init_gpm_0 },
 	{ gf100_grctx_init_gcc_0 },
@@ -179,6 +184,62 @@ gf117_grctx_pack_ppc[] = {
  * PGRAPH context implementation
  ******************************************************************************/
 
+void
+gf117_grctx_generate_dist_skip_table(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	int i;
+
+	for (i = 0; i < 8; i++)
+		nvkm_wr32(device, 0x4064d0 + (i * 0x04), 0x00000000);
+}
+
+void
+gf117_grctx_generate_rop_mapping(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	u32 data[6] = {}, data2[2] = {};
+	u8  shift, ntpcv;
+	int i;
+
+	/* Pack tile map into register format. */
+	for (i = 0; i < 32; i++)
+		data[i / 6] |= (gr->tile[i] & 0x07) << ((i % 6) * 5);
+
+	/* Magic. */
+	shift = 0;
+	ntpcv = gr->tpc_total;
+	while (!(ntpcv & (1 << 4))) {
+		ntpcv <<= 1;
+		shift++;
+	}
+
+	data2[0]  = (ntpcv << 16);
+	data2[0] |= (shift << 21);
+	data2[0] |= (((1 << (0 + 5)) % ntpcv) << 24);
+	for (i = 1; i < 7; i++)
+		data2[1] |= ((1 << (i + 5)) % ntpcv) << ((i - 1) * 5);
+
+	/* GPC_BROADCAST */
+	nvkm_wr32(device, 0x418bb8, (gr->tpc_total << 8) |
+				     gr->screen_tile_row_offset);
+	for (i = 0; i < 6; i++)
+		nvkm_wr32(device, 0x418b08 + (i * 4), data[i]);
+
+	/* GPC_BROADCAST.TP_BROADCAST */
+	nvkm_wr32(device, 0x41bfd0, (gr->tpc_total << 8) |
+				     gr->screen_tile_row_offset | data2[0]);
+	nvkm_wr32(device, 0x41bfe4, data2[1]);
+	for (i = 0; i < 6; i++)
+		nvkm_wr32(device, 0x41bf00 + (i * 4), data[i]);
+
+	/* UNK78xx */
+	nvkm_wr32(device, 0x4078bc, (gr->tpc_total << 8) |
+				     gr->screen_tile_row_offset);
+	for (i = 0; i < 6; i++)
+		nvkm_wr32(device, 0x40780c + (i * 4), data[i]);
+}
+
 void
 gf117_grctx_generate_attrib(struct gf100_grctx *info)
 {
@@ -217,50 +278,13 @@ gf117_grctx_generate_attrib(struct gf100_grctx *info)
 	}
 }
 
-static void
-gf117_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
-{
-	struct nvkm_device *device = gr->base.engine.subdev.device;
-	const struct gf100_grctx_func *grctx = gr->func->grctx;
-	u32 idle_timeout;
-	int i;
-
-	nvkm_mc_unk260(device, 0);
-
-	gf100_gr_mmio(gr, grctx->hub);
-	gf100_gr_mmio(gr, grctx->gpc);
-	gf100_gr_mmio(gr, grctx->zcull);
-	gf100_gr_mmio(gr, grctx->tpc);
-	gf100_gr_mmio(gr, grctx->ppc);
-
-	idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000);
-
-	grctx->bundle(info);
-	grctx->pagepool(info);
-	grctx->attrib(info);
-	grctx->unkn(gr);
-
-	gf100_grctx_generate_tpcid(gr);
-	gf100_grctx_generate_r406028(gr);
-	gf100_grctx_generate_r4060a8(gr);
-	gk104_grctx_generate_r418bb8(gr);
-	gf100_grctx_generate_r406800(gr);
-
-	for (i = 0; i < 8; i++)
-		nvkm_wr32(device, 0x4064d0 + (i * 0x04), 0x00000000);
-
-	gf100_gr_icmd(gr, grctx->icmd);
-	nvkm_wr32(device, 0x404154, idle_timeout);
-	gf100_gr_mthd(gr, grctx->mthd);
-	nvkm_mc_unk260(device, 1);
-}
-
 const struct gf100_grctx_func
 gf117_grctx = {
-	.main  = gf117_grctx_generate_main,
+	.main  = gf100_grctx_generate_main,
 	.unkn  = gk104_grctx_generate_unkn,
 	.hub   = gf117_grctx_pack_hub,
-	.gpc   = gf117_grctx_pack_gpc,
+	.gpc_0 = gf117_grctx_pack_gpc_0,
+	.gpc_1 = gf117_grctx_pack_gpc_1,
 	.zcull = gf100_grctx_pack_zcull,
 	.tpc   = gf117_grctx_pack_tpc,
 	.ppc   = gf117_grctx_pack_ppc,
@@ -275,4 +299,12 @@ gf117_grctx = {
 	.attrib_nr = 0x218,
 	.alpha_nr_max = 0x7ff,
 	.alpha_nr = 0x324,
+	.sm_id = gf100_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.r4060a8 = gf100_grctx_generate_r4060a8,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gf100_grctx_generate_alpha_beta_tables,
+	.max_ways_evict = gf100_grctx_generate_max_ways_evict,
+	.dist_skip_table = gf117_grctx_generate_dist_skip_table,
+	.r419cb8 = gf100_grctx_generate_r419cb8,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf119.c
index 605185b078be..0cfe46366af6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf119.c
@@ -431,15 +431,12 @@ gf119_grctx_init_crstr_0[] = {
 };
 
 static const struct gf100_gr_pack
-gf119_grctx_pack_gpc[] = {
+gf119_grctx_pack_gpc_0[] = {
 	{ gf100_grctx_init_gpc_unk_0 },
 	{ gf119_grctx_init_prop_0 },
 	{ gf119_grctx_init_gpc_unk_1 },
 	{ gf119_grctx_init_setup_0 },
 	{ gf100_grctx_init_zcull_0 },
-	{ gf119_grctx_init_crstr_0 },
-	{ gf108_grctx_init_gpm_0 },
-	{ gf100_grctx_init_gcc_0 },
 	{}
 };
 
@@ -503,7 +500,8 @@ gf119_grctx = {
 	.main  = gf100_grctx_generate_main,
 	.unkn  = gf108_grctx_generate_unkn,
 	.hub   = gf119_grctx_pack_hub,
-	.gpc   = gf119_grctx_pack_gpc,
+	.gpc_0 = gf119_grctx_pack_gpc_0,
+	.gpc_1 = gf117_grctx_pack_gpc_1,
 	.zcull = gf100_grctx_pack_zcull,
 	.tpc   = gf119_grctx_pack_tpc,
 	.icmd  = gf119_grctx_pack_icmd,
@@ -517,4 +515,11 @@ gf119_grctx = {
 	.attrib_nr = 0x218,
 	.alpha_nr_max = 0x324,
 	.alpha_nr = 0x218,
+	.sm_id = gf100_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.r4060a8 = gf100_grctx_generate_r4060a8,
+	.rop_mapping = gf100_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gf100_grctx_generate_alpha_beta_tables,
+	.max_ways_evict = gf100_grctx_generate_max_ways_evict,
+	.r419cb8 = gf100_grctx_generate_r419cb8,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c
index 825c8fd500bc..304e9d268bad 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c
@@ -739,13 +739,18 @@ gk104_grctx_init_gpm_0[] = {
 	{}
 };
 
-const struct gf100_gr_pack
-gk104_grctx_pack_gpc[] = {
+static const struct gf100_gr_pack
+gk104_grctx_pack_gpc_0[] = {
 	{ gf100_grctx_init_gpc_unk_0 },
 	{ gf119_grctx_init_prop_0 },
 	{ gf119_grctx_init_gpc_unk_1 },
 	{ gk104_grctx_init_setup_0 },
 	{ gf100_grctx_init_zcull_0 },
+	{}
+};
+
+static const struct gf100_gr_pack
+gk104_grctx_pack_gpc_1[] = {
 	{ gf119_grctx_init_crstr_0 },
 	{ gk104_grctx_init_gpm_0 },
 	{ gf100_grctx_init_gcc_0 },
@@ -840,6 +845,32 @@ gk104_grctx_pack_ppc[] = {
  * PGRAPH context implementation
  ******************************************************************************/
 
+void
+gk104_grctx_generate_r418800(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	/*XXX: Not real sure where to apply these, there doesn't seem
+	 *     to be any pattern to which chipsets it's done on.
+	 *
+	 *     Perhaps a VBIOS tweak?
+	 */
+	if (0) {
+		nvkm_mask(device, 0x418800, 0x00200000, 0x00200000);
+		nvkm_mask(device, 0x41be10, 0x00800000, 0x00800000);
+	}
+}
+
+void
+gk104_grctx_generate_patch_ltc(struct gf100_grctx *info)
+{
+	struct nvkm_device *device = info->gr->base.engine.subdev.device;
+	u32 data0 = nvkm_rd32(device, 0x17e91c);
+	u32 data1 = nvkm_rd32(device, 0x17e920);
+	/*XXX: Figure out how to modify this correctly! */
+	mmio_wr32(info, 0x17e91c, data0);
+	mmio_wr32(info, 0x17e920, data1);
+}
+
 void
 gk104_grctx_generate_bundle(struct gf100_grctx *info)
 {
@@ -881,114 +912,74 @@ gk104_grctx_generate_unkn(struct gf100_gr *gr)
 	nvkm_mask(device, 0x419c00, 0x00000008, 0x00000008);
 }
 
-void
-gk104_grctx_generate_r418bb8(struct gf100_gr *gr)
+static void
+gk104_grctx_generate_r419f78(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-	u32 data[6] = {}, data2[2] = {};
-	u8  tpcnr[GPC_MAX];
-	u8  shift, ntpcv;
-	int gpc, tpc, i;
-
-	/* calculate first set of magics */
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-
-	gpc = -1;
-	for (tpc = 0; tpc < gr->tpc_total; tpc++) {
-		do {
-			gpc = (gpc + 1) % gr->gpc_nr;
-		} while (!tpcnr[gpc]);
-		tpcnr[gpc]--;
-
-		data[tpc / 6] |= gpc << ((tpc % 6) * 5);
-	}
-
-	for (; tpc < 32; tpc++)
-		data[tpc / 6] |= 7 << ((tpc % 6) * 5);
-
-	/* and the second... */
-	shift = 0;
-	ntpcv = gr->tpc_total;
-	while (!(ntpcv & (1 << 4))) {
-		ntpcv <<= 1;
-		shift++;
-	}
-
-	data2[0]  = (ntpcv << 16);
-	data2[0] |= (shift << 21);
-	data2[0] |= (((1 << (0 + 5)) % ntpcv) << 24);
-	for (i = 1; i < 7; i++)
-		data2[1] |= ((1 << (i + 5)) % ntpcv) << ((i - 1) * 5);
-
-	/* GPC_BROADCAST */
-	nvkm_wr32(device, 0x418bb8, (gr->tpc_total << 8) |
-				     gr->screen_tile_row_offset);
-	for (i = 0; i < 6; i++)
-		nvkm_wr32(device, 0x418b08 + (i * 4), data[i]);
-
-	/* GPC_BROADCAST.TP_BROADCAST */
-	nvkm_wr32(device, 0x41bfd0, (gr->tpc_total << 8) |
-				     gr->screen_tile_row_offset | data2[0]);
-	nvkm_wr32(device, 0x41bfe4, data2[1]);
-	for (i = 0; i < 6; i++)
-		nvkm_wr32(device, 0x41bf00 + (i * 4), data[i]);
-
-	/* UNK78xx */
-	nvkm_wr32(device, 0x4078bc, (gr->tpc_total << 8) |
-				     gr->screen_tile_row_offset);
-	for (i = 0; i < 6; i++)
-		nvkm_wr32(device, 0x40780c + (i * 4), data[i]);
+	nvkm_mask(device, 0x419f78, 0x00000001, 0x00000000);
 }
 
 void
-gk104_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
+gk104_grctx_generate_gpc_tpc_nr(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-	const struct gf100_grctx_func *grctx = gr->func->grctx;
-	u32 idle_timeout;
-	int i;
-
-	nvkm_mc_unk260(device, 0);
-
-	gf100_gr_mmio(gr, grctx->hub);
-	gf100_gr_mmio(gr, grctx->gpc);
-	gf100_gr_mmio(gr, grctx->zcull);
-	gf100_gr_mmio(gr, grctx->tpc);
-	gf100_gr_mmio(gr, grctx->ppc);
-
-	idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000);
-
-	grctx->bundle(info);
-	grctx->pagepool(info);
-	grctx->attrib(info);
-	grctx->unkn(gr);
-
-	gf100_grctx_generate_tpcid(gr);
-	gf100_grctx_generate_r406028(gr);
-	gk104_grctx_generate_r418bb8(gr);
-	gf100_grctx_generate_r406800(gr);
-
-	for (i = 0; i < 8; i++)
-		nvkm_wr32(device, 0x4064d0 + (i * 0x04), 0x00000000);
-
 	nvkm_wr32(device, 0x405b00, (gr->tpc_total << 8) | gr->gpc_nr);
-	nvkm_mask(device, 0x419f78, 0x00000001, 0x00000000);
-
-	gf100_gr_icmd(gr, grctx->icmd);
-	nvkm_wr32(device, 0x404154, idle_timeout);
-	gf100_gr_mthd(gr, grctx->mthd);
-	nvkm_mc_unk260(device, 1);
+}
 
-	nvkm_mask(device, 0x418800, 0x00200000, 0x00200000);
-	nvkm_mask(device, 0x41be10, 0x00800000, 0x00800000);
+void
+gk104_grctx_generate_alpha_beta_tables(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	int i, j, gpc, ppc;
+
+	for (i = 0; i < 32; i++) {
+		u32 atarget = max_t(u32, gr->tpc_total * i / 32, 1);
+		u32 btarget = gr->tpc_total - atarget;
+		bool alpha = atarget < btarget;
+		u64 amask = 0, bmask = 0;
+
+		for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
+			for (ppc = 0; ppc < gr->func->ppc_nr; ppc++) {
+				u32 ppc_tpcs = gr->ppc_tpc_nr[gpc][ppc];
+				u32 abits, bbits, pmask;
+
+				if (alpha) {
+					abits = atarget ? ppc_tpcs : 0;
+					bbits = ppc_tpcs - abits;
+				} else {
+					bbits = btarget ? ppc_tpcs : 0;
+					abits = ppc_tpcs - bbits;
+				}
+
+				pmask = gr->ppc_tpc_mask[gpc][ppc];
+				while (ppc_tpcs-- > abits)
+					pmask &= pmask - 1;
+				amask |= (u64)pmask << (gpc * 8);
+
+				pmask ^= gr->ppc_tpc_mask[gpc][ppc];
+				bmask |= (u64)pmask << (gpc * 8);
+
+				atarget -= min(abits, atarget);
+				btarget -= min(bbits, btarget);
+				if ((abits > 0) || (bbits > 0))
+					alpha = !alpha;
+			}
+		}
+
+		for (j = 0; j < gr->gpc_nr; j += 4, amask >>= 32, bmask >>= 32) {
+			nvkm_wr32(device, 0x406800 + (i * 0x20) + j, amask);
+			nvkm_wr32(device, 0x406c00 + (i * 0x20) + j, bmask);
+		}
+	}
 }
 
 const struct gf100_grctx_func
 gk104_grctx = {
-	.main  = gk104_grctx_generate_main,
+	.main  = gf100_grctx_generate_main,
 	.unkn  = gk104_grctx_generate_unkn,
 	.hub   = gk104_grctx_pack_hub,
-	.gpc   = gk104_grctx_pack_gpc,
+	.gpc_0 = gk104_grctx_pack_gpc_0,
+	.gpc_1 = gk104_grctx_pack_gpc_1,
 	.zcull = gf100_grctx_pack_zcull,
 	.tpc   = gk104_grctx_pack_tpc,
 	.ppc   = gk104_grctx_pack_ppc,
@@ -1005,4 +996,13 @@ gk104_grctx = {
 	.attrib_nr = 0x218,
 	.alpha_nr_max = 0x7ff,
 	.alpha_nr = 0x648,
+	.patch_ltc = gk104_grctx_generate_patch_ltc,
+	.sm_id = gf100_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gk104_grctx_generate_alpha_beta_tables,
+	.dist_skip_table = gf117_grctx_generate_dist_skip_table,
+	.gpc_tpc_nr = gk104_grctx_generate_gpc_tpc_nr,
+	.r419f78 = gk104_grctx_generate_r419f78,
+	.r418800 = gk104_grctx_generate_r418800,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110.c
index 7b95ec2fe453..86547cfc38dc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110.c
@@ -704,12 +704,17 @@ gk110_grctx_init_gpc_unk_2[] = {
 };
 
 const struct gf100_gr_pack
-gk110_grctx_pack_gpc[] = {
+gk110_grctx_pack_gpc_0[] = {
 	{ gf100_grctx_init_gpc_unk_0 },
 	{ gf119_grctx_init_prop_0 },
 	{ gf119_grctx_init_gpc_unk_1 },
 	{ gk110_grctx_init_setup_0 },
 	{ gf100_grctx_init_zcull_0 },
+	{}
+};
+
+const struct gf100_gr_pack
+gk110_grctx_pack_gpc_1[] = {
 	{ gf119_grctx_init_crstr_0 },
 	{ gk104_grctx_init_gpm_0 },
 	{ gk110_grctx_init_gpc_unk_2 },
@@ -808,12 +813,20 @@ gk110_grctx_pack_ppc[] = {
  * PGRAPH context implementation
  ******************************************************************************/
 
+void
+gk110_grctx_generate_r419eb0(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, 0x419eb0, 0x00001000, 0x00001000);
+}
+
 const struct gf100_grctx_func
 gk110_grctx = {
-	.main  = gk104_grctx_generate_main,
+	.main  = gf100_grctx_generate_main,
 	.unkn  = gk104_grctx_generate_unkn,
 	.hub   = gk110_grctx_pack_hub,
-	.gpc   = gk110_grctx_pack_gpc,
+	.gpc_0 = gk110_grctx_pack_gpc_0,
+	.gpc_1 = gk110_grctx_pack_gpc_1,
 	.zcull = gf100_grctx_pack_zcull,
 	.tpc   = gk110_grctx_pack_tpc,
 	.ppc   = gk110_grctx_pack_ppc,
@@ -830,4 +843,13 @@ gk110_grctx = {
 	.attrib_nr = 0x218,
 	.alpha_nr_max = 0x7ff,
 	.alpha_nr = 0x648,
+	.patch_ltc = gk104_grctx_generate_patch_ltc,
+	.sm_id = gf100_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gk104_grctx_generate_alpha_beta_tables,
+	.dist_skip_table = gf117_grctx_generate_dist_skip_table,
+	.gpc_tpc_nr = gk104_grctx_generate_gpc_tpc_nr,
+	.r418800 = gk104_grctx_generate_r418800,
+	.r419eb0 = gk110_grctx_generate_r419eb0,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110b.c
index 048b1152da44..ebb947bd1446 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110b.c
@@ -71,10 +71,11 @@ gk110b_grctx_pack_tpc[] = {
 
 const struct gf100_grctx_func
 gk110b_grctx = {
-	.main  = gk104_grctx_generate_main,
+	.main  = gf100_grctx_generate_main,
 	.unkn  = gk104_grctx_generate_unkn,
 	.hub   = gk110_grctx_pack_hub,
-	.gpc   = gk110_grctx_pack_gpc,
+	.gpc_0 = gk110_grctx_pack_gpc_0,
+	.gpc_1 = gk110_grctx_pack_gpc_1,
 	.zcull = gf100_grctx_pack_zcull,
 	.tpc   = gk110b_grctx_pack_tpc,
 	.ppc   = gk110_grctx_pack_ppc,
@@ -91,4 +92,13 @@ gk110b_grctx = {
 	.attrib_nr = 0x218,
 	.alpha_nr_max = 0x7ff,
 	.alpha_nr = 0x648,
+	.patch_ltc = gk104_grctx_generate_patch_ltc,
+	.sm_id = gf100_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gk104_grctx_generate_alpha_beta_tables,
+	.dist_skip_table = gf117_grctx_generate_dist_skip_table,
+	.gpc_tpc_nr = gk104_grctx_generate_gpc_tpc_nr,
+	.r418800 = gk104_grctx_generate_r418800,
+	.r419eb0 = gk110_grctx_generate_r419eb0,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk208.c
index 67b7a1b43617..4d40512b5c99 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk208.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk208.c
@@ -443,12 +443,17 @@ gk208_grctx_init_gpm_0[] = {
 };
 
 static const struct gf100_gr_pack
-gk208_grctx_pack_gpc[] = {
+gk208_grctx_pack_gpc_0[] = {
 	{ gf100_grctx_init_gpc_unk_0 },
 	{ gk208_grctx_init_prop_0 },
 	{ gk208_grctx_init_gpc_unk_1 },
 	{ gk208_grctx_init_setup_0 },
 	{ gf100_grctx_init_zcull_0 },
+	{}
+};
+
+static const struct gf100_gr_pack
+gk208_grctx_pack_gpc_1[] = {
 	{ gk208_grctx_init_crstr_0 },
 	{ gk208_grctx_init_gpm_0 },
 	{ gk110_grctx_init_gpc_unk_2 },
@@ -532,10 +537,11 @@ gk208_grctx_pack_ppc[] = {
 
 const struct gf100_grctx_func
 gk208_grctx = {
-	.main  = gk104_grctx_generate_main,
+	.main  = gf100_grctx_generate_main,
 	.unkn  = gk104_grctx_generate_unkn,
 	.hub   = gk208_grctx_pack_hub,
-	.gpc   = gk208_grctx_pack_gpc,
+	.gpc_0 = gk208_grctx_pack_gpc_0,
+	.gpc_1 = gk208_grctx_pack_gpc_1,
 	.zcull = gf100_grctx_pack_zcull,
 	.tpc   = gk208_grctx_pack_tpc,
 	.ppc   = gk208_grctx_pack_ppc,
@@ -552,4 +558,12 @@ gk208_grctx = {
 	.attrib_nr = 0x218,
 	.alpha_nr_max = 0x7ff,
 	.alpha_nr = 0x648,
+	.patch_ltc = gk104_grctx_generate_patch_ltc,
+	.sm_id = gf100_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gk104_grctx_generate_alpha_beta_tables,
+	.dist_skip_table = gf117_grctx_generate_dist_skip_table,
+	.gpc_tpc_nr = gk104_grctx_generate_gpc_tpc_nr,
+	.r418800 = gk104_grctx_generate_r418800,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c
index da7c35a6a3d2..896d473dcc0f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c
@@ -42,10 +42,7 @@ gk20a_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
 
 	grctx->unkn(gr);
 
-	gf100_grctx_generate_tpcid(gr);
-	gf100_grctx_generate_r406028(gr);
-	gk104_grctx_generate_r418bb8(gr);
-	gf100_grctx_generate_r406800(gr);
+	gf100_grctx_generate_floorsweep(gr);
 
 	for (i = 0; i < 8; i++)
 		nvkm_wr32(device, 0x4064d0 + (i * 0x04), 0x00000000);
@@ -82,4 +79,8 @@ gk20a_grctx = {
 	.attrib_nr = 0x240,
 	.alpha_nr_max = 0x648 + (0x648 / 2),
 	.alpha_nr = 0x648,
+	.sm_id = gf100_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gk104_grctx_generate_alpha_beta_tables,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c
index 9b43d4ce3eaa..0b3964e6b36e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c
@@ -744,12 +744,17 @@ gm107_grctx_init_gpc_unk_2[] = {
 };
 
 static const struct gf100_gr_pack
-gm107_grctx_pack_gpc[] = {
+gm107_grctx_pack_gpc_0[] = {
 	{ gm107_grctx_init_gpc_unk_0 },
 	{ gk208_grctx_init_prop_0 },
 	{ gm107_grctx_init_gpc_unk_1 },
 	{ gm107_grctx_init_setup_0 },
 	{ gf100_grctx_init_zcull_0 },
+	{}
+};
+
+static const struct gf100_gr_pack
+gm107_grctx_pack_gpc_1[] = {
 	{ gk208_grctx_init_crstr_0 },
 	{ gk104_grctx_init_gpm_0 },
 	{ gm107_grctx_init_gpc_unk_2 },
@@ -860,6 +865,16 @@ gm107_grctx_pack_ppc[] = {
  * PGRAPH context implementation
  ******************************************************************************/
 
+static void
+gm107_grctx_generate_r419e00(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, 0x419e00, 0x00808080, 0x00808080);
+	nvkm_mask(device, 0x419ccc, 0x80000000, 0x80000000);
+	nvkm_mask(device, 0x419f80, 0x80000000, 0x80000000);
+	nvkm_mask(device, 0x419f88, 0x80000000, 0x80000000);
+}
+
 void
 gm107_grctx_generate_bundle(struct gf100_grctx *info)
 {
@@ -931,75 +946,27 @@ gm107_grctx_generate_attrib(struct gf100_grctx *info)
 }
 
 static void
-gm107_grctx_generate_tpcid(struct gf100_gr *gr)
+gm107_grctx_generate_r406500(struct gf100_gr *gr)
 {
-	struct nvkm_device *device = gr->base.engine.subdev.device;
-	int gpc, tpc, id;
-
-	for (tpc = 0, id = 0; tpc < 4; tpc++) {
-		for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-			if (tpc < gr->tpc_nr[gpc]) {
-				nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x698), id);
-				nvkm_wr32(device, GPC_UNIT(gpc, 0x0c10 + tpc * 4), id);
-				nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x088), id);
-				id++;
-			}
-
-			nvkm_wr32(device, GPC_UNIT(gpc, 0x0c08), gr->tpc_nr[gpc]);
-			nvkm_wr32(device, GPC_UNIT(gpc, 0x0c8c), gr->tpc_nr[gpc]);
-		}
-	}
+	nvkm_wr32(gr->base.engine.subdev.device, 0x406500, 0x00000001);
 }
 
-static void
-gm107_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
+void
+gm107_grctx_generate_sm_id(struct gf100_gr *gr, int gpc, int tpc, int sm)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-	const struct gf100_grctx_func *grctx = gr->func->grctx;
-	u32 idle_timeout;
-	int i;
-
-	gf100_gr_mmio(gr, grctx->hub);
-	gf100_gr_mmio(gr, grctx->gpc);
-	gf100_gr_mmio(gr, grctx->zcull);
-	gf100_gr_mmio(gr, grctx->tpc);
-	gf100_gr_mmio(gr, grctx->ppc);
-
-	idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000);
-
-	grctx->bundle(info);
-	grctx->pagepool(info);
-	grctx->attrib(info);
-	grctx->unkn(gr);
-
-	gm107_grctx_generate_tpcid(gr);
-	gf100_grctx_generate_r406028(gr);
-	gk104_grctx_generate_r418bb8(gr);
-	gf100_grctx_generate_r406800(gr);
-
-	nvkm_wr32(device, 0x4064d0, 0x00000001);
-	for (i = 1; i < 8; i++)
-		nvkm_wr32(device, 0x4064d0 + (i * 0x04), 0x00000000);
-	nvkm_wr32(device, 0x406500, 0x00000001);
-
-	nvkm_wr32(device, 0x405b00, (gr->tpc_total << 8) | gr->gpc_nr);
-
-	gf100_gr_icmd(gr, grctx->icmd);
-	nvkm_wr32(device, 0x404154, idle_timeout);
-	gf100_gr_mthd(gr, grctx->mthd);
-
-	nvkm_mask(device, 0x419e00, 0x00808080, 0x00808080);
-	nvkm_mask(device, 0x419ccc, 0x80000000, 0x80000000);
-	nvkm_mask(device, 0x419f80, 0x80000000, 0x80000000);
-	nvkm_mask(device, 0x419f88, 0x80000000, 0x80000000);
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x698), sm);
+	nvkm_wr32(device, GPC_UNIT(gpc, 0x0c10 + tpc * 4), sm);
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x088), sm);
 }
 
 const struct gf100_grctx_func
 gm107_grctx = {
-	.main  = gm107_grctx_generate_main,
+	.main  = gf100_grctx_generate_main,
 	.unkn  = gk104_grctx_generate_unkn,
 	.hub   = gm107_grctx_pack_hub,
-	.gpc   = gm107_grctx_pack_gpc,
+	.gpc_0 = gm107_grctx_pack_gpc_0,
+	.gpc_1 = gm107_grctx_pack_gpc_1,
 	.zcull = gf100_grctx_pack_zcull,
 	.tpc   = gm107_grctx_pack_tpc,
 	.ppc   = gm107_grctx_pack_ppc,
@@ -1016,4 +983,12 @@ gm107_grctx = {
 	.attrib_nr = 0xaa0,
 	.alpha_nr_max = 0x1800,
 	.alpha_nr = 0x1000,
+	.sm_id = gm107_grctx_generate_sm_id,
+	.tpc_nr = gf100_grctx_generate_tpc_nr,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.alpha_beta_tables = gk104_grctx_generate_alpha_beta_tables,
+	.dist_skip_table = gf117_grctx_generate_dist_skip_table,
+	.r406500 = gm107_grctx_generate_r406500,
+	.gpc_tpc_nr = gk104_grctx_generate_gpc_tpc_nr,
+	.r419e00 = gm107_grctx_generate_r419e00,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c
index db209d33f486..013d05a0f0f6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c
@@ -28,47 +28,34 @@
  ******************************************************************************/
 
 void
-gm200_grctx_generate_tpcid(struct gf100_gr *gr)
+gm200_grctx_generate_r419a3c(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-	int gpc, tpc, id;
+	nvkm_mask(device, 0x419a3c, 0x00000014, 0x00000000);
+}
 
-	for (tpc = 0, id = 0; tpc < TPC_MAX_PER_GPC; tpc++) {
-		for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-			if (tpc < gr->tpc_nr[gpc]) {
-				nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x698), id);
-				nvkm_wr32(device, GPC_UNIT(gpc, 0x0c10 + tpc * 4), id);
-				nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x088), id);
-				id++;
-			}
-		}
-	}
+static void
+gm200_grctx_generate_r418e94(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, 0x418e94, 0xffffffff, 0xc4230000);
+	nvkm_mask(device, 0x418e4c, 0xffffffff, 0x70000000);
 }
 
 void
-gm200_grctx_generate_405b60(struct gf100_gr *gr)
+gm200_grctx_generate_smid_config(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
 	const u32 dist_nr = DIV_ROUND_UP(gr->tpc_total, 4);
 	u32 dist[TPC_MAX / 4] = {};
 	u32 gpcs[GPC_MAX] = {};
-	u8  tpcnr[GPC_MAX];
-	int tpc, gpc, i;
+	u8  sm, i;
 
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-
-	/* won't result in the same distribution as the binary driver where
-	 * some of the gpcs have more tpcs than others, but this shall do
-	 * for the moment.  the code for earlier gpus has this issue too.
-	 */
-	for (gpc = -1, i = 0; i < gr->tpc_total; i++) {
-		do {
-			gpc = (gpc + 1) % gr->gpc_nr;
-		} while(!tpcnr[gpc]);
-		tpc = gr->tpc_nr[gpc] - tpcnr[gpc]--;
-
-		dist[i / 4] |= ((gpc << 4) | tpc) << ((i % 4) * 8);
-		gpcs[gpc] |= i << (tpc * 8);
+	for (sm = 0; sm < gr->sm_nr; sm++) {
+		const u8 gpc = gr->sm[sm].gpc;
+		const u8 tpc = gr->sm[sm].tpc;
+		dist[sm / 4] |= ((gpc << 4) | tpc) << ((sm % 4) * 8);
+		gpcs[gpc] |= sm << (tpc * 8);
 	}
 
 	for (i = 0; i < dist_nr; i++)
@@ -77,50 +64,46 @@ gm200_grctx_generate_405b60(struct gf100_gr *gr)
 		nvkm_wr32(device, 0x405ba0 + (i * 4), gpcs[i]);
 }
 
-static void
-gm200_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
+void
+gm200_grctx_generate_tpc_mask(struct gf100_gr *gr)
 {
-	struct nvkm_device *device = gr->base.engine.subdev.device;
-	const struct gf100_grctx_func *grctx = gr->func->grctx;
-	u32 idle_timeout, tmp;
-	int i;
-
-	gf100_gr_mmio(gr, gr->fuc_sw_ctx);
-
-	idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000);
-
-	grctx->bundle(info);
-	grctx->pagepool(info);
-	grctx->attrib(info);
-	grctx->unkn(gr);
-
-	gm200_grctx_generate_tpcid(gr);
-	gf100_grctx_generate_r406028(gr);
-	gk104_grctx_generate_r418bb8(gr);
-
-	for (i = 0; i < 8; i++)
-		nvkm_wr32(device, 0x4064d0 + (i * 0x04), 0x00000000);
-	nvkm_wr32(device, 0x406500, 0x00000000);
-
-	nvkm_wr32(device, 0x405b00, (gr->tpc_total << 8) | gr->gpc_nr);
-
+	u32 tmp, i;
 	for (tmp = 0, i = 0; i < gr->gpc_nr; i++)
-		tmp |= ((1 << gr->tpc_nr[i]) - 1) << (i * 4);
-	nvkm_wr32(device, 0x4041c4, tmp);
+		tmp |= ((1 << gr->tpc_nr[i]) - 1) << (i * gr->func->tpc_nr);
+	nvkm_wr32(gr->base.engine.subdev.device, 0x4041c4, tmp);
+}
 
-	gm200_grctx_generate_405b60(gr);
+void
+gm200_grctx_generate_r406500(struct gf100_gr *gr)
+{
+	nvkm_wr32(gr->base.engine.subdev.device, 0x406500, 0x00000000);
+}
 
-	gf100_gr_icmd(gr, gr->fuc_bundle);
-	nvkm_wr32(device, 0x404154, idle_timeout);
-	gf100_gr_mthd(gr, gr->fuc_method);
+void
+gm200_grctx_generate_dist_skip_table(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	u32 data[8] = {};
+	int gpc, ppc, i;
+
+	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
+		for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++) {
+			u8 ppc_tpcs = gr->ppc_tpc_nr[gpc][ppc];
+			u8 ppc_tpcm = gr->ppc_tpc_mask[gpc][ppc];
+			while (ppc_tpcs-- > gr->ppc_tpc_min)
+				ppc_tpcm &= ppc_tpcm - 1;
+			ppc_tpcm ^= gr->ppc_tpc_mask[gpc][ppc];
+			((u8 *)data)[gpc] |= ppc_tpcm;
+		}
+	}
 
-	nvkm_mask(device, 0x418e94, 0xffffffff, 0xc4230000);
-	nvkm_mask(device, 0x418e4c, 0xffffffff, 0x70000000);
+	for (i = 0; i < ARRAY_SIZE(data); i++)
+		nvkm_wr32(device, 0x4064d0 + (i * 0x04), data[i]);
 }
 
 const struct gf100_grctx_func
 gm200_grctx = {
-	.main  = gm200_grctx_generate_main,
+	.main  = gf100_grctx_generate_main,
 	.unkn  = gk104_grctx_generate_unkn,
 	.bundle = gm107_grctx_generate_bundle,
 	.bundle_size = 0x3000,
@@ -133,4 +116,13 @@ gm200_grctx = {
 	.attrib_nr = 0x400,
 	.alpha_nr_max = 0x1800,
 	.alpha_nr = 0x1000,
+	.sm_id = gm107_grctx_generate_sm_id,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.dist_skip_table = gm200_grctx_generate_dist_skip_table,
+	.r406500 = gm200_grctx_generate_r406500,
+	.gpc_tpc_nr = gk104_grctx_generate_gpc_tpc_nr,
+	.tpc_mask = gm200_grctx_generate_tpc_mask,
+	.smid_config = gm200_grctx_generate_smid_config,
+	.r418e94 = gm200_grctx_generate_r418e94,
+	.r419a3c = gm200_grctx_generate_r419a3c,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c
index e5702e3e0a5a..a1d9e114ebeb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c
@@ -21,20 +21,6 @@
  */
 #include "ctxgf100.h"
 
-static void
-gm20b_grctx_generate_r406028(struct gf100_gr *gr)
-{
-	struct nvkm_device *device = gr->base.engine.subdev.device;
-	u32 tpc_per_gpc = 0;
-	int i;
-
-	for (i = 0; i < gr->gpc_nr; i++)
-		tpc_per_gpc |= gr->tpc_nr[i] << (4 * i);
-
-	nvkm_wr32(device, 0x406028, tpc_per_gpc);
-	nvkm_wr32(device, 0x405870, tpc_per_gpc);
-}
-
 static void
 gm20b_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
 {
@@ -53,9 +39,7 @@ gm20b_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
 
 	grctx->unkn(gr);
 
-	gm200_grctx_generate_tpcid(gr);
-	gm20b_grctx_generate_r406028(gr);
-	gk104_grctx_generate_r418bb8(gr);
+	gf100_grctx_generate_floorsweep(gr);
 
 	for (i = 0; i < 8; i++)
 		nvkm_wr32(device, 0x4064d0 + (i * 0x04), 0x00000000);
@@ -68,7 +52,7 @@ gm20b_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
 		tmp |= ((1 << gr->tpc_nr[i]) - 1) << (i * 4);
 	nvkm_wr32(device, 0x4041c4, tmp);
 
-	gm200_grctx_generate_405b60(gr);
+	gm200_grctx_generate_smid_config(gr);
 
 	gf100_gr_wait_idle(gr);
 
@@ -98,4 +82,6 @@ gm20b_grctx = {
 	.attrib_nr = 0x400,
 	.alpha_nr_max = 0xc00,
 	.alpha_nr = 0x800,
+	.sm_id = gm107_grctx_generate_sm_id,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c
index 88ea322d956c..0b3326262e12 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c
@@ -36,7 +36,7 @@ gp100_grctx_generate_pagepool(struct gf100_grctx *info)
 	const int s = 8;
 	const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true);
 	mmio_refn(info, 0x40800c, 0x00000000, s, b);
-	mmio_wr32(info, 0x408010, 0x80000000);
+	mmio_wr32(info, 0x408010, 0x8007d800);
 	mmio_refn(info, 0x419004, 0x00000000, s, b);
 	mmio_wr32(info, 0x419008, 0x00000000);
 }
@@ -48,14 +48,17 @@ gp100_grctx_generate_attrib(struct gf100_grctx *info)
 	const struct gf100_grctx_func *grctx = gr->func->grctx;
 	const u32  alpha = grctx->alpha_nr;
 	const u32 attrib = grctx->attrib_nr;
-	const u32 pertpc = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max);
-	const u32   size = roundup(gr->tpc_total * pertpc, 0x80);
 	const int s = 12;
-	const int b = mmio_vram(info, size, (1 << s), false);
 	const int max_batches = 0xffff;
+	u32 size = grctx->alpha_nr_max * gr->tpc_total;
 	u32 ao = 0;
-	u32 bo = ao + grctx->alpha_nr_max * gr->tpc_total;
-	int gpc, ppc, n = 0;
+	u32 bo = ao + size;
+	int gpc, ppc, b, n = 0;
+
+	for (gpc = 0; gpc < gr->gpc_nr; gpc++)
+		size += grctx->attrib_nr_max * gr->ppc_nr[gpc] * gr->ppc_tpc_max;
+	size = ((size * 0x20) + 128) & ~127;
+	b = mmio_vram(info, size, (1 << s), false);
 
 	mmio_refn(info, 0x418810, 0x80000000, s, b);
 	mmio_refn(info, 0x419848, 0x10000000, s, b);
@@ -69,7 +72,7 @@ gp100_grctx_generate_attrib(struct gf100_grctx *info)
 	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
 		for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) {
 			const u32 as =  alpha * gr->ppc_tpc_nr[gpc][ppc];
-			const u32 bs = attrib * gr->ppc_tpc_nr[gpc][ppc];
+			const u32 bs = attrib * gr->ppc_tpc_max;
 			const u32 u = 0x418ea0 + (n * 0x04);
 			const u32 o = PPC_UNIT(gpc, ppc, 0);
 			if (!(gr->ppc_mask[gpc] & (1 << ppc)))
@@ -77,7 +80,7 @@ gp100_grctx_generate_attrib(struct gf100_grctx *info)
 			mmio_wr32(info, o + 0xc0, bs);
 			mmio_wr32(info, o + 0xf4, bo);
 			mmio_wr32(info, o + 0xf0, bs);
-			bo += grctx->attrib_nr_max * gr->ppc_tpc_nr[gpc][ppc];
+			bo += grctx->attrib_nr_max * gr->ppc_tpc_max;
 			mmio_wr32(info, o + 0xe4, as);
 			mmio_wr32(info, o + 0xf8, ao);
 			ao += grctx->alpha_nr_max * gr->ppc_tpc_nr[gpc][ppc];
@@ -89,79 +92,30 @@ gp100_grctx_generate_attrib(struct gf100_grctx *info)
 	mmio_wr32(info, 0x41befc, 0x00000000);
 }
 
-static void
-gp100_grctx_generate_405b60(struct gf100_gr *gr)
+void
+gp100_grctx_generate_smid_config(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
 	const u32 dist_nr = DIV_ROUND_UP(gr->tpc_total, 4);
-	u32 dist[TPC_MAX / 4] = {};
-	u32 gpcs[GPC_MAX * 2] = {};
-	u8  tpcnr[GPC_MAX];
-	int tpc, gpc, i;
-
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-
-	/* won't result in the same distribution as the binary driver where
-	 * some of the gpcs have more tpcs than others, but this shall do
-	 * for the moment.  the code for earlier gpus has this issue too.
-	 */
-	for (gpc = -1, i = 0; i < gr->tpc_total; i++) {
-		do {
-			gpc = (gpc + 1) % gr->gpc_nr;
-		} while(!tpcnr[gpc]);
-		tpc = gr->tpc_nr[gpc] - tpcnr[gpc]--;
-
-		dist[i / 4] |= ((gpc << 4) | tpc) << ((i % 4) * 8);
-		gpcs[gpc + (gr->gpc_nr * (tpc / 4))] |= i << (tpc * 8);
+	u32 dist[TPC_MAX / 4] = {}, gpcs[16] = {};
+	u8  sm, i;
+
+	for (sm = 0; sm < gr->sm_nr; sm++) {
+		const u8 gpc = gr->sm[sm].gpc;
+		const u8 tpc = gr->sm[sm].tpc;
+		dist[sm / 4] |= ((gpc << 4) | tpc) << ((sm % 4) * 8);
+		gpcs[gpc + (gr->func->gpc_nr * (tpc / 4))] |= sm << ((tpc % 4) * 8);
 	}
 
 	for (i = 0; i < dist_nr; i++)
 		nvkm_wr32(device, 0x405b60 + (i * 4), dist[i]);
-	for (i = 0; i < gr->gpc_nr * 2; i++)
+	for (i = 0; i < ARRAY_SIZE(gpcs); i++)
 		nvkm_wr32(device, 0x405ba0 + (i * 4), gpcs[i]);
 }
 
-void
-gp100_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
-{
-	struct nvkm_device *device = gr->base.engine.subdev.device;
-	const struct gf100_grctx_func *grctx = gr->func->grctx;
-	u32 idle_timeout, tmp;
-	int i;
-
-	gf100_gr_mmio(gr, gr->fuc_sw_ctx);
-
-	idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000);
-
-	grctx->pagepool(info);
-	grctx->bundle(info);
-	grctx->attrib(info);
-	grctx->unkn(gr);
-
-	gm200_grctx_generate_tpcid(gr);
-	gf100_grctx_generate_r406028(gr);
-	gk104_grctx_generate_r418bb8(gr);
-
-	for (i = 0; i < 8; i++)
-		nvkm_wr32(device, 0x4064d0 + (i * 0x04), 0x00000000);
-	nvkm_wr32(device, 0x406500, 0x00000000);
-
-	nvkm_wr32(device, 0x405b00, (gr->tpc_total << 8) | gr->gpc_nr);
-
-	for (tmp = 0, i = 0; i < gr->gpc_nr; i++)
-		tmp |= ((1 << gr->tpc_nr[i]) - 1) << (i * 5);
-	nvkm_wr32(device, 0x4041c4, tmp);
-
-	gp100_grctx_generate_405b60(gr);
-
-	gf100_gr_icmd(gr, gr->fuc_bundle);
-	nvkm_wr32(device, 0x404154, idle_timeout);
-	gf100_gr_mthd(gr, gr->fuc_method);
-}
-
 const struct gf100_grctx_func
 gp100_grctx = {
-	.main  = gp100_grctx_generate_main,
+	.main  = gf100_grctx_generate_main,
 	.unkn  = gk104_grctx_generate_unkn,
 	.bundle = gm107_grctx_generate_bundle,
 	.bundle_size = 0x3000,
@@ -174,4 +128,12 @@ gp100_grctx = {
 	.attrib_nr = 0x440,
 	.alpha_nr_max = 0xc00,
 	.alpha_nr = 0x800,
+	.sm_id = gm107_grctx_generate_sm_id,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.dist_skip_table = gm200_grctx_generate_dist_skip_table,
+	.r406500 = gm200_grctx_generate_r406500,
+	.gpc_tpc_nr = gk104_grctx_generate_gpc_tpc_nr,
+	.tpc_mask = gm200_grctx_generate_tpc_mask,
+	.smid_config = gp100_grctx_generate_smid_config,
+	.r419a3c = gm200_grctx_generate_r419a3c,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c
index 7a66b4c2eb18..daee17bf7d0d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c
@@ -29,6 +29,13 @@
  * PGRAPH context implementation
  ******************************************************************************/
 
+static void
+gp102_grctx_generate_r408840(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, 0x408840, 0x00000003, 0x00000000);
+}
+
 void
 gp102_grctx_generate_attrib(struct gf100_grctx *info)
 {
@@ -36,14 +43,18 @@ gp102_grctx_generate_attrib(struct gf100_grctx *info)
 	const struct gf100_grctx_func *grctx = gr->func->grctx;
 	const u32  alpha = grctx->alpha_nr;
 	const u32 attrib = grctx->attrib_nr;
-	const u32 pertpc = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max);
-	const u32   size = roundup(gr->tpc_total * pertpc, 0x80);
+	const u32   gfxp = grctx->gfxp_nr;
 	const int s = 12;
-	const int b = mmio_vram(info, size, (1 << s), false);
 	const int max_batches = 0xffff;
+	u32 size = grctx->alpha_nr_max * gr->tpc_total;
 	u32 ao = 0;
-	u32 bo = ao + grctx->alpha_nr_max * gr->tpc_total;
-	int gpc, ppc, n = 0;
+	u32 bo = ao + size;
+	int gpc, ppc, b, n = 0;
+
+	for (gpc = 0; gpc < gr->gpc_nr; gpc++)
+		size += grctx->gfxp_nr * gr->ppc_nr[gpc] * gr->ppc_tpc_max;
+	size = ((size * 0x20) + 128) & ~127;
+	b = mmio_vram(info, size, (1 << s), false);
 
 	mmio_refn(info, 0x418810, 0x80000000, s, b);
 	mmio_refn(info, 0x419848, 0x10000000, s, b);
@@ -57,17 +68,18 @@ gp102_grctx_generate_attrib(struct gf100_grctx *info)
 	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
 		for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) {
 			const u32 as =  alpha * gr->ppc_tpc_nr[gpc][ppc];
-			const u32 bs = attrib * gr->ppc_tpc_nr[gpc][ppc];
+			const u32 bs = attrib * gr->ppc_tpc_max;
+			const u32 gs =   gfxp * gr->ppc_tpc_max;
 			const u32 u = 0x418ea0 + (n * 0x04);
 			const u32 o = PPC_UNIT(gpc, ppc, 0);
 			const u32 p = GPC_UNIT(gpc, 0xc44 + (ppc * 4));
 			if (!(gr->ppc_mask[gpc] & (1 << ppc)))
 				continue;
-			mmio_wr32(info, o + 0xc0, bs);
+			mmio_wr32(info, o + 0xc0, gs);
 			mmio_wr32(info, p, bs);
 			mmio_wr32(info, o + 0xf4, bo);
 			mmio_wr32(info, o + 0xf0, bs);
-			bo += grctx->attrib_nr_max * gr->ppc_tpc_nr[gpc][ppc];
+			bo += gs;
 			mmio_wr32(info, o + 0xe4, as);
 			mmio_wr32(info, o + 0xf8, ao);
 			ao += grctx->alpha_nr_max * gr->ppc_tpc_nr[gpc][ppc];
@@ -81,7 +93,7 @@ gp102_grctx_generate_attrib(struct gf100_grctx *info)
 
 const struct gf100_grctx_func
 gp102_grctx = {
-	.main = gp100_grctx_generate_main,
+	.main = gf100_grctx_generate_main,
 	.unkn = gk104_grctx_generate_unkn,
 	.bundle = gm107_grctx_generate_bundle,
 	.bundle_size = 0x3000,
@@ -90,8 +102,18 @@ gp102_grctx = {
 	.pagepool = gp100_grctx_generate_pagepool,
 	.pagepool_size = 0x20000,
 	.attrib = gp102_grctx_generate_attrib,
-	.attrib_nr_max = 0x5d4,
+	.attrib_nr_max = 0x4b0,
 	.attrib_nr = 0x320,
 	.alpha_nr_max = 0xc00,
 	.alpha_nr = 0x800,
+	.gfxp_nr = 0xba8,
+	.sm_id = gm107_grctx_generate_sm_id,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.dist_skip_table = gm200_grctx_generate_dist_skip_table,
+	.r406500 = gm200_grctx_generate_r406500,
+	.gpc_tpc_nr = gk104_grctx_generate_gpc_tpc_nr,
+	.tpc_mask = gm200_grctx_generate_tpc_mask,
+	.smid_config = gp100_grctx_generate_smid_config,
+	.r419a3c = gm200_grctx_generate_r419a3c,
+	.r408840 = gp102_grctx_generate_r408840,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp104.c
new file mode 100644
index 000000000000..3b85e3d326b2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp104.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "ctxgf100.h"
+
+const struct gf100_grctx_func
+gp104_grctx = {
+	.main = gf100_grctx_generate_main,
+	.unkn = gk104_grctx_generate_unkn,
+	.bundle = gm107_grctx_generate_bundle,
+	.bundle_size = 0x3000,
+	.bundle_min_gpm_fifo_depth = 0x180,
+	.bundle_token_limit = 0x900,
+	.pagepool = gp100_grctx_generate_pagepool,
+	.pagepool_size = 0x20000,
+	.attrib = gp102_grctx_generate_attrib,
+	.attrib_nr_max = 0x4b0,
+	.attrib_nr = 0x320,
+	.alpha_nr_max = 0xc00,
+	.alpha_nr = 0x800,
+	.gfxp_nr = 0xba8,
+	.sm_id = gm107_grctx_generate_sm_id,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.dist_skip_table = gm200_grctx_generate_dist_skip_table,
+	.r406500 = gm200_grctx_generate_r406500,
+	.gpc_tpc_nr = gk104_grctx_generate_gpc_tpc_nr,
+	.tpc_mask = gm200_grctx_generate_tpc_mask,
+	.smid_config = gp100_grctx_generate_smid_config,
+	.r419a3c = gm200_grctx_generate_r419a3c,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp107.c
index 8da91a0b3bd2..5060c5ee5ce0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp107.c
@@ -31,7 +31,7 @@
 
 const struct gf100_grctx_func
 gp107_grctx = {
-	.main = gp100_grctx_generate_main,
+	.main = gf100_grctx_generate_main,
 	.unkn = gk104_grctx_generate_unkn,
 	.bundle = gm107_grctx_generate_bundle,
 	.bundle_size = 0x3000,
@@ -44,4 +44,13 @@ gp107_grctx = {
 	.attrib_nr = 0x540,
 	.alpha_nr_max = 0xc00,
 	.alpha_nr = 0x800,
+	.gfxp_nr = 0xe94,
+	.sm_id = gm107_grctx_generate_sm_id,
+	.rop_mapping = gf117_grctx_generate_rop_mapping,
+	.dist_skip_table = gm200_grctx_generate_dist_skip_table,
+	.r406500 = gm200_grctx_generate_r406500,
+	.gpc_tpc_nr = gk104_grctx_generate_gpc_tpc_nr,
+	.tpc_mask = gm200_grctx_generate_tpc_mask,
+	.smid_config = gp100_grctx_generate_smid_config,
+	.r419a3c = gm200_grctx_generate_r419a3c,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c
new file mode 100644
index 000000000000..0990765ef191
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c
@@ -0,0 +1,215 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "ctxgf100.h"
+
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
+static const struct gf100_gr_init
+gv100_grctx_init_sw_veid_bundle_init_0[] = {
+	{ 0x00001000, 64, 0x00100000, 0x00000008 },
+	{ 0x00000941, 64, 0x00100000, 0x00000000 },
+	{ 0x0000097e, 64, 0x00100000, 0x00000000 },
+	{ 0x0000097f, 64, 0x00100000, 0x00000100 },
+	{ 0x0000035c, 64, 0x00100000, 0x00000000 },
+	{ 0x0000035d, 64, 0x00100000, 0x00000000 },
+	{ 0x00000a08, 64, 0x00100000, 0x00000000 },
+	{ 0x00000a09, 64, 0x00100000, 0x00000000 },
+	{ 0x00000a0a, 64, 0x00100000, 0x00000000 },
+	{ 0x00000352, 64, 0x00100000, 0x00000000 },
+	{ 0x00000353, 64, 0x00100000, 0x00000000 },
+	{ 0x00000358, 64, 0x00100000, 0x00000000 },
+	{ 0x00000359, 64, 0x00100000, 0x00000000 },
+	{ 0x00000370, 64, 0x00100000, 0x00000000 },
+	{ 0x00000371, 64, 0x00100000, 0x00000000 },
+	{ 0x00000372, 64, 0x00100000, 0x000fffff },
+	{ 0x00000366, 64, 0x00100000, 0x00000000 },
+	{ 0x00000367, 64, 0x00100000, 0x00000000 },
+	{ 0x00000368, 64, 0x00100000, 0x00000fff },
+	{ 0x00000623, 64, 0x00100000, 0x00000000 },
+	{ 0x00000624, 64, 0x00100000, 0x00000000 },
+	{ 0x0001e100,  1, 0x00000001, 0x02000001 },
+	{}
+};
+
+static const struct gf100_gr_pack
+gv100_grctx_pack_sw_veid_bundle_init[] = {
+	{ gv100_grctx_init_sw_veid_bundle_init_0 },
+	{}
+};
+
+static void
+gv100_grctx_generate_attrib(struct gf100_grctx *info)
+{
+	struct gf100_gr *gr = info->gr;
+	const struct gf100_grctx_func *grctx = gr->func->grctx;
+	const u32  alpha = grctx->alpha_nr;
+	const u32 attrib = grctx->attrib_nr;
+	const u32   gfxp = grctx->gfxp_nr;
+	const int s = 12;
+	const int max_batches = 0xffff;
+	u32 size = grctx->alpha_nr_max * gr->tpc_total;
+	u32 ao = 0;
+	u32 bo = ao + size;
+	int gpc, ppc, b, n = 0;
+
+	size += grctx->gfxp_nr * gr->tpc_total;
+	size = ((size * 0x20) + 128) & ~127;
+	b = mmio_vram(info, size, (1 << s), false);
+
+	mmio_refn(info, 0x418810, 0x80000000, s, b);
+	mmio_refn(info, 0x419848, 0x10000000, s, b);
+	mmio_refn(info, 0x419c2c, 0x10000000, s, b);
+	mmio_refn(info, 0x419e00, 0x00000000, s, b);
+	mmio_wr32(info, 0x419e04, 0x80000000 | size >> 7);
+	mmio_wr32(info, 0x405830, attrib);
+	mmio_wr32(info, 0x40585c, alpha);
+	mmio_wr32(info, 0x4064c4, ((alpha / 4) << 16) | max_batches);
+
+	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
+		for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) {
+			const u32 as =  alpha * gr->ppc_tpc_nr[gpc][ppc];
+			const u32 bs = attrib * gr->ppc_tpc_nr[gpc][ppc];
+			const u32 gs =   gfxp * gr->ppc_tpc_nr[gpc][ppc];
+			const u32 u = 0x418ea0 + (n * 0x04);
+			const u32 o = PPC_UNIT(gpc, ppc, 0);
+			if (!(gr->ppc_mask[gpc] & (1 << ppc)))
+				continue;
+			mmio_wr32(info, o + 0xc0, gs);
+			mmio_wr32(info, o + 0xf4, bo);
+			mmio_wr32(info, o + 0xf0, bs);
+			bo += gs;
+			mmio_wr32(info, o + 0xe4, as);
+			mmio_wr32(info, o + 0xf8, ao);
+			ao += grctx->alpha_nr_max * gr->ppc_tpc_nr[gpc][ppc];
+			mmio_wr32(info, u, bs);
+		}
+	}
+
+	mmio_wr32(info, 0x4181e4, 0x00000100);
+	mmio_wr32(info, 0x41befc, 0x00000100);
+}
+
+static void
+gv100_grctx_generate_rop_mapping(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	u32 data;
+	int i, j;
+
+	/* Pack tile map into register format. */
+	nvkm_wr32(device, 0x418bb8, (gr->tpc_total << 8) |
+				     gr->screen_tile_row_offset);
+	for (i = 0; i < 11; i++) {
+		for (data = 0, j = 0; j < 6; j++)
+			data |= (gr->tile[i * 6 + j] & 0x1f) << (j * 5);
+		nvkm_wr32(device, 0x418b08 + (i * 4), data);
+		nvkm_wr32(device, 0x41bf00 + (i * 4), data);
+		nvkm_wr32(device, 0x40780c + (i * 4), data);
+	}
+
+	/* GPC_BROADCAST.TP_BROADCAST */
+	nvkm_wr32(device, 0x41bfd0, (gr->tpc_total << 8) |
+				     gr->screen_tile_row_offset);
+	for (i = 0, j = 1; i < 5; i++, j += 4) {
+		u8 v19 = (1 << (j + 0)) % gr->tpc_total;
+		u8 v20 = (1 << (j + 1)) % gr->tpc_total;
+		u8 v21 = (1 << (j + 2)) % gr->tpc_total;
+		u8 v22 = (1 << (j + 3)) % gr->tpc_total;
+		nvkm_wr32(device, 0x41bfb0 + (i * 4), (v22 << 24) |
+						      (v21 << 16) |
+						      (v20 <<  8) |
+						       v19);
+	}
+
+	/* UNK78xx */
+	nvkm_wr32(device, 0x4078bc, (gr->tpc_total << 8) |
+				     gr->screen_tile_row_offset);
+}
+
+static void
+gv100_grctx_generate_r400088(struct gf100_gr *gr, bool on)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, 0x400088, 0x00060000, on ? 0x00060000 : 0x00000000);
+}
+
+static void
+gv100_grctx_generate_sm_id(struct gf100_gr *gr, int gpc, int tpc, int sm)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x608), sm);
+	nvkm_wr32(device, GPC_UNIT(gpc, 0x0c10 + tpc * 4), sm);
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x088), sm);
+}
+
+static void
+gv100_grctx_generate_unkn(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, 0x41980c, 0x00000010, 0x00000010);
+	nvkm_mask(device, 0x41be08, 0x00000004, 0x00000004);
+	nvkm_mask(device, 0x4064c0, 0x80000000, 0x80000000);
+	nvkm_mask(device, 0x405800, 0x08000000, 0x08000000);
+	nvkm_mask(device, 0x419c00, 0x00000008, 0x00000008);
+}
+
+static void
+gv100_grctx_unkn88c(struct gf100_gr *gr, bool on)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	const u32 mask = 0x00000010, data = on ? mask : 0x00000000;
+	nvkm_mask(device, 0x40988c, mask, data);
+	nvkm_rd32(device, 0x40988c);
+	nvkm_mask(device, 0x41a88c, mask, data);
+	nvkm_rd32(device, 0x41a88c);
+	nvkm_mask(device, 0x408a14, mask, data);
+	nvkm_rd32(device, 0x408a14);
+}
+
+const struct gf100_grctx_func
+gv100_grctx = {
+	.unkn88c = gv100_grctx_unkn88c,
+	.main = gf100_grctx_generate_main,
+	.unkn = gv100_grctx_generate_unkn,
+	.sw_veid_bundle_init = gv100_grctx_pack_sw_veid_bundle_init,
+	.bundle = gm107_grctx_generate_bundle,
+	.bundle_size = 0x3000,
+	.bundle_min_gpm_fifo_depth = 0x180,
+	.bundle_token_limit = 0x1680,
+	.pagepool = gp100_grctx_generate_pagepool,
+	.pagepool_size = 0x20000,
+	.attrib = gv100_grctx_generate_attrib,
+	.attrib_nr_max = 0x6c0,
+	.attrib_nr = 0x480,
+	.alpha_nr_max = 0xc00,
+	.alpha_nr = 0x800,
+	.gfxp_nr = 0xd10,
+	.sm_id = gv100_grctx_generate_sm_id,
+	.rop_mapping = gv100_grctx_generate_rop_mapping,
+	.dist_skip_table = gm200_grctx_generate_dist_skip_table,
+	.r406500 = gm200_grctx_generate_r406500,
+	.gpc_tpc_nr = gk104_grctx_generate_gpc_tpc_nr,
+	.smid_config = gp100_grctx_generate_smid_config,
+	.r400088 = gv100_grctx_generate_r400088,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
index 2f8dc107047d..70d3d41e616c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
@@ -32,6 +32,7 @@
 #include <subdev/fb.h>
 #include <subdev/mc.h>
 #include <subdev/pmu.h>
+#include <subdev/therm.h>
 #include <subdev/timer.h>
 #include <engine/fifo.h>
 
@@ -91,7 +92,7 @@ gf100_gr_zbc_color_get(struct gf100_gr *gr, int format,
 	memcpy(gr->zbc_color[zbc].l2, l2, sizeof(gr->zbc_color[zbc].l2));
 	gr->zbc_color[zbc].format = format;
 	nvkm_ltc_zbc_color_get(ltc, zbc, l2);
-	gf100_gr_zbc_clear_color(gr, zbc);
+	gr->func->zbc->clear_color(gr, zbc);
 	return zbc;
 }
 
@@ -136,10 +137,16 @@ gf100_gr_zbc_depth_get(struct gf100_gr *gr, int format,
 	gr->zbc_depth[zbc].ds = ds;
 	gr->zbc_depth[zbc].l2 = l2;
 	nvkm_ltc_zbc_depth_get(ltc, zbc, l2);
-	gf100_gr_zbc_clear_depth(gr, zbc);
+	gr->func->zbc->clear_depth(gr, zbc);
 	return zbc;
 }
 
+const struct gf100_gr_func_zbc
+gf100_gr_zbc = {
+	.clear_color = gf100_gr_zbc_clear_color,
+	.clear_depth = gf100_gr_zbc_clear_depth,
+};
+
 /*******************************************************************************
  * Graphics object classes
  ******************************************************************************/
@@ -743,21 +750,31 @@ gf100_gr_zbc_init(struct gf100_gr *gr)
 	const u32 f32_1[] = { 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000,
 			      0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 };
 	struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc;
-	int index;
+	int index, c = ltc->zbc_min, d = ltc->zbc_min, s = ltc->zbc_min;
 
 	if (!gr->zbc_color[0].format) {
-		gf100_gr_zbc_color_get(gr, 1,  & zero[0],   &zero[4]);
-		gf100_gr_zbc_color_get(gr, 2,  &  one[0],    &one[4]);
-		gf100_gr_zbc_color_get(gr, 4,  &f32_0[0],  &f32_0[4]);
-		gf100_gr_zbc_color_get(gr, 4,  &f32_1[0],  &f32_1[4]);
-		gf100_gr_zbc_depth_get(gr, 1, 0x00000000, 0x00000000);
-		gf100_gr_zbc_depth_get(gr, 1, 0x3f800000, 0x3f800000);
-	}
-
-	for (index = ltc->zbc_min; index <= ltc->zbc_max; index++)
-		gf100_gr_zbc_clear_color(gr, index);
-	for (index = ltc->zbc_min; index <= ltc->zbc_max; index++)
-		gf100_gr_zbc_clear_depth(gr, index);
+		gf100_gr_zbc_color_get(gr, 1,  & zero[0],   &zero[4]); c++;
+		gf100_gr_zbc_color_get(gr, 2,  &  one[0],    &one[4]); c++;
+		gf100_gr_zbc_color_get(gr, 4,  &f32_0[0],  &f32_0[4]); c++;
+		gf100_gr_zbc_color_get(gr, 4,  &f32_1[0],  &f32_1[4]); c++;
+		gf100_gr_zbc_depth_get(gr, 1, 0x00000000, 0x00000000); d++;
+		gf100_gr_zbc_depth_get(gr, 1, 0x3f800000, 0x3f800000); d++;
+		if (gr->func->zbc->stencil_get) {
+			gr->func->zbc->stencil_get(gr, 1, 0x00, 0x00); s++;
+			gr->func->zbc->stencil_get(gr, 1, 0x01, 0x01); s++;
+			gr->func->zbc->stencil_get(gr, 1, 0xff, 0xff); s++;
+		}
+	}
+
+	for (index = c; index <= ltc->zbc_max; index++)
+		gr->func->zbc->clear_color(gr, index);
+	for (index = d; index <= ltc->zbc_max; index++)
+		gr->func->zbc->clear_depth(gr, index);
+
+	if (gr->func->zbc->clear_stencil) {
+		for (index = s; index <= ltc->zbc_max; index++)
+			gr->func->zbc->clear_stencil(gr, index);
+	}
 }
 
 /**
@@ -970,7 +987,7 @@ gf100_gr_trap_gpc_rop(struct gf100_gr *gr, int gpc)
 	nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);
 }
 
-static const struct nvkm_enum gf100_mp_warp_error[] = {
+const struct nvkm_enum gf100_mp_warp_error[] = {
 	{ 0x01, "STACK_ERROR" },
 	{ 0x02, "API_STACK_ERROR" },
 	{ 0x03, "RET_EMPTY_STACK_ERROR" },
@@ -995,7 +1012,7 @@ static const struct nvkm_enum gf100_mp_warp_error[] = {
 	{}
 };
 
-static const struct nvkm_bitfield gf100_mp_global_error[] = {
+const struct nvkm_bitfield gf100_mp_global_error[] = {
 	{ 0x00000001, "SM_TO_SM_FAULT" },
 	{ 0x00000002, "L1_ERROR" },
 	{ 0x00000004, "MULTIPLE_WARP_ERRORS" },
@@ -1009,7 +1026,7 @@ static const struct nvkm_bitfield gf100_mp_global_error[] = {
 	{}
 };
 
-static void
+void
 gf100_gr_trap_mp(struct gf100_gr *gr, int gpc, int tpc)
 {
 	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
@@ -1045,7 +1062,7 @@ gf100_gr_trap_tpc(struct gf100_gr *gr, int gpc, int tpc)
 	}
 
 	if (stat & 0x00000002) {
-		gf100_gr_trap_mp(gr, gpc, tpc);
+		gr->func->trap_mp(gr, gpc, tpc);
 		stat &= ~0x00000002;
 	}
 
@@ -1611,7 +1628,8 @@ gf100_gr_init_ctxctl_int(struct gf100_gr *gr)
 
 	/* load register lists */
 	gf100_gr_init_csdata(gr, grctx->hub, 0x409000, 0x000, 0x000000);
-	gf100_gr_init_csdata(gr, grctx->gpc, 0x41a000, 0x000, 0x418000);
+	gf100_gr_init_csdata(gr, grctx->gpc_0, 0x41a000, 0x000, 0x418000);
+	gf100_gr_init_csdata(gr, grctx->gpc_1, 0x41a000, 0x000, 0x418000);
 	gf100_gr_init_csdata(gr, grctx->tpc, 0x41a000, 0x004, 0x419800);
 	gf100_gr_init_csdata(gr, grctx->ppc, 0x41a000, 0x008, 0x41be00);
 
@@ -1651,6 +1669,97 @@ gf100_gr_init_ctxctl(struct gf100_gr *gr)
 	return ret;
 }
 
+void
+gf100_gr_oneinit_sm_id(struct gf100_gr *gr)
+{
+	int tpc, gpc;
+	for (tpc = 0; tpc < gr->tpc_max; tpc++) {
+		for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
+			if (tpc < gr->tpc_nr[gpc]) {
+				gr->sm[gr->sm_nr].gpc = gpc;
+				gr->sm[gr->sm_nr].tpc = tpc;
+				gr->sm_nr++;
+			}
+		}
+	}
+}
+
+void
+gf100_gr_oneinit_tiles(struct gf100_gr *gr)
+{
+	static const u8 primes[] = {
+		3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61
+	};
+	int init_frac[GPC_MAX], init_err[GPC_MAX], run_err[GPC_MAX], i, j;
+	u32 mul_factor, comm_denom;
+	u8  gpc_map[GPC_MAX];
+	bool sorted;
+
+	switch (gr->tpc_total) {
+	case 15: gr->screen_tile_row_offset = 0x06; break;
+	case 14: gr->screen_tile_row_offset = 0x05; break;
+	case 13: gr->screen_tile_row_offset = 0x02; break;
+	case 11: gr->screen_tile_row_offset = 0x07; break;
+	case 10: gr->screen_tile_row_offset = 0x06; break;
+	case  7:
+	case  5: gr->screen_tile_row_offset = 0x01; break;
+	case  3: gr->screen_tile_row_offset = 0x02; break;
+	case  2:
+	case  1: gr->screen_tile_row_offset = 0x01; break;
+	default: gr->screen_tile_row_offset = 0x03;
+		for (i = 0; i < ARRAY_SIZE(primes); i++) {
+			if (gr->tpc_total % primes[i]) {
+				gr->screen_tile_row_offset = primes[i];
+				break;
+			}
+		}
+		break;
+	}
+
+	/* Sort GPCs by TPC count, highest-to-lowest. */
+	for (i = 0; i < gr->gpc_nr; i++)
+		gpc_map[i] = i;
+	sorted = false;
+
+	while (!sorted) {
+		for (sorted = true, i = 0; i < gr->gpc_nr - 1; i++) {
+			if (gr->tpc_nr[gpc_map[i + 1]] >
+			    gr->tpc_nr[gpc_map[i + 0]]) {
+				u8 swap = gpc_map[i];
+				gpc_map[i + 0] = gpc_map[i + 1];
+				gpc_map[i + 1] = swap;
+				sorted = false;
+			}
+		}
+	}
+
+	/* Determine tile->GPC mapping */
+	mul_factor = gr->gpc_nr * gr->tpc_max;
+	if (mul_factor & 1)
+		mul_factor = 2;
+	else
+		mul_factor = 1;
+
+	comm_denom = gr->gpc_nr * gr->tpc_max * mul_factor;
+
+	for (i = 0; i < gr->gpc_nr; i++) {
+		init_frac[i] = gr->tpc_nr[gpc_map[i]] * gr->gpc_nr * mul_factor;
+		 init_err[i] = i * gr->tpc_max * mul_factor - comm_denom/2;
+		  run_err[i] = init_frac[i] + init_err[i];
+	}
+
+	for (i = 0; i < gr->tpc_total;) {
+		for (j = 0; j < gr->gpc_nr; j++) {
+			if ((run_err[j] * 2) >= comm_denom) {
+				gr->tile[i++] = gpc_map[j];
+				run_err[j] += init_frac[j] - comm_denom;
+			} else {
+				run_err[j] += init_frac[j];
+			}
+		}
+	}
+}
+
 static int
 gf100_gr_oneinit(struct nvkm_gr *base)
 {
@@ -1674,55 +1783,27 @@ gf100_gr_oneinit(struct nvkm_gr *base)
 	gr->gpc_nr = nvkm_rd32(device, 0x409604) & 0x0000001f;
 	for (i = 0; i < gr->gpc_nr; i++) {
 		gr->tpc_nr[i]  = nvkm_rd32(device, GPC_UNIT(i, 0x2608));
+		gr->tpc_max = max(gr->tpc_max, gr->tpc_nr[i]);
 		gr->tpc_total += gr->tpc_nr[i];
 		gr->ppc_nr[i]  = gr->func->ppc_nr;
 		for (j = 0; j < gr->ppc_nr[i]; j++) {
-			u8 mask = nvkm_rd32(device, GPC_UNIT(i, 0x0c30 + (j * 4)));
-			if (mask)
-				gr->ppc_mask[i] |= (1 << j);
-			gr->ppc_tpc_nr[i][j] = hweight8(mask);
-		}
-	}
-
-	/*XXX: these need figuring out... though it might not even matter */
-	switch (device->chipset) {
-	case 0xc0:
-		if (gr->tpc_total == 11) { /* 465, 3/4/4/0, 4 */
-			gr->screen_tile_row_offset = 0x07;
-		} else
-		if (gr->tpc_total == 14) { /* 470, 3/3/4/4, 5 */
-			gr->screen_tile_row_offset = 0x05;
-		} else
-		if (gr->tpc_total == 15) { /* 480, 3/4/4/4, 6 */
-			gr->screen_tile_row_offset = 0x06;
+			gr->ppc_tpc_mask[i][j] =
+				nvkm_rd32(device, GPC_UNIT(i, 0x0c30 + (j * 4)));
+			if (gr->ppc_tpc_mask[i][j] == 0)
+				continue;
+			gr->ppc_mask[i] |= (1 << j);
+			gr->ppc_tpc_nr[i][j] = hweight8(gr->ppc_tpc_mask[i][j]);
+			if (gr->ppc_tpc_min == 0 ||
+			    gr->ppc_tpc_min > gr->ppc_tpc_nr[i][j])
+				gr->ppc_tpc_min = gr->ppc_tpc_nr[i][j];
+			if (gr->ppc_tpc_max < gr->ppc_tpc_nr[i][j])
+				gr->ppc_tpc_max = gr->ppc_tpc_nr[i][j];
 		}
-		break;
-	case 0xc3: /* 450, 4/0/0/0, 2 */
-		gr->screen_tile_row_offset = 0x03;
-		break;
-	case 0xc4: /* 460, 3/4/0/0, 4 */
-		gr->screen_tile_row_offset = 0x01;
-		break;
-	case 0xc1: /* 2/0/0/0, 1 */
-		gr->screen_tile_row_offset = 0x01;
-		break;
-	case 0xc8: /* 4/4/3/4, 5 */
-		gr->screen_tile_row_offset = 0x06;
-		break;
-	case 0xce: /* 4/4/0/0, 4 */
-		gr->screen_tile_row_offset = 0x03;
-		break;
-	case 0xcf: /* 4/0/0/0, 3 */
-		gr->screen_tile_row_offset = 0x03;
-		break;
-	case 0xd7:
-	case 0xd9: /* 1/0/0/0, 1 */
-	case 0xea: /* gk20a */
-	case 0x12b: /* gm20b */
-		gr->screen_tile_row_offset = 0x01;
-		break;
 	}
 
+	memset(gr->tile, 0xff, sizeof(gr->tile));
+	gr->func->oneinit_tiles(gr);
+	gr->func->oneinit_sm_id(gr);
 	return 0;
 }
 
@@ -1913,6 +1994,61 @@ gf100_gr_new_(const struct gf100_gr_func *func, struct nvkm_device *device,
 	return 0;
 }
 
+void
+gf100_gr_init_400054(struct gf100_gr *gr)
+{
+	nvkm_wr32(gr->base.engine.subdev.device, 0x400054, 0x34ce3464);
+}
+
+void
+gf100_gr_init_shader_exceptions(struct gf100_gr *gr, int gpc, int tpc)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
+}
+
+void
+gf100_gr_init_tex_hww_esr(struct gf100_gr *gr, int gpc, int tpc)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
+}
+
+void
+gf100_gr_init_419eb4(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, 0x419eb4, 0x00001000, 0x00001000);
+}
+
+void
+gf100_gr_init_419cc0(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	int gpc, tpc;
+
+	nvkm_mask(device, 0x419cc0, 0x00000008, 0x00000008);
+
+	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
+		for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++)
+			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
+	}
+}
+
+void
+gf100_gr_init_40601c(struct gf100_gr *gr)
+{
+	nvkm_wr32(gr->base.engine.subdev.device, 0x40601c, 0xc0000000);
+}
+
+void
+gf100_gr_init_fecs_exceptions(struct gf100_gr *gr)
+{
+	const u32 data = gr->firmware ? 0x000e0000 : 0x000e0001;
+	nvkm_wr32(gr->base.engine.subdev.device, 0x409c24, data);
+}
+
 void
 gf100_gr_init_gpc_mmu(struct gf100_gr *gr)
 {
@@ -1920,7 +2056,7 @@ gf100_gr_init_gpc_mmu(struct gf100_gr *gr)
 	struct nvkm_fb *fb = device->fb;
 
 	nvkm_wr32(device, 0x418880, nvkm_rd32(device, 0x100c80) & 0x00000001);
-	nvkm_wr32(device, 0x4188a4, 0x00000000);
+	nvkm_wr32(device, 0x4188a4, 0x03000000);
 	nvkm_wr32(device, 0x418888, 0x00000000);
 	nvkm_wr32(device, 0x41888c, 0x00000000);
 	nvkm_wr32(device, 0x418890, 0x00000000);
@@ -1929,37 +2065,30 @@ gf100_gr_init_gpc_mmu(struct gf100_gr *gr)
 	nvkm_wr32(device, 0x4188b8, nvkm_memory_addr(fb->mmu_rd) >> 8);
 }
 
-int
-gf100_gr_init(struct gf100_gr *gr)
+void
+gf100_gr_init_num_active_ltcs(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
-	u32 data[TPC_MAX / 8] = {};
-	u8  tpcnr[GPC_MAX];
-	int gpc, tpc, rop;
-	int i;
-
-	gr->func->init_gpc_mmu(gr);
-
-	gf100_gr_mmio(gr, gr->func->mmio);
-
-	nvkm_mask(device, TPC_UNIT(0, 0, 0x05c), 0x00000001, 0x00000001);
-
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-	for (i = 0, gpc = -1; i < gr->tpc_total; i++) {
-		do {
-			gpc = (gpc + 1) % gr->gpc_nr;
-		} while (!tpcnr[gpc]);
-		tpc = gr->tpc_nr[gpc] - tpcnr[gpc]--;
+	nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));
+}
 
-		data[i / 8] |= tpc << ((i % 8) * 4);
+void
+gf100_gr_init_zcull(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
+	const u8 tile_nr = ALIGN(gr->tpc_total, 32);
+	u8 bank[GPC_MAX] = {}, gpc, i, j;
+	u32 data;
+
+	for (i = 0; i < tile_nr; i += 8) {
+		for (data = 0, j = 0; j < 8 && i + j < gr->tpc_total; j++) {
+			data |= bank[gr->tile[i + j]] << (j * 4);
+			bank[gr->tile[i + j]]++;
+		}
+		nvkm_wr32(device, GPC_BCAST(0x0980 + ((i / 8) * 4)), data);
 	}
 
-	nvkm_wr32(device, GPC_BCAST(0x0980), data[0]);
-	nvkm_wr32(device, GPC_BCAST(0x0984), data[1]);
-	nvkm_wr32(device, GPC_BCAST(0x0988), data[2]);
-	nvkm_wr32(device, GPC_BCAST(0x098c), data[3]);
-
 	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
 		nvkm_wr32(device, GPC_UNIT(gpc, 0x0914),
 			  gr->screen_tile_row_offset << 8 | gr->tpc_nr[gpc]);
@@ -1968,29 +2097,88 @@ gf100_gr_init(struct gf100_gr *gr)
 		nvkm_wr32(device, GPC_UNIT(gpc, 0x0918), magicgpc918);
 	}
 
-	if (device->chipset != 0xd7)
-		nvkm_wr32(device, GPC_BCAST(0x1bd4), magicgpc918);
+	nvkm_wr32(device, GPC_BCAST(0x1bd4), magicgpc918);
+}
+
+void
+gf100_gr_init_vsc_stream_master(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, TPC_UNIT(0, 0, 0x05c), 0x00000001, 0x00000001);
+}
+
+int
+gf100_gr_init(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	int gpc, tpc, rop;
+
+	if (gr->func->init_419bd8)
+		gr->func->init_419bd8(gr);
+
+	gr->func->init_gpc_mmu(gr);
+
+	if (gr->fuc_sw_nonctx)
+		gf100_gr_mmio(gr, gr->fuc_sw_nonctx);
 	else
-		nvkm_wr32(device, GPC_BCAST(0x3fd4), magicgpc918);
+		gf100_gr_mmio(gr, gr->func->mmio);
 
-	nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));
+	gf100_gr_wait_idle(gr);
+
+	if (gr->func->init_r405a14)
+		gr->func->init_r405a14(gr);
+
+	if (gr->func->clkgate_pack)
+		nvkm_therm_clkgate_init(device->therm, gr->func->clkgate_pack);
+
+	if (gr->func->init_bios)
+		gr->func->init_bios(gr);
+
+	gr->func->init_vsc_stream_master(gr);
+	gr->func->init_zcull(gr);
+	gr->func->init_num_active_ltcs(gr);
+	if (gr->func->init_rop_active_fbps)
+		gr->func->init_rop_active_fbps(gr);
+	if (gr->func->init_bios_2)
+		gr->func->init_bios_2(gr);
+	if (gr->func->init_swdx_pes_mask)
+		gr->func->init_swdx_pes_mask(gr);
 
 	nvkm_wr32(device, 0x400500, 0x00010001);
 
 	nvkm_wr32(device, 0x400100, 0xffffffff);
 	nvkm_wr32(device, 0x40013c, 0xffffffff);
+	nvkm_wr32(device, 0x400124, 0x00000002);
+
+	gr->func->init_fecs_exceptions(gr);
+	if (gr->func->init_ds_hww_esr_2)
+		gr->func->init_ds_hww_esr_2(gr);
 
-	nvkm_wr32(device, 0x409c24, 0x000f0000);
 	nvkm_wr32(device, 0x404000, 0xc0000000);
 	nvkm_wr32(device, 0x404600, 0xc0000000);
 	nvkm_wr32(device, 0x408030, 0xc0000000);
-	nvkm_wr32(device, 0x40601c, 0xc0000000);
+
+	if (gr->func->init_40601c)
+		gr->func->init_40601c(gr);
+
 	nvkm_wr32(device, 0x404490, 0xc0000000);
 	nvkm_wr32(device, 0x406018, 0xc0000000);
+
+	if (gr->func->init_sked_hww_esr)
+		gr->func->init_sked_hww_esr(gr);
+
 	nvkm_wr32(device, 0x405840, 0xc0000000);
 	nvkm_wr32(device, 0x405844, 0x00ffffff);
-	nvkm_mask(device, 0x419cc0, 0x00000008, 0x00000008);
-	nvkm_mask(device, 0x419eb4, 0x00001000, 0x00001000);
+
+	if (gr->func->init_419cc0)
+		gr->func->init_419cc0(gr);
+	if (gr->func->init_419eb4)
+		gr->func->init_419eb4(gr);
+	if (gr->func->init_419c9c)
+		gr->func->init_419c9c(gr);
+
+	if (gr->func->init_ppc_exceptions)
+		gr->func->init_ppc_exceptions(gr);
 
 	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
 		nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);
@@ -2000,19 +2188,20 @@ gf100_gr_init(struct gf100_gr *gr)
 		for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {
 			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
 			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
+			if (gr->func->init_tex_hww_esr)
+				gr->func->init_tex_hww_esr(gr, gpc, tpc);
 			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
+			if (gr->func->init_504430)
+				gr->func->init_504430(gr, gpc, tpc);
+			gr->func->init_shader_exceptions(gr, gpc, tpc);
 		}
 		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
 		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
 	}
 
 	for (rop = 0; rop < gr->rop_nr; rop++) {
-		nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0xc0000000);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0xc0000000);
+		nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0x40000000);
+		nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0x40000000);
 		nvkm_wr32(device, ROP_UNIT(rop, 0x204), 0xffffffff);
 		nvkm_wr32(device, ROP_UNIT(rop, 0x208), 0xffffffff);
 	}
@@ -2024,10 +2213,14 @@ gf100_gr_init(struct gf100_gr *gr)
 	nvkm_wr32(device, 0x40011c, 0xffffffff);
 	nvkm_wr32(device, 0x400134, 0xffffffff);
 
-	nvkm_wr32(device, 0x400054, 0x34ce3464);
+	if (gr->func->init_400054)
+		gr->func->init_400054(gr);
 
 	gf100_gr_zbc_init(gr);
 
+	if (gr->func->init_4188a4)
+		gr->func->init_4188a4(gr);
+
 	return gf100_gr_init_ctxctl(gr);
 }
 
@@ -2053,13 +2246,27 @@ gf100_gr_gpccs_ucode = {
 
 static const struct gf100_gr_func
 gf100_gr = {
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
 	.init = gf100_gr_init,
 	.init_gpc_mmu = gf100_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gf100_gr_init_vsc_stream_master,
+	.init_zcull = gf100_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
+	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
+	.init_40601c = gf100_gr_init_40601c,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_419eb4 = gf100_gr_init_419eb4,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_shader_exceptions = gf100_gr_init_shader_exceptions,
+	.init_400054 = gf100_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.mmio = gf100_gr_pack_mmio,
 	.fecs.ucode = &gf100_gr_fecs_ucode,
 	.gpccs.ucode = &gf100_gr_gpccs_ucode,
 	.rops = gf100_gr_rops,
 	.grctx = &gf100_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, FERMI_MEMORY_TO_MEMORY_FORMAT_A },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
index c8ec3fd97155..dc46cf0131db 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
@@ -72,6 +72,12 @@ struct gf100_gr_zbc_depth {
 	u32 l2;
 };
 
+struct gf100_gr_zbc_stencil {
+	u32 format;
+	u32 ds;
+	u32 l2;
+};
+
 struct gf100_gr {
 	const struct gf100_gr_func *func;
 	struct nvkm_gr base;
@@ -95,21 +101,33 @@ struct gf100_gr {
 
 	struct gf100_gr_zbc_color zbc_color[NVKM_LTC_MAX_ZBC_CNT];
 	struct gf100_gr_zbc_depth zbc_depth[NVKM_LTC_MAX_ZBC_CNT];
+	struct gf100_gr_zbc_stencil zbc_stencil[NVKM_LTC_MAX_ZBC_CNT];
 
 	u8 rop_nr;
 	u8 gpc_nr;
 	u8 tpc_nr[GPC_MAX];
+	u8 tpc_max;
 	u8 tpc_total;
 	u8 ppc_nr[GPC_MAX];
 	u8 ppc_mask[GPC_MAX];
+	u8 ppc_tpc_mask[GPC_MAX][4];
 	u8 ppc_tpc_nr[GPC_MAX][4];
+	u8 ppc_tpc_min;
+	u8 ppc_tpc_max;
+
+	u8 screen_tile_row_offset;
+	u8 tile[TPC_MAX];
+
+	struct {
+		u8 gpc;
+		u8 tpc;
+	} sm[TPC_MAX];
+	u8 sm_nr;
 
 	struct gf100_gr_data mmio_data[4];
 	struct gf100_gr_mmio mmio_list[4096/8];
 	u32  size;
 	u32 *data;
-
-	u8 screen_tile_row_offset;
 };
 
 int gf100_gr_ctor(const struct gf100_gr_func *, struct nvkm_device *,
@@ -118,14 +136,43 @@ int gf100_gr_new_(const struct gf100_gr_func *, struct nvkm_device *,
 		  int, struct nvkm_gr **);
 void *gf100_gr_dtor(struct nvkm_gr *);
 
+struct gf100_gr_func_zbc {
+	void (*clear_color)(struct gf100_gr *, int zbc);
+	void (*clear_depth)(struct gf100_gr *, int zbc);
+	int (*stencil_get)(struct gf100_gr *, int format,
+			   const u32 ds, const u32 l2);
+	void (*clear_stencil)(struct gf100_gr *, int zbc);
+};
+
 struct gf100_gr_func {
 	void (*dtor)(struct gf100_gr *);
+	void (*oneinit_tiles)(struct gf100_gr *);
+	void (*oneinit_sm_id)(struct gf100_gr *);
 	int (*init)(struct gf100_gr *);
+	void (*init_419bd8)(struct gf100_gr *);
 	void (*init_gpc_mmu)(struct gf100_gr *);
+	void (*init_r405a14)(struct gf100_gr *);
+	void (*init_bios)(struct gf100_gr *);
+	void (*init_vsc_stream_master)(struct gf100_gr *);
+	void (*init_zcull)(struct gf100_gr *);
+	void (*init_num_active_ltcs)(struct gf100_gr *);
 	void (*init_rop_active_fbps)(struct gf100_gr *);
-	void (*init_ppc_exceptions)(struct gf100_gr *);
+	void (*init_bios_2)(struct gf100_gr *);
 	void (*init_swdx_pes_mask)(struct gf100_gr *);
-	void (*init_num_active_ltcs)(struct gf100_gr *);
+	void (*init_fecs_exceptions)(struct gf100_gr *);
+	void (*init_ds_hww_esr_2)(struct gf100_gr *);
+	void (*init_40601c)(struct gf100_gr *);
+	void (*init_sked_hww_esr)(struct gf100_gr *);
+	void (*init_419cc0)(struct gf100_gr *);
+	void (*init_419eb4)(struct gf100_gr *);
+	void (*init_419c9c)(struct gf100_gr *);
+	void (*init_ppc_exceptions)(struct gf100_gr *);
+	void (*init_tex_hww_esr)(struct gf100_gr *, int gpc, int tpc);
+	void (*init_504430)(struct gf100_gr *, int gpc, int tpc);
+	void (*init_shader_exceptions)(struct gf100_gr *, int gpc, int tpc);
+	void (*init_400054)(struct gf100_gr *);
+	void (*init_4188a4)(struct gf100_gr *);
+	void (*trap_mp)(struct gf100_gr *, int gpc, int tpc);
 	void (*set_hww_esr_report_mask)(struct gf100_gr *);
 	const struct gf100_gr_pack *mmio;
 	struct {
@@ -135,26 +182,60 @@ struct gf100_gr_func {
 		struct gf100_gr_ucode *ucode;
 	} gpccs;
 	int (*rops)(struct gf100_gr *);
+	int gpc_nr;
+	int tpc_nr;
 	int ppc_nr;
 	const struct gf100_grctx_func *grctx;
 	const struct nvkm_therm_clkgate_pack *clkgate_pack;
+	const struct gf100_gr_func_zbc *zbc;
 	struct nvkm_sclass sclass[];
 };
 
-int gf100_gr_init(struct gf100_gr *);
 int gf100_gr_rops(struct gf100_gr *);
-
-int gk104_gr_init(struct gf100_gr *);
+void gf100_gr_oneinit_tiles(struct gf100_gr *);
+void gf100_gr_oneinit_sm_id(struct gf100_gr *);
+int gf100_gr_init(struct gf100_gr *);
+void gf100_gr_init_vsc_stream_master(struct gf100_gr *);
+void gf100_gr_init_zcull(struct gf100_gr *);
+void gf100_gr_init_num_active_ltcs(struct gf100_gr *);
+void gf100_gr_init_fecs_exceptions(struct gf100_gr *);
+void gf100_gr_init_40601c(struct gf100_gr *);
+void gf100_gr_init_419cc0(struct gf100_gr *);
+void gf100_gr_init_419eb4(struct gf100_gr *);
+void gf100_gr_init_tex_hww_esr(struct gf100_gr *, int, int);
+void gf100_gr_init_shader_exceptions(struct gf100_gr *, int, int);
+void gf100_gr_init_400054(struct gf100_gr *);
+extern const struct gf100_gr_func_zbc gf100_gr_zbc;
+
+void gf117_gr_init_zcull(struct gf100_gr *);
+
+void gk104_gr_init_vsc_stream_master(struct gf100_gr *);
 void gk104_gr_init_rop_active_fbps(struct gf100_gr *);
 void gk104_gr_init_ppc_exceptions(struct gf100_gr *);
+void gk104_gr_init_sked_hww_esr(struct gf100_gr *);
+
+void gk110_gr_init_419eb4(struct gf100_gr *);
+
+void gm107_gr_init_504430(struct gf100_gr *, int, int);
+void gm107_gr_init_shader_exceptions(struct gf100_gr *, int, int);
+void gm107_gr_init_400054(struct gf100_gr *);
 
 int gk20a_gr_init(struct gf100_gr *);
 
-int gm200_gr_init(struct gf100_gr *);
+void gm200_gr_oneinit_tiles(struct gf100_gr *);
+void gm200_gr_oneinit_sm_id(struct gf100_gr *);
 int gm200_gr_rops(struct gf100_gr *);
+void gm200_gr_init_num_active_ltcs(struct gf100_gr *);
+void gm200_gr_init_ds_hww_esr_2(struct gf100_gr *);
 
-int gp100_gr_init(struct gf100_gr *);
 void gp100_gr_init_rop_active_fbps(struct gf100_gr *);
+void gp100_gr_init_fecs_exceptions(struct gf100_gr *);
+void gp100_gr_init_shader_exceptions(struct gf100_gr *, int, int);
+void gp100_gr_zbc_clear_color(struct gf100_gr *, int);
+void gp100_gr_zbc_clear_depth(struct gf100_gr *, int);
+
+void gp102_gr_init_swdx_pes_mask(struct gf100_gr *);
+extern const struct gf100_gr_func_zbc gp102_gr_zbc;
 
 #define gf100_gr_chan(p) container_of((p), struct gf100_gr_chan, object)
 #include <core/object.h>
@@ -187,7 +268,7 @@ extern const struct nvkm_object_func gf100_fermi;
 struct gf100_gr_init {
 	u32 addr;
 	u8  count;
-	u8  pitch;
+	u32 pitch;
 	u32 data;
 };
 
@@ -257,6 +338,9 @@ extern const struct gf100_gr_init gf100_gr_init_be_0[];
 extern const struct gf100_gr_init gf100_gr_init_fe_1[];
 extern const struct gf100_gr_init gf100_gr_init_pe_1[];
 void gf100_gr_init_gpc_mmu(struct gf100_gr *);
+void gf100_gr_trap_mp(struct gf100_gr *, int, int);
+extern const struct nvkm_bitfield gf100_mp_global_error[];
+extern const struct nvkm_enum gf100_mp_warp_error[];
 
 extern const struct gf100_gr_init gf104_gr_init_ds_0[];
 extern const struct gf100_gr_init gf104_gr_init_tex_0[];
@@ -279,6 +363,7 @@ extern const struct gf100_gr_init gf117_gr_init_wwdx_0[];
 extern const struct gf100_gr_init gf117_gr_init_cbm_0[];
 
 extern const struct gf100_gr_init gk104_gr_init_main_0[];
+extern const struct gf100_gr_init gk104_gr_init_gpc_unk_2[];
 extern const struct gf100_gr_init gk104_gr_init_tpccs_0[];
 extern const struct gf100_gr_init gk104_gr_init_pe_0[];
 extern const struct gf100_gr_init gk104_gr_init_be_0[];
@@ -306,8 +391,4 @@ extern const struct gf100_gr_init gm107_gr_init_cbm_0[];
 void gm107_gr_init_bios(struct gf100_gr *);
 
 void gm200_gr_init_gpc_mmu(struct gf100_gr *);
-
-void gp100_gr_init_num_active_ltcs(struct gf100_gr *gr);
-
-void gp102_gr_init_swdx_pes_mask(struct gf100_gr *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c
index ec0f11983b23..42c2fd9fc04e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c
@@ -114,13 +114,27 @@ gf104_gr_pack_mmio[] = {
 
 static const struct gf100_gr_func
 gf104_gr = {
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
 	.init = gf100_gr_init,
 	.init_gpc_mmu = gf100_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gf100_gr_init_vsc_stream_master,
+	.init_zcull = gf100_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
+	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
+	.init_40601c = gf100_gr_init_40601c,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_419eb4 = gf100_gr_init_419eb4,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_shader_exceptions = gf100_gr_init_shader_exceptions,
+	.init_400054 = gf100_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.mmio = gf104_gr_pack_mmio,
 	.fecs.ucode = &gf100_gr_fecs_ucode,
 	.gpccs.ucode = &gf100_gr_gpccs_ucode,
 	.rops = gf100_gr_rops,
 	.grctx = &gf104_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, FERMI_MEMORY_TO_MEMORY_FORMAT_A },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c
index cc152eb74123..4731a460adc7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c
@@ -103,15 +103,36 @@ gf108_gr_pack_mmio[] = {
  * PGRAPH engine/subdev functions
  ******************************************************************************/
 
+static void
+gf108_gr_init_r405a14(struct gf100_gr *gr)
+{
+	nvkm_wr32(gr->base.engine.subdev.device, 0x405a14, 0x80000000);
+}
+
 static const struct gf100_gr_func
 gf108_gr = {
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
 	.init = gf100_gr_init,
 	.init_gpc_mmu = gf100_gr_init_gpc_mmu,
+	.init_r405a14 = gf108_gr_init_r405a14,
+	.init_vsc_stream_master = gf100_gr_init_vsc_stream_master,
+	.init_zcull = gf100_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
+	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
+	.init_40601c = gf100_gr_init_40601c,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_419eb4 = gf100_gr_init_419eb4,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_shader_exceptions = gf100_gr_init_shader_exceptions,
+	.init_400054 = gf100_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.mmio = gf108_gr_pack_mmio,
 	.fecs.ucode = &gf100_gr_fecs_ucode,
 	.gpccs.ucode = &gf100_gr_gpccs_ucode,
 	.rops = gf100_gr_rops,
 	.grctx = &gf108_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, FERMI_MEMORY_TO_MEMORY_FORMAT_A },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c
index 10d2d73ca8c3..cdf759c8cd7f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c
@@ -86,13 +86,27 @@ gf110_gr_pack_mmio[] = {
 
 static const struct gf100_gr_func
 gf110_gr = {
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
 	.init = gf100_gr_init,
 	.init_gpc_mmu = gf100_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gf100_gr_init_vsc_stream_master,
+	.init_zcull = gf100_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
+	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
+	.init_40601c = gf100_gr_init_40601c,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_419eb4 = gf100_gr_init_419eb4,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_shader_exceptions = gf100_gr_init_shader_exceptions,
+	.init_400054 = gf100_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.mmio = gf110_gr_pack_mmio,
 	.fecs.ucode = &gf100_gr_fecs_ucode,
 	.gpccs.ucode = &gf100_gr_gpccs_ucode,
 	.rops = gf100_gr_rops,
 	.grctx = &gf110_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, FERMI_MEMORY_TO_MEMORY_FORMAT_A },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c
index ac09a07c4150..a4158f84c649 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c
@@ -120,16 +120,58 @@ gf117_gr_gpccs_ucode = {
 	.data.size = sizeof(gf117_grgpc_data),
 };
 
+void
+gf117_gr_init_zcull(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
+	const u8 tile_nr = ALIGN(gr->tpc_total, 32);
+	u8 bank[GPC_MAX] = {}, gpc, i, j;
+	u32 data;
+
+	for (i = 0; i < tile_nr; i += 8) {
+		for (data = 0, j = 0; j < 8 && i + j < gr->tpc_total; j++) {
+			data |= bank[gr->tile[i + j]] << (j * 4);
+			bank[gr->tile[i + j]]++;
+		}
+		nvkm_wr32(device, GPC_BCAST(0x0980 + ((i / 8) * 4)), data);
+	}
+
+	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
+		nvkm_wr32(device, GPC_UNIT(gpc, 0x0914),
+			  gr->screen_tile_row_offset << 8 | gr->tpc_nr[gpc]);
+		nvkm_wr32(device, GPC_UNIT(gpc, 0x0910), 0x00040000 |
+							 gr->tpc_total);
+		nvkm_wr32(device, GPC_UNIT(gpc, 0x0918), magicgpc918);
+	}
+
+	nvkm_wr32(device, GPC_BCAST(0x3fd4), magicgpc918);
+}
+
 static const struct gf100_gr_func
 gf117_gr = {
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
 	.init = gf100_gr_init,
 	.init_gpc_mmu = gf100_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gf100_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
+	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
+	.init_40601c = gf100_gr_init_40601c,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_419eb4 = gf100_gr_init_419eb4,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_shader_exceptions = gf100_gr_init_shader_exceptions,
+	.init_400054 = gf100_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.mmio = gf117_gr_pack_mmio,
 	.fecs.ucode = &gf117_gr_fecs_ucode,
 	.gpccs.ucode = &gf117_gr_gpccs_ucode,
 	.rops = gf100_gr_rops,
 	.ppc_nr = 1,
 	.grctx = &gf117_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, FERMI_MEMORY_TO_MEMORY_FORMAT_A },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c
index 7f449ec6f760..4197844870b3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c
@@ -177,13 +177,27 @@ gf119_gr_pack_mmio[] = {
 
 static const struct gf100_gr_func
 gf119_gr = {
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
 	.init = gf100_gr_init,
 	.init_gpc_mmu = gf100_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gf100_gr_init_vsc_stream_master,
+	.init_zcull = gf100_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
+	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
+	.init_40601c = gf100_gr_init_40601c,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_419eb4 = gf100_gr_init_419eb4,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_shader_exceptions = gf100_gr_init_shader_exceptions,
+	.init_400054 = gf100_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.mmio = gf119_gr_pack_mmio,
 	.fecs.ucode = &gf100_gr_fecs_ucode,
 	.gpccs.ucode = &gf100_gr_gpccs_ucode,
 	.rops = gf100_gr_rops,
 	.grctx = &gf119_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, FERMI_MEMORY_TO_MEMORY_FORMAT_A },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
index 1b52fcb2c49a..477fee3e3715 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
@@ -82,6 +82,12 @@ gk104_gr_init_gpc_unk_1[] = {
 	{}
 };
 
+const struct gf100_gr_init
+gk104_gr_init_gpc_unk_2[] = {
+	{ 0x418884,   1, 0x04, 0x00000000 },
+	{}
+};
+
 const struct gf100_gr_init
 gk104_gr_init_tpccs_0[] = {
 	{ 0x419d0c,   1, 0x04, 0x00000000 },
@@ -160,6 +166,7 @@ gk104_gr_pack_mmio[] = {
 	{ gf119_gr_init_gpm_0 },
 	{ gk104_gr_init_gpc_unk_1 },
 	{ gf100_gr_init_gcc_0 },
+	{ gk104_gr_init_gpc_unk_2 },
 	{ gk104_gr_init_tpccs_0 },
 	{ gf119_gr_init_tex_0 },
 	{ gk104_gr_init_pe_0 },
@@ -380,6 +387,21 @@ gk104_clkgate_pack[] = {
  * PGRAPH engine/subdev functions
  ******************************************************************************/
 
+void
+gk104_gr_init_sked_hww_esr(struct gf100_gr *gr)
+{
+	nvkm_wr32(gr->base.engine.subdev.device, 0x407020, 0x40000000);
+}
+
+static void
+gk104_gr_init_fecs_exceptions(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_wr32(device, 0x409ffc, 0x00000000);
+	nvkm_wr32(device, 0x409c14, 0x00003e3e);
+	nvkm_wr32(device, 0x409c24, 0x000f0001);
+}
+
 void
 gk104_gr_init_rop_active_fbps(struct gf100_gr *gr)
 {
@@ -404,112 +426,11 @@ gk104_gr_init_ppc_exceptions(struct gf100_gr *gr)
 	}
 }
 
-int
-gk104_gr_init(struct gf100_gr *gr)
+void
+gk104_gr_init_vsc_stream_master(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
-	u32 data[TPC_MAX / 8] = {};
-	u8  tpcnr[GPC_MAX];
-	int gpc, tpc, rop;
-	int i;
-
-	gr->func->init_gpc_mmu(gr);
-
-	gf100_gr_mmio(gr, gr->func->mmio);
-	if (gr->func->clkgate_pack)
-		nvkm_therm_clkgate_init(gr->base.engine.subdev.device->therm,
-					gr->func->clkgate_pack);
-
 	nvkm_wr32(device, GPC_UNIT(0, 0x3018), 0x00000001);
-
-	memset(data, 0x00, sizeof(data));
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-	for (i = 0, gpc = -1; i < gr->tpc_total; i++) {
-		do {
-			gpc = (gpc + 1) % gr->gpc_nr;
-		} while (!tpcnr[gpc]);
-		tpc = gr->tpc_nr[gpc] - tpcnr[gpc]--;
-
-		data[i / 8] |= tpc << ((i % 8) * 4);
-	}
-
-	nvkm_wr32(device, GPC_BCAST(0x0980), data[0]);
-	nvkm_wr32(device, GPC_BCAST(0x0984), data[1]);
-	nvkm_wr32(device, GPC_BCAST(0x0988), data[2]);
-	nvkm_wr32(device, GPC_BCAST(0x098c), data[3]);
-
-	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0914),
-			  gr->screen_tile_row_offset << 8 | gr->tpc_nr[gpc]);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0910), 0x00040000 |
-							 gr->tpc_total);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0918), magicgpc918);
-	}
-
-	nvkm_wr32(device, GPC_BCAST(0x3fd4), magicgpc918);
-	nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));
-
-	gr->func->init_rop_active_fbps(gr);
-
-	nvkm_wr32(device, 0x400500, 0x00010001);
-
-	nvkm_wr32(device, 0x400100, 0xffffffff);
-	nvkm_wr32(device, 0x40013c, 0xffffffff);
-
-	nvkm_wr32(device, 0x409ffc, 0x00000000);
-	nvkm_wr32(device, 0x409c14, 0x00003e3e);
-	nvkm_wr32(device, 0x409c24, 0x000f0001);
-	nvkm_wr32(device, 0x404000, 0xc0000000);
-	nvkm_wr32(device, 0x404600, 0xc0000000);
-	nvkm_wr32(device, 0x408030, 0xc0000000);
-	nvkm_wr32(device, 0x404490, 0xc0000000);
-	nvkm_wr32(device, 0x406018, 0xc0000000);
-	nvkm_wr32(device, 0x407020, 0x40000000);
-	nvkm_wr32(device, 0x405840, 0xc0000000);
-	nvkm_wr32(device, 0x405844, 0x00ffffff);
-	nvkm_mask(device, 0x419cc0, 0x00000008, 0x00000008);
-	nvkm_mask(device, 0x419eb4, 0x00001000, 0x00001000);
-
-	gr->func->init_ppc_exceptions(gr);
-
-	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0900), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x1028), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0824), 0xc0000000);
-		for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
-		}
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
-	}
-
-	for (rop = 0; rop < gr->rop_nr; rop++) {
-		nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0xc0000000);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0xc0000000);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x204), 0xffffffff);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x208), 0xffffffff);
-	}
-
-	nvkm_wr32(device, 0x400108, 0xffffffff);
-	nvkm_wr32(device, 0x400138, 0xffffffff);
-	nvkm_wr32(device, 0x400118, 0xffffffff);
-	nvkm_wr32(device, 0x400130, 0xffffffff);
-	nvkm_wr32(device, 0x40011c, 0xffffffff);
-	nvkm_wr32(device, 0x400134, 0xffffffff);
-
-	nvkm_wr32(device, 0x400054, 0x34ce3464);
-
-	gf100_gr_zbc_init(gr);
-
-	return gf100_gr_init_ctxctl(gr);
 }
 
 #include "fuc/hubgk104.fuc3.h"
@@ -534,10 +455,23 @@ gk104_gr_gpccs_ucode = {
 
 static const struct gf100_gr_func
 gk104_gr = {
-	.init = gk104_gr_init,
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
 	.init_gpc_mmu = gf100_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
 	.init_rop_active_fbps = gk104_gr_init_rop_active_fbps,
+	.init_fecs_exceptions = gk104_gr_init_fecs_exceptions,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_419eb4 = gf100_gr_init_419eb4,
 	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_shader_exceptions = gf100_gr_init_shader_exceptions,
+	.init_400054 = gf100_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.mmio = gk104_gr_pack_mmio,
 	.fecs.ucode = &gk104_gr_fecs_ucode,
 	.gpccs.ucode = &gk104_gr_gpccs_ucode,
@@ -545,6 +479,7 @@ gk104_gr = {
 	.ppc_nr = 1,
 	.grctx = &gk104_grctx,
 	.clkgate_pack = gk104_clkgate_pack,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_A },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c
index 4da916a9fc73..7cd628c84e07 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c
@@ -143,6 +143,7 @@ gk110_gr_pack_mmio[] = {
 	{ gf119_gr_init_gpm_0 },
 	{ gk110_gr_init_gpc_unk_1 },
 	{ gf100_gr_init_gcc_0 },
+	{ gk104_gr_init_gpc_unk_2 },
 	{ gk104_gr_init_tpccs_0 },
 	{ gk110_gr_init_tex_0 },
 	{ gk104_gr_init_pe_0 },
@@ -334,12 +335,39 @@ gk110_gr_gpccs_ucode = {
 	.data.size = sizeof(gk110_grgpc_data),
 };
 
+void
+gk110_gr_init_419eb4(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, 0x419eb4, 0x00001000, 0x00001000);
+	nvkm_mask(device, 0x419eb4, 0x00002000, 0x00002000);
+	nvkm_mask(device, 0x419eb4, 0x00004000, 0x00004000);
+	nvkm_mask(device, 0x419eb4, 0x00008000, 0x00008000);
+	nvkm_mask(device, 0x419eb4, 0x00001000, 0x00000000);
+	nvkm_mask(device, 0x419eb4, 0x00002000, 0x00000000);
+	nvkm_mask(device, 0x419eb4, 0x00004000, 0x00000000);
+	nvkm_mask(device, 0x419eb4, 0x00008000, 0x00000000);
+}
+
 static const struct gf100_gr_func
 gk110_gr = {
-	.init = gk104_gr_init,
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
 	.init_gpc_mmu = gf100_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
 	.init_rop_active_fbps = gk104_gr_init_rop_active_fbps,
+	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_419eb4 = gk110_gr_init_419eb4,
 	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_shader_exceptions = gf100_gr_init_shader_exceptions,
+	.init_400054 = gf100_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.mmio = gk110_gr_pack_mmio,
 	.fecs.ucode = &gk110_gr_fecs_ucode,
 	.gpccs.ucode = &gk110_gr_gpccs_ucode,
@@ -347,6 +375,7 @@ gk110_gr = {
 	.ppc_nr = 2,
 	.grctx = &gk110_grctx,
 	.clkgate_pack = gk110_clkgate_pack,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c
index 1912c0bfd7ee..a38faa215635 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c
@@ -82,6 +82,7 @@ gk110b_gr_pack_mmio[] = {
 	{ gf119_gr_init_gpm_0 },
 	{ gk110_gr_init_gpc_unk_1 },
 	{ gf100_gr_init_gcc_0 },
+	{ gk104_gr_init_gpc_unk_2 },
 	{ gk104_gr_init_tpccs_0 },
 	{ gk110_gr_init_tex_0 },
 	{ gk104_gr_init_pe_0 },
@@ -102,16 +103,30 @@ gk110b_gr_pack_mmio[] = {
 
 static const struct gf100_gr_func
 gk110b_gr = {
-	.init = gk104_gr_init,
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
 	.init_gpc_mmu = gf100_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
 	.init_rop_active_fbps = gk104_gr_init_rop_active_fbps,
+	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_419eb4 = gk110_gr_init_419eb4,
 	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_shader_exceptions = gf100_gr_init_shader_exceptions,
+	.init_400054 = gf100_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.mmio = gk110b_gr_pack_mmio,
 	.fecs.ucode = &gk110_gr_fecs_ucode,
 	.gpccs.ucode = &gk110_gr_gpccs_ucode,
 	.rops = gf100_gr_rops,
 	.ppc_nr = 2,
 	.grctx = &gk110b_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c
index 1fc258163f25..58456660e603 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c
@@ -121,6 +121,7 @@ gk208_gr_pack_mmio[] = {
 	{ gf119_gr_init_gpm_0 },
 	{ gk110_gr_init_gpc_unk_1 },
 	{ gf100_gr_init_gcc_0 },
+	{ gk104_gr_init_gpc_unk_2 },
 	{ gk104_gr_init_tpccs_0 },
 	{ gk208_gr_init_tex_0 },
 	{ gk104_gr_init_pe_0 },
@@ -161,16 +162,29 @@ gk208_gr_gpccs_ucode = {
 
 static const struct gf100_gr_func
 gk208_gr = {
-	.init = gk104_gr_init,
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
 	.init_gpc_mmu = gf100_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
 	.init_rop_active_fbps = gk104_gr_init_rop_active_fbps,
+	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_419cc0 = gf100_gr_init_419cc0,
 	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_shader_exceptions = gf100_gr_init_shader_exceptions,
+	.init_400054 = gf100_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.mmio = gk208_gr_pack_mmio,
 	.fecs.ucode = &gk208_gr_fecs_ucode,
 	.gpccs.ucode = &gk208_gr_gpccs_ucode,
 	.rops = gf100_gr_rops,
 	.ppc_nr = 1,
 	.grctx = &gk208_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c
index de8b806b88fd..500cb08dd608 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c
@@ -219,11 +219,7 @@ int
 gk20a_gr_init(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
-	u32 data[TPC_MAX / 8] = {};
-	u8  tpcnr[GPC_MAX];
-	int gpc, tpc;
-	int ret, i;
+	int ret;
 
 	/* Clear SCC RAM */
 	nvkm_wr32(device, 0x40802c, 0x1);
@@ -246,31 +242,7 @@ gk20a_gr_init(struct gf100_gr *gr)
 	nvkm_mask(device, 0x503018, 0x1, 0x1);
 
 	/* Zcull init */
-	memset(data, 0x00, sizeof(data));
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-	for (i = 0, gpc = -1; i < gr->tpc_total; i++) {
-		do {
-			gpc = (gpc + 1) % gr->gpc_nr;
-		} while (!tpcnr[gpc]);
-		tpc = gr->tpc_nr[gpc] - tpcnr[gpc]--;
-
-		data[i / 8] |= tpc << ((i % 8) * 4);
-	}
-
-	nvkm_wr32(device, GPC_BCAST(0x0980), data[0]);
-	nvkm_wr32(device, GPC_BCAST(0x0984), data[1]);
-	nvkm_wr32(device, GPC_BCAST(0x0988), data[2]);
-	nvkm_wr32(device, GPC_BCAST(0x098c), data[3]);
-
-	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0914),
-			  gr->screen_tile_row_offset << 8 | gr->tpc_nr[gpc]);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0910), 0x00040000 |
-			  gr->tpc_total);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0918), magicgpc918);
-	}
-
-	nvkm_wr32(device, GPC_BCAST(0x3fd4), magicgpc918);
+	gr->func->init_zcull(gr);
 
 	gr->func->init_rop_active_fbps(gr);
 
@@ -310,12 +282,17 @@ gk20a_gr_init(struct gf100_gr *gr)
 
 static const struct gf100_gr_func
 gk20a_gr = {
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
 	.init = gk20a_gr_init,
+	.init_zcull = gf117_gr_init_zcull,
 	.init_rop_active_fbps = gk104_gr_init_rop_active_fbps,
+	.trap_mp = gf100_gr_trap_mp,
 	.set_hww_esr_report_mask = gk20a_gr_set_hww_esr_report_mask,
 	.rops = gf100_gr_rops,
 	.ppc_nr = 1,
 	.grctx = &gk20a_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_A },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c
index 2c67fac576d1..92e31d397207 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c
@@ -25,6 +25,8 @@
 #include "ctxgf100.h"
 
 #include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/init.h>
 #include <subdev/bios/P0260.h>
 #include <subdev/fb.h>
 
@@ -36,6 +38,10 @@
 
 static const struct gf100_gr_init
 gm107_gr_init_main_0[] = {
+	{ 0x40880c,   1, 0x04, 0x00000000 },
+	{ 0x408910,   1, 0x04, 0x00000000 },
+	{ 0x408984,   1, 0x04, 0x00000000 },
+	{ 0x41a8a0,   1, 0x04, 0x00000000 },
 	{ 0x400080,   1, 0x04, 0x003003c2 },
 	{ 0x400088,   1, 0x04, 0x0001bfe7 },
 	{ 0x40008c,   1, 0x04, 0x00060000 },
@@ -210,14 +216,13 @@ gm107_gr_init_cbm_0[] = {
 static const struct gf100_gr_init
 gm107_gr_init_be_0[] = {
 	{ 0x408890,   1, 0x04, 0x000000ff },
-	{ 0x40880c,   1, 0x04, 0x00000000 },
 	{ 0x408850,   1, 0x04, 0x00000004 },
 	{ 0x408878,   1, 0x04, 0x00c81603 },
 	{ 0x40887c,   1, 0x04, 0x80543432 },
 	{ 0x408880,   1, 0x04, 0x0010581e },
 	{ 0x408884,   1, 0x04, 0x00001205 },
 	{ 0x408974,   1, 0x04, 0x000000ff },
-	{ 0x408910,   9, 0x04, 0x00000000 },
+	{ 0x408914,   8, 0x04, 0x00000000 },
 	{ 0x408950,   1, 0x04, 0x00000000 },
 	{ 0x408954,   1, 0x04, 0x0000ffff },
 	{ 0x408958,   1, 0x04, 0x00000034 },
@@ -227,7 +232,6 @@ gm107_gr_init_be_0[] = {
 	{ 0x408968,   1, 0x04, 0x02808833 },
 	{ 0x40896c,   1, 0x04, 0x01f02438 },
 	{ 0x408970,   1, 0x04, 0x00012c00 },
-	{ 0x408984,   1, 0x04, 0x00000000 },
 	{ 0x408988,   1, 0x04, 0x08040201 },
 	{ 0x40898c,   1, 0x04, 0x80402010 },
 	{}
@@ -260,6 +264,7 @@ gm107_gr_pack_mmio[] = {
 	{ gf100_gr_init_gpm_0 },
 	{ gm107_gr_init_gpc_unk_1 },
 	{ gf100_gr_init_gcc_0 },
+	{ gk104_gr_init_gpc_unk_2 },
 	{ gm107_gr_init_tpccs_0 },
 	{ gm107_gr_init_tex_0 },
 	{ gm107_gr_init_pe_0 },
@@ -279,6 +284,52 @@ gm107_gr_pack_mmio[] = {
  * PGRAPH engine/subdev functions
  ******************************************************************************/
 
+void
+gm107_gr_init_400054(struct gf100_gr *gr)
+{
+	nvkm_wr32(gr->base.engine.subdev.device, 0x400054, 0x2c350f63);
+}
+
+void
+gm107_gr_init_shader_exceptions(struct gf100_gr *gr, int gpc, int tpc)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x00dffffe);
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x00000005);
+}
+
+void
+gm107_gr_init_504430(struct gf100_gr *gr, int gpc, int tpc)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x430), 0xc0000000);
+}
+
+static void
+gm107_gr_init_bios_2(struct gf100_gr *gr)
+{
+	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	struct nvkm_bios *bios = device->bios;
+	struct bit_entry bit_P;
+	if (!bit_entry(bios, 'P', &bit_P) &&
+	    bit_P.version == 2 && bit_P.length >= 0x2c) {
+		u32 data = nvbios_rd32(bios, bit_P.offset + 0x28);
+		if (data) {
+			u8 ver = nvbios_rd08(bios, data + 0x00);
+			u8 hdr = nvbios_rd08(bios, data + 0x01);
+			if (ver == 0x20 && hdr >= 8) {
+				data = nvbios_rd32(bios, data + 0x04);
+				if (data) {
+					u32 save = nvkm_rd32(device, 0x619444);
+					nvbios_init(subdev, data);
+					nvkm_wr32(device, 0x619444, save);
+				}
+			}
+		}
+	}
+}
+
 void
 gm107_gr_init_bios(struct gf100_gr *gr)
 {
@@ -308,115 +359,17 @@ gm107_gr_init_bios(struct gf100_gr *gr)
 	}
 }
 
-static int
-gm107_gr_init(struct gf100_gr *gr)
+static void
+gm107_gr_init_gpc_mmu(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
 	struct nvkm_fb *fb = device->fb;
-	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
-	u32 data[TPC_MAX / 8] = {};
-	u8  tpcnr[GPC_MAX];
-	int gpc, tpc, rop;
-	int i;
 
 	nvkm_wr32(device, GPC_BCAST(0x0880), 0x00000000);
 	nvkm_wr32(device, GPC_BCAST(0x0890), 0x00000000);
 	nvkm_wr32(device, GPC_BCAST(0x0894), 0x00000000);
 	nvkm_wr32(device, GPC_BCAST(0x08b4), nvkm_memory_addr(fb->mmu_wr) >> 8);
 	nvkm_wr32(device, GPC_BCAST(0x08b8), nvkm_memory_addr(fb->mmu_rd) >> 8);
-
-	gf100_gr_mmio(gr, gr->func->mmio);
-
-	gm107_gr_init_bios(gr);
-
-	nvkm_wr32(device, GPC_UNIT(0, 0x3018), 0x00000001);
-
-	memset(data, 0x00, sizeof(data));
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-	for (i = 0, gpc = -1; i < gr->tpc_total; i++) {
-		do {
-			gpc = (gpc + 1) % gr->gpc_nr;
-		} while (!tpcnr[gpc]);
-		tpc = gr->tpc_nr[gpc] - tpcnr[gpc]--;
-
-		data[i / 8] |= tpc << ((i % 8) * 4);
-	}
-
-	nvkm_wr32(device, GPC_BCAST(0x0980), data[0]);
-	nvkm_wr32(device, GPC_BCAST(0x0984), data[1]);
-	nvkm_wr32(device, GPC_BCAST(0x0988), data[2]);
-	nvkm_wr32(device, GPC_BCAST(0x098c), data[3]);
-
-	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0914),
-			  gr->screen_tile_row_offset << 8 | gr->tpc_nr[gpc]);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0910), 0x00040000 |
-							 gr->tpc_total);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0918), magicgpc918);
-	}
-
-	nvkm_wr32(device, GPC_BCAST(0x3fd4), magicgpc918);
-	nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));
-
-	gr->func->init_rop_active_fbps(gr);
-
-	nvkm_wr32(device, 0x400500, 0x00010001);
-
-	nvkm_wr32(device, 0x400100, 0xffffffff);
-	nvkm_wr32(device, 0x40013c, 0xffffffff);
-	nvkm_wr32(device, 0x400124, 0x00000002);
-	nvkm_wr32(device, 0x409c24, 0x000e0000);
-
-	nvkm_wr32(device, 0x404000, 0xc0000000);
-	nvkm_wr32(device, 0x404600, 0xc0000000);
-	nvkm_wr32(device, 0x408030, 0xc0000000);
-	nvkm_wr32(device, 0x404490, 0xc0000000);
-	nvkm_wr32(device, 0x406018, 0xc0000000);
-	nvkm_wr32(device, 0x407020, 0x40000000);
-	nvkm_wr32(device, 0x405840, 0xc0000000);
-	nvkm_wr32(device, 0x405844, 0x00ffffff);
-	nvkm_mask(device, 0x419cc0, 0x00000008, 0x00000008);
-
-	gr->func->init_ppc_exceptions(gr);
-
-	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0900), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x1028), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0824), 0xc0000000);
-		for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x430), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x00dffffe);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x00000005);
-		}
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
-	}
-
-	for (rop = 0; rop < gr->rop_nr; rop++) {
-		nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0x40000000);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0x40000000);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x204), 0xffffffff);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x208), 0xffffffff);
-	}
-
-	nvkm_wr32(device, 0x400108, 0xffffffff);
-	nvkm_wr32(device, 0x400138, 0xffffffff);
-	nvkm_wr32(device, 0x400118, 0xffffffff);
-	nvkm_wr32(device, 0x400130, 0xffffffff);
-	nvkm_wr32(device, 0x40011c, 0xffffffff);
-	nvkm_wr32(device, 0x400134, 0xffffffff);
-
-	nvkm_wr32(device, 0x400054, 0x2c350f63);
-
-	gf100_gr_zbc_init(gr);
-
-	return gf100_gr_init_ctxctl(gr);
 }
 
 #include "fuc/hubgm107.fuc5.h"
@@ -441,15 +394,32 @@ gm107_gr_gpccs_ucode = {
 
 static const struct gf100_gr_func
 gm107_gr = {
-	.init = gm107_gr_init,
+	.oneinit_tiles = gf100_gr_oneinit_tiles,
+	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
+	.init_gpc_mmu = gm107_gr_init_gpc_mmu,
+	.init_bios = gm107_gr_init_bios,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
 	.init_rop_active_fbps = gk104_gr_init_rop_active_fbps,
+	.init_bios_2 = gm107_gr_init_bios_2,
+	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_419cc0 = gf100_gr_init_419cc0,
 	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_504430 = gm107_gr_init_504430,
+	.init_shader_exceptions = gm107_gr_init_shader_exceptions,
+	.init_400054 = gm107_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.mmio = gm107_gr_pack_mmio,
 	.fecs.ucode = &gm107_gr_fecs_ucode,
 	.gpccs.ucode = &gm107_gr_gpccs_ucode,
 	.rops = gf100_gr_rops,
 	.ppc_nr = 2,
 	.grctx = &gm107_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c
index 6435f1257572..eff30662b984 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c
@@ -38,6 +38,22 @@ gm200_gr_rops(struct gf100_gr *gr)
 	return nvkm_rd32(gr->base.engine.subdev.device, 0x12006c);
 }
 
+void
+gm200_gr_init_ds_hww_esr_2(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_wr32(device, 0x405848, 0xc0000000);
+	nvkm_mask(device, 0x40584c, 0x00000001, 0x00000001);
+}
+
+void
+gm200_gr_init_num_active_ltcs(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));
+	nvkm_wr32(device, GPC_BCAST(0x033c), nvkm_rd32(device, 0x100804));
+}
+
 void
 gm200_gr_init_gpc_mmu(struct gf100_gr *gr)
 {
@@ -61,111 +77,51 @@ gm200_gr_init_rop_active_fbps(struct gf100_gr *gr)
 	nvkm_mask(device, 0x408958, 0x0000000f, fbp_count); /* crop */
 }
 
-int
-gm200_gr_init(struct gf100_gr *gr)
-{
-	struct nvkm_device *device = gr->base.engine.subdev.device;
-	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
-	u32 data[TPC_MAX / 8] = {};
-	u8  tpcnr[GPC_MAX];
-	int gpc, tpc, rop;
-	int i;
-
-	gr->func->init_gpc_mmu(gr);
-
-	gf100_gr_mmio(gr, gr->fuc_sw_nonctx);
-
-	gm107_gr_init_bios(gr);
-
-	nvkm_wr32(device, GPC_UNIT(0, 0x3018), 0x00000001);
-
-	memset(data, 0x00, sizeof(data));
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-	for (i = 0, gpc = -1; i < gr->tpc_total; i++) {
-		do {
-			gpc = (gpc + 1) % gr->gpc_nr;
-		} while (!tpcnr[gpc]);
-		tpc = gr->tpc_nr[gpc] - tpcnr[gpc]--;
-
-		data[i / 8] |= tpc << ((i % 8) * 4);
-	}
-
-	nvkm_wr32(device, GPC_BCAST(0x0980), data[0]);
-	nvkm_wr32(device, GPC_BCAST(0x0984), data[1]);
-	nvkm_wr32(device, GPC_BCAST(0x0988), data[2]);
-	nvkm_wr32(device, GPC_BCAST(0x098c), data[3]);
-
-	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0914),
-			  gr->screen_tile_row_offset << 8 | gr->tpc_nr[gpc]);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0910), 0x00040000 |
-							 gr->tpc_total);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0918), magicgpc918);
-	}
+static u8
+gm200_gr_tile_map_6_24[] = {
+	0, 1, 2, 3, 4, 5, 3, 4, 5, 0, 1, 2, 0, 1, 2, 3, 4, 5, 3, 4, 5, 0, 1, 2,
+};
 
-	nvkm_wr32(device, GPC_BCAST(0x3fd4), magicgpc918);
-	nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));
-	nvkm_wr32(device, GPC_BCAST(0x033c), nvkm_rd32(device, 0x100804));
+static u8
+gm200_gr_tile_map_4_16[] = {
+	0, 1, 2, 3, 2, 3, 0, 1, 3, 0, 1, 2, 1, 2, 3, 0,
+};
 
-	gr->func->init_rop_active_fbps(gr);
+static u8
+gm200_gr_tile_map_2_8[] = {
+	0, 1, 1, 0, 0, 1, 1, 0,
+};
 
-	nvkm_wr32(device, 0x400500, 0x00010001);
-	nvkm_wr32(device, 0x400100, 0xffffffff);
-	nvkm_wr32(device, 0x40013c, 0xffffffff);
-	nvkm_wr32(device, 0x400124, 0x00000002);
-	nvkm_wr32(device, 0x409c24, 0x000e0000);
-	nvkm_wr32(device, 0x405848, 0xc0000000);
-	nvkm_wr32(device, 0x40584c, 0x00000001);
-	nvkm_wr32(device, 0x404000, 0xc0000000);
-	nvkm_wr32(device, 0x404600, 0xc0000000);
-	nvkm_wr32(device, 0x408030, 0xc0000000);
-	nvkm_wr32(device, 0x404490, 0xc0000000);
-	nvkm_wr32(device, 0x406018, 0xc0000000);
-	nvkm_wr32(device, 0x407020, 0x40000000);
-	nvkm_wr32(device, 0x405840, 0xc0000000);
-	nvkm_wr32(device, 0x405844, 0x00ffffff);
-	nvkm_mask(device, 0x419cc0, 0x00000008, 0x00000008);
-
-	gr->func->init_ppc_exceptions(gr);
-
-	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0900), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x1028), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0824), 0xc0000000);
-		for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x430), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x00dffffe);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x00000005);
-		}
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
-	}
+void
+gm200_gr_oneinit_sm_id(struct gf100_gr *gr)
+{
+	/*XXX: There's a different algorithm here I've not yet figured out. */
+	gf100_gr_oneinit_sm_id(gr);
+}
 
-	for (rop = 0; rop < gr->rop_nr; rop++) {
-		nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0x40000000);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0x40000000);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x204), 0xffffffff);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x208), 0xffffffff);
+void
+gm200_gr_oneinit_tiles(struct gf100_gr *gr)
+{
+	/*XXX: Not sure what this is about.  The algorithm from NVGPU
+	 *     seems to work for all boards I tried from earlier (and
+	 *     later) GPUs except in these specific configurations.
+	 *
+	 *     Let's just hardcode them for now.
+	 */
+	if (gr->gpc_nr == 2 && gr->tpc_total == 8) {
+		memcpy(gr->tile, gm200_gr_tile_map_2_8, gr->tpc_total);
+		gr->screen_tile_row_offset = 1;
+	} else
+	if (gr->gpc_nr == 4 && gr->tpc_total == 16) {
+		memcpy(gr->tile, gm200_gr_tile_map_4_16, gr->tpc_total);
+		gr->screen_tile_row_offset = 4;
+	} else
+	if (gr->gpc_nr == 6 && gr->tpc_total == 24) {
+		memcpy(gr->tile, gm200_gr_tile_map_6_24, gr->tpc_total);
+		gr->screen_tile_row_offset = 5;
+	} else {
+		gf100_gr_oneinit_tiles(gr);
 	}
-
-	nvkm_wr32(device, 0x400108, 0xffffffff);
-	nvkm_wr32(device, 0x400138, 0xffffffff);
-	nvkm_wr32(device, 0x400118, 0xffffffff);
-	nvkm_wr32(device, 0x400130, 0xffffffff);
-	nvkm_wr32(device, 0x40011c, 0xffffffff);
-	nvkm_wr32(device, 0x400134, 0xffffffff);
-
-	nvkm_wr32(device, 0x400054, 0x2c350f63);
-
-	gf100_gr_zbc_init(gr);
-
-	return gf100_gr_init_ctxctl(gr);
 }
 
 int
@@ -208,13 +164,30 @@ gm200_gr_new_(const struct gf100_gr_func *func, struct nvkm_device *device,
 
 static const struct gf100_gr_func
 gm200_gr = {
-	.init = gm200_gr_init,
+	.oneinit_tiles = gm200_gr_oneinit_tiles,
+	.oneinit_sm_id = gm200_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
 	.init_gpc_mmu = gm200_gr_init_gpc_mmu,
+	.init_bios = gm107_gr_init_bios,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gm200_gr_init_num_active_ltcs,
 	.init_rop_active_fbps = gm200_gr_init_rop_active_fbps,
+	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
+	.init_ds_hww_esr_2 = gm200_gr_init_ds_hww_esr_2,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_419cc0 = gf100_gr_init_419cc0,
 	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_504430 = gm107_gr_init_504430,
+	.init_shader_exceptions = gm107_gr_init_shader_exceptions,
+	.init_400054 = gm107_gr_init_400054,
+	.trap_mp = gf100_gr_trap_mp,
 	.rops = gm200_gr_rops,
+	.tpc_nr = 4,
 	.ppc_nr = 2,
 	.grctx = &gm200_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm20b.c
index 69479af1d829..a667770ce3cb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm20b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm20b.c
@@ -64,13 +64,18 @@ gm20b_gr_set_hww_esr_report_mask(struct gf100_gr *gr)
 
 static const struct gf100_gr_func
 gm20b_gr = {
+	.oneinit_tiles = gm200_gr_oneinit_tiles,
+	.oneinit_sm_id = gm200_gr_oneinit_sm_id,
 	.init = gk20a_gr_init,
+	.init_zcull = gf117_gr_init_zcull,
 	.init_gpc_mmu = gm20b_gr_init_gpc_mmu,
 	.init_rop_active_fbps = gk104_gr_init_rop_active_fbps,
+	.trap_mp = gf100_gr_trap_mp,
 	.set_hww_esr_report_mask = gm20b_gr_set_hww_esr_report_mask,
 	.rops = gm200_gr_rops,
 	.ppc_nr = 1,
 	.grctx = &gm20b_grctx,
+	.zbc = &gf100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c
index 867a5f7cc5bc..9d0521ce309a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c
@@ -29,143 +29,103 @@
 /*******************************************************************************
  * PGRAPH engine/subdev functions
  ******************************************************************************/
-
 void
-gp100_gr_init_rop_active_fbps(struct gf100_gr *gr)
+gp100_gr_zbc_clear_color(struct gf100_gr *gr, int zbc)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-	/*XXX: otherwise identical to gm200 aside from mask.. do everywhere? */
-	const u32 fbp_count = nvkm_rd32(device, 0x12006c) & 0x0000000f;
-	nvkm_mask(device, 0x408850, 0x0000000f, fbp_count); /* zrop */
-	nvkm_mask(device, 0x408958, 0x0000000f, fbp_count); /* crop */
+	const int znum =  zbc - 1;
+	const u32 zoff = znum * 4;
+
+	if (gr->zbc_color[zbc].format) {
+		nvkm_wr32(device, 0x418010 + zoff, gr->zbc_color[zbc].ds[0]);
+		nvkm_wr32(device, 0x41804c + zoff, gr->zbc_color[zbc].ds[1]);
+		nvkm_wr32(device, 0x418088 + zoff, gr->zbc_color[zbc].ds[2]);
+		nvkm_wr32(device, 0x4180c4 + zoff, gr->zbc_color[zbc].ds[3]);
+	}
+
+	nvkm_mask(device, 0x418100 + ((znum / 4) * 4),
+			  0x0000007f << ((znum % 4) * 7),
+			  gr->zbc_color[zbc].format << ((znum % 4) * 7));
 }
 
 void
-gp100_gr_init_num_active_ltcs(struct gf100_gr *gr)
+gp100_gr_zbc_clear_depth(struct gf100_gr *gr, int zbc)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-
-	nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));
-	nvkm_wr32(device, GPC_BCAST(0x033c), nvkm_rd32(device, 0x100804));
+	const int znum =  zbc - 1;
+	const u32 zoff = znum * 4;
+
+	if (gr->zbc_depth[zbc].format)
+		nvkm_wr32(device, 0x418110 + zoff, gr->zbc_depth[zbc].ds);
+	nvkm_mask(device, 0x41814c + ((znum / 4) * 4),
+			  0x0000007f << ((znum % 4) * 7),
+			  gr->zbc_depth[zbc].format << ((znum % 4) * 7));
 }
 
-int
-gp100_gr_init(struct gf100_gr *gr)
+static const struct gf100_gr_func_zbc
+gp100_gr_zbc = {
+	.clear_color = gp100_gr_zbc_clear_color,
+	.clear_depth = gp100_gr_zbc_clear_depth,
+};
+
+void
+gp100_gr_init_shader_exceptions(struct gf100_gr *gr, int gpc, int tpc)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
-	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
-	u32 data[TPC_MAX / 8] = {};
-	u8  tpcnr[GPC_MAX];
-	int gpc, tpc, rop;
-	int i;
-
-	gr->func->init_gpc_mmu(gr);
-
-	gf100_gr_mmio(gr, gr->fuc_sw_nonctx);
-
-	nvkm_wr32(device, GPC_UNIT(0, 0x3018), 0x00000001);
-
-	memset(data, 0x00, sizeof(data));
-	memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
-	for (i = 0, gpc = -1; i < gr->tpc_total; i++) {
-		do {
-			gpc = (gpc + 1) % gr->gpc_nr;
-		} while (!tpcnr[gpc]);
-		tpc = gr->tpc_nr[gpc] - tpcnr[gpc]--;
-
-		data[i / 8] |= tpc << ((i % 8) * 4);
-	}
-
-	nvkm_wr32(device, GPC_BCAST(0x0980), data[0]);
-	nvkm_wr32(device, GPC_BCAST(0x0984), data[1]);
-	nvkm_wr32(device, GPC_BCAST(0x0988), data[2]);
-	nvkm_wr32(device, GPC_BCAST(0x098c), data[3]);
-
-	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0914),
-			  gr->screen_tile_row_offset << 8 | gr->tpc_nr[gpc]);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0910), 0x00040000 |
-							 gr->tpc_total);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0918), magicgpc918);
-	}
-
-	nvkm_wr32(device, GPC_BCAST(0x3fd4), magicgpc918);
-	gr->func->init_num_active_ltcs(gr);
-
-	gr->func->init_rop_active_fbps(gr);
-	if (gr->func->init_swdx_pes_mask)
-		gr->func->init_swdx_pes_mask(gr);
-
-	nvkm_wr32(device, 0x400500, 0x00010001);
-	nvkm_wr32(device, 0x400100, 0xffffffff);
-	nvkm_wr32(device, 0x40013c, 0xffffffff);
-	nvkm_wr32(device, 0x400124, 0x00000002);
-	nvkm_wr32(device, 0x409c24, 0x000f0002);
-	nvkm_wr32(device, 0x405848, 0xc0000000);
-	nvkm_mask(device, 0x40584c, 0x00000000, 0x00000001);
-	nvkm_wr32(device, 0x404000, 0xc0000000);
-	nvkm_wr32(device, 0x404600, 0xc0000000);
-	nvkm_wr32(device, 0x408030, 0xc0000000);
-	nvkm_wr32(device, 0x404490, 0xc0000000);
-	nvkm_wr32(device, 0x406018, 0xc0000000);
-	nvkm_wr32(device, 0x407020, 0x40000000);
-	nvkm_wr32(device, 0x405840, 0xc0000000);
-	nvkm_wr32(device, 0x405844, 0x00ffffff);
-	nvkm_mask(device, 0x419cc0, 0x00000008, 0x00000008);
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x00dffffe);
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x00000105);
+}
 
+static void
+gp100_gr_init_419c9c(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
 	nvkm_mask(device, 0x419c9c, 0x00010000, 0x00010000);
 	nvkm_mask(device, 0x419c9c, 0x00020000, 0x00020000);
+}
 
-	gr->func->init_ppc_exceptions(gr);
-
-	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0900), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x1028), 0xc0000000);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x0824), 0xc0000000);
-		for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x430), 0xc0000000);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x00dffffe);
-			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x00000105);
-		}
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
-		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
-	}
-
-	for (rop = 0; rop < gr->rop_nr; rop++) {
-		nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0x40000000);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0x40000000);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x204), 0xffffffff);
-		nvkm_wr32(device, ROP_UNIT(rop, 0x208), 0xffffffff);
-	}
-
-	nvkm_wr32(device, 0x400108, 0xffffffff);
-	nvkm_wr32(device, 0x400138, 0xffffffff);
-	nvkm_wr32(device, 0x400118, 0xffffffff);
-	nvkm_wr32(device, 0x400130, 0xffffffff);
-	nvkm_wr32(device, 0x40011c, 0xffffffff);
-	nvkm_wr32(device, 0x400134, 0xffffffff);
-
-	gf100_gr_zbc_init(gr);
+void
+gp100_gr_init_fecs_exceptions(struct gf100_gr *gr)
+{
+	nvkm_wr32(gr->base.engine.subdev.device, 0x409c24, 0x000f0002);
+}
 
-	return gf100_gr_init_ctxctl(gr);
+void
+gp100_gr_init_rop_active_fbps(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	/*XXX: otherwise identical to gm200 aside from mask.. do everywhere? */
+	const u32 fbp_count = nvkm_rd32(device, 0x12006c) & 0x0000000f;
+	nvkm_mask(device, 0x408850, 0x0000000f, fbp_count); /* zrop */
+	nvkm_mask(device, 0x408958, 0x0000000f, fbp_count); /* crop */
 }
 
 static const struct gf100_gr_func
 gp100_gr = {
-	.init = gp100_gr_init,
+	.oneinit_tiles = gm200_gr_oneinit_tiles,
+	.oneinit_sm_id = gm200_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
 	.init_gpc_mmu = gm200_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gm200_gr_init_num_active_ltcs,
 	.init_rop_active_fbps = gp100_gr_init_rop_active_fbps,
+	.init_fecs_exceptions = gp100_gr_init_fecs_exceptions,
+	.init_ds_hww_esr_2 = gm200_gr_init_ds_hww_esr_2,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_419c9c = gp100_gr_init_419c9c,
 	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
-	.init_num_active_ltcs = gp100_gr_init_num_active_ltcs,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_504430 = gm107_gr_init_504430,
+	.init_shader_exceptions = gp100_gr_init_shader_exceptions,
+	.trap_mp = gf100_gr_trap_mp,
 	.rops = gm200_gr_rops,
+	.gpc_nr = 6,
+	.tpc_nr = 5,
 	.ppc_nr = 2,
 	.grctx = &gp100_grctx,
+	.zbc = &gp100_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c
index 61e3a0b08559..37f7d739bf80 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c
@@ -26,6 +26,62 @@
 
 #include <nvif/class.h>
 
+static void
+gp102_gr_zbc_clear_stencil(struct gf100_gr *gr, int zbc)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	const int znum =  zbc - 1;
+	const u32 zoff = znum * 4;
+
+	if (gr->zbc_stencil[zbc].format)
+		nvkm_wr32(device, 0x41815c + zoff, gr->zbc_stencil[zbc].ds);
+	nvkm_mask(device, 0x418198 + ((znum / 4) * 4),
+			  0x0000007f << ((znum % 4) * 7),
+			  gr->zbc_stencil[zbc].format << ((znum % 4) * 7));
+}
+
+static int
+gp102_gr_zbc_stencil_get(struct gf100_gr *gr, int format,
+			 const u32 ds, const u32 l2)
+{
+	struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc;
+	int zbc = -ENOSPC, i;
+
+	for (i = ltc->zbc_min; i <= ltc->zbc_max; i++) {
+		if (gr->zbc_stencil[i].format) {
+			if (gr->zbc_stencil[i].format != format)
+				continue;
+			if (gr->zbc_stencil[i].ds != ds)
+				continue;
+			if (gr->zbc_stencil[i].l2 != l2) {
+				WARN_ON(1);
+				return -EINVAL;
+			}
+			return i;
+		} else {
+			zbc = (zbc < 0) ? i : zbc;
+		}
+	}
+
+	if (zbc < 0)
+		return zbc;
+
+	gr->zbc_stencil[zbc].format = format;
+	gr->zbc_stencil[zbc].ds = ds;
+	gr->zbc_stencil[zbc].l2 = l2;
+	nvkm_ltc_zbc_stencil_get(ltc, zbc, l2);
+	gr->func->zbc->clear_stencil(gr, zbc);
+	return zbc;
+}
+
+const struct gf100_gr_func_zbc
+gp102_gr_zbc = {
+	.clear_color = gp100_gr_zbc_clear_color,
+	.clear_depth = gp100_gr_zbc_clear_depth,
+	.stencil_get = gp102_gr_zbc_stencil_get,
+	.clear_stencil = gp102_gr_zbc_clear_stencil,
+};
+
 void
 gp102_gr_init_swdx_pes_mask(struct gf100_gr *gr)
 {
@@ -42,15 +98,30 @@ gp102_gr_init_swdx_pes_mask(struct gf100_gr *gr)
 
 static const struct gf100_gr_func
 gp102_gr = {
-	.init = gp100_gr_init,
+	.oneinit_tiles = gm200_gr_oneinit_tiles,
+	.oneinit_sm_id = gm200_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
 	.init_gpc_mmu = gm200_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gm200_gr_init_num_active_ltcs,
 	.init_rop_active_fbps = gp100_gr_init_rop_active_fbps,
-	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
 	.init_swdx_pes_mask = gp102_gr_init_swdx_pes_mask,
-	.init_num_active_ltcs = gp100_gr_init_num_active_ltcs,
+	.init_fecs_exceptions = gp100_gr_init_fecs_exceptions,
+	.init_ds_hww_esr_2 = gm200_gr_init_ds_hww_esr_2,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_504430 = gm107_gr_init_504430,
+	.init_shader_exceptions = gp100_gr_init_shader_exceptions,
+	.trap_mp = gf100_gr_trap_mp,
 	.rops = gm200_gr_rops,
+	.gpc_nr = 6,
+	.tpc_nr = 5,
 	.ppc_nr = 3,
 	.grctx = &gp102_grctx,
+	.zbc = &gp102_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c
new file mode 100644
index 000000000000..4573c914c021
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "gf100.h"
+#include "ctxgf100.h"
+
+#include <nvif/class.h>
+
+static const struct gf100_gr_func
+gp104_gr = {
+	.oneinit_tiles = gm200_gr_oneinit_tiles,
+	.oneinit_sm_id = gm200_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
+	.init_gpc_mmu = gm200_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gm200_gr_init_num_active_ltcs,
+	.init_rop_active_fbps = gp100_gr_init_rop_active_fbps,
+	.init_swdx_pes_mask = gp102_gr_init_swdx_pes_mask,
+	.init_fecs_exceptions = gp100_gr_init_fecs_exceptions,
+	.init_ds_hww_esr_2 = gm200_gr_init_ds_hww_esr_2,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_504430 = gm107_gr_init_504430,
+	.init_shader_exceptions = gp100_gr_init_shader_exceptions,
+	.trap_mp = gf100_gr_trap_mp,
+	.rops = gm200_gr_rops,
+	.gpc_nr = 6,
+	.tpc_nr = 5,
+	.ppc_nr = 3,
+	.grctx = &gp104_grctx,
+	.zbc = &gp102_gr_zbc,
+	.sclass = {
+		{ -1, -1, FERMI_TWOD_A },
+		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
+		{ -1, -1, PASCAL_B, &gf100_fermi },
+		{ -1, -1, PASCAL_COMPUTE_B },
+		{}
+	}
+};
+
+int
+gp104_gr_new(struct nvkm_device *device, int index, struct nvkm_gr **pgr)
+{
+	return gm200_gr_new_(&gp104_gr, device, index, pgr);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c
index f7272323f694..812aba91653f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c
@@ -28,15 +28,30 @@
 
 static const struct gf100_gr_func
 gp107_gr = {
-	.init = gp100_gr_init,
+	.oneinit_tiles = gm200_gr_oneinit_tiles,
+	.oneinit_sm_id = gm200_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
 	.init_gpc_mmu = gm200_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gm200_gr_init_num_active_ltcs,
 	.init_rop_active_fbps = gp100_gr_init_rop_active_fbps,
-	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
 	.init_swdx_pes_mask = gp102_gr_init_swdx_pes_mask,
-	.init_num_active_ltcs = gp100_gr_init_num_active_ltcs,
+	.init_fecs_exceptions = gp100_gr_init_fecs_exceptions,
+	.init_ds_hww_esr_2 = gm200_gr_init_ds_hww_esr_2,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_419cc0 = gf100_gr_init_419cc0,
+	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_504430 = gm107_gr_init_504430,
+	.init_shader_exceptions = gp100_gr_init_shader_exceptions,
+	.trap_mp = gf100_gr_trap_mp,
 	.rops = gm200_gr_rops,
+	.gpc_nr = 2,
+	.tpc_nr = 3,
 	.ppc_nr = 1,
 	.grctx = &gp107_grctx,
+	.zbc = &gp102_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c
index 5f3d161a0842..303dceddd4a8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c
@@ -25,24 +25,31 @@
 
 #include <nvif/class.h>
 
-static void
-gp10b_gr_init_num_active_ltcs(struct gf100_gr *gr)
-{
-	struct nvkm_device *device = gr->base.engine.subdev.device;
-
-	nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));
-}
-
 static const struct gf100_gr_func
 gp10b_gr = {
-	.init = gp100_gr_init,
+	.oneinit_tiles = gm200_gr_oneinit_tiles,
+	.oneinit_sm_id = gm200_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
 	.init_gpc_mmu = gm200_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
 	.init_rop_active_fbps = gp100_gr_init_rop_active_fbps,
+	.init_fecs_exceptions = gp100_gr_init_fecs_exceptions,
+	.init_ds_hww_esr_2 = gm200_gr_init_ds_hww_esr_2,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_419cc0 = gf100_gr_init_419cc0,
 	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
-	.init_num_active_ltcs = gp10b_gr_init_num_active_ltcs,
+	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
+	.init_504430 = gm107_gr_init_504430,
+	.init_shader_exceptions = gp100_gr_init_shader_exceptions,
+	.trap_mp = gf100_gr_trap_mp,
 	.rops = gm200_gr_rops,
+	.gpc_nr = 1,
+	.tpc_nr = 2,
 	.ppc_nr = 1,
 	.grctx = &gp102_grctx,
+	.zbc = &gp102_gr_zbc,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c
new file mode 100644
index 000000000000..19173ea19096
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "gf100.h"
+#include "ctxgf100.h"
+
+#include <nvif/class.h>
+
+static void
+gv100_gr_trap_mp(struct gf100_gr *gr, int gpc, int tpc)
+{
+	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 werr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x730));
+	u32 gerr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x734));
+	const struct nvkm_enum *warp;
+	char glob[128];
+
+	nvkm_snprintbf(glob, sizeof(glob), gf100_mp_global_error, gerr);
+	warp = nvkm_enum_find(gf100_mp_warp_error, werr & 0xffff);
+
+	nvkm_error(subdev, "GPC%i/TPC%i/MP trap: "
+			   "global %08x [%s] warp %04x [%s]\n",
+		   gpc, tpc, gerr, glob, werr, warp ? warp->name : "");
+
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x730), 0x00000000);
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x734), gerr);
+}
+
+static void
+gv100_gr_init_4188a4(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, 0x4188a4, 0x03000000, 0x03000000);
+}
+
+static void
+gv100_gr_init_shader_exceptions(struct gf100_gr *gr, int gpc, int tpc)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	int sm;
+	for (sm = 0; sm < 0x100; sm += 0x80) {
+		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x728 + sm), 0x0085eb64);
+		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x610), 0x00000001);
+		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x72c + sm), 0x00000004);
+	}
+}
+
+static void
+gv100_gr_init_504430(struct gf100_gr *gr, int gpc, int tpc)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x430), 0x403f0000);
+}
+
+static void
+gv100_gr_init_419bd8(struct gf100_gr *gr)
+{
+	struct nvkm_device *device = gr->base.engine.subdev.device;
+	nvkm_mask(device, 0x419bd8, 0x00000700, 0x00000000);
+}
+
+static const struct gf100_gr_func
+gv100_gr = {
+	.oneinit_tiles = gm200_gr_oneinit_tiles,
+	.oneinit_sm_id = gm200_gr_oneinit_sm_id,
+	.init = gf100_gr_init,
+	.init_419bd8 = gv100_gr_init_419bd8,
+	.init_gpc_mmu = gm200_gr_init_gpc_mmu,
+	.init_vsc_stream_master = gk104_gr_init_vsc_stream_master,
+	.init_zcull = gf117_gr_init_zcull,
+	.init_num_active_ltcs = gm200_gr_init_num_active_ltcs,
+	.init_rop_active_fbps = gp100_gr_init_rop_active_fbps,
+	.init_swdx_pes_mask = gp102_gr_init_swdx_pes_mask,
+	.init_fecs_exceptions = gp100_gr_init_fecs_exceptions,
+	.init_ds_hww_esr_2 = gm200_gr_init_ds_hww_esr_2,
+	.init_sked_hww_esr = gk104_gr_init_sked_hww_esr,
+	.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
+	.init_504430 = gv100_gr_init_504430,
+	.init_shader_exceptions = gv100_gr_init_shader_exceptions,
+	.init_4188a4 = gv100_gr_init_4188a4,
+	.trap_mp = gv100_gr_trap_mp,
+	.rops = gm200_gr_rops,
+	.gpc_nr = 6,
+	.tpc_nr = 5,
+	.ppc_nr = 3,
+	.grctx = &gv100_grctx,
+	.zbc = &gp102_gr_zbc,
+	.sclass = {
+		{ -1, -1, FERMI_TWOD_A },
+		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
+		{ -1, -1, VOLTA_A, &gf100_fermi },
+		{ -1, -1, VOLTA_COMPUTE_A },
+		{}
+	}
+};
+
+int
+gv100_gr_new(struct nvkm_device *device, int index, struct nvkm_gr **pgr)
+{
+	return gm200_gr_new_(&gv100_gr, device, index, pgr);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c b/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c
index 58a59b7db2e5..771e16a16267 100644
--- a/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c
+++ b/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c
@@ -506,6 +506,7 @@ nvkm_msgqueue_new(u32 version, struct nvkm_falcon *falcon,
 		break;
 	case 0x0148cdec:
 	case 0x015ccf3e:
+	case 0x0167d263:
 		ret = msgqueue_0148cdec_new(falcon, sb, queue);
 		break;
 	default:
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild
index 3f5d38d74fba..cfdffef1afb9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild
@@ -3,6 +3,7 @@ include $(src)/nvkm/subdev/bios/Kbuild
 include $(src)/nvkm/subdev/bus/Kbuild
 include $(src)/nvkm/subdev/clk/Kbuild
 include $(src)/nvkm/subdev/devinit/Kbuild
+include $(src)/nvkm/subdev/fault/Kbuild
 include $(src)/nvkm/subdev/fb/Kbuild
 include $(src)/nvkm/subdev/fuse/Kbuild
 include $(src)/nvkm/subdev/gpio/Kbuild
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c
index 7c7efa4ea0d0..3133b28f849c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c
@@ -25,7 +25,7 @@
 #include <subdev/bios/bit.h>
 #include <subdev/bios/dp.h>
 
-static u16
+u16
 nvbios_dp_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 {
 	struct bit_entry d;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pll.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pll.c
index 2ca23a9157ab..e6e804cee2bc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pll.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pll.c
@@ -193,7 +193,10 @@ pll_map_type(struct nvkm_bios *bios, u8 type, u32 *reg, u8 *ver, u8 *len)
 		data += hdr;
 		while (cnt--) {
 			if (nvbios_rd08(bios, data + 0) == type) {
-				*reg = nvbios_rd32(bios, data + 3);
+				if (*ver < 0x50)
+					*reg = nvbios_rd32(bios, data + 3);
+				else
+					*reg = 0;
 				return data;
 			}
 			data += *len;
@@ -361,6 +364,20 @@ nvbios_pll_parse(struct nvkm_bios *bios, u32 type, struct nvbios_pll *info)
 		info->min_p = nvbios_rd08(bios, data + 12);
 		info->max_p = nvbios_rd08(bios, data + 13);
 		break;
+	case 0x50:
+		info->refclk = nvbios_rd16(bios, data + 1) * 1000;
+		/* info->refclk_alt = nvbios_rd16(bios, data + 3) * 1000; */
+		info->vco1.min_freq = nvbios_rd16(bios, data + 5) * 1000;
+		info->vco1.max_freq = nvbios_rd16(bios, data + 7) * 1000;
+		info->vco1.min_inputfreq = nvbios_rd16(bios, data + 9) * 1000;
+		info->vco1.max_inputfreq = nvbios_rd16(bios, data + 11) * 1000;
+		info->vco1.min_m = nvbios_rd08(bios, data + 13);
+		info->vco1.max_m = nvbios_rd08(bios, data + 14);
+		info->vco1.min_n = nvbios_rd08(bios, data + 15);
+		info->vco1.max_n = nvbios_rd08(bios, data + 16);
+		info->min_p = nvbios_rd08(bios, data + 17);
+		info->max_p = nvbios_rd08(bios, data + 18);
+		break;
 	default:
 		nvkm_error(subdev, "unknown pll limits version 0x%02x\n", ver);
 		return -EINVAL;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowramin.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowramin.c
index 0f537c22804c..3634cd0630b8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowramin.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowramin.c
@@ -78,7 +78,10 @@ pramin_init(struct nvkm_bios *bios, const char *name)
 	 * important as we don't want to be touching vram on an
 	 * uninitialised board
 	 */
-	addr = nvkm_rd32(device, 0x619f04);
+	if (device->card_type >= GV100)
+		addr = nvkm_rd32(device, 0x625f04);
+	else
+		addr = nvkm_rd32(device, 0x619f04);
 	if (!(addr & 0x00000008)) {
 		nvkm_debug(subdev, "... not enabled\n");
 		return ERR_PTR(-ENODEV);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
index e4c8d310d870..ba6a868d4c95 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
@@ -109,18 +109,17 @@ nvkm_cstate_valid(struct nvkm_clk *clk, struct nvkm_cstate *cstate,
 
 static struct nvkm_cstate *
 nvkm_cstate_find_best(struct nvkm_clk *clk, struct nvkm_pstate *pstate,
-		      struct nvkm_cstate *start)
+		      struct nvkm_cstate *cstate)
 {
 	struct nvkm_device *device = clk->subdev.device;
 	struct nvkm_volt *volt = device->volt;
-	struct nvkm_cstate *cstate;
 	int max_volt;
 
-	if (!pstate || !start)
+	if (!pstate || !cstate)
 		return NULL;
 
 	if (!volt)
-		return start;
+		return cstate;
 
 	max_volt = volt->max_uv;
 	if (volt->max0_id != 0xff)
@@ -133,8 +132,7 @@ nvkm_cstate_find_best(struct nvkm_clk *clk, struct nvkm_pstate *pstate,
 		max_volt = min(max_volt,
 			       nvkm_volt_map(volt, volt->max2_id, clk->temp));
 
-	for (cstate = start; &cstate->head != &pstate->list;
-	     cstate = list_entry(cstate->head.prev, typeof(*cstate), head)) {
+	list_for_each_entry_from_reverse(cstate, &pstate->list, head) {
 		if (nvkm_cstate_valid(clk, cstate, max_volt, clk->temp))
 			break;
 	}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild
index eac88e3dc6e5..50a436926484 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild
@@ -12,3 +12,4 @@ nvkm-y += nvkm/subdev/devinit/mcp89.o
 nvkm-y += nvkm/subdev/devinit/gf100.o
 nvkm-y += nvkm/subdev/devinit/gm107.o
 nvkm-y += nvkm/subdev/devinit/gm200.o
+nvkm-y += nvkm/subdev/devinit/gv100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c
index 1730371933df..b80618e35491 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c
@@ -107,7 +107,7 @@ pmu_load(struct nv50_devinit *init, u8 type, bool post,
 	return pmu_exec(init, pmu.init_addr_pmu), 0;
 }
 
-static int
+int
 gm200_devinit_post(struct nvkm_devinit *base, bool post)
 {
 	struct nv50_devinit *init = nv50_devinit(base);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gv100.c
new file mode 100644
index 000000000000..fbde6828bd38
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gv100.c
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "nv50.h"
+
+#include <subdev/bios.h>
+#include <subdev/bios/pll.h>
+#include <subdev/clk/pll.h>
+
+static int
+gv100_devinit_pll_set(struct nvkm_devinit *init, u32 type, u32 freq)
+{
+	struct nvkm_subdev *subdev = &init->subdev;
+	struct nvkm_device *device = subdev->device;
+	struct nvbios_pll info;
+	int head = type - PLL_VPLL0;
+	int N, fN, M, P;
+	int ret;
+
+	ret = nvbios_pll_parse(device->bios, type, &info);
+	if (ret)
+		return ret;
+
+	ret = gt215_pll_calc(subdev, &info, freq, &N, &fN, &M, &P);
+	if (ret < 0)
+		return ret;
+
+	switch (info.type) {
+	case PLL_VPLL0:
+	case PLL_VPLL1:
+	case PLL_VPLL2:
+	case PLL_VPLL3:
+		nvkm_wr32(device, 0x00ef10 + (head * 0x40), fN << 16);
+		nvkm_wr32(device, 0x00ef04 + (head * 0x40), (P << 16) |
+							    (N <<  8) |
+							    (M <<  0));
+		break;
+	default:
+		nvkm_warn(subdev, "%08x/%dKhz unimplemented\n", type, freq);
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static const struct nvkm_devinit_func
+gv100_devinit = {
+	.preinit = gf100_devinit_preinit,
+	.init = nv50_devinit_init,
+	.post = gm200_devinit_post,
+	.pll_set = gv100_devinit_pll_set,
+	.disable = gm107_devinit_disable,
+};
+
+int
+gv100_devinit_new(struct nvkm_device *device, int index,
+		struct nvkm_devinit **pinit)
+{
+	return nv50_devinit_new_(&gv100_devinit, device, index, pinit);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h
index 315ebaff1165..9b9f0dc1e192 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h
@@ -24,4 +24,6 @@ int  gf100_devinit_pll_set(struct nvkm_devinit *, u32, u32);
 void gf100_devinit_preinit(struct nvkm_devinit *);
 
 u64  gm107_devinit_disable(struct nvkm_devinit *);
+
+int gm200_devinit_post(struct nvkm_devinit *, bool);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild
new file mode 100644
index 000000000000..45bb46fb0929
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild
@@ -0,0 +1,3 @@
+nvkm-y += nvkm/subdev/fault/base.o
+nvkm-y += nvkm/subdev/fault/gp100.o
+nvkm-y += nvkm/subdev/fault/gv100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c
new file mode 100644
index 000000000000..007bf4af33b9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c
@@ -0,0 +1,179 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "priv.h"
+
+#include <core/memory.h>
+#include <core/notify.h>
+#include <subdev/bar.h>
+#include <subdev/mmu.h>
+
+static void
+nvkm_fault_ntfy_fini(struct nvkm_event *event, int type, int index)
+{
+	struct nvkm_fault *fault = container_of(event, typeof(*fault), event);
+	fault->func->buffer.fini(fault->buffer[index]);
+}
+
+static void
+nvkm_fault_ntfy_init(struct nvkm_event *event, int type, int index)
+{
+	struct nvkm_fault *fault = container_of(event, typeof(*fault), event);
+	fault->func->buffer.init(fault->buffer[index]);
+}
+
+static int
+nvkm_fault_ntfy_ctor(struct nvkm_object *object, void *argv, u32 argc,
+		     struct nvkm_notify *notify)
+{
+	struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object);
+	if (argc == 0) {
+		notify->size  = 0;
+		notify->types = 1;
+		notify->index = buffer->id;
+		return 0;
+	}
+	return -ENOSYS;
+}
+
+static const struct nvkm_event_func
+nvkm_fault_ntfy = {
+	.ctor = nvkm_fault_ntfy_ctor,
+	.init = nvkm_fault_ntfy_init,
+	.fini = nvkm_fault_ntfy_fini,
+};
+
+static void
+nvkm_fault_intr(struct nvkm_subdev *subdev)
+{
+	struct nvkm_fault *fault = nvkm_fault(subdev);
+	return fault->func->intr(fault);
+}
+
+static int
+nvkm_fault_fini(struct nvkm_subdev *subdev, bool suspend)
+{
+	struct nvkm_fault *fault = nvkm_fault(subdev);
+	if (fault->func->fini)
+		fault->func->fini(fault);
+	return 0;
+}
+
+static int
+nvkm_fault_init(struct nvkm_subdev *subdev)
+{
+	struct nvkm_fault *fault = nvkm_fault(subdev);
+	if (fault->func->init)
+		fault->func->init(fault);
+	return 0;
+}
+
+static int
+nvkm_fault_oneinit_buffer(struct nvkm_fault *fault, int id)
+{
+	struct nvkm_subdev *subdev = &fault->subdev;
+	struct nvkm_device *device = subdev->device;
+	struct nvkm_vmm *bar2 = nvkm_bar_bar2_vmm(device);
+	struct nvkm_fault_buffer *buffer;
+	int ret;
+
+	if (!(buffer = kzalloc(sizeof(*buffer), GFP_KERNEL)))
+		return -ENOMEM;
+	buffer->fault = fault;
+	buffer->id = id;
+	buffer->entries = fault->func->buffer.entries(buffer);
+	fault->buffer[id] = buffer;
+
+	nvkm_debug(subdev, "buffer %d: %d entries\n", id, buffer->entries);
+
+	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, buffer->entries *
+			      fault->func->buffer.entry_size, 0x1000, true,
+			      &buffer->mem);
+	if (ret)
+		return ret;
+
+	ret = nvkm_vmm_get(bar2, 12, nvkm_memory_size(buffer->mem),
+			   &buffer->vma);
+	if (ret)
+		return ret;
+
+	return nvkm_memory_map(buffer->mem, 0, bar2, buffer->vma, NULL, 0);
+}
+
+static int
+nvkm_fault_oneinit(struct nvkm_subdev *subdev)
+{
+	struct nvkm_fault *fault = nvkm_fault(subdev);
+	int ret, i;
+
+	for (i = 0; i < ARRAY_SIZE(fault->buffer); i++) {
+		if (i < fault->func->buffer.nr) {
+			ret = nvkm_fault_oneinit_buffer(fault, i);
+			if (ret)
+				return ret;
+			fault->buffer_nr = i + 1;
+		}
+	}
+
+	return nvkm_event_init(&nvkm_fault_ntfy, 1, fault->buffer_nr,
+			       &fault->event);
+}
+
+static void *
+nvkm_fault_dtor(struct nvkm_subdev *subdev)
+{
+	struct nvkm_vmm *bar2 = nvkm_bar_bar2_vmm(subdev->device);
+	struct nvkm_fault *fault = nvkm_fault(subdev);
+	int i;
+
+	nvkm_event_fini(&fault->event);
+
+	for (i = 0; i < fault->buffer_nr; i++) {
+		if (fault->buffer[i]) {
+			nvkm_vmm_put(bar2, &fault->buffer[i]->vma);
+			nvkm_memory_unref(&fault->buffer[i]->mem);
+			kfree(fault->buffer[i]);
+		}
+	}
+
+	return fault;
+}
+
+static const struct nvkm_subdev_func
+nvkm_fault = {
+	.dtor = nvkm_fault_dtor,
+	.oneinit = nvkm_fault_oneinit,
+	.init = nvkm_fault_init,
+	.fini = nvkm_fault_fini,
+	.intr = nvkm_fault_intr,
+};
+
+int
+nvkm_fault_new_(const struct nvkm_fault_func *func, struct nvkm_device *device,
+		int index, struct nvkm_fault **pfault)
+{
+	struct nvkm_fault *fault;
+	if (!(fault = *pfault = kzalloc(sizeof(*fault), GFP_KERNEL)))
+		return -ENOMEM;
+	nvkm_subdev_ctor(&nvkm_fault, device, index, &fault->subdev);
+	fault->func = func;
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c
new file mode 100644
index 000000000000..5e71db2e8d75
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "priv.h"
+
+#include <subdev/mmu.h>
+
+static void
+gp100_fault_buffer_fini(struct nvkm_fault_buffer *buffer)
+{
+	struct nvkm_device *device = buffer->fault->subdev.device;
+	nvkm_mask(device, 0x002a70, 0x00000001, 0x00000000);
+}
+
+static void
+gp100_fault_buffer_init(struct nvkm_fault_buffer *buffer)
+{
+	struct nvkm_device *device = buffer->fault->subdev.device;
+	nvkm_wr32(device, 0x002a74, upper_32_bits(buffer->vma->addr));
+	nvkm_wr32(device, 0x002a70, lower_32_bits(buffer->vma->addr));
+	nvkm_mask(device, 0x002a70, 0x00000001, 0x00000001);
+}
+
+static u32
+gp100_fault_buffer_entries(struct nvkm_fault_buffer *buffer)
+{
+	return nvkm_rd32(buffer->fault->subdev.device, 0x002a78);
+}
+
+static void
+gp100_fault_intr(struct nvkm_fault *fault)
+{
+	nvkm_event_send(&fault->event, 1, 0, NULL, 0);
+}
+
+static const struct nvkm_fault_func
+gp100_fault = {
+	.intr = gp100_fault_intr,
+	.buffer.nr = 1,
+	.buffer.entry_size = 32,
+	.buffer.entries = gp100_fault_buffer_entries,
+	.buffer.init = gp100_fault_buffer_init,
+	.buffer.fini = gp100_fault_buffer_fini,
+};
+
+int
+gp100_fault_new(struct nvkm_device *device, int index,
+		struct nvkm_fault **pfault)
+{
+	return nvkm_fault_new_(&gp100_fault, device, index, pfault);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c
new file mode 100644
index 000000000000..73c7728b5969
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c
@@ -0,0 +1,206 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "priv.h"
+
+#include <core/memory.h>
+#include <subdev/mmu.h>
+#include <engine/fifo.h>
+
+static void
+gv100_fault_buffer_process(struct nvkm_fault_buffer *buffer)
+{
+	struct nvkm_device *device = buffer->fault->subdev.device;
+	struct nvkm_memory *mem = buffer->mem;
+	const u32 foff = buffer->id * 0x14;
+	u32 get = nvkm_rd32(device, 0x100e2c + foff);
+	u32 put = nvkm_rd32(device, 0x100e30 + foff);
+	if (put == get)
+		return;
+
+	nvkm_kmap(mem);
+	while (get != put) {
+		const u32   base = get * buffer->fault->func->buffer.entry_size;
+		const u32 instlo = nvkm_ro32(mem, base + 0x00);
+		const u32 insthi = nvkm_ro32(mem, base + 0x04);
+		const u32 addrlo = nvkm_ro32(mem, base + 0x08);
+		const u32 addrhi = nvkm_ro32(mem, base + 0x0c);
+		const u32 timelo = nvkm_ro32(mem, base + 0x10);
+		const u32 timehi = nvkm_ro32(mem, base + 0x14);
+		const u32  info0 = nvkm_ro32(mem, base + 0x18);
+		const u32  info1 = nvkm_ro32(mem, base + 0x1c);
+		struct nvkm_fault_data info;
+
+		if (++get == buffer->entries)
+			get = 0;
+		nvkm_wr32(device, 0x100e2c + foff, get);
+
+		info.addr   = ((u64)addrhi << 32) | addrlo;
+		info.inst   = ((u64)insthi << 32) | instlo;
+		info.time   = ((u64)timehi << 32) | timelo;
+		info.engine = (info0 & 0x000000ff);
+		info.valid  = (info1 & 0x80000000) >> 31;
+		info.gpc    = (info1 & 0x1f000000) >> 24;
+		info.hub    = (info1 & 0x00100000) >> 20;
+		info.access = (info1 & 0x000f0000) >> 16;
+		info.client = (info1 & 0x00007f00) >> 8;
+		info.reason = (info1 & 0x0000001f);
+
+		nvkm_fifo_fault(device->fifo, &info);
+	}
+	nvkm_done(mem);
+}
+
+static void
+gv100_fault_buffer_fini(struct nvkm_fault_buffer *buffer)
+{
+	struct nvkm_device *device = buffer->fault->subdev.device;
+	const u32 intr = buffer->id ? 0x08000000 : 0x20000000;
+	const u32 foff = buffer->id * 0x14;
+
+	nvkm_mask(device, 0x100a34, intr, intr);
+	nvkm_mask(device, 0x100e34 + foff, 0x80000000, 0x00000000);
+}
+
+static void
+gv100_fault_buffer_init(struct nvkm_fault_buffer *buffer)
+{
+	struct nvkm_device *device = buffer->fault->subdev.device;
+	const u32 intr = buffer->id ? 0x08000000 : 0x20000000;
+	const u32 foff = buffer->id * 0x14;
+
+	nvkm_mask(device, 0x100e34 + foff, 0xc0000000, 0x40000000);
+	nvkm_wr32(device, 0x100e28 + foff, upper_32_bits(buffer->vma->addr));
+	nvkm_wr32(device, 0x100e24 + foff, lower_32_bits(buffer->vma->addr));
+	nvkm_mask(device, 0x100e34 + foff, 0x80000000, 0x80000000);
+	nvkm_mask(device, 0x100a2c, intr, intr);
+}
+
+static u32
+gv100_fault_buffer_entries(struct nvkm_fault_buffer *buffer)
+{
+	struct nvkm_device *device = buffer->fault->subdev.device;
+	const u32 foff = buffer->id * 0x14;
+	nvkm_mask(device, 0x100e34 + foff, 0x40000000, 0x40000000);
+	return nvkm_rd32(device, 0x100e34 + foff) & 0x000fffff;
+}
+
+static int
+gv100_fault_ntfy_nrpfb(struct nvkm_notify *notify)
+{
+	struct nvkm_fault *fault = container_of(notify, typeof(*fault), nrpfb);
+	gv100_fault_buffer_process(fault->buffer[0]);
+	return NVKM_NOTIFY_KEEP;
+}
+
+static void
+gv100_fault_intr_fault(struct nvkm_fault *fault)
+{
+	struct nvkm_subdev *subdev = &fault->subdev;
+	struct nvkm_device *device = subdev->device;
+	struct nvkm_fault_data info;
+	const u32 addrlo = nvkm_rd32(device, 0x100e4c);
+	const u32 addrhi = nvkm_rd32(device, 0x100e50);
+	const u32  info0 = nvkm_rd32(device, 0x100e54);
+	const u32 insthi = nvkm_rd32(device, 0x100e58);
+	const u32  info1 = nvkm_rd32(device, 0x100e5c);
+
+	info.addr = ((u64)addrhi << 32) | addrlo;
+	info.inst = ((u64)insthi << 32) | (info0 & 0xfffff000);
+	info.time = 0;
+	info.engine = (info0 & 0x000000ff);
+	info.valid  = (info1 & 0x80000000) >> 31;
+	info.gpc    = (info1 & 0x1f000000) >> 24;
+	info.hub    = (info1 & 0x00100000) >> 20;
+	info.access = (info1 & 0x000f0000) >> 16;
+	info.client = (info1 & 0x00007f00) >> 8;
+	info.reason = (info1 & 0x0000001f);
+
+	nvkm_fifo_fault(device->fifo, &info);
+}
+
+static void
+gv100_fault_intr(struct nvkm_fault *fault)
+{
+	struct nvkm_subdev *subdev = &fault->subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 stat = nvkm_rd32(device, 0x100a20);
+
+	if (stat & 0x80000000) {
+		gv100_fault_intr_fault(fault);
+		nvkm_wr32(device, 0x100e60, 0x80000000);
+		stat &= ~0x80000000;
+	}
+
+	if (stat & 0x20000000) {
+		if (fault->buffer[0]) {
+			nvkm_event_send(&fault->event, 1, 0, NULL, 0);
+			stat &= ~0x20000000;
+		}
+	}
+
+	if (stat) {
+		nvkm_debug(subdev, "intr %08x\n", stat);
+	}
+}
+
+static void
+gv100_fault_fini(struct nvkm_fault *fault)
+{
+	nvkm_notify_put(&fault->nrpfb);
+	nvkm_mask(fault->subdev.device, 0x100a34, 0x80000000, 0x80000000);
+}
+
+static void
+gv100_fault_init(struct nvkm_fault *fault)
+{
+	nvkm_mask(fault->subdev.device, 0x100a2c, 0x80000000, 0x80000000);
+	nvkm_notify_get(&fault->nrpfb);
+}
+
+static const struct nvkm_fault_func
+gv100_fault = {
+	.init = gv100_fault_init,
+	.fini = gv100_fault_fini,
+	.intr = gv100_fault_intr,
+	.buffer.nr = 2,
+	.buffer.entry_size = 32,
+	.buffer.entries = gv100_fault_buffer_entries,
+	.buffer.init = gv100_fault_buffer_init,
+	.buffer.fini = gv100_fault_buffer_fini,
+};
+
+int
+gv100_fault_new(struct nvkm_device *device, int index,
+		struct nvkm_fault **pfault)
+{
+	struct nvkm_fault *fault;
+	int ret;
+
+	ret = nvkm_fault_new_(&gv100_fault, device, index, &fault);
+	*pfault = fault;
+	if (ret)
+		return ret;
+
+	return nvkm_notify_init(&fault->buffer[0]->object, &fault->event,
+				gv100_fault_ntfy_nrpfb, false, NULL, 0, 0,
+				&fault->nrpfb);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h
new file mode 100644
index 000000000000..44843ecf12b0
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h
@@ -0,0 +1,34 @@
+#ifndef __NVKM_FAULT_PRIV_H__
+#define __NVKM_FAULT_PRIV_H__
+#define nvkm_fault_buffer(p) container_of((p), struct nvkm_fault_buffer, object)
+#define nvkm_fault(p) container_of((p), struct nvkm_fault, subdev)
+#include <subdev/fault.h>
+
+#include <core/event.h>
+#include <core/object.h>
+
+struct nvkm_fault_buffer {
+	struct nvkm_object object;
+	struct nvkm_fault *fault;
+	int id;
+	int entries;
+	struct nvkm_memory *mem;
+	struct nvkm_vma *vma;
+};
+
+int nvkm_fault_new_(const struct nvkm_fault_func *, struct nvkm_device *,
+		    int index, struct nvkm_fault **);
+
+struct nvkm_fault_func {
+	void (*init)(struct nvkm_fault *);
+	void (*fini)(struct nvkm_fault *);
+	void (*intr)(struct nvkm_fault *);
+	struct {
+		int nr;
+		u32 entry_size;
+		u32 (*entries)(struct nvkm_fault_buffer *);
+		void (*init)(struct nvkm_fault_buffer *);
+		void (*fini)(struct nvkm_fault_buffer *);
+	} buffer;
+};
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
index b4f22cce5d43..969610951263 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
@@ -30,6 +30,7 @@ nvkm-y += nvkm/subdev/fb/gm20b.o
 nvkm-y += nvkm/subdev/fb/gp100.o
 nvkm-y += nvkm/subdev/fb/gp102.o
 nvkm-y += nvkm/subdev/fb/gp10b.o
+nvkm-y += nvkm/subdev/fb/gv100.o
 
 nvkm-y += nvkm/subdev/fb/ram.o
 nvkm-y += nvkm/subdev/fb/ramnv04.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
index cdc4e0a2cc6b..e8dc4e913494 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
@@ -46,10 +46,10 @@ gf100_fb_oneinit(struct nvkm_fb *base)
 {
 	struct gf100_fb *fb = gf100_fb(base);
 	struct nvkm_device *device = fb->base.subdev.device;
-	int ret, size = 0x1000;
+	int ret, size = 1 << (fb->base.page ? fb->base.page : 17);
 
 	size = nvkm_longopt(device->cfgopt, "MmuDebugBufferSize", size);
-	size = min(size, 0x1000);
+	size = max(size, 0x1000);
 
 	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, size, 0x1000,
 			      true, &fb->base.mmu_rd);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
index 8137e19d3292..d3b8c3367152 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
@@ -49,8 +49,6 @@ gm200_fb_init(struct nvkm_fb *base)
 	if (fb->r100c10_page)
 		nvkm_wr32(device, 0x100c10, fb->r100c10 >> 8);
 
-	nvkm_mask(device, 0x100c80, 0x00000001, 0x00000000); /* 128KiB lpg */
-
 	nvkm_wr32(device, 0x100cc8, nvkm_memory_addr(fb->base.mmu_wr) >> 8);
 	nvkm_wr32(device, 0x100ccc, nvkm_memory_addr(fb->base.mmu_rd) >> 8);
 	nvkm_mask(device, 0x100cc4, 0x00060000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c
index 147f69b30cd8..dffe1f5e1071 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c
@@ -26,7 +26,7 @@
 
 #include <core/memory.h>
 
-static void
+void
 gp100_fb_init_unkn(struct nvkm_fb *base)
 {
 	struct nvkm_device *device = gf100_fb(base)->base.subdev.device;
@@ -48,7 +48,7 @@ gp100_fb_init(struct nvkm_fb *base)
 	nvkm_wr32(device, 0x100cc8, nvkm_memory_addr(fb->base.mmu_wr) >> 8);
 	nvkm_wr32(device, 0x100ccc, nvkm_memory_addr(fb->base.mmu_rd) >> 8);
 	nvkm_mask(device, 0x100cc4, 0x00060000,
-		  max(nvkm_memory_size(fb->base.mmu_rd) >> 16, (u64)2) << 17);
+		  min(nvkm_memory_size(fb->base.mmu_rd) >> 16, (u64)2) << 17);
 }
 
 static const struct nvkm_fb_func
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gv100.c
new file mode 100644
index 000000000000..3c5e02e9794a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gv100.c
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "gf100.h"
+#include "ram.h"
+
+static int
+gv100_fb_init_page(struct nvkm_fb *fb)
+{
+	return (fb->page == 16) ? 0 : -EINVAL;
+}
+
+static const struct nvkm_fb_func
+gv100_fb = {
+	.dtor = gf100_fb_dtor,
+	.oneinit = gf100_fb_oneinit,
+	.init = gp100_fb_init,
+	.init_page = gv100_fb_init_page,
+	.init_unkn = gp100_fb_init_unkn,
+	.ram_new = gp100_ram_new,
+	.default_bigpage = 16,
+};
+
+int
+gv100_fb_new(struct nvkm_device *device, int index, struct nvkm_fb **pfb)
+{
+	return gf100_fb_new_(&gv100_fb, device, index, pfb);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
index 414a423e0e55..2857f31466bf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
@@ -68,4 +68,6 @@ int gf100_fb_oneinit(struct nvkm_fb *);
 int gf100_fb_init_page(struct nvkm_fb *);
 
 int gm200_fb_init_page(struct nvkm_fb *);
+
+void gp100_fb_init_unkn(struct nvkm_fb *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild
index 12d6f4f102cb..290ff1c425a9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild
@@ -4,3 +4,4 @@ nvkm-y += nvkm/subdev/ltc/gk104.o
 nvkm-y += nvkm/subdev/ltc/gm107.o
 nvkm-y += nvkm/subdev/ltc/gm200.o
 nvkm-y += nvkm/subdev/ltc/gp100.o
+nvkm-y += nvkm/subdev/ltc/gp102.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c
index 1f185274d3e6..23242179e600 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c
@@ -55,6 +55,14 @@ nvkm_ltc_zbc_depth_get(struct nvkm_ltc *ltc, int index, const u32 depth)
 	return index;
 }
 
+int
+nvkm_ltc_zbc_stencil_get(struct nvkm_ltc *ltc, int index, const u32 stencil)
+{
+	ltc->zbc_stencil[index] = stencil;
+	ltc->func->zbc_clear_stencil(ltc, index, stencil);
+	return index;
+}
+
 void
 nvkm_ltc_invalidate(struct nvkm_ltc *ltc)
 {
@@ -92,6 +100,8 @@ nvkm_ltc_init(struct nvkm_subdev *subdev)
 	for (i = ltc->zbc_min; i <= ltc->zbc_max; i++) {
 		ltc->func->zbc_clear_color(ltc, i, ltc->zbc_color[i]);
 		ltc->func->zbc_clear_depth(ltc, i, ltc->zbc_depth[i]);
+		if (ltc->func->zbc_clear_stencil)
+			ltc->func->zbc_clear_stencil(ltc, i, ltc->zbc_stencil[i]);
 	}
 
 	ltc->func->init(ltc);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c
index e34d42108019..e923ed76d37a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c
@@ -23,7 +23,7 @@
  */
 #include "priv.h"
 
-static void
+void
 gp100_ltc_intr(struct nvkm_ltc *ltc)
 {
 	struct nvkm_device *device = ltc->subdev.device;
@@ -38,7 +38,7 @@ gp100_ltc_intr(struct nvkm_ltc *ltc)
 	}
 }
 
-static int
+int
 gp100_ltc_oneinit(struct nvkm_ltc *ltc)
 {
 	struct nvkm_device *device = ltc->subdev.device;
@@ -48,7 +48,7 @@ gp100_ltc_oneinit(struct nvkm_ltc *ltc)
 	return 0;
 }
 
-static void
+void
 gp100_ltc_init(struct nvkm_ltc *ltc)
 {
 	/*XXX: PMU LS call to setup tagram address */
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp102.c
new file mode 100644
index 000000000000..601747ada655
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp102.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "priv.h"
+
+void
+gp102_ltc_zbc_clear_stencil(struct nvkm_ltc *ltc, int i, const u32 stencil)
+{
+	struct nvkm_device *device = ltc->subdev.device;
+	nvkm_mask(device, 0x17e338, 0x0000000f, i);
+	nvkm_wr32(device, 0x17e204, stencil);
+}
+
+static const struct nvkm_ltc_func
+gp102_ltc = {
+	.oneinit = gp100_ltc_oneinit,
+	.init = gp100_ltc_init,
+	.intr = gp100_ltc_intr,
+	.cbc_clear = gm107_ltc_cbc_clear,
+	.cbc_wait = gm107_ltc_cbc_wait,
+	.zbc = 16,
+	.zbc_clear_color = gm107_ltc_zbc_clear_color,
+	.zbc_clear_depth = gm107_ltc_zbc_clear_depth,
+	.zbc_clear_stencil = gp102_ltc_zbc_clear_stencil,
+	.invalidate = gf100_ltc_invalidate,
+	.flush = gf100_ltc_flush,
+};
+
+int
+gp102_ltc_new(struct nvkm_device *device, int index, struct nvkm_ltc **pltc)
+{
+	return nvkm_ltc_new_(&gp102_ltc, device, index, pltc);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h
index e71cc25cc775..9dcde43c0f3c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h
@@ -19,6 +19,7 @@ struct nvkm_ltc_func {
 	int zbc;
 	void (*zbc_clear_color)(struct nvkm_ltc *, int, const u32[4]);
 	void (*zbc_clear_depth)(struct nvkm_ltc *, int, const u32);
+	void (*zbc_clear_stencil)(struct nvkm_ltc *, int, const u32);
 
 	void (*invalidate)(struct nvkm_ltc *);
 	void (*flush)(struct nvkm_ltc *);
@@ -41,4 +42,8 @@ void gm107_ltc_cbc_clear(struct nvkm_ltc *, u32, u32);
 void gm107_ltc_cbc_wait(struct nvkm_ltc *);
 void gm107_ltc_zbc_clear_color(struct nvkm_ltc *, int, const u32[4]);
 void gm107_ltc_zbc_clear_depth(struct nvkm_ltc *, int, const u32);
+
+int gp100_ltc_oneinit(struct nvkm_ltc *);
+void gp100_ltc_init(struct nvkm_ltc *);
+void gp100_ltc_intr(struct nvkm_ltc *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c
index 7321ad3758c3..43db245eec9a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c
@@ -75,10 +75,28 @@ gp100_mc_intr_mask(struct nvkm_mc *base, u32 mask, u32 intr)
 	spin_unlock_irqrestore(&mc->lock, flags);
 }
 
+const struct nvkm_mc_map
+gp100_mc_intr[] = {
+	{ 0x04000000, NVKM_ENGINE_DISP },
+	{ 0x00000100, NVKM_ENGINE_FIFO },
+	{ 0x00000200, NVKM_SUBDEV_FAULT },
+	{ 0x40000000, NVKM_SUBDEV_IBUS },
+	{ 0x10000000, NVKM_SUBDEV_BUS },
+	{ 0x08000000, NVKM_SUBDEV_FB },
+	{ 0x02000000, NVKM_SUBDEV_LTC },
+	{ 0x01000000, NVKM_SUBDEV_PMU },
+	{ 0x00200000, NVKM_SUBDEV_GPIO },
+	{ 0x00200000, NVKM_SUBDEV_I2C },
+	{ 0x00100000, NVKM_SUBDEV_TIMER },
+	{ 0x00040000, NVKM_SUBDEV_THERM },
+	{ 0x00002000, NVKM_SUBDEV_FB },
+	{},
+};
+
 static const struct nvkm_mc_func
 gp100_mc = {
 	.init = nv50_mc_init,
-	.intr = gk104_mc_intr,
+	.intr = gp100_mc_intr,
 	.intr_unarm = gp100_mc_intr_unarm,
 	.intr_rearm = gp100_mc_intr_rearm,
 	.intr_mask = gp100_mc_intr_mask,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c
index 2283e3b74277..ff8629de97d6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c
@@ -34,7 +34,7 @@ gp10b_mc_init(struct nvkm_mc *mc)
 static const struct nvkm_mc_func
 gp10b_mc = {
 	.init = gp10b_mc_init,
-	.intr = gk104_mc_intr,
+	.intr = gp100_mc_intr,
 	.intr_unarm = gp100_mc_intr_unarm,
 	.intr_rearm = gp100_mc_intr_rearm,
 	.intr_mask = gp100_mc_intr_mask,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h
index 8869d79c2b59..d9e3691d45b7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h
@@ -57,4 +57,6 @@ int gp100_mc_new_(const struct nvkm_mc_func *, struct nvkm_device *, int,
 
 extern const struct nvkm_mc_map gk104_mc_intr[];
 extern const struct nvkm_mc_map gk104_mc_reset[];
+
+extern const struct nvkm_mc_map gp100_mc_intr[];
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild
index 67ee983bb026..58a24e3a0598 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild
@@ -12,6 +12,7 @@ nvkm-y += nvkm/subdev/mmu/gm200.o
 nvkm-y += nvkm/subdev/mmu/gm20b.o
 nvkm-y += nvkm/subdev/mmu/gp100.o
 nvkm-y += nvkm/subdev/mmu/gp10b.o
+nvkm-y += nvkm/subdev/mmu/gv100.o
 
 nvkm-y += nvkm/subdev/mmu/mem.o
 nvkm-y += nvkm/subdev/mmu/memnv04.o
@@ -31,6 +32,7 @@ nvkm-y += nvkm/subdev/mmu/vmmgm200.o
 nvkm-y += nvkm/subdev/mmu/vmmgm20b.o
 nvkm-y += nvkm/subdev/mmu/vmmgp100.o
 nvkm-y += nvkm/subdev/mmu/vmmgp10b.o
+nvkm-y += nvkm/subdev/mmu/vmmgv100.o
 
 nvkm-y += nvkm/subdev/mmu/umem.o
 nvkm-y += nvkm/subdev/mmu/ummu.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c
new file mode 100644
index 000000000000..f666cb57f69e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "mem.h"
+#include "vmm.h"
+
+#include <core/option.h>
+
+#include <nvif/class.h>
+
+static const struct nvkm_mmu_func
+gv100_mmu = {
+	.dma_bits = 47,
+	.mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}},
+	.mem = {{ -1,  0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map },
+	.vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gv100_vmm_new },
+	.kind = gm200_mmu_kind,
+	.kind_sys = true,
+};
+
+int
+gv100_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
+{
+	return nvkm_mmu_new_(&gv100_mmu, device, index, pmmu);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h
index da06e64d8a7d..1a3b0a3724ca 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h
@@ -236,6 +236,9 @@ int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 		  struct lock_class_key *, const char *,
 		  struct nvkm_vmm **);
+int gv100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+		  struct lock_class_key *, const char *,
+		  struct nvkm_vmm **);
 
 #define VMM_PRINT(l,v,p,f,a...) do {                                           \
 	struct nvkm_vmm *_vmm = (v);                                           \
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c
new file mode 100644
index 000000000000..2fa40c16e6d2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+#include <subdev/fb.h>
+#include <subdev/ltc.h>
+
+#include <nvif/ifc00d.h>
+#include <nvif/unpack.h>
+
+int
+gv100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
+{
+	u64 data[2], mask;
+	int ret = gp100_vmm_join(vmm, inst), i;
+	if (ret)
+		return ret;
+
+	nvkm_kmap(inst);
+	data[0] = nvkm_ro32(inst, 0x200);
+	data[1] = nvkm_ro32(inst, 0x204);
+	mask = BIT_ULL(0);
+
+	nvkm_wo32(inst, 0x21c, 0x00000000);
+
+	for (i = 0; i < 64; i++) {
+		if (mask & BIT_ULL(i)) {
+			nvkm_wo32(inst, 0x2a4 + (i * 0x10), data[1]);
+			nvkm_wo32(inst, 0x2a0 + (i * 0x10), data[0]);
+		} else {
+			nvkm_wo32(inst, 0x2a4 + (i * 0x10), 0x00000001);
+			nvkm_wo32(inst, 0x2a0 + (i * 0x10), 0x00000001);
+		}
+		nvkm_wo32(inst, 0x2a8 + (i * 0x10), 0x00000000);
+	}
+
+	nvkm_wo32(inst, 0x298, lower_32_bits(mask));
+	nvkm_wo32(inst, 0x29c, upper_32_bits(mask));
+	nvkm_done(inst);
+	return 0;
+}
+
+static const struct nvkm_vmm_func
+gv100_vmm = {
+	.join = gv100_vmm_join,
+	.part = gf100_vmm_part,
+	.aper = gf100_vmm_aper,
+	.valid = gp100_vmm_valid,
+	.flush = gp100_vmm_flush,
+	.page = {
+		{ 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx },
+		{ 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx },
+		{ 29, &gp100_vmm_desc_16[2], NVKM_VMM_PAGE_Sxxx },
+		{ 21, &gp100_vmm_desc_16[1], NVKM_VMM_PAGE_SVxC },
+		{ 16, &gp100_vmm_desc_16[0], NVKM_VMM_PAGE_SVxC },
+		{ 12, &gp100_vmm_desc_12[0], NVKM_VMM_PAGE_SVHx },
+		{}
+	}
+};
+
+int
+gv100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+	      struct lock_class_key *key, const char *name,
+	      struct nvkm_vmm **pvmm)
+{
+	return nv04_vmm_new_(&gv100_vmm, mmu, 0, addr, size,
+			     argv, argc, key, name, pvmm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp108.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp108.c
index e8c27ec700de..737a8d50a1f2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp108.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp108.c
@@ -65,3 +65,24 @@ MODULE_FIRMWARE("nvidia/gp108/nvdec/scrubber.bin");
 MODULE_FIRMWARE("nvidia/gp108/sec2/desc.bin");
 MODULE_FIRMWARE("nvidia/gp108/sec2/image.bin");
 MODULE_FIRMWARE("nvidia/gp108/sec2/sig.bin");
+
+MODULE_FIRMWARE("nvidia/gv100/acr/bl.bin");
+MODULE_FIRMWARE("nvidia/gv100/acr/unload_bl.bin");
+MODULE_FIRMWARE("nvidia/gv100/acr/ucode_load.bin");
+MODULE_FIRMWARE("nvidia/gv100/acr/ucode_unload.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/fecs_bl.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/fecs_inst.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/fecs_data.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/fecs_sig.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/gpccs_bl.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/gpccs_inst.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/gpccs_data.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/gpccs_sig.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/sw_ctx.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/sw_nonctx.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/sw_bundle_init.bin");
+MODULE_FIRMWARE("nvidia/gv100/gr/sw_method_init.bin");
+MODULE_FIRMWARE("nvidia/gv100/nvdec/scrubber.bin");
+MODULE_FIRMWARE("nvidia/gv100/sec2/desc.bin");
+MODULE_FIRMWARE("nvidia/gv100/sec2/image.bin");
+MODULE_FIRMWARE("nvidia/gv100/sec2/sig.bin");
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c
index 6f10b098676c..1e1f1c635cab 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c
@@ -80,12 +80,11 @@ acr_ls_msgqueue_post_run(struct nvkm_msgqueue *queue,
 			 struct nvkm_falcon *falcon, u32 addr_args)
 {
 	struct nvkm_device *device = falcon->owner->device;
-	u32 cmdline_size = NVKM_MSGQUEUE_CMDLINE_SIZE;
-	u8 buf[cmdline_size];
+	u8 buf[NVKM_MSGQUEUE_CMDLINE_SIZE];
 
-	memset(buf, 0, cmdline_size);
+	memset(buf, 0, sizeof(buf));
 	nvkm_msgqueue_write_cmdline(queue, buf);
-	nvkm_falcon_load_dmem(falcon, buf, addr_args, cmdline_size, 0);
+	nvkm_falcon_load_dmem(falcon, buf, addr_args, sizeof(buf), 0);
 	/* rearm the queue so it will wait for the init message */
 	nvkm_msgqueue_reinit(queue);
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c
index fea4957291da..4f1f3e890650 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c
@@ -48,7 +48,8 @@ gk104_top_oneinit(struct nvkm_top *top)
 		case 0x00000001: /* DATA */
 			inst        = (data & 0x3c000000) >> 26;
 			info->addr  = (data & 0x00fff000);
-			info->fault = (data & 0x000000f8) >> 3;
+			if (data & 0x00000004)
+				info->fault = (data & 0x000003f8) >> 3;
 			break;
 		case 0x00000002: /* ENUM */
 			if (data & 0x00000020)
diff --git a/drivers/gpu/drm/omapdrm/dss/sdi.c b/drivers/gpu/drm/omapdrm/dss/sdi.c
index 68a40ae26f5b..1e2c931f6acf 100644
--- a/drivers/gpu/drm/omapdrm/dss/sdi.c
+++ b/drivers/gpu/drm/omapdrm/dss/sdi.c
@@ -82,7 +82,7 @@ static int sdi_calc_clock_div(struct sdi_device *sdi, unsigned long pclk,
 			      struct dispc_clock_info *dispc_cinfo)
 {
 	int i;
-	struct sdi_clk_calc_ctx ctx = { .sdi = sdi };
+	struct sdi_clk_calc_ctx ctx;
 
 	/*
 	 * DSS fclk gives us very few possibilities, so finding a good pixel
@@ -95,6 +95,9 @@ static int sdi_calc_clock_div(struct sdi_device *sdi, unsigned long pclk,
 		bool ok;
 
 		memset(&ctx, 0, sizeof(ctx));
+
+		ctx.sdi = sdi;
+
 		if (pclk > 1000 * i * i * i)
 			ctx.pck_min = max(pclk - 1000 * i * i * i, 0lu);
 		else
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index 3632854c2b91..ef3b0e3571ec 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -319,6 +319,9 @@ static int omap_modeset_init(struct drm_device *dev)
 	dev->mode_config.max_width = 8192;
 	dev->mode_config.max_height = 8192;
 
+	/* We want the zpos to be normalized */
+	dev->mode_config.normalize_zpos = true;
+
 	dev->mode_config.funcs = &omap_mode_config_funcs;
 	dev->mode_config.helper_private = &omap_mode_config_helper_funcs;
 
diff --git a/drivers/gpu/drm/omapdrm/omap_plane.c b/drivers/gpu/drm/omapdrm/omap_plane.c
index 2899435cad6e..161233cbc9a0 100644
--- a/drivers/gpu/drm/omapdrm/omap_plane.c
+++ b/drivers/gpu/drm/omapdrm/omap_plane.c
@@ -65,7 +65,7 @@ static void omap_plane_atomic_update(struct drm_plane *plane,
 	info.rotation_type = OMAP_DSS_ROT_NONE;
 	info.rotation = DRM_MODE_ROTATE_0;
 	info.global_alpha = 0xff;
-	info.zorder = state->zpos;
+	info.zorder = state->normalized_zpos;
 
 	/* update scanout: */
 	omap_framebuffer_update_scanout(state->fb, state, &info);
diff --git a/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c b/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
index d964d454e4ae..2c9c9722734f 100644
--- a/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
+++ b/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
@@ -238,12 +238,6 @@ static void rpi_touchscreen_i2c_write(struct rpi_touchscreen *ts,
 
 static int rpi_touchscreen_write(struct rpi_touchscreen *ts, u16 reg, u32 val)
 {
-#if 0
-	/* The firmware uses LP DSI transactions like this to bring up
-	 * the hardware, which should be faster than using I2C to then
-	 * pass to the Toshiba.  However, I was unable to get it to
-	 * work.
-	 */
 	u8 msg[] = {
 		reg,
 		reg >> 8,
@@ -253,13 +247,7 @@ static int rpi_touchscreen_write(struct rpi_touchscreen *ts, u16 reg, u32 val)
 		val >> 24,
 	};
 
-	mipi_dsi_dcs_write_buffer(ts->dsi, msg, sizeof(msg));
-#else
-	rpi_touchscreen_i2c_write(ts, REG_WR_ADDRH, reg >> 8);
-	rpi_touchscreen_i2c_write(ts, REG_WR_ADDRL, reg);
-	rpi_touchscreen_i2c_write(ts, REG_WRITEH, val >> 8);
-	rpi_touchscreen_i2c_write(ts, REG_WRITEL, val);
-#endif
+	mipi_dsi_generic_write(ts->dsi, msg, sizeof(msg));
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/pl111/Makefile b/drivers/gpu/drm/pl111/Makefile
index 9c5e8dba8ac6..19a8189dc54f 100644
--- a/drivers/gpu/drm/pl111/Makefile
+++ b/drivers/gpu/drm/pl111/Makefile
@@ -3,6 +3,7 @@ pl111_drm-y +=	pl111_display.o \
 		pl111_versatile.o \
 		pl111_drv.o
 
+pl111_drm-$(CONFIG_ARCH_VEXPRESS) += pl111_vexpress.o
 pl111_drm-$(CONFIG_DEBUG_FS) += pl111_debugfs.o
 
 obj-$(CONFIG_DRM_PL111) += pl111_drm.o
diff --git a/drivers/gpu/drm/pl111/pl111_display.c b/drivers/gpu/drm/pl111/pl111_display.c
index 310646427907..19b0d006a54a 100644
--- a/drivers/gpu/drm/pl111/pl111_display.c
+++ b/drivers/gpu/drm/pl111/pl111_display.c
@@ -120,7 +120,8 @@ static int pl111_display_check(struct drm_simple_display_pipe *pipe,
 }
 
 static void pl111_display_enable(struct drm_simple_display_pipe *pipe,
-				 struct drm_crtc_state *cstate)
+				 struct drm_crtc_state *cstate,
+				 struct drm_plane_state *plane_state)
 {
 	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_plane *plane = &pipe->plane;
@@ -376,19 +377,13 @@ static void pl111_display_disable_vblank(struct drm_simple_display_pipe *pipe)
 	writel(0, priv->regs + priv->ienb);
 }
 
-static int pl111_display_prepare_fb(struct drm_simple_display_pipe *pipe,
-				    struct drm_plane_state *plane_state)
-{
-	return drm_gem_fb_prepare_fb(&pipe->plane, plane_state);
-}
-
 static struct drm_simple_display_pipe_funcs pl111_display_funcs = {
 	.mode_valid = pl111_mode_valid,
 	.check = pl111_display_check,
 	.enable = pl111_display_enable,
 	.disable = pl111_display_disable,
 	.update = pl111_display_update,
-	.prepare_fb = pl111_display_prepare_fb,
+	.prepare_fb = drm_gem_fb_simple_display_pipe_prepare_fb,
 };
 
 static int pl111_clk_div_choose_div(struct clk_hw *hw, unsigned long rate,
diff --git a/drivers/gpu/drm/pl111/pl111_drm.h b/drivers/gpu/drm/pl111/pl111_drm.h
index 8639b2d4ddf7..ce4501d0ab48 100644
--- a/drivers/gpu/drm/pl111/pl111_drm.h
+++ b/drivers/gpu/drm/pl111/pl111_drm.h
@@ -79,6 +79,7 @@ struct pl111_drm_dev_private {
 	const struct pl111_variant_data *variant;
 	void (*variant_display_enable) (struct drm_device *drm, u32 format);
 	void (*variant_display_disable) (struct drm_device *drm);
+	bool use_device_memory;
 };
 
 int pl111_display_init(struct drm_device *dev);
diff --git a/drivers/gpu/drm/pl111/pl111_drv.c b/drivers/gpu/drm/pl111/pl111_drv.c
index 4621259d5387..454ff0804642 100644
--- a/drivers/gpu/drm/pl111/pl111_drv.c
+++ b/drivers/gpu/drm/pl111/pl111_drv.c
@@ -60,6 +60,7 @@
 #include <linux/slab.h>
 #include <linux/of.h>
 #include <linux/of_graph.h>
+#include <linux/of_reserved_mem.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_atomic_helper.h>
@@ -207,6 +208,24 @@ static int pl111_modeset_init(struct drm_device *dev)
 	return ret;
 }
 
+static struct drm_gem_object *
+pl111_gem_import_sg_table(struct drm_device *dev,
+			  struct dma_buf_attachment *attach,
+			  struct sg_table *sgt)
+{
+	struct pl111_drm_dev_private *priv = dev->dev_private;
+
+	/*
+	 * When using device-specific reserved memory we can't import
+	 * DMA buffers: those are passed by reference in any global
+	 * memory and we can only handle a specific range of memory.
+	 */
+	if (priv->use_device_memory)
+		return ERR_PTR(-EINVAL);
+
+	return drm_gem_cma_prime_import_sg_table(dev, attach, sgt);
+}
+
 DEFINE_DRM_GEM_CMA_FOPS(drm_fops);
 
 static struct drm_driver pl111_drm_driver = {
@@ -227,7 +246,7 @@ static struct drm_driver pl111_drm_driver = {
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_import = drm_gem_prime_import,
-	.gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table,
+	.gem_prime_import_sg_table = pl111_gem_import_sg_table,
 	.gem_prime_export = drm_gem_prime_export,
 	.gem_prime_get_sg_table	= drm_gem_cma_prime_get_sg_table,
 
@@ -257,6 +276,12 @@ static int pl111_amba_probe(struct amba_device *amba_dev,
 	drm->dev_private = priv;
 	priv->variant = variant;
 
+	ret = of_reserved_mem_device_init(dev);
+	if (!ret) {
+		dev_info(dev, "using device-specific reserved memory\n");
+		priv->use_device_memory = true;
+	}
+
 	if (of_property_read_u32(dev->of_node, "max-memory-bandwidth",
 				 &priv->memory_bw)) {
 		dev_info(dev, "no max memory bandwidth specified, assume unlimited\n");
@@ -275,7 +300,8 @@ static int pl111_amba_probe(struct amba_device *amba_dev,
 	priv->regs = devm_ioremap_resource(dev, &amba_dev->res);
 	if (IS_ERR(priv->regs)) {
 		dev_err(dev, "%s failed mmio\n", __func__);
-		return PTR_ERR(priv->regs);
+		ret = PTR_ERR(priv->regs);
+		goto dev_unref;
 	}
 
 	/* This may override some variant settings */
@@ -305,11 +331,14 @@ static int pl111_amba_probe(struct amba_device *amba_dev,
 
 dev_unref:
 	drm_dev_unref(drm);
+	of_reserved_mem_device_release(dev);
+
 	return ret;
 }
 
 static int pl111_amba_remove(struct amba_device *amba_dev)
 {
+	struct device *dev = &amba_dev->dev;
 	struct drm_device *drm = amba_get_drvdata(amba_dev);
 	struct pl111_drm_dev_private *priv = drm->dev_private;
 
@@ -319,6 +348,7 @@ static int pl111_amba_remove(struct amba_device *amba_dev)
 		drm_panel_bridge_remove(priv->bridge);
 	drm_mode_config_cleanup(drm);
 	drm_dev_unref(drm);
+	of_reserved_mem_device_release(dev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/pl111/pl111_versatile.c b/drivers/gpu/drm/pl111/pl111_versatile.c
index 9302f516045e..b9baefdba38a 100644
--- a/drivers/gpu/drm/pl111/pl111_versatile.c
+++ b/drivers/gpu/drm/pl111/pl111_versatile.c
@@ -1,12 +1,14 @@
 #include <linux/amba/clcd-regs.h>
 #include <linux/device.h>
 #include <linux/of.h>
+#include <linux/of_platform.h>
 #include <linux/regmap.h>
 #include <linux/mfd/syscon.h>
 #include <linux/bitops.h>
 #include <linux/module.h>
 #include <drm/drmP.h>
 #include "pl111_versatile.h"
+#include "pl111_vexpress.h"
 #include "pl111_drm.h"
 
 static struct regmap *versatile_syscon_map;
@@ -22,6 +24,7 @@ enum versatile_clcd {
 	REALVIEW_CLCD_PB11MP,
 	REALVIEW_CLCD_PBA8,
 	REALVIEW_CLCD_PBX,
+	VEXPRESS_CLCD_V2M,
 };
 
 static const struct of_device_id versatile_clcd_of_match[] = {
@@ -53,6 +56,10 @@ static const struct of_device_id versatile_clcd_of_match[] = {
 		.compatible = "arm,realview-pbx-syscon",
 		.data = (void *)REALVIEW_CLCD_PBX,
 	},
+	{
+		.compatible = "arm,vexpress-muxfpga",
+		.data = (void *)VEXPRESS_CLCD_V2M,
+	},
 	{},
 };
 
@@ -286,12 +293,26 @@ static const struct pl111_variant_data pl111_realview = {
 	.fb_bpp = 16,
 };
 
+/*
+ * Versatile Express PL111 variant, again we just push the maximum
+ * BPP to 16 to be able to get 1024x768 without saturating the memory
+ * bus. The clockdivider also seems broken on the Versatile Express.
+ */
+static const struct pl111_variant_data pl111_vexpress = {
+	.name = "PL111 Versatile Express",
+	.formats = pl111_realview_pixel_formats,
+	.nformats = ARRAY_SIZE(pl111_realview_pixel_formats),
+	.fb_bpp = 16,
+	.broken_clockdivider = true,
+};
+
 int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv)
 {
 	const struct of_device_id *clcd_id;
 	enum versatile_clcd versatile_clcd_type;
 	struct device_node *np;
 	struct regmap *map;
+	int ret;
 
 	np = of_find_matching_node_and_match(NULL, versatile_clcd_of_match,
 					     &clcd_id);
@@ -301,7 +322,33 @@ int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv)
 	}
 	versatile_clcd_type = (enum versatile_clcd)clcd_id->data;
 
-	map = syscon_node_to_regmap(np);
+	/* Versatile Express special handling */
+	if (versatile_clcd_type == VEXPRESS_CLCD_V2M) {
+		struct platform_device *pdev;
+
+		/* Registers a driver for the muxfpga */
+		ret = vexpress_muxfpga_init();
+		if (ret) {
+			dev_err(dev, "unable to initialize muxfpga driver\n");
+			return ret;
+		}
+
+		/* Call into deep Vexpress configuration API */
+		pdev = of_find_device_by_node(np);
+		if (!pdev) {
+			dev_err(dev, "can't find the sysreg device, deferring\n");
+			return -EPROBE_DEFER;
+		}
+		map = dev_get_drvdata(&pdev->dev);
+		if (!map) {
+			dev_err(dev, "sysreg has not yet probed\n");
+			platform_device_put(pdev);
+			return -EPROBE_DEFER;
+		}
+	} else {
+		map = syscon_node_to_regmap(np);
+	}
+
 	if (IS_ERR(map)) {
 		dev_err(dev, "no Versatile syscon regmap\n");
 		return PTR_ERR(map);
@@ -340,6 +387,13 @@ int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv)
 		priv->variant_display_disable = pl111_realview_clcd_disable;
 		dev_info(dev, "set up callbacks for RealView PL111\n");
 		break;
+	case VEXPRESS_CLCD_V2M:
+		priv->variant = &pl111_vexpress;
+		dev_info(dev, "initializing Versatile Express PL111\n");
+		ret = pl111_vexpress_clcd_init(dev, priv, map);
+		if (ret)
+			return ret;
+		break;
 	default:
 		dev_info(dev, "unknown Versatile system controller\n");
 		break;
diff --git a/drivers/gpu/drm/pl111/pl111_vexpress.c b/drivers/gpu/drm/pl111/pl111_vexpress.c
new file mode 100644
index 000000000000..a534b225e31b
--- /dev/null
+++ b/drivers/gpu/drm/pl111/pl111_vexpress.c
@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Versatile Express PL111 handling
+ * Copyright (C) 2018 Linus Walleij
+ *
+ * This module binds to the "arm,vexpress-muxfpga" device on the
+ * Versatile Express configuration bus and sets up which CLCD instance
+ * gets muxed out on the DVI bridge.
+ */
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/vexpress.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include "pl111_drm.h"
+#include "pl111_vexpress.h"
+
+#define VEXPRESS_FPGAMUX_MOTHERBOARD		0x00
+#define VEXPRESS_FPGAMUX_DAUGHTERBOARD_1	0x01
+#define VEXPRESS_FPGAMUX_DAUGHTERBOARD_2	0x02
+
+int pl111_vexpress_clcd_init(struct device *dev,
+			     struct pl111_drm_dev_private *priv,
+			     struct regmap *map)
+{
+	struct device_node *root;
+	struct device_node *child;
+	struct device_node *ct_clcd = NULL;
+	bool has_coretile_clcd = false;
+	bool has_coretile_hdlcd = false;
+	bool mux_motherboard = true;
+	u32 val;
+	int ret;
+
+	/*
+	 * Check if we have a CLCD or HDLCD on the core tile by checking if a
+	 * CLCD or HDLCD is available in the root of the device tree.
+	 */
+	root = of_find_node_by_path("/");
+	if (!root)
+		return -EINVAL;
+
+	for_each_available_child_of_node(root, child) {
+		if (of_device_is_compatible(child, "arm,pl111")) {
+			has_coretile_clcd = true;
+			ct_clcd = child;
+			break;
+		}
+		if (of_device_is_compatible(child, "arm,hdlcd")) {
+			has_coretile_hdlcd = true;
+			break;
+		}
+	}
+
+	/*
+	 * If there is a coretile HDLCD and it has a driver,
+	 * do not mux the CLCD on the motherboard to the DVI.
+	 */
+	if (has_coretile_hdlcd && IS_ENABLED(CONFIG_DRM_HDLCD))
+		mux_motherboard = false;
+
+	/*
+	 * On the Vexpress CA9 we let the CLCD on the coretile
+	 * take precedence, so also in this case do not mux the
+	 * motherboard to the DVI.
+	 */
+	if (has_coretile_clcd)
+		mux_motherboard = false;
+
+	if (mux_motherboard) {
+		dev_info(dev, "DVI muxed to motherboard CLCD\n");
+		val = VEXPRESS_FPGAMUX_MOTHERBOARD;
+	} else if (ct_clcd == dev->of_node) {
+		dev_info(dev,
+			 "DVI muxed to daughterboard 1 (core tile) CLCD\n");
+		val = VEXPRESS_FPGAMUX_DAUGHTERBOARD_1;
+	} else {
+		dev_info(dev, "core tile graphics present\n");
+		dev_info(dev, "this device will be deactivated\n");
+		return -ENODEV;
+	}
+
+	ret = regmap_write(map, 0, val);
+	if (ret) {
+		dev_err(dev, "error setting DVI muxmode\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+/*
+ * This sets up the regmap pointer that will then be retrieved by
+ * the detection code in pl111_versatile.c and passed in to the
+ * pl111_vexpress_clcd_init() function above.
+ */
+static int vexpress_muxfpga_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct regmap *map;
+
+	map = devm_regmap_init_vexpress_config(&pdev->dev);
+	if (IS_ERR(map))
+		return PTR_ERR(map);
+	dev_set_drvdata(dev, map);
+
+	return 0;
+}
+
+static const struct of_device_id vexpress_muxfpga_match[] = {
+	{ .compatible = "arm,vexpress-muxfpga", }
+};
+
+static struct platform_driver vexpress_muxfpga_driver = {
+	.driver = {
+		.name = "vexpress-muxfpga",
+		.of_match_table = of_match_ptr(vexpress_muxfpga_match),
+	},
+	.probe = vexpress_muxfpga_probe,
+};
+
+int vexpress_muxfpga_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&vexpress_muxfpga_driver);
+	/* -EBUSY just means this driver is already registered */
+	if (ret == -EBUSY)
+		ret = 0;
+	return ret;
+}
diff --git a/drivers/gpu/drm/pl111/pl111_vexpress.h b/drivers/gpu/drm/pl111/pl111_vexpress.h
new file mode 100644
index 000000000000..5d3681bb4c00
--- /dev/null
+++ b/drivers/gpu/drm/pl111/pl111_vexpress.h
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-2.0
+
+struct device;
+struct pl111_drm_dev_private;
+struct regmap;
+
+#ifdef CONFIG_ARCH_VEXPRESS
+
+int pl111_vexpress_clcd_init(struct device *dev,
+			     struct pl111_drm_dev_private *priv,
+			     struct regmap *map);
+
+int vexpress_muxfpga_init(void);
+
+#else
+
+static inline int pl111_vexpress_clcd_init(struct device *dev,
+					   struct pl111_drm_dev_private *priv,
+					   struct regmap *map)
+{
+	return -ENODEV;
+}
+
+static inline int vexpress_muxfpga_init(void)
+{
+	return 0;
+}
+
+#endif
diff --git a/drivers/gpu/drm/qxl/qxl_cmd.c b/drivers/gpu/drm/qxl/qxl_cmd.c
index 01665b98c57e..208af9f37914 100644
--- a/drivers/gpu/drm/qxl/qxl_cmd.c
+++ b/drivers/gpu/drm/qxl/qxl_cmd.c
@@ -339,12 +339,9 @@ int qxl_io_update_area(struct qxl_device *qdev, struct qxl_bo *surf,
 	surface_height = surf->surf.height;
 
 	if (area->left < 0 || area->top < 0 ||
-	    area->right > surface_width || area->bottom > surface_height) {
-		qxl_io_log(qdev, "%s: not doing area update for "
-			   "%d, (%d,%d,%d,%d) (%d,%d)\n", __func__, surface_id, area->left,
-			   area->top, area->right, area->bottom, surface_width, surface_height);
+	    area->right > surface_width || area->bottom > surface_height)
 		return -EINVAL;
-	}
+
 	mutex_lock(&qdev->update_area_mutex);
 	qdev->ram_header->update_area = *area;
 	qdev->ram_header->update_surface = surface_id;
@@ -372,6 +369,7 @@ void qxl_io_flush_surfaces(struct qxl_device *qdev)
 void qxl_io_destroy_primary(struct qxl_device *qdev)
 {
 	wait_for_io_cmd(qdev, 0, QXL_IO_DESTROY_PRIMARY_ASYNC);
+	qdev->primary_created = false;
 }
 
 void qxl_io_create_primary(struct qxl_device *qdev,
@@ -397,6 +395,7 @@ void qxl_io_create_primary(struct qxl_device *qdev,
 	create->type = QXL_SURF_TYPE_PRIMARY;
 
 	wait_for_io_cmd(qdev, 0, QXL_IO_CREATE_PRIMARY_ASYNC);
+	qdev->primary_created = true;
 }
 
 void qxl_io_memslot_add(struct qxl_device *qdev, uint8_t id)
@@ -405,20 +404,6 @@ void qxl_io_memslot_add(struct qxl_device *qdev, uint8_t id)
 	wait_for_io_cmd(qdev, id, QXL_IO_MEMSLOT_ADD_ASYNC);
 }
 
-void qxl_io_log(struct qxl_device *qdev, const char *fmt, ...)
-{
-	va_list args;
-
-	va_start(args, fmt);
-	vsnprintf(qdev->ram_header->log_buf, QXL_LOG_BUF_SIZE, fmt, args);
-	va_end(args);
-	/*
-	 * DO not do a DRM output here - this will call printk, which will
-	 * call back into qxl for rendering (qxl_fb)
-	 */
-	outb(0, qdev->io_base + QXL_IO_LOG);
-}
-
 void qxl_io_reset(struct qxl_device *qdev)
 {
 	outb(0, qdev->io_base + QXL_IO_RESET);
@@ -426,19 +411,6 @@ void qxl_io_reset(struct qxl_device *qdev)
 
 void qxl_io_monitors_config(struct qxl_device *qdev)
 {
-	qxl_io_log(qdev, "%s: %d [%dx%d+%d+%d]\n", __func__,
-		   qdev->monitors_config ?
-		   qdev->monitors_config->count : -1,
-		   qdev->monitors_config && qdev->monitors_config->count ?
-		   qdev->monitors_config->heads[0].width : -1,
-		   qdev->monitors_config && qdev->monitors_config->count ?
-		   qdev->monitors_config->heads[0].height : -1,
-		   qdev->monitors_config && qdev->monitors_config->count ?
-		   qdev->monitors_config->heads[0].x : -1,
-		   qdev->monitors_config && qdev->monitors_config->count ?
-		   qdev->monitors_config->heads[0].y : -1
-		   );
-
 	wait_for_io_cmd(qdev, 0, QXL_IO_MONITORS_CONFIG_ASYNC);
 }
 
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index ecb35ed0eac8..b8cda9449241 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -48,12 +48,8 @@ static void qxl_alloc_client_monitors_config(struct qxl_device *qdev, unsigned c
 		qdev->client_monitors_config = kzalloc(
 				sizeof(struct qxl_monitors_config) +
 				sizeof(struct qxl_head) * count, GFP_KERNEL);
-		if (!qdev->client_monitors_config) {
-			qxl_io_log(qdev,
-				   "%s: allocation failure for %u heads\n",
-				   __func__, count);
+		if (!qdev->client_monitors_config)
 			return;
-		}
 	}
 	qdev->client_monitors_config->count = count;
 }
@@ -74,12 +70,8 @@ static int qxl_display_copy_rom_client_monitors_config(struct qxl_device *qdev)
 	num_monitors = qdev->rom->client_monitors_config.count;
 	crc = crc32(0, (const uint8_t *)&qdev->rom->client_monitors_config,
 		  sizeof(qdev->rom->client_monitors_config));
-	if (crc != qdev->rom->client_monitors_config_crc) {
-		qxl_io_log(qdev, "crc mismatch: have %X (%zd) != %X\n", crc,
-			   sizeof(qdev->rom->client_monitors_config),
-			   qdev->rom->client_monitors_config_crc);
+	if (crc != qdev->rom->client_monitors_config_crc)
 		return MONITORS_CONFIG_BAD_CRC;
-	}
 	if (!num_monitors) {
 		DRM_DEBUG_KMS("no client monitors configured\n");
 		return status;
@@ -170,12 +162,10 @@ void qxl_display_read_client_monitors_config(struct qxl_device *qdev)
 		udelay(5);
 	}
 	if (status == MONITORS_CONFIG_BAD_CRC) {
-		qxl_io_log(qdev, "config: bad crc\n");
 		DRM_DEBUG_KMS("ignoring client monitors config: bad crc");
 		return;
 	}
 	if (status == MONITORS_CONFIG_UNCHANGED) {
-		qxl_io_log(qdev, "config: unchanged\n");
 		DRM_DEBUG_KMS("ignoring client monitors config: unchanged");
 		return;
 	}
@@ -268,6 +258,89 @@ static int qxl_add_common_modes(struct drm_connector *connector,
 	return i - 1;
 }
 
+static void qxl_send_monitors_config(struct qxl_device *qdev)
+{
+	int i;
+
+	BUG_ON(!qdev->ram_header->monitors_config);
+
+	if (qdev->monitors_config->count == 0)
+		return;
+
+	for (i = 0 ; i < qdev->monitors_config->count ; ++i) {
+		struct qxl_head *head = &qdev->monitors_config->heads[i];
+
+		if (head->y > 8192 || head->x > 8192 ||
+		    head->width > 8192 || head->height > 8192) {
+			DRM_ERROR("head %d wrong: %dx%d+%d+%d\n",
+				  i, head->width, head->height,
+				  head->x, head->y);
+			return;
+		}
+	}
+	qxl_io_monitors_config(qdev);
+}
+
+static void qxl_crtc_update_monitors_config(struct drm_crtc *crtc,
+					    const char *reason)
+{
+	struct drm_device *dev = crtc->dev;
+	struct qxl_device *qdev = dev->dev_private;
+	struct qxl_crtc *qcrtc = to_qxl_crtc(crtc);
+	struct qxl_head head;
+	int oldcount, i = qcrtc->index;
+
+	if (!qdev->primary_created) {
+		DRM_DEBUG_KMS("no primary surface, skip (%s)\n", reason);
+		return;
+	}
+
+	if (!qdev->monitors_config ||
+	    qdev->monitors_config->max_allowed <= i)
+		return;
+
+	head.id = i;
+	head.flags = 0;
+	oldcount = qdev->monitors_config->count;
+	if (crtc->state->active) {
+		struct drm_display_mode *mode = &crtc->mode;
+		head.width = mode->hdisplay;
+		head.height = mode->vdisplay;
+		head.x = crtc->x;
+		head.y = crtc->y;
+		if (qdev->monitors_config->count < i + 1)
+			qdev->monitors_config->count = i + 1;
+	} else if (i > 0) {
+		head.width = 0;
+		head.height = 0;
+		head.x = 0;
+		head.y = 0;
+		if (qdev->monitors_config->count == i + 1)
+			qdev->monitors_config->count = i;
+	} else {
+		DRM_DEBUG_KMS("inactive head 0, skip (%s)\n", reason);
+		return;
+	}
+
+	if (head.width  == qdev->monitors_config->heads[i].width  &&
+	    head.height == qdev->monitors_config->heads[i].height &&
+	    head.x      == qdev->monitors_config->heads[i].x      &&
+	    head.y      == qdev->monitors_config->heads[i].y      &&
+	    oldcount    == qdev->monitors_config->count)
+		return;
+
+	DRM_DEBUG_KMS("head %d, %dx%d, at +%d+%d, %s (%s)\n",
+		      i, head.width, head.height, head.x, head.y,
+		      crtc->state->active ? "on" : "off", reason);
+	if (oldcount != qdev->monitors_config->count)
+		DRM_DEBUG_KMS("active heads %d -> %d (%d total)\n",
+			      oldcount, qdev->monitors_config->count,
+			      qdev->monitors_config->max_allowed);
+
+	qdev->monitors_config->heads[i] = head;
+	qxl_send_monitors_config(qdev);
+}
+
 static void qxl_crtc_atomic_flush(struct drm_crtc *crtc,
 				  struct drm_crtc_state *old_crtc_state)
 {
@@ -283,6 +356,8 @@ static void qxl_crtc_atomic_flush(struct drm_crtc *crtc,
 		drm_crtc_send_vblank_event(crtc, event);
 		spin_unlock_irqrestore(&dev->event_lock, flags);
 	}
+
+	qxl_crtc_update_monitors_config(crtc, "flush");
 }
 
 static void qxl_crtc_destroy(struct drm_crtc *crtc)
@@ -381,95 +456,19 @@ qxl_framebuffer_init(struct drm_device *dev,
 	return 0;
 }
 
-static bool qxl_crtc_mode_fixup(struct drm_crtc *crtc,
-				  const struct drm_display_mode *mode,
-				  struct drm_display_mode *adjusted_mode)
-{
-	struct drm_device *dev = crtc->dev;
-	struct qxl_device *qdev = dev->dev_private;
-
-	qxl_io_log(qdev, "%s: (%d,%d) => (%d,%d)\n",
-		   __func__,
-		   mode->hdisplay, mode->vdisplay,
-		   adjusted_mode->hdisplay,
-		   adjusted_mode->vdisplay);
-	return true;
-}
-
-static void
-qxl_send_monitors_config(struct qxl_device *qdev)
-{
-	int i;
-
-	BUG_ON(!qdev->ram_header->monitors_config);
-
-	if (qdev->monitors_config->count == 0) {
-		qxl_io_log(qdev, "%s: 0 monitors??\n", __func__);
-		return;
-	}
-	for (i = 0 ; i < qdev->monitors_config->count ; ++i) {
-		struct qxl_head *head = &qdev->monitors_config->heads[i];
-
-		if (head->y > 8192 || head->x > 8192 ||
-		    head->width > 8192 || head->height > 8192) {
-			DRM_ERROR("head %d wrong: %dx%d+%d+%d\n",
-				  i, head->width, head->height,
-				  head->x, head->y);
-			return;
-		}
-	}
-	qxl_io_monitors_config(qdev);
-}
-
-static void qxl_monitors_config_set(struct qxl_device *qdev,
-				    int index,
-				    unsigned x, unsigned y,
-				    unsigned width, unsigned height,
-				    unsigned surf_id)
-{
-	DRM_DEBUG_KMS("%d:%dx%d+%d+%d\n", index, width, height, x, y);
-	qdev->monitors_config->heads[index].x = x;
-	qdev->monitors_config->heads[index].y = y;
-	qdev->monitors_config->heads[index].width = width;
-	qdev->monitors_config->heads[index].height = height;
-	qdev->monitors_config->heads[index].surface_id = surf_id;
-
-}
-
-static void qxl_mode_set_nofb(struct drm_crtc *crtc)
-{
-	struct qxl_device *qdev = crtc->dev->dev_private;
-	struct qxl_crtc *qcrtc = to_qxl_crtc(crtc);
-	struct drm_display_mode *mode = &crtc->mode;
-
-	DRM_DEBUG("Mode set (%d,%d)\n",
-		  mode->hdisplay, mode->vdisplay);
-
-	qxl_monitors_config_set(qdev, qcrtc->index, 0, 0,
-				mode->hdisplay,	mode->vdisplay, 0);
-
-}
-
 static void qxl_crtc_atomic_enable(struct drm_crtc *crtc,
 				   struct drm_crtc_state *old_state)
 {
-	DRM_DEBUG("\n");
+	qxl_crtc_update_monitors_config(crtc, "enable");
 }
 
 static void qxl_crtc_atomic_disable(struct drm_crtc *crtc,
 				    struct drm_crtc_state *old_state)
 {
-	struct qxl_crtc *qcrtc = to_qxl_crtc(crtc);
-	struct qxl_device *qdev = crtc->dev->dev_private;
-
-	qxl_monitors_config_set(qdev, qcrtc->index, 0, 0, 0, 0, 0);
-
-	qxl_send_monitors_config(qdev);
+	qxl_crtc_update_monitors_config(crtc, "disable");
 }
 
 static const struct drm_crtc_helper_funcs qxl_crtc_helper_funcs = {
-	.mode_fixup = qxl_crtc_mode_fixup,
-	.mode_set_nofb = qxl_mode_set_nofb,
 	.atomic_flush = qxl_crtc_atomic_flush,
 	.atomic_enable = qxl_crtc_atomic_enable,
 	.atomic_disable = qxl_crtc_atomic_disable,
@@ -613,12 +612,6 @@ static void qxl_primary_atomic_disable(struct drm_plane *plane,
 	}
 }
 
-static int qxl_plane_atomic_check(struct drm_plane *plane,
-				  struct drm_plane_state *state)
-{
-	return 0;
-}
-
 static void qxl_cursor_atomic_update(struct drm_plane *plane,
 				     struct drm_plane_state *old_state)
 {
@@ -824,7 +817,6 @@ static const uint32_t qxl_cursor_plane_formats[] = {
 };
 
 static const struct drm_plane_helper_funcs qxl_cursor_helper_funcs = {
-	.atomic_check = qxl_plane_atomic_check,
 	.atomic_update = qxl_cursor_atomic_update,
 	.atomic_disable = qxl_cursor_atomic_disable,
 	.prepare_fb = qxl_plane_prepare_fb,
@@ -949,81 +941,6 @@ static int qdev_crtc_init(struct drm_device *dev, int crtc_id)
 	return r;
 }
 
-static void qxl_enc_dpms(struct drm_encoder *encoder, int mode)
-{
-	DRM_DEBUG("\n");
-}
-
-static void qxl_enc_prepare(struct drm_encoder *encoder)
-{
-	DRM_DEBUG("\n");
-}
-
-static void qxl_write_monitors_config_for_encoder(struct qxl_device *qdev,
-		struct drm_encoder *encoder)
-{
-	int i;
-	struct qxl_output *output = drm_encoder_to_qxl_output(encoder);
-	struct qxl_head *head;
-	struct drm_display_mode *mode;
-
-	BUG_ON(!encoder);
-	/* TODO: ugly, do better */
-	i = output->index;
-	if (!qdev->monitors_config ||
-	    qdev->monitors_config->max_allowed <= i) {
-		DRM_ERROR(
-		"head number too large or missing monitors config: %p, %d",
-		qdev->monitors_config,
-		qdev->monitors_config ?
-			qdev->monitors_config->max_allowed : -1);
-		return;
-	}
-	if (!encoder->crtc) {
-		DRM_ERROR("missing crtc on encoder %p\n", encoder);
-		return;
-	}
-	if (i != 0)
-		DRM_DEBUG("missing for multiple monitors: no head holes\n");
-	head = &qdev->monitors_config->heads[i];
-	head->id = i;
-	if (encoder->crtc->enabled) {
-		mode = &encoder->crtc->mode;
-		head->width = mode->hdisplay;
-		head->height = mode->vdisplay;
-		head->x = encoder->crtc->x;
-		head->y = encoder->crtc->y;
-		if (qdev->monitors_config->count < i + 1)
-			qdev->monitors_config->count = i + 1;
-	} else {
-		head->width = 0;
-		head->height = 0;
-		head->x = 0;
-		head->y = 0;
-	}
-	DRM_DEBUG_KMS("setting head %d to +%d+%d %dx%d out of %d\n",
-		      i, head->x, head->y, head->width, head->height, qdev->monitors_config->count);
-	head->flags = 0;
-	/* TODO - somewhere else to call this for multiple monitors
-	 * (config_commit?) */
-	qxl_send_monitors_config(qdev);
-}
-
-static void qxl_enc_commit(struct drm_encoder *encoder)
-{
-	struct qxl_device *qdev = encoder->dev->dev_private;
-
-	qxl_write_monitors_config_for_encoder(qdev, encoder);
-	DRM_DEBUG("\n");
-}
-
-static void qxl_enc_mode_set(struct drm_encoder *encoder,
-				struct drm_display_mode *mode,
-				struct drm_display_mode *adjusted_mode)
-{
-	DRM_DEBUG("\n");
-}
-
 static int qxl_conn_get_modes(struct drm_connector *connector)
 {
 	unsigned pwidth = 1024;
@@ -1037,7 +954,7 @@ static int qxl_conn_get_modes(struct drm_connector *connector)
 	return ret;
 }
 
-static int qxl_conn_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status qxl_conn_mode_valid(struct drm_connector *connector,
 			       struct drm_display_mode *mode)
 {
 	struct drm_device *ddev = connector->dev;
@@ -1069,10 +986,6 @@ static struct drm_encoder *qxl_best_encoder(struct drm_connector *connector)
 
 
 static const struct drm_encoder_helper_funcs qxl_enc_helper_funcs = {
-	.dpms = qxl_enc_dpms,
-	.prepare = qxl_enc_prepare,
-	.mode_set = qxl_enc_mode_set,
-	.commit = qxl_enc_commit,
 };
 
 static const struct drm_connector_helper_funcs qxl_connector_helper_funcs = {
@@ -1100,21 +1013,11 @@ static enum drm_connector_status qxl_conn_detect(
 		     qxl_head_enabled(&qdev->client_monitors_config->heads[output->index]);
 
 	DRM_DEBUG("#%d connected: %d\n", output->index, connected);
-	if (!connected)
-		qxl_monitors_config_set(qdev, output->index, 0, 0, 0, 0, 0);
 
 	return connected ? connector_status_connected
 			 : connector_status_disconnected;
 }
 
-static int qxl_conn_set_property(struct drm_connector *connector,
-				   struct drm_property *property,
-				   uint64_t value)
-{
-	DRM_DEBUG("\n");
-	return 0;
-}
-
 static void qxl_conn_destroy(struct drm_connector *connector)
 {
 	struct qxl_output *qxl_output =
@@ -1129,7 +1032,6 @@ static const struct drm_connector_funcs qxl_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.detect = qxl_conn_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = qxl_conn_set_property,
 	.destroy = qxl_conn_destroy,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 864b456080c4..01220d386b0a 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -299,9 +299,6 @@ struct qxl_device {
 	int monitors_config_height;
 };
 
-/* forward declaration for QXL_INFO_IO */
-__printf(2,3) void qxl_io_log(struct qxl_device *qdev, const char *fmt, ...);
-
 extern const struct drm_ioctl_desc qxl_ioctls[];
 extern int qxl_max_ioctl;
 
diff --git a/drivers/gpu/drm/qxl/qxl_fb.c b/drivers/gpu/drm/qxl/qxl_fb.c
index 338891401f35..9a6752606079 100644
--- a/drivers/gpu/drm/qxl/qxl_fb.c
+++ b/drivers/gpu/drm/qxl/qxl_fb.c
@@ -185,8 +185,6 @@ static int qxlfb_framebuffer_dirty(struct drm_framebuffer *fb,
 	/*
 	 * we are using a shadow draw buffer, at qdev->surface0_shadow
 	 */
-	qxl_io_log(qdev, "dirty x[%d, %d], y[%d, %d]\n", clips->x1, clips->x2,
-		   clips->y1, clips->y2);
 	image->dx = clips->x1;
 	image->dy = clips->y1;
 	image->width = clips->x2 - clips->x1;
diff --git a/drivers/gpu/drm/qxl/qxl_irq.c b/drivers/gpu/drm/qxl/qxl_irq.c
index 23a40106ab53..3bb31add6350 100644
--- a/drivers/gpu/drm/qxl/qxl_irq.c
+++ b/drivers/gpu/drm/qxl/qxl_irq.c
@@ -57,10 +57,9 @@ irqreturn_t qxl_irq_handler(int irq, void *arg)
 		 * to avoid endless loops).
 		 */
 		qdev->irq_received_error++;
-		qxl_io_log(qdev, "%s: driver is in bug mode.\n", __func__);
+		DRM_WARN("driver is in bug mode\n");
 	}
 	if (pending & QXL_INTERRUPT_CLIENT_MONITORS_CONFIG) {
-		qxl_io_log(qdev, "QXL_INTERRUPT_CLIENT_MONITORS_CONFIG\n");
 		schedule_work(&qdev->client_monitors_config_work);
 	}
 	qdev->ram_header->int_mask = QXL_INTERRUPT_MASK;
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index ee2340e31f06..86a1fb32f6db 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -105,16 +105,16 @@ static void qxl_ttm_global_fini(struct qxl_device *qdev)
 static struct vm_operations_struct qxl_ttm_vm_ops;
 static const struct vm_operations_struct *ttm_vm_ops;
 
-static int qxl_ttm_fault(struct vm_fault *vmf)
+static vm_fault_t qxl_ttm_fault(struct vm_fault *vmf)
 {
 	struct ttm_buffer_object *bo;
-	int r;
+	vm_fault_t ret;
 
 	bo = (struct ttm_buffer_object *)vmf->vma->vm_private_data;
 	if (bo == NULL)
 		return VM_FAULT_NOPAGE;
-	r = ttm_vm_ops->fault(vmf);
-	return r;
+	ret = ttm_vm_ops->fault(vmf);
+	return ret;
 }
 
 int qxl_mmap(struct file *filp, struct vm_area_struct *vma)
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index 02baaaf20e9d..efbd5816082d 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -1145,7 +1145,6 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
-	struct radeon_framebuffer *radeon_fb;
 	struct drm_framebuffer *target_fb;
 	struct drm_gem_object *obj;
 	struct radeon_bo *rbo;
@@ -1164,19 +1163,15 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 		return 0;
 	}
 
-	if (atomic) {
-		radeon_fb = to_radeon_framebuffer(fb);
+	if (atomic)
 		target_fb = fb;
-	}
-	else {
-		radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
+	else
 		target_fb = crtc->primary->fb;
-	}
 
 	/* If atomic, assume fb object is pinned & idle & fenced and
 	 * just update base pointers
 	 */
-	obj = radeon_fb->obj;
+	obj = target_fb->obj[0];
 	rbo = gem_to_radeon_bo(obj);
 	r = radeon_bo_reserve(rbo, false);
 	if (unlikely(r != 0))
@@ -1441,8 +1436,7 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(EVERGREEN_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
-		radeon_fb = to_radeon_framebuffer(fb);
-		rbo = gem_to_radeon_bo(radeon_fb->obj);
+		rbo = gem_to_radeon_bo(fb->obj[0]);
 		r = radeon_bo_reserve(rbo, false);
 		if (unlikely(r != 0))
 			return r;
@@ -1463,7 +1457,6 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
-	struct radeon_framebuffer *radeon_fb;
 	struct drm_gem_object *obj;
 	struct radeon_bo *rbo;
 	struct drm_framebuffer *target_fb;
@@ -1481,16 +1474,12 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 		return 0;
 	}
 
-	if (atomic) {
-		radeon_fb = to_radeon_framebuffer(fb);
+	if (atomic)
 		target_fb = fb;
-	}
-	else {
-		radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
+	else
 		target_fb = crtc->primary->fb;
-	}
 
-	obj = radeon_fb->obj;
+	obj = target_fb->obj[0];
 	rbo = gem_to_radeon_bo(obj);
 	r = radeon_bo_reserve(rbo, false);
 	if (unlikely(r != 0))
@@ -1641,8 +1630,7 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 3);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
-		radeon_fb = to_radeon_framebuffer(fb);
-		rbo = gem_to_radeon_bo(radeon_fb->obj);
+		rbo = gem_to_radeon_bo(fb->obj[0]);
 		r = radeon_bo_reserve(rbo, false);
 		if (unlikely(r != 0))
 			return r;
@@ -2149,11 +2137,9 @@ static void atombios_crtc_disable(struct drm_crtc *crtc)
 	atombios_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
 	if (crtc->primary->fb) {
 		int r;
-		struct radeon_framebuffer *radeon_fb;
 		struct radeon_bo *rbo;
 
-		radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
-		rbo = gem_to_radeon_bo(radeon_fb->obj);
+		rbo = gem_to_radeon_bo(crtc->primary->fb->obj[0]);
 		r = radeon_bo_reserve(rbo, false);
 		if (unlikely(r))
 			DRM_ERROR("failed to reserve rbo before unpin\n");
diff --git a/drivers/gpu/drm/radeon/radeon_atpx_handler.c b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
index 40be4068ca69..fa5fadaa9bbb 100644
--- a/drivers/gpu/drm/radeon/radeon_atpx_handler.c
+++ b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
@@ -526,7 +526,7 @@ static int radeon_atpx_init(void)
  * look up whether we are the integrated or discrete GPU (all asics).
  * Returns the client id.
  */
-static int radeon_atpx_get_client_id(struct pci_dev *pdev)
+static enum vga_switcheroo_client_id radeon_atpx_get_client_id(struct pci_dev *pdev)
 {
 	if (radeon_atpx_priv.dhandle == ACPI_HANDLE(&pdev->dev))
 		return VGA_SWITCHEROO_IGD;
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index df9469a8fdb1..2aea2bdff99b 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -852,7 +852,7 @@ static int radeon_lvds_get_modes(struct drm_connector *connector)
 	return ret;
 }
 
-static int radeon_lvds_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status radeon_lvds_mode_valid(struct drm_connector *connector,
 				  struct drm_display_mode *mode)
 {
 	struct drm_encoder *encoder = radeon_best_single_encoder(connector);
@@ -1012,7 +1012,7 @@ static int radeon_vga_get_modes(struct drm_connector *connector)
 	return ret;
 }
 
-static int radeon_vga_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status radeon_vga_mode_valid(struct drm_connector *connector,
 				  struct drm_display_mode *mode)
 {
 	struct drm_device *dev = connector->dev;
@@ -1156,7 +1156,7 @@ static int radeon_tv_get_modes(struct drm_connector *connector)
 	return 1;
 }
 
-static int radeon_tv_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status radeon_tv_mode_valid(struct drm_connector *connector,
 				struct drm_display_mode *mode)
 {
 	if ((mode->hdisplay > 1024) || (mode->vdisplay > 768))
@@ -1498,7 +1498,7 @@ static void radeon_dvi_force(struct drm_connector *connector)
 		radeon_connector->use_digital = true;
 }
 
-static int radeon_dvi_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status radeon_dvi_mode_valid(struct drm_connector *connector,
 				  struct drm_display_mode *mode)
 {
 	struct drm_device *dev = connector->dev;
@@ -1800,7 +1800,7 @@ radeon_dp_detect(struct drm_connector *connector, bool force)
 	return ret;
 }
 
-static int radeon_dp_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status radeon_dp_mode_valid(struct drm_connector *connector,
 				  struct drm_display_mode *mode)
 {
 	struct drm_device *dev = connector->dev;
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 48d0e6bd0508..59c8a6647ff2 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -1591,7 +1591,7 @@ int radeon_suspend_kms(struct drm_device *dev, bool suspend,
 	/* unpin the front buffers and cursors */
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
-		struct radeon_framebuffer *rfb = to_radeon_framebuffer(crtc->primary->fb);
+		struct drm_framebuffer *fb = crtc->primary->fb;
 		struct radeon_bo *robj;
 
 		if (radeon_crtc->cursor_bo) {
@@ -1603,10 +1603,10 @@ int radeon_suspend_kms(struct drm_device *dev, bool suspend,
 			}
 		}
 
-		if (rfb == NULL || rfb->obj == NULL) {
+		if (fb == NULL || fb->obj[0] == NULL) {
 			continue;
 		}
-		robj = gem_to_radeon_bo(rfb->obj);
+		robj = gem_to_radeon_bo(fb->obj[0]);
 		/* don't unpin kernel fb objects */
 		if (!radeon_fbdev_robj_is_fb(rdev, robj)) {
 			r = radeon_bo_reserve(robj, false);
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 26129b2b082d..9d3ac8b981da 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -32,6 +32,7 @@
 
 #include <linux/pm_runtime.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_edid.h>
@@ -478,8 +479,6 @@ static int radeon_crtc_page_flip_target(struct drm_crtc *crtc,
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
-	struct radeon_framebuffer *old_radeon_fb;
-	struct radeon_framebuffer *new_radeon_fb;
 	struct drm_gem_object *obj;
 	struct radeon_flip_work *work;
 	struct radeon_bo *new_rbo;
@@ -501,15 +500,13 @@ static int radeon_crtc_page_flip_target(struct drm_crtc *crtc,
 	work->async = (page_flip_flags & DRM_MODE_PAGE_FLIP_ASYNC) != 0;
 
 	/* schedule unpin of the old buffer */
-	old_radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
-	obj = old_radeon_fb->obj;
+	obj = crtc->primary->fb->obj[0];
 
 	/* take a reference to the old object */
 	drm_gem_object_get(obj);
 	work->old_rbo = gem_to_radeon_bo(obj);
 
-	new_radeon_fb = to_radeon_framebuffer(fb);
-	obj = new_radeon_fb->obj;
+	obj = fb->obj[0];
 	new_rbo = gem_to_radeon_bo(obj);
 
 	/* pin the new buffer */
@@ -1285,41 +1282,23 @@ void radeon_compute_pll_legacy(struct radeon_pll *pll,
 
 }
 
-static void radeon_user_framebuffer_destroy(struct drm_framebuffer *fb)
-{
-	struct radeon_framebuffer *radeon_fb = to_radeon_framebuffer(fb);
-
-	drm_gem_object_put_unlocked(radeon_fb->obj);
-	drm_framebuffer_cleanup(fb);
-	kfree(radeon_fb);
-}
-
-static int radeon_user_framebuffer_create_handle(struct drm_framebuffer *fb,
-						  struct drm_file *file_priv,
-						  unsigned int *handle)
-{
-	struct radeon_framebuffer *radeon_fb = to_radeon_framebuffer(fb);
-
-	return drm_gem_handle_create(file_priv, radeon_fb->obj, handle);
-}
-
 static const struct drm_framebuffer_funcs radeon_fb_funcs = {
-	.destroy = radeon_user_framebuffer_destroy,
-	.create_handle = radeon_user_framebuffer_create_handle,
+	.destroy = drm_gem_fb_destroy,
+	.create_handle = drm_gem_fb_create_handle,
 };
 
 int
 radeon_framebuffer_init(struct drm_device *dev,
-			struct radeon_framebuffer *rfb,
+			struct drm_framebuffer *fb,
 			const struct drm_mode_fb_cmd2 *mode_cmd,
 			struct drm_gem_object *obj)
 {
 	int ret;
-	rfb->obj = obj;
-	drm_helper_mode_fill_fb_struct(dev, &rfb->base, mode_cmd);
-	ret = drm_framebuffer_init(dev, &rfb->base, &radeon_fb_funcs);
+	fb->obj[0] = obj;
+	drm_helper_mode_fill_fb_struct(dev, fb, mode_cmd);
+	ret = drm_framebuffer_init(dev, fb, &radeon_fb_funcs);
 	if (ret) {
-		rfb->obj = NULL;
+		fb->obj[0] = NULL;
 		return ret;
 	}
 	return 0;
@@ -1331,7 +1310,7 @@ radeon_user_framebuffer_create(struct drm_device *dev,
 			       const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_gem_object *obj;
-	struct radeon_framebuffer *radeon_fb;
+	struct drm_framebuffer *fb;
 	int ret;
 
 	obj = drm_gem_object_lookup(file_priv, mode_cmd->handles[0]);
@@ -1347,20 +1326,20 @@ radeon_user_framebuffer_create(struct drm_device *dev,
 		return ERR_PTR(-EINVAL);
 	}
 
-	radeon_fb = kzalloc(sizeof(*radeon_fb), GFP_KERNEL);
-	if (radeon_fb == NULL) {
+	fb = kzalloc(sizeof(*fb), GFP_KERNEL);
+	if (fb == NULL) {
 		drm_gem_object_put_unlocked(obj);
 		return ERR_PTR(-ENOMEM);
 	}
 
-	ret = radeon_framebuffer_init(dev, radeon_fb, mode_cmd, obj);
+	ret = radeon_framebuffer_init(dev, fb, mode_cmd, obj);
 	if (ret) {
-		kfree(radeon_fb);
+		kfree(fb);
 		drm_gem_object_put_unlocked(obj);
 		return ERR_PTR(ret);
 	}
 
-	return &radeon_fb->base;
+	return fb;
 }
 
 static const struct drm_mode_config_funcs radeon_mode_funcs = {
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index b28288a781ef..2a7977a23b31 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -168,7 +168,12 @@ int radeon_no_wb;
 int radeon_modeset = -1;
 int radeon_dynclks = -1;
 int radeon_r4xx_atom = 0;
+#ifdef __powerpc__
+/* Default to PCI on PowerPC (fdo #95017) */
+int radeon_agpmode = -1;
+#else
 int radeon_agpmode = 0;
+#endif
 int radeon_vram_limit = 0;
 int radeon_gart_size = -1; /* auto */
 int radeon_benchmarking = 0;
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index 57c5404a1654..1179034024ae 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -43,7 +43,7 @@
  */
 struct radeon_fbdev {
 	struct drm_fb_helper helper;
-	struct radeon_framebuffer rfb;
+	struct drm_framebuffer fb;
 	struct radeon_device *rdev;
 };
 
@@ -246,13 +246,13 @@ static int radeonfb_create(struct drm_fb_helper *helper,
 
 	info->par = rfbdev;
 
-	ret = radeon_framebuffer_init(rdev->ddev, &rfbdev->rfb, &mode_cmd, gobj);
+	ret = radeon_framebuffer_init(rdev->ddev, &rfbdev->fb, &mode_cmd, gobj);
 	if (ret) {
 		DRM_ERROR("failed to initialize framebuffer %d\n", ret);
 		goto out;
 	}
 
-	fb = &rfbdev->rfb.base;
+	fb = &rfbdev->fb;
 
 	/* setup helper */
 	rfbdev->helper.fb = fb;
@@ -308,15 +308,15 @@ static int radeonfb_create(struct drm_fb_helper *helper,
 
 static int radeon_fbdev_destroy(struct drm_device *dev, struct radeon_fbdev *rfbdev)
 {
-	struct radeon_framebuffer *rfb = &rfbdev->rfb;
+	struct drm_framebuffer *fb = &rfbdev->fb;
 
 	drm_fb_helper_unregister_fbi(&rfbdev->helper);
 
-	if (rfb->obj) {
-		radeonfb_destroy_pinned_object(rfb->obj);
-		rfb->obj = NULL;
-		drm_framebuffer_unregister_private(&rfb->base);
-		drm_framebuffer_cleanup(&rfb->base);
+	if (fb->obj[0]) {
+		radeonfb_destroy_pinned_object(fb->obj[0]);
+		fb->obj[0] = NULL;
+		drm_framebuffer_unregister_private(fb);
+		drm_framebuffer_cleanup(fb);
 	}
 	drm_fb_helper_fini(&rfbdev->helper);
 
@@ -400,7 +400,7 @@ bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj)
 	if (!rdev->mode_info.rfbdev)
 		return false;
 
-	if (robj == gem_to_radeon_bo(rdev->mode_info.rfbdev->rfb.obj))
+	if (robj == gem_to_radeon_bo(rdev->mode_info.rfbdev->fb.obj[0]))
 		return true;
 	return false;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index 1f1856e0b1e0..35a205ae4318 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -374,7 +374,6 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
-	struct radeon_framebuffer *radeon_fb;
 	struct drm_framebuffer *target_fb;
 	struct drm_gem_object *obj;
 	struct radeon_bo *rbo;
@@ -393,14 +392,10 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 		return 0;
 	}
 
-	if (atomic) {
-		radeon_fb = to_radeon_framebuffer(fb);
+	if (atomic)
 		target_fb = fb;
-	}
-	else {
-		radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
+	else
 		target_fb = crtc->primary->fb;
-	}
 
 	switch (target_fb->format->cpp[0] * 8) {
 	case 8:
@@ -423,7 +418,7 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 	}
 
 	/* Pin framebuffer & get tilling informations */
-	obj = radeon_fb->obj;
+	obj = target_fb->obj[0];
 	rbo = gem_to_radeon_bo(obj);
 retry:
 	r = radeon_bo_reserve(rbo, false);
@@ -451,7 +446,7 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 			struct radeon_bo *old_rbo;
 			unsigned long nsize, osize;
 
-			old_rbo = gem_to_radeon_bo(to_radeon_framebuffer(fb)->obj);
+			old_rbo = gem_to_radeon_bo(fb->obj[0]);
 			osize = radeon_bo_size(old_rbo);
 			nsize = radeon_bo_size(rbo);
 			if (nsize <= osize && !radeon_bo_reserve(old_rbo, false)) {
@@ -558,8 +553,7 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(RADEON_CRTC_PITCH + radeon_crtc->crtc_offset, crtc_pitch);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
-		radeon_fb = to_radeon_framebuffer(fb);
-		rbo = gem_to_radeon_bo(radeon_fb->obj);
+		rbo = gem_to_radeon_bo(fb->obj[0]);
 		r = radeon_bo_reserve(rbo, false);
 		if (unlikely(r != 0))
 			return r;
@@ -1093,11 +1087,9 @@ static void radeon_crtc_disable(struct drm_crtc *crtc)
 	radeon_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
 	if (crtc->primary->fb) {
 		int r;
-		struct radeon_framebuffer *radeon_fb;
 		struct radeon_bo *rbo;
 
-		radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
-		rbo = gem_to_radeon_bo(radeon_fb->obj);
+		rbo = gem_to_radeon_bo(crtc->primary->fb->obj[0]);
 		r = radeon_bo_reserve(rbo, false);
 		if (unlikely(r))
 			DRM_ERROR("failed to reserve rbo before unpin\n");
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index 3243e5e01432..fd470d6bf3f4 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -46,7 +46,6 @@ struct radeon_device;
 #define to_radeon_crtc(x) container_of(x, struct radeon_crtc, base)
 #define to_radeon_connector(x) container_of(x, struct radeon_connector, base)
 #define to_radeon_encoder(x) container_of(x, struct radeon_encoder, base)
-#define to_radeon_framebuffer(x) container_of(x, struct radeon_framebuffer, base)
 
 #define RADEON_MAX_HPD_PINS 7
 #define RADEON_MAX_CRTCS 6
@@ -574,11 +573,6 @@ struct radeon_connector {
 	int enabled_attribs;
 };
 
-struct radeon_framebuffer {
-	struct drm_framebuffer base;
-	struct drm_gem_object *obj;
-};
-
 #define ENCODER_MODE_IS_DP(em) (((em) == ATOM_ENCODER_MODE_DP) || \
 				((em) == ATOM_ENCODER_MODE_DP_MST))
 
@@ -932,7 +926,7 @@ radeon_combios_encoder_crtc_scratch_regs(struct drm_encoder *encoder, int crtc);
 extern void
 radeon_combios_encoder_dpms_scratch_regs(struct drm_encoder *encoder, bool on);
 int radeon_framebuffer_init(struct drm_device *dev,
-			     struct radeon_framebuffer *rfb,
+			     struct drm_framebuffer *rfb,
 			     const struct drm_mode_fb_cmd2 *mode_cmd,
 			     struct drm_gem_object *obj);
 
diff --git a/drivers/gpu/drm/rcar-du/Makefile b/drivers/gpu/drm/rcar-du/Makefile
index 3e58ed93d5b1..2a3b8d7972b5 100644
--- a/drivers/gpu/drm/rcar-du/Makefile
+++ b/drivers/gpu/drm/rcar-du/Makefile
@@ -17,3 +17,10 @@ rcar-du-drm-$(CONFIG_DRM_RCAR_VSP)	+= rcar_du_vsp.o
 obj-$(CONFIG_DRM_RCAR_DU)		+= rcar-du-drm.o
 obj-$(CONFIG_DRM_RCAR_DW_HDMI)		+= rcar_dw_hdmi.o
 obj-$(CONFIG_DRM_RCAR_LVDS)		+= rcar_lvds.o
+
+# 'remote-endpoint' is fixed up at run-time
+DTC_FLAGS_rcar_du_of_lvds_r8a7790 += -Wno-graph_endpoint
+DTC_FLAGS_rcar_du_of_lvds_r8a7791 += -Wno-graph_endpoint
+DTC_FLAGS_rcar_du_of_lvds_r8a7793 += -Wno-graph_endpoint
+DTC_FLAGS_rcar_du_of_lvds_r8a7795 += -Wno-graph_endpoint
+DTC_FLAGS_rcar_du_of_lvds_r8a7796 += -Wno-graph_endpoint
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
index c4420538ec85..15dc9caa128b 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
@@ -691,6 +691,52 @@ static const struct drm_crtc_helper_funcs crtc_helper_funcs = {
 	.atomic_disable = rcar_du_crtc_atomic_disable,
 };
 
+static struct drm_crtc_state *
+rcar_du_crtc_atomic_duplicate_state(struct drm_crtc *crtc)
+{
+	struct rcar_du_crtc_state *state;
+	struct rcar_du_crtc_state *copy;
+
+	if (WARN_ON(!crtc->state))
+		return NULL;
+
+	state = to_rcar_crtc_state(crtc->state);
+	copy = kmemdup(state, sizeof(*state), GFP_KERNEL);
+	if (copy == NULL)
+		return NULL;
+
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &copy->state);
+
+	return &copy->state;
+}
+
+static void rcar_du_crtc_atomic_destroy_state(struct drm_crtc *crtc,
+					      struct drm_crtc_state *state)
+{
+	__drm_atomic_helper_crtc_destroy_state(state);
+	kfree(to_rcar_crtc_state(state));
+}
+
+static void rcar_du_crtc_reset(struct drm_crtc *crtc)
+{
+	struct rcar_du_crtc_state *state;
+
+	if (crtc->state) {
+		rcar_du_crtc_atomic_destroy_state(crtc, crtc->state);
+		crtc->state = NULL;
+	}
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (state == NULL)
+		return;
+
+	state->crc.source = VSP1_DU_CRC_NONE;
+	state->crc.index = 0;
+
+	crtc->state = &state->state;
+	crtc->state->crtc = crtc;
+}
+
 static int rcar_du_crtc_enable_vblank(struct drm_crtc *crtc)
 {
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
@@ -710,15 +756,111 @@ static void rcar_du_crtc_disable_vblank(struct drm_crtc *crtc)
 	rcrtc->vblank_enable = false;
 }
 
-static const struct drm_crtc_funcs crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+static int rcar_du_crtc_set_crc_source(struct drm_crtc *crtc,
+				       const char *source_name,
+				       size_t *values_cnt)
+{
+	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_crtc_state *crtc_state;
+	struct drm_atomic_state *state;
+	enum vsp1_du_crc_source source;
+	unsigned int index = 0;
+	unsigned int i;
+	int ret;
+
+	/*
+	 * Parse the source name. Supported values are "plane%u" to compute the
+	 * CRC on an input plane (%u is the plane ID), and "auto" to compute the
+	 * CRC on the composer (VSP) output.
+	 */
+	if (!source_name) {
+		source = VSP1_DU_CRC_NONE;
+	} else if (!strcmp(source_name, "auto")) {
+		source = VSP1_DU_CRC_OUTPUT;
+	} else if (strstarts(source_name, "plane")) {
+		source = VSP1_DU_CRC_PLANE;
+
+		ret = kstrtouint(source_name + strlen("plane"), 10, &index);
+		if (ret < 0)
+			return ret;
+
+		for (i = 0; i < rcrtc->vsp->num_planes; ++i) {
+			if (index == rcrtc->vsp->planes[i].plane.base.id) {
+				index = i;
+				break;
+			}
+		}
+
+		if (i >= rcrtc->vsp->num_planes)
+			return -EINVAL;
+	} else {
+		return -EINVAL;
+	}
+
+	*values_cnt = 1;
+
+	/* Perform an atomic commit to set the CRC source. */
+	drm_modeset_acquire_init(&ctx, 0);
+
+	state = drm_atomic_state_alloc(crtc->dev);
+	if (!state) {
+		ret = -ENOMEM;
+		goto unlock;
+	}
+
+	state->acquire_ctx = &ctx;
+
+retry:
+	crtc_state = drm_atomic_get_crtc_state(state, crtc);
+	if (!IS_ERR(crtc_state)) {
+		struct rcar_du_crtc_state *rcrtc_state;
+
+		rcrtc_state = to_rcar_crtc_state(crtc_state);
+		rcrtc_state->crc.source = source;
+		rcrtc_state->crc.index = index;
+
+		ret = drm_atomic_commit(state);
+	} else {
+		ret = PTR_ERR(crtc_state);
+	}
+
+	if (ret == -EDEADLK) {
+		drm_atomic_state_clear(state);
+		drm_modeset_backoff(&ctx);
+		goto retry;
+	}
+
+	drm_atomic_state_put(state);
+
+unlock:
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+
+	return 0;
+}
+
+static const struct drm_crtc_funcs crtc_funcs_gen2 = {
+	.reset = rcar_du_crtc_reset,
+	.destroy = drm_crtc_cleanup,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = rcar_du_crtc_atomic_duplicate_state,
+	.atomic_destroy_state = rcar_du_crtc_atomic_destroy_state,
+	.enable_vblank = rcar_du_crtc_enable_vblank,
+	.disable_vblank = rcar_du_crtc_disable_vblank,
+};
+
+static const struct drm_crtc_funcs crtc_funcs_gen3 = {
+	.reset = rcar_du_crtc_reset,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+	.atomic_duplicate_state = rcar_du_crtc_atomic_duplicate_state,
+	.atomic_destroy_state = rcar_du_crtc_atomic_destroy_state,
 	.enable_vblank = rcar_du_crtc_enable_vblank,
 	.disable_vblank = rcar_du_crtc_disable_vblank,
+	.set_crc_source = rcar_du_crtc_set_crc_source,
 };
 
 /* -----------------------------------------------------------------------------
@@ -767,7 +909,8 @@ static irqreturn_t rcar_du_crtc_irq(int irq, void *arg)
  * Initialization
  */
 
-int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int index)
+int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int swindex,
+			unsigned int hwindex)
 {
 	static const unsigned int mmio_offsets[] = {
 		DU0_REG_OFFSET, DU1_REG_OFFSET, DU2_REG_OFFSET, DU3_REG_OFFSET
@@ -775,7 +918,7 @@ int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int index)
 
 	struct rcar_du_device *rcdu = rgrp->dev;
 	struct platform_device *pdev = to_platform_device(rcdu->dev);
-	struct rcar_du_crtc *rcrtc = &rcdu->crtcs[index];
+	struct rcar_du_crtc *rcrtc = &rcdu->crtcs[swindex];
 	struct drm_crtc *crtc = &rcrtc->crtc;
 	struct drm_plane *primary;
 	unsigned int irqflags;
@@ -787,7 +930,7 @@ int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int index)
 
 	/* Get the CRTC clock and the optional external clock. */
 	if (rcar_du_has(rcdu, RCAR_DU_FEATURE_CRTC_IRQ_CLOCK)) {
-		sprintf(clk_name, "du.%u", index);
+		sprintf(clk_name, "du.%u", hwindex);
 		name = clk_name;
 	} else {
 		name = NULL;
@@ -795,16 +938,16 @@ int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int index)
 
 	rcrtc->clock = devm_clk_get(rcdu->dev, name);
 	if (IS_ERR(rcrtc->clock)) {
-		dev_err(rcdu->dev, "no clock for CRTC %u\n", index);
+		dev_err(rcdu->dev, "no clock for DU channel %u\n", hwindex);
 		return PTR_ERR(rcrtc->clock);
 	}
 
-	sprintf(clk_name, "dclkin.%u", index);
+	sprintf(clk_name, "dclkin.%u", hwindex);
 	clk = devm_clk_get(rcdu->dev, clk_name);
 	if (!IS_ERR(clk)) {
 		rcrtc->extclock = clk;
 	} else if (PTR_ERR(rcrtc->clock) == -EPROBE_DEFER) {
-		dev_info(rcdu->dev, "can't get external clock %u\n", index);
+		dev_info(rcdu->dev, "can't get external clock %u\n", hwindex);
 		return -EPROBE_DEFER;
 	}
 
@@ -813,16 +956,18 @@ int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int index)
 	spin_lock_init(&rcrtc->vblank_lock);
 
 	rcrtc->group = rgrp;
-	rcrtc->mmio_offset = mmio_offsets[index];
-	rcrtc->index = index;
+	rcrtc->mmio_offset = mmio_offsets[hwindex];
+	rcrtc->index = hwindex;
 
 	if (rcar_du_has(rcdu, RCAR_DU_FEATURE_VSP1_SOURCE))
 		primary = &rcrtc->vsp->planes[rcrtc->vsp_pipe].plane;
 	else
-		primary = &rgrp->planes[index % 2].plane;
+		primary = &rgrp->planes[swindex % 2].plane;
 
-	ret = drm_crtc_init_with_planes(rcdu->ddev, crtc, primary,
-					NULL, &crtc_funcs, NULL);
+	ret = drm_crtc_init_with_planes(rcdu->ddev, crtc, primary, NULL,
+					rcdu->info->gen <= 2 ?
+					&crtc_funcs_gen2 : &crtc_funcs_gen3,
+					NULL);
 	if (ret < 0)
 		return ret;
 
@@ -833,7 +978,8 @@ int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int index)
 
 	/* Register the interrupt handler. */
 	if (rcar_du_has(rcdu, RCAR_DU_FEATURE_CRTC_IRQ_CLOCK)) {
-		irq = platform_get_irq(pdev, index);
+		/* The IRQ's are associated with the CRTC (sw)index. */
+		irq = platform_get_irq(pdev, swindex);
 		irqflags = 0;
 	} else {
 		irq = platform_get_irq(pdev, 0);
@@ -841,7 +987,7 @@ int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int index)
 	}
 
 	if (irq < 0) {
-		dev_err(rcdu->dev, "no IRQ for CRTC %u\n", index);
+		dev_err(rcdu->dev, "no IRQ for CRTC %u\n", swindex);
 		return irq;
 	}
 
@@ -849,7 +995,7 @@ int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int index)
 			       dev_name(rcdu->dev), rcrtc);
 	if (ret < 0) {
 		dev_err(rcdu->dev,
-			"failed to register IRQ for CRTC %u\n", index);
+			"failed to register IRQ for CRTC %u\n", swindex);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.h b/drivers/gpu/drm/rcar-du/rcar_du_crtc.h
index fdc2bf99bda1..7680cb2636c8 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.h
@@ -21,6 +21,8 @@
 #include <drm/drmP.h>
 #include <drm/drm_crtc.h>
 
+#include <media/vsp1.h>
+
 struct rcar_du_group;
 struct rcar_du_vsp;
 
@@ -69,6 +71,19 @@ struct rcar_du_crtc {
 
 #define to_rcar_crtc(c)	container_of(c, struct rcar_du_crtc, crtc)
 
+/**
+ * struct rcar_du_crtc_state - Driver-specific CRTC state
+ * @state: base DRM CRTC state
+ * @crc: CRC computation configuration
+ */
+struct rcar_du_crtc_state {
+	struct drm_crtc_state state;
+
+	struct vsp1_du_crc_config crc;
+};
+
+#define to_rcar_crtc_state(s) container_of(s, struct rcar_du_crtc_state, state)
+
 enum rcar_du_output {
 	RCAR_DU_OUTPUT_DPAD0,
 	RCAR_DU_OUTPUT_DPAD1,
@@ -80,7 +95,8 @@ enum rcar_du_output {
 	RCAR_DU_OUTPUT_MAX,
 };
 
-int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int index);
+int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int swindex,
+			unsigned int hwindex);
 void rcar_du_crtc_suspend(struct rcar_du_crtc *rcrtc);
 void rcar_du_crtc_resume(struct rcar_du_crtc *rcrtc);
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.c b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
index 3917d839c04c..02aee6cb0e53 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_drv.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
@@ -40,7 +40,7 @@ static const struct rcar_du_device_info rzg1_du_r8a7743_info = {
 	.gen = 2,
 	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
 		  | RCAR_DU_FEATURE_EXT_CTRL_REGS,
-	.num_crtcs = 2,
+	.channels_mask = BIT(1) | BIT(0),
 	.routes = {
 		/*
 		 * R8A7743 has one RGB output and one LVDS output
@@ -61,7 +61,7 @@ static const struct rcar_du_device_info rzg1_du_r8a7745_info = {
 	.gen = 2,
 	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
 		  | RCAR_DU_FEATURE_EXT_CTRL_REGS,
-	.num_crtcs = 2,
+	.channels_mask = BIT(1) | BIT(0),
 	.routes = {
 		/*
 		 * R8A7745 has two RGB outputs
@@ -80,7 +80,7 @@ static const struct rcar_du_device_info rzg1_du_r8a7745_info = {
 static const struct rcar_du_device_info rcar_du_r8a7779_info = {
 	.gen = 2,
 	.features = 0,
-	.num_crtcs = 2,
+	.channels_mask = BIT(1) | BIT(0),
 	.routes = {
 		/*
 		 * R8A7779 has two RGB outputs and one (currently unsupported)
@@ -102,7 +102,7 @@ static const struct rcar_du_device_info rcar_du_r8a7790_info = {
 	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
 		  | RCAR_DU_FEATURE_EXT_CTRL_REGS,
 	.quirks = RCAR_DU_QUIRK_ALIGN_128B,
-	.num_crtcs = 3,
+	.channels_mask = BIT(2) | BIT(1) | BIT(0),
 	.routes = {
 		/*
 		 * R8A7790 has one RGB output, two LVDS outputs and one
@@ -129,7 +129,7 @@ static const struct rcar_du_device_info rcar_du_r8a7791_info = {
 	.gen = 2,
 	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
 		  | RCAR_DU_FEATURE_EXT_CTRL_REGS,
-	.num_crtcs = 2,
+	.channels_mask = BIT(1) | BIT(0),
 	.routes = {
 		/*
 		 * R8A779[13] has one RGB output, one LVDS output and one
@@ -151,7 +151,7 @@ static const struct rcar_du_device_info rcar_du_r8a7792_info = {
 	.gen = 2,
 	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
 		  | RCAR_DU_FEATURE_EXT_CTRL_REGS,
-	.num_crtcs = 2,
+	.channels_mask = BIT(1) | BIT(0),
 	.routes = {
 		/* R8A7792 has two RGB outputs. */
 		[RCAR_DU_OUTPUT_DPAD0] = {
@@ -169,7 +169,7 @@ static const struct rcar_du_device_info rcar_du_r8a7794_info = {
 	.gen = 2,
 	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
 		  | RCAR_DU_FEATURE_EXT_CTRL_REGS,
-	.num_crtcs = 2,
+	.channels_mask = BIT(1) | BIT(0),
 	.routes = {
 		/*
 		 * R8A7794 has two RGB outputs and one (currently unsupported)
@@ -191,7 +191,7 @@ static const struct rcar_du_device_info rcar_du_r8a7795_info = {
 	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
 		  | RCAR_DU_FEATURE_EXT_CTRL_REGS
 		  | RCAR_DU_FEATURE_VSP1_SOURCE,
-	.num_crtcs = 4,
+	.channels_mask = BIT(3) | BIT(2) | BIT(1) | BIT(0),
 	.routes = {
 		/*
 		 * R8A7795 has one RGB output, two HDMI outputs and one
@@ -215,7 +215,7 @@ static const struct rcar_du_device_info rcar_du_r8a7795_info = {
 		},
 	},
 	.num_lvds = 1,
-	.dpll_ch =  BIT(1) | BIT(2),
+	.dpll_ch =  BIT(2) | BIT(1),
 };
 
 static const struct rcar_du_device_info rcar_du_r8a7796_info = {
@@ -223,7 +223,7 @@ static const struct rcar_du_device_info rcar_du_r8a7796_info = {
 	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
 		  | RCAR_DU_FEATURE_EXT_CTRL_REGS
 		  | RCAR_DU_FEATURE_VSP1_SOURCE,
-	.num_crtcs = 3,
+	.channels_mask = BIT(2) | BIT(1) | BIT(0),
 	.routes = {
 		/*
 		 * R8A7796 has one RGB output, one LVDS output and one HDMI
@@ -246,12 +246,40 @@ static const struct rcar_du_device_info rcar_du_r8a7796_info = {
 	.dpll_ch =  BIT(1),
 };
 
+static const struct rcar_du_device_info rcar_du_r8a77965_info = {
+	.gen = 3,
+	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
+		  | RCAR_DU_FEATURE_EXT_CTRL_REGS
+		  | RCAR_DU_FEATURE_VSP1_SOURCE,
+	.channels_mask = BIT(3) | BIT(1) | BIT(0),
+	.routes = {
+		/*
+		 * R8A77965 has one RGB output, one LVDS output and one HDMI
+		 * output.
+		 */
+		[RCAR_DU_OUTPUT_DPAD0] = {
+			.possible_crtcs = BIT(2),
+			.port = 0,
+		},
+		[RCAR_DU_OUTPUT_HDMI0] = {
+			.possible_crtcs = BIT(1),
+			.port = 1,
+		},
+		[RCAR_DU_OUTPUT_LVDS0] = {
+			.possible_crtcs = BIT(0),
+			.port = 2,
+		},
+	},
+	.num_lvds = 1,
+	.dpll_ch =  BIT(1),
+};
+
 static const struct rcar_du_device_info rcar_du_r8a77970_info = {
 	.gen = 3,
 	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
 		  | RCAR_DU_FEATURE_EXT_CTRL_REGS
 		  | RCAR_DU_FEATURE_VSP1_SOURCE,
-	.num_crtcs = 1,
+	.channels_mask = BIT(0),
 	.routes = {
 		/* R8A77970 has one RGB output and one LVDS output. */
 		[RCAR_DU_OUTPUT_DPAD0] = {
@@ -277,6 +305,7 @@ static const struct of_device_id rcar_du_of_table[] = {
 	{ .compatible = "renesas,du-r8a7794", .data = &rcar_du_r8a7794_info },
 	{ .compatible = "renesas,du-r8a7795", .data = &rcar_du_r8a7795_info },
 	{ .compatible = "renesas,du-r8a7796", .data = &rcar_du_r8a7796_info },
+	{ .compatible = "renesas,du-r8a77965", .data = &rcar_du_r8a77965_info },
 	{ .compatible = "renesas,du-r8a77970", .data = &rcar_du_r8a77970_info },
 	{ }
 };
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.h b/drivers/gpu/drm/rcar-du/rcar_du_drv.h
index 5c7ec15818c7..b3a25e8e07d0 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_drv.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.h
@@ -52,7 +52,7 @@ struct rcar_du_output_routing {
  * @gen: device generation (2 or 3)
  * @features: device features (RCAR_DU_FEATURE_*)
  * @quirks: device quirks (RCAR_DU_QUIRK_*)
- * @num_crtcs: total number of CRTCs
+ * @channels_mask: bit mask of available DU channels
  * @routes: array of CRTC to output routes, indexed by output (RCAR_DU_OUTPUT_*)
  * @num_lvds: number of internal LVDS encoders
  */
@@ -60,7 +60,7 @@ struct rcar_du_device_info {
 	unsigned int gen;
 	unsigned int features;
 	unsigned int quirks;
-	unsigned int num_crtcs;
+	unsigned int channels_mask;
 	struct rcar_du_output_routing routes[RCAR_DU_OUTPUT_MAX];
 	unsigned int num_lvds;
 	unsigned int dpll_ch;
@@ -87,7 +87,6 @@ struct rcar_du_device {
 	struct rcar_du_vsp vsps[RCAR_DU_MAX_VSPS];
 
 	struct {
-		struct drm_property *alpha;
 		struct drm_property *colorkey;
 	} props;
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_group.c b/drivers/gpu/drm/rcar-du/rcar_du_group.c
index 2f37ea901873..d539cb290a35 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_group.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_group.c
@@ -46,10 +46,13 @@ void rcar_du_group_write(struct rcar_du_group *rgrp, u32 reg, u32 data)
 
 static void rcar_du_group_setup_pins(struct rcar_du_group *rgrp)
 {
-	u32 defr6 = DEFR6_CODE | DEFR6_ODPM12_DISP;
+	u32 defr6 = DEFR6_CODE;
 
-	if (rgrp->num_crtcs > 1)
-		defr6 |= DEFR6_ODPM22_DISP;
+	if (rgrp->channels_mask & BIT(0))
+		defr6 |= DEFR6_ODPM02_DISP;
+
+	if (rgrp->channels_mask & BIT(1))
+		defr6 |= DEFR6_ODPM12_DISP;
 
 	rcar_du_group_write(rgrp, DEFR6, defr6);
 }
@@ -80,10 +83,11 @@ static void rcar_du_group_setup_defr8(struct rcar_du_group *rgrp)
 		 * On Gen3 VSPD routing can't be configured, but DPAD routing
 		 * needs to be set despite having a single option available.
 		 */
-		u32 crtc = ffs(possible_crtcs) - 1;
+		unsigned int rgb_crtc = ffs(possible_crtcs) - 1;
+		struct rcar_du_crtc *crtc = &rcdu->crtcs[rgb_crtc];
 
-		if (crtc / 2 == rgrp->index)
-			defr8 |= DEFR8_DRGBS_DU(crtc);
+		if (crtc->index / 2 == rgrp->index)
+			defr8 |= DEFR8_DRGBS_DU(crtc->index);
 	}
 
 	rcar_du_group_write(rgrp, DEFR8, defr8);
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_group.h b/drivers/gpu/drm/rcar-du/rcar_du_group.h
index 5e3adc6b31b5..42105aedecc8 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_group.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_group.h
@@ -25,6 +25,7 @@ struct rcar_du_device;
  * @dev: the DU device
  * @mmio_offset: registers offset in the device memory map
  * @index: group index
+ * @channels_mask: bitmask of populated DU channels in this group
  * @num_crtcs: number of CRTCs in this group (1 or 2)
  * @use_count: number of users of the group (rcar_du_group_(get|put))
  * @used_crtcs: number of CRTCs currently in use
@@ -39,6 +40,7 @@ struct rcar_du_group {
 	unsigned int mmio_offset;
 	unsigned int index;
 
+	unsigned int channels_mask;
 	unsigned int num_crtcs;
 	unsigned int use_count;
 	unsigned int used_crtcs;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
index 0329b354bfa0..f0bc7cc0e913 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_kms.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
@@ -233,15 +233,7 @@ static int rcar_du_atomic_check(struct drm_device *dev,
 	struct rcar_du_device *rcdu = dev->dev_private;
 	int ret;
 
-	ret = drm_atomic_helper_check_modeset(dev, state);
-	if (ret)
-		return ret;
-
-	ret = drm_atomic_normalize_zpos(dev, state);
-	if (ret)
-		return ret;
-
-	ret = drm_atomic_helper_check_planes(dev, state);
+	ret = drm_atomic_helper_check(dev, state);
 	if (ret)
 		return ret;
 
@@ -415,11 +407,6 @@ static int rcar_du_encoders_init(struct rcar_du_device *rcdu)
 
 static int rcar_du_properties_init(struct rcar_du_device *rcdu)
 {
-	rcdu->props.alpha =
-		drm_property_create_range(rcdu->ddev, 0, "alpha", 0, 255);
-	if (rcdu->props.alpha == NULL)
-		return -ENOMEM;
-
 	/*
 	 * The color key is expressed as an RGB888 triplet stored in a 32-bit
 	 * integer in XRGB8888 format. Bit 24 is used as a flag to disable (0)
@@ -441,7 +428,7 @@ static int rcar_du_vsps_init(struct rcar_du_device *rcdu)
 	struct {
 		struct device_node *np;
 		unsigned int crtcs_mask;
-	} vsps[RCAR_DU_MAX_VSPS] = { { 0, }, };
+	} vsps[RCAR_DU_MAX_VSPS] = { { NULL, }, };
 	unsigned int vsps_count = 0;
 	unsigned int cells;
 	unsigned int i;
@@ -520,6 +507,8 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 	struct drm_fbdev_cma *fbdev;
 	unsigned int num_encoders;
 	unsigned int num_groups;
+	unsigned int swindex;
+	unsigned int hwindex;
 	unsigned int i;
 	int ret;
 
@@ -529,10 +518,11 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 	dev->mode_config.min_height = 0;
 	dev->mode_config.max_width = 4095;
 	dev->mode_config.max_height = 2047;
+	dev->mode_config.normalize_zpos = true;
 	dev->mode_config.funcs = &rcar_du_mode_config_funcs;
 	dev->mode_config.helper_private = &rcar_du_mode_config_helper;
 
-	rcdu->num_crtcs = rcdu->info->num_crtcs;
+	rcdu->num_crtcs = hweight8(rcdu->info->channels_mask);
 
 	ret = rcar_du_properties_init(rcdu);
 	if (ret < 0)
@@ -542,7 +532,7 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 	 * Initialize vertical blanking interrupts handling. Start with vblank
 	 * disabled for all CRTCs.
 	 */
-	ret = drm_vblank_init(dev, (1 << rcdu->info->num_crtcs) - 1);
+	ret = drm_vblank_init(dev, (1 << rcdu->num_crtcs) - 1);
 	if (ret < 0)
 		return ret;
 
@@ -557,7 +547,10 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 		rgrp->dev = rcdu;
 		rgrp->mmio_offset = mmio_offsets[i];
 		rgrp->index = i;
-		rgrp->num_crtcs = min(rcdu->num_crtcs - 2 * i, 2U);
+		/* Extract the channel mask for this group only. */
+		rgrp->channels_mask = (rcdu->info->channels_mask >> (2 * i))
+				   & GENMASK(1, 0);
+		rgrp->num_crtcs = hweight8(rgrp->channels_mask);
 
 		/*
 		 * If we have more than one CRTCs in this group pre-associate
@@ -584,10 +577,16 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 	}
 
 	/* Create the CRTCs. */
-	for (i = 0; i < rcdu->num_crtcs; ++i) {
-		struct rcar_du_group *rgrp = &rcdu->groups[i / 2];
+	for (swindex = 0, hwindex = 0; swindex < rcdu->num_crtcs; ++hwindex) {
+		struct rcar_du_group *rgrp;
+
+		/* Skip unpopulated DU channels. */
+		if (!(rcdu->info->channels_mask & BIT(hwindex)))
+			continue;
+
+		rgrp = &rcdu->groups[hwindex / 2];
 
-		ret = rcar_du_crtc_create(rgrp, i);
+		ret = rcar_du_crtc_create(rgrp, swindex++, hwindex);
 		if (ret < 0)
 			return ret;
 	}
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_of.c b/drivers/gpu/drm/rcar-du/rcar_du_of.c
index 68a0b82cb17e..afef69669bb4 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_of.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_of.c
@@ -18,6 +18,7 @@
 
 #include "rcar_du_crtc.h"
 #include "rcar_du_drv.h"
+#include "rcar_du_of.h"
 
 /* -----------------------------------------------------------------------------
  * Generic Overlay Handling
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_of.h b/drivers/gpu/drm/rcar-du/rcar_du_of.h
index c2e65a727e91..8dd3fbe96650 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_of.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_of.h
@@ -11,7 +11,7 @@
 
 struct of_device_id;
 
-#ifdef CONFIG_DRM_RCAR_LVDS
+#if IS_ENABLED(CONFIG_DRM_RCAR_LVDS)
 void __init rcar_du_of_init(const struct of_device_id *of_ids);
 #else
 static inline void rcar_du_of_init(const struct of_device_id *of_ids) { }
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.c b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
index 68556bd9dad2..c20f7ed48c8d 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_plane.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
@@ -423,7 +423,7 @@ static void rcar_du_plane_setup_mode(struct rcar_du_group *rgrp,
 		rcar_du_plane_write(rgrp, index, PnALPHAR, PnALPHAR_ABIT_0);
 	else
 		rcar_du_plane_write(rgrp, index, PnALPHAR,
-				    PnALPHAR_ABIT_X | state->alpha);
+				    PnALPHAR_ABIT_X | state->state.alpha >> 8);
 
 	pnmr = PnMR_BM_MD | state->format->pnmr;
 
@@ -692,11 +692,11 @@ static void rcar_du_plane_reset(struct drm_plane *plane)
 
 	state->hwindex = -1;
 	state->source = RCAR_DU_PLANE_MEMORY;
-	state->alpha = 255;
 	state->colorkey = RCAR_DU_COLORKEY_NONE;
 	state->state.zpos = plane->type == DRM_PLANE_TYPE_PRIMARY ? 0 : 1;
 
 	plane->state = &state->state;
+	plane->state->alpha = DRM_BLEND_ALPHA_OPAQUE;
 	plane->state->plane = plane;
 }
 
@@ -708,9 +708,7 @@ static int rcar_du_plane_atomic_set_property(struct drm_plane *plane,
 	struct rcar_du_plane_state *rstate = to_rcar_plane_state(state);
 	struct rcar_du_device *rcdu = to_rcar_plane(plane)->group->dev;
 
-	if (property == rcdu->props.alpha)
-		rstate->alpha = val;
-	else if (property == rcdu->props.colorkey)
+	if (property == rcdu->props.colorkey)
 		rstate->colorkey = val;
 	else
 		return -EINVAL;
@@ -726,9 +724,7 @@ static int rcar_du_plane_atomic_get_property(struct drm_plane *plane,
 		container_of(state, const struct rcar_du_plane_state, state);
 	struct rcar_du_device *rcdu = to_rcar_plane(plane)->group->dev;
 
-	if (property == rcdu->props.alpha)
-		*val = rstate->alpha;
-	else if (property == rcdu->props.colorkey)
+	if (property == rcdu->props.colorkey)
 		*val = rstate->colorkey;
 	else
 		return -EINVAL;
@@ -796,11 +792,10 @@ int rcar_du_planes_init(struct rcar_du_group *rgrp)
 		if (type == DRM_PLANE_TYPE_PRIMARY)
 			continue;
 
-		drm_object_attach_property(&plane->plane.base,
-					   rcdu->props.alpha, 255);
 		drm_object_attach_property(&plane->plane.base,
 					   rcdu->props.colorkey,
 					   RCAR_DU_COLORKEY_NONE);
+		drm_plane_create_alpha_property(&plane->plane);
 		drm_plane_create_zpos_property(&plane->plane, 1, 1, 7);
 	}
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.h b/drivers/gpu/drm/rcar-du/rcar_du_plane.h
index 890321b4665d..5c19c69e4691 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_plane.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.h
@@ -50,7 +50,6 @@ static inline struct rcar_du_plane *to_rcar_plane(struct drm_plane *plane)
  * @state: base DRM plane state
  * @format: information about the pixel format used by the plane
  * @hwindex: 0-based hardware plane index, -1 means unused
- * @alpha: value of the plane alpha property
  * @colorkey: value of the plane colorkey property
  */
 struct rcar_du_plane_state {
@@ -60,7 +59,6 @@ struct rcar_du_plane_state {
 	int hwindex;
 	enum rcar_du_plane_source source;
 
-	unsigned int alpha;
 	unsigned int colorkey;
 };
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_regs.h b/drivers/gpu/drm/rcar-du/rcar_du_regs.h
index d5bae99d3cfe..9dfd220ceda1 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_regs.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_regs.h
@@ -187,14 +187,14 @@
 
 #define DEFR6			0x000e8
 #define DEFR6_CODE		(0x7778 << 16)
-#define DEFR6_ODPM22_DSMR	(0 << 10)
-#define DEFR6_ODPM22_DISP	(2 << 10)
-#define DEFR6_ODPM22_CDE	(3 << 10)
-#define DEFR6_ODPM22_MASK	(3 << 10)
-#define DEFR6_ODPM12_DSMR	(0 << 8)
-#define DEFR6_ODPM12_DISP	(2 << 8)
-#define DEFR6_ODPM12_CDE	(3 << 8)
-#define DEFR6_ODPM12_MASK	(3 << 8)
+#define DEFR6_ODPM12_DSMR	(0 << 10)
+#define DEFR6_ODPM12_DISP	(2 << 10)
+#define DEFR6_ODPM12_CDE	(3 << 10)
+#define DEFR6_ODPM12_MASK	(3 << 10)
+#define DEFR6_ODPM02_DSMR	(0 << 8)
+#define DEFR6_ODPM02_DISP	(2 << 8)
+#define DEFR6_ODPM02_CDE	(3 << 8)
+#define DEFR6_ODPM02_MASK	(3 << 8)
 #define DEFR6_TCNE1		(1 << 6)
 #define DEFR6_TCNE0		(1 << 4)
 #define DEFR6_MLOS1		(1 << 2)
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
index 2c260c33840b..72eebeda518e 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
@@ -17,6 +17,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_plane_helper.h>
 
 #include <linux/bitops.h>
@@ -31,7 +32,7 @@
 #include "rcar_du_kms.h"
 #include "rcar_du_vsp.h"
 
-static void rcar_du_vsp_complete(void *private, bool completed)
+static void rcar_du_vsp_complete(void *private, bool completed, u32 crc)
 {
 	struct rcar_du_crtc *crtc = private;
 
@@ -40,6 +41,8 @@ static void rcar_du_vsp_complete(void *private, bool completed)
 
 	if (completed)
 		rcar_du_crtc_finish_page_flip(crtc);
+
+	drm_crtc_add_crc_entry(&crtc->crtc, false, 0, &crc);
 }
 
 void rcar_du_vsp_enable(struct rcar_du_crtc *crtc)
@@ -54,6 +57,7 @@ void rcar_du_vsp_enable(struct rcar_du_crtc *crtc)
 	};
 	struct rcar_du_plane_state state = {
 		.state = {
+			.alpha = DRM_BLEND_ALPHA_OPAQUE,
 			.crtc = &crtc->crtc,
 			.dst.x1 = 0,
 			.dst.y1 = 0,
@@ -67,7 +71,6 @@ void rcar_du_vsp_enable(struct rcar_du_crtc *crtc)
 		},
 		.format = rcar_du_format_info(DRM_FORMAT_ARGB8888),
 		.source = RCAR_DU_PLANE_VSPD1,
-		.alpha = 255,
 		.colorkey = 0,
 	};
 
@@ -102,7 +105,13 @@ void rcar_du_vsp_atomic_begin(struct rcar_du_crtc *crtc)
 
 void rcar_du_vsp_atomic_flush(struct rcar_du_crtc *crtc)
 {
-	vsp1_du_atomic_flush(crtc->vsp->vsp, crtc->vsp_pipe);
+	struct vsp1_du_atomic_pipe_config cfg = { { 0, } };
+	struct rcar_du_crtc_state *state;
+
+	state = to_rcar_crtc_state(crtc->crtc.state);
+	cfg.crc = state->crc;
+
+	vsp1_du_atomic_flush(crtc->vsp->vsp, crtc->vsp_pipe, &cfg);
 }
 
 /* Keep the two tables in sync. */
@@ -173,7 +182,7 @@ static void rcar_du_vsp_plane_setup(struct rcar_du_vsp_plane *plane)
 	struct vsp1_du_atomic_config cfg = {
 		.pixelformat = 0,
 		.pitch = fb->pitches[0],
-		.alpha = state->alpha,
+		.alpha = state->state.alpha >> 8,
 		.zpos = state->state.zpos,
 	};
 	unsigned int i;
@@ -237,6 +246,10 @@ static int rcar_du_vsp_plane_prepare_fb(struct drm_plane *plane,
 		}
 	}
 
+	ret = drm_gem_fb_prepare_fb(plane, state);
+	if (ret)
+		goto fail;
+
 	return 0;
 
 fail:
@@ -299,14 +312,12 @@ static const struct drm_plane_helper_funcs rcar_du_vsp_plane_helper_funcs = {
 static struct drm_plane_state *
 rcar_du_vsp_plane_atomic_duplicate_state(struct drm_plane *plane)
 {
-	struct rcar_du_vsp_plane_state *state;
 	struct rcar_du_vsp_plane_state *copy;
 
 	if (WARN_ON(!plane->state))
 		return NULL;
 
-	state = to_rcar_vsp_plane_state(plane->state);
-	copy = kmemdup(state, sizeof(*state), GFP_KERNEL);
+	copy = kzalloc(sizeof(*copy), GFP_KERNEL);
 	if (copy == NULL)
 		return NULL;
 
@@ -335,44 +346,13 @@ static void rcar_du_vsp_plane_reset(struct drm_plane *plane)
 	if (state == NULL)
 		return;
 
-	state->alpha = 255;
+	state->state.alpha = DRM_BLEND_ALPHA_OPAQUE;
 	state->state.zpos = plane->type == DRM_PLANE_TYPE_PRIMARY ? 0 : 1;
 
 	plane->state = &state->state;
 	plane->state->plane = plane;
 }
 
-static int rcar_du_vsp_plane_atomic_set_property(struct drm_plane *plane,
-	struct drm_plane_state *state, struct drm_property *property,
-	uint64_t val)
-{
-	struct rcar_du_vsp_plane_state *rstate = to_rcar_vsp_plane_state(state);
-	struct rcar_du_device *rcdu = to_rcar_vsp_plane(plane)->vsp->dev;
-
-	if (property == rcdu->props.alpha)
-		rstate->alpha = val;
-	else
-		return -EINVAL;
-
-	return 0;
-}
-
-static int rcar_du_vsp_plane_atomic_get_property(struct drm_plane *plane,
-	const struct drm_plane_state *state, struct drm_property *property,
-	uint64_t *val)
-{
-	const struct rcar_du_vsp_plane_state *rstate =
-		container_of(state, const struct rcar_du_vsp_plane_state, state);
-	struct rcar_du_device *rcdu = to_rcar_vsp_plane(plane)->vsp->dev;
-
-	if (property == rcdu->props.alpha)
-		*val = rstate->alpha;
-	else
-		return -EINVAL;
-
-	return 0;
-}
-
 static const struct drm_plane_funcs rcar_du_vsp_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
@@ -380,8 +360,6 @@ static const struct drm_plane_funcs rcar_du_vsp_plane_funcs = {
 	.destroy = drm_plane_cleanup,
 	.atomic_duplicate_state = rcar_du_vsp_plane_atomic_duplicate_state,
 	.atomic_destroy_state = rcar_du_vsp_plane_atomic_destroy_state,
-	.atomic_set_property = rcar_du_vsp_plane_atomic_set_property,
-	.atomic_get_property = rcar_du_vsp_plane_atomic_get_property,
 };
 
 int rcar_du_vsp_init(struct rcar_du_vsp *vsp, struct device_node *np,
@@ -438,8 +416,7 @@ int rcar_du_vsp_init(struct rcar_du_vsp *vsp, struct device_node *np,
 		if (type == DRM_PLANE_TYPE_PRIMARY)
 			continue;
 
-		drm_object_attach_property(&plane->plane.base,
-					   rcdu->props.alpha, 255);
+		drm_plane_create_alpha_property(&plane->plane);
 		drm_plane_create_zpos_property(&plane->plane, 1, 1,
 					       vsp->num_planes - 1);
 	}
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.h b/drivers/gpu/drm/rcar-du/rcar_du_vsp.h
index 4c5d7bbce6aa..8a8a25c8c8e8 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.h
@@ -44,15 +44,12 @@ static inline struct rcar_du_vsp_plane *to_rcar_vsp_plane(struct drm_plane *p)
  * @state: base DRM plane state
  * @format: information about the pixel format used by the plane
  * @sg_tables: scatter-gather tables for the frame buffer memory
- * @alpha: value of the plane alpha property
  */
 struct rcar_du_vsp_plane_state {
 	struct drm_plane_state state;
 
 	const struct rcar_du_format_info *format;
 	struct sg_table sg_tables[3];
-
-	unsigned int alpha;
 };
 
 static inline struct rcar_du_vsp_plane_state *
diff --git a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
index 3e8bf79bea58..080f05352195 100644
--- a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
@@ -77,13 +77,13 @@ struct rockchip_dp_device {
 	struct analogix_dp_plat_data plat_data;
 };
 
-static void analogix_dp_psr_set(struct drm_encoder *encoder, bool enabled)
+static int analogix_dp_psr_set(struct drm_encoder *encoder, bool enabled)
 {
 	struct rockchip_dp_device *dp = to_dp(encoder);
 	int ret;
 
 	if (!analogix_dp_psr_enabled(dp->adp))
-		return;
+		return 0;
 
 	DRM_DEV_DEBUG(dp->dev, "%s PSR...\n", enabled ? "Entry" : "Exit");
 
@@ -91,13 +91,13 @@ static void analogix_dp_psr_set(struct drm_encoder *encoder, bool enabled)
 					 PSR_WAIT_LINE_FLAG_TIMEOUT_MS);
 	if (ret) {
 		DRM_DEV_ERROR(dp->dev, "line flag interrupt did not arrive\n");
-		return;
+		return -ETIMEDOUT;
 	}
 
 	if (enabled)
-		analogix_dp_enable_psr(dp->adp);
+		return analogix_dp_enable_psr(dp->adp);
 	else
-		analogix_dp_disable_psr(dp->adp);
+		return analogix_dp_disable_psr(dp->adp);
 }
 
 static int rockchip_dp_pre_init(struct rockchip_dp_device *dp)
@@ -109,7 +109,7 @@ static int rockchip_dp_pre_init(struct rockchip_dp_device *dp)
 	return 0;
 }
 
-static int rockchip_dp_poweron(struct analogix_dp_plat_data *plat_data)
+static int rockchip_dp_poweron_start(struct analogix_dp_plat_data *plat_data)
 {
 	struct rockchip_dp_device *dp = to_dp(plat_data);
 	int ret;
@@ -127,7 +127,14 @@ static int rockchip_dp_poweron(struct analogix_dp_plat_data *plat_data)
 		return ret;
 	}
 
-	return rockchip_drm_psr_activate(&dp->encoder);
+	return ret;
+}
+
+static int rockchip_dp_poweron_end(struct analogix_dp_plat_data *plat_data)
+{
+	struct rockchip_dp_device *dp = to_dp(plat_data);
+
+	return rockchip_drm_psr_inhibit_put(&dp->encoder);
 }
 
 static int rockchip_dp_powerdown(struct analogix_dp_plat_data *plat_data)
@@ -135,7 +142,7 @@ static int rockchip_dp_powerdown(struct analogix_dp_plat_data *plat_data)
 	struct rockchip_dp_device *dp = to_dp(plat_data);
 	int ret;
 
-	ret = rockchip_drm_psr_deactivate(&dp->encoder);
+	ret = rockchip_drm_psr_inhibit_get(&dp->encoder);
 	if (ret != 0)
 		return ret;
 
@@ -218,6 +225,7 @@ rockchip_dp_drm_encoder_atomic_check(struct drm_encoder *encoder,
 				      struct drm_connector_state *conn_state)
 {
 	struct rockchip_crtc_state *s = to_rockchip_crtc_state(crtc_state);
+	struct drm_display_info *di = &conn_state->connector->display_info;
 
 	/*
 	 * The hardware IC designed that VOP must output the RGB10 video
@@ -229,6 +237,7 @@ rockchip_dp_drm_encoder_atomic_check(struct drm_encoder *encoder,
 
 	s->output_mode = ROCKCHIP_OUT_MODE_AAAA;
 	s->output_type = DRM_MODE_CONNECTOR_eDP;
+	s->output_bpc = di->bpc;
 
 	return 0;
 }
@@ -328,7 +337,8 @@ static int rockchip_dp_bind(struct device *dev, struct device *master,
 	dp->plat_data.encoder = &dp->encoder;
 
 	dp->plat_data.dev_type = dp->data->chip_type;
-	dp->plat_data.power_on = rockchip_dp_poweron;
+	dp->plat_data.power_on_start = rockchip_dp_poweron_start;
+	dp->plat_data.power_on_end = rockchip_dp_poweron_end;
 	dp->plat_data.power_off = rockchip_dp_powerdown;
 	dp->plat_data.get_modes = rockchip_dp_get_modes;
 
@@ -358,6 +368,8 @@ static void rockchip_dp_unbind(struct device *dev, struct device *master,
 	analogix_dp_unbind(dp->adp);
 	rockchip_drm_psr_unregister(&dp->encoder);
 	dp->encoder.funcs->destroy(&dp->encoder);
+
+	dp->adp = ERR_PTR(-ENODEV);
 }
 
 static const struct component_ops rockchip_dp_component_ops = {
@@ -381,6 +393,7 @@ static int rockchip_dp_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	dp->dev = dev;
+	dp->adp = ERR_PTR(-ENODEV);
 	dp->plat_data.panel = panel;
 
 	ret = rockchip_dp_of_probe(dp);
@@ -404,6 +417,9 @@ static int rockchip_dp_suspend(struct device *dev)
 {
 	struct rockchip_dp_device *dp = dev_get_drvdata(dev);
 
+	if (IS_ERR(dp->adp))
+		return 0;
+
 	return analogix_dp_suspend(dp->adp);
 }
 
@@ -411,6 +427,9 @@ static int rockchip_dp_resume(struct device *dev)
 {
 	struct rockchip_dp_device *dp = dev_get_drvdata(dev);
 
+	if (IS_ERR(dp->adp))
+		return 0;
+
 	return analogix_dp_resume(dp->adp);
 }
 #endif
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
index 9c064a40458b..3a6ebfc26036 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
@@ -36,6 +36,7 @@ struct rockchip_crtc_state {
 	struct drm_crtc_state base;
 	int output_type;
 	int output_mode;
+	int output_bpc;
 };
 #define to_rockchip_crtc_state(s) \
 		container_of(s, struct rockchip_crtc_state, base)
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
index e266539e04e5..d4f4118b482d 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
@@ -167,8 +167,67 @@ rockchip_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
 	return ERR_PTR(ret);
 }
 
+static void
+rockchip_drm_psr_inhibit_get_state(struct drm_atomic_state *state)
+{
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *crtc_state;
+	struct drm_encoder *encoder;
+	u32 encoder_mask = 0;
+	int i;
+
+	for_each_old_crtc_in_state(state, crtc, crtc_state, i) {
+		encoder_mask |= crtc_state->encoder_mask;
+		encoder_mask |= crtc->state->encoder_mask;
+	}
+
+	drm_for_each_encoder_mask(encoder, state->dev, encoder_mask)
+		rockchip_drm_psr_inhibit_get(encoder);
+}
+
+static void
+rockchip_drm_psr_inhibit_put_state(struct drm_atomic_state *state)
+{
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *crtc_state;
+	struct drm_encoder *encoder;
+	u32 encoder_mask = 0;
+	int i;
+
+	for_each_old_crtc_in_state(state, crtc, crtc_state, i) {
+		encoder_mask |= crtc_state->encoder_mask;
+		encoder_mask |= crtc->state->encoder_mask;
+	}
+
+	drm_for_each_encoder_mask(encoder, state->dev, encoder_mask)
+		rockchip_drm_psr_inhibit_put(encoder);
+}
+
+static void
+rockchip_atomic_helper_commit_tail_rpm(struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = old_state->dev;
+
+	rockchip_drm_psr_inhibit_get_state(old_state);
+
+	drm_atomic_helper_commit_modeset_disables(dev, old_state);
+
+	drm_atomic_helper_commit_modeset_enables(dev, old_state);
+
+	drm_atomic_helper_commit_planes(dev, old_state,
+					DRM_PLANE_COMMIT_ACTIVE_ONLY);
+
+	rockchip_drm_psr_inhibit_put_state(old_state);
+
+	drm_atomic_helper_commit_hw_done(old_state);
+
+	drm_atomic_helper_wait_for_vblanks(dev, old_state);
+
+	drm_atomic_helper_cleanup_planes(dev, old_state);
+}
+
 static const struct drm_mode_config_helper_funcs rockchip_mode_config_helpers = {
-	.atomic_commit_tail = drm_atomic_helper_commit_tail_rpm,
+	.atomic_commit_tail = rockchip_atomic_helper_commit_tail_rpm,
 };
 
 static const struct drm_mode_config_funcs rockchip_drm_mode_config_funcs = {
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
index 074db7a92809..a8db758d523e 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
@@ -357,8 +357,8 @@ rockchip_gem_create_object(struct drm_device *drm, unsigned int size,
 }
 
 /*
- * rockchip_gem_free_object - (struct drm_driver)->gem_free_object callback
- * function
+ * rockchip_gem_free_object - (struct drm_driver)->gem_free_object_unlocked
+ * callback function
  */
 void rockchip_gem_free_object(struct drm_gem_object *obj)
 {
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_psr.c b/drivers/gpu/drm/rockchip/rockchip_drm_psr.c
index b339ca943139..79d00d861a31 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_psr.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_psr.c
@@ -20,42 +20,19 @@
 
 #define PSR_FLUSH_TIMEOUT_MS	100
 
-enum psr_state {
-	PSR_FLUSH,
-	PSR_ENABLE,
-	PSR_DISABLE,
-};
-
 struct psr_drv {
 	struct list_head	list;
 	struct drm_encoder	*encoder;
 
 	struct mutex		lock;
-	bool			active;
-	enum psr_state		state;
+	int			inhibit_count;
+	bool			enabled;
 
 	struct delayed_work	flush_work;
 
-	void (*set)(struct drm_encoder *encoder, bool enable);
+	int (*set)(struct drm_encoder *encoder, bool enable);
 };
 
-static struct psr_drv *find_psr_by_crtc(struct drm_crtc *crtc)
-{
-	struct rockchip_drm_private *drm_drv = crtc->dev->dev_private;
-	struct psr_drv *psr;
-
-	mutex_lock(&drm_drv->psr_list_lock);
-	list_for_each_entry(psr, &drm_drv->psr_list, list) {
-		if (psr->encoder->crtc == crtc)
-			goto out;
-	}
-	psr = ERR_PTR(-ENODEV);
-
-out:
-	mutex_unlock(&drm_drv->psr_list_lock);
-	return psr;
-}
-
 static struct psr_drv *find_psr_by_encoder(struct drm_encoder *encoder)
 {
 	struct rockchip_drm_private *drm_drv = encoder->dev->dev_private;
@@ -73,46 +50,22 @@ static struct psr_drv *find_psr_by_encoder(struct drm_encoder *encoder)
 	return psr;
 }
 
-static void psr_set_state_locked(struct psr_drv *psr, enum psr_state state)
+static int psr_set_state_locked(struct psr_drv *psr, bool enable)
 {
-	/*
-	 * Allowed finite state machine:
-	 *
-	 *   PSR_ENABLE  < = = = = = >  PSR_FLUSH
-	 *       | ^                        |
-	 *       | |                        |
-	 *       v |                        |
-	 *   PSR_DISABLE < - - - - - - - - -
-	 */
-	if (state == psr->state || !psr->active)
-		return;
-
-	/* Already disabled in flush, change the state, but not the hardware */
-	if (state == PSR_DISABLE && psr->state == PSR_FLUSH) {
-		psr->state = state;
-		return;
-	}
+	int ret;
 
-	psr->state = state;
+	if (psr->inhibit_count > 0)
+		return -EINVAL;
 
-	/* Actually commit the state change to hardware */
-	switch (psr->state) {
-	case PSR_ENABLE:
-		psr->set(psr->encoder, true);
-		break;
+	if (enable == psr->enabled)
+		return 0;
 
-	case PSR_DISABLE:
-	case PSR_FLUSH:
-		psr->set(psr->encoder, false);
-		break;
-	}
-}
+	ret = psr->set(psr->encoder, enable);
+	if (ret)
+		return ret;
 
-static void psr_set_state(struct psr_drv *psr, enum psr_state state)
-{
-	mutex_lock(&psr->lock);
-	psr_set_state_locked(psr, state);
-	mutex_unlock(&psr->lock);
+	psr->enabled = enable;
+	return 0;
 }
 
 static void psr_flush_handler(struct work_struct *work)
@@ -120,21 +73,24 @@ static void psr_flush_handler(struct work_struct *work)
 	struct psr_drv *psr = container_of(to_delayed_work(work),
 					   struct psr_drv, flush_work);
 
-	/* If the state has changed since we initiated the flush, do nothing */
 	mutex_lock(&psr->lock);
-	if (psr->state == PSR_FLUSH)
-		psr_set_state_locked(psr, PSR_ENABLE);
+	psr_set_state_locked(psr, true);
 	mutex_unlock(&psr->lock);
 }
 
 /**
- * rockchip_drm_psr_activate - activate PSR on the given pipe
+ * rockchip_drm_psr_inhibit_put - release PSR inhibit on given encoder
  * @encoder: encoder to obtain the PSR encoder
  *
+ * Decrements PSR inhibit count on given encoder. Should be called only
+ * for a PSR inhibit count increment done before. If PSR inhibit counter
+ * reaches zero, PSR flush work is scheduled to make the hardware enter
+ * PSR mode in PSR_FLUSH_TIMEOUT_MS.
+ *
  * Returns:
  * Zero on success, negative errno on failure.
  */
-int rockchip_drm_psr_activate(struct drm_encoder *encoder)
+int rockchip_drm_psr_inhibit_put(struct drm_encoder *encoder)
 {
 	struct psr_drv *psr = find_psr_by_encoder(encoder);
 
@@ -142,21 +98,30 @@ int rockchip_drm_psr_activate(struct drm_encoder *encoder)
 		return PTR_ERR(psr);
 
 	mutex_lock(&psr->lock);
-	psr->active = true;
+	--psr->inhibit_count;
+	WARN_ON(psr->inhibit_count < 0);
+	if (!psr->inhibit_count)
+		mod_delayed_work(system_wq, &psr->flush_work,
+				 PSR_FLUSH_TIMEOUT_MS);
 	mutex_unlock(&psr->lock);
 
 	return 0;
 }
-EXPORT_SYMBOL(rockchip_drm_psr_activate);
+EXPORT_SYMBOL(rockchip_drm_psr_inhibit_put);
 
 /**
- * rockchip_drm_psr_deactivate - deactivate PSR on the given pipe
+ * rockchip_drm_psr_inhibit_get - acquire PSR inhibit on given encoder
  * @encoder: encoder to obtain the PSR encoder
  *
+ * Increments PSR inhibit count on given encoder. This function guarantees
+ * that after it returns PSR is turned off on given encoder and no PSR-related
+ * hardware state change occurs at least until a matching call to
+ * rockchip_drm_psr_inhibit_put() is done.
+ *
  * Returns:
  * Zero on success, negative errno on failure.
  */
-int rockchip_drm_psr_deactivate(struct drm_encoder *encoder)
+int rockchip_drm_psr_inhibit_get(struct drm_encoder *encoder)
 {
 	struct psr_drv *psr = find_psr_by_encoder(encoder);
 
@@ -164,37 +129,25 @@ int rockchip_drm_psr_deactivate(struct drm_encoder *encoder)
 		return PTR_ERR(psr);
 
 	mutex_lock(&psr->lock);
-	psr->active = false;
+	psr_set_state_locked(psr, false);
+	++psr->inhibit_count;
 	mutex_unlock(&psr->lock);
 	cancel_delayed_work_sync(&psr->flush_work);
 
 	return 0;
 }
-EXPORT_SYMBOL(rockchip_drm_psr_deactivate);
+EXPORT_SYMBOL(rockchip_drm_psr_inhibit_get);
 
 static void rockchip_drm_do_flush(struct psr_drv *psr)
 {
-	psr_set_state(psr, PSR_FLUSH);
-	mod_delayed_work(system_wq, &psr->flush_work, PSR_FLUSH_TIMEOUT_MS);
-}
-
-/**
- * rockchip_drm_psr_flush - flush a single pipe
- * @crtc: CRTC of the pipe to flush
- *
- * Returns:
- * 0 on success, -errno on fail
- */
-int rockchip_drm_psr_flush(struct drm_crtc *crtc)
-{
-	struct psr_drv *psr = find_psr_by_crtc(crtc);
-	if (IS_ERR(psr))
-		return PTR_ERR(psr);
+	cancel_delayed_work_sync(&psr->flush_work);
 
-	rockchip_drm_do_flush(psr);
-	return 0;
+	mutex_lock(&psr->lock);
+	if (!psr_set_state_locked(psr, false))
+		mod_delayed_work(system_wq, &psr->flush_work,
+				 PSR_FLUSH_TIMEOUT_MS);
+	mutex_unlock(&psr->lock);
 }
-EXPORT_SYMBOL(rockchip_drm_psr_flush);
 
 /**
  * rockchip_drm_psr_flush_all - force to flush all registered PSR encoders
@@ -225,11 +178,16 @@ EXPORT_SYMBOL(rockchip_drm_psr_flush_all);
  * @encoder: encoder that obtain the PSR function
  * @psr_set: call back to set PSR state
  *
+ * The function returns with PSR inhibit counter initialized with one
+ * and the caller (typically encoder driver) needs to call
+ * rockchip_drm_psr_inhibit_put() when it becomes ready to accept PSR
+ * enable request.
+ *
  * Returns:
  * Zero on success, negative errno on failure.
  */
 int rockchip_drm_psr_register(struct drm_encoder *encoder,
-			void (*psr_set)(struct drm_encoder *, bool enable))
+			int (*psr_set)(struct drm_encoder *, bool enable))
 {
 	struct rockchip_drm_private *drm_drv = encoder->dev->dev_private;
 	struct psr_drv *psr;
@@ -244,8 +202,8 @@ int rockchip_drm_psr_register(struct drm_encoder *encoder,
 	INIT_DELAYED_WORK(&psr->flush_work, psr_flush_handler);
 	mutex_init(&psr->lock);
 
-	psr->active = true;
-	psr->state = PSR_DISABLE;
+	psr->inhibit_count = 1;
+	psr->enabled = false;
 	psr->encoder = encoder;
 	psr->set = psr_set;
 
@@ -262,6 +220,11 @@ EXPORT_SYMBOL(rockchip_drm_psr_register);
  * @encoder: encoder that obtain the PSR function
  * @psr_set: call back to set PSR state
  *
+ * It is expected that the PSR inhibit counter is 1 when this function is
+ * called, which corresponds to a state when related encoder has been
+ * disconnected from any CRTCs and its driver called
+ * rockchip_drm_psr_inhibit_get() to stop the PSR logic.
+ *
  * Returns:
  * Zero on success, negative errno on failure.
  */
@@ -273,7 +236,12 @@ void rockchip_drm_psr_unregister(struct drm_encoder *encoder)
 	mutex_lock(&drm_drv->psr_list_lock);
 	list_for_each_entry_safe(psr, n, &drm_drv->psr_list, list) {
 		if (psr->encoder == encoder) {
-			cancel_delayed_work_sync(&psr->flush_work);
+			/*
+			 * Any other value would mean that the encoder
+			 * is still in use.
+			 */
+			WARN_ON(psr->inhibit_count != 1);
+
 			list_del(&psr->list);
 			kfree(psr);
 		}
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_psr.h b/drivers/gpu/drm/rockchip/rockchip_drm_psr.h
index b1ea0155e57c..860c62494496 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_psr.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_psr.h
@@ -16,13 +16,12 @@
 #define __ROCKCHIP_DRM_PSR___
 
 void rockchip_drm_psr_flush_all(struct drm_device *dev);
-int rockchip_drm_psr_flush(struct drm_crtc *crtc);
 
-int rockchip_drm_psr_activate(struct drm_encoder *encoder);
-int rockchip_drm_psr_deactivate(struct drm_encoder *encoder);
+int rockchip_drm_psr_inhibit_put(struct drm_encoder *encoder);
+int rockchip_drm_psr_inhibit_get(struct drm_encoder *encoder);
 
 int rockchip_drm_psr_register(struct drm_encoder *encoder,
-			void (*psr_set)(struct drm_encoder *, bool enable));
+			int (*psr_set)(struct drm_encoder *, bool enable));
 void rockchip_drm_psr_unregister(struct drm_encoder *encoder);
 
 #endif /* __ROCKCHIP_DRM_PSR__ */
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index 53d4afe15278..2121345a61af 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -76,6 +76,9 @@
 #define VOP_WIN_GET_YRGBADDR(vop, win) \
 		vop_readl(vop, win->base + win->phy->yrgb_mst.offset)
 
+#define VOP_WIN_TO_INDEX(vop_win) \
+	((vop_win) - (vop_win)->vop->win)
+
 #define to_vop(x) container_of(x, struct vop, crtc)
 #define to_vop_win(x) container_of(x, struct vop_win, base)
 
@@ -708,6 +711,7 @@ static void vop_plane_atomic_update(struct drm_plane *plane,
 	dma_addr_t dma_addr;
 	uint32_t val;
 	bool rb_swap;
+	int win_index = VOP_WIN_TO_INDEX(vop_win);
 	int format;
 
 	/*
@@ -777,7 +781,14 @@ static void vop_plane_atomic_update(struct drm_plane *plane,
 	rb_swap = has_rb_swapped(fb->format->format);
 	VOP_WIN_SET(vop, win, rb_swap, rb_swap);
 
-	if (fb->format->has_alpha) {
+	/*
+	 * Blending win0 with the background color doesn't seem to work
+	 * correctly. We only get the background color, no matter the contents
+	 * of the win0 framebuffer.  However, blending pre-multiplied color
+	 * with the default opaque black default background color is a no-op,
+	 * so we can just disable blending to get the correct result.
+	 */
+	if (fb->format->has_alpha && win_index > 0) {
 		VOP_WIN_SET(vop, win, dst_alpha_ctl,
 			    DST_FACTOR_M0(ALPHA_SRC_INVERSE));
 		val = SRC_ALPHA_EN(1) | SRC_COLOR_M0(ALPHA_SRC_PRE_MUL) |
@@ -925,6 +936,12 @@ static void vop_crtc_atomic_enable(struct drm_crtc *crtc,
 	if (s->output_mode == ROCKCHIP_OUT_MODE_AAAA &&
 	    !(vop_data->feature & VOP_FEATURE_OUTPUT_RGB10))
 		s->output_mode = ROCKCHIP_OUT_MODE_P888;
+
+	if (s->output_mode == ROCKCHIP_OUT_MODE_AAAA && s->output_bpc == 8)
+		VOP_REG_SET(vop, common, pre_dither_down, 1);
+	else
+		VOP_REG_SET(vop, common, pre_dither_down, 0);
+
 	VOP_REG_SET(vop, common, out_mode, s->output_mode);
 
 	VOP_REG_SET(vop, modeset, htotal_pw, (htotal << 16) | hsync_len);
@@ -1017,22 +1034,15 @@ static void vop_crtc_atomic_flush(struct drm_crtc *crtc,
 			continue;
 
 		drm_framebuffer_get(old_plane_state->fb);
+		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
 		drm_flip_work_queue(&vop->fb_unref_work, old_plane_state->fb);
 		set_bit(VOP_PENDING_FB_UNREF, &vop->pending);
-		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
 	}
 }
 
-static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
-				  struct drm_crtc_state *old_crtc_state)
-{
-	rockchip_drm_psr_flush(crtc);
-}
-
 static const struct drm_crtc_helper_funcs vop_crtc_helper_funcs = {
 	.mode_fixup = vop_crtc_mode_fixup,
 	.atomic_flush = vop_crtc_atomic_flush,
-	.atomic_begin = vop_crtc_atomic_begin,
 	.atomic_enable = vop_crtc_atomic_enable,
 	.atomic_disable = vop_crtc_atomic_disable,
 };
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
index 56bbd2e2a8ef..084acdd0019a 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
@@ -67,6 +67,7 @@ struct vop_common {
 	struct vop_reg cfg_done;
 	struct vop_reg dsp_blank;
 	struct vop_reg data_blank;
+	struct vop_reg pre_dither_down;
 	struct vop_reg dither_down;
 	struct vop_reg dither_up;
 	struct vop_reg gate_en;
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
index 2e4eea3459fe..08023d3ecb76 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
@@ -264,6 +264,7 @@ static const struct vop_common rk3288_common = {
 	.standby = VOP_REG_SYNC(RK3288_SYS_CTRL, 0x1, 22),
 	.gate_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 23),
 	.mmu_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 20),
+	.pre_dither_down = VOP_REG(RK3288_DSP_CTRL1, 0x1, 1),
 	.dither_down = VOP_REG(RK3288_DSP_CTRL1, 0xf, 1),
 	.dither_up = VOP_REG(RK3288_DSP_CTRL1, 0x1, 6),
 	.data_blank = VOP_REG(RK3288_DSP_CTRL0, 0x1, 19),
diff --git a/drivers/gpu/drm/scheduler/gpu_scheduler.c b/drivers/gpu/drm/scheduler/gpu_scheduler.c
index 0d95888ccc3e..df1578d6f42e 100644
--- a/drivers/gpu/drm/scheduler/gpu_scheduler.c
+++ b/drivers/gpu/drm/scheduler/gpu_scheduler.c
@@ -30,7 +30,7 @@
 #include <drm/spsc_queue.h>
 
 #define CREATE_TRACE_POINTS
-#include <drm/gpu_scheduler_trace.h>
+#include "gpu_scheduler_trace.h"
 
 #define to_drm_sched_job(sched_job)		\
 		container_of((sched_job), struct drm_sched_job, queue_node)
@@ -117,15 +117,15 @@ drm_sched_rq_select_entity(struct drm_sched_rq *rq)
  * @sched	The pointer to the scheduler
  * @entity	The pointer to a valid drm_sched_entity
  * @rq		The run queue this entity belongs
- * @kernel	If this is an entity for the kernel
- * @jobs	The max number of jobs in the job queue
+ * @guilty      atomic_t set to 1 when a job on this queue
+ *              is found to be guilty causing a timeout
  *
  * return 0 if succeed. negative error code on failure
 */
 int drm_sched_entity_init(struct drm_gpu_scheduler *sched,
 			  struct drm_sched_entity *entity,
 			  struct drm_sched_rq *rq,
-			  uint32_t jobs, atomic_t *guilty)
+			  atomic_t *guilty)
 {
 	if (!(sched && entity && rq))
 		return -EINVAL;
@@ -135,9 +135,10 @@ int drm_sched_entity_init(struct drm_gpu_scheduler *sched,
 	entity->rq = rq;
 	entity->sched = sched;
 	entity->guilty = guilty;
+	entity->fini_status = 0;
+	entity->last_scheduled = NULL;
 
 	spin_lock_init(&entity->rq_lock);
-	spin_lock_init(&entity->queue_lock);
 	spsc_queue_init(&entity->job_queue);
 
 	atomic_set(&entity->fence_seq, 0);
@@ -196,19 +197,30 @@ static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
 	return true;
 }
 
+static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
+				    struct dma_fence_cb *cb)
+{
+	struct drm_sched_job *job = container_of(cb, struct drm_sched_job,
+						 finish_cb);
+	drm_sched_fence_finished(job->s_fence);
+	WARN_ON(job->s_fence->parent);
+	dma_fence_put(&job->s_fence->finished);
+	job->sched->ops->free_job(job);
+}
+
+
 /**
  * Destroy a context entity
  *
  * @sched       Pointer to scheduler instance
  * @entity	The pointer to a valid scheduler entity
  *
- * Cleanup and free the allocated resources.
+ * Splitting drm_sched_entity_fini() into two functions, The first one is does the waiting,
+ * removes the entity from the runqueue and returns an error when the process was killed.
  */
-void drm_sched_entity_fini(struct drm_gpu_scheduler *sched,
+void drm_sched_entity_do_release(struct drm_gpu_scheduler *sched,
 			   struct drm_sched_entity *entity)
 {
-	int r;
-
 	if (!drm_sched_entity_is_initialized(sched, entity))
 		return;
 	/**
@@ -216,13 +228,28 @@ void drm_sched_entity_fini(struct drm_gpu_scheduler *sched,
 	 * queued IBs or discard them on SIGKILL
 	*/
 	if ((current->flags & PF_SIGNALED) && current->exit_code == SIGKILL)
-		r = -ERESTARTSYS;
+		entity->fini_status = -ERESTARTSYS;
 	else
-		r = wait_event_killable(sched->job_scheduled,
+		entity->fini_status = wait_event_killable(sched->job_scheduled,
 					drm_sched_entity_is_idle(entity));
 	drm_sched_entity_set_rq(entity, NULL);
-	if (r) {
+}
+EXPORT_SYMBOL(drm_sched_entity_do_release);
+
+/**
+ * Destroy a context entity
+ *
+ * @sched       Pointer to scheduler instance
+ * @entity	The pointer to a valid scheduler entity
+ *
+ * The second one then goes over the entity and signals all jobs with an error code.
+ */
+void drm_sched_entity_cleanup(struct drm_gpu_scheduler *sched,
+			   struct drm_sched_entity *entity)
+{
+	if (entity->fini_status) {
 		struct drm_sched_job *job;
+		int r;
 
 		/* Park the kernel for a moment to make sure it isn't processing
 		 * our enity.
@@ -240,12 +267,25 @@ void drm_sched_entity_fini(struct drm_gpu_scheduler *sched,
 			struct drm_sched_fence *s_fence = job->s_fence;
 			drm_sched_fence_scheduled(s_fence);
 			dma_fence_set_error(&s_fence->finished, -ESRCH);
-			drm_sched_fence_finished(s_fence);
-			WARN_ON(s_fence->parent);
-			dma_fence_put(&s_fence->finished);
-			sched->ops->free_job(job);
+			r = dma_fence_add_callback(entity->last_scheduled, &job->finish_cb,
+							drm_sched_entity_kill_jobs_cb);
+			if (r == -ENOENT)
+				drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb);
+			else if (r)
+				DRM_ERROR("fence add callback failed (%d)\n", r);
 		}
 	}
+
+	dma_fence_put(entity->last_scheduled);
+	entity->last_scheduled = NULL;
+}
+EXPORT_SYMBOL(drm_sched_entity_cleanup);
+
+void drm_sched_entity_fini(struct drm_gpu_scheduler *sched,
+				struct drm_sched_entity *entity)
+{
+	drm_sched_entity_do_release(sched, entity);
+	drm_sched_entity_cleanup(sched, entity);
 }
 EXPORT_SYMBOL(drm_sched_entity_fini);
 
@@ -360,6 +400,9 @@ drm_sched_entity_pop_job(struct drm_sched_entity *entity)
 	if (entity->guilty && atomic_read(entity->guilty))
 		dma_fence_set_error(&sched_job->s_fence->finished, -ECANCELED);
 
+	dma_fence_put(entity->last_scheduled);
+	entity->last_scheduled = dma_fence_get(&sched_job->s_fence->finished);
+
 	spsc_queue_pop(&entity->job_queue);
 	return sched_job;
 }
@@ -369,6 +412,10 @@ drm_sched_entity_pop_job(struct drm_sched_entity *entity)
  *
  * @sched_job		The pointer to job required to submit
  *
+ * Note: To guarantee that the order of insertion to queue matches
+ * the job's fence sequence number this function should be
+ * called with drm_sched_job_init under common lock.
+ *
  * Returns 0 for success, negative error code otherwise.
  */
 void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
@@ -379,11 +426,8 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
 
 	trace_drm_sched_job(sched_job, entity);
 
-	spin_lock(&entity->queue_lock);
 	first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
 
-	spin_unlock(&entity->queue_lock);
-
 	/* first job wakes up scheduler */
 	if (first) {
 		/* Add the entity to the run queue */
@@ -529,6 +573,7 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
 		spin_unlock(&sched->job_list_lock);
 		fence = sched->ops->run_job(s_job);
 		atomic_inc(&sched->hw_rq_count);
+
 		if (fence) {
 			s_fence->parent = dma_fence_get(fence);
 			r = dma_fence_add_callback(fence, &s_fence->cb,
@@ -548,13 +593,19 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
 }
 EXPORT_SYMBOL(drm_sched_job_recovery);
 
-/* init a sched_job with basic field */
+/**
+ * Init a sched_job with basic field
+ *
+ * Note: Refer to drm_sched_entity_push_job documentation
+ * for locking considerations.
+ */
 int drm_sched_job_init(struct drm_sched_job *job,
 		       struct drm_gpu_scheduler *sched,
 		       struct drm_sched_entity *entity,
 		       void *owner)
 {
 	job->sched = sched;
+	job->entity = entity;
 	job->s_priority = entity->rq - sched->sched_rq;
 	job->s_fence = drm_sched_fence_create(entity, owner);
 	if (!job->s_fence)
diff --git a/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h b/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h
new file mode 100644
index 000000000000..4998ad950a48
--- /dev/null
+++ b/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#if !defined(_GPU_SCHED_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _GPU_SCHED_TRACE_H_
+
+#include <linux/stringify.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+#include <drm/drmP.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM gpu_scheduler
+#define TRACE_INCLUDE_FILE gpu_scheduler_trace
+
+TRACE_EVENT(drm_sched_job,
+	    TP_PROTO(struct drm_sched_job *sched_job, struct drm_sched_entity *entity),
+	    TP_ARGS(sched_job, entity),
+	    TP_STRUCT__entry(
+			     __field(struct drm_sched_entity *, entity)
+			     __field(struct dma_fence *, fence)
+			     __field(const char *, name)
+			     __field(uint64_t, id)
+			     __field(u32, job_count)
+			     __field(int, hw_job_count)
+			     ),
+
+	    TP_fast_assign(
+			   __entry->entity = entity;
+			   __entry->id = sched_job->id;
+			   __entry->fence = &sched_job->s_fence->finished;
+			   __entry->name = sched_job->sched->name;
+			   __entry->job_count = spsc_queue_count(&entity->job_queue);
+			   __entry->hw_job_count = atomic_read(
+				   &sched_job->sched->hw_rq_count);
+			   ),
+	    TP_printk("entity=%p, id=%llu, fence=%p, ring=%s, job count:%u, hw job count:%d",
+		      __entry->entity, __entry->id,
+		      __entry->fence, __entry->name,
+		      __entry->job_count, __entry->hw_job_count)
+);
+
+TRACE_EVENT(drm_sched_process_job,
+	    TP_PROTO(struct drm_sched_fence *fence),
+	    TP_ARGS(fence),
+	    TP_STRUCT__entry(
+		    __field(struct dma_fence *, fence)
+		    ),
+
+	    TP_fast_assign(
+		    __entry->fence = &fence->finished;
+		    ),
+	    TP_printk("fence=%p signaled", __entry->fence)
+);
+
+#endif
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/scheduler
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c
index 69aab086b913..df4461648e3f 100644
--- a/drivers/gpu/drm/scheduler/sched_fence.c
+++ b/drivers/gpu/drm/scheduler/sched_fence.c
@@ -87,7 +87,7 @@ static bool drm_sched_fence_enable_signaling(struct dma_fence *f)
 }
 
 /**
- * amd_sched_fence_free - free up the fence memory
+ * drm_sched_fence_free - free up the fence memory
  *
  * @rcu: RCU callback head
  *
@@ -98,12 +98,11 @@ static void drm_sched_fence_free(struct rcu_head *rcu)
 	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
 	struct drm_sched_fence *fence = to_drm_sched_fence(f);
 
-	dma_fence_put(fence->parent);
 	kmem_cache_free(sched_fence_slab, fence);
 }
 
 /**
- * amd_sched_fence_release_scheduled - callback that fence can be freed
+ * drm_sched_fence_release_scheduled - callback that fence can be freed
  *
  * @fence: fence
  *
@@ -114,11 +113,12 @@ static void drm_sched_fence_release_scheduled(struct dma_fence *f)
 {
 	struct drm_sched_fence *fence = to_drm_sched_fence(f);
 
+	dma_fence_put(fence->parent);
 	call_rcu(&fence->finished.rcu, drm_sched_fence_free);
 }
 
 /**
- * amd_sched_fence_release_finished - drop extra reference
+ * drm_sched_fence_release_finished - drop extra reference
  *
  * @f: fence
  *
diff --git a/drivers/gpu/drm/selftests/Makefile b/drivers/gpu/drm/selftests/Makefile
index 4aebfc7f27d4..9fc349fa18e9 100644
--- a/drivers/gpu/drm/selftests/Makefile
+++ b/drivers/gpu/drm/selftests/Makefile
@@ -1 +1 @@
-obj-$(CONFIG_DRM_DEBUG_MM_SELFTEST) += test-drm_mm.o
+obj-$(CONFIG_DRM_DEBUG_SELFTEST) += test-drm_mm.o test-drm-helper.o
diff --git a/drivers/gpu/drm/selftests/drm_helper_selftests.h b/drivers/gpu/drm/selftests/drm_helper_selftests.h
new file mode 100644
index 000000000000..9771290ed228
--- /dev/null
+++ b/drivers/gpu/drm/selftests/drm_helper_selftests.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* List each unit test as selftest(name, function)
+ *
+ * The name is used as both an enum and expanded as igt__name to create
+ * a module parameter. It must be unique and legal for a C identifier.
+ *
+ * Tests are executed in order by igt/drm_selftests_helper
+ */
+selftest(check_plane_state, igt_check_plane_state)
diff --git a/drivers/gpu/drm/selftests/test-drm-helper.c b/drivers/gpu/drm/selftests/test-drm-helper.c
new file mode 100644
index 000000000000..a015712b43e8
--- /dev/null
+++ b/drivers/gpu/drm/selftests/test-drm-helper.c
@@ -0,0 +1,247 @@
+/*
+ * Test cases for the drm_kms_helper functions
+ */
+
+#define pr_fmt(fmt) "drm_kms_helper: " fmt
+
+#include <linux/module.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_modes.h>
+
+#define TESTS "drm_helper_selftests.h"
+#include "drm_selftest.h"
+
+#define FAIL(test, msg, ...) \
+	do { \
+		if (test) { \
+			pr_err("%s/%u: " msg, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
+			return -EINVAL; \
+		} \
+	} while (0)
+
+#define FAIL_ON(x) FAIL((x), "%s", "FAIL_ON(" __stringify(x) ")\n")
+
+static void set_src(struct drm_plane_state *plane_state,
+		    unsigned src_x, unsigned src_y,
+		    unsigned src_w, unsigned src_h)
+{
+	plane_state->src_x = src_x;
+	plane_state->src_y = src_y;
+	plane_state->src_w = src_w;
+	plane_state->src_h = src_h;
+}
+
+static bool check_src_eq(struct drm_plane_state *plane_state,
+			 unsigned src_x, unsigned src_y,
+			 unsigned src_w, unsigned src_h)
+{
+	if (plane_state->src.x1 < 0) {
+		pr_err("src x coordinate %x should never be below 0.\n", plane_state->src.x1);
+		drm_rect_debug_print("src: ", &plane_state->src, true);
+		return false;
+	}
+	if (plane_state->src.y1 < 0) {
+		pr_err("src y coordinate %x should never be below 0.\n", plane_state->src.y1);
+		drm_rect_debug_print("src: ", &plane_state->src, true);
+		return false;
+	}
+
+	if (plane_state->src.x1 != src_x ||
+	    plane_state->src.y1 != src_y ||
+	    drm_rect_width(&plane_state->src) != src_w ||
+	    drm_rect_height(&plane_state->src) != src_h) {
+		drm_rect_debug_print("src: ", &plane_state->src, true);
+		return false;
+	}
+
+	return true;
+}
+
+static void set_crtc(struct drm_plane_state *plane_state,
+		     int crtc_x, int crtc_y,
+		     unsigned crtc_w, unsigned crtc_h)
+{
+	plane_state->crtc_x = crtc_x;
+	plane_state->crtc_y = crtc_y;
+	plane_state->crtc_w = crtc_w;
+	plane_state->crtc_h = crtc_h;
+}
+
+static bool check_crtc_eq(struct drm_plane_state *plane_state,
+			  int crtc_x, int crtc_y,
+			  unsigned crtc_w, unsigned crtc_h)
+{
+	if (plane_state->dst.x1 != crtc_x ||
+	    plane_state->dst.y1 != crtc_y ||
+	    drm_rect_width(&plane_state->dst) != crtc_w ||
+	    drm_rect_height(&plane_state->dst) != crtc_h) {
+		drm_rect_debug_print("dst: ", &plane_state->dst, false);
+
+		return false;
+	}
+
+	return true;
+}
+
+static int igt_check_plane_state(void *ignored)
+{
+	int ret;
+
+	const struct drm_crtc_state crtc_state = {
+		.crtc = ZERO_SIZE_PTR,
+		.enable = true,
+		.active = true,
+		.mode = {
+			DRM_MODE("1024x768", 0, 65000, 1024, 1048,
+				1184, 1344, 0, 768, 771, 777, 806, 0,
+				DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC)
+		},
+	};
+	struct drm_framebuffer fb = {
+		.width = 2048,
+		.height = 2048
+	};
+	struct drm_plane_state plane_state = {
+		.crtc = ZERO_SIZE_PTR,
+		.fb = &fb,
+		.rotation = DRM_MODE_ROTATE_0
+	};
+
+	/* Simple clipping, no scaling. */
+	set_src(&plane_state, 0, 0, fb.width << 16, fb.height << 16);
+	set_crtc(&plane_state, 0, 0, fb.width, fb.height);
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  false, false);
+	FAIL(ret < 0, "Simple clipping check should pass\n");
+	FAIL_ON(!plane_state.visible);
+	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 1024 << 16, 768 << 16));
+	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
+
+	/* Rotated clipping + reflection, no scaling. */
+	plane_state.rotation = DRM_MODE_ROTATE_90 | DRM_MODE_REFLECT_X;
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  false, false);
+	FAIL(ret < 0, "Rotated clipping check should pass\n");
+	FAIL_ON(!plane_state.visible);
+	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 768 << 16, 1024 << 16));
+	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
+	plane_state.rotation = DRM_MODE_ROTATE_0;
+
+	/* Check whether positioning works correctly. */
+	set_src(&plane_state, 0, 0, 1023 << 16, 767 << 16);
+	set_crtc(&plane_state, 0, 0, 1023, 767);
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  false, false);
+	FAIL(!ret, "Should not be able to position on the crtc with can_position=false\n");
+
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  true, false);
+	FAIL(ret < 0, "Simple positioning should work\n");
+	FAIL_ON(!plane_state.visible);
+	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 1023 << 16, 767 << 16));
+	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1023, 767));
+
+	/* Simple scaling tests. */
+	set_src(&plane_state, 0, 0, 512 << 16, 384 << 16);
+	set_crtc(&plane_state, 0, 0, 1024, 768);
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  0x8001,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  false, false);
+	FAIL(!ret, "Upscaling out of range should fail.\n");
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  0x8000,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  false, false);
+	FAIL(ret < 0, "Upscaling exactly 2x should work\n");
+	FAIL_ON(!plane_state.visible);
+	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 512 << 16, 384 << 16));
+	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
+
+	set_src(&plane_state, 0, 0, 2048 << 16, 1536 << 16);
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  0x1ffff, false, false);
+	FAIL(!ret, "Downscaling out of range should fail.\n");
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  0x20000, false, false);
+	FAIL(ret < 0, "Should succeed with exact scaling limit\n");
+	FAIL_ON(!plane_state.visible);
+	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 2048 << 16, 1536 << 16));
+	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
+
+	/* Testing rounding errors. */
+	set_src(&plane_state, 0, 0, 0x40001, 0x40001);
+	set_crtc(&plane_state, 1022, 766, 4, 4);
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  0x10001,
+						  true, false);
+	FAIL(ret < 0, "Should succeed by clipping to exact multiple");
+	FAIL_ON(!plane_state.visible);
+	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 2 << 16, 2 << 16));
+	FAIL_ON(!check_crtc_eq(&plane_state, 1022, 766, 2, 2));
+
+	set_src(&plane_state, 0x20001, 0x20001, 0x4040001, 0x3040001);
+	set_crtc(&plane_state, -2, -2, 1028, 772);
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  0x10001,
+						  false, false);
+	FAIL(ret < 0, "Should succeed by clipping to exact multiple");
+	FAIL_ON(!plane_state.visible);
+	FAIL_ON(!check_src_eq(&plane_state, 0x40002, 0x40002, 1024 << 16, 768 << 16));
+	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
+
+	set_src(&plane_state, 0, 0, 0x3ffff, 0x3ffff);
+	set_crtc(&plane_state, 1022, 766, 4, 4);
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  0xffff,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  true, false);
+	FAIL(ret < 0, "Should succeed by clipping to exact multiple");
+	FAIL_ON(!plane_state.visible);
+	/* Should not be rounded to 0x20001, which would be upscaling. */
+	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 2 << 16, 2 << 16));
+	FAIL_ON(!check_crtc_eq(&plane_state, 1022, 766, 2, 2));
+
+	set_src(&plane_state, 0x1ffff, 0x1ffff, 0x403ffff, 0x303ffff);
+	set_crtc(&plane_state, -2, -2, 1028, 772);
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  0xffff,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  false, false);
+	FAIL(ret < 0, "Should succeed by clipping to exact multiple");
+	FAIL_ON(!plane_state.visible);
+	FAIL_ON(!check_src_eq(&plane_state, 0x3fffe, 0x3fffe, 1024 << 16, 768 << 16));
+	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
+
+	return 0;
+}
+
+#include "drm_selftest.c"
+
+static int __init test_drm_helper_init(void)
+{
+	int err;
+
+	err = run_selftests(selftests, ARRAY_SIZE(selftests), NULL);
+
+	return err > 0 ? 0 : err;
+}
+
+module_init(test_drm_helper_init);
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/shmobile/Kconfig b/drivers/gpu/drm/shmobile/Kconfig
index c987c826daa3..0426d66660d1 100644
--- a/drivers/gpu/drm/shmobile/Kconfig
+++ b/drivers/gpu/drm/shmobile/Kconfig
@@ -2,7 +2,6 @@ config DRM_SHMOBILE
 	tristate "DRM Support for SH Mobile"
 	depends on DRM && ARM
 	depends on ARCH_SHMOBILE || COMPILE_TEST
-	depends on FB_SH_MOBILE_MERAM || !FB_SH_MOBILE_MERAM
 	select BACKLIGHT_CLASS_DEVICE
 	select BACKLIGHT_LCD_SUPPORT
 	select DRM_KMS_HELPER
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
index e7738939a86d..40df8887fc17 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
@@ -21,8 +21,6 @@
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_plane_helper.h>
 
-#include <video/sh_mobile_meram.h>
-
 #include "shmob_drm_backlight.h"
 #include "shmob_drm_crtc.h"
 #include "shmob_drm_drv.h"
@@ -47,20 +45,12 @@ static int shmob_drm_clk_on(struct shmob_drm_device *sdev)
 		if (ret < 0)
 			return ret;
 	}
-#if 0
-	if (sdev->meram_dev && sdev->meram_dev->pdev)
-		pm_runtime_get_sync(&sdev->meram_dev->pdev->dev);
-#endif
 
 	return 0;
 }
 
 static void shmob_drm_clk_off(struct shmob_drm_device *sdev)
 {
-#if 0
-	if (sdev->meram_dev && sdev->meram_dev->pdev)
-		pm_runtime_put_sync(&sdev->meram_dev->pdev->dev);
-#endif
 	if (sdev->clock)
 		clk_disable_unprepare(sdev->clock);
 }
@@ -269,12 +259,6 @@ static void shmob_drm_crtc_stop(struct shmob_drm_crtc *scrtc)
 	if (!scrtc->started)
 		return;
 
-	/* Disable the MERAM cache. */
-	if (scrtc->cache) {
-		sh_mobile_meram_cache_free(sdev->meram, scrtc->cache);
-		scrtc->cache = NULL;
-	}
-
 	/* Stop the LCDC. */
 	shmob_drm_crtc_start_stop(scrtc, false);
 
@@ -305,7 +289,6 @@ static void shmob_drm_crtc_compute_base(struct shmob_drm_crtc *scrtc,
 {
 	struct drm_crtc *crtc = &scrtc->crtc;
 	struct drm_framebuffer *fb = crtc->primary->fb;
-	struct shmob_drm_device *sdev = crtc->dev->dev_private;
 	struct drm_gem_cma_object *gem;
 	unsigned int bpp;
 
@@ -321,11 +304,6 @@ static void shmob_drm_crtc_compute_base(struct shmob_drm_crtc *scrtc,
 			      + y / (bpp == 4 ? 2 : 1) * fb->pitches[1]
 			      + x * (bpp == 16 ? 2 : 1);
 	}
-
-	if (scrtc->cache)
-		sh_mobile_meram_cache_update(sdev->meram, scrtc->cache,
-					     scrtc->dma[0], scrtc->dma[1],
-					     &scrtc->dma[0], &scrtc->dma[1]);
 }
 
 static void shmob_drm_crtc_update_base(struct shmob_drm_crtc *scrtc)
@@ -372,9 +350,7 @@ static int shmob_drm_crtc_mode_set(struct drm_crtc *crtc,
 {
 	struct shmob_drm_crtc *scrtc = to_shmob_crtc(crtc);
 	struct shmob_drm_device *sdev = crtc->dev->dev_private;
-	const struct sh_mobile_meram_cfg *mdata = sdev->pdata->meram;
 	const struct shmob_drm_format_info *format;
-	void *cache;
 
 	format = shmob_drm_format_info(crtc->primary->fb->format->format);
 	if (format == NULL) {
@@ -386,24 +362,6 @@ static int shmob_drm_crtc_mode_set(struct drm_crtc *crtc,
 	scrtc->format = format;
 	scrtc->line_size = crtc->primary->fb->pitches[0];
 
-	if (sdev->meram) {
-		/* Enable MERAM cache if configured. We need to de-init
-		 * configured ICBs before we can re-initialize them.
-		 */
-		if (scrtc->cache) {
-			sh_mobile_meram_cache_free(sdev->meram, scrtc->cache);
-			scrtc->cache = NULL;
-		}
-
-		cache = sh_mobile_meram_cache_alloc(sdev->meram, mdata,
-						    crtc->primary->fb->pitches[0],
-						    adjusted_mode->vdisplay,
-						    format->meram,
-						    &scrtc->line_size);
-		if (!IS_ERR(cache))
-			scrtc->cache = cache;
-	}
-
 	shmob_drm_crtc_compute_base(scrtc, x, y);
 
 	return 0;
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_crtc.h b/drivers/gpu/drm/shmobile/shmob_drm_crtc.h
index f152973df11c..c11f421737dc 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_crtc.h
+++ b/drivers/gpu/drm/shmobile/shmob_drm_crtc.h
@@ -28,7 +28,6 @@ struct shmob_drm_crtc {
 	int dpms;
 
 	const struct shmob_drm_format_info *format;
-	void *cache;
 	unsigned long dma[2];
 	unsigned int line_size;
 	bool started;
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_drv.h b/drivers/gpu/drm/shmobile/shmob_drm_drv.h
index 02ea315ba69a..088a6e55fa29 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_drv.h
+++ b/drivers/gpu/drm/shmobile/shmob_drm_drv.h
@@ -23,7 +23,6 @@
 struct clk;
 struct device;
 struct drm_device;
-struct sh_mobile_meram_info;
 
 struct shmob_drm_device {
 	struct device *dev;
@@ -31,7 +30,6 @@ struct shmob_drm_device {
 
 	void __iomem *mmio;
 	struct clk *clock;
-	struct sh_mobile_meram_info *meram;
 	u32 lddckr;
 	u32 ldmt1r;
 
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_kms.c b/drivers/gpu/drm/shmobile/shmob_drm_kms.c
index d36919b14da7..447638581c08 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_kms.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_kms.c
@@ -18,8 +18,6 @@
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 
-#include <video/sh_mobile_meram.h>
-
 #include "shmob_drm_crtc.h"
 #include "shmob_drm_drv.h"
 #include "shmob_drm_kms.h"
@@ -35,55 +33,46 @@ static const struct shmob_drm_format_info shmob_drm_format_infos[] = {
 		.bpp = 16,
 		.yuv = false,
 		.lddfr = LDDFR_PKF_RGB16,
-		.meram = SH_MOBILE_MERAM_PF_RGB,
 	}, {
 		.fourcc = DRM_FORMAT_RGB888,
 		.bpp = 24,
 		.yuv = false,
 		.lddfr = LDDFR_PKF_RGB24,
-		.meram = SH_MOBILE_MERAM_PF_RGB,
 	}, {
 		.fourcc = DRM_FORMAT_ARGB8888,
 		.bpp = 32,
 		.yuv = false,
 		.lddfr = LDDFR_PKF_ARGB32,
-		.meram = SH_MOBILE_MERAM_PF_RGB,
 	}, {
 		.fourcc = DRM_FORMAT_NV12,
 		.bpp = 12,
 		.yuv = true,
 		.lddfr = LDDFR_CC | LDDFR_YF_420,
-		.meram = SH_MOBILE_MERAM_PF_NV,
 	}, {
 		.fourcc = DRM_FORMAT_NV21,
 		.bpp = 12,
 		.yuv = true,
 		.lddfr = LDDFR_CC | LDDFR_YF_420,
-		.meram = SH_MOBILE_MERAM_PF_NV,
 	}, {
 		.fourcc = DRM_FORMAT_NV16,
 		.bpp = 16,
 		.yuv = true,
 		.lddfr = LDDFR_CC | LDDFR_YF_422,
-		.meram = SH_MOBILE_MERAM_PF_NV,
 	}, {
 		.fourcc = DRM_FORMAT_NV61,
 		.bpp = 16,
 		.yuv = true,
 		.lddfr = LDDFR_CC | LDDFR_YF_422,
-		.meram = SH_MOBILE_MERAM_PF_NV,
 	}, {
 		.fourcc = DRM_FORMAT_NV24,
 		.bpp = 24,
 		.yuv = true,
 		.lddfr = LDDFR_CC | LDDFR_YF_444,
-		.meram = SH_MOBILE_MERAM_PF_NV24,
 	}, {
 		.fourcc = DRM_FORMAT_NV42,
 		.bpp = 24,
 		.yuv = true,
 		.lddfr = LDDFR_CC | LDDFR_YF_444,
-		.meram = SH_MOBILE_MERAM_PF_NV24,
 	},
 };
 
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_kms.h b/drivers/gpu/drm/shmobile/shmob_drm_kms.h
index 06d5b7caa026..753e2817dc2c 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_kms.h
+++ b/drivers/gpu/drm/shmobile/shmob_drm_kms.h
@@ -24,7 +24,6 @@ struct shmob_drm_format_info {
 	unsigned int bpp;
 	bool yuv;
 	u32 lddfr;
-	unsigned int meram;
 };
 
 const struct shmob_drm_format_info *shmob_drm_format_info(u32 fourcc);
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_plane.c b/drivers/gpu/drm/shmobile/shmob_drm_plane.c
index 97f6e4a3eb0d..1d0359f713ca 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_plane.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_plane.c
@@ -17,8 +17,6 @@
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 
-#include <video/sh_mobile_meram.h>
-
 #include "shmob_drm_drv.h"
 #include "shmob_drm_kms.h"
 #include "shmob_drm_plane.h"
diff --git a/drivers/gpu/drm/sti/Kconfig b/drivers/gpu/drm/sti/Kconfig
index cca4b3c9aeb5..1963cc1b1cc5 100644
--- a/drivers/gpu/drm/sti/Kconfig
+++ b/drivers/gpu/drm/sti/Kconfig
@@ -1,6 +1,6 @@
 config DRM_STI
 	tristate "DRM Support for STMicroelectronics SoC stiH4xx Series"
-	depends on DRM && (ARCH_STI || ARCH_MULTIPLATFORM)
+	depends on OF && DRM && (ARCH_STI || ARCH_MULTIPLATFORM)
 	select RESET_CONTROLLER
 	select DRM_KMS_HELPER
 	select DRM_GEM_CMA_HELPER
@@ -8,6 +8,5 @@ config DRM_STI
 	select DRM_PANEL
 	select FW_LOADER
 	select SND_SOC_HDMI_CODEC if SND_SOC
-	select OF
 	help
 	  Choose this option to enable DRM on STM stiH4xx chipset
diff --git a/drivers/gpu/drm/sti/sti_crtc.c b/drivers/gpu/drm/sti/sti_crtc.c
index 21e50d7b1f86..5824e6aca8f4 100644
--- a/drivers/gpu/drm/sti/sti_crtc.c
+++ b/drivers/gpu/drm/sti/sti_crtc.c
@@ -357,7 +357,7 @@ int sti_crtc_init(struct drm_device *drm_dev, struct sti_mixer *mixer,
 	res = drm_crtc_init_with_planes(drm_dev, crtc, primary, cursor,
 					&sti_crtc_funcs, NULL);
 	if (res) {
-		DRM_ERROR("Can't initialze CRTC\n");
+		DRM_ERROR("Can't initialize CRTC\n");
 		return -EINVAL;
 	}
 
diff --git a/drivers/gpu/drm/sti/sti_drv.c b/drivers/gpu/drm/sti/sti_drv.c
index 55b6967d27e1..90c46b49c931 100644
--- a/drivers/gpu/drm/sti/sti_drv.c
+++ b/drivers/gpu/drm/sti/sti_drv.c
@@ -119,30 +119,10 @@ static int sti_drm_dbg_init(struct drm_minor *minor)
 	return ret;
 }
 
-static int sti_atomic_check(struct drm_device *dev,
-			    struct drm_atomic_state *state)
-{
-	int ret;
-
-	ret = drm_atomic_helper_check_modeset(dev, state);
-	if (ret)
-		return ret;
-
-	ret = drm_atomic_normalize_zpos(dev, state);
-	if (ret)
-		return ret;
-
-	ret = drm_atomic_helper_check_planes(dev, state);
-	if (ret)
-		return ret;
-
-	return ret;
-}
-
 static const struct drm_mode_config_funcs sti_mode_config_funcs = {
 	.fb_create = drm_gem_fb_create,
 	.output_poll_changed = drm_fb_helper_output_poll_changed,
-	.atomic_check = sti_atomic_check,
+	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
@@ -160,6 +140,8 @@ static void sti_mode_config_init(struct drm_device *dev)
 	dev->mode_config.max_height = STI_MAX_FB_HEIGHT;
 
 	dev->mode_config.funcs = &sti_mode_config_funcs;
+
+	dev->mode_config.normalize_zpos = true;
 }
 
 DEFINE_DRM_GEM_CMA_FOPS(sti_driver_fops);
diff --git a/drivers/gpu/drm/sti/sti_plane.c b/drivers/gpu/drm/sti/sti_plane.c
index b074609c960a..b48cd86e0250 100644
--- a/drivers/gpu/drm/sti/sti_plane.c
+++ b/drivers/gpu/drm/sti/sti_plane.c
@@ -40,6 +40,7 @@ void sti_plane_update_fps(struct sti_plane *plane,
 			  bool new_frame,
 			  bool new_field)
 {
+	struct drm_plane_state *state = plane->drm_plane.state;
 	ktime_t now;
 	struct sti_fps_info *fps;
 	int fpks, fipks, ms_since_last, num_frames, num_fields;
@@ -66,14 +67,14 @@ void sti_plane_update_fps(struct sti_plane *plane,
 	fps->last_timestamp = now;
 	fps->last_frame_counter = fps->curr_frame_counter;
 
-	if (plane->drm_plane.fb) {
+	if (state->fb) {
 		fpks = (num_frames * 1000000) / ms_since_last;
 		snprintf(plane->fps_info.fps_str, FPS_LENGTH,
 			 "%-8s %4dx%-4d %.4s @ %3d.%-3.3d fps (%s)",
 			 plane->drm_plane.name,
-			 plane->drm_plane.fb->width,
-			 plane->drm_plane.fb->height,
-			 (char *)&plane->drm_plane.fb->format->format,
+			 state->fb->width,
+			 state->fb->height,
+			 (char *)&state->fb->format->format,
 			 fpks / 1000, fpks % 1000,
 			 sti_plane_to_str(plane));
 	}
diff --git a/drivers/gpu/drm/stm/drv.c b/drivers/gpu/drm/stm/drv.c
index 9ab00a87f7cc..8698e08313e1 100644
--- a/drivers/gpu/drm/stm/drv.c
+++ b/drivers/gpu/drm/stm/drv.c
@@ -72,8 +72,6 @@ static struct drm_driver drv_driver = {
 	.gem_prime_vmap = drm_gem_cma_prime_vmap,
 	.gem_prime_vunmap = drm_gem_cma_prime_vunmap,
 	.gem_prime_mmap = drm_gem_cma_prime_mmap,
-	.enable_vblank = ltdc_crtc_enable_vblank,
-	.disable_vblank = ltdc_crtc_disable_vblank,
 };
 
 static int drv_load(struct drm_device *ddev)
diff --git a/drivers/gpu/drm/stm/ltdc.c b/drivers/gpu/drm/stm/ltdc.c
index 1a3277e483d5..d997a6014d6c 100644
--- a/drivers/gpu/drm/stm/ltdc.c
+++ b/drivers/gpu/drm/stm/ltdc.c
@@ -392,9 +392,6 @@ static void ltdc_crtc_update_clut(struct drm_crtc *crtc)
 	u32 val;
 	int i;
 
-	if (!crtc || !crtc->state)
-		return;
-
 	if (!crtc->state->color_mgmt_changed || !crtc->state->gamma_lut)
 		return;
 
@@ -448,6 +445,43 @@ static void ltdc_crtc_atomic_disable(struct drm_crtc *crtc,
 	reg_set(ldev->regs, LTDC_SRCR, SRCR_IMR);
 }
 
+#define CLK_TOLERANCE_HZ 50
+
+static enum drm_mode_status
+ltdc_crtc_mode_valid(struct drm_crtc *crtc,
+		     const struct drm_display_mode *mode)
+{
+	struct ltdc_device *ldev = crtc_to_ltdc(crtc);
+	int target = mode->clock * 1000;
+	int target_min = target - CLK_TOLERANCE_HZ;
+	int target_max = target + CLK_TOLERANCE_HZ;
+	int result;
+
+	/*
+	 * Accept all "preferred" modes:
+	 * - this is important for panels because panel clock tolerances are
+	 *   bigger than hdmi ones and there is no reason to not accept them
+	 *   (the fps may vary a little but it is not a problem).
+	 * - the hdmi preferred mode will be accepted too, but userland will
+	 *   be able to use others hdmi "valid" modes if necessary.
+	 */
+	if (mode->type & DRM_MODE_TYPE_PREFERRED)
+		return MODE_OK;
+
+	result = clk_round_rate(ldev->pixel_clk, target);
+
+	DRM_DEBUG_DRIVER("clk rate target %d, available %d\n", target, result);
+
+	/*
+	 * Filter modes according to the clock value, particularly useful for
+	 * hdmi modes that require precise pixel clocks.
+	 */
+	if (result < target_min || result > target_max)
+		return MODE_CLOCK_RANGE;
+
+	return MODE_OK;
+}
+
 static bool ltdc_crtc_mode_fixup(struct drm_crtc *crtc,
 				 const struct drm_display_mode *mode,
 				 struct drm_display_mode *adjusted_mode)
@@ -562,6 +596,7 @@ static void ltdc_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs ltdc_crtc_helper_funcs = {
+	.mode_valid = ltdc_crtc_mode_valid,
 	.mode_fixup = ltdc_crtc_mode_fixup,
 	.mode_set_nofb = ltdc_crtc_mode_set_nofb,
 	.atomic_flush = ltdc_crtc_atomic_flush,
@@ -569,9 +604,9 @@ static const struct drm_crtc_helper_funcs ltdc_crtc_helper_funcs = {
 	.atomic_disable = ltdc_crtc_atomic_disable,
 };
 
-int ltdc_crtc_enable_vblank(struct drm_device *ddev, unsigned int pipe)
+static int ltdc_crtc_enable_vblank(struct drm_crtc *crtc)
 {
-	struct ltdc_device *ldev = ddev->dev_private;
+	struct ltdc_device *ldev = crtc_to_ltdc(crtc);
 
 	DRM_DEBUG_DRIVER("\n");
 	reg_set(ldev->regs, LTDC_IER, IER_LIE);
@@ -579,9 +614,9 @@ int ltdc_crtc_enable_vblank(struct drm_device *ddev, unsigned int pipe)
 	return 0;
 }
 
-void ltdc_crtc_disable_vblank(struct drm_device *ddev, unsigned int pipe)
+static void ltdc_crtc_disable_vblank(struct drm_crtc *crtc)
 {
-	struct ltdc_device *ldev = ddev->dev_private;
+	struct ltdc_device *ldev = crtc_to_ltdc(crtc);
 
 	DRM_DEBUG_DRIVER("\n");
 	reg_clear(ldev->regs, LTDC_IER, IER_LIE);
@@ -594,6 +629,8 @@ static const struct drm_crtc_funcs ltdc_crtc_funcs = {
 	.reset = drm_atomic_helper_crtc_reset,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+	.enable_vblank = ltdc_crtc_enable_vblank,
+	.disable_vblank = ltdc_crtc_disable_vblank,
 	.gamma_set = drm_atomic_helper_legacy_gamma_set,
 };
 
@@ -727,6 +764,8 @@ static void ltdc_plane_atomic_update(struct drm_plane *plane,
 	reg_update_bits(ldev->regs, LTDC_L1CR + lofs,
 			LXCR_LEN | LXCR_CLUTEN, val);
 
+	ldev->plane_fpsi[plane->index].counter++;
+
 	mutex_lock(&ldev->err_lock);
 	if (ldev->error_status & ISR_FUIF) {
 		DRM_DEBUG_DRIVER("Fifo underrun\n");
@@ -752,6 +791,25 @@ static void ltdc_plane_atomic_disable(struct drm_plane *plane,
 			 oldstate->crtc->base.id, plane->base.id);
 }
 
+static void ltdc_plane_atomic_print_state(struct drm_printer *p,
+					  const struct drm_plane_state *state)
+{
+	struct drm_plane *plane = state->plane;
+	struct ltdc_device *ldev = plane_to_ltdc(plane);
+	struct fps_info *fpsi = &ldev->plane_fpsi[plane->index];
+	int ms_since_last;
+	ktime_t now;
+
+	now = ktime_get();
+	ms_since_last = ktime_to_ms(ktime_sub(now, fpsi->last_timestamp));
+
+	drm_printf(p, "\tuser_updates=%dfps\n",
+		   DIV_ROUND_CLOSEST(fpsi->counter * 1000, ms_since_last));
+
+	fpsi->last_timestamp = now;
+	fpsi->counter = 0;
+}
+
 static const struct drm_plane_funcs ltdc_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
@@ -759,6 +817,7 @@ static const struct drm_plane_funcs ltdc_plane_funcs = {
 	.reset = drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+	.atomic_print_state = ltdc_plane_atomic_print_state,
 };
 
 static const struct drm_plane_helper_funcs ltdc_plane_helper_funcs = {
@@ -801,13 +860,13 @@ static struct drm_plane *ltdc_plane_create(struct drm_device *ddev,
 
 	plane = devm_kzalloc(dev, sizeof(*plane), GFP_KERNEL);
 	if (!plane)
-		return 0;
+		return NULL;
 
 	ret = drm_universal_plane_init(ddev, plane, possible_crtcs,
 				       &ltdc_plane_funcs, formats, nb_fmt,
 				       NULL, type, NULL);
 	if (ret < 0)
-		return 0;
+		return NULL;
 
 	drm_plane_helper_add(plane, &ltdc_plane_helper_funcs);
 
@@ -966,14 +1025,13 @@ int ltdc_load(struct drm_device *ddev)
 						  &bridge[i]);
 
 		/*
-		 * If at least one endpoint is ready, continue probing,
-		 * else if at least one endpoint is -EPROBE_DEFER and
-		 * there is no previous ready endpoints, defer probing.
+		 * If at least one endpoint is -EPROBE_DEFER, defer probing,
+		 * else if at least one endpoint is ready, continue probing.
 		 */
-		if (!ret)
+		if (ret == -EPROBE_DEFER)
+			return ret;
+		else if (!ret)
 			endpoint_not_ready = 0;
-		else if (ret == -EPROBE_DEFER && endpoint_not_ready)
-			endpoint_not_ready = -EPROBE_DEFER;
 	}
 
 	if (endpoint_not_ready)
diff --git a/drivers/gpu/drm/stm/ltdc.h b/drivers/gpu/drm/stm/ltdc.h
index edb268129c54..1e16d6afb0d2 100644
--- a/drivers/gpu/drm/stm/ltdc.h
+++ b/drivers/gpu/drm/stm/ltdc.h
@@ -20,6 +20,13 @@ struct ltdc_caps {
 	bool non_alpha_only_l1; /* non-native no-alpha formats on layer 1 */
 };
 
+#define LTDC_MAX_LAYER	4
+
+struct fps_info {
+	unsigned int counter;
+	ktime_t last_timestamp;
+};
+
 struct ltdc_device {
 	void __iomem *regs;
 	struct clk *pixel_clk;	/* lcd pixel clock */
@@ -27,10 +34,9 @@ struct ltdc_device {
 	struct ltdc_caps caps;
 	u32 error_status;
 	u32 irq_status;
+	struct fps_info plane_fpsi[LTDC_MAX_LAYER];
 };
 
-int ltdc_crtc_enable_vblank(struct drm_device *dev, unsigned int pipe);
-void ltdc_crtc_disable_vblank(struct drm_device *dev, unsigned int pipe);
 int ltdc_load(struct drm_device *ddev);
 void ltdc_unload(struct drm_device *ddev);
 
diff --git a/drivers/gpu/drm/sun4i/Kconfig b/drivers/gpu/drm/sun4i/Kconfig
index eee6bc0eaf97..156a865c3e6d 100644
--- a/drivers/gpu/drm/sun4i/Kconfig
+++ b/drivers/gpu/drm/sun4i/Kconfig
@@ -40,6 +40,16 @@ config DRM_SUN4I_BACKEND
 	  do some alpha blending and feed graphics to TCON. If M is
 	  selected the module will be called sun4i-backend.
 
+config DRM_SUN6I_DSI
+	tristate "Allwinner A31 MIPI-DSI Controller Support"
+	default MACH_SUN8I
+	select CRC_CCITT
+	select DRM_MIPI_DSI
+	help
+	  Choose this option if you want have an Allwinner SoC with
+	  MIPI-DSI support. If M is selected the module will be called
+	  sun6i-dsi
+
 config DRM_SUN8I_DW_HDMI
 	tristate "Support for Allwinner version of DesignWare HDMI"
 	depends on DRM_SUN4I
diff --git a/drivers/gpu/drm/sun4i/Makefile b/drivers/gpu/drm/sun4i/Makefile
index 330843ce4280..2589f4acd5ae 100644
--- a/drivers/gpu/drm/sun4i/Makefile
+++ b/drivers/gpu/drm/sun4i/Makefile
@@ -24,6 +24,9 @@ sun4i-tcon-y			+= sun4i_lvds.o
 sun4i-tcon-y			+= sun4i_tcon.o
 sun4i-tcon-y			+= sun4i_rgb.o
 
+sun6i-dsi-y			+= sun6i_mipi_dphy.o
+sun6i-dsi-y			+= sun6i_mipi_dsi.o
+
 obj-$(CONFIG_DRM_SUN4I)		+= sun4i-drm.o
 obj-$(CONFIG_DRM_SUN4I)		+= sun4i-tcon.o
 obj-$(CONFIG_DRM_SUN4I)		+= sun4i_tv.o
@@ -31,5 +34,6 @@ obj-$(CONFIG_DRM_SUN4I)		+= sun6i_drc.o
 
 obj-$(CONFIG_DRM_SUN4I_BACKEND)	+= sun4i-backend.o sun4i-frontend.o
 obj-$(CONFIG_DRM_SUN4I_HDMI)	+= sun4i-drm-hdmi.o
+obj-$(CONFIG_DRM_SUN6I_DSI)	+= sun6i-dsi.o
 obj-$(CONFIG_DRM_SUN8I_DW_HDMI)	+= sun8i-drm-hdmi.o
 obj-$(CONFIG_DRM_SUN8I_MIXER)	+= sun8i-mixer.o
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c
index 9bad54f3de38..de0a76dfa1a2 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.c
@@ -295,6 +295,15 @@ int sun4i_backend_update_layer_formats(struct sun4i_backend *backend,
 	DRM_DEBUG_DRIVER("Switching display backend interlaced mode %s\n",
 			 interlaced ? "on" : "off");
 
+	val = SUN4I_BACKEND_ATTCTL_REG0_LAY_GLBALPHA(state->alpha >> 8);
+	if (state->alpha != DRM_BLEND_ALPHA_OPAQUE)
+		val |= SUN4I_BACKEND_ATTCTL_REG0_LAY_GLBALPHA_EN;
+	regmap_update_bits(backend->engine.regs,
+			   SUN4I_BACKEND_ATTCTL_REG0(layer),
+			   SUN4I_BACKEND_ATTCTL_REG0_LAY_GLBALPHA_MASK |
+			   SUN4I_BACKEND_ATTCTL_REG0_LAY_GLBALPHA_EN,
+			   val);
+
 	if (sun4i_backend_format_is_yuv(fb->format->format))
 		return sun4i_backend_update_yuv_format(backend, layer, plane);
 
@@ -490,7 +499,7 @@ static int sun4i_backend_atomic_check(struct sunxi_engine *engine,
 		DRM_DEBUG_DRIVER("Plane FB format is %s\n",
 				 drm_get_format_name(fb->format->format,
 						     &format_name));
-		if (fb->format->has_alpha)
+		if (fb->format->has_alpha || (plane_state->alpha != DRM_BLEND_ALPHA_OPAQUE))
 			num_alpha_planes++;
 
 		if (sun4i_backend_format_is_yuv(fb->format->format)) {
@@ -548,7 +557,8 @@ static int sun4i_backend_atomic_check(struct sunxi_engine *engine,
 	}
 
 	/* We can't have an alpha plane at the lowest position */
-	if (plane_states[0]->fb->format->has_alpha)
+	if (plane_states[0]->fb->format->has_alpha ||
+	    (plane_states[0]->alpha != DRM_BLEND_ALPHA_OPAQUE))
 		return -EINVAL;
 
 	for (i = 1; i < num_planes; i++) {
@@ -560,7 +570,7 @@ static int sun4i_backend_atomic_check(struct sunxi_engine *engine,
 		 * The only alpha position is the lowest plane of the
 		 * second pipe.
 		 */
-		if (fb->format->has_alpha)
+		if (fb->format->has_alpha || (p_state->alpha != DRM_BLEND_ALPHA_OPAQUE))
 			current_pipe++;
 
 		s_state->pipe = current_pipe;
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.h b/drivers/gpu/drm/sun4i/sun4i_backend.h
index 316f2179e9e1..4caee0392fa4 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.h
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.h
@@ -68,12 +68,15 @@
 #define SUN4I_BACKEND_CKMIN_REG			0x884
 #define SUN4I_BACKEND_CKCFG_REG			0x888
 #define SUN4I_BACKEND_ATTCTL_REG0(l)		(0x890 + (0x4 * (l)))
+#define SUN4I_BACKEND_ATTCTL_REG0_LAY_GLBALPHA_MASK	GENMASK(31, 24)
+#define SUN4I_BACKEND_ATTCTL_REG0_LAY_GLBALPHA(x)		((x) << 24)
 #define SUN4I_BACKEND_ATTCTL_REG0_LAY_PIPESEL_MASK	BIT(15)
 #define SUN4I_BACKEND_ATTCTL_REG0_LAY_PIPESEL(x)		((x) << 15)
 #define SUN4I_BACKEND_ATTCTL_REG0_LAY_PRISEL_MASK	GENMASK(11, 10)
 #define SUN4I_BACKEND_ATTCTL_REG0_LAY_PRISEL(x)			((x) << 10)
 #define SUN4I_BACKEND_ATTCTL_REG0_LAY_YUVEN		BIT(2)
 #define SUN4I_BACKEND_ATTCTL_REG0_LAY_VDOEN		BIT(1)
+#define SUN4I_BACKEND_ATTCTL_REG0_LAY_GLBALPHA_EN	BIT(0)
 
 #define SUN4I_BACKEND_ATTCTL_REG1(l)		(0x8a0 + (0x4 * (l)))
 #define SUN4I_BACKEND_ATTCTL_REG1_LAY_HSCAFCT		GENMASK(15, 14)
diff --git a/drivers/gpu/drm/sun4i/sun4i_layer.c b/drivers/gpu/drm/sun4i/sun4i_layer.c
index 2949a3c912c1..750ad24de1d7 100644
--- a/drivers/gpu/drm/sun4i/sun4i_layer.c
+++ b/drivers/gpu/drm/sun4i/sun4i_layer.c
@@ -37,6 +37,7 @@ static void sun4i_backend_layer_reset(struct drm_plane *plane)
 	if (state) {
 		plane->state = &state->state;
 		plane->state->plane = plane;
+		plane->state->alpha = DRM_BLEND_ALPHA_OPAQUE;
 		plane->state->zpos = layer->id;
 	}
 }
@@ -167,6 +168,7 @@ static struct sun4i_layer *sun4i_layer_init_one(struct drm_device *drm,
 			     &sun4i_backend_layer_helper_funcs);
 	layer->backend = backend;
 
+	drm_plane_create_alpha_property(&layer->plane);
 	drm_plane_create_zpos_property(&layer->plane, 0, 0,
 				       SUN4I_BACKEND_NUM_LAYERS - 1);
 
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c
index c3d92d537240..08747fc3ee71 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c
@@ -35,6 +35,7 @@
 #include "sun4i_lvds.h"
 #include "sun4i_rgb.h"
 #include "sun4i_tcon.h"
+#include "sun6i_mipi_dsi.h"
 #include "sunxi_engine.h"
 
 static struct drm_connector *sun4i_tcon_get_connector(const struct drm_encoder *encoder)
@@ -169,6 +170,7 @@ void sun4i_tcon_set_status(struct sun4i_tcon *tcon,
 	case DRM_MODE_ENCODER_LVDS:
 		is_lvds = true;
 		/* Fallthrough */
+	case DRM_MODE_ENCODER_DSI:
 	case DRM_MODE_ENCODER_NONE:
 		channel = 0;
 		break;
@@ -201,7 +203,8 @@ void sun4i_tcon_enable_vblank(struct sun4i_tcon *tcon, bool enable)
 	DRM_DEBUG_DRIVER("%sabling VBLANK interrupt\n", enable ? "En" : "Dis");
 
 	mask = SUN4I_TCON_GINT0_VBLANK_ENABLE(0) |
-	       SUN4I_TCON_GINT0_VBLANK_ENABLE(1);
+		SUN4I_TCON_GINT0_VBLANK_ENABLE(1) |
+		SUN4I_TCON_GINT0_TCON0_TRI_FINISH_ENABLE;
 
 	if (enable)
 		val = mask;
@@ -273,6 +276,71 @@ static void sun4i_tcon0_mode_set_common(struct sun4i_tcon *tcon,
 		     SUN4I_TCON0_BASIC0_Y(mode->crtc_vdisplay));
 }
 
+static void sun4i_tcon0_mode_set_cpu(struct sun4i_tcon *tcon,
+				     struct mipi_dsi_device *device,
+				     const struct drm_display_mode *mode)
+{
+	u8 bpp = mipi_dsi_pixel_format_to_bpp(device->format);
+	u8 lanes = device->lanes;
+	u32 block_space, start_delay;
+	u32 tcon_div;
+
+	tcon->dclk_min_div = 4;
+	tcon->dclk_max_div = 127;
+
+	sun4i_tcon0_mode_set_common(tcon, mode);
+
+	regmap_update_bits(tcon->regs, SUN4I_TCON0_CTL_REG,
+			   SUN4I_TCON0_CTL_IF_MASK,
+			   SUN4I_TCON0_CTL_IF_8080);
+
+	regmap_write(tcon->regs, SUN4I_TCON_ECC_FIFO_REG,
+		     SUN4I_TCON_ECC_FIFO_EN);
+
+	regmap_write(tcon->regs, SUN4I_TCON0_CPU_IF_REG,
+		     SUN4I_TCON0_CPU_IF_MODE_DSI |
+		     SUN4I_TCON0_CPU_IF_TRI_FIFO_FLUSH |
+		     SUN4I_TCON0_CPU_IF_TRI_FIFO_EN |
+		     SUN4I_TCON0_CPU_IF_TRI_EN);
+
+	/*
+	 * This looks suspicious, but it works...
+	 *
+	 * The datasheet says that this should be set higher than 20 *
+	 * pixel cycle, but it's not clear what a pixel cycle is.
+	 */
+	regmap_read(tcon->regs, SUN4I_TCON0_DCLK_REG, &tcon_div);
+	tcon_div &= GENMASK(6, 0);
+	block_space = mode->htotal * bpp / (tcon_div * lanes);
+	block_space -= mode->hdisplay + 40;
+
+	regmap_write(tcon->regs, SUN4I_TCON0_CPU_TRI0_REG,
+		     SUN4I_TCON0_CPU_TRI0_BLOCK_SPACE(block_space) |
+		     SUN4I_TCON0_CPU_TRI0_BLOCK_SIZE(mode->hdisplay));
+
+	regmap_write(tcon->regs, SUN4I_TCON0_CPU_TRI1_REG,
+		     SUN4I_TCON0_CPU_TRI1_BLOCK_NUM(mode->vdisplay));
+
+	start_delay = (mode->crtc_vtotal - mode->crtc_vdisplay - 10 - 1);
+	start_delay = start_delay * mode->crtc_htotal * 149;
+	start_delay = start_delay / (mode->crtc_clock / 1000) / 8;
+	regmap_write(tcon->regs, SUN4I_TCON0_CPU_TRI2_REG,
+		     SUN4I_TCON0_CPU_TRI2_TRANS_START_SET(10) |
+		     SUN4I_TCON0_CPU_TRI2_START_DELAY(start_delay));
+
+	/*
+	 * The Allwinner BSP has a comment that the period should be
+	 * the display clock * 15, but uses an hardcoded 3000...
+	 */
+	regmap_write(tcon->regs, SUN4I_TCON_SAFE_PERIOD_REG,
+		     SUN4I_TCON_SAFE_PERIOD_NUM(3000) |
+		     SUN4I_TCON_SAFE_PERIOD_MODE(3));
+
+	/* Enable the output on the pins */
+	regmap_write(tcon->regs, SUN4I_TCON0_IO_TRI_REG,
+		     0xe0000000);
+}
+
 static void sun4i_tcon0_mode_set_lvds(struct sun4i_tcon *tcon,
 				      const struct drm_encoder *encoder,
 				      const struct drm_display_mode *mode)
@@ -538,7 +606,17 @@ void sun4i_tcon_mode_set(struct sun4i_tcon *tcon,
 			 const struct drm_encoder *encoder,
 			 const struct drm_display_mode *mode)
 {
+	struct sun6i_dsi *dsi;
+
 	switch (encoder->encoder_type) {
+	case DRM_MODE_ENCODER_DSI:
+		/*
+		 * This is not really elegant, but it's the "cleaner"
+		 * way I could think of...
+		 */
+		dsi = encoder_to_sun6i_dsi(encoder);
+		sun4i_tcon0_mode_set_cpu(tcon, dsi->device, mode);
+		break;
 	case DRM_MODE_ENCODER_LVDS:
 		sun4i_tcon0_mode_set_lvds(tcon, encoder, mode);
 		break;
@@ -582,7 +660,8 @@ static irqreturn_t sun4i_tcon_handler(int irq, void *private)
 	regmap_read(tcon->regs, SUN4I_TCON_GINT0_REG, &status);
 
 	if (!(status & (SUN4I_TCON_GINT0_VBLANK_INT(0) |
-			SUN4I_TCON_GINT0_VBLANK_INT(1))))
+			SUN4I_TCON_GINT0_VBLANK_INT(1) |
+			SUN4I_TCON_GINT0_TCON0_TRI_FINISH_INT)))
 		return IRQ_NONE;
 
 	drm_crtc_handle_vblank(&scrtc->crtc);
@@ -591,7 +670,8 @@ static irqreturn_t sun4i_tcon_handler(int irq, void *private)
 	/* Acknowledge the interrupt */
 	regmap_update_bits(tcon->regs, SUN4I_TCON_GINT0_REG,
 			   SUN4I_TCON_GINT0_VBLANK_INT(0) |
-			   SUN4I_TCON_GINT0_VBLANK_INT(1),
+			   SUN4I_TCON_GINT0_VBLANK_INT(1) |
+			   SUN4I_TCON_GINT0_TCON0_TRI_FINISH_INT,
 			   0);
 
 	if (engine->ops->vblank_quirk)
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.h b/drivers/gpu/drm/sun4i/sun4i_tcon.h
index 161e09427124..f6a071cd5a6f 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.h
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.h
@@ -28,13 +28,32 @@
 
 #define SUN4I_TCON_GINT0_REG			0x4
 #define SUN4I_TCON_GINT0_VBLANK_ENABLE(pipe)		BIT(31 - (pipe))
+#define SUN4I_TCON_GINT0_TCON0_TRI_FINISH_ENABLE	BIT(27)
+#define SUN4I_TCON_GINT0_TCON0_TRI_COUNTER_ENABLE	BIT(26)
 #define SUN4I_TCON_GINT0_VBLANK_INT(pipe)		BIT(15 - (pipe))
+#define SUN4I_TCON_GINT0_TCON0_TRI_FINISH_INT		BIT(11)
+#define SUN4I_TCON_GINT0_TCON0_TRI_COUNTER_INT		BIT(10)
 
 #define SUN4I_TCON_GINT1_REG			0x8
+
 #define SUN4I_TCON_FRM_CTL_REG			0x10
+#define SUN4I_TCON_FRM_CTL_EN				BIT(31)
+
+#define SUN4I_TCON_FRM_SEED_PR_REG		0x14
+#define SUN4I_TCON_FRM_SEED_PG_REG		0x18
+#define SUN4I_TCON_FRM_SEED_PB_REG		0x1c
+#define SUN4I_TCON_FRM_SEED_LR_REG		0x20
+#define SUN4I_TCON_FRM_SEED_LG_REG		0x24
+#define SUN4I_TCON_FRM_SEED_LB_REG		0x28
+#define SUN4I_TCON_FRM_TBL0_REG			0x2c
+#define SUN4I_TCON_FRM_TBL1_REG			0x30
+#define SUN4I_TCON_FRM_TBL2_REG			0x34
+#define SUN4I_TCON_FRM_TBL3_REG			0x38
 
 #define SUN4I_TCON0_CTL_REG			0x40
 #define SUN4I_TCON0_CTL_TCON_ENABLE			BIT(31)
+#define SUN4I_TCON0_CTL_IF_MASK				GENMASK(25, 24)
+#define SUN4I_TCON0_CTL_IF_8080				(1 << 24)
 #define SUN4I_TCON0_CTL_CLK_DELAY_MASK			GENMASK(8, 4)
 #define SUN4I_TCON0_CTL_CLK_DELAY(delay)		((delay << 4) & SUN4I_TCON0_CTL_CLK_DELAY_MASK)
 #define SUN4I_TCON0_CTL_SRC_SEL_MASK			GENMASK(2, 0)
@@ -61,7 +80,14 @@
 #define SUN4I_TCON0_BASIC3_V_SYNC(height)		(((height) - 1) & 0x7ff)
 
 #define SUN4I_TCON0_HV_IF_REG			0x58
+
 #define SUN4I_TCON0_CPU_IF_REG			0x60
+#define SUN4I_TCON0_CPU_IF_MODE_MASK			GENMASK(31, 28)
+#define SUN4I_TCON0_CPU_IF_MODE_DSI			(1 << 28)
+#define SUN4I_TCON0_CPU_IF_TRI_FIFO_FLUSH		BIT(16)
+#define SUN4I_TCON0_CPU_IF_TRI_FIFO_EN			BIT(2)
+#define SUN4I_TCON0_CPU_IF_TRI_EN			BIT(0)
+
 #define SUN4I_TCON0_CPU_WR_REG			0x64
 #define SUN4I_TCON0_CPU_RD0_REG			0x68
 #define SUN4I_TCON0_CPU_RDA_REG			0x6c
@@ -128,6 +154,10 @@
 
 #define SUN4I_TCON1_IO_POL_REG			0xf0
 #define SUN4I_TCON1_IO_TRI_REG			0xf4
+
+#define SUN4I_TCON_ECC_FIFO_REG			0xf8
+#define SUN4I_TCON_ECC_FIFO_EN				BIT(3)
+
 #define SUN4I_TCON_CEU_CTL_REG			0x100
 #define SUN4I_TCON_CEU_MUL_RR_REG		0x110
 #define SUN4I_TCON_CEU_MUL_RG_REG		0x114
@@ -144,6 +174,22 @@
 #define SUN4I_TCON_CEU_RANGE_R_REG		0x140
 #define SUN4I_TCON_CEU_RANGE_G_REG		0x144
 #define SUN4I_TCON_CEU_RANGE_B_REG		0x148
+
+#define SUN4I_TCON0_CPU_TRI0_REG		0x160
+#define SUN4I_TCON0_CPU_TRI0_BLOCK_SPACE(space)		((((space) - 1) & 0xfff) << 16)
+#define SUN4I_TCON0_CPU_TRI0_BLOCK_SIZE(size)		(((size) - 1) & 0xfff)
+
+#define SUN4I_TCON0_CPU_TRI1_REG		0x164
+#define SUN4I_TCON0_CPU_TRI1_BLOCK_NUM(num)		(((num) - 1) & 0xffff)
+
+#define SUN4I_TCON0_CPU_TRI2_REG		0x168
+#define SUN4I_TCON0_CPU_TRI2_START_DELAY(delay)		(((delay) & 0xffff) << 16)
+#define SUN4I_TCON0_CPU_TRI2_TRANS_START_SET(set)	((set) & 0xfff)
+
+#define SUN4I_TCON_SAFE_PERIOD_REG		0x1f0
+#define SUN4I_TCON_SAFE_PERIOD_NUM(num)			(((num) & 0xfff) << 16)
+#define SUN4I_TCON_SAFE_PERIOD_MODE(mode)		((mode) & 0x3)
+
 #define SUN4I_TCON_MUX_CTRL_REG			0x200
 
 #define SUN4I_TCON0_LVDS_ANA0_REG		0x220
diff --git a/drivers/gpu/drm/sun4i/sun6i_mipi_dphy.c b/drivers/gpu/drm/sun4i/sun6i_mipi_dphy.c
new file mode 100644
index 000000000000..e4d19431fa0e
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun6i_mipi_dphy.c
@@ -0,0 +1,292 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2016 Allwinnertech Co., Ltd.
+ * Copyright (C) 2017-2018 Bootlin
+ *
+ * Maxime Ripard <maxime.ripard@free-electrons.com>
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/of_address.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+#include "sun6i_mipi_dsi.h"
+
+#define SUN6I_DPHY_GCTL_REG		0x00
+#define SUN6I_DPHY_GCTL_LANE_NUM(n)		((((n) - 1) & 3) << 4)
+#define SUN6I_DPHY_GCTL_EN			BIT(0)
+
+#define SUN6I_DPHY_TX_CTL_REG		0x04
+#define SUN6I_DPHY_TX_CTL_HS_TX_CLK_CONT	BIT(28)
+
+#define SUN6I_DPHY_TX_TIME0_REG		0x10
+#define SUN6I_DPHY_TX_TIME0_HS_TRAIL(n)		(((n) & 0xff) << 24)
+#define SUN6I_DPHY_TX_TIME0_HS_PREPARE(n)	(((n) & 0xff) << 16)
+#define SUN6I_DPHY_TX_TIME0_LP_CLK_DIV(n)	((n) & 0xff)
+
+#define SUN6I_DPHY_TX_TIME1_REG		0x14
+#define SUN6I_DPHY_TX_TIME1_CLK_POST(n)		(((n) & 0xff) << 24)
+#define SUN6I_DPHY_TX_TIME1_CLK_PRE(n)		(((n) & 0xff) << 16)
+#define SUN6I_DPHY_TX_TIME1_CLK_ZERO(n)		(((n) & 0xff) << 8)
+#define SUN6I_DPHY_TX_TIME1_CLK_PREPARE(n)	((n) & 0xff)
+
+#define SUN6I_DPHY_TX_TIME2_REG		0x18
+#define SUN6I_DPHY_TX_TIME2_CLK_TRAIL(n)	((n) & 0xff)
+
+#define SUN6I_DPHY_TX_TIME3_REG		0x1c
+
+#define SUN6I_DPHY_TX_TIME4_REG		0x20
+#define SUN6I_DPHY_TX_TIME4_HS_TX_ANA1(n)	(((n) & 0xff) << 8)
+#define SUN6I_DPHY_TX_TIME4_HS_TX_ANA0(n)	((n) & 0xff)
+
+#define SUN6I_DPHY_ANA0_REG		0x4c
+#define SUN6I_DPHY_ANA0_REG_PWS			BIT(31)
+#define SUN6I_DPHY_ANA0_REG_DMPC		BIT(28)
+#define SUN6I_DPHY_ANA0_REG_DMPD(n)		(((n) & 0xf) << 24)
+#define SUN6I_DPHY_ANA0_REG_SLV(n)		(((n) & 7) << 12)
+#define SUN6I_DPHY_ANA0_REG_DEN(n)		(((n) & 0xf) << 8)
+
+#define SUN6I_DPHY_ANA1_REG		0x50
+#define SUN6I_DPHY_ANA1_REG_VTTMODE		BIT(31)
+#define SUN6I_DPHY_ANA1_REG_CSMPS(n)		(((n) & 3) << 28)
+#define SUN6I_DPHY_ANA1_REG_SVTT(n)		(((n) & 0xf) << 24)
+
+#define SUN6I_DPHY_ANA2_REG		0x54
+#define SUN6I_DPHY_ANA2_EN_P2S_CPU(n)		(((n) & 0xf) << 24)
+#define SUN6I_DPHY_ANA2_EN_P2S_CPU_MASK		GENMASK(27, 24)
+#define SUN6I_DPHY_ANA2_EN_CK_CPU		BIT(4)
+#define SUN6I_DPHY_ANA2_REG_ENIB		BIT(1)
+
+#define SUN6I_DPHY_ANA3_REG		0x58
+#define SUN6I_DPHY_ANA3_EN_VTTD(n)		(((n) & 0xf) << 28)
+#define SUN6I_DPHY_ANA3_EN_VTTD_MASK		GENMASK(31, 28)
+#define SUN6I_DPHY_ANA3_EN_VTTC			BIT(27)
+#define SUN6I_DPHY_ANA3_EN_DIV			BIT(26)
+#define SUN6I_DPHY_ANA3_EN_LDOC			BIT(25)
+#define SUN6I_DPHY_ANA3_EN_LDOD			BIT(24)
+#define SUN6I_DPHY_ANA3_EN_LDOR			BIT(18)
+
+#define SUN6I_DPHY_ANA4_REG		0x5c
+#define SUN6I_DPHY_ANA4_REG_DMPLVC		BIT(24)
+#define SUN6I_DPHY_ANA4_REG_DMPLVD(n)		(((n) & 0xf) << 20)
+#define SUN6I_DPHY_ANA4_REG_CKDV(n)		(((n) & 0x1f) << 12)
+#define SUN6I_DPHY_ANA4_REG_TMSC(n)		(((n) & 3) << 10)
+#define SUN6I_DPHY_ANA4_REG_TMSD(n)		(((n) & 3) << 8)
+#define SUN6I_DPHY_ANA4_REG_TXDNSC(n)		(((n) & 3) << 6)
+#define SUN6I_DPHY_ANA4_REG_TXDNSD(n)		(((n) & 3) << 4)
+#define SUN6I_DPHY_ANA4_REG_TXPUSC(n)		(((n) & 3) << 2)
+#define SUN6I_DPHY_ANA4_REG_TXPUSD(n)		((n) & 3)
+
+#define SUN6I_DPHY_DBG5_REG		0xf4
+
+int sun6i_dphy_init(struct sun6i_dphy *dphy, unsigned int lanes)
+{
+	reset_control_deassert(dphy->reset);
+	clk_prepare_enable(dphy->mod_clk);
+	clk_set_rate_exclusive(dphy->mod_clk, 150000000);
+
+	regmap_write(dphy->regs, SUN6I_DPHY_TX_CTL_REG,
+		     SUN6I_DPHY_TX_CTL_HS_TX_CLK_CONT);
+
+	regmap_write(dphy->regs, SUN6I_DPHY_TX_TIME0_REG,
+		     SUN6I_DPHY_TX_TIME0_LP_CLK_DIV(14) |
+		     SUN6I_DPHY_TX_TIME0_HS_PREPARE(6) |
+		     SUN6I_DPHY_TX_TIME0_HS_TRAIL(10));
+
+	regmap_write(dphy->regs, SUN6I_DPHY_TX_TIME1_REG,
+		     SUN6I_DPHY_TX_TIME1_CLK_PREPARE(7) |
+		     SUN6I_DPHY_TX_TIME1_CLK_ZERO(50) |
+		     SUN6I_DPHY_TX_TIME1_CLK_PRE(3) |
+		     SUN6I_DPHY_TX_TIME1_CLK_POST(10));
+
+	regmap_write(dphy->regs, SUN6I_DPHY_TX_TIME2_REG,
+		     SUN6I_DPHY_TX_TIME2_CLK_TRAIL(30));
+
+	regmap_write(dphy->regs, SUN6I_DPHY_TX_TIME3_REG, 0);
+
+	regmap_write(dphy->regs, SUN6I_DPHY_TX_TIME4_REG,
+		     SUN6I_DPHY_TX_TIME4_HS_TX_ANA0(3) |
+		     SUN6I_DPHY_TX_TIME4_HS_TX_ANA1(3));
+
+	regmap_write(dphy->regs, SUN6I_DPHY_GCTL_REG,
+		     SUN6I_DPHY_GCTL_LANE_NUM(lanes) |
+		     SUN6I_DPHY_GCTL_EN);
+
+	return 0;
+}
+
+int sun6i_dphy_power_on(struct sun6i_dphy *dphy, unsigned int lanes)
+{
+	u8 lanes_mask = GENMASK(lanes - 1, 0);
+
+	regmap_write(dphy->regs, SUN6I_DPHY_ANA0_REG,
+		     SUN6I_DPHY_ANA0_REG_PWS |
+		     SUN6I_DPHY_ANA0_REG_DMPC |
+		     SUN6I_DPHY_ANA0_REG_SLV(7) |
+		     SUN6I_DPHY_ANA0_REG_DMPD(lanes_mask) |
+		     SUN6I_DPHY_ANA0_REG_DEN(lanes_mask));
+
+	regmap_write(dphy->regs, SUN6I_DPHY_ANA1_REG,
+		     SUN6I_DPHY_ANA1_REG_CSMPS(1) |
+		     SUN6I_DPHY_ANA1_REG_SVTT(7));
+
+	regmap_write(dphy->regs, SUN6I_DPHY_ANA4_REG,
+		     SUN6I_DPHY_ANA4_REG_CKDV(1) |
+		     SUN6I_DPHY_ANA4_REG_TMSC(1) |
+		     SUN6I_DPHY_ANA4_REG_TMSD(1) |
+		     SUN6I_DPHY_ANA4_REG_TXDNSC(1) |
+		     SUN6I_DPHY_ANA4_REG_TXDNSD(1) |
+		     SUN6I_DPHY_ANA4_REG_TXPUSC(1) |
+		     SUN6I_DPHY_ANA4_REG_TXPUSD(1) |
+		     SUN6I_DPHY_ANA4_REG_DMPLVC |
+		     SUN6I_DPHY_ANA4_REG_DMPLVD(lanes_mask));
+
+	regmap_write(dphy->regs, SUN6I_DPHY_ANA2_REG,
+		     SUN6I_DPHY_ANA2_REG_ENIB);
+	udelay(5);
+
+	regmap_write(dphy->regs, SUN6I_DPHY_ANA3_REG,
+		     SUN6I_DPHY_ANA3_EN_LDOR |
+		     SUN6I_DPHY_ANA3_EN_LDOC |
+		     SUN6I_DPHY_ANA3_EN_LDOD);
+	udelay(1);
+
+	regmap_update_bits(dphy->regs, SUN6I_DPHY_ANA3_REG,
+			   SUN6I_DPHY_ANA3_EN_VTTC |
+			   SUN6I_DPHY_ANA3_EN_VTTD_MASK,
+			   SUN6I_DPHY_ANA3_EN_VTTC |
+			   SUN6I_DPHY_ANA3_EN_VTTD(lanes_mask));
+	udelay(1);
+
+	regmap_update_bits(dphy->regs, SUN6I_DPHY_ANA3_REG,
+			   SUN6I_DPHY_ANA3_EN_DIV,
+			   SUN6I_DPHY_ANA3_EN_DIV);
+	udelay(1);
+
+	regmap_update_bits(dphy->regs, SUN6I_DPHY_ANA2_REG,
+			   SUN6I_DPHY_ANA2_EN_CK_CPU,
+			   SUN6I_DPHY_ANA2_EN_CK_CPU);
+	udelay(1);
+
+	regmap_update_bits(dphy->regs, SUN6I_DPHY_ANA1_REG,
+			   SUN6I_DPHY_ANA1_REG_VTTMODE,
+			   SUN6I_DPHY_ANA1_REG_VTTMODE);
+
+	regmap_update_bits(dphy->regs, SUN6I_DPHY_ANA2_REG,
+			   SUN6I_DPHY_ANA2_EN_P2S_CPU_MASK,
+			   SUN6I_DPHY_ANA2_EN_P2S_CPU(lanes_mask));
+
+	return 0;
+}
+
+int sun6i_dphy_power_off(struct sun6i_dphy *dphy)
+{
+	regmap_update_bits(dphy->regs, SUN6I_DPHY_ANA1_REG,
+			   SUN6I_DPHY_ANA1_REG_VTTMODE, 0);
+
+	return 0;
+}
+
+int sun6i_dphy_exit(struct sun6i_dphy *dphy)
+{
+	clk_rate_exclusive_put(dphy->mod_clk);
+	clk_disable_unprepare(dphy->mod_clk);
+	reset_control_assert(dphy->reset);
+
+	return 0;
+}
+
+static struct regmap_config sun6i_dphy_regmap_config = {
+	.reg_bits	= 32,
+	.val_bits	= 32,
+	.reg_stride	= 4,
+	.max_register	= SUN6I_DPHY_DBG5_REG,
+	.name		= "mipi-dphy",
+};
+
+static const struct of_device_id sun6i_dphy_of_table[] = {
+	{ .compatible = "allwinner,sun6i-a31-mipi-dphy" },
+	{ }
+};
+
+int sun6i_dphy_probe(struct sun6i_dsi *dsi, struct device_node *node)
+{
+	struct sun6i_dphy *dphy;
+	struct resource res;
+	void __iomem *regs;
+	int ret;
+
+	if (!of_match_node(sun6i_dphy_of_table, node)) {
+		dev_err(dsi->dev, "Incompatible D-PHY\n");
+		return -EINVAL;
+	}
+
+	dphy = devm_kzalloc(dsi->dev, sizeof(*dphy), GFP_KERNEL);
+	if (!dphy)
+		return -ENOMEM;
+
+	ret = of_address_to_resource(node, 0, &res);
+	if (ret) {
+		dev_err(dsi->dev, "phy: Couldn't get our resources\n");
+		return ret;
+	}
+
+	regs = devm_ioremap_resource(dsi->dev, &res);
+	if (IS_ERR(regs)) {
+		dev_err(dsi->dev, "Couldn't map the DPHY encoder registers\n");
+		return PTR_ERR(regs);
+	}
+
+	dphy->regs = devm_regmap_init_mmio(dsi->dev, regs,
+					   &sun6i_dphy_regmap_config);
+	if (IS_ERR(dphy->regs)) {
+		dev_err(dsi->dev, "Couldn't create the DPHY encoder regmap\n");
+		return PTR_ERR(dphy->regs);
+	}
+
+	dphy->reset = of_reset_control_get_shared(node, NULL);
+	if (IS_ERR(dphy->reset)) {
+		dev_err(dsi->dev, "Couldn't get our reset line\n");
+		return PTR_ERR(dphy->reset);
+	}
+
+	dphy->bus_clk = of_clk_get_by_name(node, "bus");
+	if (IS_ERR(dphy->bus_clk)) {
+		dev_err(dsi->dev, "Couldn't get the DPHY bus clock\n");
+		ret = PTR_ERR(dphy->bus_clk);
+		goto err_free_reset;
+	}
+	regmap_mmio_attach_clk(dphy->regs, dphy->bus_clk);
+
+	dphy->mod_clk = of_clk_get_by_name(node, "mod");
+	if (IS_ERR(dphy->mod_clk)) {
+		dev_err(dsi->dev, "Couldn't get the DPHY mod clock\n");
+		ret = PTR_ERR(dphy->mod_clk);
+		goto err_free_bus;
+	}
+
+	dsi->dphy = dphy;
+
+	return 0;
+
+err_free_bus:
+	regmap_mmio_detach_clk(dphy->regs);
+	clk_put(dphy->bus_clk);
+err_free_reset:
+	reset_control_put(dphy->reset);
+	return ret;
+}
+
+int sun6i_dphy_remove(struct sun6i_dsi *dsi)
+{
+	struct sun6i_dphy *dphy = dsi->dphy;
+
+	regmap_mmio_detach_clk(dphy->regs);
+	clk_put(dphy->mod_clk);
+	clk_put(dphy->bus_clk);
+	reset_control_put(dphy->reset);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c
new file mode 100644
index 000000000000..bfbf761f0c1d
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c
@@ -0,0 +1,1107 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2016 Allwinnertech Co., Ltd.
+ * Copyright (C) 2017-2018 Bootlin
+ *
+ * Maxime Ripard <maxime.ripard@bootlin.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/crc-ccitt.h>
+#include <linux/of_address.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+#include <linux/phy/phy.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+
+#include "sun4i_drv.h"
+#include "sun6i_mipi_dsi.h"
+
+#include <video/mipi_display.h>
+
+#define SUN6I_DSI_CTL_REG		0x000
+#define SUN6I_DSI_CTL_EN			BIT(0)
+
+#define SUN6I_DSI_BASIC_CTL_REG		0x00c
+#define SUN6I_DSI_BASIC_CTL_HBP_DIS		BIT(2)
+#define SUN6I_DSI_BASIC_CTL_HSA_HSE_DIS		BIT(1)
+#define SUN6I_DSI_BASIC_CTL_VIDEO_BURST		BIT(0)
+
+#define SUN6I_DSI_BASIC_CTL0_REG	0x010
+#define SUN6I_DSI_BASIC_CTL0_HS_EOTP_EN		BIT(18)
+#define SUN6I_DSI_BASIC_CTL0_CRC_EN		BIT(17)
+#define SUN6I_DSI_BASIC_CTL0_ECC_EN		BIT(16)
+#define SUN6I_DSI_BASIC_CTL0_INST_ST		BIT(0)
+
+#define SUN6I_DSI_BASIC_CTL1_REG	0x014
+#define SUN6I_DSI_BASIC_CTL1_VIDEO_ST_DELAY(n)	(((n) & 0x1fff) << 4)
+#define SUN6I_DSI_BASIC_CTL1_VIDEO_FILL		BIT(2)
+#define SUN6I_DSI_BASIC_CTL1_VIDEO_PRECISION	BIT(1)
+#define SUN6I_DSI_BASIC_CTL1_VIDEO_MODE		BIT(0)
+
+#define SUN6I_DSI_BASIC_SIZE0_REG	0x018
+#define SUN6I_DSI_BASIC_SIZE0_VBP(n)		(((n) & 0xfff) << 16)
+#define SUN6I_DSI_BASIC_SIZE0_VSA(n)		((n) & 0xfff)
+
+#define SUN6I_DSI_BASIC_SIZE1_REG	0x01c
+#define SUN6I_DSI_BASIC_SIZE1_VT(n)		(((n) & 0xfff) << 16)
+#define SUN6I_DSI_BASIC_SIZE1_VACT(n)		((n) & 0xfff)
+
+#define SUN6I_DSI_INST_FUNC_REG(n)	(0x020 + (n) * 0x04)
+#define SUN6I_DSI_INST_FUNC_INST_MODE(n)	(((n) & 0xf) << 28)
+#define SUN6I_DSI_INST_FUNC_ESCAPE_ENTRY(n)	(((n) & 0xf) << 24)
+#define SUN6I_DSI_INST_FUNC_TRANS_PACKET(n)	(((n) & 0xf) << 20)
+#define SUN6I_DSI_INST_FUNC_LANE_CEN		BIT(4)
+#define SUN6I_DSI_INST_FUNC_LANE_DEN(n)		((n) & 0xf)
+
+#define SUN6I_DSI_INST_LOOP_SEL_REG	0x040
+
+#define SUN6I_DSI_INST_LOOP_NUM_REG(n)	(0x044 + (n) * 0x10)
+#define SUN6I_DSI_INST_LOOP_NUM_N1(n)		(((n) & 0xfff) << 16)
+#define SUN6I_DSI_INST_LOOP_NUM_N0(n)		((n) & 0xfff)
+
+#define SUN6I_DSI_INST_JUMP_SEL_REG	0x048
+
+#define SUN6I_DSI_INST_JUMP_CFG_REG(n)	(0x04c + (n) * 0x04)
+#define SUN6I_DSI_INST_JUMP_CFG_TO(n)		(((n) & 0xf) << 20)
+#define SUN6I_DSI_INST_JUMP_CFG_POINT(n)	(((n) & 0xf) << 16)
+#define SUN6I_DSI_INST_JUMP_CFG_NUM(n)		((n) & 0xffff)
+
+#define SUN6I_DSI_TRANS_START_REG	0x060
+
+#define SUN6I_DSI_TRANS_ZERO_REG	0x078
+
+#define SUN6I_DSI_TCON_DRQ_REG		0x07c
+#define SUN6I_DSI_TCON_DRQ_ENABLE_MODE		BIT(28)
+#define SUN6I_DSI_TCON_DRQ_SET(n)		((n) & 0x3ff)
+
+#define SUN6I_DSI_PIXEL_CTL0_REG	0x080
+#define SUN6I_DSI_PIXEL_CTL0_PD_PLUG_DISABLE	BIT(16)
+#define SUN6I_DSI_PIXEL_CTL0_FORMAT(n)		((n) & 0xf)
+
+#define SUN6I_DSI_PIXEL_CTL1_REG	0x084
+
+#define SUN6I_DSI_PIXEL_PH_REG		0x090
+#define SUN6I_DSI_PIXEL_PH_ECC(n)		(((n) & 0xff) << 24)
+#define SUN6I_DSI_PIXEL_PH_WC(n)		(((n) & 0xffff) << 8)
+#define SUN6I_DSI_PIXEL_PH_VC(n)		(((n) & 3) << 6)
+#define SUN6I_DSI_PIXEL_PH_DT(n)		((n) & 0x3f)
+
+#define SUN6I_DSI_PIXEL_PF0_REG		0x098
+#define SUN6I_DSI_PIXEL_PF0_CRC_FORCE(n)	((n) & 0xffff)
+
+#define SUN6I_DSI_PIXEL_PF1_REG		0x09c
+#define SUN6I_DSI_PIXEL_PF1_CRC_INIT_LINEN(n)	(((n) & 0xffff) << 16)
+#define SUN6I_DSI_PIXEL_PF1_CRC_INIT_LINE0(n)	((n) & 0xffff)
+
+#define SUN6I_DSI_SYNC_HSS_REG		0x0b0
+
+#define SUN6I_DSI_SYNC_HSE_REG		0x0b4
+
+#define SUN6I_DSI_SYNC_VSS_REG		0x0b8
+
+#define SUN6I_DSI_SYNC_VSE_REG		0x0bc
+
+#define SUN6I_DSI_BLK_HSA0_REG		0x0c0
+
+#define SUN6I_DSI_BLK_HSA1_REG		0x0c4
+#define SUN6I_DSI_BLK_PF(n)			(((n) & 0xffff) << 16)
+#define SUN6I_DSI_BLK_PD(n)			((n) & 0xff)
+
+#define SUN6I_DSI_BLK_HBP0_REG		0x0c8
+
+#define SUN6I_DSI_BLK_HBP1_REG		0x0cc
+
+#define SUN6I_DSI_BLK_HFP0_REG		0x0d0
+
+#define SUN6I_DSI_BLK_HFP1_REG		0x0d4
+
+#define SUN6I_DSI_BLK_HBLK0_REG		0x0e0
+
+#define SUN6I_DSI_BLK_HBLK1_REG		0x0e4
+
+#define SUN6I_DSI_BLK_VBLK0_REG		0x0e8
+
+#define SUN6I_DSI_BLK_VBLK1_REG		0x0ec
+
+#define SUN6I_DSI_BURST_LINE_REG	0x0f0
+#define SUN6I_DSI_BURST_LINE_SYNC_POINT(n)	(((n) & 0xffff) << 16)
+#define SUN6I_DSI_BURST_LINE_NUM(n)		((n) & 0xffff)
+
+#define SUN6I_DSI_BURST_DRQ_REG		0x0f4
+#define SUN6I_DSI_BURST_DRQ_EDGE1(n)		(((n) & 0xffff) << 16)
+#define SUN6I_DSI_BURST_DRQ_EDGE0(n)		((n) & 0xffff)
+
+#define SUN6I_DSI_CMD_CTL_REG		0x200
+#define SUN6I_DSI_CMD_CTL_RX_OVERFLOW		BIT(26)
+#define SUN6I_DSI_CMD_CTL_RX_FLAG		BIT(25)
+#define SUN6I_DSI_CMD_CTL_TX_FLAG		BIT(9)
+
+#define SUN6I_DSI_CMD_RX_REG(n)		(0x240 + (n) * 0x04)
+
+#define SUN6I_DSI_DEBUG_DATA_REG	0x2f8
+
+#define SUN6I_DSI_CMD_TX_REG(n)		(0x300 + (n) * 0x04)
+
+enum sun6i_dsi_start_inst {
+	DSI_START_LPRX,
+	DSI_START_LPTX,
+	DSI_START_HSC,
+	DSI_START_HSD,
+};
+
+enum sun6i_dsi_inst_id {
+	DSI_INST_ID_LP11	= 0,
+	DSI_INST_ID_TBA,
+	DSI_INST_ID_HSC,
+	DSI_INST_ID_HSD,
+	DSI_INST_ID_LPDT,
+	DSI_INST_ID_HSCEXIT,
+	DSI_INST_ID_NOP,
+	DSI_INST_ID_DLY,
+	DSI_INST_ID_END		= 15,
+};
+
+enum sun6i_dsi_inst_mode {
+	DSI_INST_MODE_STOP	= 0,
+	DSI_INST_MODE_TBA,
+	DSI_INST_MODE_HS,
+	DSI_INST_MODE_ESCAPE,
+	DSI_INST_MODE_HSCEXIT,
+	DSI_INST_MODE_NOP,
+};
+
+enum sun6i_dsi_inst_escape {
+	DSI_INST_ESCA_LPDT	= 0,
+	DSI_INST_ESCA_ULPS,
+	DSI_INST_ESCA_UN1,
+	DSI_INST_ESCA_UN2,
+	DSI_INST_ESCA_RESET,
+	DSI_INST_ESCA_UN3,
+	DSI_INST_ESCA_UN4,
+	DSI_INST_ESCA_UN5,
+};
+
+enum sun6i_dsi_inst_packet {
+	DSI_INST_PACK_PIXEL	= 0,
+	DSI_INST_PACK_COMMAND,
+};
+
+static const u32 sun6i_dsi_ecc_array[] = {
+	[0] = (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | BIT(7) | BIT(10) |
+	       BIT(11) | BIT(13) | BIT(16) | BIT(20) | BIT(21) | BIT(22) |
+	       BIT(23)),
+	[1] = (BIT(0) | BIT(1) | BIT(3) | BIT(4) | BIT(6) | BIT(8) | BIT(10) |
+	       BIT(12) | BIT(14) | BIT(17) | BIT(20) | BIT(21) | BIT(22) |
+	       BIT(23)),
+	[2] = (BIT(0) | BIT(2) | BIT(3) | BIT(5) | BIT(6) | BIT(9) | BIT(11) |
+	       BIT(12) | BIT(15) | BIT(18) | BIT(20) | BIT(21) | BIT(22)),
+	[3] = (BIT(1) | BIT(2) | BIT(3) | BIT(7) | BIT(8) | BIT(9) | BIT(13) |
+	       BIT(14) | BIT(15) | BIT(19) | BIT(20) | BIT(21) | BIT(23)),
+	[4] = (BIT(4) | BIT(5) | BIT(6) | BIT(7) | BIT(8) | BIT(9) | BIT(16) |
+	       BIT(17) | BIT(18) | BIT(19) | BIT(20) | BIT(22) | BIT(23)),
+	[5] = (BIT(10) | BIT(11) | BIT(12) | BIT(13) | BIT(14) | BIT(15) |
+	       BIT(16) | BIT(17) | BIT(18) | BIT(19) | BIT(21) | BIT(22) |
+	       BIT(23)),
+};
+
+static u32 sun6i_dsi_ecc_compute(unsigned int data)
+{
+	int i;
+	u8 ecc = 0;
+
+	for (i = 0; i < ARRAY_SIZE(sun6i_dsi_ecc_array); i++) {
+		u32 field = sun6i_dsi_ecc_array[i];
+		bool init = false;
+		u8 val = 0;
+		int j;
+
+		for (j = 0; j < 24; j++) {
+			if (!(BIT(j) & field))
+				continue;
+
+			if (!init) {
+				val = (BIT(j) & data) ? 1 : 0;
+				init = true;
+			} else {
+				val ^= (BIT(j) & data) ? 1 : 0;
+			}
+		}
+
+		ecc |= val << i;
+	}
+
+	return ecc;
+}
+
+static u16 sun6i_dsi_crc_compute(u8 const *buffer, size_t len)
+{
+	return crc_ccitt(0xffff, buffer, len);
+}
+
+static u16 sun6i_dsi_crc_repeat_compute(u8 pd, size_t len)
+{
+	u8 buffer[len];
+
+	memset(buffer, pd, len);
+
+	return sun6i_dsi_crc_compute(buffer, len);
+}
+
+static u32 sun6i_dsi_build_sync_pkt(u8 dt, u8 vc, u8 d0, u8 d1)
+{
+	u32 val = dt & 0x3f;
+
+	val |= (vc & 3) << 6;
+	val |= (d0 & 0xff) << 8;
+	val |= (d1 & 0xff) << 16;
+	val |= sun6i_dsi_ecc_compute(val) << 24;
+
+	return val;
+}
+
+static u32 sun6i_dsi_build_blk0_pkt(u8 vc, u16 wc)
+{
+	return sun6i_dsi_build_sync_pkt(MIPI_DSI_BLANKING_PACKET, vc,
+					wc & 0xff, wc >> 8);
+}
+
+static u32 sun6i_dsi_build_blk1_pkt(u16 pd, size_t len)
+{
+	u32 val = SUN6I_DSI_BLK_PD(pd);
+
+	return val | SUN6I_DSI_BLK_PF(sun6i_dsi_crc_repeat_compute(pd, len));
+}
+
+static void sun6i_dsi_inst_abort(struct sun6i_dsi *dsi)
+{
+	regmap_update_bits(dsi->regs, SUN6I_DSI_BASIC_CTL0_REG,
+			   SUN6I_DSI_BASIC_CTL0_INST_ST, 0);
+}
+
+static void sun6i_dsi_inst_commit(struct sun6i_dsi *dsi)
+{
+	regmap_update_bits(dsi->regs, SUN6I_DSI_BASIC_CTL0_REG,
+			   SUN6I_DSI_BASIC_CTL0_INST_ST,
+			   SUN6I_DSI_BASIC_CTL0_INST_ST);
+}
+
+static int sun6i_dsi_inst_wait_for_completion(struct sun6i_dsi *dsi)
+{
+	u32 val;
+
+	return regmap_read_poll_timeout(dsi->regs, SUN6I_DSI_BASIC_CTL0_REG,
+					val,
+					!(val & SUN6I_DSI_BASIC_CTL0_INST_ST),
+					100, 5000);
+}
+
+static void sun6i_dsi_inst_setup(struct sun6i_dsi *dsi,
+				 enum sun6i_dsi_inst_id id,
+				 enum sun6i_dsi_inst_mode mode,
+				 bool clock, u8 data,
+				 enum sun6i_dsi_inst_packet packet,
+				 enum sun6i_dsi_inst_escape escape)
+{
+	regmap_write(dsi->regs, SUN6I_DSI_INST_FUNC_REG(id),
+		     SUN6I_DSI_INST_FUNC_INST_MODE(mode) |
+		     SUN6I_DSI_INST_FUNC_ESCAPE_ENTRY(escape) |
+		     SUN6I_DSI_INST_FUNC_TRANS_PACKET(packet) |
+		     (clock ? SUN6I_DSI_INST_FUNC_LANE_CEN : 0) |
+		     SUN6I_DSI_INST_FUNC_LANE_DEN(data));
+}
+
+static void sun6i_dsi_inst_init(struct sun6i_dsi *dsi,
+				struct mipi_dsi_device *device)
+{
+	u8 lanes_mask = GENMASK(device->lanes - 1, 0);
+
+	sun6i_dsi_inst_setup(dsi, DSI_INST_ID_LP11, DSI_INST_MODE_STOP,
+			     true, lanes_mask, 0, 0);
+
+	sun6i_dsi_inst_setup(dsi, DSI_INST_ID_TBA, DSI_INST_MODE_TBA,
+			     false, 1, 0, 0);
+
+	sun6i_dsi_inst_setup(dsi, DSI_INST_ID_HSC, DSI_INST_MODE_HS,
+			     true, 0, DSI_INST_PACK_PIXEL, 0);
+
+	sun6i_dsi_inst_setup(dsi, DSI_INST_ID_HSD, DSI_INST_MODE_HS,
+			     false, lanes_mask, DSI_INST_PACK_PIXEL, 0);
+
+	sun6i_dsi_inst_setup(dsi, DSI_INST_ID_LPDT, DSI_INST_MODE_ESCAPE,
+			     false, 1, DSI_INST_PACK_COMMAND,
+			     DSI_INST_ESCA_LPDT);
+
+	sun6i_dsi_inst_setup(dsi, DSI_INST_ID_HSCEXIT, DSI_INST_MODE_HSCEXIT,
+			     true, 0, 0, 0);
+
+	sun6i_dsi_inst_setup(dsi, DSI_INST_ID_NOP, DSI_INST_MODE_STOP,
+			     false, lanes_mask, 0, 0);
+
+	sun6i_dsi_inst_setup(dsi, DSI_INST_ID_DLY, DSI_INST_MODE_NOP,
+			     true, lanes_mask, 0, 0);
+
+	regmap_write(dsi->regs, SUN6I_DSI_INST_JUMP_CFG_REG(0),
+		     SUN6I_DSI_INST_JUMP_CFG_POINT(DSI_INST_ID_NOP) |
+		     SUN6I_DSI_INST_JUMP_CFG_TO(DSI_INST_ID_HSCEXIT) |
+		     SUN6I_DSI_INST_JUMP_CFG_NUM(1));
+};
+
+static u16 sun6i_dsi_get_video_start_delay(struct sun6i_dsi *dsi,
+					   struct drm_display_mode *mode)
+{
+	return mode->vtotal - (mode->vsync_end - mode->vdisplay) + 1;
+}
+
+static void sun6i_dsi_setup_burst(struct sun6i_dsi *dsi,
+				  struct drm_display_mode *mode)
+{
+	struct mipi_dsi_device *device = dsi->device;
+	u32 val = 0;
+
+	if ((mode->hsync_end - mode->hdisplay) > 20) {
+		/* Maaaaaagic */
+		u16 drq = (mode->hsync_end - mode->hdisplay) - 20;
+
+		drq *= mipi_dsi_pixel_format_to_bpp(device->format);
+		drq /= 32;
+
+		val = (SUN6I_DSI_TCON_DRQ_ENABLE_MODE |
+		       SUN6I_DSI_TCON_DRQ_SET(drq));
+	}
+
+	regmap_write(dsi->regs, SUN6I_DSI_TCON_DRQ_REG, val);
+}
+
+static void sun6i_dsi_setup_inst_loop(struct sun6i_dsi *dsi,
+				      struct drm_display_mode *mode)
+{
+	u16 delay = 50 - 1;
+
+	regmap_write(dsi->regs, SUN6I_DSI_INST_LOOP_NUM_REG(0),
+		     SUN6I_DSI_INST_LOOP_NUM_N0(50 - 1) |
+		     SUN6I_DSI_INST_LOOP_NUM_N1(delay));
+	regmap_write(dsi->regs, SUN6I_DSI_INST_LOOP_NUM_REG(1),
+		     SUN6I_DSI_INST_LOOP_NUM_N0(50 - 1) |
+		     SUN6I_DSI_INST_LOOP_NUM_N1(delay));
+}
+
+static void sun6i_dsi_setup_format(struct sun6i_dsi *dsi,
+				   struct drm_display_mode *mode)
+{
+	struct mipi_dsi_device *device = dsi->device;
+	u32 val = SUN6I_DSI_PIXEL_PH_VC(device->channel);
+	u8 dt, fmt;
+	u16 wc;
+
+	/*
+	 * TODO: The format defines are only valid in video mode and
+	 * change in command mode.
+	 */
+	switch (device->format) {
+	case MIPI_DSI_FMT_RGB888:
+		dt = MIPI_DSI_PACKED_PIXEL_STREAM_24;
+		fmt = 8;
+		break;
+	case MIPI_DSI_FMT_RGB666:
+		dt = MIPI_DSI_PIXEL_STREAM_3BYTE_18;
+		fmt = 9;
+		break;
+	case MIPI_DSI_FMT_RGB666_PACKED:
+		dt = MIPI_DSI_PACKED_PIXEL_STREAM_18;
+		fmt = 10;
+		break;
+	case MIPI_DSI_FMT_RGB565:
+		dt = MIPI_DSI_PACKED_PIXEL_STREAM_16;
+		fmt = 11;
+		break;
+	default:
+		return;
+	}
+	val |= SUN6I_DSI_PIXEL_PH_DT(dt);
+
+	wc = mode->hdisplay * mipi_dsi_pixel_format_to_bpp(device->format) / 8;
+	val |= SUN6I_DSI_PIXEL_PH_WC(wc);
+	val |= SUN6I_DSI_PIXEL_PH_ECC(sun6i_dsi_ecc_compute(val));
+
+	regmap_write(dsi->regs, SUN6I_DSI_PIXEL_PH_REG, val);
+
+	regmap_write(dsi->regs, SUN6I_DSI_PIXEL_PF0_REG,
+		     SUN6I_DSI_PIXEL_PF0_CRC_FORCE(0xffff));
+
+	regmap_write(dsi->regs, SUN6I_DSI_PIXEL_PF1_REG,
+		     SUN6I_DSI_PIXEL_PF1_CRC_INIT_LINE0(0xffff) |
+		     SUN6I_DSI_PIXEL_PF1_CRC_INIT_LINEN(0xffff));
+
+	regmap_write(dsi->regs, SUN6I_DSI_PIXEL_CTL0_REG,
+		     SUN6I_DSI_PIXEL_CTL0_PD_PLUG_DISABLE |
+		     SUN6I_DSI_PIXEL_CTL0_FORMAT(fmt));
+}
+
+static void sun6i_dsi_setup_timings(struct sun6i_dsi *dsi,
+				    struct drm_display_mode *mode)
+{
+	struct mipi_dsi_device *device = dsi->device;
+	unsigned int Bpp = mipi_dsi_pixel_format_to_bpp(device->format) / 8;
+	u16 hbp, hfp, hsa, hblk, vblk;
+
+	regmap_write(dsi->regs, SUN6I_DSI_BASIC_CTL_REG, 0);
+
+	regmap_write(dsi->regs, SUN6I_DSI_SYNC_HSS_REG,
+		     sun6i_dsi_build_sync_pkt(MIPI_DSI_H_SYNC_START,
+					      device->channel,
+					      0, 0));
+
+	regmap_write(dsi->regs, SUN6I_DSI_SYNC_HSE_REG,
+		     sun6i_dsi_build_sync_pkt(MIPI_DSI_H_SYNC_END,
+					      device->channel,
+					      0, 0));
+
+	regmap_write(dsi->regs, SUN6I_DSI_SYNC_VSS_REG,
+		     sun6i_dsi_build_sync_pkt(MIPI_DSI_V_SYNC_START,
+					      device->channel,
+					      0, 0));
+
+	regmap_write(dsi->regs, SUN6I_DSI_SYNC_VSE_REG,
+		     sun6i_dsi_build_sync_pkt(MIPI_DSI_V_SYNC_END,
+					      device->channel,
+					      0, 0));
+
+	regmap_write(dsi->regs, SUN6I_DSI_BASIC_SIZE0_REG,
+		     SUN6I_DSI_BASIC_SIZE0_VSA(mode->vsync_end -
+					       mode->vsync_start) |
+		     SUN6I_DSI_BASIC_SIZE0_VBP(mode->vsync_start -
+					       mode->vdisplay));
+
+	regmap_write(dsi->regs, SUN6I_DSI_BASIC_SIZE1_REG,
+		     SUN6I_DSI_BASIC_SIZE1_VACT(mode->vdisplay) |
+		     SUN6I_DSI_BASIC_SIZE1_VT(mode->vtotal));
+
+	/*
+	 * A sync period is composed of a blanking packet (4 bytes +
+	 * payload + 2 bytes) and a sync event packet (4 bytes). Its
+	 * minimal size is therefore 10 bytes
+	 */
+#define HSA_PACKET_OVERHEAD	10
+	hsa = max((unsigned int)HSA_PACKET_OVERHEAD,
+		  (mode->hsync_end - mode->hsync_start) * Bpp - HSA_PACKET_OVERHEAD);
+	regmap_write(dsi->regs, SUN6I_DSI_BLK_HSA0_REG,
+		     sun6i_dsi_build_blk0_pkt(device->channel, hsa));
+	regmap_write(dsi->regs, SUN6I_DSI_BLK_HSA1_REG,
+		     sun6i_dsi_build_blk1_pkt(0, hsa));
+
+	/*
+	 * The backporch is set using a blanking packet (4 bytes +
+	 * payload + 2 bytes). Its minimal size is therefore 6 bytes
+	 */
+#define HBP_PACKET_OVERHEAD	6
+	hbp = max((unsigned int)HBP_PACKET_OVERHEAD,
+		  (mode->hsync_start - mode->hdisplay) * Bpp - HBP_PACKET_OVERHEAD);
+	regmap_write(dsi->regs, SUN6I_DSI_BLK_HBP0_REG,
+		     sun6i_dsi_build_blk0_pkt(device->channel, hbp));
+	regmap_write(dsi->regs, SUN6I_DSI_BLK_HBP1_REG,
+		     sun6i_dsi_build_blk1_pkt(0, hbp));
+
+	/*
+	 * The frontporch is set using a blanking packet (4 bytes +
+	 * payload + 2 bytes). Its minimal size is therefore 6 bytes
+	 */
+#define HFP_PACKET_OVERHEAD	6
+	hfp = max((unsigned int)HFP_PACKET_OVERHEAD,
+		  (mode->htotal - mode->hsync_end) * Bpp - HFP_PACKET_OVERHEAD);
+	regmap_write(dsi->regs, SUN6I_DSI_BLK_HFP0_REG,
+		     sun6i_dsi_build_blk0_pkt(device->channel, hfp));
+	regmap_write(dsi->regs, SUN6I_DSI_BLK_HFP1_REG,
+		     sun6i_dsi_build_blk1_pkt(0, hfp));
+
+	/*
+	 * hblk seems to be the line + porches length.
+	 */
+	hblk = mode->htotal * Bpp - hsa;
+	regmap_write(dsi->regs, SUN6I_DSI_BLK_HBLK0_REG,
+		     sun6i_dsi_build_blk0_pkt(device->channel, hblk));
+	regmap_write(dsi->regs, SUN6I_DSI_BLK_HBLK1_REG,
+		     sun6i_dsi_build_blk1_pkt(0, hblk));
+
+	/*
+	 * And I'm not entirely sure what vblk is about. The driver in
+	 * Allwinner BSP is using a rather convoluted calculation
+	 * there only for 4 lanes. However, using 0 (the !4 lanes
+	 * case) even with a 4 lanes screen seems to work...
+	 */
+	vblk = 0;
+	regmap_write(dsi->regs, SUN6I_DSI_BLK_VBLK0_REG,
+		     sun6i_dsi_build_blk0_pkt(device->channel, vblk));
+	regmap_write(dsi->regs, SUN6I_DSI_BLK_VBLK1_REG,
+		     sun6i_dsi_build_blk1_pkt(0, vblk));
+}
+
+static int sun6i_dsi_start(struct sun6i_dsi *dsi,
+			   enum sun6i_dsi_start_inst func)
+{
+	switch (func) {
+	case DSI_START_LPTX:
+		regmap_write(dsi->regs, SUN6I_DSI_INST_JUMP_SEL_REG,
+			     DSI_INST_ID_LPDT << (4 * DSI_INST_ID_LP11) |
+			     DSI_INST_ID_END  << (4 * DSI_INST_ID_LPDT));
+		break;
+	case DSI_START_LPRX:
+		regmap_write(dsi->regs, SUN6I_DSI_INST_JUMP_SEL_REG,
+			     DSI_INST_ID_LPDT << (4 * DSI_INST_ID_LP11) |
+			     DSI_INST_ID_DLY  << (4 * DSI_INST_ID_LPDT) |
+			     DSI_INST_ID_TBA  << (4 * DSI_INST_ID_DLY) |
+			     DSI_INST_ID_END  << (4 * DSI_INST_ID_TBA));
+		break;
+	case DSI_START_HSC:
+		regmap_write(dsi->regs, SUN6I_DSI_INST_JUMP_SEL_REG,
+			     DSI_INST_ID_HSC  << (4 * DSI_INST_ID_LP11) |
+			     DSI_INST_ID_END  << (4 * DSI_INST_ID_HSC));
+		break;
+	case DSI_START_HSD:
+		regmap_write(dsi->regs, SUN6I_DSI_INST_JUMP_SEL_REG,
+			     DSI_INST_ID_NOP  << (4 * DSI_INST_ID_LP11) |
+			     DSI_INST_ID_HSD  << (4 * DSI_INST_ID_NOP) |
+			     DSI_INST_ID_DLY  << (4 * DSI_INST_ID_HSD) |
+			     DSI_INST_ID_NOP  << (4 * DSI_INST_ID_DLY) |
+			     DSI_INST_ID_END  << (4 * DSI_INST_ID_HSCEXIT));
+		break;
+	default:
+		regmap_write(dsi->regs, SUN6I_DSI_INST_JUMP_SEL_REG,
+			     DSI_INST_ID_END  << (4 * DSI_INST_ID_LP11));
+		break;
+	}
+
+	sun6i_dsi_inst_abort(dsi);
+	sun6i_dsi_inst_commit(dsi);
+
+	if (func == DSI_START_HSC)
+		regmap_write_bits(dsi->regs,
+				  SUN6I_DSI_INST_FUNC_REG(DSI_INST_ID_LP11),
+				  SUN6I_DSI_INST_FUNC_LANE_CEN, 0);
+
+	return 0;
+}
+
+static void sun6i_dsi_encoder_enable(struct drm_encoder *encoder)
+{
+	struct drm_display_mode *mode = &encoder->crtc->state->adjusted_mode;
+	struct sun6i_dsi *dsi = encoder_to_sun6i_dsi(encoder);
+	struct mipi_dsi_device *device = dsi->device;
+	u16 delay;
+
+	DRM_DEBUG_DRIVER("Enabling DSI output\n");
+
+	pm_runtime_get_sync(dsi->dev);
+
+	delay = sun6i_dsi_get_video_start_delay(dsi, mode);
+	regmap_write(dsi->regs, SUN6I_DSI_BASIC_CTL1_REG,
+		     SUN6I_DSI_BASIC_CTL1_VIDEO_ST_DELAY(delay) |
+		     SUN6I_DSI_BASIC_CTL1_VIDEO_FILL |
+		     SUN6I_DSI_BASIC_CTL1_VIDEO_PRECISION |
+		     SUN6I_DSI_BASIC_CTL1_VIDEO_MODE);
+
+	sun6i_dsi_setup_burst(dsi, mode);
+	sun6i_dsi_setup_inst_loop(dsi, mode);
+	sun6i_dsi_setup_format(dsi, mode);
+	sun6i_dsi_setup_timings(dsi, mode);
+
+	sun6i_dphy_init(dsi->dphy, device->lanes);
+	sun6i_dphy_power_on(dsi->dphy, device->lanes);
+
+	if (!IS_ERR(dsi->panel))
+		drm_panel_prepare(dsi->panel);
+
+	/*
+	 * FIXME: This should be moved after the switch to HS mode.
+	 *
+	 * Unfortunately, once in HS mode, it seems like we're not
+	 * able to send DCS commands anymore, which would prevent any
+	 * panel to send any DCS command as part as their enable
+	 * method, which is quite common.
+	 *
+	 * I haven't seen any artifact due to that sub-optimal
+	 * ordering on the panels I've tested it with, so I guess this
+	 * will do for now, until that IP is better understood.
+	 */
+	if (!IS_ERR(dsi->panel))
+		drm_panel_enable(dsi->panel);
+
+	sun6i_dsi_start(dsi, DSI_START_HSC);
+
+	udelay(1000);
+
+	sun6i_dsi_start(dsi, DSI_START_HSD);
+}
+
+static void sun6i_dsi_encoder_disable(struct drm_encoder *encoder)
+{
+	struct sun6i_dsi *dsi = encoder_to_sun6i_dsi(encoder);
+
+	DRM_DEBUG_DRIVER("Disabling DSI output\n");
+
+	if (!IS_ERR(dsi->panel)) {
+		drm_panel_disable(dsi->panel);
+		drm_panel_unprepare(dsi->panel);
+	}
+
+	sun6i_dphy_power_off(dsi->dphy);
+	sun6i_dphy_exit(dsi->dphy);
+
+	pm_runtime_put(dsi->dev);
+}
+
+static int sun6i_dsi_get_modes(struct drm_connector *connector)
+{
+	struct sun6i_dsi *dsi = connector_to_sun6i_dsi(connector);
+
+	return drm_panel_get_modes(dsi->panel);
+}
+
+static struct drm_connector_helper_funcs sun6i_dsi_connector_helper_funcs = {
+	.get_modes	= sun6i_dsi_get_modes,
+};
+
+static enum drm_connector_status
+sun6i_dsi_connector_detect(struct drm_connector *connector, bool force)
+{
+	return connector_status_connected;
+}
+
+static const struct drm_connector_funcs sun6i_dsi_connector_funcs = {
+	.detect			= sun6i_dsi_connector_detect,
+	.fill_modes		= drm_helper_probe_single_connector_modes,
+	.destroy		= drm_connector_cleanup,
+	.reset			= drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state	= drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
+};
+
+static const struct drm_encoder_helper_funcs sun6i_dsi_enc_helper_funcs = {
+	.disable	= sun6i_dsi_encoder_disable,
+	.enable		= sun6i_dsi_encoder_enable,
+};
+
+static const struct drm_encoder_funcs sun6i_dsi_enc_funcs = {
+	.destroy	= drm_encoder_cleanup,
+};
+
+static u32 sun6i_dsi_dcs_build_pkt_hdr(struct sun6i_dsi *dsi,
+				       const struct mipi_dsi_msg *msg)
+{
+	u32 pkt = msg->type;
+
+	if (msg->type == MIPI_DSI_DCS_LONG_WRITE) {
+		pkt |= ((msg->tx_len + 1) & 0xffff) << 8;
+		pkt |= (((msg->tx_len + 1) >> 8) & 0xffff) << 16;
+	} else {
+		pkt |= (((u8 *)msg->tx_buf)[0] << 8);
+		if (msg->tx_len > 1)
+			pkt |= (((u8 *)msg->tx_buf)[1] << 16);
+	}
+
+	pkt |= sun6i_dsi_ecc_compute(pkt) << 24;
+
+	return pkt;
+}
+
+static int sun6i_dsi_dcs_write_short(struct sun6i_dsi *dsi,
+				     const struct mipi_dsi_msg *msg)
+{
+	regmap_write(dsi->regs, SUN6I_DSI_CMD_TX_REG(0),
+		     sun6i_dsi_dcs_build_pkt_hdr(dsi, msg));
+	regmap_write_bits(dsi->regs, SUN6I_DSI_CMD_CTL_REG,
+			  0xff, (4 - 1));
+
+	sun6i_dsi_start(dsi, DSI_START_LPTX);
+
+	return msg->tx_len;
+}
+
+static int sun6i_dsi_dcs_write_long(struct sun6i_dsi *dsi,
+				    const struct mipi_dsi_msg *msg)
+{
+	int ret, len = 0;
+	u8 *bounce;
+	u16 crc;
+
+	regmap_write(dsi->regs, SUN6I_DSI_CMD_TX_REG(0),
+		     sun6i_dsi_dcs_build_pkt_hdr(dsi, msg));
+
+	bounce = kzalloc(msg->tx_len + sizeof(crc), GFP_KERNEL);
+	if (!bounce)
+		return -ENOMEM;
+
+	memcpy(bounce, msg->tx_buf, msg->tx_len);
+	len += msg->tx_len;
+
+	crc = sun6i_dsi_crc_compute(bounce, msg->tx_len);
+	memcpy((u8 *)bounce + msg->tx_len, &crc, sizeof(crc));
+	len += sizeof(crc);
+
+	regmap_bulk_write(dsi->regs, SUN6I_DSI_CMD_TX_REG(1), bounce, len);
+	regmap_write(dsi->regs, SUN6I_DSI_CMD_CTL_REG, len + 4 - 1);
+	kfree(bounce);
+
+	sun6i_dsi_start(dsi, DSI_START_LPTX);
+
+	ret = sun6i_dsi_inst_wait_for_completion(dsi);
+	if (ret < 0) {
+		sun6i_dsi_inst_abort(dsi);
+		return ret;
+	}
+
+	/*
+	 * TODO: There's some bits (reg 0x200, bits 8/9) that
+	 * apparently can be used to check whether the data have been
+	 * sent, but I couldn't get it to work reliably.
+	 */
+	return msg->tx_len;
+}
+
+static int sun6i_dsi_dcs_read(struct sun6i_dsi *dsi,
+			      const struct mipi_dsi_msg *msg)
+{
+	u32 val;
+	int ret;
+	u8 byte0;
+
+	regmap_write(dsi->regs, SUN6I_DSI_CMD_TX_REG(0),
+		     sun6i_dsi_dcs_build_pkt_hdr(dsi, msg));
+	regmap_write(dsi->regs, SUN6I_DSI_CMD_CTL_REG,
+		     (4 - 1));
+
+	sun6i_dsi_start(dsi, DSI_START_LPRX);
+
+	ret = sun6i_dsi_inst_wait_for_completion(dsi);
+	if (ret < 0) {
+		sun6i_dsi_inst_abort(dsi);
+		return ret;
+	}
+
+	/*
+	 * TODO: There's some bits (reg 0x200, bits 24/25) that
+	 * apparently can be used to check whether the data have been
+	 * received, but I couldn't get it to work reliably.
+	 */
+	regmap_read(dsi->regs, SUN6I_DSI_CMD_CTL_REG, &val);
+	if (val & SUN6I_DSI_CMD_CTL_RX_OVERFLOW)
+		return -EIO;
+
+	regmap_read(dsi->regs, SUN6I_DSI_CMD_RX_REG(0), &val);
+	byte0 = val & 0xff;
+	if (byte0 == MIPI_DSI_RX_ACKNOWLEDGE_AND_ERROR_REPORT)
+		return -EIO;
+
+	((u8 *)msg->rx_buf)[0] = (val >> 8);
+
+	return 1;
+}
+
+static int sun6i_dsi_attach(struct mipi_dsi_host *host,
+			    struct mipi_dsi_device *device)
+{
+	struct sun6i_dsi *dsi = host_to_sun6i_dsi(host);
+
+	dsi->device = device;
+	dsi->panel = of_drm_find_panel(device->dev.of_node);
+	if (!dsi->panel)
+		return -EINVAL;
+
+	dev_info(host->dev, "Attached device %s\n", device->name);
+
+	return 0;
+}
+
+static int sun6i_dsi_detach(struct mipi_dsi_host *host,
+			    struct mipi_dsi_device *device)
+{
+	struct sun6i_dsi *dsi = host_to_sun6i_dsi(host);
+
+	dsi->panel = NULL;
+	dsi->device = NULL;
+
+	return 0;
+}
+
+static ssize_t sun6i_dsi_transfer(struct mipi_dsi_host *host,
+				  const struct mipi_dsi_msg *msg)
+{
+	struct sun6i_dsi *dsi = host_to_sun6i_dsi(host);
+	int ret;
+
+	ret = sun6i_dsi_inst_wait_for_completion(dsi);
+	if (ret < 0)
+		sun6i_dsi_inst_abort(dsi);
+
+	regmap_write(dsi->regs, SUN6I_DSI_CMD_CTL_REG,
+		     SUN6I_DSI_CMD_CTL_RX_OVERFLOW |
+		     SUN6I_DSI_CMD_CTL_RX_FLAG |
+		     SUN6I_DSI_CMD_CTL_TX_FLAG);
+
+	switch (msg->type) {
+	case MIPI_DSI_DCS_SHORT_WRITE:
+	case MIPI_DSI_DCS_SHORT_WRITE_PARAM:
+		ret = sun6i_dsi_dcs_write_short(dsi, msg);
+		break;
+
+	case MIPI_DSI_DCS_LONG_WRITE:
+		ret = sun6i_dsi_dcs_write_long(dsi, msg);
+		break;
+
+	case MIPI_DSI_DCS_READ:
+		if (msg->rx_len == 1) {
+			ret = sun6i_dsi_dcs_read(dsi, msg);
+			break;
+		}
+
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static const struct mipi_dsi_host_ops sun6i_dsi_host_ops = {
+	.attach		= sun6i_dsi_attach,
+	.detach		= sun6i_dsi_detach,
+	.transfer	= sun6i_dsi_transfer,
+};
+
+static const struct regmap_config sun6i_dsi_regmap_config = {
+	.reg_bits	= 32,
+	.val_bits	= 32,
+	.reg_stride	= 4,
+	.max_register	= SUN6I_DSI_CMD_TX_REG(255),
+	.name		= "mipi-dsi",
+};
+
+static int sun6i_dsi_bind(struct device *dev, struct device *master,
+			 void *data)
+{
+	struct drm_device *drm = data;
+	struct sun4i_drv *drv = drm->dev_private;
+	struct sun6i_dsi *dsi = dev_get_drvdata(dev);
+	int ret;
+
+	if (!dsi->panel)
+		return -EPROBE_DEFER;
+
+	dsi->drv = drv;
+
+	drm_encoder_helper_add(&dsi->encoder,
+			       &sun6i_dsi_enc_helper_funcs);
+	ret = drm_encoder_init(drm,
+			       &dsi->encoder,
+			       &sun6i_dsi_enc_funcs,
+			       DRM_MODE_ENCODER_DSI,
+			       NULL);
+	if (ret) {
+		dev_err(dsi->dev, "Couldn't initialise the DSI encoder\n");
+		return ret;
+	}
+	dsi->encoder.possible_crtcs = BIT(0);
+
+	drm_connector_helper_add(&dsi->connector,
+				 &sun6i_dsi_connector_helper_funcs);
+	ret = drm_connector_init(drm, &dsi->connector,
+				 &sun6i_dsi_connector_funcs,
+				 DRM_MODE_CONNECTOR_DSI);
+	if (ret) {
+		dev_err(dsi->dev,
+			"Couldn't initialise the DSI connector\n");
+		goto err_cleanup_connector;
+	}
+
+	drm_mode_connector_attach_encoder(&dsi->connector, &dsi->encoder);
+	drm_panel_attach(dsi->panel, &dsi->connector);
+
+	return 0;
+
+err_cleanup_connector:
+	drm_encoder_cleanup(&dsi->encoder);
+	return ret;
+}
+
+static void sun6i_dsi_unbind(struct device *dev, struct device *master,
+			    void *data)
+{
+	struct sun6i_dsi *dsi = dev_get_drvdata(dev);
+
+	drm_panel_detach(dsi->panel);
+}
+
+static const struct component_ops sun6i_dsi_ops = {
+	.bind	= sun6i_dsi_bind,
+	.unbind	= sun6i_dsi_unbind,
+};
+
+static int sun6i_dsi_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *dphy_node;
+	struct sun6i_dsi *dsi;
+	struct resource *res;
+	void __iomem *base;
+	int ret;
+
+	dsi = devm_kzalloc(dev, sizeof(*dsi), GFP_KERNEL);
+	if (!dsi)
+		return -ENOMEM;
+	dev_set_drvdata(dev, dsi);
+	dsi->dev = dev;
+	dsi->host.ops = &sun6i_dsi_host_ops;
+	dsi->host.dev = dev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base)) {
+		dev_err(dev, "Couldn't map the DSI encoder registers\n");
+		return PTR_ERR(base);
+	}
+
+	dsi->regs = devm_regmap_init_mmio_clk(dev, "bus", base,
+					      &sun6i_dsi_regmap_config);
+	if (IS_ERR(dsi->regs)) {
+		dev_err(dev, "Couldn't create the DSI encoder regmap\n");
+		return PTR_ERR(dsi->regs);
+	}
+
+	dsi->reset = devm_reset_control_get_shared(dev, NULL);
+	if (IS_ERR(dsi->reset)) {
+		dev_err(dev, "Couldn't get our reset line\n");
+		return PTR_ERR(dsi->reset);
+	}
+
+	dsi->mod_clk = devm_clk_get(dev, "mod");
+	if (IS_ERR(dsi->mod_clk)) {
+		dev_err(dev, "Couldn't get the DSI mod clock\n");
+		return PTR_ERR(dsi->mod_clk);
+	}
+
+	/*
+	 * In order to operate properly, that clock seems to be always
+	 * set to 297MHz.
+	 */
+	clk_set_rate_exclusive(dsi->mod_clk, 297000000);
+
+	dphy_node = of_parse_phandle(dev->of_node, "phys", 0);
+	ret = sun6i_dphy_probe(dsi, dphy_node);
+	of_node_put(dphy_node);
+	if (ret) {
+		dev_err(dev, "Couldn't get the MIPI D-PHY\n");
+		goto err_unprotect_clk;
+	}
+
+	pm_runtime_enable(dev);
+
+	ret = mipi_dsi_host_register(&dsi->host);
+	if (ret) {
+		dev_err(dev, "Couldn't register MIPI-DSI host\n");
+		goto err_remove_phy;
+	}
+
+	ret = component_add(&pdev->dev, &sun6i_dsi_ops);
+	if (ret) {
+		dev_err(dev, "Couldn't register our component\n");
+		goto err_remove_dsi_host;
+	}
+
+	return 0;
+
+err_remove_dsi_host:
+	mipi_dsi_host_unregister(&dsi->host);
+err_remove_phy:
+	pm_runtime_disable(dev);
+	sun6i_dphy_remove(dsi);
+err_unprotect_clk:
+	clk_rate_exclusive_put(dsi->mod_clk);
+	return ret;
+}
+
+static int sun6i_dsi_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct sun6i_dsi *dsi = dev_get_drvdata(dev);
+
+	component_del(&pdev->dev, &sun6i_dsi_ops);
+	mipi_dsi_host_unregister(&dsi->host);
+	pm_runtime_disable(dev);
+	sun6i_dphy_remove(dsi);
+	clk_rate_exclusive_put(dsi->mod_clk);
+
+	return 0;
+}
+
+static int sun6i_dsi_runtime_resume(struct device *dev)
+{
+	struct sun6i_dsi *dsi = dev_get_drvdata(dev);
+
+	reset_control_deassert(dsi->reset);
+	clk_prepare_enable(dsi->mod_clk);
+
+	/*
+	 * Enable the DSI block.
+	 *
+	 * Some part of it can only be done once we get a number of
+	 * lanes, see sun6i_dsi_inst_init
+	 */
+	regmap_write(dsi->regs, SUN6I_DSI_CTL_REG, SUN6I_DSI_CTL_EN);
+
+	regmap_write(dsi->regs, SUN6I_DSI_BASIC_CTL0_REG,
+		     SUN6I_DSI_BASIC_CTL0_ECC_EN | SUN6I_DSI_BASIC_CTL0_CRC_EN);
+
+	regmap_write(dsi->regs, SUN6I_DSI_TRANS_START_REG, 10);
+	regmap_write(dsi->regs, SUN6I_DSI_TRANS_ZERO_REG, 0);
+
+	if (dsi->device)
+		sun6i_dsi_inst_init(dsi, dsi->device);
+
+	regmap_write(dsi->regs, SUN6I_DSI_DEBUG_DATA_REG, 0xff);
+
+	return 0;
+}
+
+static int sun6i_dsi_runtime_suspend(struct device *dev)
+{
+	struct sun6i_dsi *dsi = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(dsi->mod_clk);
+	reset_control_assert(dsi->reset);
+
+	return 0;
+}
+
+static const struct dev_pm_ops sun6i_dsi_pm_ops = {
+	SET_RUNTIME_PM_OPS(sun6i_dsi_runtime_suspend,
+			   sun6i_dsi_runtime_resume,
+			   NULL)
+};
+
+static const struct of_device_id sun6i_dsi_of_table[] = {
+	{ .compatible = "allwinner,sun6i-a31-mipi-dsi" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sun6i_dsi_of_table);
+
+static struct platform_driver sun6i_dsi_platform_driver = {
+	.probe		= sun6i_dsi_probe,
+	.remove		= sun6i_dsi_remove,
+	.driver		= {
+		.name		= "sun6i-mipi-dsi",
+		.of_match_table	= sun6i_dsi_of_table,
+		.pm		= &sun6i_dsi_pm_ops,
+	},
+};
+module_platform_driver(sun6i_dsi_platform_driver);
+
+MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
+MODULE_DESCRIPTION("Allwinner A31 DSI Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.h b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.h
new file mode 100644
index 000000000000..dbbc5b3ecbda
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.h
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2016 Allwinnertech Co., Ltd.
+ * Copyright (C) 2017-2018 Bootlin
+ *
+ * Maxime Ripard <maxime.ripard@bootlin.com>
+ */
+
+#ifndef _SUN6I_MIPI_DSI_H_
+#define _SUN6I_MIPI_DSI_H_
+
+#include <drm/drm_connector.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_mipi_dsi.h>
+
+struct sun6i_dphy {
+	struct clk		*bus_clk;
+	struct clk		*mod_clk;
+	struct regmap		*regs;
+	struct reset_control	*reset;
+};
+
+struct sun6i_dsi {
+	struct drm_connector	connector;
+	struct drm_encoder	encoder;
+	struct mipi_dsi_host	host;
+
+	struct clk		*bus_clk;
+	struct clk		*mod_clk;
+	struct regmap		*regs;
+	struct reset_control	*reset;
+	struct sun6i_dphy	*dphy;
+
+	struct device		*dev;
+	struct sun4i_drv	*drv;
+	struct mipi_dsi_device	*device;
+	struct drm_panel	*panel;
+};
+
+static inline struct sun6i_dsi *host_to_sun6i_dsi(struct mipi_dsi_host *host)
+{
+	return container_of(host, struct sun6i_dsi, host);
+};
+
+static inline struct sun6i_dsi *connector_to_sun6i_dsi(struct drm_connector *connector)
+{
+	return container_of(connector, struct sun6i_dsi, connector);
+};
+
+static inline struct sun6i_dsi *encoder_to_sun6i_dsi(const struct drm_encoder *encoder)
+{
+	return container_of(encoder, struct sun6i_dsi, encoder);
+};
+
+int sun6i_dphy_probe(struct sun6i_dsi *dsi, struct device_node *node);
+int sun6i_dphy_remove(struct sun6i_dsi *dsi);
+
+int sun6i_dphy_init(struct sun6i_dphy *dphy, unsigned int lanes);
+int sun6i_dphy_power_on(struct sun6i_dphy *dphy, unsigned int lanes);
+int sun6i_dphy_power_off(struct sun6i_dphy *dphy);
+int sun6i_dphy_exit(struct sun6i_dphy *dphy);
+
+#endif /* _SUN6I_MIPI_DSI_H_ */
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index 9f83a65b5ea9..c3afe7b2237e 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -163,28 +163,89 @@ static void tegra_plane_setup_blending_legacy(struct tegra_plane *plane)
 			 BLEND_COLOR_KEY_NONE;
 	u32 blendnokey = BLEND_WEIGHT1(255) | BLEND_WEIGHT0(255);
 	struct tegra_plane_state *state;
+	u32 blending[2];
 	unsigned int i;
 
+	/* disable blending for non-overlapping case */
+	tegra_plane_writel(plane, blendnokey, DC_WIN_BLEND_NOKEY);
+	tegra_plane_writel(plane, foreground, DC_WIN_BLEND_1WIN);
+
 	state = to_tegra_plane_state(plane->base.state);
 
-	/* alpha contribution is 1 minus sum of overlapping windows */
-	for (i = 0; i < 3; i++) {
-		if (state->dependent[i])
-			background[i] |= BLEND_CONTROL_DEPENDENT;
-	}
+	if (state->opaque) {
+		/*
+		 * Since custom fix-weight blending isn't utilized and weight
+		 * of top window is set to max, we can enforce dependent
+		 * blending which in this case results in transparent bottom
+		 * window if top window is opaque and if top window enables
+		 * alpha blending, then bottom window is getting alpha value
+		 * of 1 minus the sum of alpha components of the overlapping
+		 * plane.
+		 */
+		background[0] |= BLEND_CONTROL_DEPENDENT;
+		background[1] |= BLEND_CONTROL_DEPENDENT;
 
-	/* enable alpha blending if pixel format has an alpha component */
-	if (!state->opaque)
+		/*
+		 * The region where three windows overlap is the intersection
+		 * of the two regions where two windows overlap. It contributes
+		 * to the area if all of the windows on top of it have an alpha
+		 * component.
+		 */
+		switch (state->base.normalized_zpos) {
+		case 0:
+			if (state->blending[0].alpha &&
+			    state->blending[1].alpha)
+				background[2] |= BLEND_CONTROL_DEPENDENT;
+			break;
+
+		case 1:
+			background[2] |= BLEND_CONTROL_DEPENDENT;
+			break;
+		}
+	} else {
+		/*
+		 * Enable alpha blending if pixel format has an alpha
+		 * component.
+		 */
 		foreground |= BLEND_CONTROL_ALPHA;
 
-	/*
-	 * Disable blending and assume Window A is the bottom-most window,
-	 * Window C is the top-most window and Window B is in the middle.
-	 */
-	tegra_plane_writel(plane, blendnokey, DC_WIN_BLEND_NOKEY);
-	tegra_plane_writel(plane, foreground, DC_WIN_BLEND_1WIN);
+		/*
+		 * If any of the windows on top of this window is opaque, it
+		 * will completely conceal this window within that area. If
+		 * top window has an alpha component, it is blended over the
+		 * bottom window.
+		 */
+		for (i = 0; i < 2; i++) {
+			if (state->blending[i].alpha &&
+			    state->blending[i].top)
+				background[i] |= BLEND_CONTROL_DEPENDENT;
+		}
+
+		switch (state->base.normalized_zpos) {
+		case 0:
+			if (state->blending[0].alpha &&
+			    state->blending[1].alpha)
+				background[2] |= BLEND_CONTROL_DEPENDENT;
+			break;
 
-	switch (plane->index) {
+		case 1:
+			/*
+			 * When both middle and topmost windows have an alpha,
+			 * these windows a mixed together and then the result
+			 * is blended over the bottom window.
+			 */
+			if (state->blending[0].alpha &&
+			    state->blending[0].top)
+				background[2] |= BLEND_CONTROL_ALPHA;
+
+			if (state->blending[1].alpha &&
+			    state->blending[1].top)
+				background[2] |= BLEND_CONTROL_ALPHA;
+			break;
+		}
+	}
+
+	switch (state->base.normalized_zpos) {
 	case 0:
 		tegra_plane_writel(plane, background[0], DC_WIN_BLEND_2WIN_X);
 		tegra_plane_writel(plane, background[1], DC_WIN_BLEND_2WIN_Y);
@@ -192,8 +253,21 @@ static void tegra_plane_setup_blending_legacy(struct tegra_plane *plane)
 		break;
 
 	case 1:
-		tegra_plane_writel(plane, foreground, DC_WIN_BLEND_2WIN_X);
-		tegra_plane_writel(plane, background[1], DC_WIN_BLEND_2WIN_Y);
+		/*
+		 * If window B / C is topmost, then X / Y registers are
+		 * matching the order of blending[...] state indices,
+		 * otherwise a swap is required.
+		 */
+		if (!state->blending[0].top && state->blending[1].top) {
+			blending[0] = foreground;
+			blending[1] = background[1];
+		} else {
+			blending[0] = background[0];
+			blending[1] = foreground;
+		}
+
+		tegra_plane_writel(plane, blending[0], DC_WIN_BLEND_2WIN_X);
+		tegra_plane_writel(plane, blending[1], DC_WIN_BLEND_2WIN_Y);
 		tegra_plane_writel(plane, background[2], DC_WIN_BLEND_3WIN_XY);
 		break;
 
@@ -224,6 +298,39 @@ static void tegra_plane_setup_blending(struct tegra_plane *plane,
 	tegra_plane_writel(plane, value, DC_WIN_BLEND_LAYER_CONTROL);
 }
 
+static bool
+tegra_plane_use_horizontal_filtering(struct tegra_plane *plane,
+				     const struct tegra_dc_window *window)
+{
+	struct tegra_dc *dc = plane->dc;
+
+	if (window->src.w == window->dst.w)
+		return false;
+
+	if (plane->index == 0 && dc->soc->has_win_a_without_filters)
+		return false;
+
+	return true;
+}
+
+static bool
+tegra_plane_use_vertical_filtering(struct tegra_plane *plane,
+				   const struct tegra_dc_window *window)
+{
+	struct tegra_dc *dc = plane->dc;
+
+	if (window->src.h == window->dst.h)
+		return false;
+
+	if (plane->index == 0 && dc->soc->has_win_a_without_filters)
+		return false;
+
+	if (plane->index == 2 && dc->soc->has_win_c_without_vert_filter)
+		return false;
+
+	return true;
+}
+
 static void tegra_dc_setup_window(struct tegra_plane *plane,
 				  const struct tegra_dc_window *window)
 {
@@ -361,12 +468,50 @@ static void tegra_dc_setup_window(struct tegra_plane *plane,
 	if (window->bottom_up)
 		value |= V_DIRECTION;
 
+	if (tegra_plane_use_horizontal_filtering(plane, window)) {
+		/*
+		 * Enable horizontal 6-tap filter and set filtering
+		 * coefficients to the default values defined in TRM.
+		 */
+		tegra_plane_writel(plane, 0x00008000, DC_WIN_H_FILTER_P(0));
+		tegra_plane_writel(plane, 0x3e087ce1, DC_WIN_H_FILTER_P(1));
+		tegra_plane_writel(plane, 0x3b117ac1, DC_WIN_H_FILTER_P(2));
+		tegra_plane_writel(plane, 0x591b73aa, DC_WIN_H_FILTER_P(3));
+		tegra_plane_writel(plane, 0x57256d9a, DC_WIN_H_FILTER_P(4));
+		tegra_plane_writel(plane, 0x552f668b, DC_WIN_H_FILTER_P(5));
+		tegra_plane_writel(plane, 0x73385e8b, DC_WIN_H_FILTER_P(6));
+		tegra_plane_writel(plane, 0x72435583, DC_WIN_H_FILTER_P(7));
+		tegra_plane_writel(plane, 0x714c4c8b, DC_WIN_H_FILTER_P(8));
+		tegra_plane_writel(plane, 0x70554393, DC_WIN_H_FILTER_P(9));
+		tegra_plane_writel(plane, 0x715e389b, DC_WIN_H_FILTER_P(10));
+		tegra_plane_writel(plane, 0x71662faa, DC_WIN_H_FILTER_P(11));
+		tegra_plane_writel(plane, 0x536d25ba, DC_WIN_H_FILTER_P(12));
+		tegra_plane_writel(plane, 0x55731bca, DC_WIN_H_FILTER_P(13));
+		tegra_plane_writel(plane, 0x387a11d9, DC_WIN_H_FILTER_P(14));
+		tegra_plane_writel(plane, 0x3c7c08f1, DC_WIN_H_FILTER_P(15));
+
+		value |= H_FILTER;
+	}
+
+	if (tegra_plane_use_vertical_filtering(plane, window)) {
+		unsigned int i, k;
+
+		/*
+		 * Enable vertical 2-tap filter and set filtering
+		 * coefficients to the default values defined in TRM.
+		 */
+		for (i = 0, k = 128; i < 16; i++, k -= 8)
+			tegra_plane_writel(plane, k, DC_WIN_V_FILTER_P(i));
+
+		value |= V_FILTER;
+	}
+
 	tegra_plane_writel(plane, value, DC_WIN_WIN_OPTIONS);
 
-	if (dc->soc->supports_blending)
-		tegra_plane_setup_blending(plane, window);
-	else
+	if (dc->soc->has_legacy_blending)
 		tegra_plane_setup_blending_legacy(plane);
+	else
+		tegra_plane_setup_blending(plane, window);
 }
 
 static const u32 tegra20_primary_formats[] = {
@@ -451,17 +596,18 @@ static int tegra_plane_atomic_check(struct drm_plane *plane,
 				    struct drm_plane_state *state)
 {
 	struct tegra_plane_state *plane_state = to_tegra_plane_state(state);
+	unsigned int rotation = DRM_MODE_ROTATE_0 | DRM_MODE_REFLECT_Y;
 	struct tegra_bo_tiling *tiling = &plane_state->tiling;
 	struct tegra_plane *tegra = to_tegra_plane(plane);
 	struct tegra_dc *dc = to_tegra_dc(state->crtc);
-	unsigned int format;
 	int err;
 
 	/* no need for further checks if the plane is being disabled */
 	if (!state->crtc)
 		return 0;
 
-	err = tegra_plane_format(state->fb->format->format, &format,
+	err = tegra_plane_format(state->fb->format->format,
+				 &plane_state->format,
 				 &plane_state->swap);
 	if (err < 0)
 		return err;
@@ -472,22 +618,12 @@ static int tegra_plane_atomic_check(struct drm_plane *plane,
 	 * the corresponding opaque formats. However, the opaque formats can
 	 * be emulated by disabling alpha blending for the plane.
 	 */
-	if (!dc->soc->supports_blending) {
-		if (!tegra_plane_format_has_alpha(format)) {
-			err = tegra_plane_format_get_alpha(format, &format);
-			if (err < 0)
-				return err;
-
-			plane_state->opaque = true;
-		} else {
-			plane_state->opaque = false;
-		}
-
-		tegra_plane_check_dependent(tegra, plane_state);
+	if (dc->soc->has_legacy_blending) {
+		err = tegra_plane_setup_legacy_state(tegra, plane_state);
+		if (err < 0)
+			return err;
 	}
 
-	plane_state->format = format;
-
 	err = tegra_fb_get_tiling(state->fb, tiling);
 	if (err < 0)
 		return err;
@@ -498,6 +634,13 @@ static int tegra_plane_atomic_check(struct drm_plane *plane,
 		return -EINVAL;
 	}
 
+	rotation = drm_rotation_simplify(state->rotation, rotation);
+
+	if (rotation & DRM_MODE_REFLECT_Y)
+		plane_state->bottom_up = true;
+	else
+		plane_state->bottom_up = false;
+
 	/*
 	 * Tegra doesn't support different strides for U and V planes so we
 	 * error out if the user tries to display a framebuffer with such a
@@ -558,7 +701,7 @@ static void tegra_plane_atomic_update(struct drm_plane *plane,
 	window.dst.w = drm_rect_width(&plane->state->dst);
 	window.dst.h = drm_rect_height(&plane->state->dst);
 	window.bits_per_pixel = fb->format->cpp[0] * 8;
-	window.bottom_up = tegra_fb_is_bottom_up(fb);
+	window.bottom_up = tegra_fb_is_bottom_up(fb) || state->bottom_up;
 
 	/* copy from state */
 	window.zpos = plane->state->normalized_zpos;
@@ -639,9 +782,15 @@ static struct drm_plane *tegra_primary_plane_create(struct drm_device *drm,
 	}
 
 	drm_plane_helper_add(&plane->base, &tegra_plane_helper_funcs);
+	drm_plane_create_zpos_property(&plane->base, plane->index, 0, 255);
 
-	if (dc->soc->supports_blending)
-		drm_plane_create_zpos_property(&plane->base, 0, 0, 255);
+	err = drm_plane_create_rotation_property(&plane->base,
+						 DRM_MODE_ROTATE_0,
+						 DRM_MODE_ROTATE_0 |
+						 DRM_MODE_REFLECT_Y);
+	if (err < 0)
+		dev_err(dc->dev, "failed to create rotation property: %d\n",
+			err);
 
 	return &plane->base;
 }
@@ -918,9 +1067,15 @@ static struct drm_plane *tegra_dc_overlay_plane_create(struct drm_device *drm,
 	}
 
 	drm_plane_helper_add(&plane->base, &tegra_plane_helper_funcs);
+	drm_plane_create_zpos_property(&plane->base, plane->index, 0, 255);
 
-	if (dc->soc->supports_blending)
-		drm_plane_create_zpos_property(&plane->base, 0, 0, 255);
+	err = drm_plane_create_rotation_property(&plane->base,
+						 DRM_MODE_ROTATE_0,
+						 DRM_MODE_ROTATE_0 |
+						 DRM_MODE_REFLECT_Y);
+	if (err < 0)
+		dev_err(dc->dev, "failed to create rotation property: %d\n",
+			err);
 
 	return &plane->base;
 }
@@ -1826,7 +1981,6 @@ static irqreturn_t tegra_dc_irq(int irq, void *data)
 static int tegra_dc_init(struct host1x_client *client)
 {
 	struct drm_device *drm = dev_get_drvdata(client->parent);
-	struct iommu_group *group = iommu_group_get(client->dev);
 	unsigned long flags = HOST1X_SYNCPT_CLIENT_MANAGED;
 	struct tegra_dc *dc = host1x_client_to_dc(client);
 	struct tegra_drm *tegra = drm->dev_private;
@@ -1838,20 +1992,11 @@ static int tegra_dc_init(struct host1x_client *client)
 	if (!dc->syncpt)
 		dev_warn(dc->dev, "failed to allocate syncpoint\n");
 
-	if (group && tegra->domain) {
-		if (group != tegra->group) {
-			err = iommu_attach_group(tegra->domain, group);
-			if (err < 0) {
-				dev_err(dc->dev,
-					"failed to attach to domain: %d\n",
-					err);
-				return err;
-			}
-
-			tegra->group = group;
-		}
-
-		dc->domain = tegra->domain;
+	dc->group = host1x_client_iommu_attach(client, true);
+	if (IS_ERR(dc->group)) {
+		err = PTR_ERR(dc->group);
+		dev_err(client->dev, "failed to attach to domain: %d\n", err);
+		return err;
 	}
 
 	if (dc->soc->wgrps)
@@ -1916,24 +2061,15 @@ static int tegra_dc_init(struct host1x_client *client)
 	if (!IS_ERR(primary))
 		drm_plane_cleanup(primary);
 
-	if (group && dc->domain) {
-		if (group == tegra->group) {
-			iommu_detach_group(dc->domain, group);
-			tegra->group = NULL;
-		}
-
-		dc->domain = NULL;
-	}
+	host1x_client_iommu_detach(client, dc->group);
+	host1x_syncpt_free(dc->syncpt);
 
 	return err;
 }
 
 static int tegra_dc_exit(struct host1x_client *client)
 {
-	struct drm_device *drm = dev_get_drvdata(client->parent);
-	struct iommu_group *group = iommu_group_get(client->dev);
 	struct tegra_dc *dc = host1x_client_to_dc(client);
-	struct tegra_drm *tegra = drm->dev_private;
 	int err;
 
 	devm_free_irq(dc->dev, dc->irq, dc);
@@ -1944,15 +2080,7 @@ static int tegra_dc_exit(struct host1x_client *client)
 		return err;
 	}
 
-	if (group && dc->domain) {
-		if (group == tegra->group) {
-			iommu_detach_group(dc->domain, group);
-			tegra->group = NULL;
-		}
-
-		dc->domain = NULL;
-	}
-
+	host1x_client_iommu_detach(client, dc->group);
 	host1x_syncpt_free(dc->syncpt);
 
 	return 0;
@@ -1968,7 +2096,7 @@ static const struct tegra_dc_soc_info tegra20_dc_soc_info = {
 	.supports_interlacing = false,
 	.supports_cursor = false,
 	.supports_block_linear = false,
-	.supports_blending = false,
+	.has_legacy_blending = true,
 	.pitch_align = 8,
 	.has_powergate = false,
 	.coupled_pm = true,
@@ -1978,6 +2106,8 @@ static const struct tegra_dc_soc_info tegra20_dc_soc_info = {
 	.num_overlay_formats = ARRAY_SIZE(tegra20_overlay_formats),
 	.overlay_formats = tegra20_overlay_formats,
 	.modifiers = tegra20_modifiers,
+	.has_win_a_without_filters = true,
+	.has_win_c_without_vert_filter = true,
 };
 
 static const struct tegra_dc_soc_info tegra30_dc_soc_info = {
@@ -1985,7 +2115,7 @@ static const struct tegra_dc_soc_info tegra30_dc_soc_info = {
 	.supports_interlacing = false,
 	.supports_cursor = false,
 	.supports_block_linear = false,
-	.supports_blending = false,
+	.has_legacy_blending = true,
 	.pitch_align = 8,
 	.has_powergate = false,
 	.coupled_pm = false,
@@ -1995,6 +2125,8 @@ static const struct tegra_dc_soc_info tegra30_dc_soc_info = {
 	.num_overlay_formats = ARRAY_SIZE(tegra20_overlay_formats),
 	.overlay_formats = tegra20_overlay_formats,
 	.modifiers = tegra20_modifiers,
+	.has_win_a_without_filters = false,
+	.has_win_c_without_vert_filter = false,
 };
 
 static const struct tegra_dc_soc_info tegra114_dc_soc_info = {
@@ -2002,7 +2134,7 @@ static const struct tegra_dc_soc_info tegra114_dc_soc_info = {
 	.supports_interlacing = false,
 	.supports_cursor = false,
 	.supports_block_linear = false,
-	.supports_blending = false,
+	.has_legacy_blending = true,
 	.pitch_align = 64,
 	.has_powergate = true,
 	.coupled_pm = false,
@@ -2012,6 +2144,8 @@ static const struct tegra_dc_soc_info tegra114_dc_soc_info = {
 	.num_overlay_formats = ARRAY_SIZE(tegra114_overlay_formats),
 	.overlay_formats = tegra114_overlay_formats,
 	.modifiers = tegra20_modifiers,
+	.has_win_a_without_filters = false,
+	.has_win_c_without_vert_filter = false,
 };
 
 static const struct tegra_dc_soc_info tegra124_dc_soc_info = {
@@ -2019,7 +2153,7 @@ static const struct tegra_dc_soc_info tegra124_dc_soc_info = {
 	.supports_interlacing = true,
 	.supports_cursor = true,
 	.supports_block_linear = true,
-	.supports_blending = true,
+	.has_legacy_blending = false,
 	.pitch_align = 64,
 	.has_powergate = true,
 	.coupled_pm = false,
@@ -2029,6 +2163,8 @@ static const struct tegra_dc_soc_info tegra124_dc_soc_info = {
 	.num_overlay_formats = ARRAY_SIZE(tegra124_overlay_formats),
 	.overlay_formats = tegra124_overlay_formats,
 	.modifiers = tegra124_modifiers,
+	.has_win_a_without_filters = false,
+	.has_win_c_without_vert_filter = false,
 };
 
 static const struct tegra_dc_soc_info tegra210_dc_soc_info = {
@@ -2036,7 +2172,7 @@ static const struct tegra_dc_soc_info tegra210_dc_soc_info = {
 	.supports_interlacing = true,
 	.supports_cursor = true,
 	.supports_block_linear = true,
-	.supports_blending = true,
+	.has_legacy_blending = false,
 	.pitch_align = 64,
 	.has_powergate = true,
 	.coupled_pm = false,
@@ -2046,6 +2182,8 @@ static const struct tegra_dc_soc_info tegra210_dc_soc_info = {
 	.num_overlay_formats = ARRAY_SIZE(tegra114_overlay_formats),
 	.overlay_formats = tegra114_overlay_formats,
 	.modifiers = tegra124_modifiers,
+	.has_win_a_without_filters = false,
+	.has_win_c_without_vert_filter = false,
 };
 
 static const struct tegra_windowgroup_soc tegra186_dc_wgrps[] = {
@@ -2087,7 +2225,7 @@ static const struct tegra_dc_soc_info tegra186_dc_soc_info = {
 	.supports_interlacing = true,
 	.supports_cursor = true,
 	.supports_block_linear = true,
-	.supports_blending = true,
+	.has_legacy_blending = false,
 	.pitch_align = 64,
 	.has_powergate = false,
 	.coupled_pm = false,
diff --git a/drivers/gpu/drm/tegra/dc.h b/drivers/gpu/drm/tegra/dc.h
index d2b50d32de4d..e96f582ca692 100644
--- a/drivers/gpu/drm/tegra/dc.h
+++ b/drivers/gpu/drm/tegra/dc.h
@@ -55,7 +55,7 @@ struct tegra_dc_soc_info {
 	bool supports_interlacing;
 	bool supports_cursor;
 	bool supports_block_linear;
-	bool supports_blending;
+	bool has_legacy_blending;
 	unsigned int pitch_align;
 	bool has_powergate;
 	bool coupled_pm;
@@ -67,6 +67,8 @@ struct tegra_dc_soc_info {
 	const u32 *overlay_formats;
 	unsigned int num_overlay_formats;
 	const u64 *modifiers;
+	bool has_win_a_without_filters;
+	bool has_win_c_without_vert_filter;
 };
 
 struct tegra_dc {
@@ -92,7 +94,7 @@ struct tegra_dc {
 
 	const struct tegra_dc_soc_info *soc;
 
-	struct iommu_domain *domain;
+	struct iommu_group *group;
 };
 
 static inline struct tegra_dc *
@@ -553,6 +555,9 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define  THREAD_NUM(x) (((x) & 0x1f) << 1)
 #define  THREAD_GROUP_ENABLE (1 << 0)
 
+#define DC_WIN_H_FILTER_P(p)			(0x601 + (p))
+#define DC_WIN_V_FILTER_P(p)			(0x619 + (p))
+
 #define DC_WIN_CSC_YOF				0x611
 #define DC_WIN_CSC_KYRGB			0x612
 #define DC_WIN_CSC_KUR				0x613
@@ -566,6 +571,8 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define H_DIRECTION  (1 <<  0)
 #define V_DIRECTION  (1 <<  2)
 #define COLOR_EXPAND (1 <<  6)
+#define H_FILTER     (1 <<  8)
+#define V_FILTER     (1 << 10)
 #define CSC_ENABLE   (1 << 18)
 #define WIN_ENABLE   (1 << 30)
 
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index 7afe2f635f74..776c1513e582 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -38,26 +38,11 @@ static int tegra_atomic_check(struct drm_device *drm,
 {
 	int err;
 
-	err = drm_atomic_helper_check_modeset(drm, state);
+	err = drm_atomic_helper_check(drm, state);
 	if (err < 0)
 		return err;
 
-	err = tegra_display_hub_atomic_check(drm, state);
-	if (err < 0)
-		return err;
-
-	err = drm_atomic_normalize_zpos(drm, state);
-	if (err < 0)
-		return err;
-
-	err = drm_atomic_helper_check_planes(drm, state);
-	if (err < 0)
-		return err;
-
-	if (state->legacy_cursor_update)
-		state->async_update = !drm_atomic_helper_async_check(drm, state);
-
-	return 0;
+	return tegra_display_hub_atomic_check(drm, state);
 }
 
 static const struct drm_mode_config_funcs tegra_drm_mode_config_funcs = {
@@ -113,6 +98,10 @@ static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
 			goto free;
 		}
 
+		err = iova_cache_get();
+		if (err < 0)
+			goto domain;
+
 		geometry = &tegra->domain->geometry;
 		gem_start = geometry->aperture_start;
 		gem_end = geometry->aperture_end - CARVEOUT_SZ;
@@ -151,6 +140,8 @@ static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
 
 	drm->mode_config.allow_fb_modifiers = true;
 
+	drm->mode_config.normalize_zpos = true;
+
 	drm->mode_config.funcs = &tegra_drm_mode_config_funcs;
 	drm->mode_config.helper_private = &tegra_drm_mode_config_helpers;
 
@@ -204,11 +195,14 @@ static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
 	drm_mode_config_cleanup(drm);
 
 	if (tegra->domain) {
-		iommu_domain_free(tegra->domain);
-		drm_mm_takedown(&tegra->mm);
 		mutex_destroy(&tegra->mm_lock);
+		drm_mm_takedown(&tegra->mm);
 		put_iova_domain(&tegra->carveout.domain);
+		iova_cache_put();
 	}
+domain:
+	if (tegra->domain)
+		iommu_domain_free(tegra->domain);
 free:
 	kfree(tegra);
 	return err;
@@ -230,10 +224,11 @@ static void tegra_drm_unload(struct drm_device *drm)
 		return;
 
 	if (tegra->domain) {
-		iommu_domain_free(tegra->domain);
-		drm_mm_takedown(&tegra->mm);
 		mutex_destroy(&tegra->mm_lock);
+		drm_mm_takedown(&tegra->mm);
 		put_iova_domain(&tegra->carveout.domain);
+		iova_cache_put();
+		iommu_domain_free(tegra->domain);
 	}
 
 	kfree(tegra);
@@ -313,46 +308,15 @@ static int host1x_reloc_copy_from_user(struct host1x_reloc *dest,
 	return 0;
 }
 
-static int host1x_waitchk_copy_from_user(struct host1x_waitchk *dest,
-					 struct drm_tegra_waitchk __user *src,
-					 struct drm_file *file)
-{
-	u32 cmdbuf;
-	int err;
-
-	err = get_user(cmdbuf, &src->handle);
-	if (err < 0)
-		return err;
-
-	err = get_user(dest->offset, &src->offset);
-	if (err < 0)
-		return err;
-
-	err = get_user(dest->syncpt_id, &src->syncpt);
-	if (err < 0)
-		return err;
-
-	err = get_user(dest->thresh, &src->thresh);
-	if (err < 0)
-		return err;
-
-	dest->bo = host1x_bo_lookup(file, cmdbuf);
-	if (!dest->bo)
-		return -ENOENT;
-
-	return 0;
-}
-
 int tegra_drm_submit(struct tegra_drm_context *context,
 		     struct drm_tegra_submit *args, struct drm_device *drm,
 		     struct drm_file *file)
 {
+	struct host1x_client *client = &context->client->base;
 	unsigned int num_cmdbufs = args->num_cmdbufs;
 	unsigned int num_relocs = args->num_relocs;
-	unsigned int num_waitchks = args->num_waitchks;
 	struct drm_tegra_cmdbuf __user *user_cmdbufs;
 	struct drm_tegra_reloc __user *user_relocs;
-	struct drm_tegra_waitchk __user *user_waitchks;
 	struct drm_tegra_syncpt __user *user_syncpt;
 	struct drm_tegra_syncpt syncpt;
 	struct host1x *host1x = dev_get_drvdata(drm->dev->parent);
@@ -364,7 +328,6 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 
 	user_cmdbufs = u64_to_user_ptr(args->cmdbufs);
 	user_relocs = u64_to_user_ptr(args->relocs);
-	user_waitchks = u64_to_user_ptr(args->waitchks);
 	user_syncpt = u64_to_user_ptr(args->syncpts);
 
 	/* We don't yet support other than one syncpt_incr struct per submit */
@@ -376,21 +339,20 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 		return -EINVAL;
 
 	job = host1x_job_alloc(context->channel, args->num_cmdbufs,
-			       args->num_relocs, args->num_waitchks);
+			       args->num_relocs);
 	if (!job)
 		return -ENOMEM;
 
 	job->num_relocs = args->num_relocs;
-	job->num_waitchk = args->num_waitchks;
-	job->client = (u32)args->context;
-	job->class = context->client->base.class;
+	job->client = client;
+	job->class = client->class;
 	job->serialize = true;
 
 	/*
 	 * Track referenced BOs so that they can be unreferenced after the
 	 * submission is complete.
 	 */
-	num_refs = num_cmdbufs + num_relocs * 2 + num_waitchks;
+	num_refs = num_cmdbufs + num_relocs * 2;
 
 	refs = kmalloc_array(num_refs, sizeof(*refs), GFP_KERNEL);
 	if (!refs) {
@@ -451,13 +413,13 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 		struct host1x_reloc *reloc;
 		struct tegra_bo *obj;
 
-		err = host1x_reloc_copy_from_user(&job->relocarray[num_relocs],
+		err = host1x_reloc_copy_from_user(&job->relocs[num_relocs],
 						  &user_relocs[num_relocs], drm,
 						  file);
 		if (err < 0)
 			goto fail;
 
-		reloc = &job->relocarray[num_relocs];
+		reloc = &job->relocs[num_relocs];
 		obj = host1x_to_tegra_bo(reloc->cmdbuf.bo);
 		refs[num_refs++] = &obj->gem;
 
@@ -481,30 +443,6 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 		}
 	}
 
-	/* copy and resolve waitchks from submit */
-	while (num_waitchks--) {
-		struct host1x_waitchk *wait = &job->waitchk[num_waitchks];
-		struct tegra_bo *obj;
-
-		err = host1x_waitchk_copy_from_user(
-			wait, &user_waitchks[num_waitchks], file);
-		if (err < 0)
-			goto fail;
-
-		obj = host1x_to_tegra_bo(wait->bo);
-		refs[num_refs++] = &obj->gem;
-
-		/*
-		 * The unaligned offset will cause an unaligned write during
-		 * of the waitchks patching, corrupting the commands stream.
-		 */
-		if (wait->offset & 3 ||
-		    wait->offset >= obj->gem.size) {
-			err = -EINVAL;
-			goto fail;
-		}
-	}
-
 	if (copy_from_user(&syncpt, user_syncpt, sizeof(syncpt))) {
 		err = -EFAULT;
 		goto fail;
@@ -1114,6 +1052,52 @@ int tegra_drm_unregister_client(struct tegra_drm *tegra,
 	return 0;
 }
 
+struct iommu_group *host1x_client_iommu_attach(struct host1x_client *client,
+					       bool shared)
+{
+	struct drm_device *drm = dev_get_drvdata(client->parent);
+	struct tegra_drm *tegra = drm->dev_private;
+	struct iommu_group *group = NULL;
+	int err;
+
+	if (tegra->domain) {
+		group = iommu_group_get(client->dev);
+		if (!group) {
+			dev_err(client->dev, "failed to get IOMMU group\n");
+			return ERR_PTR(-ENODEV);
+		}
+
+		if (!shared || (shared && (group != tegra->group))) {
+			err = iommu_attach_group(tegra->domain, group);
+			if (err < 0) {
+				iommu_group_put(group);
+				return ERR_PTR(err);
+			}
+
+			if (shared && !tegra->group)
+				tegra->group = group;
+		}
+	}
+
+	return group;
+}
+
+void host1x_client_iommu_detach(struct host1x_client *client,
+				struct iommu_group *group)
+{
+	struct drm_device *drm = dev_get_drvdata(client->parent);
+	struct tegra_drm *tegra = drm->dev_private;
+
+	if (group) {
+		if (group == tegra->group) {
+			iommu_detach_group(tegra->domain, group);
+			tegra->group = NULL;
+		}
+
+		iommu_group_put(group);
+	}
+}
+
 void *tegra_drm_alloc(struct tegra_drm *tegra, size_t size, dma_addr_t *dma)
 {
 	struct iova *alloc;
diff --git a/drivers/gpu/drm/tegra/drm.h b/drivers/gpu/drm/tegra/drm.h
index 4f41aaec8530..92d248784396 100644
--- a/drivers/gpu/drm/tegra/drm.h
+++ b/drivers/gpu/drm/tegra/drm.h
@@ -29,16 +29,10 @@
 
 struct reset_control;
 
-struct tegra_fb {
-	struct drm_framebuffer base;
-	struct tegra_bo **planes;
-	unsigned int num_planes;
-};
-
 #ifdef CONFIG_DRM_FBDEV_EMULATION
 struct tegra_fbdev {
 	struct drm_fb_helper base;
-	struct tegra_fb *fb;
+	struct drm_framebuffer *fb;
 };
 #endif
 
@@ -97,6 +91,7 @@ struct tegra_drm_client {
 	struct host1x_client base;
 	struct list_head list;
 
+	unsigned int version;
 	const struct tegra_drm_client_ops *ops;
 };
 
@@ -110,6 +105,10 @@ int tegra_drm_register_client(struct tegra_drm *tegra,
 			      struct tegra_drm_client *client);
 int tegra_drm_unregister_client(struct tegra_drm *tegra,
 				struct tegra_drm_client *client);
+struct iommu_group *host1x_client_iommu_attach(struct host1x_client *client,
+					       bool shared);
+void host1x_client_iommu_detach(struct host1x_client *client,
+				struct iommu_group *group);
 
 int tegra_drm_init(struct tegra_drm *tegra, struct drm_device *drm);
 int tegra_drm_exit(struct tegra_drm *tegra);
diff --git a/drivers/gpu/drm/tegra/fb.c b/drivers/gpu/drm/tegra/fb.c
index e69434909a42..4c22cdded3c2 100644
--- a/drivers/gpu/drm/tegra/fb.c
+++ b/drivers/gpu/drm/tegra/fb.c
@@ -14,11 +14,7 @@
 
 #include "drm.h"
 #include "gem.h"
-
-static inline struct tegra_fb *to_tegra_fb(struct drm_framebuffer *fb)
-{
-	return container_of(fb, struct tegra_fb, base);
-}
+#include <drm/drm_gem_framebuffer_helper.h>
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
 static inline struct tegra_fbdev *to_tegra_fbdev(struct drm_fb_helper *helper)
@@ -30,19 +26,14 @@ static inline struct tegra_fbdev *to_tegra_fbdev(struct drm_fb_helper *helper)
 struct tegra_bo *tegra_fb_get_plane(struct drm_framebuffer *framebuffer,
 				    unsigned int index)
 {
-	struct tegra_fb *fb = to_tegra_fb(framebuffer);
-
-	if (index >= framebuffer->format->num_planes)
-		return NULL;
-
-	return fb->planes[index];
+	return to_tegra_bo(drm_gem_fb_get_obj(framebuffer, index));
 }
 
 bool tegra_fb_is_bottom_up(struct drm_framebuffer *framebuffer)
 {
-	struct tegra_fb *fb = to_tegra_fb(framebuffer);
+	struct tegra_bo *bo = tegra_fb_get_plane(framebuffer, 0);
 
-	if (fb->planes[0]->flags & TEGRA_BO_BOTTOM_UP)
+	if (bo->flags & TEGRA_BO_BOTTOM_UP)
 		return true;
 
 	return false;
@@ -51,8 +42,7 @@ bool tegra_fb_is_bottom_up(struct drm_framebuffer *framebuffer)
 int tegra_fb_get_tiling(struct drm_framebuffer *framebuffer,
 			struct tegra_bo_tiling *tiling)
 {
-	struct tegra_fb *fb = to_tegra_fb(framebuffer);
-	uint64_t modifier = fb->base.modifier;
+	uint64_t modifier = framebuffer->modifier;
 
 	switch (modifier) {
 	case DRM_FORMAT_MOD_LINEAR:
@@ -102,46 +92,17 @@ int tegra_fb_get_tiling(struct drm_framebuffer *framebuffer,
 	return 0;
 }
 
-static void tegra_fb_destroy(struct drm_framebuffer *framebuffer)
-{
-	struct tegra_fb *fb = to_tegra_fb(framebuffer);
-	unsigned int i;
-
-	for (i = 0; i < fb->num_planes; i++) {
-		struct tegra_bo *bo = fb->planes[i];
-
-		if (bo) {
-			if (bo->pages)
-				vunmap(bo->vaddr);
-
-			drm_gem_object_put_unlocked(&bo->gem);
-		}
-	}
-
-	drm_framebuffer_cleanup(framebuffer);
-	kfree(fb->planes);
-	kfree(fb);
-}
-
-static int tegra_fb_create_handle(struct drm_framebuffer *framebuffer,
-				  struct drm_file *file, unsigned int *handle)
-{
-	struct tegra_fb *fb = to_tegra_fb(framebuffer);
-
-	return drm_gem_handle_create(file, &fb->planes[0]->gem, handle);
-}
-
 static const struct drm_framebuffer_funcs tegra_fb_funcs = {
-	.destroy = tegra_fb_destroy,
-	.create_handle = tegra_fb_create_handle,
+	.destroy = drm_gem_fb_destroy,
+	.create_handle = drm_gem_fb_create_handle,
 };
 
-static struct tegra_fb *tegra_fb_alloc(struct drm_device *drm,
-				       const struct drm_mode_fb_cmd2 *mode_cmd,
-				       struct tegra_bo **planes,
-				       unsigned int num_planes)
+static struct drm_framebuffer *tegra_fb_alloc(struct drm_device *drm,
+					      const struct drm_mode_fb_cmd2 *mode_cmd,
+					      struct tegra_bo **planes,
+					      unsigned int num_planes)
 {
-	struct tegra_fb *fb;
+	struct drm_framebuffer *fb;
 	unsigned int i;
 	int err;
 
@@ -149,24 +110,15 @@ static struct tegra_fb *tegra_fb_alloc(struct drm_device *drm,
 	if (!fb)
 		return ERR_PTR(-ENOMEM);
 
-	fb->planes = kzalloc(num_planes * sizeof(*planes), GFP_KERNEL);
-	if (!fb->planes) {
-		kfree(fb);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	fb->num_planes = num_planes;
+	drm_helper_mode_fill_fb_struct(drm, fb, mode_cmd);
 
-	drm_helper_mode_fill_fb_struct(drm, &fb->base, mode_cmd);
+	for (i = 0; i < fb->format->num_planes; i++)
+		fb->obj[i] = &planes[i]->gem;
 
-	for (i = 0; i < fb->num_planes; i++)
-		fb->planes[i] = planes[i];
-
-	err = drm_framebuffer_init(drm, &fb->base, &tegra_fb_funcs);
+	err = drm_framebuffer_init(drm, fb, &tegra_fb_funcs);
 	if (err < 0) {
 		dev_err(drm->dev, "failed to initialize framebuffer: %d\n",
 			err);
-		kfree(fb->planes);
 		kfree(fb);
 		return ERR_PTR(err);
 	}
@@ -181,7 +133,7 @@ struct drm_framebuffer *tegra_fb_create(struct drm_device *drm,
 	unsigned int hsub, vsub, i;
 	struct tegra_bo *planes[4];
 	struct drm_gem_object *gem;
-	struct tegra_fb *fb;
+	struct drm_framebuffer *fb;
 	int err;
 
 	hsub = drm_format_horz_chroma_subsampling(cmd->pixel_format);
@@ -217,7 +169,7 @@ struct drm_framebuffer *tegra_fb_create(struct drm_device *drm,
 		goto unreference;
 	}
 
-	return &fb->base;
+	return fb;
 
 unreference:
 	while (i--)
@@ -298,7 +250,7 @@ static int tegra_fbdev_probe(struct drm_fb_helper *helper,
 		return PTR_ERR(fbdev->fb);
 	}
 
-	fb = &fbdev->fb->base;
+	fb = fbdev->fb;
 	helper->fb = fb;
 	helper->fbdev = info;
 
@@ -398,8 +350,17 @@ static void tegra_fbdev_exit(struct tegra_fbdev *fbdev)
 {
 	drm_fb_helper_unregister_fbi(&fbdev->base);
 
-	if (fbdev->fb)
-		drm_framebuffer_remove(&fbdev->fb->base);
+	if (fbdev->fb) {
+		struct tegra_bo *bo = tegra_fb_get_plane(fbdev->fb, 0);
+
+		/* Undo the special mapping we made in fbdev probe. */
+		if (bo && bo->pages) {
+			vunmap(bo->vaddr);
+			bo->vaddr = 0;
+		}
+
+		drm_framebuffer_remove(fbdev->fb);
+	}
 
 	drm_fb_helper_fini(&fbdev->base);
 	tegra_fbdev_free(fbdev);
diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c
index 8b0b4ff64bb4..00a5c9f32254 100644
--- a/drivers/gpu/drm/tegra/gem.c
+++ b/drivers/gpu/drm/tegra/gem.c
@@ -422,14 +422,13 @@ int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm,
 	return 0;
 }
 
-static int tegra_bo_fault(struct vm_fault *vmf)
+static vm_fault_t tegra_bo_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct drm_gem_object *gem = vma->vm_private_data;
 	struct tegra_bo *bo = to_tegra_bo(gem);
 	struct page *page;
 	pgoff_t offset;
-	int err;
 
 	if (!bo->pages)
 		return VM_FAULT_SIGBUS;
@@ -437,20 +436,7 @@ static int tegra_bo_fault(struct vm_fault *vmf)
 	offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 	page = bo->pages[offset];
 
-	err = vm_insert_page(vma, vmf->address, page);
-	switch (err) {
-	case -EAGAIN:
-	case 0:
-	case -ERESTARTSYS:
-	case -EINTR:
-	case -EBUSY:
-		return VM_FAULT_NOPAGE;
-
-	case -ENOMEM:
-		return VM_FAULT_OOM;
-	}
-
-	return VM_FAULT_SIGBUS;
+	return vmf_insert_page(vma, vmf->address, page);
 }
 
 const struct vm_operations_struct tegra_bo_vm_ops = {
@@ -663,6 +649,8 @@ struct dma_buf *tegra_gem_prime_export(struct drm_device *drm,
 {
 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 
+	exp_info.exp_name = KBUILD_MODNAME;
+	exp_info.owner = drm->driver->fops->owner;
 	exp_info.ops = &tegra_gem_prime_dmabuf_ops;
 	exp_info.size = gem->size;
 	exp_info.flags = flags;
diff --git a/drivers/gpu/drm/tegra/gr2d.c b/drivers/gpu/drm/tegra/gr2d.c
index 9a8ea93016a9..673059fd2fcb 100644
--- a/drivers/gpu/drm/tegra/gr2d.c
+++ b/drivers/gpu/drm/tegra/gr2d.c
@@ -7,16 +7,25 @@
  */
 
 #include <linux/clk.h>
+#include <linux/iommu.h>
+#include <linux/of_device.h>
 
 #include "drm.h"
 #include "gem.h"
 #include "gr2d.h"
 
+struct gr2d_soc {
+	unsigned int version;
+};
+
 struct gr2d {
+	struct iommu_group *group;
 	struct tegra_drm_client client;
 	struct host1x_channel *channel;
 	struct clk *clk;
 
+	const struct gr2d_soc *soc;
+
 	DECLARE_BITMAP(addr_regs, GR2D_NUM_REGS);
 };
 
@@ -31,6 +40,7 @@ static int gr2d_init(struct host1x_client *client)
 	struct drm_device *dev = dev_get_drvdata(client->parent);
 	unsigned long flags = HOST1X_SYNCPT_HAS_BASE;
 	struct gr2d *gr2d = to_gr2d(drm);
+	int err;
 
 	gr2d->channel = host1x_channel_request(client->dev);
 	if (!gr2d->channel)
@@ -38,24 +48,48 @@ static int gr2d_init(struct host1x_client *client)
 
 	client->syncpts[0] = host1x_syncpt_request(client, flags);
 	if (!client->syncpts[0]) {
-		host1x_channel_put(gr2d->channel);
-		return -ENOMEM;
+		err = -ENOMEM;
+		dev_err(client->dev, "failed to request syncpoint: %d\n", err);
+		goto put;
+	}
+
+	gr2d->group = host1x_client_iommu_attach(client, false);
+	if (IS_ERR(gr2d->group)) {
+		err = PTR_ERR(gr2d->group);
+		dev_err(client->dev, "failed to attach to domain: %d\n", err);
+		goto free;
+	}
+
+	err = tegra_drm_register_client(dev->dev_private, drm);
+	if (err < 0) {
+		dev_err(client->dev, "failed to register client: %d\n", err);
+		goto detach;
 	}
 
-	return tegra_drm_register_client(dev->dev_private, drm);
+	return 0;
+
+detach:
+	host1x_client_iommu_detach(client, gr2d->group);
+free:
+	host1x_syncpt_free(client->syncpts[0]);
+put:
+	host1x_channel_put(gr2d->channel);
+	return err;
 }
 
 static int gr2d_exit(struct host1x_client *client)
 {
 	struct tegra_drm_client *drm = host1x_to_drm_client(client);
 	struct drm_device *dev = dev_get_drvdata(client->parent);
+	struct tegra_drm *tegra = dev->dev_private;
 	struct gr2d *gr2d = to_gr2d(drm);
 	int err;
 
-	err = tegra_drm_unregister_client(dev->dev_private, drm);
+	err = tegra_drm_unregister_client(tegra, drm);
 	if (err < 0)
 		return err;
 
+	host1x_client_iommu_detach(client, gr2d->group);
 	host1x_syncpt_free(client->syncpts[0]);
 	host1x_channel_put(gr2d->channel);
 
@@ -123,9 +157,17 @@ static const struct tegra_drm_client_ops gr2d_ops = {
 	.submit = tegra_drm_submit,
 };
 
+static const struct gr2d_soc tegra20_gr2d_soc = {
+	.version = 0x20,
+};
+
+static const struct gr2d_soc tegra30_gr2d_soc = {
+	.version = 0x30,
+};
+
 static const struct of_device_id gr2d_match[] = {
-	{ .compatible = "nvidia,tegra30-gr2d" },
-	{ .compatible = "nvidia,tegra20-gr2d" },
+	{ .compatible = "nvidia,tegra30-gr2d", .data = &tegra20_gr2d_soc },
+	{ .compatible = "nvidia,tegra20-gr2d", .data = &tegra30_gr2d_soc },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, gr2d_match);
@@ -158,6 +200,8 @@ static int gr2d_probe(struct platform_device *pdev)
 	if (!gr2d)
 		return -ENOMEM;
 
+	gr2d->soc = of_device_get_match_data(dev);
+
 	syncpts = devm_kzalloc(dev, sizeof(*syncpts), GFP_KERNEL);
 	if (!syncpts)
 		return -ENOMEM;
@@ -182,6 +226,7 @@ static int gr2d_probe(struct platform_device *pdev)
 	gr2d->client.base.num_syncpts = 1;
 
 	INIT_LIST_HEAD(&gr2d->client.list);
+	gr2d->client.version = gr2d->soc->version;
 	gr2d->client.ops = &gr2d_ops;
 
 	err = host1x_client_register(&gr2d->client.base);
diff --git a/drivers/gpu/drm/tegra/gr3d.c b/drivers/gpu/drm/tegra/gr3d.c
index 28c4ef63065b..4778ae999668 100644
--- a/drivers/gpu/drm/tegra/gr3d.c
+++ b/drivers/gpu/drm/tegra/gr3d.c
@@ -9,7 +9,9 @@
 
 #include <linux/clk.h>
 #include <linux/host1x.h>
+#include <linux/iommu.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/reset.h>
 
@@ -19,7 +21,12 @@
 #include "gem.h"
 #include "gr3d.h"
 
+struct gr3d_soc {
+	unsigned int version;
+};
+
 struct gr3d {
+	struct iommu_group *group;
 	struct tegra_drm_client client;
 	struct host1x_channel *channel;
 	struct clk *clk_secondary;
@@ -27,6 +34,8 @@ struct gr3d {
 	struct reset_control *rst_secondary;
 	struct reset_control *rst;
 
+	const struct gr3d_soc *soc;
+
 	DECLARE_BITMAP(addr_regs, GR3D_NUM_REGS);
 };
 
@@ -41,6 +50,7 @@ static int gr3d_init(struct host1x_client *client)
 	struct drm_device *dev = dev_get_drvdata(client->parent);
 	unsigned long flags = HOST1X_SYNCPT_HAS_BASE;
 	struct gr3d *gr3d = to_gr3d(drm);
+	int err;
 
 	gr3d->channel = host1x_channel_request(client->dev);
 	if (!gr3d->channel)
@@ -48,11 +58,33 @@ static int gr3d_init(struct host1x_client *client)
 
 	client->syncpts[0] = host1x_syncpt_request(client, flags);
 	if (!client->syncpts[0]) {
-		host1x_channel_put(gr3d->channel);
-		return -ENOMEM;
+		err = -ENOMEM;
+		dev_err(client->dev, "failed to request syncpoint: %d\n", err);
+		goto put;
 	}
 
-	return tegra_drm_register_client(dev->dev_private, drm);
+	gr3d->group = host1x_client_iommu_attach(client, false);
+	if (IS_ERR(gr3d->group)) {
+		err = PTR_ERR(gr3d->group);
+		dev_err(client->dev, "failed to attach to domain: %d\n", err);
+		goto free;
+	}
+
+	err = tegra_drm_register_client(dev->dev_private, drm);
+	if (err < 0) {
+		dev_err(client->dev, "failed to register client: %d\n", err);
+		goto detach;
+	}
+
+	return 0;
+
+detach:
+	host1x_client_iommu_detach(client, gr3d->group);
+free:
+	host1x_syncpt_free(client->syncpts[0]);
+put:
+	host1x_channel_put(gr3d->channel);
+	return err;
 }
 
 static int gr3d_exit(struct host1x_client *client)
@@ -66,6 +98,7 @@ static int gr3d_exit(struct host1x_client *client)
 	if (err < 0)
 		return err;
 
+	host1x_client_iommu_detach(client, gr3d->group);
 	host1x_syncpt_free(client->syncpts[0]);
 	host1x_channel_put(gr3d->channel);
 
@@ -125,10 +158,22 @@ static const struct tegra_drm_client_ops gr3d_ops = {
 	.submit = tegra_drm_submit,
 };
 
+static const struct gr3d_soc tegra20_gr3d_soc = {
+	.version = 0x20,
+};
+
+static const struct gr3d_soc tegra30_gr3d_soc = {
+	.version = 0x30,
+};
+
+static const struct gr3d_soc tegra114_gr3d_soc = {
+	.version = 0x35,
+};
+
 static const struct of_device_id tegra_gr3d_match[] = {
-	{ .compatible = "nvidia,tegra114-gr3d" },
-	{ .compatible = "nvidia,tegra30-gr3d" },
-	{ .compatible = "nvidia,tegra20-gr3d" },
+	{ .compatible = "nvidia,tegra114-gr3d", .data = &tegra114_gr3d_soc },
+	{ .compatible = "nvidia,tegra30-gr3d", .data = &tegra30_gr3d_soc },
+	{ .compatible = "nvidia,tegra20-gr3d", .data = &tegra20_gr3d_soc },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, tegra_gr3d_match);
@@ -250,6 +295,8 @@ static int gr3d_probe(struct platform_device *pdev)
 	if (!gr3d)
 		return -ENOMEM;
 
+	gr3d->soc = of_device_get_match_data(&pdev->dev);
+
 	syncpts = devm_kzalloc(&pdev->dev, sizeof(*syncpts), GFP_KERNEL);
 	if (!syncpts)
 		return -ENOMEM;
@@ -307,6 +354,7 @@ static int gr3d_probe(struct platform_device *pdev)
 	gr3d->client.base.num_syncpts = 1;
 
 	INIT_LIST_HEAD(&gr3d->client.list);
+	gr3d->client.version = gr3d->soc->version;
 	gr3d->client.ops = &gr3d_ops;
 
 	err = host1x_client_register(&gr3d->client.base);
diff --git a/drivers/gpu/drm/tegra/hub.c b/drivers/gpu/drm/tegra/hub.c
index 9a3f23d4780f..8f4fcbb515fb 100644
--- a/drivers/gpu/drm/tegra/hub.c
+++ b/drivers/gpu/drm/tegra/hub.c
@@ -687,7 +687,7 @@ void tegra_display_hub_atomic_commit(struct drm_device *drm,
 	struct device *dev = hub->client.dev;
 	int err;
 
-	hub_state = tegra_display_hub_get_state(hub, state);
+	hub_state = to_tegra_display_hub_state(hub->base.state);
 
 	if (hub_state->clk) {
 		err = clk_set_rate(hub_state->clk, hub_state->rate);
diff --git a/drivers/gpu/drm/tegra/plane.c b/drivers/gpu/drm/tegra/plane.c
index 176ef46c615c..d068e8aa3553 100644
--- a/drivers/gpu/drm/tegra/plane.c
+++ b/drivers/gpu/drm/tegra/plane.c
@@ -23,6 +23,7 @@ static void tegra_plane_destroy(struct drm_plane *plane)
 
 static void tegra_plane_reset(struct drm_plane *plane)
 {
+	struct tegra_plane *p = to_tegra_plane(plane);
 	struct tegra_plane_state *state;
 
 	if (plane->state)
@@ -35,6 +36,8 @@ static void tegra_plane_reset(struct drm_plane *plane)
 	if (state) {
 		plane->state = &state->base;
 		plane->state->plane = plane;
+		plane->state->zpos = p->index;
+		plane->state->normalized_zpos = p->index;
 	}
 }
 
@@ -53,10 +56,11 @@ tegra_plane_atomic_duplicate_state(struct drm_plane *plane)
 	copy->tiling = state->tiling;
 	copy->format = state->format;
 	copy->swap = state->swap;
+	copy->bottom_up = state->bottom_up;
 	copy->opaque = state->opaque;
 
-	for (i = 0; i < 3; i++)
-		copy->dependent[i] = state->dependent[i];
+	for (i = 0; i < 2; i++)
+		copy->blending[i] = state->blending[i];
 
 	return &copy->base;
 }
@@ -267,24 +271,8 @@ static bool __drm_format_has_alpha(u32 format)
 	return false;
 }
 
-/*
- * This is applicable to Tegra20 and Tegra30 only where the opaque formats can
- * be emulated using the alpha formats and alpha blending disabled.
- */
-bool tegra_plane_format_has_alpha(unsigned int format)
-{
-	switch (format) {
-	case WIN_COLOR_DEPTH_B5G5R5A1:
-	case WIN_COLOR_DEPTH_A1B5G5R5:
-	case WIN_COLOR_DEPTH_R8G8B8A8:
-	case WIN_COLOR_DEPTH_B8G8R8A8:
-		return true;
-	}
-
-	return false;
-}
-
-int tegra_plane_format_get_alpha(unsigned int opaque, unsigned int *alpha)
+static int tegra_plane_format_get_alpha(unsigned int opaque,
+					unsigned int *alpha)
 {
 	if (tegra_plane_format_is_yuv(opaque, NULL)) {
 		*alpha = opaque;
@@ -316,6 +304,67 @@ int tegra_plane_format_get_alpha(unsigned int opaque, unsigned int *alpha)
 	return -EINVAL;
 }
 
+/*
+ * This is applicable to Tegra20 and Tegra30 only where the opaque formats can
+ * be emulated using the alpha formats and alpha blending disabled.
+ */
+static int tegra_plane_setup_opacity(struct tegra_plane *tegra,
+				     struct tegra_plane_state *state)
+{
+	unsigned int format;
+	int err;
+
+	switch (state->format) {
+	case WIN_COLOR_DEPTH_B5G5R5A1:
+	case WIN_COLOR_DEPTH_A1B5G5R5:
+	case WIN_COLOR_DEPTH_R8G8B8A8:
+	case WIN_COLOR_DEPTH_B8G8R8A8:
+		state->opaque = false;
+		break;
+
+	default:
+		err = tegra_plane_format_get_alpha(state->format, &format);
+		if (err < 0)
+			return err;
+
+		state->format = format;
+		state->opaque = true;
+		break;
+	}
+
+	return 0;
+}
+
+static int tegra_plane_check_transparency(struct tegra_plane *tegra,
+					  struct tegra_plane_state *state)
+{
+	struct drm_plane_state *old, *plane_state;
+	struct drm_plane *plane;
+
+	old = drm_atomic_get_old_plane_state(state->base.state, &tegra->base);
+
+	/* check if zpos / transparency changed */
+	if (old->normalized_zpos == state->base.normalized_zpos &&
+	    to_tegra_plane_state(old)->opaque == state->opaque)
+		return 0;
+
+	/* include all sibling planes into this commit */
+	drm_for_each_plane(plane, tegra->base.dev) {
+		struct tegra_plane *p = to_tegra_plane(plane);
+
+		/* skip this plane and planes on different CRTCs */
+		if (p == tegra || p->dc != tegra->dc)
+			continue;
+
+		plane_state = drm_atomic_get_plane_state(state->base.state,
+							 plane);
+		if (IS_ERR(plane_state))
+			return PTR_ERR(plane_state);
+	}
+
+	return 1;
+}
+
 static unsigned int tegra_plane_get_overlap_index(struct tegra_plane *plane,
 						  struct tegra_plane *other)
 {
@@ -336,61 +385,98 @@ static unsigned int tegra_plane_get_overlap_index(struct tegra_plane *plane,
 	return index;
 }
 
-void tegra_plane_check_dependent(struct tegra_plane *tegra,
-				 struct tegra_plane_state *state)
+static void tegra_plane_update_transparency(struct tegra_plane *tegra,
+					    struct tegra_plane_state *state)
 {
-	struct drm_plane_state *old, *new;
+	struct drm_plane_state *new;
 	struct drm_plane *plane;
-	unsigned int zpos[2];
 	unsigned int i;
 
-	for (i = 0; i < 2; i++)
-		zpos[i] = 0;
-
-	for_each_oldnew_plane_in_state(state->base.state, plane, old, new, i) {
+	for_each_new_plane_in_state(state->base.state, plane, new, i) {
 		struct tegra_plane *p = to_tegra_plane(plane);
 		unsigned index;
 
 		/* skip this plane and planes on different CRTCs */
-		if (p == tegra || new->crtc != state->base.crtc)
+		if (p == tegra || p->dc != tegra->dc)
 			continue;
 
 		index = tegra_plane_get_overlap_index(tegra, p);
 
-		state->dependent[index] = false;
+		if (new->fb && __drm_format_has_alpha(new->fb->format->format))
+			state->blending[index].alpha = true;
+		else
+			state->blending[index].alpha = false;
+
+		if (new->normalized_zpos > state->base.normalized_zpos)
+			state->blending[index].top = true;
+		else
+			state->blending[index].top = false;
 
 		/*
-		 * If any of the other planes is on top of this plane and uses
-		 * a format with an alpha component, mark this plane as being
-		 * dependent, meaning it's alpha value will be 1 minus the sum
-		 * of alpha components of the overlapping planes.
+		 * Missing framebuffer means that plane is disabled, in this
+		 * case mark B / C window as top to be able to differentiate
+		 * windows indices order in regards to zPos for the middle
+		 * window X / Y registers programming.
 		 */
-		if (p->index > tegra->index) {
-			if (__drm_format_has_alpha(new->fb->format->format))
-				state->dependent[index] = true;
-
-			/* keep track of the Z position */
-			zpos[index] = p->index;
-		}
+		if (!new->fb)
+			state->blending[index].top = (index == 1);
 	}
+}
+
+static int tegra_plane_setup_transparency(struct tegra_plane *tegra,
+					  struct tegra_plane_state *state)
+{
+	struct tegra_plane_state *tegra_state;
+	struct drm_plane_state *new;
+	struct drm_plane *plane;
+	int err;
 
 	/*
-	 * The region where three windows overlap is the intersection of the
-	 * two regions where two windows overlap. It contributes to the area
-	 * if any of the windows on top of it have an alpha component.
+	 * If planes zpos / transparency changed, sibling planes blending
+	 * state may require adjustment and in this case they will be included
+	 * into this atom commit, otherwise blending state is unchanged.
 	 */
-	for (i = 0; i < 2; i++)
-		state->dependent[2] = state->dependent[2] ||
-				      state->dependent[i];
+	err = tegra_plane_check_transparency(tegra, state);
+	if (err <= 0)
+		return err;
 
 	/*
-	 * However, if any of the windows on top of this window is opaque, it
-	 * will completely conceal this window within that area, so avoid the
-	 * window from contributing to the area.
+	 * All planes are now in the atomic state, walk them up and update
+	 * transparency state for each plane.
 	 */
-	for (i = 0; i < 2; i++) {
-		if (zpos[i] > tegra->index)
-			state->dependent[2] = state->dependent[2] &&
-					      state->dependent[i];
+	drm_for_each_plane(plane, tegra->base.dev) {
+		struct tegra_plane *p = to_tegra_plane(plane);
+
+		/* skip planes on different CRTCs */
+		if (p->dc != tegra->dc)
+			continue;
+
+		new = drm_atomic_get_new_plane_state(state->base.state, plane);
+		tegra_state = to_tegra_plane_state(new);
+
+		/*
+		 * There is no need to update blending state for the disabled
+		 * plane.
+		 */
+		if (new->fb)
+			tegra_plane_update_transparency(p, tegra_state);
 	}
+
+	return 0;
+}
+
+int tegra_plane_setup_legacy_state(struct tegra_plane *tegra,
+				   struct tegra_plane_state *state)
+{
+	int err;
+
+	err = tegra_plane_setup_opacity(tegra, state);
+	if (err < 0)
+		return err;
+
+	err = tegra_plane_setup_transparency(tegra, state);
+	if (err < 0)
+		return err;
+
+	return 0;
 }
diff --git a/drivers/gpu/drm/tegra/plane.h b/drivers/gpu/drm/tegra/plane.h
index 6938719e7e5d..e79e6b4a8e0a 100644
--- a/drivers/gpu/drm/tegra/plane.h
+++ b/drivers/gpu/drm/tegra/plane.h
@@ -34,6 +34,11 @@ static inline struct tegra_plane *to_tegra_plane(struct drm_plane *plane)
 	return container_of(plane, struct tegra_plane, base);
 }
 
+struct tegra_plane_legacy_blending_state {
+	bool alpha;
+	bool top;
+};
+
 struct tegra_plane_state {
 	struct drm_plane_state base;
 
@@ -41,9 +46,11 @@ struct tegra_plane_state {
 	u32 format;
 	u32 swap;
 
+	bool bottom_up;
+
 	/* used for legacy blending support only */
+	struct tegra_plane_legacy_blending_state blending[2];
 	bool opaque;
-	bool dependent[3];
 };
 
 static inline struct tegra_plane_state *
@@ -62,9 +69,7 @@ int tegra_plane_state_add(struct tegra_plane *plane,
 
 int tegra_plane_format(u32 fourcc, u32 *format, u32 *swap);
 bool tegra_plane_format_is_yuv(unsigned int format, bool *planar);
-bool tegra_plane_format_has_alpha(unsigned int format);
-int tegra_plane_format_get_alpha(unsigned int opaque, unsigned int *alpha);
-void tegra_plane_check_dependent(struct tegra_plane *tegra,
-				 struct tegra_plane_state *state);
+int tegra_plane_setup_legacy_state(struct tegra_plane *tegra,
+				   struct tegra_plane_state *state);
 
 #endif /* TEGRA_PLANE_H */
diff --git a/drivers/gpu/drm/tegra/vic.c b/drivers/gpu/drm/tegra/vic.c
index f5794dd49f3b..9f657a63b0bb 100644
--- a/drivers/gpu/drm/tegra/vic.c
+++ b/drivers/gpu/drm/tegra/vic.c
@@ -25,6 +25,7 @@
 
 struct vic_config {
 	const char *firmware;
+	unsigned int version;
 };
 
 struct vic {
@@ -264,18 +265,21 @@ static const struct tegra_drm_client_ops vic_ops = {
 
 static const struct vic_config vic_t124_config = {
 	.firmware = NVIDIA_TEGRA_124_VIC_FIRMWARE,
+	.version = 0x40,
 };
 
 #define NVIDIA_TEGRA_210_VIC_FIRMWARE "nvidia/tegra210/vic04_ucode.bin"
 
 static const struct vic_config vic_t210_config = {
 	.firmware = NVIDIA_TEGRA_210_VIC_FIRMWARE,
+	.version = 0x21,
 };
 
 #define NVIDIA_TEGRA_186_VIC_FIRMWARE "nvidia/tegra186/vic04_ucode.bin"
 
 static const struct vic_config vic_t186_config = {
 	.firmware = NVIDIA_TEGRA_186_VIC_FIRMWARE,
+	.version = 0x18,
 };
 
 static const struct of_device_id vic_match[] = {
@@ -342,6 +346,7 @@ static int vic_probe(struct platform_device *pdev)
 	vic->dev = dev;
 
 	INIT_LIST_HEAD(&vic->client.list);
+	vic->client.version = vic->config->version;
 	vic->client.ops = &vic_ops;
 
 	err = host1x_client_register(&vic->client.base);
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index 1b278a22c8b7..1067e702c22c 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -224,7 +224,7 @@ static void tilcdc_crtc_set_clk(struct drm_crtc *crtc)
 
 	ret = clk_set_rate(priv->clk, req_rate * clkdiv);
 	clk_rate = clk_get_rate(priv->clk);
-	if (ret < 0) {
+	if (ret < 0 || tilcdc_pclk_diff(req_rate, clk_rate) > 5) {
 		/*
 		 * If we fail to set the clock rate (some architectures don't
 		 * use the common clock framework yet and may not implement
diff --git a/drivers/gpu/drm/tinydrm/core/tinydrm-core.c b/drivers/gpu/drm/tinydrm/core/tinydrm-core.c
index 4c6616278c48..24a33bf862fa 100644
--- a/drivers/gpu/drm/tinydrm/core/tinydrm-core.c
+++ b/drivers/gpu/drm/tinydrm/core/tinydrm-core.c
@@ -91,7 +91,7 @@ EXPORT_SYMBOL(tinydrm_gem_cma_prime_import_sg_table);
  * GEM object state and frees the memory used to store the object itself using
  * drm_gem_cma_free_object(). It also handles PRIME buffers which has the kernel
  * virtual address set by tinydrm_gem_cma_prime_import_sg_table(). Drivers
- * can use this as their &drm_driver->gem_free_object callback.
+ * can use this as their &drm_driver->gem_free_object_unlocked callback.
  */
 void tinydrm_gem_cma_free_object(struct drm_gem_object *gem_obj)
 {
diff --git a/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c b/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
index d1c3ce9ab294..dcd390163a4a 100644
--- a/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
+++ b/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
@@ -78,6 +78,36 @@ bool tinydrm_merge_clips(struct drm_clip_rect *dst,
 }
 EXPORT_SYMBOL(tinydrm_merge_clips);
 
+int tinydrm_fb_dirty(struct drm_framebuffer *fb,
+		     struct drm_file *file_priv,
+		     unsigned int flags, unsigned int color,
+		     struct drm_clip_rect *clips,
+		     unsigned int num_clips)
+{
+	struct tinydrm_device *tdev = fb->dev->dev_private;
+	struct drm_plane *plane = &tdev->pipe.plane;
+	int ret = 0;
+
+	drm_modeset_lock(&plane->mutex, NULL);
+
+	/* fbdev can flush even when we're not interested */
+	if (plane->state->fb == fb) {
+		mutex_lock(&tdev->dirty_lock);
+		ret = tdev->fb_dirty(fb, file_priv, flags,
+				     color, clips, num_clips);
+		mutex_unlock(&tdev->dirty_lock);
+	}
+
+	drm_modeset_unlock(&plane->mutex);
+
+	if (ret)
+		dev_err_once(fb->dev->dev,
+			     "Failed to update display %d\n", ret);
+
+	return ret;
+}
+EXPORT_SYMBOL(tinydrm_fb_dirty);
+
 /**
  * tinydrm_memcpy - Copy clip buffer
  * @dst: Destination buffer
diff --git a/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c b/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c
index 11ae950b0fc9..7e8e24d0b7a7 100644
--- a/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c
+++ b/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c
@@ -125,9 +125,8 @@ void tinydrm_display_pipe_update(struct drm_simple_display_pipe *pipe,
 	struct drm_crtc *crtc = &tdev->pipe.crtc;
 
 	if (fb && (fb != old_state->fb)) {
-		pipe->plane.fb = fb;
-		if (fb->funcs->dirty)
-			fb->funcs->dirty(fb, NULL, 0, 0, NULL, 0);
+		if (tdev->fb_dirty)
+			tdev->fb_dirty(fb, NULL, 0, 0, NULL, 0);
 	}
 
 	if (crtc->state->event) {
@@ -139,23 +138,6 @@ void tinydrm_display_pipe_update(struct drm_simple_display_pipe *pipe,
 }
 EXPORT_SYMBOL(tinydrm_display_pipe_update);
 
-/**
- * tinydrm_display_pipe_prepare_fb - Display pipe prepare_fb helper
- * @pipe: Simple display pipe
- * @plane_state: Plane state
- *
- * This function uses drm_gem_fb_prepare_fb() to check if the plane FB has an
- * dma-buf attached, extracts the exclusive fence and attaches it to plane
- * state for the atomic helper to wait on. Drivers can use this as their
- * &drm_simple_display_pipe_funcs->prepare_fb callback.
- */
-int tinydrm_display_pipe_prepare_fb(struct drm_simple_display_pipe *pipe,
-				    struct drm_plane_state *plane_state)
-{
-	return drm_gem_fb_prepare_fb(&pipe->plane, plane_state);
-}
-EXPORT_SYMBOL(tinydrm_display_pipe_prepare_fb);
-
 static int tinydrm_rotate_mode(struct drm_display_mode *mode,
 			       unsigned int rotation)
 {
diff --git a/drivers/gpu/drm/tinydrm/ili9225.c b/drivers/gpu/drm/tinydrm/ili9225.c
index a0759502b81a..841c69aba059 100644
--- a/drivers/gpu/drm/tinydrm/ili9225.c
+++ b/drivers/gpu/drm/tinydrm/ili9225.c
@@ -88,14 +88,8 @@ static int ili9225_fb_dirty(struct drm_framebuffer *fb,
 	bool full;
 	void *tr;
 
-	mutex_lock(&tdev->dirty_lock);
-
 	if (!mipi->enabled)
-		goto out_unlock;
-
-	/* fbdev can flush even when we're not interested */
-	if (tdev->pipe.plane.fb != fb)
-		goto out_unlock;
+		return 0;
 
 	full = tinydrm_merge_clips(&clip, clips, num_clips, flags,
 				   fb->width, fb->height);
@@ -108,7 +102,7 @@ static int ili9225_fb_dirty(struct drm_framebuffer *fb,
 		tr = mipi->tx_buf;
 		ret = mipi_dbi_buf_copy(mipi->tx_buf, fb, &clip, swap);
 		if (ret)
-			goto out_unlock;
+			return ret;
 	} else {
 		tr = cma_obj->vaddr;
 	}
@@ -159,24 +153,18 @@ static int ili9225_fb_dirty(struct drm_framebuffer *fb,
 	ret = mipi_dbi_command_buf(mipi, ILI9225_WRITE_DATA_TO_GRAM, tr,
 				(clip.x2 - clip.x1) * (clip.y2 - clip.y1) * 2);
 
-out_unlock:
-	mutex_unlock(&tdev->dirty_lock);
-
-	if (ret)
-		dev_err_once(fb->dev->dev, "Failed to update display %d\n",
-			     ret);
-
 	return ret;
 }
 
 static const struct drm_framebuffer_funcs ili9225_fb_funcs = {
 	.destroy	= drm_gem_fb_destroy,
 	.create_handle	= drm_gem_fb_create_handle,
-	.dirty		= ili9225_fb_dirty,
+	.dirty		= tinydrm_fb_dirty,
 };
 
 static void ili9225_pipe_enable(struct drm_simple_display_pipe *pipe,
-				struct drm_crtc_state *crtc_state)
+				struct drm_crtc_state *crtc_state,
+				struct drm_plane_state *plane_state)
 {
 	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
 	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
@@ -268,7 +256,7 @@ static void ili9225_pipe_enable(struct drm_simple_display_pipe *pipe,
 
 	ili9225_command(mipi, ILI9225_DISPLAY_CONTROL_1, 0x1017);
 
-	mipi_dbi_enable_flush(mipi);
+	mipi_dbi_enable_flush(mipi, crtc_state, plane_state);
 }
 
 static void ili9225_pipe_disable(struct drm_simple_display_pipe *pipe)
@@ -341,6 +329,8 @@ static int ili9225_init(struct device *dev, struct mipi_dbi *mipi,
 	if (ret)
 		return ret;
 
+	tdev->fb_dirty = ili9225_fb_dirty;
+
 	ret = tinydrm_display_pipe_init(tdev, pipe_funcs,
 					DRM_MODE_CONNECTOR_VIRTUAL,
 					ili9225_formats,
@@ -364,7 +354,7 @@ static const struct drm_simple_display_pipe_funcs ili9225_pipe_funcs = {
 	.enable		= ili9225_pipe_enable,
 	.disable	= ili9225_pipe_disable,
 	.update		= tinydrm_display_pipe_update,
-	.prepare_fb	= tinydrm_display_pipe_prepare_fb,
+	.prepare_fb	= drm_gem_fb_simple_display_pipe_prepare_fb,
 };
 
 static const struct drm_display_mode ili9225_mode = {
diff --git a/drivers/gpu/drm/tinydrm/mi0283qt.c b/drivers/gpu/drm/tinydrm/mi0283qt.c
index d8ed6e6f8e05..015d03f2acba 100644
--- a/drivers/gpu/drm/tinydrm/mi0283qt.c
+++ b/drivers/gpu/drm/tinydrm/mi0283qt.c
@@ -19,6 +19,7 @@
 
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_modeset_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/tinydrm/mipi-dbi.h>
 #include <drm/tinydrm/tinydrm-helpers.h>
 #include <video/mipi_display.h>
@@ -49,7 +50,8 @@
 #define ILI9341_MADCTL_MY	BIT(7)
 
 static void mi0283qt_enable(struct drm_simple_display_pipe *pipe,
-			    struct drm_crtc_state *crtc_state)
+			    struct drm_crtc_state *crtc_state,
+			    struct drm_plane_state *plane_state)
 {
 	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
 	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
@@ -83,24 +85,6 @@ static void mi0283qt_enable(struct drm_simple_display_pipe *pipe,
 	/* Memory Access Control */
 	mipi_dbi_command(mipi, MIPI_DCS_SET_PIXEL_FORMAT, MIPI_DCS_PIXEL_FMT_16BIT);
 
-	switch (mipi->rotation) {
-	default:
-		addr_mode = ILI9341_MADCTL_MV | ILI9341_MADCTL_MY |
-			    ILI9341_MADCTL_MX;
-		break;
-	case 90:
-		addr_mode = ILI9341_MADCTL_MY;
-		break;
-	case 180:
-		addr_mode = ILI9341_MADCTL_MV;
-		break;
-	case 270:
-		addr_mode = ILI9341_MADCTL_MX;
-		break;
-	}
-	addr_mode |= ILI9341_MADCTL_BGR;
-	mipi_dbi_command(mipi, MIPI_DCS_SET_ADDRESS_MODE, addr_mode);
-
 	/* Frame Rate */
 	mipi_dbi_command(mipi, ILI9341_FRMCTR1, 0x00, 0x1b);
 
@@ -126,14 +110,37 @@ static void mi0283qt_enable(struct drm_simple_display_pipe *pipe,
 	msleep(100);
 
 out_enable:
-	mipi_dbi_enable_flush(mipi);
+	/* The PiTFT (ili9340) has a hardware reset circuit that
+	 * resets only on power-on and not on each reboot through
+	 * a gpio like the rpi-display does.
+	 * As a result, we need to always apply the rotation value
+	 * regardless of the display "on/off" state.
+	 */
+	switch (mipi->rotation) {
+	default:
+		addr_mode = ILI9341_MADCTL_MV | ILI9341_MADCTL_MY |
+			    ILI9341_MADCTL_MX;
+		break;
+	case 90:
+		addr_mode = ILI9341_MADCTL_MY;
+		break;
+	case 180:
+		addr_mode = ILI9341_MADCTL_MV;
+		break;
+	case 270:
+		addr_mode = ILI9341_MADCTL_MX;
+		break;
+	}
+	addr_mode |= ILI9341_MADCTL_BGR;
+	mipi_dbi_command(mipi, MIPI_DCS_SET_ADDRESS_MODE, addr_mode);
+	mipi_dbi_enable_flush(mipi, crtc_state, plane_state);
 }
 
 static const struct drm_simple_display_pipe_funcs mi0283qt_pipe_funcs = {
 	.enable = mi0283qt_enable,
 	.disable = mipi_dbi_pipe_disable,
 	.update = tinydrm_display_pipe_update,
-	.prepare_fb = tinydrm_display_pipe_prepare_fb,
+	.prepare_fb = drm_gem_fb_simple_display_pipe_prepare_fb,
 };
 
 static const struct drm_display_mode mi0283qt_mode = {
diff --git a/drivers/gpu/drm/tinydrm/mipi-dbi.c b/drivers/gpu/drm/tinydrm/mipi-dbi.c
index 9e903812b573..4d1fb31a781f 100644
--- a/drivers/gpu/drm/tinydrm/mipi-dbi.c
+++ b/drivers/gpu/drm/tinydrm/mipi-dbi.c
@@ -219,14 +219,8 @@ static int mipi_dbi_fb_dirty(struct drm_framebuffer *fb,
 	bool full;
 	void *tr;
 
-	mutex_lock(&tdev->dirty_lock);
-
 	if (!mipi->enabled)
-		goto out_unlock;
-
-	/* fbdev can flush even when we're not interested */
-	if (tdev->pipe.plane.fb != fb)
-		goto out_unlock;
+		return 0;
 
 	full = tinydrm_merge_clips(&clip, clips, num_clips, flags,
 				   fb->width, fb->height);
@@ -239,7 +233,7 @@ static int mipi_dbi_fb_dirty(struct drm_framebuffer *fb,
 		tr = mipi->tx_buf;
 		ret = mipi_dbi_buf_copy(mipi->tx_buf, fb, &clip, swap);
 		if (ret)
-			goto out_unlock;
+			return ret;
 	} else {
 		tr = cma_obj->vaddr;
 	}
@@ -254,20 +248,13 @@ static int mipi_dbi_fb_dirty(struct drm_framebuffer *fb,
 	ret = mipi_dbi_command_buf(mipi, MIPI_DCS_WRITE_MEMORY_START, tr,
 				(clip.x2 - clip.x1) * (clip.y2 - clip.y1) * 2);
 
-out_unlock:
-	mutex_unlock(&tdev->dirty_lock);
-
-	if (ret)
-		dev_err_once(fb->dev->dev, "Failed to update display %d\n",
-			     ret);
-
 	return ret;
 }
 
 static const struct drm_framebuffer_funcs mipi_dbi_fb_funcs = {
 	.destroy	= drm_gem_fb_destroy,
 	.create_handle	= drm_gem_fb_create_handle,
-	.dirty		= mipi_dbi_fb_dirty,
+	.dirty		= tinydrm_fb_dirty,
 };
 
 /**
@@ -278,13 +265,16 @@ static const struct drm_framebuffer_funcs mipi_dbi_fb_funcs = {
  * enables the backlight. Drivers can use this in their
  * &drm_simple_display_pipe_funcs->enable callback.
  */
-void mipi_dbi_enable_flush(struct mipi_dbi *mipi)
+void mipi_dbi_enable_flush(struct mipi_dbi *mipi,
+			   struct drm_crtc_state *crtc_state,
+			   struct drm_plane_state *plane_state)
 {
-	struct drm_framebuffer *fb = mipi->tinydrm.pipe.plane.fb;
+	struct tinydrm_device *tdev = &mipi->tinydrm;
+	struct drm_framebuffer *fb = plane_state->fb;
 
 	mipi->enabled = true;
 	if (fb)
-		fb->funcs->dirty(fb, NULL, 0, 0, NULL, 0);
+		tdev->fb_dirty(fb, NULL, 0, 0, NULL, 0);
 
 	backlight_enable(mipi->backlight);
 }
@@ -381,6 +371,8 @@ int mipi_dbi_init(struct device *dev, struct mipi_dbi *mipi,
 	if (ret)
 		return ret;
 
+	tdev->fb_dirty = mipi_dbi_fb_dirty;
+
 	/* TODO: Maybe add DRM_MODE_CONNECTOR_SPI */
 	ret = tinydrm_display_pipe_init(tdev, pipe_funcs,
 					DRM_MODE_CONNECTOR_VIRTUAL,
diff --git a/drivers/gpu/drm/tinydrm/repaper.c b/drivers/gpu/drm/tinydrm/repaper.c
index 75740630c410..1ee6855212a0 100644
--- a/drivers/gpu/drm/tinydrm/repaper.c
+++ b/drivers/gpu/drm/tinydrm/repaper.c
@@ -540,14 +540,8 @@ static int repaper_fb_dirty(struct drm_framebuffer *fb,
 	clip.y1 = 0;
 	clip.y2 = fb->height;
 
-	mutex_lock(&tdev->dirty_lock);
-
 	if (!epd->enabled)
-		goto out_unlock;
-
-	/* fbdev can flush even when we're not interested */
-	if (tdev->pipe.plane.fb != fb)
-		goto out_unlock;
+		return 0;
 
 	repaper_get_temperature(epd);
 
@@ -555,16 +549,14 @@ static int repaper_fb_dirty(struct drm_framebuffer *fb,
 		  epd->factored_stage_time);
 
 	buf = kmalloc(fb->width * fb->height, GFP_KERNEL);
-	if (!buf) {
-		ret = -ENOMEM;
-		goto out_unlock;
-	}
+	if (!buf)
+		return -ENOMEM;
 
 	if (import_attach) {
 		ret = dma_buf_begin_cpu_access(import_attach->dmabuf,
 					       DMA_FROM_DEVICE);
 		if (ret)
-			goto out_unlock;
+			goto out_free;
 	}
 
 	tinydrm_xrgb8888_to_gray8(buf, cma_obj->vaddr, fb, &clip);
@@ -573,7 +565,7 @@ static int repaper_fb_dirty(struct drm_framebuffer *fb,
 		ret = dma_buf_end_cpu_access(import_attach->dmabuf,
 					     DMA_FROM_DEVICE);
 		if (ret)
-			goto out_unlock;
+			goto out_free;
 	}
 
 	repaper_gray8_to_mono_reversed(buf, fb->width, fb->height);
@@ -625,11 +617,7 @@ static int repaper_fb_dirty(struct drm_framebuffer *fb,
 			}
 	}
 
-out_unlock:
-	mutex_unlock(&tdev->dirty_lock);
-
-	if (ret)
-		DRM_DEV_ERROR(fb->dev->dev, "Failed to update display (%d)\n", ret);
+out_free:
 	kfree(buf);
 
 	return ret;
@@ -638,7 +626,7 @@ static int repaper_fb_dirty(struct drm_framebuffer *fb,
 static const struct drm_framebuffer_funcs repaper_fb_funcs = {
 	.destroy	= drm_gem_fb_destroy,
 	.create_handle	= drm_gem_fb_create_handle,
-	.dirty		= repaper_fb_dirty,
+	.dirty		= tinydrm_fb_dirty,
 };
 
 static void power_off(struct repaper_epd *epd)
@@ -659,7 +647,8 @@ static void power_off(struct repaper_epd *epd)
 }
 
 static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
-				struct drm_crtc_state *crtc_state)
+				struct drm_crtc_state *crtc_state,
+				struct drm_plane_state *plane_state)
 {
 	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
 	struct repaper_epd *epd = epd_from_tinydrm(tdev);
@@ -852,7 +841,7 @@ static const struct drm_simple_display_pipe_funcs repaper_pipe_funcs = {
 	.enable = repaper_pipe_enable,
 	.disable = repaper_pipe_disable,
 	.update = tinydrm_display_pipe_update,
-	.prepare_fb = tinydrm_display_pipe_prepare_fb,
+	.prepare_fb = drm_gem_fb_simple_display_pipe_prepare_fb,
 };
 
 static const uint32_t repaper_formats[] = {
@@ -1069,6 +1058,8 @@ static int repaper_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
+	tdev->fb_dirty = repaper_fb_dirty;
+
 	ret = tinydrm_display_pipe_init(tdev, &repaper_pipe_funcs,
 					DRM_MODE_CONNECTOR_VIRTUAL,
 					repaper_formats,
diff --git a/drivers/gpu/drm/tinydrm/st7586.c b/drivers/gpu/drm/tinydrm/st7586.c
index a6396ef9cc4a..5c29e3803ecb 100644
--- a/drivers/gpu/drm/tinydrm/st7586.c
+++ b/drivers/gpu/drm/tinydrm/st7586.c
@@ -120,14 +120,8 @@ static int st7586_fb_dirty(struct drm_framebuffer *fb,
 	int start, end;
 	int ret = 0;
 
-	mutex_lock(&tdev->dirty_lock);
-
 	if (!mipi->enabled)
-		goto out_unlock;
-
-	/* fbdev can flush even when we're not interested */
-	if (tdev->pipe.plane.fb != fb)
-		goto out_unlock;
+		return 0;
 
 	tinydrm_merge_clips(&clip, clips, num_clips, flags, fb->width,
 			    fb->height);
@@ -141,7 +135,7 @@ static int st7586_fb_dirty(struct drm_framebuffer *fb,
 
 	ret = st7586_buf_copy(mipi->tx_buf, fb, &clip);
 	if (ret)
-		goto out_unlock;
+		return ret;
 
 	/* Pixels are packed 3 per byte */
 	start = clip.x1 / 3;
@@ -158,24 +152,18 @@ static int st7586_fb_dirty(struct drm_framebuffer *fb,
 				   (u8 *)mipi->tx_buf,
 				   (end - start) * (clip.y2 - clip.y1));
 
-out_unlock:
-	mutex_unlock(&tdev->dirty_lock);
-
-	if (ret)
-		dev_err_once(fb->dev->dev, "Failed to update display %d\n",
-			     ret);
-
 	return ret;
 }
 
 static const struct drm_framebuffer_funcs st7586_fb_funcs = {
 	.destroy	= drm_gem_fb_destroy,
 	.create_handle	= drm_gem_fb_create_handle,
-	.dirty		= st7586_fb_dirty,
+	.dirty		= tinydrm_fb_dirty,
 };
 
 static void st7586_pipe_enable(struct drm_simple_display_pipe *pipe,
-			       struct drm_crtc_state *crtc_state)
+			       struct drm_crtc_state *crtc_state,
+			       struct drm_plane_state *plane_state)
 {
 	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
 	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
@@ -237,7 +225,7 @@ static void st7586_pipe_enable(struct drm_simple_display_pipe *pipe,
 
 	mipi_dbi_command(mipi, MIPI_DCS_SET_DISPLAY_ON);
 
-	mipi_dbi_enable_flush(mipi);
+	mipi_dbi_enable_flush(mipi, crtc_state, plane_state);
 }
 
 static void st7586_pipe_disable(struct drm_simple_display_pipe *pipe)
@@ -277,6 +265,8 @@ static int st7586_init(struct device *dev, struct mipi_dbi *mipi,
 	if (ret)
 		return ret;
 
+	tdev->fb_dirty = st7586_fb_dirty;
+
 	ret = tinydrm_display_pipe_init(tdev, pipe_funcs,
 					DRM_MODE_CONNECTOR_VIRTUAL,
 					st7586_formats,
@@ -300,7 +290,7 @@ static const struct drm_simple_display_pipe_funcs st7586_pipe_funcs = {
 	.enable		= st7586_pipe_enable,
 	.disable	= st7586_pipe_disable,
 	.update		= tinydrm_display_pipe_update,
-	.prepare_fb	= tinydrm_display_pipe_prepare_fb,
+	.prepare_fb	= drm_gem_fb_simple_display_pipe_prepare_fb,
 };
 
 static const struct drm_display_mode st7586_mode = {
diff --git a/drivers/gpu/drm/tinydrm/st7735r.c b/drivers/gpu/drm/tinydrm/st7735r.c
index 67d197ecfc4b..6c7b15c9da4f 100644
--- a/drivers/gpu/drm/tinydrm/st7735r.c
+++ b/drivers/gpu/drm/tinydrm/st7735r.c
@@ -37,7 +37,8 @@
 #define ST7735R_MV	BIT(5)
 
 static void jd_t18003_t01_pipe_enable(struct drm_simple_display_pipe *pipe,
-				      struct drm_crtc_state *crtc_state)
+				      struct drm_crtc_state *crtc_state,
+				      struct drm_plane_state *plane_state)
 {
 	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
 	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
@@ -98,14 +99,14 @@ static void jd_t18003_t01_pipe_enable(struct drm_simple_display_pipe *pipe,
 
 	msleep(20);
 
-	mipi_dbi_enable_flush(mipi);
+	mipi_dbi_enable_flush(mipi, crtc_state, plane_state);
 }
 
 static const struct drm_simple_display_pipe_funcs jd_t18003_t01_pipe_funcs = {
 	.enable		= jd_t18003_t01_pipe_enable,
 	.disable	= mipi_dbi_pipe_disable,
 	.update		= tinydrm_display_pipe_update,
-	.prepare_fb	= tinydrm_display_pipe_prepare_fb,
+	.prepare_fb	= drm_gem_fb_simple_display_pipe_prepare_fb,
 };
 
 static const struct drm_display_mode jd_t18003_t01_mode = {
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c
index 7c2485fe88d8..ea4d59eb8966 100644
--- a/drivers/gpu/drm/ttm/ttm_agp_backend.c
+++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 98e06f8bf23b..5d8688e522d1 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
@@ -1175,7 +1176,6 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
 	reservation_object_init(&bo->ttm_resv);
 	atomic_inc(&bo->bdev->glob->bo_count);
 	drm_vma_node_reset(&bo->vma_node);
-	bo->priority = 0;
 
 	/*
 	 * For ttm_bo_type_device buffers, allocate
diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c
index a7c232dc39cb..18d3debcc949 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_manager.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
  * Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 2ebbae6067ab..f2c167702eef 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
@@ -39,6 +40,11 @@
 #include <linux/module.h>
 #include <linux/reservation.h>
 
+struct ttm_transfer_obj {
+	struct ttm_buffer_object base;
+	struct ttm_buffer_object *bo;
+};
+
 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
 {
 	ttm_bo_mem_put(bo, &bo->mem);
@@ -454,7 +460,11 @@ EXPORT_SYMBOL(ttm_bo_move_memcpy);
 
 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 {
-	kfree(bo);
+	struct ttm_transfer_obj *fbo;
+
+	fbo = container_of(bo, struct ttm_transfer_obj, base);
+	ttm_bo_unref(&fbo->bo);
+	kfree(fbo);
 }
 
 /**
@@ -475,14 +485,15 @@ static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 				      struct ttm_buffer_object **new_obj)
 {
-	struct ttm_buffer_object *fbo;
+	struct ttm_transfer_obj *fbo;
 	int ret;
 
 	fbo = kmalloc(sizeof(*fbo), GFP_KERNEL);
 	if (!fbo)
 		return -ENOMEM;
 
-	*fbo = *bo;
+	fbo->base = *bo;
+	fbo->bo = ttm_bo_reference(bo);
 
 	/**
 	 * Fix up members that we shouldn't copy directly:
@@ -490,25 +501,25 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 	 */
 
 	atomic_inc(&bo->bdev->glob->bo_count);
-	INIT_LIST_HEAD(&fbo->ddestroy);
-	INIT_LIST_HEAD(&fbo->lru);
-	INIT_LIST_HEAD(&fbo->swap);
-	INIT_LIST_HEAD(&fbo->io_reserve_lru);
-	mutex_init(&fbo->wu_mutex);
-	fbo->moving = NULL;
-	drm_vma_node_reset(&fbo->vma_node);
-	atomic_set(&fbo->cpu_writers, 0);
-
-	kref_init(&fbo->list_kref);
-	kref_init(&fbo->kref);
-	fbo->destroy = &ttm_transfered_destroy;
-	fbo->acc_size = 0;
-	fbo->resv = &fbo->ttm_resv;
-	reservation_object_init(fbo->resv);
-	ret = reservation_object_trylock(fbo->resv);
+	INIT_LIST_HEAD(&fbo->base.ddestroy);
+	INIT_LIST_HEAD(&fbo->base.lru);
+	INIT_LIST_HEAD(&fbo->base.swap);
+	INIT_LIST_HEAD(&fbo->base.io_reserve_lru);
+	mutex_init(&fbo->base.wu_mutex);
+	fbo->base.moving = NULL;
+	drm_vma_node_reset(&fbo->base.vma_node);
+	atomic_set(&fbo->base.cpu_writers, 0);
+
+	kref_init(&fbo->base.list_kref);
+	kref_init(&fbo->base.kref);
+	fbo->base.destroy = &ttm_transfered_destroy;
+	fbo->base.acc_size = 0;
+	fbo->base.resv = &fbo->base.ttm_resv;
+	reservation_object_init(fbo->base.resv);
+	ret = reservation_object_trylock(fbo->base.resv);
 	WARN_ON(!ret);
 
-	*new_obj = fbo;
+	*new_obj = &fbo->base;
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index 8eba95b3c737..c7ece7613a6a 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index 3dca206e85f7..e73ae0d22897 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
diff --git a/drivers/gpu/drm/ttm/ttm_lock.c b/drivers/gpu/drm/ttm/ttm_lock.c
index 913f4318cdc0..20694b8a01ca 100644
--- a/drivers/gpu/drm/ttm/ttm_lock.c
+++ b/drivers/gpu/drm/ttm/ttm_lock.c
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c
index 27856c55dc84..450387c92b63 100644
--- a/drivers/gpu/drm/ttm/ttm_memory.c
+++ b/drivers/gpu/drm/ttm/ttm_memory.c
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c
index 66fc6395eb54..6ff40c041d79 100644
--- a/drivers/gpu/drm/ttm/ttm_module.c
+++ b/drivers/gpu/drm/ttm/ttm_module.c
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c
index 1aa2baa83959..74f1b1eb1f8e 100644
--- a/drivers/gpu/drm/ttm/ttm_object.c
+++ b/drivers/gpu/drm/ttm/ttm_object.c
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
  * Copyright (c) 2009-2013 VMware, Inc., Palo Alto, CA., USA
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index 7e672be987b5..a1e543972ca7 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 /**************************************************************************
  *
  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
diff --git a/drivers/gpu/drm/tve200/tve200_display.c b/drivers/gpu/drm/tve200/tve200_display.c
index db397fcb345a..e8723a2412a6 100644
--- a/drivers/gpu/drm/tve200/tve200_display.c
+++ b/drivers/gpu/drm/tve200/tve200_display.c
@@ -120,7 +120,8 @@ static int tve200_display_check(struct drm_simple_display_pipe *pipe,
 }
 
 static void tve200_display_enable(struct drm_simple_display_pipe *pipe,
-				 struct drm_crtc_state *cstate)
+				 struct drm_crtc_state *cstate,
+				 struct drm_plane_state *plane_state)
 {
 	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_plane *plane = &pipe->plane;
@@ -292,18 +293,12 @@ static void tve200_display_disable_vblank(struct drm_simple_display_pipe *pipe)
 	writel(0, priv->regs + TVE200_INT_EN);
 }
 
-static int tve200_display_prepare_fb(struct drm_simple_display_pipe *pipe,
-				    struct drm_plane_state *plane_state)
-{
-	return drm_gem_fb_prepare_fb(&pipe->plane, plane_state);
-}
-
 static const struct drm_simple_display_pipe_funcs tve200_display_funcs = {
 	.check = tve200_display_check,
 	.enable = tve200_display_enable,
 	.disable = tve200_display_disable,
 	.update = tve200_display_update,
-	.prepare_fb = tve200_display_prepare_fb,
+	.prepare_fb = drm_gem_fb_simple_display_pipe_prepare_fb,
 	.enable_vblank = tve200_display_enable_vblank,
 	.disable_vblank = tve200_display_disable_vblank,
 };
diff --git a/drivers/gpu/drm/udl/udl_connector.c b/drivers/gpu/drm/udl/udl_connector.c
index c3dc1fd20cb4..09dc585aa46f 100644
--- a/drivers/gpu/drm/udl/udl_connector.c
+++ b/drivers/gpu/drm/udl/udl_connector.c
@@ -105,7 +105,7 @@ static int udl_get_modes(struct drm_connector *connector)
 	return 0;
 }
 
-static int udl_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status udl_mode_valid(struct drm_connector *connector,
 			  struct drm_display_mode *mode)
 {
 	struct udl_device *udl = connector->dev->dev_private;
diff --git a/drivers/gpu/drm/udl/udl_dmabuf.c b/drivers/gpu/drm/udl/udl_dmabuf.c
index 2867ed155ff6..0a20695eb120 100644
--- a/drivers/gpu/drm/udl/udl_dmabuf.c
+++ b/drivers/gpu/drm/udl/udl_dmabuf.c
@@ -76,6 +76,7 @@ static struct sg_table *udl_map_dma_buf(struct dma_buf_attachment *attach,
 	struct udl_drm_dmabuf_attachment *udl_attach = attach->priv;
 	struct udl_gem_object *obj = to_udl_bo(attach->dmabuf->priv);
 	struct drm_device *dev = obj->base.dev;
+	struct udl_device *udl = dev->dev_private;
 	struct scatterlist *rd, *wr;
 	struct sg_table *sgt = NULL;
 	unsigned int i;
@@ -112,7 +113,7 @@ static struct sg_table *udl_map_dma_buf(struct dma_buf_attachment *attach,
 		return ERR_PTR(-ENOMEM);
 	}
 
-	mutex_lock(&dev->struct_mutex);
+	mutex_lock(&udl->gem_lock);
 
 	rd = obj->sg->sgl;
 	wr = sgt->sgl;
@@ -137,7 +138,7 @@ static struct sg_table *udl_map_dma_buf(struct dma_buf_attachment *attach,
 	attach->priv = udl_attach;
 
 err_unlock:
-	mutex_unlock(&dev->struct_mutex);
+	mutex_unlock(&udl->gem_lock);
 	return sgt;
 }
 
diff --git a/drivers/gpu/drm/udl/udl_drv.c b/drivers/gpu/drm/udl/udl_drv.c
index 3c45a3064726..9ef515df724b 100644
--- a/drivers/gpu/drm/udl/udl_drv.c
+++ b/drivers/gpu/drm/udl/udl_drv.c
@@ -53,7 +53,7 @@ static struct drm_driver driver = {
 	.unload = udl_driver_unload,
 
 	/* gem hooks */
-	.gem_free_object = udl_gem_free_object,
+	.gem_free_object_unlocked = udl_gem_free_object,
 	.gem_vm_ops = &udl_gem_vm_ops,
 
 	.dumb_create = udl_dumb_create,
diff --git a/drivers/gpu/drm/udl/udl_drv.h b/drivers/gpu/drm/udl/udl_drv.h
index 2a75ab80527a..55c0cc309198 100644
--- a/drivers/gpu/drm/udl/udl_drv.h
+++ b/drivers/gpu/drm/udl/udl_drv.h
@@ -54,6 +54,8 @@ struct udl_device {
 	struct usb_device *udev;
 	struct drm_crtc *crtc;
 
+	struct mutex gem_lock;
+
 	int sku_pixel_limit;
 
 	struct urb_list urbs;
diff --git a/drivers/gpu/drm/udl/udl_gem.c b/drivers/gpu/drm/udl/udl_gem.c
index dee6bd9a3dd1..9a15cce22cce 100644
--- a/drivers/gpu/drm/udl/udl_gem.c
+++ b/drivers/gpu/drm/udl/udl_gem.c
@@ -214,9 +214,10 @@ int udl_gem_mmap(struct drm_file *file, struct drm_device *dev,
 {
 	struct udl_gem_object *gobj;
 	struct drm_gem_object *obj;
+	struct udl_device *udl = dev->dev_private;
 	int ret = 0;
 
-	mutex_lock(&dev->struct_mutex);
+	mutex_lock(&udl->gem_lock);
 	obj = drm_gem_object_lookup(file, handle);
 	if (obj == NULL) {
 		ret = -ENOENT;
@@ -236,6 +237,6 @@ int udl_gem_mmap(struct drm_file *file, struct drm_device *dev,
 out:
 	drm_gem_object_put(&gobj->base);
 unlock:
-	mutex_unlock(&dev->struct_mutex);
+	mutex_unlock(&udl->gem_lock);
 	return ret;
 }
diff --git a/drivers/gpu/drm/udl/udl_main.c b/drivers/gpu/drm/udl/udl_main.c
index f1ec4528a73e..d518de8f496b 100644
--- a/drivers/gpu/drm/udl/udl_main.c
+++ b/drivers/gpu/drm/udl/udl_main.c
@@ -324,6 +324,8 @@ int udl_driver_load(struct drm_device *dev, unsigned long flags)
 	udl->ddev = dev;
 	dev->dev_private = udl;
 
+	mutex_init(&udl->gem_lock);
+
 	if (!udl_parse_vendor_descriptor(dev, udl->udev)) {
 		ret = -ENODEV;
 		DRM_ERROR("firmware not recognized. Assume incompatible device\n");
diff --git a/drivers/gpu/drm/v3d/Kconfig b/drivers/gpu/drm/v3d/Kconfig
new file mode 100644
index 000000000000..1552bf552c94
--- /dev/null
+++ b/drivers/gpu/drm/v3d/Kconfig
@@ -0,0 +1,10 @@
+config DRM_V3D
+	tristate "Broadcom V3D 3.x and newer"
+	depends on ARCH_BCM || ARCH_BCMSTB || COMPILE_TEST
+	depends on DRM
+	depends on COMMON_CLK
+	depends on MMU
+	select DRM_SCHED
+	help
+	  Choose this option if you have a system that has a Broadcom
+	  V3D 3.x or newer GPU, such as BCM7268.
diff --git a/drivers/gpu/drm/v3d/Makefile b/drivers/gpu/drm/v3d/Makefile
new file mode 100644
index 000000000000..34446e1de64f
--- /dev/null
+++ b/drivers/gpu/drm/v3d/Makefile
@@ -0,0 +1,18 @@
+# Please keep these build lists sorted!
+
+# core driver code
+v3d-y := \
+	v3d_bo.o \
+	v3d_drv.o \
+	v3d_fence.o \
+	v3d_gem.o \
+	v3d_irq.o \
+	v3d_mmu.o \
+	v3d_trace_points.o \
+	v3d_sched.o
+
+v3d-$(CONFIG_DEBUG_FS) += v3d_debugfs.o
+
+obj-$(CONFIG_DRM_V3D)  += v3d.o
+
+CFLAGS_v3d_trace_points.o := -I$(src)
diff --git a/drivers/gpu/drm/v3d/v3d_bo.c b/drivers/gpu/drm/v3d/v3d_bo.c
new file mode 100644
index 000000000000..7b1e2a549a71
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_bo.c
@@ -0,0 +1,389 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2015-2018 Broadcom */
+
+/**
+ * DOC: V3D GEM BO management support
+ *
+ * Compared to VC4 (V3D 2.x), V3D 3.3 introduces an MMU between the
+ * GPU and the bus, allowing us to use shmem objects for our storage
+ * instead of CMA.
+ *
+ * Physically contiguous objects may still be imported to V3D, but the
+ * driver doesn't allocate physically contiguous objects on its own.
+ * Display engines requiring physically contiguous allocations should
+ * look into Mesa's "renderonly" support (as used by the Mesa pl111
+ * driver) for an example of how to integrate with V3D.
+ *
+ * Long term, we should support evicting pages from the MMU when under
+ * memory pressure (thus the v3d_bo_get_pages() refcounting), but
+ * that's not a high priority since our systems tend to not have swap.
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/pfn_t.h>
+
+#include "v3d_drv.h"
+#include "uapi/drm/v3d_drm.h"
+
+/* Pins the shmem pages, fills in the .pages and .sgt fields of the BO, and maps
+ * it for DMA.
+ */
+static int
+v3d_bo_get_pages(struct v3d_bo *bo)
+{
+	struct drm_gem_object *obj = &bo->base;
+	struct drm_device *dev = obj->dev;
+	int npages = obj->size >> PAGE_SHIFT;
+	int ret = 0;
+
+	mutex_lock(&bo->lock);
+	if (bo->pages_refcount++ != 0)
+		goto unlock;
+
+	if (!obj->import_attach) {
+		bo->pages = drm_gem_get_pages(obj);
+		if (IS_ERR(bo->pages)) {
+			ret = PTR_ERR(bo->pages);
+			goto unlock;
+		}
+
+		bo->sgt = drm_prime_pages_to_sg(bo->pages, npages);
+		if (IS_ERR(bo->sgt)) {
+			ret = PTR_ERR(bo->sgt);
+			goto put_pages;
+		}
+
+		/* Map the pages for use by the GPU. */
+		dma_map_sg(dev->dev, bo->sgt->sgl,
+			   bo->sgt->nents, DMA_BIDIRECTIONAL);
+	} else {
+		bo->pages = kcalloc(npages, sizeof(*bo->pages), GFP_KERNEL);
+		if (!bo->pages)
+			goto put_pages;
+
+		drm_prime_sg_to_page_addr_arrays(bo->sgt, bo->pages,
+						 NULL, npages);
+
+		/* Note that dma-bufs come in mapped. */
+	}
+
+	mutex_unlock(&bo->lock);
+
+	return 0;
+
+put_pages:
+	drm_gem_put_pages(obj, bo->pages, true, true);
+	bo->pages = NULL;
+unlock:
+	bo->pages_refcount--;
+	mutex_unlock(&bo->lock);
+	return ret;
+}
+
+static void
+v3d_bo_put_pages(struct v3d_bo *bo)
+{
+	struct drm_gem_object *obj = &bo->base;
+
+	mutex_lock(&bo->lock);
+	if (--bo->pages_refcount == 0) {
+		if (!obj->import_attach) {
+			dma_unmap_sg(obj->dev->dev, bo->sgt->sgl,
+				     bo->sgt->nents, DMA_BIDIRECTIONAL);
+			sg_free_table(bo->sgt);
+			kfree(bo->sgt);
+			drm_gem_put_pages(obj, bo->pages, true, true);
+		} else {
+			kfree(bo->pages);
+		}
+	}
+	mutex_unlock(&bo->lock);
+}
+
+static struct v3d_bo *v3d_bo_create_struct(struct drm_device *dev,
+					   size_t unaligned_size)
+{
+	struct v3d_dev *v3d = to_v3d_dev(dev);
+	struct drm_gem_object *obj;
+	struct v3d_bo *bo;
+	size_t size = roundup(unaligned_size, PAGE_SIZE);
+	int ret;
+
+	if (size == 0)
+		return ERR_PTR(-EINVAL);
+
+	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
+	if (!bo)
+		return ERR_PTR(-ENOMEM);
+	obj = &bo->base;
+
+	INIT_LIST_HEAD(&bo->vmas);
+	INIT_LIST_HEAD(&bo->unref_head);
+	mutex_init(&bo->lock);
+
+	ret = drm_gem_object_init(dev, obj, size);
+	if (ret)
+		goto free_bo;
+
+	spin_lock(&v3d->mm_lock);
+	ret = drm_mm_insert_node_generic(&v3d->mm, &bo->node,
+					 obj->size >> PAGE_SHIFT,
+					 GMP_GRANULARITY >> PAGE_SHIFT, 0, 0);
+	spin_unlock(&v3d->mm_lock);
+	if (ret)
+		goto free_obj;
+
+	return bo;
+
+free_obj:
+	drm_gem_object_release(obj);
+free_bo:
+	kfree(bo);
+	return ERR_PTR(ret);
+}
+
+struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,
+			     size_t unaligned_size)
+{
+	struct v3d_dev *v3d = to_v3d_dev(dev);
+	struct drm_gem_object *obj;
+	struct v3d_bo *bo;
+	int ret;
+
+	bo = v3d_bo_create_struct(dev, unaligned_size);
+	if (IS_ERR(bo))
+		return bo;
+	obj = &bo->base;
+
+	bo->resv = &bo->_resv;
+	reservation_object_init(bo->resv);
+
+	ret = v3d_bo_get_pages(bo);
+	if (ret)
+		goto free_mm;
+
+	v3d_mmu_insert_ptes(bo);
+
+	mutex_lock(&v3d->bo_lock);
+	v3d->bo_stats.num_allocated++;
+	v3d->bo_stats.pages_allocated += obj->size >> PAGE_SHIFT;
+	mutex_unlock(&v3d->bo_lock);
+
+	return bo;
+
+free_mm:
+	spin_lock(&v3d->mm_lock);
+	drm_mm_remove_node(&bo->node);
+	spin_unlock(&v3d->mm_lock);
+
+	drm_gem_object_release(obj);
+	kfree(bo);
+	return ERR_PTR(ret);
+}
+
+/* Called DRM core on the last userspace/kernel unreference of the
+ * BO.
+ */
+void v3d_free_object(struct drm_gem_object *obj)
+{
+	struct v3d_dev *v3d = to_v3d_dev(obj->dev);
+	struct v3d_bo *bo = to_v3d_bo(obj);
+
+	mutex_lock(&v3d->bo_lock);
+	v3d->bo_stats.num_allocated--;
+	v3d->bo_stats.pages_allocated -= obj->size >> PAGE_SHIFT;
+	mutex_unlock(&v3d->bo_lock);
+
+	reservation_object_fini(&bo->_resv);
+
+	v3d_bo_put_pages(bo);
+
+	if (obj->import_attach)
+		drm_prime_gem_destroy(obj, bo->sgt);
+
+	v3d_mmu_remove_ptes(bo);
+	spin_lock(&v3d->mm_lock);
+	drm_mm_remove_node(&bo->node);
+	spin_unlock(&v3d->mm_lock);
+
+	mutex_destroy(&bo->lock);
+
+	drm_gem_object_release(obj);
+	kfree(bo);
+}
+
+struct reservation_object *v3d_prime_res_obj(struct drm_gem_object *obj)
+{
+	struct v3d_bo *bo = to_v3d_bo(obj);
+
+	return bo->resv;
+}
+
+static void
+v3d_set_mmap_vma_flags(struct vm_area_struct *vma)
+{
+	vma->vm_flags &= ~VM_PFNMAP;
+	vma->vm_flags |= VM_MIXEDMAP;
+	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+}
+
+int v3d_gem_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct v3d_bo *bo = to_v3d_bo(obj);
+	unsigned long pfn;
+	pgoff_t pgoff;
+	int ret;
+
+	/* We don't use vmf->pgoff since that has the fake offset: */
+	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+	pfn = page_to_pfn(bo->pages[pgoff]);
+
+	ret = vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
+
+	switch (ret) {
+	case -EAGAIN:
+	case 0:
+	case -ERESTARTSYS:
+	case -EINTR:
+	case -EBUSY:
+		/*
+		 * EBUSY is ok: this just means that another thread
+		 * already did the job.
+		 */
+		return VM_FAULT_NOPAGE;
+	case -ENOMEM:
+		return VM_FAULT_OOM;
+	default:
+		return VM_FAULT_SIGBUS;
+	}
+}
+
+int v3d_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	int ret;
+
+	ret = drm_gem_mmap(filp, vma);
+	if (ret)
+		return ret;
+
+	v3d_set_mmap_vma_flags(vma);
+
+	return ret;
+}
+
+int v3d_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+	int ret;
+
+	ret = drm_gem_mmap_obj(obj, obj->size, vma);
+	if (ret < 0)
+		return ret;
+
+	v3d_set_mmap_vma_flags(vma);
+
+	return 0;
+}
+
+struct sg_table *
+v3d_prime_get_sg_table(struct drm_gem_object *obj)
+{
+	struct v3d_bo *bo = to_v3d_bo(obj);
+	int npages = obj->size >> PAGE_SHIFT;
+
+	return drm_prime_pages_to_sg(bo->pages, npages);
+}
+
+struct drm_gem_object *
+v3d_prime_import_sg_table(struct drm_device *dev,
+			  struct dma_buf_attachment *attach,
+			  struct sg_table *sgt)
+{
+	struct drm_gem_object *obj;
+	struct v3d_bo *bo;
+
+	bo = v3d_bo_create_struct(dev, attach->dmabuf->size);
+	if (IS_ERR(bo))
+		return ERR_CAST(bo);
+	obj = &bo->base;
+
+	bo->resv = attach->dmabuf->resv;
+
+	bo->sgt = sgt;
+	v3d_bo_get_pages(bo);
+
+	v3d_mmu_insert_ptes(bo);
+
+	return obj;
+}
+
+int v3d_create_bo_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv)
+{
+	struct drm_v3d_create_bo *args = data;
+	struct v3d_bo *bo = NULL;
+	int ret;
+
+	if (args->flags != 0) {
+		DRM_INFO("unknown create_bo flags: %d\n", args->flags);
+		return -EINVAL;
+	}
+
+	bo = v3d_bo_create(dev, file_priv, PAGE_ALIGN(args->size));
+	if (IS_ERR(bo))
+		return PTR_ERR(bo);
+
+	args->offset = bo->node.start << PAGE_SHIFT;
+
+	ret = drm_gem_handle_create(file_priv, &bo->base, &args->handle);
+	drm_gem_object_put_unlocked(&bo->base);
+
+	return ret;
+}
+
+int v3d_mmap_bo_ioctl(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv)
+{
+	struct drm_v3d_mmap_bo *args = data;
+	struct drm_gem_object *gem_obj;
+	int ret;
+
+	if (args->flags != 0) {
+		DRM_INFO("unknown mmap_bo flags: %d\n", args->flags);
+		return -EINVAL;
+	}
+
+	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
+	if (!gem_obj) {
+		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
+		return -ENOENT;
+	}
+
+	ret = drm_gem_create_mmap_offset(gem_obj);
+	if (ret == 0)
+		args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
+	drm_gem_object_put_unlocked(gem_obj);
+
+	return ret;
+}
+
+int v3d_get_bo_offset_ioctl(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv)
+{
+	struct drm_v3d_get_bo_offset *args = data;
+	struct drm_gem_object *gem_obj;
+	struct v3d_bo *bo;
+
+	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
+	if (!gem_obj) {
+		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
+		return -ENOENT;
+	}
+	bo = to_v3d_bo(gem_obj);
+
+	args->offset = bo->node.start << PAGE_SHIFT;
+
+	drm_gem_object_put_unlocked(gem_obj);
+	return 0;
+}
diff --git a/drivers/gpu/drm/v3d/v3d_debugfs.c b/drivers/gpu/drm/v3d/v3d_debugfs.c
new file mode 100644
index 000000000000..4db62c545748
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_debugfs.c
@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2014-2018 Broadcom */
+
+#include <linux/circ_buf.h>
+#include <linux/ctype.h>
+#include <linux/debugfs.h>
+#include <linux/pm_runtime.h>
+#include <linux/seq_file.h>
+#include <drm/drmP.h>
+
+#include "v3d_drv.h"
+#include "v3d_regs.h"
+
+#define REGDEF(reg) { reg, #reg }
+struct v3d_reg_def {
+	u32 reg;
+	const char *name;
+};
+
+static const struct v3d_reg_def v3d_hub_reg_defs[] = {
+	REGDEF(V3D_HUB_AXICFG),
+	REGDEF(V3D_HUB_UIFCFG),
+	REGDEF(V3D_HUB_IDENT0),
+	REGDEF(V3D_HUB_IDENT1),
+	REGDEF(V3D_HUB_IDENT2),
+	REGDEF(V3D_HUB_IDENT3),
+	REGDEF(V3D_HUB_INT_STS),
+	REGDEF(V3D_HUB_INT_MSK_STS),
+};
+
+static const struct v3d_reg_def v3d_gca_reg_defs[] = {
+	REGDEF(V3D_GCA_SAFE_SHUTDOWN),
+	REGDEF(V3D_GCA_SAFE_SHUTDOWN_ACK),
+};
+
+static const struct v3d_reg_def v3d_core_reg_defs[] = {
+	REGDEF(V3D_CTL_IDENT0),
+	REGDEF(V3D_CTL_IDENT1),
+	REGDEF(V3D_CTL_IDENT2),
+	REGDEF(V3D_CTL_MISCCFG),
+	REGDEF(V3D_CTL_INT_STS),
+	REGDEF(V3D_CTL_INT_MSK_STS),
+	REGDEF(V3D_CLE_CT0CS),
+	REGDEF(V3D_CLE_CT0CA),
+	REGDEF(V3D_CLE_CT0EA),
+	REGDEF(V3D_CLE_CT1CS),
+	REGDEF(V3D_CLE_CT1CA),
+	REGDEF(V3D_CLE_CT1EA),
+
+	REGDEF(V3D_PTB_BPCA),
+	REGDEF(V3D_PTB_BPCS),
+
+	REGDEF(V3D_MMU_CTL),
+	REGDEF(V3D_MMU_VIO_ADDR),
+
+	REGDEF(V3D_GMP_STATUS),
+	REGDEF(V3D_GMP_CFG),
+	REGDEF(V3D_GMP_VIO_ADDR),
+};
+
+static int v3d_v3d_debugfs_regs(struct seq_file *m, void *unused)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct v3d_dev *v3d = to_v3d_dev(dev);
+	int i, core;
+
+	for (i = 0; i < ARRAY_SIZE(v3d_hub_reg_defs); i++) {
+		seq_printf(m, "%s (0x%04x): 0x%08x\n",
+			   v3d_hub_reg_defs[i].name, v3d_hub_reg_defs[i].reg,
+			   V3D_READ(v3d_hub_reg_defs[i].reg));
+	}
+
+	for (i = 0; i < ARRAY_SIZE(v3d_gca_reg_defs); i++) {
+		seq_printf(m, "%s (0x%04x): 0x%08x\n",
+			   v3d_gca_reg_defs[i].name, v3d_gca_reg_defs[i].reg,
+			   V3D_GCA_READ(v3d_gca_reg_defs[i].reg));
+	}
+
+	for (core = 0; core < v3d->cores; core++) {
+		for (i = 0; i < ARRAY_SIZE(v3d_core_reg_defs); i++) {
+			seq_printf(m, "core %d %s (0x%04x): 0x%08x\n",
+				   core,
+				   v3d_core_reg_defs[i].name,
+				   v3d_core_reg_defs[i].reg,
+				   V3D_CORE_READ(core,
+						 v3d_core_reg_defs[i].reg));
+		}
+	}
+
+	return 0;
+}
+
+static int v3d_v3d_debugfs_ident(struct seq_file *m, void *unused)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct v3d_dev *v3d = to_v3d_dev(dev);
+	u32 ident0, ident1, ident2, ident3, cores;
+	int ret, core;
+
+	ret = pm_runtime_get_sync(v3d->dev);
+	if (ret < 0)
+		return ret;
+
+	ident0 = V3D_READ(V3D_HUB_IDENT0);
+	ident1 = V3D_READ(V3D_HUB_IDENT1);
+	ident2 = V3D_READ(V3D_HUB_IDENT2);
+	ident3 = V3D_READ(V3D_HUB_IDENT3);
+	cores = V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_NCORES);
+
+	seq_printf(m, "Revision:   %d.%d.%d.%d\n",
+		   V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_TVER),
+		   V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_REV),
+		   V3D_GET_FIELD(ident3, V3D_HUB_IDENT3_IPREV),
+		   V3D_GET_FIELD(ident3, V3D_HUB_IDENT3_IPIDX));
+	seq_printf(m, "MMU:        %s\n",
+		   (ident2 & V3D_HUB_IDENT2_WITH_MMU) ? "yes" : "no");
+	seq_printf(m, "TFU:        %s\n",
+		   (ident1 & V3D_HUB_IDENT1_WITH_TFU) ? "yes" : "no");
+	seq_printf(m, "TSY:        %s\n",
+		   (ident1 & V3D_HUB_IDENT1_WITH_TSY) ? "yes" : "no");
+	seq_printf(m, "MSO:        %s\n",
+		   (ident1 & V3D_HUB_IDENT1_WITH_MSO) ? "yes" : "no");
+	seq_printf(m, "L3C:        %s (%dkb)\n",
+		   (ident1 & V3D_HUB_IDENT1_WITH_L3C) ? "yes" : "no",
+		   V3D_GET_FIELD(ident2, V3D_HUB_IDENT2_L3C_NKB));
+
+	for (core = 0; core < cores; core++) {
+		u32 misccfg;
+		u32 nslc, ntmu, qups;
+
+		ident0 = V3D_CORE_READ(core, V3D_CTL_IDENT0);
+		ident1 = V3D_CORE_READ(core, V3D_CTL_IDENT1);
+		ident2 = V3D_CORE_READ(core, V3D_CTL_IDENT2);
+		misccfg = V3D_CORE_READ(core, V3D_CTL_MISCCFG);
+
+		nslc = V3D_GET_FIELD(ident1, V3D_IDENT1_NSLC);
+		ntmu = V3D_GET_FIELD(ident1, V3D_IDENT1_NTMU);
+		qups = V3D_GET_FIELD(ident1, V3D_IDENT1_QUPS);
+
+		seq_printf(m, "Core %d:\n", core);
+		seq_printf(m, "  Revision:     %d.%d\n",
+			   V3D_GET_FIELD(ident0, V3D_IDENT0_VER),
+			   V3D_GET_FIELD(ident1, V3D_IDENT1_REV));
+		seq_printf(m, "  Slices:       %d\n", nslc);
+		seq_printf(m, "  TMUs:         %d\n", nslc * ntmu);
+		seq_printf(m, "  QPUs:         %d\n", nslc * qups);
+		seq_printf(m, "  Semaphores:   %d\n",
+			   V3D_GET_FIELD(ident1, V3D_IDENT1_NSEM));
+		seq_printf(m, "  BCG int:      %d\n",
+			   (ident2 & V3D_IDENT2_BCG_INT) != 0);
+		seq_printf(m, "  Override TMU: %d\n",
+			   (misccfg & V3D_MISCCFG_OVRTMUOUT) != 0);
+	}
+
+	pm_runtime_mark_last_busy(v3d->dev);
+	pm_runtime_put_autosuspend(v3d->dev);
+
+	return 0;
+}
+
+static int v3d_debugfs_bo_stats(struct seq_file *m, void *unused)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct v3d_dev *v3d = to_v3d_dev(dev);
+
+	mutex_lock(&v3d->bo_lock);
+	seq_printf(m, "allocated bos:          %d\n",
+		   v3d->bo_stats.num_allocated);
+	seq_printf(m, "allocated bo size (kb): %ld\n",
+		   (long)v3d->bo_stats.pages_allocated << (PAGE_SHIFT - 10));
+	mutex_unlock(&v3d->bo_lock);
+
+	return 0;
+}
+
+static const struct drm_info_list v3d_debugfs_list[] = {
+	{"v3d_ident", v3d_v3d_debugfs_ident, 0},
+	{"v3d_regs", v3d_v3d_debugfs_regs, 0},
+	{"bo_stats", v3d_debugfs_bo_stats, 0},
+};
+
+int
+v3d_debugfs_init(struct drm_minor *minor)
+{
+	return drm_debugfs_create_files(v3d_debugfs_list,
+					ARRAY_SIZE(v3d_debugfs_list),
+					minor->debugfs_root, minor);
+}
diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
new file mode 100644
index 000000000000..cdb582043b4f
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -0,0 +1,371 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2014-2018 Broadcom */
+
+/**
+ * DOC: Broadcom V3D Graphics Driver
+ *
+ * This driver supports the Broadcom V3D 3.3 and 4.1 OpenGL ES GPUs.
+ * For V3D 2.x support, see the VC4 driver.
+ *
+ * Currently only single-core rendering using the binner and renderer
+ * is supported.  The TFU (texture formatting unit) and V3D 4.x's CSD
+ * (compute shader dispatch) are not yet supported.
+ */
+
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_helper.h>
+
+#include "uapi/drm/v3d_drm.h"
+#include "v3d_drv.h"
+#include "v3d_regs.h"
+
+#define DRIVER_NAME "v3d"
+#define DRIVER_DESC "Broadcom V3D graphics"
+#define DRIVER_DATE "20180419"
+#define DRIVER_MAJOR 1
+#define DRIVER_MINOR 0
+#define DRIVER_PATCHLEVEL 0
+
+#ifdef CONFIG_PM
+static int v3d_runtime_suspend(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct v3d_dev *v3d = to_v3d_dev(drm);
+
+	v3d_irq_disable(v3d);
+
+	clk_disable_unprepare(v3d->clk);
+
+	return 0;
+}
+
+static int v3d_runtime_resume(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct v3d_dev *v3d = to_v3d_dev(drm);
+	int ret;
+
+	ret = clk_prepare_enable(v3d->clk);
+	if (ret != 0)
+		return ret;
+
+	/* XXX: VPM base */
+
+	v3d_mmu_set_page_table(v3d);
+	v3d_irq_enable(v3d);
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops v3d_v3d_pm_ops = {
+	SET_RUNTIME_PM_OPS(v3d_runtime_suspend, v3d_runtime_resume, NULL)
+};
+
+static int v3d_get_param_ioctl(struct drm_device *dev, void *data,
+			       struct drm_file *file_priv)
+{
+	struct v3d_dev *v3d = to_v3d_dev(dev);
+	struct drm_v3d_get_param *args = data;
+	int ret;
+	static const u32 reg_map[] = {
+		[DRM_V3D_PARAM_V3D_UIFCFG] = V3D_HUB_UIFCFG,
+		[DRM_V3D_PARAM_V3D_HUB_IDENT1] = V3D_HUB_IDENT1,
+		[DRM_V3D_PARAM_V3D_HUB_IDENT2] = V3D_HUB_IDENT2,
+		[DRM_V3D_PARAM_V3D_HUB_IDENT3] = V3D_HUB_IDENT3,
+		[DRM_V3D_PARAM_V3D_CORE0_IDENT0] = V3D_CTL_IDENT0,
+		[DRM_V3D_PARAM_V3D_CORE0_IDENT1] = V3D_CTL_IDENT1,
+		[DRM_V3D_PARAM_V3D_CORE0_IDENT2] = V3D_CTL_IDENT2,
+	};
+
+	if (args->pad != 0)
+		return -EINVAL;
+
+	/* Note that DRM_V3D_PARAM_V3D_CORE0_IDENT0 is 0, so we need
+	 * to explicitly allow it in the "the register in our
+	 * parameter map" check.
+	 */
+	if (args->param < ARRAY_SIZE(reg_map) &&
+	    (reg_map[args->param] ||
+	     args->param == DRM_V3D_PARAM_V3D_CORE0_IDENT0)) {
+		u32 offset = reg_map[args->param];
+
+		if (args->value != 0)
+			return -EINVAL;
+
+		ret = pm_runtime_get_sync(v3d->dev);
+		if (args->param >= DRM_V3D_PARAM_V3D_CORE0_IDENT0 &&
+		    args->param <= DRM_V3D_PARAM_V3D_CORE0_IDENT2) {
+			args->value = V3D_CORE_READ(0, offset);
+		} else {
+			args->value = V3D_READ(offset);
+		}
+		pm_runtime_mark_last_busy(v3d->dev);
+		pm_runtime_put_autosuspend(v3d->dev);
+		return 0;
+	}
+
+	/* Any params that aren't just register reads would go here. */
+
+	DRM_DEBUG("Unknown parameter %d\n", args->param);
+	return -EINVAL;
+}
+
+static int
+v3d_open(struct drm_device *dev, struct drm_file *file)
+{
+	struct v3d_dev *v3d = to_v3d_dev(dev);
+	struct v3d_file_priv *v3d_priv;
+	int i;
+
+	v3d_priv = kzalloc(sizeof(*v3d_priv), GFP_KERNEL);
+	if (!v3d_priv)
+		return -ENOMEM;
+
+	v3d_priv->v3d = v3d;
+
+	for (i = 0; i < V3D_MAX_QUEUES; i++) {
+		drm_sched_entity_init(&v3d->queue[i].sched,
+				      &v3d_priv->sched_entity[i],
+				      &v3d->queue[i].sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL],
+				      NULL);
+	}
+
+	file->driver_priv = v3d_priv;
+
+	return 0;
+}
+
+static void
+v3d_postclose(struct drm_device *dev, struct drm_file *file)
+{
+	struct v3d_dev *v3d = to_v3d_dev(dev);
+	struct v3d_file_priv *v3d_priv = file->driver_priv;
+	enum v3d_queue q;
+
+	for (q = 0; q < V3D_MAX_QUEUES; q++) {
+		drm_sched_entity_fini(&v3d->queue[q].sched,
+				      &v3d_priv->sched_entity[q]);
+	}
+
+	kfree(v3d_priv);
+}
+
+static const struct file_operations v3d_drm_fops = {
+	.owner = THIS_MODULE,
+	.open = drm_open,
+	.release = drm_release,
+	.unlocked_ioctl = drm_ioctl,
+	.mmap = v3d_mmap,
+	.poll = drm_poll,
+	.read = drm_read,
+	.compat_ioctl = drm_compat_ioctl,
+	.llseek = noop_llseek,
+};
+
+/* DRM_AUTH is required on SUBMIT_CL for now, while we don't have GMP
+ * protection between clients.  Note that render nodes would be be
+ * able to submit CLs that could access BOs from clients authenticated
+ * with the master node.
+ */
+static const struct drm_ioctl_desc v3d_drm_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(V3D_SUBMIT_CL, v3d_submit_cl_ioctl, DRM_RENDER_ALLOW | DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(V3D_WAIT_BO, v3d_wait_bo_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(V3D_CREATE_BO, v3d_create_bo_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(V3D_MMAP_BO, v3d_mmap_bo_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(V3D_GET_PARAM, v3d_get_param_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(V3D_GET_BO_OFFSET, v3d_get_bo_offset_ioctl, DRM_RENDER_ALLOW),
+};
+
+static const struct vm_operations_struct v3d_vm_ops = {
+	.fault = v3d_gem_fault,
+	.open = drm_gem_vm_open,
+	.close = drm_gem_vm_close,
+};
+
+static struct drm_driver v3d_drm_driver = {
+	.driver_features = (DRIVER_GEM |
+			    DRIVER_RENDER |
+			    DRIVER_PRIME |
+			    DRIVER_SYNCOBJ),
+
+	.open = v3d_open,
+	.postclose = v3d_postclose,
+
+#if defined(CONFIG_DEBUG_FS)
+	.debugfs_init = v3d_debugfs_init,
+#endif
+
+	.gem_free_object_unlocked = v3d_free_object,
+	.gem_vm_ops = &v3d_vm_ops,
+
+	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+	.gem_prime_import = drm_gem_prime_import,
+	.gem_prime_export = drm_gem_prime_export,
+	.gem_prime_res_obj = v3d_prime_res_obj,
+	.gem_prime_get_sg_table	= v3d_prime_get_sg_table,
+	.gem_prime_import_sg_table = v3d_prime_import_sg_table,
+	.gem_prime_mmap = v3d_prime_mmap,
+
+	.ioctls = v3d_drm_ioctls,
+	.num_ioctls = ARRAY_SIZE(v3d_drm_ioctls),
+	.fops = &v3d_drm_fops,
+
+	.name = DRIVER_NAME,
+	.desc = DRIVER_DESC,
+	.date = DRIVER_DATE,
+	.major = DRIVER_MAJOR,
+	.minor = DRIVER_MINOR,
+	.patchlevel = DRIVER_PATCHLEVEL,
+};
+
+static const struct of_device_id v3d_of_match[] = {
+	{ .compatible = "brcm,7268-v3d" },
+	{ .compatible = "brcm,7278-v3d" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, v3d_of_match);
+
+static int
+map_regs(struct v3d_dev *v3d, void __iomem **regs, const char *name)
+{
+	struct resource *res =
+		platform_get_resource_byname(v3d->pdev, IORESOURCE_MEM, name);
+
+	*regs = devm_ioremap_resource(v3d->dev, res);
+	return PTR_ERR_OR_ZERO(*regs);
+}
+
+static int v3d_platform_drm_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct drm_device *drm;
+	struct v3d_dev *v3d;
+	int ret;
+	u32 ident1;
+
+	dev->coherent_dma_mask = DMA_BIT_MASK(36);
+
+	v3d = kzalloc(sizeof(*v3d), GFP_KERNEL);
+	if (!v3d)
+		return -ENOMEM;
+	v3d->dev = dev;
+	v3d->pdev = pdev;
+	drm = &v3d->drm;
+
+	ret = map_regs(v3d, &v3d->bridge_regs, "bridge");
+	if (ret)
+		goto dev_free;
+
+	ret = map_regs(v3d, &v3d->hub_regs, "hub");
+	if (ret)
+		goto dev_free;
+
+	ret = map_regs(v3d, &v3d->core_regs[0], "core0");
+	if (ret)
+		goto dev_free;
+
+	ident1 = V3D_READ(V3D_HUB_IDENT1);
+	v3d->ver = (V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_TVER) * 10 +
+		    V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_REV));
+	v3d->cores = V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_NCORES);
+	WARN_ON(v3d->cores > 1); /* multicore not yet implemented */
+
+	if (v3d->ver < 41) {
+		ret = map_regs(v3d, &v3d->gca_regs, "gca");
+		if (ret)
+			goto dev_free;
+	}
+
+	v3d->mmu_scratch = dma_alloc_wc(dev, 4096, &v3d->mmu_scratch_paddr,
+					GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
+	if (!v3d->mmu_scratch) {
+		dev_err(dev, "Failed to allocate MMU scratch page\n");
+		ret = -ENOMEM;
+		goto dev_free;
+	}
+
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_autosuspend_delay(dev, 50);
+	pm_runtime_enable(dev);
+
+	ret = drm_dev_init(&v3d->drm, &v3d_drm_driver, dev);
+	if (ret)
+		goto dma_free;
+
+	platform_set_drvdata(pdev, drm);
+	drm->dev_private = v3d;
+
+	ret = v3d_gem_init(drm);
+	if (ret)
+		goto dev_destroy;
+
+	v3d_irq_init(v3d);
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		goto gem_destroy;
+
+	return 0;
+
+gem_destroy:
+	v3d_gem_destroy(drm);
+dev_destroy:
+	drm_dev_put(drm);
+dma_free:
+	dma_free_wc(dev, 4096, v3d->mmu_scratch, v3d->mmu_scratch_paddr);
+dev_free:
+	kfree(v3d);
+	return ret;
+}
+
+static int v3d_platform_drm_remove(struct platform_device *pdev)
+{
+	struct drm_device *drm = platform_get_drvdata(pdev);
+	struct v3d_dev *v3d = to_v3d_dev(drm);
+
+	drm_dev_unregister(drm);
+
+	v3d_gem_destroy(drm);
+
+	drm_dev_put(drm);
+
+	dma_free_wc(v3d->dev, 4096, v3d->mmu_scratch, v3d->mmu_scratch_paddr);
+
+	return 0;
+}
+
+static struct platform_driver v3d_platform_driver = {
+	.probe		= v3d_platform_drm_probe,
+	.remove		= v3d_platform_drm_remove,
+	.driver		= {
+		.name	= "v3d",
+		.of_match_table = v3d_of_match,
+	},
+};
+
+static int __init v3d_drm_register(void)
+{
+	return platform_driver_register(&v3d_platform_driver);
+}
+
+static void __exit v3d_drm_unregister(void)
+{
+	platform_driver_unregister(&v3d_platform_driver);
+}
+
+module_init(v3d_drm_register);
+module_exit(v3d_drm_unregister);
+
+MODULE_ALIAS("platform:v3d-drm");
+MODULE_DESCRIPTION("Broadcom V3D DRM Driver");
+MODULE_AUTHOR("Eric Anholt <eric@anholt.net>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
new file mode 100644
index 000000000000..a043ac3aae98
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -0,0 +1,294 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2015-2018 Broadcom */
+
+#include <linux/reservation.h>
+#include <drm/drmP.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_gem.h>
+#include <drm/gpu_scheduler.h>
+
+#define GMP_GRANULARITY (128 * 1024)
+
+/* Enum for each of the V3D queues.  We maintain various queue
+ * tracking as an array because at some point we'll want to support
+ * the TFU (texture formatting unit) as another queue.
+ */
+enum v3d_queue {
+	V3D_BIN,
+	V3D_RENDER,
+};
+
+#define V3D_MAX_QUEUES (V3D_RENDER + 1)
+
+struct v3d_queue_state {
+	struct drm_gpu_scheduler sched;
+
+	u64 fence_context;
+	u64 emit_seqno;
+	u64 finished_seqno;
+};
+
+struct v3d_dev {
+	struct drm_device drm;
+
+	/* Short representation (e.g. 33, 41) of the V3D tech version
+	 * and revision.
+	 */
+	int ver;
+
+	struct device *dev;
+	struct platform_device *pdev;
+	void __iomem *hub_regs;
+	void __iomem *core_regs[3];
+	void __iomem *bridge_regs;
+	void __iomem *gca_regs;
+	struct clk *clk;
+
+	/* Virtual and DMA addresses of the single shared page table. */
+	volatile u32 *pt;
+	dma_addr_t pt_paddr;
+
+	/* Virtual and DMA addresses of the MMU's scratch page.  When
+	 * a read or write is invalid in the MMU, it will be
+	 * redirected here.
+	 */
+	void *mmu_scratch;
+	dma_addr_t mmu_scratch_paddr;
+
+	/* Number of V3D cores. */
+	u32 cores;
+
+	/* Allocator managing the address space.  All units are in
+	 * number of pages.
+	 */
+	struct drm_mm mm;
+	spinlock_t mm_lock;
+
+	struct work_struct overflow_mem_work;
+
+	struct v3d_exec_info *bin_job;
+	struct v3d_exec_info *render_job;
+
+	struct v3d_queue_state queue[V3D_MAX_QUEUES];
+
+	/* Spinlock used to synchronize the overflow memory
+	 * management against bin job submission.
+	 */
+	spinlock_t job_lock;
+
+	/* Protects bo_stats */
+	struct mutex bo_lock;
+
+	/* Lock taken when resetting the GPU, to keep multiple
+	 * processes from trying to park the scheduler threads and
+	 * reset at once.
+	 */
+	struct mutex reset_lock;
+
+	struct {
+		u32 num_allocated;
+		u32 pages_allocated;
+	} bo_stats;
+};
+
+static inline struct v3d_dev *
+to_v3d_dev(struct drm_device *dev)
+{
+	return (struct v3d_dev *)dev->dev_private;
+}
+
+/* The per-fd struct, which tracks the MMU mappings. */
+struct v3d_file_priv {
+	struct v3d_dev *v3d;
+
+	struct drm_sched_entity sched_entity[V3D_MAX_QUEUES];
+};
+
+/* Tracks a mapping of a BO into a per-fd address space */
+struct v3d_vma {
+	struct v3d_page_table *pt;
+	struct list_head list; /* entry in v3d_bo.vmas */
+};
+
+struct v3d_bo {
+	struct drm_gem_object base;
+
+	struct mutex lock;
+
+	struct drm_mm_node node;
+
+	u32 pages_refcount;
+	struct page **pages;
+	struct sg_table *sgt;
+	void *vaddr;
+
+	struct list_head vmas;    /* list of v3d_vma */
+
+	/* List entry for the BO's position in
+	 * v3d_exec_info->unref_list
+	 */
+	struct list_head unref_head;
+
+	/* normally (resv == &_resv) except for imported bo's */
+	struct reservation_object *resv;
+	struct reservation_object _resv;
+};
+
+static inline struct v3d_bo *
+to_v3d_bo(struct drm_gem_object *bo)
+{
+	return (struct v3d_bo *)bo;
+}
+
+struct v3d_fence {
+	struct dma_fence base;
+	struct drm_device *dev;
+	/* v3d seqno for signaled() test */
+	u64 seqno;
+	enum v3d_queue queue;
+};
+
+static inline struct v3d_fence *
+to_v3d_fence(struct dma_fence *fence)
+{
+	return (struct v3d_fence *)fence;
+}
+
+#define V3D_READ(offset) readl(v3d->hub_regs + offset)
+#define V3D_WRITE(offset, val) writel(val, v3d->hub_regs + offset)
+
+#define V3D_BRIDGE_READ(offset) readl(v3d->bridge_regs + offset)
+#define V3D_BRIDGE_WRITE(offset, val) writel(val, v3d->bridge_regs + offset)
+
+#define V3D_GCA_READ(offset) readl(v3d->gca_regs + offset)
+#define V3D_GCA_WRITE(offset, val) writel(val, v3d->gca_regs + offset)
+
+#define V3D_CORE_READ(core, offset) readl(v3d->core_regs[core] + offset)
+#define V3D_CORE_WRITE(core, offset, val) writel(val, v3d->core_regs[core] + offset)
+
+struct v3d_job {
+	struct drm_sched_job base;
+
+	struct v3d_exec_info *exec;
+
+	/* An optional fence userspace can pass in for the job to depend on. */
+	struct dma_fence *in_fence;
+
+	/* v3d fence to be signaled by IRQ handler when the job is complete. */
+	struct dma_fence *done_fence;
+
+	/* GPU virtual addresses of the start/end of the CL job. */
+	u32 start, end;
+};
+
+struct v3d_exec_info {
+	struct v3d_dev *v3d;
+
+	struct v3d_job bin, render;
+
+	/* Fence for when the scheduler considers the binner to be
+	 * done, for render to depend on.
+	 */
+	struct dma_fence *bin_done_fence;
+
+	struct kref refcount;
+
+	/* This is the array of BOs that were looked up at the start of exec. */
+	struct v3d_bo **bo;
+	u32 bo_count;
+
+	/* List of overflow BOs used in the job that need to be
+	 * released once the job is complete.
+	 */
+	struct list_head unref_list;
+
+	/* Submitted tile memory allocation start/size, tile state. */
+	u32 qma, qms, qts;
+};
+
+/**
+ * _wait_for - magic (register) wait macro
+ *
+ * Does the right thing for modeset paths when run under kdgb or similar atomic
+ * contexts. Note that it's important that we check the condition again after
+ * having timed out, since the timeout could be due to preemption or similar and
+ * we've never had a chance to check the condition before the timeout.
+ */
+#define wait_for(COND, MS) ({ \
+	unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1;	\
+	int ret__ = 0;							\
+	while (!(COND)) {						\
+		if (time_after(jiffies, timeout__)) {			\
+			if (!(COND))					\
+				ret__ = -ETIMEDOUT;			\
+			break;						\
+		}							\
+		msleep(1);					\
+	}								\
+	ret__;								\
+})
+
+static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
+{
+	/* nsecs_to_jiffies64() does not guard against overflow */
+	if (NSEC_PER_SEC % HZ &&
+	    div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
+		return MAX_JIFFY_OFFSET;
+
+	return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
+}
+
+/* v3d_bo.c */
+void v3d_free_object(struct drm_gem_object *gem_obj);
+struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,
+			     size_t size);
+int v3d_create_bo_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
+int v3d_mmap_bo_ioctl(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv);
+int v3d_get_bo_offset_ioctl(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv);
+int v3d_gem_fault(struct vm_fault *vmf);
+int v3d_mmap(struct file *filp, struct vm_area_struct *vma);
+struct reservation_object *v3d_prime_res_obj(struct drm_gem_object *obj);
+int v3d_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
+struct sg_table *v3d_prime_get_sg_table(struct drm_gem_object *obj);
+struct drm_gem_object *v3d_prime_import_sg_table(struct drm_device *dev,
+						 struct dma_buf_attachment *attach,
+						 struct sg_table *sgt);
+
+/* v3d_debugfs.c */
+int v3d_debugfs_init(struct drm_minor *minor);
+
+/* v3d_fence.c */
+extern const struct dma_fence_ops v3d_fence_ops;
+struct dma_fence *v3d_fence_create(struct v3d_dev *v3d, enum v3d_queue queue);
+
+/* v3d_gem.c */
+int v3d_gem_init(struct drm_device *dev);
+void v3d_gem_destroy(struct drm_device *dev);
+int v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
+int v3d_wait_bo_ioctl(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv);
+void v3d_exec_put(struct v3d_exec_info *exec);
+void v3d_reset(struct v3d_dev *v3d);
+void v3d_invalidate_caches(struct v3d_dev *v3d);
+void v3d_flush_caches(struct v3d_dev *v3d);
+
+/* v3d_irq.c */
+void v3d_irq_init(struct v3d_dev *v3d);
+void v3d_irq_enable(struct v3d_dev *v3d);
+void v3d_irq_disable(struct v3d_dev *v3d);
+void v3d_irq_reset(struct v3d_dev *v3d);
+
+/* v3d_mmu.c */
+int v3d_mmu_get_offset(struct drm_file *file_priv, struct v3d_bo *bo,
+		       u32 *offset);
+int v3d_mmu_set_page_table(struct v3d_dev *v3d);
+void v3d_mmu_insert_ptes(struct v3d_bo *bo);
+void v3d_mmu_remove_ptes(struct v3d_bo *bo);
+
+/* v3d_sched.c */
+int v3d_sched_init(struct v3d_dev *v3d);
+void v3d_sched_fini(struct v3d_dev *v3d);
diff --git a/drivers/gpu/drm/v3d/v3d_fence.c b/drivers/gpu/drm/v3d/v3d_fence.c
new file mode 100644
index 000000000000..087d49c8cb12
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_fence.c
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2017-2018 Broadcom */
+
+#include "v3d_drv.h"
+
+struct dma_fence *v3d_fence_create(struct v3d_dev *v3d, enum v3d_queue queue)
+{
+	struct v3d_fence *fence;
+
+	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
+	if (!fence)
+		return ERR_PTR(-ENOMEM);
+
+	fence->dev = &v3d->drm;
+	fence->queue = queue;
+	fence->seqno = ++v3d->queue[queue].emit_seqno;
+	dma_fence_init(&fence->base, &v3d_fence_ops, &v3d->job_lock,
+		       v3d->queue[queue].fence_context, fence->seqno);
+
+	return &fence->base;
+}
+
+static const char *v3d_fence_get_driver_name(struct dma_fence *fence)
+{
+	return "v3d";
+}
+
+static const char *v3d_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct v3d_fence *f = to_v3d_fence(fence);
+
+	if (f->queue == V3D_BIN)
+		return "v3d-bin";
+	else
+		return "v3d-render";
+}
+
+static bool v3d_fence_enable_signaling(struct dma_fence *fence)
+{
+	return true;
+}
+
+static bool v3d_fence_signaled(struct dma_fence *fence)
+{
+	struct v3d_fence *f = to_v3d_fence(fence);
+	struct v3d_dev *v3d = to_v3d_dev(f->dev);
+
+	return v3d->queue[f->queue].finished_seqno >= f->seqno;
+}
+
+const struct dma_fence_ops v3d_fence_ops = {
+	.get_driver_name = v3d_fence_get_driver_name,
+	.get_timeline_name = v3d_fence_get_timeline_name,
+	.enable_signaling = v3d_fence_enable_signaling,
+	.signaled = v3d_fence_signaled,
+	.wait = dma_fence_default_wait,
+	.release = dma_fence_free,
+};
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
new file mode 100644
index 000000000000..b513f9189caf
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -0,0 +1,668 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2014-2018 Broadcom */
+
+#include <drm/drmP.h>
+#include <drm/drm_syncobj.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/sched/signal.h>
+
+#include "uapi/drm/v3d_drm.h"
+#include "v3d_drv.h"
+#include "v3d_regs.h"
+#include "v3d_trace.h"
+
+static void
+v3d_init_core(struct v3d_dev *v3d, int core)
+{
+	/* Set OVRTMUOUT, which means that the texture sampler uniform
+	 * configuration's tmu output type field is used, instead of
+	 * using the hardware default behavior based on the texture
+	 * type.  If you want the default behavior, you can still put
+	 * "2" in the indirect texture state's output_type field.
+	 */
+	V3D_CORE_WRITE(core, V3D_CTL_MISCCFG, V3D_MISCCFG_OVRTMUOUT);
+
+	/* Whenever we flush the L2T cache, we always want to flush
+	 * the whole thing.
+	 */
+	V3D_CORE_WRITE(core, V3D_CTL_L2TFLSTA, 0);
+	V3D_CORE_WRITE(core, V3D_CTL_L2TFLEND, ~0);
+}
+
+/* Sets invariant state for the HW. */
+static void
+v3d_init_hw_state(struct v3d_dev *v3d)
+{
+	v3d_init_core(v3d, 0);
+}
+
+static void
+v3d_idle_axi(struct v3d_dev *v3d, int core)
+{
+	V3D_CORE_WRITE(core, V3D_GMP_CFG, V3D_GMP_CFG_STOP_REQ);
+
+	if (wait_for((V3D_CORE_READ(core, V3D_GMP_STATUS) &
+		      (V3D_GMP_STATUS_RD_COUNT_MASK |
+		       V3D_GMP_STATUS_WR_COUNT_MASK |
+		       V3D_GMP_STATUS_CFG_BUSY)) == 0, 100)) {
+		DRM_ERROR("Failed to wait for safe GMP shutdown\n");
+	}
+}
+
+static void
+v3d_idle_gca(struct v3d_dev *v3d)
+{
+	if (v3d->ver >= 41)
+		return;
+
+	V3D_GCA_WRITE(V3D_GCA_SAFE_SHUTDOWN, V3D_GCA_SAFE_SHUTDOWN_EN);
+
+	if (wait_for((V3D_GCA_READ(V3D_GCA_SAFE_SHUTDOWN_ACK) &
+		      V3D_GCA_SAFE_SHUTDOWN_ACK_ACKED) ==
+		     V3D_GCA_SAFE_SHUTDOWN_ACK_ACKED, 100)) {
+		DRM_ERROR("Failed to wait for safe GCA shutdown\n");
+	}
+}
+
+static void
+v3d_reset_v3d(struct v3d_dev *v3d)
+{
+	int version = V3D_BRIDGE_READ(V3D_TOP_GR_BRIDGE_REVISION);
+
+	if (V3D_GET_FIELD(version, V3D_TOP_GR_BRIDGE_MAJOR) == 2) {
+		V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_0,
+				 V3D_TOP_GR_BRIDGE_SW_INIT_0_V3D_CLK_108_SW_INIT);
+		V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_0, 0);
+
+		/* GFXH-1383: The SW_INIT may cause a stray write to address 0
+		 * of the unit, so reset it to its power-on value here.
+		 */
+		V3D_WRITE(V3D_HUB_AXICFG, V3D_HUB_AXICFG_MAX_LEN_MASK);
+	} else {
+		WARN_ON_ONCE(V3D_GET_FIELD(version,
+					   V3D_TOP_GR_BRIDGE_MAJOR) != 7);
+		V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_1,
+				 V3D_TOP_GR_BRIDGE_SW_INIT_1_V3D_CLK_108_SW_INIT);
+		V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_1, 0);
+	}
+
+	v3d_init_hw_state(v3d);
+}
+
+void
+v3d_reset(struct v3d_dev *v3d)
+{
+	struct drm_device *dev = &v3d->drm;
+
+	DRM_ERROR("Resetting GPU.\n");
+	trace_v3d_reset_begin(dev);
+
+	/* XXX: only needed for safe powerdown, not reset. */
+	if (false)
+		v3d_idle_axi(v3d, 0);
+
+	v3d_idle_gca(v3d);
+	v3d_reset_v3d(v3d);
+
+	v3d_mmu_set_page_table(v3d);
+	v3d_irq_reset(v3d);
+
+	trace_v3d_reset_end(dev);
+}
+
+static void
+v3d_flush_l3(struct v3d_dev *v3d)
+{
+	if (v3d->ver < 41) {
+		u32 gca_ctrl = V3D_GCA_READ(V3D_GCA_CACHE_CTRL);
+
+		V3D_GCA_WRITE(V3D_GCA_CACHE_CTRL,
+			      gca_ctrl | V3D_GCA_CACHE_CTRL_FLUSH);
+
+		if (v3d->ver < 33) {
+			V3D_GCA_WRITE(V3D_GCA_CACHE_CTRL,
+				      gca_ctrl & ~V3D_GCA_CACHE_CTRL_FLUSH);
+		}
+	}
+}
+
+/* Invalidates the (read-only) L2 cache. */
+static void
+v3d_invalidate_l2(struct v3d_dev *v3d, int core)
+{
+	V3D_CORE_WRITE(core, V3D_CTL_L2CACTL,
+		       V3D_L2CACTL_L2CCLR |
+		       V3D_L2CACTL_L2CENA);
+}
+
+static void
+v3d_invalidate_l1td(struct v3d_dev *v3d, int core)
+{
+	V3D_CORE_WRITE(core, V3D_CTL_L2TCACTL, V3D_L2TCACTL_TMUWCF);
+	if (wait_for(!(V3D_CORE_READ(core, V3D_CTL_L2TCACTL) &
+		       V3D_L2TCACTL_L2TFLS), 100)) {
+		DRM_ERROR("Timeout waiting for L1T write combiner flush\n");
+	}
+}
+
+/* Invalidates texture L2 cachelines */
+static void
+v3d_flush_l2t(struct v3d_dev *v3d, int core)
+{
+	v3d_invalidate_l1td(v3d, core);
+
+	V3D_CORE_WRITE(core, V3D_CTL_L2TCACTL,
+		       V3D_L2TCACTL_L2TFLS |
+		       V3D_SET_FIELD(V3D_L2TCACTL_FLM_FLUSH, V3D_L2TCACTL_FLM));
+	if (wait_for(!(V3D_CORE_READ(core, V3D_CTL_L2TCACTL) &
+		       V3D_L2TCACTL_L2TFLS), 100)) {
+		DRM_ERROR("Timeout waiting for L2T flush\n");
+	}
+}
+
+/* Invalidates the slice caches.  These are read-only caches. */
+static void
+v3d_invalidate_slices(struct v3d_dev *v3d, int core)
+{
+	V3D_CORE_WRITE(core, V3D_CTL_SLCACTL,
+		       V3D_SET_FIELD(0xf, V3D_SLCACTL_TVCCS) |
+		       V3D_SET_FIELD(0xf, V3D_SLCACTL_TDCCS) |
+		       V3D_SET_FIELD(0xf, V3D_SLCACTL_UCC) |
+		       V3D_SET_FIELD(0xf, V3D_SLCACTL_ICC));
+}
+
+/* Invalidates texture L2 cachelines */
+static void
+v3d_invalidate_l2t(struct v3d_dev *v3d, int core)
+{
+	V3D_CORE_WRITE(core,
+		       V3D_CTL_L2TCACTL,
+		       V3D_L2TCACTL_L2TFLS |
+		       V3D_SET_FIELD(V3D_L2TCACTL_FLM_CLEAR, V3D_L2TCACTL_FLM));
+	if (wait_for(!(V3D_CORE_READ(core, V3D_CTL_L2TCACTL) &
+		       V3D_L2TCACTL_L2TFLS), 100)) {
+		DRM_ERROR("Timeout waiting for L2T invalidate\n");
+	}
+}
+
+void
+v3d_invalidate_caches(struct v3d_dev *v3d)
+{
+	v3d_flush_l3(v3d);
+
+	v3d_invalidate_l2(v3d, 0);
+	v3d_invalidate_slices(v3d, 0);
+	v3d_flush_l2t(v3d, 0);
+}
+
+void
+v3d_flush_caches(struct v3d_dev *v3d)
+{
+	v3d_invalidate_l1td(v3d, 0);
+	v3d_invalidate_l2t(v3d, 0);
+}
+
+static void
+v3d_attach_object_fences(struct v3d_exec_info *exec)
+{
+	struct dma_fence *out_fence = &exec->render.base.s_fence->finished;
+	struct v3d_bo *bo;
+	int i;
+
+	for (i = 0; i < exec->bo_count; i++) {
+		bo = to_v3d_bo(&exec->bo[i]->base);
+
+		/* XXX: Use shared fences for read-only objects. */
+		reservation_object_add_excl_fence(bo->resv, out_fence);
+	}
+}
+
+static void
+v3d_unlock_bo_reservations(struct drm_device *dev,
+			   struct v3d_exec_info *exec,
+			   struct ww_acquire_ctx *acquire_ctx)
+{
+	int i;
+
+	for (i = 0; i < exec->bo_count; i++) {
+		struct v3d_bo *bo = to_v3d_bo(&exec->bo[i]->base);
+
+		ww_mutex_unlock(&bo->resv->lock);
+	}
+
+	ww_acquire_fini(acquire_ctx);
+}
+
+/* Takes the reservation lock on all the BOs being referenced, so that
+ * at queue submit time we can update the reservations.
+ *
+ * We don't lock the RCL the tile alloc/state BOs, or overflow memory
+ * (all of which are on exec->unref_list).  They're entirely private
+ * to v3d, so we don't attach dma-buf fences to them.
+ */
+static int
+v3d_lock_bo_reservations(struct drm_device *dev,
+			 struct v3d_exec_info *exec,
+			 struct ww_acquire_ctx *acquire_ctx)
+{
+	int contended_lock = -1;
+	int i, ret;
+	struct v3d_bo *bo;
+
+	ww_acquire_init(acquire_ctx, &reservation_ww_class);
+
+retry:
+	if (contended_lock != -1) {
+		bo = to_v3d_bo(&exec->bo[contended_lock]->base);
+		ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock,
+						       acquire_ctx);
+		if (ret) {
+			ww_acquire_done(acquire_ctx);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < exec->bo_count; i++) {
+		if (i == contended_lock)
+			continue;
+
+		bo = to_v3d_bo(&exec->bo[i]->base);
+
+		ret = ww_mutex_lock_interruptible(&bo->resv->lock, acquire_ctx);
+		if (ret) {
+			int j;
+
+			for (j = 0; j < i; j++) {
+				bo = to_v3d_bo(&exec->bo[j]->base);
+				ww_mutex_unlock(&bo->resv->lock);
+			}
+
+			if (contended_lock != -1 && contended_lock >= i) {
+				bo = to_v3d_bo(&exec->bo[contended_lock]->base);
+
+				ww_mutex_unlock(&bo->resv->lock);
+			}
+
+			if (ret == -EDEADLK) {
+				contended_lock = i;
+				goto retry;
+			}
+
+			ww_acquire_done(acquire_ctx);
+			return ret;
+		}
+	}
+
+	ww_acquire_done(acquire_ctx);
+
+	/* Reserve space for our shared (read-only) fence references,
+	 * before we commit the CL to the hardware.
+	 */
+	for (i = 0; i < exec->bo_count; i++) {
+		bo = to_v3d_bo(&exec->bo[i]->base);
+
+		ret = reservation_object_reserve_shared(bo->resv);
+		if (ret) {
+			v3d_unlock_bo_reservations(dev, exec, acquire_ctx);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * v3d_cl_lookup_bos() - Sets up exec->bo[] with the GEM objects
+ * referenced by the job.
+ * @dev: DRM device
+ * @file_priv: DRM file for this fd
+ * @exec: V3D job being set up
+ *
+ * The command validator needs to reference BOs by their index within
+ * the submitted job's BO list.  This does the validation of the job's
+ * BO list and reference counting for the lifetime of the job.
+ *
+ * Note that this function doesn't need to unreference the BOs on
+ * failure, because that will happen at v3d_exec_cleanup() time.
+ */
+static int
+v3d_cl_lookup_bos(struct drm_device *dev,
+		  struct drm_file *file_priv,
+		  struct drm_v3d_submit_cl *args,
+		  struct v3d_exec_info *exec)
+{
+	u32 *handles;
+	int ret = 0;
+	int i;
+
+	exec->bo_count = args->bo_handle_count;
+
+	if (!exec->bo_count) {
+		/* See comment on bo_index for why we have to check
+		 * this.
+		 */
+		DRM_DEBUG("Rendering requires BOs\n");
+		return -EINVAL;
+	}
+
+	exec->bo = kvmalloc_array(exec->bo_count,
+				  sizeof(struct drm_gem_cma_object *),
+				  GFP_KERNEL | __GFP_ZERO);
+	if (!exec->bo) {
+		DRM_DEBUG("Failed to allocate validated BO pointers\n");
+		return -ENOMEM;
+	}
+
+	handles = kvmalloc_array(exec->bo_count, sizeof(u32), GFP_KERNEL);
+	if (!handles) {
+		ret = -ENOMEM;
+		DRM_DEBUG("Failed to allocate incoming GEM handles\n");
+		goto fail;
+	}
+
+	if (copy_from_user(handles,
+			   (void __user *)(uintptr_t)args->bo_handles,
+			   exec->bo_count * sizeof(u32))) {
+		ret = -EFAULT;
+		DRM_DEBUG("Failed to copy in GEM handles\n");
+		goto fail;
+	}
+
+	spin_lock(&file_priv->table_lock);
+	for (i = 0; i < exec->bo_count; i++) {
+		struct drm_gem_object *bo = idr_find(&file_priv->object_idr,
+						     handles[i]);
+		if (!bo) {
+			DRM_DEBUG("Failed to look up GEM BO %d: %d\n",
+				  i, handles[i]);
+			ret = -ENOENT;
+			spin_unlock(&file_priv->table_lock);
+			goto fail;
+		}
+		drm_gem_object_get(bo);
+		exec->bo[i] = to_v3d_bo(bo);
+	}
+	spin_unlock(&file_priv->table_lock);
+
+fail:
+	kvfree(handles);
+	return ret;
+}
+
+static void
+v3d_exec_cleanup(struct kref *ref)
+{
+	struct v3d_exec_info *exec = container_of(ref, struct v3d_exec_info,
+						  refcount);
+	struct v3d_dev *v3d = exec->v3d;
+	unsigned int i;
+	struct v3d_bo *bo, *save;
+
+	dma_fence_put(exec->bin.in_fence);
+	dma_fence_put(exec->render.in_fence);
+
+	dma_fence_put(exec->bin.done_fence);
+	dma_fence_put(exec->render.done_fence);
+
+	dma_fence_put(exec->bin_done_fence);
+
+	for (i = 0; i < exec->bo_count; i++)
+		drm_gem_object_put_unlocked(&exec->bo[i]->base);
+	kvfree(exec->bo);
+
+	list_for_each_entry_safe(bo, save, &exec->unref_list, unref_head) {
+		drm_gem_object_put_unlocked(&bo->base);
+	}
+
+	pm_runtime_mark_last_busy(v3d->dev);
+	pm_runtime_put_autosuspend(v3d->dev);
+
+	kfree(exec);
+}
+
+void v3d_exec_put(struct v3d_exec_info *exec)
+{
+	kref_put(&exec->refcount, v3d_exec_cleanup);
+}
+
+int
+v3d_wait_bo_ioctl(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv)
+{
+	int ret;
+	struct drm_v3d_wait_bo *args = data;
+	struct drm_gem_object *gem_obj;
+	struct v3d_bo *bo;
+	ktime_t start = ktime_get();
+	u64 delta_ns;
+	unsigned long timeout_jiffies =
+		nsecs_to_jiffies_timeout(args->timeout_ns);
+
+	if (args->pad != 0)
+		return -EINVAL;
+
+	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
+	if (!gem_obj) {
+		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
+		return -EINVAL;
+	}
+	bo = to_v3d_bo(gem_obj);
+
+	ret = reservation_object_wait_timeout_rcu(bo->resv,
+						  true, true,
+						  timeout_jiffies);
+
+	if (ret == 0)
+		ret = -ETIME;
+	else if (ret > 0)
+		ret = 0;
+
+	/* Decrement the user's timeout, in case we got interrupted
+	 * such that the ioctl will be restarted.
+	 */
+	delta_ns = ktime_to_ns(ktime_sub(ktime_get(), start));
+	if (delta_ns < args->timeout_ns)
+		args->timeout_ns -= delta_ns;
+	else
+		args->timeout_ns = 0;
+
+	/* Asked to wait beyond the jiffie/scheduler precision? */
+	if (ret == -ETIME && args->timeout_ns)
+		ret = -EAGAIN;
+
+	drm_gem_object_put_unlocked(gem_obj);
+
+	return ret;
+}
+
+/**
+ * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D.
+ * @dev: DRM device
+ * @data: ioctl argument
+ * @file_priv: DRM file for this fd
+ *
+ * This is the main entrypoint for userspace to submit a 3D frame to
+ * the GPU.  Userspace provides the binner command list (if
+ * applicable), and the kernel sets up the render command list to draw
+ * to the framebuffer described in the ioctl, using the command lists
+ * that the 3D engine's binner will produce.
+ */
+int
+v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
+{
+	struct v3d_dev *v3d = to_v3d_dev(dev);
+	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
+	struct drm_v3d_submit_cl *args = data;
+	struct v3d_exec_info *exec;
+	struct ww_acquire_ctx acquire_ctx;
+	struct drm_syncobj *sync_out;
+	int ret = 0;
+
+	if (args->pad != 0) {
+		DRM_INFO("pad must be zero: %d\n", args->pad);
+		return -EINVAL;
+	}
+
+	exec = kcalloc(1, sizeof(*exec), GFP_KERNEL);
+	if (!exec)
+		return -ENOMEM;
+
+	ret = pm_runtime_get_sync(v3d->dev);
+	if (ret < 0) {
+		kfree(exec);
+		return ret;
+	}
+
+	kref_init(&exec->refcount);
+
+	ret = drm_syncobj_find_fence(file_priv, args->in_sync_bcl,
+				     &exec->bin.in_fence);
+	if (ret == -EINVAL)
+		goto fail;
+
+	ret = drm_syncobj_find_fence(file_priv, args->in_sync_rcl,
+				     &exec->render.in_fence);
+	if (ret == -EINVAL)
+		goto fail;
+
+	exec->qma = args->qma;
+	exec->qms = args->qms;
+	exec->qts = args->qts;
+	exec->bin.exec = exec;
+	exec->bin.start = args->bcl_start;
+	exec->bin.end = args->bcl_end;
+	exec->render.exec = exec;
+	exec->render.start = args->rcl_start;
+	exec->render.end = args->rcl_end;
+	exec->v3d = v3d;
+	INIT_LIST_HEAD(&exec->unref_list);
+
+	ret = v3d_cl_lookup_bos(dev, file_priv, args, exec);
+	if (ret)
+		goto fail;
+
+	ret = v3d_lock_bo_reservations(dev, exec, &acquire_ctx);
+	if (ret)
+		goto fail;
+
+	if (exec->bin.start != exec->bin.end) {
+		ret = drm_sched_job_init(&exec->bin.base,
+					 &v3d->queue[V3D_BIN].sched,
+					 &v3d_priv->sched_entity[V3D_BIN],
+					 v3d_priv);
+		if (ret)
+			goto fail_unreserve;
+
+		exec->bin_done_fence =
+			dma_fence_get(&exec->bin.base.s_fence->finished);
+
+		kref_get(&exec->refcount); /* put by scheduler job completion */
+		drm_sched_entity_push_job(&exec->bin.base,
+					  &v3d_priv->sched_entity[V3D_BIN]);
+	}
+
+	ret = drm_sched_job_init(&exec->render.base,
+				 &v3d->queue[V3D_RENDER].sched,
+				 &v3d_priv->sched_entity[V3D_RENDER],
+				 v3d_priv);
+	if (ret)
+		goto fail_unreserve;
+
+	kref_get(&exec->refcount); /* put by scheduler job completion */
+	drm_sched_entity_push_job(&exec->render.base,
+				  &v3d_priv->sched_entity[V3D_RENDER]);
+
+	v3d_attach_object_fences(exec);
+
+	v3d_unlock_bo_reservations(dev, exec, &acquire_ctx);
+
+	/* Update the return sync object for the */
+	sync_out = drm_syncobj_find(file_priv, args->out_sync);
+	if (sync_out) {
+		drm_syncobj_replace_fence(sync_out,
+					  &exec->render.base.s_fence->finished);
+		drm_syncobj_put(sync_out);
+	}
+
+	v3d_exec_put(exec);
+
+	return 0;
+
+fail_unreserve:
+	v3d_unlock_bo_reservations(dev, exec, &acquire_ctx);
+fail:
+	v3d_exec_put(exec);
+
+	return ret;
+}
+
+int
+v3d_gem_init(struct drm_device *dev)
+{
+	struct v3d_dev *v3d = to_v3d_dev(dev);
+	u32 pt_size = 4096 * 1024;
+	int ret, i;
+
+	for (i = 0; i < V3D_MAX_QUEUES; i++)
+		v3d->queue[i].fence_context = dma_fence_context_alloc(1);
+
+	spin_lock_init(&v3d->mm_lock);
+	spin_lock_init(&v3d->job_lock);
+	mutex_init(&v3d->bo_lock);
+	mutex_init(&v3d->reset_lock);
+
+	/* Note: We don't allocate address 0.  Various bits of HW
+	 * treat 0 as special, such as the occlusion query counters
+	 * where 0 means "disabled".
+	 */
+	drm_mm_init(&v3d->mm, 1, pt_size / sizeof(u32) - 1);
+
+	v3d->pt = dma_alloc_wc(v3d->dev, pt_size,
+			       &v3d->pt_paddr,
+			       GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
+	if (!v3d->pt) {
+		drm_mm_takedown(&v3d->mm);
+		dev_err(v3d->dev,
+			"Failed to allocate page tables. "
+			"Please ensure you have CMA enabled.\n");
+		return -ENOMEM;
+	}
+
+	v3d_init_hw_state(v3d);
+	v3d_mmu_set_page_table(v3d);
+
+	ret = v3d_sched_init(v3d);
+	if (ret) {
+		drm_mm_takedown(&v3d->mm);
+		dma_free_coherent(v3d->dev, 4096 * 1024, (void *)v3d->pt,
+				  v3d->pt_paddr);
+	}
+
+	return 0;
+}
+
+void
+v3d_gem_destroy(struct drm_device *dev)
+{
+	struct v3d_dev *v3d = to_v3d_dev(dev);
+	enum v3d_queue q;
+
+	v3d_sched_fini(v3d);
+
+	/* Waiting for exec to finish would need to be done before
+	 * unregistering V3D.
+	 */
+	for (q = 0; q < V3D_MAX_QUEUES; q++) {
+		WARN_ON(v3d->queue[q].emit_seqno !=
+			v3d->queue[q].finished_seqno);
+	}
+
+	drm_mm_takedown(&v3d->mm);
+
+	dma_free_coherent(v3d->dev, 4096 * 1024, (void *)v3d->pt, v3d->pt_paddr);
+}
diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c
new file mode 100644
index 000000000000..77e1fa046c10
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_irq.c
@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2014-2018 Broadcom */
+
+/**
+ * DOC: Interrupt management for the V3D engine
+ *
+ * When we take a binning or rendering flush done interrupt, we need
+ * to signal the fence for that job so that the scheduler can queue up
+ * the next one and unblock any waiters.
+ *
+ * When we take the binner out of memory interrupt, we need to
+ * allocate some new memory and pass it to the binner so that the
+ * current job can make progress.
+ */
+
+#include "v3d_drv.h"
+#include "v3d_regs.h"
+
+#define V3D_CORE_IRQS ((u32)(V3D_INT_OUTOMEM |	\
+			     V3D_INT_FLDONE |	\
+			     V3D_INT_FRDONE |	\
+			     V3D_INT_GMPV))
+
+#define V3D_HUB_IRQS ((u32)(V3D_HUB_INT_MMU_WRV |	\
+			    V3D_HUB_INT_MMU_PTI |	\
+			    V3D_HUB_INT_MMU_CAP))
+
+static void
+v3d_overflow_mem_work(struct work_struct *work)
+{
+	struct v3d_dev *v3d =
+		container_of(work, struct v3d_dev, overflow_mem_work);
+	struct drm_device *dev = &v3d->drm;
+	struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024);
+	unsigned long irqflags;
+
+	if (IS_ERR(bo)) {
+		DRM_ERROR("Couldn't allocate binner overflow mem\n");
+		return;
+	}
+
+	/* We lost a race, and our work task came in after the bin job
+	 * completed and exited.  This can happen because the HW
+	 * signals OOM before it's fully OOM, so the binner might just
+	 * barely complete.
+	 *
+	 * If we lose the race and our work task comes in after a new
+	 * bin job got scheduled, that's fine.  We'll just give them
+	 * some binner pool anyway.
+	 */
+	spin_lock_irqsave(&v3d->job_lock, irqflags);
+	if (!v3d->bin_job) {
+		spin_unlock_irqrestore(&v3d->job_lock, irqflags);
+		goto out;
+	}
+
+	drm_gem_object_get(&bo->base);
+	list_add_tail(&bo->unref_head, &v3d->bin_job->unref_list);
+	spin_unlock_irqrestore(&v3d->job_lock, irqflags);
+
+	V3D_CORE_WRITE(0, V3D_PTB_BPOA, bo->node.start << PAGE_SHIFT);
+	V3D_CORE_WRITE(0, V3D_PTB_BPOS, bo->base.size);
+
+out:
+	drm_gem_object_put_unlocked(&bo->base);
+}
+
+static irqreturn_t
+v3d_irq(int irq, void *arg)
+{
+	struct v3d_dev *v3d = arg;
+	u32 intsts;
+	irqreturn_t status = IRQ_NONE;
+
+	intsts = V3D_CORE_READ(0, V3D_CTL_INT_STS);
+
+	/* Acknowledge the interrupts we're handling here. */
+	V3D_CORE_WRITE(0, V3D_CTL_INT_CLR, intsts);
+
+	if (intsts & V3D_INT_OUTOMEM) {
+		/* Note that the OOM status is edge signaled, so the
+		 * interrupt won't happen again until the we actually
+		 * add more memory.
+		 */
+		schedule_work(&v3d->overflow_mem_work);
+		status = IRQ_HANDLED;
+	}
+
+	if (intsts & V3D_INT_FLDONE) {
+		v3d->queue[V3D_BIN].finished_seqno++;
+		dma_fence_signal(v3d->bin_job->bin.done_fence);
+		status = IRQ_HANDLED;
+	}
+
+	if (intsts & V3D_INT_FRDONE) {
+		v3d->queue[V3D_RENDER].finished_seqno++;
+		dma_fence_signal(v3d->render_job->render.done_fence);
+
+		status = IRQ_HANDLED;
+	}
+
+	/* We shouldn't be triggering these if we have GMP in
+	 * always-allowed mode.
+	 */
+	if (intsts & V3D_INT_GMPV)
+		dev_err(v3d->dev, "GMP violation\n");
+
+	return status;
+}
+
+static irqreturn_t
+v3d_hub_irq(int irq, void *arg)
+{
+	struct v3d_dev *v3d = arg;
+	u32 intsts;
+	irqreturn_t status = IRQ_NONE;
+
+	intsts = V3D_READ(V3D_HUB_INT_STS);
+
+	/* Acknowledge the interrupts we're handling here. */
+	V3D_WRITE(V3D_HUB_INT_CLR, intsts);
+
+	if (intsts & (V3D_HUB_INT_MMU_WRV |
+		      V3D_HUB_INT_MMU_PTI |
+		      V3D_HUB_INT_MMU_CAP)) {
+		u32 axi_id = V3D_READ(V3D_MMU_VIO_ID);
+		u64 vio_addr = (u64)V3D_READ(V3D_MMU_VIO_ADDR) << 8;
+
+		dev_err(v3d->dev, "MMU error from client %d at 0x%08llx%s%s%s\n",
+			axi_id, (long long)vio_addr,
+			((intsts & V3D_HUB_INT_MMU_WRV) ?
+			 ", write violation" : ""),
+			((intsts & V3D_HUB_INT_MMU_PTI) ?
+			 ", pte invalid" : ""),
+			((intsts & V3D_HUB_INT_MMU_CAP) ?
+			 ", cap exceeded" : ""));
+		status = IRQ_HANDLED;
+	}
+
+	return status;
+}
+
+void
+v3d_irq_init(struct v3d_dev *v3d)
+{
+	int ret, core;
+
+	INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work);
+
+	/* Clear any pending interrupts someone might have left around
+	 * for us.
+	 */
+	for (core = 0; core < v3d->cores; core++)
+		V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS);
+	V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS);
+
+	ret = devm_request_irq(v3d->dev, platform_get_irq(v3d->pdev, 0),
+			       v3d_hub_irq, IRQF_SHARED,
+			       "v3d_hub", v3d);
+	ret = devm_request_irq(v3d->dev, platform_get_irq(v3d->pdev, 1),
+			       v3d_irq, IRQF_SHARED,
+			       "v3d_core0", v3d);
+	if (ret)
+		dev_err(v3d->dev, "IRQ setup failed: %d\n", ret);
+
+	v3d_irq_enable(v3d);
+}
+
+void
+v3d_irq_enable(struct v3d_dev *v3d)
+{
+	int core;
+
+	/* Enable our set of interrupts, masking out any others. */
+	for (core = 0; core < v3d->cores; core++) {
+		V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS);
+		V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS);
+	}
+
+	V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS);
+	V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS);
+}
+
+void
+v3d_irq_disable(struct v3d_dev *v3d)
+{
+	int core;
+
+	/* Disable all interrupts. */
+	for (core = 0; core < v3d->cores; core++)
+		V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~0);
+	V3D_WRITE(V3D_HUB_INT_MSK_SET, ~0);
+
+	/* Clear any pending interrupts we might have left. */
+	for (core = 0; core < v3d->cores; core++)
+		V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS);
+	V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS);
+
+	cancel_work_sync(&v3d->overflow_mem_work);
+}
+
+/** Reinitializes interrupt registers when a GPU reset is performed. */
+void v3d_irq_reset(struct v3d_dev *v3d)
+{
+	v3d_irq_enable(v3d);
+}
diff --git a/drivers/gpu/drm/v3d/v3d_mmu.c b/drivers/gpu/drm/v3d/v3d_mmu.c
new file mode 100644
index 000000000000..b00f97c31b70
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_mmu.c
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2017-2018 Broadcom */
+
+/**
+ * DOC: Broadcom V3D MMU
+ *
+ * The V3D 3.x hardware (compared to VC4) now includes an MMU.  It has
+ * a single level of page tables for the V3D's 4GB address space to
+ * map to AXI bus addresses, thus it could need up to 4MB of
+ * physically contiguous memory to store the PTEs.
+ *
+ * Because the 4MB of contiguous memory for page tables is precious,
+ * and switching between them is expensive, we load all BOs into the
+ * same 4GB address space.
+ *
+ * To protect clients from each other, we should use the GMP to
+ * quickly mask out (at 128kb granularity) what pages are available to
+ * each client.  This is not yet implemented.
+ */
+
+#include "v3d_drv.h"
+#include "v3d_regs.h"
+
+#define V3D_MMU_PAGE_SHIFT 12
+
+/* Note: All PTEs for the 1MB superpage must be filled with the
+ * superpage bit set.
+ */
+#define V3D_PTE_SUPERPAGE BIT(31)
+#define V3D_PTE_WRITEABLE BIT(29)
+#define V3D_PTE_VALID BIT(28)
+
+static int v3d_mmu_flush_all(struct v3d_dev *v3d)
+{
+	int ret;
+
+	/* Make sure that another flush isn't already running when we
+	 * start this one.
+	 */
+	ret = wait_for(!(V3D_READ(V3D_MMU_CTL) &
+			 V3D_MMU_CTL_TLB_CLEARING), 100);
+	if (ret)
+		dev_err(v3d->dev, "TLB clear wait idle pre-wait failed\n");
+
+	V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL) |
+		  V3D_MMU_CTL_TLB_CLEAR);
+
+	V3D_WRITE(V3D_MMUC_CONTROL,
+		  V3D_MMUC_CONTROL_FLUSH |
+		  V3D_MMUC_CONTROL_ENABLE);
+
+	ret = wait_for(!(V3D_READ(V3D_MMU_CTL) &
+			 V3D_MMU_CTL_TLB_CLEARING), 100);
+	if (ret) {
+		dev_err(v3d->dev, "TLB clear wait idle failed\n");
+		return ret;
+	}
+
+	ret = wait_for(!(V3D_READ(V3D_MMUC_CONTROL) &
+			 V3D_MMUC_CONTROL_FLUSHING), 100);
+	if (ret)
+		dev_err(v3d->dev, "MMUC flush wait idle failed\n");
+
+	return ret;
+}
+
+int v3d_mmu_set_page_table(struct v3d_dev *v3d)
+{
+	V3D_WRITE(V3D_MMU_PT_PA_BASE, v3d->pt_paddr >> V3D_MMU_PAGE_SHIFT);
+	V3D_WRITE(V3D_MMU_CTL,
+		  V3D_MMU_CTL_ENABLE |
+		  V3D_MMU_CTL_PT_INVALID |
+		  V3D_MMU_CTL_PT_INVALID_ABORT |
+		  V3D_MMU_CTL_WRITE_VIOLATION_ABORT |
+		  V3D_MMU_CTL_CAP_EXCEEDED_ABORT);
+	V3D_WRITE(V3D_MMU_ILLEGAL_ADDR,
+		  (v3d->mmu_scratch_paddr >> V3D_MMU_PAGE_SHIFT) |
+		  V3D_MMU_ILLEGAL_ADDR_ENABLE);
+	V3D_WRITE(V3D_MMUC_CONTROL, V3D_MMUC_CONTROL_ENABLE);
+
+	return v3d_mmu_flush_all(v3d);
+}
+
+void v3d_mmu_insert_ptes(struct v3d_bo *bo)
+{
+	struct v3d_dev *v3d = to_v3d_dev(bo->base.dev);
+	u32 page = bo->node.start;
+	u32 page_prot = V3D_PTE_WRITEABLE | V3D_PTE_VALID;
+	unsigned int count;
+	struct scatterlist *sgl;
+
+	for_each_sg(bo->sgt->sgl, sgl, bo->sgt->nents, count) {
+		u32 page_address = sg_dma_address(sgl) >> V3D_MMU_PAGE_SHIFT;
+		u32 pte = page_prot | page_address;
+		u32 i;
+
+		BUG_ON(page_address + (sg_dma_len(sgl) >> V3D_MMU_PAGE_SHIFT) >=
+		       BIT(24));
+
+		for (i = 0; i < sg_dma_len(sgl) >> V3D_MMU_PAGE_SHIFT; i++)
+			v3d->pt[page++] = pte + i;
+	}
+
+	WARN_ON_ONCE(page - bo->node.start !=
+		     bo->base.size >> V3D_MMU_PAGE_SHIFT);
+
+	if (v3d_mmu_flush_all(v3d))
+		dev_err(v3d->dev, "MMU flush timeout\n");
+}
+
+void v3d_mmu_remove_ptes(struct v3d_bo *bo)
+{
+	struct v3d_dev *v3d = to_v3d_dev(bo->base.dev);
+	u32 npages = bo->base.size >> V3D_MMU_PAGE_SHIFT;
+	u32 page;
+
+	for (page = bo->node.start; page < bo->node.start + npages; page++)
+		v3d->pt[page] = 0;
+
+	if (v3d_mmu_flush_all(v3d))
+		dev_err(v3d->dev, "MMU flush timeout\n");
+}
diff --git a/drivers/gpu/drm/v3d/v3d_regs.h b/drivers/gpu/drm/v3d/v3d_regs.h
new file mode 100644
index 000000000000..fc13282dfc2f
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_regs.h
@@ -0,0 +1,295 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2017-2018 Broadcom */
+
+#ifndef V3D_REGS_H
+#define V3D_REGS_H
+
+#include <linux/bitops.h>
+
+#define V3D_MASK(high, low) ((u32)GENMASK(high, low))
+/* Using the GNU statement expression extension */
+#define V3D_SET_FIELD(value, field)					\
+	({								\
+		u32 fieldval = (value) << field##_SHIFT;		\
+		WARN_ON((fieldval & ~field##_MASK) != 0);		\
+		fieldval & field##_MASK;				\
+	 })
+
+#define V3D_GET_FIELD(word, field) (((word) & field##_MASK) >>		\
+				    field##_SHIFT)
+
+/* Hub registers for shared hardware between V3D cores. */
+
+#define V3D_HUB_AXICFG                                 0x00000
+# define V3D_HUB_AXICFG_MAX_LEN_MASK                   V3D_MASK(3, 0)
+# define V3D_HUB_AXICFG_MAX_LEN_SHIFT                  0
+#define V3D_HUB_UIFCFG                                 0x00004
+#define V3D_HUB_IDENT0                                 0x00008
+
+#define V3D_HUB_IDENT1                                 0x0000c
+# define V3D_HUB_IDENT1_WITH_MSO                       BIT(19)
+# define V3D_HUB_IDENT1_WITH_TSY                       BIT(18)
+# define V3D_HUB_IDENT1_WITH_TFU                       BIT(17)
+# define V3D_HUB_IDENT1_WITH_L3C                       BIT(16)
+# define V3D_HUB_IDENT1_NHOSTS_MASK                    V3D_MASK(15, 12)
+# define V3D_HUB_IDENT1_NHOSTS_SHIFT                   12
+# define V3D_HUB_IDENT1_NCORES_MASK                    V3D_MASK(11, 8)
+# define V3D_HUB_IDENT1_NCORES_SHIFT                   8
+# define V3D_HUB_IDENT1_REV_MASK                       V3D_MASK(7, 4)
+# define V3D_HUB_IDENT1_REV_SHIFT                      4
+# define V3D_HUB_IDENT1_TVER_MASK                      V3D_MASK(3, 0)
+# define V3D_HUB_IDENT1_TVER_SHIFT                     0
+
+#define V3D_HUB_IDENT2                                 0x00010
+# define V3D_HUB_IDENT2_WITH_MMU                       BIT(8)
+# define V3D_HUB_IDENT2_L3C_NKB_MASK                   V3D_MASK(7, 0)
+# define V3D_HUB_IDENT2_L3C_NKB_SHIFT                  0
+
+#define V3D_HUB_IDENT3                                 0x00014
+# define V3D_HUB_IDENT3_IPREV_MASK                     V3D_MASK(15, 8)
+# define V3D_HUB_IDENT3_IPREV_SHIFT                    8
+# define V3D_HUB_IDENT3_IPIDX_MASK                     V3D_MASK(7, 0)
+# define V3D_HUB_IDENT3_IPIDX_SHIFT                    0
+
+#define V3D_HUB_INT_STS                                0x00050
+#define V3D_HUB_INT_SET                                0x00054
+#define V3D_HUB_INT_CLR                                0x00058
+#define V3D_HUB_INT_MSK_STS                            0x0005c
+#define V3D_HUB_INT_MSK_SET                            0x00060
+#define V3D_HUB_INT_MSK_CLR                            0x00064
+# define V3D_HUB_INT_MMU_WRV                           BIT(5)
+# define V3D_HUB_INT_MMU_PTI                           BIT(4)
+# define V3D_HUB_INT_MMU_CAP                           BIT(3)
+# define V3D_HUB_INT_MSO                               BIT(2)
+# define V3D_HUB_INT_TFUC                              BIT(1)
+# define V3D_HUB_INT_TFUF                              BIT(0)
+
+#define V3D_GCA_CACHE_CTRL                             0x0000c
+# define V3D_GCA_CACHE_CTRL_FLUSH                      BIT(0)
+
+#define V3D_GCA_SAFE_SHUTDOWN                          0x000b0
+# define V3D_GCA_SAFE_SHUTDOWN_EN                      BIT(0)
+
+#define V3D_GCA_SAFE_SHUTDOWN_ACK                      0x000b4
+# define V3D_GCA_SAFE_SHUTDOWN_ACK_ACKED               3
+
+# define V3D_TOP_GR_BRIDGE_REVISION                    0x00000
+# define V3D_TOP_GR_BRIDGE_MAJOR_MASK                  V3D_MASK(15, 8)
+# define V3D_TOP_GR_BRIDGE_MAJOR_SHIFT                 8
+# define V3D_TOP_GR_BRIDGE_MINOR_MASK                  V3D_MASK(7, 0)
+# define V3D_TOP_GR_BRIDGE_MINOR_SHIFT                 0
+
+/* 7268 reset reg */
+# define V3D_TOP_GR_BRIDGE_SW_INIT_0                   0x00008
+# define V3D_TOP_GR_BRIDGE_SW_INIT_0_V3D_CLK_108_SW_INIT BIT(0)
+/* 7278 reset reg */
+# define V3D_TOP_GR_BRIDGE_SW_INIT_1                   0x0000c
+# define V3D_TOP_GR_BRIDGE_SW_INIT_1_V3D_CLK_108_SW_INIT BIT(0)
+
+/* Per-MMU registers. */
+
+#define V3D_MMUC_CONTROL                               0x01000
+# define V3D_MMUC_CONTROL_CLEAR                        BIT(3)
+# define V3D_MMUC_CONTROL_FLUSHING                     BIT(2)
+# define V3D_MMUC_CONTROL_FLUSH                        BIT(1)
+# define V3D_MMUC_CONTROL_ENABLE                       BIT(0)
+
+#define V3D_MMU_CTL                                    0x01200
+# define V3D_MMU_CTL_CAP_EXCEEDED                      BIT(27)
+# define V3D_MMU_CTL_CAP_EXCEEDED_ABORT                BIT(26)
+# define V3D_MMU_CTL_CAP_EXCEEDED_INT                  BIT(25)
+# define V3D_MMU_CTL_CAP_EXCEEDED_EXCEPTION            BIT(24)
+# define V3D_MMU_CTL_PT_INVALID                        BIT(20)
+# define V3D_MMU_CTL_PT_INVALID_ABORT                  BIT(19)
+# define V3D_MMU_CTL_PT_INVALID_INT                    BIT(18)
+# define V3D_MMU_CTL_PT_INVALID_EXCEPTION              BIT(17)
+# define V3D_MMU_CTL_WRITE_VIOLATION                   BIT(16)
+# define V3D_MMU_CTL_WRITE_VIOLATION_ABORT             BIT(11)
+# define V3D_MMU_CTL_WRITE_VIOLATION_INT               BIT(10)
+# define V3D_MMU_CTL_WRITE_VIOLATION_EXCEPTION         BIT(9)
+# define V3D_MMU_CTL_TLB_CLEARING                      BIT(7)
+# define V3D_MMU_CTL_TLB_STATS_CLEAR                   BIT(3)
+# define V3D_MMU_CTL_TLB_CLEAR                         BIT(2)
+# define V3D_MMU_CTL_TLB_STATS_ENABLE                  BIT(1)
+# define V3D_MMU_CTL_ENABLE                            BIT(0)
+
+#define V3D_MMU_PT_PA_BASE                             0x01204
+#define V3D_MMU_HIT                                    0x01208
+#define V3D_MMU_MISSES                                 0x0120c
+#define V3D_MMU_STALLS                                 0x01210
+
+#define V3D_MMU_ADDR_CAP                               0x01214
+# define V3D_MMU_ADDR_CAP_ENABLE                       BIT(31)
+# define V3D_MMU_ADDR_CAP_MPAGE_MASK                   V3D_MASK(11, 0)
+# define V3D_MMU_ADDR_CAP_MPAGE_SHIFT                  0
+
+#define V3D_MMU_SHOOT_DOWN                             0x01218
+# define V3D_MMU_SHOOT_DOWN_SHOOTING                   BIT(29)
+# define V3D_MMU_SHOOT_DOWN_SHOOT                      BIT(28)
+# define V3D_MMU_SHOOT_DOWN_PAGE_MASK                  V3D_MASK(27, 0)
+# define V3D_MMU_SHOOT_DOWN_PAGE_SHIFT                 0
+
+#define V3D_MMU_BYPASS_START                           0x0121c
+#define V3D_MMU_BYPASS_END                             0x01220
+
+/* AXI ID of the access that faulted */
+#define V3D_MMU_VIO_ID                                 0x0122c
+
+/* Address for illegal PTEs to return */
+#define V3D_MMU_ILLEGAL_ADDR                           0x01230
+# define V3D_MMU_ILLEGAL_ADDR_ENABLE                   BIT(31)
+
+/* Address that faulted */
+#define V3D_MMU_VIO_ADDR                               0x01234
+
+/* Per-V3D-core registers */
+
+#define V3D_CTL_IDENT0                                 0x00000
+# define V3D_IDENT0_VER_MASK                           V3D_MASK(31, 24)
+# define V3D_IDENT0_VER_SHIFT                          24
+
+#define V3D_CTL_IDENT1                                 0x00004
+/* Multiples of 1kb */
+# define V3D_IDENT1_VPM_SIZE_MASK                      V3D_MASK(31, 28)
+# define V3D_IDENT1_VPM_SIZE_SHIFT                     28
+# define V3D_IDENT1_NSEM_MASK                          V3D_MASK(23, 16)
+# define V3D_IDENT1_NSEM_SHIFT                         16
+# define V3D_IDENT1_NTMU_MASK                          V3D_MASK(15, 12)
+# define V3D_IDENT1_NTMU_SHIFT                         12
+# define V3D_IDENT1_QUPS_MASK                          V3D_MASK(11, 8)
+# define V3D_IDENT1_QUPS_SHIFT                         8
+# define V3D_IDENT1_NSLC_MASK                          V3D_MASK(7, 4)
+# define V3D_IDENT1_NSLC_SHIFT                         4
+# define V3D_IDENT1_REV_MASK                           V3D_MASK(3, 0)
+# define V3D_IDENT1_REV_SHIFT                          0
+
+#define V3D_CTL_IDENT2                                 0x00008
+# define V3D_IDENT2_BCG_INT                            BIT(28)
+
+#define V3D_CTL_MISCCFG                                0x00018
+# define V3D_MISCCFG_OVRTMUOUT                         BIT(0)
+
+#define V3D_CTL_L2CACTL                                0x00020
+# define V3D_L2CACTL_L2CCLR                            BIT(2)
+# define V3D_L2CACTL_L2CDIS                            BIT(1)
+# define V3D_L2CACTL_L2CENA                            BIT(0)
+
+#define V3D_CTL_SLCACTL                                0x00024
+# define V3D_SLCACTL_TVCCS_MASK                        V3D_MASK(27, 24)
+# define V3D_SLCACTL_TVCCS_SHIFT                       24
+# define V3D_SLCACTL_TDCCS_MASK                        V3D_MASK(19, 16)
+# define V3D_SLCACTL_TDCCS_SHIFT                       16
+# define V3D_SLCACTL_UCC_MASK                          V3D_MASK(11, 8)
+# define V3D_SLCACTL_UCC_SHIFT                         8
+# define V3D_SLCACTL_ICC_MASK                          V3D_MASK(3, 0)
+# define V3D_SLCACTL_ICC_SHIFT                         0
+
+#define V3D_CTL_L2TCACTL                               0x00030
+# define V3D_L2TCACTL_TMUWCF                           BIT(8)
+# define V3D_L2TCACTL_L2T_NO_WM                        BIT(4)
+# define V3D_L2TCACTL_FLM_FLUSH                        0
+# define V3D_L2TCACTL_FLM_CLEAR                        1
+# define V3D_L2TCACTL_FLM_CLEAN                        2
+# define V3D_L2TCACTL_FLM_MASK                         V3D_MASK(2, 1)
+# define V3D_L2TCACTL_FLM_SHIFT                        1
+# define V3D_L2TCACTL_L2TFLS                           BIT(0)
+#define V3D_CTL_L2TFLSTA                               0x00034
+#define V3D_CTL_L2TFLEND                               0x00038
+
+#define V3D_CTL_INT_STS                                0x00050
+#define V3D_CTL_INT_SET                                0x00054
+#define V3D_CTL_INT_CLR                                0x00058
+#define V3D_CTL_INT_MSK_STS                            0x0005c
+#define V3D_CTL_INT_MSK_SET                            0x00060
+#define V3D_CTL_INT_MSK_CLR                            0x00064
+# define V3D_INT_QPU_MASK                              V3D_MASK(27, 16)
+# define V3D_INT_QPU_SHIFT                             16
+# define V3D_INT_GMPV                                  BIT(5)
+# define V3D_INT_TRFB                                  BIT(4)
+# define V3D_INT_SPILLUSE                              BIT(3)
+# define V3D_INT_OUTOMEM                               BIT(2)
+# define V3D_INT_FLDONE                                BIT(1)
+# define V3D_INT_FRDONE                                BIT(0)
+
+#define V3D_CLE_CT0CS                                  0x00100
+#define V3D_CLE_CT1CS                                  0x00104
+#define V3D_CLE_CTNCS(n) (V3D_CLE_CT0CS + 4 * n)
+#define V3D_CLE_CT0EA                                  0x00108
+#define V3D_CLE_CT1EA                                  0x0010c
+#define V3D_CLE_CTNEA(n) (V3D_CLE_CT0EA + 4 * n)
+#define V3D_CLE_CT0CA                                  0x00110
+#define V3D_CLE_CT1CA                                  0x00114
+#define V3D_CLE_CTNCA(n) (V3D_CLE_CT0CA + 4 * n)
+#define V3D_CLE_CT0RA                                  0x00118
+#define V3D_CLE_CT1RA                                  0x0011c
+#define V3D_CLE_CT0LC                                  0x00120
+#define V3D_CLE_CT1LC                                  0x00124
+#define V3D_CLE_CT0PC                                  0x00128
+#define V3D_CLE_CT1PC                                  0x0012c
+#define V3D_CLE_PCS                                    0x00130
+#define V3D_CLE_BFC                                    0x00134
+#define V3D_CLE_RFC                                    0x00138
+#define V3D_CLE_TFBC                                   0x0013c
+#define V3D_CLE_TFIT                                   0x00140
+#define V3D_CLE_CT1CFG                                 0x00144
+#define V3D_CLE_CT1TILECT                              0x00148
+#define V3D_CLE_CT1TSKIP                               0x0014c
+#define V3D_CLE_CT1PTCT                                0x00150
+#define V3D_CLE_CT0SYNC                                0x00154
+#define V3D_CLE_CT1SYNC                                0x00158
+#define V3D_CLE_CT0QTS                                 0x0015c
+# define V3D_CLE_CT0QTS_ENABLE                         BIT(1)
+#define V3D_CLE_CT0QBA                                 0x00160
+#define V3D_CLE_CT1QBA                                 0x00164
+#define V3D_CLE_CTNQBA(n) (V3D_CLE_CT0QBA + 4 * n)
+#define V3D_CLE_CT0QEA                                 0x00168
+#define V3D_CLE_CT1QEA                                 0x0016c
+#define V3D_CLE_CTNQEA(n) (V3D_CLE_CT0QEA + 4 * n)
+#define V3D_CLE_CT0QMA                                 0x00170
+#define V3D_CLE_CT0QMS                                 0x00174
+#define V3D_CLE_CT1QCFG                                0x00178
+/* If set without ETPROC, entirely skip tiles with no primitives. */
+# define V3D_CLE_QCFG_ETFILT                           BIT(7)
+/* If set with ETFILT, just write the clear color to tiles with no
+ * primitives.
+ */
+# define V3D_CLE_QCFG_ETPROC                           BIT(6)
+# define V3D_CLE_QCFG_ETSFLUSH                         BIT(1)
+# define V3D_CLE_QCFG_MCDIS                            BIT(0)
+
+#define V3D_PTB_BPCA                                   0x00300
+#define V3D_PTB_BPCS                                   0x00304
+#define V3D_PTB_BPOA                                   0x00308
+#define V3D_PTB_BPOS                                   0x0030c
+
+#define V3D_PTB_BXCF                                   0x00310
+# define V3D_PTB_BXCF_RWORDERDISA                      BIT(1)
+# define V3D_PTB_BXCF_CLIPDISA                         BIT(0)
+
+#define V3D_GMP_STATUS                                 0x00800
+# define V3D_GMP_STATUS_GMPRST                         BIT(31)
+# define V3D_GMP_STATUS_WR_COUNT_MASK                  V3D_MASK(30, 24)
+# define V3D_GMP_STATUS_WR_COUNT_SHIFT                 24
+# define V3D_GMP_STATUS_RD_COUNT_MASK                  V3D_MASK(22, 16)
+# define V3D_GMP_STATUS_RD_COUNT_SHIFT                 16
+# define V3D_GMP_STATUS_WR_ACTIVE                      BIT(5)
+# define V3D_GMP_STATUS_RD_ACTIVE                      BIT(4)
+# define V3D_GMP_STATUS_CFG_BUSY                       BIT(3)
+# define V3D_GMP_STATUS_CNTOVF                         BIT(2)
+# define V3D_GMP_STATUS_INVPROT                        BIT(1)
+# define V3D_GMP_STATUS_VIO                            BIT(0)
+
+#define V3D_GMP_CFG                                    0x00804
+# define V3D_GMP_CFG_LBURSTEN                          BIT(3)
+# define V3D_GMP_CFG_PGCRSEN                           BIT()
+# define V3D_GMP_CFG_STOP_REQ                          BIT(1)
+# define V3D_GMP_CFG_PROT_ENABLE                       BIT(0)
+
+#define V3D_GMP_VIO_ADDR                               0x00808
+#define V3D_GMP_VIO_TYPE                               0x0080c
+#define V3D_GMP_TABLE_ADDR                             0x00810
+#define V3D_GMP_CLEAR_LOAD                             0x00814
+#define V3D_GMP_PRESERVE_LOAD                          0x00818
+#define V3D_GMP_VALID_LINES                            0x00820
+
+#endif /* V3D_REGS_H */
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
new file mode 100644
index 000000000000..b07bece9417d
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2018 Broadcom */
+
+/**
+ * DOC: Broadcom V3D scheduling
+ *
+ * The shared DRM GPU scheduler is used to coordinate submitting jobs
+ * to the hardware.  Each DRM fd (roughly a client process) gets its
+ * own scheduler entity, which will process jobs in order.  The GPU
+ * scheduler will round-robin between clients to submit the next job.
+ *
+ * For simplicity, and in order to keep latency low for interactive
+ * jobs when bulk background jobs are queued up, we submit a new job
+ * to the HW only when it has completed the last one, instead of
+ * filling up the CT[01]Q FIFOs with jobs.  Similarly, we use
+ * v3d_job_dependency() to manage the dependency between bin and
+ * render, instead of having the clients submit jobs with using the
+ * HW's semaphores to interlock between them.
+ */
+
+#include <linux/kthread.h>
+
+#include "v3d_drv.h"
+#include "v3d_regs.h"
+#include "v3d_trace.h"
+
+static struct v3d_job *
+to_v3d_job(struct drm_sched_job *sched_job)
+{
+	return container_of(sched_job, struct v3d_job, base);
+}
+
+static void
+v3d_job_free(struct drm_sched_job *sched_job)
+{
+	struct v3d_job *job = to_v3d_job(sched_job);
+
+	v3d_exec_put(job->exec);
+}
+
+/**
+ * Returns the fences that the bin job depends on, one by one.
+ * v3d_job_run() won't be called until all of them have been signaled.
+ */
+static struct dma_fence *
+v3d_job_dependency(struct drm_sched_job *sched_job,
+		   struct drm_sched_entity *s_entity)
+{
+	struct v3d_job *job = to_v3d_job(sched_job);
+	struct v3d_exec_info *exec = job->exec;
+	enum v3d_queue q = job == &exec->bin ? V3D_BIN : V3D_RENDER;
+	struct dma_fence *fence;
+
+	fence = job->in_fence;
+	if (fence) {
+		job->in_fence = NULL;
+		return fence;
+	}
+
+	if (q == V3D_RENDER) {
+		/* If we had a bin job, the render job definitely depends on
+		 * it. We first have to wait for bin to be scheduled, so that
+		 * its done_fence is created.
+		 */
+		fence = exec->bin_done_fence;
+		if (fence) {
+			exec->bin_done_fence = NULL;
+			return fence;
+		}
+	}
+
+	/* XXX: Wait on a fence for switching the GMP if necessary,
+	 * and then do so.
+	 */
+
+	return fence;
+}
+
+static struct dma_fence *v3d_job_run(struct drm_sched_job *sched_job)
+{
+	struct v3d_job *job = to_v3d_job(sched_job);
+	struct v3d_exec_info *exec = job->exec;
+	enum v3d_queue q = job == &exec->bin ? V3D_BIN : V3D_RENDER;
+	struct v3d_dev *v3d = exec->v3d;
+	struct drm_device *dev = &v3d->drm;
+	struct dma_fence *fence;
+	unsigned long irqflags;
+
+	if (unlikely(job->base.s_fence->finished.error))
+		return NULL;
+
+	/* Lock required around bin_job update vs
+	 * v3d_overflow_mem_work().
+	 */
+	spin_lock_irqsave(&v3d->job_lock, irqflags);
+	if (q == V3D_BIN) {
+		v3d->bin_job = job->exec;
+
+		/* Clear out the overflow allocation, so we don't
+		 * reuse the overflow attached to a previous job.
+		 */
+		V3D_CORE_WRITE(0, V3D_PTB_BPOS, 0);
+	} else {
+		v3d->render_job = job->exec;
+	}
+	spin_unlock_irqrestore(&v3d->job_lock, irqflags);
+
+	/* Can we avoid this flush when q==RENDER?  We need to be
+	 * careful of scheduling, though -- imagine job0 rendering to
+	 * texture and job1 reading, and them being executed as bin0,
+	 * bin1, render0, render1, so that render1's flush at bin time
+	 * wasn't enough.
+	 */
+	v3d_invalidate_caches(v3d);
+
+	fence = v3d_fence_create(v3d, q);
+	if (!fence)
+		return fence;
+
+	if (job->done_fence)
+		dma_fence_put(job->done_fence);
+	job->done_fence = dma_fence_get(fence);
+
+	trace_v3d_submit_cl(dev, q == V3D_RENDER, to_v3d_fence(fence)->seqno,
+			    job->start, job->end);
+
+	if (q == V3D_BIN) {
+		if (exec->qma) {
+			V3D_CORE_WRITE(0, V3D_CLE_CT0QMA, exec->qma);
+			V3D_CORE_WRITE(0, V3D_CLE_CT0QMS, exec->qms);
+		}
+		if (exec->qts) {
+			V3D_CORE_WRITE(0, V3D_CLE_CT0QTS,
+				       V3D_CLE_CT0QTS_ENABLE |
+				       exec->qts);
+		}
+	} else {
+		/* XXX: Set the QCFG */
+	}
+
+	/* Set the current and end address of the control list.
+	 * Writing the end register is what starts the job.
+	 */
+	V3D_CORE_WRITE(0, V3D_CLE_CTNQBA(q), job->start);
+	V3D_CORE_WRITE(0, V3D_CLE_CTNQEA(q), job->end);
+
+	return fence;
+}
+
+static void
+v3d_job_timedout(struct drm_sched_job *sched_job)
+{
+	struct v3d_job *job = to_v3d_job(sched_job);
+	struct v3d_exec_info *exec = job->exec;
+	struct v3d_dev *v3d = exec->v3d;
+	enum v3d_queue q;
+
+	mutex_lock(&v3d->reset_lock);
+
+	/* block scheduler */
+	for (q = 0; q < V3D_MAX_QUEUES; q++) {
+		struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
+
+		kthread_park(sched->thread);
+		drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
+					       sched_job : NULL));
+	}
+
+	/* get the GPU back into the init state */
+	v3d_reset(v3d);
+
+	/* Unblock schedulers and restart their jobs. */
+	for (q = 0; q < V3D_MAX_QUEUES; q++) {
+		drm_sched_job_recovery(&v3d->queue[q].sched);
+		kthread_unpark(v3d->queue[q].sched.thread);
+	}
+
+	mutex_unlock(&v3d->reset_lock);
+}
+
+static const struct drm_sched_backend_ops v3d_sched_ops = {
+	.dependency = v3d_job_dependency,
+	.run_job = v3d_job_run,
+	.timedout_job = v3d_job_timedout,
+	.free_job = v3d_job_free
+};
+
+int
+v3d_sched_init(struct v3d_dev *v3d)
+{
+	int hw_jobs_limit = 1;
+	int job_hang_limit = 0;
+	int hang_limit_ms = 500;
+	int ret;
+
+	ret = drm_sched_init(&v3d->queue[V3D_BIN].sched,
+			     &v3d_sched_ops,
+			     hw_jobs_limit, job_hang_limit,
+			     msecs_to_jiffies(hang_limit_ms),
+			     "v3d_bin");
+	if (ret) {
+		dev_err(v3d->dev, "Failed to create bin scheduler: %d.", ret);
+		return ret;
+	}
+
+	ret = drm_sched_init(&v3d->queue[V3D_RENDER].sched,
+			     &v3d_sched_ops,
+			     hw_jobs_limit, job_hang_limit,
+			     msecs_to_jiffies(hang_limit_ms),
+			     "v3d_render");
+	if (ret) {
+		dev_err(v3d->dev, "Failed to create render scheduler: %d.",
+			ret);
+		drm_sched_fini(&v3d->queue[V3D_BIN].sched);
+		return ret;
+	}
+
+	return 0;
+}
+
+void
+v3d_sched_fini(struct v3d_dev *v3d)
+{
+	enum v3d_queue q;
+
+	for (q = 0; q < V3D_MAX_QUEUES; q++)
+		drm_sched_fini(&v3d->queue[q].sched);
+}
diff --git a/drivers/gpu/drm/v3d/v3d_trace.h b/drivers/gpu/drm/v3d/v3d_trace.h
new file mode 100644
index 000000000000..85dd351e1e09
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_trace.h
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2015-2018 Broadcom */
+
+#if !defined(_V3D_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _V3D_TRACE_H_
+
+#include <linux/stringify.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM v3d
+#define TRACE_INCLUDE_FILE v3d_trace
+
+TRACE_EVENT(v3d_submit_cl,
+	    TP_PROTO(struct drm_device *dev, bool is_render,
+		     uint64_t seqno,
+		     u32 ctnqba, u32 ctnqea),
+	    TP_ARGS(dev, is_render, seqno, ctnqba, ctnqea),
+
+	    TP_STRUCT__entry(
+			     __field(u32, dev)
+			     __field(bool, is_render)
+			     __field(u64, seqno)
+			     __field(u32, ctnqba)
+			     __field(u32, ctnqea)
+			     ),
+
+	    TP_fast_assign(
+			   __entry->dev = dev->primary->index;
+			   __entry->is_render = is_render;
+			   __entry->seqno = seqno;
+			   __entry->ctnqba = ctnqba;
+			   __entry->ctnqea = ctnqea;
+			   ),
+
+	    TP_printk("dev=%u, %s, seqno=%llu, 0x%08x..0x%08x",
+		      __entry->dev,
+		      __entry->is_render ? "RCL" : "BCL",
+		      __entry->seqno,
+		      __entry->ctnqba,
+		      __entry->ctnqea)
+);
+
+TRACE_EVENT(v3d_reset_begin,
+	    TP_PROTO(struct drm_device *dev),
+	    TP_ARGS(dev),
+
+	    TP_STRUCT__entry(
+			     __field(u32, dev)
+			     ),
+
+	    TP_fast_assign(
+			   __entry->dev = dev->primary->index;
+			   ),
+
+	    TP_printk("dev=%u",
+		      __entry->dev)
+);
+
+TRACE_EVENT(v3d_reset_end,
+	    TP_PROTO(struct drm_device *dev),
+	    TP_ARGS(dev),
+
+	    TP_STRUCT__entry(
+			     __field(u32, dev)
+			     ),
+
+	    TP_fast_assign(
+			   __entry->dev = dev->primary->index;
+			   ),
+
+	    TP_printk("dev=%u",
+		      __entry->dev)
+);
+
+#endif /* _V3D_TRACE_H_ */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/v3d/v3d_trace_points.c b/drivers/gpu/drm/v3d/v3d_trace_points.c
new file mode 100644
index 000000000000..482922d7c7e1
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_trace_points.c
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2015 Broadcom */
+
+#include "v3d_drv.h"
+
+#ifndef __CHECKER__
+#define CREATE_TRACE_POINTS
+#include "v3d_trace.h"
+#endif
diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
index c61dff594195..c8650bbcbcb3 100644
--- a/drivers/gpu/drm/vc4/vc4_crtc.c
+++ b/drivers/gpu/drm/vc4/vc4_crtc.c
@@ -42,51 +42,18 @@
 #include "vc4_drv.h"
 #include "vc4_regs.h"
 
-struct vc4_crtc {
-	struct drm_crtc base;
-	const struct vc4_crtc_data *data;
-	void __iomem *regs;
-
-	/* Timestamp at start of vblank irq - unaffected by lock delays. */
-	ktime_t t_vblank;
-
-	/* Which HVS channel we're using for our CRTC. */
-	int channel;
-
-	u8 lut_r[256];
-	u8 lut_g[256];
-	u8 lut_b[256];
-	/* Size in pixels of the COB memory allocated to this CRTC. */
-	u32 cob_size;
-
-	struct drm_pending_vblank_event *event;
-};
-
 struct vc4_crtc_state {
 	struct drm_crtc_state base;
 	/* Dlist area for this CRTC configuration. */
 	struct drm_mm_node mm;
 };
 
-static inline struct vc4_crtc *
-to_vc4_crtc(struct drm_crtc *crtc)
-{
-	return (struct vc4_crtc *)crtc;
-}
-
 static inline struct vc4_crtc_state *
 to_vc4_crtc_state(struct drm_crtc_state *crtc_state)
 {
 	return (struct vc4_crtc_state *)crtc_state;
 }
 
-struct vc4_crtc_data {
-	/* Which channel of the HVS this pixelvalve sources from. */
-	int hvs_channel;
-
-	enum vc4_encoder_type encoder_types[4];
-};
-
 #define CRTC_WRITE(offset, val) writel(val, vc4_crtc->regs + (offset))
 #define CRTC_READ(offset) readl(vc4_crtc->regs + (offset))
 
@@ -298,23 +265,21 @@ vc4_crtc_lut_load(struct drm_crtc *crtc)
 		HVS_WRITE(SCALER_GAMDATA, vc4_crtc->lut_b[i]);
 }
 
-static int
-vc4_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
-		   uint32_t size,
-		   struct drm_modeset_acquire_ctx *ctx)
+static void
+vc4_crtc_update_gamma_lut(struct drm_crtc *crtc)
 {
 	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+	struct drm_color_lut *lut = crtc->state->gamma_lut->data;
+	u32 length = drm_color_lut_size(crtc->state->gamma_lut);
 	u32 i;
 
-	for (i = 0; i < size; i++) {
-		vc4_crtc->lut_r[i] = r[i] >> 8;
-		vc4_crtc->lut_g[i] = g[i] >> 8;
-		vc4_crtc->lut_b[i] = b[i] >> 8;
+	for (i = 0; i < length; i++) {
+		vc4_crtc->lut_r[i] = drm_color_lut_extract(lut[i].red, 8);
+		vc4_crtc->lut_g[i] = drm_color_lut_extract(lut[i].green, 8);
+		vc4_crtc->lut_b[i] = drm_color_lut_extract(lut[i].blue, 8);
 	}
 
 	vc4_crtc_lut_load(crtc);
-
-	return 0;
 }
 
 static u32 vc4_get_fifo_full_level(u32 format)
@@ -699,6 +664,22 @@ static void vc4_crtc_atomic_flush(struct drm_crtc *crtc,
 	if (crtc->state->active && old_state->active)
 		vc4_crtc_update_dlist(crtc);
 
+	if (crtc->state->color_mgmt_changed) {
+		u32 dispbkgndx = HVS_READ(SCALER_DISPBKGNDX(vc4_crtc->channel));
+
+		if (crtc->state->gamma_lut) {
+			vc4_crtc_update_gamma_lut(crtc);
+			dispbkgndx |= SCALER_DISPBKGND_GAMMA;
+		} else {
+			/* Unsetting DISPBKGND_GAMMA skips the gamma lut step
+			 * in hardware, which is the same as a linear lut that
+			 * DRM expects us to use in absence of a user lut.
+			 */
+			dispbkgndx &= ~SCALER_DISPBKGND_GAMMA;
+		}
+		HVS_WRITE(SCALER_DISPBKGNDX(vc4_crtc->channel), dispbkgndx);
+	}
+
 	if (debug_dump_regs) {
 		DRM_INFO("CRTC %d HVS after:\n", drm_crtc_index(crtc));
 		vc4_hvs_dump_state(dev);
@@ -953,7 +934,7 @@ static const struct drm_crtc_funcs vc4_crtc_funcs = {
 	.reset = vc4_crtc_reset,
 	.atomic_duplicate_state = vc4_crtc_duplicate_state,
 	.atomic_destroy_state = vc4_crtc_destroy_state,
-	.gamma_set = vc4_crtc_gamma_set,
+	.gamma_set = drm_atomic_helper_legacy_gamma_set,
 	.enable_vblank = vc4_enable_vblank,
 	.disable_vblank = vc4_disable_vblank,
 };
@@ -1079,6 +1060,12 @@ static int vc4_crtc_bind(struct device *dev, struct device *master, void *data)
 	primary_plane->crtc = crtc;
 	vc4_crtc->channel = vc4_crtc->data->hvs_channel;
 	drm_mode_crtc_set_gamma_size(crtc, ARRAY_SIZE(vc4_crtc->lut_r));
+	drm_crtc_enable_color_mgmt(crtc, 0, false, crtc->gamma_size);
+
+	/* We support CTM, but only for one CRTC at a time. It's therefore
+	 * implemented as private driver state in vc4_kms, not here.
+	 */
+	drm_crtc_enable_color_mgmt(crtc, 0, true, crtc->gamma_size);
 
 	/* Set up some arbitrary number of planes.  We're not limited
 	 * by a set number of physical registers, just the space in
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index 7c95ed5c5cac..466d0a27b415 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -176,7 +176,8 @@ static struct drm_driver vc4_drm_driver = {
 			    DRIVER_GEM |
 			    DRIVER_HAVE_IRQ |
 			    DRIVER_RENDER |
-			    DRIVER_PRIME),
+			    DRIVER_PRIME |
+			    DRIVER_SYNCOBJ),
 	.lastclose = drm_fb_helper_lastclose,
 	.open = vc4_open,
 	.postclose = vc4_close,
@@ -319,8 +320,8 @@ static int vc4_drm_bind(struct device *dev)
 
 static void vc4_drm_unbind(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct drm_device *drm = platform_get_drvdata(pdev);
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct vc4_dev *vc4 = to_vc4_dev(drm);
 
 	drm_dev_unregister(drm);
 
@@ -328,6 +329,8 @@ static void vc4_drm_unbind(struct device *dev)
 
 	drm_mode_config_cleanup(drm);
 
+	drm_atomic_private_obj_fini(&vc4->ctm_manager);
+
 	drm_dev_unref(drm);
 }
 
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 1b4cd1fabf56..554a4e810d5b 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -10,6 +10,8 @@
 #include <drm/drmP.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_syncobj.h>
 
 #include "uapi/drm/vc4_drm.h"
 
@@ -193,6 +195,9 @@ struct vc4_dev {
 	} hangcheck;
 
 	struct semaphore async_modeset;
+
+	struct drm_modeset_lock ctm_state_lock;
+	struct drm_private_obj ctm_manager;
 };
 
 static inline struct vc4_dev *
@@ -392,6 +397,39 @@ to_vc4_encoder(struct drm_encoder *encoder)
 	return container_of(encoder, struct vc4_encoder, base);
 }
 
+struct vc4_crtc_data {
+	/* Which channel of the HVS this pixelvalve sources from. */
+	int hvs_channel;
+
+	enum vc4_encoder_type encoder_types[4];
+};
+
+struct vc4_crtc {
+	struct drm_crtc base;
+	const struct vc4_crtc_data *data;
+	void __iomem *regs;
+
+	/* Timestamp at start of vblank irq - unaffected by lock delays. */
+	ktime_t t_vblank;
+
+	/* Which HVS channel we're using for our CRTC. */
+	int channel;
+
+	u8 lut_r[256];
+	u8 lut_g[256];
+	u8 lut_b[256];
+	/* Size in pixels of the COB memory allocated to this CRTC. */
+	u32 cob_size;
+
+	struct drm_pending_vblank_event *event;
+};
+
+static inline struct vc4_crtc *
+to_vc4_crtc(struct drm_crtc *crtc)
+{
+	return (struct vc4_crtc *)crtc;
+}
+
 #define V3D_READ(offset) readl(vc4->v3d->regs + offset)
 #define V3D_WRITE(offset, val) writel(val, vc4->v3d->regs + offset)
 #define HVS_READ(offset) readl(vc4->hvs->regs + offset)
diff --git a/drivers/gpu/drm/vc4/vc4_dsi.c b/drivers/gpu/drm/vc4/vc4_dsi.c
index 94085f8bcd68..8aa897835118 100644
--- a/drivers/gpu/drm/vc4/vc4_dsi.c
+++ b/drivers/gpu/drm/vc4/vc4_dsi.c
@@ -753,6 +753,11 @@ static void vc4_dsi_ulps(struct vc4_dsi *dsi, bool ulps)
 			 (dsi->lanes > 2 ? DSI1_STAT_PHY_D2_STOP : 0) |
 			 (dsi->lanes > 3 ? DSI1_STAT_PHY_D3_STOP : 0));
 	int ret;
+	bool ulps_currently_enabled = (DSI_PORT_READ(PHY_AFEC0) &
+				       DSI_PORT_BIT(PHY_AFEC0_LATCH_ULPS));
+
+	if (ulps == ulps_currently_enabled)
+		return;
 
 	DSI_PORT_WRITE(STAT, stat_ulps);
 	DSI_PORT_WRITE(PHYC, DSI_PORT_READ(PHYC) | phyc_ulps);
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index 2107b0daf8ef..7910b9acedd6 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -27,6 +27,7 @@
 #include <linux/device.h>
 #include <linux/io.h>
 #include <linux/sched/signal.h>
+#include <linux/dma-fence-array.h>
 
 #include "uapi/drm/vc4_drm.h"
 #include "vc4_drv.h"
@@ -655,7 +656,8 @@ vc4_lock_bo_reservations(struct drm_device *dev,
  */
 static int
 vc4_queue_submit(struct drm_device *dev, struct vc4_exec_info *exec,
-		 struct ww_acquire_ctx *acquire_ctx)
+		 struct ww_acquire_ctx *acquire_ctx,
+		 struct drm_syncobj *out_sync)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	struct vc4_exec_info *renderjob;
@@ -678,6 +680,9 @@ vc4_queue_submit(struct drm_device *dev, struct vc4_exec_info *exec,
 	fence->seqno = exec->seqno;
 	exec->fence = &fence->base;
 
+	if (out_sync)
+		drm_syncobj_replace_fence(out_sync, exec->fence);
+
 	vc4_update_bo_seqnos(exec, seqno);
 
 	vc4_unlock_bo_reservations(dev, exec, acquire_ctx);
@@ -1113,8 +1118,10 @@ vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	struct vc4_file *vc4file = file_priv->driver_priv;
 	struct drm_vc4_submit_cl *args = data;
+	struct drm_syncobj *out_sync = NULL;
 	struct vc4_exec_info *exec;
 	struct ww_acquire_ctx acquire_ctx;
+	struct dma_fence *in_fence;
 	int ret = 0;
 
 	if ((args->flags & ~(VC4_SUBMIT_CL_USE_CLEAR_COLOR |
@@ -1126,7 +1133,7 @@ vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
 	}
 
 	if (args->pad2 != 0) {
-		DRM_DEBUG("->pad2 must be set to zero\n");
+		DRM_DEBUG("Invalid pad: 0x%08x\n", args->pad2);
 		return -EINVAL;
 	}
 
@@ -1164,6 +1171,29 @@ vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
 		}
 	}
 
+	if (args->in_sync) {
+		ret = drm_syncobj_find_fence(file_priv, args->in_sync,
+					     &in_fence);
+		if (ret)
+			goto fail;
+
+		/* When the fence (or fence array) is exclusively from our
+		 * context we can skip the wait since jobs are executed in
+		 * order of their submission through this ioctl and this can
+		 * only have fences from a prior job.
+		 */
+		if (!dma_fence_match_context(in_fence,
+					     vc4->dma_fence_context)) {
+			ret = dma_fence_wait(in_fence, true);
+			if (ret) {
+				dma_fence_put(in_fence);
+				goto fail;
+			}
+		}
+
+		dma_fence_put(in_fence);
+	}
+
 	if (exec->args->bin_cl_size != 0) {
 		ret = vc4_get_bcl(dev, exec);
 		if (ret)
@@ -1181,12 +1211,33 @@ vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
 	if (ret)
 		goto fail;
 
+	if (args->out_sync) {
+		out_sync = drm_syncobj_find(file_priv, args->out_sync);
+		if (!out_sync) {
+			ret = -EINVAL;
+			goto fail;
+		}
+
+		/* We replace the fence in out_sync in vc4_queue_submit since
+		 * the render job could execute immediately after that call.
+		 * If it finishes before our ioctl processing resumes the
+		 * render job fence could already have been freed.
+		 */
+	}
+
 	/* Clear this out of the struct we'll be putting in the queue,
 	 * since it's part of our stack.
 	 */
 	exec->args = NULL;
 
-	ret = vc4_queue_submit(dev, exec, &acquire_ctx);
+	ret = vc4_queue_submit(dev, exec, &acquire_ctx, out_sync);
+
+	/* The syncobj isn't part of the exec data and we need to free our
+	 * reference even if job submission failed.
+	 */
+	if (out_sync)
+		drm_syncobj_put(out_sync);
+
 	if (ret)
 		goto fail;
 
diff --git a/drivers/gpu/drm/vc4/vc4_hvs.c b/drivers/gpu/drm/vc4/vc4_hvs.c
index 2b62fc5b8d85..5d8c749c9749 100644
--- a/drivers/gpu/drm/vc4/vc4_hvs.c
+++ b/drivers/gpu/drm/vc4/vc4_hvs.c
@@ -58,6 +58,10 @@ static const struct {
 	HVS_REG(SCALER_DISPSTAT2),
 	HVS_REG(SCALER_DISPBASE2),
 	HVS_REG(SCALER_DISPALPHA2),
+	HVS_REG(SCALER_OLEDOFFS),
+	HVS_REG(SCALER_OLEDCOEF0),
+	HVS_REG(SCALER_OLEDCOEF1),
+	HVS_REG(SCALER_OLEDCOEF2),
 };
 
 void vc4_hvs_dump_state(struct drm_device *dev)
diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
index ba60153dddb5..8a411e5f8776 100644
--- a/drivers/gpu/drm/vc4/vc4_kms.c
+++ b/drivers/gpu/drm/vc4/vc4_kms.c
@@ -23,6 +23,117 @@
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include "vc4_drv.h"
+#include "vc4_regs.h"
+
+struct vc4_ctm_state {
+	struct drm_private_state base;
+	struct drm_color_ctm *ctm;
+	int fifo;
+};
+
+static struct vc4_ctm_state *to_vc4_ctm_state(struct drm_private_state *priv)
+{
+	return container_of(priv, struct vc4_ctm_state, base);
+}
+
+static struct vc4_ctm_state *vc4_get_ctm_state(struct drm_atomic_state *state,
+					       struct drm_private_obj *manager)
+{
+	struct drm_device *dev = state->dev;
+	struct vc4_dev *vc4 = dev->dev_private;
+	struct drm_private_state *priv_state;
+	int ret;
+
+	ret = drm_modeset_lock(&vc4->ctm_state_lock, state->acquire_ctx);
+	if (ret)
+		return ERR_PTR(ret);
+
+	priv_state = drm_atomic_get_private_obj_state(state, manager);
+	if (IS_ERR(priv_state))
+		return ERR_CAST(priv_state);
+
+	return to_vc4_ctm_state(priv_state);
+}
+
+static struct drm_private_state *
+vc4_ctm_duplicate_state(struct drm_private_obj *obj)
+{
+	struct vc4_ctm_state *state;
+
+	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
+
+	return &state->base;
+}
+
+static void vc4_ctm_destroy_state(struct drm_private_obj *obj,
+				  struct drm_private_state *state)
+{
+	struct vc4_ctm_state *ctm_state = to_vc4_ctm_state(state);
+
+	kfree(ctm_state);
+}
+
+static const struct drm_private_state_funcs vc4_ctm_state_funcs = {
+	.atomic_duplicate_state = vc4_ctm_duplicate_state,
+	.atomic_destroy_state = vc4_ctm_destroy_state,
+};
+
+/* Converts a DRM S31.32 value to the HW S0.9 format. */
+static u16 vc4_ctm_s31_32_to_s0_9(u64 in)
+{
+	u16 r;
+
+	/* Sign bit. */
+	r = in & BIT_ULL(63) ? BIT(9) : 0;
+
+	if ((in & GENMASK_ULL(62, 32)) > 0) {
+		/* We have zero integer bits so we can only saturate here. */
+		r |= GENMASK(8, 0);
+	} else {
+		/* Otherwise take the 9 most important fractional bits. */
+		r |= (in >> 23) & GENMASK(8, 0);
+	}
+
+	return r;
+}
+
+static void
+vc4_ctm_commit(struct vc4_dev *vc4, struct drm_atomic_state *state)
+{
+	struct vc4_ctm_state *ctm_state = to_vc4_ctm_state(vc4->ctm_manager.state);
+	struct drm_color_ctm *ctm = ctm_state->ctm;
+
+	if (ctm_state->fifo) {
+		HVS_WRITE(SCALER_OLEDCOEF2,
+			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[0]),
+					SCALER_OLEDCOEF2_R_TO_R) |
+			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[3]),
+					SCALER_OLEDCOEF2_R_TO_G) |
+			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[6]),
+					SCALER_OLEDCOEF2_R_TO_B));
+		HVS_WRITE(SCALER_OLEDCOEF1,
+			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[1]),
+					SCALER_OLEDCOEF1_G_TO_R) |
+			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[4]),
+					SCALER_OLEDCOEF1_G_TO_G) |
+			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[7]),
+					SCALER_OLEDCOEF1_G_TO_B));
+		HVS_WRITE(SCALER_OLEDCOEF0,
+			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[2]),
+					SCALER_OLEDCOEF0_B_TO_R) |
+			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[5]),
+					SCALER_OLEDCOEF0_B_TO_G) |
+			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[8]),
+					SCALER_OLEDCOEF0_B_TO_B));
+	}
+
+	HVS_WRITE(SCALER_OLEDOFFS,
+		  VC4_SET_FIELD(ctm_state->fifo, SCALER_OLEDOFFS_DISPFIFO));
+}
 
 static void
 vc4_atomic_complete_commit(struct drm_atomic_state *state)
@@ -36,6 +147,8 @@ vc4_atomic_complete_commit(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_modeset_disables(dev, state);
 
+	vc4_ctm_commit(vc4, state);
+
 	drm_atomic_helper_commit_planes(dev, state, 0);
 
 	drm_atomic_helper_commit_modeset_enables(dev, state);
@@ -90,6 +203,26 @@ static int vc4_atomic_commit(struct drm_device *dev,
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	int ret;
 
+	if (state->async_update) {
+		ret = down_interruptible(&vc4->async_modeset);
+		if (ret)
+			return ret;
+
+		ret = drm_atomic_helper_prepare_planes(dev, state);
+		if (ret) {
+			up(&vc4->async_modeset);
+			return ret;
+		}
+
+		drm_atomic_helper_async_commit(dev, state);
+
+		drm_atomic_helper_cleanup_planes(dev, state);
+
+		up(&vc4->async_modeset);
+
+		return 0;
+	}
+
 	ret = drm_atomic_helper_setup_commit(state, nonblock);
 	if (ret)
 		return ret;
@@ -187,9 +320,89 @@ static struct drm_framebuffer *vc4_fb_create(struct drm_device *dev,
 	return drm_gem_fb_create(dev, file_priv, mode_cmd);
 }
 
+/* Our CTM has some peculiar limitations: we can only enable it for one CRTC
+ * at a time and the HW only supports S0.9 scalars. To account for the latter,
+ * we don't allow userland to set a CTM that we have no hope of approximating.
+ */
+static int
+vc4_ctm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct vc4_ctm_state *ctm_state = NULL;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
+	struct drm_color_ctm *ctm;
+	int i;
+
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
+		/* CTM is being disabled. */
+		if (!new_crtc_state->ctm && old_crtc_state->ctm) {
+			ctm_state = vc4_get_ctm_state(state, &vc4->ctm_manager);
+			if (IS_ERR(ctm_state))
+				return PTR_ERR(ctm_state);
+			ctm_state->fifo = 0;
+		}
+	}
+
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
+		if (new_crtc_state->ctm == old_crtc_state->ctm)
+			continue;
+
+		if (!ctm_state) {
+			ctm_state = vc4_get_ctm_state(state, &vc4->ctm_manager);
+			if (IS_ERR(ctm_state))
+				return PTR_ERR(ctm_state);
+		}
+
+		/* CTM is being enabled or the matrix changed. */
+		if (new_crtc_state->ctm) {
+			/* fifo is 1-based since 0 disables CTM. */
+			int fifo = to_vc4_crtc(crtc)->channel + 1;
+
+			/* Check userland isn't trying to turn on CTM for more
+			 * than one CRTC at a time.
+			 */
+			if (ctm_state->fifo && ctm_state->fifo != fifo) {
+				DRM_DEBUG_DRIVER("Too many CTM configured\n");
+				return -EINVAL;
+			}
+
+			/* Check we can approximate the specified CTM.
+			 * We disallow scalars |c| > 1.0 since the HW has
+			 * no integer bits.
+			 */
+			ctm = new_crtc_state->ctm->data;
+			for (i = 0; i < ARRAY_SIZE(ctm->matrix); i++) {
+				u64 val = ctm->matrix[i];
+
+				val &= ~BIT_ULL(63);
+				if (val > BIT_ULL(32))
+					return -EINVAL;
+			}
+
+			ctm_state->fifo = fifo;
+			ctm_state->ctm = ctm;
+		}
+	}
+
+	return 0;
+}
+
+static int
+vc4_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
+{
+	int ret;
+
+	ret = vc4_ctm_atomic_check(dev, state);
+	if (ret < 0)
+		return ret;
+
+	return drm_atomic_helper_check(dev, state);
+}
+
 static const struct drm_mode_config_funcs vc4_mode_funcs = {
 	.output_poll_changed = drm_fb_helper_output_poll_changed,
-	.atomic_check = drm_atomic_helper_check,
+	.atomic_check = vc4_atomic_check,
 	.atomic_commit = vc4_atomic_commit,
 	.fb_create = vc4_fb_create,
 };
@@ -197,6 +410,7 @@ static const struct drm_mode_config_funcs vc4_mode_funcs = {
 int vc4_kms_load(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct vc4_ctm_state *ctm_state;
 	int ret;
 
 	sema_init(&vc4->async_modeset, 1);
@@ -217,6 +431,14 @@ int vc4_kms_load(struct drm_device *dev)
 	dev->mode_config.async_page_flip = true;
 	dev->mode_config.allow_fb_modifiers = true;
 
+	drm_modeset_lock_init(&vc4->ctm_state_lock);
+
+	ctm_state = kzalloc(sizeof(*ctm_state), GFP_KERNEL);
+	if (!ctm_state)
+		return -ENOMEM;
+	drm_atomic_private_obj_init(&vc4->ctm_manager, &ctm_state->base,
+				    &vc4_ctm_state_funcs);
+
 	drm_mode_config_reset(dev);
 
 	if (dev->mode_config.num_connector)
diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index 13dcaad06798..71d44c357d35 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -201,6 +201,7 @@ static void vc4_plane_reset(struct drm_plane *plane)
 		return;
 
 	plane->state = &vc4_state->base;
+	plane->state->alpha = DRM_BLEND_ALPHA_OPAQUE;
 	vc4_state->base.plane = plane;
 }
 
@@ -467,6 +468,7 @@ static int vc4_plane_mode_set(struct drm_plane *plane,
 	u32 ctl0_offset = vc4_state->dlist_count;
 	const struct hvs_format *format = vc4_get_hvs_format(fb->format->format);
 	int num_planes = drm_format_num_planes(format->drm);
+	bool mix_plane_alpha;
 	bool covers_screen;
 	u32 scl0, scl1, pitch0;
 	u32 lbm_size, tiling;
@@ -552,7 +554,7 @@ static int vc4_plane_mode_set(struct drm_plane *plane,
 	/* Position Word 0: Image Positions and Alpha Value */
 	vc4_state->pos0_offset = vc4_state->dlist_count;
 	vc4_dlist_write(vc4_state,
-			VC4_SET_FIELD(0xff, SCALER_POS0_FIXED_ALPHA) |
+			VC4_SET_FIELD(state->alpha >> 8, SCALER_POS0_FIXED_ALPHA) |
 			VC4_SET_FIELD(vc4_state->crtc_x, SCALER_POS0_START_X) |
 			VC4_SET_FIELD(vc4_state->crtc_y, SCALER_POS0_START_Y));
 
@@ -565,6 +567,13 @@ static int vc4_plane_mode_set(struct drm_plane *plane,
 					      SCALER_POS1_SCL_HEIGHT));
 	}
 
+	/* Don't waste cycles mixing with plane alpha if the set alpha
+	 * is opaque or there is no per-pixel alpha information.
+	 * In any case we use the alpha property value as the fixed alpha.
+	 */
+	mix_plane_alpha = state->alpha != DRM_BLEND_ALPHA_OPAQUE &&
+			  fb->format->has_alpha;
+
 	/* Position Word 2: Source Image Size, Alpha */
 	vc4_state->pos2_offset = vc4_state->dlist_count;
 	vc4_dlist_write(vc4_state,
@@ -572,6 +581,7 @@ static int vc4_plane_mode_set(struct drm_plane *plane,
 				      SCALER_POS2_ALPHA_MODE_PIPELINE :
 				      SCALER_POS2_ALPHA_MODE_FIXED,
 				      SCALER_POS2_ALPHA_MODE) |
+			(mix_plane_alpha ? SCALER_POS2_ALPHA_MIX : 0) |
 			(fb->format->has_alpha ? SCALER_POS2_ALPHA_PREMULT : 0) |
 			VC4_SET_FIELD(vc4_state->src_w[0], SCALER_POS2_WIDTH) |
 			VC4_SET_FIELD(vc4_state->src_h[0], SCALER_POS2_HEIGHT));
@@ -653,10 +663,11 @@ static int vc4_plane_mode_set(struct drm_plane *plane,
 			vc4_state->crtc_w == state->crtc->mode.hdisplay &&
 			vc4_state->crtc_h == state->crtc->mode.vdisplay;
 	/* Background fill might be necessary when the plane has per-pixel
-	 * alpha content and blends from the background or does not cover
-	 * the entire screen.
+	 * alpha content or a non-opaque plane alpha and could blend from the
+	 * background or does not cover the entire screen.
 	 */
-	vc4_state->needs_bg_fill = fb->format->has_alpha || !covers_screen;
+	vc4_state->needs_bg_fill = fb->format->has_alpha || !covers_screen ||
+				   state->alpha != DRM_BLEND_ALPHA_OPAQUE;
 
 	return 0;
 }
@@ -741,6 +752,57 @@ void vc4_plane_async_set_fb(struct drm_plane *plane, struct drm_framebuffer *fb)
 	vc4_state->dlist[vc4_state->ptr0_offset] = addr;
 }
 
+static void vc4_plane_atomic_async_update(struct drm_plane *plane,
+					  struct drm_plane_state *state)
+{
+	struct vc4_plane_state *vc4_state = to_vc4_plane_state(plane->state);
+
+	if (plane->state->fb != state->fb) {
+		vc4_plane_async_set_fb(plane, state->fb);
+		drm_atomic_set_fb_for_plane(plane->state, state->fb);
+	}
+
+	/* Set the cursor's position on the screen.  This is the
+	 * expected change from the drm_mode_cursor_universal()
+	 * helper.
+	 */
+	plane->state->crtc_x = state->crtc_x;
+	plane->state->crtc_y = state->crtc_y;
+
+	/* Allow changing the start position within the cursor BO, if
+	 * that matters.
+	 */
+	plane->state->src_x = state->src_x;
+	plane->state->src_y = state->src_y;
+
+	/* Update the display list based on the new crtc_x/y. */
+	vc4_plane_atomic_check(plane, plane->state);
+
+	/* Note that we can't just call vc4_plane_write_dlist()
+	 * because that would smash the context data that the HVS is
+	 * currently using.
+	 */
+	writel(vc4_state->dlist[vc4_state->pos0_offset],
+	       &vc4_state->hw_dlist[vc4_state->pos0_offset]);
+	writel(vc4_state->dlist[vc4_state->pos2_offset],
+	       &vc4_state->hw_dlist[vc4_state->pos2_offset]);
+	writel(vc4_state->dlist[vc4_state->ptr0_offset],
+	       &vc4_state->hw_dlist[vc4_state->ptr0_offset]);
+}
+
+static int vc4_plane_atomic_async_check(struct drm_plane *plane,
+					struct drm_plane_state *state)
+{
+	/* No configuring new scaling in the fast path. */
+	if (plane->state->crtc_w != state->crtc_w ||
+	    plane->state->crtc_h != state->crtc_h ||
+	    plane->state->src_w != state->src_w ||
+	    plane->state->src_h != state->src_h)
+		return -EINVAL;
+
+	return 0;
+}
+
 static int vc4_prepare_fb(struct drm_plane *plane,
 			  struct drm_plane_state *state)
 {
@@ -780,6 +842,8 @@ static const struct drm_plane_helper_funcs vc4_plane_helper_funcs = {
 	.atomic_update = vc4_plane_atomic_update,
 	.prepare_fb = vc4_prepare_fb,
 	.cleanup_fb = vc4_cleanup_fb,
+	.atomic_async_check = vc4_plane_atomic_async_check,
+	.atomic_async_update = vc4_plane_atomic_async_update,
 };
 
 static void vc4_plane_destroy(struct drm_plane *plane)
@@ -788,82 +852,6 @@ static void vc4_plane_destroy(struct drm_plane *plane)
 	drm_plane_cleanup(plane);
 }
 
-/* Implements immediate (non-vblank-synced) updates of the cursor
- * position, or falls back to the atomic helper otherwise.
- */
-static int
-vc4_update_plane(struct drm_plane *plane,
-		 struct drm_crtc *crtc,
-		 struct drm_framebuffer *fb,
-		 int crtc_x, int crtc_y,
-		 unsigned int crtc_w, unsigned int crtc_h,
-		 uint32_t src_x, uint32_t src_y,
-		 uint32_t src_w, uint32_t src_h,
-		 struct drm_modeset_acquire_ctx *ctx)
-{
-	struct drm_plane_state *plane_state;
-	struct vc4_plane_state *vc4_state;
-
-	if (plane != crtc->cursor)
-		goto out;
-
-	plane_state = plane->state;
-	vc4_state = to_vc4_plane_state(plane_state);
-
-	if (!plane_state)
-		goto out;
-
-	/* No configuring new scaling in the fast path. */
-	if (crtc_w != plane_state->crtc_w ||
-	    crtc_h != plane_state->crtc_h ||
-	    src_w != plane_state->src_w ||
-	    src_h != plane_state->src_h) {
-		goto out;
-	}
-
-	if (fb != plane_state->fb) {
-		drm_atomic_set_fb_for_plane(plane->state, fb);
-		vc4_plane_async_set_fb(plane, fb);
-	}
-
-	/* Set the cursor's position on the screen.  This is the
-	 * expected change from the drm_mode_cursor_universal()
-	 * helper.
-	 */
-	plane_state->crtc_x = crtc_x;
-	plane_state->crtc_y = crtc_y;
-
-	/* Allow changing the start position within the cursor BO, if
-	 * that matters.
-	 */
-	plane_state->src_x = src_x;
-	plane_state->src_y = src_y;
-
-	/* Update the display list based on the new crtc_x/y. */
-	vc4_plane_atomic_check(plane, plane_state);
-
-	/* Note that we can't just call vc4_plane_write_dlist()
-	 * because that would smash the context data that the HVS is
-	 * currently using.
-	 */
-	writel(vc4_state->dlist[vc4_state->pos0_offset],
-	       &vc4_state->hw_dlist[vc4_state->pos0_offset]);
-	writel(vc4_state->dlist[vc4_state->pos2_offset],
-	       &vc4_state->hw_dlist[vc4_state->pos2_offset]);
-	writel(vc4_state->dlist[vc4_state->ptr0_offset],
-	       &vc4_state->hw_dlist[vc4_state->ptr0_offset]);
-
-	return 0;
-
-out:
-	return drm_atomic_helper_update_plane(plane, crtc, fb,
-					      crtc_x, crtc_y,
-					      crtc_w, crtc_h,
-					      src_x, src_y,
-					      src_w, src_h,
-					      ctx);
-}
-
 static bool vc4_format_mod_supported(struct drm_plane *plane,
 				     uint32_t format,
 				     uint64_t modifier)
@@ -891,7 +879,7 @@ static bool vc4_format_mod_supported(struct drm_plane *plane,
 }
 
 static const struct drm_plane_funcs vc4_plane_funcs = {
-	.update_plane = vc4_update_plane,
+	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = vc4_plane_destroy,
 	.set_property = NULL,
@@ -939,5 +927,7 @@ struct drm_plane *vc4_plane_init(struct drm_device *dev,
 
 	drm_plane_helper_add(plane, &vc4_plane_helper_funcs);
 
+	drm_plane_create_alpha_property(plane);
+
 	return plane;
 }
diff --git a/drivers/gpu/drm/vc4/vc4_regs.h b/drivers/gpu/drm/vc4/vc4_regs.h
index a141496104a6..d1fb6fec46eb 100644
--- a/drivers/gpu/drm/vc4/vc4_regs.h
+++ b/drivers/gpu/drm/vc4/vc4_regs.h
@@ -330,6 +330,21 @@
 #define SCALER_DISPCTRL0                        0x00000040
 # define SCALER_DISPCTRLX_ENABLE		BIT(31)
 # define SCALER_DISPCTRLX_RESET			BIT(30)
+/* Generates a single frame when VSTART is seen and stops at the last
+ * pixel read from the FIFO.
+ */
+# define SCALER_DISPCTRLX_ONESHOT		BIT(29)
+/* Processes a single context in the dlist and then task switch,
+ * instead of an entire line.
+ */
+# define SCALER_DISPCTRLX_ONECTX		BIT(28)
+/* Set to have DISPSLAVE return 2 16bpp pixels and no status data. */
+# define SCALER_DISPCTRLX_FIFO32		BIT(27)
+/* Turns on output to the DISPSLAVE register instead of the normal
+ * FIFO.
+ */
+# define SCALER_DISPCTRLX_FIFOREG		BIT(26)
+
 # define SCALER_DISPCTRLX_WIDTH_MASK		VC4_MASK(23, 12)
 # define SCALER_DISPCTRLX_WIDTH_SHIFT		12
 # define SCALER_DISPCTRLX_HEIGHT_MASK		VC4_MASK(11, 0)
@@ -402,6 +417,68 @@
  */
 # define SCALER_GAMADDR_SRAMENB			BIT(30)
 
+#define SCALER_OLEDOFFS                         0x00000080
+/* Clamps R to [16,235] and G/B to [16,240]. */
+# define SCALER_OLEDOFFS_YUVCLAMP               BIT(31)
+
+/* Chooses which display FIFO the matrix applies to. */
+# define SCALER_OLEDOFFS_DISPFIFO_MASK          VC4_MASK(25, 24)
+# define SCALER_OLEDOFFS_DISPFIFO_SHIFT         24
+# define SCALER_OLEDOFFS_DISPFIFO_DISABLED      0
+# define SCALER_OLEDOFFS_DISPFIFO_0             1
+# define SCALER_OLEDOFFS_DISPFIFO_1             2
+# define SCALER_OLEDOFFS_DISPFIFO_2             3
+
+/* Offsets are 8-bit 2s-complement. */
+# define SCALER_OLEDOFFS_RED_MASK               VC4_MASK(23, 16)
+# define SCALER_OLEDOFFS_RED_SHIFT              16
+# define SCALER_OLEDOFFS_GREEN_MASK             VC4_MASK(15, 8)
+# define SCALER_OLEDOFFS_GREEN_SHIFT            8
+# define SCALER_OLEDOFFS_BLUE_MASK              VC4_MASK(7, 0)
+# define SCALER_OLEDOFFS_BLUE_SHIFT             0
+
+/* The coefficients are S0.9 fractions. */
+#define SCALER_OLEDCOEF0                        0x00000084
+# define SCALER_OLEDCOEF0_B_TO_R_MASK           VC4_MASK(29, 20)
+# define SCALER_OLEDCOEF0_B_TO_R_SHIFT          20
+# define SCALER_OLEDCOEF0_B_TO_G_MASK           VC4_MASK(19, 10)
+# define SCALER_OLEDCOEF0_B_TO_G_SHIFT          10
+# define SCALER_OLEDCOEF0_B_TO_B_MASK           VC4_MASK(9, 0)
+# define SCALER_OLEDCOEF0_B_TO_B_SHIFT          0
+
+#define SCALER_OLEDCOEF1                        0x00000088
+# define SCALER_OLEDCOEF1_G_TO_R_MASK           VC4_MASK(29, 20)
+# define SCALER_OLEDCOEF1_G_TO_R_SHIFT          20
+# define SCALER_OLEDCOEF1_G_TO_G_MASK           VC4_MASK(19, 10)
+# define SCALER_OLEDCOEF1_G_TO_G_SHIFT          10
+# define SCALER_OLEDCOEF1_G_TO_B_MASK           VC4_MASK(9, 0)
+# define SCALER_OLEDCOEF1_G_TO_B_SHIFT          0
+
+#define SCALER_OLEDCOEF2                        0x0000008c
+# define SCALER_OLEDCOEF2_R_TO_R_MASK           VC4_MASK(29, 20)
+# define SCALER_OLEDCOEF2_R_TO_R_SHIFT          20
+# define SCALER_OLEDCOEF2_R_TO_G_MASK           VC4_MASK(19, 10)
+# define SCALER_OLEDCOEF2_R_TO_G_SHIFT          10
+# define SCALER_OLEDCOEF2_R_TO_B_MASK           VC4_MASK(9, 0)
+# define SCALER_OLEDCOEF2_R_TO_B_SHIFT          0
+
+/* Slave addresses for DMAing from HVS composition output to other
+ * devices.  The top bits are valid only in !FIFO32 mode.
+ */
+#define SCALER_DISPSLAVE0                       0x000000c0
+#define SCALER_DISPSLAVE1                       0x000000c9
+#define SCALER_DISPSLAVE2                       0x000000d0
+# define SCALER_DISPSLAVE_ISSUE_VSTART          BIT(31)
+# define SCALER_DISPSLAVE_ISSUE_HSTART          BIT(30)
+/* Set when the current line has been read and an HSTART is required. */
+# define SCALER_DISPSLAVE_EOL                   BIT(26)
+/* Set when the display FIFO is empty. */
+# define SCALER_DISPSLAVE_EMPTY                 BIT(25)
+/* Set when there is RGB data ready to read. */
+# define SCALER_DISPSLAVE_VALID                 BIT(24)
+# define SCALER_DISPSLAVE_RGB_MASK              VC4_MASK(23, 0)
+# define SCALER_DISPSLAVE_RGB_SHIFT             0
+
 #define SCALER_GAMDATA                          0x000000e0
 #define SCALER_DLIST_START                      0x00002000
 #define SCALER_DLIST_SIZE                       0x00004000
@@ -767,6 +844,10 @@ enum hvs_pixel_format {
 	HVS_PIXEL_FORMAT_YCBCR_YUV420_2PLANE = 9,
 	HVS_PIXEL_FORMAT_YCBCR_YUV422_3PLANE = 10,
 	HVS_PIXEL_FORMAT_YCBCR_YUV422_2PLANE = 11,
+	HVS_PIXEL_FORMAT_H264 = 12,
+	HVS_PIXEL_FORMAT_PALETTE = 13,
+	HVS_PIXEL_FORMAT_YUV444_RGB = 14,
+	HVS_PIXEL_FORMAT_AYUV444_RGB = 15,
 };
 
 /* Note: the LSB is the rightmost character shown.  Only valid for
@@ -800,12 +881,27 @@ enum hvs_pixel_format {
 #define SCALER_CTL0_TILING_128B			2
 #define SCALER_CTL0_TILING_256B_OR_T		3
 
+#define SCALER_CTL0_ALPHA_MASK                  BIT(19)
 #define SCALER_CTL0_HFLIP                       BIT(16)
 #define SCALER_CTL0_VFLIP                       BIT(15)
 
+#define SCALER_CTL0_KEY_MODE_MASK		VC4_MASK(18, 17)
+#define SCALER_CTL0_KEY_MODE_SHIFT		17
+#define SCALER_CTL0_KEY_DISABLED		0
+#define SCALER_CTL0_KEY_LUMA_OR_COMMON_RGB	1
+#define SCALER_CTL0_KEY_MATCH			2 /* turn transparent */
+#define SCALER_CTL0_KEY_REPLACE			3 /* replace with value from key mask word 2 */
+
 #define SCALER_CTL0_ORDER_MASK			VC4_MASK(14, 13)
 #define SCALER_CTL0_ORDER_SHIFT			13
 
+#define SCALER_CTL0_RGBA_EXPAND_MASK		VC4_MASK(12, 11)
+#define SCALER_CTL0_RGBA_EXPAND_SHIFT		11
+#define SCALER_CTL0_RGBA_EXPAND_ZERO		0
+#define SCALER_CTL0_RGBA_EXPAND_LSB		1
+#define SCALER_CTL0_RGBA_EXPAND_MSB		2
+#define SCALER_CTL0_RGBA_EXPAND_ROUND		3
+
 #define SCALER_CTL0_SCL1_MASK			VC4_MASK(10, 8)
 #define SCALER_CTL0_SCL1_SHIFT			8
 
@@ -849,6 +945,7 @@ enum hvs_pixel_format {
 #define SCALER_POS2_ALPHA_MODE_FIXED_NONZERO	2
 #define SCALER_POS2_ALPHA_MODE_FIXED_OVER_0x07	3
 #define SCALER_POS2_ALPHA_PREMULT		BIT(29)
+#define SCALER_POS2_ALPHA_MIX			BIT(28)
 
 #define SCALER_POS2_HEIGHT_MASK			VC4_MASK(27, 16)
 #define SCALER_POS2_HEIGHT_SHIFT		16
diff --git a/drivers/gpu/drm/vc4/vc4_v3d.c b/drivers/gpu/drm/vc4/vc4_v3d.c
index bfc2fa73d2ae..e47e29426078 100644
--- a/drivers/gpu/drm/vc4/vc4_v3d.c
+++ b/drivers/gpu/drm/vc4/vc4_v3d.c
@@ -218,8 +218,7 @@ int vc4_v3d_get_bin_slot(struct vc4_dev *vc4)
  * overall CMA pool before they make scenes complicated enough to run
  * out of bin space.
  */
-int
-vc4_allocate_bin_bo(struct drm_device *drm)
+static int vc4_allocate_bin_bo(struct drm_device *drm)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(drm);
 	struct vc4_v3d *v3d = vc4->v3d;
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index 8cc8c34d67f5..a5edd86603d9 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -208,7 +208,7 @@ static int virtio_gpu_conn_get_modes(struct drm_connector *connector)
 	return count;
 }
 
-static int virtio_gpu_conn_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status virtio_gpu_conn_mode_valid(struct drm_connector *connector,
 				      struct drm_display_mode *mode)
 {
 	struct virtio_gpu_output *output =
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 8b770a8e02cd..09cc721160c4 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -159,14 +159,14 @@ static const struct drm_ioctl_desc vmw_ioctls[] = {
 		      DRM_RENDER_ALLOW),
 	VMW_IOCTL_DEF(VMW_CURSOR_BYPASS,
 		      vmw_kms_cursor_bypass_ioctl,
-		      DRM_MASTER | DRM_CONTROL_ALLOW),
+		      DRM_MASTER),
 
 	VMW_IOCTL_DEF(VMW_CONTROL_STREAM, vmw_overlay_ioctl,
-		      DRM_MASTER | DRM_CONTROL_ALLOW),
+		      DRM_MASTER),
 	VMW_IOCTL_DEF(VMW_CLAIM_STREAM, vmw_stream_claim_ioctl,
-		      DRM_MASTER | DRM_CONTROL_ALLOW),
+		      DRM_MASTER),
 	VMW_IOCTL_DEF(VMW_UNREF_STREAM, vmw_stream_unref_ioctl,
-		      DRM_MASTER | DRM_CONTROL_ALLOW),
+		      DRM_MASTER),
 
 	VMW_IOCTL_DEF(VMW_CREATE_CONTEXT, vmw_context_define_ioctl,
 		      DRM_AUTH | DRM_RENDER_ALLOW),
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 96fd7a03d2f8..01f2dc9e6f52 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -384,9 +384,9 @@ vmw_du_cursor_plane_atomic_update(struct drm_plane *plane,
 	hotspot_x = du->hotspot_x;
 	hotspot_y = du->hotspot_y;
 
-	if (plane->fb) {
-		hotspot_x += plane->fb->hot_x;
-		hotspot_y += plane->fb->hot_y;
+	if (plane->state->fb) {
+		hotspot_x += plane->state->fb->hot_x;
+		hotspot_y += plane->state->fb->hot_y;
 	}
 
 	du->cursor_surface = vps->surf;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index 3824595fece1..4a5907e3f560 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -281,39 +281,6 @@ drm_connector_helper_funcs vmw_ldu_connector_helper_funcs = {
  * Legacy Display Plane Functions
  */
 
-/**
- * vmw_ldu_primary_plane_cleanup_fb - Noop
- *
- * @plane:  display plane
- * @old_state: Contains the FB to clean up
- *
- * Unpins the display surface
- *
- * Returns 0 on success
- */
-static void
-vmw_ldu_primary_plane_cleanup_fb(struct drm_plane *plane,
-				 struct drm_plane_state *old_state)
-{
-}
-
-
-/**
- * vmw_ldu_primary_plane_prepare_fb - Noop
- *
- * @plane:  display plane
- * @new_state: info on the new plane state, including the FB
- *
- * Returns 0 on success
- */
-static int
-vmw_ldu_primary_plane_prepare_fb(struct drm_plane *plane,
-				 struct drm_plane_state *new_state)
-{
-	return 0;
-}
-
-
 static void
 vmw_ldu_primary_plane_atomic_update(struct drm_plane *plane,
 				    struct drm_plane_state *old_state)
@@ -373,8 +340,6 @@ static const struct
 drm_plane_helper_funcs vmw_ldu_primary_plane_helper_funcs = {
 	.atomic_check = vmw_du_primary_plane_atomic_check,
 	.atomic_update = vmw_ldu_primary_plane_atomic_update,
-	.prepare_fb = vmw_ldu_primary_plane_prepare_fb,
-	.cleanup_fb = vmw_ldu_primary_plane_cleanup_fb,
 };
 
 static const struct drm_crtc_helper_funcs vmw_ldu_crtc_helper_funcs = {
diff --git a/drivers/gpu/drm/xen/Kconfig b/drivers/gpu/drm/xen/Kconfig
new file mode 100644
index 000000000000..4cca160782ab
--- /dev/null
+++ b/drivers/gpu/drm/xen/Kconfig
@@ -0,0 +1,17 @@
+config DRM_XEN
+	bool "DRM Support for Xen guest OS"
+	depends on XEN
+	help
+	  Choose this option if you want to enable DRM support
+	  for Xen.
+
+config DRM_XEN_FRONTEND
+	tristate "Para-virtualized frontend driver for Xen guest OS"
+	depends on DRM_XEN
+	depends on DRM
+	select DRM_KMS_HELPER
+	select VIDEOMODE_HELPERS
+	select XEN_XENBUS_FRONTEND
+	help
+	  Choose this option if you want to enable a para-virtualized
+	  frontend DRM/KMS driver for Xen guest OSes.
diff --git a/drivers/gpu/drm/xen/Makefile b/drivers/gpu/drm/xen/Makefile
new file mode 100644
index 000000000000..712afff5ffc3
--- /dev/null
+++ b/drivers/gpu/drm/xen/Makefile
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0 OR MIT
+
+drm_xen_front-objs := xen_drm_front.o \
+		      xen_drm_front_kms.o \
+		      xen_drm_front_conn.o \
+		      xen_drm_front_evtchnl.o \
+		      xen_drm_front_shbuf.o \
+		      xen_drm_front_cfg.o \
+		      xen_drm_front_gem.o
+
+obj-$(CONFIG_DRM_XEN_FRONTEND) += drm_xen_front.o
diff --git a/drivers/gpu/drm/xen/xen_drm_front.c b/drivers/gpu/drm/xen/xen_drm_front.c
new file mode 100644
index 000000000000..b3786c1a4e80
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front.c
@@ -0,0 +1,839 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_gem.h>
+
+#include <linux/of_device.h>
+
+#include <xen/platform_pci.h>
+#include <xen/xen.h>
+#include <xen/xenbus.h>
+
+#include <xen/interface/io/displif.h>
+
+#include "xen_drm_front.h"
+#include "xen_drm_front_cfg.h"
+#include "xen_drm_front_evtchnl.h"
+#include "xen_drm_front_gem.h"
+#include "xen_drm_front_kms.h"
+#include "xen_drm_front_shbuf.h"
+
+struct xen_drm_front_dbuf {
+	struct list_head list;
+	u64 dbuf_cookie;
+	u64 fb_cookie;
+	struct xen_drm_front_shbuf *shbuf;
+};
+
+static int dbuf_add_to_list(struct xen_drm_front_info *front_info,
+			    struct xen_drm_front_shbuf *shbuf, u64 dbuf_cookie)
+{
+	struct xen_drm_front_dbuf *dbuf;
+
+	dbuf = kzalloc(sizeof(*dbuf), GFP_KERNEL);
+	if (!dbuf)
+		return -ENOMEM;
+
+	dbuf->dbuf_cookie = dbuf_cookie;
+	dbuf->shbuf = shbuf;
+	list_add(&dbuf->list, &front_info->dbuf_list);
+	return 0;
+}
+
+static struct xen_drm_front_dbuf *dbuf_get(struct list_head *dbuf_list,
+					   u64 dbuf_cookie)
+{
+	struct xen_drm_front_dbuf *buf, *q;
+
+	list_for_each_entry_safe(buf, q, dbuf_list, list)
+		if (buf->dbuf_cookie == dbuf_cookie)
+			return buf;
+
+	return NULL;
+}
+
+static void dbuf_flush_fb(struct list_head *dbuf_list, u64 fb_cookie)
+{
+	struct xen_drm_front_dbuf *buf, *q;
+
+	list_for_each_entry_safe(buf, q, dbuf_list, list)
+		if (buf->fb_cookie == fb_cookie)
+			xen_drm_front_shbuf_flush(buf->shbuf);
+}
+
+static void dbuf_free(struct list_head *dbuf_list, u64 dbuf_cookie)
+{
+	struct xen_drm_front_dbuf *buf, *q;
+
+	list_for_each_entry_safe(buf, q, dbuf_list, list)
+		if (buf->dbuf_cookie == dbuf_cookie) {
+			list_del(&buf->list);
+			xen_drm_front_shbuf_unmap(buf->shbuf);
+			xen_drm_front_shbuf_free(buf->shbuf);
+			kfree(buf);
+			break;
+		}
+}
+
+static void dbuf_free_all(struct list_head *dbuf_list)
+{
+	struct xen_drm_front_dbuf *buf, *q;
+
+	list_for_each_entry_safe(buf, q, dbuf_list, list) {
+		list_del(&buf->list);
+		xen_drm_front_shbuf_unmap(buf->shbuf);
+		xen_drm_front_shbuf_free(buf->shbuf);
+		kfree(buf);
+	}
+}
+
+static struct xendispl_req *
+be_prepare_req(struct xen_drm_front_evtchnl *evtchnl, u8 operation)
+{
+	struct xendispl_req *req;
+
+	req = RING_GET_REQUEST(&evtchnl->u.req.ring,
+			       evtchnl->u.req.ring.req_prod_pvt);
+	req->operation = operation;
+	req->id = evtchnl->evt_next_id++;
+	evtchnl->evt_id = req->id;
+	return req;
+}
+
+static int be_stream_do_io(struct xen_drm_front_evtchnl *evtchnl,
+			   struct xendispl_req *req)
+{
+	reinit_completion(&evtchnl->u.req.completion);
+	if (unlikely(evtchnl->state != EVTCHNL_STATE_CONNECTED))
+		return -EIO;
+
+	xen_drm_front_evtchnl_flush(evtchnl);
+	return 0;
+}
+
+static int be_stream_wait_io(struct xen_drm_front_evtchnl *evtchnl)
+{
+	if (wait_for_completion_timeout(&evtchnl->u.req.completion,
+			msecs_to_jiffies(XEN_DRM_FRONT_WAIT_BACK_MS)) <= 0)
+		return -ETIMEDOUT;
+
+	return evtchnl->u.req.resp_status;
+}
+
+int xen_drm_front_mode_set(struct xen_drm_front_drm_pipeline *pipeline,
+			   u32 x, u32 y, u32 width, u32 height,
+			   u32 bpp, u64 fb_cookie)
+{
+	struct xen_drm_front_evtchnl *evtchnl;
+	struct xen_drm_front_info *front_info;
+	struct xendispl_req *req;
+	unsigned long flags;
+	int ret;
+
+	front_info = pipeline->drm_info->front_info;
+	evtchnl = &front_info->evt_pairs[pipeline->index].req;
+	if (unlikely(!evtchnl))
+		return -EIO;
+
+	mutex_lock(&evtchnl->u.req.req_io_lock);
+
+	spin_lock_irqsave(&front_info->io_lock, flags);
+	req = be_prepare_req(evtchnl, XENDISPL_OP_SET_CONFIG);
+	req->op.set_config.x = x;
+	req->op.set_config.y = y;
+	req->op.set_config.width = width;
+	req->op.set_config.height = height;
+	req->op.set_config.bpp = bpp;
+	req->op.set_config.fb_cookie = fb_cookie;
+
+	ret = be_stream_do_io(evtchnl, req);
+	spin_unlock_irqrestore(&front_info->io_lock, flags);
+
+	if (ret == 0)
+		ret = be_stream_wait_io(evtchnl);
+
+	mutex_unlock(&evtchnl->u.req.req_io_lock);
+	return ret;
+}
+
+int xen_drm_front_dbuf_create(struct xen_drm_front_info *front_info,
+			      u64 dbuf_cookie, u32 width, u32 height,
+			      u32 bpp, u64 size, struct page **pages)
+{
+	struct xen_drm_front_evtchnl *evtchnl;
+	struct xen_drm_front_shbuf *shbuf;
+	struct xendispl_req *req;
+	struct xen_drm_front_shbuf_cfg buf_cfg;
+	unsigned long flags;
+	int ret;
+
+	evtchnl = &front_info->evt_pairs[GENERIC_OP_EVT_CHNL].req;
+	if (unlikely(!evtchnl))
+		return -EIO;
+
+	memset(&buf_cfg, 0, sizeof(buf_cfg));
+	buf_cfg.xb_dev = front_info->xb_dev;
+	buf_cfg.pages = pages;
+	buf_cfg.size = size;
+	buf_cfg.be_alloc = front_info->cfg.be_alloc;
+
+	shbuf = xen_drm_front_shbuf_alloc(&buf_cfg);
+	if (IS_ERR(shbuf))
+		return PTR_ERR(shbuf);
+
+	ret = dbuf_add_to_list(front_info, shbuf, dbuf_cookie);
+	if (ret < 0) {
+		xen_drm_front_shbuf_free(shbuf);
+		return ret;
+	}
+
+	mutex_lock(&evtchnl->u.req.req_io_lock);
+
+	spin_lock_irqsave(&front_info->io_lock, flags);
+	req = be_prepare_req(evtchnl, XENDISPL_OP_DBUF_CREATE);
+	req->op.dbuf_create.gref_directory =
+			xen_drm_front_shbuf_get_dir_start(shbuf);
+	req->op.dbuf_create.buffer_sz = size;
+	req->op.dbuf_create.dbuf_cookie = dbuf_cookie;
+	req->op.dbuf_create.width = width;
+	req->op.dbuf_create.height = height;
+	req->op.dbuf_create.bpp = bpp;
+	if (buf_cfg.be_alloc)
+		req->op.dbuf_create.flags |= XENDISPL_DBUF_FLG_REQ_ALLOC;
+
+	ret = be_stream_do_io(evtchnl, req);
+	spin_unlock_irqrestore(&front_info->io_lock, flags);
+
+	if (ret < 0)
+		goto fail;
+
+	ret = be_stream_wait_io(evtchnl);
+	if (ret < 0)
+		goto fail;
+
+	ret = xen_drm_front_shbuf_map(shbuf);
+	if (ret < 0)
+		goto fail;
+
+	mutex_unlock(&evtchnl->u.req.req_io_lock);
+	return 0;
+
+fail:
+	mutex_unlock(&evtchnl->u.req.req_io_lock);
+	dbuf_free(&front_info->dbuf_list, dbuf_cookie);
+	return ret;
+}
+
+static int xen_drm_front_dbuf_destroy(struct xen_drm_front_info *front_info,
+				      u64 dbuf_cookie)
+{
+	struct xen_drm_front_evtchnl *evtchnl;
+	struct xendispl_req *req;
+	unsigned long flags;
+	bool be_alloc;
+	int ret;
+
+	evtchnl = &front_info->evt_pairs[GENERIC_OP_EVT_CHNL].req;
+	if (unlikely(!evtchnl))
+		return -EIO;
+
+	be_alloc = front_info->cfg.be_alloc;
+
+	/*
+	 * For the backend allocated buffer release references now, so backend
+	 * can free the buffer.
+	 */
+	if (be_alloc)
+		dbuf_free(&front_info->dbuf_list, dbuf_cookie);
+
+	mutex_lock(&evtchnl->u.req.req_io_lock);
+
+	spin_lock_irqsave(&front_info->io_lock, flags);
+	req = be_prepare_req(evtchnl, XENDISPL_OP_DBUF_DESTROY);
+	req->op.dbuf_destroy.dbuf_cookie = dbuf_cookie;
+
+	ret = be_stream_do_io(evtchnl, req);
+	spin_unlock_irqrestore(&front_info->io_lock, flags);
+
+	if (ret == 0)
+		ret = be_stream_wait_io(evtchnl);
+
+	/*
+	 * Do this regardless of communication status with the backend:
+	 * if we cannot remove remote resources remove what we can locally.
+	 */
+	if (!be_alloc)
+		dbuf_free(&front_info->dbuf_list, dbuf_cookie);
+
+	mutex_unlock(&evtchnl->u.req.req_io_lock);
+	return ret;
+}
+
+int xen_drm_front_fb_attach(struct xen_drm_front_info *front_info,
+			    u64 dbuf_cookie, u64 fb_cookie, u32 width,
+			    u32 height, u32 pixel_format)
+{
+	struct xen_drm_front_evtchnl *evtchnl;
+	struct xen_drm_front_dbuf *buf;
+	struct xendispl_req *req;
+	unsigned long flags;
+	int ret;
+
+	evtchnl = &front_info->evt_pairs[GENERIC_OP_EVT_CHNL].req;
+	if (unlikely(!evtchnl))
+		return -EIO;
+
+	buf = dbuf_get(&front_info->dbuf_list, dbuf_cookie);
+	if (!buf)
+		return -EINVAL;
+
+	buf->fb_cookie = fb_cookie;
+
+	mutex_lock(&evtchnl->u.req.req_io_lock);
+
+	spin_lock_irqsave(&front_info->io_lock, flags);
+	req = be_prepare_req(evtchnl, XENDISPL_OP_FB_ATTACH);
+	req->op.fb_attach.dbuf_cookie = dbuf_cookie;
+	req->op.fb_attach.fb_cookie = fb_cookie;
+	req->op.fb_attach.width = width;
+	req->op.fb_attach.height = height;
+	req->op.fb_attach.pixel_format = pixel_format;
+
+	ret = be_stream_do_io(evtchnl, req);
+	spin_unlock_irqrestore(&front_info->io_lock, flags);
+
+	if (ret == 0)
+		ret = be_stream_wait_io(evtchnl);
+
+	mutex_unlock(&evtchnl->u.req.req_io_lock);
+	return ret;
+}
+
+int xen_drm_front_fb_detach(struct xen_drm_front_info *front_info,
+			    u64 fb_cookie)
+{
+	struct xen_drm_front_evtchnl *evtchnl;
+	struct xendispl_req *req;
+	unsigned long flags;
+	int ret;
+
+	evtchnl = &front_info->evt_pairs[GENERIC_OP_EVT_CHNL].req;
+	if (unlikely(!evtchnl))
+		return -EIO;
+
+	mutex_lock(&evtchnl->u.req.req_io_lock);
+
+	spin_lock_irqsave(&front_info->io_lock, flags);
+	req = be_prepare_req(evtchnl, XENDISPL_OP_FB_DETACH);
+	req->op.fb_detach.fb_cookie = fb_cookie;
+
+	ret = be_stream_do_io(evtchnl, req);
+	spin_unlock_irqrestore(&front_info->io_lock, flags);
+
+	if (ret == 0)
+		ret = be_stream_wait_io(evtchnl);
+
+	mutex_unlock(&evtchnl->u.req.req_io_lock);
+	return ret;
+}
+
+int xen_drm_front_page_flip(struct xen_drm_front_info *front_info,
+			    int conn_idx, u64 fb_cookie)
+{
+	struct xen_drm_front_evtchnl *evtchnl;
+	struct xendispl_req *req;
+	unsigned long flags;
+	int ret;
+
+	if (unlikely(conn_idx >= front_info->num_evt_pairs))
+		return -EINVAL;
+
+	dbuf_flush_fb(&front_info->dbuf_list, fb_cookie);
+	evtchnl = &front_info->evt_pairs[conn_idx].req;
+
+	mutex_lock(&evtchnl->u.req.req_io_lock);
+
+	spin_lock_irqsave(&front_info->io_lock, flags);
+	req = be_prepare_req(evtchnl, XENDISPL_OP_PG_FLIP);
+	req->op.pg_flip.fb_cookie = fb_cookie;
+
+	ret = be_stream_do_io(evtchnl, req);
+	spin_unlock_irqrestore(&front_info->io_lock, flags);
+
+	if (ret == 0)
+		ret = be_stream_wait_io(evtchnl);
+
+	mutex_unlock(&evtchnl->u.req.req_io_lock);
+	return ret;
+}
+
+void xen_drm_front_on_frame_done(struct xen_drm_front_info *front_info,
+				 int conn_idx, u64 fb_cookie)
+{
+	struct xen_drm_front_drm_info *drm_info = front_info->drm_info;
+
+	if (unlikely(conn_idx >= front_info->cfg.num_connectors))
+		return;
+
+	xen_drm_front_kms_on_frame_done(&drm_info->pipeline[conn_idx],
+					fb_cookie);
+}
+
+static int xen_drm_drv_dumb_create(struct drm_file *filp,
+				   struct drm_device *dev,
+				   struct drm_mode_create_dumb *args)
+{
+	struct xen_drm_front_drm_info *drm_info = dev->dev_private;
+	struct drm_gem_object *obj;
+	int ret;
+
+	/*
+	 * Dumb creation is a two stage process: first we create a fully
+	 * constructed GEM object which is communicated to the backend, and
+	 * only after that we can create GEM's handle. This is done so,
+	 * because of the possible races: once you create a handle it becomes
+	 * immediately visible to user-space, so the latter can try accessing
+	 * object without pages etc.
+	 * For details also see drm_gem_handle_create
+	 */
+	args->pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
+	args->size = args->pitch * args->height;
+
+	obj = xen_drm_front_gem_create(dev, args->size);
+	if (IS_ERR_OR_NULL(obj)) {
+		ret = PTR_ERR(obj);
+		goto fail;
+	}
+
+	ret = xen_drm_front_dbuf_create(drm_info->front_info,
+					xen_drm_front_dbuf_to_cookie(obj),
+					args->width, args->height, args->bpp,
+					args->size,
+					xen_drm_front_gem_get_pages(obj));
+	if (ret)
+		goto fail_backend;
+
+	/* This is the tail of GEM object creation */
+	ret = drm_gem_handle_create(filp, obj, &args->handle);
+	if (ret)
+		goto fail_handle;
+
+	/* Drop reference from allocate - handle holds it now */
+	drm_gem_object_put_unlocked(obj);
+	return 0;
+
+fail_handle:
+	xen_drm_front_dbuf_destroy(drm_info->front_info,
+				   xen_drm_front_dbuf_to_cookie(obj));
+fail_backend:
+	/* drop reference from allocate */
+	drm_gem_object_put_unlocked(obj);
+fail:
+	DRM_ERROR("Failed to create dumb buffer: %d\n", ret);
+	return ret;
+}
+
+static void xen_drm_drv_free_object_unlocked(struct drm_gem_object *obj)
+{
+	struct xen_drm_front_drm_info *drm_info = obj->dev->dev_private;
+	int idx;
+
+	if (drm_dev_enter(obj->dev, &idx)) {
+		xen_drm_front_dbuf_destroy(drm_info->front_info,
+					   xen_drm_front_dbuf_to_cookie(obj));
+		drm_dev_exit(idx);
+	} else {
+		dbuf_free(&drm_info->front_info->dbuf_list,
+			  xen_drm_front_dbuf_to_cookie(obj));
+	}
+
+	xen_drm_front_gem_free_object_unlocked(obj);
+}
+
+static void xen_drm_drv_release(struct drm_device *dev)
+{
+	struct xen_drm_front_drm_info *drm_info = dev->dev_private;
+	struct xen_drm_front_info *front_info = drm_info->front_info;
+
+	xen_drm_front_kms_fini(drm_info);
+
+	drm_atomic_helper_shutdown(dev);
+	drm_mode_config_cleanup(dev);
+
+	drm_dev_fini(dev);
+	kfree(dev);
+
+	if (front_info->cfg.be_alloc)
+		xenbus_switch_state(front_info->xb_dev,
+				    XenbusStateInitialising);
+
+	kfree(drm_info);
+}
+
+static const struct file_operations xen_drm_dev_fops = {
+	.owner          = THIS_MODULE,
+	.open           = drm_open,
+	.release        = drm_release,
+	.unlocked_ioctl = drm_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl   = drm_compat_ioctl,
+#endif
+	.poll           = drm_poll,
+	.read           = drm_read,
+	.llseek         = no_llseek,
+	.mmap           = xen_drm_front_gem_mmap,
+};
+
+static const struct vm_operations_struct xen_drm_drv_vm_ops = {
+	.open           = drm_gem_vm_open,
+	.close          = drm_gem_vm_close,
+};
+
+static struct drm_driver xen_drm_driver = {
+	.driver_features           = DRIVER_GEM | DRIVER_MODESET |
+				     DRIVER_PRIME | DRIVER_ATOMIC,
+	.release                   = xen_drm_drv_release,
+	.gem_vm_ops                = &xen_drm_drv_vm_ops,
+	.gem_free_object_unlocked  = xen_drm_drv_free_object_unlocked,
+	.prime_handle_to_fd        = drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle        = drm_gem_prime_fd_to_handle,
+	.gem_prime_import          = drm_gem_prime_import,
+	.gem_prime_export          = drm_gem_prime_export,
+	.gem_prime_import_sg_table = xen_drm_front_gem_import_sg_table,
+	.gem_prime_get_sg_table    = xen_drm_front_gem_get_sg_table,
+	.gem_prime_vmap            = xen_drm_front_gem_prime_vmap,
+	.gem_prime_vunmap          = xen_drm_front_gem_prime_vunmap,
+	.gem_prime_mmap            = xen_drm_front_gem_prime_mmap,
+	.dumb_create               = xen_drm_drv_dumb_create,
+	.fops                      = &xen_drm_dev_fops,
+	.name                      = "xendrm-du",
+	.desc                      = "Xen PV DRM Display Unit",
+	.date                      = "20180221",
+	.major                     = 1,
+	.minor                     = 0,
+
+};
+
+static int xen_drm_drv_init(struct xen_drm_front_info *front_info)
+{
+	struct device *dev = &front_info->xb_dev->dev;
+	struct xen_drm_front_drm_info *drm_info;
+	struct drm_device *drm_dev;
+	int ret;
+
+	DRM_INFO("Creating %s\n", xen_drm_driver.desc);
+
+	drm_info = kzalloc(sizeof(*drm_info), GFP_KERNEL);
+	if (!drm_info) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	drm_info->front_info = front_info;
+	front_info->drm_info = drm_info;
+
+	drm_dev = drm_dev_alloc(&xen_drm_driver, dev);
+	if (IS_ERR(drm_dev)) {
+		ret = PTR_ERR(drm_dev);
+		goto fail;
+	}
+
+	drm_info->drm_dev = drm_dev;
+
+	drm_dev->dev_private = drm_info;
+
+	ret = xen_drm_front_kms_init(drm_info);
+	if (ret) {
+		DRM_ERROR("Failed to initialize DRM/KMS, ret %d\n", ret);
+		goto fail_modeset;
+	}
+
+	ret = drm_dev_register(drm_dev, 0);
+	if (ret)
+		goto fail_register;
+
+	DRM_INFO("Initialized %s %d.%d.%d %s on minor %d\n",
+		 xen_drm_driver.name, xen_drm_driver.major,
+		 xen_drm_driver.minor, xen_drm_driver.patchlevel,
+		 xen_drm_driver.date, drm_dev->primary->index);
+
+	return 0;
+
+fail_register:
+	drm_dev_unregister(drm_dev);
+fail_modeset:
+	drm_kms_helper_poll_fini(drm_dev);
+	drm_mode_config_cleanup(drm_dev);
+fail:
+	kfree(drm_info);
+	return ret;
+}
+
+static void xen_drm_drv_fini(struct xen_drm_front_info *front_info)
+{
+	struct xen_drm_front_drm_info *drm_info = front_info->drm_info;
+	struct drm_device *dev;
+
+	if (!drm_info)
+		return;
+
+	dev = drm_info->drm_dev;
+	if (!dev)
+		return;
+
+	/* Nothing to do if device is already unplugged */
+	if (drm_dev_is_unplugged(dev))
+		return;
+
+	drm_kms_helper_poll_fini(dev);
+	drm_dev_unplug(dev);
+
+	front_info->drm_info = NULL;
+
+	xen_drm_front_evtchnl_free_all(front_info);
+	dbuf_free_all(&front_info->dbuf_list);
+
+	/*
+	 * If we are not using backend allocated buffers, then tell the
+	 * backend we are ready to (re)initialize. Otherwise, wait for
+	 * drm_driver.release.
+	 */
+	if (!front_info->cfg.be_alloc)
+		xenbus_switch_state(front_info->xb_dev,
+				    XenbusStateInitialising);
+}
+
+static int displback_initwait(struct xen_drm_front_info *front_info)
+{
+	struct xen_drm_front_cfg *cfg = &front_info->cfg;
+	int ret;
+
+	cfg->front_info = front_info;
+	ret = xen_drm_front_cfg_card(front_info, cfg);
+	if (ret < 0)
+		return ret;
+
+	DRM_INFO("Have %d conector(s)\n", cfg->num_connectors);
+	/* Create event channels for all connectors and publish */
+	ret = xen_drm_front_evtchnl_create_all(front_info);
+	if (ret < 0)
+		return ret;
+
+	return xen_drm_front_evtchnl_publish_all(front_info);
+}
+
+static int displback_connect(struct xen_drm_front_info *front_info)
+{
+	xen_drm_front_evtchnl_set_state(front_info, EVTCHNL_STATE_CONNECTED);
+	return xen_drm_drv_init(front_info);
+}
+
+static void displback_disconnect(struct xen_drm_front_info *front_info)
+{
+	if (!front_info->drm_info)
+		return;
+
+	/* Tell the backend to wait until we release the DRM driver. */
+	xenbus_switch_state(front_info->xb_dev, XenbusStateReconfiguring);
+
+	xen_drm_drv_fini(front_info);
+}
+
+static void displback_changed(struct xenbus_device *xb_dev,
+			      enum xenbus_state backend_state)
+{
+	struct xen_drm_front_info *front_info = dev_get_drvdata(&xb_dev->dev);
+	int ret;
+
+	DRM_DEBUG("Backend state is %s, front is %s\n",
+		  xenbus_strstate(backend_state),
+		  xenbus_strstate(xb_dev->state));
+
+	switch (backend_state) {
+	case XenbusStateReconfiguring:
+		/* fall through */
+	case XenbusStateReconfigured:
+		/* fall through */
+	case XenbusStateInitialised:
+		break;
+
+	case XenbusStateInitialising:
+		if (xb_dev->state == XenbusStateReconfiguring)
+			break;
+
+		/* recovering after backend unexpected closure */
+		displback_disconnect(front_info);
+		break;
+
+	case XenbusStateInitWait:
+		if (xb_dev->state == XenbusStateReconfiguring)
+			break;
+
+		/* recovering after backend unexpected closure */
+		displback_disconnect(front_info);
+		if (xb_dev->state != XenbusStateInitialising)
+			break;
+
+		ret = displback_initwait(front_info);
+		if (ret < 0)
+			xenbus_dev_fatal(xb_dev, ret, "initializing frontend");
+		else
+			xenbus_switch_state(xb_dev, XenbusStateInitialised);
+		break;
+
+	case XenbusStateConnected:
+		if (xb_dev->state != XenbusStateInitialised)
+			break;
+
+		ret = displback_connect(front_info);
+		if (ret < 0) {
+			displback_disconnect(front_info);
+			xenbus_dev_fatal(xb_dev, ret, "connecting backend");
+		} else {
+			xenbus_switch_state(xb_dev, XenbusStateConnected);
+		}
+		break;
+
+	case XenbusStateClosing:
+		/*
+		 * in this state backend starts freeing resources,
+		 * so let it go into closed state, so we can also
+		 * remove ours
+		 */
+		break;
+
+	case XenbusStateUnknown:
+		/* fall through */
+	case XenbusStateClosed:
+		if (xb_dev->state == XenbusStateClosed)
+			break;
+
+		displback_disconnect(front_info);
+		break;
+	}
+}
+
+static int xen_drv_probe(struct xenbus_device *xb_dev,
+			 const struct xenbus_device_id *id)
+{
+	struct xen_drm_front_info *front_info;
+	struct device *dev = &xb_dev->dev;
+	int ret;
+
+	/*
+	 * The device is not spawn from a device tree, so arch_setup_dma_ops
+	 * is not called, thus leaving the device with dummy DMA ops.
+	 * This makes the device return error on PRIME buffer import, which
+	 * is not correct: to fix this call of_dma_configure() with a NULL
+	 * node to set default DMA ops.
+	 */
+	dev->coherent_dma_mask = DMA_BIT_MASK(32);
+	ret = of_dma_configure(dev, NULL, true);
+	if (ret < 0) {
+		DRM_ERROR("Cannot setup DMA ops, ret %d", ret);
+		return ret;
+	}
+
+	front_info = devm_kzalloc(&xb_dev->dev,
+				  sizeof(*front_info), GFP_KERNEL);
+	if (!front_info)
+		return -ENOMEM;
+
+	front_info->xb_dev = xb_dev;
+	spin_lock_init(&front_info->io_lock);
+	INIT_LIST_HEAD(&front_info->dbuf_list);
+	dev_set_drvdata(&xb_dev->dev, front_info);
+
+	return xenbus_switch_state(xb_dev, XenbusStateInitialising);
+}
+
+static int xen_drv_remove(struct xenbus_device *dev)
+{
+	struct xen_drm_front_info *front_info = dev_get_drvdata(&dev->dev);
+	int to = 100;
+
+	xenbus_switch_state(dev, XenbusStateClosing);
+
+	/*
+	 * On driver removal it is disconnected from XenBus,
+	 * so no backend state change events come via .otherend_changed
+	 * callback. This prevents us from exiting gracefully, e.g.
+	 * signaling the backend to free event channels, waiting for its
+	 * state to change to XenbusStateClosed and cleaning at our end.
+	 * Normally when front driver removed backend will finally go into
+	 * XenbusStateInitWait state.
+	 *
+	 * Workaround: read backend's state manually and wait with time-out.
+	 */
+	while ((xenbus_read_unsigned(front_info->xb_dev->otherend, "state",
+				     XenbusStateUnknown) != XenbusStateInitWait) &&
+				     --to)
+		msleep(10);
+
+	if (!to) {
+		unsigned int state;
+
+		state = xenbus_read_unsigned(front_info->xb_dev->otherend,
+					     "state", XenbusStateUnknown);
+		DRM_ERROR("Backend state is %s while removing driver\n",
+			  xenbus_strstate(state));
+	}
+
+	xen_drm_drv_fini(front_info);
+	xenbus_frontend_closed(dev);
+	return 0;
+}
+
+static const struct xenbus_device_id xen_driver_ids[] = {
+	{ XENDISPL_DRIVER_NAME },
+	{ "" }
+};
+
+static struct xenbus_driver xen_driver = {
+	.ids = xen_driver_ids,
+	.probe = xen_drv_probe,
+	.remove = xen_drv_remove,
+	.otherend_changed = displback_changed,
+};
+
+static int __init xen_drv_init(void)
+{
+	/* At the moment we only support case with XEN_PAGE_SIZE == PAGE_SIZE */
+	if (XEN_PAGE_SIZE != PAGE_SIZE) {
+		DRM_ERROR(XENDISPL_DRIVER_NAME ": different kernel and Xen page sizes are not supported: XEN_PAGE_SIZE (%lu) != PAGE_SIZE (%lu)\n",
+			  XEN_PAGE_SIZE, PAGE_SIZE);
+		return -ENODEV;
+	}
+
+	if (!xen_domain())
+		return -ENODEV;
+
+	if (!xen_has_pv_devices())
+		return -ENODEV;
+
+	DRM_INFO("Registering XEN PV " XENDISPL_DRIVER_NAME "\n");
+	return xenbus_register_frontend(&xen_driver);
+}
+
+static void __exit xen_drv_fini(void)
+{
+	DRM_INFO("Unregistering XEN PV " XENDISPL_DRIVER_NAME "\n");
+	xenbus_unregister_driver(&xen_driver);
+}
+
+module_init(xen_drv_init);
+module_exit(xen_drv_fini);
+
+MODULE_DESCRIPTION("Xen para-virtualized display device frontend");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("xen:" XENDISPL_DRIVER_NAME);
diff --git a/drivers/gpu/drm/xen/xen_drm_front.h b/drivers/gpu/drm/xen/xen_drm_front.h
new file mode 100644
index 000000000000..2c2479b571ae
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front.h
@@ -0,0 +1,158 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#ifndef __XEN_DRM_FRONT_H_
+#define __XEN_DRM_FRONT_H_
+
+#include <drm/drmP.h>
+#include <drm/drm_simple_kms_helper.h>
+
+#include <linux/scatterlist.h>
+
+#include "xen_drm_front_cfg.h"
+
+/**
+ * DOC: Driver modes of operation in terms of display buffers used
+ *
+ * Depending on the requirements for the para-virtualized environment, namely
+ * requirements dictated by the accompanying DRM/(v)GPU drivers running in both
+ * host and guest environments, display buffers can be allocated by either
+ * frontend driver or backend.
+ */
+
+/**
+ * DOC: Buffers allocated by the frontend driver
+ *
+ * In this mode of operation driver allocates buffers from system memory.
+ *
+ * Note! If used with accompanying DRM/(v)GPU drivers this mode of operation
+ * may require IOMMU support on the platform, so accompanying DRM/vGPU
+ * hardware can still reach display buffer memory while importing PRIME
+ * buffers from the frontend driver.
+ */
+
+/**
+ * DOC: Buffers allocated by the backend
+ *
+ * This mode of operation is run-time configured via guest domain configuration
+ * through XenStore entries.
+ *
+ * For systems which do not provide IOMMU support, but having specific
+ * requirements for display buffers it is possible to allocate such buffers
+ * at backend side and share those with the frontend.
+ * For example, if host domain is 1:1 mapped and has DRM/GPU hardware expecting
+ * physically contiguous memory, this allows implementing zero-copying
+ * use-cases.
+ *
+ * Note, while using this scenario the following should be considered:
+ *
+ * #. If guest domain dies then pages/grants received from the backend
+ *    cannot be claimed back
+ *
+ * #. Misbehaving guest may send too many requests to the
+ *    backend exhausting its grant references and memory
+ *    (consider this from security POV)
+ */
+
+/**
+ * DOC: Driver limitations
+ *
+ * #. Only primary plane without additional properties is supported.
+ *
+ * #. Only one video mode per connector supported which is configured
+ *    via XenStore.
+ *
+ * #. All CRTCs operate at fixed frequency of 60Hz.
+ */
+
+/* timeout in ms to wait for backend to respond */
+#define XEN_DRM_FRONT_WAIT_BACK_MS	3000
+
+#ifndef GRANT_INVALID_REF
+/*
+ * Note on usage of grant reference 0 as invalid grant reference:
+ * grant reference 0 is valid, but never exposed to a PV driver,
+ * because of the fact it is already in use/reserved by the PV console.
+ */
+#define GRANT_INVALID_REF	0
+#endif
+
+struct xen_drm_front_info {
+	struct xenbus_device *xb_dev;
+	struct xen_drm_front_drm_info *drm_info;
+
+	/* to protect data between backend IO code and interrupt handler */
+	spinlock_t io_lock;
+
+	int num_evt_pairs;
+	struct xen_drm_front_evtchnl_pair *evt_pairs;
+	struct xen_drm_front_cfg cfg;
+
+	/* display buffers */
+	struct list_head dbuf_list;
+};
+
+struct xen_drm_front_drm_pipeline {
+	struct xen_drm_front_drm_info *drm_info;
+
+	int index;
+
+	struct drm_simple_display_pipe pipe;
+
+	struct drm_connector conn;
+	/* These are only for connector mode checking */
+	int width, height;
+
+	struct drm_pending_vblank_event *pending_event;
+
+	struct delayed_work pflip_to_worker;
+
+	bool conn_connected;
+};
+
+struct xen_drm_front_drm_info {
+	struct xen_drm_front_info *front_info;
+	struct drm_device *drm_dev;
+
+	struct xen_drm_front_drm_pipeline pipeline[XEN_DRM_FRONT_MAX_CRTCS];
+};
+
+static inline u64 xen_drm_front_fb_to_cookie(struct drm_framebuffer *fb)
+{
+	return (u64)fb;
+}
+
+static inline u64 xen_drm_front_dbuf_to_cookie(struct drm_gem_object *gem_obj)
+{
+	return (u64)gem_obj;
+}
+
+int xen_drm_front_mode_set(struct xen_drm_front_drm_pipeline *pipeline,
+			   u32 x, u32 y, u32 width, u32 height,
+			   u32 bpp, u64 fb_cookie);
+
+int xen_drm_front_dbuf_create(struct xen_drm_front_info *front_info,
+			      u64 dbuf_cookie, u32 width, u32 height,
+			      u32 bpp, u64 size, struct page **pages);
+
+int xen_drm_front_fb_attach(struct xen_drm_front_info *front_info,
+			    u64 dbuf_cookie, u64 fb_cookie, u32 width,
+			    u32 height, u32 pixel_format);
+
+int xen_drm_front_fb_detach(struct xen_drm_front_info *front_info,
+			    u64 fb_cookie);
+
+int xen_drm_front_page_flip(struct xen_drm_front_info *front_info,
+			    int conn_idx, u64 fb_cookie);
+
+void xen_drm_front_on_frame_done(struct xen_drm_front_info *front_info,
+				 int conn_idx, u64 fb_cookie);
+
+#endif /* __XEN_DRM_FRONT_H_ */
diff --git a/drivers/gpu/drm/xen/xen_drm_front_cfg.c b/drivers/gpu/drm/xen/xen_drm_front_cfg.c
new file mode 100644
index 000000000000..5baf2b9de93c
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_cfg.c
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#include <drm/drmP.h>
+
+#include <linux/device.h>
+
+#include <xen/interface/io/displif.h>
+#include <xen/xenbus.h>
+
+#include "xen_drm_front.h"
+#include "xen_drm_front_cfg.h"
+
+static int cfg_connector(struct xen_drm_front_info *front_info,
+			 struct xen_drm_front_cfg_connector *connector,
+			 const char *path, int index)
+{
+	char *connector_path;
+
+	connector_path = devm_kasprintf(&front_info->xb_dev->dev,
+					GFP_KERNEL, "%s/%d", path, index);
+	if (!connector_path)
+		return -ENOMEM;
+
+	if (xenbus_scanf(XBT_NIL, connector_path, XENDISPL_FIELD_RESOLUTION,
+			 "%d" XENDISPL_RESOLUTION_SEPARATOR "%d",
+			 &connector->width, &connector->height) < 0) {
+		/* either no entry configured or wrong resolution set */
+		connector->width = 0;
+		connector->height = 0;
+		return -EINVAL;
+	}
+
+	connector->xenstore_path = connector_path;
+
+	DRM_INFO("Connector %s: resolution %dx%d\n",
+		 connector_path, connector->width, connector->height);
+	return 0;
+}
+
+int xen_drm_front_cfg_card(struct xen_drm_front_info *front_info,
+			   struct xen_drm_front_cfg *cfg)
+{
+	struct xenbus_device *xb_dev = front_info->xb_dev;
+	int ret, i;
+
+	if (xenbus_read_unsigned(front_info->xb_dev->nodename,
+				 XENDISPL_FIELD_BE_ALLOC, 0)) {
+		DRM_INFO("Backend can provide display buffers\n");
+		cfg->be_alloc = true;
+	}
+
+	cfg->num_connectors = 0;
+	for (i = 0; i < ARRAY_SIZE(cfg->connectors); i++) {
+		ret = cfg_connector(front_info, &cfg->connectors[i],
+				    xb_dev->nodename, i);
+		if (ret < 0)
+			break;
+		cfg->num_connectors++;
+	}
+
+	if (!cfg->num_connectors) {
+		DRM_ERROR("No connector(s) configured at %s\n",
+			  xb_dev->nodename);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
diff --git a/drivers/gpu/drm/xen/xen_drm_front_cfg.h b/drivers/gpu/drm/xen/xen_drm_front_cfg.h
new file mode 100644
index 000000000000..aa8490ba9146
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_cfg.h
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#ifndef __XEN_DRM_FRONT_CFG_H_
+#define __XEN_DRM_FRONT_CFG_H_
+
+#include <linux/types.h>
+
+#define XEN_DRM_FRONT_MAX_CRTCS	4
+
+struct xen_drm_front_cfg_connector {
+	int width;
+	int height;
+	char *xenstore_path;
+};
+
+struct xen_drm_front_cfg {
+	struct xen_drm_front_info *front_info;
+	/* number of connectors in this configuration */
+	int num_connectors;
+	/* connector configurations */
+	struct xen_drm_front_cfg_connector connectors[XEN_DRM_FRONT_MAX_CRTCS];
+	/* set if dumb buffers are allocated externally on backend side */
+	bool be_alloc;
+};
+
+int xen_drm_front_cfg_card(struct xen_drm_front_info *front_info,
+			   struct xen_drm_front_cfg *cfg);
+
+#endif /* __XEN_DRM_FRONT_CFG_H_ */
diff --git a/drivers/gpu/drm/xen/xen_drm_front_conn.c b/drivers/gpu/drm/xen/xen_drm_front_conn.c
new file mode 100644
index 000000000000..c91ae532fa55
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_conn.c
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+
+#include <video/videomode.h>
+
+#include "xen_drm_front.h"
+#include "xen_drm_front_conn.h"
+#include "xen_drm_front_kms.h"
+
+static struct xen_drm_front_drm_pipeline *
+to_xen_drm_pipeline(struct drm_connector *connector)
+{
+	return container_of(connector, struct xen_drm_front_drm_pipeline, conn);
+}
+
+static const u32 plane_formats[] = {
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XRGB4444,
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB1555,
+};
+
+const u32 *xen_drm_front_conn_get_formats(int *format_count)
+{
+	*format_count = ARRAY_SIZE(plane_formats);
+	return plane_formats;
+}
+
+static int connector_detect(struct drm_connector *connector,
+			    struct drm_modeset_acquire_ctx *ctx,
+			    bool force)
+{
+	struct xen_drm_front_drm_pipeline *pipeline =
+			to_xen_drm_pipeline(connector);
+
+	if (drm_dev_is_unplugged(connector->dev))
+		pipeline->conn_connected = false;
+
+	return pipeline->conn_connected ? connector_status_connected :
+			connector_status_disconnected;
+}
+
+#define XEN_DRM_CRTC_VREFRESH_HZ	60
+
+static int connector_get_modes(struct drm_connector *connector)
+{
+	struct xen_drm_front_drm_pipeline *pipeline =
+			to_xen_drm_pipeline(connector);
+	struct drm_display_mode *mode;
+	struct videomode videomode;
+	int width, height;
+
+	mode = drm_mode_create(connector->dev);
+	if (!mode)
+		return 0;
+
+	memset(&videomode, 0, sizeof(videomode));
+	videomode.hactive = pipeline->width;
+	videomode.vactive = pipeline->height;
+	width = videomode.hactive + videomode.hfront_porch +
+			videomode.hback_porch + videomode.hsync_len;
+	height = videomode.vactive + videomode.vfront_porch +
+			videomode.vback_porch + videomode.vsync_len;
+	videomode.pixelclock = width * height * XEN_DRM_CRTC_VREFRESH_HZ;
+	mode->type = DRM_MODE_TYPE_PREFERRED | DRM_MODE_TYPE_DRIVER;
+
+	drm_display_mode_from_videomode(&videomode, mode);
+	drm_mode_probed_add(connector, mode);
+	return 1;
+}
+
+static const struct drm_connector_helper_funcs connector_helper_funcs = {
+	.get_modes = connector_get_modes,
+	.detect_ctx = connector_detect,
+};
+
+static const struct drm_connector_funcs connector_funcs = {
+	.dpms = drm_helper_connector_dpms,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = drm_connector_cleanup,
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+int xen_drm_front_conn_init(struct xen_drm_front_drm_info *drm_info,
+			    struct drm_connector *connector)
+{
+	struct xen_drm_front_drm_pipeline *pipeline =
+			to_xen_drm_pipeline(connector);
+
+	drm_connector_helper_add(connector, &connector_helper_funcs);
+
+	pipeline->conn_connected = true;
+
+	connector->polled = DRM_CONNECTOR_POLL_CONNECT |
+			DRM_CONNECTOR_POLL_DISCONNECT;
+
+	return drm_connector_init(drm_info->drm_dev, connector,
+				  &connector_funcs, DRM_MODE_CONNECTOR_VIRTUAL);
+}
diff --git a/drivers/gpu/drm/xen/xen_drm_front_conn.h b/drivers/gpu/drm/xen/xen_drm_front_conn.h
new file mode 100644
index 000000000000..39de7cf5adbe
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_conn.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#ifndef __XEN_DRM_FRONT_CONN_H_
+#define __XEN_DRM_FRONT_CONN_H_
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_encoder.h>
+
+#include <linux/wait.h>
+
+struct xen_drm_front_drm_info;
+
+int xen_drm_front_conn_init(struct xen_drm_front_drm_info *drm_info,
+			    struct drm_connector *connector);
+
+const u32 *xen_drm_front_conn_get_formats(int *format_count);
+
+#endif /* __XEN_DRM_FRONT_CONN_H_ */
diff --git a/drivers/gpu/drm/xen/xen_drm_front_evtchnl.c b/drivers/gpu/drm/xen/xen_drm_front_evtchnl.c
new file mode 100644
index 000000000000..945226a95e9b
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_evtchnl.c
@@ -0,0 +1,387 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#include <drm/drmP.h>
+
+#include <linux/errno.h>
+#include <linux/irq.h>
+
+#include <xen/xenbus.h>
+#include <xen/events.h>
+#include <xen/grant_table.h>
+
+#include "xen_drm_front.h"
+#include "xen_drm_front_evtchnl.h"
+
+static irqreturn_t evtchnl_interrupt_ctrl(int irq, void *dev_id)
+{
+	struct xen_drm_front_evtchnl *evtchnl = dev_id;
+	struct xen_drm_front_info *front_info = evtchnl->front_info;
+	struct xendispl_resp *resp;
+	RING_IDX i, rp;
+	unsigned long flags;
+
+	if (unlikely(evtchnl->state != EVTCHNL_STATE_CONNECTED))
+		return IRQ_HANDLED;
+
+	spin_lock_irqsave(&front_info->io_lock, flags);
+
+again:
+	rp = evtchnl->u.req.ring.sring->rsp_prod;
+	/* ensure we see queued responses up to rp */
+	virt_rmb();
+
+	for (i = evtchnl->u.req.ring.rsp_cons; i != rp; i++) {
+		resp = RING_GET_RESPONSE(&evtchnl->u.req.ring, i);
+		if (unlikely(resp->id != evtchnl->evt_id))
+			continue;
+
+		switch (resp->operation) {
+		case XENDISPL_OP_PG_FLIP:
+		case XENDISPL_OP_FB_ATTACH:
+		case XENDISPL_OP_FB_DETACH:
+		case XENDISPL_OP_DBUF_CREATE:
+		case XENDISPL_OP_DBUF_DESTROY:
+		case XENDISPL_OP_SET_CONFIG:
+			evtchnl->u.req.resp_status = resp->status;
+			complete(&evtchnl->u.req.completion);
+			break;
+
+		default:
+			DRM_ERROR("Operation %d is not supported\n",
+				  resp->operation);
+			break;
+		}
+	}
+
+	evtchnl->u.req.ring.rsp_cons = i;
+
+	if (i != evtchnl->u.req.ring.req_prod_pvt) {
+		int more_to_do;
+
+		RING_FINAL_CHECK_FOR_RESPONSES(&evtchnl->u.req.ring,
+					       more_to_do);
+		if (more_to_do)
+			goto again;
+	} else {
+		evtchnl->u.req.ring.sring->rsp_event = i + 1;
+	}
+
+	spin_unlock_irqrestore(&front_info->io_lock, flags);
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t evtchnl_interrupt_evt(int irq, void *dev_id)
+{
+	struct xen_drm_front_evtchnl *evtchnl = dev_id;
+	struct xen_drm_front_info *front_info = evtchnl->front_info;
+	struct xendispl_event_page *page = evtchnl->u.evt.page;
+	u32 cons, prod;
+	unsigned long flags;
+
+	if (unlikely(evtchnl->state != EVTCHNL_STATE_CONNECTED))
+		return IRQ_HANDLED;
+
+	spin_lock_irqsave(&front_info->io_lock, flags);
+
+	prod = page->in_prod;
+	/* ensure we see ring contents up to prod */
+	virt_rmb();
+	if (prod == page->in_cons)
+		goto out;
+
+	for (cons = page->in_cons; cons != prod; cons++) {
+		struct xendispl_evt *event;
+
+		event = &XENDISPL_IN_RING_REF(page, cons);
+		if (unlikely(event->id != evtchnl->evt_id++))
+			continue;
+
+		switch (event->type) {
+		case XENDISPL_EVT_PG_FLIP:
+			xen_drm_front_on_frame_done(front_info, evtchnl->index,
+						    event->op.pg_flip.fb_cookie);
+			break;
+		}
+	}
+	page->in_cons = cons;
+	/* ensure ring contents */
+	virt_wmb();
+
+out:
+	spin_unlock_irqrestore(&front_info->io_lock, flags);
+	return IRQ_HANDLED;
+}
+
+static void evtchnl_free(struct xen_drm_front_info *front_info,
+			 struct xen_drm_front_evtchnl *evtchnl)
+{
+	unsigned long page = 0;
+
+	if (evtchnl->type == EVTCHNL_TYPE_REQ)
+		page = (unsigned long)evtchnl->u.req.ring.sring;
+	else if (evtchnl->type == EVTCHNL_TYPE_EVT)
+		page = (unsigned long)evtchnl->u.evt.page;
+	if (!page)
+		return;
+
+	evtchnl->state = EVTCHNL_STATE_DISCONNECTED;
+
+	if (evtchnl->type == EVTCHNL_TYPE_REQ) {
+		/* release all who still waits for response if any */
+		evtchnl->u.req.resp_status = -EIO;
+		complete_all(&evtchnl->u.req.completion);
+	}
+
+	if (evtchnl->irq)
+		unbind_from_irqhandler(evtchnl->irq, evtchnl);
+
+	if (evtchnl->port)
+		xenbus_free_evtchn(front_info->xb_dev, evtchnl->port);
+
+	/* end access and free the page */
+	if (evtchnl->gref != GRANT_INVALID_REF)
+		gnttab_end_foreign_access(evtchnl->gref, 0, page);
+
+	memset(evtchnl, 0, sizeof(*evtchnl));
+}
+
+static int evtchnl_alloc(struct xen_drm_front_info *front_info, int index,
+			 struct xen_drm_front_evtchnl *evtchnl,
+			 enum xen_drm_front_evtchnl_type type)
+{
+	struct xenbus_device *xb_dev = front_info->xb_dev;
+	unsigned long page;
+	grant_ref_t gref;
+	irq_handler_t handler;
+	int ret;
+
+	memset(evtchnl, 0, sizeof(*evtchnl));
+	evtchnl->type = type;
+	evtchnl->index = index;
+	evtchnl->front_info = front_info;
+	evtchnl->state = EVTCHNL_STATE_DISCONNECTED;
+	evtchnl->gref = GRANT_INVALID_REF;
+
+	page = get_zeroed_page(GFP_NOIO | __GFP_HIGH);
+	if (!page) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	if (type == EVTCHNL_TYPE_REQ) {
+		struct xen_displif_sring *sring;
+
+		init_completion(&evtchnl->u.req.completion);
+		mutex_init(&evtchnl->u.req.req_io_lock);
+		sring = (struct xen_displif_sring *)page;
+		SHARED_RING_INIT(sring);
+		FRONT_RING_INIT(&evtchnl->u.req.ring, sring, XEN_PAGE_SIZE);
+
+		ret = xenbus_grant_ring(xb_dev, sring, 1, &gref);
+		if (ret < 0) {
+			evtchnl->u.req.ring.sring = NULL;
+			free_page(page);
+			goto fail;
+		}
+
+		handler = evtchnl_interrupt_ctrl;
+	} else {
+		ret = gnttab_grant_foreign_access(xb_dev->otherend_id,
+						  virt_to_gfn((void *)page), 0);
+		if (ret < 0) {
+			free_page(page);
+			goto fail;
+		}
+
+		evtchnl->u.evt.page = (struct xendispl_event_page *)page;
+		gref = ret;
+		handler = evtchnl_interrupt_evt;
+	}
+	evtchnl->gref = gref;
+
+	ret = xenbus_alloc_evtchn(xb_dev, &evtchnl->port);
+	if (ret < 0)
+		goto fail;
+
+	ret = bind_evtchn_to_irqhandler(evtchnl->port,
+					handler, 0, xb_dev->devicetype,
+					evtchnl);
+	if (ret < 0)
+		goto fail;
+
+	evtchnl->irq = ret;
+	return 0;
+
+fail:
+	DRM_ERROR("Failed to allocate ring: %d\n", ret);
+	return ret;
+}
+
+int xen_drm_front_evtchnl_create_all(struct xen_drm_front_info *front_info)
+{
+	struct xen_drm_front_cfg *cfg;
+	int ret, conn;
+
+	cfg = &front_info->cfg;
+
+	front_info->evt_pairs =
+			kcalloc(cfg->num_connectors,
+				sizeof(struct xen_drm_front_evtchnl_pair),
+				GFP_KERNEL);
+	if (!front_info->evt_pairs) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	for (conn = 0; conn < cfg->num_connectors; conn++) {
+		ret = evtchnl_alloc(front_info, conn,
+				    &front_info->evt_pairs[conn].req,
+				    EVTCHNL_TYPE_REQ);
+		if (ret < 0) {
+			DRM_ERROR("Error allocating control channel\n");
+			goto fail;
+		}
+
+		ret = evtchnl_alloc(front_info, conn,
+				    &front_info->evt_pairs[conn].evt,
+				    EVTCHNL_TYPE_EVT);
+		if (ret < 0) {
+			DRM_ERROR("Error allocating in-event channel\n");
+			goto fail;
+		}
+	}
+	front_info->num_evt_pairs = cfg->num_connectors;
+	return 0;
+
+fail:
+	xen_drm_front_evtchnl_free_all(front_info);
+	return ret;
+}
+
+static int evtchnl_publish(struct xenbus_transaction xbt,
+			   struct xen_drm_front_evtchnl *evtchnl,
+			   const char *path, const char *node_ring,
+			   const char *node_chnl)
+{
+	struct xenbus_device *xb_dev = evtchnl->front_info->xb_dev;
+	int ret;
+
+	/* write control channel ring reference */
+	ret = xenbus_printf(xbt, path, node_ring, "%u", evtchnl->gref);
+	if (ret < 0) {
+		xenbus_dev_error(xb_dev, ret, "writing ring-ref");
+		return ret;
+	}
+
+	/* write event channel ring reference */
+	ret = xenbus_printf(xbt, path, node_chnl, "%u", evtchnl->port);
+	if (ret < 0) {
+		xenbus_dev_error(xb_dev, ret, "writing event channel");
+		return ret;
+	}
+
+	return 0;
+}
+
+int xen_drm_front_evtchnl_publish_all(struct xen_drm_front_info *front_info)
+{
+	struct xenbus_transaction xbt;
+	struct xen_drm_front_cfg *plat_data;
+	int ret, conn;
+
+	plat_data = &front_info->cfg;
+
+again:
+	ret = xenbus_transaction_start(&xbt);
+	if (ret < 0) {
+		xenbus_dev_fatal(front_info->xb_dev, ret,
+				 "starting transaction");
+		return ret;
+	}
+
+	for (conn = 0; conn < plat_data->num_connectors; conn++) {
+		ret = evtchnl_publish(xbt, &front_info->evt_pairs[conn].req,
+				      plat_data->connectors[conn].xenstore_path,
+				      XENDISPL_FIELD_REQ_RING_REF,
+				      XENDISPL_FIELD_REQ_CHANNEL);
+		if (ret < 0)
+			goto fail;
+
+		ret = evtchnl_publish(xbt, &front_info->evt_pairs[conn].evt,
+				      plat_data->connectors[conn].xenstore_path,
+				      XENDISPL_FIELD_EVT_RING_REF,
+				      XENDISPL_FIELD_EVT_CHANNEL);
+		if (ret < 0)
+			goto fail;
+	}
+
+	ret = xenbus_transaction_end(xbt, 0);
+	if (ret < 0) {
+		if (ret == -EAGAIN)
+			goto again;
+
+		xenbus_dev_fatal(front_info->xb_dev, ret,
+				 "completing transaction");
+		goto fail_to_end;
+	}
+
+	return 0;
+
+fail:
+	xenbus_transaction_end(xbt, 1);
+
+fail_to_end:
+	xenbus_dev_fatal(front_info->xb_dev, ret, "writing Xen store");
+	return ret;
+}
+
+void xen_drm_front_evtchnl_flush(struct xen_drm_front_evtchnl *evtchnl)
+{
+	int notify;
+
+	evtchnl->u.req.ring.req_prod_pvt++;
+	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&evtchnl->u.req.ring, notify);
+	if (notify)
+		notify_remote_via_irq(evtchnl->irq);
+}
+
+void xen_drm_front_evtchnl_set_state(struct xen_drm_front_info *front_info,
+				     enum xen_drm_front_evtchnl_state state)
+{
+	unsigned long flags;
+	int i;
+
+	if (!front_info->evt_pairs)
+		return;
+
+	spin_lock_irqsave(&front_info->io_lock, flags);
+	for (i = 0; i < front_info->num_evt_pairs; i++) {
+		front_info->evt_pairs[i].req.state = state;
+		front_info->evt_pairs[i].evt.state = state;
+	}
+	spin_unlock_irqrestore(&front_info->io_lock, flags);
+}
+
+void xen_drm_front_evtchnl_free_all(struct xen_drm_front_info *front_info)
+{
+	int i;
+
+	if (!front_info->evt_pairs)
+		return;
+
+	for (i = 0; i < front_info->num_evt_pairs; i++) {
+		evtchnl_free(front_info, &front_info->evt_pairs[i].req);
+		evtchnl_free(front_info, &front_info->evt_pairs[i].evt);
+	}
+
+	kfree(front_info->evt_pairs);
+	front_info->evt_pairs = NULL;
+}
diff --git a/drivers/gpu/drm/xen/xen_drm_front_evtchnl.h b/drivers/gpu/drm/xen/xen_drm_front_evtchnl.h
new file mode 100644
index 000000000000..b0af6994332b
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_evtchnl.h
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#ifndef __XEN_DRM_FRONT_EVTCHNL_H_
+#define __XEN_DRM_FRONT_EVTCHNL_H_
+
+#include <linux/completion.h>
+#include <linux/types.h>
+
+#include <xen/interface/io/ring.h>
+#include <xen/interface/io/displif.h>
+
+/*
+ * All operations which are not connector oriented use this ctrl event channel,
+ * e.g. fb_attach/destroy which belong to a DRM device, not to a CRTC.
+ */
+#define GENERIC_OP_EVT_CHNL	0
+
+enum xen_drm_front_evtchnl_state {
+	EVTCHNL_STATE_DISCONNECTED,
+	EVTCHNL_STATE_CONNECTED,
+};
+
+enum xen_drm_front_evtchnl_type {
+	EVTCHNL_TYPE_REQ,
+	EVTCHNL_TYPE_EVT,
+};
+
+struct xen_drm_front_drm_info;
+
+struct xen_drm_front_evtchnl {
+	struct xen_drm_front_info *front_info;
+	int gref;
+	int port;
+	int irq;
+	int index;
+	enum xen_drm_front_evtchnl_state state;
+	enum xen_drm_front_evtchnl_type type;
+	/* either response id or incoming event id */
+	u16 evt_id;
+	/* next request id or next expected event id */
+	u16 evt_next_id;
+	union {
+		struct {
+			struct xen_displif_front_ring ring;
+			struct completion completion;
+			/* latest response status */
+			int resp_status;
+			/* serializer for backend IO: request/response */
+			struct mutex req_io_lock;
+		} req;
+		struct {
+			struct xendispl_event_page *page;
+		} evt;
+	} u;
+};
+
+struct xen_drm_front_evtchnl_pair {
+	struct xen_drm_front_evtchnl req;
+	struct xen_drm_front_evtchnl evt;
+};
+
+int xen_drm_front_evtchnl_create_all(struct xen_drm_front_info *front_info);
+
+int xen_drm_front_evtchnl_publish_all(struct xen_drm_front_info *front_info);
+
+void xen_drm_front_evtchnl_flush(struct xen_drm_front_evtchnl *evtchnl);
+
+void xen_drm_front_evtchnl_set_state(struct xen_drm_front_info *front_info,
+				     enum xen_drm_front_evtchnl_state state);
+
+void xen_drm_front_evtchnl_free_all(struct xen_drm_front_info *front_info);
+
+#endif /* __XEN_DRM_FRONT_EVTCHNL_H_ */
diff --git a/drivers/gpu/drm/xen/xen_drm_front_gem.c b/drivers/gpu/drm/xen/xen_drm_front_gem.c
new file mode 100644
index 000000000000..c85bfe7571cb
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_gem.c
@@ -0,0 +1,308 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#include "xen_drm_front_gem.h"
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_gem.h>
+
+#include <linux/dma-buf.h>
+#include <linux/scatterlist.h>
+#include <linux/shmem_fs.h>
+
+#include <xen/balloon.h>
+
+#include "xen_drm_front.h"
+#include "xen_drm_front_shbuf.h"
+
+struct xen_gem_object {
+	struct drm_gem_object base;
+
+	size_t num_pages;
+	struct page **pages;
+
+	/* set for buffers allocated by the backend */
+	bool be_alloc;
+
+	/* this is for imported PRIME buffer */
+	struct sg_table *sgt_imported;
+};
+
+static inline struct xen_gem_object *
+to_xen_gem_obj(struct drm_gem_object *gem_obj)
+{
+	return container_of(gem_obj, struct xen_gem_object, base);
+}
+
+static int gem_alloc_pages_array(struct xen_gem_object *xen_obj,
+				 size_t buf_size)
+{
+	xen_obj->num_pages = DIV_ROUND_UP(buf_size, PAGE_SIZE);
+	xen_obj->pages = kvmalloc_array(xen_obj->num_pages,
+					sizeof(struct page *), GFP_KERNEL);
+	return !xen_obj->pages ? -ENOMEM : 0;
+}
+
+static void gem_free_pages_array(struct xen_gem_object *xen_obj)
+{
+	kvfree(xen_obj->pages);
+	xen_obj->pages = NULL;
+}
+
+static struct xen_gem_object *gem_create_obj(struct drm_device *dev,
+					     size_t size)
+{
+	struct xen_gem_object *xen_obj;
+	int ret;
+
+	xen_obj = kzalloc(sizeof(*xen_obj), GFP_KERNEL);
+	if (!xen_obj)
+		return ERR_PTR(-ENOMEM);
+
+	ret = drm_gem_object_init(dev, &xen_obj->base, size);
+	if (ret < 0) {
+		kfree(xen_obj);
+		return ERR_PTR(ret);
+	}
+
+	return xen_obj;
+}
+
+static struct xen_gem_object *gem_create(struct drm_device *dev, size_t size)
+{
+	struct xen_drm_front_drm_info *drm_info = dev->dev_private;
+	struct xen_gem_object *xen_obj;
+	int ret;
+
+	size = round_up(size, PAGE_SIZE);
+	xen_obj = gem_create_obj(dev, size);
+	if (IS_ERR_OR_NULL(xen_obj))
+		return xen_obj;
+
+	if (drm_info->front_info->cfg.be_alloc) {
+		/*
+		 * backend will allocate space for this buffer, so
+		 * only allocate array of pointers to pages
+		 */
+		ret = gem_alloc_pages_array(xen_obj, size);
+		if (ret < 0)
+			goto fail;
+
+		/*
+		 * allocate ballooned pages which will be used to map
+		 * grant references provided by the backend
+		 */
+		ret = alloc_xenballooned_pages(xen_obj->num_pages,
+					       xen_obj->pages);
+		if (ret < 0) {
+			DRM_ERROR("Cannot allocate %zu ballooned pages: %d\n",
+				  xen_obj->num_pages, ret);
+			gem_free_pages_array(xen_obj);
+			goto fail;
+		}
+
+		xen_obj->be_alloc = true;
+		return xen_obj;
+	}
+	/*
+	 * need to allocate backing pages now, so we can share those
+	 * with the backend
+	 */
+	xen_obj->num_pages = DIV_ROUND_UP(size, PAGE_SIZE);
+	xen_obj->pages = drm_gem_get_pages(&xen_obj->base);
+	if (IS_ERR_OR_NULL(xen_obj->pages)) {
+		ret = PTR_ERR(xen_obj->pages);
+		xen_obj->pages = NULL;
+		goto fail;
+	}
+
+	return xen_obj;
+
+fail:
+	DRM_ERROR("Failed to allocate buffer with size %zu\n", size);
+	return ERR_PTR(ret);
+}
+
+struct drm_gem_object *xen_drm_front_gem_create(struct drm_device *dev,
+						size_t size)
+{
+	struct xen_gem_object *xen_obj;
+
+	xen_obj = gem_create(dev, size);
+	if (IS_ERR_OR_NULL(xen_obj))
+		return ERR_CAST(xen_obj);
+
+	return &xen_obj->base;
+}
+
+void xen_drm_front_gem_free_object_unlocked(struct drm_gem_object *gem_obj)
+{
+	struct xen_gem_object *xen_obj = to_xen_gem_obj(gem_obj);
+
+	if (xen_obj->base.import_attach) {
+		drm_prime_gem_destroy(&xen_obj->base, xen_obj->sgt_imported);
+		gem_free_pages_array(xen_obj);
+	} else {
+		if (xen_obj->pages) {
+			if (xen_obj->be_alloc) {
+				free_xenballooned_pages(xen_obj->num_pages,
+							xen_obj->pages);
+				gem_free_pages_array(xen_obj);
+			} else {
+				drm_gem_put_pages(&xen_obj->base,
+						  xen_obj->pages, true, false);
+			}
+		}
+	}
+	drm_gem_object_release(gem_obj);
+	kfree(xen_obj);
+}
+
+struct page **xen_drm_front_gem_get_pages(struct drm_gem_object *gem_obj)
+{
+	struct xen_gem_object *xen_obj = to_xen_gem_obj(gem_obj);
+
+	return xen_obj->pages;
+}
+
+struct sg_table *xen_drm_front_gem_get_sg_table(struct drm_gem_object *gem_obj)
+{
+	struct xen_gem_object *xen_obj = to_xen_gem_obj(gem_obj);
+
+	if (!xen_obj->pages)
+		return NULL;
+
+	return drm_prime_pages_to_sg(xen_obj->pages, xen_obj->num_pages);
+}
+
+struct drm_gem_object *
+xen_drm_front_gem_import_sg_table(struct drm_device *dev,
+				  struct dma_buf_attachment *attach,
+				  struct sg_table *sgt)
+{
+	struct xen_drm_front_drm_info *drm_info = dev->dev_private;
+	struct xen_gem_object *xen_obj;
+	size_t size;
+	int ret;
+
+	size = attach->dmabuf->size;
+	xen_obj = gem_create_obj(dev, size);
+	if (IS_ERR_OR_NULL(xen_obj))
+		return ERR_CAST(xen_obj);
+
+	ret = gem_alloc_pages_array(xen_obj, size);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	xen_obj->sgt_imported = sgt;
+
+	ret = drm_prime_sg_to_page_addr_arrays(sgt, xen_obj->pages,
+					       NULL, xen_obj->num_pages);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	ret = xen_drm_front_dbuf_create(drm_info->front_info,
+					xen_drm_front_dbuf_to_cookie(&xen_obj->base),
+					0, 0, 0, size, xen_obj->pages);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	DRM_DEBUG("Imported buffer of size %zu with nents %u\n",
+		  size, sgt->nents);
+
+	return &xen_obj->base;
+}
+
+static int gem_mmap_obj(struct xen_gem_object *xen_obj,
+			struct vm_area_struct *vma)
+{
+	unsigned long addr = vma->vm_start;
+	int i;
+
+	/*
+	 * clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the
+	 * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map
+	 * the whole buffer.
+	 */
+	vma->vm_flags &= ~VM_PFNMAP;
+	vma->vm_flags |= VM_MIXEDMAP;
+	vma->vm_pgoff = 0;
+	vma->vm_page_prot =
+			pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+
+	/*
+	 * vm_operations_struct.fault handler will be called if CPU access
+	 * to VM is here. For GPUs this isn't the case, because CPU
+	 * doesn't touch the memory. Insert pages now, so both CPU and GPU are
+	 * happy.
+	 * FIXME: as we insert all the pages now then no .fault handler must
+	 * be called, so don't provide one
+	 */
+	for (i = 0; i < xen_obj->num_pages; i++) {
+		int ret;
+
+		ret = vm_insert_page(vma, addr, xen_obj->pages[i]);
+		if (ret < 0) {
+			DRM_ERROR("Failed to insert pages into vma: %d\n", ret);
+			return ret;
+		}
+
+		addr += PAGE_SIZE;
+	}
+	return 0;
+}
+
+int xen_drm_front_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct xen_gem_object *xen_obj;
+	struct drm_gem_object *gem_obj;
+	int ret;
+
+	ret = drm_gem_mmap(filp, vma);
+	if (ret < 0)
+		return ret;
+
+	gem_obj = vma->vm_private_data;
+	xen_obj = to_xen_gem_obj(gem_obj);
+	return gem_mmap_obj(xen_obj, vma);
+}
+
+void *xen_drm_front_gem_prime_vmap(struct drm_gem_object *gem_obj)
+{
+	struct xen_gem_object *xen_obj = to_xen_gem_obj(gem_obj);
+
+	if (!xen_obj->pages)
+		return NULL;
+
+	return vmap(xen_obj->pages, xen_obj->num_pages,
+		    VM_MAP, pgprot_writecombine(PAGE_KERNEL));
+}
+
+void xen_drm_front_gem_prime_vunmap(struct drm_gem_object *gem_obj,
+				    void *vaddr)
+{
+	vunmap(vaddr);
+}
+
+int xen_drm_front_gem_prime_mmap(struct drm_gem_object *gem_obj,
+				 struct vm_area_struct *vma)
+{
+	struct xen_gem_object *xen_obj;
+	int ret;
+
+	ret = drm_gem_mmap_obj(gem_obj, gem_obj->size, vma);
+	if (ret < 0)
+		return ret;
+
+	xen_obj = to_xen_gem_obj(gem_obj);
+	return gem_mmap_obj(xen_obj, vma);
+}
diff --git a/drivers/gpu/drm/xen/xen_drm_front_gem.h b/drivers/gpu/drm/xen/xen_drm_front_gem.h
new file mode 100644
index 000000000000..d5ab734fdafe
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_gem.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#ifndef __XEN_DRM_FRONT_GEM_H
+#define __XEN_DRM_FRONT_GEM_H
+
+#include <drm/drmP.h>
+
+struct drm_gem_object *xen_drm_front_gem_create(struct drm_device *dev,
+						size_t size);
+
+struct drm_gem_object *
+xen_drm_front_gem_import_sg_table(struct drm_device *dev,
+				  struct dma_buf_attachment *attach,
+				  struct sg_table *sgt);
+
+struct sg_table *xen_drm_front_gem_get_sg_table(struct drm_gem_object *gem_obj);
+
+struct page **xen_drm_front_gem_get_pages(struct drm_gem_object *obj);
+
+void xen_drm_front_gem_free_object_unlocked(struct drm_gem_object *gem_obj);
+
+int xen_drm_front_gem_mmap(struct file *filp, struct vm_area_struct *vma);
+
+void *xen_drm_front_gem_prime_vmap(struct drm_gem_object *gem_obj);
+
+void xen_drm_front_gem_prime_vunmap(struct drm_gem_object *gem_obj,
+				    void *vaddr);
+
+int xen_drm_front_gem_prime_mmap(struct drm_gem_object *gem_obj,
+				 struct vm_area_struct *vma);
+
+#endif /* __XEN_DRM_FRONT_GEM_H */
diff --git a/drivers/gpu/drm/xen/xen_drm_front_kms.c b/drivers/gpu/drm/xen/xen_drm_front_kms.c
new file mode 100644
index 000000000000..a3479eb72d79
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_kms.c
@@ -0,0 +1,366 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#include "xen_drm_front_kms.h"
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+
+#include "xen_drm_front.h"
+#include "xen_drm_front_conn.h"
+
+/*
+ * Timeout in ms to wait for frame done event from the backend:
+ * must be a bit more than IO time-out
+ */
+#define FRAME_DONE_TO_MS	(XEN_DRM_FRONT_WAIT_BACK_MS + 100)
+
+static struct xen_drm_front_drm_pipeline *
+to_xen_drm_pipeline(struct drm_simple_display_pipe *pipe)
+{
+	return container_of(pipe, struct xen_drm_front_drm_pipeline, pipe);
+}
+
+static void fb_destroy(struct drm_framebuffer *fb)
+{
+	struct xen_drm_front_drm_info *drm_info = fb->dev->dev_private;
+	int idx;
+
+	if (drm_dev_enter(fb->dev, &idx)) {
+		xen_drm_front_fb_detach(drm_info->front_info,
+					xen_drm_front_fb_to_cookie(fb));
+		drm_dev_exit(idx);
+	}
+	drm_gem_fb_destroy(fb);
+}
+
+static struct drm_framebuffer_funcs fb_funcs = {
+	.destroy = fb_destroy,
+};
+
+static struct drm_framebuffer *
+fb_create(struct drm_device *dev, struct drm_file *filp,
+	  const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct xen_drm_front_drm_info *drm_info = dev->dev_private;
+	static struct drm_framebuffer *fb;
+	struct drm_gem_object *gem_obj;
+	int ret;
+
+	fb = drm_gem_fb_create_with_funcs(dev, filp, mode_cmd, &fb_funcs);
+	if (IS_ERR_OR_NULL(fb))
+		return fb;
+
+	gem_obj = drm_gem_object_lookup(filp, mode_cmd->handles[0]);
+	if (!gem_obj) {
+		DRM_ERROR("Failed to lookup GEM object\n");
+		ret = -ENOENT;
+		goto fail;
+	}
+
+	drm_gem_object_put_unlocked(gem_obj);
+
+	ret = xen_drm_front_fb_attach(drm_info->front_info,
+				      xen_drm_front_dbuf_to_cookie(gem_obj),
+				      xen_drm_front_fb_to_cookie(fb),
+				      fb->width, fb->height,
+				      fb->format->format);
+	if (ret < 0) {
+		DRM_ERROR("Back failed to attach FB %p: %d\n", fb, ret);
+		goto fail;
+	}
+
+	return fb;
+
+fail:
+	drm_gem_fb_destroy(fb);
+	return ERR_PTR(ret);
+}
+
+static const struct drm_mode_config_funcs mode_config_funcs = {
+	.fb_create = fb_create,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static void send_pending_event(struct xen_drm_front_drm_pipeline *pipeline)
+{
+	struct drm_crtc *crtc = &pipeline->pipe.crtc;
+	struct drm_device *dev = crtc->dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->event_lock, flags);
+	if (pipeline->pending_event)
+		drm_crtc_send_vblank_event(crtc, pipeline->pending_event);
+	pipeline->pending_event = NULL;
+	spin_unlock_irqrestore(&dev->event_lock, flags);
+}
+
+static void display_enable(struct drm_simple_display_pipe *pipe,
+			   struct drm_crtc_state *crtc_state,
+			   struct drm_plane_state *plane_state)
+{
+	struct xen_drm_front_drm_pipeline *pipeline =
+			to_xen_drm_pipeline(pipe);
+	struct drm_crtc *crtc = &pipe->crtc;
+	struct drm_framebuffer *fb = plane_state->fb;
+	int ret, idx;
+
+	if (!drm_dev_enter(pipe->crtc.dev, &idx))
+		return;
+
+	ret = xen_drm_front_mode_set(pipeline, crtc->x, crtc->y,
+				     fb->width, fb->height,
+				     fb->format->cpp[0] * 8,
+				     xen_drm_front_fb_to_cookie(fb));
+
+	if (ret) {
+		DRM_ERROR("Failed to enable display: %d\n", ret);
+		pipeline->conn_connected = false;
+	}
+
+	drm_dev_exit(idx);
+}
+
+static void display_disable(struct drm_simple_display_pipe *pipe)
+{
+	struct xen_drm_front_drm_pipeline *pipeline =
+			to_xen_drm_pipeline(pipe);
+	int ret = 0, idx;
+
+	if (drm_dev_enter(pipe->crtc.dev, &idx)) {
+		ret = xen_drm_front_mode_set(pipeline, 0, 0, 0, 0, 0,
+					     xen_drm_front_fb_to_cookie(NULL));
+		drm_dev_exit(idx);
+	}
+	if (ret)
+		DRM_ERROR("Failed to disable display: %d\n", ret);
+
+	/* Make sure we can restart with enabled connector next time */
+	pipeline->conn_connected = true;
+
+	/* release stalled event if any */
+	send_pending_event(pipeline);
+}
+
+void xen_drm_front_kms_on_frame_done(struct xen_drm_front_drm_pipeline *pipeline,
+				     u64 fb_cookie)
+{
+	/*
+	 * This runs in interrupt context, e.g. under
+	 * drm_info->front_info->io_lock, so we cannot call _sync version
+	 * to cancel the work
+	 */
+	cancel_delayed_work(&pipeline->pflip_to_worker);
+
+	send_pending_event(pipeline);
+}
+
+static void pflip_to_worker(struct work_struct *work)
+{
+	struct delayed_work *delayed_work = to_delayed_work(work);
+	struct xen_drm_front_drm_pipeline *pipeline =
+			container_of(delayed_work,
+				     struct xen_drm_front_drm_pipeline,
+				     pflip_to_worker);
+
+	DRM_ERROR("Frame done timed-out, releasing");
+	send_pending_event(pipeline);
+}
+
+static bool display_send_page_flip(struct drm_simple_display_pipe *pipe,
+				   struct drm_plane_state *old_plane_state)
+{
+	struct drm_plane_state *plane_state =
+			drm_atomic_get_new_plane_state(old_plane_state->state,
+						       &pipe->plane);
+
+	/*
+	 * If old_plane_state->fb is NULL and plane_state->fb is not,
+	 * then this is an atomic commit which will enable display.
+	 * If old_plane_state->fb is not NULL and plane_state->fb is,
+	 * then this is an atomic commit which will disable display.
+	 * Ignore these and do not send page flip as this framebuffer will be
+	 * sent to the backend as a part of display_set_config call.
+	 */
+	if (old_plane_state->fb && plane_state->fb) {
+		struct xen_drm_front_drm_pipeline *pipeline =
+				to_xen_drm_pipeline(pipe);
+		struct xen_drm_front_drm_info *drm_info = pipeline->drm_info;
+		int ret;
+
+		schedule_delayed_work(&pipeline->pflip_to_worker,
+				      msecs_to_jiffies(FRAME_DONE_TO_MS));
+
+		ret = xen_drm_front_page_flip(drm_info->front_info,
+					      pipeline->index,
+					      xen_drm_front_fb_to_cookie(plane_state->fb));
+		if (ret) {
+			DRM_ERROR("Failed to send page flip request to backend: %d\n", ret);
+
+			pipeline->conn_connected = false;
+			/*
+			 * Report the flip not handled, so pending event is
+			 * sent, unblocking user-space.
+			 */
+			return false;
+		}
+		/*
+		 * Signal that page flip was handled, pending event will be sent
+		 * on frame done event from the backend.
+		 */
+		return true;
+	}
+
+	return false;
+}
+
+static void display_update(struct drm_simple_display_pipe *pipe,
+			   struct drm_plane_state *old_plane_state)
+{
+	struct xen_drm_front_drm_pipeline *pipeline =
+			to_xen_drm_pipeline(pipe);
+	struct drm_crtc *crtc = &pipe->crtc;
+	struct drm_pending_vblank_event *event;
+	int idx;
+
+	event = crtc->state->event;
+	if (event) {
+		struct drm_device *dev = crtc->dev;
+		unsigned long flags;
+
+		WARN_ON(pipeline->pending_event);
+
+		spin_lock_irqsave(&dev->event_lock, flags);
+		crtc->state->event = NULL;
+
+		pipeline->pending_event = event;
+		spin_unlock_irqrestore(&dev->event_lock, flags);
+	}
+
+	if (!drm_dev_enter(pipe->crtc.dev, &idx)) {
+		send_pending_event(pipeline);
+		return;
+	}
+
+	/*
+	 * Send page flip request to the backend *after* we have event cached
+	 * above, so on page flip done event from the backend we can
+	 * deliver it and there is no race condition between this code and
+	 * event from the backend.
+	 * If this is not a page flip, e.g. no flip done event from the backend
+	 * is expected, then send now.
+	 */
+	if (!display_send_page_flip(pipe, old_plane_state))
+		send_pending_event(pipeline);
+
+	drm_dev_exit(idx);
+}
+
+static enum drm_mode_status
+display_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode *mode)
+{
+	struct xen_drm_front_drm_pipeline *pipeline =
+			container_of(crtc, struct xen_drm_front_drm_pipeline,
+				     pipe.crtc);
+
+	if (mode->hdisplay != pipeline->width)
+		return MODE_ERROR;
+
+	if (mode->vdisplay != pipeline->height)
+		return MODE_ERROR;
+
+	return MODE_OK;
+}
+
+static const struct drm_simple_display_pipe_funcs display_funcs = {
+	.mode_valid = display_mode_valid,
+	.enable = display_enable,
+	.disable = display_disable,
+	.prepare_fb = drm_gem_fb_simple_display_pipe_prepare_fb,
+	.update = display_update,
+};
+
+static int display_pipe_init(struct xen_drm_front_drm_info *drm_info,
+			     int index, struct xen_drm_front_cfg_connector *cfg,
+			     struct xen_drm_front_drm_pipeline *pipeline)
+{
+	struct drm_device *dev = drm_info->drm_dev;
+	const u32 *formats;
+	int format_count;
+	int ret;
+
+	pipeline->drm_info = drm_info;
+	pipeline->index = index;
+	pipeline->height = cfg->height;
+	pipeline->width = cfg->width;
+
+	INIT_DELAYED_WORK(&pipeline->pflip_to_worker, pflip_to_worker);
+
+	ret = xen_drm_front_conn_init(drm_info, &pipeline->conn);
+	if (ret)
+		return ret;
+
+	formats = xen_drm_front_conn_get_formats(&format_count);
+
+	return drm_simple_display_pipe_init(dev, &pipeline->pipe,
+					    &display_funcs, formats,
+					    format_count, NULL,
+					    &pipeline->conn);
+}
+
+int xen_drm_front_kms_init(struct xen_drm_front_drm_info *drm_info)
+{
+	struct drm_device *dev = drm_info->drm_dev;
+	int i, ret;
+
+	drm_mode_config_init(dev);
+
+	dev->mode_config.min_width = 0;
+	dev->mode_config.min_height = 0;
+	dev->mode_config.max_width = 4095;
+	dev->mode_config.max_height = 2047;
+	dev->mode_config.funcs = &mode_config_funcs;
+
+	for (i = 0; i < drm_info->front_info->cfg.num_connectors; i++) {
+		struct xen_drm_front_cfg_connector *cfg =
+				&drm_info->front_info->cfg.connectors[i];
+		struct xen_drm_front_drm_pipeline *pipeline =
+				&drm_info->pipeline[i];
+
+		ret = display_pipe_init(drm_info, i, cfg, pipeline);
+		if (ret) {
+			drm_mode_config_cleanup(dev);
+			return ret;
+		}
+	}
+
+	drm_mode_config_reset(dev);
+	drm_kms_helper_poll_init(dev);
+	return 0;
+}
+
+void xen_drm_front_kms_fini(struct xen_drm_front_drm_info *drm_info)
+{
+	int i;
+
+	for (i = 0; i < drm_info->front_info->cfg.num_connectors; i++) {
+		struct xen_drm_front_drm_pipeline *pipeline =
+				&drm_info->pipeline[i];
+
+		cancel_delayed_work_sync(&pipeline->pflip_to_worker);
+
+		send_pending_event(pipeline);
+	}
+}
diff --git a/drivers/gpu/drm/xen/xen_drm_front_kms.h b/drivers/gpu/drm/xen/xen_drm_front_kms.h
new file mode 100644
index 000000000000..ab2fbad4fbbf
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_kms.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#ifndef __XEN_DRM_FRONT_KMS_H_
+#define __XEN_DRM_FRONT_KMS_H_
+
+#include <linux/types.h>
+
+struct xen_drm_front_drm_info;
+struct xen_drm_front_drm_pipeline;
+
+int xen_drm_front_kms_init(struct xen_drm_front_drm_info *drm_info);
+
+void xen_drm_front_kms_fini(struct xen_drm_front_drm_info *drm_info);
+
+void xen_drm_front_kms_on_frame_done(struct xen_drm_front_drm_pipeline *pipeline,
+				     u64 fb_cookie);
+
+#endif /* __XEN_DRM_FRONT_KMS_H_ */
diff --git a/drivers/gpu/drm/xen/xen_drm_front_shbuf.c b/drivers/gpu/drm/xen/xen_drm_front_shbuf.c
new file mode 100644
index 000000000000..8099cb343ae3
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_shbuf.c
@@ -0,0 +1,414 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#include <drm/drmP.h>
+
+#if defined(CONFIG_X86)
+#include <drm/drm_cache.h>
+#endif
+#include <linux/errno.h>
+#include <linux/mm.h>
+
+#include <asm/xen/hypervisor.h>
+#include <xen/balloon.h>
+#include <xen/xen.h>
+#include <xen/xenbus.h>
+#include <xen/interface/io/ring.h>
+#include <xen/interface/io/displif.h>
+
+#include "xen_drm_front.h"
+#include "xen_drm_front_shbuf.h"
+
+struct xen_drm_front_shbuf_ops {
+	/*
+	 * Calculate number of grefs required to handle this buffer,
+	 * e.g. if grefs are required for page directory only or the buffer
+	 * pages as well.
+	 */
+	void (*calc_num_grefs)(struct xen_drm_front_shbuf *buf);
+	/* Fill page directory according to para-virtual display protocol. */
+	void (*fill_page_dir)(struct xen_drm_front_shbuf *buf);
+	/* Claim grant references for the pages of the buffer. */
+	int (*grant_refs_for_buffer)(struct xen_drm_front_shbuf *buf,
+				     grant_ref_t *priv_gref_head, int gref_idx);
+	/* Map grant references of the buffer. */
+	int (*map)(struct xen_drm_front_shbuf *buf);
+	/* Unmap grant references of the buffer. */
+	int (*unmap)(struct xen_drm_front_shbuf *buf);
+};
+
+grant_ref_t xen_drm_front_shbuf_get_dir_start(struct xen_drm_front_shbuf *buf)
+{
+	if (!buf->grefs)
+		return GRANT_INVALID_REF;
+
+	return buf->grefs[0];
+}
+
+int xen_drm_front_shbuf_map(struct xen_drm_front_shbuf *buf)
+{
+	if (buf->ops->map)
+		return buf->ops->map(buf);
+
+	/* no need to map own grant references */
+	return 0;
+}
+
+int xen_drm_front_shbuf_unmap(struct xen_drm_front_shbuf *buf)
+{
+	if (buf->ops->unmap)
+		return buf->ops->unmap(buf);
+
+	/* no need to unmap own grant references */
+	return 0;
+}
+
+void xen_drm_front_shbuf_flush(struct xen_drm_front_shbuf *buf)
+{
+#if defined(CONFIG_X86)
+	drm_clflush_pages(buf->pages, buf->num_pages);
+#endif
+}
+
+void xen_drm_front_shbuf_free(struct xen_drm_front_shbuf *buf)
+{
+	if (buf->grefs) {
+		int i;
+
+		for (i = 0; i < buf->num_grefs; i++)
+			if (buf->grefs[i] != GRANT_INVALID_REF)
+				gnttab_end_foreign_access(buf->grefs[i],
+							  0, 0UL);
+	}
+	kfree(buf->grefs);
+	kfree(buf->directory);
+	kfree(buf);
+}
+
+/*
+ * number of grefs a page can hold with respect to the
+ * struct xendispl_page_directory header
+ */
+#define XEN_DRM_NUM_GREFS_PER_PAGE ((PAGE_SIZE - \
+		offsetof(struct xendispl_page_directory, gref)) / \
+		sizeof(grant_ref_t))
+
+static int get_num_pages_dir(struct xen_drm_front_shbuf *buf)
+{
+	/* number of pages the page directory consumes itself */
+	return DIV_ROUND_UP(buf->num_pages, XEN_DRM_NUM_GREFS_PER_PAGE);
+}
+
+static void backend_calc_num_grefs(struct xen_drm_front_shbuf *buf)
+{
+	/* only for pages the page directory consumes itself */
+	buf->num_grefs = get_num_pages_dir(buf);
+}
+
+static void guest_calc_num_grefs(struct xen_drm_front_shbuf *buf)
+{
+	/*
+	 * number of pages the page directory consumes itself
+	 * plus grefs for the buffer pages
+	 */
+	buf->num_grefs = get_num_pages_dir(buf) + buf->num_pages;
+}
+
+#define xen_page_to_vaddr(page) \
+		((phys_addr_t)pfn_to_kaddr(page_to_xen_pfn(page)))
+
+static int backend_unmap(struct xen_drm_front_shbuf *buf)
+{
+	struct gnttab_unmap_grant_ref *unmap_ops;
+	int i, ret;
+
+	if (!buf->pages || !buf->backend_map_handles || !buf->grefs)
+		return 0;
+
+	unmap_ops = kcalloc(buf->num_pages, sizeof(*unmap_ops),
+			    GFP_KERNEL);
+	if (!unmap_ops) {
+		DRM_ERROR("Failed to get memory while unmapping\n");
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < buf->num_pages; i++) {
+		phys_addr_t addr;
+
+		addr = xen_page_to_vaddr(buf->pages[i]);
+		gnttab_set_unmap_op(&unmap_ops[i], addr, GNTMAP_host_map,
+				    buf->backend_map_handles[i]);
+	}
+
+	ret = gnttab_unmap_refs(unmap_ops, NULL, buf->pages,
+				buf->num_pages);
+
+	for (i = 0; i < buf->num_pages; i++) {
+		if (unlikely(unmap_ops[i].status != GNTST_okay))
+			DRM_ERROR("Failed to unmap page %d: %d\n",
+				  i, unmap_ops[i].status);
+	}
+
+	if (ret)
+		DRM_ERROR("Failed to unmap grant references, ret %d", ret);
+
+	kfree(unmap_ops);
+	kfree(buf->backend_map_handles);
+	buf->backend_map_handles = NULL;
+	return ret;
+}
+
+static int backend_map(struct xen_drm_front_shbuf *buf)
+{
+	struct gnttab_map_grant_ref *map_ops = NULL;
+	unsigned char *ptr;
+	int ret, cur_gref, cur_dir_page, cur_page, grefs_left;
+
+	map_ops = kcalloc(buf->num_pages, sizeof(*map_ops), GFP_KERNEL);
+	if (!map_ops)
+		return -ENOMEM;
+
+	buf->backend_map_handles = kcalloc(buf->num_pages,
+					   sizeof(*buf->backend_map_handles),
+					   GFP_KERNEL);
+	if (!buf->backend_map_handles) {
+		kfree(map_ops);
+		return -ENOMEM;
+	}
+
+	/*
+	 * read page directory to get grefs from the backend: for external
+	 * buffer we only allocate buf->grefs for the page directory,
+	 * so buf->num_grefs has number of pages in the page directory itself
+	 */
+	ptr = buf->directory;
+	grefs_left = buf->num_pages;
+	cur_page = 0;
+	for (cur_dir_page = 0; cur_dir_page < buf->num_grefs; cur_dir_page++) {
+		struct xendispl_page_directory *page_dir =
+				(struct xendispl_page_directory *)ptr;
+		int to_copy = XEN_DRM_NUM_GREFS_PER_PAGE;
+
+		if (to_copy > grefs_left)
+			to_copy = grefs_left;
+
+		for (cur_gref = 0; cur_gref < to_copy; cur_gref++) {
+			phys_addr_t addr;
+
+			addr = xen_page_to_vaddr(buf->pages[cur_page]);
+			gnttab_set_map_op(&map_ops[cur_page], addr,
+					  GNTMAP_host_map,
+					  page_dir->gref[cur_gref],
+					  buf->xb_dev->otherend_id);
+			cur_page++;
+		}
+
+		grefs_left -= to_copy;
+		ptr += PAGE_SIZE;
+	}
+	ret = gnttab_map_refs(map_ops, NULL, buf->pages, buf->num_pages);
+
+	/* save handles even if error, so we can unmap */
+	for (cur_page = 0; cur_page < buf->num_pages; cur_page++) {
+		buf->backend_map_handles[cur_page] = map_ops[cur_page].handle;
+		if (unlikely(map_ops[cur_page].status != GNTST_okay))
+			DRM_ERROR("Failed to map page %d: %d\n",
+				  cur_page, map_ops[cur_page].status);
+	}
+
+	if (ret) {
+		DRM_ERROR("Failed to map grant references, ret %d", ret);
+		backend_unmap(buf);
+	}
+
+	kfree(map_ops);
+	return ret;
+}
+
+static void backend_fill_page_dir(struct xen_drm_front_shbuf *buf)
+{
+	struct xendispl_page_directory *page_dir;
+	unsigned char *ptr;
+	int i, num_pages_dir;
+
+	ptr = buf->directory;
+	num_pages_dir = get_num_pages_dir(buf);
+
+	/* fill only grefs for the page directory itself */
+	for (i = 0; i < num_pages_dir - 1; i++) {
+		page_dir = (struct xendispl_page_directory *)ptr;
+
+		page_dir->gref_dir_next_page = buf->grefs[i + 1];
+		ptr += PAGE_SIZE;
+	}
+	/* last page must say there is no more pages */
+	page_dir = (struct xendispl_page_directory *)ptr;
+	page_dir->gref_dir_next_page = GRANT_INVALID_REF;
+}
+
+static void guest_fill_page_dir(struct xen_drm_front_shbuf *buf)
+{
+	unsigned char *ptr;
+	int cur_gref, grefs_left, to_copy, i, num_pages_dir;
+
+	ptr = buf->directory;
+	num_pages_dir = get_num_pages_dir(buf);
+
+	/*
+	 * while copying, skip grefs at start, they are for pages
+	 * granted for the page directory itself
+	 */
+	cur_gref = num_pages_dir;
+	grefs_left = buf->num_pages;
+	for (i = 0; i < num_pages_dir; i++) {
+		struct xendispl_page_directory *page_dir =
+				(struct xendispl_page_directory *)ptr;
+
+		if (grefs_left <= XEN_DRM_NUM_GREFS_PER_PAGE) {
+			to_copy = grefs_left;
+			page_dir->gref_dir_next_page = GRANT_INVALID_REF;
+		} else {
+			to_copy = XEN_DRM_NUM_GREFS_PER_PAGE;
+			page_dir->gref_dir_next_page = buf->grefs[i + 1];
+		}
+		memcpy(&page_dir->gref, &buf->grefs[cur_gref],
+		       to_copy * sizeof(grant_ref_t));
+		ptr += PAGE_SIZE;
+		grefs_left -= to_copy;
+		cur_gref += to_copy;
+	}
+}
+
+static int guest_grant_refs_for_buffer(struct xen_drm_front_shbuf *buf,
+				       grant_ref_t *priv_gref_head,
+				       int gref_idx)
+{
+	int i, cur_ref, otherend_id;
+
+	otherend_id = buf->xb_dev->otherend_id;
+	for (i = 0; i < buf->num_pages; i++) {
+		cur_ref = gnttab_claim_grant_reference(priv_gref_head);
+		if (cur_ref < 0)
+			return cur_ref;
+
+		gnttab_grant_foreign_access_ref(cur_ref, otherend_id,
+						xen_page_to_gfn(buf->pages[i]),
+						0);
+		buf->grefs[gref_idx++] = cur_ref;
+	}
+	return 0;
+}
+
+static int grant_references(struct xen_drm_front_shbuf *buf)
+{
+	grant_ref_t priv_gref_head;
+	int ret, i, j, cur_ref;
+	int otherend_id, num_pages_dir;
+
+	ret = gnttab_alloc_grant_references(buf->num_grefs, &priv_gref_head);
+	if (ret < 0) {
+		DRM_ERROR("Cannot allocate grant references\n");
+		return ret;
+	}
+
+	otherend_id = buf->xb_dev->otherend_id;
+	j = 0;
+	num_pages_dir = get_num_pages_dir(buf);
+	for (i = 0; i < num_pages_dir; i++) {
+		unsigned long frame;
+
+		cur_ref = gnttab_claim_grant_reference(&priv_gref_head);
+		if (cur_ref < 0)
+			return cur_ref;
+
+		frame = xen_page_to_gfn(virt_to_page(buf->directory +
+					PAGE_SIZE * i));
+		gnttab_grant_foreign_access_ref(cur_ref, otherend_id, frame, 0);
+		buf->grefs[j++] = cur_ref;
+	}
+
+	if (buf->ops->grant_refs_for_buffer) {
+		ret = buf->ops->grant_refs_for_buffer(buf, &priv_gref_head, j);
+		if (ret)
+			return ret;
+	}
+
+	gnttab_free_grant_references(priv_gref_head);
+	return 0;
+}
+
+static int alloc_storage(struct xen_drm_front_shbuf *buf)
+{
+	buf->grefs = kcalloc(buf->num_grefs, sizeof(*buf->grefs), GFP_KERNEL);
+	if (!buf->grefs)
+		return -ENOMEM;
+
+	buf->directory = kcalloc(get_num_pages_dir(buf), PAGE_SIZE, GFP_KERNEL);
+	if (!buf->directory)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/*
+ * For be allocated buffers we don't need grant_refs_for_buffer as those
+ * grant references are allocated at backend side
+ */
+static const struct xen_drm_front_shbuf_ops backend_ops = {
+	.calc_num_grefs = backend_calc_num_grefs,
+	.fill_page_dir = backend_fill_page_dir,
+	.map = backend_map,
+	.unmap = backend_unmap
+};
+
+/* For locally granted references we do not need to map/unmap the references */
+static const struct xen_drm_front_shbuf_ops local_ops = {
+	.calc_num_grefs = guest_calc_num_grefs,
+	.fill_page_dir = guest_fill_page_dir,
+	.grant_refs_for_buffer = guest_grant_refs_for_buffer,
+};
+
+struct xen_drm_front_shbuf *
+xen_drm_front_shbuf_alloc(struct xen_drm_front_shbuf_cfg *cfg)
+{
+	struct xen_drm_front_shbuf *buf;
+	int ret;
+
+	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
+	if (!buf)
+		return ERR_PTR(-ENOMEM);
+
+	if (cfg->be_alloc)
+		buf->ops = &backend_ops;
+	else
+		buf->ops = &local_ops;
+
+	buf->xb_dev = cfg->xb_dev;
+	buf->num_pages = DIV_ROUND_UP(cfg->size, PAGE_SIZE);
+	buf->pages = cfg->pages;
+
+	buf->ops->calc_num_grefs(buf);
+
+	ret = alloc_storage(buf);
+	if (ret)
+		goto fail;
+
+	ret = grant_references(buf);
+	if (ret)
+		goto fail;
+
+	buf->ops->fill_page_dir(buf);
+
+	return buf;
+
+fail:
+	xen_drm_front_shbuf_free(buf);
+	return ERR_PTR(ret);
+}
diff --git a/drivers/gpu/drm/xen/xen_drm_front_shbuf.h b/drivers/gpu/drm/xen/xen_drm_front_shbuf.h
new file mode 100644
index 000000000000..7545c692539e
--- /dev/null
+++ b/drivers/gpu/drm/xen/xen_drm_front_shbuf.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+
+/*
+ *  Xen para-virtual DRM device
+ *
+ * Copyright (C) 2016-2018 EPAM Systems Inc.
+ *
+ * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
+ */
+
+#ifndef __XEN_DRM_FRONT_SHBUF_H_
+#define __XEN_DRM_FRONT_SHBUF_H_
+
+#include <linux/kernel.h>
+#include <linux/scatterlist.h>
+
+#include <xen/grant_table.h>
+
+struct xen_drm_front_shbuf {
+	/*
+	 * number of references granted for the backend use:
+	 *  - for allocated/imported dma-buf's this holds number of grant
+	 *    references for the page directory and pages of the buffer
+	 *  - for the buffer provided by the backend this holds number of
+	 *    grant references for the page directory as grant references for
+	 *    the buffer will be provided by the backend
+	 */
+	int num_grefs;
+	grant_ref_t *grefs;
+	unsigned char *directory;
+
+	int num_pages;
+	struct page **pages;
+
+	struct xenbus_device *xb_dev;
+
+	/* these are the ops used internally depending on be_alloc mode */
+	const struct xen_drm_front_shbuf_ops *ops;
+
+	/* Xen map handles for the buffer allocated by the backend */
+	grant_handle_t *backend_map_handles;
+};
+
+struct xen_drm_front_shbuf_cfg {
+	struct xenbus_device *xb_dev;
+	size_t size;
+	struct page **pages;
+	bool be_alloc;
+};
+
+struct xen_drm_front_shbuf *
+xen_drm_front_shbuf_alloc(struct xen_drm_front_shbuf_cfg *cfg);
+
+grant_ref_t xen_drm_front_shbuf_get_dir_start(struct xen_drm_front_shbuf *buf);
+
+int xen_drm_front_shbuf_map(struct xen_drm_front_shbuf *buf);
+
+int xen_drm_front_shbuf_unmap(struct xen_drm_front_shbuf *buf);
+
+void xen_drm_front_shbuf_flush(struct xen_drm_front_shbuf *buf);
+
+void xen_drm_front_shbuf_free(struct xen_drm_front_shbuf *buf);
+
+#endif /* __XEN_DRM_FRONT_SHBUF_H_ */
diff --git a/drivers/gpu/drm/zte/zx_plane.c b/drivers/gpu/drm/zte/zx_plane.c
index 94545adac50d..d1931f5ea0b2 100644
--- a/drivers/gpu/drm/zte/zx_plane.c
+++ b/drivers/gpu/drm/zte/zx_plane.c
@@ -268,7 +268,7 @@ static void zx_plane_atomic_disable(struct drm_plane *plane,
 	struct zx_plane *zplane = to_zx_plane(plane);
 	void __iomem *hbsc = zplane->hbsc;
 
-	zx_vou_layer_disable(plane);
+	zx_vou_layer_disable(plane, old_state);
 
 	/* Disable HBSC block */
 	zx_writel_mask(hbsc + HBSC_CTRL0, HBSC_CTRL_EN, 0);
diff --git a/drivers/gpu/drm/zte/zx_vou.c b/drivers/gpu/drm/zte/zx_vou.c
index 7491813131f3..442311d31110 100644
--- a/drivers/gpu/drm/zte/zx_vou.c
+++ b/drivers/gpu/drm/zte/zx_vou.c
@@ -627,9 +627,10 @@ void zx_vou_layer_enable(struct drm_plane *plane)
 	zx_writel_mask(vou->osd + OSD_CTRL0, bits->enable, bits->enable);
 }
 
-void zx_vou_layer_disable(struct drm_plane *plane)
+void zx_vou_layer_disable(struct drm_plane *plane,
+			  struct drm_plane_state *old_state)
 {
-	struct zx_crtc *zcrtc = to_zx_crtc(plane->crtc);
+	struct zx_crtc *zcrtc = to_zx_crtc(old_state->crtc);
 	struct zx_vou_hw *vou = zcrtc->vou;
 	struct zx_plane *zplane = to_zx_plane(plane);
 	const struct vou_layer_bits *bits = zplane->bits;
diff --git a/drivers/gpu/drm/zte/zx_vou.h b/drivers/gpu/drm/zte/zx_vou.h
index 97d72bfce982..5b7f84fbb112 100644
--- a/drivers/gpu/drm/zte/zx_vou.h
+++ b/drivers/gpu/drm/zte/zx_vou.h
@@ -62,6 +62,7 @@ void zx_vou_config_dividers(struct drm_crtc *crtc,
 			    struct vou_div_config *configs, int num);
 
 void zx_vou_layer_enable(struct drm_plane *plane);
-void zx_vou_layer_disable(struct drm_plane *plane);
+void zx_vou_layer_disable(struct drm_plane *plane,
+			  struct drm_plane_state *old_state);
 
 #endif /* __ZX_VOU_H__ */
diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c
index 88a3558b7916..815bdb42e3f0 100644
--- a/drivers/gpu/host1x/bus.c
+++ b/drivers/gpu/host1x/bus.c
@@ -314,6 +314,11 @@ static int host1x_device_match(struct device *dev, struct device_driver *drv)
 	return strcmp(dev_name(dev), drv->name) == 0;
 }
 
+static int host1x_dma_configure(struct device *dev)
+{
+	return of_dma_configure(dev, dev->of_node, true);
+}
+
 static const struct dev_pm_ops host1x_device_pm_ops = {
 	.suspend = pm_generic_suspend,
 	.resume = pm_generic_resume,
@@ -326,8 +331,8 @@ static const struct dev_pm_ops host1x_device_pm_ops = {
 struct bus_type host1x_bus_type = {
 	.name = "host1x",
 	.match = host1x_device_match,
+	.dma_configure	= host1x_dma_configure,
 	.pm = &host1x_device_pm_ops,
-	.force_dma = true,
 };
 
 static void __host1x_device_del(struct host1x_device *device)
@@ -416,7 +421,7 @@ static int host1x_device_add(struct host1x *host1x,
 	device->dev.bus = &host1x_bus_type;
 	device->dev.parent = host1x->dev;
 
-	of_dma_configure(&device->dev, host1x->dev->of_node);
+	of_dma_configure(&device->dev, host1x->dev->of_node, true);
 
 	err = host1x_device_parse_dt(device, driver);
 	if (err < 0) {
diff --git a/drivers/gpu/host1x/cdma.c b/drivers/gpu/host1x/cdma.c
index 28541b280739..91df51e631b2 100644
--- a/drivers/gpu/host1x/cdma.c
+++ b/drivers/gpu/host1x/cdma.c
@@ -51,7 +51,7 @@ static void host1x_pushbuffer_destroy(struct push_buffer *pb)
 	struct host1x_cdma *cdma = pb_to_cdma(pb);
 	struct host1x *host1x = cdma_to_host1x(cdma);
 
-	if (!pb->phys)
+	if (!pb->mapped)
 		return;
 
 	if (host1x->domain) {
@@ -127,7 +127,7 @@ static int host1x_pushbuffer_init(struct push_buffer *pb)
 iommu_free_iova:
 	__free_iova(&host1x->iova, alloc);
 iommu_free_mem:
-	dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);
+	dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);
 
 	return err;
 }
@@ -247,7 +247,7 @@ static void cdma_start_timer_locked(struct host1x_cdma *cdma,
 static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
 {
 	cancel_delayed_work(&cdma->timeout.wq);
-	cdma->timeout.client = 0;
+	cdma->timeout.client = NULL;
 }
 
 /*
diff --git a/drivers/gpu/host1x/cdma.h b/drivers/gpu/host1x/cdma.h
index 286d49386be9..e97e17b82370 100644
--- a/drivers/gpu/host1x/cdma.h
+++ b/drivers/gpu/host1x/cdma.h
@@ -44,7 +44,7 @@ struct host1x_job;
 struct push_buffer {
 	void *mapped;			/* mapped pushbuffer memory */
 	dma_addr_t dma;			/* device address of pushbuffer */
-	phys_addr_t phys;		/* physical address of pushbuffer */
+	dma_addr_t phys;		/* physical address of pushbuffer */
 	u32 fence;			/* index we've written */
 	u32 pos;			/* index to write to */
 	u32 size;
@@ -58,7 +58,7 @@ struct buffer_timeout {
 	u32 syncpt_val;			/* syncpt value when completed */
 	ktime_t start_ktime;		/* starting time */
 	/* context timeout information */
-	int client;
+	struct host1x_client *client;
 };
 
 enum cdma_event {
diff --git a/drivers/gpu/host1x/debug.c b/drivers/gpu/host1x/debug.c
index dc77ec452ffc..329e4a3d8ae7 100644
--- a/drivers/gpu/host1x/debug.c
+++ b/drivers/gpu/host1x/debug.c
@@ -103,7 +103,7 @@ static void show_syncpts(struct host1x *m, struct output *o)
 
 static void show_all(struct host1x *m, struct output *o, bool show_fifo)
 {
-	int i;
+	unsigned int i;
 
 	host1x_hw_show_mlocks(m, o);
 	show_syncpts(m, o);
diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c
index 03db71173f5d..f1d5f76e9c33 100644
--- a/drivers/gpu/host1x/dev.c
+++ b/drivers/gpu/host1x/dev.c
@@ -223,10 +223,14 @@ static int host1x_probe(struct platform_device *pdev)
 		struct iommu_domain_geometry *geometry;
 		unsigned long order;
 
+		err = iova_cache_get();
+		if (err < 0)
+			goto put_group;
+
 		host->domain = iommu_domain_alloc(&platform_bus_type);
 		if (!host->domain) {
 			err = -ENOMEM;
-			goto put_group;
+			goto put_cache;
 		}
 
 		err = iommu_attach_group(host->domain, host->group);
@@ -234,6 +238,7 @@ static int host1x_probe(struct platform_device *pdev)
 			if (err == -ENODEV) {
 				iommu_domain_free(host->domain);
 				host->domain = NULL;
+				iova_cache_put();
 				iommu_group_put(host->group);
 				host->group = NULL;
 				goto skip_iommu;
@@ -308,6 +313,9 @@ static int host1x_probe(struct platform_device *pdev)
 fail_free_domain:
 	if (host->domain)
 		iommu_domain_free(host->domain);
+put_cache:
+	if (host->group)
+		iova_cache_put();
 put_group:
 	iommu_group_put(host->group);
 
@@ -328,6 +336,7 @@ static int host1x_remove(struct platform_device *pdev)
 		put_iova_domain(&host->iova);
 		iommu_detach_group(host->domain, host->group);
 		iommu_domain_free(host->domain);
+		iova_cache_put();
 		iommu_group_put(host->group);
 	}
 
diff --git a/drivers/gpu/host1x/dev.h b/drivers/gpu/host1x/dev.h
index 43e9fabb43a1..36f44ffebe73 100644
--- a/drivers/gpu/host1x/dev.h
+++ b/drivers/gpu/host1x/dev.h
@@ -78,7 +78,6 @@ struct host1x_syncpt_ops {
 	void (*load_wait_base)(struct host1x_syncpt *syncpt);
 	u32 (*load)(struct host1x_syncpt *syncpt);
 	int (*cpu_incr)(struct host1x_syncpt *syncpt);
-	int (*patch_wait)(struct host1x_syncpt *syncpt, void *patch_addr);
 	void (*assign_to_channel)(struct host1x_syncpt *syncpt,
 	                          struct host1x_channel *channel);
 	void (*enable_protection)(struct host1x *host);
@@ -183,13 +182,6 @@ static inline int host1x_hw_syncpt_cpu_incr(struct host1x *host,
 	return host->syncpt_op->cpu_incr(sp);
 }
 
-static inline int host1x_hw_syncpt_patch_wait(struct host1x *host,
-					      struct host1x_syncpt *sp,
-					      void *patch_addr)
-{
-	return host->syncpt_op->patch_wait(sp, patch_addr);
-}
-
 static inline void host1x_hw_syncpt_assign_to_channel(
 	struct host1x *host, struct host1x_syncpt *sp,
 	struct host1x_channel *ch)
diff --git a/drivers/gpu/host1x/hw/channel_hw.c b/drivers/gpu/host1x/hw/channel_hw.c
index 9af758785a11..d188f9068b91 100644
--- a/drivers/gpu/host1x/hw/channel_hw.c
+++ b/drivers/gpu/host1x/hw/channel_hw.c
@@ -104,8 +104,7 @@ static int channel_submit(struct host1x_job *job)
 	sp = host->syncpt + job->syncpt_id;
 	trace_host1x_channel_submit(dev_name(ch->dev),
 				    job->num_gathers, job->num_relocs,
-				    job->num_waitchk, job->syncpt_id,
-				    job->syncpt_incrs);
+				    job->syncpt_id, job->syncpt_incrs);
 
 	/* before error checks, return current max */
 	prev_max = job->syncpt_end = host1x_syncpt_read_max(sp);
@@ -165,7 +164,7 @@ static int channel_submit(struct host1x_job *job)
 	trace_host1x_channel_submitted(dev_name(ch->dev), prev_max, syncval);
 
 	/* schedule a submit complete interrupt */
-	err = host1x_intr_add_action(host, job->syncpt_id, syncval,
+	err = host1x_intr_add_action(host, sp, syncval,
 				     HOST1X_INTR_ACTION_SUBMIT_COMPLETE, ch,
 				     completed_waiter, NULL);
 	completed_waiter = NULL;
diff --git a/drivers/gpu/host1x/hw/syncpt_hw.c b/drivers/gpu/host1x/hw/syncpt_hw.c
index 7dfd47d74f89..a23bb3352d02 100644
--- a/drivers/gpu/host1x/hw/syncpt_hw.c
+++ b/drivers/gpu/host1x/hw/syncpt_hw.c
@@ -96,16 +96,6 @@ static int syncpt_cpu_incr(struct host1x_syncpt *sp)
 	return 0;
 }
 
-/* remove a wait pointed to by patch_addr */
-static int syncpt_patch_wait(struct host1x_syncpt *sp, void *patch_addr)
-{
-	u32 override = host1x_class_host_wait_syncpt(HOST1X_SYNCPT_RESERVED, 0);
-
-	*((u32 *)patch_addr) = override;
-
-	return 0;
-}
-
 /**
  * syncpt_assign_to_channel() - Assign syncpoint to channel
  * @sp: syncpoint
@@ -156,7 +146,6 @@ static const struct host1x_syncpt_ops host1x_syncpt_ops = {
 	.load_wait_base = syncpt_read_wait_base,
 	.load = syncpt_load,
 	.cpu_incr = syncpt_cpu_incr,
-	.patch_wait = syncpt_patch_wait,
 	.assign_to_channel = syncpt_assign_to_channel,
 	.enable_protection = syncpt_enable_protection,
 };
diff --git a/drivers/gpu/host1x/intr.c b/drivers/gpu/host1x/intr.c
index 8b4fad0ab35d..9629c009d10f 100644
--- a/drivers/gpu/host1x/intr.c
+++ b/drivers/gpu/host1x/intr.c
@@ -144,7 +144,7 @@ static const action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = {
 static void run_handlers(struct list_head completed[HOST1X_INTR_ACTION_COUNT])
 {
 	struct list_head *head = completed;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i, ++head) {
 		action_handler handler = action_handlers[i];
@@ -211,11 +211,11 @@ static void syncpt_thresh_work(struct work_struct *work)
 				host1x_syncpt_load(host->syncpt + id));
 }
 
-int host1x_intr_add_action(struct host1x *host, unsigned int id, u32 thresh,
-			   enum host1x_intr_action action, void *data,
-			   struct host1x_waitlist *waiter, void **ref)
+int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt,
+			   u32 thresh, enum host1x_intr_action action,
+			   void *data, struct host1x_waitlist *waiter,
+			   void **ref)
 {
-	struct host1x_syncpt *syncpt;
 	int queue_was_empty;
 
 	if (waiter == NULL) {
@@ -234,19 +234,17 @@ int host1x_intr_add_action(struct host1x *host, unsigned int id, u32 thresh,
 	waiter->data = data;
 	waiter->count = 1;
 
-	syncpt = host->syncpt + id;
-
 	spin_lock(&syncpt->intr.lock);
 
 	queue_was_empty = list_empty(&syncpt->intr.wait_head);
 
 	if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) {
 		/* added at head of list - new threshold value */
-		host1x_hw_intr_set_syncpt_threshold(host, id, thresh);
+		host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh);
 
 		/* added as first waiter - enable interrupt */
 		if (queue_was_empty)
-			host1x_hw_intr_enable_syncpt_intr(host, id);
+			host1x_hw_intr_enable_syncpt_intr(host, syncpt->id);
 	}
 
 	spin_unlock(&syncpt->intr.lock);
diff --git a/drivers/gpu/host1x/intr.h b/drivers/gpu/host1x/intr.h
index 1370c2bb75b8..6db96af484fe 100644
--- a/drivers/gpu/host1x/intr.h
+++ b/drivers/gpu/host1x/intr.h
@@ -22,6 +22,7 @@
 #include <linux/interrupt.h>
 #include <linux/workqueue.h>
 
+struct host1x_syncpt;
 struct host1x;
 
 enum host1x_intr_action {
@@ -75,9 +76,10 @@ struct host1x_waitlist {
  *
  * This is a non-blocking api.
  */
-int host1x_intr_add_action(struct host1x *host, unsigned int id, u32 thresh,
-	enum host1x_intr_action action, void *data,
-	struct host1x_waitlist *waiter, void **ref);
+int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt,
+			   u32 thresh, enum host1x_intr_action action,
+			   void *data, struct host1x_waitlist *waiter,
+			   void **ref);
 
 /*
  * Unreference an action submitted to host1x_intr_add_action().
diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c
index db509ab8874e..e2f4a4d93d20 100644
--- a/drivers/gpu/host1x/job.c
+++ b/drivers/gpu/host1x/job.c
@@ -34,8 +34,7 @@
 #define HOST1X_WAIT_SYNCPT_OFFSET 0x8
 
 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
-				    u32 num_cmdbufs, u32 num_relocs,
-				    u32 num_waitchks)
+				    u32 num_cmdbufs, u32 num_relocs)
 {
 	struct host1x_job *job = NULL;
 	unsigned int num_unpins = num_cmdbufs + num_relocs;
@@ -46,7 +45,6 @@ struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
 	total = sizeof(struct host1x_job) +
 		(u64)num_relocs * sizeof(struct host1x_reloc) +
 		(u64)num_unpins * sizeof(struct host1x_job_unpin_data) +
-		(u64)num_waitchks * sizeof(struct host1x_waitchk) +
 		(u64)num_cmdbufs * sizeof(struct host1x_job_gather) +
 		(u64)num_unpins * sizeof(dma_addr_t) +
 		(u64)num_unpins * sizeof(u32 *);
@@ -62,12 +60,10 @@ struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
 
 	/* Redistribute memory to the structs  */
 	mem += sizeof(struct host1x_job);
-	job->relocarray = num_relocs ? mem : NULL;
+	job->relocs = num_relocs ? mem : NULL;
 	mem += num_relocs * sizeof(struct host1x_reloc);
 	job->unpins = num_unpins ? mem : NULL;
 	mem += num_unpins * sizeof(struct host1x_job_unpin_data);
-	job->waitchk = num_waitchks ? mem : NULL;
-	mem += num_waitchks * sizeof(struct host1x_waitchk);
 	job->gathers = num_cmdbufs ? mem : NULL;
 	mem += num_cmdbufs * sizeof(struct host1x_job_gather);
 	job->addr_phys = num_unpins ? mem : NULL;
@@ -100,84 +96,18 @@ void host1x_job_put(struct host1x_job *job)
 EXPORT_SYMBOL(host1x_job_put);
 
 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo,
-			   u32 words, u32 offset)
+			   unsigned int words, unsigned int offset)
 {
-	struct host1x_job_gather *cur_gather = &job->gathers[job->num_gathers];
+	struct host1x_job_gather *gather = &job->gathers[job->num_gathers];
+
+	gather->words = words;
+	gather->bo = bo;
+	gather->offset = offset;
 
-	cur_gather->words = words;
-	cur_gather->bo = bo;
-	cur_gather->offset = offset;
 	job->num_gathers++;
 }
 EXPORT_SYMBOL(host1x_job_add_gather);
 
-/*
- * NULL an already satisfied WAIT_SYNCPT host method, by patching its
- * args in the command stream. The method data is changed to reference
- * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt
- * with a matching threshold value of 0, so is guaranteed to be popped
- * by the host HW.
- */
-static void host1x_syncpt_patch_offset(struct host1x_syncpt *sp,
-				       struct host1x_bo *h, u32 offset)
-{
-	void *patch_addr = NULL;
-
-	/* patch the wait */
-	patch_addr = host1x_bo_kmap(h, offset >> PAGE_SHIFT);
-	if (patch_addr) {
-		host1x_syncpt_patch_wait(sp,
-					 patch_addr + (offset & ~PAGE_MASK));
-		host1x_bo_kunmap(h, offset >> PAGE_SHIFT, patch_addr);
-	} else
-		pr_err("Could not map cmdbuf for wait check\n");
-}
-
-/*
- * Check driver supplied waitchk structs for syncpt thresholds
- * that have already been satisfied and NULL the comparison (to
- * avoid a wrap condition in the HW).
- */
-static int do_waitchks(struct host1x_job *job, struct host1x *host,
-		       struct host1x_job_gather *g)
-{
-	struct host1x_bo *patch = g->bo;
-	int i;
-
-	/* compare syncpt vs wait threshold */
-	for (i = 0; i < job->num_waitchk; i++) {
-		struct host1x_waitchk *wait = &job->waitchk[i];
-		struct host1x_syncpt *sp =
-			host1x_syncpt_get(host, wait->syncpt_id);
-
-		/* validate syncpt id */
-		if (wait->syncpt_id > host1x_syncpt_nb_pts(host))
-			continue;
-
-		/* skip all other gathers */
-		if (patch != wait->bo)
-			continue;
-
-		trace_host1x_syncpt_wait_check(wait->bo, wait->offset,
-					       wait->syncpt_id, wait->thresh,
-					       host1x_syncpt_read_min(sp));
-
-		if (host1x_syncpt_is_expired(sp, wait->thresh)) {
-			dev_dbg(host->dev,
-				"drop WAIT id %u (%s) thresh 0x%x, min 0x%x\n",
-				wait->syncpt_id, sp->name, wait->thresh,
-				host1x_syncpt_read_min(sp));
-
-			host1x_syncpt_patch_offset(sp, patch,
-						   g->offset + wait->offset);
-		}
-
-		wait->bo = NULL;
-	}
-
-	return 0;
-}
-
 static unsigned int pin_job(struct host1x *host, struct host1x_job *job)
 {
 	unsigned int i;
@@ -186,7 +116,7 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job)
 	job->num_unpins = 0;
 
 	for (i = 0; i < job->num_relocs; i++) {
-		struct host1x_reloc *reloc = &job->relocarray[i];
+		struct host1x_reloc *reloc = &job->relocs[i];
 		struct sg_table *sgt;
 		dma_addr_t phys_addr;
 
@@ -267,14 +197,14 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job)
 
 static int do_relocs(struct host1x_job *job, struct host1x_job_gather *g)
 {
-	int i = 0;
 	u32 last_page = ~0;
 	void *cmdbuf_page_addr = NULL;
 	struct host1x_bo *cmdbuf = g->bo;
+	unsigned int i;
 
 	/* pin & patch the relocs for one gather */
 	for (i = 0; i < job->num_relocs; i++) {
-		struct host1x_reloc *reloc = &job->relocarray[i];
+		struct host1x_reloc *reloc = &job->relocs[i];
 		u32 reloc_addr = (job->reloc_addr_phys[i] +
 				  reloc->target.offset) >> reloc->shift;
 		u32 *target;
@@ -331,17 +261,6 @@ static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
 	return true;
 }
 
-static bool check_wait(struct host1x_waitchk *wait, struct host1x_bo *cmdbuf,
-		       unsigned int offset)
-{
-	offset *= sizeof(u32);
-
-	if (wait->bo != cmdbuf || wait->offset != offset)
-		return false;
-
-	return true;
-}
-
 struct host1x_firewall {
 	struct host1x_job *job;
 	struct device *dev;
@@ -349,9 +268,6 @@ struct host1x_firewall {
 	unsigned int num_relocs;
 	struct host1x_reloc *reloc;
 
-	unsigned int num_waitchks;
-	struct host1x_waitchk *waitchk;
-
 	struct host1x_bo *cmdbuf;
 	unsigned int offset;
 
@@ -378,20 +294,6 @@ static int check_register(struct host1x_firewall *fw, unsigned long offset)
 		fw->reloc++;
 	}
 
-	if (offset == HOST1X_WAIT_SYNCPT_OFFSET) {
-		if (fw->class != HOST1X_CLASS_HOST1X)
-			return -EINVAL;
-
-		if (!fw->num_waitchks)
-			return -EINVAL;
-
-		if (!check_wait(fw->waitchk, fw->cmdbuf, fw->offset))
-			return -EINVAL;
-
-		fw->num_waitchks--;
-		fw->waitchk++;
-	}
-
 	return 0;
 }
 
@@ -550,14 +452,12 @@ static inline int copy_gathers(struct host1x_job *job, struct device *dev)
 	struct host1x_firewall fw;
 	size_t size = 0;
 	size_t offset = 0;
-	int i;
+	unsigned int i;
 
 	fw.job = job;
 	fw.dev = dev;
-	fw.reloc = job->relocarray;
+	fw.reloc = job->relocs;
 	fw.num_relocs = job->num_relocs;
-	fw.waitchk = job->waitchk;
-	fw.num_waitchks = job->num_waitchk;
 	fw.class = job->class;
 
 	for (i = 0; i < job->num_gathers; i++) {
@@ -604,8 +504,8 @@ static inline int copy_gathers(struct host1x_job *job, struct device *dev)
 		offset += g->words * sizeof(u32);
 	}
 
-	/* No relocs and waitchks should remain at this point */
-	if (fw.num_relocs || fw.num_waitchks)
+	/* No relocs should remain at this point */
+	if (fw.num_relocs)
 		return -EINVAL;
 
 	return 0;
@@ -616,19 +516,6 @@ int host1x_job_pin(struct host1x_job *job, struct device *dev)
 	int err;
 	unsigned int i, j;
 	struct host1x *host = dev_get_drvdata(dev->parent);
-	DECLARE_BITMAP(waitchk_mask, host1x_syncpt_nb_pts(host));
-
-	bitmap_zero(waitchk_mask, host1x_syncpt_nb_pts(host));
-	for (i = 0; i < job->num_waitchk; i++) {
-		u32 syncpt_id = job->waitchk[i].syncpt_id;
-
-		if (syncpt_id < host1x_syncpt_nb_pts(host))
-			set_bit(syncpt_id, waitchk_mask);
-	}
-
-	/* get current syncpt values for waitchk */
-	for_each_set_bit(i, waitchk_mask, host1x_syncpt_nb_pts(host))
-		host1x_syncpt_load(host->syncpt + i);
 
 	/* pin memory */
 	err = pin_job(host, job);
@@ -663,10 +550,6 @@ int host1x_job_pin(struct host1x_job *job, struct device *dev)
 		err = do_relocs(job, g);
 		if (err)
 			break;
-
-		err = do_waitchks(job, host, g);
-		if (err)
-			break;
 	}
 
 out:
diff --git a/drivers/gpu/host1x/job.h b/drivers/gpu/host1x/job.h
index 4bda51d503ec..188400e00192 100644
--- a/drivers/gpu/host1x/job.h
+++ b/drivers/gpu/host1x/job.h
@@ -20,10 +20,10 @@
 #define __HOST1X_JOB_H
 
 struct host1x_job_gather {
-	u32 words;
+	unsigned int words;
 	dma_addr_t base;
 	struct host1x_bo *bo;
-	u32 offset;
+	unsigned int offset;
 	bool handled;
 };
 
diff --git a/drivers/gpu/host1x/syncpt.c b/drivers/gpu/host1x/syncpt.c
index a2a952adc136..a5dbf1ba4645 100644
--- a/drivers/gpu/host1x/syncpt.c
+++ b/drivers/gpu/host1x/syncpt.c
@@ -57,8 +57,8 @@ static struct host1x_syncpt *host1x_syncpt_alloc(struct host1x *host,
 						 struct host1x_client *client,
 						 unsigned long flags)
 {
-	int i;
 	struct host1x_syncpt *sp = host->syncpt;
+	unsigned int i;
 	char *name;
 
 	mutex_lock(&host->syncpt_mutex);
@@ -255,7 +255,7 @@ int host1x_syncpt_wait(struct host1x_syncpt *sp, u32 thresh, long timeout,
 	}
 
 	/* schedule a wakeup when the syncpoint value is reached */
-	err = host1x_intr_add_action(sp->host, sp->id, thresh,
+	err = host1x_intr_add_action(sp->host, sp, thresh,
 				     HOST1X_INTR_ACTION_WAKEUP_INTERRUPTIBLE,
 				     &wq, waiter, &ref);
 	if (err)
@@ -373,12 +373,6 @@ bool host1x_syncpt_is_expired(struct host1x_syncpt *sp, u32 thresh)
 		return (s32)(current_val - thresh) >= 0;
 }
 
-/* remove a wait pointed to by patch_addr */
-int host1x_syncpt_patch_wait(struct host1x_syncpt *sp, void *patch_addr)
-{
-	return host1x_hw_syncpt_patch_wait(sp->host, sp, patch_addr);
-}
-
 int host1x_syncpt_init(struct host1x *host)
 {
 	struct host1x_syncpt_base *bases;
diff --git a/drivers/gpu/host1x/syncpt.h b/drivers/gpu/host1x/syncpt.h
index 9d88d37c2397..d98e22325e9d 100644
--- a/drivers/gpu/host1x/syncpt.h
+++ b/drivers/gpu/host1x/syncpt.h
@@ -124,7 +124,4 @@ static inline int host1x_syncpt_is_valid(struct host1x_syncpt *sp)
 	return sp->id < host1x_syncpt_nb_pts(sp->host);
 }
 
-/* Patch a wait by replacing it with a wait for syncpt 0 value 0 */
-int host1x_syncpt_patch_wait(struct host1x_syncpt *sp, void *patch_addr);
-
 #endif
diff --git a/drivers/gpu/ipu-v3/ipu-csi.c b/drivers/gpu/ipu-v3/ipu-csi.c
index caa05b0702e1..f3a5df61fb86 100644
--- a/drivers/gpu/ipu-v3/ipu-csi.c
+++ b/drivers/gpu/ipu-v3/ipu-csi.c
@@ -224,14 +224,18 @@ static int ipu_csi_set_testgen_mclk(struct ipu_csi *csi, u32 pixel_clk,
  * Find the CSI data format and data width for the given V4L2 media
  * bus pixel format code.
  */
-static int mbus_code_to_bus_cfg(struct ipu_csi_bus_config *cfg, u32 mbus_code)
+static int mbus_code_to_bus_cfg(struct ipu_csi_bus_config *cfg, u32 mbus_code,
+				enum v4l2_mbus_type mbus_type)
 {
 	switch (mbus_code) {
 	case MEDIA_BUS_FMT_BGR565_2X8_BE:
 	case MEDIA_BUS_FMT_BGR565_2X8_LE:
 	case MEDIA_BUS_FMT_RGB565_2X8_BE:
 	case MEDIA_BUS_FMT_RGB565_2X8_LE:
-		cfg->data_fmt = CSI_SENS_CONF_DATA_FMT_RGB565;
+		if (mbus_type == V4L2_MBUS_CSI2)
+			cfg->data_fmt = CSI_SENS_CONF_DATA_FMT_RGB565;
+		else
+			cfg->data_fmt = CSI_SENS_CONF_DATA_FMT_BAYER;
 		cfg->mipi_dt = MIPI_DT_RGB565;
 		cfg->data_width = IPU_CSI_DATA_WIDTH_8;
 		break;
@@ -247,6 +251,12 @@ static int mbus_code_to_bus_cfg(struct ipu_csi_bus_config *cfg, u32 mbus_code)
 		cfg->mipi_dt = MIPI_DT_RGB555;
 		cfg->data_width = IPU_CSI_DATA_WIDTH_8;
 		break;
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_BGR888_1X24:
+		cfg->data_fmt = CSI_SENS_CONF_DATA_FMT_RGB_YUV444;
+		cfg->mipi_dt = MIPI_DT_RGB888;
+		cfg->data_width = IPU_CSI_DATA_WIDTH_8;
+		break;
 	case MEDIA_BUS_FMT_UYVY8_2X8:
 		cfg->data_fmt = CSI_SENS_CONF_DATA_FMT_YUV422_UYVY;
 		cfg->mipi_dt = MIPI_DT_YUV422;
@@ -318,13 +328,17 @@ static int mbus_code_to_bus_cfg(struct ipu_csi_bus_config *cfg, u32 mbus_code)
 /*
  * Fill a CSI bus config struct from mbus_config and mbus_framefmt.
  */
-static void fill_csi_bus_cfg(struct ipu_csi_bus_config *csicfg,
+static int fill_csi_bus_cfg(struct ipu_csi_bus_config *csicfg,
 				 struct v4l2_mbus_config *mbus_cfg,
 				 struct v4l2_mbus_framefmt *mbus_fmt)
 {
+	int ret;
+
 	memset(csicfg, 0, sizeof(*csicfg));
 
-	mbus_code_to_bus_cfg(csicfg, mbus_fmt->code);
+	ret = mbus_code_to_bus_cfg(csicfg, mbus_fmt->code, mbus_cfg->type);
+	if (ret < 0)
+		return ret;
 
 	switch (mbus_cfg->type) {
 	case V4L2_MBUS_PARALLEL:
@@ -355,6 +369,8 @@ static void fill_csi_bus_cfg(struct ipu_csi_bus_config *csicfg,
 		/* will never get here, keep compiler quiet */
 		break;
 	}
+
+	return 0;
 }
 
 int ipu_csi_init_interface(struct ipu_csi *csi,
@@ -364,8 +380,11 @@ int ipu_csi_init_interface(struct ipu_csi *csi,
 	struct ipu_csi_bus_config cfg;
 	unsigned long flags;
 	u32 width, height, data = 0;
+	int ret;
 
-	fill_csi_bus_cfg(&cfg, mbus_cfg, mbus_fmt);
+	ret = fill_csi_bus_cfg(&cfg, mbus_cfg, mbus_fmt);
+	if (ret < 0)
+		return ret;
 
 	/* set default sensor frame width and height */
 	width = mbus_fmt->width;
@@ -586,11 +605,14 @@ int ipu_csi_set_mipi_datatype(struct ipu_csi *csi, u32 vc,
 	struct ipu_csi_bus_config cfg;
 	unsigned long flags;
 	u32 temp;
+	int ret;
 
 	if (vc > 3)
 		return -EINVAL;
 
-	mbus_code_to_bus_cfg(&cfg, mbus_fmt->code);
+	ret = mbus_code_to_bus_cfg(&cfg, mbus_fmt->code, V4L2_MBUS_CSI2);
+	if (ret < 0)
+		return ret;
 
 	spin_lock_irqsave(&csi->lock, flags);
 
diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
index 0000434a1fbd..a49a10437c40 100644
--- a/drivers/hid/Kconfig
+++ b/drivers/hid/Kconfig
@@ -575,6 +575,13 @@ config HID_MAYFLASH
 	Say Y here if you have HJZ Mayflash PS3 game controller adapters
 	and want to enable force feedback support.
 
+config HID_REDRAGON
+	tristate "Redragon keyboards"
+	depends on HID
+	default !EXPERT
+	---help---
+    Support for Redragon keyboards that need fix-ups to work properly.
+
 config HID_MICROSOFT
 	tristate "Microsoft non-fully HID-compliant devices"
 	depends on HID
@@ -838,6 +845,15 @@ config HID_SPEEDLINK
 	---help---
 	Support for Speedlink Vicious and Divine Cezanne mouse.
 
+config HID_STEAM
+	tristate "Steam Controller support"
+	depends on HID
+	select POWER_SUPPLY
+	---help---
+	Say Y here if you have a Steam Controller if you want to use it
+	without running the Steam Client. It supports both the wired and
+	the wireless adaptor.
+
 config HID_STEELSERIES
 	tristate "Steelseries SRW-S1 steering wheel support"
 	depends on HID
diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile
index 17a8bd97da9d..511e1cbff768 100644
--- a/drivers/hid/Makefile
+++ b/drivers/hid/Makefile
@@ -86,6 +86,7 @@ hid-picolcd-$(CONFIG_DEBUG_FS)		+= hid-picolcd_debugfs.o
 
 obj-$(CONFIG_HID_PLANTRONICS)	+= hid-plantronics.o
 obj-$(CONFIG_HID_PRIMAX)	+= hid-primax.o
+obj-$(CONFIG_HID_REDRAGON)	+= hid-redragon.o
 obj-$(CONFIG_HID_RETRODE)	+= hid-retrode.o
 obj-$(CONFIG_HID_ROCCAT)	+= hid-roccat.o hid-roccat-common.o \
 	hid-roccat-arvo.o hid-roccat-isku.o hid-roccat-kone.o \
@@ -97,6 +98,7 @@ obj-$(CONFIG_HID_SAMSUNG)	+= hid-samsung.o
 obj-$(CONFIG_HID_SMARTJOYPLUS)	+= hid-sjoy.o
 obj-$(CONFIG_HID_SONY)		+= hid-sony.o
 obj-$(CONFIG_HID_SPEEDLINK)	+= hid-speedlink.o
+obj-$(CONFIG_HID_STEAM)		+= hid-steam.o
 obj-$(CONFIG_HID_STEELSERIES)	+= hid-steelseries.o
 obj-$(CONFIG_HID_SUNPLUS)	+= hid-sunplus.o
 obj-$(CONFIG_HID_GREENASIA)	+= hid-gaff.o
diff --git a/drivers/hid/hid-alps.c b/drivers/hid/hid-alps.c
index b1eeb4839bfc..aec253b44156 100644
--- a/drivers/hid/hid-alps.c
+++ b/drivers/hid/hid-alps.c
@@ -171,7 +171,7 @@ static int t4_read_write_register(struct hid_device *hdev, u32 address,
 	int ret;
 	u16 check_sum;
 	u8 *input;
-	u8 *readbuf;
+	u8 *readbuf = NULL;
 
 	input = kzalloc(T4_FEATURE_REPORT_LEN, GFP_KERNEL);
 	if (!input)
@@ -204,8 +204,8 @@ static int t4_read_write_register(struct hid_device *hdev, u32 address,
 		goto exit;
 	}
 
-	readbuf = kzalloc(T4_FEATURE_REPORT_LEN, GFP_KERNEL);
 	if (read_flag) {
+		readbuf = kzalloc(T4_FEATURE_REPORT_LEN, GFP_KERNEL);
 		if (!readbuf) {
 			ret = -ENOMEM;
 			goto exit;
@@ -219,22 +219,24 @@ static int t4_read_write_register(struct hid_device *hdev, u32 address,
 			goto exit_readbuf;
 		}
 
+		ret = -EINVAL;
+
 		if (*(u32 *)&readbuf[6] != address) {
 			dev_err(&hdev->dev, "read register address error (%x,%x)\n",
-			*(u32 *)&readbuf[6], address);
+				*(u32 *)&readbuf[6], address);
 			goto exit_readbuf;
 		}
 
 		if (*(u16 *)&readbuf[10] != 1) {
 			dev_err(&hdev->dev, "read register size error (%x)\n",
-			*(u16 *)&readbuf[10]);
+				*(u16 *)&readbuf[10]);
 			goto exit_readbuf;
 		}
 
 		check_sum = t4_calc_check_sum(readbuf, 6, 7);
 		if (*(u16 *)&readbuf[13] != check_sum) {
 			dev_err(&hdev->dev, "read register checksum error (%x,%x)\n",
-			*(u16 *)&readbuf[13], check_sum);
+				*(u16 *)&readbuf[13], check_sum);
 			goto exit_readbuf;
 		}
 
@@ -458,17 +460,35 @@ static int __maybe_unused alps_post_reset(struct hid_device *hdev)
 	case T4:
 		ret = t4_read_write_register(hdev, T4_PRM_FEED_CONFIG_1,
 			NULL, T4_I2C_ABS, false);
+		if (ret < 0) {
+			dev_err(&hdev->dev, "failed T4_PRM_FEED_CONFIG_1 (%d)\n",
+				ret);
+			goto exit;
+		}
+
 		ret = t4_read_write_register(hdev, T4_PRM_FEED_CONFIG_4,
 			NULL, T4_FEEDCFG4_ADVANCED_ABS_ENABLE, false);
+		if (ret < 0) {
+			dev_err(&hdev->dev, "failed T4_PRM_FEED_CONFIG_4 (%d)\n",
+				ret);
+			goto exit;
+		}
 		break;
 	case U1:
 		ret = u1_read_write_register(hdev,
 			ADDRESS_U1_DEV_CTRL_1, NULL,
 			U1_TP_ABS_MODE | U1_SP_ABS_MODE, false);
+		if (ret < 0) {
+			dev_err(&hdev->dev, "failed to change TP mode (%d)\n",
+				ret);
+			goto exit;
+		}
 		break;
 	default:
 		break;
 	}
+
+exit:
 	return ret;
 }
 
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index 5d7cc6bbbac6..355dc7e49562 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -57,7 +57,9 @@ MODULE_PARM_DESC(ignore_special_drivers, "Ignore any special drivers and handle
  * Register a new report for a device.
  */
 
-struct hid_report *hid_register_report(struct hid_device *device, unsigned type, unsigned id)
+struct hid_report *hid_register_report(struct hid_device *device,
+				       unsigned int type, unsigned int id,
+				       unsigned int application)
 {
 	struct hid_report_enum *report_enum = device->report_enum + type;
 	struct hid_report *report;
@@ -78,6 +80,7 @@ struct hid_report *hid_register_report(struct hid_device *device, unsigned type,
 	report->type = type;
 	report->size = 0;
 	report->device = device;
+	report->application = application;
 	report_enum->report_id_hash[id] = report;
 
 	list_add_tail(&report->list, &report_enum->report_list);
@@ -221,11 +224,15 @@ static int hid_add_field(struct hid_parser *parser, unsigned report_type, unsign
 {
 	struct hid_report *report;
 	struct hid_field *field;
-	unsigned usages;
-	unsigned offset;
-	unsigned i;
+	unsigned int usages;
+	unsigned int offset;
+	unsigned int i;
+	unsigned int application;
+
+	application = hid_lookup_collection(parser, HID_COLLECTION_APPLICATION);
 
-	report = hid_register_report(parser->device, report_type, parser->global.report_id);
+	report = hid_register_report(parser->device, report_type,
+				     parser->global.report_id, application);
 	if (!report) {
 		hid_err(parser->device, "hid_register_report failed\n");
 		return -1;
@@ -259,7 +266,7 @@ static int hid_add_field(struct hid_parser *parser, unsigned report_type, unsign
 
 	field->physical = hid_lookup_collection(parser, HID_COLLECTION_PHYSICAL);
 	field->logical = hid_lookup_collection(parser, HID_COLLECTION_LOGICAL);
-	field->application = hid_lookup_collection(parser, HID_COLLECTION_APPLICATION);
+	field->application = application;
 
 	for (i = 0; i < usages; i++) {
 		unsigned j = i;
@@ -1798,7 +1805,7 @@ EXPORT_SYMBOL_GPL(hid_hw_stop);
  *
  * Tell underlying HW to start delivering events from the device.
  * This function should be called sometime after successful call
- * to hid_hiw_start().
+ * to hid_hw_start().
  */
 int hid_hw_open(struct hid_device *hdev)
 {
diff --git a/drivers/hid/hid-generic.c b/drivers/hid/hid-generic.c
index c25b4718de44..3b6eccbc2519 100644
--- a/drivers/hid/hid-generic.c
+++ b/drivers/hid/hid-generic.c
@@ -56,6 +56,20 @@ static bool hid_generic_match(struct hid_device *hdev,
 	return true;
 }
 
+static int hid_generic_probe(struct hid_device *hdev,
+			     const struct hid_device_id *id)
+{
+	int ret;
+
+	hdev->quirks |= HID_QUIRK_INPUT_PER_APP;
+
+	ret = hid_parse(hdev);
+	if (ret)
+		return ret;
+
+	return hid_hw_start(hdev, HID_CONNECT_DEFAULT);
+}
+
 static const struct hid_device_id hid_table[] = {
 	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_ANY, HID_ANY_ID, HID_ANY_ID) },
 	{ }
@@ -66,6 +80,7 @@ static struct hid_driver hid_generic = {
 	.name = "hid-generic",
 	.id_table = hid_table,
 	.match = hid_generic_match,
+	.probe = hid_generic_probe,
 };
 module_hid_driver(hid_generic);
 
diff --git a/drivers/hid/hid-gfrm.c b/drivers/hid/hid-gfrm.c
index 075b1c020846..cf477f8c8f4c 100644
--- a/drivers/hid/hid-gfrm.c
+++ b/drivers/hid/hid-gfrm.c
@@ -116,7 +116,7 @@ static int gfrm_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		 * those reports reach gfrm_raw_event() from hid_input_report().
 		 */
 		if (!hid_register_report(hdev, HID_INPUT_REPORT,
-					 GFRM100_SEARCH_KEY_REPORT_ID)) {
+					 GFRM100_SEARCH_KEY_REPORT_ID, 0)) {
 			ret = -ENOMEM;
 			goto done;
 		}
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index 46f5ecd11bf7..a85634fe033f 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -615,6 +615,7 @@
 #define USB_VENDOR_ID_JESS		0x0c45
 #define USB_DEVICE_ID_JESS_YUREX	0x1010
 #define USB_DEVICE_ID_ASUS_MD_5112	0x5112
+#define USB_DEVICE_ID_REDRAGON_ASURA	0x760b
 
 #define USB_VENDOR_ID_JESS2		0x0f30
 #define USB_DEVICE_ID_JESS2_COLOR_RUMBLE_PAD 0x0111
@@ -1012,6 +1013,10 @@
 #define USB_VENDOR_ID_STANTUM_SITRONIX		0x1403
 #define USB_DEVICE_ID_MTP_SITRONIX		0x5001
 
+#define USB_VENDOR_ID_VALVE			0x28de
+#define USB_DEVICE_ID_STEAM_CONTROLLER		0x1102
+#define USB_DEVICE_ID_STEAM_CONTROLLER_WIRELESS	0x1142
+
 #define USB_VENDOR_ID_STEELSERIES	0x1038
 #define USB_DEVICE_ID_STEELSERIES_SRWS1	0x1410
 
diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c
index 930652c25120..ab93dd5927c3 100644
--- a/drivers/hid/hid-input.c
+++ b/drivers/hid/hid-input.c
@@ -1110,8 +1110,31 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel
 
 	set_bit(usage->type, input->evbit);
 
-	while (usage->code <= max && test_and_set_bit(usage->code, bit))
-		usage->code = find_next_zero_bit(bit, max + 1, usage->code);
+	/*
+	 * This part is *really* controversial:
+	 * - HID aims at being generic so we should do our best to export
+	 *   all incoming events
+	 * - HID describes what events are, so there is no reason for ABS_X
+	 *   to be mapped to ABS_Y
+	 * - HID is using *_MISC+N as a default value, but nothing prevents
+	 *   *_MISC+N to overwrite a legitimate even, which confuses userspace
+	 *   (for instance ABS_MISC + 7 is ABS_MT_SLOT, which has a different
+	 *   processing)
+	 *
+	 * If devices still want to use this (at their own risk), they will
+	 * have to use the quirk HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE, but
+	 * the default should be a reliable mapping.
+	 */
+	while (usage->code <= max && test_and_set_bit(usage->code, bit)) {
+		if (device->quirks & HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE) {
+			usage->code = find_next_zero_bit(bit,
+							 max + 1,
+							 usage->code);
+		} else {
+			device->status |= HID_STAT_DUP_DETECTED;
+			goto ignore;
+		}
+	}
 
 	if (usage->code > max)
 		goto ignore;
@@ -1487,15 +1510,56 @@ static void report_features(struct hid_device *hid)
 		}
 }
 
-static struct hid_input *hidinput_allocate(struct hid_device *hid)
+static struct hid_input *hidinput_allocate(struct hid_device *hid,
+					   unsigned int application)
 {
 	struct hid_input *hidinput = kzalloc(sizeof(*hidinput), GFP_KERNEL);
 	struct input_dev *input_dev = input_allocate_device();
-	if (!hidinput || !input_dev) {
-		kfree(hidinput);
-		input_free_device(input_dev);
-		hid_err(hid, "Out of memory during hid input probe\n");
-		return NULL;
+	const char *suffix = NULL;
+
+	if (!hidinput || !input_dev)
+		goto fail;
+
+	if ((hid->quirks & HID_QUIRK_INPUT_PER_APP) &&
+	    hid->maxapplication > 1) {
+		switch (application) {
+		case HID_GD_KEYBOARD:
+			suffix = "Keyboard";
+			break;
+		case HID_GD_KEYPAD:
+			suffix = "Keypad";
+			break;
+		case HID_GD_MOUSE:
+			suffix = "Mouse";
+			break;
+		case HID_DG_STYLUS:
+			suffix = "Pen";
+			break;
+		case HID_DG_TOUCHSCREEN:
+			suffix = "Touchscreen";
+			break;
+		case HID_DG_TOUCHPAD:
+			suffix = "Touchpad";
+			break;
+		case HID_GD_SYSTEM_CONTROL:
+			suffix = "System Control";
+			break;
+		case HID_CP_CONSUMER_CONTROL:
+			suffix = "Consumer Control";
+			break;
+		case HID_GD_WIRELESS_RADIO_CTLS:
+			suffix = "Wireless Radio Control";
+			break;
+		default:
+			break;
+		}
+	}
+
+	if (suffix) {
+		hidinput->name = kasprintf(GFP_KERNEL, "%s %s",
+					   hid->name, suffix);
+		if (!hidinput->name)
+			goto fail;
 	}
 
 	input_set_drvdata(input_dev, hid);
@@ -1505,7 +1569,7 @@ static struct hid_input *hidinput_allocate(struct hid_device *hid)
 	input_dev->setkeycode = hidinput_setkeycode;
 	input_dev->getkeycode = hidinput_getkeycode;
 
-	input_dev->name = hid->name;
+	input_dev->name = hidinput->name ? hidinput->name : hid->name;
 	input_dev->phys = hid->phys;
 	input_dev->uniq = hid->uniq;
 	input_dev->id.bustype = hid->bus;
@@ -1513,10 +1577,19 @@ static struct hid_input *hidinput_allocate(struct hid_device *hid)
 	input_dev->id.product = hid->product;
 	input_dev->id.version = hid->version;
 	input_dev->dev.parent = &hid->dev;
+
 	hidinput->input = input_dev;
 	list_add_tail(&hidinput->list, &hid->inputs);
 
+	INIT_LIST_HEAD(&hidinput->reports);
+
 	return hidinput;
+
+fail:
+	kfree(hidinput);
+	input_free_device(input_dev);
+	hid_err(hid, "Out of memory during hid input probe\n");
+	return NULL;
 }
 
 static bool hidinput_has_been_populated(struct hid_input *hidinput)
@@ -1562,6 +1635,7 @@ static void hidinput_cleanup_hidinput(struct hid_device *hid,
 
 	list_del(&hidinput->list);
 	input_free_device(hidinput->input);
+	kfree(hidinput->name);
 
 	for (k = HID_INPUT_REPORT; k <= HID_OUTPUT_REPORT; k++) {
 		if (k == HID_OUTPUT_REPORT &&
@@ -1594,6 +1668,20 @@ static struct hid_input *hidinput_match(struct hid_report *report)
 	return NULL;
 }
 
+static struct hid_input *hidinput_match_application(struct hid_report *report)
+{
+	struct hid_device *hid = report->device;
+	struct hid_input *hidinput;
+
+	list_for_each_entry(hidinput, &hid->inputs, list) {
+		if (hidinput->report &&
+		    hidinput->report->application == report->application)
+			return hidinput;
+	}
+
+	return NULL;
+}
+
 static inline void hidinput_configure_usages(struct hid_input *hidinput,
 					     struct hid_report *report)
 {
@@ -1616,11 +1704,14 @@ int hidinput_connect(struct hid_device *hid, unsigned int force)
 	struct hid_driver *drv = hid->driver;
 	struct hid_report *report;
 	struct hid_input *next, *hidinput = NULL;
+	unsigned int application;
 	int i, k;
 
 	INIT_LIST_HEAD(&hid->inputs);
 	INIT_WORK(&hid->led_work, hidinput_led_worker);
 
+	hid->status &= ~HID_STAT_DUP_DETECTED;
+
 	if (!force) {
 		for (i = 0; i < hid->maxcollection; i++) {
 			struct hid_collection *col = &hid->collection[i];
@@ -1646,15 +1737,20 @@ int hidinput_connect(struct hid_device *hid, unsigned int force)
 			if (!report->maxfield)
 				continue;
 
+			application = report->application;
+
 			/*
 			 * Find the previous hidinput report attached
 			 * to this report id.
 			 */
 			if (hid->quirks & HID_QUIRK_MULTI_INPUT)
 				hidinput = hidinput_match(report);
+			else if (hid->maxapplication > 1 &&
+				 (hid->quirks & HID_QUIRK_INPUT_PER_APP))
+				hidinput = hidinput_match_application(report);
 
 			if (!hidinput) {
-				hidinput = hidinput_allocate(hid);
+				hidinput = hidinput_allocate(hid, application);
 				if (!hidinput)
 					goto out_unwind;
 			}
@@ -1663,6 +1759,9 @@ int hidinput_connect(struct hid_device *hid, unsigned int force)
 
 			if (hid->quirks & HID_QUIRK_MULTI_INPUT)
 				hidinput->report = report;
+
+			list_add_tail(&report->hidinput_list,
+				      &hidinput->reports);
 		}
 	}
 
@@ -1687,6 +1786,10 @@ int hidinput_connect(struct hid_device *hid, unsigned int force)
 		goto out_unwind;
 	}
 
+	if (hid->status & HID_STAT_DUP_DETECTED)
+		hid_dbg(hid,
+			"Some usages could not be mapped, please use HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE if this is legitimate.\n");
+
 	return 0;
 
 out_unwind:
diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c
index 42ed887ba0be..b454c4386157 100644
--- a/drivers/hid/hid-magicmouse.c
+++ b/drivers/hid/hid-magicmouse.c
@@ -531,12 +531,12 @@ static int magicmouse_probe(struct hid_device *hdev,
 
 	if (id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE)
 		report = hid_register_report(hdev, HID_INPUT_REPORT,
-			MOUSE_REPORT_ID);
+			MOUSE_REPORT_ID, 0);
 	else { /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */
 		report = hid_register_report(hdev, HID_INPUT_REPORT,
-			TRACKPAD_REPORT_ID);
+			TRACKPAD_REPORT_ID, 0);
 		report = hid_register_report(hdev, HID_INPUT_REPORT,
-			DOUBLE_REPORT_ID);
+			DOUBLE_REPORT_ID, 0);
 	}
 
 	if (!report) {
diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c
index dad2fbb0e3f8..45968f7970f8 100644
--- a/drivers/hid/hid-multitouch.c
+++ b/drivers/hid/hid-multitouch.c
@@ -81,6 +81,11 @@ MODULE_LICENSE("GPL");
 
 #define MT_BUTTONTYPE_CLICKPAD		0
 
+enum latency_mode {
+	HID_LATENCY_NORMAL = 0,
+	HID_LATENCY_HIGH = 1,
+};
+
 #define MT_IO_FLAGS_RUNNING		0
 #define MT_IO_FLAGS_ACTIVE_SLOTS	1
 #define MT_IO_FLAGS_PENDING_SLOTS	2
@@ -127,11 +132,7 @@ struct mt_device {
 	int left_button_state;	/* left button state */
 	unsigned last_slot_field;	/* the last field of a slot */
 	unsigned mt_report_id;	/* the report ID of the multitouch device */
-	__s16 inputmode;	/* InputMode HID feature, -1 if non-existent */
-	__s16 inputmode_index;	/* InputMode HID feature index in the report */
-	__s16 maxcontact_report_id;	/* Maximum Contact Number HID feature,
-				   -1 if non-existent */
-	__u8 inputmode_value;  /* InputMode HID feature value */
+	__u8 inputmode_value;	/* InputMode HID feature value */
 	__u8 num_received;	/* how many contacts we received */
 	__u8 num_expected;	/* expected last contact index */
 	__u8 maxcontacts;
@@ -415,32 +416,9 @@ static void mt_feature_mapping(struct hid_device *hdev,
 	struct mt_device *td = hid_get_drvdata(hdev);
 
 	switch (usage->hid) {
-	case HID_DG_INPUTMODE:
-		/* Ignore if value index is out of bounds. */
-		if (usage->usage_index >= field->report_count) {
-			dev_err(&hdev->dev, "HID_DG_INPUTMODE out of range\n");
-			break;
-		}
-
-		if (td->inputmode < 0) {
-			td->inputmode = field->report->id;
-			td->inputmode_index = usage->usage_index;
-		} else {
-			/*
-			 * Some elan panels wrongly declare 2 input mode
-			 * features, and silently ignore when we set the
-			 * value in the second field. Skip the second feature
-			 * and hope for the best.
-			 */
-			dev_info(&hdev->dev,
-				 "Ignoring the extra HID_DG_INPUTMODE\n");
-		}
-
-		break;
 	case HID_DG_CONTACTMAX:
 		mt_get_feature(hdev, field->report);
 
-		td->maxcontact_report_id = field->report->id;
 		td->maxcontacts = field->value[0];
 		if (!td->maxcontacts &&
 		    field->logical_maximum <= MT_MAX_MAXCONTACT)
@@ -620,13 +598,16 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi,
 			hid_map_usage(hi, usage, bit, max,
 				EV_MSC, MSC_TIMESTAMP);
 			input_set_capability(hi->input, EV_MSC, MSC_TIMESTAMP);
-			mt_store_field(usage, td, hi);
 			/* Ignore if indexes are out of bounds. */
 			if (field->index >= field->report->maxfield ||
 			    usage->usage_index >= field->report_count)
 				return 1;
 			td->scantime_index = field->index;
 			td->scantime_val_index = usage->usage_index;
+			/*
+			 * We don't set td->last_slot_field as scan time is
+			 * global to the report.
+			 */
 			return 1;
 		case HID_DG_CONTACTCOUNT:
 			/* Ignore if indexes are out of bounds. */
@@ -1181,61 +1162,100 @@ static void mt_report(struct hid_device *hid, struct hid_report *report)
 		input_sync(field->hidinput->input);
 }
 
-static void mt_set_input_mode(struct hid_device *hdev)
+static bool mt_need_to_apply_feature(struct hid_device *hdev,
+				     struct hid_field *field,
+				     struct hid_usage *usage,
+				     enum latency_mode latency,
+				     bool surface_switch,
+				     bool button_switch)
 {
 	struct mt_device *td = hid_get_drvdata(hdev);
-	struct hid_report *r;
-	struct hid_report_enum *re;
 	struct mt_class *cls = &td->mtclass;
+	struct hid_report *report = field->report;
+	unsigned int index = usage->usage_index;
 	char *buf;
 	u32 report_len;
+	int max;
 
-	if (td->inputmode < 0)
-		return;
-
-	re = &(hdev->report_enum[HID_FEATURE_REPORT]);
-	r = re->report_id_hash[td->inputmode];
-	if (r) {
+	switch (usage->hid) {
+	case HID_DG_INPUTMODE:
 		if (cls->quirks & MT_QUIRK_FORCE_GET_FEATURE) {
-			report_len = hid_report_len(r);
-			buf = hid_alloc_report_buf(r, GFP_KERNEL);
+			report_len = hid_report_len(report);
+			buf = hid_alloc_report_buf(report, GFP_KERNEL);
 			if (!buf) {
-				hid_err(hdev, "failed to allocate buffer for report\n");
-				return;
+				hid_err(hdev,
+					"failed to allocate buffer for report\n");
+				return false;
 			}
-			hid_hw_raw_request(hdev, r->id, buf, report_len,
+			hid_hw_raw_request(hdev, report->id, buf, report_len,
 					   HID_FEATURE_REPORT,
 					   HID_REQ_GET_REPORT);
 			kfree(buf);
 		}
-		r->field[0]->value[td->inputmode_index] = td->inputmode_value;
-		hid_hw_request(hdev, r, HID_REQ_SET_REPORT);
-	}
-}
 
-static void mt_set_maxcontacts(struct hid_device *hdev)
-{
-	struct mt_device *td = hid_get_drvdata(hdev);
-	struct hid_report *r;
-	struct hid_report_enum *re;
-	int fieldmax, max;
+		field->value[index] = td->inputmode_value;
+		return true;
 
-	if (td->maxcontact_report_id < 0)
-		return;
+	case HID_DG_CONTACTMAX:
+		if (td->mtclass.maxcontacts) {
+			max = min_t(int, field->logical_maximum,
+				    td->mtclass.maxcontacts);
+			if (field->value[index] != max) {
+				field->value[index] = max;
+				return true;
+			}
+		}
+		break;
 
-	if (!td->mtclass.maxcontacts)
-		return;
+	case HID_DG_LATENCYMODE:
+		field->value[index] = latency;
+		return true;
+
+	case HID_DG_SURFACESWITCH:
+		field->value[index] = surface_switch;
+		return true;
+
+	case HID_DG_BUTTONSWITCH:
+		field->value[index] = button_switch;
+		return true;
+	}
 
-	re = &hdev->report_enum[HID_FEATURE_REPORT];
-	r = re->report_id_hash[td->maxcontact_report_id];
-	if (r) {
-		max = td->mtclass.maxcontacts;
-		fieldmax = r->field[0]->logical_maximum;
-		max = min(fieldmax, max);
-		if (r->field[0]->value[0] != max) {
-			r->field[0]->value[0] = max;
-			hid_hw_request(hdev, r, HID_REQ_SET_REPORT);
+	return false; /* no need to update the report */
+}
+
+static void mt_set_modes(struct hid_device *hdev, enum latency_mode latency,
+			 bool surface_switch, bool button_switch)
+{
+	struct hid_report_enum *rep_enum;
+	struct hid_report *rep;
+	struct hid_usage *usage;
+	int i, j;
+	bool update_report;
+
+	rep_enum = &hdev->report_enum[HID_FEATURE_REPORT];
+	list_for_each_entry(rep, &rep_enum->report_list, list) {
+		update_report = false;
+
+		for (i = 0; i < rep->maxfield; i++) {
+			/* Ignore if report count is out of bounds. */
+			if (rep->field[i]->report_count < 1)
+				continue;
+
+			for (j = 0; j < rep->field[i]->maxusage; j++) {
+				usage = &rep->field[i]->usage[j];
+
+				if (mt_need_to_apply_feature(hdev,
+							     rep->field[i],
+							     usage,
+							     latency,
+							     surface_switch,
+							     button_switch))
+					update_report = true;
+			}
 		}
+
+		if (update_report)
+			hid_hw_request(hdev, rep, HID_REQ_SET_REPORT);
 	}
 }
 
@@ -1274,54 +1294,48 @@ static int mt_input_configured(struct hid_device *hdev, struct hid_input *hi)
 	struct mt_device *td = hid_get_drvdata(hdev);
 	char *name;
 	const char *suffix = NULL;
-	struct hid_field *field = hi->report->field[0];
+	unsigned int application = 0;
+	struct hid_report *report;
 	int ret;
 
-	if (hi->report->id == td->mt_report_id) {
-		ret = mt_touch_input_configured(hdev, hi);
-		if (ret)
-			return ret;
+	list_for_each_entry(report, &hi->reports, hidinput_list) {
+		application = report->application;
+		if (report->id == td->mt_report_id) {
+			ret = mt_touch_input_configured(hdev, hi);
+			if (ret)
+				return ret;
+		}
+
+		/*
+		 * some egalax touchscreens have "application == DG_TOUCHSCREEN"
+		 * for the stylus. Check this first, and then rely on
+		 * the application field.
+		 */
+		if (report->field[0]->physical == HID_DG_STYLUS) {
+			suffix = "Pen";
+			/* force BTN_STYLUS to allow tablet matching in udev */
+			__set_bit(BTN_STYLUS, hi->input->keybit);
+		}
 	}
 
-	/*
-	 * some egalax touchscreens have "application == HID_DG_TOUCHSCREEN"
-	 * for the stylus. Check this first, and then rely on the application
-	 * field.
-	 */
-	if (hi->report->field[0]->physical == HID_DG_STYLUS) {
-		suffix = "Pen";
-		/* force BTN_STYLUS to allow tablet matching in udev */
-		__set_bit(BTN_STYLUS, hi->input->keybit);
-	} else {
-		switch (field->application) {
+	if (!suffix) {
+		switch (application) {
 		case HID_GD_KEYBOARD:
-			suffix = "Keyboard";
-			break;
 		case HID_GD_KEYPAD:
-			suffix = "Keypad";
-			break;
 		case HID_GD_MOUSE:
-			suffix = "Mouse";
-			break;
-		case HID_DG_STYLUS:
-			suffix = "Pen";
-			/* force BTN_STYLUS to allow tablet matching in udev */
-			__set_bit(BTN_STYLUS, hi->input->keybit);
-			break;
-		case HID_DG_TOUCHSCREEN:
-			/* we do not set suffix = "Touchscreen" */
-			break;
 		case HID_DG_TOUCHPAD:
-			suffix = "Touchpad";
-			break;
 		case HID_GD_SYSTEM_CONTROL:
-			suffix = "System Control";
-			break;
 		case HID_CP_CONSUMER_CONTROL:
-			suffix = "Consumer Control";
-			break;
 		case HID_GD_WIRELESS_RADIO_CTLS:
-			suffix = "Wireless Radio Control";
+			/* already handled by hid core */
+			break;
+		case HID_DG_TOUCHSCREEN:
+			/* we do not set suffix = "Touchscreen" */
+			hi->input->name = hdev->name;
+			break;
+		case HID_DG_STYLUS:
+			/* force BTN_STYLUS to allow tablet matching in udev */
+			__set_bit(BTN_STYLUS, hi->input->keybit);
 			break;
 		case HID_VD_ASUS_CUSTOM_MEDIA_KEYS:
 			suffix = "Custom Media Keys";
@@ -1434,8 +1448,6 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	}
 	td->hdev = hdev;
 	td->mtclass = *mtclass;
-	td->inputmode = -1;
-	td->maxcontact_report_id = -1;
 	td->inputmode_value = MT_INPUTMODE_TOUCHSCREEN;
 	td->cc_index = -1;
 	td->scantime_index = -1;
@@ -1459,10 +1471,10 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id)
 
 	/*
 	 * This allows the driver to handle different input sensors
-	 * that emits events through different reports on the same HID
+	 * that emits events through different applications on the same HID
 	 * device.
 	 */
-	hdev->quirks |= HID_QUIRK_MULTI_INPUT;
+	hdev->quirks |= HID_QUIRK_INPUT_PER_APP;
 
 	timer_setup(&td->release_timer, mt_expired_timeout, 0);
 
@@ -1482,8 +1494,7 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		dev_warn(&hdev->dev, "Cannot allocate sysfs group for %s\n",
 				hdev->name);
 
-	mt_set_maxcontacts(hdev);
-	mt_set_input_mode(hdev);
+	mt_set_modes(hdev, HID_LATENCY_NORMAL, true, true);
 
 	/* release .fields memory as it is not used anymore */
 	devm_kfree(&hdev->dev, td->fields);
@@ -1496,8 +1507,7 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id)
 static int mt_reset_resume(struct hid_device *hdev)
 {
 	mt_release_contacts(hdev);
-	mt_set_maxcontacts(hdev);
-	mt_set_input_mode(hdev);
+	mt_set_modes(hdev, HID_LATENCY_NORMAL, true, true);
 	return 0;
 }
 
diff --git a/drivers/hid/hid-plantronics.c b/drivers/hid/hid-plantronics.c
index febb21ee190e..584b10d3fc3d 100644
--- a/drivers/hid/hid-plantronics.c
+++ b/drivers/hid/hid-plantronics.c
@@ -2,7 +2,7 @@
  *  Plantronics USB HID Driver
  *
  *  Copyright (c) 2014 JD Cole <jd.cole@plantronics.com>
- *  Copyright (c) 2015 Terry Junge <terry.junge@plantronics.com>
+ *  Copyright (c) 2015-2018 Terry Junge <terry.junge@plantronics.com>
  */
 
 /*
@@ -48,6 +48,10 @@ static int plantronics_input_mapping(struct hid_device *hdev,
 	unsigned short mapped_key;
 	unsigned long plt_type = (unsigned long)hid_get_drvdata(hdev);
 
+	/* special case for PTT products */
+	if (field->application == HID_GD_JOYSTICK)
+		goto defaulted;
+
 	/* handle volume up/down mapping */
 	/* non-standard types or multi-HID interfaces - plt_type is PID */
 	if (!(plt_type & HID_USAGE_PAGE)) {
diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c
index 587e2681a53f..249d49b6b16c 100644
--- a/drivers/hid/hid-quirks.c
+++ b/drivers/hid/hid-quirks.c
@@ -416,7 +416,6 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LCPOWER, USB_DEVICE_ID_LCPOWER_LC1000) },
 #endif
 #if IS_ENABLED(CONFIG_HID_LED)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DELCOM, USB_DEVICE_ID_DELCOM_VISUAL_IND) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_DREAM_CHEEKY, USB_DEVICE_ID_DREAM_CHEEKY_WN) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_DREAM_CHEEKY, USB_DEVICE_ID_DREAM_CHEEKY_FA) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_LUXAFOR) },
diff --git a/drivers/hid/hid-redragon.c b/drivers/hid/hid-redragon.c
new file mode 100644
index 000000000000..daf59578bf93
--- /dev/null
+++ b/drivers/hid/hid-redragon.c
@@ -0,0 +1,86 @@
+/*
+ *  HID driver for Redragon keyboards
+ *
+ *  Copyright (c) 2017 Robert Munteanu
+ *  SPDX-License-Identifier: GPL-2.0+
+ */
+
+/*
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <linux/device.h>
+#include <linux/hid.h>
+#include <linux/module.h>
+
+#include "hid-ids.h"
+
+
+/*
+ * The Redragon Asura keyboard sends an incorrect HID descriptor.
+ * At byte 100 it contains
+ *
+ *   0x81, 0x00
+ *
+ * which is Input (Data, Arr, Abs), but it should be
+ *
+ *   0x81, 0x02
+ *
+ * which is Input (Data, Var, Abs), which is consistent with the way
+ * key codes are generated.
+ */
+
+static __u8 *redragon_report_fixup(struct hid_device *hdev, __u8 *rdesc,
+	unsigned int *rsize)
+{
+	if (*rsize >= 102 && rdesc[100] == 0x81 && rdesc[101] == 0x00) {
+		dev_info(&hdev->dev, "Fixing Redragon ASURA report descriptor.\n");
+		rdesc[101] = 0x02;
+	}
+
+	return rdesc;
+}
+
+static int redragon_probe(struct hid_device *dev,
+	const struct hid_device_id *id)
+{
+	int ret;
+
+	ret = hid_parse(dev);
+	if (ret) {
+		hid_err(dev, "parse failed\n");
+		return ret;
+	}
+
+	/* do not register unused input device */
+	if (dev->maxapplication == 1)
+		return 0;
+
+	ret = hid_hw_start(dev, HID_CONNECT_DEFAULT);
+	if (ret) {
+		hid_err(dev, "hw start failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+static const struct hid_device_id redragon_devices[] = {
+	{HID_USB_DEVICE(USB_VENDOR_ID_JESS, USB_DEVICE_ID_REDRAGON_ASURA)},
+	{}
+};
+
+MODULE_DEVICE_TABLE(hid, redragon_devices);
+
+static struct hid_driver redragon_driver = {
+	.name = "redragon",
+	.id_table = redragon_devices,
+	.report_fixup = redragon_report_fixup,
+	.probe = redragon_probe
+};
+
+module_hid_driver(redragon_driver);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/hid/hid-rmi.c b/drivers/hid/hid-rmi.c
index 9c9362149641..9e33165250a3 100644
--- a/drivers/hid/hid-rmi.c
+++ b/drivers/hid/hid-rmi.c
@@ -413,6 +413,24 @@ static int rmi_event(struct hid_device *hdev, struct hid_field *field,
 	return 0;
 }
 
+static void rmi_report(struct hid_device *hid, struct hid_report *report)
+{
+	struct hid_field *field = report->field[0];
+
+	if (!(hid->claimed & HID_CLAIMED_INPUT))
+		return;
+
+	switch (report->id) {
+	case RMI_READ_DATA_REPORT_ID:
+		/* fall-through */
+	case RMI_ATTN_REPORT_ID:
+		return;
+	}
+
+	if (field && field->hidinput && field->hidinput->input)
+		input_sync(field->hidinput->input);
+}
+
 #ifdef CONFIG_PM
 static int rmi_suspend(struct hid_device *hdev, pm_message_t message)
 {
@@ -637,6 +655,7 @@ static int rmi_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	hid_set_drvdata(hdev, data);
 
 	hdev->quirks |= HID_QUIRK_NO_INIT_REPORTS;
+	hdev->quirks |= HID_QUIRK_NO_INPUT_SYNC;
 
 	ret = hid_parse(hdev);
 	if (ret) {
@@ -744,6 +763,7 @@ static struct hid_driver rmi_driver = {
 	.remove			= rmi_remove,
 	.event			= rmi_event,
 	.raw_event		= rmi_raw_event,
+	.report			= rmi_report,
 	.input_mapping		= rmi_input_mapping,
 	.input_configured	= rmi_input_configured,
 #ifdef CONFIG_PM
diff --git a/drivers/hid/hid-steam.c b/drivers/hid/hid-steam.c
new file mode 100644
index 000000000000..cb86cc834201
--- /dev/null
+++ b/drivers/hid/hid-steam.c
@@ -0,0 +1,1115 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * HID driver for Valve Steam Controller
+ *
+ * Copyright (c) 2018 Rodrigo Rivas Costa <rodrigorivascosta@gmail.com>
+ *
+ * Supports both the wired and wireless interfaces.
+ *
+ * This controller has a builtin emulation of mouse and keyboard: the right pad
+ * can be used as a mouse, the shoulder buttons are mouse buttons, A and B
+ * buttons are ENTER and ESCAPE, and so on. This is implemented as additional
+ * HID interfaces.
+ *
+ * This is known as the "lizard mode", because apparently lizards like to use
+ * the computer from the coach, without a proper mouse and keyboard.
+ *
+ * This driver will disable the lizard mode when the input device is opened
+ * and re-enable it when the input device is closed, so as not to break user
+ * mode behaviour. The lizard_mode parameter can be used to change that.
+ *
+ * There are a few user space applications (notably Steam Client) that use
+ * the hidraw interface directly to create input devices (XTest, uinput...).
+ * In order to avoid breaking them this driver creates a layered hidraw device,
+ * so it can detect when the client is running and then:
+ *  - it will not send any command to the controller.
+ *  - this input device will be disabled, to avoid double input of the same
+ *    user action.
+ *
+ * For additional functions, such as changing the right-pad margin or switching
+ * the led, you can use the user-space tool at:
+ *
+ *   https://github.com/rodrigorc/steamctrl
+ */
+
+#include <linux/device.h>
+#include <linux/input.h>
+#include <linux/hid.h>
+#include <linux/module.h>
+#include <linux/workqueue.h>
+#include <linux/mutex.h>
+#include <linux/rcupdate.h>
+#include <linux/delay.h>
+#include <linux/power_supply.h>
+#include "hid-ids.h"
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Rodrigo Rivas Costa <rodrigorivascosta@gmail.com>");
+
+static bool lizard_mode = true;
+
+static DEFINE_MUTEX(steam_devices_lock);
+static LIST_HEAD(steam_devices);
+
+#define STEAM_QUIRK_WIRELESS		BIT(0)
+
+/* Touch pads are 40 mm in diameter and 65535 units */
+#define STEAM_PAD_RESOLUTION 1638
+/* Trigger runs are about 5 mm and 256 units */
+#define STEAM_TRIGGER_RESOLUTION 51
+/* Joystick runs are about 5 mm and 256 units */
+#define STEAM_JOYSTICK_RESOLUTION 51
+
+#define STEAM_PAD_FUZZ 256
+
+/*
+ * Commands that can be sent in a feature report.
+ * Thanks to Valve for some valuable hints.
+ */
+#define STEAM_CMD_SET_MAPPINGS		0x80
+#define STEAM_CMD_CLEAR_MAPPINGS	0x81
+#define STEAM_CMD_GET_MAPPINGS		0x82
+#define STEAM_CMD_GET_ATTRIB		0x83
+#define STEAM_CMD_GET_ATTRIB_LABEL	0x84
+#define STEAM_CMD_DEFAULT_MAPPINGS	0x85
+#define STEAM_CMD_FACTORY_RESET		0x86
+#define STEAM_CMD_WRITE_REGISTER	0x87
+#define STEAM_CMD_CLEAR_REGISTER	0x88
+#define STEAM_CMD_READ_REGISTER		0x89
+#define STEAM_CMD_GET_REGISTER_LABEL	0x8a
+#define STEAM_CMD_GET_REGISTER_MAX	0x8b
+#define STEAM_CMD_GET_REGISTER_DEFAULT	0x8c
+#define STEAM_CMD_SET_MODE		0x8d
+#define STEAM_CMD_DEFAULT_MOUSE		0x8e
+#define STEAM_CMD_FORCEFEEDBAK		0x8f
+#define STEAM_CMD_REQUEST_COMM_STATUS	0xb4
+#define STEAM_CMD_GET_SERIAL		0xae
+
+/* Some useful register ids */
+#define STEAM_REG_LPAD_MODE		0x07
+#define STEAM_REG_RPAD_MODE		0x08
+#define STEAM_REG_RPAD_MARGIN		0x18
+#define STEAM_REG_LED			0x2d
+#define STEAM_REG_GYRO_MODE		0x30
+
+/* Raw event identifiers */
+#define STEAM_EV_INPUT_DATA		0x01
+#define STEAM_EV_CONNECT		0x03
+#define STEAM_EV_BATTERY		0x04
+
+/* Values for GYRO_MODE (bitmask) */
+#define STEAM_GYRO_MODE_OFF		0x0000
+#define STEAM_GYRO_MODE_STEERING	0x0001
+#define STEAM_GYRO_MODE_TILT		0x0002
+#define STEAM_GYRO_MODE_SEND_ORIENTATION	0x0004
+#define STEAM_GYRO_MODE_SEND_RAW_ACCEL		0x0008
+#define STEAM_GYRO_MODE_SEND_RAW_GYRO		0x0010
+
+/* Other random constants */
+#define STEAM_SERIAL_LEN 10
+
+struct steam_device {
+	struct list_head list;
+	spinlock_t lock;
+	struct hid_device *hdev, *client_hdev;
+	struct mutex mutex;
+	bool client_opened, input_opened;
+	struct input_dev __rcu *input;
+	unsigned long quirks;
+	struct work_struct work_connect;
+	bool connected;
+	char serial_no[STEAM_SERIAL_LEN + 1];
+	struct power_supply_desc battery_desc;
+	struct power_supply __rcu *battery;
+	u8 battery_charge;
+	u16 voltage;
+};
+
+static int steam_recv_report(struct steam_device *steam,
+		u8 *data, int size)
+{
+	struct hid_report *r;
+	u8 *buf;
+	int ret;
+
+	r = steam->hdev->report_enum[HID_FEATURE_REPORT].report_id_hash[0];
+	if (hid_report_len(r) < 64)
+		return -EINVAL;
+
+	buf = hid_alloc_report_buf(r, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	/*
+	 * The report ID is always 0, so strip the first byte from the output.
+	 * hid_report_len() is not counting the report ID, so +1 to the length
+	 * or else we get a EOVERFLOW. We are safe from a buffer overflow
+	 * because hid_alloc_report_buf() allocates +7 bytes.
+	 */
+	ret = hid_hw_raw_request(steam->hdev, 0x00,
+			buf, hid_report_len(r) + 1,
+			HID_FEATURE_REPORT, HID_REQ_GET_REPORT);
+	if (ret > 0)
+		memcpy(data, buf + 1, min(size, ret - 1));
+	kfree(buf);
+	return ret;
+}
+
+static int steam_send_report(struct steam_device *steam,
+		u8 *cmd, int size)
+{
+	struct hid_report *r;
+	u8 *buf;
+	unsigned int retries = 50;
+	int ret;
+
+	r = steam->hdev->report_enum[HID_FEATURE_REPORT].report_id_hash[0];
+	if (hid_report_len(r) < 64)
+		return -EINVAL;
+
+	buf = hid_alloc_report_buf(r, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	/* The report ID is always 0 */
+	memcpy(buf + 1, cmd, size);
+
+	/*
+	 * Sometimes the wireless controller fails with EPIPE
+	 * when sending a feature report.
+	 * Doing a HID_REQ_GET_REPORT and waiting for a while
+	 * seems to fix that.
+	 */
+	do {
+		ret = hid_hw_raw_request(steam->hdev, 0,
+				buf, size + 1,
+				HID_FEATURE_REPORT, HID_REQ_SET_REPORT);
+		if (ret != -EPIPE)
+			break;
+		msleep(20);
+	} while (--retries);
+
+	kfree(buf);
+	if (ret < 0)
+		hid_err(steam->hdev, "%s: error %d (%*ph)\n", __func__,
+				ret, size, cmd);
+	return ret;
+}
+
+static inline int steam_send_report_byte(struct steam_device *steam, u8 cmd)
+{
+	return steam_send_report(steam, &cmd, 1);
+}
+
+static int steam_write_registers(struct steam_device *steam,
+		/* u8 reg, u16 val */...)
+{
+	/* Send: 0x87 len (reg valLo valHi)* */
+	u8 reg;
+	u16 val;
+	u8 cmd[64] = {STEAM_CMD_WRITE_REGISTER, 0x00};
+	va_list args;
+
+	va_start(args, steam);
+	for (;;) {
+		reg = va_arg(args, int);
+		if (reg == 0)
+			break;
+		val = va_arg(args, int);
+		cmd[cmd[1] + 2] = reg;
+		cmd[cmd[1] + 3] = val & 0xff;
+		cmd[cmd[1] + 4] = val >> 8;
+		cmd[1] += 3;
+	}
+	va_end(args);
+
+	return steam_send_report(steam, cmd, 2 + cmd[1]);
+}
+
+static int steam_get_serial(struct steam_device *steam)
+{
+	/*
+	 * Send: 0xae 0x15 0x01
+	 * Recv: 0xae 0x15 0x01 serialnumber (10 chars)
+	 */
+	int ret;
+	u8 cmd[] = {STEAM_CMD_GET_SERIAL, 0x15, 0x01};
+	u8 reply[3 + STEAM_SERIAL_LEN + 1];
+
+	ret = steam_send_report(steam, cmd, sizeof(cmd));
+	if (ret < 0)
+		return ret;
+	ret = steam_recv_report(steam, reply, sizeof(reply));
+	if (ret < 0)
+		return ret;
+	if (reply[0] != 0xae || reply[1] != 0x15 || reply[2] != 0x01)
+		return -EIO;
+	reply[3 + STEAM_SERIAL_LEN] = 0;
+	strlcpy(steam->serial_no, reply + 3, sizeof(steam->serial_no));
+	return 0;
+}
+
+/*
+ * This command requests the wireless adaptor to post an event
+ * with the connection status. Useful if this driver is loaded when
+ * the controller is already connected.
+ */
+static inline int steam_request_conn_status(struct steam_device *steam)
+{
+	return steam_send_report_byte(steam, STEAM_CMD_REQUEST_COMM_STATUS);
+}
+
+static void steam_set_lizard_mode(struct steam_device *steam, bool enable)
+{
+	if (enable) {
+		/* enable esc, enter, cursors */
+		steam_send_report_byte(steam, STEAM_CMD_DEFAULT_MAPPINGS);
+		/* enable mouse */
+		steam_send_report_byte(steam, STEAM_CMD_DEFAULT_MOUSE);
+		steam_write_registers(steam,
+			STEAM_REG_RPAD_MARGIN, 0x01, /* enable margin */
+			0);
+	} else {
+		/* disable esc, enter, cursor */
+		steam_send_report_byte(steam, STEAM_CMD_CLEAR_MAPPINGS);
+		steam_write_registers(steam,
+			STEAM_REG_RPAD_MODE, 0x07, /* disable mouse */
+			STEAM_REG_RPAD_MARGIN, 0x00, /* disable margin */
+			0);
+	}
+}
+
+static void steam_update_lizard_mode(struct steam_device *steam)
+{
+	mutex_lock(&steam->mutex);
+	if (!steam->client_opened) {
+		if (steam->input_opened)
+			steam_set_lizard_mode(steam, false);
+		else
+			steam_set_lizard_mode(steam, lizard_mode);
+	}
+	mutex_unlock(&steam->mutex);
+}
+
+static int steam_input_open(struct input_dev *dev)
+{
+	struct steam_device *steam = input_get_drvdata(dev);
+	int ret;
+
+	ret = hid_hw_open(steam->hdev);
+	if (ret)
+		return ret;
+
+	mutex_lock(&steam->mutex);
+	steam->input_opened = true;
+	if (!steam->client_opened && lizard_mode)
+		steam_set_lizard_mode(steam, false);
+	mutex_unlock(&steam->mutex);
+	return 0;
+}
+
+static void steam_input_close(struct input_dev *dev)
+{
+	struct steam_device *steam = input_get_drvdata(dev);
+
+	mutex_lock(&steam->mutex);
+	steam->input_opened = false;
+	if (!steam->client_opened && lizard_mode)
+		steam_set_lizard_mode(steam, true);
+	mutex_unlock(&steam->mutex);
+
+	hid_hw_close(steam->hdev);
+}
+
+static enum power_supply_property steam_battery_props[] = {
+	POWER_SUPPLY_PROP_PRESENT,
+	POWER_SUPPLY_PROP_SCOPE,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_CAPACITY,
+};
+
+static int steam_battery_get_property(struct power_supply *psy,
+				enum power_supply_property psp,
+				union power_supply_propval *val)
+{
+	struct steam_device *steam = power_supply_get_drvdata(psy);
+	unsigned long flags;
+	s16 volts;
+	u8 batt;
+	int ret = 0;
+
+	spin_lock_irqsave(&steam->lock, flags);
+	volts = steam->voltage;
+	batt = steam->battery_charge;
+	spin_unlock_irqrestore(&steam->lock, flags);
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_PRESENT:
+		val->intval = 1;
+		break;
+	case POWER_SUPPLY_PROP_SCOPE:
+		val->intval = POWER_SUPPLY_SCOPE_DEVICE;
+		break;
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		val->intval = volts * 1000; /* mV -> uV */
+		break;
+	case POWER_SUPPLY_PROP_CAPACITY:
+		val->intval = batt;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static int steam_battery_register(struct steam_device *steam)
+{
+	struct power_supply *battery;
+	struct power_supply_config battery_cfg = { .drv_data = steam, };
+	unsigned long flags;
+	int ret;
+
+	steam->battery_desc.type = POWER_SUPPLY_TYPE_BATTERY;
+	steam->battery_desc.properties = steam_battery_props;
+	steam->battery_desc.num_properties = ARRAY_SIZE(steam_battery_props);
+	steam->battery_desc.get_property = steam_battery_get_property;
+	steam->battery_desc.name = devm_kasprintf(&steam->hdev->dev,
+			GFP_KERNEL, "steam-controller-%s-battery",
+			steam->serial_no);
+	if (!steam->battery_desc.name)
+		return -ENOMEM;
+
+	/* avoid the warning of 0% battery while waiting for the first info */
+	spin_lock_irqsave(&steam->lock, flags);
+	steam->voltage = 3000;
+	steam->battery_charge = 100;
+	spin_unlock_irqrestore(&steam->lock, flags);
+
+	battery = power_supply_register(&steam->hdev->dev,
+			&steam->battery_desc, &battery_cfg);
+	if (IS_ERR(battery)) {
+		ret = PTR_ERR(battery);
+		hid_err(steam->hdev,
+				"%s:power_supply_register failed with error %d\n",
+				__func__, ret);
+		return ret;
+	}
+	rcu_assign_pointer(steam->battery, battery);
+	power_supply_powers(battery, &steam->hdev->dev);
+	return 0;
+}
+
+static int steam_register(struct steam_device *steam)
+{
+	struct hid_device *hdev = steam->hdev;
+	struct input_dev *input;
+	int ret;
+
+	rcu_read_lock();
+	input = rcu_dereference(steam->input);
+	rcu_read_unlock();
+	if (input) {
+		dbg_hid("%s: already connected\n", __func__);
+		return 0;
+	}
+
+	/*
+	 * Unlikely, but getting the serial could fail, and it is not so
+	 * important, so make up a serial number and go on.
+	 */
+	if (steam_get_serial(steam) < 0)
+		strlcpy(steam->serial_no, "XXXXXXXXXX",
+				sizeof(steam->serial_no));
+
+	hid_info(hdev, "Steam Controller '%s' connected",
+			steam->serial_no);
+
+	input = input_allocate_device();
+	if (!input)
+		return -ENOMEM;
+
+	input_set_drvdata(input, steam);
+	input->dev.parent = &hdev->dev;
+	input->open = steam_input_open;
+	input->close = steam_input_close;
+
+	input->name = (steam->quirks & STEAM_QUIRK_WIRELESS) ?
+		"Wireless Steam Controller" :
+		"Steam Controller";
+	input->phys = hdev->phys;
+	input->uniq = steam->serial_no;
+	input->id.bustype = hdev->bus;
+	input->id.vendor = hdev->vendor;
+	input->id.product = hdev->product;
+	input->id.version = hdev->version;
+
+	input_set_capability(input, EV_KEY, BTN_TR2);
+	input_set_capability(input, EV_KEY, BTN_TL2);
+	input_set_capability(input, EV_KEY, BTN_TR);
+	input_set_capability(input, EV_KEY, BTN_TL);
+	input_set_capability(input, EV_KEY, BTN_Y);
+	input_set_capability(input, EV_KEY, BTN_B);
+	input_set_capability(input, EV_KEY, BTN_X);
+	input_set_capability(input, EV_KEY, BTN_A);
+	input_set_capability(input, EV_KEY, BTN_DPAD_UP);
+	input_set_capability(input, EV_KEY, BTN_DPAD_RIGHT);
+	input_set_capability(input, EV_KEY, BTN_DPAD_LEFT);
+	input_set_capability(input, EV_KEY, BTN_DPAD_DOWN);
+	input_set_capability(input, EV_KEY, BTN_SELECT);
+	input_set_capability(input, EV_KEY, BTN_MODE);
+	input_set_capability(input, EV_KEY, BTN_START);
+	input_set_capability(input, EV_KEY, BTN_GEAR_DOWN);
+	input_set_capability(input, EV_KEY, BTN_GEAR_UP);
+	input_set_capability(input, EV_KEY, BTN_THUMBR);
+	input_set_capability(input, EV_KEY, BTN_THUMBL);
+	input_set_capability(input, EV_KEY, BTN_THUMB);
+	input_set_capability(input, EV_KEY, BTN_THUMB2);
+
+	input_set_abs_params(input, ABS_HAT2Y, 0, 255, 0, 0);
+	input_set_abs_params(input, ABS_HAT2X, 0, 255, 0, 0);
+	input_set_abs_params(input, ABS_X, -32767, 32767, 0, 0);
+	input_set_abs_params(input, ABS_Y, -32767, 32767, 0, 0);
+	input_set_abs_params(input, ABS_RX, -32767, 32767,
+			STEAM_PAD_FUZZ, 0);
+	input_set_abs_params(input, ABS_RY, -32767, 32767,
+			STEAM_PAD_FUZZ, 0);
+	input_set_abs_params(input, ABS_HAT0X, -32767, 32767,
+			STEAM_PAD_FUZZ, 0);
+	input_set_abs_params(input, ABS_HAT0Y, -32767, 32767,
+			STEAM_PAD_FUZZ, 0);
+	input_abs_set_res(input, ABS_X, STEAM_JOYSTICK_RESOLUTION);
+	input_abs_set_res(input, ABS_Y, STEAM_JOYSTICK_RESOLUTION);
+	input_abs_set_res(input, ABS_RX, STEAM_PAD_RESOLUTION);
+	input_abs_set_res(input, ABS_RY, STEAM_PAD_RESOLUTION);
+	input_abs_set_res(input, ABS_HAT0X, STEAM_PAD_RESOLUTION);
+	input_abs_set_res(input, ABS_HAT0Y, STEAM_PAD_RESOLUTION);
+	input_abs_set_res(input, ABS_HAT2Y, STEAM_TRIGGER_RESOLUTION);
+	input_abs_set_res(input, ABS_HAT2X, STEAM_TRIGGER_RESOLUTION);
+
+	ret = input_register_device(input);
+	if (ret)
+		goto input_register_fail;
+
+	rcu_assign_pointer(steam->input, input);
+
+	/* ignore battery errors, we can live without it */
+	if (steam->quirks & STEAM_QUIRK_WIRELESS)
+		steam_battery_register(steam);
+
+	return 0;
+
+input_register_fail:
+	input_free_device(input);
+	return ret;
+}
+
+static void steam_unregister(struct steam_device *steam)
+{
+	struct input_dev *input;
+	struct power_supply *battery;
+
+	rcu_read_lock();
+	input = rcu_dereference(steam->input);
+	battery = rcu_dereference(steam->battery);
+	rcu_read_unlock();
+
+	if (battery) {
+		RCU_INIT_POINTER(steam->battery, NULL);
+		synchronize_rcu();
+		power_supply_unregister(battery);
+	}
+	if (input) {
+		RCU_INIT_POINTER(steam->input, NULL);
+		synchronize_rcu();
+		hid_info(steam->hdev, "Steam Controller '%s' disconnected",
+				steam->serial_no);
+		input_unregister_device(input);
+	}
+}
+
+static void steam_work_connect_cb(struct work_struct *work)
+{
+	struct steam_device *steam = container_of(work, struct steam_device,
+							work_connect);
+	unsigned long flags;
+	bool connected;
+	int ret;
+
+	spin_lock_irqsave(&steam->lock, flags);
+	connected = steam->connected;
+	spin_unlock_irqrestore(&steam->lock, flags);
+
+	if (connected) {
+		ret = steam_register(steam);
+		if (ret) {
+			hid_err(steam->hdev,
+				"%s:steam_register failed with error %d\n",
+				__func__, ret);
+		}
+	} else {
+		steam_unregister(steam);
+	}
+}
+
+static bool steam_is_valve_interface(struct hid_device *hdev)
+{
+	struct hid_report_enum *rep_enum;
+
+	/*
+	 * The wired device creates 3 interfaces:
+	 *  0: emulated mouse.
+	 *  1: emulated keyboard.
+	 *  2: the real game pad.
+	 * The wireless device creates 5 interfaces:
+	 *  0: emulated keyboard.
+	 *  1-4: slots where up to 4 real game pads will be connected to.
+	 * We know which one is the real gamepad interface because they are the
+	 * only ones with a feature report.
+	 */
+	rep_enum = &hdev->report_enum[HID_FEATURE_REPORT];
+	return !list_empty(&rep_enum->report_list);
+}
+
+static int steam_client_ll_parse(struct hid_device *hdev)
+{
+	struct steam_device *steam = hid_get_drvdata(hdev);
+
+	return hid_parse_report(hdev, steam->hdev->dev_rdesc,
+			steam->hdev->dev_rsize);
+}
+
+static int steam_client_ll_start(struct hid_device *hdev)
+{
+	return 0;
+}
+
+static void steam_client_ll_stop(struct hid_device *hdev)
+{
+}
+
+static int steam_client_ll_open(struct hid_device *hdev)
+{
+	struct steam_device *steam = hid_get_drvdata(hdev);
+	int ret;
+
+	ret = hid_hw_open(steam->hdev);
+	if (ret)
+		return ret;
+
+	mutex_lock(&steam->mutex);
+	steam->client_opened = true;
+	mutex_unlock(&steam->mutex);
+	return ret;
+}
+
+static void steam_client_ll_close(struct hid_device *hdev)
+{
+	struct steam_device *steam = hid_get_drvdata(hdev);
+
+	mutex_lock(&steam->mutex);
+	steam->client_opened = false;
+	if (steam->input_opened)
+		steam_set_lizard_mode(steam, false);
+	else
+		steam_set_lizard_mode(steam, lizard_mode);
+	mutex_unlock(&steam->mutex);
+
+	hid_hw_close(steam->hdev);
+}
+
+static int steam_client_ll_raw_request(struct hid_device *hdev,
+				unsigned char reportnum, u8 *buf,
+				size_t count, unsigned char report_type,
+				int reqtype)
+{
+	struct steam_device *steam = hid_get_drvdata(hdev);
+
+	return hid_hw_raw_request(steam->hdev, reportnum, buf, count,
+			report_type, reqtype);
+}
+
+static struct hid_ll_driver steam_client_ll_driver = {
+	.parse = steam_client_ll_parse,
+	.start = steam_client_ll_start,
+	.stop = steam_client_ll_stop,
+	.open = steam_client_ll_open,
+	.close = steam_client_ll_close,
+	.raw_request = steam_client_ll_raw_request,
+};
+
+static struct hid_device *steam_create_client_hid(struct hid_device *hdev)
+{
+	struct hid_device *client_hdev;
+
+	client_hdev = hid_allocate_device();
+	if (IS_ERR(client_hdev))
+		return client_hdev;
+
+	client_hdev->ll_driver = &steam_client_ll_driver;
+	client_hdev->dev.parent = hdev->dev.parent;
+	client_hdev->bus = hdev->bus;
+	client_hdev->vendor = hdev->vendor;
+	client_hdev->product = hdev->product;
+	client_hdev->version = hdev->version;
+	client_hdev->type = hdev->type;
+	client_hdev->country = hdev->country;
+	strlcpy(client_hdev->name, hdev->name,
+			sizeof(client_hdev->name));
+	strlcpy(client_hdev->phys, hdev->phys,
+			sizeof(client_hdev->phys));
+	/*
+	 * Since we use the same device info than the real interface to
+	 * trick userspace, we will be calling steam_probe recursively.
+	 * We need to recognize the client interface somehow.
+	 */
+	client_hdev->group = HID_GROUP_STEAM;
+	return client_hdev;
+}
+
+static int steam_probe(struct hid_device *hdev,
+				const struct hid_device_id *id)
+{
+	struct steam_device *steam;
+	int ret;
+
+	ret = hid_parse(hdev);
+	if (ret) {
+		hid_err(hdev,
+			"%s:parse of hid interface failed\n", __func__);
+		return ret;
+	}
+
+	/*
+	 * The virtual client_dev is only used for hidraw.
+	 * Also avoid the recursive probe.
+	 */
+	if (hdev->group == HID_GROUP_STEAM)
+		return hid_hw_start(hdev, HID_CONNECT_HIDRAW);
+	/*
+	 * The non-valve interfaces (mouse and keyboard emulation) are
+	 * connected without changes.
+	 */
+	if (!steam_is_valve_interface(hdev))
+		return hid_hw_start(hdev, HID_CONNECT_DEFAULT);
+
+	steam = devm_kzalloc(&hdev->dev, sizeof(*steam), GFP_KERNEL);
+	if (!steam) {
+		ret = -ENOMEM;
+		goto steam_alloc_fail;
+	}
+	steam->hdev = hdev;
+	hid_set_drvdata(hdev, steam);
+	spin_lock_init(&steam->lock);
+	mutex_init(&steam->mutex);
+	steam->quirks = id->driver_data;
+	INIT_WORK(&steam->work_connect, steam_work_connect_cb);
+
+	steam->client_hdev = steam_create_client_hid(hdev);
+	if (IS_ERR(steam->client_hdev)) {
+		ret = PTR_ERR(steam->client_hdev);
+		goto client_hdev_fail;
+	}
+	hid_set_drvdata(steam->client_hdev, steam);
+
+	/*
+	 * With the real steam controller interface, do not connect hidraw.
+	 * Instead, create the client_hid and connect that.
+	 */
+	ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT & ~HID_CONNECT_HIDRAW);
+	if (ret)
+		goto hid_hw_start_fail;
+
+	ret = hid_add_device(steam->client_hdev);
+	if (ret)
+		goto client_hdev_add_fail;
+
+	if (steam->quirks & STEAM_QUIRK_WIRELESS) {
+		ret = hid_hw_open(hdev);
+		if (ret) {
+			hid_err(hdev,
+				"%s:hid_hw_open for wireless\n",
+				__func__);
+			goto hid_hw_open_fail;
+		}
+		hid_info(hdev, "Steam wireless receiver connected");
+		steam_request_conn_status(steam);
+	} else {
+		ret = steam_register(steam);
+		if (ret) {
+			hid_err(hdev,
+				"%s:steam_register failed with error %d\n",
+				__func__, ret);
+			goto input_register_fail;
+		}
+	}
+
+	mutex_lock(&steam_devices_lock);
+	steam_update_lizard_mode(steam);
+	list_add(&steam->list, &steam_devices);
+	mutex_unlock(&steam_devices_lock);
+
+	return 0;
+
+hid_hw_open_fail:
+input_register_fail:
+client_hdev_add_fail:
+	hid_hw_stop(hdev);
+hid_hw_start_fail:
+	hid_destroy_device(steam->client_hdev);
+client_hdev_fail:
+	cancel_work_sync(&steam->work_connect);
+steam_alloc_fail:
+	hid_err(hdev, "%s: failed with error %d\n",
+			__func__, ret);
+	return ret;
+}
+
+static void steam_remove(struct hid_device *hdev)
+{
+	struct steam_device *steam = hid_get_drvdata(hdev);
+
+	if (!steam || hdev->group == HID_GROUP_STEAM) {
+		hid_hw_stop(hdev);
+		return;
+	}
+
+	mutex_lock(&steam_devices_lock);
+	list_del(&steam->list);
+	mutex_unlock(&steam_devices_lock);
+
+	hid_destroy_device(steam->client_hdev);
+	steam->client_opened = false;
+	cancel_work_sync(&steam->work_connect);
+	if (steam->quirks & STEAM_QUIRK_WIRELESS) {
+		hid_info(hdev, "Steam wireless receiver disconnected");
+		hid_hw_close(hdev);
+	}
+	hid_hw_stop(hdev);
+	steam_unregister(steam);
+}
+
+static void steam_do_connect_event(struct steam_device *steam, bool connected)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&steam->lock, flags);
+	steam->connected = connected;
+	spin_unlock_irqrestore(&steam->lock, flags);
+
+	if (schedule_work(&steam->work_connect) == 0)
+		dbg_hid("%s: connected=%d event already queued\n",
+				__func__, connected);
+}
+
+/*
+ * Some input data in the protocol has the opposite sign.
+ * Clamp the values to 32767..-32767 so that the range is
+ * symmetrical and can be negated safely.
+ */
+static inline s16 steam_le16(u8 *data)
+{
+	s16 x = (s16) le16_to_cpup((__le16 *)data);
+
+	return x == -32768 ? -32767 : x;
+}
+
+/*
+ * The size for this message payload is 60.
+ * The known values are:
+ *  (* values are not sent through wireless)
+ *  (* accelerator/gyro is disabled by default)
+ *  Offset| Type  | Mapped to |Meaning
+ * -------+-------+-----------+--------------------------
+ *  4-7   | u32   | --        | sequence number
+ *  8-10  | 24bit | see below | buttons
+ *  11    | u8    | ABS_HAT2Y | left trigger
+ *  12    | u8    | ABS_HAT2X | right trigger
+ *  13-15 | --    | --        | always 0
+ *  16-17 | s16   | ABS_X/ABS_HAT0X     | X value
+ *  18-19 | s16   | ABS_Y/ABS_HAT0Y     | Y value
+ *  20-21 | s16   | ABS_RX    | right-pad X value
+ *  22-23 | s16   | ABS_RY    | right-pad Y value
+ *  24-25 | s16   | --        | * left trigger
+ *  26-27 | s16   | --        | * right trigger
+ *  28-29 | s16   | --        | * accelerometer X value
+ *  30-31 | s16   | --        | * accelerometer Y value
+ *  32-33 | s16   | --        | * accelerometer Z value
+ *  34-35 | s16   | --        | gyro X value
+ *  36-36 | s16   | --        | gyro Y value
+ *  38-39 | s16   | --        | gyro Z value
+ *  40-41 | s16   | --        | quaternion W value
+ *  42-43 | s16   | --        | quaternion X value
+ *  44-45 | s16   | --        | quaternion Y value
+ *  46-47 | s16   | --        | quaternion Z value
+ *  48-49 | --    | --        | always 0
+ *  50-51 | s16   | --        | * left trigger (uncalibrated)
+ *  52-53 | s16   | --        | * right trigger (uncalibrated)
+ *  54-55 | s16   | --        | * joystick X value (uncalibrated)
+ *  56-57 | s16   | --        | * joystick Y value (uncalibrated)
+ *  58-59 | s16   | --        | * left-pad X value
+ *  60-61 | s16   | --        | * left-pad Y value
+ *  62-63 | u16   | --        | * battery voltage
+ *
+ * The buttons are:
+ *  Bit  | Mapped to  | Description
+ * ------+------------+--------------------------------
+ *  8.0  | BTN_TR2    | right trigger fully pressed
+ *  8.1  | BTN_TL2    | left trigger fully pressed
+ *  8.2  | BTN_TR     | right shoulder
+ *  8.3  | BTN_TL     | left shoulder
+ *  8.4  | BTN_Y      | button Y
+ *  8.5  | BTN_B      | button B
+ *  8.6  | BTN_X      | button X
+ *  8.7  | BTN_A      | button A
+ *  9.0  | BTN_DPAD_UP    | lef-pad up
+ *  9.1  | BTN_DPAD_RIGHT | lef-pad right
+ *  9.2  | BTN_DPAD_LEFT  | lef-pad left
+ *  9.3  | BTN_DPAD_DOWN  | lef-pad down
+ *  9.4  | BTN_SELECT | menu left
+ *  9.5  | BTN_MODE   | steam logo
+ *  9.6  | BTN_START  | menu right
+ *  9.7  | BTN_GEAR_DOWN | left back lever
+ * 10.0  | BTN_GEAR_UP   | right back lever
+ * 10.1  | --         | left-pad clicked
+ * 10.2  | BTN_THUMBR | right-pad clicked
+ * 10.3  | BTN_THUMB  | left-pad touched (but see explanation below)
+ * 10.4  | BTN_THUMB2 | right-pad touched
+ * 10.5  | --         | unknown
+ * 10.6  | BTN_THUMBL | joystick clicked
+ * 10.7  | --         | lpad_and_joy
+ */
+
+static void steam_do_input_event(struct steam_device *steam,
+		struct input_dev *input, u8 *data)
+{
+	/* 24 bits of buttons */
+	u8 b8, b9, b10;
+	s16 x, y;
+	bool lpad_touched, lpad_and_joy;
+
+	b8 = data[8];
+	b9 = data[9];
+	b10 = data[10];
+
+	input_report_abs(input, ABS_HAT2Y, data[11]);
+	input_report_abs(input, ABS_HAT2X, data[12]);
+
+	/*
+	 * These two bits tells how to interpret the values X and Y.
+	 * lpad_and_joy tells that the joystick and the lpad are used at the
+	 * same time.
+	 * lpad_touched tells whether X/Y are to be read as lpad coord or
+	 * joystick values.
+	 * (lpad_touched || lpad_and_joy) tells if the lpad is really touched.
+	 */
+	lpad_touched = b10 & BIT(3);
+	lpad_and_joy = b10 & BIT(7);
+	x = steam_le16(data + 16);
+	y = -steam_le16(data + 18);
+
+	input_report_abs(input, lpad_touched ? ABS_HAT0X : ABS_X, x);
+	input_report_abs(input, lpad_touched ? ABS_HAT0Y : ABS_Y, y);
+	/* Check if joystick is centered */
+	if (lpad_touched && !lpad_and_joy) {
+		input_report_abs(input, ABS_X, 0);
+		input_report_abs(input, ABS_Y, 0);
+	}
+	/* Check if lpad is untouched */
+	if (!(lpad_touched || lpad_and_joy)) {
+		input_report_abs(input, ABS_HAT0X, 0);
+		input_report_abs(input, ABS_HAT0Y, 0);
+	}
+
+	input_report_abs(input, ABS_RX, steam_le16(data + 20));
+	input_report_abs(input, ABS_RY, -steam_le16(data + 22));
+
+	input_event(input, EV_KEY, BTN_TR2, !!(b8 & BIT(0)));
+	input_event(input, EV_KEY, BTN_TL2, !!(b8 & BIT(1)));
+	input_event(input, EV_KEY, BTN_TR, !!(b8 & BIT(2)));
+	input_event(input, EV_KEY, BTN_TL, !!(b8 & BIT(3)));
+	input_event(input, EV_KEY, BTN_Y, !!(b8 & BIT(4)));
+	input_event(input, EV_KEY, BTN_B, !!(b8 & BIT(5)));
+	input_event(input, EV_KEY, BTN_X, !!(b8 & BIT(6)));
+	input_event(input, EV_KEY, BTN_A, !!(b8 & BIT(7)));
+	input_event(input, EV_KEY, BTN_SELECT, !!(b9 & BIT(4)));
+	input_event(input, EV_KEY, BTN_MODE, !!(b9 & BIT(5)));
+	input_event(input, EV_KEY, BTN_START, !!(b9 & BIT(6)));
+	input_event(input, EV_KEY, BTN_GEAR_DOWN, !!(b9 & BIT(7)));
+	input_event(input, EV_KEY, BTN_GEAR_UP, !!(b10 & BIT(0)));
+	input_event(input, EV_KEY, BTN_THUMBR, !!(b10 & BIT(2)));
+	input_event(input, EV_KEY, BTN_THUMBL, !!(b10 & BIT(6)));
+	input_event(input, EV_KEY, BTN_THUMB, lpad_touched || lpad_and_joy);
+	input_event(input, EV_KEY, BTN_THUMB2, !!(b10 & BIT(4)));
+	input_event(input, EV_KEY, BTN_DPAD_UP, !!(b9 & BIT(0)));
+	input_event(input, EV_KEY, BTN_DPAD_RIGHT, !!(b9 & BIT(1)));
+	input_event(input, EV_KEY, BTN_DPAD_LEFT, !!(b9 & BIT(2)));
+	input_event(input, EV_KEY, BTN_DPAD_DOWN, !!(b9 & BIT(3)));
+
+	input_sync(input);
+}
+
+/*
+ * The size for this message payload is 11.
+ * The known values are:
+ *  Offset| Type  | Meaning
+ * -------+-------+---------------------------
+ *  4-7   | u32   | sequence number
+ *  8-11  | --    | always 0
+ *  12-13 | u16   | voltage (mV)
+ *  14    | u8    | battery percent
+ */
+static void steam_do_battery_event(struct steam_device *steam,
+		struct power_supply *battery, u8 *data)
+{
+	unsigned long flags;
+
+	s16 volts = steam_le16(data + 12);
+	u8 batt = data[14];
+
+	/* Creating the battery may have failed */
+	rcu_read_lock();
+	battery = rcu_dereference(steam->battery);
+	if (likely(battery)) {
+		spin_lock_irqsave(&steam->lock, flags);
+		steam->voltage = volts;
+		steam->battery_charge = batt;
+		spin_unlock_irqrestore(&steam->lock, flags);
+		power_supply_changed(battery);
+	}
+	rcu_read_unlock();
+}
+
+static int steam_raw_event(struct hid_device *hdev,
+			struct hid_report *report, u8 *data,
+			int size)
+{
+	struct steam_device *steam = hid_get_drvdata(hdev);
+	struct input_dev *input;
+	struct power_supply *battery;
+
+	if (!steam)
+		return 0;
+
+	if (steam->client_opened)
+		hid_input_report(steam->client_hdev, HID_FEATURE_REPORT,
+				data, size, 0);
+	/*
+	 * All messages are size=64, all values little-endian.
+	 * The format is:
+	 *  Offset| Meaning
+	 * -------+--------------------------------------------
+	 *  0-1   | always 0x01, 0x00, maybe protocol version?
+	 *  2     | type of message
+	 *  3     | length of the real payload (not checked)
+	 *  4-n   | payload data, depends on the type
+	 *
+	 * There are these known types of message:
+	 *  0x01: input data (60 bytes)
+	 *  0x03: wireless connect/disconnect (1 byte)
+	 *  0x04: battery status (11 bytes)
+	 */
+
+	if (size != 64 || data[0] != 1 || data[1] != 0)
+		return 0;
+
+	switch (data[2]) {
+	case STEAM_EV_INPUT_DATA:
+		if (steam->client_opened)
+			return 0;
+		rcu_read_lock();
+		input = rcu_dereference(steam->input);
+		if (likely(input)) {
+			steam_do_input_event(steam, input, data);
+		} else {
+			dbg_hid("%s: input data without connect event\n",
+					__func__);
+			steam_do_connect_event(steam, true);
+		}
+		rcu_read_unlock();
+		break;
+	case STEAM_EV_CONNECT:
+		/*
+		 * The payload of this event is a single byte:
+		 *  0x01: disconnected.
+		 *  0x02: connected.
+		 */
+		switch (data[4]) {
+		case 0x01:
+			steam_do_connect_event(steam, false);
+			break;
+		case 0x02:
+			steam_do_connect_event(steam, true);
+			break;
+		}
+		break;
+	case STEAM_EV_BATTERY:
+		if (steam->quirks & STEAM_QUIRK_WIRELESS) {
+			rcu_read_lock();
+			battery = rcu_dereference(steam->battery);
+			if (likely(battery)) {
+				steam_do_battery_event(steam, battery, data);
+			} else {
+				dbg_hid(
+					"%s: battery data without connect event\n",
+					__func__);
+				steam_do_connect_event(steam, true);
+			}
+			rcu_read_unlock();
+		}
+		break;
+	}
+	return 0;
+}
+
+static int steam_param_set_lizard_mode(const char *val,
+					const struct kernel_param *kp)
+{
+	struct steam_device *steam;
+	int ret;
+
+	ret = param_set_bool(val, kp);
+	if (ret)
+		return ret;
+
+	mutex_lock(&steam_devices_lock);
+	list_for_each_entry(steam, &steam_devices, list) {
+		steam_update_lizard_mode(steam);
+	}
+	mutex_unlock(&steam_devices_lock);
+	return 0;
+}
+
+static const struct kernel_param_ops steam_lizard_mode_ops = {
+	.set	= steam_param_set_lizard_mode,
+	.get	= param_get_bool,
+};
+
+module_param_cb(lizard_mode, &steam_lizard_mode_ops, &lizard_mode, 0644);
+MODULE_PARM_DESC(lizard_mode,
+	"Enable mouse and keyboard emulation (lizard mode) when the gamepad is not in use");
+
+static const struct hid_device_id steam_controllers[] = {
+	{ /* Wired Steam Controller */
+	  HID_USB_DEVICE(USB_VENDOR_ID_VALVE,
+		USB_DEVICE_ID_STEAM_CONTROLLER)
+	},
+	{ /* Wireless Steam Controller */
+	  HID_USB_DEVICE(USB_VENDOR_ID_VALVE,
+		USB_DEVICE_ID_STEAM_CONTROLLER_WIRELESS),
+	  .driver_data = STEAM_QUIRK_WIRELESS
+	},
+	{}
+};
+
+MODULE_DEVICE_TABLE(hid, steam_controllers);
+
+static struct hid_driver steam_controller_driver = {
+	.name = "hid-steam",
+	.id_table = steam_controllers,
+	.probe = steam_probe,
+	.remove = steam_remove,
+	.raw_event = steam_raw_event,
+};
+
+module_hid_driver(steam_controller_driver);
diff --git a/drivers/hid/i2c-hid/i2c-hid.c b/drivers/hid/i2c-hid/i2c-hid.c
index cc33622253aa..c1652bb7bd15 100644
--- a/drivers/hid/i2c-hid/i2c-hid.c
+++ b/drivers/hid/i2c-hid/i2c-hid.c
@@ -131,8 +131,6 @@ static const struct i2c_hid_cmd hid_no_cmd =		{ .length = 0 };
  * static const struct i2c_hid_cmd hid_set_protocol_cmd = { I2C_HID_CMD(0x07) };
  */
 
-static DEFINE_MUTEX(i2c_hid_open_mut);
-
 /* The main device structure */
 struct i2c_hid {
 	struct i2c_client	*client;	/* i2c client */
@@ -868,6 +866,15 @@ static int i2c_hid_fetch_hid_descriptor(struct i2c_hid *ihid)
 }
 
 #ifdef CONFIG_ACPI
+static const struct acpi_device_id i2c_hid_acpi_blacklist[] = {
+	/*
+	 * The CHPN0001 ACPI device, which is used to describe the Chipone
+	 * ICN8505 controller, has a _CID of PNP0C50 but is not HID compatible.
+	 */
+	{"CHPN0001", 0 },
+	{ },
+};
+
 static int i2c_hid_acpi_pdata(struct i2c_client *client,
 		struct i2c_hid_platform_data *pdata)
 {
@@ -879,13 +886,18 @@ static int i2c_hid_acpi_pdata(struct i2c_client *client,
 	acpi_handle handle;
 
 	handle = ACPI_HANDLE(&client->dev);
-	if (!handle || acpi_bus_get_device(handle, &adev))
+	if (!handle || acpi_bus_get_device(handle, &adev)) {
+		dev_err(&client->dev, "Error could not get ACPI device\n");
+		return -ENODEV;
+	}
+
+	if (acpi_match_device_ids(adev, i2c_hid_acpi_blacklist) == 0)
 		return -ENODEV;
 
 	obj = acpi_evaluate_dsm_typed(handle, &i2c_hid_guid, 1, 1, NULL,
 				      ACPI_TYPE_INTEGER);
 	if (!obj) {
-		dev_err(&client->dev, "device _DSM execution failed\n");
+		dev_err(&client->dev, "Error _DSM call to get HID descriptor address failed\n");
 		return -ENODEV;
 	}
 
@@ -1000,11 +1012,8 @@ static int i2c_hid_probe(struct i2c_client *client,
 			goto err;
 	} else if (!platform_data) {
 		ret = i2c_hid_acpi_pdata(client, &ihid->pdata);
-		if (ret) {
-			dev_err(&client->dev,
-				"HID register address not provided\n");
+		if (ret)
 			goto err;
-		}
 	} else {
 		ihid->pdata = *platform_data;
 	}
@@ -1054,6 +1063,14 @@ static int i2c_hid_probe(struct i2c_client *client,
 	pm_runtime_enable(&client->dev);
 	device_enable_async_suspend(&client->dev);
 
+	/* Make sure there is something at this address */
+	ret = i2c_smbus_read_byte(client);
+	if (ret < 0) {
+		dev_dbg(&client->dev, "nothing at this address: %d\n", ret);
+		ret = -ENXIO;
+		goto err_pm;
+	}
+
 	ret = i2c_hid_fetch_hid_descriptor(ihid);
 	if (ret < 0)
 		goto err_pm;
diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c
index 5f947ec20dcb..0bb44d0088ed 100644
--- a/drivers/hid/wacom_wac.c
+++ b/drivers/hid/wacom_wac.c
@@ -2894,24 +2894,31 @@ static int wacom_bpt_pen(struct wacom_wac *wacom)
 	struct wacom_features *features = &wacom->features;
 	struct input_dev *input = wacom->pen_input;
 	unsigned char *data = wacom->data;
-	int prox = 0, x = 0, y = 0, p = 0, d = 0, pen = 0, btn1 = 0, btn2 = 0;
+	int x = 0, y = 0, p = 0, d = 0;
+	bool pen = false, btn1 = false, btn2 = false;
+	bool range, prox, rdy;
 
 	if (data[0] != WACOM_REPORT_PENABLED)
 	    return 0;
 
-	prox = (data[1] & 0x20) == 0x20;
+	range = (data[1] & 0x80) == 0x80;
+	prox = (data[1] & 0x40) == 0x40;
+	rdy = (data[1] & 0x20) == 0x20;
+
+	wacom->shared->stylus_in_proximity = range;
+	if (delay_pen_events(wacom))
+		return 0;
+
+	if (rdy) {
+		p = le16_to_cpup((__le16 *)&data[6]);
+		pen = data[1] & 0x01;
+		btn1 = data[1] & 0x02;
+		btn2 = data[1] & 0x04;
+	}
+	if (prox) {
+		x = le16_to_cpup((__le16 *)&data[2]);
+		y = le16_to_cpup((__le16 *)&data[4]);
 
-	/*
-	 * All reports shared between PEN and RUBBER tool must be
-	 * forced to a known starting value (zero) when transitioning to
-	 * out-of-prox.
-	 *
-	 * If not reset then, to userspace, it will look like lost events
-	 * if new tool comes in-prox with same values as previous tool sent.
-	 *
-	 * Hardware does report zero in most out-of-prox cases but not all.
-	 */
-	if (!wacom->shared->stylus_in_proximity) {
 		if (data[1] & 0x08) {
 			wacom->tool[0] = BTN_TOOL_RUBBER;
 			wacom->id[0] = ERASER_DEVICE_ID;
@@ -2919,16 +2926,9 @@ static int wacom_bpt_pen(struct wacom_wac *wacom)
 			wacom->tool[0] = BTN_TOOL_PEN;
 			wacom->id[0] = STYLUS_DEVICE_ID;
 		}
+		wacom->reporting_data = true;
 	}
-
-	wacom->shared->stylus_in_proximity = prox;
-	if (delay_pen_events(wacom))
-		return 0;
-
-	if (prox) {
-		x = le16_to_cpup((__le16 *)&data[2]);
-		y = le16_to_cpup((__le16 *)&data[4]);
-		p = le16_to_cpup((__le16 *)&data[6]);
+	if (range) {
 		/*
 		 * Convert distance from out prox to distance from tablet.
 		 * distance will be greater than distance_max once
@@ -2937,25 +2937,29 @@ static int wacom_bpt_pen(struct wacom_wac *wacom)
 		 */
 		if (data[8] <= features->distance_max)
 			d = features->distance_max - data[8];
-
-		pen = data[1] & 0x01;
-		btn1 = data[1] & 0x02;
-		btn2 = data[1] & 0x04;
 	} else {
 		wacom->id[0] = 0;
 	}
 
-	input_report_key(input, BTN_TOUCH, pen);
-	input_report_key(input, BTN_STYLUS, btn1);
-	input_report_key(input, BTN_STYLUS2, btn2);
+	if (wacom->reporting_data) {
+		input_report_key(input, BTN_TOUCH, pen);
+		input_report_key(input, BTN_STYLUS, btn1);
+		input_report_key(input, BTN_STYLUS2, btn2);
 
-	input_report_abs(input, ABS_X, x);
-	input_report_abs(input, ABS_Y, y);
-	input_report_abs(input, ABS_PRESSURE, p);
-	input_report_abs(input, ABS_DISTANCE, d);
+		if (prox || !range) {
+			input_report_abs(input, ABS_X, x);
+			input_report_abs(input, ABS_Y, y);
+		}
+		input_report_abs(input, ABS_PRESSURE, p);
+		input_report_abs(input, ABS_DISTANCE, d);
 
-	input_report_key(input, wacom->tool[0], prox); /* PEN or RUBBER */
-	input_report_abs(input, ABS_MISC, wacom->id[0]); /* TOOL ID */
+		input_report_key(input, wacom->tool[0], range); /* PEN or RUBBER */
+		input_report_abs(input, ABS_MISC, wacom->id[0]); /* TOOL ID */
+	}
+
+	if (!range) {
+		wacom->reporting_data = false;
+	}
 
 	return 1;
 }
diff --git a/drivers/hsi/clients/cmt_speech.c b/drivers/hsi/clients/cmt_speech.c
index 324cb8ec9405..a1d4b9366496 100644
--- a/drivers/hsi/clients/cmt_speech.c
+++ b/drivers/hsi/clients/cmt_speech.c
@@ -1098,7 +1098,7 @@ static void cs_hsi_stop(struct cs_hsi_iface *hi)
 	kfree(hi);
 }
 
-static int cs_char_vma_fault(struct vm_fault *vmf)
+static vm_fault_t cs_char_vma_fault(struct vm_fault *vmf)
 {
 	struct cs_char *csdata = vmf->vma->vm_private_data;
 	struct page *page;
diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c
index 72855182b191..ced041899456 100644
--- a/drivers/hv/connection.c
+++ b/drivers/hv/connection.c
@@ -63,6 +63,9 @@ static __u32 vmbus_get_next_version(__u32 current_version)
 	case (VERSION_WIN10):
 		return VERSION_WIN8_1;
 
+	case (VERSION_WIN10_V5):
+		return VERSION_WIN10;
+
 	case (VERSION_WS2008):
 	default:
 		return VERSION_INVAL;
@@ -80,9 +83,29 @@ static int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo,
 
 	msg = (struct vmbus_channel_initiate_contact *)msginfo->msg;
 
+	memset(msg, 0, sizeof(*msg));
 	msg->header.msgtype = CHANNELMSG_INITIATE_CONTACT;
 	msg->vmbus_version_requested = version;
-	msg->interrupt_page = virt_to_phys(vmbus_connection.int_page);
+
+	/*
+	 * VMBus protocol 5.0 (VERSION_WIN10_V5) requires that we must use
+	 * VMBUS_MESSAGE_CONNECTION_ID_4 for the Initiate Contact Message,
+	 * and for subsequent messages, we must use the Message Connection ID
+	 * field in the host-returned Version Response Message. And, with
+	 * VERSION_WIN10_V5, we don't use msg->interrupt_page, but we tell
+	 * the host explicitly that we still use VMBUS_MESSAGE_SINT(2) for
+	 * compatibility.
+	 *
+	 * On old hosts, we should always use VMBUS_MESSAGE_CONNECTION_ID (1).
+	 */
+	if (version >= VERSION_WIN10_V5) {
+		msg->msg_sint = VMBUS_MESSAGE_SINT;
+		vmbus_connection.msg_conn_id = VMBUS_MESSAGE_CONNECTION_ID_4;
+	} else {
+		msg->interrupt_page = virt_to_phys(vmbus_connection.int_page);
+		vmbus_connection.msg_conn_id = VMBUS_MESSAGE_CONNECTION_ID;
+	}
+
 	msg->monitor_page1 = virt_to_phys(vmbus_connection.monitor_pages[0]);
 	msg->monitor_page2 = virt_to_phys(vmbus_connection.monitor_pages[1]);
 	/*
@@ -137,6 +160,10 @@ static int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo,
 	/* Check if successful */
 	if (msginfo->response.version_response.version_supported) {
 		vmbus_connection.conn_state = CONNECTED;
+
+		if (version >= VERSION_WIN10_V5)
+			vmbus_connection.msg_conn_id =
+				msginfo->response.version_response.msg_conn_id;
 	} else {
 		return -ECONNREFUSED;
 	}
@@ -354,13 +381,14 @@ void vmbus_on_event(unsigned long data)
  */
 int vmbus_post_msg(void *buffer, size_t buflen, bool can_sleep)
 {
+	struct vmbus_channel_message_header *hdr;
 	union hv_connection_id conn_id;
 	int ret = 0;
 	int retries = 0;
 	u32 usec = 1;
 
 	conn_id.asu32 = 0;
-	conn_id.u.id = VMBUS_MESSAGE_CONNECTION_ID;
+	conn_id.u.id = vmbus_connection.msg_conn_id;
 
 	/*
 	 * hv_post_message() can have transient failures because of
@@ -372,6 +400,18 @@ int vmbus_post_msg(void *buffer, size_t buflen, bool can_sleep)
 
 		switch (ret) {
 		case HV_STATUS_INVALID_CONNECTION_ID:
+			/*
+			 * See vmbus_negotiate_version(): VMBus protocol 5.0
+			 * requires that we must use
+			 * VMBUS_MESSAGE_CONNECTION_ID_4 for the Initiate
+			 * Contact message, but on old hosts that only
+			 * support VMBus protocol 4.0 or lower, here we get
+			 * HV_STATUS_INVALID_CONNECTION_ID and we should
+			 * return an error immediately without retrying.
+			 */
+			hdr = buffer;
+			if (hdr->msgtype == CHANNELMSG_INITIATE_CONTACT)
+				return -EINVAL;
 			/*
 			 * We could get this if we send messages too
 			 * frequently.
diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h
index f761bef36e77..72eaba3d50fc 100644
--- a/drivers/hv/hyperv_vmbus.h
+++ b/drivers/hv/hyperv_vmbus.h
@@ -187,6 +187,7 @@ struct hv_input_post_message {
 
 enum {
 	VMBUS_MESSAGE_CONNECTION_ID	= 1,
+	VMBUS_MESSAGE_CONNECTION_ID_4	= 4,
 	VMBUS_MESSAGE_PORT_ID		= 1,
 	VMBUS_EVENT_CONNECTION_ID	= 2,
 	VMBUS_EVENT_PORT_ID		= 2,
@@ -302,6 +303,8 @@ struct vmbus_connection {
 	 */
 	int connect_cpu;
 
+	u32 msg_conn_id;
+
 	atomic_t offer_in_progress;
 
 	enum vmbus_connect_state conn_state;
diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c
index 8699bb969e7e..3c836c099a8f 100644
--- a/drivers/hv/ring_buffer.c
+++ b/drivers/hv/ring_buffer.c
@@ -227,6 +227,8 @@ int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
 	ring_info->ring_buffer->feature_bits.value = 1;
 
 	ring_info->ring_size = page_cnt << PAGE_SHIFT;
+	ring_info->ring_size_div10_reciprocal =
+		reciprocal_value(ring_info->ring_size / 10);
 	ring_info->ring_datasize = ring_info->ring_size -
 		sizeof(struct hv_ring_buffer);
 
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index 6ec307c93ece..f10840ad465c 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -717,15 +717,12 @@ config SENSORS_LTC2945
 	  be called ltc2945.
 
 config SENSORS_LTC2990
-	tristate "Linear Technology LTC2990 (current monitoring mode only)"
+	tristate "Linear Technology LTC2990"
 	depends on I2C
 	help
 	  If you say yes here you get support for Linear Technology LTC2990
 	  I2C System Monitor. The LTC2990 supports a combination of voltage,
-	  current and temperature monitoring, but in addition to the Vcc supply
-	  voltage and chip temperature, this driver currently only supports
-	  reading two currents by measuring two differential voltages across
-	  series resistors.
+	  current and temperature monitoring.
 
 	  This driver can also be built as a module. If so, the module will
 	  be called ltc2990.
diff --git a/drivers/hwmon/fschmd.c b/drivers/hwmon/fschmd.c
index 5e78229ade04..22d3a84f13ef 100644
--- a/drivers/hwmon/fschmd.c
+++ b/drivers/hwmon/fschmd.c
@@ -105,7 +105,7 @@ static const u8 FSCHMD_REG_VOLT[7][6] = {
 static const int FSCHMD_NO_VOLT_SENSORS[7] = { 3, 3, 3, 3, 3, 3, 6 };
 
 /*
- * minimum pwm at which the fan is driven (pwm can by increased depending on
+ * minimum pwm at which the fan is driven (pwm can be increased depending on
  * the temp. Notice that for the scy some fans share there minimum speed.
  * Also notice that with the scy the sensor order is different than with the
  * other chips, this order was in the 2.4 driver and kept for consistency.
diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c
index 32083e452cde..e88c01961948 100644
--- a/drivers/hwmon/hwmon.c
+++ b/drivers/hwmon/hwmon.c
@@ -698,6 +698,9 @@ hwmon_device_register_with_info(struct device *dev, const char *name,
 	if (chip && (!chip->ops || !chip->ops->is_visible || !chip->info))
 		return ERR_PTR(-EINVAL);
 
+	if (chip && !dev)
+		return ERR_PTR(-EINVAL);
+
 	return __hwmon_device_register(dev, name, drvdata, chip, extra_groups);
 }
 EXPORT_SYMBOL_GPL(hwmon_device_register_with_info);
diff --git a/drivers/hwmon/ibmpowernv.c b/drivers/hwmon/ibmpowernv.c
index 5ccdd0b52650..0298745d46e4 100644
--- a/drivers/hwmon/ibmpowernv.c
+++ b/drivers/hwmon/ibmpowernv.c
@@ -51,6 +51,7 @@ enum sensors {
 	POWER_SUPPLY,
 	POWER_INPUT,
 	CURRENT,
+	ENERGY,
 	MAX_SENSOR_TYPE,
 };
 
@@ -78,6 +79,7 @@ static struct sensor_group {
 	{ "in"    },
 	{ "power" },
 	{ "curr"  },
+	{ "energy" },
 };
 
 struct sensor_data {
@@ -101,9 +103,10 @@ static ssize_t show_sensor(struct device *dev, struct device_attribute *devattr,
 	struct sensor_data *sdata = container_of(devattr, struct sensor_data,
 						 dev_attr);
 	ssize_t ret;
-	u32 x;
+	u64 x;
+
+	ret =  opal_get_sensor_data_u64(sdata->id, &x);
 
-	ret = opal_get_sensor_data(sdata->id, &x);
 	if (ret)
 		return ret;
 
@@ -114,7 +117,7 @@ static ssize_t show_sensor(struct device *dev, struct device_attribute *devattr,
 	else if (sdata->type == POWER_INPUT)
 		x *= 1000000;
 
-	return sprintf(buf, "%u\n", x);
+	return sprintf(buf, "%llu\n", x);
 }
 
 static ssize_t show_label(struct device *dev, struct device_attribute *devattr,
diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c
index 3b73dee6fdc6..d3fae5a8e508 100644
--- a/drivers/hwmon/k10temp.c
+++ b/drivers/hwmon/k10temp.c
@@ -37,6 +37,10 @@ MODULE_PARM_DESC(force, "force loading on processors with erratum 319");
 /* Provide lock for writing to NB_SMU_IND_ADDR */
 static DEFINE_MUTEX(nb_smu_ind_mutex);
 
+#ifndef PCI_DEVICE_ID_AMD_15H_M70H_NB_F3
+#define PCI_DEVICE_ID_AMD_15H_M70H_NB_F3	0x15b3
+#endif
+
 #ifndef PCI_DEVICE_ID_AMD_17H_DF_F3
 #define PCI_DEVICE_ID_AMD_17H_DF_F3	0x1463
 #endif
@@ -81,6 +85,7 @@ struct k10temp_data {
 	void (*read_tempreg)(struct pci_dev *pdev, u32 *regval);
 	int temp_offset;
 	u32 temp_adjust_mask;
+	bool show_tdie;
 };
 
 struct tctl_offset {
@@ -141,17 +146,24 @@ static void read_tempreg_nb_f17(struct pci_dev *pdev, u32 *regval)
 		     F17H_M01H_REPORTED_TEMP_CTRL_OFFSET, regval);
 }
 
-static ssize_t temp1_input_show(struct device *dev,
-				struct device_attribute *attr, char *buf)
+unsigned int get_raw_temp(struct k10temp_data *data)
 {
-	struct k10temp_data *data = dev_get_drvdata(dev);
-	u32 regval;
 	unsigned int temp;
+	u32 regval;
 
 	data->read_tempreg(data->pdev, &regval);
 	temp = (regval >> 21) * 125;
 	if (regval & data->temp_adjust_mask)
 		temp -= 49000;
+	return temp;
+}
+
+static ssize_t temp1_input_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct k10temp_data *data = dev_get_drvdata(dev);
+	unsigned int temp = get_raw_temp(data);
+
 	if (temp > data->temp_offset)
 		temp -= data->temp_offset;
 	else
@@ -160,6 +172,23 @@ static ssize_t temp1_input_show(struct device *dev,
 	return sprintf(buf, "%u\n", temp);
 }
 
+static ssize_t temp2_input_show(struct device *dev,
+				struct device_attribute *devattr, char *buf)
+{
+	struct k10temp_data *data = dev_get_drvdata(dev);
+	unsigned int temp = get_raw_temp(data);
+
+	return sprintf(buf, "%u\n", temp);
+}
+
+static ssize_t temp_label_show(struct device *dev,
+			       struct device_attribute *devattr, char *buf)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+
+	return sprintf(buf, "%s\n", attr->index ? "Tctl" : "Tdie");
+}
+
 static ssize_t temp1_max_show(struct device *dev,
 			      struct device_attribute *attr, char *buf)
 {
@@ -187,16 +216,23 @@ static DEVICE_ATTR_RO(temp1_max);
 static SENSOR_DEVICE_ATTR(temp1_crit, S_IRUGO, show_temp_crit, NULL, 0);
 static SENSOR_DEVICE_ATTR(temp1_crit_hyst, S_IRUGO, show_temp_crit, NULL, 1);
 
+static SENSOR_DEVICE_ATTR(temp1_label, 0444, temp_label_show, NULL, 0);
+static DEVICE_ATTR_RO(temp2_input);
+static SENSOR_DEVICE_ATTR(temp2_label, 0444, temp_label_show, NULL, 1);
+
 static umode_t k10temp_is_visible(struct kobject *kobj,
 				  struct attribute *attr, int index)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
 	struct k10temp_data *data = dev_get_drvdata(dev);
 	struct pci_dev *pdev = data->pdev;
+	u32 reg;
 
-	if (index >= 2) {
-		u32 reg;
-
+	switch (index) {
+	case 0 ... 1:	/* temp1_input, temp1_max */
+	default:
+		break;
+	case 2 ... 3:	/* temp1_crit, temp1_crit_hyst */
 		if (!data->read_htcreg)
 			return 0;
 
@@ -208,6 +244,11 @@ static umode_t k10temp_is_visible(struct kobject *kobj,
 		data->read_htcreg(data->pdev, &reg);
 		if (!(reg & HTC_ENABLE))
 			return 0;
+		break;
+	case 4 ... 6:	/* temp1_label, temp2_input, temp2_label */
+		if (!data->show_tdie)
+			return 0;
+		break;
 	}
 	return attr->mode;
 }
@@ -217,6 +258,9 @@ static struct attribute *k10temp_attrs[] = {
 	&dev_attr_temp1_max.attr,
 	&sensor_dev_attr_temp1_crit.dev_attr.attr,
 	&sensor_dev_attr_temp1_crit_hyst.dev_attr.attr,
+	&sensor_dev_attr_temp1_label.dev_attr.attr,
+	&dev_attr_temp2_input.attr,
+	&sensor_dev_attr_temp2_label.dev_attr.attr,
 	NULL
 };
 
@@ -292,6 +336,7 @@ static int k10temp_probe(struct pci_dev *pdev,
 	} else if (boot_cpu_data.x86 == 0x17) {
 		data->temp_adjust_mask = 0x80000;
 		data->read_tempreg = read_tempreg_nb_f17;
+		data->show_tdie = true;
 	} else {
 		data->read_htcreg = read_htcreg_pci;
 		data->read_tempreg = read_tempreg_pci;
@@ -320,6 +365,7 @@ static const struct pci_device_id k10temp_id_table[] = {
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_M10H_F3) },
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_M30H_NB_F3) },
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_M60H_NB_F3) },
+	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_M70H_NB_F3) },
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_16H_NB_F3) },
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_16H_M30H_NB_F3) },
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_DF_F3) },
diff --git a/drivers/hwmon/ltc2990.c b/drivers/hwmon/ltc2990.c
index 8f8fe059ab48..2aefdc58b242 100644
--- a/drivers/hwmon/ltc2990.c
+++ b/drivers/hwmon/ltc2990.c
@@ -5,18 +5,16 @@
  * Author: Mike Looijmans <mike.looijmans@topic.nl>
  *
  * License: GPLv2
- *
- * This driver assumes the chip is wired as a dual current monitor, and
- * reports the voltage drop across two series resistors. It also reports
- * the chip's internal temperature and Vcc power supply voltage.
  */
 
+#include <linux/bitops.h>
 #include <linux/err.h>
 #include <linux/hwmon.h>
 #include <linux/hwmon-sysfs.h>
 #include <linux/i2c.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/of.h>
 
 #define LTC2990_STATUS	0x00
 #define LTC2990_CONTROL	0x01
@@ -28,45 +26,108 @@
 #define LTC2990_V4_MSB	0x0C
 #define LTC2990_VCC_MSB	0x0E
 
-#define LTC2990_CONTROL_KELVIN		BIT(7)
-#define LTC2990_CONTROL_SINGLE		BIT(6)
-#define LTC2990_CONTROL_MEASURE_ALL	(0x3 << 3)
-#define LTC2990_CONTROL_MODE_CURRENT	0x06
-#define LTC2990_CONTROL_MODE_VOLTAGE	0x07
+#define LTC2990_IN0	BIT(0)
+#define LTC2990_IN1	BIT(1)
+#define LTC2990_IN2	BIT(2)
+#define LTC2990_IN3	BIT(3)
+#define LTC2990_IN4	BIT(4)
+#define LTC2990_CURR1	BIT(5)
+#define LTC2990_CURR2	BIT(6)
+#define LTC2990_TEMP1	BIT(7)
+#define LTC2990_TEMP2	BIT(8)
+#define LTC2990_TEMP3	BIT(9)
+#define LTC2990_NONE	0
+#define LTC2990_ALL	GENMASK(9, 0)
 
-/* convert raw register value to sign-extended integer in 16-bit range */
-static int ltc2990_voltage_to_int(int raw)
-{
-	if (raw & BIT(14))
-		return -(0x4000 - (raw & 0x3FFF)) << 2;
-	else
-		return (raw & 0x3FFF) << 2;
-}
+#define LTC2990_MODE0_SHIFT	0
+#define LTC2990_MODE0_MASK	GENMASK(2, 0)
+#define LTC2990_MODE1_SHIFT	3
+#define LTC2990_MODE1_MASK	GENMASK(1, 0)
+
+/* Enabled measurements for mode bits 2..0 */
+static const int ltc2990_attrs_ena_0[] = {
+	LTC2990_IN1 | LTC2990_IN2 | LTC2990_TEMP3,
+	LTC2990_CURR1 | LTC2990_TEMP3,
+	LTC2990_CURR1 | LTC2990_IN3 | LTC2990_IN4,
+	LTC2990_TEMP2 | LTC2990_IN3 | LTC2990_IN4,
+	LTC2990_TEMP2 | LTC2990_CURR2,
+	LTC2990_TEMP2 | LTC2990_TEMP3,
+	LTC2990_CURR1 | LTC2990_CURR2,
+	LTC2990_IN1 | LTC2990_IN2 | LTC2990_IN3 | LTC2990_IN4
+};
+
+/* Enabled measurements for mode bits 4..3 */
+static const int ltc2990_attrs_ena_1[] = {
+	LTC2990_NONE,
+	LTC2990_TEMP2 | LTC2990_IN1 | LTC2990_CURR1,
+	LTC2990_TEMP3 | LTC2990_IN3 | LTC2990_CURR2,
+	LTC2990_ALL
+};
+
+struct ltc2990_data {
+	struct i2c_client *i2c;
+	u32 mode[2];
+};
 
 /* Return the converted value from the given register in uV or mC */
-static int ltc2990_get_value(struct i2c_client *i2c, u8 reg, int *result)
+static int ltc2990_get_value(struct i2c_client *i2c, int index, int *result)
 {
 	int val;
+	u8 reg;
+
+	switch (index) {
+	case LTC2990_IN0:
+		reg = LTC2990_VCC_MSB;
+		break;
+	case LTC2990_IN1:
+	case LTC2990_CURR1:
+	case LTC2990_TEMP2:
+		reg = LTC2990_V1_MSB;
+		break;
+	case LTC2990_IN2:
+		reg = LTC2990_V2_MSB;
+		break;
+	case LTC2990_IN3:
+	case LTC2990_CURR2:
+	case LTC2990_TEMP3:
+		reg = LTC2990_V3_MSB;
+		break;
+	case LTC2990_IN4:
+		reg = LTC2990_V4_MSB;
+		break;
+	case LTC2990_TEMP1:
+		reg = LTC2990_TINT_MSB;
+		break;
+	default:
+		return -EINVAL;
+	}
 
 	val = i2c_smbus_read_word_swapped(i2c, reg);
 	if (unlikely(val < 0))
 		return val;
 
-	switch (reg) {
-	case LTC2990_TINT_MSB:
-		/* internal temp, 0.0625 degrees/LSB, 13-bit  */
-		val = (val & 0x1FFF) << 3;
-		*result = (val * 1000) >> 7;
+	switch (index) {
+	case LTC2990_TEMP1:
+	case LTC2990_TEMP2:
+	case LTC2990_TEMP3:
+		/* temp, 0.0625 degrees/LSB */
+		*result = sign_extend32(val, 12) * 1000 / 16;
 		break;
-	case LTC2990_V1_MSB:
-	case LTC2990_V3_MSB:
-		 /* Vx-Vy, 19.42uV/LSB. Depends on mode. */
-		*result = ltc2990_voltage_to_int(val) * 1942 / (4 * 100);
+	case LTC2990_CURR1:
+	case LTC2990_CURR2:
+		 /* Vx-Vy, 19.42uV/LSB */
+		*result = sign_extend32(val, 14) * 1942 / 100;
 		break;
-	case LTC2990_VCC_MSB:
-		/* Vcc, 305.18μV/LSB, 2.5V offset */
-		*result = (ltc2990_voltage_to_int(val) * 30518 /
-			   (4 * 100 * 1000)) + 2500;
+	case LTC2990_IN0:
+		/* Vcc, 305.18uV/LSB, 2.5V offset */
+		*result = sign_extend32(val, 14) * 30518 / (100 * 1000) + 2500;
+		break;
+	case LTC2990_IN1:
+	case LTC2990_IN2:
+	case LTC2990_IN3:
+	case LTC2990_IN4:
+		/* Vx, 305.18uV/LSB */
+		*result = sign_extend32(val, 14) * 30518 / (100 * 1000);
 		break;
 	default:
 		return -EINVAL; /* won't happen, keep compiler happy */
@@ -79,48 +140,117 @@ static ssize_t ltc2990_show_value(struct device *dev,
 				  struct device_attribute *da, char *buf)
 {
 	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+	struct ltc2990_data *data = dev_get_drvdata(dev);
 	int value;
 	int ret;
 
-	ret = ltc2990_get_value(dev_get_drvdata(dev), attr->index, &value);
+	ret = ltc2990_get_value(data->i2c, attr->index, &value);
 	if (unlikely(ret < 0))
 		return ret;
 
 	return snprintf(buf, PAGE_SIZE, "%d\n", value);
 }
 
+static umode_t ltc2990_attrs_visible(struct kobject *kobj,
+				     struct attribute *a, int n)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct ltc2990_data *data = dev_get_drvdata(dev);
+	struct device_attribute *da =
+			container_of(a, struct device_attribute, attr);
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+
+	int attrs_mask = LTC2990_IN0 | LTC2990_TEMP1 |
+			 (ltc2990_attrs_ena_0[data->mode[0]] &
+			  ltc2990_attrs_ena_1[data->mode[1]]);
+
+	if (attr->index & attrs_mask)
+		return a->mode;
+
+	return 0;
+}
+
 static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, ltc2990_show_value, NULL,
-			  LTC2990_TINT_MSB);
+			  LTC2990_TEMP1);
+static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, ltc2990_show_value, NULL,
+			  LTC2990_TEMP2);
+static SENSOR_DEVICE_ATTR(temp3_input, S_IRUGO, ltc2990_show_value, NULL,
+			  LTC2990_TEMP3);
 static SENSOR_DEVICE_ATTR(curr1_input, S_IRUGO, ltc2990_show_value, NULL,
-			  LTC2990_V1_MSB);
+			  LTC2990_CURR1);
 static SENSOR_DEVICE_ATTR(curr2_input, S_IRUGO, ltc2990_show_value, NULL,
-			  LTC2990_V3_MSB);
+			  LTC2990_CURR2);
 static SENSOR_DEVICE_ATTR(in0_input, S_IRUGO, ltc2990_show_value, NULL,
-			  LTC2990_VCC_MSB);
+			  LTC2990_IN0);
+static SENSOR_DEVICE_ATTR(in1_input, S_IRUGO, ltc2990_show_value, NULL,
+			  LTC2990_IN1);
+static SENSOR_DEVICE_ATTR(in2_input, S_IRUGO, ltc2990_show_value, NULL,
+			  LTC2990_IN2);
+static SENSOR_DEVICE_ATTR(in3_input, S_IRUGO, ltc2990_show_value, NULL,
+			  LTC2990_IN3);
+static SENSOR_DEVICE_ATTR(in4_input, S_IRUGO, ltc2990_show_value, NULL,
+			  LTC2990_IN4);
 
 static struct attribute *ltc2990_attrs[] = {
 	&sensor_dev_attr_temp1_input.dev_attr.attr,
+	&sensor_dev_attr_temp2_input.dev_attr.attr,
+	&sensor_dev_attr_temp3_input.dev_attr.attr,
 	&sensor_dev_attr_curr1_input.dev_attr.attr,
 	&sensor_dev_attr_curr2_input.dev_attr.attr,
 	&sensor_dev_attr_in0_input.dev_attr.attr,
+	&sensor_dev_attr_in1_input.dev_attr.attr,
+	&sensor_dev_attr_in2_input.dev_attr.attr,
+	&sensor_dev_attr_in3_input.dev_attr.attr,
+	&sensor_dev_attr_in4_input.dev_attr.attr,
 	NULL,
 };
-ATTRIBUTE_GROUPS(ltc2990);
+
+static const struct attribute_group ltc2990_group = {
+	.attrs = ltc2990_attrs,
+	.is_visible = ltc2990_attrs_visible,
+};
+__ATTRIBUTE_GROUPS(ltc2990);
 
 static int ltc2990_i2c_probe(struct i2c_client *i2c,
 			     const struct i2c_device_id *id)
 {
 	int ret;
 	struct device *hwmon_dev;
+	struct ltc2990_data *data;
+	struct device_node *of_node = i2c->dev.of_node;
 
 	if (!i2c_check_functionality(i2c->adapter, I2C_FUNC_SMBUS_BYTE_DATA |
 				     I2C_FUNC_SMBUS_WORD_DATA))
 		return -ENODEV;
 
-	/* Setup continuous mode, current monitor */
+	data = devm_kzalloc(&i2c->dev, sizeof(struct ltc2990_data), GFP_KERNEL);
+	if (unlikely(!data))
+		return -ENOMEM;
+
+	data->i2c = i2c;
+
+	if (of_node) {
+		ret = of_property_read_u32_array(of_node, "lltc,meas-mode",
+						 data->mode, 2);
+		if (ret < 0)
+			return ret;
+
+		if (data->mode[0] & ~LTC2990_MODE0_MASK ||
+		    data->mode[1] & ~LTC2990_MODE1_MASK)
+			return -EINVAL;
+	} else {
+		ret = i2c_smbus_read_byte_data(i2c, LTC2990_CONTROL);
+		if (ret < 0)
+			return ret;
+
+		data->mode[0] = ret >> LTC2990_MODE0_SHIFT & LTC2990_MODE0_MASK;
+		data->mode[1] = ret >> LTC2990_MODE1_SHIFT & LTC2990_MODE1_MASK;
+	}
+
+	/* Setup continuous mode */
 	ret = i2c_smbus_write_byte_data(i2c, LTC2990_CONTROL,
-					LTC2990_CONTROL_MEASURE_ALL |
-					LTC2990_CONTROL_MODE_CURRENT);
+					data->mode[0] << LTC2990_MODE0_SHIFT |
+					data->mode[1] << LTC2990_MODE1_SHIFT);
 	if (ret < 0) {
 		dev_err(&i2c->dev, "Error: Failed to set control mode.\n");
 		return ret;
@@ -134,7 +264,7 @@ static int ltc2990_i2c_probe(struct i2c_client *i2c,
 
 	hwmon_dev = devm_hwmon_device_register_with_groups(&i2c->dev,
 							   i2c->name,
-							   i2c,
+							   data,
 							   ltc2990_groups);
 
 	return PTR_ERR_OR_ZERO(hwmon_dev);
diff --git a/drivers/hwmon/mc13783-adc.c b/drivers/hwmon/mc13783-adc.c
index 960a1db6f269..67860ad2e3d9 100644
--- a/drivers/hwmon/mc13783-adc.c
+++ b/drivers/hwmon/mc13783-adc.c
@@ -63,6 +63,10 @@ static int mc13783_adc_read(struct device *dev,
 	if (ret)
 		return ret;
 
+	/* ADIN7 subchannels */
+	if (channel >= 16)
+		channel = 7;
+
 	channel &= 0x7;
 
 	*val = (sample[channel % 4] >> (channel > 3 ? 14 : 2)) & 0x3ff;
@@ -111,6 +115,57 @@ static ssize_t mc13783_adc_read_gp(struct device *dev,
 	return sprintf(buf, "%u\n", val);
 }
 
+static ssize_t mc13783_adc_read_uid(struct device *dev,
+		struct device_attribute *devattr, char *buf)
+{
+	unsigned int val;
+	struct platform_device *pdev = to_platform_device(dev);
+	kernel_ulong_t driver_data = platform_get_device_id(pdev)->driver_data;
+	int ret = mc13783_adc_read(dev, devattr, &val);
+
+	if (ret)
+		return ret;
+
+	if (driver_data & MC13783_ADC_BPDIV2)
+		/* MC13892 have 1/2 divider, input range is [0, 4.800V] */
+		val = DIV_ROUND_CLOSEST(val * 4800, 1024);
+	else
+		/* MC13783 have 0.9 divider, input range is [0, 2.555V] */
+		val = DIV_ROUND_CLOSEST(val * 2555, 1024);
+
+	return sprintf(buf, "%u\n", val);
+}
+
+static ssize_t mc13783_adc_read_temp(struct device *dev,
+		struct device_attribute *devattr, char *buf)
+{
+	unsigned int val;
+	struct platform_device *pdev = to_platform_device(dev);
+	kernel_ulong_t driver_data = platform_get_device_id(pdev)->driver_data;
+	int ret = mc13783_adc_read(dev, devattr, &val);
+
+	if (ret)
+		return ret;
+
+	if (driver_data & MC13783_ADC_BPDIV2) {
+		/*
+		 * MC13892:
+		 * Die Temperature Read Out Code at 25C 680
+		 * Temperature change per LSB +0.4244C
+		 */
+		ret = DIV_ROUND_CLOSEST(-2635920 + val * 4244, 10);
+	} else {
+		/*
+		 * MC13783:
+		 * Die Temperature Read Out Code at 25C 282
+		 * Temperature change per LSB -1.14C
+		 */
+		ret = 346480 - 1140 * val;
+	}
+
+	return sprintf(buf, "%d\n", ret);
+}
+
 static DEVICE_ATTR_RO(name);
 static SENSOR_DEVICE_ATTR(in2_input, S_IRUGO, mc13783_adc_read_bp, NULL, 2);
 static SENSOR_DEVICE_ATTR(in5_input, S_IRUGO, mc13783_adc_read_gp, NULL, 5);
@@ -124,6 +179,9 @@ static SENSOR_DEVICE_ATTR(in12_input, S_IRUGO, mc13783_adc_read_gp, NULL, 12);
 static SENSOR_DEVICE_ATTR(in13_input, S_IRUGO, mc13783_adc_read_gp, NULL, 13);
 static SENSOR_DEVICE_ATTR(in14_input, S_IRUGO, mc13783_adc_read_gp, NULL, 14);
 static SENSOR_DEVICE_ATTR(in15_input, S_IRUGO, mc13783_adc_read_gp, NULL, 15);
+static SENSOR_DEVICE_ATTR(in16_input, S_IRUGO, mc13783_adc_read_uid, NULL, 16);
+static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO,
+			  mc13783_adc_read_temp, NULL, 17);
 
 static struct attribute *mc13783_attr_base[] = {
 	&dev_attr_name.attr,
@@ -131,6 +189,8 @@ static struct attribute *mc13783_attr_base[] = {
 	&sensor_dev_attr_in5_input.dev_attr.attr,
 	&sensor_dev_attr_in6_input.dev_attr.attr,
 	&sensor_dev_attr_in7_input.dev_attr.attr,
+	&sensor_dev_attr_in16_input.dev_attr.attr,
+	&sensor_dev_attr_temp1_input.dev_attr.attr,
 	NULL
 };
 
diff --git a/drivers/hwspinlock/Kconfig b/drivers/hwspinlock/Kconfig
index f0f467983960..e895d29500ee 100644
--- a/drivers/hwspinlock/Kconfig
+++ b/drivers/hwspinlock/Kconfig
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Generic HWSPINLOCK framework
 #
diff --git a/drivers/hwspinlock/hwspinlock_core.c b/drivers/hwspinlock/hwspinlock_core.c
index 4074441444fe..d16e6a3d38e8 100644
--- a/drivers/hwspinlock/hwspinlock_core.c
+++ b/drivers/hwspinlock/hwspinlock_core.c
@@ -1,18 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardware spinlock framework
  *
  * Copyright (C) 2010 Texas Instruments Incorporated - http://www.ti.com
  *
  * Contact: Ohad Ben-Cohen <ohad@wizery.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published
- * by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #define pr_fmt(fmt)    "%s: " fmt, __func__
@@ -71,10 +63,16 @@ static DEFINE_MUTEX(hwspinlock_tree_lock);
  * This function attempts to lock an hwspinlock, and will immediately
  * fail if the hwspinlock is already taken.
  *
- * Upon a successful return from this function, preemption (and possibly
- * interrupts) is disabled, so the caller must not sleep, and is advised to
- * release the hwspinlock as soon as possible. This is required in order to
- * minimize remote cores polling on the hardware interconnect.
+ * Caution: If the mode is HWLOCK_RAW, that means user must protect the routine
+ * of getting hardware lock with mutex or spinlock. Since in some scenarios,
+ * user need some time-consuming or sleepable operations under the hardware
+ * lock, they need one sleepable lock (like mutex) to protect the operations.
+ *
+ * If the mode is not HWLOCK_RAW, upon a successful return from this function,
+ * preemption (and possibly interrupts) is disabled, so the caller must not
+ * sleep, and is advised to release the hwspinlock as soon as possible. This is
+ * required in order to minimize remote cores polling on the hardware
+ * interconnect.
  *
  * The user decides whether local interrupts are disabled or not, and if yes,
  * whether he wants their previous state to be saved. It is up to the user
@@ -106,12 +104,20 @@ int __hwspin_trylock(struct hwspinlock *hwlock, int mode, unsigned long *flags)
 	 *    problems with hwspinlock usage (e.g. scheduler checks like
 	 *    'scheduling while atomic' etc.)
 	 */
-	if (mode == HWLOCK_IRQSTATE)
+	switch (mode) {
+	case HWLOCK_IRQSTATE:
 		ret = spin_trylock_irqsave(&hwlock->lock, *flags);
-	else if (mode == HWLOCK_IRQ)
+		break;
+	case HWLOCK_IRQ:
 		ret = spin_trylock_irq(&hwlock->lock);
-	else
+		break;
+	case HWLOCK_RAW:
+		ret = 1;
+		break;
+	default:
 		ret = spin_trylock(&hwlock->lock);
+		break;
+	}
 
 	/* is lock already taken by another context on the local cpu ? */
 	if (!ret)
@@ -122,12 +128,20 @@ int __hwspin_trylock(struct hwspinlock *hwlock, int mode, unsigned long *flags)
 
 	/* if hwlock is already taken, undo spin_trylock_* and exit */
 	if (!ret) {
-		if (mode == HWLOCK_IRQSTATE)
+		switch (mode) {
+		case HWLOCK_IRQSTATE:
 			spin_unlock_irqrestore(&hwlock->lock, *flags);
-		else if (mode == HWLOCK_IRQ)
+			break;
+		case HWLOCK_IRQ:
 			spin_unlock_irq(&hwlock->lock);
-		else
+			break;
+		case HWLOCK_RAW:
+			/* Nothing to do */
+			break;
+		default:
 			spin_unlock(&hwlock->lock);
+			break;
+		}
 
 		return -EBUSY;
 	}
@@ -160,9 +174,14 @@ EXPORT_SYMBOL_GPL(__hwspin_trylock);
  * is already taken, the function will busy loop waiting for it to
  * be released, but give up after @timeout msecs have elapsed.
  *
- * Upon a successful return from this function, preemption is disabled
- * (and possibly local interrupts, too), so the caller must not sleep,
- * and is advised to release the hwspinlock as soon as possible.
+ * Caution: If the mode is HWLOCK_RAW, that means user must protect the routine
+ * of getting hardware lock with mutex or spinlock. Since in some scenarios,
+ * user need some time-consuming or sleepable operations under the hardware
+ * lock, they need one sleepable lock (like mutex) to protect the operations.
+ *
+ * If the mode is not HWLOCK_RAW, upon a successful return from this function,
+ * preemption is disabled (and possibly local interrupts, too), so the caller
+ * must not sleep, and is advised to release the hwspinlock as soon as possible.
  * This is required in order to minimize remote cores polling on the
  * hardware interconnect.
  *
@@ -249,12 +268,20 @@ void __hwspin_unlock(struct hwspinlock *hwlock, int mode, unsigned long *flags)
 	hwlock->bank->ops->unlock(hwlock);
 
 	/* Undo the spin_trylock{_irq, _irqsave} called while locking */
-	if (mode == HWLOCK_IRQSTATE)
+	switch (mode) {
+	case HWLOCK_IRQSTATE:
 		spin_unlock_irqrestore(&hwlock->lock, *flags);
-	else if (mode == HWLOCK_IRQ)
+		break;
+	case HWLOCK_IRQ:
 		spin_unlock_irq(&hwlock->lock);
-	else
+		break;
+	case HWLOCK_RAW:
+		/* Nothing to do */
+		break;
+	default:
 		spin_unlock(&hwlock->lock);
+		break;
+	}
 }
 EXPORT_SYMBOL_GPL(__hwspin_unlock);
 
diff --git a/drivers/hwspinlock/hwspinlock_internal.h b/drivers/hwspinlock/hwspinlock_internal.h
index d26f78b8f214..9eb6bd020dc7 100644
--- a/drivers/hwspinlock/hwspinlock_internal.h
+++ b/drivers/hwspinlock/hwspinlock_internal.h
@@ -1,18 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Hardware spinlocks internal header
  *
  * Copyright (C) 2010 Texas Instruments Incorporated - http://www.ti.com
  *
  * Contact: Ohad Ben-Cohen <ohad@wizery.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published
- * by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #ifndef __HWSPINLOCK_HWSPINLOCK_H
diff --git a/drivers/hwspinlock/omap_hwspinlock.c b/drivers/hwspinlock/omap_hwspinlock.c
index ad2f8cac8487..79a4030e89d8 100644
--- a/drivers/hwspinlock/omap_hwspinlock.c
+++ b/drivers/hwspinlock/omap_hwspinlock.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * OMAP hardware spinlock driver
  *
@@ -6,15 +7,6 @@
  * Contact: Simon Que <sque@ti.com>
  *          Hari Kanigeri <h-kanigeri2@ti.com>
  *          Ohad Ben-Cohen <ohad@wizery.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/hwspinlock/qcom_hwspinlock.c b/drivers/hwspinlock/qcom_hwspinlock.c
index fa6880b8060a..6da7447d277d 100644
--- a/drivers/hwspinlock/qcom_hwspinlock.c
+++ b/drivers/hwspinlock/qcom_hwspinlock.c
@@ -1,15 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2013, The Linux Foundation. All rights reserved.
  * Copyright (c) 2015, Sony Mobile Communications AB
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/hwspinlock.h>
diff --git a/drivers/hwspinlock/sirf_hwspinlock.c b/drivers/hwspinlock/sirf_hwspinlock.c
index 16018544d431..02ef6e543cd1 100644
--- a/drivers/hwspinlock/sirf_hwspinlock.c
+++ b/drivers/hwspinlock/sirf_hwspinlock.c
@@ -1,9 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SIRF hardware spinlock driver
  *
  * Copyright (c) 2015 Cambridge Silicon Radio Limited, a CSR plc group company.
- *
- * Licensed under GPLv2.
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/hwspinlock/sprd_hwspinlock.c b/drivers/hwspinlock/sprd_hwspinlock.c
index 638e64ac18f5..dc42bf51f3e6 100644
--- a/drivers/hwspinlock/sprd_hwspinlock.c
+++ b/drivers/hwspinlock/sprd_hwspinlock.c
@@ -1,15 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Spreadtrum hardware spinlock driver
  * Copyright (C) 2017 Spreadtrum  - http://www.spreadtrum.com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
  */
 
 #include <linux/bitops.h>
diff --git a/drivers/hwspinlock/u8500_hsem.c b/drivers/hwspinlock/u8500_hsem.c
index e93eabbd660f..84a92d424e8d 100644
--- a/drivers/hwspinlock/u8500_hsem.c
+++ b/drivers/hwspinlock/u8500_hsem.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * u8500 HWSEM driver
  *
@@ -10,15 +11,6 @@
  *   Simon Que <sque@ti.com>
  *   Hari Kanigeri <h-kanigeri2@ti.com>
  *   Ohad Ben-Cohen <ohad@wizery.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
  */
 
 #include <linux/module.h>
diff --git a/drivers/hwtracing/coresight/coresight-cpu-debug.c b/drivers/hwtracing/coresight/coresight-cpu-debug.c
index 9cdb3fbc8c1f..45b2460f3166 100644
--- a/drivers/hwtracing/coresight/coresight-cpu-debug.c
+++ b/drivers/hwtracing/coresight/coresight-cpu-debug.c
@@ -1,20 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2017 Linaro Limited. All rights reserved.
  *
  * Author: Leo Yan <leo.yan@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
- *
  */
 #include <linux/amba/bus.h>
 #include <linux/coresight.h>
@@ -315,7 +303,7 @@ static void debug_dump_regs(struct debug_drvdata *drvdata)
 	}
 
 	pc = debug_adjust_pc(drvdata);
-	dev_emerg(dev, " EDPCSR:  [<%px>] %pS\n", (void *)pc, (void *)pc);
+	dev_emerg(dev, " EDPCSR:  %pS\n", (void *)pc);
 
 	if (drvdata->edcidsr_present)
 		dev_emerg(dev, " EDCIDSR: %08x\n", drvdata->edcidsr);
diff --git a/drivers/hwtracing/coresight/coresight-dynamic-replicator.c b/drivers/hwtracing/coresight/coresight-dynamic-replicator.c
index 043da86b0fe9..f6d0571ab9dd 100644
--- a/drivers/hwtracing/coresight/coresight-dynamic-replicator.c
+++ b/drivers/hwtracing/coresight/coresight-dynamic-replicator.c
@@ -1,14 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2011-2015, The Linux Foundation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/amba/bus.h>
diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c
index 580cd381adf3..9b6c55523c58 100644
--- a/drivers/hwtracing/coresight/coresight-etb10.c
+++ b/drivers/hwtracing/coresight/coresight-etb10.c
@@ -1,15 +1,8 @@
-/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
  *
  * Description: CoreSight Embedded Trace Buffer driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <asm/local.h>
diff --git a/drivers/hwtracing/coresight/coresight-etm-cp14.c b/drivers/hwtracing/coresight/coresight-etm-cp14.c
index 12a220682117..4174a8d355d2 100644
--- a/drivers/hwtracing/coresight/coresight-etm-cp14.c
+++ b/drivers/hwtracing/coresight/coresight-etm-cp14.c
@@ -1,13 +1,6 @@
-/* Copyright (c) 2012, The Linux Foundation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2012, The Linux Foundation. All rights reserved.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c
index 4e5ed6597f2f..677695635211 100644
--- a/drivers/hwtracing/coresight/coresight-etm-perf.c
+++ b/drivers/hwtracing/coresight/coresight-etm-perf.c
@@ -1,18 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright(C) 2015 Linaro Limited. All rights reserved.
  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <linux/coresight.h>
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.h b/drivers/hwtracing/coresight/coresight-etm-perf.h
index 3ffc9feb2d64..4197df4faf5e 100644
--- a/drivers/hwtracing/coresight/coresight-etm-perf.h
+++ b/drivers/hwtracing/coresight/coresight-etm-perf.h
@@ -1,18 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright(C) 2015 Linaro Limited. All rights reserved.
  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #ifndef _CORESIGHT_ETM_PERF_H
diff --git a/drivers/hwtracing/coresight/coresight-etm.h b/drivers/hwtracing/coresight/coresight-etm.h
index 70b0a248c321..79e1ad860d8a 100644
--- a/drivers/hwtracing/coresight/coresight-etm.h
+++ b/drivers/hwtracing/coresight/coresight-etm.h
@@ -1,13 +1,6 @@
-/* Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
  */
 
 #ifndef _CORESIGHT_CORESIGHT_ETM_H
@@ -175,8 +168,6 @@
  * @seq_curr_state: current value of the sequencer register.
  * @ctxid_idx: index for the context ID registers.
  * @ctxid_pid: value for the context ID to trigger on.
- * @ctxid_vpid:	Virtual PID seen by users if PID namespace is enabled, otherwise
- *		the same value of ctxid_pid.
  * @ctxid_mask: mask applicable to all the context IDs.
  * @sync_freq:	Synchronisation frequency.
  * @timestamp_event: Defines an event that requests the insertion
@@ -209,7 +200,6 @@ struct etm_config {
 	u32				seq_curr_state;
 	u8				ctxid_idx;
 	u32				ctxid_pid[ETM_MAX_CTXID_CMP];
-	u32				ctxid_vpid[ETM_MAX_CTXID_CMP];
 	u32				ctxid_mask;
 	u32				sync_freq;
 	u32				timestamp_event;
diff --git a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
index 6e547ec6fead..75487b3fad86 100644
--- a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
@@ -1,20 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright(C) 2015 Linaro Limited. All rights reserved.
  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/pid_namespace.h>
 #include <linux/pm_runtime.h>
 #include <linux/sysfs.h>
 #include "coresight-etm.h"
@@ -1036,8 +1026,15 @@ static ssize_t ctxid_pid_show(struct device *dev,
 	struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	struct etm_config *config = &drvdata->config;
 
+	/*
+	 * Don't use contextID tracing if coming from a PID namespace.  See
+	 * comment in ctxid_pid_store().
+	 */
+	if (task_active_pid_ns(current) != &init_pid_ns)
+		return -EINVAL;
+
 	spin_lock(&drvdata->spinlock);
-	val = config->ctxid_vpid[config->ctxid_idx];
+	val = config->ctxid_pid[config->ctxid_idx];
 	spin_unlock(&drvdata->spinlock);
 
 	return sprintf(buf, "%#lx\n", val);
@@ -1048,19 +1045,28 @@ static ssize_t ctxid_pid_store(struct device *dev,
 			       const char *buf, size_t size)
 {
 	int ret;
-	unsigned long vpid, pid;
+	unsigned long pid;
 	struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	struct etm_config *config = &drvdata->config;
 
-	ret = kstrtoul(buf, 16, &vpid);
+	/*
+	 * When contextID tracing is enabled the tracers will insert the
+	 * value found in the contextID register in the trace stream.  But if
+	 * a process is in a namespace the PID of that process as seen from the
+	 * namespace won't be what the kernel sees, something that makes the
+	 * feature confusing and can potentially leak kernel only information.
+	 * As such refuse to use the feature if @current is not in the initial
+	 * PID namespace.
+	 */
+	if (task_active_pid_ns(current) != &init_pid_ns)
+		return -EINVAL;
+
+	ret = kstrtoul(buf, 16, &pid);
 	if (ret)
 		return ret;
 
-	pid = coresight_vpid_to_pid(vpid);
-
 	spin_lock(&drvdata->spinlock);
 	config->ctxid_pid[config->ctxid_idx] = pid;
-	config->ctxid_vpid[config->ctxid_idx] = vpid;
 	spin_unlock(&drvdata->spinlock);
 
 	return size;
@@ -1074,6 +1080,13 @@ static ssize_t ctxid_mask_show(struct device *dev,
 	struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	struct etm_config *config = &drvdata->config;
 
+	/*
+	 * Don't use contextID tracing if coming from a PID namespace.  See
+	 * comment in ctxid_pid_store().
+	 */
+	if (task_active_pid_ns(current) != &init_pid_ns)
+		return -EINVAL;
+
 	val = config->ctxid_mask;
 	return sprintf(buf, "%#lx\n", val);
 }
@@ -1087,6 +1100,13 @@ static ssize_t ctxid_mask_store(struct device *dev,
 	struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	struct etm_config *config = &drvdata->config;
 
+	/*
+	 * Don't use contextID tracing if coming from a PID namespace.  See
+	 * comment in ctxid_pid_store().
+	 */
+	if (task_active_pid_ns(current) != &init_pid_ns)
+		return -EINVAL;
+
 	ret = kstrtoul(buf, 16, &val);
 	if (ret)
 		return ret;
diff --git a/drivers/hwtracing/coresight/coresight-etm3x.c b/drivers/hwtracing/coresight/coresight-etm3x.c
index 39f42fdd503d..7c74263c333d 100644
--- a/drivers/hwtracing/coresight/coresight-etm3x.c
+++ b/drivers/hwtracing/coresight/coresight-etm3x.c
@@ -1,15 +1,8 @@
-/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
  *
  * Description: CoreSight Program Flow Trace driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/kernel.h>
@@ -237,10 +230,8 @@ void etm_set_default(struct etm_config *config)
 
 	config->seq_curr_state = 0x0;
 	config->ctxid_idx = 0x0;
-	for (i = 0; i < ETM_MAX_CTXID_CMP; i++) {
+	for (i = 0; i < ETM_MAX_CTXID_CMP; i++)
 		config->ctxid_pid[i] = 0x0;
-		config->ctxid_vpid[i] = 0x0;
-	}
 
 	config->ctxid_mask = 0x0;
 	/* Setting default to 1024 as per TRM recommendation */
diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
index d21961710713..a0365e23678e 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
@@ -1,20 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright(C) 2015 Linaro Limited. All rights reserved.
  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/pid_namespace.h>
 #include <linux/pm_runtime.h>
 #include <linux/sysfs.h>
 #include "coresight-etm4x.h"
@@ -261,10 +251,8 @@ static ssize_t reset_store(struct device *dev,
 	}
 
 	config->ctxid_idx = 0x0;
-	for (i = 0; i < drvdata->numcidc; i++) {
+	for (i = 0; i < drvdata->numcidc; i++)
 		config->ctxid_pid[i] = 0x0;
-		config->ctxid_vpid[i] = 0x0;
-	}
 
 	config->ctxid_mask0 = 0x0;
 	config->ctxid_mask1 = 0x0;
@@ -1648,9 +1636,16 @@ static ssize_t ctxid_pid_show(struct device *dev,
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	struct etmv4_config *config = &drvdata->config;
 
+	/*
+	 * Don't use contextID tracing if coming from a PID namespace.  See
+	 * comment in ctxid_pid_store().
+	 */
+	if (task_active_pid_ns(current) != &init_pid_ns)
+		return -EINVAL;
+
 	spin_lock(&drvdata->spinlock);
 	idx = config->ctxid_idx;
-	val = (unsigned long)config->ctxid_vpid[idx];
+	val = (unsigned long)config->ctxid_pid[idx];
 	spin_unlock(&drvdata->spinlock);
 	return scnprintf(buf, PAGE_SIZE, "%#lx\n", val);
 }
@@ -1660,10 +1655,22 @@ static ssize_t ctxid_pid_store(struct device *dev,
 			       const char *buf, size_t size)
 {
 	u8 idx;
-	unsigned long vpid, pid;
+	unsigned long pid;
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	struct etmv4_config *config = &drvdata->config;
 
+	/*
+	 * When contextID tracing is enabled the tracers will insert the
+	 * value found in the contextID register in the trace stream.  But if
+	 * a process is in a namespace the PID of that process as seen from the
+	 * namespace won't be what the kernel sees, something that makes the
+	 * feature confusing and can potentially leak kernel only information.
+	 * As such refuse to use the feature if @current is not in the initial
+	 * PID namespace.
+	 */
+	if (task_active_pid_ns(current) != &init_pid_ns)
+		return -EINVAL;
+
 	/*
 	 * only implemented when ctxid tracing is enabled, i.e. at least one
 	 * ctxid comparator is implemented and ctxid is greater than 0 bits
@@ -1671,15 +1678,12 @@ static ssize_t ctxid_pid_store(struct device *dev,
 	 */
 	if (!drvdata->ctxid_size || !drvdata->numcidc)
 		return -EINVAL;
-	if (kstrtoul(buf, 16, &vpid))
+	if (kstrtoul(buf, 16, &pid))
 		return -EINVAL;
 
-	pid = coresight_vpid_to_pid(vpid);
-
 	spin_lock(&drvdata->spinlock);
 	idx = config->ctxid_idx;
 	config->ctxid_pid[idx] = (u64)pid;
-	config->ctxid_vpid[idx] = (u64)vpid;
 	spin_unlock(&drvdata->spinlock);
 	return size;
 }
@@ -1693,6 +1697,13 @@ static ssize_t ctxid_masks_show(struct device *dev,
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	struct etmv4_config *config = &drvdata->config;
 
+	/*
+	 * Don't use contextID tracing if coming from a PID namespace.  See
+	 * comment in ctxid_pid_store().
+	 */
+	if (task_active_pid_ns(current) != &init_pid_ns)
+		return -EINVAL;
+
 	spin_lock(&drvdata->spinlock);
 	val1 = config->ctxid_mask0;
 	val2 = config->ctxid_mask1;
@@ -1709,6 +1720,13 @@ static ssize_t ctxid_masks_store(struct device *dev,
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	struct etmv4_config *config = &drvdata->config;
 
+	/*
+	 * Don't use contextID tracing if coming from a PID namespace.  See
+	 * comment in ctxid_pid_store().
+	 */
+	if (task_active_pid_ns(current) != &init_pid_ns)
+		return -EINVAL;
+
 	/*
 	 * only implemented when ctxid tracing is enabled, i.e. at least one
 	 * ctxid comparator is implemented and ctxid is greater than 0 bits
diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
index cf364a514c12..9bc04c50d45b 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x.c
@@ -1,13 +1,6 @@
-/* Copyright (c) 2014, The Linux Foundation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2014, The Linux Foundation. All rights reserved.
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h
index b3b5ea7b7fb3..52786e9d8926 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x.h
+++ b/drivers/hwtracing/coresight/coresight-etm4x.h
@@ -1,13 +1,6 @@
-/* Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
  */
 
 #ifndef _CORESIGHT_CORESIGHT_ETM_H
@@ -237,8 +230,6 @@
  * @addr_type:	Current status of the comparator register.
  * @ctxid_idx:	Context ID index selector.
  * @ctxid_pid:	Value of the context ID comparator.
- * @ctxid_vpid:	Virtual PID seen by users if PID namespace is enabled, otherwise
- *		the same value of ctxid_pid.
  * @ctxid_mask0:Context ID comparator mask for comparator 0-3.
  * @ctxid_mask1:Context ID comparator mask for comparator 4-7.
  * @vmid_idx:	VM ID index selector.
@@ -281,7 +272,6 @@ struct etmv4_config {
 	u8				addr_type[ETM_MAX_SINGLE_ADDR_CMP];
 	u8				ctxid_idx;
 	u64				ctxid_pid[ETMv4_MAX_CTXID_CMP];
-	u64				ctxid_vpid[ETMv4_MAX_CTXID_CMP];
 	u32				ctxid_mask0;
 	u32				ctxid_mask1;
 	u8				vmid_idx;
diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c
index 9f8ac0bef853..448145a36675 100644
--- a/drivers/hwtracing/coresight/coresight-funnel.c
+++ b/drivers/hwtracing/coresight/coresight-funnel.c
@@ -1,15 +1,8 @@
-/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
  *
  * Description: CoreSight Funnel driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h
index f1d0e21d8cab..0e5a74dae6a6 100644
--- a/drivers/hwtracing/coresight/coresight-priv.h
+++ b/drivers/hwtracing/coresight/coresight-priv.h
@@ -1,13 +1,6 @@
-/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
  */
 
 #ifndef _CORESIGHT_PRIV_H
diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c
index 3756e71cb8f5..8d2eaaab6c2f 100644
--- a/drivers/hwtracing/coresight/coresight-replicator.c
+++ b/drivers/hwtracing/coresight/coresight-replicator.c
@@ -1,15 +1,8 @@
-/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
  *
  * Description: CoreSight Replicator driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c
index 15e7ef3891f5..c46c70aec1d5 100644
--- a/drivers/hwtracing/coresight/coresight-stm.c
+++ b/drivers/hwtracing/coresight/coresight-stm.c
@@ -1,16 +1,9 @@
-/* Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
  *
  * Description: CoreSight System Trace Macrocell driver
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
  * Initial implementation by Pratik Patel
  * (C) 2014-2015 Pratik Patel <pratikp@codeaurora.org>
  *
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c
index e2513b786242..61d849b11c26 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etf.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c
@@ -1,18 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright(C) 2016 Linaro Limited. All rights reserved.
  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <linux/circ_buf.h>
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c
index 68fbc8f7450e..02f747afa2ba 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etr.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c
@@ -1,18 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright(C) 2016 Linaro Limited. All rights reserved.
  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <linux/coresight.h>
@@ -124,10 +113,9 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
 	bool used = false;
 	unsigned long flags;
 	void __iomem *vaddr = NULL;
-	dma_addr_t paddr;
+	dma_addr_t paddr = 0;
 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
-
 	/*
 	 * If we don't have a buffer release the lock and allocate memory.
 	 * Otherwise keep the lock and move along.
@@ -164,11 +152,11 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
 		goto out;
 
 	/*
-	 * If drvdata::buf == NULL, use the memory allocated above.
+	 * If drvdata::vaddr == NULL, use the memory allocated above.
 	 * Otherwise a buffer still exists from a previous session, so
 	 * simply use that.
 	 */
-	if (drvdata->buf == NULL) {
+	if (drvdata->vaddr == NULL) {
 		used = true;
 		drvdata->vaddr = vaddr;
 		drvdata->paddr = paddr;
diff --git a/drivers/hwtracing/coresight/coresight-tmc.c b/drivers/hwtracing/coresight/coresight-tmc.c
index 0ea04f588de0..456f122df74f 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.c
+++ b/drivers/hwtracing/coresight/coresight-tmc.c
@@ -1,15 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /* Copyright (c) 2012, The Linux Foundation. All rights reserved.
  *
  * Description: CoreSight Trace Memory Controller driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h
index 8df7a813f537..dfaff077a7fc 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.h
+++ b/drivers/hwtracing/coresight/coresight-tmc.h
@@ -1,18 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright(C) 2015 Linaro Limited. All rights reserved.
  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #ifndef _CORESIGHT_TMC_H
diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c
index 805f7c2210fe..01b7457fe8fc 100644
--- a/drivers/hwtracing/coresight/coresight-tpiu.c
+++ b/drivers/hwtracing/coresight/coresight-tpiu.c
@@ -1,15 +1,8 @@
-/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved.
  *
  * Description: CoreSight Trace Port Interface Unit driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/hwtracing/coresight/coresight.c b/drivers/hwtracing/coresight/coresight.c
index 389c4baeca9d..29e834aab539 100644
--- a/drivers/hwtracing/coresight/coresight.c
+++ b/drivers/hwtracing/coresight/coresight.c
@@ -1,13 +1,6 @@
-/* Copyright (c) 2012, The Linux Foundation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2012, The Linux Foundation. All rights reserved.
  */
 
 #include <linux/kernel.h>
@@ -1026,8 +1019,10 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
 	dev_set_name(&csdev->dev, "%s", desc->pdata->name);
 
 	ret = device_register(&csdev->dev);
-	if (ret)
-		goto err_device_register;
+	if (ret) {
+		put_device(&csdev->dev);
+		goto err_kzalloc_csdev;
+	}
 
 	mutex_lock(&coresight_mutex);
 
@@ -1038,8 +1033,6 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
 
 	return csdev;
 
-err_device_register:
-	kfree(conns);
 err_kzalloc_conns:
 	kfree(refcnts);
 err_kzalloc_refcnts:
diff --git a/drivers/hwtracing/coresight/of_coresight.c b/drivers/hwtracing/coresight/of_coresight.c
index 7c375443ede6..a33a92ebe74b 100644
--- a/drivers/hwtracing/coresight/of_coresight.c
+++ b/drivers/hwtracing/coresight/of_coresight.c
@@ -1,13 +1,6 @@
-/* Copyright (c) 2012, The Linux Foundation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2012, The Linux Foundation. All rights reserved.
  */
 
 #include <linux/types.h>
diff --git a/drivers/hwtracing/intel_th/msu.c b/drivers/hwtracing/intel_th/msu.c
index ede388309376..634f58042c77 100644
--- a/drivers/hwtracing/intel_th/msu.c
+++ b/drivers/hwtracing/intel_th/msu.c
@@ -733,8 +733,8 @@ static int msc_buffer_win_alloc(struct msc *msc, unsigned int nr_blocks)
 		/* Reset the page to write-back before releasing */
 		set_memory_wb((unsigned long)win->block[i].bdesc, 1);
 #endif
-		dma_free_coherent(msc_dev(msc), size, win->block[i].bdesc,
-				  win->block[i].addr);
+		dma_free_coherent(msc_dev(msc)->parent->parent, size,
+				  win->block[i].bdesc, win->block[i].addr);
 	}
 	kfree(win);
 
@@ -769,7 +769,7 @@ static void msc_buffer_win_free(struct msc *msc, struct msc_window *win)
 		/* Reset the page to write-back before releasing */
 		set_memory_wb((unsigned long)win->block[i].bdesc, 1);
 #endif
-		dma_free_coherent(msc_dev(win->msc), PAGE_SIZE,
+		dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE,
 				  win->block[i].bdesc, win->block[i].addr);
 	}
 
diff --git a/drivers/hwtracing/stm/core.c b/drivers/hwtracing/stm/core.c
index 05386b76465e..10bcb5d73f90 100644
--- a/drivers/hwtracing/stm/core.c
+++ b/drivers/hwtracing/stm/core.c
@@ -19,6 +19,7 @@
 #include <linux/stm.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
+#include <linux/vmalloc.h>
 #include "stm.h"
 
 #include <uapi/linux/stm.h>
@@ -674,7 +675,7 @@ static void stm_device_release(struct device *dev)
 {
 	struct stm_device *stm = to_stm_device(dev);
 
-	kfree(stm);
+	vfree(stm);
 }
 
 int stm_register_device(struct device *parent, struct stm_data *stm_data,
@@ -691,7 +692,7 @@ int stm_register_device(struct device *parent, struct stm_data *stm_data,
 		return -EINVAL;
 
 	nmasters = stm_data->sw_end - stm_data->sw_start + 1;
-	stm = kzalloc(sizeof(*stm) + nmasters * sizeof(void *), GFP_KERNEL);
+	stm = vzalloc(sizeof(*stm) + nmasters * sizeof(void *));
 	if (!stm)
 		return -ENOMEM;
 
@@ -744,7 +745,7 @@ int stm_register_device(struct device *parent, struct stm_data *stm_data,
 	/* matches device_initialize() above */
 	put_device(&stm->dev);
 err_free:
-	kfree(stm);
+	vfree(stm);
 
 	return err;
 }
diff --git a/drivers/hwtracing/stm/ftrace.c b/drivers/hwtracing/stm/ftrace.c
index 7da75644c750..ce868e095410 100644
--- a/drivers/hwtracing/stm/ftrace.c
+++ b/drivers/hwtracing/stm/ftrace.c
@@ -1,16 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Simple kernel driver to link kernel Ftrace and an STM device
  * Copyright (c) 2016, Linaro Ltd.
  *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
  * STM Ftrace will be registered as a trace_export.
  */
 
diff --git a/drivers/i2c/algos/i2c-algo-bit.c b/drivers/i2c/algos/i2c-algo-bit.c
index 3df0efd69ae3..4a34f311e1ff 100644
--- a/drivers/i2c/algos/i2c-algo-bit.c
+++ b/drivers/i2c/algos/i2c-algo-bit.c
@@ -519,9 +519,7 @@ static int bit_doAddress(struct i2c_adapter *i2c_adap, struct i2c_msg *msg)
 			}
 		}
 	} else {		/* normal 7bit address	*/
-		addr = msg->addr << 1;
-		if (flags & I2C_M_RD)
-			addr |= 1;
+		addr = i2c_8bit_addr_from_msg(msg);
 		if (flags & I2C_M_REV_DIR_ADDR)
 			addr ^= 1;
 		ret = try_address(i2c_adap, addr, retries);
diff --git a/drivers/i2c/algos/i2c-algo-pca.c b/drivers/i2c/algos/i2c-algo-pca.c
index e370804ec8bc..883a290f6a4d 100644
--- a/drivers/i2c/algos/i2c-algo-pca.c
+++ b/drivers/i2c/algos/i2c-algo-pca.c
@@ -112,11 +112,8 @@ static int pca_address(struct i2c_algo_pca_data *adap,
 		       struct i2c_msg *msg)
 {
 	int sta = pca_get_con(adap);
-	int addr;
+	int addr = i2c_8bit_addr_from_msg(msg);
 
-	addr = ((0x7f & msg->addr) << 1);
-	if (msg->flags & I2C_M_RD)
-		addr |= 1;
 	DEB2("=== SLAVE ADDRESS %#04x+%c=%#04x\n",
 	     msg->addr, msg->flags & I2C_M_RD ? 'R' : 'W', addr);
 
diff --git a/drivers/i2c/algos/i2c-algo-pcf.c b/drivers/i2c/algos/i2c-algo-pcf.c
index 270d84bfc2c6..5c29a4d397cf 100644
--- a/drivers/i2c/algos/i2c-algo-pcf.c
+++ b/drivers/i2c/algos/i2c-algo-pcf.c
@@ -291,13 +291,9 @@ static int pcf_readbytes(struct i2c_adapter *i2c_adap, char *buf,
 static int pcf_doAddress(struct i2c_algo_pcf_data *adap,
 			 struct i2c_msg *msg)
 {
-	unsigned short flags = msg->flags;
-	unsigned char addr;
+	unsigned char addr = i2c_8bit_addr_from_msg(msg);
 
-	addr = msg->addr << 1;
-	if (flags & I2C_M_RD)
-		addr |= 1;
-	if (flags & I2C_M_REV_DIR_ADDR)
+	if (msg->flags & I2C_M_REV_DIR_ADDR)
 		addr ^= 1;
 	i2c_outb(adap, addr);
 
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index 8d21b9825d71..4f8df2ec87b1 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -202,8 +202,7 @@ config I2C_CHT_WC
 
 	  Note this controller is hooked up to a TI bq24292i charger-IC,
 	  combined with a FUSB302 Type-C port-controller as such it is advised
-	  to also select CONFIG_CHARGER_BQ24190=m and CONFIG_TYPEC_FUSB302=m
-	  (the fusb302 driver currently is in drivers/staging).
+	  to also select CONFIG_TYPEC_FUSB302=m.
 
 config I2C_NFORCE2
 	tristate "Nvidia nForce2, nForce3 and nForce4"
@@ -944,6 +943,7 @@ config I2C_STM32F4
 config I2C_STM32F7
 	tristate "STMicroelectronics STM32F7 I2C support"
 	depends on ARCH_STM32 || COMPILE_TEST
+	select I2C_SLAVE
 	help
 	  Enable this option to add support for STM32 I2C controller embedded
 	  in STM32F7 SoCs.
diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile
index 189e34ba050f..5a869144a0c5 100644
--- a/drivers/i2c/busses/Makefile
+++ b/drivers/i2c/busses/Makefile
@@ -94,7 +94,8 @@ obj-$(CONFIG_I2C_SIRF)		+= i2c-sirf.o
 obj-$(CONFIG_I2C_SPRD)		+= i2c-sprd.o
 obj-$(CONFIG_I2C_ST)		+= i2c-st.o
 obj-$(CONFIG_I2C_STM32F4)	+= i2c-stm32f4.o
-obj-$(CONFIG_I2C_STM32F7)	+= i2c-stm32f7.o
+i2c-stm32f7-drv-objs := i2c-stm32f7.o i2c-stm32.o
+obj-$(CONFIG_I2C_STM32F7)	+= i2c-stm32f7-drv.o
 obj-$(CONFIG_I2C_STU300)	+= i2c-stu300.o
 obj-$(CONFIG_I2C_SUN6I_P2WI)	+= i2c-sun6i-p2wi.o
 obj-$(CONFIG_I2C_SYNQUACER)	+= i2c-synquacer.o
diff --git a/drivers/i2c/busses/i2c-aspeed.c b/drivers/i2c/busses/i2c-aspeed.c
index 7d4aeb4465b3..60e4d0e939a3 100644
--- a/drivers/i2c/busses/i2c-aspeed.c
+++ b/drivers/i2c/busses/i2c-aspeed.c
@@ -335,13 +335,12 @@ static void aspeed_i2c_do_start(struct aspeed_i2c_bus *bus)
 {
 	u32 command = ASPEED_I2CD_M_START_CMD | ASPEED_I2CD_M_TX_CMD;
 	struct i2c_msg *msg = &bus->msgs[bus->msgs_index];
-	u8 slave_addr = msg->addr << 1;
+	u8 slave_addr = i2c_8bit_addr_from_msg(msg);
 
 	bus->master_state = ASPEED_I2C_MASTER_START;
 	bus->buf_index = 0;
 
 	if (msg->flags & I2C_M_RD) {
-		slave_addr |= 1;
 		command |= ASPEED_I2CD_M_RX_CMD;
 		/* Need to let the hardware know to NACK after RX. */
 		if (msg->len == 1 && !(msg->flags & I2C_M_RECV_LEN))
diff --git a/drivers/i2c/busses/i2c-at91.c b/drivers/i2c/busses/i2c-at91.c
index bfd1fdff64a9..3f3e8b3bf5ff 100644
--- a/drivers/i2c/busses/i2c-at91.c
+++ b/drivers/i2c/busses/i2c-at91.c
@@ -518,8 +518,16 @@ static irqreturn_t atmel_twi_interrupt(int irq, void *dev_id)
 	 * the RXRDY interrupt first in order to not keep garbage data in the
 	 * Receive Holding Register for the next transfer.
 	 */
-	if (irqstatus & AT91_TWI_RXRDY)
-		at91_twi_read_next_byte(dev);
+	if (irqstatus & AT91_TWI_RXRDY) {
+		/*
+		 * Read all available bytes at once by polling RXRDY usable w/
+		 * and w/o FIFO. With FIFO enabled we could also read RXFL and
+		 * avoid polling RXRDY.
+		 */
+		do {
+			at91_twi_read_next_byte(dev);
+		} while (at91_twi_read(dev, AT91_TWI_SR) & AT91_TWI_RXRDY);
+	}
 
 	/*
 	 * When a NACK condition is detected, the I2C controller sets the NACK,
diff --git a/drivers/i2c/busses/i2c-axxia.c b/drivers/i2c/busses/i2c-axxia.c
index 13f07482ec68..8e60048a33f8 100644
--- a/drivers/i2c/busses/i2c-axxia.c
+++ b/drivers/i2c/busses/i2c-axxia.c
@@ -351,13 +351,15 @@ static int axxia_i2c_xfer_msg(struct axxia_i2c_dev *idev, struct i2c_msg *msg)
 		 *   addr_2: addr[7:0]
 		 */
 		addr_1 = 0xF0 | ((msg->addr >> 7) & 0x06);
+		if (i2c_m_rd(msg))
+			addr_1 |= 1;	/* Set the R/nW bit of the address */
 		addr_2 = msg->addr & 0xFF;
 	} else {
 		/* 7-bit address
 		 *   addr_1: addr[6:0] | (R/nW)
 		 *   addr_2: dont care
 		 */
-		addr_1 = (msg->addr << 1) & 0xFF;
+		addr_1 = i2c_8bit_addr_from_msg(msg);
 		addr_2 = 0;
 	}
 
@@ -365,7 +367,6 @@ static int axxia_i2c_xfer_msg(struct axxia_i2c_dev *idev, struct i2c_msg *msg)
 		/* I2C read transfer */
 		rx_xfer = i2c_m_recv_len(msg) ? I2C_SMBUS_BLOCK_MAX : msg->len;
 		tx_xfer = 0;
-		addr_1 |= 1;	/* Set the R/nW bit of the address */
 	} else {
 		/* I2C write transfer */
 		rx_xfer = 0;
@@ -532,23 +533,23 @@ static int axxia_i2c_probe(struct platform_device *pdev)
 	if (idev->bus_clk_rate == 0)
 		idev->bus_clk_rate = 100000;	/* default clock rate */
 
+	ret = clk_prepare_enable(idev->i2c_clk);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to enable clock\n");
+		return ret;
+	}
+
 	ret = axxia_i2c_init(idev);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to initialize\n");
-		return ret;
+		goto error_disable_clk;
 	}
 
 	ret = devm_request_irq(&pdev->dev, irq, axxia_i2c_isr, 0,
 			       pdev->name, idev);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to claim IRQ%d\n", irq);
-		return ret;
-	}
-
-	ret = clk_prepare_enable(idev->i2c_clk);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to enable clock\n");
-		return ret;
+		goto error_disable_clk;
 	}
 
 	i2c_set_adapdata(&idev->adapter, idev);
@@ -563,12 +564,14 @@ static int axxia_i2c_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, idev);
 
 	ret = i2c_add_adapter(&idev->adapter);
-	if (ret) {
-		clk_disable_unprepare(idev->i2c_clk);
-		return ret;
-	}
+	if (ret)
+		goto error_disable_clk;
 
 	return 0;
+
+error_disable_clk:
+	clk_disable_unprepare(idev->i2c_clk);
+	return ret;
 }
 
 static int axxia_i2c_remove(struct platform_device *pdev)
diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c
index 27ebd90de43b..48914dfc8ce8 100644
--- a/drivers/i2c/busses/i2c-designware-common.c
+++ b/drivers/i2c/busses/i2c-designware-common.c
@@ -149,18 +149,17 @@ u32 i2c_dw_scl_lcnt(u32 ic_clk, u32 tLOW, u32 tf, int offset)
 	return ((ic_clk * (tLOW + tf) + 500000) / 1000000) - 1 + offset;
 }
 
-void __i2c_dw_enable(struct dw_i2c_dev *dev, bool enable)
-{
-	dw_writel(dev, enable, DW_IC_ENABLE);
-}
-
-void __i2c_dw_enable_and_wait(struct dw_i2c_dev *dev, bool enable)
+void __i2c_dw_disable(struct dw_i2c_dev *dev)
 {
 	int timeout = 100;
 
 	do {
-		__i2c_dw_enable(dev, enable);
-		if ((dw_readl(dev, DW_IC_ENABLE_STATUS) & 1) == enable)
+		__i2c_dw_disable_nowait(dev);
+		/*
+		 * The enable status register may be unimplemented, but
+		 * in that case this test reads zero and exits the loop.
+		 */
+		if ((dw_readl(dev, DW_IC_ENABLE_STATUS) & 1) == 0)
 			return;
 
 		/*
@@ -171,8 +170,7 @@ void __i2c_dw_enable_and_wait(struct dw_i2c_dev *dev, bool enable)
 		usleep_range(25, 250);
 	} while (timeout--);
 
-	dev_warn(dev->dev, "timeout in %sabling adapter\n",
-		 enable ? "en" : "dis");
+	dev_warn(dev->dev, "timeout in disabling adapter\n");
 }
 
 unsigned long i2c_dw_clk_rate(struct dw_i2c_dev *dev)
@@ -277,7 +275,7 @@ u32 i2c_dw_func(struct i2c_adapter *adap)
 void i2c_dw_disable(struct dw_i2c_dev *dev)
 {
 	/* Disable controller */
-	__i2c_dw_enable_and_wait(dev, false);
+	__i2c_dw_disable(dev);
 
 	/* Disable all interupts */
 	dw_writel(dev, 0, DW_IC_INTR_MASK);
diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h
index 8707c76b2fee..d690e648bc01 100644
--- a/drivers/i2c/busses/i2c-designware-core.h
+++ b/drivers/i2c/busses/i2c-designware-core.h
@@ -297,8 +297,6 @@ u32 dw_readl(struct dw_i2c_dev *dev, int offset);
 void dw_writel(struct dw_i2c_dev *dev, u32 b, int offset);
 u32 i2c_dw_scl_hcnt(u32 ic_clk, u32 tSYMBOL, u32 tf, int cond, int offset);
 u32 i2c_dw_scl_lcnt(u32 ic_clk, u32 tLOW, u32 tf, int offset);
-void __i2c_dw_enable(struct dw_i2c_dev *dev, bool enable);
-void __i2c_dw_enable_and_wait(struct dw_i2c_dev *dev, bool enable);
 unsigned long i2c_dw_clk_rate(struct dw_i2c_dev *dev);
 int i2c_dw_prepare_clk(struct dw_i2c_dev *dev, bool prepare);
 int i2c_dw_acquire_lock(struct dw_i2c_dev *dev);
@@ -309,6 +307,18 @@ u32 i2c_dw_func(struct i2c_adapter *adap);
 void i2c_dw_disable(struct dw_i2c_dev *dev);
 void i2c_dw_disable_int(struct dw_i2c_dev *dev);
 
+static inline void __i2c_dw_enable(struct dw_i2c_dev *dev)
+{
+	dw_writel(dev, 1, DW_IC_ENABLE);
+}
+
+static inline void __i2c_dw_disable_nowait(struct dw_i2c_dev *dev)
+{
+	dw_writel(dev, 0, DW_IC_ENABLE);
+}
+
+void __i2c_dw_disable(struct dw_i2c_dev *dev);
+
 extern u32 i2c_dw_read_comp_param(struct dw_i2c_dev *dev);
 extern int i2c_dw_probe(struct dw_i2c_dev *dev);
 #if IS_ENABLED(CONFIG_I2C_DESIGNWARE_SLAVE)
diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c
index 0cdba29ae0a9..27436a937492 100644
--- a/drivers/i2c/busses/i2c-designware-master.c
+++ b/drivers/i2c/busses/i2c-designware-master.c
@@ -81,7 +81,7 @@ static int i2c_dw_init_master(struct dw_i2c_dev *dev)
 	comp_param1 = dw_readl(dev, DW_IC_COMP_PARAM_1);
 
 	/* Disable the adapter */
-	__i2c_dw_enable_and_wait(dev, false);
+	__i2c_dw_disable(dev);
 
 	/* Set standard and fast speed deviders for high/low periods */
 
@@ -180,7 +180,7 @@ static void i2c_dw_xfer_init(struct dw_i2c_dev *dev)
 	u32 ic_con, ic_tar = 0;
 
 	/* Disable the adapter */
-	__i2c_dw_enable_and_wait(dev, false);
+	__i2c_dw_disable(dev);
 
 	/* If the slave address is ten bit address, enable 10BITADDR */
 	ic_con = dw_readl(dev, DW_IC_CON);
@@ -209,7 +209,7 @@ static void i2c_dw_xfer_init(struct dw_i2c_dev *dev)
 	i2c_dw_disable_int(dev);
 
 	/* Enable the adapter */
-	__i2c_dw_enable(dev, true);
+	__i2c_dw_enable(dev);
 
 	/* Dummy read to avoid the register getting stuck on Bay Trail */
 	dw_readl(dev, DW_IC_ENABLE_STATUS);
@@ -462,7 +462,7 @@ i2c_dw_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 	 * additional interrupts are a hardware bug or this driver doesn't
 	 * handle them correctly yet.
 	 */
-	__i2c_dw_enable(dev, false);
+	__i2c_dw_disable_nowait(dev);
 
 	if (dev->msg_err) {
 		ret = dev->msg_err;
diff --git a/drivers/i2c/busses/i2c-designware-slave.c b/drivers/i2c/busses/i2c-designware-slave.c
index d42558d1b002..8ce2cd368477 100644
--- a/drivers/i2c/busses/i2c-designware-slave.c
+++ b/drivers/i2c/busses/i2c-designware-slave.c
@@ -75,7 +75,7 @@ static int i2c_dw_init_slave(struct dw_i2c_dev *dev)
 	comp_param1 = dw_readl(dev, DW_IC_COMP_PARAM_1);
 
 	/* Disable the adapter. */
-	__i2c_dw_enable_and_wait(dev, false);
+	__i2c_dw_disable(dev);
 
 	/* Configure SDA Hold Time if required. */
 	reg = dw_readl(dev, DW_IC_COMP_VERSION);
@@ -119,11 +119,11 @@ static int i2c_dw_reg_slave(struct i2c_client *slave)
 	 * Set slave address in the IC_SAR register,
 	 * the address to which the DW_apb_i2c responds.
 	 */
-	__i2c_dw_enable(dev, false);
+	__i2c_dw_disable_nowait(dev);
 	dw_writel(dev, slave->addr, DW_IC_SAR);
 	dev->slave = slave;
 
-	__i2c_dw_enable(dev, true);
+	__i2c_dw_enable(dev);
 
 	dev->cmd_err = 0;
 	dev->msg_write_idx = 0;
diff --git a/drivers/i2c/busses/i2c-diolan-u2c.c b/drivers/i2c/busses/i2c-diolan-u2c.c
index f718ee4e3332..3f28317cde39 100644
--- a/drivers/i2c/busses/i2c-diolan-u2c.c
+++ b/drivers/i2c/busses/i2c-diolan-u2c.c
@@ -360,11 +360,11 @@ static int diolan_usb_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs,
 			if (ret < 0)
 				goto abort;
 		}
+		ret = diolan_i2c_put_byte_ack(dev,
+					      i2c_8bit_addr_from_msg(pmsg));
+		if (ret < 0)
+			goto abort;
 		if (pmsg->flags & I2C_M_RD) {
-			ret =
-			    diolan_i2c_put_byte_ack(dev, (pmsg->addr << 1) | 1);
-			if (ret < 0)
-				goto abort;
 			for (j = 0; j < pmsg->len; j++) {
 				u8 byte;
 				bool ack = j < pmsg->len - 1;
@@ -393,9 +393,6 @@ static int diolan_usb_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs,
 				pmsg->buf[j] = byte;
 			}
 		} else {
-			ret = diolan_i2c_put_byte_ack(dev, pmsg->addr << 1);
-			if (ret < 0)
-				goto abort;
 			for (j = 0; j < pmsg->len; j++) {
 				ret = diolan_i2c_put_byte_ack(dev,
 							      pmsg->buf[j]);
diff --git a/drivers/i2c/busses/i2c-efm32.c b/drivers/i2c/busses/i2c-efm32.c
index aa336ba89aa3..5f2bab878b2c 100644
--- a/drivers/i2c/busses/i2c-efm32.c
+++ b/drivers/i2c/busses/i2c-efm32.c
@@ -144,8 +144,7 @@ static void efm32_i2c_send_next_msg(struct efm32_i2c_ddata *ddata)
 	struct i2c_msg *cur_msg = &ddata->msgs[ddata->current_msg];
 
 	efm32_i2c_write32(ddata, REG_CMD, REG_CMD_START);
-	efm32_i2c_write32(ddata, REG_TXDATA, cur_msg->addr << 1 |
-			(cur_msg->flags & I2C_M_RD ? 1 : 0));
+	efm32_i2c_write32(ddata, REG_TXDATA, i2c_8bit_addr_from_msg(cur_msg));
 }
 
 static void efm32_i2c_send_next_byte(struct efm32_i2c_ddata *ddata)
diff --git a/drivers/i2c/busses/i2c-eg20t.c b/drivers/i2c/busses/i2c-eg20t.c
index bdeab0174fec..835d54ac2971 100644
--- a/drivers/i2c/busses/i2c-eg20t.c
+++ b/drivers/i2c/busses/i2c-eg20t.c
@@ -414,7 +414,7 @@ static s32 pch_i2c_writebytes(struct i2c_adapter *i2c_adap,
 		iowrite32(addr_8_lsb, p + PCH_I2CDR);
 	} else {
 		/* set 7 bit slave address and R/W bit as 0 */
-		iowrite32(addr << 1, p + PCH_I2CDR);
+		iowrite32(i2c_8bit_addr_from_msg(msgs), p + PCH_I2CDR);
 		if (first)
 			pch_i2c_start(adap);
 	}
@@ -538,8 +538,7 @@ static s32 pch_i2c_readbytes(struct i2c_adapter *i2c_adap, struct i2c_msg *msgs,
 		iowrite32(addr_2_msb | TEN_BIT_ADDR_MASK, p + PCH_I2CDR);
 	} else {
 		/* 7 address bits + R/W bit */
-		addr = (((addr) << 1) | (I2C_RD));
-		iowrite32(addr, p + PCH_I2CDR);
+		iowrite32(i2c_8bit_addr_from_msg(msgs), p + PCH_I2CDR);
 	}
 
 	/* check if it is the first message */
diff --git a/drivers/i2c/busses/i2c-emev2.c b/drivers/i2c/busses/i2c-emev2.c
index d2e84480fbe9..ba9b6ea48a31 100644
--- a/drivers/i2c/busses/i2c-emev2.c
+++ b/drivers/i2c/busses/i2c-emev2.c
@@ -149,7 +149,7 @@ static int __em_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msg,
 	em_clear_set_bit(priv, 0, I2C_BIT_STT0, I2C_OFS_IICC0);
 
 	/* Send slave address and R/W type */
-	writeb((msg->addr << 1) | read, priv->base + I2C_OFS_IIC0);
+	writeb(i2c_8bit_addr_from_msg(msg), priv->base + I2C_OFS_IIC0);
 
 	/* Wait for transaction */
 	status = em_i2c_wait_for_event(priv);
diff --git a/drivers/i2c/busses/i2c-exynos5.c b/drivers/i2c/busses/i2c-exynos5.c
index 12ec8484e653..de82ad8ff534 100644
--- a/drivers/i2c/busses/i2c-exynos5.c
+++ b/drivers/i2c/busses/i2c-exynos5.c
@@ -707,7 +707,7 @@ static int exynos5_i2c_xfer(struct i2c_adapter *adap,
 			struct i2c_msg *msgs, int num)
 {
 	struct exynos5_i2c *i2c = adap->algo_data;
-	int i = 0, ret = 0, stop = 0;
+	int i, ret;
 
 	if (i2c->suspended) {
 		dev_err(i2c->dev, "HS-I2C is not initialized.\n");
@@ -718,30 +718,15 @@ static int exynos5_i2c_xfer(struct i2c_adapter *adap,
 	if (ret)
 		return ret;
 
-	for (i = 0; i < num; i++, msgs++) {
-		stop = (i == num - 1);
-
-		ret = exynos5_i2c_xfer_msg(i2c, msgs, stop);
-
-		if (ret < 0)
-			goto out;
-	}
-
-	if (i == num) {
-		ret = num;
-	} else {
-		/* Only one message, cannot access the device */
-		if (i == 1)
-			ret = -EREMOTEIO;
-		else
-			ret = i;
-
-		dev_warn(i2c->dev, "xfer message failed\n");
+	for (i = 0; i < num; ++i) {
+		ret = exynos5_i2c_xfer_msg(i2c, msgs + i, i + 1 == num);
+		if (ret)
+			break;
 	}
 
- out:
 	clk_disable(i2c->clk);
-	return ret;
+
+	return ret ?: num;
 }
 
 static u32 exynos5_i2c_func(struct i2c_adapter *adap)
diff --git a/drivers/i2c/busses/i2c-gpio.c b/drivers/i2c/busses/i2c-gpio.c
index 58abb3eced58..005e6e0330c2 100644
--- a/drivers/i2c/busses/i2c-gpio.c
+++ b/drivers/i2c/busses/i2c-gpio.c
@@ -11,7 +11,7 @@
 #include <linux/delay.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
-#include <linux/i2c-gpio.h>
+#include <linux/platform_data/i2c-gpio.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/slab.h>
diff --git a/drivers/i2c/busses/i2c-hix5hd2.c b/drivers/i2c/busses/i2c-hix5hd2.c
index bb68957d3da5..061a4bfb03f4 100644
--- a/drivers/i2c/busses/i2c-hix5hd2.c
+++ b/drivers/i2c/busses/i2c-hix5hd2.c
@@ -73,7 +73,6 @@
 #define I2C_OVER_INTR		BIT(0)
 
 #define HIX5I2C_MAX_FREQ	400000		/* 400k */
-#define HIX5I2C_READ_OPERATION	0x01
 
 enum hix5hd2_i2c_state {
 	HIX5I2C_STAT_RW_ERR = -1,
@@ -311,12 +310,8 @@ static void hix5hd2_i2c_message_start(struct hix5hd2_i2c_priv *priv, int stop)
 	hix5hd2_i2c_clr_all_irq(priv);
 	hix5hd2_i2c_enable_irq(priv);
 
-	if (priv->msg->flags & I2C_M_RD)
-		writel_relaxed((priv->msg->addr << 1) | HIX5I2C_READ_OPERATION,
-			       priv->regs + HIX5I2C_TXR);
-	else
-		writel_relaxed(priv->msg->addr << 1,
-			       priv->regs + HIX5I2C_TXR);
+	writel_relaxed(i2c_8bit_addr_from_msg(priv->msg),
+		       priv->regs + HIX5I2C_TXR);
 
 	writel_relaxed(I2C_WRITE | I2C_START, priv->regs + HIX5I2C_COM);
 	spin_unlock_irqrestore(&priv->lock, flags);
@@ -377,17 +372,7 @@ static int hix5hd2_i2c_xfer(struct i2c_adapter *adap,
 			goto out;
 	}
 
-	if (i == num) {
-		ret = num;
-	} else {
-		/* Only one message, cannot access the device */
-		if (i == 1)
-			ret = -EREMOTEIO;
-		else
-			ret = i;
-
-		dev_warn(priv->dev, "xfer message failed\n");
-	}
+	ret = num;
 
 out:
 	pm_runtime_mark_last_busy(priv->dev);
@@ -471,7 +456,6 @@ static int hix5hd2_i2c_probe(struct platform_device *pdev)
 		goto err_clk;
 	}
 
-	pm_suspend_ignore_children(&pdev->dev, true);
 	pm_runtime_set_autosuspend_delay(priv->dev, MSEC_PER_SEC);
 	pm_runtime_use_autosuspend(priv->dev);
 	pm_runtime_set_active(priv->dev);
diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c
index e0d59e9ff3c6..aa726607645e 100644
--- a/drivers/i2c/busses/i2c-i801.c
+++ b/drivers/i2c/busses/i2c-i801.c
@@ -106,7 +106,7 @@
 
 #if IS_ENABLED(CONFIG_I2C_MUX_GPIO) && defined CONFIG_DMI
 #include <linux/gpio.h>
-#include <linux/i2c-mux-gpio.h>
+#include <linux/platform_data/i2c-mux-gpio.h>
 #endif
 
 /* I801 SMBus address offsets */
@@ -1710,7 +1710,7 @@ static void i801_shutdown(struct pci_dev *dev)
 	pci_write_config_byte(dev, SMBHSTCFG, priv->original_hstcfg);
 }
 
-#ifdef CONFIG_PM
+#ifdef CONFIG_PM_SLEEP
 static int i801_suspend(struct device *dev)
 {
 	struct pci_dev *pci_dev = to_pci_dev(dev);
@@ -1731,8 +1731,7 @@ static int i801_resume(struct device *dev)
 }
 #endif
 
-static UNIVERSAL_DEV_PM_OPS(i801_pm_ops, i801_suspend,
-			    i801_resume, NULL);
+static SIMPLE_DEV_PM_OPS(i801_pm_ops, i801_suspend, i801_resume);
 
 static struct pci_driver i801_driver = {
 	.name		= "i801_smbus",
diff --git a/drivers/i2c/busses/i2c-ibm_iic.c b/drivers/i2c/busses/i2c-ibm_iic.c
index 961c5f42d956..6f6e1dfe7cce 100644
--- a/drivers/i2c/busses/i2c-ibm_iic.c
+++ b/drivers/i2c/busses/i2c-ibm_iic.c
@@ -561,9 +561,6 @@ static int iic_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
 
 	DBG2("%d: iic_xfer, %d msg(s)\n", dev->idx, num);
 
-	if (!num)
-		return 0;
-
 	/* Check the sanity of the passed messages.
 	 * Uhh, generic i2c layer is more suitable place for such code...
 	 */
diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c
index e6da2c7a9a3e..6d975f5221ca 100644
--- a/drivers/i2c/busses/i2c-imx-lpi2c.c
+++ b/drivers/i2c/busses/i2c-imx-lpi2c.c
@@ -1,18 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * This is i.MX low power i2c controller driver.
  *
  * Copyright 2016 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
  */
 
 #include <linux/clk.h>
@@ -180,15 +170,13 @@ static int lpi2c_imx_start(struct lpi2c_imx_struct *lpi2c_imx,
 			   struct i2c_msg *msgs)
 {
 	unsigned int temp;
-	u8 read;
 
 	temp = readl(lpi2c_imx->base + LPI2C_MCR);
 	temp |= MCR_RRF | MCR_RTF;
 	writel(temp, lpi2c_imx->base + LPI2C_MCR);
 	writel(0x7f00, lpi2c_imx->base + LPI2C_MSR);
 
-	read = msgs->flags & I2C_M_RD;
-	temp = (msgs->addr << 1 | read) | (GEN_START << 8);
+	temp = i2c_8bit_addr_from_msg(msgs) | (GEN_START << 8);
 	writel(temp, lpi2c_imx->base + LPI2C_MTDR);
 
 	return lpi2c_imx_bus_busy(lpi2c_imx);
diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c
index d7267dd9c7bf..0207e194f84b 100644
--- a/drivers/i2c/busses/i2c-imx.c
+++ b/drivers/i2c/busses/i2c-imx.c
@@ -1,16 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  *	Copyright (C) 2002 Motorola GSG-China
  *
- *	This program is free software; you can redistribute it and/or
- *	modify it under the terms of the GNU General Public License
- *	as published by the Free Software Foundation; either version 2
- *	of the License, or (at your option) any later version.
- *
- *	This program is distributed in the hope that it will be useful,
- *	but WITHOUT ANY WARRANTY; without even the implied warranty of
- *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *	GNU General Public License for more details.
- *
  * Author:
  *	Darius Augulis, Teltonika Inc.
  *
@@ -630,7 +621,7 @@ static int i2c_imx_dma_write(struct imx_i2c_struct *i2c_imx,
 	 * Write slave address.
 	 * The first byte must be transmitted by the CPU.
 	 */
-	imx_i2c_write_reg(msgs->addr << 1, i2c_imx, IMX_I2C_I2DR);
+	imx_i2c_write_reg(i2c_8bit_addr_from_msg(msgs), i2c_imx, IMX_I2C_I2DR);
 	reinit_completion(&i2c_imx->dma->cmd_complete);
 	time_left = wait_for_completion_timeout(
 				&i2c_imx->dma->cmd_complete,
@@ -760,10 +751,10 @@ static int i2c_imx_write(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs)
 	int i, result;
 
 	dev_dbg(&i2c_imx->adapter.dev, "<%s> write slave address: addr=0x%x\n",
-		__func__, msgs->addr << 1);
+		__func__, i2c_8bit_addr_from_msg(msgs));
 
 	/* write slave address */
-	imx_i2c_write_reg(msgs->addr << 1, i2c_imx, IMX_I2C_I2DR);
+	imx_i2c_write_reg(i2c_8bit_addr_from_msg(msgs), i2c_imx, IMX_I2C_I2DR);
 	result = i2c_imx_trx_complete(i2c_imx);
 	if (result)
 		return result;
@@ -796,10 +787,10 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, bo
 
 	dev_dbg(&i2c_imx->adapter.dev,
 		"<%s> write slave address: addr=0x%x\n",
-		__func__, (msgs->addr << 1) | 0x01);
+		__func__, i2c_8bit_addr_from_msg(msgs));
 
 	/* write slave address */
-	imx_i2c_write_reg((msgs->addr << 1) | 0x01, i2c_imx, IMX_I2C_I2DR);
+	imx_i2c_write_reg(i2c_8bit_addr_from_msg(msgs), i2c_imx, IMX_I2C_I2DR);
 	result = i2c_imx_trx_complete(i2c_imx);
 	if (result)
 		return result;
diff --git a/drivers/i2c/busses/i2c-kempld.c b/drivers/i2c/busses/i2c-kempld.c
index e879190b5d1d..1c874aaa0447 100644
--- a/drivers/i2c/busses/i2c-kempld.c
+++ b/drivers/i2c/busses/i2c-kempld.c
@@ -124,15 +124,14 @@ static int kempld_i2c_process(struct kempld_i2c_data *i2c)
 		/* 10 bit address? */
 		if (i2c->msg->flags & I2C_M_TEN) {
 			addr = 0xf0 | ((i2c->msg->addr >> 7) & 0x6);
+			/* Set read bit if necessary */
+			addr |= (i2c->msg->flags & I2C_M_RD) ? 1 : 0;
 			i2c->state = STATE_ADDR10;
 		} else {
-			addr = (i2c->msg->addr << 1);
+			addr = i2c_8bit_addr_from_msg(i2c->msg);
 			i2c->state = STATE_START;
 		}
 
-		/* Set read bit if necessary */
-		addr |= (i2c->msg->flags & I2C_M_RD) ? 1 : 0;
-
 		kempld_write8(pld, KEMPLD_I2C_DATA, addr);
 		kempld_write8(pld, KEMPLD_I2C_CMD, I2C_CMD_START);
 
diff --git a/drivers/i2c/busses/i2c-mlxcpld.c b/drivers/i2c/busses/i2c-mlxcpld.c
index 4c28fa28ce76..745ed43a22d6 100644
--- a/drivers/i2c/busses/i2c-mlxcpld.c
+++ b/drivers/i2c/busses/i2c-mlxcpld.c
@@ -45,13 +45,16 @@
 #define MLXCPLD_I2C_VALID_FLAG		(I2C_M_RECV_LEN | I2C_M_RD)
 #define MLXCPLD_I2C_BUS_NUM		1
 #define MLXCPLD_I2C_DATA_REG_SZ		36
+#define MLXCPLD_I2C_DATA_SZ_BIT		BIT(5)
+#define MLXCPLD_I2C_DATA_SZ_MASK	GENMASK(6, 5)
+#define MLXCPLD_I2C_SMBUS_BLK_BIT	BIT(7)
 #define MLXCPLD_I2C_MAX_ADDR_LEN	4
 #define MLXCPLD_I2C_RETR_NUM		2
 #define MLXCPLD_I2C_XFER_TO		500000 /* usec */
 #define MLXCPLD_I2C_POLL_TIME		2000   /* usec */
 
 /* LPC I2C registers */
-#define MLXCPLD_LPCI2C_LPF_REG		0x0
+#define MLXCPLD_LPCI2C_CPBLTY_REG	0x0
 #define MLXCPLD_LPCI2C_CTRL_REG		0x1
 #define MLXCPLD_LPCI2C_HALF_CYC_REG	0x4
 #define MLXCPLD_LPCI2C_I2C_HOLD_REG	0x5
@@ -83,6 +86,7 @@ struct mlxcpld_i2c_priv {
 	struct mutex lock;
 	struct  mlxcpld_i2c_curr_xfer xfer;
 	struct device *dev;
+	bool smbus_block;
 };
 
 static void mlxcpld_i2c_lpc_write_buf(u8 *data, u8 len, u32 addr)
@@ -230,7 +234,7 @@ static void mlxcpld_i2c_set_transf_data(struct mlxcpld_i2c_priv *priv,
 	 * All upper layers currently are never use transfer with more than
 	 * 2 messages. Actually, it's also not so relevant in Mellanox systems
 	 * because of HW limitation. Max size of transfer is not more than 32
-	 * bytes in the current x86 LPCI2C bridge.
+	 * or 68 bytes in the current x86 LPCI2C bridge.
 	 */
 	priv->xfer.cmd = msgs[num - 1].flags & I2C_M_RD;
 
@@ -295,7 +299,7 @@ static int mlxcpld_i2c_wait_for_free(struct mlxcpld_i2c_priv *priv)
 static int mlxcpld_i2c_wait_for_tc(struct mlxcpld_i2c_priv *priv)
 {
 	int status, i, timeout = 0;
-	u8 datalen;
+	u8 datalen, val;
 
 	do {
 		usleep_range(MLXCPLD_I2C_POLL_TIME / 2, MLXCPLD_I2C_POLL_TIME);
@@ -324,9 +328,22 @@ static int mlxcpld_i2c_wait_for_tc(struct mlxcpld_i2c_priv *priv)
 		 * Actual read data len will be always the same as
 		 * requested len. 0xff (line pull-up) will be returned
 		 * if slave has no data to return. Thus don't read
-		 * MLXCPLD_LPCI2C_NUM_DAT_REG reg from CPLD.
+		 * MLXCPLD_LPCI2C_NUM_DAT_REG reg from CPLD.  Only in case of
+		 * SMBus block read transaction data len can be different,
+		 * check this case.
 		 */
-		datalen = priv->xfer.data_len;
+		mlxcpld_i2c_read_comm(priv, MLXCPLD_LPCI2C_NUM_ADDR_REG, &val,
+				      1);
+		if (priv->smbus_block && (val & MLXCPLD_I2C_SMBUS_BLK_BIT)) {
+			mlxcpld_i2c_read_comm(priv, MLXCPLD_LPCI2C_NUM_DAT_REG,
+					      &datalen, 1);
+			if (unlikely(datalen > (I2C_SMBUS_BLOCK_MAX + 1))) {
+				dev_err(priv->dev, "Incorrect smbus block read message len\n");
+				return -E2BIG;
+			}
+		} else {
+			datalen = priv->xfer.data_len;
+		}
 
 		mlxcpld_i2c_read_comm(priv, MLXCPLD_LPCI2C_DATA_REG,
 				      priv->xfer.msg[i].buf, datalen);
@@ -344,12 +361,20 @@ static int mlxcpld_i2c_wait_for_tc(struct mlxcpld_i2c_priv *priv)
 static void mlxcpld_i2c_xfer_msg(struct mlxcpld_i2c_priv *priv)
 {
 	int i, len = 0;
-	u8 cmd;
+	u8 cmd, val;
 
 	mlxcpld_i2c_write_comm(priv, MLXCPLD_LPCI2C_NUM_DAT_REG,
 			       &priv->xfer.data_len, 1);
-	mlxcpld_i2c_write_comm(priv, MLXCPLD_LPCI2C_NUM_ADDR_REG,
-			       &priv->xfer.addr_width, 1);
+
+	val = priv->xfer.addr_width;
+	/* Notify HW about SMBus block read transaction */
+	if (priv->smbus_block && priv->xfer.msg_num >= 2 &&
+	    priv->xfer.msg[1].len == 1 &&
+	    (priv->xfer.msg[1].flags & I2C_M_RECV_LEN) &&
+	    (priv->xfer.msg[1].flags & I2C_M_RD))
+		val |= MLXCPLD_I2C_SMBUS_BLK_BIT;
+
+	mlxcpld_i2c_write_comm(priv, MLXCPLD_LPCI2C_NUM_ADDR_REG, &val, 1);
 
 	for (i = 0; i < priv->xfer.msg_num; i++) {
 		if ((priv->xfer.msg[i].flags & I2C_M_RD) != I2C_M_RD) {
@@ -425,7 +450,14 @@ static int mlxcpld_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
 
 static u32 mlxcpld_i2c_func(struct i2c_adapter *adap)
 {
-	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | I2C_FUNC_SMBUS_BLOCK_DATA;
+	struct mlxcpld_i2c_priv *priv = i2c_get_adapdata(adap);
+
+	if (priv->smbus_block)
+		return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL |
+			I2C_FUNC_SMBUS_I2C_BLOCK | I2C_FUNC_SMBUS_BLOCK_DATA;
+	else
+		return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL |
+			I2C_FUNC_SMBUS_I2C_BLOCK;
 }
 
 static const struct i2c_algorithm mlxcpld_i2c_algo = {
@@ -440,6 +472,13 @@ static const struct i2c_adapter_quirks mlxcpld_i2c_quirks = {
 	.max_comb_1st_msg_len = 4,
 };
 
+static const struct i2c_adapter_quirks mlxcpld_i2c_quirks_ext = {
+	.flags = I2C_AQ_COMB_WRITE_THEN_READ,
+	.max_read_len = MLXCPLD_I2C_DATA_REG_SZ * 2 - MLXCPLD_I2C_MAX_ADDR_LEN,
+	.max_write_len = MLXCPLD_I2C_DATA_REG_SZ * 2,
+	.max_comb_1st_msg_len = 4,
+};
+
 static struct i2c_adapter mlxcpld_i2c_adapter = {
 	.owner          = THIS_MODULE,
 	.name           = "i2c-mlxcpld",
@@ -454,6 +493,7 @@ static int mlxcpld_i2c_probe(struct platform_device *pdev)
 {
 	struct mlxcpld_i2c_priv *priv;
 	int err;
+	u8 val;
 
 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -466,6 +506,16 @@ static int mlxcpld_i2c_probe(struct platform_device *pdev)
 
 	/* Register with i2c layer */
 	mlxcpld_i2c_adapter.timeout = usecs_to_jiffies(MLXCPLD_I2C_XFER_TO);
+	/* Read capability register */
+	mlxcpld_i2c_read_comm(priv, MLXCPLD_LPCI2C_CPBLTY_REG, &val, 1);
+	/* Check support for extended transaction length */
+	if ((val & MLXCPLD_I2C_DATA_SZ_MASK) == MLXCPLD_I2C_DATA_SZ_BIT)
+		mlxcpld_i2c_adapter.quirks = &mlxcpld_i2c_quirks_ext;
+	/* Check support for smbus block transaction */
+	if (val & MLXCPLD_I2C_SMBUS_BLK_BIT)
+		priv->smbus_block = true;
+	if (pdev->id >= -1)
+		mlxcpld_i2c_adapter.nr = pdev->id;
 	priv->adap = mlxcpld_i2c_adapter;
 	priv->adap.dev.parent = &pdev->dev;
 	priv->base_addr = MLXPLAT_CPLD_LPC_I2C_BASE_ADDR;
diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c
index cf23a746cc17..1e57f58fcb00 100644
--- a/drivers/i2c/busses/i2c-mt65xx.c
+++ b/drivers/i2c/busses/i2c-mt65xx.c
@@ -27,6 +27,7 @@
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
 #include <linux/scatterlist.h>
@@ -734,7 +735,6 @@ static int mtk_i2c_parse_dt(struct device_node *np, struct mtk_i2c *i2c)
 
 static int mtk_i2c_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *of_id;
 	int ret = 0;
 	struct mtk_i2c *i2c;
 	struct clk *clk;
@@ -761,11 +761,7 @@ static int mtk_i2c_probe(struct platform_device *pdev)
 
 	init_completion(&i2c->msg_complete);
 
-	of_id = of_match_node(mtk_i2c_of_match, pdev->dev.of_node);
-	if (!of_id)
-		return -EINVAL;
-
-	i2c->dev_comp = of_id->data;
+	i2c->dev_comp = of_device_get_match_data(&pdev->dev);
 	i2c->adap.dev.of_node = pdev->dev.of_node;
 	i2c->dev = &pdev->dev;
 	i2c->adap.dev.parent = &pdev->dev;
diff --git a/drivers/i2c/busses/i2c-mxs.c b/drivers/i2c/busses/i2c-mxs.c
index e617bd600794..642c58946d8d 100644
--- a/drivers/i2c/busses/i2c-mxs.c
+++ b/drivers/i2c/busses/i2c-mxs.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * Freescale MXS I2C bus driver
  *
@@ -7,12 +8,6 @@
  * based on a (non-working) driver which was:
  *
  * Copyright (C) 2009-2010 Freescale Semiconductor, Inc. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
  */
 
 #include <linux/slab.h>
@@ -180,9 +175,10 @@ static int mxs_i2c_dma_setup_xfer(struct i2c_adapter *adap,
 	struct dma_async_tx_descriptor *desc;
 	struct mxs_i2c_dev *i2c = i2c_get_adapdata(adap);
 
+	i2c->addr_data = i2c_8bit_addr_from_msg(msg);
+
 	if (msg->flags & I2C_M_RD) {
 		i2c->dma_read = true;
-		i2c->addr_data = (msg->addr << 1) | I2C_SMBUS_READ;
 
 		/*
 		 * SELECT command.
@@ -240,7 +236,6 @@ static int mxs_i2c_dma_setup_xfer(struct i2c_adapter *adap,
 		}
 	} else {
 		i2c->dma_read = false;
-		i2c->addr_data = (msg->addr << 1) | I2C_SMBUS_WRITE;
 
 		/*
 		 * WRITE command.
@@ -371,7 +366,7 @@ static int mxs_i2c_pio_setup_xfer(struct i2c_adapter *adap,
 			struct i2c_msg *msg, uint32_t flags)
 {
 	struct mxs_i2c_dev *i2c = i2c_get_adapdata(adap);
-	uint32_t addr_data = msg->addr << 1;
+	uint32_t addr_data = i2c_8bit_addr_from_msg(msg);
 	uint32_t data = 0;
 	int i, ret, xlen = 0, xmit = 0;
 	uint32_t start;
@@ -411,8 +406,6 @@ static int mxs_i2c_pio_setup_xfer(struct i2c_adapter *adap,
 		 */
 		BUG_ON(msg->len > 4);
 
-		addr_data |= I2C_SMBUS_READ;
-
 		/* SELECT command. */
 		mxs_i2c_pio_trigger_write_cmd(i2c, MXS_CMD_I2C_SELECT,
 					      addr_data);
@@ -450,7 +443,6 @@ static int mxs_i2c_pio_setup_xfer(struct i2c_adapter *adap,
 		 * fast enough. It is possible to transfer arbitrary amount
 		 * of data using PIO write.
 		 */
-		addr_data |= I2C_SMBUS_WRITE;
 
 		/*
 		 * The LSB of data buffer is the first byte blasted across
diff --git a/drivers/i2c/busses/i2c-nomadik.c b/drivers/i2c/busses/i2c-nomadik.c
index 49c7c0c91486..0ed5a41804dc 100644
--- a/drivers/i2c/busses/i2c-nomadik.c
+++ b/drivers/i2c/busses/i2c-nomadik.c
@@ -1012,8 +1012,6 @@ static int nmk_i2c_probe(struct amba_device *adev, const struct amba_id *id)
 		goto err_no_mem;
 	}
 
-	pm_suspend_ignore_children(&adev->dev, true);
-
 	dev->clk = devm_clk_get(&adev->dev, NULL);
 	if (IS_ERR(dev->clk)) {
 		dev_err(&adev->dev, "could not get i2c clock\n");
diff --git a/drivers/i2c/busses/i2c-ocores.c b/drivers/i2c/busses/i2c-ocores.c
index 8c42ca7107b2..88444ef74943 100644
--- a/drivers/i2c/busses/i2c-ocores.c
+++ b/drivers/i2c/busses/i2c-ocores.c
@@ -1,6 +1,6 @@
 /*
  * i2c-ocores.c: I2C bus driver for OpenCores I2C controller
- * (http://www.opencores.org/projects.cgi/web/i2c/overview).
+ * (https://opencores.org/project/i2c/overview)
  *
  * Peter Korsgaard <jacmet@sunsite.dk>
  *
@@ -21,7 +21,7 @@
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/wait.h>
-#include <linux/i2c-ocores.h>
+#include <linux/platform_data/i2c-ocores.h>
 #include <linux/slab.h>
 #include <linux/io.h>
 #include <linux/log2.h>
@@ -222,10 +222,7 @@ static int ocores_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
 	i2c->nmsgs = num;
 	i2c->state = STATE_START;
 
-	oc_setreg(i2c, OCI2C_DATA,
-			(i2c->msg->addr << 1) |
-			((i2c->msg->flags & I2C_M_RD) ? 1:0));
-
+	oc_setreg(i2c, OCI2C_DATA, i2c_8bit_addr_from_msg(i2c->msg));
 	oc_setreg(i2c, OCI2C_CMD, OCI2C_CMD_START);
 
 	if (wait_event_timeout(i2c->wait, (i2c->state == STATE_ERROR) ||
diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c
index b9172f08fd05..65d06a819307 100644
--- a/drivers/i2c/busses/i2c-omap.c
+++ b/drivers/i2c/busses/i2c-omap.c
@@ -36,7 +36,7 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/slab.h>
-#include <linux/i2c-omap.h>
+#include <linux/platform_data/i2c-omap.h>
 #include <linux/pm_runtime.h>
 #include <linux/pinctrl/consumer.h>
 
diff --git a/drivers/i2c/busses/i2c-opal.c b/drivers/i2c/busses/i2c-opal.c
index 0aabb7eca0c5..dc2a23f4fb52 100644
--- a/drivers/i2c/busses/i2c-opal.c
+++ b/drivers/i2c/busses/i2c-opal.c
@@ -94,8 +94,6 @@ static int i2c_opal_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
 	 */
 	memset(&req, 0, sizeof(req));
 	switch(num) {
-	case 0:
-		return 0;
 	case 1:
 		req.type = (msgs[0].flags & I2C_M_RD) ?
 			OPAL_I2C_RAW_READ : OPAL_I2C_RAW_WRITE;
@@ -114,8 +112,6 @@ static int i2c_opal_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
 		req.size = cpu_to_be32(msgs[1].len);
 		req.buffer_ra = cpu_to_be64(__pa(msgs[1].buf));
 		break;
-	default:
-		return -EOPNOTSUPP;
 	}
 
 	rc = i2c_opal_send_request(opal_id, &req);
diff --git a/drivers/i2c/busses/i2c-pasemi.c b/drivers/i2c/busses/i2c-pasemi.c
index df1dbc92a024..55fd5c6f3cca 100644
--- a/drivers/i2c/busses/i2c-pasemi.c
+++ b/drivers/i2c/busses/i2c-pasemi.c
@@ -121,7 +121,7 @@ static int pasemi_i2c_xfer_msg(struct i2c_adapter *adapter,
 
 	read = msg->flags & I2C_M_RD ? 1 : 0;
 
-	TXFIFO_WR(smbus, MTXFIFO_START | (msg->addr << 1) | read);
+	TXFIFO_WR(smbus, MTXFIFO_START | i2c_8bit_addr_from_msg(msg));
 
 	if (read) {
 		TXFIFO_WR(smbus, msg->len | MTXFIFO_READ |
diff --git a/drivers/i2c/busses/i2c-pca-platform.c b/drivers/i2c/busses/i2c-pca-platform.c
index bc2707ffd409..de3fe6e828cb 100644
--- a/drivers/i2c/busses/i2c-pca-platform.c
+++ b/drivers/i2c/busses/i2c-pca-platform.c
@@ -20,7 +20,7 @@
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
 #include <linux/i2c-algo-pca.h>
-#include <linux/i2c-pca-platform.h>
+#include <linux/platform_data/i2c-pca-platform.h>
 #include <linux/gpio.h>
 #include <linux/gpio/consumer.h>
 #include <linux/io.h>
diff --git a/drivers/i2c/busses/i2c-pnx.c b/drivers/i2c/busses/i2c-pnx.c
index a542041df0cd..6e0e546ef83f 100644
--- a/drivers/i2c/busses/i2c-pnx.c
+++ b/drivers/i2c/busses/i2c-pnx.c
@@ -18,7 +18,6 @@
 #include <linux/timer.h>
 #include <linux/completion.h>
 #include <linux/platform_device.h>
-#include <linux/i2c-pnx.h>
 #include <linux/io.h>
 #include <linux/err.h>
 #include <linux/clk.h>
@@ -29,6 +28,26 @@
 #define I2C_PNX_SPEED_KHZ_DEFAULT	100
 #define I2C_PNX_REGION_SIZE		0x100
 
+struct i2c_pnx_mif {
+	int			ret;		/* Return value */
+	int			mode;		/* Interface mode */
+	struct completion	complete;	/* I/O completion */
+	struct timer_list	timer;		/* Timeout */
+	u8 *			buf;		/* Data buffer */
+	int			len;		/* Length of data buffer */
+	int			order;		/* RX Bytes to order via TX */
+};
+
+struct i2c_pnx_algo_data {
+	void __iomem		*ioaddr;
+	struct i2c_pnx_mif	mif;
+	int			last;
+	struct clk		*clk;
+	struct i2c_adapter	adapter;
+	int			irq;
+	u32			timeout;
+};
+
 enum {
 	mstatus_tdi = 0x00000001,
 	mstatus_afi = 0x00000002,
diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c
index 904dfec7ab96..4f793b5d0c3b 100644
--- a/drivers/i2c/busses/i2c-qup.c
+++ b/drivers/i2c/busses/i2c-qup.c
@@ -136,8 +136,13 @@
  */
 #define TOUT_MIN			2
 
+/* I2C Frequency Modes */
+#define I2C_STANDARD_FREQ		100000
+#define I2C_FAST_MODE_FREQ		400000
+#define I2C_FAST_MODE_PLUS_FREQ		1000000
+
 /* Default values. Use these if FW query fails */
-#define DEFAULT_CLK_FREQ 100000
+#define DEFAULT_CLK_FREQ I2C_STANDARD_FREQ
 #define DEFAULT_SRC_CLK 20000000
 
 /*
@@ -150,6 +155,10 @@
 /* TAG length for DATA READ in RX FIFO  */
 #define READ_RX_TAGS_LEN		2
 
+static unsigned int scl_freq;
+module_param_named(scl_freq, scl_freq, uint, 0444);
+MODULE_PARM_DESC(scl_freq, "SCL frequency override");
+
 /*
  * count: no of blocks
  * pos: current block number
@@ -453,7 +462,7 @@ static void qup_i2c_write_tx_fifo_v1(struct qup_i2c_dev *qup)
 {
 	struct qup_i2c_block *blk = &qup->blk;
 	struct i2c_msg *msg = qup->msg;
-	u32 addr = msg->addr << 1;
+	u32 addr = i2c_8bit_addr_from_msg(msg);
 	u32 qup_tag;
 	int idx;
 	u32 val;
@@ -1648,6 +1657,14 @@ static void qup_i2c_disable_clocks(struct qup_i2c_dev *qup)
 	clk_disable_unprepare(qup->pclk);
 }
 
+#if IS_ENABLED(CONFIG_ACPI)
+static const struct acpi_device_id qup_i2c_acpi_match[] = {
+	{ "QCOM8010"},
+	{ },
+};
+MODULE_DEVICE_TABLE(acpi, qup_i2c_acpi_match);
+#endif
+
 static int qup_i2c_probe(struct platform_device *pdev)
 {
 	static const int blk_sizes[] = {4, 16, 32};
@@ -1669,10 +1686,15 @@ static int qup_i2c_probe(struct platform_device *pdev)
 	init_completion(&qup->xfer);
 	platform_set_drvdata(pdev, qup);
 
-	ret = device_property_read_u32(qup->dev, "clock-frequency", &clk_freq);
-	if (ret) {
-		dev_notice(qup->dev, "using default clock-frequency %d",
-			DEFAULT_CLK_FREQ);
+	if (scl_freq) {
+		dev_notice(qup->dev, "Using override frequency of %u\n", scl_freq);
+		clk_freq = scl_freq;
+	} else {
+		ret = device_property_read_u32(qup->dev, "clock-frequency", &clk_freq);
+		if (ret) {
+			dev_notice(qup->dev, "using default clock-frequency %d",
+				DEFAULT_CLK_FREQ);
+		}
 	}
 
 	if (of_device_is_compatible(pdev->dev.of_node, "qcom,i2c-qup-v1.1.1")) {
@@ -1682,7 +1704,10 @@ static int qup_i2c_probe(struct platform_device *pdev)
 	} else {
 		qup->adap.algo = &qup_i2c_algo_v2;
 		is_qup_v1 = false;
-		ret = qup_i2c_req_dma(qup);
+		if (acpi_match_device(qup_i2c_acpi_match, qup->dev))
+			goto nodma;
+		else
+			ret = qup_i2c_req_dma(qup);
 
 		if (ret == -EPROBE_DEFER)
 			goto fail_dma;
@@ -1734,8 +1759,8 @@ static int qup_i2c_probe(struct platform_device *pdev)
 	}
 
 nodma:
-	/* We support frequencies up to FAST Mode (400KHz) */
-	if (!clk_freq || clk_freq > 400000) {
+	/* We support frequencies up to FAST Mode Plus (1MHz) */
+	if (!clk_freq || clk_freq > I2C_FAST_MODE_PLUS_FREQ) {
 		dev_err(qup->dev, "clock frequency not supported %d\n",
 			clk_freq);
 		return -EINVAL;
@@ -1839,9 +1864,15 @@ static int qup_i2c_probe(struct platform_device *pdev)
 	size = QUP_INPUT_FIFO_SIZE(io_mode);
 	qup->in_fifo_sz = qup->in_blk_sz * (2 << size);
 
-	fs_div = ((src_clk_freq / clk_freq) / 2) - 3;
 	hs_div = 3;
-	qup->clk_ctl = (hs_div << 8) | (fs_div & 0xff);
+	if (clk_freq <= I2C_STANDARD_FREQ) {
+		fs_div = ((src_clk_freq / clk_freq) / 2) - 3;
+		qup->clk_ctl = (hs_div << 8) | (fs_div & 0xff);
+	} else {
+		/* 33%/66% duty cycle */
+		fs_div = ((src_clk_freq / clk_freq) - 6) * 2 / 3;
+		qup->clk_ctl = ((fs_div / 2) << 16) | (hs_div << 8) | (fs_div & 0xff);
+	}
 
 	/*
 	 * Time it takes for a byte to be clocked out on the bus.
@@ -1959,14 +1990,6 @@ static const struct of_device_id qup_i2c_dt_match[] = {
 };
 MODULE_DEVICE_TABLE(of, qup_i2c_dt_match);
 
-#if IS_ENABLED(CONFIG_ACPI)
-static const struct acpi_device_id qup_i2c_acpi_match[] = {
-	{ "QCOM8010"},
-	{ },
-};
-MODULE_DEVICE_TABLE(acpi, qup_i2c_acpi_match);
-#endif
-
 static struct platform_driver qup_i2c_driver = {
 	.probe  = qup_i2c_probe,
 	.remove = qup_i2c_remove,
diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c
index c6915b835396..5e310efd9446 100644
--- a/drivers/i2c/busses/i2c-rcar.c
+++ b/drivers/i2c/busses/i2c-rcar.c
@@ -329,7 +329,7 @@ static void rcar_i2c_prepare_msg(struct rcar_i2c_priv *priv)
 	if (priv->msgs_left == 1)
 		priv->flags |= ID_LAST_MSG;
 
-	rcar_i2c_write(priv, ICMAR, (priv->msg->addr << 1) | read);
+	rcar_i2c_write(priv, ICMAR, i2c_8bit_addr_from_msg(priv->msg));
 	/*
 	 * We don't have a test case but the HW engineers say that the write order
 	 * of ICMSR and ICMCR depends on whether we issue START or REP_START. Since
@@ -542,6 +542,8 @@ static void rcar_i2c_irq_recv(struct rcar_i2c_priv *priv, u32 msr)
 	 * If next received data is the _LAST_, go to STOP phase. Might be
 	 * overwritten by REP START when setting up a new msg. Not elegant
 	 * but the only stable sequence for REP START I have found so far.
+	 * If you want to change this code, make sure sending one transfer with
+	 * four messages (WR-RD-WR-RD) works!
 	 */
 	if (priv->pos + 1 >= msg->len)
 		rcar_i2c_write(priv, ICMCR, RCAR_BUS_PHASE_STOP);
diff --git a/drivers/i2c/busses/i2c-riic.c b/drivers/i2c/busses/i2c-riic.c
index 95c2f1ce3cad..5f1fca7880b1 100644
--- a/drivers/i2c/busses/i2c-riic.c
+++ b/drivers/i2c/busses/i2c-riic.c
@@ -167,15 +167,14 @@ static irqreturn_t riic_tdre_isr(int irq, void *data)
 		return IRQ_NONE;
 
 	if (riic->bytes_left == RIIC_INIT_MSG) {
-		val = !!(riic->msg->flags & I2C_M_RD);
-		if (val)
+		if (riic->msg->flags & I2C_M_RD)
 			/* On read, switch over to receive interrupt */
 			riic_clear_set_bit(riic, ICIER_TIE, ICIER_RIE, RIIC_ICIER);
 		else
 			/* On write, initialize length */
 			riic->bytes_left = riic->msg->len;
 
-		val |= (riic->msg->addr << 1);
+		val = i2c_8bit_addr_from_msg(riic->msg);
 	} else {
 		val = *riic->buf;
 		riic->buf++;
diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c
index e1a18d989f83..b8a2728dd4b6 100644
--- a/drivers/i2c/busses/i2c-rk3x.c
+++ b/drivers/i2c/busses/i2c-rk3x.c
@@ -1326,8 +1326,6 @@ static int rk3x_i2c_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto err_clk_notifier;
 
-	dev_info(&pdev->dev, "Initialized RK3xxx I2C bus at %p\n", i2c->regs);
-
 	return 0;
 
 err_clk_notifier:
diff --git a/drivers/i2c/busses/i2c-robotfuzz-osif.c b/drivers/i2c/busses/i2c-robotfuzz-osif.c
index 9c0f52b7ff7e..d848cf515234 100644
--- a/drivers/i2c/busses/i2c-robotfuzz-osif.c
+++ b/drivers/i2c/busses/i2c-robotfuzz-osif.c
@@ -62,27 +62,24 @@ static int osif_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs,
 {
 	struct osif_priv *priv = adapter->algo_data;
 	struct i2c_msg *pmsg;
-	int ret = 0;
-	int i, cmd;
+	int ret;
+	int i;
 
-	for (i = 0; ret >= 0 && i < num; i++) {
+	for (i = 0; i < num; i++) {
 		pmsg = &msgs[i];
 
 		if (pmsg->flags & I2C_M_RD) {
-			cmd = OSIFI2C_READ;
-
-			ret = osif_usb_read(adapter, cmd, pmsg->flags,
-					    pmsg->addr, pmsg->buf,
-					    pmsg->len);
+			ret = osif_usb_read(adapter, OSIFI2C_READ,
+					    pmsg->flags, pmsg->addr,
+					    pmsg->buf, pmsg->len);
 			if (ret != pmsg->len) {
 				dev_err(&adapter->dev, "failure reading data\n");
 				return -EREMOTEIO;
 			}
 		} else {
-			cmd = OSIFI2C_WRITE;
-
-			ret = osif_usb_write(adapter, cmd, pmsg->flags,
-					     pmsg->addr, pmsg->buf, pmsg->len);
+			ret = osif_usb_write(adapter, OSIFI2C_WRITE,
+					     pmsg->flags, pmsg->addr,
+					     pmsg->buf, pmsg->len);
 			if (ret != pmsg->len) {
 				dev_err(&adapter->dev, "failure writing data\n");
 				return -EREMOTEIO;
diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c
index 5d97510ee48b..9fe2b6951895 100644
--- a/drivers/i2c/busses/i2c-s3c2410.c
+++ b/drivers/i2c/busses/i2c-s3c2410.c
@@ -154,8 +154,6 @@ static const struct of_device_id s3c24xx_i2c_match[] = {
 	{ .compatible = "samsung,s3c2440-i2c", .data = (void *)QUIRK_S3C2440 },
 	{ .compatible = "samsung,s3c2440-hdmiphy-i2c",
 	  .data = (void *)(QUIRK_S3C2440 | QUIRK_HDMIPHY | QUIRK_NO_GPIO) },
-	{ .compatible = "samsung,exynos5440-i2c",
-	  .data = (void *)(QUIRK_S3C2440 | QUIRK_NO_GPIO) },
 	{ .compatible = "samsung,exynos5-sata-phy-i2c",
 	  .data = (void *)(QUIRK_S3C2440 | QUIRK_POLL | QUIRK_NO_GPIO) },
 	{},
diff --git a/drivers/i2c/busses/i2c-sh_mobile.c b/drivers/i2c/busses/i2c-sh_mobile.c
index d856bc211715..5fda4188a9e5 100644
--- a/drivers/i2c/busses/i2c-sh_mobile.c
+++ b/drivers/i2c/busses/i2c-sh_mobile.c
@@ -899,17 +899,6 @@ static int sh_mobile_i2c_probe(struct platform_device *dev)
 	if (resource_size(res) > 0x17)
 		pd->flags |= IIC_FLAG_HAS_ICIC67;
 
-	/* Enable Runtime PM for this device.
-	 *
-	 * Also tell the Runtime PM core to ignore children
-	 * for this device since it is valid for us to suspend
-	 * this I2C master driver even though the slave devices
-	 * on the I2C bus may not be suspended.
-	 *
-	 * The state of the I2C hardware bus is unaffected by
-	 * the Runtime PM state.
-	 */
-	pm_suspend_ignore_children(&dev->dev, true);
 	pm_runtime_enable(&dev->dev);
 	pm_runtime_get_sync(&dev->dev);
 
diff --git a/drivers/i2c/busses/i2c-stm32.c b/drivers/i2c/busses/i2c-stm32.c
new file mode 100644
index 000000000000..d75fbcbf02ef
--- /dev/null
+++ b/drivers/i2c/busses/i2c-stm32.c
@@ -0,0 +1,153 @@
+/*
+ * i2c-stm32.c
+ *
+ * Copyright (C) M'boumba Cedric Madianga 2017
+ * Author: M'boumba Cedric Madianga <cedric.madianga@gmail.com>
+ *
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include "i2c-stm32.h"
+
+/* Functions for DMA support */
+struct stm32_i2c_dma *stm32_i2c_dma_request(struct device *dev,
+					    dma_addr_t phy_addr,
+					    u32 txdr_offset,
+					    u32 rxdr_offset)
+{
+	struct stm32_i2c_dma *dma;
+	struct dma_slave_config dma_sconfig;
+	int ret;
+
+	dma = devm_kzalloc(dev, sizeof(*dma), GFP_KERNEL);
+	if (!dma)
+		return NULL;
+
+	/* Request and configure I2C TX dma channel */
+	dma->chan_tx = dma_request_slave_channel(dev, "tx");
+	if (!dma->chan_tx) {
+		dev_dbg(dev, "can't request DMA tx channel\n");
+		ret = -EINVAL;
+		goto fail_al;
+	}
+
+	memset(&dma_sconfig, 0, sizeof(dma_sconfig));
+	dma_sconfig.dst_addr = phy_addr + txdr_offset;
+	dma_sconfig.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+	dma_sconfig.dst_maxburst = 1;
+	dma_sconfig.direction = DMA_MEM_TO_DEV;
+	ret = dmaengine_slave_config(dma->chan_tx, &dma_sconfig);
+	if (ret < 0) {
+		dev_err(dev, "can't configure tx channel\n");
+		goto fail_tx;
+	}
+
+	/* Request and configure I2C RX dma channel */
+	dma->chan_rx = dma_request_slave_channel(dev, "rx");
+	if (!dma->chan_rx) {
+		dev_err(dev, "can't request DMA rx channel\n");
+		ret = -EINVAL;
+		goto fail_tx;
+	}
+
+	memset(&dma_sconfig, 0, sizeof(dma_sconfig));
+	dma_sconfig.src_addr = phy_addr + rxdr_offset;
+	dma_sconfig.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+	dma_sconfig.src_maxburst = 1;
+	dma_sconfig.direction = DMA_DEV_TO_MEM;
+	ret = dmaengine_slave_config(dma->chan_rx, &dma_sconfig);
+	if (ret < 0) {
+		dev_err(dev, "can't configure rx channel\n");
+		goto fail_rx;
+	}
+
+	init_completion(&dma->dma_complete);
+
+	dev_info(dev, "using %s (tx) and %s (rx) for DMA transfers\n",
+		 dma_chan_name(dma->chan_tx), dma_chan_name(dma->chan_rx));
+
+	return dma;
+
+fail_rx:
+	dma_release_channel(dma->chan_rx);
+fail_tx:
+	dma_release_channel(dma->chan_tx);
+fail_al:
+	devm_kfree(dev, dma);
+	dev_info(dev, "can't use DMA\n");
+
+	return NULL;
+}
+
+void stm32_i2c_dma_free(struct stm32_i2c_dma *dma)
+{
+	dma->dma_buf = 0;
+	dma->dma_len = 0;
+
+	dma_release_channel(dma->chan_tx);
+	dma->chan_tx = NULL;
+
+	dma_release_channel(dma->chan_rx);
+	dma->chan_rx = NULL;
+
+	dma->chan_using = NULL;
+}
+
+int stm32_i2c_prep_dma_xfer(struct device *dev, struct stm32_i2c_dma *dma,
+			    bool rd_wr, u32 len, u8 *buf,
+			    dma_async_tx_callback callback,
+			    void *dma_async_param)
+{
+	struct dma_async_tx_descriptor *txdesc;
+	struct device *chan_dev;
+	int ret;
+
+	if (rd_wr) {
+		dma->chan_using = dma->chan_rx;
+		dma->dma_transfer_dir = DMA_DEV_TO_MEM;
+		dma->dma_data_dir = DMA_FROM_DEVICE;
+	} else {
+		dma->chan_using = dma->chan_tx;
+		dma->dma_transfer_dir = DMA_MEM_TO_DEV;
+		dma->dma_data_dir = DMA_TO_DEVICE;
+	}
+
+	dma->dma_len = len;
+	chan_dev = dma->chan_using->device->dev;
+
+	dma->dma_buf = dma_map_single(chan_dev, buf, dma->dma_len,
+				      dma->dma_data_dir);
+	if (dma_mapping_error(chan_dev, dma->dma_buf)) {
+		dev_err(dev, "DMA mapping failed\n");
+		return -EINVAL;
+	}
+
+	txdesc = dmaengine_prep_slave_single(dma->chan_using, dma->dma_buf,
+					     dma->dma_len,
+					     dma->dma_transfer_dir,
+					     DMA_PREP_INTERRUPT);
+	if (!txdesc) {
+		dev_err(dev, "Not able to get desc for DMA xfer\n");
+		ret = -EINVAL;
+		goto err;
+	}
+
+	reinit_completion(&dma->dma_complete);
+
+	txdesc->callback = callback;
+	txdesc->callback_param = dma_async_param;
+	ret = dma_submit_error(dmaengine_submit(txdesc));
+	if (ret < 0) {
+		dev_err(dev, "DMA submit failed\n");
+		goto err;
+	}
+
+	dma_async_issue_pending(dma->chan_using);
+
+	return 0;
+
+err:
+	dma_unmap_single(chan_dev, dma->dma_buf, dma->dma_len,
+			 dma->dma_data_dir);
+	return ret;
+}
diff --git a/drivers/i2c/busses/i2c-stm32.h b/drivers/i2c/busses/i2c-stm32.h
index d4f9cef251ac..868755f82f88 100644
--- a/drivers/i2c/busses/i2c-stm32.h
+++ b/drivers/i2c/busses/i2c-stm32.h
@@ -11,6 +11,10 @@
 #ifndef _I2C_STM32_H
 #define _I2C_STM32_H
 
+#include <linux/dma-direction.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+
 enum stm32_i2c_speed {
 	STM32_I2C_SPEED_STANDARD, /* 100 kHz */
 	STM32_I2C_SPEED_FAST, /* 400 kHz */
@@ -18,4 +22,37 @@ enum stm32_i2c_speed {
 	STM32_I2C_SPEED_END,
 };
 
+/**
+ * struct stm32_i2c_dma - DMA specific data
+ * @chan_tx: dma channel for TX transfer
+ * @chan_rx: dma channel for RX transfer
+ * @chan_using: dma channel used for the current transfer (TX or RX)
+ * @dma_buf: dma buffer
+ * @dma_len: dma buffer len
+ * @dma_transfer_dir: dma transfer direction indicator
+ * @dma_data_dir: dma transfer mode indicator
+ * @dma_complete: dma transfer completion
+ */
+struct stm32_i2c_dma {
+	struct dma_chan *chan_tx;
+	struct dma_chan *chan_rx;
+	struct dma_chan *chan_using;
+	dma_addr_t dma_buf;
+	unsigned int dma_len;
+	enum dma_transfer_direction dma_transfer_dir;
+	enum dma_data_direction dma_data_dir;
+	struct completion dma_complete;
+};
+
+struct stm32_i2c_dma *stm32_i2c_dma_request(struct device *dev,
+					    dma_addr_t phy_addr,
+					    u32 txdr_offset, u32 rxdr_offset);
+
+void stm32_i2c_dma_free(struct stm32_i2c_dma *dma);
+
+int stm32_i2c_prep_dma_xfer(struct device *dev, struct stm32_i2c_dma *dma,
+			    bool rd_wr, u32 len, u8 *buf,
+			    dma_async_tx_callback callback,
+			    void *dma_async_param);
+
 #endif /* _I2C_STM32_H */
diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c
index f273e28c39db..62d023e737d9 100644
--- a/drivers/i2c/busses/i2c-stm32f7.c
+++ b/drivers/i2c/busses/i2c-stm32f7.c
@@ -35,6 +35,9 @@
 /* STM32F7 I2C registers */
 #define STM32F7_I2C_CR1				0x00
 #define STM32F7_I2C_CR2				0x04
+#define STM32F7_I2C_OAR1			0x08
+#define STM32F7_I2C_OAR2			0x0C
+#define STM32F7_I2C_PECR			0x20
 #define STM32F7_I2C_TIMINGR			0x10
 #define STM32F7_I2C_ISR				0x18
 #define STM32F7_I2C_ICR				0x1C
@@ -42,6 +45,10 @@
 #define STM32F7_I2C_TXDR			0x28
 
 /* STM32F7 I2C control 1 */
+#define STM32F7_I2C_CR1_PECEN			BIT(23)
+#define STM32F7_I2C_CR1_SBC			BIT(16)
+#define STM32F7_I2C_CR1_RXDMAEN			BIT(15)
+#define STM32F7_I2C_CR1_TXDMAEN			BIT(14)
 #define STM32F7_I2C_CR1_ANFOFF			BIT(12)
 #define STM32F7_I2C_CR1_ERRIE			BIT(7)
 #define STM32F7_I2C_CR1_TCIE			BIT(6)
@@ -57,34 +64,77 @@
 						| STM32F7_I2C_CR1_NACKIE \
 						| STM32F7_I2C_CR1_RXIE \
 						| STM32F7_I2C_CR1_TXIE)
+#define STM32F7_I2C_XFER_IRQ_MASK		(STM32F7_I2C_CR1_TCIE \
+						| STM32F7_I2C_CR1_STOPIE \
+						| STM32F7_I2C_CR1_NACKIE \
+						| STM32F7_I2C_CR1_RXIE \
+						| STM32F7_I2C_CR1_TXIE)
 
 /* STM32F7 I2C control 2 */
+#define STM32F7_I2C_CR2_PECBYTE			BIT(26)
 #define STM32F7_I2C_CR2_RELOAD			BIT(24)
 #define STM32F7_I2C_CR2_NBYTES_MASK		GENMASK(23, 16)
 #define STM32F7_I2C_CR2_NBYTES(n)		(((n) & 0xff) << 16)
 #define STM32F7_I2C_CR2_NACK			BIT(15)
 #define STM32F7_I2C_CR2_STOP			BIT(14)
 #define STM32F7_I2C_CR2_START			BIT(13)
+#define STM32F7_I2C_CR2_HEAD10R			BIT(12)
+#define STM32F7_I2C_CR2_ADD10			BIT(11)
 #define STM32F7_I2C_CR2_RD_WRN			BIT(10)
+#define STM32F7_I2C_CR2_SADD10_MASK		GENMASK(9, 0)
+#define STM32F7_I2C_CR2_SADD10(n)		(((n) & \
+						STM32F7_I2C_CR2_SADD10_MASK))
 #define STM32F7_I2C_CR2_SADD7_MASK		GENMASK(7, 1)
 #define STM32F7_I2C_CR2_SADD7(n)		(((n) & 0x7f) << 1)
 
+/* STM32F7 I2C Own Address 1 */
+#define STM32F7_I2C_OAR1_OA1EN			BIT(15)
+#define STM32F7_I2C_OAR1_OA1MODE		BIT(10)
+#define STM32F7_I2C_OAR1_OA1_10_MASK		GENMASK(9, 0)
+#define STM32F7_I2C_OAR1_OA1_10(n)		(((n) & \
+						STM32F7_I2C_OAR1_OA1_10_MASK))
+#define STM32F7_I2C_OAR1_OA1_7_MASK		GENMASK(7, 1)
+#define STM32F7_I2C_OAR1_OA1_7(n)		(((n) & 0x7f) << 1)
+#define STM32F7_I2C_OAR1_MASK			(STM32F7_I2C_OAR1_OA1_7_MASK \
+						| STM32F7_I2C_OAR1_OA1_10_MASK \
+						| STM32F7_I2C_OAR1_OA1EN \
+						| STM32F7_I2C_OAR1_OA1MODE)
+
+/* STM32F7 I2C Own Address 2 */
+#define STM32F7_I2C_OAR2_OA2EN			BIT(15)
+#define STM32F7_I2C_OAR2_OA2MSK_MASK		GENMASK(10, 8)
+#define STM32F7_I2C_OAR2_OA2MSK(n)		(((n) & 0x7) << 8)
+#define STM32F7_I2C_OAR2_OA2_7_MASK		GENMASK(7, 1)
+#define STM32F7_I2C_OAR2_OA2_7(n)		(((n) & 0x7f) << 1)
+#define STM32F7_I2C_OAR2_MASK			(STM32F7_I2C_OAR2_OA2MSK_MASK \
+						| STM32F7_I2C_OAR2_OA2_7_MASK \
+						| STM32F7_I2C_OAR2_OA2EN)
+
 /* STM32F7 I2C Interrupt Status */
+#define STM32F7_I2C_ISR_ADDCODE_MASK		GENMASK(23, 17)
+#define STM32F7_I2C_ISR_ADDCODE_GET(n) \
+				(((n) & STM32F7_I2C_ISR_ADDCODE_MASK) >> 17)
+#define STM32F7_I2C_ISR_DIR			BIT(16)
 #define STM32F7_I2C_ISR_BUSY			BIT(15)
+#define STM32F7_I2C_ISR_PECERR			BIT(11)
 #define STM32F7_I2C_ISR_ARLO			BIT(9)
 #define STM32F7_I2C_ISR_BERR			BIT(8)
 #define STM32F7_I2C_ISR_TCR			BIT(7)
 #define STM32F7_I2C_ISR_TC			BIT(6)
 #define STM32F7_I2C_ISR_STOPF			BIT(5)
 #define STM32F7_I2C_ISR_NACKF			BIT(4)
+#define STM32F7_I2C_ISR_ADDR			BIT(3)
 #define STM32F7_I2C_ISR_RXNE			BIT(2)
 #define STM32F7_I2C_ISR_TXIS			BIT(1)
+#define STM32F7_I2C_ISR_TXE			BIT(0)
 
 /* STM32F7 I2C Interrupt Clear */
+#define STM32F7_I2C_ICR_PECCF			BIT(11)
 #define STM32F7_I2C_ICR_ARLOCF			BIT(9)
 #define STM32F7_I2C_ICR_BERRCF			BIT(8)
 #define STM32F7_I2C_ICR_STOPCF			BIT(5)
 #define STM32F7_I2C_ICR_NACKCF			BIT(4)
+#define STM32F7_I2C_ICR_ADDRCF			BIT(3)
 
 /* STM32F7 I2C Timing */
 #define STM32F7_I2C_TIMINGR_PRESC(n)		(((n) & 0xf) << 28)
@@ -94,6 +144,8 @@
 #define STM32F7_I2C_TIMINGR_SCLL(n)		((n) & 0xff)
 
 #define STM32F7_I2C_MAX_LEN			0xff
+#define STM32F7_I2C_DMA_LEN_MIN			0x16
+#define STM32F7_I2C_MAX_SLAVE			0x2
 
 #define STM32F7_I2C_DNF_DEFAULT			0
 #define STM32F7_I2C_DNF_MAX			16
@@ -159,11 +211,12 @@ struct stm32f7_i2c_setup {
 
 /**
  * struct stm32f7_i2c_timings - private I2C output parameters
- * @prec: Prescaler value
+ * @node: List entry
+ * @presc: Prescaler value
  * @scldel: Data setup time
  * @sdadel: Data hold time
  * @sclh: SCL high period (master mode)
- * @sclh: SCL low period (master mode)
+ * @scll: SCL low period (master mode)
  */
 struct stm32f7_i2c_timings {
 	struct list_head node;
@@ -176,18 +229,30 @@ struct stm32f7_i2c_timings {
 
 /**
  * struct stm32f7_i2c_msg - client specific data
- * @addr: 8-bit slave addr, including r/w bit
+ * @addr: 8-bit or 10-bit slave addr, including r/w bit
  * @count: number of bytes to be transferred
  * @buf: data buffer
  * @result: result of the transfer
  * @stop: last I2C msg to be sent, i.e. STOP to be generated
+ * @smbus: boolean to know if the I2C IP is used in SMBus mode
+ * @size: type of SMBus protocol
+ * @read_write: direction of SMBus protocol
+ * SMBus block read and SMBus block write - block read process call protocols
+ * @smbus_buf: buffer to be used for SMBus protocol transfer. It will
+ * contain a maximum of 32 bytes of data + byte command + byte count + PEC
+ * This buffer has to be 32-bit aligned to be compliant with memory address
+ * register in DMA mode.
  */
 struct stm32f7_i2c_msg {
-	u8 addr;
+	u16 addr;
 	u32 count;
 	u8 *buf;
 	int result;
 	bool stop;
+	bool smbus;
+	int size;
+	char read_write;
+	u8 smbus_buf[I2C_SMBUS_BLOCK_MAX + 3] __aligned(4);
 };
 
 /**
@@ -204,6 +269,13 @@ struct stm32f7_i2c_msg {
  * @f7_msg: customized i2c msg for driver usage
  * @setup: I2C timing input setup
  * @timing: I2C computed timings
+ * @slave: list of slave devices registered on the I2C bus
+ * @slave_running: slave device currently used
+ * @slave_dir: transfer direction for the current slave device
+ * @master_mode: boolean to know in which mode the I2C is running (master or
+ * slave)
+ * @dma: dma data
+ * @use_dma: boolean to know if dma is used in the current transfer
  */
 struct stm32f7_i2c_dev {
 	struct i2c_adapter adap;
@@ -218,6 +290,12 @@ struct stm32f7_i2c_dev {
 	struct stm32f7_i2c_msg f7_msg;
 	struct stm32f7_i2c_setup setup;
 	struct stm32f7_i2c_timings timing;
+	struct i2c_client *slave[STM32F7_I2C_MAX_SLAVE];
+	struct i2c_client *slave_running;
+	u32 slave_dir;
+	bool master_mode;
+	struct stm32_i2c_dma *dma;
+	bool use_dma;
 };
 
 /**
@@ -283,6 +361,11 @@ static inline void stm32f7_i2c_clr_bits(void __iomem *reg, u32 mask)
 	writel_relaxed(readl_relaxed(reg) & ~mask, reg);
 }
 
+static void stm32f7_i2c_disable_irq(struct stm32f7_i2c_dev *i2c_dev, u32 mask)
+{
+	stm32f7_i2c_clr_bits(i2c_dev->base + STM32F7_I2C_CR1, mask);
+}
+
 static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev,
 				      struct stm32f7_i2c_setup *setup,
 				      struct stm32f7_i2c_timings *output)
@@ -524,6 +607,25 @@ static int stm32f7_i2c_setup_timing(struct stm32f7_i2c_dev *i2c_dev,
 	return 0;
 }
 
+static void stm32f7_i2c_disable_dma_req(struct stm32f7_i2c_dev *i2c_dev)
+{
+	void __iomem *base = i2c_dev->base;
+	u32 mask = STM32F7_I2C_CR1_RXDMAEN | STM32F7_I2C_CR1_TXDMAEN;
+
+	stm32f7_i2c_clr_bits(base + STM32F7_I2C_CR1, mask);
+}
+
+static void stm32f7_i2c_dma_callback(void *arg)
+{
+	struct stm32f7_i2c_dev *i2c_dev = (struct stm32f7_i2c_dev *)arg;
+	struct stm32_i2c_dma *dma = i2c_dev->dma;
+	struct device *dev = dma->chan_using->device->dev;
+
+	stm32f7_i2c_disable_dma_req(i2c_dev);
+	dma_unmap_single(dev, dma->dma_buf, dma->dma_len, dma->dma_data_dir);
+	complete(&dma->dma_complete);
+}
+
 static void stm32f7_i2c_hw_config(struct stm32f7_i2c_dev *i2c_dev)
 {
 	struct stm32f7_i2c_timings *t = &i2c_dev->timing;
@@ -567,6 +669,9 @@ static void stm32f7_i2c_read_rx_data(struct stm32f7_i2c_dev *i2c_dev)
 	if (f7_msg->count) {
 		*f7_msg->buf++ = readb_relaxed(base + STM32F7_I2C_RXDR);
 		f7_msg->count--;
+	} else {
+		/* Flush RX buffer has no data is expected */
+		readb_relaxed(base + STM32F7_I2C_RXDR);
 	}
 }
 
@@ -575,6 +680,9 @@ static void stm32f7_i2c_reload(struct stm32f7_i2c_dev *i2c_dev)
 	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
 	u32 cr2;
 
+	if (i2c_dev->use_dma)
+		f7_msg->count -= STM32F7_I2C_MAX_LEN;
+
 	cr2 = readl_relaxed(i2c_dev->base + STM32F7_I2C_CR2);
 
 	cr2 &= ~STM32F7_I2C_CR2_NBYTES_MASK;
@@ -588,6 +696,43 @@ static void stm32f7_i2c_reload(struct stm32f7_i2c_dev *i2c_dev)
 	writel_relaxed(cr2, i2c_dev->base + STM32F7_I2C_CR2);
 }
 
+static void stm32f7_i2c_smbus_reload(struct stm32f7_i2c_dev *i2c_dev)
+{
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	u32 cr2;
+	u8 *val;
+
+	/*
+	 * For I2C_SMBUS_BLOCK_DATA && I2C_SMBUS_BLOCK_PROC_CALL, the first
+	 * data received inform us how many data will follow.
+	 */
+	stm32f7_i2c_read_rx_data(i2c_dev);
+
+	/*
+	 * Update NBYTES with the value read to continue the transfer
+	 */
+	val = f7_msg->buf - sizeof(u8);
+	f7_msg->count = *val;
+	cr2 = readl_relaxed(i2c_dev->base + STM32F7_I2C_CR2);
+	cr2 &= ~(STM32F7_I2C_CR2_NBYTES_MASK | STM32F7_I2C_CR2_RELOAD);
+	cr2 |= STM32F7_I2C_CR2_NBYTES(f7_msg->count);
+	writel_relaxed(cr2, i2c_dev->base + STM32F7_I2C_CR2);
+}
+
+static int stm32f7_i2c_release_bus(struct i2c_adapter *i2c_adap)
+{
+	struct stm32f7_i2c_dev *i2c_dev = i2c_get_adapdata(i2c_adap);
+
+	dev_info(i2c_dev->dev, "Trying to recover bus\n");
+
+	stm32f7_i2c_clr_bits(i2c_dev->base + STM32F7_I2C_CR1,
+			     STM32F7_I2C_CR1_PE);
+
+	stm32f7_i2c_hw_config(i2c_dev);
+
+	return 0;
+}
+
 static int stm32f7_i2c_wait_free_bus(struct stm32f7_i2c_dev *i2c_dev)
 {
 	u32 status;
@@ -597,12 +742,18 @@ static int stm32f7_i2c_wait_free_bus(struct stm32f7_i2c_dev *i2c_dev)
 					 status,
 					 !(status & STM32F7_I2C_ISR_BUSY),
 					 10, 1000);
+	if (!ret)
+		return 0;
+
+	dev_info(i2c_dev->dev, "bus busy\n");
+
+	ret = stm32f7_i2c_release_bus(&i2c_dev->adap);
 	if (ret) {
-		dev_dbg(i2c_dev->dev, "bus busy\n");
-		ret = -EBUSY;
+		dev_err(i2c_dev->dev, "Failed to recover the bus (%d)\n", ret);
+		return ret;
 	}
 
-	return ret;
+	return -EBUSY;
 }
 
 static void stm32f7_i2c_xfer_msg(struct stm32f7_i2c_dev *i2c_dev,
@@ -611,6 +762,7 @@ static void stm32f7_i2c_xfer_msg(struct stm32f7_i2c_dev *i2c_dev,
 	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
 	void __iomem *base = i2c_dev->base;
 	u32 cr1, cr2;
+	int ret;
 
 	f7_msg->addr = msg->addr;
 	f7_msg->buf = msg->buf;
@@ -629,8 +781,15 @@ static void stm32f7_i2c_xfer_msg(struct stm32f7_i2c_dev *i2c_dev,
 		cr2 |= STM32F7_I2C_CR2_RD_WRN;
 
 	/* Set slave address */
-	cr2 &= ~STM32F7_I2C_CR2_SADD7_MASK;
-	cr2 |= STM32F7_I2C_CR2_SADD7(f7_msg->addr);
+	cr2 &= ~(STM32F7_I2C_CR2_HEAD10R | STM32F7_I2C_CR2_ADD10);
+	if (msg->flags & I2C_M_TEN) {
+		cr2 &= ~STM32F7_I2C_CR2_SADD10_MASK;
+		cr2 |= STM32F7_I2C_CR2_SADD10(f7_msg->addr);
+		cr2 |= STM32F7_I2C_CR2_ADD10;
+	} else {
+		cr2 &= ~STM32F7_I2C_CR2_SADD7_MASK;
+		cr2 |= STM32F7_I2C_CR2_SADD7(f7_msg->addr);
+	}
 
 	/* Set nb bytes to transfer and reload if needed */
 	cr2 &= ~(STM32F7_I2C_CR2_NBYTES_MASK | STM32F7_I2C_CR2_RELOAD);
@@ -645,16 +804,286 @@ static void stm32f7_i2c_xfer_msg(struct stm32f7_i2c_dev *i2c_dev,
 	cr1 |= STM32F7_I2C_CR1_ERRIE | STM32F7_I2C_CR1_TCIE |
 		STM32F7_I2C_CR1_STOPIE | STM32F7_I2C_CR1_NACKIE;
 
-	/* Clear TX/RX interrupt */
+	/* Clear DMA req and TX/RX interrupt */
+	cr1 &= ~(STM32F7_I2C_CR1_RXIE | STM32F7_I2C_CR1_TXIE |
+			STM32F7_I2C_CR1_RXDMAEN | STM32F7_I2C_CR1_TXDMAEN);
+
+	/* Configure DMA or enable RX/TX interrupt */
+	i2c_dev->use_dma = false;
+	if (i2c_dev->dma && f7_msg->count >= STM32F7_I2C_DMA_LEN_MIN) {
+		ret = stm32_i2c_prep_dma_xfer(i2c_dev->dev, i2c_dev->dma,
+					      msg->flags & I2C_M_RD,
+					      f7_msg->count, f7_msg->buf,
+					      stm32f7_i2c_dma_callback,
+					      i2c_dev);
+		if (!ret)
+			i2c_dev->use_dma = true;
+		else
+			dev_warn(i2c_dev->dev, "can't use DMA\n");
+	}
+
+	if (!i2c_dev->use_dma) {
+		if (msg->flags & I2C_M_RD)
+			cr1 |= STM32F7_I2C_CR1_RXIE;
+		else
+			cr1 |= STM32F7_I2C_CR1_TXIE;
+	} else {
+		if (msg->flags & I2C_M_RD)
+			cr1 |= STM32F7_I2C_CR1_RXDMAEN;
+		else
+			cr1 |= STM32F7_I2C_CR1_TXDMAEN;
+	}
+
+	/* Configure Start/Repeated Start */
+	cr2 |= STM32F7_I2C_CR2_START;
+
+	i2c_dev->master_mode = true;
+
+	/* Write configurations registers */
+	writel_relaxed(cr1, base + STM32F7_I2C_CR1);
+	writel_relaxed(cr2, base + STM32F7_I2C_CR2);
+}
+
+static int stm32f7_i2c_smbus_xfer_msg(struct stm32f7_i2c_dev *i2c_dev,
+				      unsigned short flags, u8 command,
+				      union i2c_smbus_data *data)
+{
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	struct device *dev = i2c_dev->dev;
+	void __iomem *base = i2c_dev->base;
+	u32 cr1, cr2;
+	int i, ret;
+
+	f7_msg->result = 0;
+	reinit_completion(&i2c_dev->complete);
+
+	cr2 = readl_relaxed(base + STM32F7_I2C_CR2);
+	cr1 = readl_relaxed(base + STM32F7_I2C_CR1);
+
+	/* Set transfer direction */
+	cr2 &= ~STM32F7_I2C_CR2_RD_WRN;
+	if (f7_msg->read_write)
+		cr2 |= STM32F7_I2C_CR2_RD_WRN;
+
+	/* Set slave address */
+	cr2 &= ~(STM32F7_I2C_CR2_ADD10 | STM32F7_I2C_CR2_SADD7_MASK);
+	cr2 |= STM32F7_I2C_CR2_SADD7(f7_msg->addr);
+
+	f7_msg->smbus_buf[0] = command;
+	switch (f7_msg->size) {
+	case I2C_SMBUS_QUICK:
+		f7_msg->stop = true;
+		f7_msg->count = 0;
+		break;
+	case I2C_SMBUS_BYTE:
+		f7_msg->stop = true;
+		f7_msg->count = 1;
+		break;
+	case I2C_SMBUS_BYTE_DATA:
+		if (f7_msg->read_write) {
+			f7_msg->stop = false;
+			f7_msg->count = 1;
+			cr2 &= ~STM32F7_I2C_CR2_RD_WRN;
+		} else {
+			f7_msg->stop = true;
+			f7_msg->count = 2;
+			f7_msg->smbus_buf[1] = data->byte;
+		}
+		break;
+	case I2C_SMBUS_WORD_DATA:
+		if (f7_msg->read_write) {
+			f7_msg->stop = false;
+			f7_msg->count = 1;
+			cr2 &= ~STM32F7_I2C_CR2_RD_WRN;
+		} else {
+			f7_msg->stop = true;
+			f7_msg->count = 3;
+			f7_msg->smbus_buf[1] = data->word & 0xff;
+			f7_msg->smbus_buf[2] = data->word >> 8;
+		}
+		break;
+	case I2C_SMBUS_BLOCK_DATA:
+		if (f7_msg->read_write) {
+			f7_msg->stop = false;
+			f7_msg->count = 1;
+			cr2 &= ~STM32F7_I2C_CR2_RD_WRN;
+		} else {
+			f7_msg->stop = true;
+			if (data->block[0] > I2C_SMBUS_BLOCK_MAX ||
+			    !data->block[0]) {
+				dev_err(dev, "Invalid block write size %d\n",
+					data->block[0]);
+				return -EINVAL;
+			}
+			f7_msg->count = data->block[0] + 2;
+			for (i = 1; i < f7_msg->count; i++)
+				f7_msg->smbus_buf[i] = data->block[i - 1];
+		}
+		break;
+	case I2C_SMBUS_PROC_CALL:
+		f7_msg->stop = false;
+		f7_msg->count = 3;
+		f7_msg->smbus_buf[1] = data->word & 0xff;
+		f7_msg->smbus_buf[2] = data->word >> 8;
+		cr2 &= ~STM32F7_I2C_CR2_RD_WRN;
+		f7_msg->read_write = I2C_SMBUS_READ;
+		break;
+	case I2C_SMBUS_BLOCK_PROC_CALL:
+		f7_msg->stop = false;
+		if (data->block[0] > I2C_SMBUS_BLOCK_MAX - 1) {
+			dev_err(dev, "Invalid block write size %d\n",
+				data->block[0]);
+			return -EINVAL;
+		}
+		f7_msg->count = data->block[0] + 2;
+		for (i = 1; i < f7_msg->count; i++)
+			f7_msg->smbus_buf[i] = data->block[i - 1];
+		cr2 &= ~STM32F7_I2C_CR2_RD_WRN;
+		f7_msg->read_write = I2C_SMBUS_READ;
+		break;
+	default:
+		dev_err(dev, "Unsupported smbus protocol %d\n", f7_msg->size);
+		return -EOPNOTSUPP;
+	}
+
+	f7_msg->buf = f7_msg->smbus_buf;
+
+	/* Configure PEC */
+	if ((flags & I2C_CLIENT_PEC) && f7_msg->size != I2C_SMBUS_QUICK) {
+		cr1 |= STM32F7_I2C_CR1_PECEN;
+		cr2 |= STM32F7_I2C_CR2_PECBYTE;
+		if (!f7_msg->read_write)
+			f7_msg->count++;
+	} else {
+		cr1 &= ~STM32F7_I2C_CR1_PECEN;
+		cr2 &= ~STM32F7_I2C_CR2_PECBYTE;
+	}
+
+	/* Set number of bytes to be transferred */
+	cr2 &= ~(STM32F7_I2C_CR2_NBYTES_MASK | STM32F7_I2C_CR2_RELOAD);
+	cr2 |= STM32F7_I2C_CR2_NBYTES(f7_msg->count);
+
+	/* Enable NACK, STOP, error and transfer complete interrupts */
+	cr1 |= STM32F7_I2C_CR1_ERRIE | STM32F7_I2C_CR1_TCIE |
+		STM32F7_I2C_CR1_STOPIE | STM32F7_I2C_CR1_NACKIE;
+
+	/* Clear DMA req and TX/RX interrupt */
+	cr1 &= ~(STM32F7_I2C_CR1_RXIE | STM32F7_I2C_CR1_TXIE |
+			STM32F7_I2C_CR1_RXDMAEN | STM32F7_I2C_CR1_TXDMAEN);
+
+	/* Configure DMA or enable RX/TX interrupt */
+	i2c_dev->use_dma = false;
+	if (i2c_dev->dma && f7_msg->count >= STM32F7_I2C_DMA_LEN_MIN) {
+		ret = stm32_i2c_prep_dma_xfer(i2c_dev->dev, i2c_dev->dma,
+					      cr2 & STM32F7_I2C_CR2_RD_WRN,
+					      f7_msg->count, f7_msg->buf,
+					      stm32f7_i2c_dma_callback,
+					      i2c_dev);
+		if (!ret)
+			i2c_dev->use_dma = true;
+		else
+			dev_warn(i2c_dev->dev, "can't use DMA\n");
+	}
+
+	if (!i2c_dev->use_dma) {
+		if (cr2 & STM32F7_I2C_CR2_RD_WRN)
+			cr1 |= STM32F7_I2C_CR1_RXIE;
+		else
+			cr1 |= STM32F7_I2C_CR1_TXIE;
+	} else {
+		if (cr2 & STM32F7_I2C_CR2_RD_WRN)
+			cr1 |= STM32F7_I2C_CR1_RXDMAEN;
+		else
+			cr1 |= STM32F7_I2C_CR1_TXDMAEN;
+	}
+
+	/* Set Start bit */
+	cr2 |= STM32F7_I2C_CR2_START;
+
+	i2c_dev->master_mode = true;
+
+	/* Write configurations registers */
+	writel_relaxed(cr1, base + STM32F7_I2C_CR1);
+	writel_relaxed(cr2, base + STM32F7_I2C_CR2);
+
+	return 0;
+}
+
+static void stm32f7_i2c_smbus_rep_start(struct stm32f7_i2c_dev *i2c_dev)
+{
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	void __iomem *base = i2c_dev->base;
+	u32 cr1, cr2;
+	int ret;
+
+	cr2 = readl_relaxed(base + STM32F7_I2C_CR2);
+	cr1 = readl_relaxed(base + STM32F7_I2C_CR1);
+
+	/* Set transfer direction */
+	cr2 |= STM32F7_I2C_CR2_RD_WRN;
+
+	switch (f7_msg->size) {
+	case I2C_SMBUS_BYTE_DATA:
+		f7_msg->count = 1;
+		break;
+	case I2C_SMBUS_WORD_DATA:
+	case I2C_SMBUS_PROC_CALL:
+		f7_msg->count = 2;
+		break;
+	case I2C_SMBUS_BLOCK_DATA:
+	case I2C_SMBUS_BLOCK_PROC_CALL:
+		f7_msg->count = 1;
+		cr2 |= STM32F7_I2C_CR2_RELOAD;
+		break;
+	}
+
+	f7_msg->buf = f7_msg->smbus_buf;
+	f7_msg->stop = true;
+
+	/* Add one byte for PEC if needed */
+	if (cr1 & STM32F7_I2C_CR1_PECEN)
+		f7_msg->count++;
+
+	/* Set number of bytes to be transferred */
+	cr2 &= ~(STM32F7_I2C_CR2_NBYTES_MASK);
+	cr2 |= STM32F7_I2C_CR2_NBYTES(f7_msg->count);
+
+	/*
+	 * Configure RX/TX interrupt:
+	 */
 	cr1 &= ~(STM32F7_I2C_CR1_RXIE | STM32F7_I2C_CR1_TXIE);
+	cr1 |= STM32F7_I2C_CR1_RXIE;
 
-	/* Enable RX/TX interrupt according to msg direction */
-	if (msg->flags & I2C_M_RD)
+	/*
+	 * Configure DMA or enable RX/TX interrupt:
+	 * For I2C_SMBUS_BLOCK_DATA and I2C_SMBUS_BLOCK_PROC_CALL we don't use
+	 * dma as we don't know in advance how many data will be received
+	 */
+	cr1 &= ~(STM32F7_I2C_CR1_RXIE | STM32F7_I2C_CR1_TXIE |
+		 STM32F7_I2C_CR1_RXDMAEN | STM32F7_I2C_CR1_TXDMAEN);
+
+	i2c_dev->use_dma = false;
+	if (i2c_dev->dma && f7_msg->count >= STM32F7_I2C_DMA_LEN_MIN &&
+	    f7_msg->size != I2C_SMBUS_BLOCK_DATA &&
+	    f7_msg->size != I2C_SMBUS_BLOCK_PROC_CALL) {
+		ret = stm32_i2c_prep_dma_xfer(i2c_dev->dev, i2c_dev->dma,
+					      cr2 & STM32F7_I2C_CR2_RD_WRN,
+					      f7_msg->count, f7_msg->buf,
+					      stm32f7_i2c_dma_callback,
+					      i2c_dev);
+
+		if (!ret)
+			i2c_dev->use_dma = true;
+		else
+			dev_warn(i2c_dev->dev, "can't use DMA\n");
+	}
+
+	if (!i2c_dev->use_dma)
 		cr1 |= STM32F7_I2C_CR1_RXIE;
 	else
-		cr1 |= STM32F7_I2C_CR1_TXIE;
+		cr1 |= STM32F7_I2C_CR1_RXDMAEN;
 
-	/* Configure Start/Repeated Start */
+	/* Configure Repeated Start */
 	cr2 |= STM32F7_I2C_CR2_START;
 
 	/* Write configurations registers */
@@ -662,9 +1091,278 @@ static void stm32f7_i2c_xfer_msg(struct stm32f7_i2c_dev *i2c_dev,
 	writel_relaxed(cr2, base + STM32F7_I2C_CR2);
 }
 
-static void stm32f7_i2c_disable_irq(struct stm32f7_i2c_dev *i2c_dev, u32 mask)
+static int stm32f7_i2c_smbus_check_pec(struct stm32f7_i2c_dev *i2c_dev)
 {
-	stm32f7_i2c_clr_bits(i2c_dev->base + STM32F7_I2C_CR1, mask);
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	u8 count, internal_pec, received_pec;
+
+	internal_pec = readl_relaxed(i2c_dev->base + STM32F7_I2C_PECR);
+
+	switch (f7_msg->size) {
+	case I2C_SMBUS_BYTE:
+	case I2C_SMBUS_BYTE_DATA:
+		received_pec = f7_msg->smbus_buf[1];
+		break;
+	case I2C_SMBUS_WORD_DATA:
+	case I2C_SMBUS_PROC_CALL:
+		received_pec = f7_msg->smbus_buf[2];
+		break;
+	case I2C_SMBUS_BLOCK_DATA:
+	case I2C_SMBUS_BLOCK_PROC_CALL:
+		count = f7_msg->smbus_buf[0];
+		received_pec = f7_msg->smbus_buf[count];
+		break;
+	default:
+		dev_err(i2c_dev->dev, "Unsupported smbus protocol for PEC\n");
+		return -EINVAL;
+	}
+
+	if (internal_pec != received_pec) {
+		dev_err(i2c_dev->dev, "Bad PEC 0x%02x vs. 0x%02x\n",
+			internal_pec, received_pec);
+		return -EBADMSG;
+	}
+
+	return 0;
+}
+
+static bool stm32f7_i2c_is_addr_match(struct i2c_client *slave, u32 addcode)
+{
+	u32 addr;
+
+	if (!slave)
+		return false;
+
+	if (slave->flags & I2C_CLIENT_TEN) {
+		/*
+		 * For 10-bit addr, addcode = 11110XY with
+		 * X = Bit 9 of slave address
+		 * Y = Bit 8 of slave address
+		 */
+		addr = slave->addr >> 8;
+		addr |= 0x78;
+		if (addr == addcode)
+			return true;
+	} else {
+		addr = slave->addr & 0x7f;
+		if (addr == addcode)
+			return true;
+	}
+
+	return false;
+}
+
+static void stm32f7_i2c_slave_start(struct stm32f7_i2c_dev *i2c_dev)
+{
+	struct i2c_client *slave = i2c_dev->slave_running;
+	void __iomem *base = i2c_dev->base;
+	u32 mask;
+	u8 value = 0;
+
+	if (i2c_dev->slave_dir) {
+		/* Notify i2c slave that new read transfer is starting */
+		i2c_slave_event(slave, I2C_SLAVE_READ_REQUESTED, &value);
+
+		/*
+		 * Disable slave TX config in case of I2C combined message
+		 * (I2C Write followed by I2C Read)
+		 */
+		mask = STM32F7_I2C_CR2_RELOAD;
+		stm32f7_i2c_clr_bits(base + STM32F7_I2C_CR2, mask);
+		mask = STM32F7_I2C_CR1_SBC | STM32F7_I2C_CR1_RXIE |
+		       STM32F7_I2C_CR1_TCIE;
+		stm32f7_i2c_clr_bits(base + STM32F7_I2C_CR1, mask);
+
+		/* Enable TX empty, STOP, NACK interrupts */
+		mask =  STM32F7_I2C_CR1_STOPIE | STM32F7_I2C_CR1_NACKIE |
+			STM32F7_I2C_CR1_TXIE;
+		stm32f7_i2c_set_bits(base + STM32F7_I2C_CR1, mask);
+
+	} else {
+		/* Notify i2c slave that new write transfer is starting */
+		i2c_slave_event(slave, I2C_SLAVE_WRITE_REQUESTED, &value);
+
+		/* Set reload mode to be able to ACK/NACK each received byte */
+		mask = STM32F7_I2C_CR2_RELOAD;
+		stm32f7_i2c_set_bits(base + STM32F7_I2C_CR2, mask);
+
+		/*
+		 * Set STOP, NACK, RX empty and transfer complete interrupts.*
+		 * Set Slave Byte Control to be able to ACK/NACK each data
+		 * byte received
+		 */
+		mask =  STM32F7_I2C_CR1_STOPIE | STM32F7_I2C_CR1_NACKIE |
+			STM32F7_I2C_CR1_SBC | STM32F7_I2C_CR1_RXIE |
+			STM32F7_I2C_CR1_TCIE;
+		stm32f7_i2c_set_bits(base + STM32F7_I2C_CR1, mask);
+	}
+}
+
+static void stm32f7_i2c_slave_addr(struct stm32f7_i2c_dev *i2c_dev)
+{
+	void __iomem *base = i2c_dev->base;
+	u32 isr, addcode, dir, mask;
+	int i;
+
+	isr = readl_relaxed(i2c_dev->base + STM32F7_I2C_ISR);
+	addcode = STM32F7_I2C_ISR_ADDCODE_GET(isr);
+	dir = isr & STM32F7_I2C_ISR_DIR;
+
+	for (i = 0; i < STM32F7_I2C_MAX_SLAVE; i++) {
+		if (stm32f7_i2c_is_addr_match(i2c_dev->slave[i], addcode)) {
+			i2c_dev->slave_running = i2c_dev->slave[i];
+			i2c_dev->slave_dir = dir;
+
+			/* Start I2C slave processing */
+			stm32f7_i2c_slave_start(i2c_dev);
+
+			/* Clear ADDR flag */
+			mask = STM32F7_I2C_ICR_ADDRCF;
+			writel_relaxed(mask, base + STM32F7_I2C_ICR);
+			break;
+		}
+	}
+}
+
+static int stm32f7_i2c_get_slave_id(struct stm32f7_i2c_dev *i2c_dev,
+				    struct i2c_client *slave, int *id)
+{
+	int i;
+
+	for (i = 0; i < STM32F7_I2C_MAX_SLAVE; i++) {
+		if (i2c_dev->slave[i] == slave) {
+			*id = i;
+			return 0;
+		}
+	}
+
+	dev_err(i2c_dev->dev, "Slave 0x%x not registered\n", slave->addr);
+
+	return -ENODEV;
+}
+
+static int stm32f7_i2c_get_free_slave_id(struct stm32f7_i2c_dev *i2c_dev,
+					 struct i2c_client *slave, int *id)
+{
+	struct device *dev = i2c_dev->dev;
+	int i;
+
+	/*
+	 * slave[0] supports 7-bit and 10-bit slave address
+	 * slave[1] supports 7-bit slave address only
+	 */
+	for (i = 0; i < STM32F7_I2C_MAX_SLAVE; i++) {
+		if (i == 1 && (slave->flags & I2C_CLIENT_PEC))
+			continue;
+		if (!i2c_dev->slave[i]) {
+			*id = i;
+			return 0;
+		}
+	}
+
+	dev_err(dev, "Slave 0x%x could not be registered\n", slave->addr);
+
+	return -EINVAL;
+}
+
+static bool stm32f7_i2c_is_slave_registered(struct stm32f7_i2c_dev *i2c_dev)
+{
+	int i;
+
+	for (i = 0; i < STM32F7_I2C_MAX_SLAVE; i++) {
+		if (i2c_dev->slave[i])
+			return true;
+	}
+
+	return false;
+}
+
+static bool stm32f7_i2c_is_slave_busy(struct stm32f7_i2c_dev *i2c_dev)
+{
+	int i, busy;
+
+	busy = 0;
+	for (i = 0; i < STM32F7_I2C_MAX_SLAVE; i++) {
+		if (i2c_dev->slave[i])
+			busy++;
+	}
+
+	return i == busy;
+}
+
+static irqreturn_t stm32f7_i2c_slave_isr_event(struct stm32f7_i2c_dev *i2c_dev)
+{
+	void __iomem *base = i2c_dev->base;
+	u32 cr2, status, mask;
+	u8 val;
+	int ret;
+
+	status = readl_relaxed(i2c_dev->base + STM32F7_I2C_ISR);
+
+	/* Slave transmitter mode */
+	if (status & STM32F7_I2C_ISR_TXIS) {
+		i2c_slave_event(i2c_dev->slave_running,
+				I2C_SLAVE_READ_PROCESSED,
+				&val);
+
+		/* Write data byte */
+		writel_relaxed(val, base + STM32F7_I2C_TXDR);
+	}
+
+	/* Transfer Complete Reload for Slave receiver mode */
+	if (status & STM32F7_I2C_ISR_TCR || status & STM32F7_I2C_ISR_RXNE) {
+		/*
+		 * Read data byte then set NBYTES to receive next byte or NACK
+		 * the current received byte
+		 */
+		val = readb_relaxed(i2c_dev->base + STM32F7_I2C_RXDR);
+		ret = i2c_slave_event(i2c_dev->slave_running,
+				      I2C_SLAVE_WRITE_RECEIVED,
+				      &val);
+		if (!ret) {
+			cr2 = readl_relaxed(i2c_dev->base + STM32F7_I2C_CR2);
+			cr2 |= STM32F7_I2C_CR2_NBYTES(1);
+			writel_relaxed(cr2, i2c_dev->base + STM32F7_I2C_CR2);
+		} else {
+			mask = STM32F7_I2C_CR2_NACK;
+			stm32f7_i2c_set_bits(base + STM32F7_I2C_CR2, mask);
+		}
+	}
+
+	/* NACK received */
+	if (status & STM32F7_I2C_ISR_NACKF) {
+		dev_dbg(i2c_dev->dev, "<%s>: Receive NACK\n", __func__);
+		writel_relaxed(STM32F7_I2C_ICR_NACKCF, base + STM32F7_I2C_ICR);
+	}
+
+	/* STOP received */
+	if (status & STM32F7_I2C_ISR_STOPF) {
+		/* Disable interrupts */
+		stm32f7_i2c_disable_irq(i2c_dev, STM32F7_I2C_XFER_IRQ_MASK);
+
+		if (i2c_dev->slave_dir) {
+			/*
+			 * Flush TX buffer in order to not used the byte in
+			 * TXDR for the next transfer
+			 */
+			mask = STM32F7_I2C_ISR_TXE;
+			stm32f7_i2c_set_bits(base + STM32F7_I2C_ISR, mask);
+		}
+
+		/* Clear STOP flag */
+		writel_relaxed(STM32F7_I2C_ICR_STOPCF, base + STM32F7_I2C_ICR);
+
+		/* Notify i2c slave that a STOP flag has been detected */
+		i2c_slave_event(i2c_dev->slave_running, I2C_SLAVE_STOP, &val);
+
+		i2c_dev->slave_running = NULL;
+	}
+
+	/* Address match received */
+	if (status & STM32F7_I2C_ISR_ADDR)
+		stm32f7_i2c_slave_addr(i2c_dev);
+
+	return IRQ_HANDLED;
 }
 
 static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data)
@@ -673,6 +1371,13 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data)
 	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
 	void __iomem *base = i2c_dev->base;
 	u32 status, mask;
+	int ret = IRQ_HANDLED;
+
+	/* Check if the interrupt if for a slave device */
+	if (!i2c_dev->master_mode) {
+		ret = stm32f7_i2c_slave_isr_event(i2c_dev);
+		return ret;
+	}
 
 	status = readl_relaxed(i2c_dev->base + STM32F7_I2C_ISR);
 
@@ -694,12 +1399,21 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data)
 	/* STOP detection flag */
 	if (status & STM32F7_I2C_ISR_STOPF) {
 		/* Disable interrupts */
-		stm32f7_i2c_disable_irq(i2c_dev, STM32F7_I2C_ALL_IRQ_MASK);
+		if (stm32f7_i2c_is_slave_registered(i2c_dev))
+			mask = STM32F7_I2C_XFER_IRQ_MASK;
+		else
+			mask = STM32F7_I2C_ALL_IRQ_MASK;
+		stm32f7_i2c_disable_irq(i2c_dev, mask);
 
 		/* Clear STOP flag */
 		writel_relaxed(STM32F7_I2C_ICR_STOPCF, base + STM32F7_I2C_ICR);
 
-		complete(&i2c_dev->complete);
+		if (i2c_dev->use_dma) {
+			ret = IRQ_WAKE_THREAD;
+		} else {
+			i2c_dev->master_mode = false;
+			complete(&i2c_dev->complete);
+		}
 	}
 
 	/* Transfer complete */
@@ -707,6 +1421,10 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data)
 		if (f7_msg->stop) {
 			mask = STM32F7_I2C_CR2_STOP;
 			stm32f7_i2c_set_bits(base + STM32F7_I2C_CR2, mask);
+		} else if (i2c_dev->use_dma) {
+			ret = IRQ_WAKE_THREAD;
+		} else if (f7_msg->smbus) {
+			stm32f7_i2c_smbus_rep_start(i2c_dev);
 		} else {
 			i2c_dev->msg_id++;
 			i2c_dev->msg++;
@@ -714,13 +1432,50 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data)
 		}
 	}
 
+	if (status & STM32F7_I2C_ISR_TCR) {
+		if (f7_msg->smbus)
+			stm32f7_i2c_smbus_reload(i2c_dev);
+		else
+			stm32f7_i2c_reload(i2c_dev);
+	}
+
+	return ret;
+}
+
+static irqreturn_t stm32f7_i2c_isr_event_thread(int irq, void *data)
+{
+	struct stm32f7_i2c_dev *i2c_dev = data;
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	struct stm32_i2c_dma *dma = i2c_dev->dma;
+	u32 status;
+	int ret;
+
 	/*
-	 * Transfer Complete Reload: 255 data bytes have been transferred
-	 * We have to prepare the I2C controller to transfer the remaining
-	 * data.
+	 * Wait for dma transfer completion before sending next message or
+	 * notity the end of xfer to the client
 	 */
-	if (status & STM32F7_I2C_ISR_TCR)
-		stm32f7_i2c_reload(i2c_dev);
+	ret = wait_for_completion_timeout(&i2c_dev->dma->dma_complete, HZ);
+	if (!ret) {
+		dev_dbg(i2c_dev->dev, "<%s>: Timed out\n", __func__);
+		stm32f7_i2c_disable_dma_req(i2c_dev);
+		dmaengine_terminate_all(dma->chan_using);
+		f7_msg->result = -ETIMEDOUT;
+	}
+
+	status = readl_relaxed(i2c_dev->base + STM32F7_I2C_ISR);
+
+	if (status & STM32F7_I2C_ISR_TC) {
+		if (f7_msg->smbus) {
+			stm32f7_i2c_smbus_rep_start(i2c_dev);
+		} else {
+			i2c_dev->msg_id++;
+			i2c_dev->msg++;
+			stm32f7_i2c_xfer_msg(i2c_dev, i2c_dev->msg);
+		}
+	} else {
+		i2c_dev->master_mode = false;
+		complete(&i2c_dev->complete);
+	}
 
 	return IRQ_HANDLED;
 }
@@ -731,7 +1486,8 @@ static irqreturn_t stm32f7_i2c_isr_error(int irq, void *data)
 	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
 	void __iomem *base = i2c_dev->base;
 	struct device *dev = i2c_dev->dev;
-	u32 status;
+	struct stm32_i2c_dma *dma = i2c_dev->dma;
+	u32 mask, status;
 
 	status = readl_relaxed(i2c_dev->base + STM32F7_I2C_ISR);
 
@@ -739,6 +1495,7 @@ static irqreturn_t stm32f7_i2c_isr_error(int irq, void *data)
 	if (status & STM32F7_I2C_ISR_BERR) {
 		dev_err(dev, "<%s>: Bus error\n", __func__);
 		writel_relaxed(STM32F7_I2C_ICR_BERRCF, base + STM32F7_I2C_ICR);
+		stm32f7_i2c_release_bus(&i2c_dev->adap);
 		f7_msg->result = -EIO;
 	}
 
@@ -749,8 +1506,26 @@ static irqreturn_t stm32f7_i2c_isr_error(int irq, void *data)
 		f7_msg->result = -EAGAIN;
 	}
 
-	stm32f7_i2c_disable_irq(i2c_dev, STM32F7_I2C_ALL_IRQ_MASK);
+	if (status & STM32F7_I2C_ISR_PECERR) {
+		dev_err(dev, "<%s>: PEC error in reception\n", __func__);
+		writel_relaxed(STM32F7_I2C_ICR_PECCF, base + STM32F7_I2C_ICR);
+		f7_msg->result = -EINVAL;
+	}
+
+	/* Disable interrupts */
+	if (stm32f7_i2c_is_slave_registered(i2c_dev))
+		mask = STM32F7_I2C_XFER_IRQ_MASK;
+	else
+		mask = STM32F7_I2C_ALL_IRQ_MASK;
+	stm32f7_i2c_disable_irq(i2c_dev, mask);
+
+	/* Disable dma */
+	if (i2c_dev->use_dma) {
+		stm32f7_i2c_disable_dma_req(i2c_dev);
+		dmaengine_terminate_all(dma->chan_using);
+	}
 
+	i2c_dev->master_mode = false;
 	complete(&i2c_dev->complete);
 
 	return IRQ_HANDLED;
@@ -761,12 +1536,14 @@ static int stm32f7_i2c_xfer(struct i2c_adapter *i2c_adap,
 {
 	struct stm32f7_i2c_dev *i2c_dev = i2c_get_adapdata(i2c_adap);
 	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	struct stm32_i2c_dma *dma = i2c_dev->dma;
 	unsigned long time_left;
 	int ret;
 
 	i2c_dev->msg = msgs;
 	i2c_dev->msg_num = num;
 	i2c_dev->msg_id = 0;
+	f7_msg->smbus = false;
 
 	ret = clk_enable(i2c_dev->clk);
 	if (ret) {
@@ -787,6 +1564,8 @@ static int stm32f7_i2c_xfer(struct i2c_adapter *i2c_adap,
 	if (!time_left) {
 		dev_dbg(i2c_dev->dev, "Access to slave 0x%x timed out\n",
 			i2c_dev->msg->addr);
+		if (i2c_dev->use_dma)
+			dmaengine_terminate_all(dma->chan_using);
 		ret = -ETIMEDOUT;
 	}
 
@@ -796,14 +1575,209 @@ static int stm32f7_i2c_xfer(struct i2c_adapter *i2c_adap,
 	return (ret < 0) ? ret : num;
 }
 
+static int stm32f7_i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
+				  unsigned short flags, char read_write,
+				  u8 command, int size,
+				  union i2c_smbus_data *data)
+{
+	struct stm32f7_i2c_dev *i2c_dev = i2c_get_adapdata(adapter);
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	struct stm32_i2c_dma *dma = i2c_dev->dma;
+	struct device *dev = i2c_dev->dev;
+	unsigned long timeout;
+	int i, ret;
+
+	f7_msg->addr = addr;
+	f7_msg->size = size;
+	f7_msg->read_write = read_write;
+	f7_msg->smbus = true;
+
+	ret = clk_enable(i2c_dev->clk);
+	if (ret) {
+		dev_err(i2c_dev->dev, "Failed to enable clock\n");
+		return ret;
+	}
+
+	ret = stm32f7_i2c_wait_free_bus(i2c_dev);
+	if (ret)
+		goto clk_free;
+
+	ret = stm32f7_i2c_smbus_xfer_msg(i2c_dev, flags, command, data);
+	if (ret)
+		goto clk_free;
+
+	timeout = wait_for_completion_timeout(&i2c_dev->complete,
+					      i2c_dev->adap.timeout);
+	ret = f7_msg->result;
+	if (ret)
+		goto clk_free;
+
+	if (!timeout) {
+		dev_dbg(dev, "Access to slave 0x%x timed out\n", f7_msg->addr);
+		if (i2c_dev->use_dma)
+			dmaengine_terminate_all(dma->chan_using);
+		ret = -ETIMEDOUT;
+		goto clk_free;
+	}
+
+	/* Check PEC */
+	if ((flags & I2C_CLIENT_PEC) && size != I2C_SMBUS_QUICK && read_write) {
+		ret = stm32f7_i2c_smbus_check_pec(i2c_dev);
+		if (ret)
+			goto clk_free;
+	}
+
+	if (read_write && size != I2C_SMBUS_QUICK) {
+		switch (size) {
+		case I2C_SMBUS_BYTE:
+		case I2C_SMBUS_BYTE_DATA:
+			data->byte = f7_msg->smbus_buf[0];
+		break;
+		case I2C_SMBUS_WORD_DATA:
+		case I2C_SMBUS_PROC_CALL:
+			data->word = f7_msg->smbus_buf[0] |
+				(f7_msg->smbus_buf[1] << 8);
+		break;
+		case I2C_SMBUS_BLOCK_DATA:
+		case I2C_SMBUS_BLOCK_PROC_CALL:
+		for (i = 0; i <= f7_msg->smbus_buf[0]; i++)
+			data->block[i] = f7_msg->smbus_buf[i];
+		break;
+		default:
+			dev_err(dev, "Unsupported smbus transaction\n");
+			ret = -EINVAL;
+		}
+	}
+
+clk_free:
+	clk_disable(i2c_dev->clk);
+	return ret;
+}
+
+static int stm32f7_i2c_reg_slave(struct i2c_client *slave)
+{
+	struct stm32f7_i2c_dev *i2c_dev = i2c_get_adapdata(slave->adapter);
+	void __iomem *base = i2c_dev->base;
+	struct device *dev = i2c_dev->dev;
+	u32 oar1, oar2, mask;
+	int id, ret;
+
+	if (slave->flags & I2C_CLIENT_PEC) {
+		dev_err(dev, "SMBus PEC not supported in slave mode\n");
+		return -EINVAL;
+	}
+
+	if (stm32f7_i2c_is_slave_busy(i2c_dev)) {
+		dev_err(dev, "Too much slave registered\n");
+		return -EBUSY;
+	}
+
+	ret = stm32f7_i2c_get_free_slave_id(i2c_dev, slave, &id);
+	if (ret)
+		return ret;
+
+	if (!(stm32f7_i2c_is_slave_registered(i2c_dev))) {
+		ret = clk_enable(i2c_dev->clk);
+		if (ret) {
+			dev_err(dev, "Failed to enable clock\n");
+			return ret;
+		}
+	}
+
+	if (id == 0) {
+		/* Configure Own Address 1 */
+		oar1 = readl_relaxed(i2c_dev->base + STM32F7_I2C_OAR1);
+		oar1 &= ~STM32F7_I2C_OAR1_MASK;
+		if (slave->flags & I2C_CLIENT_TEN) {
+			oar1 |= STM32F7_I2C_OAR1_OA1_10(slave->addr);
+			oar1 |= STM32F7_I2C_OAR1_OA1MODE;
+		} else {
+			oar1 |= STM32F7_I2C_OAR1_OA1_7(slave->addr);
+		}
+		oar1 |= STM32F7_I2C_OAR1_OA1EN;
+		i2c_dev->slave[id] = slave;
+		writel_relaxed(oar1, i2c_dev->base + STM32F7_I2C_OAR1);
+	} else if (id == 1) {
+		/* Configure Own Address 2 */
+		oar2 = readl_relaxed(i2c_dev->base + STM32F7_I2C_OAR2);
+		oar2 &= ~STM32F7_I2C_OAR2_MASK;
+		if (slave->flags & I2C_CLIENT_TEN) {
+			ret = -EOPNOTSUPP;
+			goto exit;
+		}
+
+		oar2 |= STM32F7_I2C_OAR2_OA2_7(slave->addr);
+		oar2 |= STM32F7_I2C_OAR2_OA2EN;
+		i2c_dev->slave[id] = slave;
+		writel_relaxed(oar2, i2c_dev->base + STM32F7_I2C_OAR2);
+	} else {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	/* Enable ACK */
+	stm32f7_i2c_clr_bits(base + STM32F7_I2C_CR2, STM32F7_I2C_CR2_NACK);
+
+	/* Enable Address match interrupt, error interrupt and enable I2C  */
+	mask = STM32F7_I2C_CR1_ADDRIE | STM32F7_I2C_CR1_ERRIE |
+		STM32F7_I2C_CR1_PE;
+	stm32f7_i2c_set_bits(base + STM32F7_I2C_CR1, mask);
+
+	return 0;
+
+exit:
+	if (!(stm32f7_i2c_is_slave_registered(i2c_dev)))
+		clk_disable(i2c_dev->clk);
+
+	return ret;
+}
+
+static int stm32f7_i2c_unreg_slave(struct i2c_client *slave)
+{
+	struct stm32f7_i2c_dev *i2c_dev = i2c_get_adapdata(slave->adapter);
+	void __iomem *base = i2c_dev->base;
+	u32 mask;
+	int id, ret;
+
+	ret = stm32f7_i2c_get_slave_id(i2c_dev, slave, &id);
+	if (ret)
+		return ret;
+
+	WARN_ON(!i2c_dev->slave[id]);
+
+	if (id == 0) {
+		mask = STM32F7_I2C_OAR1_OA1EN;
+		stm32f7_i2c_clr_bits(base + STM32F7_I2C_OAR1, mask);
+	} else {
+		mask = STM32F7_I2C_OAR2_OA2EN;
+		stm32f7_i2c_clr_bits(base + STM32F7_I2C_OAR2, mask);
+	}
+
+	i2c_dev->slave[id] = NULL;
+
+	if (!(stm32f7_i2c_is_slave_registered(i2c_dev))) {
+		stm32f7_i2c_disable_irq(i2c_dev, STM32F7_I2C_ALL_IRQ_MASK);
+		clk_disable(i2c_dev->clk);
+	}
+
+	return 0;
+}
+
 static u32 stm32f7_i2c_func(struct i2c_adapter *adap)
 {
-	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+	return I2C_FUNC_I2C | I2C_FUNC_10BIT_ADDR | I2C_FUNC_SLAVE |
+		I2C_FUNC_SMBUS_QUICK | I2C_FUNC_SMBUS_BYTE |
+		I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA |
+		I2C_FUNC_SMBUS_BLOCK_DATA | I2C_FUNC_SMBUS_BLOCK_PROC_CALL |
+		I2C_FUNC_SMBUS_PROC_CALL | I2C_FUNC_SMBUS_PEC;
 }
 
 static struct i2c_algorithm stm32f7_i2c_algo = {
 	.master_xfer = stm32f7_i2c_xfer,
+	.smbus_xfer = stm32f7_i2c_smbus_xfer,
 	.functionality = stm32f7_i2c_func,
+	.reg_slave = stm32f7_i2c_reg_slave,
+	.unreg_slave = stm32f7_i2c_unreg_slave,
 };
 
 static int stm32f7_i2c_probe(struct platform_device *pdev)
@@ -815,6 +1789,7 @@ static int stm32f7_i2c_probe(struct platform_device *pdev)
 	u32 irq_error, irq_event, clk_rate, rise_time, fall_time;
 	struct i2c_adapter *adap;
 	struct reset_control *rst;
+	dma_addr_t phy_addr;
 	int ret;
 
 	i2c_dev = devm_kzalloc(&pdev->dev, sizeof(*i2c_dev), GFP_KERNEL);
@@ -825,6 +1800,7 @@ static int stm32f7_i2c_probe(struct platform_device *pdev)
 	i2c_dev->base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(i2c_dev->base))
 		return PTR_ERR(i2c_dev->base);
+	phy_addr = (dma_addr_t)res->start;
 
 	irq_event = irq_of_parse_and_map(np, 0);
 	if (!irq_event) {
@@ -871,8 +1847,11 @@ static int stm32f7_i2c_probe(struct platform_device *pdev)
 
 	i2c_dev->dev = &pdev->dev;
 
-	ret = devm_request_irq(&pdev->dev, irq_event, stm32f7_i2c_isr_event, 0,
-			       pdev->name, i2c_dev);
+	ret = devm_request_threaded_irq(&pdev->dev, irq_event,
+					stm32f7_i2c_isr_event,
+					stm32f7_i2c_isr_event_thread,
+					IRQF_ONESHOT,
+					pdev->name, i2c_dev);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to request irq event %i\n",
 			irq_event);
@@ -924,6 +1903,11 @@ static int stm32f7_i2c_probe(struct platform_device *pdev)
 
 	init_completion(&i2c_dev->complete);
 
+	/* Init DMA config if supported */
+	i2c_dev->dma = stm32_i2c_dma_request(i2c_dev->dev, phy_addr,
+					     STM32F7_I2C_TXDR,
+					     STM32F7_I2C_RXDR);
+
 	ret = i2c_add_adapter(adap);
 	if (ret)
 		goto clk_free;
@@ -946,6 +1930,11 @@ static int stm32f7_i2c_remove(struct platform_device *pdev)
 {
 	struct stm32f7_i2c_dev *i2c_dev = platform_get_drvdata(pdev);
 
+	if (i2c_dev->dma) {
+		stm32_i2c_dma_free(i2c_dev->dma);
+		i2c_dev->dma = NULL;
+	}
+
 	i2c_del_adapter(&i2c_dev->adap);
 
 	clk_unprepare(i2c_dev->clk);
diff --git a/drivers/i2c/busses/i2c-stu300.c b/drivers/i2c/busses/i2c-stu300.c
index dc63236b45b2..e866c481bfc3 100644
--- a/drivers/i2c/busses/i2c-stu300.c
+++ b/drivers/i2c/busses/i2c-stu300.c
@@ -602,20 +602,24 @@ static int stu300_send_address(struct stu300_dev *dev,
 	u32 val;
 	int ret;
 
-	if (msg->flags & I2C_M_TEN)
+	if (msg->flags & I2C_M_TEN) {
 		/* This is probably how 10 bit addresses look */
 		val = (0xf0 | (((u32) msg->addr & 0x300) >> 7)) &
 			I2C_DR_D_MASK;
-	else
-		val = ((msg->addr << 1) & I2C_DR_D_MASK);
+		if (msg->flags & I2C_M_RD)
+			/* This is the direction bit */
+			val |= 0x01;
+	} else {
+		val = i2c_8bit_addr_from_msg(msg);
+	}
 
-	if (msg->flags & I2C_M_RD) {
-		/* This is the direction bit */
-		val |= 0x01;
-		if (resend)
+	if (resend) {
+		if (msg->flags & I2C_M_RD)
 			dev_dbg(&dev->pdev->dev, "read resend\n");
-	} else if (resend)
-		dev_dbg(&dev->pdev->dev, "write resend\n");
+		else
+			dev_dbg(&dev->pdev->dev, "write resend\n");
+	}
+
 	stu300_wr8(val, dev->virtbase + I2C_DR);
 
 	/* For 10bit addressing, await 10bit request (EVENT 9) */
diff --git a/drivers/i2c/busses/i2c-synquacer.c b/drivers/i2c/busses/i2c-synquacer.c
index a021f866d8c2..915f5edbab33 100644
--- a/drivers/i2c/busses/i2c-synquacer.c
+++ b/drivers/i2c/busses/i2c-synquacer.c
@@ -509,7 +509,7 @@ static int synquacer_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
 
 	dev_dbg(i2c->dev, "calculated timeout %d ms\n", i2c->timeout_ms);
 
-	for (retry = 0; retry < adap->retries; retry++) {
+	for (retry = 0; retry <= adap->retries; retry++) {
 		ret = synquacer_i2c_doxfer(i2c, msgs, num);
 		if (ret != -EAGAIN)
 			return ret;
diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c
index 60292d243e24..5fccd1f1bca8 100644
--- a/drivers/i2c/busses/i2c-tegra.c
+++ b/drivers/i2c/busses/i2c-tegra.c
@@ -173,7 +173,6 @@ struct tegra_i2c_hw_feature {
  * @msg_buf_remaining: size of unsent data in the message buffer
  * @msg_read: identifies read transfers
  * @bus_clk_rate: current i2c bus clock rate
- * @is_suspended: prevents i2c controller accesses after suspend is called
  */
 struct tegra_i2c_dev {
 	struct device *dev;
@@ -194,7 +193,6 @@ struct tegra_i2c_dev {
 	int msg_read;
 	u32 bus_clk_rate;
 	u16 clk_divisor_non_hs_mode;
-	bool is_suspended;
 	bool is_multimaster_mode;
 	spinlock_t xfer_lock;
 };
@@ -734,9 +732,6 @@ static int tegra_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[],
 	int i;
 	int ret = 0;
 
-	if (i2c_dev->is_suspended)
-		return -EBUSY;
-
 	ret = pm_runtime_get_sync(i2c_dev->dev);
 	if (ret < 0) {
 		dev_err(i2c_dev->dev, "runtime resume failed %d\n", ret);
@@ -1051,37 +1046,9 @@ static int tegra_i2c_remove(struct platform_device *pdev)
 }
 
 #ifdef CONFIG_PM_SLEEP
-static int tegra_i2c_suspend(struct device *dev)
-{
-	struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev);
-
-	i2c_lock_adapter(&i2c_dev->adapter);
-	i2c_dev->is_suspended = true;
-	i2c_unlock_adapter(&i2c_dev->adapter);
-
-	return 0;
-}
-
-static int tegra_i2c_resume(struct device *dev)
-{
-	struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev);
-	int ret;
-
-	i2c_lock_adapter(&i2c_dev->adapter);
-
-	ret = tegra_i2c_init(i2c_dev);
-	if (!ret)
-		i2c_dev->is_suspended = false;
-
-	i2c_unlock_adapter(&i2c_dev->adapter);
-
-	return ret;
-}
-
 static const struct dev_pm_ops tegra_i2c_pm = {
 	SET_RUNTIME_PM_OPS(tegra_i2c_runtime_suspend, tegra_i2c_runtime_resume,
 			   NULL)
-	SET_SYSTEM_SLEEP_PM_OPS(tegra_i2c_suspend, tegra_i2c_resume)
 };
 #define TEGRA_I2C_PM	(&tegra_i2c_pm)
 #else
diff --git a/drivers/i2c/busses/i2c-xiic.c b/drivers/i2c/busses/i2c-xiic.c
index c80527816ad0..9a71e50d21f1 100644
--- a/drivers/i2c/busses/i2c-xiic.c
+++ b/drivers/i2c/busses/i2c-xiic.c
@@ -33,7 +33,7 @@
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/wait.h>
-#include <linux/i2c-xiic.h>
+#include <linux/platform_data/i2c-xiic.h>
 #include <linux/io.h>
 #include <linux/slab.h>
 #include <linux/of.h>
@@ -143,12 +143,6 @@ struct xiic_i2c {
 
 #define XIIC_TX_RX_INTERRUPTS (XIIC_INTR_RX_FULL_MASK | XIIC_TX_INTERRUPTS)
 
-/* The following constants are used with the following macros to specify the
- * operation, a read or write operation.
- */
-#define XIIC_READ_OPERATION  1
-#define XIIC_WRITE_OPERATION 0
-
 /*
  * Tx Fifo upper bit masks.
  */
@@ -415,7 +409,7 @@ static irqreturn_t xiic_process(int irq, void *dev_id)
 		clr |= XIIC_INTR_RX_FULL_MASK;
 		if (!i2c->rx_msg) {
 			dev_dbg(i2c->adap.dev.parent,
-				"%s unexpexted RX IRQ\n", __func__);
+				"%s unexpected RX IRQ\n", __func__);
 			xiic_clear_rx_fifo(i2c);
 			goto out;
 		}
@@ -470,7 +464,7 @@ static irqreturn_t xiic_process(int irq, void *dev_id)
 
 		if (!i2c->tx_msg) {
 			dev_dbg(i2c->adap.dev.parent,
-				"%s unexpexted TX IRQ\n", __func__);
+				"%s unexpected TX IRQ\n", __func__);
 			goto out;
 		}
 
@@ -556,8 +550,7 @@ static void xiic_start_recv(struct xiic_i2c *i2c)
 	if (!(msg->flags & I2C_M_NOSTART))
 		/* write the address */
 		xiic_setreg16(i2c, XIIC_DTR_REG_OFFSET,
-			(msg->addr << 1) | XIIC_READ_OPERATION |
-			XIIC_TX_DYN_START_MASK);
+			i2c_8bit_addr_from_msg(msg) | XIIC_TX_DYN_START_MASK);
 
 	xiic_irq_clr_en(i2c, XIIC_INTR_BNB_MASK);
 
@@ -585,7 +578,7 @@ static void xiic_start_send(struct xiic_i2c *i2c)
 
 	if (!(msg->flags & I2C_M_NOSTART)) {
 		/* write the address */
-		u16 data = ((msg->addr << 1) & 0xfe) | XIIC_WRITE_OPERATION |
+		u16 data = i2c_8bit_addr_from_msg(msg) |
 			XIIC_TX_DYN_START_MASK;
 		if ((i2c->nmsgs == 1) && msg->len == 0)
 			/* no data and last message -> add STOP */
diff --git a/drivers/i2c/busses/i2c-xlp9xx.c b/drivers/i2c/busses/i2c-xlp9xx.c
index eb8913eba0c5..1f41a4f89c08 100644
--- a/drivers/i2c/busses/i2c-xlp9xx.c
+++ b/drivers/i2c/busses/i2c-xlp9xx.c
@@ -10,6 +10,7 @@
 #include <linux/clk.h>
 #include <linux/completion.h>
 #include <linux/i2c.h>
+#include <linux/i2c-smbus.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
@@ -84,6 +85,8 @@ struct xlp9xx_i2c_dev {
 	struct device *dev;
 	struct i2c_adapter adapter;
 	struct completion msg_complete;
+	struct i2c_smbus_alert_setup alert_data;
+	struct i2c_client *ara;
 	int irq;
 	bool msg_read;
 	bool len_recv;
@@ -155,9 +158,30 @@ static void xlp9xx_i2c_fill_tx_fifo(struct xlp9xx_i2c_dev *priv)
 	priv->msg_buf += len;
 }
 
+static void xlp9xx_i2c_update_rlen(struct xlp9xx_i2c_dev *priv)
+{
+	u32 val, len;
+
+	/*
+	 * Update receive length. Re-read len to get the latest value,
+	 * and then add 4 to have a minimum value that can be safely
+	 * written. This is to account for the byte read above, the
+	 * transfer in progress and any delays in the register I/O
+	 */
+	val = xlp9xx_read_i2c_reg(priv, XLP9XX_I2C_CTRL);
+	len = xlp9xx_read_i2c_reg(priv, XLP9XX_I2C_FIFOWCNT) &
+				  XLP9XX_I2C_FIFO_WCNT_MASK;
+	len = max_t(u32, priv->msg_len, len + 4);
+	if (len >= I2C_SMBUS_BLOCK_MAX + 2)
+		return;
+	val = (val & ~XLP9XX_I2C_CTRL_MCTLEN_MASK) |
+			(len << XLP9XX_I2C_CTRL_MCTLEN_SHIFT);
+	xlp9xx_write_i2c_reg(priv, XLP9XX_I2C_CTRL, val);
+}
+
 static void xlp9xx_i2c_drain_rx_fifo(struct xlp9xx_i2c_dev *priv)
 {
-	u32 len, i, val;
+	u32 len, i;
 	u8 rlen, *buf = priv->msg_buf;
 
 	len = xlp9xx_read_i2c_reg(priv, XLP9XX_I2C_FIFOWCNT) &
@@ -167,21 +191,20 @@ static void xlp9xx_i2c_drain_rx_fifo(struct xlp9xx_i2c_dev *priv)
 	if (priv->len_recv) {
 		/* read length byte */
 		rlen = xlp9xx_read_i2c_reg(priv, XLP9XX_I2C_MRXFIFO);
-		*buf++ = rlen;
-		len--;
-
-		if (priv->client_pec)
-			++rlen;
-		/* update remaining bytes and message length */
-		priv->msg_buf_remaining = rlen;
-		priv->msg_len = rlen + 1;
+		if (rlen > I2C_SMBUS_BLOCK_MAX || rlen == 0) {
+			rlen = 0;	/*abort transfer */
+			priv->msg_buf_remaining = 0;
+			priv->msg_len = 0;
+		} else {
+			*buf++ = rlen;
+			if (priv->client_pec)
+				++rlen; /* account for error check byte */
+			/* update remaining bytes and message length */
+			priv->msg_buf_remaining = rlen;
+			priv->msg_len = rlen + 1;
+		}
+		xlp9xx_i2c_update_rlen(priv);
 		priv->len_recv = false;
-
-		/* Update transfer length to read only actual data */
-		val = xlp9xx_read_i2c_reg(priv, XLP9XX_I2C_CTRL);
-		val = (val & ~XLP9XX_I2C_CTRL_MCTLEN_MASK) |
-			((rlen + 1) << XLP9XX_I2C_CTRL_MCTLEN_SHIFT);
-		xlp9xx_write_i2c_reg(priv, XLP9XX_I2C_CTRL, val);
 	} else {
 		len = min(priv->msg_buf_remaining, len);
 		for (i = 0; i < len; i++, buf++)
@@ -300,10 +323,6 @@ static int xlp9xx_i2c_xfer_msg(struct xlp9xx_i2c_dev *priv, struct i2c_msg *msg,
 	xlp9xx_write_i2c_reg(priv, XLP9XX_I2C_MFIFOCTRL,
 			     XLP9XX_I2C_MFIFOCTRL_RST);
 
-	/* set FIFO threshold if reading */
-	if (priv->msg_read)
-		xlp9xx_i2c_update_rx_fifo_thres(priv);
-
 	/* set slave addr */
 	xlp9xx_write_i2c_reg(priv, XLP9XX_I2C_SLAVEADDR,
 			     (msg->addr << XLP9XX_I2C_SLAVEADDR_ADDR_SHIFT) |
@@ -322,9 +341,13 @@ static int xlp9xx_i2c_xfer_msg(struct xlp9xx_i2c_dev *priv, struct i2c_msg *msg,
 		val &= ~XLP9XX_I2C_CTRL_ADDMODE;
 
 	priv->len_recv = msg->flags & I2C_M_RECV_LEN;
-	len = priv->len_recv ? XLP9XX_I2C_FIFO_SIZE : msg->len;
+	len = priv->len_recv ? I2C_SMBUS_BLOCK_MAX + 2 : msg->len;
 	priv->client_pec = msg->flags & I2C_CLIENT_PEC;
 
+	/* set FIFO threshold if reading */
+	if (priv->msg_read)
+		xlp9xx_i2c_update_rx_fifo_thres(priv);
+
 	/* set data length to be transferred */
 	val = (val & ~XLP9XX_I2C_CTRL_MCTLEN_MASK) |
 	      (len << XLP9XX_I2C_CTRL_MCTLEN_SHIFT);
@@ -378,8 +401,11 @@ static int xlp9xx_i2c_xfer_msg(struct xlp9xx_i2c_dev *priv, struct i2c_msg *msg,
 	}
 
 	/* update msg->len with actual received length */
-	if (msg->flags & I2C_M_RECV_LEN)
+	if (msg->flags & I2C_M_RECV_LEN) {
+		if (!priv->msg_len)
+			return -EPROTO;
 		msg->len = priv->msg_len;
+	}
 	return 0;
 }
 
@@ -447,6 +473,19 @@ static int xlp9xx_i2c_get_frequency(struct platform_device *pdev,
 	return 0;
 }
 
+static int xlp9xx_i2c_smbus_setup(struct xlp9xx_i2c_dev *priv,
+				  struct platform_device *pdev)
+{
+	if (!priv->alert_data.irq)
+		return -EINVAL;
+
+	priv->ara = i2c_setup_smbus_alert(&priv->adapter, &priv->alert_data);
+	if (!priv->ara)
+		return -ENODEV;
+
+	return 0;
+}
+
 static int xlp9xx_i2c_probe(struct platform_device *pdev)
 {
 	struct xlp9xx_i2c_dev *priv;
@@ -467,6 +506,10 @@ static int xlp9xx_i2c_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "invalid irq!\n");
 		return priv->irq;
 	}
+	/* SMBAlert irq */
+	priv->alert_data.irq = platform_get_irq(pdev, 1);
+	if (priv->alert_data.irq <= 0)
+		priv->alert_data.irq = 0;
 
 	xlp9xx_i2c_get_frequency(pdev, priv);
 	xlp9xx_i2c_init(priv);
@@ -493,6 +536,10 @@ static int xlp9xx_i2c_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
+	err = xlp9xx_i2c_smbus_setup(priv, pdev);
+	if (err)
+		dev_dbg(&pdev->dev, "No active SMBus alert %d\n", err);
+
 	platform_set_drvdata(pdev, priv);
 	dev_dbg(&pdev->dev, "I2C bus:%d added\n", priv->adapter.nr);
 
diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c
index 1ba40bb2b966..31d16ada6e7d 100644
--- a/drivers/i2c/i2c-core-base.c
+++ b/drivers/i2c/i2c-core-base.c
@@ -363,7 +363,7 @@ static int i2c_device_probe(struct device *dev)
 		goto err_clear_wakeup_irq;
 
 	status = dev_pm_domain_attach(&client->dev, true);
-	if (status == -EPROBE_DEFER)
+	if (status)
 		goto err_clear_wakeup_irq;
 
 	/*
@@ -717,10 +717,6 @@ i2c_new_device(struct i2c_adapter *adap, struct i2c_board_info const *info)
 	client->adapter = adap;
 
 	client->dev.platform_data = info->platform_data;
-
-	if (info->archdata)
-		client->dev.archdata = *info->archdata;
-
 	client->flags = info->flags;
 	client->addr = info->addr;
 
@@ -746,7 +742,7 @@ i2c_new_device(struct i2c_adapter *adap, struct i2c_board_info const *info)
 	client->dev.parent = &client->adapter->dev;
 	client->dev.bus = &i2c_bus_type;
 	client->dev.type = &i2c_client_type;
-	client->dev.of_node = info->of_node;
+	client->dev.of_node = of_node_get(info->of_node);
 	client->dev.fwnode = info->fwnode;
 
 	i2c_dev_set_name(adap, client, info);
@@ -757,7 +753,7 @@ i2c_new_device(struct i2c_adapter *adap, struct i2c_board_info const *info)
 			dev_err(&adap->dev,
 				"Failed to add properties to client %s: %d\n",
 				client->name, status);
-			goto out_err;
+			goto out_err_put_of_node;
 		}
 	}
 
@@ -773,6 +769,8 @@ i2c_new_device(struct i2c_adapter *adap, struct i2c_board_info const *info)
 out_free_props:
 	if (info->properties)
 		device_remove_properties(&client->dev);
+out_err_put_of_node:
+	of_node_put(info->of_node);
 out_err:
 	dev_err(&adap->dev,
 		"Failed to register i2c client %s at 0x%02x (%d)\n",
diff --git a/drivers/i2c/i2c-core-of.c b/drivers/i2c/i2c-core-of.c
index c405270a98b4..6cb7ad608bcd 100644
--- a/drivers/i2c/i2c-core-of.c
+++ b/drivers/i2c/i2c-core-of.c
@@ -22,53 +22,64 @@
 
 #include "i2c-core.h"
 
-static struct i2c_client *of_i2c_register_device(struct i2c_adapter *adap,
-						 struct device_node *node)
+int of_i2c_get_board_info(struct device *dev, struct device_node *node,
+			  struct i2c_board_info *info)
 {
-	struct i2c_client *client;
-	struct i2c_board_info info = {};
-	struct dev_archdata dev_ad = {};
 	u32 addr;
 	int ret;
 
-	dev_dbg(&adap->dev, "of_i2c: register %pOF\n", node);
+	memset(info, 0, sizeof(*info));
 
-	if (of_modalias_node(node, info.type, sizeof(info.type)) < 0) {
-		dev_err(&adap->dev, "of_i2c: modalias failure on %pOF\n",
-			node);
-		return ERR_PTR(-EINVAL);
+	if (of_modalias_node(node, info->type, sizeof(info->type)) < 0) {
+		dev_err(dev, "of_i2c: modalias failure on %pOF\n", node);
+		return -EINVAL;
 	}
 
 	ret = of_property_read_u32(node, "reg", &addr);
 	if (ret) {
-		dev_err(&adap->dev, "of_i2c: invalid reg on %pOF\n", node);
-		return ERR_PTR(ret);
+		dev_err(dev, "of_i2c: invalid reg on %pOF\n", node);
+		return ret;
 	}
 
 	if (addr & I2C_TEN_BIT_ADDRESS) {
 		addr &= ~I2C_TEN_BIT_ADDRESS;
-		info.flags |= I2C_CLIENT_TEN;
+		info->flags |= I2C_CLIENT_TEN;
 	}
 
 	if (addr & I2C_OWN_SLAVE_ADDRESS) {
 		addr &= ~I2C_OWN_SLAVE_ADDRESS;
-		info.flags |= I2C_CLIENT_SLAVE;
+		info->flags |= I2C_CLIENT_SLAVE;
 	}
 
-	info.addr = addr;
-	info.archdata = &dev_ad;
-	info.of_node = of_node_get(node);
+	info->addr = addr;
+	info->of_node = node;
 
 	if (of_property_read_bool(node, "host-notify"))
-		info.flags |= I2C_CLIENT_HOST_NOTIFY;
+		info->flags |= I2C_CLIENT_HOST_NOTIFY;
 
 	if (of_get_property(node, "wakeup-source", NULL))
-		info.flags |= I2C_CLIENT_WAKE;
+		info->flags |= I2C_CLIENT_WAKE;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(of_i2c_get_board_info);
+
+static struct i2c_client *of_i2c_register_device(struct i2c_adapter *adap,
+						 struct device_node *node)
+{
+	struct i2c_client *client;
+	struct i2c_board_info info;
+	int ret;
+
+	dev_dbg(&adap->dev, "of_i2c: register %pOF\n", node);
+
+	ret = of_i2c_get_board_info(&adap->dev, node, &info);
+	if (ret)
+		return ERR_PTR(ret);
 
 	client = i2c_new_device(adap, &info);
 	if (!client) {
 		dev_err(&adap->dev, "of_i2c: Failure registering %pOF\n", node);
-		of_node_put(node);
 		return ERR_PTR(-EINVAL);
 	}
 	return client;
diff --git a/drivers/i2c/i2c-core-smbus.c b/drivers/i2c/i2c-core-smbus.c
index b5aec33002c3..f3f683041e7f 100644
--- a/drivers/i2c/i2c-core-smbus.c
+++ b/drivers/i2c/i2c-core-smbus.c
@@ -466,6 +466,8 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr,
 	status = i2c_transfer(adapter, msg, num);
 	if (status < 0)
 		return status;
+	if (status != num)
+		return -EIO;
 
 	/* Check PEC if last message is a read */
 	if (i && (msg[num-1].flags & I2C_M_RD)) {
diff --git a/drivers/i2c/i2c-mux.c b/drivers/i2c/i2c-mux.c
index 9669ca4937b8..300ab4b672e4 100644
--- a/drivers/i2c/i2c-mux.c
+++ b/drivers/i2c/i2c-mux.c
@@ -418,7 +418,7 @@ int i2c_mux_add_adapter(struct i2c_mux_core *muxc,
 	snprintf(symlink_name, sizeof(symlink_name), "channel-%u", chan_id);
 	WARN(sysfs_create_link(&muxc->dev->kobj, &priv->adap.dev.kobj,
 			       symlink_name),
-	     "can't create symlink for channel %u\n", chan_id);
+	     "can't create symlink to channel %u\n", chan_id);
 	dev_info(&parent->dev, "Added multiplexed i2c bus %d\n",
 		 i2c_adapter_id(&priv->adap));
 
diff --git a/drivers/i2c/muxes/i2c-demux-pinctrl.c b/drivers/i2c/muxes/i2c-demux-pinctrl.c
index 33ce032cb701..035032e20327 100644
--- a/drivers/i2c/muxes/i2c-demux-pinctrl.c
+++ b/drivers/i2c/muxes/i2c-demux-pinctrl.c
@@ -18,6 +18,7 @@
 #include <linux/of.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/sysfs.h>
 
@@ -105,7 +106,7 @@ static int i2c_demux_activate_master(struct i2c_demux_pinctrl_priv *priv, u32 ne
 	priv->cur_adap.owner = THIS_MODULE;
 	priv->cur_adap.algo = &priv->algo;
 	priv->cur_adap.algo_data = priv;
-	priv->cur_adap.dev.parent = priv->dev;
+	priv->cur_adap.dev.parent = &adap->dev;
 	priv->cur_adap.class = adap->class;
 	priv->cur_adap.retries = adap->retries;
 	priv->cur_adap.timeout = adap->timeout;
@@ -254,6 +255,8 @@ static int i2c_demux_pinctrl_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, priv);
 
+	pm_runtime_no_callbacks(&pdev->dev);
+
 	/* switch to first parent as active master */
 	i2c_demux_activate_master(priv, 0);
 
diff --git a/drivers/i2c/muxes/i2c-mux-gpio.c b/drivers/i2c/muxes/i2c-mux-gpio.c
index 1a9973ede443..15a7cc0459fb 100644
--- a/drivers/i2c/muxes/i2c-mux-gpio.c
+++ b/drivers/i2c/muxes/i2c-mux-gpio.c
@@ -10,7 +10,7 @@
 
 #include <linux/i2c.h>
 #include <linux/i2c-mux.h>
-#include <linux/i2c-mux-gpio.h>
+#include <linux/platform_data/i2c-mux-gpio.h>
 #include <linux/platform_device.h>
 #include <linux/module.h>
 #include <linux/slab.h>
diff --git a/drivers/i2c/muxes/i2c-mux-ltc4306.c b/drivers/i2c/muxes/i2c-mux-ltc4306.c
index 311b1cced0c0..a9af93259b19 100644
--- a/drivers/i2c/muxes/i2c-mux-ltc4306.c
+++ b/drivers/i2c/muxes/i2c-mux-ltc4306.c
@@ -206,8 +206,7 @@ static const struct of_device_id ltc4306_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, ltc4306_of_match);
 
-static int ltc4306_probe(struct i2c_client *client,
-			 const struct i2c_device_id *id)
+static int ltc4306_probe(struct i2c_client *client)
 {
 	struct i2c_adapter *adap = to_i2c_adapter(client->dev.parent);
 	const struct chip_desc *chip;
@@ -221,7 +220,7 @@ static int ltc4306_probe(struct i2c_client *client,
 	chip = of_device_get_match_data(&client->dev);
 
 	if (!chip)
-		chip = &chips[id->driver_data];
+		chip = &chips[i2c_match_id(ltc4306_id, client)->driver_data];
 
 	idle_disc = device_property_read_bool(&client->dev,
 					      "i2c-mux-idle-disconnect");
@@ -310,7 +309,7 @@ static struct i2c_driver ltc4306_driver = {
 		.name	= "ltc4306",
 		.of_match_table = of_match_ptr(ltc4306_of_match),
 	},
-	.probe		= ltc4306_probe,
+	.probe_new	= ltc4306_probe,
 	.remove		= ltc4306_remove,
 	.id_table	= ltc4306_id,
 };
diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c
index 09bafd3e68fa..fbc748027087 100644
--- a/drivers/i2c/muxes/i2c-mux-pca954x.c
+++ b/drivers/i2c/muxes/i2c-mux-pca954x.c
@@ -36,6 +36,7 @@
  */
 
 #include <linux/device.h>
+#include <linux/delay.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/i2c-mux.h>
@@ -373,7 +374,6 @@ static int pca954x_probe(struct i2c_client *client,
 	int num, force, class;
 	struct i2c_mux_core *muxc;
 	struct pca954x *data;
-	const struct of_device_id *match;
 	int ret;
 
 	if (!i2c_check_functionality(adap, I2C_FUNC_SMBUS_BYTE))
@@ -389,15 +389,19 @@ static int pca954x_probe(struct i2c_client *client,
 	i2c_set_clientdata(client, muxc);
 	data->client = client;
 
-	/* Get the mux out of reset if a reset GPIO is specified. */
-	gpio = devm_gpiod_get_optional(&client->dev, "reset", GPIOD_OUT_LOW);
+	/* Reset the mux if a reset GPIO is specified. */
+	gpio = devm_gpiod_get_optional(&client->dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(gpio))
 		return PTR_ERR(gpio);
+	if (gpio) {
+		udelay(1);
+		gpiod_set_value_cansleep(gpio, 0);
+		/* Give the chip some time to recover. */
+		udelay(1);
+	}
 
-	match = of_match_device(of_match_ptr(pca954x_of_match), &client->dev);
-	if (match)
-		data->chip = of_device_get_match_data(&client->dev);
-	else
+	data->chip = of_device_get_match_data(&client->dev);
+	if (!data->chip)
 		data->chip = &chips[id->driver_data];
 
 	if (data->chip->id.manufacturer_id != I2C_DEVICE_ID_NONE) {
diff --git a/drivers/i2c/muxes/i2c-mux-reg.c b/drivers/i2c/muxes/i2c-mux-reg.c
index c948e5a4cb04..f624ed64a47b 100644
--- a/drivers/i2c/muxes/i2c-mux-reg.c
+++ b/drivers/i2c/muxes/i2c-mux-reg.c
@@ -127,10 +127,8 @@ static int i2c_mux_reg_probe_dt(struct regmux *mux,
 	values = devm_kzalloc(&pdev->dev,
 			      sizeof(*mux->data.values) * mux->data.n_values,
 			      GFP_KERNEL);
-	if (!values) {
-		dev_err(&pdev->dev, "Cannot allocate values array");
+	if (!values)
 		return -ENOMEM;
-	}
 
 	for_each_child_of_node(np, child) {
 		of_property_read_u32(child, "reg", values + i);
diff --git a/drivers/ide/ide-atapi.c b/drivers/ide/ide-atapi.c
index 0e6bc631a1ca..8b2b72b93885 100644
--- a/drivers/ide/ide-atapi.c
+++ b/drivers/ide/ide-atapi.c
@@ -92,7 +92,7 @@ int ide_queue_pc_tail(ide_drive_t *drive, struct gendisk *disk,
 	struct request *rq;
 	int error;
 
-	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
 	ide_req(rq)->type = ATA_PRIV_MISC;
 	rq->special = (char *)pc;
 
diff --git a/drivers/ide/ide-cd.c b/drivers/ide/ide-cd.c
index 5a8e8e3c22cd..5f178384876f 100644
--- a/drivers/ide/ide-cd.c
+++ b/drivers/ide/ide-cd.c
@@ -437,7 +437,7 @@ int ide_cd_queue_pc(ide_drive_t *drive, const unsigned char *cmd,
 		bool delay = false;
 
 		rq = blk_get_request(drive->queue,
-			write ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN,  __GFP_RECLAIM);
+			write ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
 		memcpy(scsi_req(rq)->cmd, cmd, BLK_MAX_CDB);
 		ide_req(rq)->type = ATA_PRIV_PC;
 		rq->rq_flags |= rq_flags;
@@ -1426,21 +1426,8 @@ static int idecd_capacity_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int idecd_capacity_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, idecd_capacity_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations idecd_capacity_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= idecd_capacity_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static ide_proc_entry_t idecd_proc[] = {
-	{ "capacity", S_IFREG|S_IRUGO, &idecd_capacity_proc_fops },
+	{ "capacity", S_IFREG|S_IRUGO, idecd_capacity_proc_show },
 	{}
 };
 
diff --git a/drivers/ide/ide-cd_ioctl.c b/drivers/ide/ide-cd_ioctl.c
index 2acca12b9c94..b1322400887b 100644
--- a/drivers/ide/ide-cd_ioctl.c
+++ b/drivers/ide/ide-cd_ioctl.c
@@ -304,7 +304,7 @@ int ide_cdrom_reset(struct cdrom_device_info *cdi)
 	struct request *rq;
 	int ret;
 
-	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
 	ide_req(rq)->type = ATA_PRIV_MISC;
 	rq->rq_flags = RQF_QUIET;
 	blk_execute_rq(drive->queue, cd->disk, rq, 0);
diff --git a/drivers/ide/ide-devsets.c b/drivers/ide/ide-devsets.c
index 4e20747af32e..f4f8afdf8bbe 100644
--- a/drivers/ide/ide-devsets.c
+++ b/drivers/ide/ide-devsets.c
@@ -166,7 +166,7 @@ int ide_devset_execute(ide_drive_t *drive, const struct ide_devset *setting,
 	if (!(setting->flags & DS_SYNC))
 		return setting->set(drive, arg);
 
-	rq = blk_get_request(q, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	rq = blk_get_request(q, REQ_OP_DRV_IN, 0);
 	ide_req(rq)->type = ATA_PRIV_MISC;
 	scsi_req(rq)->cmd_len = 5;
 	scsi_req(rq)->cmd[0] = REQ_DEVSET_EXEC;
diff --git a/drivers/ide/ide-disk.c b/drivers/ide/ide-disk.c
index f1a7c58fe418..e3b4e659082d 100644
--- a/drivers/ide/ide-disk.c
+++ b/drivers/ide/ide-disk.c
@@ -478,7 +478,7 @@ static int set_multcount(ide_drive_t *drive, int arg)
 	if (drive->special_flags & IDE_SFLAG_SET_MULTMODE)
 		return -EBUSY;
 
-	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
 	ide_req(rq)->type = ATA_PRIV_TASKFILE;
 
 	drive->mult_req = arg;
diff --git a/drivers/ide/ide-disk_proc.c b/drivers/ide/ide-disk_proc.c
index 82a36ced4e96..95d239b2f646 100644
--- a/drivers/ide/ide-disk_proc.c
+++ b/drivers/ide/ide-disk_proc.c
@@ -52,19 +52,6 @@ static int idedisk_cache_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int idedisk_cache_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, idedisk_cache_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations idedisk_cache_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= idedisk_cache_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int idedisk_capacity_proc_show(struct seq_file *m, void *v)
 {
 	ide_drive_t*drive = (ide_drive_t *)m->private;
@@ -73,19 +60,6 @@ static int idedisk_capacity_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int idedisk_capacity_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, idedisk_capacity_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations idedisk_capacity_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= idedisk_capacity_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int __idedisk_proc_show(struct seq_file *m, ide_drive_t *drive, u8 sub_cmd)
 {
 	u8 *buf;
@@ -114,43 +88,17 @@ static int idedisk_sv_proc_show(struct seq_file *m, void *v)
 	return __idedisk_proc_show(m, m->private, ATA_SMART_READ_VALUES);
 }
 
-static int idedisk_sv_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, idedisk_sv_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations idedisk_sv_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= idedisk_sv_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int idedisk_st_proc_show(struct seq_file *m, void *v)
 {
 	return __idedisk_proc_show(m, m->private, ATA_SMART_READ_THRESHOLDS);
 }
 
-static int idedisk_st_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, idedisk_st_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations idedisk_st_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= idedisk_st_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 ide_proc_entry_t ide_disk_proc[] = {
-	{ "cache",	  S_IFREG|S_IRUGO, &idedisk_cache_proc_fops	},
-	{ "capacity",	  S_IFREG|S_IRUGO, &idedisk_capacity_proc_fops	},
-	{ "geometry",	  S_IFREG|S_IRUGO, &ide_geometry_proc_fops	},
-	{ "smart_values", S_IFREG|S_IRUSR, &idedisk_sv_proc_fops	},
-	{ "smart_thresholds", S_IFREG|S_IRUSR, &idedisk_st_proc_fops	},
+	{ "cache",	  S_IFREG|S_IRUGO, idedisk_cache_proc_show	},
+	{ "capacity",	  S_IFREG|S_IRUGO, idedisk_capacity_proc_show	},
+	{ "geometry",	  S_IFREG|S_IRUGO, ide_geometry_proc_show	},
+	{ "smart_values", S_IFREG|S_IRUSR, idedisk_sv_proc_show		},
+	{ "smart_thresholds", S_IFREG|S_IRUSR, idedisk_st_proc_show	},
 	{}
 };
 
diff --git a/drivers/ide/ide-dma.c b/drivers/ide/ide-dma.c
index 54d4d78ca46a..6f344654ef22 100644
--- a/drivers/ide/ide-dma.c
+++ b/drivers/ide/ide-dma.c
@@ -180,7 +180,6 @@ EXPORT_SYMBOL_GPL(ide_dma_unmap_sg);
 void ide_dma_off_quietly(ide_drive_t *drive)
 {
 	drive->dev_flags &= ~IDE_DFLAG_USING_DMA;
-	ide_toggle_bounce(drive, 0);
 
 	drive->hwif->dma_ops->dma_host_set(drive, 0);
 }
@@ -211,7 +210,6 @@ EXPORT_SYMBOL(ide_dma_off);
 void ide_dma_on(ide_drive_t *drive)
 {
 	drive->dev_flags |= IDE_DFLAG_USING_DMA;
-	ide_toggle_bounce(drive, 1);
 
 	drive->hwif->dma_ops->dma_host_set(drive, 1);
 }
diff --git a/drivers/ide/ide-floppy_proc.c b/drivers/ide/ide-floppy_proc.c
index 471457ebea67..7f697ddb5fe5 100644
--- a/drivers/ide/ide-floppy_proc.c
+++ b/drivers/ide/ide-floppy_proc.c
@@ -14,22 +14,9 @@ static int idefloppy_capacity_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int idefloppy_capacity_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, idefloppy_capacity_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations idefloppy_capacity_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= idefloppy_capacity_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 ide_proc_entry_t ide_floppy_proc[] = {
-	{ "capacity",	S_IFREG|S_IRUGO, &idefloppy_capacity_proc_fops	},
-	{ "geometry",	S_IFREG|S_IRUGO, &ide_geometry_proc_fops	},
+	{ "capacity",	S_IFREG|S_IRUGO, idefloppy_capacity_proc_show	},
+	{ "geometry",	S_IFREG|S_IRUGO, ide_geometry_proc_show		},
 	{}
 };
 
diff --git a/drivers/ide/ide-ioctls.c b/drivers/ide/ide-ioctls.c
index 3661abb16a5f..af5119a73689 100644
--- a/drivers/ide/ide-ioctls.c
+++ b/drivers/ide/ide-ioctls.c
@@ -125,7 +125,7 @@ static int ide_cmd_ioctl(ide_drive_t *drive, unsigned long arg)
 	if (NULL == (void *) arg) {
 		struct request *rq;
 
-		rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+		rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
 		ide_req(rq)->type = ATA_PRIV_TASKFILE;
 		blk_execute_rq(drive->queue, NULL, rq, 0);
 		err = scsi_req(rq)->result ? -EIO : 0;
@@ -222,7 +222,7 @@ static int generic_drive_reset(ide_drive_t *drive)
 	struct request *rq;
 	int ret = 0;
 
-	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
 	ide_req(rq)->type = ATA_PRIV_MISC;
 	scsi_req(rq)->cmd_len = 1;
 	scsi_req(rq)->cmd[0] = REQ_DRIVE_RESET;
diff --git a/drivers/ide/ide-lib.c b/drivers/ide/ide-lib.c
index e1180fa46196..78cb79eddc8b 100644
--- a/drivers/ide/ide-lib.c
+++ b/drivers/ide/ide-lib.c
@@ -6,32 +6,6 @@
 #include <linux/ide.h>
 #include <linux/bitops.h>
 
-/**
- *	ide_toggle_bounce	-	handle bounce buffering
- *	@drive: drive to update
- *	@on: on/off boolean
- *
- *	Enable or disable bounce buffering for the device. Drives move
- *	between PIO and DMA and that changes the rules we need.
- */
-
-void ide_toggle_bounce(ide_drive_t *drive, int on)
-{
-	u64 addr = BLK_BOUNCE_HIGH;	/* dma64_addr_t */
-
-	if (!PCI_DMA_BUS_IS_PHYS) {
-		addr = BLK_BOUNCE_ANY;
-	} else if (on && drive->media == ide_disk) {
-		struct device *dev = drive->hwif->dev;
-
-		if (dev && dev->dma_mask)
-			addr = *dev->dma_mask;
-	}
-
-	if (drive->queue)
-		blk_queue_bounce_limit(drive->queue, addr);
-}
-
 u64 ide_get_lba_addr(struct ide_cmd *cmd, int lba48)
 {
 	struct ide_taskfile *tf = &cmd->tf;
diff --git a/drivers/ide/ide-park.c b/drivers/ide/ide-park.c
index 6465bcc7cea6..622f0edb3945 100644
--- a/drivers/ide/ide-park.c
+++ b/drivers/ide/ide-park.c
@@ -32,7 +32,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout)
 	}
 	spin_unlock_irq(&hwif->lock);
 
-	rq = blk_get_request(q, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	rq = blk_get_request(q, REQ_OP_DRV_IN, 0);
 	scsi_req(rq)->cmd[0] = REQ_PARK_HEADS;
 	scsi_req(rq)->cmd_len = 1;
 	ide_req(rq)->type = ATA_PRIV_MISC;
@@ -47,7 +47,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout)
 	 * Make sure that *some* command is sent to the drive after the
 	 * timeout has expired, so power management will be reenabled.
 	 */
-	rq = blk_get_request(q, REQ_OP_DRV_IN, GFP_NOWAIT);
+	rq = blk_get_request(q, REQ_OP_DRV_IN, BLK_MQ_REQ_NOWAIT);
 	if (IS_ERR(rq))
 		goto out;
 
diff --git a/drivers/ide/ide-pm.c b/drivers/ide/ide-pm.c
index ad8a125defdd..59217aa1d1fb 100644
--- a/drivers/ide/ide-pm.c
+++ b/drivers/ide/ide-pm.c
@@ -19,7 +19,7 @@ int generic_ide_suspend(struct device *dev, pm_message_t mesg)
 	}
 
 	memset(&rqpm, 0, sizeof(rqpm));
-	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
 	ide_req(rq)->type = ATA_PRIV_PM_SUSPEND;
 	rq->special = &rqpm;
 	rqpm.pm_step = IDE_PM_START_SUSPEND;
@@ -90,8 +90,7 @@ int generic_ide_resume(struct device *dev)
 	}
 
 	memset(&rqpm, 0, sizeof(rqpm));
-	rq = blk_get_request_flags(drive->queue, REQ_OP_DRV_IN,
-				   BLK_MQ_REQ_PREEMPT);
+	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_PREEMPT);
 	ide_req(rq)->type = ATA_PRIV_PM_RESUME;
 	rq->special = &rqpm;
 	rqpm.pm_step = IDE_PM_START_RESUME;
diff --git a/drivers/ide/ide-probe.c b/drivers/ide/ide-probe.c
index 2019e66eada7..56d7bc228cb3 100644
--- a/drivers/ide/ide-probe.c
+++ b/drivers/ide/ide-probe.c
@@ -796,8 +796,7 @@ static int ide_init_queue(ide_drive_t *drive)
 	 * This will be fixed once we teach pci_map_sg() about our boundary
 	 * requirements, hopefully soon. *FIXME*
 	 */
-	if (!PCI_DMA_BUS_IS_PHYS)
-		max_sg_entries >>= 1;
+	max_sg_entries >>= 1;
 #endif /* CONFIG_PCI */
 
 	blk_queue_max_segments(q, max_sg_entries);
@@ -805,9 +804,6 @@ static int ide_init_queue(ide_drive_t *drive)
 	/* assign drive queue */
 	drive->queue = q;
 
-	/* needs drive->queue to be set */
-	ide_toggle_bounce(drive, 1);
-
 	return 0;
 }
 
diff --git a/drivers/ide/ide-proc.c b/drivers/ide/ide-proc.c
index 863db44c7916..45c997430332 100644
--- a/drivers/ide/ide-proc.c
+++ b/drivers/ide/ide-proc.c
@@ -56,19 +56,6 @@ static int ide_imodel_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int ide_imodel_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ide_imodel_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations ide_imodel_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= ide_imodel_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int ide_mate_proc_show(struct seq_file *m, void *v)
 {
 	ide_hwif_t	*hwif = (ide_hwif_t *) m->private;
@@ -80,19 +67,6 @@ static int ide_mate_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int ide_mate_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ide_mate_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations ide_mate_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= ide_mate_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int ide_channel_proc_show(struct seq_file *m, void *v)
 {
 	ide_hwif_t	*hwif = (ide_hwif_t *) m->private;
@@ -101,19 +75,6 @@ static int ide_channel_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int ide_channel_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ide_channel_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations ide_channel_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= ide_channel_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int ide_identify_proc_show(struct seq_file *m, void *v)
 {
 	ide_drive_t *drive = (ide_drive_t *)m->private;
@@ -141,19 +102,6 @@ static int ide_identify_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int ide_identify_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ide_identify_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations ide_identify_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= ide_identify_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 /**
  *	ide_find_setting	-	find a specific setting
  *	@st: setting table pointer
@@ -441,27 +389,14 @@ static const struct file_operations ide_settings_proc_fops = {
 	.write		= ide_settings_proc_write,
 };
 
-static int ide_capacity_proc_show(struct seq_file *m, void *v)
+int ide_capacity_proc_show(struct seq_file *m, void *v)
 {
 	seq_printf(m, "%llu\n", (long long)0x7fffffff);
 	return 0;
 }
+EXPORT_SYMBOL_GPL(ide_capacity_proc_show);
 
-static int ide_capacity_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ide_capacity_proc_show, NULL);
-}
-
-const struct file_operations ide_capacity_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= ide_capacity_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-EXPORT_SYMBOL_GPL(ide_capacity_proc_fops);
-
-static int ide_geometry_proc_show(struct seq_file *m, void *v)
+int ide_geometry_proc_show(struct seq_file *m, void *v)
 {
 	ide_drive_t	*drive = (ide_drive_t *) m->private;
 
@@ -471,20 +406,7 @@ static int ide_geometry_proc_show(struct seq_file *m, void *v)
 			drive->bios_cyl, drive->bios_head, drive->bios_sect);
 	return 0;
 }
-
-static int ide_geometry_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ide_geometry_proc_show, PDE_DATA(inode));
-}
-
-const struct file_operations ide_geometry_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= ide_geometry_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-EXPORT_SYMBOL(ide_geometry_proc_fops);
+EXPORT_SYMBOL(ide_geometry_proc_show);
 
 static int ide_dmodel_proc_show(struct seq_file *seq, void *v)
 {
@@ -495,19 +417,6 @@ static int ide_dmodel_proc_show(struct seq_file *seq, void *v)
 	return 0;
 }
 
-static int ide_dmodel_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ide_dmodel_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations ide_dmodel_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= ide_dmodel_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int ide_driver_proc_show(struct seq_file *m, void *v)
 {
 	ide_drive_t		*drive = (ide_drive_t *)m->private;
@@ -523,65 +432,6 @@ static int ide_driver_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int ide_driver_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ide_driver_proc_show, PDE_DATA(inode));
-}
-
-static int ide_replace_subdriver(ide_drive_t *drive, const char *driver)
-{
-	struct device *dev = &drive->gendev;
-	int ret = 1;
-	int err;
-
-	device_release_driver(dev);
-	/* FIXME: device can still be in use by previous driver */
-	strlcpy(drive->driver_req, driver, sizeof(drive->driver_req));
-	err = device_attach(dev);
-	if (err < 0)
-		printk(KERN_WARNING "IDE: %s: device_attach error: %d\n",
-			__func__, err);
-	drive->driver_req[0] = 0;
-	if (dev->driver == NULL) {
-		err = device_attach(dev);
-		if (err < 0)
-			printk(KERN_WARNING
-				"IDE: %s: device_attach(2) error: %d\n",
-				__func__, err);
-	}
-	if (dev->driver && !strcmp(dev->driver->name, driver))
-		ret = 0;
-
-	return ret;
-}
-
-static ssize_t ide_driver_proc_write(struct file *file, const char __user *buffer,
-				     size_t count, loff_t *pos)
-{
-	ide_drive_t	*drive = PDE_DATA(file_inode(file));
-	char name[32];
-
-	if (!capable(CAP_SYS_ADMIN))
-		return -EACCES;
-	if (count > 31)
-		count = 31;
-	if (copy_from_user(name, buffer, count))
-		return -EFAULT;
-	name[count] = '\0';
-	if (ide_replace_subdriver(drive, name))
-		return -EINVAL;
-	return count;
-}
-
-static const struct file_operations ide_driver_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= ide_driver_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-	.write		= ide_driver_proc_write,
-};
-
 static int ide_media_proc_show(struct seq_file *m, void *v)
 {
 	ide_drive_t	*drive = (ide_drive_t *) m->private;
@@ -613,11 +463,10 @@ static const struct file_operations ide_media_proc_fops = {
 };
 
 static ide_proc_entry_t generic_drive_entries[] = {
-	{ "driver",	S_IFREG|S_IRUGO,	 &ide_driver_proc_fops	},
-	{ "identify",	S_IFREG|S_IRUSR,	 &ide_identify_proc_fops},
-	{ "media",	S_IFREG|S_IRUGO,	 &ide_media_proc_fops	},
-	{ "model",	S_IFREG|S_IRUGO,	 &ide_dmodel_proc_fops	},
-	{ "settings",	S_IFREG|S_IRUSR|S_IWUSR, &ide_settings_proc_fops},
+	{ "driver",	S_IFREG|S_IRUGO,	 ide_driver_proc_show	},
+	{ "identify",	S_IFREG|S_IRUSR,	 ide_identify_proc_show	},
+	{ "media",	S_IFREG|S_IRUGO,	 ide_media_proc_show	},
+	{ "model",	S_IFREG|S_IRUGO,	 ide_dmodel_proc_show	},
 	{}
 };
 
@@ -628,7 +477,7 @@ static void ide_add_proc_entries(struct proc_dir_entry *dir, ide_proc_entry_t *p
 	if (!dir || !p)
 		return;
 	while (p->name != NULL) {
-		ent = proc_create_data(p->name, p->mode, dir, p->proc_fops, data);
+		ent = proc_create_single_data(p->name, p->mode, dir, p->show, data);
 		if (!ent) return;
 		p++;
 	}
@@ -693,8 +542,12 @@ void ide_proc_port_register_devices(ide_hwif_t *hwif)
 			continue;
 
 		drive->proc = proc_mkdir(drive->name, parent);
-		if (drive->proc)
+		if (drive->proc) {
 			ide_add_proc_entries(drive->proc, generic_drive_entries, drive);
+			proc_create_data("setting", S_IFREG|S_IRUSR|S_IWUSR,
+					drive->proc, &ide_settings_proc_fops,
+					drive);
+		}
 		sprintf(name, "ide%d/%s", (drive->name[2]-'a')/2, drive->name);
 		ent = proc_symlink(drive->name, proc_ide_root, name);
 		if (!ent) return;
@@ -704,6 +557,7 @@ void ide_proc_port_register_devices(ide_hwif_t *hwif)
 void ide_proc_unregister_device(ide_drive_t *drive)
 {
 	if (drive->proc) {
+		remove_proc_entry("settings", drive->proc);
 		ide_remove_proc_entries(drive->proc, generic_drive_entries);
 		remove_proc_entry(drive->name, proc_ide_root);
 		remove_proc_entry(drive->name, drive->hwif->proc);
@@ -712,9 +566,9 @@ void ide_proc_unregister_device(ide_drive_t *drive)
 }
 
 static ide_proc_entry_t hwif_entries[] = {
-	{ "channel",	S_IFREG|S_IRUGO,	&ide_channel_proc_fops	},
-	{ "mate",	S_IFREG|S_IRUGO,	&ide_mate_proc_fops	},
-	{ "model",	S_IFREG|S_IRUGO,	&ide_imodel_proc_fops	},
+	{ "channel",	S_IFREG|S_IRUGO,	ide_channel_proc_show	},
+	{ "mate",	S_IFREG|S_IRUGO,	ide_mate_proc_show	},
+	{ "model",	S_IFREG|S_IRUGO,	ide_imodel_proc_show	},
 	{}
 };
 
diff --git a/drivers/ide/ide-tape.c b/drivers/ide/ide-tape.c
index fd57e8ccc47a..aee7b46d2330 100644
--- a/drivers/ide/ide-tape.c
+++ b/drivers/ide/ide-tape.c
@@ -854,7 +854,7 @@ static int idetape_queue_rw_tail(ide_drive_t *drive, int cmd, int size)
 	BUG_ON(cmd != REQ_IDETAPE_READ && cmd != REQ_IDETAPE_WRITE);
 	BUG_ON(size < 0 || size % tape->blk_size);
 
-	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
 	ide_req(rq)->type = ATA_PRIV_MISC;
 	scsi_req(rq)->cmd[13] = cmd;
 	rq->rq_disk = tape->disk;
@@ -862,7 +862,7 @@ static int idetape_queue_rw_tail(ide_drive_t *drive, int cmd, int size)
 
 	if (size) {
 		ret = blk_rq_map_kern(drive->queue, rq, tape->buf, size,
-				      __GFP_RECLAIM);
+				      GFP_NOIO);
 		if (ret)
 			goto out_put;
 	}
@@ -1847,22 +1847,9 @@ static int idetape_name_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int idetape_name_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, idetape_name_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations idetape_name_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= idetape_name_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static ide_proc_entry_t idetape_proc[] = {
-	{ "capacity",	S_IFREG|S_IRUGO,	&ide_capacity_proc_fops	},
-	{ "name",	S_IFREG|S_IRUGO,	&idetape_name_proc_fops	},
+	{ "capacity",	S_IFREG|S_IRUGO,	ide_capacity_proc_show	},
+	{ "name",	S_IFREG|S_IRUGO,	idetape_name_proc_show	},
 	{}
 };
 
diff --git a/drivers/ide/ide-taskfile.c b/drivers/ide/ide-taskfile.c
index abe0822dd429..c034cd965831 100644
--- a/drivers/ide/ide-taskfile.c
+++ b/drivers/ide/ide-taskfile.c
@@ -431,7 +431,7 @@ int ide_raw_taskfile(ide_drive_t *drive, struct ide_cmd *cmd, u8 *buf,
 
 	rq = blk_get_request(drive->queue,
 		(cmd->tf_flags & IDE_TFLAG_WRITE) ?
-			REQ_OP_DRV_OUT : REQ_OP_DRV_IN, __GFP_RECLAIM);
+			REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
 	ide_req(rq)->type = ATA_PRIV_TASKFILE;
 
 	/*
@@ -442,7 +442,7 @@ int ide_raw_taskfile(ide_drive_t *drive, struct ide_cmd *cmd, u8 *buf,
 	 */
 	if (nsect) {
 		error = blk_rq_map_kern(drive->queue, rq, buf,
-					nsect * SECTOR_SIZE, __GFP_RECLAIM);
+					nsect * SECTOR_SIZE, GFP_NOIO);
 		if (error)
 			goto put_req;
 	}
diff --git a/drivers/iio/Kconfig b/drivers/iio/Kconfig
index b3c8c6ef0dff..d08aeb41cd07 100644
--- a/drivers/iio/Kconfig
+++ b/drivers/iio/Kconfig
@@ -70,6 +70,7 @@ config IIO_TRIGGERED_EVENT
 
 source "drivers/iio/accel/Kconfig"
 source "drivers/iio/adc/Kconfig"
+source "drivers/iio/afe/Kconfig"
 source "drivers/iio/amplifiers/Kconfig"
 source "drivers/iio/chemical/Kconfig"
 source "drivers/iio/common/Kconfig"
@@ -92,6 +93,7 @@ source "drivers/iio/potentiometer/Kconfig"
 source "drivers/iio/potentiostat/Kconfig"
 source "drivers/iio/pressure/Kconfig"
 source "drivers/iio/proximity/Kconfig"
+source "drivers/iio/resolver/Kconfig"
 source "drivers/iio/temperature/Kconfig"
 
 endif # IIO
diff --git a/drivers/iio/Makefile b/drivers/iio/Makefile
index b16b2e9ddc40..cb5993251381 100644
--- a/drivers/iio/Makefile
+++ b/drivers/iio/Makefile
@@ -15,6 +15,7 @@ obj-$(CONFIG_IIO_TRIGGERED_EVENT) += industrialio-triggered-event.o
 
 obj-y += accel/
 obj-y += adc/
+obj-y += afe/
 obj-y += amplifiers/
 obj-y += buffer/
 obj-y += chemical/
@@ -35,5 +36,6 @@ obj-y += potentiometer/
 obj-y += potentiostat/
 obj-y += pressure/
 obj-y += proximity/
+obj-y += resolver/
 obj-y += temperature/
 obj-y += trigger/
diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig
index c6d9517d7611..62ae7e5abcfa 100644
--- a/drivers/iio/accel/Kconfig
+++ b/drivers/iio/accel/Kconfig
@@ -5,6 +5,30 @@
 
 menu "Accelerometers"
 
+config ADIS16201
+        tristate "Analog Devices ADIS16201 Dual-Axis Digital Inclinometer and Accelerometer"
+        depends on SPI
+        select IIO_ADIS_LIB
+        select IIO_ADIS_LIB_BUFFER if IIO_BUFFER
+        help
+          Say Y here to build support for Analog Devices adis16201 dual-axis
+          digital inclinometer and accelerometer.
+
+          To compile this driver as a module, say M here: the module will
+          be called adis16201.
+
+config ADIS16209
+        tristate "Analog Devices ADIS16209 Dual-Axis Digital Inclinometer and Accelerometer"
+        depends on SPI
+        select IIO_ADIS_LIB
+        select IIO_ADIS_LIB_BUFFER if IIO_BUFFER
+        help
+          Say Y here to build support for Analog Devices adis16209 dual-axis digital inclinometer
+          and accelerometer.
+
+          To compile this driver as a module, say M here: the module will be
+          called adis16209.
+
 config ADXL345
 	tristate
 
diff --git a/drivers/iio/accel/Makefile b/drivers/iio/accel/Makefile
index 368aedb6377a..636d4d1b2990 100644
--- a/drivers/iio/accel/Makefile
+++ b/drivers/iio/accel/Makefile
@@ -4,6 +4,8 @@
 #
 
 # When adding new entries keep the list in alphabetical order
+obj-$(CONFIG_ADIS16201) += adis16201.o
+obj-$(CONFIG_ADIS16209) += adis16209.o
 obj-$(CONFIG_ADXL345) += adxl345_core.o
 obj-$(CONFIG_ADXL345_I2C) += adxl345_i2c.o
 obj-$(CONFIG_ADXL345_SPI) += adxl345_spi.o
diff --git a/drivers/iio/accel/adis16201.c b/drivers/iio/accel/adis16201.c
new file mode 100644
index 000000000000..4c1d482ea73a
--- /dev/null
+++ b/drivers/iio/accel/adis16201.c
@@ -0,0 +1,321 @@
+/*
+ * ADIS16201 Dual-Axis Digital Inclinometer and Accelerometer
+ *
+ * Copyright 2010 Analog Devices Inc.
+ *
+ * Licensed under the GPL-2 or later.
+ */
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/spi/spi.h>
+#include <linux/sysfs.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/imu/adis.h>
+
+#define ADIS16201_STARTUP_DELAY_MS			220
+#define ADIS16201_FLASH_CNT				0x00
+
+/* Data Output Register Information */
+#define ADIS16201_SUPPLY_OUT_REG			0x02
+#define ADIS16201_XACCL_OUT_REG				0x04
+#define ADIS16201_YACCL_OUT_REG				0x06
+#define ADIS16201_AUX_ADC_REG				0x08
+#define ADIS16201_TEMP_OUT_REG				0x0A
+#define ADIS16201_XINCL_OUT_REG				0x0C
+#define ADIS16201_YINCL_OUT_REG				0x0E
+
+/* Calibration Register Definition */
+#define ADIS16201_XACCL_OFFS_REG			0x10
+#define ADIS16201_YACCL_OFFS_REG			0x12
+#define ADIS16201_XACCL_SCALE_REG			0x14
+#define ADIS16201_YACCL_SCALE_REG			0x16
+#define ADIS16201_XINCL_OFFS_REG			0x18
+#define ADIS16201_YINCL_OFFS_REG			0x1A
+#define ADIS16201_XINCL_SCALE_REG			0x1C
+#define ADIS16201_YINCL_SCALE_REG			0x1E
+
+/* Alarm Register Definition */
+#define ADIS16201_ALM_MAG1_REG				0x20
+#define ADIS16201_ALM_MAG2_REG				0x22
+#define ADIS16201_ALM_SMPL1_REG				0x24
+#define ADIS16201_ALM_SMPL2_REG				0x26
+#define ADIS16201_ALM_CTRL_REG				0x28
+
+#define ADIS16201_AUX_DAC_REG				0x30
+#define ADIS16201_GPIO_CTRL_REG				0x32
+#define ADIS16201_SMPL_PRD_REG				0x36
+/* Operation, filter configuration */
+#define ADIS16201_AVG_CNT_REG				0x38
+#define ADIS16201_SLP_CNT_REG				0x3A
+
+/* Miscellaneous Control Register Definition */
+#define ADIS16201_MSC_CTRL_REG				0x34
+#define  ADIS16201_MSC_CTRL_SELF_TEST_EN		BIT(8)
+/* Data-ready enable: 1 = enabled, 0 = disabled */
+#define  ADIS16201_MSC_CTRL_DATA_RDY_EN			BIT(2)
+/* Data-ready polarity: 1 = active high, 0 = active low */
+#define  ADIS16201_MSC_CTRL_ACTIVE_DATA_RDY_HIGH	BIT(1)
+/* Data-ready line selection: 1 = DIO1, 0 = DIO0 */
+#define  ADIS16201_MSC_CTRL_DATA_RDY_DIO1		BIT(0)
+
+/* Diagnostics System Status Register Definition */
+#define ADIS16201_DIAG_STAT_REG				0x3C
+#define  ADIS16201_DIAG_STAT_ALARM2			BIT(9)
+#define  ADIS16201_DIAG_STAT_ALARM1			BIT(8)
+#define  ADIS16201_DIAG_STAT_SPI_FAIL_BIT		3
+#define  ADIS16201_DIAG_STAT_FLASH_UPT_FAIL_BIT		2
+/* Power supply above 3.625 V */
+#define  ADIS16201_DIAG_STAT_POWER_HIGH_BIT		1
+/* Power supply below 3.15 V */
+#define  ADIS16201_DIAG_STAT_POWER_LOW_BIT		0
+
+/* System Command Register Definition */
+#define ADIS16201_GLOB_CMD_REG				0x3E
+#define  ADIS16201_GLOB_CMD_SW_RESET			BIT(7)
+#define  ADIS16201_GLOB_CMD_FACTORY_RESET		BIT(1)
+
+#define ADIS16201_ERROR_ACTIVE				BIT(14)
+
+enum adis16201_scan {
+	ADIS16201_SCAN_ACC_X,
+	ADIS16201_SCAN_ACC_Y,
+	ADIS16201_SCAN_INCLI_X,
+	ADIS16201_SCAN_INCLI_Y,
+	ADIS16201_SCAN_SUPPLY,
+	ADIS16201_SCAN_AUX_ADC,
+	ADIS16201_SCAN_TEMP,
+};
+
+static const u8 adis16201_addresses[] = {
+	[ADIS16201_SCAN_ACC_X] = ADIS16201_XACCL_OFFS_REG,
+	[ADIS16201_SCAN_ACC_Y] = ADIS16201_YACCL_OFFS_REG,
+	[ADIS16201_SCAN_INCLI_X] = ADIS16201_XINCL_OFFS_REG,
+	[ADIS16201_SCAN_INCLI_Y] = ADIS16201_YINCL_OFFS_REG,
+};
+
+static int adis16201_read_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      int *val, int *val2,
+			      long mask)
+{
+	struct adis *st = iio_priv(indio_dev);
+	int ret;
+	int bits;
+	u8 addr;
+	s16 val16;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		return adis_single_conversion(indio_dev, chan,
+					      ADIS16201_ERROR_ACTIVE, val);
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->type) {
+		case IIO_VOLTAGE:
+			if (chan->channel == 0) {
+			/* Voltage base units are mV hence 1.22 mV */
+				*val = 1;
+				*val2 = 220000;
+			} else {
+			/* Voltage base units are mV hence 0.61 mV */
+				*val = 0;
+				*val2 = 610000;
+			}
+			return IIO_VAL_INT_PLUS_MICRO;
+		case IIO_TEMP:
+			*val = -470;
+			*val2 = 0;
+			return IIO_VAL_INT_PLUS_MICRO;
+		case IIO_ACCEL:
+			/*
+			 * IIO base unit for sensitivity of accelerometer
+			 * is milli g.
+			 * 1 LSB represents 0.244 mg.
+			 */
+			*val = 0;
+			*val2 = IIO_G_TO_M_S_2(462400);
+			return IIO_VAL_INT_PLUS_NANO;
+		case IIO_INCLI:
+			*val = 0;
+			*val2 = 100000;
+			return IIO_VAL_INT_PLUS_MICRO;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case IIO_CHAN_INFO_OFFSET:
+		/*
+		 * The raw ADC value is 1278 when the temperature
+		 * is 25 degrees and the scale factor per milli
+		 * degree celcius is -470.
+		 */
+		*val = 25000 / -470 - 1278;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_CALIBBIAS:
+		switch (chan->type) {
+		case IIO_ACCEL:
+			bits = 12;
+			break;
+		case IIO_INCLI:
+			bits = 9;
+			break;
+		default:
+			return -EINVAL;
+		}
+		addr = adis16201_addresses[chan->scan_index];
+		ret = adis_read_reg_16(st, addr, &val16);
+		if (ret)
+			return ret;
+
+		*val = sign_extend32(val16, bits - 1);
+		return IIO_VAL_INT;
+	}
+
+	return -EINVAL;
+}
+
+static int adis16201_write_raw(struct iio_dev *indio_dev,
+			       struct iio_chan_spec const *chan,
+			       int val,
+			       int val2,
+			       long mask)
+{
+	struct adis *st = iio_priv(indio_dev);
+	int m;
+
+	if (mask != IIO_CHAN_INFO_CALIBBIAS)
+		return -EINVAL;
+
+	switch (chan->type) {
+	case IIO_ACCEL:
+		m = GENMASK(11, 0);
+		break;
+	case IIO_INCLI:
+		m = GENMASK(8, 0);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return adis_write_reg_16(st, adis16201_addresses[chan->scan_index],
+				 val & m);
+}
+
+static const struct iio_chan_spec adis16201_channels[] = {
+	ADIS_SUPPLY_CHAN(ADIS16201_SUPPLY_OUT_REG, ADIS16201_SCAN_SUPPLY, 0,
+			 12),
+	ADIS_TEMP_CHAN(ADIS16201_TEMP_OUT_REG, ADIS16201_SCAN_TEMP, 0, 12),
+	ADIS_ACCEL_CHAN(X, ADIS16201_XACCL_OUT_REG, ADIS16201_SCAN_ACC_X,
+			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
+	ADIS_ACCEL_CHAN(Y, ADIS16201_YACCL_OUT_REG, ADIS16201_SCAN_ACC_Y,
+			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
+	ADIS_AUX_ADC_CHAN(ADIS16201_AUX_ADC_REG, ADIS16201_SCAN_AUX_ADC, 0, 12),
+	ADIS_INCLI_CHAN(X, ADIS16201_XINCL_OUT_REG, ADIS16201_SCAN_INCLI_X,
+			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
+	ADIS_INCLI_CHAN(X, ADIS16201_YINCL_OUT_REG, ADIS16201_SCAN_INCLI_Y,
+			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
+	IIO_CHAN_SOFT_TIMESTAMP(7)
+};
+
+static const struct iio_info adis16201_info = {
+	.read_raw = adis16201_read_raw,
+	.write_raw = adis16201_write_raw,
+	.update_scan_mode = adis_update_scan_mode,
+};
+
+static const char * const adis16201_status_error_msgs[] = {
+	[ADIS16201_DIAG_STAT_SPI_FAIL_BIT] = "SPI failure",
+	[ADIS16201_DIAG_STAT_FLASH_UPT_FAIL_BIT] = "Flash update failed",
+	[ADIS16201_DIAG_STAT_POWER_HIGH_BIT] = "Power supply above 3.625V",
+	[ADIS16201_DIAG_STAT_POWER_LOW_BIT] = "Power supply below 3.15V",
+};
+
+static const struct adis_data adis16201_data = {
+	.read_delay = 20,
+	.msc_ctrl_reg = ADIS16201_MSC_CTRL_REG,
+	.glob_cmd_reg = ADIS16201_GLOB_CMD_REG,
+	.diag_stat_reg = ADIS16201_DIAG_STAT_REG,
+
+	.self_test_mask = ADIS16201_MSC_CTRL_SELF_TEST_EN,
+	.self_test_no_autoclear = true,
+	.startup_delay = ADIS16201_STARTUP_DELAY_MS,
+
+	.status_error_msgs = adis16201_status_error_msgs,
+	.status_error_mask = BIT(ADIS16201_DIAG_STAT_SPI_FAIL_BIT) |
+		BIT(ADIS16201_DIAG_STAT_FLASH_UPT_FAIL_BIT) |
+		BIT(ADIS16201_DIAG_STAT_POWER_HIGH_BIT) |
+		BIT(ADIS16201_DIAG_STAT_POWER_LOW_BIT),
+};
+
+static int adis16201_probe(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev;
+	struct adis *st;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	spi_set_drvdata(spi, indio_dev);
+
+	indio_dev->name = spi->dev.driver->name;
+	indio_dev->dev.parent = &spi->dev;
+	indio_dev->info = &adis16201_info;
+
+	indio_dev->channels = adis16201_channels;
+	indio_dev->num_channels = ARRAY_SIZE(adis16201_channels);
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	ret = adis_init(st, indio_dev, spi, &adis16201_data);
+	if (ret)
+		return ret;
+
+	ret = adis_setup_buffer_and_trigger(st, indio_dev, NULL);
+	if (ret)
+		return ret;
+
+	ret = adis_initial_startup(st);
+	if (ret)
+		goto error_cleanup_buffer_trigger;
+
+	ret = iio_device_register(indio_dev);
+	if (ret < 0)
+		goto error_cleanup_buffer_trigger;
+
+	return 0;
+
+error_cleanup_buffer_trigger:
+	adis_cleanup_buffer_and_trigger(st, indio_dev);
+	return ret;
+}
+
+static int adis16201_remove(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev = spi_get_drvdata(spi);
+	struct adis *st = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+	adis_cleanup_buffer_and_trigger(st, indio_dev);
+
+	return 0;
+}
+
+static struct spi_driver adis16201_driver = {
+	.driver = {
+		.name = "adis16201",
+	},
+	.probe = adis16201_probe,
+	.remove = adis16201_remove,
+};
+module_spi_driver(adis16201_driver);
+
+MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
+MODULE_DESCRIPTION("Analog Devices ADIS16201 Dual-Axis Digital Inclinometer and Accelerometer");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("spi:adis16201");
diff --git a/drivers/iio/accel/adis16209.c b/drivers/iio/accel/adis16209.c
new file mode 100644
index 000000000000..f2dc3a5f0463
--- /dev/null
+++ b/drivers/iio/accel/adis16209.c
@@ -0,0 +1,330 @@
+/*
+ * ADIS16209 Dual-Axis Digital Inclinometer and Accelerometer
+ *
+ * Copyright 2010 Analog Devices Inc.
+ *
+ * Licensed under the GPL-2 or later.
+ */
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/spi/spi.h>
+#include <linux/slab.h>
+#include <linux/sysfs.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/imu/adis.h>
+
+#define ADIS16209_STARTUP_DELAY_MS	220
+#define ADIS16209_FLASH_CNT_REG		0x00
+
+/* Data Output Register Definitions */
+#define ADIS16209_SUPPLY_OUT_REG	0x02
+#define ADIS16209_XACCL_OUT_REG		0x04
+#define ADIS16209_YACCL_OUT_REG		0x06
+/* Output, auxiliary ADC input */
+#define ADIS16209_AUX_ADC_REG		0x08
+/* Output, temperature */
+#define ADIS16209_TEMP_OUT_REG		0x0A
+/* Output, +/- 90 degrees X-axis inclination */
+#define ADIS16209_XINCL_OUT_REG		0x0C
+#define ADIS16209_YINCL_OUT_REG		0x0E
+/* Output, +/-180 vertical rotational position */
+#define ADIS16209_ROT_OUT_REG		0x10
+
+/*
+ * Calibration Register Definitions.
+ * Acceleration, inclination or rotation offset null.
+ */
+#define ADIS16209_XACCL_NULL_REG	0x12
+#define ADIS16209_YACCL_NULL_REG	0x14
+#define ADIS16209_XINCL_NULL_REG	0x16
+#define ADIS16209_YINCL_NULL_REG	0x18
+#define ADIS16209_ROT_NULL_REG		0x1A
+
+/* Alarm Register Definitions */
+#define ADIS16209_ALM_MAG1_REG		0x20
+#define ADIS16209_ALM_MAG2_REG		0x22
+#define ADIS16209_ALM_SMPL1_REG		0x24
+#define ADIS16209_ALM_SMPL2_REG		0x26
+#define ADIS16209_ALM_CTRL_REG		0x28
+
+#define ADIS16209_AUX_DAC_REG		0x30
+#define ADIS16209_GPIO_CTRL_REG		0x32
+#define ADIS16209_SMPL_PRD_REG		0x36
+#define ADIS16209_AVG_CNT_REG		0x38
+#define ADIS16209_SLP_CNT_REG		0x3A
+
+#define ADIS16209_MSC_CTRL_REG			0x34
+#define  ADIS16209_MSC_CTRL_PWRUP_SELF_TEST	BIT(10)
+#define  ADIS16209_MSC_CTRL_SELF_TEST_EN	BIT(8)
+#define  ADIS16209_MSC_CTRL_DATA_RDY_EN		BIT(2)
+/* Data-ready polarity: 1 = active high, 0 = active low */
+#define  ADIS16209_MSC_CTRL_ACTIVE_HIGH		BIT(1)
+#define  ADIS16209_MSC_CTRL_DATA_RDY_DIO2	BIT(0)
+
+#define ADIS16209_STAT_REG			0x3C
+#define  ADIS16209_STAT_ALARM2			BIT(9)
+#define  ADIS16209_STAT_ALARM1			BIT(8)
+#define  ADIS16209_STAT_SELFTEST_FAIL_BIT	5
+#define  ADIS16209_STAT_SPI_FAIL_BIT		3
+#define  ADIS16209_STAT_FLASH_UPT_FAIL_BIT	2
+/* Power supply above 3.625 V */
+#define  ADIS16209_STAT_POWER_HIGH_BIT		1
+/* Power supply below 3.15 V */
+#define  ADIS16209_STAT_POWER_LOW_BIT		0
+
+#define ADIS16209_CMD_REG			0x3E
+#define  ADIS16209_CMD_SW_RESET			BIT(7)
+#define  ADIS16209_CMD_CLEAR_STAT		BIT(4)
+#define  ADIS16209_CMD_FACTORY_CAL		BIT(1)
+
+#define ADIS16209_ERROR_ACTIVE			BIT(14)
+
+enum adis16209_scan {
+	ADIS16209_SCAN_SUPPLY,
+	ADIS16209_SCAN_ACC_X,
+	ADIS16209_SCAN_ACC_Y,
+	ADIS16209_SCAN_AUX_ADC,
+	ADIS16209_SCAN_TEMP,
+	ADIS16209_SCAN_INCLI_X,
+	ADIS16209_SCAN_INCLI_Y,
+	ADIS16209_SCAN_ROT,
+};
+
+static const u8 adis16209_addresses[8][1] = {
+	[ADIS16209_SCAN_SUPPLY] = { },
+	[ADIS16209_SCAN_AUX_ADC] = { },
+	[ADIS16209_SCAN_ACC_X] = { ADIS16209_XACCL_NULL_REG },
+	[ADIS16209_SCAN_ACC_Y] = { ADIS16209_YACCL_NULL_REG },
+	[ADIS16209_SCAN_INCLI_X] = { ADIS16209_XINCL_NULL_REG },
+	[ADIS16209_SCAN_INCLI_Y] = { ADIS16209_YINCL_NULL_REG },
+	[ADIS16209_SCAN_ROT] = { },
+	[ADIS16209_SCAN_TEMP] = { },
+};
+
+static int adis16209_write_raw(struct iio_dev *indio_dev,
+			       struct iio_chan_spec const *chan,
+			       int val,
+			       int val2,
+			       long mask)
+{
+	struct adis *st = iio_priv(indio_dev);
+	int m;
+
+	if (mask != IIO_CHAN_INFO_CALIBBIAS)
+		return -EINVAL;
+
+	switch (chan->type) {
+	case IIO_ACCEL:
+	case IIO_INCLI:
+		m = GENMASK(13, 0);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return adis_write_reg_16(st, adis16209_addresses[chan->scan_index][0],
+				 val & m);
+}
+
+static int adis16209_read_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      int *val, int *val2,
+			      long mask)
+{
+	struct adis *st = iio_priv(indio_dev);
+	int ret;
+	int bits;
+	u8 addr;
+	s16 val16;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		return adis_single_conversion(indio_dev, chan,
+			ADIS16209_ERROR_ACTIVE, val);
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->type) {
+		case IIO_VOLTAGE:
+			*val = 0;
+			switch (chan->channel) {
+			case 0:
+				*val2 = 305180; /* 0.30518 mV */
+				break;
+			case 1:
+				*val2 = 610500; /* 0.6105 mV */
+				break;
+			default:
+				return -EINVAL;
+			}
+			return IIO_VAL_INT_PLUS_MICRO;
+		case IIO_TEMP:
+			*val = -470;
+			*val2 = 0;
+			return IIO_VAL_INT_PLUS_MICRO;
+		case IIO_ACCEL:
+			/*
+			 * IIO base unit for sensitivity of accelerometer
+			 * is milli g.
+			 * 1 LSB represents 0.244 mg.
+			 */
+			*val = 0;
+			*val2 = IIO_G_TO_M_S_2(244140);
+			return IIO_VAL_INT_PLUS_NANO;
+		case IIO_INCLI:
+		case IIO_ROT:
+			/*
+			 * IIO base units for rotation are degrees.
+			 * 1 LSB represents 0.025 milli degrees.
+			 */
+			*val = 0;
+			*val2 = 25000;
+			return IIO_VAL_INT_PLUS_MICRO;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case IIO_CHAN_INFO_OFFSET:
+		/*
+		 * The raw ADC value is 0x4FE when the temperature
+		 * is 45 degrees and the scale factor per milli
+		 * degree celcius is -470.
+		 */
+		*val = 25000 / -470 - 0x4FE;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_CALIBBIAS:
+		switch (chan->type) {
+		case IIO_ACCEL:
+			bits = 14;
+			break;
+		default:
+			return -EINVAL;
+		}
+		addr = adis16209_addresses[chan->scan_index][0];
+		ret = adis_read_reg_16(st, addr, &val16);
+		if (ret)
+			return ret;
+
+		*val = sign_extend32(val16, bits - 1);
+		return IIO_VAL_INT;
+	}
+	return -EINVAL;
+}
+
+static const struct iio_chan_spec adis16209_channels[] = {
+	ADIS_SUPPLY_CHAN(ADIS16209_SUPPLY_OUT_REG, ADIS16209_SCAN_SUPPLY,
+			 0, 14),
+	ADIS_TEMP_CHAN(ADIS16209_TEMP_OUT_REG, ADIS16209_SCAN_TEMP, 0, 12),
+	ADIS_ACCEL_CHAN(X, ADIS16209_XACCL_OUT_REG, ADIS16209_SCAN_ACC_X,
+			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
+	ADIS_ACCEL_CHAN(Y, ADIS16209_YACCL_OUT_REG, ADIS16209_SCAN_ACC_Y,
+			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
+	ADIS_AUX_ADC_CHAN(ADIS16209_AUX_ADC_REG, ADIS16209_SCAN_AUX_ADC, 0, 12),
+	ADIS_INCLI_CHAN(X, ADIS16209_XINCL_OUT_REG, ADIS16209_SCAN_INCLI_X,
+			0, 0, 14),
+	ADIS_INCLI_CHAN(Y, ADIS16209_YINCL_OUT_REG, ADIS16209_SCAN_INCLI_Y,
+			0, 0, 14),
+	ADIS_ROT_CHAN(X, ADIS16209_ROT_OUT_REG, ADIS16209_SCAN_ROT, 0, 0, 14),
+	IIO_CHAN_SOFT_TIMESTAMP(8)
+};
+
+static const struct iio_info adis16209_info = {
+	.read_raw = adis16209_read_raw,
+	.write_raw = adis16209_write_raw,
+	.update_scan_mode = adis_update_scan_mode,
+};
+
+static const char * const adis16209_status_error_msgs[] = {
+	[ADIS16209_STAT_SELFTEST_FAIL_BIT] = "Self test failure",
+	[ADIS16209_STAT_SPI_FAIL_BIT] = "SPI failure",
+	[ADIS16209_STAT_FLASH_UPT_FAIL_BIT] = "Flash update failed",
+	[ADIS16209_STAT_POWER_HIGH_BIT] = "Power supply above 3.625V",
+	[ADIS16209_STAT_POWER_LOW_BIT] = "Power supply below 3.15V",
+};
+
+static const struct adis_data adis16209_data = {
+	.read_delay = 30,
+	.msc_ctrl_reg = ADIS16209_MSC_CTRL_REG,
+	.glob_cmd_reg = ADIS16209_CMD_REG,
+	.diag_stat_reg = ADIS16209_STAT_REG,
+
+	.self_test_mask = ADIS16209_MSC_CTRL_SELF_TEST_EN,
+	.self_test_no_autoclear = true,
+	.startup_delay = ADIS16209_STARTUP_DELAY_MS,
+
+	.status_error_msgs = adis16209_status_error_msgs,
+	.status_error_mask = BIT(ADIS16209_STAT_SELFTEST_FAIL_BIT) |
+		BIT(ADIS16209_STAT_SPI_FAIL_BIT) |
+		BIT(ADIS16209_STAT_FLASH_UPT_FAIL_BIT) |
+		BIT(ADIS16209_STAT_POWER_HIGH_BIT) |
+		BIT(ADIS16209_STAT_POWER_LOW_BIT),
+};
+
+static int adis16209_probe(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev;
+	struct adis *st;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	spi_set_drvdata(spi, indio_dev);
+
+	indio_dev->name = spi->dev.driver->name;
+	indio_dev->dev.parent = &spi->dev;
+	indio_dev->info = &adis16209_info;
+	indio_dev->channels = adis16209_channels;
+	indio_dev->num_channels = ARRAY_SIZE(adis16209_channels);
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	ret = adis_init(st, indio_dev, spi, &adis16209_data);
+	if (ret)
+		return ret;
+
+	ret = adis_setup_buffer_and_trigger(st, indio_dev, NULL);
+	if (ret)
+		return ret;
+
+	ret = adis_initial_startup(st);
+	if (ret)
+		goto error_cleanup_buffer_trigger;
+	ret = iio_device_register(indio_dev);
+	if (ret)
+		goto error_cleanup_buffer_trigger;
+
+	return 0;
+
+error_cleanup_buffer_trigger:
+	adis_cleanup_buffer_and_trigger(st, indio_dev);
+	return ret;
+}
+
+static int adis16209_remove(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev = spi_get_drvdata(spi);
+	struct adis *st = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+	adis_cleanup_buffer_and_trigger(st, indio_dev);
+
+	return 0;
+}
+
+static struct spi_driver adis16209_driver = {
+	.driver = {
+		.name = "adis16209",
+	},
+	.probe = adis16209_probe,
+	.remove = adis16209_remove,
+};
+module_spi_driver(adis16209_driver);
+
+MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
+MODULE_DESCRIPTION("Analog Devices ADIS16209 Dual-Axis Digital Inclinometer and Accelerometer");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("spi:adis16209");
diff --git a/drivers/iio/accel/bmc150-accel-core.c b/drivers/iio/accel/bmc150-accel-core.c
index 208f2d9f0e8a..383c802eb5b8 100644
--- a/drivers/iio/accel/bmc150-accel-core.c
+++ b/drivers/iio/accel/bmc150-accel-core.c
@@ -837,29 +837,12 @@ static int bmc150_accel_fifo_transfer(struct bmc150_accel_data *data,
 	int sample_length = 3 * 2;
 	int ret;
 	int total_length = samples * sample_length;
-	int i;
-	size_t step = regmap_get_raw_read_max(data->regmap);
-
-	if (!step || step > total_length)
-		step = total_length;
-	else if (step < total_length)
-		step = sample_length;
-
-	/*
-	 * Seems we have a bus with size limitation so we have to execute
-	 * multiple reads
-	 */
-	for (i = 0; i < total_length; i += step) {
-		ret = regmap_raw_read(data->regmap, BMC150_ACCEL_REG_FIFO_DATA,
-				      &buffer[i], step);
-		if (ret)
-			break;
-	}
 
+	ret = regmap_raw_read(data->regmap, BMC150_ACCEL_REG_FIFO_DATA,
+			      buffer, total_length);
 	if (ret)
 		dev_err(dev,
-			"Error transferring data from fifo in single steps of %zu\n",
-			step);
+			"Error transferring data from fifo: %d\n", ret);
 
 	return ret;
 }
diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c
index 7a2da7f9d4dc..7e3d82cff3d5 100644
--- a/drivers/iio/accel/mma8452.c
+++ b/drivers/iio/accel/mma8452.c
@@ -106,6 +106,7 @@ struct mma8452_data {
 	u8 ctrl_reg1;
 	u8 data_cfg;
 	const struct mma_chip_info *chip_info;
+	int sleep_val;
 };
 
  /**
@@ -193,7 +194,11 @@ static int mma8452_drdy(struct mma8452_data *data)
 		if ((ret & MMA8452_STATUS_DRDY) == MMA8452_STATUS_DRDY)
 			return 0;
 
-		msleep(20);
+		if (data->sleep_val <= 20)
+			usleep_range(data->sleep_val * 250,
+				     data->sleep_val * 500);
+		else
+			msleep(20);
 	}
 
 	dev_err(&data->client->dev, "data not ready\n");
@@ -544,6 +549,18 @@ static int mma8452_read_raw(struct iio_dev *indio_dev,
 	return -EINVAL;
 }
 
+static int mma8452_calculate_sleep(struct mma8452_data *data)
+{
+	int ret, i = mma8452_get_odr_index(data);
+
+	if (mma8452_samp_freq[i][0] > 0)
+		ret = 1000 / mma8452_samp_freq[i][0];
+	else
+		ret = 1000;
+
+	return ret == 0 ? 1 : ret;
+}
+
 static int mma8452_standby(struct mma8452_data *data)
 {
 	return i2c_smbus_write_byte_data(data->client, MMA8452_CTRL_REG1,
@@ -700,6 +717,8 @@ static int mma8452_write_raw(struct iio_dev *indio_dev,
 		data->ctrl_reg1 &= ~MMA8452_CTRL_DR_MASK;
 		data->ctrl_reg1 |= i << MMA8452_CTRL_DR_SHIFT;
 
+		data->sleep_val = mma8452_calculate_sleep(data);
+
 		ret = mma8452_change_config(data, MMA8452_CTRL_REG1,
 					    data->ctrl_reg1);
 		break;
@@ -1593,6 +1612,9 @@ static int mma8452_probe(struct i2c_client *client,
 
 	data->ctrl_reg1 = MMA8452_CTRL_ACTIVE |
 			  (MMA8452_CTRL_DR_DEFAULT << MMA8452_CTRL_DR_SHIFT);
+
+	data->sleep_val = mma8452_calculate_sleep(data);
+
 	ret = i2c_smbus_write_byte_data(client, MMA8452_CTRL_REG1,
 					data->ctrl_reg1);
 	if (ret < 0)
diff --git a/drivers/iio/accel/sca3000.c b/drivers/iio/accel/sca3000.c
index f33dadf7b262..4dceb75e3586 100644
--- a/drivers/iio/accel/sca3000.c
+++ b/drivers/iio/accel/sca3000.c
@@ -1277,7 +1277,7 @@ static int sca3000_configure_ring(struct iio_dev *indio_dev)
 {
 	struct iio_buffer *buffer;
 
-	buffer = iio_kfifo_allocate();
+	buffer = devm_iio_kfifo_allocate(&indio_dev->dev);
 	if (!buffer)
 		return -ENOMEM;
 
@@ -1287,11 +1287,6 @@ static int sca3000_configure_ring(struct iio_dev *indio_dev)
 	return 0;
 }
 
-static void sca3000_unconfigure_ring(struct iio_dev *indio_dev)
-{
-	iio_kfifo_free(indio_dev->buffer);
-}
-
 static inline
 int __sca3000_hw_ring_state_set(struct iio_dev *indio_dev, bool state)
 {
@@ -1486,7 +1481,9 @@ static int sca3000_probe(struct spi_device *spi)
 	}
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	sca3000_configure_ring(indio_dev);
+	ret = sca3000_configure_ring(indio_dev);
+	if (ret)
+		return ret;
 
 	if (spi->irq) {
 		ret = request_threaded_irq(spi->irq,
@@ -1546,8 +1543,6 @@ static int sca3000_remove(struct spi_device *spi)
 	if (spi->irq)
 		free_irq(spi->irq, indio_dev);
 
-	sca3000_unconfigure_ring(indio_dev);
-
 	return 0;
 }
 
diff --git a/drivers/iio/accel/st_accel_i2c.c b/drivers/iio/accel/st_accel_i2c.c
index 6bdec8c451e0..056dddb27236 100644
--- a/drivers/iio/accel/st_accel_i2c.c
+++ b/drivers/iio/accel/st_accel_i2c.c
@@ -107,6 +107,7 @@ MODULE_DEVICE_TABLE(of, st_accel_of_match);
 
 #ifdef CONFIG_ACPI
 static const struct acpi_device_id st_accel_acpi_match[] = {
+	{"SMO8840", LNG2DM},
 	{"SMO8A90", LNG2DM},
 	{ },
 };
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index 15606f237480..9da79070357c 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -158,6 +158,7 @@ config AT91_SAMA5D2_ADC
 	depends on ARCH_AT91 || COMPILE_TEST
 	depends on HAS_IOMEM
 	depends on HAS_DMA
+	select IIO_BUFFER
 	select IIO_TRIGGERED_BUFFER
 	help
 	  Say yes here to build support for Atmel SAMA5D2 ADC which is
diff --git a/drivers/iio/adc/ad7791.c b/drivers/iio/adc/ad7791.c
index 70fbf92f9827..a9ff0695ddf7 100644
--- a/drivers/iio/adc/ad7791.c
+++ b/drivers/iio/adc/ad7791.c
@@ -153,6 +153,17 @@ struct ad7791_state {
 	const struct ad7791_chip_info *info;
 };
 
+static const int ad7791_sample_freq_avail[8][2] = {
+	[AD7791_FILTER_RATE_120] =  { 120, 0 },
+	[AD7791_FILTER_RATE_100] =  { 100, 0 },
+	[AD7791_FILTER_RATE_33_3] = { 33,  300000 },
+	[AD7791_FILTER_RATE_20] =   { 20,  0 },
+	[AD7791_FILTER_RATE_16_6] = { 16,  600000 },
+	[AD7791_FILTER_RATE_16_7] = { 16,  700000 },
+	[AD7791_FILTER_RATE_13_3] = { 13,  300000 },
+	[AD7791_FILTER_RATE_9_5] =  { 9,   500000 },
+};
+
 static struct ad7791_state *ad_sigma_delta_to_ad7791(struct ad_sigma_delta *sd)
 {
 	return container_of(sd, struct ad7791_state, sd);
@@ -202,6 +213,7 @@ static int ad7791_read_raw(struct iio_dev *indio_dev,
 {
 	struct ad7791_state *st = iio_priv(indio_dev);
 	bool unipolar = !!(st->mode & AD7791_MODE_UNIPOLAR);
+	unsigned int rate;
 
 	switch (info) {
 	case IIO_CHAN_INFO_RAW:
@@ -239,63 +251,56 @@ static int ad7791_read_raw(struct iio_dev *indio_dev,
 			*val2 = chan->scan_type.realbits - 1;
 
 		return IIO_VAL_FRACTIONAL_LOG2;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		rate = st->filter & AD7791_FILTER_RATE_MASK;
+		*val = ad7791_sample_freq_avail[rate][0];
+		*val2 = ad7791_sample_freq_avail[rate][1];
+		return IIO_VAL_INT_PLUS_MICRO;
 	}
 
 	return -EINVAL;
 }
 
-static const char * const ad7791_sample_freq_avail[] = {
-	[AD7791_FILTER_RATE_120] = "120",
-	[AD7791_FILTER_RATE_100] = "100",
-	[AD7791_FILTER_RATE_33_3] = "33.3",
-	[AD7791_FILTER_RATE_20] = "20",
-	[AD7791_FILTER_RATE_16_6] = "16.6",
-	[AD7791_FILTER_RATE_16_7] = "16.7",
-	[AD7791_FILTER_RATE_13_3] = "13.3",
-	[AD7791_FILTER_RATE_9_5] = "9.5",
-};
-
-static ssize_t ad7791_read_frequency(struct device *dev,
-	struct device_attribute *attr, char *buf)
+static int ad7791_write_raw(struct iio_dev *indio_dev,
+	struct iio_chan_spec const *chan, int val, int val2, long mask)
 {
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ad7791_state *st = iio_priv(indio_dev);
-	unsigned int rate = st->filter & AD7791_FILTER_RATE_MASK;
-
-	return sprintf(buf, "%s\n", ad7791_sample_freq_avail[rate]);
-}
-
-static ssize_t ad7791_write_frequency(struct device *dev,
-	struct device_attribute *attr, const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7791_state *st = iio_priv(indio_dev);
-	int i, ret;
-
-	i = sysfs_match_string(ad7791_sample_freq_avail, buf);
-	if (i < 0)
-		return i;
+	int ret, i;
 
 	ret = iio_device_claim_direct_mode(indio_dev);
 	if (ret)
 		return ret;
-	st->filter &= ~AD7791_FILTER_RATE_MASK;
-	st->filter |= i;
-	ad_sd_write_reg(&st->sd, AD7791_REG_FILTER, sizeof(st->filter),
-			st->filter);
-	iio_device_release_direct_mode(indio_dev);
 
-	return len;
-}
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		for (i = 0; i < ARRAY_SIZE(ad7791_sample_freq_avail); i++) {
+			if (ad7791_sample_freq_avail[i][0] == val &&
+			    ad7791_sample_freq_avail[i][1] == val2)
+				break;
+		}
+
+		if (i == ARRAY_SIZE(ad7791_sample_freq_avail)) {
+			ret = -EINVAL;
+			break;
+		}
+
+		st->filter &= ~AD7791_FILTER_RATE_MASK;
+		st->filter |= i;
+		ad_sd_write_reg(&st->sd, AD7791_REG_FILTER,
+				sizeof(st->filter),
+				st->filter);
+		break;
+	default:
+		ret = -EINVAL;
+	}
 
-static IIO_DEV_ATTR_SAMP_FREQ(S_IWUSR | S_IRUGO,
-		ad7791_read_frequency,
-		ad7791_write_frequency);
+	iio_device_release_direct_mode(indio_dev);
+	return ret;
+}
 
 static IIO_CONST_ATTR_SAMP_FREQ_AVAIL("120 100 33.3 20 16.7 16.6 13.3 9.5");
 
 static struct attribute *ad7791_attributes[] = {
-	&iio_dev_attr_sampling_frequency.dev_attr.attr,
 	&iio_const_attr_sampling_frequency_available.dev_attr.attr,
 	NULL
 };
@@ -306,12 +311,14 @@ static const struct attribute_group ad7791_attribute_group = {
 
 static const struct iio_info ad7791_info = {
 	.read_raw = &ad7791_read_raw,
+	.write_raw = &ad7791_write_raw,
 	.attrs = &ad7791_attribute_group,
 	.validate_trigger = ad_sd_validate_trigger,
 };
 
 static const struct iio_info ad7791_no_filter_info = {
 	.read_raw = &ad7791_read_raw,
+	.write_raw = &ad7791_write_raw,
 	.validate_trigger = ad_sd_validate_trigger,
 };
 
diff --git a/drivers/iio/adc/ad7793.c b/drivers/iio/adc/ad7793.c
index 801afb61310b..d4bbe5b53318 100644
--- a/drivers/iio/adc/ad7793.c
+++ b/drivers/iio/adc/ad7793.c
@@ -348,55 +348,6 @@ static const u16 ad7793_sample_freq_avail[16] = {0, 470, 242, 123, 62, 50, 39,
 static const u16 ad7797_sample_freq_avail[16] = {0, 0, 0, 123, 62, 50, 0,
 					33, 0, 17, 16, 12, 10, 8, 6, 4};
 
-static ssize_t ad7793_read_frequency(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7793_state *st = iio_priv(indio_dev);
-
-	return sprintf(buf, "%d\n",
-	       st->chip_info->sample_freq_avail[AD7793_MODE_RATE(st->mode)]);
-}
-
-static ssize_t ad7793_write_frequency(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7793_state *st = iio_priv(indio_dev);
-	long lval;
-	int i, ret;
-
-	ret = kstrtol(buf, 10, &lval);
-	if (ret)
-		return ret;
-
-	if (lval == 0)
-		return -EINVAL;
-
-	for (i = 0; i < 16; i++)
-		if (lval == st->chip_info->sample_freq_avail[i])
-			break;
-	if (i == 16)
-		return -EINVAL;
-
-	ret = iio_device_claim_direct_mode(indio_dev);
-	if (ret)
-		return ret;
-	st->mode &= ~AD7793_MODE_RATE(-1);
-	st->mode |= AD7793_MODE_RATE(i);
-	ad_sd_write_reg(&st->sd, AD7793_REG_MODE, sizeof(st->mode), st->mode);
-	iio_device_release_direct_mode(indio_dev);
-
-	return len;
-}
-
-static IIO_DEV_ATTR_SAMP_FREQ(S_IWUSR | S_IRUGO,
-		ad7793_read_frequency,
-		ad7793_write_frequency);
-
 static IIO_CONST_ATTR_SAMP_FREQ_AVAIL(
 	"470 242 123 62 50 39 33 19 17 16 12 10 8 6 4");
 
@@ -424,7 +375,6 @@ static IIO_DEVICE_ATTR_NAMED(in_m_in_scale_available,
 		ad7793_show_scale_available, NULL, 0);
 
 static struct attribute *ad7793_attributes[] = {
-	&iio_dev_attr_sampling_frequency.dev_attr.attr,
 	&iio_const_attr_sampling_frequency_available.dev_attr.attr,
 	&iio_dev_attr_in_m_in_scale_available.dev_attr.attr,
 	NULL
@@ -435,7 +385,6 @@ static const struct attribute_group ad7793_attribute_group = {
 };
 
 static struct attribute *ad7797_attributes[] = {
-	&iio_dev_attr_sampling_frequency.dev_attr.attr,
 	&iio_const_attr_sampling_frequency_available_ad7797.dev_attr.attr,
 	NULL
 };
@@ -505,6 +454,10 @@ static int ad7793_read_raw(struct iio_dev *indio_dev,
 			*val -= offset;
 		}
 		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*val = st->chip_info
+			       ->sample_freq_avail[AD7793_MODE_RATE(st->mode)];
+		return IIO_VAL_INT;
 	}
 	return -EINVAL;
 }
@@ -542,6 +495,26 @@ static int ad7793_write_raw(struct iio_dev *indio_dev,
 				break;
 			}
 		break;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (!val) {
+			ret = -EINVAL;
+			break;
+		}
+
+		for (i = 0; i < 16; i++)
+			if (val == st->chip_info->sample_freq_avail[i])
+				break;
+
+		if (i == 16) {
+			ret = -EINVAL;
+			break;
+		}
+
+		st->mode &= ~AD7793_MODE_RATE(-1);
+		st->mode |= AD7793_MODE_RATE(i);
+		ad_sd_write_reg(&st->sd, AD7793_REG_MODE, sizeof(st->mode),
+				st->mode);
+		break;
 	default:
 		ret = -EINVAL;
 	}
diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c
index 4eff8351ce29..8729d6524b4d 100644
--- a/drivers/iio/adc/at91-sama5d2_adc.c
+++ b/drivers/iio/adc/at91-sama5d2_adc.c
@@ -333,6 +333,27 @@ static const struct iio_chan_spec at91_adc_channels[] = {
 				+ AT91_SAMA5D2_DIFF_CHAN_CNT + 1),
 };
 
+static int at91_adc_chan_xlate(struct iio_dev *indio_dev, int chan)
+{
+	int i;
+
+	for (i = 0; i < indio_dev->num_channels; i++) {
+		if (indio_dev->channels[i].scan_index == chan)
+			return i;
+	}
+	return -EINVAL;
+}
+
+static inline struct iio_chan_spec const *
+at91_adc_chan_get(struct iio_dev *indio_dev, int chan)
+{
+	int index = at91_adc_chan_xlate(indio_dev, chan);
+
+	if (index < 0)
+		return NULL;
+	return indio_dev->channels + index;
+}
+
 static int at91_adc_configure_trigger(struct iio_trigger *trig, bool state)
 {
 	struct iio_dev *indio = iio_trigger_get_drvdata(trig);
@@ -350,8 +371,10 @@ static int at91_adc_configure_trigger(struct iio_trigger *trig, bool state)
 	at91_adc_writel(st, AT91_SAMA5D2_TRGR, status);
 
 	for_each_set_bit(bit, indio->active_scan_mask, indio->num_channels) {
-		struct iio_chan_spec const *chan = indio->channels + bit;
+		struct iio_chan_spec const *chan = at91_adc_chan_get(indio, bit);
 
+		if (!chan)
+			continue;
 		if (state) {
 			at91_adc_writel(st, AT91_SAMA5D2_CHER,
 					BIT(chan->channel));
@@ -448,7 +471,11 @@ static int at91_adc_dma_start(struct iio_dev *indio_dev)
 
 	for_each_set_bit(bit, indio_dev->active_scan_mask,
 			 indio_dev->num_channels) {
-		struct iio_chan_spec const *chan = indio_dev->channels + bit;
+		struct iio_chan_spec const *chan =
+					 at91_adc_chan_get(indio_dev, bit);
+
+		if (!chan)
+			continue;
 
 		st->dma_st.rx_buf_sz += chan->scan_type.storagebits / 8;
 	}
@@ -526,8 +553,11 @@ static int at91_adc_buffer_predisable(struct iio_dev *indio_dev)
 	 */
 	for_each_set_bit(bit, indio_dev->active_scan_mask,
 			 indio_dev->num_channels) {
-		struct iio_chan_spec const *chan = indio_dev->channels + bit;
+		struct iio_chan_spec const *chan =
+					at91_adc_chan_get(indio_dev, bit);
 
+		if (!chan)
+			continue;
 		if (st->dma_st.dma_chan)
 			at91_adc_readl(st, chan->address);
 	}
@@ -587,8 +617,11 @@ static void at91_adc_trigger_handler_nodma(struct iio_dev *indio_dev,
 
 	for_each_set_bit(bit, indio_dev->active_scan_mask,
 			 indio_dev->num_channels) {
-		struct iio_chan_spec const *chan = indio_dev->channels + bit;
+		struct iio_chan_spec const *chan =
+					at91_adc_chan_get(indio_dev, bit);
 
+		if (!chan)
+			continue;
 		st->buffer[i] = at91_adc_readl(st, chan->address);
 		i++;
 	}
diff --git a/drivers/iio/adc/imx7d_adc.c b/drivers/iio/adc/imx7d_adc.c
index cfab31162845..ad6764fb2a23 100644
--- a/drivers/iio/adc/imx7d_adc.c
+++ b/drivers/iio/adc/imx7d_adc.c
@@ -604,5 +604,5 @@ static struct platform_driver imx7d_adc_driver = {
 module_platform_driver(imx7d_adc_driver);
 
 MODULE_AUTHOR("Haibo Chen <haibo.chen@freescale.com>");
-MODULE_DESCRIPTION("Freeacale IMX7D ADC driver");
+MODULE_DESCRIPTION("Freescale IMX7D ADC driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/meson_saradc.c b/drivers/iio/adc/meson_saradc.c
index ede955d9b2a4..2948909f3ee3 100644
--- a/drivers/iio/adc/meson_saradc.c
+++ b/drivers/iio/adc/meson_saradc.c
@@ -219,15 +219,19 @@ enum meson_sar_adc_chan7_mux_sel {
 	CHAN7_MUX_CH7_INPUT = 0x7,
 };
 
-struct meson_sar_adc_data {
+struct meson_sar_adc_param {
 	bool					has_bl30_integration;
 	unsigned long				clock_rate;
 	u32					bandgap_reg;
 	unsigned int				resolution;
-	const char				*name;
 	const struct regmap_config		*regmap_config;
 };
 
+struct meson_sar_adc_data {
+	const struct meson_sar_adc_param	*param;
+	const char				*name;
+};
+
 struct meson_sar_adc_priv {
 	struct regmap				*regmap;
 	struct regulator			*vref;
@@ -276,7 +280,7 @@ static int meson_sar_adc_calib_val(struct iio_dev *indio_dev, int val)
 	/* use val_calib = scale * val_raw + offset calibration function */
 	tmp = div_s64((s64)val * priv->calibscale, MILLION) + priv->calibbias;
 
-	return clamp(tmp, 0, (1 << priv->data->resolution) - 1);
+	return clamp(tmp, 0, (1 << priv->data->param->resolution) - 1);
 }
 
 static int meson_sar_adc_wait_busy_clear(struct iio_dev *indio_dev)
@@ -328,7 +332,7 @@ static int meson_sar_adc_read_raw_sample(struct iio_dev *indio_dev,
 	}
 
 	fifo_val = FIELD_GET(MESON_SAR_ADC_FIFO_RD_SAMPLE_VALUE_MASK, regval);
-	fifo_val &= GENMASK(priv->data->resolution - 1, 0);
+	fifo_val &= GENMASK(priv->data->param->resolution - 1, 0);
 	*val = meson_sar_adc_calib_val(indio_dev, fifo_val);
 
 	return 0;
@@ -447,7 +451,7 @@ static int meson_sar_adc_lock(struct iio_dev *indio_dev)
 
 	mutex_lock(&indio_dev->mlock);
 
-	if (priv->data->has_bl30_integration) {
+	if (priv->data->param->has_bl30_integration) {
 		/* prevent BL30 from using the SAR ADC while we are using it */
 		regmap_update_bits(priv->regmap, MESON_SAR_ADC_DELAY,
 				MESON_SAR_ADC_DELAY_KERNEL_BUSY,
@@ -475,7 +479,7 @@ static void meson_sar_adc_unlock(struct iio_dev *indio_dev)
 {
 	struct meson_sar_adc_priv *priv = iio_priv(indio_dev);
 
-	if (priv->data->has_bl30_integration)
+	if (priv->data->param->has_bl30_integration)
 		/* allow BL30 to use the SAR ADC again */
 		regmap_update_bits(priv->regmap, MESON_SAR_ADC_DELAY,
 				MESON_SAR_ADC_DELAY_KERNEL_BUSY, 0);
@@ -559,7 +563,7 @@ static int meson_sar_adc_iio_info_read_raw(struct iio_dev *indio_dev,
 		}
 
 		*val = ret / 1000;
-		*val2 = priv->data->resolution;
+		*val2 = priv->data->param->resolution;
 		return IIO_VAL_FRACTIONAL_LOG2;
 
 	case IIO_CHAN_INFO_CALIBBIAS:
@@ -632,7 +636,7 @@ static int meson_sar_adc_init(struct iio_dev *indio_dev)
 	 */
 	meson_sar_adc_set_chan7_mux(indio_dev, CHAN7_MUX_CH7_INPUT);
 
-	if (priv->data->has_bl30_integration) {
+	if (priv->data->param->has_bl30_integration) {
 		/*
 		 * leave sampling delay and the input clocks as configured by
 		 * BL30 to make sure BL30 gets the values it expects when
@@ -712,7 +716,7 @@ static int meson_sar_adc_init(struct iio_dev *indio_dev)
 		return ret;
 	}
 
-	ret = clk_set_rate(priv->adc_clk, priv->data->clock_rate);
+	ret = clk_set_rate(priv->adc_clk, priv->data->param->clock_rate);
 	if (ret) {
 		dev_err(indio_dev->dev.parent,
 			"failed to set adc clock rate\n");
@@ -725,14 +729,15 @@ static int meson_sar_adc_init(struct iio_dev *indio_dev)
 static void meson_sar_adc_set_bandgap(struct iio_dev *indio_dev, bool on_off)
 {
 	struct meson_sar_adc_priv *priv = iio_priv(indio_dev);
+	const struct meson_sar_adc_param *param = priv->data->param;
 	u32 enable_mask;
 
-	if (priv->data->bandgap_reg == MESON_SAR_ADC_REG11)
+	if (param->bandgap_reg == MESON_SAR_ADC_REG11)
 		enable_mask = MESON_SAR_ADC_REG11_BANDGAP_EN;
 	else
 		enable_mask = MESON_SAR_ADC_DELTA_10_TS_VBG_EN;
 
-	regmap_update_bits(priv->regmap, priv->data->bandgap_reg, enable_mask,
+	regmap_update_bits(priv->regmap, param->bandgap_reg, enable_mask,
 			   on_off ? enable_mask : 0);
 }
 
@@ -844,8 +849,8 @@ static int meson_sar_adc_calib(struct iio_dev *indio_dev)
 	int ret, nominal0, nominal1, value0, value1;
 
 	/* use points 25% and 75% for calibration */
-	nominal0 = (1 << priv->data->resolution) / 4;
-	nominal1 = (1 << priv->data->resolution) * 3 / 4;
+	nominal0 = (1 << priv->data->param->resolution) / 4;
+	nominal1 = (1 << priv->data->param->resolution) * 3 / 4;
 
 	meson_sar_adc_set_chan7_mux(indio_dev, CHAN7_MUX_VDD_DIV4);
 	usleep_range(10, 20);
@@ -883,51 +888,60 @@ static const struct iio_info meson_sar_adc_iio_info = {
 	.read_raw = meson_sar_adc_iio_info_read_raw,
 };
 
-static const struct meson_sar_adc_data meson_sar_adc_meson8_data = {
-	.has_bl30_integration = false,
-	.clock_rate = 1150000,
-	.bandgap_reg = MESON_SAR_ADC_DELTA_10,
-	.regmap_config = &meson_sar_adc_regmap_config_meson8,
-	.resolution = 10,
-	.name = "meson-meson8-saradc",
-};
-
-static const struct meson_sar_adc_data meson_sar_adc_meson8b_data = {
+static const struct meson_sar_adc_param meson_sar_adc_meson8_param = {
 	.has_bl30_integration = false,
 	.clock_rate = 1150000,
 	.bandgap_reg = MESON_SAR_ADC_DELTA_10,
 	.regmap_config = &meson_sar_adc_regmap_config_meson8,
 	.resolution = 10,
-	.name = "meson-meson8b-saradc",
 };
 
-static const struct meson_sar_adc_data meson_sar_adc_gxbb_data = {
+static const struct meson_sar_adc_param meson_sar_adc_gxbb_param = {
 	.has_bl30_integration = true,
 	.clock_rate = 1200000,
 	.bandgap_reg = MESON_SAR_ADC_REG11,
 	.regmap_config = &meson_sar_adc_regmap_config_gxbb,
 	.resolution = 10,
-	.name = "meson-gxbb-saradc",
 };
 
-static const struct meson_sar_adc_data meson_sar_adc_gxl_data = {
+static const struct meson_sar_adc_param meson_sar_adc_gxl_param = {
 	.has_bl30_integration = true,
 	.clock_rate = 1200000,
 	.bandgap_reg = MESON_SAR_ADC_REG11,
 	.regmap_config = &meson_sar_adc_regmap_config_gxbb,
 	.resolution = 12,
+};
+
+static const struct meson_sar_adc_data meson_sar_adc_meson8_data = {
+	.param = &meson_sar_adc_meson8_param,
+	.name = "meson-meson8-saradc",
+};
+
+static const struct meson_sar_adc_data meson_sar_adc_meson8b_data = {
+	.param = &meson_sar_adc_meson8_param,
+	.name = "meson-meson8b-saradc",
+};
+
+static const struct meson_sar_adc_data meson_sar_adc_gxbb_data = {
+	.param = &meson_sar_adc_gxbb_param,
+	.name = "meson-gxbb-saradc",
+};
+
+static const struct meson_sar_adc_data meson_sar_adc_gxl_data = {
+	.param = &meson_sar_adc_gxl_param,
 	.name = "meson-gxl-saradc",
 };
 
 static const struct meson_sar_adc_data meson_sar_adc_gxm_data = {
-	.has_bl30_integration = true,
-	.clock_rate = 1200000,
-	.bandgap_reg = MESON_SAR_ADC_REG11,
-	.regmap_config = &meson_sar_adc_regmap_config_gxbb,
-	.resolution = 12,
+	.param = &meson_sar_adc_gxl_param,
 	.name = "meson-gxm-saradc",
 };
 
+static const struct meson_sar_adc_data meson_sar_adc_axg_data = {
+	.param = &meson_sar_adc_gxl_param,
+	.name = "meson-axg-saradc",
+};
+
 static const struct of_device_id meson_sar_adc_of_match[] = {
 	{
 		.compatible = "amlogic,meson8-saradc",
@@ -946,6 +960,9 @@ static const struct of_device_id meson_sar_adc_of_match[] = {
 	}, {
 		.compatible = "amlogic,meson-gxm-saradc",
 		.data = &meson_sar_adc_gxm_data,
+	}, {
+		.compatible = "amlogic,meson-axg-saradc",
+		.data = &meson_sar_adc_axg_data,
 	},
 	{},
 };
@@ -1001,7 +1018,7 @@ static int meson_sar_adc_probe(struct platform_device *pdev)
 		return ret;
 
 	priv->regmap = devm_regmap_init_mmio(&pdev->dev, base,
-					     priv->data->regmap_config);
+					     priv->data->param->regmap_config);
 	if (IS_ERR(priv->regmap))
 		return PTR_ERR(priv->regmap);
 
diff --git a/drivers/iio/adc/stm32-adc-core.c b/drivers/iio/adc/stm32-adc-core.c
index 40be7d9fadbf..ca432e7b6ff1 100644
--- a/drivers/iio/adc/stm32-adc-core.c
+++ b/drivers/iio/adc/stm32-adc-core.c
@@ -34,9 +34,6 @@
 #define STM32F4_ADC_ADCPRE_SHIFT	16
 #define STM32F4_ADC_ADCPRE_MASK		GENMASK(17, 16)
 
-/* STM32 F4 maximum analog clock rate (from datasheet) */
-#define STM32F4_ADC_MAX_CLK_RATE	36000000
-
 /* STM32H7 - common registers for all ADC instances */
 #define STM32H7_ADC_CSR			(STM32_ADCX_COMN_OFFSET + 0x00)
 #define STM32H7_ADC_CCR			(STM32_ADCX_COMN_OFFSET + 0x08)
@@ -51,9 +48,6 @@
 #define STM32H7_CKMODE_SHIFT		16
 #define STM32H7_CKMODE_MASK		GENMASK(17, 16)
 
-/* STM32 H7 maximum analog clock rate (from datasheet) */
-#define STM32H7_ADC_MAX_CLK_RATE	36000000
-
 /**
  * stm32_adc_common_regs - stm32 common registers, compatible dependent data
  * @csr:	common status register offset
@@ -74,15 +68,17 @@ struct stm32_adc_priv;
  * stm32_adc_priv_cfg - stm32 core compatible configuration data
  * @regs:	common registers for all instances
  * @clk_sel:	clock selection routine
+ * @max_clk_rate_hz: maximum analog clock rate (Hz, from datasheet)
  */
 struct stm32_adc_priv_cfg {
 	const struct stm32_adc_common_regs *regs;
 	int (*clk_sel)(struct platform_device *, struct stm32_adc_priv *);
+	u32 max_clk_rate_hz;
 };
 
 /**
  * struct stm32_adc_priv - stm32 ADC core private data
- * @irq:		irq for ADC block
+ * @irq:		irq(s) for ADC block
  * @domain:		irq domain reference
  * @aclk:		clock reference for the analog circuitry
  * @bclk:		bus clock common for all ADCs, depends on part used
@@ -91,7 +87,7 @@ struct stm32_adc_priv_cfg {
  * @common:		common data for all ADC instances
  */
 struct stm32_adc_priv {
-	int				irq;
+	int				irq[STM32_ADC_MAX_ADCS];
 	struct irq_domain		*domain;
 	struct clk			*aclk;
 	struct clk			*bclk;
@@ -133,7 +129,7 @@ static int stm32f4_adc_clk_sel(struct platform_device *pdev,
 	}
 
 	for (i = 0; i < ARRAY_SIZE(stm32f4_pclk_div); i++) {
-		if ((rate / stm32f4_pclk_div[i]) <= STM32F4_ADC_MAX_CLK_RATE)
+		if ((rate / stm32f4_pclk_div[i]) <= priv->cfg->max_clk_rate_hz)
 			break;
 	}
 	if (i >= ARRAY_SIZE(stm32f4_pclk_div)) {
@@ -222,7 +218,7 @@ static int stm32h7_adc_clk_sel(struct platform_device *pdev,
 			if (ckmode)
 				continue;
 
-			if ((rate / div) <= STM32H7_ADC_MAX_CLK_RATE)
+			if ((rate / div) <= priv->cfg->max_clk_rate_hz)
 				goto out;
 		}
 	}
@@ -242,7 +238,7 @@ static int stm32h7_adc_clk_sel(struct platform_device *pdev,
 		if (!ckmode)
 			continue;
 
-		if ((rate / div) <= STM32H7_ADC_MAX_CLK_RATE)
+		if ((rate / div) <= priv->cfg->max_clk_rate_hz)
 			goto out;
 	}
 
@@ -328,11 +324,24 @@ static int stm32_adc_irq_probe(struct platform_device *pdev,
 			       struct stm32_adc_priv *priv)
 {
 	struct device_node *np = pdev->dev.of_node;
+	unsigned int i;
+
+	for (i = 0; i < STM32_ADC_MAX_ADCS; i++) {
+		priv->irq[i] = platform_get_irq(pdev, i);
+		if (priv->irq[i] < 0) {
+			/*
+			 * At least one interrupt must be provided, make others
+			 * optional:
+			 * - stm32f4/h7 shares a common interrupt.
+			 * - stm32mp1, has one line per ADC (either for ADC1,
+			 *   ADC2 or both).
+			 */
+			if (i && priv->irq[i] == -ENXIO)
+				continue;
+			dev_err(&pdev->dev, "failed to get irq\n");
 
-	priv->irq = platform_get_irq(pdev, 0);
-	if (priv->irq < 0) {
-		dev_err(&pdev->dev, "failed to get irq\n");
-		return priv->irq;
+			return priv->irq[i];
+		}
 	}
 
 	priv->domain = irq_domain_add_simple(np, STM32_ADC_MAX_ADCS, 0,
@@ -343,8 +352,12 @@ static int stm32_adc_irq_probe(struct platform_device *pdev,
 		return -ENOMEM;
 	}
 
-	irq_set_chained_handler(priv->irq, stm32_adc_irq_handler);
-	irq_set_handler_data(priv->irq, priv);
+	for (i = 0; i < STM32_ADC_MAX_ADCS; i++) {
+		if (priv->irq[i] < 0)
+			continue;
+		irq_set_chained_handler(priv->irq[i], stm32_adc_irq_handler);
+		irq_set_handler_data(priv->irq[i], priv);
+	}
 
 	return 0;
 }
@@ -353,11 +366,17 @@ static void stm32_adc_irq_remove(struct platform_device *pdev,
 				 struct stm32_adc_priv *priv)
 {
 	int hwirq;
+	unsigned int i;
 
 	for (hwirq = 0; hwirq < STM32_ADC_MAX_ADCS; hwirq++)
 		irq_dispose_mapping(irq_find_mapping(priv->domain, hwirq));
 	irq_domain_remove(priv->domain);
-	irq_set_chained_handler(priv->irq, NULL);
+
+	for (i = 0; i < STM32_ADC_MAX_ADCS; i++) {
+		if (priv->irq[i] < 0)
+			continue;
+		irq_set_chained_handler(priv->irq[i], NULL);
+	}
 }
 
 static int stm32_adc_probe(struct platform_device *pdev)
@@ -497,11 +516,19 @@ static int stm32_adc_remove(struct platform_device *pdev)
 static const struct stm32_adc_priv_cfg stm32f4_adc_priv_cfg = {
 	.regs = &stm32f4_adc_common_regs,
 	.clk_sel = stm32f4_adc_clk_sel,
+	.max_clk_rate_hz = 36000000,
 };
 
 static const struct stm32_adc_priv_cfg stm32h7_adc_priv_cfg = {
 	.regs = &stm32h7_adc_common_regs,
 	.clk_sel = stm32h7_adc_clk_sel,
+	.max_clk_rate_hz = 36000000,
+};
+
+static const struct stm32_adc_priv_cfg stm32mp1_adc_priv_cfg = {
+	.regs = &stm32h7_adc_common_regs,
+	.clk_sel = stm32h7_adc_clk_sel,
+	.max_clk_rate_hz = 40000000,
 };
 
 static const struct of_device_id stm32_adc_of_match[] = {
@@ -511,6 +538,9 @@ static const struct of_device_id stm32_adc_of_match[] = {
 	}, {
 		.compatible = "st,stm32h7-adc-core",
 		.data = (void *)&stm32h7_adc_priv_cfg
+	}, {
+		.compatible = "st,stm32mp1-adc-core",
+		.data = (void *)&stm32mp1_adc_priv_cfg
 	}, {
 	},
 };
diff --git a/drivers/iio/adc/stm32-adc.c b/drivers/iio/adc/stm32-adc.c
index 9a2583caedaa..378411853d75 100644
--- a/drivers/iio/adc/stm32-adc.c
+++ b/drivers/iio/adc/stm32-adc.c
@@ -84,6 +84,7 @@
 #define STM32H7_ADC_CALFACT2		0xC8
 
 /* STM32H7_ADC_ISR - bit fields */
+#define STM32MP1_VREGREADY		BIT(12)
 #define STM32H7_EOC			BIT(2)
 #define STM32H7_ADRDY			BIT(0)
 
@@ -249,6 +250,7 @@ struct stm32_adc;
  * @adc_info:		per instance input channels definitions
  * @trigs:		external trigger sources
  * @clk_required:	clock is required
+ * @has_vregready:	vregready status flag presence
  * @selfcalib:		optional routine for self-calibration
  * @prepare:		optional prepare routine (power-up, enable)
  * @start_conv:		routine to start conversions
@@ -261,6 +263,7 @@ struct stm32_adc_cfg {
 	const struct stm32_adc_info	*adc_info;
 	struct stm32_adc_trig_info	*trigs;
 	bool clk_required;
+	bool has_vregready;
 	int (*selfcalib)(struct stm32_adc *);
 	int (*prepare)(struct stm32_adc *);
 	void (*start_conv)(struct stm32_adc *, bool dma);
@@ -695,8 +698,12 @@ static void stm32h7_adc_stop_conv(struct stm32_adc *adc)
 	stm32_adc_clr_bits(adc, STM32H7_ADC_CFGR, STM32H7_DMNGT_MASK);
 }
 
-static void stm32h7_adc_exit_pwr_down(struct stm32_adc *adc)
+static int stm32h7_adc_exit_pwr_down(struct stm32_adc *adc)
 {
+	struct iio_dev *indio_dev = iio_priv_to_dev(adc);
+	int ret;
+	u32 val;
+
 	/* Exit deep power down, then enable ADC voltage regulator */
 	stm32_adc_clr_bits(adc, STM32H7_ADC_CR, STM32H7_DEEPPWD);
 	stm32_adc_set_bits(adc, STM32H7_ADC_CR, STM32H7_ADVREGEN);
@@ -705,7 +712,20 @@ static void stm32h7_adc_exit_pwr_down(struct stm32_adc *adc)
 		stm32_adc_set_bits(adc, STM32H7_ADC_CR, STM32H7_BOOST);
 
 	/* Wait for startup time */
-	usleep_range(10, 20);
+	if (!adc->cfg->has_vregready) {
+		usleep_range(10, 20);
+		return 0;
+	}
+
+	ret = stm32_adc_readl_poll_timeout(STM32H7_ADC_ISR, val,
+					   val & STM32MP1_VREGREADY, 100,
+					   STM32_ADC_TIMEOUT_US);
+	if (ret) {
+		stm32_adc_set_bits(adc, STM32H7_ADC_CR, STM32H7_DEEPPWD);
+		dev_err(&indio_dev->dev, "Failed to exit power down\n");
+	}
+
+	return ret;
 }
 
 static void stm32h7_adc_enter_pwr_down(struct stm32_adc *adc)
@@ -888,7 +908,9 @@ static int stm32h7_adc_selfcalib(struct stm32_adc *adc)
 	int ret;
 	u32 val;
 
-	stm32h7_adc_exit_pwr_down(adc);
+	ret = stm32h7_adc_exit_pwr_down(adc);
+	if (ret)
+		return ret;
 
 	/*
 	 * Select calibration mode:
@@ -952,7 +974,10 @@ static int stm32h7_adc_prepare(struct stm32_adc *adc)
 {
 	int ret;
 
-	stm32h7_adc_exit_pwr_down(adc);
+	ret = stm32h7_adc_exit_pwr_down(adc);
+	if (ret)
+		return ret;
+
 	stm32_adc_writel(adc, STM32H7_ADC_DIFSEL, adc->difsel);
 
 	ret = stm32h7_adc_enable(adc);
@@ -1944,9 +1969,23 @@ static const struct stm32_adc_cfg stm32h7_adc_cfg = {
 	.smp_cycles = stm32h7_adc_smp_cycles,
 };
 
+static const struct stm32_adc_cfg stm32mp1_adc_cfg = {
+	.regs = &stm32h7_adc_regspec,
+	.adc_info = &stm32h7_adc_info,
+	.trigs = stm32h7_adc_trigs,
+	.has_vregready = true,
+	.selfcalib = stm32h7_adc_selfcalib,
+	.start_conv = stm32h7_adc_start_conv,
+	.stop_conv = stm32h7_adc_stop_conv,
+	.prepare = stm32h7_adc_prepare,
+	.unprepare = stm32h7_adc_unprepare,
+	.smp_cycles = stm32h7_adc_smp_cycles,
+};
+
 static const struct of_device_id stm32_adc_of_match[] = {
 	{ .compatible = "st,stm32f4-adc", .data = (void *)&stm32f4_adc_cfg },
 	{ .compatible = "st,stm32h7-adc", .data = (void *)&stm32h7_adc_cfg },
+	{ .compatible = "st,stm32mp1-adc", .data = (void *)&stm32mp1_adc_cfg },
 	{},
 };
 MODULE_DEVICE_TABLE(of, stm32_adc_of_match);
diff --git a/drivers/iio/adc/stm32-dfsdm-adc.c b/drivers/iio/adc/stm32-dfsdm-adc.c
index 01422d11753c..fcd4a1c00ca0 100644
--- a/drivers/iio/adc/stm32-dfsdm-adc.c
+++ b/drivers/iio/adc/stm32-dfsdm-adc.c
@@ -8,11 +8,11 @@
 
 #include <linux/dmaengine.h>
 #include <linux/dma-mapping.h>
-#include <linux/interrupt.h>
+#include <linux/iio/adc/stm32-dfsdm-adc.h>
 #include <linux/iio/buffer.h>
 #include <linux/iio/hw-consumer.h>
-#include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
@@ -144,6 +144,7 @@ static int stm32_dfsdm_set_osrs(struct stm32_dfsdm_filter *fl,
 	 * Leave as soon as if exact resolution if reached.
 	 * Otherwise the higher resolution below 32 bits is kept.
 	 */
+	fl->res = 0;
 	for (fosr = 1; fosr <= DFSDM_MAX_FL_OVERSAMPLING; fosr++) {
 		for (iosr = 1; iosr <= DFSDM_MAX_INT_OVERSAMPLING; iosr++) {
 			if (fast)
@@ -193,7 +194,7 @@ static int stm32_dfsdm_set_osrs(struct stm32_dfsdm_filter *fl,
 		}
 	}
 
-	if (!fl->fosr)
+	if (!fl->res)
 		return -EINVAL;
 
 	return 0;
@@ -253,7 +254,8 @@ static int stm32_dfsdm_start_filter(struct stm32_dfsdm *dfsdm,
 				  DFSDM_CR1_RSWSTART(1));
 }
 
-static void stm32_dfsdm_stop_filter(struct stm32_dfsdm *dfsdm, unsigned int fl_id)
+static void stm32_dfsdm_stop_filter(struct stm32_dfsdm *dfsdm,
+				    unsigned int fl_id)
 {
 	/* Disable conversion */
 	regmap_update_bits(dfsdm->regmap, DFSDM_CR1(fl_id),
@@ -337,7 +339,7 @@ static int stm32_dfsdm_channel_parse_of(struct stm32_dfsdm *dfsdm,
 					    "st,adc-channel-types", chan_idx,
 					    &of_str);
 	if (!ret) {
-		val  = stm32_dfsdm_str2val(of_str, stm32_dfsdm_chan_type);
+		val = stm32_dfsdm_str2val(of_str, stm32_dfsdm_chan_type);
 		if (val < 0)
 			return val;
 	} else {
@@ -349,7 +351,7 @@ static int stm32_dfsdm_channel_parse_of(struct stm32_dfsdm *dfsdm,
 					    "st,adc-channel-clk-src", chan_idx,
 					    &of_str);
 	if (!ret) {
-		val  = stm32_dfsdm_str2val(of_str, stm32_dfsdm_chan_src);
+		val = stm32_dfsdm_str2val(of_str, stm32_dfsdm_chan_src);
 		if (val < 0)
 			return val;
 	} else {
@@ -770,7 +772,7 @@ static int stm32_dfsdm_write_raw(struct iio_dev *indio_dev,
 	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
 	struct stm32_dfsdm_filter *fl = &adc->dfsdm->fl_list[adc->fl_id];
 	struct stm32_dfsdm_channel *ch = &adc->dfsdm->ch_list[chan->channel];
-	unsigned int spi_freq = adc->spi_freq;
+	unsigned int spi_freq;
 	int ret = -EINVAL;
 
 	switch (mask) {
@@ -784,8 +786,18 @@ static int stm32_dfsdm_write_raw(struct iio_dev *indio_dev,
 	case IIO_CHAN_INFO_SAMP_FREQ:
 		if (!val)
 			return -EINVAL;
-		if (ch->src != DFSDM_CHANNEL_SPI_CLOCK_EXTERNAL)
+
+		switch (ch->src) {
+		case DFSDM_CHANNEL_SPI_CLOCK_INTERNAL:
 			spi_freq = adc->dfsdm->spi_master_freq;
+			break;
+		case DFSDM_CHANNEL_SPI_CLOCK_INTERNAL_DIV2_FALLING:
+		case DFSDM_CHANNEL_SPI_CLOCK_INTERNAL_DIV2_RISING:
+			spi_freq = adc->dfsdm->spi_master_freq / 2;
+			break;
+		default:
+			spi_freq = adc->spi_freq;
+		}
 
 		if (spi_freq % val)
 			dev_warn(&indio_dev->dev,
@@ -1093,7 +1105,6 @@ static int stm32_dfsdm_adc_probe(struct platform_device *pdev)
 	char *name;
 	int ret, irq, val;
 
-
 	dev_data = of_device_get_match_data(dev);
 	iio = devm_iio_device_alloc(dev, sizeof(*adc));
 	if (!iio) {
@@ -1111,8 +1122,8 @@ static int stm32_dfsdm_adc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, adc);
 
 	ret = of_property_read_u32(dev->of_node, "reg", &adc->fl_id);
-	if (ret != 0) {
-		dev_err(dev, "Missing reg property\n");
+	if (ret != 0 || adc->fl_id >= adc->dfsdm->num_fls) {
+		dev_err(dev, "Missing or bad reg property\n");
 		return -EINVAL;
 	}
 
@@ -1161,7 +1172,6 @@ static int stm32_dfsdm_adc_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto err_cleanup;
 
-	dev_err(dev, "of_platform_populate\n");
 	if (dev_data->type == DFSDM_AUDIO) {
 		ret = of_platform_populate(np, NULL, NULL, dev);
 		if (ret < 0) {
diff --git a/drivers/iio/adc/stm32-dfsdm-core.c b/drivers/iio/adc/stm32-dfsdm-core.c
index e50efdcc41ff..bf089f5d6225 100644
--- a/drivers/iio/adc/stm32-dfsdm-core.c
+++ b/drivers/iio/adc/stm32-dfsdm-core.c
@@ -25,6 +25,8 @@ struct stm32_dfsdm_dev_data {
 
 #define STM32H7_DFSDM_NUM_FILTERS	4
 #define STM32H7_DFSDM_NUM_CHANNELS	8
+#define STM32MP1_DFSDM_NUM_FILTERS	6
+#define STM32MP1_DFSDM_NUM_CHANNELS	8
 
 static bool stm32_dfsdm_volatile_reg(struct device *dev, unsigned int reg)
 {
@@ -61,6 +63,21 @@ static const struct stm32_dfsdm_dev_data stm32h7_dfsdm_data = {
 	.regmap_cfg = &stm32h7_dfsdm_regmap_cfg,
 };
 
+static const struct regmap_config stm32mp1_dfsdm_regmap_cfg = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = sizeof(u32),
+	.max_register = 0x7fc,
+	.volatile_reg = stm32_dfsdm_volatile_reg,
+	.fast_io = true,
+};
+
+static const struct stm32_dfsdm_dev_data stm32mp1_dfsdm_data = {
+	.num_filters = STM32MP1_DFSDM_NUM_FILTERS,
+	.num_channels = STM32MP1_DFSDM_NUM_CHANNELS,
+	.regmap_cfg = &stm32mp1_dfsdm_regmap_cfg,
+};
+
 struct dfsdm_priv {
 	struct platform_device *pdev; /* platform device */
 
@@ -227,6 +244,11 @@ static int stm32_dfsdm_parse_of(struct platform_device *pdev,
 	}
 
 	priv->spi_clk_out_div = div_u64_rem(clk_freq, spi_freq, &rem) - 1;
+	if (!priv->spi_clk_out_div) {
+		/* spi_clk_out_div == 0 means ckout is OFF */
+		dev_err(&pdev->dev, "spi-max-frequency not achievable\n");
+		return -EINVAL;
+	}
 	priv->dfsdm.spi_master_freq = spi_freq;
 
 	if (rem) {
@@ -243,6 +265,10 @@ static const struct of_device_id stm32_dfsdm_of_match[] = {
 		.compatible = "st,stm32h7-dfsdm",
 		.data = &stm32h7_dfsdm_data,
 	},
+	{
+		.compatible = "st,stm32mp1-dfsdm",
+		.data = &stm32mp1_dfsdm_data,
+	},
 	{}
 };
 MODULE_DEVICE_TABLE(of, stm32_dfsdm_of_match);
diff --git a/drivers/iio/adc/stx104.c b/drivers/iio/adc/stx104.c
index 17b021f33180..0662ca199eb0 100644
--- a/drivers/iio/adc/stx104.c
+++ b/drivers/iio/adc/stx104.c
@@ -233,6 +233,16 @@ static int stx104_gpio_get(struct gpio_chip *chip, unsigned int offset)
 	return !!(inb(stx104gpio->base) & BIT(offset));
 }
 
+static int stx104_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask,
+	unsigned long *bits)
+{
+	struct stx104_gpio *const stx104gpio = gpiochip_get_data(chip);
+
+	*bits = inb(stx104gpio->base);
+
+	return 0;
+}
+
 static void stx104_gpio_set(struct gpio_chip *chip, unsigned int offset,
 	int value)
 {
@@ -342,6 +352,7 @@ static int stx104_probe(struct device *dev, unsigned int id)
 	stx104gpio->chip.direction_input = stx104_gpio_direction_input;
 	stx104gpio->chip.direction_output = stx104_gpio_direction_output;
 	stx104gpio->chip.get = stx104_gpio_get;
+	stx104gpio->chip.get_multiple = stx104_gpio_get_multiple;
 	stx104gpio->chip.set = stx104_gpio_set;
 	stx104gpio->chip.set_multiple = stx104_gpio_set_multiple;
 	stx104gpio->base = base[id] + 3;
diff --git a/drivers/iio/adc/ti-ads8688.c b/drivers/iio/adc/ti-ads8688.c
index 079f133144b0..184d686ebd99 100644
--- a/drivers/iio/adc/ti-ads8688.c
+++ b/drivers/iio/adc/ti-ads8688.c
@@ -17,6 +17,9 @@
 #include <linux/of.h>
 
 #include <linux/iio/iio.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
 #include <linux/iio/sysfs.h>
 
 #define ADS8688_CMD_REG(x)		(x << 8)
@@ -155,6 +158,13 @@ static const struct attribute_group ads8688_attribute_group = {
 	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW)		\
 			      | BIT(IIO_CHAN_INFO_SCALE)	\
 			      | BIT(IIO_CHAN_INFO_OFFSET),	\
+	.scan_index = index,					\
+	.scan_type = {						\
+		.sign = 'u',					\
+		.realbits = 16,					\
+		.storagebits = 16,				\
+		.endianness = IIO_BE,				\
+	},							\
 }
 
 static const struct iio_chan_spec ads8684_channels[] = {
@@ -371,6 +381,28 @@ static const struct iio_info ads8688_info = {
 	.attrs = &ads8688_attribute_group,
 };
 
+static irqreturn_t ads8688_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	u16 buffer[8];
+	int i, j = 0;
+
+	for (i = 0; i < indio_dev->masklength; i++) {
+		if (!test_bit(i, indio_dev->active_scan_mask))
+			continue;
+		buffer[j] = ads8688_read(indio_dev, i);
+		j++;
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev, buffer,
+			pf->timestamp);
+
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
 static const struct ads8688_chip_info ads8688_chip_info_tbl[] = {
 	[ID_ADS8684] = {
 		.channels = ads8684_channels,
@@ -402,7 +434,7 @@ static int ads8688_probe(struct spi_device *spi)
 
 		ret = regulator_get_voltage(st->reg);
 		if (ret < 0)
-			goto error_out;
+			goto err_regulator_disable;
 
 		st->vref_mv = ret / 1000;
 	} else {
@@ -430,13 +462,22 @@ static int ads8688_probe(struct spi_device *spi)
 
 	mutex_init(&st->lock);
 
+	ret = iio_triggered_buffer_setup(indio_dev, NULL, ads8688_trigger_handler, NULL);
+	if (ret < 0) {
+		dev_err(&spi->dev, "iio triggered buffer setup failed\n");
+		goto err_regulator_disable;
+	}
+
 	ret = iio_device_register(indio_dev);
 	if (ret)
-		goto error_out;
+		goto err_buffer_cleanup;
 
 	return 0;
 
-error_out:
+err_buffer_cleanup:
+	iio_triggered_buffer_cleanup(indio_dev);
+
+err_regulator_disable:
 	if (!IS_ERR(st->reg))
 		regulator_disable(st->reg);
 
@@ -449,6 +490,7 @@ static int ads8688_remove(struct spi_device *spi)
 	struct ads8688_state *st = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
 
 	if (!IS_ERR(st->reg))
 		regulator_disable(st->reg);
diff --git a/drivers/iio/afe/Kconfig b/drivers/iio/afe/Kconfig
new file mode 100644
index 000000000000..c91eef04825a
--- /dev/null
+++ b/drivers/iio/afe/Kconfig
@@ -0,0 +1,19 @@
+#
+# Analog Front End drivers
+#
+# When adding new entries keep the list in alphabetical order
+
+menu "Analog Front Ends"
+
+config IIO_RESCALE
+	tristate "IIO rescale"
+	depends on OF || COMPILE_TEST
+	help
+	  Say yes here to build support for the IIO rescaling
+	  that handles voltage dividers, current sense shunts and
+	  current sense amplifiers.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called iio-rescale.
+
+endmenu
diff --git a/drivers/iio/afe/Makefile b/drivers/iio/afe/Makefile
new file mode 100644
index 000000000000..5fabb7bcac47
--- /dev/null
+++ b/drivers/iio/afe/Makefile
@@ -0,0 +1,6 @@
+#
+# Makefile for industrial I/O Analog Front Ends (AFE)
+#
+
+# When adding new entries keep the list in alphabetical order
+obj-$(CONFIG_IIO_RESCALE) += iio-rescale.o
diff --git a/drivers/iio/afe/iio-rescale.c b/drivers/iio/afe/iio-rescale.c
new file mode 100644
index 000000000000..e9ceee66d1e7
--- /dev/null
+++ b/drivers/iio/afe/iio-rescale.c
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * IIO rescale driver
+ *
+ * Copyright (C) 2018 Axentia Technologies AB
+ *
+ * Author: Peter Rosin <peda@axentia.se>
+ */
+
+#include <linux/err.h>
+#include <linux/gcd.h>
+#include <linux/iio/consumer.h>
+#include <linux/iio/iio.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+
+struct rescale;
+
+struct rescale_cfg {
+	enum iio_chan_type type;
+	int (*props)(struct device *dev, struct rescale *rescale);
+};
+
+struct rescale {
+	const struct rescale_cfg *cfg;
+	struct iio_channel *source;
+	struct iio_chan_spec chan;
+	struct iio_chan_spec_ext_info *ext_info;
+	s32 numerator;
+	s32 denominator;
+};
+
+static int rescale_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
+{
+	struct rescale *rescale = iio_priv(indio_dev);
+	unsigned long long tmp;
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		return iio_read_channel_raw(rescale->source, val);
+
+	case IIO_CHAN_INFO_SCALE:
+		ret = iio_read_channel_scale(rescale->source, val, val2);
+		switch (ret) {
+		case IIO_VAL_FRACTIONAL:
+			*val *= rescale->numerator;
+			*val2 *= rescale->denominator;
+			return ret;
+		case IIO_VAL_INT:
+			*val *= rescale->numerator;
+			if (rescale->denominator == 1)
+				return ret;
+			*val2 = rescale->denominator;
+			return IIO_VAL_FRACTIONAL;
+		case IIO_VAL_FRACTIONAL_LOG2:
+			tmp = *val * 1000000000LL;
+			do_div(tmp, rescale->denominator);
+			tmp *= rescale->numerator;
+			do_div(tmp, 1000000000LL);
+			*val = tmp;
+			return ret;
+		default:
+			return -EOPNOTSUPP;
+		}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int rescale_read_avail(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      const int **vals, int *type, int *length,
+			      long mask)
+{
+	struct rescale *rescale = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		*type = IIO_VAL_INT;
+		return iio_read_avail_channel_raw(rescale->source,
+						  vals, length);
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info rescale_info = {
+	.read_raw = rescale_read_raw,
+	.read_avail = rescale_read_avail,
+};
+
+static ssize_t rescale_read_ext_info(struct iio_dev *indio_dev,
+				     uintptr_t private,
+				     struct iio_chan_spec const *chan,
+				     char *buf)
+{
+	struct rescale *rescale = iio_priv(indio_dev);
+
+	return iio_read_channel_ext_info(rescale->source,
+					 rescale->ext_info[private].name,
+					 buf);
+}
+
+static ssize_t rescale_write_ext_info(struct iio_dev *indio_dev,
+				      uintptr_t private,
+				      struct iio_chan_spec const *chan,
+				      const char *buf, size_t len)
+{
+	struct rescale *rescale = iio_priv(indio_dev);
+
+	return iio_write_channel_ext_info(rescale->source,
+					  rescale->ext_info[private].name,
+					  buf, len);
+}
+
+static int rescale_configure_channel(struct device *dev,
+				     struct rescale *rescale)
+{
+	struct iio_chan_spec *chan = &rescale->chan;
+	struct iio_chan_spec const *schan = rescale->source->channel;
+
+	chan->indexed = 1;
+	chan->output = schan->output;
+	chan->ext_info = rescale->ext_info;
+	chan->type = rescale->cfg->type;
+
+	if (!iio_channel_has_info(schan, IIO_CHAN_INFO_RAW) ||
+	    !iio_channel_has_info(schan, IIO_CHAN_INFO_SCALE)) {
+		dev_err(dev, "source channel does not support raw/scale\n");
+		return -EINVAL;
+	}
+
+	chan->info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+		BIT(IIO_CHAN_INFO_SCALE);
+
+	if (iio_channel_has_available(schan, IIO_CHAN_INFO_RAW))
+		chan->info_mask_separate_available |= BIT(IIO_CHAN_INFO_RAW);
+
+	return 0;
+}
+
+static int rescale_current_sense_amplifier_props(struct device *dev,
+						 struct rescale *rescale)
+{
+	u32 sense;
+	u32 gain_mult = 1;
+	u32 gain_div = 1;
+	u32 factor;
+	int ret;
+
+	ret = device_property_read_u32(dev, "sense-resistor-micro-ohms",
+				       &sense);
+	if (ret) {
+		dev_err(dev, "failed to read the sense resistance: %d\n", ret);
+		return ret;
+	}
+
+	device_property_read_u32(dev, "sense-gain-mult", &gain_mult);
+	device_property_read_u32(dev, "sense-gain-div", &gain_div);
+
+	/*
+	 * Calculate the scaling factor, 1 / (gain * sense), or
+	 * gain_div / (gain_mult * sense), while trying to keep the
+	 * numerator/denominator from overflowing.
+	 */
+	factor = gcd(sense, 1000000);
+	rescale->numerator = 1000000 / factor;
+	rescale->denominator = sense / factor;
+
+	factor = gcd(rescale->numerator, gain_mult);
+	rescale->numerator /= factor;
+	rescale->denominator *= gain_mult / factor;
+
+	factor = gcd(rescale->denominator, gain_div);
+	rescale->numerator *= gain_div / factor;
+	rescale->denominator /= factor;
+
+	return 0;
+}
+
+static int rescale_current_sense_shunt_props(struct device *dev,
+					     struct rescale *rescale)
+{
+	u32 shunt;
+	u32 factor;
+	int ret;
+
+	ret = device_property_read_u32(dev, "shunt-resistor-micro-ohms",
+				       &shunt);
+	if (ret) {
+		dev_err(dev, "failed to read the shunt resistance: %d\n", ret);
+		return ret;
+	}
+
+	factor = gcd(shunt, 1000000);
+	rescale->numerator = 1000000 / factor;
+	rescale->denominator = shunt / factor;
+
+	return 0;
+}
+
+static int rescale_voltage_divider_props(struct device *dev,
+					 struct rescale *rescale)
+{
+	int ret;
+	u32 factor;
+
+	ret = device_property_read_u32(dev, "output-ohms",
+				       &rescale->denominator);
+	if (ret) {
+		dev_err(dev, "failed to read output-ohms: %d\n", ret);
+		return ret;
+	}
+
+	ret = device_property_read_u32(dev, "full-ohms",
+				       &rescale->numerator);
+	if (ret) {
+		dev_err(dev, "failed to read full-ohms: %d\n", ret);
+		return ret;
+	}
+
+	factor = gcd(rescale->numerator, rescale->denominator);
+	rescale->numerator /= factor;
+	rescale->denominator /= factor;
+
+	return 0;
+}
+
+enum rescale_variant {
+	CURRENT_SENSE_AMPLIFIER,
+	CURRENT_SENSE_SHUNT,
+	VOLTAGE_DIVIDER,
+};
+
+static const struct rescale_cfg rescale_cfg[] = {
+	[CURRENT_SENSE_AMPLIFIER] = {
+		.type = IIO_CURRENT,
+		.props = rescale_current_sense_amplifier_props,
+	},
+	[CURRENT_SENSE_SHUNT] = {
+		.type = IIO_CURRENT,
+		.props = rescale_current_sense_shunt_props,
+	},
+	[VOLTAGE_DIVIDER] = {
+		.type = IIO_VOLTAGE,
+		.props = rescale_voltage_divider_props,
+	},
+};
+
+static const struct of_device_id rescale_match[] = {
+	{ .compatible = "current-sense-amplifier",
+	  .data = &rescale_cfg[CURRENT_SENSE_AMPLIFIER], },
+	{ .compatible = "current-sense-shunt",
+	  .data = &rescale_cfg[CURRENT_SENSE_SHUNT], },
+	{ .compatible = "voltage-divider",
+	  .data = &rescale_cfg[VOLTAGE_DIVIDER], },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, rescale_match);
+
+static int rescale_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct iio_dev *indio_dev;
+	struct iio_channel *source;
+	struct rescale *rescale;
+	int sizeof_ext_info;
+	int sizeof_priv;
+	int i;
+	int ret;
+
+	source = devm_iio_channel_get(dev, NULL);
+	if (IS_ERR(source)) {
+		if (PTR_ERR(source) != -EPROBE_DEFER)
+			dev_err(dev, "failed to get source channel\n");
+		return PTR_ERR(source);
+	}
+
+	sizeof_ext_info = iio_get_channel_ext_info_count(source);
+	if (sizeof_ext_info) {
+		sizeof_ext_info += 1; /* one extra entry for the sentinel */
+		sizeof_ext_info *= sizeof(*rescale->ext_info);
+	}
+
+	sizeof_priv = sizeof(*rescale) + sizeof_ext_info;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof_priv);
+	if (!indio_dev)
+		return -ENOMEM;
+
+	rescale = iio_priv(indio_dev);
+
+	rescale->cfg = of_device_get_match_data(dev);
+	rescale->numerator = 1;
+	rescale->denominator = 1;
+
+	ret = rescale->cfg->props(dev, rescale);
+	if (ret)
+		return ret;
+
+	if (!rescale->numerator || !rescale->denominator) {
+		dev_err(dev, "invalid scaling factor.\n");
+		return -EINVAL;
+	}
+
+	platform_set_drvdata(pdev, indio_dev);
+
+	rescale->source = source;
+
+	indio_dev->name = dev_name(dev);
+	indio_dev->dev.parent = dev;
+	indio_dev->info = &rescale_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = &rescale->chan;
+	indio_dev->num_channels = 1;
+	if (sizeof_ext_info) {
+		rescale->ext_info = devm_kmemdup(dev,
+						 source->channel->ext_info,
+						 sizeof_ext_info, GFP_KERNEL);
+		if (!rescale->ext_info)
+			return -ENOMEM;
+
+		for (i = 0; rescale->ext_info[i].name; ++i) {
+			struct iio_chan_spec_ext_info *ext_info =
+				&rescale->ext_info[i];
+
+			if (source->channel->ext_info[i].read)
+				ext_info->read = rescale_read_ext_info;
+			if (source->channel->ext_info[i].write)
+				ext_info->write = rescale_write_ext_info;
+			ext_info->private = i;
+		}
+	}
+
+	ret = rescale_configure_channel(dev, rescale);
+	if (ret)
+		return ret;
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static struct platform_driver rescale_driver = {
+	.probe = rescale_probe,
+	.driver = {
+		.name = "iio-rescale",
+		.of_match_table = rescale_match,
+	},
+};
+module_platform_driver(rescale_driver);
+
+MODULE_DESCRIPTION("IIO rescale driver");
+MODULE_AUTHOR("Peter Rosin <peda@axentia.se>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/amplifiers/ad8366.c b/drivers/iio/amplifiers/ad8366.c
index 43667866321e..0138337aedd1 100644
--- a/drivers/iio/amplifiers/ad8366.c
+++ b/drivers/iio/amplifiers/ad8366.c
@@ -161,12 +161,14 @@ static int ad8366_probe(struct spi_device *spi)
 	indio_dev->channels = ad8366_channels;
 	indio_dev->num_channels = ARRAY_SIZE(ad8366_channels);
 
+	ret = ad8366_write(indio_dev, 0 , 0);
+	if (ret < 0)
+		goto error_disable_reg;
+
 	ret = iio_device_register(indio_dev);
 	if (ret)
 		goto error_disable_reg;
 
-	ad8366_write(indio_dev, 0, 0);
-
 	return 0;
 
 error_disable_reg:
diff --git a/drivers/iio/buffer/industrialio-buffer-dma.c b/drivers/iio/buffer/industrialio-buffer-dma.c
index 05e0c353e089..b32bf57910ca 100644
--- a/drivers/iio/buffer/industrialio-buffer-dma.c
+++ b/drivers/iio/buffer/industrialio-buffer-dma.c
@@ -587,7 +587,7 @@ EXPORT_SYMBOL_GPL(iio_dma_buffer_set_bytes_per_datum);
  * Should be used as the set_length callback for iio_buffer_access_ops
  * struct for DMA buffers.
  */
-int iio_dma_buffer_set_length(struct iio_buffer *buffer, int length)
+int iio_dma_buffer_set_length(struct iio_buffer *buffer, unsigned int length)
 {
 	/* Avoid an invalid state */
 	if (length < 2)
diff --git a/drivers/iio/buffer/kfifo_buf.c b/drivers/iio/buffer/kfifo_buf.c
index 047fe757ab97..70c302a93d7f 100644
--- a/drivers/iio/buffer/kfifo_buf.c
+++ b/drivers/iio/buffer/kfifo_buf.c
@@ -22,11 +22,18 @@ struct iio_kfifo {
 #define iio_to_kfifo(r) container_of(r, struct iio_kfifo, buffer)
 
 static inline int __iio_allocate_kfifo(struct iio_kfifo *buf,
-				int bytes_per_datum, int length)
+			size_t bytes_per_datum, unsigned int length)
 {
 	if ((length == 0) || (bytes_per_datum == 0))
 		return -EINVAL;
 
+	/*
+	 * Make sure we don't overflow an unsigned int after kfifo rounds up to
+	 * the next power of 2.
+	 */
+	if (roundup_pow_of_two(length) > UINT_MAX / bytes_per_datum)
+		return -EINVAL;
+
 	return __kfifo_alloc((struct __kfifo *)&buf->kf, length,
 			     bytes_per_datum, GFP_KERNEL);
 }
@@ -67,7 +74,7 @@ static int iio_set_bytes_per_datum_kfifo(struct iio_buffer *r, size_t bpd)
 	return 0;
 }
 
-static int iio_set_length_kfifo(struct iio_buffer *r, int length)
+static int iio_set_length_kfifo(struct iio_buffer *r, unsigned int length)
 {
 	/* Avoid an invalid state */
 	if (length < 2)
diff --git a/drivers/iio/chemical/atlas-ph-sensor.c b/drivers/iio/chemical/atlas-ph-sensor.c
index abfc4bbc4cfc..a406ad31b096 100644
--- a/drivers/iio/chemical/atlas-ph-sensor.c
+++ b/drivers/iio/chemical/atlas-ph-sensor.c
@@ -61,9 +61,9 @@
 #define ATLAS_REG_ORP_CALIB_STATUS	0x0d
 #define ATLAS_REG_ORP_DATA		0x0e
 
-#define ATLAS_PH_INT_TIME_IN_US		450000
-#define ATLAS_EC_INT_TIME_IN_US		650000
-#define ATLAS_ORP_INT_TIME_IN_US	450000
+#define ATLAS_PH_INT_TIME_IN_MS		450
+#define ATLAS_EC_INT_TIME_IN_MS		650
+#define ATLAS_ORP_INT_TIME_IN_MS	450
 
 enum {
 	ATLAS_PH_SM,
@@ -270,21 +270,21 @@ static struct atlas_device atlas_devices[] = {
 				.num_channels = 3,
 				.data_reg = ATLAS_REG_PH_DATA,
 				.calibration = &atlas_check_ph_calibration,
-				.delay = ATLAS_PH_INT_TIME_IN_US,
+				.delay = ATLAS_PH_INT_TIME_IN_MS,
 	},
 	[ATLAS_EC_SM] = {
 				.channels = atlas_ec_channels,
 				.num_channels = 5,
 				.data_reg = ATLAS_REG_EC_DATA,
 				.calibration = &atlas_check_ec_calibration,
-				.delay = ATLAS_EC_INT_TIME_IN_US,
+				.delay = ATLAS_EC_INT_TIME_IN_MS,
 	},
 	[ATLAS_ORP_SM] = {
 				.channels = atlas_orp_channels,
 				.num_channels = 2,
 				.data_reg = ATLAS_REG_ORP_DATA,
 				.calibration = &atlas_check_orp_calibration,
-				.delay = ATLAS_ORP_INT_TIME_IN_US,
+				.delay = ATLAS_ORP_INT_TIME_IN_MS,
 	},
 };
 
@@ -393,7 +393,7 @@ static int atlas_read_measurement(struct atlas_data *data, int reg, __be32 *val)
 	}
 
 	if (suspended)
-		usleep_range(data->chip->delay, data->chip->delay + 100000);
+		msleep(data->chip->delay);
 
 	ret = regmap_bulk_read(data->regmap, reg, (u8 *) val, sizeof(*val));
 
diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c
index 705cb3e72663..89cb0066a6e0 100644
--- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c
+++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c
@@ -19,6 +19,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/iio/buffer.h>
+#include <linux/iio/common/cros_ec_sensors_core.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/kfifo_buf.h>
 #include <linux/iio/trigger_consumer.h>
@@ -31,8 +32,6 @@
 #include <linux/slab.h>
 #include <linux/sysfs.h>
 
-#include "cros_ec_sensors_core.h"
-
 #define CROS_EC_SENSORS_MAX_CHANNELS 4
 
 /* State data for ec_sensors iio driver. */
diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c
index a620eb5ce202..414cc43c287e 100644
--- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c
+++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c
@@ -16,6 +16,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/iio/buffer.h>
+#include <linux/iio/common/cros_ec_sensors_core.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/kfifo_buf.h>
 #include <linux/iio/trigger_consumer.h>
@@ -27,8 +28,6 @@
 #include <linux/sysfs.h>
 #include <linux/platform_device.h>
 
-#include "cros_ec_sensors_core.h"
-
 static char *cros_ec_loc[] = {
 	[MOTIONSENSE_LOC_BASE] = "base",
 	[MOTIONSENSE_LOC_LID] = "lid",
@@ -448,8 +447,7 @@ EXPORT_SYMBOL_GPL(cros_ec_sensors_core_write);
 
 static int __maybe_unused cros_ec_sensors_prepare(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct cros_ec_sensors_core_state *st = iio_priv(indio_dev);
 
 	if (st->curr_sampl_freq == 0)
@@ -471,8 +469,7 @@ static int __maybe_unused cros_ec_sensors_prepare(struct device *dev)
 
 static void __maybe_unused cros_ec_sensors_complete(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct cros_ec_sensors_core_state *st = iio_priv(indio_dev);
 
 	if (st->curr_sampl_freq == 0)
diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.h b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.h
deleted file mode 100644
index 2edf68dc7336..000000000000
--- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.h
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * ChromeOS EC sensor hub
- *
- * Copyright (C) 2016 Google, Inc
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#ifndef __CROS_EC_SENSORS_CORE_H
-#define __CROS_EC_SENSORS_CORE_H
-
-#include <linux/irqreturn.h>
-
-enum {
-	CROS_EC_SENSOR_X,
-	CROS_EC_SENSOR_Y,
-	CROS_EC_SENSOR_Z,
-	CROS_EC_SENSOR_MAX_AXIS,
-};
-
-/* EC returns sensor values using signed 16 bit registers */
-#define CROS_EC_SENSOR_BITS 16
-
-/*
- * 4 16 bit channels are allowed.
- * Good enough for current sensors, they use up to 3 16 bit vectors.
- */
-#define CROS_EC_SAMPLE_SIZE  (sizeof(s64) * 2)
-
-/* Minimum sampling period to use when device is suspending */
-#define CROS_EC_MIN_SUSPEND_SAMPLING_FREQUENCY 1000  /* 1 second */
-
-/**
- * struct cros_ec_sensors_core_state - state data for EC sensors IIO driver
- * @ec:				cros EC device structure
- * @cmd_lock:			lock used to prevent simultaneous access to the
- *				commands.
- * @msg:			cros EC command structure
- * @param:			motion sensor parameters structure
- * @resp:			motion sensor response structure
- * @type:			type of motion sensor
- * @loc:			location where the motion sensor is placed
- * @calib:			calibration parameters. Note that trigger
- *				captured data will always provide the calibrated
- *				data
- * @samples:			static array to hold data from a single capture.
- *				For each channel we need 2 bytes, except for
- *				the timestamp. The timestamp is always last and
- *				is always 8-byte aligned.
- * @read_ec_sensors_data:	function used for accessing sensors values
- * @cuur_sampl_freq:		current sampling period
- */
-struct cros_ec_sensors_core_state {
-	struct cros_ec_device *ec;
-	struct mutex cmd_lock;
-
-	struct cros_ec_command *msg;
-	struct ec_params_motion_sense param;
-	struct ec_response_motion_sense *resp;
-
-	enum motionsensor_type type;
-	enum motionsensor_location loc;
-
-	s16 calib[CROS_EC_SENSOR_MAX_AXIS];
-
-	u8 samples[CROS_EC_SAMPLE_SIZE];
-
-	int (*read_ec_sensors_data)(struct iio_dev *indio_dev,
-				    unsigned long scan_mask, s16 *data);
-
-	int curr_sampl_freq;
-};
-
-/**
- * cros_ec_sensors_read_lpc() - retrieve data from EC shared memory
- * @indio_dev:	pointer to IIO device
- * @scan_mask:	bitmap of the sensor indices to scan
- * @data:	location to store data
- *
- * This is the safe function for reading the EC data. It guarantees that the
- * data sampled was not modified by the EC while being read.
- *
- * Return: 0 on success, -errno on failure.
- */
-int cros_ec_sensors_read_lpc(struct iio_dev *indio_dev, unsigned long scan_mask,
-			     s16 *data);
-
-/**
- * cros_ec_sensors_read_cmd() - retrieve data using the EC command protocol
- * @indio_dev:	pointer to IIO device
- * @scan_mask:	bitmap of the sensor indices to scan
- * @data:	location to store data
- *
- * Return: 0 on success, -errno on failure.
- */
-int cros_ec_sensors_read_cmd(struct iio_dev *indio_dev, unsigned long scan_mask,
-			     s16 *data);
-
-/**
- * cros_ec_sensors_core_init() - basic initialization of the core structure
- * @pdev:		platform device created for the sensors
- * @indio_dev:		iio device structure of the device
- * @physical_device:	true if the device refers to a physical device
- *
- * Return: 0 on success, -errno on failure.
- */
-int cros_ec_sensors_core_init(struct platform_device *pdev,
-			      struct iio_dev *indio_dev, bool physical_device);
-
-/**
- * cros_ec_sensors_capture() - the trigger handler function
- * @irq:	the interrupt number.
- * @p:		a pointer to the poll function.
- *
- * On a trigger event occurring, if the pollfunc is attached then this
- * handler is called as a threaded interrupt (and hence may sleep). It
- * is responsible for grabbing data from the device and pushing it into
- * the associated buffer.
- *
- * Return: IRQ_HANDLED
- */
-irqreturn_t cros_ec_sensors_capture(int irq, void *p);
-
-/**
- * cros_ec_motion_send_host_cmd() - send motion sense host command
- * @st:		pointer to state information for device
- * @opt_length:	optional length to reduce the response size, useful on the data
- *		path. Otherwise, the maximal allowed response size is used
- *
- * When called, the sub-command is assumed to be set in param->cmd.
- *
- * Return: 0 on success, -errno on failure.
- */
-int cros_ec_motion_send_host_cmd(struct cros_ec_sensors_core_state *st,
-				 u16 opt_length);
-
-/**
- * cros_ec_sensors_core_read() - function to request a value from the sensor
- * @st:		pointer to state information for device
- * @chan:	channel specification structure table
- * @val:	will contain one element making up the returned value
- * @val2:	will contain another element making up the returned value
- * @mask:	specifies which values to be requested
- *
- * Return:	the type of value returned by the device
- */
-int cros_ec_sensors_core_read(struct cros_ec_sensors_core_state *st,
-			      struct iio_chan_spec const *chan,
-			      int *val, int *val2, long mask);
-
-/**
- * cros_ec_sensors_core_write() - function to write a value to the sensor
- * @st:		pointer to state information for device
- * @chan:	channel specification structure table
- * @val:	first part of value to write
- * @val2:	second part of value to write
- * @mask:	specifies which values to write
- *
- * Return:	the type of value returned by the device
- */
-int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st,
-			       struct iio_chan_spec const *chan,
-			       int val, int val2, long mask);
-
-extern const struct dev_pm_ops cros_ec_sensors_pm_ops;
-
-/* List of extended channel specification for all sensors */
-extern const struct iio_chan_spec_ext_info cros_ec_sensors_ext_info[];
-
-#endif  /* __CROS_EC_SENSORS_CORE_H */
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
index cfb6588565ba..1e10c0af2f2c 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
+++ b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
@@ -178,14 +178,14 @@ int hid_sensor_power_state(struct hid_sensor_common *st, bool state)
 #ifdef CONFIG_PM
 	int ret;
 
-	atomic_set(&st->user_requested_state, state);
-
 	if (atomic_add_unless(&st->runtime_pm_enable, 1, 1))
 		pm_runtime_enable(&st->pdev->dev);
 
-	if (state)
+	if (state) {
+		atomic_inc(&st->user_requested_state);
 		ret = pm_runtime_get_sync(&st->pdev->dev);
-	else {
+	} else {
+		atomic_dec(&st->user_requested_state);
 		pm_runtime_mark_last_busy(&st->pdev->dev);
 		pm_runtime_use_autosuspend(&st->pdev->dev);
 		ret = pm_runtime_put_autosuspend(&st->pdev->dev);
@@ -304,8 +304,7 @@ EXPORT_SYMBOL(hid_sensor_setup_trigger);
 
 static int __maybe_unused hid_sensor_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct hid_sensor_common *attrb = iio_device_get_drvdata(indio_dev);
 
 	return _hid_sensor_power_state(attrb, false);
@@ -313,8 +312,7 @@ static int __maybe_unused hid_sensor_suspend(struct device *dev)
 
 static int __maybe_unused hid_sensor_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct hid_sensor_common *attrb = iio_device_get_drvdata(indio_dev);
 	schedule_work(&attrb->work);
 	return 0;
@@ -322,8 +320,7 @@ static int __maybe_unused hid_sensor_resume(struct device *dev)
 
 static int __maybe_unused hid_sensor_runtime_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct hid_sensor_common *attrb = iio_device_get_drvdata(indio_dev);
 	return _hid_sensor_power_state(attrb, true);
 }
diff --git a/drivers/iio/dac/Kconfig b/drivers/iio/dac/Kconfig
index 76db0768e454..06e90debb9f5 100644
--- a/drivers/iio/dac/Kconfig
+++ b/drivers/iio/dac/Kconfig
@@ -131,16 +131,31 @@ config LTC2632
 	  module will be called ltc2632.
 
 config AD5686
-	tristate "Analog Devices AD5686R/AD5685R/AD5684R DAC SPI driver"
+	tristate
+
+config AD5686_SPI
+	tristate "Analog Devices AD5686 and similar multi-channel DACs (SPI)"
 	depends on SPI
+	select AD5686
 	help
-	  Say yes here to build support for Analog Devices AD5686R, AD5685R,
-	  AD5684R, AD5791 Voltage Output Digital to
-	  Analog Converter.
+	  Say yes here to build support for Analog Devices AD5672R, AD5676,
+	  AD5676R, AD5684, AD5684R, AD5684R, AD5685R, AD5686, AD5686R.
+	  Voltage Output Digital to Analog Converter.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called ad5686.
 
+config AD5696_I2C
+	tristate "Analog Devices AD5696 and similar multi-channel DACs (I2C)"
+	depends on I2C
+	select AD5686
+	help
+	  Say yes here to build support for Analog Devices AD5671R, AD5675R,
+	  AD5694, AD5694R, AD5695R, AD5696, AD5696R Voltage Output Digital to
+	  Analog Converter.
+	  To compile this driver as a module, choose M here: the module will be
+	  called ad5696.
+
 config AD5755
 	tristate "Analog Devices AD5755/AD5755-1/AD5757/AD5735/AD5737 DAC driver"
 	depends on SPI_MASTER
@@ -321,6 +336,16 @@ config TI_DAC082S085
 
 	  If compiled as a module, it will be called ti-dac082s085.
 
+config TI_DAC5571
+	tristate "Texas Instruments 8/10/12/16-bit 1/2/4-channel DAC driver"
+	depends on I2C
+	help
+	  Driver for the Texas Instruments
+	  DAC5571, DAC6571, DAC7571, DAC5574, DAC6574, DAC7574, DAC5573,
+	  DAC6573, DAC7573, DAC8571, DAC8574.
+
+	  If compiled as a module, it will be called ti-dac5571.
+
 config VF610_DAC
 	tristate "Vybrid vf610 DAC driver"
 	depends on OF
diff --git a/drivers/iio/dac/Makefile b/drivers/iio/dac/Makefile
index 81e710ed7491..57aa230d34ab 100644
--- a/drivers/iio/dac/Makefile
+++ b/drivers/iio/dac/Makefile
@@ -20,6 +20,8 @@ obj-$(CONFIG_AD5761) += ad5761.o
 obj-$(CONFIG_AD5764) += ad5764.o
 obj-$(CONFIG_AD5791) += ad5791.o
 obj-$(CONFIG_AD5686) += ad5686.o
+obj-$(CONFIG_AD5686_SPI) += ad5686-spi.o
+obj-$(CONFIG_AD5696_I2C) += ad5696-i2c.o
 obj-$(CONFIG_AD7303) += ad7303.o
 obj-$(CONFIG_AD8801) += ad8801.o
 obj-$(CONFIG_CIO_DAC) += cio-dac.o
@@ -35,4 +37,5 @@ obj-$(CONFIG_MCP4922) += mcp4922.o
 obj-$(CONFIG_STM32_DAC_CORE) += stm32-dac-core.o
 obj-$(CONFIG_STM32_DAC) += stm32-dac.o
 obj-$(CONFIG_TI_DAC082S085) += ti-dac082s085.o
+obj-$(CONFIG_TI_DAC5571) += ti-dac5571.o
 obj-$(CONFIG_VF610_DAC) += vf610_dac.o
diff --git a/drivers/iio/dac/ad5686-spi.c b/drivers/iio/dac/ad5686-spi.c
new file mode 100644
index 000000000000..1df9143f55e9
--- /dev/null
+++ b/drivers/iio/dac/ad5686-spi.c
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * AD5672R, AD5676, AD5676R, AD5681R, AD5682R, AD5683, AD5683R,
+ * AD5684, AD5684R, AD5685R, AD5686, AD5686R
+ * Digital to analog converters driver
+ *
+ * Copyright 2018 Analog Devices Inc.
+ */
+
+#include "ad5686.h"
+
+#include <linux/module.h>
+#include <linux/spi/spi.h>
+
+static int ad5686_spi_write(struct ad5686_state *st,
+			    u8 cmd, u8 addr, u16 val)
+{
+	struct spi_device *spi = to_spi_device(st->dev);
+	u8 tx_len, *buf;
+
+	switch (st->chip_info->regmap_type) {
+	case AD5683_REGMAP:
+		st->data[0].d32 = cpu_to_be32(AD5686_CMD(cmd) |
+					      AD5683_DATA(val));
+		buf = &st->data[0].d8[1];
+		tx_len = 3;
+		break;
+	case AD5686_REGMAP:
+		st->data[0].d32 = cpu_to_be32(AD5686_CMD(cmd) |
+					      AD5686_ADDR(addr) |
+					      val);
+		buf = &st->data[0].d8[1];
+		tx_len = 3;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return spi_write(spi, buf, tx_len);
+}
+
+static int ad5686_spi_read(struct ad5686_state *st, u8 addr)
+{
+	struct spi_transfer t[] = {
+		{
+			.tx_buf = &st->data[0].d8[1],
+			.len = 3,
+			.cs_change = 1,
+		}, {
+			.tx_buf = &st->data[1].d8[1],
+			.rx_buf = &st->data[2].d8[1],
+			.len = 3,
+		},
+	};
+	struct spi_device *spi = to_spi_device(st->dev);
+	u8 cmd = 0;
+	int ret;
+
+	if (st->chip_info->regmap_type == AD5686_REGMAP)
+		cmd = AD5686_CMD_READBACK_ENABLE;
+	else if (st->chip_info->regmap_type == AD5683_REGMAP)
+		cmd = AD5686_CMD_READBACK_ENABLE_V2;
+
+	st->data[0].d32 = cpu_to_be32(AD5686_CMD(cmd) |
+				      AD5686_ADDR(addr));
+	st->data[1].d32 = cpu_to_be32(AD5686_CMD(AD5686_CMD_NOOP));
+
+	ret = spi_sync_transfer(spi, t, ARRAY_SIZE(t));
+	if (ret < 0)
+		return ret;
+
+	return be32_to_cpu(st->data[2].d32);
+}
+
+static int ad5686_spi_probe(struct spi_device *spi)
+{
+	const struct spi_device_id *id = spi_get_device_id(spi);
+
+	return ad5686_probe(&spi->dev, id->driver_data, id->name,
+			    ad5686_spi_write, ad5686_spi_read);
+}
+
+static int ad5686_spi_remove(struct spi_device *spi)
+{
+	return ad5686_remove(&spi->dev);
+}
+
+static const struct spi_device_id ad5686_spi_id[] = {
+	{"ad5672r", ID_AD5672R},
+	{"ad5676", ID_AD5676},
+	{"ad5676r", ID_AD5676R},
+	{"ad5681r", ID_AD5681R},
+	{"ad5682r", ID_AD5682R},
+	{"ad5683", ID_AD5683},
+	{"ad5683r", ID_AD5683R},
+	{"ad5684", ID_AD5684},
+	{"ad5684r", ID_AD5684R},
+	{"ad5685", ID_AD5685R}, /* Does not exist */
+	{"ad5685r", ID_AD5685R},
+	{"ad5686", ID_AD5686},
+	{"ad5686r", ID_AD5686R},
+	{}
+};
+MODULE_DEVICE_TABLE(spi, ad5686_spi_id);
+
+static struct spi_driver ad5686_spi_driver = {
+	.driver = {
+		.name = "ad5686",
+	},
+	.probe = ad5686_spi_probe,
+	.remove = ad5686_spi_remove,
+	.id_table = ad5686_spi_id,
+};
+
+module_spi_driver(ad5686_spi_driver);
+
+MODULE_AUTHOR("Stefan Popa <stefan.popa@analog.com>");
+MODULE_DESCRIPTION("Analog Devices AD5686 and similar multi-channel DACs");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/dac/ad5686.c b/drivers/iio/dac/ad5686.c
index 20254df7f9c7..e136f0fd38f0 100644
--- a/drivers/iio/dac/ad5686.c
+++ b/drivers/iio/dac/ad5686.c
@@ -1,9 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * AD5686R, AD5685R, AD5684R Digital to analog converters  driver
  *
  * Copyright 2011 Analog Devices Inc.
- *
- * Licensed under the GPL-2.
  */
 
 #include <linux/interrupt.h>
@@ -11,7 +10,6 @@
 #include <linux/device.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
-#include <linux/spi/spi.h>
 #include <linux/slab.h>
 #include <linux/sysfs.h>
 #include <linux/regulator/consumer.h>
@@ -19,116 +17,7 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
-#define AD5686_DAC_CHANNELS			4
-
-#define AD5686_ADDR(x)				((x) << 16)
-#define AD5686_CMD(x)				((x) << 20)
-
-#define AD5686_ADDR_DAC(chan)		(0x1 << (chan))
-#define AD5686_ADDR_ALL_DAC			0xF
-
-#define AD5686_CMD_NOOP				0x0
-#define AD5686_CMD_WRITE_INPUT_N		0x1
-#define AD5686_CMD_UPDATE_DAC_N			0x2
-#define AD5686_CMD_WRITE_INPUT_N_UPDATE_N	0x3
-#define AD5686_CMD_POWERDOWN_DAC		0x4
-#define AD5686_CMD_LDAC_MASK			0x5
-#define AD5686_CMD_RESET			0x6
-#define AD5686_CMD_INTERNAL_REFER_SETUP		0x7
-#define AD5686_CMD_DAISY_CHAIN_ENABLE		0x8
-#define AD5686_CMD_READBACK_ENABLE		0x9
-
-#define AD5686_LDAC_PWRDN_NONE			0x0
-#define AD5686_LDAC_PWRDN_1K			0x1
-#define AD5686_LDAC_PWRDN_100K			0x2
-#define AD5686_LDAC_PWRDN_3STATE		0x3
-
-/**
- * struct ad5686_chip_info - chip specific information
- * @int_vref_mv:	AD5620/40/60: the internal reference voltage
- * @channel:		channel specification
-*/
-
-struct ad5686_chip_info {
-	u16				int_vref_mv;
-	struct iio_chan_spec		channel[AD5686_DAC_CHANNELS];
-};
-
-/**
- * struct ad5446_state - driver instance specific data
- * @spi:		spi_device
- * @chip_info:		chip model specific constants, available modes etc
- * @reg:		supply regulator
- * @vref_mv:		actual reference voltage used
- * @pwr_down_mask:	power down mask
- * @pwr_down_mode:	current power down mode
- * @data:		spi transfer buffers
- */
-
-struct ad5686_state {
-	struct spi_device		*spi;
-	const struct ad5686_chip_info	*chip_info;
-	struct regulator		*reg;
-	unsigned short			vref_mv;
-	unsigned			pwr_down_mask;
-	unsigned			pwr_down_mode;
-	/*
-	 * DMA (thus cache coherency maintenance) requires the
-	 * transfer buffers to live in their own cache lines.
-	 */
-
-	union {
-		__be32 d32;
-		u8 d8[4];
-	} data[3] ____cacheline_aligned;
-};
-
-/**
- * ad5686_supported_device_ids:
- */
-
-enum ad5686_supported_device_ids {
-	ID_AD5684,
-	ID_AD5685,
-	ID_AD5686,
-};
-static int ad5686_spi_write(struct ad5686_state *st,
-			     u8 cmd, u8 addr, u16 val, u8 shift)
-{
-	val <<= shift;
-
-	st->data[0].d32 = cpu_to_be32(AD5686_CMD(cmd) |
-			      AD5686_ADDR(addr) |
-			      val);
-
-	return spi_write(st->spi, &st->data[0].d8[1], 3);
-}
-
-static int ad5686_spi_read(struct ad5686_state *st, u8 addr)
-{
-	struct spi_transfer t[] = {
-		{
-			.tx_buf = &st->data[0].d8[1],
-			.len = 3,
-			.cs_change = 1,
-		}, {
-			.tx_buf = &st->data[1].d8[1],
-			.rx_buf = &st->data[2].d8[1],
-			.len = 3,
-		},
-	};
-	int ret;
-
-	st->data[0].d32 = cpu_to_be32(AD5686_CMD(AD5686_CMD_READBACK_ENABLE) |
-			      AD5686_ADDR(addr));
-	st->data[1].d32 = cpu_to_be32(AD5686_CMD(AD5686_CMD_NOOP));
-
-	ret = spi_sync_transfer(st->spi, t, ARRAY_SIZE(t));
-	if (ret < 0)
-		return ret;
-
-	return be32_to_cpu(st->data[2].d32);
-}
+#include "ad5686.h"
 
 static const char * const ad5686_powerdown_modes[] = {
 	"1kohm_to_gnd",
@@ -137,7 +26,7 @@ static const char * const ad5686_powerdown_modes[] = {
 };
 
 static int ad5686_get_powerdown_mode(struct iio_dev *indio_dev,
-	const struct iio_chan_spec *chan)
+				     const struct iio_chan_spec *chan)
 {
 	struct ad5686_state *st = iio_priv(indio_dev);
 
@@ -145,7 +34,8 @@ static int ad5686_get_powerdown_mode(struct iio_dev *indio_dev,
 }
 
 static int ad5686_set_powerdown_mode(struct iio_dev *indio_dev,
-	const struct iio_chan_spec *chan, unsigned int mode)
+				     const struct iio_chan_spec *chan,
+				     unsigned int mode)
 {
 	struct ad5686_state *st = iio_priv(indio_dev);
 
@@ -163,21 +53,25 @@ static const struct iio_enum ad5686_powerdown_mode_enum = {
 };
 
 static ssize_t ad5686_read_dac_powerdown(struct iio_dev *indio_dev,
-	uintptr_t private, const struct iio_chan_spec *chan, char *buf)
+		uintptr_t private, const struct iio_chan_spec *chan, char *buf)
 {
 	struct ad5686_state *st = iio_priv(indio_dev);
 
 	return sprintf(buf, "%d\n", !!(st->pwr_down_mask &
-			(0x3 << (chan->channel * 2))));
+				       (0x3 << (chan->channel * 2))));
 }
 
 static ssize_t ad5686_write_dac_powerdown(struct iio_dev *indio_dev,
-	 uintptr_t private, const struct iio_chan_spec *chan, const char *buf,
-	 size_t len)
+					  uintptr_t private,
+					  const struct iio_chan_spec *chan,
+					  const char *buf,
+					  size_t len)
 {
 	bool readin;
 	int ret;
 	struct ad5686_state *st = iio_priv(indio_dev);
+	unsigned int val, ref_bit_msk;
+	u8 shift;
 
 	ret = strtobool(buf, &readin);
 	if (ret)
@@ -188,8 +82,28 @@ static ssize_t ad5686_write_dac_powerdown(struct iio_dev *indio_dev,
 	else
 		st->pwr_down_mask &= ~(0x3 << (chan->channel * 2));
 
-	ret = ad5686_spi_write(st, AD5686_CMD_POWERDOWN_DAC, 0,
-			       st->pwr_down_mask & st->pwr_down_mode, 0);
+	switch (st->chip_info->regmap_type) {
+	case AD5683_REGMAP:
+		shift = 13;
+		ref_bit_msk = AD5683_REF_BIT_MSK;
+		break;
+	case AD5686_REGMAP:
+		shift = 0;
+		ref_bit_msk = 0;
+		break;
+	case AD5693_REGMAP:
+		shift = 13;
+		ref_bit_msk = AD5693_REF_BIT_MSK;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	val = ((st->pwr_down_mask & st->pwr_down_mode) << shift);
+	if (!st->use_internal_vref)
+		val |= ref_bit_msk;
+
+	ret = st->write(st, AD5686_CMD_POWERDOWN_DAC, 0, val);
 
 	return ret ? ret : len;
 }
@@ -206,7 +120,7 @@ static int ad5686_read_raw(struct iio_dev *indio_dev,
 	switch (m) {
 	case IIO_CHAN_INFO_RAW:
 		mutex_lock(&indio_dev->mlock);
-		ret = ad5686_spi_read(st, chan->address);
+		ret = st->read(st, chan->address);
 		mutex_unlock(&indio_dev->mlock);
 		if (ret < 0)
 			return ret;
@@ -221,10 +135,10 @@ static int ad5686_read_raw(struct iio_dev *indio_dev,
 }
 
 static int ad5686_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
+			    struct iio_chan_spec const *chan,
+			    int val,
+			    int val2,
+			    long mask)
 {
 	struct ad5686_state *st = iio_priv(indio_dev);
 	int ret;
@@ -235,11 +149,10 @@ static int ad5686_write_raw(struct iio_dev *indio_dev,
 			return -EINVAL;
 
 		mutex_lock(&indio_dev->mlock);
-		ret = ad5686_spi_write(st,
-				 AD5686_CMD_WRITE_INPUT_N_UPDATE_N,
-				 chan->address,
-				 val,
-				 chan->scan_type.shift);
+		ret = st->write(st,
+				AD5686_CMD_WRITE_INPUT_N_UPDATE_N,
+				chan->address,
+				val << chan->scan_type.shift);
 		mutex_unlock(&indio_dev->mlock);
 		break;
 	default:
@@ -266,14 +179,14 @@ static const struct iio_chan_spec_ext_info ad5686_ext_info[] = {
 	{ },
 };
 
-#define AD5868_CHANNEL(chan, bits, _shift) {			\
+#define AD5868_CHANNEL(chan, addr, bits, _shift) {		\
 		.type = IIO_VOLTAGE,				\
 		.indexed = 1,					\
 		.output = 1,					\
 		.channel = chan,				\
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),\
-		.address = AD5686_ADDR_DAC(chan),		\
+		.address = addr,				\
 		.scan_type = {					\
 			.sign = 'u',				\
 			.realbits = (bits),			\
@@ -283,45 +196,191 @@ static const struct iio_chan_spec_ext_info ad5686_ext_info[] = {
 		.ext_info = ad5686_ext_info,			\
 }
 
+#define DECLARE_AD5693_CHANNELS(name, bits, _shift)		\
+static struct iio_chan_spec name[] = {				\
+		AD5868_CHANNEL(0, 0, bits, _shift),		\
+}
+
+#define DECLARE_AD5686_CHANNELS(name, bits, _shift)		\
+static struct iio_chan_spec name[] = {				\
+		AD5868_CHANNEL(0, 1, bits, _shift),		\
+		AD5868_CHANNEL(1, 2, bits, _shift),		\
+		AD5868_CHANNEL(2, 4, bits, _shift),		\
+		AD5868_CHANNEL(3, 8, bits, _shift),		\
+}
+
+#define DECLARE_AD5676_CHANNELS(name, bits, _shift)		\
+static struct iio_chan_spec name[] = {				\
+		AD5868_CHANNEL(0, 0, bits, _shift),		\
+		AD5868_CHANNEL(1, 1, bits, _shift),		\
+		AD5868_CHANNEL(2, 2, bits, _shift),		\
+		AD5868_CHANNEL(3, 3, bits, _shift),		\
+		AD5868_CHANNEL(4, 4, bits, _shift),		\
+		AD5868_CHANNEL(5, 5, bits, _shift),		\
+		AD5868_CHANNEL(6, 6, bits, _shift),		\
+		AD5868_CHANNEL(7, 7, bits, _shift),		\
+}
+
+DECLARE_AD5676_CHANNELS(ad5672_channels, 12, 4);
+DECLARE_AD5676_CHANNELS(ad5676_channels, 16, 0);
+DECLARE_AD5686_CHANNELS(ad5684_channels, 12, 4);
+DECLARE_AD5686_CHANNELS(ad5685r_channels, 14, 2);
+DECLARE_AD5686_CHANNELS(ad5686_channels, 16, 0);
+DECLARE_AD5693_CHANNELS(ad5693_channels, 16, 0);
+DECLARE_AD5693_CHANNELS(ad5692r_channels, 14, 2);
+DECLARE_AD5693_CHANNELS(ad5691r_channels, 12, 4);
+
 static const struct ad5686_chip_info ad5686_chip_info_tbl[] = {
+	[ID_AD5671R] = {
+		.channels = ad5672_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 8,
+		.regmap_type = AD5686_REGMAP,
+	},
+	[ID_AD5672R] = {
+		.channels = ad5672_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 8,
+		.regmap_type = AD5686_REGMAP,
+	},
+	[ID_AD5675R] = {
+		.channels = ad5676_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 8,
+		.regmap_type = AD5686_REGMAP,
+	},
+	[ID_AD5676] = {
+		.channels = ad5676_channels,
+		.num_channels = 8,
+		.regmap_type = AD5686_REGMAP,
+	},
+	[ID_AD5676R] = {
+		.channels = ad5676_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 8,
+		.regmap_type = AD5686_REGMAP,
+	},
+	[ID_AD5681R] = {
+		.channels = ad5691r_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 1,
+		.regmap_type = AD5683_REGMAP,
+	},
+	[ID_AD5682R] = {
+		.channels = ad5692r_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 1,
+		.regmap_type = AD5683_REGMAP,
+	},
+	[ID_AD5683] = {
+		.channels = ad5693_channels,
+		.num_channels = 1,
+		.regmap_type = AD5683_REGMAP,
+	},
+	[ID_AD5683R] = {
+		.channels = ad5693_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 1,
+		.regmap_type = AD5683_REGMAP,
+	},
 	[ID_AD5684] = {
-		.channel[0] = AD5868_CHANNEL(0, 12, 4),
-		.channel[1] = AD5868_CHANNEL(1, 12, 4),
-		.channel[2] = AD5868_CHANNEL(2, 12, 4),
-		.channel[3] = AD5868_CHANNEL(3, 12, 4),
+		.channels = ad5684_channels,
+		.num_channels = 4,
+		.regmap_type = AD5686_REGMAP,
+	},
+	[ID_AD5684R] = {
+		.channels = ad5684_channels,
 		.int_vref_mv = 2500,
+		.num_channels = 4,
+		.regmap_type = AD5686_REGMAP,
 	},
-	[ID_AD5685] = {
-		.channel[0] = AD5868_CHANNEL(0, 14, 2),
-		.channel[1] = AD5868_CHANNEL(1, 14, 2),
-		.channel[2] = AD5868_CHANNEL(2, 14, 2),
-		.channel[3] = AD5868_CHANNEL(3, 14, 2),
+	[ID_AD5685R] = {
+		.channels = ad5685r_channels,
 		.int_vref_mv = 2500,
+		.num_channels = 4,
+		.regmap_type = AD5686_REGMAP,
 	},
 	[ID_AD5686] = {
-		.channel[0] = AD5868_CHANNEL(0, 16, 0),
-		.channel[1] = AD5868_CHANNEL(1, 16, 0),
-		.channel[2] = AD5868_CHANNEL(2, 16, 0),
-		.channel[3] = AD5868_CHANNEL(3, 16, 0),
+		.channels = ad5686_channels,
+		.num_channels = 4,
+		.regmap_type = AD5686_REGMAP,
+	},
+	[ID_AD5686R] = {
+		.channels = ad5686_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 4,
+		.regmap_type = AD5686_REGMAP,
+	},
+	[ID_AD5691R] = {
+		.channels = ad5691r_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 1,
+		.regmap_type = AD5693_REGMAP,
+	},
+	[ID_AD5692R] = {
+		.channels = ad5692r_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 1,
+		.regmap_type = AD5693_REGMAP,
+	},
+	[ID_AD5693] = {
+		.channels = ad5693_channels,
+		.num_channels = 1,
+		.regmap_type = AD5693_REGMAP,
+	},
+	[ID_AD5693R] = {
+		.channels = ad5693_channels,
 		.int_vref_mv = 2500,
+		.num_channels = 1,
+		.regmap_type = AD5693_REGMAP,
+	},
+	[ID_AD5694] = {
+		.channels = ad5684_channels,
+		.num_channels = 4,
+		.regmap_type = AD5686_REGMAP,
+	},
+	[ID_AD5694R] = {
+		.channels = ad5684_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 4,
+		.regmap_type = AD5686_REGMAP,
+	},
+	[ID_AD5696] = {
+		.channels = ad5686_channels,
+		.num_channels = 4,
+		.regmap_type = AD5686_REGMAP,
+	},
+	[ID_AD5696R] = {
+		.channels = ad5686_channels,
+		.int_vref_mv = 2500,
+		.num_channels = 4,
+		.regmap_type = AD5686_REGMAP,
 	},
 };
 
-
-static int ad5686_probe(struct spi_device *spi)
+int ad5686_probe(struct device *dev,
+		 enum ad5686_supported_device_ids chip_type,
+		 const char *name, ad5686_write_func write,
+		 ad5686_read_func read)
 {
 	struct ad5686_state *st;
 	struct iio_dev *indio_dev;
-	int ret, voltage_uv = 0;
+	unsigned int val, ref_bit_msk;
+	u8 cmd;
+	int ret, i, voltage_uv = 0;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (indio_dev == NULL)
 		return  -ENOMEM;
 
 	st = iio_priv(indio_dev);
-	spi_set_drvdata(spi, indio_dev);
+	dev_set_drvdata(dev, indio_dev);
+
+	st->dev = dev;
+	st->write = write;
+	st->read = read;
 
-	st->reg = devm_regulator_get_optional(&spi->dev, "vcc");
+	st->reg = devm_regulator_get_optional(dev, "vcc");
 	if (!IS_ERR(st->reg)) {
 		ret = regulator_enable(st->reg);
 		if (ret)
@@ -334,28 +393,47 @@ static int ad5686_probe(struct spi_device *spi)
 		voltage_uv = ret;
 	}
 
-	st->chip_info =
-		&ad5686_chip_info_tbl[spi_get_device_id(spi)->driver_data];
+	st->chip_info = &ad5686_chip_info_tbl[chip_type];
 
 	if (voltage_uv)
 		st->vref_mv = voltage_uv / 1000;
 	else
 		st->vref_mv = st->chip_info->int_vref_mv;
 
-	st->spi = spi;
-
 	/* Set all the power down mode for all channels to 1K pulldown */
-	st->pwr_down_mode = 0x55;
+	for (i = 0; i < st->chip_info->num_channels; i++)
+		st->pwr_down_mode |= (0x01 << (i * 2));
 
-	indio_dev->dev.parent = &spi->dev;
-	indio_dev->name = spi_get_device_id(spi)->name;
+	indio_dev->dev.parent = dev;
+	indio_dev->name = name;
 	indio_dev->info = &ad5686_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->channels = st->chip_info->channel;
-	indio_dev->num_channels = AD5686_DAC_CHANNELS;
+	indio_dev->channels = st->chip_info->channels;
+	indio_dev->num_channels = st->chip_info->num_channels;
+
+	switch (st->chip_info->regmap_type) {
+	case AD5683_REGMAP:
+		cmd = AD5686_CMD_CONTROL_REG;
+		ref_bit_msk = AD5683_REF_BIT_MSK;
+		st->use_internal_vref = !voltage_uv;
+		break;
+	case AD5686_REGMAP:
+		cmd = AD5686_CMD_INTERNAL_REFER_SETUP;
+		ref_bit_msk = 0;
+		break;
+	case AD5693_REGMAP:
+		cmd = AD5686_CMD_CONTROL_REG;
+		ref_bit_msk = AD5693_REF_BIT_MSK;
+		st->use_internal_vref = !voltage_uv;
+		break;
+	default:
+		ret = -EINVAL;
+		goto error_disable_reg;
+	}
+
+	val = (voltage_uv | ref_bit_msk);
 
-	ret = ad5686_spi_write(st, AD5686_CMD_INTERNAL_REFER_SETUP, 0,
-				!!voltage_uv, 0);
+	ret = st->write(st, cmd, 0, !!val);
 	if (ret)
 		goto error_disable_reg;
 
@@ -370,10 +448,11 @@ static int ad5686_probe(struct spi_device *spi)
 		regulator_disable(st->reg);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(ad5686_probe);
 
-static int ad5686_remove(struct spi_device *spi)
+int ad5686_remove(struct device *dev)
 {
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct ad5686_state *st = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
@@ -382,24 +461,7 @@ static int ad5686_remove(struct spi_device *spi)
 
 	return 0;
 }
-
-static const struct spi_device_id ad5686_id[] = {
-	{"ad5684", ID_AD5684},
-	{"ad5685", ID_AD5685},
-	{"ad5686", ID_AD5686},
-	{}
-};
-MODULE_DEVICE_TABLE(spi, ad5686_id);
-
-static struct spi_driver ad5686_driver = {
-	.driver = {
-		   .name = "ad5686",
-		   },
-	.probe = ad5686_probe,
-	.remove = ad5686_remove,
-	.id_table = ad5686_id,
-};
-module_spi_driver(ad5686_driver);
+EXPORT_SYMBOL_GPL(ad5686_remove);
 
 MODULE_AUTHOR("Michael Hennerich <hennerich@blackfin.uclinux.org>");
 MODULE_DESCRIPTION("Analog Devices AD5686/85/84 DAC");
diff --git a/drivers/iio/dac/ad5686.h b/drivers/iio/dac/ad5686.h
new file mode 100644
index 000000000000..d05cda9f1edd
--- /dev/null
+++ b/drivers/iio/dac/ad5686.h
@@ -0,0 +1,145 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * This file is part of AD5686 DAC driver
+ *
+ * Copyright 2018 Analog Devices Inc.
+ */
+
+#ifndef __DRIVERS_IIO_DAC_AD5686_H__
+#define __DRIVERS_IIO_DAC_AD5686_H__
+
+#include <linux/types.h>
+#include <linux/cache.h>
+#include <linux/mutex.h>
+#include <linux/kernel.h>
+
+#define AD5683_DATA(x)				((x) << 4)
+#define AD5686_ADDR(x)				((x) << 16)
+#define AD5686_CMD(x)				((x) << 20)
+
+#define AD5686_ADDR_DAC(chan)			(0x1 << (chan))
+#define AD5686_ADDR_ALL_DAC			0xF
+
+#define AD5686_CMD_NOOP				0x0
+#define AD5686_CMD_WRITE_INPUT_N		0x1
+#define AD5686_CMD_UPDATE_DAC_N			0x2
+#define AD5686_CMD_WRITE_INPUT_N_UPDATE_N	0x3
+#define AD5686_CMD_POWERDOWN_DAC		0x4
+#define AD5686_CMD_LDAC_MASK			0x5
+#define AD5686_CMD_RESET			0x6
+#define AD5686_CMD_INTERNAL_REFER_SETUP		0x7
+#define AD5686_CMD_DAISY_CHAIN_ENABLE		0x8
+#define AD5686_CMD_READBACK_ENABLE		0x9
+
+#define AD5686_LDAC_PWRDN_NONE			0x0
+#define AD5686_LDAC_PWRDN_1K			0x1
+#define AD5686_LDAC_PWRDN_100K			0x2
+#define AD5686_LDAC_PWRDN_3STATE		0x3
+
+#define AD5686_CMD_CONTROL_REG			0x4
+#define AD5686_CMD_READBACK_ENABLE_V2		0x5
+#define AD5683_REF_BIT_MSK			BIT(12)
+#define AD5693_REF_BIT_MSK			BIT(12)
+
+/**
+ * ad5686_supported_device_ids:
+ */
+enum ad5686_supported_device_ids {
+	ID_AD5671R,
+	ID_AD5672R,
+	ID_AD5675R,
+	ID_AD5676,
+	ID_AD5676R,
+	ID_AD5681R,
+	ID_AD5682R,
+	ID_AD5683,
+	ID_AD5683R,
+	ID_AD5684,
+	ID_AD5684R,
+	ID_AD5685R,
+	ID_AD5686,
+	ID_AD5686R,
+	ID_AD5691R,
+	ID_AD5692R,
+	ID_AD5693,
+	ID_AD5693R,
+	ID_AD5694,
+	ID_AD5694R,
+	ID_AD5695R,
+	ID_AD5696,
+	ID_AD5696R,
+};
+
+enum ad5686_regmap_type {
+	AD5683_REGMAP,
+	AD5686_REGMAP,
+	AD5693_REGMAP
+};
+
+struct ad5686_state;
+
+typedef int (*ad5686_write_func)(struct ad5686_state *st,
+				 u8 cmd, u8 addr, u16 val);
+
+typedef int (*ad5686_read_func)(struct ad5686_state *st, u8 addr);
+
+/**
+ * struct ad5686_chip_info - chip specific information
+ * @int_vref_mv:	AD5620/40/60: the internal reference voltage
+ * @num_channels:	number of channels
+ * @channel:		channel specification
+ * @regmap_type:	register map layout variant
+ */
+
+struct ad5686_chip_info {
+	u16				int_vref_mv;
+	unsigned int			num_channels;
+	struct iio_chan_spec		*channels;
+	enum ad5686_regmap_type		regmap_type;
+};
+
+/**
+ * struct ad5446_state - driver instance specific data
+ * @spi:		spi_device
+ * @chip_info:		chip model specific constants, available modes etc
+ * @reg:		supply regulator
+ * @vref_mv:		actual reference voltage used
+ * @pwr_down_mask:	power down mask
+ * @pwr_down_mode:	current power down mode
+ * @use_internal_vref:	set to true if the internal reference voltage is used
+ * @data:		spi transfer buffers
+ */
+
+struct ad5686_state {
+	struct device			*dev;
+	const struct ad5686_chip_info	*chip_info;
+	struct regulator		*reg;
+	unsigned short			vref_mv;
+	unsigned int			pwr_down_mask;
+	unsigned int			pwr_down_mode;
+	ad5686_write_func		write;
+	ad5686_read_func		read;
+	bool				use_internal_vref;
+
+	/*
+	 * DMA (thus cache coherency maintenance) requires the
+	 * transfer buffers to live in their own cache lines.
+	 */
+
+	union {
+		__be32 d32;
+		__be16 d16;
+		u8 d8[4];
+	} data[3] ____cacheline_aligned;
+};
+
+
+int ad5686_probe(struct device *dev,
+		 enum ad5686_supported_device_ids chip_type,
+		 const char *name, ad5686_write_func write,
+		 ad5686_read_func read);
+
+int ad5686_remove(struct device *dev);
+
+
+#endif /* __DRIVERS_IIO_DAC_AD5686_H__ */
diff --git a/drivers/iio/dac/ad5696-i2c.c b/drivers/iio/dac/ad5696-i2c.c
new file mode 100644
index 000000000000..d18735d7d938
--- /dev/null
+++ b/drivers/iio/dac/ad5696-i2c.c
@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * AD5671R, AD5675R, AD5691R, AD5692R, AD5693, AD5693R,
+ * AD5694, AD5694R, AD5695R, AD5696, AD5696R
+ * Digital to analog converters driver
+ *
+ * Copyright 2018 Analog Devices Inc.
+ */
+
+#include "ad5686.h"
+
+#include <linux/module.h>
+#include <linux/i2c.h>
+
+static int ad5686_i2c_read(struct ad5686_state *st, u8 addr)
+{
+	struct i2c_client *i2c = to_i2c_client(st->dev);
+	struct i2c_msg msg[2] = {
+		{
+			.addr = i2c->addr,
+			.flags = i2c->flags,
+			.len = 3,
+			.buf = &st->data[0].d8[1],
+		},
+		{
+			.addr = i2c->addr,
+			.flags = i2c->flags | I2C_M_RD,
+			.len = 2,
+			.buf = (char *)&st->data[0].d16,
+		},
+	};
+	int ret;
+
+	st->data[0].d32 = cpu_to_be32(AD5686_CMD(AD5686_CMD_NOOP) |
+				      AD5686_ADDR(addr) |
+				      0x00);
+
+	ret = i2c_transfer(i2c->adapter, msg, 2);
+	if (ret < 0)
+		return ret;
+
+	return be16_to_cpu(st->data[0].d16);
+}
+
+static int ad5686_i2c_write(struct ad5686_state *st,
+			    u8 cmd, u8 addr, u16 val)
+{
+	struct i2c_client *i2c = to_i2c_client(st->dev);
+	int ret;
+
+	st->data[0].d32 = cpu_to_be32(AD5686_CMD(cmd) | AD5686_ADDR(addr)
+				      | val);
+
+	ret = i2c_master_send(i2c, &st->data[0].d8[1], 3);
+	if (ret < 0)
+		return ret;
+
+	return (ret != 3) ? -EIO : 0;
+}
+
+static int ad5686_i2c_probe(struct i2c_client *i2c,
+			    const struct i2c_device_id *id)
+{
+	return ad5686_probe(&i2c->dev, id->driver_data, id->name,
+			    ad5686_i2c_write, ad5686_i2c_read);
+}
+
+static int ad5686_i2c_remove(struct i2c_client *i2c)
+{
+	return ad5686_remove(&i2c->dev);
+}
+
+static const struct i2c_device_id ad5686_i2c_id[] = {
+	{"ad5671r", ID_AD5671R},
+	{"ad5675r", ID_AD5675R},
+	{"ad5691r", ID_AD5691R},
+	{"ad5692r", ID_AD5692R},
+	{"ad5693", ID_AD5693},
+	{"ad5693r", ID_AD5693R},
+	{"ad5694", ID_AD5694},
+	{"ad5694r", ID_AD5694R},
+	{"ad5695r", ID_AD5695R},
+	{"ad5696", ID_AD5696},
+	{"ad5696r", ID_AD5696R},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, ad5686_i2c_id);
+
+static struct i2c_driver ad5686_i2c_driver = {
+	.driver = {
+		.name = "ad5696",
+	},
+	.probe = ad5686_i2c_probe,
+	.remove = ad5686_i2c_remove,
+	.id_table = ad5686_i2c_id,
+};
+
+module_i2c_driver(ad5686_i2c_driver);
+
+MODULE_AUTHOR("Stefan Popa <stefan.popa@analog.com>");
+MODULE_DESCRIPTION("Analog Devices AD5686 and similar multi-channel DACs");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/dac/ltc2632.c b/drivers/iio/dac/ltc2632.c
index af2ddd0dd341..cca278eaa138 100644
--- a/drivers/iio/dac/ltc2632.c
+++ b/drivers/iio/dac/ltc2632.c
@@ -2,6 +2,7 @@
  * LTC2632 Digital to analog convertors spi driver
  *
  * Copyright 2017 Maxime Roussin-B�langer
+ * expanded by Silvan Murer <silvan.murer@gmail.com>
  *
  * Licensed under the GPL-2.
  */
@@ -10,6 +11,7 @@
 #include <linux/spi/spi.h>
 #include <linux/module.h>
 #include <linux/iio/iio.h>
+#include <linux/regulator/consumer.h>
 
 #define LTC2632_DAC_CHANNELS                    2
 
@@ -28,7 +30,7 @@
 /**
  * struct ltc2632_chip_info - chip specific information
  * @channels:		channel spec for the DAC
- * @vref_mv:		reference voltage
+ * @vref_mv:		internal reference voltage
  */
 struct ltc2632_chip_info {
 	const struct iio_chan_spec *channels;
@@ -39,10 +41,14 @@ struct ltc2632_chip_info {
  * struct ltc2632_state - driver instance specific data
  * @spi_dev:			pointer to the spi_device struct
  * @powerdown_cache_mask	used to show current channel powerdown state
+ * @vref_mv			used reference voltage (internal or external)
+ * @vref_reg		regulator for the reference voltage
  */
 struct ltc2632_state {
 	struct spi_device *spi_dev;
 	unsigned int powerdown_cache_mask;
+	int vref_mv;
+	struct regulator *vref_reg;
 };
 
 enum ltc2632_supported_device_ids {
@@ -90,7 +96,7 @@ static int ltc2632_read_raw(struct iio_dev *indio_dev,
 
 	switch (m) {
 	case IIO_CHAN_INFO_SCALE:
-		*val = chip_info->vref_mv;
+		*val = st->vref_mv;
 		*val2 = chan->scan_type.realbits;
 		return IIO_VAL_FRACTIONAL_LOG2;
 	}
@@ -246,6 +252,45 @@ static int ltc2632_probe(struct spi_device *spi)
 	chip_info = (struct ltc2632_chip_info *)
 			spi_get_device_id(spi)->driver_data;
 
+	st->vref_reg = devm_regulator_get_optional(&spi->dev, "vref");
+	if (PTR_ERR(st->vref_reg) == -ENODEV) {
+		/* use internal reference voltage */
+		st->vref_reg = NULL;
+		st->vref_mv = chip_info->vref_mv;
+
+		ret = ltc2632_spi_write(spi, LTC2632_CMD_INTERNAL_REFER,
+				0, 0, 0);
+		if (ret) {
+			dev_err(&spi->dev,
+				"Set internal reference command failed, %d\n",
+				ret);
+			return ret;
+		}
+	} else if (IS_ERR(st->vref_reg)) {
+		dev_err(&spi->dev,
+				"Error getting voltage reference regulator\n");
+		return PTR_ERR(st->vref_reg);
+	} else {
+		/* use external reference voltage */
+		ret = regulator_enable(st->vref_reg);
+		if (ret) {
+			dev_err(&spi->dev,
+				"enable reference regulator failed, %d\n",
+				ret);
+			return ret;
+		}
+		st->vref_mv = regulator_get_voltage(st->vref_reg) / 1000;
+
+		ret = ltc2632_spi_write(spi, LTC2632_CMD_EXTERNAL_REFER,
+				0, 0, 0);
+		if (ret) {
+			dev_err(&spi->dev,
+				"Set external reference command failed, %d\n",
+				ret);
+			return ret;
+		}
+	}
+
 	indio_dev->dev.parent = &spi->dev;
 	indio_dev->name = dev_of_node(&spi->dev) ? dev_of_node(&spi->dev)->name
 						 : spi_get_device_id(spi)->name;
@@ -254,14 +299,20 @@ static int ltc2632_probe(struct spi_device *spi)
 	indio_dev->channels = chip_info->channels;
 	indio_dev->num_channels = LTC2632_DAC_CHANNELS;
 
-	ret = ltc2632_spi_write(spi, LTC2632_CMD_INTERNAL_REFER, 0, 0, 0);
-	if (ret) {
-		dev_err(&spi->dev,
-			"Set internal reference command failed, %d\n", ret);
-		return ret;
-	}
+	return iio_device_register(indio_dev);
+}
 
-	return devm_iio_device_register(&spi->dev, indio_dev);
+static int ltc2632_remove(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev = spi_get_drvdata(spi);
+	struct ltc2632_state *st = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+
+	if (st->vref_reg)
+		regulator_disable(st->vref_reg);
+
+	return 0;
 }
 
 static const struct spi_device_id ltc2632_id[] = {
@@ -275,15 +326,6 @@ static const struct spi_device_id ltc2632_id[] = {
 };
 MODULE_DEVICE_TABLE(spi, ltc2632_id);
 
-static struct spi_driver ltc2632_driver = {
-	.driver		= {
-		.name	= "ltc2632",
-	},
-	.probe		= ltc2632_probe,
-	.id_table	= ltc2632_id,
-};
-module_spi_driver(ltc2632_driver);
-
 static const struct of_device_id ltc2632_of_match[] = {
 	{
 		.compatible = "lltc,ltc2632-l12",
@@ -308,6 +350,17 @@ static const struct of_device_id ltc2632_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, ltc2632_of_match);
 
+static struct spi_driver ltc2632_driver = {
+	.driver		= {
+		.name	= "ltc2632",
+		.of_match_table = of_match_ptr(ltc2632_of_match),
+	},
+	.probe		= ltc2632_probe,
+	.remove		= ltc2632_remove,
+	.id_table	= ltc2632_id,
+};
+module_spi_driver(ltc2632_driver);
+
 MODULE_AUTHOR("Maxime Roussin-Belanger <maxime.roussinbelanger@gmail.com>");
 MODULE_DESCRIPTION("LTC2632 DAC SPI driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/dac/ti-dac5571.c b/drivers/iio/dac/ti-dac5571.c
new file mode 100644
index 000000000000..dd21eebed6a8
--- /dev/null
+++ b/drivers/iio/dac/ti-dac5571.c
@@ -0,0 +1,439 @@
+/*
+ * ti-dac5571.c - Texas Instruments 8/10/12-bit 1/4-channel DAC driver
+ *
+ * Copyright (C) 2018 Prevas A/S
+ *
+ * http://www.ti.com/lit/ds/symlink/dac5571.pdf
+ * http://www.ti.com/lit/ds/symlink/dac6571.pdf
+ * http://www.ti.com/lit/ds/symlink/dac7571.pdf
+ * http://www.ti.com/lit/ds/symlink/dac5574.pdf
+ * http://www.ti.com/lit/ds/symlink/dac6574.pdf
+ * http://www.ti.com/lit/ds/symlink/dac7574.pdf
+ * http://www.ti.com/lit/ds/symlink/dac5573.pdf
+ * http://www.ti.com/lit/ds/symlink/dac6573.pdf
+ * http://www.ti.com/lit/ds/symlink/dac7573.pdf
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License (version 2) as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/iio/iio.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+
+enum chip_id {
+	single_8bit, single_10bit, single_12bit,
+	quad_8bit, quad_10bit, quad_12bit
+};
+
+struct dac5571_spec {
+	u8 num_channels;
+	u8 resolution;
+};
+
+static const struct dac5571_spec dac5571_spec[] = {
+	[single_8bit]  = {.num_channels = 1, .resolution =  8},
+	[single_10bit] = {.num_channels = 1, .resolution = 10},
+	[single_12bit] = {.num_channels = 1, .resolution = 12},
+	[quad_8bit]    = {.num_channels = 4, .resolution =  8},
+	[quad_10bit]   = {.num_channels = 4, .resolution = 10},
+	[quad_12bit]   = {.num_channels = 4, .resolution = 12},
+};
+
+struct dac5571_data {
+	struct i2c_client *client;
+	int id;
+	struct mutex lock;
+	struct regulator *vref;
+	u16 val[4];
+	bool powerdown;
+	u8 powerdown_mode;
+	struct dac5571_spec const *spec;
+	int (*dac5571_cmd)(struct dac5571_data *data, int channel, u16 val);
+	int (*dac5571_pwrdwn)(struct dac5571_data *data, int channel, u8 pwrdwn);
+	u8 buf[3] ____cacheline_aligned;
+};
+
+#define DAC5571_POWERDOWN(mode)		((mode) + 1)
+#define DAC5571_POWERDOWN_FLAG		BIT(0)
+#define DAC5571_CHANNEL_SELECT		1
+#define DAC5571_LOADMODE_DIRECT		BIT(4)
+#define DAC5571_SINGLE_PWRDWN_BITS	4
+#define DAC5571_QUAD_PWRDWN_BITS	6
+
+static int dac5571_cmd_single(struct dac5571_data *data, int channel, u16 val)
+{
+	unsigned int shift;
+
+	shift = 12 - data->spec->resolution;
+	data->buf[1] = val << shift;
+	data->buf[0] = val >> (8 - shift);
+
+	if (i2c_master_send(data->client, data->buf, 2) != 2)
+		return -EIO;
+
+	return 0;
+}
+
+static int dac5571_cmd_quad(struct dac5571_data *data, int channel, u16 val)
+{
+	unsigned int shift;
+
+	shift = 16 - data->spec->resolution;
+	data->buf[2] = val << shift;
+	data->buf[1] = (val >> (8 - shift));
+	data->buf[0] = (channel << DAC5571_CHANNEL_SELECT) |
+		       DAC5571_LOADMODE_DIRECT;
+
+	if (i2c_master_send(data->client, data->buf, 3) != 3)
+		return -EIO;
+
+	return 0;
+}
+
+static int dac5571_pwrdwn_single(struct dac5571_data *data, int channel, u8 pwrdwn)
+{
+	unsigned int shift;
+
+	shift = 12 - data->spec->resolution;
+	data->buf[1] = 0;
+	data->buf[0] = pwrdwn << DAC5571_SINGLE_PWRDWN_BITS;
+
+	if (i2c_master_send(data->client, data->buf, 2) != 2)
+		return -EIO;
+
+	return 0;
+}
+
+static int dac5571_pwrdwn_quad(struct dac5571_data *data, int channel, u8 pwrdwn)
+{
+	unsigned int shift;
+
+	shift = 16 - data->spec->resolution;
+	data->buf[2] = 0;
+	data->buf[1] = pwrdwn << DAC5571_QUAD_PWRDWN_BITS;
+	data->buf[0] = (channel << DAC5571_CHANNEL_SELECT) |
+		       DAC5571_LOADMODE_DIRECT | DAC5571_POWERDOWN_FLAG;
+
+	if (i2c_master_send(data->client, data->buf, 3) != 3)
+		return -EIO;
+
+	return 0;
+}
+
+static const char *const dac5571_powerdown_modes[] = {
+	"1kohm_to_gnd", "100kohm_to_gnd", "three_state",
+};
+
+static int dac5571_get_powerdown_mode(struct iio_dev *indio_dev,
+				      const struct iio_chan_spec *chan)
+{
+	struct dac5571_data *data = iio_priv(indio_dev);
+
+	return data->powerdown_mode;
+}
+
+static int dac5571_set_powerdown_mode(struct iio_dev *indio_dev,
+				      const struct iio_chan_spec *chan,
+				      unsigned int mode)
+{
+	struct dac5571_data *data = iio_priv(indio_dev);
+	int ret = 0;
+
+	if (data->powerdown_mode == mode)
+		return 0;
+
+	mutex_lock(&data->lock);
+	if (data->powerdown) {
+		ret = data->dac5571_pwrdwn(data, chan->channel,
+					   DAC5571_POWERDOWN(mode));
+		if (ret)
+			goto out;
+	}
+	data->powerdown_mode = mode;
+
+ out:
+	mutex_unlock(&data->lock);
+
+	return ret;
+}
+
+static const struct iio_enum dac5571_powerdown_mode = {
+	.items = dac5571_powerdown_modes,
+	.num_items = ARRAY_SIZE(dac5571_powerdown_modes),
+	.get = dac5571_get_powerdown_mode,
+	.set = dac5571_set_powerdown_mode,
+};
+
+static ssize_t dac5571_read_powerdown(struct iio_dev *indio_dev,
+				      uintptr_t private,
+				      const struct iio_chan_spec *chan,
+				      char *buf)
+{
+	struct dac5571_data *data = iio_priv(indio_dev);
+
+	return sprintf(buf, "%d\n", data->powerdown);
+}
+
+static ssize_t dac5571_write_powerdown(struct iio_dev *indio_dev,
+				       uintptr_t private,
+				       const struct iio_chan_spec *chan,
+				       const char *buf, size_t len)
+{
+	struct dac5571_data *data = iio_priv(indio_dev);
+	bool powerdown;
+	int ret;
+
+	ret = strtobool(buf, &powerdown);
+	if (ret)
+		return ret;
+
+	if (data->powerdown == powerdown)
+		return len;
+
+	mutex_lock(&data->lock);
+	if (powerdown)
+		ret = data->dac5571_pwrdwn(data, chan->channel,
+			    DAC5571_POWERDOWN(data->powerdown_mode));
+	else
+		ret = data->dac5571_cmd(data, chan->channel, data->val[0]);
+	if (ret)
+		goto out;
+
+	data->powerdown = powerdown;
+
+ out:
+	mutex_unlock(&data->lock);
+
+	return ret ? ret : len;
+}
+
+
+static const struct iio_chan_spec_ext_info dac5571_ext_info[] = {
+	{
+		.name	   = "powerdown",
+		.read	   = dac5571_read_powerdown,
+		.write	   = dac5571_write_powerdown,
+		.shared	   = IIO_SHARED_BY_TYPE,
+	},
+	IIO_ENUM("powerdown_mode", IIO_SHARED_BY_TYPE, &dac5571_powerdown_mode),
+	IIO_ENUM_AVAILABLE("powerdown_mode", &dac5571_powerdown_mode),
+	{},
+};
+
+#define dac5571_CHANNEL(chan, name) {				\
+	.type = IIO_VOLTAGE,					\
+	.channel = (chan),					\
+	.address = (chan),					\
+	.indexed = true,					\
+	.output = true,						\
+	.datasheet_name = name,					\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+	.ext_info = dac5571_ext_info,				\
+}
+
+static const struct iio_chan_spec dac5571_channels[] = {
+	dac5571_CHANNEL(0, "A"),
+	dac5571_CHANNEL(1, "B"),
+	dac5571_CHANNEL(2, "C"),
+	dac5571_CHANNEL(3, "D"),
+};
+
+static int dac5571_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
+{
+	struct dac5571_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		*val = data->val[chan->channel];
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_SCALE:
+		ret = regulator_get_voltage(data->vref);
+		if (ret < 0)
+			return ret;
+
+		*val = ret / 1000;
+		*val2 = data->spec->resolution;
+		return IIO_VAL_FRACTIONAL_LOG2;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int dac5571_write_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
+{
+	struct dac5571_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		if (data->val[chan->channel] == val)
+			return 0;
+
+		if (val >= (1 << data->spec->resolution) || val < 0)
+			return -EINVAL;
+
+		if (data->powerdown)
+			return -EBUSY;
+
+		mutex_lock(&data->lock);
+		ret = data->dac5571_cmd(data, chan->channel, val);
+		if (ret == 0)
+			data->val[chan->channel] = val;
+		mutex_unlock(&data->lock);
+		return ret;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int dac5571_write_raw_get_fmt(struct iio_dev *indio_dev,
+				     struct iio_chan_spec const *chan,
+				     long mask)
+{
+	return IIO_VAL_INT;
+}
+
+static const struct iio_info dac5571_info = {
+	.read_raw = dac5571_read_raw,
+	.write_raw = dac5571_write_raw,
+	.write_raw_get_fmt = dac5571_write_raw_get_fmt,
+};
+
+static int dac5571_probe(struct i2c_client *client,
+			 const struct i2c_device_id *id)
+{
+	struct device *dev = &client->dev;
+	const struct dac5571_spec *spec;
+	struct dac5571_data *data;
+	struct iio_dev *indio_dev;
+	int ret, i;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	i2c_set_clientdata(client, indio_dev);
+	data->client = client;
+
+	indio_dev->dev.parent = dev;
+	indio_dev->dev.of_node = client->dev.of_node;
+	indio_dev->info = &dac5571_info;
+	indio_dev->name = id->name;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = dac5571_channels;
+
+	spec = &dac5571_spec[id->driver_data];
+	indio_dev->num_channels = spec->num_channels;
+	data->spec = spec;
+
+	data->vref = devm_regulator_get(dev, "vref");
+	if (IS_ERR(data->vref))
+		return PTR_ERR(data->vref);
+
+	ret = regulator_enable(data->vref);
+	if (ret < 0)
+		return ret;
+
+	mutex_init(&data->lock);
+
+	switch (spec->num_channels) {
+	case 1:
+		data->dac5571_cmd = dac5571_cmd_single;
+		data->dac5571_pwrdwn = dac5571_pwrdwn_single;
+		break;
+	case 4:
+		data->dac5571_cmd = dac5571_cmd_quad;
+		data->dac5571_pwrdwn = dac5571_pwrdwn_quad;
+		break;
+	default:
+		goto err;
+	}
+
+	for (i = 0; i < spec->num_channels; i++) {
+		ret = data->dac5571_cmd(data, i, 0);
+		if (ret) {
+			dev_err(dev, "failed to initialize channel %d to 0\n", i);
+			goto err;
+		}
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret)
+		goto err;
+
+	return 0;
+
+ err:
+	regulator_disable(data->vref);
+	return ret;
+}
+
+static int dac5571_remove(struct i2c_client *i2c)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(i2c);
+	struct dac5571_data *data = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+	regulator_disable(data->vref);
+
+	return 0;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id dac5571_of_id[] = {
+	{.compatible = "ti,dac5571"},
+	{.compatible = "ti,dac6571"},
+	{.compatible = "ti,dac7571"},
+	{.compatible = "ti,dac5574"},
+	{.compatible = "ti,dac6574"},
+	{.compatible = "ti,dac7574"},
+	{.compatible = "ti,dac5573"},
+	{.compatible = "ti,dac6573"},
+	{.compatible = "ti,dac7573"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, dac5571_of_id);
+#endif
+
+static const struct i2c_device_id dac5571_id[] = {
+	{"dac5571", single_8bit},
+	{"dac6571", single_10bit},
+	{"dac7571", single_12bit},
+	{"dac5574", quad_8bit},
+	{"dac6574", quad_10bit},
+	{"dac7574", quad_12bit},
+	{"dac5573", quad_8bit},
+	{"dac6573", quad_10bit},
+	{"dac7573", quad_12bit},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, dac5571_id);
+
+static struct i2c_driver dac5571_driver = {
+	.driver = {
+		   .name = "ti-dac5571",
+	},
+	.probe	  = dac5571_probe,
+	.remove   = dac5571_remove,
+	.id_table = dac5571_id,
+};
+module_i2c_driver(dac5571_driver);
+
+MODULE_AUTHOR("Sean Nyekjaer <sean.nyekjaer@prevas.dk>");
+MODULE_DESCRIPTION("Texas Instruments 8/10/12-bit 1/4-channel DAC driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c
index 7d64be353403..f9c0624505a2 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c
@@ -24,6 +24,7 @@
 #include <linux/spinlock.h>
 #include <linux/iio/iio.h>
 #include <linux/acpi.h>
+#include <linux/platform_device.h>
 #include "inv_mpu_iio.h"
 
 /*
@@ -52,6 +53,7 @@ static const struct inv_mpu6050_reg_map reg_set_6500 = {
 	.raw_accl               = INV_MPU6050_REG_RAW_ACCEL,
 	.temperature            = INV_MPU6050_REG_TEMPERATURE,
 	.int_enable             = INV_MPU6050_REG_INT_ENABLE,
+	.int_status             = INV_MPU6050_REG_INT_STATUS,
 	.pwr_mgmt_1             = INV_MPU6050_REG_PWR_MGMT_1,
 	.pwr_mgmt_2             = INV_MPU6050_REG_PWR_MGMT_2,
 	.int_pin_cfg		= INV_MPU6050_REG_INT_PIN_CFG,
@@ -86,6 +88,7 @@ static const struct inv_mpu6050_chip_config chip_config_6050 = {
 	.gyro_fifo_enable = false,
 	.accl_fifo_enable = false,
 	.accl_fs = INV_MPU6050_FS_02G,
+	.user_ctrl = 0,
 };
 
 /* Indexed by enum inv_devices */
@@ -120,6 +123,12 @@ static const struct inv_mpu6050_hw hw_info[] = {
 		.reg = &reg_set_6500,
 		.config = &chip_config_6050,
 	},
+	{
+		.whoami = INV_MPU9255_WHOAMI_VALUE,
+		.name = "MPU9255",
+		.reg = &reg_set_6500,
+		.config = &chip_config_6050,
+	},
 	{
 		.whoami = INV_ICM20608_WHOAMI_VALUE,
 		.name = "ICM20608",
@@ -168,7 +177,7 @@ int inv_mpu6050_switch_engine(struct inv_mpu6050_state *st, bool en, u32 mask)
 		return result;
 
 	if (en) {
-		/* Wait for output stabilize */
+		/* Wait for output to stabilize */
 		msleep(INV_MPU6050_TEMP_UP_TIME);
 		if (mask == INV_MPU6050_BIT_PWR_GYRO_STBY) {
 			/* switch internal clock to PLL */
@@ -185,26 +194,29 @@ int inv_mpu6050_switch_engine(struct inv_mpu6050_state *st, bool en, u32 mask)
 
 int inv_mpu6050_set_power_itg(struct inv_mpu6050_state *st, bool power_on)
 {
-	int result = 0;
+	int result;
 
 	if (power_on) {
-		if (!st->powerup_count)
+		if (!st->powerup_count) {
 			result = regmap_write(st->map, st->reg->pwr_mgmt_1, 0);
-		if (!result)
-			st->powerup_count++;
+			if (result)
+				return result;
+			usleep_range(INV_MPU6050_REG_UP_TIME_MIN,
+				     INV_MPU6050_REG_UP_TIME_MAX);
+		}
+		st->powerup_count++;
 	} else {
-		st->powerup_count--;
-		if (!st->powerup_count)
+		if (st->powerup_count == 1) {
 			result = regmap_write(st->map, st->reg->pwr_mgmt_1,
 					      INV_MPU6050_BIT_SLEEP);
+			if (result)
+				return result;
+		}
+		st->powerup_count--;
 	}
 
-	if (result)
-		return result;
-
-	if (power_on)
-		usleep_range(INV_MPU6050_REG_UP_TIME_MIN,
-			     INV_MPU6050_REG_UP_TIME_MAX);
+	dev_dbg(regmap_get_device(st->map), "set power %d, count=%u\n",
+		power_on, st->powerup_count);
 
 	return 0;
 }
@@ -262,26 +274,33 @@ static int inv_mpu6050_init_config(struct iio_dev *indio_dev)
 	d = (INV_MPU6050_FSR_2000DPS << INV_MPU6050_GYRO_CONFIG_FSR_SHIFT);
 	result = regmap_write(st->map, st->reg->gyro_config, d);
 	if (result)
-		return result;
+		goto error_power_off;
 
 	result = inv_mpu6050_set_lpf_regs(st, INV_MPU6050_FILTER_20HZ);
 	if (result)
-		return result;
+		goto error_power_off;
 
 	d = INV_MPU6050_ONE_K_HZ / INV_MPU6050_INIT_FIFO_RATE - 1;
 	result = regmap_write(st->map, st->reg->sample_rate_div, d);
 	if (result)
-		return result;
+		goto error_power_off;
 
 	d = (INV_MPU6050_FS_02G << INV_MPU6050_ACCL_CONFIG_FSR_SHIFT);
 	result = regmap_write(st->map, st->reg->accl_config, d);
+	if (result)
+		goto error_power_off;
+
+	result = regmap_write(st->map, st->reg->int_pin_cfg, st->irq_mask);
 	if (result)
 		return result;
 
 	memcpy(&st->chip_config, hw_info[st->chip_type].config,
 	       sizeof(struct inv_mpu6050_chip_config));
-	result = inv_mpu6050_set_power_itg(st, false);
 
+	return inv_mpu6050_set_power_itg(st, false);
+
+error_power_off:
+	inv_mpu6050_set_power_itg(st, false);
 	return result;
 }
 
@@ -314,6 +333,65 @@ static int inv_mpu6050_sensor_show(struct inv_mpu6050_state  *st, int reg,
 	return IIO_VAL_INT;
 }
 
+static int inv_mpu6050_read_channel_data(struct iio_dev *indio_dev,
+					 struct iio_chan_spec const *chan,
+					 int *val)
+{
+	struct inv_mpu6050_state *st = iio_priv(indio_dev);
+	int result;
+	int ret;
+
+	result = inv_mpu6050_set_power_itg(st, true);
+	if (result)
+		return result;
+
+	switch (chan->type) {
+	case IIO_ANGL_VEL:
+		result = inv_mpu6050_switch_engine(st, true,
+				INV_MPU6050_BIT_PWR_GYRO_STBY);
+		if (result)
+			goto error_power_off;
+		ret = inv_mpu6050_sensor_show(st, st->reg->raw_gyro,
+					      chan->channel2, val);
+		result = inv_mpu6050_switch_engine(st, false,
+				INV_MPU6050_BIT_PWR_GYRO_STBY);
+		if (result)
+			goto error_power_off;
+		break;
+	case IIO_ACCEL:
+		result = inv_mpu6050_switch_engine(st, true,
+				INV_MPU6050_BIT_PWR_ACCL_STBY);
+		if (result)
+			goto error_power_off;
+		ret = inv_mpu6050_sensor_show(st, st->reg->raw_accl,
+					      chan->channel2, val);
+		result = inv_mpu6050_switch_engine(st, false,
+				INV_MPU6050_BIT_PWR_ACCL_STBY);
+		if (result)
+			goto error_power_off;
+		break;
+	case IIO_TEMP:
+		/* wait for stablization */
+		msleep(INV_MPU6050_SENSOR_UP_TIME);
+		ret = inv_mpu6050_sensor_show(st, st->reg->temperature,
+					      IIO_MOD_X, val);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	result = inv_mpu6050_set_power_itg(st, false);
+	if (result)
+		goto error_power_off;
+
+	return ret;
+
+error_power_off:
+	inv_mpu6050_set_power_itg(st, false);
+	return result;
+}
+
 static int
 inv_mpu6050_read_raw(struct iio_dev *indio_dev,
 		     struct iio_chan_spec const *chan,
@@ -324,63 +402,14 @@ inv_mpu6050_read_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-	{
-		int result;
-
-		ret = IIO_VAL_INT;
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
 		mutex_lock(&st->lock);
-		result = iio_device_claim_direct_mode(indio_dev);
-		if (result)
-			goto error_read_raw_unlock;
-		result = inv_mpu6050_set_power_itg(st, true);
-		if (result)
-			goto error_read_raw_release;
-		switch (chan->type) {
-		case IIO_ANGL_VEL:
-			result = inv_mpu6050_switch_engine(st, true,
-					INV_MPU6050_BIT_PWR_GYRO_STBY);
-			if (result)
-				goto error_read_raw_power_off;
-			ret = inv_mpu6050_sensor_show(st, st->reg->raw_gyro,
-						      chan->channel2, val);
-			result = inv_mpu6050_switch_engine(st, false,
-					INV_MPU6050_BIT_PWR_GYRO_STBY);
-			if (result)
-				goto error_read_raw_power_off;
-			break;
-		case IIO_ACCEL:
-			result = inv_mpu6050_switch_engine(st, true,
-					INV_MPU6050_BIT_PWR_ACCL_STBY);
-			if (result)
-				goto error_read_raw_power_off;
-			ret = inv_mpu6050_sensor_show(st, st->reg->raw_accl,
-						      chan->channel2, val);
-			result = inv_mpu6050_switch_engine(st, false,
-					INV_MPU6050_BIT_PWR_ACCL_STBY);
-			if (result)
-				goto error_read_raw_power_off;
-			break;
-		case IIO_TEMP:
-			/* wait for stablization */
-			msleep(INV_MPU6050_SENSOR_UP_TIME);
-			ret = inv_mpu6050_sensor_show(st, st->reg->temperature,
-						IIO_MOD_X, val);
-			break;
-		default:
-			ret = -EINVAL;
-			break;
-		}
-error_read_raw_power_off:
-		result |= inv_mpu6050_set_power_itg(st, false);
-error_read_raw_release:
-		iio_device_release_direct_mode(indio_dev);
-error_read_raw_unlock:
+		ret = inv_mpu6050_read_channel_data(indio_dev, chan, val);
 		mutex_unlock(&st->lock);
-		if (result)
-			return result;
-
+		iio_device_release_direct_mode(indio_dev);
 		return ret;
-	}
 	case IIO_CHAN_INFO_SCALE:
 		switch (chan->type) {
 		case IIO_ANGL_VEL:
@@ -502,17 +531,18 @@ static int inv_mpu6050_write_raw(struct iio_dev *indio_dev,
 	struct inv_mpu6050_state  *st = iio_priv(indio_dev);
 	int result;
 
-	mutex_lock(&st->lock);
 	/*
 	 * we should only update scale when the chip is disabled, i.e.
 	 * not running
 	 */
 	result = iio_device_claim_direct_mode(indio_dev);
 	if (result)
-		goto error_write_raw_unlock;
+		return result;
+
+	mutex_lock(&st->lock);
 	result = inv_mpu6050_set_power_itg(st, true);
 	if (result)
-		goto error_write_raw_release;
+		goto error_write_raw_unlock;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_SCALE:
@@ -551,10 +581,9 @@ static int inv_mpu6050_write_raw(struct iio_dev *indio_dev,
 	}
 
 	result |= inv_mpu6050_set_power_itg(st, false);
-error_write_raw_release:
-	iio_device_release_direct_mode(indio_dev);
 error_write_raw_unlock:
 	mutex_unlock(&st->lock);
+	iio_device_release_direct_mode(indio_dev);
 
 	return result;
 }
@@ -613,17 +642,18 @@ inv_mpu6050_fifo_rate_store(struct device *dev, struct device_attribute *attr,
 	    fifo_rate > INV_MPU6050_MAX_FIFO_RATE)
 		return -EINVAL;
 
+	result = iio_device_claim_direct_mode(indio_dev);
+	if (result)
+		return result;
+
 	mutex_lock(&st->lock);
 	if (fifo_rate == st->chip_config.fifo_rate) {
 		result = 0;
 		goto fifo_rate_fail_unlock;
 	}
-	result = iio_device_claim_direct_mode(indio_dev);
-	if (result)
-		goto fifo_rate_fail_unlock;
 	result = inv_mpu6050_set_power_itg(st, true);
 	if (result)
-		goto fifo_rate_fail_release;
+		goto fifo_rate_fail_unlock;
 
 	d = INV_MPU6050_ONE_K_HZ / fifo_rate - 1;
 	result = regmap_write(st->map, st->reg->sample_rate_div, d);
@@ -637,10 +667,9 @@ inv_mpu6050_fifo_rate_store(struct device *dev, struct device_attribute *attr,
 
 fifo_rate_fail_power_off:
 	result |= inv_mpu6050_set_power_itg(st, false);
-fifo_rate_fail_release:
-	iio_device_release_direct_mode(indio_dev);
 fifo_rate_fail_unlock:
 	mutex_unlock(&st->lock);
+	iio_device_release_direct_mode(indio_dev);
 	if (result)
 		return result;
 
@@ -769,7 +798,14 @@ static const struct iio_chan_spec inv_mpu_channels[] = {
 	INV_MPU6050_CHAN(IIO_ACCEL, IIO_MOD_Z, INV_MPU6050_SCAN_ACCL_Z),
 };
 
-/* constant IIO attribute */
+/*
+ * The user can choose any frequency between INV_MPU6050_MIN_FIFO_RATE and
+ * INV_MPU6050_MAX_FIFO_RATE, but only these frequencies are matched by the
+ * low-pass filter. Specifically, each of these sampling rates are about twice
+ * the bandwidth of a corresponding low-pass filter, which should eliminate
+ * aliasing following the Nyquist principle. By picking a frequency different
+ * from these, the user risks aliasing effects.
+ */
 static IIO_CONST_ATTR_SAMP_FREQ_AVAIL("10 20 50 100 200 500");
 static IIO_CONST_ATTR(in_anglvel_scale_available,
 					  "0.000133090 0.000266181 0.000532362 0.001064724");
@@ -850,14 +886,11 @@ static int inv_check_and_setup_chip(struct inv_mpu6050_state *st)
 	msleep(INV_MPU6050_POWER_UP_TIME);
 
 	/*
-	 * toggle power state. After reset, the sleep bit could be on
-	 * or off depending on the OTP settings. Toggling power would
+	 * Turn power on. After reset, the sleep bit could be on
+	 * or off depending on the OTP settings. Turning power on
 	 * make it in a definite state as well as making the hardware
 	 * state align with the software state
 	 */
-	result = inv_mpu6050_set_power_itg(st, false);
-	if (result)
-		return result;
 	result = inv_mpu6050_set_power_itg(st, true);
 	if (result)
 		return result;
@@ -865,13 +898,17 @@ static int inv_check_and_setup_chip(struct inv_mpu6050_state *st)
 	result = inv_mpu6050_switch_engine(st, false,
 					   INV_MPU6050_BIT_PWR_ACCL_STBY);
 	if (result)
-		return result;
+		goto error_power_off;
 	result = inv_mpu6050_switch_engine(st, false,
 					   INV_MPU6050_BIT_PWR_GYRO_STBY);
 	if (result)
-		return result;
+		goto error_power_off;
 
-	return 0;
+	return inv_mpu6050_set_power_itg(st, false);
+
+error_power_off:
+	inv_mpu6050_set_power_itg(st, false);
+	return result;
 }
 
 int inv_mpu_core_probe(struct regmap *regmap, int irq, const char *name,
@@ -882,6 +919,8 @@ int inv_mpu_core_probe(struct regmap *regmap, int irq, const char *name,
 	struct inv_mpu6050_platform_data *pdata;
 	struct device *dev = regmap_get_device(regmap);
 	int result;
+	struct irq_data *desc;
+	int irq_type;
 
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
@@ -913,20 +952,43 @@ int inv_mpu_core_probe(struct regmap *regmap, int irq, const char *name,
 		st->plat_data = *pdata;
 	}
 
+	desc = irq_get_irq_data(irq);
+	if (!desc) {
+		dev_err(dev, "Could not find IRQ %d\n", irq);
+		return -EINVAL;
+	}
+
+	irq_type = irqd_get_trigger_type(desc);
+	if (irq_type == IRQF_TRIGGER_RISING)
+		st->irq_mask = INV_MPU6050_ACTIVE_HIGH;
+	else if (irq_type == IRQF_TRIGGER_FALLING)
+		st->irq_mask = INV_MPU6050_ACTIVE_LOW;
+	else if (irq_type == IRQF_TRIGGER_HIGH)
+		st->irq_mask = INV_MPU6050_ACTIVE_HIGH |
+			INV_MPU6050_LATCH_INT_EN;
+	else if (irq_type == IRQF_TRIGGER_LOW)
+		st->irq_mask = INV_MPU6050_ACTIVE_LOW |
+			INV_MPU6050_LATCH_INT_EN;
+	else {
+		dev_err(dev, "Invalid interrupt type 0x%x specified\n",
+			irq_type);
+		return -EINVAL;
+	}
+
 	/* power is turned on inside check chip type*/
 	result = inv_check_and_setup_chip(st);
 	if (result)
 		return result;
 
-	if (inv_mpu_bus_setup)
-		inv_mpu_bus_setup(indio_dev);
-
 	result = inv_mpu6050_init_config(indio_dev);
 	if (result) {
 		dev_err(dev, "Could not initialize device.\n");
 		return result;
 	}
 
+	if (inv_mpu_bus_setup)
+		inv_mpu_bus_setup(indio_dev);
+
 	dev_set_drvdata(dev, indio_dev);
 	indio_dev->dev.parent = dev;
 	/* name will be NULL when enumerated via ACPI */
@@ -940,50 +1002,32 @@ int inv_mpu_core_probe(struct regmap *regmap, int irq, const char *name,
 	indio_dev->info = &mpu_info;
 	indio_dev->modes = INDIO_BUFFER_TRIGGERED;
 
-	result = iio_triggered_buffer_setup(indio_dev,
-					    inv_mpu6050_irq_handler,
-					    inv_mpu6050_read_fifo,
-					    NULL);
+	result = devm_iio_triggered_buffer_setup(dev, indio_dev,
+						 inv_mpu6050_irq_handler,
+						 inv_mpu6050_read_fifo,
+						 NULL);
 	if (result) {
 		dev_err(dev, "configure buffer fail %d\n", result);
 		return result;
 	}
-	result = inv_mpu6050_probe_trigger(indio_dev);
+	result = inv_mpu6050_probe_trigger(indio_dev, irq_type);
 	if (result) {
 		dev_err(dev, "trigger probe fail %d\n", result);
-		goto out_unreg_ring;
+		return result;
 	}
 
 	INIT_KFIFO(st->timestamps);
 	spin_lock_init(&st->time_stamp_lock);
-	result = iio_device_register(indio_dev);
+	result = devm_iio_device_register(dev, indio_dev);
 	if (result) {
 		dev_err(dev, "IIO register fail %d\n", result);
-		goto out_remove_trigger;
+		return result;
 	}
 
 	return 0;
-
-out_remove_trigger:
-	inv_mpu6050_remove_trigger(st);
-out_unreg_ring:
-	iio_triggered_buffer_cleanup(indio_dev);
-	return result;
 }
 EXPORT_SYMBOL_GPL(inv_mpu_core_probe);
 
-int inv_mpu_core_remove(struct device  *dev)
-{
-	struct iio_dev *indio_dev = dev_get_drvdata(dev);
-
-	iio_device_unregister(indio_dev);
-	inv_mpu6050_remove_trigger(iio_priv(indio_dev));
-	iio_triggered_buffer_cleanup(indio_dev);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(inv_mpu_core_remove);
-
 #ifdef CONFIG_PM_SLEEP
 
 static int inv_mpu_resume(struct device *dev)
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
index fcd7a92b6cf8..495409d56207 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
@@ -29,25 +29,18 @@ static int inv_mpu6050_select_bypass(struct i2c_mux_core *muxc, u32 chan_id)
 {
 	struct iio_dev *indio_dev = i2c_mux_priv(muxc);
 	struct inv_mpu6050_state *st = iio_priv(indio_dev);
-	int ret = 0;
+	int ret;
 
-	/* Use the same mutex which was used everywhere to protect power-op */
 	mutex_lock(&st->lock);
-	if (!st->powerup_count) {
-		ret = regmap_write(st->map, st->reg->pwr_mgmt_1, 0);
-		if (ret)
-			goto write_error;
 
-		usleep_range(INV_MPU6050_REG_UP_TIME_MIN,
-			     INV_MPU6050_REG_UP_TIME_MAX);
-	}
-	if (!ret) {
-		st->powerup_count++;
-		ret = regmap_write(st->map, st->reg->int_pin_cfg,
-				   INV_MPU6050_INT_PIN_CFG |
-				   INV_MPU6050_BIT_BYPASS_EN);
-	}
-write_error:
+	ret = inv_mpu6050_set_power_itg(st, true);
+	if (ret)
+		goto error_unlock;
+
+	ret = regmap_write(st->map, st->reg->int_pin_cfg,
+			   st->irq_mask | INV_MPU6050_BIT_BYPASS_EN);
+
+error_unlock:
 	mutex_unlock(&st->lock);
 
 	return ret;
@@ -59,12 +52,11 @@ static int inv_mpu6050_deselect_bypass(struct i2c_mux_core *muxc, u32 chan_id)
 	struct inv_mpu6050_state *st = iio_priv(indio_dev);
 
 	mutex_lock(&st->lock);
-	/* It doesn't really mattter, if any of the calls fails */
-	regmap_write(st->map, st->reg->int_pin_cfg, INV_MPU6050_INT_PIN_CFG);
-	st->powerup_count--;
-	if (!st->powerup_count)
-		regmap_write(st->map, st->reg->pwr_mgmt_1,
-			     INV_MPU6050_BIT_SLEEP);
+
+	/* It doesn't really matter if any of the calls fail */
+	regmap_write(st->map, st->reg->int_pin_cfg, st->irq_mask);
+	inv_mpu6050_set_power_itg(st, false);
+
 	mutex_unlock(&st->lock);
 
 	return 0;
@@ -133,29 +125,32 @@ static int inv_mpu_probe(struct i2c_client *client,
 		return result;
 
 	st = iio_priv(dev_get_drvdata(&client->dev));
-	st->muxc = i2c_mux_alloc(client->adapter, &client->dev,
-				 1, 0, I2C_MUX_LOCKED | I2C_MUX_GATE,
-				 inv_mpu6050_select_bypass,
-				 inv_mpu6050_deselect_bypass);
-	if (!st->muxc) {
-		result = -ENOMEM;
-		goto out_unreg_device;
+	switch (st->chip_type) {
+	case INV_ICM20608:
+		/* no i2c auxiliary bus on the chip */
+		break;
+	default:
+		/* declare i2c auxiliary bus */
+		st->muxc = i2c_mux_alloc(client->adapter, &client->dev,
+					 1, 0, I2C_MUX_LOCKED | I2C_MUX_GATE,
+					 inv_mpu6050_select_bypass,
+					 inv_mpu6050_deselect_bypass);
+		if (!st->muxc)
+			return -ENOMEM;
+		st->muxc->priv = dev_get_drvdata(&client->dev);
+		result = i2c_mux_add_adapter(st->muxc, 0, 0, 0);
+		if (result)
+			return result;
+		result = inv_mpu_acpi_create_mux_client(client);
+		if (result)
+			goto out_del_mux;
+		break;
 	}
-	st->muxc->priv = dev_get_drvdata(&client->dev);
-	result = i2c_mux_add_adapter(st->muxc, 0, 0, 0);
-	if (result)
-		goto out_unreg_device;
-
-	result = inv_mpu_acpi_create_mux_client(client);
-	if (result)
-		goto out_del_mux;
 
 	return 0;
 
 out_del_mux:
 	i2c_mux_del_adapters(st->muxc);
-out_unreg_device:
-	inv_mpu_core_remove(&client->dev);
 	return result;
 }
 
@@ -164,10 +159,12 @@ static int inv_mpu_remove(struct i2c_client *client)
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct inv_mpu6050_state *st = iio_priv(indio_dev);
 
-	inv_mpu_acpi_delete_mux_client(client);
-	i2c_mux_del_adapters(st->muxc);
+	if (st->muxc) {
+		inv_mpu_acpi_delete_mux_client(client);
+		i2c_mux_del_adapters(st->muxc);
+	}
 
-	return inv_mpu_core_remove(&client->dev);
+	return 0;
 }
 
 /*
@@ -179,6 +176,7 @@ static const struct i2c_device_id inv_mpu_id[] = {
 	{"mpu6500", INV_MPU6500},
 	{"mpu9150", INV_MPU9150},
 	{"mpu9250", INV_MPU9250},
+	{"mpu9255", INV_MPU9255},
 	{"icm20608", INV_ICM20608},
 	{}
 };
@@ -202,6 +200,10 @@ static const struct of_device_id inv_of_match[] = {
 		.compatible = "invensense,mpu9250",
 		.data = (void *)INV_MPU9250
 	},
+	{
+		.compatible = "invensense,mpu9255",
+		.data = (void *)INV_MPU9255
+	},
 	{
 		.compatible = "invensense,icm20608",
 		.data = (void *)INV_ICM20608
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h b/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h
index 065794162d65..c54da777945d 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h
@@ -40,6 +40,7 @@
  *  @raw_accl:		Address of first accel register.
  *  @temperature:	temperature register
  *  @int_enable:	Interrupt enable register.
+ *  @int_status:	Interrupt status register.
  *  @pwr_mgmt_1:	Controls chip's power state and clock source.
  *  @pwr_mgmt_2:	Controls power state of individual sensors.
  *  @int_pin_cfg;	Controls interrupt pin configuration.
@@ -60,6 +61,7 @@ struct inv_mpu6050_reg_map {
 	u8 raw_accl;
 	u8 temperature;
 	u8 int_enable;
+	u8 int_status;
 	u8 pwr_mgmt_1;
 	u8 pwr_mgmt_2;
 	u8 int_pin_cfg;
@@ -74,6 +76,7 @@ enum inv_devices {
 	INV_MPU6000,
 	INV_MPU9150,
 	INV_MPU9250,
+	INV_MPU9255,
 	INV_ICM20608,
 	INV_NUM_PARTS
 };
@@ -94,6 +97,7 @@ struct inv_mpu6050_chip_config {
 	unsigned int accl_fifo_enable:1;
 	unsigned int gyro_fifo_enable:1;
 	u16 fifo_rate;
+	u8 user_ctrl;
 };
 
 /**
@@ -125,6 +129,7 @@ struct inv_mpu6050_hw {
  *  @timestamps:        kfifo queue to store time stamp.
  *  @map		regmap pointer.
  *  @irq		interrupt number.
+ *  @irq_mask		the int_pin_cfg mask to configure interrupt type.
  */
 struct inv_mpu6050_state {
 #define TIMESTAMP_FIFO_SIZE 16
@@ -143,6 +148,8 @@ struct inv_mpu6050_state {
 	DECLARE_KFIFO(timestamps, long long, TIMESTAMP_FIFO_SIZE);
 	struct regmap *map;
 	int irq;
+	u8 irq_mask;
+	unsigned skip_samples;
 };
 
 /*register and associated bit definition*/
@@ -166,6 +173,9 @@ struct inv_mpu6050_state {
 #define INV_MPU6050_REG_TEMPERATURE         0x41
 #define INV_MPU6050_REG_RAW_GYRO            0x43
 
+#define INV_MPU6050_REG_INT_STATUS          0x3A
+#define INV_MPU6050_BIT_RAW_DATA_RDY_INT    0x01
+
 #define INV_MPU6050_REG_USER_CTRL           0x6A
 #define INV_MPU6050_BIT_FIFO_RST            0x04
 #define INV_MPU6050_BIT_DMP_RST             0x08
@@ -215,8 +225,12 @@ struct inv_mpu6050_state {
 #define INV_MPU6050_OUTPUT_DATA_SIZE         24
 
 #define INV_MPU6050_REG_INT_PIN_CFG	0x37
+#define INV_MPU6050_ACTIVE_HIGH		0x00
+#define INV_MPU6050_ACTIVE_LOW		0x80
+/* enable level triggering */
+#define INV_MPU6050_LATCH_INT_EN	0x20
 #define INV_MPU6050_BIT_BYPASS_EN	0x2
-#define INV_MPU6050_INT_PIN_CFG		0
+
 
 /* init parameters */
 #define INV_MPU6050_INIT_FIFO_RATE           50
@@ -232,6 +246,7 @@ struct inv_mpu6050_state {
 #define INV_MPU6500_WHOAMI_VALUE		0x70
 #define INV_MPU9150_WHOAMI_VALUE		0x68
 #define INV_MPU9250_WHOAMI_VALUE		0x71
+#define INV_MPU9255_WHOAMI_VALUE		0x73
 #define INV_ICM20608_WHOAMI_VALUE		0xAF
 
 /* scan element definition */
@@ -287,8 +302,7 @@ enum inv_mpu6050_clock_sel_e {
 
 irqreturn_t inv_mpu6050_irq_handler(int irq, void *p);
 irqreturn_t inv_mpu6050_read_fifo(int irq, void *p);
-int inv_mpu6050_probe_trigger(struct iio_dev *indio_dev);
-void inv_mpu6050_remove_trigger(struct inv_mpu6050_state *st);
+int inv_mpu6050_probe_trigger(struct iio_dev *indio_dev, int irq_type);
 int inv_reset_fifo(struct iio_dev *indio_dev);
 int inv_mpu6050_switch_engine(struct inv_mpu6050_state *st, bool en, u32 mask);
 int inv_mpu6050_write_reg(struct inv_mpu6050_state *st, int reg, u8 val);
@@ -297,6 +311,4 @@ int inv_mpu_acpi_create_mux_client(struct i2c_client *client);
 void inv_mpu_acpi_delete_mux_client(struct i2c_client *client);
 int inv_mpu_core_probe(struct regmap *regmap, int irq, const char *name,
 		int (*inv_mpu_bus_setup)(struct iio_dev *), int chip_type);
-int inv_mpu_core_remove(struct device *dev);
-int inv_mpu6050_set_power_itg(struct inv_mpu6050_state *st, bool power_on);
 extern const struct dev_pm_ops inv_mpu_pmops;
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c
index ff81c6aa009d..1795418438e4 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c
@@ -51,13 +51,14 @@ int inv_reset_fifo(struct iio_dev *indio_dev)
 	if (result)
 		goto reset_fifo_fail;
 	/* disable fifo reading */
-	result = regmap_write(st->map, st->reg->user_ctrl, 0);
+	result = regmap_write(st->map, st->reg->user_ctrl,
+			      st->chip_config.user_ctrl);
 	if (result)
 		goto reset_fifo_fail;
 
 	/* reset FIFO*/
-	result = regmap_write(st->map, st->reg->user_ctrl,
-			      INV_MPU6050_BIT_FIFO_RST);
+	d = st->chip_config.user_ctrl | INV_MPU6050_BIT_FIFO_RST;
+	result = regmap_write(st->map, st->reg->user_ctrl, d);
 	if (result)
 		goto reset_fifo_fail;
 
@@ -72,9 +73,9 @@ int inv_reset_fifo(struct iio_dev *indio_dev)
 		if (result)
 			return result;
 	}
-	/* enable FIFO reading and I2C master interface*/
-	result = regmap_write(st->map, st->reg->user_ctrl,
-			      INV_MPU6050_BIT_FIFO_EN);
+	/* enable FIFO reading */
+	d = st->chip_config.user_ctrl | INV_MPU6050_BIT_FIFO_EN;
+	result = regmap_write(st->map, st->reg->user_ctrl, d);
 	if (result)
 		goto reset_fifo_fail;
 	/* enable sensor output to FIFO */
@@ -127,8 +128,23 @@ irqreturn_t inv_mpu6050_read_fifo(int irq, void *p)
 	u8 data[INV_MPU6050_OUTPUT_DATA_SIZE];
 	u16 fifo_count;
 	s64 timestamp;
+	int int_status;
 
 	mutex_lock(&st->lock);
+
+	/* ack interrupt and check status */
+	result = regmap_read(st->map, st->reg->int_status, &int_status);
+	if (result) {
+		dev_err(regmap_get_device(st->map),
+			"failed to ack interrupt\n");
+		goto flush_fifo;
+	}
+	if (!(int_status & INV_MPU6050_BIT_RAW_DATA_RDY_INT)) {
+		dev_warn(regmap_get_device(st->map),
+			"spurious interrupt with status 0x%x\n", int_status);
+		goto end_session;
+	}
+
 	if (!(st->chip_config.accl_fifo_enable |
 		st->chip_config.gyro_fifo_enable))
 		goto end_session;
@@ -140,7 +156,7 @@ irqreturn_t inv_mpu6050_read_fifo(int irq, void *p)
 		bytes_per_datum += INV_MPU6050_BYTES_PER_3AXIS_SENSOR;
 
 	/*
-	 * read fifo_count register to know how many bytes inside FIFO
+	 * read fifo_count register to know how many bytes are inside the FIFO
 	 * right now
 	 */
 	result = regmap_bulk_read(st->map, st->reg->fifo_count_h, data,
@@ -150,7 +166,7 @@ irqreturn_t inv_mpu6050_read_fifo(int irq, void *p)
 	fifo_count = be16_to_cpup((__be16 *)(&data[0]));
 	if (fifo_count < bytes_per_datum)
 		goto end_session;
-	/* fifo count can't be odd number, if it is odd, reset fifo*/
+	/* fifo count can't be an odd number. If it is odd, reset the FIFO. */
 	if (fifo_count & 1)
 		goto flush_fifo;
 	if (fifo_count >  INV_MPU6050_FIFO_THRESHOLD)
@@ -159,7 +175,7 @@ irqreturn_t inv_mpu6050_read_fifo(int irq, void *p)
 	if (kfifo_len(&st->timestamps) >
 	    fifo_count / bytes_per_datum + INV_MPU6050_TIME_STAMP_TOR)
 		goto flush_fifo;
-	while (fifo_count >= bytes_per_datum) {
+	do {
 		result = regmap_bulk_read(st->map, st->reg->fifo_r_w,
 					  data, bytes_per_datum);
 		if (result)
@@ -170,12 +186,15 @@ irqreturn_t inv_mpu6050_read_fifo(int irq, void *p)
 		if (result == 0)
 			timestamp = 0;
 
-		result = iio_push_to_buffers_with_timestamp(indio_dev, data,
-							    timestamp);
-		if (result)
-			goto flush_fifo;
+		/* skip first samples if needed */
+		if (st->skip_samples)
+			st->skip_samples--;
+		else
+			iio_push_to_buffers_with_timestamp(indio_dev, data,
+							   timestamp);
+
 		fifo_count -= bytes_per_datum;
-	}
+	} while (fifo_count >= bytes_per_datum);
 
 end_session:
 	mutex_unlock(&st->lock);
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c
index 74506e5ac0db..227f50afff22 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c
@@ -31,8 +31,9 @@ static int inv_mpu_i2c_disable(struct iio_dev *indio_dev)
 	if (ret)
 		return ret;
 
-	ret = regmap_write(st->map, INV_MPU6050_REG_USER_CTRL,
-			   INV_MPU6050_BIT_I2C_IF_DIS);
+	st->chip_config.user_ctrl |= INV_MPU6050_BIT_I2C_IF_DIS;
+	ret = regmap_write(st->map, st->reg->user_ctrl,
+			   st->chip_config.user_ctrl);
 	if (ret) {
 		inv_mpu6050_set_power_itg(st, false);
 		return ret;
@@ -69,11 +70,6 @@ static int inv_mpu_probe(struct spi_device *spi)
 				  inv_mpu_i2c_disable, chip_type);
 }
 
-static int inv_mpu_remove(struct spi_device *spi)
-{
-	return inv_mpu_core_remove(&spi->dev);
-}
-
 /*
  * device id table is used to identify what device can be
  * supported by this driver
@@ -83,6 +79,7 @@ static const struct spi_device_id inv_mpu_id[] = {
 	{"mpu6500", INV_MPU6500},
 	{"mpu9150", INV_MPU9150},
 	{"mpu9250", INV_MPU9250},
+	{"mpu9255", INV_MPU9255},
 	{"icm20608", INV_ICM20608},
 	{}
 };
@@ -97,7 +94,6 @@ MODULE_DEVICE_TABLE(acpi, inv_acpi_match);
 
 static struct spi_driver inv_mpu_driver = {
 	.probe		=	inv_mpu_probe,
-	.remove		=	inv_mpu_remove,
 	.id_table	=	inv_mpu_id,
 	.driver = {
 		.acpi_match_table = ACPI_PTR(inv_acpi_match),
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c
index f963f9fc98c0..6c3e1652a687 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_trigger.c
@@ -49,49 +49,66 @@ static int inv_mpu6050_set_enable(struct iio_dev *indio_dev, bool enable)
 		if (result)
 			return result;
 		inv_scan_query(indio_dev);
+		st->skip_samples = 0;
 		if (st->chip_config.gyro_fifo_enable) {
 			result = inv_mpu6050_switch_engine(st, true,
 					INV_MPU6050_BIT_PWR_GYRO_STBY);
 			if (result)
-				return result;
+				goto error_power_off;
+			/* gyro first sample is out of specs, skip it */
+			st->skip_samples = 1;
 		}
 		if (st->chip_config.accl_fifo_enable) {
 			result = inv_mpu6050_switch_engine(st, true,
 					INV_MPU6050_BIT_PWR_ACCL_STBY);
 			if (result)
-				return result;
+				goto error_gyro_off;
 		}
 		result = inv_reset_fifo(indio_dev);
 		if (result)
-			return result;
+			goto error_accl_off;
 	} else {
 		result = regmap_write(st->map, st->reg->fifo_en, 0);
 		if (result)
-			return result;
+			goto error_accl_off;
 
 		result = regmap_write(st->map, st->reg->int_enable, 0);
 		if (result)
-			return result;
+			goto error_accl_off;
 
-		result = regmap_write(st->map, st->reg->user_ctrl, 0);
+		result = regmap_write(st->map, st->reg->user_ctrl,
+				      st->chip_config.user_ctrl);
 		if (result)
-			return result;
+			goto error_accl_off;
 
 		result = inv_mpu6050_switch_engine(st, false,
-					INV_MPU6050_BIT_PWR_GYRO_STBY);
+					INV_MPU6050_BIT_PWR_ACCL_STBY);
 		if (result)
-			return result;
+			goto error_accl_off;
 
 		result = inv_mpu6050_switch_engine(st, false,
-					INV_MPU6050_BIT_PWR_ACCL_STBY);
+					INV_MPU6050_BIT_PWR_GYRO_STBY);
 		if (result)
-			return result;
+			goto error_gyro_off;
+
 		result = inv_mpu6050_set_power_itg(st, false);
 		if (result)
-			return result;
+			goto error_power_off;
 	}
 
 	return 0;
+
+error_accl_off:
+	if (st->chip_config.accl_fifo_enable)
+		inv_mpu6050_switch_engine(st, false,
+					  INV_MPU6050_BIT_PWR_ACCL_STBY);
+error_gyro_off:
+	if (st->chip_config.gyro_fifo_enable)
+		inv_mpu6050_switch_engine(st, false,
+					  INV_MPU6050_BIT_PWR_GYRO_STBY);
+error_power_off:
+	inv_mpu6050_set_power_itg(st, false);
+	return result;
 }
 
 /**
@@ -117,7 +134,7 @@ static const struct iio_trigger_ops inv_mpu_trigger_ops = {
 	.set_trigger_state = &inv_mpu_data_rdy_trigger_set_state,
 };
 
-int inv_mpu6050_probe_trigger(struct iio_dev *indio_dev)
+int inv_mpu6050_probe_trigger(struct iio_dev *indio_dev, int irq_type)
 {
 	int ret;
 	struct inv_mpu6050_state *st = iio_priv(indio_dev);
@@ -131,7 +148,7 @@ int inv_mpu6050_probe_trigger(struct iio_dev *indio_dev)
 
 	ret = devm_request_irq(&indio_dev->dev, st->irq,
 			       &iio_trigger_generic_data_rdy_poll,
-			       IRQF_TRIGGER_RISING,
+			       irq_type,
 			       "inv_mpu",
 			       st->trig);
 	if (ret)
@@ -141,7 +158,7 @@ int inv_mpu6050_probe_trigger(struct iio_dev *indio_dev)
 	st->trig->ops = &inv_mpu_trigger_ops;
 	iio_trigger_set_drvdata(st->trig, indio_dev);
 
-	ret = iio_trigger_register(st->trig);
+	ret = devm_iio_trigger_register(&indio_dev->dev, st->trig);
 	if (ret)
 		return ret;
 
@@ -149,8 +166,3 @@ int inv_mpu6050_probe_trigger(struct iio_dev *indio_dev)
 
 	return 0;
 }
-
-void inv_mpu6050_remove_trigger(struct inv_mpu6050_state *st)
-{
-	iio_trigger_unregister(st->trig);
-}
diff --git a/drivers/iio/imu/st_lsm6dsx/Kconfig b/drivers/iio/imu/st_lsm6dsx/Kconfig
index 14f2eb6e9fb7..ccc817e17eb8 100644
--- a/drivers/iio/imu/st_lsm6dsx/Kconfig
+++ b/drivers/iio/imu/st_lsm6dsx/Kconfig
@@ -8,7 +8,8 @@ config IIO_ST_LSM6DSX
 	select IIO_ST_LSM6DSX_SPI if (SPI_MASTER)
 	help
 	  Say yes here to build support for STMicroelectronics LSM6DSx imu
-	  sensor. Supported devices: lsm6ds3, lsm6ds3h, lsm6dsl, lsm6dsm
+	  sensor. Supported devices: lsm6ds3, lsm6ds3h, lsm6dsl, lsm6dsm,
+	  ism330dlc
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called st_lsm6dsx.
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
index a3cc7cd97026..edcd838037cd 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
@@ -18,12 +18,14 @@
 #define ST_LSM6DS3H_DEV_NAME	"lsm6ds3h"
 #define ST_LSM6DSL_DEV_NAME	"lsm6dsl"
 #define ST_LSM6DSM_DEV_NAME	"lsm6dsm"
+#define ST_ISM330DLC_DEV_NAME	"ism330dlc"
 
 enum st_lsm6dsx_hw_id {
 	ST_LSM6DS3_ID,
 	ST_LSM6DS3H_ID,
 	ST_LSM6DSL_ID,
 	ST_LSM6DSM_ID,
+	ST_ISM330DLC_ID,
 	ST_LSM6DSX_MAX_ID,
 };
 
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
index 1045e025e92b..4994f920a836 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
@@ -1,10 +1,10 @@
 /*
  * STMicroelectronics st_lsm6dsx FIFO buffer library driver
  *
- * LSM6DS3/LSM6DS3H/LSM6DSL/LSM6DSM: The FIFO buffer can be configured
- * to store data from gyroscope and accelerometer. Samples are queued
- * without any tag according to a specific pattern based on 'FIFO data sets'
- * (6 bytes each):
+ * LSM6DS3/LSM6DS3H/LSM6DSL/LSM6DSM/ISM330DLC: The FIFO buffer can be
+ * configured to store data from gyroscope and accelerometer. Samples are
+ * queued without any tag according to a specific pattern based on
+ * 'FIFO data sets' (6 bytes each):
  *  - 1st data set is reserved for gyroscope data
  *  - 2nd data set is reserved for accelerometer data
  * The FIFO pattern changes depending on the ODRs and decimation factors
@@ -276,7 +276,7 @@ static inline int st_lsm6dsx_read_block(struct st_lsm6dsx_hw *hw, u8 *data,
 #define ST_LSM6DSX_IIO_BUFF_SIZE	(ALIGN(ST_LSM6DSX_SAMPLE_SIZE, \
 					       sizeof(s64)) + sizeof(s64))
 /**
- * st_lsm6dsx_read_fifo() - LSM6DS3-LSM6DS3H-LSM6DSL-LSM6DSM read FIFO routine
+ * st_lsm6dsx_read_fifo() - hw FIFO read routine
  * @hw: Pointer to instance of struct st_lsm6dsx_hw.
  *
  * Read samples from the hw FIFO and push them to IIO buffers.
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
index 8656d72ef4ee..aebbe0ddd8d8 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
@@ -17,7 +17,7 @@
  *   - Gyroscope supported full-scale [dps]: +-125/+-245/+-500/+-1000/+-2000
  *   - FIFO size: 8KB
  *
- * - LSM6DS3H/LSM6DSL/LSM6DSM:
+ * - LSM6DS3H/LSM6DSL/LSM6DSM/ISM330DLC:
  *   - Accelerometer/Gyroscope supported ODR [Hz]: 13, 26, 52, 104, 208, 416
  *   - Accelerometer supported full-scale [g]: +-2/+-4/+-8/+-16
  *   - Gyroscope supported full-scale [dps]: +-125/+-245/+-500/+-1000/+-2000
@@ -252,6 +252,7 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 		.id = {
 			[0] = ST_LSM6DSL_ID,
 			[1] = ST_LSM6DSM_ID,
+			[2] = ST_ISM330DLC_ID,
 		},
 		.decimator = {
 			[ST_LSM6DSX_ID_ACC] = {
@@ -266,11 +267,11 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 		.fifo_ops = {
 			.fifo_th = {
 				.addr = 0x06,
-				.mask = GENMASK(11, 0),
+				.mask = GENMASK(10, 0),
 			},
 			.fifo_diff = {
 				.addr = 0x3a,
-				.mask = GENMASK(11, 0),
+				.mask = GENMASK(10, 0),
 			},
 			.th_wl = 3, /* 1LSB = 2B */
 		},
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c
index 41525dd2aab7..377c4e9997da 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c
@@ -57,6 +57,10 @@ static const struct of_device_id st_lsm6dsx_i2c_of_match[] = {
 		.compatible = "st,lsm6dsm",
 		.data = (void *)ST_LSM6DSM_ID,
 	},
+	{
+		.compatible = "st,ism330dlc",
+		.data = (void *)ST_ISM330DLC_ID,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, st_lsm6dsx_i2c_of_match);
@@ -66,6 +70,7 @@ static const struct i2c_device_id st_lsm6dsx_i2c_id_table[] = {
 	{ ST_LSM6DS3H_DEV_NAME, ST_LSM6DS3H_ID },
 	{ ST_LSM6DSL_DEV_NAME, ST_LSM6DSL_ID },
 	{ ST_LSM6DSM_DEV_NAME, ST_LSM6DSM_ID },
+	{ ST_ISM330DLC_DEV_NAME, ST_ISM330DLC_ID },
 	{},
 };
 MODULE_DEVICE_TABLE(i2c, st_lsm6dsx_i2c_id_table);
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
index 2c8135834479..fec5c6ce7eb7 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
@@ -57,6 +57,10 @@ static const struct of_device_id st_lsm6dsx_spi_of_match[] = {
 		.compatible = "st,lsm6dsm",
 		.data = (void *)ST_LSM6DSM_ID,
 	},
+	{
+		.compatible = "st,ism330dlc",
+		.data = (void *)ST_ISM330DLC_ID,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, st_lsm6dsx_spi_of_match);
@@ -66,6 +70,7 @@ static const struct spi_device_id st_lsm6dsx_spi_id_table[] = {
 	{ ST_LSM6DS3H_DEV_NAME, ST_LSM6DS3H_ID },
 	{ ST_LSM6DSL_DEV_NAME, ST_LSM6DSL_ID },
 	{ ST_LSM6DSM_DEV_NAME, ST_LSM6DSM_ID },
+	{ ST_ISM330DLC_DEV_NAME, ST_ISM330DLC_ID },
 	{},
 };
 MODULE_DEVICE_TABLE(spi, st_lsm6dsx_spi_id_table);
diff --git a/drivers/iio/light/Kconfig b/drivers/iio/light/Kconfig
index 074e50657366..c7ef8d1862d6 100644
--- a/drivers/iio/light/Kconfig
+++ b/drivers/iio/light/Kconfig
@@ -409,6 +409,14 @@ config TSL2583
 	 Provides support for the TAOS tsl2580, tsl2581 and tsl2583 devices.
 	 Access ALS data via iio, sysfs.
 
+config TSL2772
+	tristate "TAOS TSL/TMD2x71 and TSL/TMD2x72 Family of light and proximity sensors"
+	depends on I2C
+	help
+	 Support for: tsl2571, tsl2671, tmd2671, tsl2771, tmd2771, tsl2572, tsl2672,
+	 tmd2672, tsl2772, tmd2772 devices.
+	 Provides iio_events and direct access via sysfs.
+
 config TSL4531
 	tristate "TAOS TSL4531 ambient light sensors"
 	depends on I2C
diff --git a/drivers/iio/light/Makefile b/drivers/iio/light/Makefile
index f1777036d4f8..80943af5d627 100644
--- a/drivers/iio/light/Makefile
+++ b/drivers/iio/light/Makefile
@@ -40,6 +40,7 @@ obj-$(CONFIG_ST_UVIS25_SPI)	+= st_uvis25_spi.o
 obj-$(CONFIG_TCS3414)		+= tcs3414.o
 obj-$(CONFIG_TCS3472)		+= tcs3472.o
 obj-$(CONFIG_TSL2583)		+= tsl2583.o
+obj-$(CONFIG_TSL2772)		+= tsl2772.o
 obj-$(CONFIG_TSL4531)		+= tsl4531.o
 obj-$(CONFIG_US5182D)		+= us5182d.o
 obj-$(CONFIG_VCNL4000)		+= vcnl4000.o
diff --git a/drivers/iio/light/cros_ec_light_prox.c b/drivers/iio/light/cros_ec_light_prox.c
index 8e8a0e7f78d1..fd1609e975ab 100644
--- a/drivers/iio/light/cros_ec_light_prox.c
+++ b/drivers/iio/light/cros_ec_light_prox.c
@@ -16,6 +16,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/iio/buffer.h>
+#include <linux/iio/common/cros_ec_sensors_core.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/kfifo_buf.h>
 #include <linux/iio/trigger.h>
@@ -29,8 +30,6 @@
 #include <linux/slab.h>
 #include <linux/sysfs.h>
 
-#include "../common/cros_ec_sensors/cros_ec_sensors_core.h"
-
 /*
  * We only represent one entry for light or proximity. EC is merging different
  * light sensors to return the what the eye would see. For proximity, we
diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c
index f2e50edaa242..4b5d9988f025 100644
--- a/drivers/iio/light/tsl2583.c
+++ b/drivers/iio/light/tsl2583.c
@@ -600,7 +600,7 @@ static ssize_t in_illuminance_lux_table_store(struct device *dev,
 
 static IIO_CONST_ATTR(in_illuminance_calibscale_available, "1 8 16 111");
 static IIO_CONST_ATTR(in_illuminance_integration_time_available,
-		      "0.000050 0.000100 0.000150 0.000200 0.000250 0.000300 0.000350 0.000400 0.000450 0.000500 0.000550 0.000600 0.000650");
+		      "0.050 0.100 0.150 0.200 0.250 0.300 0.350 0.400 0.450 0.500 0.550 0.600 0.650");
 static IIO_DEVICE_ATTR_RW(in_illuminance_input_target, 0);
 static IIO_DEVICE_ATTR_WO(in_illuminance_calibrate, 0);
 static IIO_DEVICE_ATTR_RW(in_illuminance_lux_table, 0);
diff --git a/drivers/iio/light/tsl2772.c b/drivers/iio/light/tsl2772.c
new file mode 100644
index 000000000000..34d42a2504c9
--- /dev/null
+++ b/drivers/iio/light/tsl2772.c
@@ -0,0 +1,1800 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Device driver for monitoring ambient light intensity in (lux) and proximity
+ * detection (prox) within the TAOS TSL2571, TSL2671, TMD2671, TSL2771, TMD2771,
+ * TSL2572, TSL2672, TMD2672, TSL2772, and TMD2772 devices.
+ *
+ * Copyright (c) 2012, TAOS Corporation.
+ * Copyright (c) 2017-2018 Brian Masney <masneyb@onstation.org>
+ */
+
+#include <linux/delay.h>
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/iio/events.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/platform_data/tsl2772.h>
+
+/* Cal defs */
+#define PROX_STAT_CAL			0
+#define PROX_STAT_SAMP			1
+#define MAX_SAMPLES_CAL			200
+
+/* TSL2772 Device ID */
+#define TRITON_ID			0x00
+#define SWORDFISH_ID			0x30
+#define HALIBUT_ID			0x20
+
+/* Lux calculation constants */
+#define TSL2772_LUX_CALC_OVER_FLOW	65535
+
+/*
+ * TAOS Register definitions - Note: depending on device, some of these register
+ * are not used and the register address is benign.
+ */
+
+/* Register offsets */
+#define TSL2772_MAX_CONFIG_REG		16
+
+/* Device Registers and Masks */
+#define TSL2772_CNTRL			0x00
+#define TSL2772_ALS_TIME		0X01
+#define TSL2772_PRX_TIME		0x02
+#define TSL2772_WAIT_TIME		0x03
+#define TSL2772_ALS_MINTHRESHLO		0X04
+#define TSL2772_ALS_MINTHRESHHI		0X05
+#define TSL2772_ALS_MAXTHRESHLO		0X06
+#define TSL2772_ALS_MAXTHRESHHI		0X07
+#define TSL2772_PRX_MINTHRESHLO		0X08
+#define TSL2772_PRX_MINTHRESHHI		0X09
+#define TSL2772_PRX_MAXTHRESHLO		0X0A
+#define TSL2772_PRX_MAXTHRESHHI		0X0B
+#define TSL2772_PERSISTENCE		0x0C
+#define TSL2772_ALS_PRX_CONFIG		0x0D
+#define TSL2772_PRX_COUNT		0x0E
+#define TSL2772_GAIN			0x0F
+#define TSL2772_NOTUSED			0x10
+#define TSL2772_REVID			0x11
+#define TSL2772_CHIPID			0x12
+#define TSL2772_STATUS			0x13
+#define TSL2772_ALS_CHAN0LO		0x14
+#define TSL2772_ALS_CHAN0HI		0x15
+#define TSL2772_ALS_CHAN1LO		0x16
+#define TSL2772_ALS_CHAN1HI		0x17
+#define TSL2772_PRX_LO			0x18
+#define TSL2772_PRX_HI			0x19
+
+/* tsl2772 cmd reg masks */
+#define TSL2772_CMD_REG			0x80
+#define TSL2772_CMD_SPL_FN		0x60
+#define TSL2772_CMD_REPEAT_PROTO	0x00
+#define TSL2772_CMD_AUTOINC_PROTO	0x20
+
+#define TSL2772_CMD_PROX_INT_CLR	0X05
+#define TSL2772_CMD_ALS_INT_CLR		0x06
+#define TSL2772_CMD_PROXALS_INT_CLR	0X07
+
+/* tsl2772 cntrl reg masks */
+#define TSL2772_CNTL_ADC_ENBL		0x02
+#define TSL2772_CNTL_PWR_ON		0x01
+
+/* tsl2772 status reg masks */
+#define TSL2772_STA_ADC_VALID		0x01
+#define TSL2772_STA_PRX_VALID		0x02
+#define TSL2772_STA_ADC_PRX_VALID	(TSL2772_STA_ADC_VALID | \
+					 TSL2772_STA_PRX_VALID)
+#define TSL2772_STA_ALS_INTR		0x10
+#define TSL2772_STA_PRX_INTR		0x20
+
+/* tsl2772 cntrl reg masks */
+#define TSL2772_CNTL_REG_CLEAR		0x00
+#define TSL2772_CNTL_PROX_INT_ENBL	0X20
+#define TSL2772_CNTL_ALS_INT_ENBL	0X10
+#define TSL2772_CNTL_WAIT_TMR_ENBL	0X08
+#define TSL2772_CNTL_PROX_DET_ENBL	0X04
+#define TSL2772_CNTL_PWRON		0x01
+#define TSL2772_CNTL_ALSPON_ENBL	0x03
+#define TSL2772_CNTL_INTALSPON_ENBL	0x13
+#define TSL2772_CNTL_PROXPON_ENBL	0x0F
+#define TSL2772_CNTL_INTPROXPON_ENBL	0x2F
+
+#define TSL2772_ALS_GAIN_TRIM_MIN	250
+#define TSL2772_ALS_GAIN_TRIM_MAX	4000
+
+/* Device family members */
+enum {
+	tsl2571,
+	tsl2671,
+	tmd2671,
+	tsl2771,
+	tmd2771,
+	tsl2572,
+	tsl2672,
+	tmd2672,
+	tsl2772,
+	tmd2772
+};
+
+enum {
+	TSL2772_CHIP_UNKNOWN = 0,
+	TSL2772_CHIP_WORKING = 1,
+	TSL2772_CHIP_SUSPENDED = 2
+};
+
+/* Per-device data */
+struct tsl2772_als_info {
+	u16 als_ch0;
+	u16 als_ch1;
+	u16 lux;
+};
+
+struct tsl2772_chip_info {
+	int chan_table_elements;
+	struct iio_chan_spec channel_with_events[4];
+	struct iio_chan_spec channel_without_events[4];
+	const struct iio_info *info;
+};
+
+struct tsl2772_chip {
+	kernel_ulong_t id;
+	struct mutex prox_mutex;
+	struct mutex als_mutex;
+	struct i2c_client *client;
+	u16 prox_data;
+	struct tsl2772_als_info als_cur_info;
+	struct tsl2772_settings settings;
+	struct tsl2772_platform_data *pdata;
+	int als_gain_time_scale;
+	int als_saturation;
+	int tsl2772_chip_status;
+	u8 tsl2772_config[TSL2772_MAX_CONFIG_REG];
+	const struct tsl2772_chip_info	*chip_info;
+	const struct iio_info *info;
+	s64 event_timestamp;
+	/*
+	 * This structure is intentionally large to accommodate
+	 * updates via sysfs.
+	 * Sized to 9 = max 8 segments + 1 termination segment
+	 */
+	struct tsl2772_lux tsl2772_device_lux[TSL2772_MAX_LUX_TABLE_SIZE];
+};
+
+/*
+ * Different devices require different coefficents, and these numbers were
+ * derived from the 'Lux Equation' section of the various device datasheets.
+ * All of these coefficients assume a Glass Attenuation (GA) factor of 1.
+ * The coefficients are multiplied by 1000 to avoid floating point operations.
+ * The two rows in each table correspond to the Lux1 and Lux2 equations from
+ * the datasheets.
+ */
+static const struct tsl2772_lux tsl2x71_lux_table[TSL2772_DEF_LUX_TABLE_SZ] = {
+	{ 53000, 106000 },
+	{ 31800,  53000 },
+	{ 0,          0 },
+};
+
+static const struct tsl2772_lux tmd2x71_lux_table[TSL2772_DEF_LUX_TABLE_SZ] = {
+	{ 24000,  48000 },
+	{ 14400,  24000 },
+	{ 0,          0 },
+};
+
+static const struct tsl2772_lux tsl2x72_lux_table[TSL2772_DEF_LUX_TABLE_SZ] = {
+	{ 60000, 112200 },
+	{ 37800,  60000 },
+	{     0,      0 },
+};
+
+static const struct tsl2772_lux tmd2x72_lux_table[TSL2772_DEF_LUX_TABLE_SZ] = {
+	{ 20000,  35000 },
+	{ 12600,  20000 },
+	{     0,      0 },
+};
+
+static const struct tsl2772_lux *tsl2772_default_lux_table_group[] = {
+	[tsl2571] = tsl2x71_lux_table,
+	[tsl2671] = tsl2x71_lux_table,
+	[tmd2671] = tmd2x71_lux_table,
+	[tsl2771] = tsl2x71_lux_table,
+	[tmd2771] = tmd2x71_lux_table,
+	[tsl2572] = tsl2x72_lux_table,
+	[tsl2672] = tsl2x72_lux_table,
+	[tmd2672] = tmd2x72_lux_table,
+	[tsl2772] = tsl2x72_lux_table,
+	[tmd2772] = tmd2x72_lux_table,
+};
+
+static const struct tsl2772_settings tsl2772_default_settings = {
+	.als_time = 255, /* 2.72 / 2.73 ms */
+	.als_gain = 0,
+	.prox_time = 255, /* 2.72 / 2.73 ms */
+	.prox_gain = 0,
+	.wait_time = 255,
+	.als_prox_config = 0,
+	.als_gain_trim = 1000,
+	.als_cal_target = 150,
+	.als_persistence = 1,
+	.als_interrupt_en = false,
+	.als_thresh_low = 200,
+	.als_thresh_high = 256,
+	.prox_persistence = 1,
+	.prox_interrupt_en = false,
+	.prox_thres_low  = 0,
+	.prox_thres_high = 512,
+	.prox_max_samples_cal = 30,
+	.prox_pulse_count = 8,
+	.prox_diode = TSL2772_DIODE1,
+	.prox_power = TSL2772_100_mA
+};
+
+static const s16 tsl2772_als_gain[] = {
+	1,
+	8,
+	16,
+	120
+};
+
+static const s16 tsl2772_prox_gain[] = {
+	1,
+	2,
+	4,
+	8
+};
+
+static const int tsl2772_int_time_avail[][6] = {
+	[tsl2571] = { 0, 2720, 0, 2720, 0, 696000 },
+	[tsl2671] = { 0, 2720, 0, 2720, 0, 696000 },
+	[tmd2671] = { 0, 2720, 0, 2720, 0, 696000 },
+	[tsl2771] = { 0, 2720, 0, 2720, 0, 696000 },
+	[tmd2771] = { 0, 2720, 0, 2720, 0, 696000 },
+	[tsl2572] = { 0, 2730, 0, 2730, 0, 699000 },
+	[tsl2672] = { 0, 2730, 0, 2730, 0, 699000 },
+	[tmd2672] = { 0, 2730, 0, 2730, 0, 699000 },
+	[tsl2772] = { 0, 2730, 0, 2730, 0, 699000 },
+	[tmd2772] = { 0, 2730, 0, 2730, 0, 699000 },
+};
+
+static int tsl2772_int_calibscale_avail[] = { 1, 8, 16, 120 };
+
+static int tsl2772_prox_calibscale_avail[] = { 1, 2, 4, 8 };
+
+/* Channel variations */
+enum {
+	ALS,
+	PRX,
+	ALSPRX,
+	PRX2,
+	ALSPRX2,
+};
+
+static const u8 device_channel_config[] = {
+	[tsl2571] = ALS,
+	[tsl2671] = PRX,
+	[tmd2671] = PRX,
+	[tsl2771] = ALSPRX,
+	[tmd2771] = ALSPRX,
+	[tsl2572] = ALS,
+	[tsl2672] = PRX2,
+	[tmd2672] = PRX2,
+	[tsl2772] = ALSPRX2,
+	[tmd2772] = ALSPRX2
+};
+
+static int tsl2772_read_status(struct tsl2772_chip *chip)
+{
+	int ret;
+
+	ret = i2c_smbus_read_byte_data(chip->client,
+				       TSL2772_CMD_REG | TSL2772_STATUS);
+	if (ret < 0)
+		dev_err(&chip->client->dev,
+			"%s: failed to read STATUS register: %d\n", __func__,
+			ret);
+
+	return ret;
+}
+
+static int tsl2772_write_control_reg(struct tsl2772_chip *chip, u8 data)
+{
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(chip->client,
+					TSL2772_CMD_REG | TSL2772_CNTRL, data);
+	if (ret < 0) {
+		dev_err(&chip->client->dev,
+			"%s: failed to write to control register %x: %d\n",
+			__func__, data, ret);
+	}
+
+	return ret;
+}
+
+static int tsl2772_read_autoinc_regs(struct tsl2772_chip *chip, int lower_reg,
+				     int upper_reg)
+{
+	u8 buf[2];
+	int ret;
+
+	ret = i2c_smbus_write_byte(chip->client,
+				   TSL2772_CMD_REG | TSL2772_CMD_AUTOINC_PROTO |
+				   lower_reg);
+	if (ret < 0) {
+		dev_err(&chip->client->dev,
+			"%s: failed to enable auto increment protocol: %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	ret = i2c_smbus_read_byte_data(chip->client,
+				       TSL2772_CMD_REG | lower_reg);
+	if (ret < 0) {
+		dev_err(&chip->client->dev,
+			"%s: failed to read from register %x: %d\n", __func__,
+			lower_reg, ret);
+		return ret;
+	}
+	buf[0] = ret;
+
+	ret = i2c_smbus_read_byte_data(chip->client,
+				       TSL2772_CMD_REG | upper_reg);
+	if (ret < 0) {
+		dev_err(&chip->client->dev,
+			"%s: failed to read from register %x: %d\n", __func__,
+			upper_reg, ret);
+		return ret;
+	}
+	buf[1] = ret;
+
+	ret = i2c_smbus_write_byte(chip->client,
+				   TSL2772_CMD_REG | TSL2772_CMD_REPEAT_PROTO |
+				   lower_reg);
+	if (ret < 0) {
+		dev_err(&chip->client->dev,
+			"%s: failed to enable repeated byte protocol: %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	return le16_to_cpup((const __le16 *)&buf[0]);
+}
+
+/**
+ * tsl2772_get_lux() - Reads and calculates current lux value.
+ * @indio_dev:	pointer to IIO device
+ *
+ * The raw ch0 and ch1 values of the ambient light sensed in the last
+ * integration cycle are read from the device. The raw values are multiplied
+ * by a device-specific scale factor, and divided by the integration time and
+ * device gain. The code supports multiple lux equations through the lux table
+ * coefficients. A lux gain trim is applied to each lux equation, and then the
+ * maximum lux within the interval 0..65535 is selected.
+ */
+static int tsl2772_get_lux(struct iio_dev *indio_dev)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+	struct tsl2772_lux *p;
+	int max_lux, ret;
+	bool overflow;
+
+	mutex_lock(&chip->als_mutex);
+
+	if (chip->tsl2772_chip_status != TSL2772_CHIP_WORKING) {
+		dev_err(&chip->client->dev, "%s: device is not enabled\n",
+			__func__);
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
+	ret = tsl2772_read_status(chip);
+	if (ret < 0)
+		goto out_unlock;
+
+	if (!(ret & TSL2772_STA_ADC_VALID)) {
+		dev_err(&chip->client->dev,
+			"%s: data not valid yet\n", __func__);
+		ret = chip->als_cur_info.lux; /* return LAST VALUE */
+		goto out_unlock;
+	}
+
+	ret = tsl2772_read_autoinc_regs(chip, TSL2772_ALS_CHAN0LO,
+					TSL2772_ALS_CHAN0HI);
+	if (ret < 0)
+		goto out_unlock;
+	chip->als_cur_info.als_ch0 = ret;
+
+	ret = tsl2772_read_autoinc_regs(chip, TSL2772_ALS_CHAN1LO,
+					TSL2772_ALS_CHAN1HI);
+	if (ret < 0)
+		goto out_unlock;
+	chip->als_cur_info.als_ch1 = ret;
+
+	if (chip->als_cur_info.als_ch0 >= chip->als_saturation) {
+		max_lux = TSL2772_LUX_CALC_OVER_FLOW;
+		goto update_struct_with_max_lux;
+	}
+
+	if (!chip->als_cur_info.als_ch0) {
+		/* have no data, so return LAST VALUE */
+		ret = chip->als_cur_info.lux;
+		goto out_unlock;
+	}
+
+	max_lux = 0;
+	overflow = false;
+	for (p = (struct tsl2772_lux *)chip->tsl2772_device_lux; p->ch0 != 0;
+	     p++) {
+		int lux;
+
+		lux = ((chip->als_cur_info.als_ch0 * p->ch0) -
+		       (chip->als_cur_info.als_ch1 * p->ch1)) /
+			chip->als_gain_time_scale;
+
+		/*
+		 * The als_gain_trim can have a value within the range 250..4000
+		 * and is a multiplier for the lux. A trim of 1000 makes no
+		 * changes to the lux, less than 1000 scales it down, and
+		 * greater than 1000 scales it up.
+		 */
+		lux = (lux * chip->settings.als_gain_trim) / 1000;
+
+		if (lux > TSL2772_LUX_CALC_OVER_FLOW) {
+			overflow = true;
+			continue;
+		}
+
+		max_lux = max(max_lux, lux);
+	}
+
+	if (overflow && max_lux == 0)
+		max_lux = TSL2772_LUX_CALC_OVER_FLOW;
+
+update_struct_with_max_lux:
+	chip->als_cur_info.lux = max_lux;
+	ret = max_lux;
+
+out_unlock:
+	mutex_unlock(&chip->als_mutex);
+
+	return ret;
+}
+
+/**
+ * tsl2772_get_prox() - Reads proximity data registers and updates
+ *                      chip->prox_data.
+ *
+ * @indio_dev:	pointer to IIO device
+ */
+static int tsl2772_get_prox(struct iio_dev *indio_dev)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&chip->prox_mutex);
+
+	ret = tsl2772_read_status(chip);
+	if (ret < 0)
+		goto prox_poll_err;
+
+	switch (chip->id) {
+	case tsl2571:
+	case tsl2671:
+	case tmd2671:
+	case tsl2771:
+	case tmd2771:
+		if (!(ret & TSL2772_STA_ADC_VALID)) {
+			ret = -EINVAL;
+			goto prox_poll_err;
+		}
+		break;
+	case tsl2572:
+	case tsl2672:
+	case tmd2672:
+	case tsl2772:
+	case tmd2772:
+		if (!(ret & TSL2772_STA_PRX_VALID)) {
+			ret = -EINVAL;
+			goto prox_poll_err;
+		}
+		break;
+	}
+
+	ret = tsl2772_read_autoinc_regs(chip, TSL2772_PRX_LO, TSL2772_PRX_HI);
+	if (ret < 0)
+		goto prox_poll_err;
+	chip->prox_data = ret;
+
+prox_poll_err:
+	mutex_unlock(&chip->prox_mutex);
+
+	return ret;
+}
+
+/**
+ * tsl2772_defaults() - Populates the device nominal operating parameters
+ *                      with those provided by a 'platform' data struct or
+ *                      with prefined defaults.
+ *
+ * @chip:               pointer to device structure.
+ */
+static void tsl2772_defaults(struct tsl2772_chip *chip)
+{
+	/* If Operational settings defined elsewhere.. */
+	if (chip->pdata && chip->pdata->platform_default_settings)
+		memcpy(&chip->settings, chip->pdata->platform_default_settings,
+		       sizeof(tsl2772_default_settings));
+	else
+		memcpy(&chip->settings, &tsl2772_default_settings,
+		       sizeof(tsl2772_default_settings));
+
+	/* Load up the proper lux table. */
+	if (chip->pdata && chip->pdata->platform_lux_table[0].ch0 != 0)
+		memcpy(chip->tsl2772_device_lux,
+		       chip->pdata->platform_lux_table,
+		       sizeof(chip->pdata->platform_lux_table));
+	else
+		memcpy(chip->tsl2772_device_lux,
+		       tsl2772_default_lux_table_group[chip->id],
+		       TSL2772_DEFAULT_TABLE_BYTES);
+}
+
+/**
+ * tsl2772_als_calibrate() -	Obtain single reading and calculate
+ *                              the als_gain_trim.
+ *
+ * @indio_dev:	pointer to IIO device
+ */
+static int tsl2772_als_calibrate(struct iio_dev *indio_dev)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+	int ret, lux_val;
+
+	ret = i2c_smbus_read_byte_data(chip->client,
+				       TSL2772_CMD_REG | TSL2772_CNTRL);
+	if (ret < 0) {
+		dev_err(&chip->client->dev,
+			"%s: failed to read from the CNTRL register\n",
+			__func__);
+		return ret;
+	}
+
+	if ((ret & (TSL2772_CNTL_ADC_ENBL | TSL2772_CNTL_PWR_ON))
+			!= (TSL2772_CNTL_ADC_ENBL | TSL2772_CNTL_PWR_ON)) {
+		dev_err(&chip->client->dev,
+			"%s: Device is not powered on and/or ADC is not enabled\n",
+			__func__);
+		return -EINVAL;
+	} else if ((ret & TSL2772_STA_ADC_VALID) != TSL2772_STA_ADC_VALID) {
+		dev_err(&chip->client->dev,
+			"%s: The two ADC channels have not completed an integration cycle\n",
+			__func__);
+		return -ENODATA;
+	}
+
+	lux_val = tsl2772_get_lux(indio_dev);
+	if (lux_val < 0) {
+		dev_err(&chip->client->dev,
+			"%s: failed to get lux\n", __func__);
+		return lux_val;
+	}
+
+	ret = (chip->settings.als_cal_target * chip->settings.als_gain_trim) /
+			lux_val;
+	if (ret < TSL2772_ALS_GAIN_TRIM_MIN || ret > TSL2772_ALS_GAIN_TRIM_MAX)
+		return -ERANGE;
+
+	chip->settings.als_gain_trim = ret;
+
+	return ret;
+}
+
+static int tsl2772_chip_on(struct iio_dev *indio_dev)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+	int ret, i, als_count, als_time_us;
+	u8 *dev_reg, reg_val;
+
+	/* Non calculated parameters */
+	chip->tsl2772_config[TSL2772_ALS_TIME] = chip->settings.als_time;
+	chip->tsl2772_config[TSL2772_PRX_TIME] = chip->settings.prox_time;
+	chip->tsl2772_config[TSL2772_WAIT_TIME] = chip->settings.wait_time;
+	chip->tsl2772_config[TSL2772_ALS_PRX_CONFIG] =
+		chip->settings.als_prox_config;
+
+	chip->tsl2772_config[TSL2772_ALS_MINTHRESHLO] =
+		(chip->settings.als_thresh_low) & 0xFF;
+	chip->tsl2772_config[TSL2772_ALS_MINTHRESHHI] =
+		(chip->settings.als_thresh_low >> 8) & 0xFF;
+	chip->tsl2772_config[TSL2772_ALS_MAXTHRESHLO] =
+		(chip->settings.als_thresh_high) & 0xFF;
+	chip->tsl2772_config[TSL2772_ALS_MAXTHRESHHI] =
+		(chip->settings.als_thresh_high >> 8) & 0xFF;
+	chip->tsl2772_config[TSL2772_PERSISTENCE] =
+		(chip->settings.prox_persistence & 0xFF) << 4 |
+		(chip->settings.als_persistence & 0xFF);
+
+	chip->tsl2772_config[TSL2772_PRX_COUNT] =
+			chip->settings.prox_pulse_count;
+	chip->tsl2772_config[TSL2772_PRX_MINTHRESHLO] =
+			(chip->settings.prox_thres_low) & 0xFF;
+	chip->tsl2772_config[TSL2772_PRX_MINTHRESHHI] =
+			(chip->settings.prox_thres_low >> 8) & 0xFF;
+	chip->tsl2772_config[TSL2772_PRX_MAXTHRESHLO] =
+			(chip->settings.prox_thres_high) & 0xFF;
+	chip->tsl2772_config[TSL2772_PRX_MAXTHRESHHI] =
+			(chip->settings.prox_thres_high >> 8) & 0xFF;
+
+	/* and make sure we're not already on */
+	if (chip->tsl2772_chip_status == TSL2772_CHIP_WORKING) {
+		/* if forcing a register update - turn off, then on */
+		dev_info(&chip->client->dev, "device is already enabled\n");
+		return -EINVAL;
+	}
+
+	/* Set the gain based on tsl2772_settings struct */
+	chip->tsl2772_config[TSL2772_GAIN] =
+		(chip->settings.als_gain & 0xFF) |
+		((chip->settings.prox_gain & 0xFF) << 2) |
+		(chip->settings.prox_diode << 4) |
+		(chip->settings.prox_power << 6);
+
+	/* set chip time scaling and saturation */
+	als_count = 256 - chip->settings.als_time;
+	als_time_us = als_count * tsl2772_int_time_avail[chip->id][3];
+	chip->als_saturation = als_count * 768; /* 75% of full scale */
+	chip->als_gain_time_scale = als_time_us *
+		tsl2772_als_gain[chip->settings.als_gain];
+
+	/*
+	 * TSL2772 Specific power-on / adc enable sequence
+	 * Power on the device 1st.
+	 */
+	ret = tsl2772_write_control_reg(chip, TSL2772_CNTL_PWR_ON);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Use the following shadow copy for our delay before enabling ADC.
+	 * Write all the registers.
+	 */
+	for (i = 0, dev_reg = chip->tsl2772_config;
+			i < TSL2772_MAX_CONFIG_REG; i++) {
+		int reg = TSL2772_CMD_REG + i;
+
+		ret = i2c_smbus_write_byte_data(chip->client, reg,
+						*dev_reg++);
+		if (ret < 0) {
+			dev_err(&chip->client->dev,
+				"%s: failed to write to register %x: %d\n",
+				__func__, reg, ret);
+			return ret;
+		}
+	}
+
+	/* Power-on settling time */
+	usleep_range(3000, 3500);
+
+	reg_val = TSL2772_CNTL_PWR_ON | TSL2772_CNTL_ADC_ENBL |
+		  TSL2772_CNTL_PROX_DET_ENBL;
+	if (chip->settings.als_interrupt_en)
+		reg_val |= TSL2772_CNTL_ALS_INT_ENBL;
+	if (chip->settings.prox_interrupt_en)
+		reg_val |= TSL2772_CNTL_PROX_INT_ENBL;
+
+	ret = tsl2772_write_control_reg(chip, reg_val);
+	if (ret < 0)
+		return ret;
+
+	ret = i2c_smbus_write_byte(chip->client,
+				   TSL2772_CMD_REG | TSL2772_CMD_SPL_FN |
+				   TSL2772_CMD_PROXALS_INT_CLR);
+	if (ret < 0) {
+		dev_err(&chip->client->dev,
+			"%s: failed to clear interrupt status: %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	chip->tsl2772_chip_status = TSL2772_CHIP_WORKING;
+
+	return ret;
+}
+
+static int tsl2772_chip_off(struct iio_dev *indio_dev)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+
+	/* turn device off */
+	chip->tsl2772_chip_status = TSL2772_CHIP_SUSPENDED;
+	return tsl2772_write_control_reg(chip, 0x00);
+}
+
+/**
+ * tsl2772_invoke_change - power cycle the device to implement the user
+ *                         parameters
+ * @indio_dev:	pointer to IIO device
+ *
+ * Obtain and lock both ALS and PROX resources, determine and save device state
+ * (On/Off), cycle device to implement updated parameter, put device back into
+ * proper state, and unlock resource.
+ */
+static int tsl2772_invoke_change(struct iio_dev *indio_dev)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+	int device_status = chip->tsl2772_chip_status;
+	int ret;
+
+	mutex_lock(&chip->als_mutex);
+	mutex_lock(&chip->prox_mutex);
+
+	if (device_status == TSL2772_CHIP_WORKING) {
+		ret = tsl2772_chip_off(indio_dev);
+		if (ret < 0)
+			goto unlock;
+	}
+
+	ret = tsl2772_chip_on(indio_dev);
+
+unlock:
+	mutex_unlock(&chip->prox_mutex);
+	mutex_unlock(&chip->als_mutex);
+
+	return ret;
+}
+
+static int tsl2772_prox_cal(struct iio_dev *indio_dev)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+	int prox_history[MAX_SAMPLES_CAL + 1];
+	int i, ret, mean, max, sample_sum;
+
+	if (chip->settings.prox_max_samples_cal < 1 ||
+	    chip->settings.prox_max_samples_cal > MAX_SAMPLES_CAL)
+		return -EINVAL;
+
+	for (i = 0; i < chip->settings.prox_max_samples_cal; i++) {
+		usleep_range(15000, 17500);
+		ret = tsl2772_get_prox(indio_dev);
+		if (ret < 0)
+			return ret;
+
+		prox_history[i] = chip->prox_data;
+	}
+
+	sample_sum = 0;
+	max = INT_MIN;
+	for (i = 0; i < chip->settings.prox_max_samples_cal; i++) {
+		sample_sum += prox_history[i];
+		max = max(max, prox_history[i]);
+	}
+	mean = sample_sum / chip->settings.prox_max_samples_cal;
+
+	chip->settings.prox_thres_high = (max << 1) - mean;
+
+	return tsl2772_invoke_change(indio_dev);
+}
+
+static int tsl2772_read_avail(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      const int **vals, int *type, int *length,
+			      long mask)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_CALIBSCALE:
+		if (chan->type == IIO_INTENSITY) {
+			*length = ARRAY_SIZE(tsl2772_int_calibscale_avail);
+			*vals = tsl2772_int_calibscale_avail;
+		} else {
+			*length = ARRAY_SIZE(tsl2772_prox_calibscale_avail);
+			*vals = tsl2772_prox_calibscale_avail;
+		}
+		*type = IIO_VAL_INT;
+		return IIO_AVAIL_LIST;
+	case IIO_CHAN_INFO_INT_TIME:
+		*length = ARRAY_SIZE(tsl2772_int_time_avail[chip->id]);
+		*vals = tsl2772_int_time_avail[chip->id];
+		*type = IIO_VAL_INT_PLUS_MICRO;
+		return IIO_AVAIL_RANGE;
+	}
+
+	return -EINVAL;
+}
+
+static ssize_t in_illuminance0_target_input_show(struct device *dev,
+						 struct device_attribute *attr,
+						 char *buf)
+{
+	struct tsl2772_chip *chip = iio_priv(dev_to_iio_dev(dev));
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", chip->settings.als_cal_target);
+}
+
+static ssize_t in_illuminance0_target_input_store(struct device *dev,
+						  struct device_attribute *attr,
+						  const char *buf, size_t len)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+	u16 value;
+	int ret;
+
+	if (kstrtou16(buf, 0, &value))
+		return -EINVAL;
+
+	chip->settings.als_cal_target = value;
+	ret = tsl2772_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
+
+	return len;
+}
+
+static ssize_t in_illuminance0_calibrate_store(struct device *dev,
+					       struct device_attribute *attr,
+					       const char *buf, size_t len)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	bool value;
+	int ret;
+
+	if (kstrtobool(buf, &value) || !value)
+		return -EINVAL;
+
+	ret = tsl2772_als_calibrate(indio_dev);
+	if (ret < 0)
+		return ret;
+
+	ret = tsl2772_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
+
+	return len;
+}
+
+static ssize_t in_illuminance0_lux_table_show(struct device *dev,
+					      struct device_attribute *attr,
+					      char *buf)
+{
+	struct tsl2772_chip *chip = iio_priv(dev_to_iio_dev(dev));
+	int i = 0;
+	int offset = 0;
+
+	while (i < TSL2772_MAX_LUX_TABLE_SIZE) {
+		offset += snprintf(buf + offset, PAGE_SIZE, "%u,%u,",
+			chip->tsl2772_device_lux[i].ch0,
+			chip->tsl2772_device_lux[i].ch1);
+		if (chip->tsl2772_device_lux[i].ch0 == 0) {
+			/*
+			 * We just printed the first "0" entry.
+			 * Now get rid of the extra "," and break.
+			 */
+			offset--;
+			break;
+		}
+		i++;
+	}
+
+	offset += snprintf(buf + offset, PAGE_SIZE, "\n");
+	return offset;
+}
+
+static ssize_t in_illuminance0_lux_table_store(struct device *dev,
+					       struct device_attribute *attr,
+					       const char *buf, size_t len)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+	int value[ARRAY_SIZE(chip->tsl2772_device_lux) * 2 + 1];
+	int n, ret;
+
+	get_options(buf, ARRAY_SIZE(value), value);
+
+	/*
+	 * We now have an array of ints starting at value[1], and
+	 * enumerated by value[0].
+	 * We expect each group of two ints to be one table entry,
+	 * and the last table entry is all 0.
+	 */
+	n = value[0];
+	if ((n % 2) || n < 4 ||
+	    n > ((ARRAY_SIZE(chip->tsl2772_device_lux) - 1) * 2))
+		return -EINVAL;
+
+	if ((value[(n - 1)] | value[n]) != 0)
+		return -EINVAL;
+
+	if (chip->tsl2772_chip_status == TSL2772_CHIP_WORKING) {
+		ret = tsl2772_chip_off(indio_dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* Zero out the table */
+	memset(chip->tsl2772_device_lux, 0, sizeof(chip->tsl2772_device_lux));
+	memcpy(chip->tsl2772_device_lux, &value[1], (value[0] * 4));
+
+	ret = tsl2772_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
+
+	return len;
+}
+
+static ssize_t in_proximity0_calibrate_store(struct device *dev,
+					     struct device_attribute *attr,
+					     const char *buf, size_t len)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	bool value;
+	int ret;
+
+	if (kstrtobool(buf, &value) || !value)
+		return -EINVAL;
+
+	ret = tsl2772_prox_cal(indio_dev);
+	if (ret < 0)
+		return ret;
+
+	ret = tsl2772_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
+
+	return len;
+}
+
+static int tsl2772_read_interrupt_config(struct iio_dev *indio_dev,
+					 const struct iio_chan_spec *chan,
+					 enum iio_event_type type,
+					 enum iio_event_direction dir)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+
+	if (chan->type == IIO_INTENSITY)
+		return chip->settings.als_interrupt_en;
+	else
+		return chip->settings.prox_interrupt_en;
+}
+
+static int tsl2772_write_interrupt_config(struct iio_dev *indio_dev,
+					  const struct iio_chan_spec *chan,
+					  enum iio_event_type type,
+					  enum iio_event_direction dir,
+					  int val)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+
+	if (chan->type == IIO_INTENSITY)
+		chip->settings.als_interrupt_en = val ? true : false;
+	else
+		chip->settings.prox_interrupt_en = val ? true : false;
+
+	return tsl2772_invoke_change(indio_dev);
+}
+
+static int tsl2772_write_event_value(struct iio_dev *indio_dev,
+				     const struct iio_chan_spec *chan,
+				     enum iio_event_type type,
+				     enum iio_event_direction dir,
+				     enum iio_event_info info,
+				     int val, int val2)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+	int ret = -EINVAL, count, persistence;
+	u8 time;
+
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		if (chan->type == IIO_INTENSITY) {
+			switch (dir) {
+			case IIO_EV_DIR_RISING:
+				chip->settings.als_thresh_high = val;
+				ret = 0;
+				break;
+			case IIO_EV_DIR_FALLING:
+				chip->settings.als_thresh_low = val;
+				ret = 0;
+				break;
+			default:
+				break;
+			}
+		} else {
+			switch (dir) {
+			case IIO_EV_DIR_RISING:
+				chip->settings.prox_thres_high = val;
+				ret = 0;
+				break;
+			case IIO_EV_DIR_FALLING:
+				chip->settings.prox_thres_low = val;
+				ret = 0;
+				break;
+			default:
+				break;
+			}
+		}
+		break;
+	case IIO_EV_INFO_PERIOD:
+		if (chan->type == IIO_INTENSITY)
+			time = chip->settings.als_time;
+		else
+			time = chip->settings.prox_time;
+
+		count = 256 - time;
+		persistence = ((val * 1000000) + val2) /
+			(count * tsl2772_int_time_avail[chip->id][3]);
+
+		if (chan->type == IIO_INTENSITY) {
+			/* ALS filter values are 1, 2, 3, 5, 10, 15, ..., 60 */
+			if (persistence > 3)
+				persistence = (persistence / 5) + 3;
+
+			chip->settings.als_persistence = persistence;
+		} else {
+			chip->settings.prox_persistence = persistence;
+		}
+
+		ret = 0;
+		break;
+	default:
+		break;
+	}
+
+	if (ret < 0)
+		return ret;
+
+	return tsl2772_invoke_change(indio_dev);
+}
+
+static int tsl2772_read_event_value(struct iio_dev *indio_dev,
+				    const struct iio_chan_spec *chan,
+				    enum iio_event_type type,
+				    enum iio_event_direction dir,
+				    enum iio_event_info info,
+				    int *val, int *val2)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+	int filter_delay, persistence;
+	u8 time;
+
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		if (chan->type == IIO_INTENSITY) {
+			switch (dir) {
+			case IIO_EV_DIR_RISING:
+				*val = chip->settings.als_thresh_high;
+				return IIO_VAL_INT;
+			case IIO_EV_DIR_FALLING:
+				*val = chip->settings.als_thresh_low;
+				return IIO_VAL_INT;
+			default:
+				return -EINVAL;
+			}
+		} else {
+			switch (dir) {
+			case IIO_EV_DIR_RISING:
+				*val = chip->settings.prox_thres_high;
+				return IIO_VAL_INT;
+			case IIO_EV_DIR_FALLING:
+				*val = chip->settings.prox_thres_low;
+				return IIO_VAL_INT;
+			default:
+				return -EINVAL;
+			}
+		}
+		break;
+	case IIO_EV_INFO_PERIOD:
+		if (chan->type == IIO_INTENSITY) {
+			time = chip->settings.als_time;
+			persistence = chip->settings.als_persistence;
+
+			/* ALS filter values are 1, 2, 3, 5, 10, 15, ..., 60 */
+			if (persistence > 3)
+				persistence = (persistence - 3) * 5;
+		} else {
+			time = chip->settings.prox_time;
+			persistence = chip->settings.prox_persistence;
+		}
+
+		filter_delay = persistence * (256 - time) *
+			tsl2772_int_time_avail[chip->id][3];
+
+		*val = filter_delay / 1000000;
+		*val2 = filter_delay % 1000000;
+		return IIO_VAL_INT_PLUS_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int tsl2772_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val,
+			    int *val2,
+			    long mask)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_PROCESSED:
+		switch (chan->type) {
+		case IIO_LIGHT:
+			tsl2772_get_lux(indio_dev);
+			*val = chip->als_cur_info.lux;
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_RAW:
+		switch (chan->type) {
+		case IIO_INTENSITY:
+			tsl2772_get_lux(indio_dev);
+			if (chan->channel == 0)
+				*val = chip->als_cur_info.als_ch0;
+			else
+				*val = chip->als_cur_info.als_ch1;
+			return IIO_VAL_INT;
+		case IIO_PROXIMITY:
+			tsl2772_get_prox(indio_dev);
+			*val = chip->prox_data;
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case IIO_CHAN_INFO_CALIBSCALE:
+		if (chan->type == IIO_LIGHT)
+			*val = tsl2772_als_gain[chip->settings.als_gain];
+		else
+			*val = tsl2772_prox_gain[chip->settings.prox_gain];
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_CALIBBIAS:
+		*val = chip->settings.als_gain_trim;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_INT_TIME:
+		*val = 0;
+		*val2 = (256 - chip->settings.als_time) *
+			tsl2772_int_time_avail[chip->id][3];
+		return IIO_VAL_INT_PLUS_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int tsl2772_write_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int val,
+			     int val2,
+			     long mask)
+{
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_CALIBSCALE:
+		if (chan->type == IIO_INTENSITY) {
+			switch (val) {
+			case 1:
+				chip->settings.als_gain = 0;
+				break;
+			case 8:
+				chip->settings.als_gain = 1;
+				break;
+			case 16:
+				chip->settings.als_gain = 2;
+				break;
+			case 120:
+				chip->settings.als_gain = 3;
+				break;
+			default:
+				return -EINVAL;
+			}
+		} else {
+			switch (val) {
+			case 1:
+				chip->settings.prox_gain = 0;
+				break;
+			case 2:
+				chip->settings.prox_gain = 1;
+				break;
+			case 4:
+				chip->settings.prox_gain = 2;
+				break;
+			case 8:
+				chip->settings.prox_gain = 3;
+				break;
+			default:
+				return -EINVAL;
+			}
+		}
+		break;
+	case IIO_CHAN_INFO_CALIBBIAS:
+		if (val < TSL2772_ALS_GAIN_TRIM_MIN ||
+		    val > TSL2772_ALS_GAIN_TRIM_MAX)
+			return -EINVAL;
+
+		chip->settings.als_gain_trim = val;
+		break;
+	case IIO_CHAN_INFO_INT_TIME:
+		if (val != 0 || val2 < tsl2772_int_time_avail[chip->id][1] ||
+		    val2 > tsl2772_int_time_avail[chip->id][5])
+			return -EINVAL;
+
+		chip->settings.als_time = 256 -
+			(val2 / tsl2772_int_time_avail[chip->id][3]);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return tsl2772_invoke_change(indio_dev);
+}
+
+static DEVICE_ATTR_RW(in_illuminance0_target_input);
+
+static DEVICE_ATTR_WO(in_illuminance0_calibrate);
+
+static DEVICE_ATTR_WO(in_proximity0_calibrate);
+
+static DEVICE_ATTR_RW(in_illuminance0_lux_table);
+
+/* Use the default register values to identify the Taos device */
+static int tsl2772_device_id_verif(int id, int target)
+{
+	switch (target) {
+	case tsl2571:
+	case tsl2671:
+	case tsl2771:
+		return (id & 0xf0) == TRITON_ID;
+	case tmd2671:
+	case tmd2771:
+		return (id & 0xf0) == HALIBUT_ID;
+	case tsl2572:
+	case tsl2672:
+	case tmd2672:
+	case tsl2772:
+	case tmd2772:
+		return (id & 0xf0) == SWORDFISH_ID;
+	}
+
+	return -EINVAL;
+}
+
+static irqreturn_t tsl2772_event_handler(int irq, void *private)
+{
+	struct iio_dev *indio_dev = private;
+	struct tsl2772_chip *chip = iio_priv(indio_dev);
+	s64 timestamp = iio_get_time_ns(indio_dev);
+	int ret;
+
+	ret = tsl2772_read_status(chip);
+	if (ret < 0)
+		return IRQ_HANDLED;
+
+	/* What type of interrupt do we need to process */
+	if (ret & TSL2772_STA_PRX_INTR) {
+		iio_push_event(indio_dev,
+			       IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY,
+						    0,
+						    IIO_EV_TYPE_THRESH,
+						    IIO_EV_DIR_EITHER),
+			       timestamp);
+	}
+
+	if (ret & TSL2772_STA_ALS_INTR) {
+		iio_push_event(indio_dev,
+			       IIO_UNMOD_EVENT_CODE(IIO_LIGHT,
+						    0,
+						    IIO_EV_TYPE_THRESH,
+						    IIO_EV_DIR_EITHER),
+			       timestamp);
+	}
+
+	ret = i2c_smbus_write_byte(chip->client,
+				   TSL2772_CMD_REG | TSL2772_CMD_SPL_FN |
+				   TSL2772_CMD_PROXALS_INT_CLR);
+	if (ret < 0)
+		dev_err(&chip->client->dev,
+			"%s: failed to clear interrupt status: %d\n",
+			__func__, ret);
+
+	return IRQ_HANDLED;
+}
+
+static struct attribute *tsl2772_ALS_device_attrs[] = {
+	&dev_attr_in_illuminance0_target_input.attr,
+	&dev_attr_in_illuminance0_calibrate.attr,
+	&dev_attr_in_illuminance0_lux_table.attr,
+	NULL
+};
+
+static struct attribute *tsl2772_PRX_device_attrs[] = {
+	&dev_attr_in_proximity0_calibrate.attr,
+	NULL
+};
+
+static struct attribute *tsl2772_ALSPRX_device_attrs[] = {
+	&dev_attr_in_illuminance0_target_input.attr,
+	&dev_attr_in_illuminance0_calibrate.attr,
+	&dev_attr_in_illuminance0_lux_table.attr,
+	NULL
+};
+
+static struct attribute *tsl2772_PRX2_device_attrs[] = {
+	&dev_attr_in_proximity0_calibrate.attr,
+	NULL
+};
+
+static struct attribute *tsl2772_ALSPRX2_device_attrs[] = {
+	&dev_attr_in_illuminance0_target_input.attr,
+	&dev_attr_in_illuminance0_calibrate.attr,
+	&dev_attr_in_illuminance0_lux_table.attr,
+	&dev_attr_in_proximity0_calibrate.attr,
+	NULL
+};
+
+static const struct attribute_group tsl2772_device_attr_group_tbl[] = {
+	[ALS] = {
+		.attrs = tsl2772_ALS_device_attrs,
+	},
+	[PRX] = {
+		.attrs = tsl2772_PRX_device_attrs,
+	},
+	[ALSPRX] = {
+		.attrs = tsl2772_ALSPRX_device_attrs,
+	},
+	[PRX2] = {
+		.attrs = tsl2772_PRX2_device_attrs,
+	},
+	[ALSPRX2] = {
+		.attrs = tsl2772_ALSPRX2_device_attrs,
+	},
+};
+
+#define TSL2772_DEVICE_INFO(type)[type] = \
+	{ \
+		.attrs = &tsl2772_device_attr_group_tbl[type], \
+		.read_raw = &tsl2772_read_raw, \
+		.read_avail = &tsl2772_read_avail, \
+		.write_raw = &tsl2772_write_raw, \
+		.read_event_value = &tsl2772_read_event_value, \
+		.write_event_value = &tsl2772_write_event_value, \
+		.read_event_config = &tsl2772_read_interrupt_config, \
+		.write_event_config = &tsl2772_write_interrupt_config, \
+	}
+
+static const struct iio_info tsl2772_device_info[] = {
+	TSL2772_DEVICE_INFO(ALS),
+	TSL2772_DEVICE_INFO(PRX),
+	TSL2772_DEVICE_INFO(ALSPRX),
+	TSL2772_DEVICE_INFO(PRX2),
+	TSL2772_DEVICE_INFO(ALSPRX2),
+};
+
+static const struct iio_event_spec tsl2772_events[] = {
+	{
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_RISING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE),
+	}, {
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_FALLING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE),
+	}, {
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_EITHER,
+		.mask_separate = BIT(IIO_EV_INFO_PERIOD) |
+			BIT(IIO_EV_INFO_ENABLE),
+	},
+};
+
+static const struct tsl2772_chip_info tsl2772_chip_info_tbl[] = {
+	[ALS] = {
+		.channel_with_events = {
+			{
+			.type = IIO_LIGHT,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE) |
+				BIT(IIO_CHAN_INFO_CALIBBIAS),
+			.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			.event_spec = tsl2772_events,
+			.num_event_specs = ARRAY_SIZE(tsl2772_events),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 1,
+			},
+		},
+		.channel_without_events = {
+			{
+			.type = IIO_LIGHT,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE) |
+				BIT(IIO_CHAN_INFO_CALIBBIAS),
+			.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 1,
+			},
+		},
+		.chan_table_elements = 3,
+		.info = &tsl2772_device_info[ALS],
+	},
+	[PRX] = {
+		.channel_with_events = {
+			{
+			.type = IIO_PROXIMITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+			.event_spec = tsl2772_events,
+			.num_event_specs = ARRAY_SIZE(tsl2772_events),
+			},
+		},
+		.channel_without_events = {
+			{
+			.type = IIO_PROXIMITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+			},
+		},
+		.chan_table_elements = 1,
+		.info = &tsl2772_device_info[PRX],
+	},
+	[ALSPRX] = {
+		.channel_with_events = {
+			{
+			.type = IIO_LIGHT,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE) |
+				BIT(IIO_CHAN_INFO_CALIBBIAS),
+			.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			.event_spec = tsl2772_events,
+			.num_event_specs = ARRAY_SIZE(tsl2772_events),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 1,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+			}, {
+			.type = IIO_PROXIMITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+			.event_spec = tsl2772_events,
+			.num_event_specs = ARRAY_SIZE(tsl2772_events),
+			},
+		},
+		.channel_without_events = {
+			{
+			.type = IIO_LIGHT,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE) |
+				BIT(IIO_CHAN_INFO_CALIBBIAS),
+			.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 1,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+			}, {
+			.type = IIO_PROXIMITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+			},
+		},
+		.chan_table_elements = 4,
+		.info = &tsl2772_device_info[ALSPRX],
+	},
+	[PRX2] = {
+		.channel_with_events = {
+			{
+			.type = IIO_PROXIMITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			.event_spec = tsl2772_events,
+			.num_event_specs = ARRAY_SIZE(tsl2772_events),
+			},
+		},
+		.channel_without_events = {
+			{
+			.type = IIO_PROXIMITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			},
+		},
+		.chan_table_elements = 1,
+		.info = &tsl2772_device_info[PRX2],
+	},
+	[ALSPRX2] = {
+		.channel_with_events = {
+			{
+			.type = IIO_LIGHT,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE) |
+				BIT(IIO_CHAN_INFO_CALIBBIAS),
+			.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			.event_spec = tsl2772_events,
+			.num_event_specs = ARRAY_SIZE(tsl2772_events),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 1,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+			}, {
+			.type = IIO_PROXIMITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			.event_spec = tsl2772_events,
+			.num_event_specs = ARRAY_SIZE(tsl2772_events),
+			},
+		},
+		.channel_without_events = {
+			{
+			.type = IIO_LIGHT,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE) |
+				BIT(IIO_CHAN_INFO_CALIBBIAS),
+			.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_INT_TIME) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			}, {
+			.type = IIO_INTENSITY,
+			.indexed = 1,
+			.channel = 1,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+			}, {
+			.type = IIO_PROXIMITY,
+			.indexed = 1,
+			.channel = 0,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_CALIBSCALE),
+			},
+		},
+		.chan_table_elements = 4,
+		.info = &tsl2772_device_info[ALSPRX2],
+	},
+};
+
+static int tsl2772_probe(struct i2c_client *clientp,
+			 const struct i2c_device_id *id)
+{
+	struct iio_dev *indio_dev;
+	struct tsl2772_chip *chip;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&clientp->dev, sizeof(*chip));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	chip = iio_priv(indio_dev);
+	chip->client = clientp;
+	i2c_set_clientdata(clientp, indio_dev);
+
+	ret = i2c_smbus_read_byte_data(chip->client,
+				       TSL2772_CMD_REG | TSL2772_CHIPID);
+	if (ret < 0)
+		return ret;
+
+	if (tsl2772_device_id_verif(ret, id->driver_data) <= 0) {
+		dev_info(&chip->client->dev,
+			 "%s: i2c device found does not match expected id\n",
+				__func__);
+		return -EINVAL;
+	}
+
+	ret = i2c_smbus_write_byte(clientp, TSL2772_CMD_REG | TSL2772_CNTRL);
+	if (ret < 0) {
+		dev_err(&clientp->dev,
+			"%s: Failed to write to CMD register: %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	mutex_init(&chip->als_mutex);
+	mutex_init(&chip->prox_mutex);
+
+	chip->tsl2772_chip_status = TSL2772_CHIP_UNKNOWN;
+	chip->pdata = dev_get_platdata(&clientp->dev);
+	chip->id = id->driver_data;
+	chip->chip_info =
+		&tsl2772_chip_info_tbl[device_channel_config[id->driver_data]];
+
+	indio_dev->info = chip->chip_info->info;
+	indio_dev->dev.parent = &clientp->dev;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->name = chip->client->name;
+	indio_dev->num_channels = chip->chip_info->chan_table_elements;
+
+	if (clientp->irq) {
+		indio_dev->channels = chip->chip_info->channel_with_events;
+
+		ret = devm_request_threaded_irq(&clientp->dev, clientp->irq,
+						NULL,
+						&tsl2772_event_handler,
+						IRQF_TRIGGER_FALLING |
+						IRQF_ONESHOT,
+						"TSL2772_event",
+						indio_dev);
+		if (ret) {
+			dev_err(&clientp->dev,
+				"%s: irq request failed\n", __func__);
+			return ret;
+		}
+	} else {
+		indio_dev->channels = chip->chip_info->channel_without_events;
+	}
+
+	tsl2772_defaults(chip);
+	ret = tsl2772_chip_on(indio_dev);
+	if (ret < 0)
+		return ret;
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		tsl2772_chip_off(indio_dev);
+		dev_err(&clientp->dev,
+			"%s: iio registration failed\n", __func__);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int tsl2772_suspend(struct device *dev)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+
+	return tsl2772_chip_off(indio_dev);
+}
+
+static int tsl2772_resume(struct device *dev)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+
+	return tsl2772_chip_on(indio_dev);
+}
+
+static int tsl2772_remove(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+	tsl2772_chip_off(indio_dev);
+
+	iio_device_unregister(indio_dev);
+
+	return 0;
+}
+
+static const struct i2c_device_id tsl2772_idtable[] = {
+	{ "tsl2571", tsl2571 },
+	{ "tsl2671", tsl2671 },
+	{ "tmd2671", tmd2671 },
+	{ "tsl2771", tsl2771 },
+	{ "tmd2771", tmd2771 },
+	{ "tsl2572", tsl2572 },
+	{ "tsl2672", tsl2672 },
+	{ "tmd2672", tmd2672 },
+	{ "tsl2772", tsl2772 },
+	{ "tmd2772", tmd2772 },
+	{}
+};
+
+MODULE_DEVICE_TABLE(i2c, tsl2772_idtable);
+
+static const struct of_device_id tsl2772_of_match[] = {
+	{ .compatible = "amstaos,tsl2571" },
+	{ .compatible = "amstaos,tsl2671" },
+	{ .compatible = "amstaos,tmd2671" },
+	{ .compatible = "amstaos,tsl2771" },
+	{ .compatible = "amstaos,tmd2771" },
+	{ .compatible = "amstaos,tsl2572" },
+	{ .compatible = "amstaos,tsl2672" },
+	{ .compatible = "amstaos,tmd2672" },
+	{ .compatible = "amstaos,tsl2772" },
+	{ .compatible = "amstaos,tmd2772" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, tsl2772_of_match);
+
+static const struct dev_pm_ops tsl2772_pm_ops = {
+	.suspend = tsl2772_suspend,
+	.resume  = tsl2772_resume,
+};
+
+static struct i2c_driver tsl2772_driver = {
+	.driver = {
+		.name = "tsl2772",
+		.of_match_table = tsl2772_of_match,
+		.pm = &tsl2772_pm_ops,
+	},
+	.id_table = tsl2772_idtable,
+	.probe = tsl2772_probe,
+	.remove = tsl2772_remove,
+};
+
+module_i2c_driver(tsl2772_driver);
+
+MODULE_AUTHOR("J. August Brenner <Jon.Brenner@ams.com>");
+MODULE_AUTHOR("Brian Masney <masneyb@onstation.org>");
+MODULE_DESCRIPTION("TAOS tsl2772 ambient and proximity light sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/magnetometer/mag3110.c b/drivers/iio/magnetometer/mag3110.c
index b34ace76d31b..f063355480ba 100644
--- a/drivers/iio/magnetometer/mag3110.c
+++ b/drivers/iio/magnetometer/mag3110.c
@@ -26,6 +26,7 @@
 #define MAG3110_OUT_Y 0x03
 #define MAG3110_OUT_Z 0x05
 #define MAG3110_WHO_AM_I 0x07
+#define MAG3110_SYSMOD 0x08
 #define MAG3110_OFF_X 0x09 /* MSB first */
 #define MAG3110_OFF_Y 0x0b
 #define MAG3110_OFF_Z 0x0d
@@ -39,6 +40,8 @@
 #define MAG3110_CTRL_DR_SHIFT 5
 #define MAG3110_CTRL_DR_DEFAULT 0
 
+#define MAG3110_SYSMOD_MODE_MASK GENMASK(1, 0)
+
 #define MAG3110_CTRL_TM BIT(1) /* trigger single measurement */
 #define MAG3110_CTRL_AC BIT(0) /* continuous measurements */
 
@@ -52,17 +55,20 @@ struct mag3110_data {
 	struct i2c_client *client;
 	struct mutex lock;
 	u8 ctrl_reg1;
+	int sleep_val;
 };
 
 static int mag3110_request(struct mag3110_data *data)
 {
 	int ret, tries = 150;
 
-	/* trigger measurement */
-	ret = i2c_smbus_write_byte_data(data->client, MAG3110_CTRL_REG1,
-		data->ctrl_reg1 | MAG3110_CTRL_TM);
-	if (ret < 0)
-		return ret;
+	if ((data->ctrl_reg1 & MAG3110_CTRL_AC) == 0) {
+		/* trigger measurement */
+		ret = i2c_smbus_write_byte_data(data->client, MAG3110_CTRL_REG1,
+			data->ctrl_reg1 | MAG3110_CTRL_TM);
+		if (ret < 0)
+			return ret;
+	}
 
 	while (tries-- > 0) {
 		ret = i2c_smbus_read_byte_data(data->client, MAG3110_STATUS);
@@ -71,7 +77,11 @@ static int mag3110_request(struct mag3110_data *data)
 		/* wait for data ready */
 		if ((ret & MAG3110_STATUS_DRDY) == MAG3110_STATUS_DRDY)
 			break;
-		msleep(20);
+
+		if (data->sleep_val <= 20)
+			usleep_range(data->sleep_val * 250, data->sleep_val * 500);
+		else
+			msleep(20);
 	}
 
 	if (tries < 0) {
@@ -144,6 +154,117 @@ static int mag3110_get_samp_freq_index(struct mag3110_data *data,
 		val2);
 }
 
+static int mag3110_calculate_sleep(struct mag3110_data *data)
+{
+	int ret, i = data->ctrl_reg1 >> MAG3110_CTRL_DR_SHIFT;
+
+	if (mag3110_samp_freq[i][0] > 0)
+		ret = 1000 / mag3110_samp_freq[i][0];
+	else
+		ret = 1000;
+
+	return ret == 0 ? 1 : ret;
+}
+
+static int mag3110_standby(struct mag3110_data *data)
+{
+	return i2c_smbus_write_byte_data(data->client, MAG3110_CTRL_REG1,
+		data->ctrl_reg1 & ~MAG3110_CTRL_AC);
+}
+
+static int mag3110_wait_standby(struct mag3110_data *data)
+{
+	int ret, tries = 30;
+
+	/*
+	 * Takes up to 1/ODR to come out of active mode into stby
+	 * Longest expected period is 12.5seconds.
+	 * We'll sleep for 500ms between checks
+	 */
+	while (tries-- > 0) {
+		ret = i2c_smbus_read_byte_data(data->client, MAG3110_SYSMOD);
+		if (ret < 0) {
+			dev_err(&data->client->dev, "i2c error\n");
+			return ret;
+		}
+		/* wait for standby */
+		if ((ret & MAG3110_SYSMOD_MODE_MASK) == 0)
+			break;
+
+		msleep_interruptible(500);
+	}
+
+	if (tries < 0) {
+		dev_err(&data->client->dev, "device not entering standby mode\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int mag3110_active(struct mag3110_data *data)
+{
+	return i2c_smbus_write_byte_data(data->client, MAG3110_CTRL_REG1,
+					 data->ctrl_reg1);
+}
+
+/* returns >0 if active, 0 if in standby and <0 on error */
+static int mag3110_is_active(struct mag3110_data *data)
+{
+	int reg;
+
+	reg = i2c_smbus_read_byte_data(data->client, MAG3110_CTRL_REG1);
+	if (reg < 0)
+		return reg;
+
+	return reg & MAG3110_CTRL_AC;
+}
+
+static int mag3110_change_config(struct mag3110_data *data, u8 reg, u8 val)
+{
+	int ret;
+	int is_active;
+
+	mutex_lock(&data->lock);
+
+	is_active = mag3110_is_active(data);
+	if (is_active < 0) {
+		ret = is_active;
+		goto fail;
+	}
+
+	/* config can only be changed when in standby */
+	if (is_active > 0) {
+		ret = mag3110_standby(data);
+		if (ret < 0)
+			goto fail;
+	}
+
+	/*
+	 * After coming out of active we must wait for the part
+	 * to transition to STBY. This can take up to 1 /ODR to occur
+	 */
+	ret = mag3110_wait_standby(data);
+	if (ret < 0)
+		goto fail;
+
+	ret = i2c_smbus_write_byte_data(data->client, reg, val);
+	if (ret < 0)
+		goto fail;
+
+	if (is_active > 0) {
+		ret = mag3110_active(data);
+		if (ret < 0)
+			goto fail;
+	}
+
+	ret = 0;
+fail:
+	mutex_unlock(&data->lock);
+
+	return ret;
+}
+
 static int mag3110_read_raw(struct iio_dev *indio_dev,
 			    struct iio_chan_spec const *chan,
 			    int *val, int *val2, long mask)
@@ -235,11 +356,15 @@ static int mag3110_write_raw(struct iio_dev *indio_dev,
 			ret = -EINVAL;
 			break;
 		}
-
-		data->ctrl_reg1 &= ~MAG3110_CTRL_DR_MASK;
+		data->ctrl_reg1 &= 0xff & ~MAG3110_CTRL_DR_MASK
+					& ~MAG3110_CTRL_AC;
 		data->ctrl_reg1 |= rate << MAG3110_CTRL_DR_SHIFT;
-		ret = i2c_smbus_write_byte_data(data->client,
-			MAG3110_CTRL_REG1, data->ctrl_reg1);
+		data->sleep_val = mag3110_calculate_sleep(data);
+		if (data->sleep_val < 40)
+			data->ctrl_reg1 |= MAG3110_CTRL_AC;
+
+		ret = mag3110_change_config(data, MAG3110_CTRL_REG1,
+					    data->ctrl_reg1);
 		break;
 	case IIO_CHAN_INFO_CALIBBIAS:
 		if (val < -10000 || val > 10000) {
@@ -337,12 +462,6 @@ static const struct iio_info mag3110_info = {
 
 static const unsigned long mag3110_scan_masks[] = {0x7, 0xf, 0};
 
-static int mag3110_standby(struct mag3110_data *data)
-{
-	return i2c_smbus_write_byte_data(data->client, MAG3110_CTRL_REG1,
-		data->ctrl_reg1 & ~MAG3110_CTRL_AC);
-}
-
 static int mag3110_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
@@ -374,8 +493,11 @@ static int mag3110_probe(struct i2c_client *client,
 	indio_dev->available_scan_masks = mag3110_scan_masks;
 
 	data->ctrl_reg1 = MAG3110_CTRL_DR_DEFAULT << MAG3110_CTRL_DR_SHIFT;
-	ret = i2c_smbus_write_byte_data(client, MAG3110_CTRL_REG1,
-		data->ctrl_reg1);
+	data->sleep_val = mag3110_calculate_sleep(data);
+	if (data->sleep_val < 40)
+		data->ctrl_reg1 |= MAG3110_CTRL_AC;
+
+	ret = mag3110_change_config(data, MAG3110_CTRL_REG1, data->ctrl_reg1);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/iio/potentiometer/mcp4018.c b/drivers/iio/potentiometer/mcp4018.c
index 601b25d1f387..320a7c929777 100644
--- a/drivers/iio/potentiometer/mcp4018.c
+++ b/drivers/iio/potentiometer/mcp4018.c
@@ -99,6 +99,23 @@ static const struct iio_info mcp4018_info = {
 	.write_raw = mcp4018_write_raw,
 };
 
+static const struct i2c_device_id mcp4018_id[] = {
+	{ "mcp4017-502", MCP4018_502 },
+	{ "mcp4017-103", MCP4018_103 },
+	{ "mcp4017-503", MCP4018_503 },
+	{ "mcp4017-104", MCP4018_104 },
+	{ "mcp4018-502", MCP4018_502 },
+	{ "mcp4018-103", MCP4018_103 },
+	{ "mcp4018-503", MCP4018_503 },
+	{ "mcp4018-104", MCP4018_104 },
+	{ "mcp4019-502", MCP4018_502 },
+	{ "mcp4019-103", MCP4018_103 },
+	{ "mcp4019-503", MCP4018_503 },
+	{ "mcp4019-104", MCP4018_104 },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, mcp4018_id);
+
 #ifdef CONFIG_OF
 
 #define MCP4018_COMPATIBLE(of_compatible, cfg) {	\
@@ -125,8 +142,7 @@ MODULE_DEVICE_TABLE(of, mcp4018_of_match);
 
 #endif
 
-static int mcp4018_probe(struct i2c_client *client,
-			 const struct i2c_device_id *id)
+static int mcp4018_probe(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
 	struct mcp4018_data *data;
@@ -150,7 +166,7 @@ static int mcp4018_probe(struct i2c_client *client,
 	if (match)
 		data->cfg = of_device_get_match_data(dev);
 	else
-		data->cfg = &mcp4018_cfg[id->driver_data];
+		data->cfg = &mcp4018_cfg[i2c_match_id(mcp4018_id, client)->driver_data];
 
 	indio_dev->dev.parent = dev;
 	indio_dev->info = &mcp4018_info;
@@ -161,29 +177,12 @@ static int mcp4018_probe(struct i2c_client *client,
 	return devm_iio_device_register(dev, indio_dev);
 }
 
-static const struct i2c_device_id mcp4018_id[] = {
-	{ "mcp4017-502", MCP4018_502 },
-	{ "mcp4017-103", MCP4018_103 },
-	{ "mcp4017-503", MCP4018_503 },
-	{ "mcp4017-104", MCP4018_104 },
-	{ "mcp4018-502", MCP4018_502 },
-	{ "mcp4018-103", MCP4018_103 },
-	{ "mcp4018-503", MCP4018_503 },
-	{ "mcp4018-104", MCP4018_104 },
-	{ "mcp4019-502", MCP4018_502 },
-	{ "mcp4019-103", MCP4018_103 },
-	{ "mcp4019-503", MCP4018_503 },
-	{ "mcp4019-104", MCP4018_104 },
-	{}
-};
-MODULE_DEVICE_TABLE(i2c, mcp4018_id);
-
 static struct i2c_driver mcp4018_driver = {
 	.driver = {
 		.name	= "mcp4018",
 		.of_match_table = of_match_ptr(mcp4018_of_match),
 	},
-	.probe		= mcp4018_probe,
+	.probe_new	= mcp4018_probe,
 	.id_table	= mcp4018_id,
 };
 
diff --git a/drivers/iio/potentiometer/mcp4531.c b/drivers/iio/potentiometer/mcp4531.c
index 114ab876fcc6..df894af6cccb 100644
--- a/drivers/iio/potentiometer/mcp4531.c
+++ b/drivers/iio/potentiometer/mcp4531.c
@@ -209,6 +209,75 @@ static const struct iio_info mcp4531_info = {
 	.write_raw = mcp4531_write_raw,
 };
 
+static const struct i2c_device_id mcp4531_id[] = {
+	{ "mcp4531-502", MCP453x_502 },
+	{ "mcp4531-103", MCP453x_103 },
+	{ "mcp4531-503", MCP453x_503 },
+	{ "mcp4531-104", MCP453x_104 },
+	{ "mcp4532-502", MCP453x_502 },
+	{ "mcp4532-103", MCP453x_103 },
+	{ "mcp4532-503", MCP453x_503 },
+	{ "mcp4532-104", MCP453x_104 },
+	{ "mcp4541-502", MCP454x_502 },
+	{ "mcp4541-103", MCP454x_103 },
+	{ "mcp4541-503", MCP454x_503 },
+	{ "mcp4541-104", MCP454x_104 },
+	{ "mcp4542-502", MCP454x_502 },
+	{ "mcp4542-103", MCP454x_103 },
+	{ "mcp4542-503", MCP454x_503 },
+	{ "mcp4542-104", MCP454x_104 },
+	{ "mcp4551-502", MCP455x_502 },
+	{ "mcp4551-103", MCP455x_103 },
+	{ "mcp4551-503", MCP455x_503 },
+	{ "mcp4551-104", MCP455x_104 },
+	{ "mcp4552-502", MCP455x_502 },
+	{ "mcp4552-103", MCP455x_103 },
+	{ "mcp4552-503", MCP455x_503 },
+	{ "mcp4552-104", MCP455x_104 },
+	{ "mcp4561-502", MCP456x_502 },
+	{ "mcp4561-103", MCP456x_103 },
+	{ "mcp4561-503", MCP456x_503 },
+	{ "mcp4561-104", MCP456x_104 },
+	{ "mcp4562-502", MCP456x_502 },
+	{ "mcp4562-103", MCP456x_103 },
+	{ "mcp4562-503", MCP456x_503 },
+	{ "mcp4562-104", MCP456x_104 },
+	{ "mcp4631-502", MCP463x_502 },
+	{ "mcp4631-103", MCP463x_103 },
+	{ "mcp4631-503", MCP463x_503 },
+	{ "mcp4631-104", MCP463x_104 },
+	{ "mcp4632-502", MCP463x_502 },
+	{ "mcp4632-103", MCP463x_103 },
+	{ "mcp4632-503", MCP463x_503 },
+	{ "mcp4632-104", MCP463x_104 },
+	{ "mcp4641-502", MCP464x_502 },
+	{ "mcp4641-103", MCP464x_103 },
+	{ "mcp4641-503", MCP464x_503 },
+	{ "mcp4641-104", MCP464x_104 },
+	{ "mcp4642-502", MCP464x_502 },
+	{ "mcp4642-103", MCP464x_103 },
+	{ "mcp4642-503", MCP464x_503 },
+	{ "mcp4642-104", MCP464x_104 },
+	{ "mcp4651-502", MCP465x_502 },
+	{ "mcp4651-103", MCP465x_103 },
+	{ "mcp4651-503", MCP465x_503 },
+	{ "mcp4651-104", MCP465x_104 },
+	{ "mcp4652-502", MCP465x_502 },
+	{ "mcp4652-103", MCP465x_103 },
+	{ "mcp4652-503", MCP465x_503 },
+	{ "mcp4652-104", MCP465x_104 },
+	{ "mcp4661-502", MCP466x_502 },
+	{ "mcp4661-103", MCP466x_103 },
+	{ "mcp4661-503", MCP466x_503 },
+	{ "mcp4661-104", MCP466x_104 },
+	{ "mcp4662-502", MCP466x_502 },
+	{ "mcp4662-103", MCP466x_103 },
+	{ "mcp4662-503", MCP466x_503 },
+	{ "mcp4662-104", MCP466x_104 },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, mcp4531_id);
+
 #ifdef CONFIG_OF
 
 #define MCP4531_COMPATIBLE(of_compatible, cfg) {	\
@@ -286,8 +355,7 @@ static const struct of_device_id mcp4531_of_match[] = {
 MODULE_DEVICE_TABLE(of, mcp4531_of_match);
 #endif
 
-static int mcp4531_probe(struct i2c_client *client,
-			 const struct i2c_device_id *id)
+static int mcp4531_probe(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
 	struct mcp4531_data *data;
@@ -311,7 +379,7 @@ static int mcp4531_probe(struct i2c_client *client,
 	if (match)
 		data->cfg = of_device_get_match_data(dev);
 	else
-		data->cfg = &mcp4531_cfg[id->driver_data];
+		data->cfg = &mcp4531_cfg[i2c_match_id(mcp4531_id, client)->driver_data];
 
 	indio_dev->dev.parent = dev;
 	indio_dev->info = &mcp4531_info;
@@ -322,81 +390,12 @@ static int mcp4531_probe(struct i2c_client *client,
 	return devm_iio_device_register(dev, indio_dev);
 }
 
-static const struct i2c_device_id mcp4531_id[] = {
-	{ "mcp4531-502", MCP453x_502 },
-	{ "mcp4531-103", MCP453x_103 },
-	{ "mcp4531-503", MCP453x_503 },
-	{ "mcp4531-104", MCP453x_104 },
-	{ "mcp4532-502", MCP453x_502 },
-	{ "mcp4532-103", MCP453x_103 },
-	{ "mcp4532-503", MCP453x_503 },
-	{ "mcp4532-104", MCP453x_104 },
-	{ "mcp4541-502", MCP454x_502 },
-	{ "mcp4541-103", MCP454x_103 },
-	{ "mcp4541-503", MCP454x_503 },
-	{ "mcp4541-104", MCP454x_104 },
-	{ "mcp4542-502", MCP454x_502 },
-	{ "mcp4542-103", MCP454x_103 },
-	{ "mcp4542-503", MCP454x_503 },
-	{ "mcp4542-104", MCP454x_104 },
-	{ "mcp4551-502", MCP455x_502 },
-	{ "mcp4551-103", MCP455x_103 },
-	{ "mcp4551-503", MCP455x_503 },
-	{ "mcp4551-104", MCP455x_104 },
-	{ "mcp4552-502", MCP455x_502 },
-	{ "mcp4552-103", MCP455x_103 },
-	{ "mcp4552-503", MCP455x_503 },
-	{ "mcp4552-104", MCP455x_104 },
-	{ "mcp4561-502", MCP456x_502 },
-	{ "mcp4561-103", MCP456x_103 },
-	{ "mcp4561-503", MCP456x_503 },
-	{ "mcp4561-104", MCP456x_104 },
-	{ "mcp4562-502", MCP456x_502 },
-	{ "mcp4562-103", MCP456x_103 },
-	{ "mcp4562-503", MCP456x_503 },
-	{ "mcp4562-104", MCP456x_104 },
-	{ "mcp4631-502", MCP463x_502 },
-	{ "mcp4631-103", MCP463x_103 },
-	{ "mcp4631-503", MCP463x_503 },
-	{ "mcp4631-104", MCP463x_104 },
-	{ "mcp4632-502", MCP463x_502 },
-	{ "mcp4632-103", MCP463x_103 },
-	{ "mcp4632-503", MCP463x_503 },
-	{ "mcp4632-104", MCP463x_104 },
-	{ "mcp4641-502", MCP464x_502 },
-	{ "mcp4641-103", MCP464x_103 },
-	{ "mcp4641-503", MCP464x_503 },
-	{ "mcp4641-104", MCP464x_104 },
-	{ "mcp4642-502", MCP464x_502 },
-	{ "mcp4642-103", MCP464x_103 },
-	{ "mcp4642-503", MCP464x_503 },
-	{ "mcp4642-104", MCP464x_104 },
-	{ "mcp4651-502", MCP465x_502 },
-	{ "mcp4651-103", MCP465x_103 },
-	{ "mcp4651-503", MCP465x_503 },
-	{ "mcp4651-104", MCP465x_104 },
-	{ "mcp4652-502", MCP465x_502 },
-	{ "mcp4652-103", MCP465x_103 },
-	{ "mcp4652-503", MCP465x_503 },
-	{ "mcp4652-104", MCP465x_104 },
-	{ "mcp4661-502", MCP466x_502 },
-	{ "mcp4661-103", MCP466x_103 },
-	{ "mcp4661-503", MCP466x_503 },
-	{ "mcp4661-104", MCP466x_104 },
-	{ "mcp4662-502", MCP466x_502 },
-	{ "mcp4662-103", MCP466x_103 },
-	{ "mcp4662-503", MCP466x_503 },
-	{ "mcp4662-104", MCP466x_104 },
-	{}
-};
-MODULE_DEVICE_TABLE(i2c, mcp4531_id);
-
 static struct i2c_driver mcp4531_driver = {
 	.driver = {
 		.name	= "mcp4531",
 		.of_match_table = of_match_ptr(mcp4531_of_match),
 	},
-	.probe		= mcp4531_probe,
+	.probe_new	= mcp4531_probe,
 	.id_table	= mcp4531_id,
 };
 
diff --git a/drivers/iio/potentiostat/lmp91000.c b/drivers/iio/potentiostat/lmp91000.c
index 85714055cc74..90e895adf997 100644
--- a/drivers/iio/potentiostat/lmp91000.c
+++ b/drivers/iio/potentiostat/lmp91000.c
@@ -411,12 +411,14 @@ static int lmp91000_remove(struct i2c_client *client)
 
 static const struct of_device_id lmp91000_of_match[] = {
 	{ .compatible = "ti,lmp91000", },
+	{ .compatible = "ti,lmp91002", },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, lmp91000_of_match);
 
 static const struct i2c_device_id lmp91000_id[] = {
 	{ "lmp91000", 0 },
+	{ "lmp91002", 0 },
 	{}
 };
 MODULE_DEVICE_TABLE(i2c, lmp91000_id);
diff --git a/drivers/iio/pressure/cros_ec_baro.c b/drivers/iio/pressure/cros_ec_baro.c
index 4599fde4dd25..87c07af9181f 100644
--- a/drivers/iio/pressure/cros_ec_baro.c
+++ b/drivers/iio/pressure/cros_ec_baro.c
@@ -16,6 +16,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/iio/buffer.h>
+#include <linux/iio/common/cros_ec_sensors_core.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/kfifo_buf.h>
 #include <linux/iio/trigger.h>
@@ -28,8 +29,6 @@
 #include <linux/slab.h>
 #include <linux/platform_device.h>
 
-#include "../common/cros_ec_sensors/cros_ec_sensors_core.h"
-
 /*
  * One channel for pressure, the other for timestamp.
  */
diff --git a/drivers/iio/resolver/Kconfig b/drivers/iio/resolver/Kconfig
new file mode 100644
index 000000000000..2ced9f22aa70
--- /dev/null
+++ b/drivers/iio/resolver/Kconfig
@@ -0,0 +1,17 @@
+#
+# Resolver/Synchro drivers
+#
+menu "Resolver to digital converters"
+
+config AD2S1200
+	tristate "Analog Devices ad2s1200/ad2s1205 driver"
+	depends on SPI
+	depends on GPIOLIB || COMPILE_TEST
+	help
+	  Say yes here to build support for Analog Devices spi resolver
+	  to digital converters, ad2s1200 and ad2s1205, provides direct access
+	  via sysfs.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ad2s1200.
+endmenu
diff --git a/drivers/iio/resolver/Makefile b/drivers/iio/resolver/Makefile
new file mode 100644
index 000000000000..4e1dccae07e7
--- /dev/null
+++ b/drivers/iio/resolver/Makefile
@@ -0,0 +1,5 @@
+#
+# Makefile for Resolver/Synchro drivers
+#
+
+obj-$(CONFIG_AD2S1200) += ad2s1200.o
diff --git a/drivers/iio/resolver/ad2s1200.c b/drivers/iio/resolver/ad2s1200.c
new file mode 100644
index 000000000000..28e618af9939
--- /dev/null
+++ b/drivers/iio/resolver/ad2s1200.c
@@ -0,0 +1,210 @@
+/*
+ * ad2s1200.c simple support for the ADI Resolver to Digital Converters:
+ * AD2S1200/1205
+ *
+ * Copyright (c) 2018-2018 David Veenstra <davidjulianveenstra@gmail.com>
+ * Copyright (c) 2010-2010 Analog Devices Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/spi/spi.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+
+#define DRV_NAME "ad2s1200"
+
+/* input clock on serial interface */
+#define AD2S1200_HZ	8192000
+/* clock period in nano second */
+#define AD2S1200_TSCLK	(1000000000 / AD2S1200_HZ)
+
+/**
+ * struct ad2s1200_state - driver instance specific data.
+ * @lock:	protects both the GPIO pins and the rx buffer.
+ * @sdev:	spi device.
+ * @sample:	GPIO pin SAMPLE.
+ * @rdvel:	GPIO pin RDVEL.
+ * @rx:		buffer for spi transfers.
+ */
+struct ad2s1200_state {
+	struct mutex lock;
+	struct spi_device *sdev;
+	struct gpio_desc *sample;
+	struct gpio_desc *rdvel;
+	__be16 rx ____cacheline_aligned;
+};
+
+static int ad2s1200_read_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int *val,
+			     int *val2,
+			     long m)
+{
+	struct ad2s1200_state *st = iio_priv(indio_dev);
+	int ret;
+
+	switch (m) {
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->type) {
+		case IIO_ANGL:
+			/* 2 * Pi / (2^12 - 1) ~= 0.001534355 */
+			*val = 0;
+			*val2 = 1534355;
+			return IIO_VAL_INT_PLUS_NANO;
+		case IIO_ANGL_VEL:
+			/* 2 * Pi ~= 6.283185 */
+			*val = 6;
+			*val2 = 283185;
+			return IIO_VAL_INT_PLUS_MICRO;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case IIO_CHAN_INFO_RAW:
+		mutex_lock(&st->lock);
+		gpiod_set_value(st->sample, 0);
+
+		/* delay (6 * AD2S1200_TSCLK + 20) nano seconds */
+		udelay(1);
+		gpiod_set_value(st->sample, 1);
+		gpiod_set_value(st->rdvel, !!(chan->type == IIO_ANGL));
+
+		ret = spi_read(st->sdev, &st->rx, 2);
+		if (ret < 0) {
+			mutex_unlock(&st->lock);
+			return ret;
+		}
+
+		switch (chan->type) {
+		case IIO_ANGL:
+			*val = be16_to_cpup(&st->rx) >> 4;
+			break;
+		case IIO_ANGL_VEL:
+			*val = sign_extend32(be16_to_cpup(&st->rx) >> 4, 11);
+			break;
+		default:
+			mutex_unlock(&st->lock);
+			return -EINVAL;
+		}
+
+		/* delay (2 * AD2S1200_TSCLK + 20) ns for sample pulse */
+		udelay(1);
+		mutex_unlock(&st->lock);
+
+		return IIO_VAL_INT;
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
+static const struct iio_chan_spec ad2s1200_channels[] = {
+	{
+		.type = IIO_ANGL,
+		.indexed = 1,
+		.channel = 0,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),
+	}, {
+		.type = IIO_ANGL_VEL,
+		.indexed = 1,
+		.channel = 0,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),
+	}
+};
+
+static const struct iio_info ad2s1200_info = {
+	.read_raw = ad2s1200_read_raw,
+};
+
+static int ad2s1200_probe(struct spi_device *spi)
+{
+	struct ad2s1200_state *st;
+	struct iio_dev *indio_dev;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	spi_set_drvdata(spi, indio_dev);
+	st = iio_priv(indio_dev);
+	mutex_init(&st->lock);
+	st->sdev = spi;
+
+	st->sample = devm_gpiod_get(&spi->dev, "adi,sample", GPIOD_OUT_LOW);
+	if (IS_ERR(st->sample)) {
+		dev_err(&spi->dev, "Failed to claim SAMPLE gpio: err=%ld\n",
+			PTR_ERR(st->sample));
+		return PTR_ERR(st->sample);
+	}
+
+	st->rdvel = devm_gpiod_get(&spi->dev, "adi,rdvel", GPIOD_OUT_LOW);
+	if (IS_ERR(st->rdvel)) {
+		dev_err(&spi->dev, "Failed to claim RDVEL gpio: err=%ld\n",
+			PTR_ERR(st->rdvel));
+		return PTR_ERR(st->rdvel);
+	}
+
+	indio_dev->dev.parent = &spi->dev;
+	indio_dev->info = &ad2s1200_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = ad2s1200_channels;
+	indio_dev->num_channels = ARRAY_SIZE(ad2s1200_channels);
+	indio_dev->name = spi_get_device_id(spi)->name;
+
+	spi->max_speed_hz = AD2S1200_HZ;
+	spi->mode = SPI_MODE_3;
+	ret = spi_setup(spi);
+
+	if (ret < 0) {
+		dev_err(&spi->dev, "spi_setup failed!\n");
+		return ret;
+	}
+
+	return devm_iio_device_register(&spi->dev, indio_dev);
+}
+
+static const struct of_device_id ad2s1200_of_match[] = {
+	{ .compatible = "adi,ad2s1200", },
+	{ .compatible = "adi,ad2s1205", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ad2s1200_of_match);
+
+static const struct spi_device_id ad2s1200_id[] = {
+	{ "ad2s1200" },
+	{ "ad2s1205" },
+	{}
+};
+MODULE_DEVICE_TABLE(spi, ad2s1200_id);
+
+static struct spi_driver ad2s1200_driver = {
+	.driver = {
+		.name = DRV_NAME,
+		.of_match_table = of_match_ptr(ad2s1200_of_match),
+	},
+	.probe = ad2s1200_probe,
+	.id_table = ad2s1200_id,
+};
+module_spi_driver(ad2s1200_driver);
+
+MODULE_AUTHOR("David Veenstra <davidjulianveenstra@gmail.com>");
+MODULE_AUTHOR("Graff Yang <graff.yang@gmail.com>");
+MODULE_DESCRIPTION("Analog Devices AD2S1200/1205 Resolver to Digital SPI driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index 2a972ed6851b..b03af54367c0 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -35,6 +35,17 @@ config INFINIBAND_USER_ACCESS
 	  libibverbs, libibcm and a hardware driver library from
 	  rdma-core <https://github.com/linux-rdma/rdma-core>.
 
+config INFINIBAND_USER_ACCESS_UCM
+	bool "Userspace CM (UCM, DEPRECATED)"
+	depends on BROKEN
+	depends on INFINIBAND_USER_ACCESS
+	help
+	  The UCM module has known security flaws, which no one is
+	  interested to fix. The user-space part of this code was
+	  dropped from the upstream a long time ago.
+
+	  This option is DEPRECATED and planned to be removed.
+
 config INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI
 	bool "Allow experimental legacy verbs in new ioctl uAPI  (EXPERIMENTAL)"
 	depends on INFINIBAND_USER_ACCESS
diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
index dda9e856e3fa..8d42373a2d8a 100644
--- a/drivers/infiniband/core/Makefile
+++ b/drivers/infiniband/core/Makefile
@@ -5,15 +5,16 @@ user_access-$(CONFIG_INFINIBAND_ADDR_TRANS)	:= rdma_ucm.o
 obj-$(CONFIG_INFINIBAND) +=		ib_core.o ib_cm.o iw_cm.o \
 					$(infiniband-y)
 obj-$(CONFIG_INFINIBAND_USER_MAD) +=	ib_umad.o
-obj-$(CONFIG_INFINIBAND_USER_ACCESS) +=	ib_uverbs.o ib_ucm.o \
-					$(user_access-y)
+obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o $(user_access-y)
+obj-$(CONFIG_INFINIBAND_USER_ACCESS_UCM) += ib_ucm.o $(user_access-y)
 
 ib_core-y :=			packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \
 				device.o fmr_pool.o cache.o netlink.o \
 				roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \
 				multicast.o mad.o smi.o agent.o mad_rmpp.o \
-				security.o nldev.o restrack.o
+				nldev.o restrack.o
 
+ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o
 ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o
 ib_core-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o
 ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index 88a7542d8c7b..4f32c4062fb6 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -56,7 +56,6 @@ struct addr_req {
 	struct sockaddr_storage src_addr;
 	struct sockaddr_storage dst_addr;
 	struct rdma_dev_addr *addr;
-	struct rdma_addr_client *client;
 	void *context;
 	void (*callback)(int status, struct sockaddr *src_addr,
 			 struct rdma_dev_addr *addr, void *context);
@@ -68,11 +67,8 @@ struct addr_req {
 
 static atomic_t ib_nl_addr_request_seq = ATOMIC_INIT(0);
 
-static void process_req(struct work_struct *work);
-
-static DEFINE_MUTEX(lock);
+static DEFINE_SPINLOCK(lock);
 static LIST_HEAD(req_list);
-static DECLARE_DELAYED_WORK(work, process_req);
 static struct workqueue_struct *addr_wq;
 
 static const struct nla_policy ib_nl_addr_policy[LS_NLA_TYPE_MAX] = {
@@ -112,7 +108,7 @@ static void ib_nl_process_good_ip_rsep(const struct nlmsghdr *nlh)
 			memcpy(&gid, nla_data(curr), nla_len(curr));
 	}
 
-	mutex_lock(&lock);
+	spin_lock_bh(&lock);
 	list_for_each_entry(req, &req_list, list) {
 		if (nlh->nlmsg_seq != req->seq)
 			continue;
@@ -122,7 +118,7 @@ static void ib_nl_process_good_ip_rsep(const struct nlmsghdr *nlh)
 		found = 1;
 		break;
 	}
-	mutex_unlock(&lock);
+	spin_unlock_bh(&lock);
 
 	if (!found)
 		pr_info("Couldn't find request waiting for DGID: %pI6\n",
@@ -223,28 +219,6 @@ int rdma_addr_size_kss(struct __kernel_sockaddr_storage *addr)
 }
 EXPORT_SYMBOL(rdma_addr_size_kss);
 
-static struct rdma_addr_client self;
-
-void rdma_addr_register_client(struct rdma_addr_client *client)
-{
-	atomic_set(&client->refcount, 1);
-	init_completion(&client->comp);
-}
-EXPORT_SYMBOL(rdma_addr_register_client);
-
-static inline void put_client(struct rdma_addr_client *client)
-{
-	if (atomic_dec_and_test(&client->refcount))
-		complete(&client->comp);
-}
-
-void rdma_addr_unregister_client(struct rdma_addr_client *client)
-{
-	put_client(client);
-	wait_for_completion(&client->comp);
-}
-EXPORT_SYMBOL(rdma_addr_unregister_client);
-
 void rdma_copy_addr(struct rdma_dev_addr *dev_addr,
 		    const struct net_device *dev,
 		    const unsigned char *dst_dev_addr)
@@ -302,7 +276,7 @@ int rdma_translate_ip(const struct sockaddr *addr,
 }
 EXPORT_SYMBOL(rdma_translate_ip);
 
-static void set_timeout(struct delayed_work *delayed_work, unsigned long time)
+static void set_timeout(struct addr_req *req, unsigned long time)
 {
 	unsigned long delay;
 
@@ -310,23 +284,15 @@ static void set_timeout(struct delayed_work *delayed_work, unsigned long time)
 	if ((long)delay < 0)
 		delay = 0;
 
-	mod_delayed_work(addr_wq, delayed_work, delay);
+	mod_delayed_work(addr_wq, &req->work, delay);
 }
 
 static void queue_req(struct addr_req *req)
 {
-	struct addr_req *temp_req;
-
-	mutex_lock(&lock);
-	list_for_each_entry_reverse(temp_req, &req_list, list) {
-		if (time_after_eq(req->timeout, temp_req->timeout))
-			break;
-	}
-
-	list_add(&req->list, &temp_req->list);
-
-	set_timeout(&req->work, req->timeout);
-	mutex_unlock(&lock);
+	spin_lock_bh(&lock);
+	list_add_tail(&req->list, &req_list);
+	set_timeout(req, req->timeout);
+	spin_unlock_bh(&lock);
 }
 
 static int ib_nl_fetch_ha(const struct dst_entry *dst,
@@ -584,7 +550,6 @@ static void process_one_req(struct work_struct *_work)
 	struct addr_req *req;
 	struct sockaddr *src_in, *dst_in;
 
-	mutex_lock(&lock);
 	req = container_of(_work, struct addr_req, work.work);
 
 	if (req->status == -ENODATA) {
@@ -596,72 +561,33 @@ static void process_one_req(struct work_struct *_work)
 			req->status = -ETIMEDOUT;
 		} else if (req->status == -ENODATA) {
 			/* requeue the work for retrying again */
-			set_timeout(&req->work, req->timeout);
-			mutex_unlock(&lock);
+			spin_lock_bh(&lock);
+			if (!list_empty(&req->list))
+				set_timeout(req, req->timeout);
+			spin_unlock_bh(&lock);
 			return;
 		}
 	}
-	list_del(&req->list);
-	mutex_unlock(&lock);
-
-	/*
-	 * Although the work will normally have been canceled by the
-	 * workqueue, it can still be requeued as long as it is on the
-	 * req_list, so it could have been requeued before we grabbed &lock.
-	 * We need to cancel it after it is removed from req_list to really be
-	 * sure it is safe to free.
-	 */
-	cancel_delayed_work(&req->work);
 
 	req->callback(req->status, (struct sockaddr *)&req->src_addr,
 		req->addr, req->context);
-	put_client(req->client);
-	kfree(req);
-}
-
-static void process_req(struct work_struct *work)
-{
-	struct addr_req *req, *temp_req;
-	struct sockaddr *src_in, *dst_in;
-	struct list_head done_list;
-
-	INIT_LIST_HEAD(&done_list);
-
-	mutex_lock(&lock);
-	list_for_each_entry_safe(req, temp_req, &req_list, list) {
-		if (req->status == -ENODATA) {
-			src_in = (struct sockaddr *) &req->src_addr;
-			dst_in = (struct sockaddr *) &req->dst_addr;
-			req->status = addr_resolve(src_in, dst_in, req->addr,
-						   true, req->seq);
-			if (req->status && time_after_eq(jiffies, req->timeout))
-				req->status = -ETIMEDOUT;
-			else if (req->status == -ENODATA) {
-				set_timeout(&req->work, req->timeout);
-				continue;
-			}
-		}
-		list_move_tail(&req->list, &done_list);
-	}
-
-	mutex_unlock(&lock);
-
-	list_for_each_entry_safe(req, temp_req, &done_list, list) {
-		list_del(&req->list);
-		/* It is safe to cancel other work items from this work item
-		 * because at a time there can be only one work item running
-		 * with this single threaded work queue.
+	req->callback = NULL;
+
+	spin_lock_bh(&lock);
+	if (!list_empty(&req->list)) {
+		/*
+		 * Although the work will normally have been canceled by the
+		 * workqueue, it can still be requeued as long as it is on the
+		 * req_list.
 		 */
 		cancel_delayed_work(&req->work);
-		req->callback(req->status, (struct sockaddr *) &req->src_addr,
-			req->addr, req->context);
-		put_client(req->client);
+		list_del_init(&req->list);
 		kfree(req);
 	}
+	spin_unlock_bh(&lock);
 }
 
-int rdma_resolve_ip(struct rdma_addr_client *client,
-		    struct sockaddr *src_addr, struct sockaddr *dst_addr,
+int rdma_resolve_ip(struct sockaddr *src_addr, struct sockaddr *dst_addr,
 		    struct rdma_dev_addr *addr, int timeout_ms,
 		    void (*callback)(int status, struct sockaddr *src_addr,
 				     struct rdma_dev_addr *addr, void *context),
@@ -693,8 +619,6 @@ int rdma_resolve_ip(struct rdma_addr_client *client,
 	req->addr = addr;
 	req->callback = callback;
 	req->context = context;
-	req->client = client;
-	atomic_inc(&client->refcount);
 	INIT_DELAYED_WORK(&req->work, process_one_req);
 	req->seq = (u32)atomic_inc_return(&ib_nl_addr_request_seq);
 
@@ -710,7 +634,6 @@ int rdma_resolve_ip(struct rdma_addr_client *client,
 		break;
 	default:
 		ret = req->status;
-		atomic_dec(&client->refcount);
 		goto err;
 	}
 	return ret;
@@ -742,18 +665,36 @@ int rdma_resolve_ip_route(struct sockaddr *src_addr,
 void rdma_addr_cancel(struct rdma_dev_addr *addr)
 {
 	struct addr_req *req, *temp_req;
+	struct addr_req *found = NULL;
 
-	mutex_lock(&lock);
+	spin_lock_bh(&lock);
 	list_for_each_entry_safe(req, temp_req, &req_list, list) {
 		if (req->addr == addr) {
-			req->status = -ECANCELED;
-			req->timeout = jiffies;
-			list_move(&req->list, &req_list);
-			set_timeout(&req->work, req->timeout);
+			/*
+			 * Removing from the list means we take ownership of
+			 * the req
+			 */
+			list_del_init(&req->list);
+			found = req;
 			break;
 		}
 	}
-	mutex_unlock(&lock);
+	spin_unlock_bh(&lock);
+
+	if (!found)
+		return;
+
+	/*
+	 * sync canceling the work after removing it from the req_list
+	 * guarentees no work is running and none will be started.
+	 */
+	cancel_delayed_work_sync(&found->work);
+
+	if (found->callback)
+		found->callback(-ECANCELED, (struct sockaddr *)&found->src_addr,
+			      found->addr, found->context);
+
+	kfree(found);
 }
 EXPORT_SYMBOL(rdma_addr_cancel);
 
@@ -791,8 +732,8 @@ int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
 	dev_addr.net = &init_net;
 
 	init_completion(&ctx.comp);
-	ret = rdma_resolve_ip(&self, &sgid_addr._sockaddr, &dgid_addr._sockaddr,
-			&dev_addr, 1000, resolve_cb, &ctx);
+	ret = rdma_resolve_ip(&sgid_addr._sockaddr, &dgid_addr._sockaddr,
+			      &dev_addr, 1000, resolve_cb, &ctx);
 	if (ret)
 		return ret;
 
@@ -810,11 +751,17 @@ int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
 static int netevent_callback(struct notifier_block *self, unsigned long event,
 	void *ctx)
 {
+	struct addr_req *req;
+
 	if (event == NETEVENT_NEIGH_UPDATE) {
 		struct neighbour *neigh = ctx;
 
-		if (neigh->nud_state & NUD_VALID)
-			set_timeout(&work, jiffies);
+		if (neigh->nud_state & NUD_VALID) {
+			spin_lock_bh(&lock);
+			list_for_each_entry(req, &req_list, list)
+				set_timeout(req, jiffies);
+			spin_unlock_bh(&lock);
+		}
 	}
 	return 0;
 }
@@ -830,14 +777,13 @@ int addr_init(void)
 		return -ENOMEM;
 
 	register_netevent_notifier(&nb);
-	rdma_addr_register_client(&self);
 
 	return 0;
 }
 
 void addr_cleanup(void)
 {
-	rdma_addr_unregister_client(&self);
 	unregister_netevent_notifier(&nb);
 	destroy_workqueue(addr_wq);
+	WARN_ON(!list_empty(&req_list));
 }
diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c
index fb2d347f760f..2bdfc4b4a15c 100644
--- a/drivers/infiniband/core/cache.c
+++ b/drivers/infiniband/core/cache.c
@@ -125,6 +125,16 @@ const char *ib_cache_gid_type_str(enum ib_gid_type gid_type)
 }
 EXPORT_SYMBOL(ib_cache_gid_type_str);
 
+/** rdma_is_zero_gid - Check if given GID is zero or not.
+ * @gid:	GID to check
+ * Returns true if given GID is zero, returns false otherwise.
+ */
+bool rdma_is_zero_gid(const union ib_gid *gid)
+{
+	return !memcmp(gid, &zgid, sizeof(*gid));
+}
+EXPORT_SYMBOL(rdma_is_zero_gid);
+
 int ib_cache_gid_parse_type_str(const char *buf)
 {
 	unsigned int i;
@@ -149,6 +159,11 @@ int ib_cache_gid_parse_type_str(const char *buf)
 }
 EXPORT_SYMBOL(ib_cache_gid_parse_type_str);
 
+static struct ib_gid_table *rdma_gid_table(struct ib_device *device, u8 port)
+{
+	return device->cache.ports[port - rdma_start_port(device)].gid;
+}
+
 static void del_roce_gid(struct ib_device *device, u8 port_num,
 			 struct ib_gid_table *table, int ix)
 {
@@ -231,7 +246,7 @@ static int add_modify_gid(struct ib_gid_table *table,
 		 * So ignore such behavior for IB link layer and don't
 		 * fail the call, but don't add such entry to GID cache.
 		 */
-		if (!memcmp(gid, &zgid, sizeof(*gid)))
+		if (rdma_is_zero_gid(gid))
 			return 0;
 	}
 
@@ -264,7 +279,7 @@ static void del_gid(struct ib_device *ib_dev, u8 port,
 
 	if (rdma_protocol_roce(ib_dev, port))
 		del_roce_gid(ib_dev, port, table, ix);
-	memcpy(&table->data_vec[ix].gid, &zgid, sizeof(zgid));
+	memset(&table->data_vec[ix].gid, 0, sizeof(table->data_vec[ix].gid));
 	memset(&table->data_vec[ix].attr, 0, sizeof(table->data_vec[ix].attr));
 	table->data_vec[ix].context = NULL;
 }
@@ -363,10 +378,10 @@ static int __ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
 	 * IB spec version 1.3 section 4.1.1 point (6) and
 	 * section 12.7.10 and section 12.7.20
 	 */
-	if (!memcmp(gid, &zgid, sizeof(*gid)))
+	if (rdma_is_zero_gid(gid))
 		return -EINVAL;
 
-	table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid;
+	table = rdma_gid_table(ib_dev, port);
 
 	mutex_lock(&table->lock);
 
@@ -433,7 +448,7 @@ _ib_cache_gid_del(struct ib_device *ib_dev, u8 port,
 	int ret = 0;
 	int ix;
 
-	table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid;
+	table = rdma_gid_table(ib_dev, port);
 
 	mutex_lock(&table->lock);
 
@@ -472,7 +487,7 @@ int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,
 	int ix;
 	bool deleted = false;
 
-	table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid;
+	table = rdma_gid_table(ib_dev, port);
 
 	mutex_lock(&table->lock);
 
@@ -496,7 +511,7 @@ static int __ib_cache_gid_get(struct ib_device *ib_dev, u8 port, int index,
 {
 	struct ib_gid_table *table;
 
-	table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid;
+	table = rdma_gid_table(ib_dev, port);
 
 	if (index < 0 || index >= table->sz)
 		return -EINVAL;
@@ -589,7 +604,7 @@ int ib_find_cached_gid_by_port(struct ib_device *ib_dev,
 	if (!rdma_is_port_valid(ib_dev, port))
 		return -ENOENT;
 
-	table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid;
+	table = rdma_gid_table(ib_dev, port);
 
 	if (ndev)
 		mask |= GID_ATTR_FIND_MASK_NETDEV;
@@ -647,7 +662,7 @@ static int ib_cache_gid_find_by_filter(struct ib_device *ib_dev,
 	    !rdma_protocol_roce(ib_dev, port))
 		return -EPROTONOSUPPORT;
 
-	table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid;
+	table = rdma_gid_table(ib_dev, port);
 
 	read_lock_irqsave(&table->rwlock, flags);
 	for (i = 0; i < table->sz; i++) {
@@ -724,8 +739,7 @@ static void cleanup_gid_table_port(struct ib_device *ib_dev, u8 port,
 
 	mutex_lock(&table->lock);
 	for (i = 0; i < table->sz; ++i) {
-		if (memcmp(&table->data_vec[i].gid, &zgid,
-			   sizeof(table->data_vec[i].gid))) {
+		if (!rdma_is_zero_gid(&table->data_vec[i].gid)) {
 			del_gid(ib_dev, port, table, i);
 			deleted = true;
 		}
@@ -747,7 +761,7 @@ void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
 	unsigned int gid_type;
 	unsigned long mask;
 
-	table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid;
+	table = rdma_gid_table(ib_dev, port);
 
 	mask = GID_ATTR_FIND_MASK_GID_TYPE |
 	       GID_ATTR_FIND_MASK_DEFAULT |
@@ -772,8 +786,8 @@ void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
 	}
 }
 
-static int gid_table_reserve_default(struct ib_device *ib_dev, u8 port,
-				     struct ib_gid_table *table)
+static void gid_table_reserve_default(struct ib_device *ib_dev, u8 port,
+				      struct ib_gid_table *table)
 {
 	unsigned int i;
 	unsigned long roce_gid_type_mask;
@@ -783,8 +797,7 @@ static int gid_table_reserve_default(struct ib_device *ib_dev, u8 port,
 	roce_gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
 	num_default_gids = hweight_long(roce_gid_type_mask);
 	for (i = 0; i < num_default_gids && i < table->sz; i++) {
-		struct ib_gid_table_entry *entry =
-			&table->data_vec[i];
+		struct ib_gid_table_entry *entry = &table->data_vec[i];
 
 		entry->props |= GID_TABLE_ENTRY_DEFAULT;
 		current_gid = find_next_bit(&roce_gid_type_mask,
@@ -792,59 +805,42 @@ static int gid_table_reserve_default(struct ib_device *ib_dev, u8 port,
 					    current_gid);
 		entry->attr.gid_type = current_gid++;
 	}
+}
 
-	return 0;
+
+static void gid_table_release_one(struct ib_device *ib_dev)
+{
+	struct ib_gid_table *table;
+	u8 port;
+
+	for (port = 0; port < ib_dev->phys_port_cnt; port++) {
+		table = ib_dev->cache.ports[port].gid;
+		release_gid_table(table);
+		ib_dev->cache.ports[port].gid = NULL;
+	}
 }
 
 static int _gid_table_setup_one(struct ib_device *ib_dev)
 {
 	u8 port;
 	struct ib_gid_table *table;
-	int err = 0;
 
 	for (port = 0; port < ib_dev->phys_port_cnt; port++) {
 		u8 rdma_port = port + rdma_start_port(ib_dev);
 
-		table =
-			alloc_gid_table(
+		table =	alloc_gid_table(
 				ib_dev->port_immutable[rdma_port].gid_tbl_len);
-		if (!table) {
-			err = -ENOMEM;
+		if (!table)
 			goto rollback_table_setup;
-		}
 
-		err = gid_table_reserve_default(ib_dev,
-						port + rdma_start_port(ib_dev),
-						table);
-		if (err)
-			goto rollback_table_setup;
+		gid_table_reserve_default(ib_dev, rdma_port, table);
 		ib_dev->cache.ports[port].gid = table;
 	}
-
 	return 0;
 
 rollback_table_setup:
-	for (port = 0; port < ib_dev->phys_port_cnt; port++) {
-		table = ib_dev->cache.ports[port].gid;
-
-		cleanup_gid_table_port(ib_dev, port + rdma_start_port(ib_dev),
-				       table);
-		release_gid_table(table);
-	}
-
-	return err;
-}
-
-static void gid_table_release_one(struct ib_device *ib_dev)
-{
-	struct ib_gid_table *table;
-	u8 port;
-
-	for (port = 0; port < ib_dev->phys_port_cnt; port++) {
-		table = ib_dev->cache.ports[port].gid;
-		release_gid_table(table);
-		ib_dev->cache.ports[port].gid = NULL;
-	}
+	gid_table_release_one(ib_dev);
+	return -ENOMEM;
 }
 
 static void gid_table_cleanup_one(struct ib_device *ib_dev)
@@ -886,7 +882,7 @@ int ib_get_cached_gid(struct ib_device *device,
 	if (!rdma_is_port_valid(device, port_num))
 		return -EINVAL;
 
-	table = device->cache.ports[port_num - rdma_start_port(device)].gid;
+	table = rdma_gid_table(device, port_num);
 	read_lock_irqsave(&table->rwlock, flags);
 	res = __ib_cache_gid_get(device, port_num, index, gid, gid_attr);
 	read_unlock_irqrestore(&table->rwlock, flags);
@@ -1104,7 +1100,7 @@ static int config_non_roce_gid_cache(struct ib_device *device,
 
 	gid_attr.device = device;
 	gid_attr.port_num = port;
-	table = device->cache.ports[port - rdma_start_port(device)].gid;
+	table = rdma_gid_table(device, port);
 
 	mutex_lock(&table->lock);
 	for (i = 0; i < gid_tbl_len; ++i) {
@@ -1137,7 +1133,7 @@ static void ib_cache_update(struct ib_device *device,
 	if (!rdma_is_port_valid(device, port))
 		return;
 
-	table = device->cache.ports[port - rdma_start_port(device)].gid;
+	table = rdma_gid_table(device, port);
 
 	tprops = kmalloc(sizeof *tprops, GFP_KERNEL);
 	if (!tprops)
@@ -1299,13 +1295,3 @@ void ib_cache_cleanup_one(struct ib_device *device)
 	flush_workqueue(ib_wq);
 	gid_table_cleanup_one(device);
 }
-
-void __init ib_cache_setup(void)
-{
-	roce_gid_mgmt_init();
-}
-
-void __exit ib_cache_cleanup(void)
-{
-	roce_gid_mgmt_cleanup();
-}
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index a92e1a5c202b..7df4c7173607 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -452,6 +452,32 @@ static void cm_set_private_data(struct cm_id_private *cm_id_priv,
 	cm_id_priv->private_data_len = private_data_len;
 }
 
+static int cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
+			      struct ib_grh *grh, struct cm_av *av)
+{
+	struct rdma_ah_attr new_ah_attr;
+	int ret;
+
+	av->port = port;
+	av->pkey_index = wc->pkey_index;
+
+	/*
+	 * av->ah_attr might be initialized based on past wc during incoming
+	 * connect request or while sending out connect request. So initialize
+	 * a new ah_attr on stack. If initialization fails, old ah_attr is
+	 * used for sending any responses. If initialization is successful,
+	 * than new ah_attr is used by overwriting old one.
+	 */
+	ret = ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
+				      port->port_num, wc,
+				      grh, &new_ah_attr);
+	if (ret)
+		return ret;
+
+	memcpy(&av->ah_attr, &new_ah_attr, sizeof(new_ah_attr));
+	return 0;
+}
+
 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
 				   struct ib_grh *grh, struct cm_av *av)
 {
@@ -509,6 +535,7 @@ static struct cm_port *get_cm_port_from_path(struct sa_path_rec *path)
 static int cm_init_av_by_path(struct sa_path_rec *path, struct cm_av *av,
 			      struct cm_id_private *cm_id_priv)
 {
+	struct rdma_ah_attr new_ah_attr;
 	struct cm_device *cm_dev;
 	struct cm_port *port;
 	int ret;
@@ -524,15 +551,26 @@ static int cm_init_av_by_path(struct sa_path_rec *path, struct cm_av *av,
 		return ret;
 
 	av->port = port;
+
+	/*
+	 * av->ah_attr might be initialized based on wc or during
+	 * request processing time. So initialize a new ah_attr on stack.
+	 * If initialization fails, old ah_attr is used for sending any
+	 * responses. If initialization is successful, than new ah_attr
+	 * is used by overwriting the old one.
+	 */
 	ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
-					&av->ah_attr);
+					&new_ah_attr);
 	if (ret)
 		return ret;
 
 	av->timeout = path->packet_life_time + 1;
 
 	ret = add_cm_id_to_port_list(cm_id_priv, av, port);
-	return ret;
+	if (ret)
+		return ret;
+	memcpy(&av->ah_attr, &new_ah_attr, sizeof(new_ah_attr));
+	return 0;
 }
 
 static int cm_alloc_id(struct cm_id_private *cm_id_priv)
@@ -3189,12 +3227,6 @@ static int cm_lap_handler(struct cm_work *work)
 	if (!cm_id_priv)
 		return -EINVAL;
 
-	ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
-				      work->mad_recv_wc->recv_buf.grh,
-				      &cm_id_priv->av);
-	if (ret)
-		goto deref;
-
 	param = &work->cm_event.param.lap_rcvd;
 	memset(&work->path[0], 0, sizeof(work->path[1]));
 	cm_path_set_rec_type(work->port->cm_dev->ib_device,
@@ -3239,10 +3271,16 @@ static int cm_lap_handler(struct cm_work *work)
 		goto unlock;
 	}
 
-	cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
-	cm_id_priv->tid = lap_msg->hdr.tid;
+	ret = cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
+				 work->mad_recv_wc->recv_buf.grh,
+				 &cm_id_priv->av);
+	if (ret)
+		goto unlock;
+
 	cm_init_av_by_path(param->alternate_path, &cm_id_priv->alt_av,
 			   cm_id_priv);
+	cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
+	cm_id_priv->tid = lap_msg->hdr.tid;
 	ret = atomic_inc_and_test(&cm_id_priv->work_count);
 	if (!ret)
 		list_add_tail(&work->list, &cm_id_priv->work_list);
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index a693fcd4c513..441555a35525 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -146,6 +146,34 @@ const void *rdma_consumer_reject_data(struct rdma_cm_id *id,
 }
 EXPORT_SYMBOL(rdma_consumer_reject_data);
 
+/**
+ * rdma_iw_cm_id() - return the iw_cm_id pointer for this cm_id.
+ * @id: Communication Identifier
+ */
+struct iw_cm_id *rdma_iw_cm_id(struct rdma_cm_id *id)
+{
+	struct rdma_id_private *id_priv;
+
+	id_priv = container_of(id, struct rdma_id_private, id);
+	if (id->device->node_type == RDMA_NODE_RNIC)
+		return id_priv->cm_id.iw;
+	return NULL;
+}
+EXPORT_SYMBOL(rdma_iw_cm_id);
+
+/**
+ * rdma_res_to_id() - return the rdma_cm_id pointer for this restrack.
+ * @res: rdma resource tracking entry pointer
+ */
+struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res)
+{
+	struct rdma_id_private *id_priv =
+		container_of(res, struct rdma_id_private, res);
+
+	return &id_priv->id;
+}
+EXPORT_SYMBOL(rdma_res_to_id);
+
 static void cma_add_one(struct ib_device *device);
 static void cma_remove_one(struct ib_device *device, void *client_data);
 
@@ -156,7 +184,6 @@ static struct ib_client cma_client = {
 };
 
 static struct ib_sa_client sa_client;
-static struct rdma_addr_client addr_client;
 static LIST_HEAD(dev_list);
 static LIST_HEAD(listen_any_list);
 static DEFINE_MUTEX(lock);
@@ -2936,7 +2963,7 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
 		if (dst_addr->sa_family == AF_IB) {
 			ret = cma_resolve_ib_addr(id_priv);
 		} else {
-			ret = rdma_resolve_ip(&addr_client, cma_src_addr(id_priv),
+			ret = rdma_resolve_ip(cma_src_addr(id_priv),
 					      dst_addr, &id->route.addr.dev_addr,
 					      timeout_ms, addr_handler, id_priv);
 		}
@@ -4573,7 +4600,6 @@ static int __init cma_init(void)
 		goto err_wq;
 
 	ib_sa_register_client(&sa_client);
-	rdma_addr_register_client(&addr_client);
 	register_netdevice_notifier(&cma_nb);
 
 	ret = ib_register_client(&cma_client);
@@ -4587,7 +4613,6 @@ static int __init cma_init(void)
 
 err:
 	unregister_netdevice_notifier(&cma_nb);
-	rdma_addr_unregister_client(&addr_client);
 	ib_sa_unregister_client(&sa_client);
 err_wq:
 	destroy_workqueue(cma_wq);
@@ -4600,7 +4625,6 @@ static void __exit cma_cleanup(void)
 	rdma_nl_unregister(RDMA_NL_RDMA_CM);
 	ib_unregister_client(&cma_client);
 	unregister_netdevice_notifier(&cma_nb);
-	rdma_addr_unregister_client(&addr_client);
 	ib_sa_unregister_client(&sa_client);
 	unregister_pernet_subsys(&cma_pernet_operations);
 	destroy_workqueue(cma_wq);
diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index 54163a6e4067..fae417a391fb 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -88,9 +88,6 @@ int  ib_device_register_sysfs(struct ib_device *device,
 						   u8, struct kobject *));
 void ib_device_unregister_sysfs(struct ib_device *device);
 
-void ib_cache_setup(void);
-void ib_cache_cleanup(void);
-
 typedef void (*roce_netdev_callback)(struct ib_device *device, u8 port,
 	      struct net_device *idev, void *cookie);
 
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index ea9fbcfb21bd..84f51386e1e3 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -1225,7 +1225,7 @@ static int __init ib_core_init(void)
 
 	nldev_init();
 	rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table);
-	ib_cache_setup();
+	roce_gid_mgmt_init();
 
 	return 0;
 
@@ -1248,7 +1248,7 @@ static int __init ib_core_init(void)
 
 static void __exit ib_core_cleanup(void)
 {
-	ib_cache_cleanup();
+	roce_gid_mgmt_cleanup();
 	nldev_exit();
 	rdma_nl_unregister(RDMA_NL_LS);
 	unregister_lsm_notifier(&ibdev_lsm_nb);
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index eb567765f45c..340c7bea45ab 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -98,8 +98,83 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
 	[RDMA_NLDEV_ATTR_NDEV_INDEX]		= { .type = NLA_U32 },
 	[RDMA_NLDEV_ATTR_NDEV_NAME]		= { .type = NLA_NUL_STRING,
 						    .len = IFNAMSIZ },
+	[RDMA_NLDEV_ATTR_DRIVER]		= { .type = NLA_NESTED },
+	[RDMA_NLDEV_ATTR_DRIVER_ENTRY]		= { .type = NLA_NESTED },
+	[RDMA_NLDEV_ATTR_DRIVER_STRING]		= { .type = NLA_NUL_STRING,
+				    .len = RDMA_NLDEV_ATTR_ENTRY_STRLEN },
+	[RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE]	= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_DRIVER_S32]		= { .type = NLA_S32 },
+	[RDMA_NLDEV_ATTR_DRIVER_U32]		= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_DRIVER_S64]		= { .type = NLA_S64 },
+	[RDMA_NLDEV_ATTR_DRIVER_U64]		= { .type = NLA_U64 },
 };
 
+static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
+				      enum rdma_nldev_print_type print_type)
+{
+	if (nla_put_string(msg, RDMA_NLDEV_ATTR_DRIVER_STRING, name))
+		return -EMSGSIZE;
+	if (print_type != RDMA_NLDEV_PRINT_TYPE_UNSPEC &&
+	    nla_put_u8(msg, RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE, print_type))
+		return -EMSGSIZE;
+
+	return 0;
+}
+
+static int _rdma_nl_put_driver_u32(struct sk_buff *msg, const char *name,
+				   enum rdma_nldev_print_type print_type,
+				   u32 value)
+{
+	if (put_driver_name_print_type(msg, name, print_type))
+		return -EMSGSIZE;
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DRIVER_U32, value))
+		return -EMSGSIZE;
+
+	return 0;
+}
+
+static int _rdma_nl_put_driver_u64(struct sk_buff *msg, const char *name,
+				   enum rdma_nldev_print_type print_type,
+				   u64 value)
+{
+	if (put_driver_name_print_type(msg, name, print_type))
+		return -EMSGSIZE;
+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_DRIVER_U64, value,
+			      RDMA_NLDEV_ATTR_PAD))
+		return -EMSGSIZE;
+
+	return 0;
+}
+
+int rdma_nl_put_driver_u32(struct sk_buff *msg, const char *name, u32 value)
+{
+	return _rdma_nl_put_driver_u32(msg, name, RDMA_NLDEV_PRINT_TYPE_UNSPEC,
+				       value);
+}
+EXPORT_SYMBOL(rdma_nl_put_driver_u32);
+
+int rdma_nl_put_driver_u32_hex(struct sk_buff *msg, const char *name,
+			       u32 value)
+{
+	return _rdma_nl_put_driver_u32(msg, name, RDMA_NLDEV_PRINT_TYPE_HEX,
+				       value);
+}
+EXPORT_SYMBOL(rdma_nl_put_driver_u32_hex);
+
+int rdma_nl_put_driver_u64(struct sk_buff *msg, const char *name, u64 value)
+{
+	return _rdma_nl_put_driver_u64(msg, name, RDMA_NLDEV_PRINT_TYPE_UNSPEC,
+				       value);
+}
+EXPORT_SYMBOL(rdma_nl_put_driver_u64);
+
+int rdma_nl_put_driver_u64_hex(struct sk_buff *msg, const char *name, u64 value)
+{
+	return _rdma_nl_put_driver_u64(msg, name, RDMA_NLDEV_PRINT_TYPE_HEX,
+				       value);
+}
+EXPORT_SYMBOL(rdma_nl_put_driver_u64_hex);
+
 static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)
 {
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device->index))
@@ -122,7 +197,8 @@ static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)
 
 	BUILD_BUG_ON(sizeof(device->attrs.device_cap_flags) != sizeof(u64));
 	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS,
-			      device->attrs.device_cap_flags, 0))
+			      device->attrs.device_cap_flags,
+			      RDMA_NLDEV_ATTR_PAD))
 		return -EMSGSIZE;
 
 	ib_get_device_fw_str(device, fw);
@@ -131,10 +207,12 @@ static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)
 		return -EMSGSIZE;
 
 	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_NODE_GUID,
-			      be64_to_cpu(device->node_guid), 0))
+			      be64_to_cpu(device->node_guid),
+			      RDMA_NLDEV_ATTR_PAD))
 		return -EMSGSIZE;
 	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SYS_IMAGE_GUID,
-			      be64_to_cpu(device->attrs.sys_image_guid), 0))
+			      be64_to_cpu(device->attrs.sys_image_guid),
+			      RDMA_NLDEV_ATTR_PAD))
 		return -EMSGSIZE;
 	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_NODE_TYPE, device->node_type))
 		return -EMSGSIZE;
@@ -161,11 +239,11 @@ static int fill_port_info(struct sk_buff *msg,
 
 	BUILD_BUG_ON(sizeof(attr.port_cap_flags) > sizeof(u64));
 	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS,
-			      (u64)attr.port_cap_flags, 0))
+			      (u64)attr.port_cap_flags, RDMA_NLDEV_ATTR_PAD))
 		return -EMSGSIZE;
 	if (rdma_protocol_ib(device, port) &&
 	    nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SUBNET_PREFIX,
-			      attr.subnet_prefix, 0))
+			      attr.subnet_prefix, RDMA_NLDEV_ATTR_PAD))
 		return -EMSGSIZE;
 	if (rdma_protocol_ib(device, port)) {
 		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_LID, attr.lid))
@@ -209,8 +287,8 @@ static int fill_res_info_entry(struct sk_buff *msg,
 
 	if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME, name))
 		goto err;
-	if (nla_put_u64_64bit(msg,
-			      RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr, 0))
+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr,
+			      RDMA_NLDEV_ATTR_PAD))
 		goto err;
 
 	nla_nest_end(msg, entry_attr);
@@ -282,6 +360,7 @@ static int fill_res_qp_entry(struct sk_buff *msg, struct netlink_callback *cb,
 			     struct rdma_restrack_entry *res, uint32_t port)
 {
 	struct ib_qp *qp = container_of(res, struct ib_qp, res);
+	struct rdma_restrack_root *resroot = &qp->device->res;
 	struct ib_qp_init_attr qp_init_attr;
 	struct nlattr *entry_attr;
 	struct ib_qp_attr qp_attr;
@@ -331,6 +410,9 @@ static int fill_res_qp_entry(struct sk_buff *msg, struct netlink_callback *cb,
 	if (fill_res_name_pid(msg, res))
 		goto err;
 
+	if (resroot->fill_res_entry(msg, res))
+		goto err;
+
 	nla_nest_end(msg, entry_attr);
 	return 0;
 
@@ -346,6 +428,7 @@ static int fill_res_cm_id_entry(struct sk_buff *msg,
 {
 	struct rdma_id_private *id_priv =
 				container_of(res, struct rdma_id_private, res);
+	struct rdma_restrack_root *resroot = &id_priv->id.device->res;
 	struct rdma_cm_id *cm_id = &id_priv->id;
 	struct nlattr *entry_attr;
 
@@ -387,6 +470,9 @@ static int fill_res_cm_id_entry(struct sk_buff *msg,
 	if (fill_res_name_pid(msg, res))
 		goto err;
 
+	if (resroot->fill_res_entry(msg, res))
+		goto err;
+
 	nla_nest_end(msg, entry_attr);
 	return 0;
 
@@ -400,6 +486,7 @@ static int fill_res_cq_entry(struct sk_buff *msg, struct netlink_callback *cb,
 			     struct rdma_restrack_entry *res, uint32_t port)
 {
 	struct ib_cq *cq = container_of(res, struct ib_cq, res);
+	struct rdma_restrack_root *resroot = &cq->device->res;
 	struct nlattr *entry_attr;
 
 	entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_CQ_ENTRY);
@@ -409,7 +496,7 @@ static int fill_res_cq_entry(struct sk_buff *msg, struct netlink_callback *cb,
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQE, cq->cqe))
 		goto err;
 	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT,
-			      atomic_read(&cq->usecnt), 0))
+			      atomic_read(&cq->usecnt), RDMA_NLDEV_ATTR_PAD))
 		goto err;
 
 	/* Poll context is only valid for kernel CQs */
@@ -420,6 +507,9 @@ static int fill_res_cq_entry(struct sk_buff *msg, struct netlink_callback *cb,
 	if (fill_res_name_pid(msg, res))
 		goto err;
 
+	if (resroot->fill_res_entry(msg, res))
+		goto err;
+
 	nla_nest_end(msg, entry_attr);
 	return 0;
 
@@ -433,6 +523,7 @@ static int fill_res_mr_entry(struct sk_buff *msg, struct netlink_callback *cb,
 			     struct rdma_restrack_entry *res, uint32_t port)
 {
 	struct ib_mr *mr = container_of(res, struct ib_mr, res);
+	struct rdma_restrack_root *resroot = &mr->pd->device->res;
 	struct nlattr *entry_attr;
 
 	entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_MR_ENTRY);
@@ -444,17 +535,18 @@ static int fill_res_mr_entry(struct sk_buff *msg, struct netlink_callback *cb,
 			goto err;
 		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LKEY, mr->lkey))
 			goto err;
-		if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_IOVA,
-				      mr->iova, 0))
-			goto err;
 	}
 
-	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr->length, 0))
+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr->length,
+			      RDMA_NLDEV_ATTR_PAD))
 		goto err;
 
 	if (fill_res_name_pid(msg, res))
 		goto err;
 
+	if (resroot->fill_res_entry(msg, res))
+		goto err;
+
 	nla_nest_end(msg, entry_attr);
 	return 0;
 
@@ -468,6 +560,7 @@ static int fill_res_pd_entry(struct sk_buff *msg, struct netlink_callback *cb,
 			     struct rdma_restrack_entry *res, uint32_t port)
 {
 	struct ib_pd *pd = container_of(res, struct ib_pd, res);
+	struct rdma_restrack_root *resroot = &pd->device->res;
 	struct nlattr *entry_attr;
 
 	entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_PD_ENTRY);
@@ -484,7 +577,7 @@ static int fill_res_pd_entry(struct sk_buff *msg, struct netlink_callback *cb,
 			goto err;
 	}
 	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT,
-			      atomic_read(&pd->usecnt), 0))
+			      atomic_read(&pd->usecnt), RDMA_NLDEV_ATTR_PAD))
 		goto err;
 	if ((pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) &&
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,
@@ -494,6 +587,9 @@ static int fill_res_pd_entry(struct sk_buff *msg, struct netlink_callback *cb,
 	if (fill_res_name_pid(msg, res))
 		goto err;
 
+	if (resroot->fill_res_entry(msg, res))
+		goto err;
+
 	nla_nest_end(msg, entry_attr);
 	return 0;
 
diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c
index efddd13e3edb..172b517dc7b9 100644
--- a/drivers/infiniband/core/restrack.c
+++ b/drivers/infiniband/core/restrack.c
@@ -12,9 +12,16 @@
 
 #include "cma_priv.h"
 
+static int fill_res_noop(struct sk_buff *msg,
+			 struct rdma_restrack_entry *entry)
+{
+	return 0;
+}
+
 void rdma_restrack_init(struct rdma_restrack_root *res)
 {
 	init_rwsem(&res->rwsem);
+	res->fill_res_entry = fill_res_noop;
 }
 
 static const char *type2str(enum rdma_restrack_type type)
diff --git a/drivers/infiniband/core/roce_gid_mgmt.c b/drivers/infiniband/core/roce_gid_mgmt.c
index c0e4fd55e2cc..a4fbdc5d28fa 100644
--- a/drivers/infiniband/core/roce_gid_mgmt.c
+++ b/drivers/infiniband/core/roce_gid_mgmt.c
@@ -44,8 +44,6 @@
 
 static struct workqueue_struct *gid_cache_wq;
 
-static struct workqueue_struct *gid_cache_wq;
-
 enum gid_op_type {
 	GID_DEL = 0,
 	GID_ADD
diff --git a/drivers/infiniband/core/security.c b/drivers/infiniband/core/security.c
index b61dda6b04fc..9b0bea8303e0 100644
--- a/drivers/infiniband/core/security.c
+++ b/drivers/infiniband/core/security.c
@@ -30,8 +30,6 @@
  * SOFTWARE.
  */
 
-#ifdef CONFIG_SECURITY_INFINIBAND
-
 #include <linux/security.h>
 #include <linux/completion.h>
 #include <linux/list.h>
@@ -751,5 +749,3 @@ int ib_mad_enforce_security(struct ib_mad_agent_private *map, u16 pkey_index)
 				       pkey_index,
 				       map->agent.security);
 }
-
-#endif /* CONFIG_SECURITY_INFINIBAND */
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index 2b6c9b516070..54ab6335c48d 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -64,8 +64,6 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
 	}
 
 	sg_free_table(&umem->sg_head);
-	return;
-
 }
 
 /**
@@ -119,16 +117,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
 	umem->length     = size;
 	umem->address    = addr;
 	umem->page_shift = PAGE_SHIFT;
-	/*
-	 * We ask for writable memory if any of the following
-	 * access flags are set.  "Local write" and "remote write"
-	 * obviously require write access.  "Remote atomic" can do
-	 * things like fetch and add, which will modify memory, and
-	 * "MW bind" can change permissions by binding a window.
-	 */
-	umem->writable  = !!(access &
-		(IB_ACCESS_LOCAL_WRITE   | IB_ACCESS_REMOTE_WRITE |
-		 IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND));
+	umem->writable   = ib_access_writable(access);
 
 	if (access & IB_ACCESS_ON_DEMAND) {
 		ret = ib_umem_odp_get(context, umem, access);
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 21a887c9523b..e74262ee104c 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -2948,6 +2948,28 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,
 		memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz);
 		memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz);
 		break;
+	case IB_FLOW_SPEC_GRE:
+		ib_filter_sz = offsetof(struct ib_flow_gre_filter, real_sz);
+		actual_filter_sz = spec_filter_size(kern_spec_mask,
+						    kern_filter_sz,
+						    ib_filter_sz);
+		if (actual_filter_sz <= 0)
+			return -EINVAL;
+		ib_spec->gre.size = sizeof(struct ib_flow_spec_gre);
+		memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz);
+		memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz);
+		break;
+	case IB_FLOW_SPEC_MPLS:
+		ib_filter_sz = offsetof(struct ib_flow_mpls_filter, real_sz);
+		actual_filter_sz = spec_filter_size(kern_spec_mask,
+						    kern_filter_sz,
+						    ib_filter_sz);
+		if (actual_filter_sz <= 0)
+			return -EINVAL;
+		ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls);
+		memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz);
+		memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz);
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -3507,6 +3529,7 @@ int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file,
 					   uflow_res);
 		if (err)
 			goto err_free;
+
 		flow_attr->size +=
 			((union ib_flow_spec *) ib_spec)->size;
 		cmd.flow_attr.size -= ((struct ib_uverbs_flow_spec *)kern_spec)->size;
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index f6c739ec8b62..20b9f31052bf 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -185,12 +185,65 @@ static void bnxt_re_shutdown(void *p)
 	bnxt_re_ib_unreg(rdev, false);
 }
 
+static void bnxt_re_stop_irq(void *handle)
+{
+	struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;
+	struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw;
+	struct bnxt_qplib_nq *nq;
+	int indx;
+
+	for (indx = BNXT_RE_NQ_IDX; indx < rdev->num_msix; indx++) {
+		nq = &rdev->nq[indx - 1];
+		bnxt_qplib_nq_stop_irq(nq, false);
+	}
+
+	bnxt_qplib_rcfw_stop_irq(rcfw, false);
+}
+
+static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
+{
+	struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;
+	struct bnxt_msix_entry *msix_ent = rdev->msix_entries;
+	struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw;
+	struct bnxt_qplib_nq *nq;
+	int indx, rc;
+
+	if (!ent) {
+		/* Not setting the f/w timeout bit in rcfw.
+		 * During the driver unload the first command
+		 * to f/w will timeout and that will set the
+		 * timeout bit.
+		 */
+		dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n");
+		return;
+	}
+
+	/* Vectors may change after restart, so update with new vectors
+	 * in device sctructure.
+	 */
+	for (indx = 0; indx < rdev->num_msix; indx++)
+		rdev->msix_entries[indx].vector = ent[indx].vector;
+
+	bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,
+				  false);
+	for (indx = BNXT_RE_NQ_IDX ; indx < rdev->num_msix; indx++) {
+		nq = &rdev->nq[indx - 1];
+		rc = bnxt_qplib_nq_start_irq(nq, indx - 1,
+					     msix_ent[indx].vector, false);
+		if (rc)
+			dev_warn(rdev_to_dev(rdev),
+				 "Failed to reinit NQ index %d\n", indx - 1);
+	}
+}
+
 static struct bnxt_ulp_ops bnxt_re_ulp_ops = {
 	.ulp_async_notifier = NULL,
 	.ulp_stop = bnxt_re_stop,
 	.ulp_start = bnxt_re_start,
 	.ulp_sriov_config = bnxt_re_sriov_config,
-	.ulp_shutdown = bnxt_re_shutdown
+	.ulp_shutdown = bnxt_re_shutdown,
+	.ulp_irq_stop = bnxt_re_stop_irq,
+	.ulp_irq_restart = bnxt_re_start_irq
 };
 
 /* RoCE -> Net driver */
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 3a78faba8d91..50d8f1fc98d5 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -336,22 +336,32 @@ static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
 	return IRQ_HANDLED;
 }
 
+void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill)
+{
+	tasklet_disable(&nq->worker);
+	/* Mask h/w interrupt */
+	NQ_DB(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
+	/* Sync with last running IRQ handler */
+	synchronize_irq(nq->vector);
+	if (kill)
+		tasklet_kill(&nq->worker);
+	if (nq->requested) {
+		irq_set_affinity_hint(nq->vector, NULL);
+		free_irq(nq->vector, nq);
+		nq->requested = false;
+	}
+}
+
 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
 {
 	if (nq->cqn_wq) {
 		destroy_workqueue(nq->cqn_wq);
 		nq->cqn_wq = NULL;
 	}
+
 	/* Make sure the HW is stopped! */
-	synchronize_irq(nq->vector);
-	tasklet_disable(&nq->worker);
-	tasklet_kill(&nq->worker);
+	bnxt_qplib_nq_stop_irq(nq, true);
 
-	if (nq->requested) {
-		irq_set_affinity_hint(nq->vector, NULL);
-		free_irq(nq->vector, nq);
-		nq->requested = false;
-	}
 	if (nq->bar_reg_iomem)
 		iounmap(nq->bar_reg_iomem);
 	nq->bar_reg_iomem = NULL;
@@ -361,6 +371,40 @@ void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
 	nq->vector = 0;
 }
 
+int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
+			    int msix_vector, bool need_init)
+{
+	int rc;
+
+	if (nq->requested)
+		return -EFAULT;
+
+	nq->vector = msix_vector;
+	if (need_init)
+		tasklet_init(&nq->worker, bnxt_qplib_service_nq,
+			     (unsigned long)nq);
+	else
+		tasklet_enable(&nq->worker);
+
+	snprintf(nq->name, sizeof(nq->name), "bnxt_qplib_nq-%d", nq_indx);
+	rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, nq->name, nq);
+	if (rc)
+		return rc;
+
+	cpumask_clear(&nq->mask);
+	cpumask_set_cpu(nq_indx, &nq->mask);
+	rc = irq_set_affinity_hint(nq->vector, &nq->mask);
+	if (rc) {
+		dev_warn(&nq->pdev->dev,
+			 "QPLIB: set affinity failed; vector: %d nq_idx: %d\n",
+			 nq->vector, nq_indx);
+	}
+	nq->requested = true;
+	NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
+
+	return rc;
+}
+
 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
 			 int nq_idx, int msix_vector, int bar_reg_offset,
 			 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
@@ -372,41 +416,17 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
 	resource_size_t nq_base;
 	int rc = -1;
 
-	nq->pdev = pdev;
-	nq->vector = msix_vector;
 	if (cqn_handler)
 		nq->cqn_handler = cqn_handler;
 
 	if (srqn_handler)
 		nq->srqn_handler = srqn_handler;
 
-	tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
-
 	/* Have a task to schedule CQ notifiers in post send case */
 	nq->cqn_wq  = create_singlethread_workqueue("bnxt_qplib_nq");
 	if (!nq->cqn_wq)
-		goto fail;
-
-	nq->requested = false;
-	memset(nq->name, 0, 32);
-	sprintf(nq->name, "bnxt_qplib_nq-%d", nq_idx);
-	rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, nq->name, nq);
-	if (rc) {
-		dev_err(&nq->pdev->dev,
-			"Failed to request IRQ for NQ: %#x", rc);
-		goto fail;
-	}
-
-	cpumask_clear(&nq->mask);
-	cpumask_set_cpu(nq_idx, &nq->mask);
-	rc = irq_set_affinity_hint(nq->vector, &nq->mask);
-	if (rc) {
-		dev_warn(&nq->pdev->dev,
-			 "QPLIB: set affinity failed; vector: %d nq_idx: %d\n",
-			 nq->vector, nq_idx);
-	}
+		return -ENOMEM;
 
-	nq->requested = true;
 	nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
 	nq->bar_reg_off = bar_reg_offset;
 	nq_base = pci_resource_start(pdev, nq->bar_reg);
@@ -419,7 +439,13 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
 		rc = -ENOMEM;
 		goto fail;
 	}
-	NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
+
+	rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true);
+	if (rc) {
+		dev_err(&nq->pdev->dev,
+			"QPLIB: Failed to request irq for nq-idx %d", nq_idx);
+		goto fail;
+	}
 
 	return 0;
 fail:
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index ade9f13c0fd1..72352ca80ace 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -467,7 +467,10 @@ struct bnxt_qplib_nq_work {
 	struct bnxt_qplib_cq    *cq;
 };
 
+void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill);
 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq);
+int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
+			    int msix_vector, bool need_init);
 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
 			 int nq_idx, int msix_vector, int bar_reg_offset,
 			 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
index 80027a494730..2852d350ada1 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
@@ -582,19 +582,29 @@ int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
 	return -ENOMEM;
 }
 
-void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
+void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill)
 {
-	unsigned long indx;
-
-	/* Make sure the HW channel is stopped! */
-	synchronize_irq(rcfw->vector);
 	tasklet_disable(&rcfw->worker);
-	tasklet_kill(&rcfw->worker);
+	/* Mask h/w interrupts */
+	CREQ_DB(rcfw->creq_bar_reg_iomem, rcfw->creq.cons,
+		rcfw->creq.max_elements);
+	/* Sync with last running IRQ-handler */
+	synchronize_irq(rcfw->vector);
+	if (kill)
+		tasklet_kill(&rcfw->worker);
 
 	if (rcfw->requested) {
 		free_irq(rcfw->vector, rcfw);
 		rcfw->requested = false;
 	}
+}
+
+void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
+{
+	unsigned long indx;
+
+	bnxt_qplib_rcfw_stop_irq(rcfw, true);
+
 	if (rcfw->cmdq_bar_reg_iomem)
 		iounmap(rcfw->cmdq_bar_reg_iomem);
 	rcfw->cmdq_bar_reg_iomem = NULL;
@@ -614,6 +624,31 @@ void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
 	rcfw->vector = 0;
 }
 
+int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,
+			      bool need_init)
+{
+	int rc;
+
+	if (rcfw->requested)
+		return -EFAULT;
+
+	rcfw->vector = msix_vector;
+	if (need_init)
+		tasklet_init(&rcfw->worker,
+			     bnxt_qplib_service_creq, (unsigned long)rcfw);
+	else
+		tasklet_enable(&rcfw->worker);
+	rc = request_irq(rcfw->vector, bnxt_qplib_creq_irq, 0,
+			 "bnxt_qplib_creq", rcfw);
+	if (rc)
+		return rc;
+	rcfw->requested = true;
+	CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, rcfw->creq.cons,
+		      rcfw->creq.max_elements);
+
+	return 0;
+}
+
 int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
 				   struct bnxt_qplib_rcfw *rcfw,
 				   int msix_vector,
@@ -675,27 +710,17 @@ int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
 	rcfw->creq_qp_event_processed = 0;
 	rcfw->creq_func_event_processed = 0;
 
-	rcfw->vector = msix_vector;
 	if (aeq_handler)
 		rcfw->aeq_handler = aeq_handler;
+	init_waitqueue_head(&rcfw->waitq);
 
-	tasklet_init(&rcfw->worker, bnxt_qplib_service_creq,
-		     (unsigned long)rcfw);
-
-	rcfw->requested = false;
-	rc = request_irq(rcfw->vector, bnxt_qplib_creq_irq, 0,
-			 "bnxt_qplib_creq", rcfw);
+	rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true);
 	if (rc) {
 		dev_err(&rcfw->pdev->dev,
 			"QPLIB: Failed to request IRQ for CREQ rc = 0x%x", rc);
 		bnxt_qplib_disable_rcfw_channel(rcfw);
 		return rc;
 	}
-	rcfw->requested = true;
-
-	init_waitqueue_head(&rcfw->waitq);
-
-	CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, 0, rcfw->creq.max_elements);
 
 	init.cmdq_pbl = cpu_to_le64(rcfw->cmdq.pbl[PBL_LVL_0].pg_map_arr[0]);
 	init.cmdq_size_cmdq_lvl = cpu_to_le16(
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
index c7cce2e4185e..46416dfe8830 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
@@ -195,7 +195,10 @@ struct bnxt_qplib_rcfw {
 void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
 int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
 				  struct bnxt_qplib_rcfw *rcfw, int qp_tbl_sz);
+void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill);
 void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
+int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,
+			      bool need_init);
 int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
 				   struct bnxt_qplib_rcfw *rcfw,
 				   int msix_vector,
diff --git a/drivers/infiniband/hw/cxgb4/Makefile b/drivers/infiniband/hw/cxgb4/Makefile
index fa40b685831b..9edd92023e18 100644
--- a/drivers/infiniband/hw/cxgb4/Makefile
+++ b/drivers/infiniband/hw/cxgb4/Makefile
@@ -3,4 +3,5 @@ ccflags-y += -Idrivers/net/ethernet/chelsio/libcxgb
 
 obj-$(CONFIG_INFINIBAND_CXGB4) += iw_cxgb4.o
 
-iw_cxgb4-y :=  device.o cm.o provider.o mem.o cq.o qp.o resource.o ev.o id_table.o
+iw_cxgb4-y :=  device.o cm.o provider.o mem.o cq.o qp.o resource.o ev.o id_table.o \
+	       restrack.o
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index 4cf17c650c36..0912fa026327 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -3210,6 +3210,7 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 
 	ep->com.cm_id = cm_id;
 	ref_cm_id(&ep->com);
+	cm_id->provider_data = ep;
 	ep->com.dev = dev;
 	ep->com.qp = get_qhp(dev, conn_param->qpn);
 	if (!ep->com.qp) {
diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index 831027717121..870649ff049c 100644
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -55,6 +55,7 @@
 #include <rdma/iw_cm.h>
 #include <rdma/rdma_netlink.h>
 #include <rdma/iw_portmap.h>
+#include <rdma/restrack.h>
 
 #include "cxgb4.h"
 #include "cxgb4_uld.h"
@@ -1082,4 +1083,8 @@ extern int use_dsgl;
 void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey);
 struct c4iw_wr_wait *c4iw_alloc_wr_wait(gfp_t gfp);
 
+typedef int c4iw_restrack_func(struct sk_buff *msg,
+			       struct rdma_restrack_entry *res);
+extern c4iw_restrack_func *c4iw_restrack_funcs[RDMA_RESTRACK_MAX];
+
 #endif
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index 0b9cc73c3ded..1feade8bb4b3 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -551,6 +551,13 @@ static struct net_device *get_netdev(struct ib_device *dev, u8 port)
 	return ndev;
 }
 
+static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res)
+{
+	return (res->type < ARRAY_SIZE(c4iw_restrack_funcs) &&
+		c4iw_restrack_funcs[res->type]) ?
+		c4iw_restrack_funcs[res->type](msg, res) : 0;
+}
+
 void c4iw_register_device(struct work_struct *work)
 {
 	int ret;
@@ -645,6 +652,7 @@ void c4iw_register_device(struct work_struct *work)
 	dev->ibdev.iwcm->add_ref = c4iw_qp_add_ref;
 	dev->ibdev.iwcm->rem_ref = c4iw_qp_rem_ref;
 	dev->ibdev.iwcm->get_qp = c4iw_get_qp;
+	dev->ibdev.res.fill_res_entry = fill_res_entry;
 	memcpy(dev->ibdev.iwcm->ifname, dev->rdev.lldi.ports[0]->name,
 	       sizeof(dev->ibdev.iwcm->ifname));
 
diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c
index ae167b686608..4106eed1b8fb 100644
--- a/drivers/infiniband/hw/cxgb4/qp.c
+++ b/drivers/infiniband/hw/cxgb4/qp.c
@@ -1297,8 +1297,7 @@ static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe,
 
 	set_wr_txq(skb, CPL_PRIORITY_DATA, qhp->ep->txq_idx);
 
-	wqe = __skb_put(skb, sizeof(*wqe));
-	memset(wqe, 0, sizeof *wqe);
+	wqe = __skb_put_zero(skb, sizeof(*wqe));
 	wqe->op_compl = cpu_to_be32(FW_WR_OP_V(FW_RI_INIT_WR));
 	wqe->flowid_len16 = cpu_to_be32(
 		FW_WR_FLOWID_V(qhp->ep->hwtid) |
@@ -1421,8 +1420,7 @@ static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
 
 	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
 
-	wqe = __skb_put(skb, sizeof(*wqe));
-	memset(wqe, 0, sizeof *wqe);
+	wqe = __skb_put_zero(skb, sizeof(*wqe));
 	wqe->op_compl = cpu_to_be32(
 		FW_WR_OP_V(FW_RI_INIT_WR) |
 		FW_WR_COMPL_F);
@@ -1487,8 +1485,7 @@ static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
 	}
 	set_wr_txq(skb, CPL_PRIORITY_DATA, qhp->ep->txq_idx);
 
-	wqe = __skb_put(skb, sizeof(*wqe));
-	memset(wqe, 0, sizeof *wqe);
+	wqe = __skb_put_zero(skb, sizeof(*wqe));
 	wqe->op_compl = cpu_to_be32(
 		FW_WR_OP_V(FW_RI_INIT_WR) |
 		FW_WR_COMPL_F);
diff --git a/drivers/infiniband/hw/cxgb4/restrack.c b/drivers/infiniband/hw/cxgb4/restrack.c
new file mode 100644
index 000000000000..9a7520ee41e0
--- /dev/null
+++ b/drivers/infiniband/hw/cxgb4/restrack.c
@@ -0,0 +1,501 @@
+/*
+ * Copyright (c) 2018 Chelsio, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <rdma/rdma_cm.h>
+
+#include "iw_cxgb4.h"
+#include <rdma/restrack.h>
+#include <uapi/rdma/rdma_netlink.h>
+
+static int fill_sq(struct sk_buff *msg, struct t4_wq *wq)
+{
+	/* WQ+SQ */
+	if (rdma_nl_put_driver_u32(msg, "sqid", wq->sq.qid))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "flushed", wq->flushed))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "memsize", wq->sq.memsize))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "cidx", wq->sq.cidx))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "pidx", wq->sq.pidx))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "wq_pidx", wq->sq.wq_pidx))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "flush_cidx", wq->sq.flush_cidx))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "in_use", wq->sq.in_use))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "size", wq->sq.size))
+		goto err;
+	if (rdma_nl_put_driver_u32_hex(msg, "flags", wq->sq.flags))
+		goto err;
+	return 0;
+err:
+	return -EMSGSIZE;
+}
+
+static int fill_rq(struct sk_buff *msg, struct t4_wq *wq)
+{
+	/* RQ */
+	if (rdma_nl_put_driver_u32(msg, "rqid", wq->rq.qid))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "memsize", wq->rq.memsize))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "cidx", wq->rq.cidx))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "pidx", wq->rq.pidx))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "wq_pidx", wq->rq.wq_pidx))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "msn", wq->rq.msn))
+		goto err;
+	if (rdma_nl_put_driver_u32_hex(msg, "rqt_hwaddr", wq->rq.rqt_hwaddr))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "rqt_size", wq->rq.rqt_size))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "in_use", wq->rq.in_use))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "size", wq->rq.size))
+		goto err;
+	return 0;
+err:
+	return -EMSGSIZE;
+}
+
+static int fill_swsqe(struct sk_buff *msg, struct t4_sq *sq, u16 idx,
+		      struct t4_swsqe *sqe)
+{
+	if (rdma_nl_put_driver_u32(msg, "idx", idx))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "opcode", sqe->opcode))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "complete", sqe->complete))
+		goto err;
+	if (sqe->complete &&
+	    rdma_nl_put_driver_u32(msg, "cqe_status", CQE_STATUS(&sqe->cqe)))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "signaled", sqe->signaled))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "flushed", sqe->flushed))
+		goto err;
+	return 0;
+err:
+	return -EMSGSIZE;
+}
+
+/*
+ * Dump the first and last pending sqes.
+ */
+static int fill_swsqes(struct sk_buff *msg, struct t4_sq *sq,
+		       u16 first_idx, struct t4_swsqe *first_sqe,
+		       u16 last_idx, struct t4_swsqe *last_sqe)
+{
+	if (!first_sqe)
+		goto out;
+	if (fill_swsqe(msg, sq, first_idx, first_sqe))
+		goto err;
+	if (!last_sqe)
+		goto out;
+	if (fill_swsqe(msg, sq, last_idx, last_sqe))
+		goto err;
+out:
+	return 0;
+err:
+	return -EMSGSIZE;
+}
+
+static int fill_res_qp_entry(struct sk_buff *msg,
+			     struct rdma_restrack_entry *res)
+{
+	struct ib_qp *ibqp = container_of(res, struct ib_qp, res);
+	struct t4_swsqe *fsp = NULL, *lsp = NULL;
+	struct c4iw_qp *qhp = to_c4iw_qp(ibqp);
+	u16 first_sq_idx = 0, last_sq_idx = 0;
+	struct t4_swsqe first_sqe, last_sqe;
+	struct nlattr *table_attr;
+	struct t4_wq wq;
+
+	/* User qp state is not available, so don't dump user qps */
+	if (qhp->ucontext)
+		return 0;
+
+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
+	if (!table_attr)
+		goto err;
+
+	/* Get a consistent snapshot */
+	spin_lock_irq(&qhp->lock);
+	wq = qhp->wq;
+
+	/* If there are any pending sqes, copy the first and last */
+	if (wq.sq.cidx != wq.sq.pidx) {
+		first_sq_idx = wq.sq.cidx;
+		first_sqe = qhp->wq.sq.sw_sq[first_sq_idx];
+		fsp = &first_sqe;
+		last_sq_idx = wq.sq.pidx;
+		if (last_sq_idx-- == 0)
+			last_sq_idx = wq.sq.size - 1;
+		if (last_sq_idx != first_sq_idx) {
+			last_sqe = qhp->wq.sq.sw_sq[last_sq_idx];
+			lsp = &last_sqe;
+		}
+	}
+	spin_unlock_irq(&qhp->lock);
+
+	if (fill_sq(msg, &wq))
+		goto err_cancel_table;
+
+	if (fill_swsqes(msg, &wq.sq, first_sq_idx, fsp, last_sq_idx, lsp))
+		goto err_cancel_table;
+
+	if (fill_rq(msg, &wq))
+		goto err_cancel_table;
+
+	nla_nest_end(msg, table_attr);
+	return 0;
+
+err_cancel_table:
+	nla_nest_cancel(msg, table_attr);
+err:
+	return -EMSGSIZE;
+}
+
+union union_ep {
+	struct c4iw_listen_ep lep;
+	struct c4iw_ep ep;
+};
+
+static int fill_res_ep_entry(struct sk_buff *msg,
+			     struct rdma_restrack_entry *res)
+{
+	struct rdma_cm_id *cm_id = rdma_res_to_id(res);
+	struct nlattr *table_attr;
+	struct c4iw_ep_common *epcp;
+	struct c4iw_listen_ep *listen_ep = NULL;
+	struct c4iw_ep *ep = NULL;
+	struct iw_cm_id *iw_cm_id;
+	union union_ep *uep;
+
+	iw_cm_id = rdma_iw_cm_id(cm_id);
+	if (!iw_cm_id)
+		return 0;
+	epcp = (struct c4iw_ep_common *)iw_cm_id->provider_data;
+	if (!epcp)
+		return 0;
+	uep = kcalloc(1, sizeof(*uep), GFP_KERNEL);
+	if (!uep)
+		return 0;
+
+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
+	if (!table_attr)
+		goto err_free_uep;
+
+	/* Get a consistent snapshot */
+	mutex_lock(&epcp->mutex);
+	if (epcp->state == LISTEN) {
+		uep->lep = *(struct c4iw_listen_ep *)epcp;
+		mutex_unlock(&epcp->mutex);
+		listen_ep = &uep->lep;
+		epcp = &listen_ep->com;
+	} else {
+		uep->ep = *(struct c4iw_ep *)epcp;
+		mutex_unlock(&epcp->mutex);
+		ep = &uep->ep;
+		epcp = &ep->com;
+	}
+
+	if (rdma_nl_put_driver_u32(msg, "state", epcp->state))
+		goto err_cancel_table;
+	if (rdma_nl_put_driver_u64_hex(msg, "flags", epcp->flags))
+		goto err_cancel_table;
+	if (rdma_nl_put_driver_u64_hex(msg, "history", epcp->history))
+		goto err_cancel_table;
+
+	if (epcp->state == LISTEN) {
+		if (rdma_nl_put_driver_u32(msg, "stid", listen_ep->stid))
+			goto err_cancel_table;
+		if (rdma_nl_put_driver_u32(msg, "backlog", listen_ep->backlog))
+			goto err_cancel_table;
+	} else {
+		if (rdma_nl_put_driver_u32(msg, "hwtid", ep->hwtid))
+			goto err_cancel_table;
+		if (rdma_nl_put_driver_u32(msg, "ord", ep->ord))
+			goto err_cancel_table;
+		if (rdma_nl_put_driver_u32(msg, "ird", ep->ird))
+			goto err_cancel_table;
+		if (rdma_nl_put_driver_u32(msg, "emss", ep->emss))
+			goto err_cancel_table;
+
+		if (!ep->parent_ep && rdma_nl_put_driver_u32(msg, "atid",
+							     ep->atid))
+			goto err_cancel_table;
+	}
+	nla_nest_end(msg, table_attr);
+	kfree(uep);
+	return 0;
+
+err_cancel_table:
+	nla_nest_cancel(msg, table_attr);
+err_free_uep:
+	kfree(uep);
+	return -EMSGSIZE;
+}
+
+static int fill_cq(struct sk_buff *msg, struct t4_cq *cq)
+{
+	if (rdma_nl_put_driver_u32(msg, "cqid", cq->cqid))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "memsize", cq->memsize))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "size", cq->size))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "cidx", cq->cidx))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "cidx_inc", cq->cidx_inc))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "sw_cidx", cq->sw_cidx))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "sw_pidx", cq->sw_pidx))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "sw_in_use", cq->sw_in_use))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "vector", cq->vector))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "gen", cq->gen))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "error", cq->error))
+		goto err;
+	if (rdma_nl_put_driver_u64_hex(msg, "bits_type_ts",
+					 be64_to_cpu(cq->bits_type_ts)))
+		goto err;
+	if (rdma_nl_put_driver_u64_hex(msg, "flags", cq->flags))
+		goto err;
+
+	return 0;
+
+err:
+	return -EMSGSIZE;
+}
+
+static int fill_cqe(struct sk_buff *msg, struct t4_cqe *cqe, u16 idx,
+		    const char *qstr)
+{
+	if (rdma_nl_put_driver_u32(msg, qstr, idx))
+		goto err;
+	if (rdma_nl_put_driver_u32_hex(msg, "header",
+					 be32_to_cpu(cqe->header)))
+		goto err;
+	if (rdma_nl_put_driver_u32(msg, "len", be32_to_cpu(cqe->len)))
+		goto err;
+	if (rdma_nl_put_driver_u32_hex(msg, "wrid_hi",
+					 be32_to_cpu(cqe->u.gen.wrid_hi)))
+		goto err;
+	if (rdma_nl_put_driver_u32_hex(msg, "wrid_low",
+					 be32_to_cpu(cqe->u.gen.wrid_low)))
+		goto err;
+	if (rdma_nl_put_driver_u64_hex(msg, "bits_type_ts",
+					 be64_to_cpu(cqe->bits_type_ts)))
+		goto err;
+
+	return 0;
+
+err:
+	return -EMSGSIZE;
+}
+
+static int fill_hwcqes(struct sk_buff *msg, struct t4_cq *cq,
+		       struct t4_cqe *cqes)
+{
+	u16 idx;
+
+	idx = (cq->cidx > 0) ? cq->cidx - 1 : cq->size - 1;
+	if (fill_cqe(msg, cqes, idx, "hwcq_idx"))
+		goto err;
+	idx = cq->cidx;
+	if (fill_cqe(msg, cqes + 1, idx, "hwcq_idx"))
+		goto err;
+
+	return 0;
+err:
+	return -EMSGSIZE;
+}
+
+static int fill_swcqes(struct sk_buff *msg, struct t4_cq *cq,
+		       struct t4_cqe *cqes)
+{
+	u16 idx;
+
+	if (!cq->sw_in_use)
+		return 0;
+
+	idx = cq->sw_cidx;
+	if (fill_cqe(msg, cqes, idx, "swcq_idx"))
+		goto err;
+	if (cq->sw_in_use == 1)
+		goto out;
+	idx = (cq->sw_pidx > 0) ? cq->sw_pidx - 1 : cq->size - 1;
+	if (fill_cqe(msg, cqes + 1, idx, "swcq_idx"))
+		goto err;
+out:
+	return 0;
+err:
+	return -EMSGSIZE;
+}
+
+static int fill_res_cq_entry(struct sk_buff *msg,
+			     struct rdma_restrack_entry *res)
+{
+	struct ib_cq *ibcq = container_of(res, struct ib_cq, res);
+	struct c4iw_cq *chp = to_c4iw_cq(ibcq);
+	struct nlattr *table_attr;
+	struct t4_cqe hwcqes[2];
+	struct t4_cqe swcqes[2];
+	struct t4_cq cq;
+	u16 idx;
+
+	/* User cq state is not available, so don't dump user cqs */
+	if (ibcq->uobject)
+		return 0;
+
+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
+	if (!table_attr)
+		goto err;
+
+	/* Get a consistent snapshot */
+	spin_lock_irq(&chp->lock);
+
+	/* t4_cq struct */
+	cq = chp->cq;
+
+	/* get 2 hw cqes: cidx-1, and cidx */
+	idx = (cq.cidx > 0) ? cq.cidx - 1 : cq.size - 1;
+	hwcqes[0] = chp->cq.queue[idx];
+
+	idx = cq.cidx;
+	hwcqes[1] = chp->cq.queue[idx];
+
+	/* get first and last sw cqes */
+	if (cq.sw_in_use) {
+		swcqes[0] = chp->cq.sw_queue[cq.sw_cidx];
+		if (cq.sw_in_use > 1) {
+			idx = (cq.sw_pidx > 0) ? cq.sw_pidx - 1 : cq.size - 1;
+			swcqes[1] = chp->cq.sw_queue[idx];
+		}
+	}
+
+	spin_unlock_irq(&chp->lock);
+
+	if (fill_cq(msg, &cq))
+		goto err_cancel_table;
+
+	if (fill_swcqes(msg, &cq, swcqes))
+		goto err_cancel_table;
+
+	if (fill_hwcqes(msg, &cq, hwcqes))
+		goto err_cancel_table;
+
+	nla_nest_end(msg, table_attr);
+	return 0;
+
+err_cancel_table:
+	nla_nest_cancel(msg, table_attr);
+err:
+	return -EMSGSIZE;
+}
+
+static int fill_res_mr_entry(struct sk_buff *msg,
+			     struct rdma_restrack_entry *res)
+{
+	struct ib_mr *ibmr = container_of(res, struct ib_mr, res);
+	struct c4iw_mr *mhp = to_c4iw_mr(ibmr);
+	struct c4iw_dev *dev = mhp->rhp;
+	u32 stag = mhp->attr.stag;
+	struct nlattr *table_attr;
+	struct fw_ri_tpte tpte;
+	int ret;
+
+	if (!stag)
+		return 0;
+
+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
+	if (!table_attr)
+		goto err;
+
+	ret = cxgb4_read_tpte(dev->rdev.lldi.ports[0], stag, (__be32 *)&tpte);
+	if (ret) {
+		dev_err(&dev->rdev.lldi.pdev->dev,
+			"%s cxgb4_read_tpte err %d\n", __func__, ret);
+		return 0;
+	}
+
+	if (rdma_nl_put_driver_u32_hex(msg, "idx", stag >> 8))
+		goto err_cancel_table;
+	if (rdma_nl_put_driver_u32(msg, "valid",
+			FW_RI_TPTE_VALID_G(ntohl(tpte.valid_to_pdid))))
+		goto err_cancel_table;
+	if (rdma_nl_put_driver_u32_hex(msg, "key", stag & 0xff))
+		goto err_cancel_table;
+	if (rdma_nl_put_driver_u32(msg, "state",
+			FW_RI_TPTE_STAGSTATE_G(ntohl(tpte.valid_to_pdid))))
+		goto err_cancel_table;
+	if (rdma_nl_put_driver_u32(msg, "pdid",
+			FW_RI_TPTE_PDID_G(ntohl(tpte.valid_to_pdid))))
+		goto err_cancel_table;
+	if (rdma_nl_put_driver_u32_hex(msg, "perm",
+			FW_RI_TPTE_PERM_G(ntohl(tpte.locread_to_qpid))))
+		goto err_cancel_table;
+	if (rdma_nl_put_driver_u32(msg, "ps",
+			FW_RI_TPTE_PS_G(ntohl(tpte.locread_to_qpid))))
+		goto err_cancel_table;
+	if (rdma_nl_put_driver_u64(msg, "len",
+		      ((u64)ntohl(tpte.len_hi) << 32) | ntohl(tpte.len_lo)))
+		goto err_cancel_table;
+	if (rdma_nl_put_driver_u32_hex(msg, "pbl_addr",
+			FW_RI_TPTE_PBLADDR_G(ntohl(tpte.nosnoop_pbladdr))))
+		goto err_cancel_table;
+
+	nla_nest_end(msg, table_attr);
+	return 0;
+
+err_cancel_table:
+	nla_nest_cancel(msg, table_attr);
+err:
+	return -EMSGSIZE;
+}
+
+c4iw_restrack_func *c4iw_restrack_funcs[RDMA_RESTRACK_MAX] = {
+	[RDMA_RESTRACK_QP]	= fill_res_qp_entry,
+	[RDMA_RESTRACK_CM_ID]	= fill_res_ep_entry,
+	[RDMA_RESTRACK_CQ]	= fill_res_cq_entry,
+	[RDMA_RESTRACK_MR]	= fill_res_mr_entry,
+};
diff --git a/drivers/infiniband/hw/hfi1/Makefile b/drivers/infiniband/hw/hfi1/Makefile
index ce4010bad982..f451ba912f47 100644
--- a/drivers/infiniband/hw/hfi1/Makefile
+++ b/drivers/infiniband/hw/hfi1/Makefile
@@ -14,7 +14,15 @@ hfi1-y := affinity.o chip.o device.o driver.o efivar.o \
 	qp.o qsfp.o rc.o ruc.o sdma.o sysfs.o trace.o \
 	uc.o ud.o user_exp_rcv.o user_pages.o user_sdma.o verbs.o \
 	verbs_txreq.o vnic_main.o vnic_sdma.o
-hfi1-$(CONFIG_DEBUG_FS) += debugfs.o
+
+ifdef CONFIG_DEBUG_FS
+hfi1-y += debugfs.o
+ifdef CONFIG_FAULT_INJECTION
+ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
+hfi1-y += fault.o
+endif
+endif
+endif
 
 CFLAGS_trace.o = -I$(src)
 ifdef MVERSION
diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c
index b5fab55cc275..fbe7198a715a 100644
--- a/drivers/infiniband/hw/hfi1/affinity.c
+++ b/drivers/infiniband/hw/hfi1/affinity.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2017 Intel Corporation.
+ * Copyright(c) 2015 - 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -77,6 +77,58 @@ static inline void init_cpu_mask_set(struct cpu_mask_set *set)
 	set->gen = 0;
 }
 
+/* Increment generation of CPU set if needed */
+static void _cpu_mask_set_gen_inc(struct cpu_mask_set *set)
+{
+	if (cpumask_equal(&set->mask, &set->used)) {
+		/*
+		 * We've used up all the CPUs, bump up the generation
+		 * and reset the 'used' map
+		 */
+		set->gen++;
+		cpumask_clear(&set->used);
+	}
+}
+
+static void _cpu_mask_set_gen_dec(struct cpu_mask_set *set)
+{
+	if (cpumask_empty(&set->used) && set->gen) {
+		set->gen--;
+		cpumask_copy(&set->used, &set->mask);
+	}
+}
+
+/* Get the first CPU from the list of unused CPUs in a CPU set data structure */
+static int cpu_mask_set_get_first(struct cpu_mask_set *set, cpumask_var_t diff)
+{
+	int cpu;
+
+	if (!diff || !set)
+		return -EINVAL;
+
+	_cpu_mask_set_gen_inc(set);
+
+	/* Find out CPUs left in CPU mask */
+	cpumask_andnot(diff, &set->mask, &set->used);
+
+	cpu = cpumask_first(diff);
+	if (cpu >= nr_cpu_ids) /* empty */
+		cpu = -EINVAL;
+	else
+		cpumask_set_cpu(cpu, &set->used);
+
+	return cpu;
+}
+
+static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu)
+{
+	if (!set)
+		return;
+
+	cpumask_clear_cpu(cpu, &set->used);
+	_cpu_mask_set_gen_dec(set);
+}
+
 /* Initialize non-HT cpu cores mask */
 void init_real_cpu_mask(void)
 {
@@ -156,7 +208,13 @@ int node_affinity_init(void)
 	return 0;
 }
 
-void node_affinity_destroy(void)
+static void node_affinity_destroy(struct hfi1_affinity_node *entry)
+{
+	free_percpu(entry->comp_vect_affinity);
+	kfree(entry);
+}
+
+void node_affinity_destroy_all(void)
 {
 	struct list_head *pos, *q;
 	struct hfi1_affinity_node *entry;
@@ -166,7 +224,7 @@ void node_affinity_destroy(void)
 		entry = list_entry(pos, struct hfi1_affinity_node,
 				   list);
 		list_del(pos);
-		kfree(entry);
+		node_affinity_destroy(entry);
 	}
 	mutex_unlock(&node_affinity.lock);
 	kfree(hfi1_per_node_cntr);
@@ -180,6 +238,7 @@ static struct hfi1_affinity_node *node_affinity_allocate(int node)
 	if (!entry)
 		return NULL;
 	entry->node = node;
+	entry->comp_vect_affinity = alloc_percpu(u16);
 	INIT_LIST_HEAD(&entry->list);
 
 	return entry;
@@ -209,6 +268,341 @@ static struct hfi1_affinity_node *node_affinity_lookup(int node)
 	return NULL;
 }
 
+static int per_cpu_affinity_get(cpumask_var_t possible_cpumask,
+				u16 __percpu *comp_vect_affinity)
+{
+	int curr_cpu;
+	u16 cntr;
+	u16 prev_cntr;
+	int ret_cpu;
+
+	if (!possible_cpumask) {
+		ret_cpu = -EINVAL;
+		goto fail;
+	}
+
+	if (!comp_vect_affinity) {
+		ret_cpu = -EINVAL;
+		goto fail;
+	}
+
+	ret_cpu = cpumask_first(possible_cpumask);
+	if (ret_cpu >= nr_cpu_ids) {
+		ret_cpu = -EINVAL;
+		goto fail;
+	}
+
+	prev_cntr = *per_cpu_ptr(comp_vect_affinity, ret_cpu);
+	for_each_cpu(curr_cpu, possible_cpumask) {
+		cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
+
+		if (cntr < prev_cntr) {
+			ret_cpu = curr_cpu;
+			prev_cntr = cntr;
+		}
+	}
+
+	*per_cpu_ptr(comp_vect_affinity, ret_cpu) += 1;
+
+fail:
+	return ret_cpu;
+}
+
+static int per_cpu_affinity_put_max(cpumask_var_t possible_cpumask,
+				    u16 __percpu *comp_vect_affinity)
+{
+	int curr_cpu;
+	int max_cpu;
+	u16 cntr;
+	u16 prev_cntr;
+
+	if (!possible_cpumask)
+		return -EINVAL;
+
+	if (!comp_vect_affinity)
+		return -EINVAL;
+
+	max_cpu = cpumask_first(possible_cpumask);
+	if (max_cpu >= nr_cpu_ids)
+		return -EINVAL;
+
+	prev_cntr = *per_cpu_ptr(comp_vect_affinity, max_cpu);
+	for_each_cpu(curr_cpu, possible_cpumask) {
+		cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
+
+		if (cntr > prev_cntr) {
+			max_cpu = curr_cpu;
+			prev_cntr = cntr;
+		}
+	}
+
+	*per_cpu_ptr(comp_vect_affinity, max_cpu) -= 1;
+
+	return max_cpu;
+}
+
+/*
+ * Non-interrupt CPUs are used first, then interrupt CPUs.
+ * Two already allocated cpu masks must be passed.
+ */
+static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd,
+				  struct hfi1_affinity_node *entry,
+				  cpumask_var_t non_intr_cpus,
+				  cpumask_var_t available_cpus)
+	__must_hold(&node_affinity.lock)
+{
+	int cpu;
+	struct cpu_mask_set *set = dd->comp_vect;
+
+	lockdep_assert_held(&node_affinity.lock);
+	if (!non_intr_cpus) {
+		cpu = -1;
+		goto fail;
+	}
+
+	if (!available_cpus) {
+		cpu = -1;
+		goto fail;
+	}
+
+	/* Available CPUs for pinning completion vectors */
+	_cpu_mask_set_gen_inc(set);
+	cpumask_andnot(available_cpus, &set->mask, &set->used);
+
+	/* Available CPUs without SDMA engine interrupts */
+	cpumask_andnot(non_intr_cpus, available_cpus,
+		       &entry->def_intr.used);
+
+	/* If there are non-interrupt CPUs available, use them first */
+	if (!cpumask_empty(non_intr_cpus))
+		cpu = cpumask_first(non_intr_cpus);
+	else /* Otherwise, use interrupt CPUs */
+		cpu = cpumask_first(available_cpus);
+
+	if (cpu >= nr_cpu_ids) { /* empty */
+		cpu = -1;
+		goto fail;
+	}
+	cpumask_set_cpu(cpu, &set->used);
+
+fail:
+	return cpu;
+}
+
+static void _dev_comp_vect_cpu_put(struct hfi1_devdata *dd, int cpu)
+{
+	struct cpu_mask_set *set = dd->comp_vect;
+
+	if (cpu < 0)
+		return;
+
+	cpu_mask_set_put(set, cpu);
+}
+
+/* _dev_comp_vect_mappings_destroy() is reentrant */
+static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd)
+{
+	int i, cpu;
+
+	if (!dd->comp_vect_mappings)
+		return;
+
+	for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
+		cpu = dd->comp_vect_mappings[i];
+		_dev_comp_vect_cpu_put(dd, cpu);
+		dd->comp_vect_mappings[i] = -1;
+		hfi1_cdbg(AFFINITY,
+			  "[%s] Release CPU %d from completion vector %d",
+			  rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), cpu, i);
+	}
+
+	kfree(dd->comp_vect_mappings);
+	dd->comp_vect_mappings = NULL;
+}
+
+/*
+ * This function creates the table for looking up CPUs for completion vectors.
+ * num_comp_vectors needs to have been initilized before calling this function.
+ */
+static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd,
+					  struct hfi1_affinity_node *entry)
+	__must_hold(&node_affinity.lock)
+{
+	int i, cpu, ret;
+	cpumask_var_t non_intr_cpus;
+	cpumask_var_t available_cpus;
+
+	lockdep_assert_held(&node_affinity.lock);
+
+	if (!zalloc_cpumask_var(&non_intr_cpus, GFP_KERNEL))
+		return -ENOMEM;
+
+	if (!zalloc_cpumask_var(&available_cpus, GFP_KERNEL)) {
+		free_cpumask_var(non_intr_cpus);
+		return -ENOMEM;
+	}
+
+	dd->comp_vect_mappings = kcalloc(dd->comp_vect_possible_cpus,
+					 sizeof(*dd->comp_vect_mappings),
+					 GFP_KERNEL);
+	if (!dd->comp_vect_mappings) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+	for (i = 0; i < dd->comp_vect_possible_cpus; i++)
+		dd->comp_vect_mappings[i] = -1;
+
+	for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
+		cpu = _dev_comp_vect_cpu_get(dd, entry, non_intr_cpus,
+					     available_cpus);
+		if (cpu < 0) {
+			ret = -EINVAL;
+			goto fail;
+		}
+
+		dd->comp_vect_mappings[i] = cpu;
+		hfi1_cdbg(AFFINITY,
+			  "[%s] Completion Vector %d -> CPU %d",
+			  rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), i, cpu);
+	}
+
+	return 0;
+
+fail:
+	free_cpumask_var(available_cpus);
+	free_cpumask_var(non_intr_cpus);
+	_dev_comp_vect_mappings_destroy(dd);
+
+	return ret;
+}
+
+int hfi1_comp_vectors_set_up(struct hfi1_devdata *dd)
+{
+	int ret;
+	struct hfi1_affinity_node *entry;
+
+	mutex_lock(&node_affinity.lock);
+	entry = node_affinity_lookup(dd->node);
+	if (!entry) {
+		ret = -EINVAL;
+		goto unlock;
+	}
+	ret = _dev_comp_vect_mappings_create(dd, entry);
+unlock:
+	mutex_unlock(&node_affinity.lock);
+
+	return ret;
+}
+
+void hfi1_comp_vectors_clean_up(struct hfi1_devdata *dd)
+{
+	_dev_comp_vect_mappings_destroy(dd);
+}
+
+int hfi1_comp_vect_mappings_lookup(struct rvt_dev_info *rdi, int comp_vect)
+{
+	struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);
+	struct hfi1_devdata *dd = dd_from_dev(verbs_dev);
+
+	if (!dd->comp_vect_mappings)
+		return -EINVAL;
+	if (comp_vect >= dd->comp_vect_possible_cpus)
+		return -EINVAL;
+
+	return dd->comp_vect_mappings[comp_vect];
+}
+
+/*
+ * It assumes dd->comp_vect_possible_cpus is available.
+ */
+static int _dev_comp_vect_cpu_mask_init(struct hfi1_devdata *dd,
+					struct hfi1_affinity_node *entry,
+					bool first_dev_init)
+	__must_hold(&node_affinity.lock)
+{
+	int i, j, curr_cpu;
+	int possible_cpus_comp_vect = 0;
+	struct cpumask *dev_comp_vect_mask = &dd->comp_vect->mask;
+
+	lockdep_assert_held(&node_affinity.lock);
+	/*
+	 * If there's only one CPU available for completion vectors, then
+	 * there will only be one completion vector available. Othewise,
+	 * the number of completion vector available will be the number of
+	 * available CPUs divide it by the number of devices in the
+	 * local NUMA node.
+	 */
+	if (cpumask_weight(&entry->comp_vect_mask) == 1) {
+		possible_cpus_comp_vect = 1;
+		dd_dev_warn(dd,
+			    "Number of kernel receive queues is too large for completion vector affinity to be effective\n");
+	} else {
+		possible_cpus_comp_vect +=
+			cpumask_weight(&entry->comp_vect_mask) /
+				       hfi1_per_node_cntr[dd->node];
+
+		/*
+		 * If the completion vector CPUs available doesn't divide
+		 * evenly among devices, then the first device device to be
+		 * initialized gets an extra CPU.
+		 */
+		if (first_dev_init &&
+		    cpumask_weight(&entry->comp_vect_mask) %
+		    hfi1_per_node_cntr[dd->node] != 0)
+			possible_cpus_comp_vect++;
+	}
+
+	dd->comp_vect_possible_cpus = possible_cpus_comp_vect;
+
+	/* Reserving CPUs for device completion vector */
+	for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
+		curr_cpu = per_cpu_affinity_get(&entry->comp_vect_mask,
+						entry->comp_vect_affinity);
+		if (curr_cpu < 0)
+			goto fail;
+
+		cpumask_set_cpu(curr_cpu, dev_comp_vect_mask);
+	}
+
+	hfi1_cdbg(AFFINITY,
+		  "[%s] Completion vector affinity CPU set(s) %*pbl",
+		  rvt_get_ibdev_name(&(dd)->verbs_dev.rdi),
+		  cpumask_pr_args(dev_comp_vect_mask));
+
+	return 0;
+
+fail:
+	for (j = 0; j < i; j++)
+		per_cpu_affinity_put_max(&entry->comp_vect_mask,
+					 entry->comp_vect_affinity);
+
+	return curr_cpu;
+}
+
+/*
+ * It assumes dd->comp_vect_possible_cpus is available.
+ */
+static void _dev_comp_vect_cpu_mask_clean_up(struct hfi1_devdata *dd,
+					     struct hfi1_affinity_node *entry)
+	__must_hold(&node_affinity.lock)
+{
+	int i, cpu;
+
+	lockdep_assert_held(&node_affinity.lock);
+	if (!dd->comp_vect_possible_cpus)
+		return;
+
+	for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
+		cpu = per_cpu_affinity_put_max(&dd->comp_vect->mask,
+					       entry->comp_vect_affinity);
+		/* Clearing CPU in device completion vector cpu mask */
+		if (cpu >= 0)
+			cpumask_clear_cpu(cpu, &dd->comp_vect->mask);
+	}
+
+	dd->comp_vect_possible_cpus = 0;
+}
+
 /*
  * Interrupt affinity.
  *
@@ -225,7 +619,8 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
 	int node = pcibus_to_node(dd->pcidev->bus);
 	struct hfi1_affinity_node *entry;
 	const struct cpumask *local_mask;
-	int curr_cpu, possible, i;
+	int curr_cpu, possible, i, ret;
+	bool new_entry = false;
 
 	if (node < 0)
 		node = numa_node_id();
@@ -247,11 +642,14 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
 		if (!entry) {
 			dd_dev_err(dd,
 				   "Unable to allocate global affinity node\n");
-			mutex_unlock(&node_affinity.lock);
-			return -ENOMEM;
+			ret = -ENOMEM;
+			goto fail;
 		}
+		new_entry = true;
+
 		init_cpu_mask_set(&entry->def_intr);
 		init_cpu_mask_set(&entry->rcv_intr);
+		cpumask_clear(&entry->comp_vect_mask);
 		cpumask_clear(&entry->general_intr_mask);
 		/* Use the "real" cpu mask of this node as the default */
 		cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask,
@@ -304,10 +702,64 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
 					     &entry->general_intr_mask);
 		}
 
-		node_affinity_add_tail(entry);
+		/* Determine completion vector CPUs for the entire node */
+		cpumask_and(&entry->comp_vect_mask,
+			    &node_affinity.real_cpu_mask, local_mask);
+		cpumask_andnot(&entry->comp_vect_mask,
+			       &entry->comp_vect_mask,
+			       &entry->rcv_intr.mask);
+		cpumask_andnot(&entry->comp_vect_mask,
+			       &entry->comp_vect_mask,
+			       &entry->general_intr_mask);
+
+		/*
+		 * If there ends up being 0 CPU cores leftover for completion
+		 * vectors, use the same CPU core as the general/control
+		 * context.
+		 */
+		if (cpumask_weight(&entry->comp_vect_mask) == 0)
+			cpumask_copy(&entry->comp_vect_mask,
+				     &entry->general_intr_mask);
 	}
+
+	ret = _dev_comp_vect_cpu_mask_init(dd, entry, new_entry);
+	if (ret < 0)
+		goto fail;
+
+	if (new_entry)
+		node_affinity_add_tail(entry);
+
 	mutex_unlock(&node_affinity.lock);
+
 	return 0;
+
+fail:
+	if (new_entry)
+		node_affinity_destroy(entry);
+	mutex_unlock(&node_affinity.lock);
+	return ret;
+}
+
+void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd)
+{
+	struct hfi1_affinity_node *entry;
+
+	if (dd->node < 0)
+		return;
+
+	mutex_lock(&node_affinity.lock);
+	entry = node_affinity_lookup(dd->node);
+	if (!entry)
+		goto unlock;
+
+	/*
+	 * Free device completion vector CPUs to be used by future
+	 * completion vectors
+	 */
+	_dev_comp_vect_cpu_mask_clean_up(dd, entry);
+unlock:
+	mutex_unlock(&node_affinity.lock);
+	dd->node = -1;
 }
 
 /*
@@ -456,17 +908,12 @@ static int get_irq_affinity(struct hfi1_devdata *dd,
 		if (!zalloc_cpumask_var(&diff, GFP_KERNEL))
 			return -ENOMEM;
 
-		if (cpumask_equal(&set->mask, &set->used)) {
-			/*
-			 * We've used up all the CPUs, bump up the generation
-			 * and reset the 'used' map
-			 */
-			set->gen++;
-			cpumask_clear(&set->used);
+		cpu = cpu_mask_set_get_first(set, diff);
+		if (cpu < 0) {
+			free_cpumask_var(diff);
+			dd_dev_err(dd, "Failure to obtain CPU for IRQ\n");
+			return cpu;
 		}
-		cpumask_andnot(diff, &set->mask, &set->used);
-		cpu = cpumask_first(diff);
-		cpumask_set_cpu(cpu, &set->used);
 
 		free_cpumask_var(diff);
 	}
@@ -526,10 +973,7 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
 
 	if (set) {
 		cpumask_andnot(&set->used, &set->used, &msix->mask);
-		if (cpumask_empty(&set->used) && set->gen) {
-			set->gen--;
-			cpumask_copy(&set->used, &set->mask);
-		}
+		_cpu_mask_set_gen_dec(set);
 	}
 
 	irq_set_affinity_hint(msix->irq, NULL);
@@ -640,10 +1084,7 @@ int hfi1_get_proc_affinity(int node)
 	 * If we've used all available HW threads, clear the mask and start
 	 * overloading.
 	 */
-	if (cpumask_equal(&set->mask, &set->used)) {
-		set->gen++;
-		cpumask_clear(&set->used);
-	}
+	_cpu_mask_set_gen_inc(set);
 
 	/*
 	 * If NUMA node has CPUs used by interrupt handlers, include them in the
@@ -767,11 +1208,7 @@ void hfi1_put_proc_affinity(int cpu)
 		return;
 
 	mutex_lock(&affinity->lock);
-	cpumask_clear_cpu(cpu, &set->used);
+	cpu_mask_set_put(set, cpu);
 	hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);
-	if (cpumask_empty(&set->used) && set->gen) {
-		set->gen--;
-		cpumask_copy(&set->used, &set->mask);
-	}
 	mutex_unlock(&affinity->lock);
 }
diff --git a/drivers/infiniband/hw/hfi1/affinity.h b/drivers/infiniband/hw/hfi1/affinity.h
index 2a1e374169c0..6a7e6ea4e426 100644
--- a/drivers/infiniband/hw/hfi1/affinity.h
+++ b/drivers/infiniband/hw/hfi1/affinity.h
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2017 Intel Corporation.
+ * Copyright(c) 2015 - 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -98,9 +98,11 @@ void hfi1_put_proc_affinity(int cpu);
 
 struct hfi1_affinity_node {
 	int node;
+	u16 __percpu *comp_vect_affinity;
 	struct cpu_mask_set def_intr;
 	struct cpu_mask_set rcv_intr;
 	struct cpumask general_intr_mask;
+	struct cpumask comp_vect_mask;
 	struct list_head list;
 };
 
@@ -116,7 +118,11 @@ struct hfi1_affinity_node_list {
 };
 
 int node_affinity_init(void);
-void node_affinity_destroy(void);
+void node_affinity_destroy_all(void);
 extern struct hfi1_affinity_node_list node_affinity;
+void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd);
+int hfi1_comp_vect_mappings_lookup(struct rvt_dev_info *rdi, int comp_vect);
+int hfi1_comp_vectors_set_up(struct hfi1_devdata *dd);
+void hfi1_comp_vectors_clean_up(struct hfi1_devdata *dd);
 
 #endif /* _HFI1_AFFINITY_H */
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index e6bdd0c1e80a..68580cb2ae1e 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2017 Intel Corporation.
+ * Copyright(c) 2015 - 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -65,6 +65,7 @@
 #include "aspm.h"
 #include "affinity.h"
 #include "debugfs.h"
+#include "fault.h"
 
 #define NUM_IB_PORTS 1
 
@@ -1032,8 +1033,8 @@ static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
 				  u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
 static void read_vc_remote_link_width(struct hfi1_devdata *dd,
 				      u8 *remote_tx_rate, u16 *link_widths);
-static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
-				     u8 *flag_bits, u16 *link_widths);
+static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits,
+				    u8 *flag_bits, u16 *link_widths);
 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
 				  u8 *device_rev);
 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
@@ -6354,6 +6355,18 @@ static void handle_8051_request(struct hfi1_pportdata *ppd)
 		dd_dev_info(dd, "8051 request: request 0x%x not supported\n",
 			    type);
 		hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
+		break;
+	case HREQ_LCB_RESET:
+		/* Put the LCB, RX FPE and TX FPE into reset */
+		write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_INTO_RESET);
+		/* Make sure the write completed */
+		(void)read_csr(dd, DCC_CFG_RESET);
+		/* Hold the reset long enough to take effect */
+		udelay(1);
+		/* Take the LCB, RX FPE and TX FPE out of reset */
+		write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET);
+		hreq_response(dd, HREQ_SUCCESS, 0);
+
 		break;
 	case HREQ_CONFIG_DONE:
 		hreq_response(dd, HREQ_SUCCESS, 0);
@@ -6465,8 +6478,7 @@ static void lcb_shutdown(struct hfi1_devdata *dd, int abort)
 	dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN);
 	reg = read_csr(dd, DCC_CFG_RESET);
 	write_csr(dd, DCC_CFG_RESET, reg |
-		  (1ull << DCC_CFG_RESET_RESET_LCB_SHIFT) |
-		  (1ull << DCC_CFG_RESET_RESET_RX_FPE_SHIFT));
+		  DCC_CFG_RESET_RESET_LCB | DCC_CFG_RESET_RESET_RX_FPE);
 	(void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */
 	if (!abort) {
 		udelay(1);    /* must hold for the longer of 16cclks or 20ns */
@@ -6531,7 +6543,7 @@ static void _dc_start(struct hfi1_devdata *dd)
 			   __func__);
 
 	/* Take away reset for LCB and RX FPE (set in lcb_shutdown). */
-	write_csr(dd, DCC_CFG_RESET, 0x10);
+	write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET);
 	/* lcb_shutdown() with abort=1 does not restore these */
 	write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
 	dd->dc_shutdown = 0;
@@ -7352,7 +7364,7 @@ static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width,
 	u8 misc_bits, local_flags;
 	u16 active_tx, active_rx;
 
-	read_vc_local_link_width(dd, &misc_bits, &local_flags, &widths);
+	read_vc_local_link_mode(dd, &misc_bits, &local_flags, &widths);
 	tx = widths >> 12;
 	rx = (widths >> 8) & 0xf;
 
@@ -8824,29 +8836,29 @@ static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu,
 				GENERAL_CONFIG, frame);
 }
 
-static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
-				     u8 *flag_bits, u16 *link_widths)
+static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits,
+				    u8 *flag_bits, u16 *link_widths)
 {
 	u32 frame;
 
-	read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
+	read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG,
 			 &frame);
 	*misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK;
 	*flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK;
 	*link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
 }
 
-static int write_vc_local_link_width(struct hfi1_devdata *dd,
-				     u8 misc_bits,
-				     u8 flag_bits,
-				     u16 link_widths)
+static int write_vc_local_link_mode(struct hfi1_devdata *dd,
+				    u8 misc_bits,
+				    u8 flag_bits,
+				    u16 link_widths)
 {
 	u32 frame;
 
 	frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT
 		| (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT
 		| (u32)link_widths << LINK_WIDTH_SHIFT;
-	return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
+	return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG,
 		     frame);
 }
 
@@ -9316,8 +9328,16 @@ static int set_local_link_attributes(struct hfi1_pportdata *ppd)
 	if (loopback == LOOPBACK_SERDES)
 		misc_bits |= 1 << LOOPBACK_SERDES_CONFIG_BIT_MASK_SHIFT;
 
-	ret = write_vc_local_link_width(dd, misc_bits, 0,
-					opa_to_vc_link_widths(
+	/*
+	 * An external device configuration request is used to reset the LCB
+	 * to retry to obtain operational lanes when the first attempt is
+	 * unsuccesful.
+	 */
+	if (dd->dc8051_ver >= dc8051_ver(1, 25, 0))
+		misc_bits |= 1 << EXT_CFG_LCB_RESET_SUPPORTED_SHIFT;
+
+	ret = write_vc_local_link_mode(dd, misc_bits, 0,
+				       opa_to_vc_link_widths(
 						ppd->link_width_enabled));
 	if (ret != HCMD_SUCCESS)
 		goto set_local_link_attributes_fail;
@@ -10495,9 +10515,9 @@ u32 driver_pstate(struct hfi1_pportdata *ppd)
 	case HLS_DN_OFFLINE:
 		return OPA_PORTPHYSSTATE_OFFLINE;
 	case HLS_VERIFY_CAP:
-		return IB_PORTPHYSSTATE_POLLING;
+		return IB_PORTPHYSSTATE_TRAINING;
 	case HLS_GOING_UP:
-		return IB_PORTPHYSSTATE_POLLING;
+		return IB_PORTPHYSSTATE_TRAINING;
 	case HLS_GOING_OFFLINE:
 		return OPA_PORTPHYSSTATE_OFFLINE;
 	case HLS_LINK_COOLDOWN:
@@ -15022,13 +15042,6 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 	if (ret < 0)
 		goto bail_cleanup;
 
-	/* verify that reads actually work, save revision for reset check */
-	dd->revision = read_csr(dd, CCE_REVISION);
-	if (dd->revision == ~(u64)0) {
-		dd_dev_err(dd, "cannot read chip CSRs\n");
-		ret = -EINVAL;
-		goto bail_cleanup;
-	}
 	dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT)
 			& CCE_REVISION_CHIP_REV_MAJOR_MASK;
 	dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT)
@@ -15224,6 +15237,10 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 	if (ret)
 		goto bail_cleanup;
 
+	ret = hfi1_comp_vectors_set_up(dd);
+	if (ret)
+		goto bail_clear_intr;
+
 	/* set up LCB access - must be after set_up_interrupts() */
 	init_lcb_access(dd);
 
@@ -15266,6 +15283,7 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 bail_free_cntrs:
 	free_cntrs(dd);
 bail_clear_intr:
+	hfi1_comp_vectors_clean_up(dd);
 	hfi1_clean_up_interrupts(dd);
 bail_cleanup:
 	hfi1_pcie_ddcleanup(dd);
diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h
index c0d70f255050..fdf389e46e19 100644
--- a/drivers/infiniband/hw/hfi1/chip.h
+++ b/drivers/infiniband/hw/hfi1/chip.h
@@ -196,6 +196,15 @@
 #define LSTATE_ARMED   0x3
 #define LSTATE_ACTIVE  0x4
 
+/* DCC_CFG_RESET reset states */
+#define LCB_RX_FPE_TX_FPE_INTO_RESET   (DCC_CFG_RESET_RESET_LCB    | \
+					DCC_CFG_RESET_RESET_TX_FPE | \
+					DCC_CFG_RESET_RESET_RX_FPE | \
+					DCC_CFG_RESET_ENABLE_CCLK_BCC)
+					/* 0x17 */
+
+#define LCB_RX_FPE_TX_FPE_OUT_OF_RESET  DCC_CFG_RESET_ENABLE_CCLK_BCC /* 0x10 */
+
 /* DC8051_STS_CUR_STATE port values (physical link states) */
 #define PLS_DISABLED			   0x30
 #define PLS_OFFLINE				   0x90
@@ -283,6 +292,7 @@
 #define HREQ_SET_TX_EQ_ABS	0x04
 #define HREQ_SET_TX_EQ_REL	0x05
 #define HREQ_ENABLE		0x06
+#define HREQ_LCB_RESET		0x07
 #define HREQ_CONFIG_DONE	0xfe
 #define HREQ_INTERFACE_TEST	0xff
 
@@ -383,7 +393,7 @@
 #define TX_SETTINGS		     0x06
 #define VERIFY_CAP_LOCAL_PHY	     0x07
 #define VERIFY_CAP_LOCAL_FABRIC	     0x08
-#define VERIFY_CAP_LOCAL_LINK_WIDTH  0x09
+#define VERIFY_CAP_LOCAL_LINK_MODE   0x09
 #define LOCAL_DEVICE_ID		     0x0a
 #define RESERVED_REGISTERS	     0x0b
 #define LOCAL_LNI_INFO		     0x0c
@@ -584,8 +594,9 @@ enum {
 #define LOOPBACK_LCB	2
 #define LOOPBACK_CABLE	3	/* external cable */
 
-/* set up serdes bit in MISC_CONFIG_BITS */
+/* set up bits in MISC_CONFIG_BITS */
 #define LOOPBACK_SERDES_CONFIG_BIT_MASK_SHIFT 0
+#define EXT_CFG_LCB_RESET_SUPPORTED_SHIFT     3
 
 /* read and write hardware registers */
 u64 read_csr(const struct hfi1_devdata *dd, u32 offset);
diff --git a/drivers/infiniband/hw/hfi1/chip_registers.h b/drivers/infiniband/hw/hfi1/chip_registers.h
index 793514f1d15f..da598b5fe8f6 100644
--- a/drivers/infiniband/hw/hfi1/chip_registers.h
+++ b/drivers/infiniband/hw/hfi1/chip_registers.h
@@ -97,8 +97,11 @@
 #define DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT 32
 #define DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK 0x700000000ull
 #define DCC_CFG_RESET (DCC_CSRS + 0x000000000000)
-#define DCC_CFG_RESET_RESET_LCB_SHIFT 0
-#define DCC_CFG_RESET_RESET_RX_FPE_SHIFT 2
+#define DCC_CFG_RESET_RESET_LCB          BIT_ULL(0)
+#define DCC_CFG_RESET_RESET_TX_FPE       BIT_ULL(1)
+#define DCC_CFG_RESET_RESET_RX_FPE       BIT_ULL(2)
+#define DCC_CFG_RESET_RESET_8051         BIT_ULL(3)
+#define DCC_CFG_RESET_ENABLE_CCLK_BCC    BIT_ULL(4)
 #define DCC_CFG_SC_VL_TABLE_15_0 (DCC_CSRS + 0x000000000028)
 #define DCC_CFG_SC_VL_TABLE_15_0_ENTRY0_SHIFT 0
 #define DCC_CFG_SC_VL_TABLE_15_0_ENTRY10_SHIFT 40
diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c
index 852173bf05d0..9f992ae36c89 100644
--- a/drivers/infiniband/hw/hfi1/debugfs.c
+++ b/drivers/infiniband/hw/hfi1/debugfs.c
@@ -60,15 +60,13 @@
 #include "device.h"
 #include "qp.h"
 #include "sdma.h"
+#include "fault.h"
 
 static struct dentry *hfi1_dbg_root;
 
 /* wrappers to enforce srcu in seq file */
-static ssize_t hfi1_seq_read(
-	struct file *file,
-	char __user *buf,
-	size_t size,
-	loff_t *ppos)
+ssize_t hfi1_seq_read(struct file *file, char __user *buf, size_t size,
+		      loff_t *ppos)
 {
 	struct dentry *d = file->f_path.dentry;
 	ssize_t r;
@@ -81,10 +79,7 @@ static ssize_t hfi1_seq_read(
 	return r;
 }
 
-static loff_t hfi1_seq_lseek(
-	struct file *file,
-	loff_t offset,
-	int whence)
+loff_t hfi1_seq_lseek(struct file *file, loff_t offset, int whence)
 {
 	struct dentry *d = file->f_path.dentry;
 	loff_t r;
@@ -100,48 +95,6 @@ static loff_t hfi1_seq_lseek(
 #define private2dd(file) (file_inode(file)->i_private)
 #define private2ppd(file) (file_inode(file)->i_private)
 
-#define DEBUGFS_SEQ_FILE_OPS(name) \
-static const struct seq_operations _##name##_seq_ops = { \
-	.start = _##name##_seq_start, \
-	.next  = _##name##_seq_next, \
-	.stop  = _##name##_seq_stop, \
-	.show  = _##name##_seq_show \
-}
-
-#define DEBUGFS_SEQ_FILE_OPEN(name) \
-static int _##name##_open(struct inode *inode, struct file *s) \
-{ \
-	struct seq_file *seq; \
-	int ret; \
-	ret =  seq_open(s, &_##name##_seq_ops); \
-	if (ret) \
-		return ret; \
-	seq = s->private_data; \
-	seq->private = inode->i_private; \
-	return 0; \
-}
-
-#define DEBUGFS_FILE_OPS(name) \
-static const struct file_operations _##name##_file_ops = { \
-	.owner   = THIS_MODULE, \
-	.open    = _##name##_open, \
-	.read    = hfi1_seq_read, \
-	.llseek  = hfi1_seq_lseek, \
-	.release = seq_release \
-}
-
-#define DEBUGFS_FILE_CREATE(name, parent, data, ops, mode)	\
-do { \
-	struct dentry *ent; \
-	ent = debugfs_create_file(name, mode, parent, \
-		data, ops); \
-	if (!ent) \
-		pr_warn("create of %s failed\n", name); \
-} while (0)
-
-#define DEBUGFS_SEQ_FILE_CREATE(name, parent, data) \
-	DEBUGFS_FILE_CREATE(#name, parent, data, &_##name##_file_ops, S_IRUGO)
-
 static void *_opcode_stats_seq_start(struct seq_file *s, loff_t *pos)
 {
 	struct hfi1_opcode_stats_perctx *opstats;
@@ -1160,232 +1113,6 @@ DEBUGFS_SEQ_FILE_OPS(sdma_cpu_list);
 DEBUGFS_SEQ_FILE_OPEN(sdma_cpu_list)
 DEBUGFS_FILE_OPS(sdma_cpu_list);
 
-#ifdef CONFIG_FAULT_INJECTION
-static void *_fault_stats_seq_start(struct seq_file *s, loff_t *pos)
-{
-	struct hfi1_opcode_stats_perctx *opstats;
-
-	if (*pos >= ARRAY_SIZE(opstats->stats))
-		return NULL;
-	return pos;
-}
-
-static void *_fault_stats_seq_next(struct seq_file *s, void *v, loff_t *pos)
-{
-	struct hfi1_opcode_stats_perctx *opstats;
-
-	++*pos;
-	if (*pos >= ARRAY_SIZE(opstats->stats))
-		return NULL;
-	return pos;
-}
-
-static void _fault_stats_seq_stop(struct seq_file *s, void *v)
-{
-}
-
-static int _fault_stats_seq_show(struct seq_file *s, void *v)
-{
-	loff_t *spos = v;
-	loff_t i = *spos, j;
-	u64 n_packets = 0, n_bytes = 0;
-	struct hfi1_ibdev *ibd = (struct hfi1_ibdev *)s->private;
-	struct hfi1_devdata *dd = dd_from_dev(ibd);
-	struct hfi1_ctxtdata *rcd;
-
-	for (j = 0; j < dd->first_dyn_alloc_ctxt; j++) {
-		rcd = hfi1_rcd_get_by_index(dd, j);
-		if (rcd) {
-			n_packets += rcd->opstats->stats[i].n_packets;
-			n_bytes += rcd->opstats->stats[i].n_bytes;
-		}
-		hfi1_rcd_put(rcd);
-	}
-	for_each_possible_cpu(j) {
-		struct hfi1_opcode_stats_perctx *sp =
-			per_cpu_ptr(dd->tx_opstats, j);
-
-		n_packets += sp->stats[i].n_packets;
-		n_bytes += sp->stats[i].n_bytes;
-	}
-	if (!n_packets && !n_bytes)
-		return SEQ_SKIP;
-	if (!ibd->fault_opcode->n_rxfaults[i] &&
-	    !ibd->fault_opcode->n_txfaults[i])
-		return SEQ_SKIP;
-	seq_printf(s, "%02llx %llu/%llu (faults rx:%llu faults: tx:%llu)\n", i,
-		   (unsigned long long)n_packets,
-		   (unsigned long long)n_bytes,
-		   (unsigned long long)ibd->fault_opcode->n_rxfaults[i],
-		   (unsigned long long)ibd->fault_opcode->n_txfaults[i]);
-	return 0;
-}
-
-DEBUGFS_SEQ_FILE_OPS(fault_stats);
-DEBUGFS_SEQ_FILE_OPEN(fault_stats);
-DEBUGFS_FILE_OPS(fault_stats);
-
-static void fault_exit_opcode_debugfs(struct hfi1_ibdev *ibd)
-{
-	debugfs_remove_recursive(ibd->fault_opcode->dir);
-	kfree(ibd->fault_opcode);
-	ibd->fault_opcode = NULL;
-}
-
-static int fault_init_opcode_debugfs(struct hfi1_ibdev *ibd)
-{
-	struct dentry *parent = ibd->hfi1_ibdev_dbg;
-
-	ibd->fault_opcode = kzalloc(sizeof(*ibd->fault_opcode), GFP_KERNEL);
-	if (!ibd->fault_opcode)
-		return -ENOMEM;
-
-	ibd->fault_opcode->attr.interval = 1;
-	ibd->fault_opcode->attr.require_end = ULONG_MAX;
-	ibd->fault_opcode->attr.stacktrace_depth = 32;
-	ibd->fault_opcode->attr.dname = NULL;
-	ibd->fault_opcode->attr.verbose = 0;
-	ibd->fault_opcode->fault_by_opcode = false;
-	ibd->fault_opcode->opcode = 0;
-	ibd->fault_opcode->mask = 0xff;
-
-	ibd->fault_opcode->dir =
-		fault_create_debugfs_attr("fault_opcode",
-					  parent,
-					  &ibd->fault_opcode->attr);
-	if (IS_ERR(ibd->fault_opcode->dir)) {
-		kfree(ibd->fault_opcode);
-		return -ENOENT;
-	}
-
-	DEBUGFS_SEQ_FILE_CREATE(fault_stats, ibd->fault_opcode->dir, ibd);
-	if (!debugfs_create_bool("fault_by_opcode", 0600,
-				 ibd->fault_opcode->dir,
-				 &ibd->fault_opcode->fault_by_opcode))
-		goto fail;
-	if (!debugfs_create_x8("opcode", 0600, ibd->fault_opcode->dir,
-			       &ibd->fault_opcode->opcode))
-		goto fail;
-	if (!debugfs_create_x8("mask", 0600, ibd->fault_opcode->dir,
-			       &ibd->fault_opcode->mask))
-		goto fail;
-
-	return 0;
-fail:
-	fault_exit_opcode_debugfs(ibd);
-	return -ENOMEM;
-}
-
-static void fault_exit_packet_debugfs(struct hfi1_ibdev *ibd)
-{
-	debugfs_remove_recursive(ibd->fault_packet->dir);
-	kfree(ibd->fault_packet);
-	ibd->fault_packet = NULL;
-}
-
-static int fault_init_packet_debugfs(struct hfi1_ibdev *ibd)
-{
-	struct dentry *parent = ibd->hfi1_ibdev_dbg;
-
-	ibd->fault_packet = kzalloc(sizeof(*ibd->fault_packet), GFP_KERNEL);
-	if (!ibd->fault_packet)
-		return -ENOMEM;
-
-	ibd->fault_packet->attr.interval = 1;
-	ibd->fault_packet->attr.require_end = ULONG_MAX;
-	ibd->fault_packet->attr.stacktrace_depth = 32;
-	ibd->fault_packet->attr.dname = NULL;
-	ibd->fault_packet->attr.verbose = 0;
-	ibd->fault_packet->fault_by_packet = false;
-
-	ibd->fault_packet->dir =
-		fault_create_debugfs_attr("fault_packet",
-					  parent,
-					  &ibd->fault_opcode->attr);
-	if (IS_ERR(ibd->fault_packet->dir)) {
-		kfree(ibd->fault_packet);
-		return -ENOENT;
-	}
-
-	if (!debugfs_create_bool("fault_by_packet", 0600,
-				 ibd->fault_packet->dir,
-				 &ibd->fault_packet->fault_by_packet))
-		goto fail;
-	if (!debugfs_create_u64("fault_stats", 0400,
-				ibd->fault_packet->dir,
-				&ibd->fault_packet->n_faults))
-		goto fail;
-
-	return 0;
-fail:
-	fault_exit_packet_debugfs(ibd);
-	return -ENOMEM;
-}
-
-static void fault_exit_debugfs(struct hfi1_ibdev *ibd)
-{
-	fault_exit_opcode_debugfs(ibd);
-	fault_exit_packet_debugfs(ibd);
-}
-
-static int fault_init_debugfs(struct hfi1_ibdev *ibd)
-{
-	int ret = 0;
-
-	ret = fault_init_opcode_debugfs(ibd);
-	if (ret)
-		return ret;
-
-	ret = fault_init_packet_debugfs(ibd);
-	if (ret)
-		fault_exit_opcode_debugfs(ibd);
-
-	return ret;
-}
-
-bool hfi1_dbg_fault_suppress_err(struct hfi1_ibdev *ibd)
-{
-	return ibd->fault_suppress_err;
-}
-
-bool hfi1_dbg_fault_opcode(struct rvt_qp *qp, u32 opcode, bool rx)
-{
-	bool ret = false;
-	struct hfi1_ibdev *ibd = to_idev(qp->ibqp.device);
-
-	if (!ibd->fault_opcode || !ibd->fault_opcode->fault_by_opcode)
-		return false;
-	if (ibd->fault_opcode->opcode != (opcode & ibd->fault_opcode->mask))
-		return false;
-	ret = should_fail(&ibd->fault_opcode->attr, 1);
-	if (ret) {
-		trace_hfi1_fault_opcode(qp, opcode);
-		if (rx)
-			ibd->fault_opcode->n_rxfaults[opcode]++;
-		else
-			ibd->fault_opcode->n_txfaults[opcode]++;
-	}
-	return ret;
-}
-
-bool hfi1_dbg_fault_packet(struct hfi1_packet *packet)
-{
-	struct rvt_dev_info *rdi = &packet->rcd->ppd->dd->verbs_dev.rdi;
-	struct hfi1_ibdev *ibd = dev_from_rdi(rdi);
-	bool ret = false;
-
-	if (!ibd->fault_packet || !ibd->fault_packet->fault_by_packet)
-		return false;
-
-	ret = should_fail(&ibd->fault_packet->attr, 1);
-	if (ret) {
-		++ibd->fault_packet->n_faults;
-		trace_hfi1_fault_packet(packet);
-	}
-	return ret;
-}
-#endif
-
 void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd)
 {
 	char name[sizeof("port0counters") + 1];
@@ -1438,21 +1165,14 @@ void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd)
 					    S_IRUGO : S_IRUGO | S_IWUSR);
 		}
 
-#ifdef CONFIG_FAULT_INJECTION
-	debugfs_create_bool("fault_suppress_err", 0600,
-			    ibd->hfi1_ibdev_dbg,
-			    &ibd->fault_suppress_err);
-	fault_init_debugfs(ibd);
-#endif
+	hfi1_fault_init_debugfs(ibd);
 }
 
 void hfi1_dbg_ibdev_exit(struct hfi1_ibdev *ibd)
 {
 	if (!hfi1_dbg_root)
 		goto out;
-#ifdef CONFIG_FAULT_INJECTION
-	fault_exit_debugfs(ibd);
-#endif
+	hfi1_fault_exit_debugfs(ibd);
 	debugfs_remove(ibd->hfi1_ibdev_link);
 	debugfs_remove_recursive(ibd->hfi1_ibdev_dbg);
 out:
diff --git a/drivers/infiniband/hw/hfi1/debugfs.h b/drivers/infiniband/hw/hfi1/debugfs.h
index 38c38a98156d..d5d824459fcc 100644
--- a/drivers/infiniband/hw/hfi1/debugfs.h
+++ b/drivers/infiniband/hw/hfi1/debugfs.h
@@ -1,7 +1,7 @@
 #ifndef _HFI1_DEBUGFS_H
 #define _HFI1_DEBUGFS_H
 /*
- * Copyright(c) 2015, 2016 Intel Corporation.
+ * Copyright(c) 2015, 2016, 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -48,51 +48,59 @@
  */
 
 struct hfi1_ibdev;
-#ifdef CONFIG_DEBUG_FS
-void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd);
-void hfi1_dbg_ibdev_exit(struct hfi1_ibdev *ibd);
-void hfi1_dbg_init(void);
-void hfi1_dbg_exit(void);
-
-#ifdef CONFIG_FAULT_INJECTION
-#include <linux/fault-inject.h>
-struct fault_opcode {
-	struct fault_attr attr;
-	struct dentry *dir;
-	bool fault_by_opcode;
-	u64 n_rxfaults[256];
-	u64 n_txfaults[256];
-	u8 opcode;
-	u8 mask;
-};
 
-struct fault_packet {
-	struct fault_attr attr;
-	struct dentry *dir;
-	bool fault_by_packet;
-	u64 n_faults;
-};
+#define DEBUGFS_FILE_CREATE(name, parent, data, ops, mode)	\
+do { \
+	struct dentry *ent; \
+	const char *__name = name; \
+	ent = debugfs_create_file(__name, mode, parent, \
+		data, ops); \
+	if (!ent) \
+		pr_warn("create of %s failed\n", __name); \
+} while (0)
 
-bool hfi1_dbg_fault_opcode(struct rvt_qp *qp, u32 opcode, bool rx);
-bool hfi1_dbg_fault_packet(struct hfi1_packet *packet);
-bool hfi1_dbg_fault_suppress_err(struct hfi1_ibdev *ibd);
-#else
-static inline bool hfi1_dbg_fault_packet(struct hfi1_packet *packet)
-{
-	return false;
+#define DEBUGFS_SEQ_FILE_OPS(name) \
+static const struct seq_operations _##name##_seq_ops = { \
+	.start = _##name##_seq_start, \
+	.next  = _##name##_seq_next, \
+	.stop  = _##name##_seq_stop, \
+	.show  = _##name##_seq_show \
 }
 
-static inline bool hfi1_dbg_fault_opcode(struct rvt_qp *qp,
-					 u32 opcode, bool rx)
-{
-	return false;
+#define DEBUGFS_SEQ_FILE_OPEN(name) \
+static int _##name##_open(struct inode *inode, struct file *s) \
+{ \
+	struct seq_file *seq; \
+	int ret; \
+	ret =  seq_open(s, &_##name##_seq_ops); \
+	if (ret) \
+		return ret; \
+	seq = s->private_data; \
+	seq->private = inode->i_private; \
+	return 0; \
 }
 
-static inline bool hfi1_dbg_fault_suppress_err(struct hfi1_ibdev *ibd)
-{
-	return false;
+#define DEBUGFS_FILE_OPS(name) \
+static const struct file_operations _##name##_file_ops = { \
+	.owner   = THIS_MODULE, \
+	.open    = _##name##_open, \
+	.read    = hfi1_seq_read, \
+	.llseek  = hfi1_seq_lseek, \
+	.release = seq_release \
 }
-#endif
+
+#define DEBUGFS_SEQ_FILE_CREATE(name, parent, data) \
+	DEBUGFS_FILE_CREATE(#name, parent, data, &_##name##_file_ops, 0444)
+
+ssize_t hfi1_seq_read(struct file *file, char __user *buf, size_t size,
+		      loff_t *ppos);
+loff_t hfi1_seq_lseek(struct file *file, loff_t offset, int whence);
+
+#ifdef CONFIG_DEBUG_FS
+void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd);
+void hfi1_dbg_ibdev_exit(struct hfi1_ibdev *ibd);
+void hfi1_dbg_init(void);
+void hfi1_dbg_exit(void);
 
 #else
 static inline void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd)
@@ -110,22 +118,6 @@ static inline void hfi1_dbg_init(void)
 static inline void hfi1_dbg_exit(void)
 {
 }
-
-static inline bool hfi1_dbg_fault_packet(struct hfi1_packet *packet)
-{
-	return false;
-}
-
-static inline bool hfi1_dbg_fault_opcode(struct rvt_qp *qp,
-					 u32 opcode, bool rx)
-{
-	return false;
-}
-
-static inline bool hfi1_dbg_fault_suppress_err(struct hfi1_ibdev *ibd)
-{
-	return false;
-}
 #endif
 
 #endif                          /* _HFI1_DEBUGFS_H */
diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c
index bd837a048bf4..94dca95db04f 100644
--- a/drivers/infiniband/hw/hfi1/driver.c
+++ b/drivers/infiniband/hw/hfi1/driver.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015-2017 Intel Corporation.
+ * Copyright(c) 2015-2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -61,6 +61,7 @@
 #include "sdma.h"
 #include "debugfs.h"
 #include "vnic.h"
+#include "fault.h"
 
 #undef pr_fmt
 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
@@ -1482,38 +1483,51 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)
 	struct hfi1_pportdata *ppd = rcd->ppd;
 	struct hfi1_ibport *ibp = &ppd->ibport_data;
 	u8 l4;
-	u8 grh_len;
 
 	packet->hdr = (struct hfi1_16b_header *)
 			hfi1_get_16B_header(packet->rcd->dd,
 					    packet->rhf_addr);
-	packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
-
 	l4 = hfi1_16B_get_l4(packet->hdr);
 	if (l4 == OPA_16B_L4_IB_LOCAL) {
-		grh_len = 0;
 		packet->ohdr = packet->ebuf;
 		packet->grh = NULL;
+		packet->opcode = ib_bth_get_opcode(packet->ohdr);
+		packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
+		/* hdr_len_by_opcode already has an IB LRH factored in */
+		packet->hlen = hdr_len_by_opcode[packet->opcode] +
+			(LRH_16B_BYTES - LRH_9B_BYTES);
+		packet->migrated = opa_bth_is_migration(packet->ohdr);
 	} else if (l4 == OPA_16B_L4_IB_GLOBAL) {
 		u32 vtf;
+		u8 grh_len = sizeof(struct ib_grh);
 
-		grh_len = sizeof(struct ib_grh);
 		packet->ohdr = packet->ebuf + grh_len;
 		packet->grh = packet->ebuf;
+		packet->opcode = ib_bth_get_opcode(packet->ohdr);
+		packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
+		/* hdr_len_by_opcode already has an IB LRH factored in */
+		packet->hlen = hdr_len_by_opcode[packet->opcode] +
+			(LRH_16B_BYTES - LRH_9B_BYTES) + grh_len;
+		packet->migrated = opa_bth_is_migration(packet->ohdr);
+
 		if (packet->grh->next_hdr != IB_GRH_NEXT_HDR)
 			goto drop;
 		vtf = be32_to_cpu(packet->grh->version_tclass_flow);
 		if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
 			goto drop;
+	} else if (l4 == OPA_16B_L4_FM) {
+		packet->mgmt = packet->ebuf;
+		packet->ohdr = NULL;
+		packet->grh = NULL;
+		packet->opcode = IB_OPCODE_UD_SEND_ONLY;
+		packet->pad = OPA_16B_L4_FM_PAD;
+		packet->hlen = OPA_16B_L4_FM_HLEN;
+		packet->migrated = false;
 	} else {
 		goto drop;
 	}
 
 	/* Query commonly used fields from packet header */
-	packet->opcode = ib_bth_get_opcode(packet->ohdr);
-	/* hdr_len_by_opcode already has an IB LRH factored in */
-	packet->hlen = hdr_len_by_opcode[packet->opcode] +
-		(LRH_16B_BYTES - LRH_9B_BYTES) + grh_len;
 	packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES;
 	packet->slid = hfi1_16B_get_slid(packet->hdr);
 	packet->dlid = hfi1_16B_get_dlid(packet->hdr);
@@ -1523,10 +1537,8 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)
 					    16B);
 	packet->sc = hfi1_16B_get_sc(packet->hdr);
 	packet->sl = ibp->sc_to_sl[packet->sc];
-	packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
 	packet->extra_byte = SIZE_OF_LT;
 	packet->pkey = hfi1_16B_get_pkey(packet->hdr);
-	packet->migrated = opa_bth_is_migration(packet->ohdr);
 
 	if (hfi1_bypass_ingress_pkt_check(packet))
 		goto drop;
@@ -1565,10 +1577,10 @@ void handle_eflags(struct hfi1_packet *packet)
  */
 int process_receive_ib(struct hfi1_packet *packet)
 {
-	if (unlikely(hfi1_dbg_fault_packet(packet)))
+	if (hfi1_setup_9B_packet(packet))
 		return RHF_RCV_CONTINUE;
 
-	if (hfi1_setup_9B_packet(packet))
+	if (unlikely(hfi1_dbg_should_fault_rx(packet)))
 		return RHF_RCV_CONTINUE;
 
 	trace_hfi1_rcvhdr(packet);
@@ -1642,7 +1654,8 @@ int process_receive_error(struct hfi1_packet *packet)
 	/* KHdrHCRCErr -- KDETH packet with a bad HCRC */
 	if (unlikely(
 		 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) &&
-		 rhf_rcv_type_err(packet->rhf) == 3))
+		 (rhf_rcv_type_err(packet->rhf) == RHF_RCV_TYPE_ERROR ||
+		  packet->rhf & RHF_DC_ERR)))
 		return RHF_RCV_CONTINUE;
 
 	hfi1_setup_ib_header(packet);
@@ -1657,10 +1670,10 @@ int process_receive_error(struct hfi1_packet *packet)
 
 int kdeth_process_expected(struct hfi1_packet *packet)
 {
-	if (unlikely(hfi1_dbg_fault_packet(packet)))
+	hfi1_setup_9B_packet(packet);
+	if (unlikely(hfi1_dbg_should_fault_rx(packet)))
 		return RHF_RCV_CONTINUE;
 
-	hfi1_setup_ib_header(packet);
 	if (unlikely(rhf_err_flags(packet->rhf)))
 		handle_eflags(packet);
 
@@ -1671,11 +1684,11 @@ int kdeth_process_expected(struct hfi1_packet *packet)
 
 int kdeth_process_eager(struct hfi1_packet *packet)
 {
-	hfi1_setup_ib_header(packet);
+	hfi1_setup_9B_packet(packet);
+	if (unlikely(hfi1_dbg_should_fault_rx(packet)))
+		return RHF_RCV_CONTINUE;
 	if (unlikely(rhf_err_flags(packet->rhf)))
 		handle_eflags(packet);
-	if (unlikely(hfi1_dbg_fault_packet(packet)))
-		return RHF_RCV_CONTINUE;
 
 	dd_dev_err(packet->rcd->dd,
 		   "Unhandled eager packet received. Dropping.\n");
diff --git a/drivers/infiniband/hw/hfi1/exp_rcv.c b/drivers/infiniband/hw/hfi1/exp_rcv.c
index 0af91675acc6..1be49a0d9c11 100644
--- a/drivers/infiniband/hw/hfi1/exp_rcv.c
+++ b/drivers/infiniband/hw/hfi1/exp_rcv.c
@@ -52,12 +52,23 @@
  * exp_tid_group_init - initialize exp_tid_set
  * @set - the set
  */
-void hfi1_exp_tid_group_init(struct exp_tid_set *set)
+static void hfi1_exp_tid_set_init(struct exp_tid_set *set)
 {
 	INIT_LIST_HEAD(&set->list);
 	set->count = 0;
 }
 
+/**
+ * hfi1_exp_tid_group_init - initialize rcd expected receive
+ * @rcd - the rcd
+ */
+void hfi1_exp_tid_group_init(struct hfi1_ctxtdata *rcd)
+{
+	hfi1_exp_tid_set_init(&rcd->tid_group_list);
+	hfi1_exp_tid_set_init(&rcd->tid_used_list);
+	hfi1_exp_tid_set_init(&rcd->tid_full_list);
+}
+
 /**
  * alloc_ctxt_rcv_groups - initialize expected receive groups
  * @rcd - the context to add the groupings to
@@ -68,13 +79,17 @@ int hfi1_alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd)
 	u32 tidbase;
 	struct tid_group *grp;
 	int i;
+	u32 ngroups;
 
+	ngroups = rcd->expected_count / dd->rcv_entries.group_size;
+	rcd->groups =
+		kcalloc_node(ngroups, sizeof(*rcd->groups),
+			     GFP_KERNEL, rcd->numa_id);
+	if (!rcd->groups)
+		return -ENOMEM;
 	tidbase = rcd->expected_base;
-	for (i = 0; i < rcd->expected_count /
-		     dd->rcv_entries.group_size; i++) {
-		grp = kzalloc(sizeof(*grp), GFP_KERNEL);
-		if (!grp)
-			goto bail;
+	for (i = 0; i < ngroups; i++) {
+		grp = &rcd->groups[i];
 		grp->size = dd->rcv_entries.group_size;
 		grp->base = tidbase;
 		tid_group_add_tail(grp, &rcd->tid_group_list);
@@ -82,9 +97,6 @@ int hfi1_alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd)
 	}
 
 	return 0;
-bail:
-	hfi1_free_ctxt_rcv_groups(rcd);
-	return -ENOMEM;
 }
 
 /**
@@ -100,15 +112,12 @@ int hfi1_alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd)
  */
 void hfi1_free_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd)
 {
-	struct tid_group *grp, *gptr;
-
 	WARN_ON(!EXP_TID_SET_EMPTY(rcd->tid_full_list));
 	WARN_ON(!EXP_TID_SET_EMPTY(rcd->tid_used_list));
 
-	list_for_each_entry_safe(grp, gptr, &rcd->tid_group_list.list, list) {
-		tid_group_remove(grp, &rcd->tid_group_list);
-		kfree(grp);
-	}
+	kfree(rcd->groups);
+	rcd->groups = NULL;
+	hfi1_exp_tid_group_init(rcd);
 
 	hfi1_clear_tids(rcd);
 }
diff --git a/drivers/infiniband/hw/hfi1/exp_rcv.h b/drivers/infiniband/hw/hfi1/exp_rcv.h
index 08719047628a..f25362015095 100644
--- a/drivers/infiniband/hw/hfi1/exp_rcv.h
+++ b/drivers/infiniband/hw/hfi1/exp_rcv.h
@@ -183,8 +183,30 @@ static inline u32 rcventry2tidinfo(u32 rcventry)
 		EXP_TID_SET(CTRL, 1 << (rcventry - pair));
 }
 
+/**
+ * hfi1_tid_group_to_idx - convert an index to a group
+ * @rcd - the receive context
+ * @grp - the group pointer
+ */
+static inline u16
+hfi1_tid_group_to_idx(struct hfi1_ctxtdata *rcd, struct tid_group *grp)
+{
+	return grp - &rcd->groups[0];
+}
+
+/**
+ * hfi1_idx_to_tid_group - convert a group to an index
+ * @rcd - the receive context
+ * @idx - the index
+ */
+static inline struct tid_group *
+hfi1_idx_to_tid_group(struct hfi1_ctxtdata *rcd, u16 idx)
+{
+	return &rcd->groups[idx];
+}
+
 int hfi1_alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd);
 void hfi1_free_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd);
-void hfi1_exp_tid_group_init(struct exp_tid_set *set);
+void hfi1_exp_tid_group_init(struct hfi1_ctxtdata *rcd);
 
 #endif /* _HFI1_EXP_RCV_H */
diff --git a/drivers/infiniband/hw/hfi1/fault.c b/drivers/infiniband/hw/hfi1/fault.c
new file mode 100644
index 000000000000..e2290f32c8d9
--- /dev/null
+++ b/drivers/infiniband/hw/hfi1/fault.c
@@ -0,0 +1,375 @@
+/*
+ * Copyright(c) 2018 Intel Corporation.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * BSD LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  - Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  - Neither the name of Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/bitmap.h>
+
+#include "debugfs.h"
+#include "fault.h"
+#include "trace.h"
+
+#define HFI1_FAULT_DIR_TX   BIT(0)
+#define HFI1_FAULT_DIR_RX   BIT(1)
+#define HFI1_FAULT_DIR_TXRX (HFI1_FAULT_DIR_TX | HFI1_FAULT_DIR_RX)
+
+static void *_fault_stats_seq_start(struct seq_file *s, loff_t *pos)
+{
+	struct hfi1_opcode_stats_perctx *opstats;
+
+	if (*pos >= ARRAY_SIZE(opstats->stats))
+		return NULL;
+	return pos;
+}
+
+static void *_fault_stats_seq_next(struct seq_file *s, void *v, loff_t *pos)
+{
+	struct hfi1_opcode_stats_perctx *opstats;
+
+	++*pos;
+	if (*pos >= ARRAY_SIZE(opstats->stats))
+		return NULL;
+	return pos;
+}
+
+static void _fault_stats_seq_stop(struct seq_file *s, void *v)
+{
+}
+
+static int _fault_stats_seq_show(struct seq_file *s, void *v)
+{
+	loff_t *spos = v;
+	loff_t i = *spos, j;
+	u64 n_packets = 0, n_bytes = 0;
+	struct hfi1_ibdev *ibd = (struct hfi1_ibdev *)s->private;
+	struct hfi1_devdata *dd = dd_from_dev(ibd);
+	struct hfi1_ctxtdata *rcd;
+
+	for (j = 0; j < dd->first_dyn_alloc_ctxt; j++) {
+		rcd = hfi1_rcd_get_by_index(dd, j);
+		if (rcd) {
+			n_packets += rcd->opstats->stats[i].n_packets;
+			n_bytes += rcd->opstats->stats[i].n_bytes;
+		}
+		hfi1_rcd_put(rcd);
+	}
+	for_each_possible_cpu(j) {
+		struct hfi1_opcode_stats_perctx *sp =
+			per_cpu_ptr(dd->tx_opstats, j);
+
+		n_packets += sp->stats[i].n_packets;
+		n_bytes += sp->stats[i].n_bytes;
+	}
+	if (!n_packets && !n_bytes)
+		return SEQ_SKIP;
+	if (!ibd->fault->n_rxfaults[i] && !ibd->fault->n_txfaults[i])
+		return SEQ_SKIP;
+	seq_printf(s, "%02llx %llu/%llu (faults rx:%llu faults: tx:%llu)\n", i,
+		   (unsigned long long)n_packets,
+		   (unsigned long long)n_bytes,
+		   (unsigned long long)ibd->fault->n_rxfaults[i],
+		   (unsigned long long)ibd->fault->n_txfaults[i]);
+	return 0;
+}
+
+DEBUGFS_SEQ_FILE_OPS(fault_stats);
+DEBUGFS_SEQ_FILE_OPEN(fault_stats);
+DEBUGFS_FILE_OPS(fault_stats);
+
+static int fault_opcodes_open(struct inode *inode, struct file *file)
+{
+	file->private_data = inode->i_private;
+	return nonseekable_open(inode, file);
+}
+
+static ssize_t fault_opcodes_write(struct file *file, const char __user *buf,
+				   size_t len, loff_t *pos)
+{
+	ssize_t ret = 0;
+	/* 1280 = 256 opcodes * 4 chars/opcode + 255 commas + NULL */
+	size_t copy, datalen = 1280;
+	char *data, *token, *ptr, *end;
+	struct fault *fault = file->private_data;
+
+	data = kcalloc(datalen, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+	copy = min(len, datalen - 1);
+	if (copy_from_user(data, buf, copy))
+		return -EFAULT;
+
+	ret = debugfs_file_get(file->f_path.dentry);
+	if (unlikely(ret))
+		return ret;
+	ptr = data;
+	token = ptr;
+	for (ptr = data; *ptr; ptr = end + 1, token = ptr) {
+		char *dash;
+		unsigned long range_start, range_end, i;
+		bool remove = false;
+
+		end = strchr(ptr, ',');
+		if (end)
+			*end = '\0';
+		if (token[0] == '-') {
+			remove = true;
+			token++;
+		}
+		dash = strchr(token, '-');
+		if (dash)
+			*dash = '\0';
+		if (kstrtoul(token, 0, &range_start))
+			break;
+		if (dash) {
+			token = dash + 1;
+			if (kstrtoul(token, 0, &range_end))
+				break;
+		} else {
+			range_end = range_start;
+		}
+		if (range_start == range_end && range_start == -1UL) {
+			bitmap_zero(fault->opcodes, sizeof(fault->opcodes) *
+				    BITS_PER_BYTE);
+			break;
+		}
+		for (i = range_start; i <= range_end; i++) {
+			if (remove)
+				clear_bit(i, fault->opcodes);
+			else
+				set_bit(i, fault->opcodes);
+		}
+		if (!end)
+			break;
+	}
+	ret = len;
+
+	debugfs_file_put(file->f_path.dentry);
+	kfree(data);
+	return ret;
+}
+
+static ssize_t fault_opcodes_read(struct file *file, char __user *buf,
+				  size_t len, loff_t *pos)
+{
+	ssize_t ret = 0;
+	char *data;
+	size_t datalen = 1280, size = 0; /* see fault_opcodes_write() */
+	unsigned long bit = 0, zero = 0;
+	struct fault *fault = file->private_data;
+	size_t bitsize = sizeof(fault->opcodes) * BITS_PER_BYTE;
+
+	data = kcalloc(datalen, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+	ret = debugfs_file_get(file->f_path.dentry);
+	if (unlikely(ret))
+		return ret;
+	bit = find_first_bit(fault->opcodes, bitsize);
+	while (bit < bitsize) {
+		zero = find_next_zero_bit(fault->opcodes, bitsize, bit);
+		if (zero - 1 != bit)
+			size += snprintf(data + size,
+					 datalen - size - 1,
+					 "0x%lx-0x%lx,", bit, zero - 1);
+		else
+			size += snprintf(data + size,
+					 datalen - size - 1, "0x%lx,",
+					 bit);
+		bit = find_next_bit(fault->opcodes, bitsize, zero);
+	}
+	debugfs_file_put(file->f_path.dentry);
+	data[size - 1] = '\n';
+	data[size] = '\0';
+	ret = simple_read_from_buffer(buf, len, pos, data, size);
+	kfree(data);
+	return ret;
+}
+
+static const struct file_operations __fault_opcodes_fops = {
+	.owner = THIS_MODULE,
+	.open = fault_opcodes_open,
+	.read = fault_opcodes_read,
+	.write = fault_opcodes_write,
+	.llseek = no_llseek
+};
+
+void hfi1_fault_exit_debugfs(struct hfi1_ibdev *ibd)
+{
+	if (ibd->fault)
+		debugfs_remove_recursive(ibd->fault->dir);
+	kfree(ibd->fault);
+	ibd->fault = NULL;
+}
+
+int hfi1_fault_init_debugfs(struct hfi1_ibdev *ibd)
+{
+	struct dentry *parent = ibd->hfi1_ibdev_dbg;
+
+	ibd->fault = kzalloc(sizeof(*ibd->fault), GFP_KERNEL);
+	if (!ibd->fault)
+		return -ENOMEM;
+
+	ibd->fault->attr.interval = 1;
+	ibd->fault->attr.require_end = ULONG_MAX;
+	ibd->fault->attr.stacktrace_depth = 32;
+	ibd->fault->attr.dname = NULL;
+	ibd->fault->attr.verbose = 0;
+	ibd->fault->enable = false;
+	ibd->fault->opcode = false;
+	ibd->fault->fault_skip = 0;
+	ibd->fault->skip = 0;
+	ibd->fault->direction = HFI1_FAULT_DIR_TXRX;
+	ibd->fault->suppress_err = false;
+	bitmap_zero(ibd->fault->opcodes,
+		    sizeof(ibd->fault->opcodes) * BITS_PER_BYTE);
+
+	ibd->fault->dir =
+		fault_create_debugfs_attr("fault", parent,
+					  &ibd->fault->attr);
+	if (IS_ERR(ibd->fault->dir)) {
+		kfree(ibd->fault);
+		ibd->fault = NULL;
+		return -ENOENT;
+	}
+
+	DEBUGFS_SEQ_FILE_CREATE(fault_stats, ibd->fault->dir, ibd);
+	if (!debugfs_create_bool("enable", 0600, ibd->fault->dir,
+				 &ibd->fault->enable))
+		goto fail;
+	if (!debugfs_create_bool("suppress_err", 0600,
+				 ibd->fault->dir,
+				 &ibd->fault->suppress_err))
+		goto fail;
+	if (!debugfs_create_bool("opcode_mode", 0600, ibd->fault->dir,
+				 &ibd->fault->opcode))
+		goto fail;
+	if (!debugfs_create_file("opcodes", 0600, ibd->fault->dir,
+				 ibd->fault, &__fault_opcodes_fops))
+		goto fail;
+	if (!debugfs_create_u64("skip_pkts", 0600,
+				ibd->fault->dir,
+				&ibd->fault->fault_skip))
+		goto fail;
+	if (!debugfs_create_u64("skip_usec", 0600,
+				ibd->fault->dir,
+				&ibd->fault->fault_skip_usec))
+		goto fail;
+	if (!debugfs_create_u8("direction", 0600, ibd->fault->dir,
+			       &ibd->fault->direction))
+		goto fail;
+
+	return 0;
+fail:
+	hfi1_fault_exit_debugfs(ibd);
+	return -ENOMEM;
+}
+
+bool hfi1_dbg_fault_suppress_err(struct hfi1_ibdev *ibd)
+{
+	if (ibd->fault)
+		return ibd->fault->suppress_err;
+	return false;
+}
+
+static bool __hfi1_should_fault(struct hfi1_ibdev *ibd, u32 opcode,
+				u8 direction)
+{
+	bool ret = false;
+
+	if (!ibd->fault || !ibd->fault->enable)
+		return false;
+	if (!(ibd->fault->direction & direction))
+		return false;
+	if (ibd->fault->opcode) {
+		if (bitmap_empty(ibd->fault->opcodes,
+				 (sizeof(ibd->fault->opcodes) *
+				  BITS_PER_BYTE)))
+			return false;
+		if (!(test_bit(opcode, ibd->fault->opcodes)))
+			return false;
+	}
+	if (ibd->fault->fault_skip_usec &&
+	    time_before(jiffies, ibd->fault->skip_usec))
+		return false;
+	if (ibd->fault->fault_skip && ibd->fault->skip) {
+		ibd->fault->skip--;
+		return false;
+	}
+	ret = should_fail(&ibd->fault->attr, 1);
+	if (ret) {
+		ibd->fault->skip = ibd->fault->fault_skip;
+		ibd->fault->skip_usec = jiffies +
+			usecs_to_jiffies(ibd->fault->fault_skip_usec);
+	}
+	return ret;
+}
+
+bool hfi1_dbg_should_fault_tx(struct rvt_qp *qp, u32 opcode)
+{
+	struct hfi1_ibdev *ibd = to_idev(qp->ibqp.device);
+
+	if (__hfi1_should_fault(ibd, opcode, HFI1_FAULT_DIR_TX)) {
+		trace_hfi1_fault_opcode(qp, opcode);
+		ibd->fault->n_txfaults[opcode]++;
+		return true;
+	}
+	return false;
+}
+
+bool hfi1_dbg_should_fault_rx(struct hfi1_packet *packet)
+{
+	struct hfi1_ibdev *ibd = &packet->rcd->dd->verbs_dev;
+
+	if (__hfi1_should_fault(ibd, packet->opcode, HFI1_FAULT_DIR_RX)) {
+		trace_hfi1_fault_packet(packet);
+		ibd->fault->n_rxfaults[packet->opcode]++;
+		return true;
+	}
+	return false;
+}
diff --git a/drivers/infiniband/hw/hfi1/fault.h b/drivers/infiniband/hw/hfi1/fault.h
new file mode 100644
index 000000000000..a83382700a7c
--- /dev/null
+++ b/drivers/infiniband/hw/hfi1/fault.h
@@ -0,0 +1,109 @@
+#ifndef _HFI1_FAULT_H
+#define _HFI1_FAULT_H
+/*
+ * Copyright(c) 2018 Intel Corporation.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * BSD LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  - Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  - Neither the name of Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#include <linux/fault-inject.h>
+#include <linux/dcache.h>
+#include <linux/bitops.h>
+#include <linux/kernel.h>
+#include <rdma/rdma_vt.h>
+
+#include "hfi.h"
+
+struct hfi1_ibdev;
+
+#if defined(CONFIG_FAULT_INJECTION) && defined(CONFIG_FAULT_INJECTION_DEBUG_FS)
+struct fault {
+	struct fault_attr attr;
+	struct dentry *dir;
+	u64 n_rxfaults[(1U << BITS_PER_BYTE)];
+	u64 n_txfaults[(1U << BITS_PER_BYTE)];
+	u64 fault_skip;
+	u64 skip;
+	u64 fault_skip_usec;
+	unsigned long skip_usec;
+	unsigned long opcodes[(1U << BITS_PER_BYTE) / BITS_PER_LONG];
+	bool enable;
+	bool suppress_err;
+	bool opcode;
+	u8 direction;
+};
+
+int hfi1_fault_init_debugfs(struct hfi1_ibdev *ibd);
+bool hfi1_dbg_should_fault_tx(struct rvt_qp *qp, u32 opcode);
+bool hfi1_dbg_should_fault_rx(struct hfi1_packet *packet);
+bool hfi1_dbg_fault_suppress_err(struct hfi1_ibdev *ibd);
+void hfi1_fault_exit_debugfs(struct hfi1_ibdev *ibd);
+
+#else
+
+static inline int hfi1_fault_init_debugfs(struct hfi1_ibdev *ibd)
+{
+	return 0;
+}
+
+static inline bool hfi1_dbg_should_fault_rx(struct hfi1_packet *packet)
+{
+	return false;
+}
+
+static inline bool hfi1_dbg_should_fault_tx(struct rvt_qp *qp,
+					    u32 opcode)
+{
+	return false;
+}
+
+static inline bool hfi1_dbg_fault_suppress_err(struct hfi1_ibdev *ibd)
+{
+	return false;
+}
+
+static inline void hfi1_fault_exit_debugfs(struct hfi1_ibdev *ibd)
+{
+}
+#endif
+#endif /* _HFI1_FAULT_H */
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c
index da4aa1a95b11..c9d23c37a371 100644
--- a/drivers/infiniband/hw/hfi1/file_ops.c
+++ b/drivers/infiniband/hw/hfi1/file_ops.c
@@ -110,7 +110,7 @@ static int set_ctxt_pkey(struct hfi1_ctxtdata *uctxt, unsigned long arg);
 static int ctxt_reset(struct hfi1_ctxtdata *uctxt);
 static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt,
 		       unsigned long arg);
-static int vma_fault(struct vm_fault *vmf);
+static vm_fault_t vma_fault(struct vm_fault *vmf);
 static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,
 			    unsigned long arg);
 
@@ -591,7 +591,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
  * Local (non-chip) user memory is not mapped right away but as it is
  * accessed by the user-level code.
  */
-static int vma_fault(struct vm_fault *vmf)
+static vm_fault_t vma_fault(struct vm_fault *vmf)
 {
 	struct page *page;
 
@@ -689,8 +689,8 @@ static int hfi1_file_close(struct inode *inode, struct file *fp)
 	 * checks to default and disable the send context.
 	 */
 	if (uctxt->sc) {
-		set_pio_integrity(uctxt->sc);
 		sc_disable(uctxt->sc);
+		set_pio_integrity(uctxt->sc);
 	}
 
 	hfi1_free_ctxt_rcv_groups(uctxt);
diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index cac2c62bc42d..5eb3bf0849c7 100644
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -1,7 +1,7 @@
 #ifndef _HFI1_KERNEL_H
 #define _HFI1_KERNEL_H
 /*
- * Copyright(c) 2015-2017 Intel Corporation.
+ * Copyright(c) 2015-2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -231,13 +231,15 @@ struct hfi1_ctxtdata {
 	/* job key */
 	u16 jkey;
 	/* number of RcvArray groups for this context. */
-	u32 rcv_array_groups;
+	u16 rcv_array_groups;
 	/* index of first eager TID entry. */
-	u32 eager_base;
+	u16 eager_base;
 	/* number of expected TID entries */
-	u32 expected_count;
+	u16 expected_count;
 	/* index of first expected TID entry. */
-	u32 expected_base;
+	u16 expected_base;
+	/* array of tid_groups */
+	struct tid_group  *groups;
 
 	struct exp_tid_set tid_group_list;
 	struct exp_tid_set tid_used_list;
@@ -282,7 +284,7 @@ struct hfi1_ctxtdata {
 	/* interrupt handling */
 	u64 imask;	/* clear interrupt mask */
 	int ireg;	/* clear interrupt register */
-	unsigned numa_id; /* numa node of this context */
+	int numa_id; /* numa node of this context */
 	/* verbs rx_stats per rcd */
 	struct hfi1_opcode_stats_perctx *opstats;
 
@@ -333,6 +335,7 @@ struct hfi1_packet {
 	struct rvt_qp *qp;
 	struct ib_other_headers *ohdr;
 	struct ib_grh *grh;
+	struct opa_16b_mgmt *mgmt;
 	u64 rhf;
 	u32 maxcnt;
 	u32 rhqoff;
@@ -392,10 +395,17 @@ struct hfi1_packet {
  */
 #define OPA_16B_L4_9B		0x00
 #define OPA_16B_L2_TYPE		0x02
+#define OPA_16B_L4_FM		0x08
 #define OPA_16B_L4_IB_LOCAL	0x09
 #define OPA_16B_L4_IB_GLOBAL	0x0A
 #define OPA_16B_L4_ETHR		OPA_VNIC_L4_ETHR
 
+/*
+ * OPA 16B Management
+ */
+#define OPA_16B_L4_FM_PAD	3  /* fixed 3B pad */
+#define OPA_16B_L4_FM_HLEN	24 /* 16B(16) + L4_FM(8) */
+
 static inline u8 hfi1_16B_get_l4(struct hfi1_16b_header *hdr)
 {
 	return (u8)(hdr->lrh[2] & OPA_16B_L4_MASK);
@@ -472,6 +482,27 @@ static inline u8 hfi1_16B_bth_get_pad(struct ib_other_headers *ohdr)
 		   OPA_16B_BTH_PAD_MASK);
 }
 
+/*
+ * 16B Management
+ */
+#define OPA_16B_MGMT_QPN_MASK	0xFFFFFF
+static inline u32 hfi1_16B_get_dest_qpn(struct opa_16b_mgmt *mgmt)
+{
+	return be32_to_cpu(mgmt->dest_qpn) & OPA_16B_MGMT_QPN_MASK;
+}
+
+static inline u32 hfi1_16B_get_src_qpn(struct opa_16b_mgmt *mgmt)
+{
+	return be32_to_cpu(mgmt->src_qpn) & OPA_16B_MGMT_QPN_MASK;
+}
+
+static inline void hfi1_16B_set_qpn(struct opa_16b_mgmt *mgmt,
+				    u32 dest_qp, u32 src_qp)
+{
+	mgmt->dest_qpn = cpu_to_be32(dest_qp & OPA_16B_MGMT_QPN_MASK);
+	mgmt->src_qpn = cpu_to_be32(src_qp & OPA_16B_MGMT_QPN_MASK);
+}
+
 struct rvt_sge_state;
 
 /*
@@ -880,9 +911,9 @@ typedef void (*hfi1_make_req)(struct rvt_qp *qp,
 #define RHF_RCV_REPROCESS 2	/* stop. retain this packet */
 
 struct rcv_array_data {
-	u8 group_size;
 	u16 ngroups;
 	u16 nctxt_extra;
+	u8 group_size;
 };
 
 struct per_vl_data {
@@ -1263,6 +1294,9 @@ struct hfi1_devdata {
 
 	/* Save the enabled LCB error bits */
 	u64 lcb_err_en;
+	struct cpu_mask_set *comp_vect;
+	int *comp_vect_mappings;
+	u32 comp_vect_possible_cpus;
 
 	/*
 	 * Capability to have different send engines simply by changing a
@@ -1856,6 +1890,7 @@ struct cc_state *get_cc_state_protected(struct hfi1_pportdata *ppd)
 #define HFI1_HAS_SDMA_TIMEOUT  0x8
 #define HFI1_HAS_SEND_DMA      0x10   /* Supports Send DMA */
 #define HFI1_FORCED_FREEZE     0x80   /* driver forced freeze mode */
+#define HFI1_SHUTDOWN          0x100  /* device is shutting down */
 
 /* IB dword length mask in PBC (lower 11 bits); same for all chips */
 #define HFI1_PBC_LENGTH_MASK                     ((1 << 11) - 1)
@@ -2048,7 +2083,9 @@ static inline u64 hfi1_pkt_default_send_ctxt_mask(struct hfi1_devdata *dd,
 	| SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_LONG_BYPASS_PACKETS_SMASK
 	| SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_LONG_IB_PACKETS_SMASK
 	| SEND_CTXT_CHECK_ENABLE_DISALLOW_BAD_PKT_LEN_SMASK
+#ifndef CONFIG_FAULT_INJECTION
 	| SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_TEST_SMASK
+#endif
 	| SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_SMALL_BYPASS_PACKETS_SMASK
 	| SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_SMALL_IB_PACKETS_SMASK
 	| SEND_CTXT_CHECK_ENABLE_DISALLOW_RAW_IPV6_SMASK
@@ -2061,7 +2098,11 @@ static inline u64 hfi1_pkt_default_send_ctxt_mask(struct hfi1_devdata *dd,
 	| SEND_CTXT_CHECK_ENABLE_CHECK_ENABLE_SMASK;
 
 	if (ctxt_type == SC_USER)
-		base_sc_integrity |= HFI1_PKT_USER_SC_INTEGRITY;
+		base_sc_integrity |=
+#ifndef CONFIG_FAULT_INJECTION
+			SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_TEST_SMASK |
+#endif
+			HFI1_PKT_USER_SC_INTEGRITY;
 	else
 		base_sc_integrity |= HFI1_PKT_KERNEL_SC_INTEGRITY;
 
diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c
index 6309edf811df..3feecf926322 100644
--- a/drivers/infiniband/hw/hfi1/init.c
+++ b/drivers/infiniband/hw/hfi1/init.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015-2017 Intel Corporation.
+ * Copyright(c) 2015 - 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -361,9 +361,7 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
 		}
 
 		INIT_LIST_HEAD(&rcd->qp_wait_list);
-		hfi1_exp_tid_group_init(&rcd->tid_group_list);
-		hfi1_exp_tid_group_init(&rcd->tid_used_list);
-		hfi1_exp_tid_group_init(&rcd->tid_full_list);
+		hfi1_exp_tid_group_init(rcd);
 		rcd->ppd = ppd;
 		rcd->dd = dd;
 		__set_bit(0, rcd->in_use_ctxts);
@@ -1058,6 +1056,10 @@ static void shutdown_device(struct hfi1_devdata *dd)
 	unsigned pidx;
 	int i;
 
+	if (dd->flags & HFI1_SHUTDOWN)
+		return;
+	dd->flags |= HFI1_SHUTDOWN;
+
 	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 		ppd = dd->pport + pidx;
 
@@ -1240,6 +1242,8 @@ static void hfi1_clean_devdata(struct hfi1_devdata *dd)
 	dd->rcv_limit     = NULL;
 	dd->send_schedule = NULL;
 	dd->tx_opstats    = NULL;
+	kfree(dd->comp_vect);
+	dd->comp_vect = NULL;
 	sdma_clean(dd, dd->num_sdma);
 	rvt_dealloc_device(&dd->verbs_dev.rdi);
 }
@@ -1296,6 +1300,7 @@ struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, size_t extra)
 		dd->unit = ret;
 		list_add(&dd->list, &hfi1_dev_list);
 	}
+	dd->node = -1;
 
 	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
 	idr_preload_end();
@@ -1348,6 +1353,12 @@ struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, size_t extra)
 		goto bail;
 	}
 
+	dd->comp_vect = kzalloc(sizeof(*dd->comp_vect), GFP_KERNEL);
+	if (!dd->comp_vect) {
+		ret = -ENOMEM;
+		goto bail;
+	}
+
 	kobject_init(&dd->kobj, &hfi1_devdata_type);
 	return dd;
 
@@ -1391,6 +1402,7 @@ void hfi1_disable_after_error(struct hfi1_devdata *dd)
 
 static void remove_one(struct pci_dev *);
 static int init_one(struct pci_dev *, const struct pci_device_id *);
+static void shutdown_one(struct pci_dev *);
 
 #define DRIVER_LOAD_MSG "Intel " DRIVER_NAME " loaded: "
 #define PFX DRIVER_NAME ": "
@@ -1407,6 +1419,7 @@ static struct pci_driver hfi1_pci_driver = {
 	.name = DRIVER_NAME,
 	.probe = init_one,
 	.remove = remove_one,
+	.shutdown = shutdown_one,
 	.id_table = hfi1_pci_tbl,
 	.err_handler = &hfi1_pci_err_handler,
 };
@@ -1515,7 +1528,7 @@ module_init(hfi1_mod_init);
 static void __exit hfi1_mod_cleanup(void)
 {
 	pci_unregister_driver(&hfi1_pci_driver);
-	node_affinity_destroy();
+	node_affinity_destroy_all();
 	hfi1_wss_exit();
 	hfi1_dbg_exit();
 
@@ -1599,6 +1612,8 @@ static void cleanup_device_data(struct hfi1_devdata *dd)
 static void postinit_cleanup(struct hfi1_devdata *dd)
 {
 	hfi1_start_cleanup(dd);
+	hfi1_comp_vectors_clean_up(dd);
+	hfi1_dev_affinity_clean_up(dd);
 
 	hfi1_pcie_ddcleanup(dd);
 	hfi1_pcie_cleanup(dd->pcidev);
@@ -1816,6 +1831,13 @@ static void remove_one(struct pci_dev *pdev)
 	postinit_cleanup(dd);
 }
 
+static void shutdown_one(struct pci_dev *pdev)
+{
+	struct hfi1_devdata *dd = pci_get_drvdata(pdev);
+
+	shutdown_device(dd);
+}
+
 /**
  * hfi1_create_rcvhdrq - create a receive header queue
  * @dd: the hfi1_ib device
diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c
index e9962c65c68f..0307405491e0 100644
--- a/drivers/infiniband/hw/hfi1/mad.c
+++ b/drivers/infiniband/hw/hfi1/mad.c
@@ -1238,7 +1238,7 @@ static int port_states_transition_allowed(struct hfi1_pportdata *ppd,
 }
 
 static int set_port_states(struct hfi1_pportdata *ppd, struct opa_smp *smp,
-			   u32 logical_state, u32 phys_state)
+			   u32 logical_state, u32 phys_state, int local_mad)
 {
 	struct hfi1_devdata *dd = ppd->dd;
 	u32 link_state;
@@ -1314,7 +1314,7 @@ static int set_port_states(struct hfi1_pportdata *ppd, struct opa_smp *smp,
 		 * Don't send a reply if the response would be sent
 		 * through the disabled port.
 		 */
-		if (link_state == HLS_DN_DISABLE && smp->hop_cnt)
+		if (link_state == HLS_DN_DISABLE && !local_mad)
 			return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
 		break;
 	case IB_PORT_ARMED:
@@ -1350,7 +1350,7 @@ static int set_port_states(struct hfi1_pportdata *ppd, struct opa_smp *smp,
  */
 static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len, u32 max_len)
+				   u32 *resp_len, u32 max_len, int local_mad)
 {
 	struct opa_port_info *pi = (struct opa_port_info *)data;
 	struct ib_event event;
@@ -1634,7 +1634,7 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 	 */
 
 	if (!invalid) {
-		ret = set_port_states(ppd, smp, ls_new, ps_new);
+		ret = set_port_states(ppd, smp, ls_new, ps_new, local_mad);
 		if (ret)
 			return ret;
 	}
@@ -2085,7 +2085,7 @@ static int __subn_get_opa_psi(struct opa_smp *smp, u32 am, u8 *data,
 
 static int __subn_set_opa_psi(struct opa_smp *smp, u32 am, u8 *data,
 			      struct ib_device *ibdev, u8 port,
-			      u32 *resp_len, u32 max_len)
+			      u32 *resp_len, u32 max_len, int local_mad)
 {
 	u32 nports = OPA_AM_NPORT(am);
 	u32 start_of_sm_config = OPA_AM_START_SM_CFG(am);
@@ -2122,7 +2122,7 @@ static int __subn_set_opa_psi(struct opa_smp *smp, u32 am, u8 *data,
 	}
 
 	if (!invalid) {
-		ret = set_port_states(ppd, smp, ls_new, ps_new);
+		ret = set_port_states(ppd, smp, ls_new, ps_new, local_mad);
 		if (ret)
 			return ret;
 	}
@@ -3424,6 +3424,7 @@ static int pma_get_opa_errorinfo(struct opa_pma_mad *pmp,
 		pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)pmp);
 	}
+	rsp->port_number = port;
 
 	/* PortRcvErrorInfo */
 	rsp->port_rcv_ei.status_and_code =
@@ -4190,7 +4191,7 @@ static int subn_get_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,
 
 static int subn_set_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,
 			    u8 *data, struct ib_device *ibdev, u8 port,
-			    u32 *resp_len, u32 max_len)
+			    u32 *resp_len, u32 max_len, int local_mad)
 {
 	int ret;
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
@@ -4198,7 +4199,7 @@ static int subn_set_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,
 	switch (attr_id) {
 	case IB_SMP_ATTR_PORT_INFO:
 		ret = __subn_set_opa_portinfo(smp, am, data, ibdev, port,
-					      resp_len, max_len);
+					      resp_len, max_len, local_mad);
 		break;
 	case IB_SMP_ATTR_PKEY_TABLE:
 		ret = __subn_set_opa_pkeytable(smp, am, data, ibdev, port,
@@ -4222,7 +4223,7 @@ static int subn_set_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,
 		break;
 	case OPA_ATTRIB_ID_PORT_STATE_INFO:
 		ret = __subn_set_opa_psi(smp, am, data, ibdev, port,
-					 resp_len, max_len);
+					 resp_len, max_len, local_mad);
 		break;
 	case OPA_ATTRIB_ID_BUFFER_CONTROL_TABLE:
 		ret = __subn_set_opa_bct(smp, am, data, ibdev, port,
@@ -4314,7 +4315,7 @@ static int subn_get_opa_aggregate(struct opa_smp *smp,
 
 static int subn_set_opa_aggregate(struct opa_smp *smp,
 				  struct ib_device *ibdev, u8 port,
-				  u32 *resp_len)
+				  u32 *resp_len, int local_mad)
 {
 	int i;
 	u32 num_attr = be32_to_cpu(smp->attr_mod) & 0x000000ff;
@@ -4344,7 +4345,9 @@ static int subn_set_opa_aggregate(struct opa_smp *smp,
 		}
 
 		(void)subn_set_opa_sma(agg->attr_id, smp, am, agg->data,
-				       ibdev, port, NULL, (u32)agg_data_len);
+				       ibdev, port, NULL, (u32)agg_data_len,
+				       local_mad);
+
 		if (smp->status & IB_SMP_INVALID_FIELD)
 			break;
 		if (smp->status & ~IB_SMP_DIRECTION) {
@@ -4519,7 +4522,7 @@ static int hfi1_pkey_validation_pma(struct hfi1_ibport *ibp,
 static int process_subn_opa(struct ib_device *ibdev, int mad_flags,
 			    u8 port, const struct opa_mad *in_mad,
 			    struct opa_mad *out_mad,
-			    u32 *resp_len)
+			    u32 *resp_len, int local_mad)
 {
 	struct opa_smp *smp = (struct opa_smp *)out_mad;
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
@@ -4588,11 +4591,11 @@ static int process_subn_opa(struct ib_device *ibdev, int mad_flags,
 		default:
 			ret = subn_set_opa_sma(attr_id, smp, am, data,
 					       ibdev, port, resp_len,
-					       data_size);
+					       data_size, local_mad);
 			break;
 		case OPA_ATTRIB_ID_AGGREGATE:
 			ret = subn_set_opa_aggregate(smp, ibdev, port,
-						     resp_len);
+						     resp_len, local_mad);
 			break;
 		}
 		break;
@@ -4832,6 +4835,7 @@ static int hfi1_process_opa_mad(struct ib_device *ibdev, int mad_flags,
 {
 	int ret;
 	int pkey_idx;
+	int local_mad = 0;
 	u32 resp_len = 0;
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
 
@@ -4846,13 +4850,14 @@ static int hfi1_process_opa_mad(struct ib_device *ibdev, int mad_flags,
 	switch (in_mad->mad_hdr.mgmt_class) {
 	case IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE:
 	case IB_MGMT_CLASS_SUBN_LID_ROUTED:
-		if (is_local_mad(ibp, in_mad, in_wc)) {
+		local_mad = is_local_mad(ibp, in_mad, in_wc);
+		if (local_mad) {
 			ret = opa_local_smp_check(ibp, in_wc);
 			if (ret)
 				return IB_MAD_RESULT_FAILURE;
 		}
 		ret = process_subn_opa(ibdev, mad_flags, port, in_mad,
-				       out_mad, &resp_len);
+				       out_mad, &resp_len, local_mad);
 		goto bail;
 	case IB_MGMT_CLASS_PERF_MGMT:
 		ret = hfi1_pkey_validation_pma(ibp, in_mad, in_wc);
diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c
index c1c982908b4b..4d4371bf2c7c 100644
--- a/drivers/infiniband/hw/hfi1/pcie.c
+++ b/drivers/infiniband/hw/hfi1/pcie.c
@@ -56,11 +56,6 @@
 #include "chip_registers.h"
 #include "aspm.h"
 
-/* link speed vector for Gen3 speed - not in Linux headers */
-#define GEN1_SPEED_VECTOR 0x1
-#define GEN2_SPEED_VECTOR 0x2
-#define GEN3_SPEED_VECTOR 0x3
-
 /*
  * This file contains PCIe utility routines.
  */
@@ -183,6 +178,14 @@ int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev)
 		return -ENOMEM;
 	}
 	dd_dev_info(dd, "UC base1: %p for %x\n", dd->kregbase1, RCV_ARRAY);
+
+	/* verify that reads actually work, save revision for reset check */
+	dd->revision = readq(dd->kregbase1 + CCE_REVISION);
+	if (dd->revision == ~(u64)0) {
+		dd_dev_err(dd, "Cannot read chip CSRs\n");
+		goto nomem;
+	}
+
 	dd->chip_rcv_array_count = readq(dd->kregbase1 + RCV_ARRAY_CNT);
 	dd_dev_info(dd, "RcvArray count: %u\n", dd->chip_rcv_array_count);
 	dd->base2_start  = RCV_ARRAY + dd->chip_rcv_array_count * 8;
@@ -262,7 +265,7 @@ static u32 extract_speed(u16 linkstat)
 	case PCI_EXP_LNKSTA_CLS_5_0GB:
 		speed = 5000; /* Gen 2, 5GHz */
 		break;
-	case GEN3_SPEED_VECTOR:
+	case PCI_EXP_LNKSTA_CLS_8_0GB:
 		speed = 8000; /* Gen 3, 8GHz */
 		break;
 	}
@@ -317,7 +320,7 @@ int pcie_speeds(struct hfi1_devdata *dd)
 		return ret;
 	}
 
-	if ((linkcap & PCI_EXP_LNKCAP_SLS) != GEN3_SPEED_VECTOR) {
+	if ((linkcap & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_8_0GB) {
 		dd_dev_info(dd,
 			    "This HFI is not Gen3 capable, max speed 0x%x, need 0x3\n",
 			    linkcap & PCI_EXP_LNKCAP_SLS);
@@ -694,9 +697,6 @@ const struct pci_error_handlers hfi1_pci_err_handler = {
 /* gasket block secondary bus reset delay */
 #define SBR_DELAY_US 200000	/* 200ms */
 
-/* mask for PCIe capability register lnkctl2 target link speed */
-#define LNKCTL2_TARGET_LINK_SPEED_MASK 0xf
-
 static uint pcie_target = 3;
 module_param(pcie_target, uint, S_IRUGO);
 MODULE_PARM_DESC(pcie_target, "PCIe target speed (0 skip, 1-3 Gen1-3)");
@@ -1045,13 +1045,13 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd)
 		return 0;
 
 	if (pcie_target == 1) {			/* target Gen1 */
-		target_vector = GEN1_SPEED_VECTOR;
+		target_vector = PCI_EXP_LNKCTL2_TLS_2_5GT;
 		target_speed = 2500;
 	} else if (pcie_target == 2) {		/* target Gen2 */
-		target_vector = GEN2_SPEED_VECTOR;
+		target_vector = PCI_EXP_LNKCTL2_TLS_5_0GT;
 		target_speed = 5000;
 	} else if (pcie_target == 3) {		/* target Gen3 */
-		target_vector = GEN3_SPEED_VECTOR;
+		target_vector = PCI_EXP_LNKCTL2_TLS_8_0GT;
 		target_speed = 8000;
 	} else {
 		/* off or invalid target - skip */
@@ -1290,8 +1290,8 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd)
 	dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__,
 		    (u32)lnkctl2);
 	/* only write to parent if target is not as high as ours */
-	if ((lnkctl2 & LNKCTL2_TARGET_LINK_SPEED_MASK) < target_vector) {
-		lnkctl2 &= ~LNKCTL2_TARGET_LINK_SPEED_MASK;
+	if ((lnkctl2 & PCI_EXP_LNKCTL2_TLS) < target_vector) {
+		lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS;
 		lnkctl2 |= target_vector;
 		dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__,
 			    (u32)lnkctl2);
@@ -1316,7 +1316,7 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd)
 
 	dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__,
 		    (u32)lnkctl2);
-	lnkctl2 &= ~LNKCTL2_TARGET_LINK_SPEED_MASK;
+	lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS;
 	lnkctl2 |= target_vector;
 	dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__,
 		    (u32)lnkctl2);
diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c
index 40dac4d16eb8..9cac15d10c4f 100644
--- a/drivers/infiniband/hw/hfi1/pio.c
+++ b/drivers/infiniband/hw/hfi1/pio.c
@@ -50,8 +50,6 @@
 #include "qp.h"
 #include "trace.h"
 
-#define SC_CTXT_PACKET_EGRESS_TIMEOUT 350 /* in chip cycles */
-
 #define SC(name) SEND_CTXT_##name
 /*
  * Send Context functions
@@ -961,15 +959,40 @@ void sc_disable(struct send_context *sc)
 }
 
 /* return SendEgressCtxtStatus.PacketOccupancy */
-#define packet_occupancy(r) \
-	(((r) & SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_PACKET_OCCUPANCY_SMASK)\
-	>> SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_PACKET_OCCUPANCY_SHIFT)
+static u64 packet_occupancy(u64 reg)
+{
+	return (reg &
+		SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_PACKET_OCCUPANCY_SMASK)
+		>> SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_PACKET_OCCUPANCY_SHIFT;
+}
 
 /* is egress halted on the context? */
-#define egress_halted(r) \
-	((r) & SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_HALT_STATUS_SMASK)
+static bool egress_halted(u64 reg)
+{
+	return !!(reg & SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_HALT_STATUS_SMASK);
+}
 
-/* wait for packet egress, optionally pause for credit return  */
+/* is the send context halted? */
+static bool is_sc_halted(struct hfi1_devdata *dd, u32 hw_context)
+{
+	return !!(read_kctxt_csr(dd, hw_context, SC(STATUS)) &
+		  SC(STATUS_CTXT_HALTED_SMASK));
+}
+
+/**
+ * sc_wait_for_packet_egress
+ * @sc: valid send context
+ * @pause: wait for credit return
+ *
+ * Wait for packet egress, optionally pause for credit return
+ *
+ * Egress halt and Context halt are not necessarily the same thing, so
+ * check for both.
+ *
+ * NOTE: The context halt bit may not be set immediately.  Because of this,
+ * it is necessary to check the SW SFC_HALTED bit (set in the IRQ) and the HW
+ * context bit to determine if the context is halted.
+ */
 static void sc_wait_for_packet_egress(struct send_context *sc, int pause)
 {
 	struct hfi1_devdata *dd = sc->dd;
@@ -981,8 +1004,9 @@ static void sc_wait_for_packet_egress(struct send_context *sc, int pause)
 		reg_prev = reg;
 		reg = read_csr(dd, sc->hw_context * 8 +
 			       SEND_EGRESS_CTXT_STATUS);
-		/* done if egress is stopped */
-		if (egress_halted(reg))
+		/* done if any halt bits, SW or HW are set */
+		if (sc->flags & SCF_HALTED ||
+		    is_sc_halted(dd, sc->hw_context) || egress_halted(reg))
 			break;
 		reg = packet_occupancy(reg);
 		if (reg == 0)
diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c
index da58046a02ea..1a1a47ac53c6 100644
--- a/drivers/infiniband/hw/hfi1/rc.c
+++ b/drivers/infiniband/hw/hfi1/rc.c
@@ -2012,7 +2012,7 @@ void process_becn(struct hfi1_pportdata *ppd, u8 sl, u32 rlid, u32 lqpn,
 		unsigned long nsec = 1024 * ccti_timer;
 
 		hrtimer_start(&cca_timer->hrtimer, ns_to_ktime(nsec),
-			      HRTIMER_MODE_REL);
+			      HRTIMER_MODE_REL_PINNED);
 	}
 
 	spin_unlock_irqrestore(&ppd->cca_timer_lock, flags);
@@ -2123,7 +2123,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
 	/* OK, process the packet. */
 	switch (opcode) {
 	case OP(SEND_FIRST):
-		ret = hfi1_rvt_get_rwqe(qp, 0);
+		ret = rvt_get_rwqe(qp, false);
 		if (ret < 0)
 			goto nack_op_err;
 		if (!ret)
@@ -2149,7 +2149,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
 
 	case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
 		/* consume RWQE */
-		ret = hfi1_rvt_get_rwqe(qp, 1);
+		ret = rvt_get_rwqe(qp, true);
 		if (ret < 0)
 			goto nack_op_err;
 		if (!ret)
@@ -2159,7 +2159,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
 	case OP(SEND_ONLY):
 	case OP(SEND_ONLY_WITH_IMMEDIATE):
 	case OP(SEND_ONLY_WITH_INVALIDATE):
-		ret = hfi1_rvt_get_rwqe(qp, 0);
+		ret = rvt_get_rwqe(qp, false);
 		if (ret < 0)
 			goto nack_op_err;
 		if (!ret)
@@ -2271,7 +2271,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
 			goto send_middle;
 		else if (opcode == OP(RDMA_WRITE_ONLY))
 			goto no_immediate_data;
-		ret = hfi1_rvt_get_rwqe(qp, 1);
+		ret = rvt_get_rwqe(qp, true);
 		if (ret < 0)
 			goto nack_op_err;
 		if (!ret) {
diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c
index c0071ca4147a..ef4c566e206f 100644
--- a/drivers/infiniband/hw/hfi1/ruc.c
+++ b/drivers/infiniband/hw/hfi1/ruc.c
@@ -53,156 +53,6 @@
 #include "verbs_txreq.h"
 #include "trace.h"
 
-/*
- * Validate a RWQE and fill in the SGE state.
- * Return 1 if OK.
- */
-static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe)
-{
-	int i, j, ret;
-	struct ib_wc wc;
-	struct rvt_lkey_table *rkt;
-	struct rvt_pd *pd;
-	struct rvt_sge_state *ss;
-
-	rkt = &to_idev(qp->ibqp.device)->rdi.lkey_table;
-	pd = ibpd_to_rvtpd(qp->ibqp.srq ? qp->ibqp.srq->pd : qp->ibqp.pd);
-	ss = &qp->r_sge;
-	ss->sg_list = qp->r_sg_list;
-	qp->r_len = 0;
-	for (i = j = 0; i < wqe->num_sge; i++) {
-		if (wqe->sg_list[i].length == 0)
-			continue;
-		/* Check LKEY */
-		ret = rvt_lkey_ok(rkt, pd, j ? &ss->sg_list[j - 1] : &ss->sge,
-				  NULL, &wqe->sg_list[i],
-				  IB_ACCESS_LOCAL_WRITE);
-		if (unlikely(ret <= 0))
-			goto bad_lkey;
-		qp->r_len += wqe->sg_list[i].length;
-		j++;
-	}
-	ss->num_sge = j;
-	ss->total_len = qp->r_len;
-	ret = 1;
-	goto bail;
-
-bad_lkey:
-	while (j) {
-		struct rvt_sge *sge = --j ? &ss->sg_list[j - 1] : &ss->sge;
-
-		rvt_put_mr(sge->mr);
-	}
-	ss->num_sge = 0;
-	memset(&wc, 0, sizeof(wc));
-	wc.wr_id = wqe->wr_id;
-	wc.status = IB_WC_LOC_PROT_ERR;
-	wc.opcode = IB_WC_RECV;
-	wc.qp = &qp->ibqp;
-	/* Signal solicited completion event. */
-	rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1);
-	ret = 0;
-bail:
-	return ret;
-}
-
-/**
- * hfi1_rvt_get_rwqe - copy the next RWQE into the QP's RWQE
- * @qp: the QP
- * @wr_id_only: update qp->r_wr_id only, not qp->r_sge
- *
- * Return -1 if there is a local error, 0 if no RWQE is available,
- * otherwise return 1.
- *
- * Can be called from interrupt level.
- */
-int hfi1_rvt_get_rwqe(struct rvt_qp *qp, int wr_id_only)
-{
-	unsigned long flags;
-	struct rvt_rq *rq;
-	struct rvt_rwq *wq;
-	struct rvt_srq *srq;
-	struct rvt_rwqe *wqe;
-	void (*handler)(struct ib_event *, void *);
-	u32 tail;
-	int ret;
-
-	if (qp->ibqp.srq) {
-		srq = ibsrq_to_rvtsrq(qp->ibqp.srq);
-		handler = srq->ibsrq.event_handler;
-		rq = &srq->rq;
-	} else {
-		srq = NULL;
-		handler = NULL;
-		rq = &qp->r_rq;
-	}
-
-	spin_lock_irqsave(&rq->lock, flags);
-	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
-		ret = 0;
-		goto unlock;
-	}
-
-	wq = rq->wq;
-	tail = wq->tail;
-	/* Validate tail before using it since it is user writable. */
-	if (tail >= rq->size)
-		tail = 0;
-	if (unlikely(tail == wq->head)) {
-		ret = 0;
-		goto unlock;
-	}
-	/* Make sure entry is read after head index is read. */
-	smp_rmb();
-	wqe = rvt_get_rwqe_ptr(rq, tail);
-	/*
-	 * Even though we update the tail index in memory, the verbs
-	 * consumer is not supposed to post more entries until a
-	 * completion is generated.
-	 */
-	if (++tail >= rq->size)
-		tail = 0;
-	wq->tail = tail;
-	if (!wr_id_only && !init_sge(qp, wqe)) {
-		ret = -1;
-		goto unlock;
-	}
-	qp->r_wr_id = wqe->wr_id;
-
-	ret = 1;
-	set_bit(RVT_R_WRID_VALID, &qp->r_aflags);
-	if (handler) {
-		u32 n;
-
-		/*
-		 * Validate head pointer value and compute
-		 * the number of remaining WQEs.
-		 */
-		n = wq->head;
-		if (n >= rq->size)
-			n = 0;
-		if (n < tail)
-			n += rq->size - tail;
-		else
-			n -= tail;
-		if (n < srq->limit) {
-			struct ib_event ev;
-
-			srq->limit = 0;
-			spin_unlock_irqrestore(&rq->lock, flags);
-			ev.device = qp->ibqp.device;
-			ev.element.srq = qp->ibqp.srq;
-			ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
-			handler(&ev, srq->ibsrq.srq_context);
-			goto bail;
-		}
-	}
-unlock:
-	spin_unlock_irqrestore(&rq->lock, flags);
-bail:
-	return ret;
-}
-
 static int gid_ok(union ib_gid *gid, __be64 gid_prefix, __be64 id)
 {
 	return (gid->global.interface_id == id &&
@@ -423,7 +273,7 @@ static void ruc_loopback(struct rvt_qp *sqp)
 		/* FALLTHROUGH */
 	case IB_WR_SEND:
 send:
-		ret = hfi1_rvt_get_rwqe(qp, 0);
+		ret = rvt_get_rwqe(qp, false);
 		if (ret < 0)
 			goto op_err;
 		if (!ret)
@@ -435,7 +285,7 @@ static void ruc_loopback(struct rvt_qp *sqp)
 			goto inv_err;
 		wc.wc_flags = IB_WC_WITH_IMM;
 		wc.ex.imm_data = wqe->wr.ex.imm_data;
-		ret = hfi1_rvt_get_rwqe(qp, 1);
+		ret = rvt_get_rwqe(qp, true);
 		if (ret < 0)
 			goto op_err;
 		if (!ret)
diff --git a/drivers/infiniband/hw/hfi1/trace.c b/drivers/infiniband/hw/hfi1/trace.c
index 89bd9851065b..7c8aed0ffc07 100644
--- a/drivers/infiniband/hw/hfi1/trace.c
+++ b/drivers/infiniband/hw/hfi1/trace.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2017 Intel Corporation.
+ * Copyright(c) 2015 - 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -63,13 +63,20 @@ static u8 __get_ib_hdr_len(struct ib_header *hdr)
 
 static u8 __get_16b_hdr_len(struct hfi1_16b_header *hdr)
 {
-	struct ib_other_headers *ohdr;
+	struct ib_other_headers *ohdr = NULL;
 	u8 opcode;
+	u8 l4 = hfi1_16B_get_l4(hdr);
+
+	if (l4 == OPA_16B_L4_FM) {
+		opcode = IB_OPCODE_UD_SEND_ONLY;
+		return (8 + 8); /* No BTH */
+	}
 
-	if (hfi1_16B_get_l4(hdr) == OPA_16B_L4_IB_LOCAL)
+	if (l4 == OPA_16B_L4_IB_LOCAL)
 		ohdr = &hdr->u.oth;
 	else
 		ohdr = &hdr->u.l.oth;
+
 	opcode = ib_bth_get_opcode(ohdr);
 	return hdr_len_by_opcode[opcode] == 0 ?
 	       0 : hdr_len_by_opcode[opcode] - (12 + 8 + 8);
@@ -234,17 +241,24 @@ const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass,
 #define BTH_16B_PRN \
 	"op:0x%.2x,%s se:%d m:%d pad:%d tver:%d " \
 	"qpn:0x%.6x a:%d psn:0x%.8x"
-const char *hfi1_trace_fmt_bth(struct trace_seq *p, bool bypass,
-			       u8 ack, bool becn, bool fecn, u8 mig,
-			       u8 se, u8 pad, u8 opcode, const char *opname,
-			       u8 tver, u16 pkey, u32 psn, u32 qpn)
+#define L4_FM_16B_PRN \
+	"op:0x%.2x,%s dest_qpn:0x%.6x src_qpn:0x%.6x"
+const char *hfi1_trace_fmt_rest(struct trace_seq *p, bool bypass, u8 l4,
+				u8 ack, bool becn, bool fecn, u8 mig,
+				u8 se, u8 pad, u8 opcode, const char *opname,
+				u8 tver, u16 pkey, u32 psn, u32 qpn,
+				u32 dest_qpn, u32 src_qpn)
 {
 	const char *ret = trace_seq_buffer_ptr(p);
 
 	if (bypass)
-		trace_seq_printf(p, BTH_16B_PRN,
-				 opcode, opname,
-				 se, mig, pad, tver, qpn, ack, psn);
+		if (l4 == OPA_16B_L4_FM)
+			trace_seq_printf(p, L4_FM_16B_PRN,
+					 opcode, opname, dest_qpn, src_qpn);
+		else
+			trace_seq_printf(p, BTH_16B_PRN,
+					 opcode, opname,
+					 se, mig, pad, tver, qpn, ack, psn);
 
 	else
 		trace_seq_printf(p, BTH_9B_PRN,
@@ -258,12 +272,17 @@ const char *hfi1_trace_fmt_bth(struct trace_seq *p, bool bypass,
 
 const char *parse_everbs_hdrs(
 	struct trace_seq *p,
-	u8 opcode,
+	u8 opcode, u8 l4, u32 dest_qpn, u32 src_qpn,
 	void *ehdrs)
 {
 	union ib_ehdrs *eh = ehdrs;
 	const char *ret = trace_seq_buffer_ptr(p);
 
+	if (l4 == OPA_16B_L4_FM) {
+		trace_seq_printf(p, "mgmt pkt");
+		goto out;
+	}
+
 	switch (opcode) {
 	/* imm */
 	case OP(RC, SEND_LAST_WITH_IMMEDIATE):
@@ -334,6 +353,7 @@ const char *parse_everbs_hdrs(
 				 be32_to_cpu(eh->ieth));
 		break;
 	}
+out:
 	trace_seq_putc(p, 0);
 	return ret;
 }
@@ -374,6 +394,7 @@ const char *print_u32_array(
 	return ret;
 }
 
+__hfi1_trace_fn(AFFINITY);
 __hfi1_trace_fn(PKT);
 __hfi1_trace_fn(PROC);
 __hfi1_trace_fn(SDMA);
diff --git a/drivers/infiniband/hw/hfi1/trace_dbg.h b/drivers/infiniband/hw/hfi1/trace_dbg.h
index 0e7d929530c5..e62171fb7379 100644
--- a/drivers/infiniband/hw/hfi1/trace_dbg.h
+++ b/drivers/infiniband/hw/hfi1/trace_dbg.h
@@ -1,5 +1,5 @@
 /*
-* Copyright(c) 2015, 2016 Intel Corporation.
+* Copyright(c) 2015 - 2018 Intel Corporation.
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
@@ -113,6 +113,7 @@ void __hfi1_trace_##lvl(const char *func, char *fmt, ...)		\
  * hfi1_cdbg(LVL, fmt, ...); as well as take care of all
  * the debugfs stuff.
  */
+__hfi1_trace_def(AFFINITY);
 __hfi1_trace_def(PKT);
 __hfi1_trace_def(PROC);
 __hfi1_trace_def(SDMA);
diff --git a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
index 2847626d3819..1dc2c28fc96e 100644
--- a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
+++ b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
@@ -96,7 +96,9 @@ __print_symbolic(opcode,                                   \
 	ib_opcode_name(CNP))
 
 u8 ibhdr_exhdr_len(struct ib_header *hdr);
-const char *parse_everbs_hdrs(struct trace_seq *p, u8 opcode, void *ehdrs);
+const char *parse_everbs_hdrs(struct trace_seq *p, u8 opcode,
+			      u8 l4, u32 dest_qpn, u32 src_qpn,
+			      void *ehdrs);
 u8 hfi1_trace_opa_hdr_len(struct hfi1_opa_header *opah);
 u8 hfi1_trace_packet_hdr_len(struct hfi1_packet *packet);
 const char *hfi1_trace_get_packet_l4_str(u8 l4);
@@ -123,14 +125,16 @@ const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass,
 			       u8 rc, u8 sc, u8 sl, u16 entropy,
 			       u16 len, u16 pkey, u32 dlid, u32 slid);
 
-const char *hfi1_trace_fmt_bth(struct trace_seq *p, bool bypass,
-			       u8 ack, bool becn, bool fecn, u8 mig,
-			       u8 se, u8 pad, u8 opcode, const char *opname,
-			       u8 tver, u16 pkey, u32 psn, u32 qpn);
+const char *hfi1_trace_fmt_rest(struct trace_seq *p, bool bypass, u8 l4,
+				u8 ack, bool becn, bool fecn, u8 mig,
+				u8 se, u8 pad, u8 opcode, const char *opname,
+				u8 tver, u16 pkey, u32 psn, u32 qpn,
+				u32 dest_qpn, u32 src_qpn);
 
 const char *hfi1_trace_get_packet_l2_str(u8 l2);
 
-#define __parse_ib_ehdrs(op, ehdrs) parse_everbs_hdrs(p, op, ehdrs)
+#define __parse_ib_ehdrs(op, l4, dest_qpn, src_qpn, ehdrs) \
+			 parse_everbs_hdrs(p, op, l4, dest_qpn, src_qpn, ehdrs)
 
 #define lrh_name(lrh) { HFI1_##lrh, #lrh }
 #define show_lnh(lrh)                    \
@@ -169,6 +173,8 @@ DECLARE_EVENT_CLASS(hfi1_input_ibhdr_template,
 			__field(u32, psn)
 			__field(u32, qpn)
 			__field(u32, slid)
+			__field(u32, dest_qpn)
+			__field(u32, src_qpn)
 			/* extended headers */
 			__dynamic_array(u8, ehdrs,
 					hfi1_trace_packet_hdr_len(packet))
@@ -178,6 +184,8 @@ DECLARE_EVENT_CLASS(hfi1_input_ibhdr_template,
 
 			__entry->etype = packet->etype;
 			__entry->l2 = hfi1_16B_get_l2(packet->hdr);
+			__entry->dest_qpn = 0;
+			__entry->src_qpn = 0;
 			if (__entry->etype == RHF_RCV_TYPE_BYPASS) {
 				hfi1_trace_parse_16b_hdr(packet->hdr,
 							 &__entry->age,
@@ -192,16 +200,23 @@ DECLARE_EVENT_CLASS(hfi1_input_ibhdr_template,
 							 &__entry->dlid,
 							 &__entry->slid);
 
-				  hfi1_trace_parse_16b_bth(packet->ohdr,
-							   &__entry->ack,
-							   &__entry->mig,
-							   &__entry->opcode,
-							   &__entry->pad,
-							   &__entry->se,
-							   &__entry->tver,
-							   &__entry->psn,
-							   &__entry->qpn);
+				if (__entry->l4 == OPA_16B_L4_FM) {
+					__entry->opcode = IB_OPCODE_UD_SEND_ONLY;
+					__entry->dest_qpn = hfi1_16B_get_dest_qpn(packet->mgmt);
+					__entry->src_qpn = hfi1_16B_get_src_qpn(packet->mgmt);
+				}  else {
+					hfi1_trace_parse_16b_bth(packet->ohdr,
+								 &__entry->ack,
+								 &__entry->mig,
+								 &__entry->opcode,
+								 &__entry->pad,
+								 &__entry->se,
+								 &__entry->tver,
+								 &__entry->psn,
+								 &__entry->qpn);
+				}
 			} else {
+				__entry->l4 = OPA_16B_L4_9B;
 				hfi1_trace_parse_9b_hdr(packet->hdr, sc5,
 							&__entry->lnh,
 							&__entry->lver,
@@ -223,8 +238,9 @@ DECLARE_EVENT_CLASS(hfi1_input_ibhdr_template,
 							  &__entry->pkey,
 							  &__entry->psn,
 							  &__entry->qpn);
-				}
-				/* extended headers */
+			}
+			/* extended headers */
+			if (__entry->l4 != OPA_16B_L4_FM)
 				memcpy(__get_dynamic_array(ehdrs),
 				       &packet->ohdr->u,
 				       __get_dynamic_array_len(ehdrs));
@@ -253,25 +269,31 @@ DECLARE_EVENT_CLASS(hfi1_input_ibhdr_template,
 						 __entry->pkey,
 						 __entry->dlid,
 						 __entry->slid),
-			      hfi1_trace_fmt_bth(p,
-						 __entry->etype ==
+			      hfi1_trace_fmt_rest(p,
+						  __entry->etype ==
 							RHF_RCV_TYPE_BYPASS,
-						 __entry->ack,
-						 __entry->becn,
-						 __entry->fecn,
-						 __entry->mig,
-						 __entry->se,
-						 __entry->pad,
-						 __entry->opcode,
-						 show_ib_opcode(__entry->opcode),
-						 __entry->tver,
-						 __entry->pkey,
-						 __entry->psn,
-						 __entry->qpn),
+						  __entry->l4,
+						  __entry->ack,
+						  __entry->becn,
+						  __entry->fecn,
+						  __entry->mig,
+						  __entry->se,
+						  __entry->pad,
+						  __entry->opcode,
+						  show_ib_opcode(__entry->opcode),
+						  __entry->tver,
+						  __entry->pkey,
+						  __entry->psn,
+						  __entry->qpn,
+						  __entry->dest_qpn,
+						  __entry->src_qpn),
 			      /* extended headers */
 			      __get_dynamic_array_len(ehdrs),
 			      __parse_ib_ehdrs(
 					__entry->opcode,
+					__entry->l4,
+					__entry->dest_qpn,
+					__entry->src_qpn,
 					(void *)__get_dynamic_array(ehdrs))
 			     )
 );
@@ -310,6 +332,8 @@ DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template,
 			__field(u32, psn)
 			__field(u32, qpn)
 			__field(u32, slid)
+			__field(u32, dest_qpn)
+			__field(u32, src_qpn)
 			/* extended headers */
 			__dynamic_array(u8, ehdrs,
 					hfi1_trace_opa_hdr_len(opah))
@@ -320,6 +344,8 @@ DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template,
 			DD_DEV_ASSIGN(dd);
 
 			__entry->hdr_type = opah->hdr_type;
+			__entry->dest_qpn = 0;
+			__entry->src_qpn = 0;
 			if (__entry->hdr_type)  {
 				hfi1_trace_parse_16b_hdr(&opah->opah,
 							 &__entry->age,
@@ -334,19 +360,26 @@ DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template,
 							 &__entry->dlid,
 							 &__entry->slid);
 
-				if (__entry->l4 == OPA_16B_L4_IB_LOCAL)
-					ohdr = &opah->opah.u.oth;
-				else
-					ohdr = &opah->opah.u.l.oth;
-				hfi1_trace_parse_16b_bth(ohdr,
-							 &__entry->ack,
-							 &__entry->mig,
-							 &__entry->opcode,
-							 &__entry->pad,
-							 &__entry->se,
-							 &__entry->tver,
-							 &__entry->psn,
-							 &__entry->qpn);
+				if (__entry->l4 == OPA_16B_L4_FM) {
+					ohdr = NULL;
+					__entry->opcode = IB_OPCODE_UD_SEND_ONLY;
+					__entry->dest_qpn = hfi1_16B_get_dest_qpn(&opah->opah.u.mgmt);
+					__entry->src_qpn = hfi1_16B_get_src_qpn(&opah->opah.u.mgmt);
+				} else {
+					if (__entry->l4 == OPA_16B_L4_IB_LOCAL)
+						ohdr = &opah->opah.u.oth;
+					else
+						ohdr = &opah->opah.u.l.oth;
+					hfi1_trace_parse_16b_bth(ohdr,
+								 &__entry->ack,
+								 &__entry->mig,
+								 &__entry->opcode,
+								 &__entry->pad,
+								 &__entry->se,
+								 &__entry->tver,
+								 &__entry->psn,
+								 &__entry->qpn);
+				}
 			} else {
 				__entry->l4 = OPA_16B_L4_9B;
 				hfi1_trace_parse_9b_hdr(&opah->ibh, sc5,
@@ -376,8 +409,9 @@ DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template,
 			}
 
 			/* extended headers */
-			memcpy(__get_dynamic_array(ehdrs),
-			       &ohdr->u, __get_dynamic_array_len(ehdrs));
+			if (__entry->l4 != OPA_16B_L4_FM)
+				memcpy(__get_dynamic_array(ehdrs),
+				       &ohdr->u, __get_dynamic_array_len(ehdrs));
 		    ),
 		    TP_printk("[%s] (%s) %s %s hlen:%d %s",
 			      __get_str(dev),
@@ -399,24 +433,30 @@ DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template,
 						 __entry->pkey,
 						 __entry->dlid,
 						 __entry->slid),
-			      hfi1_trace_fmt_bth(p,
-						 !!__entry->hdr_type,
-						 __entry->ack,
-						 __entry->becn,
-						 __entry->fecn,
-						 __entry->mig,
-						 __entry->se,
-						 __entry->pad,
-						 __entry->opcode,
-						 show_ib_opcode(__entry->opcode),
-						 __entry->tver,
-						 __entry->pkey,
-						 __entry->psn,
-						 __entry->qpn),
+			      hfi1_trace_fmt_rest(p,
+						  !!__entry->hdr_type,
+						  __entry->l4,
+						  __entry->ack,
+						  __entry->becn,
+						  __entry->fecn,
+						  __entry->mig,
+						  __entry->se,
+						  __entry->pad,
+						  __entry->opcode,
+						  show_ib_opcode(__entry->opcode),
+						  __entry->tver,
+						  __entry->pkey,
+						  __entry->psn,
+						  __entry->qpn,
+						  __entry->dest_qpn,
+						  __entry->src_qpn),
 			      /* extended headers */
 			      __get_dynamic_array_len(ehdrs),
 			      __parse_ib_ehdrs(
 					__entry->opcode,
+					__entry->l4,
+					__entry->dest_qpn,
+					__entry->src_qpn,
 					(void *)__get_dynamic_array(ehdrs))
 			     )
 );
diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c
index 9d7a3110c14c..b7b671017e59 100644
--- a/drivers/infiniband/hw/hfi1/uc.c
+++ b/drivers/infiniband/hw/hfi1/uc.c
@@ -397,7 +397,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet)
 		if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) {
 			qp->r_sge = qp->s_rdma_read_sge;
 		} else {
-			ret = hfi1_rvt_get_rwqe(qp, 0);
+			ret = rvt_get_rwqe(qp, false);
 			if (ret < 0)
 				goto op_err;
 			if (!ret)
@@ -542,7 +542,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet)
 		if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) {
 			rvt_put_ss(&qp->s_rdma_read_sge);
 		} else {
-			ret = hfi1_rvt_get_rwqe(qp, 1);
+			ret = rvt_get_rwqe(qp, true);
 			if (ret < 0)
 				goto op_err;
 			if (!ret)
diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c
index 69c17a5ef038..1ab332f1866e 100644
--- a/drivers/infiniband/hw/hfi1/ud.c
+++ b/drivers/infiniband/hw/hfi1/ud.c
@@ -163,7 +163,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 	} else {
 		int ret;
 
-		ret = hfi1_rvt_get_rwqe(qp, 0);
+		ret = rvt_get_rwqe(qp, false);
 		if (ret < 0) {
 			rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
 			goto bail_unlock;
@@ -399,16 +399,30 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 	struct hfi1_pportdata *ppd;
 	struct hfi1_ibport *ibp;
 	u32 dlid, slid, nwords, extra_bytes;
+	u32 dest_qp = wqe->ud_wr.remote_qpn;
+	u32 src_qp = qp->ibqp.qp_num;
 	u16 len, pkey;
 	u8 l4, sc5;
+	bool is_mgmt = false;
 
 	ibp = to_iport(qp->ibqp.device, qp->port_num);
 	ppd = ppd_from_ibp(ibp);
 	ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr;
-	/* header size in dwords 16B LRH+BTH+DETH = (16+12+8)/4. */
-	ps->s_txreq->hdr_dwords = 9;
-	if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
-		ps->s_txreq->hdr_dwords++;
+
+	/*
+	 * Build 16B Management Packet if either the destination
+	 * or source queue pair number is 0 or 1.
+	 */
+	if (dest_qp == 0 || src_qp == 0 || dest_qp == 1 || src_qp == 1) {
+		/* header size in dwords 16B LRH+L4_FM = (16+8)/4. */
+		ps->s_txreq->hdr_dwords = 6;
+		is_mgmt = true;
+	} else {
+		/* header size in dwords 16B LRH+BTH+DETH = (16+12+8)/4. */
+		ps->s_txreq->hdr_dwords = 9;
+		if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
+			ps->s_txreq->hdr_dwords++;
+	}
 
 	/* SW provides space for CRC and LT for bypass packets. */
 	extra_bytes = hfi1_get_16b_padding((ps->s_txreq->hdr_dwords << 2),
@@ -453,7 +467,14 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 		slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) &
 			   ((1 << ppd->lmc) - 1));
 
-	hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, true);
+	if (is_mgmt) {
+		l4 = OPA_16B_L4_FM;
+		pkey = hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index);
+		hfi1_16B_set_qpn(&ps->s_txreq->phdr.hdr.opah.u.mgmt,
+				 dest_qp, src_qp);
+	} else {
+		hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, true);
+	}
 	/* Convert dwords to flits */
 	len = (ps->s_txreq->hdr_dwords + nwords) >> 1;
 
@@ -845,10 +866,8 @@ static int opa_smp_check(struct hfi1_ibport *ibp, u16 pkey, u8 sc5,
  */
 void hfi1_ud_rcv(struct hfi1_packet *packet)
 {
-	struct ib_other_headers *ohdr = packet->ohdr;
 	u32 hdrsize = packet->hlen;
 	struct ib_wc wc;
-	u32 qkey;
 	u32 src_qp;
 	u16 pkey;
 	int mgmt_pkey_idx = -1;
@@ -864,27 +883,35 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 	u32 dlid = packet->dlid;
 	u32 slid = packet->slid;
 	u8 extra_bytes;
+	u8 l4 = 0;
 	bool dlid_is_permissive;
 	bool slid_is_permissive;
+	bool solicited = false;
 
 	extra_bytes = packet->pad + packet->extra_byte + (SIZE_OF_CRC << 2);
-	qkey = ib_get_qkey(ohdr);
-	src_qp = ib_get_sqpn(ohdr);
 
 	if (packet->etype == RHF_RCV_TYPE_BYPASS) {
 		u32 permissive_lid =
 			opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B);
 
+		l4 = hfi1_16B_get_l4(packet->hdr);
 		pkey = hfi1_16B_get_pkey(packet->hdr);
 		dlid_is_permissive = (dlid == permissive_lid);
 		slid_is_permissive = (slid == permissive_lid);
 	} else {
-		pkey = ib_bth_get_pkey(ohdr);
+		pkey = ib_bth_get_pkey(packet->ohdr);
 		dlid_is_permissive = (dlid == be16_to_cpu(IB_LID_PERMISSIVE));
 		slid_is_permissive = (slid == be16_to_cpu(IB_LID_PERMISSIVE));
 	}
 	sl_from_sc = ibp->sc_to_sl[sc5];
 
+	if (likely(l4 != OPA_16B_L4_FM)) {
+		src_qp = ib_get_sqpn(packet->ohdr);
+		solicited = ib_bth_is_solicited(packet->ohdr);
+	} else {
+		src_qp = hfi1_16B_get_src_qpn(packet->mgmt);
+	}
+
 	process_ecn(qp, packet, (opcode != IB_OPCODE_CNP));
 	/*
 	 * Get the number of bytes the message was padded by
@@ -922,8 +949,9 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 			if (mgmt_pkey_idx < 0)
 				goto drop;
 		}
-		if (unlikely(qkey != qp->qkey)) /* Silent drop */
-			return;
+		if (unlikely(l4 != OPA_16B_L4_FM &&
+			     ib_get_qkey(packet->ohdr) != qp->qkey))
+			return; /* Silent drop */
 
 		/* Drop invalid MAD packets (see 13.5.3.1). */
 		if (unlikely(qp->ibqp.qp_num == 1 &&
@@ -950,7 +978,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 
 	if (qp->ibqp.qp_num > 1 &&
 	    opcode == IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE) {
-		wc.ex.imm_data = ohdr->u.ud.imm_data;
+		wc.ex.imm_data = packet->ohdr->u.ud.imm_data;
 		wc.wc_flags = IB_WC_WITH_IMM;
 		tlen -= sizeof(u32);
 	} else if (opcode == IB_OPCODE_UD_SEND_ONLY) {
@@ -974,7 +1002,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 	} else {
 		int ret;
 
-		ret = hfi1_rvt_get_rwqe(qp, 0);
+		ret = rvt_get_rwqe(qp, false);
 		if (ret < 0) {
 			rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
 			return;
@@ -1047,8 +1075,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 		dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1);
 	wc.port_num = qp->port_num;
 	/* Signal completion event if the solicited bit is set. */
-	rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
-		     ib_bth_is_solicited(ohdr));
+	rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, solicited);
 	return;
 
 drop:
diff --git a/drivers/infiniband/hw/hfi1/user_exp_rcv.c b/drivers/infiniband/hw/hfi1/user_exp_rcv.c
index 0d5330b7353d..6a4c5142515a 100644
--- a/drivers/infiniband/hw/hfi1/user_exp_rcv.c
+++ b/drivers/infiniband/hw/hfi1/user_exp_rcv.c
@@ -437,7 +437,6 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
 				hfi1_cdbg(TID,
 					  "Failed to program RcvArray entries %d",
 					  ret);
-				ret = -EFAULT;
 				goto unlock;
 			} else if (ret > 0) {
 				if (grp->used == grp->size)
diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index c8cf4d4984d3..08991874c0e2 100644
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2017 Intel Corporation.
+ * Copyright(c) 2015 - 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -63,6 +63,8 @@
 #include "verbs_txreq.h"
 #include "debugfs.h"
 #include "vnic.h"
+#include "fault.h"
+#include "affinity.h"
 
 static unsigned int hfi1_lkey_table_size = 16;
 module_param_named(lkey_table_size, hfi1_lkey_table_size, uint,
@@ -615,7 +617,12 @@ static inline void hfi1_handle_packet(struct hfi1_packet *packet,
 			wake_up(&mcast->wait);
 	} else {
 		/* Get the destination QP number. */
-		qp_num = ib_bth_get_qpn(packet->ohdr);
+		if (packet->etype == RHF_RCV_TYPE_BYPASS &&
+		    hfi1_16B_get_l4(packet->hdr) == OPA_16B_L4_FM)
+			qp_num = hfi1_16B_get_dest_qpn(packet->mgmt);
+		else
+			qp_num = ib_bth_get_qpn(packet->ohdr);
+
 		rcu_read_lock();
 		packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
 		if (!packet->qp)
@@ -624,10 +631,6 @@ static inline void hfi1_handle_packet(struct hfi1_packet *packet,
 		if (hfi1_do_pkey_check(packet))
 			goto unlock_drop;
 
-		if (unlikely(hfi1_dbg_fault_opcode(packet->qp, packet->opcode,
-						   true)))
-			goto unlock_drop;
-
 		spin_lock_irqsave(&packet->qp->r_lock, flags);
 		packet_handler = qp_ok(packet);
 		if (likely(packet_handler))
@@ -934,8 +937,7 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 			else
 				pbc |= (ib_is_sc5(sc5) << PBC_DC_INFO_SHIFT);
 
-			if (unlikely(hfi1_dbg_fault_opcode(qp, ps->opcode,
-							   false)))
+			if (unlikely(hfi1_dbg_should_fault_tx(qp, ps->opcode)))
 				pbc = hfi1_fault_tx(qp, ps->opcode, pbc);
 			pbc = create_pbc(ppd,
 					 pbc,
@@ -1088,7 +1090,8 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 			pbc |= PBC_PACKET_BYPASS | PBC_INSERT_BYPASS_ICRC;
 		else
 			pbc |= (ib_is_sc5(sc5) << PBC_DC_INFO_SHIFT);
-		if (unlikely(hfi1_dbg_fault_opcode(qp, ps->opcode, false)))
+
+		if (unlikely(hfi1_dbg_should_fault_tx(qp, ps->opcode)))
 			pbc = hfi1_fault_tx(qp, ps->opcode, pbc);
 		pbc = create_pbc(ppd, pbc, qp->srate_mbps, vl, plen);
 	}
@@ -1310,21 +1313,23 @@ int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 {
 	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
 	struct hfi1_qp_priv *priv = qp->priv;
-	struct ib_other_headers *ohdr;
+	struct ib_other_headers *ohdr = NULL;
 	send_routine sr;
 	int ret;
 	u16 pkey;
 	u32 slid;
+	u8 l4 = 0;
 
 	/* locate the pkey within the headers */
 	if (ps->s_txreq->phdr.hdr.hdr_type) {
 		struct hfi1_16b_header *hdr = &ps->s_txreq->phdr.hdr.opah;
-		u8 l4 = hfi1_16B_get_l4(hdr);
 
-		if (l4 == OPA_16B_L4_IB_GLOBAL)
-			ohdr = &hdr->u.l.oth;
-		else
+		l4 = hfi1_16B_get_l4(hdr);
+		if (l4 == OPA_16B_L4_IB_LOCAL)
 			ohdr = &hdr->u.oth;
+		else if (l4 == OPA_16B_L4_IB_GLOBAL)
+			ohdr = &hdr->u.l.oth;
+
 		slid = hfi1_16B_get_slid(hdr);
 		pkey = hfi1_16B_get_pkey(hdr);
 	} else {
@@ -1339,7 +1344,11 @@ int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		pkey = ib_bth_get_pkey(ohdr);
 	}
 
-	ps->opcode = ib_bth_get_opcode(ohdr);
+	if (likely(l4 != OPA_16B_L4_FM))
+		ps->opcode = ib_bth_get_opcode(ohdr);
+	else
+		ps->opcode = IB_OPCODE_UD_SEND_ONLY;
+
 	sr = get_send_routine(qp, ps);
 	ret = egress_pkey_check(dd->pport, slid, pkey,
 				priv->s_sc, qp->s_pkey_index);
@@ -1937,11 +1946,11 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
 	dd->verbs_dev.rdi.driver_f.modify_qp = hfi1_modify_qp;
 	dd->verbs_dev.rdi.driver_f.notify_restart_rc = hfi1_restart_rc;
 	dd->verbs_dev.rdi.driver_f.check_send_wqe = hfi1_check_send_wqe;
+	dd->verbs_dev.rdi.driver_f.comp_vect_cpu_lookup =
+						hfi1_comp_vect_mappings_lookup;
 
 	/* completeion queue */
-	snprintf(dd->verbs_dev.rdi.dparms.cq_name,
-		 sizeof(dd->verbs_dev.rdi.dparms.cq_name),
-		 "hfi1_cq%d", dd->unit);
+	dd->verbs_dev.rdi.ibdev.num_comp_vectors = dd->comp_vect_possible_cpus;
 	dd->verbs_dev.rdi.dparms.node = dd->node;
 
 	/* misc settings */
diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h
index 2d787b8346ca..a4d06502f06d 100644
--- a/drivers/infiniband/hw/hfi1/verbs.h
+++ b/drivers/infiniband/hw/hfi1/verbs.h
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2017 Intel Corporation.
+ * Copyright(c) 2015 - 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -110,6 +110,12 @@ enum {
 #define LRH_9B_BYTES (FIELD_SIZEOF(struct ib_header, lrh))
 #define LRH_9B_DWORDS (LRH_9B_BYTES / sizeof(u32))
 
+/* 24Bits for qpn, upper 8Bits reserved */
+struct opa_16b_mgmt {
+	__be32 dest_qpn;
+	__be32 src_qpn;
+};
+
 struct hfi1_16b_header {
 	u32 lrh[4];
 	union {
@@ -118,6 +124,7 @@ struct hfi1_16b_header {
 			struct ib_other_headers oth;
 		} l;
 		struct ib_other_headers oth;
+		struct opa_16b_mgmt mgmt;
 	} u;
 } __packed;
 
@@ -227,9 +234,7 @@ struct hfi1_ibdev {
 	/* per HFI symlinks to above */
 	struct dentry *hfi1_ibdev_link;
 #ifdef CONFIG_FAULT_INJECTION
-	struct fault_opcode *fault_opcode;
-	struct fault_packet *fault_packet;
-	bool fault_suppress_err;
+	struct fault *fault;
 #endif
 #endif
 };
@@ -330,8 +335,6 @@ void hfi1_ud_rcv(struct hfi1_packet *packet);
 
 int hfi1_lookup_pkey_idx(struct hfi1_ibport *ibp, u16 pkey);
 
-int hfi1_rvt_get_rwqe(struct rvt_qp *qp, int wr_id_only);
-
 void hfi1_migrate_qp(struct rvt_qp *qp);
 
 int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index fb305b7f99a8..412297d4b86c 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -100,6 +100,9 @@
 #define SERV_TYPE_UC				2
 #define SERV_TYPE_UD				3
 
+/* Configure to HW for PAGE_SIZE larger than 4KB */
+#define PG_SHIFT_OFFSET				(PAGE_SHIFT - 12)
+
 #define PAGES_SHIFT_8				8
 #define PAGES_SHIFT_16				16
 #define PAGES_SHIFT_24				24
@@ -211,6 +214,7 @@ enum {
 struct hns_roce_uar {
 	u64		pfn;
 	unsigned long	index;
+	unsigned long	logic_idx;
 };
 
 struct hns_roce_ucontext {
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 1f0965bb64ee..dd716ed60661 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -481,8 +481,8 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 			       V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
 		roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
 			       V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
-		roce_set_field(sq_db.parameter, V2_DB_PARAMETER_CONS_IDX_M,
-			       V2_DB_PARAMETER_CONS_IDX_S,
+		roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
+			       V2_DB_PARAMETER_IDX_S,
 			       qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
 		roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
 			       V2_DB_PARAMETER_SL_S, qp->sl);
@@ -1031,40 +1031,40 @@ static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
 
 	roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,
 		       CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,
-		       hr_dev->caps.qpc_ba_pg_sz);
+		       hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,
 		       CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,
-		       hr_dev->caps.qpc_buf_pg_sz);
+		       hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,
 		       CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,
 		       qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);
 
 	roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,
 		       CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,
-		       hr_dev->caps.srqc_ba_pg_sz);
+		       hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,
 		       CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,
-		       hr_dev->caps.srqc_buf_pg_sz);
+		       hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,
 		       CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,
 		       srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);
 
 	roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,
 		       CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,
-		       hr_dev->caps.cqc_ba_pg_sz);
+		       hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,
 		       CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,
-		       hr_dev->caps.cqc_buf_pg_sz);
+		       hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,
 		       CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,
 		       cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);
 
 	roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,
 		       CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,
-		       hr_dev->caps.mpt_ba_pg_sz);
+		       hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,
 		       CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,
-		       hr_dev->caps.mpt_buf_pg_sz);
+		       hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,
 		       CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
 		       mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
@@ -1359,7 +1359,8 @@ static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
 		       HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num);
 	roce_set_field(mpt_entry->byte_4_pd_hop_st,
 		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
-		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, mr->pbl_ba_pg_sz);
+		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
+		       mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
 		       V2_MPT_BYTE_4_PD_S, mr->pd);
 	mpt_entry->byte_4_pd_hop_st = cpu_to_le32(mpt_entry->byte_4_pd_hop_st);
@@ -1435,7 +1436,8 @@ static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
 
 	roce_set_field(mpt_entry->byte_64_buf_pa1,
 		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
-		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, mr->pbl_buf_pg_sz);
+		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
+		       mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
 	mpt_entry->byte_64_buf_pa1 = cpu_to_le32(mpt_entry->byte_64_buf_pa1);
 
 	return 0;
@@ -1616,11 +1618,11 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
 	roce_set_field(cq_context->byte_24_pgsz_addr,
 		       V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
 		       V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
-		       hr_dev->caps.cqe_ba_pg_sz);
+		       hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(cq_context->byte_24_pgsz_addr,
 		       V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
 		       V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
-		       hr_dev->caps.cqe_buf_pg_sz);
+		       hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET);
 
 	cq_context->cqe_ba = (u32)(dma_handle >> 3);
 
@@ -2719,7 +2721,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 	roce_set_field(context->byte_16_buf_ba_pg_sz,
 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
-		       hr_dev->caps.mtt_ba_pg_sz);
+		       hr_dev->caps.mtt_ba_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
@@ -2727,7 +2729,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 	roce_set_field(context->byte_16_buf_ba_pg_sz,
 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
-		       hr_dev->caps.mtt_buf_pg_sz);
+		       hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET);
 	roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
@@ -4161,12 +4163,14 @@ static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
 	/* set eqe_ba_pg_sz */
 	roce_set_field(eqc->byte_8,
 		       HNS_ROCE_EQC_BA_PG_SZ_M,
-		       HNS_ROCE_EQC_BA_PG_SZ_S, eq->eqe_ba_pg_sz);
+		       HNS_ROCE_EQC_BA_PG_SZ_S,
+		       eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET);
 
 	/* set eqe_buf_pg_sz */
 	roce_set_field(eqc->byte_8,
 		       HNS_ROCE_EQC_BUF_PG_SZ_M,
-		       HNS_ROCE_EQC_BUF_PG_SZ_S, eq->eqe_buf_pg_sz);
+		       HNS_ROCE_EQC_BUF_PG_SZ_S,
+		       eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET);
 
 	/* set eq_producer_idx */
 	roce_set_field(eqc->byte_8,
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
index 182b6726f783..2caeb4cdad5c 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
@@ -76,7 +76,7 @@
 #define HNS_ROCE_V2_PAGE_SIZE_SUPPORTED		0xFFFFF000
 #define HNS_ROCE_V2_MAX_INNER_MTPT_NUM		2
 #define HNS_ROCE_INVALID_LKEY			0x100
-#define HNS_ROCE_CMQ_TX_TIMEOUT			200
+#define HNS_ROCE_CMQ_TX_TIMEOUT			30000
 
 #define HNS_ROCE_CONTEXT_HOP_NUM		1
 #define HNS_ROCE_MTT_HOP_NUM			1
@@ -897,8 +897,8 @@ struct hns_roce_v2_mpt_entry {
 #define	V2_DB_BYTE_4_CMD_S 24
 #define V2_DB_BYTE_4_CMD_M GENMASK(27, 24)
 
-#define V2_DB_PARAMETER_CONS_IDX_S 0
-#define V2_DB_PARAMETER_CONS_IDX_M GENMASK(15, 0)
+#define V2_DB_PARAMETER_IDX_S 0
+#define V2_DB_PARAMETER_IDX_M GENMASK(15, 0)
 
 #define V2_DB_PARAMETER_SL_S 16
 #define V2_DB_PARAMETER_SL_M GENMASK(18, 16)
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 96fb6a9ed93c..c614f9182b1a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -99,7 +99,6 @@ static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(attr->device);
 	struct ib_gid_attr zattr = { };
-	union ib_gid zgid = { {0} };
 	u8 port = attr->port_num - 1;
 	unsigned long flags;
 	int ret;
diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c
index f7256d88d38f..d1fe0e7957e3 100644
--- a/drivers/infiniband/hw/hns/hns_roce_mr.c
+++ b/drivers/infiniband/hw/hns/hns_roce_mr.c
@@ -1007,12 +1007,6 @@ struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	}
 
 	n = ib_umem_page_count(mr->umem);
-	if (mr->umem->page_shift != HNS_ROCE_HEM_PAGE_SHIFT) {
-		dev_err(dev, "Just support 4K page size but is 0x%lx now!\n",
-			BIT(mr->umem->page_shift));
-		ret = -EINVAL;
-		goto err_umem;
-	}
 
 	if (!hr_dev->caps.pbl_hop_num) {
 		if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) {
diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c
index 4b41e041799c..b9f2c871ff9a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_pd.c
+++ b/drivers/infiniband/hw/hns/hns_roce_pd.c
@@ -107,13 +107,15 @@ int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)
 	int ret = 0;
 
 	/* Using bitmap to manager UAR index */
-	ret = hns_roce_bitmap_alloc(&hr_dev->uar_table.bitmap, &uar->index);
+	ret = hns_roce_bitmap_alloc(&hr_dev->uar_table.bitmap, &uar->logic_idx);
 	if (ret == -1)
 		return -ENOMEM;
 
-	if (uar->index > 0)
-		uar->index = (uar->index - 1) %
+	if (uar->logic_idx > 0 && hr_dev->caps.phy_num_uars > 1)
+		uar->index = (uar->logic_idx - 1) %
 			     (hr_dev->caps.phy_num_uars - 1) + 1;
+	else
+		uar->index = 0;
 
 	if (!dev_is_pci(hr_dev->dev)) {
 		res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0);
@@ -132,7 +134,7 @@ int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)
 
 void hns_roce_uar_free(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)
 {
-	hns_roce_bitmap_free(&hr_dev->uar_table.bitmap, uar->index,
+	hns_roce_bitmap_free(&hr_dev->uar_table.bitmap, uar->logic_idx,
 			     BITMAP_NO_RR);
 }
 
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c
index f7c6fd9ff6e2..7b2655128b9f 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c
@@ -1519,18 +1519,13 @@ static void i40iw_add_hte_node(struct i40iw_cm_core *cm_core,
 
 /**
  * i40iw_find_port - find port that matches reference port
- * @port: port number
+ * @hte: ptr to accelerated or non-accelerated list
  * @accelerated_list: flag for accelerated vs non-accelerated list
  */
-static bool i40iw_find_port(struct i40iw_cm_core *cm_core, u16 port,
-			    bool accelerated_list)
+static bool i40iw_find_port(struct list_head *hte, u16 port)
 {
-	struct list_head *hte;
 	struct i40iw_cm_node *cm_node;
 
-	hte = accelerated_list ?
-	      &cm_core->accelerated_list : &cm_core->non_accelerated_list;
-
 	list_for_each_entry(cm_node, hte, list) {
 		if (cm_node->loc_port == port)
 			return true;
@@ -1540,35 +1535,32 @@ static bool i40iw_find_port(struct i40iw_cm_core *cm_core, u16 port,
 
 /**
  * i40iw_port_in_use - determine if port is in use
+ * @cm_core: cm's core
  * @port: port number
- * @active_side: flag for listener side vs active side
  */
-static bool i40iw_port_in_use(struct i40iw_cm_core *cm_core, u16 port, bool active_side)
+bool i40iw_port_in_use(struct i40iw_cm_core *cm_core, u16 port)
 {
 	struct i40iw_cm_listener *listen_node;
 	unsigned long flags;
-	bool ret = false;
 
-	if (active_side) {
-		spin_lock_irqsave(&cm_core->ht_lock, flags);
-		ret = i40iw_find_port(cm_core, port, true);
-		if (!ret)
-			ret = i40iw_find_port(cm_core, port, false);
-		if (!ret)
-			clear_bit(port, cm_core->active_side_ports);
+	spin_lock_irqsave(&cm_core->ht_lock, flags);
+	if (i40iw_find_port(&cm_core->accelerated_list, port) ||
+	    i40iw_find_port(&cm_core->non_accelerated_list, port)) {
 		spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-	} else {
-		spin_lock_irqsave(&cm_core->listen_list_lock, flags);
-		list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
-			if (listen_node->loc_port == port) {
-				ret = true;
-				break;
-			}
+		return true;
+	}
+	spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+
+	spin_lock_irqsave(&cm_core->listen_list_lock, flags);
+	list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
+		if (listen_node->loc_port == port) {
+			spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
+			return true;
 		}
-		spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
 	}
+	spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
 
-	return ret;
+	return false;
 }
 
 /**
@@ -1788,7 +1780,7 @@ static enum i40iw_status_code i40iw_add_mqh_4(
 					    &ifa->ifa_address,
 					    rdma_vlan_dev_vlan_id(dev),
 					    dev->dev_addr);
-				child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_ATOMIC);
+				child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);
 				cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
 				i40iw_debug(&iwdev->sc_dev,
 					    I40IW_DEBUG_CM,
@@ -1917,7 +1909,7 @@ static int i40iw_dec_refcnt_listen(struct i40iw_cm_core *cm_core,
 		spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
 
 		if (listener->iwdev) {
-			if (apbvt_del && !i40iw_port_in_use(cm_core, listener->loc_port, false))
+			if (apbvt_del)
 				i40iw_manage_apbvt(listener->iwdev,
 						   listener->loc_port,
 						   I40IW_MANAGE_APBVT_DEL);
@@ -2298,7 +2290,7 @@ static void i40iw_rem_ref_cm_node(struct i40iw_cm_node *cm_node)
 	if (cm_node->listener) {
 		i40iw_dec_refcnt_listen(cm_core, cm_node->listener, 0, true);
 	} else {
-		if (!i40iw_port_in_use(cm_core, cm_node->loc_port, true) && cm_node->apbvt_set) {
+		if (cm_node->apbvt_set) {
 			i40iw_manage_apbvt(cm_node->iwdev,
 					   cm_node->loc_port,
 					   I40IW_MANAGE_APBVT_DEL);
@@ -2872,7 +2864,7 @@ static struct i40iw_cm_listener *i40iw_make_listen_node(
 
 	if (!listener) {
 		/* create a CM listen node (1/2 node to compare incoming traffic to) */
-		listener = kzalloc(sizeof(*listener), GFP_ATOMIC);
+		listener = kzalloc(sizeof(*listener), GFP_KERNEL);
 		if (!listener)
 			return NULL;
 		cm_core->stats_listen_nodes_created++;
@@ -3244,6 +3236,7 @@ void i40iw_setup_cm_core(struct i40iw_device *iwdev)
 
 	spin_lock_init(&cm_core->ht_lock);
 	spin_lock_init(&cm_core->listen_list_lock);
+	spin_lock_init(&cm_core->apbvt_lock);
 
 	cm_core->event_wq = alloc_ordered_workqueue("iwewq",
 						    WQ_MEM_RECLAIM);
@@ -3811,7 +3804,6 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	struct sockaddr_in6 *laddr6;
 	struct sockaddr_in6 *raddr6;
 	int ret = 0;
-	unsigned long flags;
 
 	ibqp = i40iw_get_qp(cm_id->device, conn_param->qpn);
 	if (!ibqp)
@@ -3882,15 +3874,10 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 		cm_node->qhash_set = true;
 	}
 
-	spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags);
-	if (!test_and_set_bit(cm_info.loc_port, iwdev->cm_core.active_side_ports)) {
-		spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);
-		if (i40iw_manage_apbvt(iwdev, cm_info.loc_port, I40IW_MANAGE_APBVT_ADD)) {
-			ret =  -EINVAL;
-			goto err;
-		}
-	} else {
-		spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);
+	if (i40iw_manage_apbvt(iwdev, cm_info.loc_port,
+			       I40IW_MANAGE_APBVT_ADD)) {
+		ret =  -EINVAL;
+		goto err;
 	}
 
 	cm_node->apbvt_set = true;
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.h b/drivers/infiniband/hw/i40iw/i40iw_cm.h
index 78ba36ae2bbe..66dc1ba03389 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_cm.h
@@ -413,8 +413,9 @@ struct i40iw_cm_core {
 
 	spinlock_t ht_lock; /* manage hash table */
 	spinlock_t listen_list_lock; /* listen list */
+	spinlock_t apbvt_lock; /*manage apbvt entries*/
 
-	unsigned long active_side_ports[BITS_TO_LONGS(MAX_PORTS)];
+	unsigned long ports_in_use[BITS_TO_LONGS(MAX_PORTS)];
 
 	u64	stats_nodes_created;
 	u64	stats_nodes_destroyed;
@@ -457,4 +458,5 @@ void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev,
 void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr,
 				   struct i40iw_cm_info *nfo,
 				   bool disconnect_all);
+bool i40iw_port_in_use(struct i40iw_cm_core *cm_core, u16 port);
 #endif /* I40IW_CM_H */
diff --git a/drivers/infiniband/hw/i40iw/i40iw_hw.c b/drivers/infiniband/hw/i40iw/i40iw_hw.c
index c9f62ca7643c..2836c5420d60 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_hw.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_hw.c
@@ -443,13 +443,37 @@ void i40iw_process_aeq(struct i40iw_device *iwdev)
 int i40iw_manage_apbvt(struct i40iw_device *iwdev, u16 accel_local_port, bool add_port)
 {
 	struct i40iw_apbvt_info *info;
-	enum i40iw_status_code status;
 	struct i40iw_cqp_request *cqp_request;
 	struct cqp_commands_info *cqp_info;
+	unsigned long flags;
+	struct i40iw_cm_core *cm_core = &iwdev->cm_core;
+	enum i40iw_status_code status = 0;
+	bool in_use;
+
+	/* apbvt_lock is held across CQP delete APBVT OP (non-waiting) to
+	 * protect against race where add APBVT CQP can race ahead of the delete
+	 * APBVT for same port.
+	 */
+	spin_lock_irqsave(&cm_core->apbvt_lock, flags);
+
+	if (!add_port) {
+		in_use = i40iw_port_in_use(cm_core, accel_local_port);
+		if (in_use)
+			goto exit;
+		clear_bit(accel_local_port, cm_core->ports_in_use);
+	} else {
+		in_use = test_and_set_bit(accel_local_port,
+					  cm_core->ports_in_use);
+		spin_unlock_irqrestore(&cm_core->apbvt_lock, flags);
+		if (in_use)
+			return 0;
+	}
 
 	cqp_request = i40iw_get_cqp_request(&iwdev->cqp, add_port);
-	if (!cqp_request)
-		return -ENOMEM;
+	if (!cqp_request) {
+		status = -ENOMEM;
+		goto exit;
+	}
 
 	cqp_info = &cqp_request->info;
 	info = &cqp_info->in.u.manage_apbvt_entry.info;
@@ -465,6 +489,10 @@ int i40iw_manage_apbvt(struct i40iw_device *iwdev, u16 accel_local_port, bool ad
 	status = i40iw_handle_cqp_op(iwdev, cqp_request);
 	if (status)
 		i40iw_pr_err("CQP-OP Manage APBVT entry fail");
+exit:
+	if (!add_port)
+		spin_unlock_irqrestore(&cm_core->apbvt_lock, flags);
+
 	return status;
 }
 
diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c
index 05001e6da1f8..68095f00d08f 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_main.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_main.c
@@ -1757,7 +1757,7 @@ static void i40iw_l2param_change(struct i40e_info *ldev, struct i40e_client *cli
 		return;
 
 
-	work = kzalloc(sizeof(*work), GFP_ATOMIC);
+	work = kzalloc(sizeof(*work), GFP_KERNEL);
 	if (!work)
 		return;
 
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 5b70744f414a..bf12394c13c1 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -276,7 +276,7 @@ static int mlx4_ib_add_gid(const union ib_gid *gid,
 			found = i;
 			break;
 		}
-		if (free < 0 && !memcmp(&port_gid_table->gids[i].gid, &zgid, sizeof(*gid)))
+		if (free < 0 && rdma_is_zero_gid(&port_gid_table->gids[i].gid))
 			free = i; /* HW has space */
 	}
 
@@ -345,7 +345,8 @@ static int mlx4_ib_del_gid(const struct ib_gid_attr *attr, void **context)
 		if (!ctx->refcount) {
 			unsigned int real_index = ctx->real_index;
 
-			memcpy(&port_gid_table->gids[real_index].gid, &zgid, sizeof(zgid));
+			memset(&port_gid_table->gids[real_index].gid, 0,
+			       sizeof(port_gid_table->gids[real_index].gid));
 			kfree(port_gid_table->gids[real_index].ctx);
 			port_gid_table->gids[real_index].ctx = NULL;
 			hw_update = 1;
diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index 61d8b06375bb..ed1f253faf97 100644
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -367,6 +367,40 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va,
 	return block_shift;
 }
 
+static struct ib_umem *mlx4_get_umem_mr(struct ib_ucontext *context, u64 start,
+					u64 length, u64 virt_addr,
+					int access_flags)
+{
+	/*
+	 * Force registering the memory as writable if the underlying pages
+	 * are writable.  This is so rereg can change the access permissions
+	 * from readable to writable without having to run through ib_umem_get
+	 * again
+	 */
+	if (!ib_access_writable(access_flags)) {
+		struct vm_area_struct *vma;
+
+		down_read(&current->mm->mmap_sem);
+		/*
+		 * FIXME: Ideally this would iterate over all the vmas that
+		 * cover the memory, but for now it requires a single vma to
+		 * entirely cover the MR to support RO mappings.
+		 */
+		vma = find_vma(current->mm, start);
+		if (vma && vma->vm_end >= start + length &&
+		    vma->vm_start <= start) {
+			if (vma->vm_flags & VM_WRITE)
+				access_flags |= IB_ACCESS_LOCAL_WRITE;
+		} else {
+			access_flags |= IB_ACCESS_LOCAL_WRITE;
+		}
+
+		up_read(&current->mm->mmap_sem);
+	}
+
+	return ib_umem_get(context, start, length, access_flags, 0);
+}
+
 struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 				  u64 virt_addr, int access_flags,
 				  struct ib_udata *udata)
@@ -381,10 +415,8 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	/* Force registering the memory as writable. */
-	/* Used for memory re-registeration. HCA protects the access */
-	mr->umem = ib_umem_get(pd->uobject->context, start, length,
-			       access_flags | IB_ACCESS_LOCAL_WRITE, 0);
+	mr->umem = mlx4_get_umem_mr(pd->uobject->context, start, length,
+				    virt_addr, access_flags);
 	if (IS_ERR(mr->umem)) {
 		err = PTR_ERR(mr->umem);
 		goto err_free;
@@ -454,6 +486,9 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,
 	}
 
 	if (flags & IB_MR_REREG_ACCESS) {
+		if (ib_access_writable(mr_access_flags) && !mmr->umem->writable)
+			return -EPERM;
+
 		err = mlx4_mr_hw_change_access(dev->dev, *pmpt_entry,
 					       convert_access(mr_access_flags));
 
@@ -467,10 +502,9 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,
 
 		mlx4_mr_rereg_mem_cleanup(dev->dev, &mmr->mmr);
 		ib_umem_release(mmr->umem);
-		mmr->umem = ib_umem_get(mr->uobject->context, start, length,
-					mr_access_flags |
-					IB_ACCESS_LOCAL_WRITE,
-					0);
+		mmr->umem =
+			mlx4_get_umem_mr(mr->uobject->context, start, length,
+					 virt_addr, mr_access_flags);
 		if (IS_ERR(mmr->umem)) {
 			err = PTR_ERR(mmr->umem);
 			/* Prevent mlx4_ib_dereg_mr from free'ing invalid pointer */
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 199648adac74..cd2c08c45334 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -3078,7 +3078,7 @@ static int fill_gid_by_hw_index(struct mlx4_ib_dev *ibdev, u8 port_num,
 	memcpy(gid, &port_gid_table->gids[index].gid, sizeof(*gid));
 	*gid_type = port_gid_table->gids[index].gid_type;
 	spin_unlock_irqrestore(&iboe->lock, flags);
-	if (!memcmp(gid, &zgid, sizeof(*gid)))
+	if (rdma_is_zero_gid(gid))
 		return -ENOENT;
 
 	return 0;
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 77d257ec899b..ad39d64b8108 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -637,7 +637,7 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq,
 }
 
 static int poll_soft_wc(struct mlx5_ib_cq *cq, int num_entries,
-			struct ib_wc *wc)
+			struct ib_wc *wc, bool is_fatal_err)
 {
 	struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
 	struct mlx5_ib_wc *soft_wc, *next;
@@ -650,6 +650,10 @@ static int poll_soft_wc(struct mlx5_ib_cq *cq, int num_entries,
 		mlx5_ib_dbg(dev, "polled software generated completion on CQ 0x%x\n",
 			    cq->mcq.cqn);
 
+		if (unlikely(is_fatal_err)) {
+			soft_wc->wc.status = IB_WC_WR_FLUSH_ERR;
+			soft_wc->wc.vendor_err = MLX5_CQE_SYNDROME_WR_FLUSH_ERR;
+		}
 		wc[npolled++] = soft_wc->wc;
 		list_del(&soft_wc->list);
 		kfree(soft_wc);
@@ -670,12 +674,17 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 
 	spin_lock_irqsave(&cq->lock, flags);
 	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
-		mlx5_ib_poll_sw_comp(cq, num_entries, wc, &npolled);
+		/* make sure no soft wqe's are waiting */
+		if (unlikely(!list_empty(&cq->wc_list)))
+			soft_polled = poll_soft_wc(cq, num_entries, wc, true);
+
+		mlx5_ib_poll_sw_comp(cq, num_entries - soft_polled,
+				     wc + soft_polled, &npolled);
 		goto out;
 	}
 
 	if (unlikely(!list_empty(&cq->wc_list)))
-		soft_polled = poll_soft_wc(cq, num_entries, wc);
+		soft_polled = poll_soft_wc(cq, num_entries, wc, false);
 
 	for (npolled = 0; npolled < num_entries - soft_polled; npolled++) {
 		if (mlx5_poll_one(cq, &cur_qp, wc + soft_polled + npolled))
@@ -742,6 +751,28 @@ static int alloc_cq_frag_buf(struct mlx5_ib_dev *dev,
 	return 0;
 }
 
+enum {
+	MLX5_CQE_RES_FORMAT_HASH = 0,
+	MLX5_CQE_RES_FORMAT_CSUM = 1,
+	MLX5_CQE_RES_FORMAT_CSUM_STRIDX = 3,
+};
+
+static int mini_cqe_res_format_to_hw(struct mlx5_ib_dev *dev, u8 format)
+{
+	switch (format) {
+	case MLX5_IB_CQE_RES_FORMAT_HASH:
+		return MLX5_CQE_RES_FORMAT_HASH;
+	case MLX5_IB_CQE_RES_FORMAT_CSUM:
+		return MLX5_CQE_RES_FORMAT_CSUM;
+	case MLX5_IB_CQE_RES_FORMAT_CSUM_STRIDX:
+		if (MLX5_CAP_GEN(dev->mdev, mini_cqe_resp_stride_index))
+			return MLX5_CQE_RES_FORMAT_CSUM_STRIDX;
+		return -EOPNOTSUPP;
+	default:
+		return -EINVAL;
+	}
+}
+
 static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 			  struct ib_ucontext *context, struct mlx5_ib_cq *cq,
 			  int entries, u32 **cqb,
@@ -807,6 +838,8 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 	*index = to_mucontext(context)->bfregi.sys_pages[0];
 
 	if (ucmd.cqe_comp_en == 1) {
+		int mini_cqe_format;
+
 		if (!((*cqe_size == 128 &&
 		       MLX5_CAP_GEN(dev->mdev, cqe_compression_128)) ||
 		      (*cqe_size == 64  &&
@@ -817,20 +850,18 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 			goto err_cqb;
 		}
 
-		if (unlikely(!ucmd.cqe_comp_res_format ||
-			     !(ucmd.cqe_comp_res_format <
-			       MLX5_IB_CQE_RES_RESERVED) ||
-			     (ucmd.cqe_comp_res_format &
-			      (ucmd.cqe_comp_res_format - 1)))) {
-			err = -EOPNOTSUPP;
-			mlx5_ib_warn(dev, "CQE compression res format %d is not supported!\n",
-				     ucmd.cqe_comp_res_format);
+		mini_cqe_format =
+			mini_cqe_res_format_to_hw(dev,
+						  ucmd.cqe_comp_res_format);
+		if (mini_cqe_format < 0) {
+			err = mini_cqe_format;
+			mlx5_ib_dbg(dev, "CQE compression res format %d error: %d\n",
+				    ucmd.cqe_comp_res_format, err);
 			goto err_cqb;
 		}
 
 		MLX5_SET(cqc, cqc, cqe_comp_en, 1);
-		MLX5_SET(cqc, cqc, mini_cqe_res_format,
-			 ilog2(ucmd.cqe_comp_res_format));
+		MLX5_SET(cqc, cqc, mini_cqe_res_format, mini_cqe_format);
 	}
 
 	if (ucmd.flags & MLX5_IB_CREATE_CQ_FLAGS_CQE_128B_PAD) {
@@ -849,7 +880,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 	return 0;
 
 err_cqb:
-	kfree(*cqb);
+	kvfree(*cqb);
 
 err_db:
 	mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 69716a7ea993..50fd6ed9a929 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -982,13 +982,21 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	}
 
 	if (field_avail(typeof(resp), cqe_comp_caps, uhw->outlen)) {
-		resp.cqe_comp_caps.max_num =
-			MLX5_CAP_GEN(dev->mdev, cqe_compression) ?
-			MLX5_CAP_GEN(dev->mdev, cqe_compression_max_num) : 0;
-		resp.cqe_comp_caps.supported_format =
-			MLX5_IB_CQE_RES_FORMAT_HASH |
-			MLX5_IB_CQE_RES_FORMAT_CSUM;
 		resp.response_length += sizeof(resp.cqe_comp_caps);
+
+		if (MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
+			resp.cqe_comp_caps.max_num =
+				MLX5_CAP_GEN(dev->mdev,
+					     cqe_compression_max_num);
+
+			resp.cqe_comp_caps.supported_format =
+				MLX5_IB_CQE_RES_FORMAT_HASH |
+				MLX5_IB_CQE_RES_FORMAT_CSUM;
+
+			if (MLX5_CAP_GEN(dev->mdev, mini_cqe_resp_stride_index))
+				resp.cqe_comp_caps.supported_format |=
+					MLX5_IB_CQE_RES_FORMAT_CSUM_STRIDX;
+		}
 	}
 
 	if (field_avail(typeof(resp), packet_pacing_caps, uhw->outlen) &&
@@ -1084,6 +1092,14 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre))
 			resp.tunnel_offloads_caps |=
 				MLX5_IB_TUNNELED_OFFLOADS_GRE;
+		if (MLX5_CAP_GEN(mdev, flex_parser_protocols) &
+		    MLX5_FLEX_PROTO_CW_MPLS_GRE)
+			resp.tunnel_offloads_caps |=
+				MLX5_IB_TUNNELED_OFFLOADS_MPLS_GRE;
+		if (MLX5_CAP_GEN(mdev, flex_parser_protocols) &
+		    MLX5_FLEX_PROTO_CW_MPLS_UDP)
+			resp.tunnel_offloads_caps |=
+				MLX5_IB_TUNNELED_OFFLOADS_MPLS_UDP;
 	}
 
 	if (uhw->outlen) {
@@ -2051,10 +2067,6 @@ static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
 	if (err)
 		return err;
 
-	mlx5_ib_dbg(dev, "mapped clock info at 0x%lx, PA 0x%llx\n",
-		    vma->vm_start,
-		    (unsigned long long)pfn << PAGE_SHIFT);
-
 	return mlx5_ib_set_vma_data(vma, context);
 }
 
@@ -2149,15 +2161,14 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
 	err = io_remap_pfn_range(vma, vma->vm_start, pfn,
 				 PAGE_SIZE, vma->vm_page_prot);
 	if (err) {
-		mlx5_ib_err(dev, "io_remap_pfn_range failed with error=%d, vm_start=0x%lx, pfn=%pa, mmap_cmd=%s\n",
-			    err, vma->vm_start, &pfn, mmap_cmd2str(cmd));
+		mlx5_ib_err(dev,
+			    "io_remap_pfn_range failed with error=%d, mmap_cmd=%s\n",
+			    err, mmap_cmd2str(cmd));
 		err = -EAGAIN;
 		goto err;
 	}
 
 	pa = pfn << PAGE_SHIFT;
-	mlx5_ib_dbg(dev, "mapped %s at 0x%lx, PA %pa\n", mmap_cmd2str(cmd),
-		    vma->vm_start, &pa);
 
 	err = mlx5_ib_set_vma_data(vma, context);
 	if (err)
@@ -2243,10 +2254,6 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm
 		if (io_remap_pfn_range(vma, vma->vm_start, pfn,
 				       PAGE_SIZE, vma->vm_page_prot))
 			return -EAGAIN;
-
-		mlx5_ib_dbg(dev, "mapped internal timer at 0x%lx, PA 0x%llx\n",
-			    vma->vm_start,
-			    (unsigned long long)pfn << PAGE_SHIFT);
 		break;
 	case MLX5_IB_MMAP_CLOCK_INFO:
 		return mlx5_ib_mmap_clock_info_page(dev, vma, context);
@@ -2386,7 +2393,8 @@ static int mlx5_ib_dealloc_pd(struct ib_pd *pd)
 enum {
 	MATCH_CRITERIA_ENABLE_OUTER_BIT,
 	MATCH_CRITERIA_ENABLE_MISC_BIT,
-	MATCH_CRITERIA_ENABLE_INNER_BIT
+	MATCH_CRITERIA_ENABLE_INNER_BIT,
+	MATCH_CRITERIA_ENABLE_MISC2_BIT
 };
 
 #define HEADER_IS_ZERO(match_criteria, headers)			           \
@@ -2406,6 +2414,9 @@ static u8 get_match_criteria_enable(u32 *match_criteria)
 	match_criteria_enable |=
 		(!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
 		MATCH_CRITERIA_ENABLE_INNER_BIT;
+	match_criteria_enable |=
+		(!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
+		MATCH_CRITERIA_ENABLE_MISC2_BIT;
 
 	return match_criteria_enable;
 }
@@ -2440,6 +2451,27 @@ static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
 	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
 }
 
+static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
+{
+	if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
+	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
+		return -EOPNOTSUPP;
+
+	if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
+	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
+		return -EOPNOTSUPP;
+
+	if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
+	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
+		return -EOPNOTSUPP;
+
+	if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
+	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
 #define LAST_ETH_FIELD vlan_tag
 #define LAST_IB_FIELD sl
 #define LAST_IPV4_FIELD tos
@@ -2448,6 +2480,7 @@ static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
 #define LAST_TUNNEL_FIELD tunnel_id
 #define LAST_FLOW_TAG_FIELD tag_id
 #define LAST_DROP_FIELD size
+#define LAST_DROP_FIELD size
 
 /* Field is the last supported field */
 #define FIELDS_NOT_SUPPORTED(filter, field)\
@@ -2479,12 +2512,16 @@ static int parse_flow_flow_action(const union ib_flow_spec *ib_spec,
 static int parse_flow_attr(struct mlx5_core_dev *mdev, u32 *match_c,
 			   u32 *match_v, const union ib_flow_spec *ib_spec,
 			   const struct ib_flow_attr *flow_attr,
-			   struct mlx5_flow_act *action)
+			   struct mlx5_flow_act *action, u32 prev_type)
 {
 	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
 					   misc_parameters);
 	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
 					   misc_parameters);
+	void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
+					    misc_parameters_2);
+	void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
+					    misc_parameters_2);
 	void *headers_c;
 	void *headers_v;
 	int match_ipv;
@@ -2689,6 +2726,93 @@ static int parse_flow_attr(struct mlx5_core_dev *mdev, u32 *match_c,
 		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
 			 ntohs(ib_spec->tcp_udp.val.dst_port));
 		break;
+	case IB_FLOW_SPEC_GRE:
+		if (ib_spec->gre.mask.c_ks_res0_ver)
+			return -EOPNOTSUPP;
+
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
+			 0xff);
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
+			 IPPROTO_GRE);
+
+		MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
+			 0xffff);
+		MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
+			 ntohs(ib_spec->gre.val.protocol));
+
+		memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
+				    gre_key_h),
+		       &ib_spec->gre.mask.key,
+		       sizeof(ib_spec->gre.mask.key));
+		memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
+				    gre_key_h),
+		       &ib_spec->gre.val.key,
+		       sizeof(ib_spec->gre.val.key));
+		break;
+	case IB_FLOW_SPEC_MPLS:
+		switch (prev_type) {
+		case IB_FLOW_SPEC_UDP:
+			if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+						   ft_field_support.outer_first_mpls_over_udp),
+						   &ib_spec->mpls.mask.tag))
+				return -EOPNOTSUPP;
+
+			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
+					    outer_first_mpls_over_udp),
+			       &ib_spec->mpls.val.tag,
+			       sizeof(ib_spec->mpls.val.tag));
+			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
+					    outer_first_mpls_over_udp),
+			       &ib_spec->mpls.mask.tag,
+			       sizeof(ib_spec->mpls.mask.tag));
+			break;
+		case IB_FLOW_SPEC_GRE:
+			if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+						   ft_field_support.outer_first_mpls_over_gre),
+						   &ib_spec->mpls.mask.tag))
+				return -EOPNOTSUPP;
+
+			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
+					    outer_first_mpls_over_gre),
+			       &ib_spec->mpls.val.tag,
+			       sizeof(ib_spec->mpls.val.tag));
+			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
+					    outer_first_mpls_over_gre),
+			       &ib_spec->mpls.mask.tag,
+			       sizeof(ib_spec->mpls.mask.tag));
+			break;
+		default:
+			if (ib_spec->type & IB_FLOW_SPEC_INNER) {
+				if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+							   ft_field_support.inner_first_mpls),
+							   &ib_spec->mpls.mask.tag))
+					return -EOPNOTSUPP;
+
+				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
+						    inner_first_mpls),
+				       &ib_spec->mpls.val.tag,
+				       sizeof(ib_spec->mpls.val.tag));
+				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
+						    inner_first_mpls),
+				       &ib_spec->mpls.mask.tag,
+				       sizeof(ib_spec->mpls.mask.tag));
+			} else {
+				if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+							   ft_field_support.outer_first_mpls),
+							   &ib_spec->mpls.mask.tag))
+					return -EOPNOTSUPP;
+
+				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
+						    outer_first_mpls),
+				       &ib_spec->mpls.val.tag,
+				       sizeof(ib_spec->mpls.val.tag));
+				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
+						    outer_first_mpls),
+				       &ib_spec->mpls.mask.tag,
+				       sizeof(ib_spec->mpls.mask.tag));
+			}
+		}
+		break;
 	case IB_FLOW_SPEC_VXLAN_TUNNEL:
 		if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
 					 LAST_TUNNEL_FIELD))
@@ -3020,6 +3144,7 @@ static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
 	struct mlx5_flow_destination *rule_dst = dst;
 	const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
 	unsigned int spec_index;
+	u32 prev_type = 0;
 	int err = 0;
 	int dest_num = 1;
 	bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
@@ -3039,10 +3164,12 @@ static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
 	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
 		err = parse_flow_attr(dev->mdev, spec->match_criteria,
 				      spec->match_value,
-				      ib_flow, flow_attr, &flow_act);
+				      ib_flow, flow_attr, &flow_act,
+				      prev_type);
 		if (err < 0)
 			goto free;
 
+		prev_type = ((union ib_flow_spec *)ib_flow)->type;
 		ib_flow += ((union ib_flow_spec *)ib_flow)->size;
 	}
 
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 2193dc1765fb..a4f1f638509f 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -54,6 +54,7 @@ enum {
 
 enum {
 	MLX5_IB_SQ_STRIDE	= 6,
+	MLX5_IB_SQ_UMR_INLINE_THRESHOLD = 64,
 };
 
 static const u32 mlx5_ib_opcode[] = {
@@ -302,7 +303,9 @@ static int sq_overhead(struct ib_qp_init_attr *attr)
 			max(sizeof(struct mlx5_wqe_atomic_seg) +
 			    sizeof(struct mlx5_wqe_raddr_seg),
 			    sizeof(struct mlx5_wqe_umr_ctrl_seg) +
-			    sizeof(struct mlx5_mkey_seg));
+			    sizeof(struct mlx5_mkey_seg) +
+			    MLX5_IB_SQ_UMR_INLINE_THRESHOLD /
+			    MLX5_IB_UMR_OCTOWORD);
 		break;
 
 	case IB_QPT_XRC_TGT:
@@ -3641,13 +3644,15 @@ static __be64 sig_mkey_mask(void)
 }
 
 static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
-			    struct mlx5_ib_mr *mr)
+			    struct mlx5_ib_mr *mr, bool umr_inline)
 {
 	int size = mr->ndescs * mr->desc_size;
 
 	memset(umr, 0, sizeof(*umr));
 
 	umr->flags = MLX5_UMR_CHECK_NOT_FREE;
+	if (umr_inline)
+		umr->flags |= MLX5_UMR_INLINE;
 	umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
 	umr->mkey_mask = frwr_mkey_mask();
 }
@@ -3831,6 +3836,24 @@ static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
 	dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
 }
 
+static void set_reg_umr_inline_seg(void *seg, struct mlx5_ib_qp *qp,
+				   struct mlx5_ib_mr *mr, int mr_list_size)
+{
+	void *qend = qp->sq.qend;
+	void *addr = mr->descs;
+	int copy;
+
+	if (unlikely(seg + mr_list_size > qend)) {
+		copy = qend - seg;
+		memcpy(seg, addr, copy);
+		addr += copy;
+		mr_list_size -= copy;
+		seg = mlx5_get_send_wqe(qp, 0);
+	}
+	memcpy(seg, addr, mr_list_size);
+	seg += mr_list_size;
+}
+
 static __be32 send_ieth(struct ib_send_wr *wr)
 {
 	switch (wr->opcode) {
@@ -4225,6 +4248,8 @@ static int set_reg_wr(struct mlx5_ib_qp *qp,
 {
 	struct mlx5_ib_mr *mr = to_mmr(wr->mr);
 	struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
+	int mr_list_size = mr->ndescs * mr->desc_size;
+	bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
 
 	if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
 		mlx5_ib_warn(to_mdev(qp->ibqp.device),
@@ -4232,7 +4257,7 @@ static int set_reg_wr(struct mlx5_ib_qp *qp,
 		return -EINVAL;
 	}
 
-	set_reg_umr_seg(*seg, mr);
+	set_reg_umr_seg(*seg, mr, umr_inline);
 	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
 	*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
 	if (unlikely((*seg == qp->sq.qend)))
@@ -4244,10 +4269,14 @@ static int set_reg_wr(struct mlx5_ib_qp *qp,
 	if (unlikely((*seg == qp->sq.qend)))
 		*seg = mlx5_get_send_wqe(qp, 0);
 
-	set_reg_data_seg(*seg, mr, pd);
-	*seg += sizeof(struct mlx5_wqe_data_seg);
-	*size += (sizeof(struct mlx5_wqe_data_seg) / 16);
-
+	if (umr_inline) {
+		set_reg_umr_inline_seg(*seg, qp, mr, mr_list_size);
+		*size += get_xlt_octo(mr_list_size);
+	} else {
+		set_reg_data_seg(*seg, mr, pd);
+		*seg += sizeof(struct mlx5_wqe_data_seg);
+		*size += (sizeof(struct mlx5_wqe_data_seg) / 16);
+	}
 	return 0;
 }
 
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index 3f9afc02d166..988aace89430 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -2577,7 +2577,7 @@ static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
 	u32 pbes_in_page;
 
 	if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
-		DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
+		DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
 		return -ENOMEM;
 	}
 
diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h
index 46072455130c..3461df002f81 100644
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -1228,6 +1228,7 @@ static inline struct qib_ibport *to_iport(struct ib_device *ibdev, u8 port)
 #define QIB_BADINTR           0x8000 /* severe interrupt problems */
 #define QIB_DCA_ENABLED       0x10000 /* Direct Cache Access enabled */
 #define QIB_HAS_QSFP          0x20000 /* device (card instance) has QSFP */
+#define QIB_SHUTDOWN          0x40000 /* device is shutting down */
 
 /*
  * values for ppd->lflags (_ib_port_ related flags)
@@ -1423,8 +1424,7 @@ u64 qib_sps_ints(void);
 /*
  * dma_addr wrappers - all 0's invalid for hw
  */
-dma_addr_t qib_map_page(struct pci_dev *, struct page *, unsigned long,
-			  size_t, int);
+int qib_map_page(struct pci_dev *d, struct page *p, dma_addr_t *daddr);
 struct pci_dev *qib_get_pci_dev(struct rvt_dev_info *rdi);
 
 /*
diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c
index 6a8800b65047..98e1ce14fa2a 100644
--- a/drivers/infiniband/hw/qib/qib_file_ops.c
+++ b/drivers/infiniband/hw/qib/qib_file_ops.c
@@ -364,6 +364,8 @@ static int qib_tid_update(struct qib_ctxtdata *rcd, struct file *fp,
 		goto done;
 	}
 	for (i = 0; i < cnt; i++, vaddr += PAGE_SIZE) {
+		dma_addr_t daddr;
+
 		for (; ntids--; tid++) {
 			if (tid == tidcnt)
 				tid = 0;
@@ -380,12 +382,14 @@ static int qib_tid_update(struct qib_ctxtdata *rcd, struct file *fp,
 			ret = -ENOMEM;
 			break;
 		}
+		ret = qib_map_page(dd->pcidev, pagep[i], &daddr);
+		if (ret)
+			break;
+
 		tidlist[i] = tid + tidoff;
 		/* we "know" system pages and TID pages are same size */
 		dd->pageshadow[ctxttid + tid] = pagep[i];
-		dd->physshadow[ctxttid + tid] =
-			qib_map_page(dd->pcidev, pagep[i], 0, PAGE_SIZE,
-				     PCI_DMA_FROMDEVICE);
+		dd->physshadow[ctxttid + tid] = daddr;
 		/*
 		 * don't need atomic or it's overhead
 		 */
@@ -868,7 +872,7 @@ static int mmap_rcvegrbufs(struct vm_area_struct *vma,
 /*
  * qib_file_vma_fault - handle a VMA page fault.
  */
-static int qib_file_vma_fault(struct vm_fault *vmf)
+static vm_fault_t qib_file_vma_fault(struct vm_fault *vmf)
 {
 	struct page *page;
 
diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c
index 6c68f8a97018..015520289735 100644
--- a/drivers/infiniband/hw/qib/qib_init.c
+++ b/drivers/infiniband/hw/qib/qib_init.c
@@ -841,6 +841,10 @@ static void qib_shutdown_device(struct qib_devdata *dd)
 	struct qib_pportdata *ppd;
 	unsigned pidx;
 
+	if (dd->flags & QIB_SHUTDOWN)
+		return;
+	dd->flags |= QIB_SHUTDOWN;
+
 	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 		ppd = dd->pport + pidx;
 
@@ -1182,6 +1186,7 @@ void qib_disable_after_error(struct qib_devdata *dd)
 
 static void qib_remove_one(struct pci_dev *);
 static int qib_init_one(struct pci_dev *, const struct pci_device_id *);
+static void qib_shutdown_one(struct pci_dev *);
 
 #define DRIVER_LOAD_MSG "Intel " QIB_DRV_NAME " loaded: "
 #define PFX QIB_DRV_NAME ": "
@@ -1199,6 +1204,7 @@ static struct pci_driver qib_driver = {
 	.name = QIB_DRV_NAME,
 	.probe = qib_init_one,
 	.remove = qib_remove_one,
+	.shutdown = qib_shutdown_one,
 	.id_table = qib_pci_tbl,
 	.err_handler = &qib_pci_err_handler,
 };
@@ -1549,6 +1555,13 @@ static void qib_remove_one(struct pci_dev *pdev)
 	qib_postinit_cleanup(dd);
 }
 
+static void qib_shutdown_one(struct pci_dev *pdev)
+{
+	struct qib_devdata *dd = pci_get_drvdata(pdev);
+
+	qib_shutdown_device(dd);
+}
+
 /**
  * qib_create_rcvhdrq - create a receive header queue
  * @dd: the qlogic_ib device
diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c
index c9955d48c50f..f35fdeb14347 100644
--- a/drivers/infiniband/hw/qib/qib_rc.c
+++ b/drivers/infiniband/hw/qib/qib_rc.c
@@ -1828,7 +1828,7 @@ void qib_rc_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
 	/* OK, process the packet. */
 	switch (opcode) {
 	case OP(SEND_FIRST):
-		ret = qib_get_rwqe(qp, 0);
+		ret = rvt_get_rwqe(qp, false);
 		if (ret < 0)
 			goto nack_op_err;
 		if (!ret)
@@ -1849,7 +1849,7 @@ void qib_rc_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
 
 	case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
 		/* consume RWQE */
-		ret = qib_get_rwqe(qp, 1);
+		ret = rvt_get_rwqe(qp, true);
 		if (ret < 0)
 			goto nack_op_err;
 		if (!ret)
@@ -1858,7 +1858,7 @@ void qib_rc_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
 
 	case OP(SEND_ONLY):
 	case OP(SEND_ONLY_WITH_IMMEDIATE):
-		ret = qib_get_rwqe(qp, 0);
+		ret = rvt_get_rwqe(qp, false);
 		if (ret < 0)
 			goto nack_op_err;
 		if (!ret)
@@ -1949,7 +1949,7 @@ void qib_rc_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
 			goto send_middle;
 		else if (opcode == OP(RDMA_WRITE_ONLY))
 			goto no_immediate_data;
-		ret = qib_get_rwqe(qp, 1);
+		ret = rvt_get_rwqe(qp, true);
 		if (ret < 0)
 			goto nack_op_err;
 		if (!ret) {
diff --git a/drivers/infiniband/hw/qib/qib_ruc.c b/drivers/infiniband/hw/qib/qib_ruc.c
index 4662cc7bde92..f8a7de795beb 100644
--- a/drivers/infiniband/hw/qib/qib_ruc.c
+++ b/drivers/infiniband/hw/qib/qib_ruc.c
@@ -37,156 +37,6 @@
 #include "qib.h"
 #include "qib_mad.h"
 
-/*
- * Validate a RWQE and fill in the SGE state.
- * Return 1 if OK.
- */
-static int qib_init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe)
-{
-	int i, j, ret;
-	struct ib_wc wc;
-	struct rvt_lkey_table *rkt;
-	struct rvt_pd *pd;
-	struct rvt_sge_state *ss;
-
-	rkt = &to_idev(qp->ibqp.device)->rdi.lkey_table;
-	pd = ibpd_to_rvtpd(qp->ibqp.srq ? qp->ibqp.srq->pd : qp->ibqp.pd);
-	ss = &qp->r_sge;
-	ss->sg_list = qp->r_sg_list;
-	qp->r_len = 0;
-	for (i = j = 0; i < wqe->num_sge; i++) {
-		if (wqe->sg_list[i].length == 0)
-			continue;
-		/* Check LKEY */
-		ret = rvt_lkey_ok(rkt, pd, j ? &ss->sg_list[j - 1] : &ss->sge,
-				  NULL, &wqe->sg_list[i],
-				  IB_ACCESS_LOCAL_WRITE);
-		if (unlikely(ret <= 0))
-			goto bad_lkey;
-		qp->r_len += wqe->sg_list[i].length;
-		j++;
-	}
-	ss->num_sge = j;
-	ss->total_len = qp->r_len;
-	ret = 1;
-	goto bail;
-
-bad_lkey:
-	while (j) {
-		struct rvt_sge *sge = --j ? &ss->sg_list[j - 1] : &ss->sge;
-
-		rvt_put_mr(sge->mr);
-	}
-	ss->num_sge = 0;
-	memset(&wc, 0, sizeof(wc));
-	wc.wr_id = wqe->wr_id;
-	wc.status = IB_WC_LOC_PROT_ERR;
-	wc.opcode = IB_WC_RECV;
-	wc.qp = &qp->ibqp;
-	/* Signal solicited completion event. */
-	rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1);
-	ret = 0;
-bail:
-	return ret;
-}
-
-/**
- * qib_get_rwqe - copy the next RWQE into the QP's RWQE
- * @qp: the QP
- * @wr_id_only: update qp->r_wr_id only, not qp->r_sge
- *
- * Return -1 if there is a local error, 0 if no RWQE is available,
- * otherwise return 1.
- *
- * Can be called from interrupt level.
- */
-int qib_get_rwqe(struct rvt_qp *qp, int wr_id_only)
-{
-	unsigned long flags;
-	struct rvt_rq *rq;
-	struct rvt_rwq *wq;
-	struct rvt_srq *srq;
-	struct rvt_rwqe *wqe;
-	void (*handler)(struct ib_event *, void *);
-	u32 tail;
-	int ret;
-
-	if (qp->ibqp.srq) {
-		srq = ibsrq_to_rvtsrq(qp->ibqp.srq);
-		handler = srq->ibsrq.event_handler;
-		rq = &srq->rq;
-	} else {
-		srq = NULL;
-		handler = NULL;
-		rq = &qp->r_rq;
-	}
-
-	spin_lock_irqsave(&rq->lock, flags);
-	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
-		ret = 0;
-		goto unlock;
-	}
-
-	wq = rq->wq;
-	tail = wq->tail;
-	/* Validate tail before using it since it is user writable. */
-	if (tail >= rq->size)
-		tail = 0;
-	if (unlikely(tail == wq->head)) {
-		ret = 0;
-		goto unlock;
-	}
-	/* Make sure entry is read after head index is read. */
-	smp_rmb();
-	wqe = rvt_get_rwqe_ptr(rq, tail);
-	/*
-	 * Even though we update the tail index in memory, the verbs
-	 * consumer is not supposed to post more entries until a
-	 * completion is generated.
-	 */
-	if (++tail >= rq->size)
-		tail = 0;
-	wq->tail = tail;
-	if (!wr_id_only && !qib_init_sge(qp, wqe)) {
-		ret = -1;
-		goto unlock;
-	}
-	qp->r_wr_id = wqe->wr_id;
-
-	ret = 1;
-	set_bit(RVT_R_WRID_VALID, &qp->r_aflags);
-	if (handler) {
-		u32 n;
-
-		/*
-		 * Validate head pointer value and compute
-		 * the number of remaining WQEs.
-		 */
-		n = wq->head;
-		if (n >= rq->size)
-			n = 0;
-		if (n < tail)
-			n += rq->size - tail;
-		else
-			n -= tail;
-		if (n < srq->limit) {
-			struct ib_event ev;
-
-			srq->limit = 0;
-			spin_unlock_irqrestore(&rq->lock, flags);
-			ev.device = qp->ibqp.device;
-			ev.element.srq = qp->ibqp.srq;
-			ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
-			handler(&ev, srq->ibsrq.srq_context);
-			goto bail;
-		}
-	}
-unlock:
-	spin_unlock_irqrestore(&rq->lock, flags);
-bail:
-	return ret;
-}
-
 /*
  * Switch to alternate path.
  * The QP s_lock should be held and interrupts disabled.
@@ -419,7 +269,7 @@ static void qib_ruc_loopback(struct rvt_qp *sqp)
 		wc.ex.imm_data = wqe->wr.ex.imm_data;
 		/* FALLTHROUGH */
 	case IB_WR_SEND:
-		ret = qib_get_rwqe(qp, 0);
+		ret = rvt_get_rwqe(qp, false);
 		if (ret < 0)
 			goto op_err;
 		if (!ret)
@@ -431,7 +281,7 @@ static void qib_ruc_loopback(struct rvt_qp *sqp)
 			goto inv_err;
 		wc.wc_flags = IB_WC_WITH_IMM;
 		wc.ex.imm_data = wqe->wr.ex.imm_data;
-		ret = qib_get_rwqe(qp, 1);
+		ret = rvt_get_rwqe(qp, true);
 		if (ret < 0)
 			goto op_err;
 		if (!ret)
diff --git a/drivers/infiniband/hw/qib/qib_uc.c b/drivers/infiniband/hw/qib/qib_uc.c
index 840eec6ebc33..3e54bc11e0ae 100644
--- a/drivers/infiniband/hw/qib/qib_uc.c
+++ b/drivers/infiniband/hw/qib/qib_uc.c
@@ -335,7 +335,7 @@ void qib_uc_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
 		if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags))
 			qp->r_sge = qp->s_rdma_read_sge;
 		else {
-			ret = qib_get_rwqe(qp, 0);
+			ret = rvt_get_rwqe(qp, false);
 			if (ret < 0)
 				goto op_err;
 			if (!ret)
@@ -471,7 +471,7 @@ void qib_uc_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
 		if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags))
 			rvt_put_ss(&qp->s_rdma_read_sge);
 		else {
-			ret = qib_get_rwqe(qp, 1);
+			ret = rvt_get_rwqe(qp, true);
 			if (ret < 0)
 				goto op_err;
 			if (!ret)
diff --git a/drivers/infiniband/hw/qib/qib_ud.c b/drivers/infiniband/hw/qib/qib_ud.c
index 3e4ff77260c2..f8d029a2390f 100644
--- a/drivers/infiniband/hw/qib/qib_ud.c
+++ b/drivers/infiniband/hw/qib/qib_ud.c
@@ -139,7 +139,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 	else {
 		int ret;
 
-		ret = qib_get_rwqe(qp, 0);
+		ret = rvt_get_rwqe(qp, false);
 		if (ret < 0) {
 			rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
 			goto bail_unlock;
@@ -534,7 +534,7 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
 	else {
 		int ret;
 
-		ret = qib_get_rwqe(qp, 0);
+		ret = rvt_get_rwqe(qp, false);
 		if (ret < 0) {
 			rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
 			return;
diff --git a/drivers/infiniband/hw/qib/qib_user_pages.c b/drivers/infiniband/hw/qib/qib_user_pages.c
index ce83ba9a12ef..16543d5e80c3 100644
--- a/drivers/infiniband/hw/qib/qib_user_pages.c
+++ b/drivers/infiniband/hw/qib/qib_user_pages.c
@@ -99,23 +99,27 @@ static int __qib_get_user_pages(unsigned long start_page, size_t num_pages,
  *
  * I'm sure we won't be so lucky with other iommu's, so FIXME.
  */
-dma_addr_t qib_map_page(struct pci_dev *hwdev, struct page *page,
-			unsigned long offset, size_t size, int direction)
+int qib_map_page(struct pci_dev *hwdev, struct page *page, dma_addr_t *daddr)
 {
 	dma_addr_t phys;
 
-	phys = pci_map_page(hwdev, page, offset, size, direction);
+	phys = pci_map_page(hwdev, page, 0, PAGE_SIZE, PCI_DMA_FROMDEVICE);
+	if (pci_dma_mapping_error(hwdev, phys))
+		return -ENOMEM;
 
-	if (phys == 0) {
-		pci_unmap_page(hwdev, phys, size, direction);
-		phys = pci_map_page(hwdev, page, offset, size, direction);
+	if (!phys) {
+		pci_unmap_page(hwdev, phys, PAGE_SIZE, PCI_DMA_FROMDEVICE);
+		phys = pci_map_page(hwdev, page, 0, PAGE_SIZE,
+				    PCI_DMA_FROMDEVICE);
+		if (pci_dma_mapping_error(hwdev, phys))
+			return -ENOMEM;
 		/*
 		 * FIXME: If we get 0 again, we should keep this page,
 		 * map another, then free the 0 page.
 		 */
 	}
-
-	return phys;
+	*daddr = phys;
+	return 0;
 }
 
 /**
diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c
index 3977abbc83ad..14b4057a2b8f 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2013 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2012 - 2018 Intel Corporation.  All rights reserved.
  * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
  * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
  *
@@ -1631,10 +1631,6 @@ int qib_register_ib_device(struct qib_devdata *dd)
 	dd->verbs_dev.rdi.dparms.core_cap_flags = RDMA_CORE_PORT_IBA_IB;
 	dd->verbs_dev.rdi.dparms.max_mad_size = IB_MGMT_MAD_SIZE;
 
-	snprintf(dd->verbs_dev.rdi.dparms.cq_name,
-		 sizeof(dd->verbs_dev.rdi.dparms.cq_name),
-		 "qib_cq%d", dd->unit);
-
 	qib_fill_device_attr(dd);
 
 	ppd = dd->pport;
diff --git a/drivers/infiniband/hw/qib/qib_verbs.h b/drivers/infiniband/hw/qib/qib_verbs.h
index f887737ac142..f9a46768a19a 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.h
+++ b/drivers/infiniband/hw/qib/qib_verbs.h
@@ -321,8 +321,6 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
 
 void mr_rcu_callback(struct rcu_head *list);
 
-int qib_get_rwqe(struct rvt_qp *qp, int wr_id_only);
-
 void qib_migrate_qp(struct rvt_qp *qp);
 
 int qib_ruc_check_hdr(struct qib_ibport *ibp, struct ib_header *hdr,
diff --git a/drivers/infiniband/sw/rdmavt/Kconfig b/drivers/infiniband/sw/rdmavt/Kconfig
index 2b5513da7e83..98e798007f75 100644
--- a/drivers/infiniband/sw/rdmavt/Kconfig
+++ b/drivers/infiniband/sw/rdmavt/Kconfig
@@ -1,6 +1,6 @@
 config INFINIBAND_RDMAVT
 	tristate "RDMA verbs transport library"
-	depends on 64BIT
+	depends on 64BIT && ARCH_DMA_ADDR_T_64BIT
 	depends on PCI
 	select DMA_VIRT_OPS
 	---help---
diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c
index fb52b669bfce..4f1544ad4aff 100644
--- a/drivers/infiniband/sw/rdmavt/cq.c
+++ b/drivers/infiniband/sw/rdmavt/cq.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2016 Intel Corporation.
+ * Copyright(c) 2016 - 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -47,11 +47,12 @@
 
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
-#include <linux/kthread.h>
 #include "cq.h"
 #include "vt.h"
 #include "trace.h"
 
+static struct workqueue_struct *comp_vector_wq;
+
 /**
  * rvt_cq_enter - add a new entry to the completion queue
  * @cq: completion queue
@@ -124,20 +125,17 @@ void rvt_cq_enter(struct rvt_cq *cq, struct ib_wc *entry, bool solicited)
 		 * This will cause send_complete() to be called in
 		 * another thread.
 		 */
-		spin_lock(&cq->rdi->n_cqs_lock);
-		if (likely(cq->rdi->worker)) {
-			cq->notify = RVT_CQ_NONE;
-			cq->triggered++;
-			kthread_queue_work(cq->rdi->worker, &cq->comptask);
-		}
-		spin_unlock(&cq->rdi->n_cqs_lock);
+		cq->notify = RVT_CQ_NONE;
+		cq->triggered++;
+		queue_work_on(cq->comp_vector_cpu, comp_vector_wq,
+			      &cq->comptask);
 	}
 
 	spin_unlock_irqrestore(&cq->lock, flags);
 }
 EXPORT_SYMBOL(rvt_cq_enter);
 
-static void send_complete(struct kthread_work *work)
+static void send_complete(struct work_struct *work)
 {
 	struct rvt_cq *cq = container_of(work, struct rvt_cq, comptask);
 
@@ -189,6 +187,7 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
 	struct ib_cq *ret;
 	u32 sz;
 	unsigned int entries = attr->cqe;
+	int comp_vector = attr->comp_vector;
 
 	if (attr->flags)
 		return ERR_PTR(-EINVAL);
@@ -196,6 +195,11 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
 	if (entries < 1 || entries > rdi->dparms.props.max_cqe)
 		return ERR_PTR(-EINVAL);
 
+	if (comp_vector < 0)
+		comp_vector = 0;
+
+	comp_vector = comp_vector % rdi->ibdev.num_comp_vectors;
+
 	/* Allocate the completion queue structure. */
 	cq = kzalloc_node(sizeof(*cq), GFP_KERNEL, rdi->dparms.node);
 	if (!cq)
@@ -264,14 +268,22 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
 	 * an error.
 	 */
 	cq->rdi = rdi;
+	if (rdi->driver_f.comp_vect_cpu_lookup)
+		cq->comp_vector_cpu =
+			rdi->driver_f.comp_vect_cpu_lookup(rdi, comp_vector);
+	else
+		cq->comp_vector_cpu =
+			cpumask_first(cpumask_of_node(rdi->dparms.node));
+
 	cq->ibcq.cqe = entries;
 	cq->notify = RVT_CQ_NONE;
 	spin_lock_init(&cq->lock);
-	kthread_init_work(&cq->comptask, send_complete);
+	INIT_WORK(&cq->comptask, send_complete);
 	cq->queue = wc;
 
 	ret = &cq->ibcq;
 
+	trace_rvt_create_cq(cq, attr);
 	goto done;
 
 bail_ip:
@@ -297,7 +309,7 @@ int rvt_destroy_cq(struct ib_cq *ibcq)
 	struct rvt_cq *cq = ibcq_to_rvtcq(ibcq);
 	struct rvt_dev_info *rdi = cq->rdi;
 
-	kthread_flush_work(&cq->comptask);
+	flush_work(&cq->comptask);
 	spin_lock_irq(&rdi->n_cqs_lock);
 	rdi->n_cqs_allocated--;
 	spin_unlock_irq(&rdi->n_cqs_lock);
@@ -507,24 +519,13 @@ int rvt_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
  *
  * Return: 0 on success
  */
-int rvt_driver_cq_init(struct rvt_dev_info *rdi)
+int rvt_driver_cq_init(void)
 {
-	int cpu;
-	struct kthread_worker *worker;
-
-	if (rdi->worker)
-		return 0;
-
-	spin_lock_init(&rdi->n_cqs_lock);
-
-	cpu = cpumask_first(cpumask_of_node(rdi->dparms.node));
-	worker = kthread_create_worker_on_cpu(cpu, 0,
-					      "%s", rdi->dparms.cq_name);
-	if (IS_ERR(worker))
-		return PTR_ERR(worker);
+	comp_vector_wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_CPU_INTENSIVE,
+					 0, "rdmavt_cq");
+	if (!comp_vector_wq)
+		return -ENOMEM;
 
-	set_user_nice(worker->task, MIN_NICE);
-	rdi->worker = worker;
 	return 0;
 }
 
@@ -532,19 +533,8 @@ int rvt_driver_cq_init(struct rvt_dev_info *rdi)
  * rvt_cq_exit - tear down cq reources
  * @rdi: rvt dev structure
  */
-void rvt_cq_exit(struct rvt_dev_info *rdi)
+void rvt_cq_exit(void)
 {
-	struct kthread_worker *worker;
-
-	/* block future queuing from send_complete() */
-	spin_lock_irq(&rdi->n_cqs_lock);
-	worker = rdi->worker;
-	if (!worker) {
-		spin_unlock_irq(&rdi->n_cqs_lock);
-		return;
-	}
-	rdi->worker = NULL;
-	spin_unlock_irq(&rdi->n_cqs_lock);
-
-	kthread_destroy_worker(worker);
+	destroy_workqueue(comp_vector_wq);
+	comp_vector_wq = NULL;
 }
diff --git a/drivers/infiniband/sw/rdmavt/cq.h b/drivers/infiniband/sw/rdmavt/cq.h
index 6182c29eff66..72184b1c176b 100644
--- a/drivers/infiniband/sw/rdmavt/cq.h
+++ b/drivers/infiniband/sw/rdmavt/cq.h
@@ -2,7 +2,7 @@
 #define DEF_RVTCQ_H
 
 /*
- * Copyright(c) 2016 Intel Corporation.
+ * Copyright(c) 2016 - 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -59,6 +59,6 @@ int rvt_destroy_cq(struct ib_cq *ibcq);
 int rvt_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags);
 int rvt_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata);
 int rvt_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry);
-int rvt_driver_cq_init(struct rvt_dev_info *rdi);
-void rvt_cq_exit(struct rvt_dev_info *rdi);
+int rvt_driver_cq_init(void);
+void rvt_cq_exit(void);
 #endif          /* DEF_RVTCQ_H */
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index c82e6bb3d77c..40046135c509 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -1987,6 +1987,155 @@ int rvt_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
 	return 0;
 }
 
+/*
+ * Validate a RWQE and fill in the SGE state.
+ * Return 1 if OK.
+ */
+static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe)
+{
+	int i, j, ret;
+	struct ib_wc wc;
+	struct rvt_lkey_table *rkt;
+	struct rvt_pd *pd;
+	struct rvt_sge_state *ss;
+	struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
+
+	rkt = &rdi->lkey_table;
+	pd = ibpd_to_rvtpd(qp->ibqp.srq ? qp->ibqp.srq->pd : qp->ibqp.pd);
+	ss = &qp->r_sge;
+	ss->sg_list = qp->r_sg_list;
+	qp->r_len = 0;
+	for (i = j = 0; i < wqe->num_sge; i++) {
+		if (wqe->sg_list[i].length == 0)
+			continue;
+		/* Check LKEY */
+		ret = rvt_lkey_ok(rkt, pd, j ? &ss->sg_list[j - 1] : &ss->sge,
+				  NULL, &wqe->sg_list[i],
+				  IB_ACCESS_LOCAL_WRITE);
+		if (unlikely(ret <= 0))
+			goto bad_lkey;
+		qp->r_len += wqe->sg_list[i].length;
+		j++;
+	}
+	ss->num_sge = j;
+	ss->total_len = qp->r_len;
+	return 1;
+
+bad_lkey:
+	while (j) {
+		struct rvt_sge *sge = --j ? &ss->sg_list[j - 1] : &ss->sge;
+
+		rvt_put_mr(sge->mr);
+	}
+	ss->num_sge = 0;
+	memset(&wc, 0, sizeof(wc));
+	wc.wr_id = wqe->wr_id;
+	wc.status = IB_WC_LOC_PROT_ERR;
+	wc.opcode = IB_WC_RECV;
+	wc.qp = &qp->ibqp;
+	/* Signal solicited completion event. */
+	rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1);
+	return 0;
+}
+
+/**
+ * rvt_get_rwqe - copy the next RWQE into the QP's RWQE
+ * @qp: the QP
+ * @wr_id_only: update qp->r_wr_id only, not qp->r_sge
+ *
+ * Return -1 if there is a local error, 0 if no RWQE is available,
+ * otherwise return 1.
+ *
+ * Can be called from interrupt level.
+ */
+int rvt_get_rwqe(struct rvt_qp *qp, bool wr_id_only)
+{
+	unsigned long flags;
+	struct rvt_rq *rq;
+	struct rvt_rwq *wq;
+	struct rvt_srq *srq;
+	struct rvt_rwqe *wqe;
+	void (*handler)(struct ib_event *, void *);
+	u32 tail;
+	int ret;
+
+	if (qp->ibqp.srq) {
+		srq = ibsrq_to_rvtsrq(qp->ibqp.srq);
+		handler = srq->ibsrq.event_handler;
+		rq = &srq->rq;
+	} else {
+		srq = NULL;
+		handler = NULL;
+		rq = &qp->r_rq;
+	}
+
+	spin_lock_irqsave(&rq->lock, flags);
+	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
+		ret = 0;
+		goto unlock;
+	}
+
+	wq = rq->wq;
+	tail = wq->tail;
+	/* Validate tail before using it since it is user writable. */
+	if (tail >= rq->size)
+		tail = 0;
+	if (unlikely(tail == wq->head)) {
+		ret = 0;
+		goto unlock;
+	}
+	/* Make sure entry is read after head index is read. */
+	smp_rmb();
+	wqe = rvt_get_rwqe_ptr(rq, tail);
+	/*
+	 * Even though we update the tail index in memory, the verbs
+	 * consumer is not supposed to post more entries until a
+	 * completion is generated.
+	 */
+	if (++tail >= rq->size)
+		tail = 0;
+	wq->tail = tail;
+	if (!wr_id_only && !init_sge(qp, wqe)) {
+		ret = -1;
+		goto unlock;
+	}
+	qp->r_wr_id = wqe->wr_id;
+
+	ret = 1;
+	set_bit(RVT_R_WRID_VALID, &qp->r_aflags);
+	if (handler) {
+		u32 n;
+
+		/*
+		 * Validate head pointer value and compute
+		 * the number of remaining WQEs.
+		 */
+		n = wq->head;
+		if (n >= rq->size)
+			n = 0;
+		if (n < tail)
+			n += rq->size - tail;
+		else
+			n -= tail;
+		if (n < srq->limit) {
+			struct ib_event ev;
+
+			srq->limit = 0;
+			spin_unlock_irqrestore(&rq->lock, flags);
+			ev.device = qp->ibqp.device;
+			ev.element.srq = qp->ibqp.srq;
+			ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
+			handler(&ev, srq->ibsrq.srq_context);
+			goto bail;
+		}
+	}
+unlock:
+	spin_unlock_irqrestore(&rq->lock, flags);
+bail:
+	return ret;
+}
+EXPORT_SYMBOL(rvt_get_rwqe);
+
 /**
  * qp_comm_est - handle trap with QP established
  * @qp: the QP
@@ -2076,7 +2225,7 @@ void rvt_add_rnr_timer(struct rvt_qp *qp, u32 aeth)
 	to = rvt_aeth_to_usec(aeth);
 	trace_rvt_rnrnak_add(qp, to);
 	hrtimer_start(&qp->s_rnr_timer,
-		      ns_to_ktime(1000 * to), HRTIMER_MODE_REL);
+		      ns_to_ktime(1000 * to), HRTIMER_MODE_REL_PINNED);
 }
 EXPORT_SYMBOL(rvt_add_rnr_timer);
 
diff --git a/drivers/infiniband/sw/rdmavt/trace_cq.h b/drivers/infiniband/sw/rdmavt/trace_cq.h
index a315850aa9bb..df8e1adbef9d 100644
--- a/drivers/infiniband/sw/rdmavt/trace_cq.h
+++ b/drivers/infiniband/sw/rdmavt/trace_cq.h
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2016 Intel Corporation.
+ * Copyright(c) 2016 - 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -71,6 +71,39 @@ __print_symbolic(opcode,                                      \
 	wc_opcode_name(RECV),                                 \
 	wc_opcode_name(RECV_RDMA_WITH_IMM))
 
+#define CQ_ATTR_PRINT \
+"[%s] user cq %s cqe %u comp_vector %d comp_vector_cpu %d flags %x"
+
+DECLARE_EVENT_CLASS(rvt_cq_template,
+		    TP_PROTO(struct rvt_cq *cq,
+			     const struct ib_cq_init_attr *attr),
+		    TP_ARGS(cq, attr),
+		    TP_STRUCT__entry(RDI_DEV_ENTRY(cq->rdi)
+				     __field(struct rvt_mmap_info *, ip)
+				     __field(unsigned int, cqe)
+				     __field(int, comp_vector)
+				     __field(int, comp_vector_cpu)
+				     __field(u32, flags)
+				     ),
+		    TP_fast_assign(RDI_DEV_ASSIGN(cq->rdi)
+				   __entry->ip = cq->ip;
+				   __entry->cqe = attr->cqe;
+				   __entry->comp_vector = attr->comp_vector;
+				   __entry->comp_vector_cpu =
+							cq->comp_vector_cpu;
+				   __entry->flags = attr->flags;
+				   ),
+		    TP_printk(CQ_ATTR_PRINT, __get_str(dev),
+			      __entry->ip ? "true" : "false", __entry->cqe,
+			      __entry->comp_vector, __entry->comp_vector_cpu,
+			      __entry->flags
+			      )
+);
+
+DEFINE_EVENT(rvt_cq_template, rvt_create_cq,
+	     TP_PROTO(struct rvt_cq *cq, const struct ib_cq_init_attr *attr),
+	     TP_ARGS(cq, attr));
+
 #define CQ_PRN \
 "[%s] idx %u wr_id %llx status %u opcode %u,%s length %u qpn %x"
 
diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c
index 434199d0bc96..17e4abc067af 100644
--- a/drivers/infiniband/sw/rdmavt/vt.c
+++ b/drivers/infiniband/sw/rdmavt/vt.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2016 Intel Corporation.
+ * Copyright(c) 2016 - 2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -49,6 +49,7 @@
 #include <linux/kernel.h>
 #include <linux/dma-mapping.h>
 #include "vt.h"
+#include "cq.h"
 #include "trace.h"
 
 #define RVT_UVERBS_ABI_VERSION 2
@@ -58,21 +59,18 @@ MODULE_DESCRIPTION("RDMA Verbs Transport Library");
 
 static int rvt_init(void)
 {
-	/*
-	 * rdmavt does not need to do anything special when it starts up. All it
-	 * needs to do is sit and wait until a driver attempts registration.
-	 */
-	return 0;
+	int ret = rvt_driver_cq_init();
+
+	if (ret)
+		pr_err("Error in driver CQ init.\n");
+
+	return ret;
 }
 module_init(rvt_init);
 
 static void rvt_cleanup(void)
 {
-	/*
-	 * Nothing to do at exit time either. The module won't be able to be
-	 * removed until all drivers are gone which means all the dev structs
-	 * are gone so there is really nothing to do.
-	 */
+	rvt_cq_exit();
 }
 module_exit(rvt_cleanup);
 
@@ -777,11 +775,7 @@ int rvt_register_device(struct rvt_dev_info *rdi, u32 driver_id)
 	}
 
 	/* Completion queues */
-	ret = rvt_driver_cq_init(rdi);
-	if (ret) {
-		pr_err("Error in driver CQ init.\n");
-		goto bail_mr;
-	}
+	spin_lock_init(&rdi->n_cqs_lock);
 
 	/* DMA Operations */
 	rdi->ibdev.dev.dma_ops = rdi->ibdev.dev.dma_ops ? : &dma_virt_ops;
@@ -829,14 +823,15 @@ int rvt_register_device(struct rvt_dev_info *rdi, u32 driver_id)
 		(1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)         |
 		(1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV);
 	rdi->ibdev.node_type = RDMA_NODE_IB_CA;
-	rdi->ibdev.num_comp_vectors = 1;
+	if (!rdi->ibdev.num_comp_vectors)
+		rdi->ibdev.num_comp_vectors = 1;
 
 	rdi->ibdev.driver_id = driver_id;
 	/* We are now good to announce we exist */
 	ret =  ib_register_device(&rdi->ibdev, rdi->driver_f.port_callback);
 	if (ret) {
 		rvt_pr_err(rdi, "Failed to register driver with ib core.\n");
-		goto bail_cq;
+		goto bail_mr;
 	}
 
 	rvt_create_mad_agents(rdi);
@@ -844,9 +839,6 @@ int rvt_register_device(struct rvt_dev_info *rdi, u32 driver_id)
 	rvt_pr_info(rdi, "Registration with rdmavt done.\n");
 	return ret;
 
-bail_cq:
-	rvt_cq_exit(rdi);
-
 bail_mr:
 	rvt_mr_exit(rdi);
 
@@ -870,7 +862,6 @@ void rvt_unregister_device(struct rvt_dev_info *rdi)
 	rvt_free_mad_agents(rdi);
 
 	ib_unregister_device(&rdi->ibdev);
-	rvt_cq_exit(rdi);
 	rvt_mr_exit(rdi);
 	rvt_qp_exit(rdi);
 }
diff --git a/drivers/infiniband/sw/rxe/Kconfig b/drivers/infiniband/sw/rxe/Kconfig
index bad4a576d7cf..67ae960ab523 100644
--- a/drivers/infiniband/sw/rxe/Kconfig
+++ b/drivers/infiniband/sw/rxe/Kconfig
@@ -1,6 +1,7 @@
 config RDMA_RXE
 	tristate "Software RDMA over Ethernet (RoCE) driver"
 	depends on INET && PCI && INFINIBAND
+	depends on !64BIT || ARCH_DMA_ADDR_T_64BIT
 	select NET_UDP_TUNNEL
 	select CRYPTO_CRC32
 	select DMA_VIRT_OPS
diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index e493fdbd61c6..7121e1b1eb89 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -291,7 +291,7 @@ static int rxe_init(struct rxe_dev *rxe)
 	return err;
 }
 
-int rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)
+void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)
 {
 	struct rxe_port *port = &rxe->port;
 	enum ib_mtu mtu;
@@ -303,10 +303,7 @@ int rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)
 
 	port->attr.active_mtu = mtu;
 	port->mtu_cap = ib_mtu_enum_to_int(mtu);
-
-	return 0;
 }
-EXPORT_SYMBOL(rxe_set_mtu);
 
 /* called by ifc layer to create new rxe device.
  * The caller should allocate memory for rxe by calling ib_alloc_device.
@@ -321,9 +318,7 @@ int rxe_add(struct rxe_dev *rxe, unsigned int mtu)
 	if (err)
 		goto err1;
 
-	err = rxe_set_mtu(rxe, mtu);
-	if (err)
-		goto err1;
+	rxe_set_mtu(rxe, mtu);
 
 	err = rxe_register_device(rxe);
 	if (err)
@@ -335,7 +330,6 @@ int rxe_add(struct rxe_dev *rxe, unsigned int mtu)
 	rxe_dev_put(rxe);
 	return err;
 }
-EXPORT_SYMBOL(rxe_add);
 
 /* called by the ifc layer to remove a device */
 void rxe_remove(struct rxe_dev *rxe)
@@ -344,7 +338,6 @@ void rxe_remove(struct rxe_dev *rxe)
 
 	rxe_dev_put(rxe);
 }
-EXPORT_SYMBOL(rxe_remove);
 
 static int __init rxe_module_init(void)
 {
diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h
index 561ad307c6ec..d9ec2de68738 100644
--- a/drivers/infiniband/sw/rxe/rxe.h
+++ b/drivers/infiniband/sw/rxe/rxe.h
@@ -92,13 +92,13 @@ static inline u32 rxe_crc32(struct rxe_dev *rxe,
 	return retval;
 }
 
-int rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
+void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
 
 int rxe_add(struct rxe_dev *rxe, unsigned int mtu);
 void rxe_remove(struct rxe_dev *rxe);
 void rxe_remove_all(void);
 
-int rxe_rcv(struct sk_buff *skb);
+void rxe_rcv(struct sk_buff *skb);
 
 static inline void rxe_dev_put(struct rxe_dev *rxe)
 {
diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c
index 6cdc40ed8a9f..98d470d1f3fc 100644
--- a/drivers/infiniband/sw/rxe/rxe_comp.c
+++ b/drivers/infiniband/sw/rxe/rxe_comp.c
@@ -355,10 +355,9 @@ static inline enum comp_state do_read(struct rxe_qp *qp,
 				      struct rxe_pkt_info *pkt,
 				      struct rxe_send_wqe *wqe)
 {
-	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
 	int ret;
 
-	ret = copy_data(rxe, qp->pd, IB_ACCESS_LOCAL_WRITE,
+	ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE,
 			&wqe->dma, payload_addr(pkt),
 			payload_size(pkt), to_mem_obj, NULL);
 	if (ret)
@@ -374,12 +373,11 @@ static inline enum comp_state do_atomic(struct rxe_qp *qp,
 					struct rxe_pkt_info *pkt,
 					struct rxe_send_wqe *wqe)
 {
-	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
 	int ret;
 
 	u64 atomic_orig = atmack_orig(pkt);
 
-	ret = copy_data(rxe, qp->pd, IB_ACCESS_LOCAL_WRITE,
+	ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE,
 			&wqe->dma, &atomic_orig,
 			sizeof(u64), to_mem_obj, NULL);
 	if (ret)
@@ -661,7 +659,6 @@ int rxe_completer(void *arg)
 			    qp->qp_timeout_jiffies)
 				mod_timer(&qp->retrans_timer,
 					  jiffies + qp->qp_timeout_jiffies);
-			WARN_ON_ONCE(skb);
 			goto exit;
 
 		case COMPST_ERROR_RETRY:
@@ -675,7 +672,6 @@ int rxe_completer(void *arg)
 
 			/* there is nothing to retry in this case */
 			if (!wqe || (wqe->state == wqe_state_posted)) {
-				WARN_ON_ONCE(skb);
 				goto exit;
 			}
 
@@ -704,7 +700,6 @@ int rxe_completer(void *arg)
 					skb = NULL;
 				}
 
-				WARN_ON_ONCE(skb);
 				goto exit;
 
 			} else {
@@ -748,7 +743,6 @@ int rxe_completer(void *arg)
 				skb = NULL;
 			}
 
-			WARN_ON_ONCE(skb);
 			goto exit;
 		}
 	}
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index b71023c1c58b..a51ece596c43 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -106,20 +106,20 @@ enum copy_direction {
 	from_mem_obj,
 };
 
-int rxe_mem_init_dma(struct rxe_dev *rxe, struct rxe_pd *pd,
+int rxe_mem_init_dma(struct rxe_pd *pd,
 		     int access, struct rxe_mem *mem);
 
-int rxe_mem_init_user(struct rxe_dev *rxe, struct rxe_pd *pd, u64 start,
+int rxe_mem_init_user(struct rxe_pd *pd, u64 start,
 		      u64 length, u64 iova, int access, struct ib_udata *udata,
 		      struct rxe_mem *mr);
 
-int rxe_mem_init_fast(struct rxe_dev *rxe, struct rxe_pd *pd,
+int rxe_mem_init_fast(struct rxe_pd *pd,
 		      int max_pages, struct rxe_mem *mem);
 
 int rxe_mem_copy(struct rxe_mem *mem, u64 iova, void *addr,
 		 int length, enum copy_direction dir, u32 *crcp);
 
-int copy_data(struct rxe_dev *rxe, struct rxe_pd *pd, int access,
+int copy_data(struct rxe_pd *pd, int access,
 	      struct rxe_dma_info *dma, void *addr, int length,
 	      enum copy_direction dir, u32 *crcp);
 
@@ -143,7 +143,7 @@ void rxe_mem_cleanup(struct rxe_pool_entry *arg);
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
 
 /* rxe_net.c */
-int rxe_loopback(struct sk_buff *skb);
+void rxe_loopback(struct sk_buff *skb);
 int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb);
 struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 				int paylen, struct rxe_pkt_info *pkt);
@@ -268,7 +268,8 @@ static inline int rxe_xmit_packet(struct rxe_dev *rxe, struct rxe_qp *qp,
 
 	if (pkt->mask & RXE_LOOPBACK_MASK) {
 		memcpy(SKB_TO_PKT(skb), pkt, sizeof(*pkt));
-		err = rxe_loopback(skb);
+		rxe_loopback(skb);
+		err = 0;
 	} else {
 		err = rxe_send(pkt, skb);
 	}
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index 5c2684bf430f..dff605fdf60f 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -107,7 +107,7 @@ void rxe_mem_cleanup(struct rxe_pool_entry *arg)
 	}
 }
 
-static int rxe_mem_alloc(struct rxe_dev *rxe, struct rxe_mem *mem, int num_buf)
+static int rxe_mem_alloc(struct rxe_mem *mem, int num_buf)
 {
 	int i;
 	int num_map;
@@ -145,7 +145,7 @@ static int rxe_mem_alloc(struct rxe_dev *rxe, struct rxe_mem *mem, int num_buf)
 	return -ENOMEM;
 }
 
-int rxe_mem_init_dma(struct rxe_dev *rxe, struct rxe_pd *pd,
+int rxe_mem_init_dma(struct rxe_pd *pd,
 		     int access, struct rxe_mem *mem)
 {
 	rxe_mem_init(access, mem);
@@ -158,7 +158,7 @@ int rxe_mem_init_dma(struct rxe_dev *rxe, struct rxe_pd *pd,
 	return 0;
 }
 
-int rxe_mem_init_user(struct rxe_dev *rxe, struct rxe_pd *pd, u64 start,
+int rxe_mem_init_user(struct rxe_pd *pd, u64 start,
 		      u64 length, u64 iova, int access, struct ib_udata *udata,
 		      struct rxe_mem *mem)
 {
@@ -184,7 +184,7 @@ int rxe_mem_init_user(struct rxe_dev *rxe, struct rxe_pd *pd, u64 start,
 
 	rxe_mem_init(access, mem);
 
-	err = rxe_mem_alloc(rxe, mem, num_buf);
+	err = rxe_mem_alloc(mem, num_buf);
 	if (err) {
 		pr_warn("err %d from rxe_mem_alloc\n", err);
 		ib_umem_release(umem);
@@ -236,7 +236,7 @@ int rxe_mem_init_user(struct rxe_dev *rxe, struct rxe_pd *pd, u64 start,
 	return err;
 }
 
-int rxe_mem_init_fast(struct rxe_dev *rxe, struct rxe_pd *pd,
+int rxe_mem_init_fast(struct rxe_pd *pd,
 		      int max_pages, struct rxe_mem *mem)
 {
 	int err;
@@ -246,7 +246,7 @@ int rxe_mem_init_fast(struct rxe_dev *rxe, struct rxe_pd *pd,
 	/* In fastreg, we also set the rkey */
 	mem->ibmr.rkey = mem->ibmr.lkey;
 
-	err = rxe_mem_alloc(rxe, mem, max_pages);
+	err = rxe_mem_alloc(mem, max_pages);
 	if (err)
 		goto err1;
 
@@ -434,7 +434,6 @@ int rxe_mem_copy(struct rxe_mem *mem, u64 iova, void *addr, int length,
  * under the control of a dma descriptor
  */
 int copy_data(
-	struct rxe_dev		*rxe,
 	struct rxe_pd		*pd,
 	int			access,
 	struct rxe_dma_info	*dma,
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 9da6e37fb70c..59ec6d918ed4 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -276,9 +276,12 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
 	pkt->mask = RXE_GRH_MASK;
 	pkt->paylen = be16_to_cpu(udph->len) - sizeof(*udph);
 
-	return rxe_rcv(skb);
+	rxe_rcv(skb);
+
+	return 0;
 drop:
 	kfree_skb(skb);
+
 	return 0;
 }
 
@@ -315,7 +318,7 @@ static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port,
 	return sock;
 }
 
-void rxe_release_udp_tunnel(struct socket *sk)
+static void rxe_release_udp_tunnel(struct socket *sk)
 {
 	if (sk)
 		udp_tunnel_sock_release(sk);
@@ -517,9 +520,9 @@ int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb)
 	return 0;
 }
 
-int rxe_loopback(struct sk_buff *skb)
+void rxe_loopback(struct sk_buff *skb)
 {
-	return rxe_rcv(skb);
+	rxe_rcv(skb);
 }
 
 static inline int addr_same(struct rxe_dev *rxe, struct rxe_av *av)
@@ -562,11 +565,9 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 
 	pkt->rxe	= rxe;
 	pkt->port_num	= port_num;
-	pkt->hdr	= skb_put(skb, paylen);
+	pkt->hdr	= skb_put_zero(skb, paylen);
 	pkt->mask	|= RXE_GRH_MASK;
 
-	memset(pkt->hdr, 0, paylen);
-
 	dev_put(ndev);
 	return skb;
 }
@@ -622,7 +623,6 @@ void rxe_remove_all(void)
 	}
 	spin_unlock_bh(&dev_list_lock);
 }
-EXPORT_SYMBOL(rxe_remove_all);
 
 static void rxe_port_event(struct rxe_dev *rxe,
 			   enum ib_event_type event)
@@ -707,7 +707,7 @@ static int rxe_notify(struct notifier_block *not_blk,
 	return NOTIFY_OK;
 }
 
-struct notifier_block rxe_net_notifier = {
+static struct notifier_block rxe_net_notifier = {
 	.notifier_call = rxe_notify,
 };
 
diff --git a/drivers/infiniband/sw/rxe/rxe_net.h b/drivers/infiniband/sw/rxe/rxe_net.h
index 728d8c71b36a..106c586dbb26 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.h
+++ b/drivers/infiniband/sw/rxe/rxe_net.h
@@ -43,9 +43,6 @@ struct rxe_recv_sockets {
 	struct socket *sk6;
 };
 
-extern struct notifier_block rxe_net_notifier;
-void rxe_release_udp_tunnel(struct socket *sk);
-
 struct rxe_dev *rxe_net_add(struct net_device *ndev);
 
 int rxe_net_init(void);
diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
index dd80c7d9074a..dfba44a40f0b 100644
--- a/drivers/infiniband/sw/rxe/rxe_recv.c
+++ b/drivers/infiniband/sw/rxe/rxe_recv.c
@@ -311,7 +311,7 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)
 		 * increase the users of the skb then post to the next qp
 		 */
 		if (mce->qp_list.next != &mcg->qp_list)
-			refcount_inc(&skb->users);
+			skb_get(skb);
 
 		pkt->qp = qp;
 		rxe_add_ref(qp);
@@ -345,7 +345,7 @@ static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb)
 }
 
 /* rxe_rcv is called from the interface driver */
-int rxe_rcv(struct sk_buff *skb)
+void rxe_rcv(struct sk_buff *skb)
 {
 	int err;
 	struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
@@ -403,12 +403,11 @@ int rxe_rcv(struct sk_buff *skb)
 	else
 		rxe_rcv_pkt(rxe, pkt, skb);
 
-	return 0;
+	return;
 
 drop:
 	if (pkt->qp)
 		rxe_drop_ref(pkt->qp);
 
 	kfree_skb(skb);
-	return 0;
 }
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index 785199990457..f30eeba3f772 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -490,7 +490,7 @@ static int fill_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 			wqe->dma.resid -= paylen;
 			wqe->dma.sge_offset += paylen;
 		} else {
-			err = copy_data(rxe, qp->pd, 0, &wqe->dma,
+			err = copy_data(qp->pd, 0, &wqe->dma,
 					payload_addr(pkt), paylen,
 					from_mem_obj,
 					&crc);
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index 955ff3b6da9c..5b57de30dee4 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -511,9 +511,8 @@ static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr,
 				     int data_len)
 {
 	int err;
-	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
 
-	err = copy_data(rxe, qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,
+	err = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,
 			data_addr, data_len, to_mem_obj, NULL);
 	if (unlikely(err))
 		return (err == -ENOSPC) ? RESPST_ERR_LENGTH
@@ -987,7 +986,7 @@ static int send_atomic_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
 	memset((unsigned char *)SKB_TO_PKT(skb) + sizeof(ack_pkt), 0,
 	       sizeof(skb->cb) - sizeof(ack_pkt));
 
-	refcount_inc(&skb->users);
+	skb_get(skb);
 	res->type = RXE_ATOMIC_MASK;
 	res->atomic.skb = skb;
 	res->first_psn = ack_pkt.psn;
@@ -1121,23 +1120,12 @@ static enum resp_states duplicate_request(struct rxe_qp *qp,
 		/* Find the operation in our list of responder resources. */
 		res = find_resource(qp, pkt->psn);
 		if (res) {
-			struct sk_buff *skb_copy;
-
-			skb_copy = skb_clone(res->atomic.skb, GFP_ATOMIC);
-			if (skb_copy) {
-				rxe_add_ref(qp); /* for the new SKB */
-			} else {
-				pr_warn("Couldn't clone atomic resp\n");
-				rc = RESPST_CLEANUP;
-				goto out;
-			}
-
+			skb_get(res->atomic.skb);
 			/* Resend the result. */
 			rc = rxe_xmit_packet(to_rdev(qp->ibqp.device), qp,
-					     pkt, skb_copy);
+					     pkt, res->atomic.skb);
 			if (rc) {
 				pr_err("Failed resending result. This flow is not handled - skb ignored\n");
-				rxe_drop_ref(qp);
 				rc = RESPST_CLEANUP;
 				goto out;
 			}
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 73a00a1c06f6..9deafc3aa6af 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -1003,7 +1003,7 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)
 
 	rxe_add_ref(pd);
 
-	err = rxe_mem_init_dma(rxe, pd, access, mr);
+	err = rxe_mem_init_dma(pd, access, mr);
 	if (err)
 		goto err2;
 
@@ -1038,7 +1038,7 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,
 
 	rxe_add_ref(pd);
 
-	err = rxe_mem_init_user(rxe, pd, start, length, iova,
+	err = rxe_mem_init_user(pd, start, length, iova,
 				access, udata, mr);
 	if (err)
 		goto err3;
@@ -1086,7 +1086,7 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd,
 
 	rxe_add_ref(pd);
 
-	err = rxe_mem_init_fast(rxe, pd, max_num_sg, mr);
+	err = rxe_mem_init_fast(pd, max_num_sg, mr);
 	if (err)
 		goto err2;
 
diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h
index 308e0ce49289..a50b062ed13e 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -415,6 +415,7 @@ struct ipoib_ah {
 	struct list_head   list;
 	struct kref	   ref;
 	unsigned	   last_send;
+	int  		   valid;
 };
 
 struct ipoib_path {
@@ -431,7 +432,6 @@ struct ipoib_path {
 
 	struct rb_node	      rb_node;
 	struct list_head      list;
-	int  		      valid;
 };
 
 struct ipoib_neigh {
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index cf291f90b58f..788bb9573f1f 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -697,7 +697,8 @@ void ipoib_mark_paths_invalid(struct net_device *dev)
 		ipoib_dbg(priv, "mark path LID 0x%08x GID %pI6 invalid\n",
 			  be32_to_cpu(sa_path_get_dlid(&path->pathrec)),
 			  path->pathrec.dgid.raw);
-		path->valid =  0;
+		if (path->ah)
+			path->ah->valid = 0;
 	}
 
 	spin_unlock_irq(&priv->lock);
@@ -833,7 +834,7 @@ static void path_rec_completion(int status,
 			while ((skb = __skb_dequeue(&neigh->queue)))
 				__skb_queue_tail(&skqueue, skb);
 		}
-		path->valid = 1;
+		path->ah->valid = 1;
 	}
 
 	path->query = NULL;
@@ -926,6 +927,24 @@ static int path_rec_start(struct net_device *dev,
 	return 0;
 }
 
+static void neigh_refresh_path(struct ipoib_neigh *neigh, u8 *daddr,
+			       struct net_device *dev)
+{
+	struct ipoib_dev_priv *priv = ipoib_priv(dev);
+	struct ipoib_path *path;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->lock, flags);
+
+	path = __path_find(dev, daddr + 4);
+	if (!path)
+		goto out;
+	if (!path->query)
+		path_rec_start(dev, path);
+out:
+	spin_unlock_irqrestore(&priv->lock, flags);
+}
+
 static struct ipoib_neigh *neigh_add_path(struct sk_buff *skb, u8 *daddr,
 					  struct net_device *dev)
 {
@@ -963,7 +982,7 @@ static struct ipoib_neigh *neigh_add_path(struct sk_buff *skb, u8 *daddr,
 
 	list_add_tail(&neigh->list, &path->neigh_list);
 
-	if (path->ah) {
+	if (path->ah && path->ah->valid) {
 		kref_get(&path->ah->ref);
 		neigh->ah = path->ah;
 
@@ -1034,7 +1053,7 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
 		goto drop_and_unlock;
 
 	path = __path_find(dev, phdr->hwaddr + 4);
-	if (!path || !path->valid) {
+	if (!path || !path->ah || !path->ah->valid) {
 		int new_path = 0;
 
 		if (!path) {
@@ -1069,7 +1088,7 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
 		return;
 	}
 
-	if (path->ah) {
+	if (path->ah && path->ah->valid) {
 		ipoib_dbg(priv, "Send unicast ARP to %08x\n",
 			  be32_to_cpu(sa_path_get_dlid(&path->pathrec)));
 
@@ -1161,10 +1180,12 @@ static netdev_tx_t ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)
 			ipoib_cm_send(dev, skb, ipoib_cm_get(neigh));
 			goto unref;
 		}
-	} else if (neigh->ah) {
+	} else if (neigh->ah && neigh->ah->valid) {
 		neigh->ah->last_send = rn->send(dev, skb, neigh->ah->ah,
 						IPOIB_QPN(phdr->hwaddr));
 		goto unref;
+	} else if (neigh->ah) {
+		neigh_refresh_path(neigh, phdr->hwaddr, dev);
 	}
 
 	if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index 9b3f47ae2016..6709328d90f8 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -886,7 +886,6 @@ void ipoib_mcast_restart_task(struct work_struct *work)
 	struct netdev_hw_addr *ha;
 	struct ipoib_mcast *mcast, *tmcast;
 	LIST_HEAD(remove_list);
-	unsigned long flags;
 	struct ib_sa_mcmember_rec rec;
 
 	if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags))
@@ -898,9 +897,8 @@ void ipoib_mcast_restart_task(struct work_struct *work)
 
 	ipoib_dbg_mcast(priv, "restarting multicast task\n");
 
-	local_irq_save(flags);
-	netif_addr_lock(dev);
-	spin_lock(&priv->lock);
+	netif_addr_lock_bh(dev);
+	spin_lock_irq(&priv->lock);
 
 	/*
 	 * Unfortunately, the networking core only gives us a list of all of
@@ -978,9 +976,8 @@ void ipoib_mcast_restart_task(struct work_struct *work)
 		}
 	}
 
-	spin_unlock(&priv->lock);
-	netif_addr_unlock(dev);
-	local_irq_restore(flags);
+	spin_unlock_irq(&priv->lock);
+	netif_addr_unlock_bh(dev);
 
 	ipoib_mcast_remove_list(&remove_list);
 
@@ -988,9 +985,9 @@ void ipoib_mcast_restart_task(struct work_struct *work)
 	 * Double check that we are still up
 	 */
 	if (test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) {
-		spin_lock_irqsave(&priv->lock, flags);
+		spin_lock_irq(&priv->lock);
 		__ipoib_mcast_schedule_join_thread(priv, NULL, 0);
-		spin_unlock_irqrestore(&priv->lock, flags);
+		spin_unlock_irq(&priv->lock);
 	}
 }
 
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index fff40b097947..6a55b870b863 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -886,15 +886,9 @@ isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_des
 }
 
 static void
-isert_create_send_desc(struct isert_conn *isert_conn,
-		       struct isert_cmd *isert_cmd,
-		       struct iser_tx_desc *tx_desc)
+__isert_create_send_desc(struct isert_device *device,
+			 struct iser_tx_desc *tx_desc)
 {
-	struct isert_device *device = isert_conn->device;
-	struct ib_device *ib_dev = device->ib_device;
-
-	ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
-				   ISER_HEADERS_LEN, DMA_TO_DEVICE);
 
 	memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl));
 	tx_desc->iser_header.flags = ISCSI_CTRL;
@@ -907,6 +901,20 @@ isert_create_send_desc(struct isert_conn *isert_conn,
 	}
 }
 
+static void
+isert_create_send_desc(struct isert_conn *isert_conn,
+		       struct isert_cmd *isert_cmd,
+		       struct iser_tx_desc *tx_desc)
+{
+	struct isert_device *device = isert_conn->device;
+	struct ib_device *ib_dev = device->ib_device;
+
+	ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
+				   ISER_HEADERS_LEN, DMA_TO_DEVICE);
+
+	__isert_create_send_desc(device, tx_desc);
+}
+
 static int
 isert_init_tx_hdrs(struct isert_conn *isert_conn,
 		   struct iser_tx_desc *tx_desc)
@@ -994,7 +1002,7 @@ isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
 	struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc;
 	int ret;
 
-	isert_create_send_desc(isert_conn, NULL, tx_desc);
+	__isert_create_send_desc(device, tx_desc);
 
 	memcpy(&tx_desc->iscsi_header, &login->rsp[0],
 	       sizeof(struct iscsi_hdr));
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.c
index 4be3aef40bd2..267da8215e08 100644
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.c
+++ b/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.c
@@ -443,17 +443,16 @@ static u8 opa_vnic_get_rc(struct __opa_veswport_info *info,
 }
 
 /* opa_vnic_calc_entropy - calculate the packet entropy */
-u8 opa_vnic_calc_entropy(struct opa_vnic_adapter *adapter, struct sk_buff *skb)
+u8 opa_vnic_calc_entropy(struct sk_buff *skb)
 {
-	u16 hash16;
-
-	/*
-	 * Get flow based 16-bit hash and then XOR the upper and lower bytes
-	 * to get the entropy.
-	 * __skb_tx_hash limits qcount to 16 bits. Hence, get 15-bit hash.
-	 */
-	hash16 = __skb_tx_hash(adapter->netdev, skb, BIT(15));
-	return (u8)((hash16 >> 8) ^ (hash16 & 0xff));
+	u32 hash = skb_get_hash(skb);
+
+	/* store XOR of all bytes in lower 8 bits */
+	hash ^= hash >> 8;
+	hash ^= hash >> 16;
+
+	/* return lower 8 bits as entropy */
+	return (u8)(hash & 0xFF);
 }
 
 /* opa_vnic_get_def_port - get default port based on entropy */
@@ -490,7 +489,7 @@ void opa_vnic_encap_skb(struct opa_vnic_adapter *adapter, struct sk_buff *skb)
 
 	hdr = skb_push(skb, OPA_VNIC_HDR_LEN);
 
-	entropy = opa_vnic_calc_entropy(adapter, skb);
+	entropy = opa_vnic_calc_entropy(skb);
 	def_port = opa_vnic_get_def_port(adapter, entropy);
 	len = opa_vnic_wire_length(skb);
 	dlid = opa_vnic_get_dlid(adapter, skb, def_port);
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_internal.h b/drivers/infiniband/ulp/opa_vnic/opa_vnic_internal.h
index afd95f432262..43ac61ffef4a 100644
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_internal.h
+++ b/drivers/infiniband/ulp/opa_vnic/opa_vnic_internal.h
@@ -299,7 +299,7 @@ struct opa_vnic_adapter *opa_vnic_add_netdev(struct ib_device *ibdev,
 void opa_vnic_rem_netdev(struct opa_vnic_adapter *adapter);
 void opa_vnic_encap_skb(struct opa_vnic_adapter *adapter, struct sk_buff *skb);
 u8 opa_vnic_get_vl(struct opa_vnic_adapter *adapter, struct sk_buff *skb);
-u8 opa_vnic_calc_entropy(struct opa_vnic_adapter *adapter, struct sk_buff *skb);
+u8 opa_vnic_calc_entropy(struct sk_buff *skb);
 void opa_vnic_process_vema_config(struct opa_vnic_adapter *adapter);
 void opa_vnic_release_mac_tbl(struct opa_vnic_adapter *adapter);
 void opa_vnic_query_mac_tbl(struct opa_vnic_adapter *adapter,
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c
index ce57e0f10289..0c8aec62a425 100644
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c
+++ b/drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c
@@ -104,7 +104,7 @@ static u16 opa_vnic_select_queue(struct net_device *netdev, struct sk_buff *skb,
 
 	/* pass entropy and vl as metadata in skb */
 	mdata = skb_push(skb, sizeof(*mdata));
-	mdata->entropy =  opa_vnic_calc_entropy(adapter, skb);
+	mdata->entropy = opa_vnic_calc_entropy(skb);
 	mdata->vl = opa_vnic_get_vl(adapter, skb);
 	rc = adapter->rn_ops->ndo_select_queue(netdev, skb,
 					       accel_priv, fallback);
diff --git a/drivers/infiniband/ulp/srpt/Kconfig b/drivers/infiniband/ulp/srpt/Kconfig
index 25bf6955b6d0..fb8b7182f05e 100644
--- a/drivers/infiniband/ulp/srpt/Kconfig
+++ b/drivers/infiniband/ulp/srpt/Kconfig
@@ -1,6 +1,6 @@
 config INFINIBAND_SRPT
 	tristate "InfiniBand SCSI RDMA Protocol target support"
-	depends on INFINIBAND_ADDR_TRANS && TARGET_CORE
+	depends on INFINIBAND && INFINIBAND_ADDR_TRANS && TARGET_CORE
 	---help---
 
 	  Support for the SCSI RDMA Protocol (SRP) Target driver. The
diff --git a/drivers/input/input.c b/drivers/input/input.c
index 9785546420a7..6365c1958264 100644
--- a/drivers/input/input.c
+++ b/drivers/input/input.c
@@ -1943,8 +1943,7 @@ void input_set_capability(struct input_dev *dev, unsigned int type, unsigned int
 		break;
 
 	default:
-		pr_err("input_set_capability: unknown type %u (code %u)\n",
-		       type, code);
+		pr_err("%s: unknown type %u (code %u)\n", __func__, type, code);
 		dump_stack();
 		return;
 	}
diff --git a/drivers/input/joystick/as5011.c b/drivers/input/joystick/as5011.c
index 005d852a06e9..f051993c568e 100644
--- a/drivers/input/joystick/as5011.c
+++ b/drivers/input/joystick/as5011.c
@@ -269,9 +269,7 @@ static int as5011_probe(struct i2c_client *client,
 	input_dev->id.bustype = BUS_I2C;
 	input_dev->dev.parent = &client->dev;
 
-	__set_bit(EV_KEY, input_dev->evbit);
-	__set_bit(EV_ABS, input_dev->evbit);
-	__set_bit(BTN_JOYSTICK, input_dev->keybit);
+	input_set_capability(input_dev, EV_KEY, BTN_JOYSTICK);
 
 	input_set_abs_params(input_dev, ABS_X,
 		AS5011_MIN_AXIS, AS5011_MAX_AXIS, AS5011_FUZZ, AS5011_FLAT);
diff --git a/drivers/input/joystick/gamecon.c b/drivers/input/joystick/gamecon.c
index 2ffb2e8bdc3b..4e10ffdf8a36 100644
--- a/drivers/input/joystick/gamecon.c
+++ b/drivers/input/joystick/gamecon.c
@@ -862,7 +862,7 @@ static int gc_setup_pad(struct gc *gc, int idx, int pad_type)
 
 	case GC_N64:
 		for (i = 0; i < 10; i++)
-			__set_bit(gc_n64_btn[i], input_dev->keybit);
+			input_set_capability(input_dev, EV_KEY, gc_n64_btn[i]);
 
 		for (i = 0; i < 2; i++) {
 			input_set_abs_params(input_dev, ABS_X + i, -127, 126, 0, 2);
@@ -879,26 +879,27 @@ static int gc_setup_pad(struct gc *gc, int idx, int pad_type)
 		break;
 
 	case GC_SNESMOUSE:
-		__set_bit(BTN_LEFT, input_dev->keybit);
-		__set_bit(BTN_RIGHT, input_dev->keybit);
-		__set_bit(REL_X, input_dev->relbit);
-		__set_bit(REL_Y, input_dev->relbit);
+		input_set_capability(input_dev, EV_KEY, BTN_LEFT);
+		input_set_capability(input_dev, EV_KEY, BTN_RIGHT);
+		input_set_capability(input_dev, EV_REL, REL_X);
+		input_set_capability(input_dev, EV_REL, REL_Y);
 		break;
 
 	case GC_SNES:
 		for (i = 4; i < 8; i++)
-			__set_bit(gc_snes_btn[i], input_dev->keybit);
+			input_set_capability(input_dev, EV_KEY, gc_snes_btn[i]);
 		/* fall through */
 	case GC_NES:
 		for (i = 0; i < 4; i++)
-			__set_bit(gc_snes_btn[i], input_dev->keybit);
+			input_set_capability(input_dev, EV_KEY, gc_snes_btn[i]);
 		break;
 
 	case GC_MULTI2:
-		__set_bit(BTN_THUMB, input_dev->keybit);
+		input_set_capability(input_dev, EV_KEY, BTN_THUMB);
 		/* fall through */
 	case GC_MULTI:
-		__set_bit(BTN_TRIGGER, input_dev->keybit);
+		input_set_capability(input_dev, EV_KEY, BTN_TRIGGER);
+		/* fall through */
 		break;
 
 	case GC_PSX:
@@ -906,15 +907,17 @@ static int gc_setup_pad(struct gc *gc, int idx, int pad_type)
 			input_set_abs_params(input_dev,
 					     gc_psx_abs[i], 4, 252, 0, 2);
 		for (i = 0; i < 12; i++)
-			__set_bit(gc_psx_btn[i], input_dev->keybit);
+			input_set_capability(input_dev, EV_KEY, gc_psx_btn[i]);
+		break;
 
 		break;
 
 	case GC_DDR:
 		for (i = 0; i < 4; i++)
-			__set_bit(gc_psx_ddr_btn[i], input_dev->keybit);
+			input_set_capability(input_dev, EV_KEY,
+					     gc_psx_ddr_btn[i]);
 		for (i = 0; i < 12; i++)
-			__set_bit(gc_psx_btn[i], input_dev->keybit);
+			input_set_capability(input_dev, EV_KEY, gc_psx_btn[i]);
 
 		break;
 	}
diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c
index 06e9650b3b30..f5d2b5147dd4 100644
--- a/drivers/input/joystick/xpad.c
+++ b/drivers/input/joystick/xpad.c
@@ -86,8 +86,10 @@
 
 #define XPAD_PKT_LEN 64
 
-/* xbox d-pads should map to buttons, as is required for DDR pads
-   but we map them to axes when possible to simplify things */
+/*
+ * xbox d-pads should map to buttons, as is required for DDR pads
+ * but we map them to axes when possible to simplify things
+ */
 #define MAP_DPAD_TO_BUTTONS		(1 << 0)
 #define MAP_TRIGGERS_TO_BUTTONS		(1 << 1)
 #define MAP_STICKS_TO_NULL		(1 << 2)
@@ -387,15 +389,15 @@ static const signed short xpad_abs_triggers[] = {
  * match against vendor id as well. Wired Xbox 360 devices have protocol 1,
  * wireless controllers have protocol 129.
  */
-#define XPAD_XBOX360_VENDOR_PROTOCOL(vend,pr) \
+#define XPAD_XBOX360_VENDOR_PROTOCOL(vend, pr) \
 	.match_flags = USB_DEVICE_ID_MATCH_VENDOR | USB_DEVICE_ID_MATCH_INT_INFO, \
 	.idVendor = (vend), \
 	.bInterfaceClass = USB_CLASS_VENDOR_SPEC, \
 	.bInterfaceSubClass = 93, \
 	.bInterfaceProtocol = (pr)
 #define XPAD_XBOX360_VENDOR(vend) \
-	{ XPAD_XBOX360_VENDOR_PROTOCOL(vend,1) }, \
-	{ XPAD_XBOX360_VENDOR_PROTOCOL(vend,129) }
+	{ XPAD_XBOX360_VENDOR_PROTOCOL((vend), 1) }, \
+	{ XPAD_XBOX360_VENDOR_PROTOCOL((vend), 129) }
 
 /* The Xbox One controller uses subclass 71 and protocol 208. */
 #define XPAD_XBOXONE_VENDOR_PROTOCOL(vend, pr) \
@@ -405,7 +407,7 @@ static const signed short xpad_abs_triggers[] = {
 	.bInterfaceSubClass = 71, \
 	.bInterfaceProtocol = (pr)
 #define XPAD_XBOXONE_VENDOR(vend) \
-	{ XPAD_XBOXONE_VENDOR_PROTOCOL(vend, 208) }
+	{ XPAD_XBOXONE_VENDOR_PROTOCOL((vend), 208) }
 
 static const struct usb_device_id xpad_table[] = {
 	{ USB_INTERFACE_INFO('X', 'B', 0) },	/* X-Box USB-IF not approved class */
@@ -1573,7 +1575,6 @@ static void xpad_close(struct input_dev *dev)
 static void xpad_set_up_abs(struct input_dev *input_dev, signed short abs)
 {
 	struct usb_xpad *xpad = input_get_drvdata(input_dev);
-	set_bit(abs, input_dev->absbit);
 
 	switch (abs) {
 	case ABS_X:
@@ -1593,6 +1594,9 @@ static void xpad_set_up_abs(struct input_dev *input_dev, signed short abs)
 	case ABS_HAT0Y:	/* the d-pad (only if dpad is mapped to axes */
 		input_set_abs_params(input_dev, abs, -1, 1, 0, 0);
 		break;
+	default:
+		input_set_abs_params(input_dev, abs, 0, 0, 0, 0);
+		break;
 	}
 }
 
@@ -1633,10 +1637,7 @@ static int xpad_init_input(struct usb_xpad *xpad)
 		input_dev->close = xpad_close;
 	}
 
-	__set_bit(EV_KEY, input_dev->evbit);
-
 	if (!(xpad->mapping & MAP_STICKS_TO_NULL)) {
-		__set_bit(EV_ABS, input_dev->evbit);
 		/* set up axes */
 		for (i = 0; xpad_abs[i] >= 0; i++)
 			xpad_set_up_abs(input_dev, xpad_abs[i]);
@@ -1644,21 +1645,22 @@ static int xpad_init_input(struct usb_xpad *xpad)
 
 	/* set up standard buttons */
 	for (i = 0; xpad_common_btn[i] >= 0; i++)
-		__set_bit(xpad_common_btn[i], input_dev->keybit);
+		input_set_capability(input_dev, EV_KEY, xpad_common_btn[i]);
 
 	/* set up model-specific ones */
 	if (xpad->xtype == XTYPE_XBOX360 || xpad->xtype == XTYPE_XBOX360W ||
 	    xpad->xtype == XTYPE_XBOXONE) {
 		for (i = 0; xpad360_btn[i] >= 0; i++)
-			__set_bit(xpad360_btn[i], input_dev->keybit);
+			input_set_capability(input_dev, EV_KEY, xpad360_btn[i]);
 	} else {
 		for (i = 0; xpad_btn[i] >= 0; i++)
-			__set_bit(xpad_btn[i], input_dev->keybit);
+			input_set_capability(input_dev, EV_KEY, xpad_btn[i]);
 	}
 
 	if (xpad->mapping & MAP_DPAD_TO_BUTTONS) {
 		for (i = 0; xpad_btn_pad[i] >= 0; i++)
-			__set_bit(xpad_btn_pad[i], input_dev->keybit);
+			input_set_capability(input_dev, EV_KEY,
+					     xpad_btn_pad[i]);
 	}
 
 	/*
@@ -1675,7 +1677,8 @@ static int xpad_init_input(struct usb_xpad *xpad)
 
 	if (xpad->mapping & MAP_TRIGGERS_TO_BUTTONS) {
 		for (i = 0; xpad_btn_triggers[i] >= 0; i++)
-			__set_bit(xpad_btn_triggers[i], input_dev->keybit);
+			input_set_capability(input_dev, EV_KEY,
+					     xpad_btn_triggers[i]);
 	} else {
 		for (i = 0; xpad_abs_triggers[i] >= 0; i++)
 			xpad_set_up_abs(input_dev, xpad_abs_triggers[i]);
diff --git a/drivers/input/keyboard/Kconfig b/drivers/input/keyboard/Kconfig
index 2b469cc47a78..6bd97ffee761 100644
--- a/drivers/input/keyboard/Kconfig
+++ b/drivers/input/keyboard/Kconfig
@@ -747,4 +747,13 @@ config KEYBOARD_BCM
 	  To compile this driver as a module, choose M here: the
 	  module will be called bcm-keypad.
 
+config KEYBOARD_MTK_PMIC
+	tristate "MediaTek PMIC keys support"
+	depends on MFD_MT6397
+	help
+	  Say Y here if you want to use the pmic keys (powerkey/homekey).
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called pmic-keys.
+
 endif
diff --git a/drivers/input/keyboard/Makefile b/drivers/input/keyboard/Makefile
index 8fab920afa58..182e92985dbf 100644
--- a/drivers/input/keyboard/Makefile
+++ b/drivers/input/keyboard/Makefile
@@ -40,6 +40,7 @@ obj-$(CONFIG_KEYBOARD_MATRIX)		+= matrix_keypad.o
 obj-$(CONFIG_KEYBOARD_MAX7359)		+= max7359_keypad.o
 obj-$(CONFIG_KEYBOARD_MCS)		+= mcs_touchkey.o
 obj-$(CONFIG_KEYBOARD_MPR121)		+= mpr121_touchkey.o
+obj-$(CONFIG_KEYBOARD_MTK_PMIC) 	+= mtk-pmic-keys.o
 obj-$(CONFIG_KEYBOARD_NEWTON)		+= newtonkbd.o
 obj-$(CONFIG_KEYBOARD_NOMADIK)		+= nomadik-ske-keypad.o
 obj-$(CONFIG_KEYBOARD_NSPIRE)		+= nspire-keypad.o
diff --git a/drivers/input/keyboard/cros_ec_keyb.c b/drivers/input/keyboard/cros_ec_keyb.c
index 79eb29550c34..489ddd37bd4e 100644
--- a/drivers/input/keyboard/cros_ec_keyb.c
+++ b/drivers/input/keyboard/cros_ec_keyb.c
@@ -244,24 +244,35 @@ static int cros_ec_keyb_work(struct notifier_block *nb,
 
 	switch (ckdev->ec->event_data.event_type) {
 	case EC_MKBP_EVENT_KEY_MATRIX:
-		/*
-		 * If EC is not the wake source, discard key state changes
-		 * during suspend.
-		 */
-		if (queued_during_suspend)
-			return NOTIFY_OK;
+		if (device_may_wakeup(ckdev->dev)) {
+			pm_wakeup_event(ckdev->dev, 0);
+		} else {
+			/*
+			 * If keyboard is not wake enabled, discard key state
+			 * changes during suspend. Switches will be re-checked
+			 * in cros_ec_keyb_resume() to be sure nothing is lost.
+			 */
+			if (queued_during_suspend)
+				return NOTIFY_OK;
+		}
 
 		if (ckdev->ec->event_size != ckdev->cols) {
 			dev_err(ckdev->dev,
 				"Discarded incomplete key matrix event.\n");
 			return NOTIFY_OK;
 		}
+
 		cros_ec_keyb_process(ckdev,
 				     ckdev->ec->event_data.data.key_matrix,
 				     ckdev->ec->event_size);
 		break;
 
 	case EC_MKBP_EVENT_SYSRQ:
+		if (device_may_wakeup(ckdev->dev))
+			pm_wakeup_event(ckdev->dev, 0);
+		else if (queued_during_suspend)
+			return NOTIFY_OK;
+
 		val = get_unaligned_le32(&ckdev->ec->event_data.data.sysrq);
 		dev_dbg(ckdev->dev, "sysrq code from EC: %#x\n", val);
 		handle_sysrq(val);
@@ -269,12 +280,9 @@ static int cros_ec_keyb_work(struct notifier_block *nb,
 
 	case EC_MKBP_EVENT_BUTTON:
 	case EC_MKBP_EVENT_SWITCH:
-		/*
-		 * If EC is not the wake source, discard key state
-		 * changes during suspend. Switches will be re-checked in
-		 * cros_ec_keyb_resume() to be sure nothing is lost.
-		 */
-		if (queued_during_suspend)
+		if (device_may_wakeup(ckdev->dev))
+			pm_wakeup_event(ckdev->dev, 0);
+		else if (queued_during_suspend)
 			return NOTIFY_OK;
 
 		if (ckdev->ec->event_data.event_type == EC_MKBP_EVENT_BUTTON) {
@@ -639,6 +647,7 @@ static int cros_ec_keyb_probe(struct platform_device *pdev)
 		return err;
 	}
 
+	device_init_wakeup(ckdev->dev, true);
 	return 0;
 }
 
diff --git a/drivers/input/keyboard/mtk-pmic-keys.c b/drivers/input/keyboard/mtk-pmic-keys.c
new file mode 100644
index 000000000000..02c67a1749fc
--- /dev/null
+++ b/drivers/input/keyboard/mtk-pmic-keys.c
@@ -0,0 +1,339 @@
+/*
+ * Copyright (C) 2017 MediaTek, Inc.
+ *
+ * Author: Chen Zhong <chen.zhong@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/input.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+#include <linux/mfd/mt6323/registers.h>
+#include <linux/mfd/mt6397/registers.h>
+#include <linux/mfd/mt6397/core.h>
+
+#define MTK_PMIC_PWRKEY_RST_EN_MASK	0x1
+#define MTK_PMIC_PWRKEY_RST_EN_SHIFT	6
+#define MTK_PMIC_HOMEKEY_RST_EN_MASK	0x1
+#define MTK_PMIC_HOMEKEY_RST_EN_SHIFT	5
+#define MTK_PMIC_RST_DU_MASK		0x3
+#define MTK_PMIC_RST_DU_SHIFT		8
+
+#define MTK_PMIC_PWRKEY_RST		\
+	(MTK_PMIC_PWRKEY_RST_EN_MASK << MTK_PMIC_PWRKEY_RST_EN_SHIFT)
+#define MTK_PMIC_HOMEKEY_RST		\
+	(MTK_PMIC_HOMEKEY_RST_EN_MASK << MTK_PMIC_HOMEKEY_RST_EN_SHIFT)
+
+#define MTK_PMIC_PWRKEY_INDEX	0
+#define MTK_PMIC_HOMEKEY_INDEX	1
+#define MTK_PMIC_MAX_KEY_COUNT	2
+
+struct mtk_pmic_keys_regs {
+	u32 deb_reg;
+	u32 deb_mask;
+	u32 intsel_reg;
+	u32 intsel_mask;
+};
+
+#define MTK_PMIC_KEYS_REGS(_deb_reg, _deb_mask,		\
+	_intsel_reg, _intsel_mask)			\
+{							\
+	.deb_reg		= _deb_reg,		\
+	.deb_mask		= _deb_mask,		\
+	.intsel_reg		= _intsel_reg,		\
+	.intsel_mask		= _intsel_mask,		\
+}
+
+struct mtk_pmic_regs {
+	const struct mtk_pmic_keys_regs keys_regs[MTK_PMIC_MAX_KEY_COUNT];
+	u32 pmic_rst_reg;
+};
+
+static const struct mtk_pmic_regs mt6397_regs = {
+	.keys_regs[MTK_PMIC_PWRKEY_INDEX] =
+		MTK_PMIC_KEYS_REGS(MT6397_CHRSTATUS,
+		0x8, MT6397_INT_RSV, 0x10),
+	.keys_regs[MTK_PMIC_HOMEKEY_INDEX] =
+		MTK_PMIC_KEYS_REGS(MT6397_OCSTATUS2,
+		0x10, MT6397_INT_RSV, 0x8),
+	.pmic_rst_reg = MT6397_TOP_RST_MISC,
+};
+
+static const struct mtk_pmic_regs mt6323_regs = {
+	.keys_regs[MTK_PMIC_PWRKEY_INDEX] =
+		MTK_PMIC_KEYS_REGS(MT6323_CHRSTATUS,
+		0x2, MT6323_INT_MISC_CON, 0x10),
+	.keys_regs[MTK_PMIC_HOMEKEY_INDEX] =
+		MTK_PMIC_KEYS_REGS(MT6323_CHRSTATUS,
+		0x4, MT6323_INT_MISC_CON, 0x8),
+	.pmic_rst_reg = MT6323_TOP_RST_MISC,
+};
+
+struct mtk_pmic_keys_info {
+	struct mtk_pmic_keys *keys;
+	const struct mtk_pmic_keys_regs *regs;
+	unsigned int keycode;
+	int irq;
+	bool wakeup:1;
+};
+
+struct mtk_pmic_keys {
+	struct input_dev *input_dev;
+	struct device *dev;
+	struct regmap *regmap;
+	struct mtk_pmic_keys_info keys[MTK_PMIC_MAX_KEY_COUNT];
+};
+
+enum mtk_pmic_keys_lp_mode {
+	LP_DISABLE,
+	LP_ONEKEY,
+	LP_TWOKEY,
+};
+
+static void mtk_pmic_keys_lp_reset_setup(struct mtk_pmic_keys *keys,
+		u32 pmic_rst_reg)
+{
+	int ret;
+	u32 long_press_mode, long_press_debounce;
+
+	ret = of_property_read_u32(keys->dev->of_node,
+		"power-off-time-sec", &long_press_debounce);
+	if (ret)
+		long_press_debounce = 0;
+
+	regmap_update_bits(keys->regmap, pmic_rst_reg,
+			   MTK_PMIC_RST_DU_MASK << MTK_PMIC_RST_DU_SHIFT,
+			   long_press_debounce << MTK_PMIC_RST_DU_SHIFT);
+
+	ret = of_property_read_u32(keys->dev->of_node,
+		"mediatek,long-press-mode", &long_press_mode);
+	if (ret)
+		long_press_mode = LP_DISABLE;
+
+	switch (long_press_mode) {
+	case LP_ONEKEY:
+		regmap_update_bits(keys->regmap, pmic_rst_reg,
+				   MTK_PMIC_PWRKEY_RST,
+				   MTK_PMIC_PWRKEY_RST);
+		regmap_update_bits(keys->regmap, pmic_rst_reg,
+				   MTK_PMIC_HOMEKEY_RST,
+				   0);
+		break;
+	case LP_TWOKEY:
+		regmap_update_bits(keys->regmap, pmic_rst_reg,
+				   MTK_PMIC_PWRKEY_RST,
+				   MTK_PMIC_PWRKEY_RST);
+		regmap_update_bits(keys->regmap, pmic_rst_reg,
+				   MTK_PMIC_HOMEKEY_RST,
+				   MTK_PMIC_HOMEKEY_RST);
+		break;
+	case LP_DISABLE:
+		regmap_update_bits(keys->regmap, pmic_rst_reg,
+				   MTK_PMIC_PWRKEY_RST,
+				   0);
+		regmap_update_bits(keys->regmap, pmic_rst_reg,
+				   MTK_PMIC_HOMEKEY_RST,
+				   0);
+		break;
+	default:
+		break;
+	}
+}
+
+static irqreturn_t mtk_pmic_keys_irq_handler_thread(int irq, void *data)
+{
+	struct mtk_pmic_keys_info *info = data;
+	u32 key_deb, pressed;
+
+	regmap_read(info->keys->regmap, info->regs->deb_reg, &key_deb);
+
+	key_deb &= info->regs->deb_mask;
+
+	pressed = !key_deb;
+
+	input_report_key(info->keys->input_dev, info->keycode, pressed);
+	input_sync(info->keys->input_dev);
+
+	dev_dbg(info->keys->dev, "(%s) key =%d using PMIC\n",
+		 pressed ? "pressed" : "released", info->keycode);
+
+	return IRQ_HANDLED;
+}
+
+static int mtk_pmic_key_setup(struct mtk_pmic_keys *keys,
+		struct mtk_pmic_keys_info *info)
+{
+	int ret;
+
+	info->keys = keys;
+
+	ret = regmap_update_bits(keys->regmap, info->regs->intsel_reg,
+				 info->regs->intsel_mask,
+				 info->regs->intsel_mask);
+	if (ret < 0)
+		return ret;
+
+	ret = devm_request_threaded_irq(keys->dev, info->irq, NULL,
+					mtk_pmic_keys_irq_handler_thread,
+					IRQF_ONESHOT | IRQF_TRIGGER_HIGH,
+					"mtk-pmic-keys", info);
+	if (ret) {
+		dev_err(keys->dev, "Failed to request IRQ: %d: %d\n",
+			info->irq, ret);
+		return ret;
+	}
+
+	input_set_capability(keys->input_dev, EV_KEY, info->keycode);
+
+	return 0;
+}
+
+static int __maybe_unused mtk_pmic_keys_suspend(struct device *dev)
+{
+	struct mtk_pmic_keys *keys = dev_get_drvdata(dev);
+	int index;
+
+	for (index = 0; index < MTK_PMIC_MAX_KEY_COUNT; index++) {
+		if (keys->keys[index].wakeup)
+			enable_irq_wake(keys->keys[index].irq);
+	}
+
+	return 0;
+}
+
+static int __maybe_unused mtk_pmic_keys_resume(struct device *dev)
+{
+	struct mtk_pmic_keys *keys = dev_get_drvdata(dev);
+	int index;
+
+	for (index = 0; index < MTK_PMIC_MAX_KEY_COUNT; index++) {
+		if (keys->keys[index].wakeup)
+			disable_irq_wake(keys->keys[index].irq);
+	}
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(mtk_pmic_keys_pm_ops, mtk_pmic_keys_suspend,
+			mtk_pmic_keys_resume);
+
+static const struct of_device_id of_mtk_pmic_keys_match_tbl[] = {
+	{
+		.compatible = "mediatek,mt6397-keys",
+		.data = &mt6397_regs,
+	}, {
+		.compatible = "mediatek,mt6323-keys",
+		.data = &mt6323_regs,
+	}, {
+		/* sentinel */
+	}
+};
+MODULE_DEVICE_TABLE(of, of_mtk_pmic_keys_match_tbl);
+
+static int mtk_pmic_keys_probe(struct platform_device *pdev)
+{
+	int error, index = 0;
+	unsigned int keycount;
+	struct mt6397_chip *pmic_chip = dev_get_drvdata(pdev->dev.parent);
+	struct device_node *node = pdev->dev.of_node, *child;
+	struct mtk_pmic_keys *keys;
+	const struct mtk_pmic_regs *mtk_pmic_regs;
+	struct input_dev *input_dev;
+	const struct of_device_id *of_id =
+		of_match_device(of_mtk_pmic_keys_match_tbl, &pdev->dev);
+
+	keys = devm_kzalloc(&pdev->dev, sizeof(*keys), GFP_KERNEL);
+	if (!keys)
+		return -ENOMEM;
+
+	keys->dev = &pdev->dev;
+	keys->regmap = pmic_chip->regmap;
+	mtk_pmic_regs = of_id->data;
+
+	keys->input_dev = input_dev = devm_input_allocate_device(keys->dev);
+	if (!input_dev) {
+		dev_err(keys->dev, "input allocate device fail.\n");
+		return -ENOMEM;
+	}
+
+	input_dev->name = "mtk-pmic-keys";
+	input_dev->id.bustype = BUS_HOST;
+	input_dev->id.vendor = 0x0001;
+	input_dev->id.product = 0x0001;
+	input_dev->id.version = 0x0001;
+
+	keycount = of_get_available_child_count(node);
+	if (keycount > MTK_PMIC_MAX_KEY_COUNT) {
+		dev_err(keys->dev, "too many keys defined (%d)\n", keycount);
+		return -EINVAL;
+	}
+
+	for_each_child_of_node(node, child) {
+		keys->keys[index].regs = &mtk_pmic_regs->keys_regs[index];
+
+		keys->keys[index].irq = platform_get_irq(pdev, index);
+		if (keys->keys[index].irq < 0)
+			return keys->keys[index].irq;
+
+		error = of_property_read_u32(child,
+			"linux,keycodes", &keys->keys[index].keycode);
+		if (error) {
+			dev_err(keys->dev,
+				"failed to read key:%d linux,keycode property: %d\n",
+				index, error);
+			return error;
+		}
+
+		if (of_property_read_bool(child, "wakeup-source"))
+			keys->keys[index].wakeup = true;
+
+		error = mtk_pmic_key_setup(keys, &keys->keys[index]);
+		if (error)
+			return error;
+
+		index++;
+	}
+
+	error = input_register_device(input_dev);
+	if (error) {
+		dev_err(&pdev->dev,
+			"register input device failed (%d)\n", error);
+		return error;
+	}
+
+	mtk_pmic_keys_lp_reset_setup(keys, mtk_pmic_regs->pmic_rst_reg);
+
+	platform_set_drvdata(pdev, keys);
+
+	return 0;
+}
+
+static struct platform_driver pmic_keys_pdrv = {
+	.probe = mtk_pmic_keys_probe,
+	.driver = {
+		   .name = "mtk-pmic-keys",
+		   .of_match_table = of_mtk_pmic_keys_match_tbl,
+		   .pm = &mtk_pmic_keys_pm_ops,
+	},
+};
+
+module_platform_driver(pmic_keys_pdrv);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Chen Zhong <chen.zhong@mediatek.com>");
+MODULE_DESCRIPTION("MTK pmic-keys driver v0.1");
diff --git a/drivers/input/misc/ati_remote2.c b/drivers/input/misc/ati_remote2.c
index ded5b84e336d..d8fd58fdf050 100644
--- a/drivers/input/misc/ati_remote2.c
+++ b/drivers/input/misc/ati_remote2.c
@@ -22,7 +22,7 @@ MODULE_LICENSE("GPL");
 /*
  * ATI Remote Wonder II Channel Configuration
  *
- * The remote control can by assigned one of sixteen "channels" in order to facilitate
+ * The remote control can be assigned one of sixteen "channels" in order to facilitate
  * the use of multiple remote controls within range of each other.
  * A remote's "channel" may be altered by pressing and holding the "PC" button for
  * approximately 3 seconds, after which the button will slowly flash the count of the
diff --git a/drivers/input/misc/hp_sdc_rtc.c b/drivers/input/misc/hp_sdc_rtc.c
index 49b34de0aed4..47eb8ca729fe 100644
--- a/drivers/input/misc/hp_sdc_rtc.c
+++ b/drivers/input/misc/hp_sdc_rtc.c
@@ -509,18 +509,6 @@ static int hp_sdc_rtc_proc_show(struct seq_file *m, void *v)
 #undef NY
 }
 
-static int hp_sdc_rtc_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, hp_sdc_rtc_proc_show, NULL);
-}
-
-static const struct file_operations hp_sdc_rtc_proc_fops = {
-	.open		= hp_sdc_rtc_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int hp_sdc_rtc_ioctl(struct file *file, 
 			    unsigned int cmd, unsigned long arg)
 {
@@ -713,7 +701,7 @@ static int __init hp_sdc_rtc_init(void)
 	if (misc_register(&hp_sdc_rtc_dev) != 0)
 		printk(KERN_INFO "Could not register misc. dev for i8042 rtc\n");
 
-        proc_create("driver/rtc", 0, NULL, &hp_sdc_rtc_proc_fops);
+        proc_create_single("driver/rtc", 0, NULL, hp_sdc_rtc_proc_show);
 
 	printk(KERN_INFO "HP i8042 SDC + MSM-58321 RTC support loaded "
 			 "(RTC v " RTC_VERSION ")\n");
diff --git a/drivers/input/mouse/Kconfig b/drivers/input/mouse/Kconfig
index 89ebb8f39fee..f27f23f2d99a 100644
--- a/drivers/input/mouse/Kconfig
+++ b/drivers/input/mouse/Kconfig
@@ -133,6 +133,18 @@ config MOUSE_PS2_ELANTECH
 
 	  If unsure, say N.
 
+config MOUSE_PS2_ELANTECH_SMBUS
+	bool "Elantech PS/2 SMbus companion" if EXPERT
+	default y
+	depends on MOUSE_PS2 && MOUSE_PS2_ELANTECH
+	depends on I2C=y || I2C=MOUSE_PS2
+	select MOUSE_PS2_SMBUS
+	help
+	  Say Y here if you have a Elantech touchpad connected to
+	  to an SMBus, but enumerated through PS/2.
+
+	  If unsure, say Y.
+
 config MOUSE_PS2_SENTELIC
 	bool "Sentelic Finger Sensing Pad PS/2 protocol extension"
 	depends on MOUSE_PS2
diff --git a/drivers/input/mouse/alps.c b/drivers/input/mouse/alps.c
index 38f9501acdf0..cb5579716dba 100644
--- a/drivers/input/mouse/alps.c
+++ b/drivers/input/mouse/alps.c
@@ -2049,14 +2049,11 @@ static int alps_hw_init_v1_v2(struct psmouse *psmouse)
 	return 0;
 }
 
-static int alps_hw_init_v6(struct psmouse *psmouse)
+/* Must be in passthrough mode when calling this function */
+static int alps_trackstick_enter_extended_mode_v3_v6(struct psmouse *psmouse)
 {
 	unsigned char param[2] = {0xC8, 0x14};
 
-	/* Enter passthrough mode to let trackpoint enter 6byte raw mode */
-	if (alps_passthrough_mode_v2(psmouse, true))
-		return -1;
-
 	if (ps2_command(&psmouse->ps2dev, NULL, PSMOUSE_CMD_SETSCALE11) ||
 	    ps2_command(&psmouse->ps2dev, NULL, PSMOUSE_CMD_SETSCALE11) ||
 	    ps2_command(&psmouse->ps2dev, NULL, PSMOUSE_CMD_SETSCALE11) ||
@@ -2064,9 +2061,25 @@ static int alps_hw_init_v6(struct psmouse *psmouse)
 	    ps2_command(&psmouse->ps2dev, &param[1], PSMOUSE_CMD_SETRATE))
 		return -1;
 
+	return 0;
+}
+
+static int alps_hw_init_v6(struct psmouse *psmouse)
+{
+	int ret;
+
+	/* Enter passthrough mode to let trackpoint enter 6byte raw mode */
+	if (alps_passthrough_mode_v2(psmouse, true))
+		return -1;
+
+	ret = alps_trackstick_enter_extended_mode_v3_v6(psmouse);
+
 	if (alps_passthrough_mode_v2(psmouse, false))
 		return -1;
 
+	if (ret)
+		return ret;
+
 	if (alps_absolute_mode_v6(psmouse)) {
 		psmouse_err(psmouse, "Failed to enable absolute mode\n");
 		return -1;
@@ -2140,10 +2153,18 @@ static int alps_probe_trackstick_v3_v7(struct psmouse *psmouse, int reg_base)
 
 static int alps_setup_trackstick_v3(struct psmouse *psmouse, int reg_base)
 {
-	struct ps2dev *ps2dev = &psmouse->ps2dev;
 	int ret = 0;
+	int reg_val;
 	unsigned char param[4];
 
+	/*
+	 * We need to configure trackstick to report data for touchpad in
+	 * extended format. And also we need to tell touchpad to expect data
+	 * from trackstick in extended format. Without this configuration
+	 * trackstick packets sent from touchpad are in basic format which is
+	 * different from what we expect.
+	 */
+
 	if (alps_passthrough_mode_v3(psmouse, reg_base, true))
 		return -EIO;
 
@@ -2161,39 +2182,36 @@ static int alps_setup_trackstick_v3(struct psmouse *psmouse, int reg_base)
 		ret = -ENODEV;
 	} else {
 		psmouse_dbg(psmouse, "trackstick E7 report: %3ph\n", param);
-
-		/*
-		 * Not sure what this does, but it is absolutely
-		 * essential. Without it, the touchpad does not
-		 * work at all and the trackstick just emits normal
-		 * PS/2 packets.
-		 */
-		if (ps2_command(ps2dev, NULL, PSMOUSE_CMD_SETSCALE11) ||
-		    ps2_command(ps2dev, NULL, PSMOUSE_CMD_SETSCALE11) ||
-		    ps2_command(ps2dev, NULL, PSMOUSE_CMD_SETSCALE11) ||
-		    alps_command_mode_send_nibble(psmouse, 0x9) ||
-		    alps_command_mode_send_nibble(psmouse, 0x4)) {
-			psmouse_err(psmouse,
-				    "Error sending magic E6 sequence\n");
+		if (alps_trackstick_enter_extended_mode_v3_v6(psmouse)) {
+			psmouse_err(psmouse, "Failed to enter into trackstick extended mode\n");
 			ret = -EIO;
-			goto error;
 		}
+	}
+
+	if (alps_passthrough_mode_v3(psmouse, reg_base, false))
+		return -EIO;
+
+	if (ret)
+		return ret;
 
+	if (alps_enter_command_mode(psmouse))
+		return -EIO;
+
+	reg_val = alps_command_mode_read_reg(psmouse, reg_base + 0x08);
+	if (reg_val == -1) {
+		ret = -EIO;
+	} else {
 		/*
-		 * This ensures the trackstick packets are in the format
-		 * supported by this driver. If bit 1 isn't set the packet
-		 * format is different.
+		 * Tell touchpad that trackstick is now in extended mode.
+		 * If bit 1 isn't set the packet format is different.
 		 */
-		if (alps_enter_command_mode(psmouse) ||
-		    alps_command_mode_write_reg(psmouse,
-						reg_base + 0x08, 0x82) ||
-		    alps_exit_command_mode(psmouse))
+		reg_val |= BIT(1);
+		if (__alps_command_mode_write_reg(psmouse, reg_val))
 			ret = -EIO;
 	}
 
-error:
-	if (alps_passthrough_mode_v3(psmouse, reg_base, false))
-		ret = -EIO;
+	if (alps_exit_command_mode(psmouse))
+		return -EIO;
 
 	return ret;
 }
diff --git a/drivers/input/mouse/elan_i2c_core.c b/drivers/input/mouse/elan_i2c_core.c
index 75e757520ef0..2690a4b7ed53 100644
--- a/drivers/input/mouse/elan_i2c_core.c
+++ b/drivers/input/mouse/elan_i2c_core.c
@@ -36,6 +36,7 @@
 #include <linux/jiffies.h>
 #include <linux/completion.h>
 #include <linux/of.h>
+#include <linux/property.h>
 #include <linux/regulator/consumer.h>
 #include <asm/unaligned.h>
 
@@ -51,6 +52,7 @@
 #define ETP_MAX_FINGERS		5
 #define ETP_FINGER_DATA_LEN	5
 #define ETP_REPORT_ID		0x5D
+#define ETP_TP_REPORT_ID	0x5E
 #define ETP_REPORT_ID_OFFSET	2
 #define ETP_TOUCH_INFO_OFFSET	3
 #define ETP_FINGER_DATA_OFFSET	4
@@ -61,6 +63,7 @@
 struct elan_tp_data {
 	struct i2c_client	*client;
 	struct input_dev	*input;
+	struct input_dev	*tp_input; /* trackpoint input node */
 	struct regulator	*vcc;
 
 	const struct elan_transport_ops *ops;
@@ -930,6 +933,33 @@ static void elan_report_absolute(struct elan_tp_data *data, u8 *packet)
 	input_sync(input);
 }
 
+static void elan_report_trackpoint(struct elan_tp_data *data, u8 *report)
+{
+	struct input_dev *input = data->tp_input;
+	u8 *packet = &report[ETP_REPORT_ID_OFFSET + 1];
+	int x, y;
+
+	if (!data->tp_input) {
+		dev_warn_once(&data->client->dev,
+			      "received a trackpoint report while no trackpoint device has been created. Please report upstream.\n");
+		return;
+	}
+
+	input_report_key(input, BTN_LEFT, packet[0] & 0x01);
+	input_report_key(input, BTN_RIGHT, packet[0] & 0x02);
+	input_report_key(input, BTN_MIDDLE, packet[0] & 0x04);
+
+	if ((packet[3] & 0x0F) == 0x06) {
+		x = packet[4] - (int)((packet[1] ^ 0x80) << 1);
+		y = (int)((packet[2] ^ 0x80) << 1) - packet[5];
+
+		input_report_rel(input, REL_X, x);
+		input_report_rel(input, REL_Y, y);
+	}
+
+	input_sync(input);
+}
+
 static irqreturn_t elan_isr(int irq, void *dev_id)
 {
 	struct elan_tp_data *data = dev_id;
@@ -951,11 +981,17 @@ static irqreturn_t elan_isr(int irq, void *dev_id)
 	if (error)
 		goto out;
 
-	if (report[ETP_REPORT_ID_OFFSET] != ETP_REPORT_ID)
+	switch (report[ETP_REPORT_ID_OFFSET]) {
+	case ETP_REPORT_ID:
+		elan_report_absolute(data, report);
+		break;
+	case ETP_TP_REPORT_ID:
+		elan_report_trackpoint(data, report);
+		break;
+	default:
 		dev_err(dev, "invalid report id data (%x)\n",
 			report[ETP_REPORT_ID_OFFSET]);
-	else
-		elan_report_absolute(data, report);
+	}
 
 out:
 	return IRQ_HANDLED;
@@ -966,6 +1002,36 @@ static irqreturn_t elan_isr(int irq, void *dev_id)
  * Elan initialization functions
  ******************************************************************
  */
+
+static int elan_setup_trackpoint_input_device(struct elan_tp_data *data)
+{
+	struct device *dev = &data->client->dev;
+	struct input_dev *input;
+
+	input = devm_input_allocate_device(dev);
+	if (!input)
+		return -ENOMEM;
+
+	input->name = "Elan TrackPoint";
+	input->id.bustype = BUS_I2C;
+	input->id.vendor = ELAN_VENDOR_ID;
+	input->id.product = data->product_id;
+	input_set_drvdata(input, data);
+
+	input_set_capability(input, EV_REL, REL_X);
+	input_set_capability(input, EV_REL, REL_Y);
+	input_set_capability(input, EV_KEY, BTN_LEFT);
+	input_set_capability(input, EV_KEY, BTN_RIGHT);
+	input_set_capability(input, EV_KEY, BTN_MIDDLE);
+
+	__set_bit(INPUT_PROP_POINTER, input->propbit);
+	__set_bit(INPUT_PROP_POINTING_STICK, input->propbit);
+
+	data->tp_input = input;
+
+	return 0;
+}
+
 static int elan_setup_input_device(struct elan_tp_data *data)
 {
 	struct device *dev = &data->client->dev;
@@ -1140,6 +1206,12 @@ static int elan_probe(struct i2c_client *client,
 	if (error)
 		return error;
 
+	if (device_property_read_bool(&client->dev, "elan,trackpoint")) {
+		error = elan_setup_trackpoint_input_device(data);
+		if (error)
+			return error;
+	}
+
 	/*
 	 * Platform code (ACPI, DTS) should normally set up interrupt
 	 * for us, but in case it did not let's fall back to using falling
@@ -1177,6 +1249,16 @@ static int elan_probe(struct i2c_client *client,
 		return error;
 	}
 
+	if (data->tp_input) {
+		error = input_register_device(data->tp_input);
+		if (error) {
+			dev_err(&client->dev,
+				"failed to register TrackPoint input device: %d\n",
+				error);
+			return error;
+		}
+	}
+
 	/*
 	 * Systems using device tree should set up wakeup via DTS,
 	 * the rest will configure device as wakeup source by default.
diff --git a/drivers/input/mouse/elan_i2c_smbus.c b/drivers/input/mouse/elan_i2c_smbus.c
index 29f99529b187..cfcb32559925 100644
--- a/drivers/input/mouse/elan_i2c_smbus.c
+++ b/drivers/input/mouse/elan_i2c_smbus.c
@@ -130,7 +130,7 @@ static int elan_smbus_get_baseline_data(struct i2c_client *client,
 					bool max_baseline, u8 *value)
 {
 	int error;
-	u8 val[3];
+	u8 val[I2C_SMBUS_BLOCK_MAX] = {0};
 
 	error = i2c_smbus_read_block_data(client,
 					  max_baseline ?
@@ -149,7 +149,7 @@ static int elan_smbus_get_version(struct i2c_client *client,
 				  bool iap, u8 *version)
 {
 	int error;
-	u8 val[3];
+	u8 val[I2C_SMBUS_BLOCK_MAX] = {0};
 
 	error = i2c_smbus_read_block_data(client,
 					  iap ? ETP_SMBUS_IAP_VERSION_CMD :
@@ -170,7 +170,7 @@ static int elan_smbus_get_sm_version(struct i2c_client *client,
 				     u8 *clickpad)
 {
 	int error;
-	u8 val[3];
+	u8 val[I2C_SMBUS_BLOCK_MAX] = {0};
 
 	error = i2c_smbus_read_block_data(client,
 					  ETP_SMBUS_SM_VERSION_CMD, val);
@@ -188,7 +188,7 @@ static int elan_smbus_get_sm_version(struct i2c_client *client,
 static int elan_smbus_get_product_id(struct i2c_client *client, u16 *id)
 {
 	int error;
-	u8 val[3];
+	u8 val[I2C_SMBUS_BLOCK_MAX] = {0};
 
 	error = i2c_smbus_read_block_data(client,
 					  ETP_SMBUS_UNIQUEID_CMD, val);
@@ -205,7 +205,7 @@ static int elan_smbus_get_checksum(struct i2c_client *client,
 				   bool iap, u16 *csum)
 {
 	int error;
-	u8 val[3];
+	u8 val[I2C_SMBUS_BLOCK_MAX] = {0};
 
 	error = i2c_smbus_read_block_data(client,
 					  iap ? ETP_SMBUS_FW_CHECKSUM_CMD :
@@ -226,7 +226,7 @@ static int elan_smbus_get_max(struct i2c_client *client,
 {
 	int ret;
 	int error;
-	u8 val[3];
+	u8 val[I2C_SMBUS_BLOCK_MAX] = {0};
 
 	ret = i2c_smbus_read_block_data(client, ETP_SMBUS_RANGE_CMD, val);
 	if (ret != 3) {
@@ -246,7 +246,7 @@ static int elan_smbus_get_resolution(struct i2c_client *client,
 {
 	int ret;
 	int error;
-	u8 val[3];
+	u8 val[I2C_SMBUS_BLOCK_MAX] = {0};
 
 	ret = i2c_smbus_read_block_data(client, ETP_SMBUS_RESOLUTION_CMD, val);
 	if (ret != 3) {
@@ -267,7 +267,7 @@ static int elan_smbus_get_num_traces(struct i2c_client *client,
 {
 	int ret;
 	int error;
-	u8 val[3];
+	u8 val[I2C_SMBUS_BLOCK_MAX] = {0};
 
 	ret = i2c_smbus_read_block_data(client, ETP_SMBUS_XY_TRACENUM_CMD, val);
 	if (ret != 3) {
@@ -294,7 +294,7 @@ static int elan_smbus_iap_get_mode(struct i2c_client *client,
 {
 	int error;
 	u16 constant;
-	u8 val[3];
+	u8 val[I2C_SMBUS_BLOCK_MAX] = {0};
 
 	error = i2c_smbus_read_block_data(client, ETP_SMBUS_IAP_CTRL_CMD, val);
 	if (error < 0) {
@@ -345,7 +345,7 @@ static int elan_smbus_prepare_fw_update(struct i2c_client *client)
 	int len;
 	int error;
 	enum tp_mode mode;
-	u8 val[3];
+	u8 val[I2C_SMBUS_BLOCK_MAX] = {0};
 	u8 cmd[4] = {0x0F, 0x78, 0x00, 0x06};
 	u16 password;
 
@@ -419,7 +419,7 @@ static int elan_smbus_write_fw_block(struct i2c_client *client,
 	struct device *dev = &client->dev;
 	int error;
 	u16 result;
-	u8 val[3];
+	u8 val[I2C_SMBUS_BLOCK_MAX] = {0};
 
 	/*
 	 * Due to the limitation of smbus protocol limiting
diff --git a/drivers/input/mouse/elantech.c b/drivers/input/mouse/elantech.c
index db47a5e1d114..fb4d902c4403 100644
--- a/drivers/input/mouse/elantech.c
+++ b/drivers/input/mouse/elantech.c
@@ -14,17 +14,20 @@
 #include <linux/dmi.h>
 #include <linux/slab.h>
 #include <linux/module.h>
+#include <linux/i2c.h>
 #include <linux/input.h>
 #include <linux/input/mt.h>
+#include <linux/platform_device.h>
 #include <linux/serio.h>
 #include <linux/libps2.h>
 #include <asm/unaligned.h>
 #include "psmouse.h"
 #include "elantech.h"
+#include "elan_i2c.h"
 
 #define elantech_debug(fmt, ...)					\
 	do {								\
-		if (etd->debug)						\
+		if (etd->info.debug)					\
 			psmouse_printk(KERN_DEBUG, psmouse,		\
 					fmt, ##__VA_ARGS__);		\
 	} while (0)
@@ -105,7 +108,7 @@ static int elantech_read_reg(struct psmouse *psmouse, unsigned char reg,
 	if (reg > 0x11 && reg < 0x20)
 		return -1;
 
-	switch (etd->hw_version) {
+	switch (etd->info.hw_version) {
 	case 1:
 		if (ps2_sliced_command(&psmouse->ps2dev, ETP_REGISTER_READ) ||
 		    ps2_sliced_command(&psmouse->ps2dev, reg) ||
@@ -137,7 +140,7 @@ static int elantech_read_reg(struct psmouse *psmouse, unsigned char reg,
 
 	if (rc)
 		psmouse_err(psmouse, "failed to read register 0x%02x.\n", reg);
-	else if (etd->hw_version != 4)
+	else if (etd->info.hw_version != 4)
 		*val = param[0];
 	else
 		*val = param[1];
@@ -160,7 +163,7 @@ static int elantech_write_reg(struct psmouse *psmouse, unsigned char reg,
 	if (reg > 0x11 && reg < 0x20)
 		return -1;
 
-	switch (etd->hw_version) {
+	switch (etd->info.hw_version) {
 	case 1:
 		if (ps2_sliced_command(&psmouse->ps2dev, ETP_REGISTER_WRITE) ||
 		    ps2_sliced_command(&psmouse->ps2dev, reg) ||
@@ -237,7 +240,7 @@ static void elantech_report_absolute_v1(struct psmouse *psmouse)
 	unsigned char *packet = psmouse->packet;
 	int fingers;
 
-	if (etd->fw_version < 0x020000) {
+	if (etd->info.fw_version < 0x020000) {
 		/*
 		 * byte 0:  D   U  p1  p2   1  p3   R   L
 		 * byte 1:  f   0  th  tw  x9  x8  y9  y8
@@ -252,7 +255,7 @@ static void elantech_report_absolute_v1(struct psmouse *psmouse)
 		fingers = (packet[0] & 0xc0) >> 6;
 	}
 
-	if (etd->jumpy_cursor) {
+	if (etd->info.jumpy_cursor) {
 		if (fingers != 1) {
 			etd->single_finger_reports = 0;
 		} else if (etd->single_finger_reports < 2) {
@@ -282,8 +285,8 @@ static void elantech_report_absolute_v1(struct psmouse *psmouse)
 
 	psmouse_report_standard_buttons(dev, packet[0]);
 
-	if (etd->fw_version < 0x020000 &&
-	    (etd->capabilities[0] & ETP_CAP_HAS_ROCKER)) {
+	if (etd->info.fw_version < 0x020000 &&
+	    (etd->info.capabilities[0] & ETP_CAP_HAS_ROCKER)) {
 		/* rocker up */
 		input_report_key(dev, BTN_FORWARD, packet[0] & 0x40);
 		/* rocker down */
@@ -391,7 +394,7 @@ static void elantech_report_absolute_v2(struct psmouse *psmouse)
 	input_report_key(dev, BTN_TOOL_TRIPLETAP, fingers == 3);
 	input_report_key(dev, BTN_TOOL_QUADTAP, fingers == 4);
 	psmouse_report_standard_buttons(dev, packet[0]);
-	if (etd->reports_pressure) {
+	if (etd->info.reports_pressure) {
 		input_report_abs(dev, ABS_PRESSURE, pres);
 		input_report_abs(dev, ABS_TOOL_WIDTH, width);
 	}
@@ -444,7 +447,7 @@ static void elantech_report_trackpoint(struct psmouse *psmouse,
 
 	default:
 		/* Dump unexpected packet sequences if debug=1 (default) */
-		if (etd->debug == 1)
+		if (etd->info.debug == 1)
 			elantech_packet_dump(psmouse);
 
 		break;
@@ -523,7 +526,7 @@ static void elantech_report_absolute_v3(struct psmouse *psmouse,
 	input_report_key(dev, BTN_TOOL_TRIPLETAP, fingers == 3);
 
 	/* For clickpads map both buttons to BTN_LEFT */
-	if (etd->fw_version & 0x001000)
+	if (etd->info.fw_version & 0x001000)
 		input_report_key(dev, BTN_LEFT, packet[0] & 0x03);
 	else
 		psmouse_report_standard_buttons(dev, packet[0]);
@@ -541,7 +544,7 @@ static void elantech_input_sync_v4(struct psmouse *psmouse)
 	unsigned char *packet = psmouse->packet;
 
 	/* For clickpads map both buttons to BTN_LEFT */
-	if (etd->fw_version & 0x001000)
+	if (etd->info.fw_version & 0x001000)
 		input_report_key(dev, BTN_LEFT, packet[0] & 0x03);
 	else
 		psmouse_report_standard_buttons(dev, packet[0]);
@@ -669,7 +672,7 @@ static int elantech_packet_check_v1(struct psmouse *psmouse)
 	unsigned char p1, p2, p3;
 
 	/* Parity bits are placed differently */
-	if (etd->fw_version < 0x020000) {
+	if (etd->info.fw_version < 0x020000) {
 		/* byte 0:  D   U  p1  p2   1  p3   R   L */
 		p1 = (packet[0] & 0x20) >> 5;
 		p2 = (packet[0] & 0x10) >> 4;
@@ -714,7 +717,7 @@ static int elantech_packet_check_v2(struct psmouse *psmouse)
 	 * With all three cases, if the constant bits are not exactly what I
 	 * expected, I consider them invalid.
 	 */
-	if (etd->reports_pressure)
+	if (etd->info.reports_pressure)
 		return (packet[0] & 0x0c) == 0x04 &&
 		       (packet[3] & 0x0f) == 0x02;
 
@@ -751,7 +754,7 @@ static int elantech_packet_check_v3(struct psmouse *psmouse)
 	 * If the hardware flag 'crc_enabled' is set the packets have
 	 * different signatures.
 	 */
-	if (etd->crc_enabled) {
+	if (etd->info.crc_enabled) {
 		if ((packet[3] & 0x09) == 0x08)
 			return PACKET_V3_HEAD;
 
@@ -782,7 +785,7 @@ static int elantech_packet_check_v4(struct psmouse *psmouse)
 		return PACKET_TRACKPOINT;
 
 	/* This represents the version of IC body. */
-	ic_version = (etd->fw_version & 0x0f0000) >> 16;
+	ic_version = (etd->info.fw_version & 0x0f0000) >> 16;
 
 	/*
 	 * Sanity check based on the constant bits of a packet.
@@ -791,9 +794,9 @@ static int elantech_packet_check_v4(struct psmouse *psmouse)
 	 * the IC body, but are the same for every packet,
 	 * regardless of the type.
 	 */
-	if (etd->crc_enabled)
+	if (etd->info.crc_enabled)
 		sanity_check = ((packet[3] & 0x08) == 0x00);
-	else if (ic_version == 7 && etd->samples[1] == 0x2A)
+	else if (ic_version == 7 && etd->info.samples[1] == 0x2A)
 		sanity_check = ((packet[3] & 0x1c) == 0x10);
 	else
 		sanity_check = ((packet[0] & 0x0c) == 0x04 &&
@@ -827,12 +830,12 @@ static psmouse_ret_t elantech_process_byte(struct psmouse *psmouse)
 	if (psmouse->pktcnt < psmouse->pktsize)
 		return PSMOUSE_GOOD_DATA;
 
-	if (etd->debug > 1)
+	if (etd->info.debug > 1)
 		elantech_packet_dump(psmouse);
 
-	switch (etd->hw_version) {
+	switch (etd->info.hw_version) {
 	case 1:
-		if (etd->paritycheck && !elantech_packet_check_v1(psmouse))
+		if (etd->info.paritycheck && !elantech_packet_check_v1(psmouse))
 			return PSMOUSE_BAD_DATA;
 
 		elantech_report_absolute_v1(psmouse);
@@ -843,7 +846,7 @@ static psmouse_ret_t elantech_process_byte(struct psmouse *psmouse)
 		if (elantech_debounce_check_v2(psmouse))
 			return PSMOUSE_FULL_PACKET;
 
-		if (etd->paritycheck && !elantech_packet_check_v2(psmouse))
+		if (etd->info.paritycheck && !elantech_packet_check_v2(psmouse))
 			return PSMOUSE_BAD_DATA;
 
 		elantech_report_absolute_v2(psmouse);
@@ -916,7 +919,7 @@ static int elantech_set_absolute_mode(struct psmouse *psmouse)
 	int tries = ETP_READ_BACK_TRIES;
 	int rc = 0;
 
-	switch (etd->hw_version) {
+	switch (etd->info.hw_version) {
 	case 1:
 		etd->reg_10 = 0x16;
 		etd->reg_11 = 0x8f;
@@ -939,7 +942,7 @@ static int elantech_set_absolute_mode(struct psmouse *psmouse)
 		break;
 
 	case 3:
-		if (etd->set_hw_resolution)
+		if (etd->info.set_hw_resolution)
 			etd->reg_10 = 0x0b;
 		else
 			etd->reg_10 = 0x01;
@@ -976,7 +979,7 @@ static int elantech_set_absolute_mode(struct psmouse *psmouse)
 		if (rc) {
 			psmouse_err(psmouse,
 				    "failed to read back register 0x10.\n");
-		} else if (etd->hw_version == 1 &&
+		} else if (etd->info.hw_version == 1 &&
 			   !(val & ETP_R10_ABSOLUTE_MODE)) {
 			psmouse_err(psmouse,
 				    "touchpad refuses to switch to absolute mode.\n");
@@ -997,10 +1000,11 @@ static int elantech_set_range(struct psmouse *psmouse,
 			      unsigned int *width)
 {
 	struct elantech_data *etd = psmouse->private;
+	struct elantech_device_info *info = &etd->info;
 	unsigned char param[3];
 	unsigned char traces;
 
-	switch (etd->hw_version) {
+	switch (info->hw_version) {
 	case 1:
 		*x_min = ETP_XMIN_V1;
 		*y_min = ETP_YMIN_V1;
@@ -1009,9 +1013,9 @@ static int elantech_set_range(struct psmouse *psmouse,
 		break;
 
 	case 2:
-		if (etd->fw_version == 0x020800 ||
-		    etd->fw_version == 0x020b00 ||
-		    etd->fw_version == 0x020030) {
+		if (info->fw_version == 0x020800 ||
+		    info->fw_version == 0x020b00 ||
+		    info->fw_version == 0x020030) {
 			*x_min = ETP_XMIN_V2;
 			*y_min = ETP_YMIN_V2;
 			*x_max = ETP_XMAX_V2;
@@ -1020,35 +1024,35 @@ static int elantech_set_range(struct psmouse *psmouse,
 			int i;
 			int fixed_dpi;
 
-			i = (etd->fw_version > 0x020800 &&
-			     etd->fw_version < 0x020900) ? 1 : 2;
+			i = (info->fw_version > 0x020800 &&
+			     info->fw_version < 0x020900) ? 1 : 2;
 
-			if (etd->send_cmd(psmouse, ETP_FW_ID_QUERY, param))
+			if (info->send_cmd(psmouse, ETP_FW_ID_QUERY, param))
 				return -1;
 
 			fixed_dpi = param[1] & 0x10;
 
-			if (((etd->fw_version >> 16) == 0x14) && fixed_dpi) {
-				if (etd->send_cmd(psmouse, ETP_SAMPLE_QUERY, param))
+			if (((info->fw_version >> 16) == 0x14) && fixed_dpi) {
+				if (info->send_cmd(psmouse, ETP_SAMPLE_QUERY, param))
 					return -1;
 
-				*x_max = (etd->capabilities[1] - i) * param[1] / 2;
-				*y_max = (etd->capabilities[2] - i) * param[2] / 2;
-			} else if (etd->fw_version == 0x040216) {
+				*x_max = (info->capabilities[1] - i) * param[1] / 2;
+				*y_max = (info->capabilities[2] - i) * param[2] / 2;
+			} else if (info->fw_version == 0x040216) {
 				*x_max = 819;
 				*y_max = 405;
-			} else if (etd->fw_version == 0x040219 || etd->fw_version == 0x040215) {
+			} else if (info->fw_version == 0x040219 || info->fw_version == 0x040215) {
 				*x_max = 900;
 				*y_max = 500;
 			} else {
-				*x_max = (etd->capabilities[1] - i) * 64;
-				*y_max = (etd->capabilities[2] - i) * 64;
+				*x_max = (info->capabilities[1] - i) * 64;
+				*y_max = (info->capabilities[2] - i) * 64;
 			}
 		}
 		break;
 
 	case 3:
-		if (etd->send_cmd(psmouse, ETP_FW_ID_QUERY, param))
+		if (info->send_cmd(psmouse, ETP_FW_ID_QUERY, param))
 			return -1;
 
 		*x_max = (0x0f & param[0]) << 8 | param[1];
@@ -1056,12 +1060,12 @@ static int elantech_set_range(struct psmouse *psmouse,
 		break;
 
 	case 4:
-		if (etd->send_cmd(psmouse, ETP_FW_ID_QUERY, param))
+		if (info->send_cmd(psmouse, ETP_FW_ID_QUERY, param))
 			return -1;
 
 		*x_max = (0x0f & param[0]) << 8 | param[1];
 		*y_max = (0xf0 & param[0]) << 4 | param[2];
-		traces = etd->capabilities[1];
+		traces = info->capabilities[1];
 		if ((traces < 2) || (traces > *x_max))
 			return -1;
 
@@ -1083,7 +1087,8 @@ static unsigned int elantech_convert_res(unsigned int val)
 
 static int elantech_get_resolution_v4(struct psmouse *psmouse,
 				      unsigned int *x_res,
-				      unsigned int *y_res)
+				      unsigned int *y_res,
+				      unsigned int *bus)
 {
 	unsigned char param[3];
 
@@ -1092,6 +1097,7 @@ static int elantech_get_resolution_v4(struct psmouse *psmouse,
 
 	*x_res = elantech_convert_res(param[1] & 0x0f);
 	*y_res = elantech_convert_res((param[1] & 0xf0) >> 4);
+	*bus = param[2];
 
 	return 0;
 }
@@ -1140,7 +1146,7 @@ static void elantech_set_buttonpad_prop(struct psmouse *psmouse)
 	struct input_dev *dev = psmouse->dev;
 	struct elantech_data *etd = psmouse->private;
 
-	if (etd->fw_version & 0x001000) {
+	if (etd->info.fw_version & 0x001000) {
 		__set_bit(INPUT_PROP_BUTTONPAD, dev->propbit);
 		__clear_bit(BTN_RIGHT, dev->keybit);
 	}
@@ -1176,8 +1182,8 @@ static int elantech_set_input_params(struct psmouse *psmouse)
 {
 	struct input_dev *dev = psmouse->dev;
 	struct elantech_data *etd = psmouse->private;
+	struct elantech_device_info *info = &etd->info;
 	unsigned int x_min = 0, y_min = 0, x_max = 0, y_max = 0, width = 0;
-	unsigned int x_res = 31, y_res = 31;
 
 	if (elantech_set_range(psmouse, &x_min, &y_min, &x_max, &y_max, &width))
 		return -1;
@@ -1197,11 +1203,11 @@ static int elantech_set_input_params(struct psmouse *psmouse)
 	__set_bit(BTN_TOOL_DOUBLETAP, dev->keybit);
 	__set_bit(BTN_TOOL_TRIPLETAP, dev->keybit);
 
-	switch (etd->hw_version) {
+	switch (info->hw_version) {
 	case 1:
 		/* Rocker button */
-		if (etd->fw_version < 0x020000 &&
-		    (etd->capabilities[0] & ETP_CAP_HAS_ROCKER)) {
+		if (info->fw_version < 0x020000 &&
+		    (info->capabilities[0] & ETP_CAP_HAS_ROCKER)) {
 			__set_bit(BTN_FORWARD, dev->keybit);
 			__set_bit(BTN_BACK, dev->keybit);
 		}
@@ -1214,11 +1220,11 @@ static int elantech_set_input_params(struct psmouse *psmouse)
 		__set_bit(INPUT_PROP_SEMI_MT, dev->propbit);
 		/* fall through */
 	case 3:
-		if (etd->hw_version == 3)
+		if (info->hw_version == 3)
 			elantech_set_buttonpad_prop(psmouse);
 		input_set_abs_params(dev, ABS_X, x_min, x_max, 0, 0);
 		input_set_abs_params(dev, ABS_Y, y_min, y_max, 0, 0);
-		if (etd->reports_pressure) {
+		if (info->reports_pressure) {
 			input_set_abs_params(dev, ABS_PRESSURE, ETP_PMIN_V2,
 					     ETP_PMAX_V2, 0, 0);
 			input_set_abs_params(dev, ABS_TOOL_WIDTH, ETP_WMIN_V2,
@@ -1230,13 +1236,6 @@ static int elantech_set_input_params(struct psmouse *psmouse)
 		break;
 
 	case 4:
-		if (elantech_get_resolution_v4(psmouse, &x_res, &y_res)) {
-			/*
-			 * if query failed, print a warning and leave the values
-			 * zero to resemble synaptics.c behavior.
-			 */
-			psmouse_warn(psmouse, "couldn't query resolution data.\n");
-		}
 		elantech_set_buttonpad_prop(psmouse);
 		__set_bit(BTN_TOOL_QUADTAP, dev->keybit);
 		/* For X to recognize me as touchpad. */
@@ -1265,11 +1264,11 @@ static int elantech_set_input_params(struct psmouse *psmouse)
 		break;
 	}
 
-	input_abs_set_res(dev, ABS_X, x_res);
-	input_abs_set_res(dev, ABS_Y, y_res);
-	if (etd->hw_version > 1) {
-		input_abs_set_res(dev, ABS_MT_POSITION_X, x_res);
-		input_abs_set_res(dev, ABS_MT_POSITION_Y, y_res);
+	input_abs_set_res(dev, ABS_X, info->x_res);
+	input_abs_set_res(dev, ABS_Y, info->y_res);
+	if (info->hw_version > 1) {
+		input_abs_set_res(dev, ABS_MT_POSITION_X, info->x_res);
+		input_abs_set_res(dev, ABS_MT_POSITION_Y, info->y_res);
 	}
 
 	etd->y_max = y_max;
@@ -1317,7 +1316,7 @@ static ssize_t elantech_set_int_attr(struct psmouse *psmouse,
 		return err;
 
 	/* Do we need to preserve some bits for version 2 hardware too? */
-	if (etd->hw_version == 1) {
+	if (etd->info.hw_version == 1) {
 		if (attr->reg == 0x10)
 			/* Force absolute mode always on */
 			value |= ETP_R10_ABSOLUTE_MODE;
@@ -1337,11 +1336,22 @@ static ssize_t elantech_set_int_attr(struct psmouse *psmouse,
 		.field_offset = offsetof(struct elantech_data, _name),	\
 		.reg = _register,					\
 	};								\
-	PSMOUSE_DEFINE_ATTR(_name, S_IWUSR | S_IRUGO,			\
+	PSMOUSE_DEFINE_ATTR(_name, 0644,				\
 			    &elantech_attr_##_name,			\
 			    elantech_show_int_attr,			\
 			    elantech_set_int_attr)
 
+#define ELANTECH_INFO_ATTR(_name)					       \
+	static struct elantech_attr_data elantech_attr_##_name = {	       \
+		.field_offset = offsetof(struct elantech_data, info) +	       \
+				offsetof(struct elantech_device_info, _name),  \
+		.reg = 0,						       \
+	};								       \
+	PSMOUSE_DEFINE_ATTR(_name, 0644,				       \
+			    &elantech_attr_##_name,			       \
+			    elantech_show_int_attr,			       \
+			    elantech_set_int_attr)
+
 ELANTECH_INT_ATTR(reg_07, 0x07);
 ELANTECH_INT_ATTR(reg_10, 0x10);
 ELANTECH_INT_ATTR(reg_11, 0x11);
@@ -1352,9 +1362,9 @@ ELANTECH_INT_ATTR(reg_23, 0x23);
 ELANTECH_INT_ATTR(reg_24, 0x24);
 ELANTECH_INT_ATTR(reg_25, 0x25);
 ELANTECH_INT_ATTR(reg_26, 0x26);
-ELANTECH_INT_ATTR(debug, 0);
-ELANTECH_INT_ATTR(paritycheck, 0);
-ELANTECH_INT_ATTR(crc_enabled, 0);
+ELANTECH_INFO_ATTR(debug);
+ELANTECH_INFO_ATTR(paritycheck);
+ELANTECH_INFO_ATTR(crc_enabled);
 
 static struct attribute *elantech_attrs[] = {
 	&psmouse_attr_reg_07.dattr.attr,
@@ -1469,6 +1479,12 @@ static void elantech_disconnect(struct psmouse *psmouse)
 {
 	struct elantech_data *etd = psmouse->private;
 
+	/*
+	 * We might have left a breadcrumb when trying to
+	 * set up SMbus companion.
+	 */
+	psmouse_smbus_cleanup(psmouse);
+
 	if (etd->tp_dev)
 		input_unregister_device(etd->tp_dev);
 	sysfs_remove_group(&psmouse->ps2dev.serio->dev.kobj,
@@ -1588,25 +1604,25 @@ static const struct dmi_system_id no_hw_res_dmi_table[] = {
 /*
  * determine hardware version and set some properties according to it.
  */
-static int elantech_set_properties(struct elantech_data *etd)
+static int elantech_set_properties(struct elantech_device_info *info)
 {
 	/* This represents the version of IC body. */
-	int ver = (etd->fw_version & 0x0f0000) >> 16;
+	int ver = (info->fw_version & 0x0f0000) >> 16;
 
 	/* Early version of Elan touchpads doesn't obey the rule. */
-	if (etd->fw_version < 0x020030 || etd->fw_version == 0x020600)
-		etd->hw_version = 1;
+	if (info->fw_version < 0x020030 || info->fw_version == 0x020600)
+		info->hw_version = 1;
 	else {
 		switch (ver) {
 		case 2:
 		case 4:
-			etd->hw_version = 2;
+			info->hw_version = 2;
 			break;
 		case 5:
-			etd->hw_version = 3;
+			info->hw_version = 3;
 			break;
 		case 6 ... 15:
-			etd->hw_version = 4;
+			info->hw_version = 4;
 			break;
 		default:
 			return -1;
@@ -1614,100 +1630,88 @@ static int elantech_set_properties(struct elantech_data *etd)
 	}
 
 	/* decide which send_cmd we're gonna use early */
-	etd->send_cmd = etd->hw_version >= 3 ? elantech_send_cmd :
-					       synaptics_send_cmd;
+	info->send_cmd = info->hw_version >= 3 ? elantech_send_cmd :
+						 synaptics_send_cmd;
 
 	/* Turn on packet checking by default */
-	etd->paritycheck = 1;
+	info->paritycheck = 1;
 
 	/*
 	 * This firmware suffers from misreporting coordinates when
 	 * a touch action starts causing the mouse cursor or scrolled page
 	 * to jump. Enable a workaround.
 	 */
-	etd->jumpy_cursor =
-		(etd->fw_version == 0x020022 || etd->fw_version == 0x020600);
+	info->jumpy_cursor =
+		(info->fw_version == 0x020022 || info->fw_version == 0x020600);
 
-	if (etd->hw_version > 1) {
+	if (info->hw_version > 1) {
 		/* For now show extra debug information */
-		etd->debug = 1;
+		info->debug = 1;
 
-		if (etd->fw_version >= 0x020800)
-			etd->reports_pressure = true;
+		if (info->fw_version >= 0x020800)
+			info->reports_pressure = true;
 	}
 
 	/*
 	 * The signatures of v3 and v4 packets change depending on the
 	 * value of this hardware flag.
 	 */
-	etd->crc_enabled = (etd->fw_version & 0x4000) == 0x4000 ||
-			   dmi_check_system(elantech_dmi_force_crc_enabled);
+	info->crc_enabled = (info->fw_version & 0x4000) == 0x4000 ||
+			     dmi_check_system(elantech_dmi_force_crc_enabled);
 
 	/* Enable real hardware resolution on hw_version 3 ? */
-	etd->set_hw_resolution = !dmi_check_system(no_hw_res_dmi_table);
+	info->set_hw_resolution = !dmi_check_system(no_hw_res_dmi_table);
 
 	return 0;
 }
 
-/*
- * Initialize the touchpad and create sysfs entries
- */
-int elantech_init(struct psmouse *psmouse)
+static int elantech_query_info(struct psmouse *psmouse,
+			       struct elantech_device_info *info)
 {
-	struct elantech_data *etd;
-	int i;
-	int error = -EINVAL;
 	unsigned char param[3];
-	struct input_dev *tp_dev;
 
-	psmouse->private = etd = kzalloc(sizeof(struct elantech_data), GFP_KERNEL);
-	if (!etd)
-		return -ENOMEM;
-
-	psmouse_reset(psmouse);
-
-	etd->parity[0] = 1;
-	for (i = 1; i < 256; i++)
-		etd->parity[i] = etd->parity[i & (i - 1)] ^ 1;
+	memset(info, 0, sizeof(*info));
 
 	/*
 	 * Do the version query again so we can store the result
 	 */
 	if (synaptics_send_cmd(psmouse, ETP_FW_VERSION_QUERY, param)) {
 		psmouse_err(psmouse, "failed to query firmware version.\n");
-		goto init_fail;
+		return -EINVAL;
 	}
-	etd->fw_version = (param[0] << 16) | (param[1] << 8) | param[2];
+	info->fw_version = (param[0] << 16) | (param[1] << 8) | param[2];
 
-	if (elantech_set_properties(etd)) {
+	if (elantech_set_properties(info)) {
 		psmouse_err(psmouse, "unknown hardware version, aborting...\n");
-		goto init_fail;
+		return -EINVAL;
 	}
 	psmouse_info(psmouse,
 		     "assuming hardware version %d (with firmware version 0x%02x%02x%02x)\n",
-		     etd->hw_version, param[0], param[1], param[2]);
+		     info->hw_version, param[0], param[1], param[2]);
 
-	if (etd->send_cmd(psmouse, ETP_CAPABILITIES_QUERY,
-	    etd->capabilities)) {
+	if (info->send_cmd(psmouse, ETP_CAPABILITIES_QUERY,
+	    info->capabilities)) {
 		psmouse_err(psmouse, "failed to query capabilities.\n");
-		goto init_fail;
+		return -EINVAL;
 	}
 	psmouse_info(psmouse,
 		     "Synaptics capabilities query result 0x%02x, 0x%02x, 0x%02x.\n",
-		     etd->capabilities[0], etd->capabilities[1],
-		     etd->capabilities[2]);
+		     info->capabilities[0], info->capabilities[1],
+		     info->capabilities[2]);
 
-	if (etd->hw_version != 1) {
-		if (etd->send_cmd(psmouse, ETP_SAMPLE_QUERY, etd->samples)) {
+	if (info->hw_version != 1) {
+		if (info->send_cmd(psmouse, ETP_SAMPLE_QUERY, info->samples)) {
 			psmouse_err(psmouse, "failed to query sample data\n");
-			goto init_fail;
+			return -EINVAL;
 		}
 		psmouse_info(psmouse,
 			     "Elan sample query result %02x, %02x, %02x\n",
-			     etd->samples[0], etd->samples[1], etd->samples[2]);
+			     info->samples[0],
+			     info->samples[1],
+			     info->samples[2]);
 	}
 
-	if (etd->samples[1] == 0x74 && etd->hw_version == 0x03) {
+	if (info->samples[1] == 0x74 && info->hw_version == 0x03) {
 		/*
 		 * This module has a bug which makes absolute mode
 		 * unusable, so let's abort so we'll be using standard
@@ -1715,16 +1719,181 @@ int elantech_init(struct psmouse *psmouse)
 		 */
 		psmouse_info(psmouse,
 			     "absolute mode broken, forcing standard PS/2 protocol\n");
+		return -ENODEV;
+	}
+
+	/* The MSB indicates the presence of the trackpoint */
+	info->has_trackpoint = (info->capabilities[0] & 0x80) == 0x80;
+
+	info->x_res = 31;
+	info->y_res = 31;
+	if (info->hw_version == 4) {
+		if (elantech_get_resolution_v4(psmouse,
+					       &info->x_res,
+					       &info->y_res,
+					       &info->bus)) {
+			psmouse_warn(psmouse,
+				     "failed to query resolution data.\n");
+		}
+	}
+
+	return 0;
+}
+
+#if defined(CONFIG_MOUSE_PS2_ELANTECH_SMBUS)
+
+/*
+ * The newest Elantech device can use a secondary bus (over SMBus) which
+ * provides a better bandwidth and allow a better control of the touchpads.
+ * This is used to decide if we need to use this bus or not.
+ */
+enum {
+	ELANTECH_SMBUS_NOT_SET = -1,
+	ELANTECH_SMBUS_OFF,
+	ELANTECH_SMBUS_ON,
+};
+
+static int elantech_smbus = IS_ENABLED(CONFIG_MOUSE_ELAN_I2C_SMBUS) ?
+		ELANTECH_SMBUS_NOT_SET : ELANTECH_SMBUS_OFF;
+module_param_named(elantech_smbus, elantech_smbus, int, 0644);
+MODULE_PARM_DESC(elantech_smbus, "Use a secondary bus for the Elantech device.");
+
+static int elantech_create_smbus(struct psmouse *psmouse,
+				 struct elantech_device_info *info,
+				 bool leave_breadcrumbs)
+{
+	const struct property_entry i2c_properties[] = {
+		PROPERTY_ENTRY_BOOL("elan,trackpoint"),
+		{ },
+	};
+	struct i2c_board_info smbus_board = {
+		I2C_BOARD_INFO("elan_i2c", 0x15),
+		.flags = I2C_CLIENT_HOST_NOTIFY,
+	};
+
+	if (info->has_trackpoint)
+		smbus_board.properties = i2c_properties;
+
+	return psmouse_smbus_init(psmouse, &smbus_board, NULL, 0, false,
+				  leave_breadcrumbs);
+}
+
+/**
+ * elantech_setup_smbus - called once the PS/2 devices are enumerated
+ * and decides to instantiate a SMBus InterTouch device.
+ */
+static int elantech_setup_smbus(struct psmouse *psmouse,
+				struct elantech_device_info *info,
+				bool leave_breadcrumbs)
+{
+	int error;
+
+	if (elantech_smbus == ELANTECH_SMBUS_OFF)
+		return -ENXIO;
+
+	if (elantech_smbus == ELANTECH_SMBUS_NOT_SET) {
+		/*
+		 * New ICs are enabled by default.
+		 * Old ICs are up to the user to decide.
+		 */
+		if (!ETP_NEW_IC_SMBUS_HOST_NOTIFY(info->fw_version))
+			return -ENXIO;
+	}
+
+	psmouse_info(psmouse, "Trying to set up SMBus access\n");
+
+	error = elantech_create_smbus(psmouse, info, leave_breadcrumbs);
+	if (error) {
+		if (error == -EAGAIN)
+			psmouse_info(psmouse, "SMbus companion is not ready yet\n");
+		else
+			psmouse_err(psmouse, "unable to create intertouch device\n");
+
+		return error;
+	}
+
+	return 0;
+}
+
+static bool elantech_use_host_notify(struct psmouse *psmouse,
+				     struct elantech_device_info *info)
+{
+	if (ETP_NEW_IC_SMBUS_HOST_NOTIFY(info->fw_version))
+		return true;
+
+	switch (info->bus) {
+	case ETP_BUS_PS2_ONLY:
+		/* expected case */
+		break;
+	case ETP_BUS_SMB_ALERT_ONLY:
+		/* fall-through  */
+	case ETP_BUS_PS2_SMB_ALERT:
+		psmouse_dbg(psmouse, "Ignoring SMBus provider through alert protocol.\n");
+		break;
+	case ETP_BUS_SMB_HST_NTFY_ONLY:
+		/* fall-through  */
+	case ETP_BUS_PS2_SMB_HST_NTFY:
+		return true;
+	default:
+		psmouse_dbg(psmouse,
+			    "Ignoring SMBus bus provider %d.\n",
+			    info->bus);
+	}
+
+	return false;
+}
+
+int elantech_init_smbus(struct psmouse *psmouse)
+{
+	struct elantech_device_info info;
+	int error = -EINVAL;
+
+	psmouse_reset(psmouse);
+
+	error = elantech_query_info(psmouse, &info);
+	if (error)
+		goto init_fail;
+
+	if (info.hw_version < 4) {
+		error = -ENXIO;
 		goto init_fail;
 	}
 
+	return elantech_create_smbus(psmouse, &info, false);
+ init_fail:
+	psmouse_reset(psmouse);
+	return error;
+}
+#endif /* CONFIG_MOUSE_PS2_ELANTECH_SMBUS */
+
+/*
+ * Initialize the touchpad and create sysfs entries
+ */
+static int elantech_setup_ps2(struct psmouse *psmouse,
+			      struct elantech_device_info *info)
+{
+	struct elantech_data *etd;
+	int i;
+	int error = -EINVAL;
+	struct input_dev *tp_dev;
+
+	psmouse->private = etd = kzalloc(sizeof(*etd), GFP_KERNEL);
+	if (!etd)
+		return -ENOMEM;
+
+	etd->info = *info;
+
+	etd->parity[0] = 1;
+	for (i = 1; i < 256; i++)
+		etd->parity[i] = etd->parity[i & (i - 1)] ^ 1;
+
 	if (elantech_set_absolute_mode(psmouse)) {
 		psmouse_err(psmouse,
 			    "failed to put touchpad into absolute mode.\n");
 		goto init_fail;
 	}
 
-	if (etd->fw_version == 0x381f17) {
+	if (info->fw_version == 0x381f17) {
 		etd->original_set_rate = psmouse->set_rate;
 		psmouse->set_rate = elantech_set_rate_restore_reg_07;
 	}
@@ -1743,8 +1912,7 @@ int elantech_init(struct psmouse *psmouse)
 		goto init_fail;
 	}
 
-	/* The MSB indicates the presence of the trackpoint */
-	if ((etd->capabilities[0] & 0x80) == 0x80) {
+	if (info->has_trackpoint) {
 		tp_dev = input_allocate_device();
 
 		if (!tp_dev) {
@@ -1780,7 +1948,7 @@ int elantech_init(struct psmouse *psmouse)
 	psmouse->protocol_handler = elantech_process_byte;
 	psmouse->disconnect = elantech_disconnect;
 	psmouse->reconnect = elantech_reconnect;
-	psmouse->pktsize = etd->hw_version > 1 ? 6 : 4;
+	psmouse->pktsize = info->hw_version > 1 ? 6 : 4;
 
 	return 0;
  init_fail_tp_reg:
@@ -1789,7 +1957,70 @@ int elantech_init(struct psmouse *psmouse)
 	sysfs_remove_group(&psmouse->ps2dev.serio->dev.kobj,
 			   &elantech_attr_group);
  init_fail:
-	psmouse_reset(psmouse);
 	kfree(etd);
 	return error;
 }
+
+int elantech_init_ps2(struct psmouse *psmouse)
+{
+	struct elantech_device_info info;
+	int error = -EINVAL;
+
+	psmouse_reset(psmouse);
+
+	error = elantech_query_info(psmouse, &info);
+	if (error)
+		goto init_fail;
+
+	error = elantech_setup_ps2(psmouse, &info);
+	if (error)
+		goto init_fail;
+
+	return 0;
+ init_fail:
+	psmouse_reset(psmouse);
+	return error;
+}
+
+int elantech_init(struct psmouse *psmouse)
+{
+	struct elantech_device_info info;
+	int error = -EINVAL;
+
+	psmouse_reset(psmouse);
+
+	error = elantech_query_info(psmouse, &info);
+	if (error)
+		goto init_fail;
+
+#if defined(CONFIG_MOUSE_PS2_ELANTECH_SMBUS)
+
+	if (elantech_use_host_notify(psmouse, &info)) {
+		if (!IS_ENABLED(CONFIG_MOUSE_ELAN_I2C_SMBUS) ||
+		    !IS_ENABLED(CONFIG_MOUSE_PS2_ELANTECH_SMBUS)) {
+			psmouse_warn(psmouse,
+				     "The touchpad can support a better bus than the too old PS/2 protocol. "
+				     "Make sure MOUSE_PS2_ELANTECH_SMBUS and MOUSE_ELAN_I2C_SMBUS are enabled to get a better touchpad experience.\n");
+		}
+		error = elantech_setup_smbus(psmouse, &info, true);
+		if (!error)
+			return PSMOUSE_ELANTECH_SMBUS;
+	}
+
+#endif /* CONFIG_MOUSE_PS2_ELANTECH_SMBUS */
+
+	error = elantech_setup_ps2(psmouse, &info);
+	if (error < 0) {
+		/*
+		 * Not using any flavor of Elantech support, so clean up
+		 * SMbus breadcrumbs, if any.
+		 */
+		psmouse_smbus_cleanup(psmouse);
+		goto init_fail;
+	}
+
+	return PSMOUSE_ELANTECH;
+ init_fail:
+	psmouse_reset(psmouse);
+	return error;
+}
diff --git a/drivers/input/mouse/elantech.h b/drivers/input/mouse/elantech.h
index e1cbf409d9c8..119727085a60 100644
--- a/drivers/input/mouse/elantech.h
+++ b/drivers/input/mouse/elantech.h
@@ -106,6 +106,30 @@
  */
 #define ETP_WEIGHT_VALUE		5
 
+/*
+ * Bus information on 3rd byte of query ETP_RESOLUTION_QUERY(0x04)
+ */
+#define ETP_BUS_PS2_ONLY		0
+#define ETP_BUS_SMB_ALERT_ONLY		1
+#define ETP_BUS_SMB_HST_NTFY_ONLY	2
+#define ETP_BUS_PS2_SMB_ALERT		3
+#define ETP_BUS_PS2_SMB_HST_NTFY	4
+
+/*
+ * New ICs are either using SMBus Host Notify or just plain PS2.
+ *
+ * ETP_FW_VERSION_QUERY is:
+ * Byte 1:
+ *  - bit 0..3: IC BODY
+ * Byte 2:
+ *  - bit 4: HiddenButton
+ *  - bit 5: PS2_SMBUS_NOTIFY
+ *  - bit 6: PS2CRCCheck
+ */
+#define ETP_NEW_IC_SMBUS_HOST_NOTIFY(fw_version)	\
+		((((fw_version) & 0x0f2000) == 0x0f2000) && \
+		 ((fw_version) & 0x0000ff) > 0)
+
 /*
  * The base position for one finger, v4 hardware
  */
@@ -114,6 +138,25 @@ struct finger_pos {
 	unsigned int y;
 };
 
+struct elantech_device_info {
+	unsigned char capabilities[3];
+	unsigned char samples[3];
+	unsigned char debug;
+	unsigned char hw_version;
+	unsigned int fw_version;
+	unsigned int x_res;
+	unsigned int y_res;
+	unsigned int bus;
+	bool paritycheck;
+	bool jumpy_cursor;
+	bool reports_pressure;
+	bool crc_enabled;
+	bool set_hw_resolution;
+	bool has_trackpoint;
+	int (*send_cmd)(struct psmouse *psmouse, unsigned char c,
+			unsigned char *param);
+};
+
 struct elantech_data {
 	struct input_dev *tp_dev;	/* Relative device for trackpoint */
 	char tp_phys[32];
@@ -127,27 +170,18 @@ struct elantech_data {
 	unsigned char reg_24;
 	unsigned char reg_25;
 	unsigned char reg_26;
-	unsigned char debug;
-	unsigned char capabilities[3];
-	unsigned char samples[3];
-	bool paritycheck;
-	bool jumpy_cursor;
-	bool reports_pressure;
-	bool crc_enabled;
-	bool set_hw_resolution;
-	unsigned char hw_version;
-	unsigned int fw_version;
 	unsigned int single_finger_reports;
 	unsigned int y_max;
 	unsigned int width;
 	struct finger_pos mt[ETP_MAX_FINGERS];
 	unsigned char parity[256];
-	int (*send_cmd)(struct psmouse *psmouse, unsigned char c, unsigned char *param);
+	struct elantech_device_info info;
 	void (*original_set_rate)(struct psmouse *psmouse, unsigned int rate);
 };
 
 #ifdef CONFIG_MOUSE_PS2_ELANTECH
 int elantech_detect(struct psmouse *psmouse, bool set_properties);
+int elantech_init_ps2(struct psmouse *psmouse);
 int elantech_init(struct psmouse *psmouse);
 #else
 static inline int elantech_detect(struct psmouse *psmouse, bool set_properties)
@@ -158,6 +192,19 @@ static inline int elantech_init(struct psmouse *psmouse)
 {
 	return -ENOSYS;
 }
+static inline int elantech_init_ps2(struct psmouse *psmouse)
+{
+	return -ENOSYS;
+}
 #endif /* CONFIG_MOUSE_PS2_ELANTECH */
 
+#if defined(CONFIG_MOUSE_PS2_ELANTECH_SMBUS)
+int elantech_init_smbus(struct psmouse *psmouse);
+#else
+static inline int elantech_init_smbus(struct psmouse *psmouse)
+{
+	return -ENOSYS;
+}
+#endif /* CONFIG_MOUSE_PS2_ELANTECH_SMBUS */
+
 #endif
diff --git a/drivers/input/mouse/psmouse-base.c b/drivers/input/mouse/psmouse-base.c
index 8900c3166ebf..5ff5b1952be0 100644
--- a/drivers/input/mouse/psmouse-base.c
+++ b/drivers/input/mouse/psmouse-base.c
@@ -856,7 +856,17 @@ static const struct psmouse_protocol psmouse_protocols[] = {
 		.name		= "ETPS/2",
 		.alias		= "elantech",
 		.detect		= elantech_detect,
-		.init		= elantech_init,
+		.init		= elantech_init_ps2,
+	},
+#endif
+#ifdef CONFIG_MOUSE_PS2_ELANTECH_SMBUS
+	{
+		.type		= PSMOUSE_ELANTECH_SMBUS,
+		.name		= "ETSMBus",
+		.alias		= "elantech-smbus",
+		.detect		= elantech_detect,
+		.init		= elantech_init_smbus,
+		.smbus_companion = true,
 	},
 #endif
 #ifdef CONFIG_MOUSE_PS2_SENTELIC
@@ -1158,8 +1168,13 @@ static int psmouse_extensions(struct psmouse *psmouse,
 	/* Try Elantech touchpad */
 	if (max_proto > PSMOUSE_IMEX &&
 	    psmouse_try_protocol(psmouse, PSMOUSE_ELANTECH,
-				 &max_proto, set_properties, true)) {
-		return PSMOUSE_ELANTECH;
+				 &max_proto, set_properties, false)) {
+		if (!set_properties)
+			return PSMOUSE_ELANTECH;
+
+		ret = elantech_init(psmouse);
+		if (ret >= 0)
+			return ret;
 	}
 
 	if (max_proto > PSMOUSE_IMEX) {
diff --git a/drivers/input/mouse/psmouse-smbus.c b/drivers/input/mouse/psmouse-smbus.c
index c7ac24d119c1..852d4b486ddb 100644
--- a/drivers/input/mouse/psmouse-smbus.c
+++ b/drivers/input/mouse/psmouse-smbus.c
@@ -23,6 +23,7 @@ struct psmouse_smbus_dev {
 	struct i2c_client *client;
 	struct list_head node;
 	bool dead;
+	bool need_deactivate;
 };
 
 static LIST_HEAD(psmouse_smbus_list);
@@ -118,7 +119,10 @@ static psmouse_ret_t psmouse_smbus_process_byte(struct psmouse *psmouse)
 
 static int psmouse_smbus_reconnect(struct psmouse *psmouse)
 {
-	psmouse_deactivate(psmouse);
+	struct psmouse_smbus_dev *smbdev = psmouse->private;
+
+	if (smbdev->need_deactivate)
+		psmouse_deactivate(psmouse);
 
 	return 0;
 }
@@ -225,6 +229,7 @@ void psmouse_smbus_cleanup(struct psmouse *psmouse)
 int psmouse_smbus_init(struct psmouse *psmouse,
 		       const struct i2c_board_info *board,
 		       const void *pdata, size_t pdata_size,
+		       bool need_deactivate,
 		       bool leave_breadcrumbs)
 {
 	struct psmouse_smbus_dev *smbdev;
@@ -236,13 +241,20 @@ int psmouse_smbus_init(struct psmouse *psmouse,
 
 	smbdev->psmouse = psmouse;
 	smbdev->board = *board;
+	smbdev->need_deactivate = need_deactivate;
 
-	smbdev->board.platform_data = kmemdup(pdata, pdata_size, GFP_KERNEL);
-	if (!smbdev->board.platform_data) {
-		kfree(smbdev);
-		return -ENOMEM;
+	if (pdata) {
+		smbdev->board.platform_data = kmemdup(pdata, pdata_size,
+						      GFP_KERNEL);
+		if (!smbdev->board.platform_data) {
+			kfree(smbdev);
+			return -ENOMEM;
+		}
 	}
 
+	if (need_deactivate)
+		psmouse_deactivate(psmouse);
+
 	psmouse->private = smbdev;
 	psmouse->protocol_handler = psmouse_smbus_process_byte;
 	psmouse->reconnect = psmouse_smbus_reconnect;
@@ -250,8 +262,6 @@ int psmouse_smbus_init(struct psmouse *psmouse,
 	psmouse->disconnect = psmouse_smbus_disconnect;
 	psmouse->resync_time = 0;
 
-	psmouse_deactivate(psmouse);
-
 	mutex_lock(&psmouse_smbus_mutex);
 	list_add_tail(&smbdev->node, &psmouse_smbus_list);
 	mutex_unlock(&psmouse_smbus_mutex);
diff --git a/drivers/input/mouse/psmouse.h b/drivers/input/mouse/psmouse.h
index 71ac50082c8b..64c3a5d3fb3e 100644
--- a/drivers/input/mouse/psmouse.h
+++ b/drivers/input/mouse/psmouse.h
@@ -68,6 +68,7 @@ enum psmouse_type {
 	PSMOUSE_VMMOUSE,
 	PSMOUSE_BYD,
 	PSMOUSE_SYNAPTICS_SMBUS,
+	PSMOUSE_ELANTECH_SMBUS,
 	PSMOUSE_AUTO		/* This one should always be last */
 };
 
@@ -224,6 +225,7 @@ struct i2c_board_info;
 int psmouse_smbus_init(struct psmouse *psmouse,
 		       const struct i2c_board_info *board,
 		       const void *pdata, size_t pdata_size,
+		       bool need_deactivate,
 		       bool leave_breadcrumbs);
 void psmouse_smbus_cleanup(struct psmouse *psmouse);
 
diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c
index 60f2c463d1cc..55d33500d55e 100644
--- a/drivers/input/mouse/synaptics.c
+++ b/drivers/input/mouse/synaptics.c
@@ -172,6 +172,12 @@ static const char * const smbus_pnp_ids[] = {
 	"LEN0048", /* X1 Carbon 3 */
 	"LEN0046", /* X250 */
 	"LEN004a", /* W541 */
+	"LEN0071", /* T480 */
+	"LEN0072", /* X1 Carbon Gen 5 (2017) - Elan/ALPS trackpoint */
+	"LEN0073", /* X1 Carbon G5 (Elantech) */
+	"LEN0092", /* X1 Carbon 6 */
+	"LEN0096", /* X280 */
+	"LEN0097", /* X280 -> ALPS trackpoint */
 	"LEN200f", /* T450s */
 	NULL
 };
@@ -1748,7 +1754,7 @@ static int synaptics_create_intertouch(struct psmouse *psmouse,
 	};
 
 	return psmouse_smbus_init(psmouse, &intertouch_board,
-				  &pdata, sizeof(pdata),
+				  &pdata, sizeof(pdata), true,
 				  leave_breadcrumbs);
 }
 
diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig
index 3e613afa10b4..32267c1afebc 100644
--- a/drivers/input/touchscreen/Kconfig
+++ b/drivers/input/touchscreen/Kconfig
@@ -164,6 +164,17 @@ config TOUCHSCREEN_CHIPONE_ICN8318
 	  To compile this driver as a module, choose M here: the
 	  module will be called chipone_icn8318.
 
+config TOUCHSCREEN_CHIPONE_ICN8505
+	tristate "chipone icn8505 touchscreen controller"
+	depends on I2C && ACPI
+	help
+	  Say Y here if you have a ChipOne icn8505 based I2C touchscreen.
+
+	  If unsure, say N.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called chipone_icn8505.
+
 config TOUCHSCREEN_CY8CTMG110
 	tristate "cy8ctmg110 touchscreen"
 	depends on I2C
diff --git a/drivers/input/touchscreen/Makefile b/drivers/input/touchscreen/Makefile
index dddae7973436..fd4fd32fb73f 100644
--- a/drivers/input/touchscreen/Makefile
+++ b/drivers/input/touchscreen/Makefile
@@ -19,6 +19,7 @@ obj-$(CONFIG_TOUCHSCREEN_ATMEL_MXT)	+= atmel_mxt_ts.o
 obj-$(CONFIG_TOUCHSCREEN_AUO_PIXCIR)	+= auo-pixcir-ts.o
 obj-$(CONFIG_TOUCHSCREEN_BU21013)	+= bu21013_ts.o
 obj-$(CONFIG_TOUCHSCREEN_CHIPONE_ICN8318)	+= chipone_icn8318.o
+obj-$(CONFIG_TOUCHSCREEN_CHIPONE_ICN8505)	+= chipone_icn8505.o
 obj-$(CONFIG_TOUCHSCREEN_CY8CTMG110)	+= cy8ctmg110_ts.o
 obj-$(CONFIG_TOUCHSCREEN_CYTTSP_CORE)	+= cyttsp_core.o
 obj-$(CONFIG_TOUCHSCREEN_CYTTSP_I2C)	+= cyttsp_i2c.o cyttsp_i2c_common.o
diff --git a/drivers/input/touchscreen/atmel_mxt_ts.c b/drivers/input/touchscreen/atmel_mxt_ts.c
index 09194721aed2..54fe190fd4bc 100644
--- a/drivers/input/touchscreen/atmel_mxt_ts.c
+++ b/drivers/input/touchscreen/atmel_mxt_ts.c
@@ -194,6 +194,8 @@ enum t100_type {
 
 /* Delay times */
 #define MXT_BACKUP_TIME		50	/* msec */
+#define MXT_RESET_GPIO_TIME	20	/* msec */
+#define MXT_RESET_INVALID_CHG	100	/* msec */
 #define MXT_RESET_TIME		200	/* msec */
 #define MXT_RESET_TIMEOUT	3000	/* msec */
 #define MXT_CRC_TIMEOUT		1000	/* msec */
@@ -1208,7 +1210,7 @@ static int mxt_soft_reset(struct mxt_data *data)
 		return ret;
 
 	/* Ignore CHG line for 100ms after reset */
-	msleep(100);
+	msleep(MXT_RESET_INVALID_CHG);
 
 	mxt_acquire_irq(data);
 
@@ -2999,142 +3001,6 @@ static int mxt_parse_device_properties(struct mxt_data *data)
 	return 0;
 }
 
-#ifdef CONFIG_ACPI
-
-struct mxt_acpi_platform_data {
-	const char *hid;
-	const struct property_entry *props;
-};
-
-static unsigned int samus_touchpad_buttons[] = {
-	KEY_RESERVED,
-	KEY_RESERVED,
-	KEY_RESERVED,
-	BTN_LEFT
-};
-
-static const struct property_entry samus_touchpad_props[] = {
-	PROPERTY_ENTRY_U32_ARRAY("linux,gpio-keymap", samus_touchpad_buttons),
-	{ }
-};
-
-static struct mxt_acpi_platform_data samus_platform_data[] = {
-	{
-		/* Touchpad */
-		.hid	= "ATML0000",
-		.props	= samus_touchpad_props,
-	},
-	{
-		/* Touchscreen */
-		.hid	= "ATML0001",
-	},
-	{ }
-};
-
-static unsigned int chromebook_tp_buttons[] = {
-	KEY_RESERVED,
-	KEY_RESERVED,
-	KEY_RESERVED,
-	KEY_RESERVED,
-	KEY_RESERVED,
-	BTN_LEFT
-};
-
-static const struct property_entry chromebook_tp_props[] = {
-	PROPERTY_ENTRY_U32_ARRAY("linux,gpio-keymap", chromebook_tp_buttons),
-	{ }
-};
-
-static struct mxt_acpi_platform_data chromebook_platform_data[] = {
-	{
-		/* Touchpad */
-		.hid	= "ATML0000",
-		.props	= chromebook_tp_props,
-	},
-	{
-		/* Touchscreen */
-		.hid	= "ATML0001",
-	},
-	{ }
-};
-
-static const struct dmi_system_id mxt_dmi_table[] = {
-	{
-		/* 2015 Google Pixel */
-		.ident = "Chromebook Pixel 2",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "GOOGLE"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "Samus"),
-		},
-		.driver_data = samus_platform_data,
-	},
-	{
-		/* Samsung Chromebook Pro */
-		.ident = "Samsung Chromebook Pro",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Google"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "Caroline"),
-		},
-		.driver_data = samus_platform_data,
-	},
-	{
-		/* Other Google Chromebooks */
-		.ident = "Chromebook",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "GOOGLE"),
-		},
-		.driver_data = chromebook_platform_data,
-	},
-	{ }
-};
-
-static int mxt_prepare_acpi_properties(struct i2c_client *client)
-{
-	struct acpi_device *adev;
-	const struct dmi_system_id *system_id;
-	const struct mxt_acpi_platform_data *acpi_pdata;
-
-	adev = ACPI_COMPANION(&client->dev);
-	if (!adev)
-		return -ENOENT;
-
-	system_id = dmi_first_match(mxt_dmi_table);
-	if (!system_id)
-		return -ENOENT;
-
-	acpi_pdata = system_id->driver_data;
-	if (!acpi_pdata)
-		return -ENOENT;
-
-	while (acpi_pdata->hid) {
-		if (!strcmp(acpi_device_hid(adev), acpi_pdata->hid)) {
-			/*
-			 * Remove previously installed properties if we
-			 * are probing this device not for the very first
-			 * time.
-			 */
-			device_remove_properties(&client->dev);
-
-			/*
-			 * Now install the platform-specific properties
-			 * that are missing from ACPI.
-			 */
-			device_add_properties(&client->dev, acpi_pdata->props);
-			break;
-		}
-
-		acpi_pdata++;
-	}
-
-	return 0;
-}
-#else
-static int mxt_prepare_acpi_properties(struct i2c_client *client)
-{
-	return -ENOENT;
-}
-#endif
-
 static const struct dmi_system_id chromebook_T9_suspend_dmi[] = {
 	{
 		.matches = {
@@ -3155,6 +3021,18 @@ static int mxt_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	struct mxt_data *data;
 	int error;
 
+	/*
+	 * Ignore devices that do not have device properties attached to
+	 * them, as we need help determining whether we are dealing with
+	 * touch screen or touchpad.
+	 *
+	 * So far on x86 the only users of Atmel touch controllers are
+	 * Chromebooks, and chromeos_laptop driver will ensure that
+	 * necessary properties are provided (if firmware does not do that).
+	 */
+	if (!device_property_present(&client->dev, "compatible"))
+		return -ENXIO;
+
 	/*
 	 * Ignore ACPI devices representing bootloader mode.
 	 *
@@ -3186,10 +3064,6 @@ static int mxt_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	data->suspend_mode = dmi_check_system(chromebook_T9_suspend_dmi) ?
 		MXT_SUSPEND_T9_CTRL : MXT_SUSPEND_DEEP_SLEEP;
 
-	error = mxt_prepare_acpi_properties(client);
-	if (error && error != -ENOENT)
-		return error;
-
 	error = mxt_parse_device_properties(data);
 	if (error)
 		return error;
@@ -3210,20 +3084,14 @@ static int mxt_probe(struct i2c_client *client, const struct i2c_device_id *id)
 		return error;
 	}
 
+	disable_irq(client->irq);
+
 	if (data->reset_gpio) {
-		data->in_bootloader = true;
-		msleep(MXT_RESET_TIME);
-		reinit_completion(&data->bl_completion);
+		msleep(MXT_RESET_GPIO_TIME);
 		gpiod_set_value(data->reset_gpio, 1);
-		error = mxt_wait_for_completion(data, &data->bl_completion,
-						MXT_RESET_TIMEOUT);
-		if (error)
-			return error;
-		data->in_bootloader = false;
+		msleep(MXT_RESET_INVALID_CHG);
 	}
 
-	disable_irq(client->irq);
-
 	error = mxt_initialize(data);
 	if (error)
 		return error;
diff --git a/drivers/input/touchscreen/chipone_icn8505.c b/drivers/input/touchscreen/chipone_icn8505.c
new file mode 100644
index 000000000000..c768186ce856
--- /dev/null
+++ b/drivers/input/touchscreen/chipone_icn8505.c
@@ -0,0 +1,520 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Driver for ChipOne icn8505 i2c touchscreen controller
+ *
+ * Copyright (c) 2015-2018 Red Hat Inc.
+ *
+ * Red Hat authors:
+ * Hans de Goede <hdegoede@redhat.com>
+ */
+
+#include <asm/unaligned.h>
+#include <linux/acpi.h>
+#include <linux/crc32.h>
+#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/interrupt.h>
+#include <linux/i2c.h>
+#include <linux/input.h>
+#include <linux/input/mt.h>
+#include <linux/input/touchscreen.h>
+#include <linux/module.h>
+
+/* Normal operation mode defines */
+#define ICN8505_REG_ADDR_WIDTH		16
+
+#define ICN8505_REG_POWER		0x0004
+#define ICN8505_REG_TOUCHDATA		0x1000
+#define ICN8505_REG_CONFIGDATA		0x8000
+
+/* ICN8505_REG_POWER commands */
+#define ICN8505_POWER_ACTIVE		0x00
+#define ICN8505_POWER_MONITOR		0x01
+#define ICN8505_POWER_HIBERNATE		0x02
+/*
+ * The Android driver uses these to turn on/off the charger filter, but the
+ * filter is way too aggressive making e.g. onscreen keyboards unusable.
+ */
+#define ICN8505_POWER_ENA_CHARGER_MODE	0x55
+#define ICN8505_POWER_DIS_CHARGER_MODE	0x66
+
+#define ICN8505_MAX_TOUCHES		10
+
+/* Programming mode defines */
+#define ICN8505_PROG_I2C_ADDR		0x30
+#define ICN8505_PROG_REG_ADDR_WIDTH	24
+
+#define MAX_FW_UPLOAD_TRIES		3
+
+struct icn8505_touch {
+	u8 slot;
+	u8 x[2];
+	u8 y[2];
+	u8 pressure;	/* Seems more like finger width then pressure really */
+	u8 event;
+/* The difference between 2 and 3 is unclear */
+#define ICN8505_EVENT_NO_DATA	1 /* No finger seen yet since wakeup */
+#define ICN8505_EVENT_UPDATE1	2 /* New or updated coordinates */
+#define ICN8505_EVENT_UPDATE2	3 /* New or updated coordinates */
+#define ICN8505_EVENT_END	4 /* Finger lifted */
+} __packed;
+
+struct icn8505_touch_data {
+	u8 softbutton;
+	u8 touch_count;
+	struct icn8505_touch touches[ICN8505_MAX_TOUCHES];
+} __packed;
+
+struct icn8505_data {
+	struct i2c_client *client;
+	struct input_dev *input;
+	struct gpio_desc *wake_gpio;
+	struct touchscreen_properties prop;
+	char firmware_name[32];
+};
+
+static int icn8505_read_xfer(struct i2c_client *client, u16 i2c_addr,
+			     int reg_addr, int reg_addr_width,
+			     void *data, int len, bool silent)
+{
+	u8 buf[3];
+	int i, ret;
+	struct i2c_msg msg[2] = {
+		{
+			.addr = i2c_addr,
+			.buf = buf,
+			.len = reg_addr_width / 8,
+		},
+		{
+			.addr = i2c_addr,
+			.flags = I2C_M_RD,
+			.buf = data,
+			.len = len,
+		}
+	};
+
+	for (i = 0; i < (reg_addr_width / 8); i++)
+		buf[i] = (reg_addr >> (reg_addr_width - (i + 1) * 8)) & 0xff;
+
+	ret = i2c_transfer(client->adapter, msg, 2);
+	if (ret != ARRAY_SIZE(msg)) {
+		if (ret >= 0)
+			ret = -EIO;
+		if (!silent)
+			dev_err(&client->dev,
+				"Error reading addr %#x reg %#x: %d\n",
+				i2c_addr, reg_addr, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int icn8505_write_xfer(struct i2c_client *client, u16 i2c_addr,
+			      int reg_addr, int reg_addr_width,
+			      const void *data, int len, bool silent)
+{
+	u8 buf[3 + 32]; /* 3 bytes for 24 bit reg-addr + 32 bytes max len */
+	int i, ret;
+	struct i2c_msg msg = {
+		.addr = i2c_addr,
+		.buf = buf,
+		.len = reg_addr_width / 8 + len,
+	};
+
+	if (WARN_ON(len > 32))
+		return -EINVAL;
+
+	for (i = 0; i < (reg_addr_width / 8); i++)
+		buf[i] = (reg_addr >> (reg_addr_width - (i + 1) * 8)) & 0xff;
+
+	memcpy(buf + reg_addr_width / 8, data, len);
+
+	ret = i2c_transfer(client->adapter, &msg, 1);
+	if (ret != 1) {
+		if (ret >= 0)
+			ret = -EIO;
+		if (!silent)
+			dev_err(&client->dev,
+				"Error writing addr %#x reg %#x: %d\n",
+				i2c_addr, reg_addr, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int icn8505_read_data(struct icn8505_data *icn8505, int reg,
+			     void *buf, int len)
+{
+	return icn8505_read_xfer(icn8505->client, icn8505->client->addr, reg,
+				 ICN8505_REG_ADDR_WIDTH, buf, len, false);
+}
+
+static int icn8505_read_reg_silent(struct icn8505_data *icn8505, int reg)
+{
+	u8 buf;
+	int error;
+
+	error = icn8505_read_xfer(icn8505->client, icn8505->client->addr, reg,
+				  ICN8505_REG_ADDR_WIDTH, &buf, 1, true);
+	if (error)
+		return error;
+
+	return buf;
+}
+
+static int icn8505_write_reg(struct icn8505_data *icn8505, int reg, u8 val)
+{
+	return icn8505_write_xfer(icn8505->client, icn8505->client->addr, reg,
+				  ICN8505_REG_ADDR_WIDTH, &val, 1, false);
+}
+
+static int icn8505_read_prog_data(struct icn8505_data *icn8505, int reg,
+				  void *buf, int len)
+{
+	return icn8505_read_xfer(icn8505->client, ICN8505_PROG_I2C_ADDR, reg,
+				 ICN8505_PROG_REG_ADDR_WIDTH, buf, len, false);
+}
+
+static int icn8505_write_prog_data(struct icn8505_data *icn8505, int reg,
+				   const void *buf, int len)
+{
+	return icn8505_write_xfer(icn8505->client, ICN8505_PROG_I2C_ADDR, reg,
+				  ICN8505_PROG_REG_ADDR_WIDTH, buf, len, false);
+}
+
+static int icn8505_write_prog_reg(struct icn8505_data *icn8505, int reg, u8 val)
+{
+	return icn8505_write_xfer(icn8505->client, ICN8505_PROG_I2C_ADDR, reg,
+				  ICN8505_PROG_REG_ADDR_WIDTH, &val, 1, false);
+}
+
+/*
+ * Note this function uses a number of magic register addresses and values,
+ * there are deliberately no defines for these because the algorithm is taken
+ * from the icn85xx Android driver and I do not want to make up possibly wrong
+ * names for the addresses and/or values.
+ */
+static int icn8505_try_fw_upload(struct icn8505_data *icn8505,
+				 const struct firmware *fw)
+{
+	struct device *dev = &icn8505->client->dev;
+	size_t offset, count;
+	int error;
+	u8 buf[4];
+	u32 crc;
+
+	/* Put the controller in programming mode */
+	error = icn8505_write_prog_reg(icn8505, 0xcc3355, 0x5a);
+	if (error)
+		return error;
+
+	usleep_range(2000, 5000);
+
+	error = icn8505_write_prog_reg(icn8505, 0x040400, 0x01);
+	if (error)
+		return error;
+
+	usleep_range(2000, 5000);
+
+	error = icn8505_read_prog_data(icn8505, 0x040002, buf, 1);
+	if (error)
+		return error;
+
+	if (buf[0] != 0x85) {
+		dev_err(dev, "Failed to enter programming mode\n");
+		return -ENODEV;
+	}
+
+	usleep_range(1000, 5000);
+
+	/* Enable CRC mode */
+	error = icn8505_write_prog_reg(icn8505, 0x40028, 1);
+	if (error)
+		return error;
+
+	/* Send the firmware to SRAM */
+	for (offset = 0; offset < fw->size; offset += count) {
+		count = min_t(size_t, fw->size - offset, 32);
+		error = icn8505_write_prog_data(icn8505, offset,
+					      fw->data + offset, count);
+		if (error)
+			return error;
+	}
+
+	/* Disable CRC mode */
+	error = icn8505_write_prog_reg(icn8505, 0x40028, 0);
+	if (error)
+		return error;
+
+	/* Get and check length and CRC */
+	error = icn8505_read_prog_data(icn8505, 0x40034, buf, 2);
+	if (error)
+		return error;
+
+	if (get_unaligned_le16(buf) != fw->size) {
+		dev_warn(dev, "Length mismatch after uploading fw\n");
+		return -EIO;
+	}
+
+	error = icn8505_read_prog_data(icn8505, 0x4002c, buf, 4);
+	if (error)
+		return error;
+
+	crc = crc32_be(0, fw->data, fw->size);
+	if (get_unaligned_le32(buf) != crc) {
+		dev_warn(dev, "CRC mismatch after uploading fw\n");
+		return -EIO;
+	}
+
+	/* Boot controller from SRAM */
+	error = icn8505_write_prog_reg(icn8505, 0x40400, 0x03);
+	if (error)
+		return error;
+
+	usleep_range(2000, 5000);
+	return 0;
+}
+
+static int icn8505_upload_fw(struct icn8505_data *icn8505)
+{
+	struct device *dev = &icn8505->client->dev;
+	const struct firmware *fw;
+	int i, error;
+
+	/*
+	 * Always load the firmware, even if we don't need it at boot, we
+	 * we may need it at resume. Having loaded it once will make the
+	 * firmware class code cache it at suspend/resume.
+	 */
+	error = request_firmware(&fw, icn8505->firmware_name, dev);
+	if (error) {
+		dev_err(dev, "Firmware request error %d\n", error);
+		return error;
+	}
+
+	/* Check if the controller is not already up and running */
+	if (icn8505_read_reg_silent(icn8505, 0x000a) == 0x85)
+		goto success;
+
+	for (i = 1; i <= MAX_FW_UPLOAD_TRIES; i++) {
+		error = icn8505_try_fw_upload(icn8505, fw);
+		if (!error)
+			goto success;
+
+		dev_err(dev, "Failed to upload firmware: %d (attempt %d/%d)\n",
+			error, i, MAX_FW_UPLOAD_TRIES);
+		usleep_range(2000, 5000);
+	}
+
+success:
+	release_firmware(fw);
+	return error;
+}
+
+static bool icn8505_touch_active(u8 event)
+{
+	return event == ICN8505_EVENT_UPDATE1 ||
+	       event == ICN8505_EVENT_UPDATE2;
+}
+
+static irqreturn_t icn8505_irq(int irq, void *dev_id)
+{
+	struct icn8505_data *icn8505 = dev_id;
+	struct device *dev = &icn8505->client->dev;
+	struct icn8505_touch_data touch_data;
+	int i, error;
+
+	error = icn8505_read_data(icn8505, ICN8505_REG_TOUCHDATA,
+				  &touch_data, sizeof(touch_data));
+	if (error) {
+		dev_err(dev, "Error reading touch data: %d\n", error);
+		return IRQ_HANDLED;
+	}
+
+	if (touch_data.touch_count > ICN8505_MAX_TOUCHES) {
+		dev_warn(dev, "Too many touches %d > %d\n",
+			 touch_data.touch_count, ICN8505_MAX_TOUCHES);
+		touch_data.touch_count = ICN8505_MAX_TOUCHES;
+	}
+
+	for (i = 0; i < touch_data.touch_count; i++) {
+		struct icn8505_touch *touch = &touch_data.touches[i];
+		bool act = icn8505_touch_active(touch->event);
+
+		input_mt_slot(icn8505->input, touch->slot);
+		input_mt_report_slot_state(icn8505->input, MT_TOOL_FINGER, act);
+		if (!act)
+			continue;
+
+		touchscreen_report_pos(icn8505->input, &icn8505->prop,
+				       get_unaligned_le16(touch->x),
+				       get_unaligned_le16(touch->y),
+				       true);
+	}
+
+	input_mt_sync_frame(icn8505->input);
+	input_report_key(icn8505->input, KEY_LEFTMETA,
+			 touch_data.softbutton == 1);
+	input_sync(icn8505->input);
+
+	return IRQ_HANDLED;
+}
+
+static int icn8505_probe_acpi(struct icn8505_data *icn8505, struct device *dev)
+{
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	const char *subsys = "unknown";
+	struct acpi_device *adev;
+	union acpi_object *obj;
+	acpi_status status;
+
+	adev = ACPI_COMPANION(dev);
+	if (!adev)
+		return -ENODEV;
+
+	status = acpi_evaluate_object(adev->handle, "_SUB", NULL, &buffer);
+	if (ACPI_SUCCESS(status)) {
+		obj = buffer.pointer;
+		if (obj->type == ACPI_TYPE_STRING)
+			subsys = obj->string.pointer;
+		else
+			dev_warn(dev, "Warning ACPI _SUB did not return a string\n");
+	} else {
+		dev_warn(dev, "Warning ACPI _SUB failed: %#x\n", status);
+		buffer.pointer = NULL;
+	}
+
+	snprintf(icn8505->firmware_name, sizeof(icn8505->firmware_name),
+		 "chipone/icn8505-%s.fw", subsys);
+
+	kfree(buffer.pointer);
+	return 0;
+}
+
+static int icn8505_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct icn8505_data *icn8505;
+	struct input_dev *input;
+	__le16 resolution[2];
+	int error;
+
+	if (!client->irq) {
+		dev_err(dev, "No irq specified\n");
+		return -EINVAL;
+	}
+
+	icn8505 = devm_kzalloc(dev, sizeof(*icn8505), GFP_KERNEL);
+	if (!icn8505)
+		return -ENOMEM;
+
+	input = devm_input_allocate_device(dev);
+	if (!input)
+		return -ENOMEM;
+
+	input->name = client->name;
+	input->id.bustype = BUS_I2C;
+
+	input_set_capability(input, EV_ABS, ABS_MT_POSITION_X);
+	input_set_capability(input, EV_ABS, ABS_MT_POSITION_Y);
+	input_set_capability(input, EV_KEY, KEY_LEFTMETA);
+
+	icn8505->client = client;
+	icn8505->input = input;
+	input_set_drvdata(input, icn8505);
+
+	error = icn8505_probe_acpi(icn8505, dev);
+	if (error)
+		return error;
+
+	error = icn8505_upload_fw(icn8505);
+	if (error)
+		return error;
+
+	error = icn8505_read_data(icn8505, ICN8505_REG_CONFIGDATA,
+				resolution, sizeof(resolution));
+	if (error) {
+		dev_err(dev, "Error reading resolution: %d\n", error);
+		return error;
+	}
+
+	input_set_abs_params(input, ABS_MT_POSITION_X, 0,
+			     le16_to_cpu(resolution[0]) - 1, 0, 0);
+	input_set_abs_params(input, ABS_MT_POSITION_Y, 0,
+			     le16_to_cpu(resolution[1]) - 1, 0, 0);
+
+	touchscreen_parse_properties(input, true, &icn8505->prop);
+	if (!input_abs_get_max(input, ABS_MT_POSITION_X) ||
+	    !input_abs_get_max(input, ABS_MT_POSITION_Y)) {
+		dev_err(dev, "Error touchscreen-size-x and/or -y missing\n");
+		return -EINVAL;
+	}
+
+	error = input_mt_init_slots(input, ICN8505_MAX_TOUCHES,
+				  INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED);
+	if (error)
+		return error;
+
+	error = devm_request_threaded_irq(dev, client->irq, NULL, icn8505_irq,
+					IRQF_ONESHOT, client->name, icn8505);
+	if (error) {
+		dev_err(dev, "Error requesting irq: %d\n", error);
+		return error;
+	}
+
+	error = input_register_device(input);
+	if (error)
+		return error;
+
+	i2c_set_clientdata(client, icn8505);
+	return 0;
+}
+
+static int __maybe_unused icn8505_suspend(struct device *dev)
+{
+	struct icn8505_data *icn8505 = i2c_get_clientdata(to_i2c_client(dev));
+
+	disable_irq(icn8505->client->irq);
+
+	icn8505_write_reg(icn8505, ICN8505_REG_POWER, ICN8505_POWER_HIBERNATE);
+
+	return 0;
+}
+
+static int __maybe_unused icn8505_resume(struct device *dev)
+{
+	struct icn8505_data *icn8505 = i2c_get_clientdata(to_i2c_client(dev));
+	int error;
+
+	error = icn8505_upload_fw(icn8505);
+	if (error)
+		return error;
+
+	enable_irq(icn8505->client->irq);
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(icn8505_pm_ops, icn8505_suspend, icn8505_resume);
+
+static const struct acpi_device_id icn8505_acpi_match[] = {
+	{ "CHPN0001" },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, icn8505_acpi_match);
+
+static struct i2c_driver icn8505_driver = {
+	.driver = {
+		.name	= "chipone_icn8505",
+		.pm	= &icn8505_pm_ops,
+		.acpi_match_table = icn8505_acpi_match,
+	},
+	.probe_new = icn8505_probe,
+};
+
+module_i2c_driver(icn8505_driver);
+
+MODULE_DESCRIPTION("ChipOne icn8505 I2C Touchscreen Driver");
+MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/input/touchscreen/mk712.c b/drivers/input/touchscreen/mk712.c
index bd5352824f77..c179060525ae 100644
--- a/drivers/input/touchscreen/mk712.c
+++ b/drivers/input/touchscreen/mk712.c
@@ -17,7 +17,7 @@
  * found in Gateway AOL Connected Touchpad computers.
  *
  * Documentation for ICS MK712 can be found at:
- *	http://www.idt.com/products/getDoc.cfm?docID=18713923
+ *	https://www.idt.com/general-parts/mk712-touch-screen-controller
  */
 
 /*
diff --git a/drivers/input/touchscreen/ti_am335x_tsc.c b/drivers/input/touchscreen/ti_am335x_tsc.c
index f1043ae71dcc..b86c1e5fbc11 100644
--- a/drivers/input/touchscreen/ti_am335x_tsc.c
+++ b/drivers/input/touchscreen/ti_am335x_tsc.c
@@ -34,6 +34,8 @@
 #define SEQ_SETTLE		275
 #define MAX_12BIT		((1 << 12) - 1)
 
+#define TSC_IRQENB_MASK		(IRQENB_FIFO0THRES | IRQENB_EOS | IRQENB_HW_PEN)
+
 static const int config_pins[] = {
 	STEPCONFIG_XPP,
 	STEPCONFIG_XNN,
@@ -274,6 +276,7 @@ static irqreturn_t titsc_irq(int irq, void *dev)
 	if (status & IRQENB_HW_PEN) {
 		ts_dev->pen_down = true;
 		irqclr |= IRQENB_HW_PEN;
+		pm_stay_awake(ts_dev->mfd_tscadc->dev);
 	}
 
 	if (status & IRQENB_PENUP) {
@@ -283,6 +286,7 @@ static irqreturn_t titsc_irq(int irq, void *dev)
 			input_report_key(input_dev, BTN_TOUCH, 0);
 			input_report_abs(input_dev, ABS_PRESSURE, 0);
 			input_sync(input_dev);
+			pm_relax(ts_dev->mfd_tscadc->dev);
 		} else {
 			ts_dev->pen_down = true;
 		}
@@ -432,6 +436,7 @@ static int titsc_probe(struct platform_device *pdev)
 		goto err_free_mem;
 	}
 
+	titsc_writel(ts_dev, REG_IRQSTATUS, TSC_IRQENB_MASK);
 	titsc_writel(ts_dev, REG_IRQENABLE, IRQENB_FIFO0THRES);
 	titsc_writel(ts_dev, REG_IRQENABLE, IRQENB_EOS);
 	err = titsc_config_wires(ts_dev);
@@ -495,6 +500,7 @@ static int __maybe_unused titsc_suspend(struct device *dev)
 
 	tscadc_dev = ti_tscadc_dev_get(to_platform_device(dev));
 	if (device_may_wakeup(tscadc_dev->dev)) {
+		titsc_writel(ts_dev, REG_IRQSTATUS, TSC_IRQENB_MASK);
 		idle = titsc_readl(ts_dev, REG_IRQENABLE);
 		titsc_writel(ts_dev, REG_IRQENABLE,
 				(idle | IRQENB_HW_PEN));
@@ -513,6 +519,7 @@ static int __maybe_unused titsc_resume(struct device *dev)
 		titsc_writel(ts_dev, REG_IRQWAKEUP,
 				0x00);
 		titsc_writel(ts_dev, REG_IRQCLR, IRQENB_HW_PEN);
+		pm_relax(ts_dev->mfd_tscadc->dev);
 	}
 	titsc_step_config(ts_dev);
 	titsc_writel(ts_dev, REG_FIFO0THR,
diff --git a/drivers/input/touchscreen/usbtouchscreen.c b/drivers/input/touchscreen/usbtouchscreen.c
index c6cf90868503..d61570d64ee7 100644
--- a/drivers/input/touchscreen/usbtouchscreen.c
+++ b/drivers/input/touchscreen/usbtouchscreen.c
@@ -440,6 +440,8 @@ static int panjit_read_data(struct usbtouch_usb *dev, unsigned char *pkt)
 #define MTOUCHUSB_RESET                 7
 #define MTOUCHUSB_REQ_CTRLLR_ID         10
 
+#define MTOUCHUSB_REQ_CTRLLR_ID_LEN	16
+
 static int mtouch_read_data(struct usbtouch_usb *dev, unsigned char *pkt)
 {
 	if (hwcalib_xy) {
@@ -454,11 +456,93 @@ static int mtouch_read_data(struct usbtouch_usb *dev, unsigned char *pkt)
 	return 1;
 }
 
+struct mtouch_priv {
+	u8 fw_rev_major;
+	u8 fw_rev_minor;
+};
+
+static ssize_t mtouch_firmware_rev_show(struct device *dev,
+				struct device_attribute *attr, char *output)
+{
+	struct usb_interface *intf = to_usb_interface(dev);
+	struct usbtouch_usb *usbtouch = usb_get_intfdata(intf);
+	struct mtouch_priv *priv = usbtouch->priv;
+
+	return scnprintf(output, PAGE_SIZE, "%1x.%1x\n",
+			 priv->fw_rev_major, priv->fw_rev_minor);
+}
+static DEVICE_ATTR(firmware_rev, 0444, mtouch_firmware_rev_show, NULL);
+
+static struct attribute *mtouch_attrs[] = {
+	&dev_attr_firmware_rev.attr,
+	NULL
+};
+
+static const struct attribute_group mtouch_attr_group = {
+	.attrs = mtouch_attrs,
+};
+
+static int mtouch_get_fw_revision(struct usbtouch_usb *usbtouch)
+{
+	struct usb_device *udev = interface_to_usbdev(usbtouch->interface);
+	struct mtouch_priv *priv = usbtouch->priv;
+	u8 *buf;
+	int ret;
+
+	buf = kzalloc(MTOUCHUSB_REQ_CTRLLR_ID_LEN, GFP_NOIO);
+	if (!buf)
+		return -ENOMEM;
+
+	ret = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
+			      MTOUCHUSB_REQ_CTRLLR_ID,
+			      USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+			      0, 0, buf, MTOUCHUSB_REQ_CTRLLR_ID_LEN,
+			      USB_CTRL_SET_TIMEOUT);
+	if (ret != MTOUCHUSB_REQ_CTRLLR_ID_LEN) {
+		dev_warn(&usbtouch->interface->dev,
+			 "Failed to read FW rev: %d\n", ret);
+		ret = ret < 0 ? ret : -EIO;
+		goto free;
+	}
+
+	priv->fw_rev_major = buf[3];
+	priv->fw_rev_minor = buf[4];
+
+	ret = 0;
+
+free:
+	kfree(buf);
+	return ret;
+}
+
+static int mtouch_alloc(struct usbtouch_usb *usbtouch)
+{
+	int ret;
+
+	usbtouch->priv = kmalloc(sizeof(struct mtouch_priv), GFP_KERNEL);
+	if (!usbtouch->priv)
+		return -ENOMEM;
+
+	ret = sysfs_create_group(&usbtouch->interface->dev.kobj,
+				 &mtouch_attr_group);
+	if (ret) {
+		kfree(usbtouch->priv);
+		usbtouch->priv = NULL;
+		return ret;
+	}
+
+	return 0;
+}
+
 static int mtouch_init(struct usbtouch_usb *usbtouch)
 {
 	int ret, i;
 	struct usb_device *udev = interface_to_usbdev(usbtouch->interface);
 
+	ret = mtouch_get_fw_revision(usbtouch);
+	if (ret)
+		return ret;
+
 	ret = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
 	                      MTOUCHUSB_RESET,
 	                      USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
@@ -492,6 +576,14 @@ static int mtouch_init(struct usbtouch_usb *usbtouch)
 
 	return 0;
 }
+
+static void mtouch_exit(struct usbtouch_usb *usbtouch)
+{
+	struct mtouch_priv *priv = usbtouch->priv;
+
+	sysfs_remove_group(&usbtouch->interface->dev.kobj, &mtouch_attr_group);
+	kfree(priv);
+}
 #endif
 
 
@@ -1119,7 +1211,9 @@ static struct usbtouch_device_info usbtouch_dev_info[] = {
 		.max_yc		= 0x4000,
 		.rept_size	= 11,
 		.read_data	= mtouch_read_data,
+		.alloc		= mtouch_alloc,
 		.init		= mtouch_init,
+		.exit		= mtouch_exit,
 	},
 #endif
 
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index df171cb85822..8ea77efb2e29 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -23,7 +23,7 @@ config IOMMU_IO_PGTABLE
 config IOMMU_IO_PGTABLE_LPAE
 	bool "ARMv7/v8 Long Descriptor Format"
 	select IOMMU_IO_PGTABLE
-	depends on HAS_DMA && (ARM || ARM64 || (COMPILE_TEST && !GENERIC_ATOMIC64))
+	depends on ARM || ARM64 || (COMPILE_TEST && !GENERIC_ATOMIC64)
 	help
 	  Enable support for the ARM long descriptor pagetable format.
 	  This allocator supports 4K/2M/1G, 16K/32M and 64K/512M page
@@ -42,7 +42,7 @@ config IOMMU_IO_PGTABLE_LPAE_SELFTEST
 config IOMMU_IO_PGTABLE_ARMV7S
 	bool "ARMv7/v8 Short Descriptor Format"
 	select IOMMU_IO_PGTABLE
-	depends on HAS_DMA && (ARM || ARM64 || COMPILE_TEST)
+	depends on ARM || ARM64 || COMPILE_TEST
 	help
 	  Enable support for the ARM Short-descriptor pagetable format.
 	  This supports 32-bit virtual and physical addresses mapped using
@@ -146,6 +146,7 @@ config INTEL_IOMMU
 	select DMA_DIRECT_OPS
 	select IOMMU_API
 	select IOMMU_IOVA
+	select NEED_DMA_MAP_STATE
 	select DMAR_TABLE
 	help
 	  DMA remapping (DMAR) devices support enables independent address
@@ -376,7 +377,6 @@ config QCOM_IOMMU
 	# Note: iommu drivers cannot (yet?) be built as modules
 	bool "Qualcomm IOMMU Support"
 	depends on ARCH_QCOM || (COMPILE_TEST && !GENERIC_ATOMIC64)
-	depends on HAS_DMA
 	select IOMMU_API
 	select IOMMU_IO_PGTABLE_LPAE
 	select ARM_DMA_USE_IOMMU
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 8fb8c737fffe..1912e9106fbe 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -354,6 +354,9 @@ static bool pci_iommuv2_capable(struct pci_dev *pdev)
 	};
 	int i, pos;
 
+	if (pci_ats_disabled())
+		return false;
+
 	for (i = 0; i < 3; ++i) {
 		pos = pci_find_ext_capability(pdev, caps[i]);
 		if (pos == 0)
@@ -544,7 +547,7 @@ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id,
 static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
 {
 	struct device *dev = iommu->iommu.dev;
-	int type, devid, domid, flags;
+	int type, devid, pasid, flags, tag;
 	volatile u32 *event = __evt;
 	int count = 0;
 	u64 address;
@@ -552,7 +555,7 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
 retry:
 	type    = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
 	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
-	domid   = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
+	pasid   = PPR_PASID(*(u64 *)&event[0]);
 	flags   = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
 	address = (u64)(((u64)event[3]) << 32) | event[2];
 
@@ -567,7 +570,7 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
 	}
 
 	if (type == EVENT_TYPE_IO_FAULT) {
-		amd_iommu_report_page_fault(devid, domid, address, flags);
+		amd_iommu_report_page_fault(devid, pasid, address, flags);
 		return;
 	} else {
 		dev_err(dev, "AMD-Vi: Event logged [");
@@ -575,10 +578,9 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
 
 	switch (type) {
 	case EVENT_TYPE_ILL_DEV:
-		dev_err(dev, "ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
-			"address=0x%016llx flags=0x%04x]\n",
+		dev_err(dev, "ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n",
 			PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
-			address, flags);
+			pasid, address, flags);
 		dump_dte_entry(devid);
 		break;
 	case EVENT_TYPE_DEV_TAB_ERR:
@@ -588,34 +590,38 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
 			address, flags);
 		break;
 	case EVENT_TYPE_PAGE_TAB_ERR:
-		dev_err(dev, "PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
-			"domain=0x%04x address=0x%016llx flags=0x%04x]\n",
+		dev_err(dev, "PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x domain=0x%04x address=0x%016llx flags=0x%04x]\n",
 			PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
-			domid, address, flags);
+			pasid, address, flags);
 		break;
 	case EVENT_TYPE_ILL_CMD:
 		dev_err(dev, "ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
 		dump_command(address);
 		break;
 	case EVENT_TYPE_CMD_HARD_ERR:
-		dev_err(dev, "COMMAND_HARDWARE_ERROR address=0x%016llx "
-			"flags=0x%04x]\n", address, flags);
+		dev_err(dev, "COMMAND_HARDWARE_ERROR address=0x%016llx flags=0x%04x]\n",
+			address, flags);
 		break;
 	case EVENT_TYPE_IOTLB_INV_TO:
-		dev_err(dev, "IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
-			"address=0x%016llx]\n",
+		dev_err(dev, "IOTLB_INV_TIMEOUT device=%02x:%02x.%x address=0x%016llx]\n",
 			PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 			address);
 		break;
 	case EVENT_TYPE_INV_DEV_REQ:
-		dev_err(dev, "INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
-			"address=0x%016llx flags=0x%04x]\n",
+		dev_err(dev, "INVALID_DEVICE_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n",
 			PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
-			address, flags);
+			pasid, address, flags);
+		break;
+	case EVENT_TYPE_INV_PPR_REQ:
+		pasid = ((event[0] >> 16) & 0xFFFF)
+			| ((event[1] << 6) & 0xF0000);
+		tag = event[1] & 0x03FF;
+		dev_err(dev, "INVALID_PPR_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n",
+			PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
+			pasid, address, flags);
 		break;
 	default:
-		dev_err(dev, KERN_ERR "UNKNOWN event[0]=0x%08x event[1]=0x%08x "
-			"event[2]=0x%08x event[3]=0x%08x\n",
+		dev_err(dev, "UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n",
 			event[0], event[1], event[2], event[3]);
 	}
 
@@ -1911,15 +1917,6 @@ static void do_detach(struct iommu_dev_data *dev_data)
 	struct amd_iommu *iommu;
 	u16 alias;
 
-	/*
-	 * First check if the device is still attached. It might already
-	 * be detached from its domain because the generic
-	 * iommu_detach_group code detached it and we try again here in
-	 * our alias handling.
-	 */
-	if (!dev_data->domain)
-		return;
-
 	iommu = amd_iommu_rlookup_table[dev_data->devid];
 	alias = dev_data->alias;
 
@@ -1939,8 +1936,8 @@ static void do_detach(struct iommu_dev_data *dev_data)
 }
 
 /*
- * If a device is not yet associated with a domain, this function does
- * assigns it visible for the hardware
+ * If a device is not yet associated with a domain, this function makes the
+ * device visible in the domain
  */
 static int __attach_device(struct iommu_dev_data *dev_data,
 			   struct protection_domain *domain)
@@ -2061,8 +2058,8 @@ static bool pci_pri_tlp_required(struct pci_dev *pdev)
 }
 
 /*
- * If a device is not yet associated with a domain, this function
- * assigns it visible for the hardware
+ * If a device is not yet associated with a domain, this function makes the
+ * device visible in the domain
  */
 static int attach_device(struct device *dev,
 			 struct protection_domain *domain)
@@ -2124,9 +2121,6 @@ static void __detach_device(struct iommu_dev_data *dev_data)
 	 */
 	WARN_ON(!irqs_disabled());
 
-	if (WARN_ON(!dev_data->domain))
-		return;
-
 	domain = dev_data->domain;
 
 	spin_lock(&domain->lock);
@@ -2148,6 +2142,15 @@ static void detach_device(struct device *dev)
 	dev_data = get_dev_data(dev);
 	domain   = dev_data->domain;
 
+	/*
+	 * First check if the device is still attached. It might already
+	 * be detached from its domain because the generic
+	 * iommu_detach_group code detached it and we try again here in
+	 * our alias handling.
+	 */
+	if (WARN_ON(!dev_data->domain))
+		return;
+
 	/* lock device table */
 	spin_lock_irqsave(&amd_iommu_devtable_lock, flags);
 	__detach_device(dev_data);
@@ -2793,6 +2796,7 @@ static void cleanup_domain(struct protection_domain *domain)
 	while (!list_empty(&domain->dev_list)) {
 		entry = list_first_entry(&domain->dev_list,
 					 struct iommu_dev_data, list);
+		BUG_ON(!entry->domain);
 		__detach_device(entry);
 	}
 
@@ -3523,9 +3527,11 @@ int amd_iommu_device_info(struct pci_dev *pdev,
 
 	memset(info, 0, sizeof(*info));
 
-	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
-	if (pos)
-		info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
+	if (!pci_ats_disabled()) {
+		pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
+		if (pos)
+			info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
+	}
 
 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
 	if (pos)
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 1c9b080276c9..986cbe0cc189 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -133,6 +133,7 @@
 #define EVENT_TYPE_CMD_HARD_ERR	0x6
 #define EVENT_TYPE_IOTLB_INV_TO	0x7
 #define EVENT_TYPE_INV_DEV_REQ	0x8
+#define EVENT_TYPE_INV_PPR_REQ	0x9
 #define EVENT_DEVID_MASK	0xffff
 #define EVENT_DEVID_SHIFT	0
 #define EVENT_DOMID_MASK	0xffff
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 460bed4fc5b1..4321f7704b23 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -1618,17 +1618,13 @@ irqreturn_t dmar_fault(int irq, void *dev_id)
 	int reg, fault_index;
 	u32 fault_status;
 	unsigned long flag;
-	bool ratelimited;
 	static DEFINE_RATELIMIT_STATE(rs,
 				      DEFAULT_RATELIMIT_INTERVAL,
 				      DEFAULT_RATELIMIT_BURST);
 
-	/* Disable printing, simply clear the fault when ratelimited */
-	ratelimited = !__ratelimit(&rs);
-
 	raw_spin_lock_irqsave(&iommu->register_lock, flag);
 	fault_status = readl(iommu->reg + DMAR_FSTS_REG);
-	if (fault_status && !ratelimited)
+	if (fault_status && __ratelimit(&rs))
 		pr_err("DRHD: handling fault status reg %x\n", fault_status);
 
 	/* TBD: ignore advanced fault log currently */
@@ -1638,6 +1634,8 @@ irqreturn_t dmar_fault(int irq, void *dev_id)
 	fault_index = dma_fsts_fault_record_index(fault_status);
 	reg = cap_fault_reg_offset(iommu->cap);
 	while (1) {
+		/* Disable printing, simply clear the fault when ratelimited */
+		bool ratelimited = !__ratelimit(&rs);
 		u8 fault_reason;
 		u16 source_id;
 		u64 guest_addr;
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 749d8f235346..89e49a429c57 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -485,37 +485,14 @@ static int dmar_forcedac;
 static int intel_iommu_strict;
 static int intel_iommu_superpage = 1;
 static int intel_iommu_ecs = 1;
-static int intel_iommu_pasid28;
 static int iommu_identity_mapping;
 
 #define IDENTMAP_ALL		1
 #define IDENTMAP_GFX		2
 #define IDENTMAP_AZALIA		4
 
-/* Broadwell and Skylake have broken ECS support — normal so-called "second
- * level" translation of DMA requests-without-PASID doesn't actually happen
- * unless you also set the NESTE bit in an extended context-entry. Which of
- * course means that SVM doesn't work because it's trying to do nested
- * translation of the physical addresses it finds in the process page tables,
- * through the IOVA->phys mapping found in the "second level" page tables.
- *
- * The VT-d specification was retroactively changed to change the definition
- * of the capability bits and pretend that Broadwell/Skylake never happened...
- * but unfortunately the wrong bit was changed. It's ECS which is broken, but
- * for some reason it was the PASID capability bit which was redefined (from
- * bit 28 on BDW/SKL to bit 40 in future).
- *
- * So our test for ECS needs to eschew those implementations which set the old
- * PASID capabiity bit 28, since those are the ones on which ECS is broken.
- * Unless we are working around the 'pasid28' limitations, that is, by putting
- * the device into passthrough mode for normal DMA and thus masking the bug.
- */
-#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
-			    (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
-/* PASID support is thus enabled if ECS is enabled and *either* of the old
- * or new capability bits are set. */
-#define pasid_enabled(iommu) (ecs_enabled(iommu) &&			\
-			      (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
+#define ecs_enabled(iommu)	(intel_iommu_ecs && ecap_ecs(iommu->ecap))
+#define pasid_enabled(iommu)	(ecs_enabled(iommu) && ecap_pasid(iommu->ecap))
 
 int intel_iommu_gfx_mapped;
 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
@@ -578,11 +555,6 @@ static int __init intel_iommu_setup(char *str)
 			printk(KERN_INFO
 				"Intel-IOMMU: disable extended context table support\n");
 			intel_iommu_ecs = 0;
-		} else if (!strncmp(str, "pasid28", 7)) {
-			printk(KERN_INFO
-				"Intel-IOMMU: enable pre-production PASID support\n");
-			intel_iommu_pasid28 = 1;
-			iommu_identity_mapping |= IDENTMAP_GFX;
 		} else if (!strncmp(str, "tboot_noforce", 13)) {
 			printk(KERN_INFO
 				"Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
@@ -1606,6 +1578,18 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
 		iommu_flush_dev_iotlb(domain, addr, mask);
 }
 
+/* Notification for newly created mappings */
+static inline void __mapping_notify_one(struct intel_iommu *iommu,
+					struct dmar_domain *domain,
+					unsigned long pfn, unsigned int pages)
+{
+	/* It's a non-present to present mapping. Only flush if caching mode */
+	if (cap_caching_mode(iommu->cap))
+		iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
+	else
+		iommu_flush_write_buffer(iommu);
+}
+
 static void iommu_flush_iova(struct iova_domain *iovad)
 {
 	struct dmar_domain *domain;
@@ -2340,18 +2324,47 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
 	return 0;
 }
 
+static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
+                         struct scatterlist *sg, unsigned long phys_pfn,
+                         unsigned long nr_pages, int prot)
+{
+       int ret;
+       struct intel_iommu *iommu;
+
+       /* Do the real mapping first */
+       ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
+       if (ret)
+               return ret;
+
+       /* Notify about the new mapping */
+       if (domain_type_is_vm(domain)) {
+	       /* VM typed domains can have more than one IOMMUs */
+	       int iommu_id;
+	       for_each_domain_iommu(iommu_id, domain) {
+		       iommu = g_iommus[iommu_id];
+		       __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
+	       }
+       } else {
+	       /* General domains only have one IOMMU */
+	       iommu = domain_get_iommu(domain);
+	       __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
+       }
+
+       return 0;
+}
+
 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
 				    struct scatterlist *sg, unsigned long nr_pages,
 				    int prot)
 {
-	return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
+	return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
 }
 
 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
 				     unsigned long phys_pfn, unsigned long nr_pages,
 				     int prot)
 {
-	return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
+	return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
 }
 
 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
@@ -2459,7 +2472,8 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
 	if (dev && dev_is_pci(dev)) {
 		struct pci_dev *pdev = to_pci_dev(info->dev);
 
-		if (ecap_dev_iotlb_support(iommu->ecap) &&
+		if (!pci_ats_disabled() &&
+		    ecap_dev_iotlb_support(iommu->ecap) &&
 		    pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
 		    dmar_find_matched_atsr_unit(pdev))
 			info->ats_supported = 1;
@@ -2533,7 +2547,7 @@ static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
 	struct device_domain_info *info = NULL;
 	struct dmar_domain *domain = NULL;
 	struct intel_iommu *iommu;
-	u16 req_id, dma_alias;
+	u16 dma_alias;
 	unsigned long flags;
 	u8 bus, devfn;
 
@@ -2541,8 +2555,6 @@ static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
 	if (!iommu)
 		return NULL;
 
-	req_id = ((u16)bus << 8) | devfn;
-
 	if (dev_is_pci(dev)) {
 		struct pci_dev *pdev = to_pci_dev(dev);
 
@@ -2656,9 +2668,9 @@ static int iommu_domain_identity_map(struct dmar_domain *domain,
 	 */
 	dma_pte_clear_range(domain, first_vpfn, last_vpfn);
 
-	return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
-				  last_vpfn - first_vpfn + 1,
-				  DMA_PTE_READ|DMA_PTE_WRITE);
+	return __domain_mapping(domain, first_vpfn, NULL,
+				first_vpfn, last_vpfn - first_vpfn + 1,
+				DMA_PTE_READ|DMA_PTE_WRITE);
 }
 
 static int domain_prepare_identity_map(struct device *dev,
@@ -3625,14 +3637,6 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
 	if (ret)
 		goto error;
 
-	/* it's a non-present to present mapping. Only flush if caching mode */
-	if (cap_caching_mode(iommu->cap))
-		iommu_flush_iotlb_psi(iommu, domain,
-				      mm_to_dma_pfn(iova_pfn),
-				      size, 0, 1);
-	else
-		iommu_flush_write_buffer(iommu);
-
 	start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
 	start_paddr += paddr & ~PAGE_MASK;
 	return start_paddr;
@@ -3819,12 +3823,6 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele
 		return 0;
 	}
 
-	/* it's a non-present to present mapping. Only flush if caching mode */
-	if (cap_caching_mode(iommu->cap))
-		iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
-	else
-		iommu_flush_write_buffer(iommu);
-
 	return nelems;
 }
 
diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index e8cd984cf9c8..45f6e581cd56 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -319,7 +319,7 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
 	} else
 		pasid_max = 1 << 20;
 
-	if ((flags & SVM_FLAG_SUPERVISOR_MODE)) {
+	if (flags & SVM_FLAG_SUPERVISOR_MODE) {
 		if (!ecap_srs(iommu->ecap))
 			return -EINVAL;
 	} else if (pasid) {
diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c
index 10e4a3d11c02..50e3a9fcf43e 100644
--- a/drivers/iommu/io-pgtable-arm-v7s.c
+++ b/drivers/iommu/io-pgtable-arm-v7s.c
@@ -898,8 +898,7 @@ static int __init arm_v7s_do_selftests(void)
 
 	/* Full unmap */
 	iova = 0;
-	i = find_first_bit(&cfg.pgsize_bitmap, BITS_PER_LONG);
-	while (i != BITS_PER_LONG) {
+	for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) {
 		size = 1UL << i;
 
 		if (ops->unmap(ops, iova, size) != size)
@@ -916,8 +915,6 @@ static int __init arm_v7s_do_selftests(void)
 			return __FAIL(ops);
 
 		iova += SZ_16M;
-		i++;
-		i = find_next_bit(&cfg.pgsize_bitmap, BITS_PER_LONG, i);
 	}
 
 	free_io_pgtable_ops(ops);
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index 39c2a056da21..010a254305dd 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -231,12 +231,17 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
 				    struct io_pgtable_cfg *cfg)
 {
 	struct device *dev = cfg->iommu_dev;
+	int order = get_order(size);
+	struct page *p;
 	dma_addr_t dma;
-	void *pages = alloc_pages_exact(size, gfp | __GFP_ZERO);
+	void *pages;
 
-	if (!pages)
+	VM_BUG_ON((gfp & __GFP_HIGHMEM));
+	p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order);
+	if (!p)
 		return NULL;
 
+	pages = page_address(p);
 	if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) {
 		dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE);
 		if (dma_mapping_error(dev, dma))
@@ -256,7 +261,7 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
 	dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
 	dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
 out_free:
-	free_pages_exact(pages, size);
+	__free_pages(p, order);
 	return NULL;
 }
 
@@ -266,7 +271,7 @@ static void __arm_lpae_free_pages(void *pages, size_t size,
 	if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA))
 		dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
 				 size, DMA_TO_DEVICE);
-	free_pages_exact(pages, size);
+	free_pages((unsigned long)pages, get_order(size));
 }
 
 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep,
@@ -1120,8 +1125,7 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
 
 		/* Full unmap */
 		iova = 0;
-		j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG);
-		while (j != BITS_PER_LONG) {
+		for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
 			size = 1UL << j;
 
 			if (ops->unmap(ops, iova, size) != size)
@@ -1138,8 +1142,6 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
 				return __FAIL(ops, i);
 
 			iova += SZ_1G;
-			j++;
-			j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j);
 		}
 
 		free_io_pgtable_ops(ops);
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index d2aa23202bb9..63b37563db7e 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -116,9 +116,11 @@ static void __iommu_detach_group(struct iommu_domain *domain,
 static int __init iommu_set_def_domain_type(char *str)
 {
 	bool pt;
+	int ret;
 
-	if (!str || strtobool(str, &pt))
-		return -EINVAL;
+	ret = kstrtobool(str, &pt);
+	if (ret)
+		return ret;
 
 	iommu_def_domain_type = pt ? IOMMU_DOMAIN_IDENTITY : IOMMU_DOMAIN_DMA;
 	return 0;
@@ -322,7 +324,6 @@ static struct kobj_type iommu_group_ktype = {
 
 /**
  * iommu_group_alloc - Allocate a new group
- * @name: Optional name to associate with group, visible in sysfs
  *
  * This function is called by an iommu driver to allocate a new iommu
  * group.  The iommu group represents the minimum granularity of the iommu.
diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c
index 65b9c99707f8..fe88a4880d3a 100644
--- a/drivers/iommu/qcom_iommu.c
+++ b/drivers/iommu/qcom_iommu.c
@@ -885,16 +885,14 @@ static int qcom_iommu_device_remove(struct platform_device *pdev)
 
 static int __maybe_unused qcom_iommu_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+	struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev);
 
 	return qcom_iommu_enable_clocks(qcom_iommu);
 }
 
 static int __maybe_unused qcom_iommu_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+	struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev);
 
 	qcom_iommu_disable_clocks(qcom_iommu);
 
diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
index b62f790ad1ba..89ec24c6952c 100644
--- a/drivers/iommu/tegra-gart.c
+++ b/drivers/iommu/tegra-gart.c
@@ -72,6 +72,8 @@ struct gart_domain {
 
 static struct gart_device *gart_handle; /* unique for a system */
 
+static bool gart_debug;
+
 #define GART_PTE(_pfn)						\
 	(GART_ENTRY_PHYS_ADDR_VALID | ((_pfn) << PAGE_SHIFT))
 
@@ -271,6 +273,7 @@ static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
 	struct gart_device *gart = gart_domain->gart;
 	unsigned long flags;
 	unsigned long pfn;
+	unsigned long pte;
 
 	if (!gart_iova_range_valid(gart, iova, bytes))
 		return -EINVAL;
@@ -282,6 +285,14 @@ static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
 		spin_unlock_irqrestore(&gart->pte_lock, flags);
 		return -EINVAL;
 	}
+	if (gart_debug) {
+		pte = gart_read_pte(gart, iova);
+		if (pte & GART_ENTRY_PHYS_ADDR_VALID) {
+			spin_unlock_irqrestore(&gart->pte_lock, flags);
+			dev_err(gart->dev, "Page entry is in-use\n");
+			return -EBUSY;
+		}
+	}
 	gart_set_pte(gart, iova, GART_PTE(pfn));
 	FLUSH_GART_REGS(gart);
 	spin_unlock_irqrestore(&gart->pte_lock, flags);
@@ -302,7 +313,7 @@ static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
 	gart_set_pte(gart, iova, 0);
 	FLUSH_GART_REGS(gart);
 	spin_unlock_irqrestore(&gart->pte_lock, flags);
-	return 0;
+	return bytes;
 }
 
 static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
@@ -515,7 +526,9 @@ static void __exit tegra_gart_exit(void)
 
 subsys_initcall(tegra_gart_init);
 module_exit(tegra_gart_exit);
+module_param(gart_debug, bool, 0644);
 
+MODULE_PARM_DESC(gart_debug, "Enable GART debugging");
 MODULE_DESCRIPTION("IOMMU API for GART in Tegra20");
 MODULE_AUTHOR("Hiroshi DOYU <hdoyu@nvidia.com>");
 MODULE_ALIAS("platform:tegra-gart");
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index 5ed465ab1c76..15f268f646bf 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -27,7 +27,7 @@ obj-$(CONFIG_ARM_GIC)			+= irq-gic.o irq-gic-common.o
 obj-$(CONFIG_ARM_GIC_PM)		+= irq-gic-pm.o
 obj-$(CONFIG_ARCH_REALVIEW)		+= irq-gic-realview.o
 obj-$(CONFIG_ARM_GIC_V2M)		+= irq-gic-v2m.o
-obj-$(CONFIG_ARM_GIC_V3)		+= irq-gic-v3.o irq-gic-common.o
+obj-$(CONFIG_ARM_GIC_V3)		+= irq-gic-v3.o irq-gic-v3-mbi.o irq-gic-common.o
 obj-$(CONFIG_ARM_GIC_V3_ITS)		+= irq-gic-v3-its.o irq-gic-v3-its-platform-msi.o irq-gic-v4.o
 obj-$(CONFIG_ARM_GIC_V3_ITS_PCI)	+= irq-gic-v3-its-pci-msi.o
 obj-$(CONFIG_ARM_GIC_V3_ITS_FSL_MC)	+= irq-gic-v3-its-fsl-mc-msi.o
diff --git a/drivers/irqchip/irq-gic-v3-mbi.c b/drivers/irqchip/irq-gic-v3-mbi.c
new file mode 100644
index 000000000000..ad70e7c416e3
--- /dev/null
+++ b/drivers/irqchip/irq-gic-v3-mbi.c
@@ -0,0 +1,331 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 ARM Limited, All Rights Reserved.
+ * Author: Marc Zyngier <marc.zyngier@arm.com>
+ */
+
+#define pr_fmt(fmt) "GICv3: " fmt
+
+#include <linux/dma-iommu.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/msi.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include <linux/irqchip/arm-gic-v3.h>
+
+struct mbi_range {
+	u32			spi_start;
+	u32			nr_spis;
+	unsigned long		*bm;
+};
+
+static struct mutex		mbi_lock;
+static phys_addr_t		mbi_phys_base;
+static struct mbi_range		*mbi_ranges;
+static unsigned int		mbi_range_nr;
+
+static struct irq_chip mbi_irq_chip = {
+	.name			= "MBI",
+	.irq_mask		= irq_chip_mask_parent,
+	.irq_unmask		= irq_chip_unmask_parent,
+	.irq_eoi		= irq_chip_eoi_parent,
+	.irq_set_type		= irq_chip_set_type_parent,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+};
+
+static int mbi_irq_gic_domain_alloc(struct irq_domain *domain,
+				       unsigned int virq,
+				       irq_hw_number_t hwirq)
+{
+	struct irq_fwspec fwspec;
+	struct irq_data *d;
+	int err;
+
+	/*
+	 * Using ACPI? There is no MBI support in the spec, you
+	 * shouldn't even be here.
+	 */
+	if (!is_of_node(domain->parent->fwnode))
+		return -EINVAL;
+
+	/*
+	 * Let's default to edge. This is consistent with traditional
+	 * MSIs, and systems requiring level signaling will just
+	 * enforce the trigger on their own.
+	 */
+	fwspec.fwnode = domain->parent->fwnode;
+	fwspec.param_count = 3;
+	fwspec.param[0] = 0;
+	fwspec.param[1] = hwirq - 32;
+	fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
+
+	err = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
+	if (err)
+		return err;
+
+	d = irq_domain_get_irq_data(domain->parent, virq);
+	return d->chip->irq_set_type(d, IRQ_TYPE_EDGE_RISING);
+}
+
+static void mbi_free_msi(struct mbi_range *mbi, unsigned int hwirq,
+			 int nr_irqs)
+{
+	mutex_lock(&mbi_lock);
+	bitmap_release_region(mbi->bm, hwirq - mbi->spi_start,
+			      get_count_order(nr_irqs));
+	mutex_unlock(&mbi_lock);
+}
+
+static int mbi_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
+				   unsigned int nr_irqs, void *args)
+{
+	struct mbi_range *mbi = NULL;
+	int hwirq, offset, i, err = 0;
+
+	mutex_lock(&mbi_lock);
+	for (i = 0; i < mbi_range_nr; i++) {
+		offset = bitmap_find_free_region(mbi_ranges[i].bm,
+						 mbi_ranges[i].nr_spis,
+						 get_count_order(nr_irqs));
+		if (offset >= 0) {
+			mbi = &mbi_ranges[i];
+			break;
+		}
+	}
+	mutex_unlock(&mbi_lock);
+
+	if (!mbi)
+		return -ENOSPC;
+
+	hwirq = mbi->spi_start + offset;
+
+	for (i = 0; i < nr_irqs; i++) {
+		err = mbi_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
+		if (err)
+			goto fail;
+
+		irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
+					      &mbi_irq_chip, mbi);
+	}
+
+	return 0;
+
+fail:
+	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
+	mbi_free_msi(mbi, hwirq, nr_irqs);
+	return err;
+}
+
+static void mbi_irq_domain_free(struct irq_domain *domain,
+				unsigned int virq, unsigned int nr_irqs)
+{
+	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
+	struct mbi_range *mbi = irq_data_get_irq_chip_data(d);
+
+	mbi_free_msi(mbi, d->hwirq, nr_irqs);
+	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
+}
+
+static const struct irq_domain_ops mbi_domain_ops = {
+	.alloc			= mbi_irq_domain_alloc,
+	.free			= mbi_irq_domain_free,
+};
+
+static void mbi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+	msg[0].address_hi = upper_32_bits(mbi_phys_base + GICD_SETSPI_NSR);
+	msg[0].address_lo = lower_32_bits(mbi_phys_base + GICD_SETSPI_NSR);
+	msg[0].data = data->parent_data->hwirq;
+
+	iommu_dma_map_msi_msg(data->irq, msg);
+}
+
+#ifdef CONFIG_PCI_MSI
+/* PCI-specific irqchip */
+static void mbi_mask_msi_irq(struct irq_data *d)
+{
+	pci_msi_mask_irq(d);
+	irq_chip_mask_parent(d);
+}
+
+static void mbi_unmask_msi_irq(struct irq_data *d)
+{
+	pci_msi_unmask_irq(d);
+	irq_chip_unmask_parent(d);
+}
+
+static struct irq_chip mbi_msi_irq_chip = {
+	.name			= "MSI",
+	.irq_mask		= mbi_mask_msi_irq,
+	.irq_unmask		= mbi_unmask_msi_irq,
+	.irq_eoi		= irq_chip_eoi_parent,
+	.irq_compose_msi_msg	= mbi_compose_msi_msg,
+	.irq_write_msi_msg	= pci_msi_domain_write_msg,
+};
+
+static struct msi_domain_info mbi_msi_domain_info = {
+	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+		   MSI_FLAG_PCI_MSIX | MSI_FLAG_MULTI_PCI_MSI),
+	.chip	= &mbi_msi_irq_chip,
+};
+
+static int mbi_allocate_pci_domain(struct irq_domain *nexus_domain,
+				   struct irq_domain **pci_domain)
+{
+	*pci_domain = pci_msi_create_irq_domain(nexus_domain->parent->fwnode,
+						&mbi_msi_domain_info,
+						nexus_domain);
+	if (!*pci_domain)
+		return -ENOMEM;
+
+	return 0;
+}
+#else
+static int mbi_allocate_pci_domain(struct irq_domain *nexus_domain,
+				   struct irq_domain **pci_domain)
+{
+	*pci_domain = NULL;
+	return 0;
+}
+#endif
+
+static void mbi_compose_mbi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+	mbi_compose_msi_msg(data, msg);
+
+	msg[1].address_hi = upper_32_bits(mbi_phys_base + GICD_CLRSPI_NSR);
+	msg[1].address_lo = lower_32_bits(mbi_phys_base + GICD_CLRSPI_NSR);
+	msg[1].data = data->parent_data->hwirq;
+
+	iommu_dma_map_msi_msg(data->irq, &msg[1]);
+}
+
+/* Platform-MSI specific irqchip */
+static struct irq_chip mbi_pmsi_irq_chip = {
+	.name			= "pMSI",
+	.irq_set_type		= irq_chip_set_type_parent,
+	.irq_compose_msi_msg	= mbi_compose_mbi_msg,
+	.flags			= IRQCHIP_SUPPORTS_LEVEL_MSI,
+};
+
+static struct msi_domain_ops mbi_pmsi_ops = {
+};
+
+static struct msi_domain_info mbi_pmsi_domain_info = {
+	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+		   MSI_FLAG_LEVEL_CAPABLE),
+	.ops	= &mbi_pmsi_ops,
+	.chip	= &mbi_pmsi_irq_chip,
+};
+
+static int mbi_allocate_domains(struct irq_domain *parent)
+{
+	struct irq_domain *nexus_domain, *pci_domain, *plat_domain;
+	int err;
+
+	nexus_domain = irq_domain_create_tree(parent->fwnode,
+					      &mbi_domain_ops, NULL);
+	if (!nexus_domain)
+		return -ENOMEM;
+
+	irq_domain_update_bus_token(nexus_domain, DOMAIN_BUS_NEXUS);
+	nexus_domain->parent = parent;
+
+	err = mbi_allocate_pci_domain(nexus_domain, &pci_domain);
+
+	plat_domain = platform_msi_create_irq_domain(parent->fwnode,
+						     &mbi_pmsi_domain_info,
+						     nexus_domain);
+
+	if (err || !plat_domain) {
+		if (plat_domain)
+			irq_domain_remove(plat_domain);
+		if (pci_domain)
+			irq_domain_remove(pci_domain);
+		irq_domain_remove(nexus_domain);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+int __init mbi_init(struct fwnode_handle *fwnode, struct irq_domain *parent)
+{
+	struct device_node *np;
+	const __be32 *reg;
+	int ret, n;
+
+	np = to_of_node(fwnode);
+
+	if (!of_property_read_bool(np, "msi-controller"))
+		return 0;
+
+	n = of_property_count_elems_of_size(np, "mbi-ranges", sizeof(u32));
+	if (n <= 0 || n % 2)
+		return -EINVAL;
+
+	mbi_range_nr = n / 2;
+	mbi_ranges = kcalloc(mbi_range_nr, sizeof(*mbi_ranges), GFP_KERNEL);
+	if (!mbi_ranges)
+		return -ENOMEM;
+
+	for (n = 0; n < mbi_range_nr; n++) {
+		ret = of_property_read_u32_index(np, "mbi-ranges", n * 2,
+						 &mbi_ranges[n].spi_start);
+		if (ret)
+			goto err_free_mbi;
+		ret = of_property_read_u32_index(np, "mbi-ranges", n * 2 + 1,
+						 &mbi_ranges[n].nr_spis);
+		if (ret)
+			goto err_free_mbi;
+
+		mbi_ranges[n].bm = kcalloc(BITS_TO_LONGS(mbi_ranges[n].nr_spis),
+					   sizeof(long), GFP_KERNEL);
+		if (!mbi_ranges[n].bm) {
+			ret = -ENOMEM;
+			goto err_free_mbi;
+		}
+		pr_info("MBI range [%d:%d]\n", mbi_ranges[n].spi_start,
+			mbi_ranges[n].spi_start + mbi_ranges[n].nr_spis - 1);
+	}
+
+	reg = of_get_property(np, "mbi-alias", NULL);
+	if (reg) {
+		mbi_phys_base = of_translate_address(np, reg);
+		if (mbi_phys_base == OF_BAD_ADDR) {
+			ret = -ENXIO;
+			goto err_free_mbi;
+		}
+	} else {
+		struct resource res;
+
+		if (of_address_to_resource(np, 0, &res)) {
+			ret = -ENXIO;
+			goto err_free_mbi;
+		}
+
+		mbi_phys_base = res.start;
+	}
+
+	pr_info("Using MBI frame %pa\n", &mbi_phys_base);
+
+	ret = mbi_allocate_domains(parent);
+	if (ret)
+		goto err_free_mbi;
+
+	return 0;
+
+err_free_mbi:
+	if (mbi_ranges) {
+		for (n = 0; n < mbi_range_nr; n++)
+			kfree(mbi_ranges[n].bm);
+		kfree(mbi_ranges);
+	}
+
+	return ret;
+}
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index e5d101418390..5a67ec084588 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -1099,6 +1099,7 @@ static int __init gic_init_bases(void __iomem *dist_base,
 
 	gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
 						 &gic_data);
+	irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
 	gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
 	gic_data.rdists.has_vlpis = true;
 	gic_data.rdists.has_direct_lpi = true;
@@ -1112,6 +1113,12 @@ static int __init gic_init_bases(void __iomem *dist_base,
 	pr_info("Distributor has %sRange Selector support\n",
 		gic_data.has_rss ? "" : "no ");
 
+	if (typer & GICD_TYPER_MBIS) {
+		err = mbi_init(handle, gic_data.domain);
+		if (err)
+			pr_err("Failed to initialize MBIs\n");
+	}
+
 	set_handle_irq(gic_handle_irq);
 
 	gic_update_vlpi_properties();
diff --git a/drivers/irqchip/irq-meson-gpio.c b/drivers/irqchip/irq-meson-gpio.c
index a59bdbc0b9bb..7b531fd075b8 100644
--- a/drivers/irqchip/irq-meson-gpio.c
+++ b/drivers/irqchip/irq-meson-gpio.c
@@ -63,11 +63,16 @@ static const struct meson_gpio_irq_params gxl_params = {
 	.nr_hwirq = 110,
 };
 
+static const struct meson_gpio_irq_params axg_params = {
+	.nr_hwirq = 100,
+};
+
 static const struct of_device_id meson_irq_gpio_matches[] = {
 	{ .compatible = "amlogic,meson8-gpio-intc", .data = &meson8_params },
 	{ .compatible = "amlogic,meson8b-gpio-intc", .data = &meson8b_params },
 	{ .compatible = "amlogic,meson-gxbb-gpio-intc", .data = &gxbb_params },
 	{ .compatible = "amlogic,meson-gxl-gpio-intc", .data = &gxl_params },
+	{ .compatible = "amlogic,meson-axg-gpio-intc", .data = &axg_params },
 	{ }
 };
 
diff --git a/drivers/irqchip/irq-mvebu-gicp.c b/drivers/irqchip/irq-mvebu-gicp.c
index 17a4a7b6cdbb..4e17f7081efc 100644
--- a/drivers/irqchip/irq-mvebu-gicp.c
+++ b/drivers/irqchip/irq-mvebu-gicp.c
@@ -19,8 +19,6 @@
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 
-#include "irq-mvebu-gicp.h"
-
 #define GICP_SETSPI_NSR_OFFSET	0x0
 #define GICP_CLRSPI_NSR_OFFSET	0x8
 
@@ -55,34 +53,18 @@ static int gicp_idx_to_spi(struct mvebu_gicp *gicp, int idx)
 	return -EINVAL;
 }
 
-int mvebu_gicp_get_doorbells(struct device_node *dn, phys_addr_t *setspi,
-			     phys_addr_t *clrspi)
-{
-	struct platform_device *pdev;
-	struct mvebu_gicp *gicp;
-
-	pdev = of_find_device_by_node(dn);
-	if (!pdev)
-		return -ENODEV;
-
-	gicp = platform_get_drvdata(pdev);
-	if (!gicp)
-		return -ENODEV;
-
-	*setspi = gicp->res->start + GICP_SETSPI_NSR_OFFSET;
-	*clrspi = gicp->res->start + GICP_CLRSPI_NSR_OFFSET;
-
-	return 0;
-}
-
 static void gicp_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 {
 	struct mvebu_gicp *gicp = data->chip_data;
 	phys_addr_t setspi = gicp->res->start + GICP_SETSPI_NSR_OFFSET;
-
-	msg->data = data->hwirq;
-	msg->address_lo = lower_32_bits(setspi);
-	msg->address_hi = upper_32_bits(setspi);
+	phys_addr_t clrspi = gicp->res->start + GICP_CLRSPI_NSR_OFFSET;
+
+	msg[0].data = data->hwirq;
+	msg[0].address_lo = lower_32_bits(setspi);
+	msg[0].address_hi = upper_32_bits(setspi);
+	msg[1].data = data->hwirq;
+	msg[1].address_lo = lower_32_bits(clrspi);
+	msg[1].address_hi = upper_32_bits(clrspi);
 }
 
 static struct irq_chip gicp_irq_chip = {
@@ -170,13 +152,15 @@ static const struct irq_domain_ops gicp_domain_ops = {
 static struct irq_chip gicp_msi_irq_chip = {
 	.name		= "GICP",
 	.irq_set_type	= irq_chip_set_type_parent,
+	.flags		= IRQCHIP_SUPPORTS_LEVEL_MSI,
 };
 
 static struct msi_domain_ops gicp_msi_ops = {
 };
 
 static struct msi_domain_info gicp_msi_domain_info = {
-	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
+	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+		   MSI_FLAG_LEVEL_CAPABLE),
 	.ops	= &gicp_msi_ops,
 	.chip	= &gicp_msi_irq_chip,
 };
diff --git a/drivers/irqchip/irq-mvebu-gicp.h b/drivers/irqchip/irq-mvebu-gicp.h
deleted file mode 100644
index eaa12fb72102..000000000000
--- a/drivers/irqchip/irq-mvebu-gicp.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __MVEBU_GICP_H__
-#define __MVEBU_GICP_H__
-
-#include <linux/types.h>
-
-struct device_node;
-
-int mvebu_gicp_get_doorbells(struct device_node *dn, phys_addr_t *setspi,
-			     phys_addr_t *clrspi);
-
-#endif /* __MVEBU_GICP_H__ */
diff --git a/drivers/irqchip/irq-mvebu-icu.c b/drivers/irqchip/irq-mvebu-icu.c
index e18c48d3a92e..13063339b416 100644
--- a/drivers/irqchip/irq-mvebu-icu.c
+++ b/drivers/irqchip/irq-mvebu-icu.c
@@ -21,8 +21,6 @@
 
 #include <dt-bindings/interrupt-controller/mvebu-icu.h>
 
-#include "irq-mvebu-gicp.h"
-
 /* ICU registers */
 #define ICU_SETSPI_NSR_AL	0x10
 #define ICU_SETSPI_NSR_AH	0x14
@@ -43,6 +41,7 @@ struct mvebu_icu {
 	void __iomem *base;
 	struct irq_domain *domain;
 	struct device *dev;
+	atomic_t initialized;
 };
 
 struct mvebu_icu_irq_data {
@@ -51,6 +50,18 @@ struct mvebu_icu_irq_data {
 	unsigned int type;
 };
 
+static void mvebu_icu_init(struct mvebu_icu *icu, struct msi_msg *msg)
+{
+	if (atomic_cmpxchg(&icu->initialized, false, true))
+		return;
+
+	/* Set Clear/Set ICU SPI message address in AP */
+	writel_relaxed(msg[0].address_hi, icu->base + ICU_SETSPI_NSR_AH);
+	writel_relaxed(msg[0].address_lo, icu->base + ICU_SETSPI_NSR_AL);
+	writel_relaxed(msg[1].address_hi, icu->base + ICU_CLRSPI_NSR_AH);
+	writel_relaxed(msg[1].address_lo, icu->base + ICU_CLRSPI_NSR_AL);
+}
+
 static void mvebu_icu_write_msg(struct msi_desc *desc, struct msi_msg *msg)
 {
 	struct irq_data *d = irq_get_irq_data(desc->irq);
@@ -59,6 +70,8 @@ static void mvebu_icu_write_msg(struct msi_desc *desc, struct msi_msg *msg)
 	unsigned int icu_int;
 
 	if (msg->address_lo || msg->address_hi) {
+		/* One off initialization */
+		mvebu_icu_init(icu, msg);
 		/* Configure the ICU with irq number & type */
 		icu_int = msg->data | ICU_INT_ENABLE;
 		if (icu_irqd->type & IRQ_TYPE_EDGE_RISING)
@@ -197,9 +210,7 @@ static int mvebu_icu_probe(struct platform_device *pdev)
 	struct device_node *node = pdev->dev.of_node;
 	struct device_node *gicp_dn;
 	struct resource *res;
-	phys_addr_t setspi, clrspi;
-	u32 i, icu_int;
-	int ret;
+	int i;
 
 	icu = devm_kzalloc(&pdev->dev, sizeof(struct mvebu_icu),
 			   GFP_KERNEL);
@@ -242,22 +253,12 @@ static int mvebu_icu_probe(struct platform_device *pdev)
 	if (!gicp_dn)
 		return -ENODEV;
 
-	ret = mvebu_gicp_get_doorbells(gicp_dn, &setspi, &clrspi);
-	if (ret)
-		return ret;
-
-	/* Set Clear/Set ICU SPI message address in AP */
-	writel_relaxed(upper_32_bits(setspi), icu->base + ICU_SETSPI_NSR_AH);
-	writel_relaxed(lower_32_bits(setspi), icu->base + ICU_SETSPI_NSR_AL);
-	writel_relaxed(upper_32_bits(clrspi), icu->base + ICU_CLRSPI_NSR_AH);
-	writel_relaxed(lower_32_bits(clrspi), icu->base + ICU_CLRSPI_NSR_AL);
-
 	/*
 	 * Clean all ICU interrupts with type SPI_NSR, required to
 	 * avoid unpredictable SPI assignments done by firmware.
 	 */
 	for (i = 0 ; i < ICU_MAX_IRQS ; i++) {
-		icu_int = readl(icu->base + ICU_INT_CFG(i));
+		u32 icu_int = readl_relaxed(icu->base + ICU_INT_CFG(i));
 		if ((icu_int >> ICU_GROUP_SHIFT) == ICU_GRP_NSR)
 			writel_relaxed(0x0, icu->base + ICU_INT_CFG(i));
 	}
diff --git a/drivers/isdn/capi/capi.c b/drivers/isdn/capi/capi.c
index 19cd93783c87..baa1ee2bc2ac 100644
--- a/drivers/isdn/capi/capi.c
+++ b/drivers/isdn/capi/capi.c
@@ -1340,19 +1340,6 @@ static int capi20_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int capi20_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, capi20_proc_show, NULL);
-}
-
-static const struct file_operations capi20_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= capi20_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 /*
  * /proc/capi/capi20ncci:
  *  applid ncci
@@ -1373,23 +1360,10 @@ static int capi20ncci_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int capi20ncci_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, capi20ncci_proc_show, NULL);
-}
-
-static const struct file_operations capi20ncci_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= capi20ncci_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static void __init proc_init(void)
 {
-	proc_create("capi/capi20", 0, NULL, &capi20_proc_fops);
-	proc_create("capi/capi20ncci", 0, NULL, &capi20ncci_proc_fops);
+	proc_create_single("capi/capi20", 0, NULL, capi20_proc_show);
+	proc_create_single("capi/capi20ncci", 0, NULL, capi20ncci_proc_show);
 }
 
 static void __exit proc_exit(void)
diff --git a/drivers/isdn/capi/capidrv.c b/drivers/isdn/capi/capidrv.c
index 49fef08858c5..7ac51798949d 100644
--- a/drivers/isdn/capi/capidrv.c
+++ b/drivers/isdn/capi/capidrv.c
@@ -2460,22 +2460,9 @@ static int capidrv_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int capidrv_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, capidrv_proc_show, NULL);
-}
-
-static const struct file_operations capidrv_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= capidrv_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static void __init proc_init(void)
 {
-	proc_create("capi/capidrv", 0, NULL, &capidrv_proc_fops);
+	proc_create_single("capi/capidrv", 0, NULL, capidrv_proc_show);
 }
 
 static void __exit proc_exit(void)
diff --git a/drivers/isdn/capi/kcapi.c b/drivers/isdn/capi/kcapi.c
index 46c189ad8d94..0ff517d3c98f 100644
--- a/drivers/isdn/capi/kcapi.c
+++ b/drivers/isdn/capi/kcapi.c
@@ -534,7 +534,8 @@ int attach_capi_ctr(struct capi_ctr *ctr)
 	init_waitqueue_head(&ctr->state_wait_queue);
 
 	sprintf(ctr->procfn, "capi/controllers/%d", ctr->cnr);
-	ctr->procent = proc_create_data(ctr->procfn, 0, NULL, ctr->proc_fops, ctr);
+	ctr->procent = proc_create_single_data(ctr->procfn, 0, NULL,
+			ctr->proc_show, ctr);
 
 	ncontrollers++;
 
diff --git a/drivers/isdn/capi/kcapi_proc.c b/drivers/isdn/capi/kcapi_proc.c
index 68db3c5a1063..c94bd12c0f7c 100644
--- a/drivers/isdn/capi/kcapi_proc.c
+++ b/drivers/isdn/capi/kcapi_proc.c
@@ -108,32 +108,6 @@ static const struct seq_operations seq_contrstats_ops = {
 	.show	= contrstats_show,
 };
 
-static int seq_controller_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &seq_controller_ops);
-}
-
-static int seq_contrstats_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &seq_contrstats_ops);
-}
-
-static const struct file_operations proc_controller_ops = {
-	.owner		= THIS_MODULE,
-	.open		= seq_controller_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
-static const struct file_operations proc_contrstats_ops = {
-	.owner		= THIS_MODULE,
-	.open		= seq_contrstats_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
 // /proc/capi/applications:
 //      applid l3cnt dblkcnt dblklen #ncci recvqueuelen
 // /proc/capi/applstats:
@@ -216,34 +190,6 @@ static const struct seq_operations seq_applstats_ops = {
 	.show	= applstats_show,
 };
 
-static int
-seq_applications_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &seq_applications_ops);
-}
-
-static int
-seq_applstats_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &seq_applstats_ops);
-}
-
-static const struct file_operations proc_applications_ops = {
-	.owner		= THIS_MODULE,
-	.open		= seq_applications_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
-static const struct file_operations proc_applstats_ops = {
-	.owner		= THIS_MODULE,
-	.open		= seq_applstats_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
 // ---------------------------------------------------------------------------
 
 static void *capi_driver_start(struct seq_file *seq, loff_t *pos)
@@ -279,22 +225,6 @@ static const struct seq_operations seq_capi_driver_ops = {
 	.show	= capi_driver_show,
 };
 
-static int
-seq_capi_driver_open(struct inode *inode, struct file *file)
-{
-	int err;
-	err = seq_open(file, &seq_capi_driver_ops);
-	return err;
-}
-
-static const struct file_operations proc_driver_ops = {
-	.owner		= THIS_MODULE,
-	.open		= seq_capi_driver_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
 // ---------------------------------------------------------------------------
 
 void __init
@@ -302,11 +232,11 @@ kcapi_proc_init(void)
 {
 	proc_mkdir("capi",             NULL);
 	proc_mkdir("capi/controllers", NULL);
-	proc_create("capi/controller",   0, NULL, &proc_controller_ops);
-	proc_create("capi/contrstats",   0, NULL, &proc_contrstats_ops);
-	proc_create("capi/applications", 0, NULL, &proc_applications_ops);
-	proc_create("capi/applstats",    0, NULL, &proc_applstats_ops);
-	proc_create("capi/driver",       0, NULL, &proc_driver_ops);
+	proc_create_seq("capi/controller",   0, NULL, &seq_controller_ops);
+	proc_create_seq("capi/contrstats",   0, NULL, &seq_contrstats_ops);
+	proc_create_seq("capi/applications", 0, NULL, &seq_applications_ops);
+	proc_create_seq("capi/applstats",    0, NULL, &seq_applstats_ops);
+	proc_create_seq("capi/driver",       0, NULL, &seq_capi_driver_ops);
 }
 
 void __exit
diff --git a/drivers/isdn/gigaset/capi.c b/drivers/isdn/gigaset/capi.c
index ccec7778cad2..56748af78c04 100644
--- a/drivers/isdn/gigaset/capi.c
+++ b/drivers/isdn/gigaset/capi.c
@@ -2437,19 +2437,6 @@ static int gigaset_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int gigaset_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, gigaset_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations gigaset_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= gigaset_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 /**
  * gigaset_isdn_regdev() - register device to LL
  * @cs:		device descriptor structure.
@@ -2479,7 +2466,7 @@ int gigaset_isdn_regdev(struct cardstate *cs, const char *isdnid)
 	iif->ctr.release_appl  = gigaset_release_appl;
 	iif->ctr.send_message  = gigaset_send_message;
 	iif->ctr.procinfo      = gigaset_procinfo;
-	iif->ctr.proc_fops = &gigaset_proc_fops;
+	iif->ctr.proc_show     = gigaset_proc_show,
 	INIT_LIST_HEAD(&iif->appls);
 	skb_queue_head_init(&iif->sendqueue);
 	atomic_set(&iif->sendqlen, 0);
diff --git a/drivers/isdn/hardware/avm/avmcard.h b/drivers/isdn/hardware/avm/avmcard.h
index c95712dbfa9f..cdfa89c71997 100644
--- a/drivers/isdn/hardware/avm/avmcard.h
+++ b/drivers/isdn/hardware/avm/avmcard.h
@@ -556,7 +556,7 @@ u16  b1_send_message(struct capi_ctr *ctrl, struct sk_buff *skb);
 void b1_parse_version(avmctrl_info *card);
 irqreturn_t b1_interrupt(int interrupt, void *devptr);
 
-extern const struct file_operations b1ctl_proc_fops;
+int b1_proc_show(struct seq_file *m, void *v);
 
 avmcard_dmainfo *avmcard_dma_alloc(char *name, struct pci_dev *,
 				   long rsize, long ssize);
@@ -576,6 +576,6 @@ void b1dma_register_appl(struct capi_ctr *ctrl,
 			 capi_register_params *rp);
 void b1dma_release_appl(struct capi_ctr *ctrl, u16 appl);
 u16  b1dma_send_message(struct capi_ctr *ctrl, struct sk_buff *skb);
-extern const struct file_operations b1dmactl_proc_fops;
+int b1dma_proc_show(struct seq_file *m, void *v);
 
 #endif /* _AVMCARD_H_ */
diff --git a/drivers/isdn/hardware/avm/b1.c b/drivers/isdn/hardware/avm/b1.c
index b1833d08a5fe..5ee5489d3f15 100644
--- a/drivers/isdn/hardware/avm/b1.c
+++ b/drivers/isdn/hardware/avm/b1.c
@@ -637,7 +637,7 @@ irqreturn_t b1_interrupt(int interrupt, void *devptr)
 }
 
 /* ------------------------------------------------------------- */
-static int b1ctl_proc_show(struct seq_file *m, void *v)
+int b1_proc_show(struct seq_file *m, void *v)
 {
 	struct capi_ctr *ctrl = m->private;
 	avmctrl_info *cinfo = (avmctrl_info *)(ctrl->driverdata);
@@ -699,20 +699,7 @@ static int b1ctl_proc_show(struct seq_file *m, void *v)
 
 	return 0;
 }
-
-static int b1ctl_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, b1ctl_proc_show, PDE_DATA(inode));
-}
-
-const struct file_operations b1ctl_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= b1ctl_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-EXPORT_SYMBOL(b1ctl_proc_fops);
+EXPORT_SYMBOL(b1_proc_show);
 
 /* ------------------------------------------------------------- */
 
diff --git a/drivers/isdn/hardware/avm/b1dma.c b/drivers/isdn/hardware/avm/b1dma.c
index 9538a9e5e1a8..6a3dc9937ce5 100644
--- a/drivers/isdn/hardware/avm/b1dma.c
+++ b/drivers/isdn/hardware/avm/b1dma.c
@@ -858,7 +858,7 @@ u16 b1dma_send_message(struct capi_ctr *ctrl, struct sk_buff *skb)
 
 /* ------------------------------------------------------------- */
 
-static int b1dmactl_proc_show(struct seq_file *m, void *v)
+int b1dma_proc_show(struct seq_file *m, void *v)
 {
 	struct capi_ctr *ctrl = m->private;
 	avmctrl_info *cinfo = (avmctrl_info *)(ctrl->driverdata);
@@ -941,20 +941,7 @@ static int b1dmactl_proc_show(struct seq_file *m, void *v)
 
 	return 0;
 }
-
-static int b1dmactl_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, b1dmactl_proc_show, PDE_DATA(inode));
-}
-
-const struct file_operations b1dmactl_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= b1dmactl_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-EXPORT_SYMBOL(b1dmactl_proc_fops);
+EXPORT_SYMBOL(b1dma_proc_show);
 
 /* ------------------------------------------------------------- */
 
diff --git a/drivers/isdn/hardware/avm/b1isa.c b/drivers/isdn/hardware/avm/b1isa.c
index 54e871a47387..cdfea72e0ef6 100644
--- a/drivers/isdn/hardware/avm/b1isa.c
+++ b/drivers/isdn/hardware/avm/b1isa.c
@@ -121,7 +121,7 @@ static int b1isa_probe(struct pci_dev *pdev)
 	cinfo->capi_ctrl.load_firmware = b1_load_firmware;
 	cinfo->capi_ctrl.reset_ctr     = b1_reset_ctr;
 	cinfo->capi_ctrl.procinfo      = b1isa_procinfo;
-	cinfo->capi_ctrl.proc_fops = &b1ctl_proc_fops;
+	cinfo->capi_ctrl.proc_show     = b1_proc_show;
 	strcpy(cinfo->capi_ctrl.name, card->name);
 
 	retval = attach_capi_ctr(&cinfo->capi_ctrl);
diff --git a/drivers/isdn/hardware/avm/b1pci.c b/drivers/isdn/hardware/avm/b1pci.c
index ac4863c2ecbc..b76b57a82c02 100644
--- a/drivers/isdn/hardware/avm/b1pci.c
+++ b/drivers/isdn/hardware/avm/b1pci.c
@@ -112,7 +112,7 @@ static int b1pci_probe(struct capicardparams *p, struct pci_dev *pdev)
 	cinfo->capi_ctrl.load_firmware = b1_load_firmware;
 	cinfo->capi_ctrl.reset_ctr     = b1_reset_ctr;
 	cinfo->capi_ctrl.procinfo      = b1pci_procinfo;
-	cinfo->capi_ctrl.proc_fops = &b1ctl_proc_fops;
+	cinfo->capi_ctrl.proc_show     = b1_proc_show;
 	strcpy(cinfo->capi_ctrl.name, card->name);
 	cinfo->capi_ctrl.owner         = THIS_MODULE;
 
@@ -251,7 +251,7 @@ static int b1pciv4_probe(struct capicardparams *p, struct pci_dev *pdev)
 	cinfo->capi_ctrl.load_firmware = b1dma_load_firmware;
 	cinfo->capi_ctrl.reset_ctr     = b1dma_reset_ctr;
 	cinfo->capi_ctrl.procinfo      = b1pciv4_procinfo;
-	cinfo->capi_ctrl.proc_fops = &b1dmactl_proc_fops;
+	cinfo->capi_ctrl.proc_show     = b1dma_proc_show;
 	strcpy(cinfo->capi_ctrl.name, card->name);
 
 	retval = attach_capi_ctr(&cinfo->capi_ctrl);
diff --git a/drivers/isdn/hardware/avm/b1pcmcia.c b/drivers/isdn/hardware/avm/b1pcmcia.c
index 6b0d19d963d5..3aca16e62902 100644
--- a/drivers/isdn/hardware/avm/b1pcmcia.c
+++ b/drivers/isdn/hardware/avm/b1pcmcia.c
@@ -108,7 +108,7 @@ static int b1pcmcia_add_card(unsigned int port, unsigned irq,
 	cinfo->capi_ctrl.load_firmware = b1_load_firmware;
 	cinfo->capi_ctrl.reset_ctr     = b1_reset_ctr;
 	cinfo->capi_ctrl.procinfo      = b1pcmcia_procinfo;
-	cinfo->capi_ctrl.proc_fops = &b1ctl_proc_fops;
+	cinfo->capi_ctrl.proc_show     = b1_proc_show;
 	strcpy(cinfo->capi_ctrl.name, card->name);
 
 	retval = attach_capi_ctr(&cinfo->capi_ctrl);
diff --git a/drivers/isdn/hardware/avm/c4.c b/drivers/isdn/hardware/avm/c4.c
index 034cabac699d..ac72cd204c4d 100644
--- a/drivers/isdn/hardware/avm/c4.c
+++ b/drivers/isdn/hardware/avm/c4.c
@@ -1127,19 +1127,6 @@ static int c4_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int c4_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, c4_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations c4_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= c4_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 /* ------------------------------------------------------------- */
 
 static int c4_add_card(struct capicardparams *p, struct pci_dev *dev,
@@ -1211,7 +1198,7 @@ static int c4_add_card(struct capicardparams *p, struct pci_dev *dev,
 		cinfo->capi_ctrl.load_firmware = c4_load_firmware;
 		cinfo->capi_ctrl.reset_ctr     = c4_reset_ctr;
 		cinfo->capi_ctrl.procinfo      = c4_procinfo;
-		cinfo->capi_ctrl.proc_fops = &c4_proc_fops;
+		cinfo->capi_ctrl.proc_show     = c4_proc_show;
 		strcpy(cinfo->capi_ctrl.name, card->name);
 
 		retval = attach_capi_ctr(&cinfo->capi_ctrl);
diff --git a/drivers/isdn/hardware/avm/t1isa.c b/drivers/isdn/hardware/avm/t1isa.c
index 9f80d20ced87..2153619c5b31 100644
--- a/drivers/isdn/hardware/avm/t1isa.c
+++ b/drivers/isdn/hardware/avm/t1isa.c
@@ -430,7 +430,7 @@ static int t1isa_probe(struct pci_dev *pdev, int cardnr)
 	cinfo->capi_ctrl.load_firmware = t1isa_load_firmware;
 	cinfo->capi_ctrl.reset_ctr     = t1isa_reset_ctr;
 	cinfo->capi_ctrl.procinfo      = t1isa_procinfo;
-	cinfo->capi_ctrl.proc_fops = &b1ctl_proc_fops;
+	cinfo->capi_ctrl.proc_show     = b1_proc_show;
 	strcpy(cinfo->capi_ctrl.name, card->name);
 
 	retval = attach_capi_ctr(&cinfo->capi_ctrl);
diff --git a/drivers/isdn/hardware/avm/t1pci.c b/drivers/isdn/hardware/avm/t1pci.c
index 2180b1685691..f5ed1d5004c9 100644
--- a/drivers/isdn/hardware/avm/t1pci.c
+++ b/drivers/isdn/hardware/avm/t1pci.c
@@ -119,7 +119,7 @@ static int t1pci_add_card(struct capicardparams *p, struct pci_dev *pdev)
 	cinfo->capi_ctrl.load_firmware = b1dma_load_firmware;
 	cinfo->capi_ctrl.reset_ctr     = b1dma_reset_ctr;
 	cinfo->capi_ctrl.procinfo      = t1pci_procinfo;
-	cinfo->capi_ctrl.proc_fops = &b1dmactl_proc_fops;
+	cinfo->capi_ctrl.proc_show     = b1dma_proc_show;
 	strcpy(cinfo->capi_ctrl.name, card->name);
 
 	retval = attach_capi_ctr(&cinfo->capi_ctrl);
diff --git a/drivers/isdn/hardware/eicon/capimain.c b/drivers/isdn/hardware/eicon/capimain.c
index be36d82004d6..f9244dc1c3c9 100644
--- a/drivers/isdn/hardware/eicon/capimain.c
+++ b/drivers/isdn/hardware/eicon/capimain.c
@@ -90,19 +90,6 @@ static int diva_ctl_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int diva_ctl_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, diva_ctl_proc_show, NULL);
-}
-
-static const struct file_operations diva_ctl_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= diva_ctl_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 /*
  * set additional os settings in capi_ctr struct
  */
@@ -111,7 +98,7 @@ void diva_os_set_controller_struct(struct capi_ctr *ctrl)
 	ctrl->driver_name = DRIVERLNAME;
 	ctrl->load_firmware = NULL;
 	ctrl->reset_ctr = NULL;
-	ctrl->proc_fops = &diva_ctl_proc_fops;
+	ctrl->proc_show = diva_ctl_proc_show;
 	ctrl->owner = THIS_MODULE;
 }
 
diff --git a/drivers/isdn/hardware/eicon/diva_didd.c b/drivers/isdn/hardware/eicon/diva_didd.c
index fab6ccfb00d5..60e79257dd5f 100644
--- a/drivers/isdn/hardware/eicon/diva_didd.c
+++ b/drivers/isdn/hardware/eicon/diva_didd.c
@@ -78,26 +78,13 @@ static int divadidd_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int divadidd_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, divadidd_proc_show, NULL);
-}
-
-static const struct file_operations divadidd_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= divadidd_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int __init create_proc(void)
 {
 	proc_net_eicon = proc_mkdir("eicon", init_net.proc_net);
 
 	if (proc_net_eicon) {
-		proc_didd = proc_create(DRIVERLNAME, S_IRUGO, proc_net_eicon,
-					&divadidd_proc_fops);
+		proc_didd = proc_create_single(DRIVERLNAME, S_IRUGO,
+				proc_net_eicon, divadidd_proc_show);
 		return (1);
 	}
 	return (0);
diff --git a/drivers/isdn/hardware/eicon/divasi.c b/drivers/isdn/hardware/eicon/divasi.c
index 525518c945fe..e7081e0c0e35 100644
--- a/drivers/isdn/hardware/eicon/divasi.c
+++ b/drivers/isdn/hardware/eicon/divasi.c
@@ -101,23 +101,10 @@ static int um_idi_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int um_idi_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, um_idi_proc_show, NULL);
-}
-
-static const struct file_operations um_idi_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= um_idi_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int __init create_um_idi_proc(void)
 {
-	um_idi_proc_entry = proc_create(DRIVERLNAME, S_IRUGO, proc_net_eicon,
-					&um_idi_proc_fops);
+	um_idi_proc_entry = proc_create_single(DRIVERLNAME, S_IRUGO,
+			proc_net_eicon, um_idi_proc_show);
 	if (!um_idi_proc_entry)
 		return (0);
 	return (1);
diff --git a/drivers/isdn/hysdn/hycapi.c b/drivers/isdn/hysdn/hycapi.c
index eac0f51a0f60..a2c15cd7bf67 100644
--- a/drivers/isdn/hysdn/hycapi.c
+++ b/drivers/isdn/hysdn/hycapi.c
@@ -467,19 +467,6 @@ static int hycapi_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int hycapi_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, hycapi_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations hycapi_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= hycapi_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 /**************************************************************
 hycapi_load_firmware
 
@@ -774,7 +761,7 @@ hycapi_capi_create(hysdn_card *card)
 		ctrl->load_firmware = hycapi_load_firmware;
 		ctrl->reset_ctr     = hycapi_reset_ctr;
 		ctrl->procinfo      = hycapi_procinfo;
-		ctrl->proc_fops = &hycapi_proc_fops;
+		ctrl->proc_show     = hycapi_proc_show;
 		strcpy(ctrl->name, cinfo->cardname);
 		ctrl->owner = THIS_MODULE;
 
diff --git a/drivers/isdn/mISDN/socket.c b/drivers/isdn/mISDN/socket.c
index 1f8f489b4167..98f90aadd141 100644
--- a/drivers/isdn/mISDN/socket.c
+++ b/drivers/isdn/mISDN/socket.c
@@ -588,7 +588,7 @@ static const struct proto_ops data_sock_ops = {
 	.getname	= data_sock_getname,
 	.sendmsg	= mISDN_sock_sendmsg,
 	.recvmsg	= mISDN_sock_recvmsg,
-	.poll		= datagram_poll,
+	.poll_mask	= datagram_poll_mask,
 	.listen		= sock_no_listen,
 	.shutdown	= sock_no_shutdown,
 	.setsockopt	= data_sock_setsockopt,
@@ -745,7 +745,6 @@ static const struct proto_ops base_sock_ops = {
 	.getname	= sock_no_getname,
 	.sendmsg	= sock_no_sendmsg,
 	.recvmsg	= sock_no_recvmsg,
-	.poll		= sock_no_poll,
 	.listen		= sock_no_listen,
 	.shutdown	= sock_no_shutdown,
 	.setsockopt	= sock_no_setsockopt,
diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig
index 2c896c0e69e1..6e3a998f3370 100644
--- a/drivers/leds/Kconfig
+++ b/drivers/leds/Kconfig
@@ -104,6 +104,19 @@ config LEDS_CPCAP
 	  This option enables support for LEDs offered by Motorola's
 	  CPCAP PMIC.
 
+config LEDS_CR0014114
+	tristate "LED Support for Crane CR0014114"
+	depends on LEDS_CLASS
+	depends on SPI
+	depends on OF
+	help
+	  This option enables support for CR0014114 LED Board which
+	  is widely used in vending machines produced by
+	  Crane Merchandising Systems.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called leds-cr0014114.
+
 config LEDS_LM3530
 	tristate "LCD Backlight driver for LM3530"
 	depends on LEDS_CLASS
@@ -145,6 +158,15 @@ config LEDS_LM3692X
 	  This option enables support for the TI LM3692x family
 	  of white LED string drivers used for backlighting.
 
+config LEDS_LM3601X
+	tristate "LED support for LM3601x Chips"
+	depends on LEDS_CLASS && I2C
+	depends on LEDS_CLASS_FLASH
+	select REGMAP_I2C
+	help
+	  This option enables support for the TI LM3601x family
+	  of flash, torch and indicator classes.
+
 config LEDS_LOCOMO
 	tristate "LED Support for Locomo device"
 	depends on LEDS_CLASS
@@ -647,6 +669,17 @@ config LEDS_IS31FL32XX
 	  LED controllers. They are I2C devices with multiple constant-current
 	  channels, each with independent 256-level PWM control.
 
+config LEDS_SC27XX_BLTC
+	tristate "LED support for the SC27xx breathing light controller"
+	depends on LEDS_CLASS && MFD_SC27XX_PMIC
+	depends on OF
+	help
+	  Say Y here to include support for the SC27xx breathing light controller
+	  LEDs.
+
+	  This driver can also be built as a module. If so the module will be
+	  called leds-sc27xx-bltc.
+
 comment "LED driver for blink(1) USB RGB LED is under Special HID drivers (HID_THINGM)"
 
 config LEDS_BLINKM
diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile
index 91eca81cae82..420b5d2cfa62 100644
--- a/drivers/leds/Makefile
+++ b/drivers/leds/Makefile
@@ -76,8 +76,11 @@ obj-$(CONFIG_LEDS_MLXREG)		+= leds-mlxreg.o
 obj-$(CONFIG_LEDS_NIC78BX)		+= leds-nic78bx.o
 obj-$(CONFIG_LEDS_MT6323)		+= leds-mt6323.o
 obj-$(CONFIG_LEDS_LM3692X)		+= leds-lm3692x.o
+obj-$(CONFIG_LEDS_SC27XX_BLTC)		+= leds-sc27xx-bltc.o
+obj-$(CONFIG_LEDS_LM3601X)		+= leds-lm3601x.o
 
 # LED SPI Drivers
+obj-$(CONFIG_LEDS_CR0014114)		+= leds-cr0014114.o
 obj-$(CONFIG_LEDS_DAC124S085)		+= leds-dac124s085.o
 
 # LED Userspace Drivers
diff --git a/drivers/leds/led-class.c b/drivers/leds/led-class.c
index b0e2d55acbd6..3c7e3487b373 100644
--- a/drivers/leds/led-class.c
+++ b/drivers/leds/led-class.c
@@ -260,10 +260,14 @@ int of_led_classdev_register(struct device *parent, struct device_node *np,
 	if (ret < 0)
 		return ret;
 
+	mutex_init(&led_cdev->led_access);
+	mutex_lock(&led_cdev->led_access);
 	led_cdev->dev = device_create_with_groups(leds_class, parent, 0,
 				led_cdev, led_cdev->groups, "%s", name);
-	if (IS_ERR(led_cdev->dev))
+	if (IS_ERR(led_cdev->dev)) {
+		mutex_unlock(&led_cdev->led_access);
 		return PTR_ERR(led_cdev->dev);
+	}
 	led_cdev->dev->of_node = np;
 
 	if (ret)
@@ -274,6 +278,7 @@ int of_led_classdev_register(struct device *parent, struct device_node *np,
 		ret = led_add_brightness_hw_changed(led_cdev);
 		if (ret) {
 			device_unregister(led_cdev->dev);
+			mutex_unlock(&led_cdev->led_access);
 			return ret;
 		}
 	}
@@ -285,7 +290,6 @@ int of_led_classdev_register(struct device *parent, struct device_node *np,
 #ifdef CONFIG_LEDS_BRIGHTNESS_HW_CHANGED
 	led_cdev->brightness_hw_changed = -1;
 #endif
-	mutex_init(&led_cdev->led_access);
 	/* add to the list of leds */
 	down_write(&leds_list_lock);
 	list_add_tail(&led_cdev->node, &leds_list);
@@ -302,6 +306,8 @@ int of_led_classdev_register(struct device *parent, struct device_node *np,
 	led_trigger_set_default(led_cdev);
 #endif
 
+	mutex_unlock(&led_cdev->led_access);
+
 	dev_dbg(parent, "Registered led device: %s\n",
 			led_cdev->name);
 
diff --git a/drivers/leds/leds-cr0014114.c b/drivers/leds/leds-cr0014114.c
new file mode 100644
index 000000000000..a4b1c1dcce7f
--- /dev/null
+++ b/drivers/leds/leds-cr0014114.c
@@ -0,0 +1,315 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2018 Crane Merchandising Systems. All rights reserved.
+// Copyright (C) 2018 Oleh Kravchenko <oleg@kaa.org.ua>
+
+#include <linux/delay.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/spi/spi.h>
+#include <linux/workqueue.h>
+#include <uapi/linux/uleds.h>
+
+/*
+ *  CR0014114 SPI protocol descrtiption:
+ *  +----+-----------------------------------+----+
+ *  | CMD|             BRIGHTNESS            |CRC |
+ *  +----+-----------------------------------+----+
+ *  |    | LED0| LED1| LED2| LED3| LED4| LED5|    |
+ *  |    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+    |
+ *  |    |R|G|B|R|G|B|R|G|B|R|G|B|R|G|B|R|G|B|    |
+ *  | 1  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1  |
+ *  |    |1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|1|    |
+ *  |    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+    |
+ *  |    |               18                  |    |
+ *  +----+-----------------------------------+----+
+ *  |                    20                       |
+ *  +---------------------------------------------+
+ *
+ *  PS: Boards can be connected to the chain:
+ *      SPI -> board0 -> board1 -> board2 ..
+ */
+
+/* CR0014114 SPI commands */
+#define CR_SET_BRIGHTNESS	0x80
+#define CR_INIT_REENUMERATE	0x81
+#define CR_NEXT_REENUMERATE	0x82
+
+/* CR0014114 default settings */
+#define CR_MAX_BRIGHTNESS	GENMASK(6, 0)
+#define CR_FW_DELAY_MSEC	10
+#define CR_RECOUNT_DELAY	(HZ * 3600)
+
+struct cr0014114_led {
+	char			name[LED_MAX_NAME_SIZE];
+	struct cr0014114	*priv;
+	struct led_classdev	ldev;
+	u8			brightness;
+};
+
+struct cr0014114 {
+	bool			do_recount;
+	size_t			count;
+	struct delayed_work	work;
+	struct device		*dev;
+	struct mutex		lock;
+	struct spi_device	*spi;
+	u8			*buf;
+	unsigned long		delay;
+	struct cr0014114_led	leds[];
+};
+
+static void cr0014114_calc_crc(u8 *buf, const size_t len)
+{
+	size_t	i;
+	u8	crc;
+
+	for (i = 1, crc = 1; i < len - 1; i++)
+		crc += buf[i];
+	crc |= BIT(7);
+
+	/* special case when CRC matches the SPI commands */
+	if (crc == CR_SET_BRIGHTNESS ||
+	    crc == CR_INIT_REENUMERATE ||
+	    crc == CR_NEXT_REENUMERATE)
+		crc = 0xfe;
+
+	buf[len - 1] = crc;
+}
+
+static int cr0014114_recount(struct cr0014114 *priv)
+{
+	int	ret;
+	size_t	i;
+	u8	cmd;
+
+	dev_dbg(priv->dev, "LEDs recount is started\n");
+
+	cmd = CR_INIT_REENUMERATE;
+	ret = spi_write(priv->spi, &cmd, sizeof(cmd));
+	if (ret)
+		goto err;
+
+	cmd = CR_NEXT_REENUMERATE;
+	for (i = 0; i < priv->count; i++) {
+		msleep(CR_FW_DELAY_MSEC);
+
+		ret = spi_write(priv->spi, &cmd, sizeof(cmd));
+		if (ret)
+			goto err;
+	}
+
+err:
+	dev_dbg(priv->dev, "LEDs recount is finished\n");
+
+	if (ret)
+		dev_err(priv->dev, "with error %d", ret);
+
+	return ret;
+}
+
+static int cr0014114_sync(struct cr0014114 *priv)
+{
+	int		ret;
+	size_t		i;
+	unsigned long	udelay, now = jiffies;
+
+	/* to avoid SPI mistiming with firmware we should wait some time */
+	if (time_after(priv->delay, now)) {
+		udelay = jiffies_to_usecs(priv->delay - now);
+		usleep_range(udelay, udelay + 1);
+	}
+
+	if (unlikely(priv->do_recount)) {
+		ret = cr0014114_recount(priv);
+		if (ret)
+			goto err;
+
+		priv->do_recount = false;
+		msleep(CR_FW_DELAY_MSEC);
+	}
+
+	priv->buf[0] = CR_SET_BRIGHTNESS;
+	for (i = 0; i < priv->count; i++)
+		priv->buf[i + 1] = priv->leds[i].brightness;
+	cr0014114_calc_crc(priv->buf, priv->count + 2);
+	ret = spi_write(priv->spi, priv->buf, priv->count + 2);
+
+err:
+	priv->delay = jiffies + msecs_to_jiffies(CR_FW_DELAY_MSEC);
+
+	return ret;
+}
+
+static void cr0014114_recount_work(struct work_struct *work)
+{
+	int			ret;
+	struct cr0014114	*priv = container_of(work,
+						     struct cr0014114,
+						     work.work);
+
+	mutex_lock(&priv->lock);
+	priv->do_recount = true;
+	ret = cr0014114_sync(priv);
+	mutex_unlock(&priv->lock);
+
+	if (ret)
+		dev_warn(priv->dev, "sync of LEDs failed %d\n", ret);
+
+	schedule_delayed_work(&priv->work, CR_RECOUNT_DELAY);
+}
+
+static int cr0014114_set_sync(struct led_classdev *ldev,
+			      enum led_brightness brightness)
+{
+	int			ret;
+	struct cr0014114_led    *led = container_of(ldev,
+						    struct cr0014114_led,
+						    ldev);
+
+	dev_dbg(led->priv->dev, "Set brightness of %s to %d\n",
+		led->name, brightness);
+
+	mutex_lock(&led->priv->lock);
+	led->brightness = (u8)brightness;
+	ret = cr0014114_sync(led->priv);
+	mutex_unlock(&led->priv->lock);
+
+	return ret;
+}
+
+static int cr0014114_probe_dt(struct cr0014114 *priv)
+{
+	size_t			i = 0;
+	struct cr0014114_led	*led;
+	struct fwnode_handle	*child;
+	struct device_node	*np;
+	int			ret;
+	const char		*str;
+
+	device_for_each_child_node(priv->dev, child) {
+		np = to_of_node(child);
+		led = &priv->leds[i];
+
+		ret = fwnode_property_read_string(child, "label", &str);
+		if (ret)
+			snprintf(led->name, sizeof(led->name),
+				 "cr0014114::");
+		else
+			snprintf(led->name, sizeof(led->name),
+				 "cr0014114:%s", str);
+
+		fwnode_property_read_string(child, "linux,default-trigger",
+					    &led->ldev.default_trigger);
+
+		led->priv			  = priv;
+		led->ldev.name			  = led->name;
+		led->ldev.max_brightness	  = CR_MAX_BRIGHTNESS;
+		led->ldev.brightness_set_blocking = cr0014114_set_sync;
+
+		ret = devm_of_led_classdev_register(priv->dev, np,
+						    &led->ldev);
+		if (ret) {
+			dev_err(priv->dev,
+				"failed to register LED device %s, err %d",
+				led->name, ret);
+			fwnode_handle_put(child);
+			return ret;
+		}
+
+		led->ldev.dev->of_node = np;
+
+		i++;
+	}
+
+	return 0;
+}
+
+static int cr0014114_probe(struct spi_device *spi)
+{
+	struct cr0014114	*priv;
+	size_t			count;
+	int			ret;
+
+	count = device_get_child_node_count(&spi->dev);
+	if (!count) {
+		dev_err(&spi->dev, "LEDs are not defined in device tree!");
+		return -ENODEV;
+	}
+
+	priv = devm_kzalloc(&spi->dev,
+			    sizeof(*priv) + sizeof(*priv->leds) * count,
+			    GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->buf = devm_kzalloc(&spi->dev, count + 2, GFP_KERNEL);
+	if (!priv->buf)
+		return -ENOMEM;
+
+	mutex_init(&priv->lock);
+	INIT_DELAYED_WORK(&priv->work, cr0014114_recount_work);
+	priv->count	= count;
+	priv->dev	= &spi->dev;
+	priv->spi	= spi;
+	priv->delay	= jiffies -
+			  msecs_to_jiffies(CR_FW_DELAY_MSEC);
+
+	priv->do_recount = true;
+	ret = cr0014114_sync(priv);
+	if (ret) {
+		dev_err(priv->dev, "first recount failed %d\n", ret);
+		return ret;
+	}
+
+	priv->do_recount = true;
+	ret = cr0014114_sync(priv);
+	if (ret) {
+		dev_err(priv->dev, "second recount failed %d\n", ret);
+		return ret;
+	}
+
+	ret = cr0014114_probe_dt(priv);
+	if (ret)
+		return ret;
+
+	/* setup recount work to workaround buggy firmware */
+	schedule_delayed_work(&priv->work, CR_RECOUNT_DELAY);
+
+	spi_set_drvdata(spi, priv);
+
+	return 0;
+}
+
+static int cr0014114_remove(struct spi_device *spi)
+{
+	struct cr0014114 *priv = spi_get_drvdata(spi);
+
+	cancel_delayed_work_sync(&priv->work);
+	mutex_destroy(&priv->lock);
+
+	return 0;
+}
+
+static const struct of_device_id cr0014114_dt_ids[] = {
+	{ .compatible = "crane,cr0014114", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, cr0014114_dt_ids);
+
+static struct spi_driver cr0014114_driver = {
+	.probe		= cr0014114_probe,
+	.remove		= cr0014114_remove,
+	.driver = {
+		.name		= KBUILD_MODNAME,
+		.of_match_table	= cr0014114_dt_ids,
+	},
+};
+
+module_spi_driver(cr0014114_driver);
+
+MODULE_AUTHOR("Oleh Kravchenko <oleg@kaa.org.ua>");
+MODULE_DESCRIPTION("cr0014114 LED driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("spi:cr0014114");
diff --git a/drivers/leds/leds-lm3601x.c b/drivers/leds/leds-lm3601x.c
new file mode 100644
index 000000000000..081aa71e43a3
--- /dev/null
+++ b/drivers/leds/leds-lm3601x.c
@@ -0,0 +1,487 @@
+// SPDX-License-Identifier: GPL-2.0
+// Flash and torch driver for Texas Instruments LM3601X LED
+// Flash driver chip family
+// Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/leds.h>
+#include <linux/led-class-flash.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <uapi/linux/uleds.h>
+
+#define LM3601X_LED_IR		0x0
+#define LM3601X_LED_TORCH	0x1
+
+/* Registers */
+#define LM3601X_ENABLE_REG	0x01
+#define LM3601X_CFG_REG		0x02
+#define LM3601X_LED_FLASH_REG	0x03
+#define LM3601X_LED_TORCH_REG	0x04
+#define LM3601X_FLAGS_REG	0x05
+#define LM3601X_DEV_ID_REG	0x06
+
+#define LM3601X_SW_RESET	BIT(7)
+
+/* Enable Mode bits */
+#define LM3601X_MODE_STANDBY	0x00
+#define LM3601X_MODE_IR_DRV	BIT(0)
+#define LM3601X_MODE_TORCH	BIT(1)
+#define LM3601X_MODE_STROBE	(BIT(0) | BIT(1))
+#define LM3601X_STRB_EN		BIT(2)
+#define LM3601X_STRB_EDGE_TRIG	BIT(3)
+#define LM3601X_IVFM_EN		BIT(4)
+
+#define LM36010_BOOST_LIMIT_28	BIT(5)
+#define LM36010_BOOST_FREQ_4MHZ	BIT(6)
+#define LM36010_BOOST_MODE_PASS	BIT(7)
+
+/* Flag Mask */
+#define LM3601X_FLASH_TIME_OUT	BIT(0)
+#define LM3601X_UVLO_FAULT	BIT(1)
+#define LM3601X_THERM_SHUTDOWN	BIT(2)
+#define LM3601X_THERM_CURR	BIT(3)
+#define LM36010_CURR_LIMIT	BIT(4)
+#define LM3601X_SHORT_FAULT	BIT(5)
+#define LM3601X_IVFM_TRIP	BIT(6)
+#define LM36010_OVP_FAULT	BIT(7)
+
+#define LM3601X_MAX_TORCH_I_UA	376000
+#define LM3601X_MIN_TORCH_I_UA	2400
+#define LM3601X_TORCH_REG_DIV	2965
+
+#define LM3601X_MAX_STROBE_I_UA	1500000
+#define LM3601X_MIN_STROBE_I_UA	11000
+#define LM3601X_STROBE_REG_DIV	11800
+
+#define LM3601X_TIMEOUT_MASK	0x1e
+#define LM3601X_ENABLE_MASK	(LM3601X_MODE_IR_DRV | LM3601X_MODE_TORCH)
+
+#define LM3601X_LOWER_STEP_US	40000
+#define LM3601X_UPPER_STEP_US	200000
+#define LM3601X_MIN_TIMEOUT_US	40000
+#define LM3601X_MAX_TIMEOUT_US	1600000
+#define LM3601X_TIMEOUT_XOVER_US 400000
+
+enum lm3601x_type {
+	CHIP_LM36010 = 0,
+	CHIP_LM36011,
+};
+
+/**
+ * struct lm3601x_led -
+ * @fled_cdev: flash LED class device pointer
+ * @client: Pointer to the I2C client
+ * @regmap: Devices register map
+ * @lock: Lock for reading/writing the device
+ * @led_name: LED label for the Torch or IR LED
+ * @flash_timeout: the timeout for the flash
+ * @last_flag: last known flags register value
+ * @torch_current_max: maximum current for the torch
+ * @flash_current_max: maximum current for the flash
+ * @max_flash_timeout: maximum timeout for the flash
+ * @led_mode: The mode to enable either IR or Torch
+ */
+struct lm3601x_led {
+	struct led_classdev_flash fled_cdev;
+	struct i2c_client *client;
+	struct regmap *regmap;
+	struct mutex lock;
+
+	char led_name[LED_MAX_NAME_SIZE];
+
+	unsigned int flash_timeout;
+	unsigned int last_flag;
+
+	u32 torch_current_max;
+	u32 flash_current_max;
+	u32 max_flash_timeout;
+
+	u32 led_mode;
+};
+
+static const struct reg_default lm3601x_regmap_defs[] = {
+	{ LM3601X_ENABLE_REG, 0x20 },
+	{ LM3601X_CFG_REG, 0x15 },
+	{ LM3601X_LED_FLASH_REG, 0x00 },
+	{ LM3601X_LED_TORCH_REG, 0x00 },
+};
+
+static bool lm3601x_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case LM3601X_FLAGS_REG:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct regmap_config lm3601x_regmap = {
+	.reg_bits = 8,
+	.val_bits = 8,
+
+	.max_register = LM3601X_DEV_ID_REG,
+	.reg_defaults = lm3601x_regmap_defs,
+	.num_reg_defaults = ARRAY_SIZE(lm3601x_regmap_defs),
+	.cache_type = REGCACHE_RBTREE,
+	.volatile_reg = lm3601x_volatile_reg,
+};
+
+static struct lm3601x_led *fled_cdev_to_led(struct led_classdev_flash *fled_cdev)
+{
+	return container_of(fled_cdev, struct lm3601x_led, fled_cdev);
+}
+
+static int lm3601x_read_faults(struct lm3601x_led *led)
+{
+	int flags_val;
+	int ret;
+
+	ret = regmap_read(led->regmap, LM3601X_FLAGS_REG, &flags_val);
+	if (ret < 0)
+		return -EIO;
+
+	led->last_flag = 0;
+
+	if (flags_val & LM36010_OVP_FAULT)
+		led->last_flag |= LED_FAULT_OVER_VOLTAGE;
+
+	if (flags_val & (LM3601X_THERM_SHUTDOWN | LM3601X_THERM_CURR))
+		led->last_flag |= LED_FAULT_OVER_TEMPERATURE;
+
+	if (flags_val & LM3601X_SHORT_FAULT)
+		led->last_flag |= LED_FAULT_SHORT_CIRCUIT;
+
+	if (flags_val & LM36010_CURR_LIMIT)
+		led->last_flag |= LED_FAULT_OVER_CURRENT;
+
+	if (flags_val & LM3601X_UVLO_FAULT)
+		led->last_flag |= LED_FAULT_UNDER_VOLTAGE;
+
+	if (flags_val & LM3601X_IVFM_TRIP)
+		led->last_flag |= LED_FAULT_INPUT_VOLTAGE;
+
+	if (flags_val & LM3601X_THERM_SHUTDOWN)
+		led->last_flag |= LED_FAULT_LED_OVER_TEMPERATURE;
+
+	return led->last_flag;
+}
+
+static int lm3601x_brightness_set(struct led_classdev *cdev,
+					enum led_brightness brightness)
+{
+	struct led_classdev_flash *fled_cdev = lcdev_to_flcdev(cdev);
+	struct lm3601x_led *led = fled_cdev_to_led(fled_cdev);
+	int ret, led_mode_val;
+
+	mutex_lock(&led->lock);
+
+	ret = lm3601x_read_faults(led);
+	if (ret < 0)
+		goto out;
+
+	if (led->led_mode == LM3601X_LED_TORCH)
+		led_mode_val = LM3601X_MODE_TORCH;
+	else
+		led_mode_val = LM3601X_MODE_IR_DRV;
+
+	if (brightness == LED_OFF) {
+		ret = regmap_update_bits(led->regmap, LM3601X_ENABLE_REG,
+					led_mode_val, LED_OFF);
+		goto out;
+	}
+
+	ret = regmap_write(led->regmap, LM3601X_LED_TORCH_REG, brightness);
+	if (ret < 0)
+		goto out;
+
+	ret = regmap_update_bits(led->regmap, LM3601X_ENABLE_REG,
+				LM3601X_MODE_TORCH | LM3601X_MODE_IR_DRV,
+				led_mode_val);
+out:
+	mutex_unlock(&led->lock);
+	return ret;
+}
+
+static int lm3601x_strobe_set(struct led_classdev_flash *fled_cdev,
+				bool state)
+{
+	struct lm3601x_led *led = fled_cdev_to_led(fled_cdev);
+	int timeout_reg_val;
+	int current_timeout;
+	int ret;
+
+	mutex_lock(&led->lock);
+
+	ret = regmap_read(led->regmap, LM3601X_CFG_REG, &current_timeout);
+	if (ret < 0)
+		goto out;
+
+	if (led->flash_timeout >= LM3601X_TIMEOUT_XOVER_US)
+		timeout_reg_val = led->flash_timeout / LM3601X_UPPER_STEP_US + 0x07;
+	else
+		timeout_reg_val = led->flash_timeout / LM3601X_LOWER_STEP_US - 0x01;
+
+	if (led->flash_timeout != current_timeout)
+		ret = regmap_update_bits(led->regmap, LM3601X_CFG_REG,
+					LM3601X_TIMEOUT_MASK, timeout_reg_val);
+
+	if (state)
+		ret = regmap_update_bits(led->regmap, LM3601X_ENABLE_REG,
+					LM3601X_MODE_TORCH | LM3601X_MODE_IR_DRV,
+					LM3601X_MODE_STROBE);
+	else
+		ret = regmap_update_bits(led->regmap, LM3601X_ENABLE_REG,
+					LM3601X_MODE_STROBE, LED_OFF);
+
+	ret = lm3601x_read_faults(led);
+out:
+	mutex_unlock(&led->lock);
+	return ret;
+}
+
+static int lm3601x_flash_brightness_set(struct led_classdev_flash *fled_cdev,
+					u32 brightness)
+{
+	struct lm3601x_led *led = fled_cdev_to_led(fled_cdev);
+	u8 brightness_val;
+	int ret;
+
+	mutex_lock(&led->lock);
+	ret = lm3601x_read_faults(led);
+	if (ret < 0)
+		goto out;
+
+	if (brightness == LED_OFF) {
+		ret = regmap_update_bits(led->regmap, LM3601X_ENABLE_REG,
+					LM3601X_MODE_STROBE, LED_OFF);
+		goto out;
+	}
+
+	brightness_val = brightness / LM3601X_STROBE_REG_DIV;
+
+	ret = regmap_write(led->regmap, LM3601X_LED_FLASH_REG, brightness_val);
+out:
+	mutex_unlock(&led->lock);
+	return ret;
+}
+
+static int lm3601x_flash_timeout_set(struct led_classdev_flash *fled_cdev,
+				u32 timeout)
+{
+	struct lm3601x_led *led = fled_cdev_to_led(fled_cdev);
+
+	mutex_lock(&led->lock);
+
+	led->flash_timeout = timeout;
+
+	mutex_unlock(&led->lock);
+
+	return 0;
+}
+
+static int lm3601x_strobe_get(struct led_classdev_flash *fled_cdev, bool *state)
+{
+	struct lm3601x_led *led = fled_cdev_to_led(fled_cdev);
+	int strobe_state;
+	int ret;
+
+	mutex_lock(&led->lock);
+
+	ret = regmap_read(led->regmap, LM3601X_ENABLE_REG, &strobe_state);
+	if (ret < 0)
+		goto out;
+
+	*state = strobe_state & LM3601X_MODE_STROBE;
+
+out:
+	mutex_unlock(&led->lock);
+	return ret;
+}
+
+static int lm3601x_flash_fault_get(struct led_classdev_flash *fled_cdev,
+				u32 *fault)
+{
+	struct lm3601x_led *led = fled_cdev_to_led(fled_cdev);
+
+	lm3601x_read_faults(led);
+
+	*fault = led->last_flag;
+
+	return 0;
+}
+
+static const struct led_flash_ops flash_ops = {
+	.flash_brightness_set	= lm3601x_flash_brightness_set,
+	.strobe_set		= lm3601x_strobe_set,
+	.strobe_get		= lm3601x_strobe_get,
+	.timeout_set		= lm3601x_flash_timeout_set,
+	.fault_get		= lm3601x_flash_fault_get,
+};
+
+static int lm3601x_register_leds(struct lm3601x_led *led)
+{
+	struct led_classdev *led_cdev;
+	struct led_flash_setting *setting;
+
+	led->fled_cdev.ops = &flash_ops;
+
+	setting = &led->fled_cdev.timeout;
+	setting->min = LM3601X_MIN_TIMEOUT_US;
+	setting->max = led->max_flash_timeout;
+	setting->step = LM3601X_LOWER_STEP_US;
+	setting->val = led->max_flash_timeout;
+
+	setting = &led->fled_cdev.brightness;
+	setting->min = LM3601X_MIN_STROBE_I_UA;
+	setting->max = led->flash_current_max;
+	setting->step = LM3601X_TORCH_REG_DIV;
+	setting->val = led->flash_current_max;
+
+	led_cdev = &led->fled_cdev.led_cdev;
+	led_cdev->name = led->led_name;
+	led_cdev->brightness_set_blocking = lm3601x_brightness_set;
+	led_cdev->max_brightness = DIV_ROUND_UP(led->torch_current_max,
+						LM3601X_TORCH_REG_DIV);
+	led_cdev->flags |= LED_DEV_CAP_FLASH;
+
+	return led_classdev_flash_register(&led->client->dev, &led->fled_cdev);
+}
+
+static int lm3601x_parse_node(struct lm3601x_led *led)
+{
+	struct fwnode_handle *child = NULL;
+	int ret = -ENODEV;
+	const char *name;
+
+	child = device_get_next_child_node(&led->client->dev, child);
+	if (!child) {
+		dev_err(&led->client->dev, "No LED Child node\n");
+		return ret;
+	}
+
+	ret = fwnode_property_read_u32(child, "reg", &led->led_mode);
+	if (ret) {
+		dev_err(&led->client->dev, "reg DT property missing\n");
+		goto out_err;
+	}
+
+	if (led->led_mode > LM3601X_LED_TORCH ||
+	    led->led_mode < LM3601X_LED_IR) {
+		dev_warn(&led->client->dev, "Invalid led mode requested\n");
+		ret = -EINVAL;
+		goto out_err;
+	}
+
+	ret = fwnode_property_read_string(child, "label", &name);
+	if (ret) {
+		if (led->led_mode == LM3601X_LED_TORCH)
+			name = "torch";
+		else
+			name = "infrared";
+	}
+
+	snprintf(led->led_name, sizeof(led->led_name),
+		"%s:%s", led->client->name, name);
+
+	ret = fwnode_property_read_u32(child, "led-max-microamp",
+					&led->torch_current_max);
+	if (ret) {
+		dev_warn(&led->client->dev,
+			"led-max-microamp DT property missing\n");
+		goto out_err;
+	}
+
+	ret = fwnode_property_read_u32(child, "flash-max-microamp",
+				&led->flash_current_max);
+	if (ret) {
+		dev_warn(&led->client->dev,
+			 "flash-max-microamp DT property missing\n");
+		goto out_err;
+	}
+
+	ret = fwnode_property_read_u32(child, "flash-max-timeout-us",
+				&led->max_flash_timeout);
+	if (ret) {
+		dev_warn(&led->client->dev,
+			 "flash-max-timeout-us DT property missing\n");
+		goto out_err;
+	}
+
+out_err:
+	fwnode_handle_put(child);
+	return ret;
+}
+
+static int lm3601x_probe(struct i2c_client *client)
+{
+	struct lm3601x_led *led;
+	int ret;
+
+	led = devm_kzalloc(&client->dev, sizeof(*led), GFP_KERNEL);
+	if (!led)
+		return -ENOMEM;
+
+	led->client = client;
+	i2c_set_clientdata(client, led);
+
+	ret = lm3601x_parse_node(led);
+	if (ret)
+		return -ENODEV;
+
+	led->regmap = devm_regmap_init_i2c(client, &lm3601x_regmap);
+	if (IS_ERR(led->regmap)) {
+		ret = PTR_ERR(led->regmap);
+		dev_err(&client->dev,
+			"Failed to allocate register map: %d\n", ret);
+		return ret;
+	}
+
+	mutex_init(&led->lock);
+
+	return lm3601x_register_leds(led);
+}
+
+static int lm3601x_remove(struct i2c_client *client)
+{
+	struct lm3601x_led *led = i2c_get_clientdata(client);
+
+	led_classdev_flash_unregister(&led->fled_cdev);
+	mutex_destroy(&led->lock);
+
+	return regmap_update_bits(led->regmap, LM3601X_ENABLE_REG,
+			   LM3601X_ENABLE_MASK,
+			   LM3601X_MODE_STANDBY);
+}
+
+static const struct i2c_device_id lm3601x_id[] = {
+	{ "LM36010", CHIP_LM36010 },
+	{ "LM36011", CHIP_LM36011 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, lm3601x_id);
+
+static const struct of_device_id of_lm3601x_leds_match[] = {
+	{ .compatible = "ti,lm36010", },
+	{ .compatible = "ti,lm36011", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, of_lm3601x_leds_match);
+
+static struct i2c_driver lm3601x_i2c_driver = {
+	.driver = {
+		.name = "lm3601x",
+		.of_match_table = of_lm3601x_leds_match,
+	},
+	.probe_new = lm3601x_probe,
+	.remove = lm3601x_remove,
+	.id_table = lm3601x_id,
+};
+module_i2c_driver(lm3601x_i2c_driver);
+
+MODULE_DESCRIPTION("Texas Instruments Flash Lighting driver for LM3601X");
+MODULE_AUTHOR("Dan Murphy <dmurphy@ti.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/leds/leds-sc27xx-bltc.c b/drivers/leds/leds-sc27xx-bltc.c
new file mode 100644
index 000000000000..9d9b7aab843f
--- /dev/null
+++ b/drivers/leds/leds-sc27xx-bltc.c
@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2018 Spreadtrum Communications Inc.
+
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <uapi/linux/uleds.h>
+
+/* PMIC global control register definition */
+#define SC27XX_MODULE_EN0	0xc08
+#define SC27XX_CLK_EN0		0xc18
+#define SC27XX_RGB_CTRL		0xebc
+
+#define SC27XX_BLTC_EN		BIT(9)
+#define SC27XX_RTC_EN		BIT(7)
+#define SC27XX_RGB_PD		BIT(0)
+
+/* Breathing light controller register definition */
+#define SC27XX_LEDS_CTRL	0x00
+#define SC27XX_LEDS_PRESCALE	0x04
+#define SC27XX_LEDS_DUTY	0x08
+#define SC27XX_LEDS_CURVE0	0x0c
+#define SC27XX_LEDS_CURVE1	0x10
+
+#define SC27XX_CTRL_SHIFT	4
+#define SC27XX_LED_RUN		BIT(0)
+#define SC27XX_LED_TYPE		BIT(1)
+
+#define SC27XX_DUTY_SHIFT	8
+#define SC27XX_DUTY_MASK	GENMASK(15, 0)
+#define SC27XX_MOD_MASK		GENMASK(7, 0)
+
+#define SC27XX_LEDS_OFFSET	0x10
+#define SC27XX_LEDS_MAX		3
+
+struct sc27xx_led {
+	char name[LED_MAX_NAME_SIZE];
+	struct led_classdev ldev;
+	struct sc27xx_led_priv *priv;
+	u8 line;
+	bool active;
+};
+
+struct sc27xx_led_priv {
+	struct sc27xx_led leds[SC27XX_LEDS_MAX];
+	struct regmap *regmap;
+	struct mutex lock;
+	u32 base;
+};
+
+#define to_sc27xx_led(ldev) \
+	container_of(ldev, struct sc27xx_led, ldev)
+
+static int sc27xx_led_init(struct regmap *regmap)
+{
+	int err;
+
+	err = regmap_update_bits(regmap, SC27XX_MODULE_EN0, SC27XX_BLTC_EN,
+				 SC27XX_BLTC_EN);
+	if (err)
+		return err;
+
+	err = regmap_update_bits(regmap, SC27XX_CLK_EN0, SC27XX_RTC_EN,
+				 SC27XX_RTC_EN);
+	if (err)
+		return err;
+
+	return regmap_update_bits(regmap, SC27XX_RGB_CTRL, SC27XX_RGB_PD, 0);
+}
+
+static u32 sc27xx_led_get_offset(struct sc27xx_led *leds)
+{
+	return leds->priv->base + SC27XX_LEDS_OFFSET * leds->line;
+}
+
+static int sc27xx_led_enable(struct sc27xx_led *leds, enum led_brightness value)
+{
+	u32 base = sc27xx_led_get_offset(leds);
+	u32 ctrl_base = leds->priv->base + SC27XX_LEDS_CTRL;
+	u8 ctrl_shift = SC27XX_CTRL_SHIFT * leds->line;
+	struct regmap *regmap = leds->priv->regmap;
+	int err;
+
+	err = regmap_update_bits(regmap, base + SC27XX_LEDS_DUTY,
+				 SC27XX_DUTY_MASK,
+				 (value << SC27XX_DUTY_SHIFT) |
+				 SC27XX_MOD_MASK);
+	if (err)
+		return err;
+
+	return regmap_update_bits(regmap, ctrl_base,
+			(SC27XX_LED_RUN | SC27XX_LED_TYPE) << ctrl_shift,
+			(SC27XX_LED_RUN | SC27XX_LED_TYPE) << ctrl_shift);
+}
+
+static int sc27xx_led_disable(struct sc27xx_led *leds)
+{
+	struct regmap *regmap = leds->priv->regmap;
+	u32 ctrl_base = leds->priv->base + SC27XX_LEDS_CTRL;
+	u8 ctrl_shift = SC27XX_CTRL_SHIFT * leds->line;
+
+	return regmap_update_bits(regmap, ctrl_base,
+			(SC27XX_LED_RUN | SC27XX_LED_TYPE) << ctrl_shift, 0);
+}
+
+static int sc27xx_led_set(struct led_classdev *ldev, enum led_brightness value)
+{
+	struct sc27xx_led *leds = to_sc27xx_led(ldev);
+	int err;
+
+	mutex_lock(&leds->priv->lock);
+
+	if (value == LED_OFF)
+		err = sc27xx_led_disable(leds);
+	else
+		err = sc27xx_led_enable(leds, value);
+
+	mutex_unlock(&leds->priv->lock);
+
+	return err;
+}
+
+static int sc27xx_led_register(struct device *dev, struct sc27xx_led_priv *priv)
+{
+	int i, err;
+
+	err = sc27xx_led_init(priv->regmap);
+	if (err)
+		return err;
+
+	for (i = 0; i < SC27XX_LEDS_MAX; i++) {
+		struct sc27xx_led *led = &priv->leds[i];
+
+		if (!led->active)
+			continue;
+
+		led->line = i;
+		led->priv = priv;
+		led->ldev.name = led->name;
+		led->ldev.brightness_set_blocking = sc27xx_led_set;
+
+		err = devm_led_classdev_register(dev, &led->ldev);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int sc27xx_led_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node, *child;
+	struct sc27xx_led_priv *priv;
+	const char *str;
+	u32 base, count, reg;
+	int err;
+
+	count = of_get_child_count(np);
+	if (!count || count > SC27XX_LEDS_MAX)
+		return -EINVAL;
+
+	err = of_property_read_u32(np, "reg", &base);
+	if (err) {
+		dev_err(dev, "fail to get reg of property\n");
+		return err;
+	}
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, priv);
+	mutex_init(&priv->lock);
+	priv->base = base;
+	priv->regmap = dev_get_regmap(dev->parent, NULL);
+	if (!priv->regmap) {
+		err = -ENODEV;
+		dev_err(dev, "failed to get regmap: %d\n", err);
+		return err;
+	}
+
+	for_each_child_of_node(np, child) {
+		err = of_property_read_u32(child, "reg", &reg);
+		if (err) {
+			of_node_put(child);
+			mutex_destroy(&priv->lock);
+			return err;
+		}
+
+		if (reg >= SC27XX_LEDS_MAX || priv->leds[reg].active) {
+			of_node_put(child);
+			mutex_destroy(&priv->lock);
+			return -EINVAL;
+		}
+
+		priv->leds[reg].active = true;
+
+		err = of_property_read_string(child, "label", &str);
+		if (err)
+			snprintf(priv->leds[reg].name, LED_MAX_NAME_SIZE,
+				 "sc27xx::");
+		else
+			snprintf(priv->leds[reg].name, LED_MAX_NAME_SIZE,
+				 "sc27xx:%s", str);
+	}
+
+	err = sc27xx_led_register(dev, priv);
+	if (err)
+		mutex_destroy(&priv->lock);
+
+	return err;
+}
+
+static int sc27xx_led_remove(struct platform_device *pdev)
+{
+	struct sc27xx_led_priv *priv = platform_get_drvdata(pdev);
+
+	mutex_destroy(&priv->lock);
+	return 0;
+}
+
+static const struct of_device_id sc27xx_led_of_match[] = {
+	{ .compatible = "sprd,sc2731-bltc", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sc27xx_led_of_match);
+
+static struct platform_driver sc27xx_led_driver = {
+	.driver = {
+		.name = "sprd-bltc",
+		.of_match_table = sc27xx_led_of_match,
+	},
+	.probe = sc27xx_led_probe,
+	.remove = sc27xx_led_remove,
+};
+
+module_platform_driver(sc27xx_led_driver);
+
+MODULE_DESCRIPTION("Spreadtrum SC27xx breathing light controller driver");
+MODULE_AUTHOR("Xiaotong Lu <xiaotong.lu@spreadtrum.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/leds/leds-wm831x-status.c b/drivers/leds/leds-wm831x-status.c
index be93b20e792a..c5798b92e4d3 100644
--- a/drivers/leds/leds-wm831x-status.c
+++ b/drivers/leds/leds-wm831x-status.c
@@ -188,24 +188,14 @@ static ssize_t wm831x_status_src_store(struct device *dev,
 {
 	struct led_classdev *led_cdev = dev_get_drvdata(dev);
 	struct wm831x_status *led = to_wm831x_status(led_cdev);
-	char name[20];
 	int i;
-	size_t len;
 
-	name[sizeof(name) - 1] = '\0';
-	strncpy(name, buf, sizeof(name) - 1);
-	len = strlen(name);
-
-	if (len && name[len - 1] == '\n')
-		name[len - 1] = '\0';
-
-	for (i = 0; i < ARRAY_SIZE(led_src_texts); i++) {
-		if (!strcmp(name, led_src_texts[i])) {
-			mutex_lock(&led->mutex);
-			led->src = i;
-			mutex_unlock(&led->mutex);
-			wm831x_status_set(led);
-		}
+	i = sysfs_match_string(led_src_texts, buf);
+	if (i >= 0) {
+		mutex_lock(&led->mutex);
+		led->src = i;
+		mutex_unlock(&led->mutex);
+		wm831x_status_set(led);
 	}
 
 	return size;
diff --git a/drivers/lightnvm/pblk-core.c b/drivers/lightnvm/pblk-core.c
index 94d5d97c9d8a..934341b10493 100644
--- a/drivers/lightnvm/pblk-core.c
+++ b/drivers/lightnvm/pblk-core.c
@@ -40,7 +40,7 @@ static void pblk_line_mark_bb(struct work_struct *work)
 	}
 
 	kfree(ppa);
-	mempool_free(line_ws, pblk->gen_ws_pool);
+	mempool_free(line_ws, &pblk->gen_ws_pool);
 }
 
 static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line,
@@ -102,7 +102,7 @@ static void pblk_end_io_erase(struct nvm_rq *rqd)
 	struct pblk *pblk = rqd->private;
 
 	__pblk_end_io_erase(pblk, rqd);
-	mempool_free(rqd, pblk->e_rq_pool);
+	mempool_free(rqd, &pblk->e_rq_pool);
 }
 
 /*
@@ -237,15 +237,15 @@ struct nvm_rq *pblk_alloc_rqd(struct pblk *pblk, int type)
 	switch (type) {
 	case PBLK_WRITE:
 	case PBLK_WRITE_INT:
-		pool = pblk->w_rq_pool;
+		pool = &pblk->w_rq_pool;
 		rq_size = pblk_w_rq_size;
 		break;
 	case PBLK_READ:
-		pool = pblk->r_rq_pool;
+		pool = &pblk->r_rq_pool;
 		rq_size = pblk_g_rq_size;
 		break;
 	default:
-		pool = pblk->e_rq_pool;
+		pool = &pblk->e_rq_pool;
 		rq_size = pblk_g_rq_size;
 	}
 
@@ -265,13 +265,13 @@ void pblk_free_rqd(struct pblk *pblk, struct nvm_rq *rqd, int type)
 	case PBLK_WRITE:
 		kfree(((struct pblk_c_ctx *)nvm_rq_to_pdu(rqd))->lun_bitmap);
 	case PBLK_WRITE_INT:
-		pool = pblk->w_rq_pool;
+		pool = &pblk->w_rq_pool;
 		break;
 	case PBLK_READ:
-		pool = pblk->r_rq_pool;
+		pool = &pblk->r_rq_pool;
 		break;
 	case PBLK_ERASE:
-		pool = pblk->e_rq_pool;
+		pool = &pblk->e_rq_pool;
 		break;
 	default:
 		pr_err("pblk: trying to free unknown rqd type\n");
@@ -292,7 +292,7 @@ void pblk_bio_free_pages(struct pblk *pblk, struct bio *bio, int off,
 
 	for (i = off; i < nr_pages + off; i++) {
 		bv = bio->bi_io_vec[i];
-		mempool_free(bv.bv_page, pblk->page_bio_pool);
+		mempool_free(bv.bv_page, &pblk->page_bio_pool);
 	}
 }
 
@@ -304,12 +304,12 @@ int pblk_bio_add_pages(struct pblk *pblk, struct bio *bio, gfp_t flags,
 	int i, ret;
 
 	for (i = 0; i < nr_pages; i++) {
-		page = mempool_alloc(pblk->page_bio_pool, flags);
+		page = mempool_alloc(&pblk->page_bio_pool, flags);
 
 		ret = bio_add_pc_page(q, bio, page, PBLK_EXPOSED_PAGE_SIZE, 0);
 		if (ret != PBLK_EXPOSED_PAGE_SIZE) {
 			pr_err("pblk: could not add page to bio\n");
-			mempool_free(page, pblk->page_bio_pool);
+			mempool_free(page, &pblk->page_bio_pool);
 			goto err;
 		}
 	}
@@ -1593,7 +1593,7 @@ static void pblk_line_put_ws(struct work_struct *work)
 	struct pblk_line *line = line_put_ws->line;
 
 	__pblk_line_put(pblk, line);
-	mempool_free(line_put_ws, pblk->gen_ws_pool);
+	mempool_free(line_put_ws, &pblk->gen_ws_pool);
 }
 
 void pblk_line_put(struct kref *ref)
@@ -1610,7 +1610,7 @@ void pblk_line_put_wq(struct kref *ref)
 	struct pblk *pblk = line->pblk;
 	struct pblk_line_ws *line_put_ws;
 
-	line_put_ws = mempool_alloc(pblk->gen_ws_pool, GFP_ATOMIC);
+	line_put_ws = mempool_alloc(&pblk->gen_ws_pool, GFP_ATOMIC);
 	if (!line_put_ws)
 		return;
 
@@ -1752,7 +1752,7 @@ void pblk_line_close_ws(struct work_struct *work)
 	struct pblk_line *line = line_ws->line;
 
 	pblk_line_close(pblk, line);
-	mempool_free(line_ws, pblk->gen_ws_pool);
+	mempool_free(line_ws, &pblk->gen_ws_pool);
 }
 
 void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
@@ -1761,7 +1761,7 @@ void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
 {
 	struct pblk_line_ws *line_ws;
 
-	line_ws = mempool_alloc(pblk->gen_ws_pool, gfp_mask);
+	line_ws = mempool_alloc(&pblk->gen_ws_pool, gfp_mask);
 
 	line_ws->pblk = pblk;
 	line_ws->line = line;
diff --git a/drivers/lightnvm/pblk-init.c b/drivers/lightnvm/pblk-init.c
index 91a5bc2556a3..9a984abd3dfe 100644
--- a/drivers/lightnvm/pblk-init.c
+++ b/drivers/lightnvm/pblk-init.c
@@ -23,7 +23,7 @@
 static struct kmem_cache *pblk_ws_cache, *pblk_rec_cache, *pblk_g_rq_cache,
 				*pblk_w_rq_cache;
 static DECLARE_RWSEM(pblk_lock);
-struct bio_set *pblk_bio_set;
+struct bio_set pblk_bio_set;
 
 static int pblk_rw_io(struct request_queue *q, struct pblk *pblk,
 			  struct bio *bio)
@@ -341,7 +341,7 @@ static int pblk_core_init(struct pblk *pblk)
 {
 	struct nvm_tgt_dev *dev = pblk->dev;
 	struct nvm_geo *geo = &dev->geo;
-	int max_write_ppas;
+	int ret, max_write_ppas;
 
 	atomic64_set(&pblk->user_wa, 0);
 	atomic64_set(&pblk->pad_wa, 0);
@@ -375,33 +375,33 @@ static int pblk_core_init(struct pblk *pblk)
 		goto fail_free_pad_dist;
 
 	/* Internal bios can be at most the sectors signaled by the device. */
-	pblk->page_bio_pool = mempool_create_page_pool(NVM_MAX_VLBA, 0);
-	if (!pblk->page_bio_pool)
+	ret = mempool_init_page_pool(&pblk->page_bio_pool, NVM_MAX_VLBA, 0);
+	if (ret)
 		goto free_global_caches;
 
-	pblk->gen_ws_pool = mempool_create_slab_pool(PBLK_GEN_WS_POOL_SIZE,
-							pblk_ws_cache);
-	if (!pblk->gen_ws_pool)
+	ret = mempool_init_slab_pool(&pblk->gen_ws_pool, PBLK_GEN_WS_POOL_SIZE,
+				     pblk_ws_cache);
+	if (ret)
 		goto free_page_bio_pool;
 
-	pblk->rec_pool = mempool_create_slab_pool(geo->all_luns,
-							pblk_rec_cache);
-	if (!pblk->rec_pool)
+	ret = mempool_init_slab_pool(&pblk->rec_pool, geo->all_luns,
+				     pblk_rec_cache);
+	if (ret)
 		goto free_gen_ws_pool;
 
-	pblk->r_rq_pool = mempool_create_slab_pool(geo->all_luns,
-							pblk_g_rq_cache);
-	if (!pblk->r_rq_pool)
+	ret = mempool_init_slab_pool(&pblk->r_rq_pool, geo->all_luns,
+				     pblk_g_rq_cache);
+	if (ret)
 		goto free_rec_pool;
 
-	pblk->e_rq_pool = mempool_create_slab_pool(geo->all_luns,
-							pblk_g_rq_cache);
-	if (!pblk->e_rq_pool)
+	ret = mempool_init_slab_pool(&pblk->e_rq_pool, geo->all_luns,
+				     pblk_g_rq_cache);
+	if (ret)
 		goto free_r_rq_pool;
 
-	pblk->w_rq_pool = mempool_create_slab_pool(geo->all_luns,
-							pblk_w_rq_cache);
-	if (!pblk->w_rq_pool)
+	ret = mempool_init_slab_pool(&pblk->w_rq_pool, geo->all_luns,
+				     pblk_w_rq_cache);
+	if (ret)
 		goto free_e_rq_pool;
 
 	pblk->close_wq = alloc_workqueue("pblk-close-wq",
@@ -433,17 +433,17 @@ static int pblk_core_init(struct pblk *pblk)
 free_close_wq:
 	destroy_workqueue(pblk->close_wq);
 free_w_rq_pool:
-	mempool_destroy(pblk->w_rq_pool);
+	mempool_exit(&pblk->w_rq_pool);
 free_e_rq_pool:
-	mempool_destroy(pblk->e_rq_pool);
+	mempool_exit(&pblk->e_rq_pool);
 free_r_rq_pool:
-	mempool_destroy(pblk->r_rq_pool);
+	mempool_exit(&pblk->r_rq_pool);
 free_rec_pool:
-	mempool_destroy(pblk->rec_pool);
+	mempool_exit(&pblk->rec_pool);
 free_gen_ws_pool:
-	mempool_destroy(pblk->gen_ws_pool);
+	mempool_exit(&pblk->gen_ws_pool);
 free_page_bio_pool:
-	mempool_destroy(pblk->page_bio_pool);
+	mempool_exit(&pblk->page_bio_pool);
 free_global_caches:
 	pblk_free_global_caches(pblk);
 fail_free_pad_dist:
@@ -462,12 +462,12 @@ static void pblk_core_free(struct pblk *pblk)
 	if (pblk->bb_wq)
 		destroy_workqueue(pblk->bb_wq);
 
-	mempool_destroy(pblk->page_bio_pool);
-	mempool_destroy(pblk->gen_ws_pool);
-	mempool_destroy(pblk->rec_pool);
-	mempool_destroy(pblk->r_rq_pool);
-	mempool_destroy(pblk->e_rq_pool);
-	mempool_destroy(pblk->w_rq_pool);
+	mempool_exit(&pblk->page_bio_pool);
+	mempool_exit(&pblk->gen_ws_pool);
+	mempool_exit(&pblk->rec_pool);
+	mempool_exit(&pblk->r_rq_pool);
+	mempool_exit(&pblk->e_rq_pool);
+	mempool_exit(&pblk->w_rq_pool);
 
 	pblk_free_global_caches(pblk);
 	kfree(pblk->pad_dist);
@@ -1297,18 +1297,18 @@ static int __init pblk_module_init(void)
 {
 	int ret;
 
-	pblk_bio_set = bioset_create(BIO_POOL_SIZE, 0, 0);
-	if (!pblk_bio_set)
-		return -ENOMEM;
+	ret = bioset_init(&pblk_bio_set, BIO_POOL_SIZE, 0, 0);
+	if (ret)
+		return ret;
 	ret = nvm_register_tgt_type(&tt_pblk);
 	if (ret)
-		bioset_free(pblk_bio_set);
+		bioset_exit(&pblk_bio_set);
 	return ret;
 }
 
 static void pblk_module_exit(void)
 {
-	bioset_free(pblk_bio_set);
+	bioset_exit(&pblk_bio_set);
 	nvm_unregister_tgt_type(&tt_pblk);
 }
 
diff --git a/drivers/lightnvm/pblk-read.c b/drivers/lightnvm/pblk-read.c
index 9eee10f69df0..c844ffb6aead 100644
--- a/drivers/lightnvm/pblk-read.c
+++ b/drivers/lightnvm/pblk-read.c
@@ -294,7 +294,7 @@ static int pblk_partial_read_bio(struct pblk *pblk, struct nvm_rq *rqd,
 		kunmap_atomic(src_p);
 		kunmap_atomic(dst_p);
 
-		mempool_free(src_bv.bv_page, pblk->page_bio_pool);
+		mempool_free(src_bv.bv_page, &pblk->page_bio_pool);
 
 		hole = find_next_zero_bit(read_bitmap, nr_secs, hole + 1);
 	} while (hole < nr_secs);
@@ -429,7 +429,7 @@ int pblk_submit_read(struct pblk *pblk, struct bio *bio)
 		struct bio *int_bio = NULL;
 
 		/* Clone read bio to deal with read errors internally */
-		int_bio = bio_clone_fast(bio, GFP_KERNEL, pblk_bio_set);
+		int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set);
 		if (!int_bio) {
 			pr_err("pblk: could not clone read bio\n");
 			goto fail_end_io;
diff --git a/drivers/lightnvm/pblk-recovery.c b/drivers/lightnvm/pblk-recovery.c
index 3e079c2afa6e..364ad52a5bfb 100644
--- a/drivers/lightnvm/pblk-recovery.c
+++ b/drivers/lightnvm/pblk-recovery.c
@@ -60,7 +60,7 @@ void pblk_submit_rec(struct work_struct *work)
 		goto err;
 	}
 
-	mempool_free(recovery, pblk->rec_pool);
+	mempool_free(recovery, &pblk->rec_pool);
 	return;
 
 err:
diff --git a/drivers/lightnvm/pblk-write.c b/drivers/lightnvm/pblk-write.c
index 3e6f1ebd743a..aef7fa2d401d 100644
--- a/drivers/lightnvm/pblk-write.c
+++ b/drivers/lightnvm/pblk-write.c
@@ -122,7 +122,7 @@ static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd)
 	if (unlikely(nr_ppas == 1))
 		ppa_list = &rqd->ppa_addr;
 
-	recovery = mempool_alloc(pblk->rec_pool, GFP_ATOMIC);
+	recovery = mempool_alloc(&pblk->rec_pool, GFP_ATOMIC);
 
 	INIT_LIST_HEAD(&recovery->failed);
 
@@ -134,7 +134,7 @@ static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd)
 		/* Logic error */
 		if (bit > c_ctx->nr_valid) {
 			WARN_ONCE(1, "pblk: corrupted write request\n");
-			mempool_free(recovery, pblk->rec_pool);
+			mempool_free(recovery, &pblk->rec_pool);
 			goto out;
 		}
 
@@ -142,7 +142,7 @@ static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd)
 		entry = pblk_rb_sync_scan_entry(&pblk->rwb, &ppa);
 		if (!entry) {
 			pr_err("pblk: could not scan entry on write failure\n");
-			mempool_free(recovery, pblk->rec_pool);
+			mempool_free(recovery, &pblk->rec_pool);
 			goto out;
 		}
 
@@ -156,7 +156,7 @@ static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd)
 	ret = pblk_recov_setup_rq(pblk, c_ctx, recovery, comp_bits, c_entries);
 	if (ret) {
 		pr_err("pblk: could not recover from write failure\n");
-		mempool_free(recovery, pblk->rec_pool);
+		mempool_free(recovery, &pblk->rec_pool);
 		goto out;
 	}
 
diff --git a/drivers/lightnvm/pblk.h b/drivers/lightnvm/pblk.h
index 9c682acfc5d1..feafa4de264d 100644
--- a/drivers/lightnvm/pblk.h
+++ b/drivers/lightnvm/pblk.h
@@ -664,12 +664,12 @@ struct pblk {
 
 	struct list_head compl_list;
 
-	mempool_t *page_bio_pool;
-	mempool_t *gen_ws_pool;
-	mempool_t *rec_pool;
-	mempool_t *r_rq_pool;
-	mempool_t *w_rq_pool;
-	mempool_t *e_rq_pool;
+	mempool_t page_bio_pool;
+	mempool_t gen_ws_pool;
+	mempool_t rec_pool;
+	mempool_t r_rq_pool;
+	mempool_t w_rq_pool;
+	mempool_t e_rq_pool;
 
 	struct workqueue_struct *close_wq;
 	struct workqueue_struct *bb_wq;
@@ -841,7 +841,7 @@ void pblk_write_should_kick(struct pblk *pblk);
 /*
  * pblk read path
  */
-extern struct bio_set *pblk_bio_set;
+extern struct bio_set pblk_bio_set;
 int pblk_submit_read(struct pblk *pblk, struct bio *bio);
 int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq);
 /*
diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c
index 433dbeddfcf9..25c1ce811053 100644
--- a/drivers/macintosh/via-pmu.c
+++ b/drivers/macintosh/via-pmu.c
@@ -191,10 +191,10 @@ static int init_pmu(void);
 static void pmu_start(void);
 static irqreturn_t via_pmu_interrupt(int irq, void *arg);
 static irqreturn_t gpio1_interrupt(int irq, void *arg);
-static const struct file_operations pmu_info_proc_fops;
-static const struct file_operations pmu_irqstats_proc_fops;
+static int pmu_info_proc_show(struct seq_file *m, void *v);
+static int pmu_irqstats_proc_show(struct seq_file *m, void *v);
+static int pmu_battery_proc_show(struct seq_file *m, void *v);
 static void pmu_pass_intr(unsigned char *data, int len);
-static const struct file_operations pmu_battery_proc_fops;
 static const struct file_operations pmu_options_proc_fops;
 
 #ifdef CONFIG_ADB
@@ -274,7 +274,7 @@ int __init find_via_pmu(void)
 	u64 taddr;
 	const u32 *reg;
 
-	if (via != 0)
+	if (via)
 		return 1;
 	vias = of_find_node_by_name(NULL, "via-pmu");
 	if (vias == NULL)
@@ -511,13 +511,15 @@ static int __init via_pmu_dev_init(void)
 		for (i=0; i<pmu_battery_count; i++) {
 			char title[16];
 			sprintf(title, "battery_%ld", i);
-			proc_pmu_batt[i] = proc_create_data(title, 0, proc_pmu_root,
-					&pmu_battery_proc_fops, (void *)i);
+			proc_pmu_batt[i] = proc_create_single_data(title, 0,
+					proc_pmu_root, pmu_battery_proc_show,
+					(void *)i);
 		}
 
-		proc_pmu_info = proc_create("info", 0, proc_pmu_root, &pmu_info_proc_fops);
-		proc_pmu_irqstats = proc_create("interrupts", 0, proc_pmu_root,
-						&pmu_irqstats_proc_fops);
+		proc_pmu_info = proc_create_single("info", 0, proc_pmu_root,
+				pmu_info_proc_show);
+		proc_pmu_irqstats = proc_create_single("interrupts", 0,
+				proc_pmu_root, pmu_irqstats_proc_show);
 		proc_pmu_options = proc_create("options", 0600, proc_pmu_root,
 						&pmu_options_proc_fops);
 	}
@@ -811,19 +813,6 @@ static int pmu_info_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int pmu_info_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, pmu_info_proc_show, NULL);
-}
-
-static const struct file_operations pmu_info_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= pmu_info_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int pmu_irqstats_proc_show(struct seq_file *m, void *v)
 {
 	int i;
@@ -848,19 +837,6 @@ static int pmu_irqstats_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int pmu_irqstats_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, pmu_irqstats_proc_show, NULL);
-}
-
-static const struct file_operations pmu_irqstats_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= pmu_irqstats_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int pmu_battery_proc_show(struct seq_file *m, void *v)
 {
 	long batnum = (long)m->private;
@@ -875,19 +851,6 @@ static int pmu_battery_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int pmu_battery_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, pmu_battery_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations pmu_battery_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= pmu_battery_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int pmu_options_proc_show(struct seq_file *m, void *v)
 {
 #if defined(CONFIG_SUSPEND) && defined(CONFIG_PPC32)
@@ -1152,7 +1115,7 @@ pmu_queue_request(struct adb_request *req)
 	req->complete = 0;
 
 	spin_lock_irqsave(&pmu_lock, flags);
-	if (current_req != 0) {
+	if (current_req) {
 		last_req->next = req;
 		last_req = req;
 	} else {
@@ -1227,7 +1190,7 @@ pmu_start(void)
 	/* assert pmu_state == idle */
 	/* get the packet to send */
 	req = current_req;
-	if (req == 0 || pmu_state != idle
+	if (!req || pmu_state != idle
 	    || (/*req->reply_expected && */req_awaiting_reply))
 		return;
 
@@ -1382,7 +1345,7 @@ pmu_handle_data(unsigned char *data, int len)
 	if ((1 << pirq) & PMU_INT_ADB) {
 		if ((data[0] & PMU_INT_ADB_AUTO) == 0) {
 			struct adb_request *req = req_awaiting_reply;
-			if (req == 0) {
+			if (!req) {
 				printk(KERN_ERR "PMU: extra ADB reply\n");
 				return;
 			}
@@ -1749,7 +1712,7 @@ pmu_shutdown(void)
 int
 pmu_present(void)
 {
-	return via != 0;
+	return via != NULL;
 }
 
 #if defined(CONFIG_SUSPEND) && defined(CONFIG_PPC32)
@@ -2081,7 +2044,7 @@ pmu_open(struct inode *inode, struct file *file)
 	unsigned long flags;
 
 	pp = kmalloc(sizeof(struct pmu_private), GFP_KERNEL);
-	if (pp == 0)
+	if (!pp)
 		return -ENOMEM;
 	pp->rb_get = pp->rb_put = 0;
 	spin_lock_init(&pp->lock);
@@ -2107,7 +2070,7 @@ pmu_read(struct file *file, char __user *buf,
 	unsigned long flags;
 	int ret = 0;
 
-	if (count < 1 || pp == 0)
+	if (count < 1 || !pp)
 		return -EINVAL;
 	if (!access_ok(VERIFY_WRITE, buf, count))
 		return -EFAULT;
@@ -2164,7 +2127,7 @@ pmu_fpoll(struct file *filp, poll_table *wait)
 	__poll_t mask = 0;
 	unsigned long flags;
 	
-	if (pp == 0)
+	if (!pp)
 		return 0;
 	poll_wait(filp, &pp->wait, wait);
 	spin_lock_irqsave(&pp->lock, flags);
@@ -2180,7 +2143,7 @@ pmu_release(struct inode *inode, struct file *file)
 	struct pmu_private *pp = file->private_data;
 	unsigned long flags;
 
-	if (pp != 0) {
+	if (pp) {
 		file->private_data = NULL;
 		spin_lock_irqsave(&all_pvt_lock, flags);
 		list_del(&pp->list);
diff --git a/drivers/macintosh/windfarm_pm121.c b/drivers/macintosh/windfarm_pm121.c
index 4d72d8f58cb6..97634e030991 100644
--- a/drivers/macintosh/windfarm_pm121.c
+++ b/drivers/macintosh/windfarm_pm121.c
@@ -710,7 +710,7 @@ static void pm121_create_cpu_fans(void)
 	wf_cpu_pid_init(&pm121_cpu_state->pid, &pid_param);
 
 	pr_debug("pm121: CPU Fan control initialized.\n");
-	pr_debug("       ttarged=%d.%03d, tmax=%d.%03d, min=%d RPM, max=%d RPM,\n",
+	pr_debug("       ttarget=%d.%03d, tmax=%d.%03d, min=%d RPM, max=%d RPM,\n",
 		 FIX32TOPRINT(pid_param.ttarget), FIX32TOPRINT(pid_param.tmax),
 		 pid_param.min, pid_param.max);
 
diff --git a/drivers/macintosh/windfarm_pm81.c b/drivers/macintosh/windfarm_pm81.c
index d9ea45581b9e..346c91744442 100644
--- a/drivers/macintosh/windfarm_pm81.c
+++ b/drivers/macintosh/windfarm_pm81.c
@@ -455,7 +455,7 @@ static void wf_smu_create_cpu_fans(void)
 	wf_cpu_pid_init(&wf_smu_cpu_fans->pid, &pid_param);
 
 	DBG("wf: CPU Fan control initialized.\n");
-	DBG("    ttarged=%d.%03d, tmax=%d.%03d, min=%d RPM, max=%d RPM\n",
+	DBG("    ttarget=%d.%03d, tmax=%d.%03d, min=%d RPM, max=%d RPM\n",
 	    FIX32TOPRINT(pid_param.ttarget), FIX32TOPRINT(pid_param.tmax),
 	    pid_param.min, pid_param.max);
 
diff --git a/drivers/macintosh/windfarm_pm91.c b/drivers/macintosh/windfarm_pm91.c
index 7fd73dcb2b0a..a8a47c62a74b 100644
--- a/drivers/macintosh/windfarm_pm91.c
+++ b/drivers/macintosh/windfarm_pm91.c
@@ -200,7 +200,7 @@ static void wf_smu_create_cpu_fans(void)
 	wf_cpu_pid_init(&wf_smu_cpu_fans->pid, &pid_param);
 
 	DBG("wf: CPU Fan control initialized.\n");
-	DBG("    ttarged=%d.%03d, tmax=%d.%03d, min=%d RPM, max=%d RPM\n",
+	DBG("    ttarget=%d.%03d, tmax=%d.%03d, min=%d RPM, max=%d RPM\n",
 	    FIX32TOPRINT(pid_param.ttarget), FIX32TOPRINT(pid_param.tmax),
 	    pid_param.min, pid_param.max);
 
diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig
index a2bb27446dce..725dce5ba62d 100644
--- a/drivers/mailbox/Kconfig
+++ b/drivers/mailbox/Kconfig
@@ -162,7 +162,6 @@ config XGENE_SLIMPRO_MBOX
 config BCM_PDC_MBOX
 	tristate "Broadcom FlexSparx DMA Mailbox"
 	depends on ARCH_BCM_IPROC || COMPILE_TEST
-	depends on HAS_DMA
 	help
 	  Mailbox implementation for the Broadcom FlexSparx DMA ring manager,
 	  which provides access to various offload engines on Broadcom
@@ -172,7 +171,6 @@ config BCM_FLEXRM_MBOX
 	tristate "Broadcom FlexRM Mailbox"
 	depends on ARM64
 	depends on ARCH_BCM_IPROC || COMPILE_TEST
-	depends on HAS_DMA
 	select GENERIC_MSI_IRQ_DOMAIN
 	default m if ARCH_BCM_IPROC
 	help
diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c
index 3ef7f036ceea..fc3c237daef2 100644
--- a/drivers/mailbox/pcc.c
+++ b/drivers/mailbox/pcc.c
@@ -373,33 +373,24 @@ static const struct mbox_chan_ops pcc_chan_ops = {
 };
 
 /**
- * parse_pcc_subspace - Parse the PCC table and verify PCC subspace
- *		entries. There should be one entry per PCC client.
+ * parse_pcc_subspaces -- Count PCC subspaces defined
  * @header: Pointer to the ACPI subtable header under the PCCT.
  * @end: End of subtable entry.
  *
- * Return: 0 for Success, else errno.
+ * Return: If we find a PCC subspace entry of a valid type, return 0.
+ *	Otherwise, return -EINVAL.
  *
  * This gets called for each entry in the PCC table.
  */
 static int parse_pcc_subspace(struct acpi_subtable_header *header,
 		const unsigned long end)
 {
-	struct acpi_pcct_hw_reduced *pcct_ss;
-
-	if (pcc_mbox_ctrl.num_chans <= MAX_PCC_SUBSPACES) {
-		pcct_ss = (struct acpi_pcct_hw_reduced *) header;
+	struct acpi_pcct_subspace *ss = (struct acpi_pcct_subspace *) header;
 
-		if ((pcct_ss->header.type !=
-				ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE)
-		    && (pcct_ss->header.type !=
-				ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE_TYPE2)) {
-			pr_err("Incorrect PCC Subspace type detected\n");
-			return -EINVAL;
-		}
-	}
+	if (ss->header.type < ACPI_PCCT_TYPE_RESERVED)
+		return 0;
 
-	return 0;
+	return -EINVAL;
 }
 
 /**
@@ -449,8 +440,8 @@ static int __init acpi_pcc_probe(void)
 	struct acpi_table_header *pcct_tbl;
 	struct acpi_subtable_header *pcct_entry;
 	struct acpi_table_pcct *acpi_pcct_tbl;
+	struct acpi_subtable_proc proc[ACPI_PCCT_TYPE_RESERVED];
 	int count, i, rc;
-	int sum = 0;
 	acpi_status status = AE_OK;
 
 	/* Search for PCCT */
@@ -459,43 +450,41 @@ static int __init acpi_pcc_probe(void)
 	if (ACPI_FAILURE(status) || !pcct_tbl)
 		return -ENODEV;
 
-	count = acpi_table_parse_entries(ACPI_SIG_PCCT,
-			sizeof(struct acpi_table_pcct),
-			ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE,
-			parse_pcc_subspace, MAX_PCC_SUBSPACES);
-	sum += (count > 0) ? count : 0;
-
-	count = acpi_table_parse_entries(ACPI_SIG_PCCT,
-			sizeof(struct acpi_table_pcct),
-			ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE_TYPE2,
-			parse_pcc_subspace, MAX_PCC_SUBSPACES);
-	sum += (count > 0) ? count : 0;
+	/* Set up the subtable handlers */
+	for (i = ACPI_PCCT_TYPE_GENERIC_SUBSPACE;
+	     i < ACPI_PCCT_TYPE_RESERVED; i++) {
+		proc[i].id = i;
+		proc[i].count = 0;
+		proc[i].handler = parse_pcc_subspace;
+	}
 
-	if (sum == 0 || sum >= MAX_PCC_SUBSPACES) {
-		pr_err("Error parsing PCC subspaces from PCCT\n");
+	count = acpi_table_parse_entries_array(ACPI_SIG_PCCT,
+			sizeof(struct acpi_table_pcct), proc,
+			ACPI_PCCT_TYPE_RESERVED, MAX_PCC_SUBSPACES);
+	if (count == 0 || count > MAX_PCC_SUBSPACES) {
+		pr_warn("Invalid PCCT: %d PCC subspaces\n", count);
 		return -EINVAL;
 	}
 
-	pcc_mbox_channels = kzalloc(sizeof(struct mbox_chan) *
-			sum, GFP_KERNEL);
+	pcc_mbox_channels = kzalloc(sizeof(struct mbox_chan) * count, GFP_KERNEL);
 	if (!pcc_mbox_channels) {
 		pr_err("Could not allocate space for PCC mbox channels\n");
 		return -ENOMEM;
 	}
 
-	pcc_doorbell_vaddr = kcalloc(sum, sizeof(void *), GFP_KERNEL);
+	pcc_doorbell_vaddr = kcalloc(count, sizeof(void *), GFP_KERNEL);
 	if (!pcc_doorbell_vaddr) {
 		rc = -ENOMEM;
 		goto err_free_mbox;
 	}
 
-	pcc_doorbell_ack_vaddr = kcalloc(sum, sizeof(void *), GFP_KERNEL);
+	pcc_doorbell_ack_vaddr = kcalloc(count, sizeof(void *), GFP_KERNEL);
 	if (!pcc_doorbell_ack_vaddr) {
 		rc = -ENOMEM;
 		goto err_free_db_vaddr;
 	}
 
-	pcc_doorbell_irq = kcalloc(sum, sizeof(int), GFP_KERNEL);
+	pcc_doorbell_irq = kcalloc(count, sizeof(int), GFP_KERNEL);
 	if (!pcc_doorbell_irq) {
 		rc = -ENOMEM;
 		goto err_free_db_ack_vaddr;
@@ -509,18 +498,24 @@ static int __init acpi_pcc_probe(void)
 	if (acpi_pcct_tbl->flags & ACPI_PCCT_DOORBELL)
 		pcc_mbox_ctrl.txdone_irq = true;
 
-	for (i = 0; i < sum; i++) {
+	for (i = 0; i < count; i++) {
 		struct acpi_generic_address *db_reg;
-		struct acpi_pcct_hw_reduced *pcct_ss;
+		struct acpi_pcct_subspace *pcct_ss;
 		pcc_mbox_channels[i].con_priv = pcct_entry;
 
-		pcct_ss = (struct acpi_pcct_hw_reduced *) pcct_entry;
+		if (pcct_entry->type == ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE ||
+		    pcct_entry->type == ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE_TYPE2) {
+			struct acpi_pcct_hw_reduced *pcct_hrss;
+
+			pcct_hrss = (struct acpi_pcct_hw_reduced *) pcct_entry;
 
-		if (pcc_mbox_ctrl.txdone_irq) {
-			rc = pcc_parse_subspace_irq(i, pcct_ss);
-			if (rc < 0)
-				goto err;
+			if (pcc_mbox_ctrl.txdone_irq) {
+				rc = pcc_parse_subspace_irq(i, pcct_hrss);
+				if (rc < 0)
+					goto err;
+			}
 		}
+		pcct_ss = (struct acpi_pcct_subspace *) pcct_entry;
 
 		/* If doorbell is in system memory cache the virt address */
 		db_reg = &pcct_ss->doorbell_register;
@@ -531,7 +526,7 @@ static int __init acpi_pcc_probe(void)
 			((unsigned long) pcct_entry + pcct_entry->length);
 	}
 
-	pcc_mbox_ctrl.num_chans = sum;
+	pcc_mbox_ctrl.num_chans = count;
 
 	pr_info("Detected %d PCC Subspaces\n", pcc_mbox_ctrl.num_chans);
 
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index edff083f7c4e..8b8c123cae66 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -334,6 +334,17 @@ config DM_CACHE_SMQ
          of less memory utilization, improved performance and increased
          adaptability in the face of changing workloads.
 
+config DM_WRITECACHE
+	tristate "Writecache target"
+	depends on BLK_DEV_DM
+	---help---
+	   The writecache target caches writes on persistent memory or SSD.
+	   It is intended for databases or other programs that need extremely
+	   low commit latency.
+
+	   The writecache target doesn't cache reads because reads are supposed
+	   to be cached in standard RAM.
+
 config DM_ERA
        tristate "Era target (EXPERIMENTAL)"
        depends on BLK_DEV_DM
diff --git a/drivers/md/Makefile b/drivers/md/Makefile
index 63255f3ebd97..822f4e8753bc 100644
--- a/drivers/md/Makefile
+++ b/drivers/md/Makefile
@@ -67,6 +67,7 @@ obj-$(CONFIG_DM_ERA)		+= dm-era.o
 obj-$(CONFIG_DM_LOG_WRITES)	+= dm-log-writes.o
 obj-$(CONFIG_DM_INTEGRITY)	+= dm-integrity.o
 obj-$(CONFIG_DM_ZONED)		+= dm-zoned.o
+obj-$(CONFIG_DM_WRITECACHE)	+= dm-writecache.o
 
 ifeq ($(CONFIG_DM_UEVENT),y)
 dm-mod-objs			+= dm-uevent.o
diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index 3a0cfb237af9..d6bf294f3907 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -269,7 +269,7 @@ struct bcache_device {
 	atomic_t		*stripe_sectors_dirty;
 	unsigned long		*full_dirty_stripes;
 
-	struct bio_set		*bio_split;
+	struct bio_set		bio_split;
 
 	unsigned		data_csum:1;
 
@@ -345,6 +345,7 @@ struct cached_dev {
 
 	struct keybuf		writeback_keys;
 
+	struct task_struct	*status_update_thread;
 	/*
 	 * Order the write-half of writeback operations strongly in dispatch
 	 * order.  (Maintain LBA order; don't allow reads completing out of
@@ -392,6 +393,7 @@ struct cached_dev {
 #define DEFAULT_CACHED_DEV_ERROR_LIMIT	64
 	atomic_t		io_errors;
 	unsigned		error_limit;
+	unsigned		offline_seconds;
 
 	char			backing_dev_name[BDEVNAME_SIZE];
 };
@@ -528,9 +530,9 @@ struct cache_set {
 	struct closure		sb_write;
 	struct semaphore	sb_write_mutex;
 
-	mempool_t		*search;
-	mempool_t		*bio_meta;
-	struct bio_set		*bio_split;
+	mempool_t		search;
+	mempool_t		bio_meta;
+	struct bio_set		bio_split;
 
 	/* For the btree cache */
 	struct shrinker		shrink;
@@ -655,7 +657,7 @@ struct cache_set {
 	 * A btree node on disk could have too many bsets for an iterator to fit
 	 * on the stack - have to dynamically allocate them
 	 */
-	mempool_t		*fill_iter;
+	mempool_t		fill_iter;
 
 	struct bset_sort_state	sort;
 
@@ -956,8 +958,6 @@ void bch_prio_write(struct cache *);
 void bch_write_bdev_super(struct cached_dev *, struct closure *);
 
 extern struct workqueue_struct *bcache_wq;
-extern const char * const bch_cache_modes[];
-extern const char * const bch_stop_on_failure_modes[];
 extern struct mutex bch_register_lock;
 extern struct list_head bch_cache_sets;
 
diff --git a/drivers/md/bcache/bset.c b/drivers/md/bcache/bset.c
index 579c696a5fe0..f3403b45bc28 100644
--- a/drivers/md/bcache/bset.c
+++ b/drivers/md/bcache/bset.c
@@ -1118,8 +1118,7 @@ struct bkey *bch_btree_iter_next_filter(struct btree_iter *iter,
 
 void bch_bset_sort_state_free(struct bset_sort_state *state)
 {
-	if (state->pool)
-		mempool_destroy(state->pool);
+	mempool_exit(&state->pool);
 }
 
 int bch_bset_sort_state_init(struct bset_sort_state *state, unsigned page_order)
@@ -1129,11 +1128,7 @@ int bch_bset_sort_state_init(struct bset_sort_state *state, unsigned page_order)
 	state->page_order = page_order;
 	state->crit_factor = int_sqrt(1 << page_order);
 
-	state->pool = mempool_create_page_pool(1, page_order);
-	if (!state->pool)
-		return -ENOMEM;
-
-	return 0;
+	return mempool_init_page_pool(&state->pool, 1, page_order);
 }
 EXPORT_SYMBOL(bch_bset_sort_state_init);
 
@@ -1191,7 +1186,7 @@ static void __btree_sort(struct btree_keys *b, struct btree_iter *iter,
 
 		BUG_ON(order > state->page_order);
 
-		outp = mempool_alloc(state->pool, GFP_NOIO);
+		outp = mempool_alloc(&state->pool, GFP_NOIO);
 		out = page_address(outp);
 		used_mempool = true;
 		order = state->page_order;
@@ -1220,7 +1215,7 @@ static void __btree_sort(struct btree_keys *b, struct btree_iter *iter,
 	}
 
 	if (used_mempool)
-		mempool_free(virt_to_page(out), state->pool);
+		mempool_free(virt_to_page(out), &state->pool);
 	else
 		free_pages((unsigned long) out, order);
 
diff --git a/drivers/md/bcache/bset.h b/drivers/md/bcache/bset.h
index 0c24280f3b98..b867f2200495 100644
--- a/drivers/md/bcache/bset.h
+++ b/drivers/md/bcache/bset.h
@@ -347,7 +347,7 @@ static inline struct bkey *bch_bset_search(struct btree_keys *b,
 /* Sorting */
 
 struct bset_sort_state {
-	mempool_t		*pool;
+	mempool_t		pool;
 
 	unsigned		page_order;
 	unsigned		crit_factor;
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
index 17936b2dc7d6..2a0968c04e21 100644
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -204,7 +204,7 @@ void bch_btree_node_read_done(struct btree *b)
 	struct bset *i = btree_bset_first(b);
 	struct btree_iter *iter;
 
-	iter = mempool_alloc(b->c->fill_iter, GFP_NOIO);
+	iter = mempool_alloc(&b->c->fill_iter, GFP_NOIO);
 	iter->size = b->c->sb.bucket_size / b->c->sb.block_size;
 	iter->used = 0;
 
@@ -271,7 +271,7 @@ void bch_btree_node_read_done(struct btree *b)
 		bch_bset_init_next(&b->keys, write_block(b),
 				   bset_magic(&b->c->sb));
 out:
-	mempool_free(iter, b->c->fill_iter);
+	mempool_free(iter, &b->c->fill_iter);
 	return;
 err:
 	set_btree_node_io_error(b);
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index 2ddf8515e6a5..9612873afee2 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -17,12 +17,12 @@
 void bch_bbio_free(struct bio *bio, struct cache_set *c)
 {
 	struct bbio *b = container_of(bio, struct bbio, bio);
-	mempool_free(b, c->bio_meta);
+	mempool_free(b, &c->bio_meta);
 }
 
 struct bio *bch_bbio_alloc(struct cache_set *c)
 {
-	struct bbio *b = mempool_alloc(c->bio_meta, GFP_NOIO);
+	struct bbio *b = mempool_alloc(&c->bio_meta, GFP_NOIO);
 	struct bio *bio = &b->bio;
 
 	bio_init(bio, bio->bi_inline_vecs, bucket_pages(c));
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index 8e3e8655ed63..ae67f5fa8047 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -213,7 +213,7 @@ static void bch_data_insert_start(struct closure *cl)
 	do {
 		unsigned i;
 		struct bkey *k;
-		struct bio_set *split = op->c->bio_split;
+		struct bio_set *split = &op->c->bio_split;
 
 		/* 1 for the device pointer and 1 for the chksum */
 		if (bch_keylist_realloc(&op->insert_keys,
@@ -548,7 +548,7 @@ static int cache_lookup_fn(struct btree_op *op, struct btree *b, struct bkey *k)
 
 	n = bio_next_split(bio, min_t(uint64_t, INT_MAX,
 				      KEY_OFFSET(k) - bio->bi_iter.bi_sector),
-			   GFP_NOIO, s->d->bio_split);
+			   GFP_NOIO, &s->d->bio_split);
 
 	bio_key = &container_of(n, struct bbio, bio)->key;
 	bch_bkey_copy_single_ptr(bio_key, k, ptr);
@@ -707,7 +707,7 @@ static void search_free(struct closure *cl)
 
 	bio_complete(s);
 	closure_debug_destroy(cl);
-	mempool_free(s, s->d->c->search);
+	mempool_free(s, &s->d->c->search);
 }
 
 static inline struct search *search_alloc(struct bio *bio,
@@ -715,7 +715,7 @@ static inline struct search *search_alloc(struct bio *bio,
 {
 	struct search *s;
 
-	s = mempool_alloc(d->c->search, GFP_NOIO);
+	s = mempool_alloc(&d->c->search, GFP_NOIO);
 
 	closure_init(&s->cl, NULL);
 	do_bio_hook(s, bio, request_endio);
@@ -864,7 +864,7 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s,
 	s->cache_missed = 1;
 
 	if (s->cache_miss || s->iop.bypass) {
-		miss = bio_next_split(bio, sectors, GFP_NOIO, s->d->bio_split);
+		miss = bio_next_split(bio, sectors, GFP_NOIO, &s->d->bio_split);
 		ret = miss == bio ? MAP_DONE : MAP_CONTINUE;
 		goto out_submit;
 	}
@@ -887,14 +887,14 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s,
 
 	s->iop.replace = true;
 
-	miss = bio_next_split(bio, sectors, GFP_NOIO, s->d->bio_split);
+	miss = bio_next_split(bio, sectors, GFP_NOIO, &s->d->bio_split);
 
 	/* btree_search_recurse()'s btree iterator is no good anymore */
 	ret = miss == bio ? MAP_DONE : -EINTR;
 
 	cache_bio = bio_alloc_bioset(GFP_NOWAIT,
 			DIV_ROUND_UP(s->insert_bio_sectors, PAGE_SECTORS),
-			dc->disk.bio_split);
+			&dc->disk.bio_split);
 	if (!cache_bio)
 		goto out_submit;
 
@@ -1008,7 +1008,7 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
 			struct bio *flush;
 
 			flush = bio_alloc_bioset(GFP_NOIO, 0,
-						 dc->disk.bio_split);
+						 &dc->disk.bio_split);
 			if (!flush) {
 				s->iop.status = BLK_STS_RESOURCE;
 				goto insert_data;
@@ -1021,7 +1021,7 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
 			closure_bio_submit(s->iop.c, flush, cl);
 		}
 	} else {
-		s->iop.bio = bio_clone_fast(bio, GFP_NOIO, dc->disk.bio_split);
+		s->iop.bio = bio_clone_fast(bio, GFP_NOIO, &dc->disk.bio_split);
 		/* I/O request sent to backing device */
 		bio->bi_end_io = backing_request_endio;
 		closure_bio_submit(s->iop.c, bio, cl);
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index 3dea06b41d43..a31e55bcc4e5 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -37,24 +37,6 @@ static const char invalid_uuid[] = {
 	0xc8, 0x50, 0xfc, 0x5e, 0xcb, 0x16, 0xcd, 0x99
 };
 
-/* Default is -1; we skip past it for struct cached_dev's cache mode */
-const char * const bch_cache_modes[] = {
-	"default",
-	"writethrough",
-	"writeback",
-	"writearound",
-	"none",
-	NULL
-};
-
-/* Default is -1; we skip past it for stop_when_cache_set_failed */
-const char * const bch_stop_on_failure_modes[] = {
-	"default",
-	"auto",
-	"always",
-	NULL
-};
-
 static struct kobject *bcache_kobj;
 struct mutex bch_register_lock;
 LIST_HEAD(bch_cache_sets);
@@ -654,6 +636,11 @@ static int ioctl_dev(struct block_device *b, fmode_t mode,
 		     unsigned int cmd, unsigned long arg)
 {
 	struct bcache_device *d = b->bd_disk->private_data;
+	struct cached_dev *dc = container_of(d, struct cached_dev, disk);
+
+	if (dc->io_disable)
+		return -EIO;
+
 	return d->ioctl(d, mode, cmd, arg);
 }
 
@@ -766,8 +753,7 @@ static void bcache_device_free(struct bcache_device *d)
 		put_disk(d->disk);
 	}
 
-	if (d->bio_split)
-		bioset_free(d->bio_split);
+	bioset_exit(&d->bio_split);
 	kvfree(d->full_dirty_stripes);
 	kvfree(d->stripe_sectors_dirty);
 
@@ -809,9 +795,8 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size,
 	if (idx < 0)
 		return idx;
 
-	if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio),
-					   BIOSET_NEED_BVECS |
-					   BIOSET_NEED_RESCUER)) ||
+	if (bioset_init(&d->bio_split, 4, offsetof(struct bbio, bio),
+			BIOSET_NEED_BVECS|BIOSET_NEED_RESCUER) ||
 	    !(d->disk = alloc_disk(BCACHE_MINORS))) {
 		ida_simple_remove(&bcache_device_idx, idx);
 		return -ENOMEM;
@@ -864,6 +849,44 @@ static void calc_cached_dev_sectors(struct cache_set *c)
 	c->cached_dev_sectors = sectors;
 }
 
+#define BACKING_DEV_OFFLINE_TIMEOUT 5
+static int cached_dev_status_update(void *arg)
+{
+	struct cached_dev *dc = arg;
+	struct request_queue *q;
+
+	/*
+	 * If this delayed worker is stopping outside, directly quit here.
+	 * dc->io_disable might be set via sysfs interface, so check it
+	 * here too.
+	 */
+	while (!kthread_should_stop() && !dc->io_disable) {
+		q = bdev_get_queue(dc->bdev);
+		if (blk_queue_dying(q))
+			dc->offline_seconds++;
+		else
+			dc->offline_seconds = 0;
+
+		if (dc->offline_seconds >= BACKING_DEV_OFFLINE_TIMEOUT) {
+			pr_err("%s: device offline for %d seconds",
+			       dc->backing_dev_name,
+			       BACKING_DEV_OFFLINE_TIMEOUT);
+			pr_err("%s: disable I/O request due to backing "
+			       "device offline", dc->disk.name);
+			dc->io_disable = true;
+			/* let others know earlier that io_disable is true */
+			smp_mb();
+			bcache_device_stop(&dc->disk);
+			break;
+		}
+		schedule_timeout_interruptible(HZ);
+	}
+
+	wait_for_kthread_stop();
+	return 0;
+}
+
+
 void bch_cached_dev_run(struct cached_dev *dc)
 {
 	struct bcache_device *d = &dc->disk;
@@ -906,6 +929,14 @@ void bch_cached_dev_run(struct cached_dev *dc)
 	if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") ||
 	    sysfs_create_link(&disk_to_dev(d->disk)->kobj, &d->kobj, "bcache"))
 		pr_debug("error creating sysfs link");
+
+	dc->status_update_thread = kthread_run(cached_dev_status_update,
+					       dc, "bcache_status_update");
+	if (IS_ERR(dc->status_update_thread)) {
+		pr_warn("failed to create bcache_status_update kthread, "
+			"continue to run without monitoring backing "
+			"device status");
+	}
 }
 
 /*
@@ -1139,6 +1170,8 @@ static void cached_dev_free(struct closure *cl)
 		kthread_stop(dc->writeback_thread);
 	if (dc->writeback_write_wq)
 		destroy_workqueue(dc->writeback_write_wq);
+	if (!IS_ERR_OR_NULL(dc->status_update_thread))
+		kthread_stop(dc->status_update_thread);
 
 	if (atomic_read(&dc->running))
 		bd_unlink_disk_holder(dc->bdev, dc->disk.disk);
@@ -1465,14 +1498,10 @@ static void cache_set_free(struct closure *cl)
 
 	if (c->moving_gc_wq)
 		destroy_workqueue(c->moving_gc_wq);
-	if (c->bio_split)
-		bioset_free(c->bio_split);
-	if (c->fill_iter)
-		mempool_destroy(c->fill_iter);
-	if (c->bio_meta)
-		mempool_destroy(c->bio_meta);
-	if (c->search)
-		mempool_destroy(c->search);
+	bioset_exit(&c->bio_split);
+	mempool_exit(&c->fill_iter);
+	mempool_exit(&c->bio_meta);
+	mempool_exit(&c->search);
 	kfree(c->devices);
 
 	mutex_lock(&bch_register_lock);
@@ -1683,21 +1712,17 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)
 	INIT_LIST_HEAD(&c->btree_cache_freed);
 	INIT_LIST_HEAD(&c->data_buckets);
 
-	c->search = mempool_create_slab_pool(32, bch_search_cache);
-	if (!c->search)
-		goto err;
-
 	iter_size = (sb->bucket_size / sb->block_size + 1) *
 		sizeof(struct btree_iter_set);
 
 	if (!(c->devices = kzalloc(c->nr_uuids * sizeof(void *), GFP_KERNEL)) ||
-	    !(c->bio_meta = mempool_create_kmalloc_pool(2,
-				sizeof(struct bbio) + sizeof(struct bio_vec) *
-				bucket_pages(c))) ||
-	    !(c->fill_iter = mempool_create_kmalloc_pool(1, iter_size)) ||
-	    !(c->bio_split = bioset_create(4, offsetof(struct bbio, bio),
-					   BIOSET_NEED_BVECS |
-					   BIOSET_NEED_RESCUER)) ||
+	    mempool_init_slab_pool(&c->search, 32, bch_search_cache) ||
+	    mempool_init_kmalloc_pool(&c->bio_meta, 2,
+				      sizeof(struct bbio) + sizeof(struct bio_vec) *
+				      bucket_pages(c)) ||
+	    mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size) ||
+	    bioset_init(&c->bio_split, 4, offsetof(struct bbio, bio),
+			BIOSET_NEED_BVECS|BIOSET_NEED_RESCUER) ||
 	    !(c->uuids = alloc_bucket_pages(GFP_KERNEL, c)) ||
 	    !(c->moving_gc_wq = alloc_workqueue("bcache_gc",
 						WQ_MEM_RECLAIM, 0)) ||
diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c
index dfeef583ee50..8ccbc8f3b3af 100644
--- a/drivers/md/bcache/sysfs.c
+++ b/drivers/md/bcache/sysfs.c
@@ -16,6 +16,22 @@
 #include <linux/sort.h>
 #include <linux/sched/clock.h>
 
+/* Default is -1; we skip past it for struct cached_dev's cache mode */
+static const char * const bch_cache_modes[] = {
+	"writethrough",
+	"writeback",
+	"writearound",
+	"none",
+	NULL
+};
+
+/* Default is -1; we skip past it for stop_when_cache_set_failed */
+static const char * const bch_stop_on_failure_modes[] = {
+	"auto",
+	"always",
+	NULL
+};
+
 static const char * const cache_replacement_policies[] = {
 	"lru",
 	"fifo",
@@ -114,6 +130,20 @@ rw_attribute(btree_shrinker_disabled);
 rw_attribute(copy_gc_enabled);
 rw_attribute(size);
 
+static ssize_t bch_snprint_string_list(char *buf, size_t size, const char * const list[],
+			    size_t selected)
+{
+	char *out = buf;
+	size_t i;
+
+	for (i = 0; list[i]; i++)
+		out += snprintf(out, buf + size - out,
+				i == selected ? "[%s] " : "%s ", list[i]);
+
+	out[-1] = '\n';
+	return out - buf;
+}
+
 SHOW(__bch_cached_dev)
 {
 	struct cached_dev *dc = container_of(kobj, struct cached_dev,
@@ -124,12 +154,12 @@ SHOW(__bch_cached_dev)
 
 	if (attr == &sysfs_cache_mode)
 		return bch_snprint_string_list(buf, PAGE_SIZE,
-					       bch_cache_modes + 1,
+					       bch_cache_modes,
 					       BDEV_CACHE_MODE(&dc->sb));
 
 	if (attr == &sysfs_stop_when_cache_set_failed)
 		return bch_snprint_string_list(buf, PAGE_SIZE,
-					       bch_stop_on_failure_modes + 1,
+					       bch_stop_on_failure_modes,
 					       dc->stop_when_cache_set_failed);
 
 
@@ -253,8 +283,7 @@ STORE(__cached_dev)
 		bch_cached_dev_run(dc);
 
 	if (attr == &sysfs_cache_mode) {
-		v = bch_read_string_list(buf, bch_cache_modes + 1);
-
+		v = __sysfs_match_string(bch_cache_modes, -1, buf);
 		if (v < 0)
 			return v;
 
@@ -265,8 +294,7 @@ STORE(__cached_dev)
 	}
 
 	if (attr == &sysfs_stop_when_cache_set_failed) {
-		v = bch_read_string_list(buf, bch_stop_on_failure_modes + 1);
-
+		v = __sysfs_match_string(bch_stop_on_failure_modes, -1, buf);
 		if (v < 0)
 			return v;
 
@@ -635,6 +663,7 @@ SHOW_LOCKED(bch_cache_set)
 STORE(__bch_cache_set)
 {
 	struct cache_set *c = container_of(kobj, struct cache_set, kobj);
+	ssize_t v;
 
 	if (attr == &sysfs_unregister)
 		bch_cache_set_unregister(c);
@@ -698,8 +727,7 @@ STORE(__bch_cache_set)
 		      c->congested_write_threshold_us);
 
 	if (attr == &sysfs_errors) {
-		ssize_t v = bch_read_string_list(buf, error_actions);
-
+		v = __sysfs_match_string(error_actions, -1, buf);
 		if (v < 0)
 			return v;
 
@@ -714,8 +742,7 @@ STORE(__bch_cache_set)
 		c->error_decay = strtoul_or_return(buf) / 88;
 
 	if (attr == &sysfs_io_disable) {
-		int v = strtoul_or_return(buf);
-
+		v = strtoul_or_return(buf);
 		if (v) {
 			if (test_and_set_bit(CACHE_SET_IO_DISABLE,
 					     &c->flags))
@@ -929,6 +956,7 @@ SHOW_LOCKED(bch_cache)
 STORE(__bch_cache)
 {
 	struct cache *ca = container_of(kobj, struct cache, kobj);
+	ssize_t v;
 
 	if (attr == &sysfs_discard) {
 		bool v = strtoul_or_return(buf);
@@ -943,8 +971,7 @@ STORE(__bch_cache)
 	}
 
 	if (attr == &sysfs_cache_replacement_policy) {
-		ssize_t v = bch_read_string_list(buf, cache_replacement_policies);
-
+		v = __sysfs_match_string(cache_replacement_policies, -1, buf);
 		if (v < 0)
 			return v;
 
diff --git a/drivers/md/bcache/util.c b/drivers/md/bcache/util.c
index 74febd5230df..fc479b026d6d 100644
--- a/drivers/md/bcache/util.c
+++ b/drivers/md/bcache/util.c
@@ -120,41 +120,6 @@ ssize_t bch_hprint(char *buf, int64_t v)
 		return sprintf(buf, "%llu.%i%c", q, t * 10 / 1024, units[u]);
 }
 
-ssize_t bch_snprint_string_list(char *buf, size_t size, const char * const list[],
-			    size_t selected)
-{
-	char *out = buf;
-	size_t i;
-
-	for (i = 0; list[i]; i++)
-		out += snprintf(out, buf + size - out,
-				i == selected ? "[%s] " : "%s ", list[i]);
-
-	out[-1] = '\n';
-	return out - buf;
-}
-
-ssize_t bch_read_string_list(const char *buf, const char * const list[])
-{
-	size_t i;
-	char *s, *d = kstrndup(buf, PAGE_SIZE - 1, GFP_KERNEL);
-	if (!d)
-		return -ENOMEM;
-
-	s = strim(d);
-
-	for (i = 0; list[i]; i++)
-		if (!strcmp(list[i], s))
-			break;
-
-	kfree(d);
-
-	if (!list[i])
-		return -EINVAL;
-
-	return i;
-}
-
 bool bch_is_zero(const char *p, size_t n)
 {
 	size_t i;
diff --git a/drivers/md/bcache/util.h b/drivers/md/bcache/util.h
index 268024529edd..cced87f8eb27 100644
--- a/drivers/md/bcache/util.h
+++ b/drivers/md/bcache/util.h
@@ -365,11 +365,6 @@ ssize_t bch_hprint(char *buf, int64_t v);
 bool bch_is_zero(const char *p, size_t n);
 int bch_parse_uuid(const char *s, char *uuid);
 
-ssize_t bch_snprint_string_list(char *buf, size_t size, const char * const list[],
-			    size_t selected);
-
-ssize_t bch_read_string_list(const char *buf, const char * const list[]);
-
 struct time_stats {
 	spinlock_t	lock;
 	/*
diff --git a/drivers/md/dm-bio-prison-v1.c b/drivers/md/dm-bio-prison-v1.c
index 874841f0fc83..8e33a3808368 100644
--- a/drivers/md/dm-bio-prison-v1.c
+++ b/drivers/md/dm-bio-prison-v1.c
@@ -19,7 +19,7 @@
 
 struct dm_bio_prison {
 	spinlock_t lock;
-	mempool_t *cell_pool;
+	mempool_t cell_pool;
 	struct rb_root cells;
 };
 
@@ -34,14 +34,15 @@ static struct kmem_cache *_cell_cache;
 struct dm_bio_prison *dm_bio_prison_create(void)
 {
 	struct dm_bio_prison *prison = kmalloc(sizeof(*prison), GFP_KERNEL);
+	int ret;
 
 	if (!prison)
 		return NULL;
 
 	spin_lock_init(&prison->lock);
 
-	prison->cell_pool = mempool_create_slab_pool(MIN_CELLS, _cell_cache);
-	if (!prison->cell_pool) {
+	ret = mempool_init_slab_pool(&prison->cell_pool, MIN_CELLS, _cell_cache);
+	if (ret) {
 		kfree(prison);
 		return NULL;
 	}
@@ -54,21 +55,21 @@ EXPORT_SYMBOL_GPL(dm_bio_prison_create);
 
 void dm_bio_prison_destroy(struct dm_bio_prison *prison)
 {
-	mempool_destroy(prison->cell_pool);
+	mempool_exit(&prison->cell_pool);
 	kfree(prison);
 }
 EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);
 
 struct dm_bio_prison_cell *dm_bio_prison_alloc_cell(struct dm_bio_prison *prison, gfp_t gfp)
 {
-	return mempool_alloc(prison->cell_pool, gfp);
+	return mempool_alloc(&prison->cell_pool, gfp);
 }
 EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);
 
 void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
 			     struct dm_bio_prison_cell *cell)
 {
-	mempool_free(cell, prison->cell_pool);
+	mempool_free(cell, &prison->cell_pool);
 }
 EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);
 
diff --git a/drivers/md/dm-bio-prison-v2.c b/drivers/md/dm-bio-prison-v2.c
index 8ce3a1a588cf..601b1569206a 100644
--- a/drivers/md/dm-bio-prison-v2.c
+++ b/drivers/md/dm-bio-prison-v2.c
@@ -21,7 +21,7 @@ struct dm_bio_prison_v2 {
 	struct workqueue_struct *wq;
 
 	spinlock_t lock;
-	mempool_t *cell_pool;
+	mempool_t cell_pool;
 	struct rb_root cells;
 };
 
@@ -36,6 +36,7 @@ static struct kmem_cache *_cell_cache;
 struct dm_bio_prison_v2 *dm_bio_prison_create_v2(struct workqueue_struct *wq)
 {
 	struct dm_bio_prison_v2 *prison = kmalloc(sizeof(*prison), GFP_KERNEL);
+	int ret;
 
 	if (!prison)
 		return NULL;
@@ -43,8 +44,8 @@ struct dm_bio_prison_v2 *dm_bio_prison_create_v2(struct workqueue_struct *wq)
 	prison->wq = wq;
 	spin_lock_init(&prison->lock);
 
-	prison->cell_pool = mempool_create_slab_pool(MIN_CELLS, _cell_cache);
-	if (!prison->cell_pool) {
+	ret = mempool_init_slab_pool(&prison->cell_pool, MIN_CELLS, _cell_cache);
+	if (ret) {
 		kfree(prison);
 		return NULL;
 	}
@@ -57,21 +58,21 @@ EXPORT_SYMBOL_GPL(dm_bio_prison_create_v2);
 
 void dm_bio_prison_destroy_v2(struct dm_bio_prison_v2 *prison)
 {
-	mempool_destroy(prison->cell_pool);
+	mempool_exit(&prison->cell_pool);
 	kfree(prison);
 }
 EXPORT_SYMBOL_GPL(dm_bio_prison_destroy_v2);
 
 struct dm_bio_prison_cell_v2 *dm_bio_prison_alloc_cell_v2(struct dm_bio_prison_v2 *prison, gfp_t gfp)
 {
-	return mempool_alloc(prison->cell_pool, gfp);
+	return mempool_alloc(&prison->cell_pool, gfp);
 }
 EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell_v2);
 
 void dm_bio_prison_free_cell_v2(struct dm_bio_prison_v2 *prison,
 				struct dm_bio_prison_cell_v2 *cell)
 {
-	mempool_free(cell, prison->cell_pool);
+	mempool_free(cell, &prison->cell_pool);
 }
 EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell_v2);
 
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index da208638fba4..001c71248246 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -447,9 +447,9 @@ struct cache {
 	struct work_struct migration_worker;
 	struct delayed_work waker;
 	struct dm_bio_prison_v2 *prison;
-	struct bio_set *bs;
+	struct bio_set bs;
 
-	mempool_t *migration_pool;
+	mempool_t migration_pool;
 
 	struct dm_cache_policy *policy;
 	unsigned policy_nr_args;
@@ -550,7 +550,7 @@ static struct dm_cache_migration *alloc_migration(struct cache *cache)
 {
 	struct dm_cache_migration *mg;
 
-	mg = mempool_alloc(cache->migration_pool, GFP_NOWAIT);
+	mg = mempool_alloc(&cache->migration_pool, GFP_NOWAIT);
 	if (!mg)
 		return NULL;
 
@@ -569,7 +569,7 @@ static void free_migration(struct dm_cache_migration *mg)
 	if (atomic_dec_and_test(&cache->nr_allocated_migrations))
 		wake_up(&cache->migration_wait);
 
-	mempool_free(mg, cache->migration_pool);
+	mempool_free(mg, &cache->migration_pool);
 }
 
 /*----------------------------------------------------------------*/
@@ -924,7 +924,7 @@ static void issue_op(struct bio *bio, void *context)
 static void remap_to_origin_and_cache(struct cache *cache, struct bio *bio,
 				      dm_oblock_t oblock, dm_cblock_t cblock)
 {
-	struct bio *origin_bio = bio_clone_fast(bio, GFP_NOIO, cache->bs);
+	struct bio *origin_bio = bio_clone_fast(bio, GFP_NOIO, &cache->bs);
 
 	BUG_ON(!origin_bio);
 
@@ -2011,7 +2011,7 @@ static void destroy(struct cache *cache)
 {
 	unsigned i;
 
-	mempool_destroy(cache->migration_pool);
+	mempool_exit(&cache->migration_pool);
 
 	if (cache->prison)
 		dm_bio_prison_destroy_v2(cache->prison);
@@ -2047,8 +2047,7 @@ static void destroy(struct cache *cache)
 		kfree(cache->ctr_args[i]);
 	kfree(cache->ctr_args);
 
-	if (cache->bs)
-		bioset_free(cache->bs);
+	bioset_exit(&cache->bs);
 
 	kfree(cache);
 }
@@ -2498,8 +2497,8 @@ static int cache_create(struct cache_args *ca, struct cache **result)
 	cache->features = ca->features;
 	if (writethrough_mode(cache)) {
 		/* Create bioset for writethrough bios issued to origin */
-		cache->bs = bioset_create(BIO_POOL_SIZE, 0, 0);
-		if (!cache->bs)
+		r = bioset_init(&cache->bs, BIO_POOL_SIZE, 0, 0);
+		if (r)
 			goto bad;
 	}
 
@@ -2630,9 +2629,9 @@ static int cache_create(struct cache_args *ca, struct cache **result)
 		goto bad;
 	}
 
-	cache->migration_pool = mempool_create_slab_pool(MIGRATION_POOL_SIZE,
-							 migration_cache);
-	if (!cache->migration_pool) {
+	r = mempool_init_slab_pool(&cache->migration_pool, MIGRATION_POOL_SIZE,
+				   migration_cache);
+	if (r) {
 		*error = "Error creating cache's migration mempool";
 		goto bad;
 	}
diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h
index 3222e21cbbf8..f21c5d21bf1b 100644
--- a/drivers/md/dm-core.h
+++ b/drivers/md/dm-core.h
@@ -91,8 +91,8 @@ struct mapped_device {
 	/*
 	 * io objects are allocated from here.
 	 */
-	struct bio_set *io_bs;
-	struct bio_set *bs;
+	struct bio_set io_bs;
+	struct bio_set bs;
 
 	/*
 	 * freeze/thaw support require holding onto a super block
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 44ff473dab3e..eaf6b279accb 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -143,14 +143,14 @@ struct crypt_config {
 	 * pool for per bio private data, crypto requests,
 	 * encryption requeusts/buffer pages and integrity tags
 	 */
-	mempool_t *req_pool;
-	mempool_t *page_pool;
-	mempool_t *tag_pool;
+	mempool_t req_pool;
+	mempool_t page_pool;
+	mempool_t tag_pool;
 	unsigned tag_pool_max_sectors;
 
 	struct percpu_counter n_allocated_pages;
 
-	struct bio_set *bs;
+	struct bio_set bs;
 	struct mutex bio_alloc_lock;
 
 	struct workqueue_struct *io_queue;
@@ -1245,7 +1245,7 @@ static void crypt_alloc_req_skcipher(struct crypt_config *cc,
 	unsigned key_index = ctx->cc_sector & (cc->tfms_count - 1);
 
 	if (!ctx->r.req)
-		ctx->r.req = mempool_alloc(cc->req_pool, GFP_NOIO);
+		ctx->r.req = mempool_alloc(&cc->req_pool, GFP_NOIO);
 
 	skcipher_request_set_tfm(ctx->r.req, cc->cipher_tfm.tfms[key_index]);
 
@@ -1262,7 +1262,7 @@ static void crypt_alloc_req_aead(struct crypt_config *cc,
 				 struct convert_context *ctx)
 {
 	if (!ctx->r.req_aead)
-		ctx->r.req_aead = mempool_alloc(cc->req_pool, GFP_NOIO);
+		ctx->r.req_aead = mempool_alloc(&cc->req_pool, GFP_NOIO);
 
 	aead_request_set_tfm(ctx->r.req_aead, cc->cipher_tfm.tfms_aead[0]);
 
@@ -1290,7 +1290,7 @@ static void crypt_free_req_skcipher(struct crypt_config *cc,
 	struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
 
 	if ((struct skcipher_request *)(io + 1) != req)
-		mempool_free(req, cc->req_pool);
+		mempool_free(req, &cc->req_pool);
 }
 
 static void crypt_free_req_aead(struct crypt_config *cc,
@@ -1299,7 +1299,7 @@ static void crypt_free_req_aead(struct crypt_config *cc,
 	struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
 
 	if ((struct aead_request *)(io + 1) != req)
-		mempool_free(req, cc->req_pool);
+		mempool_free(req, &cc->req_pool);
 }
 
 static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_bio)
@@ -1409,7 +1409,7 @@ static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size)
 	if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
 		mutex_lock(&cc->bio_alloc_lock);
 
-	clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs);
+	clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, &cc->bs);
 	if (!clone)
 		goto out;
 
@@ -1418,7 +1418,7 @@ static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size)
 	remaining_size = size;
 
 	for (i = 0; i < nr_iovecs; i++) {
-		page = mempool_alloc(cc->page_pool, gfp_mask);
+		page = mempool_alloc(&cc->page_pool, gfp_mask);
 		if (!page) {
 			crypt_free_buffer_pages(cc, clone);
 			bio_put(clone);
@@ -1453,7 +1453,7 @@ static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
 
 	bio_for_each_segment_all(bv, clone, i) {
 		BUG_ON(!bv->bv_page);
-		mempool_free(bv->bv_page, cc->page_pool);
+		mempool_free(bv->bv_page, &cc->page_pool);
 	}
 }
 
@@ -1492,7 +1492,7 @@ static void crypt_dec_pending(struct dm_crypt_io *io)
 		crypt_free_req(cc, io->ctx.r.req, base_bio);
 
 	if (unlikely(io->integrity_metadata_from_pool))
-		mempool_free(io->integrity_metadata, io->cc->tag_pool);
+		mempool_free(io->integrity_metadata, &io->cc->tag_pool);
 	else
 		kfree(io->integrity_metadata);
 
@@ -1565,7 +1565,7 @@ static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
 	 * biovecs we don't need to worry about the block layer
 	 * modifying the biovec array; so leverage bio_clone_fast().
 	 */
-	clone = bio_clone_fast(io->base_bio, gfp, cc->bs);
+	clone = bio_clone_fast(io->base_bio, gfp, &cc->bs);
 	if (!clone)
 		return 1;
 
@@ -2219,17 +2219,16 @@ static void crypt_dtr(struct dm_target *ti)
 
 	crypt_free_tfms(cc);
 
-	if (cc->bs)
-		bioset_free(cc->bs);
+	bioset_exit(&cc->bs);
 
-	mempool_destroy(cc->page_pool);
-	mempool_destroy(cc->req_pool);
-	mempool_destroy(cc->tag_pool);
-
-	if (cc->page_pool)
+	if (mempool_initialized(&cc->page_pool))
 		WARN_ON(percpu_counter_sum(&cc->n_allocated_pages) != 0);
 	percpu_counter_destroy(&cc->n_allocated_pages);
 
+	mempool_exit(&cc->page_pool);
+	mempool_exit(&cc->req_pool);
+	mempool_exit(&cc->tag_pool);
+
 	if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
 		cc->iv_gen_ops->dtr(cc);
 
@@ -2743,8 +2742,6 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		iv_size_padding = align_mask;
 	}
 
-	ret = -ENOMEM;
-
 	/*  ...| IV + padding | original IV | original sec. number | bio tag offset | */
 	additional_req_size = sizeof(struct dm_crypt_request) +
 		iv_size_padding + cc->iv_size +
@@ -2752,8 +2749,8 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		sizeof(uint64_t) +
 		sizeof(unsigned int);
 
-	cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + additional_req_size);
-	if (!cc->req_pool) {
+	ret = mempool_init_kmalloc_pool(&cc->req_pool, MIN_IOS, cc->dmreq_start + additional_req_size);
+	if (ret) {
 		ti->error = "Cannot allocate crypt request mempool";
 		goto bad;
 	}
@@ -2762,14 +2759,14 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size,
 		      ARCH_KMALLOC_MINALIGN);
 
-	cc->page_pool = mempool_create(BIO_MAX_PAGES, crypt_page_alloc, crypt_page_free, cc);
-	if (!cc->page_pool) {
+	ret = mempool_init(&cc->page_pool, BIO_MAX_PAGES, crypt_page_alloc, crypt_page_free, cc);
+	if (ret) {
 		ti->error = "Cannot allocate page mempool";
 		goto bad;
 	}
 
-	cc->bs = bioset_create(MIN_IOS, 0, BIOSET_NEED_BVECS);
-	if (!cc->bs) {
+	ret = bioset_init(&cc->bs, MIN_IOS, 0, BIOSET_NEED_BVECS);
+	if (ret) {
 		ti->error = "Cannot allocate crypt bioset";
 		goto bad;
 	}
@@ -2806,11 +2803,10 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		if (!cc->tag_pool_max_sectors)
 			cc->tag_pool_max_sectors = 1;
 
-		cc->tag_pool = mempool_create_kmalloc_pool(MIN_IOS,
+		ret = mempool_init_kmalloc_pool(&cc->tag_pool, MIN_IOS,
 			cc->tag_pool_max_sectors * cc->on_disk_tag_size);
-		if (!cc->tag_pool) {
+		if (ret) {
 			ti->error = "Cannot allocate integrity tags mempool";
-			ret = -ENOMEM;
 			goto bad;
 		}
 
@@ -2903,7 +2899,7 @@ static int crypt_map(struct dm_target *ti, struct bio *bio)
 				GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN)))) {
 			if (bio_sectors(bio) > cc->tag_pool_max_sectors)
 				dm_accept_partial_bio(bio, cc->tag_pool_max_sectors);
-			io->integrity_metadata = mempool_alloc(cc->tag_pool, GFP_NOIO);
+			io->integrity_metadata = mempool_alloc(&cc->tag_pool, GFP_NOIO);
 			io->integrity_metadata_from_pool = true;
 		}
 	}
diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
index 514fb4aec5d1..fc68c7aaef8e 100644
--- a/drivers/md/dm-integrity.c
+++ b/drivers/md/dm-integrity.c
@@ -142,7 +142,7 @@ struct dm_integrity_c {
 	unsigned tag_size;
 	__s8 log2_tag_size;
 	sector_t start;
-	mempool_t *journal_io_mempool;
+	mempool_t journal_io_mempool;
 	struct dm_io_client *io;
 	struct dm_bufio_client *bufio;
 	struct workqueue_struct *metadata_wq;
@@ -1817,7 +1817,7 @@ static void complete_copy_from_journal(unsigned long error, void *context)
 	struct journal_completion *comp = io->comp;
 	struct dm_integrity_c *ic = comp->ic;
 	remove_range(ic, &io->range);
-	mempool_free(io, ic->journal_io_mempool);
+	mempool_free(io, &ic->journal_io_mempool);
 	if (unlikely(error != 0))
 		dm_integrity_io_error(ic, "copying from journal", -EIO);
 	complete_journal_op(comp);
@@ -1886,7 +1886,7 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start,
 			}
 			next_loop = k - 1;
 
-			io = mempool_alloc(ic->journal_io_mempool, GFP_NOIO);
+			io = mempool_alloc(&ic->journal_io_mempool, GFP_NOIO);
 			io->comp = &comp;
 			io->range.logical_sector = sec;
 			io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block;
@@ -1918,7 +1918,7 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start,
 				if (j == k) {
 					remove_range_unlocked(ic, &io->range);
 					spin_unlock_irq(&ic->endio_wait.lock);
-					mempool_free(io, ic->journal_io_mempool);
+					mempool_free(io, &ic->journal_io_mempool);
 					goto skip_io;
 				}
 				for (l = j; l < k; l++) {
@@ -2980,9 +2980,8 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv)
 		goto bad;
 	}
 
-	ic->journal_io_mempool = mempool_create_slab_pool(JOURNAL_IO_MEMPOOL, journal_io_cache);
-	if (!ic->journal_io_mempool) {
-		r = -ENOMEM;
+	r = mempool_init_slab_pool(&ic->journal_io_mempool, JOURNAL_IO_MEMPOOL, journal_io_cache);
+	if (r) {
 		ti->error = "Cannot allocate mempool";
 		goto bad;
 	}
@@ -3196,7 +3195,7 @@ static void dm_integrity_dtr(struct dm_target *ti)
 		destroy_workqueue(ic->writer_wq);
 	if (ic->bufio)
 		dm_bufio_client_destroy(ic->bufio);
-	mempool_destroy(ic->journal_io_mempool);
+	mempool_exit(&ic->journal_io_mempool);
 	if (ic->io)
 		dm_io_client_destroy(ic->io);
 	if (ic->dev)
diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
index a8d914d5abbe..53c6ed0eaa1f 100644
--- a/drivers/md/dm-io.c
+++ b/drivers/md/dm-io.c
@@ -22,8 +22,8 @@
 #define DM_IO_MAX_REGIONS	BITS_PER_LONG
 
 struct dm_io_client {
-	mempool_t *pool;
-	struct bio_set *bios;
+	mempool_t pool;
+	struct bio_set bios;
 };
 
 /*
@@ -49,32 +49,33 @@ struct dm_io_client *dm_io_client_create(void)
 {
 	struct dm_io_client *client;
 	unsigned min_ios = dm_get_reserved_bio_based_ios();
+	int ret;
 
 	client = kmalloc(sizeof(*client), GFP_KERNEL);
 	if (!client)
 		return ERR_PTR(-ENOMEM);
 
-	client->pool = mempool_create_slab_pool(min_ios, _dm_io_cache);
-	if (!client->pool)
+	ret = mempool_init_slab_pool(&client->pool, min_ios, _dm_io_cache);
+	if (ret)
 		goto bad;
 
-	client->bios = bioset_create(min_ios, 0, BIOSET_NEED_BVECS);
-	if (!client->bios)
+	ret = bioset_init(&client->bios, min_ios, 0, BIOSET_NEED_BVECS);
+	if (ret)
 		goto bad;
 
 	return client;
 
    bad:
-	mempool_destroy(client->pool);
+	mempool_exit(&client->pool);
 	kfree(client);
-	return ERR_PTR(-ENOMEM);
+	return ERR_PTR(ret);
 }
 EXPORT_SYMBOL(dm_io_client_create);
 
 void dm_io_client_destroy(struct dm_io_client *client)
 {
-	mempool_destroy(client->pool);
-	bioset_free(client->bios);
+	mempool_exit(&client->pool);
+	bioset_exit(&client->bios);
 	kfree(client);
 }
 EXPORT_SYMBOL(dm_io_client_destroy);
@@ -120,7 +121,7 @@ static void complete_io(struct io *io)
 		invalidate_kernel_vmap_range(io->vma_invalidate_address,
 					     io->vma_invalidate_size);
 
-	mempool_free(io, io->client->pool);
+	mempool_free(io, &io->client->pool);
 	fn(error_bits, context);
 }
 
@@ -344,7 +345,7 @@ static void do_region(int op, int op_flags, unsigned region,
 					  dm_sector_div_up(remaining, (PAGE_SIZE >> SECTOR_SHIFT)));
 		}
 
-		bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios);
+		bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, &io->client->bios);
 		bio->bi_iter.bi_sector = where->sector + (where->count - remaining);
 		bio_set_dev(bio, where->bdev);
 		bio->bi_end_io = endio;
@@ -442,7 +443,7 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions,
 
 	init_completion(&sio.wait);
 
-	io = mempool_alloc(client->pool, GFP_NOIO);
+	io = mempool_alloc(&client->pool, GFP_NOIO);
 	io->error_bits = 0;
 	atomic_set(&io->count, 1); /* see dispatch_io() */
 	io->client = client;
@@ -474,7 +475,7 @@ static int async_io(struct dm_io_client *client, unsigned int num_regions,
 		return -EIO;
 	}
 
-	io = mempool_alloc(client->pool, GFP_NOIO);
+	io = mempool_alloc(&client->pool, GFP_NOIO);
 	io->error_bits = 0;
 	atomic_set(&io->count, 1); /* see dispatch_io() */
 	io->client = client;
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index 5acf77de5945..b810ea77e6b1 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -1344,7 +1344,8 @@ static int table_load(struct file *filp, struct dm_ioctl *param, size_t param_si
 			goto err_unlock_md_type;
 		}
 	} else if (!is_valid_type(dm_get_md_type(md), dm_table_get_type(t))) {
-		DMWARN("can't change device type after initial table load.");
+		DMWARN("can't change device type (old=%u vs new=%u) after initial table load.",
+		       dm_get_md_type(md), dm_table_get_type(t));
 		r = -EINVAL;
 		goto err_unlock_md_type;
 	}
diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c
index e6e7c686646d..c89a675a2aac 100644
--- a/drivers/md/dm-kcopyd.c
+++ b/drivers/md/dm-kcopyd.c
@@ -47,7 +47,7 @@ struct dm_kcopyd_client {
 	wait_queue_head_t destroyq;
 	atomic_t nr_jobs;
 
-	mempool_t *job_pool;
+	mempool_t job_pool;
 
 	struct workqueue_struct *kcopyd_wq;
 	struct work_struct kcopyd_work;
@@ -479,7 +479,7 @@ static int run_complete_job(struct kcopyd_job *job)
 	 */
 	if (job->master_job == job) {
 		mutex_destroy(&job->lock);
-		mempool_free(job, kc->job_pool);
+		mempool_free(job, &kc->job_pool);
 	}
 	fn(read_err, write_err, context);
 
@@ -751,7 +751,7 @@ int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
 	 * Allocate an array of jobs consisting of one master job
 	 * followed by SPLIT_COUNT sub jobs.
 	 */
-	job = mempool_alloc(kc->job_pool, GFP_NOIO);
+	job = mempool_alloc(&kc->job_pool, GFP_NOIO);
 	mutex_init(&job->lock);
 
 	/*
@@ -835,7 +835,7 @@ void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc,
 {
 	struct kcopyd_job *job;
 
-	job = mempool_alloc(kc->job_pool, GFP_NOIO);
+	job = mempool_alloc(&kc->job_pool, GFP_NOIO);
 
 	memset(job, 0, sizeof(struct kcopyd_job));
 	job->kc = kc;
@@ -879,7 +879,7 @@ int kcopyd_cancel(struct kcopyd_job *job, int block)
  *---------------------------------------------------------------*/
 struct dm_kcopyd_client *dm_kcopyd_client_create(struct dm_kcopyd_throttle *throttle)
 {
-	int r = -ENOMEM;
+	int r;
 	struct dm_kcopyd_client *kc;
 
 	kc = kmalloc(sizeof(*kc), GFP_KERNEL);
@@ -892,14 +892,16 @@ struct dm_kcopyd_client *dm_kcopyd_client_create(struct dm_kcopyd_throttle *thro
 	INIT_LIST_HEAD(&kc->pages_jobs);
 	kc->throttle = throttle;
 
-	kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache);
-	if (!kc->job_pool)
+	r = mempool_init_slab_pool(&kc->job_pool, MIN_JOBS, _job_cache);
+	if (r)
 		goto bad_slab;
 
 	INIT_WORK(&kc->kcopyd_work, do_work);
 	kc->kcopyd_wq = alloc_workqueue("kcopyd", WQ_MEM_RECLAIM, 0);
-	if (!kc->kcopyd_wq)
+	if (!kc->kcopyd_wq) {
+		r = -ENOMEM;
 		goto bad_workqueue;
+	}
 
 	kc->pages = NULL;
 	kc->nr_reserved_pages = kc->nr_free_pages = 0;
@@ -923,7 +925,7 @@ struct dm_kcopyd_client *dm_kcopyd_client_create(struct dm_kcopyd_throttle *thro
 bad_client_pages:
 	destroy_workqueue(kc->kcopyd_wq);
 bad_workqueue:
-	mempool_destroy(kc->job_pool);
+	mempool_exit(&kc->job_pool);
 bad_slab:
 	kfree(kc);
 
@@ -942,7 +944,7 @@ void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc)
 	destroy_workqueue(kc->kcopyd_wq);
 	dm_io_client_destroy(kc->io_client);
 	client_free_pages(kc);
-	mempool_destroy(kc->job_pool);
+	mempool_exit(&kc->job_pool);
 	kfree(kc);
 }
 EXPORT_SYMBOL(dm_kcopyd_client_destroy);
diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c
index 775c06d953b7..d10964d41fd7 100644
--- a/drivers/md/dm-linear.c
+++ b/drivers/md/dm-linear.c
@@ -185,9 +185,24 @@ static size_t linear_dax_copy_from_iter(struct dm_target *ti, pgoff_t pgoff,
 	return dax_copy_from_iter(dax_dev, pgoff, addr, bytes, i);
 }
 
+static size_t linear_dax_copy_to_iter(struct dm_target *ti, pgoff_t pgoff,
+		void *addr, size_t bytes, struct iov_iter *i)
+{
+	struct linear_c *lc = ti->private;
+	struct block_device *bdev = lc->dev->bdev;
+	struct dax_device *dax_dev = lc->dev->dax_dev;
+	sector_t dev_sector, sector = pgoff * PAGE_SECTORS;
+
+	dev_sector = linear_map_sector(ti, sector);
+	if (bdev_dax_pgoff(bdev, dev_sector, ALIGN(bytes, PAGE_SIZE), &pgoff))
+		return 0;
+	return dax_copy_to_iter(dax_dev, pgoff, addr, bytes, i);
+}
+
 #else
 #define linear_dax_direct_access NULL
 #define linear_dax_copy_from_iter NULL
+#define linear_dax_copy_to_iter NULL
 #endif
 
 static struct target_type linear_target = {
@@ -204,6 +219,7 @@ static struct target_type linear_target = {
 	.iterate_devices = linear_iterate_devices,
 	.direct_access = linear_dax_direct_access,
 	.dax_copy_from_iter = linear_dax_copy_from_iter,
+	.dax_copy_to_iter = linear_dax_copy_to_iter,
 };
 
 int __init dm_linear_init(void)
diff --git a/drivers/md/dm-log-userspace-base.c b/drivers/md/dm-log-userspace-base.c
index 53b7b06d0aa8..52090bee17c2 100644
--- a/drivers/md/dm-log-userspace-base.c
+++ b/drivers/md/dm-log-userspace-base.c
@@ -76,7 +76,7 @@ struct log_c {
 	 */
 	uint32_t integrated_flush;
 
-	mempool_t *flush_entry_pool;
+	mempool_t flush_entry_pool;
 };
 
 static struct kmem_cache *_flush_entry_cache;
@@ -249,11 +249,10 @@ static int userspace_ctr(struct dm_dirty_log *log, struct dm_target *ti,
 		goto out;
 	}
 
-	lc->flush_entry_pool = mempool_create_slab_pool(FLUSH_ENTRY_POOL_SIZE,
-							_flush_entry_cache);
-	if (!lc->flush_entry_pool) {
+	r = mempool_init_slab_pool(&lc->flush_entry_pool, FLUSH_ENTRY_POOL_SIZE,
+				   _flush_entry_cache);
+	if (r) {
 		DMERR("Failed to create flush_entry_pool");
-		r = -ENOMEM;
 		goto out;
 	}
 
@@ -313,7 +312,7 @@ static int userspace_ctr(struct dm_dirty_log *log, struct dm_target *ti,
 out:
 	kfree(devices_rdata);
 	if (r) {
-		mempool_destroy(lc->flush_entry_pool);
+		mempool_exit(&lc->flush_entry_pool);
 		kfree(lc);
 		kfree(ctr_str);
 	} else {
@@ -342,7 +341,7 @@ static void userspace_dtr(struct dm_dirty_log *log)
 	if (lc->log_dev)
 		dm_put_device(lc->ti, lc->log_dev);
 
-	mempool_destroy(lc->flush_entry_pool);
+	mempool_exit(&lc->flush_entry_pool);
 
 	kfree(lc->usr_argv_str);
 	kfree(lc);
@@ -570,7 +569,7 @@ static int userspace_flush(struct dm_dirty_log *log)
 	int mark_list_is_empty;
 	int clear_list_is_empty;
 	struct dm_dirty_log_flush_entry *fe, *tmp_fe;
-	mempool_t *flush_entry_pool = lc->flush_entry_pool;
+	mempool_t *flush_entry_pool = &lc->flush_entry_pool;
 
 	spin_lock_irqsave(&lc->flush_lock, flags);
 	list_splice_init(&lc->mark_list, &mark_list);
@@ -653,7 +652,7 @@ static void userspace_mark_region(struct dm_dirty_log *log, region_t region)
 	struct dm_dirty_log_flush_entry *fe;
 
 	/* Wait for an allocation, but _never_ fail */
-	fe = mempool_alloc(lc->flush_entry_pool, GFP_NOIO);
+	fe = mempool_alloc(&lc->flush_entry_pool, GFP_NOIO);
 	BUG_ON(!fe);
 
 	spin_lock_irqsave(&lc->flush_lock, flags);
@@ -687,7 +686,7 @@ static void userspace_clear_region(struct dm_dirty_log *log, region_t region)
 	 * to cause the region to be resync'ed when the
 	 * device is activated next time.
 	 */
-	fe = mempool_alloc(lc->flush_entry_pool, GFP_ATOMIC);
+	fe = mempool_alloc(&lc->flush_entry_pool, GFP_ATOMIC);
 	if (!fe) {
 		DMERR("Failed to allocate memory to clear region.");
 		return;
diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c
index c90c7c08a77f..9ea2b0291f20 100644
--- a/drivers/md/dm-log-writes.c
+++ b/drivers/md/dm-log-writes.c
@@ -962,9 +962,23 @@ static size_t log_writes_dax_copy_from_iter(struct dm_target *ti,
 dax_copy:
 	return dax_copy_from_iter(lc->dev->dax_dev, pgoff, addr, bytes, i);
 }
+
+static size_t log_writes_dax_copy_to_iter(struct dm_target *ti,
+					  pgoff_t pgoff, void *addr, size_t bytes,
+					  struct iov_iter *i)
+{
+	struct log_writes_c *lc = ti->private;
+	sector_t sector = pgoff * PAGE_SECTORS;
+
+	if (bdev_dax_pgoff(lc->dev->bdev, sector, ALIGN(bytes, PAGE_SIZE), &pgoff))
+		return 0;
+	return dax_copy_to_iter(lc->dev->dax_dev, pgoff, addr, bytes, i);
+}
+
 #else
 #define log_writes_dax_direct_access NULL
 #define log_writes_dax_copy_from_iter NULL
+#define log_writes_dax_copy_to_iter NULL
 #endif
 
 static struct target_type log_writes_target = {
@@ -982,6 +996,7 @@ static struct target_type log_writes_target = {
 	.io_hints = log_writes_io_hints,
 	.direct_access = log_writes_dax_direct_access,
 	.dax_copy_from_iter = log_writes_dax_copy_from_iter,
+	.dax_copy_to_iter = log_writes_dax_copy_to_iter,
 };
 
 static int __init dm_log_writes_init(void)
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index 203a0419d2b0..d94ba6f72ff5 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -520,7 +520,8 @@ static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
 
 	bdev = pgpath->path.dev->bdev;
 	q = bdev_get_queue(bdev);
-	clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE, GFP_ATOMIC);
+	clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE,
+			BLK_MQ_REQ_NOWAIT);
 	if (IS_ERR(clone)) {
 		/* EBUSY, ENODEV or EWOULDBLOCK: requeue */
 		if (blk_queue_dying(q)) {
diff --git a/drivers/md/dm-region-hash.c b/drivers/md/dm-region-hash.c
index 85c32b22a420..43149eb49375 100644
--- a/drivers/md/dm-region-hash.c
+++ b/drivers/md/dm-region-hash.c
@@ -63,7 +63,7 @@ struct dm_region_hash {
 
 	/* hash table */
 	rwlock_t hash_lock;
-	mempool_t *region_pool;
+	mempool_t region_pool;
 	unsigned mask;
 	unsigned nr_buckets;
 	unsigned prime;
@@ -169,6 +169,7 @@ struct dm_region_hash *dm_region_hash_create(
 	struct dm_region_hash *rh;
 	unsigned nr_buckets, max_buckets;
 	size_t i;
+	int ret;
 
 	/*
 	 * Calculate a suitable number of buckets for our hash
@@ -220,9 +221,9 @@ struct dm_region_hash *dm_region_hash_create(
 	INIT_LIST_HEAD(&rh->failed_recovered_regions);
 	rh->flush_failure = 0;
 
-	rh->region_pool = mempool_create_kmalloc_pool(MIN_REGIONS,
-						      sizeof(struct dm_region));
-	if (!rh->region_pool) {
+	ret = mempool_init_kmalloc_pool(&rh->region_pool, MIN_REGIONS,
+					sizeof(struct dm_region));
+	if (ret) {
 		vfree(rh->buckets);
 		kfree(rh);
 		rh = ERR_PTR(-ENOMEM);
@@ -242,14 +243,14 @@ void dm_region_hash_destroy(struct dm_region_hash *rh)
 		list_for_each_entry_safe(reg, nreg, rh->buckets + h,
 					 hash_list) {
 			BUG_ON(atomic_read(&reg->pending));
-			mempool_free(reg, rh->region_pool);
+			mempool_free(reg, &rh->region_pool);
 		}
 	}
 
 	if (rh->log)
 		dm_dirty_log_destroy(rh->log);
 
-	mempool_destroy(rh->region_pool);
+	mempool_exit(&rh->region_pool);
 	vfree(rh->buckets);
 	kfree(rh);
 }
@@ -287,7 +288,7 @@ static struct dm_region *__rh_alloc(struct dm_region_hash *rh, region_t region)
 {
 	struct dm_region *reg, *nreg;
 
-	nreg = mempool_alloc(rh->region_pool, GFP_ATOMIC);
+	nreg = mempool_alloc(&rh->region_pool, GFP_ATOMIC);
 	if (unlikely(!nreg))
 		nreg = kmalloc(sizeof(*nreg), GFP_NOIO | __GFP_NOFAIL);
 
@@ -303,7 +304,7 @@ static struct dm_region *__rh_alloc(struct dm_region_hash *rh, region_t region)
 	reg = __rh_lookup(rh, region);
 	if (reg)
 		/* We lost the race. */
-		mempool_free(nreg, rh->region_pool);
+		mempool_free(nreg, &rh->region_pool);
 	else {
 		__rh_insert(rh, nreg);
 		if (nreg->state == DM_RH_CLEAN) {
@@ -481,17 +482,17 @@ void dm_rh_update_states(struct dm_region_hash *rh, int errors_handled)
 	list_for_each_entry_safe(reg, next, &recovered, list) {
 		rh->log->type->clear_region(rh->log, reg->key);
 		complete_resync_work(reg, 1);
-		mempool_free(reg, rh->region_pool);
+		mempool_free(reg, &rh->region_pool);
 	}
 
 	list_for_each_entry_safe(reg, next, &failed_recovered, list) {
 		complete_resync_work(reg, errors_handled ? 0 : 1);
-		mempool_free(reg, rh->region_pool);
+		mempool_free(reg, &rh->region_pool);
 	}
 
 	list_for_each_entry_safe(reg, next, &clean, list) {
 		rh->log->type->clear_region(rh->log, reg->key);
-		mempool_free(reg, rh->region_pool);
+		mempool_free(reg, &rh->region_pool);
 	}
 
 	rh->log->type->flush(rh->log);
diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c
index bf0b840645cc..6e547b8dd298 100644
--- a/drivers/md/dm-rq.c
+++ b/drivers/md/dm-rq.c
@@ -406,7 +406,7 @@ static blk_status_t dm_dispatch_clone_request(struct request *clone, struct requ
 	if (blk_queue_io_stat(clone->q))
 		clone->rq_flags |= RQF_IO_STAT;
 
-	clone->start_time = jiffies;
+	clone->start_time_ns = ktime_get_ns();
 	r = blk_insert_cloned_request(clone->q, clone);
 	if (r != BLK_STS_OK && r != BLK_STS_RESOURCE && r != BLK_STS_DEV_RESOURCE)
 		/* must complete clone in terms of original request */
@@ -433,7 +433,7 @@ static int setup_clone(struct request *clone, struct request *rq,
 {
 	int r;
 
-	r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask,
+	r = blk_rq_prep_clone(clone, rq, &tio->md->bs, gfp_mask,
 			      dm_rq_bio_constructor, tio);
 	if (r)
 		return r;
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index 216035be5661..b11ddc55f297 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -87,7 +87,7 @@ struct dm_snapshot {
 	 */
 	struct list_head out_of_order_list;
 
-	mempool_t *pending_pool;
+	mempool_t pending_pool;
 
 	struct dm_exception_table pending;
 	struct dm_exception_table complete;
@@ -682,7 +682,7 @@ static void free_completed_exception(struct dm_exception *e)
 
 static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s)
 {
-	struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool,
+	struct dm_snap_pending_exception *pe = mempool_alloc(&s->pending_pool,
 							     GFP_NOIO);
 
 	atomic_inc(&s->pending_exceptions_count);
@@ -695,7 +695,7 @@ static void free_pending_exception(struct dm_snap_pending_exception *pe)
 {
 	struct dm_snapshot *s = pe->snap;
 
-	mempool_free(pe, s->pending_pool);
+	mempool_free(pe, &s->pending_pool);
 	smp_mb__before_atomic();
 	atomic_dec(&s->pending_exceptions_count);
 }
@@ -1196,10 +1196,9 @@ static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		goto bad_kcopyd;
 	}
 
-	s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache);
-	if (!s->pending_pool) {
+	r = mempool_init_slab_pool(&s->pending_pool, MIN_IOS, pending_cache);
+	if (r) {
 		ti->error = "Could not allocate mempool for pending exceptions";
-		r = -ENOMEM;
 		goto bad_pending_pool;
 	}
 
@@ -1259,7 +1258,7 @@ static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	unregister_snapshot(s);
 
 bad_load_and_register:
-	mempool_destroy(s->pending_pool);
+	mempool_exit(&s->pending_pool);
 
 bad_pending_pool:
 	dm_kcopyd_client_destroy(s->kcopyd_client);
@@ -1355,7 +1354,7 @@ static void snapshot_dtr(struct dm_target *ti)
 	while (atomic_read(&s->pending_exceptions_count))
 		msleep(1);
 	/*
-	 * Ensure instructions in mempool_destroy aren't reordered
+	 * Ensure instructions in mempool_exit aren't reordered
 	 * before atomic_read.
 	 */
 	smp_mb();
@@ -1367,7 +1366,7 @@ static void snapshot_dtr(struct dm_target *ti)
 
 	__free_exceptions(s);
 
-	mempool_destroy(s->pending_pool);
+	mempool_exit(&s->pending_pool);
 
 	dm_exception_store_destroy(s->store);
 
diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c
index fe7fb9b1aec3..8547d7594338 100644
--- a/drivers/md/dm-stripe.c
+++ b/drivers/md/dm-stripe.c
@@ -354,9 +354,29 @@ static size_t stripe_dax_copy_from_iter(struct dm_target *ti, pgoff_t pgoff,
 	return dax_copy_from_iter(dax_dev, pgoff, addr, bytes, i);
 }
 
+static size_t stripe_dax_copy_to_iter(struct dm_target *ti, pgoff_t pgoff,
+		void *addr, size_t bytes, struct iov_iter *i)
+{
+	sector_t dev_sector, sector = pgoff * PAGE_SECTORS;
+	struct stripe_c *sc = ti->private;
+	struct dax_device *dax_dev;
+	struct block_device *bdev;
+	uint32_t stripe;
+
+	stripe_map_sector(sc, sector, &stripe, &dev_sector);
+	dev_sector += sc->stripe[stripe].physical_start;
+	dax_dev = sc->stripe[stripe].dev->dax_dev;
+	bdev = sc->stripe[stripe].dev->bdev;
+
+	if (bdev_dax_pgoff(bdev, dev_sector, ALIGN(bytes, PAGE_SIZE), &pgoff))
+		return 0;
+	return dax_copy_to_iter(dax_dev, pgoff, addr, bytes, i);
+}
+
 #else
 #define stripe_dax_direct_access NULL
 #define stripe_dax_copy_from_iter NULL
+#define stripe_dax_copy_to_iter NULL
 #endif
 
 /*
@@ -478,6 +498,7 @@ static struct target_type stripe_target = {
 	.io_hints = stripe_io_hints,
 	.direct_access = stripe_dax_direct_access,
 	.dax_copy_from_iter = stripe_dax_copy_from_iter,
+	.dax_copy_to_iter = stripe_dax_copy_to_iter,
 };
 
 int __init dm_stripe_init(void)
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index b11107497d2e..6c923824ec91 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -260,7 +260,7 @@ struct pool {
 	struct dm_deferred_set *all_io_ds;
 
 	struct dm_thin_new_mapping *next_mapping;
-	mempool_t *mapping_pool;
+	mempool_t mapping_pool;
 
 	process_bio_fn process_bio;
 	process_bio_fn process_discard;
@@ -917,7 +917,7 @@ static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
 {
 	cell_error(m->tc->pool, m->cell);
 	list_del(&m->list);
-	mempool_free(m, m->tc->pool->mapping_pool);
+	mempool_free(m, &m->tc->pool->mapping_pool);
 }
 
 static void process_prepared_mapping(struct dm_thin_new_mapping *m)
@@ -961,7 +961,7 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m)
 
 out:
 	list_del(&m->list);
-	mempool_free(m, pool->mapping_pool);
+	mempool_free(m, &pool->mapping_pool);
 }
 
 /*----------------------------------------------------------------*/
@@ -971,7 +971,7 @@ static void free_discard_mapping(struct dm_thin_new_mapping *m)
 	struct thin_c *tc = m->tc;
 	if (m->cell)
 		cell_defer_no_holder(tc, m->cell);
-	mempool_free(m, tc->pool->mapping_pool);
+	mempool_free(m, &tc->pool->mapping_pool);
 }
 
 static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)
@@ -999,7 +999,7 @@ static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m)
 		bio_endio(m->bio);
 
 	cell_defer_no_holder(tc, m->cell);
-	mempool_free(m, tc->pool->mapping_pool);
+	mempool_free(m, &tc->pool->mapping_pool);
 }
 
 /*----------------------------------------------------------------*/
@@ -1092,7 +1092,7 @@ static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m)
 		metadata_operation_failed(pool, "dm_thin_remove_range", r);
 		bio_io_error(m->bio);
 		cell_defer_no_holder(tc, m->cell);
-		mempool_free(m, pool->mapping_pool);
+		mempool_free(m, &pool->mapping_pool);
 		return;
 	}
 
@@ -1105,7 +1105,7 @@ static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m)
 		metadata_operation_failed(pool, "dm_pool_inc_data_range", r);
 		bio_io_error(m->bio);
 		cell_defer_no_holder(tc, m->cell);
-		mempool_free(m, pool->mapping_pool);
+		mempool_free(m, &pool->mapping_pool);
 		return;
 	}
 
@@ -1150,7 +1150,7 @@ static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping *m)
 		bio_endio(m->bio);
 
 	cell_defer_no_holder(tc, m->cell);
-	mempool_free(m, pool->mapping_pool);
+	mempool_free(m, &pool->mapping_pool);
 }
 
 static void process_prepared(struct pool *pool, struct list_head *head,
@@ -1196,7 +1196,7 @@ static int ensure_next_mapping(struct pool *pool)
 	if (pool->next_mapping)
 		return 0;
 
-	pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC);
+	pool->next_mapping = mempool_alloc(&pool->mapping_pool, GFP_ATOMIC);
 
 	return pool->next_mapping ? 0 : -ENOMEM;
 }
@@ -2835,8 +2835,8 @@ static void __pool_destroy(struct pool *pool)
 		destroy_workqueue(pool->wq);
 
 	if (pool->next_mapping)
-		mempool_free(pool->next_mapping, pool->mapping_pool);
-	mempool_destroy(pool->mapping_pool);
+		mempool_free(pool->next_mapping, &pool->mapping_pool);
+	mempool_exit(&pool->mapping_pool);
 	dm_deferred_set_destroy(pool->shared_read_ds);
 	dm_deferred_set_destroy(pool->all_io_ds);
 	kfree(pool);
@@ -2931,11 +2931,11 @@ static struct pool *pool_create(struct mapped_device *pool_md,
 	}
 
 	pool->next_mapping = NULL;
-	pool->mapping_pool = mempool_create_slab_pool(MAPPING_POOL_SIZE,
-						      _new_mapping_cache);
-	if (!pool->mapping_pool) {
+	r = mempool_init_slab_pool(&pool->mapping_pool, MAPPING_POOL_SIZE,
+				   _new_mapping_cache);
+	if (r) {
 		*error = "Error creating pool's mapping mempool";
-		err_p = ERR_PTR(-ENOMEM);
+		err_p = ERR_PTR(r);
 		goto bad_mapping_pool;
 	}
 
@@ -2955,7 +2955,7 @@ static struct pool *pool_create(struct mapped_device *pool_md,
 	return pool;
 
 bad_sort_array:
-	mempool_destroy(pool->mapping_pool);
+	mempool_exit(&pool->mapping_pool);
 bad_mapping_pool:
 	dm_deferred_set_destroy(pool->all_io_ds);
 bad_all_io_ds:
diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c
index e13f90832b6b..684af08d0747 100644
--- a/drivers/md/dm-verity-fec.c
+++ b/drivers/md/dm-verity-fec.c
@@ -309,13 +309,13 @@ static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
 	unsigned n;
 
 	if (!fio->rs)
-		fio->rs = mempool_alloc(v->fec->rs_pool, GFP_NOIO);
+		fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO);
 
 	fec_for_each_prealloc_buffer(n) {
 		if (fio->bufs[n])
 			continue;
 
-		fio->bufs[n] = mempool_alloc(v->fec->prealloc_pool, GFP_NOWAIT);
+		fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT);
 		if (unlikely(!fio->bufs[n])) {
 			DMERR("failed to allocate FEC buffer");
 			return -ENOMEM;
@@ -327,7 +327,7 @@ static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
 		if (fio->bufs[n])
 			continue;
 
-		fio->bufs[n] = mempool_alloc(v->fec->extra_pool, GFP_NOWAIT);
+		fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT);
 		/* we can manage with even one buffer if necessary */
 		if (unlikely(!fio->bufs[n]))
 			break;
@@ -335,7 +335,7 @@ static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
 	fio->nbufs = n;
 
 	if (!fio->output)
-		fio->output = mempool_alloc(v->fec->output_pool, GFP_NOIO);
+		fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO);
 
 	return 0;
 }
@@ -493,15 +493,15 @@ void verity_fec_finish_io(struct dm_verity_io *io)
 	if (!verity_fec_is_enabled(io->v))
 		return;
 
-	mempool_free(fio->rs, f->rs_pool);
+	mempool_free(fio->rs, &f->rs_pool);
 
 	fec_for_each_prealloc_buffer(n)
-		mempool_free(fio->bufs[n], f->prealloc_pool);
+		mempool_free(fio->bufs[n], &f->prealloc_pool);
 
 	fec_for_each_extra_buffer(fio, n)
-		mempool_free(fio->bufs[n], f->extra_pool);
+		mempool_free(fio->bufs[n], &f->extra_pool);
 
-	mempool_free(fio->output, f->output_pool);
+	mempool_free(fio->output, &f->output_pool);
 }
 
 /*
@@ -549,9 +549,9 @@ void verity_fec_dtr(struct dm_verity *v)
 	if (!verity_fec_is_enabled(v))
 		goto out;
 
-	mempool_destroy(f->rs_pool);
-	mempool_destroy(f->prealloc_pool);
-	mempool_destroy(f->extra_pool);
+	mempool_exit(&f->rs_pool);
+	mempool_exit(&f->prealloc_pool);
+	mempool_exit(&f->extra_pool);
 	kmem_cache_destroy(f->cache);
 
 	if (f->data_bufio)
@@ -570,7 +570,7 @@ static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data)
 {
 	struct dm_verity *v = (struct dm_verity *)pool_data;
 
-	return init_rs(8, 0x11d, 0, 1, v->fec->roots);
+	return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask);
 }
 
 static void fec_rs_free(void *element, void *pool_data)
@@ -675,6 +675,7 @@ int verity_fec_ctr(struct dm_verity *v)
 	struct dm_verity_fec *f = v->fec;
 	struct dm_target *ti = v->ti;
 	u64 hash_blocks;
+	int ret;
 
 	if (!verity_fec_is_enabled(v)) {
 		verity_fec_dtr(v);
@@ -770,11 +771,11 @@ int verity_fec_ctr(struct dm_verity *v)
 	}
 
 	/* Preallocate an rs_control structure for each worker thread */
-	f->rs_pool = mempool_create(num_online_cpus(), fec_rs_alloc,
-				    fec_rs_free, (void *) v);
-	if (!f->rs_pool) {
+	ret = mempool_init(&f->rs_pool, num_online_cpus(), fec_rs_alloc,
+			   fec_rs_free, (void *) v);
+	if (ret) {
 		ti->error = "Cannot allocate RS pool";
-		return -ENOMEM;
+		return ret;
 	}
 
 	f->cache = kmem_cache_create("dm_verity_fec_buffers",
@@ -786,26 +787,26 @@ int verity_fec_ctr(struct dm_verity *v)
 	}
 
 	/* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */
-	f->prealloc_pool = mempool_create_slab_pool(num_online_cpus() *
-						    DM_VERITY_FEC_BUF_PREALLOC,
-						    f->cache);
-	if (!f->prealloc_pool) {
+	ret = mempool_init_slab_pool(&f->prealloc_pool, num_online_cpus() *
+				     DM_VERITY_FEC_BUF_PREALLOC,
+				     f->cache);
+	if (ret) {
 		ti->error = "Cannot allocate FEC buffer prealloc pool";
-		return -ENOMEM;
+		return ret;
 	}
 
-	f->extra_pool = mempool_create_slab_pool(0, f->cache);
-	if (!f->extra_pool) {
+	ret = mempool_init_slab_pool(&f->extra_pool, 0, f->cache);
+	if (ret) {
 		ti->error = "Cannot allocate FEC buffer extra pool";
-		return -ENOMEM;
+		return ret;
 	}
 
 	/* Preallocate an output buffer for each thread */
-	f->output_pool = mempool_create_kmalloc_pool(num_online_cpus(),
-						     1 << v->data_dev_block_bits);
-	if (!f->output_pool) {
+	ret = mempool_init_kmalloc_pool(&f->output_pool, num_online_cpus(),
+					1 << v->data_dev_block_bits);
+	if (ret) {
 		ti->error = "Cannot allocate FEC output pool";
-		return -ENOMEM;
+		return ret;
 	}
 
 	/* Reserve space for our per-bio data */
diff --git a/drivers/md/dm-verity-fec.h b/drivers/md/dm-verity-fec.h
index bb31ce87a933..6ad803b2b36c 100644
--- a/drivers/md/dm-verity-fec.h
+++ b/drivers/md/dm-verity-fec.h
@@ -46,10 +46,10 @@ struct dm_verity_fec {
 	sector_t hash_blocks;	/* blocks covered after v->hash_start */
 	unsigned char roots;	/* number of parity bytes, M-N of RS(M, N) */
 	unsigned char rsn;	/* N of RS(M, N) */
-	mempool_t *rs_pool;	/* mempool for fio->rs */
-	mempool_t *prealloc_pool;	/* mempool for preallocated buffers */
-	mempool_t *extra_pool;	/* mempool for extra buffers */
-	mempool_t *output_pool;	/* mempool for output */
+	mempool_t rs_pool;	/* mempool for fio->rs */
+	mempool_t prealloc_pool;	/* mempool for preallocated buffers */
+	mempool_t extra_pool;	/* mempool for extra buffers */
+	mempool_t output_pool;	/* mempool for output */
 	struct kmem_cache *cache;	/* cache for buffers */
 };
 
diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c
new file mode 100644
index 000000000000..b95f1f0e199d
--- /dev/null
+++ b/drivers/md/dm-writecache.c
@@ -0,0 +1,2323 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Red Hat. All rights reserved.
+ *
+ * This file is released under the GPL.
+ */
+
+#include <linux/device-mapper.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/vmalloc.h>
+#include <linux/kthread.h>
+#include <linux/swait.h>
+#include <linux/dm-io.h>
+#include <linux/dm-kcopyd.h>
+#include <linux/dax.h>
+#include <linux/pfn_t.h>
+#include <linux/libnvdimm.h>
+
+#define DM_MSG_PREFIX "writecache"
+
+#define HIGH_WATERMARK			50
+#define LOW_WATERMARK			45
+#define MAX_WRITEBACK_JOBS		0
+#define ENDIO_LATENCY			16
+#define WRITEBACK_LATENCY		64
+#define AUTOCOMMIT_BLOCKS_SSD		65536
+#define AUTOCOMMIT_BLOCKS_PMEM		64
+#define AUTOCOMMIT_MSEC			1000
+
+#define BITMAP_GRANULARITY	65536
+#if BITMAP_GRANULARITY < PAGE_SIZE
+#undef BITMAP_GRANULARITY
+#define BITMAP_GRANULARITY	PAGE_SIZE
+#endif
+
+/*
+ * API for optimized flushing of persistent memory:
+ * On X86, non-temporal stores are more efficient than cache flushing.
+ * On ARM64, cache flushing is more efficient.
+ *
+ * This API is a candidate for being elevated out of DM but for now
+ * it serves to cleanly allow optimized flushing without excessive
+ * branching throughout this target's code.
+ */
+#if defined(CONFIG_X86_64)
+
+#define __pmem_assign(dest, src, uniq)				\
+do {								\
+	typeof(dest) uniq = (src);				\
+	memcpy_flushcache(&(dest), &uniq, sizeof(dest));	\
+} while (0)
+
+#define pmem_assign(dest, src)					\
+	__pmem_assign(dest, src, __UNIQUE_ID(pmem_assign))
+
+static void pmem_memcpy(void *dest, void *src, size_t len)
+{
+	memcpy_flushcache(dest, src, len);
+}
+
+static void pmem_flush(void *dest, size_t len)
+{
+}
+
+static void pmem_commit(void)
+{
+	wmb();
+}
+
+#else
+
+#define pmem_assign(dest, src)	WRITE_ONCE(dest, src)
+
+static void pmem_memcpy(void *dest, void *src, size_t len)
+{
+	memcpy(dest, src, len);
+}
+
+static void pmem_flush(void *dest, size_t len)
+{
+	arch_wb_cache_pmem(dest, len);
+}
+
+static void pmem_commit(void)
+{
+}
+
+#endif
+
+#if defined(__HAVE_ARCH_MEMCPY_MCSAFE) && defined(CONFIG_ARCH_HAS_PMEM_API)
+#define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
+#endif
+
+#define MEMORY_SUPERBLOCK_MAGIC		0x23489321
+#define MEMORY_SUPERBLOCK_VERSION	1
+
+struct wc_memory_entry {
+	__le64 original_sector;
+	__le64 seq_count;
+};
+
+struct wc_memory_superblock {
+	union {
+		struct {
+			__le32 magic;
+			__le32 version;
+			__le32 block_size;
+			__le32 pad;
+			__le64 n_blocks;
+			__le64 seq_count;
+		};
+		__le64 padding[8];
+	};
+	struct wc_memory_entry entries[0];
+};
+
+struct wc_entry {
+	struct rb_node rb_node;
+	struct list_head lru;
+	unsigned short wc_list_contiguous;
+	bool write_in_progress
+#if BITS_PER_LONG == 64
+		:1
+#endif
+	;
+	unsigned long index
+#if BITS_PER_LONG == 64
+		:47
+#endif
+	;
+#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
+	uint64_t original_sector;
+	uint64_t seq_count;
+#endif
+};
+
+#define WC_MODE_PMEM(wc)			((wc)->pmem_mode)
+#define WC_MODE_FUA(wc)				((wc)->writeback_fua)
+#define WC_MODE_SORT_FREELIST(wc)		(!WC_MODE_PMEM(wc))
+
+struct dm_writecache {
+	struct mutex lock;
+	struct list_head lru;
+	union {
+		struct list_head freelist;
+		struct {
+			struct rb_root freetree;
+			struct wc_entry *current_free;
+		};
+	};
+	struct rb_root tree;
+
+	size_t freelist_size;
+	size_t writeback_size;
+	size_t freelist_high_watermark;
+	size_t freelist_low_watermark;
+
+	unsigned uncommitted_blocks;
+	unsigned autocommit_blocks;
+	unsigned max_writeback_jobs;
+
+	int error;
+
+	unsigned long autocommit_jiffies;
+	struct timer_list autocommit_timer;
+	struct swait_queue_head freelist_wait;
+
+	atomic_t bio_in_progress[2];
+	struct swait_queue_head bio_in_progress_wait[2];
+
+	struct dm_target *ti;
+	struct dm_dev *dev;
+	struct dm_dev *ssd_dev;
+	void *memory_map;
+	uint64_t memory_map_size;
+	size_t metadata_sectors;
+	size_t n_blocks;
+	uint64_t seq_count;
+	void *block_start;
+	struct wc_entry *entries;
+	unsigned block_size;
+	unsigned char block_size_bits;
+
+	bool pmem_mode:1;
+	bool writeback_fua:1;
+
+	bool overwrote_committed:1;
+	bool memory_vmapped:1;
+
+	bool high_wm_percent_set:1;
+	bool low_wm_percent_set:1;
+	bool max_writeback_jobs_set:1;
+	bool autocommit_blocks_set:1;
+	bool autocommit_time_set:1;
+	bool writeback_fua_set:1;
+	bool flush_on_suspend:1;
+
+	unsigned writeback_all;
+	struct workqueue_struct *writeback_wq;
+	struct work_struct writeback_work;
+	struct work_struct flush_work;
+
+	struct dm_io_client *dm_io;
+
+	struct swait_queue_head endio_thread_wait;
+	struct list_head endio_list;
+	struct task_struct *endio_thread;
+
+	struct task_struct *flush_thread;
+	struct completion flush_completion;
+	struct bio *flush_bio;
+
+	struct bio_set bio_set;
+	mempool_t *copy_pool;
+
+	struct dm_kcopyd_client *dm_kcopyd;
+	unsigned long *dirty_bitmap;
+	unsigned dirty_bitmap_size;
+};
+
+#define WB_LIST_INLINE		16
+
+struct writeback_struct {
+	struct list_head endio_entry;
+	struct dm_writecache *wc;
+	struct wc_entry **wc_list;
+	unsigned wc_list_n;
+	unsigned page_offset;
+	struct page *page;
+	struct wc_entry *wc_list_inline[WB_LIST_INLINE];
+	struct bio bio;
+};
+
+struct copy_struct {
+	struct list_head endio_entry;
+	struct dm_writecache *wc;
+	struct wc_entry *e;
+	unsigned n_entries;
+	int error;
+};
+
+DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(dm_writecache_throttle,
+					    "A percentage of time allocated for data copying");
+
+static void wc_lock(struct dm_writecache *wc)
+{
+	mutex_lock(&wc->lock);
+}
+
+static void wc_unlock(struct dm_writecache *wc)
+{
+	mutex_unlock(&wc->lock);
+}
+
+#if IS_ENABLED(CONFIG_DAX_DRIVER)
+static int persistent_memory_claim(struct dm_writecache *wc)
+{
+	int r;
+	loff_t s;
+	long p, da;
+	pfn_t pfn;
+	int id;
+	struct page **pages;
+
+	wc->memory_vmapped = false;
+
+	if (!wc->ssd_dev->dax_dev) {
+		r = -EOPNOTSUPP;
+		goto err1;
+	}
+	s = wc->memory_map_size;
+	p = s >> PAGE_SHIFT;
+	if (!p) {
+		r = -EINVAL;
+		goto err1;
+	}
+	if (p != s >> PAGE_SHIFT) {
+		r = -EOVERFLOW;
+		goto err1;
+	}
+
+	id = dax_read_lock();
+
+	da = dax_direct_access(wc->ssd_dev->dax_dev, 0, p, &wc->memory_map, &pfn);
+	if (da < 0) {
+		wc->memory_map = NULL;
+		r = da;
+		goto err2;
+	}
+	if (!pfn_t_has_page(pfn)) {
+		wc->memory_map = NULL;
+		r = -EOPNOTSUPP;
+		goto err2;
+	}
+	if (da != p) {
+		long i;
+		wc->memory_map = NULL;
+		pages = kvmalloc(p * sizeof(struct page *), GFP_KERNEL);
+		if (!pages) {
+			r = -ENOMEM;
+			goto err2;
+		}
+		i = 0;
+		do {
+			long daa;
+			void *dummy_addr;
+			daa = dax_direct_access(wc->ssd_dev->dax_dev, i, p - i,
+						&dummy_addr, &pfn);
+			if (daa <= 0) {
+				r = daa ? daa : -EINVAL;
+				goto err3;
+			}
+			if (!pfn_t_has_page(pfn)) {
+				r = -EOPNOTSUPP;
+				goto err3;
+			}
+			while (daa-- && i < p) {
+				pages[i++] = pfn_t_to_page(pfn);
+				pfn.val++;
+			}
+		} while (i < p);
+		wc->memory_map = vmap(pages, p, VM_MAP, PAGE_KERNEL);
+		if (!wc->memory_map) {
+			r = -ENOMEM;
+			goto err3;
+		}
+		kvfree(pages);
+		wc->memory_vmapped = true;
+	}
+
+	dax_read_unlock(id);
+	return 0;
+err3:
+	kvfree(pages);
+err2:
+	dax_read_unlock(id);
+err1:
+	return r;
+}
+#else
+static int persistent_memory_claim(struct dm_writecache *wc)
+{
+	BUG();
+}
+#endif
+
+static void persistent_memory_release(struct dm_writecache *wc)
+{
+	if (wc->memory_vmapped)
+		vunmap(wc->memory_map);
+}
+
+static struct page *persistent_memory_page(void *addr)
+{
+	if (is_vmalloc_addr(addr))
+		return vmalloc_to_page(addr);
+	else
+		return virt_to_page(addr);
+}
+
+static unsigned persistent_memory_page_offset(void *addr)
+{
+	return (unsigned long)addr & (PAGE_SIZE - 1);
+}
+
+static void persistent_memory_flush_cache(void *ptr, size_t size)
+{
+	if (is_vmalloc_addr(ptr))
+		flush_kernel_vmap_range(ptr, size);
+}
+
+static void persistent_memory_invalidate_cache(void *ptr, size_t size)
+{
+	if (is_vmalloc_addr(ptr))
+		invalidate_kernel_vmap_range(ptr, size);
+}
+
+static struct wc_memory_superblock *sb(struct dm_writecache *wc)
+{
+	return wc->memory_map;
+}
+
+static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e)
+{
+	if (is_power_of_2(sizeof(struct wc_entry)) && 0)
+		return &sb(wc)->entries[e - wc->entries];
+	else
+		return &sb(wc)->entries[e->index];
+}
+
+static void *memory_data(struct dm_writecache *wc, struct wc_entry *e)
+{
+	return (char *)wc->block_start + (e->index << wc->block_size_bits);
+}
+
+static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e)
+{
+	return wc->metadata_sectors +
+		((sector_t)e->index << (wc->block_size_bits - SECTOR_SHIFT));
+}
+
+static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e)
+{
+#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
+	return e->original_sector;
+#else
+	return le64_to_cpu(memory_entry(wc, e)->original_sector);
+#endif
+}
+
+static uint64_t read_seq_count(struct dm_writecache *wc, struct wc_entry *e)
+{
+#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
+	return e->seq_count;
+#else
+	return le64_to_cpu(memory_entry(wc, e)->seq_count);
+#endif
+}
+
+static void clear_seq_count(struct dm_writecache *wc, struct wc_entry *e)
+{
+#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
+	e->seq_count = -1;
+#endif
+	pmem_assign(memory_entry(wc, e)->seq_count, cpu_to_le64(-1));
+}
+
+static void write_original_sector_seq_count(struct dm_writecache *wc, struct wc_entry *e,
+					    uint64_t original_sector, uint64_t seq_count)
+{
+	struct wc_memory_entry me;
+#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
+	e->original_sector = original_sector;
+	e->seq_count = seq_count;
+#endif
+	me.original_sector = cpu_to_le64(original_sector);
+	me.seq_count = cpu_to_le64(seq_count);
+	pmem_assign(*memory_entry(wc, e), me);
+}
+
+#define writecache_error(wc, err, msg, arg...)				\
+do {									\
+	if (!cmpxchg(&(wc)->error, 0, err))				\
+		DMERR(msg, ##arg);					\
+	swake_up(&(wc)->freelist_wait);					\
+} while (0)
+
+#define writecache_has_error(wc)	(unlikely(READ_ONCE((wc)->error)))
+
+static void writecache_flush_all_metadata(struct dm_writecache *wc)
+{
+	if (WC_MODE_PMEM(wc))
+		pmem_flush(sb(wc), offsetof(struct wc_memory_superblock, entries[wc->n_blocks]));
+	else
+		memset(wc->dirty_bitmap, -1, wc->dirty_bitmap_size);
+}
+
+static void writecache_flush_region(struct dm_writecache *wc, void *ptr, size_t size)
+{
+	if (WC_MODE_PMEM(wc))
+		pmem_flush(ptr, size);
+	else
+		__set_bit(((char *)ptr - (char *)wc->memory_map) / BITMAP_GRANULARITY,
+			  wc->dirty_bitmap);
+}
+
+static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev);
+
+struct io_notify {
+	struct dm_writecache *wc;
+	struct completion c;
+	atomic_t count;
+};
+
+static void writecache_notify_io(unsigned long error, void *context)
+{
+	struct io_notify *endio = context;
+
+	if (unlikely(error != 0))
+		writecache_error(endio->wc, -EIO, "error writing metadata");
+	BUG_ON(atomic_read(&endio->count) <= 0);
+	if (atomic_dec_and_test(&endio->count))
+		complete(&endio->c);
+}
+
+static void ssd_commit_flushed(struct dm_writecache *wc)
+{
+	struct dm_io_region region;
+	struct dm_io_request req;
+	struct io_notify endio = {
+		wc,
+		COMPLETION_INITIALIZER_ONSTACK(endio.c),
+		ATOMIC_INIT(1),
+	};
+	unsigned bitmap_bits = wc->dirty_bitmap_size * BITS_PER_LONG;
+	unsigned i = 0;
+
+	while (1) {
+		unsigned j;
+		i = find_next_bit(wc->dirty_bitmap, bitmap_bits, i);
+		if (unlikely(i == bitmap_bits))
+			break;
+		j = find_next_zero_bit(wc->dirty_bitmap, bitmap_bits, i);
+
+		region.bdev = wc->ssd_dev->bdev;
+		region.sector = (sector_t)i * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
+		region.count = (sector_t)(j - i) * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
+
+		if (unlikely(region.sector >= wc->metadata_sectors))
+			break;
+		if (unlikely(region.sector + region.count > wc->metadata_sectors))
+			region.count = wc->metadata_sectors - region.sector;
+
+		atomic_inc(&endio.count);
+		req.bi_op = REQ_OP_WRITE;
+		req.bi_op_flags = REQ_SYNC;
+		req.mem.type = DM_IO_VMA;
+		req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY;
+		req.client = wc->dm_io;
+		req.notify.fn = writecache_notify_io;
+		req.notify.context = &endio;
+
+		/* writing via async dm-io (implied by notify.fn above) won't return an error */
+	        (void) dm_io(&req, 1, &region, NULL);
+		i = j;
+	}
+
+	writecache_notify_io(0, &endio);
+	wait_for_completion_io(&endio.c);
+
+	writecache_disk_flush(wc, wc->ssd_dev);
+
+	memset(wc->dirty_bitmap, 0, wc->dirty_bitmap_size);
+}
+
+static void writecache_commit_flushed(struct dm_writecache *wc)
+{
+	if (WC_MODE_PMEM(wc))
+		pmem_commit();
+	else
+		ssd_commit_flushed(wc);
+}
+
+static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
+{
+	int r;
+	struct dm_io_region region;
+	struct dm_io_request req;
+
+	region.bdev = dev->bdev;
+	region.sector = 0;
+	region.count = 0;
+	req.bi_op = REQ_OP_WRITE;
+	req.bi_op_flags = REQ_PREFLUSH;
+	req.mem.type = DM_IO_KMEM;
+	req.mem.ptr.addr = NULL;
+	req.client = wc->dm_io;
+	req.notify.fn = NULL;
+
+	r = dm_io(&req, 1, &region, NULL);
+	if (unlikely(r))
+		writecache_error(wc, r, "error flushing metadata: %d", r);
+}
+
+static void writecache_wait_for_ios(struct dm_writecache *wc, int direction)
+{
+	swait_event(wc->bio_in_progress_wait[direction],
+		   !atomic_read(&wc->bio_in_progress[direction]));
+}
+
+#define WFE_RETURN_FOLLOWING	1
+#define WFE_LOWEST_SEQ		2
+
+static struct wc_entry *writecache_find_entry(struct dm_writecache *wc,
+					      uint64_t block, int flags)
+{
+	struct wc_entry *e;
+	struct rb_node *node = wc->tree.rb_node;
+
+	if (unlikely(!node))
+		return NULL;
+
+	while (1) {
+		e = container_of(node, struct wc_entry, rb_node);
+		if (read_original_sector(wc, e) == block)
+			break;
+		node = (read_original_sector(wc, e) >= block ?
+			e->rb_node.rb_left : e->rb_node.rb_right);
+		if (unlikely(!node)) {
+			if (!(flags & WFE_RETURN_FOLLOWING)) {
+				return NULL;
+			}
+			if (read_original_sector(wc, e) >= block) {
+				break;
+			} else {
+				node = rb_next(&e->rb_node);
+				if (unlikely(!node)) {
+					return NULL;
+				}
+				e = container_of(node, struct wc_entry, rb_node);
+				break;
+			}
+		}
+	}
+
+	while (1) {
+		struct wc_entry *e2;
+		if (flags & WFE_LOWEST_SEQ)
+			node = rb_prev(&e->rb_node);
+		else
+			node = rb_next(&e->rb_node);
+		if (!node)
+			return e;
+		e2 = container_of(node, struct wc_entry, rb_node);
+		if (read_original_sector(wc, e2) != block)
+			return e;
+		e = e2;
+	}
+}
+
+static void writecache_insert_entry(struct dm_writecache *wc, struct wc_entry *ins)
+{
+	struct wc_entry *e;
+	struct rb_node **node = &wc->tree.rb_node, *parent = NULL;
+
+	while (*node) {
+		e = container_of(*node, struct wc_entry, rb_node);
+		parent = &e->rb_node;
+		if (read_original_sector(wc, e) > read_original_sector(wc, ins))
+			node = &parent->rb_left;
+		else
+			node = &parent->rb_right;
+	}
+	rb_link_node(&ins->rb_node, parent, node);
+	rb_insert_color(&ins->rb_node, &wc->tree);
+	list_add(&ins->lru, &wc->lru);
+}
+
+static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e)
+{
+	list_del(&e->lru);
+	rb_erase(&e->rb_node, &wc->tree);
+}
+
+static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry *e)
+{
+	if (WC_MODE_SORT_FREELIST(wc)) {
+		struct rb_node **node = &wc->freetree.rb_node, *parent = NULL;
+		if (unlikely(!*node))
+			wc->current_free = e;
+		while (*node) {
+			parent = *node;
+			if (&e->rb_node < *node)
+				node = &parent->rb_left;
+			else
+				node = &parent->rb_right;
+		}
+		rb_link_node(&e->rb_node, parent, node);
+		rb_insert_color(&e->rb_node, &wc->freetree);
+	} else {
+		list_add_tail(&e->lru, &wc->freelist);
+	}
+	wc->freelist_size++;
+}
+
+static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc)
+{
+	struct wc_entry *e;
+
+	if (WC_MODE_SORT_FREELIST(wc)) {
+		struct rb_node *next;
+		if (unlikely(!wc->current_free))
+			return NULL;
+		e = wc->current_free;
+		next = rb_next(&e->rb_node);
+		rb_erase(&e->rb_node, &wc->freetree);
+		if (unlikely(!next))
+			next = rb_first(&wc->freetree);
+		wc->current_free = next ? container_of(next, struct wc_entry, rb_node) : NULL;
+	} else {
+		if (unlikely(list_empty(&wc->freelist)))
+			return NULL;
+		e = container_of(wc->freelist.next, struct wc_entry, lru);
+		list_del(&e->lru);
+	}
+	wc->freelist_size--;
+	if (unlikely(wc->freelist_size <= wc->freelist_high_watermark))
+		queue_work(wc->writeback_wq, &wc->writeback_work);
+
+	return e;
+}
+
+static void writecache_free_entry(struct dm_writecache *wc, struct wc_entry *e)
+{
+	writecache_unlink(wc, e);
+	writecache_add_to_freelist(wc, e);
+	clear_seq_count(wc, e);
+	writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
+	if (unlikely(swait_active(&wc->freelist_wait)))
+		swake_up(&wc->freelist_wait);
+}
+
+static void writecache_wait_on_freelist(struct dm_writecache *wc)
+{
+	DECLARE_SWAITQUEUE(wait);
+
+	prepare_to_swait(&wc->freelist_wait, &wait, TASK_UNINTERRUPTIBLE);
+	wc_unlock(wc);
+	io_schedule();
+	finish_swait(&wc->freelist_wait, &wait);
+	wc_lock(wc);
+}
+
+static void writecache_poison_lists(struct dm_writecache *wc)
+{
+	/*
+	 * Catch incorrect access to these values while the device is suspended.
+	 */
+	memset(&wc->tree, -1, sizeof wc->tree);
+	wc->lru.next = LIST_POISON1;
+	wc->lru.prev = LIST_POISON2;
+	wc->freelist.next = LIST_POISON1;
+	wc->freelist.prev = LIST_POISON2;
+}
+
+static void writecache_flush_entry(struct dm_writecache *wc, struct wc_entry *e)
+{
+	writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
+	if (WC_MODE_PMEM(wc))
+		writecache_flush_region(wc, memory_data(wc, e), wc->block_size);
+}
+
+static bool writecache_entry_is_committed(struct dm_writecache *wc, struct wc_entry *e)
+{
+	return read_seq_count(wc, e) < wc->seq_count;
+}
+
+static void writecache_flush(struct dm_writecache *wc)
+{
+	struct wc_entry *e, *e2;
+	bool need_flush_after_free;
+
+	wc->uncommitted_blocks = 0;
+	del_timer(&wc->autocommit_timer);
+
+	if (list_empty(&wc->lru))
+		return;
+
+	e = container_of(wc->lru.next, struct wc_entry, lru);
+	if (writecache_entry_is_committed(wc, e)) {
+		if (wc->overwrote_committed) {
+			writecache_wait_for_ios(wc, WRITE);
+			writecache_disk_flush(wc, wc->ssd_dev);
+			wc->overwrote_committed = false;
+		}
+		return;
+	}
+	while (1) {
+		writecache_flush_entry(wc, e);
+		if (unlikely(e->lru.next == &wc->lru))
+			break;
+		e2 = container_of(e->lru.next, struct wc_entry, lru);
+		if (writecache_entry_is_committed(wc, e2))
+			break;
+		e = e2;
+		cond_resched();
+	}
+	writecache_commit_flushed(wc);
+
+	writecache_wait_for_ios(wc, WRITE);
+
+	wc->seq_count++;
+	pmem_assign(sb(wc)->seq_count, cpu_to_le64(wc->seq_count));
+	writecache_flush_region(wc, &sb(wc)->seq_count, sizeof sb(wc)->seq_count);
+	writecache_commit_flushed(wc);
+
+	wc->overwrote_committed = false;
+
+	need_flush_after_free = false;
+	while (1) {
+		/* Free another committed entry with lower seq-count */
+		struct rb_node *rb_node = rb_prev(&e->rb_node);
+
+		if (rb_node) {
+			e2 = container_of(rb_node, struct wc_entry, rb_node);
+			if (read_original_sector(wc, e2) == read_original_sector(wc, e) &&
+			    likely(!e2->write_in_progress)) {
+				writecache_free_entry(wc, e2);
+				need_flush_after_free = true;
+			}
+		}
+		if (unlikely(e->lru.prev == &wc->lru))
+			break;
+		e = container_of(e->lru.prev, struct wc_entry, lru);
+		cond_resched();
+	}
+
+	if (need_flush_after_free)
+		writecache_commit_flushed(wc);
+}
+
+static void writecache_flush_work(struct work_struct *work)
+{
+	struct dm_writecache *wc = container_of(work, struct dm_writecache, flush_work);
+	wc_lock(wc);
+	writecache_flush(wc);
+	wc_unlock(wc);
+}
+
+static void writecache_autocommit_timer(struct timer_list *t)
+{
+	struct dm_writecache *wc = from_timer(wc, t, autocommit_timer);
+	if (!writecache_has_error(wc))
+		queue_work(wc->writeback_wq, &wc->flush_work);
+}
+
+static void writecache_schedule_autocommit(struct dm_writecache *wc)
+{
+	if (!timer_pending(&wc->autocommit_timer))
+		mod_timer(&wc->autocommit_timer, jiffies + wc->autocommit_jiffies);
+}
+
+static void writecache_discard(struct dm_writecache *wc, sector_t start, sector_t end)
+{
+	struct wc_entry *e;
+	bool discarded_something = false;
+
+	e = writecache_find_entry(wc, start, WFE_RETURN_FOLLOWING | WFE_LOWEST_SEQ);
+	if (unlikely(!e))
+		return;
+
+	while (read_original_sector(wc, e) < end) {
+		struct rb_node *node = rb_next(&e->rb_node);
+
+		if (likely(!e->write_in_progress)) {
+			if (!discarded_something) {
+				writecache_wait_for_ios(wc, READ);
+				writecache_wait_for_ios(wc, WRITE);
+				discarded_something = true;
+			}
+			writecache_free_entry(wc, e);
+		}
+
+		if (!node)
+			break;
+
+		e = container_of(node, struct wc_entry, rb_node);
+	}
+
+	if (discarded_something)
+		writecache_commit_flushed(wc);
+}
+
+static bool writecache_wait_for_writeback(struct dm_writecache *wc)
+{
+	if (wc->writeback_size) {
+		writecache_wait_on_freelist(wc);
+		return true;
+	}
+	return false;
+}
+
+static void writecache_suspend(struct dm_target *ti)
+{
+	struct dm_writecache *wc = ti->private;
+	bool flush_on_suspend;
+
+	del_timer_sync(&wc->autocommit_timer);
+
+	wc_lock(wc);
+	writecache_flush(wc);
+	flush_on_suspend = wc->flush_on_suspend;
+	if (flush_on_suspend) {
+		wc->flush_on_suspend = false;
+		wc->writeback_all++;
+		queue_work(wc->writeback_wq, &wc->writeback_work);
+	}
+	wc_unlock(wc);
+
+	flush_workqueue(wc->writeback_wq);
+
+	wc_lock(wc);
+	if (flush_on_suspend)
+		wc->writeback_all--;
+	while (writecache_wait_for_writeback(wc));
+
+	if (WC_MODE_PMEM(wc))
+		persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
+
+	writecache_poison_lists(wc);
+
+	wc_unlock(wc);
+}
+
+static int writecache_alloc_entries(struct dm_writecache *wc)
+{
+	size_t b;
+
+	if (wc->entries)
+		return 0;
+	wc->entries = vmalloc(sizeof(struct wc_entry) * wc->n_blocks);
+	if (!wc->entries)
+		return -ENOMEM;
+	for (b = 0; b < wc->n_blocks; b++) {
+		struct wc_entry *e = &wc->entries[b];
+		e->index = b;
+		e->write_in_progress = false;
+	}
+
+	return 0;
+}
+
+static void writecache_resume(struct dm_target *ti)
+{
+	struct dm_writecache *wc = ti->private;
+	size_t b;
+	bool need_flush = false;
+	__le64 sb_seq_count;
+	int r;
+
+	wc_lock(wc);
+
+	if (WC_MODE_PMEM(wc))
+		persistent_memory_invalidate_cache(wc->memory_map, wc->memory_map_size);
+
+	wc->tree = RB_ROOT;
+	INIT_LIST_HEAD(&wc->lru);
+	if (WC_MODE_SORT_FREELIST(wc)) {
+		wc->freetree = RB_ROOT;
+		wc->current_free = NULL;
+	} else {
+		INIT_LIST_HEAD(&wc->freelist);
+	}
+	wc->freelist_size = 0;
+
+	r = memcpy_mcsafe(&sb_seq_count, &sb(wc)->seq_count, sizeof(uint64_t));
+	if (r) {
+		writecache_error(wc, r, "hardware memory error when reading superblock: %d", r);
+		sb_seq_count = cpu_to_le64(0);
+	}
+	wc->seq_count = le64_to_cpu(sb_seq_count);
+
+#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
+	for (b = 0; b < wc->n_blocks; b++) {
+		struct wc_entry *e = &wc->entries[b];
+		struct wc_memory_entry wme;
+		if (writecache_has_error(wc)) {
+			e->original_sector = -1;
+			e->seq_count = -1;
+			continue;
+		}
+		r = memcpy_mcsafe(&wme, memory_entry(wc, e), sizeof(struct wc_memory_entry));
+		if (r) {
+			writecache_error(wc, r, "hardware memory error when reading metadata entry %lu: %d",
+					 (unsigned long)b, r);
+			e->original_sector = -1;
+			e->seq_count = -1;
+		} else {
+			e->original_sector = le64_to_cpu(wme.original_sector);
+			e->seq_count = le64_to_cpu(wme.seq_count);
+		}
+	}
+#endif
+	for (b = 0; b < wc->n_blocks; b++) {
+		struct wc_entry *e = &wc->entries[b];
+		if (!writecache_entry_is_committed(wc, e)) {
+			if (read_seq_count(wc, e) != -1) {
+erase_this:
+				clear_seq_count(wc, e);
+				need_flush = true;
+			}
+			writecache_add_to_freelist(wc, e);
+		} else {
+			struct wc_entry *old;
+
+			old = writecache_find_entry(wc, read_original_sector(wc, e), 0);
+			if (!old) {
+				writecache_insert_entry(wc, e);
+			} else {
+				if (read_seq_count(wc, old) == read_seq_count(wc, e)) {
+					writecache_error(wc, -EINVAL,
+						 "two identical entries, position %llu, sector %llu, sequence %llu",
+						 (unsigned long long)b, (unsigned long long)read_original_sector(wc, e),
+						 (unsigned long long)read_seq_count(wc, e));
+				}
+				if (read_seq_count(wc, old) > read_seq_count(wc, e)) {
+					goto erase_this;
+				} else {
+					writecache_free_entry(wc, old);
+					writecache_insert_entry(wc, e);
+					need_flush = true;
+				}
+			}
+		}
+		cond_resched();
+	}
+
+	if (need_flush) {
+		writecache_flush_all_metadata(wc);
+		writecache_commit_flushed(wc);
+	}
+
+	wc_unlock(wc);
+}
+
+static int process_flush_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
+{
+	if (argc != 1)
+		return -EINVAL;
+
+	wc_lock(wc);
+	if (dm_suspended(wc->ti)) {
+		wc_unlock(wc);
+		return -EBUSY;
+	}
+	if (writecache_has_error(wc)) {
+		wc_unlock(wc);
+		return -EIO;
+	}
+
+	writecache_flush(wc);
+	wc->writeback_all++;
+	queue_work(wc->writeback_wq, &wc->writeback_work);
+	wc_unlock(wc);
+
+	flush_workqueue(wc->writeback_wq);
+
+	wc_lock(wc);
+	wc->writeback_all--;
+	if (writecache_has_error(wc)) {
+		wc_unlock(wc);
+		return -EIO;
+	}
+	wc_unlock(wc);
+
+	return 0;
+}
+
+static int process_flush_on_suspend_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
+{
+	if (argc != 1)
+		return -EINVAL;
+
+	wc_lock(wc);
+	wc->flush_on_suspend = true;
+	wc_unlock(wc);
+
+	return 0;
+}
+
+static int writecache_message(struct dm_target *ti, unsigned argc, char **argv,
+			      char *result, unsigned maxlen)
+{
+	int r = -EINVAL;
+	struct dm_writecache *wc = ti->private;
+
+	if (!strcasecmp(argv[0], "flush"))
+		r = process_flush_mesg(argc, argv, wc);
+	else if (!strcasecmp(argv[0], "flush_on_suspend"))
+		r = process_flush_on_suspend_mesg(argc, argv, wc);
+	else
+		DMERR("unrecognised message received: %s", argv[0]);
+
+	return r;
+}
+
+static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data)
+{
+	void *buf;
+	unsigned long flags;
+	unsigned size;
+	int rw = bio_data_dir(bio);
+	unsigned remaining_size = wc->block_size;
+
+	do {
+		struct bio_vec bv = bio_iter_iovec(bio, bio->bi_iter);
+		buf = bvec_kmap_irq(&bv, &flags);
+		size = bv.bv_len;
+		if (unlikely(size > remaining_size))
+			size = remaining_size;
+
+		if (rw == READ) {
+			int r;
+			r = memcpy_mcsafe(buf, data, size);
+			flush_dcache_page(bio_page(bio));
+			if (unlikely(r)) {
+				writecache_error(wc, r, "hardware memory error when reading data: %d", r);
+				bio->bi_status = BLK_STS_IOERR;
+			}
+		} else {
+			flush_dcache_page(bio_page(bio));
+			pmem_memcpy(data, buf, size);
+		}
+
+		bvec_kunmap_irq(buf, &flags);
+
+		data = (char *)data + size;
+		remaining_size -= size;
+		bio_advance(bio, size);
+	} while (unlikely(remaining_size));
+}
+
+static int writecache_flush_thread(void *data)
+{
+	struct dm_writecache *wc = data;
+
+	while (!kthread_should_stop()) {
+		struct bio *bio = wc->flush_bio;
+
+		if (likely(bio)) {
+			if (bio_op(bio) == REQ_OP_DISCARD)
+				writecache_discard(wc, bio->bi_iter.bi_sector, bio_end_sector(bio));
+			else
+				writecache_flush(wc);
+		}
+
+		set_current_state(TASK_INTERRUPTIBLE);
+		/* for debugging - catch uninitialized use */
+		wc->flush_bio = (void *)0x600 + POISON_POINTER_DELTA;
+		complete(&wc->flush_completion);
+
+		schedule();
+	}
+
+	set_current_state(TASK_RUNNING);
+
+	return 0;
+}
+
+static void writecache_offload_bio(struct dm_writecache *wc, struct bio *bio)
+{
+	wc->flush_bio = bio;
+	reinit_completion(&wc->flush_completion);
+	wake_up_process(wc->flush_thread);
+	wait_for_completion_io(&wc->flush_completion);
+}
+
+static int writecache_map(struct dm_target *ti, struct bio *bio)
+{
+	struct wc_entry *e;
+	struct dm_writecache *wc = ti->private;
+
+	bio->bi_private = NULL;
+
+	wc_lock(wc);
+
+	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
+		if (writecache_has_error(wc))
+			goto unlock_error;
+		if (WC_MODE_PMEM(wc))
+			writecache_flush(wc);
+		else
+			writecache_offload_bio(wc, bio);
+		goto unlock_ok;
+	}
+
+	bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
+
+	if (unlikely((((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) &
+				(wc->block_size / 512 - 1)) != 0)) {
+		DMERR("I/O is not aligned, sector %llu, size %u, block size %u",
+		      (unsigned long long)bio->bi_iter.bi_sector,
+		      bio->bi_iter.bi_size, wc->block_size);
+		goto unlock_error;
+	}
+
+	if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
+		if (writecache_has_error(wc))
+			goto unlock_error;
+		if (WC_MODE_PMEM(wc))
+			writecache_discard(wc, bio->bi_iter.bi_sector, bio_end_sector(bio));
+		else
+			writecache_offload_bio(wc, bio);
+		goto unlock_remap_origin;
+	}
+
+	if (bio_data_dir(bio) == READ) {
+read_next_block:
+		e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
+		if (e && read_original_sector(wc, e) == bio->bi_iter.bi_sector) {
+			if (WC_MODE_PMEM(wc)) {
+				bio_copy_block(wc, bio, memory_data(wc, e));
+				if (bio->bi_iter.bi_size)
+					goto read_next_block;
+				goto unlock_ok;
+			} else {
+				dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT);
+				bio_set_dev(bio, wc->ssd_dev->bdev);
+				bio->bi_iter.bi_sector = cache_sector(wc, e);
+				if (!writecache_entry_is_committed(wc, e))
+					writecache_wait_for_ios(wc, WRITE);
+				goto unlock_remap;
+			}
+		} else {
+			if (e) {
+				sector_t next_boundary =
+					read_original_sector(wc, e) - bio->bi_iter.bi_sector;
+				if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) {
+					dm_accept_partial_bio(bio, next_boundary);
+				}
+			}
+			goto unlock_remap_origin;
+		}
+	} else {
+		do {
+			if (writecache_has_error(wc))
+				goto unlock_error;
+			e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
+			if (e) {
+				if (!writecache_entry_is_committed(wc, e))
+					goto bio_copy;
+				if (!WC_MODE_PMEM(wc) && !e->write_in_progress) {
+					wc->overwrote_committed = true;
+					goto bio_copy;
+				}
+			}
+			e = writecache_pop_from_freelist(wc);
+			if (unlikely(!e)) {
+				writecache_wait_on_freelist(wc);
+				continue;
+			}
+			write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count);
+			writecache_insert_entry(wc, e);
+			wc->uncommitted_blocks++;
+bio_copy:
+			if (WC_MODE_PMEM(wc)) {
+				bio_copy_block(wc, bio, memory_data(wc, e));
+			} else {
+				dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT);
+				bio_set_dev(bio, wc->ssd_dev->bdev);
+				bio->bi_iter.bi_sector = cache_sector(wc, e);
+				if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
+					wc->uncommitted_blocks = 0;
+					queue_work(wc->writeback_wq, &wc->flush_work);
+				} else {
+					writecache_schedule_autocommit(wc);
+				}
+				goto unlock_remap;
+			}
+		} while (bio->bi_iter.bi_size);
+
+		if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks))
+			writecache_flush(wc);
+		else
+			writecache_schedule_autocommit(wc);
+		goto unlock_ok;
+	}
+
+unlock_remap_origin:
+	bio_set_dev(bio, wc->dev->bdev);
+	wc_unlock(wc);
+	return DM_MAPIO_REMAPPED;
+
+unlock_remap:
+	/* make sure that writecache_end_io decrements bio_in_progress: */
+	bio->bi_private = (void *)1;
+	atomic_inc(&wc->bio_in_progress[bio_data_dir(bio)]);
+	wc_unlock(wc);
+	return DM_MAPIO_REMAPPED;
+
+unlock_ok:
+	wc_unlock(wc);
+	bio_endio(bio);
+	return DM_MAPIO_SUBMITTED;
+
+unlock_error:
+	wc_unlock(wc);
+	bio_io_error(bio);
+	return DM_MAPIO_SUBMITTED;
+}
+
+static int writecache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status)
+{
+	struct dm_writecache *wc = ti->private;
+
+	if (bio->bi_private != NULL) {
+		int dir = bio_data_dir(bio);
+		if (atomic_dec_and_test(&wc->bio_in_progress[dir]))
+			if (unlikely(swait_active(&wc->bio_in_progress_wait[dir])))
+				swake_up(&wc->bio_in_progress_wait[dir]);
+	}
+	return 0;
+}
+
+static int writecache_iterate_devices(struct dm_target *ti,
+				      iterate_devices_callout_fn fn, void *data)
+{
+	struct dm_writecache *wc = ti->private;
+
+	return fn(ti, wc->dev, 0, ti->len, data);
+}
+
+static void writecache_io_hints(struct dm_target *ti, struct queue_limits *limits)
+{
+	struct dm_writecache *wc = ti->private;
+
+	if (limits->logical_block_size < wc->block_size)
+		limits->logical_block_size = wc->block_size;
+
+	if (limits->physical_block_size < wc->block_size)
+		limits->physical_block_size = wc->block_size;
+
+	if (limits->io_min < wc->block_size)
+		limits->io_min = wc->block_size;
+}
+
+
+static void writecache_writeback_endio(struct bio *bio)
+{
+	struct writeback_struct *wb = container_of(bio, struct writeback_struct, bio);
+	struct dm_writecache *wc = wb->wc;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&wc->endio_thread_wait.lock, flags);
+	list_add_tail(&wb->endio_entry, &wc->endio_list);
+	swake_up_locked(&wc->endio_thread_wait);
+	raw_spin_unlock_irqrestore(&wc->endio_thread_wait.lock, flags);
+}
+
+static void writecache_copy_endio(int read_err, unsigned long write_err, void *ptr)
+{
+	struct copy_struct *c = ptr;
+	struct dm_writecache *wc = c->wc;
+
+	c->error = likely(!(read_err | write_err)) ? 0 : -EIO;
+
+	raw_spin_lock_irq(&wc->endio_thread_wait.lock);
+	list_add_tail(&c->endio_entry, &wc->endio_list);
+	swake_up_locked(&wc->endio_thread_wait);
+	raw_spin_unlock_irq(&wc->endio_thread_wait.lock);
+}
+
+static void __writecache_endio_pmem(struct dm_writecache *wc, struct list_head *list)
+{
+	unsigned i;
+	struct writeback_struct *wb;
+	struct wc_entry *e;
+	unsigned long n_walked = 0;
+
+	do {
+		wb = list_entry(list->next, struct writeback_struct, endio_entry);
+		list_del(&wb->endio_entry);
+
+		if (unlikely(wb->bio.bi_status != BLK_STS_OK))
+			writecache_error(wc, blk_status_to_errno(wb->bio.bi_status),
+					"write error %d", wb->bio.bi_status);
+		i = 0;
+		do {
+			e = wb->wc_list[i];
+			BUG_ON(!e->write_in_progress);
+			e->write_in_progress = false;
+			INIT_LIST_HEAD(&e->lru);
+			if (!writecache_has_error(wc))
+				writecache_free_entry(wc, e);
+			BUG_ON(!wc->writeback_size);
+			wc->writeback_size--;
+			n_walked++;
+			if (unlikely(n_walked >= ENDIO_LATENCY)) {
+				writecache_commit_flushed(wc);
+				wc_unlock(wc);
+				wc_lock(wc);
+				n_walked = 0;
+			}
+		} while (++i < wb->wc_list_n);
+
+		if (wb->wc_list != wb->wc_list_inline)
+			kfree(wb->wc_list);
+		bio_put(&wb->bio);
+	} while (!list_empty(list));
+}
+
+static void __writecache_endio_ssd(struct dm_writecache *wc, struct list_head *list)
+{
+	struct copy_struct *c;
+	struct wc_entry *e;
+
+	do {
+		c = list_entry(list->next, struct copy_struct, endio_entry);
+		list_del(&c->endio_entry);
+
+		if (unlikely(c->error))
+			writecache_error(wc, c->error, "copy error");
+
+		e = c->e;
+		do {
+			BUG_ON(!e->write_in_progress);
+			e->write_in_progress = false;
+			INIT_LIST_HEAD(&e->lru);
+			if (!writecache_has_error(wc))
+				writecache_free_entry(wc, e);
+
+			BUG_ON(!wc->writeback_size);
+			wc->writeback_size--;
+			e++;
+		} while (--c->n_entries);
+		mempool_free(c, wc->copy_pool);
+	} while (!list_empty(list));
+}
+
+static int writecache_endio_thread(void *data)
+{
+	struct dm_writecache *wc = data;
+
+	while (1) {
+		DECLARE_SWAITQUEUE(wait);
+		struct list_head list;
+
+		raw_spin_lock_irq(&wc->endio_thread_wait.lock);
+continue_locked:
+		if (!list_empty(&wc->endio_list))
+			goto pop_from_list;
+		set_current_state(TASK_INTERRUPTIBLE);
+		__prepare_to_swait(&wc->endio_thread_wait, &wait);
+		raw_spin_unlock_irq(&wc->endio_thread_wait.lock);
+
+		if (unlikely(kthread_should_stop())) {
+			finish_swait(&wc->endio_thread_wait, &wait);
+			break;
+		}
+
+		schedule();
+
+		raw_spin_lock_irq(&wc->endio_thread_wait.lock);
+		__finish_swait(&wc->endio_thread_wait, &wait);
+		goto continue_locked;
+
+pop_from_list:
+		list = wc->endio_list;
+		list.next->prev = list.prev->next = &list;
+		INIT_LIST_HEAD(&wc->endio_list);
+		raw_spin_unlock_irq(&wc->endio_thread_wait.lock);
+
+		if (!WC_MODE_FUA(wc))
+			writecache_disk_flush(wc, wc->dev);
+
+		wc_lock(wc);
+
+		if (WC_MODE_PMEM(wc)) {
+			__writecache_endio_pmem(wc, &list);
+		} else {
+			__writecache_endio_ssd(wc, &list);
+			writecache_wait_for_ios(wc, READ);
+		}
+
+		writecache_commit_flushed(wc);
+
+		wc_unlock(wc);
+	}
+
+	return 0;
+}
+
+static bool wc_add_block(struct writeback_struct *wb, struct wc_entry *e, gfp_t gfp)
+{
+	struct dm_writecache *wc = wb->wc;
+	unsigned block_size = wc->block_size;
+	void *address = memory_data(wc, e);
+
+	persistent_memory_flush_cache(address, block_size);
+	return bio_add_page(&wb->bio, persistent_memory_page(address),
+			    block_size, persistent_memory_page_offset(address)) != 0;
+}
+
+struct writeback_list {
+	struct list_head list;
+	size_t size;
+};
+
+static void __writeback_throttle(struct dm_writecache *wc, struct writeback_list *wbl)
+{
+	if (unlikely(wc->max_writeback_jobs)) {
+		if (READ_ONCE(wc->writeback_size) - wbl->size >= wc->max_writeback_jobs) {
+			wc_lock(wc);
+			while (wc->writeback_size - wbl->size >= wc->max_writeback_jobs)
+				writecache_wait_on_freelist(wc);
+			wc_unlock(wc);
+		}
+	}
+	cond_resched();
+}
+
+static void __writecache_writeback_pmem(struct dm_writecache *wc, struct writeback_list *wbl)
+{
+	struct wc_entry *e, *f;
+	struct bio *bio;
+	struct writeback_struct *wb;
+	unsigned max_pages;
+
+	while (wbl->size) {
+		wbl->size--;
+		e = container_of(wbl->list.prev, struct wc_entry, lru);
+		list_del(&e->lru);
+
+		max_pages = e->wc_list_contiguous;
+
+		bio = bio_alloc_bioset(GFP_NOIO, max_pages, &wc->bio_set);
+		wb = container_of(bio, struct writeback_struct, bio);
+		wb->wc = wc;
+		wb->bio.bi_end_io = writecache_writeback_endio;
+		bio_set_dev(&wb->bio, wc->dev->bdev);
+		wb->bio.bi_iter.bi_sector = read_original_sector(wc, e);
+		wb->page_offset = PAGE_SIZE;
+		if (max_pages <= WB_LIST_INLINE ||
+		    unlikely(!(wb->wc_list = kmalloc(max_pages * sizeof(struct wc_entry *),
+						     GFP_NOIO | __GFP_NORETRY |
+						     __GFP_NOMEMALLOC | __GFP_NOWARN)))) {
+			wb->wc_list = wb->wc_list_inline;
+			max_pages = WB_LIST_INLINE;
+		}
+
+		BUG_ON(!wc_add_block(wb, e, GFP_NOIO));
+
+		wb->wc_list[0] = e;
+		wb->wc_list_n = 1;
+
+		while (wbl->size && wb->wc_list_n < max_pages) {
+			f = container_of(wbl->list.prev, struct wc_entry, lru);
+			if (read_original_sector(wc, f) !=
+			    read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
+				break;
+			if (!wc_add_block(wb, f, GFP_NOWAIT | __GFP_NOWARN))
+				break;
+			wbl->size--;
+			list_del(&f->lru);
+			wb->wc_list[wb->wc_list_n++] = f;
+			e = f;
+		}
+		bio_set_op_attrs(&wb->bio, REQ_OP_WRITE, WC_MODE_FUA(wc) * REQ_FUA);
+		if (writecache_has_error(wc)) {
+			bio->bi_status = BLK_STS_IOERR;
+			bio_endio(&wb->bio);
+		} else {
+			submit_bio(&wb->bio);
+		}
+
+		__writeback_throttle(wc, wbl);
+	}
+}
+
+static void __writecache_writeback_ssd(struct dm_writecache *wc, struct writeback_list *wbl)
+{
+	struct wc_entry *e, *f;
+	struct dm_io_region from, to;
+	struct copy_struct *c;
+
+	while (wbl->size) {
+		unsigned n_sectors;
+
+		wbl->size--;
+		e = container_of(wbl->list.prev, struct wc_entry, lru);
+		list_del(&e->lru);
+
+		n_sectors = e->wc_list_contiguous << (wc->block_size_bits - SECTOR_SHIFT);
+
+		from.bdev = wc->ssd_dev->bdev;
+		from.sector = cache_sector(wc, e);
+		from.count = n_sectors;
+		to.bdev = wc->dev->bdev;
+		to.sector = read_original_sector(wc, e);
+		to.count = n_sectors;
+
+		c = mempool_alloc(wc->copy_pool, GFP_NOIO);
+		c->wc = wc;
+		c->e = e;
+		c->n_entries = e->wc_list_contiguous;
+
+		while ((n_sectors -= wc->block_size >> SECTOR_SHIFT)) {
+			wbl->size--;
+			f = container_of(wbl->list.prev, struct wc_entry, lru);
+			BUG_ON(f != e + 1);
+			list_del(&f->lru);
+			e = f;
+		}
+
+		dm_kcopyd_copy(wc->dm_kcopyd, &from, 1, &to, 0, writecache_copy_endio, c);
+
+		__writeback_throttle(wc, wbl);
+	}
+}
+
+static void writecache_writeback(struct work_struct *work)
+{
+	struct dm_writecache *wc = container_of(work, struct dm_writecache, writeback_work);
+	struct blk_plug plug;
+	struct wc_entry *e, *f, *g;
+	struct rb_node *node, *next_node;
+	struct list_head skipped;
+	struct writeback_list wbl;
+	unsigned long n_walked;
+
+	wc_lock(wc);
+restart:
+	if (writecache_has_error(wc)) {
+		wc_unlock(wc);
+		return;
+	}
+
+	if (unlikely(wc->writeback_all)) {
+		if (writecache_wait_for_writeback(wc))
+			goto restart;
+	}
+
+	if (wc->overwrote_committed) {
+		writecache_wait_for_ios(wc, WRITE);
+	}
+
+	n_walked = 0;
+	INIT_LIST_HEAD(&skipped);
+	INIT_LIST_HEAD(&wbl.list);
+	wbl.size = 0;
+	while (!list_empty(&wc->lru) &&
+	       (wc->writeback_all ||
+		wc->freelist_size + wc->writeback_size <= wc->freelist_high_watermark)) {
+
+		n_walked++;
+		if (unlikely(n_walked > WRITEBACK_LATENCY) &&
+		    likely(!wc->writeback_all) && likely(!dm_suspended(wc->ti))) {
+			queue_work(wc->writeback_wq, &wc->writeback_work);
+			break;
+		}
+
+		e = container_of(wc->lru.prev, struct wc_entry, lru);
+		BUG_ON(e->write_in_progress);
+		if (unlikely(!writecache_entry_is_committed(wc, e))) {
+			writecache_flush(wc);
+		}
+		node = rb_prev(&e->rb_node);
+		if (node) {
+			f = container_of(node, struct wc_entry, rb_node);
+			if (unlikely(read_original_sector(wc, f) ==
+				     read_original_sector(wc, e))) {
+				BUG_ON(!f->write_in_progress);
+				list_del(&e->lru);
+				list_add(&e->lru, &skipped);
+				cond_resched();
+				continue;
+			}
+		}
+		wc->writeback_size++;
+		list_del(&e->lru);
+		list_add(&e->lru, &wbl.list);
+		wbl.size++;
+		e->write_in_progress = true;
+		e->wc_list_contiguous = 1;
+
+		f = e;
+
+		while (1) {
+			next_node = rb_next(&f->rb_node);
+			if (unlikely(!next_node))
+				break;
+			g = container_of(next_node, struct wc_entry, rb_node);
+			if (read_original_sector(wc, g) ==
+			    read_original_sector(wc, f)) {
+				f = g;
+				continue;
+			}
+			if (read_original_sector(wc, g) !=
+			    read_original_sector(wc, f) + (wc->block_size >> SECTOR_SHIFT))
+				break;
+			if (unlikely(g->write_in_progress))
+				break;
+			if (unlikely(!writecache_entry_is_committed(wc, g)))
+				break;
+
+			if (!WC_MODE_PMEM(wc)) {
+				if (g != f + 1)
+					break;
+			}
+
+			n_walked++;
+			//if (unlikely(n_walked > WRITEBACK_LATENCY) && likely(!wc->writeback_all))
+			//	break;
+
+			wc->writeback_size++;
+			list_del(&g->lru);
+			list_add(&g->lru, &wbl.list);
+			wbl.size++;
+			g->write_in_progress = true;
+			g->wc_list_contiguous = BIO_MAX_PAGES;
+			f = g;
+			e->wc_list_contiguous++;
+			if (unlikely(e->wc_list_contiguous == BIO_MAX_PAGES))
+				break;
+		}
+		cond_resched();
+	}
+
+	if (!list_empty(&skipped)) {
+		list_splice_tail(&skipped, &wc->lru);
+		/*
+		 * If we didn't do any progress, we must wait until some
+		 * writeback finishes to avoid burning CPU in a loop
+		 */
+		if (unlikely(!wbl.size))
+			writecache_wait_for_writeback(wc);
+	}
+
+	wc_unlock(wc);
+
+	blk_start_plug(&plug);
+
+	if (WC_MODE_PMEM(wc))
+		__writecache_writeback_pmem(wc, &wbl);
+	else
+		__writecache_writeback_ssd(wc, &wbl);
+
+	blk_finish_plug(&plug);
+
+	if (unlikely(wc->writeback_all)) {
+		wc_lock(wc);
+		while (writecache_wait_for_writeback(wc));
+		wc_unlock(wc);
+	}
+}
+
+static int calculate_memory_size(uint64_t device_size, unsigned block_size,
+				 size_t *n_blocks_p, size_t *n_metadata_blocks_p)
+{
+	uint64_t n_blocks, offset;
+	struct wc_entry e;
+
+	n_blocks = device_size;
+	do_div(n_blocks, block_size + sizeof(struct wc_memory_entry));
+
+	while (1) {
+		if (!n_blocks)
+			return -ENOSPC;
+		/* Verify the following entries[n_blocks] won't overflow */
+		if (n_blocks >= ((size_t)-sizeof(struct wc_memory_superblock) /
+				 sizeof(struct wc_memory_entry)))
+			return -EFBIG;
+		offset = offsetof(struct wc_memory_superblock, entries[n_blocks]);
+		offset = (offset + block_size - 1) & ~(uint64_t)(block_size - 1);
+		if (offset + n_blocks * block_size <= device_size)
+			break;
+		n_blocks--;
+	}
+
+	/* check if the bit field overflows */
+	e.index = n_blocks;
+	if (e.index != n_blocks)
+		return -EFBIG;
+
+	if (n_blocks_p)
+		*n_blocks_p = n_blocks;
+	if (n_metadata_blocks_p)
+		*n_metadata_blocks_p = offset >> __ffs(block_size);
+	return 0;
+}
+
+static int init_memory(struct dm_writecache *wc)
+{
+	size_t b;
+	int r;
+
+	r = calculate_memory_size(wc->memory_map_size, wc->block_size, &wc->n_blocks, NULL);
+	if (r)
+		return r;
+
+	r = writecache_alloc_entries(wc);
+	if (r)
+		return r;
+
+	for (b = 0; b < ARRAY_SIZE(sb(wc)->padding); b++)
+		pmem_assign(sb(wc)->padding[b], cpu_to_le64(0));
+	pmem_assign(sb(wc)->version, cpu_to_le32(MEMORY_SUPERBLOCK_VERSION));
+	pmem_assign(sb(wc)->block_size, cpu_to_le32(wc->block_size));
+	pmem_assign(sb(wc)->n_blocks, cpu_to_le64(wc->n_blocks));
+	pmem_assign(sb(wc)->seq_count, cpu_to_le64(0));
+
+	for (b = 0; b < wc->n_blocks; b++)
+		write_original_sector_seq_count(wc, &wc->entries[b], -1, -1);
+
+	writecache_flush_all_metadata(wc);
+	writecache_commit_flushed(wc);
+	pmem_assign(sb(wc)->magic, cpu_to_le32(MEMORY_SUPERBLOCK_MAGIC));
+	writecache_flush_region(wc, &sb(wc)->magic, sizeof sb(wc)->magic);
+	writecache_commit_flushed(wc);
+
+	return 0;
+}
+
+static void writecache_dtr(struct dm_target *ti)
+{
+	struct dm_writecache *wc = ti->private;
+
+	if (!wc)
+		return;
+
+	if (wc->endio_thread)
+		kthread_stop(wc->endio_thread);
+
+	if (wc->flush_thread)
+		kthread_stop(wc->flush_thread);
+
+	bioset_exit(&wc->bio_set);
+
+	mempool_destroy(wc->copy_pool);
+
+	if (wc->writeback_wq)
+		destroy_workqueue(wc->writeback_wq);
+
+	if (wc->dev)
+		dm_put_device(ti, wc->dev);
+
+	if (wc->ssd_dev)
+		dm_put_device(ti, wc->ssd_dev);
+
+	if (wc->entries)
+		vfree(wc->entries);
+
+	if (wc->memory_map) {
+		if (WC_MODE_PMEM(wc))
+			persistent_memory_release(wc);
+		else
+			vfree(wc->memory_map);
+	}
+
+	if (wc->dm_kcopyd)
+		dm_kcopyd_client_destroy(wc->dm_kcopyd);
+
+	if (wc->dm_io)
+		dm_io_client_destroy(wc->dm_io);
+
+	if (wc->dirty_bitmap)
+		vfree(wc->dirty_bitmap);
+
+	kfree(wc);
+}
+
+static int writecache_ctr(struct dm_target *ti, unsigned argc, char **argv)
+{
+	struct dm_writecache *wc;
+	struct dm_arg_set as;
+	const char *string;
+	unsigned opt_params;
+	size_t offset, data_size;
+	int i, r;
+	char dummy;
+	int high_wm_percent = HIGH_WATERMARK;
+	int low_wm_percent = LOW_WATERMARK;
+	uint64_t x;
+	struct wc_memory_superblock s;
+
+	static struct dm_arg _args[] = {
+		{0, 10, "Invalid number of feature args"},
+	};
+
+	as.argc = argc;
+	as.argv = argv;
+
+	wc = kzalloc(sizeof(struct dm_writecache), GFP_KERNEL);
+	if (!wc) {
+		ti->error = "Cannot allocate writecache structure";
+		r = -ENOMEM;
+		goto bad;
+	}
+	ti->private = wc;
+	wc->ti = ti;
+
+	mutex_init(&wc->lock);
+	writecache_poison_lists(wc);
+	init_swait_queue_head(&wc->freelist_wait);
+	timer_setup(&wc->autocommit_timer, writecache_autocommit_timer, 0);
+
+	for (i = 0; i < 2; i++) {
+		atomic_set(&wc->bio_in_progress[i], 0);
+		init_swait_queue_head(&wc->bio_in_progress_wait[i]);
+	}
+
+	wc->dm_io = dm_io_client_create();
+	if (!wc->dm_io) {
+		r = -ENOMEM;
+		ti->error = "Unable to allocate dm-io client";
+		goto bad;
+	}
+
+	wc->writeback_wq = alloc_workqueue("writecache-writeabck", WQ_MEM_RECLAIM, 1);
+	if (!wc->writeback_wq) {
+		r = -ENOMEM;
+		ti->error = "Could not allocate writeback workqueue";
+		goto bad;
+	}
+	INIT_WORK(&wc->writeback_work, writecache_writeback);
+	INIT_WORK(&wc->flush_work, writecache_flush_work);
+
+	init_swait_queue_head(&wc->endio_thread_wait);
+	INIT_LIST_HEAD(&wc->endio_list);
+	wc->endio_thread = kthread_create(writecache_endio_thread, wc, "writecache_endio");
+	if (IS_ERR(wc->endio_thread)) {
+		r = PTR_ERR(wc->endio_thread);
+		wc->endio_thread = NULL;
+		ti->error = "Couldn't spawn endio thread";
+		goto bad;
+	}
+	wake_up_process(wc->endio_thread);
+
+	/*
+	 * Parse the mode (pmem or ssd)
+	 */
+	string = dm_shift_arg(&as);
+	if (!string)
+		goto bad_arguments;
+
+	if (!strcasecmp(string, "s")) {
+		wc->pmem_mode = false;
+	} else if (!strcasecmp(string, "p")) {
+#if defined(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_DAX_DRIVER)
+		wc->pmem_mode = true;
+		wc->writeback_fua = true;
+#else
+		/*
+		 * If the architecture doesn't support persistent memory or
+		 * the kernel doesn't support any DAX drivers, this driver can
+		 * only be used in SSD-only mode.
+		 */
+		r = -EOPNOTSUPP;
+		ti->error = "Persistent memory or DAX not supported on this system";
+		goto bad;
+#endif
+	} else {
+		goto bad_arguments;
+	}
+
+	if (WC_MODE_PMEM(wc)) {
+		r = bioset_init(&wc->bio_set, BIO_POOL_SIZE,
+					    offsetof(struct writeback_struct, bio),
+					    BIOSET_NEED_BVECS);
+		if (r) {
+			ti->error = "Could not allocate bio set";
+			goto bad;
+		}
+	} else {
+		wc->copy_pool = mempool_create_kmalloc_pool(1, sizeof(struct copy_struct));
+		if (!wc->copy_pool) {
+			r = -ENOMEM;
+			ti->error = "Could not allocate mempool";
+			goto bad;
+		}
+	}
+
+	/*
+	 * Parse the origin data device
+	 */
+	string = dm_shift_arg(&as);
+	if (!string)
+		goto bad_arguments;
+	r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->dev);
+	if (r) {
+		ti->error = "Origin data device lookup failed";
+		goto bad;
+	}
+
+	/*
+	 * Parse cache data device (be it pmem or ssd)
+	 */
+	string = dm_shift_arg(&as);
+	if (!string)
+		goto bad_arguments;
+
+	r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->ssd_dev);
+	if (r) {
+		ti->error = "Cache data device lookup failed";
+		goto bad;
+	}
+	wc->memory_map_size = i_size_read(wc->ssd_dev->bdev->bd_inode);
+
+	if (WC_MODE_PMEM(wc)) {
+		r = persistent_memory_claim(wc);
+		if (r) {
+			ti->error = "Unable to map persistent memory for cache";
+			goto bad;
+		}
+	}
+
+	/*
+	 * Parse the cache block size
+	 */
+	string = dm_shift_arg(&as);
+	if (!string)
+		goto bad_arguments;
+	if (sscanf(string, "%u%c", &wc->block_size, &dummy) != 1 ||
+	    wc->block_size < 512 || wc->block_size > PAGE_SIZE ||
+	    (wc->block_size & (wc->block_size - 1))) {
+		r = -EINVAL;
+		ti->error = "Invalid block size";
+		goto bad;
+	}
+	wc->block_size_bits = __ffs(wc->block_size);
+
+	wc->max_writeback_jobs = MAX_WRITEBACK_JOBS;
+	wc->autocommit_blocks = !WC_MODE_PMEM(wc) ? AUTOCOMMIT_BLOCKS_SSD : AUTOCOMMIT_BLOCKS_PMEM;
+	wc->autocommit_jiffies = msecs_to_jiffies(AUTOCOMMIT_MSEC);
+
+	/*
+	 * Parse optional arguments
+	 */
+	r = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
+	if (r)
+		goto bad;
+
+	while (opt_params) {
+		string = dm_shift_arg(&as), opt_params--;
+		if (!strcasecmp(string, "high_watermark") && opt_params >= 1) {
+			string = dm_shift_arg(&as), opt_params--;
+			if (sscanf(string, "%d%c", &high_wm_percent, &dummy) != 1)
+				goto invalid_optional;
+			if (high_wm_percent < 0 || high_wm_percent > 100)
+				goto invalid_optional;
+			wc->high_wm_percent_set = true;
+		} else if (!strcasecmp(string, "low_watermark") && opt_params >= 1) {
+			string = dm_shift_arg(&as), opt_params--;
+			if (sscanf(string, "%d%c", &low_wm_percent, &dummy) != 1)
+				goto invalid_optional;
+			if (low_wm_percent < 0 || low_wm_percent > 100)
+				goto invalid_optional;
+			wc->low_wm_percent_set = true;
+		} else if (!strcasecmp(string, "writeback_jobs") && opt_params >= 1) {
+			string = dm_shift_arg(&as), opt_params--;
+			if (sscanf(string, "%u%c", &wc->max_writeback_jobs, &dummy) != 1)
+				goto invalid_optional;
+			wc->max_writeback_jobs_set = true;
+		} else if (!strcasecmp(string, "autocommit_blocks") && opt_params >= 1) {
+			string = dm_shift_arg(&as), opt_params--;
+			if (sscanf(string, "%u%c", &wc->autocommit_blocks, &dummy) != 1)
+				goto invalid_optional;
+			wc->autocommit_blocks_set = true;
+		} else if (!strcasecmp(string, "autocommit_time") && opt_params >= 1) {
+			unsigned autocommit_msecs;
+			string = dm_shift_arg(&as), opt_params--;
+			if (sscanf(string, "%u%c", &autocommit_msecs, &dummy) != 1)
+				goto invalid_optional;
+			if (autocommit_msecs > 3600000)
+				goto invalid_optional;
+			wc->autocommit_jiffies = jiffies_to_msecs(autocommit_msecs);
+			wc->autocommit_time_set = true;
+		} else if (!strcasecmp(string, "fua")) {
+			if (WC_MODE_PMEM(wc)) {
+				wc->writeback_fua = true;
+				wc->writeback_fua_set = true;
+			} else goto invalid_optional;
+		} else if (!strcasecmp(string, "nofua")) {
+			if (WC_MODE_PMEM(wc)) {
+				wc->writeback_fua = false;
+				wc->writeback_fua_set = true;
+			} else goto invalid_optional;
+		} else {
+invalid_optional:
+			r = -EINVAL;
+			ti->error = "Invalid optional argument";
+			goto bad;
+		}
+	}
+
+	if (!WC_MODE_PMEM(wc)) {
+		struct dm_io_region region;
+		struct dm_io_request req;
+		size_t n_blocks, n_metadata_blocks;
+		uint64_t n_bitmap_bits;
+
+		init_completion(&wc->flush_completion);
+		wc->flush_thread = kthread_create(writecache_flush_thread, wc, "dm_writecache_flush");
+		if (IS_ERR(wc->flush_thread)) {
+			r = PTR_ERR(wc->flush_thread);
+			wc->flush_thread = NULL;
+			ti->error = "Couldn't spawn endio thread";
+			goto bad;
+		}
+		writecache_offload_bio(wc, NULL);
+
+		r = calculate_memory_size(wc->memory_map_size, wc->block_size,
+					  &n_blocks, &n_metadata_blocks);
+		if (r) {
+			ti->error = "Invalid device size";
+			goto bad;
+		}
+
+		n_bitmap_bits = (((uint64_t)n_metadata_blocks << wc->block_size_bits) +
+				 BITMAP_GRANULARITY - 1) / BITMAP_GRANULARITY;
+		/* this is limitation of test_bit functions */
+		if (n_bitmap_bits > 1U << 31) {
+			r = -EFBIG;
+			ti->error = "Invalid device size";
+			goto bad;
+		}
+
+		wc->memory_map = vmalloc(n_metadata_blocks << wc->block_size_bits);
+		if (!wc->memory_map) {
+			r = -ENOMEM;
+			ti->error = "Unable to allocate memory for metadata";
+			goto bad;
+		}
+
+		wc->dm_kcopyd = dm_kcopyd_client_create(&dm_kcopyd_throttle);
+		if (!wc->dm_kcopyd) {
+			r = -ENOMEM;
+			ti->error = "Unable to allocate dm-kcopyd client";
+			goto bad;
+		}
+
+		wc->metadata_sectors = n_metadata_blocks << (wc->block_size_bits - SECTOR_SHIFT);
+		wc->dirty_bitmap_size = (n_bitmap_bits + BITS_PER_LONG - 1) /
+			BITS_PER_LONG * sizeof(unsigned long);
+		wc->dirty_bitmap = vzalloc(wc->dirty_bitmap_size);
+		if (!wc->dirty_bitmap) {
+			r = -ENOMEM;
+			ti->error = "Unable to allocate dirty bitmap";
+			goto bad;
+		}
+
+		region.bdev = wc->ssd_dev->bdev;
+		region.sector = 0;
+		region.count = wc->metadata_sectors;
+		req.bi_op = REQ_OP_READ;
+		req.bi_op_flags = REQ_SYNC;
+		req.mem.type = DM_IO_VMA;
+		req.mem.ptr.vma = (char *)wc->memory_map;
+		req.client = wc->dm_io;
+		req.notify.fn = NULL;
+
+		r = dm_io(&req, 1, &region, NULL);
+		if (r) {
+			ti->error = "Unable to read metadata";
+			goto bad;
+		}
+	}
+
+	r = memcpy_mcsafe(&s, sb(wc), sizeof(struct wc_memory_superblock));
+	if (r) {
+		ti->error = "Hardware memory error when reading superblock";
+		goto bad;
+	}
+	if (!le32_to_cpu(s.magic) && !le32_to_cpu(s.version)) {
+		r = init_memory(wc);
+		if (r) {
+			ti->error = "Unable to initialize device";
+			goto bad;
+		}
+		r = memcpy_mcsafe(&s, sb(wc), sizeof(struct wc_memory_superblock));
+		if (r) {
+			ti->error = "Hardware memory error when reading superblock";
+			goto bad;
+		}
+	}
+
+	if (le32_to_cpu(s.magic) != MEMORY_SUPERBLOCK_MAGIC) {
+		ti->error = "Invalid magic in the superblock";
+		r = -EINVAL;
+		goto bad;
+	}
+
+	if (le32_to_cpu(s.version) != MEMORY_SUPERBLOCK_VERSION) {
+		ti->error = "Invalid version in the superblock";
+		r = -EINVAL;
+		goto bad;
+	}
+
+	if (le32_to_cpu(s.block_size) != wc->block_size) {
+		ti->error = "Block size does not match superblock";
+		r = -EINVAL;
+		goto bad;
+	}
+
+	wc->n_blocks = le64_to_cpu(s.n_blocks);
+
+	offset = wc->n_blocks * sizeof(struct wc_memory_entry);
+	if (offset / sizeof(struct wc_memory_entry) != le64_to_cpu(sb(wc)->n_blocks)) {
+overflow:
+		ti->error = "Overflow in size calculation";
+		r = -EINVAL;
+		goto bad;
+	}
+	offset += sizeof(struct wc_memory_superblock);
+	if (offset < sizeof(struct wc_memory_superblock))
+		goto overflow;
+	offset = (offset + wc->block_size - 1) & ~(size_t)(wc->block_size - 1);
+	data_size = wc->n_blocks * (size_t)wc->block_size;
+	if (!offset || (data_size / wc->block_size != wc->n_blocks) ||
+	    (offset + data_size < offset))
+		goto overflow;
+	if (offset + data_size > wc->memory_map_size) {
+		ti->error = "Memory area is too small";
+		r = -EINVAL;
+		goto bad;
+	}
+
+	wc->metadata_sectors = offset >> SECTOR_SHIFT;
+	wc->block_start = (char *)sb(wc) + offset;
+
+	x = (uint64_t)wc->n_blocks * (100 - high_wm_percent);
+	x += 50;
+	do_div(x, 100);
+	wc->freelist_high_watermark = x;
+	x = (uint64_t)wc->n_blocks * (100 - low_wm_percent);
+	x += 50;
+	do_div(x, 100);
+	wc->freelist_low_watermark = x;
+
+	r = writecache_alloc_entries(wc);
+	if (r) {
+		ti->error = "Cannot allocate memory";
+		goto bad;
+	}
+
+	ti->num_flush_bios = 1;
+	ti->flush_supported = true;
+	ti->num_discard_bios = 1;
+
+	if (WC_MODE_PMEM(wc))
+		persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
+
+	return 0;
+
+bad_arguments:
+	r = -EINVAL;
+	ti->error = "Bad arguments";
+bad:
+	writecache_dtr(ti);
+	return r;
+}
+
+static void writecache_status(struct dm_target *ti, status_type_t type,
+			      unsigned status_flags, char *result, unsigned maxlen)
+{
+	struct dm_writecache *wc = ti->private;
+	unsigned extra_args;
+	unsigned sz = 0;
+	uint64_t x;
+
+	switch (type) {
+	case STATUSTYPE_INFO:
+		DMEMIT("%ld %llu %llu %llu", (long)writecache_has_error(wc),
+		       (unsigned long long)wc->n_blocks, (unsigned long long)wc->freelist_size,
+		       (unsigned long long)wc->writeback_size);
+		break;
+	case STATUSTYPE_TABLE:
+		DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's',
+				wc->dev->name, wc->ssd_dev->name, wc->block_size);
+		extra_args = 0;
+		if (wc->high_wm_percent_set)
+			extra_args += 2;
+		if (wc->low_wm_percent_set)
+			extra_args += 2;
+		if (wc->max_writeback_jobs_set)
+			extra_args += 2;
+		if (wc->autocommit_blocks_set)
+			extra_args += 2;
+		if (wc->autocommit_time_set)
+			extra_args += 2;
+		if (wc->writeback_fua_set)
+			extra_args++;
+
+		DMEMIT("%u", extra_args);
+		if (wc->high_wm_percent_set) {
+			x = (uint64_t)wc->freelist_high_watermark * 100;
+			x += wc->n_blocks / 2;
+			do_div(x, (size_t)wc->n_blocks);
+			DMEMIT(" high_watermark %u", 100 - (unsigned)x);
+		}
+		if (wc->low_wm_percent_set) {
+			x = (uint64_t)wc->freelist_low_watermark * 100;
+			x += wc->n_blocks / 2;
+			do_div(x, (size_t)wc->n_blocks);
+			DMEMIT(" low_watermark %u", 100 - (unsigned)x);
+		}
+		if (wc->max_writeback_jobs_set) {
+			DMEMIT(" writeback_jobs %u", wc->max_writeback_jobs);
+		}
+		if (wc->autocommit_blocks_set) {
+			DMEMIT(" autocommit_blocks %u", wc->autocommit_blocks);
+		}
+		if (wc->autocommit_time_set) {
+			DMEMIT(" autocommit_time %u", jiffies_to_msecs(wc->autocommit_jiffies));
+		}
+		if (wc->writeback_fua_set) {
+			DMEMIT(" %sfua", wc->writeback_fua ? "" : "no");
+		}
+		break;
+	}
+}
+
+static struct target_type writecache_target = {
+	.name			= "writecache",
+	.version		= {1, 0, 0},
+	.module			= THIS_MODULE,
+	.ctr			= writecache_ctr,
+	.dtr			= writecache_dtr,
+	.status			= writecache_status,
+	.postsuspend		= writecache_suspend,
+	.resume			= writecache_resume,
+	.message		= writecache_message,
+	.map			= writecache_map,
+	.end_io			= writecache_end_io,
+	.iterate_devices	= writecache_iterate_devices,
+	.io_hints		= writecache_io_hints,
+};
+
+static int __init dm_writecache_init(void)
+{
+	int r;
+
+	r = dm_register_target(&writecache_target);
+	if (r < 0) {
+		DMERR("register failed %d", r);
+		return r;
+	}
+
+	return 0;
+}
+
+static void __exit dm_writecache_exit(void)
+{
+	dm_unregister_target(&writecache_target);
+}
+
+module_init(dm_writecache_init);
+module_exit(dm_writecache_exit);
+
+MODULE_DESCRIPTION(DM_NAME " writecache target");
+MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-zoned-target.c b/drivers/md/dm-zoned-target.c
index e73b0776683c..30602d15ad9a 100644
--- a/drivers/md/dm-zoned-target.c
+++ b/drivers/md/dm-zoned-target.c
@@ -57,7 +57,7 @@ struct dmz_target {
 	struct workqueue_struct *chunk_wq;
 
 	/* For cloned BIOs to zones */
-	struct bio_set		*bio_set;
+	struct bio_set		bio_set;
 
 	/* For flush */
 	spinlock_t		flush_lock;
@@ -121,7 +121,7 @@ static int dmz_submit_read_bio(struct dmz_target *dmz, struct dm_zone *zone,
 	}
 
 	/* Partial BIO: we need to clone the BIO */
-	clone = bio_clone_fast(bio, GFP_NOIO, dmz->bio_set);
+	clone = bio_clone_fast(bio, GFP_NOIO, &dmz->bio_set);
 	if (!clone)
 		return -ENOMEM;
 
@@ -779,10 +779,9 @@ static int dmz_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	ti->len = (sector_t)dmz_nr_chunks(dmz->metadata) << dev->zone_nr_sectors_shift;
 
 	/* Zone BIO */
-	dmz->bio_set = bioset_create(DMZ_MIN_BIOS, 0, 0);
-	if (!dmz->bio_set) {
+	ret = bioset_init(&dmz->bio_set, DMZ_MIN_BIOS, 0, 0);
+	if (ret) {
 		ti->error = "Create BIO set failed";
-		ret = -ENOMEM;
 		goto err_meta;
 	}
 
@@ -828,7 +827,7 @@ static int dmz_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	destroy_workqueue(dmz->chunk_wq);
 err_bio:
 	mutex_destroy(&dmz->chunk_lock);
-	bioset_free(dmz->bio_set);
+	bioset_exit(&dmz->bio_set);
 err_meta:
 	dmz_dtr_metadata(dmz->metadata);
 err_dev:
@@ -858,7 +857,7 @@ static void dmz_dtr(struct dm_target *ti)
 
 	dmz_dtr_metadata(dmz->metadata);
 
-	bioset_free(dmz->bio_set);
+	bioset_exit(&dmz->bio_set);
 
 	dmz_put_zoned_device(ti);
 
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 0a7b0107ca78..68fad4b544b1 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -148,8 +148,8 @@ static int dm_numa_node = DM_NUMA_NODE;
  * For mempools pre-allocation at the table loading time.
  */
 struct dm_md_mempools {
-	struct bio_set *bs;
-	struct bio_set *io_bs;
+	struct bio_set bs;
+	struct bio_set io_bs;
 };
 
 struct table_device {
@@ -537,7 +537,7 @@ static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
 	struct dm_target_io *tio;
 	struct bio *clone;
 
-	clone = bio_alloc_bioset(GFP_NOIO, 0, md->io_bs);
+	clone = bio_alloc_bioset(GFP_NOIO, 0, &md->io_bs);
 	if (!clone)
 		return NULL;
 
@@ -572,7 +572,7 @@ static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *t
 		/* the dm_target_io embedded in ci->io is available */
 		tio = &ci->io->tio;
 	} else {
-		struct bio *clone = bio_alloc_bioset(gfp_mask, 0, ci->io->md->bs);
+		struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs);
 		if (!clone)
 			return NULL;
 
@@ -1089,6 +1089,30 @@ static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
 	return ret;
 }
 
+static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,
+		void *addr, size_t bytes, struct iov_iter *i)
+{
+	struct mapped_device *md = dax_get_private(dax_dev);
+	sector_t sector = pgoff * PAGE_SECTORS;
+	struct dm_target *ti;
+	long ret = 0;
+	int srcu_idx;
+
+	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
+
+	if (!ti)
+		goto out;
+	if (!ti->type->dax_copy_to_iter) {
+		ret = copy_to_iter(addr, bytes, i);
+		goto out;
+	}
+	ret = ti->type->dax_copy_to_iter(ti, pgoff, addr, bytes, i);
+ out:
+	dm_put_live_table(md, srcu_idx);
+
+	return ret;
+}
+
 /*
  * A target may call dm_accept_partial_bio only from the map routine.  It is
  * allowed for all bio types except REQ_PREFLUSH and REQ_OP_ZONE_RESET.
@@ -1583,7 +1607,7 @@ static blk_qc_t __split_and_process_bio(struct mapped_device *md,
 				 * won't be affected by this reassignment.
 				 */
 				struct bio *b = bio_clone_bioset(bio, GFP_NOIO,
-								 md->queue->bio_split);
+								 &md->queue->bio_split);
 				ci.io->orig_bio = b;
 				bio_advance(bio, (bio_sectors(bio) - ci.sector_count) << 9);
 				bio_chain(b, bio);
@@ -1785,10 +1809,8 @@ static void cleanup_mapped_device(struct mapped_device *md)
 		destroy_workqueue(md->wq);
 	if (md->kworker_task)
 		kthread_stop(md->kworker_task);
-	if (md->bs)
-		bioset_free(md->bs);
-	if (md->io_bs)
-		bioset_free(md->io_bs);
+	bioset_exit(&md->bs);
+	bioset_exit(&md->io_bs);
 
 	if (md->dax_dev) {
 		kill_dax(md->dax_dev);
@@ -1965,16 +1987,10 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
 		 * If so, reload bioset because front_pad may have changed
 		 * because a different table was loaded.
 		 */
-		if (md->bs) {
-			bioset_free(md->bs);
-			md->bs = NULL;
-		}
-		if (md->io_bs) {
-			bioset_free(md->io_bs);
-			md->io_bs = NULL;
-		}
+		bioset_exit(&md->bs);
+		bioset_exit(&md->io_bs);
 
-	} else if (md->bs) {
+	} else if (bioset_initialized(&md->bs)) {
 		/*
 		 * There's no need to reload with request-based dm
 		 * because the size of front_pad doesn't change.
@@ -1986,12 +2002,14 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
 		goto out;
 	}
 
-	BUG_ON(!p || md->bs || md->io_bs);
+	BUG_ON(!p ||
+	       bioset_initialized(&md->bs) ||
+	       bioset_initialized(&md->io_bs));
 
 	md->bs = p->bs;
-	p->bs = NULL;
+	memset(&p->bs, 0, sizeof(p->bs));
 	md->io_bs = p->io_bs;
-	p->io_bs = NULL;
+	memset(&p->io_bs, 0, sizeof(p->io_bs));
 out:
 	/* mempool bind completed, no longer need any mempools in the table */
 	dm_table_free_md_mempools(t);
@@ -2905,6 +2923,7 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_qu
 	struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
 	unsigned int pool_size = 0;
 	unsigned int front_pad, io_front_pad;
+	int ret;
 
 	if (!pools)
 		return NULL;
@@ -2916,10 +2935,10 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_qu
 		pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
 		front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
 		io_front_pad = roundup(front_pad,  __alignof__(struct dm_io)) + offsetof(struct dm_io, tio);
-		pools->io_bs = bioset_create(pool_size, io_front_pad, 0);
-		if (!pools->io_bs)
+		ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0);
+		if (ret)
 			goto out;
-		if (integrity && bioset_integrity_create(pools->io_bs, pool_size))
+		if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))
 			goto out;
 		break;
 	case DM_TYPE_REQUEST_BASED:
@@ -2932,11 +2951,11 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_qu
 		BUG();
 	}
 
-	pools->bs = bioset_create(pool_size, front_pad, 0);
-	if (!pools->bs)
+	ret = bioset_init(&pools->bs, pool_size, front_pad, 0);
+	if (ret)
 		goto out;
 
-	if (integrity && bioset_integrity_create(pools->bs, pool_size))
+	if (integrity && bioset_integrity_create(&pools->bs, pool_size))
 		goto out;
 
 	return pools;
@@ -2952,10 +2971,8 @@ void dm_free_md_mempools(struct dm_md_mempools *pools)
 	if (!pools)
 		return;
 
-	if (pools->bs)
-		bioset_free(pools->bs);
-	if (pools->io_bs)
-		bioset_free(pools->io_bs);
+	bioset_exit(&pools->bs);
+	bioset_exit(&pools->io_bs);
 
 	kfree(pools);
 }
@@ -3134,6 +3151,7 @@ static const struct block_device_operations dm_blk_dops = {
 static const struct dax_operations dm_dax_ops = {
 	.direct_access = dm_dax_direct_access,
 	.copy_from_iter = dm_dax_copy_from_iter,
+	.copy_to_iter = dm_dax_copy_to_iter,
 };
 
 /*
diff --git a/drivers/md/md-faulty.c b/drivers/md/md-faulty.c
index 38264b38420f..c2fdf899de14 100644
--- a/drivers/md/md-faulty.c
+++ b/drivers/md/md-faulty.c
@@ -214,7 +214,7 @@ static bool faulty_make_request(struct mddev *mddev, struct bio *bio)
 		}
 	}
 	if (failit) {
-		struct bio *b = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
+		struct bio *b = bio_clone_fast(bio, GFP_NOIO, &mddev->bio_set);
 
 		bio_set_dev(b, conf->rdev->bdev);
 		b->bi_private = bio;
diff --git a/drivers/md/md-linear.c b/drivers/md/md-linear.c
index 4964323d936b..d45c697c0ebe 100644
--- a/drivers/md/md-linear.c
+++ b/drivers/md/md-linear.c
@@ -269,7 +269,7 @@ static bool linear_make_request(struct mddev *mddev, struct bio *bio)
 	if (unlikely(bio_end_sector(bio) > end_sector)) {
 		/* This bio crosses a device boundary, so we have to split it */
 		struct bio *split = bio_split(bio, end_sector - bio_sector,
-					      GFP_NOIO, mddev->bio_set);
+					      GFP_NOIO, &mddev->bio_set);
 		bio_chain(split, bio);
 		generic_make_request(bio);
 		bio = split;
diff --git a/drivers/md/md-multipath.c b/drivers/md/md-multipath.c
index 0a7e99d62c69..f71fcdb9b39c 100644
--- a/drivers/md/md-multipath.c
+++ b/drivers/md/md-multipath.c
@@ -80,7 +80,7 @@ static void multipath_end_bh_io(struct multipath_bh *mp_bh, blk_status_t status)
 
 	bio->bi_status = status;
 	bio_endio(bio);
-	mempool_free(mp_bh, conf->pool);
+	mempool_free(mp_bh, &conf->pool);
 }
 
 static void multipath_end_request(struct bio *bio)
@@ -117,7 +117,7 @@ static bool multipath_make_request(struct mddev *mddev, struct bio * bio)
 		return true;
 	}
 
-	mp_bh = mempool_alloc(conf->pool, GFP_NOIO);
+	mp_bh = mempool_alloc(&conf->pool, GFP_NOIO);
 
 	mp_bh->master_bio = bio;
 	mp_bh->mddev = mddev;
@@ -125,7 +125,7 @@ static bool multipath_make_request(struct mddev *mddev, struct bio * bio)
 	mp_bh->path = multipath_map(conf);
 	if (mp_bh->path < 0) {
 		bio_io_error(bio);
-		mempool_free(mp_bh, conf->pool);
+		mempool_free(mp_bh, &conf->pool);
 		return true;
 	}
 	multipath = conf->multipaths + mp_bh->path;
@@ -378,6 +378,7 @@ static int multipath_run (struct mddev *mddev)
 	struct multipath_info *disk;
 	struct md_rdev *rdev;
 	int working_disks;
+	int ret;
 
 	if (md_check_no_bitmap(mddev))
 		return -EINVAL;
@@ -431,9 +432,9 @@ static int multipath_run (struct mddev *mddev)
 	}
 	mddev->degraded = conf->raid_disks - working_disks;
 
-	conf->pool = mempool_create_kmalloc_pool(NR_RESERVED_BUFS,
-						 sizeof(struct multipath_bh));
-	if (conf->pool == NULL)
+	ret = mempool_init_kmalloc_pool(&conf->pool, NR_RESERVED_BUFS,
+					sizeof(struct multipath_bh));
+	if (ret)
 		goto out_free_conf;
 
 	mddev->thread = md_register_thread(multipathd, mddev,
@@ -455,7 +456,7 @@ static int multipath_run (struct mddev *mddev)
 	return 0;
 
 out_free_conf:
-	mempool_destroy(conf->pool);
+	mempool_exit(&conf->pool);
 	kfree(conf->multipaths);
 	kfree(conf);
 	mddev->private = NULL;
@@ -467,7 +468,7 @@ static void multipath_free(struct mddev *mddev, void *priv)
 {
 	struct mpconf *conf = priv;
 
-	mempool_destroy(conf->pool);
+	mempool_exit(&conf->pool);
 	kfree(conf->multipaths);
 	kfree(conf);
 }
diff --git a/drivers/md/md-multipath.h b/drivers/md/md-multipath.h
index 0adb941f485a..b3099e5fc4d7 100644
--- a/drivers/md/md-multipath.h
+++ b/drivers/md/md-multipath.h
@@ -13,7 +13,7 @@ struct mpconf {
 	spinlock_t		device_lock;
 	struct list_head	retry_list;
 
-	mempool_t		*pool;
+	mempool_t		pool;
 };
 
 /*
diff --git a/drivers/md/md.c b/drivers/md/md.c
index c208c01f63a5..411f60d591d1 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -130,6 +130,24 @@ static inline int speed_max(struct mddev *mddev)
 		mddev->sync_speed_max : sysctl_speed_limit_max;
 }
 
+static void * flush_info_alloc(gfp_t gfp_flags, void *data)
+{
+        return kzalloc(sizeof(struct flush_info), gfp_flags);
+}
+static void flush_info_free(void *flush_info, void *data)
+{
+        kfree(flush_info);
+}
+
+static void * flush_bio_alloc(gfp_t gfp_flags, void *data)
+{
+	return kzalloc(sizeof(struct flush_bio), gfp_flags);
+}
+static void flush_bio_free(void *flush_bio, void *data)
+{
+	kfree(flush_bio);
+}
+
 static struct ctl_table_header *raid_table_header;
 
 static struct ctl_table raid_table[] = {
@@ -193,10 +211,10 @@ struct bio *bio_alloc_mddev(gfp_t gfp_mask, int nr_iovecs,
 {
 	struct bio *b;
 
-	if (!mddev || !mddev->bio_set)
+	if (!mddev || !bioset_initialized(&mddev->bio_set))
 		return bio_alloc(gfp_mask, nr_iovecs);
 
-	b = bio_alloc_bioset(gfp_mask, nr_iovecs, mddev->bio_set);
+	b = bio_alloc_bioset(gfp_mask, nr_iovecs, &mddev->bio_set);
 	if (!b)
 		return NULL;
 	return b;
@@ -205,10 +223,10 @@ EXPORT_SYMBOL_GPL(bio_alloc_mddev);
 
 static struct bio *md_bio_alloc_sync(struct mddev *mddev)
 {
-	if (!mddev || !mddev->sync_set)
+	if (!mddev || !bioset_initialized(&mddev->sync_set))
 		return bio_alloc(GFP_NOIO, 1);
 
-	return bio_alloc_bioset(GFP_NOIO, 1, mddev->sync_set);
+	return bio_alloc_bioset(GFP_NOIO, 1, &mddev->sync_set);
 }
 
 /*
@@ -412,30 +430,53 @@ static int md_congested(void *data, int bits)
 /*
  * Generic flush handling for md
  */
+static void submit_flushes(struct work_struct *ws)
+{
+	struct flush_info *fi = container_of(ws, struct flush_info, flush_work);
+	struct mddev *mddev = fi->mddev;
+	struct bio *bio = fi->bio;
+
+	bio->bi_opf &= ~REQ_PREFLUSH;
+	md_handle_request(mddev, bio);
+
+	mempool_free(fi, mddev->flush_pool);
+}
 
-static void md_end_flush(struct bio *bio)
+static void md_end_flush(struct bio *fbio)
 {
-	struct md_rdev *rdev = bio->bi_private;
-	struct mddev *mddev = rdev->mddev;
+	struct flush_bio *fb = fbio->bi_private;
+	struct md_rdev *rdev = fb->rdev;
+	struct flush_info *fi = fb->fi;
+	struct bio *bio = fi->bio;
+	struct mddev *mddev = fi->mddev;
 
 	rdev_dec_pending(rdev, mddev);
 
-	if (atomic_dec_and_test(&mddev->flush_pending)) {
-		/* The pre-request flush has finished */
-		queue_work(md_wq, &mddev->flush_work);
+	if (atomic_dec_and_test(&fi->flush_pending)) {
+		if (bio->bi_iter.bi_size == 0)
+			/* an empty barrier - all done */
+			bio_endio(bio);
+		else {
+			INIT_WORK(&fi->flush_work, submit_flushes);
+			queue_work(md_wq, &fi->flush_work);
+		}
 	}
-	bio_put(bio);
-}
 
-static void md_submit_flush_data(struct work_struct *ws);
+	mempool_free(fb, mddev->flush_bio_pool);
+	bio_put(fbio);
+}
 
-static void submit_flushes(struct work_struct *ws)
+void md_flush_request(struct mddev *mddev, struct bio *bio)
 {
-	struct mddev *mddev = container_of(ws, struct mddev, flush_work);
 	struct md_rdev *rdev;
+	struct flush_info *fi;
+
+	fi = mempool_alloc(mddev->flush_pool, GFP_NOIO);
+
+	fi->bio = bio;
+	fi->mddev = mddev;
+	atomic_set(&fi->flush_pending, 1);
 
-	INIT_WORK(&mddev->flush_work, md_submit_flush_data);
-	atomic_set(&mddev->flush_pending, 1);
 	rcu_read_lock();
 	rdev_for_each_rcu(rdev, mddev)
 		if (rdev->raid_disk >= 0 &&
@@ -445,59 +486,39 @@ static void submit_flushes(struct work_struct *ws)
 			 * we reclaim rcu_read_lock
 			 */
 			struct bio *bi;
+			struct flush_bio *fb;
 			atomic_inc(&rdev->nr_pending);
 			atomic_inc(&rdev->nr_pending);
 			rcu_read_unlock();
+
+			fb = mempool_alloc(mddev->flush_bio_pool, GFP_NOIO);
+			fb->fi = fi;
+			fb->rdev = rdev;
+
 			bi = bio_alloc_mddev(GFP_NOIO, 0, mddev);
-			bi->bi_end_io = md_end_flush;
-			bi->bi_private = rdev;
 			bio_set_dev(bi, rdev->bdev);
+			bi->bi_end_io = md_end_flush;
+			bi->bi_private = fb;
 			bi->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
-			atomic_inc(&mddev->flush_pending);
+
+			atomic_inc(&fi->flush_pending);
 			submit_bio(bi);
+
 			rcu_read_lock();
 			rdev_dec_pending(rdev, mddev);
 		}
 	rcu_read_unlock();
-	if (atomic_dec_and_test(&mddev->flush_pending))
-		queue_work(md_wq, &mddev->flush_work);
-}
-
-static void md_submit_flush_data(struct work_struct *ws)
-{
-	struct mddev *mddev = container_of(ws, struct mddev, flush_work);
-	struct bio *bio = mddev->flush_bio;
-
-	/*
-	 * must reset flush_bio before calling into md_handle_request to avoid a
-	 * deadlock, because other bios passed md_handle_request suspend check
-	 * could wait for this and below md_handle_request could wait for those
-	 * bios because of suspend check
-	 */
-	mddev->flush_bio = NULL;
-	wake_up(&mddev->sb_wait);
 
-	if (bio->bi_iter.bi_size == 0)
-		/* an empty barrier - all done */
-		bio_endio(bio);
-	else {
-		bio->bi_opf &= ~REQ_PREFLUSH;
-		md_handle_request(mddev, bio);
+	if (atomic_dec_and_test(&fi->flush_pending)) {
+		if (bio->bi_iter.bi_size == 0)
+			/* an empty barrier - all done */
+			bio_endio(bio);
+		else {
+			INIT_WORK(&fi->flush_work, submit_flushes);
+			queue_work(md_wq, &fi->flush_work);
+		}
 	}
 }
-
-void md_flush_request(struct mddev *mddev, struct bio *bio)
-{
-	spin_lock_irq(&mddev->lock);
-	wait_event_lock_irq(mddev->sb_wait,
-			    !mddev->flush_bio,
-			    mddev->lock);
-	mddev->flush_bio = bio;
-	spin_unlock_irq(&mddev->lock);
-
-	INIT_WORK(&mddev->flush_work, submit_flushes);
-	queue_work(md_wq, &mddev->flush_work);
-}
 EXPORT_SYMBOL(md_flush_request);
 
 static inline struct mddev *mddev_get(struct mddev *mddev)
@@ -510,7 +531,10 @@ static void mddev_delayed_delete(struct work_struct *ws);
 
 static void mddev_put(struct mddev *mddev)
 {
-	struct bio_set *bs = NULL, *sync_bs = NULL;
+	struct bio_set bs, sync_bs;
+
+	memset(&bs, 0, sizeof(bs));
+	memset(&sync_bs, 0, sizeof(sync_bs));
 
 	if (!atomic_dec_and_lock(&mddev->active, &all_mddevs_lock))
 		return;
@@ -521,8 +545,8 @@ static void mddev_put(struct mddev *mddev)
 		list_del_init(&mddev->all_mddevs);
 		bs = mddev->bio_set;
 		sync_bs = mddev->sync_set;
-		mddev->bio_set = NULL;
-		mddev->sync_set = NULL;
+		memset(&mddev->bio_set, 0, sizeof(mddev->bio_set));
+		memset(&mddev->sync_set, 0, sizeof(mddev->sync_set));
 		if (mddev->gendisk) {
 			/* We did a probe so need to clean up.  Call
 			 * queue_work inside the spinlock so that
@@ -535,10 +559,8 @@ static void mddev_put(struct mddev *mddev)
 			kfree(mddev);
 	}
 	spin_unlock(&all_mddevs_lock);
-	if (bs)
-		bioset_free(bs);
-	if (sync_bs)
-		bioset_free(sync_bs);
+	bioset_exit(&bs);
+	bioset_exit(&sync_bs);
 }
 
 static void md_safemode_timeout(struct timer_list *t);
@@ -555,7 +577,6 @@ void mddev_init(struct mddev *mddev)
 	atomic_set(&mddev->openers, 0);
 	atomic_set(&mddev->active_io, 0);
 	spin_lock_init(&mddev->lock);
-	atomic_set(&mddev->flush_pending, 0);
 	init_waitqueue_head(&mddev->sb_wait);
 	init_waitqueue_head(&mddev->recovery_wait);
 	mddev->reshape_position = MaxSector;
@@ -2123,7 +2144,7 @@ int md_integrity_register(struct mddev *mddev)
 			       bdev_get_integrity(reference->bdev));
 
 	pr_debug("md: data integrity enabled on %s\n", mdname(mddev));
-	if (bioset_integrity_create(mddev->bio_set, BIO_POOL_SIZE)) {
+	if (bioset_integrity_create(&mddev->bio_set, BIO_POOL_SIZE)) {
 		pr_err("md: failed to create integrity pool for %s\n",
 		       mdname(mddev));
 		return -EINVAL;
@@ -2853,7 +2874,8 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
 			err = 0;
 		}
 	} else if (cmd_match(buf, "re-add")) {
-		if (test_bit(Faulty, &rdev->flags) && (rdev->raid_disk == -1)) {
+		if (test_bit(Faulty, &rdev->flags) && (rdev->raid_disk == -1) &&
+			rdev->saved_raid_disk >= 0) {
 			/* clear_bit is performed _after_ all the devices
 			 * have their local Faulty bit cleared. If any writes
 			 * happen in the meantime in the local node, they
@@ -5497,14 +5519,28 @@ int md_run(struct mddev *mddev)
 		sysfs_notify_dirent_safe(rdev->sysfs_state);
 	}
 
-	if (mddev->bio_set == NULL) {
-		mddev->bio_set = bioset_create(BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
-		if (!mddev->bio_set)
-			return -ENOMEM;
+	if (!bioset_initialized(&mddev->bio_set)) {
+		err = bioset_init(&mddev->bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
+		if (err)
+			return err;
+	}
+	if (!bioset_initialized(&mddev->sync_set)) {
+		err = bioset_init(&mddev->sync_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
+		if (err)
+			goto abort;
 	}
-	if (mddev->sync_set == NULL) {
-		mddev->sync_set = bioset_create(BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
-		if (!mddev->sync_set) {
+	if (mddev->flush_pool == NULL) {
+		mddev->flush_pool = mempool_create(NR_FLUSH_INFOS, flush_info_alloc,
+						flush_info_free, mddev);
+		if (!mddev->flush_pool) {
+			err = -ENOMEM;
+			goto abort;
+		}
+	}
+	if (mddev->flush_bio_pool == NULL) {
+		mddev->flush_bio_pool = mempool_create(NR_FLUSH_BIOS, flush_bio_alloc,
+						flush_bio_free, mddev);
+		if (!mddev->flush_bio_pool) {
 			err = -ENOMEM;
 			goto abort;
 		}
@@ -5668,14 +5704,16 @@ int md_run(struct mddev *mddev)
 	return 0;
 
 abort:
-	if (mddev->bio_set) {
-		bioset_free(mddev->bio_set);
-		mddev->bio_set = NULL;
+	if (mddev->flush_bio_pool) {
+		mempool_destroy(mddev->flush_bio_pool);
+		mddev->flush_bio_pool = NULL;
 	}
-	if (mddev->sync_set) {
-		bioset_free(mddev->sync_set);
-		mddev->sync_set = NULL;
+	if (mddev->flush_pool){
+		mempool_destroy(mddev->flush_pool);
+		mddev->flush_pool = NULL;
 	}
+	bioset_exit(&mddev->bio_set);
+	bioset_exit(&mddev->sync_set);
 
 	return err;
 }
@@ -5888,14 +5926,16 @@ void md_stop(struct mddev *mddev)
 	 * This is called from dm-raid
 	 */
 	__md_stop(mddev);
-	if (mddev->bio_set) {
-		bioset_free(mddev->bio_set);
-		mddev->bio_set = NULL;
+	if (mddev->flush_bio_pool) {
+		mempool_destroy(mddev->flush_bio_pool);
+		mddev->flush_bio_pool = NULL;
 	}
-	if (mddev->sync_set) {
-		bioset_free(mddev->sync_set);
-		mddev->sync_set = NULL;
+	if (mddev->flush_pool) {
+		mempool_destroy(mddev->flush_pool);
+		mddev->flush_pool = NULL;
 	}
+	bioset_exit(&mddev->bio_set);
+	bioset_exit(&mddev->sync_set);
 }
 
 EXPORT_SYMBOL_GPL(md_stop);
@@ -6524,6 +6564,9 @@ static int hot_remove_disk(struct mddev *mddev, dev_t dev)
 	char b[BDEVNAME_SIZE];
 	struct md_rdev *rdev;
 
+	if (!mddev->pers)
+		return -ENODEV;
+
 	rdev = find_rdev(mddev, dev);
 	if (!rdev)
 		return -ENXIO;
@@ -8641,6 +8684,7 @@ static int remove_and_add_spares(struct mddev *mddev,
 			if (mddev->pers->hot_remove_disk(
 				    mddev, rdev) == 0) {
 				sysfs_unlink_rdev(mddev, rdev);
+				rdev->saved_raid_disk = rdev->raid_disk;
 				rdev->raid_disk = -1;
 				removed++;
 			}
diff --git a/drivers/md/md.h b/drivers/md/md.h
index fbc925cce810..2d148bdaba74 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -252,6 +252,19 @@ enum mddev_sb_flags {
 	MD_SB_NEED_REWRITE,	/* metadata write needs to be repeated */
 };
 
+#define NR_FLUSH_INFOS 8
+#define NR_FLUSH_BIOS 64
+struct flush_info {
+	struct bio			*bio;
+	struct mddev			*mddev;
+	struct work_struct		flush_work;
+	atomic_t			flush_pending;
+};
+struct flush_bio {
+	struct flush_info *fi;
+	struct md_rdev *rdev;
+};
+
 struct mddev {
 	void				*private;
 	struct md_personality		*pers;
@@ -452,18 +465,13 @@ struct mddev {
 
 	struct attribute_group		*to_remove;
 
-	struct bio_set			*bio_set;
-	struct bio_set			*sync_set; /* for sync operations like
+	struct bio_set			bio_set;
+	struct bio_set			sync_set; /* for sync operations like
 						   * metadata and bitmap writes
 						   */
 
-	/* Generic flush handling.
-	 * The last to finish preflush schedules a worker to submit
-	 * the rest of the request (without the REQ_PREFLUSH flag).
-	 */
-	struct bio *flush_bio;
-	atomic_t flush_pending;
-	struct work_struct flush_work;
+	mempool_t			*flush_pool;
+	mempool_t			*flush_bio_pool;
 	struct work_struct event_work;	/* used by dm to report failure event */
 	void (*sync_super)(struct mddev *mddev, struct md_rdev *rdev);
 	struct md_cluster_info		*cluster_info;
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
index 584c10347267..65ae47a02218 100644
--- a/drivers/md/raid0.c
+++ b/drivers/md/raid0.c
@@ -479,7 +479,7 @@ static void raid0_handle_discard(struct mddev *mddev, struct bio *bio)
 	if (bio_end_sector(bio) > zone->zone_end) {
 		struct bio *split = bio_split(bio,
 			zone->zone_end - bio->bi_iter.bi_sector, GFP_NOIO,
-			mddev->bio_set);
+			&mddev->bio_set);
 		bio_chain(split, bio);
 		generic_make_request(bio);
 		bio = split;
@@ -582,7 +582,8 @@ static bool raid0_make_request(struct mddev *mddev, struct bio *bio)
 	sector = bio_sector;
 
 	if (sectors < bio_sectors(bio)) {
-		struct bio *split = bio_split(bio, sectors, GFP_NOIO, mddev->bio_set);
+		struct bio *split = bio_split(bio, sectors, GFP_NOIO,
+					      &mddev->bio_set);
 		bio_chain(split, bio);
 		generic_make_request(bio);
 		bio = split;
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index e9e3308cb0a7..0b344d087581 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -221,7 +221,7 @@ static void free_r1bio(struct r1bio *r1_bio)
 	struct r1conf *conf = r1_bio->mddev->private;
 
 	put_all_bios(conf, r1_bio);
-	mempool_free(r1_bio, conf->r1bio_pool);
+	mempool_free(r1_bio, &conf->r1bio_pool);
 }
 
 static void put_buf(struct r1bio *r1_bio)
@@ -236,7 +236,7 @@ static void put_buf(struct r1bio *r1_bio)
 			rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev);
 	}
 
-	mempool_free(r1_bio, conf->r1buf_pool);
+	mempool_free(r1_bio, &conf->r1buf_pool);
 
 	lower_barrier(conf, sect);
 }
@@ -1178,7 +1178,7 @@ alloc_r1bio(struct mddev *mddev, struct bio *bio)
 	struct r1conf *conf = mddev->private;
 	struct r1bio *r1_bio;
 
-	r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);
+	r1_bio = mempool_alloc(&conf->r1bio_pool, GFP_NOIO);
 	/* Ensure no bio records IO_BLOCKED */
 	memset(r1_bio->bios, 0, conf->raid_disks * sizeof(r1_bio->bios[0]));
 	init_r1bio(r1_bio, mddev, bio);
@@ -1268,7 +1268,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio,
 
 	if (max_sectors < bio_sectors(bio)) {
 		struct bio *split = bio_split(bio, max_sectors,
-					      gfp, conf->bio_split);
+					      gfp, &conf->bio_split);
 		bio_chain(split, bio);
 		generic_make_request(bio);
 		bio = split;
@@ -1278,7 +1278,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio,
 
 	r1_bio->read_disk = rdisk;
 
-	read_bio = bio_clone_fast(bio, gfp, mddev->bio_set);
+	read_bio = bio_clone_fast(bio, gfp, &mddev->bio_set);
 
 	r1_bio->bios[rdisk] = read_bio;
 
@@ -1439,7 +1439,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio,
 
 	if (max_sectors < bio_sectors(bio)) {
 		struct bio *split = bio_split(bio, max_sectors,
-					      GFP_NOIO, conf->bio_split);
+					      GFP_NOIO, &conf->bio_split);
 		bio_chain(split, bio);
 		generic_make_request(bio);
 		bio = split;
@@ -1479,9 +1479,9 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio,
 
 		if (r1_bio->behind_master_bio)
 			mbio = bio_clone_fast(r1_bio->behind_master_bio,
-					      GFP_NOIO, mddev->bio_set);
+					      GFP_NOIO, &mddev->bio_set);
 		else
-			mbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
+			mbio = bio_clone_fast(bio, GFP_NOIO, &mddev->bio_set);
 
 		if (r1_bio->behind_master_bio) {
 			if (test_bit(WriteMostly, &conf->mirrors[i].rdev->flags))
@@ -1657,8 +1657,7 @@ static void close_sync(struct r1conf *conf)
 		_allow_barrier(conf, idx);
 	}
 
-	mempool_destroy(conf->r1buf_pool);
-	conf->r1buf_pool = NULL;
+	mempool_exit(&conf->r1buf_pool);
 }
 
 static int raid1_spare_active(struct mddev *mddev)
@@ -2348,10 +2347,10 @@ static int narrow_write_error(struct r1bio *r1_bio, int i)
 		if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
 			wbio = bio_clone_fast(r1_bio->behind_master_bio,
 					      GFP_NOIO,
-					      mddev->bio_set);
+					      &mddev->bio_set);
 		} else {
 			wbio = bio_clone_fast(r1_bio->master_bio, GFP_NOIO,
-					      mddev->bio_set);
+					      &mddev->bio_set);
 		}
 
 		bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
@@ -2450,7 +2449,6 @@ static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
 	struct mddev *mddev = conf->mddev;
 	struct bio *bio;
 	struct md_rdev *rdev;
-	sector_t bio_sector;
 
 	clear_bit(R1BIO_ReadError, &r1_bio->state);
 	/* we got a read error. Maybe the drive is bad.  Maybe just
@@ -2463,7 +2461,6 @@ static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
 	 */
 
 	bio = r1_bio->bios[r1_bio->read_disk];
-	bio_sector = conf->mirrors[r1_bio->read_disk].rdev->data_offset + r1_bio->sector;
 	bio_put(bio);
 	r1_bio->bios[r1_bio->read_disk] = NULL;
 
@@ -2474,6 +2471,8 @@ static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
 		fix_read_error(conf, r1_bio->read_disk,
 			       r1_bio->sector, r1_bio->sectors);
 		unfreeze_array(conf);
+	} else if (mddev->ro == 0 && test_bit(FailFast, &rdev->flags)) {
+		md_error(mddev, rdev);
 	} else {
 		r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED;
 	}
@@ -2564,17 +2563,15 @@ static int init_resync(struct r1conf *conf)
 	int buffs;
 
 	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
-	BUG_ON(conf->r1buf_pool);
-	conf->r1buf_pool = mempool_create(buffs, r1buf_pool_alloc, r1buf_pool_free,
-					  conf->poolinfo);
-	if (!conf->r1buf_pool)
-		return -ENOMEM;
-	return 0;
+	BUG_ON(mempool_initialized(&conf->r1buf_pool));
+
+	return mempool_init(&conf->r1buf_pool, buffs, r1buf_pool_alloc,
+			    r1buf_pool_free, conf->poolinfo);
 }
 
 static struct r1bio *raid1_alloc_init_r1buf(struct r1conf *conf)
 {
-	struct r1bio *r1bio = mempool_alloc(conf->r1buf_pool, GFP_NOIO);
+	struct r1bio *r1bio = mempool_alloc(&conf->r1buf_pool, GFP_NOIO);
 	struct resync_pages *rps;
 	struct bio *bio;
 	int i;
@@ -2617,7 +2614,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
 	int idx = sector_to_idx(sector_nr);
 	int page_idx = 0;
 
-	if (!conf->r1buf_pool)
+	if (!mempool_initialized(&conf->r1buf_pool))
 		if (init_resync(conf))
 			return 0;
 
@@ -2953,14 +2950,13 @@ static struct r1conf *setup_conf(struct mddev *mddev)
 	if (!conf->poolinfo)
 		goto abort;
 	conf->poolinfo->raid_disks = mddev->raid_disks * 2;
-	conf->r1bio_pool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
-					  r1bio_pool_free,
-					  conf->poolinfo);
-	if (!conf->r1bio_pool)
+	err = mempool_init(&conf->r1bio_pool, NR_RAID1_BIOS, r1bio_pool_alloc,
+			   r1bio_pool_free, conf->poolinfo);
+	if (err)
 		goto abort;
 
-	conf->bio_split = bioset_create(BIO_POOL_SIZE, 0, 0);
-	if (!conf->bio_split)
+	err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);
+	if (err)
 		goto abort;
 
 	conf->poolinfo->mddev = mddev;
@@ -3033,7 +3029,7 @@ static struct r1conf *setup_conf(struct mddev *mddev)
 
  abort:
 	if (conf) {
-		mempool_destroy(conf->r1bio_pool);
+		mempool_exit(&conf->r1bio_pool);
 		kfree(conf->mirrors);
 		safe_put_page(conf->tmppage);
 		kfree(conf->poolinfo);
@@ -3041,8 +3037,7 @@ static struct r1conf *setup_conf(struct mddev *mddev)
 		kfree(conf->nr_waiting);
 		kfree(conf->nr_queued);
 		kfree(conf->barrier);
-		if (conf->bio_split)
-			bioset_free(conf->bio_split);
+		bioset_exit(&conf->bio_split);
 		kfree(conf);
 	}
 	return ERR_PTR(err);
@@ -3144,7 +3139,7 @@ static void raid1_free(struct mddev *mddev, void *priv)
 {
 	struct r1conf *conf = priv;
 
-	mempool_destroy(conf->r1bio_pool);
+	mempool_exit(&conf->r1bio_pool);
 	kfree(conf->mirrors);
 	safe_put_page(conf->tmppage);
 	kfree(conf->poolinfo);
@@ -3152,8 +3147,7 @@ static void raid1_free(struct mddev *mddev, void *priv)
 	kfree(conf->nr_waiting);
 	kfree(conf->nr_queued);
 	kfree(conf->barrier);
-	if (conf->bio_split)
-		bioset_free(conf->bio_split);
+	bioset_exit(&conf->bio_split);
 	kfree(conf);
 }
 
@@ -3199,13 +3193,17 @@ static int raid1_reshape(struct mddev *mddev)
 	 * At the same time, we "pack" the devices so that all the missing
 	 * devices have the higher raid_disk numbers.
 	 */
-	mempool_t *newpool, *oldpool;
+	mempool_t newpool, oldpool;
 	struct pool_info *newpoolinfo;
 	struct raid1_info *newmirrors;
 	struct r1conf *conf = mddev->private;
 	int cnt, raid_disks;
 	unsigned long flags;
 	int d, d2;
+	int ret;
+
+	memset(&newpool, 0, sizeof(newpool));
+	memset(&oldpool, 0, sizeof(oldpool));
 
 	/* Cannot change chunk_size, layout, or level */
 	if (mddev->chunk_sectors != mddev->new_chunk_sectors ||
@@ -3237,17 +3235,17 @@ static int raid1_reshape(struct mddev *mddev)
 	newpoolinfo->mddev = mddev;
 	newpoolinfo->raid_disks = raid_disks * 2;
 
-	newpool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
-				 r1bio_pool_free, newpoolinfo);
-	if (!newpool) {
+	ret = mempool_init(&newpool, NR_RAID1_BIOS, r1bio_pool_alloc,
+			   r1bio_pool_free, newpoolinfo);
+	if (ret) {
 		kfree(newpoolinfo);
-		return -ENOMEM;
+		return ret;
 	}
 	newmirrors = kzalloc(sizeof(struct raid1_info) * raid_disks * 2,
 			     GFP_KERNEL);
 	if (!newmirrors) {
 		kfree(newpoolinfo);
-		mempool_destroy(newpool);
+		mempool_exit(&newpool);
 		return -ENOMEM;
 	}
 
@@ -3287,7 +3285,7 @@ static int raid1_reshape(struct mddev *mddev)
 	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
 	md_wakeup_thread(mddev->thread);
 
-	mempool_destroy(oldpool);
+	mempool_exit(&oldpool);
 	return 0;
 }
 
diff --git a/drivers/md/raid1.h b/drivers/md/raid1.h
index eb84bc68e2fd..e7ccad898736 100644
--- a/drivers/md/raid1.h
+++ b/drivers/md/raid1.h
@@ -118,10 +118,10 @@ struct r1conf {
 	 * mempools - it changes when the array grows or shrinks
 	 */
 	struct pool_info	*poolinfo;
-	mempool_t		*r1bio_pool;
-	mempool_t		*r1buf_pool;
+	mempool_t		r1bio_pool;
+	mempool_t		r1buf_pool;
 
-	struct bio_set		*bio_split;
+	struct bio_set		bio_split;
 
 	/* temporary buffer to synchronous IO when attempting to repair
 	 * a read error.
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 3c60774c8430..1147ae59e3b6 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -255,9 +255,11 @@ static void r10buf_pool_free(void *__r10_bio, void *data)
 	for (j = conf->copies; j--; ) {
 		struct bio *bio = r10bio->devs[j].bio;
 
-		rp = get_resync_pages(bio);
-		resync_free_pages(rp);
-		bio_put(bio);
+		if (bio) {
+			rp = get_resync_pages(bio);
+			resync_free_pages(rp);
+			bio_put(bio);
+		}
 
 		bio = r10bio->devs[j].repl_bio;
 		if (bio)
@@ -291,14 +293,14 @@ static void free_r10bio(struct r10bio *r10_bio)
 	struct r10conf *conf = r10_bio->mddev->private;
 
 	put_all_bios(conf, r10_bio);
-	mempool_free(r10_bio, conf->r10bio_pool);
+	mempool_free(r10_bio, &conf->r10bio_pool);
 }
 
 static void put_buf(struct r10bio *r10_bio)
 {
 	struct r10conf *conf = r10_bio->mddev->private;
 
-	mempool_free(r10_bio, conf->r10buf_pool);
+	mempool_free(r10_bio, &conf->r10buf_pool);
 
 	lower_barrier(conf);
 }
@@ -1204,7 +1206,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio,
 				   (unsigned long long)r10_bio->sector);
 	if (max_sectors < bio_sectors(bio)) {
 		struct bio *split = bio_split(bio, max_sectors,
-					      gfp, conf->bio_split);
+					      gfp, &conf->bio_split);
 		bio_chain(split, bio);
 		generic_make_request(bio);
 		bio = split;
@@ -1213,7 +1215,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio,
 	}
 	slot = r10_bio->read_slot;
 
-	read_bio = bio_clone_fast(bio, gfp, mddev->bio_set);
+	read_bio = bio_clone_fast(bio, gfp, &mddev->bio_set);
 
 	r10_bio->devs[slot].bio = read_bio;
 	r10_bio->devs[slot].rdev = rdev;
@@ -1261,7 +1263,7 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,
 	} else
 		rdev = conf->mirrors[devnum].rdev;
 
-	mbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
+	mbio = bio_clone_fast(bio, GFP_NOIO, &mddev->bio_set);
 	if (replacement)
 		r10_bio->devs[n_copy].repl_bio = mbio;
 	else
@@ -1509,7 +1511,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio,
 
 	if (r10_bio->sectors < bio_sectors(bio)) {
 		struct bio *split = bio_split(bio, r10_bio->sectors,
-					      GFP_NOIO, conf->bio_split);
+					      GFP_NOIO, &conf->bio_split);
 		bio_chain(split, bio);
 		generic_make_request(bio);
 		bio = split;
@@ -1533,7 +1535,7 @@ static void __make_request(struct mddev *mddev, struct bio *bio, int sectors)
 	struct r10conf *conf = mddev->private;
 	struct r10bio *r10_bio;
 
-	r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
+	r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO);
 
 	r10_bio->master_bio = bio;
 	r10_bio->sectors = sectors;
@@ -1732,8 +1734,7 @@ static void close_sync(struct r10conf *conf)
 	wait_barrier(conf);
 	allow_barrier(conf);
 
-	mempool_destroy(conf->r10buf_pool);
-	conf->r10buf_pool = NULL;
+	mempool_exit(&conf->r10buf_pool);
 }
 
 static int raid10_spare_active(struct mddev *mddev)
@@ -2363,7 +2364,7 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
 {
 	int sect = 0; /* Offset from r10_bio->sector */
 	int sectors = r10_bio->sectors;
-	struct md_rdev*rdev;
+	struct md_rdev *rdev;
 	int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
 	int d = r10_bio->devs[r10_bio->read_slot].devnum;
 
@@ -2583,7 +2584,7 @@ static int narrow_write_error(struct r10bio *r10_bio, int i)
 		if (sectors > sect_to_write)
 			sectors = sect_to_write;
 		/* Write at 'sector' for 'sectors' */
-		wbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
+		wbio = bio_clone_fast(bio, GFP_NOIO, &mddev->bio_set);
 		bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors);
 		wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector);
 		wbio->bi_iter.bi_sector = wsector +
@@ -2816,25 +2817,25 @@ static void raid10d(struct md_thread *thread)
 
 static int init_resync(struct r10conf *conf)
 {
-	int buffs;
-	int i;
+	int ret, buffs, i;
 
 	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
-	BUG_ON(conf->r10buf_pool);
+	BUG_ON(mempool_initialized(&conf->r10buf_pool));
 	conf->have_replacement = 0;
 	for (i = 0; i < conf->geo.raid_disks; i++)
 		if (conf->mirrors[i].replacement)
 			conf->have_replacement = 1;
-	conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
-	if (!conf->r10buf_pool)
-		return -ENOMEM;
+	ret = mempool_init(&conf->r10buf_pool, buffs,
+			   r10buf_pool_alloc, r10buf_pool_free, conf);
+	if (ret)
+		return ret;
 	conf->next_resync = 0;
 	return 0;
 }
 
 static struct r10bio *raid10_alloc_init_r10buf(struct r10conf *conf)
 {
-	struct r10bio *r10bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
+	struct r10bio *r10bio = mempool_alloc(&conf->r10buf_pool, GFP_NOIO);
 	struct rsync_pages *rp;
 	struct bio *bio;
 	int nalloc;
@@ -2945,7 +2946,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 	sector_t chunk_mask = conf->geo.chunk_mask;
 	int page_idx = 0;
 
-	if (!conf->r10buf_pool)
+	if (!mempool_initialized(&conf->r10buf_pool))
 		if (init_resync(conf))
 			return 0;
 
@@ -3699,13 +3700,13 @@ static struct r10conf *setup_conf(struct mddev *mddev)
 
 	conf->geo = geo;
 	conf->copies = copies;
-	conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
-					   r10bio_pool_free, conf);
-	if (!conf->r10bio_pool)
+	err = mempool_init(&conf->r10bio_pool, NR_RAID10_BIOS, r10bio_pool_alloc,
+			   r10bio_pool_free, conf);
+	if (err)
 		goto out;
 
-	conf->bio_split = bioset_create(BIO_POOL_SIZE, 0, 0);
-	if (!conf->bio_split)
+	err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);
+	if (err)
 		goto out;
 
 	calc_sectors(conf, mddev->dev_sectors);
@@ -3733,6 +3734,7 @@ static struct r10conf *setup_conf(struct mddev *mddev)
 	init_waitqueue_head(&conf->wait_barrier);
 	atomic_set(&conf->nr_pending, 0);
 
+	err = -ENOMEM;
 	conf->thread = md_register_thread(raid10d, mddev, "raid10");
 	if (!conf->thread)
 		goto out;
@@ -3742,11 +3744,10 @@ static struct r10conf *setup_conf(struct mddev *mddev)
 
  out:
 	if (conf) {
-		mempool_destroy(conf->r10bio_pool);
+		mempool_exit(&conf->r10bio_pool);
 		kfree(conf->mirrors);
 		safe_put_page(conf->tmppage);
-		if (conf->bio_split)
-			bioset_free(conf->bio_split);
+		bioset_exit(&conf->bio_split);
 		kfree(conf);
 	}
 	return ERR_PTR(err);
@@ -3953,7 +3954,7 @@ static int raid10_run(struct mddev *mddev)
 
 out_free_conf:
 	md_unregister_thread(&mddev->thread);
-	mempool_destroy(conf->r10bio_pool);
+	mempool_exit(&conf->r10bio_pool);
 	safe_put_page(conf->tmppage);
 	kfree(conf->mirrors);
 	kfree(conf);
@@ -3966,13 +3967,12 @@ static void raid10_free(struct mddev *mddev, void *priv)
 {
 	struct r10conf *conf = priv;
 
-	mempool_destroy(conf->r10bio_pool);
+	mempool_exit(&conf->r10bio_pool);
 	safe_put_page(conf->tmppage);
 	kfree(conf->mirrors);
 	kfree(conf->mirrors_old);
 	kfree(conf->mirrors_new);
-	if (conf->bio_split)
-		bioset_free(conf->bio_split);
+	bioset_exit(&conf->bio_split);
 	kfree(conf);
 }
 
@@ -4543,7 +4543,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
 		 * on all the target devices.
 		 */
 		// FIXME
-		mempool_free(r10_bio, conf->r10buf_pool);
+		mempool_free(r10_bio, &conf->r10buf_pool);
 		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
 		return sectors_done;
 	}
diff --git a/drivers/md/raid10.h b/drivers/md/raid10.h
index e2e8840de9bf..d3eaaf3eb1bc 100644
--- a/drivers/md/raid10.h
+++ b/drivers/md/raid10.h
@@ -93,10 +93,10 @@ struct r10conf {
 						   */
 	wait_queue_head_t	wait_barrier;
 
-	mempool_t		*r10bio_pool;
-	mempool_t		*r10buf_pool;
+	mempool_t		r10bio_pool;
+	mempool_t		r10buf_pool;
 	struct page		*tmppage;
-	struct bio_set		*bio_split;
+	struct bio_set		bio_split;
 
 	/* When taking over an array from a different personality, we store
 	 * the new thread here until we fully activate the array.
diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
index 3c65f52b68f5..2b775abf377b 100644
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -125,9 +125,9 @@ struct r5l_log {
 	struct list_head no_mem_stripes;   /* pending stripes, -ENOMEM */
 
 	struct kmem_cache *io_kc;
-	mempool_t *io_pool;
-	struct bio_set *bs;
-	mempool_t *meta_pool;
+	mempool_t io_pool;
+	struct bio_set bs;
+	mempool_t meta_pool;
 
 	struct md_thread *reclaim_thread;
 	unsigned long reclaim_target;	/* number of space that need to be
@@ -579,7 +579,7 @@ static void r5l_log_endio(struct bio *bio)
 		md_error(log->rdev->mddev, log->rdev);
 
 	bio_put(bio);
-	mempool_free(io->meta_page, log->meta_pool);
+	mempool_free(io->meta_page, &log->meta_pool);
 
 	spin_lock_irqsave(&log->io_list_lock, flags);
 	__r5l_set_io_unit_state(io, IO_UNIT_IO_END);
@@ -748,7 +748,7 @@ static void r5l_submit_current_io(struct r5l_log *log)
 
 static struct bio *r5l_bio_alloc(struct r5l_log *log)
 {
-	struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, log->bs);
+	struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, &log->bs);
 
 	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
 	bio_set_dev(bio, log->rdev->bdev);
@@ -780,7 +780,7 @@ static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
 	struct r5l_io_unit *io;
 	struct r5l_meta_block *block;
 
-	io = mempool_alloc(log->io_pool, GFP_ATOMIC);
+	io = mempool_alloc(&log->io_pool, GFP_ATOMIC);
 	if (!io)
 		return NULL;
 	memset(io, 0, sizeof(*io));
@@ -791,7 +791,7 @@ static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
 	bio_list_init(&io->flush_barriers);
 	io->state = IO_UNIT_RUNNING;
 
-	io->meta_page = mempool_alloc(log->meta_pool, GFP_NOIO);
+	io->meta_page = mempool_alloc(&log->meta_pool, GFP_NOIO);
 	block = page_address(io->meta_page);
 	clear_page(block);
 	block->magic = cpu_to_le32(R5LOG_MAGIC);
@@ -1223,7 +1223,7 @@ static bool r5l_complete_finished_ios(struct r5l_log *log)
 		log->next_checkpoint = io->log_start;
 
 		list_del(&io->log_sibling);
-		mempool_free(io, log->io_pool);
+		mempool_free(io, &log->io_pool);
 		r5l_run_no_mem_stripe(log);
 
 		found = true;
@@ -1647,7 +1647,7 @@ static int r5l_recovery_allocate_ra_pool(struct r5l_log *log,
 {
 	struct page *page;
 
-	ctx->ra_bio = bio_alloc_bioset(GFP_KERNEL, BIO_MAX_PAGES, log->bs);
+	ctx->ra_bio = bio_alloc_bioset(GFP_KERNEL, BIO_MAX_PAGES, &log->bs);
 	if (!ctx->ra_bio)
 		return -ENOMEM;
 
@@ -3066,6 +3066,7 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
 	struct request_queue *q = bdev_get_queue(rdev->bdev);
 	struct r5l_log *log;
 	char b[BDEVNAME_SIZE];
+	int ret;
 
 	pr_debug("md/raid:%s: using device %s as journal\n",
 		 mdname(conf->mddev), bdevname(rdev->bdev, b));
@@ -3111,16 +3112,16 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
 	if (!log->io_kc)
 		goto io_kc;
 
-	log->io_pool = mempool_create_slab_pool(R5L_POOL_SIZE, log->io_kc);
-	if (!log->io_pool)
+	ret = mempool_init_slab_pool(&log->io_pool, R5L_POOL_SIZE, log->io_kc);
+	if (ret)
 		goto io_pool;
 
-	log->bs = bioset_create(R5L_POOL_SIZE, 0, BIOSET_NEED_BVECS);
-	if (!log->bs)
+	ret = bioset_init(&log->bs, R5L_POOL_SIZE, 0, BIOSET_NEED_BVECS);
+	if (ret)
 		goto io_bs;
 
-	log->meta_pool = mempool_create_page_pool(R5L_POOL_SIZE, 0);
-	if (!log->meta_pool)
+	ret = mempool_init_page_pool(&log->meta_pool, R5L_POOL_SIZE, 0);
+	if (ret)
 		goto out_mempool;
 
 	spin_lock_init(&log->tree_lock);
@@ -3155,11 +3156,11 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
 	rcu_assign_pointer(conf->log, NULL);
 	md_unregister_thread(&log->reclaim_thread);
 reclaim_thread:
-	mempool_destroy(log->meta_pool);
+	mempool_exit(&log->meta_pool);
 out_mempool:
-	bioset_free(log->bs);
+	bioset_exit(&log->bs);
 io_bs:
-	mempool_destroy(log->io_pool);
+	mempool_exit(&log->io_pool);
 io_pool:
 	kmem_cache_destroy(log->io_kc);
 io_kc:
@@ -3178,9 +3179,9 @@ void r5l_exit_log(struct r5conf *conf)
 	wake_up(&conf->mddev->sb_wait);
 	flush_work(&log->disable_writeback_work);
 	md_unregister_thread(&log->reclaim_thread);
-	mempool_destroy(log->meta_pool);
-	bioset_free(log->bs);
-	mempool_destroy(log->io_pool);
+	mempool_exit(&log->meta_pool);
+	bioset_exit(&log->bs);
+	mempool_exit(&log->io_pool);
 	kmem_cache_destroy(log->io_kc);
 	kfree(log);
 }
diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c
index 42890a08375b..3a7c36326589 100644
--- a/drivers/md/raid5-ppl.c
+++ b/drivers/md/raid5-ppl.c
@@ -105,9 +105,9 @@ struct ppl_conf {
 	atomic64_t seq;		/* current log write sequence number */
 
 	struct kmem_cache *io_kc;
-	mempool_t *io_pool;
-	struct bio_set *bs;
-	struct bio_set *flush_bs;
+	mempool_t io_pool;
+	struct bio_set bs;
+	struct bio_set flush_bs;
 
 	/* used only for recovery */
 	int recovered_entries;
@@ -244,7 +244,7 @@ static struct ppl_io_unit *ppl_new_iounit(struct ppl_log *log,
 	struct ppl_header *pplhdr;
 	struct page *header_page;
 
-	io = mempool_alloc(ppl_conf->io_pool, GFP_NOWAIT);
+	io = mempool_alloc(&ppl_conf->io_pool, GFP_NOWAIT);
 	if (!io)
 		return NULL;
 
@@ -503,7 +503,7 @@ static void ppl_submit_iounit(struct ppl_io_unit *io)
 			struct bio *prev = bio;
 
 			bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES,
-					       ppl_conf->bs);
+					       &ppl_conf->bs);
 			bio->bi_opf = prev->bi_opf;
 			bio_copy_dev(bio, prev);
 			bio->bi_iter.bi_sector = bio_end_sector(prev);
@@ -570,7 +570,7 @@ static void ppl_io_unit_finished(struct ppl_io_unit *io)
 	list_del(&io->log_sibling);
 	spin_unlock(&log->io_list_lock);
 
-	mempool_free(io, ppl_conf->io_pool);
+	mempool_free(io, &ppl_conf->io_pool);
 
 	spin_lock(&ppl_conf->no_mem_stripes_lock);
 	if (!list_empty(&ppl_conf->no_mem_stripes)) {
@@ -642,7 +642,7 @@ static void ppl_do_flush(struct ppl_io_unit *io)
 			struct bio *bio;
 			char b[BDEVNAME_SIZE];
 
-			bio = bio_alloc_bioset(GFP_NOIO, 0, ppl_conf->flush_bs);
+			bio = bio_alloc_bioset(GFP_NOIO, 0, &ppl_conf->flush_bs);
 			bio_set_dev(bio, bdev);
 			bio->bi_private = io;
 			bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
@@ -1246,11 +1246,9 @@ static void __ppl_exit_log(struct ppl_conf *ppl_conf)
 
 	kfree(ppl_conf->child_logs);
 
-	if (ppl_conf->bs)
-		bioset_free(ppl_conf->bs);
-	if (ppl_conf->flush_bs)
-		bioset_free(ppl_conf->flush_bs);
-	mempool_destroy(ppl_conf->io_pool);
+	bioset_exit(&ppl_conf->bs);
+	bioset_exit(&ppl_conf->flush_bs);
+	mempool_exit(&ppl_conf->io_pool);
 	kmem_cache_destroy(ppl_conf->io_kc);
 
 	kfree(ppl_conf);
@@ -1387,24 +1385,18 @@ int ppl_init_log(struct r5conf *conf)
 		goto err;
 	}
 
-	ppl_conf->io_pool = mempool_create(conf->raid_disks, ppl_io_pool_alloc,
-					   ppl_io_pool_free, ppl_conf->io_kc);
-	if (!ppl_conf->io_pool) {
-		ret = -ENOMEM;
+	ret = mempool_init(&ppl_conf->io_pool, conf->raid_disks, ppl_io_pool_alloc,
+			   ppl_io_pool_free, ppl_conf->io_kc);
+	if (ret)
 		goto err;
-	}
 
-	ppl_conf->bs = bioset_create(conf->raid_disks, 0, BIOSET_NEED_BVECS);
-	if (!ppl_conf->bs) {
-		ret = -ENOMEM;
+	ret = bioset_init(&ppl_conf->bs, conf->raid_disks, 0, BIOSET_NEED_BVECS);
+	if (ret)
 		goto err;
-	}
 
-	ppl_conf->flush_bs = bioset_create(conf->raid_disks, 0, 0);
-	if (!ppl_conf->flush_bs) {
-		ret = -ENOMEM;
+	ret = bioset_init(&ppl_conf->flush_bs, conf->raid_disks, 0, 0);
+	if (ret)
 		goto err;
-	}
 
 	ppl_conf->count = conf->raid_disks;
 	ppl_conf->child_logs = kcalloc(ppl_conf->count, sizeof(struct ppl_log),
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index be117d0a65a8..73489446bbcb 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -1139,6 +1139,9 @@ static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
 			bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
 			bi->bi_io_vec[0].bv_offset = 0;
 			bi->bi_iter.bi_size = STRIPE_SIZE;
+			bi->bi_write_hint = sh->dev[i].write_hint;
+			if (!rrdev)
+				sh->dev[i].write_hint = RWF_WRITE_LIFE_NOT_SET;
 			/*
 			 * If this is discard request, set bi_vcnt 0. We don't
 			 * want to confuse SCSI because SCSI will replace payload
@@ -1190,6 +1193,8 @@ static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
 			rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
 			rbi->bi_io_vec[0].bv_offset = 0;
 			rbi->bi_iter.bi_size = STRIPE_SIZE;
+			rbi->bi_write_hint = sh->dev[i].write_hint;
+			sh->dev[i].write_hint = RWF_WRITE_LIFE_NOT_SET;
 			/*
 			 * If this is discard request, set bi_vcnt 0. We don't
 			 * want to confuse SCSI because SCSI will replace payload
@@ -3204,6 +3209,7 @@ static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx,
 		(unsigned long long)sh->sector);
 
 	spin_lock_irq(&sh->stripe_lock);
+	sh->dev[dd_idx].write_hint = bi->bi_write_hint;
 	/* Don't allow new IO added to stripes in batch list */
 	if (sh->batch_head)
 		goto overlap;
@@ -4614,15 +4620,15 @@ static void break_stripe_batch_list(struct stripe_head *head_sh,
 
 		sh->check_state = head_sh->check_state;
 		sh->reconstruct_state = head_sh->reconstruct_state;
+		spin_lock_irq(&sh->stripe_lock);
+		sh->batch_head = NULL;
+		spin_unlock_irq(&sh->stripe_lock);
 		for (i = 0; i < sh->disks; i++) {
 			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
 				do_wakeup = 1;
 			sh->dev[i].flags = head_sh->dev[i].flags &
 				(~((1 << R5_WriteError) | (1 << R5_Overlap)));
 		}
-		spin_lock_irq(&sh->stripe_lock);
-		sh->batch_head = NULL;
-		spin_unlock_irq(&sh->stripe_lock);
 		if (handle_flags == 0 ||
 		    sh->state & handle_flags)
 			set_bit(STRIPE_HANDLE, &sh->state);
@@ -5192,7 +5198,7 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
 	/*
 	 * use bio_clone_fast to make a copy of the bio
 	 */
-	align_bi = bio_clone_fast(raid_bio, GFP_NOIO, mddev->bio_set);
+	align_bi = bio_clone_fast(raid_bio, GFP_NOIO, &mddev->bio_set);
 	if (!align_bi)
 		return 0;
 	/*
@@ -5277,7 +5283,7 @@ static struct bio *chunk_aligned_read(struct mddev *mddev, struct bio *raid_bio)
 
 	if (sectors < bio_sectors(raid_bio)) {
 		struct r5conf *conf = mddev->private;
-		split = bio_split(raid_bio, sectors, GFP_NOIO, conf->bio_split);
+		split = bio_split(raid_bio, sectors, GFP_NOIO, &conf->bio_split);
 		bio_chain(split, raid_bio);
 		generic_make_request(raid_bio);
 		raid_bio = split;
@@ -6773,8 +6779,7 @@ static void free_conf(struct r5conf *conf)
 		if (conf->disks[i].extra_page)
 			put_page(conf->disks[i].extra_page);
 	kfree(conf->disks);
-	if (conf->bio_split)
-		bioset_free(conf->bio_split);
+	bioset_exit(&conf->bio_split);
 	kfree(conf->stripe_hashtbl);
 	kfree(conf->pending_data);
 	kfree(conf);
@@ -6853,6 +6858,7 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 	int i;
 	int group_cnt, worker_cnt_per_group;
 	struct r5worker_group *new_group;
+	int ret;
 
 	if (mddev->new_level != 5
 	    && mddev->new_level != 4
@@ -6950,8 +6956,8 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 			goto abort;
 	}
 
-	conf->bio_split = bioset_create(BIO_POOL_SIZE, 0, 0);
-	if (!conf->bio_split)
+	ret = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);
+	if (ret)
 		goto abort;
 	conf->mddev = mddev;
 
diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h
index 3f8da26032ac..8474c224127b 100644
--- a/drivers/md/raid5.h
+++ b/drivers/md/raid5.h
@@ -257,6 +257,7 @@ struct stripe_head {
 		sector_t	sector;			/* sector of this page */
 		unsigned long	flags;
 		u32		log_checksum;
+		unsigned short	write_hint;
 	} dev[1]; /* allocated with extra space depending of RAID geometry */
 };
 
@@ -669,7 +670,7 @@ struct r5conf {
 	int			pool_size; /* number of disks in stripeheads in pool */
 	spinlock_t		device_lock;
 	struct disk_info	*disks;
-	struct bio_set		*bio_split;
+	struct bio_set		bio_split;
 
 	/* When taking over an array from a different personality, we store
 	 * the new thread here until we fully activate the array.
diff --git a/drivers/media/Kconfig b/drivers/media/Kconfig
index 37124c3b8c2a..8add62a18293 100644
--- a/drivers/media/Kconfig
+++ b/drivers/media/Kconfig
@@ -78,13 +78,13 @@ config MEDIA_SDR_SUPPORT
 	  Say Y when you have a software defined radio device.
 
 config MEDIA_CEC_SUPPORT
-       bool "HDMI CEC support"
-       ---help---
-	 Enable support for HDMI CEC (Consumer Electronics Control),
-	 which is an optional HDMI feature.
+	bool "HDMI CEC support"
+	---help---
+	  Enable support for HDMI CEC (Consumer Electronics Control),
+	  which is an optional HDMI feature.
 
-	 Say Y when you have an HDMI receiver, transmitter or a USB CEC
-	 adapter that supports HDMI CEC.
+	  Say Y when you have an HDMI receiver, transmitter or a USB CEC
+	  adapter that supports HDMI CEC.
 
 source "drivers/media/cec/Kconfig"
 
diff --git a/drivers/media/cec/cec-adap.c b/drivers/media/cec/cec-adap.c
index 002ed4c90371..b7fad0ec5710 100644
--- a/drivers/media/cec/cec-adap.c
+++ b/drivers/media/cec/cec-adap.c
@@ -339,12 +339,19 @@ static void cec_data_cancel(struct cec_data *data)
 			data->adap->transmit_queue_sz--;
 	}
 
-	/* Mark it as an error */
-	data->msg.tx_ts = ktime_get_ns();
-	data->msg.tx_status |= CEC_TX_STATUS_ERROR |
-			       CEC_TX_STATUS_MAX_RETRIES;
-	data->msg.tx_error_cnt++;
-	data->attempts = 0;
+	if (data->msg.tx_status & CEC_TX_STATUS_OK) {
+		/* Mark the canceled RX as a timeout */
+		data->msg.rx_ts = ktime_get_ns();
+		data->msg.rx_status = CEC_RX_STATUS_TIMEOUT;
+	} else {
+		/* Mark the canceled TX as an error */
+		data->msg.tx_ts = ktime_get_ns();
+		data->msg.tx_status |= CEC_TX_STATUS_ERROR |
+				       CEC_TX_STATUS_MAX_RETRIES;
+		data->msg.tx_error_cnt++;
+		data->attempts = 0;
+	}
+
 	/* Queue transmitted message for monitoring purposes */
 	cec_queue_msg_monitor(data->adap, &data->msg, 1);
 
diff --git a/drivers/media/cec/cec-core.c b/drivers/media/cec/cec-core.c
index b0c87f9ea08f..b278ab90b387 100644
--- a/drivers/media/cec/cec-core.c
+++ b/drivers/media/cec/cec-core.c
@@ -322,7 +322,7 @@ struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops,
 	adap->rc->allowed_protocols = RC_PROTO_BIT_CEC;
 	adap->rc->priv = adap;
 	adap->rc->map_name = RC_MAP_CEC;
-	adap->rc->timeout = MS_TO_NS(100);
+	adap->rc->timeout = MS_TO_NS(550);
 #endif
 	return adap;
 }
diff --git a/drivers/media/cec/cec-pin-error-inj.c b/drivers/media/cec/cec-pin-error-inj.c
index aaa899a175ce..c0088d3b8e3d 100644
--- a/drivers/media/cec/cec-pin-error-inj.c
+++ b/drivers/media/cec/cec-pin-error-inj.c
@@ -81,10 +81,9 @@ bool cec_pin_error_inj_parse_line(struct cec_adapter *adap, char *line)
 	u64 *error;
 	u8 *args;
 	bool has_op;
-	u32 op;
+	u8 op;
 	u8 mode;
 	u8 pos;
-	u8 v;
 
 	p = skip_spaces(p);
 	token = strsep(&p, delims);
@@ -146,12 +145,18 @@ bool cec_pin_error_inj_parse_line(struct cec_adapter *adap, char *line)
 	comma = strchr(token, ',');
 	if (comma)
 		*comma++ = '\0';
-	if (!strcmp(token, "any"))
-		op = CEC_ERROR_INJ_OP_ANY;
-	else if (!kstrtou8(token, 0, &v))
-		op = v;
-	else
+	if (!strcmp(token, "any")) {
+		has_op = false;
+		error = pin->error_inj + CEC_ERROR_INJ_OP_ANY;
+		args = pin->error_inj_args[CEC_ERROR_INJ_OP_ANY];
+	} else if (!kstrtou8(token, 0, &op)) {
+		has_op = true;
+		error = pin->error_inj + op;
+		args = pin->error_inj_args[op];
+	} else {
 		return false;
+	}
+
 	mode = CEC_ERROR_INJ_MODE_ONCE;
 	if (comma) {
 		if (!strcmp(comma, "off"))
@@ -166,10 +171,6 @@ bool cec_pin_error_inj_parse_line(struct cec_adapter *adap, char *line)
 			return false;
 	}
 
-	error = pin->error_inj + op;
-	args = pin->error_inj_args[op];
-	has_op = op <= 0xff;
-
 	token = strsep(&p, delims);
 	if (p) {
 		p = skip_spaces(p);
@@ -203,16 +204,18 @@ bool cec_pin_error_inj_parse_line(struct cec_adapter *adap, char *line)
 		mode_mask = CEC_ERROR_INJ_MODE_MASK << mode_offset;
 		arg_idx = cec_error_inj_cmds[i].arg_idx;
 
-		if (mode_offset == CEC_ERROR_INJ_RX_ARB_LOST_OFFSET ||
-		    mode_offset == CEC_ERROR_INJ_TX_ADD_BYTES_OFFSET)
-			is_bit_pos = false;
-
 		if (mode_offset == CEC_ERROR_INJ_RX_ARB_LOST_OFFSET) {
 			if (has_op)
 				return false;
 			if (!has_pos)
 				pos = 0x0f;
+			is_bit_pos = false;
+		} else if (mode_offset == CEC_ERROR_INJ_TX_ADD_BYTES_OFFSET) {
+			if (!has_pos || !pos)
+				return false;
+			is_bit_pos = false;
 		}
+
 		if (arg_idx >= 0 && is_bit_pos) {
 			if (!has_pos || pos >= 160)
 				return false;
diff --git a/drivers/media/cec/cec-pin.c b/drivers/media/cec/cec-pin.c
index 2a5df99735fa..6e311424f0dc 100644
--- a/drivers/media/cec/cec-pin.c
+++ b/drivers/media/cec/cec-pin.c
@@ -119,7 +119,7 @@ static void cec_pin_update(struct cec_pin *pin, bool v, bool force)
 
 		if (pin->work_pin_events_dropped) {
 			pin->work_pin_events_dropped = false;
-			v |= CEC_PIN_EVENT_FL_DROPPED;
+			ev |= CEC_PIN_EVENT_FL_DROPPED;
 		}
 		pin->work_pin_events[pin->work_pin_events_wr] = ev;
 		pin->work_pin_ts[pin->work_pin_events_wr] = ktime_get();
diff --git a/drivers/media/common/b2c2/flexcop-fe-tuner.c b/drivers/media/common/b2c2/flexcop-fe-tuner.c
index a1ce3e8eb1d3..aac1aadb0cb1 100644
--- a/drivers/media/common/b2c2/flexcop-fe-tuner.c
+++ b/drivers/media/common/b2c2/flexcop-fe-tuner.c
@@ -495,7 +495,6 @@ static int airstar_atsc2_attach(struct flexcop_device *fc,
 /* AirStar ATSC 3rd generation */
 #if FE_SUPPORTED(LGDT330X)
 static struct lgdt330x_config air2pc_atsc_hd5000_config = {
-	.demod_address       = 0x59,
 	.demod_chip          = LGDT3303,
 	.serial_mpeg         = 0x04,
 	.clock_polarity_flip = 1,
@@ -504,7 +503,8 @@ static struct lgdt330x_config air2pc_atsc_hd5000_config = {
 static int airstar_atsc3_attach(struct flexcop_device *fc,
 	struct i2c_adapter *i2c)
 {
-	fc->fe = dvb_attach(lgdt330x_attach, &air2pc_atsc_hd5000_config, i2c);
+	fc->fe = dvb_attach(lgdt330x_attach, &air2pc_atsc_hd5000_config,
+			    0x59, i2c);
 	if (!fc->fe)
 		return 0;
 
diff --git a/drivers/media/common/b2c2/flexcop-i2c.c b/drivers/media/common/b2c2/flexcop-i2c.c
index 564da6fa900d..6675b605eb6f 100644
--- a/drivers/media/common/b2c2/flexcop-i2c.c
+++ b/drivers/media/common/b2c2/flexcop-i2c.c
@@ -105,40 +105,36 @@ static int flexcop_i2c_write4(struct flexcop_device *fc,
 }
 
 int flexcop_i2c_request(struct flexcop_i2c_adapter *i2c,
-		flexcop_access_op_t op, u8 chipaddr, u8 addr, u8 *buf, u16 len)
+			flexcop_access_op_t op, u8 chipaddr,
+			u8 start_addr, u8 *buf, u16 size)
 {
 	int ret;
-
-#ifdef DUMP_I2C_MESSAGES
-	int i;
-#endif
+	int len = size;
+	u8 *p;
+	u8 addr = start_addr;
 
 	u16 bytes_to_transfer;
 	flexcop_ibi_value r100;
 
-	deb_i2c("op = %d\n",op);
+	deb_i2c("port %d %s(%02x): register %02x, size: %d\n",
+		i2c->port,
+		op == FC_READ ? "rd" : "wr",
+		chipaddr, start_addr, size);
 	r100.raw = 0;
 	r100.tw_sm_c_100.chipaddr = chipaddr;
 	r100.tw_sm_c_100.twoWS_rw = op;
 	r100.tw_sm_c_100.twoWS_port_reg = i2c->port;
 
-#ifdef DUMP_I2C_MESSAGES
-	printk(KERN_DEBUG "%d ", i2c->port);
-	if (op == FC_READ)
-		printk(KERN_CONT "rd(");
-	else
-		printk(KERN_CONT "wr(");
-	printk(KERN_CONT "%02x): %02x ", chipaddr, addr);
-#endif
-
 	/* in that case addr is the only value ->
 	 * we write it twice as baseaddr and val0
 	 * BBTI is doing it like that for ISL6421 at least */
 	if (i2c->no_base_addr && len == 0 && op == FC_WRITE) {
-		buf = &addr;
+		buf = &start_addr;
 		len = 1;
 	}
 
+	p = buf;
+
 	while (len != 0) {
 		bytes_to_transfer = len > 4 ? 4 : len;
 
@@ -146,26 +142,21 @@ int flexcop_i2c_request(struct flexcop_i2c_adapter *i2c,
 		r100.tw_sm_c_100.baseaddr = addr;
 
 		if (op == FC_READ)
-			ret = flexcop_i2c_read4(i2c, r100, buf);
+			ret = flexcop_i2c_read4(i2c, r100, p);
 		else
-			ret = flexcop_i2c_write4(i2c->fc, r100, buf);
-
-#ifdef DUMP_I2C_MESSAGES
-		for (i = 0; i < bytes_to_transfer; i++)
-			printk(KERN_CONT "%02x ", buf[i]);
-#endif
+			ret = flexcop_i2c_write4(i2c->fc, r100, p);
 
 		if (ret < 0)
 			return ret;
 
-		buf  += bytes_to_transfer;
+		p  += bytes_to_transfer;
 		addr += bytes_to_transfer;
 		len  -= bytes_to_transfer;
 	}
-
-#ifdef DUMP_I2C_MESSAGES
-	printk(KERN_CONT "\n");
-#endif
+	deb_i2c_dump("port %d %s(%02x): register %02x: %*ph\n",
+		i2c->port,
+		op == FC_READ ? "rd" : "wr",
+		chipaddr, start_addr, size, buf);
 
 	return 0;
 }
diff --git a/drivers/media/common/b2c2/flexcop.c b/drivers/media/common/b2c2/flexcop.c
index 2e0ab55cd67e..cbaa61f10d5f 100644
--- a/drivers/media/common/b2c2/flexcop.c
+++ b/drivers/media/common/b2c2/flexcop.c
@@ -42,7 +42,7 @@ int b2c2_flexcop_debug;
 EXPORT_SYMBOL_GPL(b2c2_flexcop_debug);
 module_param_named(debug, b2c2_flexcop_debug,  int, 0644);
 MODULE_PARM_DESC(debug,
-		"set debug level (1=info,2=tuner,4=i2c,8=ts,16=sram,32=reg (|-able))."
+		"set debug level (1=info,2=tuner,4=i2c,8=ts,16=sram,32=reg,64=i2cdump (|-able))."
 		DEBSTATUS);
 #undef DEBSTATUS
 
diff --git a/drivers/media/common/b2c2/flexcop.h b/drivers/media/common/b2c2/flexcop.h
index 911ece59ea02..486fe2380b92 100644
--- a/drivers/media/common/b2c2/flexcop.h
+++ b/drivers/media/common/b2c2/flexcop.h
@@ -26,5 +26,6 @@ extern int b2c2_flexcop_debug;
 #define deb_ts(args...) dprintk(0x08, args)
 #define deb_sram(args...) dprintk(0x10, args)
 #define deb_rdump(args...) dprintk(0x20, args)
+#define deb_i2c_dump(args...) dprintk(0x40, args)
 
 #endif
diff --git a/drivers/media/common/saa7146/saa7146_i2c.c b/drivers/media/common/saa7146/saa7146_i2c.c
index f9e099d812c8..3feddc52c446 100644
--- a/drivers/media/common/saa7146/saa7146_i2c.c
+++ b/drivers/media/common/saa7146/saa7146_i2c.c
@@ -308,7 +308,7 @@ static int saa7146_i2c_transfer(struct saa7146_dev *dev, const struct i2c_msg *m
 	/* prepare the message(s), get number of u32s to transfer */
 	count = saa7146_i2c_msg_prepare(msgs, num, buffer);
 	if ( 0 > count ) {
-		err = -1;
+		err = -EIO;
 		goto out;
 	}
 
@@ -360,7 +360,7 @@ static int saa7146_i2c_transfer(struct saa7146_dev *dev, const struct i2c_msg *m
 	/* if any things had to be read, get the results */
 	if ( 0 != saa7146_i2c_msg_cleanup(msgs, num, buffer)) {
 		DEB_I2C("could not cleanup i2c-message\n");
-		err = -1;
+		err = -EIO;
 		goto out;
 	}
 
diff --git a/drivers/media/common/siano/smscoreapi.c b/drivers/media/common/siano/smscoreapi.c
index b5dcc6d1fe90..3b02cb570a6e 100644
--- a/drivers/media/common/siano/smscoreapi.c
+++ b/drivers/media/common/siano/smscoreapi.c
@@ -415,8 +415,8 @@ EXPORT_SYMBOL_GPL(smscore_get_board_id);
 
 struct smscore_registry_entry_t {
 	struct list_head entry;
-	char			devpath[32];
-	int				mode;
+	char devpath[32];
+	int mode;
 	enum sms_device_type_st	type;
 };
 
@@ -442,7 +442,7 @@ static struct smscore_registry_entry_t *smscore_find_registry(char *devpath)
 	     next != &g_smscore_registry;
 	     next = next->next) {
 		entry = (struct smscore_registry_entry_t *) next;
-		if (!strcmp(entry->devpath, devpath)) {
+		if (!strncmp(entry->devpath, devpath, sizeof(entry->devpath))) {
 			kmutex_unlock(&g_smscore_registrylock);
 			return entry;
 		}
@@ -450,7 +450,7 @@ static struct smscore_registry_entry_t *smscore_find_registry(char *devpath)
 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
 	if (entry) {
 		entry->mode = default_mode;
-		strcpy(entry->devpath, devpath);
+		strlcpy(entry->devpath, devpath, sizeof(entry->devpath));
 		list_add(&entry->entry, &g_smscore_registry);
 	} else
 		pr_err("failed to create smscore_registry.\n");
@@ -649,6 +649,7 @@ smscore_buffer_t *smscore_createbuffer(u8 *buffer, void *common_buffer,
  */
 int smscore_register_device(struct smsdevice_params_t *params,
 			    struct smscore_device_t **coredev,
+			    gfp_t gfp_buf_flags,
 			    void *mdev)
 {
 	struct smscore_device_t *dev;
@@ -661,6 +662,7 @@ int smscore_register_device(struct smsdevice_params_t *params,
 #ifdef CONFIG_MEDIA_CONTROLLER_DVB
 	dev->media_dev = mdev;
 #endif
+	dev->gfp_buf_flags = gfp_buf_flags;
 
 	/* init list entry so it could be safe in smscore_unregister_device */
 	INIT_LIST_HEAD(&dev->entry);
@@ -697,7 +699,7 @@ int smscore_register_device(struct smsdevice_params_t *params,
 		buffer = dma_alloc_coherent(params->device,
 					    dev->common_buffer_size,
 					    &dev->common_buffer_phys,
-					    GFP_KERNEL | GFP_DMA);
+					    GFP_KERNEL | dev->gfp_buf_flags);
 	if (!buffer) {
 		smscore_unregister_device(dev);
 		return -ENOMEM;
@@ -733,7 +735,7 @@ int smscore_register_device(struct smsdevice_params_t *params,
 	dev->postload_handler = params->postload_handler;
 
 	dev->device_flags = params->flags;
-	strcpy(dev->devpath, params->devpath);
+	strlcpy(dev->devpath, params->devpath, sizeof(dev->devpath));
 
 	smscore_registry_settype(dev->devpath, params->device_type);
 
@@ -792,7 +794,7 @@ static int smscore_init_ir(struct smscore_device_t *coredev)
 		else {
 			buffer = kmalloc(sizeof(struct sms_msg_data2) +
 						SMS_DMA_ALIGNMENT,
-						GFP_KERNEL | GFP_DMA);
+						GFP_KERNEL | coredev->gfp_buf_flags);
 			if (buffer) {
 				struct sms_msg_data2 *msg =
 				(struct sms_msg_data2 *)
@@ -933,7 +935,7 @@ static int smscore_load_firmware_family2(struct smscore_device_t *coredev,
 	}
 
 	/* PAGE_SIZE buffer shall be enough and dma aligned */
-	msg = kmalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
+	msg = kmalloc(PAGE_SIZE, GFP_KERNEL | coredev->gfp_buf_flags);
 	if (!msg)
 		return -ENOMEM;
 
@@ -1168,7 +1170,7 @@ static int smscore_load_firmware_from_file(struct smscore_device_t *coredev,
 	}
 	pr_debug("read fw %s, buffer size=0x%zx\n", fw_filename, fw->size);
 	fw_buf = kmalloc(ALIGN(fw->size + sizeof(struct sms_firmware),
-			 SMS_ALLOC_ALIGNMENT), GFP_KERNEL | GFP_DMA);
+			 SMS_ALLOC_ALIGNMENT), GFP_KERNEL | coredev->gfp_buf_flags);
 	if (!fw_buf) {
 		pr_err("failed to allocate firmware buffer\n");
 		rc = -ENOMEM;
@@ -1260,7 +1262,7 @@ EXPORT_SYMBOL_GPL(smscore_unregister_device);
 static int smscore_detect_mode(struct smscore_device_t *coredev)
 {
 	void *buffer = kmalloc(sizeof(struct sms_msg_hdr) + SMS_DMA_ALIGNMENT,
-			       GFP_KERNEL | GFP_DMA);
+			       GFP_KERNEL | coredev->gfp_buf_flags);
 	struct sms_msg_hdr *msg =
 		(struct sms_msg_hdr *) SMS_ALIGN_ADDRESS(buffer);
 	int rc;
@@ -1309,7 +1311,7 @@ static int smscore_init_device(struct smscore_device_t *coredev, int mode)
 	int rc = 0;
 
 	buffer = kmalloc(sizeof(struct sms_msg_data) +
-			SMS_DMA_ALIGNMENT, GFP_KERNEL | GFP_DMA);
+			SMS_DMA_ALIGNMENT, GFP_KERNEL | coredev->gfp_buf_flags);
 	if (!buffer)
 		return -ENOMEM;
 
@@ -1398,7 +1400,7 @@ int smscore_set_device_mode(struct smscore_device_t *coredev, int mode)
 		coredev->device_flags &= ~SMS_DEVICE_NOT_READY;
 
 		buffer = kmalloc(sizeof(struct sms_msg_data) +
-				 SMS_DMA_ALIGNMENT, GFP_KERNEL | GFP_DMA);
+				 SMS_DMA_ALIGNMENT, GFP_KERNEL | coredev->gfp_buf_flags);
 		if (buffer) {
 			struct sms_msg_data *msg = (struct sms_msg_data *) SMS_ALIGN_ADDRESS(buffer);
 
@@ -1971,7 +1973,7 @@ int smscore_gpio_configure(struct smscore_device_t *coredev, u8 pin_num,
 	total_len = sizeof(struct sms_msg_hdr) + (sizeof(u32) * 6);
 
 	buffer = kmalloc(total_len + SMS_DMA_ALIGNMENT,
-			GFP_KERNEL | GFP_DMA);
+			GFP_KERNEL | coredev->gfp_buf_flags);
 	if (!buffer)
 		return -ENOMEM;
 
@@ -2043,7 +2045,7 @@ int smscore_gpio_set_level(struct smscore_device_t *coredev, u8 pin_num,
 			(3 * sizeof(u32)); /* keep it 3 ! */
 
 	buffer = kmalloc(total_len + SMS_DMA_ALIGNMENT,
-			GFP_KERNEL | GFP_DMA);
+			GFP_KERNEL | coredev->gfp_buf_flags);
 	if (!buffer)
 		return -ENOMEM;
 
@@ -2091,7 +2093,7 @@ int smscore_gpio_get_level(struct smscore_device_t *coredev, u8 pin_num,
 	total_len = sizeof(struct sms_msg_hdr) + (2 * sizeof(u32));
 
 	buffer = kmalloc(total_len + SMS_DMA_ALIGNMENT,
-			GFP_KERNEL | GFP_DMA);
+			GFP_KERNEL | coredev->gfp_buf_flags);
 	if (!buffer)
 		return -ENOMEM;
 
diff --git a/drivers/media/common/siano/smscoreapi.h b/drivers/media/common/siano/smscoreapi.h
index 134c69f7ea7b..eb58853008c9 100644
--- a/drivers/media/common/siano/smscoreapi.h
+++ b/drivers/media/common/siano/smscoreapi.h
@@ -190,6 +190,8 @@ struct smscore_device_t {
 
 	int mode, modes_supported;
 
+	gfp_t gfp_buf_flags;
+
 	/* host <--> device messages */
 	struct completion version_ex_done, data_download_done, trigger_done;
 	struct completion data_validity_done, device_ready_done;
@@ -1125,6 +1127,7 @@ extern void smscore_unregister_hotplug(hotplug_t hotplug);
 
 extern int smscore_register_device(struct smsdevice_params_t *params,
 				   struct smscore_device_t **coredev,
+				   gfp_t gfp_buf_flags,
 				   void *mdev);
 extern void smscore_unregister_device(struct smscore_device_t *coredev);
 
diff --git a/drivers/media/common/siano/smsendian.c b/drivers/media/common/siano/smsendian.c
index bfe831c10b1c..b95a631f23f9 100644
--- a/drivers/media/common/siano/smsendian.c
+++ b/drivers/media/common/siano/smsendian.c
@@ -35,7 +35,7 @@ void smsendian_handle_tx_message(void *buffer)
 	switch (msg->x_msg_header.msg_type) {
 	case MSG_SMS_DATA_DOWNLOAD_REQ:
 	{
-		msg->msg_data[0] = le32_to_cpu(msg->msg_data[0]);
+		msg->msg_data[0] = le32_to_cpu((__force __le32)(msg->msg_data[0]));
 		break;
 	}
 
@@ -44,7 +44,7 @@ void smsendian_handle_tx_message(void *buffer)
 				sizeof(struct sms_msg_hdr))/4;
 
 		for (i = 0; i < msg_words; i++)
-			msg->msg_data[i] = le32_to_cpu(msg->msg_data[i]);
+			msg->msg_data[i] = le32_to_cpu((__force __le32)msg->msg_data[i]);
 
 		break;
 	}
@@ -64,7 +64,7 @@ void smsendian_handle_rx_message(void *buffer)
 	{
 		struct sms_version_res *ver =
 			(struct sms_version_res *) msg;
-		ver->chip_model = le16_to_cpu(ver->chip_model);
+		ver->chip_model = le16_to_cpu((__force __le16)ver->chip_model);
 		break;
 	}
 
@@ -81,7 +81,7 @@ void smsendian_handle_rx_message(void *buffer)
 				sizeof(struct sms_msg_hdr))/4;
 
 		for (i = 0; i < msg_words; i++)
-			msg->msg_data[i] = le32_to_cpu(msg->msg_data[i]);
+			msg->msg_data[i] = le32_to_cpu((__force __le32)msg->msg_data[i]);
 
 		break;
 	}
@@ -95,9 +95,9 @@ void smsendian_handle_message_header(void *msg)
 #ifdef __BIG_ENDIAN
 	struct sms_msg_hdr *phdr = (struct sms_msg_hdr *)msg;
 
-	phdr->msg_type = le16_to_cpu(phdr->msg_type);
-	phdr->msg_length = le16_to_cpu(phdr->msg_length);
-	phdr->msg_flags = le16_to_cpu(phdr->msg_flags);
+	phdr->msg_type = le16_to_cpu((__force __le16)phdr->msg_type);
+	phdr->msg_length = le16_to_cpu((__force __le16)phdr->msg_length);
+	phdr->msg_flags = le16_to_cpu((__force __le16)phdr->msg_flags);
 #endif /* __BIG_ENDIAN */
 }
 EXPORT_SYMBOL_GPL(smsendian_handle_message_header);
diff --git a/drivers/media/common/videobuf2/Kconfig b/drivers/media/common/videobuf2/Kconfig
index 17c32ea58395..4ed11b46676a 100644
--- a/drivers/media/common/videobuf2/Kconfig
+++ b/drivers/media/common/videobuf2/Kconfig
@@ -12,7 +12,6 @@ config VIDEOBUF2_MEMOPS
 
 config VIDEOBUF2_DMA_CONTIG
 	tristate
-	depends on HAS_DMA
 	select VIDEOBUF2_CORE
 	select VIDEOBUF2_MEMOPS
 	select DMA_SHARED_BUFFER
@@ -25,7 +24,6 @@ config VIDEOBUF2_VMALLOC
 
 config VIDEOBUF2_DMA_SG
 	tristate
-	depends on HAS_DMA
 	select VIDEOBUF2_CORE
 	select VIDEOBUF2_MEMOPS
 
diff --git a/drivers/media/common/videobuf2/videobuf2-core.c b/drivers/media/common/videobuf2/videobuf2-core.c
index d3f7bb33a54d..f32ec7342ef0 100644
--- a/drivers/media/common/videobuf2/videobuf2-core.c
+++ b/drivers/media/common/videobuf2/videobuf2-core.c
@@ -916,9 +916,12 @@ void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
 	dprintk(4, "done processing on buffer %d, state: %d\n",
 			vb->index, state);
 
-	/* sync buffers */
-	for (plane = 0; plane < vb->num_planes; ++plane)
-		call_void_memop(vb, finish, vb->planes[plane].mem_priv);
+	if (state != VB2_BUF_STATE_QUEUED &&
+	    state != VB2_BUF_STATE_REQUEUEING) {
+		/* sync buffers */
+		for (plane = 0; plane < vb->num_planes; ++plane)
+			call_void_memop(vb, finish, vb->planes[plane].mem_priv);
+	}
 
 	spin_lock_irqsave(&q->done_lock, flags);
 	if (state == VB2_BUF_STATE_QUEUED ||
diff --git a/drivers/media/dvb-core/dmxdev.c b/drivers/media/dvb-core/dmxdev.c
index 61a750fae465..cb078d688c70 100644
--- a/drivers/media/dvb-core/dmxdev.c
+++ b/drivers/media/dvb-core/dmxdev.c
@@ -622,7 +622,7 @@ static int dvb_dmxdev_start_feed(struct dmxdev *dmxdev,
 				 struct dmxdev_filter *filter,
 				 struct dmxdev_feed *feed)
 {
-	ktime_t timeout = 0;
+	ktime_t timeout = ktime_set(0, 0);
 	struct dmx_pes_filter_params *para = &filter->params.pes;
 	enum dmx_output otype;
 	int ret;
diff --git a/drivers/media/dvb-core/dvb_ca_en50221.c b/drivers/media/dvb-core/dvb_ca_en50221.c
index 97365a863519..1310526b0d49 100644
--- a/drivers/media/dvb-core/dvb_ca_en50221.c
+++ b/drivers/media/dvb-core/dvb_ca_en50221.c
@@ -31,6 +31,7 @@
 #include <linux/slab.h>
 #include <linux/list.h>
 #include <linux/module.h>
+#include <linux/nospec.h>
 #include <linux/vmalloc.h>
 #include <linux/delay.h>
 #include <linux/spinlock.h>
@@ -1476,6 +1477,7 @@ static ssize_t dvb_ca_en50221_io_write(struct file *file,
 
 	if (slot >= ca->slot_count)
 		return -EINVAL;
+	slot = array_index_nospec(slot, ca->slot_count);
 	sl = &ca->slot_info[slot];
 
 	/* check if the slot is actually running */
diff --git a/drivers/media/dvb-core/dvb_frontend.c b/drivers/media/dvb-core/dvb_frontend.c
index e33414975065..ce25aef39008 100644
--- a/drivers/media/dvb-core/dvb_frontend.c
+++ b/drivers/media/dvb-core/dvb_frontend.c
@@ -190,7 +190,7 @@ dtv_property_legacy_params_sync(struct dvb_frontend *fe,
 
 static bool has_get_frontend(struct dvb_frontend *fe)
 {
-	return fe->ops.get_frontend != NULL;
+	return fe->ops.get_frontend;
 }
 
 /*
@@ -272,11 +272,23 @@ static void dvb_frontend_add_event(struct dvb_frontend *fe,
 
 	mutex_unlock(&events->mtx);
 
-	wake_up_interruptible (&events->wait_queue);
+	wake_up_interruptible(&events->wait_queue);
+}
+
+static int dvb_frontend_test_event(struct dvb_frontend_private *fepriv,
+				   struct dvb_fe_events *events)
+{
+	int ret;
+
+	up(&fepriv->sem);
+	ret = events->eventw != events->eventr;
+	down(&fepriv->sem);
+
+	return ret;
 }
 
 static int dvb_frontend_get_event(struct dvb_frontend *fe,
-			    struct dvb_frontend_event *event, int flags)
+				  struct dvb_frontend_event *event, int flags)
 {
 	struct dvb_frontend_private *fepriv = fe->frontend_priv;
 	struct dvb_fe_events *events = &fepriv->events;
@@ -294,13 +306,8 @@ static int dvb_frontend_get_event(struct dvb_frontend *fe,
 		if (flags & O_NONBLOCK)
 			return -EWOULDBLOCK;
 
-		up(&fepriv->sem);
-
-		ret = wait_event_interruptible (events->wait_queue,
-						events->eventw != events->eventr);
-
-		if (down_interruptible (&fepriv->sem))
-			return -ERESTARTSYS;
+		ret = wait_event_interruptible(events->wait_queue,
+					       dvb_frontend_test_event(fepriv, events));
 
 		if (ret < 0)
 			return ret;
@@ -327,8 +334,8 @@ static void dvb_frontend_clear_events(struct dvb_frontend *fe)
 static void dvb_frontend_init(struct dvb_frontend *fe)
 {
 	dev_dbg(fe->dvb->device,
-			"%s: initialising adapter %i frontend %i (%s)...\n",
-			__func__, fe->dvb->num, fe->id, fe->ops.info.name);
+		"%s: initialising adapter %i frontend %i (%s)...\n",
+		__func__, fe->dvb->num, fe->id, fe->ops.info.name);
 
 	if (fe->ops.init)
 		fe->ops.init(fe);
@@ -358,14 +365,14 @@ static void dvb_frontend_swzigzag_update_delay(struct dvb_frontend_private *fepr
 	dev_dbg(fe->dvb->device, "%s:\n", __func__);
 
 	if (locked)
-		(fepriv->quality) = (fepriv->quality * 220 + 36*256) / 256;
+		(fepriv->quality) = (fepriv->quality * 220 + 36 * 256) / 256;
 	else
 		(fepriv->quality) = (fepriv->quality * 220 + 0) / 256;
 
 	q2 = fepriv->quality - 128;
 	q2 *= q2;
 
-	fepriv->delay = fepriv->min_delay + q2 * HZ / (128*128);
+	fepriv->delay = fepriv->min_delay + q2 * HZ / (128 * 128);
 }
 
 /**
@@ -393,7 +400,7 @@ static int dvb_frontend_swzigzag_autotune(struct dvb_frontend *fe, int check_wra
 			 (c->inversion == INVERSION_AUTO));
 
 	/* setup parameters correctly */
-	while(!ready) {
+	while (!ready) {
 		/* calculate the lnb_drift */
 		fepriv->lnb_drift = fepriv->auto_step * fepriv->step_size;
 
@@ -405,7 +412,7 @@ static int dvb_frontend_swzigzag_autotune(struct dvb_frontend *fe, int check_wra
 		}
 
 		/* perform inversion and +/- zigzag */
-		switch(fepriv->auto_sub_step) {
+		switch (fepriv->auto_sub_step) {
 		case 0:
 			/* try with the current inversion and current drift setting */
 			ready = 1;
@@ -450,11 +457,11 @@ static int dvb_frontend_swzigzag_autotune(struct dvb_frontend *fe, int check_wra
 		return 1;
 	}
 
-	dev_dbg(fe->dvb->device, "%s: drift:%i inversion:%i auto_step:%i " \
-			"auto_sub_step:%i started_auto_step:%i\n",
-			__func__, fepriv->lnb_drift, fepriv->inversion,
-			fepriv->auto_step, fepriv->auto_sub_step,
-			fepriv->started_auto_step);
+	dev_dbg(fe->dvb->device,
+		"%s: drift:%i inversion:%i auto_step:%i auto_sub_step:%i started_auto_step:%i\n",
+		__func__, fepriv->lnb_drift, fepriv->inversion,
+		fepriv->auto_step, fepriv->auto_sub_step,
+		fepriv->started_auto_step);
 
 	/* set the frontend itself */
 	c->frequency += fepriv->lnb_drift;
@@ -485,7 +492,7 @@ static void dvb_frontend_swzigzag(struct dvb_frontend *fe)
 
 	/* if we've got no parameters, just keep idling */
 	if (fepriv->state & FESTATE_IDLE) {
-		fepriv->delay = 3*HZ;
+		fepriv->delay = 3 * HZ;
 		fepriv->quality = 0;
 		return;
 	}
@@ -502,7 +509,7 @@ static void dvb_frontend_swzigzag(struct dvb_frontend *fe)
 			else
 				fepriv->state = FESTATE_TUNED;
 		}
-		fepriv->delay = 3*HZ;
+		fepriv->delay = 3 * HZ;
 		fepriv->quality = 0;
 		return;
 	}
@@ -591,7 +598,7 @@ static void dvb_frontend_swzigzag(struct dvb_frontend *fe)
 		}
 		fepriv->check_wrapped = 1;
 
-		/* if we've just retuned, enter the ZIGZAG_FAST state.
+		/* if we've just re-tuned, enter the ZIGZAG_FAST state.
 		 * This ensures we cannot return from an
 		 * FE_SET_FRONTEND ioctl before the first frontend tune
 		 * occurs */
@@ -658,7 +665,7 @@ static int dvb_frontend_thread(void *data)
 
 	fepriv->check_wrapped = 0;
 	fepriv->quality = 0;
-	fepriv->delay = 3*HZ;
+	fepriv->delay = 3 * HZ;
 	fepriv->status = 0;
 	fepriv->wakeup = 0;
 	fepriv->reinitialise = 0;
@@ -670,8 +677,9 @@ static int dvb_frontend_thread(void *data)
 		up(&fepriv->sem);	    /* is locked when we enter the thread... */
 restart:
 		wait_event_interruptible_timeout(fepriv->wait_queue,
-			dvb_frontend_should_wakeup(fe) || kthread_should_stop()
-				|| freezing(current),
+						 dvb_frontend_should_wakeup(fe) ||
+						 kthread_should_stop() ||
+						 freezing(current),
 			fepriv->delay);
 
 		if (kthread_should_stop() || dvb_frontend_is_exiting(fe)) {
@@ -820,8 +828,8 @@ static void dvb_frontend_stop(struct dvb_frontend *fe)
 	/* paranoia check in case a signal arrived */
 	if (fepriv->thread)
 		dev_warn(fe->dvb->device,
-				"dvb_frontend_stop: warning: thread %p won't exit\n",
-				fepriv->thread);
+			 "dvb_frontend_stop: warning: thread %p won't exit\n",
+			 fepriv->thread);
 }
 
 /*
@@ -858,12 +866,12 @@ static int dvb_frontend_start(struct dvb_frontend *fe)
 		if (fe->exit == DVB_FE_NO_EXIT)
 			return 0;
 		else
-			dvb_frontend_stop (fe);
+			dvb_frontend_stop(fe);
 	}
 
 	if (signal_pending(current))
 		return -EINTR;
-	if (down_interruptible (&fepriv->sem))
+	if (down_interruptible(&fepriv->sem))
 		return -EINTR;
 
 	fepriv->state = FESTATE_IDLE;
@@ -872,12 +880,12 @@ static int dvb_frontend_start(struct dvb_frontend *fe)
 	mb();
 
 	fe_thread = kthread_run(dvb_frontend_thread, fe,
-		"kdvb-ad-%i-fe-%i", fe->dvb->num,fe->id);
+				"kdvb-ad-%i-fe-%i", fe->dvb->num, fe->id);
 	if (IS_ERR(fe_thread)) {
 		ret = PTR_ERR(fe_thread);
 		dev_warn(fe->dvb->device,
-				"dvb_frontend_start: failed to start kthread (%d)\n",
-				ret);
+			 "dvb_frontend_start: failed to start kthread (%d)\n",
+			 ret);
 		up(&fepriv->sem);
 		return ret;
 	}
@@ -886,7 +894,7 @@ static int dvb_frontend_start(struct dvb_frontend *fe)
 }
 
 static void dvb_frontend_get_frequency_limits(struct dvb_frontend *fe,
-					u32 *freq_min, u32 *freq_max)
+					      u32 *freq_min, u32 *freq_max)
 {
 	*freq_min = max(fe->ops.info.frequency_min, fe->ops.tuner_ops.info.frequency_min);
 
@@ -898,8 +906,9 @@ static void dvb_frontend_get_frequency_limits(struct dvb_frontend *fe,
 		*freq_max = min(fe->ops.info.frequency_max, fe->ops.tuner_ops.info.frequency_max);
 
 	if (*freq_min == 0 || *freq_max == 0)
-		dev_warn(fe->dvb->device, "DVB: adapter %i frontend %u frequency limits undefined - fix the driver\n",
-				fe->dvb->num, fe->id);
+		dev_warn(fe->dvb->device,
+			 "DVB: adapter %i frontend %u frequency limits undefined - fix the driver\n",
+			 fe->dvb->num, fe->id);
 }
 
 static int dvb_frontend_check_parameters(struct dvb_frontend *fe)
@@ -913,8 +922,8 @@ static int dvb_frontend_check_parameters(struct dvb_frontend *fe)
 	if ((freq_min && c->frequency < freq_min) ||
 	    (freq_max && c->frequency > freq_max)) {
 		dev_warn(fe->dvb->device, "DVB: adapter %i frontend %i frequency %u out of range (%u..%u)\n",
-				fe->dvb->num, fe->id, c->frequency,
-				freq_min, freq_max);
+			 fe->dvb->num, fe->id, c->frequency,
+			 freq_min, freq_max);
 		return -EINVAL;
 	}
 
@@ -930,9 +939,9 @@ static int dvb_frontend_check_parameters(struct dvb_frontend *fe)
 		    (fe->ops.info.symbol_rate_max &&
 		     c->symbol_rate > fe->ops.info.symbol_rate_max)) {
 			dev_warn(fe->dvb->device, "DVB: adapter %i frontend %i symbol rate %u out of range (%u..%u)\n",
-					fe->dvb->num, fe->id, c->symbol_rate,
-					fe->ops.info.symbol_rate_min,
-					fe->ops.info.symbol_rate_max);
+				 fe->dvb->num, fe->id, c->symbol_rate,
+				 fe->ops.info.symbol_rate_min,
+				 fe->ops.info.symbol_rate_max);
 			return -EINVAL;
 		}
 	default:
@@ -953,7 +962,7 @@ static int dvb_frontend_clear_cache(struct dvb_frontend *fe)
 	c->delivery_system = delsys;
 
 	dev_dbg(fe->dvb->device, "%s: Clearing cache for delivery system %d\n",
-			__func__, c->delivery_system);
+		__func__, c->delivery_system);
 
 	c->transmission_mode = TRANSMISSION_MODE_AUTO;
 	c->bandwidth_hz = 0;	/* AUTO */
@@ -973,7 +982,7 @@ static int dvb_frontend_clear_cache(struct dvb_frontend *fe)
 	c->isdbt_sb_subchannel = 0;
 	c->isdbt_sb_segment_idx = 0;
 	c->isdbt_sb_segment_count = 0;
-	c->isdbt_layer_enabled = 0;
+	c->isdbt_layer_enabled = 7;	/* All layers (A,B,C) */
 	for (i = 0; i < 3; i++) {
 		c->layer[i].fec = FEC_AUTO;
 		c->layer[i].modulation = QAM_AUTO;
@@ -1178,8 +1187,8 @@ static int dtv_property_cache_sync(struct dvb_frontend *fe,
 		break;
 	case DVBV3_UNKNOWN:
 		dev_err(fe->dvb->device,
-				"%s: doesn't know how to handle a DVBv3 call to delivery system %i\n",
-				__func__, c->delivery_system);
+			"%s: doesn't know how to handle a DVBv3 call to delivery system %i\n",
+			__func__, c->delivery_system);
 		return -EINVAL;
 	}
 
@@ -1200,8 +1209,8 @@ dtv_property_legacy_params_sync(struct dvb_frontend *fe,
 	switch (dvbv3_type(c->delivery_system)) {
 	case DVBV3_UNKNOWN:
 		dev_err(fe->dvb->device,
-				"%s: doesn't know how to handle a DVBv3 call to delivery system %i\n",
-				__func__, c->delivery_system);
+			"%s: doesn't know how to handle a DVBv3 call to delivery system %i\n",
+			__func__, c->delivery_system);
 		return -EINVAL;
 	case DVBV3_QPSK:
 		dev_dbg(fe->dvb->device, "%s: Preparing QPSK req\n", __func__);
@@ -1293,7 +1302,7 @@ static int dtv_property_process_get(struct dvb_frontend *fe,
 {
 	int ncaps;
 
-	switch(tvp->cmd) {
+	switch (tvp->cmd) {
 	case DTV_ENUM_DELSYS:
 		ncaps = 0;
 		while (ncaps < MAX_DELSYS && fe->ops.delsys[ncaps]) {
@@ -1622,8 +1631,8 @@ static int dvbv5_set_delivery_system(struct dvb_frontend *fe,
 		if (fe->ops.delsys[ncaps] == desired_system) {
 			c->delivery_system = desired_system;
 			dev_dbg(fe->dvb->device,
-					"%s: Changing delivery system to %d\n",
-					__func__, desired_system);
+				"%s: Changing delivery system to %d\n",
+				__func__, desired_system);
 			return 0;
 		}
 		ncaps++;
@@ -1715,8 +1724,8 @@ static int dvbv3_set_delivery_system(struct dvb_frontend *fe)
 	 */
 	if (is_dvbv3_delsys(c->delivery_system)) {
 		dev_dbg(fe->dvb->device,
-				"%s: Using delivery system to %d\n",
-				__func__, c->delivery_system);
+			"%s: Using delivery system to %d\n",
+			__func__, c->delivery_system);
 		return 0;
 	}
 
@@ -1756,8 +1765,8 @@ static int dvbv3_set_delivery_system(struct dvb_frontend *fe)
  * Zero on success, negative errno on failure.
  */
 static int dtv_property_process_set(struct dvb_frontend *fe,
-					struct file *file,
-					u32 cmd, u32 data)
+				    struct file *file,
+				    u32 cmd, u32 data)
 {
 	int r = 0;
 	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
@@ -1765,11 +1774,11 @@ static int dtv_property_process_set(struct dvb_frontend *fe,
 	/** Dump DTV command name and value*/
 	if (!cmd || cmd > DTV_MAX_COMMAND)
 		dev_warn(fe->dvb->device, "%s: SET cmd 0x%08x undefined\n",
-				 __func__, cmd);
+			 __func__, cmd);
 	else
 		dev_dbg(fe->dvb->device,
-				"%s: SET cmd 0x%08x (%s) to 0x%08x\n",
-				__func__, cmd, dtv_cmds[cmd].name, data);
+			"%s: SET cmd 0x%08x (%s) to 0x%08x\n",
+			__func__, cmd, dtv_cmds[cmd].name, data);
 	switch (cmd) {
 	case DTV_CLEAR:
 		/*
@@ -1819,12 +1828,12 @@ static int dtv_property_process_set(struct dvb_frontend *fe,
 	case DTV_VOLTAGE:
 		c->voltage = data;
 		r = dvb_frontend_handle_ioctl(file, FE_SET_VOLTAGE,
-			(void *)c->voltage);
+					      (void *)c->voltage);
 		break;
 	case DTV_TONE:
 		c->sectone = data;
 		r = dvb_frontend_handle_ioctl(file, FE_SET_TONE,
-			(void *)c->sectone);
+					      (void *)c->sectone);
 		break;
 	case DTV_CODE_RATE_HP:
 		c->code_rate_HP = data;
@@ -2045,8 +2054,8 @@ static int dvb_frontend_handle_compat_ioctl(struct file *file, unsigned int cmd,
 
 		for (i = 0; i < tvps->num; i++) {
 			err = dtv_property_process_set(fe, file,
-							(tvp + i)->cmd,
-							(tvp + i)->u.data);
+						       (tvp + i)->cmd,
+						       (tvp + i)->u.data);
 			if (err < 0) {
 				kfree(tvp);
 				return err;
@@ -2265,7 +2274,6 @@ static int dtv_set_frontend(struct dvb_frontend *fe)
 	return 0;
 }
 
-
 static int dvb_frontend_handle_ioctl(struct file *file,
 				     unsigned int cmd, void *parg)
 {
@@ -2300,8 +2308,8 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 
 		for (i = 0; i < tvps->num; i++) {
 			err = dtv_property_process_set(fe, file,
-							(tvp + i)->cmd,
-							(tvp + i)->u.data);
+						       (tvp + i)->cmd,
+						       (tvp + i)->u.data);
 			if (err < 0) {
 				kfree(tvp);
 				return err;
@@ -2365,7 +2373,7 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 	}
 
 	case FE_GET_INFO: {
-		struct dvb_frontend_info* info = parg;
+		struct dvb_frontend_info *info = parg;
 
 		memcpy(info, &fe->ops.info, sizeof(struct dvb_frontend_info));
 		dvb_frontend_get_frequency_limits(fe, &info->frequency_min, &info->frequency_max);
@@ -2396,12 +2404,12 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 			break;
 		default:
 			dev_err(fe->dvb->device,
-					"%s: doesn't know how to handle a DVBv3 call to delivery system %i\n",
-					__func__, c->delivery_system);
+				"%s: doesn't know how to handle a DVBv3 call to delivery system %i\n",
+				__func__, c->delivery_system);
 			fe->ops.info.type = FE_OFDM;
 		}
 		dev_dbg(fe->dvb->device, "%s: current delivery system on cache: %d, V3 type: %d\n",
-				 __func__, c->delivery_system, fe->ops.info.type);
+			__func__, c->delivery_system, fe->ops.info.type);
 
 		/* Set CAN_INVERSION_AUTO bit on in other than oneshot mode */
 		if (!(fepriv->tune_mode_flags & FE_TUNE_MODE_ONESHOT))
@@ -2417,7 +2425,7 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 		 * that user get signal state from previous tuning */
 		if (fepriv->state == FESTATE_RETUNE ||
 		    fepriv->state == FESTATE_ERROR) {
-			err=0;
+			err = 0;
 			*status = 0;
 			break;
 		}
@@ -2485,11 +2493,11 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 
 	case FE_ENABLE_HIGH_LNB_VOLTAGE:
 		if (fe->ops.enable_high_lnb_voltage)
-			err = fe->ops.enable_high_lnb_voltage(fe, (long) parg);
+			err = fe->ops.enable_high_lnb_voltage(fe, (long)parg);
 		break;
 
 	case FE_SET_FRONTEND_TUNE_MODE:
-		fepriv->tune_mode_flags = (unsigned long) parg;
+		fepriv->tune_mode_flags = (unsigned long)parg;
 		err = 0;
 		break;
 
@@ -2518,11 +2526,12 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 			 * initialization, so parg is 8 bits and does not
 			 * include the initialization or start bit
 			 */
-			unsigned long swcmd = ((unsigned long) parg) << 1;
+			unsigned long swcmd = ((unsigned long)parg) << 1;
 			ktime_t nexttime;
 			ktime_t tv[10];
 			int i;
 			u8 last = 1;
+
 			if (dvb_frontend_debug)
 				dprintk("%s switch command: 0x%04lx\n",
 					__func__, swcmd);
@@ -2537,7 +2546,7 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 
 			for (i = 0; i < 9; i++) {
 				if (dvb_frontend_debug)
-					tv[i+1] = ktime_get_boottime();
+					tv[i + 1] = ktime_get_boottime();
 				if ((swcmd & 0x01) != last) {
 					/* set voltage to (last ? 13V : 18V) */
 					fe->ops.set_voltage(fe, (last) ? SEC_VOLTAGE_13 : SEC_VOLTAGE_18);
@@ -2552,7 +2561,7 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 					__func__, fe->dvb->num);
 				for (i = 1; i < 10; i++)
 					pr_info("%d: %d\n", i,
-					(int) ktime_us_delta(tv[i], tv[i-1]));
+						(int)ktime_us_delta(tv[i], tv[i - 1]));
 			}
 			err = 0;
 			fepriv->state = FESTATE_DISEQC;
@@ -2611,7 +2620,7 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 		err = dtv_set_frontend(fe);
 		break;
 	case FE_GET_EVENT:
-		err = dvb_frontend_get_event (fe, parg, file->f_flags);
+		err = dvb_frontend_get_event(fe, parg, file->f_flags);
 		break;
 
 	case FE_GET_FRONTEND: {
@@ -2634,7 +2643,6 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 	return err;
 }
 
-
 static __poll_t dvb_frontend_poll(struct file *file, struct poll_table_struct *wait)
 {
 	struct dvb_device *dvbdev = file->private_data;
@@ -2643,7 +2651,7 @@ static __poll_t dvb_frontend_poll(struct file *file, struct poll_table_struct *w
 
 	dev_dbg_ratelimited(fe->dvb->device, "%s:\n", __func__);
 
-	poll_wait (file, &fepriv->events.wait_queue, wait);
+	poll_wait(file, &fepriv->events.wait_queue, wait);
 
 	if (fepriv->events.eventw != fepriv->events.eventr)
 		return (EPOLLIN | EPOLLRDNORM | EPOLLPRI);
@@ -2664,9 +2672,9 @@ static int dvb_frontend_open(struct inode *inode, struct file *file)
 		return -ENODEV;
 
 	if (adapter->mfe_shared) {
-		mutex_lock (&adapter->mfe_lock);
+		mutex_lock(&adapter->mfe_lock);
 
-		if (adapter->mfe_dvbdev == NULL)
+		if (!adapter->mfe_dvbdev)
 			adapter->mfe_dvbdev = dvbdev;
 
 		else if (adapter->mfe_dvbdev != dvbdev) {
@@ -2678,23 +2686,23 @@ static int dvb_frontend_open(struct inode *inode, struct file *file)
 				*mfepriv = mfe->frontend_priv;
 			int mferetry = (dvb_mfe_wait_time << 1);
 
-			mutex_unlock (&adapter->mfe_lock);
+			mutex_unlock(&adapter->mfe_lock);
 			while (mferetry-- && (mfedev->users != -1 ||
-					mfepriv->thread != NULL)) {
-				if(msleep_interruptible(500)) {
-					if(signal_pending(current))
+					      mfepriv->thread)) {
+				if (msleep_interruptible(500)) {
+					if (signal_pending(current))
 						return -EINTR;
 				}
 			}
 
-			mutex_lock (&adapter->mfe_lock);
-			if(adapter->mfe_dvbdev != dvbdev) {
+			mutex_lock(&adapter->mfe_lock);
+			if (adapter->mfe_dvbdev != dvbdev) {
 				mfedev = adapter->mfe_dvbdev;
 				mfe = mfedev->priv;
 				mfepriv = mfe->frontend_priv;
 				if (mfedev->users != -1 ||
-						mfepriv->thread != NULL) {
-					mutex_unlock (&adapter->mfe_lock);
+				    mfepriv->thread) {
+					mutex_unlock(&adapter->mfe_lock);
 					return -EBUSY;
 				}
 				adapter->mfe_dvbdev = dvbdev;
@@ -2715,7 +2723,7 @@ static int dvb_frontend_open(struct inode *inode, struct file *file)
 		fepriv->reinitialise = 1;
 	}
 
-	if ((ret = dvb_generic_open (inode, file)) < 0)
+	if ((ret = dvb_generic_open(inode, file)) < 0)
 		goto err1;
 
 	if ((file->f_flags & O_ACCMODE) != O_RDONLY) {
@@ -2725,6 +2733,7 @@ static int dvb_frontend_open(struct inode *inode, struct file *file)
 		fepriv->voltage = -1;
 
 #ifdef CONFIG_MEDIA_CONTROLLER_DVB
+		mutex_lock(&fe->dvb->mdev_lock);
 		if (fe->dvb->mdev) {
 			mutex_lock(&fe->dvb->mdev->graph_mutex);
 			if (fe->dvb->mdev->enable_source)
@@ -2733,13 +2742,15 @@ static int dvb_frontend_open(struct inode *inode, struct file *file)
 							   &fepriv->pipe);
 			mutex_unlock(&fe->dvb->mdev->graph_mutex);
 			if (ret) {
+				mutex_unlock(&fe->dvb->mdev_lock);
 				dev_err(fe->dvb->device,
 					"Tuner is busy. Error %d\n", ret);
 				goto err2;
 			}
 		}
+		mutex_unlock(&fe->dvb->mdev_lock);
 #endif
-		ret = dvb_frontend_start (fe);
+		ret = dvb_frontend_start(fe);
 		if (ret)
 			goto err3;
 
@@ -2750,17 +2761,19 @@ static int dvb_frontend_open(struct inode *inode, struct file *file)
 	dvb_frontend_get(fe);
 
 	if (adapter->mfe_shared)
-		mutex_unlock (&adapter->mfe_lock);
+		mutex_unlock(&adapter->mfe_lock);
 	return ret;
 
 err3:
 #ifdef CONFIG_MEDIA_CONTROLLER_DVB
+	mutex_lock(&fe->dvb->mdev_lock);
 	if (fe->dvb->mdev) {
 		mutex_lock(&fe->dvb->mdev->graph_mutex);
 		if (fe->dvb->mdev->disable_source)
 			fe->dvb->mdev->disable_source(dvbdev->entity);
 		mutex_unlock(&fe->dvb->mdev->graph_mutex);
 	}
+	mutex_unlock(&fe->dvb->mdev_lock);
 err2:
 #endif
 	dvb_generic_release(inode, file);
@@ -2769,7 +2782,7 @@ static int dvb_frontend_open(struct inode *inode, struct file *file)
 		fe->ops.ts_bus_ctrl(fe, 0);
 err0:
 	if (adapter->mfe_shared)
-		mutex_unlock (&adapter->mfe_lock);
+		mutex_unlock(&adapter->mfe_lock);
 	return ret;
 }
 
@@ -2787,17 +2800,19 @@ static int dvb_frontend_release(struct inode *inode, struct file *file)
 		mb();
 	}
 
-	ret = dvb_generic_release (inode, file);
+	ret = dvb_generic_release(inode, file);
 
 	if (dvbdev->users == -1) {
 		wake_up(&fepriv->wait_queue);
 #ifdef CONFIG_MEDIA_CONTROLLER_DVB
+		mutex_lock(&fe->dvb->mdev_lock);
 		if (fe->dvb->mdev) {
 			mutex_lock(&fe->dvb->mdev->graph_mutex);
 			if (fe->dvb->mdev->disable_source)
 				fe->dvb->mdev->disable_source(dvbdev->entity);
 			mutex_unlock(&fe->dvb->mdev->graph_mutex);
 		}
+		mutex_unlock(&fe->dvb->mdev_lock);
 #endif
 		if (fe->exit != DVB_FE_NO_EXIT)
 			wake_up(&dvbdev->wait_queue);
@@ -2827,7 +2842,7 @@ int dvb_frontend_suspend(struct dvb_frontend *fe)
 	int ret = 0;
 
 	dev_dbg(fe->dvb->device, "%s: adap=%d fe=%d\n", __func__, fe->dvb->num,
-			fe->id);
+		fe->id);
 
 	if (fe->ops.tuner_ops.suspend)
 		ret = fe->ops.tuner_ops.suspend(fe);
@@ -2847,7 +2862,7 @@ int dvb_frontend_resume(struct dvb_frontend *fe)
 	int ret = 0;
 
 	dev_dbg(fe->dvb->device, "%s: adap=%d fe=%d\n", __func__, fe->dvb->num,
-			fe->id);
+		fe->id);
 
 	fe->exit = DVB_FE_DEVICE_RESUME;
 	if (fe->ops.init)
@@ -2871,14 +2886,14 @@ int dvb_frontend_resume(struct dvb_frontend *fe)
 }
 EXPORT_SYMBOL(dvb_frontend_resume);
 
-int dvb_register_frontend(struct dvb_adapter* dvb,
-			  struct dvb_frontend* fe)
+int dvb_register_frontend(struct dvb_adapter *dvb,
+			  struct dvb_frontend *fe)
 {
 	struct dvb_frontend_private *fepriv;
 	const struct dvb_device dvbdev_template = {
 		.users = ~0,
 		.writers = 1,
-		.readers = (~0)-1,
+		.readers = (~0) - 1,
 		.fops = &dvb_frontend_fops,
 #if defined(CONFIG_MEDIA_CONTROLLER_DVB)
 		.name = fe->ops.info.name,
@@ -2891,7 +2906,7 @@ int dvb_register_frontend(struct dvb_adapter* dvb,
 		return -ERESTARTSYS;
 
 	fe->frontend_priv = kzalloc(sizeof(struct dvb_frontend_private), GFP_KERNEL);
-	if (fe->frontend_priv == NULL) {
+	if (!fe->frontend_priv) {
 		mutex_unlock(&frontend_mutex);
 		return -ENOMEM;
 	}
@@ -2907,18 +2922,18 @@ int dvb_register_frontend(struct dvb_adapter* dvb,
 	dvb_frontend_get(fe);
 
 	sema_init(&fepriv->sem, 1);
-	init_waitqueue_head (&fepriv->wait_queue);
-	init_waitqueue_head (&fepriv->events.wait_queue);
+	init_waitqueue_head(&fepriv->wait_queue);
+	init_waitqueue_head(&fepriv->events.wait_queue);
 	mutex_init(&fepriv->events.mtx);
 	fe->dvb = dvb;
 	fepriv->inversion = INVERSION_OFF;
 
 	dev_info(fe->dvb->device,
-			"DVB: registering adapter %i frontend %i (%s)...\n",
-			fe->dvb->num, fe->id, fe->ops.info.name);
+		 "DVB: registering adapter %i frontend %i (%s)...\n",
+		 fe->dvb->num, fe->id, fe->ops.info.name);
 
-	dvb_register_device (fe->dvb, &fepriv->dvbdev, &dvbdev_template,
-			     fe, DVB_DEVICE_FRONTEND, 0);
+	dvb_register_device(fe->dvb, &fepriv->dvbdev, &dvbdev_template,
+			    fe, DVB_DEVICE_FRONTEND, 0);
 
 	/*
 	 * Initialize the cache to the proper values according with the
@@ -2933,9 +2948,10 @@ int dvb_register_frontend(struct dvb_adapter* dvb,
 }
 EXPORT_SYMBOL(dvb_register_frontend);
 
-int dvb_unregister_frontend(struct dvb_frontend* fe)
+int dvb_unregister_frontend(struct dvb_frontend *fe)
 {
 	struct dvb_frontend_private *fepriv = fe->frontend_priv;
+
 	dev_dbg(fe->dvb->device, "%s:\n", __func__);
 
 	mutex_lock(&frontend_mutex);
@@ -2960,7 +2976,7 @@ static void dvb_frontend_invoke_release(struct dvb_frontend *fe,
 	}
 }
 
-void dvb_frontend_detach(struct dvb_frontend* fe)
+void dvb_frontend_detach(struct dvb_frontend *fe)
 {
 	dvb_frontend_invoke_release(fe, fe->ops.release_sec);
 	dvb_frontend_invoke_release(fe, fe->ops.tuner_ops.release);
diff --git a/drivers/media/dvb-core/dvb_net.c b/drivers/media/dvb-core/dvb_net.c
index ba39f9942e1d..10f78109bb3f 100644
--- a/drivers/media/dvb-core/dvb_net.c
+++ b/drivers/media/dvb-core/dvb_net.c
@@ -1005,7 +1005,7 @@ static int dvb_net_sec_callback(const u8 *buffer1, size_t buffer1_len,
 	return 0;
 }
 
-static int dvb_net_tx(struct sk_buff *skb, struct net_device *dev)
+static netdev_tx_t dvb_net_tx(struct sk_buff *skb, struct net_device *dev)
 {
 	dev_kfree_skb(skb);
 	return NETDEV_TX_OK;
diff --git a/drivers/media/dvb-core/dvbdev.c b/drivers/media/dvb-core/dvbdev.c
index 787fe06df217..64d6793674b9 100644
--- a/drivers/media/dvb-core/dvbdev.c
+++ b/drivers/media/dvb-core/dvbdev.c
@@ -859,6 +859,10 @@ int dvb_register_adapter(struct dvb_adapter *adap, const char *name,
 	adap->mfe_dvbdev = NULL;
 	mutex_init (&adap->mfe_lock);
 
+#ifdef CONFIG_MEDIA_CONTROLLER_DVB
+	mutex_init(&adap->mdev_lock);
+#endif
+
 	list_add_tail (&adap->list_head, &dvb_adapter_list);
 
 	mutex_unlock(&dvbdev_register_lock);
diff --git a/drivers/media/dvb-frontends/Kconfig b/drivers/media/dvb-frontends/Kconfig
index 0712069fd9fe..55e36a4f5215 100644
--- a/drivers/media/dvb-frontends/Kconfig
+++ b/drivers/media/dvb-frontends/Kconfig
@@ -903,7 +903,7 @@ comment "Common Interface (EN50221) controller drivers"
 	depends on DVB_CORE
 
 config DVB_CXD2099
-	tristate "CXD2099AR Common Interface driver"
+	tristate "Sony CXD2099AR Common Interface driver"
 	depends on DVB_CORE && I2C
 	select REGMAP_I2C
 	default m if !MEDIA_SUBDRV_AUTOSELECT
diff --git a/drivers/media/dvb-frontends/as102_fe.h b/drivers/media/dvb-frontends/as102_fe.h
index a7c91430ca3d..98d33d5ce872 100644
--- a/drivers/media/dvb-frontends/as102_fe.h
+++ b/drivers/media/dvb-frontends/as102_fe.h
@@ -1,6 +1,6 @@
 /*
  * Abilis Systems Single DVB-T Receiver
- * Copyright (C) 2014 Mauro Carvalho Chehab <m.chehab@samsung.com>
+ * Copyright (C) 2014 Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
diff --git a/drivers/media/dvb-frontends/au8522_decoder.c b/drivers/media/dvb-frontends/au8522_decoder.c
index 343dc92ef54e..f285096a48f0 100644
--- a/drivers/media/dvb-frontends/au8522_decoder.c
+++ b/drivers/media/dvb-frontends/au8522_decoder.c
@@ -280,14 +280,12 @@ static void setup_decoder_defaults(struct au8522_state *state, bool is_svideo)
 			AU8522_TOREGAAGC_REG0E5H_CVBS);
 	au8522_writereg(state, AU8522_REG016H, AU8522_REG016H_CVBS);
 
-	if (is_svideo) {
-		/* Despite what the table says, for the HVR-950q we still need
-		   to be in CVBS mode for the S-Video input (reason unknown). */
-		/* filter_coef_type = 3; */
-		filter_coef_type = 5;
-	} else {
-		filter_coef_type = 5;
-	}
+	/*
+	 * Despite what the table says, for the HVR-950q we still need
+	 * to be in CVBS mode for the S-Video input (reason unknown).
+	 */
+	/* filter_coef_type = 3; */
+	filter_coef_type = 5;
 
 	/* Load the Video Decoder Filter Coefficients */
 	for (i = 0; i < NUM_FILTER_COEF; i++) {
diff --git a/drivers/media/dvb-frontends/cx24116.c b/drivers/media/dvb-frontends/cx24116.c
index 786c56a4ef76..2dbc7349d870 100644
--- a/drivers/media/dvb-frontends/cx24116.c
+++ b/drivers/media/dvb-frontends/cx24116.c
@@ -1456,7 +1456,7 @@ static int cx24116_tune(struct dvb_frontend *fe, bool re_tune,
 	return cx24116_read_status(fe, status);
 }
 
-static int cx24116_get_algo(struct dvb_frontend *fe)
+static enum dvbfe_algo cx24116_get_algo(struct dvb_frontend *fe)
 {
 	return DVBFE_ALGO_HW;
 }
diff --git a/drivers/media/dvb-frontends/cx24117.c b/drivers/media/dvb-frontends/cx24117.c
index 8935114b75f3..ba55d75d916c 100644
--- a/drivers/media/dvb-frontends/cx24117.c
+++ b/drivers/media/dvb-frontends/cx24117.c
@@ -1555,7 +1555,7 @@ static int cx24117_tune(struct dvb_frontend *fe, bool re_tune,
 	return cx24117_read_status(fe, status);
 }
 
-static int cx24117_get_algo(struct dvb_frontend *fe)
+static enum dvbfe_algo cx24117_get_algo(struct dvb_frontend *fe)
 {
 	return DVBFE_ALGO_HW;
 }
diff --git a/drivers/media/dvb-frontends/cx24120.c b/drivers/media/dvb-frontends/cx24120.c
index 810f68acd69b..ccbabdae6a69 100644
--- a/drivers/media/dvb-frontends/cx24120.c
+++ b/drivers/media/dvb-frontends/cx24120.c
@@ -1491,7 +1491,7 @@ static int cx24120_tune(struct dvb_frontend *fe, bool re_tune,
 	return cx24120_read_status(fe, status);
 }
 
-static int cx24120_get_algo(struct dvb_frontend *fe)
+static enum dvbfe_algo cx24120_get_algo(struct dvb_frontend *fe)
 {
 	return DVBFE_ALGO_HW;
 }
diff --git a/drivers/media/dvb-frontends/cx24123.c b/drivers/media/dvb-frontends/cx24123.c
index 228ba1f4bf63..bf33e7390aaf 100644
--- a/drivers/media/dvb-frontends/cx24123.c
+++ b/drivers/media/dvb-frontends/cx24123.c
@@ -1005,7 +1005,7 @@ static int cx24123_tune(struct dvb_frontend *fe,
 	return retval;
 }
 
-static int cx24123_get_algo(struct dvb_frontend *fe)
+static enum dvbfe_algo cx24123_get_algo(struct dvb_frontend *fe)
 {
 	return DVBFE_ALGO_HW;
 }
diff --git a/drivers/media/dvb-frontends/cxd2099.c b/drivers/media/dvb-frontends/cxd2099.c
index c0a5849b76bb..4a0ce3037fd6 100644
--- a/drivers/media/dvb-frontends/cxd2099.c
+++ b/drivers/media/dvb-frontends/cxd2099.c
@@ -1,5 +1,5 @@
 /*
- * cxd2099.c: Driver for the CXD2099AR Common Interface Controller
+ * cxd2099.c: Driver for the Sony CXD2099AR Common Interface Controller
  *
  * Copyright (C) 2010-2013 Digital Devices GmbH
  *
@@ -699,6 +699,6 @@ static struct i2c_driver cxd2099_driver = {
 
 module_i2c_driver(cxd2099_driver);
 
-MODULE_DESCRIPTION("CXD2099AR Common Interface controller driver");
+MODULE_DESCRIPTION("Sony CXD2099AR Common Interface controller driver");
 MODULE_AUTHOR("Ralph Metzler");
 MODULE_LICENSE("GPL");
diff --git a/drivers/media/dvb-frontends/cxd2099.h b/drivers/media/dvb-frontends/cxd2099.h
index 8fa45a4c615a..ec1910dec3f3 100644
--- a/drivers/media/dvb-frontends/cxd2099.h
+++ b/drivers/media/dvb-frontends/cxd2099.h
@@ -1,5 +1,5 @@
 /*
- * cxd2099.h: Driver for the CXD2099AR Common Interface Controller
+ * cxd2099.h: Driver for the Sony CXD2099AR Common Interface Controller
  *
  * Copyright (C) 2010-2011 Digital Devices GmbH
  *
diff --git a/drivers/media/dvb-frontends/cxd2820r_core.c b/drivers/media/dvb-frontends/cxd2820r_core.c
index f6ebbb47b9b2..3e0d8cbd76da 100644
--- a/drivers/media/dvb-frontends/cxd2820r_core.c
+++ b/drivers/media/dvb-frontends/cxd2820r_core.c
@@ -403,7 +403,7 @@ static enum dvbfe_search cxd2820r_search(struct dvb_frontend *fe)
 	return DVBFE_ALGO_SEARCH_ERROR;
 }
 
-static int cxd2820r_get_frontend_algo(struct dvb_frontend *fe)
+static enum dvbfe_algo cxd2820r_get_frontend_algo(struct dvb_frontend *fe)
 {
 	return DVBFE_ALGO_CUSTOM;
 }
diff --git a/drivers/media/dvb-frontends/cxd2880/cxd2880_tnrdmd_driver_version.h b/drivers/media/dvb-frontends/cxd2880/cxd2880_tnrdmd_driver_version.h
index fab55038b37b..c6d6c8dd16a1 100644
--- a/drivers/media/dvb-frontends/cxd2880/cxd2880_tnrdmd_driver_version.h
+++ b/drivers/media/dvb-frontends/cxd2880/cxd2880_tnrdmd_driver_version.h
@@ -7,6 +7,6 @@
  * Copyright (C) 2016, 2017, 2018 Sony Semiconductor Solutions Corporation
  */
 
-#define CXD2880_TNRDMD_DRIVER_VERSION "1.4.1 - 1.0.4"
+#define CXD2880_TNRDMD_DRIVER_VERSION "1.4.1 - 1.0.5"
 
-#define CXD2880_TNRDMD_DRIVER_RELEASE_DATE "2018-01-17"
+#define CXD2880_TNRDMD_DRIVER_RELEASE_DATE "2018-04-25"
diff --git a/drivers/media/dvb-frontends/cxd2880/cxd2880_top.c b/drivers/media/dvb-frontends/cxd2880/cxd2880_top.c
index d474dc1b05da..bd9101e246d5 100644
--- a/drivers/media/dvb-frontends/cxd2880/cxd2880_top.c
+++ b/drivers/media/dvb-frontends/cxd2880/cxd2880_top.c
@@ -520,6 +520,15 @@ static int cxd2880_init(struct dvb_frontend *fe)
 		pr_err("cxd2880 integ init failed %d\n", ret);
 		return ret;
 	}
+
+	ret = cxd2880_tnrdmd_set_cfg(&priv->tnrdmd,
+				     CXD2880_TNRDMD_CFG_TSPIN_CURRENT,
+				     0x00);
+	if (ret) {
+		mutex_unlock(priv->spi_mutex);
+		pr_err("cxd2880 set config failed %d\n", ret);
+		return ret;
+	}
 	mutex_unlock(priv->spi_mutex);
 
 	pr_debug("OK.\n");
@@ -1126,7 +1135,7 @@ static int cxd2880_get_stats(struct dvb_frontend *fe,
 	priv = fe->demodulator_priv;
 	c = &fe->dtv_property_cache;
 
-	if (!(status & FE_HAS_LOCK)) {
+	if (!(status & FE_HAS_LOCK) || !(status & FE_HAS_CARRIER)) {
 		c->pre_bit_error.len = 1;
 		c->pre_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
 		c->pre_bit_count.len = 1;
@@ -1345,7 +1354,8 @@ static int cxd2880_read_status(struct dvb_frontend *fe,
 
 	pr_debug("status %d\n", *status);
 
-	if (priv->s == 0 && (*status & FE_HAS_LOCK)) {
+	if (priv->s == 0 && (*status & FE_HAS_LOCK) &&
+	    (*status & FE_HAS_CARRIER)) {
 		mutex_lock(priv->spi_mutex);
 		if (c->delivery_system == SYS_DVBT) {
 			ret = cxd2880_set_ber_per_period_t(fe);
diff --git a/drivers/media/dvb-frontends/dvb-pll.c b/drivers/media/dvb-frontends/dvb-pll.c
index 5553b89b804e..e3894ff403d7 100644
--- a/drivers/media/dvb-frontends/dvb-pll.c
+++ b/drivers/media/dvb-frontends/dvb-pll.c
@@ -533,6 +533,45 @@ static const struct dvb_pll_desc dvb_pll_alps_tdee4 = {
 	}
 };
 
+/* Infineon TUA6034 ISDB-T, used in Friio */
+/* CP cur. 50uA, AGC takeover: 103dBuV, PORT3 on */
+static const struct dvb_pll_desc dvb_pll_tua6034_friio = {
+	.name   = "Infineon TUA6034 ISDB-T (Friio)",
+	.min    =  90000000,
+	.max    = 770000000,
+	.iffreq =  57000000,
+	.initdata = (u8[]){ 4, 0x9a, 0x50, 0xb2, 0x08 },
+	.sleepdata = (u8[]){ 4, 0x9a, 0x70, 0xb3, 0x0b },
+	.count = 3,
+	.entries = {
+		{ 170000000, 142857, 0xba, 0x09 },
+		{ 470000000, 142857, 0xba, 0x0a },
+		{ 770000000, 142857, 0xb2, 0x08 },
+	}
+};
+
+/* Philips TDA6651 ISDB-T, used in Earthsoft PT1 */
+static const struct dvb_pll_desc dvb_pll_tda665x_earth_pt1 = {
+	.name   = "Philips TDA6651 ISDB-T (EarthSoft PT1)",
+	.min    =  90000000,
+	.max    = 770000000,
+	.iffreq =  57000000,
+	.initdata = (u8[]){ 5, 0x0e, 0x7f, 0xc1, 0x80, 0x80 },
+	.count = 10,
+	.entries = {
+		{ 140000000, 142857, 0xc1, 0x81 },
+		{ 170000000, 142857, 0xc1, 0xa1 },
+		{ 220000000, 142857, 0xc1, 0x62 },
+		{ 330000000, 142857, 0xc1, 0xa2 },
+		{ 402000000, 142857, 0xc1, 0xe2 },
+		{ 450000000, 142857, 0xc1, 0x64 },
+		{ 550000000, 142857, 0xc1, 0x84 },
+		{ 600000000, 142857, 0xc1, 0xa4 },
+		{ 700000000, 142857, 0xc1, 0xc4 },
+		{ 770000000, 142857, 0xc1, 0xe4 },
+	}
+};
+
 /* ----------------------------------------------------------- */
 
 static const struct dvb_pll_desc *pll_list[] = {
@@ -556,6 +595,8 @@ static const struct dvb_pll_desc *pll_list[] = {
 	[DVB_PLL_SAMSUNG_TDTC9251DH0]    = &dvb_pll_samsung_tdtc9251dh0,
 	[DVB_PLL_SAMSUNG_TBDU18132]	 = &dvb_pll_samsung_tbdu18132,
 	[DVB_PLL_SAMSUNG_TBMU24112]      = &dvb_pll_samsung_tbmu24112,
+	[DVB_PLL_TUA6034_FRIIO]          = &dvb_pll_tua6034_friio,
+	[DVB_PLL_TDA665X_EARTH_PT1]      = &dvb_pll_tda665x_earth_pt1,
 };
 
 /* ----------------------------------------------------------- */
@@ -827,6 +868,75 @@ struct dvb_frontend *dvb_pll_attach(struct dvb_frontend *fe, int pll_addr,
 }
 EXPORT_SYMBOL(dvb_pll_attach);
 
+
+static int
+dvb_pll_probe(struct i2c_client *client, const struct i2c_device_id *id)
+{
+	struct dvb_pll_config *cfg;
+	struct dvb_frontend *fe;
+	unsigned int desc_id;
+
+	cfg = client->dev.platform_data;
+	fe = cfg->fe;
+	i2c_set_clientdata(client, fe);
+	desc_id = (unsigned int) id->driver_data;
+
+	if (!dvb_pll_attach(fe, client->addr, client->adapter, desc_id))
+		return -ENOMEM;
+
+	dev_info(&client->dev, "DVB Simple Tuner attached.\n");
+	return 0;
+}
+
+static int dvb_pll_remove(struct i2c_client *client)
+{
+	struct dvb_frontend *fe;
+
+	fe = i2c_get_clientdata(client);
+	dvb_pll_release(fe);
+	return 0;
+}
+
+
+static const struct i2c_device_id dvb_pll_id[] = {
+	{"dtt7579",		DVB_PLL_THOMSON_DTT7579},
+	{"dtt759x",		DVB_PLL_THOMSON_DTT759X},
+	{"z201",		DVB_PLL_LG_Z201},
+	{"unknown_1",		DVB_PLL_UNKNOWN_1},
+	{"tua6010xs",		DVB_PLL_TUA6010XS},
+	{"env57h1xd5",		DVB_PLL_ENV57H1XD5},
+	{"tua6034",		DVB_PLL_TUA6034},
+	{"tda665x",		DVB_PLL_TDA665X},
+	{"tded4",		DVB_PLL_TDED4},
+	{"tdhu2",		DVB_PLL_TDHU2},
+	{"tbmv",		DVB_PLL_SAMSUNG_TBMV},
+	{"sd1878_tda8261",	DVB_PLL_PHILIPS_SD1878_TDA8261},
+	{"opera1",		DVB_PLL_OPERA1},
+	{"dtos403ih102a",	DVB_PLL_SAMSUNG_DTOS403IH102A},
+	{"tdtc9251dh0",		DVB_PLL_SAMSUNG_TDTC9251DH0},
+	{"tbdu18132",		DVB_PLL_SAMSUNG_TBDU18132},
+	{"tbmu24112",		DVB_PLL_SAMSUNG_TBMU24112},
+	{"tdee4",		DVB_PLL_TDEE4},
+	{"dtt7520x",		DVB_PLL_THOMSON_DTT7520X},
+	{"tua6034_friio",	DVB_PLL_TUA6034_FRIIO},
+	{"tda665x_earthpt1",	DVB_PLL_TDA665X_EARTH_PT1},
+	{}
+};
+
+
+MODULE_DEVICE_TABLE(i2c, dvb_pll_id);
+
+static struct i2c_driver dvb_pll_driver = {
+	.driver = {
+		.name = "dvb_pll",
+	},
+	.probe    = dvb_pll_probe,
+	.remove   = dvb_pll_remove,
+	.id_table = dvb_pll_id,
+};
+
+module_i2c_driver(dvb_pll_driver);
+
 MODULE_DESCRIPTION("dvb pll library");
 MODULE_AUTHOR("Gerd Knorr");
 MODULE_LICENSE("GPL");
diff --git a/drivers/media/dvb-frontends/dvb-pll.h b/drivers/media/dvb-frontends/dvb-pll.h
index ca885e71d2f0..973a66a82e27 100644
--- a/drivers/media/dvb-frontends/dvb-pll.h
+++ b/drivers/media/dvb-frontends/dvb-pll.h
@@ -29,6 +29,12 @@
 #define DVB_PLL_SAMSUNG_TBMU24112      17
 #define DVB_PLL_TDEE4		       18
 #define DVB_PLL_THOMSON_DTT7520X       19
+#define DVB_PLL_TUA6034_FRIIO          20
+#define DVB_PLL_TDA665X_EARTH_PT1      21
+
+struct dvb_pll_config {
+	struct dvb_frontend *fe;
+};
 
 #if IS_REACHABLE(CONFIG_DVB_PLL)
 /**
diff --git a/drivers/media/dvb-frontends/l64781.c b/drivers/media/dvb-frontends/l64781.c
index e056f36e6f0c..249c18761e6e 100644
--- a/drivers/media/dvb-frontends/l64781.c
+++ b/drivers/media/dvb-frontends/l64781.c
@@ -546,7 +546,7 @@ struct dvb_frontend* l64781_attach(const struct l64781_config* config,
 
 	/* Responds to all reads with 0 */
 	if (l64781_readreg(state, 0x1a) != 0) {
-		dprintk("Read 1 returned unexpcted value\n");
+		dprintk("Read 1 returned unexpected value\n");
 		goto error;
 	}
 
@@ -555,7 +555,7 @@ struct dvb_frontend* l64781_attach(const struct l64781_config* config,
 
 	/* Responds with register default value */
 	if (l64781_readreg(state, 0x1a) != 0xa1) {
-		dprintk("Read 2 returned unexpcted value\n");
+		dprintk("Read 2 returned unexpected value\n");
 		goto error;
 	}
 
diff --git a/drivers/media/dvb-frontends/lgdt3306a.c b/drivers/media/dvb-frontends/lgdt3306a.c
index 7eb4e1469d20..32de824476db 100644
--- a/drivers/media/dvb-frontends/lgdt3306a.c
+++ b/drivers/media/dvb-frontends/lgdt3306a.c
@@ -1784,7 +1784,7 @@ static int lgdt3306a_get_tune_settings(struct dvb_frontend *fe,
 	return 0;
 }
 
-static int lgdt3306a_search(struct dvb_frontend *fe)
+static enum dvbfe_search lgdt3306a_search(struct dvb_frontend *fe)
 {
 	enum fe_status status = 0;
 	int ret;
diff --git a/drivers/media/dvb-frontends/lgdt330x.c b/drivers/media/dvb-frontends/lgdt330x.c
index 8ad03bd81af5..f6731738b073 100644
--- a/drivers/media/dvb-frontends/lgdt330x.c
+++ b/drivers/media/dvb-frontends/lgdt330x.c
@@ -47,50 +47,50 @@
 
 static int debug;
 module_param(debug, int, 0644);
-MODULE_PARM_DESC(debug,"Turn on/off lgdt330x frontend debugging (default:off).");
-#define dprintk(args...) \
-do { \
-if (debug) printk(KERN_DEBUG "lgdt330x: " args); \
+MODULE_PARM_DESC(debug, "Turn on/off lgdt330x frontend debugging (default:off).");
+
+#define dprintk(state, fmt, arg...) do {				\
+	if (debug)							\
+		dev_printk(KERN_DEBUG, &state->client->dev, fmt, ##arg);\
 } while (0)
 
-struct lgdt330x_state
-{
-	struct i2c_adapter* i2c;
+struct lgdt330x_state {
+	struct i2c_client *client;
 
 	/* Configuration settings */
-	const struct lgdt330x_config* config;
+	struct lgdt330x_config config;
 
 	struct dvb_frontend frontend;
 
 	/* Demodulator private data */
 	enum fe_modulation current_modulation;
-	u32 snr; /* Result of last SNR calculation */
+	u32 snr;	/* Result of last SNR calculation */
+	u16 ucblocks;
+	unsigned long last_stats_time;
 
 	/* Tuner private data */
 	u32 current_frequency;
 };
 
-static int i2c_write_demod_bytes (struct lgdt330x_state* state,
-				  u8 *buf, /* data bytes to send */
-				  int len  /* number of bytes to send */ )
+static int i2c_write_demod_bytes(struct lgdt330x_state *state,
+				 const u8 *buf, /* data bytes to send */
+				 int len  /* number of bytes to send */)
 {
-	struct i2c_msg msg =
-		{ .addr = state->config->demod_address,
-		  .flags = 0,
-		  .buf = buf,
-		  .len = 2 };
 	int i;
 	int err;
 
-	for (i=0; i<len-1; i+=2){
-		if ((err = i2c_transfer(state->i2c, &msg, 1)) != 1) {
-			printk(KERN_WARNING "lgdt330x: %s error (addr %02x <- %02x, err = %i)\n", __func__, msg.buf[0], msg.buf[1], err);
+	for (i = 0; i < len - 1; i += 2) {
+		err = i2c_master_send(state->client, buf, 2);
+		if (err != 2) {
+			dev_warn(&state->client->dev,
+				 "%s: error (addr %02x <- %02x, err = %i)\n",
+				__func__, buf[0], buf[1], err);
 			if (err < 0)
 				return err;
 			else
 				return -EREMOTEIO;
 		}
-		msg.buf += 2;
+		buf += 2;
 	}
 	return 0;
 }
@@ -99,21 +99,30 @@ static int i2c_write_demod_bytes (struct lgdt330x_state* state,
  * This routine writes the register (reg) to the demod bus
  * then reads the data returned for (len) bytes.
  */
-
 static int i2c_read_demod_bytes(struct lgdt330x_state *state,
 				enum I2C_REG reg, u8 *buf, int len)
 {
-	u8 wr [] = { reg };
-	struct i2c_msg msg [] = {
-		{ .addr = state->config->demod_address,
-		  .flags = 0, .buf = wr,  .len = 1 },
-		{ .addr = state->config->demod_address,
-		  .flags = I2C_M_RD, .buf = buf, .len = len },
+	u8 wr[] = { reg };
+	struct i2c_msg msg[] = {
+		{
+			.addr = state->client->addr,
+			.flags = 0,
+			.buf = wr,
+			.len = 1
+		}, {
+			.addr = state->client->addr,
+			.flags = I2C_M_RD,
+			.buf = buf,
+			.len = len
+		},
 	};
 	int ret;
-	ret = i2c_transfer(state->i2c, msg, 2);
+
+	ret = i2c_transfer(state->client->adapter, msg, 2);
 	if (ret != 2) {
-		printk(KERN_WARNING "lgdt330x: %s: addr 0x%02x select 0x%02x error (ret == %i)\n", __func__, state->config->demod_address, reg, ret);
+		dev_warn(&state->client->dev,
+			 "%s: addr 0x%02x select 0x%02x error (ret == %i)\n",
+			 __func__, state->client->addr, reg, ret);
 		if (ret >= 0)
 			ret = -EIO;
 	} else {
@@ -123,19 +132,21 @@ static int i2c_read_demod_bytes(struct lgdt330x_state *state,
 }
 
 /* Software reset */
-static int lgdt3302_SwReset(struct lgdt330x_state* state)
+static int lgdt3302_sw_reset(struct lgdt330x_state *state)
 {
 	u8 ret;
 	u8 reset[] = {
 		IRQ_MASK,
-		0x00 /* bit 6 is active low software reset
-		      *	bits 5-0 are 1 to mask interrupts */
+		/*
+		 * bit 6 is active low software reset
+		 * bits 5-0 are 1 to mask interrupts
+		 */
+		0x00
 	};
 
 	ret = i2c_write_demod_bytes(state,
 				    reset, sizeof(reset));
 	if (ret == 0) {
-
 		/* force reset high (inactive) and unmask interrupts */
 		reset[1] = 0x7f;
 		ret = i2c_write_demod_bytes(state,
@@ -144,7 +155,7 @@ static int lgdt3302_SwReset(struct lgdt330x_state* state)
 	return ret;
 }
 
-static int lgdt3303_SwReset(struct lgdt330x_state* state)
+static int lgdt3303_sw_reset(struct lgdt330x_state *state)
 {
 	u8 ret;
 	u8 reset[] = {
@@ -155,7 +166,6 @@ static int lgdt3303_SwReset(struct lgdt330x_state* state)
 	ret = i2c_write_demod_bytes(state,
 				    reset, sizeof(reset));
 	if (ret == 0) {
-
 		/* force reset high (inactive) */
 		reset[1] = 0x01;
 		ret = i2c_write_demod_bytes(state,
@@ -164,81 +174,94 @@ static int lgdt3303_SwReset(struct lgdt330x_state* state)
 	return ret;
 }
 
-static int lgdt330x_SwReset(struct lgdt330x_state* state)
+static int lgdt330x_sw_reset(struct lgdt330x_state *state)
 {
-	switch (state->config->demod_chip) {
+	switch (state->config.demod_chip) {
 	case LGDT3302:
-		return lgdt3302_SwReset(state);
+		return lgdt3302_sw_reset(state);
 	case LGDT3303:
-		return lgdt3303_SwReset(state);
+		return lgdt3303_sw_reset(state);
 	default:
 		return -ENODEV;
 	}
 }
 
-static int lgdt330x_init(struct dvb_frontend* fe)
+static int lgdt330x_init(struct dvb_frontend *fe)
 {
-	/* Hardware reset is done using gpio[0] of cx23880x chip.
-	 * I'd like to do it here, but don't know how to find chip address.
-	 * cx88-cards.c arranges for the reset bit to be inactive (high).
-	 * Maybe there needs to be a callable function in cx88-core or
-	 * the caller of this function needs to do it. */
-
+	struct lgdt330x_state *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	char  *chip_name;
+	int    err;
 	/*
 	 * Array of byte pairs <address, value>
 	 * to initialize each different chip
 	 */
-	static u8 lgdt3302_init_data[] = {
-		/* Use 50MHz parameter values from spec sheet since xtal is 50 */
-		/* Change the value of NCOCTFV[25:0] of carrier
-		   recovery center frequency register */
+	static const u8 lgdt3302_init_data[] = {
+		/* Use 50MHz param values from spec sheet since xtal is 50 */
+		/*
+		 * Change the value of NCOCTFV[25:0] of carrier
+		 * recovery center frequency register
+		 */
 		VSB_CARRIER_FREQ0, 0x00,
 		VSB_CARRIER_FREQ1, 0x87,
 		VSB_CARRIER_FREQ2, 0x8e,
 		VSB_CARRIER_FREQ3, 0x01,
-		/* Change the TPCLK pin polarity
-		   data is valid on falling clock */
+		/*
+		 * Change the TPCLK pin polarity
+		 * data is valid on falling clock
+		 */
 		DEMUX_CONTROL, 0xfb,
-		/* Change the value of IFBW[11:0] of
-		   AGC IF/RF loop filter bandwidth register */
+		/*
+		 * Change the value of IFBW[11:0] of
+		 * AGC IF/RF loop filter bandwidth register
+		 */
 		AGC_RF_BANDWIDTH0, 0x40,
 		AGC_RF_BANDWIDTH1, 0x93,
 		AGC_RF_BANDWIDTH2, 0x00,
-		/* Change the value of bit 6, 'nINAGCBY' and
-		   'NSSEL[1:0] of ACG function control register 2 */
+		/*
+		 * Change the value of bit 6, 'nINAGCBY' and
+		 * 'NSSEL[1:0] of ACG function control register 2
+		 */
 		AGC_FUNC_CTRL2, 0xc6,
-		/* Change the value of bit 6 'RFFIX'
-		   of AGC function control register 3 */
+		/*
+		 * Change the value of bit 6 'RFFIX'
+		 * of AGC function control register 3
+		 */
 		AGC_FUNC_CTRL3, 0x40,
-		/* Set the value of 'INLVTHD' register 0x2a/0x2c
-		   to 0x7fe */
+		/*
+		 * Set the value of 'INLVTHD' register 0x2a/0x2c
+		 * to 0x7fe
+		 */
 		AGC_DELAY0, 0x07,
 		AGC_DELAY2, 0xfe,
-		/* Change the value of IAGCBW[15:8]
-		   of inner AGC loop filter bandwidth */
+		/*
+		 * Change the value of IAGCBW[15:8]
+		 * of inner AGC loop filter bandwidth
+		 */
 		AGC_LOOP_BANDWIDTH0, 0x08,
 		AGC_LOOP_BANDWIDTH1, 0x9a
 	};
-
-	static u8 lgdt3303_init_data[] = {
+	static const u8 lgdt3303_init_data[] = {
 		0x4c, 0x14
 	};
-
-	static u8 flip_1_lgdt3303_init_data[] = {
+	static const u8 flip_1_lgdt3303_init_data[] = {
 		0x4c, 0x14,
 		0x87, 0xf3
 	};
-
-	static u8 flip_2_lgdt3303_init_data[] = {
+	static const u8 flip_2_lgdt3303_init_data[] = {
 		0x4c, 0x14,
 		0x87, 0xda
 	};
 
-	struct lgdt330x_state* state = fe->demodulator_priv;
-	char  *chip_name;
-	int    err;
+	/*
+	 * Hardware reset is done using gpio[0] of cx23880x chip.
+	 * I'd like to do it here, but don't know how to find chip address.
+	 * cx88-cards.c arranges for the reset bit to be inactive (high).
+	 * Maybe there needs to be a callable function in cx88-core or
+	 * the caller of this function needs to do it.
+	 */
 
-	switch (state->config->demod_chip) {
+	switch (state->config.demod_chip) {
 	case LGDT3302:
 		chip_name = "LGDT3302";
 		err = i2c_write_demod_bytes(state, lgdt3302_init_data,
@@ -246,16 +269,16 @@ static int lgdt330x_init(struct dvb_frontend* fe)
 		break;
 	case LGDT3303:
 		chip_name = "LGDT3303";
-		switch (state->config->clock_polarity_flip) {
+		switch (state->config.clock_polarity_flip) {
 		case 2:
 			err = i2c_write_demod_bytes(state,
-					flip_2_lgdt3303_init_data,
-					sizeof(flip_2_lgdt3303_init_data));
+						    flip_2_lgdt3303_init_data,
+						    sizeof(flip_2_lgdt3303_init_data));
 			break;
 		case 1:
 			err = i2c_write_demod_bytes(state,
-					flip_1_lgdt3303_init_data,
-					sizeof(flip_1_lgdt3303_init_data));
+						    flip_1_lgdt3303_init_data,
+						    sizeof(flip_1_lgdt3303_init_data));
 			break;
 		case 0:
 		default:
@@ -265,70 +288,55 @@ static int lgdt330x_init(struct dvb_frontend* fe)
 		break;
 	default:
 		chip_name = "undefined";
-		printk (KERN_WARNING "Only LGDT3302 and LGDT3303 are supported chips.\n");
+		dev_warn(&state->client->dev,
+			 "Only LGDT3302 and LGDT3303 are supported chips.\n");
 		err = -ENODEV;
 	}
-	dprintk("%s entered as %s\n", __func__, chip_name);
+	dprintk(state, "Initialized the %s chip\n", chip_name);
 	if (err < 0)
 		return err;
-	return lgdt330x_SwReset(state);
-}
 
-static int lgdt330x_read_ber(struct dvb_frontend* fe, u32* ber)
-{
-	*ber = 0; /* Not supplied by the demod chips */
-	return 0;
+	p->cnr.len = 1;
+	p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	p->block_error.len = 1;
+	p->block_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	p->block_count.len = 1;
+	p->block_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	state->last_stats_time = 0;
+
+	return lgdt330x_sw_reset(state);
 }
 
-static int lgdt330x_read_ucblocks(struct dvb_frontend* fe, u32* ucblocks)
+static int lgdt330x_read_ucblocks(struct dvb_frontend *fe, u32 *ucblocks)
 {
-	struct lgdt330x_state* state = fe->demodulator_priv;
-	int err;
-	u8 buf[2];
+	struct lgdt330x_state *state = fe->demodulator_priv;
 
-	*ucblocks = 0;
+	*ucblocks = state->ucblocks;
 
-	switch (state->config->demod_chip) {
-	case LGDT3302:
-		err = i2c_read_demod_bytes(state, LGDT3302_PACKET_ERR_COUNTER1,
-					   buf, sizeof(buf));
-		break;
-	case LGDT3303:
-		err = i2c_read_demod_bytes(state, LGDT3303_PACKET_ERR_COUNTER1,
-					   buf, sizeof(buf));
-		break;
-	default:
-		printk(KERN_WARNING
-		       "Only LGDT3302 and LGDT3303 are supported chips.\n");
-		err = -ENODEV;
-	}
-	if (err < 0)
-		return err;
-
-	*ucblocks = (buf[0] << 8) | buf[1];
 	return 0;
 }
 
 static int lgdt330x_set_parameters(struct dvb_frontend *fe)
 {
 	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	struct lgdt330x_state *state = fe->demodulator_priv;
 	/*
 	 * Array of byte pairs <address, value>
 	 * to initialize 8VSB for lgdt3303 chip 50 MHz IF
 	 */
-	static u8 lgdt3303_8vsb_44_data[] = {
+	static const u8 lgdt3303_8vsb_44_data[] = {
 		0x04, 0x00,
 		0x0d, 0x40,
 		0x0e, 0x87,
 		0x0f, 0x8e,
 		0x10, 0x01,
-		0x47, 0x8b };
-
+		0x47, 0x8b
+	};
 	/*
 	 * Array of byte pairs <address, value>
 	 * to initialize QAM for lgdt3303 chip
 	 */
-	static u8 lgdt3303_qam_data[] = {
+	static const u8 lgdt3303_qam_data[] = {
 		0x04, 0x00,
 		0x0d, 0x00,
 		0x0e, 0x00,
@@ -339,98 +347,105 @@ static int lgdt330x_set_parameters(struct dvb_frontend *fe)
 		0x48, 0x66,
 		0x4d, 0x1a,
 		0x49, 0x08,
-		0x4a, 0x9b };
-
-	struct lgdt330x_state* state = fe->demodulator_priv;
-
-	static u8 top_ctrl_cfg[]   = { TOP_CONTROL, 0x03 };
+		0x4a, 0x9b
+	};
+	u8 top_ctrl_cfg[]   = { TOP_CONTROL, 0x03 };
 
 	int err = 0;
 	/* Change only if we are actually changing the modulation */
 	if (state->current_modulation != p->modulation) {
 		switch (p->modulation) {
 		case VSB_8:
-			dprintk("%s: VSB_8 MODE\n", __func__);
+			dprintk(state, "VSB_8 MODE\n");
 
 			/* Select VSB mode */
 			top_ctrl_cfg[1] = 0x03;
 
 			/* Select ANT connector if supported by card */
-			if (state->config->pll_rf_set)
-				state->config->pll_rf_set(fe, 1);
+			if (state->config.pll_rf_set)
+				state->config.pll_rf_set(fe, 1);
 
-			if (state->config->demod_chip == LGDT3303) {
-				err = i2c_write_demod_bytes(state, lgdt3303_8vsb_44_data,
+			if (state->config.demod_chip == LGDT3303) {
+				err = i2c_write_demod_bytes(state,
+							    lgdt3303_8vsb_44_data,
 							    sizeof(lgdt3303_8vsb_44_data));
 			}
 			break;
 
 		case QAM_64:
-			dprintk("%s: QAM_64 MODE\n", __func__);
+			dprintk(state, "QAM_64 MODE\n");
 
 			/* Select QAM_64 mode */
 			top_ctrl_cfg[1] = 0x00;
 
 			/* Select CABLE connector if supported by card */
-			if (state->config->pll_rf_set)
-				state->config->pll_rf_set(fe, 0);
+			if (state->config.pll_rf_set)
+				state->config.pll_rf_set(fe, 0);
 
-			if (state->config->demod_chip == LGDT3303) {
-				err = i2c_write_demod_bytes(state, lgdt3303_qam_data,
-											sizeof(lgdt3303_qam_data));
+			if (state->config.demod_chip == LGDT3303) {
+				err = i2c_write_demod_bytes(state,
+							    lgdt3303_qam_data,
+							    sizeof(lgdt3303_qam_data));
 			}
 			break;
 
 		case QAM_256:
-			dprintk("%s: QAM_256 MODE\n", __func__);
+			dprintk(state, "QAM_256 MODE\n");
 
 			/* Select QAM_256 mode */
 			top_ctrl_cfg[1] = 0x01;
 
 			/* Select CABLE connector if supported by card */
-			if (state->config->pll_rf_set)
-				state->config->pll_rf_set(fe, 0);
+			if (state->config.pll_rf_set)
+				state->config.pll_rf_set(fe, 0);
 
-			if (state->config->demod_chip == LGDT3303) {
-				err = i2c_write_demod_bytes(state, lgdt3303_qam_data,
-											sizeof(lgdt3303_qam_data));
+			if (state->config.demod_chip == LGDT3303) {
+				err = i2c_write_demod_bytes(state,
+							    lgdt3303_qam_data,
+							    sizeof(lgdt3303_qam_data));
 			}
 			break;
 		default:
-			printk(KERN_WARNING "lgdt330x: %s: Modulation type(%d) UNSUPPORTED\n", __func__, p->modulation);
+			dev_warn(&state->client->dev,
+				 "%s: Modulation type(%d) UNSUPPORTED\n",
+				 __func__, p->modulation);
 			return -1;
 		}
 		if (err < 0)
-			printk(KERN_WARNING "lgdt330x: %s: error blasting bytes to lgdt3303 for modulation type(%d)\n",
-			       __func__, p->modulation);
+			dev_warn(&state->client->dev,
+				 "%s: error blasting bytes to lgdt3303 for modulation type(%d)\n",
+				 __func__, p->modulation);
 
 		/*
-		 * select serial or parallel MPEG harware interface
+		 * select serial or parallel MPEG hardware interface
 		 * Serial:   0x04 for LGDT3302 or 0x40 for LGDT3303
 		 * Parallel: 0x00
 		 */
-		top_ctrl_cfg[1] |= state->config->serial_mpeg;
+		top_ctrl_cfg[1] |= state->config.serial_mpeg;
 
 		/* Select the requested mode */
 		i2c_write_demod_bytes(state, top_ctrl_cfg,
 				      sizeof(top_ctrl_cfg));
-		if (state->config->set_ts_params)
-			state->config->set_ts_params(fe, 0);
+		if (state->config.set_ts_params)
+			state->config.set_ts_params(fe, 0);
 		state->current_modulation = p->modulation;
 	}
 
 	/* Tune to the specified frequency */
 	if (fe->ops.tuner_ops.set_params) {
 		fe->ops.tuner_ops.set_params(fe);
-		if (fe->ops.i2c_gate_ctrl) fe->ops.i2c_gate_ctrl(fe, 0);
+		if (fe->ops.i2c_gate_ctrl)
+			fe->ops.i2c_gate_ctrl(fe, 0);
 	}
 
 	/* Keep track of the new frequency */
-	/* FIXME this is the wrong way to do this...           */
-	/* The tuner is shared with the video4linux analog API */
+	/*
+	 * FIXME this is the wrong way to do this...
+	 * The tuner is shared with the video4linux analog API
+	 */
 	state->current_frequency = p->frequency;
 
-	lgdt330x_SwReset(state);
+	lgdt330x_sw_reset(state);
 	return 0;
 }
 
@@ -443,20 +458,191 @@ static int lgdt330x_get_frontend(struct dvb_frontend *fe,
 	return 0;
 }
 
+/*
+ * Calculate SNR estimation (scaled by 2^24)
+ *
+ * 8-VSB SNR equations from LGDT3302 and LGDT3303 datasheets, QAM
+ * equations from LGDT3303 datasheet.  VSB is the same between the '02
+ * and '03, so maybe QAM is too?  Perhaps someone with a newer datasheet
+ * that has QAM information could verify?
+ *
+ * For 8-VSB: (two ways, take your pick)
+ * LGDT3302:
+ *   SNR_EQ = 10 * log10(25 * 24^2 / EQ_MSE)
+ * LGDT3303:
+ *   SNR_EQ = 10 * log10(25 * 32^2 / EQ_MSE)
+ * LGDT3302 & LGDT3303:
+ *   SNR_PT = 10 * log10(25 * 32^2 / PT_MSE)  (we use this one)
+ * For 64-QAM:
+ *   SNR    = 10 * log10( 688128   / MSEQAM)
+ * For 256-QAM:
+ *   SNR    = 10 * log10( 696320   / MSEQAM)
+ *
+ * We re-write the snr equation as:
+ *   SNR * 2^24 = 10*(c - intlog10(MSE))
+ * Where for 256-QAM, c = log10(696320) * 2^24, and so on.
+ */
+static u32 calculate_snr(u32 mse, u32 c)
+{
+	if (mse == 0) /* No signal */
+		return 0;
+
+	mse = intlog10(mse);
+	if (mse > c) {
+		/*
+		 * Negative SNR, which is possible, but realisticly the
+		 * demod will lose lock before the signal gets this bad.
+		 * The API only allows for unsigned values, so just return 0
+		 */
+		return 0;
+	}
+	return 10 * (c - mse);
+}
+
+static int lgdt3302_read_snr(struct dvb_frontend *fe)
+{
+	struct lgdt330x_state *state = fe->demodulator_priv;
+	u8 buf[5];	/* read data buffer */
+	u32 noise;	/* noise value */
+	u32 c;		/* per-modulation SNR calculation constant */
+
+	switch (state->current_modulation) {
+	case VSB_8:
+		i2c_read_demod_bytes(state, LGDT3302_EQPH_ERR0, buf, 5);
+#ifdef USE_EQMSE
+		/* Use Equalizer Mean-Square Error Register */
+		/* SNR for ranges from -15.61 to +41.58 */
+		noise = ((buf[0] & 7) << 16) | (buf[1] << 8) | buf[2];
+		c = 69765745; /* log10(25*24^2)*2^24 */
+#else
+		/* Use Phase Tracker Mean-Square Error Register */
+		/* SNR for ranges from -13.11 to +44.08 */
+		noise = ((buf[0] & 7 << 3) << 13) | (buf[3] << 8) | buf[4];
+		c = 73957994; /* log10(25*32^2)*2^24 */
+#endif
+		break;
+	case QAM_64:
+	case QAM_256:
+		i2c_read_demod_bytes(state, CARRIER_MSEQAM1, buf, 2);
+		noise = ((buf[0] & 3) << 8) | buf[1];
+		c = state->current_modulation == QAM_64 ? 97939837 : 98026066;
+		/* log10(688128)*2^24 and log10(696320)*2^24 */
+		break;
+	default:
+		dev_err(&state->client->dev,
+			"%s: Modulation set to unsupported value\n",
+			__func__);
+
+		state->snr = 0;
+
+		return -EREMOTEIO; /* return -EDRIVER_IS_GIBBERED; */
+	}
+
+	state->snr = calculate_snr(noise, c);
+
+	dprintk(state, "noise = 0x%08x, snr = %d.%02d dB\n", noise,
+		state->snr >> 24, (((state->snr >> 8) & 0xffff) * 100) >> 16);
+
+	return 0;
+}
+
+static int lgdt3303_read_snr(struct dvb_frontend *fe)
+{
+	struct lgdt330x_state *state = fe->demodulator_priv;
+	u8 buf[5];	/* read data buffer */
+	u32 noise;	/* noise value */
+	u32 c;		/* per-modulation SNR calculation constant */
+
+	switch (state->current_modulation) {
+	case VSB_8:
+		i2c_read_demod_bytes(state, LGDT3303_EQPH_ERR0, buf, 5);
+#ifdef USE_EQMSE
+		/* Use Equalizer Mean-Square Error Register */
+		/* SNR for ranges from -16.12 to +44.08 */
+		noise = ((buf[0] & 0x78) << 13) | (buf[1] << 8) | buf[2];
+		c = 73957994; /* log10(25*32^2)*2^24 */
+#else
+		/* Use Phase Tracker Mean-Square Error Register */
+		/* SNR for ranges from -13.11 to +44.08 */
+		noise = ((buf[0] & 7) << 16) | (buf[3] << 8) | buf[4];
+		c = 73957994; /* log10(25*32^2)*2^24 */
+#endif
+		break;
+	case QAM_64:
+	case QAM_256:
+		i2c_read_demod_bytes(state, CARRIER_MSEQAM1, buf, 2);
+		noise = (buf[0] << 8) | buf[1];
+		c = state->current_modulation == QAM_64 ? 97939837 : 98026066;
+		/* log10(688128)*2^24 and log10(696320)*2^24 */
+		break;
+	default:
+		dev_err(&state->client->dev,
+			"%s: Modulation set to unsupported value\n",
+			__func__);
+		state->snr = 0;
+		return -EREMOTEIO; /* return -EDRIVER_IS_GIBBERED; */
+	}
+
+	state->snr = calculate_snr(noise, c);
+
+	dprintk(state, "noise = 0x%08x, snr = %d.%02d dB\n", noise,
+		state->snr >> 24, (((state->snr >> 8) & 0xffff) * 100) >> 16);
+
+	return 0;
+}
+
+static int lgdt330x_read_snr(struct dvb_frontend *fe, u16 *snr)
+{
+	struct lgdt330x_state *state = fe->demodulator_priv;
+
+	*snr = (state->snr) >> 16; /* Convert from 8.24 fixed-point to 8.8 */
+
+	return 0;
+}
+
+static int lgdt330x_read_signal_strength(struct dvb_frontend *fe, u16 *strength)
+{
+	/* Calculate Strength from SNR up to 35dB */
+	/*
+	 * Even though the SNR can go higher than 35dB, there is some comfort
+	 * factor in having a range of strong signals that can show at 100%
+	 */
+	struct lgdt330x_state *state = fe->demodulator_priv;
+	u16 snr;
+	int ret;
+
+	ret = fe->ops.read_snr(fe, &snr);
+	if (ret != 0)
+		return ret;
+	/* Rather than use the 8.8 value snr, use state->snr which is 8.24 */
+	/* scale the range 0 - 35*2^24 into 0 - 65535 */
+	if (state->snr >= 8960 * 0x10000)
+		*strength = 0xffff;
+	else
+		*strength = state->snr / 8960;
+
+	return 0;
+}
+
+
 static int lgdt3302_read_status(struct dvb_frontend *fe,
 				enum fe_status *status)
 {
-	struct lgdt330x_state* state = fe->demodulator_priv;
+	struct lgdt330x_state *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
 	u8 buf[3];
+	int err;
 
 	*status = 0; /* Reset status result */
 
 	/* AGC status register */
 	i2c_read_demod_bytes(state, AGC_STATUS, buf, 1);
-	dprintk("%s: AGC_STATUS = 0x%02x\n", __func__, buf[0]);
-	if ((buf[0] & 0x0c) == 0x8){
-		/* Test signal does not exist flag */
-		/* as well as the AGC lock flag.   */
+	dprintk(state, "AGC_STATUS = 0x%02x\n", buf[0]);
+	if ((buf[0] & 0x0c) == 0x8) {
+		/*
+		 * Test signal does not exist flag
+		 * as well as the AGC lock flag.
+		 */
 		*status |= FE_HAS_SIGNAL;
 	}
 
@@ -465,25 +651,24 @@ static int lgdt3302_read_status(struct dvb_frontend *fe,
 	 * to see that status bit in the IRQ_STATUS register.
 	 * This is done in SwReset();
 	 */
+
 	/* signal status */
 	i2c_read_demod_bytes(state, TOP_CONTROL, buf, sizeof(buf));
-	dprintk("%s: TOP_CONTROL = 0x%02x, IRO_MASK = 0x%02x, IRQ_STATUS = 0x%02x\n", __func__, buf[0], buf[1], buf[2]);
-
+	dprintk(state,
+		"TOP_CONTROL = 0x%02x, IRO_MASK = 0x%02x, IRQ_STATUS = 0x%02x\n",
+		buf[0], buf[1], buf[2]);
 
 	/* sync status */
-	if ((buf[2] & 0x03) == 0x01) {
+	if ((buf[2] & 0x03) == 0x01)
 		*status |= FE_HAS_SYNC;
-	}
 
 	/* FEC error status */
-	if ((buf[2] & 0x0c) == 0x08) {
-		*status |= FE_HAS_LOCK;
-		*status |= FE_HAS_VITERBI;
-	}
+	if ((buf[2] & 0x0c) == 0x08)
+		*status |= FE_HAS_LOCK | FE_HAS_VITERBI;
 
 	/* Carrier Recovery Lock Status Register */
 	i2c_read_demod_bytes(state, CARRIER_LOCK, buf, 1);
-	dprintk("%s: CARRIER_LOCK = 0x%02x\n", __func__, buf[0]);
+	dprintk(state, "CARRIER_LOCK = 0x%02x\n", buf[0]);
 	switch (state->current_modulation) {
 	case QAM_256:
 	case QAM_64:
@@ -496,7 +681,48 @@ static int lgdt3302_read_status(struct dvb_frontend *fe,
 			*status |= FE_HAS_CARRIER;
 		break;
 	default:
-		printk(KERN_WARNING "lgdt330x: %s: Modulation set to unsupported value\n", __func__);
+		dev_warn(&state->client->dev,
+			 "%s: Modulation set to unsupported value\n",
+			 __func__);
+	}
+
+	if (!(*status & FE_HAS_LOCK)) {
+		p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->block_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->block_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		return 0;
+	}
+
+	if (state->last_stats_time &&
+	    time_is_after_jiffies(state->last_stats_time))
+		return 0;
+
+	state->last_stats_time = jiffies + msecs_to_jiffies(1000);
+
+	err = lgdt3302_read_snr(fe);
+	if (!err) {
+		p->cnr.stat[0].scale = FE_SCALE_DECIBEL;
+		p->cnr.stat[0].svalue = (((u64)state->snr) * 1000) >> 24;
+	} else {
+		p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	}
+
+	err = i2c_read_demod_bytes(state, LGDT3302_PACKET_ERR_COUNTER1,
+					   buf, sizeof(buf));
+	if (!err) {
+		state->ucblocks = (buf[0] << 8) | buf[1];
+
+		dprintk(state, "UCB = 0x%02x\n", state->ucblocks);
+
+		p->block_error.stat[0].uvalue += state->ucblocks;
+		/* FIXME: what's the basis for block count */
+		p->block_count.stat[0].uvalue += 10000;
+
+		p->block_error.stat[0].scale = FE_SCALE_COUNTER;
+		p->block_count.stat[0].scale = FE_SCALE_COUNTER;
+	} else {
+		p->block_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->block_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
 	}
 
 	return 0;
@@ -505,9 +731,10 @@ static int lgdt3302_read_status(struct dvb_frontend *fe,
 static int lgdt3303_read_status(struct dvb_frontend *fe,
 				enum fe_status *status)
 {
-	struct lgdt330x_state* state = fe->demodulator_priv;
-	int err;
+	struct lgdt330x_state *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
 	u8 buf[3];
+	int err;
 
 	*status = 0; /* Reset status result */
 
@@ -516,16 +743,18 @@ static int lgdt3303_read_status(struct dvb_frontend *fe,
 	if (err < 0)
 		return err;
 
-	dprintk("%s: AGC_STATUS = 0x%02x\n", __func__, buf[0]);
-	if ((buf[0] & 0x21) == 0x01){
-		/* Test input signal does not exist flag */
-		/* as well as the AGC lock flag.   */
+	dprintk(state, "AGC_STATUS = 0x%02x\n", buf[0]);
+	if ((buf[0] & 0x21) == 0x01) {
+		/*
+		 * Test input signal does not exist flag
+		 * as well as the AGC lock flag.
+		 */
 		*status |= FE_HAS_SIGNAL;
 	}
 
 	/* Carrier Recovery Lock Status Register */
 	i2c_read_demod_bytes(state, CARRIER_LOCK, buf, 1);
-	dprintk("%s: CARRIER_LOCK = 0x%02x\n", __func__, buf[0]);
+	dprintk(state, "CARRIER_LOCK = 0x%02x\n", buf[0]);
 	switch (state->current_modulation) {
 	case QAM_256:
 	case QAM_64:
@@ -535,6 +764,8 @@ static int lgdt3303_read_status(struct dvb_frontend *fe,
 		else
 			break;
 		i2c_read_demod_bytes(state, 0x8a, buf, 1);
+		dprintk(state, "QAM LOCK = 0x%02x\n", buf[0]);
+
 		if ((buf[0] & 0x04) == 0x04)
 			*status |= FE_HAS_SYNC;
 		if ((buf[0] & 0x01) == 0x01)
@@ -548,168 +779,64 @@ static int lgdt3303_read_status(struct dvb_frontend *fe,
 		else
 			break;
 		i2c_read_demod_bytes(state, 0x38, buf, 1);
+		dprintk(state, "8-VSB LOCK = 0x%02x\n", buf[0]);
+
 		if ((buf[0] & 0x02) == 0x00)
 			*status |= FE_HAS_SYNC;
-		if ((buf[0] & 0x01) == 0x01) {
-			*status |= FE_HAS_LOCK;
-			*status |= FE_HAS_VITERBI;
-		}
+		if ((buf[0] & 0xfd) == 0x01)
+			*status |= FE_HAS_VITERBI | FE_HAS_LOCK;
 		break;
 	default:
-		printk(KERN_WARNING "lgdt330x: %s: Modulation set to unsupported value\n", __func__);
+		dev_warn(&state->client->dev,
+			 "%s: Modulation set to unsupported value\n",
+			 __func__);
 	}
-	return 0;
-}
-
-/* Calculate SNR estimation (scaled by 2^24)
-
-   8-VSB SNR equations from LGDT3302 and LGDT3303 datasheets, QAM
-   equations from LGDT3303 datasheet.  VSB is the same between the '02
-   and '03, so maybe QAM is too?  Perhaps someone with a newer datasheet
-   that has QAM information could verify?
-
-   For 8-VSB: (two ways, take your pick)
-   LGDT3302:
-     SNR_EQ = 10 * log10(25 * 24^2 / EQ_MSE)
-   LGDT3303:
-     SNR_EQ = 10 * log10(25 * 32^2 / EQ_MSE)
-   LGDT3302 & LGDT3303:
-     SNR_PT = 10 * log10(25 * 32^2 / PT_MSE)  (we use this one)
-   For 64-QAM:
-     SNR    = 10 * log10( 688128   / MSEQAM)
-   For 256-QAM:
-     SNR    = 10 * log10( 696320   / MSEQAM)
-
-   We re-write the snr equation as:
-     SNR * 2^24 = 10*(c - intlog10(MSE))
-   Where for 256-QAM, c = log10(696320) * 2^24, and so on. */
 
-static u32 calculate_snr(u32 mse, u32 c)
-{
-	if (mse == 0) /* No signal */
+	if (!(*status & FE_HAS_LOCK)) {
+		p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->block_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->block_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
 		return 0;
+	}
 
-	mse = intlog10(mse);
-	if (mse > c) {
-		/* Negative SNR, which is possible, but realisticly the
-		demod will lose lock before the signal gets this bad.  The
-		API only allows for unsigned values, so just return 0 */
+	if (state->last_stats_time &&
+	    time_is_after_jiffies(state->last_stats_time))
 		return 0;
-	}
-	return 10*(c - mse);
-}
 
-static int lgdt3302_read_snr(struct dvb_frontend* fe, u16* snr)
-{
-	struct lgdt330x_state* state = (struct lgdt330x_state*) fe->demodulator_priv;
-	u8 buf[5];	/* read data buffer */
-	u32 noise;	/* noise value */
-	u32 c;		/* per-modulation SNR calculation constant */
+	state->last_stats_time = jiffies + msecs_to_jiffies(1000);
 
-	switch(state->current_modulation) {
-	case VSB_8:
-		i2c_read_demod_bytes(state, LGDT3302_EQPH_ERR0, buf, 5);
-#ifdef USE_EQMSE
-		/* Use Equalizer Mean-Square Error Register */
-		/* SNR for ranges from -15.61 to +41.58 */
-		noise = ((buf[0] & 7) << 16) | (buf[1] << 8) | buf[2];
-		c = 69765745; /* log10(25*24^2)*2^24 */
-#else
-		/* Use Phase Tracker Mean-Square Error Register */
-		/* SNR for ranges from -13.11 to +44.08 */
-		noise = ((buf[0] & 7<<3) << 13) | (buf[3] << 8) | buf[4];
-		c = 73957994; /* log10(25*32^2)*2^24 */
-#endif
-		break;
-	case QAM_64:
-	case QAM_256:
-		i2c_read_demod_bytes(state, CARRIER_MSEQAM1, buf, 2);
-		noise = ((buf[0] & 3) << 8) | buf[1];
-		c = state->current_modulation == QAM_64 ? 97939837 : 98026066;
-		/* log10(688128)*2^24 and log10(696320)*2^24 */
-		break;
-	default:
-		printk(KERN_ERR "lgdt330x: %s: Modulation set to unsupported value\n",
-		       __func__);
-		return -EREMOTEIO; /* return -EDRIVER_IS_GIBBERED; */
+	err = lgdt3303_read_snr(fe);
+	if (!err) {
+		p->cnr.stat[0].scale = FE_SCALE_DECIBEL;
+		p->cnr.stat[0].svalue = (((u64)state->snr) * 1000) >> 24;
+	} else {
+		p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
 	}
 
-	state->snr = calculate_snr(noise, c);
-	*snr = (state->snr) >> 16; /* Convert from 8.24 fixed-point to 8.8 */
-
-	dprintk("%s: noise = 0x%08x, snr = %d.%02d dB\n", __func__, noise,
-		state->snr >> 24, (((state->snr>>8) & 0xffff) * 100) >> 16);
+	err = i2c_read_demod_bytes(state, LGDT3303_PACKET_ERR_COUNTER1,
+					   buf, sizeof(buf));
+	if (!err) {
+		state->ucblocks = (buf[0] << 8) | buf[1];
 
-	return 0;
-}
+		dprintk(state, "UCB = 0x%02x\n", state->ucblocks);
 
-static int lgdt3303_read_snr(struct dvb_frontend* fe, u16* snr)
-{
-	struct lgdt330x_state* state = (struct lgdt330x_state*) fe->demodulator_priv;
-	u8 buf[5];	/* read data buffer */
-	u32 noise;	/* noise value */
-	u32 c;		/* per-modulation SNR calculation constant */
+		p->block_error.stat[0].uvalue += state->ucblocks;
+		/* FIXME: what's the basis for block count */
+		p->block_count.stat[0].uvalue += 10000;
 
-	switch(state->current_modulation) {
-	case VSB_8:
-		i2c_read_demod_bytes(state, LGDT3303_EQPH_ERR0, buf, 5);
-#ifdef USE_EQMSE
-		/* Use Equalizer Mean-Square Error Register */
-		/* SNR for ranges from -16.12 to +44.08 */
-		noise = ((buf[0] & 0x78) << 13) | (buf[1] << 8) | buf[2];
-		c = 73957994; /* log10(25*32^2)*2^24 */
-#else
-		/* Use Phase Tracker Mean-Square Error Register */
-		/* SNR for ranges from -13.11 to +44.08 */
-		noise = ((buf[0] & 7) << 16) | (buf[3] << 8) | buf[4];
-		c = 73957994; /* log10(25*32^2)*2^24 */
-#endif
-		break;
-	case QAM_64:
-	case QAM_256:
-		i2c_read_demod_bytes(state, CARRIER_MSEQAM1, buf, 2);
-		noise = (buf[0] << 8) | buf[1];
-		c = state->current_modulation == QAM_64 ? 97939837 : 98026066;
-		/* log10(688128)*2^24 and log10(696320)*2^24 */
-		break;
-	default:
-		printk(KERN_ERR "lgdt330x: %s: Modulation set to unsupported value\n",
-		       __func__);
-		return -EREMOTEIO; /* return -EDRIVER_IS_GIBBERED; */
+		p->block_error.stat[0].scale = FE_SCALE_COUNTER;
+		p->block_count.stat[0].scale = FE_SCALE_COUNTER;
+	} else {
+		p->block_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->block_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
 	}
 
-	state->snr = calculate_snr(noise, c);
-	*snr = (state->snr) >> 16; /* Convert from 8.24 fixed-point to 8.8 */
-
-	dprintk("%s: noise = 0x%08x, snr = %d.%02d dB\n", __func__, noise,
-		state->snr >> 24, (((state->snr >> 8) & 0xffff) * 100) >> 16);
-
 	return 0;
 }
 
-static int lgdt330x_read_signal_strength(struct dvb_frontend* fe, u16* strength)
-{
-	/* Calculate Strength from SNR up to 35dB */
-	/* Even though the SNR can go higher than 35dB, there is some comfort */
-	/* factor in having a range of strong signals that can show at 100%   */
-	struct lgdt330x_state* state = (struct lgdt330x_state*) fe->demodulator_priv;
-	u16 snr;
-	int ret;
-
-	ret = fe->ops.read_snr(fe, &snr);
-	if (ret != 0)
-		return ret;
-	/* Rather than use the 8.8 value snr, use state->snr which is 8.24 */
-	/* scale the range 0 - 35*2^24 into 0 - 65535 */
-	if (state->snr >= 8960 * 0x10000)
-		*strength = 0xffff;
-	else
-		*strength = state->snr / 8960;
-
-	return 0;
-}
-
-static int lgdt330x_get_tune_settings(struct dvb_frontend* fe, struct dvb_frontend_tune_settings* fe_tune_settings)
+static int
+lgdt330x_get_tune_settings(struct dvb_frontend *fe,
+			   struct dvb_frontend_tune_settings *fe_tune_settings)
 {
 	/* I have no idea about this - it may not be needed */
 	fe_tune_settings->min_delay_ms = 500;
@@ -718,43 +845,63 @@ static int lgdt330x_get_tune_settings(struct dvb_frontend* fe, struct dvb_fronte
 	return 0;
 }
 
-static void lgdt330x_release(struct dvb_frontend* fe)
+static void lgdt330x_release(struct dvb_frontend *fe)
 {
-	struct lgdt330x_state* state = (struct lgdt330x_state*) fe->demodulator_priv;
-	kfree(state);
+	struct lgdt330x_state *state = fe->demodulator_priv;
+	struct i2c_client *client = state->client;
+
+	dev_dbg(&client->dev, "\n");
+
+	i2c_unregister_device(client);
+}
+
+static struct dvb_frontend *lgdt330x_get_dvb_frontend(struct i2c_client *client)
+{
+	struct lgdt330x_state *state = i2c_get_clientdata(client);
+
+	dev_dbg(&client->dev, "\n");
+
+	return &state->frontend;
 }
 
 static const struct dvb_frontend_ops lgdt3302_ops;
 static const struct dvb_frontend_ops lgdt3303_ops;
 
-struct dvb_frontend* lgdt330x_attach(const struct lgdt330x_config* config,
-				     struct i2c_adapter* i2c)
+static int lgdt330x_probe(struct i2c_client *client,
+			  const struct i2c_device_id *id)
 {
-	struct lgdt330x_state* state = NULL;
+	struct lgdt330x_state *state = NULL;
 	u8 buf[1];
 
 	/* Allocate memory for the internal state */
-	state = kzalloc(sizeof(struct lgdt330x_state), GFP_KERNEL);
-	if (state == NULL)
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
 		goto error;
 
 	/* Setup the state */
-	state->config = config;
-	state->i2c = i2c;
+	memcpy(&state->config, client->dev.platform_data,
+	       sizeof(state->config));
+	i2c_set_clientdata(client, state);
+	state->client = client;
 
 	/* Create dvb_frontend */
-	switch (config->demod_chip) {
+	switch (state->config.demod_chip) {
 	case LGDT3302:
-		memcpy(&state->frontend.ops, &lgdt3302_ops, sizeof(struct dvb_frontend_ops));
+		memcpy(&state->frontend.ops, &lgdt3302_ops,
+		       sizeof(struct dvb_frontend_ops));
 		break;
 	case LGDT3303:
-		memcpy(&state->frontend.ops, &lgdt3303_ops, sizeof(struct dvb_frontend_ops));
+		memcpy(&state->frontend.ops, &lgdt3303_ops,
+		       sizeof(struct dvb_frontend_ops));
 		break;
 	default:
 		goto error;
 	}
 	state->frontend.demodulator_priv = state;
 
+	/* Setup get frontend callback */
+	state->config.get_dvb_frontend = lgdt330x_get_dvb_frontend;
+
 	/* Verify communication with demod chip */
 	if (i2c_read_demod_bytes(state, 2, buf, 1))
 		goto error;
@@ -762,21 +909,44 @@ struct dvb_frontend* lgdt330x_attach(const struct lgdt330x_config* config,
 	state->current_frequency = -1;
 	state->current_modulation = -1;
 
-	return &state->frontend;
+	dev_info(&state->client->dev,
+		"Demod loaded for LGDT330%s chip\n",
+		state->config.demod_chip == LGDT3302 ? "2" : "3");
+
+	return 0;
 
 error:
 	kfree(state);
-	dprintk("%s: ERROR\n",__func__);
-	return NULL;
+	if (debug)
+		dev_printk(KERN_DEBUG, &client->dev, "Error loading lgdt330x driver\n");
+	return -ENODEV;
+}
+struct dvb_frontend *lgdt330x_attach(const struct lgdt330x_config *_config,
+				     u8 demod_address,
+				     struct i2c_adapter *i2c)
+{
+	struct i2c_client *client;
+	struct i2c_board_info board_info = {};
+	struct lgdt330x_config config = *_config;
+
+	strlcpy(board_info.type, "lgdt330x", sizeof(board_info.type));
+	board_info.addr = demod_address;
+	board_info.platform_data = &config;
+	client = i2c_new_device(i2c, &board_info);
+	if (!client || !client->dev.driver)
+		return NULL;
+
+	return lgdt330x_get_dvb_frontend(client);
 }
+EXPORT_SYMBOL(lgdt330x_attach);
 
 static const struct dvb_frontend_ops lgdt3302_ops = {
 	.delsys = { SYS_ATSC, SYS_DVBC_ANNEX_B },
 	.info = {
-		.name= "LG Electronics LGDT3302 VSB/QAM Frontend",
-		.frequency_min= 54000000,
-		.frequency_max= 858000000,
-		.frequency_stepsize= 62500,
+		.name = "LG Electronics LGDT3302 VSB/QAM Frontend",
+		.frequency_min = 54000000,
+		.frequency_max = 858000000,
+		.frequency_stepsize = 62500,
 		.symbol_rate_min    = 5056941,	/* QAM 64 */
 		.symbol_rate_max    = 10762000,	/* VSB 8  */
 		.caps = FE_CAN_QAM_64 | FE_CAN_QAM_256 | FE_CAN_8VSB
@@ -786,9 +956,8 @@ static const struct dvb_frontend_ops lgdt3302_ops = {
 	.get_frontend         = lgdt330x_get_frontend,
 	.get_tune_settings    = lgdt330x_get_tune_settings,
 	.read_status          = lgdt3302_read_status,
-	.read_ber             = lgdt330x_read_ber,
 	.read_signal_strength = lgdt330x_read_signal_strength,
-	.read_snr             = lgdt3302_read_snr,
+	.read_snr             = lgdt330x_read_snr,
 	.read_ucblocks        = lgdt330x_read_ucblocks,
 	.release              = lgdt330x_release,
 };
@@ -796,10 +965,10 @@ static const struct dvb_frontend_ops lgdt3302_ops = {
 static const struct dvb_frontend_ops lgdt3303_ops = {
 	.delsys = { SYS_ATSC, SYS_DVBC_ANNEX_B },
 	.info = {
-		.name= "LG Electronics LGDT3303 VSB/QAM Frontend",
-		.frequency_min= 54000000,
-		.frequency_max= 858000000,
-		.frequency_stepsize= 62500,
+		.name = "LG Electronics LGDT3303 VSB/QAM Frontend",
+		.frequency_min = 54000000,
+		.frequency_max = 858000000,
+		.frequency_stepsize = 62500,
 		.symbol_rate_min    = 5056941,	/* QAM 64 */
 		.symbol_rate_max    = 10762000,	/* VSB 8  */
 		.caps = FE_CAN_QAM_64 | FE_CAN_QAM_256 | FE_CAN_8VSB
@@ -809,15 +978,42 @@ static const struct dvb_frontend_ops lgdt3303_ops = {
 	.get_frontend         = lgdt330x_get_frontend,
 	.get_tune_settings    = lgdt330x_get_tune_settings,
 	.read_status          = lgdt3303_read_status,
-	.read_ber             = lgdt330x_read_ber,
 	.read_signal_strength = lgdt330x_read_signal_strength,
-	.read_snr             = lgdt3303_read_snr,
+	.read_snr             = lgdt330x_read_snr,
 	.read_ucblocks        = lgdt330x_read_ucblocks,
 	.release              = lgdt330x_release,
 };
 
+static int lgdt330x_remove(struct i2c_client *client)
+{
+	struct lgdt330x_state *state = i2c_get_clientdata(client);
+
+	dev_dbg(&client->dev, "\n");
+
+	kfree(state);
+
+	return 0;
+}
+
+static const struct i2c_device_id lgdt330x_id_table[] = {
+	{"lgdt330x", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, lgdt330x_id_table);
+
+static struct i2c_driver lgdt330x_driver = {
+	.driver = {
+		.name	= "lgdt330x",
+		.suppress_bind_attrs = true,
+	},
+	.probe		= lgdt330x_probe,
+	.remove		= lgdt330x_remove,
+	.id_table	= lgdt330x_id_table,
+};
+
+module_i2c_driver(lgdt330x_driver);
+
+
 MODULE_DESCRIPTION("LGDT330X (ATSC 8VSB & ITU-T J.83 AnnexB 64/256 QAM) Demodulator Driver");
 MODULE_AUTHOR("Wilson Michaels");
 MODULE_LICENSE("GPL");
-
-EXPORT_SYMBOL(lgdt330x_attach);
diff --git a/drivers/media/dvb-frontends/lgdt330x.h b/drivers/media/dvb-frontends/lgdt330x.h
index 61434cbecd2c..8ab6b39d02a8 100644
--- a/drivers/media/dvb-frontends/lgdt330x.h
+++ b/drivers/media/dvb-frontends/lgdt330x.h
@@ -26,34 +26,41 @@ typedef enum lg_chip_t {
 		LGDT3303
 }lg_chip_type;
 
+/**
+ * struct lgdt330x_config - contains lgdt330x configuration
+ *
+ * @demod_chip:		LG demodulator chip LGDT3302 or LGDT3303
+ * @serial_mpeg:	MPEG hardware interface - 0:parallel 1:serial
+ * @pll_rf_set:		Callback function to set PLL interface
+ * @set_ts_params:	Callback function to set device param for start_dma
+ * @clock_polarity_flip:
+ *	Flip the polarity of the mpeg data transfer clock using alternate
+ *	init data.
+ *	This option applies ONLY to LGDT3303 - 0:disabled (default) 1:enabled
+ * @get_dvb_frontend:
+ *	returns the frontend associated with this I2C client.
+ *	Filled by the driver.
+ */
 struct lgdt330x_config
 {
-	/* The demodulator's i2c address */
-	u8 demod_address;
-
-	/* LG demodulator chip LGDT3302 or LGDT3303 */
 	lg_chip_type demod_chip;
-
-	/* MPEG hardware interface - 0:parallel 1:serial */
 	int serial_mpeg;
-
-	/* PLL interface */
 	int (*pll_rf_set) (struct dvb_frontend* fe, int index);
-
-	/* Need to set device param for start_dma */
 	int (*set_ts_params)(struct dvb_frontend* fe, int is_punctured);
-
-	/* Flip the polarity of the mpeg data transfer clock using alternate init data
-	 * This option applies ONLY to LGDT3303 - 0:disabled (default) 1:enabled */
 	int clock_polarity_flip;
+
+	struct dvb_frontend* (*get_dvb_frontend)(struct i2c_client *);
 };
 
 #if IS_REACHABLE(CONFIG_DVB_LGDT330X)
-extern struct dvb_frontend* lgdt330x_attach(const struct lgdt330x_config* config,
-					    struct i2c_adapter* i2c);
+struct dvb_frontend *lgdt330x_attach(const struct lgdt330x_config *config,
+				     u8 demod_address,
+				     struct i2c_adapter *i2c);
 #else
-static inline struct dvb_frontend* lgdt330x_attach(const struct lgdt330x_config* config,
-					    struct i2c_adapter* i2c)
+static inline
+struct dvb_frontend *lgdt330x_attach(const struct lgdt330x_config *config,
+				     u8 demod_address,
+				     struct i2c_adapter *i2c)
 {
 	printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__);
 	return NULL;
diff --git a/drivers/media/dvb-frontends/mb86a20s.c b/drivers/media/dvb-frontends/mb86a20s.c
index 36e95196dff4..c3b1b88e2e7a 100644
--- a/drivers/media/dvb-frontends/mb86a20s.c
+++ b/drivers/media/dvb-frontends/mb86a20s.c
@@ -2055,7 +2055,7 @@ static void mb86a20s_release(struct dvb_frontend *fe)
 	kfree(state);
 }
 
-static int mb86a20s_get_frontend_algo(struct dvb_frontend *fe)
+static enum dvbfe_algo mb86a20s_get_frontend_algo(struct dvb_frontend *fe)
 {
 	return DVBFE_ALGO_HW;
 }
diff --git a/drivers/media/dvb-frontends/mxl5xx.c b/drivers/media/dvb-frontends/mxl5xx.c
index 483ee7d6198e..274d8fca0763 100644
--- a/drivers/media/dvb-frontends/mxl5xx.c
+++ b/drivers/media/dvb-frontends/mxl5xx.c
@@ -375,7 +375,7 @@ static void release(struct dvb_frontend *fe)
 	kfree(state);
 }
 
-static int get_algo(struct dvb_frontend *fe)
+static enum dvbfe_algo get_algo(struct dvb_frontend *fe)
 {
 	return DVBFE_ALGO_HW;
 }
diff --git a/drivers/media/dvb-frontends/s921.c b/drivers/media/dvb-frontends/s921.c
index 2d75ede77aca..6c9015236655 100644
--- a/drivers/media/dvb-frontends/s921.c
+++ b/drivers/media/dvb-frontends/s921.c
@@ -464,7 +464,7 @@ static int s921_tune(struct dvb_frontend *fe,
 	return rc;
 }
 
-static int s921_get_algo(struct dvb_frontend *fe)
+static enum dvbfe_algo s921_get_algo(struct dvb_frontend *fe)
 {
 	return DVBFE_ALGO_HW;
 }
diff --git a/drivers/media/dvb-frontends/stv0910.c b/drivers/media/dvb-frontends/stv0910.c
index 52355c14fd64..41444fa1c0bb 100644
--- a/drivers/media/dvb-frontends/stv0910.c
+++ b/drivers/media/dvb-frontends/stv0910.c
@@ -1200,7 +1200,6 @@ static int probe(struct stv *state)
 	write_reg(state, RSTV0910_P1_TSCFGM, 0xC0); /* Manual speed */
 	write_reg(state, RSTV0910_P1_TSCFGL, 0x20);
 
-	/* Speed = 67.5 MHz */
 	write_reg(state, RSTV0910_P1_TSSPEED, state->tsspeed);
 
 	write_reg(state, RSTV0910_P2_TSCFGH, state->tscfgh | 0x01);
@@ -1208,7 +1207,6 @@ static int probe(struct stv *state)
 	write_reg(state, RSTV0910_P2_TSCFGM, 0xC0); /* Manual speed */
 	write_reg(state, RSTV0910_P2_TSCFGL, 0x20);
 
-	/* Speed = 67.5 MHz */
 	write_reg(state, RSTV0910_P2_TSSPEED, state->tsspeed);
 
 	/* Reset stream merger */
@@ -1220,6 +1218,12 @@ static int probe(struct stv *state)
 	write_reg(state, RSTV0910_P1_I2CRPT, state->i2crpt);
 	write_reg(state, RSTV0910_P2_I2CRPT, state->i2crpt);
 
+	write_reg(state, RSTV0910_P1_TSINSDELM, 0x17);
+	write_reg(state, RSTV0910_P1_TSINSDELL, 0xff);
+
+	write_reg(state, RSTV0910_P2_TSINSDELM, 0x17);
+	write_reg(state, RSTV0910_P2_TSINSDELL, 0xff);
+
 	init_diseqc(state);
 	return 0;
 }
@@ -1320,7 +1324,7 @@ static int read_snr(struct dvb_frontend *fe)
 
 	if (!get_signal_to_noise(state, &snrval)) {
 		p->cnr.stat[0].scale = FE_SCALE_DECIBEL;
-		p->cnr.stat[0].uvalue = 100 * snrval; /* fix scale */
+		p->cnr.stat[0].svalue = 100 * snrval; /* fix scale */
 	} else {
 		p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
 	}
@@ -1633,7 +1637,7 @@ static int tune(struct dvb_frontend *fe, bool re_tune,
 	return 0;
 }
 
-static int get_algo(struct dvb_frontend *fe)
+static enum dvbfe_algo get_algo(struct dvb_frontend *fe)
 {
 	return DVBFE_ALGO_HW;
 }
@@ -1784,7 +1788,8 @@ struct dvb_frontend *stv0910_attach(struct i2c_adapter *i2c,
 	state->tscfgh = 0x20 | (cfg->parallel ? 0 : 0x40);
 	state->tsgeneral = (cfg->parallel == 2) ? 0x02 : 0x00;
 	state->i2crpt = 0x0A | ((cfg->rptlvl & 0x07) << 4);
-	state->tsspeed = 0x28;
+	/* use safe tsspeed value if unspecified through stv0910_cfg */
+	state->tsspeed = (cfg->tsspeed ? cfg->tsspeed : 0x28);
 	state->nr = nr;
 	state->regoff = state->nr ? 0 : 0x200;
 	state->search_range = 16000000;
diff --git a/drivers/media/dvb-frontends/stv0910.h b/drivers/media/dvb-frontends/stv0910.h
index fccd8d9b665f..f37171b7a2de 100644
--- a/drivers/media/dvb-frontends/stv0910.h
+++ b/drivers/media/dvb-frontends/stv0910.h
@@ -10,6 +10,7 @@ struct stv0910_cfg {
 	u8  parallel;
 	u8  rptlvl;
 	u8  single;
+	u8  tsspeed;
 };
 
 #if IS_REACHABLE(CONFIG_DVB_STV0910)
diff --git a/drivers/media/dvb-frontends/tc90522.c b/drivers/media/dvb-frontends/tc90522.c
index 5572b39614d5..7abf6b0916ed 100644
--- a/drivers/media/dvb-frontends/tc90522.c
+++ b/drivers/media/dvb-frontends/tc90522.c
@@ -1,17 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Toshiba TC90522 Demodulator
  *
  * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /*
@@ -352,7 +343,7 @@ static int tc90522t_get_frontend(struct dvb_frontend *fe,
 	mode = 1;
 	ret = reg_read(state, 0xb0, val, 1);
 	if (ret == 0) {
-		mode = (val[0] & 0xc0) >> 2;
+		mode = (val[0] & 0xc0) >> 6;
 		c->transmission_mode = tm_conv[mode];
 		c->guard_interval = (val[0] & 0x30) >> 4;
 	}
@@ -379,7 +370,7 @@ static int tc90522t_get_frontend(struct dvb_frontend *fe,
 		}
 
 		/* layer B */
-		v = (val[3] & 0x03) << 1 | (val[4] & 0xc0) >> 6;
+		v = (val[3] & 0x03) << 2 | (val[4] & 0xc0) >> 6;
 		if (v == 0x0f)
 			c->layer[1].segment_count = 0;
 		else {
diff --git a/drivers/media/dvb-frontends/tc90522.h b/drivers/media/dvb-frontends/tc90522.h
index 10e585f32499..ac0e2ab51924 100644
--- a/drivers/media/dvb-frontends/tc90522.h
+++ b/drivers/media/dvb-frontends/tc90522.h
@@ -1,17 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Toshiba TC90522 Demodulator
  *
  * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /*
diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
index 541f0d28afd8..341452fe98df 100644
--- a/drivers/media/i2c/Kconfig
+++ b/drivers/media/i2c/Kconfig
@@ -575,6 +575,17 @@ config VIDEO_APTINA_PLL
 config VIDEO_SMIAPP_PLL
 	tristate
 
+config VIDEO_IMX258
+	tristate "Sony IMX258 sensor support"
+	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
+	depends on MEDIA_CAMERA_SUPPORT
+	---help---
+	  This is a Video4Linux2 sensor-level driver for the Sony
+	  IMX258 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called imx258.
+
 config VIDEO_IMX274
 	tristate "Sony IMX274 sensor support"
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
@@ -688,6 +699,18 @@ config VIDEO_OV5695
 	  To compile this driver as a module, choose M here: the
 	  module will be called ov5695.
 
+config VIDEO_OV7251
+	tristate "OmniVision OV7251 sensor support"
+	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
+	depends on MEDIA_CAMERA_SUPPORT
+	select V4L2_FWNODE
+	help
+	  This is a Video4Linux2 sensor-level driver for the OmniVision
+	  OV7251 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ov7251.
+
 config VIDEO_OV772X
 	tristate "OmniVision OV772x sensor support"
 	depends on I2C && VIDEO_V4L2
@@ -974,6 +997,19 @@ config VIDEO_M52790
 
 	 To compile this driver as a module, choose M here: the
 	 module will be called m52790.
+
+config VIDEO_I2C
+	tristate "I2C transport video support"
+	depends on VIDEO_V4L2 && I2C
+	select VIDEOBUF2_VMALLOC
+	---help---
+	  Enable the I2C transport video support which supports the
+	  following:
+	   * Panasonic AMG88xx Grid-Eye Sensors
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called video-i2c
+
 endmenu
 
 menu "Sensors used on soc_camera driver"
diff --git a/drivers/media/i2c/Makefile b/drivers/media/i2c/Makefile
index ea34aee1a85a..d679d57cd3b3 100644
--- a/drivers/media/i2c/Makefile
+++ b/drivers/media/i2c/Makefile
@@ -70,6 +70,7 @@ obj-$(CONFIG_VIDEO_OV5647) += ov5647.o
 obj-$(CONFIG_VIDEO_OV5670) += ov5670.o
 obj-$(CONFIG_VIDEO_OV5695) += ov5695.o
 obj-$(CONFIG_VIDEO_OV6650) += ov6650.o
+obj-$(CONFIG_VIDEO_OV7251) += ov7251.o
 obj-$(CONFIG_VIDEO_OV7640) += ov7640.o
 obj-$(CONFIG_VIDEO_OV7670) += ov7670.o
 obj-$(CONFIG_VIDEO_OV772X) += ov772x.o
@@ -96,9 +97,11 @@ obj-$(CONFIG_VIDEO_LM3646)	+= lm3646.o
 obj-$(CONFIG_VIDEO_SMIAPP_PLL)	+= smiapp-pll.o
 obj-$(CONFIG_VIDEO_AK881X)		+= ak881x.o
 obj-$(CONFIG_VIDEO_IR_I2C)  += ir-kbd-i2c.o
+obj-$(CONFIG_VIDEO_I2C)		+= video-i2c.o
 obj-$(CONFIG_VIDEO_ML86V7667)	+= ml86v7667.o
 obj-$(CONFIG_VIDEO_OV2659)	+= ov2659.o
 obj-$(CONFIG_VIDEO_TC358743)	+= tc358743.o
+obj-$(CONFIG_VIDEO_IMX258)	+= imx258.o
 obj-$(CONFIG_VIDEO_IMX274)	+= imx274.o
 
 obj-$(CONFIG_SDR_MAX2175) += max2175.o
diff --git a/drivers/media/i2c/adv748x/adv748x-afe.c b/drivers/media/i2c/adv748x/adv748x-afe.c
index 61514bae7e5c..edd25e895e5d 100644
--- a/drivers/media/i2c/adv748x/adv748x-afe.c
+++ b/drivers/media/i2c/adv748x/adv748x-afe.c
@@ -321,17 +321,17 @@ static const struct v4l2_subdev_video_ops adv748x_afe_video_ops = {
 static int adv748x_afe_propagate_pixelrate(struct adv748x_afe *afe)
 {
 	struct v4l2_subdev *tx;
-	unsigned int width, height, fps;
 
 	tx = adv748x_get_remote_sd(&afe->pads[ADV748X_AFE_SOURCE]);
 	if (!tx)
 		return -ENOLINK;
 
-	width = 720;
-	height = afe->curr_norm & V4L2_STD_525_60 ? 480 : 576;
-	fps = afe->curr_norm & V4L2_STD_525_60 ? 30 : 25;
-
-	return adv748x_csi2_set_pixelrate(tx, width * height * fps);
+	/*
+	 * The ADV748x ADC sampling frequency is twice the externally supplied
+	 * clock whose frequency is required to be 28.63636 MHz. It oversamples
+	 * with a factor of 4 resulting in a pixel rate of 14.3180180 MHz.
+	 */
+	return adv748x_csi2_set_pixelrate(tx, 14318180);
 }
 
 static int adv748x_afe_enum_mbus_code(struct v4l2_subdev *sd,
diff --git a/drivers/media/i2c/adv748x/adv748x-hdmi.c b/drivers/media/i2c/adv748x/adv748x-hdmi.c
index 10d229a4f088..aecc2a84dfec 100644
--- a/drivers/media/i2c/adv748x/adv748x-hdmi.c
+++ b/drivers/media/i2c/adv748x/adv748x-hdmi.c
@@ -402,8 +402,6 @@ static int adv748x_hdmi_propagate_pixelrate(struct adv748x_hdmi *hdmi)
 {
 	struct v4l2_subdev *tx;
 	struct v4l2_dv_timings timings;
-	struct v4l2_bt_timings *bt = &timings.bt;
-	unsigned int fps;
 
 	tx = adv748x_get_remote_sd(&hdmi->pads[ADV748X_HDMI_SOURCE]);
 	if (!tx)
@@ -411,11 +409,7 @@ static int adv748x_hdmi_propagate_pixelrate(struct adv748x_hdmi *hdmi)
 
 	adv748x_hdmi_query_dv_timings(&hdmi->sd, &timings);
 
-	fps = DIV_ROUND_CLOSEST_ULL(bt->pixelclock,
-				    V4L2_DV_BT_FRAME_WIDTH(bt) *
-				    V4L2_DV_BT_FRAME_HEIGHT(bt));
-
-	return adv748x_csi2_set_pixelrate(tx, bt->width * bt->height * fps);
+	return adv748x_csi2_set_pixelrate(tx, timings.bt.pixelclock);
 }
 
 static int adv748x_hdmi_enum_mbus_code(struct v4l2_subdev *sd,
diff --git a/drivers/media/i2c/adv7511.c b/drivers/media/i2c/adv7511.c
index d23505a411ee..5731751d3f2a 100644
--- a/drivers/media/i2c/adv7511.c
+++ b/drivers/media/i2c/adv7511.c
@@ -732,8 +732,8 @@ static int adv7511_cec_adap_enable(struct cec_adapter *adap, bool enable)
 		/* power up cec section */
 		adv7511_cec_write_and_or(sd, 0x4e, 0xfc, 0x01);
 		/* legacy mode and clear all rx buffers */
+		adv7511_cec_write(sd, 0x4a, 0x00);
 		adv7511_cec_write(sd, 0x4a, 0x07);
-		adv7511_cec_write(sd, 0x4a, 0);
 		adv7511_cec_write_and_or(sd, 0x11, 0xfe, 0); /* initially disable tx */
 		/* enabled irqs: */
 		/* tx: ready */
@@ -831,8 +831,8 @@ static int adv7511_cec_adap_transmit(struct cec_adapter *adap, u8 attempts,
 	 */
 	adv7511_cec_write_and_or(sd, 0x12, ~0x70, max(1, attempts - 1) << 4);
 
-	/* blocking, clear cec tx irq status */
-	adv7511_wr_and_or(sd, 0x97, 0xc7, 0x38);
+	/* clear cec tx irq status */
+	adv7511_wr(sd, 0x97, 0x38);
 
 	/* write data */
 	for (i = 0; i < len; i++)
@@ -917,9 +917,6 @@ static void adv7511_set_isr(struct v4l2_subdev *sd, bool enable)
 	else if (adv7511_have_hotplug(sd))
 		irqs |= MASK_ADV7511_EDID_RDY_INT;
 
-	adv7511_wr_and_or(sd, 0x95, 0xc0,
-			  (state->cec_enabled_adap && enable) ? 0x39 : 0x00);
-
 	/*
 	 * This i2c write can fail (approx. 1 in 1000 writes). But it
 	 * is essential that this register is correct, so retry it
@@ -933,9 +930,11 @@ static void adv7511_set_isr(struct v4l2_subdev *sd, bool enable)
 		irqs_rd = adv7511_rd(sd, 0x94);
 	} while (retries-- && irqs_rd != irqs);
 
-	if (irqs_rd == irqs)
-		return;
-	v4l2_err(sd, "Could not set interrupts: hw failure?\n");
+	if (irqs_rd != irqs)
+		v4l2_err(sd, "Could not set interrupts: hw failure?\n");
+
+	adv7511_wr_and_or(sd, 0x95, 0xc0,
+			  (state->cec_enabled_adap && enable) ? 0x39 : 0x00);
 }
 
 /* Interrupt handler */
@@ -982,8 +981,8 @@ static int adv7511_isr(struct v4l2_subdev *sd, u32 status, bool *handled)
 			for (i = 0; i < msg.len; i++)
 				msg.msg[i] = adv7511_cec_read(sd, i + 0x15);
 
-			adv7511_cec_write(sd, 0x4a, 1); /* toggle to re-enable rx 1 */
-			adv7511_cec_write(sd, 0x4a, 0);
+			adv7511_cec_write(sd, 0x4a, 0); /* toggle to re-enable rx 1 */
+			adv7511_cec_write(sd, 0x4a, 1);
 			cec_received_msg(state->cec_adap, &msg);
 		}
 	}
@@ -1778,6 +1777,7 @@ static void adv7511_init_setup(struct v4l2_subdev *sd)
 
 	/* legacy mode */
 	adv7511_cec_write(sd, 0x4a, 0x00);
+	adv7511_cec_write(sd, 0x4a, 0x07);
 
 	if (cec_clk % 750000 != 0)
 		v4l2_err(sd, "%s: cec_clk %d, not multiple of 750 Khz\n",
diff --git a/drivers/media/i2c/imx258.c b/drivers/media/i2c/imx258.c
new file mode 100644
index 000000000000..f3b124723aa0
--- /dev/null
+++ b/drivers/media/i2c/imx258.c
@@ -0,0 +1,1318 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2018 Intel Corporation
+
+#include <linux/acpi.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <asm/unaligned.h>
+
+#define IMX258_REG_VALUE_08BIT		1
+#define IMX258_REG_VALUE_16BIT		2
+
+#define IMX258_REG_MODE_SELECT		0x0100
+#define IMX258_MODE_STANDBY		0x00
+#define IMX258_MODE_STREAMING		0x01
+
+/* Chip ID */
+#define IMX258_REG_CHIP_ID		0x0016
+#define IMX258_CHIP_ID			0x0258
+
+/* V_TIMING internal */
+#define IMX258_VTS_30FPS		0x0c98
+#define IMX258_VTS_30FPS_2K		0x0638
+#define IMX258_VTS_30FPS_VGA		0x034c
+#define IMX258_VTS_MAX			0xffff
+
+/*Frame Length Line*/
+#define IMX258_FLL_MIN			0x08a6
+#define IMX258_FLL_MAX			0xffff
+#define IMX258_FLL_STEP			1
+#define IMX258_FLL_DEFAULT		0x0c98
+
+/* HBLANK control - read only */
+#define IMX258_PPL_DEFAULT		5352
+
+/* Exposure control */
+#define IMX258_REG_EXPOSURE		0x0202
+#define IMX258_EXPOSURE_MIN		4
+#define IMX258_EXPOSURE_STEP		1
+#define IMX258_EXPOSURE_DEFAULT		0x640
+#define IMX258_EXPOSURE_MAX		65535
+
+/* Analog gain control */
+#define IMX258_REG_ANALOG_GAIN		0x0204
+#define IMX258_ANA_GAIN_MIN		0
+#define IMX258_ANA_GAIN_MAX		0x1fff
+#define IMX258_ANA_GAIN_STEP		1
+#define IMX258_ANA_GAIN_DEFAULT		0x0
+
+/* Digital gain control */
+#define IMX258_REG_GR_DIGITAL_GAIN	0x020e
+#define IMX258_REG_R_DIGITAL_GAIN	0x0210
+#define IMX258_REG_B_DIGITAL_GAIN	0x0212
+#define IMX258_REG_GB_DIGITAL_GAIN	0x0214
+#define IMX258_DGTL_GAIN_MIN		0
+#define IMX258_DGTL_GAIN_MAX		4096	/* Max = 0xFFF */
+#define IMX258_DGTL_GAIN_DEFAULT	1024
+#define IMX258_DGTL_GAIN_STEP		1
+
+/* Test Pattern Control */
+#define IMX258_REG_TEST_PATTERN		0x0600
+#define IMX258_TEST_PATTERN_DISABLE	0
+#define IMX258_TEST_PATTERN_SOLID_COLOR	1
+#define IMX258_TEST_PATTERN_COLOR_BARS	2
+#define IMX258_TEST_PATTERN_GREY_COLOR	3
+#define IMX258_TEST_PATTERN_PN9		4
+
+/* Orientation */
+#define REG_MIRROR_FLIP_CONTROL		0x0101
+#define REG_CONFIG_MIRROR_FLIP		0x03
+#define REG_CONFIG_FLIP_TEST_PATTERN	0x02
+
+struct imx258_reg {
+	u16 address;
+	u8 val;
+};
+
+struct imx258_reg_list {
+	u32 num_of_regs;
+	const struct imx258_reg *regs;
+};
+
+/* Link frequency config */
+struct imx258_link_freq_config {
+	u32 pixels_per_line;
+
+	/* PLL registers for this link frequency */
+	struct imx258_reg_list reg_list;
+};
+
+/* Mode : resolution and related config&values */
+struct imx258_mode {
+	/* Frame width */
+	u32 width;
+	/* Frame height */
+	u32 height;
+
+	/* V-timing */
+	u32 vts_def;
+	u32 vts_min;
+
+	/* Index of Link frequency config to be used */
+	u32 link_freq_index;
+	/* Default register values */
+	struct imx258_reg_list reg_list;
+};
+
+/* 4208x3118 needs 1267Mbps/lane, 4 lanes */
+static const struct imx258_reg mipi_data_rate_1267mbps[] = {
+	{ 0x0301, 0x05 },
+	{ 0x0303, 0x02 },
+	{ 0x0305, 0x03 },
+	{ 0x0306, 0x00 },
+	{ 0x0307, 0xC6 },
+	{ 0x0309, 0x0A },
+	{ 0x030B, 0x01 },
+	{ 0x030D, 0x02 },
+	{ 0x030E, 0x00 },
+	{ 0x030F, 0xD8 },
+	{ 0x0310, 0x00 },
+	{ 0x0820, 0x13 },
+	{ 0x0821, 0x4C },
+	{ 0x0822, 0xCC },
+	{ 0x0823, 0xCC },
+};
+
+static const struct imx258_reg mipi_data_rate_640mbps[] = {
+	{ 0x0301, 0x05 },
+	{ 0x0303, 0x02 },
+	{ 0x0305, 0x03 },
+	{ 0x0306, 0x00 },
+	{ 0x0307, 0x64 },
+	{ 0x0309, 0x0A },
+	{ 0x030B, 0x01 },
+	{ 0x030D, 0x02 },
+	{ 0x030E, 0x00 },
+	{ 0x030F, 0xD8 },
+	{ 0x0310, 0x00 },
+	{ 0x0820, 0x0A },
+	{ 0x0821, 0x00 },
+	{ 0x0822, 0x00 },
+	{ 0x0823, 0x00 },
+};
+
+static const struct imx258_reg mode_4208x3118_regs[] = {
+	{ 0x0136, 0x13 },
+	{ 0x0137, 0x33 },
+	{ 0x3051, 0x00 },
+	{ 0x3052, 0x00 },
+	{ 0x4E21, 0x14 },
+	{ 0x6B11, 0xCF },
+	{ 0x7FF0, 0x08 },
+	{ 0x7FF1, 0x0F },
+	{ 0x7FF2, 0x08 },
+	{ 0x7FF3, 0x1B },
+	{ 0x7FF4, 0x23 },
+	{ 0x7FF5, 0x60 },
+	{ 0x7FF6, 0x00 },
+	{ 0x7FF7, 0x01 },
+	{ 0x7FF8, 0x00 },
+	{ 0x7FF9, 0x78 },
+	{ 0x7FFA, 0x00 },
+	{ 0x7FFB, 0x00 },
+	{ 0x7FFC, 0x00 },
+	{ 0x7FFD, 0x00 },
+	{ 0x7FFE, 0x00 },
+	{ 0x7FFF, 0x03 },
+	{ 0x7F76, 0x03 },
+	{ 0x7F77, 0xFE },
+	{ 0x7FA8, 0x03 },
+	{ 0x7FA9, 0xFE },
+	{ 0x7B24, 0x81 },
+	{ 0x7B25, 0x00 },
+	{ 0x6564, 0x07 },
+	{ 0x6B0D, 0x41 },
+	{ 0x653D, 0x04 },
+	{ 0x6B05, 0x8C },
+	{ 0x6B06, 0xF9 },
+	{ 0x6B08, 0x65 },
+	{ 0x6B09, 0xFC },
+	{ 0x6B0A, 0xCF },
+	{ 0x6B0B, 0xD2 },
+	{ 0x6700, 0x0E },
+	{ 0x6707, 0x0E },
+	{ 0x9104, 0x00 },
+	{ 0x4648, 0x7F },
+	{ 0x7420, 0x00 },
+	{ 0x7421, 0x1C },
+	{ 0x7422, 0x00 },
+	{ 0x7423, 0xD7 },
+	{ 0x5F04, 0x00 },
+	{ 0x5F05, 0xED },
+	{ 0x0112, 0x0A },
+	{ 0x0113, 0x0A },
+	{ 0x0114, 0x03 },
+	{ 0x0342, 0x14 },
+	{ 0x0343, 0xE8 },
+	{ 0x0340, 0x0C },
+	{ 0x0341, 0x50 },
+	{ 0x0344, 0x00 },
+	{ 0x0345, 0x00 },
+	{ 0x0346, 0x00 },
+	{ 0x0347, 0x00 },
+	{ 0x0348, 0x10 },
+	{ 0x0349, 0x6F },
+	{ 0x034A, 0x0C },
+	{ 0x034B, 0x2E },
+	{ 0x0381, 0x01 },
+	{ 0x0383, 0x01 },
+	{ 0x0385, 0x01 },
+	{ 0x0387, 0x01 },
+	{ 0x0900, 0x00 },
+	{ 0x0901, 0x11 },
+	{ 0x0401, 0x00 },
+	{ 0x0404, 0x00 },
+	{ 0x0405, 0x10 },
+	{ 0x0408, 0x00 },
+	{ 0x0409, 0x00 },
+	{ 0x040A, 0x00 },
+	{ 0x040B, 0x00 },
+	{ 0x040C, 0x10 },
+	{ 0x040D, 0x70 },
+	{ 0x040E, 0x0C },
+	{ 0x040F, 0x30 },
+	{ 0x3038, 0x00 },
+	{ 0x303A, 0x00 },
+	{ 0x303B, 0x10 },
+	{ 0x300D, 0x00 },
+	{ 0x034C, 0x10 },
+	{ 0x034D, 0x70 },
+	{ 0x034E, 0x0C },
+	{ 0x034F, 0x30 },
+	{ 0x0350, 0x01 },
+	{ 0x0202, 0x0C },
+	{ 0x0203, 0x46 },
+	{ 0x0204, 0x00 },
+	{ 0x0205, 0x00 },
+	{ 0x020E, 0x01 },
+	{ 0x020F, 0x00 },
+	{ 0x0210, 0x01 },
+	{ 0x0211, 0x00 },
+	{ 0x0212, 0x01 },
+	{ 0x0213, 0x00 },
+	{ 0x0214, 0x01 },
+	{ 0x0215, 0x00 },
+	{ 0x7BCD, 0x00 },
+	{ 0x94DC, 0x20 },
+	{ 0x94DD, 0x20 },
+	{ 0x94DE, 0x20 },
+	{ 0x95DC, 0x20 },
+	{ 0x95DD, 0x20 },
+	{ 0x95DE, 0x20 },
+	{ 0x7FB0, 0x00 },
+	{ 0x9010, 0x3E },
+	{ 0x9419, 0x50 },
+	{ 0x941B, 0x50 },
+	{ 0x9519, 0x50 },
+	{ 0x951B, 0x50 },
+	{ 0x3030, 0x00 },
+	{ 0x3032, 0x00 },
+	{ 0x0220, 0x00 },
+};
+
+static const struct imx258_reg mode_2104_1560_regs[] = {
+	{ 0x0136, 0x13 },
+	{ 0x0137, 0x33 },
+	{ 0x3051, 0x00 },
+	{ 0x3052, 0x00 },
+	{ 0x4E21, 0x14 },
+	{ 0x6B11, 0xCF },
+	{ 0x7FF0, 0x08 },
+	{ 0x7FF1, 0x0F },
+	{ 0x7FF2, 0x08 },
+	{ 0x7FF3, 0x1B },
+	{ 0x7FF4, 0x23 },
+	{ 0x7FF5, 0x60 },
+	{ 0x7FF6, 0x00 },
+	{ 0x7FF7, 0x01 },
+	{ 0x7FF8, 0x00 },
+	{ 0x7FF9, 0x78 },
+	{ 0x7FFA, 0x00 },
+	{ 0x7FFB, 0x00 },
+	{ 0x7FFC, 0x00 },
+	{ 0x7FFD, 0x00 },
+	{ 0x7FFE, 0x00 },
+	{ 0x7FFF, 0x03 },
+	{ 0x7F76, 0x03 },
+	{ 0x7F77, 0xFE },
+	{ 0x7FA8, 0x03 },
+	{ 0x7FA9, 0xFE },
+	{ 0x7B24, 0x81 },
+	{ 0x7B25, 0x00 },
+	{ 0x6564, 0x07 },
+	{ 0x6B0D, 0x41 },
+	{ 0x653D, 0x04 },
+	{ 0x6B05, 0x8C },
+	{ 0x6B06, 0xF9 },
+	{ 0x6B08, 0x65 },
+	{ 0x6B09, 0xFC },
+	{ 0x6B0A, 0xCF },
+	{ 0x6B0B, 0xD2 },
+	{ 0x6700, 0x0E },
+	{ 0x6707, 0x0E },
+	{ 0x9104, 0x00 },
+	{ 0x4648, 0x7F },
+	{ 0x7420, 0x00 },
+	{ 0x7421, 0x1C },
+	{ 0x7422, 0x00 },
+	{ 0x7423, 0xD7 },
+	{ 0x5F04, 0x00 },
+	{ 0x5F05, 0xED },
+	{ 0x0112, 0x0A },
+	{ 0x0113, 0x0A },
+	{ 0x0114, 0x03 },
+	{ 0x0342, 0x14 },
+	{ 0x0343, 0xE8 },
+	{ 0x0340, 0x06 },
+	{ 0x0341, 0x38 },
+	{ 0x0344, 0x00 },
+	{ 0x0345, 0x00 },
+	{ 0x0346, 0x00 },
+	{ 0x0347, 0x00 },
+	{ 0x0348, 0x10 },
+	{ 0x0349, 0x6F },
+	{ 0x034A, 0x0C },
+	{ 0x034B, 0x2E },
+	{ 0x0381, 0x01 },
+	{ 0x0383, 0x01 },
+	{ 0x0385, 0x01 },
+	{ 0x0387, 0x01 },
+	{ 0x0900, 0x01 },
+	{ 0x0901, 0x12 },
+	{ 0x0401, 0x01 },
+	{ 0x0404, 0x00 },
+	{ 0x0405, 0x20 },
+	{ 0x0408, 0x00 },
+	{ 0x0409, 0x02 },
+	{ 0x040A, 0x00 },
+	{ 0x040B, 0x00 },
+	{ 0x040C, 0x10 },
+	{ 0x040D, 0x6A },
+	{ 0x040E, 0x06 },
+	{ 0x040F, 0x18 },
+	{ 0x3038, 0x00 },
+	{ 0x303A, 0x00 },
+	{ 0x303B, 0x10 },
+	{ 0x300D, 0x00 },
+	{ 0x034C, 0x08 },
+	{ 0x034D, 0x38 },
+	{ 0x034E, 0x06 },
+	{ 0x034F, 0x18 },
+	{ 0x0350, 0x01 },
+	{ 0x0202, 0x06 },
+	{ 0x0203, 0x2E },
+	{ 0x0204, 0x00 },
+	{ 0x0205, 0x00 },
+	{ 0x020E, 0x01 },
+	{ 0x020F, 0x00 },
+	{ 0x0210, 0x01 },
+	{ 0x0211, 0x00 },
+	{ 0x0212, 0x01 },
+	{ 0x0213, 0x00 },
+	{ 0x0214, 0x01 },
+	{ 0x0215, 0x00 },
+	{ 0x7BCD, 0x01 },
+	{ 0x94DC, 0x20 },
+	{ 0x94DD, 0x20 },
+	{ 0x94DE, 0x20 },
+	{ 0x95DC, 0x20 },
+	{ 0x95DD, 0x20 },
+	{ 0x95DE, 0x20 },
+	{ 0x7FB0, 0x00 },
+	{ 0x9010, 0x3E },
+	{ 0x9419, 0x50 },
+	{ 0x941B, 0x50 },
+	{ 0x9519, 0x50 },
+	{ 0x951B, 0x50 },
+	{ 0x3030, 0x00 },
+	{ 0x3032, 0x00 },
+	{ 0x0220, 0x00 },
+};
+
+static const struct imx258_reg mode_1048_780_regs[] = {
+	{ 0x0136, 0x13 },
+	{ 0x0137, 0x33 },
+	{ 0x3051, 0x00 },
+	{ 0x3052, 0x00 },
+	{ 0x4E21, 0x14 },
+	{ 0x6B11, 0xCF },
+	{ 0x7FF0, 0x08 },
+	{ 0x7FF1, 0x0F },
+	{ 0x7FF2, 0x08 },
+	{ 0x7FF3, 0x1B },
+	{ 0x7FF4, 0x23 },
+	{ 0x7FF5, 0x60 },
+	{ 0x7FF6, 0x00 },
+	{ 0x7FF7, 0x01 },
+	{ 0x7FF8, 0x00 },
+	{ 0x7FF9, 0x78 },
+	{ 0x7FFA, 0x00 },
+	{ 0x7FFB, 0x00 },
+	{ 0x7FFC, 0x00 },
+	{ 0x7FFD, 0x00 },
+	{ 0x7FFE, 0x00 },
+	{ 0x7FFF, 0x03 },
+	{ 0x7F76, 0x03 },
+	{ 0x7F77, 0xFE },
+	{ 0x7FA8, 0x03 },
+	{ 0x7FA9, 0xFE },
+	{ 0x7B24, 0x81 },
+	{ 0x7B25, 0x00 },
+	{ 0x6564, 0x07 },
+	{ 0x6B0D, 0x41 },
+	{ 0x653D, 0x04 },
+	{ 0x6B05, 0x8C },
+	{ 0x6B06, 0xF9 },
+	{ 0x6B08, 0x65 },
+	{ 0x6B09, 0xFC },
+	{ 0x6B0A, 0xCF },
+	{ 0x6B0B, 0xD2 },
+	{ 0x6700, 0x0E },
+	{ 0x6707, 0x0E },
+	{ 0x9104, 0x00 },
+	{ 0x4648, 0x7F },
+	{ 0x7420, 0x00 },
+	{ 0x7421, 0x1C },
+	{ 0x7422, 0x00 },
+	{ 0x7423, 0xD7 },
+	{ 0x5F04, 0x00 },
+	{ 0x5F05, 0xED },
+	{ 0x0112, 0x0A },
+	{ 0x0113, 0x0A },
+	{ 0x0114, 0x03 },
+	{ 0x0342, 0x14 },
+	{ 0x0343, 0xE8 },
+	{ 0x0340, 0x03 },
+	{ 0x0341, 0x4C },
+	{ 0x0344, 0x00 },
+	{ 0x0345, 0x00 },
+	{ 0x0346, 0x00 },
+	{ 0x0347, 0x00 },
+	{ 0x0348, 0x10 },
+	{ 0x0349, 0x6F },
+	{ 0x034A, 0x0C },
+	{ 0x034B, 0x2E },
+	{ 0x0381, 0x01 },
+	{ 0x0383, 0x01 },
+	{ 0x0385, 0x01 },
+	{ 0x0387, 0x01 },
+	{ 0x0900, 0x01 },
+	{ 0x0901, 0x14 },
+	{ 0x0401, 0x01 },
+	{ 0x0404, 0x00 },
+	{ 0x0405, 0x40 },
+	{ 0x0408, 0x00 },
+	{ 0x0409, 0x06 },
+	{ 0x040A, 0x00 },
+	{ 0x040B, 0x00 },
+	{ 0x040C, 0x10 },
+	{ 0x040D, 0x64 },
+	{ 0x040E, 0x03 },
+	{ 0x040F, 0x0C },
+	{ 0x3038, 0x00 },
+	{ 0x303A, 0x00 },
+	{ 0x303B, 0x10 },
+	{ 0x300D, 0x00 },
+	{ 0x034C, 0x04 },
+	{ 0x034D, 0x18 },
+	{ 0x034E, 0x03 },
+	{ 0x034F, 0x0C },
+	{ 0x0350, 0x01 },
+	{ 0x0202, 0x03 },
+	{ 0x0203, 0x42 },
+	{ 0x0204, 0x00 },
+	{ 0x0205, 0x00 },
+	{ 0x020E, 0x01 },
+	{ 0x020F, 0x00 },
+	{ 0x0210, 0x01 },
+	{ 0x0211, 0x00 },
+	{ 0x0212, 0x01 },
+	{ 0x0213, 0x00 },
+	{ 0x0214, 0x01 },
+	{ 0x0215, 0x00 },
+	{ 0x7BCD, 0x00 },
+	{ 0x94DC, 0x20 },
+	{ 0x94DD, 0x20 },
+	{ 0x94DE, 0x20 },
+	{ 0x95DC, 0x20 },
+	{ 0x95DD, 0x20 },
+	{ 0x95DE, 0x20 },
+	{ 0x7FB0, 0x00 },
+	{ 0x9010, 0x3E },
+	{ 0x9419, 0x50 },
+	{ 0x941B, 0x50 },
+	{ 0x9519, 0x50 },
+	{ 0x951B, 0x50 },
+	{ 0x3030, 0x00 },
+	{ 0x3032, 0x00 },
+	{ 0x0220, 0x00 },
+};
+
+static const char * const imx258_test_pattern_menu[] = {
+	"Disabled",
+	"Color Bars",
+	"Solid Color",
+	"Grey Color Bars",
+	"PN9"
+};
+
+static const int imx258_test_pattern_val[] = {
+	IMX258_TEST_PATTERN_DISABLE,
+	IMX258_TEST_PATTERN_COLOR_BARS,
+	IMX258_TEST_PATTERN_SOLID_COLOR,
+	IMX258_TEST_PATTERN_GREY_COLOR,
+	IMX258_TEST_PATTERN_PN9,
+};
+
+/* Configurations for supported link frequencies */
+#define IMX258_LINK_FREQ_634MHZ	633600000ULL
+#define IMX258_LINK_FREQ_320MHZ	320000000ULL
+
+enum {
+	IMX258_LINK_FREQ_1267MBPS,
+	IMX258_LINK_FREQ_640MBPS,
+};
+
+/*
+ * pixel_rate = link_freq * data-rate * nr_of_lanes / bits_per_sample
+ * data rate => double data rate; number of lanes => 4; bits per pixel => 10
+ */
+static u64 link_freq_to_pixel_rate(u64 f)
+{
+	f *= 2 * 4;
+	do_div(f, 10);
+
+	return f;
+}
+
+/* Menu items for LINK_FREQ V4L2 control */
+static const s64 link_freq_menu_items[] = {
+	IMX258_LINK_FREQ_634MHZ,
+	IMX258_LINK_FREQ_320MHZ,
+};
+
+/* Link frequency configs */
+static const struct imx258_link_freq_config link_freq_configs[] = {
+	[IMX258_LINK_FREQ_1267MBPS] = {
+		.pixels_per_line = IMX258_PPL_DEFAULT,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mipi_data_rate_1267mbps),
+			.regs = mipi_data_rate_1267mbps,
+		}
+	},
+	[IMX258_LINK_FREQ_640MBPS] = {
+		.pixels_per_line = IMX258_PPL_DEFAULT,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mipi_data_rate_640mbps),
+			.regs = mipi_data_rate_640mbps,
+		}
+	},
+};
+
+/* Mode configs */
+static const struct imx258_mode supported_modes[] = {
+	{
+		.width = 4208,
+		.height = 3118,
+		.vts_def = IMX258_VTS_30FPS,
+		.vts_min = IMX258_VTS_30FPS,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_4208x3118_regs),
+			.regs = mode_4208x3118_regs,
+		},
+		.link_freq_index = IMX258_LINK_FREQ_1267MBPS,
+	},
+	{
+		.width = 2104,
+		.height = 1560,
+		.vts_def = IMX258_VTS_30FPS_2K,
+		.vts_min = IMX258_VTS_30FPS_2K,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_2104_1560_regs),
+			.regs = mode_2104_1560_regs,
+		},
+		.link_freq_index = IMX258_LINK_FREQ_640MBPS,
+	},
+	{
+		.width = 1048,
+		.height = 780,
+		.vts_def = IMX258_VTS_30FPS_VGA,
+		.vts_min = IMX258_VTS_30FPS_VGA,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_1048_780_regs),
+			.regs = mode_1048_780_regs,
+		},
+		.link_freq_index = IMX258_LINK_FREQ_640MBPS,
+	},
+};
+
+struct imx258 {
+	struct v4l2_subdev sd;
+	struct media_pad pad;
+
+	struct v4l2_ctrl_handler ctrl_handler;
+	/* V4L2 Controls */
+	struct v4l2_ctrl *link_freq;
+	struct v4l2_ctrl *pixel_rate;
+	struct v4l2_ctrl *vblank;
+	struct v4l2_ctrl *hblank;
+	struct v4l2_ctrl *exposure;
+
+	/* Current mode */
+	const struct imx258_mode *cur_mode;
+
+	/*
+	 * Mutex for serialized access:
+	 * Protect sensor module set pad format and start/stop streaming safely.
+	 */
+	struct mutex mutex;
+
+	/* Streaming on/off */
+	bool streaming;
+};
+
+static inline struct imx258 *to_imx258(struct v4l2_subdev *_sd)
+{
+	return container_of(_sd, struct imx258, sd);
+}
+
+/* Read registers up to 2 at a time */
+static int imx258_read_reg(struct imx258 *imx258, u16 reg, u32 len, u32 *val)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&imx258->sd);
+	struct i2c_msg msgs[2];
+	u8 addr_buf[2] = { reg >> 8, reg & 0xff };
+	u8 data_buf[4] = { 0, };
+	int ret;
+
+	if (len > 4)
+		return -EINVAL;
+
+	/* Write register address */
+	msgs[0].addr = client->addr;
+	msgs[0].flags = 0;
+	msgs[0].len = ARRAY_SIZE(addr_buf);
+	msgs[0].buf = addr_buf;
+
+	/* Read data from register */
+	msgs[1].addr = client->addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = len;
+	msgs[1].buf = &data_buf[4 - len];
+
+	ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
+	if (ret != ARRAY_SIZE(msgs))
+		return -EIO;
+
+	*val = get_unaligned_be32(data_buf);
+
+	return 0;
+}
+
+/* Write registers up to 2 at a time */
+static int imx258_write_reg(struct imx258 *imx258, u16 reg, u32 len, u32 val)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&imx258->sd);
+	u8 buf[6];
+
+	if (len > 4)
+		return -EINVAL;
+
+	put_unaligned_be16(reg, buf);
+	put_unaligned_be32(val << (8 * (4 - len)), buf + 2);
+	if (i2c_master_send(client, buf, len + 2) != len + 2)
+		return -EIO;
+
+	return 0;
+}
+
+/* Write a list of registers */
+static int imx258_write_regs(struct imx258 *imx258,
+			     const struct imx258_reg *regs, u32 len)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&imx258->sd);
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i < len; i++) {
+		ret = imx258_write_reg(imx258, regs[i].address, 1,
+					regs[i].val);
+		if (ret) {
+			dev_err_ratelimited(
+				&client->dev,
+				"Failed to write reg 0x%4.4x. error = %d\n",
+				regs[i].address, ret);
+
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/* Open sub-device */
+static int imx258_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct v4l2_mbus_framefmt *try_fmt =
+		v4l2_subdev_get_try_format(sd, fh->pad, 0);
+
+	/* Initialize try_fmt */
+	try_fmt->width = supported_modes[0].width;
+	try_fmt->height = supported_modes[0].height;
+	try_fmt->code = MEDIA_BUS_FMT_SGRBG10_1X10;
+	try_fmt->field = V4L2_FIELD_NONE;
+
+	return 0;
+}
+
+static int imx258_update_digital_gain(struct imx258 *imx258, u32 len, u32 val)
+{
+	int ret;
+
+	ret = imx258_write_reg(imx258, IMX258_REG_GR_DIGITAL_GAIN,
+				IMX258_REG_VALUE_16BIT,
+				val);
+	if (ret)
+		return ret;
+	ret = imx258_write_reg(imx258, IMX258_REG_GB_DIGITAL_GAIN,
+				IMX258_REG_VALUE_16BIT,
+				val);
+	if (ret)
+		return ret;
+	ret = imx258_write_reg(imx258, IMX258_REG_R_DIGITAL_GAIN,
+				IMX258_REG_VALUE_16BIT,
+				val);
+	if (ret)
+		return ret;
+	ret = imx258_write_reg(imx258, IMX258_REG_B_DIGITAL_GAIN,
+				IMX258_REG_VALUE_16BIT,
+				val);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static int imx258_set_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct imx258 *imx258 =
+		container_of(ctrl->handler, struct imx258, ctrl_handler);
+	struct i2c_client *client = v4l2_get_subdevdata(&imx258->sd);
+	int ret = 0;
+
+	/*
+	 * Applying V4L2 control value only happens
+	 * when power is up for streaming
+	 */
+	if (pm_runtime_get_if_in_use(&client->dev) == 0)
+		return 0;
+
+	switch (ctrl->id) {
+	case V4L2_CID_ANALOGUE_GAIN:
+		ret = imx258_write_reg(imx258, IMX258_REG_ANALOG_GAIN,
+				IMX258_REG_VALUE_16BIT,
+				ctrl->val);
+		break;
+	case V4L2_CID_EXPOSURE:
+		ret = imx258_write_reg(imx258, IMX258_REG_EXPOSURE,
+				IMX258_REG_VALUE_16BIT,
+				ctrl->val);
+		break;
+	case V4L2_CID_DIGITAL_GAIN:
+		ret = imx258_update_digital_gain(imx258, IMX258_REG_VALUE_16BIT,
+				ctrl->val);
+		break;
+	case V4L2_CID_TEST_PATTERN:
+		ret = imx258_write_reg(imx258, IMX258_REG_TEST_PATTERN,
+				IMX258_REG_VALUE_16BIT,
+				imx258_test_pattern_val[ctrl->val]);
+
+		ret = imx258_write_reg(imx258, REG_MIRROR_FLIP_CONTROL,
+				IMX258_REG_VALUE_08BIT,
+				ctrl->val == imx258_test_pattern_val
+				[IMX258_TEST_PATTERN_DISABLE] ?
+				REG_CONFIG_MIRROR_FLIP :
+				REG_CONFIG_FLIP_TEST_PATTERN);
+		break;
+	default:
+		dev_info(&client->dev,
+			 "ctrl(id:0x%x,val:0x%x) is not handled\n",
+			 ctrl->id, ctrl->val);
+		ret = -EINVAL;
+		break;
+	}
+
+	pm_runtime_put(&client->dev);
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops imx258_ctrl_ops = {
+	.s_ctrl = imx258_set_ctrl,
+};
+
+static int imx258_enum_mbus_code(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_pad_config *cfg,
+				  struct v4l2_subdev_mbus_code_enum *code)
+{
+	/* Only one bayer order(GRBG) is supported */
+	if (code->index > 0)
+		return -EINVAL;
+
+	code->code = MEDIA_BUS_FMT_SGRBG10_1X10;
+
+	return 0;
+}
+
+static int imx258_enum_frame_size(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_pad_config *cfg,
+				  struct v4l2_subdev_frame_size_enum *fse)
+{
+	if (fse->index >= ARRAY_SIZE(supported_modes))
+		return -EINVAL;
+
+	if (fse->code != MEDIA_BUS_FMT_SGRBG10_1X10)
+		return -EINVAL;
+
+	fse->min_width = supported_modes[fse->index].width;
+	fse->max_width = fse->min_width;
+	fse->min_height = supported_modes[fse->index].height;
+	fse->max_height = fse->min_height;
+
+	return 0;
+}
+
+static void imx258_update_pad_format(const struct imx258_mode *mode,
+				     struct v4l2_subdev_format *fmt)
+{
+	fmt->format.width = mode->width;
+	fmt->format.height = mode->height;
+	fmt->format.code = MEDIA_BUS_FMT_SGRBG10_1X10;
+	fmt->format.field = V4L2_FIELD_NONE;
+}
+
+static int __imx258_get_pad_format(struct imx258 *imx258,
+				   struct v4l2_subdev_pad_config *cfg,
+				   struct v4l2_subdev_format *fmt)
+{
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY)
+		fmt->format = *v4l2_subdev_get_try_format(&imx258->sd, cfg,
+							  fmt->pad);
+	else
+		imx258_update_pad_format(imx258->cur_mode, fmt);
+
+	return 0;
+}
+
+static int imx258_get_pad_format(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_format *fmt)
+{
+	struct imx258 *imx258 = to_imx258(sd);
+	int ret;
+
+	mutex_lock(&imx258->mutex);
+	ret = __imx258_get_pad_format(imx258, cfg, fmt);
+	mutex_unlock(&imx258->mutex);
+
+	return ret;
+}
+
+static int imx258_set_pad_format(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_format *fmt)
+{
+	struct imx258 *imx258 = to_imx258(sd);
+	const struct imx258_mode *mode;
+	struct v4l2_mbus_framefmt *framefmt;
+	s32 vblank_def;
+	s32 vblank_min;
+	s64 h_blank;
+	s64 pixel_rate;
+	s64 link_freq;
+
+	mutex_lock(&imx258->mutex);
+
+	/* Only one raw bayer(GBRG) order is supported */
+	fmt->format.code = MEDIA_BUS_FMT_SGRBG10_1X10;
+
+	mode = v4l2_find_nearest_size(supported_modes,
+		ARRAY_SIZE(supported_modes), width, height,
+		fmt->format.width, fmt->format.height);
+	imx258_update_pad_format(mode, fmt);
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
+		framefmt = v4l2_subdev_get_try_format(sd, cfg, fmt->pad);
+		*framefmt = fmt->format;
+	} else {
+		imx258->cur_mode = mode;
+		__v4l2_ctrl_s_ctrl(imx258->link_freq, mode->link_freq_index);
+
+		link_freq = link_freq_menu_items[mode->link_freq_index];
+		pixel_rate = link_freq_to_pixel_rate(link_freq);
+		__v4l2_ctrl_s_ctrl_int64(imx258->pixel_rate, pixel_rate);
+		/* Update limits and set FPS to default */
+		vblank_def = imx258->cur_mode->vts_def -
+			     imx258->cur_mode->height;
+		vblank_min = imx258->cur_mode->vts_min -
+			     imx258->cur_mode->height;
+		__v4l2_ctrl_modify_range(
+			imx258->vblank, vblank_min,
+			IMX258_VTS_MAX - imx258->cur_mode->height, 1,
+			vblank_def);
+		__v4l2_ctrl_s_ctrl(imx258->vblank, vblank_def);
+		h_blank =
+			link_freq_configs[mode->link_freq_index].pixels_per_line
+			 - imx258->cur_mode->width;
+		__v4l2_ctrl_modify_range(imx258->hblank, h_blank,
+					 h_blank, 1, h_blank);
+	}
+
+	mutex_unlock(&imx258->mutex);
+
+	return 0;
+}
+
+/* Start streaming */
+static int imx258_start_streaming(struct imx258 *imx258)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&imx258->sd);
+	const struct imx258_reg_list *reg_list;
+	int ret, link_freq_index;
+
+	/* Setup PLL */
+	link_freq_index = imx258->cur_mode->link_freq_index;
+	reg_list = &link_freq_configs[link_freq_index].reg_list;
+	ret = imx258_write_regs(imx258, reg_list->regs, reg_list->num_of_regs);
+	if (ret) {
+		dev_err(&client->dev, "%s failed to set plls\n", __func__);
+		return ret;
+	}
+
+	/* Apply default values of current mode */
+	reg_list = &imx258->cur_mode->reg_list;
+	ret = imx258_write_regs(imx258, reg_list->regs, reg_list->num_of_regs);
+	if (ret) {
+		dev_err(&client->dev, "%s failed to set mode\n", __func__);
+		return ret;
+	}
+
+	/* Set Orientation be 180 degree */
+	ret = imx258_write_reg(imx258, REG_MIRROR_FLIP_CONTROL,
+			       IMX258_REG_VALUE_08BIT, REG_CONFIG_MIRROR_FLIP);
+	if (ret) {
+		dev_err(&client->dev, "%s failed to set orientation\n",
+			__func__);
+		return ret;
+	}
+
+	/* Apply customized values from user */
+	ret =  __v4l2_ctrl_handler_setup(imx258->sd.ctrl_handler);
+	if (ret)
+		return ret;
+
+	/* set stream on register */
+	return imx258_write_reg(imx258, IMX258_REG_MODE_SELECT,
+				IMX258_REG_VALUE_08BIT,
+				IMX258_MODE_STREAMING);
+}
+
+/* Stop streaming */
+static int imx258_stop_streaming(struct imx258 *imx258)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&imx258->sd);
+	int ret;
+
+	/* set stream off register */
+	ret = imx258_write_reg(imx258, IMX258_REG_MODE_SELECT,
+		IMX258_REG_VALUE_08BIT, IMX258_MODE_STANDBY);
+	if (ret)
+		dev_err(&client->dev, "%s failed to set stream\n", __func__);
+
+	/*
+	 * Return success even if it was an error, as there is nothing the
+	 * caller can do about it.
+	 */
+	return 0;
+}
+
+static int imx258_set_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct imx258 *imx258 = to_imx258(sd);
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	int ret = 0;
+
+	mutex_lock(&imx258->mutex);
+	if (imx258->streaming == enable) {
+		mutex_unlock(&imx258->mutex);
+		return 0;
+	}
+
+	if (enable) {
+		ret = pm_runtime_get_sync(&client->dev);
+		if (ret < 0) {
+			pm_runtime_put_noidle(&client->dev);
+			goto err_unlock;
+		}
+
+		/*
+		 * Apply default & customized values
+		 * and then start streaming.
+		 */
+		ret = imx258_start_streaming(imx258);
+		if (ret)
+			goto err_rpm_put;
+	} else {
+		imx258_stop_streaming(imx258);
+		pm_runtime_put(&client->dev);
+	}
+
+	imx258->streaming = enable;
+	mutex_unlock(&imx258->mutex);
+
+	return ret;
+
+err_rpm_put:
+	pm_runtime_put(&client->dev);
+err_unlock:
+	mutex_unlock(&imx258->mutex);
+
+	return ret;
+}
+
+static int __maybe_unused imx258_suspend(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx258 *imx258 = to_imx258(sd);
+
+	if (imx258->streaming)
+		imx258_stop_streaming(imx258);
+
+	return 0;
+}
+
+static int __maybe_unused imx258_resume(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx258 *imx258 = to_imx258(sd);
+	int ret;
+
+	if (imx258->streaming) {
+		ret = imx258_start_streaming(imx258);
+		if (ret)
+			goto error;
+	}
+
+	return 0;
+
+error:
+	imx258_stop_streaming(imx258);
+	imx258->streaming = 0;
+	return ret;
+}
+
+/* Verify chip ID */
+static int imx258_identify_module(struct imx258 *imx258)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&imx258->sd);
+	int ret;
+	u32 val;
+
+	ret = imx258_read_reg(imx258, IMX258_REG_CHIP_ID,
+			      IMX258_REG_VALUE_16BIT, &val);
+	if (ret) {
+		dev_err(&client->dev, "failed to read chip id %x\n",
+			IMX258_CHIP_ID);
+		return ret;
+	}
+
+	if (val != IMX258_CHIP_ID) {
+		dev_err(&client->dev, "chip id mismatch: %x!=%x\n",
+			IMX258_CHIP_ID, val);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static const struct v4l2_subdev_video_ops imx258_video_ops = {
+	.s_stream = imx258_set_stream,
+};
+
+static const struct v4l2_subdev_pad_ops imx258_pad_ops = {
+	.enum_mbus_code = imx258_enum_mbus_code,
+	.get_fmt = imx258_get_pad_format,
+	.set_fmt = imx258_set_pad_format,
+	.enum_frame_size = imx258_enum_frame_size,
+};
+
+static const struct v4l2_subdev_ops imx258_subdev_ops = {
+	.video = &imx258_video_ops,
+	.pad = &imx258_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops imx258_internal_ops = {
+	.open = imx258_open,
+};
+
+/* Initialize control handlers */
+static int imx258_init_controls(struct imx258 *imx258)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&imx258->sd);
+	struct v4l2_ctrl_handler *ctrl_hdlr;
+	s64 vblank_def;
+	s64 vblank_min;
+	s64 pixel_rate_min;
+	s64 pixel_rate_max;
+	int ret;
+
+	ctrl_hdlr = &imx258->ctrl_handler;
+	ret = v4l2_ctrl_handler_init(ctrl_hdlr, 8);
+	if (ret)
+		return ret;
+
+	mutex_init(&imx258->mutex);
+	ctrl_hdlr->lock = &imx258->mutex;
+	imx258->link_freq = v4l2_ctrl_new_int_menu(ctrl_hdlr,
+				&imx258_ctrl_ops,
+				V4L2_CID_LINK_FREQ,
+				ARRAY_SIZE(link_freq_menu_items) - 1,
+				0,
+				link_freq_menu_items);
+
+	if (imx258->link_freq)
+		imx258->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	pixel_rate_max = link_freq_to_pixel_rate(link_freq_menu_items[0]);
+	pixel_rate_min = link_freq_to_pixel_rate(link_freq_menu_items[1]);
+	/* By default, PIXEL_RATE is read only */
+	imx258->pixel_rate = v4l2_ctrl_new_std(ctrl_hdlr, &imx258_ctrl_ops,
+				V4L2_CID_PIXEL_RATE,
+				pixel_rate_min, pixel_rate_max,
+				1, pixel_rate_max);
+
+
+	vblank_def = imx258->cur_mode->vts_def - imx258->cur_mode->height;
+	vblank_min = imx258->cur_mode->vts_min - imx258->cur_mode->height;
+	imx258->vblank = v4l2_ctrl_new_std(
+				ctrl_hdlr, &imx258_ctrl_ops, V4L2_CID_VBLANK,
+				vblank_min,
+				IMX258_VTS_MAX - imx258->cur_mode->height, 1,
+				vblank_def);
+
+	if (imx258->vblank)
+		imx258->vblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	imx258->hblank = v4l2_ctrl_new_std(
+				ctrl_hdlr, &imx258_ctrl_ops, V4L2_CID_HBLANK,
+				IMX258_PPL_DEFAULT - imx258->cur_mode->width,
+				IMX258_PPL_DEFAULT - imx258->cur_mode->width,
+				1,
+				IMX258_PPL_DEFAULT - imx258->cur_mode->width);
+
+	if (imx258->hblank)
+		imx258->hblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	imx258->exposure = v4l2_ctrl_new_std(
+				ctrl_hdlr, &imx258_ctrl_ops,
+				V4L2_CID_EXPOSURE, IMX258_EXPOSURE_MIN,
+				IMX258_EXPOSURE_MAX, IMX258_EXPOSURE_STEP,
+				IMX258_EXPOSURE_DEFAULT);
+
+	v4l2_ctrl_new_std(ctrl_hdlr, &imx258_ctrl_ops, V4L2_CID_ANALOGUE_GAIN,
+				IMX258_ANA_GAIN_MIN, IMX258_ANA_GAIN_MAX,
+				IMX258_ANA_GAIN_STEP, IMX258_ANA_GAIN_DEFAULT);
+
+	v4l2_ctrl_new_std(ctrl_hdlr, &imx258_ctrl_ops, V4L2_CID_DIGITAL_GAIN,
+				IMX258_DGTL_GAIN_MIN, IMX258_DGTL_GAIN_MAX,
+				IMX258_DGTL_GAIN_STEP,
+				IMX258_DGTL_GAIN_DEFAULT);
+
+	v4l2_ctrl_new_std_menu_items(ctrl_hdlr, &imx258_ctrl_ops,
+				V4L2_CID_TEST_PATTERN,
+				ARRAY_SIZE(imx258_test_pattern_menu) - 1,
+				0, 0, imx258_test_pattern_menu);
+
+	if (ctrl_hdlr->error) {
+		ret = ctrl_hdlr->error;
+		dev_err(&client->dev, "%s control init failed (%d)\n",
+				__func__, ret);
+		goto error;
+	}
+
+	imx258->sd.ctrl_handler = ctrl_hdlr;
+
+	return 0;
+
+error:
+	v4l2_ctrl_handler_free(ctrl_hdlr);
+	mutex_destroy(&imx258->mutex);
+
+	return ret;
+}
+
+static void imx258_free_controls(struct imx258 *imx258)
+{
+	v4l2_ctrl_handler_free(imx258->sd.ctrl_handler);
+	mutex_destroy(&imx258->mutex);
+}
+
+static int imx258_probe(struct i2c_client *client)
+{
+	struct imx258 *imx258;
+	int ret;
+	u32 val = 0;
+
+	device_property_read_u32(&client->dev, "clock-frequency", &val);
+	if (val != 19200000)
+		return -EINVAL;
+
+	imx258 = devm_kzalloc(&client->dev, sizeof(*imx258), GFP_KERNEL);
+	if (!imx258)
+		return -ENOMEM;
+
+	/* Initialize subdev */
+	v4l2_i2c_subdev_init(&imx258->sd, client, &imx258_subdev_ops);
+
+	/* Check module identity */
+	ret = imx258_identify_module(imx258);
+	if (ret)
+		return ret;
+
+	/* Set default mode to max resolution */
+	imx258->cur_mode = &supported_modes[0];
+
+	ret = imx258_init_controls(imx258);
+	if (ret)
+		return ret;
+
+	/* Initialize subdev */
+	imx258->sd.internal_ops = &imx258_internal_ops;
+	imx258->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	imx258->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
+
+	/* Initialize source pad */
+	imx258->pad.flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&imx258->sd.entity, 1, &imx258->pad);
+	if (ret)
+		goto error_handler_free;
+
+	ret = v4l2_async_register_subdev_sensor_common(&imx258->sd);
+	if (ret < 0)
+		goto error_media_entity;
+
+	pm_runtime_set_active(&client->dev);
+	pm_runtime_enable(&client->dev);
+	pm_runtime_idle(&client->dev);
+
+	return 0;
+
+error_media_entity:
+	media_entity_cleanup(&imx258->sd.entity);
+
+error_handler_free:
+	imx258_free_controls(imx258);
+
+	return ret;
+}
+
+static int imx258_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx258 *imx258 = to_imx258(sd);
+
+	v4l2_async_unregister_subdev(sd);
+	media_entity_cleanup(&sd->entity);
+	imx258_free_controls(imx258);
+
+	pm_runtime_disable(&client->dev);
+	pm_runtime_set_suspended(&client->dev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops imx258_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(imx258_suspend, imx258_resume)
+};
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id imx258_acpi_ids[] = {
+	{ "SONY258A" },
+	{ /* sentinel */ }
+};
+
+MODULE_DEVICE_TABLE(acpi, imx258_acpi_ids);
+#endif
+
+static struct i2c_driver imx258_i2c_driver = {
+	.driver = {
+		.name = "imx258",
+		.pm = &imx258_pm_ops,
+		.acpi_match_table = ACPI_PTR(imx258_acpi_ids),
+	},
+	.probe_new = imx258_probe,
+	.remove = imx258_remove,
+};
+
+module_i2c_driver(imx258_i2c_driver);
+
+MODULE_AUTHOR("Yeh, Andy <andy.yeh@intel.com>");
+MODULE_AUTHOR("Chiang, Alan <alanx.chiang@intel.com>");
+MODULE_AUTHOR("Chen, Jason <jasonx.z.chen@intel.com>");
+MODULE_DESCRIPTION("Sony IMX258 sensor driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/i2c/imx274.c b/drivers/media/i2c/imx274.c
index daec33f4196a..63fb94e7da37 100644
--- a/drivers/media/i2c/imx274.c
+++ b/drivers/media/i2c/imx274.c
@@ -87,7 +87,7 @@
 #define IMX274_SHR_LIMIT_CONST			(4)
 
 /*
- * Constants for sensor reset delay
+ * Min and max sensor reset delay (microseconds)
  */
 #define IMX274_RESET_DELAY1			(2000)
 #define IMX274_RESET_DELAY2			(2200)
@@ -107,15 +107,15 @@
 /*
  * IMX274 register definitions
  */
-#define IMX274_FRAME_LENGTH_ADDR_1		0x30FA /* VMAX, MSB */
-#define IMX274_FRAME_LENGTH_ADDR_2		0x30F9 /* VMAX */
-#define IMX274_FRAME_LENGTH_ADDR_3		0x30F8 /* VMAX, LSB */
+#define IMX274_SHR_REG_MSB			0x300D /* SHR */
+#define IMX274_SHR_REG_LSB			0x300C /* SHR */
 #define IMX274_SVR_REG_MSB			0x300F /* SVR */
 #define IMX274_SVR_REG_LSB			0x300E /* SVR */
+#define IMX274_VMAX_REG_1			0x30FA /* VMAX, MSB */
+#define IMX274_VMAX_REG_2			0x30F9 /* VMAX */
+#define IMX274_VMAX_REG_3			0x30F8 /* VMAX, LSB */
 #define IMX274_HMAX_REG_MSB			0x30F7 /* HMAX */
 #define IMX274_HMAX_REG_LSB			0x30F6 /* HMAX */
-#define IMX274_COARSE_TIME_ADDR_MSB		0x300D /* SHR */
-#define IMX274_COARSE_TIME_ADDR_LSB		0x300C /* SHR */
 #define IMX274_ANALOG_GAIN_ADDR_LSB		0x300A /* ANALOG GAIN LSB */
 #define IMX274_ANALOG_GAIN_ADDR_MSB		0x300B /* ANALOG GAIN MSB */
 #define IMX274_DIGITAL_GAIN_REG			0x3012 /* Digital Gain */
@@ -144,22 +144,13 @@ enum imx274_mode {
 	IMX274_MODE_3840X2160,
 	IMX274_MODE_1920X1080,
 	IMX274_MODE_1280X720,
-
-	IMX274_MODE_START_STREAM_1,
-	IMX274_MODE_START_STREAM_2,
-	IMX274_MODE_START_STREAM_3,
-	IMX274_MODE_START_STREAM_4,
-	IMX274_MODE_STOP_STREAM
 };
 
 /*
  * imx274 format related structure
  */
 struct imx274_frmfmt {
-	u32 mbus_code;
-	enum v4l2_colorspace colorspace;
 	struct v4l2_frmsize_discrete size;
-	enum imx274_mode mode;
 };
 
 /*
@@ -489,24 +480,15 @@ static const struct reg_8 *mode_table[] = {
 	[IMX274_MODE_3840X2160]		= imx274_mode1_3840x2160_raw10,
 	[IMX274_MODE_1920X1080]		= imx274_mode3_1920x1080_raw10,
 	[IMX274_MODE_1280X720]		= imx274_mode5_1280x720_raw10,
-
-	[IMX274_MODE_START_STREAM_1]	= imx274_start_1,
-	[IMX274_MODE_START_STREAM_2]	= imx274_start_2,
-	[IMX274_MODE_START_STREAM_3]	= imx274_start_3,
-	[IMX274_MODE_START_STREAM_4]	= imx274_start_4,
-	[IMX274_MODE_STOP_STREAM]	= imx274_stop,
 };
 
 /*
  * imx274 format related structure
  */
 static const struct imx274_frmfmt imx274_formats[] = {
-	{MEDIA_BUS_FMT_SRGGB10_1X10, V4L2_COLORSPACE_SRGB, {3840, 2160},
-		IMX274_MODE_3840X2160},
-	{MEDIA_BUS_FMT_SRGGB10_1X10, V4L2_COLORSPACE_SRGB, {1920, 1080},
-		IMX274_MODE_1920X1080},
-	{MEDIA_BUS_FMT_SRGGB10_1X10, V4L2_COLORSPACE_SRGB, {1280, 720},
-		IMX274_MODE_1280X720},
+	{ {3840, 2160} },
+	{ {1920, 1080} },
+	{ {1280,  720} },
 };
 
 /*
@@ -737,11 +719,11 @@ static int imx274_mode_regs(struct stimx274 *priv, int mode)
 {
 	int err = 0;
 
-	err = imx274_write_table(priv, mode_table[IMX274_MODE_START_STREAM_1]);
+	err = imx274_write_table(priv, imx274_start_1);
 	if (err)
 		return err;
 
-	err = imx274_write_table(priv, mode_table[IMX274_MODE_START_STREAM_2]);
+	err = imx274_write_table(priv, imx274_start_2);
 	if (err)
 		return err;
 
@@ -766,7 +748,7 @@ static int imx274_start_stream(struct stimx274 *priv)
 	 * give it 1 extra ms for margin
 	 */
 	msleep_range(11);
-	err = imx274_write_table(priv, mode_table[IMX274_MODE_START_STREAM_3]);
+	err = imx274_write_table(priv, imx274_start_3);
 	if (err)
 		return err;
 
@@ -776,7 +758,7 @@ static int imx274_start_stream(struct stimx274 *priv)
 	 * give it 1 extra ms for margin
 	 */
 	msleep_range(8);
-	err = imx274_write_table(priv, mode_table[IMX274_MODE_START_STREAM_4]);
+	err = imx274_write_table(priv, imx274_start_4);
 	if (err)
 		return err;
 
@@ -890,9 +872,8 @@ static int imx274_set_fmt(struct v4l2_subdev *sd,
 	int index;
 
 	dev_dbg(&client->dev,
-		"%s: width = %d height = %d code = %d mbus_code = %d\n",
-		__func__, fmt->width, fmt->height, fmt->code,
-		imx274_formats[imx274->mode_index].mbus_code);
+		"%s: width = %d height = %d code = %d\n",
+		__func__, fmt->width, fmt->height, fmt->code);
 
 	mutex_lock(&imx274->lock);
 
@@ -971,7 +952,7 @@ static int imx274_s_frame_interval(struct v4l2_subdev *sd,
 	if (!ret) {
 		/*
 		 * exposure time range is decided by frame interval
-		 * need to update it after frame interal changes
+		 * need to update it after frame interval changes
 		 */
 		min = IMX274_MIN_EXPOSURE_TIME;
 		max = fi->interval.numerator * 1000000
@@ -984,7 +965,7 @@ static int imx274_s_frame_interval(struct v4l2_subdev *sd,
 		}
 
 		/* update exposure time accordingly */
-		imx274_set_exposure(imx274, imx274->ctrls.exposure->val);
+		imx274_set_exposure(imx274, ctrl->val);
 
 		dev_dbg(&imx274->client->dev, "set frame interval to %uus\n",
 			fi->interval.numerator * 1000000
@@ -1088,8 +1069,7 @@ static int imx274_s_stream(struct v4l2_subdev *sd, int on)
 			goto fail;
 	} else {
 		/* stop stream */
-		ret = imx274_write_table(imx274,
-					 mode_table[IMX274_MODE_STOP_STREAM]);
+		ret = imx274_write_table(imx274, imx274_stop);
 		if (ret)
 			goto fail;
 	}
@@ -1133,15 +1113,15 @@ static int imx274_get_frame_length(struct stimx274 *priv, u32 *val)
 	svr = (reg_val[1] << IMX274_SHIFT_8_BITS) + reg_val[0];
 
 	/* vmax */
-	err = imx274_read_reg(priv, IMX274_FRAME_LENGTH_ADDR_3, &reg_val[0]);
+	err = imx274_read_reg(priv, IMX274_VMAX_REG_3, &reg_val[0]);
 	if (err)
 		goto fail;
 
-	err = imx274_read_reg(priv, IMX274_FRAME_LENGTH_ADDR_2, &reg_val[1]);
+	err = imx274_read_reg(priv, IMX274_VMAX_REG_2, &reg_val[1]);
 	if (err)
 		goto fail;
 
-	err = imx274_read_reg(priv, IMX274_FRAME_LENGTH_ADDR_1, &reg_val[2]);
+	err = imx274_read_reg(priv, IMX274_VMAX_REG_1, &reg_val[2]);
 	if (err)
 		goto fail;
 
@@ -1300,10 +1280,10 @@ static int imx274_set_gain(struct stimx274 *priv, struct v4l2_ctrl *ctrl)
 static inline void imx274_calculate_coarse_time_regs(struct reg_8 regs[2],
 						     u32 coarse_time)
 {
-	regs->addr = IMX274_COARSE_TIME_ADDR_MSB;
+	regs->addr = IMX274_SHR_REG_MSB;
 	regs->val = (coarse_time >> IMX274_SHIFT_8_BITS)
 			& IMX274_MASK_LSB_8_BITS;
-	(regs + 1)->addr = IMX274_COARSE_TIME_ADDR_LSB;
+	(regs + 1)->addr = IMX274_SHR_REG_LSB;
 	(regs + 1)->val = (coarse_time) & IMX274_MASK_LSB_8_BITS;
 }
 
@@ -1471,13 +1451,13 @@ static int imx274_set_test_pattern(struct stimx274 *priv, int val)
 static inline void imx274_calculate_frame_length_regs(struct reg_8 regs[3],
 						      u32 frame_length)
 {
-	regs->addr = IMX274_FRAME_LENGTH_ADDR_1;
+	regs->addr = IMX274_VMAX_REG_1;
 	regs->val = (frame_length >> IMX274_SHIFT_16_BITS)
 			& IMX274_MASK_LSB_4_BITS;
-	(regs + 1)->addr = IMX274_FRAME_LENGTH_ADDR_2;
+	(regs + 1)->addr = IMX274_VMAX_REG_2;
 	(regs + 1)->val = (frame_length >> IMX274_SHIFT_8_BITS)
 			& IMX274_MASK_LSB_8_BITS;
-	(regs + 2)->addr = IMX274_FRAME_LENGTH_ADDR_3;
+	(regs + 2)->addr = IMX274_VMAX_REG_3;
 	(regs + 2)->val = (frame_length) & IMX274_MASK_LSB_8_BITS;
 }
 
@@ -1786,7 +1766,7 @@ static int imx274_remove(struct i2c_client *client)
 	struct stimx274 *imx274 = to_imx274(sd);
 
 	/* stop stream */
-	imx274_write_table(imx274, mode_table[IMX274_MODE_STOP_STREAM]);
+	imx274_write_table(imx274, imx274_stop);
 
 	v4l2_async_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&imx274->ctrls.handler);
diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c
index a7e23bcf845c..a14a74e6b986 100644
--- a/drivers/media/i2c/ir-kbd-i2c.c
+++ b/drivers/media/i2c/ir-kbd-i2c.c
@@ -739,6 +739,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	struct rc_dev *rc = NULL;
 	struct i2c_adapter *adap = client->adapter;
 	unsigned short addr = client->addr;
+	bool probe_tx = (id->driver_data & FLAG_TX) != 0;
 	int err;
 
 	if ((id->driver_data & FLAG_HDPVR) && !enable_hdpvr) {
@@ -800,6 +801,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 		rc_proto    = RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC6_MCE |
 							RC_PROTO_BIT_RC6_6A_32;
 		ir_codes    = RC_MAP_HAUPPAUGE;
+		probe_tx = true;
 		break;
 	}
 
@@ -892,7 +894,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 
 	INIT_DELAYED_WORK(&ir->work, ir_work);
 
-	if (id->driver_data & FLAG_TX) {
+	if (probe_tx) {
 		ir->tx_c = i2c_new_dummy(client->adapter, 0x70);
 		if (!ir->tx_c) {
 			dev_err(&client->dev, "failed to setup tx i2c address");
diff --git a/drivers/media/i2c/ov13858.c b/drivers/media/i2c/ov13858.c
index 3dbcae257164..a66f6201f53c 100644
--- a/drivers/media/i2c/ov13858.c
+++ b/drivers/media/i2c/ov13858.c
@@ -1796,7 +1796,6 @@ MODULE_DEVICE_TABLE(acpi, ov13858_acpi_ids);
 static struct i2c_driver ov13858_i2c_driver = {
 	.driver = {
 		.name = "ov13858",
-		.owner = THIS_MODULE,
 		.pm = &ov13858_pm_ops,
 		.acpi_match_table = ACPI_PTR(ov13858_acpi_ids),
 	},
diff --git a/drivers/media/i2c/ov2640.c b/drivers/media/i2c/ov2640.c
index 4c3b92763243..beb722065152 100644
--- a/drivers/media/i2c/ov2640.c
+++ b/drivers/media/i2c/ov2640.c
@@ -307,6 +307,10 @@ struct ov2640_priv {
 
 	struct gpio_desc *resetb_gpio;
 	struct gpio_desc *pwdn_gpio;
+
+	struct mutex lock; /* lock to protect streaming and power_count */
+	bool streaming;
+	int power_count;
 };
 
 /*
@@ -709,9 +713,20 @@ static int ov2640_s_ctrl(struct v4l2_ctrl *ctrl)
 	struct v4l2_subdev *sd =
 		&container_of(ctrl->handler, struct ov2640_priv, hdl)->subdev;
 	struct i2c_client  *client = v4l2_get_subdevdata(sd);
+	struct ov2640_priv *priv = to_ov2640(client);
 	u8 val;
 	int ret;
 
+	/* v4l2_ctrl_lock() locks our own mutex */
+
+	/*
+	 * If the device is not powered up by the host driver, do not apply any
+	 * controls to H/W at this time. Instead the controls will be restored
+	 * when the streaming is started.
+	 */
+	if (!priv->power_count)
+		return 0;
+
 	ret = i2c_smbus_write_byte_data(client, BANK_SEL, BANK_SEL_SENS);
 	if (ret < 0)
 		return ret;
@@ -763,12 +778,9 @@ static int ov2640_s_register(struct v4l2_subdev *sd,
 }
 #endif
 
-static int ov2640_s_power(struct v4l2_subdev *sd, int on)
+static void ov2640_set_power(struct ov2640_priv *priv, int on)
 {
 #ifdef CONFIG_GPIOLIB
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct ov2640_priv *priv = to_ov2640(client);
-
 	if (priv->pwdn_gpio)
 		gpiod_direction_output(priv->pwdn_gpio, !on);
 	if (on && priv->resetb_gpio) {
@@ -778,6 +790,25 @@ static int ov2640_s_power(struct v4l2_subdev *sd, int on)
 		gpiod_set_value(priv->resetb_gpio, 0);
 	}
 #endif
+}
+
+static int ov2640_s_power(struct v4l2_subdev *sd, int on)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct ov2640_priv *priv = to_ov2640(client);
+
+	mutex_lock(&priv->lock);
+
+	/*
+	 * If the power count is modified from 0 to != 0 or from != 0 to 0,
+	 * update the power state.
+	 */
+	if (priv->power_count == !on)
+		ov2640_set_power(priv, on);
+	priv->power_count += on ? 1 : -1;
+	WARN_ON(priv->power_count < 0);
+	mutex_unlock(&priv->lock);
+
 	return 0;
 }
 
@@ -798,16 +829,13 @@ static const struct ov2640_win_size *ov2640_select_win(u32 width, u32 height)
 static int ov2640_set_params(struct i2c_client *client,
 			     const struct ov2640_win_size *win, u32 code)
 {
-	struct ov2640_priv       *priv = to_ov2640(client);
 	const struct regval_list *selected_cfmt_regs;
 	u8 val;
 	int ret;
 
-	/* select win */
-	priv->win = win;
+	if (!win)
+		return -EINVAL;
 
-	/* select format */
-	priv->cfmt_code = 0;
 	switch (code) {
 	case MEDIA_BUS_FMT_RGB565_2X8_BE:
 		dev_dbg(&client->dev, "%s: Selected cfmt RGB565 BE", __func__);
@@ -846,13 +874,13 @@ static int ov2640_set_params(struct i2c_client *client,
 		goto err;
 
 	/* select preamble */
-	dev_dbg(&client->dev, "%s: Set size to %s", __func__, priv->win->name);
+	dev_dbg(&client->dev, "%s: Set size to %s", __func__, win->name);
 	ret = ov2640_write_array(client, ov2640_size_change_preamble_regs);
 	if (ret < 0)
 		goto err;
 
 	/* set size win */
-	ret = ov2640_write_array(client, priv->win->regs);
+	ret = ov2640_write_array(client, win->regs);
 	if (ret < 0)
 		goto err;
 
@@ -872,14 +900,11 @@ static int ov2640_set_params(struct i2c_client *client,
 	if (ret < 0)
 		goto err;
 
-	priv->cfmt_code = code;
-
 	return 0;
 
 err:
 	dev_err(&client->dev, "%s: Error %d", __func__, ret);
 	ov2640_reset(client);
-	priv->win = NULL;
 
 	return ret;
 }
@@ -915,11 +940,15 @@ static int ov2640_set_fmt(struct v4l2_subdev *sd,
 {
 	struct v4l2_mbus_framefmt *mf = &format->format;
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct ov2640_priv *priv = to_ov2640(client);
 	const struct ov2640_win_size *win;
+	int ret = 0;
 
 	if (format->pad)
 		return -EINVAL;
 
+	mutex_lock(&priv->lock);
+
 	/* select suitable win */
 	win = ov2640_select_win(mf->width, mf->height);
 	mf->width	= win->width;
@@ -941,10 +970,24 @@ static int ov2640_set_fmt(struct v4l2_subdev *sd,
 		break;
 	}
 
-	if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE)
-		return ov2640_set_params(client, win, mf->code);
-	cfg->try_fmt = *mf;
-	return 0;
+	if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+		struct ov2640_priv *priv = to_ov2640(client);
+
+		if (priv->streaming) {
+			ret = -EBUSY;
+			goto out;
+		}
+		/* select win */
+		priv->win = win;
+		/* select format */
+		priv->cfmt_code = mf->code;
+	} else {
+		cfg->try_fmt = *mf;
+	}
+out:
+	mutex_unlock(&priv->lock);
+
+	return ret;
 }
 
 static int ov2640_enum_mbus_code(struct v4l2_subdev *sd,
@@ -979,6 +1022,28 @@ static int ov2640_get_selection(struct v4l2_subdev *sd,
 	}
 }
 
+static int ov2640_s_stream(struct v4l2_subdev *sd, int on)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct ov2640_priv *priv = to_ov2640(client);
+	int ret = 0;
+
+	mutex_lock(&priv->lock);
+	if (priv->streaming == !on) {
+		if (on) {
+			ret = ov2640_set_params(client, priv->win,
+						priv->cfmt_code);
+			if (!ret)
+				ret = __v4l2_ctrl_handler_setup(&priv->hdl);
+		}
+	}
+	if (!ret)
+		priv->streaming = on;
+	mutex_unlock(&priv->lock);
+
+	return ret;
+}
+
 static int ov2640_video_probe(struct i2c_client *client)
 {
 	struct ov2640_priv *priv = to_ov2640(client);
@@ -1014,8 +1079,6 @@ static int ov2640_video_probe(struct i2c_client *client)
 		 "%s Product ID %0x:%0x Manufacturer ID %x:%x\n",
 		 devname, pid, ver, midh, midl);
 
-	ret = v4l2_ctrl_handler_setup(&priv->hdl);
-
 done:
 	ov2640_s_power(&priv->subdev, 0);
 	return ret;
@@ -1040,9 +1103,14 @@ static const struct v4l2_subdev_pad_ops ov2640_subdev_pad_ops = {
 	.set_fmt	= ov2640_set_fmt,
 };
 
+static const struct v4l2_subdev_video_ops ov2640_subdev_video_ops = {
+	.s_stream = ov2640_s_stream,
+};
+
 static const struct v4l2_subdev_ops ov2640_subdev_ops = {
 	.core	= &ov2640_subdev_core_ops,
 	.pad	= &ov2640_subdev_pad_ops,
+	.video	= &ov2640_subdev_video_ops,
 };
 
 static int ov2640_probe_dt(struct i2c_client *client,
@@ -1116,7 +1184,9 @@ static int ov2640_probe(struct i2c_client *client,
 
 	v4l2_i2c_subdev_init(&priv->subdev, client, &ov2640_subdev_ops);
 	priv->subdev.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	mutex_init(&priv->lock);
 	v4l2_ctrl_handler_init(&priv->hdl, 2);
+	priv->hdl.lock = &priv->lock;
 	v4l2_ctrl_new_std(&priv->hdl, &ov2640_ctrl_ops,
 			V4L2_CID_VFLIP, 0, 1, 1, 0);
 	v4l2_ctrl_new_std(&priv->hdl, &ov2640_ctrl_ops,
@@ -1150,6 +1220,7 @@ static int ov2640_probe(struct i2c_client *client,
 	media_entity_cleanup(&priv->subdev.entity);
 err_hdl:
 	v4l2_ctrl_handler_free(&priv->hdl);
+	mutex_destroy(&priv->lock);
 err_clk:
 	clk_disable_unprepare(priv->clk);
 	return ret;
@@ -1161,6 +1232,7 @@ static int ov2640_remove(struct i2c_client *client)
 
 	v4l2_async_unregister_subdev(&priv->subdev);
 	v4l2_ctrl_handler_free(&priv->hdl);
+	mutex_destroy(&priv->lock);
 	media_entity_cleanup(&priv->subdev.entity);
 	v4l2_device_unregister_subdev(&priv->subdev);
 	clk_disable_unprepare(priv->clk);
diff --git a/drivers/media/i2c/ov5640.c b/drivers/media/i2c/ov5640.c
index 852026baa2e7..f6e40cc9745c 100644
--- a/drivers/media/i2c/ov5640.c
+++ b/drivers/media/i2c/ov5640.c
@@ -60,6 +60,8 @@
 #define OV5640_REG_AEC_PK_MANUAL	0x3503
 #define OV5640_REG_AEC_PK_REAL_GAIN	0x350a
 #define OV5640_REG_AEC_PK_VTS		0x350c
+#define OV5640_REG_TIMING_DVPHO		0x3808
+#define OV5640_REG_TIMING_DVPVO		0x380a
 #define OV5640_REG_TIMING_HTS		0x380c
 #define OV5640_REG_TIMING_VTS		0x380e
 #define OV5640_REG_TIMING_TC_REG21	0x3821
@@ -91,6 +93,9 @@
 #define OV5640_REG_SDE_CTRL5		0x5585
 #define OV5640_REG_AVG_READOUT		0x56a1
 
+#define OV5640_SCLK2X_ROOT_DIVIDER_DEFAULT	1
+#define OV5640_SCLK_ROOT_DIVIDER_DEFAULT	2
+
 enum ov5640_mode_id {
 	OV5640_MODE_QCIF_176_144 = 0,
 	OV5640_MODE_QVGA_320_240,
@@ -165,8 +170,10 @@ struct reg_value {
 struct ov5640_mode_info {
 	enum ov5640_mode_id id;
 	enum ov5640_downsize_mode dn_mode;
-	u32 width;
-	u32 height;
+	u32 hact;
+	u32 htot;
+	u32 vact;
+	u32 vtot;
 	const struct reg_value *reg_data;
 	u32 reg_data_size;
 };
@@ -187,6 +194,7 @@ struct ov5640_ctrls {
 		struct v4l2_ctrl *gain;
 	};
 	struct v4l2_ctrl *brightness;
+	struct v4l2_ctrl *light_freq;
 	struct v4l2_ctrl *saturation;
 	struct v4l2_ctrl *contrast;
 	struct v4l2_ctrl *hue;
@@ -248,7 +256,7 @@ static const struct reg_value ov5640_init_setting_30fps_VGA[] = {
 	{0x3103, 0x11, 0, 0}, {0x3008, 0x82, 0, 5}, {0x3008, 0x42, 0, 0},
 	{0x3103, 0x03, 0, 0}, {0x3017, 0x00, 0, 0}, {0x3018, 0x00, 0, 0},
 	{0x3034, 0x18, 0, 0}, {0x3035, 0x14, 0, 0}, {0x3036, 0x38, 0, 0},
-	{0x3037, 0x13, 0, 0}, {0x3108, 0x01, 0, 0}, {0x3630, 0x36, 0, 0},
+	{0x3037, 0x13, 0, 0}, {0x3630, 0x36, 0, 0},
 	{0x3631, 0x0e, 0, 0}, {0x3632, 0xe2, 0, 0}, {0x3633, 0x12, 0, 0},
 	{0x3621, 0xe0, 0, 0}, {0x3704, 0xa0, 0, 0}, {0x3703, 0x5a, 0, 0},
 	{0x3715, 0x78, 0, 0}, {0x3717, 0x01, 0, 0}, {0x370b, 0x60, 0, 0},
@@ -265,9 +273,7 @@ static const struct reg_value ov5640_init_setting_30fps_VGA[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x02, 0, 0}, {0x3809, 0x80, 0, 0}, {0x380a, 0x01, 0, 0},
-	{0x380b, 0xe0, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x03, 0, 0}, {0x380f, 0xd8, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -339,9 +345,7 @@ static const struct reg_value ov5640_setting_30fps_VGA_640_480[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x02, 0, 0}, {0x3809, 0x80, 0, 0}, {0x380a, 0x01, 0, 0},
-	{0x380b, 0xe0, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x04, 0, 0}, {0x380f, 0x38, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -360,9 +364,7 @@ static const struct reg_value ov5640_setting_15fps_VGA_640_480[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x02, 0, 0}, {0x3809, 0x80, 0, 0}, {0x380a, 0x01, 0, 0},
-	{0x380b, 0xe0, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x03, 0, 0}, {0x380f, 0xd8, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -381,9 +383,7 @@ static const struct reg_value ov5640_setting_30fps_XGA_1024_768[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x02, 0, 0}, {0x3809, 0x80, 0, 0}, {0x380a, 0x01, 0, 0},
-	{0x380b, 0xe0, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x04, 0, 0}, {0x380f, 0x38, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -393,8 +393,7 @@ static const struct reg_value ov5640_setting_30fps_XGA_1024_768[] = {
 	{0x4001, 0x02, 0, 0}, {0x4004, 0x02, 0, 0}, {0x4713, 0x03, 0, 0},
 	{0x4407, 0x04, 0, 0}, {0x460b, 0x35, 0, 0}, {0x460c, 0x22, 0, 0},
 	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0}, {0x3503, 0x00, 0, 0},
-	{0x3808, 0x04, 0, 0}, {0x3809, 0x00, 0, 0}, {0x380a, 0x03, 0, 0},
-	{0x380b, 0x00, 0, 0}, {0x3035, 0x12, 0, 0},
+	{0x3035, 0x12, 0, 0},
 };
 
 static const struct reg_value ov5640_setting_15fps_XGA_1024_768[] = {
@@ -404,9 +403,7 @@ static const struct reg_value ov5640_setting_15fps_XGA_1024_768[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x02, 0, 0}, {0x3809, 0x80, 0, 0}, {0x380a, 0x01, 0, 0},
-	{0x380b, 0xe0, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x03, 0, 0}, {0x380f, 0xd8, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -415,8 +412,7 @@ static const struct reg_value ov5640_setting_15fps_XGA_1024_768[] = {
 	{0x3a0d, 0x04, 0, 0}, {0x3a14, 0x03, 0, 0}, {0x3a15, 0xd8, 0, 0},
 	{0x4001, 0x02, 0, 0}, {0x4004, 0x02, 0, 0}, {0x4713, 0x03, 0, 0},
 	{0x4407, 0x04, 0, 0}, {0x460b, 0x35, 0, 0}, {0x460c, 0x22, 0, 0},
-	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0}, {0x3808, 0x04, 0, 0},
-	{0x3809, 0x00, 0, 0}, {0x380a, 0x03, 0, 0}, {0x380b, 0x00, 0, 0},
+	{0x3824, 0x02, 0, 0}, {0x5001, 0xa3, 0, 0},
 };
 
 static const struct reg_value ov5640_setting_30fps_QVGA_320_240[] = {
@@ -426,9 +422,7 @@ static const struct reg_value ov5640_setting_30fps_QVGA_320_240[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x01, 0, 0}, {0x3809, 0x40, 0, 0}, {0x380a, 0x00, 0, 0},
-	{0x380b, 0xf0, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x03, 0, 0}, {0x380f, 0xd8, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -447,9 +441,7 @@ static const struct reg_value ov5640_setting_15fps_QVGA_320_240[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x01, 0, 0}, {0x3809, 0x40, 0, 0}, {0x380a, 0x00, 0, 0},
-	{0x380b, 0xf0, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x03, 0, 0}, {0x380f, 0xd8, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -468,9 +460,7 @@ static const struct reg_value ov5640_setting_30fps_QCIF_176_144[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x00, 0, 0}, {0x3809, 0xb0, 0, 0}, {0x380a, 0x00, 0, 0},
-	{0x380b, 0x90, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x03, 0, 0}, {0x380f, 0xd8, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -489,9 +479,7 @@ static const struct reg_value ov5640_setting_15fps_QCIF_176_144[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x00, 0, 0}, {0x3809, 0xb0, 0, 0}, {0x380a, 0x00, 0, 0},
-	{0x380b, 0x90, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x03, 0, 0}, {0x380f, 0xd8, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -510,9 +498,7 @@ static const struct reg_value ov5640_setting_30fps_NTSC_720_480[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x02, 0, 0}, {0x3809, 0xd0, 0, 0}, {0x380a, 0x01, 0, 0},
-	{0x380b, 0xe0, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x03, 0, 0}, {0x380f, 0xd8, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x3c, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -531,9 +517,7 @@ static const struct reg_value ov5640_setting_15fps_NTSC_720_480[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x02, 0, 0}, {0x3809, 0xd0, 0, 0}, {0x380a, 0x01, 0, 0},
-	{0x380b, 0xe0, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x03, 0, 0}, {0x380f, 0xd8, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x3c, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -552,9 +536,7 @@ static const struct reg_value ov5640_setting_30fps_PAL_720_576[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x02, 0, 0}, {0x3809, 0xd0, 0, 0}, {0x380a, 0x02, 0, 0},
-	{0x380b, 0x40, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x03, 0, 0}, {0x380f, 0xd8, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x38, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -573,9 +555,7 @@ static const struct reg_value ov5640_setting_15fps_PAL_720_576[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x04, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9b, 0, 0},
-	{0x3808, 0x02, 0, 0}, {0x3809, 0xd0, 0, 0}, {0x380a, 0x02, 0, 0},
-	{0x380b, 0x40, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x68, 0, 0},
-	{0x380e, 0x03, 0, 0}, {0x380f, 0xd8, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x38, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x06, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -595,9 +575,7 @@ static const struct reg_value ov5640_setting_30fps_720P_1280_720[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0xfa, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x06, 0, 0}, {0x3807, 0xa9, 0, 0},
-	{0x3808, 0x05, 0, 0}, {0x3809, 0x00, 0, 0}, {0x380a, 0x02, 0, 0},
-	{0x380b, 0xd0, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x64, 0, 0},
-	{0x380e, 0x02, 0, 0}, {0x380f, 0xe4, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x04, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x02, 0, 0},
@@ -617,9 +595,7 @@ static const struct reg_value ov5640_setting_15fps_720P_1280_720[] = {
 	{0x3815, 0x31, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0xfa, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x06, 0, 0}, {0x3807, 0xa9, 0, 0},
-	{0x3808, 0x05, 0, 0}, {0x3809, 0x00, 0, 0}, {0x380a, 0x02, 0, 0},
-	{0x380b, 0xd0, 0, 0}, {0x380c, 0x07, 0, 0}, {0x380d, 0x64, 0, 0},
-	{0x380e, 0x02, 0, 0}, {0x380f, 0xe4, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x04, 0, 0},
 	{0x3618, 0x00, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3709, 0x52, 0, 0}, {0x370c, 0x03, 0, 0}, {0x3a02, 0x02, 0, 0},
@@ -639,9 +615,7 @@ static const struct reg_value ov5640_setting_30fps_1080P_1920_1080[] = {
 	{0x3815, 0x11, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x00, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9f, 0, 0},
-	{0x3808, 0x0a, 0, 0}, {0x3809, 0x20, 0, 0}, {0x380a, 0x07, 0, 0},
-	{0x380b, 0x98, 0, 0}, {0x380c, 0x0b, 0, 0}, {0x380d, 0x1c, 0, 0},
-	{0x380e, 0x07, 0, 0}, {0x380f, 0xb0, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x04, 0, 0},
 	{0x3618, 0x04, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x21, 0, 0},
 	{0x3709, 0x12, 0, 0}, {0x370c, 0x00, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -655,10 +629,8 @@ static const struct reg_value ov5640_setting_30fps_1080P_1920_1080[] = {
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3800, 0x01, 0, 0}, {0x3801, 0x50, 0, 0}, {0x3802, 0x01, 0, 0},
 	{0x3803, 0xb2, 0, 0}, {0x3804, 0x08, 0, 0}, {0x3805, 0xef, 0, 0},
-	{0x3806, 0x05, 0, 0}, {0x3807, 0xf1, 0, 0}, {0x3808, 0x07, 0, 0},
-	{0x3809, 0x80, 0, 0}, {0x380a, 0x04, 0, 0}, {0x380b, 0x38, 0, 0},
-	{0x380c, 0x09, 0, 0}, {0x380d, 0xc4, 0, 0}, {0x380e, 0x04, 0, 0},
-	{0x380f, 0x60, 0, 0}, {0x3612, 0x2b, 0, 0}, {0x3708, 0x64, 0, 0},
+	{0x3806, 0x05, 0, 0}, {0x3807, 0xf1, 0, 0},
+	{0x3612, 0x2b, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3a02, 0x04, 0, 0}, {0x3a03, 0x60, 0, 0}, {0x3a08, 0x01, 0, 0},
 	{0x3a09, 0x50, 0, 0}, {0x3a0a, 0x01, 0, 0}, {0x3a0b, 0x18, 0, 0},
 	{0x3a0e, 0x03, 0, 0}, {0x3a0d, 0x04, 0, 0}, {0x3a14, 0x04, 0, 0},
@@ -676,9 +648,7 @@ static const struct reg_value ov5640_setting_15fps_1080P_1920_1080[] = {
 	{0x3815, 0x11, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x00, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9f, 0, 0},
-	{0x3808, 0x0a, 0, 0}, {0x3809, 0x20, 0, 0}, {0x380a, 0x07, 0, 0},
-	{0x380b, 0x98, 0, 0}, {0x380c, 0x0b, 0, 0}, {0x380d, 0x1c, 0, 0},
-	{0x380e, 0x07, 0, 0}, {0x380f, 0xb0, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x04, 0, 0},
 	{0x3618, 0x04, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x21, 0, 0},
 	{0x3709, 0x12, 0, 0}, {0x370c, 0x00, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -692,10 +662,8 @@ static const struct reg_value ov5640_setting_15fps_1080P_1920_1080[] = {
 	{0x3c09, 0x1c, 0, 0}, {0x3c0a, 0x9c, 0, 0}, {0x3c0b, 0x40, 0, 0},
 	{0x3800, 0x01, 0, 0}, {0x3801, 0x50, 0, 0}, {0x3802, 0x01, 0, 0},
 	{0x3803, 0xb2, 0, 0}, {0x3804, 0x08, 0, 0}, {0x3805, 0xef, 0, 0},
-	{0x3806, 0x05, 0, 0}, {0x3807, 0xf1, 0, 0}, {0x3808, 0x07, 0, 0},
-	{0x3809, 0x80, 0, 0}, {0x380a, 0x04, 0, 0}, {0x380b, 0x38, 0, 0},
-	{0x380c, 0x09, 0, 0}, {0x380d, 0xc4, 0, 0}, {0x380e, 0x04, 0, 0},
-	{0x380f, 0x60, 0, 0}, {0x3612, 0x2b, 0, 0}, {0x3708, 0x64, 0, 0},
+	{0x3806, 0x05, 0, 0}, {0x3807, 0xf1, 0, 0},
+	{0x3612, 0x2b, 0, 0}, {0x3708, 0x64, 0, 0},
 	{0x3a02, 0x04, 0, 0}, {0x3a03, 0x60, 0, 0}, {0x3a08, 0x01, 0, 0},
 	{0x3a09, 0x50, 0, 0}, {0x3a0a, 0x01, 0, 0}, {0x3a0b, 0x18, 0, 0},
 	{0x3a0e, 0x03, 0, 0}, {0x3a0d, 0x04, 0, 0}, {0x3a14, 0x04, 0, 0},
@@ -712,9 +680,7 @@ static const struct reg_value ov5640_setting_15fps_QSXGA_2592_1944[] = {
 	{0x3815, 0x11, 0, 0}, {0x3800, 0x00, 0, 0}, {0x3801, 0x00, 0, 0},
 	{0x3802, 0x00, 0, 0}, {0x3803, 0x00, 0, 0}, {0x3804, 0x0a, 0, 0},
 	{0x3805, 0x3f, 0, 0}, {0x3806, 0x07, 0, 0}, {0x3807, 0x9f, 0, 0},
-	{0x3808, 0x0a, 0, 0}, {0x3809, 0x20, 0, 0}, {0x380a, 0x07, 0, 0},
-	{0x380b, 0x98, 0, 0}, {0x380c, 0x0b, 0, 0}, {0x380d, 0x1c, 0, 0},
-	{0x380e, 0x07, 0, 0}, {0x380f, 0xb0, 0, 0}, {0x3810, 0x00, 0, 0},
+	{0x3810, 0x00, 0, 0},
 	{0x3811, 0x10, 0, 0}, {0x3812, 0x00, 0, 0}, {0x3813, 0x04, 0, 0},
 	{0x3618, 0x04, 0, 0}, {0x3612, 0x29, 0, 0}, {0x3708, 0x21, 0, 0},
 	{0x3709, 0x12, 0, 0}, {0x370c, 0x00, 0, 0}, {0x3a02, 0x03, 0, 0},
@@ -728,66 +694,84 @@ static const struct reg_value ov5640_setting_15fps_QSXGA_2592_1944[] = {
 
 /* power-on sensor init reg table */
 static const struct ov5640_mode_info ov5640_mode_init_data = {
-	0, SUBSAMPLING, 640, 480, ov5640_init_setting_30fps_VGA,
+	0, SUBSAMPLING, 640, 1896, 480, 984,
+	ov5640_init_setting_30fps_VGA,
 	ARRAY_SIZE(ov5640_init_setting_30fps_VGA),
 };
 
 static const struct ov5640_mode_info
 ov5640_mode_data[OV5640_NUM_FRAMERATES][OV5640_NUM_MODES] = {
 	{
-		{OV5640_MODE_QCIF_176_144, SUBSAMPLING, 176, 144,
+		{OV5640_MODE_QCIF_176_144, SUBSAMPLING,
+		 176, 1896, 144, 984,
 		 ov5640_setting_15fps_QCIF_176_144,
 		 ARRAY_SIZE(ov5640_setting_15fps_QCIF_176_144)},
-		{OV5640_MODE_QVGA_320_240, SUBSAMPLING, 320,  240,
+		{OV5640_MODE_QVGA_320_240, SUBSAMPLING,
+		 320, 1896, 240, 984,
 		 ov5640_setting_15fps_QVGA_320_240,
 		 ARRAY_SIZE(ov5640_setting_15fps_QVGA_320_240)},
-		{OV5640_MODE_VGA_640_480, SUBSAMPLING, 640,  480,
+		{OV5640_MODE_VGA_640_480, SUBSAMPLING,
+		 640, 1896, 480, 1080,
 		 ov5640_setting_15fps_VGA_640_480,
 		 ARRAY_SIZE(ov5640_setting_15fps_VGA_640_480)},
-		{OV5640_MODE_NTSC_720_480, SUBSAMPLING, 720, 480,
+		{OV5640_MODE_NTSC_720_480, SUBSAMPLING,
+		 720, 1896, 480, 984,
 		 ov5640_setting_15fps_NTSC_720_480,
 		 ARRAY_SIZE(ov5640_setting_15fps_NTSC_720_480)},
-		{OV5640_MODE_PAL_720_576, SUBSAMPLING, 720, 576,
+		{OV5640_MODE_PAL_720_576, SUBSAMPLING,
+		 720, 1896, 576, 984,
 		 ov5640_setting_15fps_PAL_720_576,
 		 ARRAY_SIZE(ov5640_setting_15fps_PAL_720_576)},
-		{OV5640_MODE_XGA_1024_768, SUBSAMPLING, 1024, 768,
+		{OV5640_MODE_XGA_1024_768, SUBSAMPLING,
+		 1024, 1896, 768, 1080,
 		 ov5640_setting_15fps_XGA_1024_768,
 		 ARRAY_SIZE(ov5640_setting_15fps_XGA_1024_768)},
-		{OV5640_MODE_720P_1280_720, SUBSAMPLING, 1280, 720,
+		{OV5640_MODE_720P_1280_720, SUBSAMPLING,
+		 1280, 1892, 720, 740,
 		 ov5640_setting_15fps_720P_1280_720,
 		 ARRAY_SIZE(ov5640_setting_15fps_720P_1280_720)},
-		{OV5640_MODE_1080P_1920_1080, SCALING, 1920, 1080,
+		{OV5640_MODE_1080P_1920_1080, SCALING,
+		 1920, 2500, 1080, 1120,
 		 ov5640_setting_15fps_1080P_1920_1080,
 		 ARRAY_SIZE(ov5640_setting_15fps_1080P_1920_1080)},
-		{OV5640_MODE_QSXGA_2592_1944, SCALING, 2592, 1944,
+		{OV5640_MODE_QSXGA_2592_1944, SCALING,
+		 2592, 2844, 1944, 1968,
 		 ov5640_setting_15fps_QSXGA_2592_1944,
 		 ARRAY_SIZE(ov5640_setting_15fps_QSXGA_2592_1944)},
 	}, {
-		{OV5640_MODE_QCIF_176_144, SUBSAMPLING, 176, 144,
+		{OV5640_MODE_QCIF_176_144, SUBSAMPLING,
+		 176, 1896, 144, 984,
 		 ov5640_setting_30fps_QCIF_176_144,
 		 ARRAY_SIZE(ov5640_setting_30fps_QCIF_176_144)},
-		{OV5640_MODE_QVGA_320_240, SUBSAMPLING, 320,  240,
+		{OV5640_MODE_QVGA_320_240, SUBSAMPLING,
+		 320, 1896, 240, 984,
 		 ov5640_setting_30fps_QVGA_320_240,
 		 ARRAY_SIZE(ov5640_setting_30fps_QVGA_320_240)},
-		{OV5640_MODE_VGA_640_480, SUBSAMPLING, 640,  480,
+		{OV5640_MODE_VGA_640_480, SUBSAMPLING,
+		 640, 1896, 480, 1080,
 		 ov5640_setting_30fps_VGA_640_480,
 		 ARRAY_SIZE(ov5640_setting_30fps_VGA_640_480)},
-		{OV5640_MODE_NTSC_720_480, SUBSAMPLING, 720, 480,
+		{OV5640_MODE_NTSC_720_480, SUBSAMPLING,
+		 720, 1896, 480, 984,
 		 ov5640_setting_30fps_NTSC_720_480,
 		 ARRAY_SIZE(ov5640_setting_30fps_NTSC_720_480)},
-		{OV5640_MODE_PAL_720_576, SUBSAMPLING, 720, 576,
+		{OV5640_MODE_PAL_720_576, SUBSAMPLING,
+		 720, 1896, 576, 984,
 		 ov5640_setting_30fps_PAL_720_576,
 		 ARRAY_SIZE(ov5640_setting_30fps_PAL_720_576)},
-		{OV5640_MODE_XGA_1024_768, SUBSAMPLING, 1024, 768,
+		{OV5640_MODE_XGA_1024_768, SUBSAMPLING,
+		 1024, 1896, 768, 1080,
 		 ov5640_setting_30fps_XGA_1024_768,
 		 ARRAY_SIZE(ov5640_setting_30fps_XGA_1024_768)},
-		{OV5640_MODE_720P_1280_720, SUBSAMPLING, 1280, 720,
+		{OV5640_MODE_720P_1280_720, SUBSAMPLING,
+		 1280, 1892, 720, 740,
 		 ov5640_setting_30fps_720P_1280_720,
 		 ARRAY_SIZE(ov5640_setting_30fps_720P_1280_720)},
-		{OV5640_MODE_1080P_1920_1080, SCALING, 1920, 1080,
+		{OV5640_MODE_1080P_1920_1080, SCALING,
+		 1920, 2500, 1080, 1120,
 		 ov5640_setting_30fps_1080P_1920_1080,
 		 ARRAY_SIZE(ov5640_setting_30fps_1080P_1920_1080)},
-		{OV5640_MODE_QSXGA_2592_1944, -1, 0, 0, NULL, 0},
+		{OV5640_MODE_QSXGA_2592_1944, -1, 0, 0, 0, 0, NULL, 0},
 	},
 };
 
@@ -1377,6 +1361,30 @@ static int ov5640_set_virtual_channel(struct ov5640_dev *sensor)
 	return ov5640_write_reg(sensor, OV5640_REG_DEBUG_MODE, temp);
 }
 
+static int ov5640_set_timings(struct ov5640_dev *sensor,
+			      const struct ov5640_mode_info *mode)
+{
+	int ret;
+
+	ret = ov5640_write_reg16(sensor, OV5640_REG_TIMING_DVPHO, mode->hact);
+	if (ret < 0)
+		return ret;
+
+	ret = ov5640_write_reg16(sensor, OV5640_REG_TIMING_DVPVO, mode->vact);
+	if (ret < 0)
+		return ret;
+
+	ret = ov5640_write_reg16(sensor, OV5640_REG_TIMING_HTS, mode->htot);
+	if (ret < 0)
+		return ret;
+
+	ret = ov5640_write_reg16(sensor, OV5640_REG_TIMING_VTS, mode->vtot);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
 static const struct ov5640_mode_info *
 ov5640_find_mode(struct ov5640_dev *sensor, enum ov5640_frame_rate fr,
 		 int width, int height, bool nearest)
@@ -1390,10 +1398,10 @@ ov5640_find_mode(struct ov5640_dev *sensor, enum ov5640_frame_rate fr,
 		if (!mode->reg_data)
 			continue;
 
-		if ((nearest && mode->width <= width &&
-		     mode->height <= height) ||
-		    (!nearest && mode->width == width &&
-		     mode->height == height))
+		if ((nearest && mode->hact <= width &&
+		     mode->vact <= height) ||
+		    (!nearest && mode->hact == width &&
+		     mode->vact == height))
 			break;
 	}
 
@@ -1568,7 +1576,8 @@ static int ov5640_set_mode_exposure_calc(struct ov5640_dev *sensor,
  * change mode directly
  */
 static int ov5640_set_mode_direct(struct ov5640_dev *sensor,
-				  const struct ov5640_mode_info *mode)
+				  const struct ov5640_mode_info *mode,
+				  s32 exposure)
 {
 	int ret;
 
@@ -1584,7 +1593,8 @@ static int ov5640_set_mode_direct(struct ov5640_dev *sensor,
 	ret = __v4l2_ctrl_s_ctrl(sensor->ctrls.auto_gain, 1);
 	if (ret)
 		return ret;
-	return __v4l2_ctrl_s_ctrl(sensor->ctrls.auto_exp, V4L2_EXPOSURE_AUTO);
+
+	return __v4l2_ctrl_s_ctrl(sensor->ctrls.auto_exp, exposure);
 }
 
 static int ov5640_set_mode(struct ov5640_dev *sensor,
@@ -1592,6 +1602,7 @@ static int ov5640_set_mode(struct ov5640_dev *sensor,
 {
 	const struct ov5640_mode_info *mode = sensor->current_mode;
 	enum ov5640_downsize_mode dn_mode, orig_dn_mode;
+	s32 exposure;
 	int ret;
 
 	dn_mode = mode->dn_mode;
@@ -1601,7 +1612,9 @@ static int ov5640_set_mode(struct ov5640_dev *sensor,
 	ret = __v4l2_ctrl_s_ctrl(sensor->ctrls.auto_gain, 0);
 	if (ret)
 		return ret;
-	ret = __v4l2_ctrl_s_ctrl(sensor->ctrls.auto_exp, V4L2_EXPOSURE_MANUAL);
+
+	exposure = sensor->ctrls.auto_exp->val;
+	ret = ov5640_set_exposure(sensor, V4L2_EXPOSURE_MANUAL);
 	if (ret)
 		return ret;
 
@@ -1617,9 +1630,13 @@ static int ov5640_set_mode(struct ov5640_dev *sensor,
 		 * change inside subsampling or scaling
 		 * download firmware directly
 		 */
-		ret = ov5640_set_mode_direct(sensor, mode);
+		ret = ov5640_set_mode_direct(sensor, mode, exposure);
 	}
 
+	if (ret < 0)
+		return ret;
+
+	ret = ov5640_set_timings(sensor, mode);
 	if (ret < 0)
 		return ret;
 
@@ -1654,6 +1671,12 @@ static int ov5640_restore_mode(struct ov5640_dev *sensor)
 	if (ret < 0)
 		return ret;
 
+	ret = ov5640_mod_reg(sensor, OV5640_REG_SYS_ROOT_DIVIDER, 0x3f,
+			     (ilog2(OV5640_SCLK2X_ROOT_DIVIDER_DEFAULT) << 2) |
+			     ilog2(OV5640_SCLK_ROOT_DIVIDER_DEFAULT));
+	if (ret)
+		return ret;
+
 	/* now restore the last capture mode */
 	ret = ov5640_set_mode(sensor, &ov5640_mode_init_data);
 	if (ret < 0)
@@ -1871,8 +1894,8 @@ static int ov5640_try_fmt_internal(struct v4l2_subdev *sd,
 	mode = ov5640_find_mode(sensor, fr, fmt->width, fmt->height, true);
 	if (!mode)
 		return -EINVAL;
-	fmt->width = mode->width;
-	fmt->height = mode->height;
+	fmt->width = mode->hact;
+	fmt->height = mode->vact;
 
 	if (new_mode)
 		*new_mode = mode;
@@ -2155,6 +2178,21 @@ static int ov5640_set_ctrl_test_pattern(struct ov5640_dev *sensor, int value)
 			      0xa4, value ? 0xa4 : 0);
 }
 
+static int ov5640_set_ctrl_light_freq(struct ov5640_dev *sensor, int value)
+{
+	int ret;
+
+	ret = ov5640_mod_reg(sensor, OV5640_REG_HZ5060_CTRL01, BIT(7),
+			     (value == V4L2_CID_POWER_LINE_FREQUENCY_AUTO) ?
+			     0 : BIT(7));
+	if (ret)
+		return ret;
+
+	return ov5640_mod_reg(sensor, OV5640_REG_HZ5060_CTRL00, BIT(2),
+			      (value == V4L2_CID_POWER_LINE_FREQUENCY_50HZ) ?
+			      BIT(2) : 0);
+}
+
 static int ov5640_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct v4l2_subdev *sd = ctrl_to_sd(ctrl);
@@ -2223,6 +2261,9 @@ static int ov5640_s_ctrl(struct v4l2_ctrl *ctrl)
 	case V4L2_CID_TEST_PATTERN:
 		ret = ov5640_set_ctrl_test_pattern(sensor, ctrl->val);
 		break;
+	case V4L2_CID_POWER_LINE_FREQUENCY:
+		ret = ov5640_set_ctrl_light_freq(sensor, ctrl->val);
+		break;
 	default:
 		ret = -EINVAL;
 		break;
@@ -2285,6 +2326,12 @@ static int ov5640_init_controls(struct ov5640_dev *sensor)
 					     ARRAY_SIZE(test_pattern_menu) - 1,
 					     0, 0, test_pattern_menu);
 
+	ctrls->light_freq =
+		v4l2_ctrl_new_std_menu(hdl, ops,
+				       V4L2_CID_POWER_LINE_FREQUENCY,
+				       V4L2_CID_POWER_LINE_FREQUENCY_AUTO, 0,
+				       V4L2_CID_POWER_LINE_FREQUENCY_50HZ);
+
 	if (hdl->error) {
 		ret = hdl->error;
 		goto free_ctrls;
@@ -2315,10 +2362,10 @@ static int ov5640_enum_frame_size(struct v4l2_subdev *sd,
 		return -EINVAL;
 
 	fse->min_width =
-		ov5640_mode_data[0][fse->index].width;
+		ov5640_mode_data[0][fse->index].hact;
 	fse->max_width = fse->min_width;
 	fse->min_height =
-		ov5640_mode_data[0][fse->index].height;
+		ov5640_mode_data[0][fse->index].vact;
 	fse->max_height = fse->min_height;
 
 	return 0;
@@ -2382,14 +2429,14 @@ static int ov5640_s_frame_interval(struct v4l2_subdev *sd,
 	mode = sensor->current_mode;
 
 	frame_rate = ov5640_try_frame_interval(sensor, &fi->interval,
-					       mode->width, mode->height);
+					       mode->hact, mode->vact);
 	if (frame_rate < 0)
 		frame_rate = OV5640_15_FPS;
 
 	sensor->current_fr = frame_rate;
 	sensor->frame_interval = fi->interval;
-	sensor->current_mode = ov5640_find_mode(sensor, frame_rate, mode->width,
-						mode->height, true);
+	sensor->current_mode = ov5640_find_mode(sensor, frame_rate, mode->hact,
+						mode->vact, true);
 	sensor->pending_mode_change = true;
 out:
 	mutex_unlock(&sensor->lock);
@@ -2536,8 +2583,8 @@ static int ov5640_probe(struct i2c_client *client,
 
 	sensor->ae_target = 52;
 
-	endpoint = fwnode_graph_get_next_endpoint(
-		of_fwnode_handle(client->dev.of_node), NULL);
+	endpoint = fwnode_graph_get_next_endpoint(dev_fwnode(&client->dev),
+						  NULL);
 	if (!endpoint) {
 		dev_err(dev, "endpoint node not found\n");
 		return -EINVAL;
diff --git a/drivers/media/i2c/ov5645.c b/drivers/media/i2c/ov5645.c
index 4e3142a7e5a7..b3f762578f7f 100644
--- a/drivers/media/i2c/ov5645.c
+++ b/drivers/media/i2c/ov5645.c
@@ -600,11 +600,13 @@ static int ov5645_write_reg(struct ov5645 *ov5645, u16 reg, u8 val)
 	regbuf[2] = val;
 
 	ret = i2c_master_send(ov5645->i2c_client, regbuf, 3);
-	if (ret < 0)
+	if (ret < 0) {
 		dev_err(ov5645->dev, "%s: write reg error %d: reg=%x, val=%x\n",
 			__func__, ret, reg, val);
+		return ret;
+	}
 
-	return ret;
+	return 0;
 }
 
 static int ov5645_read_reg(struct ov5645 *ov5645, u16 reg, u8 *val)
diff --git a/drivers/media/i2c/ov5695.c b/drivers/media/i2c/ov5695.c
index 9be38a0a2046..9a80decd93d3 100644
--- a/drivers/media/i2c/ov5695.c
+++ b/drivers/media/i2c/ov5695.c
@@ -1385,7 +1385,6 @@ MODULE_DEVICE_TABLE(of, ov5695_of_match);
 static struct i2c_driver ov5695_i2c_driver = {
 	.driver = {
 		.name = "ov5695",
-		.owner = THIS_MODULE,
 		.pm = &ov5695_pm_ops,
 		.of_match_table = of_match_ptr(ov5695_of_match),
 	},
diff --git a/drivers/media/i2c/ov7251.c b/drivers/media/i2c/ov7251.c
new file mode 100644
index 000000000000..d3ebb7529fca
--- /dev/null
+++ b/drivers/media/i2c/ov7251.c
@@ -0,0 +1,1503 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for the OV7251 camera sensor.
+ *
+ * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2017-2018, Linaro Ltd.
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-subdev.h>
+
+#define OV7251_SC_MODE_SELECT		0x0100
+#define OV7251_SC_MODE_SELECT_SW_STANDBY	0x0
+#define OV7251_SC_MODE_SELECT_STREAMING		0x1
+
+#define OV7251_CHIP_ID_HIGH		0x300a
+#define OV7251_CHIP_ID_HIGH_BYTE	0x77
+#define OV7251_CHIP_ID_LOW		0x300b
+#define OV7251_CHIP_ID_LOW_BYTE		0x50
+#define OV7251_SC_GP_IO_IN1		0x3029
+#define OV7251_AEC_EXPO_0		0x3500
+#define OV7251_AEC_EXPO_1		0x3501
+#define OV7251_AEC_EXPO_2		0x3502
+#define OV7251_AEC_AGC_ADJ_0		0x350a
+#define OV7251_AEC_AGC_ADJ_1		0x350b
+#define OV7251_TIMING_FORMAT1		0x3820
+#define OV7251_TIMING_FORMAT1_VFLIP	BIT(2)
+#define OV7251_TIMING_FORMAT2		0x3821
+#define OV7251_TIMING_FORMAT2_MIRROR	BIT(2)
+#define OV7251_PRE_ISP_00		0x5e00
+#define OV7251_PRE_ISP_00_TEST_PATTERN	BIT(7)
+
+struct reg_value {
+	u16 reg;
+	u8 val;
+};
+
+struct ov7251_mode_info {
+	u32 width;
+	u32 height;
+	const struct reg_value *data;
+	u32 data_size;
+	u32 pixel_clock;
+	u32 link_freq;
+	u16 exposure_max;
+	u16 exposure_def;
+	struct v4l2_fract timeperframe;
+};
+
+struct ov7251 {
+	struct i2c_client *i2c_client;
+	struct device *dev;
+	struct v4l2_subdev sd;
+	struct media_pad pad;
+	struct v4l2_fwnode_endpoint ep;
+	struct v4l2_mbus_framefmt fmt;
+	struct v4l2_rect crop;
+	struct clk *xclk;
+	u32 xclk_freq;
+
+	struct regulator *io_regulator;
+	struct regulator *core_regulator;
+	struct regulator *analog_regulator;
+
+	const struct ov7251_mode_info *current_mode;
+
+	struct v4l2_ctrl_handler ctrls;
+	struct v4l2_ctrl *pixel_clock;
+	struct v4l2_ctrl *link_freq;
+	struct v4l2_ctrl *exposure;
+	struct v4l2_ctrl *gain;
+
+	/* Cached register values */
+	u8 aec_pk_manual;
+	u8 pre_isp_00;
+	u8 timing_format1;
+	u8 timing_format2;
+
+	struct mutex lock; /* lock to protect power state, ctrls and mode */
+	bool power_on;
+
+	struct gpio_desc *enable_gpio;
+};
+
+static inline struct ov7251 *to_ov7251(struct v4l2_subdev *sd)
+{
+	return container_of(sd, struct ov7251, sd);
+}
+
+static const struct reg_value ov7251_global_init_setting[] = {
+	{ 0x0103, 0x01 },
+	{ 0x303b, 0x02 },
+};
+
+static const struct reg_value ov7251_setting_vga_30fps[] = {
+	{ 0x3005, 0x00 },
+	{ 0x3012, 0xc0 },
+	{ 0x3013, 0xd2 },
+	{ 0x3014, 0x04 },
+	{ 0x3016, 0xf0 },
+	{ 0x3017, 0xf0 },
+	{ 0x3018, 0xf0 },
+	{ 0x301a, 0xf0 },
+	{ 0x301b, 0xf0 },
+	{ 0x301c, 0xf0 },
+	{ 0x3023, 0x05 },
+	{ 0x3037, 0xf0 },
+	{ 0x3098, 0x04 }, /* pll2 pre divider */
+	{ 0x3099, 0x28 }, /* pll2 multiplier */
+	{ 0x309a, 0x05 }, /* pll2 sys divider */
+	{ 0x309b, 0x04 }, /* pll2 adc divider */
+	{ 0x309d, 0x00 }, /* pll2 divider */
+	{ 0x30b0, 0x0a }, /* pll1 pix divider */
+	{ 0x30b1, 0x01 }, /* pll1 divider */
+	{ 0x30b3, 0x64 }, /* pll1 multiplier */
+	{ 0x30b4, 0x03 }, /* pll1 pre divider */
+	{ 0x30b5, 0x05 }, /* pll1 mipi divider */
+	{ 0x3106, 0xda },
+	{ 0x3503, 0x07 },
+	{ 0x3509, 0x10 },
+	{ 0x3600, 0x1c },
+	{ 0x3602, 0x62 },
+	{ 0x3620, 0xb7 },
+	{ 0x3622, 0x04 },
+	{ 0x3626, 0x21 },
+	{ 0x3627, 0x30 },
+	{ 0x3630, 0x44 },
+	{ 0x3631, 0x35 },
+	{ 0x3634, 0x60 },
+	{ 0x3636, 0x00 },
+	{ 0x3662, 0x01 },
+	{ 0x3663, 0x70 },
+	{ 0x3664, 0x50 },
+	{ 0x3666, 0x0a },
+	{ 0x3669, 0x1a },
+	{ 0x366a, 0x00 },
+	{ 0x366b, 0x50 },
+	{ 0x3673, 0x01 },
+	{ 0x3674, 0xff },
+	{ 0x3675, 0x03 },
+	{ 0x3705, 0xc1 },
+	{ 0x3709, 0x40 },
+	{ 0x373c, 0x08 },
+	{ 0x3742, 0x00 },
+	{ 0x3757, 0xb3 },
+	{ 0x3788, 0x00 },
+	{ 0x37a8, 0x01 },
+	{ 0x37a9, 0xc0 },
+	{ 0x3800, 0x00 },
+	{ 0x3801, 0x04 },
+	{ 0x3802, 0x00 },
+	{ 0x3803, 0x04 },
+	{ 0x3804, 0x02 },
+	{ 0x3805, 0x8b },
+	{ 0x3806, 0x01 },
+	{ 0x3807, 0xeb },
+	{ 0x3808, 0x02 }, /* width high */
+	{ 0x3809, 0x80 }, /* width low */
+	{ 0x380a, 0x01 }, /* height high */
+	{ 0x380b, 0xe0 }, /* height low */
+	{ 0x380c, 0x03 }, /* total horiz timing high */
+	{ 0x380d, 0xa0 }, /* total horiz timing low */
+	{ 0x380e, 0x06 }, /* total vertical timing high */
+	{ 0x380f, 0xbc }, /* total vertical timing low */
+	{ 0x3810, 0x00 },
+	{ 0x3811, 0x04 },
+	{ 0x3812, 0x00 },
+	{ 0x3813, 0x05 },
+	{ 0x3814, 0x11 },
+	{ 0x3815, 0x11 },
+	{ 0x3820, 0x40 },
+	{ 0x3821, 0x00 },
+	{ 0x382f, 0x0e },
+	{ 0x3832, 0x00 },
+	{ 0x3833, 0x05 },
+	{ 0x3834, 0x00 },
+	{ 0x3835, 0x0c },
+	{ 0x3837, 0x00 },
+	{ 0x3b80, 0x00 },
+	{ 0x3b81, 0xa5 },
+	{ 0x3b82, 0x10 },
+	{ 0x3b83, 0x00 },
+	{ 0x3b84, 0x08 },
+	{ 0x3b85, 0x00 },
+	{ 0x3b86, 0x01 },
+	{ 0x3b87, 0x00 },
+	{ 0x3b88, 0x00 },
+	{ 0x3b89, 0x00 },
+	{ 0x3b8a, 0x00 },
+	{ 0x3b8b, 0x05 },
+	{ 0x3b8c, 0x00 },
+	{ 0x3b8d, 0x00 },
+	{ 0x3b8e, 0x00 },
+	{ 0x3b8f, 0x1a },
+	{ 0x3b94, 0x05 },
+	{ 0x3b95, 0xf2 },
+	{ 0x3b96, 0x40 },
+	{ 0x3c00, 0x89 },
+	{ 0x3c01, 0x63 },
+	{ 0x3c02, 0x01 },
+	{ 0x3c03, 0x00 },
+	{ 0x3c04, 0x00 },
+	{ 0x3c05, 0x03 },
+	{ 0x3c06, 0x00 },
+	{ 0x3c07, 0x06 },
+	{ 0x3c0c, 0x01 },
+	{ 0x3c0d, 0xd0 },
+	{ 0x3c0e, 0x02 },
+	{ 0x3c0f, 0x0a },
+	{ 0x4001, 0x42 },
+	{ 0x4004, 0x04 },
+	{ 0x4005, 0x00 },
+	{ 0x404e, 0x01 },
+	{ 0x4300, 0xff },
+	{ 0x4301, 0x00 },
+	{ 0x4315, 0x00 },
+	{ 0x4501, 0x48 },
+	{ 0x4600, 0x00 },
+	{ 0x4601, 0x4e },
+	{ 0x4801, 0x0f },
+	{ 0x4806, 0x0f },
+	{ 0x4819, 0xaa },
+	{ 0x4823, 0x3e },
+	{ 0x4837, 0x19 },
+	{ 0x4a0d, 0x00 },
+	{ 0x4a47, 0x7f },
+	{ 0x4a49, 0xf0 },
+	{ 0x4a4b, 0x30 },
+	{ 0x5000, 0x85 },
+	{ 0x5001, 0x80 },
+};
+
+static const struct reg_value ov7251_setting_vga_60fps[] = {
+	{ 0x3005, 0x00 },
+	{ 0x3012, 0xc0 },
+	{ 0x3013, 0xd2 },
+	{ 0x3014, 0x04 },
+	{ 0x3016, 0x10 },
+	{ 0x3017, 0x00 },
+	{ 0x3018, 0x00 },
+	{ 0x301a, 0x00 },
+	{ 0x301b, 0x00 },
+	{ 0x301c, 0x00 },
+	{ 0x3023, 0x05 },
+	{ 0x3037, 0xf0 },
+	{ 0x3098, 0x04 }, /* pll2 pre divider */
+	{ 0x3099, 0x28 }, /* pll2 multiplier */
+	{ 0x309a, 0x05 }, /* pll2 sys divider */
+	{ 0x309b, 0x04 }, /* pll2 adc divider */
+	{ 0x309d, 0x00 }, /* pll2 divider */
+	{ 0x30b0, 0x0a }, /* pll1 pix divider */
+	{ 0x30b1, 0x01 }, /* pll1 divider */
+	{ 0x30b3, 0x64 }, /* pll1 multiplier */
+	{ 0x30b4, 0x03 }, /* pll1 pre divider */
+	{ 0x30b5, 0x05 }, /* pll1 mipi divider */
+	{ 0x3106, 0xda },
+	{ 0x3503, 0x07 },
+	{ 0x3509, 0x10 },
+	{ 0x3600, 0x1c },
+	{ 0x3602, 0x62 },
+	{ 0x3620, 0xb7 },
+	{ 0x3622, 0x04 },
+	{ 0x3626, 0x21 },
+	{ 0x3627, 0x30 },
+	{ 0x3630, 0x44 },
+	{ 0x3631, 0x35 },
+	{ 0x3634, 0x60 },
+	{ 0x3636, 0x00 },
+	{ 0x3662, 0x01 },
+	{ 0x3663, 0x70 },
+	{ 0x3664, 0x50 },
+	{ 0x3666, 0x0a },
+	{ 0x3669, 0x1a },
+	{ 0x366a, 0x00 },
+	{ 0x366b, 0x50 },
+	{ 0x3673, 0x01 },
+	{ 0x3674, 0xff },
+	{ 0x3675, 0x03 },
+	{ 0x3705, 0xc1 },
+	{ 0x3709, 0x40 },
+	{ 0x373c, 0x08 },
+	{ 0x3742, 0x00 },
+	{ 0x3757, 0xb3 },
+	{ 0x3788, 0x00 },
+	{ 0x37a8, 0x01 },
+	{ 0x37a9, 0xc0 },
+	{ 0x3800, 0x00 },
+	{ 0x3801, 0x04 },
+	{ 0x3802, 0x00 },
+	{ 0x3803, 0x04 },
+	{ 0x3804, 0x02 },
+	{ 0x3805, 0x8b },
+	{ 0x3806, 0x01 },
+	{ 0x3807, 0xeb },
+	{ 0x3808, 0x02 }, /* width high */
+	{ 0x3809, 0x80 }, /* width low */
+	{ 0x380a, 0x01 }, /* height high */
+	{ 0x380b, 0xe0 }, /* height low */
+	{ 0x380c, 0x03 }, /* total horiz timing high */
+	{ 0x380d, 0xa0 }, /* total horiz timing low */
+	{ 0x380e, 0x03 }, /* total vertical timing high */
+	{ 0x380f, 0x5c }, /* total vertical timing low */
+	{ 0x3810, 0x00 },
+	{ 0x3811, 0x04 },
+	{ 0x3812, 0x00 },
+	{ 0x3813, 0x05 },
+	{ 0x3814, 0x11 },
+	{ 0x3815, 0x11 },
+	{ 0x3820, 0x40 },
+	{ 0x3821, 0x00 },
+	{ 0x382f, 0x0e },
+	{ 0x3832, 0x00 },
+	{ 0x3833, 0x05 },
+	{ 0x3834, 0x00 },
+	{ 0x3835, 0x0c },
+	{ 0x3837, 0x00 },
+	{ 0x3b80, 0x00 },
+	{ 0x3b81, 0xa5 },
+	{ 0x3b82, 0x10 },
+	{ 0x3b83, 0x00 },
+	{ 0x3b84, 0x08 },
+	{ 0x3b85, 0x00 },
+	{ 0x3b86, 0x01 },
+	{ 0x3b87, 0x00 },
+	{ 0x3b88, 0x00 },
+	{ 0x3b89, 0x00 },
+	{ 0x3b8a, 0x00 },
+	{ 0x3b8b, 0x05 },
+	{ 0x3b8c, 0x00 },
+	{ 0x3b8d, 0x00 },
+	{ 0x3b8e, 0x00 },
+	{ 0x3b8f, 0x1a },
+	{ 0x3b94, 0x05 },
+	{ 0x3b95, 0xf2 },
+	{ 0x3b96, 0x40 },
+	{ 0x3c00, 0x89 },
+	{ 0x3c01, 0x63 },
+	{ 0x3c02, 0x01 },
+	{ 0x3c03, 0x00 },
+	{ 0x3c04, 0x00 },
+	{ 0x3c05, 0x03 },
+	{ 0x3c06, 0x00 },
+	{ 0x3c07, 0x06 },
+	{ 0x3c0c, 0x01 },
+	{ 0x3c0d, 0xd0 },
+	{ 0x3c0e, 0x02 },
+	{ 0x3c0f, 0x0a },
+	{ 0x4001, 0x42 },
+	{ 0x4004, 0x04 },
+	{ 0x4005, 0x00 },
+	{ 0x404e, 0x01 },
+	{ 0x4300, 0xff },
+	{ 0x4301, 0x00 },
+	{ 0x4315, 0x00 },
+	{ 0x4501, 0x48 },
+	{ 0x4600, 0x00 },
+	{ 0x4601, 0x4e },
+	{ 0x4801, 0x0f },
+	{ 0x4806, 0x0f },
+	{ 0x4819, 0xaa },
+	{ 0x4823, 0x3e },
+	{ 0x4837, 0x19 },
+	{ 0x4a0d, 0x00 },
+	{ 0x4a47, 0x7f },
+	{ 0x4a49, 0xf0 },
+	{ 0x4a4b, 0x30 },
+	{ 0x5000, 0x85 },
+	{ 0x5001, 0x80 },
+};
+
+static const struct reg_value ov7251_setting_vga_90fps[] = {
+	{ 0x3005, 0x00 },
+	{ 0x3012, 0xc0 },
+	{ 0x3013, 0xd2 },
+	{ 0x3014, 0x04 },
+	{ 0x3016, 0x10 },
+	{ 0x3017, 0x00 },
+	{ 0x3018, 0x00 },
+	{ 0x301a, 0x00 },
+	{ 0x301b, 0x00 },
+	{ 0x301c, 0x00 },
+	{ 0x3023, 0x05 },
+	{ 0x3037, 0xf0 },
+	{ 0x3098, 0x04 }, /* pll2 pre divider */
+	{ 0x3099, 0x28 }, /* pll2 multiplier */
+	{ 0x309a, 0x05 }, /* pll2 sys divider */
+	{ 0x309b, 0x04 }, /* pll2 adc divider */
+	{ 0x309d, 0x00 }, /* pll2 divider */
+	{ 0x30b0, 0x0a }, /* pll1 pix divider */
+	{ 0x30b1, 0x01 }, /* pll1 divider */
+	{ 0x30b3, 0x64 }, /* pll1 multiplier */
+	{ 0x30b4, 0x03 }, /* pll1 pre divider */
+	{ 0x30b5, 0x05 }, /* pll1 mipi divider */
+	{ 0x3106, 0xda },
+	{ 0x3503, 0x07 },
+	{ 0x3509, 0x10 },
+	{ 0x3600, 0x1c },
+	{ 0x3602, 0x62 },
+	{ 0x3620, 0xb7 },
+	{ 0x3622, 0x04 },
+	{ 0x3626, 0x21 },
+	{ 0x3627, 0x30 },
+	{ 0x3630, 0x44 },
+	{ 0x3631, 0x35 },
+	{ 0x3634, 0x60 },
+	{ 0x3636, 0x00 },
+	{ 0x3662, 0x01 },
+	{ 0x3663, 0x70 },
+	{ 0x3664, 0x50 },
+	{ 0x3666, 0x0a },
+	{ 0x3669, 0x1a },
+	{ 0x366a, 0x00 },
+	{ 0x366b, 0x50 },
+	{ 0x3673, 0x01 },
+	{ 0x3674, 0xff },
+	{ 0x3675, 0x03 },
+	{ 0x3705, 0xc1 },
+	{ 0x3709, 0x40 },
+	{ 0x373c, 0x08 },
+	{ 0x3742, 0x00 },
+	{ 0x3757, 0xb3 },
+	{ 0x3788, 0x00 },
+	{ 0x37a8, 0x01 },
+	{ 0x37a9, 0xc0 },
+	{ 0x3800, 0x00 },
+	{ 0x3801, 0x04 },
+	{ 0x3802, 0x00 },
+	{ 0x3803, 0x04 },
+	{ 0x3804, 0x02 },
+	{ 0x3805, 0x8b },
+	{ 0x3806, 0x01 },
+	{ 0x3807, 0xeb },
+	{ 0x3808, 0x02 }, /* width high */
+	{ 0x3809, 0x80 }, /* width low */
+	{ 0x380a, 0x01 }, /* height high */
+	{ 0x380b, 0xe0 }, /* height low */
+	{ 0x380c, 0x03 }, /* total horiz timing high */
+	{ 0x380d, 0xa0 }, /* total horiz timing low */
+	{ 0x380e, 0x02 }, /* total vertical timing high */
+	{ 0x380f, 0x3c }, /* total vertical timing low */
+	{ 0x3810, 0x00 },
+	{ 0x3811, 0x04 },
+	{ 0x3812, 0x00 },
+	{ 0x3813, 0x05 },
+	{ 0x3814, 0x11 },
+	{ 0x3815, 0x11 },
+	{ 0x3820, 0x40 },
+	{ 0x3821, 0x00 },
+	{ 0x382f, 0x0e },
+	{ 0x3832, 0x00 },
+	{ 0x3833, 0x05 },
+	{ 0x3834, 0x00 },
+	{ 0x3835, 0x0c },
+	{ 0x3837, 0x00 },
+	{ 0x3b80, 0x00 },
+	{ 0x3b81, 0xa5 },
+	{ 0x3b82, 0x10 },
+	{ 0x3b83, 0x00 },
+	{ 0x3b84, 0x08 },
+	{ 0x3b85, 0x00 },
+	{ 0x3b86, 0x01 },
+	{ 0x3b87, 0x00 },
+	{ 0x3b88, 0x00 },
+	{ 0x3b89, 0x00 },
+	{ 0x3b8a, 0x00 },
+	{ 0x3b8b, 0x05 },
+	{ 0x3b8c, 0x00 },
+	{ 0x3b8d, 0x00 },
+	{ 0x3b8e, 0x00 },
+	{ 0x3b8f, 0x1a },
+	{ 0x3b94, 0x05 },
+	{ 0x3b95, 0xf2 },
+	{ 0x3b96, 0x40 },
+	{ 0x3c00, 0x89 },
+	{ 0x3c01, 0x63 },
+	{ 0x3c02, 0x01 },
+	{ 0x3c03, 0x00 },
+	{ 0x3c04, 0x00 },
+	{ 0x3c05, 0x03 },
+	{ 0x3c06, 0x00 },
+	{ 0x3c07, 0x06 },
+	{ 0x3c0c, 0x01 },
+	{ 0x3c0d, 0xd0 },
+	{ 0x3c0e, 0x02 },
+	{ 0x3c0f, 0x0a },
+	{ 0x4001, 0x42 },
+	{ 0x4004, 0x04 },
+	{ 0x4005, 0x00 },
+	{ 0x404e, 0x01 },
+	{ 0x4300, 0xff },
+	{ 0x4301, 0x00 },
+	{ 0x4315, 0x00 },
+	{ 0x4501, 0x48 },
+	{ 0x4600, 0x00 },
+	{ 0x4601, 0x4e },
+	{ 0x4801, 0x0f },
+	{ 0x4806, 0x0f },
+	{ 0x4819, 0xaa },
+	{ 0x4823, 0x3e },
+	{ 0x4837, 0x19 },
+	{ 0x4a0d, 0x00 },
+	{ 0x4a47, 0x7f },
+	{ 0x4a49, 0xf0 },
+	{ 0x4a4b, 0x30 },
+	{ 0x5000, 0x85 },
+	{ 0x5001, 0x80 },
+};
+
+static const s64 link_freq[] = {
+	240000000,
+};
+
+static const struct ov7251_mode_info ov7251_mode_info_data[] = {
+	{
+		.width = 640,
+		.height = 480,
+		.data = ov7251_setting_vga_30fps,
+		.data_size = ARRAY_SIZE(ov7251_setting_vga_30fps),
+		.pixel_clock = 48000000,
+		.link_freq = 0, /* an index in link_freq[] */
+		.exposure_max = 1704,
+		.exposure_def = 504,
+		.timeperframe = {
+			.numerator = 100,
+			.denominator = 3000
+		}
+	},
+	{
+		.width = 640,
+		.height = 480,
+		.data = ov7251_setting_vga_60fps,
+		.data_size = ARRAY_SIZE(ov7251_setting_vga_60fps),
+		.pixel_clock = 48000000,
+		.link_freq = 0, /* an index in link_freq[] */
+		.exposure_max = 840,
+		.exposure_def = 504,
+		.timeperframe = {
+			.numerator = 100,
+			.denominator = 6014
+		}
+	},
+	{
+		.width = 640,
+		.height = 480,
+		.data = ov7251_setting_vga_90fps,
+		.data_size = ARRAY_SIZE(ov7251_setting_vga_90fps),
+		.pixel_clock = 48000000,
+		.link_freq = 0, /* an index in link_freq[] */
+		.exposure_max = 552,
+		.exposure_def = 504,
+		.timeperframe = {
+			.numerator = 100,
+			.denominator = 9043
+		}
+	},
+};
+
+static int ov7251_regulators_enable(struct ov7251 *ov7251)
+{
+	int ret;
+
+	/* OV7251 power up sequence requires core regulator
+	 * to be enabled not earlier than io regulator
+	 */
+
+	ret = regulator_enable(ov7251->io_regulator);
+	if (ret < 0) {
+		dev_err(ov7251->dev, "set io voltage failed\n");
+		return ret;
+	}
+
+	ret = regulator_enable(ov7251->analog_regulator);
+	if (ret) {
+		dev_err(ov7251->dev, "set analog voltage failed\n");
+		goto err_disable_io;
+	}
+
+	ret = regulator_enable(ov7251->core_regulator);
+	if (ret) {
+		dev_err(ov7251->dev, "set core voltage failed\n");
+		goto err_disable_analog;
+	}
+
+	return 0;
+
+err_disable_analog:
+	regulator_disable(ov7251->analog_regulator);
+
+err_disable_io:
+	regulator_disable(ov7251->io_regulator);
+
+	return ret;
+}
+
+static void ov7251_regulators_disable(struct ov7251 *ov7251)
+{
+	int ret;
+
+	ret = regulator_disable(ov7251->core_regulator);
+	if (ret < 0)
+		dev_err(ov7251->dev, "core regulator disable failed\n");
+
+	ret = regulator_disable(ov7251->analog_regulator);
+	if (ret < 0)
+		dev_err(ov7251->dev, "analog regulator disable failed\n");
+
+	ret = regulator_disable(ov7251->io_regulator);
+	if (ret < 0)
+		dev_err(ov7251->dev, "io regulator disable failed\n");
+}
+
+static int ov7251_write_reg(struct ov7251 *ov7251, u16 reg, u8 val)
+{
+	u8 regbuf[3];
+	int ret;
+
+	regbuf[0] = reg >> 8;
+	regbuf[1] = reg & 0xff;
+	regbuf[2] = val;
+
+	ret = i2c_master_send(ov7251->i2c_client, regbuf, 3);
+	if (ret < 0) {
+		dev_err(ov7251->dev, "%s: write reg error %d: reg=%x, val=%x\n",
+			__func__, ret, reg, val);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ov7251_write_seq_regs(struct ov7251 *ov7251, u16 reg, u8 *val,
+				 u8 num)
+{
+	u8 regbuf[5];
+	u8 nregbuf = sizeof(reg) + num * sizeof(*val);
+	int ret = 0;
+
+	if (nregbuf > sizeof(regbuf))
+		return -EINVAL;
+
+	regbuf[0] = reg >> 8;
+	regbuf[1] = reg & 0xff;
+
+	memcpy(regbuf + 2, val, num);
+
+	ret = i2c_master_send(ov7251->i2c_client, regbuf, nregbuf);
+	if (ret < 0) {
+		dev_err(ov7251->dev,
+			"%s: write seq regs error %d: first reg=%x\n",
+			__func__, ret, reg);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ov7251_read_reg(struct ov7251 *ov7251, u16 reg, u8 *val)
+{
+	u8 regbuf[2];
+	int ret;
+
+	regbuf[0] = reg >> 8;
+	regbuf[1] = reg & 0xff;
+
+	ret = i2c_master_send(ov7251->i2c_client, regbuf, 2);
+	if (ret < 0) {
+		dev_err(ov7251->dev, "%s: write reg error %d: reg=%x\n",
+			__func__, ret, reg);
+		return ret;
+	}
+
+	ret = i2c_master_recv(ov7251->i2c_client, val, 1);
+	if (ret < 0) {
+		dev_err(ov7251->dev, "%s: read reg error %d: reg=%x\n",
+			__func__, ret, reg);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ov7251_set_exposure(struct ov7251 *ov7251, s32 exposure)
+{
+	u16 reg;
+	u8 val[3];
+
+	reg = OV7251_AEC_EXPO_0;
+	val[0] = (exposure & 0xf000) >> 12; /* goes to OV7251_AEC_EXPO_0 */
+	val[1] = (exposure & 0x0ff0) >> 4;  /* goes to OV7251_AEC_EXPO_1 */
+	val[2] = (exposure & 0x000f) << 4;  /* goes to OV7251_AEC_EXPO_2 */
+
+	return ov7251_write_seq_regs(ov7251, reg, val, 3);
+}
+
+static int ov7251_set_gain(struct ov7251 *ov7251, s32 gain)
+{
+	u16 reg;
+	u8 val[2];
+
+	reg = OV7251_AEC_AGC_ADJ_0;
+	val[0] = (gain & 0x0300) >> 8; /* goes to OV7251_AEC_AGC_ADJ_0 */
+	val[1] = gain & 0xff;          /* goes to OV7251_AEC_AGC_ADJ_1 */
+
+	return ov7251_write_seq_regs(ov7251, reg, val, 2);
+}
+
+static int ov7251_set_register_array(struct ov7251 *ov7251,
+				     const struct reg_value *settings,
+				     unsigned int num_settings)
+{
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i < num_settings; ++i, ++settings) {
+		ret = ov7251_write_reg(ov7251, settings->reg, settings->val);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int ov7251_set_power_on(struct ov7251 *ov7251)
+{
+	int ret;
+	u32 wait_us;
+
+	ret = ov7251_regulators_enable(ov7251);
+	if (ret < 0)
+		return ret;
+
+	ret = clk_prepare_enable(ov7251->xclk);
+	if (ret < 0) {
+		dev_err(ov7251->dev, "clk prepare enable failed\n");
+		ov7251_regulators_disable(ov7251);
+		return ret;
+	}
+
+	gpiod_set_value_cansleep(ov7251->enable_gpio, 1);
+
+	/* wait at least 65536 external clock cycles */
+	wait_us = DIV_ROUND_UP(65536 * 1000,
+			       DIV_ROUND_UP(ov7251->xclk_freq, 1000));
+	usleep_range(wait_us, wait_us + 1000);
+
+	return 0;
+}
+
+static void ov7251_set_power_off(struct ov7251 *ov7251)
+{
+	clk_disable_unprepare(ov7251->xclk);
+	gpiod_set_value_cansleep(ov7251->enable_gpio, 0);
+	ov7251_regulators_disable(ov7251);
+}
+
+static int ov7251_s_power(struct v4l2_subdev *sd, int on)
+{
+	struct ov7251 *ov7251 = to_ov7251(sd);
+	int ret = 0;
+
+	mutex_lock(&ov7251->lock);
+
+	/* If the power state is not modified - no work to do. */
+	if (ov7251->power_on == !!on)
+		goto exit;
+
+	if (on) {
+		ret = ov7251_set_power_on(ov7251);
+		if (ret < 0)
+			goto exit;
+
+		ret = ov7251_set_register_array(ov7251,
+					ov7251_global_init_setting,
+					ARRAY_SIZE(ov7251_global_init_setting));
+		if (ret < 0) {
+			dev_err(ov7251->dev, "could not set init registers\n");
+			ov7251_set_power_off(ov7251);
+			goto exit;
+		}
+
+		ov7251->power_on = true;
+	} else {
+		ov7251_set_power_off(ov7251);
+		ov7251->power_on = false;
+	}
+
+exit:
+	mutex_unlock(&ov7251->lock);
+
+	return ret;
+}
+
+static int ov7251_set_hflip(struct ov7251 *ov7251, s32 value)
+{
+	u8 val = ov7251->timing_format2;
+	int ret;
+
+	if (value)
+		val |= OV7251_TIMING_FORMAT2_MIRROR;
+	else
+		val &= ~OV7251_TIMING_FORMAT2_MIRROR;
+
+	ret = ov7251_write_reg(ov7251, OV7251_TIMING_FORMAT2, val);
+	if (!ret)
+		ov7251->timing_format2 = val;
+
+	return ret;
+}
+
+static int ov7251_set_vflip(struct ov7251 *ov7251, s32 value)
+{
+	u8 val = ov7251->timing_format1;
+	int ret;
+
+	if (value)
+		val |= OV7251_TIMING_FORMAT1_VFLIP;
+	else
+		val &= ~OV7251_TIMING_FORMAT1_VFLIP;
+
+	ret = ov7251_write_reg(ov7251, OV7251_TIMING_FORMAT1, val);
+	if (!ret)
+		ov7251->timing_format1 = val;
+
+	return ret;
+}
+
+static int ov7251_set_test_pattern(struct ov7251 *ov7251, s32 value)
+{
+	u8 val = ov7251->pre_isp_00;
+	int ret;
+
+	if (value)
+		val |= OV7251_PRE_ISP_00_TEST_PATTERN;
+	else
+		val &= ~OV7251_PRE_ISP_00_TEST_PATTERN;
+
+	ret = ov7251_write_reg(ov7251, OV7251_PRE_ISP_00, val);
+	if (!ret)
+		ov7251->pre_isp_00 = val;
+
+	return ret;
+}
+
+static const char * const ov7251_test_pattern_menu[] = {
+	"Disabled",
+	"Vertical Pattern Bars",
+};
+
+static int ov7251_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct ov7251 *ov7251 = container_of(ctrl->handler,
+					     struct ov7251, ctrls);
+	int ret;
+
+	/* v4l2_ctrl_lock() locks our mutex */
+
+	if (!ov7251->power_on)
+		return 0;
+
+	switch (ctrl->id) {
+	case V4L2_CID_EXPOSURE:
+		ret = ov7251_set_exposure(ov7251, ctrl->val);
+		break;
+	case V4L2_CID_GAIN:
+		ret = ov7251_set_gain(ov7251, ctrl->val);
+		break;
+	case V4L2_CID_TEST_PATTERN:
+		ret = ov7251_set_test_pattern(ov7251, ctrl->val);
+		break;
+	case V4L2_CID_HFLIP:
+		ret = ov7251_set_hflip(ov7251, ctrl->val);
+		break;
+	case V4L2_CID_VFLIP:
+		ret = ov7251_set_vflip(ov7251, ctrl->val);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops ov7251_ctrl_ops = {
+	.s_ctrl = ov7251_s_ctrl,
+};
+
+static int ov7251_enum_mbus_code(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->index > 0)
+		return -EINVAL;
+
+	code->code = MEDIA_BUS_FMT_Y10_1X10;
+
+	return 0;
+}
+
+static int ov7251_enum_frame_size(struct v4l2_subdev *subdev,
+				  struct v4l2_subdev_pad_config *cfg,
+				  struct v4l2_subdev_frame_size_enum *fse)
+{
+	if (fse->code != MEDIA_BUS_FMT_Y10_1X10)
+		return -EINVAL;
+
+	if (fse->index >= ARRAY_SIZE(ov7251_mode_info_data))
+		return -EINVAL;
+
+	fse->min_width = ov7251_mode_info_data[fse->index].width;
+	fse->max_width = ov7251_mode_info_data[fse->index].width;
+	fse->min_height = ov7251_mode_info_data[fse->index].height;
+	fse->max_height = ov7251_mode_info_data[fse->index].height;
+
+	return 0;
+}
+
+static int ov7251_enum_frame_ival(struct v4l2_subdev *subdev,
+				  struct v4l2_subdev_pad_config *cfg,
+				  struct v4l2_subdev_frame_interval_enum *fie)
+{
+	unsigned int index = fie->index;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(ov7251_mode_info_data); i++) {
+		if (fie->width != ov7251_mode_info_data[i].width ||
+		    fie->height != ov7251_mode_info_data[i].height)
+			continue;
+
+		if (index-- == 0) {
+			fie->interval = ov7251_mode_info_data[i].timeperframe;
+			return 0;
+		}
+	}
+
+	return -EINVAL;
+}
+
+static struct v4l2_mbus_framefmt *
+__ov7251_get_pad_format(struct ov7251 *ov7251,
+			struct v4l2_subdev_pad_config *cfg,
+			unsigned int pad,
+			enum v4l2_subdev_format_whence which)
+{
+	switch (which) {
+	case V4L2_SUBDEV_FORMAT_TRY:
+		return v4l2_subdev_get_try_format(&ov7251->sd, cfg, pad);
+	case V4L2_SUBDEV_FORMAT_ACTIVE:
+		return &ov7251->fmt;
+	default:
+		return NULL;
+	}
+}
+
+static int ov7251_get_format(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_format *format)
+{
+	struct ov7251 *ov7251 = to_ov7251(sd);
+
+	mutex_lock(&ov7251->lock);
+	format->format = *__ov7251_get_pad_format(ov7251, cfg, format->pad,
+						  format->which);
+	mutex_unlock(&ov7251->lock);
+
+	return 0;
+}
+
+static struct v4l2_rect *
+__ov7251_get_pad_crop(struct ov7251 *ov7251, struct v4l2_subdev_pad_config *cfg,
+		      unsigned int pad, enum v4l2_subdev_format_whence which)
+{
+	switch (which) {
+	case V4L2_SUBDEV_FORMAT_TRY:
+		return v4l2_subdev_get_try_crop(&ov7251->sd, cfg, pad);
+	case V4L2_SUBDEV_FORMAT_ACTIVE:
+		return &ov7251->crop;
+	default:
+		return NULL;
+	}
+}
+
+static inline u32 avg_fps(const struct v4l2_fract *t)
+{
+	return (t->denominator + (t->numerator >> 1)) / t->numerator;
+}
+
+static const struct ov7251_mode_info *
+ov7251_find_mode_by_ival(struct ov7251 *ov7251, struct v4l2_fract *timeperframe)
+{
+	const struct ov7251_mode_info *mode = ov7251->current_mode;
+	unsigned int fps_req = avg_fps(timeperframe);
+	unsigned int max_dist_match = (unsigned int) -1;
+	unsigned int i, n = 0;
+
+	for (i = 0; i < ARRAY_SIZE(ov7251_mode_info_data); i++) {
+		unsigned int dist;
+		unsigned int fps_tmp;
+
+		if (mode->width != ov7251_mode_info_data[i].width ||
+		    mode->height != ov7251_mode_info_data[i].height)
+			continue;
+
+		fps_tmp = avg_fps(&ov7251_mode_info_data[i].timeperframe);
+
+		dist = abs(fps_req - fps_tmp);
+
+		if (dist < max_dist_match) {
+			n = i;
+			max_dist_match = dist;
+		}
+	}
+
+	return &ov7251_mode_info_data[n];
+}
+
+static int ov7251_set_format(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_format *format)
+{
+	struct ov7251 *ov7251 = to_ov7251(sd);
+	struct v4l2_mbus_framefmt *__format;
+	struct v4l2_rect *__crop;
+	const struct ov7251_mode_info *new_mode;
+	int ret = 0;
+
+	mutex_lock(&ov7251->lock);
+
+	__crop = __ov7251_get_pad_crop(ov7251, cfg, format->pad, format->which);
+
+	new_mode = v4l2_find_nearest_size(ov7251_mode_info_data,
+				ARRAY_SIZE(ov7251_mode_info_data),
+				width, height,
+				format->format.width, format->format.height);
+
+	__crop->width = new_mode->width;
+	__crop->height = new_mode->height;
+
+	if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+		ret = __v4l2_ctrl_s_ctrl_int64(ov7251->pixel_clock,
+					       new_mode->pixel_clock);
+		if (ret < 0)
+			goto exit;
+
+		ret = __v4l2_ctrl_s_ctrl(ov7251->link_freq,
+					 new_mode->link_freq);
+		if (ret < 0)
+			goto exit;
+
+		ret = __v4l2_ctrl_modify_range(ov7251->exposure,
+					       1, new_mode->exposure_max,
+					       1, new_mode->exposure_def);
+		if (ret < 0)
+			goto exit;
+
+		ret = __v4l2_ctrl_s_ctrl(ov7251->exposure,
+					 new_mode->exposure_def);
+		if (ret < 0)
+			goto exit;
+
+		ret = __v4l2_ctrl_s_ctrl(ov7251->gain, 16);
+		if (ret < 0)
+			goto exit;
+
+		ov7251->current_mode = new_mode;
+	}
+
+	__format = __ov7251_get_pad_format(ov7251, cfg, format->pad,
+					   format->which);
+	__format->width = __crop->width;
+	__format->height = __crop->height;
+	__format->code = MEDIA_BUS_FMT_Y10_1X10;
+	__format->field = V4L2_FIELD_NONE;
+	__format->colorspace = V4L2_COLORSPACE_SRGB;
+	__format->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(__format->colorspace);
+	__format->quantization = V4L2_MAP_QUANTIZATION_DEFAULT(true,
+				__format->colorspace, __format->ycbcr_enc);
+	__format->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(__format->colorspace);
+
+	format->format = *__format;
+
+exit:
+	mutex_unlock(&ov7251->lock);
+
+	return ret;
+}
+
+static int ov7251_entity_init_cfg(struct v4l2_subdev *subdev,
+				  struct v4l2_subdev_pad_config *cfg)
+{
+	struct v4l2_subdev_format fmt = {
+		.which = cfg ? V4L2_SUBDEV_FORMAT_TRY
+			     : V4L2_SUBDEV_FORMAT_ACTIVE,
+		.format = {
+			.width = 640,
+			.height = 480
+		}
+	};
+
+	ov7251_set_format(subdev, cfg, &fmt);
+
+	return 0;
+}
+
+static int ov7251_get_selection(struct v4l2_subdev *sd,
+				struct v4l2_subdev_pad_config *cfg,
+				struct v4l2_subdev_selection *sel)
+{
+	struct ov7251 *ov7251 = to_ov7251(sd);
+
+	if (sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
+
+	mutex_lock(&ov7251->lock);
+	sel->r = *__ov7251_get_pad_crop(ov7251, cfg, sel->pad,
+					sel->which);
+	mutex_unlock(&ov7251->lock);
+
+	return 0;
+}
+
+static int ov7251_s_stream(struct v4l2_subdev *subdev, int enable)
+{
+	struct ov7251 *ov7251 = to_ov7251(subdev);
+	int ret;
+
+	mutex_lock(&ov7251->lock);
+
+	if (enable) {
+		ret = ov7251_set_register_array(ov7251,
+					ov7251->current_mode->data,
+					ov7251->current_mode->data_size);
+		if (ret < 0) {
+			dev_err(ov7251->dev, "could not set mode %dx%d\n",
+				ov7251->current_mode->width,
+				ov7251->current_mode->height);
+			goto exit;
+		}
+		ret = __v4l2_ctrl_handler_setup(&ov7251->ctrls);
+		if (ret < 0) {
+			dev_err(ov7251->dev, "could not sync v4l2 controls\n");
+			goto exit;
+		}
+		ret = ov7251_write_reg(ov7251, OV7251_SC_MODE_SELECT,
+				       OV7251_SC_MODE_SELECT_STREAMING);
+	} else {
+		ret = ov7251_write_reg(ov7251, OV7251_SC_MODE_SELECT,
+				       OV7251_SC_MODE_SELECT_SW_STANDBY);
+	}
+
+exit:
+	mutex_unlock(&ov7251->lock);
+
+	return ret;
+}
+
+static int ov7251_get_frame_interval(struct v4l2_subdev *subdev,
+				     struct v4l2_subdev_frame_interval *fi)
+{
+	struct ov7251 *ov7251 = to_ov7251(subdev);
+
+	mutex_lock(&ov7251->lock);
+	fi->interval = ov7251->current_mode->timeperframe;
+	mutex_unlock(&ov7251->lock);
+
+	return 0;
+}
+
+static int ov7251_set_frame_interval(struct v4l2_subdev *subdev,
+				     struct v4l2_subdev_frame_interval *fi)
+{
+	struct ov7251 *ov7251 = to_ov7251(subdev);
+	const struct ov7251_mode_info *new_mode;
+	int ret = 0;
+
+	mutex_lock(&ov7251->lock);
+	new_mode = ov7251_find_mode_by_ival(ov7251, &fi->interval);
+
+	if (new_mode != ov7251->current_mode) {
+		ret = __v4l2_ctrl_s_ctrl_int64(ov7251->pixel_clock,
+					       new_mode->pixel_clock);
+		if (ret < 0)
+			goto exit;
+
+		ret = __v4l2_ctrl_s_ctrl(ov7251->link_freq,
+					 new_mode->link_freq);
+		if (ret < 0)
+			goto exit;
+
+		ret = __v4l2_ctrl_modify_range(ov7251->exposure,
+					       1, new_mode->exposure_max,
+					       1, new_mode->exposure_def);
+		if (ret < 0)
+			goto exit;
+
+		ret = __v4l2_ctrl_s_ctrl(ov7251->exposure,
+					 new_mode->exposure_def);
+		if (ret < 0)
+			goto exit;
+
+		ret = __v4l2_ctrl_s_ctrl(ov7251->gain, 16);
+		if (ret < 0)
+			goto exit;
+
+		ov7251->current_mode = new_mode;
+	}
+
+	fi->interval = ov7251->current_mode->timeperframe;
+
+exit:
+	mutex_unlock(&ov7251->lock);
+
+	return ret;
+}
+
+static const struct v4l2_subdev_core_ops ov7251_core_ops = {
+	.s_power = ov7251_s_power,
+};
+
+static const struct v4l2_subdev_video_ops ov7251_video_ops = {
+	.s_stream = ov7251_s_stream,
+	.g_frame_interval = ov7251_get_frame_interval,
+	.s_frame_interval = ov7251_set_frame_interval,
+};
+
+static const struct v4l2_subdev_pad_ops ov7251_subdev_pad_ops = {
+	.init_cfg = ov7251_entity_init_cfg,
+	.enum_mbus_code = ov7251_enum_mbus_code,
+	.enum_frame_size = ov7251_enum_frame_size,
+	.enum_frame_interval = ov7251_enum_frame_ival,
+	.get_fmt = ov7251_get_format,
+	.set_fmt = ov7251_set_format,
+	.get_selection = ov7251_get_selection,
+};
+
+static const struct v4l2_subdev_ops ov7251_subdev_ops = {
+	.core = &ov7251_core_ops,
+	.video = &ov7251_video_ops,
+	.pad = &ov7251_subdev_pad_ops,
+};
+
+static int ov7251_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct fwnode_handle *endpoint;
+	struct ov7251 *ov7251;
+	u8 chip_id_high, chip_id_low, chip_rev;
+	int ret;
+
+	ov7251 = devm_kzalloc(dev, sizeof(struct ov7251), GFP_KERNEL);
+	if (!ov7251)
+		return -ENOMEM;
+
+	ov7251->i2c_client = client;
+	ov7251->dev = dev;
+
+	endpoint = fwnode_graph_get_next_endpoint(dev_fwnode(dev), NULL);
+	if (!endpoint) {
+		dev_err(dev, "endpoint node not found\n");
+		return -EINVAL;
+	}
+
+	ret = v4l2_fwnode_endpoint_parse(endpoint, &ov7251->ep);
+	fwnode_handle_put(endpoint);
+	if (ret < 0) {
+		dev_err(dev, "parsing endpoint node failed\n");
+		return ret;
+	}
+
+	if (ov7251->ep.bus_type != V4L2_MBUS_CSI2) {
+		dev_err(dev, "invalid bus type (%u), must be CSI2 (%u)\n",
+			ov7251->ep.bus_type, V4L2_MBUS_CSI2);
+		return -EINVAL;
+	}
+
+	/* get system clock (xclk) */
+	ov7251->xclk = devm_clk_get(dev, "xclk");
+	if (IS_ERR(ov7251->xclk)) {
+		dev_err(dev, "could not get xclk");
+		return PTR_ERR(ov7251->xclk);
+	}
+
+	ret = fwnode_property_read_u32(dev_fwnode(dev), "clock-frequency",
+				       &ov7251->xclk_freq);
+	if (ret) {
+		dev_err(dev, "could not get xclk frequency\n");
+		return ret;
+	}
+
+	/* external clock must be 24MHz, allow 1% tolerance */
+	if (ov7251->xclk_freq < 23760000 || ov7251->xclk_freq > 24240000) {
+		dev_err(dev, "external clock frequency %u is not supported\n",
+			ov7251->xclk_freq);
+		return -EINVAL;
+	}
+
+	ret = clk_set_rate(ov7251->xclk, ov7251->xclk_freq);
+	if (ret) {
+		dev_err(dev, "could not set xclk frequency\n");
+		return ret;
+	}
+
+	ov7251->io_regulator = devm_regulator_get(dev, "vdddo");
+	if (IS_ERR(ov7251->io_regulator)) {
+		dev_err(dev, "cannot get io regulator\n");
+		return PTR_ERR(ov7251->io_regulator);
+	}
+
+	ov7251->core_regulator = devm_regulator_get(dev, "vddd");
+	if (IS_ERR(ov7251->core_regulator)) {
+		dev_err(dev, "cannot get core regulator\n");
+		return PTR_ERR(ov7251->core_regulator);
+	}
+
+	ov7251->analog_regulator = devm_regulator_get(dev, "vdda");
+	if (IS_ERR(ov7251->analog_regulator)) {
+		dev_err(dev, "cannot get analog regulator\n");
+		return PTR_ERR(ov7251->analog_regulator);
+	}
+
+	ov7251->enable_gpio = devm_gpiod_get(dev, "enable", GPIOD_OUT_HIGH);
+	if (IS_ERR(ov7251->enable_gpio)) {
+		dev_err(dev, "cannot get enable gpio\n");
+		return PTR_ERR(ov7251->enable_gpio);
+	}
+
+	mutex_init(&ov7251->lock);
+
+	v4l2_ctrl_handler_init(&ov7251->ctrls, 7);
+	ov7251->ctrls.lock = &ov7251->lock;
+
+	v4l2_ctrl_new_std(&ov7251->ctrls, &ov7251_ctrl_ops,
+			  V4L2_CID_HFLIP, 0, 1, 1, 0);
+	v4l2_ctrl_new_std(&ov7251->ctrls, &ov7251_ctrl_ops,
+			  V4L2_CID_VFLIP, 0, 1, 1, 0);
+	ov7251->exposure = v4l2_ctrl_new_std(&ov7251->ctrls, &ov7251_ctrl_ops,
+					     V4L2_CID_EXPOSURE, 1, 32, 1, 32);
+	ov7251->gain = v4l2_ctrl_new_std(&ov7251->ctrls, &ov7251_ctrl_ops,
+					 V4L2_CID_GAIN, 16, 1023, 1, 16);
+	v4l2_ctrl_new_std_menu_items(&ov7251->ctrls, &ov7251_ctrl_ops,
+				     V4L2_CID_TEST_PATTERN,
+				     ARRAY_SIZE(ov7251_test_pattern_menu) - 1,
+				     0, 0, ov7251_test_pattern_menu);
+	ov7251->pixel_clock = v4l2_ctrl_new_std(&ov7251->ctrls,
+						&ov7251_ctrl_ops,
+						V4L2_CID_PIXEL_RATE,
+						1, INT_MAX, 1, 1);
+	ov7251->link_freq = v4l2_ctrl_new_int_menu(&ov7251->ctrls,
+						   &ov7251_ctrl_ops,
+						   V4L2_CID_LINK_FREQ,
+						   ARRAY_SIZE(link_freq) - 1,
+						   0, link_freq);
+	if (ov7251->link_freq)
+		ov7251->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	ov7251->sd.ctrl_handler = &ov7251->ctrls;
+
+	if (ov7251->ctrls.error) {
+		dev_err(dev, "%s: control initialization error %d\n",
+			__func__, ov7251->ctrls.error);
+		ret = ov7251->ctrls.error;
+		goto free_ctrl;
+	}
+
+	v4l2_i2c_subdev_init(&ov7251->sd, client, &ov7251_subdev_ops);
+	ov7251->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	ov7251->pad.flags = MEDIA_PAD_FL_SOURCE;
+	ov7251->sd.dev = &client->dev;
+	ov7251->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
+
+	ret = media_entity_pads_init(&ov7251->sd.entity, 1, &ov7251->pad);
+	if (ret < 0) {
+		dev_err(dev, "could not register media entity\n");
+		goto free_ctrl;
+	}
+
+	ret = ov7251_s_power(&ov7251->sd, true);
+	if (ret < 0) {
+		dev_err(dev, "could not power up OV7251\n");
+		goto free_entity;
+	}
+
+	ret = ov7251_read_reg(ov7251, OV7251_CHIP_ID_HIGH, &chip_id_high);
+	if (ret < 0 || chip_id_high != OV7251_CHIP_ID_HIGH_BYTE) {
+		dev_err(dev, "could not read ID high\n");
+		ret = -ENODEV;
+		goto power_down;
+	}
+	ret = ov7251_read_reg(ov7251, OV7251_CHIP_ID_LOW, &chip_id_low);
+	if (ret < 0 || chip_id_low != OV7251_CHIP_ID_LOW_BYTE) {
+		dev_err(dev, "could not read ID low\n");
+		ret = -ENODEV;
+		goto power_down;
+	}
+
+	ret = ov7251_read_reg(ov7251, OV7251_SC_GP_IO_IN1, &chip_rev);
+	if (ret < 0) {
+		dev_err(dev, "could not read revision\n");
+		ret = -ENODEV;
+		goto power_down;
+	}
+	chip_rev >>= 4;
+
+	dev_info(dev, "OV7251 revision %x (%s) detected at address 0x%02x\n",
+		 chip_rev,
+		 chip_rev == 0x4 ? "1A / 1B" :
+		 chip_rev == 0x5 ? "1C / 1D" :
+		 chip_rev == 0x6 ? "1E" :
+		 chip_rev == 0x7 ? "1F" : "unknown",
+		 client->addr);
+
+	ret = ov7251_read_reg(ov7251, OV7251_PRE_ISP_00,
+			      &ov7251->pre_isp_00);
+	if (ret < 0) {
+		dev_err(dev, "could not read test pattern value\n");
+		ret = -ENODEV;
+		goto power_down;
+	}
+
+	ret = ov7251_read_reg(ov7251, OV7251_TIMING_FORMAT1,
+			      &ov7251->timing_format1);
+	if (ret < 0) {
+		dev_err(dev, "could not read vflip value\n");
+		ret = -ENODEV;
+		goto power_down;
+	}
+
+	ret = ov7251_read_reg(ov7251, OV7251_TIMING_FORMAT2,
+			      &ov7251->timing_format2);
+	if (ret < 0) {
+		dev_err(dev, "could not read hflip value\n");
+		ret = -ENODEV;
+		goto power_down;
+	}
+
+	ov7251_s_power(&ov7251->sd, false);
+
+	ret = v4l2_async_register_subdev(&ov7251->sd);
+	if (ret < 0) {
+		dev_err(dev, "could not register v4l2 device\n");
+		goto free_entity;
+	}
+
+	ov7251_entity_init_cfg(&ov7251->sd, NULL);
+
+	return 0;
+
+power_down:
+	ov7251_s_power(&ov7251->sd, false);
+free_entity:
+	media_entity_cleanup(&ov7251->sd.entity);
+free_ctrl:
+	v4l2_ctrl_handler_free(&ov7251->ctrls);
+	mutex_destroy(&ov7251->lock);
+
+	return ret;
+}
+
+static int ov7251_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct ov7251 *ov7251 = to_ov7251(sd);
+
+	v4l2_async_unregister_subdev(&ov7251->sd);
+	media_entity_cleanup(&ov7251->sd.entity);
+	v4l2_ctrl_handler_free(&ov7251->ctrls);
+	mutex_destroy(&ov7251->lock);
+
+	return 0;
+}
+
+static const struct of_device_id ov7251_of_match[] = {
+	{ .compatible = "ovti,ov7251" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, ov7251_of_match);
+
+static struct i2c_driver ov7251_i2c_driver = {
+	.driver = {
+		.of_match_table = ov7251_of_match,
+		.name  = "ov7251",
+	},
+	.probe_new  = ov7251_probe,
+	.remove = ov7251_remove,
+};
+
+module_i2c_driver(ov7251_i2c_driver);
+
+MODULE_DESCRIPTION("Omnivision OV7251 Camera Driver");
+MODULE_AUTHOR("Todor Tomov <todor.tomov@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/i2c/ov772x.c b/drivers/media/i2c/ov772x.c
index b62860c89439..e2550708abc8 100644
--- a/drivers/media/i2c/ov772x.c
+++ b/drivers/media/i2c/ov772x.c
@@ -1035,7 +1035,7 @@ static int ov772x_set_params(struct ov772x_priv *priv,
 
 	/* Set COM8. */
 	if (priv->band_filter) {
-		ret = ov772x_mask_set(client, COM8, BNDF_ON_OFF, 1);
+		ret = ov772x_mask_set(client, COM8, BNDF_ON_OFF, BNDF_ON_OFF);
 		if (!ret)
 			ret = ov772x_mask_set(client, BDBASE,
 					      0xff, 256 - priv->band_filter);
diff --git a/drivers/media/i2c/ov7740.c b/drivers/media/i2c/ov7740.c
index 01f578785e79..605f3e25ad82 100644
--- a/drivers/media/i2c/ov7740.c
+++ b/drivers/media/i2c/ov7740.c
@@ -953,7 +953,7 @@ static int ov7740_init_controls(struct ov7740 *ov7740)
 	struct v4l2_ctrl_handler *ctrl_hdlr = &ov7740->ctrl_handler;
 	int ret;
 
-	ret = v4l2_ctrl_handler_init(ctrl_hdlr, 2);
+	ret = v4l2_ctrl_handler_init(ctrl_hdlr, 12);
 	if (ret < 0)
 		return ret;
 
@@ -980,27 +980,39 @@ static int ov7740_init_controls(struct ov7740 *ov7740)
 					V4L2_CID_HFLIP, 0, 1, 1, 0);
 	ov7740->vflip = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
 					V4L2_CID_VFLIP, 0, 1, 1, 0);
+
 	ov7740->gain = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
 				       V4L2_CID_GAIN, 0, 1023, 1, 500);
+	if (ov7740->gain)
+		ov7740->gain->flags |= V4L2_CTRL_FLAG_VOLATILE;
+
 	ov7740->auto_gain = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
 					    V4L2_CID_AUTOGAIN, 0, 1, 1, 1);
+
 	ov7740->exposure = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
 					   V4L2_CID_EXPOSURE, 0, 65535, 1, 500);
+	if (ov7740->exposure)
+		ov7740->exposure->flags |= V4L2_CTRL_FLAG_VOLATILE;
+
 	ov7740->auto_exposure = v4l2_ctrl_new_std_menu(ctrl_hdlr,
 					&ov7740_ctrl_ops,
 					V4L2_CID_EXPOSURE_AUTO,
 					V4L2_EXPOSURE_MANUAL, 0,
 					V4L2_EXPOSURE_AUTO);
 
-	ov7740->gain->flags |= V4L2_CTRL_FLAG_VOLATILE;
-	ov7740->exposure->flags |= V4L2_CTRL_FLAG_VOLATILE;
-
 	v4l2_ctrl_auto_cluster(3, &ov7740->auto_wb, 0, false);
 	v4l2_ctrl_auto_cluster(2, &ov7740->auto_gain, 0, true);
 	v4l2_ctrl_auto_cluster(2, &ov7740->auto_exposure,
 			       V4L2_EXPOSURE_MANUAL, false);
 	v4l2_ctrl_cluster(2, &ov7740->hflip);
 
+	if (ctrl_hdlr->error) {
+		ret = ctrl_hdlr->error;
+		dev_err(&client->dev, "controls initialisation failed (%d)\n",
+			ret);
+		goto error;
+	}
+
 	ret = v4l2_ctrl_handler_setup(ctrl_hdlr);
 	if (ret) {
 		dev_err(&client->dev, "%s control init failed (%d)\n",
@@ -1074,7 +1086,7 @@ static int ov7740_probe(struct i2c_client *client,
 
 #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
 	sd->internal_ops = &ov7740_subdev_internal_ops;
-	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS;
 #endif
 
 #if defined(CONFIG_MEDIA_CONTROLLER)
diff --git a/drivers/media/i2c/smiapp/smiapp-core.c b/drivers/media/i2c/smiapp/smiapp-core.c
index 3b7ace395ee6..e1f8208581aa 100644
--- a/drivers/media/i2c/smiapp/smiapp-core.c
+++ b/drivers/media/i2c/smiapp/smiapp-core.c
@@ -1001,7 +1001,7 @@ static int smiapp_read_nvm(struct smiapp_sensor *sensor,
 		if (rval)
 			goto out;
 
-		for (i = 0; i < 1000; i++) {
+		for (i = 1000; i > 0; i--) {
 			rval = smiapp_read(
 				sensor,
 				SMIAPP_REG_U8_DATA_TRANSFER_IF_1_STATUS, &s);
@@ -1012,11 +1012,10 @@ static int smiapp_read_nvm(struct smiapp_sensor *sensor,
 			if (s & SMIAPP_DATA_TRANSFER_IF_1_STATUS_RD_READY)
 				break;
 
-			if (--i == 0) {
-				rval = -ETIMEDOUT;
-				goto out;
-			}
-
+		}
+		if (!i) {
+			rval = -ETIMEDOUT;
+			goto out;
 		}
 
 		for (i = 0; i < SMIAPP_NVM_PAGE_SIZE; i++) {
diff --git a/drivers/media/i2c/tda1997x.c b/drivers/media/i2c/tda1997x.c
index 3021913c28fa..039a92c3294a 100644
--- a/drivers/media/i2c/tda1997x.c
+++ b/drivers/media/i2c/tda1997x.c
@@ -2444,7 +2444,7 @@ static int tda1997x_pcm_startup(struct snd_pcm_substream *substream,
 				struct snd_soc_dai *dai)
 {
 	struct tda1997x_state *state = snd_soc_dai_get_drvdata(dai);
-	struct snd_soc_codec *codec = dai->codec;
+	struct snd_soc_component *component = dai->component;
 	struct snd_pcm_runtime *rtd = substream->runtime;
 	int rate, err;
 
@@ -2452,11 +2452,11 @@ static int tda1997x_pcm_startup(struct snd_pcm_substream *substream,
 	err = snd_pcm_hw_constraint_minmax(rtd, SNDRV_PCM_HW_PARAM_RATE,
 					   rate, rate);
 	if (err < 0) {
-		dev_err(codec->dev, "failed to constrain samplerate to %dHz\n",
+		dev_err(component->dev, "failed to constrain samplerate to %dHz\n",
 			rate);
 		return err;
 	}
-	dev_info(codec->dev, "set samplerate constraint to %dHz\n", rate);
+	dev_info(component->dev, "set samplerate constraint to %dHz\n", rate);
 
 	return 0;
 }
@@ -2479,20 +2479,22 @@ static struct snd_soc_dai_driver tda1997x_audio_dai = {
 	.ops = &tda1997x_dai_ops,
 };
 
-static int tda1997x_codec_probe(struct snd_soc_codec *codec)
+static int tda1997x_codec_probe(struct snd_soc_component *component)
 {
 	return 0;
 }
 
-static int tda1997x_codec_remove(struct snd_soc_codec *codec)
+static void tda1997x_codec_remove(struct snd_soc_component *component)
 {
-	return 0;
 }
 
-static struct snd_soc_codec_driver tda1997x_codec_driver = {
-	.probe = tda1997x_codec_probe,
-	.remove = tda1997x_codec_remove,
-	.reg_word_size = sizeof(u16),
+static struct snd_soc_component_driver tda1997x_codec_driver = {
+	.probe			= tda1997x_codec_probe,
+	.remove			= tda1997x_codec_remove,
+	.idle_bias_on		= 1,
+	.use_pmdown_time	= 1,
+	.endianness		= 1,
+	.non_legacy_dai_naming	= 1,
 };
 
 static int tda1997x_probe(struct i2c_client *client,
@@ -2567,7 +2569,7 @@ static int tda1997x_probe(struct i2c_client *client,
 	snprintf(sd->name, sizeof(sd->name), "%s %d-%04x",
 		 id->name, i2c_adapter_id(client->adapter),
 		 client->addr);
-	sd->flags = V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS;
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS;
 	sd->entity.function = MEDIA_ENT_F_DTV_DECODER;
 	sd->entity.ops = &tda1997x_media_ops;
 
@@ -2721,7 +2723,7 @@ static int tda1997x_probe(struct i2c_client *client,
 		state->pads);
 	if (ret) {
 		v4l_err(client, "failed entity_init: %d", ret);
-		goto err_free_mutex;
+		goto err_free_handler;
 	}
 
 	ret = v4l2_async_register_subdev(sd);
@@ -2737,7 +2739,7 @@ static int tda1997x_probe(struct i2c_client *client,
 		else
 			formats = SNDRV_PCM_FMTBIT_S16_LE;
 		tda1997x_audio_dai.capture.formats = formats;
-		ret = snd_soc_register_codec(&state->client->dev,
+		ret = devm_snd_soc_register_component(&state->client->dev,
 					     &tda1997x_codec_driver,
 					     &tda1997x_audio_dai, 1);
 		if (ret) {
@@ -2782,7 +2784,6 @@ static int tda1997x_remove(struct i2c_client *client)
 	struct tda1997x_platform_data *pdata = &state->pdata;
 
 	if (pdata->audout_format) {
-		snd_soc_unregister_codec(&client->dev);
 		mutex_destroy(&state->audio_lock);
 	}
 
diff --git a/drivers/media/i2c/tvp5150.c b/drivers/media/i2c/tvp5150.c
index 1734ed4ede33..b162c2fe62c3 100644
--- a/drivers/media/i2c/tvp5150.c
+++ b/drivers/media/i2c/tvp5150.c
@@ -319,136 +319,136 @@ struct i2c_reg_value {
 /* Default values as sugested at TVP5150AM1 datasheet */
 static const struct i2c_reg_value tvp5150_init_default[] = {
 	{ /* 0x00 */
-		TVP5150_VD_IN_SRC_SEL_1,0x00
+		TVP5150_VD_IN_SRC_SEL_1, 0x00
 	},
 	{ /* 0x01 */
-		TVP5150_ANAL_CHL_CTL,0x15
+		TVP5150_ANAL_CHL_CTL, 0x15
 	},
 	{ /* 0x02 */
-		TVP5150_OP_MODE_CTL,0x00
+		TVP5150_OP_MODE_CTL, 0x00
 	},
 	{ /* 0x03 */
-		TVP5150_MISC_CTL,0x01
+		TVP5150_MISC_CTL, 0x01
 	},
 	{ /* 0x06 */
-		TVP5150_COLOR_KIL_THSH_CTL,0x10
+		TVP5150_COLOR_KIL_THSH_CTL, 0x10
 	},
 	{ /* 0x07 */
-		TVP5150_LUMA_PROC_CTL_1,0x60
+		TVP5150_LUMA_PROC_CTL_1, 0x60
 	},
 	{ /* 0x08 */
-		TVP5150_LUMA_PROC_CTL_2,0x00
+		TVP5150_LUMA_PROC_CTL_2, 0x00
 	},
 	{ /* 0x09 */
-		TVP5150_BRIGHT_CTL,0x80
+		TVP5150_BRIGHT_CTL, 0x80
 	},
 	{ /* 0x0a */
-		TVP5150_SATURATION_CTL,0x80
+		TVP5150_SATURATION_CTL, 0x80
 	},
 	{ /* 0x0b */
-		TVP5150_HUE_CTL,0x00
+		TVP5150_HUE_CTL, 0x00
 	},
 	{ /* 0x0c */
-		TVP5150_CONTRAST_CTL,0x80
+		TVP5150_CONTRAST_CTL, 0x80
 	},
 	{ /* 0x0d */
-		TVP5150_DATA_RATE_SEL,0x47
+		TVP5150_DATA_RATE_SEL, 0x47
 	},
 	{ /* 0x0e */
-		TVP5150_LUMA_PROC_CTL_3,0x00
+		TVP5150_LUMA_PROC_CTL_3, 0x00
 	},
 	{ /* 0x0f */
-		TVP5150_CONF_SHARED_PIN,0x08
+		TVP5150_CONF_SHARED_PIN, 0x08
 	},
 	{ /* 0x11 */
-		TVP5150_ACT_VD_CROP_ST_MSB,0x00
+		TVP5150_ACT_VD_CROP_ST_MSB, 0x00
 	},
 	{ /* 0x12 */
-		TVP5150_ACT_VD_CROP_ST_LSB,0x00
+		TVP5150_ACT_VD_CROP_ST_LSB, 0x00
 	},
 	{ /* 0x13 */
-		TVP5150_ACT_VD_CROP_STP_MSB,0x00
+		TVP5150_ACT_VD_CROP_STP_MSB, 0x00
 	},
 	{ /* 0x14 */
-		TVP5150_ACT_VD_CROP_STP_LSB,0x00
+		TVP5150_ACT_VD_CROP_STP_LSB, 0x00
 	},
 	{ /* 0x15 */
-		TVP5150_GENLOCK,0x01
+		TVP5150_GENLOCK, 0x01
 	},
 	{ /* 0x16 */
-		TVP5150_HORIZ_SYNC_START,0x80
+		TVP5150_HORIZ_SYNC_START, 0x80
 	},
 	{ /* 0x18 */
-		TVP5150_VERT_BLANKING_START,0x00
+		TVP5150_VERT_BLANKING_START, 0x00
 	},
 	{ /* 0x19 */
-		TVP5150_VERT_BLANKING_STOP,0x00
+		TVP5150_VERT_BLANKING_STOP, 0x00
 	},
 	{ /* 0x1a */
-		TVP5150_CHROMA_PROC_CTL_1,0x0c
+		TVP5150_CHROMA_PROC_CTL_1, 0x0c
 	},
 	{ /* 0x1b */
-		TVP5150_CHROMA_PROC_CTL_2,0x14
+		TVP5150_CHROMA_PROC_CTL_2, 0x14
 	},
 	{ /* 0x1c */
-		TVP5150_INT_RESET_REG_B,0x00
+		TVP5150_INT_RESET_REG_B, 0x00
 	},
 	{ /* 0x1d */
-		TVP5150_INT_ENABLE_REG_B,0x00
+		TVP5150_INT_ENABLE_REG_B, 0x00
 	},
 	{ /* 0x1e */
-		TVP5150_INTT_CONFIG_REG_B,0x00
+		TVP5150_INTT_CONFIG_REG_B, 0x00
 	},
 	{ /* 0x28 */
-		TVP5150_VIDEO_STD,0x00
+		TVP5150_VIDEO_STD, 0x00
 	},
 	{ /* 0x2e */
-		TVP5150_MACROVISION_ON_CTR,0x0f
+		TVP5150_MACROVISION_ON_CTR, 0x0f
 	},
 	{ /* 0x2f */
-		TVP5150_MACROVISION_OFF_CTR,0x01
+		TVP5150_MACROVISION_OFF_CTR, 0x01
 	},
 	{ /* 0xbb */
-		TVP5150_TELETEXT_FIL_ENA,0x00
+		TVP5150_TELETEXT_FIL_ENA, 0x00
 	},
 	{ /* 0xc0 */
-		TVP5150_INT_STATUS_REG_A,0x00
+		TVP5150_INT_STATUS_REG_A, 0x00
 	},
 	{ /* 0xc1 */
-		TVP5150_INT_ENABLE_REG_A,0x00
+		TVP5150_INT_ENABLE_REG_A, 0x00
 	},
 	{ /* 0xc2 */
-		TVP5150_INT_CONF,0x04
+		TVP5150_INT_CONF, 0x04
 	},
 	{ /* 0xc8 */
-		TVP5150_FIFO_INT_THRESHOLD,0x80
+		TVP5150_FIFO_INT_THRESHOLD, 0x80
 	},
 	{ /* 0xc9 */
-		TVP5150_FIFO_RESET,0x00
+		TVP5150_FIFO_RESET, 0x00
 	},
 	{ /* 0xca */
-		TVP5150_LINE_NUMBER_INT,0x00
+		TVP5150_LINE_NUMBER_INT, 0x00
 	},
 	{ /* 0xcb */
-		TVP5150_PIX_ALIGN_REG_LOW,0x4e
+		TVP5150_PIX_ALIGN_REG_LOW, 0x4e
 	},
 	{ /* 0xcc */
-		TVP5150_PIX_ALIGN_REG_HIGH,0x00
+		TVP5150_PIX_ALIGN_REG_HIGH, 0x00
 	},
 	{ /* 0xcd */
-		TVP5150_FIFO_OUT_CTRL,0x01
+		TVP5150_FIFO_OUT_CTRL, 0x01
 	},
 	{ /* 0xcf */
-		TVP5150_FULL_FIELD_ENA,0x00
+		TVP5150_FULL_FIELD_ENA, 0x00
 	},
 	{ /* 0xd0 */
-		TVP5150_LINE_MODE_INI,0x00
+		TVP5150_LINE_MODE_INI, 0x00
 	},
 	{ /* 0xfc */
-		TVP5150_FULL_FIELD_MODE_REG,0x7f
+		TVP5150_FULL_FIELD_MODE_REG, 0x7f
 	},
 	{ /* end of data */
-		0xff,0xff
+		0xff, 0xff
 	}
 };
 
@@ -456,27 +456,27 @@ static const struct i2c_reg_value tvp5150_init_default[] = {
 static const struct i2c_reg_value tvp5150_init_enable[] = {
 	{
 		TVP5150_CONF_SHARED_PIN, 2
-	},{	/* Automatic offset and AGC enabled */
+	}, {	/* Automatic offset and AGC enabled */
 		TVP5150_ANAL_CHL_CTL, 0x15
-	},{	/* Activate YCrCb output 0x9 or 0xd ? */
+	}, {	/* Activate YCrCb output 0x9 or 0xd ? */
 		TVP5150_MISC_CTL, TVP5150_MISC_CTL_GPCL |
 				  TVP5150_MISC_CTL_INTREQ_OE |
 				  TVP5150_MISC_CTL_YCBCR_OE |
 				  TVP5150_MISC_CTL_SYNC_OE |
 				  TVP5150_MISC_CTL_VBLANK |
 				  TVP5150_MISC_CTL_CLOCK_OE,
-	},{	/* Activates video std autodetection for all standards */
+	}, {	/* Activates video std autodetection for all standards */
 		TVP5150_AUTOSW_MSK, 0x0
-	},{	/* Default format: 0x47. For 4:2:2: 0x40 */
+	}, {	/* Default format: 0x47. For 4:2:2: 0x40 */
 		TVP5150_DATA_RATE_SEL, 0x47
-	},{
+	}, {
 		TVP5150_CHROMA_PROC_CTL_1, 0x0c
-	},{
+	}, {
 		TVP5150_CHROMA_PROC_CTL_2, 0x54
-	},{	/* Non documented, but initialized on WinTV USB2 */
+	}, {	/* Non documented, but initialized on WinTV USB2 */
 		0x27, 0x20
-	},{
-		0xff,0xff
+	}, {
+		0xff, 0xff
 	}
 };
 
@@ -500,78 +500,80 @@ struct i2c_vbi_ram_value {
  * and so on. There are 16 possible locations from 0 to 15.
  */
 
-static struct i2c_vbi_ram_value vbi_ram_default[] =
-{
-	/* FIXME: Current api doesn't handle all VBI types, those not
-	   yet supported are placed under #if 0 */
+static struct i2c_vbi_ram_value vbi_ram_default[] = {
+
+	/*
+	 * FIXME: Current api doesn't handle all VBI types, those not
+	 * yet supported are placed under #if 0
+	 */
 #if 0
 	[0] = {0x010, /* Teletext, SECAM, WST System A */
-		{V4L2_SLICED_TELETEXT_SECAM,6,23,1},
+		{V4L2_SLICED_TELETEXT_SECAM, 6, 23, 1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0xe7, 0x2e, 0x20, 0x26,
 		  0xe6, 0xb4, 0x0e, 0x00, 0x00, 0x00, 0x10, 0x00 }
 	},
 #endif
 	[1] = {0x030, /* Teletext, PAL, WST System B */
-		{V4L2_SLICED_TELETEXT_B,6,22,1},
+		{V4L2_SLICED_TELETEXT_B, 6, 22, 1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0x27, 0x2e, 0x20, 0x2b,
 		  0xa6, 0x72, 0x10, 0x00, 0x00, 0x00, 0x10, 0x00 }
 	},
 #if 0
 	[2] = {0x050, /* Teletext, PAL, WST System C */
-		{V4L2_SLICED_TELETEXT_PAL_C,6,22,1},
+		{V4L2_SLICED_TELETEXT_PAL_C, 6, 22, 1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0xe7, 0x2e, 0x20, 0x22,
 		  0xa6, 0x98, 0x0d, 0x00, 0x00, 0x00, 0x10, 0x00 }
 	},
 	[3] = {0x070, /* Teletext, NTSC, WST System B */
-		{V4L2_SLICED_TELETEXT_NTSC_B,10,21,1},
+		{V4L2_SLICED_TELETEXT_NTSC_B, 10, 21, 1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0x27, 0x2e, 0x20, 0x23,
 		  0x69, 0x93, 0x0d, 0x00, 0x00, 0x00, 0x10, 0x00 }
 	},
 	[4] = {0x090, /* Tetetext, NTSC NABTS System C */
-		{V4L2_SLICED_TELETEXT_NTSC_C,10,21,1},
+		{V4L2_SLICED_TELETEXT_NTSC_C, 10, 21, 1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0xe7, 0x2e, 0x20, 0x22,
 		  0x69, 0x93, 0x0d, 0x00, 0x00, 0x00, 0x15, 0x00 }
 	},
 	[5] = {0x0b0, /* Teletext, NTSC-J, NABTS System D */
-		{V4L2_SLICED_TELETEXT_NTSC_D,10,21,1},
+		{V4L2_SLICED_TELETEXT_NTSC_D, 10, 21, 1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0xa7, 0x2e, 0x20, 0x23,
 		  0x69, 0x93, 0x0d, 0x00, 0x00, 0x00, 0x10, 0x00 }
 	},
 	[6] = {0x0d0, /* Closed Caption, PAL/SECAM */
-		{V4L2_SLICED_CAPTION_625,22,22,1},
+		{V4L2_SLICED_CAPTION_625, 22, 22, 1},
 		{ 0xaa, 0x2a, 0xff, 0x3f, 0x04, 0x51, 0x6e, 0x02,
 		  0xa6, 0x7b, 0x09, 0x00, 0x00, 0x00, 0x27, 0x00 }
 	},
 #endif
 	[7] = {0x0f0, /* Closed Caption, NTSC */
-		{V4L2_SLICED_CAPTION_525,21,21,1},
+		{V4L2_SLICED_CAPTION_525, 21, 21, 1},
 		{ 0xaa, 0x2a, 0xff, 0x3f, 0x04, 0x51, 0x6e, 0x02,
 		  0x69, 0x8c, 0x09, 0x00, 0x00, 0x00, 0x27, 0x00 }
 	},
 	[8] = {0x110, /* Wide Screen Signal, PAL/SECAM */
-		{V4L2_SLICED_WSS_625,23,23,1},
+		{V4L2_SLICED_WSS_625, 23, 23, 1},
 		{ 0x5b, 0x55, 0xc5, 0xff, 0x00, 0x71, 0x6e, 0x42,
 		  0xa6, 0xcd, 0x0f, 0x00, 0x00, 0x00, 0x3a, 0x00 }
 	},
 #if 0
 	[9] = {0x130, /* Wide Screen Signal, NTSC C */
-		{V4L2_SLICED_WSS_525,20,20,1},
+		{V4L2_SLICED_WSS_525, 20, 20, 1},
 		{ 0x38, 0x00, 0x3f, 0x00, 0x00, 0x71, 0x6e, 0x43,
 		  0x69, 0x7c, 0x08, 0x00, 0x00, 0x00, 0x39, 0x00 }
 	},
 	[10] = {0x150, /* Vertical Interval Timecode (VITC), PAL/SECAM */
-		{V4l2_SLICED_VITC_625,6,22,0},
+		{V4l2_SLICED_VITC_625, 6, 22, 0},
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x6d, 0x49,
 		  0xa6, 0x85, 0x08, 0x00, 0x00, 0x00, 0x4c, 0x00 }
 	},
 	[11] = {0x170, /* Vertical Interval Timecode (VITC), NTSC */
-		{V4l2_SLICED_VITC_525,10,20,0},
+		{V4l2_SLICED_VITC_525, 10, 20, 0},
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x6d, 0x49,
 		  0x69, 0x94, 0x08, 0x00, 0x00, 0x00, 0x4c, 0x00 }
 	},
 #endif
 	[12] = {0x190, /* Video Program System (VPS), PAL */
-		{V4L2_SLICED_VPS,16,16,0},
+		{V4L2_SLICED_VPS, 16, 16, 0},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0xba, 0xce, 0x2b, 0x0d,
 		  0xa6, 0xda, 0x0b, 0x00, 0x00, 0x00, 0x60, 0x00 }
 	},
@@ -623,7 +625,7 @@ static int tvp5150_g_sliced_vbi_cap(struct v4l2_subdev *sd,
 	int line, i;
 
 	dev_dbg_lvl(sd->dev, 1, debug, "g_sliced_vbi_cap\n");
-	memset(cap, 0, sizeof *cap);
+	memset(cap, 0, sizeof(*cap));
 
 	for (i = 0; i < ARRAY_SIZE(vbi_ram_default); i++) {
 		const struct i2c_vbi_ram_value *regs = &vbi_ram_default[i];
@@ -655,7 +657,7 @@ static int tvp5150_g_sliced_vbi_cap(struct v4l2_subdev *sd,
  *	MSB = field2
  */
 static int tvp5150_set_vbi(struct v4l2_subdev *sd,
-			unsigned int type,u8 flags, int line,
+			unsigned int type, u8 flags, int line,
 			const int fields)
 {
 	struct tvp5150 *decoder = to_tvp5150(sd);
@@ -1101,11 +1103,14 @@ static int tvp5150_s_routing(struct v4l2_subdev *sd,
 
 static int tvp5150_s_raw_fmt(struct v4l2_subdev *sd, struct v4l2_vbi_format *fmt)
 {
-	/* this is for capturing 36 raw vbi lines
-	   if there's a way to cut off the beginning 2 vbi lines
-	   with the tvp5150 then the vbi line count could be lowered
-	   to 17 lines/field again, although I couldn't find a register
-	   which could do that cropping */
+	/*
+	 * this is for capturing 36 raw vbi lines
+	 * if there's a way to cut off the beginning 2 vbi lines
+	 * with the tvp5150 then the vbi line count could be lowered
+	 * to 17 lines/field again, although I couldn't find a register
+	 * which could do that cropping
+	 */
+
 	if (fmt->sample_format == V4L2_PIX_FMT_GREY)
 		tvp5150_write(sd, TVP5150_LUMA_PROC_CTL_1, 0x70);
 	if (fmt->count[0] == 18 && fmt->count[1] == 18) {
diff --git a/drivers/media/i2c/video-i2c.c b/drivers/media/i2c/video-i2c.c
new file mode 100644
index 000000000000..0b347cc19aa5
--- /dev/null
+++ b/drivers/media/i2c/video-i2c.c
@@ -0,0 +1,564 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * video-i2c.c - Support for I2C transport video devices
+ *
+ * Copyright (C) 2018 Matt Ranostay <matt.ranostay@konsulko.com>
+ *
+ * Supported:
+ * - Panasonic AMG88xx Grid-Eye Sensors
+ */
+
+#include <linux/delay.h>
+#include <linux/freezer.h>
+#include <linux/kthread.h>
+#include <linux/i2c.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_device.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/videodev2.h>
+#include <media/v4l2-common.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-ioctl.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/videobuf2-vmalloc.h>
+
+#define VIDEO_I2C_DRIVER	"video-i2c"
+
+struct video_i2c_chip;
+
+struct video_i2c_buffer {
+	struct vb2_v4l2_buffer vb;
+	struct list_head list;
+};
+
+struct video_i2c_data {
+	struct i2c_client *client;
+	const struct video_i2c_chip *chip;
+	struct mutex lock;
+	spinlock_t slock;
+	unsigned int sequence;
+	struct mutex queue_lock;
+
+	struct v4l2_device v4l2_dev;
+	struct video_device vdev;
+	struct vb2_queue vb_vidq;
+
+	struct task_struct *kthread_vid_cap;
+	struct list_head vid_cap_active;
+};
+
+static const struct v4l2_fmtdesc amg88xx_format = {
+	.pixelformat = V4L2_PIX_FMT_Y12,
+};
+
+static const struct v4l2_frmsize_discrete amg88xx_size = {
+	.width = 8,
+	.height = 8,
+};
+
+struct video_i2c_chip {
+	/* video dimensions */
+	const struct v4l2_fmtdesc *format;
+	const struct v4l2_frmsize_discrete *size;
+
+	/* max frames per second */
+	unsigned int max_fps;
+
+	/* pixel buffer size */
+	unsigned int buffer_size;
+
+	/* pixel size in bits */
+	unsigned int bpp;
+
+	/* xfer function */
+	int (*xfer)(struct video_i2c_data *data, char *buf);
+};
+
+static int amg88xx_xfer(struct video_i2c_data *data, char *buf)
+{
+	struct i2c_client *client = data->client;
+	struct i2c_msg msg[2];
+	u8 reg = 0x80;
+	int ret;
+
+	msg[0].addr = client->addr;
+	msg[0].flags = 0;
+	msg[0].len = 1;
+	msg[0].buf  = (char *)&reg;
+
+	msg[1].addr = client->addr;
+	msg[1].flags = I2C_M_RD;
+	msg[1].len = data->chip->buffer_size;
+	msg[1].buf = (char *)buf;
+
+	ret = i2c_transfer(client->adapter, msg, 2);
+
+	return (ret == 2) ? 0 : -EIO;
+}
+
+#define AMG88XX		0
+
+static const struct video_i2c_chip video_i2c_chip[] = {
+	[AMG88XX] = {
+		.size		= &amg88xx_size,
+		.format		= &amg88xx_format,
+		.max_fps	= 10,
+		.buffer_size	= 128,
+		.bpp		= 16,
+		.xfer		= &amg88xx_xfer,
+	},
+};
+
+static const struct v4l2_file_operations video_i2c_fops = {
+	.owner		= THIS_MODULE,
+	.open		= v4l2_fh_open,
+	.release	= vb2_fop_release,
+	.poll		= vb2_fop_poll,
+	.read		= vb2_fop_read,
+	.mmap		= vb2_fop_mmap,
+	.unlocked_ioctl = video_ioctl2,
+};
+
+static int queue_setup(struct vb2_queue *vq,
+		       unsigned int *nbuffers, unsigned int *nplanes,
+		       unsigned int sizes[], struct device *alloc_devs[])
+{
+	struct video_i2c_data *data = vb2_get_drv_priv(vq);
+	unsigned int size = data->chip->buffer_size;
+
+	if (vq->num_buffers + *nbuffers < 2)
+		*nbuffers = 2;
+
+	if (*nplanes)
+		return sizes[0] < size ? -EINVAL : 0;
+
+	*nplanes = 1;
+	sizes[0] = size;
+
+	return 0;
+}
+
+static int buffer_prepare(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct video_i2c_data *data = vb2_get_drv_priv(vb->vb2_queue);
+	unsigned int size = data->chip->buffer_size;
+
+	if (vb2_plane_size(vb, 0) < size)
+		return -EINVAL;
+
+	vbuf->field = V4L2_FIELD_NONE;
+	vb2_set_plane_payload(vb, 0, size);
+
+	return 0;
+}
+
+static void buffer_queue(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct video_i2c_data *data = vb2_get_drv_priv(vb->vb2_queue);
+	struct video_i2c_buffer *buf =
+			container_of(vbuf, struct video_i2c_buffer, vb);
+
+	spin_lock(&data->slock);
+	list_add_tail(&buf->list, &data->vid_cap_active);
+	spin_unlock(&data->slock);
+}
+
+static int video_i2c_thread_vid_cap(void *priv)
+{
+	struct video_i2c_data *data = priv;
+	unsigned int delay = msecs_to_jiffies(1000 / data->chip->max_fps);
+
+	set_freezable();
+
+	do {
+		unsigned long start_jiffies = jiffies;
+		struct video_i2c_buffer *vid_cap_buf = NULL;
+		int schedule_delay;
+
+		try_to_freeze();
+
+		spin_lock(&data->slock);
+
+		if (!list_empty(&data->vid_cap_active)) {
+			vid_cap_buf = list_last_entry(&data->vid_cap_active,
+						 struct video_i2c_buffer, list);
+			list_del(&vid_cap_buf->list);
+		}
+
+		spin_unlock(&data->slock);
+
+		if (vid_cap_buf) {
+			struct vb2_buffer *vb2_buf = &vid_cap_buf->vb.vb2_buf;
+			void *vbuf = vb2_plane_vaddr(vb2_buf, 0);
+			int ret;
+
+			ret = data->chip->xfer(data, vbuf);
+			vb2_buf->timestamp = ktime_get_ns();
+			vid_cap_buf->vb.sequence = data->sequence++;
+			vb2_buffer_done(vb2_buf, ret ?
+				VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE);
+		}
+
+		schedule_delay = delay - (jiffies - start_jiffies);
+
+		if (time_after(jiffies, start_jiffies + delay))
+			schedule_delay = delay;
+
+		schedule_timeout_interruptible(schedule_delay);
+	} while (!kthread_should_stop());
+
+	return 0;
+}
+
+static void video_i2c_del_list(struct vb2_queue *vq, enum vb2_buffer_state state)
+{
+	struct video_i2c_data *data = vb2_get_drv_priv(vq);
+	struct video_i2c_buffer *buf, *tmp;
+
+	spin_lock(&data->slock);
+
+	list_for_each_entry_safe(buf, tmp, &data->vid_cap_active, list) {
+		list_del(&buf->list);
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
+	}
+
+	spin_unlock(&data->slock);
+}
+
+static int start_streaming(struct vb2_queue *vq, unsigned int count)
+{
+	struct video_i2c_data *data = vb2_get_drv_priv(vq);
+
+	if (data->kthread_vid_cap)
+		return 0;
+
+	data->sequence = 0;
+	data->kthread_vid_cap = kthread_run(video_i2c_thread_vid_cap, data,
+					    "%s-vid-cap", data->v4l2_dev.name);
+	if (!IS_ERR(data->kthread_vid_cap))
+		return 0;
+
+	video_i2c_del_list(vq, VB2_BUF_STATE_QUEUED);
+
+	return PTR_ERR(data->kthread_vid_cap);
+}
+
+static void stop_streaming(struct vb2_queue *vq)
+{
+	struct video_i2c_data *data = vb2_get_drv_priv(vq);
+
+	if (data->kthread_vid_cap == NULL)
+		return;
+
+	kthread_stop(data->kthread_vid_cap);
+	data->kthread_vid_cap = NULL;
+
+	video_i2c_del_list(vq, VB2_BUF_STATE_ERROR);
+}
+
+static struct vb2_ops video_i2c_video_qops = {
+	.queue_setup		= queue_setup,
+	.buf_prepare		= buffer_prepare,
+	.buf_queue		= buffer_queue,
+	.start_streaming	= start_streaming,
+	.stop_streaming		= stop_streaming,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
+};
+
+static int video_i2c_querycap(struct file *file, void  *priv,
+				struct v4l2_capability *vcap)
+{
+	struct video_i2c_data *data = video_drvdata(file);
+	struct i2c_client *client = data->client;
+
+	strlcpy(vcap->driver, data->v4l2_dev.name, sizeof(vcap->driver));
+	strlcpy(vcap->card, data->vdev.name, sizeof(vcap->card));
+
+	sprintf(vcap->bus_info, "I2C:%d-%d", client->adapter->nr, client->addr);
+
+	return 0;
+}
+
+static int video_i2c_g_input(struct file *file, void *fh, unsigned int *inp)
+{
+	*inp = 0;
+
+	return 0;
+}
+
+static int video_i2c_s_input(struct file *file, void *fh, unsigned int inp)
+{
+	return (inp > 0) ? -EINVAL : 0;
+}
+
+static int video_i2c_enum_input(struct file *file, void *fh,
+				  struct v4l2_input *vin)
+{
+	if (vin->index > 0)
+		return -EINVAL;
+
+	strlcpy(vin->name, "Camera", sizeof(vin->name));
+
+	vin->type = V4L2_INPUT_TYPE_CAMERA;
+
+	return 0;
+}
+
+static int video_i2c_enum_fmt_vid_cap(struct file *file, void *fh,
+					struct v4l2_fmtdesc *fmt)
+{
+	struct video_i2c_data *data = video_drvdata(file);
+	enum v4l2_buf_type type = fmt->type;
+
+	if (fmt->index > 0)
+		return -EINVAL;
+
+	*fmt = *data->chip->format;
+	fmt->type = type;
+
+	return 0;
+}
+
+static int video_i2c_enum_framesizes(struct file *file, void *fh,
+				       struct v4l2_frmsizeenum *fsize)
+{
+	const struct video_i2c_data *data = video_drvdata(file);
+	const struct v4l2_frmsize_discrete *size = data->chip->size;
+
+	/* currently only one frame size is allowed */
+	if (fsize->index > 0)
+		return -EINVAL;
+
+	if (fsize->pixel_format != data->chip->format->pixelformat)
+		return -EINVAL;
+
+	fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE;
+	fsize->discrete.width = size->width;
+	fsize->discrete.height = size->height;
+
+	return 0;
+}
+
+static int video_i2c_enum_frameintervals(struct file *file, void *priv,
+					   struct v4l2_frmivalenum *fe)
+{
+	const struct video_i2c_data *data = video_drvdata(file);
+	const struct v4l2_frmsize_discrete *size = data->chip->size;
+
+	if (fe->index > 0)
+		return -EINVAL;
+
+	if (fe->width != size->width || fe->height != size->height)
+		return -EINVAL;
+
+	fe->type = V4L2_FRMIVAL_TYPE_DISCRETE;
+	fe->discrete.numerator = 1;
+	fe->discrete.denominator = data->chip->max_fps;
+
+	return 0;
+}
+
+static int video_i2c_try_fmt_vid_cap(struct file *file, void *fh,
+				       struct v4l2_format *fmt)
+{
+	const struct video_i2c_data *data = video_drvdata(file);
+	const struct v4l2_frmsize_discrete *size = data->chip->size;
+	struct v4l2_pix_format *pix = &fmt->fmt.pix;
+	unsigned int bpp = data->chip->bpp / 8;
+
+	pix->width = size->width;
+	pix->height = size->height;
+	pix->pixelformat = data->chip->format->pixelformat;
+	pix->field = V4L2_FIELD_NONE;
+	pix->bytesperline = pix->width * bpp;
+	pix->sizeimage = pix->bytesperline * pix->height;
+	pix->colorspace = V4L2_COLORSPACE_RAW;
+
+	return 0;
+}
+
+static int video_i2c_s_fmt_vid_cap(struct file *file, void *fh,
+				     struct v4l2_format *fmt)
+{
+	struct video_i2c_data *data = video_drvdata(file);
+
+	if (vb2_is_busy(&data->vb_vidq))
+		return -EBUSY;
+
+	return video_i2c_try_fmt_vid_cap(file, fh, fmt);
+}
+
+static int video_i2c_g_parm(struct file *filp, void *priv,
+			      struct v4l2_streamparm *parm)
+{
+	struct video_i2c_data *data = video_drvdata(filp);
+
+	if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	parm->parm.capture.readbuffers = 1;
+	parm->parm.capture.capability = V4L2_CAP_TIMEPERFRAME;
+	parm->parm.capture.timeperframe.numerator = 1;
+	parm->parm.capture.timeperframe.denominator = data->chip->max_fps;
+
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops video_i2c_ioctl_ops = {
+	.vidioc_querycap		= video_i2c_querycap,
+	.vidioc_g_input			= video_i2c_g_input,
+	.vidioc_s_input			= video_i2c_s_input,
+	.vidioc_enum_input		= video_i2c_enum_input,
+	.vidioc_enum_fmt_vid_cap	= video_i2c_enum_fmt_vid_cap,
+	.vidioc_enum_framesizes		= video_i2c_enum_framesizes,
+	.vidioc_enum_frameintervals	= video_i2c_enum_frameintervals,
+	.vidioc_g_fmt_vid_cap		= video_i2c_try_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap		= video_i2c_s_fmt_vid_cap,
+	.vidioc_g_parm			= video_i2c_g_parm,
+	.vidioc_s_parm			= video_i2c_g_parm,
+	.vidioc_try_fmt_vid_cap		= video_i2c_try_fmt_vid_cap,
+	.vidioc_reqbufs			= vb2_ioctl_reqbufs,
+	.vidioc_create_bufs		= vb2_ioctl_create_bufs,
+	.vidioc_prepare_buf		= vb2_ioctl_prepare_buf,
+	.vidioc_querybuf		= vb2_ioctl_querybuf,
+	.vidioc_qbuf			= vb2_ioctl_qbuf,
+	.vidioc_dqbuf			= vb2_ioctl_dqbuf,
+	.vidioc_streamon		= vb2_ioctl_streamon,
+	.vidioc_streamoff		= vb2_ioctl_streamoff,
+};
+
+static void video_i2c_release(struct video_device *vdev)
+{
+	kfree(video_get_drvdata(vdev));
+}
+
+static int video_i2c_probe(struct i2c_client *client,
+			     const struct i2c_device_id *id)
+{
+	struct video_i2c_data *data;
+	struct v4l2_device *v4l2_dev;
+	struct vb2_queue *queue;
+	int ret = -ENODEV;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	if (dev_fwnode(&client->dev))
+		data->chip = device_get_match_data(&client->dev);
+	else if (id)
+		data->chip = &video_i2c_chip[id->driver_data];
+	else
+		goto error_free_device;
+
+	data->client = client;
+	v4l2_dev = &data->v4l2_dev;
+	strlcpy(v4l2_dev->name, VIDEO_I2C_DRIVER, sizeof(v4l2_dev->name));
+
+	ret = v4l2_device_register(&client->dev, v4l2_dev);
+	if (ret < 0)
+		goto error_free_device;
+
+	mutex_init(&data->lock);
+	mutex_init(&data->queue_lock);
+
+	queue = &data->vb_vidq;
+	queue->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	queue->io_modes = VB2_DMABUF | VB2_MMAP | VB2_USERPTR | VB2_READ;
+	queue->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	queue->drv_priv = data;
+	queue->buf_struct_size = sizeof(struct video_i2c_buffer);
+	queue->min_buffers_needed = 1;
+	queue->ops = &video_i2c_video_qops;
+	queue->mem_ops = &vb2_vmalloc_memops;
+
+	ret = vb2_queue_init(queue);
+	if (ret < 0)
+		goto error_unregister_device;
+
+	data->vdev.queue = queue;
+	data->vdev.queue->lock = &data->queue_lock;
+
+	snprintf(data->vdev.name, sizeof(data->vdev.name),
+				 "I2C %d-%d Transport Video",
+				 client->adapter->nr, client->addr);
+
+	data->vdev.v4l2_dev = v4l2_dev;
+	data->vdev.fops = &video_i2c_fops;
+	data->vdev.lock = &data->lock;
+	data->vdev.ioctl_ops = &video_i2c_ioctl_ops;
+	data->vdev.release = video_i2c_release;
+	data->vdev.device_caps = V4L2_CAP_VIDEO_CAPTURE |
+				 V4L2_CAP_READWRITE | V4L2_CAP_STREAMING;
+
+	spin_lock_init(&data->slock);
+	INIT_LIST_HEAD(&data->vid_cap_active);
+
+	video_set_drvdata(&data->vdev, data);
+	i2c_set_clientdata(client, data);
+
+	ret = video_register_device(&data->vdev, VFL_TYPE_GRABBER, -1);
+	if (ret < 0)
+		goto error_unregister_device;
+
+	return 0;
+
+error_unregister_device:
+	v4l2_device_unregister(v4l2_dev);
+	mutex_destroy(&data->lock);
+	mutex_destroy(&data->queue_lock);
+
+error_free_device:
+	kfree(data);
+
+	return ret;
+}
+
+static int video_i2c_remove(struct i2c_client *client)
+{
+	struct video_i2c_data *data = i2c_get_clientdata(client);
+
+	video_unregister_device(&data->vdev);
+	v4l2_device_unregister(&data->v4l2_dev);
+
+	mutex_destroy(&data->lock);
+	mutex_destroy(&data->queue_lock);
+
+	return 0;
+}
+
+static const struct i2c_device_id video_i2c_id_table[] = {
+	{ "amg88xx", AMG88XX },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, video_i2c_id_table);
+
+static const struct of_device_id video_i2c_of_match[] = {
+	{ .compatible = "panasonic,amg88xx", .data = &video_i2c_chip[AMG88XX] },
+	{}
+};
+MODULE_DEVICE_TABLE(of, video_i2c_of_match);
+
+static struct i2c_driver video_i2c_driver = {
+	.driver = {
+		.name	= VIDEO_I2C_DRIVER,
+		.of_match_table = video_i2c_of_match,
+	},
+	.probe		= video_i2c_probe,
+	.remove		= video_i2c_remove,
+	.id_table	= video_i2c_id_table,
+};
+
+module_i2c_driver(video_i2c_driver);
+
+MODULE_AUTHOR("Matt Ranostay <matt.ranostay@konsulko.com>");
+MODULE_DESCRIPTION("I2C transport video support");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/media-device.c b/drivers/media/media-device.c
index 35e81f7c0d2f..ae59c3177555 100644
--- a/drivers/media/media-device.c
+++ b/drivers/media/media-device.c
@@ -54,9 +54,10 @@ static int media_device_close(struct file *filp)
 	return 0;
 }
 
-static int media_device_get_info(struct media_device *dev,
-				 struct media_device_info *info)
+static long media_device_get_info(struct media_device *dev, void *arg)
 {
+	struct media_device_info *info = arg;
+
 	memset(info, 0, sizeof(*info));
 
 	if (dev->driver_name[0])
@@ -93,9 +94,9 @@ static struct media_entity *find_entity(struct media_device *mdev, u32 id)
 	return NULL;
 }
 
-static long media_device_enum_entities(struct media_device *mdev,
-				       struct media_entity_desc *entd)
+static long media_device_enum_entities(struct media_device *mdev, void *arg)
 {
+	struct media_entity_desc *entd = arg;
 	struct media_entity *ent;
 
 	ent = find_entity(mdev, entd->id);
@@ -146,9 +147,9 @@ static void media_device_kpad_to_upad(const struct media_pad *kpad,
 	upad->flags = kpad->flags;
 }
 
-static long media_device_enum_links(struct media_device *mdev,
-				    struct media_links_enum *links)
+static long media_device_enum_links(struct media_device *mdev, void *arg)
 {
+	struct media_links_enum *links = arg;
 	struct media_entity *entity;
 
 	entity = find_entity(mdev, links->entity);
@@ -195,9 +196,9 @@ static long media_device_enum_links(struct media_device *mdev,
 	return 0;
 }
 
-static long media_device_setup_link(struct media_device *mdev,
-				    struct media_link_desc *linkd)
+static long media_device_setup_link(struct media_device *mdev, void *arg)
 {
+	struct media_link_desc *linkd = arg;
 	struct media_link *link = NULL;
 	struct media_entity *source;
 	struct media_entity *sink;
@@ -225,9 +226,9 @@ static long media_device_setup_link(struct media_device *mdev,
 	return __media_entity_setup_link(link, linkd->flags);
 }
 
-static long media_device_get_topology(struct media_device *mdev,
-				      struct media_v2_topology *topo)
+static long media_device_get_topology(struct media_device *mdev, void *arg)
 {
+	struct media_v2_topology *topo = arg;
 	struct media_entity *entity;
 	struct media_interface *intf;
 	struct media_pad *pad;
diff --git a/drivers/media/mmc/siano/smssdio.c b/drivers/media/mmc/siano/smssdio.c
index fee2d710bbf8..b9e40d4ca0e8 100644
--- a/drivers/media/mmc/siano/smssdio.c
+++ b/drivers/media/mmc/siano/smssdio.c
@@ -279,7 +279,7 @@ static int smssdio_probe(struct sdio_func *func,
 		goto free;
 	}
 
-	ret = smscore_register_device(&params, &smsdev->coredev, NULL);
+	ret = smscore_register_device(&params, &smsdev->coredev, GFP_DMA, NULL);
 	if (ret < 0)
 		goto free;
 
diff --git a/drivers/media/pci/Kconfig b/drivers/media/pci/Kconfig
index 5932e225f9c0..1f09123e2bf9 100644
--- a/drivers/media/pci/Kconfig
+++ b/drivers/media/pci/Kconfig
@@ -16,7 +16,6 @@ source "drivers/media/pci/sta2x11/Kconfig"
 source "drivers/media/pci/tw5864/Kconfig"
 source "drivers/media/pci/tw68/Kconfig"
 source "drivers/media/pci/tw686x/Kconfig"
-source "drivers/media/pci/zoran/Kconfig"
 endif
 
 if MEDIA_ANALOG_TV_SUPPORT
diff --git a/drivers/media/pci/Makefile b/drivers/media/pci/Makefile
index 1c5ab07a8cff..984fa247096d 100644
--- a/drivers/media/pci/Makefile
+++ b/drivers/media/pci/Makefile
@@ -18,7 +18,6 @@ obj-y        +=	ttpci/		\
 		intel/
 
 obj-$(CONFIG_VIDEO_IVTV) += ivtv/
-obj-$(CONFIG_VIDEO_ZORAN) += zoran/
 obj-$(CONFIG_VIDEO_CX18) += cx18/
 obj-$(CONFIG_VIDEO_CX23885) += cx23885/
 obj-$(CONFIG_VIDEO_CX25821) += cx25821/
diff --git a/drivers/media/pci/bt8xx/bttv-risc.c b/drivers/media/pci/bt8xx/bttv-risc.c
index 3859dde98be2..6a6be0b49f70 100644
--- a/drivers/media/pci/bt8xx/bttv-risc.c
+++ b/drivers/media/pci/bt8xx/bttv-risc.c
@@ -189,20 +189,21 @@ bttv_risc_planar(struct bttv *btv, struct btcx_riscmem *risc,
 				yoffset -= sg_dma_len(ysg);
 				ysg = sg_next(ysg);
 			}
-			while (uoffset && uoffset >= sg_dma_len(usg)) {
-				uoffset -= sg_dma_len(usg);
-				usg = sg_next(usg);
-			}
-			while (voffset && voffset >= sg_dma_len(vsg)) {
-				voffset -= sg_dma_len(vsg);
-				vsg = sg_next(vsg);
-			}
 
 			/* calculate max number of bytes we can write */
 			ylen = todo;
 			if (yoffset + ylen > sg_dma_len(ysg))
 				ylen = sg_dma_len(ysg) - yoffset;
 			if (chroma) {
+				while (uoffset && uoffset >= sg_dma_len(usg)) {
+					uoffset -= sg_dma_len(usg);
+					usg = sg_next(usg);
+				}
+				while (voffset && voffset >= sg_dma_len(vsg)) {
+					voffset -= sg_dma_len(vsg);
+					vsg = sg_next(vsg);
+				}
+
 				if (uoffset + (ylen>>hshift) > sg_dma_len(usg))
 					ylen = (sg_dma_len(usg) - uoffset) << hshift;
 				if (voffset + (ylen>>hshift) > sg_dma_len(vsg))
diff --git a/drivers/media/pci/bt8xx/dst.c b/drivers/media/pci/bt8xx/dst.c
index 4f0bba9e4c48..2e33b7236672 100644
--- a/drivers/media/pci/bt8xx/dst.c
+++ b/drivers/media/pci/bt8xx/dst.c
@@ -1657,7 +1657,7 @@ static int dst_tune_frontend(struct dvb_frontend* fe,
 	return 0;
 }
 
-static int dst_get_tuning_algo(struct dvb_frontend *fe)
+static enum dvbfe_algo dst_get_tuning_algo(struct dvb_frontend *fe)
 {
 	return dst_algo ? DVBFE_ALGO_HW : DVBFE_ALGO_SW;
 }
diff --git a/drivers/media/pci/bt8xx/dvb-bt8xx.c b/drivers/media/pci/bt8xx/dvb-bt8xx.c
index f60d69ac515b..5ef6e2051d45 100644
--- a/drivers/media/pci/bt8xx/dvb-bt8xx.c
+++ b/drivers/media/pci/bt8xx/dvb-bt8xx.c
@@ -575,7 +575,6 @@ static struct mt352_config digitv_alps_tded4_config = {
 };
 
 static struct lgdt330x_config tdvs_tua6034_config = {
-	.demod_address    = 0x0e,
 	.demod_chip       = LGDT3303,
 	.serial_mpeg      = 0x40, /* TPSERIAL for 3303 in TOP_CONTROL */
 };
@@ -614,7 +613,8 @@ static void frontend_init(struct dvb_bt8xx_card *card, u32 type)
 
 	case BTTV_BOARD_DVICO_FUSIONHDTV_5_LITE:
 		lgdt330x_reset(card);
-		card->fe = dvb_attach(lgdt330x_attach, &tdvs_tua6034_config, card->i2c_adapter);
+		card->fe = dvb_attach(lgdt330x_attach, &tdvs_tua6034_config,
+				      0x0e, card->i2c_adapter);
 		if (card->fe != NULL) {
 			dvb_attach(simple_tuner_attach, card->fe,
 				   card->i2c_adapter, 0x61,
diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c
index 019fac49db5b..94b996ff12a9 100644
--- a/drivers/media/pci/cx23885/cx23885-core.c
+++ b/drivers/media/pci/cx23885/cx23885-core.c
@@ -422,19 +422,30 @@ static void cx23885_wakeup(struct cx23885_tsport *port,
 			   struct cx23885_dmaqueue *q, u32 count)
 {
 	struct cx23885_buffer *buf;
-
-	if (list_empty(&q->active))
-		return;
-	buf = list_entry(q->active.next,
-			 struct cx23885_buffer, queue);
-
-	buf->vb.vb2_buf.timestamp = ktime_get_ns();
-	buf->vb.sequence = q->count++;
-	dprintk(1, "[%p/%d] wakeup reg=%d buf=%d\n", buf,
-		buf->vb.vb2_buf.index,
-		count, q->count);
-	list_del(&buf->queue);
-	vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
+	int count_delta;
+	int max_buf_done = 5; /* service maximum five buffers */
+
+	do {
+		if (list_empty(&q->active))
+			return;
+		buf = list_entry(q->active.next,
+				 struct cx23885_buffer, queue);
+
+		buf->vb.vb2_buf.timestamp = ktime_get_ns();
+		buf->vb.sequence = q->count++;
+		if (count != (q->count % 65536)) {
+			dprintk(1, "[%p/%d] wakeup reg=%d buf=%d\n", buf,
+				buf->vb.vb2_buf.index, count, q->count);
+		} else {
+			dprintk(7, "[%p/%d] wakeup reg=%d buf=%d\n", buf,
+				buf->vb.vb2_buf.index, count, q->count);
+		}
+		list_del(&buf->queue);
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
+		max_buf_done--;
+		/* count register is 16 bits so apply modulo appropriately */
+		count_delta = ((int)count - (int)(q->count % 65536));
+	} while ((count_delta > 0) && (max_buf_done > 0));
 }
 
 int cx23885_sram_channel_setup(struct cx23885_dev *dev,
@@ -590,6 +601,25 @@ static void cx23885_risc_disasm(struct cx23885_tsport *port,
 	}
 }
 
+static void cx23885_clear_bridge_error(struct cx23885_dev *dev)
+{
+	uint32_t reg1_val = cx_read(TC_REQ); /* read-only */
+	uint32_t reg2_val = cx_read(TC_REQ_SET);
+
+	if (reg1_val && reg2_val) {
+		cx_write(TC_REQ, reg1_val);
+		cx_write(TC_REQ_SET, reg2_val);
+		cx_read(VID_B_DMA);
+		cx_read(VBI_B_DMA);
+		cx_read(VID_C_DMA);
+		cx_read(VBI_C_DMA);
+
+		dev_info(&dev->pci->dev,
+			"dma in progress detected 0x%08x 0x%08x, clearing\n",
+			reg1_val, reg2_val);
+	}
+}
+
 static void cx23885_shutdown(struct cx23885_dev *dev)
 {
 	/* disable RISC controller */
@@ -635,6 +665,8 @@ static void cx23885_reset(struct cx23885_dev *dev)
 	cx_write(CLK_DELAY, cx_read(CLK_DELAY) & 0x80000000);
 	cx_write(PAD_CTRL, 0x00500300);
 
+	/* clear dma in progress */
+	cx23885_clear_bridge_error(dev);
 	mdelay(100);
 
 	cx23885_sram_channel_setup(dev, &dev->sram_channels[SRAM_CH01],
@@ -651,6 +683,11 @@ static void cx23885_reset(struct cx23885_dev *dev)
 	cx23885_sram_channel_setup(dev, &dev->sram_channels[SRAM_CH09], 128, 0);
 
 	cx23885_gpio_setup(dev);
+
+	cx23885_irq_get_mask(dev);
+
+	/* clear dma in progress */
+	cx23885_clear_bridge_error(dev);
 }
 
 
@@ -665,6 +702,8 @@ static int cx23885_pci_quirks(struct cx23885_dev *dev)
 	if (dev->bridge == CX23885_BRIDGE_885)
 		cx_clear(RDR_TLCTL0, 1 << 4);
 
+	/* clear dma in progress */
+	cx23885_clear_bridge_error(dev);
 	return 0;
 }
 
@@ -1329,6 +1368,18 @@ static void cx23885_tsport_reg_dump(struct cx23885_tsport *port)
 		port->reg_ts_clk_en, cx_read(port->reg_ts_clk_en));
 	dprintk(1, "%s() ts_int_msk(0x%08X)     0x%08x\n", __func__,
 		port->reg_ts_int_msk, cx_read(port->reg_ts_int_msk));
+	dprintk(1, "%s() ts_int_status(0x%08X)  0x%08x\n", __func__,
+		port->reg_ts_int_stat, cx_read(port->reg_ts_int_stat));
+	dprintk(1, "%s() PCI_INT_STAT           0x%08X\n", __func__,
+		cx_read(PCI_INT_STAT));
+	dprintk(1, "%s() VID_B_INT_MSTAT        0x%08X\n", __func__,
+		cx_read(VID_B_INT_MSTAT));
+	dprintk(1, "%s() VID_B_INT_SSTAT        0x%08X\n", __func__,
+		cx_read(VID_B_INT_SSTAT));
+	dprintk(1, "%s() VID_C_INT_MSTAT        0x%08X\n", __func__,
+		cx_read(VID_C_INT_MSTAT));
+	dprintk(1, "%s() VID_C_INT_SSTAT        0x%08X\n", __func__,
+		cx_read(VID_C_INT_SSTAT));
 }
 
 int cx23885_start_dma(struct cx23885_tsport *port,
@@ -1341,6 +1392,9 @@ int cx23885_start_dma(struct cx23885_tsport *port,
 	dprintk(1, "%s() w: %d, h: %d, f: %d\n", __func__,
 		dev->width, dev->height, dev->field);
 
+	/* clear dma in progress */
+	cx23885_clear_bridge_error(dev);
+
 	/* Stop the fifo and risc engine for this port */
 	cx_clear(port->reg_dma_ctl, port->dma_ctl_val);
 
@@ -1410,8 +1464,15 @@ int cx23885_start_dma(struct cx23885_tsport *port,
 		reg = reg | 0xa;
 		cx_write(PAD_CTRL, reg);
 
-		/* FIXME and these two registers should be documented. */
+		/* Sets MOE_CLK_DIS to disable MoE clock */
+		/* sets MCLK_DLY_SEL/BCLK_DLY_SEL to 1 buffer delay each */
 		cx_write(CLK_DELAY, cx_read(CLK_DELAY) | 0x80000011);
+
+		/* ALT_GPIO_ALT_SET: GPIO[0]
+		 * IR_ALT_TX_SEL: GPIO[1]
+		 * GPIO1_ALT_SEL: VIP_656_DATA[0]
+		 * GPIO0_ALT_SEL: VIP_656_CLK
+		 */
 		cx_write(ALT_PIN_OUT_SEL, 0x10100045);
 	}
 
@@ -1421,16 +1482,26 @@ int cx23885_start_dma(struct cx23885_tsport *port,
 	case CX23885_BRIDGE_888:
 		/* enable irqs */
 		dprintk(1, "%s() enabling TS int's and DMA\n", __func__);
+		/* clear dma in progress */
+		cx23885_clear_bridge_error(dev);
 		cx_set(port->reg_ts_int_msk,  port->ts_int_msk_val);
 		cx_set(port->reg_dma_ctl, port->dma_ctl_val);
+
+		/* clear dma in progress */
+		cx23885_clear_bridge_error(dev);
 		cx23885_irq_add(dev, port->pci_irqmask);
 		cx23885_irq_enable_all(dev);
+
+		/* clear dma in progress */
+		cx23885_clear_bridge_error(dev);
 		break;
 	default:
 		BUG();
 	}
 
 	cx_set(DEV_CNTRL2, (1<<5)); /* Enable RISC controller */
+	/* clear dma in progress */
+	cx23885_clear_bridge_error(dev);
 
 	if (cx23885_boards[dev->board].portb == CX23885_MPEG_ENCODER)
 		cx23885_av_clk(dev, 1);
@@ -1438,6 +1509,11 @@ int cx23885_start_dma(struct cx23885_tsport *port,
 	if (debug > 4)
 		cx23885_tsport_reg_dump(port);
 
+	cx23885_irq_get_mask(dev);
+
+	/* clear dma in progress */
+	cx23885_clear_bridge_error(dev);
+
 	return 0;
 }
 
@@ -1445,15 +1521,28 @@ static int cx23885_stop_dma(struct cx23885_tsport *port)
 {
 	struct cx23885_dev *dev = port->dev;
 	u32 reg;
+	int delay = 0;
+	uint32_t reg1_val;
+	uint32_t reg2_val;
 
 	dprintk(1, "%s()\n", __func__);
 
 	/* Stop interrupts and DMA */
 	cx_clear(port->reg_ts_int_msk, port->ts_int_msk_val);
 	cx_clear(port->reg_dma_ctl, port->dma_ctl_val);
+	/* just in case wait for any dma to complete before allowing dealloc */
+	mdelay(20);
+	for (delay = 0; delay < 100; delay++) {
+		reg1_val = cx_read(TC_REQ);
+		reg2_val = cx_read(TC_REQ_SET);
+		if (reg1_val == 0 || reg2_val == 0)
+			break;
+		mdelay(1);
+	}
+	dev_dbg(&dev->pci->dev, "delay=%d reg1=0x%08x reg2=0x%08x\n",
+		delay, reg1_val, reg2_val);
 
 	if (cx23885_boards[dev->board].portb == CX23885_MPEG_ENCODER) {
-
 		reg = cx_read(PAD_CTRL);
 
 		/* Set TS1_OE */
@@ -1464,7 +1553,6 @@ static int cx23885_stop_dma(struct cx23885_tsport *port)
 		cx_write(PAD_CTRL, reg);
 		cx_write(port->reg_src_sel, 0);
 		cx_write(port->reg_gen_ctrl, 8);
-
 	}
 
 	if (cx23885_boards[dev->board].portb == CX23885_MPEG_ENCODER)
@@ -1693,6 +1781,12 @@ static irqreturn_t cx23885_irq(int irq, void *dev_id)
 
 	pci_status = cx_read(PCI_INT_STAT);
 	pci_mask = cx23885_irq_get_mask(dev);
+	if ((pci_status & pci_mask) == 0) {
+		dprintk(7, "pci_status: 0x%08x  pci_mask: 0x%08x\n",
+			pci_status, pci_mask);
+		goto out;
+	}
+
 	vida_status = cx_read(VID_A_INT_STAT);
 	vida_mask = cx_read(VID_A_INT_MSK);
 	audint_status = cx_read(AUDIO_INT_INT_STAT);
@@ -1702,7 +1796,9 @@ static irqreturn_t cx23885_irq(int irq, void *dev_id)
 	ts2_status = cx_read(VID_C_INT_STAT);
 	ts2_mask = cx_read(VID_C_INT_MSK);
 
-	if ((pci_status == 0) && (ts2_status == 0) && (ts1_status == 0))
+	if (((pci_status & pci_mask) == 0) &&
+		((ts2_status & ts2_mask) == 0) &&
+		((ts1_status & ts1_mask) == 0))
 		goto out;
 
 	vida_count = cx_read(VID_A_GPCNT);
@@ -1829,7 +1925,7 @@ static irqreturn_t cx23885_irq(int irq, void *dev_id)
 	}
 
 	if (handled)
-		cx_write(PCI_INT_STAT, pci_status);
+		cx_write(PCI_INT_STAT, pci_status & pci_mask);
 out:
 	return IRQ_RETVAL(handled);
 }
diff --git a/drivers/media/pci/cx23885/cx23885-dvb.c b/drivers/media/pci/cx23885/cx23885-dvb.c
index 114d9bcbe4f4..7d52173073d6 100644
--- a/drivers/media/pci/cx23885/cx23885-dvb.c
+++ b/drivers/media/pci/cx23885/cx23885-dvb.c
@@ -252,7 +252,6 @@ static struct mt2131_config hauppauge_generic_tunerconfig = {
 };
 
 static struct lgdt330x_config fusionhdtv_5_express = {
-	.demod_address = 0x0e,
 	.demod_chip = LGDT3303,
 	.serial_mpeg = 0x40,
 };
@@ -1321,8 +1320,9 @@ static int dvb_register(struct cx23885_tsport *port)
 	case CX23885_BOARD_DVICO_FUSIONHDTV_5_EXP:
 		i2c_bus = &dev->i2c_bus[0];
 		fe0->dvb.frontend = dvb_attach(lgdt330x_attach,
-						&fusionhdtv_5_express,
-						&i2c_bus->i2c_adap);
+					       &fusionhdtv_5_express,
+					       0x0e,
+					       &i2c_bus->i2c_adap);
 		if (fe0->dvb.frontend == NULL)
 			break;
 		dvb_attach(simple_tuner_attach, fe0->dvb.frontend,
diff --git a/drivers/media/pci/cx23885/cx23885-reg.h b/drivers/media/pci/cx23885/cx23885-reg.h
index 2d3cbafe2402..08cec8d91742 100644
--- a/drivers/media/pci/cx23885/cx23885-reg.h
+++ b/drivers/media/pci/cx23885/cx23885-reg.h
@@ -288,6 +288,18 @@ Channel manager Data Structure entry = 20 DWORD
 #define AUDIO_EXT_INT_MSTAT	0x00040068
 #define AUDIO_EXT_INT_SSTAT	0x0004006C
 
+/* Bits [7:0] set in both TC_REQ and TC_REQ_SET
+ * indicate a stall in the RISC engine for a
+ * particular rider traffic class. This causes
+ * the 885 and 888 bridges (unknown about 887)
+ * to become inoperable. Setting bits in
+ * TC_REQ_SET resets the corresponding bits
+ * in TC_REQ (and TC_REQ_SET) allowing
+ * operation to continue.
+ */
+#define TC_REQ		0x00040090
+#define TC_REQ_SET	0x00040094
+
 #define RDR_CFG0	0x00050000
 #define RDR_CFG1	0x00050004
 #define RDR_CFG2	0x00050008
@@ -386,6 +398,8 @@ Channel manager Data Structure entry = 20 DWORD
 #define VID_B_PIXEL_FRMT	0x00130184
 
 /* Video C Interface */
+#define VID_C_DMA		0x00130200
+#define VBI_C_DMA		0x00130208
 #define VID_C_GPCNT		0x00130220
 #define VID_C_GPCNT_CTL		0x00130230
 #define VBI_C_GPCNT_CTL		0x00130234
diff --git a/drivers/media/pci/cx88/cx88-dvb.c b/drivers/media/pci/cx88/cx88-dvb.c
index 2f886140dd2e..ccfde28d4af2 100644
--- a/drivers/media/pci/cx88/cx88-dvb.c
+++ b/drivers/media/pci/cx88/cx88-dvb.c
@@ -411,21 +411,18 @@ static int lgdt330x_set_ts_param(struct dvb_frontend *fe, int is_punctured)
 }
 
 static struct lgdt330x_config fusionhdtv_3_gold = {
-	.demod_address = 0x0e,
 	.demod_chip    = LGDT3302,
 	.serial_mpeg   = 0x04, /* TPSERIAL for 3302 in TOP_CONTROL */
 	.set_ts_params = lgdt330x_set_ts_param,
 };
 
 static const struct lgdt330x_config fusionhdtv_5_gold = {
-	.demod_address = 0x0e,
 	.demod_chip    = LGDT3303,
 	.serial_mpeg   = 0x40, /* TPSERIAL for 3303 in TOP_CONTROL */
 	.set_ts_params = lgdt330x_set_ts_param,
 };
 
 static const struct lgdt330x_config pchdtv_hd5500 = {
-	.demod_address = 0x59,
 	.demod_chip    = LGDT3303,
 	.serial_mpeg   = 0x40, /* TPSERIAL for 3303 in TOP_CONTROL */
 	.set_ts_params = lgdt330x_set_ts_param,
@@ -1237,6 +1234,7 @@ static int dvb_register(struct cx8802_dev *dev)
 		fusionhdtv_3_gold.pll_rf_set = lgdt330x_pll_rf_set;
 		fe0->dvb.frontend = dvb_attach(lgdt330x_attach,
 					       &fusionhdtv_3_gold,
+					       0x0e,
 					       &core->i2c_adap);
 		if (fe0->dvb.frontend) {
 			if (!dvb_attach(simple_tuner_attach, fe0->dvb.frontend,
@@ -1255,6 +1253,7 @@ static int dvb_register(struct cx8802_dev *dev)
 		mdelay(200);
 		fe0->dvb.frontend = dvb_attach(lgdt330x_attach,
 					       &fusionhdtv_3_gold,
+					       0x0e,
 					       &core->i2c_adap);
 		if (fe0->dvb.frontend) {
 			if (!dvb_attach(simple_tuner_attach, fe0->dvb.frontend,
@@ -1273,6 +1272,7 @@ static int dvb_register(struct cx8802_dev *dev)
 		mdelay(200);
 		fe0->dvb.frontend = dvb_attach(lgdt330x_attach,
 					       &fusionhdtv_5_gold,
+					       0x0e,
 					       &core->i2c_adap);
 		if (fe0->dvb.frontend) {
 			if (!dvb_attach(simple_tuner_attach, fe0->dvb.frontend,
@@ -1294,6 +1294,7 @@ static int dvb_register(struct cx8802_dev *dev)
 		mdelay(200);
 		fe0->dvb.frontend = dvb_attach(lgdt330x_attach,
 					       &pchdtv_hd5500,
+					       0x59,
 					       &core->i2c_adap);
 		if (fe0->dvb.frontend) {
 			if (!dvb_attach(simple_tuner_attach, fe0->dvb.frontend,
diff --git a/drivers/media/pci/cx88/cx88-input.c b/drivers/media/pci/cx88/cx88-input.c
index 6f4e6923a91a..2f5debce4905 100644
--- a/drivers/media/pci/cx88/cx88-input.c
+++ b/drivers/media/pci/cx88/cx88-input.c
@@ -180,7 +180,8 @@ static enum hrtimer_restart cx88_ir_work(struct hrtimer *timer)
 	struct cx88_IR *ir = container_of(timer, struct cx88_IR, timer);
 
 	cx88_ir_handle_key(ir);
-	missed = hrtimer_forward_now(&ir->timer, ir->polling * 1000000LL);
+	missed = hrtimer_forward_now(&ir->timer,
+				     ktime_set(0, ir->polling * 1000000));
 	if (missed > 1)
 		ir_dprintk("Missed ticks %ld\n", missed - 1);
 
@@ -200,7 +201,8 @@ static int __cx88_ir_start(void *priv)
 	if (ir->polling) {
 		hrtimer_init(&ir->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 		ir->timer.function = cx88_ir_work;
-		hrtimer_start(&ir->timer, ir->polling * 1000000LL,
+		hrtimer_start(&ir->timer,
+			      ktime_set(0, ir->polling * 1000000),
 			      HRTIMER_MODE_REL);
 	}
 	if (ir->sampling) {
@@ -632,8 +634,9 @@ void cx88_i2c_init_ir(struct cx88_core *core)
 		memset(&core->init_data, 0, sizeof(core->init_data));
 
 		if (*addrp == 0x71) {
-			/* Hauppauge XVR */
-			core->init_data.name = "cx88 Hauppauge XVR remote";
+			/* Hauppauge Z8F0811 */
+			strlcpy(info.type, "ir_z8f0811_haup", I2C_NAME_SIZE);
+			core->init_data.name = core->board.name;
 			core->init_data.ir_codes = RC_MAP_HAUPPAUGE;
 			core->init_data.type = RC_PROTO_BIT_RC5 |
 				RC_PROTO_BIT_RC6_MCE | RC_PROTO_BIT_RC6_6A_32;
diff --git a/drivers/media/pci/cx88/cx88-vbi.c b/drivers/media/pci/cx88/cx88-vbi.c
index c637679b01b2..58489ea0c1da 100644
--- a/drivers/media/pci/cx88/cx88-vbi.c
+++ b/drivers/media/pci/cx88/cx88-vbi.c
@@ -178,7 +178,6 @@ static void buffer_queue(struct vb2_buffer *vb)
 
 	if (list_empty(&q->active)) {
 		list_add_tail(&buf->list, &q->active);
-		cx8800_start_vbi_dma(dev, q, buf);
 		dprintk(2, "[%p/%d] vbi_queue - first active\n",
 			buf, buf->vb.vb2_buf.index);
 
diff --git a/drivers/media/pci/ddbridge/Kconfig b/drivers/media/pci/ddbridge/Kconfig
index a422dde2f34a..16faef265e97 100644
--- a/drivers/media/pci/ddbridge/Kconfig
+++ b/drivers/media/pci/ddbridge/Kconfig
@@ -14,6 +14,7 @@ config DVB_DDBRIDGE
 	select MEDIA_TUNER_TDA18212 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_MXL5XX if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_CXD2099 if MEDIA_SUBDRV_AUTOSELECT
+	select DVB_DUMMY_FE if MEDIA_SUBDRV_AUTOSELECT
 	---help---
 	  Support for cards with the Digital Devices PCI express bridge:
 	  - Octopus PCIe Bridge
diff --git a/drivers/media/pci/ddbridge/Makefile b/drivers/media/pci/ddbridge/Makefile
index 745b37d07558..9b9e35f171b7 100644
--- a/drivers/media/pci/ddbridge/Makefile
+++ b/drivers/media/pci/ddbridge/Makefile
@@ -4,7 +4,7 @@
 #
 
 ddbridge-objs := ddbridge-main.o ddbridge-core.o ddbridge-ci.o \
-		ddbridge-hw.o ddbridge-i2c.o ddbridge-max.o
+		ddbridge-hw.o ddbridge-i2c.o ddbridge-max.o ddbridge-mci.o
 
 obj-$(CONFIG_DVB_DDBRIDGE) += ddbridge.o
 
diff --git a/drivers/media/pci/ddbridge/ddbridge-ci.c b/drivers/media/pci/ddbridge/ddbridge-ci.c
index a9dbc4ebf94f..cfe23d02e561 100644
--- a/drivers/media/pci/ddbridge/ddbridge-ci.c
+++ b/drivers/media/pci/ddbridge/ddbridge-ci.c
@@ -164,7 +164,7 @@ static struct dvb_ca_en50221 en_templ = {
 
 static void ci_attach(struct ddb_port *port)
 {
-	struct ddb_ci *ci = NULL;
+	struct ddb_ci *ci;
 
 	ci = kzalloc(sizeof(*ci), GFP_KERNEL);
 	if (!ci)
diff --git a/drivers/media/pci/ddbridge/ddbridge-core.c b/drivers/media/pci/ddbridge/ddbridge-core.c
index 90687eff5909..d5b0d1eaf3ad 100644
--- a/drivers/media/pci/ddbridge/ddbridge-core.c
+++ b/drivers/media/pci/ddbridge/ddbridge-core.c
@@ -54,6 +54,7 @@
 #include "stv6111.h"
 #include "lnbh25.h"
 #include "cxd2099.h"
+#include "dvb_dummy_fe.h"
 
 /****************************************************************************/
 
@@ -68,11 +69,53 @@ module_param(adapter_alloc, int, 0444);
 MODULE_PARM_DESC(adapter_alloc,
 		 "0-one adapter per io, 1-one per tab with io, 2-one per tab, 3-one for all");
 
+static int ci_bitrate = 70000;
+module_param(ci_bitrate, int, 0444);
+MODULE_PARM_DESC(ci_bitrate, " Bitrate in KHz for output to CI.");
+
+static int ts_loop = -1;
+module_param(ts_loop, int, 0444);
+MODULE_PARM_DESC(ts_loop, "TS in/out test loop on port ts_loop");
+
+static int xo2_speed = 2;
+module_param(xo2_speed, int, 0444);
+MODULE_PARM_DESC(xo2_speed, "default transfer speed for xo2 based duoflex, 0=55,1=75,2=90,3=104 MBit/s, default=2, use attribute to change for individual cards");
+
+#ifdef __arm__
+static int alt_dma = 1;
+#else
+static int alt_dma;
+#endif
+module_param(alt_dma, int, 0444);
+MODULE_PARM_DESC(alt_dma, "use alternative DMA buffer handling");
+
+static int no_init;
+module_param(no_init, int, 0444);
+MODULE_PARM_DESC(no_init, "do not initialize most devices");
+
+static int stv0910_single;
+module_param(stv0910_single, int, 0444);
+MODULE_PARM_DESC(stv0910_single, "use stv0910 cards as single demods");
+
+static int dma_buf_num = 8;
+module_param(dma_buf_num, int, 0444);
+MODULE_PARM_DESC(dma_buf_num, "Number of DMA buffers, possible values: 8-32");
+
+static int dma_buf_size = 21;
+module_param(dma_buf_size, int, 0444);
+MODULE_PARM_DESC(dma_buf_size,
+		 "DMA buffer size as multiple of 128*47, possible values: 1-43");
+
+static int dummy_tuner;
+module_param(dummy_tuner, int, 0444);
+MODULE_PARM_DESC(dummy_tuner,
+		 "attach dummy tuner to port 0 on Octopus V3 or Octopus Mini cards");
+
 /****************************************************************************/
 
 static DEFINE_MUTEX(redirect_lock);
 
-struct workqueue_struct *ddb_wq;
+static struct workqueue_struct *ddb_wq;
 
 static struct ddb *ddbs[DDB_MAX_ADAPTER];
 
@@ -80,6 +123,16 @@ static struct ddb *ddbs[DDB_MAX_ADAPTER];
 /****************************************************************************/
 /****************************************************************************/
 
+struct ddb_irq *ddb_irq_set(struct ddb *dev, u32 link, u32 nr,
+			    void (*handler)(void *), void *data)
+{
+	struct ddb_irq *irq = &dev->link[link].irq[nr];
+
+	irq->handler = handler;
+	irq->data = data;
+	return irq;
+}
+
 static void ddb_set_dma_table(struct ddb_io *io)
 {
 	struct ddb *dev = io->port->dev;
@@ -410,13 +463,11 @@ static void ddb_output_start(struct ddb_output *output)
 	struct ddb *dev = output->port->dev;
 	u32 con = 0x11c, con2 = 0;
 
-	if (output->dma) {
-		spin_lock_irq(&output->dma->lock);
-		output->dma->cbuf = 0;
-		output->dma->coff = 0;
-		output->dma->stat = 0;
-		ddbwritel(dev, 0, DMA_BUFFER_CONTROL(output->dma));
-	}
+	spin_lock_irq(&output->dma->lock);
+	output->dma->cbuf = 0;
+	output->dma->coff = 0;
+	output->dma->stat = 0;
+	ddbwritel(dev, 0, DMA_BUFFER_CONTROL(output->dma));
 
 	if (output->port->input[0]->port->class == DDB_PORT_LOOP)
 		con = (1UL << 13) | 0x14;
@@ -429,36 +480,29 @@ static void ddb_output_start(struct ddb_output *output)
 	ddbwritel(dev, con, TS_CONTROL(output));
 	ddbwritel(dev, con2, TS_CONTROL2(output));
 
-	if (output->dma) {
-		ddbwritel(dev, output->dma->bufval,
-			  DMA_BUFFER_SIZE(output->dma));
-		ddbwritel(dev, 0, DMA_BUFFER_ACK(output->dma));
-		ddbwritel(dev, 1, DMA_BASE_READ);
-		ddbwritel(dev, 7, DMA_BUFFER_CONTROL(output->dma));
-	}
+	ddbwritel(dev, output->dma->bufval,
+		  DMA_BUFFER_SIZE(output->dma));
+	ddbwritel(dev, 0, DMA_BUFFER_ACK(output->dma));
+	ddbwritel(dev, 1, DMA_BASE_READ);
+	ddbwritel(dev, 7, DMA_BUFFER_CONTROL(output->dma));
 
 	ddbwritel(dev, con | 1, TS_CONTROL(output));
 
-	if (output->dma) {
-		output->dma->running = 1;
-		spin_unlock_irq(&output->dma->lock);
-	}
+	output->dma->running = 1;
+	spin_unlock_irq(&output->dma->lock);
 }
 
 static void ddb_output_stop(struct ddb_output *output)
 {
 	struct ddb *dev = output->port->dev;
 
-	if (output->dma)
-		spin_lock_irq(&output->dma->lock);
+	spin_lock_irq(&output->dma->lock);
 
 	ddbwritel(dev, 0, TS_CONTROL(output));
 
-	if (output->dma) {
-		ddbwritel(dev, 0, DMA_BUFFER_CONTROL(output->dma));
-		output->dma->running = 0;
-		spin_unlock_irq(&output->dma->lock);
-	}
+	ddbwritel(dev, 0, DMA_BUFFER_CONTROL(output->dma));
+	output->dma->running = 0;
+	spin_unlock_irq(&output->dma->lock);
 }
 
 static void ddb_input_stop(struct ddb_input *input)
@@ -466,45 +510,42 @@ static void ddb_input_stop(struct ddb_input *input)
 	struct ddb *dev = input->port->dev;
 	u32 tag = DDB_LINK_TAG(input->port->lnr);
 
-	if (input->dma)
-		spin_lock_irq(&input->dma->lock);
+	spin_lock_irq(&input->dma->lock);
+
 	ddbwritel(dev, 0, tag | TS_CONTROL(input));
-	if (input->dma) {
-		ddbwritel(dev, 0, DMA_BUFFER_CONTROL(input->dma));
-		input->dma->running = 0;
-		spin_unlock_irq(&input->dma->lock);
-	}
+
+	ddbwritel(dev, 0, DMA_BUFFER_CONTROL(input->dma));
+	input->dma->running = 0;
+	spin_unlock_irq(&input->dma->lock);
 }
 
 static void ddb_input_start(struct ddb_input *input)
 {
 	struct ddb *dev = input->port->dev;
 
-	if (input->dma) {
-		spin_lock_irq(&input->dma->lock);
-		input->dma->cbuf = 0;
-		input->dma->coff = 0;
-		input->dma->stat = 0;
-		ddbwritel(dev, 0, DMA_BUFFER_CONTROL(input->dma));
-	}
+	spin_lock_irq(&input->dma->lock);
+	input->dma->cbuf = 0;
+	input->dma->coff = 0;
+	input->dma->stat = 0;
+	ddbwritel(dev, 0, DMA_BUFFER_CONTROL(input->dma));
+
 	ddbwritel(dev, 0, TS_CONTROL(input));
 	ddbwritel(dev, 2, TS_CONTROL(input));
 	ddbwritel(dev, 0, TS_CONTROL(input));
 
-	if (input->dma) {
-		ddbwritel(dev, input->dma->bufval,
-			  DMA_BUFFER_SIZE(input->dma));
-		ddbwritel(dev, 0, DMA_BUFFER_ACK(input->dma));
-		ddbwritel(dev, 1, DMA_BASE_WRITE);
-		ddbwritel(dev, 3, DMA_BUFFER_CONTROL(input->dma));
-	}
+	ddbwritel(dev, input->dma->bufval,
+		  DMA_BUFFER_SIZE(input->dma));
+	ddbwritel(dev, 0, DMA_BUFFER_ACK(input->dma));
+	ddbwritel(dev, 1, DMA_BASE_WRITE);
+	ddbwritel(dev, 3, DMA_BUFFER_CONTROL(input->dma));
 
 	ddbwritel(dev, 0x09, TS_CONTROL(input));
 
-	if (input->dma) {
-		input->dma->running = 1;
-		spin_unlock_irq(&input->dma->lock);
-	}
+	if (input->port->type == DDB_TUNER_DUMMY)
+		ddbwritel(dev, 0x000fff01, TS_CONTROL2(input));
+
+	input->dma->running = 1;
+	spin_unlock_irq(&input->dma->lock);
 }
 
 static void ddb_input_start_all(struct ddb_input *input)
@@ -549,12 +590,12 @@ static u32 ddb_output_free(struct ddb_output *output)
 
 	if (output->dma->cbuf != idx) {
 		if ((((output->dma->cbuf + 1) % output->dma->num) == idx) &&
-		    (output->dma->size - output->dma->coff <= 188))
+		    (output->dma->size - output->dma->coff <= (2 * 188)))
 			return 0;
 		return 188;
 	}
 	diff = off - output->dma->coff;
-	if (diff <= 0 || diff > 188)
+	if (diff <= 0 || diff > (2 * 188))
 		return 188;
 	return 0;
 }
@@ -1142,6 +1183,7 @@ static const struct stv0910_cfg stv0910_p = {
 	.parallel = 1,
 	.rptlvl   = 4,
 	.clk      = 30000000,
+	.tsspeed  = 0x28,
 };
 
 static const struct lnbh25_config lnbh25_cfg = {
@@ -1149,7 +1191,7 @@ static const struct lnbh25_config lnbh25_cfg = {
 	.data2_config = LNBH25_TEN
 };
 
-static int demod_attach_stv0910(struct ddb_input *input, int type)
+static int demod_attach_stv0910(struct ddb_input *input, int type, int tsfast)
 {
 	struct i2c_adapter *i2c = &input->port->i2c->adap;
 	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
@@ -1162,6 +1204,12 @@ static int demod_attach_stv0910(struct ddb_input *input, int type)
 
 	if (type)
 		cfg.parallel = 2;
+
+	if (tsfast) {
+		dev_info(dev, "Enabling stv0910 higher speed TS\n");
+		cfg.tsspeed = 0x10;
+	}
+
 	dvb->fe = dvb_attach(stv0910_attach, i2c, &cfg, (input->nr & 1));
 	if (!dvb->fe) {
 		cfg.adr = 0x6c;
@@ -1208,6 +1256,20 @@ static int tuner_attach_stv6111(struct ddb_input *input, int type)
 	return 0;
 }
 
+static int demod_attach_dummy(struct ddb_input *input)
+{
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct device *dev = input->port->dev->dev;
+
+	dvb->fe = dvb_attach(dvb_dummy_fe_qam_attach);
+	if (!dvb->fe) {
+		dev_err(dev, "QAM dummy attach failed!\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
 static int start_feed(struct dvb_demux_feed *dvbdmxfeed)
 {
 	struct dvb_demux *dvbdmx = dvbdmxfeed->demux;
@@ -1383,7 +1445,25 @@ static int dvb_input_attach(struct ddb_input *input)
 	struct ddb_port *port = input->port;
 	struct dvb_adapter *adap = dvb->adap;
 	struct dvb_demux *dvbdemux = &dvb->demux;
-	int par = 0, osc24 = 0;
+	struct ddb_ids *devids = &input->port->dev->link[input->port->lnr].ids;
+	int par = 0, osc24 = 0, tsfast = 0;
+
+	/*
+	 * Determine if bridges with stv0910 demods can run with fast TS and
+	 * thus support high bandwidth transponders.
+	 * STV0910_PR and STV0910_P tuner types covers all relevant bridges,
+	 * namely the CineS2 V7(A) and the Octopus CI S2 Pro/Advanced. All
+	 * DuoFlex S2 V4(A) have type=DDB_TUNER_DVBS_STV0910 without any suffix
+	 * and are limited by the serial link to the bridge, thus won't work
+	 * in fast TS mode.
+	 */
+	if (port->nr == 0 &&
+	    (port->type == DDB_TUNER_DVBS_STV0910_PR ||
+	     port->type == DDB_TUNER_DVBS_STV0910_P)) {
+		/* fast TS on port 0 requires FPGA version >= 1.7 */
+		if ((devids->hwid & 0x00ffffff) >= 0x00010007)
+			tsfast = 1;
+	}
 
 	dvb->attached = 0x01;
 
@@ -1440,19 +1520,19 @@ static int dvb_input_attach(struct ddb_input *input)
 			goto err_tuner;
 		break;
 	case DDB_TUNER_DVBS_STV0910:
-		if (demod_attach_stv0910(input, 0) < 0)
+		if (demod_attach_stv0910(input, 0, tsfast) < 0)
 			goto err_detach;
 		if (tuner_attach_stv6111(input, 0) < 0)
 			goto err_tuner;
 		break;
 	case DDB_TUNER_DVBS_STV0910_PR:
-		if (demod_attach_stv0910(input, 1) < 0)
+		if (demod_attach_stv0910(input, 1, tsfast) < 0)
 			goto err_detach;
 		if (tuner_attach_stv6111(input, 1) < 0)
 			goto err_tuner;
 		break;
 	case DDB_TUNER_DVBS_STV0910_P:
-		if (demod_attach_stv0910(input, 0) < 0)
+		if (demod_attach_stv0910(input, 0, tsfast) < 0)
 			goto err_detach;
 		if (tuner_attach_stv6111(input, 1) < 0)
 			goto err_tuner;
@@ -1500,6 +1580,14 @@ static int dvb_input_attach(struct ddb_input *input)
 		if (tuner_attach_tda18212(input, port->type) < 0)
 			goto err_tuner;
 		break;
+	case DDB_TUNER_DUMMY:
+		if (demod_attach_dummy(input) < 0)
+			goto err_detach;
+		break;
+	case DDB_TUNER_MCI:
+		if (ddb_fe_attach_mci(input) < 0)
+			goto err_detach;
+		break;
 	default:
 		return 0;
 	}
@@ -1762,6 +1850,15 @@ static void ddb_port_probe(struct ddb_port *port)
 
 	/* Handle missing ports and ports without I2C */
 
+	if (dummy_tuner && !port->nr &&
+	    dev->link[0].ids.device == 0x0005) {
+		port->name = "DUMMY";
+		port->class = DDB_PORT_TUNER;
+		port->type = DDB_TUNER_DUMMY;
+		port->type_name = "DUMMY";
+		return;
+	}
+
 	if (port->nr == ts_loop) {
 		port->name = "TS LOOP";
 		port->class = DDB_PORT_LOOP;
@@ -1786,6 +1883,16 @@ static void ddb_port_probe(struct ddb_port *port)
 		return;
 	}
 
+	if (dev->link[l].info->type == DDB_OCTOPUS_MCI) {
+		if (port->nr >= dev->link[l].info->mci)
+			return;
+		port->name = "DUAL MCI";
+		port->type_name = "MCI";
+		port->class = DDB_PORT_TUNER;
+		port->type = DDB_TUNER_MCI;
+		return;
+	}
+
 	if (port->nr > 1 && dev->link[l].info->type == DDB_OCTOPUS_CI) {
 		port->name = "CI internal";
 		port->type_name = "INTERNAL";
@@ -2081,39 +2188,39 @@ static void input_work(struct work_struct *work)
 	spin_unlock_irqrestore(&dma->lock, flags);
 }
 
-static void input_handler(unsigned long data)
+static void input_handler(void *data)
 {
 	struct ddb_input *input = (struct ddb_input *)data;
 	struct ddb_dma *dma = input->dma;
 
-	/*
-	 * If there is no input connected, input_tasklet() will
-	 * just copy pointers and ACK. So, there is no need to go
-	 * through the tasklet scheduler.
-	 */
-	if (input->redi)
-		queue_work(ddb_wq, &dma->work);
-	else
-		input_work(&dma->work);
+	queue_work(ddb_wq, &dma->work);
 }
 
-static void output_handler(unsigned long data)
+static void output_work(struct work_struct *work)
 {
-	struct ddb_output *output = (struct ddb_output *)data;
-	struct ddb_dma *dma = output->dma;
+	struct ddb_dma *dma = container_of(work, struct ddb_dma, work);
+	struct ddb_output *output = (struct ddb_output *)dma->io;
 	struct ddb *dev = output->port->dev;
+	unsigned long flags;
 
-	spin_lock(&dma->lock);
-	if (!dma->running) {
-		spin_unlock(&dma->lock);
-		return;
-	}
+	spin_lock_irqsave(&dma->lock, flags);
+	if (!dma->running)
+		goto unlock_exit;
 	dma->stat = ddbreadl(dev, DMA_BUFFER_CURRENT(dma));
 	dma->ctrl = ddbreadl(dev, DMA_BUFFER_CONTROL(dma));
 	if (output->redi)
 		output_ack_input(output, output->redi);
 	wake_up(&dma->wq);
-	spin_unlock(&dma->lock);
+unlock_exit:
+	spin_unlock_irqrestore(&dma->lock, flags);
+}
+
+static void output_handler(void *data)
+{
+	struct ddb_output *output = (struct ddb_output *)data;
+	struct ddb_dma *dma = output->dma;
+
+	queue_work(ddb_wq, &dma->work);
 }
 
 /****************************************************************************/
@@ -2146,18 +2253,19 @@ static void ddb_dma_init(struct ddb_io *io, int nr, int out)
 	spin_lock_init(&dma->lock);
 	init_waitqueue_head(&dma->wq);
 	if (out) {
+		INIT_WORK(&dma->work, output_work);
 		dma->regs = rm->odma->base + rm->odma->size * nr;
 		dma->bufregs = rm->odma_buf->base + rm->odma_buf->size * nr;
-		dma->num = OUTPUT_DMA_BUFS;
-		dma->size = OUTPUT_DMA_SIZE;
-		dma->div = OUTPUT_DMA_IRQ_DIV;
+		dma->num = dma_buf_num;
+		dma->size = dma_buf_size * 128 * 47;
+		dma->div = 1;
 	} else {
 		INIT_WORK(&dma->work, input_work);
 		dma->regs = rm->idma->base + rm->idma->size * nr;
 		dma->bufregs = rm->idma_buf->base + rm->idma_buf->size * nr;
-		dma->num = INPUT_DMA_BUFS;
-		dma->size = INPUT_DMA_SIZE;
-		dma->div = INPUT_DMA_IRQ_DIV;
+		dma->num = dma_buf_num;
+		dma->size = dma_buf_size * 128 * 47;
+		dma->div = 1;
 	}
 	ddbwritel(io->port->dev, 0, DMA_BUFFER_ACK(dma));
 	dev_dbg(io->port->dev->dev, "init link %u, io %u, dma %u, dmaregs %08x bufregs %08x\n",
@@ -2190,8 +2298,7 @@ static void ddb_input_init(struct ddb_port *port, int nr, int pnr, int anr)
 		dev_dbg(dev->dev, "init link %u, input %u, handler %u\n",
 			port->lnr, nr, dma_nr + base);
 
-		dev->handler[0][dma_nr + base] = input_handler;
-		dev->handler_data[0][dma_nr + base] = (unsigned long)input;
+		ddb_irq_set(dev, 0, dma_nr + base, &input_handler, input);
 		ddb_dma_init(input, dma_nr, 0);
 	}
 }
@@ -2216,8 +2323,7 @@ static void ddb_output_init(struct ddb_port *port, int nr)
 		const struct ddb_regmap *rm0 = io_regmap(output, 0);
 		u32 base = rm0->irq_base_odma;
 
-		dev->handler[0][nr + base] = output_handler;
-		dev->handler_data[0][nr + base] = (unsigned long)output;
+		ddb_irq_set(dev, 0, nr + base, &output_handler, output);
 		ddb_dma_init(output, nr, 1);
 	}
 }
@@ -2329,6 +2435,7 @@ void ddb_ports_init(struct ddb *dev)
 				break;
 			case DDB_OCTOPUS_MAX:
 			case DDB_OCTOPUS_MAX_CT:
+			case DDB_OCTOPUS_MCI:
 				ddb_input_init(port, 2 * i, 0, 2 * p);
 				ddb_input_init(port, 2 * i + 1, 1, 2 * p + 1);
 				break;
@@ -2361,73 +2468,62 @@ void ddb_ports_release(struct ddb *dev)
 /****************************************************************************/
 
 #define IRQ_HANDLE(_nr) \
-	do { if ((s & (1UL << ((_nr) & 0x1f))) && dev->handler[0][_nr]) \
-		dev->handler[0][_nr](dev->handler_data[0][_nr]); } \
+	do { if ((s & (1UL << ((_nr) & 0x1f))) && \
+		 dev->link[0].irq[_nr].handler) \
+		dev->link[0].irq[_nr].handler(dev->link[0].irq[_nr].data); } \
 	while (0)
 
+#define IRQ_HANDLE_NIBBLE(_shift) {		     \
+	if (s & (0x0000000f << ((_shift) & 0x1f))) { \
+		IRQ_HANDLE(0 + (_shift));	     \
+		IRQ_HANDLE(1 + (_shift));	     \
+		IRQ_HANDLE(2 + (_shift));	     \
+		IRQ_HANDLE(3 + (_shift));	     \
+	}					     \
+}
+
+#define IRQ_HANDLE_BYTE(_shift) {		     \
+	if (s & (0x000000ff << ((_shift) & 0x1f))) { \
+		IRQ_HANDLE(0 + (_shift));	     \
+		IRQ_HANDLE(1 + (_shift));	     \
+		IRQ_HANDLE(2 + (_shift));	     \
+		IRQ_HANDLE(3 + (_shift));	     \
+		IRQ_HANDLE(4 + (_shift));	     \
+		IRQ_HANDLE(5 + (_shift));	     \
+		IRQ_HANDLE(6 + (_shift));	     \
+		IRQ_HANDLE(7 + (_shift));	     \
+	}					     \
+}
+
 static void irq_handle_msg(struct ddb *dev, u32 s)
 {
 	dev->i2c_irq++;
-	IRQ_HANDLE(0);
-	IRQ_HANDLE(1);
-	IRQ_HANDLE(2);
-	IRQ_HANDLE(3);
+	IRQ_HANDLE_NIBBLE(0);
 }
 
 static void irq_handle_io(struct ddb *dev, u32 s)
 {
 	dev->ts_irq++;
-	if ((s & 0x000000f0)) {
-		IRQ_HANDLE(4);
-		IRQ_HANDLE(5);
-		IRQ_HANDLE(6);
-		IRQ_HANDLE(7);
-	}
-	if ((s & 0x0000ff00)) {
-		IRQ_HANDLE(8);
-		IRQ_HANDLE(9);
-		IRQ_HANDLE(10);
-		IRQ_HANDLE(11);
-		IRQ_HANDLE(12);
-		IRQ_HANDLE(13);
-		IRQ_HANDLE(14);
-		IRQ_HANDLE(15);
-	}
-	if ((s & 0x00ff0000)) {
-		IRQ_HANDLE(16);
-		IRQ_HANDLE(17);
-		IRQ_HANDLE(18);
-		IRQ_HANDLE(19);
-		IRQ_HANDLE(20);
-		IRQ_HANDLE(21);
-		IRQ_HANDLE(22);
-		IRQ_HANDLE(23);
-	}
-	if ((s & 0xff000000)) {
-		IRQ_HANDLE(24);
-		IRQ_HANDLE(25);
-		IRQ_HANDLE(26);
-		IRQ_HANDLE(27);
-		IRQ_HANDLE(28);
-		IRQ_HANDLE(29);
-		IRQ_HANDLE(30);
-		IRQ_HANDLE(31);
-	}
+	IRQ_HANDLE_NIBBLE(4);
+	IRQ_HANDLE_BYTE(8);
+	IRQ_HANDLE_BYTE(16);
+	IRQ_HANDLE_BYTE(24);
 }
 
 irqreturn_t ddb_irq_handler0(int irq, void *dev_id)
 {
 	struct ddb *dev = (struct ddb *)dev_id;
-	u32 s = ddbreadl(dev, INTERRUPT_STATUS);
+	u32 mask = 0x8fffff00;
+	u32 s = mask & ddbreadl(dev, INTERRUPT_STATUS);
 
+	if (!s)
+		return IRQ_NONE;
 	do {
 		if (s & 0x80000000)
 			return IRQ_NONE;
-		if (!(s & 0xfffff00))
-			return IRQ_NONE;
-		ddbwritel(dev, s & 0xfffff00, INTERRUPT_ACK);
+		ddbwritel(dev, s, INTERRUPT_ACK);
 		irq_handle_io(dev, s);
-	} while ((s = ddbreadl(dev, INTERRUPT_STATUS)));
+	} while ((s = mask & ddbreadl(dev, INTERRUPT_STATUS)));
 
 	return IRQ_HANDLED;
 }
@@ -2435,16 +2531,17 @@ irqreturn_t ddb_irq_handler0(int irq, void *dev_id)
 irqreturn_t ddb_irq_handler1(int irq, void *dev_id)
 {
 	struct ddb *dev = (struct ddb *)dev_id;
-	u32 s = ddbreadl(dev, INTERRUPT_STATUS);
+	u32 mask = 0x8000000f;
+	u32 s = mask & ddbreadl(dev, INTERRUPT_STATUS);
 
+	if (!s)
+		return IRQ_NONE;
 	do {
 		if (s & 0x80000000)
 			return IRQ_NONE;
-		if (!(s & 0x0000f))
-			return IRQ_NONE;
-		ddbwritel(dev, s & 0x0000f, INTERRUPT_ACK);
+		ddbwritel(dev, s, INTERRUPT_ACK);
 		irq_handle_msg(dev, s);
-	} while ((s = ddbreadl(dev, INTERRUPT_STATUS)));
+	} while ((s = mask & ddbreadl(dev, INTERRUPT_STATUS)));
 
 	return IRQ_HANDLED;
 }
@@ -3027,7 +3124,7 @@ static struct class ddb_class = {
 	.devnode        = ddb_devnode,
 };
 
-int ddb_class_create(void)
+static int ddb_class_create(void)
 {
 	ddb_major = register_chrdev(0, DDB_NAME, &ddb_fops);
 	if (ddb_major < 0)
@@ -3037,7 +3134,7 @@ int ddb_class_create(void)
 	return 0;
 }
 
-void ddb_class_destroy(void)
+static void ddb_class_destroy(void)
 {
 	class_unregister(&ddb_class);
 	unregister_chrdev(ddb_major, DDB_NAME);
@@ -3173,7 +3270,7 @@ static void tempmon_setfan(struct ddb_link *link)
 	ddblwritel(link, (pwm << 8), TEMPMON_FANCONTROL);
 }
 
-static void temp_handler(unsigned long data)
+static void temp_handler(void *data)
 {
 	struct ddb_link *link = (struct ddb_link *)data;
 
@@ -3196,8 +3293,7 @@ static int tempmon_init(struct ddb_link *link, int first_time)
 		memcpy(link->temp_tab, temperature_table,
 		       sizeof(temperature_table));
 	}
-	dev->handler[l][link->info->tempmon_irq] = temp_handler;
-	dev->handler_data[l][link->info->tempmon_irq] = (unsigned long)link;
+	ddb_irq_set(dev, l, link->info->tempmon_irq, temp_handler, link);
 	ddblwritel(link, (TEMPMON_CONTROL_OVERTEMP | TEMPMON_CONTROL_AUTOSCAN |
 			  TEMPMON_CONTROL_INTENABLE),
 		   TEMPMON_CONTROL);
@@ -3309,3 +3405,38 @@ void ddb_unmap(struct ddb *dev)
 		iounmap(dev->regs);
 	vfree(dev);
 }
+
+int ddb_exit_ddbridge(int stage, int error)
+{
+	switch (stage) {
+	default:
+	case 2:
+		destroy_workqueue(ddb_wq);
+		/* fall-through */
+	case 1:
+		ddb_class_destroy();
+		break;
+	}
+
+	return error;
+}
+
+int ddb_init_ddbridge(void)
+{
+	if (dma_buf_num < 8)
+		dma_buf_num = 8;
+	if (dma_buf_num > 32)
+		dma_buf_num = 32;
+	if (dma_buf_size < 1)
+		dma_buf_size = 1;
+	if (dma_buf_size > 43)
+		dma_buf_size = 43;
+
+	if (ddb_class_create() < 0)
+		return -1;
+	ddb_wq = alloc_workqueue("ddbridge", 0, 0);
+	if (!ddb_wq)
+		return ddb_exit_ddbridge(1, -1);
+
+	return 0;
+}
diff --git a/drivers/media/pci/ddbridge/ddbridge-hw.c b/drivers/media/pci/ddbridge/ddbridge-hw.c
index c6d14925e2fc..1d3ee6accdd5 100644
--- a/drivers/media/pci/ddbridge/ddbridge-hw.c
+++ b/drivers/media/pci/ddbridge/ddbridge-hw.c
@@ -311,6 +311,16 @@ static const struct ddb_info ddb_s2_48 = {
 	.tempmon_irq = 24,
 };
 
+static const struct ddb_info ddb_s2x_48 = {
+	.type     = DDB_OCTOPUS_MCI,
+	.name     = "Digital Devices MAX SX8",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x00,
+	.tempmon_irq = 24,
+	.mci      = 4
+};
+
 /****************************************************************************/
 /****************************************************************************/
 /****************************************************************************/
@@ -346,6 +356,7 @@ static const struct ddb_device_id ddb_device_ids[] = {
 	DDB_DEVID(0x0008, 0x0036, ddb_isdbt_8),
 	DDB_DEVID(0x0008, 0x0037, ddb_c2t2i_v0_8),
 	DDB_DEVID(0x0008, 0x0038, ddb_c2t2i_8),
+	DDB_DEVID(0x0009, 0x0025, ddb_s2x_48),
 	DDB_DEVID(0x0006, 0x0039, ddb_ctv7),
 	DDB_DEVID(0x0011, 0x0040, ddb_ci),
 	DDB_DEVID(0x0011, 0x0041, ddb_cis),
diff --git a/drivers/media/pci/ddbridge/ddbridge-i2c.c b/drivers/media/pci/ddbridge/ddbridge-i2c.c
index 82a9a0e806fc..667340c86ea7 100644
--- a/drivers/media/pci/ddbridge/ddbridge-i2c.c
+++ b/drivers/media/pci/ddbridge/ddbridge-i2c.c
@@ -147,7 +147,7 @@ void ddb_i2c_release(struct ddb *dev)
 	}
 }
 
-static void i2c_handler(unsigned long priv)
+static void i2c_handler(void *priv)
 {
 	struct ddb_i2c *i2c = (struct ddb_i2c *)priv;
 
@@ -210,8 +210,7 @@ int ddb_i2c_init(struct ddb *dev)
 			if (!(dev->link[l].info->i2c_mask & (1 << i)))
 				continue;
 			i2c = &dev->i2c[num];
-			dev->handler_data[l][i + base] = (unsigned long)i2c;
-			dev->handler[l][i + base] = i2c_handler;
+			ddb_irq_set(dev, l, i + base, i2c_handler, i2c);
 			stat = ddb_i2c_add(dev, i2c, regmap, l, i, num);
 			if (stat)
 				break;
diff --git a/drivers/media/pci/ddbridge/ddbridge-main.c b/drivers/media/pci/ddbridge/ddbridge-main.c
index 26497d6b1395..f4748cfd904b 100644
--- a/drivers/media/pci/ddbridge/ddbridge-main.c
+++ b/drivers/media/pci/ddbridge/ddbridge-main.c
@@ -55,34 +55,6 @@ MODULE_PARM_DESC(msi, "Control MSI interrupts: 0-disable (default), 1-enable");
 #endif
 #endif
 
-int ci_bitrate = 70000;
-module_param(ci_bitrate, int, 0444);
-MODULE_PARM_DESC(ci_bitrate, " Bitrate in KHz for output to CI.");
-
-int ts_loop = -1;
-module_param(ts_loop, int, 0444);
-MODULE_PARM_DESC(ts_loop, "TS in/out test loop on port ts_loop");
-
-int xo2_speed = 2;
-module_param(xo2_speed, int, 0444);
-MODULE_PARM_DESC(xo2_speed, "default transfer speed for xo2 based duoflex, 0=55,1=75,2=90,3=104 MBit/s, default=2, use attribute to change for individual cards");
-
-#ifdef __arm__
-int alt_dma = 1;
-#else
-int alt_dma;
-#endif
-module_param(alt_dma, int, 0444);
-MODULE_PARM_DESC(alt_dma, "use alternative DMA buffer handling");
-
-int no_init;
-module_param(no_init, int, 0444);
-MODULE_PARM_DESC(no_init, "do not initialize most devices");
-
-int stv0910_single;
-module_param(stv0910_single, int, 0444);
-MODULE_PARM_DESC(stv0910_single, "use stv0910 cards as single demods");
-
 /****************************************************************************/
 /****************************************************************************/
 /****************************************************************************/
@@ -93,18 +65,22 @@ static void ddb_irq_disable(struct ddb *dev)
 	ddbwritel(dev, 0, MSI1_ENABLE);
 }
 
-static void ddb_irq_exit(struct ddb *dev)
+static void ddb_msi_exit(struct ddb *dev)
 {
-	ddb_irq_disable(dev);
-	if (dev->msi == 2)
-		free_irq(dev->pdev->irq + 1, dev);
-	free_irq(dev->pdev->irq, dev);
 #ifdef CONFIG_PCI_MSI
 	if (dev->msi)
-		pci_disable_msi(dev->pdev);
+		pci_free_irq_vectors(dev->pdev);
 #endif
 }
 
+static void ddb_irq_exit(struct ddb *dev)
+{
+	ddb_irq_disable(dev);
+	if (dev->msi == 2)
+		free_irq(pci_irq_vector(dev->pdev, 1), dev);
+	free_irq(pci_irq_vector(dev->pdev, 0), dev);
+}
+
 static void ddb_remove(struct pci_dev *pdev)
 {
 	struct ddb *dev = (struct ddb *)pci_get_drvdata(pdev);
@@ -114,6 +90,7 @@ static void ddb_remove(struct pci_dev *pdev)
 	ddb_i2c_release(dev);
 
 	ddb_irq_exit(dev);
+	ddb_msi_exit(dev);
 	ddb_ports_release(dev);
 	ddb_buffers_free(dev);
 
@@ -128,7 +105,8 @@ static void ddb_irq_msi(struct ddb *dev, int nr)
 	int stat;
 
 	if (msi && pci_msi_enabled()) {
-		stat = pci_alloc_irq_vectors(dev->pdev, 1, nr, PCI_IRQ_MSI);
+		stat = pci_alloc_irq_vectors(dev->pdev, 1, nr,
+					     PCI_IRQ_MSI | PCI_IRQ_MSIX);
 		if (stat >= 1) {
 			dev->msi = stat;
 			dev_info(dev->dev, "using %d MSI interrupt(s)\n",
@@ -160,21 +138,24 @@ static int ddb_irq_init(struct ddb *dev)
 	if (dev->msi)
 		irq_flag = 0;
 	if (dev->msi == 2) {
-		stat = request_irq(dev->pdev->irq, ddb_irq_handler0,
-				   irq_flag, "ddbridge", (void *)dev);
+		stat = request_irq(pci_irq_vector(dev->pdev, 0),
+				   ddb_irq_handler0, irq_flag, "ddbridge",
+				   (void *)dev);
 		if (stat < 0)
 			return stat;
-		stat = request_irq(dev->pdev->irq + 1, ddb_irq_handler1,
-				   irq_flag, "ddbridge", (void *)dev);
+		stat = request_irq(pci_irq_vector(dev->pdev, 1),
+				   ddb_irq_handler1, irq_flag, "ddbridge",
+				   (void *)dev);
 		if (stat < 0) {
-			free_irq(dev->pdev->irq, dev);
+			free_irq(pci_irq_vector(dev->pdev, 0), dev);
 			return stat;
 		}
 	} else
 #endif
 	{
-		stat = request_irq(dev->pdev->irq, ddb_irq_handler,
-				   irq_flag, "ddbridge", (void *)dev);
+		stat = request_irq(pci_irq_vector(dev->pdev, 0),
+				   ddb_irq_handler, irq_flag, "ddbridge",
+				   (void *)dev);
 		if (stat < 0)
 			return stat;
 	}
@@ -217,6 +198,7 @@ static int ddb_probe(struct pci_dev *pdev,
 	dev->link[0].ids.device = id->device;
 	dev->link[0].ids.subvendor = id->subvendor;
 	dev->link[0].ids.subdevice = pdev->subsystem_device;
+	dev->link[0].ids.devid = (id->device << 16) | id->vendor;
 
 	dev->link[0].dev = dev;
 	dev->link[0].info = get_ddb_info(id->vendor, id->device,
@@ -258,8 +240,7 @@ static int ddb_probe(struct pci_dev *pdev,
 	ddb_irq_exit(dev);
 fail0:
 	dev_err(&pdev->dev, "fail0\n");
-	if (dev->msi)
-		pci_disable_msi(dev->pdev);
+	ddb_msi_exit(dev);
 fail:
 	dev_err(&pdev->dev, "fail\n");
 
@@ -283,6 +264,7 @@ static const struct pci_device_id ddb_id_table[] = {
 	DDB_DEVICE_ANY(0x0006),
 	DDB_DEVICE_ANY(0x0007),
 	DDB_DEVICE_ANY(0x0008),
+	DDB_DEVICE_ANY(0x0009),
 	DDB_DEVICE_ANY(0x0011),
 	DDB_DEVICE_ANY(0x0012),
 	DDB_DEVICE_ANY(0x0013),
@@ -310,32 +292,25 @@ static struct pci_driver ddb_pci_driver = {
 
 static __init int module_init_ddbridge(void)
 {
-	int stat = -1;
+	int stat;
 
 	pr_info("Digital Devices PCIE bridge driver "
 		DDBRIDGE_VERSION
 		", Copyright (C) 2010-17 Digital Devices GmbH\n");
-	if (ddb_class_create() < 0)
-		return -1;
-	ddb_wq = create_workqueue("ddbridge");
-	if (!ddb_wq)
-		goto exit1;
+	stat = ddb_init_ddbridge();
+	if (stat < 0)
+		return stat;
 	stat = pci_register_driver(&ddb_pci_driver);
 	if (stat < 0)
-		goto exit2;
-	return stat;
-exit2:
-	destroy_workqueue(ddb_wq);
-exit1:
-	ddb_class_destroy();
+		ddb_exit_ddbridge(0, stat);
+
 	return stat;
 }
 
 static __exit void module_exit_ddbridge(void)
 {
 	pci_unregister_driver(&ddb_pci_driver);
-	destroy_workqueue(ddb_wq);
-	ddb_class_destroy();
+	ddb_exit_ddbridge(0, 0);
 }
 
 module_init(module_init_ddbridge);
diff --git a/drivers/media/pci/ddbridge/ddbridge-max.c b/drivers/media/pci/ddbridge/ddbridge-max.c
index dc6b81488746..739e4b444cf4 100644
--- a/drivers/media/pci/ddbridge/ddbridge-max.c
+++ b/drivers/media/pci/ddbridge/ddbridge-max.c
@@ -33,6 +33,7 @@
 #include "ddbridge.h"
 #include "ddbridge-regs.h"
 #include "ddbridge-io.h"
+#include "ddbridge-mci.h"
 
 #include "ddbridge-max.h"
 #include "mxl5xx.h"
@@ -452,3 +453,44 @@ int ddb_fe_attach_mxl5xx(struct ddb_input *input)
 	dvb->input = tuner;
 	return 0;
 }
+
+/******************************************************************************/
+/* MAX MCI related functions */
+
+int ddb_fe_attach_mci(struct ddb_input *input)
+{
+	struct ddb *dev = input->port->dev;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct ddb_port *port = input->port;
+	struct ddb_link *link = &dev->link[port->lnr];
+	int demod, tuner;
+
+	demod = input->nr;
+	tuner = demod & 3;
+	if (fmode == 3)
+		tuner = 0;
+	dvb->fe = ddb_mci_attach(input, 0, demod, &dvb->set_input);
+	if (!dvb->fe) {
+		dev_err(dev->dev, "No MAXSX8 found!\n");
+		return -ENODEV;
+	}
+	if (!dvb->set_input) {
+		dev_err(dev->dev, "No MCI set_input function pointer!\n");
+		return -ENODEV;
+	}
+	if (input->nr < 4) {
+		lnb_command(dev, port->lnr, input->nr, LNB_CMD_INIT);
+		lnb_set_voltage(dev, port->lnr, input->nr, SEC_VOLTAGE_OFF);
+	}
+	ddb_lnb_init_fmode(dev, link, fmode);
+
+	dvb->fe->ops.set_voltage = max_set_voltage;
+	dvb->fe->ops.enable_high_lnb_voltage = max_enable_high_lnb_voltage;
+	dvb->fe->ops.set_tone = max_set_tone;
+	dvb->diseqc_send_master_cmd = dvb->fe->ops.diseqc_send_master_cmd;
+	dvb->fe->ops.diseqc_send_master_cmd = max_send_master_cmd;
+	dvb->fe->ops.diseqc_send_burst = max_send_burst;
+	dvb->fe->sec_priv = input;
+	dvb->input = tuner;
+	return 0;
+}
diff --git a/drivers/media/pci/ddbridge/ddbridge-max.h b/drivers/media/pci/ddbridge/ddbridge-max.h
index bf8bf38739f6..82efc53baa94 100644
--- a/drivers/media/pci/ddbridge/ddbridge-max.h
+++ b/drivers/media/pci/ddbridge/ddbridge-max.h
@@ -25,5 +25,6 @@
 
 int ddb_lnb_init_fmode(struct ddb *dev, struct ddb_link *link, u32 fm);
 int ddb_fe_attach_mxl5xx(struct ddb_input *input);
+int ddb_fe_attach_mci(struct ddb_input *input);
 
 #endif /* _DDBRIDGE_MAX_H */
diff --git a/drivers/media/pci/ddbridge/ddbridge-mci.c b/drivers/media/pci/ddbridge/ddbridge-mci.c
new file mode 100644
index 000000000000..4ac634fc96e4
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-mci.c
@@ -0,0 +1,551 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ddbridge-mci.c: Digital Devices microcode interface
+ *
+ * Copyright (C) 2017 Digital Devices GmbH
+ *                    Ralph Metzler <rjkm@metzlerbros.de>
+ *                    Marcus Metzler <mocm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include "ddbridge.h"
+#include "ddbridge-io.h"
+#include "ddbridge-mci.h"
+
+static LIST_HEAD(mci_list);
+
+static const u32 MCLK = (1550000000 / 12);
+static const u32 MAX_DEMOD_LDPC_BITRATE = (1550000000 / 6);
+static const u32 MAX_LDPC_BITRATE = (720000000);
+
+struct mci_base {
+	struct list_head     mci_list;
+	void                *key;
+	struct ddb_link     *link;
+	struct completion    completion;
+
+	struct device       *dev;
+	struct mutex         tuner_lock; /* concurrent tuner access lock */
+	u8                   adr;
+	struct mutex         mci_lock; /* concurrent MCI access lock */
+	int                  count;
+
+	u8                   tuner_use_count[MCI_TUNER_MAX];
+	u8                   assigned_demod[MCI_DEMOD_MAX];
+	u32                  used_ldpc_bitrate[MCI_DEMOD_MAX];
+	u8                   demod_in_use[MCI_DEMOD_MAX];
+	u32                  iq_mode;
+};
+
+struct mci {
+	struct mci_base     *base;
+	struct dvb_frontend  fe;
+	int                  nr;
+	int                  demod;
+	int                  tuner;
+	int                  first_time_lock;
+	int                  started;
+	struct mci_result    signal_info;
+
+	u32                  bb_mode;
+};
+
+static int mci_reset(struct mci *state)
+{
+	struct ddb_link *link = state->base->link;
+	u32 status = 0;
+	u32 timeout = 40;
+
+	ddblwritel(link, MCI_CONTROL_RESET, MCI_CONTROL);
+	ddblwritel(link, 0, MCI_CONTROL + 4); /* 1= no internal init */
+	msleep(300);
+	ddblwritel(link, 0, MCI_CONTROL);
+
+	while (1) {
+		status = ddblreadl(link, MCI_CONTROL);
+		if ((status & MCI_CONTROL_READY) == MCI_CONTROL_READY)
+			break;
+		if (--timeout == 0)
+			break;
+		msleep(50);
+	}
+	if ((status & MCI_CONTROL_READY) == 0)
+		return -1;
+	if (link->ids.device == 0x0009)
+		ddblwritel(link, SX8_TSCONFIG_MODE_NORMAL, SX8_TSCONFIG);
+	return 0;
+}
+
+static int mci_config(struct mci *state, u32 config)
+{
+	struct ddb_link *link = state->base->link;
+
+	if (link->ids.device != 0x0009)
+		return -EINVAL;
+	ddblwritel(link, config, SX8_TSCONFIG);
+	return 0;
+}
+
+static int _mci_cmd_unlocked(struct mci *state,
+			     u32 *cmd, u32 cmd_len,
+			     u32 *res, u32 res_len)
+{
+	struct ddb_link *link = state->base->link;
+	u32 i, val;
+	unsigned long stat;
+
+	val = ddblreadl(link, MCI_CONTROL);
+	if (val & (MCI_CONTROL_RESET | MCI_CONTROL_START_COMMAND))
+		return -EIO;
+	if (cmd && cmd_len)
+		for (i = 0; i < cmd_len; i++)
+			ddblwritel(link, cmd[i], MCI_COMMAND + i * 4);
+	val |= (MCI_CONTROL_START_COMMAND | MCI_CONTROL_ENABLE_DONE_INTERRUPT);
+	ddblwritel(link, val, MCI_CONTROL);
+
+	stat = wait_for_completion_timeout(&state->base->completion, HZ);
+	if (stat == 0) {
+		dev_warn(state->base->dev, "MCI-%d: MCI timeout\n", state->nr);
+		return -EIO;
+	}
+	if (res && res_len)
+		for (i = 0; i < res_len; i++)
+			res[i] = ddblreadl(link, MCI_RESULT + i * 4);
+	return 0;
+}
+
+static int mci_cmd(struct mci *state,
+		   struct mci_command *command,
+		   struct mci_result *result)
+{
+	int stat;
+
+	mutex_lock(&state->base->mci_lock);
+	stat = _mci_cmd_unlocked(state,
+				 (u32 *)command, sizeof(*command) / sizeof(u32),
+				 (u32 *)result, sizeof(*result) / sizeof(u32));
+	mutex_unlock(&state->base->mci_lock);
+	return stat;
+}
+
+static void mci_handler(void *priv)
+{
+	struct mci_base *base = (struct mci_base *)priv;
+
+	complete(&base->completion);
+}
+
+static void release(struct dvb_frontend *fe)
+{
+	struct mci *state = fe->demodulator_priv;
+
+	state->base->count--;
+	if (state->base->count == 0) {
+		list_del(&state->base->mci_list);
+		kfree(state->base);
+	}
+	kfree(state);
+}
+
+static int read_status(struct dvb_frontend *fe, enum fe_status *status)
+{
+	int stat;
+	struct mci *state = fe->demodulator_priv;
+	struct mci_command cmd;
+	struct mci_result res;
+
+	cmd.command = MCI_CMD_GETSTATUS;
+	cmd.demod = state->demod;
+	stat = mci_cmd(state, &cmd, &res);
+	if (stat)
+		return stat;
+	*status = 0x00;
+	if (res.status == SX8_DEMOD_WAIT_MATYPE)
+		*status = 0x0f;
+	if (res.status == SX8_DEMOD_LOCKED)
+		*status = 0x1f;
+	return stat;
+}
+
+static int mci_set_tuner(struct dvb_frontend *fe, u32 tuner, u32 on)
+{
+	struct mci *state = fe->demodulator_priv;
+	struct mci_command cmd;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.tuner = state->tuner;
+	cmd.command = on ? SX8_CMD_INPUT_ENABLE : SX8_CMD_INPUT_DISABLE;
+	return mci_cmd(state, &cmd, NULL);
+}
+
+static int stop(struct dvb_frontend *fe)
+{
+	struct mci *state = fe->demodulator_priv;
+	struct mci_command cmd;
+	u32 input = state->tuner;
+
+	memset(&cmd, 0, sizeof(cmd));
+	if (state->demod != DEMOD_UNUSED) {
+		cmd.command = MCI_CMD_STOP;
+		cmd.demod = state->demod;
+		mci_cmd(state, &cmd, NULL);
+		if (state->base->iq_mode) {
+			cmd.command = MCI_CMD_STOP;
+			cmd.demod = state->demod;
+			cmd.output = 0;
+			mci_cmd(state, &cmd, NULL);
+			mci_config(state, SX8_TSCONFIG_MODE_NORMAL);
+		}
+	}
+	mutex_lock(&state->base->tuner_lock);
+	state->base->tuner_use_count[input]--;
+	if (!state->base->tuner_use_count[input])
+		mci_set_tuner(fe, input, 0);
+	if (state->demod < MCI_DEMOD_MAX)
+		state->base->demod_in_use[state->demod] = 0;
+	state->base->used_ldpc_bitrate[state->nr] = 0;
+	state->demod = DEMOD_UNUSED;
+	state->base->assigned_demod[state->nr] = DEMOD_UNUSED;
+	state->base->iq_mode = 0;
+	mutex_unlock(&state->base->tuner_lock);
+	state->started = 0;
+	return 0;
+}
+
+static int start(struct dvb_frontend *fe, u32 flags, u32 modmask, u32 ts_config)
+{
+	struct mci *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	u32 used_ldpc_bitrate = 0, free_ldpc_bitrate;
+	u32 used_demods = 0;
+	struct mci_command cmd;
+	u32 input = state->tuner;
+	u32 bits_per_symbol = 0;
+	int i, stat = 0;
+
+	if (p->symbol_rate >= (MCLK / 2))
+		flags &= ~1;
+	if ((flags & 3) == 0)
+		return -EINVAL;
+
+	if (flags & 2) {
+		u32 tmp = modmask;
+
+		bits_per_symbol = 1;
+		while (tmp & 1) {
+			tmp >>= 1;
+			bits_per_symbol++;
+		}
+	}
+
+	mutex_lock(&state->base->tuner_lock);
+	if (state->base->iq_mode) {
+		stat = -EBUSY;
+		goto unlock;
+	}
+	for (i = 0; i < MCI_DEMOD_MAX; i++) {
+		used_ldpc_bitrate += state->base->used_ldpc_bitrate[i];
+		if (state->base->demod_in_use[i])
+			used_demods++;
+	}
+	if (used_ldpc_bitrate >= MAX_LDPC_BITRATE ||
+	    ((ts_config & SX8_TSCONFIG_MODE_MASK) >
+	     SX8_TSCONFIG_MODE_NORMAL && used_demods > 0)) {
+		stat = -EBUSY;
+		goto unlock;
+	}
+	free_ldpc_bitrate = MAX_LDPC_BITRATE - used_ldpc_bitrate;
+	if (free_ldpc_bitrate > MAX_DEMOD_LDPC_BITRATE)
+		free_ldpc_bitrate = MAX_DEMOD_LDPC_BITRATE;
+
+	while (p->symbol_rate * bits_per_symbol > free_ldpc_bitrate)
+		bits_per_symbol--;
+
+	if (bits_per_symbol < 2) {
+		stat = -EBUSY;
+		goto unlock;
+	}
+	i = (p->symbol_rate > (MCLK / 2)) ? 3 : 7;
+	while (i >= 0 && state->base->demod_in_use[i])
+		i--;
+	if (i < 0) {
+		stat = -EBUSY;
+		goto unlock;
+	}
+	state->base->demod_in_use[i] = 1;
+	state->base->used_ldpc_bitrate[state->nr] = p->symbol_rate
+						    * bits_per_symbol;
+	state->demod = i;
+	state->base->assigned_demod[state->nr] = i;
+
+	if (!state->base->tuner_use_count[input])
+		mci_set_tuner(fe, input, 1);
+	state->base->tuner_use_count[input]++;
+	state->base->iq_mode = (ts_config > 1);
+unlock:
+	mutex_unlock(&state->base->tuner_lock);
+	if (stat)
+		return stat;
+	memset(&cmd, 0, sizeof(cmd));
+
+	if (state->base->iq_mode) {
+		cmd.command = SX8_CMD_SELECT_IQOUT;
+		cmd.demod = state->demod;
+		cmd.output = 0;
+		mci_cmd(state, &cmd, NULL);
+		mci_config(state, ts_config);
+	}
+	if (p->stream_id != NO_STREAM_ID_FILTER && p->stream_id != 0x80000000)
+		flags |= 0x80;
+	dev_dbg(state->base->dev, "MCI-%d: tuner=%d demod=%d\n",
+		state->nr, state->tuner, state->demod);
+	cmd.command = MCI_CMD_SEARCH_DVBS;
+	cmd.dvbs2_search.flags = flags;
+	cmd.dvbs2_search.s2_modulation_mask =
+		modmask & ((1 << (bits_per_symbol - 1)) - 1);
+	cmd.dvbs2_search.retry = 2;
+	cmd.dvbs2_search.frequency = p->frequency * 1000;
+	cmd.dvbs2_search.symbol_rate = p->symbol_rate;
+	cmd.dvbs2_search.scrambling_sequence_index =
+		p->scrambling_sequence_index;
+	cmd.dvbs2_search.input_stream_id =
+		(p->stream_id != NO_STREAM_ID_FILTER) ? p->stream_id : 0;
+	cmd.tuner = state->tuner;
+	cmd.demod = state->demod;
+	cmd.output = state->nr;
+	if (p->stream_id == 0x80000000)
+		cmd.output |= 0x80;
+	stat = mci_cmd(state, &cmd, NULL);
+	if (stat)
+		stop(fe);
+	return stat;
+}
+
+static int start_iq(struct dvb_frontend *fe, u32 ts_config)
+{
+	struct mci *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	u32 used_demods = 0;
+	struct mci_command cmd;
+	u32 input = state->tuner;
+	int i, stat = 0;
+
+	mutex_lock(&state->base->tuner_lock);
+	if (state->base->iq_mode) {
+		stat = -EBUSY;
+		goto unlock;
+	}
+	for (i = 0; i < MCI_DEMOD_MAX; i++)
+		if (state->base->demod_in_use[i])
+			used_demods++;
+	if (used_demods > 0) {
+		stat = -EBUSY;
+		goto unlock;
+	}
+	state->demod = 0;
+	state->base->assigned_demod[state->nr] = 0;
+	if (!state->base->tuner_use_count[input])
+		mci_set_tuner(fe, input, 1);
+	state->base->tuner_use_count[input]++;
+	state->base->iq_mode = (ts_config > 1);
+unlock:
+	mutex_unlock(&state->base->tuner_lock);
+	if (stat)
+		return stat;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.command = SX8_CMD_START_IQ;
+	cmd.dvbs2_search.frequency = p->frequency * 1000;
+	cmd.dvbs2_search.symbol_rate = p->symbol_rate;
+	cmd.tuner = state->tuner;
+	cmd.demod = state->demod;
+	cmd.output = 7;
+	mci_config(state, ts_config);
+	stat = mci_cmd(state, &cmd, NULL);
+	if (stat)
+		stop(fe);
+	return stat;
+}
+
+static int set_parameters(struct dvb_frontend *fe)
+{
+	int stat = 0;
+	struct mci *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	u32 ts_config, iq_mode = 0, isi;
+
+	if (state->started)
+		stop(fe);
+
+	isi = p->stream_id;
+	if (isi != NO_STREAM_ID_FILTER)
+		iq_mode = (isi & 0x30000000) >> 28;
+
+	switch (iq_mode) {
+	case 1:
+		ts_config = (SX8_TSCONFIG_TSHEADER | SX8_TSCONFIG_MODE_IQ);
+		break;
+	case 2:
+		ts_config = (SX8_TSCONFIG_TSHEADER | SX8_TSCONFIG_MODE_IQ);
+		break;
+	default:
+		ts_config = SX8_TSCONFIG_MODE_NORMAL;
+		break;
+	}
+
+	if (iq_mode != 2) {
+		u32 flags = 3;
+		u32 mask = 3;
+
+		if (p->modulation == APSK_16 ||
+		    p->modulation == APSK_32) {
+			flags = 2;
+			mask = 15;
+		}
+		stat = start(fe, flags, mask, ts_config);
+	} else {
+		stat = start_iq(fe, ts_config);
+	}
+
+	if (!stat) {
+		state->started = 1;
+		state->first_time_lock = 1;
+		state->signal_info.status = SX8_DEMOD_WAIT_SIGNAL;
+	}
+
+	return stat;
+}
+
+static int tune(struct dvb_frontend *fe, bool re_tune,
+		unsigned int mode_flags,
+		unsigned int *delay, enum fe_status *status)
+{
+	int r;
+
+	if (re_tune) {
+		r = set_parameters(fe);
+		if (r)
+			return r;
+	}
+	r = read_status(fe, status);
+	if (r)
+		return r;
+
+	if (*status & FE_HAS_LOCK)
+		return 0;
+	*delay = HZ / 10;
+	return 0;
+}
+
+static enum dvbfe_algo get_algo(struct dvb_frontend *fe)
+{
+	return DVBFE_ALGO_HW;
+}
+
+static int set_input(struct dvb_frontend *fe, int input)
+{
+	struct mci *state = fe->demodulator_priv;
+
+	state->tuner = input;
+	dev_dbg(state->base->dev, "MCI-%d: input=%d\n", state->nr, input);
+	return 0;
+}
+
+static struct dvb_frontend_ops mci_ops = {
+	.delsys = { SYS_DVBS, SYS_DVBS2 },
+	.info = {
+		.name			= "Digital Devices MaxSX8 MCI DVB-S/S2/S2X",
+		.frequency_min		= 950000,
+		.frequency_max		= 2150000,
+		.frequency_stepsize	= 0,
+		.frequency_tolerance	= 0,
+		.symbol_rate_min	= 100000,
+		.symbol_rate_max	= 100000000,
+		.caps			= FE_CAN_INVERSION_AUTO |
+					  FE_CAN_FEC_AUTO       |
+					  FE_CAN_QPSK           |
+					  FE_CAN_2G_MODULATION  |
+					  FE_CAN_MULTISTREAM,
+	},
+	.get_frontend_algo		= get_algo,
+	.tune				= tune,
+	.release			= release,
+	.read_status			= read_status,
+};
+
+static struct mci_base *match_base(void *key)
+{
+	struct mci_base *p;
+
+	list_for_each_entry(p, &mci_list, mci_list)
+		if (p->key == key)
+			return p;
+	return NULL;
+}
+
+static int probe(struct mci *state)
+{
+	mci_reset(state);
+	return 0;
+}
+
+struct dvb_frontend
+*ddb_mci_attach(struct ddb_input *input,
+		int mci_type, int nr,
+		int (**fn_set_input)(struct dvb_frontend *fe, int input))
+{
+	struct ddb_port *port = input->port;
+	struct ddb *dev = port->dev;
+	struct ddb_link *link = &dev->link[port->lnr];
+	struct mci_base *base;
+	struct mci *state;
+	void *key = mci_type ? (void *)port : (void *)link;
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	base = match_base(key);
+	if (base) {
+		base->count++;
+		state->base = base;
+	} else {
+		base = kzalloc(sizeof(*base), GFP_KERNEL);
+		if (!base)
+			goto fail;
+		base->key = key;
+		base->count = 1;
+		base->link = link;
+		base->dev = dev->dev;
+		mutex_init(&base->mci_lock);
+		mutex_init(&base->tuner_lock);
+		ddb_irq_set(dev, link->nr, 0, mci_handler, base);
+		init_completion(&base->completion);
+		state->base = base;
+		if (probe(state) < 0) {
+			kfree(base);
+			goto fail;
+		}
+		list_add(&base->mci_list, &mci_list);
+	}
+	state->fe.ops = mci_ops;
+	state->fe.demodulator_priv = state;
+	state->nr = nr;
+	*fn_set_input = set_input;
+
+	state->tuner = nr;
+	state->demod = nr;
+
+	return &state->fe;
+fail:
+	kfree(state);
+	return NULL;
+}
diff --git a/drivers/media/pci/ddbridge/ddbridge-mci.h b/drivers/media/pci/ddbridge/ddbridge-mci.h
new file mode 100644
index 000000000000..209cc2b92dff
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-mci.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ddbridge-mci.h: Digital Devices micro code interface
+ *
+ * Copyright (C) 2017 Digital Devices GmbH
+ *                    Marcus Metzler <mocm@metzlerbros.de>
+ *                    Ralph Metzler <rjkm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DDBRIDGE_MCI_H_
+#define _DDBRIDGE_MCI_H_
+
+#define MCI_DEMOD_MAX                       8
+#define MCI_TUNER_MAX                       4
+#define DEMOD_UNUSED                        (0xFF)
+
+#define MCI_CONTROL                         (0x500)
+#define MCI_COMMAND                         (0x600)
+#define MCI_RESULT                          (0x680)
+
+#define MCI_COMMAND_SIZE                    (0x80)
+#define MCI_RESULT_SIZE                     (0x80)
+
+#define MCI_CONTROL_START_COMMAND           (0x00000001)
+#define MCI_CONTROL_ENABLE_DONE_INTERRUPT   (0x00000002)
+#define MCI_CONTROL_RESET                   (0x00008000)
+#define MCI_CONTROL_READY                   (0x00010000)
+
+#define SX8_TSCONFIG                        (0x280)
+
+#define SX8_TSCONFIG_MODE_MASK              (0x00000003)
+#define SX8_TSCONFIG_MODE_OFF               (0x00000000)
+#define SX8_TSCONFIG_MODE_NORMAL            (0x00000001)
+#define SX8_TSCONFIG_MODE_IQ                (0x00000003)
+
+#define SX8_TSCONFIG_TSHEADER               (0x00000004)
+#define SX8_TSCONFIG_BURST                  (0x00000008)
+
+#define SX8_TSCONFIG_BURSTSIZE_MASK         (0x00000030)
+#define SX8_TSCONFIG_BURSTSIZE_2K           (0x00000000)
+#define SX8_TSCONFIG_BURSTSIZE_4K           (0x00000010)
+#define SX8_TSCONFIG_BURSTSIZE_8K           (0x00000020)
+#define SX8_TSCONFIG_BURSTSIZE_16K          (0x00000030)
+
+#define SX8_DEMOD_STOPPED       (0)
+#define SX8_DEMOD_IQ_MODE       (1)
+#define SX8_DEMOD_WAIT_SIGNAL   (2)
+#define SX8_DEMOD_WAIT_MATYPE   (3)
+#define SX8_DEMOD_TIMEOUT       (14)
+#define SX8_DEMOD_LOCKED        (15)
+
+#define MCI_CMD_STOP            (0x01)
+#define MCI_CMD_GETSTATUS       (0x02)
+#define MCI_CMD_GETSIGNALINFO   (0x03)
+#define MCI_CMD_RFPOWER         (0x04)
+
+#define MCI_CMD_SEARCH_DVBS     (0x10)
+
+#define MCI_CMD_GET_IQSYMBOL    (0x30)
+
+#define SX8_CMD_INPUT_ENABLE    (0x40)
+#define SX8_CMD_INPUT_DISABLE   (0x41)
+#define SX8_CMD_START_IQ        (0x42)
+#define SX8_CMD_STOP_IQ         (0x43)
+#define SX8_CMD_SELECT_IQOUT    (0x44)
+#define SX8_CMD_SELECT_TSOUT    (0x45)
+
+#define SX8_ERROR_UNSUPPORTED   (0x80)
+
+#define SX8_SUCCESS(status)     (status < SX8_ERROR_UNSUPPORTED)
+
+#define SX8_CMD_DIAG_READ8      (0xE0)
+#define SX8_CMD_DIAG_READ32     (0xE1)
+#define SX8_CMD_DIAG_WRITE8     (0xE2)
+#define SX8_CMD_DIAG_WRITE32    (0xE3)
+
+#define SX8_CMD_DIAG_READRF     (0xE8)
+#define SX8_CMD_DIAG_WRITERF    (0xE9)
+
+struct mci_command {
+	union {
+		u32 command_word;
+		struct {
+			u8 command;
+			u8 tuner;
+			u8 demod;
+			u8 output;
+		};
+	};
+	union {
+		u32 params[31];
+		struct {
+			u8  flags;
+			u8  s2_modulation_mask;
+			u8  rsvd1;
+			u8  retry;
+			u32 frequency;
+			u32 symbol_rate;
+			u8  input_stream_id;
+			u8  rsvd2[3];
+			u32 scrambling_sequence_index;
+		} dvbs2_search;
+	};
+};
+
+struct mci_result {
+	union {
+		u32 status_word;
+		struct {
+			u8 status;
+			u8 rsvd;
+			u16 time;
+		};
+	};
+	union {
+		u32 result[27];
+		struct {
+			u8  standard;
+			/* puncture rate for DVB-S */
+			u8  pls_code;
+			/* 7-6: rolloff, 5-2: rsrvd, 1:short, 0:pilots */
+			u8  roll_off;
+			u8  rsvd;
+			u32 frequency;
+			u32 symbol_rate;
+			s16 channel_power;
+			s16 band_power;
+			s16 signal_to_noise;
+			s16 rsvd2;
+			u32 packet_errors;
+			u32 ber_numerator;
+			u32 ber_denominator;
+		} dvbs2_signal_info;
+		struct {
+			u8 i_symbol;
+			u8 q_symbol;
+		} dvbs2_signal_iq;
+	};
+	u32 version[4];
+};
+
+struct dvb_frontend
+*ddb_mci_attach(struct ddb_input *input,
+		int mci_type, int nr,
+		int (**fn_set_input)(struct dvb_frontend *fe, int input));
+
+#endif /* _DDBRIDGE_MCI_H_ */
diff --git a/drivers/media/pci/ddbridge/ddbridge-regs.h b/drivers/media/pci/ddbridge/ddbridge-regs.h
index 23d74ff83fe4..b978b5991940 100644
--- a/drivers/media/pci/ddbridge/ddbridge-regs.h
+++ b/drivers/media/pci/ddbridge/ddbridge-regs.h
@@ -17,6 +17,9 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
+#ifndef __DDBRIDGE_REGS_H__
+#define __DDBRIDGE_REGS_H__
+
 /* ------------------------------------------------------------------------- */
 /* SPI Controller */
 
@@ -154,3 +157,4 @@
 #define LNB_BUF_LEVEL(i)		(LNB_BASE + (i) * 0x20 + 0x10)
 #define LNB_BUF_WRITE(i)		(LNB_BASE + (i) * 0x20 + 0x14)
 
+#endif /* __DDBRIDGE_REGS_H__ */
diff --git a/drivers/media/pci/ddbridge/ddbridge.h b/drivers/media/pci/ddbridge/ddbridge.h
index f223dc6c9963..a66b1125cc74 100644
--- a/drivers/media/pci/ddbridge/ddbridge.h
+++ b/drivers/media/pci/ddbridge/ddbridge.h
@@ -63,7 +63,7 @@
 #include <media/dvb_ca_en50221.h>
 #include <media/dvb_net.h>
 
-#define DDBRIDGE_VERSION "0.9.32-integrated"
+#define DDBRIDGE_VERSION "0.9.33-integrated"
 
 #define DDB_MAX_I2C    32
 #define DDB_MAX_PORT   32
@@ -112,11 +112,12 @@ struct ddb_ids {
 
 struct ddb_info {
 	int   type;
-#define DDB_NONE         0
-#define DDB_OCTOPUS      1
-#define DDB_OCTOPUS_CI   2
-#define DDB_OCTOPUS_MAX  5
+#define DDB_NONE            0
+#define DDB_OCTOPUS         1
+#define DDB_OCTOPUS_CI      2
+#define DDB_OCTOPUS_MAX     5
 #define DDB_OCTOPUS_MAX_CT  6
+#define DDB_OCTOPUS_MCI     9
 	char *name;
 	u32   i2c_mask;
 	u8    port_num;
@@ -133,23 +134,12 @@ struct ddb_info {
 #define TS_QUIRK_REVERSED 2
 #define TS_QUIRK_ALT_OSC  8
 	u32   tempmon_irq;
+	u8    mci;
 	const struct ddb_regmap *regmap;
 };
 
-/* DMA_SIZE MUST be smaller than 256k and
- * MUST be divisible by 188 and 128 !!!
- */
-
 #define DMA_MAX_BUFS 32      /* hardware table limit */
 
-#define INPUT_DMA_BUFS 8
-#define INPUT_DMA_SIZE (128 * 47 * 21)
-#define INPUT_DMA_IRQ_DIV 1
-
-#define OUTPUT_DMA_BUFS 8
-#define OUTPUT_DMA_SIZE (128 * 47 * 21)
-#define OUTPUT_DMA_IRQ_DIV 1
-
 struct ddb;
 struct ddb_port;
 
@@ -248,6 +238,7 @@ struct ddb_port {
 	char                   *name;
 	char                   *type_name;
 	u32                     type;
+#define DDB_TUNER_DUMMY          0xffffffff
 #define DDB_TUNER_NONE           0
 #define DDB_TUNER_DVBS_ST        1
 #define DDB_TUNER_DVBS_ST_AA     2
@@ -264,6 +255,7 @@ struct ddb_port {
 #define DDB_CI_EXTERNAL_XO2_B    13
 #define DDB_TUNER_DVBS_STV0910_PR 14
 #define DDB_TUNER_DVBC2T2I_SONY_P 15
+#define DDB_TUNER_MCI            16
 
 #define DDB_TUNER_XO2            32
 #define DDB_TUNER_DVBS_STV0910   (DDB_TUNER_XO2 + 0)
@@ -305,6 +297,11 @@ struct ddb_lnb {
 	u32                    fmode;
 };
 
+struct ddb_irq {
+	void                   (*handler)(void *);
+	void                  *data;
+};
+
 struct ddb_link {
 	struct ddb            *dev;
 	const struct ddb_info *info;
@@ -319,6 +316,7 @@ struct ddb_link {
 	spinlock_t             temp_lock; /* lock temp chip access */
 	int                    overtemperature_error;
 	u8                     temp_tab[11];
+	struct ddb_irq         irq[256];
 };
 
 struct ddb {
@@ -343,9 +341,6 @@ struct ddb {
 	struct ddb_dma           idma[DDB_MAX_INPUT];
 	struct ddb_dma           odma[DDB_MAX_OUTPUT];
 
-	void                     (*handler[4][256])(unsigned long);
-	unsigned long            handler_data[4][256];
-
 	struct device           *ddb_dev;
 	u32                      ddb_dev_users;
 	u32                      nr;
@@ -368,16 +363,9 @@ int ddbridge_flashread(struct ddb *dev, u32 link, u8 *buf, u32 addr, u32 len);
 
 /****************************************************************************/
 
-/* ddbridge-main.c (modparams) */
-extern int ci_bitrate;
-extern int ts_loop;
-extern int xo2_speed;
-extern int alt_dma;
-extern int no_init;
-extern int stv0910_single;
-extern struct workqueue_struct *ddb_wq;
-
 /* ddbridge-core.c */
+struct ddb_irq *ddb_irq_set(struct ddb *dev, u32 link, u32 nr,
+			    void (*handler)(void *), void *data);
 void ddb_ports_detach(struct ddb *dev);
 void ddb_ports_release(struct ddb *dev);
 void ddb_buffers_free(struct ddb *dev);
@@ -389,9 +377,9 @@ void ddb_ports_init(struct ddb *dev);
 int ddb_buffers_alloc(struct ddb *dev);
 int ddb_ports_attach(struct ddb *dev);
 int ddb_device_create(struct ddb *dev);
-int ddb_class_create(void);
-void ddb_class_destroy(void);
 int ddb_init(struct ddb *dev);
 void ddb_unmap(struct ddb *dev);
+int ddb_exit_ddbridge(int stage, int error);
+int ddb_init_ddbridge(void);
 
 #endif /* DDBRIDGE_H */
diff --git a/drivers/media/pci/dt3155/Kconfig b/drivers/media/pci/dt3155/Kconfig
index 5145e0dfa2aa..858b0f2f15be 100644
--- a/drivers/media/pci/dt3155/Kconfig
+++ b/drivers/media/pci/dt3155/Kconfig
@@ -1,7 +1,6 @@
 config VIDEO_DT3155
 	tristate "DT3155 frame grabber"
 	depends on PCI && VIDEO_DEV && VIDEO_V4L2
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	default n
 	---help---
diff --git a/drivers/media/pci/intel/ipu3/Kconfig b/drivers/media/pci/intel/ipu3/Kconfig
index a82d3fe277d2..715f77651482 100644
--- a/drivers/media/pci/intel/ipu3/Kconfig
+++ b/drivers/media/pci/intel/ipu3/Kconfig
@@ -2,18 +2,16 @@ config VIDEO_IPU3_CIO2
 	tristate "Intel ipu3-cio2 driver"
 	depends on VIDEO_V4L2 && PCI
 	depends on VIDEO_V4L2_SUBDEV_API
-	depends on X86 || COMPILE_TEST
+	depends on (X86 && ACPI) || COMPILE_TEST
 	depends on MEDIA_CONTROLLER
-	depends on HAS_DMA
-	depends on ACPI
 	select V4L2_FWNODE
 	select VIDEOBUF2_DMA_SG
 
 	---help---
-	This is the Intel IPU3 CIO2 CSI-2 receiver unit, found in Intel
-	Skylake and Kaby Lake SoCs and used for capturing images and
-	video from a camera sensor.
+	  This is the Intel IPU3 CIO2 CSI-2 receiver unit, found in Intel
+	  Skylake and Kaby Lake SoCs and used for capturing images and
+	  video from a camera sensor.
 
-	Say Y or M here if you have a Skylake/Kaby Lake SoC with MIPI CSI-2
-	connected camera.
-	The module will be called ipu3-cio2.
+	  Say Y or M here if you have a Skylake/Kaby Lake SoC with MIPI CSI-2
+	  connected camera.
+	  The module will be called ipu3-cio2.
diff --git a/drivers/media/pci/intel/ipu3/ipu3-cio2.c b/drivers/media/pci/intel/ipu3/ipu3-cio2.c
index 7d768ec0f824..29027159eced 100644
--- a/drivers/media/pci/intel/ipu3/ipu3-cio2.c
+++ b/drivers/media/pci/intel/ipu3/ipu3-cio2.c
@@ -640,18 +640,10 @@ static const char *const cio2_port_errs[] = {
 	"PKT2LONG",
 };
 
-static irqreturn_t cio2_irq(int irq, void *cio2_ptr)
+static void cio2_irq_handle_once(struct cio2_device *cio2, u32 int_status)
 {
-	struct cio2_device *cio2 = cio2_ptr;
 	void __iomem *const base = cio2->base;
 	struct device *dev = &cio2->pci_dev->dev;
-	u32 int_status, int_clear;
-
-	int_status = readl(base + CIO2_REG_INT_STS);
-	int_clear = int_status;
-
-	if (!int_status)
-		return IRQ_NONE;
 
 	if (int_status & CIO2_INT_IOOE) {
 		/*
@@ -770,9 +762,29 @@ static irqreturn_t cio2_irq(int irq, void *cio2_ptr)
 		int_status &= ~(CIO2_INT_IOIE | CIO2_INT_IOIRQ);
 	}
 
-	writel(int_clear, base + CIO2_REG_INT_STS);
 	if (int_status)
 		dev_warn(dev, "unknown interrupt 0x%x on INT\n", int_status);
+}
+
+static irqreturn_t cio2_irq(int irq, void *cio2_ptr)
+{
+	struct cio2_device *cio2 = cio2_ptr;
+	void __iomem *const base = cio2->base;
+	struct device *dev = &cio2->pci_dev->dev;
+	u32 int_status;
+
+	int_status = readl(base + CIO2_REG_INT_STS);
+	dev_dbg(dev, "isr enter - interrupt status 0x%x\n", int_status);
+	if (!int_status)
+		return IRQ_NONE;
+
+	do {
+		writel(int_status, base + CIO2_REG_INT_STS);
+		cio2_irq_handle_once(cio2, int_status);
+		int_status = readl(base + CIO2_REG_INT_STS);
+		if (int_status)
+			dev_dbg(dev, "pending status 0x%x\n", int_status);
+	} while (int_status);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/media/pci/mantis/mantis_uart.c b/drivers/media/pci/mantis/mantis_uart.c
index 18f81c135996..b7765687e0c3 100644
--- a/drivers/media/pci/mantis/mantis_uart.c
+++ b/drivers/media/pci/mantis/mantis_uart.c
@@ -92,6 +92,7 @@ static void mantis_uart_work(struct work_struct *work)
 {
 	struct mantis_pci *mantis = container_of(work, struct mantis_pci, uart_work);
 	u32 stat;
+	unsigned long timeout;
 
 	stat = mmread(MANTIS_UART_STAT);
 
@@ -102,9 +103,15 @@ static void mantis_uart_work(struct work_struct *work)
 	 * MANTIS_UART_RXFIFO_DATA is only set if at least
 	 * config->bytes + 1 bytes are in the FIFO.
 	 */
+
+	/* FIXME: is 10ms good enough ? */
+	timeout = jiffies +  msecs_to_jiffies(10);
 	while (stat & MANTIS_UART_RXFIFO_DATA) {
 		mantis_uart_read(mantis);
 		stat = mmread(MANTIS_UART_STAT);
+
+		if (!time_is_after_jiffies(timeout))
+			break;
 	}
 
 	/* re-enable UART (RX) interrupt */
diff --git a/drivers/media/pci/meye/Kconfig b/drivers/media/pci/meye/Kconfig
index b4bf848be5a0..2e60334ffef5 100644
--- a/drivers/media/pci/meye/Kconfig
+++ b/drivers/media/pci/meye/Kconfig
@@ -1,6 +1,7 @@
 config VIDEO_MEYE
 	tristate "Sony Vaio Picturebook Motion Eye Video For Linux"
-	depends on PCI && SONY_LAPTOP && VIDEO_V4L2
+	depends on PCI && VIDEO_V4L2
+	depends on SONY_LAPTOP || COMPILE_TEST
 	---help---
 	  This is the video4linux driver for the Motion Eye camera found
 	  in the Vaio Picturebook laptops. Please read the material in
diff --git a/drivers/media/pci/ngene/ngene-cards.c b/drivers/media/pci/ngene/ngene-cards.c
index 65fc8f23ad86..7a106bc11a2b 100644
--- a/drivers/media/pci/ngene/ngene-cards.c
+++ b/drivers/media/pci/ngene/ngene-cards.c
@@ -137,11 +137,6 @@ static int tuner_attach_stv6110(struct ngene_channel *chan)
 		chan->dev->card_info->tuner_config[chan->number];
 	const struct stv6110x_devctl *ctl;
 
-	if (chan->number < 2)
-		i2c = &chan->dev->channel[0].i2c_adapter;
-	else
-		i2c = &chan->dev->channel[1].i2c_adapter;
-
 	ctl = dvb_attach(stv6110x_attach, chan->fe, tunerconf, i2c);
 	if (ctl == NULL) {
 		dev_err(pdev, "No STV6110X found!\n");
@@ -304,14 +299,6 @@ static int demod_attach_stv0900(struct ngene_channel *chan)
 	struct stv090x_config *feconf = (struct stv090x_config *)
 		chan->dev->card_info->fe_config[chan->number];
 
-	/* tuner 1+2: i2c adapter #0, tuner 3+4: i2c adapter #1 */
-	/* Note: Both adapters share the same i2c bus, but the demod     */
-	/*       driver requires that each demod has its own i2c adapter */
-	if (chan->number < 2)
-		i2c = &chan->dev->channel[0].i2c_adapter;
-	else
-		i2c = &chan->dev->channel[1].i2c_adapter;
-
 	chan->fe = dvb_attach(stv090x_attach, feconf, i2c,
 			(chan->number & 1) == 0 ? STV090x_DEMODULATOR_0
 						: STV090x_DEMODULATOR_1);
@@ -340,6 +327,7 @@ static struct stv0910_cfg stv0910_p = {
 	.parallel = 1,
 	.rptlvl   = 4,
 	.clk      = 30000000,
+	.tsspeed  = 0x28,
 };
 
 static struct lnbh25_config lnbh25_cfg = {
@@ -721,7 +709,6 @@ static int cineS2_probe(struct ngene_channel *chan)
 
 
 static struct lgdt330x_config aver_m780 = {
-	.demod_address = 0xb2 >> 1,
 	.demod_chip    = LGDT3303,
 	.serial_mpeg   = 0x00, /* PARALLEL */
 	.clock_polarity_flip = 1,
@@ -738,7 +725,8 @@ static int demod_attach_lg330x(struct ngene_channel *chan)
 {
 	struct device *pdev = &chan->dev->pci_dev->dev;
 
-	chan->fe = dvb_attach(lgdt330x_attach, &aver_m780, &chan->i2c_adapter);
+	chan->fe = dvb_attach(lgdt330x_attach, &aver_m780,
+			      0xb2 >> 1, &chan->i2c_adapter);
 	if (chan->fe == NULL) {
 		dev_err(pdev, "No LGDT330x found!\n");
 		return -ENODEV;
diff --git a/drivers/media/pci/ngene/ngene-dvb.c b/drivers/media/pci/ngene/ngene-dvb.c
index fee89b9ed9c1..5147e83397a1 100644
--- a/drivers/media/pci/ngene/ngene-dvb.c
+++ b/drivers/media/pci/ngene/ngene-dvb.c
@@ -40,7 +40,7 @@
 
 static int ci_tsfix = 1;
 module_param(ci_tsfix, int, 0444);
-MODULE_PARM_DESC(ci_tsfix, "Detect and fix TS buffer offset shifs in conjunction with CI expansions (default: 1/enabled)");
+MODULE_PARM_DESC(ci_tsfix, "Detect and fix TS buffer offset shifts in conjunction with CI expansions (default: 1/enabled)");
 
 /****************************************************************************/
 /* COMMAND API interface ****************************************************/
diff --git a/drivers/media/pci/pt1/Kconfig b/drivers/media/pci/pt1/Kconfig
index 24501d5bf70d..2718b4c6b7c6 100644
--- a/drivers/media/pci/pt1/Kconfig
+++ b/drivers/media/pci/pt1/Kconfig
@@ -1,6 +1,9 @@
 config DVB_PT1
 	tristate "PT1 cards"
 	depends on DVB_CORE && PCI && I2C
+	select DVB_TC90522 if MEDIA_SUBDRV_AUTOSELECT
+	select DVB_PLL if MEDIA_SUBDRV_AUTOSELECT
+	select MEDIA_TUNER_QM1D1B0004 if MEDIA_SUBDRV_AUTOSELECT
 	help
 	  Support for Earthsoft PT1 PCI cards.
 
diff --git a/drivers/media/pci/pt1/Makefile b/drivers/media/pci/pt1/Makefile
index ab873ae088a0..bc491e08dd63 100644
--- a/drivers/media/pci/pt1/Makefile
+++ b/drivers/media/pci/pt1/Makefile
@@ -1,5 +1,6 @@
-earth-pt1-objs := pt1.o va1j5jf8007s.o va1j5jf8007t.o
+earth-pt1-objs := pt1.o
 
 obj-$(CONFIG_DVB_PT1) += earth-pt1.o
 
 ccflags-y += -Idrivers/media/dvb-frontends
+ccflags-y += -Idrivers/media/tuners
diff --git a/drivers/media/pci/pt1/pt1.c b/drivers/media/pci/pt1/pt1.c
index 4f6867af8311..5708f69622cc 100644
--- a/drivers/media/pci/pt1/pt1.c
+++ b/drivers/media/pci/pt1/pt1.c
@@ -18,7 +18,10 @@
  */
 
 #include <linux/kernel.h>
+#include <linux/sched.h>
 #include <linux/sched/signal.h>
+#include <linux/hrtimer.h>
+#include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
@@ -26,6 +29,8 @@
 #include <linux/kthread.h>
 #include <linux/freezer.h>
 #include <linux/ratelimit.h>
+#include <linux/string.h>
+#include <linux/i2c.h>
 
 #include <media/dvbdev.h>
 #include <media/dvb_demux.h>
@@ -33,8 +38,9 @@
 #include <media/dvb_net.h>
 #include <media/dvb_frontend.h>
 
-#include "va1j5jf8007t.h"
-#include "va1j5jf8007s.h"
+#include "tc90522.h"
+#include "qm1d1b0004.h"
+#include "dvb-pll.h"
 
 #define DRIVER_NAME "earth-pt1"
 
@@ -63,6 +69,11 @@ struct pt1_table {
 	struct pt1_buffer bufs[PT1_NR_BUFS];
 };
 
+enum pt1_fe_clk {
+	PT1_FE_CLK_20MHZ,	/* PT1 */
+	PT1_FE_CLK_25MHZ,	/* PT2 */
+};
+
 #define PT1_NR_ADAPS 4
 
 struct pt1_adapter;
@@ -81,6 +92,8 @@ struct pt1 {
 	struct mutex lock;
 	int power;
 	int reset;
+
+	enum pt1_fe_clk fe_clk;
 };
 
 struct pt1_adapter {
@@ -97,6 +110,8 @@ struct pt1_adapter {
 	int users;
 	struct dmxdev dmxdev;
 	struct dvb_frontend *fe;
+	struct i2c_client *demod_i2c_client;
+	struct i2c_client *tuner_i2c_client;
 	int (*orig_set_voltage)(struct dvb_frontend *fe,
 				enum fe_sec_voltage voltage);
 	int (*orig_sleep)(struct dvb_frontend *fe);
@@ -106,6 +121,145 @@ struct pt1_adapter {
 	int sleep;
 };
 
+union pt1_tuner_config {
+	struct qm1d1b0004_config qm1d1b0004;
+	struct dvb_pll_config tda6651;
+};
+
+struct pt1_config {
+	struct i2c_board_info demod_info;
+	struct tc90522_config demod_cfg;
+
+	struct i2c_board_info tuner_info;
+	union pt1_tuner_config tuner_cfg;
+};
+
+static const struct pt1_config pt1_configs[PT1_NR_ADAPS] = {
+	{
+		.demod_info = {
+			I2C_BOARD_INFO(TC90522_I2C_DEV_SAT, 0x1b),
+		},
+		.tuner_info = {
+			I2C_BOARD_INFO("qm1d1b0004", 0x60),
+		},
+	},
+	{
+		.demod_info = {
+			I2C_BOARD_INFO(TC90522_I2C_DEV_TER, 0x1a),
+		},
+		.tuner_info = {
+			I2C_BOARD_INFO("tda665x_earthpt1", 0x61),
+		},
+	},
+	{
+		.demod_info = {
+			I2C_BOARD_INFO(TC90522_I2C_DEV_SAT, 0x19),
+		},
+		.tuner_info = {
+			I2C_BOARD_INFO("qm1d1b0004", 0x60),
+		},
+	},
+	{
+		.demod_info = {
+			I2C_BOARD_INFO(TC90522_I2C_DEV_TER, 0x18),
+		},
+		.tuner_info = {
+			I2C_BOARD_INFO("tda665x_earthpt1", 0x61),
+		},
+	},
+};
+
+static const u8 va1j5jf8007s_20mhz_configs[][2] = {
+	{0x04, 0x02}, {0x0d, 0x55}, {0x11, 0x40}, {0x13, 0x80}, {0x17, 0x01},
+	{0x1c, 0x0a}, {0x1d, 0xaa}, {0x1e, 0x20}, {0x1f, 0x88}, {0x51, 0xb0},
+	{0x52, 0x89}, {0x53, 0xb3}, {0x5a, 0x2d}, {0x5b, 0xd3}, {0x85, 0x69},
+	{0x87, 0x04}, {0x8e, 0x02}, {0xa3, 0xf7}, {0xa5, 0xc0},
+};
+
+static const u8 va1j5jf8007s_25mhz_configs[][2] = {
+	{0x04, 0x02}, {0x11, 0x40}, {0x13, 0x80}, {0x17, 0x01}, {0x1c, 0x0a},
+	{0x1d, 0xaa}, {0x1e, 0x20}, {0x1f, 0x88}, {0x51, 0xb0}, {0x52, 0x89},
+	{0x53, 0xb3}, {0x5a, 0x2d}, {0x5b, 0xd3}, {0x85, 0x69}, {0x87, 0x04},
+	{0x8e, 0x26}, {0xa3, 0xf7}, {0xa5, 0xc0},
+};
+
+static const u8 va1j5jf8007t_20mhz_configs[][2] = {
+	{0x03, 0x90}, {0x14, 0x8f}, {0x1c, 0x2a}, {0x1d, 0xa8}, {0x1e, 0xa2},
+	{0x22, 0x83}, {0x31, 0x0d}, {0x32, 0xe0}, {0x39, 0xd3}, {0x3a, 0x00},
+	{0x3b, 0x11}, {0x3c, 0x3f},
+	{0x5c, 0x40}, {0x5f, 0x80}, {0x75, 0x02}, {0x76, 0x4e}, {0x77, 0x03},
+	{0xef, 0x01}
+};
+
+static const u8 va1j5jf8007t_25mhz_configs[][2] = {
+	{0x03, 0x90}, {0x1c, 0x2a}, {0x1d, 0xa8}, {0x1e, 0xa2}, {0x22, 0x83},
+	{0x3a, 0x04}, {0x3b, 0x11}, {0x3c, 0x3f}, {0x5c, 0x40}, {0x5f, 0x80},
+	{0x75, 0x0a}, {0x76, 0x4c}, {0x77, 0x03}, {0xef, 0x01}
+};
+
+static int config_demod(struct i2c_client *cl, enum pt1_fe_clk clk)
+{
+	int ret;
+	u8 buf[2] = {0x01, 0x80};
+	bool is_sat;
+	const u8 (*cfg_data)[2];
+	int i, len;
+
+	ret = i2c_master_send(cl, buf, 2);
+	if (ret < 0)
+		return ret;
+	usleep_range(30000, 50000);
+
+	is_sat = !strncmp(cl->name, TC90522_I2C_DEV_SAT,
+			  strlen(TC90522_I2C_DEV_SAT));
+	if (is_sat) {
+		struct i2c_msg msg[2];
+		u8 wbuf, rbuf;
+
+		wbuf = 0x07;
+		msg[0].addr = cl->addr;
+		msg[0].flags = 0;
+		msg[0].len = 1;
+		msg[0].buf = &wbuf;
+
+		msg[1].addr = cl->addr;
+		msg[1].flags = I2C_M_RD;
+		msg[1].len = 1;
+		msg[1].buf = &rbuf;
+		ret = i2c_transfer(cl->adapter, msg, 2);
+		if (ret < 0)
+			return ret;
+		if (rbuf != 0x41)
+			return -EIO;
+	}
+
+	/* frontend init */
+	if (clk == PT1_FE_CLK_20MHZ) {
+		if (is_sat) {
+			cfg_data = va1j5jf8007s_20mhz_configs;
+			len = ARRAY_SIZE(va1j5jf8007s_20mhz_configs);
+		} else {
+			cfg_data = va1j5jf8007t_20mhz_configs;
+			len = ARRAY_SIZE(va1j5jf8007t_20mhz_configs);
+		}
+	} else {
+		if (is_sat) {
+			cfg_data = va1j5jf8007s_25mhz_configs;
+			len = ARRAY_SIZE(va1j5jf8007s_25mhz_configs);
+		} else {
+			cfg_data = va1j5jf8007t_25mhz_configs;
+			len = ARRAY_SIZE(va1j5jf8007t_25mhz_configs);
+		}
+	}
+
+	for (i = 0; i < len; i++) {
+		ret = i2c_master_send(cl, cfg_data[i], 2);
+		if (ret < 0)
+			return ret;
+	}
+	return 0;
+}
+
 static void pt1_write_reg(struct pt1 *pt1, int reg, u32 data)
 {
 	writel(data, pt1->regs + reg * 4);
@@ -171,7 +325,7 @@ static int pt1_unlock(struct pt1 *pt1)
 	for (i = 0; i < 3; i++) {
 		if (pt1_read_reg(pt1, 0) & 0x80000000)
 			return 0;
-		schedule_timeout_uninterruptible((HZ + 999) / 1000);
+		usleep_range(1000, 2000);
 	}
 	dev_err(&pt1->pdev->dev, "could not unlock\n");
 	return -EIO;
@@ -185,7 +339,7 @@ static int pt1_reset_pci(struct pt1 *pt1)
 	for (i = 0; i < 10; i++) {
 		if (pt1_read_reg(pt1, 0) & 0x00000001)
 			return 0;
-		schedule_timeout_uninterruptible((HZ + 999) / 1000);
+		usleep_range(1000, 2000);
 	}
 	dev_err(&pt1->pdev->dev, "could not reset PCI\n");
 	return -EIO;
@@ -199,7 +353,7 @@ static int pt1_reset_ram(struct pt1 *pt1)
 	for (i = 0; i < 10; i++) {
 		if (pt1_read_reg(pt1, 0) & 0x00000002)
 			return 0;
-		schedule_timeout_uninterruptible((HZ + 999) / 1000);
+		usleep_range(1000, 2000);
 	}
 	dev_err(&pt1->pdev->dev, "could not reset RAM\n");
 	return -EIO;
@@ -216,7 +370,7 @@ static int pt1_do_enable_ram(struct pt1 *pt1)
 			if ((pt1_read_reg(pt1, 0) & 0x00000004) != status)
 				return 0;
 		}
-		schedule_timeout_uninterruptible((HZ + 999) / 1000);
+		usleep_range(1000, 2000);
 	}
 	dev_err(&pt1->pdev->dev, "could not enable RAM\n");
 	return -EIO;
@@ -226,7 +380,7 @@ static int pt1_enable_ram(struct pt1 *pt1)
 {
 	int i, ret;
 	int phase;
-	schedule_timeout_uninterruptible((HZ + 999) / 1000);
+	usleep_range(1000, 2000);
 	phase = pt1->pdev->device == 0x211a ? 128 : 166;
 	for (i = 0; i < phase; i++) {
 		ret = pt1_do_enable_ram(pt1);
@@ -311,16 +465,31 @@ static int pt1_thread(void *data)
 {
 	struct pt1 *pt1;
 	struct pt1_buffer_page *page;
+	bool was_frozen;
+
+#define PT1_FETCH_DELAY 10
+#define PT1_FETCH_DELAY_DELTA 2
 
 	pt1 = data;
 	set_freezable();
 
-	while (!kthread_should_stop()) {
-		try_to_freeze();
+	while (!kthread_freezable_should_stop(&was_frozen)) {
+		if (was_frozen) {
+			int i;
+
+			for (i = 0; i < PT1_NR_ADAPS; i++)
+				pt1_set_stream(pt1, i, !!pt1->adaps[i]->users);
+		}
 
 		page = pt1->tables[pt1->table_index].bufs[pt1->buf_index].page;
 		if (!pt1_filter(pt1, page)) {
-			schedule_timeout_interruptible((HZ + 999) / 1000);
+			ktime_t delay;
+
+			delay = ktime_set(0, PT1_FETCH_DELAY * NSEC_PER_MSEC);
+			set_current_state(TASK_INTERRUPTIBLE);
+			schedule_hrtimeout_range(&delay,
+					PT1_FETCH_DELAY_DELTA * NSEC_PER_MSEC,
+					HRTIMER_MODE_REL);
 			continue;
 		}
 
@@ -556,7 +725,7 @@ pt1_update_power(struct pt1 *pt1)
 		adap = pt1->adaps[i];
 		switch (adap->voltage) {
 		case SEC_VOLTAGE_13: /* actually 11V */
-			bits |= 1 << 1;
+			bits |= 1 << 2;
 			break;
 		case SEC_VOLTAGE_18: /* actually 15V */
 			bits |= 1 << 1 | 1 << 2;
@@ -589,30 +758,33 @@ static int pt1_set_voltage(struct dvb_frontend *fe, enum fe_sec_voltage voltage)
 static int pt1_sleep(struct dvb_frontend *fe)
 {
 	struct pt1_adapter *adap;
+	int ret;
 
 	adap = container_of(fe->dvb, struct pt1_adapter, adap);
-	adap->sleep = 1;
-	pt1_update_power(adap->pt1);
 
+	ret = 0;
 	if (adap->orig_sleep)
-		return adap->orig_sleep(fe);
-	else
-		return 0;
+		ret = adap->orig_sleep(fe);
+
+	adap->sleep = 1;
+	pt1_update_power(adap->pt1);
+	return ret;
 }
 
 static int pt1_wakeup(struct dvb_frontend *fe)
 {
 	struct pt1_adapter *adap;
+	int ret;
 
 	adap = container_of(fe->dvb, struct pt1_adapter, adap);
 	adap->sleep = 0;
 	pt1_update_power(adap->pt1);
-	schedule_timeout_uninterruptible((HZ + 999) / 1000);
+	usleep_range(1000, 2000);
 
-	if (adap->orig_init)
-		return adap->orig_init(fe);
-	else
-		return 0;
+	ret = config_demod(adap->demod_i2c_client, adap->pt1->fe_clk);
+	if (ret == 0 && adap->orig_init)
+		ret = adap->orig_init(fe);
+	return ret;
 }
 
 static void pt1_free_adapter(struct pt1_adapter *adap)
@@ -735,6 +907,8 @@ static int pt1_init_adapters(struct pt1 *pt1)
 static void pt1_cleanup_frontend(struct pt1_adapter *adap)
 {
 	dvb_unregister_frontend(adap->fe);
+	dvb_module_release(adap->tuner_i2c_client);
+	dvb_module_release(adap->demod_i2c_client);
 }
 
 static int pt1_init_frontend(struct pt1_adapter *adap, struct dvb_frontend *fe)
@@ -763,112 +937,70 @@ static void pt1_cleanup_frontends(struct pt1 *pt1)
 		pt1_cleanup_frontend(pt1->adaps[i]);
 }
 
-struct pt1_config {
-	struct va1j5jf8007s_config va1j5jf8007s_config;
-	struct va1j5jf8007t_config va1j5jf8007t_config;
-};
-
-static const struct pt1_config pt1_configs[2] = {
-	{
-		{
-			.demod_address = 0x1b,
-			.frequency = VA1J5JF8007S_20MHZ,
-		},
-		{
-			.demod_address = 0x1a,
-			.frequency = VA1J5JF8007T_20MHZ,
-		},
-	}, {
-		{
-			.demod_address = 0x19,
-			.frequency = VA1J5JF8007S_20MHZ,
-		},
-		{
-			.demod_address = 0x18,
-			.frequency = VA1J5JF8007T_20MHZ,
-		},
-	},
-};
-
-static const struct pt1_config pt2_configs[2] = {
-	{
-		{
-			.demod_address = 0x1b,
-			.frequency = VA1J5JF8007S_25MHZ,
-		},
-		{
-			.demod_address = 0x1a,
-			.frequency = VA1J5JF8007T_25MHZ,
-		},
-	}, {
-		{
-			.demod_address = 0x19,
-			.frequency = VA1J5JF8007S_25MHZ,
-		},
-		{
-			.demod_address = 0x18,
-			.frequency = VA1J5JF8007T_25MHZ,
-		},
-	},
-};
-
 static int pt1_init_frontends(struct pt1 *pt1)
 {
-	int i, j;
-	struct i2c_adapter *i2c_adap;
-	const struct pt1_config *configs, *config;
-	struct dvb_frontend *fe[4];
+	int i;
 	int ret;
 
-	i = 0;
-	j = 0;
-
-	i2c_adap = &pt1->i2c_adap;
-	configs = pt1->pdev->device == 0x211a ? pt1_configs : pt2_configs;
-	do {
-		config = &configs[i / 2];
-
-		fe[i] = va1j5jf8007s_attach(&config->va1j5jf8007s_config,
-					    i2c_adap);
-		if (!fe[i]) {
-			ret = -ENODEV; /* This does not sound nice... */
-			goto err;
-		}
-		i++;
-
-		fe[i] = va1j5jf8007t_attach(&config->va1j5jf8007t_config,
-					    i2c_adap);
-		if (!fe[i]) {
-			ret = -ENODEV;
-			goto err;
+	for (i = 0; i < ARRAY_SIZE(pt1_configs); i++) {
+		const struct i2c_board_info *info;
+		struct tc90522_config dcfg;
+		struct i2c_client *cl;
+
+		info = &pt1_configs[i].demod_info;
+		dcfg = pt1_configs[i].demod_cfg;
+		dcfg.tuner_i2c = NULL;
+
+		ret = -ENODEV;
+		cl = dvb_module_probe("tc90522", info->type, &pt1->i2c_adap,
+				      info->addr, &dcfg);
+		if (!cl)
+			goto fe_unregister;
+		pt1->adaps[i]->demod_i2c_client = cl;
+
+		if (!strncmp(cl->name, TC90522_I2C_DEV_SAT,
+			     strlen(TC90522_I2C_DEV_SAT))) {
+			struct qm1d1b0004_config tcfg;
+
+			info = &pt1_configs[i].tuner_info;
+			tcfg = pt1_configs[i].tuner_cfg.qm1d1b0004;
+			tcfg.fe = dcfg.fe;
+			cl = dvb_module_probe("qm1d1b0004",
+					      info->type, dcfg.tuner_i2c,
+					      info->addr, &tcfg);
+		} else {
+			struct dvb_pll_config tcfg;
+
+			info = &pt1_configs[i].tuner_info;
+			tcfg = pt1_configs[i].tuner_cfg.tda6651;
+			tcfg.fe = dcfg.fe;
+			cl = dvb_module_probe("dvb_pll",
+					      info->type, dcfg.tuner_i2c,
+					      info->addr, &tcfg);
 		}
-		i++;
-
-		ret = va1j5jf8007s_prepare(fe[i - 2]);
-		if (ret < 0)
-			goto err;
+		if (!cl)
+			goto demod_release;
+		pt1->adaps[i]->tuner_i2c_client = cl;
 
-		ret = va1j5jf8007t_prepare(fe[i - 1]);
-		if (ret < 0)
-			goto err;
-
-	} while (i < 4);
-
-	do {
-		ret = pt1_init_frontend(pt1->adaps[j], fe[j]);
+		ret = pt1_init_frontend(pt1->adaps[i], dcfg.fe);
 		if (ret < 0)
-			goto err;
-	} while (++j < 4);
+			goto tuner_release;
+	}
 
 	return 0;
 
-err:
-	while (i-- > j)
-		fe[i]->ops.release(fe[i]);
-
-	while (j--)
-		dvb_unregister_frontend(fe[j]);
-
+tuner_release:
+	dvb_module_release(pt1->adaps[i]->tuner_i2c_client);
+demod_release:
+	dvb_module_release(pt1->adaps[i]->demod_i2c_client);
+fe_unregister:
+	dev_warn(&pt1->pdev->dev, "failed to init FE(%d).\n", i);
+	i--;
+	for (; i >= 0; i--) {
+		dvb_unregister_frontend(pt1->adaps[i]->fe);
+		dvb_module_release(pt1->adaps[i]->tuner_i2c_client);
+		dvb_module_release(pt1->adaps[i]->demod_i2c_client);
+	}
 	return ret;
 }
 
@@ -954,7 +1086,7 @@ static int pt1_i2c_end(struct pt1 *pt1, int addr)
 	do {
 		if (signal_pending(current))
 			return -EINTR;
-		schedule_timeout_interruptible((HZ + 999) / 1000);
+		usleep_range(1000, 2000);
 	} while (pt1_read_reg(pt1, 0) & 0x00000080);
 	return 0;
 }
@@ -1052,6 +1184,98 @@ static void pt1_i2c_init(struct pt1 *pt1)
 		pt1_i2c_emit(pt1, i, 0, 0, 1, 1, 0);
 }
 
+#ifdef CONFIG_PM_SLEEP
+
+static int pt1_suspend(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pt1 *pt1 = pci_get_drvdata(pdev);
+
+	pt1_init_streams(pt1);
+	pt1_disable_ram(pt1);
+	pt1->power = 0;
+	pt1->reset = 1;
+	pt1_update_power(pt1);
+	return 0;
+}
+
+static int pt1_resume(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pt1 *pt1 = pci_get_drvdata(pdev);
+	int ret;
+	int i;
+
+	pt1->power = 0;
+	pt1->reset = 1;
+	pt1_update_power(pt1);
+
+	pt1_i2c_init(pt1);
+	pt1_i2c_wait(pt1);
+
+	ret = pt1_sync(pt1);
+	if (ret < 0)
+		goto resume_err;
+
+	pt1_identify(pt1);
+
+	ret = pt1_unlock(pt1);
+	if (ret < 0)
+		goto resume_err;
+
+	ret = pt1_reset_pci(pt1);
+	if (ret < 0)
+		goto resume_err;
+
+	ret = pt1_reset_ram(pt1);
+	if (ret < 0)
+		goto resume_err;
+
+	ret = pt1_enable_ram(pt1);
+	if (ret < 0)
+		goto resume_err;
+
+	pt1_init_streams(pt1);
+
+	pt1->power = 1;
+	pt1_update_power(pt1);
+	msleep(20);
+
+	pt1->reset = 0;
+	pt1_update_power(pt1);
+	usleep_range(1000, 2000);
+
+	for (i = 0; i < PT1_NR_ADAPS; i++)
+		dvb_frontend_reinitialise(pt1->adaps[i]->fe);
+
+	pt1_init_table_count(pt1);
+	for (i = 0; i < pt1_nr_tables; i++) {
+		int j;
+
+		for (j = 0; j < PT1_NR_BUFS; j++)
+			pt1->tables[i].bufs[j].page->upackets[PT1_NR_UPACKETS-1]
+				= 0;
+		pt1_increment_table_count(pt1);
+	}
+	pt1_register_tables(pt1, pt1->tables[0].addr >> PT1_PAGE_SHIFT);
+
+	pt1->table_index = 0;
+	pt1->buf_index = 0;
+	for (i = 0; i < PT1_NR_ADAPS; i++) {
+		pt1->adaps[i]->upacket_count = 0;
+		pt1->adaps[i]->packet_count = 0;
+		pt1->adaps[i]->st_count = -1;
+	}
+
+	return 0;
+
+resume_err:
+	dev_info(&pt1->pdev->dev, "failed to resume PT1/PT2.");
+	return 0;	/* resume anyway */
+}
+
+#endif /* CONFIG_PM_SLEEP */
+
 static void pt1_remove(struct pci_dev *pdev)
 {
 	struct pt1 *pt1;
@@ -1112,6 +1336,8 @@ static int pt1_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	mutex_init(&pt1->lock);
 	pt1->pdev = pdev;
 	pt1->regs = regs;
+	pt1->fe_clk = (pdev->device == 0x211a) ?
+				PT1_FE_CLK_20MHZ : PT1_FE_CLK_25MHZ;
 	pci_set_drvdata(pdev, pt1);
 
 	ret = pt1_init_adapters(pt1);
@@ -1163,11 +1389,11 @@ static int pt1_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	pt1->power = 1;
 	pt1_update_power(pt1);
-	schedule_timeout_uninterruptible((HZ + 49) / 50);
+	msleep(20);
 
 	pt1->reset = 0;
 	pt1_update_power(pt1);
-	schedule_timeout_uninterruptible((HZ + 999) / 1000);
+	usleep_range(1000, 2000);
 
 	ret = pt1_init_frontends(pt1);
 	if (ret < 0)
@@ -1210,11 +1436,16 @@ static const struct pci_device_id pt1_id_table[] = {
 };
 MODULE_DEVICE_TABLE(pci, pt1_id_table);
 
+static SIMPLE_DEV_PM_OPS(pt1_pm_ops, pt1_suspend, pt1_resume);
+
 static struct pci_driver pt1_driver = {
 	.name		= DRIVER_NAME,
 	.probe		= pt1_probe,
 	.remove		= pt1_remove,
 	.id_table	= pt1_id_table,
+#ifdef CONFIG_PM_SLEEP
+	.driver.pm	= &pt1_pm_ops,
+#endif
 };
 
 module_pci_driver(pt1_driver);
diff --git a/drivers/media/pci/pt1/va1j5jf8007s.c b/drivers/media/pci/pt1/va1j5jf8007s.c
deleted file mode 100644
index f49867aef054..000000000000
--- a/drivers/media/pci/pt1/va1j5jf8007s.c
+++ /dev/null
@@ -1,732 +0,0 @@
-/*
- * ISDB-S driver for VA1J5JF8007/VA1J5JF8011
- *
- * Copyright (C) 2009 HIRANO Takahito <hiranotaka@zng.info>
- *
- * based on pt1dvr - http://pt1dvr.sourceforge.jp/
- *	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/i2c.h>
-#include <media/dvb_frontend.h>
-#include "va1j5jf8007s.h"
-
-enum va1j5jf8007s_tune_state {
-	VA1J5JF8007S_IDLE,
-	VA1J5JF8007S_SET_FREQUENCY_1,
-	VA1J5JF8007S_SET_FREQUENCY_2,
-	VA1J5JF8007S_SET_FREQUENCY_3,
-	VA1J5JF8007S_CHECK_FREQUENCY,
-	VA1J5JF8007S_SET_MODULATION,
-	VA1J5JF8007S_CHECK_MODULATION,
-	VA1J5JF8007S_SET_TS_ID,
-	VA1J5JF8007S_CHECK_TS_ID,
-	VA1J5JF8007S_TRACK,
-};
-
-struct va1j5jf8007s_state {
-	const struct va1j5jf8007s_config *config;
-	struct i2c_adapter *adap;
-	struct dvb_frontend fe;
-	enum va1j5jf8007s_tune_state tune_state;
-};
-
-static int va1j5jf8007s_read_snr(struct dvb_frontend *fe, u16 *snr)
-{
-	struct va1j5jf8007s_state *state;
-	u8 addr;
-	int i;
-	u8 write_buf[1], read_buf[1];
-	struct i2c_msg msgs[2];
-	s32 word, x1, x2, x3, x4, x5, y;
-
-	state = fe->demodulator_priv;
-	addr = state->config->demod_address;
-
-	word = 0;
-	for (i = 0; i < 2; i++) {
-		write_buf[0] = 0xbc + i;
-
-		msgs[0].addr = addr;
-		msgs[0].flags = 0;
-		msgs[0].len = sizeof(write_buf);
-		msgs[0].buf = write_buf;
-
-		msgs[1].addr = addr;
-		msgs[1].flags = I2C_M_RD;
-		msgs[1].len = sizeof(read_buf);
-		msgs[1].buf = read_buf;
-
-		if (i2c_transfer(state->adap, msgs, 2) != 2)
-			return -EREMOTEIO;
-
-		word <<= 8;
-		word |= read_buf[0];
-	}
-
-	word -= 3000;
-	if (word < 0)
-		word = 0;
-
-	x1 = int_sqrt(word << 16) * ((15625ll << 21) / 1000000);
-	x2 = (s64)x1 * x1 >> 31;
-	x3 = (s64)x2 * x1 >> 31;
-	x4 = (s64)x2 * x2 >> 31;
-	x5 = (s64)x4 * x1 >> 31;
-
-	y = (58857ll << 23) / 1000;
-	y -= (s64)x1 * ((89565ll << 24) / 1000) >> 30;
-	y += (s64)x2 * ((88977ll << 24) / 1000) >> 28;
-	y -= (s64)x3 * ((50259ll << 25) / 1000) >> 27;
-	y += (s64)x4 * ((14341ll << 27) / 1000) >> 27;
-	y -= (s64)x5 * ((16346ll << 30) / 10000) >> 28;
-
-	*snr = y < 0 ? 0 : y >> 15;
-	return 0;
-}
-
-static int va1j5jf8007s_get_frontend_algo(struct dvb_frontend *fe)
-{
-	return DVBFE_ALGO_HW;
-}
-
-static int
-va1j5jf8007s_read_status(struct dvb_frontend *fe, enum fe_status *status)
-{
-	struct va1j5jf8007s_state *state;
-
-	state = fe->demodulator_priv;
-
-	switch (state->tune_state) {
-	case VA1J5JF8007S_IDLE:
-	case VA1J5JF8007S_SET_FREQUENCY_1:
-	case VA1J5JF8007S_SET_FREQUENCY_2:
-	case VA1J5JF8007S_SET_FREQUENCY_3:
-	case VA1J5JF8007S_CHECK_FREQUENCY:
-		*status = 0;
-		return 0;
-
-
-	case VA1J5JF8007S_SET_MODULATION:
-	case VA1J5JF8007S_CHECK_MODULATION:
-		*status |= FE_HAS_SIGNAL;
-		return 0;
-
-	case VA1J5JF8007S_SET_TS_ID:
-	case VA1J5JF8007S_CHECK_TS_ID:
-		*status |= FE_HAS_SIGNAL | FE_HAS_CARRIER;
-		return 0;
-
-	case VA1J5JF8007S_TRACK:
-		*status |= FE_HAS_SIGNAL | FE_HAS_CARRIER | FE_HAS_LOCK;
-		return 0;
-	}
-
-	BUG();
-}
-
-struct va1j5jf8007s_cb_map {
-	u32 frequency;
-	u8 cb;
-};
-
-static const struct va1j5jf8007s_cb_map va1j5jf8007s_cb_maps[] = {
-	{  986000, 0xb2 },
-	{ 1072000, 0xd2 },
-	{ 1154000, 0xe2 },
-	{ 1291000, 0x20 },
-	{ 1447000, 0x40 },
-	{ 1615000, 0x60 },
-	{ 1791000, 0x80 },
-	{ 1972000, 0xa0 },
-};
-
-static u8 va1j5jf8007s_lookup_cb(u32 frequency)
-{
-	int i;
-	const struct va1j5jf8007s_cb_map *map;
-
-	for (i = 0; i < ARRAY_SIZE(va1j5jf8007s_cb_maps); i++) {
-		map = &va1j5jf8007s_cb_maps[i];
-		if (frequency < map->frequency)
-			return map->cb;
-	}
-	return 0xc0;
-}
-
-static int va1j5jf8007s_set_frequency_1(struct va1j5jf8007s_state *state)
-{
-	u32 frequency;
-	u16 word;
-	u8 buf[6];
-	struct i2c_msg msg;
-
-	frequency = state->fe.dtv_property_cache.frequency;
-
-	word = (frequency + 500) / 1000;
-	if (frequency < 1072000)
-		word = (word << 1 & ~0x1f) | (word & 0x0f);
-
-	buf[0] = 0xfe;
-	buf[1] = 0xc0;
-	buf[2] = 0x40 | word >> 8;
-	buf[3] = word;
-	buf[4] = 0xe0;
-	buf[5] = va1j5jf8007s_lookup_cb(frequency);
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1)
-		return -EREMOTEIO;
-
-	return 0;
-}
-
-static int va1j5jf8007s_set_frequency_2(struct va1j5jf8007s_state *state)
-{
-	u8 buf[3];
-	struct i2c_msg msg;
-
-	buf[0] = 0xfe;
-	buf[1] = 0xc0;
-	buf[2] = 0xe4;
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1)
-		return -EREMOTEIO;
-
-	return 0;
-}
-
-static int va1j5jf8007s_set_frequency_3(struct va1j5jf8007s_state *state)
-{
-	u32 frequency;
-	u8 buf[4];
-	struct i2c_msg msg;
-
-	frequency = state->fe.dtv_property_cache.frequency;
-
-	buf[0] = 0xfe;
-	buf[1] = 0xc0;
-	buf[2] = 0xf4;
-	buf[3] = va1j5jf8007s_lookup_cb(frequency) | 0x4;
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1)
-		return -EREMOTEIO;
-
-	return 0;
-}
-
-static int
-va1j5jf8007s_check_frequency(struct va1j5jf8007s_state *state, int *lock)
-{
-	u8 addr;
-	u8 write_buf[2], read_buf[1];
-	struct i2c_msg msgs[2];
-
-	addr = state->config->demod_address;
-
-	write_buf[0] = 0xfe;
-	write_buf[1] = 0xc1;
-
-	msgs[0].addr = addr;
-	msgs[0].flags = 0;
-	msgs[0].len = sizeof(write_buf);
-	msgs[0].buf = write_buf;
-
-	msgs[1].addr = addr;
-	msgs[1].flags = I2C_M_RD;
-	msgs[1].len = sizeof(read_buf);
-	msgs[1].buf = read_buf;
-
-	if (i2c_transfer(state->adap, msgs, 2) != 2)
-		return -EREMOTEIO;
-
-	*lock = read_buf[0] & 0x40;
-	return 0;
-}
-
-static int va1j5jf8007s_set_modulation(struct va1j5jf8007s_state *state)
-{
-	u8 buf[2];
-	struct i2c_msg msg;
-
-	buf[0] = 0x03;
-	buf[1] = 0x01;
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1)
-		return -EREMOTEIO;
-
-	return 0;
-}
-
-static int
-va1j5jf8007s_check_modulation(struct va1j5jf8007s_state *state, int *lock)
-{
-	u8 addr;
-	u8 write_buf[1], read_buf[1];
-	struct i2c_msg msgs[2];
-
-	addr = state->config->demod_address;
-
-	write_buf[0] = 0xc3;
-
-	msgs[0].addr = addr;
-	msgs[0].flags = 0;
-	msgs[0].len = sizeof(write_buf);
-	msgs[0].buf = write_buf;
-
-	msgs[1].addr = addr;
-	msgs[1].flags = I2C_M_RD;
-	msgs[1].len = sizeof(read_buf);
-	msgs[1].buf = read_buf;
-
-	if (i2c_transfer(state->adap, msgs, 2) != 2)
-		return -EREMOTEIO;
-
-	*lock = !(read_buf[0] & 0x10);
-	return 0;
-}
-
-static int
-va1j5jf8007s_set_ts_id(struct va1j5jf8007s_state *state)
-{
-	u32 ts_id;
-	u8 buf[3];
-	struct i2c_msg msg;
-
-	ts_id = state->fe.dtv_property_cache.stream_id;
-	if (!ts_id || ts_id == NO_STREAM_ID_FILTER)
-		return 0;
-
-	buf[0] = 0x8f;
-	buf[1] = ts_id >> 8;
-	buf[2] = ts_id;
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1)
-		return -EREMOTEIO;
-
-	return 0;
-}
-
-static int
-va1j5jf8007s_check_ts_id(struct va1j5jf8007s_state *state, int *lock)
-{
-	u8 addr;
-	u8 write_buf[1], read_buf[2];
-	struct i2c_msg msgs[2];
-	u32 ts_id;
-
-	ts_id = state->fe.dtv_property_cache.stream_id;
-	if (!ts_id || ts_id == NO_STREAM_ID_FILTER) {
-		*lock = 1;
-		return 0;
-	}
-
-	addr = state->config->demod_address;
-
-	write_buf[0] = 0xe6;
-
-	msgs[0].addr = addr;
-	msgs[0].flags = 0;
-	msgs[0].len = sizeof(write_buf);
-	msgs[0].buf = write_buf;
-
-	msgs[1].addr = addr;
-	msgs[1].flags = I2C_M_RD;
-	msgs[1].len = sizeof(read_buf);
-	msgs[1].buf = read_buf;
-
-	if (i2c_transfer(state->adap, msgs, 2) != 2)
-		return -EREMOTEIO;
-
-	*lock = (read_buf[0] << 8 | read_buf[1]) == ts_id;
-	return 0;
-}
-
-static int
-va1j5jf8007s_tune(struct dvb_frontend *fe,
-		  bool re_tune,
-		  unsigned int mode_flags,  unsigned int *delay,
-		  enum fe_status *status)
-{
-	struct va1j5jf8007s_state *state;
-	int ret;
-	int lock = 0;
-
-	state = fe->demodulator_priv;
-
-	if (re_tune)
-		state->tune_state = VA1J5JF8007S_SET_FREQUENCY_1;
-
-	switch (state->tune_state) {
-	case VA1J5JF8007S_IDLE:
-		*delay = 3 * HZ;
-		*status = 0;
-		return 0;
-
-	case VA1J5JF8007S_SET_FREQUENCY_1:
-		ret = va1j5jf8007s_set_frequency_1(state);
-		if (ret < 0)
-			return ret;
-
-		state->tune_state = VA1J5JF8007S_SET_FREQUENCY_2;
-		*delay = 0;
-		*status = 0;
-		return 0;
-
-	case VA1J5JF8007S_SET_FREQUENCY_2:
-		ret = va1j5jf8007s_set_frequency_2(state);
-		if (ret < 0)
-			return ret;
-
-		state->tune_state = VA1J5JF8007S_SET_FREQUENCY_3;
-		*delay = (HZ + 99) / 100;
-		*status = 0;
-		return 0;
-
-	case VA1J5JF8007S_SET_FREQUENCY_3:
-		ret = va1j5jf8007s_set_frequency_3(state);
-		if (ret < 0)
-			return ret;
-
-		state->tune_state = VA1J5JF8007S_CHECK_FREQUENCY;
-		*delay = 0;
-		*status = 0;
-		return 0;
-
-	case VA1J5JF8007S_CHECK_FREQUENCY:
-		ret = va1j5jf8007s_check_frequency(state, &lock);
-		if (ret < 0)
-			return ret;
-
-		if (!lock)  {
-			*delay = (HZ + 999) / 1000;
-			*status = 0;
-			return 0;
-		}
-
-		state->tune_state = VA1J5JF8007S_SET_MODULATION;
-		*delay = 0;
-		*status = FE_HAS_SIGNAL;
-		return 0;
-
-	case VA1J5JF8007S_SET_MODULATION:
-		ret = va1j5jf8007s_set_modulation(state);
-		if (ret < 0)
-			return ret;
-
-		state->tune_state = VA1J5JF8007S_CHECK_MODULATION;
-		*delay = 0;
-		*status = FE_HAS_SIGNAL;
-		return 0;
-
-	case VA1J5JF8007S_CHECK_MODULATION:
-		ret = va1j5jf8007s_check_modulation(state, &lock);
-		if (ret < 0)
-			return ret;
-
-		if (!lock)  {
-			*delay = (HZ + 49) / 50;
-			*status = FE_HAS_SIGNAL;
-			return 0;
-		}
-
-		state->tune_state = VA1J5JF8007S_SET_TS_ID;
-		*delay = 0;
-		*status = FE_HAS_SIGNAL | FE_HAS_CARRIER;
-		return 0;
-
-	case VA1J5JF8007S_SET_TS_ID:
-		ret = va1j5jf8007s_set_ts_id(state);
-		if (ret < 0)
-			return ret;
-
-		state->tune_state = VA1J5JF8007S_CHECK_TS_ID;
-		return 0;
-
-	case VA1J5JF8007S_CHECK_TS_ID:
-		ret = va1j5jf8007s_check_ts_id(state, &lock);
-		if (ret < 0)
-			return ret;
-
-		if (!lock)  {
-			*delay = (HZ + 99) / 100;
-			*status = FE_HAS_SIGNAL | FE_HAS_CARRIER;
-			return 0;
-		}
-
-		state->tune_state = VA1J5JF8007S_TRACK;
-		/* fall through */
-
-	case VA1J5JF8007S_TRACK:
-		*delay = 3 * HZ;
-		*status = FE_HAS_SIGNAL | FE_HAS_CARRIER | FE_HAS_LOCK;
-		return 0;
-	}
-
-	BUG();
-}
-
-static int va1j5jf8007s_init_frequency(struct va1j5jf8007s_state *state)
-{
-	u8 buf[4];
-	struct i2c_msg msg;
-
-	buf[0] = 0xfe;
-	buf[1] = 0xc0;
-	buf[2] = 0xf0;
-	buf[3] = 0x04;
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1)
-		return -EREMOTEIO;
-
-	return 0;
-}
-
-static int va1j5jf8007s_set_sleep(struct va1j5jf8007s_state *state, int sleep)
-{
-	u8 buf[2];
-	struct i2c_msg msg;
-
-	buf[0] = 0x17;
-	buf[1] = sleep ? 0x01 : 0x00;
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1)
-		return -EREMOTEIO;
-
-	return 0;
-}
-
-static int va1j5jf8007s_sleep(struct dvb_frontend *fe)
-{
-	struct va1j5jf8007s_state *state;
-	int ret;
-
-	state = fe->demodulator_priv;
-
-	ret = va1j5jf8007s_init_frequency(state);
-	if (ret < 0)
-		return ret;
-
-	return va1j5jf8007s_set_sleep(state, 1);
-}
-
-static int va1j5jf8007s_init(struct dvb_frontend *fe)
-{
-	struct va1j5jf8007s_state *state;
-
-	state = fe->demodulator_priv;
-	state->tune_state = VA1J5JF8007S_IDLE;
-
-	return va1j5jf8007s_set_sleep(state, 0);
-}
-
-static void va1j5jf8007s_release(struct dvb_frontend *fe)
-{
-	struct va1j5jf8007s_state *state;
-	state = fe->demodulator_priv;
-	kfree(state);
-}
-
-static const struct dvb_frontend_ops va1j5jf8007s_ops = {
-	.delsys = { SYS_ISDBS },
-	.info = {
-		.name = "VA1J5JF8007/VA1J5JF8011 ISDB-S",
-		.frequency_min = 950000,
-		.frequency_max = 2150000,
-		.frequency_stepsize = 1000,
-		.caps = FE_CAN_INVERSION_AUTO | FE_CAN_FEC_AUTO |
-			FE_CAN_QAM_AUTO | FE_CAN_TRANSMISSION_MODE_AUTO |
-			FE_CAN_GUARD_INTERVAL_AUTO | FE_CAN_HIERARCHY_AUTO |
-			FE_CAN_MULTISTREAM,
-	},
-
-	.read_snr = va1j5jf8007s_read_snr,
-	.get_frontend_algo = va1j5jf8007s_get_frontend_algo,
-	.read_status = va1j5jf8007s_read_status,
-	.tune = va1j5jf8007s_tune,
-	.sleep = va1j5jf8007s_sleep,
-	.init = va1j5jf8007s_init,
-	.release = va1j5jf8007s_release,
-};
-
-static int va1j5jf8007s_prepare_1(struct va1j5jf8007s_state *state)
-{
-	u8 addr;
-	u8 write_buf[1], read_buf[1];
-	struct i2c_msg msgs[2];
-
-	addr = state->config->demod_address;
-
-	write_buf[0] = 0x07;
-
-	msgs[0].addr = addr;
-	msgs[0].flags = 0;
-	msgs[0].len = sizeof(write_buf);
-	msgs[0].buf = write_buf;
-
-	msgs[1].addr = addr;
-	msgs[1].flags = I2C_M_RD;
-	msgs[1].len = sizeof(read_buf);
-	msgs[1].buf = read_buf;
-
-	if (i2c_transfer(state->adap, msgs, 2) != 2)
-		return -EREMOTEIO;
-
-	if (read_buf[0] != 0x41)
-		return -EIO;
-
-	return 0;
-}
-
-static const u8 va1j5jf8007s_20mhz_prepare_bufs[][2] = {
-	{0x04, 0x02}, {0x0d, 0x55}, {0x11, 0x40}, {0x13, 0x80}, {0x17, 0x01},
-	{0x1c, 0x0a}, {0x1d, 0xaa}, {0x1e, 0x20}, {0x1f, 0x88}, {0x51, 0xb0},
-	{0x52, 0x89}, {0x53, 0xb3}, {0x5a, 0x2d}, {0x5b, 0xd3}, {0x85, 0x69},
-	{0x87, 0x04}, {0x8e, 0x02}, {0xa3, 0xf7}, {0xa5, 0xc0},
-};
-
-static const u8 va1j5jf8007s_25mhz_prepare_bufs[][2] = {
-	{0x04, 0x02}, {0x11, 0x40}, {0x13, 0x80}, {0x17, 0x01}, {0x1c, 0x0a},
-	{0x1d, 0xaa}, {0x1e, 0x20}, {0x1f, 0x88}, {0x51, 0xb0}, {0x52, 0x89},
-	{0x53, 0xb3}, {0x5a, 0x2d}, {0x5b, 0xd3}, {0x85, 0x69}, {0x87, 0x04},
-	{0x8e, 0x26}, {0xa3, 0xf7}, {0xa5, 0xc0},
-};
-
-static int va1j5jf8007s_prepare_2(struct va1j5jf8007s_state *state)
-{
-	const u8 (*bufs)[2];
-	int size;
-	u8 addr;
-	u8 buf[2];
-	struct i2c_msg msg;
-	int i;
-
-	switch (state->config->frequency) {
-	case VA1J5JF8007S_20MHZ:
-		bufs = va1j5jf8007s_20mhz_prepare_bufs;
-		size = ARRAY_SIZE(va1j5jf8007s_20mhz_prepare_bufs);
-		break;
-	case VA1J5JF8007S_25MHZ:
-		bufs = va1j5jf8007s_25mhz_prepare_bufs;
-		size = ARRAY_SIZE(va1j5jf8007s_25mhz_prepare_bufs);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	addr = state->config->demod_address;
-
-	msg.addr = addr;
-	msg.flags = 0;
-	msg.len = 2;
-	msg.buf = buf;
-	for (i = 0; i < size; i++) {
-		memcpy(buf, bufs[i], sizeof(buf));
-		if (i2c_transfer(state->adap, &msg, 1) != 1)
-			return -EREMOTEIO;
-	}
-
-	return 0;
-}
-
-/* must be called after va1j5jf8007t_attach */
-int va1j5jf8007s_prepare(struct dvb_frontend *fe)
-{
-	struct va1j5jf8007s_state *state;
-	int ret;
-
-	state = fe->demodulator_priv;
-
-	ret = va1j5jf8007s_prepare_1(state);
-	if (ret < 0)
-		return ret;
-
-	ret = va1j5jf8007s_prepare_2(state);
-	if (ret < 0)
-		return ret;
-
-	return va1j5jf8007s_init_frequency(state);
-}
-
-struct dvb_frontend *
-va1j5jf8007s_attach(const struct va1j5jf8007s_config *config,
-		    struct i2c_adapter *adap)
-{
-	struct va1j5jf8007s_state *state;
-	struct dvb_frontend *fe;
-	u8 buf[2];
-	struct i2c_msg msg;
-
-	state = kzalloc(sizeof(struct va1j5jf8007s_state), GFP_KERNEL);
-	if (!state)
-		return NULL;
-
-	state->config = config;
-	state->adap = adap;
-
-	fe = &state->fe;
-	memcpy(&fe->ops, &va1j5jf8007s_ops, sizeof(struct dvb_frontend_ops));
-	fe->demodulator_priv = state;
-
-	buf[0] = 0x01;
-	buf[1] = 0x80;
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1) {
-		kfree(state);
-		return NULL;
-	}
-
-	return fe;
-}
diff --git a/drivers/media/pci/pt1/va1j5jf8007s.h b/drivers/media/pci/pt1/va1j5jf8007s.h
deleted file mode 100644
index f8ce5609095d..000000000000
--- a/drivers/media/pci/pt1/va1j5jf8007s.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * ISDB-S driver for VA1J5JF8007/VA1J5JF8011
- *
- * Copyright (C) 2009 HIRANO Takahito <hiranotaka@zng.info>
- *
- * based on pt1dvr - http://pt1dvr.sourceforge.jp/
- *	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#ifndef VA1J5JF8007S_H
-#define VA1J5JF8007S_H
-
-enum va1j5jf8007s_frequency {
-	VA1J5JF8007S_20MHZ,
-	VA1J5JF8007S_25MHZ,
-};
-
-struct va1j5jf8007s_config {
-	u8 demod_address;
-	enum va1j5jf8007s_frequency frequency;
-};
-
-struct i2c_adapter;
-
-struct dvb_frontend *
-va1j5jf8007s_attach(const struct va1j5jf8007s_config *config,
-		    struct i2c_adapter *adap);
-
-/* must be called after va1j5jf8007t_attach */
-int va1j5jf8007s_prepare(struct dvb_frontend *fe);
-
-#endif
diff --git a/drivers/media/pci/pt1/va1j5jf8007t.c b/drivers/media/pci/pt1/va1j5jf8007t.c
deleted file mode 100644
index a52984a6f9b3..000000000000
--- a/drivers/media/pci/pt1/va1j5jf8007t.c
+++ /dev/null
@@ -1,532 +0,0 @@
-/*
- * ISDB-T driver for VA1J5JF8007/VA1J5JF8011
- *
- * Copyright (C) 2009 HIRANO Takahito <hiranotaka@zng.info>
- *
- * based on pt1dvr - http://pt1dvr.sourceforge.jp/
- *	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/i2c.h>
-#include <media/dvb_frontend.h>
-#include <media/dvb_math.h>
-#include "va1j5jf8007t.h"
-
-enum va1j5jf8007t_tune_state {
-	VA1J5JF8007T_IDLE,
-	VA1J5JF8007T_SET_FREQUENCY,
-	VA1J5JF8007T_CHECK_FREQUENCY,
-	VA1J5JF8007T_SET_MODULATION,
-	VA1J5JF8007T_CHECK_MODULATION,
-	VA1J5JF8007T_TRACK,
-	VA1J5JF8007T_ABORT,
-};
-
-struct va1j5jf8007t_state {
-	const struct va1j5jf8007t_config *config;
-	struct i2c_adapter *adap;
-	struct dvb_frontend fe;
-	enum va1j5jf8007t_tune_state tune_state;
-};
-
-static int va1j5jf8007t_read_snr(struct dvb_frontend *fe, u16 *snr)
-{
-	struct va1j5jf8007t_state *state;
-	u8 addr;
-	int i;
-	u8 write_buf[1], read_buf[1];
-	struct i2c_msg msgs[2];
-	s32 word, x, y;
-
-	state = fe->demodulator_priv;
-	addr = state->config->demod_address;
-
-	word = 0;
-	for (i = 0; i < 3; i++) {
-		write_buf[0] = 0x8b + i;
-
-		msgs[0].addr = addr;
-		msgs[0].flags = 0;
-		msgs[0].len = sizeof(write_buf);
-		msgs[0].buf = write_buf;
-
-		msgs[1].addr = addr;
-		msgs[1].flags = I2C_M_RD;
-		msgs[1].len = sizeof(read_buf);
-		msgs[1].buf = read_buf;
-
-		if (i2c_transfer(state->adap, msgs, 2) != 2)
-			return -EREMOTEIO;
-
-		word <<= 8;
-		word |= read_buf[0];
-	}
-
-	if (!word)
-		return -EIO;
-
-	x = 10 * (intlog10(0x540000 * 100 / word) - (2 << 24));
-	y = (24ll << 46) / 1000000;
-	y = ((s64)y * x >> 30) - (16ll << 40) / 10000;
-	y = ((s64)y * x >> 29) + (398ll << 35) / 10000;
-	y = ((s64)y * x >> 30) + (5491ll << 29) / 10000;
-	y = ((s64)y * x >> 30) + (30965ll << 23) / 10000;
-	*snr = y >> 15;
-	return 0;
-}
-
-static int va1j5jf8007t_get_frontend_algo(struct dvb_frontend *fe)
-{
-	return DVBFE_ALGO_HW;
-}
-
-static int
-va1j5jf8007t_read_status(struct dvb_frontend *fe, enum fe_status *status)
-{
-	struct va1j5jf8007t_state *state;
-
-	state = fe->demodulator_priv;
-
-	switch (state->tune_state) {
-	case VA1J5JF8007T_IDLE:
-	case VA1J5JF8007T_SET_FREQUENCY:
-	case VA1J5JF8007T_CHECK_FREQUENCY:
-		*status = 0;
-		return 0;
-
-
-	case VA1J5JF8007T_SET_MODULATION:
-	case VA1J5JF8007T_CHECK_MODULATION:
-	case VA1J5JF8007T_ABORT:
-		*status |= FE_HAS_SIGNAL;
-		return 0;
-
-	case VA1J5JF8007T_TRACK:
-		*status |= FE_HAS_SIGNAL | FE_HAS_CARRIER | FE_HAS_LOCK;
-		return 0;
-	}
-
-	BUG();
-}
-
-struct va1j5jf8007t_cb_map {
-	u32 frequency;
-	u8 cb;
-};
-
-static const struct va1j5jf8007t_cb_map va1j5jf8007t_cb_maps[] = {
-	{  90000000, 0x80 },
-	{ 140000000, 0x81 },
-	{ 170000000, 0xa1 },
-	{ 220000000, 0x62 },
-	{ 330000000, 0xa2 },
-	{ 402000000, 0xe2 },
-	{ 450000000, 0x64 },
-	{ 550000000, 0x84 },
-	{ 600000000, 0xa4 },
-	{ 700000000, 0xc4 },
-};
-
-static u8 va1j5jf8007t_lookup_cb(u32 frequency)
-{
-	int i;
-	const struct va1j5jf8007t_cb_map *map;
-
-	for (i = 0; i < ARRAY_SIZE(va1j5jf8007t_cb_maps); i++) {
-		map = &va1j5jf8007t_cb_maps[i];
-		if (frequency < map->frequency)
-			return map->cb;
-	}
-	return 0xe4;
-}
-
-static int va1j5jf8007t_set_frequency(struct va1j5jf8007t_state *state)
-{
-	u32 frequency;
-	u16 word;
-	u8 buf[6];
-	struct i2c_msg msg;
-
-	frequency = state->fe.dtv_property_cache.frequency;
-
-	word = (frequency + 71428) / 142857 + 399;
-	buf[0] = 0xfe;
-	buf[1] = 0xc2;
-	buf[2] = word >> 8;
-	buf[3] = word;
-	buf[4] = 0x80;
-	buf[5] = va1j5jf8007t_lookup_cb(frequency);
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1)
-		return -EREMOTEIO;
-
-	return 0;
-}
-
-static int
-va1j5jf8007t_check_frequency(struct va1j5jf8007t_state *state, int *lock)
-{
-	u8 addr;
-	u8 write_buf[2], read_buf[1];
-	struct i2c_msg msgs[2];
-
-	addr = state->config->demod_address;
-
-	write_buf[0] = 0xfe;
-	write_buf[1] = 0xc3;
-
-	msgs[0].addr = addr;
-	msgs[0].flags = 0;
-	msgs[0].len = sizeof(write_buf);
-	msgs[0].buf = write_buf;
-
-	msgs[1].addr = addr;
-	msgs[1].flags = I2C_M_RD;
-	msgs[1].len = sizeof(read_buf);
-	msgs[1].buf = read_buf;
-
-	if (i2c_transfer(state->adap, msgs, 2) != 2)
-		return -EREMOTEIO;
-
-	*lock = read_buf[0] & 0x40;
-	return 0;
-}
-
-static int va1j5jf8007t_set_modulation(struct va1j5jf8007t_state *state)
-{
-	u8 buf[2];
-	struct i2c_msg msg;
-
-	buf[0] = 0x01;
-	buf[1] = 0x40;
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1)
-		return -EREMOTEIO;
-
-	return 0;
-}
-
-static int va1j5jf8007t_check_modulation(struct va1j5jf8007t_state *state,
-					 int *lock, int *retry)
-{
-	u8 addr;
-	u8 write_buf[1], read_buf[1];
-	struct i2c_msg msgs[2];
-
-	addr = state->config->demod_address;
-
-	write_buf[0] = 0x80;
-
-	msgs[0].addr = addr;
-	msgs[0].flags = 0;
-	msgs[0].len = sizeof(write_buf);
-	msgs[0].buf = write_buf;
-
-	msgs[1].addr = addr;
-	msgs[1].flags = I2C_M_RD;
-	msgs[1].len = sizeof(read_buf);
-	msgs[1].buf = read_buf;
-
-	if (i2c_transfer(state->adap, msgs, 2) != 2)
-		return -EREMOTEIO;
-
-	*lock = !(read_buf[0] & 0x10);
-	*retry = read_buf[0] & 0x80;
-	return 0;
-}
-
-static int
-va1j5jf8007t_tune(struct dvb_frontend *fe,
-		  bool re_tune,
-		  unsigned int mode_flags,  unsigned int *delay,
-		  enum fe_status *status)
-{
-	struct va1j5jf8007t_state *state;
-	int ret;
-	int lock = 0, retry = 0;
-
-	state = fe->demodulator_priv;
-
-	if (re_tune)
-		state->tune_state = VA1J5JF8007T_SET_FREQUENCY;
-
-	switch (state->tune_state) {
-	case VA1J5JF8007T_IDLE:
-		*delay = 3 * HZ;
-		*status = 0;
-		return 0;
-
-	case VA1J5JF8007T_SET_FREQUENCY:
-		ret = va1j5jf8007t_set_frequency(state);
-		if (ret < 0)
-			return ret;
-
-		state->tune_state = VA1J5JF8007T_CHECK_FREQUENCY;
-		*delay = 0;
-		*status = 0;
-		return 0;
-
-	case VA1J5JF8007T_CHECK_FREQUENCY:
-		ret = va1j5jf8007t_check_frequency(state, &lock);
-		if (ret < 0)
-			return ret;
-
-		if (!lock)  {
-			*delay = (HZ + 999) / 1000;
-			*status = 0;
-			return 0;
-		}
-
-		state->tune_state = VA1J5JF8007T_SET_MODULATION;
-		*delay = 0;
-		*status = FE_HAS_SIGNAL;
-		return 0;
-
-	case VA1J5JF8007T_SET_MODULATION:
-		ret = va1j5jf8007t_set_modulation(state);
-		if (ret < 0)
-			return ret;
-
-		state->tune_state = VA1J5JF8007T_CHECK_MODULATION;
-		*delay = 0;
-		*status = FE_HAS_SIGNAL;
-		return 0;
-
-	case VA1J5JF8007T_CHECK_MODULATION:
-		ret = va1j5jf8007t_check_modulation(state, &lock, &retry);
-		if (ret < 0)
-			return ret;
-
-		if (!lock)  {
-			if (!retry)  {
-				state->tune_state = VA1J5JF8007T_ABORT;
-				*delay = 3 * HZ;
-				*status = FE_HAS_SIGNAL;
-				return 0;
-			}
-			*delay = (HZ + 999) / 1000;
-			*status = FE_HAS_SIGNAL;
-			return 0;
-		}
-
-		state->tune_state = VA1J5JF8007T_TRACK;
-		/* fall through */
-
-	case VA1J5JF8007T_TRACK:
-		*delay = 3 * HZ;
-		*status = FE_HAS_SIGNAL | FE_HAS_CARRIER | FE_HAS_LOCK;
-		return 0;
-
-	case VA1J5JF8007T_ABORT:
-		*delay = 3 * HZ;
-		*status = FE_HAS_SIGNAL;
-		return 0;
-	}
-
-	BUG();
-}
-
-static int va1j5jf8007t_init_frequency(struct va1j5jf8007t_state *state)
-{
-	u8 buf[7];
-	struct i2c_msg msg;
-
-	buf[0] = 0xfe;
-	buf[1] = 0xc2;
-	buf[2] = 0x01;
-	buf[3] = 0x8f;
-	buf[4] = 0xc1;
-	buf[5] = 0x80;
-	buf[6] = 0x80;
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1)
-		return -EREMOTEIO;
-
-	return 0;
-}
-
-static int va1j5jf8007t_set_sleep(struct va1j5jf8007t_state *state, int sleep)
-{
-	u8 buf[2];
-	struct i2c_msg msg;
-
-	buf[0] = 0x03;
-	buf[1] = sleep ? 0x90 : 0x80;
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1)
-		return -EREMOTEIO;
-
-	return 0;
-}
-
-static int va1j5jf8007t_sleep(struct dvb_frontend *fe)
-{
-	struct va1j5jf8007t_state *state;
-	int ret;
-
-	state = fe->demodulator_priv;
-
-	ret = va1j5jf8007t_init_frequency(state);
-	if (ret < 0)
-		return ret;
-
-	return va1j5jf8007t_set_sleep(state, 1);
-}
-
-static int va1j5jf8007t_init(struct dvb_frontend *fe)
-{
-	struct va1j5jf8007t_state *state;
-
-	state = fe->demodulator_priv;
-	state->tune_state = VA1J5JF8007T_IDLE;
-
-	return va1j5jf8007t_set_sleep(state, 0);
-}
-
-static void va1j5jf8007t_release(struct dvb_frontend *fe)
-{
-	struct va1j5jf8007t_state *state;
-	state = fe->demodulator_priv;
-	kfree(state);
-}
-
-static const struct dvb_frontend_ops va1j5jf8007t_ops = {
-	.delsys = { SYS_ISDBT },
-	.info = {
-		.name = "VA1J5JF8007/VA1J5JF8011 ISDB-T",
-		.frequency_min = 90000000,
-		.frequency_max = 770000000,
-		.frequency_stepsize = 142857,
-		.caps = FE_CAN_INVERSION_AUTO | FE_CAN_FEC_AUTO |
-			FE_CAN_QAM_AUTO | FE_CAN_TRANSMISSION_MODE_AUTO |
-			FE_CAN_GUARD_INTERVAL_AUTO | FE_CAN_HIERARCHY_AUTO,
-	},
-
-	.read_snr = va1j5jf8007t_read_snr,
-	.get_frontend_algo = va1j5jf8007t_get_frontend_algo,
-	.read_status = va1j5jf8007t_read_status,
-	.tune = va1j5jf8007t_tune,
-	.sleep = va1j5jf8007t_sleep,
-	.init = va1j5jf8007t_init,
-	.release = va1j5jf8007t_release,
-};
-
-static const u8 va1j5jf8007t_20mhz_prepare_bufs[][2] = {
-	{0x03, 0x90}, {0x14, 0x8f}, {0x1c, 0x2a}, {0x1d, 0xa8}, {0x1e, 0xa2},
-	{0x22, 0x83}, {0x31, 0x0d}, {0x32, 0xe0}, {0x39, 0xd3}, {0x3a, 0x00},
-	{0x5c, 0x40}, {0x5f, 0x80}, {0x75, 0x02}, {0x76, 0x4e}, {0x77, 0x03},
-	{0xef, 0x01}
-};
-
-static const u8 va1j5jf8007t_25mhz_prepare_bufs[][2] = {
-	{0x03, 0x90}, {0x1c, 0x2a}, {0x1d, 0xa8}, {0x1e, 0xa2}, {0x22, 0x83},
-	{0x3a, 0x00}, {0x5c, 0x40}, {0x5f, 0x80}, {0x75, 0x0a}, {0x76, 0x4c},
-	{0x77, 0x03}, {0xef, 0x01}
-};
-
-int va1j5jf8007t_prepare(struct dvb_frontend *fe)
-{
-	struct va1j5jf8007t_state *state;
-	const u8 (*bufs)[2];
-	int size;
-	u8 buf[2];
-	struct i2c_msg msg;
-	int i;
-
-	state = fe->demodulator_priv;
-
-	switch (state->config->frequency) {
-	case VA1J5JF8007T_20MHZ:
-		bufs = va1j5jf8007t_20mhz_prepare_bufs;
-		size = ARRAY_SIZE(va1j5jf8007t_20mhz_prepare_bufs);
-		break;
-	case VA1J5JF8007T_25MHZ:
-		bufs = va1j5jf8007t_25mhz_prepare_bufs;
-		size = ARRAY_SIZE(va1j5jf8007t_25mhz_prepare_bufs);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	for (i = 0; i < size; i++) {
-		memcpy(buf, bufs[i], sizeof(buf));
-		if (i2c_transfer(state->adap, &msg, 1) != 1)
-			return -EREMOTEIO;
-	}
-
-	return va1j5jf8007t_init_frequency(state);
-}
-
-struct dvb_frontend *
-va1j5jf8007t_attach(const struct va1j5jf8007t_config *config,
-		    struct i2c_adapter *adap)
-{
-	struct va1j5jf8007t_state *state;
-	struct dvb_frontend *fe;
-	u8 buf[2];
-	struct i2c_msg msg;
-
-	state = kzalloc(sizeof(struct va1j5jf8007t_state), GFP_KERNEL);
-	if (!state)
-		return NULL;
-
-	state->config = config;
-	state->adap = adap;
-
-	fe = &state->fe;
-	memcpy(&fe->ops, &va1j5jf8007t_ops, sizeof(struct dvb_frontend_ops));
-	fe->demodulator_priv = state;
-
-	buf[0] = 0x01;
-	buf[1] = 0x80;
-
-	msg.addr = state->config->demod_address;
-	msg.flags = 0;
-	msg.len = sizeof(buf);
-	msg.buf = buf;
-
-	if (i2c_transfer(state->adap, &msg, 1) != 1) {
-		kfree(state);
-		return NULL;
-	}
-
-	return fe;
-}
diff --git a/drivers/media/pci/pt1/va1j5jf8007t.h b/drivers/media/pci/pt1/va1j5jf8007t.h
deleted file mode 100644
index 95eb7d294d20..000000000000
--- a/drivers/media/pci/pt1/va1j5jf8007t.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * ISDB-T driver for VA1J5JF8007/VA1J5JF8011
- *
- * Copyright (C) 2009 HIRANO Takahito <hiranotaka@zng.info>
- *
- * based on pt1dvr - http://pt1dvr.sourceforge.jp/
- *	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#ifndef VA1J5JF8007T_H
-#define VA1J5JF8007T_H
-
-enum va1j5jf8007t_frequency {
-	VA1J5JF8007T_20MHZ,
-	VA1J5JF8007T_25MHZ,
-};
-
-struct va1j5jf8007t_config {
-	u8 demod_address;
-	enum va1j5jf8007t_frequency frequency;
-};
-
-struct i2c_adapter;
-
-struct dvb_frontend *
-va1j5jf8007t_attach(const struct va1j5jf8007t_config *config,
-		    struct i2c_adapter *adap);
-
-/* must be called after va1j5jf8007s_attach */
-int va1j5jf8007t_prepare(struct dvb_frontend *fe);
-
-#endif
diff --git a/drivers/media/pci/pt3/pt3.c b/drivers/media/pci/pt3/pt3.c
index da74828805bc..90273b4d772f 100644
--- a/drivers/media/pci/pt3/pt3.c
+++ b/drivers/media/pci/pt3/pt3.c
@@ -1,17 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Earthsoft PT3 driver
  *
  * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/freezer.h>
@@ -376,25 +367,22 @@ static int pt3_fe_init(struct pt3_board *pt3)
 
 static int pt3_attach_fe(struct pt3_board *pt3, int i)
 {
-	struct i2c_board_info info;
+	const struct i2c_board_info *info;
 	struct tc90522_config cfg;
 	struct i2c_client *cl;
 	struct dvb_adapter *dvb_adap;
 	int ret;
 
-	info = adap_conf[i].demod_info;
+	info = &adap_conf[i].demod_info;
 	cfg = adap_conf[i].demod_cfg;
 	cfg.tuner_i2c = NULL;
-	info.platform_data = &cfg;
 
 	ret = -ENODEV;
-	request_module("tc90522");
-	cl = i2c_new_device(&pt3->i2c_adap, &info);
-	if (!cl || !cl->dev.driver)
+	cl = dvb_module_probe("tc90522", info->type, &pt3->i2c_adap,
+			      info->addr, &cfg);
+	if (!cl)
 		return -ENODEV;
 	pt3->adaps[i]->i2c_demod = cl;
-	if (!try_module_get(cl->dev.driver->owner))
-		goto err_demod_i2c_unregister_device;
 
 	if (!strncmp(cl->name, TC90522_I2C_DEV_SAT,
 		     strlen(TC90522_I2C_DEV_SAT))) {
@@ -402,41 +390,33 @@ static int pt3_attach_fe(struct pt3_board *pt3, int i)
 
 		tcfg = adap_conf[i].tuner_cfg.qm1d1c0042;
 		tcfg.fe = cfg.fe;
-		info = adap_conf[i].tuner_info;
-		info.platform_data = &tcfg;
-		request_module("qm1d1c0042");
-		cl = i2c_new_device(cfg.tuner_i2c, &info);
+		info = &adap_conf[i].tuner_info;
+		cl = dvb_module_probe("qm1d1c0042", info->type, cfg.tuner_i2c,
+				      info->addr, &tcfg);
 	} else {
 		struct mxl301rf_config tcfg;
 
 		tcfg = adap_conf[i].tuner_cfg.mxl301rf;
 		tcfg.fe = cfg.fe;
-		info = adap_conf[i].tuner_info;
-		info.platform_data = &tcfg;
-		request_module("mxl301rf");
-		cl = i2c_new_device(cfg.tuner_i2c, &info);
+		info = &adap_conf[i].tuner_info;
+		cl = dvb_module_probe("mxl301rf", info->type, cfg.tuner_i2c,
+				      info->addr, &tcfg);
 	}
-	if (!cl || !cl->dev.driver)
-		goto err_demod_module_put;
+	if (!cl)
+		goto err_demod_module_release;
 	pt3->adaps[i]->i2c_tuner = cl;
-	if (!try_module_get(cl->dev.driver->owner))
-		goto err_tuner_i2c_unregister_device;
 
 	dvb_adap = &pt3->adaps[one_adapter ? 0 : i]->dvb_adap;
 	ret = dvb_register_frontend(dvb_adap, cfg.fe);
 	if (ret < 0)
-		goto err_tuner_module_put;
+		goto err_tuner_module_release;
 	pt3->adaps[i]->fe = cfg.fe;
 	return 0;
 
-err_tuner_module_put:
-	module_put(pt3->adaps[i]->i2c_tuner->dev.driver->owner);
-err_tuner_i2c_unregister_device:
-	i2c_unregister_device(pt3->adaps[i]->i2c_tuner);
-err_demod_module_put:
-	module_put(pt3->adaps[i]->i2c_demod->dev.driver->owner);
-err_demod_i2c_unregister_device:
-	i2c_unregister_device(pt3->adaps[i]->i2c_demod);
+err_tuner_module_release:
+	dvb_module_release(pt3->adaps[i]->i2c_tuner);
+err_demod_module_release:
+	dvb_module_release(pt3->adaps[i]->i2c_demod);
 
 	return ret;
 }
@@ -464,7 +444,7 @@ static int pt3_fetch_thread(void *data)
 
 		pt3_proc_dma(adap);
 
-		delay = PT3_FETCH_DELAY * NSEC_PER_MSEC;
+		delay = ktime_set(0, PT3_FETCH_DELAY * NSEC_PER_MSEC);
 		set_current_state(TASK_UNINTERRUPTIBLE);
 		freezable_schedule_hrtimeout_range(&delay,
 					PT3_FETCH_DELAY_DELTA * NSEC_PER_MSEC,
@@ -630,14 +610,8 @@ static void pt3_cleanup_adapter(struct pt3_board *pt3, int index)
 		adap->fe->callback = NULL;
 		if (adap->fe->frontend_priv)
 			dvb_unregister_frontend(adap->fe);
-		if (adap->i2c_tuner) {
-			module_put(adap->i2c_tuner->dev.driver->owner);
-			i2c_unregister_device(adap->i2c_tuner);
-		}
-		if (adap->i2c_demod) {
-			module_put(adap->i2c_demod->dev.driver->owner);
-			i2c_unregister_device(adap->i2c_demod);
-		}
+		dvb_module_release(adap->i2c_tuner);
+		dvb_module_release(adap->i2c_demod);
 	}
 	pt3_free_dmabuf(adap);
 	dvb_dmxdev_release(&adap->dmxdev);
diff --git a/drivers/media/pci/pt3/pt3.h b/drivers/media/pci/pt3/pt3.h
index fbe8d9b847b0..495891a4ee17 100644
--- a/drivers/media/pci/pt3/pt3.h
+++ b/drivers/media/pci/pt3/pt3.h
@@ -1,17 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Earthsoft PT3 driver
  *
  * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #ifndef PT3_H
diff --git a/drivers/media/pci/pt3/pt3_dma.c b/drivers/media/pci/pt3/pt3_dma.c
index f0ce90437fac..de677b90ea5c 100644
--- a/drivers/media/pci/pt3/pt3_dma.c
+++ b/drivers/media/pci/pt3/pt3_dma.c
@@ -1,17 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Earthsoft PT3 driver
  *
  * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 #include <linux/dma-mapping.h>
 #include <linux/kernel.h>
diff --git a/drivers/media/pci/pt3/pt3_i2c.c b/drivers/media/pci/pt3/pt3_i2c.c
index b66138c7b364..b02be789a8c5 100644
--- a/drivers/media/pci/pt3/pt3_i2c.c
+++ b/drivers/media/pci/pt3/pt3_i2c.c
@@ -1,17 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Earthsoft PT3 driver
  *
  * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 #include <linux/delay.h>
 #include <linux/device.h>
diff --git a/drivers/media/pci/saa7164/saa7164-core.c b/drivers/media/pci/saa7164/saa7164-core.c
index fca36a4910c2..d697e1ad929c 100644
--- a/drivers/media/pci/saa7164/saa7164-core.c
+++ b/drivers/media/pci/saa7164/saa7164-core.c
@@ -1122,23 +1122,11 @@ static int saa7164_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int saa7164_proc_open(struct inode *inode, struct file *filp)
-{
-	return single_open(filp, saa7164_proc_show, NULL);
-}
-
-static const struct file_operations saa7164_proc_fops = {
-	.open		= saa7164_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int saa7164_proc_create(void)
 {
 	struct proc_dir_entry *pe;
 
-	pe = proc_create("saa7164", S_IRUGO, NULL, &saa7164_proc_fops);
+	pe = proc_create_single("saa7164", S_IRUGO, NULL, saa7164_proc_show);
 	if (!pe)
 		return -ENOMEM;
 
diff --git a/drivers/media/pci/saa7164/saa7164-fw.c b/drivers/media/pci/saa7164/saa7164-fw.c
index ef4906406ebf..a50461861133 100644
--- a/drivers/media/pci/saa7164/saa7164-fw.c
+++ b/drivers/media/pci/saa7164/saa7164-fw.c
@@ -426,7 +426,8 @@ int saa7164_downloadfirmware(struct saa7164_dev *dev)
 			__func__, fw->size);
 
 		if (fw->size != fwlength) {
-			printk(KERN_ERR "xc5000: firmware incorrect size\n");
+			printk(KERN_ERR "saa7164: firmware incorrect size %zu != %u\n",
+				fw->size, fwlength);
 			ret = -ENOMEM;
 			goto out;
 		}
diff --git a/drivers/media/pci/solo6x10/Kconfig b/drivers/media/pci/solo6x10/Kconfig
index 0fb91dc7ca73..d9e06a6bf1eb 100644
--- a/drivers/media/pci/solo6x10/Kconfig
+++ b/drivers/media/pci/solo6x10/Kconfig
@@ -1,7 +1,6 @@
 config VIDEO_SOLO6X10
 	tristate "Bluecherry / Softlogic 6x10 capture cards (MPEG-4/H.264)"
 	depends on PCI && VIDEO_DEV && SND && I2C
-	depends on HAS_DMA
 	select BITREVERSE
 	select FONT_SUPPORT
 	select FONT_8x16
diff --git a/drivers/media/pci/sta2x11/Kconfig b/drivers/media/pci/sta2x11/Kconfig
index e03587b1af71..4407b9f881e4 100644
--- a/drivers/media/pci/sta2x11/Kconfig
+++ b/drivers/media/pci/sta2x11/Kconfig
@@ -1,7 +1,6 @@
 config STA2X11_VIP
 	tristate "STA2X11 VIP Video For Linux"
-	depends on STA2X11
-	depends on HAS_DMA
+	depends on STA2X11 || COMPILE_TEST
 	select VIDEO_ADV7180 if MEDIA_SUBDRV_AUTOSELECT
 	select VIDEOBUF2_DMA_CONTIG
 	depends on PCI && VIDEO_V4L2 && VIRT_TO_BUS
diff --git a/drivers/media/pci/sta2x11/sta2x11_vip.c b/drivers/media/pci/sta2x11/sta2x11_vip.c
index dd199bfc1d45..069c4a853346 100644
--- a/drivers/media/pci/sta2x11/sta2x11_vip.c
+++ b/drivers/media/pci/sta2x11/sta2x11_vip.c
@@ -908,10 +908,10 @@ static int sta2x11_vip_init_controls(struct sta2x11_vip *vip)
 static int vip_gpio_reserve(struct device *dev, int pin, int dir,
 			    const char *name)
 {
-	int ret;
+	int ret = -ENODEV;
 
-	if (pin == -1)
-		return 0;
+	if (!gpio_is_valid(pin))
+		return ret;
 
 	ret = gpio_request(pin, name);
 	if (ret) {
@@ -946,7 +946,7 @@ static int vip_gpio_reserve(struct device *dev, int pin, int dir,
  */
 static void vip_gpio_release(struct device *dev, int pin, const char *name)
 {
-	if (pin != -1) {
+	if (gpio_is_valid(pin)) {
 		dev_dbg(dev, "releasing pin %d (%s)\n",	pin, name);
 		gpio_unexport(pin);
 		gpio_free(pin);
@@ -1003,25 +1003,24 @@ static int sta2x11_vip_init_one(struct pci_dev *pdev,
 	if (ret)
 		goto disable;
 
-	if (config->reset_pin >= 0) {
-		ret = vip_gpio_reserve(&pdev->dev, config->reset_pin, 0,
-				       config->reset_name);
-		if (ret) {
-			vip_gpio_release(&pdev->dev, config->pwr_pin,
-					 config->pwr_name);
-			goto disable;
-		}
+	ret = vip_gpio_reserve(&pdev->dev, config->reset_pin, 0,
+			       config->reset_name);
+	if (ret) {
+		vip_gpio_release(&pdev->dev, config->pwr_pin,
+				 config->pwr_name);
+		goto disable;
 	}
-	if (config->pwr_pin != -1) {
+
+	if (gpio_is_valid(config->pwr_pin)) {
 		/* Datasheet says 5ms between PWR and RST */
 		usleep_range(5000, 25000);
-		ret = gpio_direction_output(config->pwr_pin, 1);
+		gpio_direction_output(config->pwr_pin, 1);
 	}
 
-	if (config->reset_pin != -1) {
+	if (gpio_is_valid(config->reset_pin)) {
 		/* Datasheet says 5ms between PWR and RST */
 		usleep_range(5000, 25000);
-		ret = gpio_direction_output(config->reset_pin, 1);
+		gpio_direction_output(config->reset_pin, 1);
 	}
 	usleep_range(5000, 25000);
 
diff --git a/drivers/media/pci/tw5864/Kconfig b/drivers/media/pci/tw5864/Kconfig
index 87c8f327e2d4..760fb11dfeae 100644
--- a/drivers/media/pci/tw5864/Kconfig
+++ b/drivers/media/pci/tw5864/Kconfig
@@ -1,7 +1,6 @@
 config VIDEO_TW5864
 	tristate "Techwell TW5864 video/audio grabber and encoder"
 	depends on VIDEO_DEV && PCI && VIDEO_V4L2
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	---help---
 	  Support for boards based on Techwell TW5864 chip which provides
diff --git a/drivers/media/pci/tw686x/Kconfig b/drivers/media/pci/tw686x/Kconfig
index 34ff37712313..da8bfee71b44 100644
--- a/drivers/media/pci/tw686x/Kconfig
+++ b/drivers/media/pci/tw686x/Kconfig
@@ -1,7 +1,6 @@
 config VIDEO_TW686X
 	tristate "Intersil/Techwell TW686x video capture cards"
 	depends on PCI && VIDEO_DEV && VIDEO_V4L2 && SND
-	depends on HAS_DMA
 	select VIDEOBUF2_VMALLOC
 	select VIDEOBUF2_DMA_CONTIG
 	select VIDEOBUF2_DMA_SG
diff --git a/drivers/media/pci/tw686x/tw686x-video.c b/drivers/media/pci/tw686x/tw686x-video.c
index c3fafa97b2d0..0ea8dd44026c 100644
--- a/drivers/media/pci/tw686x/tw686x-video.c
+++ b/drivers/media/pci/tw686x/tw686x-video.c
@@ -1228,7 +1228,8 @@ int tw686x_video_init(struct tw686x_dev *dev)
 		vc->vidq.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
 		vc->vidq.min_buffers_needed = 2;
 		vc->vidq.lock = &vc->vb_mutex;
-		vc->vidq.gfp_flags = GFP_DMA32;
+		vc->vidq.gfp_flags = dev->dma_mode != TW686X_DMA_MODE_MEMCPY ?
+				     GFP_DMA32 : 0;
 		vc->vidq.dev = &dev->pci_dev->dev;
 
 		err = vb2_queue_init(&vc->vidq);
diff --git a/drivers/media/pci/zoran/Kconfig b/drivers/media/pci/zoran/Kconfig
deleted file mode 100644
index 39ec35bd21a5..000000000000
--- a/drivers/media/pci/zoran/Kconfig
+++ /dev/null
@@ -1,75 +0,0 @@
-config VIDEO_ZORAN
-	tristate "Zoran ZR36057/36067 Video For Linux"
-	depends on PCI && I2C_ALGOBIT && VIDEO_V4L2 && VIRT_TO_BUS
-	depends on !ALPHA
-	help
-	  Say Y for support for MJPEG capture cards based on the Zoran
-	  36057/36067 PCI controller chipset. This includes the Iomega
-	  Buz, Pinnacle DC10+ and the Linux Media Labs LML33. There is
-	  a driver homepage at <http://mjpeg.sf.net/driver-zoran/>. For
-	  more information, check <file:Documentation/video4linux/Zoran>.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called zr36067.
-
-config VIDEO_ZORAN_DC30
-	tristate "Pinnacle/Miro DC30(+) support"
-	depends on VIDEO_ZORAN
-	select VIDEO_ADV7175 if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_VPX3220 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  Support for the Pinnacle/Miro DC30(+) MJPEG capture/playback
-	  card. This also supports really old DC10 cards based on the
-	  zr36050 MJPEG codec and zr36016 VFE.
-
-config VIDEO_ZORAN_ZR36060
-	tristate "Zoran ZR36060"
-	depends on VIDEO_ZORAN
-	help
-	  Say Y to support Zoran boards based on 36060 chips.
-	  This includes Iomega Buz, Pinnacle DC10, Linux media Labs 33
-	  and 33 R10 and AverMedia 6 boards.
-
-config VIDEO_ZORAN_BUZ
-	tristate "Iomega Buz support"
-	depends on VIDEO_ZORAN_ZR36060
-	select VIDEO_SAA711X if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_SAA7185 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  Support for the Iomega Buz MJPEG capture/playback card.
-
-config VIDEO_ZORAN_DC10
-	tristate "Pinnacle/Miro DC10(+) support"
-	depends on VIDEO_ZORAN_ZR36060
-	select VIDEO_SAA7110 if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_ADV7175 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  Support for the Pinnacle/Miro DC10(+) MJPEG capture/playback
-	  card.
-
-config VIDEO_ZORAN_LML33
-	tristate "Linux Media Labs LML33 support"
-	depends on VIDEO_ZORAN_ZR36060
-	select VIDEO_BT819 if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_BT856 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  Support for the Linux Media Labs LML33 MJPEG capture/playback
-	  card.
-
-config VIDEO_ZORAN_LML33R10
-	tristate "Linux Media Labs LML33R10 support"
-	depends on VIDEO_ZORAN_ZR36060
-	select VIDEO_SAA711X if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_ADV7170 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  support for the Linux Media Labs LML33R10 MJPEG capture/playback
-	  card.
-
-config VIDEO_ZORAN_AVS6EYES
-	tristate "AverMedia 6 Eyes support"
-	depends on VIDEO_ZORAN_ZR36060
-	select VIDEO_BT856 if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_BT866 if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_KS0127 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  Support for the AverMedia 6 Eyes video surveillance card.
diff --git a/drivers/media/pci/zoran/Makefile b/drivers/media/pci/zoran/Makefile
deleted file mode 100644
index 21ac29a71458..000000000000
--- a/drivers/media/pci/zoran/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-zr36067-objs	:=	zoran_procfs.o zoran_device.o \
-			zoran_driver.o zoran_card.o
-
-obj-$(CONFIG_VIDEO_ZORAN) += zr36067.o videocodec.o
-obj-$(CONFIG_VIDEO_ZORAN_DC30) += zr36050.o zr36016.o
-obj-$(CONFIG_VIDEO_ZORAN_ZR36060) += zr36060.o
diff --git a/drivers/media/pci/zoran/videocodec.c b/drivers/media/pci/zoran/videocodec.c
deleted file mode 100644
index 5ff23ef89215..000000000000
--- a/drivers/media/pci/zoran/videocodec.c
+++ /dev/null
@@ -1,403 +0,0 @@
-/*
- * VIDEO MOTION CODECs internal API for video devices
- *
- * Interface for MJPEG (and maybe later MPEG/WAVELETS) codec's
- * bound to a master device.
- *
- * (c) 2002 Wolfgang Scherr <scherr@net4you.at>
- *
- * $Id: videocodec.c,v 1.1.2.8 2003/03/29 07:16:04 rbultje Exp $
- *
- * ------------------------------------------------------------------------
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * ------------------------------------------------------------------------
- */
-
-#define VIDEOCODEC_VERSION "v0.2"
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/types.h>
-#include <linux/slab.h>
-
-// kernel config is here (procfs flag)
-
-#ifdef CONFIG_PROC_FS
-#include <linux/proc_fs.h>
-#include <linux/seq_file.h>
-#include <linux/uaccess.h>
-#endif
-
-#include "videocodec.h"
-
-static int debug;
-module_param(debug, int, 0);
-MODULE_PARM_DESC(debug, "Debug level (0-4)");
-
-#define dprintk(num, format, args...) \
-	do { \
-		if (debug >= num) \
-			printk(format, ##args); \
-	} while (0)
-
-struct attached_list {
-	struct videocodec *codec;
-	struct attached_list *next;
-};
-
-struct codec_list {
-	const struct videocodec *codec;
-	int attached;
-	struct attached_list *list;
-	struct codec_list *next;
-};
-
-static struct codec_list *codeclist_top = NULL;
-
-/* ================================================= */
-/* function prototypes of the master/slave interface */
-/* ================================================= */
-
-struct videocodec *
-videocodec_attach (struct videocodec_master *master)
-{
-	struct codec_list *h = codeclist_top;
-	struct attached_list *a, *ptr;
-	struct videocodec *codec;
-	int res;
-
-	if (!master) {
-		dprintk(1, KERN_ERR "videocodec_attach: no data\n");
-		return NULL;
-	}
-
-	dprintk(2,
-		"videocodec_attach: '%s', flags %lx, magic %lx\n",
-		master->name, master->flags, master->magic);
-
-	if (!h) {
-		dprintk(1,
-			KERN_ERR
-			"videocodec_attach: no device available\n");
-		return NULL;
-	}
-
-	while (h) {
-		// attach only if the slave has at least the flags
-		// expected by the master
-		if ((master->flags & h->codec->flags) == master->flags) {
-			dprintk(4, "videocodec_attach: try '%s'\n",
-				h->codec->name);
-
-			if (!try_module_get(h->codec->owner))
-				return NULL;
-
-			codec = kmemdup(h->codec, sizeof(struct videocodec),
-					GFP_KERNEL);
-			if (!codec) {
-				dprintk(1,
-					KERN_ERR
-					"videocodec_attach: no mem\n");
-				goto out_module_put;
-			}
-
-			res = strlen(codec->name);
-			snprintf(codec->name + res, sizeof(codec->name) - res,
-				 "[%d]", h->attached);
-			codec->master_data = master;
-			res = codec->setup(codec);
-			if (res == 0) {
-				dprintk(3, "videocodec_attach '%s'\n",
-					codec->name);
-				ptr = kzalloc(sizeof(struct attached_list), GFP_KERNEL);
-				if (!ptr) {
-					dprintk(1,
-						KERN_ERR
-						"videocodec_attach: no memory\n");
-					goto out_kfree;
-				}
-				ptr->codec = codec;
-
-				a = h->list;
-				if (!a) {
-					h->list = ptr;
-					dprintk(4,
-						"videocodec: first element\n");
-				} else {
-					while (a->next)
-						a = a->next;	// find end
-					a->next = ptr;
-					dprintk(4,
-						"videocodec: in after '%s'\n",
-						h->codec->name);
-				}
-
-				h->attached += 1;
-				return codec;
-			} else {
-				kfree(codec);
-			}
-		}
-		h = h->next;
-	}
-
-	dprintk(1, KERN_ERR "videocodec_attach: no codec found!\n");
-	return NULL;
-
- out_module_put:
-	module_put(h->codec->owner);
- out_kfree:
-	kfree(codec);
-	return NULL;
-}
-
-int
-videocodec_detach (struct videocodec *codec)
-{
-	struct codec_list *h = codeclist_top;
-	struct attached_list *a, *prev;
-	int res;
-
-	if (!codec) {
-		dprintk(1, KERN_ERR "videocodec_detach: no data\n");
-		return -EINVAL;
-	}
-
-	dprintk(2,
-		"videocodec_detach: '%s', type: %x, flags %lx, magic %lx\n",
-		codec->name, codec->type, codec->flags, codec->magic);
-
-	if (!h) {
-		dprintk(1,
-			KERN_ERR "videocodec_detach: no device left...\n");
-		return -ENXIO;
-	}
-
-	while (h) {
-		a = h->list;
-		prev = NULL;
-		while (a) {
-			if (codec == a->codec) {
-				res = a->codec->unset(a->codec);
-				if (res >= 0) {
-					dprintk(3,
-						"videocodec_detach: '%s'\n",
-						a->codec->name);
-					a->codec->master_data = NULL;
-				} else {
-					dprintk(1,
-						KERN_ERR
-						"videocodec_detach: '%s'\n",
-						a->codec->name);
-					a->codec->master_data = NULL;
-				}
-				if (prev == NULL) {
-					h->list = a->next;
-					dprintk(4,
-						"videocodec: delete first\n");
-				} else {
-					prev->next = a->next;
-					dprintk(4,
-						"videocodec: delete middle\n");
-				}
-				module_put(a->codec->owner);
-				kfree(a->codec);
-				kfree(a);
-				h->attached -= 1;
-				return 0;
-			}
-			prev = a;
-			a = a->next;
-		}
-		h = h->next;
-	}
-
-	dprintk(1, KERN_ERR "videocodec_detach: given codec not found!\n");
-	return -EINVAL;
-}
-
-int
-videocodec_register (const struct videocodec *codec)
-{
-	struct codec_list *ptr, *h = codeclist_top;
-
-	if (!codec) {
-		dprintk(1, KERN_ERR "videocodec_register: no data!\n");
-		return -EINVAL;
-	}
-
-	dprintk(2,
-		"videocodec: register '%s', type: %x, flags %lx, magic %lx\n",
-		codec->name, codec->type, codec->flags, codec->magic);
-
-	ptr = kzalloc(sizeof(struct codec_list), GFP_KERNEL);
-	if (!ptr) {
-		dprintk(1, KERN_ERR "videocodec_register: no memory\n");
-		return -ENOMEM;
-	}
-	ptr->codec = codec;
-
-	if (!h) {
-		codeclist_top = ptr;
-		dprintk(4, "videocodec: hooked in as first element\n");
-	} else {
-		while (h->next)
-			h = h->next;	// find the end
-		h->next = ptr;
-		dprintk(4, "videocodec: hooked in after '%s'\n",
-			h->codec->name);
-	}
-
-	return 0;
-}
-
-int
-videocodec_unregister (const struct videocodec *codec)
-{
-	struct codec_list *prev = NULL, *h = codeclist_top;
-
-	if (!codec) {
-		dprintk(1, KERN_ERR "videocodec_unregister: no data!\n");
-		return -EINVAL;
-	}
-
-	dprintk(2,
-		"videocodec: unregister '%s', type: %x, flags %lx, magic %lx\n",
-		codec->name, codec->type, codec->flags, codec->magic);
-
-	if (!h) {
-		dprintk(1,
-			KERN_ERR
-			"videocodec_unregister: no device left...\n");
-		return -ENXIO;
-	}
-
-	while (h) {
-		if (codec == h->codec) {
-			if (h->attached) {
-				dprintk(1,
-					KERN_ERR
-					"videocodec: '%s' is used\n",
-					h->codec->name);
-				return -EBUSY;
-			}
-			dprintk(3, "videocodec: unregister '%s' is ok.\n",
-				h->codec->name);
-			if (prev == NULL) {
-				codeclist_top = h->next;
-				dprintk(4,
-					"videocodec: delete first element\n");
-			} else {
-				prev->next = h->next;
-				dprintk(4,
-					"videocodec: delete middle element\n");
-			}
-			kfree(h);
-			return 0;
-		}
-		prev = h;
-		h = h->next;
-	}
-
-	dprintk(1,
-		KERN_ERR
-		"videocodec_unregister: given codec not found!\n");
-	return -EINVAL;
-}
-
-#ifdef CONFIG_PROC_FS
-static int proc_videocodecs_show(struct seq_file *m, void *v)
-{
-	struct codec_list *h = codeclist_top;
-	struct attached_list *a;
-
-	seq_printf(m, "<S>lave or attached <M>aster name  type flags    magic    ");
-	seq_printf(m, "(connected as)\n");
-
-	while (h) {
-		seq_printf(m, "S %32s %04x %08lx %08lx (TEMPLATE)\n",
-			      h->codec->name, h->codec->type,
-			      h->codec->flags, h->codec->magic);
-		a = h->list;
-		while (a) {
-			seq_printf(m, "M %32s %04x %08lx %08lx (%s)\n",
-				      a->codec->master_data->name,
-				      a->codec->master_data->type,
-				      a->codec->master_data->flags,
-				      a->codec->master_data->magic,
-				      a->codec->name);
-			a = a->next;
-		}
-		h = h->next;
-	}
-
-	return 0;
-}
-
-static int proc_videocodecs_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, proc_videocodecs_show, NULL);
-}
-
-static const struct file_operations videocodecs_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= proc_videocodecs_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-#endif
-
-/* ===================== */
-/* hook in driver module */
-/* ===================== */
-static int __init
-videocodec_init (void)
-{
-#ifdef CONFIG_PROC_FS
-	static struct proc_dir_entry *videocodec_proc_entry;
-#endif
-
-	printk(KERN_INFO "Linux video codec intermediate layer: %s\n",
-	       VIDEOCODEC_VERSION);
-
-#ifdef CONFIG_PROC_FS
-	videocodec_proc_entry = proc_create("videocodecs", 0, NULL, &videocodecs_proc_fops);
-	if (!videocodec_proc_entry) {
-		dprintk(1, KERN_ERR "videocodec: can't init procfs.\n");
-	}
-#endif
-	return 0;
-}
-
-static void __exit
-videocodec_exit (void)
-{
-#ifdef CONFIG_PROC_FS
-	remove_proc_entry("videocodecs", NULL);
-#endif
-}
-
-EXPORT_SYMBOL(videocodec_attach);
-EXPORT_SYMBOL(videocodec_detach);
-EXPORT_SYMBOL(videocodec_register);
-EXPORT_SYMBOL(videocodec_unregister);
-
-module_init(videocodec_init);
-module_exit(videocodec_exit);
-
-MODULE_AUTHOR("Wolfgang Scherr <scherr@net4you.at>");
-MODULE_DESCRIPTION("Intermediate API module for video codecs "
-		   VIDEOCODEC_VERSION);
-MODULE_LICENSE("GPL");
diff --git a/drivers/media/pci/zoran/videocodec.h b/drivers/media/pci/zoran/videocodec.h
deleted file mode 100644
index 8ed5a0f7ac01..000000000000
--- a/drivers/media/pci/zoran/videocodec.h
+++ /dev/null
@@ -1,349 +0,0 @@
-/*
- * VIDEO MOTION CODECs internal API for video devices
- *
- * Interface for MJPEG (and maybe later MPEG/WAVELETS) codec's
- * bound to a master device.
- *
- * (c) 2002 Wolfgang Scherr <scherr@net4you.at>
- *
- * $Id: videocodec.h,v 1.1.2.4 2003/01/14 21:15:03 rbultje Exp $
- *
- * ------------------------------------------------------------------------
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * ------------------------------------------------------------------------
- */
-
-/* =================== */
-/* general description */
-/* =================== */
-
-/* Should ease the (re-)usage of drivers supporting cards with (different)
-   video codecs. The codecs register to this module their functionality,
-   and the processors (masters) can attach to them if they fit.
-
-   The codecs are typically have a "strong" binding to their master - so I
-   don't think it makes sense to have a full blown interfacing as with e.g.
-   i2c. If you have an other opinion, let's discuss & implement it :-)))
-
-   Usage:
-
-   The slave has just to setup the videocodec structure and use two functions:
-   videocodec_register(codecdata);
-   videocodec_unregister(codecdata);
-   The best is just calling them at module (de-)initialisation.
-
-   The master sets up the structure videocodec_master and calls:
-   codecdata=videocodec_attach(master_codecdata);
-   videocodec_detach(codecdata);
-
-   The slave is called during attach/detach via functions setup previously
-   during register. At that time, the master_data pointer is set up
-   and the slave can access any io registers of the master device (in the case
-   the slave is bound to it). Otherwise it doesn't need this functions and
-   therfor they may not be initialized.
-
-   The other functions are just for convenience, as they are for sure used by
-   most/all of the codecs. The last ones may be omitted, too.
-
-   See the structure declaration below for more information and which data has
-   to be set up for the master and the slave.
-
-   ----------------------------------------------------------------------------
-   The master should have "knowledge" of the slave and vice versa.  So the data
-   structures sent to/from slave via set_data/get_data set_image/get_image are
-   device dependent and vary between MJPEG/MPEG/WAVELET/... devices. (!!!!)
-   ----------------------------------------------------------------------------
-*/
-
-
-/* ========================================== */
-/* description of the videocodec_io structure */
-/* ========================================== */
-
-/*
-   ==== master setup ====
-   name -> name of the device structure for reference and debugging
-   master_data ->  data ref. for the master (e.g. the zr36055,57,67)
-   readreg -> ref. to read-fn from register (setup by master, used by slave)
-   writereg -> ref. to write-fn to register (setup by master, used by slave)
-	       this two functions do the lowlevel I/O job
-
-   ==== slave functionality setup ====
-   slave_data -> data ref. for the slave (e.g. the zr36050,60)
-   check -> fn-ref. checks availability of an device, returns -EIO on failure or
-	    the type on success
-	    this makes espcecially sense if a driver module supports more than
-	    one codec which may be quite similar to access, nevertheless it
-	    is good for a first functionality check
-
-   -- main functions you always need for compression/decompression --
-
-   set_mode -> this fn-ref. resets the entire codec, and sets up the mode
-	       with the last defined norm/size (or device default if not
-	       available) - it returns 0 if the mode is possible
-   set_size -> this fn-ref. sets the norm and image size for
-	       compression/decompression (returns 0 on success)
-	       the norm param is defined in videodev2.h (V4L2_STD_*)
-
-   additional setup may be available, too - but the codec should work with
-   some default values even without this
-
-   set_data -> sets device-specific data (tables, quality etc.)
-   get_data -> query device-specific data (tables, quality etc.)
-
-   if the device delivers interrupts, they may be setup/handled here
-   setup_interrupt -> codec irq setup (not needed for 36050/60)
-   handle_interrupt -> codec irq handling (not needed for 36050/60)
-
-   if the device delivers pictures, they may be handled here
-   put_image -> puts image data to the codec (not needed for 36050/60)
-   get_image -> gets image data from the codec (not needed for 36050/60)
-		the calls include frame numbers and flags (even/odd/...)
-		if needed and a flag which allows blocking until its ready
-*/
-
-/* ============== */
-/* user interface */
-/* ============== */
-
-/*
-   Currently there is only a information display planned, as the layer
-   is not visible for the user space at all.
-
-   Information is available via procfs. The current entry is "/proc/videocodecs"
-   but it makes sense to "hide" it in the /proc/video tree of v4l(2) --TODO--.
-
-A example for such an output is:
-
-<S>lave or attached <M>aster name  type flags    magic    (connected as)
-S                          zr36050 0002 0000d001 00000000 (TEMPLATE)
-M                       zr36055[0] 0001 0000c001 00000000 (zr36050[0])
-M                       zr36055[1] 0001 0000c001 00000000 (zr36050[1])
-
-*/
-
-
-/* =============================================== */
-/* special defines for the videocodec_io structure */
-/* =============================================== */
-
-#ifndef __LINUX_VIDEOCODEC_H
-#define __LINUX_VIDEOCODEC_H
-
-#include <linux/videodev2.h>
-
-#define CODEC_DO_COMPRESSION 0
-#define CODEC_DO_EXPANSION   1
-
-/* this are the current codec flags I think they are needed */
-/*  -> type value in structure */
-#define CODEC_FLAG_JPEG      0x00000001L	// JPEG codec
-#define CODEC_FLAG_MPEG      0x00000002L	// MPEG1/2/4 codec
-#define CODEC_FLAG_DIVX      0x00000004L	// DIVX codec
-#define CODEC_FLAG_WAVELET   0x00000008L	// WAVELET codec
-					  // room for other types
-
-#define CODEC_FLAG_MAGIC     0x00000800L	// magic key must match
-#define CODEC_FLAG_HARDWARE  0x00001000L	// is a hardware codec
-#define CODEC_FLAG_VFE       0x00002000L	// has direct video frontend
-#define CODEC_FLAG_ENCODER   0x00004000L	// compression capability
-#define CODEC_FLAG_DECODER   0x00008000L	// decompression capability
-#define CODEC_FLAG_NEEDIRQ   0x00010000L	// needs irq handling
-#define CODEC_FLAG_RDWRPIC   0x00020000L	// handles picture I/O
-
-/* a list of modes, some are just examples (is there any HW?) */
-#define CODEC_MODE_BJPG      0x0001	// Baseline JPEG
-#define CODEC_MODE_LJPG      0x0002	// Lossless JPEG
-#define CODEC_MODE_MPEG1     0x0003	// MPEG 1
-#define CODEC_MODE_MPEG2     0x0004	// MPEG 2
-#define CODEC_MODE_MPEG4     0x0005	// MPEG 4
-#define CODEC_MODE_MSDIVX    0x0006	// MS DivX
-#define CODEC_MODE_ODIVX     0x0007	// Open DivX
-#define CODEC_MODE_WAVELET   0x0008	// Wavelet
-
-/* this are the current codec types I want to implement */
-/*  -> type value in structure */
-#define CODEC_TYPE_NONE    0
-#define CODEC_TYPE_L64702  1
-#define CODEC_TYPE_ZR36050 2
-#define CODEC_TYPE_ZR36016 3
-#define CODEC_TYPE_ZR36060 4
-
-/* the type of data may be enhanced by future implementations (data-fn.'s) */
-/*  -> used in command                                                     */
-#define CODEC_G_STATUS         0x0000	/* codec status (query only) */
-#define CODEC_S_CODEC_MODE     0x0001	/* codec mode (baseline JPEG, MPEG1,... */
-#define CODEC_G_CODEC_MODE     0x8001
-#define CODEC_S_VFE            0x0002	/* additional video frontend setup */
-#define CODEC_G_VFE            0x8002
-#define CODEC_S_MMAP           0x0003	/* MMAP setup (if available) */
-
-#define CODEC_S_JPEG_TDS_BYTE  0x0010	/* target data size in bytes */
-#define CODEC_G_JPEG_TDS_BYTE  0x8010
-#define CODEC_S_JPEG_SCALE     0x0011	/* scaling factor for quant. tables */
-#define CODEC_G_JPEG_SCALE     0x8011
-#define CODEC_S_JPEG_HDT_DATA  0x0018	/* huffman-tables */
-#define CODEC_G_JPEG_HDT_DATA  0x8018
-#define CODEC_S_JPEG_QDT_DATA  0x0019	/* quantizing-tables */
-#define CODEC_G_JPEG_QDT_DATA  0x8019
-#define CODEC_S_JPEG_APP_DATA  0x001A	/* APP marker */
-#define CODEC_G_JPEG_APP_DATA  0x801A
-#define CODEC_S_JPEG_COM_DATA  0x001B	/* COM marker */
-#define CODEC_G_JPEG_COM_DATA  0x801B
-
-#define CODEC_S_PRIVATE        0x1000	/* "private" commands start here */
-#define CODEC_G_PRIVATE        0x9000
-
-#define CODEC_G_FLAG           0x8000	/* this is how 'get' is detected */
-
-/* types of transfer, directly user space or a kernel buffer (image-fn.'s) */
-/*  -> used in get_image, put_image                                        */
-#define CODEC_TRANSFER_KERNEL 0	/* use "memcopy" */
-#define CODEC_TRANSFER_USER   1	/* use "to/from_user" */
-
-
-/* ========================= */
-/* the structures itself ... */
-/* ========================= */
-
-struct vfe_polarity {
-	unsigned int vsync_pol:1;
-	unsigned int hsync_pol:1;
-	unsigned int field_pol:1;
-	unsigned int blank_pol:1;
-	unsigned int subimg_pol:1;
-	unsigned int poe_pol:1;
-	unsigned int pvalid_pol:1;
-	unsigned int vclk_pol:1;
-};
-
-struct vfe_settings {
-	__u32 x, y;		/* Offsets into image */
-	__u32 width, height;	/* Area to capture */
-	__u16 decimation;	/* Decimation divider */
-	__u16 flags;		/* Flags for capture */
-	__u16 quality;		/* quality of the video */
-};
-
-struct tvnorm {
-	u16 Wt, Wa, HStart, HSyncStart, Ht, Ha, VStart;
-};
-
-struct jpeg_com_marker {
-	int len; /* number of usable bytes in data */
-	char data[60];
-};
-
-struct jpeg_app_marker {
-	int appn; /* number app segment */
-	int len; /* number of usable bytes in data */
-	char data[60];
-};
-
-struct videocodec {
-	struct module *owner;
-	/* -- filled in by slave device during register -- */
-	char name[32];
-	unsigned long magic;	/* may be used for client<->master attaching */
-	unsigned long flags;	/* functionality flags */
-	unsigned int type;	/* codec type */
-
-	/* -- these is filled in later during master device attach -- */
-
-	struct videocodec_master *master_data;
-
-	/* -- these are filled in by the slave device during register -- */
-
-	void *data;		/* private slave data */
-
-	/* attach/detach client functions (indirect call) */
-	int (*setup) (struct videocodec * codec);
-	int (*unset) (struct videocodec * codec);
-
-	/* main functions, every client needs them for sure! */
-	// set compression or decompression (or freeze, stop, standby, etc)
-	int (*set_mode) (struct videocodec * codec,
-			 int mode);
-	// setup picture size and norm (for the codec's video frontend)
-	int (*set_video) (struct videocodec * codec,
-			  struct tvnorm * norm,
-			  struct vfe_settings * cap,
-			  struct vfe_polarity * pol);
-	// other control commands, also mmap setup etc.
-	int (*control) (struct videocodec * codec,
-			int type,
-			int size,
-			void *data);
-
-	/* additional setup/query/processing (may be NULL pointer) */
-	// interrupt setup / handling (for irq's delivered by master)
-	int (*setup_interrupt) (struct videocodec * codec,
-				long mode);
-	int (*handle_interrupt) (struct videocodec * codec,
-				 int source,
-				 long flag);
-	// picture interface (if any)
-	long (*put_image) (struct videocodec * codec,
-			   int tr_type,
-			   int block,
-			   long *fr_num,
-			   long *flag,
-			   long size,
-			   void *buf);
-	long (*get_image) (struct videocodec * codec,
-			   int tr_type,
-			   int block,
-			   long *fr_num,
-			   long *flag,
-			   long size,
-			   void *buf);
-};
-
-struct videocodec_master {
-	/* -- filled in by master device for registration -- */
-	char name[32];
-	unsigned long magic;	/* may be used for client<->master attaching */
-	unsigned long flags;	/* functionality flags */
-	unsigned int type;	/* master type */
-
-	void *data;		/* private master data */
-
-	 __u32(*readreg) (struct videocodec * codec,
-			  __u16 reg);
-	void (*writereg) (struct videocodec * codec,
-			  __u16 reg,
-			  __u32 value);
-};
-
-
-/* ================================================= */
-/* function prototypes of the master/slave interface */
-/* ================================================= */
-
-/* attach and detach commands for the master */
-// * master structure needs to be kmalloc'ed before calling attach
-//   and free'd after calling detach
-// * returns pointer on success, NULL on failure
-extern struct videocodec *videocodec_attach(struct videocodec_master *);
-// * 0 on success, <0 (errno) on failure
-extern int videocodec_detach(struct videocodec *);
-
-/* register and unregister commands for the slaves */
-// * 0 on success, <0 (errno) on failure
-extern int videocodec_register(const struct videocodec *);
-// * 0 on success, <0 (errno) on failure
-extern int videocodec_unregister(const struct videocodec *);
-
-/* the other calls are directly done via the videocodec structure! */
-
-#endif				/*ifndef __LINUX_VIDEOCODEC_H */
diff --git a/drivers/media/pci/zoran/zoran.h b/drivers/media/pci/zoran/zoran.h
deleted file mode 100644
index 9bb3c21aa275..000000000000
--- a/drivers/media/pci/zoran/zoran.h
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * zoran - Iomega Buz driver
- *
- * Copyright (C) 1999 Rainer Johanni <Rainer@Johanni.de>
- *
- * based on
- *
- * zoran.0.0.3 Copyright (C) 1998 Dave Perks <dperks@ibm.net>
- *
- * and
- *
- * bttv - Bt848 frame grabber driver
- * Copyright (C) 1996,97,98 Ralph  Metzler (rjkm@thp.uni-koeln.de)
- *                        & Marcus Metzler (mocm@thp.uni-koeln.de)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#ifndef _BUZ_H_
-#define _BUZ_H_
-
-#include <media/v4l2-device.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-fh.h>
-
-struct zoran_sync {
-	unsigned long frame;	/* number of buffer that has been free'd */
-	unsigned long length;	/* number of code bytes in buffer (capture only) */
-	unsigned long seq;	/* frame sequence number */
-	struct timeval timestamp;	/* timestamp */
-};
-
-
-#define ZORAN_NAME    "ZORAN"	/* name of the device */
-
-#define ZR_DEVNAME(zr) ((zr)->name)
-
-#define   BUZ_MAX_WIDTH   (zr->timing->Wa)
-#define   BUZ_MAX_HEIGHT  (zr->timing->Ha)
-#define   BUZ_MIN_WIDTH    32	/* never display less than 32 pixels */
-#define   BUZ_MIN_HEIGHT   24	/* never display less than 24 rows */
-
-#define BUZ_NUM_STAT_COM    4
-#define BUZ_MASK_STAT_COM   3
-
-#define BUZ_MAX_FRAME     256	/* Must be a power of 2 */
-#define BUZ_MASK_FRAME    255	/* Must be BUZ_MAX_FRAME-1 */
-
-#define BUZ_MAX_INPUT       16
-
-#if VIDEO_MAX_FRAME <= 32
-#   define   V4L_MAX_FRAME   32
-#elif VIDEO_MAX_FRAME <= 64
-#   define   V4L_MAX_FRAME   64
-#else
-#   error   "Too many video frame buffers to handle"
-#endif
-#define   V4L_MASK_FRAME   (V4L_MAX_FRAME - 1)
-
-#define MAX_FRAME (BUZ_MAX_FRAME > VIDEO_MAX_FRAME ? BUZ_MAX_FRAME : VIDEO_MAX_FRAME)
-
-#include "zr36057.h"
-
-enum card_type {
-	UNKNOWN = -1,
-
-	/* Pinnacle/Miro */
-	DC10_old,		/* DC30 like */
-	DC10_new,		/* DC10plus like */
-	DC10plus,
-	DC30,
-	DC30plus,
-
-	/* Linux Media Labs */
-	LML33,
-	LML33R10,
-
-	/* Iomega */
-	BUZ,
-
-	/* AverMedia */
-	AVS6EYES,
-
-	/* total number of cards */
-	NUM_CARDS
-};
-
-enum zoran_codec_mode {
-	BUZ_MODE_IDLE,		/* nothing going on */
-	BUZ_MODE_MOTION_COMPRESS,	/* grabbing frames */
-	BUZ_MODE_MOTION_DECOMPRESS,	/* playing frames */
-	BUZ_MODE_STILL_COMPRESS,	/* still frame conversion */
-	BUZ_MODE_STILL_DECOMPRESS	/* still frame conversion */
-};
-
-enum zoran_buffer_state {
-	BUZ_STATE_USER,		/* buffer is owned by application */
-	BUZ_STATE_PEND,		/* buffer is queued in pend[] ready to feed to I/O */
-	BUZ_STATE_DMA,		/* buffer is queued in dma[] for I/O */
-	BUZ_STATE_DONE		/* buffer is ready to return to application */
-};
-
-enum zoran_map_mode {
-	ZORAN_MAP_MODE_RAW,
-	ZORAN_MAP_MODE_JPG_REC,
-#define ZORAN_MAP_MODE_JPG ZORAN_MAP_MODE_JPG_REC
-	ZORAN_MAP_MODE_JPG_PLAY,
-};
-
-enum gpio_type {
-	ZR_GPIO_JPEG_SLEEP = 0,
-	ZR_GPIO_JPEG_RESET,
-	ZR_GPIO_JPEG_FRAME,
-	ZR_GPIO_VID_DIR,
-	ZR_GPIO_VID_EN,
-	ZR_GPIO_VID_RESET,
-	ZR_GPIO_CLK_SEL1,
-	ZR_GPIO_CLK_SEL2,
-	ZR_GPIO_MAX,
-};
-
-enum gpcs_type {
-	GPCS_JPEG_RESET = 0,
-	GPCS_JPEG_START,
-	GPCS_MAX,
-};
-
-struct zoran_format {
-	char *name;
-	__u32 fourcc;
-	int colorspace;
-	int depth;
-	__u32 flags;
-	__u32 vfespfr;
-};
-/* flags */
-#define ZORAN_FORMAT_COMPRESSED 1<<0
-#define ZORAN_FORMAT_OVERLAY    1<<1
-#define ZORAN_FORMAT_CAPTURE	1<<2
-#define ZORAN_FORMAT_PLAYBACK	1<<3
-
-/* overlay-settings */
-struct zoran_overlay_settings {
-	int is_set;
-	int x, y, width, height;	/* position */
-	int clipcount;		/* position and number of clips */
-	const struct zoran_format *format;	/* overlay format */
-};
-
-/* v4l-capture settings */
-struct zoran_v4l_settings {
-	int width, height, bytesperline;	/* capture size */
-	const struct zoran_format *format;	/* capture format */
-};
-
-/* jpg-capture/-playback settings */
-struct zoran_jpg_settings {
-	int decimation;		/* this bit is used to set everything to default */
-	int HorDcm, VerDcm, TmpDcm;	/* capture decimation settings (TmpDcm=1 means both fields) */
-	int field_per_buff, odd_even;	/* field-settings (odd_even=1 (+TmpDcm=1) means top-field-first) */
-	int img_x, img_y, img_width, img_height;	/* crop settings (subframe capture) */
-	struct v4l2_jpegcompression jpg_comp;	/* JPEG-specific capture settings */
-};
-
-struct zoran_fh;
-
-struct zoran_mapping {
-	struct zoran_fh *fh;
-	atomic_t count;
-};
-
-struct zoran_buffer {
-	struct zoran_mapping *map;
-	enum zoran_buffer_state state;	/* state: unused/pending/dma/done */
-	struct zoran_sync bs;		/* DONE: info to return to application */
-	union {
-		struct {
-			__le32 *frag_tab;	/* addresses of frag table */
-			u32 frag_tab_bus;	/* same value cached to save time in ISR */
-		} jpg;
-		struct {
-			char *fbuffer;		/* virtual address of frame buffer */
-			unsigned long fbuffer_phys;/* physical address of frame buffer */
-			unsigned long fbuffer_bus;/* bus address of frame buffer */
-		} v4l;
-	};
-};
-
-enum zoran_lock_activity {
-	ZORAN_FREE,		/* free for use */
-	ZORAN_ACTIVE,		/* active but unlocked */
-	ZORAN_LOCKED,		/* locked */
-};
-
-/* buffer collections */
-struct zoran_buffer_col {
-	enum zoran_lock_activity active;	/* feature currently in use? */
-	unsigned int num_buffers, buffer_size;
-	struct zoran_buffer buffer[MAX_FRAME];	/* buffers */
-	u8 allocated;		/* Flag if buffers are allocated  */
-	u8 need_contiguous;	/* Flag if contiguous buffers are needed */
-	/* only applies to jpg buffers, raw buffers are always contiguous */
-};
-
-struct zoran;
-
-/* zoran_fh contains per-open() settings */
-struct zoran_fh {
-	struct v4l2_fh fh;
-	struct zoran *zr;
-
-	enum zoran_map_mode map_mode;		/* Flag which bufferset will map by next mmap() */
-
-	struct zoran_overlay_settings overlay_settings;
-	u32 *overlay_mask;			/* overlay mask */
-	enum zoran_lock_activity overlay_active;/* feature currently in use? */
-
-	struct zoran_buffer_col buffers;	/* buffers' info */
-
-	struct zoran_v4l_settings v4l_settings;	/* structure with a lot of things to play with */
-	struct zoran_jpg_settings jpg_settings;	/* structure with a lot of things to play with */
-};
-
-struct card_info {
-	enum card_type type;
-	char name[32];
-	const char *i2c_decoder;	/* i2c decoder device */
-	const unsigned short *addrs_decoder;
-	const char *i2c_encoder;	/* i2c encoder device */
-	const unsigned short *addrs_encoder;
-	u16 video_vfe, video_codec;			/* videocodec types */
-	u16 audio_chip;					/* audio type */
-
-	int inputs;		/* number of video inputs */
-	struct input {
-		int muxsel;
-		char name[32];
-	} input[BUZ_MAX_INPUT];
-
-	v4l2_std_id norms;
-	struct tvnorm *tvn[3];	/* supported TV norms */
-
-	u32 jpeg_int;		/* JPEG interrupt */
-	u32 vsync_int;		/* VSYNC interrupt */
-	s8 gpio[ZR_GPIO_MAX];
-	u8 gpcs[GPCS_MAX];
-
-	struct vfe_polarity vfe_pol;
-	u8 gpio_pol[ZR_GPIO_MAX];
-
-	/* is the /GWS line connected? */
-	u8 gws_not_connected;
-
-	/* avs6eyes mux setting */
-	u8 input_mux;
-
-	void (*init) (struct zoran * zr);
-};
-
-struct zoran {
-	struct v4l2_device v4l2_dev;
-	struct v4l2_ctrl_handler hdl;
-	struct video_device *video_dev;
-
-	struct i2c_adapter i2c_adapter;	/* */
-	struct i2c_algo_bit_data i2c_algo;	/* */
-	u32 i2cbr;
-
-	struct v4l2_subdev *decoder;	/* video decoder sub-device */
-	struct v4l2_subdev *encoder;	/* video encoder sub-device */
-
-	struct videocodec *codec;	/* video codec */
-	struct videocodec *vfe;	/* video front end */
-
-	struct mutex lock;	/* file ops serialize lock */
-
-	u8 initialized;		/* flag if zoran has been correctly initialized */
-	int user;		/* number of current users */
-	struct card_info card;
-	struct tvnorm *timing;
-
-	unsigned short id;	/* number of this device */
-	char name[32];		/* name of this device */
-	struct pci_dev *pci_dev;	/* PCI device */
-	unsigned char revision;	/* revision of zr36057 */
-	unsigned char __iomem *zr36057_mem;/* pointer to mapped IO memory */
-
-	spinlock_t spinlock;	/* Spinlock */
-
-	/* Video for Linux parameters */
-	int input;	/* card's norm and input */
-	v4l2_std_id norm;
-
-	/* Current buffer params */
-	void    *vbuf_base;
-	int     vbuf_height, vbuf_width;
-	int     vbuf_depth;
-	int     vbuf_bytesperline;
-
-	struct zoran_overlay_settings overlay_settings;
-	u32 *overlay_mask;	/* overlay mask */
-	enum zoran_lock_activity overlay_active;	/* feature currently in use? */
-
-	wait_queue_head_t v4l_capq;
-
-	int v4l_overlay_active;	/* Overlay grab is activated */
-	int v4l_memgrab_active;	/* Memory grab is activated */
-
-	int v4l_grab_frame;	/* Frame number being currently grabbed */
-#define NO_GRAB_ACTIVE (-1)
-	unsigned long v4l_grab_seq;	/* Number of frames grabbed */
-	struct zoran_v4l_settings v4l_settings;	/* structure with a lot of things to play with */
-
-	/* V4L grab queue of frames pending */
-	unsigned long v4l_pend_head;
-	unsigned long v4l_pend_tail;
-	unsigned long v4l_sync_tail;
-	int v4l_pend[V4L_MAX_FRAME];
-	struct zoran_buffer_col v4l_buffers;	/* V4L buffers' info */
-
-	/* Buz MJPEG parameters */
-	enum zoran_codec_mode codec_mode;	/* status of codec */
-	struct zoran_jpg_settings jpg_settings;	/* structure with a lot of things to play with */
-
-	wait_queue_head_t jpg_capq;	/* wait here for grab to finish */
-
-	/* grab queue counts/indices, mask with BUZ_MASK_STAT_COM before using as index */
-	/* (dma_head - dma_tail) is number active in DMA, must be <= BUZ_NUM_STAT_COM */
-	/* (value & BUZ_MASK_STAT_COM) corresponds to index in stat_com table */
-	unsigned long jpg_que_head;	/* Index where to put next buffer which is queued */
-	unsigned long jpg_dma_head;	/* Index of next buffer which goes into stat_com  */
-	unsigned long jpg_dma_tail;	/* Index of last buffer in stat_com               */
-	unsigned long jpg_que_tail;	/* Index of last buffer in queue                  */
-	unsigned long jpg_seq_num;	/* count of frames since grab/play started        */
-	unsigned long jpg_err_seq;	/* last seq_num before error                      */
-	unsigned long jpg_err_shift;
-	unsigned long jpg_queued_num;	/* count of frames queued since grab/play started */
-
-	/* zr36057's code buffer table */
-	__le32 *stat_com;		/* stat_com[i] is indexed by dma_head/tail & BUZ_MASK_STAT_COM */
-
-	/* (value & BUZ_MASK_FRAME) corresponds to index in pend[] queue */
-	int jpg_pend[BUZ_MAX_FRAME];
-
-	/* array indexed by frame number */
-	struct zoran_buffer_col jpg_buffers;	/* MJPEG buffers' info */
-
-	/* Additional stuff for testing */
-#ifdef CONFIG_PROC_FS
-	struct proc_dir_entry *zoran_proc;
-#else
-	void *zoran_proc;
-#endif
-	int testing;
-	int jpeg_error;
-	int intr_counter_GIRQ1;
-	int intr_counter_GIRQ0;
-	int intr_counter_CodRepIRQ;
-	int intr_counter_JPEGRepIRQ;
-	int field_counter;
-	int IRQ1_in;
-	int IRQ1_out;
-	int JPEG_in;
-	int JPEG_out;
-	int JPEG_0;
-	int JPEG_1;
-	int END_event_missed;
-	int JPEG_missed;
-	int JPEG_error;
-	int num_errors;
-	int JPEG_max_missed;
-	int JPEG_min_missed;
-
-	u32 last_isr;
-	unsigned long frame_num;
-
-	wait_queue_head_t test_q;
-};
-
-static inline struct zoran *to_zoran(struct v4l2_device *v4l2_dev)
-{
-	return container_of(v4l2_dev, struct zoran, v4l2_dev);
-}
-
-/* There was something called _ALPHA_BUZ that used the PCI address instead of
- * the kernel iomapped address for btread/btwrite.  */
-#define btwrite(dat,adr)    writel((dat), zr->zr36057_mem+(adr))
-#define btread(adr)         readl(zr->zr36057_mem+(adr))
-
-#define btand(dat,adr)      btwrite((dat) & btread(adr), adr)
-#define btor(dat,adr)       btwrite((dat) | btread(adr), adr)
-#define btaor(dat,mask,adr) btwrite((dat) | ((mask) & btread(adr)), adr)
-
-#endif
diff --git a/drivers/media/pci/zoran/zoran_card.c b/drivers/media/pci/zoran/zoran_card.c
deleted file mode 100644
index a6b9ebd20263..000000000000
--- a/drivers/media/pci/zoran/zoran_card.c
+++ /dev/null
@@ -1,1524 +0,0 @@
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles card-specific data and detection
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/delay.h>
-
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/vmalloc.h>
-#include <linux/slab.h>
-
-#include <linux/proc_fs.h>
-#include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
-#include <linux/videodev2.h>
-#include <linux/spinlock.h>
-#include <linux/sem.h>
-#include <linux/kmod.h>
-#include <linux/wait.h>
-
-#include <linux/pci.h>
-#include <linux/interrupt.h>
-#include <linux/mutex.h>
-#include <linux/io.h>
-#include <media/v4l2-common.h>
-#include <media/i2c/bt819.h>
-
-#include "videocodec.h"
-#include "zoran.h"
-#include "zoran_card.h"
-#include "zoran_device.h"
-#include "zoran_procfs.h"
-
-extern const struct zoran_format zoran_formats[];
-
-static int card[BUZ_MAX] = { [0 ... (BUZ_MAX-1)] = -1 };
-module_param_array(card, int, NULL, 0444);
-MODULE_PARM_DESC(card, "Card type");
-
-/*
-   The video mem address of the video card.
-   The driver has a little database for some videocards
-   to determine it from there. If your video card is not in there
-   you have either to give it to the driver as a parameter
-   or set in in a VIDIOCSFBUF ioctl
- */
-
-static unsigned long vidmem;	/* default = 0 - Video memory base address */
-module_param_hw(vidmem, ulong, iomem, 0444);
-MODULE_PARM_DESC(vidmem, "Default video memory base address");
-
-/*
-   Default input and video norm at startup of the driver.
-*/
-
-static unsigned int default_input;	/* default 0 = Composite, 1 = S-Video */
-module_param(default_input, uint, 0444);
-MODULE_PARM_DESC(default_input,
-		 "Default input (0=Composite, 1=S-Video, 2=Internal)");
-
-static int default_mux = 1;	/* 6 Eyes input selection */
-module_param(default_mux, int, 0644);
-MODULE_PARM_DESC(default_mux,
-		 "Default 6 Eyes mux setting (Input selection)");
-
-static int default_norm;	/* default 0 = PAL, 1 = NTSC 2 = SECAM */
-module_param(default_norm, int, 0444);
-MODULE_PARM_DESC(default_norm, "Default norm (0=PAL, 1=NTSC, 2=SECAM)");
-
-/* /dev/videoN, -1 for autodetect */
-static int video_nr[BUZ_MAX] = { [0 ... (BUZ_MAX-1)] = -1 };
-module_param_array(video_nr, int, NULL, 0444);
-MODULE_PARM_DESC(video_nr, "Video device number (-1=Auto)");
-
-int v4l_nbufs = 4;
-int v4l_bufsize = 864;		/* Everybody should be able to work with this setting */
-module_param(v4l_nbufs, int, 0644);
-MODULE_PARM_DESC(v4l_nbufs, "Maximum number of V4L buffers to use");
-module_param(v4l_bufsize, int, 0644);
-MODULE_PARM_DESC(v4l_bufsize, "Maximum size per V4L buffer (in kB)");
-
-int jpg_nbufs = 32;
-int jpg_bufsize = 512;		/* max size for 100% quality full-PAL frame */
-module_param(jpg_nbufs, int, 0644);
-MODULE_PARM_DESC(jpg_nbufs, "Maximum number of JPG buffers to use");
-module_param(jpg_bufsize, int, 0644);
-MODULE_PARM_DESC(jpg_bufsize, "Maximum size per JPG buffer (in kB)");
-
-int pass_through = 0;		/* 1=Pass through TV signal when device is not used */
-				/* 0=Show color bar when device is not used (LML33: only if lml33dpath=1) */
-module_param(pass_through, int, 0644);
-MODULE_PARM_DESC(pass_through,
-		 "Pass TV signal through to TV-out when idling");
-
-int zr36067_debug = 1;
-module_param_named(debug, zr36067_debug, int, 0644);
-MODULE_PARM_DESC(debug, "Debug level (0-5)");
-
-#define ZORAN_VERSION "0.10.1"
-
-MODULE_DESCRIPTION("Zoran-36057/36067 JPEG codec driver");
-MODULE_AUTHOR("Serguei Miridonov");
-MODULE_LICENSE("GPL");
-MODULE_VERSION(ZORAN_VERSION);
-
-#define ZR_DEVICE(subven, subdev, data)	{ \
-	.vendor = PCI_VENDOR_ID_ZORAN, .device = PCI_DEVICE_ID_ZORAN_36057, \
-	.subvendor = (subven), .subdevice = (subdev), .driver_data = (data) }
-
-static const struct pci_device_id zr36067_pci_tbl[] = {
-	ZR_DEVICE(PCI_VENDOR_ID_MIRO, PCI_DEVICE_ID_MIRO_DC10PLUS, DC10plus),
-	ZR_DEVICE(PCI_VENDOR_ID_MIRO, PCI_DEVICE_ID_MIRO_DC30PLUS, DC30plus),
-	ZR_DEVICE(PCI_VENDOR_ID_ELECTRONICDESIGNGMBH, PCI_DEVICE_ID_LML_33R10, LML33R10),
-	ZR_DEVICE(PCI_VENDOR_ID_IOMEGA, PCI_DEVICE_ID_IOMEGA_BUZ, BUZ),
-	ZR_DEVICE(PCI_ANY_ID, PCI_ANY_ID, NUM_CARDS),
-	{0}
-};
-MODULE_DEVICE_TABLE(pci, zr36067_pci_tbl);
-
-static unsigned int zoran_num;		/* number of cards found */
-
-/* videocodec bus functions ZR36060 */
-static u32
-zr36060_read (struct videocodec *codec,
-	      u16                reg)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-	__u32 data;
-
-	if (post_office_wait(zr)
-	    || post_office_write(zr, 0, 1, reg >> 8)
-	    || post_office_write(zr, 0, 2, reg & 0xff)) {
-		return -1;
-	}
-
-	data = post_office_read(zr, 0, 3) & 0xff;
-	return data;
-}
-
-static void
-zr36060_write (struct videocodec *codec,
-	       u16                reg,
-	       u32                val)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-
-	if (post_office_wait(zr)
-	    || post_office_write(zr, 0, 1, reg >> 8)
-	    || post_office_write(zr, 0, 2, reg & 0xff)) {
-		return;
-	}
-
-	post_office_write(zr, 0, 3, val & 0xff);
-}
-
-/* videocodec bus functions ZR36050 */
-static u32
-zr36050_read (struct videocodec *codec,
-	      u16                reg)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-	__u32 data;
-
-	if (post_office_wait(zr)
-	    || post_office_write(zr, 1, 0, reg >> 2)) {	// reg. HIGHBYTES
-		return -1;
-	}
-
-	data = post_office_read(zr, 0, reg & 0x03) & 0xff;	// reg. LOWBYTES + read
-	return data;
-}
-
-static void
-zr36050_write (struct videocodec *codec,
-	       u16                reg,
-	       u32                val)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-
-	if (post_office_wait(zr)
-	    || post_office_write(zr, 1, 0, reg >> 2)) {	// reg. HIGHBYTES
-		return;
-	}
-
-	post_office_write(zr, 0, reg & 0x03, val & 0xff);	// reg. LOWBYTES + wr. data
-}
-
-/* videocodec bus functions ZR36016 */
-static u32
-zr36016_read (struct videocodec *codec,
-	      u16                reg)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-	__u32 data;
-
-	if (post_office_wait(zr)) {
-		return -1;
-	}
-
-	data = post_office_read(zr, 2, reg & 0x03) & 0xff;	// read
-	return data;
-}
-
-/* hack for in zoran_device.c */
-void
-zr36016_write (struct videocodec *codec,
-	       u16                reg,
-	       u32                val)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-
-	if (post_office_wait(zr)) {
-		return;
-	}
-
-	post_office_write(zr, 2, reg & 0x03, val & 0x0ff);	// wr. data
-}
-
-/*
- * Board specific information
- */
-
-static void
-dc10_init (struct zoran *zr)
-{
-	dprintk(3, KERN_DEBUG "%s: %s\n", ZR_DEVNAME(zr), __func__);
-
-	/* Pixel clock selection */
-	GPIO(zr, 4, 0);
-	GPIO(zr, 5, 1);
-	/* Enable the video bus sync signals */
-	GPIO(zr, 7, 0);
-}
-
-static void
-dc10plus_init (struct zoran *zr)
-{
-	dprintk(3, KERN_DEBUG "%s: %s\n", ZR_DEVNAME(zr), __func__);
-}
-
-static void
-buz_init (struct zoran *zr)
-{
-	dprintk(3, KERN_DEBUG "%s: %s\n", ZR_DEVNAME(zr), __func__);
-
-	/* some stuff from Iomega */
-	pci_write_config_dword(zr->pci_dev, 0xfc, 0x90680f15);
-	pci_write_config_dword(zr->pci_dev, 0x0c, 0x00012020);
-	pci_write_config_dword(zr->pci_dev, 0xe8, 0xc0200000);
-}
-
-static void
-lml33_init (struct zoran *zr)
-{
-	dprintk(3, KERN_DEBUG "%s: %s\n", ZR_DEVNAME(zr), __func__);
-
-	GPIO(zr, 2, 1);		// Set Composite input/output
-}
-
-static void
-avs6eyes_init (struct zoran *zr)
-{
-	// AverMedia 6-Eyes original driver by Christer Weinigel
-
-	// Lifted straight from Christer's old driver and
-	// modified slightly by Martin Samuelsson.
-
-	int mux = default_mux; /* 1 = BT866, 7 = VID1 */
-
-	GPIO(zr, 4, 1); /* Bt866 SLEEP on */
-	udelay(2);
-
-	GPIO(zr, 0, 1); /* ZR36060 /RESET on */
-	GPIO(zr, 1, 0); /* ZR36060 /SLEEP on */
-	GPIO(zr, 2, mux & 1);   /* MUX S0 */
-	GPIO(zr, 3, 0); /* /FRAME on */
-	GPIO(zr, 4, 0); /* Bt866 SLEEP off */
-	GPIO(zr, 5, mux & 2);   /* MUX S1 */
-	GPIO(zr, 6, 0); /* ? */
-	GPIO(zr, 7, mux & 4);   /* MUX S2 */
-
-}
-
-static char *
-codecid_to_modulename (u16 codecid)
-{
-	char *name = NULL;
-
-	switch (codecid) {
-	case CODEC_TYPE_ZR36060:
-		name = "zr36060";
-		break;
-	case CODEC_TYPE_ZR36050:
-		name = "zr36050";
-		break;
-	case CODEC_TYPE_ZR36016:
-		name = "zr36016";
-		break;
-	}
-
-	return name;
-}
-
-// struct tvnorm {
-//      u16 Wt, Wa, HStart, HSyncStart, Ht, Ha, VStart;
-// };
-
-static struct tvnorm f50sqpixel = { 944, 768, 83, 880, 625, 576, 16 };
-static struct tvnorm f60sqpixel = { 780, 640, 51, 716, 525, 480, 12 };
-static struct tvnorm f50ccir601 = { 864, 720, 75, 804, 625, 576, 18 };
-static struct tvnorm f60ccir601 = { 858, 720, 57, 788, 525, 480, 16 };
-
-static struct tvnorm f50ccir601_lml33 = { 864, 720, 75+34, 804, 625, 576, 18 };
-static struct tvnorm f60ccir601_lml33 = { 858, 720, 57+34, 788, 525, 480, 16 };
-
-/* The DC10 (57/16/50) uses VActive as HSync, so HStart must be 0 */
-static struct tvnorm f50sqpixel_dc10 = { 944, 768, 0, 880, 625, 576, 0 };
-static struct tvnorm f60sqpixel_dc10 = { 780, 640, 0, 716, 525, 480, 12 };
-
-/* FIXME: I cannot swap U and V in saa7114, so i do one
- * pixel left shift in zoran (75 -> 74)
- * (Maxim Yevtyushkin <max@linuxmedialabs.com>) */
-static struct tvnorm f50ccir601_lm33r10 = { 864, 720, 74+54, 804, 625, 576, 18 };
-static struct tvnorm f60ccir601_lm33r10 = { 858, 720, 56+54, 788, 525, 480, 16 };
-
-/* FIXME: The ks0127 seem incapable of swapping U and V, too, which is why I
- * copy Maxim's left shift hack for the 6 Eyes.
- *
- * Christer's driver used the unshifted norms, though...
- * /Sam  */
-static struct tvnorm f50ccir601_avs6eyes = { 864, 720, 74, 804, 625, 576, 18 };
-static struct tvnorm f60ccir601_avs6eyes = { 858, 720, 56, 788, 525, 480, 16 };
-
-static const unsigned short vpx3220_addrs[] = { 0x43, 0x47, I2C_CLIENT_END };
-static const unsigned short saa7110_addrs[] = { 0x4e, 0x4f, I2C_CLIENT_END };
-static const unsigned short saa7111_addrs[] = { 0x25, 0x24, I2C_CLIENT_END };
-static const unsigned short saa7114_addrs[] = { 0x21, 0x20, I2C_CLIENT_END };
-static const unsigned short adv717x_addrs[] = { 0x6a, 0x6b, 0x2a, 0x2b, I2C_CLIENT_END };
-static const unsigned short ks0127_addrs[] = { 0x6c, 0x6d, I2C_CLIENT_END };
-static const unsigned short saa7185_addrs[] = { 0x44, I2C_CLIENT_END };
-static const unsigned short bt819_addrs[] = { 0x45, I2C_CLIENT_END };
-static const unsigned short bt856_addrs[] = { 0x44, I2C_CLIENT_END };
-static const unsigned short bt866_addrs[] = { 0x44, I2C_CLIENT_END };
-
-static struct card_info zoran_cards[NUM_CARDS] = {
-	{
-		.type = DC10_old,
-		.name = "DC10(old)",
-		.i2c_decoder = "vpx3220a",
-		.addrs_decoder = vpx3220_addrs,
-		.video_codec = CODEC_TYPE_ZR36050,
-		.video_vfe = CODEC_TYPE_ZR36016,
-
-		.inputs = 3,
-		.input = {
-			{ 1, "Composite" },
-			{ 2, "S-Video" },
-			{ 0, "Internal/comp" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-			&f50sqpixel_dc10,
-			&f60sqpixel_dc10,
-			&f50sqpixel_dc10
-		},
-		.jpeg_int = 0,
-		.vsync_int = ZR36057_ISR_GIRQ1,
-		.gpio = { 2, 1, -1, 3, 7, 0, 4, 5 },
-		.gpio_pol = { 0, 0, 0, 1, 0, 0, 0, 0 },
-		.gpcs = { -1, 0 },
-		.vfe_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gws_not_connected = 0,
-		.input_mux = 0,
-		.init = &dc10_init,
-	}, {
-		.type = DC10_new,
-		.name = "DC10(new)",
-		.i2c_decoder = "saa7110",
-		.addrs_decoder = saa7110_addrs,
-		.i2c_encoder = "adv7175",
-		.addrs_encoder = adv717x_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 3,
-		.input = {
-				{ 0, "Composite" },
-				{ 7, "S-Video" },
-				{ 5, "Internal/comp" }
-			},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-				&f50sqpixel,
-				&f60sqpixel,
-				&f50sqpixel},
-		.jpeg_int = ZR36057_ISR_GIRQ0,
-		.vsync_int = ZR36057_ISR_GIRQ1,
-		.gpio = { 3, 0, 6, 1, 2, -1, 4, 5 },
-		.gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gpcs = { -1, 1},
-		.vfe_pol = { 1, 1, 1, 1, 0, 0, 0, 0 },
-		.gws_not_connected = 0,
-		.input_mux = 0,
-		.init = &dc10plus_init,
-	}, {
-		.type = DC10plus,
-		.name = "DC10plus",
-		.i2c_decoder = "saa7110",
-		.addrs_decoder = saa7110_addrs,
-		.i2c_encoder = "adv7175",
-		.addrs_encoder = adv717x_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 3,
-		.input = {
-			{ 0, "Composite" },
-			{ 7, "S-Video" },
-			{ 5, "Internal/comp" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-			&f50sqpixel,
-			&f60sqpixel,
-			&f50sqpixel
-		},
-		.jpeg_int = ZR36057_ISR_GIRQ0,
-		.vsync_int = ZR36057_ISR_GIRQ1,
-		.gpio = { 3, 0, 6, 1, 2, -1, 4, 5 },
-		.gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gpcs = { -1, 1 },
-		.vfe_pol = { 1, 1, 1, 1, 0, 0, 0, 0 },
-		.gws_not_connected = 0,
-		.input_mux = 0,
-		.init = &dc10plus_init,
-	}, {
-		.type = DC30,
-		.name = "DC30",
-		.i2c_decoder = "vpx3220a",
-		.addrs_decoder = vpx3220_addrs,
-		.i2c_encoder = "adv7175",
-		.addrs_encoder = adv717x_addrs,
-		.video_codec = CODEC_TYPE_ZR36050,
-		.video_vfe = CODEC_TYPE_ZR36016,
-
-		.inputs = 3,
-		.input = {
-			{ 1, "Composite" },
-			{ 2, "S-Video" },
-			{ 0, "Internal/comp" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-			&f50sqpixel_dc10,
-			&f60sqpixel_dc10,
-			&f50sqpixel_dc10
-		},
-		.jpeg_int = 0,
-		.vsync_int = ZR36057_ISR_GIRQ1,
-		.gpio = { 2, 1, -1, 3, 7, 0, 4, 5 },
-		.gpio_pol = { 0, 0, 0, 1, 0, 0, 0, 0 },
-		.gpcs = { -1, 0 },
-		.vfe_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gws_not_connected = 0,
-		.input_mux = 0,
-		.init = &dc10_init,
-	}, {
-		.type = DC30plus,
-		.name = "DC30plus",
-		.i2c_decoder = "vpx3220a",
-		.addrs_decoder = vpx3220_addrs,
-		.i2c_encoder = "adv7175",
-		.addrs_encoder = adv717x_addrs,
-		.video_codec = CODEC_TYPE_ZR36050,
-		.video_vfe = CODEC_TYPE_ZR36016,
-
-		.inputs = 3,
-		.input = {
-			{ 1, "Composite" },
-			{ 2, "S-Video" },
-			{ 0, "Internal/comp" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-			&f50sqpixel_dc10,
-			&f60sqpixel_dc10,
-			&f50sqpixel_dc10
-		},
-		.jpeg_int = 0,
-		.vsync_int = ZR36057_ISR_GIRQ1,
-		.gpio = { 2, 1, -1, 3, 7, 0, 4, 5 },
-		.gpio_pol = { 0, 0, 0, 1, 0, 0, 0, 0 },
-		.gpcs = { -1, 0 },
-		.vfe_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gws_not_connected = 0,
-		.input_mux = 0,
-		.init = &dc10_init,
-	}, {
-		.type = LML33,
-		.name = "LML33",
-		.i2c_decoder = "bt819a",
-		.addrs_decoder = bt819_addrs,
-		.i2c_encoder = "bt856",
-		.addrs_encoder = bt856_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 2,
-		.input = {
-			{ 0, "Composite" },
-			{ 7, "S-Video" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL,
-		.tvn = {
-			&f50ccir601_lml33,
-			&f60ccir601_lml33,
-			NULL
-		},
-		.jpeg_int = ZR36057_ISR_GIRQ1,
-		.vsync_int = ZR36057_ISR_GIRQ0,
-		.gpio = { 1, -1, 3, 5, 7, -1, -1, -1 },
-		.gpio_pol = { 0, 0, 0, 0, 1, 0, 0, 0 },
-		.gpcs = { 3, 1 },
-		.vfe_pol = { 1, 1, 0, 0, 0, 1, 0, 0 },
-		.gws_not_connected = 1,
-		.input_mux = 0,
-		.init = &lml33_init,
-	}, {
-		.type = LML33R10,
-		.name = "LML33R10",
-		.i2c_decoder = "saa7114",
-		.addrs_decoder = saa7114_addrs,
-		.i2c_encoder = "adv7170",
-		.addrs_encoder = adv717x_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 2,
-		.input = {
-			{ 0, "Composite" },
-			{ 7, "S-Video" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL,
-		.tvn = {
-			&f50ccir601_lm33r10,
-			&f60ccir601_lm33r10,
-			NULL
-		},
-		.jpeg_int = ZR36057_ISR_GIRQ1,
-		.vsync_int = ZR36057_ISR_GIRQ0,
-		.gpio = { 1, -1, 3, 5, 7, -1, -1, -1 },
-		.gpio_pol = { 0, 0, 0, 0, 1, 0, 0, 0 },
-		.gpcs = { 3, 1 },
-		.vfe_pol = { 1, 1, 0, 0, 0, 1, 0, 0 },
-		.gws_not_connected = 1,
-		.input_mux = 0,
-		.init = &lml33_init,
-	}, {
-		.type = BUZ,
-		.name = "Buz",
-		.i2c_decoder = "saa7111",
-		.addrs_decoder = saa7111_addrs,
-		.i2c_encoder = "saa7185",
-		.addrs_encoder = saa7185_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 2,
-		.input = {
-			{ 3, "Composite" },
-			{ 7, "S-Video" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-			&f50ccir601,
-			&f60ccir601,
-			&f50ccir601
-		},
-		.jpeg_int = ZR36057_ISR_GIRQ1,
-		.vsync_int = ZR36057_ISR_GIRQ0,
-		.gpio = { 1, -1, 3, -1, -1, -1, -1, -1 },
-		.gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gpcs = { 3, 1 },
-		.vfe_pol = { 1, 1, 0, 0, 0, 1, 0, 0 },
-		.gws_not_connected = 1,
-		.input_mux = 0,
-		.init = &buz_init,
-	}, {
-		.type = AVS6EYES,
-		.name = "6-Eyes",
-		/* AverMedia chose not to brand the 6-Eyes. Thus it
-		   can't be autodetected, and requires card=x. */
-		.i2c_decoder = "ks0127",
-		.addrs_decoder = ks0127_addrs,
-		.i2c_encoder = "bt866",
-		.addrs_encoder = bt866_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 10,
-		.input = {
-			{ 0, "Composite 1" },
-			{ 1, "Composite 2" },
-			{ 2, "Composite 3" },
-			{ 4, "Composite 4" },
-			{ 5, "Composite 5" },
-			{ 6, "Composite 6" },
-			{ 8, "S-Video 1" },
-			{ 9, "S-Video 2" },
-			{10, "S-Video 3" },
-			{15, "YCbCr" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL,
-		.tvn = {
-			&f50ccir601_avs6eyes,
-			&f60ccir601_avs6eyes,
-			NULL
-		},
-		.jpeg_int = ZR36057_ISR_GIRQ1,
-		.vsync_int = ZR36057_ISR_GIRQ0,
-		.gpio = { 1, 0, 3, -1, -1, -1, -1, -1 },// Validity unknown /Sam
-		.gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 }, // Validity unknown /Sam
-		.gpcs = { 3, 1 },			// Validity unknown /Sam
-		.vfe_pol = { 1, 0, 0, 0, 0, 1, 0, 0 },  // Validity unknown /Sam
-		.gws_not_connected = 1,
-		.input_mux = 1,
-		.init = &avs6eyes_init,
-	}
-
-};
-
-/*
- * I2C functions
- */
-/* software I2C functions */
-static int
-zoran_i2c_getsda (void *data)
-{
-	struct zoran *zr = (struct zoran *) data;
-
-	return (btread(ZR36057_I2CBR) >> 1) & 1;
-}
-
-static int
-zoran_i2c_getscl (void *data)
-{
-	struct zoran *zr = (struct zoran *) data;
-
-	return btread(ZR36057_I2CBR) & 1;
-}
-
-static void
-zoran_i2c_setsda (void *data,
-		  int   state)
-{
-	struct zoran *zr = (struct zoran *) data;
-
-	if (state)
-		zr->i2cbr |= 2;
-	else
-		zr->i2cbr &= ~2;
-	btwrite(zr->i2cbr, ZR36057_I2CBR);
-}
-
-static void
-zoran_i2c_setscl (void *data,
-		  int   state)
-{
-	struct zoran *zr = (struct zoran *) data;
-
-	if (state)
-		zr->i2cbr |= 1;
-	else
-		zr->i2cbr &= ~1;
-	btwrite(zr->i2cbr, ZR36057_I2CBR);
-}
-
-static const struct i2c_algo_bit_data zoran_i2c_bit_data_template = {
-	.setsda = zoran_i2c_setsda,
-	.setscl = zoran_i2c_setscl,
-	.getsda = zoran_i2c_getsda,
-	.getscl = zoran_i2c_getscl,
-	.udelay = 10,
-	.timeout = 100,
-};
-
-static int
-zoran_register_i2c (struct zoran *zr)
-{
-	zr->i2c_algo = zoran_i2c_bit_data_template;
-	zr->i2c_algo.data = zr;
-	strlcpy(zr->i2c_adapter.name, ZR_DEVNAME(zr),
-		sizeof(zr->i2c_adapter.name));
-	i2c_set_adapdata(&zr->i2c_adapter, &zr->v4l2_dev);
-	zr->i2c_adapter.algo_data = &zr->i2c_algo;
-	zr->i2c_adapter.dev.parent = &zr->pci_dev->dev;
-	return i2c_bit_add_bus(&zr->i2c_adapter);
-}
-
-static void
-zoran_unregister_i2c (struct zoran *zr)
-{
-	i2c_del_adapter(&zr->i2c_adapter);
-}
-
-/* Check a zoran_params struct for correctness, insert default params */
-
-int
-zoran_check_jpg_settings (struct zoran              *zr,
-			  struct zoran_jpg_settings *settings,
-			  int try)
-{
-	int err = 0, err0 = 0;
-
-	dprintk(4,
-		KERN_DEBUG
-		"%s: %s - dec: %d, Hdcm: %d, Vdcm: %d, Tdcm: %d\n",
-		ZR_DEVNAME(zr), __func__, settings->decimation, settings->HorDcm,
-		settings->VerDcm, settings->TmpDcm);
-	dprintk(4,
-		KERN_DEBUG
-		"%s: %s - x: %d, y: %d, w: %d, y: %d\n",
-		ZR_DEVNAME(zr), __func__, settings->img_x, settings->img_y,
-		settings->img_width, settings->img_height);
-	/* Check decimation, set default values for decimation = 1, 2, 4 */
-	switch (settings->decimation) {
-	case 1:
-
-		settings->HorDcm = 1;
-		settings->VerDcm = 1;
-		settings->TmpDcm = 1;
-		settings->field_per_buff = 2;
-		settings->img_x = 0;
-		settings->img_y = 0;
-		settings->img_width = BUZ_MAX_WIDTH;
-		settings->img_height = BUZ_MAX_HEIGHT / 2;
-		break;
-	case 2:
-
-		settings->HorDcm = 2;
-		settings->VerDcm = 1;
-		settings->TmpDcm = 2;
-		settings->field_per_buff = 1;
-		settings->img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
-		settings->img_y = 0;
-		settings->img_width =
-		    (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
-		settings->img_height = BUZ_MAX_HEIGHT / 2;
-		break;
-	case 4:
-
-		if (zr->card.type == DC10_new) {
-			dprintk(1,
-				KERN_DEBUG
-				"%s: %s - HDec by 4 is not supported on the DC10\n",
-				ZR_DEVNAME(zr), __func__);
-			err0++;
-			break;
-		}
-
-		settings->HorDcm = 4;
-		settings->VerDcm = 2;
-		settings->TmpDcm = 2;
-		settings->field_per_buff = 1;
-		settings->img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
-		settings->img_y = 0;
-		settings->img_width =
-		    (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
-		settings->img_height = BUZ_MAX_HEIGHT / 2;
-		break;
-	case 0:
-
-		/* We have to check the data the user has set */
-
-		if (settings->HorDcm != 1 && settings->HorDcm != 2 &&
-		    (zr->card.type == DC10_new || settings->HorDcm != 4)) {
-			settings->HorDcm = clamp(settings->HorDcm, 1, 2);
-			err0++;
-		}
-		if (settings->VerDcm != 1 && settings->VerDcm != 2) {
-			settings->VerDcm = clamp(settings->VerDcm, 1, 2);
-			err0++;
-		}
-		if (settings->TmpDcm != 1 && settings->TmpDcm != 2) {
-			settings->TmpDcm = clamp(settings->TmpDcm, 1, 2);
-			err0++;
-		}
-		if (settings->field_per_buff != 1 &&
-		    settings->field_per_buff != 2) {
-			settings->field_per_buff = clamp(settings->field_per_buff, 1, 2);
-			err0++;
-		}
-		if (settings->img_x < 0) {
-			settings->img_x = 0;
-			err0++;
-		}
-		if (settings->img_y < 0) {
-			settings->img_y = 0;
-			err0++;
-		}
-		if (settings->img_width < 0 || settings->img_width > BUZ_MAX_WIDTH) {
-			settings->img_width = clamp(settings->img_width, 0, (int)BUZ_MAX_WIDTH);
-			err0++;
-		}
-		if (settings->img_height < 0 || settings->img_height > BUZ_MAX_HEIGHT / 2) {
-			settings->img_height = clamp(settings->img_height, 0, BUZ_MAX_HEIGHT / 2);
-			err0++;
-		}
-		if (settings->img_x + settings->img_width > BUZ_MAX_WIDTH) {
-			settings->img_x = BUZ_MAX_WIDTH - settings->img_width;
-			err0++;
-		}
-		if (settings->img_y + settings->img_height > BUZ_MAX_HEIGHT / 2) {
-			settings->img_y = BUZ_MAX_HEIGHT / 2 - settings->img_height;
-			err0++;
-		}
-		if (settings->img_width % (16 * settings->HorDcm) != 0) {
-			settings->img_width -= settings->img_width % (16 * settings->HorDcm);
-			if (settings->img_width == 0)
-				settings->img_width = 16 * settings->HorDcm;
-			err0++;
-		}
-		if (settings->img_height % (8 * settings->VerDcm) != 0) {
-			settings->img_height -= settings->img_height % (8 * settings->VerDcm);
-			if (settings->img_height == 0)
-				settings->img_height = 8 * settings->VerDcm;
-			err0++;
-		}
-
-		if (!try && err0) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - error in params for decimation = 0\n",
-				ZR_DEVNAME(zr), __func__);
-			err++;
-		}
-		break;
-	default:
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - decimation = %d, must be 0, 1, 2 or 4\n",
-			ZR_DEVNAME(zr), __func__, settings->decimation);
-		err++;
-		break;
-	}
-
-	if (settings->jpg_comp.quality > 100)
-		settings->jpg_comp.quality = 100;
-	if (settings->jpg_comp.quality < 5)
-		settings->jpg_comp.quality = 5;
-	if (settings->jpg_comp.APPn < 0)
-		settings->jpg_comp.APPn = 0;
-	if (settings->jpg_comp.APPn > 15)
-		settings->jpg_comp.APPn = 15;
-	if (settings->jpg_comp.APP_len < 0)
-		settings->jpg_comp.APP_len = 0;
-	if (settings->jpg_comp.APP_len > 60)
-		settings->jpg_comp.APP_len = 60;
-	if (settings->jpg_comp.COM_len < 0)
-		settings->jpg_comp.COM_len = 0;
-	if (settings->jpg_comp.COM_len > 60)
-		settings->jpg_comp.COM_len = 60;
-	if (err)
-		return -EINVAL;
-	return 0;
-}
-
-void
-zoran_open_init_params (struct zoran *zr)
-{
-	int i;
-
-	/* User must explicitly set a window */
-	zr->overlay_settings.is_set = 0;
-	zr->overlay_mask = NULL;
-	zr->overlay_active = ZORAN_FREE;
-
-	zr->v4l_memgrab_active = 0;
-	zr->v4l_overlay_active = 0;
-	zr->v4l_grab_frame = NO_GRAB_ACTIVE;
-	zr->v4l_grab_seq = 0;
-	zr->v4l_settings.width = 192;
-	zr->v4l_settings.height = 144;
-	zr->v4l_settings.format = &zoran_formats[7];	/* YUY2 - YUV-4:2:2 packed */
-	zr->v4l_settings.bytesperline =
-	    zr->v4l_settings.width *
-	    ((zr->v4l_settings.format->depth + 7) / 8);
-
-	/* DMA ring stuff for V4L */
-	zr->v4l_pend_tail = 0;
-	zr->v4l_pend_head = 0;
-	zr->v4l_sync_tail = 0;
-	zr->v4l_buffers.active = ZORAN_FREE;
-	for (i = 0; i < VIDEO_MAX_FRAME; i++) {
-		zr->v4l_buffers.buffer[i].state = BUZ_STATE_USER;	/* nothing going on */
-	}
-	zr->v4l_buffers.allocated = 0;
-
-	for (i = 0; i < BUZ_MAX_FRAME; i++) {
-		zr->jpg_buffers.buffer[i].state = BUZ_STATE_USER;	/* nothing going on */
-	}
-	zr->jpg_buffers.active = ZORAN_FREE;
-	zr->jpg_buffers.allocated = 0;
-	/* Set necessary params and call zoran_check_jpg_settings to set the defaults */
-	zr->jpg_settings.decimation = 1;
-	zr->jpg_settings.jpg_comp.quality = 50;	/* default compression factor 8 */
-	if (zr->card.type != BUZ)
-		zr->jpg_settings.odd_even = 1;
-	else
-		zr->jpg_settings.odd_even = 0;
-	zr->jpg_settings.jpg_comp.APPn = 0;
-	zr->jpg_settings.jpg_comp.APP_len = 0;	/* No APPn marker */
-	memset(zr->jpg_settings.jpg_comp.APP_data, 0,
-	       sizeof(zr->jpg_settings.jpg_comp.APP_data));
-	zr->jpg_settings.jpg_comp.COM_len = 0;	/* No COM marker */
-	memset(zr->jpg_settings.jpg_comp.COM_data, 0,
-	       sizeof(zr->jpg_settings.jpg_comp.COM_data));
-	zr->jpg_settings.jpg_comp.jpeg_markers =
-	    V4L2_JPEG_MARKER_DHT | V4L2_JPEG_MARKER_DQT;
-	i = zoran_check_jpg_settings(zr, &zr->jpg_settings, 0);
-	if (i)
-		dprintk(1, KERN_ERR "%s: %s internal error\n",
-			ZR_DEVNAME(zr), __func__);
-
-	clear_interrupt_counters(zr);
-	zr->testing = 0;
-}
-
-static void test_interrupts (struct zoran *zr)
-{
-	DEFINE_WAIT(wait);
-	int timeout, icr;
-
-	clear_interrupt_counters(zr);
-
-	zr->testing = 1;
-	icr = btread(ZR36057_ICR);
-	btwrite(0x78000000 | ZR36057_ICR_IntPinEn, ZR36057_ICR);
-	prepare_to_wait(&zr->test_q, &wait, TASK_INTERRUPTIBLE);
-	timeout = schedule_timeout(HZ);
-	finish_wait(&zr->test_q, &wait);
-	btwrite(0, ZR36057_ICR);
-	btwrite(0x78000000, ZR36057_ISR);
-	zr->testing = 0;
-	dprintk(5, KERN_INFO "%s: Testing interrupts...\n", ZR_DEVNAME(zr));
-	if (timeout) {
-		dprintk(1, ": time spent: %d\n", 1 * HZ - timeout);
-	}
-	if (zr36067_debug > 1)
-		print_interrupts(zr);
-	btwrite(icr, ZR36057_ICR);
-}
-
-static int zr36057_init (struct zoran *zr)
-{
-	int j, err;
-
-	dprintk(1,
-		KERN_INFO
-		"%s: %s - initializing card[%d], zr=%p\n",
-		ZR_DEVNAME(zr), __func__, zr->id, zr);
-
-	/* default setup of all parameters which will persist between opens */
-	zr->user = 0;
-
-	init_waitqueue_head(&zr->v4l_capq);
-	init_waitqueue_head(&zr->jpg_capq);
-	init_waitqueue_head(&zr->test_q);
-	zr->jpg_buffers.allocated = 0;
-	zr->v4l_buffers.allocated = 0;
-
-	zr->vbuf_base = (void *) vidmem;
-	zr->vbuf_width = 0;
-	zr->vbuf_height = 0;
-	zr->vbuf_depth = 0;
-	zr->vbuf_bytesperline = 0;
-
-	/* Avoid nonsense settings from user for default input/norm */
-	if (default_norm < 0 || default_norm > 2)
-		default_norm = 0;
-	if (default_norm == 0) {
-		zr->norm = V4L2_STD_PAL;
-		zr->timing = zr->card.tvn[0];
-	} else if (default_norm == 1) {
-		zr->norm = V4L2_STD_NTSC;
-		zr->timing = zr->card.tvn[1];
-	} else {
-		zr->norm = V4L2_STD_SECAM;
-		zr->timing = zr->card.tvn[2];
-	}
-	if (zr->timing == NULL) {
-		dprintk(1,
-			KERN_WARNING
-			"%s: %s - default TV standard not supported by hardware. PAL will be used.\n",
-			ZR_DEVNAME(zr), __func__);
-		zr->norm = V4L2_STD_PAL;
-		zr->timing = zr->card.tvn[0];
-	}
-
-	if (default_input > zr->card.inputs-1) {
-		dprintk(1,
-			KERN_WARNING
-			"%s: default_input value %d out of range (0-%d)\n",
-			ZR_DEVNAME(zr), default_input, zr->card.inputs-1);
-		default_input = 0;
-	}
-	zr->input = default_input;
-
-	/* default setup (will be repeated at every open) */
-	zoran_open_init_params(zr);
-
-	/* allocate memory *before* doing anything to the hardware
-	 * in case allocation fails */
-	zr->stat_com = kzalloc(BUZ_NUM_STAT_COM * 4, GFP_KERNEL);
-	zr->video_dev = video_device_alloc();
-	if (!zr->stat_com || !zr->video_dev) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - kmalloc (STAT_COM) failed\n",
-			ZR_DEVNAME(zr), __func__);
-		err = -ENOMEM;
-		goto exit_free;
-	}
-	for (j = 0; j < BUZ_NUM_STAT_COM; j++) {
-		zr->stat_com[j] = cpu_to_le32(1); /* mark as unavailable to zr36057 */
-	}
-
-	/*
-	 *   Now add the template and register the device unit.
-	 */
-	*zr->video_dev = zoran_template;
-	zr->video_dev->v4l2_dev = &zr->v4l2_dev;
-	zr->video_dev->lock = &zr->lock;
-	strcpy(zr->video_dev->name, ZR_DEVNAME(zr));
-	/* It's not a mem2mem device, but you can both capture and output from
-	   one and the same device. This should really be split up into two
-	   device nodes, but that's a job for another day. */
-	zr->video_dev->vfl_dir = VFL_DIR_M2M;
-	err = video_register_device(zr->video_dev, VFL_TYPE_GRABBER, video_nr[zr->id]);
-	if (err < 0)
-		goto exit_free;
-	video_set_drvdata(zr->video_dev, zr);
-
-	zoran_init_hardware(zr);
-	if (zr36067_debug > 2)
-		detect_guest_activity(zr);
-	test_interrupts(zr);
-	if (!pass_through) {
-		decoder_call(zr, video, s_stream, 0);
-		encoder_call(zr, video, s_routing, 2, 0, 0);
-	}
-
-	zr->zoran_proc = NULL;
-	zr->initialized = 1;
-	return 0;
-
-exit_free:
-	kfree(zr->stat_com);
-	kfree(zr->video_dev);
-	return err;
-}
-
-static void zoran_remove(struct pci_dev *pdev)
-{
-	struct v4l2_device *v4l2_dev = dev_get_drvdata(&pdev->dev);
-	struct zoran *zr = to_zoran(v4l2_dev);
-
-	if (!zr->initialized)
-		goto exit_free;
-
-	/* unregister videocodec bus */
-	if (zr->codec) {
-		struct videocodec_master *master = zr->codec->master_data;
-
-		videocodec_detach(zr->codec);
-		kfree(master);
-	}
-	if (zr->vfe) {
-		struct videocodec_master *master = zr->vfe->master_data;
-
-		videocodec_detach(zr->vfe);
-		kfree(master);
-	}
-
-	/* unregister i2c bus */
-	zoran_unregister_i2c(zr);
-	/* disable PCI bus-mastering */
-	zoran_set_pci_master(zr, 0);
-	/* put chip into reset */
-	btwrite(0, ZR36057_SPGPPCR);
-	free_irq(zr->pci_dev->irq, zr);
-	/* unmap and free memory */
-	kfree(zr->stat_com);
-	zoran_proc_cleanup(zr);
-	iounmap(zr->zr36057_mem);
-	pci_disable_device(zr->pci_dev);
-	video_unregister_device(zr->video_dev);
-exit_free:
-	v4l2_ctrl_handler_free(&zr->hdl);
-	v4l2_device_unregister(&zr->v4l2_dev);
-	kfree(zr);
-}
-
-void
-zoran_vdev_release (struct video_device *vdev)
-{
-	kfree(vdev);
-}
-
-static struct videocodec_master *zoran_setup_videocodec(struct zoran *zr,
-							int type)
-{
-	struct videocodec_master *m = NULL;
-
-	m = kmalloc(sizeof(struct videocodec_master), GFP_KERNEL);
-	if (!m) {
-		dprintk(1, KERN_ERR "%s: %s - no memory\n",
-			ZR_DEVNAME(zr), __func__);
-		return m;
-	}
-
-	/* magic and type are unused for master struct. Makes sense only at
-	   codec structs.
-	   In the past, .type were initialized to the old V4L1 .hardware
-	   value, as VID_HARDWARE_ZR36067
-	 */
-	m->magic = 0L;
-	m->type = 0;
-
-	m->flags = CODEC_FLAG_ENCODER | CODEC_FLAG_DECODER;
-	strlcpy(m->name, ZR_DEVNAME(zr), sizeof(m->name));
-	m->data = zr;
-
-	switch (type)
-	{
-	case CODEC_TYPE_ZR36060:
-		m->readreg = zr36060_read;
-		m->writereg = zr36060_write;
-		m->flags |= CODEC_FLAG_JPEG | CODEC_FLAG_VFE;
-		break;
-	case CODEC_TYPE_ZR36050:
-		m->readreg = zr36050_read;
-		m->writereg = zr36050_write;
-		m->flags |= CODEC_FLAG_JPEG;
-		break;
-	case CODEC_TYPE_ZR36016:
-		m->readreg = zr36016_read;
-		m->writereg = zr36016_write;
-		m->flags |= CODEC_FLAG_VFE;
-		break;
-	}
-
-	return m;
-}
-
-static void zoran_subdev_notify(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
-{
-	struct zoran *zr = to_zoran(sd->v4l2_dev);
-
-	/* Bt819 needs to reset its FIFO buffer using #FRST pin and
-	   LML33 card uses GPIO(7) for that. */
-	if (cmd == BT819_FIFO_RESET_LOW)
-		GPIO(zr, 7, 0);
-	else if (cmd == BT819_FIFO_RESET_HIGH)
-		GPIO(zr, 7, 1);
-}
-
-/*
- *   Scan for a Buz card (actually for the PCI controller ZR36057),
- *   request the irq and map the io memory
- */
-static int zoran_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-	unsigned char latency, need_latency;
-	struct zoran *zr;
-	int result;
-	struct videocodec_master *master_vfe = NULL;
-	struct videocodec_master *master_codec = NULL;
-	int card_num;
-	char *codec_name, *vfe_name;
-	unsigned int nr;
-
-
-	nr = zoran_num++;
-	if (nr >= BUZ_MAX) {
-		dprintk(1, KERN_ERR "%s: driver limited to %d card(s) maximum\n",
-			ZORAN_NAME, BUZ_MAX);
-		return -ENOENT;
-	}
-
-	zr = kzalloc(sizeof(struct zoran), GFP_KERNEL);
-	if (!zr) {
-		dprintk(1, KERN_ERR "%s: %s - kzalloc failed\n",
-			ZORAN_NAME, __func__);
-		return -ENOMEM;
-	}
-	zr->v4l2_dev.notify = zoran_subdev_notify;
-	if (v4l2_device_register(&pdev->dev, &zr->v4l2_dev))
-		goto zr_free_mem;
-	zr->pci_dev = pdev;
-	zr->id = nr;
-	snprintf(ZR_DEVNAME(zr), sizeof(ZR_DEVNAME(zr)), "MJPEG[%u]", zr->id);
-	if (v4l2_ctrl_handler_init(&zr->hdl, 10))
-		goto zr_unreg;
-	zr->v4l2_dev.ctrl_handler = &zr->hdl;
-	spin_lock_init(&zr->spinlock);
-	mutex_init(&zr->lock);
-	if (pci_enable_device(pdev))
-		goto zr_unreg;
-	zr->revision = zr->pci_dev->revision;
-
-	dprintk(1,
-		KERN_INFO
-		"%s: Zoran ZR360%c7 (rev %d), irq: %d, memory: 0x%08llx\n",
-		ZR_DEVNAME(zr), zr->revision < 2 ? '5' : '6', zr->revision,
-		zr->pci_dev->irq, (uint64_t)pci_resource_start(zr->pci_dev, 0));
-	if (zr->revision >= 2) {
-		dprintk(1,
-			KERN_INFO
-			"%s: Subsystem vendor=0x%04x id=0x%04x\n",
-			ZR_DEVNAME(zr), zr->pci_dev->subsystem_vendor,
-			zr->pci_dev->subsystem_device);
-	}
-
-	/* Use auto-detected card type? */
-	if (card[nr] == -1) {
-		if (zr->revision < 2) {
-			dprintk(1,
-				KERN_ERR
-				"%s: No card type specified, please use the card=X module parameter\n",
-				ZR_DEVNAME(zr));
-			dprintk(1,
-				KERN_ERR
-				"%s: It is not possible to auto-detect ZR36057 based cards\n",
-				ZR_DEVNAME(zr));
-			goto zr_unreg;
-		}
-
-		card_num = ent->driver_data;
-		if (card_num >= NUM_CARDS) {
-			dprintk(1,
-				KERN_ERR
-				"%s: Unknown card, try specifying card=X module parameter\n",
-				ZR_DEVNAME(zr));
-			goto zr_unreg;
-		}
-		dprintk(3,
-			KERN_DEBUG
-			"%s: %s() - card %s detected\n",
-			ZR_DEVNAME(zr), __func__, zoran_cards[card_num].name);
-	} else {
-		card_num = card[nr];
-		if (card_num >= NUM_CARDS || card_num < 0) {
-			dprintk(1,
-				KERN_ERR
-				"%s: User specified card type %d out of range (0 .. %d)\n",
-				ZR_DEVNAME(zr), card_num, NUM_CARDS - 1);
-			goto zr_unreg;
-		}
-	}
-
-	/* even though we make this a non pointer and thus
-	 * theoretically allow for making changes to this struct
-	 * on a per-individual card basis at runtime, this is
-	 * strongly discouraged. This structure is intended to
-	 * keep general card information, no settings or anything */
-	zr->card = zoran_cards[card_num];
-	snprintf(ZR_DEVNAME(zr), sizeof(ZR_DEVNAME(zr)),
-		 "%s[%u]", zr->card.name, zr->id);
-
-	zr->zr36057_mem = pci_ioremap_bar(zr->pci_dev, 0);
-	if (!zr->zr36057_mem) {
-		dprintk(1, KERN_ERR "%s: %s() - ioremap failed\n",
-			ZR_DEVNAME(zr), __func__);
-		goto zr_unreg;
-	}
-
-	result = request_irq(zr->pci_dev->irq, zoran_irq,
-			     IRQF_SHARED, ZR_DEVNAME(zr), zr);
-	if (result < 0) {
-		if (result == -EINVAL) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - bad irq number or handler\n",
-				ZR_DEVNAME(zr), __func__);
-		} else if (result == -EBUSY) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - IRQ %d busy, change your PnP config in BIOS\n",
-				ZR_DEVNAME(zr), __func__, zr->pci_dev->irq);
-		} else {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - can't assign irq, error code %d\n",
-				ZR_DEVNAME(zr), __func__, result);
-		}
-		goto zr_unmap;
-	}
-
-	/* set PCI latency timer */
-	pci_read_config_byte(zr->pci_dev, PCI_LATENCY_TIMER,
-			     &latency);
-	need_latency = zr->revision > 1 ? 32 : 48;
-	if (latency != need_latency) {
-		dprintk(2, KERN_INFO "%s: Changing PCI latency from %d to %d\n",
-			ZR_DEVNAME(zr), latency, need_latency);
-		pci_write_config_byte(zr->pci_dev, PCI_LATENCY_TIMER,
-				      need_latency);
-	}
-
-	zr36057_restart(zr);
-	/* i2c */
-	dprintk(2, KERN_INFO "%s: Initializing i2c bus...\n",
-		ZR_DEVNAME(zr));
-
-	if (zoran_register_i2c(zr) < 0) {
-		dprintk(1, KERN_ERR "%s: %s - can't initialize i2c bus\n",
-			ZR_DEVNAME(zr), __func__);
-		goto zr_free_irq;
-	}
-
-	zr->decoder = v4l2_i2c_new_subdev(&zr->v4l2_dev,
-		&zr->i2c_adapter, zr->card.i2c_decoder,
-		0, zr->card.addrs_decoder);
-
-	if (zr->card.i2c_encoder)
-		zr->encoder = v4l2_i2c_new_subdev(&zr->v4l2_dev,
-			&zr->i2c_adapter, zr->card.i2c_encoder,
-			0, zr->card.addrs_encoder);
-
-	dprintk(2,
-		KERN_INFO "%s: Initializing videocodec bus...\n",
-		ZR_DEVNAME(zr));
-
-	if (zr->card.video_codec) {
-		codec_name = codecid_to_modulename(zr->card.video_codec);
-		if (codec_name) {
-			result = request_module(codec_name);
-			if (result) {
-				dprintk(1,
-					KERN_ERR
-					"%s: failed to load modules %s: %d\n",
-					ZR_DEVNAME(zr), codec_name, result);
-			}
-		}
-	}
-	if (zr->card.video_vfe) {
-		vfe_name = codecid_to_modulename(zr->card.video_vfe);
-		if (vfe_name) {
-			result = request_module(vfe_name);
-			if (result < 0) {
-				dprintk(1,
-					KERN_ERR
-					"%s: failed to load modules %s: %d\n",
-					ZR_DEVNAME(zr), vfe_name, result);
-			}
-		}
-	}
-
-	/* reset JPEG codec */
-	jpeg_codec_sleep(zr, 1);
-	jpeg_codec_reset(zr);
-	/* video bus enabled */
-	/* display codec revision */
-	if (zr->card.video_codec != 0) {
-		master_codec = zoran_setup_videocodec(zr, zr->card.video_codec);
-		if (!master_codec)
-			goto zr_unreg_i2c;
-		zr->codec = videocodec_attach(master_codec);
-		if (!zr->codec) {
-			dprintk(1, KERN_ERR "%s: %s - no codec found\n",
-				ZR_DEVNAME(zr), __func__);
-			goto zr_free_codec;
-		}
-		if (zr->codec->type != zr->card.video_codec) {
-			dprintk(1, KERN_ERR "%s: %s - wrong codec\n",
-				ZR_DEVNAME(zr), __func__);
-			goto zr_detach_codec;
-		}
-	}
-	if (zr->card.video_vfe != 0) {
-		master_vfe = zoran_setup_videocodec(zr, zr->card.video_vfe);
-		if (!master_vfe)
-			goto zr_detach_codec;
-		zr->vfe = videocodec_attach(master_vfe);
-		if (!zr->vfe) {
-			dprintk(1, KERN_ERR "%s: %s - no VFE found\n",
-				ZR_DEVNAME(zr), __func__);
-			goto zr_free_vfe;
-		}
-		if (zr->vfe->type != zr->card.video_vfe) {
-			dprintk(1, KERN_ERR "%s: %s = wrong VFE\n",
-				ZR_DEVNAME(zr), __func__);
-			goto zr_detach_vfe;
-		}
-	}
-
-	/* take care of Natoma chipset and a revision 1 zr36057 */
-	if ((pci_pci_problems & PCIPCI_NATOMA) && zr->revision <= 1) {
-		zr->jpg_buffers.need_contiguous = 1;
-		dprintk(1, KERN_INFO
-			"%s: ZR36057/Natoma bug, max. buffer size is 128K\n",
-			ZR_DEVNAME(zr));
-	}
-
-	if (zr36057_init(zr) < 0)
-		goto zr_detach_vfe;
-
-	zoran_proc_init(zr);
-
-	return 0;
-
-zr_detach_vfe:
-	videocodec_detach(zr->vfe);
-zr_free_vfe:
-	kfree(master_vfe);
-zr_detach_codec:
-	videocodec_detach(zr->codec);
-zr_free_codec:
-	kfree(master_codec);
-zr_unreg_i2c:
-	zoran_unregister_i2c(zr);
-zr_free_irq:
-	btwrite(0, ZR36057_SPGPPCR);
-	free_irq(zr->pci_dev->irq, zr);
-zr_unmap:
-	iounmap(zr->zr36057_mem);
-zr_unreg:
-	v4l2_ctrl_handler_free(&zr->hdl);
-	v4l2_device_unregister(&zr->v4l2_dev);
-zr_free_mem:
-	kfree(zr);
-
-	return -ENODEV;
-}
-
-static struct pci_driver zoran_driver = {
-	.name = "zr36067",
-	.id_table = zr36067_pci_tbl,
-	.probe = zoran_probe,
-	.remove = zoran_remove,
-};
-
-static int __init zoran_init(void)
-{
-	int res;
-
-	printk(KERN_INFO "Zoran MJPEG board driver version %s\n",
-	       ZORAN_VERSION);
-
-	/* check the parameters we have been given, adjust if necessary */
-	if (v4l_nbufs < 2)
-		v4l_nbufs = 2;
-	if (v4l_nbufs > VIDEO_MAX_FRAME)
-		v4l_nbufs = VIDEO_MAX_FRAME;
-	/* The user specfies the in KB, we want them in byte
-	 * (and page aligned) */
-	v4l_bufsize = PAGE_ALIGN(v4l_bufsize * 1024);
-	if (v4l_bufsize < 32768)
-		v4l_bufsize = 32768;
-	/* 2 MB is arbitrary but sufficient for the maximum possible images */
-	if (v4l_bufsize > 2048 * 1024)
-		v4l_bufsize = 2048 * 1024;
-	if (jpg_nbufs < 4)
-		jpg_nbufs = 4;
-	if (jpg_nbufs > BUZ_MAX_FRAME)
-		jpg_nbufs = BUZ_MAX_FRAME;
-	jpg_bufsize = PAGE_ALIGN(jpg_bufsize * 1024);
-	if (jpg_bufsize < 8192)
-		jpg_bufsize = 8192;
-	if (jpg_bufsize > (512 * 1024))
-		jpg_bufsize = 512 * 1024;
-	/* Use parameter for vidmem or try to find a video card */
-	if (vidmem) {
-		dprintk(1,
-			KERN_INFO
-			"%s: Using supplied video memory base address @ 0x%lx\n",
-			ZORAN_NAME, vidmem);
-	}
-
-	/* some mainboards might not do PCI-PCI data transfer well */
-	if (pci_pci_problems & (PCIPCI_FAIL|PCIAGP_FAIL|PCIPCI_ALIMAGIK)) {
-		dprintk(1,
-			KERN_WARNING
-			"%s: chipset does not support reliable PCI-PCI DMA\n",
-			ZORAN_NAME);
-	}
-
-	res = pci_register_driver(&zoran_driver);
-	if (res) {
-		dprintk(1,
-			KERN_ERR
-			"%s: Unable to register ZR36057 driver\n",
-			ZORAN_NAME);
-		return res;
-	}
-
-	return 0;
-}
-
-static void __exit zoran_exit(void)
-{
-	pci_unregister_driver(&zoran_driver);
-}
-
-module_init(zoran_init);
-module_exit(zoran_exit);
diff --git a/drivers/media/pci/zoran/zoran_card.h b/drivers/media/pci/zoran/zoran_card.h
deleted file mode 100644
index 0cdb7d34926d..000000000000
--- a/drivers/media/pci/zoran/zoran_card.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles card-specific data and detection
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#ifndef __ZORAN_CARD_H__
-#define __ZORAN_CARD_H__
-
-extern int zr36067_debug;
-
-#define dprintk(num, format, args...) \
-	do { \
-		if (zr36067_debug >= num) \
-			printk(format, ##args); \
-	} while (0)
-
-/* Anybody who uses more than four? */
-#define BUZ_MAX 4
-
-extern const struct video_device zoran_template;
-
-extern int zoran_check_jpg_settings(struct zoran *zr,
-				    struct zoran_jpg_settings *settings,
-				    int try);
-extern void zoran_open_init_params(struct zoran *zr);
-extern void zoran_vdev_release(struct video_device *vdev);
-
-void zr36016_write(struct videocodec *codec, u16 reg, u32 val);
-
-#endif				/* __ZORAN_CARD_H__ */
diff --git a/drivers/media/pci/zoran/zoran_device.c b/drivers/media/pci/zoran/zoran_device.c
deleted file mode 100644
index 40adceebca7e..000000000000
--- a/drivers/media/pci/zoran/zoran_device.c
+++ /dev/null
@@ -1,1619 +0,0 @@
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles device access (PCI/I2C/codec/...)
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/vmalloc.h>
-#include <linux/ktime.h>
-#include <linux/sched/signal.h>
-
-#include <linux/interrupt.h>
-#include <linux/proc_fs.h>
-#include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
-#include <linux/videodev2.h>
-#include <media/v4l2-common.h>
-#include <linux/spinlock.h>
-#include <linux/sem.h>
-
-#include <linux/pci.h>
-#include <linux/delay.h>
-#include <linux/wait.h>
-
-#include <asm/byteorder.h>
-#include <asm/io.h>
-
-#include "videocodec.h"
-#include "zoran.h"
-#include "zoran_device.h"
-#include "zoran_card.h"
-
-#define IRQ_MASK ( ZR36057_ISR_GIRQ0 | \
-		   ZR36057_ISR_GIRQ1 | \
-		   ZR36057_ISR_JPEGRepIRQ )
-
-static bool lml33dpath;		/* default = 0
-				 * 1 will use digital path in capture
-				 * mode instead of analog. It can be
-				 * used for picture adjustments using
-				 * tool like xawtv while watching image
-				 * on TV monitor connected to the output.
-				 * However, due to absence of 75 Ohm
-				 * load on Bt819 input, there will be
-				 * some image imperfections */
-
-module_param(lml33dpath, bool, 0644);
-MODULE_PARM_DESC(lml33dpath,
-		 "Use digital path capture mode (on LML33 cards)");
-
-static void
-zr36057_init_vfe (struct zoran *zr);
-
-/*
- * General Purpose I/O and Guest bus access
- */
-
-/*
- * This is a bit tricky. When a board lacks a GPIO function, the corresponding
- * GPIO bit number in the card_info structure is set to 0.
- */
-
-void
-GPIO (struct zoran *zr,
-      int           bit,
-      unsigned int  value)
-{
-	u32 reg;
-	u32 mask;
-
-	/* Make sure the bit number is legal
-	 * A bit number of -1 (lacking) gives a mask of 0,
-	 * making it harmless */
-	mask = (1 << (24 + bit)) & 0xff000000;
-	reg = btread(ZR36057_GPPGCR1) & ~mask;
-	if (value) {
-		reg |= mask;
-	}
-	btwrite(reg, ZR36057_GPPGCR1);
-	udelay(1);
-}
-
-/*
- * Wait til post office is no longer busy
- */
-
-int
-post_office_wait (struct zoran *zr)
-{
-	u32 por;
-
-//      while (((por = btread(ZR36057_POR)) & (ZR36057_POR_POPen | ZR36057_POR_POTime)) == ZR36057_POR_POPen) {
-	while ((por = btread(ZR36057_POR)) & ZR36057_POR_POPen) {
-		/* wait for something to happen */
-	}
-	if ((por & ZR36057_POR_POTime) && !zr->card.gws_not_connected) {
-		/* In LML33/BUZ \GWS line is not connected, so it has always timeout set */
-		dprintk(1, KERN_INFO "%s: pop timeout %08x\n", ZR_DEVNAME(zr),
-			por);
-		return -1;
-	}
-
-	return 0;
-}
-
-int
-post_office_write (struct zoran *zr,
-		   unsigned int  guest,
-		   unsigned int  reg,
-		   unsigned int  value)
-{
-	u32 por;
-
-	por =
-	    ZR36057_POR_PODir | ZR36057_POR_POTime | ((guest & 7) << 20) |
-	    ((reg & 7) << 16) | (value & 0xFF);
-	btwrite(por, ZR36057_POR);
-
-	return post_office_wait(zr);
-}
-
-int
-post_office_read (struct zoran *zr,
-		  unsigned int  guest,
-		  unsigned int  reg)
-{
-	u32 por;
-
-	por = ZR36057_POR_POTime | ((guest & 7) << 20) | ((reg & 7) << 16);
-	btwrite(por, ZR36057_POR);
-	if (post_office_wait(zr) < 0) {
-		return -1;
-	}
-
-	return btread(ZR36057_POR) & 0xFF;
-}
-
-/*
- * detect guests
- */
-
-static void
-dump_guests (struct zoran *zr)
-{
-	if (zr36067_debug > 2) {
-		int i, guest[8];
-
-		for (i = 1; i < 8; i++) {	// Don't read jpeg codec here
-			guest[i] = post_office_read(zr, i, 0);
-		}
-
-		printk(KERN_INFO "%s: Guests: %*ph\n",
-		       ZR_DEVNAME(zr), 8, guest);
-	}
-}
-
-void
-detect_guest_activity (struct zoran *zr)
-{
-	int timeout, i, j, res, guest[8], guest0[8], change[8][3];
-	ktime_t t0, t1;
-
-	dump_guests(zr);
-	printk(KERN_INFO "%s: Detecting guests activity, please wait...\n",
-	       ZR_DEVNAME(zr));
-	for (i = 1; i < 8; i++) {	// Don't read jpeg codec here
-		guest0[i] = guest[i] = post_office_read(zr, i, 0);
-	}
-
-	timeout = 0;
-	j = 0;
-	t0 = ktime_get();
-	while (timeout < 10000) {
-		udelay(10);
-		timeout++;
-		for (i = 1; (i < 8) && (j < 8); i++) {
-			res = post_office_read(zr, i, 0);
-			if (res != guest[i]) {
-				t1 = ktime_get();
-				change[j][0] = ktime_to_us(ktime_sub(t1, t0));
-				t0 = t1;
-				change[j][1] = i;
-				change[j][2] = res;
-				j++;
-				guest[i] = res;
-			}
-		}
-		if (j >= 8)
-			break;
-	}
-
-	printk(KERN_INFO "%s: Guests: %*ph\n", ZR_DEVNAME(zr), 8, guest0);
-
-	if (j == 0) {
-		printk(KERN_INFO "%s: No activity detected.\n", ZR_DEVNAME(zr));
-		return;
-	}
-	for (i = 0; i < j; i++) {
-		printk(KERN_INFO "%s: %6d: %d => 0x%02x\n", ZR_DEVNAME(zr),
-		       change[i][0], change[i][1], change[i][2]);
-	}
-}
-
-/*
- * JPEG Codec access
- */
-
-void
-jpeg_codec_sleep (struct zoran *zr,
-		  int           sleep)
-{
-	GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_SLEEP], !sleep);
-	if (!sleep) {
-		dprintk(3,
-			KERN_DEBUG
-			"%s: jpeg_codec_sleep() - wake GPIO=0x%08x\n",
-			ZR_DEVNAME(zr), btread(ZR36057_GPPGCR1));
-		udelay(500);
-	} else {
-		dprintk(3,
-			KERN_DEBUG
-			"%s: jpeg_codec_sleep() - sleep GPIO=0x%08x\n",
-			ZR_DEVNAME(zr), btread(ZR36057_GPPGCR1));
-		udelay(2);
-	}
-}
-
-int
-jpeg_codec_reset (struct zoran *zr)
-{
-	/* Take the codec out of sleep */
-	jpeg_codec_sleep(zr, 0);
-
-	if (zr->card.gpcs[GPCS_JPEG_RESET] != 0xff) {
-		post_office_write(zr, zr->card.gpcs[GPCS_JPEG_RESET], 0,
-				  0);
-		udelay(2);
-	} else {
-		GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_RESET], 0);
-		udelay(2);
-		GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_RESET], 1);
-		udelay(2);
-	}
-
-	return 0;
-}
-
-/*
- *   Set the registers for the size we have specified. Don't bother
- *   trying to understand this without the ZR36057 manual in front of
- *   you [AC].
- *
- *   PS: The manual is free for download in .pdf format from
- *   www.zoran.com - nicely done those folks.
- */
-
-static void
-zr36057_adjust_vfe (struct zoran          *zr,
-		    enum zoran_codec_mode  mode)
-{
-	u32 reg;
-
-	switch (mode) {
-	case BUZ_MODE_MOTION_DECOMPRESS:
-		btand(~ZR36057_VFESPFR_ExtFl, ZR36057_VFESPFR);
-		reg = btread(ZR36057_VFEHCR);
-		if ((reg & (1 << 10)) && zr->card.type != LML33R10) {
-			reg += ((1 << 10) | 1);
-		}
-		btwrite(reg, ZR36057_VFEHCR);
-		break;
-	case BUZ_MODE_MOTION_COMPRESS:
-	case BUZ_MODE_IDLE:
-	default:
-		if ((zr->norm & V4L2_STD_NTSC) ||
-		    (zr->card.type == LML33R10 &&
-		     (zr->norm & V4L2_STD_PAL)))
-			btand(~ZR36057_VFESPFR_ExtFl, ZR36057_VFESPFR);
-		else
-			btor(ZR36057_VFESPFR_ExtFl, ZR36057_VFESPFR);
-		reg = btread(ZR36057_VFEHCR);
-		if (!(reg & (1 << 10)) && zr->card.type != LML33R10) {
-			reg -= ((1 << 10) | 1);
-		}
-		btwrite(reg, ZR36057_VFEHCR);
-		break;
-	}
-}
-
-/*
- * set geometry
- */
-
-static void
-zr36057_set_vfe (struct zoran              *zr,
-		 int                        video_width,
-		 int                        video_height,
-		 const struct zoran_format *format)
-{
-	struct tvnorm *tvn;
-	unsigned HStart, HEnd, VStart, VEnd;
-	unsigned DispMode;
-	unsigned VidWinWid, VidWinHt;
-	unsigned hcrop1, hcrop2, vcrop1, vcrop2;
-	unsigned Wa, We, Ha, He;
-	unsigned X, Y, HorDcm, VerDcm;
-	u32 reg;
-	unsigned mask_line_size;
-
-	tvn = zr->timing;
-
-	Wa = tvn->Wa;
-	Ha = tvn->Ha;
-
-	dprintk(2, KERN_INFO "%s: set_vfe() - width = %d, height = %d\n",
-		ZR_DEVNAME(zr), video_width, video_height);
-
-	if (video_width < BUZ_MIN_WIDTH ||
-	    video_height < BUZ_MIN_HEIGHT ||
-	    video_width > Wa || video_height > Ha) {
-		dprintk(1, KERN_ERR "%s: set_vfe: w=%d h=%d not valid\n",
-			ZR_DEVNAME(zr), video_width, video_height);
-		return;
-	}
-
-	/**** zr36057 ****/
-
-	/* horizontal */
-	VidWinWid = video_width;
-	X = DIV_ROUND_UP(VidWinWid * 64, tvn->Wa);
-	We = (VidWinWid * 64) / X;
-	HorDcm = 64 - X;
-	hcrop1 = 2 * ((tvn->Wa - We) / 4);
-	hcrop2 = tvn->Wa - We - hcrop1;
-	HStart = tvn->HStart ? tvn->HStart : 1;
-	/* (Ronald) Original comment:
-	 * "| 1 Doesn't have any effect, tested on both a DC10 and a DC10+"
-	 * this is false. It inverses chroma values on the LML33R10 (so Cr
-	 * suddenly is shown as Cb and reverse, really cool effect if you
-	 * want to see blue faces, not useful otherwise). So don't use |1.
-	 * However, the DC10 has '0' as HStart, but does need |1, so we
-	 * use a dirty check...
-	 */
-	HEnd = HStart + tvn->Wa - 1;
-	HStart += hcrop1;
-	HEnd -= hcrop2;
-	reg = ((HStart & ZR36057_VFEHCR_Hmask) << ZR36057_VFEHCR_HStart)
-	    | ((HEnd & ZR36057_VFEHCR_Hmask) << ZR36057_VFEHCR_HEnd);
-	if (zr->card.vfe_pol.hsync_pol)
-		reg |= ZR36057_VFEHCR_HSPol;
-	btwrite(reg, ZR36057_VFEHCR);
-
-	/* Vertical */
-	DispMode = !(video_height > BUZ_MAX_HEIGHT / 2);
-	VidWinHt = DispMode ? video_height : video_height / 2;
-	Y = DIV_ROUND_UP(VidWinHt * 64 * 2, tvn->Ha);
-	He = (VidWinHt * 64) / Y;
-	VerDcm = 64 - Y;
-	vcrop1 = (tvn->Ha / 2 - He) / 2;
-	vcrop2 = tvn->Ha / 2 - He - vcrop1;
-	VStart = tvn->VStart;
-	VEnd = VStart + tvn->Ha / 2;	// - 1; FIXME SnapShot times out with -1 in 768*576 on the DC10 - LP
-	VStart += vcrop1;
-	VEnd -= vcrop2;
-	reg = ((VStart & ZR36057_VFEVCR_Vmask) << ZR36057_VFEVCR_VStart)
-	    | ((VEnd & ZR36057_VFEVCR_Vmask) << ZR36057_VFEVCR_VEnd);
-	if (zr->card.vfe_pol.vsync_pol)
-		reg |= ZR36057_VFEVCR_VSPol;
-	btwrite(reg, ZR36057_VFEVCR);
-
-	/* scaler and pixel format */
-	reg = 0;
-	reg |= (HorDcm << ZR36057_VFESPFR_HorDcm);
-	reg |= (VerDcm << ZR36057_VFESPFR_VerDcm);
-	reg |= (DispMode << ZR36057_VFESPFR_DispMode);
-	/* RJ: I don't know, why the following has to be the opposite
-	 * of the corresponding ZR36060 setting, but only this way
-	 * we get the correct colors when uncompressing to the screen  */
-	//reg |= ZR36057_VFESPFR_VCLKPol; /**/
-	/* RJ: Don't know if that is needed for NTSC also */
-	if (!(zr->norm & V4L2_STD_NTSC))
-		reg |= ZR36057_VFESPFR_ExtFl;	// NEEDED!!!!!!! Wolfgang
-	reg |= ZR36057_VFESPFR_TopField;
-	if (HorDcm >= 48) {
-		reg |= 3 << ZR36057_VFESPFR_HFilter;	/* 5 tap filter */
-	} else if (HorDcm >= 32) {
-		reg |= 2 << ZR36057_VFESPFR_HFilter;	/* 4 tap filter */
-	} else if (HorDcm >= 16) {
-		reg |= 1 << ZR36057_VFESPFR_HFilter;	/* 3 tap filter */
-	}
-	reg |= format->vfespfr;
-	btwrite(reg, ZR36057_VFESPFR);
-
-	/* display configuration */
-	reg = (16 << ZR36057_VDCR_MinPix)
-	    | (VidWinHt << ZR36057_VDCR_VidWinHt)
-	    | (VidWinWid << ZR36057_VDCR_VidWinWid);
-	if (pci_pci_problems & PCIPCI_TRITON)
-		// || zr->revision < 1) // Revision 1 has also Triton support
-		reg &= ~ZR36057_VDCR_Triton;
-	else
-		reg |= ZR36057_VDCR_Triton;
-	btwrite(reg, ZR36057_VDCR);
-
-	/* (Ronald) don't write this if overlay_mask = NULL */
-	if (zr->overlay_mask) {
-		/* Write overlay clipping mask data, but don't enable overlay clipping */
-		/* RJ: since this makes only sense on the screen, we use
-		 * zr->overlay_settings.width instead of video_width */
-
-		mask_line_size = (BUZ_MAX_WIDTH + 31) / 32;
-		reg = virt_to_bus(zr->overlay_mask);
-		btwrite(reg, ZR36057_MMTR);
-		reg = virt_to_bus(zr->overlay_mask + mask_line_size);
-		btwrite(reg, ZR36057_MMBR);
-		reg =
-		    mask_line_size - (zr->overlay_settings.width +
-				      31) / 32;
-		if (DispMode == 0)
-			reg += mask_line_size;
-		reg <<= ZR36057_OCR_MaskStride;
-		btwrite(reg, ZR36057_OCR);
-	}
-
-	zr36057_adjust_vfe(zr, zr->codec_mode);
-}
-
-/*
- * Switch overlay on or off
- */
-
-void
-zr36057_overlay (struct zoran *zr,
-		 int           on)
-{
-	u32 reg;
-
-	if (on) {
-		/* do the necessary settings ... */
-		btand(~ZR36057_VDCR_VidEn, ZR36057_VDCR);	/* switch it off first */
-
-		zr36057_set_vfe(zr,
-				zr->overlay_settings.width,
-				zr->overlay_settings.height,
-				zr->overlay_settings.format);
-
-		/* Start and length of each line MUST be 4-byte aligned.
-		 * This should be already checked before the call to this routine.
-		 * All error messages are internal driver checking only! */
-
-		/* video display top and bottom registers */
-		reg = (long) zr->vbuf_base +
-		    zr->overlay_settings.x *
-		    ((zr->overlay_settings.format->depth + 7) / 8) +
-		    zr->overlay_settings.y *
-		    zr->vbuf_bytesperline;
-		btwrite(reg, ZR36057_VDTR);
-		if (reg & 3)
-			dprintk(1,
-				KERN_ERR
-				"%s: zr36057_overlay() - video_address not aligned\n",
-				ZR_DEVNAME(zr));
-		if (zr->overlay_settings.height > BUZ_MAX_HEIGHT / 2)
-			reg += zr->vbuf_bytesperline;
-		btwrite(reg, ZR36057_VDBR);
-
-		/* video stride, status, and frame grab register */
-		reg = zr->vbuf_bytesperline -
-		    zr->overlay_settings.width *
-		    ((zr->overlay_settings.format->depth + 7) / 8);
-		if (zr->overlay_settings.height > BUZ_MAX_HEIGHT / 2)
-			reg += zr->vbuf_bytesperline;
-		if (reg & 3)
-			dprintk(1,
-				KERN_ERR
-				"%s: zr36057_overlay() - video_stride not aligned\n",
-				ZR_DEVNAME(zr));
-		reg = (reg << ZR36057_VSSFGR_DispStride);
-		reg |= ZR36057_VSSFGR_VidOvf;	/* clear overflow status */
-		btwrite(reg, ZR36057_VSSFGR);
-
-		/* Set overlay clipping */
-		if (zr->overlay_settings.clipcount > 0)
-			btor(ZR36057_OCR_OvlEnable, ZR36057_OCR);
-
-		/* ... and switch it on */
-		btor(ZR36057_VDCR_VidEn, ZR36057_VDCR);
-	} else {
-		/* Switch it off */
-		btand(~ZR36057_VDCR_VidEn, ZR36057_VDCR);
-	}
-}
-
-/*
- * The overlay mask has one bit for each pixel on a scan line,
- *  and the maximum window size is BUZ_MAX_WIDTH * BUZ_MAX_HEIGHT pixels.
- */
-
-void write_overlay_mask(struct zoran_fh *fh, struct v4l2_clip *vp, int count)
-{
-	struct zoran *zr = fh->zr;
-	unsigned mask_line_size = (BUZ_MAX_WIDTH + 31) / 32;
-	u32 *mask;
-	int x, y, width, height;
-	unsigned i, j, k;
-
-	/* fill mask with one bits */
-	memset(fh->overlay_mask, ~0, mask_line_size * 4 * BUZ_MAX_HEIGHT);
-
-	for (i = 0; i < count; ++i) {
-		/* pick up local copy of clip */
-		x = vp[i].c.left;
-		y = vp[i].c.top;
-		width = vp[i].c.width;
-		height = vp[i].c.height;
-
-		/* trim clips that extend beyond the window */
-		if (x < 0) {
-			width += x;
-			x = 0;
-		}
-		if (y < 0) {
-			height += y;
-			y = 0;
-		}
-		if (x + width > fh->overlay_settings.width) {
-			width = fh->overlay_settings.width - x;
-		}
-		if (y + height > fh->overlay_settings.height) {
-			height = fh->overlay_settings.height - y;
-		}
-
-		/* ignore degenerate clips */
-		if (height <= 0) {
-			continue;
-		}
-		if (width <= 0) {
-			continue;
-		}
-
-		/* apply clip for each scan line */
-		for (j = 0; j < height; ++j) {
-			/* reset bit for each pixel */
-			/* this can be optimized later if need be */
-			mask = fh->overlay_mask + (y + j) * mask_line_size;
-			for (k = 0; k < width; ++k) {
-				mask[(x + k) / 32] &=
-				    ~((u32) 1 << (x + k) % 32);
-			}
-		}
-	}
-}
-
-/* Enable/Disable uncompressed memory grabbing of the 36057 */
-
-void
-zr36057_set_memgrab (struct zoran *zr,
-		     int           mode)
-{
-	if (mode) {
-		/* We only check SnapShot and not FrameGrab here.  SnapShot==1
-		 * means a capture is already in progress, but FrameGrab==1
-		 * doesn't necessary mean that.  It's more correct to say a 1
-		 * to 0 transition indicates a capture completed.  If a
-		 * capture is pending when capturing is tuned off, FrameGrab
-		 * will be stuck at 1 until capturing is turned back on.
-		 */
-		if (btread(ZR36057_VSSFGR) & ZR36057_VSSFGR_SnapShot)
-			dprintk(1,
-				KERN_WARNING
-				"%s: zr36057_set_memgrab(1) with SnapShot on!?\n",
-				ZR_DEVNAME(zr));
-
-		/* switch on VSync interrupts */
-		btwrite(IRQ_MASK, ZR36057_ISR);	// Clear Interrupts
-		btor(zr->card.vsync_int, ZR36057_ICR);	// SW
-
-		/* enable SnapShot */
-		btor(ZR36057_VSSFGR_SnapShot, ZR36057_VSSFGR);
-
-		/* Set zr36057 video front end  and enable video */
-		zr36057_set_vfe(zr, zr->v4l_settings.width,
-				zr->v4l_settings.height,
-				zr->v4l_settings.format);
-
-		zr->v4l_memgrab_active = 1;
-	} else {
-		/* switch off VSync interrupts */
-		btand(~zr->card.vsync_int, ZR36057_ICR);	// SW
-
-		zr->v4l_memgrab_active = 0;
-		zr->v4l_grab_frame = NO_GRAB_ACTIVE;
-
-		/* reenable grabbing to screen if it was running */
-		if (zr->v4l_overlay_active) {
-			zr36057_overlay(zr, 1);
-		} else {
-			btand(~ZR36057_VDCR_VidEn, ZR36057_VDCR);
-			btand(~ZR36057_VSSFGR_SnapShot, ZR36057_VSSFGR);
-		}
-	}
-}
-
-int
-wait_grab_pending (struct zoran *zr)
-{
-	unsigned long flags;
-
-	/* wait until all pending grabs are finished */
-
-	if (!zr->v4l_memgrab_active)
-		return 0;
-
-	wait_event_interruptible(zr->v4l_capq,
-			(zr->v4l_pend_tail == zr->v4l_pend_head));
-	if (signal_pending(current))
-		return -ERESTARTSYS;
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-	zr36057_set_memgrab(zr, 0);
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	return 0;
-}
-
-/*****************************************************************************
- *                                                                           *
- *  Set up the Buz-specific MJPEG part                                       *
- *                                                                           *
- *****************************************************************************/
-
-static inline void
-set_frame (struct zoran *zr,
-	   int           val)
-{
-	GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_FRAME], val);
-}
-
-static void
-set_videobus_dir (struct zoran *zr,
-		  int           val)
-{
-	switch (zr->card.type) {
-	case LML33:
-	case LML33R10:
-		if (!lml33dpath)
-			GPIO(zr, 5, val);
-		else
-			GPIO(zr, 5, 1);
-		break;
-	default:
-		GPIO(zr, zr->card.gpio[ZR_GPIO_VID_DIR],
-		     zr->card.gpio_pol[ZR_GPIO_VID_DIR] ? !val : val);
-		break;
-	}
-}
-
-static void
-init_jpeg_queue (struct zoran *zr)
-{
-	int i;
-
-	/* re-initialize DMA ring stuff */
-	zr->jpg_que_head = 0;
-	zr->jpg_dma_head = 0;
-	zr->jpg_dma_tail = 0;
-	zr->jpg_que_tail = 0;
-	zr->jpg_seq_num = 0;
-	zr->JPEG_error = 0;
-	zr->num_errors = 0;
-	zr->jpg_err_seq = 0;
-	zr->jpg_err_shift = 0;
-	zr->jpg_queued_num = 0;
-	for (i = 0; i < zr->jpg_buffers.num_buffers; i++) {
-		zr->jpg_buffers.buffer[i].state = BUZ_STATE_USER;	/* nothing going on */
-	}
-	for (i = 0; i < BUZ_NUM_STAT_COM; i++) {
-		zr->stat_com[i] = cpu_to_le32(1);	/* mark as unavailable to zr36057 */
-	}
-}
-
-static void
-zr36057_set_jpg (struct zoran          *zr,
-		 enum zoran_codec_mode  mode)
-{
-	struct tvnorm *tvn;
-	u32 reg;
-
-	tvn = zr->timing;
-
-	/* assert P_Reset, disable code transfer, deassert Active */
-	btwrite(0, ZR36057_JPC);
-
-	/* MJPEG compression mode */
-	switch (mode) {
-
-	case BUZ_MODE_MOTION_COMPRESS:
-	default:
-		reg = ZR36057_JMC_MJPGCmpMode;
-		break;
-
-	case BUZ_MODE_MOTION_DECOMPRESS:
-		reg = ZR36057_JMC_MJPGExpMode;
-		reg |= ZR36057_JMC_SyncMstr;
-		/* RJ: The following is experimental - improves the output to screen */
-		//if(zr->jpg_settings.VFIFO_FB) reg |= ZR36057_JMC_VFIFO_FB; // No, it doesn't. SM
-		break;
-
-	case BUZ_MODE_STILL_COMPRESS:
-		reg = ZR36057_JMC_JPGCmpMode;
-		break;
-
-	case BUZ_MODE_STILL_DECOMPRESS:
-		reg = ZR36057_JMC_JPGExpMode;
-		break;
-
-	}
-	reg |= ZR36057_JMC_JPG;
-	if (zr->jpg_settings.field_per_buff == 1)
-		reg |= ZR36057_JMC_Fld_per_buff;
-	btwrite(reg, ZR36057_JMC);
-
-	/* vertical */
-	btor(ZR36057_VFEVCR_VSPol, ZR36057_VFEVCR);
-	reg = (6 << ZR36057_VSP_VsyncSize) |
-	      (tvn->Ht << ZR36057_VSP_FrmTot);
-	btwrite(reg, ZR36057_VSP);
-	reg = ((zr->jpg_settings.img_y + tvn->VStart) << ZR36057_FVAP_NAY) |
-	      (zr->jpg_settings.img_height << ZR36057_FVAP_PAY);
-	btwrite(reg, ZR36057_FVAP);
-
-	/* horizontal */
-	if (zr->card.vfe_pol.hsync_pol)
-		btor(ZR36057_VFEHCR_HSPol, ZR36057_VFEHCR);
-	else
-		btand(~ZR36057_VFEHCR_HSPol, ZR36057_VFEHCR);
-	reg = ((tvn->HSyncStart) << ZR36057_HSP_HsyncStart) |
-	      (tvn->Wt << ZR36057_HSP_LineTot);
-	btwrite(reg, ZR36057_HSP);
-	reg = ((zr->jpg_settings.img_x +
-		tvn->HStart + 4) << ZR36057_FHAP_NAX) |
-	      (zr->jpg_settings.img_width << ZR36057_FHAP_PAX);
-	btwrite(reg, ZR36057_FHAP);
-
-	/* field process parameters */
-	if (zr->jpg_settings.odd_even)
-		reg = ZR36057_FPP_Odd_Even;
-	else
-		reg = 0;
-
-	btwrite(reg, ZR36057_FPP);
-
-	/* Set proper VCLK Polarity, else colors will be wrong during playback */
-	//btor(ZR36057_VFESPFR_VCLKPol, ZR36057_VFESPFR);
-
-	/* code base address */
-	reg = virt_to_bus(zr->stat_com);
-	btwrite(reg, ZR36057_JCBA);
-
-	/* FIFO threshold (FIFO is 160. double words) */
-	/* NOTE: decimal values here */
-	switch (mode) {
-
-	case BUZ_MODE_STILL_COMPRESS:
-	case BUZ_MODE_MOTION_COMPRESS:
-		if (zr->card.type != BUZ)
-			reg = 140;
-		else
-			reg = 60;
-		break;
-
-	case BUZ_MODE_STILL_DECOMPRESS:
-	case BUZ_MODE_MOTION_DECOMPRESS:
-		reg = 20;
-		break;
-
-	default:
-		reg = 80;
-		break;
-
-	}
-	btwrite(reg, ZR36057_JCFT);
-	zr36057_adjust_vfe(zr, mode);
-
-}
-
-void
-print_interrupts (struct zoran *zr)
-{
-	int res, noerr = 0;
-
-	printk(KERN_INFO "%s: interrupts received:", ZR_DEVNAME(zr));
-	if ((res = zr->field_counter) < -1 || res > 1) {
-		printk(KERN_CONT " FD:%d", res);
-	}
-	if ((res = zr->intr_counter_GIRQ1) != 0) {
-		printk(KERN_CONT " GIRQ1:%d", res);
-		noerr++;
-	}
-	if ((res = zr->intr_counter_GIRQ0) != 0) {
-		printk(KERN_CONT " GIRQ0:%d", res);
-		noerr++;
-	}
-	if ((res = zr->intr_counter_CodRepIRQ) != 0) {
-		printk(KERN_CONT " CodRepIRQ:%d", res);
-		noerr++;
-	}
-	if ((res = zr->intr_counter_JPEGRepIRQ) != 0) {
-		printk(KERN_CONT " JPEGRepIRQ:%d", res);
-		noerr++;
-	}
-	if (zr->JPEG_max_missed) {
-		printk(KERN_CONT " JPEG delays: max=%d min=%d", zr->JPEG_max_missed,
-		       zr->JPEG_min_missed);
-	}
-	if (zr->END_event_missed) {
-		printk(KERN_CONT " ENDs missed: %d", zr->END_event_missed);
-	}
-	//if (zr->jpg_queued_num) {
-	printk(KERN_CONT " queue_state=%ld/%ld/%ld/%ld", zr->jpg_que_tail,
-	       zr->jpg_dma_tail, zr->jpg_dma_head, zr->jpg_que_head);
-	//}
-	if (!noerr) {
-		printk(KERN_CONT ": no interrupts detected.");
-	}
-	printk(KERN_CONT "\n");
-}
-
-void
-clear_interrupt_counters (struct zoran *zr)
-{
-	zr->intr_counter_GIRQ1 = 0;
-	zr->intr_counter_GIRQ0 = 0;
-	zr->intr_counter_CodRepIRQ = 0;
-	zr->intr_counter_JPEGRepIRQ = 0;
-	zr->field_counter = 0;
-	zr->IRQ1_in = 0;
-	zr->IRQ1_out = 0;
-	zr->JPEG_in = 0;
-	zr->JPEG_out = 0;
-	zr->JPEG_0 = 0;
-	zr->JPEG_1 = 0;
-	zr->END_event_missed = 0;
-	zr->JPEG_missed = 0;
-	zr->JPEG_max_missed = 0;
-	zr->JPEG_min_missed = 0x7fffffff;
-}
-
-static u32
-count_reset_interrupt (struct zoran *zr)
-{
-	u32 isr;
-
-	if ((isr = btread(ZR36057_ISR) & 0x78000000)) {
-		if (isr & ZR36057_ISR_GIRQ1) {
-			btwrite(ZR36057_ISR_GIRQ1, ZR36057_ISR);
-			zr->intr_counter_GIRQ1++;
-		}
-		if (isr & ZR36057_ISR_GIRQ0) {
-			btwrite(ZR36057_ISR_GIRQ0, ZR36057_ISR);
-			zr->intr_counter_GIRQ0++;
-		}
-		if (isr & ZR36057_ISR_CodRepIRQ) {
-			btwrite(ZR36057_ISR_CodRepIRQ, ZR36057_ISR);
-			zr->intr_counter_CodRepIRQ++;
-		}
-		if (isr & ZR36057_ISR_JPEGRepIRQ) {
-			btwrite(ZR36057_ISR_JPEGRepIRQ, ZR36057_ISR);
-			zr->intr_counter_JPEGRepIRQ++;
-		}
-	}
-	return isr;
-}
-
-void
-jpeg_start (struct zoran *zr)
-{
-	int reg;
-
-	zr->frame_num = 0;
-
-	/* deassert P_reset, disable code transfer, deassert Active */
-	btwrite(ZR36057_JPC_P_Reset, ZR36057_JPC);
-	/* stop flushing the internal code buffer */
-	btand(~ZR36057_MCTCR_CFlush, ZR36057_MCTCR);
-	/* enable code transfer */
-	btor(ZR36057_JPC_CodTrnsEn, ZR36057_JPC);
-
-	/* clear IRQs */
-	btwrite(IRQ_MASK, ZR36057_ISR);
-	/* enable the JPEG IRQs */
-	btwrite(zr->card.jpeg_int |
-			ZR36057_ICR_JPEGRepIRQ |
-			ZR36057_ICR_IntPinEn,
-		ZR36057_ICR);
-
-	set_frame(zr, 0);	// \FRAME
-
-	/* set the JPEG codec guest ID */
-	reg = (zr->card.gpcs[1] << ZR36057_JCGI_JPEGuestID) |
-	       (0 << ZR36057_JCGI_JPEGuestReg);
-	btwrite(reg, ZR36057_JCGI);
-
-	if (zr->card.video_vfe == CODEC_TYPE_ZR36016 &&
-	    zr->card.video_codec == CODEC_TYPE_ZR36050) {
-		/* Enable processing on the ZR36016 */
-		if (zr->vfe)
-			zr36016_write(zr->vfe, 0, 1);
-
-		/* load the address of the GO register in the ZR36050 latch */
-		post_office_write(zr, 0, 0, 0);
-	}
-
-	/* assert Active */
-	btor(ZR36057_JPC_Active, ZR36057_JPC);
-
-	/* enable the Go generation */
-	btor(ZR36057_JMC_Go_en, ZR36057_JMC);
-	udelay(30);
-
-	set_frame(zr, 1);	// /FRAME
-
-	dprintk(3, KERN_DEBUG "%s: jpeg_start\n", ZR_DEVNAME(zr));
-}
-
-void
-zr36057_enable_jpg (struct zoran          *zr,
-		    enum zoran_codec_mode  mode)
-{
-	struct vfe_settings cap;
-	int field_size =
-	    zr->jpg_buffers.buffer_size / zr->jpg_settings.field_per_buff;
-
-	zr->codec_mode = mode;
-
-	cap.x = zr->jpg_settings.img_x;
-	cap.y = zr->jpg_settings.img_y;
-	cap.width = zr->jpg_settings.img_width;
-	cap.height = zr->jpg_settings.img_height;
-	cap.decimation =
-	    zr->jpg_settings.HorDcm | (zr->jpg_settings.VerDcm << 8);
-	cap.quality = zr->jpg_settings.jpg_comp.quality;
-
-	switch (mode) {
-
-	case BUZ_MODE_MOTION_COMPRESS: {
-		struct jpeg_app_marker app;
-		struct jpeg_com_marker com;
-
-		/* In motion compress mode, the decoder output must be enabled, and
-		 * the video bus direction set to input.
-		 */
-		set_videobus_dir(zr, 0);
-		decoder_call(zr, video, s_stream, 1);
-		encoder_call(zr, video, s_routing, 0, 0, 0);
-
-		/* Take the JPEG codec and the VFE out of sleep */
-		jpeg_codec_sleep(zr, 0);
-
-		/* set JPEG app/com marker */
-		app.appn = zr->jpg_settings.jpg_comp.APPn;
-		app.len = zr->jpg_settings.jpg_comp.APP_len;
-		memcpy(app.data, zr->jpg_settings.jpg_comp.APP_data, 60);
-		zr->codec->control(zr->codec, CODEC_S_JPEG_APP_DATA,
-				   sizeof(struct jpeg_app_marker), &app);
-
-		com.len = zr->jpg_settings.jpg_comp.COM_len;
-		memcpy(com.data, zr->jpg_settings.jpg_comp.COM_data, 60);
-		zr->codec->control(zr->codec, CODEC_S_JPEG_COM_DATA,
-				   sizeof(struct jpeg_com_marker), &com);
-
-		/* Setup the JPEG codec */
-		zr->codec->control(zr->codec, CODEC_S_JPEG_TDS_BYTE,
-				   sizeof(int), &field_size);
-		zr->codec->set_video(zr->codec, zr->timing, &cap,
-				     &zr->card.vfe_pol);
-		zr->codec->set_mode(zr->codec, CODEC_DO_COMPRESSION);
-
-		/* Setup the VFE */
-		if (zr->vfe) {
-			zr->vfe->control(zr->vfe, CODEC_S_JPEG_TDS_BYTE,
-					 sizeof(int), &field_size);
-			zr->vfe->set_video(zr->vfe, zr->timing, &cap,
-					   &zr->card.vfe_pol);
-			zr->vfe->set_mode(zr->vfe, CODEC_DO_COMPRESSION);
-		}
-
-		init_jpeg_queue(zr);
-		zr36057_set_jpg(zr, mode);	// \P_Reset, ... Video param, FIFO
-
-		clear_interrupt_counters(zr);
-		dprintk(2, KERN_INFO "%s: enable_jpg(MOTION_COMPRESS)\n",
-			ZR_DEVNAME(zr));
-		break;
-	}
-
-	case BUZ_MODE_MOTION_DECOMPRESS:
-		/* In motion decompression mode, the decoder output must be disabled, and
-		 * the video bus direction set to output.
-		 */
-		decoder_call(zr, video, s_stream, 0);
-		set_videobus_dir(zr, 1);
-		encoder_call(zr, video, s_routing, 1, 0, 0);
-
-		/* Take the JPEG codec and the VFE out of sleep */
-		jpeg_codec_sleep(zr, 0);
-		/* Setup the VFE */
-		if (zr->vfe) {
-			zr->vfe->set_video(zr->vfe, zr->timing, &cap,
-					   &zr->card.vfe_pol);
-			zr->vfe->set_mode(zr->vfe, CODEC_DO_EXPANSION);
-		}
-		/* Setup the JPEG codec */
-		zr->codec->set_video(zr->codec, zr->timing, &cap,
-				     &zr->card.vfe_pol);
-		zr->codec->set_mode(zr->codec, CODEC_DO_EXPANSION);
-
-		init_jpeg_queue(zr);
-		zr36057_set_jpg(zr, mode);	// \P_Reset, ... Video param, FIFO
-
-		clear_interrupt_counters(zr);
-		dprintk(2, KERN_INFO "%s: enable_jpg(MOTION_DECOMPRESS)\n",
-			ZR_DEVNAME(zr));
-		break;
-
-	case BUZ_MODE_IDLE:
-	default:
-		/* shut down processing */
-		btand(~(zr->card.jpeg_int | ZR36057_ICR_JPEGRepIRQ),
-		      ZR36057_ICR);
-		btwrite(zr->card.jpeg_int | ZR36057_ICR_JPEGRepIRQ,
-			ZR36057_ISR);
-		btand(~ZR36057_JMC_Go_en, ZR36057_JMC);	// \Go_en
-
-		msleep(50);
-
-		set_videobus_dir(zr, 0);
-		set_frame(zr, 1);	// /FRAME
-		btor(ZR36057_MCTCR_CFlush, ZR36057_MCTCR);	// /CFlush
-		btwrite(0, ZR36057_JPC);	// \P_Reset,\CodTrnsEn,\Active
-		btand(~ZR36057_JMC_VFIFO_FB, ZR36057_JMC);
-		btand(~ZR36057_JMC_SyncMstr, ZR36057_JMC);
-		jpeg_codec_reset(zr);
-		jpeg_codec_sleep(zr, 1);
-		zr36057_adjust_vfe(zr, mode);
-
-		decoder_call(zr, video, s_stream, 1);
-		encoder_call(zr, video, s_routing, 0, 0, 0);
-
-		dprintk(2, KERN_INFO "%s: enable_jpg(IDLE)\n", ZR_DEVNAME(zr));
-		break;
-
-	}
-}
-
-/* when this is called the spinlock must be held */
-void
-zoran_feed_stat_com (struct zoran *zr)
-{
-	/* move frames from pending queue to DMA */
-
-	int frame, i, max_stat_com;
-
-	max_stat_com =
-	    (zr->jpg_settings.TmpDcm ==
-	     1) ? BUZ_NUM_STAT_COM : (BUZ_NUM_STAT_COM >> 1);
-
-	while ((zr->jpg_dma_head - zr->jpg_dma_tail) < max_stat_com &&
-	       zr->jpg_dma_head < zr->jpg_que_head) {
-
-		frame = zr->jpg_pend[zr->jpg_dma_head & BUZ_MASK_FRAME];
-		if (zr->jpg_settings.TmpDcm == 1) {
-			/* fill 1 stat_com entry */
-			i = (zr->jpg_dma_head -
-			     zr->jpg_err_shift) & BUZ_MASK_STAT_COM;
-			if (!(zr->stat_com[i] & cpu_to_le32(1)))
-				break;
-			zr->stat_com[i] =
-			    cpu_to_le32(zr->jpg_buffers.buffer[frame].jpg.frag_tab_bus);
-		} else {
-			/* fill 2 stat_com entries */
-			i = ((zr->jpg_dma_head -
-			      zr->jpg_err_shift) & 1) * 2;
-			if (!(zr->stat_com[i] & cpu_to_le32(1)))
-				break;
-			zr->stat_com[i] =
-			    cpu_to_le32(zr->jpg_buffers.buffer[frame].jpg.frag_tab_bus);
-			zr->stat_com[i + 1] =
-			    cpu_to_le32(zr->jpg_buffers.buffer[frame].jpg.frag_tab_bus);
-		}
-		zr->jpg_buffers.buffer[frame].state = BUZ_STATE_DMA;
-		zr->jpg_dma_head++;
-
-	}
-	if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS)
-		zr->jpg_queued_num++;
-}
-
-/* when this is called the spinlock must be held */
-static void
-zoran_reap_stat_com (struct zoran *zr)
-{
-	/* move frames from DMA queue to done queue */
-
-	int i;
-	u32 stat_com;
-	unsigned int seq;
-	unsigned int dif;
-	struct zoran_buffer *buffer;
-	int frame;
-
-	/* In motion decompress we don't have a hardware frame counter,
-	 * we just count the interrupts here */
-
-	if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS) {
-		zr->jpg_seq_num++;
-	}
-	while (zr->jpg_dma_tail < zr->jpg_dma_head) {
-		if (zr->jpg_settings.TmpDcm == 1)
-			i = (zr->jpg_dma_tail -
-			     zr->jpg_err_shift) & BUZ_MASK_STAT_COM;
-		else
-			i = ((zr->jpg_dma_tail -
-			      zr->jpg_err_shift) & 1) * 2 + 1;
-
-		stat_com = le32_to_cpu(zr->stat_com[i]);
-
-		if ((stat_com & 1) == 0) {
-			return;
-		}
-		frame = zr->jpg_pend[zr->jpg_dma_tail & BUZ_MASK_FRAME];
-		buffer = &zr->jpg_buffers.buffer[frame];
-		v4l2_get_timestamp(&buffer->bs.timestamp);
-
-		if (zr->codec_mode == BUZ_MODE_MOTION_COMPRESS) {
-			buffer->bs.length = (stat_com & 0x7fffff) >> 1;
-
-			/* update sequence number with the help of the counter in stat_com */
-
-			seq = ((stat_com >> 24) + zr->jpg_err_seq) & 0xff;
-			dif = (seq - zr->jpg_seq_num) & 0xff;
-			zr->jpg_seq_num += dif;
-		} else {
-			buffer->bs.length = 0;
-		}
-		buffer->bs.seq =
-		    zr->jpg_settings.TmpDcm ==
-		    2 ? (zr->jpg_seq_num >> 1) : zr->jpg_seq_num;
-		buffer->state = BUZ_STATE_DONE;
-
-		zr->jpg_dma_tail++;
-	}
-}
-
-static void zoran_restart(struct zoran *zr)
-{
-	/* Now the stat_comm buffer is ready for restart */
-	unsigned int status = 0;
-	int mode;
-
-	if (zr->codec_mode == BUZ_MODE_MOTION_COMPRESS) {
-		decoder_call(zr, video, g_input_status, &status);
-		mode = CODEC_DO_COMPRESSION;
-	} else {
-		status = V4L2_IN_ST_NO_SIGNAL;
-		mode = CODEC_DO_EXPANSION;
-	}
-	if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS ||
-	    !(status & V4L2_IN_ST_NO_SIGNAL)) {
-		/********** RESTART code *************/
-		jpeg_codec_reset(zr);
-		zr->codec->set_mode(zr->codec, mode);
-		zr36057_set_jpg(zr, zr->codec_mode);
-		jpeg_start(zr);
-
-		if (zr->num_errors <= 8)
-			dprintk(2, KERN_INFO "%s: Restart\n",
-				ZR_DEVNAME(zr));
-
-		zr->JPEG_missed = 0;
-		zr->JPEG_error = 2;
-		/********** End RESTART code ***********/
-	}
-}
-
-static void
-error_handler (struct zoran *zr,
-	       u32           astat,
-	       u32           stat)
-{
-	int i;
-
-	/* This is JPEG error handling part */
-	if (zr->codec_mode != BUZ_MODE_MOTION_COMPRESS &&
-	    zr->codec_mode != BUZ_MODE_MOTION_DECOMPRESS) {
-		return;
-	}
-
-	if ((stat & 1) == 0 &&
-	    zr->codec_mode == BUZ_MODE_MOTION_COMPRESS &&
-	    zr->jpg_dma_tail - zr->jpg_que_tail >= zr->jpg_buffers.num_buffers) {
-		/* No free buffers... */
-		zoran_reap_stat_com(zr);
-		zoran_feed_stat_com(zr);
-		wake_up_interruptible(&zr->jpg_capq);
-		zr->JPEG_missed = 0;
-		return;
-	}
-
-	if (zr->JPEG_error == 1) {
-		zoran_restart(zr);
-		return;
-	}
-
-	/*
-	 * First entry: error just happened during normal operation
-	 *
-	 * In BUZ_MODE_MOTION_COMPRESS:
-	 *
-	 * Possible glitch in TV signal. In this case we should
-	 * stop the codec and wait for good quality signal before
-	 * restarting it to avoid further problems
-	 *
-	 * In BUZ_MODE_MOTION_DECOMPRESS:
-	 *
-	 * Bad JPEG frame: we have to mark it as processed (codec crashed
-	 * and was not able to do it itself), and to remove it from queue.
-	 */
-	btand(~ZR36057_JMC_Go_en, ZR36057_JMC);
-	udelay(1);
-	stat = stat | (post_office_read(zr, 7, 0) & 3) << 8;
-	btwrite(0, ZR36057_JPC);
-	btor(ZR36057_MCTCR_CFlush, ZR36057_MCTCR);
-	jpeg_codec_reset(zr);
-	jpeg_codec_sleep(zr, 1);
-	zr->JPEG_error = 1;
-	zr->num_errors++;
-
-	/* Report error */
-	if (zr36067_debug > 1 && zr->num_errors <= 8) {
-		long frame;
-		int j;
-
-		frame = zr->jpg_pend[zr->jpg_dma_tail & BUZ_MASK_FRAME];
-		printk(KERN_ERR
-		       "%s: JPEG error stat=0x%08x(0x%08x) queue_state=%ld/%ld/%ld/%ld seq=%ld frame=%ld. Codec stopped. ",
-		       ZR_DEVNAME(zr), stat, zr->last_isr,
-		       zr->jpg_que_tail, zr->jpg_dma_tail,
-		       zr->jpg_dma_head, zr->jpg_que_head,
-		       zr->jpg_seq_num, frame);
-		printk(KERN_INFO "stat_com frames:");
-		for (j = 0; j < BUZ_NUM_STAT_COM; j++) {
-			for (i = 0; i < zr->jpg_buffers.num_buffers; i++) {
-				if (le32_to_cpu(zr->stat_com[j]) == zr->jpg_buffers.buffer[i].jpg.frag_tab_bus)
-					printk(KERN_CONT "% d->%d", j, i);
-			}
-		}
-		printk(KERN_CONT "\n");
-	}
-	/* Find an entry in stat_com and rotate contents */
-	if (zr->jpg_settings.TmpDcm == 1)
-		i = (zr->jpg_dma_tail - zr->jpg_err_shift) & BUZ_MASK_STAT_COM;
-	else
-		i = ((zr->jpg_dma_tail - zr->jpg_err_shift) & 1) * 2;
-	if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS) {
-		/* Mimic zr36067 operation */
-		zr->stat_com[i] |= cpu_to_le32(1);
-		if (zr->jpg_settings.TmpDcm != 1)
-			zr->stat_com[i + 1] |= cpu_to_le32(1);
-		/* Refill */
-		zoran_reap_stat_com(zr);
-		zoran_feed_stat_com(zr);
-		wake_up_interruptible(&zr->jpg_capq);
-		/* Find an entry in stat_com again after refill */
-		if (zr->jpg_settings.TmpDcm == 1)
-			i = (zr->jpg_dma_tail - zr->jpg_err_shift) & BUZ_MASK_STAT_COM;
-		else
-			i = ((zr->jpg_dma_tail - zr->jpg_err_shift) & 1) * 2;
-	}
-	if (i) {
-		/* Rotate stat_comm entries to make current entry first */
-		int j;
-		__le32 bus_addr[BUZ_NUM_STAT_COM];
-
-		/* Here we are copying the stat_com array, which
-		 * is already in little endian format, so
-		 * no endian conversions here
-		 */
-		memcpy(bus_addr, zr->stat_com, sizeof(bus_addr));
-
-		for (j = 0; j < BUZ_NUM_STAT_COM; j++)
-			zr->stat_com[j] = bus_addr[(i + j) & BUZ_MASK_STAT_COM];
-
-		zr->jpg_err_shift += i;
-		zr->jpg_err_shift &= BUZ_MASK_STAT_COM;
-	}
-	if (zr->codec_mode == BUZ_MODE_MOTION_COMPRESS)
-		zr->jpg_err_seq = zr->jpg_seq_num;	/* + 1; */
-	zoran_restart(zr);
-}
-
-irqreturn_t
-zoran_irq (int             irq,
-	   void           *dev_id)
-{
-	u32 stat, astat;
-	int count;
-	struct zoran *zr;
-	unsigned long flags;
-
-	zr = dev_id;
-	count = 0;
-
-	if (zr->testing) {
-		/* Testing interrupts */
-		spin_lock_irqsave(&zr->spinlock, flags);
-		while ((stat = count_reset_interrupt(zr))) {
-			if (count++ > 100) {
-				btand(~ZR36057_ICR_IntPinEn, ZR36057_ICR);
-				dprintk(1,
-					KERN_ERR
-					"%s: IRQ lockup while testing, isr=0x%08x, cleared int mask\n",
-					ZR_DEVNAME(zr), stat);
-				wake_up_interruptible(&zr->test_q);
-			}
-		}
-		zr->last_isr = stat;
-		spin_unlock_irqrestore(&zr->spinlock, flags);
-		return IRQ_HANDLED;
-	}
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-	while (1) {
-		/* get/clear interrupt status bits */
-		stat = count_reset_interrupt(zr);
-		astat = stat & IRQ_MASK;
-		if (!astat) {
-			break;
-		}
-		dprintk(4,
-			KERN_DEBUG
-			"zoran_irq: astat: 0x%08x, mask: 0x%08x\n",
-			astat, btread(ZR36057_ICR));
-		if (astat & zr->card.vsync_int) {	// SW
-
-			if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS ||
-			    zr->codec_mode == BUZ_MODE_MOTION_COMPRESS) {
-				/* count missed interrupts */
-				zr->JPEG_missed++;
-			}
-			//post_office_read(zr,1,0);
-			/* Interrupts may still happen when
-			 * zr->v4l_memgrab_active is switched off.
-			 * We simply ignore them */
-
-			if (zr->v4l_memgrab_active) {
-				/* A lot more checks should be here ... */
-				if ((btread(ZR36057_VSSFGR) & ZR36057_VSSFGR_SnapShot) == 0)
-					dprintk(1,
-						KERN_WARNING
-						"%s: BuzIRQ with SnapShot off ???\n",
-						ZR_DEVNAME(zr));
-
-				if (zr->v4l_grab_frame != NO_GRAB_ACTIVE) {
-					/* There is a grab on a frame going on, check if it has finished */
-					if ((btread(ZR36057_VSSFGR) & ZR36057_VSSFGR_FrameGrab) == 0) {
-						/* it is finished, notify the user */
-
-						zr->v4l_buffers.buffer[zr->v4l_grab_frame].state = BUZ_STATE_DONE;
-						zr->v4l_buffers.buffer[zr->v4l_grab_frame].bs.seq = zr->v4l_grab_seq;
-						v4l2_get_timestamp(&zr->v4l_buffers.buffer[zr->v4l_grab_frame].bs.timestamp);
-						zr->v4l_grab_frame = NO_GRAB_ACTIVE;
-						zr->v4l_pend_tail++;
-					}
-				}
-
-				if (zr->v4l_grab_frame == NO_GRAB_ACTIVE)
-					wake_up_interruptible(&zr->v4l_capq);
-
-				/* Check if there is another grab queued */
-
-				if (zr->v4l_grab_frame == NO_GRAB_ACTIVE &&
-				    zr->v4l_pend_tail != zr->v4l_pend_head) {
-					int frame = zr->v4l_pend[zr->v4l_pend_tail & V4L_MASK_FRAME];
-					u32 reg;
-
-					zr->v4l_grab_frame = frame;
-
-					/* Set zr36057 video front end and enable video */
-
-					/* Buffer address */
-
-					reg = zr->v4l_buffers.buffer[frame].v4l.fbuffer_bus;
-					btwrite(reg, ZR36057_VDTR);
-					if (zr->v4l_settings.height > BUZ_MAX_HEIGHT / 2)
-						reg += zr->v4l_settings.bytesperline;
-					btwrite(reg, ZR36057_VDBR);
-
-					/* video stride, status, and frame grab register */
-					reg = 0;
-					if (zr->v4l_settings.height > BUZ_MAX_HEIGHT / 2)
-						reg += zr->v4l_settings.bytesperline;
-					reg = (reg << ZR36057_VSSFGR_DispStride);
-					reg |= ZR36057_VSSFGR_VidOvf;
-					reg |= ZR36057_VSSFGR_SnapShot;
-					reg |= ZR36057_VSSFGR_FrameGrab;
-					btwrite(reg, ZR36057_VSSFGR);
-
-					btor(ZR36057_VDCR_VidEn,
-					     ZR36057_VDCR);
-				}
-			}
-
-			/* even if we don't grab, we do want to increment
-			 * the sequence counter to see lost frames */
-			zr->v4l_grab_seq++;
-		}
-#if (IRQ_MASK & ZR36057_ISR_CodRepIRQ)
-		if (astat & ZR36057_ISR_CodRepIRQ) {
-			zr->intr_counter_CodRepIRQ++;
-			IDEBUG(printk(KERN_DEBUG "%s: ZR36057_ISR_CodRepIRQ\n",
-				ZR_DEVNAME(zr)));
-			btand(~ZR36057_ICR_CodRepIRQ, ZR36057_ICR);
-		}
-#endif				/* (IRQ_MASK & ZR36057_ISR_CodRepIRQ) */
-
-#if (IRQ_MASK & ZR36057_ISR_JPEGRepIRQ)
-		if ((astat & ZR36057_ISR_JPEGRepIRQ) &&
-		    (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS ||
-		     zr->codec_mode == BUZ_MODE_MOTION_COMPRESS)) {
-			if (zr36067_debug > 1 && (!zr->frame_num || zr->JPEG_error)) {
-				char sv[BUZ_NUM_STAT_COM + 1];
-				int i;
-
-				printk(KERN_INFO
-				       "%s: first frame ready: state=0x%08x odd_even=%d field_per_buff=%d delay=%d\n",
-				       ZR_DEVNAME(zr), stat,
-				       zr->jpg_settings.odd_even,
-				       zr->jpg_settings.field_per_buff,
-				       zr->JPEG_missed);
-
-				for (i = 0; i < BUZ_NUM_STAT_COM; i++)
-					sv[i] = le32_to_cpu(zr->stat_com[i]) & 1 ? '1' : '0';
-				sv[BUZ_NUM_STAT_COM] = 0;
-				printk(KERN_INFO
-				       "%s: stat_com=%s queue_state=%ld/%ld/%ld/%ld\n",
-				       ZR_DEVNAME(zr), sv,
-				       zr->jpg_que_tail,
-				       zr->jpg_dma_tail,
-				       zr->jpg_dma_head,
-				       zr->jpg_que_head);
-			} else {
-				/* Get statistics */
-				if (zr->JPEG_missed > zr->JPEG_max_missed)
-					zr->JPEG_max_missed = zr->JPEG_missed;
-				if (zr->JPEG_missed < zr->JPEG_min_missed)
-					zr->JPEG_min_missed = zr->JPEG_missed;
-			}
-
-			if (zr36067_debug > 2 && zr->frame_num < 6) {
-				int i;
-
-				printk(KERN_INFO "%s: seq=%ld stat_com:",
-				       ZR_DEVNAME(zr), zr->jpg_seq_num);
-				for (i = 0; i < 4; i++) {
-					printk(KERN_CONT " %08x",
-					       le32_to_cpu(zr->stat_com[i]));
-				}
-				printk(KERN_CONT "\n");
-			}
-			zr->frame_num++;
-			zr->JPEG_missed = 0;
-			zr->JPEG_error = 0;
-			zoran_reap_stat_com(zr);
-			zoran_feed_stat_com(zr);
-			wake_up_interruptible(&zr->jpg_capq);
-		}
-#endif				/* (IRQ_MASK & ZR36057_ISR_JPEGRepIRQ) */
-
-		/* DATERR, too many fields missed, error processing */
-		if ((astat & zr->card.jpeg_int) ||
-		    zr->JPEG_missed > 25 ||
-		    zr->JPEG_error == 1	||
-		    ((zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS) &&
-		     (zr->frame_num && (zr->JPEG_missed > zr->jpg_settings.field_per_buff)))) {
-			error_handler(zr, astat, stat);
-		}
-
-		count++;
-		if (count > 10) {
-			dprintk(2, KERN_WARNING "%s: irq loop %d\n",
-				ZR_DEVNAME(zr), count);
-			if (count > 20) {
-				btand(~ZR36057_ICR_IntPinEn, ZR36057_ICR);
-				dprintk(2,
-					KERN_ERR
-					"%s: IRQ lockup, cleared int mask\n",
-					ZR_DEVNAME(zr));
-				break;
-			}
-		}
-		zr->last_isr = stat;
-	}
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	return IRQ_HANDLED;
-}
-
-void
-zoran_set_pci_master (struct zoran *zr,
-		      int           set_master)
-{
-	if (set_master) {
-		pci_set_master(zr->pci_dev);
-	} else {
-		u16 command;
-
-		pci_read_config_word(zr->pci_dev, PCI_COMMAND, &command);
-		command &= ~PCI_COMMAND_MASTER;
-		pci_write_config_word(zr->pci_dev, PCI_COMMAND, command);
-	}
-}
-
-void
-zoran_init_hardware (struct zoran *zr)
-{
-	/* Enable bus-mastering */
-	zoran_set_pci_master(zr, 1);
-
-	/* Initialize the board */
-	if (zr->card.init) {
-		zr->card.init(zr);
-	}
-
-	decoder_call(zr, core, init, 0);
-	decoder_call(zr, video, s_std, zr->norm);
-	decoder_call(zr, video, s_routing,
-		zr->card.input[zr->input].muxsel, 0, 0);
-
-	encoder_call(zr, core, init, 0);
-	encoder_call(zr, video, s_std_output, zr->norm);
-	encoder_call(zr, video, s_routing, 0, 0, 0);
-
-	/* toggle JPEG codec sleep to sync PLL */
-	jpeg_codec_sleep(zr, 1);
-	jpeg_codec_sleep(zr, 0);
-
-	/*
-	 * set individual interrupt enables (without GIRQ1)
-	 * but don't global enable until zoran_open()
-	 */
-	zr36057_init_vfe(zr);
-
-	zr36057_enable_jpg(zr, BUZ_MODE_IDLE);
-
-	btwrite(IRQ_MASK, ZR36057_ISR);	// Clears interrupts
-}
-
-void
-zr36057_restart (struct zoran *zr)
-{
-	btwrite(0, ZR36057_SPGPPCR);
-	mdelay(1);
-	btor(ZR36057_SPGPPCR_SoftReset, ZR36057_SPGPPCR);
-	mdelay(1);
-
-	/* assert P_Reset */
-	btwrite(0, ZR36057_JPC);
-	/* set up GPIO direction - all output */
-	btwrite(ZR36057_SPGPPCR_SoftReset | 0, ZR36057_SPGPPCR);
-
-	/* set up GPIO pins and guest bus timing */
-	btwrite((0x81 << 24) | 0x8888, ZR36057_GPPGCR1);
-}
-
-/*
- * initialize video front end
- */
-
-static void
-zr36057_init_vfe (struct zoran *zr)
-{
-	u32 reg;
-
-	reg = btread(ZR36057_VFESPFR);
-	reg |= ZR36057_VFESPFR_LittleEndian;
-	reg &= ~ZR36057_VFESPFR_VCLKPol;
-	reg |= ZR36057_VFESPFR_ExtFl;
-	reg |= ZR36057_VFESPFR_TopField;
-	btwrite(reg, ZR36057_VFESPFR);
-	reg = btread(ZR36057_VDCR);
-	if (pci_pci_problems & PCIPCI_TRITON)
-		// || zr->revision < 1) // Revision 1 has also Triton support
-		reg &= ~ZR36057_VDCR_Triton;
-	else
-		reg |= ZR36057_VDCR_Triton;
-	btwrite(reg, ZR36057_VDCR);
-}
diff --git a/drivers/media/pci/zoran/zoran_device.h b/drivers/media/pci/zoran/zoran_device.h
deleted file mode 100644
index a507aaad4ebb..000000000000
--- a/drivers/media/pci/zoran/zoran_device.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles card-specific data and detection
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#ifndef __ZORAN_DEVICE_H__
-#define __ZORAN_DEVICE_H__
-
-/* general purpose I/O */
-extern void GPIO(struct zoran *zr,
-		 int bit,
-		 unsigned int value);
-
-/* codec (or actually: guest bus) access */
-extern int post_office_wait(struct zoran *zr);
-extern int post_office_write(struct zoran *zr,
-			     unsigned guest,
-			     unsigned reg,
-			     unsigned value);
-extern int post_office_read(struct zoran *zr,
-			    unsigned guest,
-			    unsigned reg);
-
-extern void detect_guest_activity(struct zoran *zr);
-
-extern void jpeg_codec_sleep(struct zoran *zr,
-			     int sleep);
-extern int jpeg_codec_reset(struct zoran *zr);
-
-/* zr360x7 access to raw capture */
-extern void zr36057_overlay(struct zoran *zr,
-			    int on);
-extern void write_overlay_mask(struct zoran_fh *fh,
-			       struct v4l2_clip *vp,
-			       int count);
-extern void zr36057_set_memgrab(struct zoran *zr,
-				int mode);
-extern int wait_grab_pending(struct zoran *zr);
-
-/* interrupts */
-extern void print_interrupts(struct zoran *zr);
-extern void clear_interrupt_counters(struct zoran *zr);
-extern irqreturn_t zoran_irq(int irq, void *dev_id);
-
-/* JPEG codec access */
-extern void jpeg_start(struct zoran *zr);
-extern void zr36057_enable_jpg(struct zoran *zr,
-			       enum zoran_codec_mode mode);
-extern void zoran_feed_stat_com(struct zoran *zr);
-
-/* general */
-extern void zoran_set_pci_master(struct zoran *zr,
-				 int set_master);
-extern void zoran_init_hardware(struct zoran *zr);
-extern void zr36057_restart(struct zoran *zr);
-
-extern const struct zoran_format zoran_formats[];
-
-extern int v4l_nbufs;
-extern int v4l_bufsize;
-extern int jpg_nbufs;
-extern int jpg_bufsize;
-extern int pass_through;
-
-/* i2c */
-#define decoder_call(zr, o, f, args...) \
-	v4l2_subdev_call(zr->decoder, o, f, ##args)
-#define encoder_call(zr, o, f, args...) \
-	v4l2_subdev_call(zr->encoder, o, f, ##args)
-
-#endif				/* __ZORAN_DEVICE_H__ */
diff --git a/drivers/media/pci/zoran/zoran_driver.c b/drivers/media/pci/zoran/zoran_driver.c
deleted file mode 100644
index 14f9c0e26a1c..000000000000
--- a/drivers/media/pci/zoran/zoran_driver.c
+++ /dev/null
@@ -1,2849 +0,0 @@
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Changes for BUZ by Wolfgang Scherr <scherr@net4you.net>
- *
- * Changes for DC10/DC30 by Laurent Pinchart <laurent.pinchart@skynet.be>
- *
- * Changes for LML33R10 by Maxim Yevtyushkin <max@linuxmedialabs.com>
- *
- * Changes for videodev2/v4l2 by Ronald Bultje <rbultje@ronald.bitfreak.net>
- *
- * Based on
- *
- * Miro DC10 driver
- * Copyright (C) 1999 Wolfgang Scherr <scherr@net4you.net>
- *
- * Iomega Buz driver version 1.0
- * Copyright (C) 1999 Rainer Johanni <Rainer@Johanni.de>
- *
- * buz.0.0.3
- * Copyright (C) 1998 Dave Perks <dperks@ibm.net>
- *
- * bttv - Bt848 frame grabber driver
- * Copyright (C) 1996,97,98 Ralph  Metzler (rjkm@thp.uni-koeln.de)
- *                        & Marcus Metzler (mocm@thp.uni-koeln.de)
- *
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <linux/pci.h>
-#include <linux/vmalloc.h>
-#include <linux/wait.h>
-
-#include <linux/interrupt.h>
-#include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
-
-#include <linux/spinlock.h>
-
-#include <linux/videodev2.h>
-#include <media/v4l2-common.h>
-#include <media/v4l2-ioctl.h>
-#include <media/v4l2-event.h>
-#include "videocodec.h"
-
-#include <asm/byteorder.h>
-#include <asm/io.h>
-#include <linux/uaccess.h>
-#include <linux/proc_fs.h>
-
-#include <linux/mutex.h>
-#include "zoran.h"
-#include "zoran_device.h"
-#include "zoran_card.h"
-
-
-const struct zoran_format zoran_formats[] = {
-	{
-		.name = "15-bit RGB LE",
-		.fourcc = V4L2_PIX_FMT_RGB555,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 15,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB555|ZR36057_VFESPFR_ErrDif|
-			   ZR36057_VFESPFR_LittleEndian,
-	}, {
-		.name = "15-bit RGB BE",
-		.fourcc = V4L2_PIX_FMT_RGB555X,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 15,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB555|ZR36057_VFESPFR_ErrDif,
-	}, {
-		.name = "16-bit RGB LE",
-		.fourcc = V4L2_PIX_FMT_RGB565,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 16,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB565|ZR36057_VFESPFR_ErrDif|
-			   ZR36057_VFESPFR_LittleEndian,
-	}, {
-		.name = "16-bit RGB BE",
-		.fourcc = V4L2_PIX_FMT_RGB565X,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 16,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB565|ZR36057_VFESPFR_ErrDif,
-	}, {
-		.name = "24-bit RGB",
-		.fourcc = V4L2_PIX_FMT_BGR24,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 24,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB888|ZR36057_VFESPFR_Pack24,
-	}, {
-		.name = "32-bit RGB LE",
-		.fourcc = V4L2_PIX_FMT_BGR32,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 32,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB888|ZR36057_VFESPFR_LittleEndian,
-	}, {
-		.name = "32-bit RGB BE",
-		.fourcc = V4L2_PIX_FMT_RGB32,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 32,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB888,
-	}, {
-		.name = "4:2:2, packed, YUYV",
-		.fourcc = V4L2_PIX_FMT_YUYV,
-		.colorspace = V4L2_COLORSPACE_SMPTE170M,
-		.depth = 16,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_YUV422,
-	}, {
-		.name = "4:2:2, packed, UYVY",
-		.fourcc = V4L2_PIX_FMT_UYVY,
-		.colorspace = V4L2_COLORSPACE_SMPTE170M,
-		.depth = 16,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_YUV422|ZR36057_VFESPFR_LittleEndian,
-	}, {
-		.name = "Hardware-encoded Motion-JPEG",
-		.fourcc = V4L2_PIX_FMT_MJPEG,
-		.colorspace = V4L2_COLORSPACE_SMPTE170M,
-		.depth = 0,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_PLAYBACK |
-			 ZORAN_FORMAT_COMPRESSED,
-	}
-};
-#define NUM_FORMATS ARRAY_SIZE(zoran_formats)
-
-	/* small helper function for calculating buffersizes for v4l2
-	 * we calculate the nearest higher power-of-two, which
-	 * will be the recommended buffersize */
-static __u32
-zoran_v4l2_calc_bufsize (struct zoran_jpg_settings *settings)
-{
-	__u8 div = settings->VerDcm * settings->HorDcm * settings->TmpDcm;
-	__u32 num = (1024 * 512) / (div);
-	__u32 result = 2;
-
-	num--;
-	while (num) {
-		num >>= 1;
-		result <<= 1;
-	}
-
-	if (result > jpg_bufsize)
-		return jpg_bufsize;
-	if (result < 8192)
-		return 8192;
-	return result;
-}
-
-/* forward references */
-static void v4l_fbuffer_free(struct zoran_fh *fh);
-static void jpg_fbuffer_free(struct zoran_fh *fh);
-
-/* Set mapping mode */
-static void map_mode_raw(struct zoran_fh *fh)
-{
-	fh->map_mode = ZORAN_MAP_MODE_RAW;
-	fh->buffers.buffer_size = v4l_bufsize;
-	fh->buffers.num_buffers = v4l_nbufs;
-}
-static void map_mode_jpg(struct zoran_fh *fh, int play)
-{
-	fh->map_mode = play ? ZORAN_MAP_MODE_JPG_PLAY : ZORAN_MAP_MODE_JPG_REC;
-	fh->buffers.buffer_size = jpg_bufsize;
-	fh->buffers.num_buffers = jpg_nbufs;
-}
-static inline const char *mode_name(enum zoran_map_mode mode)
-{
-	return mode == ZORAN_MAP_MODE_RAW ? "V4L" : "JPG";
-}
-
-/*
- *   Allocate the V4L grab buffers
- *
- *   These have to be pysically contiguous.
- */
-
-static int v4l_fbuffer_alloc(struct zoran_fh *fh)
-{
-	struct zoran *zr = fh->zr;
-	int i, off;
-	unsigned char *mem;
-
-	for (i = 0; i < fh->buffers.num_buffers; i++) {
-		if (fh->buffers.buffer[i].v4l.fbuffer)
-			dprintk(2,
-				KERN_WARNING
-				"%s: %s - buffer %d already allocated!?\n",
-				ZR_DEVNAME(zr), __func__, i);
-
-		//udelay(20);
-		mem = kmalloc(fh->buffers.buffer_size,
-			      GFP_KERNEL | __GFP_NOWARN);
-		if (!mem) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - kmalloc for V4L buf %d failed\n",
-				ZR_DEVNAME(zr), __func__, i);
-			v4l_fbuffer_free(fh);
-			return -ENOBUFS;
-		}
-		fh->buffers.buffer[i].v4l.fbuffer = mem;
-		fh->buffers.buffer[i].v4l.fbuffer_phys = virt_to_phys(mem);
-		fh->buffers.buffer[i].v4l.fbuffer_bus = virt_to_bus(mem);
-		for (off = 0; off < fh->buffers.buffer_size;
-		     off += PAGE_SIZE)
-			SetPageReserved(virt_to_page(mem + off));
-		dprintk(4,
-			KERN_INFO
-			"%s: %s - V4L frame %d mem %p (bus: 0x%llx)\n",
-			ZR_DEVNAME(zr), __func__, i, mem,
-			(unsigned long long)virt_to_bus(mem));
-	}
-
-	fh->buffers.allocated = 1;
-
-	return 0;
-}
-
-/* free the V4L grab buffers */
-static void v4l_fbuffer_free(struct zoran_fh *fh)
-{
-	struct zoran *zr = fh->zr;
-	int i, off;
-	unsigned char *mem;
-
-	dprintk(4, KERN_INFO "%s: %s\n", ZR_DEVNAME(zr), __func__);
-
-	for (i = 0; i < fh->buffers.num_buffers; i++) {
-		if (!fh->buffers.buffer[i].v4l.fbuffer)
-			continue;
-
-		mem = fh->buffers.buffer[i].v4l.fbuffer;
-		for (off = 0; off < fh->buffers.buffer_size;
-		     off += PAGE_SIZE)
-			ClearPageReserved(virt_to_page(mem + off));
-		kfree(fh->buffers.buffer[i].v4l.fbuffer);
-		fh->buffers.buffer[i].v4l.fbuffer = NULL;
-	}
-
-	fh->buffers.allocated = 0;
-}
-
-/*
- *   Allocate the MJPEG grab buffers.
- *
- *   If a Natoma chipset is present and this is a revision 1 zr36057,
- *   each MJPEG buffer needs to be physically contiguous.
- *   (RJ: This statement is from Dave Perks' original driver,
- *   I could never check it because I have a zr36067)
- *
- *   RJ: The contents grab buffers needs never be accessed in the driver.
- *       Therefore there is no need to allocate them with vmalloc in order
- *       to get a contiguous virtual memory space.
- *       I don't understand why many other drivers first allocate them with
- *       vmalloc (which uses internally also get_zeroed_page, but delivers you
- *       virtual addresses) and then again have to make a lot of efforts
- *       to get the physical address.
- *
- *   Ben Capper:
- *       On big-endian architectures (such as ppc) some extra steps
- *       are needed. When reading and writing to the stat_com array
- *       and fragment buffers, the device expects to see little-
- *       endian values. The use of cpu_to_le32() and le32_to_cpu()
- *       in this function (and one or two others in zoran_device.c)
- *       ensure that these values are always stored in little-endian
- *       form, regardless of architecture. The zr36057 does Very Bad
- *       Things on big endian architectures if the stat_com array
- *       and fragment buffers are not little-endian.
- */
-
-static int jpg_fbuffer_alloc(struct zoran_fh *fh)
-{
-	struct zoran *zr = fh->zr;
-	int i, j, off;
-	u8 *mem;
-
-	for (i = 0; i < fh->buffers.num_buffers; i++) {
-		if (fh->buffers.buffer[i].jpg.frag_tab)
-			dprintk(2,
-				KERN_WARNING
-				"%s: %s - buffer %d already allocated!?\n",
-				ZR_DEVNAME(zr), __func__, i);
-
-		/* Allocate fragment table for this buffer */
-
-		mem = (void *)get_zeroed_page(GFP_KERNEL);
-		if (!mem) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - get_zeroed_page (frag_tab) failed for buffer %d\n",
-				ZR_DEVNAME(zr), __func__, i);
-			jpg_fbuffer_free(fh);
-			return -ENOBUFS;
-		}
-		fh->buffers.buffer[i].jpg.frag_tab = (__le32 *)mem;
-		fh->buffers.buffer[i].jpg.frag_tab_bus = virt_to_bus(mem);
-
-		if (fh->buffers.need_contiguous) {
-			mem = kmalloc(fh->buffers.buffer_size, GFP_KERNEL);
-			if (mem == NULL) {
-				dprintk(1,
-					KERN_ERR
-					"%s: %s - kmalloc failed for buffer %d\n",
-					ZR_DEVNAME(zr), __func__, i);
-				jpg_fbuffer_free(fh);
-				return -ENOBUFS;
-			}
-			fh->buffers.buffer[i].jpg.frag_tab[0] =
-				cpu_to_le32(virt_to_bus(mem));
-			fh->buffers.buffer[i].jpg.frag_tab[1] =
-				cpu_to_le32((fh->buffers.buffer_size >> 1) | 1);
-			for (off = 0; off < fh->buffers.buffer_size; off += PAGE_SIZE)
-				SetPageReserved(virt_to_page(mem + off));
-		} else {
-			/* jpg_bufsize is already page aligned */
-			for (j = 0; j < fh->buffers.buffer_size / PAGE_SIZE; j++) {
-				mem = (void *)get_zeroed_page(GFP_KERNEL);
-				if (mem == NULL) {
-					dprintk(1,
-						KERN_ERR
-						"%s: %s - get_zeroed_page failed for buffer %d\n",
-						ZR_DEVNAME(zr), __func__, i);
-					jpg_fbuffer_free(fh);
-					return -ENOBUFS;
-				}
-
-				fh->buffers.buffer[i].jpg.frag_tab[2 * j] =
-					cpu_to_le32(virt_to_bus(mem));
-				fh->buffers.buffer[i].jpg.frag_tab[2 * j + 1] =
-					cpu_to_le32((PAGE_SIZE >> 2) << 1);
-				SetPageReserved(virt_to_page(mem));
-			}
-
-			fh->buffers.buffer[i].jpg.frag_tab[2 * j - 1] |= cpu_to_le32(1);
-		}
-	}
-
-	dprintk(4,
-		KERN_DEBUG "%s: %s - %d KB allocated\n",
-		ZR_DEVNAME(zr), __func__,
-		(fh->buffers.num_buffers * fh->buffers.buffer_size) >> 10);
-
-	fh->buffers.allocated = 1;
-
-	return 0;
-}
-
-/* free the MJPEG grab buffers */
-static void jpg_fbuffer_free(struct zoran_fh *fh)
-{
-	struct zoran *zr = fh->zr;
-	int i, j, off;
-	unsigned char *mem;
-	__le32 frag_tab;
-	struct zoran_buffer *buffer;
-
-	dprintk(4, KERN_DEBUG "%s: %s\n", ZR_DEVNAME(zr), __func__);
-
-	for (i = 0, buffer = &fh->buffers.buffer[0];
-	     i < fh->buffers.num_buffers; i++, buffer++) {
-		if (!buffer->jpg.frag_tab)
-			continue;
-
-		if (fh->buffers.need_contiguous) {
-			frag_tab = buffer->jpg.frag_tab[0];
-
-			if (frag_tab) {
-				mem = bus_to_virt(le32_to_cpu(frag_tab));
-				for (off = 0; off < fh->buffers.buffer_size; off += PAGE_SIZE)
-					ClearPageReserved(virt_to_page(mem + off));
-				kfree(mem);
-				buffer->jpg.frag_tab[0] = 0;
-				buffer->jpg.frag_tab[1] = 0;
-			}
-		} else {
-			for (j = 0; j < fh->buffers.buffer_size / PAGE_SIZE; j++) {
-				frag_tab = buffer->jpg.frag_tab[2 * j];
-
-				if (!frag_tab)
-					break;
-				ClearPageReserved(virt_to_page(bus_to_virt(le32_to_cpu(frag_tab))));
-				free_page((unsigned long)bus_to_virt(le32_to_cpu(frag_tab)));
-				buffer->jpg.frag_tab[2 * j] = 0;
-				buffer->jpg.frag_tab[2 * j + 1] = 0;
-			}
-		}
-
-		free_page((unsigned long)buffer->jpg.frag_tab);
-		buffer->jpg.frag_tab = NULL;
-	}
-
-	fh->buffers.allocated = 0;
-}
-
-/*
- *   V4L Buffer grabbing
- */
-
-static int
-zoran_v4l_set_format (struct zoran_fh           *fh,
-		      int                        width,
-		      int                        height,
-		      const struct zoran_format *format)
-{
-	struct zoran *zr = fh->zr;
-	int bpp;
-
-	/* Check size and format of the grab wanted */
-
-	if (height < BUZ_MIN_HEIGHT || width < BUZ_MIN_WIDTH ||
-	    height > BUZ_MAX_HEIGHT || width > BUZ_MAX_WIDTH) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - wrong frame size (%dx%d)\n",
-			ZR_DEVNAME(zr), __func__, width, height);
-		return -EINVAL;
-	}
-
-	bpp = (format->depth + 7) / 8;
-
-	/* Check against available buffer size */
-	if (height * width * bpp > fh->buffers.buffer_size) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - video buffer size (%d kB) is too small\n",
-			ZR_DEVNAME(zr), __func__, fh->buffers.buffer_size >> 10);
-		return -EINVAL;
-	}
-
-	/* The video front end needs 4-byte alinged line sizes */
-
-	if ((bpp == 2 && (width & 1)) || (bpp == 3 && (width & 3))) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - wrong frame alignment\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	fh->v4l_settings.width = width;
-	fh->v4l_settings.height = height;
-	fh->v4l_settings.format = format;
-	fh->v4l_settings.bytesperline = bpp * fh->v4l_settings.width;
-
-	return 0;
-}
-
-static int zoran_v4l_queue_frame(struct zoran_fh *fh, int num)
-{
-	struct zoran *zr = fh->zr;
-	unsigned long flags;
-	int res = 0;
-
-	if (!fh->buffers.allocated) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - buffers not yet allocated\n",
-			ZR_DEVNAME(zr), __func__);
-		res = -ENOMEM;
-	}
-
-	/* No grabbing outside the buffer range! */
-	if (num >= fh->buffers.num_buffers || num < 0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - buffer %d is out of range\n",
-			ZR_DEVNAME(zr), __func__, num);
-		res = -EINVAL;
-	}
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-
-	if (fh->buffers.active == ZORAN_FREE) {
-		if (zr->v4l_buffers.active == ZORAN_FREE) {
-			zr->v4l_buffers = fh->buffers;
-			fh->buffers.active = ZORAN_ACTIVE;
-		} else {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - another session is already capturing\n",
-				ZR_DEVNAME(zr), __func__);
-			res = -EBUSY;
-		}
-	}
-
-	/* make sure a grab isn't going on currently with this buffer */
-	if (!res) {
-		switch (zr->v4l_buffers.buffer[num].state) {
-		default:
-		case BUZ_STATE_PEND:
-			if (zr->v4l_buffers.active == ZORAN_FREE) {
-				fh->buffers.active = ZORAN_FREE;
-				zr->v4l_buffers.allocated = 0;
-			}
-			res = -EBUSY;	/* what are you doing? */
-			break;
-		case BUZ_STATE_DONE:
-			dprintk(2,
-				KERN_WARNING
-				"%s: %s - queueing buffer %d in state DONE!?\n",
-				ZR_DEVNAME(zr), __func__, num);
-			/* fall through */
-		case BUZ_STATE_USER:
-			/* since there is at least one unused buffer there's room for at least
-			 * one more pend[] entry */
-			zr->v4l_pend[zr->v4l_pend_head++ & V4L_MASK_FRAME] = num;
-			zr->v4l_buffers.buffer[num].state = BUZ_STATE_PEND;
-			zr->v4l_buffers.buffer[num].bs.length =
-			    fh->v4l_settings.bytesperline *
-			    zr->v4l_settings.height;
-			fh->buffers.buffer[num] = zr->v4l_buffers.buffer[num];
-			break;
-		}
-	}
-
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	if (!res && zr->v4l_buffers.active == ZORAN_FREE)
-		zr->v4l_buffers.active = fh->buffers.active;
-
-	return res;
-}
-
-/*
- * Sync on a V4L buffer
- */
-
-static int v4l_sync(struct zoran_fh *fh, int frame)
-{
-	struct zoran *zr = fh->zr;
-	unsigned long flags;
-
-	if (fh->buffers.active == ZORAN_FREE) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - no grab active for this session\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	/* check passed-in frame number */
-	if (frame >= fh->buffers.num_buffers || frame < 0) {
-		dprintk(1,
-			KERN_ERR "%s: %s - frame %d is invalid\n",
-			ZR_DEVNAME(zr), __func__, frame);
-		return -EINVAL;
-	}
-
-	/* Check if is buffer was queued at all */
-	if (zr->v4l_buffers.buffer[frame].state == BUZ_STATE_USER) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - attempt to sync on a buffer which was not queued?\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EPROTO;
-	}
-
-	mutex_unlock(&zr->lock);
-	/* wait on this buffer to get ready */
-	if (!wait_event_interruptible_timeout(zr->v4l_capq,
-		(zr->v4l_buffers.buffer[frame].state != BUZ_STATE_PEND), 10*HZ)) {
-		mutex_lock(&zr->lock);
-		return -ETIME;
-	}
-	mutex_lock(&zr->lock);
-	if (signal_pending(current))
-		return -ERESTARTSYS;
-
-	/* buffer should now be in BUZ_STATE_DONE */
-	if (zr->v4l_buffers.buffer[frame].state != BUZ_STATE_DONE)
-		dprintk(2,
-			KERN_ERR "%s: %s - internal state error\n",
-			ZR_DEVNAME(zr), __func__);
-
-	zr->v4l_buffers.buffer[frame].state = BUZ_STATE_USER;
-	fh->buffers.buffer[frame] = zr->v4l_buffers.buffer[frame];
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-
-	/* Check if streaming capture has finished */
-	if (zr->v4l_pend_tail == zr->v4l_pend_head) {
-		zr36057_set_memgrab(zr, 0);
-		if (zr->v4l_buffers.active == ZORAN_ACTIVE) {
-			fh->buffers.active = zr->v4l_buffers.active = ZORAN_FREE;
-			zr->v4l_buffers.allocated = 0;
-		}
-	}
-
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	return 0;
-}
-
-/*
- *   Queue a MJPEG buffer for capture/playback
- */
-
-static int zoran_jpg_queue_frame(struct zoran_fh *fh, int num,
-				 enum zoran_codec_mode mode)
-{
-	struct zoran *zr = fh->zr;
-	unsigned long flags;
-	int res = 0;
-
-	/* Check if buffers are allocated */
-	if (!fh->buffers.allocated) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - buffers not yet allocated\n",
-			ZR_DEVNAME(zr), __func__);
-		return -ENOMEM;
-	}
-
-	/* No grabbing outside the buffer range! */
-	if (num >= fh->buffers.num_buffers || num < 0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - buffer %d out of range\n",
-			ZR_DEVNAME(zr), __func__, num);
-		return -EINVAL;
-	}
-
-	/* what is the codec mode right now? */
-	if (zr->codec_mode == BUZ_MODE_IDLE) {
-		zr->jpg_settings = fh->jpg_settings;
-	} else if (zr->codec_mode != mode) {
-		/* wrong codec mode active - invalid */
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - codec in wrong mode\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	if (fh->buffers.active == ZORAN_FREE) {
-		if (zr->jpg_buffers.active == ZORAN_FREE) {
-			zr->jpg_buffers = fh->buffers;
-			fh->buffers.active = ZORAN_ACTIVE;
-		} else {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - another session is already capturing\n",
-				ZR_DEVNAME(zr), __func__);
-			res = -EBUSY;
-		}
-	}
-
-	if (!res && zr->codec_mode == BUZ_MODE_IDLE) {
-		/* Ok load up the jpeg codec */
-		zr36057_enable_jpg(zr, mode);
-	}
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-
-	if (!res) {
-		switch (zr->jpg_buffers.buffer[num].state) {
-		case BUZ_STATE_DONE:
-			dprintk(2,
-				KERN_WARNING
-				"%s: %s - queing frame in BUZ_STATE_DONE state!?\n",
-				ZR_DEVNAME(zr), __func__);
-			/* fall through */
-		case BUZ_STATE_USER:
-			/* since there is at least one unused buffer there's room for at
-			 *least one more pend[] entry */
-			zr->jpg_pend[zr->jpg_que_head++ & BUZ_MASK_FRAME] = num;
-			zr->jpg_buffers.buffer[num].state = BUZ_STATE_PEND;
-			fh->buffers.buffer[num] = zr->jpg_buffers.buffer[num];
-			zoran_feed_stat_com(zr);
-			break;
-		default:
-		case BUZ_STATE_DMA:
-		case BUZ_STATE_PEND:
-			if (zr->jpg_buffers.active == ZORAN_FREE) {
-				fh->buffers.active = ZORAN_FREE;
-				zr->jpg_buffers.allocated = 0;
-			}
-			res = -EBUSY;	/* what are you doing? */
-			break;
-		}
-	}
-
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	if (!res && zr->jpg_buffers.active == ZORAN_FREE)
-		zr->jpg_buffers.active = fh->buffers.active;
-
-	return res;
-}
-
-static int jpg_qbuf(struct zoran_fh *fh, int frame, enum zoran_codec_mode mode)
-{
-	struct zoran *zr = fh->zr;
-	int res = 0;
-
-	/* Does the user want to stop streaming? */
-	if (frame < 0) {
-		if (zr->codec_mode == mode) {
-			if (fh->buffers.active == ZORAN_FREE) {
-				dprintk(1,
-					KERN_ERR
-					"%s: %s(-1) - session not active\n",
-					ZR_DEVNAME(zr), __func__);
-				return -EINVAL;
-			}
-			fh->buffers.active = zr->jpg_buffers.active = ZORAN_FREE;
-			zr->jpg_buffers.allocated = 0;
-			zr36057_enable_jpg(zr, BUZ_MODE_IDLE);
-			return 0;
-		} else {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - stop streaming but not in streaming mode\n",
-				ZR_DEVNAME(zr), __func__);
-			return -EINVAL;
-		}
-	}
-
-	if ((res = zoran_jpg_queue_frame(fh, frame, mode)))
-		return res;
-
-	/* Start the jpeg codec when the first frame is queued  */
-	if (!res && zr->jpg_que_head == 1)
-		jpeg_start(zr);
-
-	return res;
-}
-
-/*
- *   Sync on a MJPEG buffer
- */
-
-static int jpg_sync(struct zoran_fh *fh, struct zoran_sync *bs)
-{
-	struct zoran *zr = fh->zr;
-	unsigned long flags;
-	int frame;
-
-	if (fh->buffers.active == ZORAN_FREE) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - capture is not currently active\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-	if (zr->codec_mode != BUZ_MODE_MOTION_DECOMPRESS &&
-	    zr->codec_mode != BUZ_MODE_MOTION_COMPRESS) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - codec not in streaming mode\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-	mutex_unlock(&zr->lock);
-	if (!wait_event_interruptible_timeout(zr->jpg_capq,
-			(zr->jpg_que_tail != zr->jpg_dma_tail ||
-			 zr->jpg_dma_tail == zr->jpg_dma_head),
-			10*HZ)) {
-		int isr;
-
-		btand(~ZR36057_JMC_Go_en, ZR36057_JMC);
-		udelay(1);
-		zr->codec->control(zr->codec, CODEC_G_STATUS,
-					   sizeof(isr), &isr);
-		mutex_lock(&zr->lock);
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - timeout: codec isr=0x%02x\n",
-			ZR_DEVNAME(zr), __func__, isr);
-
-		return -ETIME;
-
-	}
-	mutex_lock(&zr->lock);
-	if (signal_pending(current))
-		return -ERESTARTSYS;
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-
-	if (zr->jpg_dma_tail != zr->jpg_dma_head)
-		frame = zr->jpg_pend[zr->jpg_que_tail++ & BUZ_MASK_FRAME];
-	else
-		frame = zr->jpg_pend[zr->jpg_que_tail & BUZ_MASK_FRAME];
-
-	/* buffer should now be in BUZ_STATE_DONE */
-	if (zr->jpg_buffers.buffer[frame].state != BUZ_STATE_DONE)
-		dprintk(2,
-			KERN_ERR "%s: %s - internal state error\n",
-			ZR_DEVNAME(zr), __func__);
-
-	*bs = zr->jpg_buffers.buffer[frame].bs;
-	bs->frame = frame;
-	zr->jpg_buffers.buffer[frame].state = BUZ_STATE_USER;
-	fh->buffers.buffer[frame] = zr->jpg_buffers.buffer[frame];
-
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	return 0;
-}
-
-static void zoran_open_init_session(struct zoran_fh *fh)
-{
-	int i;
-	struct zoran *zr = fh->zr;
-
-	/* Per default, map the V4L Buffers */
-	map_mode_raw(fh);
-
-	/* take over the card's current settings */
-	fh->overlay_settings = zr->overlay_settings;
-	fh->overlay_settings.is_set = 0;
-	fh->overlay_settings.format = zr->overlay_settings.format;
-	fh->overlay_active = ZORAN_FREE;
-
-	/* v4l settings */
-	fh->v4l_settings = zr->v4l_settings;
-	/* jpg settings */
-	fh->jpg_settings = zr->jpg_settings;
-
-	/* buffers */
-	memset(&fh->buffers, 0, sizeof(fh->buffers));
-	for (i = 0; i < MAX_FRAME; i++) {
-		fh->buffers.buffer[i].state = BUZ_STATE_USER;	/* nothing going on */
-		fh->buffers.buffer[i].bs.frame = i;
-	}
-	fh->buffers.allocated = 0;
-	fh->buffers.active = ZORAN_FREE;
-}
-
-static void zoran_close_end_session(struct zoran_fh *fh)
-{
-	struct zoran *zr = fh->zr;
-
-	/* overlay */
-	if (fh->overlay_active != ZORAN_FREE) {
-		fh->overlay_active = zr->overlay_active = ZORAN_FREE;
-		zr->v4l_overlay_active = 0;
-		if (!zr->v4l_memgrab_active)
-			zr36057_overlay(zr, 0);
-		zr->overlay_mask = NULL;
-	}
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
-		/* v4l capture */
-		if (fh->buffers.active != ZORAN_FREE) {
-			unsigned long flags;
-
-			spin_lock_irqsave(&zr->spinlock, flags);
-			zr36057_set_memgrab(zr, 0);
-			zr->v4l_buffers.allocated = 0;
-			zr->v4l_buffers.active = fh->buffers.active = ZORAN_FREE;
-			spin_unlock_irqrestore(&zr->spinlock, flags);
-		}
-
-		/* v4l buffers */
-		if (fh->buffers.allocated)
-			v4l_fbuffer_free(fh);
-	} else {
-		/* jpg capture */
-		if (fh->buffers.active != ZORAN_FREE) {
-			zr36057_enable_jpg(zr, BUZ_MODE_IDLE);
-			zr->jpg_buffers.allocated = 0;
-			zr->jpg_buffers.active = fh->buffers.active = ZORAN_FREE;
-		}
-
-		/* jpg buffers */
-		if (fh->buffers.allocated)
-			jpg_fbuffer_free(fh);
-	}
-}
-
-/*
- *   Open a zoran card. Right now the flags stuff is just playing
- */
-
-static int zoran_open(struct file *file)
-{
-	struct zoran *zr = video_drvdata(file);
-	struct zoran_fh *fh;
-	int res, first_open = 0;
-
-	dprintk(2, KERN_INFO "%s: %s(%s, pid=[%d]), users(-)=%d\n",
-		ZR_DEVNAME(zr), __func__, current->comm, task_pid_nr(current), zr->user + 1);
-
-	mutex_lock(&zr->lock);
-
-	if (zr->user >= 2048) {
-		dprintk(1, KERN_ERR "%s: too many users (%d) on device\n",
-			ZR_DEVNAME(zr), zr->user);
-		res = -EBUSY;
-		goto fail_unlock;
-	}
-
-	/* now, create the open()-specific file_ops struct */
-	fh = kzalloc(sizeof(struct zoran_fh), GFP_KERNEL);
-	if (!fh) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - allocation of zoran_fh failed\n",
-			ZR_DEVNAME(zr), __func__);
-		res = -ENOMEM;
-		goto fail_unlock;
-	}
-	v4l2_fh_init(&fh->fh, video_devdata(file));
-
-	/* used to be BUZ_MAX_WIDTH/HEIGHT, but that gives overflows
-	 * on norm-change! */
-	fh->overlay_mask =
-	    kmalloc(((768 + 31) / 32) * 576 * 4, GFP_KERNEL);
-	if (!fh->overlay_mask) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - allocation of overlay_mask failed\n",
-			ZR_DEVNAME(zr), __func__);
-		res = -ENOMEM;
-		goto fail_fh;
-	}
-
-	if (zr->user++ == 0)
-		first_open = 1;
-
-	/* default setup - TODO: look at flags */
-	if (first_open) {	/* First device open */
-		zr36057_restart(zr);
-		zoran_open_init_params(zr);
-		zoran_init_hardware(zr);
-
-		btor(ZR36057_ICR_IntPinEn, ZR36057_ICR);
-	}
-
-	/* set file_ops stuff */
-	file->private_data = fh;
-	fh->zr = zr;
-	zoran_open_init_session(fh);
-	v4l2_fh_add(&fh->fh);
-	mutex_unlock(&zr->lock);
-
-	return 0;
-
-fail_fh:
-	v4l2_fh_exit(&fh->fh);
-	kfree(fh);
-fail_unlock:
-	mutex_unlock(&zr->lock);
-
-	dprintk(2, KERN_INFO "%s: open failed (%d), users(-)=%d\n",
-		ZR_DEVNAME(zr), res, zr->user);
-
-	return res;
-}
-
-static int
-zoran_close(struct file  *file)
-{
-	struct zoran_fh *fh = file->private_data;
-	struct zoran *zr = fh->zr;
-
-	dprintk(2, KERN_INFO "%s: %s(%s, pid=[%d]), users(+)=%d\n",
-		ZR_DEVNAME(zr), __func__, current->comm, task_pid_nr(current), zr->user - 1);
-
-	/* kernel locks (fs/device.c), so don't do that ourselves
-	 * (prevents deadlocks) */
-	mutex_lock(&zr->lock);
-
-	zoran_close_end_session(fh);
-
-	if (zr->user-- == 1) {	/* Last process */
-		/* Clean up JPEG process */
-		wake_up_interruptible(&zr->jpg_capq);
-		zr36057_enable_jpg(zr, BUZ_MODE_IDLE);
-		zr->jpg_buffers.allocated = 0;
-		zr->jpg_buffers.active = ZORAN_FREE;
-
-		/* disable interrupts */
-		btand(~ZR36057_ICR_IntPinEn, ZR36057_ICR);
-
-		if (zr36067_debug > 1)
-			print_interrupts(zr);
-
-		/* Overlay off */
-		zr->v4l_overlay_active = 0;
-		zr36057_overlay(zr, 0);
-		zr->overlay_mask = NULL;
-
-		/* capture off */
-		wake_up_interruptible(&zr->v4l_capq);
-		zr36057_set_memgrab(zr, 0);
-		zr->v4l_buffers.allocated = 0;
-		zr->v4l_buffers.active = ZORAN_FREE;
-		zoran_set_pci_master(zr, 0);
-
-		if (!pass_through) {	/* Switch to color bar */
-			decoder_call(zr, video, s_stream, 0);
-			encoder_call(zr, video, s_routing, 2, 0, 0);
-		}
-	}
-	mutex_unlock(&zr->lock);
-
-	v4l2_fh_del(&fh->fh);
-	v4l2_fh_exit(&fh->fh);
-	kfree(fh->overlay_mask);
-	kfree(fh);
-
-	dprintk(4, KERN_INFO "%s: %s done\n", ZR_DEVNAME(zr), __func__);
-
-	return 0;
-}
-
-static int setup_fbuffer(struct zoran_fh *fh,
-	       void                      *base,
-	       const struct zoran_format *fmt,
-	       int                        width,
-	       int                        height,
-	       int                        bytesperline)
-{
-	struct zoran *zr = fh->zr;
-
-	/* (Ronald) v4l/v4l2 guidelines */
-	if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RAWIO))
-		return -EPERM;
-
-	/* Don't allow frame buffer overlay if PCI or AGP is buggy, or on
-	   ALi Magik (that needs very low latency while the card needs a
-	   higher value always) */
-
-	if (pci_pci_problems & (PCIPCI_FAIL | PCIAGP_FAIL | PCIPCI_ALIMAGIK))
-		return -ENXIO;
-
-	/* we need a bytesperline value, even if not given */
-	if (!bytesperline)
-		bytesperline = width * ((fmt->depth + 7) & ~7) / 8;
-
-#if 0
-	if (zr->overlay_active) {
-		/* dzjee... stupid users... don't even bother to turn off
-		 * overlay before changing the memory location...
-		 * normally, we would return errors here. However, one of
-		 * the tools that does this is... xawtv! and since xawtv
-		 * is used by +/- 99% of the users, we'd rather be user-
-		 * friendly and silently do as if nothing went wrong */
-		dprintk(3,
-			KERN_ERR
-			"%s: %s - forced overlay turnoff because framebuffer changed\n",
-			ZR_DEVNAME(zr), __func__);
-		zr36057_overlay(zr, 0);
-	}
-#endif
-
-	if (!(fmt->flags & ZORAN_FORMAT_OVERLAY)) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - no valid overlay format given\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-	if (height <= 0 || width <= 0 || bytesperline <= 0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - invalid height/width/bpl value (%d|%d|%d)\n",
-			ZR_DEVNAME(zr), __func__, width, height, bytesperline);
-		return -EINVAL;
-	}
-	if (bytesperline & 3) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - bytesperline (%d) must be 4-byte aligned\n",
-			ZR_DEVNAME(zr), __func__, bytesperline);
-		return -EINVAL;
-	}
-
-	zr->vbuf_base = (void *) ((unsigned long) base & ~3);
-	zr->vbuf_height = height;
-	zr->vbuf_width = width;
-	zr->vbuf_depth = fmt->depth;
-	zr->overlay_settings.format = fmt;
-	zr->vbuf_bytesperline = bytesperline;
-
-	/* The user should set new window parameters */
-	zr->overlay_settings.is_set = 0;
-
-	return 0;
-}
-
-
-static int setup_window(struct zoran_fh *fh,
-			int x,
-			int y,
-			int width,
-			int height,
-			struct v4l2_clip __user *clips,
-			unsigned int clipcount,
-			void __user *bitmap)
-{
-	struct zoran *zr = fh->zr;
-	struct v4l2_clip *vcp = NULL;
-	int on, end;
-
-
-	if (!zr->vbuf_base) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - frame buffer has to be set first\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	if (!fh->overlay_settings.format) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - no overlay format set\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	if (clipcount > 2048) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - invalid clipcount\n",
-			 ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	/*
-	 * The video front end needs 4-byte alinged line sizes, we correct that
-	 * silently here if necessary
-	 */
-	if (zr->vbuf_depth == 15 || zr->vbuf_depth == 16) {
-		end = (x + width) & ~1;	/* round down */
-		x = (x + 1) & ~1;	/* round up */
-		width = end - x;
-	}
-
-	if (zr->vbuf_depth == 24) {
-		end = (x + width) & ~3;	/* round down */
-		x = (x + 3) & ~3;	/* round up */
-		width = end - x;
-	}
-
-	if (width > BUZ_MAX_WIDTH)
-		width = BUZ_MAX_WIDTH;
-	if (height > BUZ_MAX_HEIGHT)
-		height = BUZ_MAX_HEIGHT;
-
-	/* Check for invalid parameters */
-	if (width < BUZ_MIN_WIDTH || height < BUZ_MIN_HEIGHT ||
-	    width > BUZ_MAX_WIDTH || height > BUZ_MAX_HEIGHT) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - width = %d or height = %d invalid\n",
-			ZR_DEVNAME(zr), __func__, width, height);
-		return -EINVAL;
-	}
-
-	fh->overlay_settings.x = x;
-	fh->overlay_settings.y = y;
-	fh->overlay_settings.width = width;
-	fh->overlay_settings.height = height;
-	fh->overlay_settings.clipcount = clipcount;
-
-	/*
-	 * If an overlay is running, we have to switch it off
-	 * and switch it on again in order to get the new settings in effect.
-	 *
-	 * We also want to avoid that the overlay mask is written
-	 * when an overlay is running.
-	 */
-
-	on = zr->v4l_overlay_active && !zr->v4l_memgrab_active &&
-	    zr->overlay_active != ZORAN_FREE &&
-	    fh->overlay_active != ZORAN_FREE;
-	if (on)
-		zr36057_overlay(zr, 0);
-
-	/*
-	 *   Write the overlay mask if clips are wanted.
-	 *   We prefer a bitmap.
-	 */
-	if (bitmap) {
-		/* fake value - it just means we want clips */
-		fh->overlay_settings.clipcount = 1;
-
-		if (copy_from_user(fh->overlay_mask, bitmap,
-				   (width * height + 7) / 8)) {
-			return -EFAULT;
-		}
-	} else if (clipcount) {
-		/* write our own bitmap from the clips */
-		vcp = vmalloc(sizeof(struct v4l2_clip) * (clipcount + 4));
-		if (vcp == NULL) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - Alloc of clip mask failed\n",
-				ZR_DEVNAME(zr), __func__);
-			return -ENOMEM;
-		}
-		if (copy_from_user
-		    (vcp, clips, sizeof(struct v4l2_clip) * clipcount)) {
-			vfree(vcp);
-			return -EFAULT;
-		}
-		write_overlay_mask(fh, vcp, clipcount);
-		vfree(vcp);
-	}
-
-	fh->overlay_settings.is_set = 1;
-	if (fh->overlay_active != ZORAN_FREE &&
-	    zr->overlay_active != ZORAN_FREE)
-		zr->overlay_settings = fh->overlay_settings;
-
-	if (on)
-		zr36057_overlay(zr, 1);
-
-	/* Make sure the changes come into effect */
-	return wait_grab_pending(zr);
-}
-
-static int setup_overlay(struct zoran_fh *fh, int on)
-{
-	struct zoran *zr = fh->zr;
-
-	/* If there is nothing to do, return immediately */
-	if ((on && fh->overlay_active != ZORAN_FREE) ||
-	    (!on && fh->overlay_active == ZORAN_FREE))
-		return 0;
-
-	/* check whether we're touching someone else's overlay */
-	if (on && zr->overlay_active != ZORAN_FREE &&
-	    fh->overlay_active == ZORAN_FREE) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - overlay is already active for another session\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EBUSY;
-	}
-	if (!on && zr->overlay_active != ZORAN_FREE &&
-	    fh->overlay_active == ZORAN_FREE) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - you cannot cancel someone else's session\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EPERM;
-	}
-
-	if (on == 0) {
-		zr->overlay_active = fh->overlay_active = ZORAN_FREE;
-		zr->v4l_overlay_active = 0;
-		/* When a grab is running, the video simply
-		 * won't be switched on any more */
-		if (!zr->v4l_memgrab_active)
-			zr36057_overlay(zr, 0);
-		zr->overlay_mask = NULL;
-	} else {
-		if (!zr->vbuf_base || !fh->overlay_settings.is_set) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - buffer or window not set\n",
-				ZR_DEVNAME(zr), __func__);
-			return -EINVAL;
-		}
-		if (!fh->overlay_settings.format) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - no overlay format set\n",
-				ZR_DEVNAME(zr), __func__);
-			return -EINVAL;
-		}
-		zr->overlay_active = fh->overlay_active = ZORAN_LOCKED;
-		zr->v4l_overlay_active = 1;
-		zr->overlay_mask = fh->overlay_mask;
-		zr->overlay_settings = fh->overlay_settings;
-		if (!zr->v4l_memgrab_active)
-			zr36057_overlay(zr, 1);
-		/* When a grab is running, the video will be
-		 * switched on when grab is finished */
-	}
-
-	/* Make sure the changes come into effect */
-	return wait_grab_pending(zr);
-}
-
-/* get the status of a buffer in the clients buffer queue */
-static int zoran_v4l2_buffer_status(struct zoran_fh *fh,
-				    struct v4l2_buffer *buf, int num)
-{
-	struct zoran *zr = fh->zr;
-	unsigned long flags;
-
-	buf->flags = V4L2_BUF_FLAG_MAPPED | V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:
-		/* check range */
-		if (num < 0 || num >= fh->buffers.num_buffers ||
-		    !fh->buffers.allocated) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - wrong number or buffers not allocated\n",
-				ZR_DEVNAME(zr), __func__);
-			return -EINVAL;
-		}
-
-		spin_lock_irqsave(&zr->spinlock, flags);
-		dprintk(3,
-			KERN_DEBUG
-			"%s: %s() - raw active=%c, buffer %d: state=%c, map=%c\n",
-			ZR_DEVNAME(zr), __func__,
-			"FAL"[fh->buffers.active], num,
-			"UPMD"[zr->v4l_buffers.buffer[num].state],
-			fh->buffers.buffer[num].map ? 'Y' : 'N');
-		spin_unlock_irqrestore(&zr->spinlock, flags);
-
-		buf->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-		buf->length = fh->buffers.buffer_size;
-
-		/* get buffer */
-		buf->bytesused = fh->buffers.buffer[num].bs.length;
-		if (fh->buffers.buffer[num].state == BUZ_STATE_DONE ||
-		    fh->buffers.buffer[num].state == BUZ_STATE_USER) {
-			buf->sequence = fh->buffers.buffer[num].bs.seq;
-			buf->flags |= V4L2_BUF_FLAG_DONE;
-			buf->timestamp = fh->buffers.buffer[num].bs.timestamp;
-		} else {
-			buf->flags |= V4L2_BUF_FLAG_QUEUED;
-		}
-
-		if (fh->v4l_settings.height <= BUZ_MAX_HEIGHT / 2)
-			buf->field = V4L2_FIELD_TOP;
-		else
-			buf->field = V4L2_FIELD_INTERLACED;
-
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-
-		/* check range */
-		if (num < 0 || num >= fh->buffers.num_buffers ||
-		    !fh->buffers.allocated) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - wrong number or buffers not allocated\n",
-				ZR_DEVNAME(zr), __func__);
-			return -EINVAL;
-		}
-
-		buf->type = (fh->map_mode == ZORAN_MAP_MODE_JPG_REC) ?
-			      V4L2_BUF_TYPE_VIDEO_CAPTURE :
-			      V4L2_BUF_TYPE_VIDEO_OUTPUT;
-		buf->length = fh->buffers.buffer_size;
-
-		/* these variables are only written after frame has been captured */
-		if (fh->buffers.buffer[num].state == BUZ_STATE_DONE ||
-		    fh->buffers.buffer[num].state == BUZ_STATE_USER) {
-			buf->sequence = fh->buffers.buffer[num].bs.seq;
-			buf->timestamp = fh->buffers.buffer[num].bs.timestamp;
-			buf->bytesused = fh->buffers.buffer[num].bs.length;
-			buf->flags |= V4L2_BUF_FLAG_DONE;
-		} else {
-			buf->flags |= V4L2_BUF_FLAG_QUEUED;
-		}
-
-		/* which fields are these? */
-		if (fh->jpg_settings.TmpDcm != 1)
-			buf->field = fh->jpg_settings.odd_even ?
-				V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM;
-		else
-			buf->field = fh->jpg_settings.odd_even ?
-				V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT;
-
-		break;
-
-	default:
-
-		dprintk(5,
-			KERN_ERR
-			"%s: %s - invalid buffer type|map_mode (%d|%d)\n",
-			ZR_DEVNAME(zr), __func__, buf->type, fh->map_mode);
-		return -EINVAL;
-	}
-
-	buf->memory = V4L2_MEMORY_MMAP;
-	buf->index = num;
-	buf->m.offset = buf->length * num;
-
-	return 0;
-}
-
-static int
-zoran_set_norm (struct zoran *zr,
-		v4l2_std_id norm)
-{
-	int on;
-
-	if (zr->v4l_buffers.active != ZORAN_FREE ||
-	    zr->jpg_buffers.active != ZORAN_FREE) {
-		dprintk(1,
-			KERN_WARNING
-			"%s: %s called while in playback/capture mode\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EBUSY;
-	}
-
-	if (!(norm & zr->card.norms)) {
-		dprintk(1,
-			KERN_ERR "%s: %s - unsupported norm %llx\n",
-			ZR_DEVNAME(zr), __func__, norm);
-		return -EINVAL;
-	}
-
-	if (norm & V4L2_STD_SECAM)
-		zr->timing = zr->card.tvn[2];
-	else if (norm & V4L2_STD_NTSC)
-		zr->timing = zr->card.tvn[1];
-	else
-		zr->timing = zr->card.tvn[0];
-
-	/* We switch overlay off and on since a change in the
-	 * norm needs different VFE settings */
-	on = zr->overlay_active && !zr->v4l_memgrab_active;
-	if (on)
-		zr36057_overlay(zr, 0);
-
-	decoder_call(zr, video, s_std, norm);
-	encoder_call(zr, video, s_std_output, norm);
-
-	if (on)
-		zr36057_overlay(zr, 1);
-
-	/* Make sure the changes come into effect */
-	zr->norm = norm;
-
-	return 0;
-}
-
-static int
-zoran_set_input (struct zoran *zr,
-		 int           input)
-{
-	if (input == zr->input) {
-		return 0;
-	}
-
-	if (zr->v4l_buffers.active != ZORAN_FREE ||
-	    zr->jpg_buffers.active != ZORAN_FREE) {
-		dprintk(1,
-			KERN_WARNING
-			"%s: %s called while in playback/capture mode\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EBUSY;
-	}
-
-	if (input < 0 || input >= zr->card.inputs) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - unsupported input %d\n",
-			ZR_DEVNAME(zr), __func__, input);
-		return -EINVAL;
-	}
-
-	zr->input = input;
-
-	decoder_call(zr, video, s_routing,
-			zr->card.input[input].muxsel, 0, 0);
-
-	return 0;
-}
-
-/*
- *   ioctl routine
- */
-
-static int zoran_querycap(struct file *file, void *__fh, struct v4l2_capability *cap)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	strncpy(cap->card, ZR_DEVNAME(zr), sizeof(cap->card)-1);
-	strncpy(cap->driver, "zoran", sizeof(cap->driver)-1);
-	snprintf(cap->bus_info, sizeof(cap->bus_info), "PCI:%s",
-		 pci_name(zr->pci_dev));
-	cap->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE |
-			   V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_VIDEO_OVERLAY;
-	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
-	return 0;
-}
-
-static int zoran_enum_fmt(struct zoran *zr, struct v4l2_fmtdesc *fmt, int flag)
-{
-	unsigned int num, i;
-
-	for (num = i = 0; i < NUM_FORMATS; i++) {
-		if (zoran_formats[i].flags & flag && num++ == fmt->index) {
-			strncpy(fmt->description, zoran_formats[i].name,
-				sizeof(fmt->description) - 1);
-			/* fmt struct pre-zeroed, so adding '\0' not needed */
-			fmt->pixelformat = zoran_formats[i].fourcc;
-			if (zoran_formats[i].flags & ZORAN_FORMAT_COMPRESSED)
-				fmt->flags |= V4L2_FMT_FLAG_COMPRESSED;
-			return 0;
-		}
-	}
-	return -EINVAL;
-}
-
-static int zoran_enum_fmt_vid_cap(struct file *file, void *__fh,
-					    struct v4l2_fmtdesc *f)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	return zoran_enum_fmt(zr, f, ZORAN_FORMAT_CAPTURE);
-}
-
-static int zoran_enum_fmt_vid_out(struct file *file, void *__fh,
-					    struct v4l2_fmtdesc *f)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	return zoran_enum_fmt(zr, f, ZORAN_FORMAT_PLAYBACK);
-}
-
-static int zoran_enum_fmt_vid_overlay(struct file *file, void *__fh,
-					    struct v4l2_fmtdesc *f)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	return zoran_enum_fmt(zr, f, ZORAN_FORMAT_OVERLAY);
-}
-
-static int zoran_g_fmt_vid_out(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-
-	fmt->fmt.pix.width = fh->jpg_settings.img_width / fh->jpg_settings.HorDcm;
-	fmt->fmt.pix.height = fh->jpg_settings.img_height * 2 /
-		(fh->jpg_settings.VerDcm * fh->jpg_settings.TmpDcm);
-	fmt->fmt.pix.sizeimage = zoran_v4l2_calc_bufsize(&fh->jpg_settings);
-	fmt->fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG;
-	if (fh->jpg_settings.TmpDcm == 1)
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT);
-	else
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM);
-	fmt->fmt.pix.bytesperline = 0;
-	fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
-
-	return 0;
-}
-
-static int zoran_g_fmt_vid_cap(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	if (fh->map_mode != ZORAN_MAP_MODE_RAW)
-		return zoran_g_fmt_vid_out(file, fh, fmt);
-
-	fmt->fmt.pix.width = fh->v4l_settings.width;
-	fmt->fmt.pix.height = fh->v4l_settings.height;
-	fmt->fmt.pix.sizeimage = fh->v4l_settings.bytesperline *
-					fh->v4l_settings.height;
-	fmt->fmt.pix.pixelformat = fh->v4l_settings.format->fourcc;
-	fmt->fmt.pix.colorspace = fh->v4l_settings.format->colorspace;
-	fmt->fmt.pix.bytesperline = fh->v4l_settings.bytesperline;
-	if (BUZ_MAX_HEIGHT < (fh->v4l_settings.height * 2))
-		fmt->fmt.pix.field = V4L2_FIELD_INTERLACED;
-	else
-		fmt->fmt.pix.field = V4L2_FIELD_TOP;
-	return 0;
-}
-
-static int zoran_g_fmt_vid_overlay(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	fmt->fmt.win.w.left = fh->overlay_settings.x;
-	fmt->fmt.win.w.top = fh->overlay_settings.y;
-	fmt->fmt.win.w.width = fh->overlay_settings.width;
-	fmt->fmt.win.w.height = fh->overlay_settings.height;
-	if (fh->overlay_settings.width * 2 > BUZ_MAX_HEIGHT)
-		fmt->fmt.win.field = V4L2_FIELD_INTERLACED;
-	else
-		fmt->fmt.win.field = V4L2_FIELD_TOP;
-
-	return 0;
-}
-
-static int zoran_try_fmt_vid_overlay(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	if (fmt->fmt.win.w.width > BUZ_MAX_WIDTH)
-		fmt->fmt.win.w.width = BUZ_MAX_WIDTH;
-	if (fmt->fmt.win.w.width < BUZ_MIN_WIDTH)
-		fmt->fmt.win.w.width = BUZ_MIN_WIDTH;
-	if (fmt->fmt.win.w.height > BUZ_MAX_HEIGHT)
-		fmt->fmt.win.w.height = BUZ_MAX_HEIGHT;
-	if (fmt->fmt.win.w.height < BUZ_MIN_HEIGHT)
-		fmt->fmt.win.w.height = BUZ_MIN_HEIGHT;
-
-	return 0;
-}
-
-static int zoran_try_fmt_vid_out(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	struct zoran_jpg_settings settings;
-	int res = 0;
-
-	if (fmt->fmt.pix.pixelformat != V4L2_PIX_FMT_MJPEG)
-		return -EINVAL;
-
-	settings = fh->jpg_settings;
-
-	/* we actually need to set 'real' parameters now */
-	if ((fmt->fmt.pix.height * 2) > BUZ_MAX_HEIGHT)
-		settings.TmpDcm = 1;
-	else
-		settings.TmpDcm = 2;
-	settings.decimation = 0;
-	if (fmt->fmt.pix.height <= fh->jpg_settings.img_height / 2)
-		settings.VerDcm = 2;
-	else
-		settings.VerDcm = 1;
-	if (fmt->fmt.pix.width <= fh->jpg_settings.img_width / 4)
-		settings.HorDcm = 4;
-	else if (fmt->fmt.pix.width <= fh->jpg_settings.img_width / 2)
-		settings.HorDcm = 2;
-	else
-		settings.HorDcm = 1;
-	if (settings.TmpDcm == 1)
-		settings.field_per_buff = 2;
-	else
-		settings.field_per_buff = 1;
-
-	if (settings.HorDcm > 1) {
-		settings.img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
-		settings.img_width = (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
-	} else {
-		settings.img_x = 0;
-		settings.img_width = BUZ_MAX_WIDTH;
-	}
-
-	/* check */
-	res = zoran_check_jpg_settings(zr, &settings, 1);
-	if (res)
-		return res;
-
-	/* tell the user what we actually did */
-	fmt->fmt.pix.width = settings.img_width / settings.HorDcm;
-	fmt->fmt.pix.height = settings.img_height * 2 /
-		(settings.TmpDcm * settings.VerDcm);
-	if (settings.TmpDcm == 1)
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT);
-	else
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM);
-
-	fmt->fmt.pix.sizeimage = zoran_v4l2_calc_bufsize(&settings);
-	fmt->fmt.pix.bytesperline = 0;
-	fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
-	return res;
-}
-
-static int zoran_try_fmt_vid_cap(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int bpp;
-	int i;
-
-	if (fmt->fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG)
-		return zoran_try_fmt_vid_out(file, fh, fmt);
-
-	for (i = 0; i < NUM_FORMATS; i++)
-		if (zoran_formats[i].fourcc == fmt->fmt.pix.pixelformat)
-			break;
-
-	if (i == NUM_FORMATS)
-		return -EINVAL;
-
-	bpp = DIV_ROUND_UP(zoran_formats[i].depth, 8);
-	v4l_bound_align_image(
-		&fmt->fmt.pix.width, BUZ_MIN_WIDTH, BUZ_MAX_WIDTH, bpp == 2 ? 1 : 2,
-		&fmt->fmt.pix.height, BUZ_MIN_HEIGHT, BUZ_MAX_HEIGHT, 0, 0);
-	return 0;
-}
-
-static int zoran_s_fmt_vid_overlay(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	int res;
-
-	dprintk(3, "x=%d, y=%d, w=%d, h=%d, cnt=%d, map=0x%p\n",
-			fmt->fmt.win.w.left, fmt->fmt.win.w.top,
-			fmt->fmt.win.w.width,
-			fmt->fmt.win.w.height,
-			fmt->fmt.win.clipcount,
-			fmt->fmt.win.bitmap);
-	res = setup_window(fh, fmt->fmt.win.w.left, fmt->fmt.win.w.top,
-			   fmt->fmt.win.w.width, fmt->fmt.win.w.height,
-			   (struct v4l2_clip __user *)fmt->fmt.win.clips,
-			   fmt->fmt.win.clipcount, fmt->fmt.win.bitmap);
-	return res;
-}
-
-static int zoran_s_fmt_vid_out(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	__le32 printformat = __cpu_to_le32(fmt->fmt.pix.pixelformat);
-	struct zoran_jpg_settings settings;
-	int res = 0;
-
-	dprintk(3, "size=%dx%d, fmt=0x%x (%4.4s)\n",
-			fmt->fmt.pix.width, fmt->fmt.pix.height,
-			fmt->fmt.pix.pixelformat,
-			(char *) &printformat);
-	if (fmt->fmt.pix.pixelformat != V4L2_PIX_FMT_MJPEG)
-		return -EINVAL;
-
-	if (fh->buffers.allocated) {
-		dprintk(1, KERN_ERR "%s: VIDIOC_S_FMT - cannot change capture mode\n",
-			ZR_DEVNAME(zr));
-		res = -EBUSY;
-		return res;
-	}
-
-	settings = fh->jpg_settings;
-
-	/* we actually need to set 'real' parameters now */
-	if (fmt->fmt.pix.height * 2 > BUZ_MAX_HEIGHT)
-		settings.TmpDcm = 1;
-	else
-		settings.TmpDcm = 2;
-	settings.decimation = 0;
-	if (fmt->fmt.pix.height <= fh->jpg_settings.img_height / 2)
-		settings.VerDcm = 2;
-	else
-		settings.VerDcm = 1;
-	if (fmt->fmt.pix.width <= fh->jpg_settings.img_width / 4)
-		settings.HorDcm = 4;
-	else if (fmt->fmt.pix.width <= fh->jpg_settings.img_width / 2)
-		settings.HorDcm = 2;
-	else
-		settings.HorDcm = 1;
-	if (settings.TmpDcm == 1)
-		settings.field_per_buff = 2;
-	else
-		settings.field_per_buff = 1;
-
-	if (settings.HorDcm > 1) {
-		settings.img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
-		settings.img_width = (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
-	} else {
-		settings.img_x = 0;
-		settings.img_width = BUZ_MAX_WIDTH;
-	}
-
-	/* check */
-	res = zoran_check_jpg_settings(zr, &settings, 0);
-	if (res)
-		return res;
-
-	/* it's ok, so set them */
-	fh->jpg_settings = settings;
-
-	map_mode_jpg(fh, fmt->type == V4L2_BUF_TYPE_VIDEO_OUTPUT);
-	fh->buffers.buffer_size = zoran_v4l2_calc_bufsize(&fh->jpg_settings);
-
-	/* tell the user what we actually did */
-	fmt->fmt.pix.width = settings.img_width / settings.HorDcm;
-	fmt->fmt.pix.height = settings.img_height * 2 /
-		(settings.TmpDcm * settings.VerDcm);
-	if (settings.TmpDcm == 1)
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT);
-	else
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM);
-	fmt->fmt.pix.bytesperline = 0;
-	fmt->fmt.pix.sizeimage = fh->buffers.buffer_size;
-	fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
-	return res;
-}
-
-static int zoran_s_fmt_vid_cap(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int i;
-	int res = 0;
-
-	if (fmt->fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG)
-		return zoran_s_fmt_vid_out(file, fh, fmt);
-
-	for (i = 0; i < NUM_FORMATS; i++)
-		if (fmt->fmt.pix.pixelformat == zoran_formats[i].fourcc)
-			break;
-	if (i == NUM_FORMATS) {
-		dprintk(1, KERN_ERR "%s: VIDIOC_S_FMT - unknown/unsupported format 0x%x\n",
-			ZR_DEVNAME(zr), fmt->fmt.pix.pixelformat);
-		return -EINVAL;
-	}
-
-	if ((fh->map_mode != ZORAN_MAP_MODE_RAW && fh->buffers.allocated) ||
-	    fh->buffers.active != ZORAN_FREE) {
-		dprintk(1, KERN_ERR "%s: VIDIOC_S_FMT - cannot change capture mode\n",
-				ZR_DEVNAME(zr));
-		res = -EBUSY;
-		return res;
-	}
-	if (fmt->fmt.pix.height > BUZ_MAX_HEIGHT)
-		fmt->fmt.pix.height = BUZ_MAX_HEIGHT;
-	if (fmt->fmt.pix.width > BUZ_MAX_WIDTH)
-		fmt->fmt.pix.width = BUZ_MAX_WIDTH;
-
-	map_mode_raw(fh);
-
-	res = zoran_v4l_set_format(fh, fmt->fmt.pix.width, fmt->fmt.pix.height,
-				   &zoran_formats[i]);
-	if (res)
-		return res;
-
-	/* tell the user the results/missing stuff */
-	fmt->fmt.pix.bytesperline = fh->v4l_settings.bytesperline;
-	fmt->fmt.pix.sizeimage = fh->v4l_settings.height * fh->v4l_settings.bytesperline;
-	fmt->fmt.pix.colorspace = fh->v4l_settings.format->colorspace;
-	if (BUZ_MAX_HEIGHT < (fh->v4l_settings.height * 2))
-		fmt->fmt.pix.field = V4L2_FIELD_INTERLACED;
-	else
-		fmt->fmt.pix.field = V4L2_FIELD_TOP;
-	return res;
-}
-
-static int zoran_g_fbuf(struct file *file, void *__fh,
-		struct v4l2_framebuffer *fb)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	memset(fb, 0, sizeof(*fb));
-	fb->base = zr->vbuf_base;
-	fb->fmt.width = zr->vbuf_width;
-	fb->fmt.height = zr->vbuf_height;
-	if (zr->overlay_settings.format)
-		fb->fmt.pixelformat = fh->overlay_settings.format->fourcc;
-	fb->fmt.bytesperline = zr->vbuf_bytesperline;
-	fb->fmt.colorspace = V4L2_COLORSPACE_SRGB;
-	fb->fmt.field = V4L2_FIELD_INTERLACED;
-	fb->capability = V4L2_FBUF_CAP_LIST_CLIPPING;
-
-	return 0;
-}
-
-static int zoran_s_fbuf(struct file *file, void *__fh,
-		const struct v4l2_framebuffer *fb)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int i, res = 0;
-	__le32 printformat = __cpu_to_le32(fb->fmt.pixelformat);
-
-	for (i = 0; i < NUM_FORMATS; i++)
-		if (zoran_formats[i].fourcc == fb->fmt.pixelformat)
-			break;
-	if (i == NUM_FORMATS) {
-		dprintk(1, KERN_ERR "%s: VIDIOC_S_FBUF - format=0x%x (%4.4s) not allowed\n",
-			ZR_DEVNAME(zr), fb->fmt.pixelformat,
-			(char *)&printformat);
-		return -EINVAL;
-	}
-
-	res = setup_fbuffer(fh, fb->base, &zoran_formats[i], fb->fmt.width,
-			    fb->fmt.height, fb->fmt.bytesperline);
-
-	return res;
-}
-
-static int zoran_overlay(struct file *file, void *__fh, unsigned int on)
-{
-	struct zoran_fh *fh = __fh;
-	int res;
-
-	res = setup_overlay(fh, on);
-
-	return res;
-}
-
-static int zoran_streamoff(struct file *file, void *__fh, enum v4l2_buf_type type);
-
-static int zoran_reqbufs(struct file *file, void *__fh, struct v4l2_requestbuffers *req)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0;
-
-	if (req->memory != V4L2_MEMORY_MMAP) {
-		dprintk(2,
-				KERN_ERR
-				"%s: only MEMORY_MMAP capture is supported, not %d\n",
-				ZR_DEVNAME(zr), req->memory);
-		return -EINVAL;
-	}
-
-	if (req->count == 0)
-		return zoran_streamoff(file, fh, req->type);
-
-	if (fh->buffers.allocated) {
-		dprintk(2,
-				KERN_ERR
-				"%s: VIDIOC_REQBUFS - buffers already allocated\n",
-				ZR_DEVNAME(zr));
-		res = -EBUSY;
-		return res;
-	}
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW &&
-	    req->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) {
-		/* control user input */
-		if (req->count < 2)
-			req->count = 2;
-		if (req->count > v4l_nbufs)
-			req->count = v4l_nbufs;
-
-		/* The next mmap will map the V4L buffers */
-		map_mode_raw(fh);
-		fh->buffers.num_buffers = req->count;
-
-		if (v4l_fbuffer_alloc(fh)) {
-			res = -ENOMEM;
-			return res;
-		}
-	} else if (fh->map_mode == ZORAN_MAP_MODE_JPG_REC ||
-		   fh->map_mode == ZORAN_MAP_MODE_JPG_PLAY) {
-		/* we need to calculate size ourselves now */
-		if (req->count < 4)
-			req->count = 4;
-		if (req->count > jpg_nbufs)
-			req->count = jpg_nbufs;
-
-		/* The next mmap will map the MJPEG buffers */
-		map_mode_jpg(fh, req->type == V4L2_BUF_TYPE_VIDEO_OUTPUT);
-		fh->buffers.num_buffers = req->count;
-		fh->buffers.buffer_size = zoran_v4l2_calc_bufsize(&fh->jpg_settings);
-
-		if (jpg_fbuffer_alloc(fh)) {
-			res = -ENOMEM;
-			return res;
-		}
-	} else {
-		dprintk(1,
-				KERN_ERR
-				"%s: VIDIOC_REQBUFS - unknown type %d\n",
-				ZR_DEVNAME(zr), req->type);
-		res = -EINVAL;
-		return res;
-	}
-	return res;
-}
-
-static int zoran_querybuf(struct file *file, void *__fh, struct v4l2_buffer *buf)
-{
-	struct zoran_fh *fh = __fh;
-	int res;
-
-	res = zoran_v4l2_buffer_status(fh, buf, buf->index);
-
-	return res;
-}
-
-static int zoran_qbuf(struct file *file, void *__fh, struct v4l2_buffer *buf)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0, codec_mode, buf_type;
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:
-		if (buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
-			dprintk(1, KERN_ERR
-				"%s: VIDIOC_QBUF - invalid buf->type=%d for map_mode=%d\n",
-				ZR_DEVNAME(zr), buf->type, fh->map_mode);
-			res = -EINVAL;
-			return res;
-		}
-
-		res = zoran_v4l_queue_frame(fh, buf->index);
-		if (res)
-			return res;
-		if (!zr->v4l_memgrab_active && fh->buffers.active == ZORAN_LOCKED)
-			zr36057_set_memgrab(zr, 1);
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-		if (fh->map_mode == ZORAN_MAP_MODE_JPG_PLAY) {
-			buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
-			codec_mode = BUZ_MODE_MOTION_DECOMPRESS;
-		} else {
-			buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-			codec_mode = BUZ_MODE_MOTION_COMPRESS;
-		}
-
-		if (buf->type != buf_type) {
-			dprintk(1, KERN_ERR
-				"%s: VIDIOC_QBUF - invalid buf->type=%d for map_mode=%d\n",
-				ZR_DEVNAME(zr), buf->type, fh->map_mode);
-			res = -EINVAL;
-			return res;
-		}
-
-		res = zoran_jpg_queue_frame(fh, buf->index, codec_mode);
-		if (res != 0)
-			return res;
-		if (zr->codec_mode == BUZ_MODE_IDLE &&
-		    fh->buffers.active == ZORAN_LOCKED)
-			zr36057_enable_jpg(zr, codec_mode);
-
-		break;
-
-	default:
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_QBUF - unsupported type %d\n",
-			ZR_DEVNAME(zr), buf->type);
-		res = -EINVAL;
-		break;
-	}
-	return res;
-}
-
-static int zoran_dqbuf(struct file *file, void *__fh, struct v4l2_buffer *buf)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0, buf_type, num = -1;	/* compiler borks here (?) */
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:
-		if (buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
-			dprintk(1, KERN_ERR
-				"%s: VIDIOC_QBUF - invalid buf->type=%d for map_mode=%d\n",
-				ZR_DEVNAME(zr), buf->type, fh->map_mode);
-			res = -EINVAL;
-			return res;
-		}
-
-		num = zr->v4l_pend[zr->v4l_sync_tail & V4L_MASK_FRAME];
-		if (file->f_flags & O_NONBLOCK &&
-		    zr->v4l_buffers.buffer[num].state != BUZ_STATE_DONE) {
-			res = -EAGAIN;
-			return res;
-		}
-		res = v4l_sync(fh, num);
-		if (res)
-			return res;
-		zr->v4l_sync_tail++;
-		res = zoran_v4l2_buffer_status(fh, buf, num);
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-	{
-		struct zoran_sync bs;
-
-		if (fh->map_mode == ZORAN_MAP_MODE_JPG_PLAY)
-			buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
-		else
-			buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-		if (buf->type != buf_type) {
-			dprintk(1, KERN_ERR
-				"%s: VIDIOC_QBUF - invalid buf->type=%d for map_mode=%d\n",
-				ZR_DEVNAME(zr), buf->type, fh->map_mode);
-			res = -EINVAL;
-			return res;
-		}
-
-		num = zr->jpg_pend[zr->jpg_que_tail & BUZ_MASK_FRAME];
-
-		if (file->f_flags & O_NONBLOCK &&
-		    zr->jpg_buffers.buffer[num].state != BUZ_STATE_DONE) {
-			res = -EAGAIN;
-			return res;
-		}
-		bs.frame = 0; /* suppress compiler warning */
-		res = jpg_sync(fh, &bs);
-		if (res)
-			return res;
-		res = zoran_v4l2_buffer_status(fh, buf, bs.frame);
-		break;
-	}
-
-	default:
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_DQBUF - unsupported type %d\n",
-			ZR_DEVNAME(zr), buf->type);
-		res = -EINVAL;
-		break;
-	}
-	return res;
-}
-
-static int zoran_streamon(struct file *file, void *__fh, enum v4l2_buf_type type)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0;
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:	/* raw capture */
-		if (zr->v4l_buffers.active != ZORAN_ACTIVE ||
-		    fh->buffers.active != ZORAN_ACTIVE) {
-			res = -EBUSY;
-			return res;
-		}
-
-		zr->v4l_buffers.active = fh->buffers.active = ZORAN_LOCKED;
-		zr->v4l_settings = fh->v4l_settings;
-
-		zr->v4l_sync_tail = zr->v4l_pend_tail;
-		if (!zr->v4l_memgrab_active &&
-		    zr->v4l_pend_head != zr->v4l_pend_tail) {
-			zr36057_set_memgrab(zr, 1);
-		}
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-		/* what is the codec mode right now? */
-		if (zr->jpg_buffers.active != ZORAN_ACTIVE ||
-		    fh->buffers.active != ZORAN_ACTIVE) {
-			res = -EBUSY;
-			return res;
-		}
-
-		zr->jpg_buffers.active = fh->buffers.active = ZORAN_LOCKED;
-
-		if (zr->jpg_que_head != zr->jpg_que_tail) {
-			/* Start the jpeg codec when the first frame is queued  */
-			jpeg_start(zr);
-		}
-		break;
-
-	default:
-		dprintk(1,
-			KERN_ERR
-			"%s: VIDIOC_STREAMON - invalid map mode %d\n",
-			ZR_DEVNAME(zr), fh->map_mode);
-		res = -EINVAL;
-		break;
-	}
-	return res;
-}
-
-static int zoran_streamoff(struct file *file, void *__fh, enum v4l2_buf_type type)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int i, res = 0;
-	unsigned long flags;
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:	/* raw capture */
-		if (fh->buffers.active == ZORAN_FREE &&
-		    zr->v4l_buffers.active != ZORAN_FREE) {
-			res = -EPERM;	/* stay off other's settings! */
-			return res;
-		}
-		if (zr->v4l_buffers.active == ZORAN_FREE)
-			return res;
-
-		spin_lock_irqsave(&zr->spinlock, flags);
-		/* unload capture */
-		if (zr->v4l_memgrab_active) {
-
-			zr36057_set_memgrab(zr, 0);
-		}
-
-		for (i = 0; i < fh->buffers.num_buffers; i++)
-			zr->v4l_buffers.buffer[i].state = BUZ_STATE_USER;
-		fh->buffers = zr->v4l_buffers;
-
-		zr->v4l_buffers.active = fh->buffers.active = ZORAN_FREE;
-
-		zr->v4l_grab_seq = 0;
-		zr->v4l_pend_head = zr->v4l_pend_tail = 0;
-		zr->v4l_sync_tail = 0;
-
-		spin_unlock_irqrestore(&zr->spinlock, flags);
-
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-		if (fh->buffers.active == ZORAN_FREE &&
-		    zr->jpg_buffers.active != ZORAN_FREE) {
-			res = -EPERM;	/* stay off other's settings! */
-			return res;
-		}
-		if (zr->jpg_buffers.active == ZORAN_FREE)
-			return res;
-
-		res = jpg_qbuf(fh, -1,
-			     (fh->map_mode == ZORAN_MAP_MODE_JPG_REC) ?
-			     BUZ_MODE_MOTION_COMPRESS :
-			     BUZ_MODE_MOTION_DECOMPRESS);
-		if (res)
-			return res;
-		break;
-	default:
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_STREAMOFF - invalid map mode %d\n",
-			ZR_DEVNAME(zr), fh->map_mode);
-		res = -EINVAL;
-		break;
-	}
-	return res;
-}
-static int zoran_g_std(struct file *file, void *__fh, v4l2_std_id *std)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	*std = zr->norm;
-	return 0;
-}
-
-static int zoran_s_std(struct file *file, void *__fh, v4l2_std_id std)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0;
-
-	res = zoran_set_norm(zr, std);
-	if (res)
-		return res;
-
-	res = wait_grab_pending(zr);
-	return res;
-}
-
-static int zoran_enum_input(struct file *file, void *__fh,
-				 struct v4l2_input *inp)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	if (inp->index >= zr->card.inputs)
-		return -EINVAL;
-
-	strncpy(inp->name, zr->card.input[inp->index].name,
-		sizeof(inp->name) - 1);
-	inp->type = V4L2_INPUT_TYPE_CAMERA;
-	inp->std = V4L2_STD_ALL;
-
-	/* Get status of video decoder */
-	decoder_call(zr, video, g_input_status, &inp->status);
-	return 0;
-}
-
-static int zoran_g_input(struct file *file, void *__fh, unsigned int *input)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	*input = zr->input;
-
-	return 0;
-}
-
-static int zoran_s_input(struct file *file, void *__fh, unsigned int input)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res;
-
-	res = zoran_set_input(zr, input);
-	if (res)
-		return res;
-
-	/* Make sure the changes come into effect */
-	res = wait_grab_pending(zr);
-	return res;
-}
-
-static int zoran_enum_output(struct file *file, void *__fh,
-				  struct v4l2_output *outp)
-{
-	if (outp->index != 0)
-		return -EINVAL;
-
-	outp->index = 0;
-	outp->type = V4L2_OUTPUT_TYPE_ANALOGVGAOVERLAY;
-	strncpy(outp->name, "Autodetect", sizeof(outp->name)-1);
-
-	return 0;
-}
-
-static int zoran_g_output(struct file *file, void *__fh, unsigned int *output)
-{
-	*output = 0;
-
-	return 0;
-}
-
-static int zoran_s_output(struct file *file, void *__fh, unsigned int output)
-{
-	if (output != 0)
-		return -EINVAL;
-
-	return 0;
-}
-
-/* cropping (sub-frame capture) */
-static int zoran_g_selection(struct file *file, void *__fh, struct v4l2_selection *sel)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
-	    sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return -EINVAL;
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_G_SELECTION - subcapture only supported for compressed capture\n",
-			ZR_DEVNAME(zr));
-		return -EINVAL;
-	}
-
-	switch (sel->target) {
-	case V4L2_SEL_TGT_CROP:
-		sel->r.top = fh->jpg_settings.img_y;
-		sel->r.left = fh->jpg_settings.img_x;
-		sel->r.width = fh->jpg_settings.img_width;
-		sel->r.height = fh->jpg_settings.img_height;
-		break;
-	case V4L2_SEL_TGT_CROP_DEFAULT:
-		sel->r.top = sel->r.left = 0;
-		sel->r.width = BUZ_MIN_WIDTH;
-		sel->r.height = BUZ_MIN_HEIGHT;
-		break;
-	case V4L2_SEL_TGT_CROP_BOUNDS:
-		sel->r.top = sel->r.left = 0;
-		sel->r.width = BUZ_MAX_WIDTH;
-		sel->r.height = BUZ_MAX_HEIGHT;
-		break;
-	default:
-		return -EINVAL;
-	}
-	return 0;
-}
-
-static int zoran_s_selection(struct file *file, void *__fh, struct v4l2_selection *sel)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	struct zoran_jpg_settings settings;
-	int res;
-
-	if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
-	    sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return -EINVAL;
-
-	if (sel->target != V4L2_SEL_TGT_CROP)
-		return -EINVAL;
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_S_SELECTION - subcapture only supported for compressed capture\n",
-			ZR_DEVNAME(zr));
-		return -EINVAL;
-	}
-
-	settings = fh->jpg_settings;
-
-	if (fh->buffers.allocated) {
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_S_SELECTION - cannot change settings while active\n",
-			ZR_DEVNAME(zr));
-		return -EBUSY;
-	}
-
-	/* move into a form that we understand */
-	settings.img_x = sel->r.left;
-	settings.img_y = sel->r.top;
-	settings.img_width = sel->r.width;
-	settings.img_height = sel->r.height;
-
-	/* check validity */
-	res = zoran_check_jpg_settings(zr, &settings, 0);
-	if (res)
-		return res;
-
-	/* accept */
-	fh->jpg_settings = settings;
-	return res;
-}
-
-static int zoran_g_jpegcomp(struct file *file, void *__fh,
-					struct v4l2_jpegcompression *params)
-{
-	struct zoran_fh *fh = __fh;
-	memset(params, 0, sizeof(*params));
-
-	params->quality = fh->jpg_settings.jpg_comp.quality;
-	params->APPn = fh->jpg_settings.jpg_comp.APPn;
-	memcpy(params->APP_data,
-	       fh->jpg_settings.jpg_comp.APP_data,
-	       fh->jpg_settings.jpg_comp.APP_len);
-	params->APP_len = fh->jpg_settings.jpg_comp.APP_len;
-	memcpy(params->COM_data,
-	       fh->jpg_settings.jpg_comp.COM_data,
-	       fh->jpg_settings.jpg_comp.COM_len);
-	params->COM_len = fh->jpg_settings.jpg_comp.COM_len;
-	params->jpeg_markers =
-	    fh->jpg_settings.jpg_comp.jpeg_markers;
-
-	return 0;
-}
-
-static int zoran_s_jpegcomp(struct file *file, void *__fh,
-					const struct v4l2_jpegcompression *params)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0;
-	struct zoran_jpg_settings settings;
-
-	settings = fh->jpg_settings;
-
-	settings.jpg_comp = *params;
-
-	if (fh->buffers.active != ZORAN_FREE) {
-		dprintk(1, KERN_WARNING
-			"%s: VIDIOC_S_JPEGCOMP called while in playback/capture mode\n",
-			ZR_DEVNAME(zr));
-		res = -EBUSY;
-		return res;
-	}
-
-	res = zoran_check_jpg_settings(zr, &settings, 0);
-	if (res)
-		return res;
-	if (!fh->buffers.allocated)
-		fh->buffers.buffer_size =
-			zoran_v4l2_calc_bufsize(&fh->jpg_settings);
-	fh->jpg_settings.jpg_comp = settings.jpg_comp;
-	return res;
-}
-
-static __poll_t
-zoran_poll (struct file *file,
-	    poll_table  *wait)
-{
-	struct zoran_fh *fh = file->private_data;
-	struct zoran *zr = fh->zr;
-	__poll_t res = v4l2_ctrl_poll(file, wait);
-	int frame;
-	unsigned long flags;
-
-	/* we should check whether buffers are ready to be synced on
-	 * (w/o waits - O_NONBLOCK) here
-	 * if ready for read (sync), return EPOLLIN|EPOLLRDNORM,
-	 * if ready for write (sync), return EPOLLOUT|EPOLLWRNORM,
-	 * if error, return EPOLLERR,
-	 * if no buffers queued or so, return EPOLLNVAL
-	 */
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:
-		poll_wait(file, &zr->v4l_capq, wait);
-		frame = zr->v4l_pend[zr->v4l_sync_tail & V4L_MASK_FRAME];
-
-		spin_lock_irqsave(&zr->spinlock, flags);
-		dprintk(3,
-			KERN_DEBUG
-			"%s: %s() raw - active=%c, sync_tail=%lu/%c, pend_tail=%lu, pend_head=%lu\n",
-			ZR_DEVNAME(zr), __func__,
-			"FAL"[fh->buffers.active], zr->v4l_sync_tail,
-			"UPMD"[zr->v4l_buffers.buffer[frame].state],
-			zr->v4l_pend_tail, zr->v4l_pend_head);
-		/* Process is the one capturing? */
-		if (fh->buffers.active != ZORAN_FREE &&
-		    /* Buffer ready to DQBUF? */
-		    zr->v4l_buffers.buffer[frame].state == BUZ_STATE_DONE)
-			res |= EPOLLIN | EPOLLRDNORM;
-		spin_unlock_irqrestore(&zr->spinlock, flags);
-
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-		poll_wait(file, &zr->jpg_capq, wait);
-		frame = zr->jpg_pend[zr->jpg_que_tail & BUZ_MASK_FRAME];
-
-		spin_lock_irqsave(&zr->spinlock, flags);
-		dprintk(3,
-			KERN_DEBUG
-			"%s: %s() jpg - active=%c, que_tail=%lu/%c, que_head=%lu, dma=%lu/%lu\n",
-			ZR_DEVNAME(zr), __func__,
-			"FAL"[fh->buffers.active], zr->jpg_que_tail,
-			"UPMD"[zr->jpg_buffers.buffer[frame].state],
-			zr->jpg_que_head, zr->jpg_dma_tail, zr->jpg_dma_head);
-		if (fh->buffers.active != ZORAN_FREE &&
-		    zr->jpg_buffers.buffer[frame].state == BUZ_STATE_DONE) {
-			if (fh->map_mode == ZORAN_MAP_MODE_JPG_REC)
-				res |= EPOLLIN | EPOLLRDNORM;
-			else
-				res |= EPOLLOUT | EPOLLWRNORM;
-		}
-		spin_unlock_irqrestore(&zr->spinlock, flags);
-
-		break;
-
-	default:
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - internal error, unknown map_mode=%d\n",
-			ZR_DEVNAME(zr), __func__, fh->map_mode);
-		res |= EPOLLERR;
-	}
-
-	return res;
-}
-
-
-/*
- * This maps the buffers to user space.
- *
- * Depending on the state of fh->map_mode
- * the V4L or the MJPEG buffers are mapped
- * per buffer or all together
- *
- * Note that we need to connect to some
- * unmap signal event to unmap the de-allocate
- * the buffer accordingly (zoran_vm_close())
- */
-
-static void
-zoran_vm_open (struct vm_area_struct *vma)
-{
-	struct zoran_mapping *map = vma->vm_private_data;
-	atomic_inc(&map->count);
-}
-
-static void
-zoran_vm_close (struct vm_area_struct *vma)
-{
-	struct zoran_mapping *map = vma->vm_private_data;
-	struct zoran_fh *fh = map->fh;
-	struct zoran *zr = fh->zr;
-	int i;
-
-	dprintk(3, KERN_INFO "%s: %s - munmap(%s)\n", ZR_DEVNAME(zr),
-		__func__, mode_name(fh->map_mode));
-
-	for (i = 0; i < fh->buffers.num_buffers; i++) {
-		if (fh->buffers.buffer[i].map == map)
-			fh->buffers.buffer[i].map = NULL;
-	}
-	kfree(map);
-
-	/* Any buffers still mapped? */
-	for (i = 0; i < fh->buffers.num_buffers; i++) {
-		if (fh->buffers.buffer[i].map) {
-			return;
-		}
-	}
-
-	dprintk(3, KERN_INFO "%s: %s - free %s buffers\n", ZR_DEVNAME(zr),
-		__func__, mode_name(fh->map_mode));
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
-		if (fh->buffers.active != ZORAN_FREE) {
-			unsigned long flags;
-
-			spin_lock_irqsave(&zr->spinlock, flags);
-			zr36057_set_memgrab(zr, 0);
-			zr->v4l_buffers.allocated = 0;
-			zr->v4l_buffers.active = fh->buffers.active = ZORAN_FREE;
-			spin_unlock_irqrestore(&zr->spinlock, flags);
-		}
-		v4l_fbuffer_free(fh);
-	} else {
-		if (fh->buffers.active != ZORAN_FREE) {
-			jpg_qbuf(fh, -1, zr->codec_mode);
-			zr->jpg_buffers.allocated = 0;
-			zr->jpg_buffers.active = fh->buffers.active = ZORAN_FREE;
-		}
-		jpg_fbuffer_free(fh);
-	}
-}
-
-static const struct vm_operations_struct zoran_vm_ops = {
-	.open = zoran_vm_open,
-	.close = zoran_vm_close,
-};
-
-static int
-zoran_mmap (struct file           *file,
-	    struct vm_area_struct *vma)
-{
-	struct zoran_fh *fh = file->private_data;
-	struct zoran *zr = fh->zr;
-	unsigned long size = (vma->vm_end - vma->vm_start);
-	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
-	int i, j;
-	unsigned long page, start = vma->vm_start, todo, pos, fraglen;
-	int first, last;
-	struct zoran_mapping *map;
-	int res = 0;
-
-	dprintk(3,
-		KERN_INFO "%s: %s(%s) of 0x%08lx-0x%08lx (size=%lu)\n",
-		ZR_DEVNAME(zr), __func__,
-		mode_name(fh->map_mode), vma->vm_start, vma->vm_end, size);
-
-	if (!(vma->vm_flags & VM_SHARED) || !(vma->vm_flags & VM_READ) ||
-	    !(vma->vm_flags & VM_WRITE)) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - no MAP_SHARED/PROT_{READ,WRITE} given\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	if (!fh->buffers.allocated) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s(%s) - buffers not yet allocated\n",
-			ZR_DEVNAME(zr), __func__, mode_name(fh->map_mode));
-		res = -ENOMEM;
-		return res;
-	}
-
-	first = offset / fh->buffers.buffer_size;
-	last = first - 1 + size / fh->buffers.buffer_size;
-	if (offset % fh->buffers.buffer_size != 0 ||
-	    size % fh->buffers.buffer_size != 0 || first < 0 ||
-	    last < 0 || first >= fh->buffers.num_buffers ||
-	    last >= fh->buffers.buffer_size) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s(%s) - offset=%lu or size=%lu invalid for bufsize=%d and numbufs=%d\n",
-			ZR_DEVNAME(zr), __func__, mode_name(fh->map_mode), offset, size,
-			fh->buffers.buffer_size,
-			fh->buffers.num_buffers);
-		res = -EINVAL;
-		return res;
-	}
-
-	/* Check if any buffers are already mapped */
-	for (i = first; i <= last; i++) {
-		if (fh->buffers.buffer[i].map) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s(%s) - buffer %d already mapped\n",
-				ZR_DEVNAME(zr), __func__, mode_name(fh->map_mode), i);
-			res = -EBUSY;
-			return res;
-		}
-	}
-
-	/* map these buffers */
-	map = kmalloc(sizeof(struct zoran_mapping), GFP_KERNEL);
-	if (!map) {
-		res = -ENOMEM;
-		return res;
-	}
-	map->fh = fh;
-	atomic_set(&map->count, 1);
-
-	vma->vm_ops = &zoran_vm_ops;
-	vma->vm_flags |= VM_DONTEXPAND;
-	vma->vm_private_data = map;
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
-		for (i = first; i <= last; i++) {
-			todo = size;
-			if (todo > fh->buffers.buffer_size)
-				todo = fh->buffers.buffer_size;
-			page = fh->buffers.buffer[i].v4l.fbuffer_phys;
-			if (remap_pfn_range(vma, start, page >> PAGE_SHIFT,
-							todo, PAGE_SHARED)) {
-				dprintk(1,
-					KERN_ERR
-					"%s: %s(V4L) - remap_pfn_range failed\n",
-					ZR_DEVNAME(zr), __func__);
-				res = -EAGAIN;
-				return res;
-			}
-			size -= todo;
-			start += todo;
-			fh->buffers.buffer[i].map = map;
-			if (size == 0)
-				break;
-		}
-	} else {
-		for (i = first; i <= last; i++) {
-			for (j = 0;
-			     j < fh->buffers.buffer_size / PAGE_SIZE;
-			     j++) {
-				fraglen =
-				    (le32_to_cpu(fh->buffers.buffer[i].jpg.
-				     frag_tab[2 * j + 1]) & ~1) << 1;
-				todo = size;
-				if (todo > fraglen)
-					todo = fraglen;
-				pos =
-				    le32_to_cpu(fh->buffers.
-				    buffer[i].jpg.frag_tab[2 * j]);
-				/* should just be pos on i386 */
-				page = virt_to_phys(bus_to_virt(pos))
-								>> PAGE_SHIFT;
-				if (remap_pfn_range(vma, start, page,
-							todo, PAGE_SHARED)) {
-					dprintk(1,
-						KERN_ERR
-						"%s: %s(V4L) - remap_pfn_range failed\n",
-						ZR_DEVNAME(zr), __func__);
-					res = -EAGAIN;
-					return res;
-				}
-				size -= todo;
-				start += todo;
-				if (size == 0)
-					break;
-				if (le32_to_cpu(fh->buffers.buffer[i].jpg.
-				    frag_tab[2 * j + 1]) & 1)
-					break;	/* was last fragment */
-			}
-			fh->buffers.buffer[i].map = map;
-			if (size == 0)
-				break;
-
-		}
-	}
-	return res;
-}
-
-static const struct v4l2_ioctl_ops zoran_ioctl_ops = {
-	.vidioc_querycap		    = zoran_querycap,
-	.vidioc_s_selection		    = zoran_s_selection,
-	.vidioc_g_selection		    = zoran_g_selection,
-	.vidioc_enum_input		    = zoran_enum_input,
-	.vidioc_g_input			    = zoran_g_input,
-	.vidioc_s_input			    = zoran_s_input,
-	.vidioc_enum_output		    = zoran_enum_output,
-	.vidioc_g_output		    = zoran_g_output,
-	.vidioc_s_output		    = zoran_s_output,
-	.vidioc_g_fbuf			    = zoran_g_fbuf,
-	.vidioc_s_fbuf			    = zoran_s_fbuf,
-	.vidioc_g_std			    = zoran_g_std,
-	.vidioc_s_std			    = zoran_s_std,
-	.vidioc_g_jpegcomp		    = zoran_g_jpegcomp,
-	.vidioc_s_jpegcomp		    = zoran_s_jpegcomp,
-	.vidioc_overlay			    = zoran_overlay,
-	.vidioc_reqbufs			    = zoran_reqbufs,
-	.vidioc_querybuf		    = zoran_querybuf,
-	.vidioc_qbuf			    = zoran_qbuf,
-	.vidioc_dqbuf			    = zoran_dqbuf,
-	.vidioc_streamon		    = zoran_streamon,
-	.vidioc_streamoff		    = zoran_streamoff,
-	.vidioc_enum_fmt_vid_cap	    = zoran_enum_fmt_vid_cap,
-	.vidioc_enum_fmt_vid_out	    = zoran_enum_fmt_vid_out,
-	.vidioc_enum_fmt_vid_overlay	    = zoran_enum_fmt_vid_overlay,
-	.vidioc_g_fmt_vid_cap		    = zoran_g_fmt_vid_cap,
-	.vidioc_g_fmt_vid_out               = zoran_g_fmt_vid_out,
-	.vidioc_g_fmt_vid_overlay           = zoran_g_fmt_vid_overlay,
-	.vidioc_s_fmt_vid_cap		    = zoran_s_fmt_vid_cap,
-	.vidioc_s_fmt_vid_out               = zoran_s_fmt_vid_out,
-	.vidioc_s_fmt_vid_overlay           = zoran_s_fmt_vid_overlay,
-	.vidioc_try_fmt_vid_cap		    = zoran_try_fmt_vid_cap,
-	.vidioc_try_fmt_vid_out		    = zoran_try_fmt_vid_out,
-	.vidioc_try_fmt_vid_overlay	    = zoran_try_fmt_vid_overlay,
-	.vidioc_subscribe_event             = v4l2_ctrl_subscribe_event,
-	.vidioc_unsubscribe_event           = v4l2_event_unsubscribe,
-};
-
-static const struct v4l2_file_operations zoran_fops = {
-	.owner = THIS_MODULE,
-	.open = zoran_open,
-	.release = zoran_close,
-	.unlocked_ioctl = video_ioctl2,
-	.mmap = zoran_mmap,
-	.poll = zoran_poll,
-};
-
-const struct video_device zoran_template = {
-	.name = ZORAN_NAME,
-	.fops = &zoran_fops,
-	.ioctl_ops = &zoran_ioctl_ops,
-	.release = &zoran_vdev_release,
-	.tvnorms = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM,
-};
-
diff --git a/drivers/media/pci/zoran/zoran_procfs.c b/drivers/media/pci/zoran/zoran_procfs.c
deleted file mode 100644
index 78ac8f853748..000000000000
--- a/drivers/media/pci/zoran/zoran_procfs.c
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles the procFS entries (/proc/ZORAN[%d])
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/vmalloc.h>
-
-#include <linux/proc_fs.h>
-#include <linux/pci.h>
-#include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
-#include <linux/videodev2.h>
-#include <linux/spinlock.h>
-#include <linux/sem.h>
-#include <linux/seq_file.h>
-
-#include <linux/ctype.h>
-#include <linux/poll.h>
-#include <asm/io.h>
-
-#include "videocodec.h"
-#include "zoran.h"
-#include "zoran_procfs.h"
-#include "zoran_card.h"
-
-#ifdef CONFIG_PROC_FS
-struct procfs_params_zr36067 {
-	char *name;
-	short reg;
-	u32 mask;
-	short bit;
-};
-
-static const struct procfs_params_zr36067 zr67[] = {
-	{"HSPol", 0x000, 1, 30},
-	{"HStart", 0x000, 0x3ff, 10},
-	{"HEnd", 0x000, 0x3ff, 0},
-
-	{"VSPol", 0x004, 1, 30},
-	{"VStart", 0x004, 0x3ff, 10},
-	{"VEnd", 0x004, 0x3ff, 0},
-
-	{"ExtFl", 0x008, 1, 26},
-	{"TopField", 0x008, 1, 25},
-	{"VCLKPol", 0x008, 1, 24},
-	{"DupFld", 0x008, 1, 20},
-	{"LittleEndian", 0x008, 1, 0},
-
-	{"HsyncStart", 0x10c, 0xffff, 16},
-	{"LineTot", 0x10c, 0xffff, 0},
-
-	{"NAX", 0x110, 0xffff, 16},
-	{"PAX", 0x110, 0xffff, 0},
-
-	{"NAY", 0x114, 0xffff, 16},
-	{"PAY", 0x114, 0xffff, 0},
-
-	/* {"",,,}, */
-
-	{NULL, 0, 0, 0},
-};
-
-static void
-setparam (struct zoran *zr,
-	  char         *name,
-	  char         *sval)
-{
-	int i = 0, reg0, reg, val;
-
-	while (zr67[i].name != NULL) {
-		if (!strncmp(name, zr67[i].name, strlen(zr67[i].name))) {
-			reg = reg0 = btread(zr67[i].reg);
-			reg &= ~(zr67[i].mask << zr67[i].bit);
-			if (!isdigit(sval[0]))
-				break;
-			val = simple_strtoul(sval, NULL, 0);
-			if ((val & ~zr67[i].mask))
-				break;
-			reg |= (val & zr67[i].mask) << zr67[i].bit;
-			dprintk(4,
-				KERN_INFO
-				"%s: setparam: setting ZR36067 register 0x%03x: 0x%08x=>0x%08x %s=%d\n",
-				ZR_DEVNAME(zr), zr67[i].reg, reg0, reg,
-				zr67[i].name, val);
-			btwrite(reg, zr67[i].reg);
-			break;
-		}
-		i++;
-	}
-}
-
-static int zoran_show(struct seq_file *p, void *v)
-{
-	struct zoran *zr = p->private;
-	int i;
-
-	seq_printf(p, "ZR36067 registers:\n");
-	for (i = 0; i < 0x130; i += 16)
-		seq_printf(p, "%03X %08X  %08X  %08X  %08X \n", i,
-			   btread(i), btread(i+4), btread(i+8), btread(i+12));
-	return 0;
-}
-
-static int zoran_open(struct inode *inode, struct file *file)
-{
-	struct zoran *data = PDE_DATA(inode);
-	return single_open(file, zoran_show, data);
-}
-
-static ssize_t zoran_write(struct file *file, const char __user *buffer,
-			size_t count, loff_t *ppos)
-{
-	struct zoran *zr = PDE_DATA(file_inode(file));
-	char *string, *sp;
-	char *line, *ldelim, *varname, *svar, *tdelim;
-
-	if (count > 32768)	/* Stupidity filter */
-		return -EINVAL;
-
-	string = sp = vmalloc(count + 1);
-	if (!string) {
-		dprintk(1,
-			KERN_ERR
-			"%s: write_proc: can not allocate memory\n",
-			ZR_DEVNAME(zr));
-		return -ENOMEM;
-	}
-	if (copy_from_user(string, buffer, count)) {
-		vfree (string);
-		return -EFAULT;
-	}
-	string[count] = 0;
-	dprintk(4, KERN_INFO "%s: write_proc: name=%pD count=%zu zr=%p\n",
-		ZR_DEVNAME(zr), file, count, zr);
-	ldelim = " \t\n";
-	tdelim = "=";
-	line = strpbrk(sp, ldelim);
-	while (line) {
-		*line = 0;
-		svar = strpbrk(sp, tdelim);
-		if (svar) {
-			*svar = 0;
-			varname = sp;
-			svar++;
-			setparam(zr, varname, svar);
-		}
-		sp = line + 1;
-		line = strpbrk(sp, ldelim);
-	}
-	vfree(string);
-
-	return count;
-}
-
-static const struct file_operations zoran_operations = {
-	.owner		= THIS_MODULE,
-	.open		= zoran_open,
-	.read		= seq_read,
-	.write		= zoran_write,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-#endif
-
-int
-zoran_proc_init (struct zoran *zr)
-{
-#ifdef CONFIG_PROC_FS
-	char name[8];
-
-	snprintf(name, 7, "zoran%d", zr->id);
-	zr->zoran_proc = proc_create_data(name, 0, NULL, &zoran_operations, zr);
-	if (zr->zoran_proc != NULL) {
-		dprintk(2,
-			KERN_INFO
-			"%s: procfs entry /proc/%s allocated. data=%p\n",
-			ZR_DEVNAME(zr), name, zr);
-	} else {
-		dprintk(1, KERN_ERR "%s: Unable to initialise /proc/%s\n",
-			ZR_DEVNAME(zr), name);
-		return 1;
-	}
-#endif
-	return 0;
-}
-
-void
-zoran_proc_cleanup (struct zoran *zr)
-{
-#ifdef CONFIG_PROC_FS
-	char name[8];
-
-	snprintf(name, 7, "zoran%d", zr->id);
-	if (zr->zoran_proc)
-		remove_proc_entry(name, NULL);
-	zr->zoran_proc = NULL;
-#endif
-}
diff --git a/drivers/media/pci/zoran/zoran_procfs.h b/drivers/media/pci/zoran/zoran_procfs.h
deleted file mode 100644
index 0ac7cb0011f2..000000000000
--- a/drivers/media/pci/zoran/zoran_procfs.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles card-specific data and detection
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#ifndef __ZORAN_PROCFS_H__
-#define __ZORAN_PROCFS_H__
-
-extern int zoran_proc_init(struct zoran *zr);
-extern void zoran_proc_cleanup(struct zoran *zr);
-
-#endif				/* __ZORAN_PROCFS_H__ */
diff --git a/drivers/media/pci/zoran/zr36016.c b/drivers/media/pci/zoran/zr36016.c
deleted file mode 100644
index 8736b9d8d97e..000000000000
--- a/drivers/media/pci/zoran/zr36016.c
+++ /dev/null
@@ -1,516 +0,0 @@
-/*
- * Zoran ZR36016 basic configuration functions
- *
- * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
- *
- * $Id: zr36016.c,v 1.1.2.14 2003/08/20 19:46:55 rbultje Exp $
- *
- * ------------------------------------------------------------------------
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * ------------------------------------------------------------------------
- */
-
-#define ZR016_VERSION "v0.7"
-
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-
-#include <linux/types.h>
-#include <linux/wait.h>
-
-/* I/O commands, error codes */
-#include <asm/io.h>
-
-/* v4l  API */
-
-/* headerfile of this module */
-#include "zr36016.h"
-
-/* codec io API */
-#include "videocodec.h"
-
-/* it doesn't make sense to have more than 20 or so,
-  just to prevent some unwanted loops */
-#define MAX_CODECS 20
-
-/* amount of chips attached via this driver */
-static int zr36016_codecs;
-
-/* debugging is available via module parameter */
-static int debug;
-module_param(debug, int, 0);
-MODULE_PARM_DESC(debug, "Debug level (0-4)");
-
-#define dprintk(num, format, args...) \
-	do { \
-		if (debug >= num) \
-			printk(format, ##args); \
-	} while (0)
-
-/* =========================================================================
-   Local hardware I/O functions:
-
-   read/write via codec layer (registers are located in the master device)
-   ========================================================================= */
-
-/* read and write functions */
-static u8
-zr36016_read (struct zr36016 *ptr,
-	      u16             reg)
-{
-	u8 value = 0;
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->readreg)
-		value =
-		    (ptr->codec->master_data->
-		     readreg(ptr->codec, reg)) & 0xFF;
-	else
-		dprintk(1,
-			KERN_ERR "%s: invalid I/O setup, nothing read!\n",
-			ptr->name);
-
-	dprintk(4, "%s: reading from 0x%04x: %02x\n", ptr->name, reg,
-		value);
-
-	return value;
-}
-
-static void
-zr36016_write (struct zr36016 *ptr,
-	       u16             reg,
-	       u8              value)
-{
-	dprintk(4, "%s: writing 0x%02x to 0x%04x\n", ptr->name, value,
-		reg);
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->writereg) {
-		ptr->codec->master_data->writereg(ptr->codec, reg, value);
-	} else
-		dprintk(1,
-			KERN_ERR
-			"%s: invalid I/O setup, nothing written!\n",
-			ptr->name);
-}
-
-/* indirect read and write functions */
-/* the 016 supports auto-addr-increment, but
- * writing it all time cost not much and is safer... */
-static u8
-zr36016_readi (struct zr36016 *ptr,
-	       u16             reg)
-{
-	u8 value = 0;
-
-	// just in case something is wrong...
-	if ((ptr->codec->master_data->writereg) &&
-	    (ptr->codec->master_data->readreg)) {
-		ptr->codec->master_data->writereg(ptr->codec, ZR016_IADDR, reg & 0x0F);	// ADDR
-		value = (ptr->codec->master_data->readreg(ptr->codec, ZR016_IDATA)) & 0xFF;	// DATA
-	} else
-		dprintk(1,
-			KERN_ERR
-			"%s: invalid I/O setup, nothing read (i)!\n",
-			ptr->name);
-
-	dprintk(4, "%s: reading indirect from 0x%04x: %02x\n", ptr->name,
-		reg, value);
-	return value;
-}
-
-static void
-zr36016_writei (struct zr36016 *ptr,
-		u16             reg,
-		u8              value)
-{
-	dprintk(4, "%s: writing indirect 0x%02x to 0x%04x\n", ptr->name,
-		value, reg);
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->writereg) {
-		ptr->codec->master_data->writereg(ptr->codec, ZR016_IADDR, reg & 0x0F);	// ADDR
-		ptr->codec->master_data->writereg(ptr->codec, ZR016_IDATA, value & 0x0FF);	// DATA
-	} else
-		dprintk(1,
-			KERN_ERR
-			"%s: invalid I/O setup, nothing written (i)!\n",
-			ptr->name);
-}
-
-/* =========================================================================
-   Local helper function:
-
-   version read
-   ========================================================================= */
-
-/* version kept in datastructure */
-static u8
-zr36016_read_version (struct zr36016 *ptr)
-{
-	ptr->version = zr36016_read(ptr, 0) >> 4;
-	return ptr->version;
-}
-
-/* =========================================================================
-   Local helper function:
-
-   basic test of "connectivity", writes/reads to/from PAX-Lo register
-   ========================================================================= */
-
-static int
-zr36016_basic_test (struct zr36016 *ptr)
-{
-	if (debug) {
-		int i;
-		zr36016_writei(ptr, ZR016I_PAX_LO, 0x55);
-		dprintk(1, KERN_INFO "%s: registers: ", ptr->name);
-		for (i = 0; i <= 0x0b; i++)
-			dprintk(1, "%02x ", zr36016_readi(ptr, i));
-		dprintk(1, "\n");
-	}
-	// for testing just write 0, then the default value to a register and read
-	// it back in both cases
-	zr36016_writei(ptr, ZR016I_PAX_LO, 0x00);
-	if (zr36016_readi(ptr, ZR016I_PAX_LO) != 0x0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, can't connect to vfe processor!\n",
-			ptr->name);
-		return -ENXIO;
-	}
-	zr36016_writei(ptr, ZR016I_PAX_LO, 0x0d0);
-	if (zr36016_readi(ptr, ZR016I_PAX_LO) != 0x0d0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, can't connect to vfe processor!\n",
-			ptr->name);
-		return -ENXIO;
-	}
-	// we allow version numbers from 0-3, should be enough, though
-	zr36016_read_version(ptr);
-	if (ptr->version & 0x0c) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, suspicious version %d found...\n",
-			ptr->name, ptr->version);
-		return -ENXIO;
-	}
-
-	return 0;		/* looks good! */
-}
-
-/* =========================================================================
-   Local helper function:
-
-   simple loop for pushing the init datasets - NO USE --
-   ========================================================================= */
-
-#if 0
-static int zr36016_pushit (struct zr36016 *ptr,
-			   u16             startreg,
-			   u16             len,
-			   const char     *data)
-{
-	int i=0;
-
-	dprintk(4, "%s: write data block to 0x%04x (len=%d)\n",
-		ptr->name, startreg,len);
-	while (i<len) {
-		zr36016_writei(ptr, startreg++,  data[i++]);
-	}
-
-	return i;
-}
-#endif
-
-/* =========================================================================
-   Basic datasets & init:
-
-   //TODO//
-   ========================================================================= */
-
-static void
-zr36016_init (struct zr36016 *ptr)
-{
-	// stop any processing
-	zr36016_write(ptr, ZR016_GOSTOP, 0);
-
-	// mode setup (yuv422 in and out, compression/expansuon due to mode)
-	zr36016_write(ptr, ZR016_MODE,
-		      ZR016_YUV422 | ZR016_YUV422_YUV422 |
-		      (ptr->mode == CODEC_DO_COMPRESSION ?
-		       ZR016_COMPRESSION : ZR016_EXPANSION));
-
-	// misc setup
-	zr36016_writei(ptr, ZR016I_SETUP1,
-		       (ptr->xdec ? (ZR016_HRFL | ZR016_HORZ) : 0) |
-		       (ptr->ydec ? ZR016_VERT : 0) | ZR016_CNTI);
-	zr36016_writei(ptr, ZR016I_SETUP2, ZR016_CCIR);
-
-	// Window setup
-	// (no extra offset for now, norm defines offset, default width height)
-	zr36016_writei(ptr, ZR016I_PAX_HI, ptr->width >> 8);
-	zr36016_writei(ptr, ZR016I_PAX_LO, ptr->width & 0xFF);
-	zr36016_writei(ptr, ZR016I_PAY_HI, ptr->height >> 8);
-	zr36016_writei(ptr, ZR016I_PAY_LO, ptr->height & 0xFF);
-	zr36016_writei(ptr, ZR016I_NAX_HI, ptr->xoff >> 8);
-	zr36016_writei(ptr, ZR016I_NAX_LO, ptr->xoff & 0xFF);
-	zr36016_writei(ptr, ZR016I_NAY_HI, ptr->yoff >> 8);
-	zr36016_writei(ptr, ZR016I_NAY_LO, ptr->yoff & 0xFF);
-
-	/* shall we continue now, please? */
-	zr36016_write(ptr, ZR016_GOSTOP, 1);
-}
-
-/* =========================================================================
-   CODEC API FUNCTIONS
-
-   this functions are accessed by the master via the API structure
-   ========================================================================= */
-
-/* set compression/expansion mode and launches codec -
-   this should be the last call from the master before starting processing */
-static int
-zr36016_set_mode (struct videocodec *codec,
-		  int                mode)
-{
-	struct zr36016 *ptr = (struct zr36016 *) codec->data;
-
-	dprintk(2, "%s: set_mode %d call\n", ptr->name, mode);
-
-	if ((mode != CODEC_DO_EXPANSION) && (mode != CODEC_DO_COMPRESSION))
-		return -EINVAL;
-
-	ptr->mode = mode;
-	zr36016_init(ptr);
-
-	return 0;
-}
-
-/* set picture size */
-static int
-zr36016_set_video (struct videocodec   *codec,
-		   struct tvnorm       *norm,
-		   struct vfe_settings *cap,
-		   struct vfe_polarity *pol)
-{
-	struct zr36016 *ptr = (struct zr36016 *) codec->data;
-
-	dprintk(2, "%s: set_video %d.%d, %d/%d-%dx%d (0x%x) call\n",
-		ptr->name, norm->HStart, norm->VStart,
-		cap->x, cap->y, cap->width, cap->height,
-		cap->decimation);
-
-	/* if () return -EINVAL;
-	 * trust the master driver that it knows what it does - so
-	 * we allow invalid startx/y for now ... */
-	ptr->width = cap->width;
-	ptr->height = cap->height;
-	/* (Ronald) This is ugly. zoran_device.c, line 387
-	 * already mentions what happens if HStart is even
-	 * (blue faces, etc., cr/cb inversed). There's probably
-	 * some good reason why HStart is 0 instead of 1, so I'm
-	 * leaving it to this for now, but really... This can be
-	 * done a lot simpler */
-	ptr->xoff = (norm->HStart ? norm->HStart : 1) + cap->x;
-	/* Something to note here (I don't understand it), setting
-	 * VStart too high will cause the codec to 'not work'. I
-	 * really don't get it. values of 16 (VStart) already break
-	 * it here. Just '0' seems to work. More testing needed! */
-	ptr->yoff = norm->VStart + cap->y;
-	/* (Ronald) dzjeeh, can't this thing do hor_decimation = 4? */
-	ptr->xdec = ((cap->decimation & 0xff) == 1) ? 0 : 1;
-	ptr->ydec = (((cap->decimation >> 8) & 0xff) == 1) ? 0 : 1;
-
-	return 0;
-}
-
-/* additional control functions */
-static int
-zr36016_control (struct videocodec *codec,
-		 int                type,
-		 int                size,
-		 void              *data)
-{
-	struct zr36016 *ptr = (struct zr36016 *) codec->data;
-	int *ival = (int *) data;
-
-	dprintk(2, "%s: control %d call with %d byte\n", ptr->name, type,
-		size);
-
-	switch (type) {
-	case CODEC_G_STATUS:	/* get last status - we don't know it ... */
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = 0;
-		break;
-
-	case CODEC_G_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = 0;
-		break;
-
-	case CODEC_S_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		if (*ival != 0)
-			return -EINVAL;
-		/* not needed, do nothing */
-		return 0;
-
-	case CODEC_G_VFE:
-	case CODEC_S_VFE:
-		return 0;
-
-	case CODEC_S_MMAP:
-		/* not available, give an error */
-		return -ENXIO;
-
-	default:
-		return -EINVAL;
-	}
-
-	return size;
-}
-
-/* =========================================================================
-   Exit and unregister function:
-
-   Deinitializes Zoran's JPEG processor
-   ========================================================================= */
-
-static int
-zr36016_unset (struct videocodec *codec)
-{
-	struct zr36016 *ptr = codec->data;
-
-	if (ptr) {
-		/* do wee need some codec deinit here, too ???? */
-
-		dprintk(1, "%s: finished codec #%d\n", ptr->name,
-			ptr->num);
-		kfree(ptr);
-		codec->data = NULL;
-
-		zr36016_codecs--;
-		return 0;
-	}
-
-	return -EFAULT;
-}
-
-/* =========================================================================
-   Setup and registry function:
-
-   Initializes Zoran's JPEG processor
-
-   Also sets pixel size, average code size, mode (compr./decompr.)
-   (the given size is determined by the processor with the video interface)
-   ========================================================================= */
-
-static int
-zr36016_setup (struct videocodec *codec)
-{
-	struct zr36016 *ptr;
-	int res;
-
-	dprintk(2, "zr36016: initializing VFE subsystem #%d.\n",
-		zr36016_codecs);
-
-	if (zr36016_codecs == MAX_CODECS) {
-		dprintk(1,
-			KERN_ERR "zr36016: Can't attach more codecs!\n");
-		return -ENOSPC;
-	}
-	//mem structure init
-	codec->data = ptr = kzalloc(sizeof(struct zr36016), GFP_KERNEL);
-	if (NULL == ptr) {
-		dprintk(1, KERN_ERR "zr36016: Can't get enough memory!\n");
-		return -ENOMEM;
-	}
-
-	snprintf(ptr->name, sizeof(ptr->name), "zr36016[%d]",
-		 zr36016_codecs);
-	ptr->num = zr36016_codecs++;
-	ptr->codec = codec;
-
-	//testing
-	res = zr36016_basic_test(ptr);
-	if (res < 0) {
-		zr36016_unset(codec);
-		return res;
-	}
-	//final setup
-	ptr->mode = CODEC_DO_COMPRESSION;
-	ptr->width = 768;
-	ptr->height = 288;
-	ptr->xdec = 1;
-	ptr->ydec = 0;
-	zr36016_init(ptr);
-
-	dprintk(1, KERN_INFO "%s: codec v%d attached and running\n",
-		ptr->name, ptr->version);
-
-	return 0;
-}
-
-static const struct videocodec zr36016_codec = {
-	.owner = THIS_MODULE,
-	.name = "zr36016",
-	.magic = 0L,		// magic not used
-	.flags =
-	    CODEC_FLAG_HARDWARE | CODEC_FLAG_VFE | CODEC_FLAG_ENCODER |
-	    CODEC_FLAG_DECODER,
-	.type = CODEC_TYPE_ZR36016,
-	.setup = zr36016_setup,	// functionality
-	.unset = zr36016_unset,
-	.set_mode = zr36016_set_mode,
-	.set_video = zr36016_set_video,
-	.control = zr36016_control,
-	// others are not used
-};
-
-/* =========================================================================
-   HOOK IN DRIVER AS KERNEL MODULE
-   ========================================================================= */
-
-static int __init
-zr36016_init_module (void)
-{
-	//dprintk(1, "ZR36016 driver %s\n",ZR016_VERSION);
-	zr36016_codecs = 0;
-	return videocodec_register(&zr36016_codec);
-}
-
-static void __exit
-zr36016_cleanup_module (void)
-{
-	if (zr36016_codecs) {
-		dprintk(1,
-			"zr36016: something's wrong - %d codecs left somehow.\n",
-			zr36016_codecs);
-	}
-	videocodec_unregister(&zr36016_codec);
-}
-
-module_init(zr36016_init_module);
-module_exit(zr36016_cleanup_module);
-
-MODULE_AUTHOR("Wolfgang Scherr <scherr@net4you.at>");
-MODULE_DESCRIPTION("Driver module for ZR36016 video frontends "
-		   ZR016_VERSION);
-MODULE_LICENSE("GPL");
diff --git a/drivers/media/pci/zoran/zr36016.h b/drivers/media/pci/zoran/zr36016.h
deleted file mode 100644
index 784bcf5727b8..000000000000
--- a/drivers/media/pci/zoran/zr36016.h
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Zoran ZR36016 basic configuration functions - header file
- *
- * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
- *
- * $Id: zr36016.h,v 1.1.2.3 2003/01/14 21:18:07 rbultje Exp $
- *
- * ------------------------------------------------------------------------
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * ------------------------------------------------------------------------
- */
-
-#ifndef ZR36016_H
-#define ZR36016_H
-
-/* data stored for each zoran jpeg codec chip */
-struct zr36016 {
-	char name[32];
-	int num;
-	/* io datastructure */
-	struct videocodec *codec;
-	// coder status
-	__u8 version;
-	// actual coder setup
-	int mode;
-
-	__u16 xoff;
-	__u16 yoff;
-	__u16 width;
-	__u16 height;
-	__u16 xdec;
-	__u16 ydec;
-};
-
-/* direct  register addresses */
-#define ZR016_GOSTOP      0x00
-#define ZR016_MODE        0x01
-#define ZR016_IADDR       0x02
-#define ZR016_IDATA       0x03
-
-/* indirect  register addresses */
-#define ZR016I_SETUP1     0x00
-#define ZR016I_SETUP2     0x01
-#define ZR016I_NAX_LO     0x02
-#define ZR016I_NAX_HI     0x03
-#define ZR016I_PAX_LO     0x04
-#define ZR016I_PAX_HI     0x05
-#define ZR016I_NAY_LO     0x06
-#define ZR016I_NAY_HI     0x07
-#define ZR016I_PAY_LO     0x08
-#define ZR016I_PAY_HI     0x09
-#define ZR016I_NOL_LO     0x0a
-#define ZR016I_NOL_HI     0x0b
-
-/* possible values for mode register */
-#define ZR016_RGB444_YUV444  0x00
-#define ZR016_RGB444_YUV422  0x01
-#define ZR016_RGB444_YUV411  0x02
-#define ZR016_RGB444_Y400    0x03
-#define ZR016_RGB444_RGB444  0x04
-#define ZR016_YUV444_YUV444  0x08
-#define ZR016_YUV444_YUV422  0x09
-#define ZR016_YUV444_YUV411  0x0a
-#define ZR016_YUV444_Y400    0x0b
-#define ZR016_YUV444_RGB444  0x0c
-#define ZR016_YUV422_YUV422  0x11
-#define ZR016_YUV422_YUV411  0x12
-#define ZR016_YUV422_Y400    0x13
-#define ZR016_YUV411_YUV411  0x16
-#define ZR016_YUV411_Y400    0x17
-#define ZR016_4444_4444      0x19
-#define ZR016_100_100        0x1b
-
-#define ZR016_RGB444         0x00
-#define ZR016_YUV444         0x20
-#define ZR016_YUV422         0x40
-
-#define ZR016_COMPRESSION    0x80
-#define ZR016_EXPANSION      0x80
-
-/* possible values for setup 1 register */
-#define ZR016_CKRT           0x80
-#define ZR016_VERT           0x40
-#define ZR016_HORZ           0x20
-#define ZR016_HRFL           0x10
-#define ZR016_DSFL           0x08
-#define ZR016_SBFL           0x04
-#define ZR016_RSTR           0x02
-#define ZR016_CNTI           0x01
-
-/* possible values for setup 2 register */
-#define ZR016_SYEN           0x40
-#define ZR016_CCIR           0x04
-#define ZR016_SIGN           0x02
-#define ZR016_YMCS           0x01
-
-#endif				/*fndef ZR36016_H */
diff --git a/drivers/media/pci/zoran/zr36050.c b/drivers/media/pci/zoran/zr36050.c
deleted file mode 100644
index 5ebfc16672f3..000000000000
--- a/drivers/media/pci/zoran/zr36050.c
+++ /dev/null
@@ -1,896 +0,0 @@
-/*
- * Zoran ZR36050 basic configuration functions
- *
- * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
- *
- * $Id: zr36050.c,v 1.1.2.11 2003/08/03 14:54:53 rbultje Exp $
- *
- * ------------------------------------------------------------------------
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * ------------------------------------------------------------------------
- */
-
-#define ZR050_VERSION "v0.7.1"
-
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-
-#include <linux/types.h>
-#include <linux/wait.h>
-
-/* I/O commands, error codes */
-#include <asm/io.h>
-
-/* headerfile of this module */
-#include "zr36050.h"
-
-/* codec io API */
-#include "videocodec.h"
-
-/* it doesn't make sense to have more than 20 or so,
-  just to prevent some unwanted loops */
-#define MAX_CODECS 20
-
-/* amount of chips attached via this driver */
-static int zr36050_codecs;
-
-/* debugging is available via module parameter */
-static int debug;
-module_param(debug, int, 0);
-MODULE_PARM_DESC(debug, "Debug level (0-4)");
-
-#define dprintk(num, format, args...) \
-	do { \
-		if (debug >= num) \
-			printk(format, ##args); \
-	} while (0)
-
-/* =========================================================================
-   Local hardware I/O functions:
-
-   read/write via codec layer (registers are located in the master device)
-   ========================================================================= */
-
-/* read and write functions */
-static u8
-zr36050_read (struct zr36050 *ptr,
-	      u16             reg)
-{
-	u8 value = 0;
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->readreg)
-		value = (ptr->codec->master_data->readreg(ptr->codec,
-							  reg)) & 0xFF;
-	else
-		dprintk(1,
-			KERN_ERR "%s: invalid I/O setup, nothing read!\n",
-			ptr->name);
-
-	dprintk(4, "%s: reading from 0x%04x: %02x\n", ptr->name, reg,
-		value);
-
-	return value;
-}
-
-static void
-zr36050_write (struct zr36050 *ptr,
-	       u16             reg,
-	       u8              value)
-{
-	dprintk(4, "%s: writing 0x%02x to 0x%04x\n", ptr->name, value,
-		reg);
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->writereg)
-		ptr->codec->master_data->writereg(ptr->codec, reg, value);
-	else
-		dprintk(1,
-			KERN_ERR
-			"%s: invalid I/O setup, nothing written!\n",
-			ptr->name);
-}
-
-/* =========================================================================
-   Local helper function:
-
-   status read
-   ========================================================================= */
-
-/* status is kept in datastructure */
-static u8
-zr36050_read_status1 (struct zr36050 *ptr)
-{
-	ptr->status1 = zr36050_read(ptr, ZR050_STATUS_1);
-
-	zr36050_read(ptr, 0);
-	return ptr->status1;
-}
-
-/* =========================================================================
-   Local helper function:
-
-   scale factor read
-   ========================================================================= */
-
-/* scale factor is kept in datastructure */
-static u16
-zr36050_read_scalefactor (struct zr36050 *ptr)
-{
-	ptr->scalefact = (zr36050_read(ptr, ZR050_SF_HI) << 8) |
-			 (zr36050_read(ptr, ZR050_SF_LO) & 0xFF);
-
-	/* leave 0 selected for an eventually GO from master */
-	zr36050_read(ptr, 0);
-	return ptr->scalefact;
-}
-
-/* =========================================================================
-   Local helper function:
-
-   wait if codec is ready to proceed (end of processing) or time is over
-   ========================================================================= */
-
-static void
-zr36050_wait_end (struct zr36050 *ptr)
-{
-	int i = 0;
-
-	while (!(zr36050_read_status1(ptr) & 0x4)) {
-		udelay(1);
-		if (i++ > 200000) {	// 200ms, there is for sure something wrong!!!
-			dprintk(1,
-				"%s: timeout at wait_end (last status: 0x%02x)\n",
-				ptr->name, ptr->status1);
-			break;
-		}
-	}
-}
-
-/* =========================================================================
-   Local helper function:
-
-   basic test of "connectivity", writes/reads to/from memory the SOF marker
-   ========================================================================= */
-
-static int
-zr36050_basic_test (struct zr36050 *ptr)
-{
-	zr36050_write(ptr, ZR050_SOF_IDX, 0x00);
-	zr36050_write(ptr, ZR050_SOF_IDX + 1, 0x00);
-	if ((zr36050_read(ptr, ZR050_SOF_IDX) |
-	     zr36050_read(ptr, ZR050_SOF_IDX + 1)) != 0x0000) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, can't connect to jpeg processor!\n",
-			ptr->name);
-		return -ENXIO;
-	}
-	zr36050_write(ptr, ZR050_SOF_IDX, 0xff);
-	zr36050_write(ptr, ZR050_SOF_IDX + 1, 0xc0);
-	if (((zr36050_read(ptr, ZR050_SOF_IDX) << 8) |
-	     zr36050_read(ptr, ZR050_SOF_IDX + 1)) != 0xffc0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, can't connect to jpeg processor!\n",
-			ptr->name);
-		return -ENXIO;
-	}
-
-	zr36050_wait_end(ptr);
-	if ((ptr->status1 & 0x4) == 0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, jpeg processor failed (end flag)!\n",
-			ptr->name);
-		return -EBUSY;
-	}
-
-	return 0;		/* looks good! */
-}
-
-/* =========================================================================
-   Local helper function:
-
-   simple loop for pushing the init datasets
-   ========================================================================= */
-
-static int
-zr36050_pushit (struct zr36050 *ptr,
-		u16             startreg,
-		u16             len,
-		const char     *data)
-{
-	int i = 0;
-
-	dprintk(4, "%s: write data block to 0x%04x (len=%d)\n", ptr->name,
-		startreg, len);
-	while (i < len) {
-		zr36050_write(ptr, startreg++, data[i++]);
-	}
-
-	return i;
-}
-
-/* =========================================================================
-   Basic datasets:
-
-   jpeg baseline setup data (you find it on lots places in internet, or just
-   extract it from any regular .jpg image...)
-
-   Could be variable, but until it's not needed it they are just fixed to save
-   memory. Otherwise expand zr36050 structure with arrays, push the values to
-   it and initialize from there, as e.g. the linux zr36057/60 driver does it.
-   ========================================================================= */
-
-static const char zr36050_dqt[0x86] = {
-	0xff, 0xdb,		//Marker: DQT
-	0x00, 0x84,		//Length: 2*65+2
-	0x00,			//Pq,Tq first table
-	0x10, 0x0b, 0x0c, 0x0e, 0x0c, 0x0a, 0x10, 0x0e,
-	0x0d, 0x0e, 0x12, 0x11, 0x10, 0x13, 0x18, 0x28,
-	0x1a, 0x18, 0x16, 0x16, 0x18, 0x31, 0x23, 0x25,
-	0x1d, 0x28, 0x3a, 0x33, 0x3d, 0x3c, 0x39, 0x33,
-	0x38, 0x37, 0x40, 0x48, 0x5c, 0x4e, 0x40, 0x44,
-	0x57, 0x45, 0x37, 0x38, 0x50, 0x6d, 0x51, 0x57,
-	0x5f, 0x62, 0x67, 0x68, 0x67, 0x3e, 0x4d, 0x71,
-	0x79, 0x70, 0x64, 0x78, 0x5c, 0x65, 0x67, 0x63,
-	0x01,			//Pq,Tq second table
-	0x11, 0x12, 0x12, 0x18, 0x15, 0x18, 0x2f, 0x1a,
-	0x1a, 0x2f, 0x63, 0x42, 0x38, 0x42, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63
-};
-
-static const char zr36050_dht[0x1a4] = {
-	0xff, 0xc4,		//Marker: DHT
-	0x01, 0xa2,		//Length: 2*AC, 2*DC
-	0x00,			//DC first table
-	0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
-	0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
-	0x01,			//DC second table
-	0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
-	0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
-	0x10,			//AC first table
-	0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03,
-	0x05, 0x05, 0x04, 0x04, 0x00, 0x00,
-	0x01, 0x7D, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11,
-	0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61,
-	0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1,
-	0x08, 0x23, 0x42, 0xB1, 0xC1, 0x15, 0x52, 0xD1, 0xF0, 0x24,
-	0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17,
-	0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x34,
-	0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
-	0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
-	0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
-	0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
-	0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88,
-	0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99,
-	0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8,
-	0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9,
-	0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8,
-	0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9,
-	0xDA, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
-	0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
-	0xF8, 0xF9, 0xFA,
-	0x11,			//AC second table
-	0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04,
-	0x07, 0x05, 0x04, 0x04, 0x00, 0x01,
-	0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04,
-	0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71,
-	0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91,
-	0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, 0x15, 0x62,
-	0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25,
-	0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, 0x27, 0x28, 0x29, 0x2A,
-	0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
-	0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
-	0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
-	0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
-	0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
-	0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98,
-	0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
-	0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8,
-	0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
-	0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8,
-	0xD9, 0xDA, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
-	0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8,
-	0xF9, 0xFA
-};
-
-/* jpeg baseline setup, this is just fixed in this driver (YUV pictures) */
-#define NO_OF_COMPONENTS          0x3	//Y,U,V
-#define BASELINE_PRECISION        0x8	//MCU size (?)
-static const char zr36050_tq[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's QT
-static const char zr36050_td[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's DC
-static const char zr36050_ta[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's AC
-
-/* horizontal 422 decimation setup (maybe we support 411 or so later, too) */
-static const char zr36050_decimation_h[8] = { 2, 1, 1, 0, 0, 0, 0, 0 };
-static const char zr36050_decimation_v[8] = { 1, 1, 1, 0, 0, 0, 0, 0 };
-
-/* =========================================================================
-   Local helper functions:
-
-   calculation and setup of parameter-dependent JPEG baseline segments
-   (needed for compression only)
-   ========================================================================= */
-
-/* ------------------------------------------------------------------------- */
-
-/* SOF (start of frame) segment depends on width, height and sampling ratio
-			 of each color component */
-
-static int
-zr36050_set_sof (struct zr36050 *ptr)
-{
-	char sof_data[34];	// max. size of register set
-	int i;
-
-	dprintk(3, "%s: write SOF (%dx%d, %d components)\n", ptr->name,
-		ptr->width, ptr->height, NO_OF_COMPONENTS);
-	sof_data[0] = 0xff;
-	sof_data[1] = 0xc0;
-	sof_data[2] = 0x00;
-	sof_data[3] = (3 * NO_OF_COMPONENTS) + 8;
-	sof_data[4] = BASELINE_PRECISION;	// only '8' possible with zr36050
-	sof_data[5] = (ptr->height) >> 8;
-	sof_data[6] = (ptr->height) & 0xff;
-	sof_data[7] = (ptr->width) >> 8;
-	sof_data[8] = (ptr->width) & 0xff;
-	sof_data[9] = NO_OF_COMPONENTS;
-	for (i = 0; i < NO_OF_COMPONENTS; i++) {
-		sof_data[10 + (i * 3)] = i;	// index identifier
-		sof_data[11 + (i * 3)] = (ptr->h_samp_ratio[i] << 4) | (ptr->v_samp_ratio[i]);	// sampling ratios
-		sof_data[12 + (i * 3)] = zr36050_tq[i];	// Q table selection
-	}
-	return zr36050_pushit(ptr, ZR050_SOF_IDX,
-			      (3 * NO_OF_COMPONENTS) + 10, sof_data);
-}
-
-/* ------------------------------------------------------------------------- */
-
-/* SOS (start of scan) segment depends on the used scan components
-			of each color component */
-
-static int
-zr36050_set_sos (struct zr36050 *ptr)
-{
-	char sos_data[16];	// max. size of register set
-	int i;
-
-	dprintk(3, "%s: write SOS\n", ptr->name);
-	sos_data[0] = 0xff;
-	sos_data[1] = 0xda;
-	sos_data[2] = 0x00;
-	sos_data[3] = 2 + 1 + (2 * NO_OF_COMPONENTS) + 3;
-	sos_data[4] = NO_OF_COMPONENTS;
-	for (i = 0; i < NO_OF_COMPONENTS; i++) {
-		sos_data[5 + (i * 2)] = i;	// index
-		sos_data[6 + (i * 2)] = (zr36050_td[i] << 4) | zr36050_ta[i];	// AC/DC tbl.sel.
-	}
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 2] = 00;	// scan start
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 3] = 0x3F;
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 4] = 00;
-	return zr36050_pushit(ptr, ZR050_SOS1_IDX,
-			      4 + 1 + (2 * NO_OF_COMPONENTS) + 3,
-			      sos_data);
-}
-
-/* ------------------------------------------------------------------------- */
-
-/* DRI (define restart interval) */
-
-static int
-zr36050_set_dri (struct zr36050 *ptr)
-{
-	char dri_data[6];	// max. size of register set
-
-	dprintk(3, "%s: write DRI\n", ptr->name);
-	dri_data[0] = 0xff;
-	dri_data[1] = 0xdd;
-	dri_data[2] = 0x00;
-	dri_data[3] = 0x04;
-	dri_data[4] = ptr->dri >> 8;
-	dri_data[5] = ptr->dri & 0xff;
-	return zr36050_pushit(ptr, ZR050_DRI_IDX, 6, dri_data);
-}
-
-/* =========================================================================
-   Setup function:
-
-   Setup compression/decompression of Zoran's JPEG processor
-   ( see also zoran 36050 manual )
-
-   ... sorry for the spaghetti code ...
-   ========================================================================= */
-static void
-zr36050_init (struct zr36050 *ptr)
-{
-	int sum = 0;
-	long bitcnt, tmp;
-
-	if (ptr->mode == CODEC_DO_COMPRESSION) {
-		dprintk(2, "%s: COMPRESSION SETUP\n", ptr->name);
-
-		/* 050 communicates with 057 in master mode */
-		zr36050_write(ptr, ZR050_HARDWARE, ZR050_HW_MSTR);
-
-		/* encoding table preload for compression */
-		zr36050_write(ptr, ZR050_MODE,
-			      ZR050_MO_COMP | ZR050_MO_TLM);
-		zr36050_write(ptr, ZR050_OPTIONS, 0);
-
-		/* disable all IRQs */
-		zr36050_write(ptr, ZR050_INT_REQ_0, 0);
-		zr36050_write(ptr, ZR050_INT_REQ_1, 3);	// low 2 bits always 1
-
-		/* volume control settings */
-		/*zr36050_write(ptr, ZR050_MBCV, ptr->max_block_vol);*/
-		zr36050_write(ptr, ZR050_SF_HI, ptr->scalefact >> 8);
-		zr36050_write(ptr, ZR050_SF_LO, ptr->scalefact & 0xff);
-
-		zr36050_write(ptr, ZR050_AF_HI, 0xff);
-		zr36050_write(ptr, ZR050_AF_M, 0xff);
-		zr36050_write(ptr, ZR050_AF_LO, 0xff);
-
-		/* setup the variable jpeg tables */
-		sum += zr36050_set_sof(ptr);
-		sum += zr36050_set_sos(ptr);
-		sum += zr36050_set_dri(ptr);
-
-		/* setup the fixed jpeg tables - maybe variable, though -
-		 * (see table init section above) */
-		dprintk(3, "%s: write DQT, DHT, APP\n", ptr->name);
-		sum += zr36050_pushit(ptr, ZR050_DQT_IDX,
-				      sizeof(zr36050_dqt), zr36050_dqt);
-		sum += zr36050_pushit(ptr, ZR050_DHT_IDX,
-				      sizeof(zr36050_dht), zr36050_dht);
-		zr36050_write(ptr, ZR050_APP_IDX, 0xff);
-		zr36050_write(ptr, ZR050_APP_IDX + 1, 0xe0 + ptr->app.appn);
-		zr36050_write(ptr, ZR050_APP_IDX + 2, 0x00);
-		zr36050_write(ptr, ZR050_APP_IDX + 3, ptr->app.len + 2);
-		sum += zr36050_pushit(ptr, ZR050_APP_IDX + 4, 60,
-				      ptr->app.data) + 4;
-		zr36050_write(ptr, ZR050_COM_IDX, 0xff);
-		zr36050_write(ptr, ZR050_COM_IDX + 1, 0xfe);
-		zr36050_write(ptr, ZR050_COM_IDX + 2, 0x00);
-		zr36050_write(ptr, ZR050_COM_IDX + 3, ptr->com.len + 2);
-		sum += zr36050_pushit(ptr, ZR050_COM_IDX + 4, 60,
-				      ptr->com.data) + 4;
-
-		/* do the internal huffman table preload */
-		zr36050_write(ptr, ZR050_MARKERS_EN, ZR050_ME_DHTI);
-
-		zr36050_write(ptr, ZR050_GO, 1);	// launch codec
-		zr36050_wait_end(ptr);
-		dprintk(2, "%s: Status after table preload: 0x%02x\n",
-			ptr->name, ptr->status1);
-
-		if ((ptr->status1 & 0x4) == 0) {
-			dprintk(1, KERN_ERR "%s: init aborted!\n",
-				ptr->name);
-			return;	// something is wrong, its timed out!!!!
-		}
-
-		/* setup misc. data for compression (target code sizes) */
-
-		/* size of compressed code to reach without header data */
-		sum = ptr->real_code_vol - sum;
-		bitcnt = sum << 3;	/* need the size in bits */
-
-		tmp = bitcnt >> 16;
-		dprintk(3,
-			"%s: code: csize=%d, tot=%d, bit=%ld, highbits=%ld\n",
-			ptr->name, sum, ptr->real_code_vol, bitcnt, tmp);
-		zr36050_write(ptr, ZR050_TCV_NET_HI, tmp >> 8);
-		zr36050_write(ptr, ZR050_TCV_NET_MH, tmp & 0xff);
-		tmp = bitcnt & 0xffff;
-		zr36050_write(ptr, ZR050_TCV_NET_ML, tmp >> 8);
-		zr36050_write(ptr, ZR050_TCV_NET_LO, tmp & 0xff);
-
-		bitcnt -= bitcnt >> 7;	// bits without stuffing
-		bitcnt -= ((bitcnt * 5) >> 6);	// bits without eob
-
-		tmp = bitcnt >> 16;
-		dprintk(3, "%s: code: nettobit=%ld, highnettobits=%ld\n",
-			ptr->name, bitcnt, tmp);
-		zr36050_write(ptr, ZR050_TCV_DATA_HI, tmp >> 8);
-		zr36050_write(ptr, ZR050_TCV_DATA_MH, tmp & 0xff);
-		tmp = bitcnt & 0xffff;
-		zr36050_write(ptr, ZR050_TCV_DATA_ML, tmp >> 8);
-		zr36050_write(ptr, ZR050_TCV_DATA_LO, tmp & 0xff);
-
-		/* compression setup with or without bitrate control */
-		zr36050_write(ptr, ZR050_MODE,
-			      ZR050_MO_COMP | ZR050_MO_PASS2 |
-			      (ptr->bitrate_ctrl ? ZR050_MO_BRC : 0));
-
-		/* this headers seem to deliver "valid AVI" jpeg frames */
-		zr36050_write(ptr, ZR050_MARKERS_EN,
-			      ZR050_ME_DQT | ZR050_ME_DHT |
-			      ((ptr->app.len > 0) ? ZR050_ME_APP : 0) |
-			      ((ptr->com.len > 0) ? ZR050_ME_COM : 0));
-	} else {
-		dprintk(2, "%s: EXPANSION SETUP\n", ptr->name);
-
-		/* 050 communicates with 055 in master mode */
-		zr36050_write(ptr, ZR050_HARDWARE,
-			      ZR050_HW_MSTR | ZR050_HW_CFIS_2_CLK);
-
-		/* encoding table preload */
-		zr36050_write(ptr, ZR050_MODE, ZR050_MO_TLM);
-
-		/* disable all IRQs */
-		zr36050_write(ptr, ZR050_INT_REQ_0, 0);
-		zr36050_write(ptr, ZR050_INT_REQ_1, 3);	// low 2 bits always 1
-
-		dprintk(3, "%s: write DHT\n", ptr->name);
-		zr36050_pushit(ptr, ZR050_DHT_IDX, sizeof(zr36050_dht),
-			       zr36050_dht);
-
-		/* do the internal huffman table preload */
-		zr36050_write(ptr, ZR050_MARKERS_EN, ZR050_ME_DHTI);
-
-		zr36050_write(ptr, ZR050_GO, 1);	// launch codec
-		zr36050_wait_end(ptr);
-		dprintk(2, "%s: Status after table preload: 0x%02x\n",
-			ptr->name, ptr->status1);
-
-		if ((ptr->status1 & 0x4) == 0) {
-			dprintk(1, KERN_ERR "%s: init aborted!\n",
-				ptr->name);
-			return;	// something is wrong, its timed out!!!!
-		}
-
-		/* setup misc. data for expansion */
-		zr36050_write(ptr, ZR050_MODE, 0);
-		zr36050_write(ptr, ZR050_MARKERS_EN, 0);
-	}
-
-	/* adr on selected, to allow GO from master */
-	zr36050_read(ptr, 0);
-}
-
-/* =========================================================================
-   CODEC API FUNCTIONS
-
-   this functions are accessed by the master via the API structure
-   ========================================================================= */
-
-/* set compression/expansion mode and launches codec -
-   this should be the last call from the master before starting processing */
-static int
-zr36050_set_mode (struct videocodec *codec,
-		  int                mode)
-{
-	struct zr36050 *ptr = (struct zr36050 *) codec->data;
-
-	dprintk(2, "%s: set_mode %d call\n", ptr->name, mode);
-
-	if ((mode != CODEC_DO_EXPANSION) && (mode != CODEC_DO_COMPRESSION))
-		return -EINVAL;
-
-	ptr->mode = mode;
-	zr36050_init(ptr);
-
-	return 0;
-}
-
-/* set picture size (norm is ignored as the codec doesn't know about it) */
-static int
-zr36050_set_video (struct videocodec   *codec,
-		   struct tvnorm       *norm,
-		   struct vfe_settings *cap,
-		   struct vfe_polarity *pol)
-{
-	struct zr36050 *ptr = (struct zr36050 *) codec->data;
-	int size;
-
-	dprintk(2, "%s: set_video %d.%d, %d/%d-%dx%d (0x%x) q%d call\n",
-		ptr->name, norm->HStart, norm->VStart,
-		cap->x, cap->y, cap->width, cap->height,
-		cap->decimation, cap->quality);
-	/* if () return -EINVAL;
-	 * trust the master driver that it knows what it does - so
-	 * we allow invalid startx/y and norm for now ... */
-	ptr->width = cap->width / (cap->decimation & 0xff);
-	ptr->height = cap->height / ((cap->decimation >> 8) & 0xff);
-
-	/* (KM) JPEG quality */
-	size = ptr->width * ptr->height;
-	size *= 16; /* size in bits */
-	/* apply quality setting */
-	size = size * cap->quality / 200;
-
-	/* Minimum: 1kb */
-	if (size < 8192)
-		size = 8192;
-	/* Maximum: 7/8 of code buffer */
-	if (size > ptr->total_code_vol * 7)
-		size = ptr->total_code_vol * 7;
-
-	ptr->real_code_vol = size >> 3; /* in bytes */
-
-	/* Set max_block_vol here (previously in zr36050_init, moved
-	 * here for consistency with zr36060 code */
-	zr36050_write(ptr, ZR050_MBCV, ptr->max_block_vol);
-
-	return 0;
-}
-
-/* additional control functions */
-static int
-zr36050_control (struct videocodec *codec,
-		 int                type,
-		 int                size,
-		 void              *data)
-{
-	struct zr36050 *ptr = (struct zr36050 *) codec->data;
-	int *ival = (int *) data;
-
-	dprintk(2, "%s: control %d call with %d byte\n", ptr->name, type,
-		size);
-
-	switch (type) {
-	case CODEC_G_STATUS:	/* get last status */
-		if (size != sizeof(int))
-			return -EFAULT;
-		zr36050_read_status1(ptr);
-		*ival = ptr->status1;
-		break;
-
-	case CODEC_G_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = CODEC_MODE_BJPG;
-		break;
-
-	case CODEC_S_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		if (*ival != CODEC_MODE_BJPG)
-			return -EINVAL;
-		/* not needed, do nothing */
-		return 0;
-
-	case CODEC_G_VFE:
-	case CODEC_S_VFE:
-		/* not needed, do nothing */
-		return 0;
-
-	case CODEC_S_MMAP:
-		/* not available, give an error */
-		return -ENXIO;
-
-	case CODEC_G_JPEG_TDS_BYTE:	/* get target volume in byte */
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = ptr->total_code_vol;
-		break;
-
-	case CODEC_S_JPEG_TDS_BYTE:	/* get target volume in byte */
-		if (size != sizeof(int))
-			return -EFAULT;
-		ptr->total_code_vol = *ival;
-		/* (Kieran Morrissey)
-		 * code copied from zr36060.c to ensure proper bitrate */
-		ptr->real_code_vol = (ptr->total_code_vol * 6) >> 3;
-		break;
-
-	case CODEC_G_JPEG_SCALE:	/* get scaling factor */
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = zr36050_read_scalefactor(ptr);
-		break;
-
-	case CODEC_S_JPEG_SCALE:	/* set scaling factor */
-		if (size != sizeof(int))
-			return -EFAULT;
-		ptr->scalefact = *ival;
-		break;
-
-	case CODEC_G_JPEG_APP_DATA: {	/* get appn marker data */
-		struct jpeg_app_marker *app = data;
-
-		if (size != sizeof(struct jpeg_app_marker))
-			return -EFAULT;
-
-		*app = ptr->app;
-		break;
-	}
-
-	case CODEC_S_JPEG_APP_DATA: {	 /* set appn marker data */
-		struct jpeg_app_marker *app = data;
-
-		if (size != sizeof(struct jpeg_app_marker))
-			return -EFAULT;
-
-		ptr->app = *app;
-		break;
-	}
-
-	case CODEC_G_JPEG_COM_DATA: {	/* get comment marker data */
-		struct jpeg_com_marker *com = data;
-
-		if (size != sizeof(struct jpeg_com_marker))
-			return -EFAULT;
-
-		*com = ptr->com;
-		break;
-	}
-
-	case CODEC_S_JPEG_COM_DATA: {	/* set comment marker data */
-		struct jpeg_com_marker *com = data;
-
-		if (size != sizeof(struct jpeg_com_marker))
-			return -EFAULT;
-
-		ptr->com = *com;
-		break;
-	}
-
-	default:
-		return -EINVAL;
-	}
-
-	return size;
-}
-
-/* =========================================================================
-   Exit and unregister function:
-
-   Deinitializes Zoran's JPEG processor
-   ========================================================================= */
-
-static int
-zr36050_unset (struct videocodec *codec)
-{
-	struct zr36050 *ptr = codec->data;
-
-	if (ptr) {
-		/* do wee need some codec deinit here, too ???? */
-
-		dprintk(1, "%s: finished codec #%d\n", ptr->name,
-			ptr->num);
-		kfree(ptr);
-		codec->data = NULL;
-
-		zr36050_codecs--;
-		return 0;
-	}
-
-	return -EFAULT;
-}
-
-/* =========================================================================
-   Setup and registry function:
-
-   Initializes Zoran's JPEG processor
-
-   Also sets pixel size, average code size, mode (compr./decompr.)
-   (the given size is determined by the processor with the video interface)
-   ========================================================================= */
-
-static int
-zr36050_setup (struct videocodec *codec)
-{
-	struct zr36050 *ptr;
-	int res;
-
-	dprintk(2, "zr36050: initializing MJPEG subsystem #%d.\n",
-		zr36050_codecs);
-
-	if (zr36050_codecs == MAX_CODECS) {
-		dprintk(1,
-			KERN_ERR "zr36050: Can't attach more codecs!\n");
-		return -ENOSPC;
-	}
-	//mem structure init
-	codec->data = ptr = kzalloc(sizeof(struct zr36050), GFP_KERNEL);
-	if (NULL == ptr) {
-		dprintk(1, KERN_ERR "zr36050: Can't get enough memory!\n");
-		return -ENOMEM;
-	}
-
-	snprintf(ptr->name, sizeof(ptr->name), "zr36050[%d]",
-		 zr36050_codecs);
-	ptr->num = zr36050_codecs++;
-	ptr->codec = codec;
-
-	//testing
-	res = zr36050_basic_test(ptr);
-	if (res < 0) {
-		zr36050_unset(codec);
-		return res;
-	}
-	//final setup
-	memcpy(ptr->h_samp_ratio, zr36050_decimation_h, 8);
-	memcpy(ptr->v_samp_ratio, zr36050_decimation_v, 8);
-
-	ptr->bitrate_ctrl = 0;	/* 0 or 1 - fixed file size flag
-				 * (what is the difference?) */
-	ptr->mode = CODEC_DO_COMPRESSION;
-	ptr->width = 384;
-	ptr->height = 288;
-	ptr->total_code_vol = 16000;
-	ptr->max_block_vol = 240;
-	ptr->scalefact = 0x100;
-	ptr->dri = 1;
-
-	/* no app/com marker by default */
-	ptr->app.appn = 0;
-	ptr->app.len = 0;
-	ptr->com.len = 0;
-
-	zr36050_init(ptr);
-
-	dprintk(1, KERN_INFO "%s: codec attached and running\n",
-		ptr->name);
-
-	return 0;
-}
-
-static const struct videocodec zr36050_codec = {
-	.owner = THIS_MODULE,
-	.name = "zr36050",
-	.magic = 0L,		// magic not used
-	.flags =
-	    CODEC_FLAG_JPEG | CODEC_FLAG_HARDWARE | CODEC_FLAG_ENCODER |
-	    CODEC_FLAG_DECODER,
-	.type = CODEC_TYPE_ZR36050,
-	.setup = zr36050_setup,	// functionality
-	.unset = zr36050_unset,
-	.set_mode = zr36050_set_mode,
-	.set_video = zr36050_set_video,
-	.control = zr36050_control,
-	// others are not used
-};
-
-/* =========================================================================
-   HOOK IN DRIVER AS KERNEL MODULE
-   ========================================================================= */
-
-static int __init
-zr36050_init_module (void)
-{
-	//dprintk(1, "ZR36050 driver %s\n",ZR050_VERSION);
-	zr36050_codecs = 0;
-	return videocodec_register(&zr36050_codec);
-}
-
-static void __exit
-zr36050_cleanup_module (void)
-{
-	if (zr36050_codecs) {
-		dprintk(1,
-			"zr36050: something's wrong - %d codecs left somehow.\n",
-			zr36050_codecs);
-	}
-	videocodec_unregister(&zr36050_codec);
-}
-
-module_init(zr36050_init_module);
-module_exit(zr36050_cleanup_module);
-
-MODULE_AUTHOR("Wolfgang Scherr <scherr@net4you.at>");
-MODULE_DESCRIPTION("Driver module for ZR36050 jpeg processors "
-		   ZR050_VERSION);
-MODULE_LICENSE("GPL");
diff --git a/drivers/media/pci/zoran/zr36050.h b/drivers/media/pci/zoran/zr36050.h
deleted file mode 100644
index 9236486d3c2b..000000000000
--- a/drivers/media/pci/zoran/zr36050.h
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Zoran ZR36050 basic configuration functions - header file
- *
- * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
- *
- * $Id: zr36050.h,v 1.1.2.2 2003/01/14 21:18:22 rbultje Exp $
- *
- * ------------------------------------------------------------------------
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * ------------------------------------------------------------------------
- */
-
-#ifndef ZR36050_H
-#define ZR36050_H
-
-#include "videocodec.h"
-
-/* data stored for each zoran jpeg codec chip */
-struct zr36050 {
-	char name[32];
-	int num;
-	/* io datastructure */
-	struct videocodec *codec;
-	// last coder status
-	__u8 status1;
-	// actual coder setup
-	int mode;
-
-	__u16 width;
-	__u16 height;
-
-	__u16 bitrate_ctrl;
-
-	__u32 total_code_vol;
-	__u32 real_code_vol;
-	__u16 max_block_vol;
-
-	__u8 h_samp_ratio[8];
-	__u8 v_samp_ratio[8];
-	__u16 scalefact;
-	__u16 dri;
-
-	/* com/app marker */
-	struct jpeg_com_marker com;
-	struct jpeg_app_marker app;
-};
-
-/* zr36050 register addresses */
-#define ZR050_GO                  0x000
-#define ZR050_HARDWARE            0x002
-#define ZR050_MODE                0x003
-#define ZR050_OPTIONS             0x004
-#define ZR050_MBCV                0x005
-#define ZR050_MARKERS_EN          0x006
-#define ZR050_INT_REQ_0           0x007
-#define ZR050_INT_REQ_1           0x008
-#define ZR050_TCV_NET_HI          0x009
-#define ZR050_TCV_NET_MH          0x00a
-#define ZR050_TCV_NET_ML          0x00b
-#define ZR050_TCV_NET_LO          0x00c
-#define ZR050_TCV_DATA_HI         0x00d
-#define ZR050_TCV_DATA_MH         0x00e
-#define ZR050_TCV_DATA_ML         0x00f
-#define ZR050_TCV_DATA_LO         0x010
-#define ZR050_SF_HI               0x011
-#define ZR050_SF_LO               0x012
-#define ZR050_AF_HI               0x013
-#define ZR050_AF_M                0x014
-#define ZR050_AF_LO               0x015
-#define ZR050_ACV_HI              0x016
-#define ZR050_ACV_MH              0x017
-#define ZR050_ACV_ML              0x018
-#define ZR050_ACV_LO              0x019
-#define ZR050_ACT_HI              0x01a
-#define ZR050_ACT_MH              0x01b
-#define ZR050_ACT_ML              0x01c
-#define ZR050_ACT_LO              0x01d
-#define ZR050_ACV_TRUN_HI         0x01e
-#define ZR050_ACV_TRUN_MH         0x01f
-#define ZR050_ACV_TRUN_ML         0x020
-#define ZR050_ACV_TRUN_LO         0x021
-#define ZR050_STATUS_0            0x02e
-#define ZR050_STATUS_1            0x02f
-
-#define ZR050_SOF_IDX             0x040
-#define ZR050_SOS1_IDX            0x07a
-#define ZR050_SOS2_IDX            0x08a
-#define ZR050_SOS3_IDX            0x09a
-#define ZR050_SOS4_IDX            0x0aa
-#define ZR050_DRI_IDX             0x0c0
-#define ZR050_DNL_IDX             0x0c6
-#define ZR050_DQT_IDX             0x0cc
-#define ZR050_DHT_IDX             0x1d4
-#define ZR050_APP_IDX             0x380
-#define ZR050_COM_IDX             0x3c0
-
-/* zr36050 hardware register bits */
-
-#define ZR050_HW_BSWD                0x80
-#define ZR050_HW_MSTR                0x40
-#define ZR050_HW_DMA                 0x20
-#define ZR050_HW_CFIS_1_CLK          0x00
-#define ZR050_HW_CFIS_2_CLK          0x04
-#define ZR050_HW_CFIS_3_CLK          0x08
-#define ZR050_HW_CFIS_4_CLK          0x0C
-#define ZR050_HW_CFIS_5_CLK          0x10
-#define ZR050_HW_CFIS_6_CLK          0x14
-#define ZR050_HW_CFIS_7_CLK          0x18
-#define ZR050_HW_CFIS_8_CLK          0x1C
-#define ZR050_HW_BELE                0x01
-
-/* zr36050 mode register bits */
-
-#define ZR050_MO_COMP                0x80
-#define ZR050_MO_ATP                 0x40
-#define ZR050_MO_PASS2               0x20
-#define ZR050_MO_TLM                 0x10
-#define ZR050_MO_DCONLY              0x08
-#define ZR050_MO_BRC                 0x04
-
-#define ZR050_MO_ATP                 0x40
-#define ZR050_MO_PASS2               0x20
-#define ZR050_MO_TLM                 0x10
-#define ZR050_MO_DCONLY              0x08
-
-/* zr36050 option register bits */
-
-#define ZR050_OP_NSCN_1              0x00
-#define ZR050_OP_NSCN_2              0x20
-#define ZR050_OP_NSCN_3              0x40
-#define ZR050_OP_NSCN_4              0x60
-#define ZR050_OP_NSCN_5              0x80
-#define ZR050_OP_NSCN_6              0xA0
-#define ZR050_OP_NSCN_7              0xC0
-#define ZR050_OP_NSCN_8              0xE0
-#define ZR050_OP_OVF                 0x10
-
-
-/* zr36050 markers-enable register bits */
-
-#define ZR050_ME_APP                 0x80
-#define ZR050_ME_COM                 0x40
-#define ZR050_ME_DRI                 0x20
-#define ZR050_ME_DQT                 0x10
-#define ZR050_ME_DHT                 0x08
-#define ZR050_ME_DNL                 0x04
-#define ZR050_ME_DQTI                0x02
-#define ZR050_ME_DHTI                0x01
-
-/* zr36050 status0/1 register bit masks */
-
-#define ZR050_ST_RST_MASK            0x20
-#define ZR050_ST_SOF_MASK            0x02
-#define ZR050_ST_SOS_MASK            0x02
-#define ZR050_ST_DATRDY_MASK         0x80
-#define ZR050_ST_MRKDET_MASK         0x40
-#define ZR050_ST_RFM_MASK            0x10
-#define ZR050_ST_RFD_MASK            0x08
-#define ZR050_ST_END_MASK            0x04
-#define ZR050_ST_TCVOVF_MASK         0x02
-#define ZR050_ST_DATOVF_MASK         0x01
-
-/* pixel component idx */
-
-#define ZR050_Y_COMPONENT         0
-#define ZR050_U_COMPONENT         1
-#define ZR050_V_COMPONENT         2
-
-#endif				/*fndef ZR36050_H */
diff --git a/drivers/media/pci/zoran/zr36057.h b/drivers/media/pci/zoran/zr36057.h
deleted file mode 100644
index c8acb21dcb5c..000000000000
--- a/drivers/media/pci/zoran/zr36057.h
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * zr36057.h - zr36057 register offsets
- *
- * Copyright (C) 1998 Dave Perks <dperks@ibm.net>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#ifndef _ZR36057_H_
-#define _ZR36057_H_
-
-
-/* Zoran ZR36057 registers */
-
-#define ZR36057_VFEHCR          0x000	/* Video Front End, Horizontal Configuration Register */
-#define ZR36057_VFEHCR_HSPol            (1<<30)
-#define ZR36057_VFEHCR_HStart           10
-#define ZR36057_VFEHCR_HEnd		0
-#define ZR36057_VFEHCR_Hmask		0x3ff
-
-#define ZR36057_VFEVCR          0x004	/* Video Front End, Vertical Configuration Register */
-#define ZR36057_VFEVCR_VSPol            (1<<30)
-#define ZR36057_VFEVCR_VStart           10
-#define ZR36057_VFEVCR_VEnd		0
-#define ZR36057_VFEVCR_Vmask		0x3ff
-
-#define ZR36057_VFESPFR         0x008	/* Video Front End, Scaler and Pixel Format Register */
-#define ZR36057_VFESPFR_ExtFl           (1<<26)
-#define ZR36057_VFESPFR_TopField        (1<<25)
-#define ZR36057_VFESPFR_VCLKPol         (1<<24)
-#define ZR36057_VFESPFR_HFilter         21
-#define ZR36057_VFESPFR_HorDcm          14
-#define ZR36057_VFESPFR_VerDcm          8
-#define ZR36057_VFESPFR_DispMode        6
-#define ZR36057_VFESPFR_YUV422          (0<<3)
-#define ZR36057_VFESPFR_RGB888          (1<<3)
-#define ZR36057_VFESPFR_RGB565          (2<<3)
-#define ZR36057_VFESPFR_RGB555          (3<<3)
-#define ZR36057_VFESPFR_ErrDif          (1<<2)
-#define ZR36057_VFESPFR_Pack24          (1<<1)
-#define ZR36057_VFESPFR_LittleEndian    (1<<0)
-
-#define ZR36057_VDTR            0x00c	/* Video Display "Top" Register */
-
-#define ZR36057_VDBR            0x010	/* Video Display "Bottom" Register */
-
-#define ZR36057_VSSFGR          0x014	/* Video Stride, Status, and Frame Grab Register */
-#define ZR36057_VSSFGR_DispStride       16
-#define ZR36057_VSSFGR_VidOvf           (1<<8)
-#define ZR36057_VSSFGR_SnapShot         (1<<1)
-#define ZR36057_VSSFGR_FrameGrab        (1<<0)
-
-#define ZR36057_VDCR            0x018	/* Video Display Configuration Register */
-#define ZR36057_VDCR_VidEn              (1<<31)
-#define ZR36057_VDCR_MinPix             24
-#define ZR36057_VDCR_Triton             (1<<24)
-#define ZR36057_VDCR_VidWinHt           12
-#define ZR36057_VDCR_VidWinWid          0
-
-#define ZR36057_MMTR            0x01c	/* Masking Map "Top" Register */
-
-#define ZR36057_MMBR            0x020	/* Masking Map "Bottom" Register */
-
-#define ZR36057_OCR             0x024	/* Overlay Control Register */
-#define ZR36057_OCR_OvlEnable           (1 << 15)
-#define ZR36057_OCR_MaskStride          0
-
-#define ZR36057_SPGPPCR         0x028	/* System, PCI, and General Purpose Pins Control Register */
-#define ZR36057_SPGPPCR_SoftReset	(1<<24)
-
-#define ZR36057_GPPGCR1         0x02c	/* General Purpose Pins and GuestBus Control Register (1) */
-
-#define ZR36057_MCSAR           0x030	/* MPEG Code Source Address Register */
-
-#define ZR36057_MCTCR           0x034	/* MPEG Code Transfer Control Register */
-#define ZR36057_MCTCR_CodTime           (1 << 30)
-#define ZR36057_MCTCR_CEmpty            (1 << 29)
-#define ZR36057_MCTCR_CFlush            (1 << 28)
-#define ZR36057_MCTCR_CodGuestID	20
-#define ZR36057_MCTCR_CodGuestReg	16
-
-#define ZR36057_MCMPR           0x038	/* MPEG Code Memory Pointer Register */
-
-#define ZR36057_ISR             0x03c	/* Interrupt Status Register */
-#define ZR36057_ISR_GIRQ1               (1<<30)
-#define ZR36057_ISR_GIRQ0               (1<<29)
-#define ZR36057_ISR_CodRepIRQ           (1<<28)
-#define ZR36057_ISR_JPEGRepIRQ          (1<<27)
-
-#define ZR36057_ICR             0x040	/* Interrupt Control Register */
-#define ZR36057_ICR_GIRQ1               (1<<30)
-#define ZR36057_ICR_GIRQ0               (1<<29)
-#define ZR36057_ICR_CodRepIRQ           (1<<28)
-#define ZR36057_ICR_JPEGRepIRQ          (1<<27)
-#define ZR36057_ICR_IntPinEn            (1<<24)
-
-#define ZR36057_I2CBR           0x044	/* I2C Bus Register */
-#define ZR36057_I2CBR_SDA		(1<<1)
-#define ZR36057_I2CBR_SCL		(1<<0)
-
-#define ZR36057_JMC             0x100	/* JPEG Mode and Control */
-#define ZR36057_JMC_JPG                 (1 << 31)
-#define ZR36057_JMC_JPGExpMode          (0 << 29)
-#define ZR36057_JMC_JPGCmpMode          (1 << 29)
-#define ZR36057_JMC_MJPGExpMode         (2 << 29)
-#define ZR36057_JMC_MJPGCmpMode         (3 << 29)
-#define ZR36057_JMC_RTBUSY_FB           (1 << 6)
-#define ZR36057_JMC_Go_en               (1 << 5)
-#define ZR36057_JMC_SyncMstr            (1 << 4)
-#define ZR36057_JMC_Fld_per_buff        (1 << 3)
-#define ZR36057_JMC_VFIFO_FB            (1 << 2)
-#define ZR36057_JMC_CFIFO_FB            (1 << 1)
-#define ZR36057_JMC_Stll_LitEndian      (1 << 0)
-
-#define ZR36057_JPC             0x104	/* JPEG Process Control */
-#define ZR36057_JPC_P_Reset             (1 << 7)
-#define ZR36057_JPC_CodTrnsEn           (1 << 5)
-#define ZR36057_JPC_Active              (1 << 0)
-
-#define ZR36057_VSP             0x108	/* Vertical Sync Parameters */
-#define ZR36057_VSP_VsyncSize           16
-#define ZR36057_VSP_FrmTot              0
-
-#define ZR36057_HSP             0x10c	/* Horizontal Sync Parameters */
-#define ZR36057_HSP_HsyncStart          16
-#define ZR36057_HSP_LineTot             0
-
-#define ZR36057_FHAP            0x110	/* Field Horizontal Active Portion */
-#define ZR36057_FHAP_NAX                16
-#define ZR36057_FHAP_PAX                0
-
-#define ZR36057_FVAP            0x114	/* Field Vertical Active Portion */
-#define ZR36057_FVAP_NAY                16
-#define ZR36057_FVAP_PAY                0
-
-#define ZR36057_FPP             0x118	/* Field Process Parameters */
-#define ZR36057_FPP_Odd_Even            (1 << 0)
-
-#define ZR36057_JCBA            0x11c	/* JPEG Code Base Address */
-
-#define ZR36057_JCFT            0x120	/* JPEG Code FIFO Threshold */
-
-#define ZR36057_JCGI            0x124	/* JPEG Codec Guest ID */
-#define ZR36057_JCGI_JPEGuestID         4
-#define ZR36057_JCGI_JPEGuestReg        0
-
-#define ZR36057_GCR2            0x12c	/* GuestBus Control Register (2) */
-
-#define ZR36057_POR             0x200	/* Post Office Register */
-#define ZR36057_POR_POPen               (1<<25)
-#define ZR36057_POR_POTime              (1<<24)
-#define ZR36057_POR_PODir               (1<<23)
-
-#define ZR36057_STR             0x300	/* "Still" Transfer Register */
-
-#endif
diff --git a/drivers/media/pci/zoran/zr36060.c b/drivers/media/pci/zoran/zr36060.c
deleted file mode 100644
index 2c2e8130fc96..000000000000
--- a/drivers/media/pci/zoran/zr36060.c
+++ /dev/null
@@ -1,1006 +0,0 @@
-/*
- * Zoran ZR36060 basic configuration functions
- *
- * Copyright (C) 2002 Laurent Pinchart <laurent.pinchart@skynet.be>
- *
- * $Id: zr36060.c,v 1.1.2.22 2003/05/06 09:35:36 rbultje Exp $
- *
- * ------------------------------------------------------------------------
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * ------------------------------------------------------------------------
- */
-
-#define ZR060_VERSION "v0.7"
-
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-
-#include <linux/types.h>
-#include <linux/wait.h>
-
-/* I/O commands, error codes */
-#include <asm/io.h>
-
-/* headerfile of this module */
-#include "zr36060.h"
-
-/* codec io API */
-#include "videocodec.h"
-
-/* it doesn't make sense to have more than 20 or so,
-  just to prevent some unwanted loops */
-#define MAX_CODECS 20
-
-/* amount of chips attached via this driver */
-static int zr36060_codecs;
-
-static bool low_bitrate;
-module_param(low_bitrate, bool, 0);
-MODULE_PARM_DESC(low_bitrate, "Buz compatibility option, halves bitrate");
-
-/* debugging is available via module parameter */
-static int debug;
-module_param(debug, int, 0);
-MODULE_PARM_DESC(debug, "Debug level (0-4)");
-
-#define dprintk(num, format, args...) \
-	do { \
-		if (debug >= num) \
-			printk(format, ##args); \
-	} while (0)
-
-/* =========================================================================
-   Local hardware I/O functions:
-
-   read/write via codec layer (registers are located in the master device)
-   ========================================================================= */
-
-/* read and write functions */
-static u8
-zr36060_read (struct zr36060 *ptr,
-	      u16             reg)
-{
-	u8 value = 0;
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->readreg)
-		value = (ptr->codec->master_data->readreg(ptr->codec,
-							  reg)) & 0xff;
-	else
-		dprintk(1,
-			KERN_ERR "%s: invalid I/O setup, nothing read!\n",
-			ptr->name);
-
-	//dprintk(4, "%s: reading from 0x%04x: %02x\n",ptr->name,reg,value);
-
-	return value;
-}
-
-static void
-zr36060_write(struct zr36060 *ptr,
-	      u16             reg,
-	      u8              value)
-{
-	//dprintk(4, "%s: writing 0x%02x to 0x%04x\n",ptr->name,value,reg);
-	dprintk(4, "0x%02x @0x%04x\n", value, reg);
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->writereg)
-		ptr->codec->master_data->writereg(ptr->codec, reg, value);
-	else
-		dprintk(1,
-			KERN_ERR
-			"%s: invalid I/O setup, nothing written!\n",
-			ptr->name);
-}
-
-/* =========================================================================
-   Local helper function:
-
-   status read
-   ========================================================================= */
-
-/* status is kept in datastructure */
-static u8
-zr36060_read_status (struct zr36060 *ptr)
-{
-	ptr->status = zr36060_read(ptr, ZR060_CFSR);
-
-	zr36060_read(ptr, 0);
-	return ptr->status;
-}
-
-/* =========================================================================
-   Local helper function:
-
-   scale factor read
-   ========================================================================= */
-
-/* scale factor is kept in datastructure */
-static u16
-zr36060_read_scalefactor (struct zr36060 *ptr)
-{
-	ptr->scalefact = (zr36060_read(ptr, ZR060_SF_HI) << 8) |
-			 (zr36060_read(ptr, ZR060_SF_LO) & 0xFF);
-
-	/* leave 0 selected for an eventually GO from master */
-	zr36060_read(ptr, 0);
-	return ptr->scalefact;
-}
-
-/* =========================================================================
-   Local helper function:
-
-   wait if codec is ready to proceed (end of processing) or time is over
-   ========================================================================= */
-
-static void
-zr36060_wait_end (struct zr36060 *ptr)
-{
-	int i = 0;
-
-	while (zr36060_read_status(ptr) & ZR060_CFSR_Busy) {
-		udelay(1);
-		if (i++ > 200000) {	// 200ms, there is for sure something wrong!!!
-			dprintk(1,
-				"%s: timeout at wait_end (last status: 0x%02x)\n",
-				ptr->name, ptr->status);
-			break;
-		}
-	}
-}
-
-/* =========================================================================
-   Local helper function:
-
-   basic test of "connectivity", writes/reads to/from memory the SOF marker
-   ========================================================================= */
-
-static int
-zr36060_basic_test (struct zr36060 *ptr)
-{
-	if ((zr36060_read(ptr, ZR060_IDR_DEV) != 0x33) &&
-	    (zr36060_read(ptr, ZR060_IDR_REV) != 0x01)) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, can't connect to jpeg processor!\n",
-			ptr->name);
-		return -ENXIO;
-	}
-
-	zr36060_wait_end(ptr);
-	if (ptr->status & ZR060_CFSR_Busy) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, jpeg processor failed (end flag)!\n",
-			ptr->name);
-		return -EBUSY;
-	}
-
-	return 0;		/* looks good! */
-}
-
-/* =========================================================================
-   Local helper function:
-
-   simple loop for pushing the init datasets
-   ========================================================================= */
-
-static int
-zr36060_pushit (struct zr36060 *ptr,
-		u16             startreg,
-		u16             len,
-		const char     *data)
-{
-	int i = 0;
-
-	dprintk(4, "%s: write data block to 0x%04x (len=%d)\n", ptr->name,
-		startreg, len);
-	while (i < len) {
-		zr36060_write(ptr, startreg++, data[i++]);
-	}
-
-	return i;
-}
-
-/* =========================================================================
-   Basic datasets:
-
-   jpeg baseline setup data (you find it on lots places in internet, or just
-   extract it from any regular .jpg image...)
-
-   Could be variable, but until it's not needed it they are just fixed to save
-   memory. Otherwise expand zr36060 structure with arrays, push the values to
-   it and initialize from there, as e.g. the linux zr36057/60 driver does it.
-   ========================================================================= */
-
-static const char zr36060_dqt[0x86] = {
-	0xff, 0xdb,		//Marker: DQT
-	0x00, 0x84,		//Length: 2*65+2
-	0x00,			//Pq,Tq first table
-	0x10, 0x0b, 0x0c, 0x0e, 0x0c, 0x0a, 0x10, 0x0e,
-	0x0d, 0x0e, 0x12, 0x11, 0x10, 0x13, 0x18, 0x28,
-	0x1a, 0x18, 0x16, 0x16, 0x18, 0x31, 0x23, 0x25,
-	0x1d, 0x28, 0x3a, 0x33, 0x3d, 0x3c, 0x39, 0x33,
-	0x38, 0x37, 0x40, 0x48, 0x5c, 0x4e, 0x40, 0x44,
-	0x57, 0x45, 0x37, 0x38, 0x50, 0x6d, 0x51, 0x57,
-	0x5f, 0x62, 0x67, 0x68, 0x67, 0x3e, 0x4d, 0x71,
-	0x79, 0x70, 0x64, 0x78, 0x5c, 0x65, 0x67, 0x63,
-	0x01,			//Pq,Tq second table
-	0x11, 0x12, 0x12, 0x18, 0x15, 0x18, 0x2f, 0x1a,
-	0x1a, 0x2f, 0x63, 0x42, 0x38, 0x42, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63
-};
-
-static const char zr36060_dht[0x1a4] = {
-	0xff, 0xc4,		//Marker: DHT
-	0x01, 0xa2,		//Length: 2*AC, 2*DC
-	0x00,			//DC first table
-	0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
-	0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
-	0x01,			//DC second table
-	0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
-	0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
-	0x10,			//AC first table
-	0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03,
-	0x05, 0x05, 0x04, 0x04, 0x00, 0x00,
-	0x01, 0x7D, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11,
-	0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61,
-	0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1,
-	0x08, 0x23, 0x42, 0xB1, 0xC1, 0x15, 0x52, 0xD1, 0xF0, 0x24,
-	0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17,
-	0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x34,
-	0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
-	0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
-	0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
-	0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
-	0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88,
-	0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99,
-	0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8,
-	0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9,
-	0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8,
-	0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9,
-	0xDA, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
-	0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
-	0xF8, 0xF9, 0xFA,
-	0x11,			//AC second table
-	0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04,
-	0x07, 0x05, 0x04, 0x04, 0x00, 0x01,
-	0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04,
-	0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71,
-	0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91,
-	0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, 0x15, 0x62,
-	0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25,
-	0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, 0x27, 0x28, 0x29, 0x2A,
-	0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
-	0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
-	0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
-	0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
-	0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
-	0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98,
-	0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
-	0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8,
-	0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
-	0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8,
-	0xD9, 0xDA, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
-	0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8,
-	0xF9, 0xFA
-};
-
-/* jpeg baseline setup, this is just fixed in this driver (YUV pictures) */
-#define NO_OF_COMPONENTS          0x3	//Y,U,V
-#define BASELINE_PRECISION        0x8	//MCU size (?)
-static const char zr36060_tq[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's QT
-static const char zr36060_td[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's DC
-static const char zr36060_ta[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's AC
-
-/* horizontal 422 decimation setup (maybe we support 411 or so later, too) */
-static const char zr36060_decimation_h[8] = { 2, 1, 1, 0, 0, 0, 0, 0 };
-static const char zr36060_decimation_v[8] = { 1, 1, 1, 0, 0, 0, 0, 0 };
-
-/* =========================================================================
-   Local helper functions:
-
-   calculation and setup of parameter-dependent JPEG baseline segments
-   (needed for compression only)
-   ========================================================================= */
-
-/* ------------------------------------------------------------------------- */
-
-/* SOF (start of frame) segment depends on width, height and sampling ratio
-			 of each color component */
-
-static int
-zr36060_set_sof (struct zr36060 *ptr)
-{
-	char sof_data[34];	// max. size of register set
-	int i;
-
-	dprintk(3, "%s: write SOF (%dx%d, %d components)\n", ptr->name,
-		ptr->width, ptr->height, NO_OF_COMPONENTS);
-	sof_data[0] = 0xff;
-	sof_data[1] = 0xc0;
-	sof_data[2] = 0x00;
-	sof_data[3] = (3 * NO_OF_COMPONENTS) + 8;
-	sof_data[4] = BASELINE_PRECISION;	// only '8' possible with zr36060
-	sof_data[5] = (ptr->height) >> 8;
-	sof_data[6] = (ptr->height) & 0xff;
-	sof_data[7] = (ptr->width) >> 8;
-	sof_data[8] = (ptr->width) & 0xff;
-	sof_data[9] = NO_OF_COMPONENTS;
-	for (i = 0; i < NO_OF_COMPONENTS; i++) {
-		sof_data[10 + (i * 3)] = i;	// index identifier
-		sof_data[11 + (i * 3)] = (ptr->h_samp_ratio[i] << 4) |
-					 (ptr->v_samp_ratio[i]); // sampling ratios
-		sof_data[12 + (i * 3)] = zr36060_tq[i];	// Q table selection
-	}
-	return zr36060_pushit(ptr, ZR060_SOF_IDX,
-			      (3 * NO_OF_COMPONENTS) + 10, sof_data);
-}
-
-/* ------------------------------------------------------------------------- */
-
-/* SOS (start of scan) segment depends on the used scan components
-			of each color component */
-
-static int
-zr36060_set_sos (struct zr36060 *ptr)
-{
-	char sos_data[16];	// max. size of register set
-	int i;
-
-	dprintk(3, "%s: write SOS\n", ptr->name);
-	sos_data[0] = 0xff;
-	sos_data[1] = 0xda;
-	sos_data[2] = 0x00;
-	sos_data[3] = 2 + 1 + (2 * NO_OF_COMPONENTS) + 3;
-	sos_data[4] = NO_OF_COMPONENTS;
-	for (i = 0; i < NO_OF_COMPONENTS; i++) {
-		sos_data[5 + (i * 2)] = i;	// index
-		sos_data[6 + (i * 2)] = (zr36060_td[i] << 4) |
-					zr36060_ta[i]; // AC/DC tbl.sel.
-	}
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 2] = 00;	// scan start
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 3] = 0x3f;
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 4] = 00;
-	return zr36060_pushit(ptr, ZR060_SOS_IDX,
-			      4 + 1 + (2 * NO_OF_COMPONENTS) + 3,
-			      sos_data);
-}
-
-/* ------------------------------------------------------------------------- */
-
-/* DRI (define restart interval) */
-
-static int
-zr36060_set_dri (struct zr36060 *ptr)
-{
-	char dri_data[6];	// max. size of register set
-
-	dprintk(3, "%s: write DRI\n", ptr->name);
-	dri_data[0] = 0xff;
-	dri_data[1] = 0xdd;
-	dri_data[2] = 0x00;
-	dri_data[3] = 0x04;
-	dri_data[4] = (ptr->dri) >> 8;
-	dri_data[5] = (ptr->dri) & 0xff;
-	return zr36060_pushit(ptr, ZR060_DRI_IDX, 6, dri_data);
-}
-
-/* =========================================================================
-   Setup function:
-
-   Setup compression/decompression of Zoran's JPEG processor
-   ( see also zoran 36060 manual )
-
-   ... sorry for the spaghetti code ...
-   ========================================================================= */
-static void
-zr36060_init (struct zr36060 *ptr)
-{
-	int sum = 0;
-	long bitcnt, tmp;
-
-	if (ptr->mode == CODEC_DO_COMPRESSION) {
-		dprintk(2, "%s: COMPRESSION SETUP\n", ptr->name);
-
-		zr36060_write(ptr, ZR060_LOAD, ZR060_LOAD_SyncRst);
-
-		/* 060 communicates with 067 in master mode */
-		zr36060_write(ptr, ZR060_CIR, ZR060_CIR_CodeMstr);
-
-		/* Compression with or without variable scale factor */
-		/*FIXME: What about ptr->bitrate_ctrl? */
-		zr36060_write(ptr, ZR060_CMR,
-			      ZR060_CMR_Comp | ZR060_CMR_Pass2 |
-			      ZR060_CMR_BRB);
-
-		/* Must be zero */
-		zr36060_write(ptr, ZR060_MBZ, 0x00);
-		zr36060_write(ptr, ZR060_TCR_HI, 0x00);
-		zr36060_write(ptr, ZR060_TCR_LO, 0x00);
-
-		/* Disable all IRQs - no DataErr means autoreset */
-		zr36060_write(ptr, ZR060_IMR, 0);
-
-		/* volume control settings */
-		zr36060_write(ptr, ZR060_SF_HI, ptr->scalefact >> 8);
-		zr36060_write(ptr, ZR060_SF_LO, ptr->scalefact & 0xff);
-
-		zr36060_write(ptr, ZR060_AF_HI, 0xff);
-		zr36060_write(ptr, ZR060_AF_M, 0xff);
-		zr36060_write(ptr, ZR060_AF_LO, 0xff);
-
-		/* setup the variable jpeg tables */
-		sum += zr36060_set_sof(ptr);
-		sum += zr36060_set_sos(ptr);
-		sum += zr36060_set_dri(ptr);
-
-		/* setup the fixed jpeg tables - maybe variable, though -
-		 * (see table init section above) */
-		sum +=
-		    zr36060_pushit(ptr, ZR060_DQT_IDX, sizeof(zr36060_dqt),
-				   zr36060_dqt);
-		sum +=
-		    zr36060_pushit(ptr, ZR060_DHT_IDX, sizeof(zr36060_dht),
-				   zr36060_dht);
-		zr36060_write(ptr, ZR060_APP_IDX, 0xff);
-		zr36060_write(ptr, ZR060_APP_IDX + 1, 0xe0 + ptr->app.appn);
-		zr36060_write(ptr, ZR060_APP_IDX + 2, 0x00);
-		zr36060_write(ptr, ZR060_APP_IDX + 3, ptr->app.len + 2);
-		sum += zr36060_pushit(ptr, ZR060_APP_IDX + 4, 60,
-				      ptr->app.data) + 4;
-		zr36060_write(ptr, ZR060_COM_IDX, 0xff);
-		zr36060_write(ptr, ZR060_COM_IDX + 1, 0xfe);
-		zr36060_write(ptr, ZR060_COM_IDX + 2, 0x00);
-		zr36060_write(ptr, ZR060_COM_IDX + 3, ptr->com.len + 2);
-		sum += zr36060_pushit(ptr, ZR060_COM_IDX + 4, 60,
-				      ptr->com.data) + 4;
-
-		/* setup misc. data for compression (target code sizes) */
-
-		/* size of compressed code to reach without header data */
-		sum = ptr->real_code_vol - sum;
-		bitcnt = sum << 3;	/* need the size in bits */
-
-		tmp = bitcnt >> 16;
-		dprintk(3,
-			"%s: code: csize=%d, tot=%d, bit=%ld, highbits=%ld\n",
-			ptr->name, sum, ptr->real_code_vol, bitcnt, tmp);
-		zr36060_write(ptr, ZR060_TCV_NET_HI, tmp >> 8);
-		zr36060_write(ptr, ZR060_TCV_NET_MH, tmp & 0xff);
-		tmp = bitcnt & 0xffff;
-		zr36060_write(ptr, ZR060_TCV_NET_ML, tmp >> 8);
-		zr36060_write(ptr, ZR060_TCV_NET_LO, tmp & 0xff);
-
-		bitcnt -= bitcnt >> 7;	// bits without stuffing
-		bitcnt -= ((bitcnt * 5) >> 6);	// bits without eob
-
-		tmp = bitcnt >> 16;
-		dprintk(3, "%s: code: nettobit=%ld, highnettobits=%ld\n",
-			ptr->name, bitcnt, tmp);
-		zr36060_write(ptr, ZR060_TCV_DATA_HI, tmp >> 8);
-		zr36060_write(ptr, ZR060_TCV_DATA_MH, tmp & 0xff);
-		tmp = bitcnt & 0xffff;
-		zr36060_write(ptr, ZR060_TCV_DATA_ML, tmp >> 8);
-		zr36060_write(ptr, ZR060_TCV_DATA_LO, tmp & 0xff);
-
-		/* JPEG markers to be included in the compressed stream */
-		zr36060_write(ptr, ZR060_MER,
-			      ZR060_MER_DQT | ZR060_MER_DHT |
-			      ((ptr->com.len > 0) ? ZR060_MER_Com : 0) |
-			      ((ptr->app.len > 0) ? ZR060_MER_App : 0));
-
-		/* Setup the Video Frontend */
-		/* Limit pixel range to 16..235 as per CCIR-601 */
-		zr36060_write(ptr, ZR060_VCR, ZR060_VCR_Range);
-
-	} else {
-		dprintk(2, "%s: EXPANSION SETUP\n", ptr->name);
-
-		zr36060_write(ptr, ZR060_LOAD, ZR060_LOAD_SyncRst);
-
-		/* 060 communicates with 067 in master mode */
-		zr36060_write(ptr, ZR060_CIR, ZR060_CIR_CodeMstr);
-
-		/* Decompression */
-		zr36060_write(ptr, ZR060_CMR, 0);
-
-		/* Must be zero */
-		zr36060_write(ptr, ZR060_MBZ, 0x00);
-		zr36060_write(ptr, ZR060_TCR_HI, 0x00);
-		zr36060_write(ptr, ZR060_TCR_LO, 0x00);
-
-		/* Disable all IRQs - no DataErr means autoreset */
-		zr36060_write(ptr, ZR060_IMR, 0);
-
-		/* setup misc. data for expansion */
-		zr36060_write(ptr, ZR060_MER, 0);
-
-		/* setup the fixed jpeg tables - maybe variable, though -
-		 * (see table init section above) */
-		zr36060_pushit(ptr, ZR060_DHT_IDX, sizeof(zr36060_dht),
-			       zr36060_dht);
-
-		/* Setup the Video Frontend */
-		//zr36060_write(ptr, ZR060_VCR, ZR060_VCR_FIExt);
-		//this doesn't seem right and doesn't work...
-		zr36060_write(ptr, ZR060_VCR, ZR060_VCR_Range);
-	}
-
-	/* Load the tables */
-	zr36060_write(ptr, ZR060_LOAD,
-		      ZR060_LOAD_SyncRst | ZR060_LOAD_Load);
-	zr36060_wait_end(ptr);
-	dprintk(2, "%s: Status after table preload: 0x%02x\n", ptr->name,
-		ptr->status);
-
-	if (ptr->status & ZR060_CFSR_Busy) {
-		dprintk(1, KERN_ERR "%s: init aborted!\n", ptr->name);
-		return;		// something is wrong, its timed out!!!!
-	}
-}
-
-/* =========================================================================
-   CODEC API FUNCTIONS
-
-   this functions are accessed by the master via the API structure
-   ========================================================================= */
-
-/* set compression/expansion mode and launches codec -
-   this should be the last call from the master before starting processing */
-static int
-zr36060_set_mode (struct videocodec *codec,
-		  int                mode)
-{
-	struct zr36060 *ptr = (struct zr36060 *) codec->data;
-
-	dprintk(2, "%s: set_mode %d call\n", ptr->name, mode);
-
-	if ((mode != CODEC_DO_EXPANSION) && (mode != CODEC_DO_COMPRESSION))
-		return -EINVAL;
-
-	ptr->mode = mode;
-	zr36060_init(ptr);
-
-	return 0;
-}
-
-/* set picture size (norm is ignored as the codec doesn't know about it) */
-static int
-zr36060_set_video (struct videocodec   *codec,
-		   struct tvnorm       *norm,
-		   struct vfe_settings *cap,
-		   struct vfe_polarity *pol)
-{
-	struct zr36060 *ptr = (struct zr36060 *) codec->data;
-	u32 reg;
-	int size;
-
-	dprintk(2, "%s: set_video %d/%d-%dx%d (%%%d) call\n", ptr->name,
-		cap->x, cap->y, cap->width, cap->height, cap->decimation);
-
-	/* if () return -EINVAL;
-	 * trust the master driver that it knows what it does - so
-	 * we allow invalid startx/y and norm for now ... */
-	ptr->width = cap->width / (cap->decimation & 0xff);
-	ptr->height = cap->height / (cap->decimation >> 8);
-
-	zr36060_write(ptr, ZR060_LOAD, ZR060_LOAD_SyncRst);
-
-	/* Note that VSPol/HSPol bits in zr36060 have the opposite
-	 * meaning of their zr360x7 counterparts with the same names
-	 * N.b. for VSPol this is only true if FIVEdge = 0 (default,
-	 * left unchanged here - in accordance with datasheet).
-	*/
-	reg = (!pol->vsync_pol ? ZR060_VPR_VSPol : 0)
-	    | (!pol->hsync_pol ? ZR060_VPR_HSPol : 0)
-	    | (pol->field_pol ? ZR060_VPR_FIPol : 0)
-	    | (pol->blank_pol ? ZR060_VPR_BLPol : 0)
-	    | (pol->subimg_pol ? ZR060_VPR_SImgPol : 0)
-	    | (pol->poe_pol ? ZR060_VPR_PoePol : 0)
-	    | (pol->pvalid_pol ? ZR060_VPR_PValPol : 0)
-	    | (pol->vclk_pol ? ZR060_VPR_VCLKPol : 0);
-	zr36060_write(ptr, ZR060_VPR, reg);
-
-	reg = 0;
-	switch (cap->decimation & 0xff) {
-	default:
-	case 1:
-		break;
-
-	case 2:
-		reg |= ZR060_SR_HScale2;
-		break;
-
-	case 4:
-		reg |= ZR060_SR_HScale4;
-		break;
-	}
-
-	switch (cap->decimation >> 8) {
-	default:
-	case 1:
-		break;
-
-	case 2:
-		reg |= ZR060_SR_VScale;
-		break;
-	}
-	zr36060_write(ptr, ZR060_SR, reg);
-
-	zr36060_write(ptr, ZR060_BCR_Y, 0x00);
-	zr36060_write(ptr, ZR060_BCR_U, 0x80);
-	zr36060_write(ptr, ZR060_BCR_V, 0x80);
-
-	/* sync generator */
-
-	reg = norm->Ht - 1;	/* Vtotal */
-	zr36060_write(ptr, ZR060_SGR_VTOTAL_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SGR_VTOTAL_LO, (reg >> 0) & 0xff);
-
-	reg = norm->Wt - 1;	/* Htotal */
-	zr36060_write(ptr, ZR060_SGR_HTOTAL_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SGR_HTOTAL_LO, (reg >> 0) & 0xff);
-
-	reg = 6 - 1;		/* VsyncSize */
-	zr36060_write(ptr, ZR060_SGR_VSYNC, reg);
-
-	//reg   = 30 - 1;               /* HsyncSize */
-///*CP*/        reg = (zr->params.norm == 1 ? 57 : 68);
-	reg = 68;
-	zr36060_write(ptr, ZR060_SGR_HSYNC, reg);
-
-	reg = norm->VStart - 1;	/* BVstart */
-	zr36060_write(ptr, ZR060_SGR_BVSTART, reg);
-
-	reg += norm->Ha / 2;	/* BVend */
-	zr36060_write(ptr, ZR060_SGR_BVEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SGR_BVEND_LO, (reg >> 0) & 0xff);
-
-	reg = norm->HStart - 1;	/* BHstart */
-	zr36060_write(ptr, ZR060_SGR_BHSTART, reg);
-
-	reg += norm->Wa;	/* BHend */
-	zr36060_write(ptr, ZR060_SGR_BHEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SGR_BHEND_LO, (reg >> 0) & 0xff);
-
-	/* active area */
-	reg = cap->y + norm->VStart;	/* Vstart */
-	zr36060_write(ptr, ZR060_AAR_VSTART_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_AAR_VSTART_LO, (reg >> 0) & 0xff);
-
-	reg += cap->height;	/* Vend */
-	zr36060_write(ptr, ZR060_AAR_VEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_AAR_VEND_LO, (reg >> 0) & 0xff);
-
-	reg = cap->x + norm->HStart;	/* Hstart */
-	zr36060_write(ptr, ZR060_AAR_HSTART_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_AAR_HSTART_LO, (reg >> 0) & 0xff);
-
-	reg += cap->width;	/* Hend */
-	zr36060_write(ptr, ZR060_AAR_HEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_AAR_HEND_LO, (reg >> 0) & 0xff);
-
-	/* subimage area */
-	reg = norm->VStart - 4;	/* SVstart */
-	zr36060_write(ptr, ZR060_SWR_VSTART_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SWR_VSTART_LO, (reg >> 0) & 0xff);
-
-	reg += norm->Ha / 2 + 8;	/* SVend */
-	zr36060_write(ptr, ZR060_SWR_VEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SWR_VEND_LO, (reg >> 0) & 0xff);
-
-	reg = norm->HStart /*+ 64 */  - 4;	/* SHstart */
-	zr36060_write(ptr, ZR060_SWR_HSTART_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SWR_HSTART_LO, (reg >> 0) & 0xff);
-
-	reg += norm->Wa + 8;	/* SHend */
-	zr36060_write(ptr, ZR060_SWR_HEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SWR_HEND_LO, (reg >> 0) & 0xff);
-
-	size = ptr->width * ptr->height;
-	/* Target compressed field size in bits: */
-	size = size * 16;	/* uncompressed size in bits */
-	/* (Ronald) by default, quality = 100 is a compression
-	 * ratio 1:2. Setting low_bitrate (insmod option) sets
-	 * it to 1:4 (instead of 1:2, zr36060 max) as limit because the
-	 * buz can't handle more at decimation=1... Use low_bitrate if
-	 * you have a Buz, unless you know what you're doing */
-	size = size * cap->quality / (low_bitrate ? 400 : 200);
-	/* Lower limit (arbitrary, 1 KB) */
-	if (size < 8192)
-		size = 8192;
-	/* Upper limit: 7/8 of the code buffers */
-	if (size > ptr->total_code_vol * 7)
-		size = ptr->total_code_vol * 7;
-
-	ptr->real_code_vol = size >> 3;	/* in bytes */
-
-	/* the MBCVR is the *maximum* block volume, according to the
-	 * JPEG ISO specs, this shouldn't be used, since that allows
-	 * for the best encoding quality. So set it to it's max value */
-	reg = ptr->max_block_vol;
-	zr36060_write(ptr, ZR060_MBCVR, reg);
-
-	return 0;
-}
-
-/* additional control functions */
-static int
-zr36060_control (struct videocodec *codec,
-		 int                type,
-		 int                size,
-		 void              *data)
-{
-	struct zr36060 *ptr = (struct zr36060 *) codec->data;
-	int *ival = (int *) data;
-
-	dprintk(2, "%s: control %d call with %d byte\n", ptr->name, type,
-		size);
-
-	switch (type) {
-	case CODEC_G_STATUS:	/* get last status */
-		if (size != sizeof(int))
-			return -EFAULT;
-		zr36060_read_status(ptr);
-		*ival = ptr->status;
-		break;
-
-	case CODEC_G_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = CODEC_MODE_BJPG;
-		break;
-
-	case CODEC_S_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		if (*ival != CODEC_MODE_BJPG)
-			return -EINVAL;
-		/* not needed, do nothing */
-		return 0;
-
-	case CODEC_G_VFE:
-	case CODEC_S_VFE:
-		/* not needed, do nothing */
-		return 0;
-
-	case CODEC_S_MMAP:
-		/* not available, give an error */
-		return -ENXIO;
-
-	case CODEC_G_JPEG_TDS_BYTE:	/* get target volume in byte */
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = ptr->total_code_vol;
-		break;
-
-	case CODEC_S_JPEG_TDS_BYTE:	/* get target volume in byte */
-		if (size != sizeof(int))
-			return -EFAULT;
-		ptr->total_code_vol = *ival;
-		ptr->real_code_vol = (ptr->total_code_vol * 6) >> 3;
-		break;
-
-	case CODEC_G_JPEG_SCALE:	/* get scaling factor */
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = zr36060_read_scalefactor(ptr);
-		break;
-
-	case CODEC_S_JPEG_SCALE:	/* set scaling factor */
-		if (size != sizeof(int))
-			return -EFAULT;
-		ptr->scalefact = *ival;
-		break;
-
-	case CODEC_G_JPEG_APP_DATA: {	/* get appn marker data */
-		struct jpeg_app_marker *app = data;
-
-		if (size != sizeof(struct jpeg_app_marker))
-			return -EFAULT;
-
-		*app = ptr->app;
-		break;
-	}
-
-	case CODEC_S_JPEG_APP_DATA: {	/* set appn marker data */
-		struct jpeg_app_marker *app = data;
-
-		if (size != sizeof(struct jpeg_app_marker))
-			return -EFAULT;
-
-		ptr->app = *app;
-		break;
-	}
-
-	case CODEC_G_JPEG_COM_DATA: {	/* get comment marker data */
-		struct jpeg_com_marker *com = data;
-
-		if (size != sizeof(struct jpeg_com_marker))
-			return -EFAULT;
-
-		*com = ptr->com;
-		break;
-	}
-
-	case CODEC_S_JPEG_COM_DATA: {	/* set comment marker data */
-		struct jpeg_com_marker *com = data;
-
-		if (size != sizeof(struct jpeg_com_marker))
-			return -EFAULT;
-
-		ptr->com = *com;
-		break;
-	}
-
-	default:
-		return -EINVAL;
-	}
-
-	return size;
-}
-
-/* =========================================================================
-   Exit and unregister function:
-
-   Deinitializes Zoran's JPEG processor
-   ========================================================================= */
-
-static int
-zr36060_unset (struct videocodec *codec)
-{
-	struct zr36060 *ptr = codec->data;
-
-	if (ptr) {
-		/* do wee need some codec deinit here, too ???? */
-
-		dprintk(1, "%s: finished codec #%d\n", ptr->name,
-			ptr->num);
-		kfree(ptr);
-		codec->data = NULL;
-
-		zr36060_codecs--;
-		return 0;
-	}
-
-	return -EFAULT;
-}
-
-/* =========================================================================
-   Setup and registry function:
-
-   Initializes Zoran's JPEG processor
-
-   Also sets pixel size, average code size, mode (compr./decompr.)
-   (the given size is determined by the processor with the video interface)
-   ========================================================================= */
-
-static int
-zr36060_setup (struct videocodec *codec)
-{
-	struct zr36060 *ptr;
-	int res;
-
-	dprintk(2, "zr36060: initializing MJPEG subsystem #%d.\n",
-		zr36060_codecs);
-
-	if (zr36060_codecs == MAX_CODECS) {
-		dprintk(1,
-			KERN_ERR "zr36060: Can't attach more codecs!\n");
-		return -ENOSPC;
-	}
-	//mem structure init
-	codec->data = ptr = kzalloc(sizeof(struct zr36060), GFP_KERNEL);
-	if (NULL == ptr) {
-		dprintk(1, KERN_ERR "zr36060: Can't get enough memory!\n");
-		return -ENOMEM;
-	}
-
-	snprintf(ptr->name, sizeof(ptr->name), "zr36060[%d]",
-		 zr36060_codecs);
-	ptr->num = zr36060_codecs++;
-	ptr->codec = codec;
-
-	//testing
-	res = zr36060_basic_test(ptr);
-	if (res < 0) {
-		zr36060_unset(codec);
-		return res;
-	}
-	//final setup
-	memcpy(ptr->h_samp_ratio, zr36060_decimation_h, 8);
-	memcpy(ptr->v_samp_ratio, zr36060_decimation_v, 8);
-
-	ptr->bitrate_ctrl = 0;	/* 0 or 1 - fixed file size flag
-				 * (what is the difference?) */
-	ptr->mode = CODEC_DO_COMPRESSION;
-	ptr->width = 384;
-	ptr->height = 288;
-	ptr->total_code_vol = 16000;	/* CHECKME */
-	ptr->real_code_vol = (ptr->total_code_vol * 6) >> 3;
-	ptr->max_block_vol = 240;	/* CHECKME, was 120 is 240 */
-	ptr->scalefact = 0x100;
-	ptr->dri = 1;		/* CHECKME, was 8 is 1 */
-
-	/* by default, no COM or APP markers - app should set those */
-	ptr->com.len = 0;
-	ptr->app.appn = 0;
-	ptr->app.len = 0;
-
-	zr36060_init(ptr);
-
-	dprintk(1, KERN_INFO "%s: codec attached and running\n",
-		ptr->name);
-
-	return 0;
-}
-
-static const struct videocodec zr36060_codec = {
-	.owner = THIS_MODULE,
-	.name = "zr36060",
-	.magic = 0L,		// magic not used
-	.flags =
-	    CODEC_FLAG_JPEG | CODEC_FLAG_HARDWARE | CODEC_FLAG_ENCODER |
-	    CODEC_FLAG_DECODER | CODEC_FLAG_VFE,
-	.type = CODEC_TYPE_ZR36060,
-	.setup = zr36060_setup,	// functionality
-	.unset = zr36060_unset,
-	.set_mode = zr36060_set_mode,
-	.set_video = zr36060_set_video,
-	.control = zr36060_control,
-	// others are not used
-};
-
-/* =========================================================================
-   HOOK IN DRIVER AS KERNEL MODULE
-   ========================================================================= */
-
-static int __init
-zr36060_init_module (void)
-{
-	//dprintk(1, "zr36060 driver %s\n",ZR060_VERSION);
-	zr36060_codecs = 0;
-	return videocodec_register(&zr36060_codec);
-}
-
-static void __exit
-zr36060_cleanup_module (void)
-{
-	if (zr36060_codecs) {
-		dprintk(1,
-			"zr36060: something's wrong - %d codecs left somehow.\n",
-			zr36060_codecs);
-	}
-
-	/* however, we can't just stay alive */
-	videocodec_unregister(&zr36060_codec);
-}
-
-module_init(zr36060_init_module);
-module_exit(zr36060_cleanup_module);
-
-MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@skynet.be>");
-MODULE_DESCRIPTION("Driver module for ZR36060 jpeg processors "
-		   ZR060_VERSION);
-MODULE_LICENSE("GPL");
diff --git a/drivers/media/pci/zoran/zr36060.h b/drivers/media/pci/zoran/zr36060.h
deleted file mode 100644
index 82911757ba78..000000000000
--- a/drivers/media/pci/zoran/zr36060.h
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Zoran ZR36060 basic configuration functions - header file
- *
- * Copyright (C) 2002 Laurent Pinchart <laurent.pinchart@skynet.be>
- *
- * $Id: zr36060.h,v 1.1.1.1.2.3 2003/01/14 21:18:47 rbultje Exp $
- *
- * ------------------------------------------------------------------------
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * ------------------------------------------------------------------------
- */
-
-#ifndef ZR36060_H
-#define ZR36060_H
-
-#include "videocodec.h"
-
-/* data stored for each zoran jpeg codec chip */
-struct zr36060 {
-	char name[32];
-	int num;
-	/* io datastructure */
-	struct videocodec *codec;
-	// last coder status
-	__u8 status;
-	// actual coder setup
-	int mode;
-
-	__u16 width;
-	__u16 height;
-
-	__u16 bitrate_ctrl;
-
-	__u32 total_code_vol;
-	__u32 real_code_vol;
-	__u16 max_block_vol;
-
-	__u8 h_samp_ratio[8];
-	__u8 v_samp_ratio[8];
-	__u16 scalefact;
-	__u16 dri;
-
-	/* app/com marker data */
-	struct jpeg_app_marker app;
-	struct jpeg_com_marker com;
-};
-
-/* ZR36060 register addresses */
-#define ZR060_LOAD			0x000
-#define ZR060_CFSR			0x001
-#define ZR060_CIR			0x002
-#define ZR060_CMR			0x003
-#define ZR060_MBZ			0x004
-#define ZR060_MBCVR			0x005
-#define ZR060_MER			0x006
-#define ZR060_IMR			0x007
-#define ZR060_ISR			0x008
-#define ZR060_TCV_NET_HI		0x009
-#define ZR060_TCV_NET_MH		0x00a
-#define ZR060_TCV_NET_ML		0x00b
-#define ZR060_TCV_NET_LO		0x00c
-#define ZR060_TCV_DATA_HI		0x00d
-#define ZR060_TCV_DATA_MH		0x00e
-#define ZR060_TCV_DATA_ML		0x00f
-#define ZR060_TCV_DATA_LO		0x010
-#define ZR060_SF_HI			0x011
-#define ZR060_SF_LO			0x012
-#define ZR060_AF_HI			0x013
-#define ZR060_AF_M			0x014
-#define ZR060_AF_LO			0x015
-#define ZR060_ACV_HI			0x016
-#define ZR060_ACV_MH			0x017
-#define ZR060_ACV_ML			0x018
-#define ZR060_ACV_LO			0x019
-#define ZR060_ACT_HI			0x01a
-#define ZR060_ACT_MH			0x01b
-#define ZR060_ACT_ML			0x01c
-#define ZR060_ACT_LO			0x01d
-#define ZR060_ACV_TRUN_HI		0x01e
-#define ZR060_ACV_TRUN_MH		0x01f
-#define ZR060_ACV_TRUN_ML		0x020
-#define ZR060_ACV_TRUN_LO		0x021
-#define ZR060_IDR_DEV			0x022
-#define ZR060_IDR_REV			0x023
-#define ZR060_TCR_HI			0x024
-#define ZR060_TCR_LO			0x025
-#define ZR060_VCR			0x030
-#define ZR060_VPR			0x031
-#define ZR060_SR			0x032
-#define ZR060_BCR_Y			0x033
-#define ZR060_BCR_U			0x034
-#define ZR060_BCR_V			0x035
-#define ZR060_SGR_VTOTAL_HI		0x036
-#define ZR060_SGR_VTOTAL_LO		0x037
-#define ZR060_SGR_HTOTAL_HI		0x038
-#define ZR060_SGR_HTOTAL_LO		0x039
-#define ZR060_SGR_VSYNC			0x03a
-#define ZR060_SGR_HSYNC			0x03b
-#define ZR060_SGR_BVSTART		0x03c
-#define ZR060_SGR_BHSTART		0x03d
-#define ZR060_SGR_BVEND_HI		0x03e
-#define ZR060_SGR_BVEND_LO		0x03f
-#define ZR060_SGR_BHEND_HI		0x040
-#define ZR060_SGR_BHEND_LO		0x041
-#define ZR060_AAR_VSTART_HI		0x042
-#define ZR060_AAR_VSTART_LO		0x043
-#define ZR060_AAR_VEND_HI		0x044
-#define ZR060_AAR_VEND_LO		0x045
-#define ZR060_AAR_HSTART_HI		0x046
-#define ZR060_AAR_HSTART_LO		0x047
-#define ZR060_AAR_HEND_HI		0x048
-#define ZR060_AAR_HEND_LO		0x049
-#define ZR060_SWR_VSTART_HI		0x04a
-#define ZR060_SWR_VSTART_LO		0x04b
-#define ZR060_SWR_VEND_HI		0x04c
-#define ZR060_SWR_VEND_LO		0x04d
-#define ZR060_SWR_HSTART_HI		0x04e
-#define ZR060_SWR_HSTART_LO		0x04f
-#define ZR060_SWR_HEND_HI		0x050
-#define ZR060_SWR_HEND_LO		0x051
-
-#define ZR060_SOF_IDX			0x060
-#define ZR060_SOS_IDX			0x07a
-#define ZR060_DRI_IDX			0x0c0
-#define ZR060_DQT_IDX			0x0cc
-#define ZR060_DHT_IDX			0x1d4
-#define ZR060_APP_IDX			0x380
-#define ZR060_COM_IDX			0x3c0
-
-/* ZR36060 LOAD register bits */
-
-#define ZR060_LOAD_Load			(1 << 7)
-#define ZR060_LOAD_SyncRst		(1 << 0)
-
-/* ZR36060 Code FIFO Status register bits */
-
-#define ZR060_CFSR_Busy			(1 << 7)
-#define ZR060_CFSR_CBusy		(1 << 2)
-#define ZR060_CFSR_CFIFO		(3 << 0)
-
-/* ZR36060 Code Interface register */
-
-#define ZR060_CIR_Code16		(1 << 7)
-#define ZR060_CIR_Endian		(1 << 6)
-#define ZR060_CIR_CFIS			(1 << 2)
-#define ZR060_CIR_CodeMstr		(1 << 0)
-
-/* ZR36060 Codec Mode register */
-
-#define ZR060_CMR_Comp			(1 << 7)
-#define ZR060_CMR_ATP			(1 << 6)
-#define ZR060_CMR_Pass2			(1 << 5)
-#define ZR060_CMR_TLM			(1 << 4)
-#define ZR060_CMR_BRB			(1 << 2)
-#define ZR060_CMR_FSF			(1 << 1)
-
-/* ZR36060 Markers Enable register */
-
-#define ZR060_MER_App			(1 << 7)
-#define ZR060_MER_Com			(1 << 6)
-#define ZR060_MER_DRI			(1 << 5)
-#define ZR060_MER_DQT			(1 << 4)
-#define ZR060_MER_DHT			(1 << 3)
-
-/* ZR36060 Interrupt Mask register */
-
-#define ZR060_IMR_EOAV			(1 << 3)
-#define ZR060_IMR_EOI			(1 << 2)
-#define ZR060_IMR_End			(1 << 1)
-#define ZR060_IMR_DataErr		(1 << 0)
-
-/* ZR36060 Interrupt Status register */
-
-#define ZR060_ISR_ProCnt		(3 << 6)
-#define ZR060_ISR_EOAV			(1 << 3)
-#define ZR060_ISR_EOI			(1 << 2)
-#define ZR060_ISR_End			(1 << 1)
-#define ZR060_ISR_DataErr		(1 << 0)
-
-/* ZR36060 Video Control register */
-
-#define ZR060_VCR_Video8		(1 << 7)
-#define ZR060_VCR_Range			(1 << 6)
-#define ZR060_VCR_FIDet			(1 << 3)
-#define ZR060_VCR_FIVedge		(1 << 2)
-#define ZR060_VCR_FIExt			(1 << 1)
-#define ZR060_VCR_SyncMstr		(1 << 0)
-
-/* ZR36060 Video Polarity register */
-
-#define ZR060_VPR_VCLKPol		(1 << 7)
-#define ZR060_VPR_PValPol		(1 << 6)
-#define ZR060_VPR_PoePol		(1 << 5)
-#define ZR060_VPR_SImgPol		(1 << 4)
-#define ZR060_VPR_BLPol			(1 << 3)
-#define ZR060_VPR_FIPol			(1 << 2)
-#define ZR060_VPR_HSPol			(1 << 1)
-#define ZR060_VPR_VSPol			(1 << 0)
-
-/* ZR36060 Scaling register */
-
-#define ZR060_SR_VScale			(1 << 2)
-#define ZR060_SR_HScale2		(1 << 0)
-#define ZR060_SR_HScale4		(2 << 0)
-
-#endif				/*fndef ZR36060_H */
diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig
index c7a1cf8a1b01..2728376b04b5 100644
--- a/drivers/media/platform/Kconfig
+++ b/drivers/media/platform/Kconfig
@@ -26,6 +26,7 @@ config VIDEO_VIA_CAMERA
 #
 # Platform multimedia device configuration
 #
+source "drivers/media/platform/cadence/Kconfig"
 
 source "drivers/media/platform/davinci/Kconfig"
 
@@ -34,7 +35,7 @@ source "drivers/media/platform/omap/Kconfig"
 config VIDEO_SH_VOU
 	tristate "SuperH VOU video output driver"
 	depends on MEDIA_CAMERA_SUPPORT
-	depends on VIDEO_DEV && I2C && HAS_DMA
+	depends on VIDEO_DEV && I2C
 	depends on ARCH_SHMOBILE || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	help
@@ -42,7 +43,7 @@ config VIDEO_SH_VOU
 
 config VIDEO_VIU
 	tristate "Freescale VIU Video Driver"
-	depends on VIDEO_V4L2 && PPC_MPC512x
+	depends on VIDEO_V4L2 && (PPC_MPC512x || COMPILE_TEST) && I2C
 	select VIDEOBUF_DMA_CONTIG
 	default y
 	---help---
@@ -62,10 +63,10 @@ config VIDEO_MUX
 
 config VIDEO_OMAP3
 	tristate "OMAP 3 Camera support"
-	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API && ARCH_OMAP3
-	depends on HAS_DMA && OF
-	depends on OMAP_IOMMU
-	select ARM_DMA_USE_IOMMU
+	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
+	depends on (ARCH_OMAP3 && OMAP_IOMMU) || COMPILE_TEST
+	depends on COMMON_CLK && OF
+	select ARM_DMA_USE_IOMMU if OMAP_IOMMU
 	select VIDEOBUF2_DMA_CONTIG
 	select MFD_SYSCON
 	select V4L2_FWNODE
@@ -80,7 +81,7 @@ config VIDEO_OMAP3_DEBUG
 
 config VIDEO_PXA27x
 	tristate "PXA27x Quick Capture Interface driver"
-	depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA
+	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on PXA27x || COMPILE_TEST
 	select VIDEOBUF2_DMA_SG
 	select SG_SPLIT
@@ -90,7 +91,7 @@ config VIDEO_PXA27x
 
 config VIDEO_QCOM_CAMSS
 	tristate "Qualcomm 8x16 V4L2 Camera Subsystem driver"
-	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && HAS_DMA
+	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on (ARCH_QCOM && IOMMU_DMA) || COMPILE_TEST
 	select VIDEOBUF2_DMA_SG
 	select V4L2_FWNODE
@@ -100,7 +101,6 @@ config VIDEO_S3C_CAMIF
 	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
 	depends on PM
 	depends on ARCH_S3C64XX || PLAT_S3C24XX || COMPILE_TEST
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	---help---
 	  This is a v4l2 driver for s3c24xx and s3c64xx SoC series camera
@@ -111,7 +111,7 @@ config VIDEO_S3C_CAMIF
 
 config VIDEO_STM32_DCMI
 	tristate "STM32 Digital Camera Memory Interface (DCMI) support"
-	depends on VIDEO_V4L2 && OF && HAS_DMA
+	depends on VIDEO_V4L2 && OF
 	depends on ARCH_STM32 || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
@@ -124,7 +124,7 @@ config VIDEO_STM32_DCMI
 
 config VIDEO_RENESAS_CEU
 	tristate "Renesas Capture Engine Unit (CEU) driver"
-	depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA
+	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_SHMOBILE || ARCH_R7S72100 || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
@@ -142,7 +142,6 @@ config VIDEO_TI_CAL
 	tristate "TI CAL (Camera Adaptation Layer) driver"
 	depends on VIDEO_DEV && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on SOC_DRA7XX || COMPILE_TEST
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
 	default n
@@ -170,7 +169,6 @@ if V4L_MEM2MEM_DRIVERS
 config VIDEO_CODA
 	tristate "Chips&Media Coda multi-standard codec IP"
 	depends on VIDEO_DEV && VIDEO_V4L2 && (ARCH_MXC || COMPILE_TEST)
-	depends on HAS_DMA
 	select SRAM
 	select VIDEOBUF2_DMA_CONTIG
 	select VIDEOBUF2_VMALLOC
@@ -188,7 +186,6 @@ config VIDEO_MEDIATEK_JPEG
 	depends on MTK_IOMMU_V1 || COMPILE_TEST
 	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_MEDIATEK || COMPILE_TEST
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
 	---help---
@@ -200,7 +197,7 @@ config VIDEO_MEDIATEK_JPEG
 
 config VIDEO_MEDIATEK_VPU
 	tristate "Mediatek Video Processor Unit"
-	depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA
+	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 	---help---
 	    This driver provides downloading VPU firmware and
@@ -216,7 +213,6 @@ config VIDEO_MEDIATEK_MDP
 	depends on MTK_IOMMU || COMPILE_TEST
 	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_MEDIATEK || COMPILE_TEST
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
 	select VIDEO_MEDIATEK_VPU
@@ -231,7 +227,7 @@ config VIDEO_MEDIATEK_MDP
 config VIDEO_MEDIATEK_VCODEC
 	tristate "Mediatek Video Codec driver"
 	depends on MTK_IOMMU || COMPILE_TEST
-	depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA
+	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
@@ -247,7 +243,7 @@ config VIDEO_MEDIATEK_VCODEC
 
 config VIDEO_MEM2MEM_DEINTERLACE
 	tristate "Deinterlace support"
-	depends on VIDEO_DEV && VIDEO_V4L2 && DMA_ENGINE
+	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
@@ -258,7 +254,6 @@ config VIDEO_SAMSUNG_S5P_G2D
 	tristate "Samsung S5P and EXYNOS4 G2D 2d graphics accelerator driver"
 	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
 	default n
@@ -270,7 +265,6 @@ config VIDEO_SAMSUNG_S5P_JPEG
 	tristate "Samsung S5P/Exynos3250/Exynos4 JPEG codec driver"
 	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
 	---help---
@@ -281,7 +275,6 @@ config VIDEO_SAMSUNG_S5P_MFC
 	tristate "Samsung S5P MFC Video Codec"
 	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	default n
 	help
@@ -291,7 +284,6 @@ config VIDEO_MX2_EMMAPRP
 	tristate "MX2 eMMa-PrP support"
 	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on SOC_IMX27 || COMPILE_TEST
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
 	help
@@ -303,7 +295,6 @@ config VIDEO_SAMSUNG_EXYNOS_GSC
 	tristate "Samsung Exynos G-Scaler driver"
 	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_EXYNOS || COMPILE_TEST
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
 	help
@@ -312,7 +303,6 @@ config VIDEO_SAMSUNG_EXYNOS_GSC
 config VIDEO_STI_BDISP
 	tristate "STMicroelectronics BDISP 2D blitter driver"
 	depends on VIDEO_DEV && VIDEO_V4L2
-	depends on HAS_DMA
 	depends on ARCH_STI || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
@@ -322,7 +312,6 @@ config VIDEO_STI_BDISP
 config VIDEO_STI_HVA
 	tristate "STMicroelectronics HVA multi-format video encoder V4L2 driver"
 	depends on VIDEO_DEV && VIDEO_V4L2
-	depends on HAS_DMA
 	depends on ARCH_STI || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
@@ -349,7 +338,6 @@ config VIDEO_STI_DELTA
 	tristate "STMicroelectronics DELTA multi-format video decoder V4L2 driver"
 	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_STI || COMPILE_TEST
-	depends on HAS_DMA
 	help
 		This V4L2 driver enables DELTA multi-format video decoder
 		of STMicroelectronics STiH4xx SoC series allowing hardware
@@ -395,7 +383,7 @@ config VIDEO_SH_VEU
 
 config VIDEO_RENESAS_FDP1
 	tristate "Renesas Fine Display Processor"
-	depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA
+	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_SHMOBILE || COMPILE_TEST
 	depends on (!ARCH_RENESAS && !VIDEO_RENESAS_FCP) || VIDEO_RENESAS_FCP
 	select VIDEOBUF2_DMA_CONTIG
@@ -409,7 +397,7 @@ config VIDEO_RENESAS_FDP1
 
 config VIDEO_RENESAS_JPU
 	tristate "Renesas JPEG Processing Unit"
-	depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA
+	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_RENESAS || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
@@ -434,8 +422,8 @@ config VIDEO_RENESAS_FCP
 
 config VIDEO_RENESAS_VSP1
 	tristate "Renesas VSP1 Video Processing Engine"
-	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && HAS_DMA
-	depends on (ARCH_RENESAS && OF) || COMPILE_TEST
+	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
+	depends on ARCH_RENESAS || COMPILE_TEST
 	depends on (!ARM64 && !VIDEO_RENESAS_FCP) || VIDEO_RENESAS_FCP
 	select VIDEOBUF2_DMA_CONTIG
 	select VIDEOBUF2_VMALLOC
@@ -447,7 +435,7 @@ config VIDEO_RENESAS_VSP1
 
 config VIDEO_ROCKCHIP_RGA
 	tristate "Rockchip Raster 2d Graphic Acceleration Unit"
-	depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA
+	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_ROCKCHIP || COMPILE_TEST
 	select VIDEOBUF2_DMA_SG
 	select V4L2_MEM2MEM_DEV
@@ -464,7 +452,6 @@ config VIDEO_TI_VPE
 	tristate "TI VPE (Video Processing Engine) driver"
 	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on SOC_DRA7XX || COMPILE_TEST
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
 	select VIDEO_TI_VPDMA
@@ -483,7 +470,7 @@ config VIDEO_TI_VPE_DEBUG
 
 config VIDEO_QCOM_VENUS
 	tristate "Qualcomm Venus V4L2 encoder/decoder driver"
-	depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA
+	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on (ARCH_QCOM && IOMMU_DMA) || COMPILE_TEST
 	select QCOM_MDT_LOADER if ARCH_QCOM
 	select QCOM_SCM if ARCH_QCOM
@@ -558,7 +545,7 @@ config VIDEO_MESON_AO_CEC
 
 config CEC_GPIO
 	tristate "Generic GPIO-based CEC driver"
-	depends on PREEMPT
+	depends on PREEMPT || COMPILE_TEST
 	select CEC_CORE
 	select CEC_PIN
 	select GPIOLIB
@@ -625,7 +612,7 @@ if SDR_PLATFORM_DRIVERS
 
 config VIDEO_RCAR_DRIF
 	tristate "Renesas Digitial Radio Interface (DRIF)"
-	depends on VIDEO_V4L2 && HAS_DMA
+	depends on VIDEO_V4L2
 	depends on ARCH_RENESAS || COMPILE_TEST
 	select VIDEOBUF2_VMALLOC
 	---help---
diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile
index 932515df4477..04bc1502a30e 100644
--- a/drivers/media/platform/Makefile
+++ b/drivers/media/platform/Makefile
@@ -3,6 +3,7 @@
 # Makefile for the video capture/playback device drivers.
 #
 
+obj-$(CONFIG_VIDEO_CADENCE)		+= cadence/
 obj-$(CONFIG_VIDEO_VIA_CAMERA) += via-camera.o
 obj-$(CONFIG_VIDEO_CAFE_CCIC) += marvell-ccic/
 obj-$(CONFIG_VIDEO_MMP_CAMERA) += marvell-ccic/
diff --git a/drivers/media/platform/am437x/Kconfig b/drivers/media/platform/am437x/Kconfig
index 160e77e9a0fb..f4ce1176e4dc 100644
--- a/drivers/media/platform/am437x/Kconfig
+++ b/drivers/media/platform/am437x/Kconfig
@@ -1,6 +1,6 @@
 config VIDEO_AM437X_VPFE
 	tristate "TI AM437x VPFE video capture driver"
-	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && HAS_DMA
+	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on SOC_AM43XX || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
diff --git a/drivers/media/platform/am437x/am437x-vpfe.c b/drivers/media/platform/am437x/am437x-vpfe.c
index 601ae6487617..58ebc2220d0e 100644
--- a/drivers/media/platform/am437x/am437x-vpfe.c
+++ b/drivers/media/platform/am437x/am437x-vpfe.c
@@ -2662,8 +2662,7 @@ static void vpfe_save_context(struct vpfe_ccdc *ccdc)
 
 static int vpfe_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct vpfe_device *vpfe = platform_get_drvdata(pdev);
+	struct vpfe_device *vpfe = dev_get_drvdata(dev);
 	struct vpfe_ccdc *ccdc = &vpfe->ccdc;
 
 	/* if streaming has not started we don't care */
@@ -2720,8 +2719,7 @@ static void vpfe_restore_context(struct vpfe_ccdc *ccdc)
 
 static int vpfe_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct vpfe_device *vpfe = platform_get_drvdata(pdev);
+	struct vpfe_device *vpfe = dev_get_drvdata(dev);
 	struct vpfe_ccdc *ccdc = &vpfe->ccdc;
 
 	/* if streaming has not started we don't care */
diff --git a/drivers/media/platform/atmel/Kconfig b/drivers/media/platform/atmel/Kconfig
index 55de751e5f51..a211ef20f77e 100644
--- a/drivers/media/platform/atmel/Kconfig
+++ b/drivers/media/platform/atmel/Kconfig
@@ -1,6 +1,6 @@
 config VIDEO_ATMEL_ISC
 	tristate "ATMEL Image Sensor Controller (ISC) support"
-	depends on VIDEO_V4L2 && COMMON_CLK && VIDEO_V4L2_SUBDEV_API && HAS_DMA
+	depends on VIDEO_V4L2 && COMMON_CLK && VIDEO_V4L2_SUBDEV_API
 	depends on ARCH_AT91 || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select REGMAP_MMIO
@@ -11,7 +11,7 @@ config VIDEO_ATMEL_ISC
 
 config VIDEO_ATMEL_ISI
 	tristate "ATMEL Image Sensor Interface (ISI) support"
-	depends on VIDEO_V4L2 && OF && HAS_DMA
+	depends on VIDEO_V4L2 && OF
 	depends on ARCH_AT91 || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
diff --git a/drivers/media/platform/cadence/Kconfig b/drivers/media/platform/cadence/Kconfig
new file mode 100644
index 000000000000..3bf0f2454384
--- /dev/null
+++ b/drivers/media/platform/cadence/Kconfig
@@ -0,0 +1,34 @@
+config VIDEO_CADENCE
+	bool "Cadence Video Devices"
+	help
+	  If you have a media device designed by Cadence, say Y.
+
+	  Note that this option doesn't include new drivers in the kernel:
+	  saying N will just cause Kconfig to skip all the questions about
+	  Cadence media devices.
+
+if VIDEO_CADENCE
+
+config VIDEO_CADENCE_CSI2RX
+	tristate "Cadence MIPI-CSI2 RX Controller"
+	depends on MEDIA_CONTROLLER
+	depends on VIDEO_V4L2_SUBDEV_API
+	select V4L2_FWNODE
+	help
+	  Support for the Cadence MIPI CSI2 Receiver controller.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called cdns-csi2rx.
+
+config VIDEO_CADENCE_CSI2TX
+	tristate "Cadence MIPI-CSI2 TX Controller"
+	depends on MEDIA_CONTROLLER
+	depends on VIDEO_V4L2_SUBDEV_API
+	select V4L2_FWNODE
+	help
+	  Support for the Cadence MIPI CSI2 Transceiver controller.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called cdns-csi2tx.
+
+endif
diff --git a/drivers/media/platform/cadence/Makefile b/drivers/media/platform/cadence/Makefile
new file mode 100644
index 000000000000..be59a8728a01
--- /dev/null
+++ b/drivers/media/platform/cadence/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_VIDEO_CADENCE_CSI2RX)	+= cdns-csi2rx.o
+obj-$(CONFIG_VIDEO_CADENCE_CSI2TX)	+= cdns-csi2tx.o
diff --git a/drivers/media/platform/cadence/cdns-csi2rx.c b/drivers/media/platform/cadence/cdns-csi2rx.c
new file mode 100644
index 000000000000..a0f02916006b
--- /dev/null
+++ b/drivers/media/platform/cadence/cdns-csi2rx.c
@@ -0,0 +1,498 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Driver for Cadence MIPI-CSI2 RX Controller v1.3
+ *
+ * Copyright (C) 2017 Cadence Design Systems Inc.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-subdev.h>
+
+#define CSI2RX_DEVICE_CFG_REG			0x000
+
+#define CSI2RX_SOFT_RESET_REG			0x004
+#define CSI2RX_SOFT_RESET_PROTOCOL			BIT(1)
+#define CSI2RX_SOFT_RESET_FRONT				BIT(0)
+
+#define CSI2RX_STATIC_CFG_REG			0x008
+#define CSI2RX_STATIC_CFG_DLANE_MAP(llane, plane)	((plane) << (16 + (llane) * 4))
+#define CSI2RX_STATIC_CFG_LANES_MASK			GENMASK(11, 8)
+
+#define CSI2RX_STREAM_BASE(n)		(((n) + 1) * 0x100)
+
+#define CSI2RX_STREAM_CTRL_REG(n)		(CSI2RX_STREAM_BASE(n) + 0x000)
+#define CSI2RX_STREAM_CTRL_START			BIT(0)
+
+#define CSI2RX_STREAM_DATA_CFG_REG(n)		(CSI2RX_STREAM_BASE(n) + 0x008)
+#define CSI2RX_STREAM_DATA_CFG_EN_VC_SELECT		BIT(31)
+#define CSI2RX_STREAM_DATA_CFG_VC_SELECT(n)		BIT((n) + 16)
+
+#define CSI2RX_STREAM_CFG_REG(n)		(CSI2RX_STREAM_BASE(n) + 0x00c)
+#define CSI2RX_STREAM_CFG_FIFO_MODE_LARGE_BUF		(1 << 8)
+
+#define CSI2RX_LANES_MAX	4
+#define CSI2RX_STREAMS_MAX	4
+
+enum csi2rx_pads {
+	CSI2RX_PAD_SINK,
+	CSI2RX_PAD_SOURCE_STREAM0,
+	CSI2RX_PAD_SOURCE_STREAM1,
+	CSI2RX_PAD_SOURCE_STREAM2,
+	CSI2RX_PAD_SOURCE_STREAM3,
+	CSI2RX_PAD_MAX,
+};
+
+struct csi2rx_priv {
+	struct device			*dev;
+	unsigned int			count;
+
+	/*
+	 * Used to prevent race conditions between multiple,
+	 * concurrent calls to start and stop.
+	 */
+	struct mutex			lock;
+
+	void __iomem			*base;
+	struct clk			*sys_clk;
+	struct clk			*p_clk;
+	struct clk			*pixel_clk[CSI2RX_STREAMS_MAX];
+	struct phy			*dphy;
+
+	u8				lanes[CSI2RX_LANES_MAX];
+	u8				num_lanes;
+	u8				max_lanes;
+	u8				max_streams;
+	bool				has_internal_dphy;
+
+	struct v4l2_subdev		subdev;
+	struct v4l2_async_notifier	notifier;
+	struct media_pad		pads[CSI2RX_PAD_MAX];
+
+	/* Remote source */
+	struct v4l2_async_subdev	asd;
+	struct v4l2_subdev		*source_subdev;
+	int				source_pad;
+};
+
+static inline
+struct csi2rx_priv *v4l2_subdev_to_csi2rx(struct v4l2_subdev *subdev)
+{
+	return container_of(subdev, struct csi2rx_priv, subdev);
+}
+
+static void csi2rx_reset(struct csi2rx_priv *csi2rx)
+{
+	writel(CSI2RX_SOFT_RESET_PROTOCOL | CSI2RX_SOFT_RESET_FRONT,
+	       csi2rx->base + CSI2RX_SOFT_RESET_REG);
+
+	udelay(10);
+
+	writel(0, csi2rx->base + CSI2RX_SOFT_RESET_REG);
+}
+
+static int csi2rx_start(struct csi2rx_priv *csi2rx)
+{
+	unsigned int i;
+	unsigned long lanes_used = 0;
+	u32 reg;
+	int ret;
+
+	ret = clk_prepare_enable(csi2rx->p_clk);
+	if (ret)
+		return ret;
+
+	csi2rx_reset(csi2rx);
+
+	reg = csi2rx->num_lanes << 8;
+	for (i = 0; i < csi2rx->num_lanes; i++) {
+		reg |= CSI2RX_STATIC_CFG_DLANE_MAP(i, csi2rx->lanes[i]);
+		set_bit(csi2rx->lanes[i], &lanes_used);
+	}
+
+	/*
+	 * Even the unused lanes need to be mapped. In order to avoid
+	 * to map twice to the same physical lane, keep the lanes used
+	 * in the previous loop, and only map unused physical lanes to
+	 * the rest of our logical lanes.
+	 */
+	for (i = csi2rx->num_lanes; i < csi2rx->max_lanes; i++) {
+		unsigned int idx = find_first_zero_bit(&lanes_used,
+						       sizeof(lanes_used));
+		set_bit(idx, &lanes_used);
+		reg |= CSI2RX_STATIC_CFG_DLANE_MAP(i, i + 1);
+	}
+
+	writel(reg, csi2rx->base + CSI2RX_STATIC_CFG_REG);
+
+	ret = v4l2_subdev_call(csi2rx->source_subdev, video, s_stream, true);
+	if (ret)
+		goto err_disable_pclk;
+
+	/*
+	 * Create a static mapping between the CSI virtual channels
+	 * and the output stream.
+	 *
+	 * This should be enhanced, but v4l2 lacks the support for
+	 * changing that mapping dynamically.
+	 *
+	 * We also cannot enable and disable independent streams here,
+	 * hence the reference counting.
+	 */
+	for (i = 0; i < csi2rx->max_streams; i++) {
+		ret = clk_prepare_enable(csi2rx->pixel_clk[i]);
+		if (ret)
+			goto err_disable_pixclk;
+
+		writel(CSI2RX_STREAM_CFG_FIFO_MODE_LARGE_BUF,
+		       csi2rx->base + CSI2RX_STREAM_CFG_REG(i));
+
+		writel(CSI2RX_STREAM_DATA_CFG_EN_VC_SELECT |
+		       CSI2RX_STREAM_DATA_CFG_VC_SELECT(i),
+		       csi2rx->base + CSI2RX_STREAM_DATA_CFG_REG(i));
+
+		writel(CSI2RX_STREAM_CTRL_START,
+		       csi2rx->base + CSI2RX_STREAM_CTRL_REG(i));
+	}
+
+	ret = clk_prepare_enable(csi2rx->sys_clk);
+	if (ret)
+		goto err_disable_pixclk;
+
+	clk_disable_unprepare(csi2rx->p_clk);
+
+	return 0;
+
+err_disable_pixclk:
+	for (; i > 0; i--)
+		clk_disable_unprepare(csi2rx->pixel_clk[i - 1]);
+
+err_disable_pclk:
+	clk_disable_unprepare(csi2rx->p_clk);
+
+	return ret;
+}
+
+static void csi2rx_stop(struct csi2rx_priv *csi2rx)
+{
+	unsigned int i;
+
+	clk_prepare_enable(csi2rx->p_clk);
+	clk_disable_unprepare(csi2rx->sys_clk);
+
+	for (i = 0; i < csi2rx->max_streams; i++) {
+		writel(0, csi2rx->base + CSI2RX_STREAM_CTRL_REG(i));
+
+		clk_disable_unprepare(csi2rx->pixel_clk[i]);
+	}
+
+	clk_disable_unprepare(csi2rx->p_clk);
+
+	if (v4l2_subdev_call(csi2rx->source_subdev, video, s_stream, false))
+		dev_warn(csi2rx->dev, "Couldn't disable our subdev\n");
+}
+
+static int csi2rx_s_stream(struct v4l2_subdev *subdev, int enable)
+{
+	struct csi2rx_priv *csi2rx = v4l2_subdev_to_csi2rx(subdev);
+	int ret = 0;
+
+	mutex_lock(&csi2rx->lock);
+
+	if (enable) {
+		/*
+		 * If we're not the first users, there's no need to
+		 * enable the whole controller.
+		 */
+		if (!csi2rx->count) {
+			ret = csi2rx_start(csi2rx);
+			if (ret)
+				goto out;
+		}
+
+		csi2rx->count++;
+	} else {
+		csi2rx->count--;
+
+		/*
+		 * Let the last user turn off the lights.
+		 */
+		if (!csi2rx->count)
+			csi2rx_stop(csi2rx);
+	}
+
+out:
+	mutex_unlock(&csi2rx->lock);
+	return ret;
+}
+
+static const struct v4l2_subdev_video_ops csi2rx_video_ops = {
+	.s_stream	= csi2rx_s_stream,
+};
+
+static const struct v4l2_subdev_ops csi2rx_subdev_ops = {
+	.video		= &csi2rx_video_ops,
+};
+
+static int csi2rx_async_bound(struct v4l2_async_notifier *notifier,
+			      struct v4l2_subdev *s_subdev,
+			      struct v4l2_async_subdev *asd)
+{
+	struct v4l2_subdev *subdev = notifier->sd;
+	struct csi2rx_priv *csi2rx = v4l2_subdev_to_csi2rx(subdev);
+
+	csi2rx->source_pad = media_entity_get_fwnode_pad(&s_subdev->entity,
+							 s_subdev->fwnode,
+							 MEDIA_PAD_FL_SOURCE);
+	if (csi2rx->source_pad < 0) {
+		dev_err(csi2rx->dev, "Couldn't find output pad for subdev %s\n",
+			s_subdev->name);
+		return csi2rx->source_pad;
+	}
+
+	csi2rx->source_subdev = s_subdev;
+
+	dev_dbg(csi2rx->dev, "Bound %s pad: %d\n", s_subdev->name,
+		csi2rx->source_pad);
+
+	return media_create_pad_link(&csi2rx->source_subdev->entity,
+				     csi2rx->source_pad,
+				     &csi2rx->subdev.entity, 0,
+				     MEDIA_LNK_FL_ENABLED |
+				     MEDIA_LNK_FL_IMMUTABLE);
+}
+
+static const struct v4l2_async_notifier_operations csi2rx_notifier_ops = {
+	.bound		= csi2rx_async_bound,
+};
+
+static int csi2rx_get_resources(struct csi2rx_priv *csi2rx,
+				struct platform_device *pdev)
+{
+	struct resource *res;
+	unsigned char i;
+	u32 dev_cfg;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	csi2rx->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(csi2rx->base))
+		return PTR_ERR(csi2rx->base);
+
+	csi2rx->sys_clk = devm_clk_get(&pdev->dev, "sys_clk");
+	if (IS_ERR(csi2rx->sys_clk)) {
+		dev_err(&pdev->dev, "Couldn't get sys clock\n");
+		return PTR_ERR(csi2rx->sys_clk);
+	}
+
+	csi2rx->p_clk = devm_clk_get(&pdev->dev, "p_clk");
+	if (IS_ERR(csi2rx->p_clk)) {
+		dev_err(&pdev->dev, "Couldn't get P clock\n");
+		return PTR_ERR(csi2rx->p_clk);
+	}
+
+	csi2rx->dphy = devm_phy_optional_get(&pdev->dev, "dphy");
+	if (IS_ERR(csi2rx->dphy)) {
+		dev_err(&pdev->dev, "Couldn't get external D-PHY\n");
+		return PTR_ERR(csi2rx->dphy);
+	}
+
+	/*
+	 * FIXME: Once we'll have external D-PHY support, the check
+	 * will need to be removed.
+	 */
+	if (csi2rx->dphy) {
+		dev_err(&pdev->dev, "External D-PHY not supported yet\n");
+		return -EINVAL;
+	}
+
+	clk_prepare_enable(csi2rx->p_clk);
+	dev_cfg = readl(csi2rx->base + CSI2RX_DEVICE_CFG_REG);
+	clk_disable_unprepare(csi2rx->p_clk);
+
+	csi2rx->max_lanes = dev_cfg & 7;
+	if (csi2rx->max_lanes > CSI2RX_LANES_MAX) {
+		dev_err(&pdev->dev, "Invalid number of lanes: %u\n",
+			csi2rx->max_lanes);
+		return -EINVAL;
+	}
+
+	csi2rx->max_streams = (dev_cfg >> 4) & 7;
+	if (csi2rx->max_streams > CSI2RX_STREAMS_MAX) {
+		dev_err(&pdev->dev, "Invalid number of streams: %u\n",
+			csi2rx->max_streams);
+		return -EINVAL;
+	}
+
+	csi2rx->has_internal_dphy = dev_cfg & BIT(3) ? true : false;
+
+	/*
+	 * FIXME: Once we'll have internal D-PHY support, the check
+	 * will need to be removed.
+	 */
+	if (csi2rx->has_internal_dphy) {
+		dev_err(&pdev->dev, "Internal D-PHY not supported yet\n");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < csi2rx->max_streams; i++) {
+		char clk_name[16];
+
+		snprintf(clk_name, sizeof(clk_name), "pixel_if%u_clk", i);
+		csi2rx->pixel_clk[i] = devm_clk_get(&pdev->dev, clk_name);
+		if (IS_ERR(csi2rx->pixel_clk[i])) {
+			dev_err(&pdev->dev, "Couldn't get clock %s\n", clk_name);
+			return PTR_ERR(csi2rx->pixel_clk[i]);
+		}
+	}
+
+	return 0;
+}
+
+static int csi2rx_parse_dt(struct csi2rx_priv *csi2rx)
+{
+	struct v4l2_fwnode_endpoint v4l2_ep;
+	struct fwnode_handle *fwh;
+	struct device_node *ep;
+	int ret;
+
+	ep = of_graph_get_endpoint_by_regs(csi2rx->dev->of_node, 0, 0);
+	if (!ep)
+		return -EINVAL;
+
+	fwh = of_fwnode_handle(ep);
+	ret = v4l2_fwnode_endpoint_parse(fwh, &v4l2_ep);
+	if (ret) {
+		dev_err(csi2rx->dev, "Could not parse v4l2 endpoint\n");
+		of_node_put(ep);
+		return ret;
+	}
+
+	if (v4l2_ep.bus_type != V4L2_MBUS_CSI2) {
+		dev_err(csi2rx->dev, "Unsupported media bus type: 0x%x\n",
+			v4l2_ep.bus_type);
+		of_node_put(ep);
+		return -EINVAL;
+	}
+
+	memcpy(csi2rx->lanes, v4l2_ep.bus.mipi_csi2.data_lanes,
+	       sizeof(csi2rx->lanes));
+	csi2rx->num_lanes = v4l2_ep.bus.mipi_csi2.num_data_lanes;
+	if (csi2rx->num_lanes > csi2rx->max_lanes) {
+		dev_err(csi2rx->dev, "Unsupported number of data-lanes: %d\n",
+			csi2rx->num_lanes);
+		of_node_put(ep);
+		return -EINVAL;
+	}
+
+	csi2rx->asd.match.fwnode = fwnode_graph_get_remote_port_parent(fwh);
+	csi2rx->asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
+	of_node_put(ep);
+
+	csi2rx->notifier.subdevs = devm_kzalloc(csi2rx->dev,
+						sizeof(*csi2rx->notifier.subdevs),
+						GFP_KERNEL);
+	if (!csi2rx->notifier.subdevs)
+		return -ENOMEM;
+
+	csi2rx->notifier.subdevs[0] = &csi2rx->asd;
+	csi2rx->notifier.num_subdevs = 1;
+	csi2rx->notifier.ops = &csi2rx_notifier_ops;
+
+	return v4l2_async_subdev_notifier_register(&csi2rx->subdev,
+						   &csi2rx->notifier);
+}
+
+static int csi2rx_probe(struct platform_device *pdev)
+{
+	struct csi2rx_priv *csi2rx;
+	unsigned int i;
+	int ret;
+
+	csi2rx = kzalloc(sizeof(*csi2rx), GFP_KERNEL);
+	if (!csi2rx)
+		return -ENOMEM;
+	platform_set_drvdata(pdev, csi2rx);
+	csi2rx->dev = &pdev->dev;
+	mutex_init(&csi2rx->lock);
+
+	ret = csi2rx_get_resources(csi2rx, pdev);
+	if (ret)
+		goto err_free_priv;
+
+	ret = csi2rx_parse_dt(csi2rx);
+	if (ret)
+		goto err_free_priv;
+
+	csi2rx->subdev.owner = THIS_MODULE;
+	csi2rx->subdev.dev = &pdev->dev;
+	v4l2_subdev_init(&csi2rx->subdev, &csi2rx_subdev_ops);
+	v4l2_set_subdevdata(&csi2rx->subdev, &pdev->dev);
+	snprintf(csi2rx->subdev.name, V4L2_SUBDEV_NAME_SIZE, "%s.%s",
+		 KBUILD_MODNAME, dev_name(&pdev->dev));
+
+	/* Create our media pads */
+	csi2rx->subdev.entity.function = MEDIA_ENT_F_VID_IF_BRIDGE;
+	csi2rx->pads[CSI2RX_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
+	for (i = CSI2RX_PAD_SOURCE_STREAM0; i < CSI2RX_PAD_MAX; i++)
+		csi2rx->pads[i].flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&csi2rx->subdev.entity, CSI2RX_PAD_MAX,
+				     csi2rx->pads);
+	if (ret)
+		goto err_free_priv;
+
+	ret = v4l2_async_register_subdev(&csi2rx->subdev);
+	if (ret < 0)
+		goto err_free_priv;
+
+	dev_info(&pdev->dev,
+		 "Probed CSI2RX with %u/%u lanes, %u streams, %s D-PHY\n",
+		 csi2rx->num_lanes, csi2rx->max_lanes, csi2rx->max_streams,
+		 csi2rx->has_internal_dphy ? "internal" : "no");
+
+	return 0;
+
+err_free_priv:
+	kfree(csi2rx);
+	return ret;
+}
+
+static int csi2rx_remove(struct platform_device *pdev)
+{
+	struct csi2rx_priv *csi2rx = platform_get_drvdata(pdev);
+
+	v4l2_async_unregister_subdev(&csi2rx->subdev);
+	kfree(csi2rx);
+
+	return 0;
+}
+
+static const struct of_device_id csi2rx_of_table[] = {
+	{ .compatible = "cdns,csi2rx" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, csi2rx_of_table);
+
+static struct platform_driver csi2rx_driver = {
+	.probe	= csi2rx_probe,
+	.remove	= csi2rx_remove,
+
+	.driver	= {
+		.name		= "cdns-csi2rx",
+		.of_match_table	= csi2rx_of_table,
+	},
+};
+module_platform_driver(csi2rx_driver);
+MODULE_AUTHOR("Maxime Ripard <maxime.ripard@bootlin.com>");
+MODULE_DESCRIPTION("Cadence CSI2-RX controller");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/cadence/cdns-csi2tx.c b/drivers/media/platform/cadence/cdns-csi2tx.c
new file mode 100644
index 000000000000..dfa1d88d955b
--- /dev/null
+++ b/drivers/media/platform/cadence/cdns-csi2tx.c
@@ -0,0 +1,563 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Driver for Cadence MIPI-CSI2 TX Controller
+ *
+ * Copyright (C) 2017-2018 Cadence Design Systems Inc.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-subdev.h>
+
+#define CSI2TX_DEVICE_CONFIG_REG	0x00
+#define CSI2TX_DEVICE_CONFIG_STREAMS_MASK	GENMASK(6, 4)
+#define CSI2TX_DEVICE_CONFIG_HAS_DPHY		BIT(3)
+#define CSI2TX_DEVICE_CONFIG_LANES_MASK		GENMASK(2, 0)
+
+#define CSI2TX_CONFIG_REG		0x20
+#define CSI2TX_CONFIG_CFG_REQ			BIT(2)
+#define CSI2TX_CONFIG_SRST_REQ			BIT(1)
+
+#define CSI2TX_DPHY_CFG_REG		0x28
+#define CSI2TX_DPHY_CFG_CLK_RESET		BIT(16)
+#define CSI2TX_DPHY_CFG_LANE_RESET(n)		BIT((n) + 12)
+#define CSI2TX_DPHY_CFG_MODE_MASK		GENMASK(9, 8)
+#define CSI2TX_DPHY_CFG_MODE_LPDT		(2 << 8)
+#define CSI2TX_DPHY_CFG_MODE_HS			(1 << 8)
+#define CSI2TX_DPHY_CFG_MODE_ULPS		(0 << 8)
+#define CSI2TX_DPHY_CFG_CLK_ENABLE		BIT(4)
+#define CSI2TX_DPHY_CFG_LANE_ENABLE(n)		BIT(n)
+
+#define CSI2TX_DPHY_CLK_WAKEUP_REG	0x2c
+#define CSI2TX_DPHY_CLK_WAKEUP_ULPS_CYCLES(n)	((n) & 0xffff)
+
+#define CSI2TX_DT_CFG_REG(n)		(0x80 + (n) * 8)
+#define CSI2TX_DT_CFG_DT(n)			(((n) & 0x3f) << 2)
+
+#define CSI2TX_DT_FORMAT_REG(n)		(0x84 + (n) * 8)
+#define CSI2TX_DT_FORMAT_BYTES_PER_LINE(n)	(((n) & 0xffff) << 16)
+#define CSI2TX_DT_FORMAT_MAX_LINE_NUM(n)	((n) & 0xffff)
+
+#define CSI2TX_STREAM_IF_CFG_REG(n)	(0x100 + (n) * 4)
+#define CSI2TX_STREAM_IF_CFG_FILL_LEVEL(n)	((n) & 0x1f)
+
+#define CSI2TX_LANES_MAX	4
+#define CSI2TX_STREAMS_MAX	4
+
+enum csi2tx_pads {
+	CSI2TX_PAD_SOURCE,
+	CSI2TX_PAD_SINK_STREAM0,
+	CSI2TX_PAD_SINK_STREAM1,
+	CSI2TX_PAD_SINK_STREAM2,
+	CSI2TX_PAD_SINK_STREAM3,
+	CSI2TX_PAD_MAX,
+};
+
+struct csi2tx_fmt {
+	u32	mbus;
+	u32	dt;
+	u32	bpp;
+};
+
+struct csi2tx_priv {
+	struct device			*dev;
+	unsigned int			count;
+
+	/*
+	 * Used to prevent race conditions between multiple,
+	 * concurrent calls to start and stop.
+	 */
+	struct mutex			lock;
+
+	void __iomem			*base;
+
+	struct clk			*esc_clk;
+	struct clk			*p_clk;
+	struct clk			*pixel_clk[CSI2TX_STREAMS_MAX];
+
+	struct v4l2_subdev		subdev;
+	struct media_pad		pads[CSI2TX_PAD_MAX];
+	struct v4l2_mbus_framefmt	pad_fmts[CSI2TX_PAD_MAX];
+
+	bool				has_internal_dphy;
+	u8				lanes[CSI2TX_LANES_MAX];
+	unsigned int			num_lanes;
+	unsigned int			max_lanes;
+	unsigned int			max_streams;
+};
+
+static const struct csi2tx_fmt csi2tx_formats[] = {
+	{
+		.mbus	= MEDIA_BUS_FMT_UYVY8_1X16,
+		.bpp	= 2,
+		.dt	= 0x1e,
+	},
+	{
+		.mbus	= MEDIA_BUS_FMT_RGB888_1X24,
+		.bpp	= 3,
+		.dt	= 0x24,
+	},
+};
+
+static const struct v4l2_mbus_framefmt fmt_default = {
+	.width		= 1280,
+	.height		= 720,
+	.code		= MEDIA_BUS_FMT_RGB888_1X24,
+	.field		= V4L2_FIELD_NONE,
+	.colorspace	= V4L2_COLORSPACE_DEFAULT,
+};
+
+static inline
+struct csi2tx_priv *v4l2_subdev_to_csi2tx(struct v4l2_subdev *subdev)
+{
+	return container_of(subdev, struct csi2tx_priv, subdev);
+}
+
+static const struct csi2tx_fmt *csi2tx_get_fmt_from_mbus(u32 mbus)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(csi2tx_formats); i++)
+		if (csi2tx_formats[i].mbus == mbus)
+			return &csi2tx_formats[i];
+
+	return NULL;
+}
+
+static int csi2tx_enum_mbus_code(struct v4l2_subdev *subdev,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->pad || code->index >= ARRAY_SIZE(csi2tx_formats))
+		return -EINVAL;
+
+	code->code = csi2tx_formats[code->index].mbus;
+
+	return 0;
+}
+
+static struct v4l2_mbus_framefmt *
+__csi2tx_get_pad_format(struct v4l2_subdev *subdev,
+			struct v4l2_subdev_pad_config *cfg,
+			struct v4l2_subdev_format *fmt)
+{
+	struct csi2tx_priv *csi2tx = v4l2_subdev_to_csi2tx(subdev);
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_get_try_format(subdev, cfg,
+						  fmt->pad);
+
+	return &csi2tx->pad_fmts[fmt->pad];
+}
+
+static int csi2tx_get_pad_format(struct v4l2_subdev *subdev,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_format *fmt)
+{
+	const struct v4l2_mbus_framefmt *format;
+
+	/* Multiplexed pad? */
+	if (fmt->pad == CSI2TX_PAD_SOURCE)
+		return -EINVAL;
+
+	format = __csi2tx_get_pad_format(subdev, cfg, fmt);
+	if (!format)
+		return -EINVAL;
+
+	fmt->format = *format;
+
+	return 0;
+}
+
+static int csi2tx_set_pad_format(struct v4l2_subdev *subdev,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_format *fmt)
+{
+	const struct v4l2_mbus_framefmt *src_format = &fmt->format;
+	struct v4l2_mbus_framefmt *dst_format;
+
+	/* Multiplexed pad? */
+	if (fmt->pad == CSI2TX_PAD_SOURCE)
+		return -EINVAL;
+
+	if (!csi2tx_get_fmt_from_mbus(fmt->format.code))
+		src_format = &fmt_default;
+
+	dst_format = __csi2tx_get_pad_format(subdev, cfg, fmt);
+	if (!dst_format)
+		return -EINVAL;
+
+	*dst_format = *src_format;
+
+	return 0;
+}
+
+static const struct v4l2_subdev_pad_ops csi2tx_pad_ops = {
+	.enum_mbus_code	= csi2tx_enum_mbus_code,
+	.get_fmt	= csi2tx_get_pad_format,
+	.set_fmt	= csi2tx_set_pad_format,
+};
+
+static void csi2tx_reset(struct csi2tx_priv *csi2tx)
+{
+	writel(CSI2TX_CONFIG_SRST_REQ, csi2tx->base + CSI2TX_CONFIG_REG);
+
+	udelay(10);
+}
+
+static int csi2tx_start(struct csi2tx_priv *csi2tx)
+{
+	struct media_entity *entity = &csi2tx->subdev.entity;
+	struct media_link *link;
+	unsigned int i;
+	u32 reg;
+
+	csi2tx_reset(csi2tx);
+
+	writel(CSI2TX_CONFIG_CFG_REQ, csi2tx->base + CSI2TX_CONFIG_REG);
+
+	udelay(10);
+
+	/* Configure our PPI interface with the D-PHY */
+	writel(CSI2TX_DPHY_CLK_WAKEUP_ULPS_CYCLES(32),
+	       csi2tx->base + CSI2TX_DPHY_CLK_WAKEUP_REG);
+
+	/* Put our lanes (clock and data) out of reset */
+	reg = CSI2TX_DPHY_CFG_CLK_RESET | CSI2TX_DPHY_CFG_MODE_LPDT;
+	for (i = 0; i < csi2tx->num_lanes; i++)
+		reg |= CSI2TX_DPHY_CFG_LANE_RESET(csi2tx->lanes[i]);
+	writel(reg, csi2tx->base + CSI2TX_DPHY_CFG_REG);
+
+	udelay(10);
+
+	/* Enable our (clock and data) lanes */
+	reg |= CSI2TX_DPHY_CFG_CLK_ENABLE;
+	for (i = 0; i < csi2tx->num_lanes; i++)
+		reg |= CSI2TX_DPHY_CFG_LANE_ENABLE(csi2tx->lanes[i]);
+	writel(reg, csi2tx->base + CSI2TX_DPHY_CFG_REG);
+
+	udelay(10);
+
+	/* Switch to HS mode */
+	reg &= ~CSI2TX_DPHY_CFG_MODE_MASK;
+	writel(reg | CSI2TX_DPHY_CFG_MODE_HS,
+	       csi2tx->base + CSI2TX_DPHY_CFG_REG);
+
+	udelay(10);
+
+	/*
+	 * Create a static mapping between the CSI virtual channels
+	 * and the input streams.
+	 *
+	 * This should be enhanced, but v4l2 lacks the support for
+	 * changing that mapping dynamically at the moment.
+	 *
+	 * We're protected from the userspace setting up links at the
+	 * same time by the upper layer having called
+	 * media_pipeline_start().
+	 */
+	list_for_each_entry(link, &entity->links, list) {
+		struct v4l2_mbus_framefmt *mfmt;
+		const struct csi2tx_fmt *fmt;
+		unsigned int stream;
+		int pad_idx = -1;
+
+		/* Only consider our enabled input pads */
+		for (i = CSI2TX_PAD_SINK_STREAM0; i < CSI2TX_PAD_MAX; i++) {
+			struct media_pad *pad = &csi2tx->pads[i];
+
+			if ((pad == link->sink) &&
+			    (link->flags & MEDIA_LNK_FL_ENABLED)) {
+				pad_idx = i;
+				break;
+			}
+		}
+
+		if (pad_idx < 0)
+			continue;
+
+		mfmt = &csi2tx->pad_fmts[pad_idx];
+		fmt = csi2tx_get_fmt_from_mbus(mfmt->code);
+		if (!fmt)
+			continue;
+
+		stream = pad_idx - CSI2TX_PAD_SINK_STREAM0;
+
+		/*
+		 * We use the stream ID there, but it's wrong.
+		 *
+		 * A stream could very well send a data type that is
+		 * not equal to its stream ID. We need to find a
+		 * proper way to address it.
+		 */
+		writel(CSI2TX_DT_CFG_DT(fmt->dt),
+		       csi2tx->base + CSI2TX_DT_CFG_REG(stream));
+
+		writel(CSI2TX_DT_FORMAT_BYTES_PER_LINE(mfmt->width * fmt->bpp) |
+		       CSI2TX_DT_FORMAT_MAX_LINE_NUM(mfmt->height + 1),
+		       csi2tx->base + CSI2TX_DT_FORMAT_REG(stream));
+
+		/*
+		 * TODO: This needs to be calculated based on the
+		 * output CSI2 clock rate.
+		 */
+		writel(CSI2TX_STREAM_IF_CFG_FILL_LEVEL(4),
+		       csi2tx->base + CSI2TX_STREAM_IF_CFG_REG(stream));
+	}
+
+	/* Disable the configuration mode */
+	writel(0, csi2tx->base + CSI2TX_CONFIG_REG);
+
+	return 0;
+}
+
+static void csi2tx_stop(struct csi2tx_priv *csi2tx)
+{
+	writel(CSI2TX_CONFIG_CFG_REQ | CSI2TX_CONFIG_SRST_REQ,
+	       csi2tx->base + CSI2TX_CONFIG_REG);
+}
+
+static int csi2tx_s_stream(struct v4l2_subdev *subdev, int enable)
+{
+	struct csi2tx_priv *csi2tx = v4l2_subdev_to_csi2tx(subdev);
+	int ret = 0;
+
+	mutex_lock(&csi2tx->lock);
+
+	if (enable) {
+		/*
+		 * If we're not the first users, there's no need to
+		 * enable the whole controller.
+		 */
+		if (!csi2tx->count) {
+			ret = csi2tx_start(csi2tx);
+			if (ret)
+				goto out;
+		}
+
+		csi2tx->count++;
+	} else {
+		csi2tx->count--;
+
+		/*
+		 * Let the last user turn off the lights.
+		 */
+		if (!csi2tx->count)
+			csi2tx_stop(csi2tx);
+	}
+
+out:
+	mutex_unlock(&csi2tx->lock);
+	return ret;
+}
+
+static const struct v4l2_subdev_video_ops csi2tx_video_ops = {
+	.s_stream	= csi2tx_s_stream,
+};
+
+static const struct v4l2_subdev_ops csi2tx_subdev_ops = {
+	.pad		= &csi2tx_pad_ops,
+	.video		= &csi2tx_video_ops,
+};
+
+static int csi2tx_get_resources(struct csi2tx_priv *csi2tx,
+				struct platform_device *pdev)
+{
+	struct resource *res;
+	unsigned int i;
+	u32 dev_cfg;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	csi2tx->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(csi2tx->base))
+		return PTR_ERR(csi2tx->base);
+
+	csi2tx->p_clk = devm_clk_get(&pdev->dev, "p_clk");
+	if (IS_ERR(csi2tx->p_clk)) {
+		dev_err(&pdev->dev, "Couldn't get p_clk\n");
+		return PTR_ERR(csi2tx->p_clk);
+	}
+
+	csi2tx->esc_clk = devm_clk_get(&pdev->dev, "esc_clk");
+	if (IS_ERR(csi2tx->esc_clk)) {
+		dev_err(&pdev->dev, "Couldn't get the esc_clk\n");
+		return PTR_ERR(csi2tx->esc_clk);
+	}
+
+	clk_prepare_enable(csi2tx->p_clk);
+	dev_cfg = readl(csi2tx->base + CSI2TX_DEVICE_CONFIG_REG);
+	clk_disable_unprepare(csi2tx->p_clk);
+
+	csi2tx->max_lanes = dev_cfg & CSI2TX_DEVICE_CONFIG_LANES_MASK;
+	if (csi2tx->max_lanes > CSI2TX_LANES_MAX) {
+		dev_err(&pdev->dev, "Invalid number of lanes: %u\n",
+			csi2tx->max_lanes);
+		return -EINVAL;
+	}
+
+	csi2tx->max_streams = (dev_cfg & CSI2TX_DEVICE_CONFIG_STREAMS_MASK) >> 4;
+	if (csi2tx->max_streams > CSI2TX_STREAMS_MAX) {
+		dev_err(&pdev->dev, "Invalid number of streams: %u\n",
+			csi2tx->max_streams);
+		return -EINVAL;
+	}
+
+	csi2tx->has_internal_dphy = !!(dev_cfg & CSI2TX_DEVICE_CONFIG_HAS_DPHY);
+
+	for (i = 0; i < csi2tx->max_streams; i++) {
+		char clk_name[16];
+
+		snprintf(clk_name, sizeof(clk_name), "pixel_if%u_clk", i);
+		csi2tx->pixel_clk[i] = devm_clk_get(&pdev->dev, clk_name);
+		if (IS_ERR(csi2tx->pixel_clk[i])) {
+			dev_err(&pdev->dev, "Couldn't get clock %s\n",
+				clk_name);
+			return PTR_ERR(csi2tx->pixel_clk[i]);
+		}
+	}
+
+	return 0;
+}
+
+static int csi2tx_check_lanes(struct csi2tx_priv *csi2tx)
+{
+	struct v4l2_fwnode_endpoint v4l2_ep;
+	struct device_node *ep;
+	int ret;
+
+	ep = of_graph_get_endpoint_by_regs(csi2tx->dev->of_node, 0, 0);
+	if (!ep)
+		return -EINVAL;
+
+	ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep), &v4l2_ep);
+	if (ret) {
+		dev_err(csi2tx->dev, "Could not parse v4l2 endpoint\n");
+		goto out;
+	}
+
+	if (v4l2_ep.bus_type != V4L2_MBUS_CSI2) {
+		dev_err(csi2tx->dev, "Unsupported media bus type: 0x%x\n",
+			v4l2_ep.bus_type);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	csi2tx->num_lanes = v4l2_ep.bus.mipi_csi2.num_data_lanes;
+	if (csi2tx->num_lanes > csi2tx->max_lanes) {
+		dev_err(csi2tx->dev,
+			"Current configuration uses more lanes than supported\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	memcpy(csi2tx->lanes, v4l2_ep.bus.mipi_csi2.data_lanes,
+	       sizeof(csi2tx->lanes));
+
+out:
+	of_node_put(ep);
+	return ret;
+}
+
+static int csi2tx_probe(struct platform_device *pdev)
+{
+	struct csi2tx_priv *csi2tx;
+	unsigned int i;
+	int ret;
+
+	csi2tx = kzalloc(sizeof(*csi2tx), GFP_KERNEL);
+	if (!csi2tx)
+		return -ENOMEM;
+	platform_set_drvdata(pdev, csi2tx);
+	mutex_init(&csi2tx->lock);
+	csi2tx->dev = &pdev->dev;
+
+	ret = csi2tx_get_resources(csi2tx, pdev);
+	if (ret)
+		goto err_free_priv;
+
+	v4l2_subdev_init(&csi2tx->subdev, &csi2tx_subdev_ops);
+	csi2tx->subdev.owner = THIS_MODULE;
+	csi2tx->subdev.dev = &pdev->dev;
+	csi2tx->subdev.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	snprintf(csi2tx->subdev.name, V4L2_SUBDEV_NAME_SIZE, "%s.%s",
+		 KBUILD_MODNAME, dev_name(&pdev->dev));
+
+	ret = csi2tx_check_lanes(csi2tx);
+	if (ret)
+		goto err_free_priv;
+
+	/* Create our media pads */
+	csi2tx->subdev.entity.function = MEDIA_ENT_F_VID_IF_BRIDGE;
+	csi2tx->pads[CSI2TX_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
+	for (i = CSI2TX_PAD_SINK_STREAM0; i < CSI2TX_PAD_MAX; i++)
+		csi2tx->pads[i].flags = MEDIA_PAD_FL_SINK;
+
+	/*
+	 * Only the input pads are considered to have a format at the
+	 * moment. The CSI link can multiplex various streams with
+	 * different formats, and we can't expose this in v4l2 right
+	 * now.
+	 */
+	for (i = CSI2TX_PAD_SINK_STREAM0; i < CSI2TX_PAD_MAX; i++)
+		csi2tx->pad_fmts[i] = fmt_default;
+
+	ret = media_entity_pads_init(&csi2tx->subdev.entity, CSI2TX_PAD_MAX,
+				     csi2tx->pads);
+	if (ret)
+		goto err_free_priv;
+
+	ret = v4l2_async_register_subdev(&csi2tx->subdev);
+	if (ret < 0)
+		goto err_free_priv;
+
+	dev_info(&pdev->dev,
+		 "Probed CSI2TX with %u/%u lanes, %u streams, %s D-PHY\n",
+		 csi2tx->num_lanes, csi2tx->max_lanes, csi2tx->max_streams,
+		 csi2tx->has_internal_dphy ? "internal" : "no");
+
+	return 0;
+
+err_free_priv:
+	kfree(csi2tx);
+	return ret;
+}
+
+static int csi2tx_remove(struct platform_device *pdev)
+{
+	struct csi2tx_priv *csi2tx = platform_get_drvdata(pdev);
+
+	v4l2_async_unregister_subdev(&csi2tx->subdev);
+	kfree(csi2tx);
+
+	return 0;
+}
+
+static const struct of_device_id csi2tx_of_table[] = {
+	{ .compatible = "cdns,csi2tx" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, csi2tx_of_table);
+
+static struct platform_driver csi2tx_driver = {
+	.probe	= csi2tx_probe,
+	.remove	= csi2tx_remove,
+
+	.driver	= {
+		.name		= "cdns-csi2tx",
+		.of_match_table	= csi2tx_of_table,
+	},
+};
+module_platform_driver(csi2tx_driver);
+MODULE_AUTHOR("Maxime Ripard <maxime.ripard@bootlin.com>");
+MODULE_DESCRIPTION("Cadence CSI2-TX controller");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/cec-gpio/cec-gpio.c b/drivers/media/platform/cec-gpio/cec-gpio.c
index f1f28cf5c751..69f8242209c2 100644
--- a/drivers/media/platform/cec-gpio/cec-gpio.c
+++ b/drivers/media/platform/cec-gpio/cec-gpio.c
@@ -158,7 +158,7 @@ static int cec_gpio_probe(struct platform_device *pdev)
 
 	cec->dev = dev;
 
-	cec->cec_gpio = devm_gpiod_get(dev, "cec", GPIOD_IN);
+	cec->cec_gpio = devm_gpiod_get(dev, "cec", GPIOD_OUT_HIGH_OPEN_DRAIN);
 	if (IS_ERR(cec->cec_gpio))
 		return PTR_ERR(cec->cec_gpio);
 	cec->cec_irq = gpiod_to_irq(cec->cec_gpio);
diff --git a/drivers/media/platform/coda/coda-common.c b/drivers/media/platform/coda/coda-common.c
index 04e35d70ce2e..c7631e117dd3 100644
--- a/drivers/media/platform/coda/coda-common.c
+++ b/drivers/media/platform/coda/coda-common.c
@@ -779,16 +779,27 @@ static int coda_s_fmt_vid_cap(struct file *file, void *priv,
 	r.width = q_data_src->width;
 	r.height = q_data_src->height;
 
-	return coda_s_fmt(ctx, f, &r);
+	ret = coda_s_fmt(ctx, f, &r);
+	if (ret)
+		return ret;
+
+	if (ctx->inst_type != CODA_INST_ENCODER)
+		return 0;
+
+	ctx->colorspace = f->fmt.pix.colorspace;
+	ctx->xfer_func = f->fmt.pix.xfer_func;
+	ctx->ycbcr_enc = f->fmt.pix.ycbcr_enc;
+	ctx->quantization = f->fmt.pix.quantization;
+
+	return 0;
 }
 
 static int coda_s_fmt_vid_out(struct file *file, void *priv,
 			      struct v4l2_format *f)
 {
 	struct coda_ctx *ctx = fh_to_ctx(priv);
-	struct coda_q_data *q_data_src;
 	struct v4l2_format f_cap;
-	struct v4l2_rect r;
+	struct vb2_queue *dst_vq;
 	int ret;
 
 	ret = coda_try_fmt_vid_out(file, priv, f);
@@ -799,28 +810,34 @@ static int coda_s_fmt_vid_out(struct file *file, void *priv,
 	if (ret)
 		return ret;
 
+	if (ctx->inst_type != CODA_INST_DECODER)
+		return 0;
+
 	ctx->colorspace = f->fmt.pix.colorspace;
 	ctx->xfer_func = f->fmt.pix.xfer_func;
 	ctx->ycbcr_enc = f->fmt.pix.ycbcr_enc;
 	ctx->quantization = f->fmt.pix.quantization;
 
+	dst_vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
+	if (!dst_vq)
+		return -EINVAL;
+
+	/*
+	 * Setting the capture queue format is not possible while the capture
+	 * queue is still busy. This is not an error, but the user will have to
+	 * make sure themselves that the capture format is set correctly before
+	 * starting the output queue again.
+	 */
+	if (vb2_is_busy(dst_vq))
+		return 0;
+
 	memset(&f_cap, 0, sizeof(f_cap));
 	f_cap.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
 	coda_g_fmt(file, priv, &f_cap);
 	f_cap.fmt.pix.width = f->fmt.pix.width;
 	f_cap.fmt.pix.height = f->fmt.pix.height;
 
-	ret = coda_try_fmt_vid_cap(file, priv, &f_cap);
-	if (ret)
-		return ret;
-
-	q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
-	r.left = 0;
-	r.top = 0;
-	r.width = q_data_src->width;
-	r.height = q_data_src->height;
-
-	return coda_s_fmt(ctx, &f_cap, &r);
+	return coda_s_fmt_vid_cap(file, priv, &f_cap);
 }
 
 static int coda_reqbufs(struct file *file, void *priv,
diff --git a/drivers/media/platform/davinci/Kconfig b/drivers/media/platform/davinci/Kconfig
index 55982e681d77..06b5e581f25f 100644
--- a/drivers/media/platform/davinci/Kconfig
+++ b/drivers/media/platform/davinci/Kconfig
@@ -2,7 +2,6 @@ config VIDEO_DAVINCI_VPIF_DISPLAY
 	tristate "TI DaVinci VPIF V4L2-Display driver"
 	depends on VIDEO_V4L2
 	depends on ARCH_DAVINCI || COMPILE_TEST
-	depends on HAS_DMA
 	depends on I2C
 	select VIDEOBUF2_DMA_CONTIG
 	select VIDEO_ADV7343 if MEDIA_SUBDRV_AUTOSELECT
@@ -19,7 +18,6 @@ config VIDEO_DAVINCI_VPIF_CAPTURE
 	tristate "TI DaVinci VPIF video capture driver"
 	depends on VIDEO_V4L2
 	depends on ARCH_DAVINCI || COMPILE_TEST
-	depends on HAS_DMA
 	depends on I2C
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
@@ -35,7 +33,6 @@ config VIDEO_DM6446_CCDC
 	tristate "TI DM6446 CCDC video capture driver"
 	depends on VIDEO_V4L2
 	depends on ARCH_DAVINCI || COMPILE_TEST
-	depends on HAS_DMA
 	depends on I2C
 	select VIDEOBUF_DMA_CONTIG
 	help
@@ -52,7 +49,6 @@ config VIDEO_DM355_CCDC
 	tristate "TI DM355 CCDC video capture driver"
 	depends on VIDEO_V4L2
 	depends on ARCH_DAVINCI || COMPILE_TEST
-	depends on HAS_DMA
 	depends on I2C
 	select VIDEOBUF_DMA_CONTIG
 	help
@@ -67,8 +63,8 @@ config VIDEO_DM355_CCDC
 
 config VIDEO_DM365_ISIF
 	tristate "TI DM365 ISIF video capture driver"
-	depends on VIDEO_V4L2 && ARCH_DAVINCI
-	depends on HAS_DMA
+	depends on VIDEO_V4L2
+	depends on ARCH_DAVINCI || COMPILE_TEST
 	depends on I2C
 	select VIDEOBUF_DMA_CONTIG
 	help
@@ -81,8 +77,8 @@ config VIDEO_DM365_ISIF
 
 config VIDEO_DAVINCI_VPBE_DISPLAY
 	tristate "TI DaVinci VPBE V4L2-Display driver"
-	depends on VIDEO_V4L2 && ARCH_DAVINCI
-	depends on HAS_DMA
+	depends on VIDEO_V4L2
+	depends on ARCH_DAVINCI || COMPILE_TEST
 	depends on I2C
 	select VIDEOBUF2_DMA_CONTIG
 	help
diff --git a/drivers/media/platform/davinci/isif.c b/drivers/media/platform/davinci/isif.c
index d5ff58494c1e..f924e76e2fbf 100644
--- a/drivers/media/platform/davinci/isif.c
+++ b/drivers/media/platform/davinci/isif.c
@@ -31,8 +31,6 @@
 #include <linux/err.h>
 #include <linux/module.h>
 
-#include <mach/mux.h>
-
 #include <media/davinci/isif.h>
 #include <media/davinci/vpss.h>
 
@@ -1029,7 +1027,7 @@ static int isif_probe(struct platform_device *pdev)
 {
 	void (*setup_pinmux)(void);
 	struct resource	*res;
-	void *__iomem addr;
+	void __iomem *addr;
 	int status = 0, i;
 
 	/* Platform data holds setup_pinmux function ptr */
diff --git a/drivers/media/platform/davinci/vpbe.c b/drivers/media/platform/davinci/vpbe.c
index 7f6462562579..18c035ef84cf 100644
--- a/drivers/media/platform/davinci/vpbe.c
+++ b/drivers/media/platform/davinci/vpbe.c
@@ -51,7 +51,7 @@ MODULE_AUTHOR("Texas Instruments");
 
 /**
  * vpbe_current_encoder_info - Get config info for current encoder
- * @vpbe_dev - vpbe device ptr
+ * @vpbe_dev: vpbe device ptr
  *
  * Return ptr to current encoder config info
  */
@@ -68,8 +68,8 @@ vpbe_current_encoder_info(struct vpbe_device *vpbe_dev)
 /**
  * vpbe_find_encoder_sd_index - Given a name find encoder sd index
  *
- * @vpbe_config - ptr to vpbe cfg
- * @output_index - index used by application
+ * @cfg: ptr to vpbe cfg
+ * @index: index used by application
  *
  * Return sd index of the encoder
  */
@@ -94,8 +94,8 @@ static int vpbe_find_encoder_sd_index(struct vpbe_config *cfg,
 
 /**
  * vpbe_g_cropcap - Get crop capabilities of the display
- * @vpbe_dev - vpbe device ptr
- * @cropcap - cropcap is a ptr to struct v4l2_cropcap
+ * @vpbe_dev: vpbe device ptr
+ * @cropcap: cropcap is a ptr to struct v4l2_cropcap
  *
  * Update the crop capabilities in crop cap for current
  * mode
@@ -116,8 +116,8 @@ static int vpbe_g_cropcap(struct vpbe_device *vpbe_dev,
 
 /**
  * vpbe_enum_outputs - enumerate outputs
- * @vpbe_dev - vpbe device ptr
- * @output - ptr to v4l2_output structure
+ * @vpbe_dev: vpbe device ptr
+ * @output: ptr to v4l2_output structure
  *
  * Enumerates the outputs available at the vpbe display
  * returns the status, -EINVAL if end of output list
@@ -212,8 +212,8 @@ static int vpbe_get_std_info_by_name(struct vpbe_device *vpbe_dev,
 
 /**
  * vpbe_set_output - Set output
- * @vpbe_dev - vpbe device ptr
- * @index - index of output
+ * @vpbe_dev: vpbe device ptr
+ * @index: index of output
  *
  * Set vpbe output to the output specified by the index
  */
@@ -308,7 +308,7 @@ static int vpbe_set_default_output(struct vpbe_device *vpbe_dev)
 
 /**
  * vpbe_get_output - Get output
- * @vpbe_dev - vpbe device ptr
+ * @vpbe_dev: vpbe device ptr
  *
  * return current vpbe output to the the index
  */
@@ -317,7 +317,7 @@ static unsigned int vpbe_get_output(struct vpbe_device *vpbe_dev)
 	return vpbe_dev->current_out_index;
 }
 
-/**
+/*
  * vpbe_s_dv_timings - Set the given preset timings in the encoder
  *
  * Sets the timings if supported by the current encoder. Return the status.
@@ -369,7 +369,7 @@ static int vpbe_s_dv_timings(struct vpbe_device *vpbe_dev,
 	return ret;
 }
 
-/**
+/*
  * vpbe_g_dv_timings - Get the timings in the current encoder
  *
  * Get the timings in the current encoder. Return the status. 0 - success
@@ -394,7 +394,7 @@ static int vpbe_g_dv_timings(struct vpbe_device *vpbe_dev,
 	return -EINVAL;
 }
 
-/**
+/*
  * vpbe_enum_dv_timings - Enumerate the dv timings in the current encoder
  *
  * Get the timings in the current encoder. Return the status. 0 - success
@@ -426,7 +426,7 @@ static int vpbe_enum_dv_timings(struct vpbe_device *vpbe_dev,
 	return 0;
 }
 
-/**
+/*
  * vpbe_s_std - Set the given standard in the encoder
  *
  * Sets the standard if supported by the current encoder. Return the status.
@@ -465,7 +465,7 @@ static int vpbe_s_std(struct vpbe_device *vpbe_dev, v4l2_std_id std_id)
 	return ret;
 }
 
-/**
+/*
  * vpbe_g_std - Get the standard in the current encoder
  *
  * Get the standard in the current encoder. Return the status. 0 - success
@@ -488,7 +488,7 @@ static int vpbe_g_std(struct vpbe_device *vpbe_dev, v4l2_std_id *std_id)
 	return -EINVAL;
 }
 
-/**
+/*
  * vpbe_set_mode - Set mode in the current encoder using mode info
  *
  * Use the mode string to decide what timings to set in the encoder
@@ -572,7 +572,8 @@ static int platform_device_get(struct device *dev, void *data)
 
 /**
  * vpbe_initialize() - Initialize the vpbe display controller
- * @vpbe_dev - vpbe device ptr
+ * @dev: Master and slave device ptr
+ * @vpbe_dev: vpbe device ptr
  *
  * Master frame buffer device drivers calls this to initialize vpbe
  * display controller. This will then registers v4l2 device and the sub
@@ -769,7 +770,8 @@ static int vpbe_initialize(struct device *dev, struct vpbe_device *vpbe_dev)
 
 /**
  * vpbe_deinitialize() - de-initialize the vpbe display controller
- * @dev - Master and slave device ptr
+ * @dev: Master and slave device ptr
+ * @vpbe_dev: vpbe device ptr
  *
  * vpbe_master and slave frame buffer devices calls this to de-initialize
  * the display controller. It is called when master and slave device
diff --git a/drivers/media/platform/davinci/vpbe_display.c b/drivers/media/platform/davinci/vpbe_display.c
index 6aabd21fe69f..b0eb3d899eb4 100644
--- a/drivers/media/platform/davinci/vpbe_display.c
+++ b/drivers/media/platform/davinci/vpbe_display.c
@@ -26,7 +26,10 @@
 #include <linux/slab.h>
 
 #include <asm/pgtable.h>
+
+#ifdef CONFIG_ARCH_DAVINCI
 #include <mach/cputype.h>
+#endif
 
 #include <media/v4l2-dev.h>
 #include <media/v4l2-common.h>
@@ -53,8 +56,7 @@ static int vpbe_set_osd_display_params(struct vpbe_display *disp_dev,
 static int venc_is_second_field(struct vpbe_display *disp_dev)
 {
 	struct vpbe_device *vpbe_dev = disp_dev->vpbe_dev;
-	int ret;
-	int val;
+	int ret, val;
 
 	ret = v4l2_subdev_call(vpbe_dev->venc,
 			       core,
@@ -64,6 +66,7 @@ static int venc_is_second_field(struct vpbe_display *disp_dev)
 	if (ret < 0) {
 		v4l2_err(&vpbe_dev->v4l2_dev,
 			 "Error in getting Field ID 0\n");
+		return 1;
 	}
 	return val;
 }
@@ -282,7 +285,7 @@ static int vpbe_start_streaming(struct vb2_queue *vq, unsigned int count)
 	struct osd_state *osd_device = layer->disp_dev->osd_device;
 	int ret;
 
-	 osd_device->ops.disable_layer(osd_device, layer->layer_info.id);
+	osd_device->ops.disable_layer(osd_device, layer->layer_info.id);
 
 	/* Get the next frame from the buffer queue */
 	layer->next_frm = layer->cur_frm = list_entry(layer->dma_queue.next,
@@ -564,7 +567,7 @@ static void vpbe_disp_check_window_params(struct vpbe_display *disp_dev,
 
 }
 
-/**
+/*
  * vpbe_try_format()
  * If user application provides width and height, and have bytesperline set
  * to zero, driver calculates bytesperline and sizeimage based on hardware
@@ -929,7 +932,7 @@ static int vpbe_display_try_fmt(struct file *file, void *priv,
 
 }
 
-/**
+/*
  * vpbe_display_s_std - Set the given standard in the encoder
  *
  * Sets the standard if supported by the current encoder. Return the status.
@@ -961,7 +964,7 @@ static int vpbe_display_s_std(struct file *file, void *priv,
 	return 0;
 }
 
-/**
+/*
  * vpbe_display_g_std - Get the standard in the current encoder
  *
  * Get the standard in the current encoder. Return the status. 0 - success
@@ -984,7 +987,7 @@ static int vpbe_display_g_std(struct file *file, void *priv,
 	return -EINVAL;
 }
 
-/**
+/*
  * vpbe_display_enum_output - enumerate outputs
  *
  * Enumerates the outputs available at the vpbe display
@@ -1013,7 +1016,7 @@ static int vpbe_display_enum_output(struct file *file, void *priv,
 	return 0;
 }
 
-/**
+/*
  * vpbe_display_s_output - Set output to
  * the output specified by the index
  */
@@ -1042,7 +1045,7 @@ static int vpbe_display_s_output(struct file *file, void *priv,
 	return 0;
 }
 
-/**
+/*
  * vpbe_display_g_output - Get output from subdevice
  * for a given by the index
  */
@@ -1059,7 +1062,7 @@ static int vpbe_display_g_output(struct file *file, void *priv,
 	return 0;
 }
 
-/**
+/*
  * vpbe_display_enum_dv_timings - Enumerate the dv timings
  *
  * enum the timings in the current encoder. Return the status. 0 - success
@@ -1089,7 +1092,7 @@ vpbe_display_enum_dv_timings(struct file *file, void *priv,
 	return 0;
 }
 
-/**
+/*
  * vpbe_display_s_dv_timings - Set the dv timings
  *
  * Set the timings in the current encoder. Return the status. 0 - success
@@ -1122,7 +1125,7 @@ vpbe_display_s_dv_timings(struct file *file, void *priv,
 	return 0;
 }
 
-/**
+/*
  * vpbe_display_g_dv_timings - Set the dv timings
  *
  * Get the timings in the current encoder. Return the status. 0 - success
@@ -1351,9 +1354,9 @@ static int register_device(struct vpbe_layer *vpbe_display_layer,
 	v4l2_info(&disp_dev->vpbe_dev->v4l2_dev,
 		  "Trying to register VPBE display device.\n");
 	v4l2_info(&disp_dev->vpbe_dev->v4l2_dev,
-		  "layer=%x,layer->video_dev=%x\n",
-		  (int)vpbe_display_layer,
-		  (int)&vpbe_display_layer->video_dev);
+		  "layer=%p,layer->video_dev=%p\n",
+		  vpbe_display_layer,
+		  &vpbe_display_layer->video_dev);
 
 	vpbe_display_layer->video_dev.queue = &vpbe_display_layer->buffer_queue;
 	err = video_register_device(&vpbe_display_layer->video_dev,
diff --git a/drivers/media/platform/davinci/vpbe_osd.c b/drivers/media/platform/davinci/vpbe_osd.c
index 66449791c70c..7f610320426d 100644
--- a/drivers/media/platform/davinci/vpbe_osd.c
+++ b/drivers/media/platform/davinci/vpbe_osd.c
@@ -24,8 +24,10 @@
 #include <linux/clk.h>
 #include <linux/slab.h>
 
+#ifdef CONFIG_ARCH_DAVINCI
 #include <mach/cputype.h>
 #include <mach/hardware.h>
+#endif
 
 #include <media/davinci/vpss.h>
 #include <media/v4l2-device.h>
@@ -122,10 +124,10 @@ static inline u32 osd_modify(struct osd_state *sd, u32 mask, u32 val,
 
 /**
  * _osd_dm6446_vid0_pingpong() - field inversion fix for DM6446
- * @sd - ptr to struct osd_state
- * @field_inversion - inversion flag
- * @fb_base_phys - frame buffer address
- * @lconfig - ptr to layer config
+ * @sd: ptr to struct osd_state
+ * @field_inversion: inversion flag
+ * @fb_base_phys: frame buffer address
+ * @lconfig: ptr to layer config
  *
  * This routine implements a workaround for the field signal inversion silicon
  * erratum described in Advisory 1.3.8 for the DM6446.  The fb_base_phys and
@@ -782,9 +784,9 @@ static void osd_get_layer_config(struct osd_state *sd, enum osd_layer layer,
 
 /**
  * try_layer_config() - Try a specific configuration for the layer
- * @sd  - ptr to struct osd_state
- * @layer - layer to configure
- * @lconfig - layer configuration to try
+ * @sd: ptr to struct osd_state
+ * @layer: layer to configure
+ * @lconfig: layer configuration to try
  *
  * If the requested lconfig is completely rejected and the value of lconfig on
  * exit is the current lconfig, then try_layer_config() returns 1.  Otherwise,
@@ -844,9 +846,10 @@ static int try_layer_config(struct osd_state *sd, enum osd_layer layer,
 
 	/* DM6446: */
 	/* only one OSD window at a time can use RGB pixel formats */
-	  if ((osd->vpbe_type == VPBE_VERSION_1) &&
-		  is_osd_win(layer) && is_rgb_pixfmt(lconfig->pixfmt)) {
+	if ((osd->vpbe_type == VPBE_VERSION_1) &&
+	    is_osd_win(layer) && is_rgb_pixfmt(lconfig->pixfmt)) {
 		enum osd_pix_format pixfmt;
+
 		if (layer == WIN_OSD0)
 			pixfmt = osd->win[WIN_OSD1].lconfig.pixfmt;
 		else
diff --git a/drivers/media/platform/davinci/vpbe_venc.c b/drivers/media/platform/davinci/vpbe_venc.c
index 3a4e78595149..ba157827192c 100644
--- a/drivers/media/platform/davinci/vpbe_venc.c
+++ b/drivers/media/platform/davinci/vpbe_venc.c
@@ -21,8 +21,11 @@
 #include <linux/videodev2.h>
 #include <linux/slab.h>
 
+#ifdef CONFIG_ARCH_DAVINCI
 #include <mach/hardware.h>
 #include <mach/mux.h>
+#endif
+
 #include <linux/platform_data/i2c-davinci.h>
 
 #include <linux/io.h>
@@ -224,7 +227,6 @@ venc_enable_vpss_clock(int venc_type,
  */
 static int venc_set_ntsc(struct v4l2_subdev *sd)
 {
-	u32 val;
 	struct venc_state *venc = to_state(sd);
 	struct venc_platform_data *pdata = venc->pdata;
 
@@ -241,7 +243,7 @@ static int venc_set_ntsc(struct v4l2_subdev *sd)
 	if (venc->venc_type == VPBE_VERSION_3) {
 		venc_write(sd, VENC_CLKCTL, 0x01);
 		venc_write(sd, VENC_VIDCTL, 0);
-		val = vdaccfg_write(sd, VDAC_CONFIG_SD_V3);
+		vdaccfg_write(sd, VDAC_CONFIG_SD_V3);
 	} else if (venc->venc_type == VPBE_VERSION_2) {
 		venc_write(sd, VENC_CLKCTL, 0x01);
 		venc_write(sd, VENC_VIDCTL, 0);
@@ -604,10 +606,9 @@ static int venc_device_get(struct device *dev, void *data)
 struct v4l2_subdev *venc_sub_dev_init(struct v4l2_device *v4l2_dev,
 		const char *venc_name)
 {
-	struct venc_state *venc;
-	int err;
+	struct venc_state *venc = NULL;
 
-	err = bus_for_each_dev(&platform_bus_type, NULL, &venc,
+	bus_for_each_dev(&platform_bus_type, NULL, &venc,
 			venc_device_get);
 	if (venc == NULL)
 		return NULL;
diff --git a/drivers/media/platform/davinci/vpfe_capture.c b/drivers/media/platform/davinci/vpfe_capture.c
index 6f44abf7fa31..8613358ed245 100644
--- a/drivers/media/platform/davinci/vpfe_capture.c
+++ b/drivers/media/platform/davinci/vpfe_capture.c
@@ -1509,7 +1509,7 @@ static int vpfe_streamon(struct file *file, void *priv,
 unlock_out:
 	mutex_unlock(&vpfe_dev->lock);
 streamoff:
-	ret = videobuf_streamoff(&vpfe_dev->buffer_queue);
+	videobuf_streamoff(&vpfe_dev->buffer_queue);
 	return ret;
 }
 
diff --git a/drivers/media/platform/exynos4-is/Kconfig b/drivers/media/platform/exynos4-is/Kconfig
index 7b2c49e5a592..c8e5ad8f8294 100644
--- a/drivers/media/platform/exynos4-is/Kconfig
+++ b/drivers/media/platform/exynos4-is/Kconfig
@@ -41,11 +41,10 @@ config VIDEO_S5P_MIPI_CSIS
 	  To compile this driver as a module, choose M here: the
 	  module will be called s5p-csis.
 
-if SOC_EXYNOS4412 || SOC_EXYNOS5250
-
 config VIDEO_EXYNOS_FIMC_LITE
 	tristate "EXYNOS FIMC-LITE camera interface driver"
 	depends on I2C
+	depends on SOC_EXYNOS4412 || SOC_EXYNOS5250 || COMPILE_TEST
 	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select VIDEO_EXYNOS4_IS_COMMON
@@ -55,7 +54,6 @@ config VIDEO_EXYNOS_FIMC_LITE
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called exynos-fimc-lite.
-endif
 
 config VIDEO_EXYNOS4_FIMC_IS
 	tristate "EXYNOS4x12 FIMC-IS (Imaging Subsystem) driver"
diff --git a/drivers/media/platform/exynos4-is/fimc-lite-reg.c b/drivers/media/platform/exynos4-is/fimc-lite-reg.c
index f0acc550d065..16565a0b4bf1 100644
--- a/drivers/media/platform/exynos4-is/fimc-lite-reg.c
+++ b/drivers/media/platform/exynos4-is/fimc-lite-reg.c
@@ -254,7 +254,7 @@ void flite_hw_set_dma_window(struct fimc_lite *dev, struct flite_frame *f)
 	/* Maximum output pixel size */
 	cfg = readl(dev->regs + FLITE_REG_CIOCAN);
 	cfg &= ~FLITE_REG_CIOCAN_MASK;
-	cfg = (f->f_height << 16) | f->f_width;
+	cfg |= (f->f_height << 16) | f->f_width;
 	writel(cfg, dev->regs + FLITE_REG_CIOCAN);
 
 	/* DMA offsets */
diff --git a/drivers/media/platform/fsl-viu.c b/drivers/media/platform/fsl-viu.c
index 200c47c69a75..e41510ce69a4 100644
--- a/drivers/media/platform/fsl-viu.c
+++ b/drivers/media/platform/fsl-viu.c
@@ -36,6 +36,12 @@
 #define DRV_NAME		"fsl_viu"
 #define VIU_VERSION		"0.5.1"
 
+/* Allow building this driver with COMPILE_TEST */
+#ifndef CONFIG_PPC
+#define out_be32(v, a)	iowrite32be(a, (void __iomem *)v)
+#define in_be32(a)	ioread32be((void __iomem *)a)
+#endif
+
 #define BUFFER_TIMEOUT		msecs_to_jiffies(500)  /* 0.5 seconds */
 
 #define	VIU_VID_MEM_LIMIT	4	/* Video memory limit, in Mb */
@@ -128,7 +134,7 @@ struct viu_dev {
 	int			dma_done;
 
 	/* Hardware register area */
-	struct viu_reg		*vr;
+	struct viu_reg __iomem	*vr;
 
 	/* Interrupt vector */
 	int			irq;
@@ -229,7 +235,7 @@ enum status_config {
 
 static irqreturn_t viu_intr(int irq, void *dev_id);
 
-struct viu_fmt *format_by_fourcc(int fourcc)
+static struct viu_fmt *format_by_fourcc(int fourcc)
 {
 	int i;
 
@@ -242,9 +248,9 @@ struct viu_fmt *format_by_fourcc(int fourcc)
 	return NULL;
 }
 
-void viu_start_dma(struct viu_dev *dev)
+static void viu_start_dma(struct viu_dev *dev)
 {
-	struct viu_reg *vr = dev->vr;
+	struct viu_reg __iomem *vr = dev->vr;
 
 	dev->field = 0;
 
@@ -253,9 +259,9 @@ void viu_start_dma(struct viu_dev *dev)
 	out_be32(&vr->status_cfg, INT_FIELD_EN);
 }
 
-void viu_stop_dma(struct viu_dev *dev)
+static void viu_stop_dma(struct viu_dev *dev)
 {
-	struct viu_reg *vr = dev->vr;
+	struct viu_reg __iomem *vr = dev->vr;
 	int cnt = 100;
 	u32 status_cfg;
 
@@ -290,7 +296,7 @@ static int restart_video_queue(struct viu_dmaqueue *vidq)
 {
 	struct viu_buf *buf, *prev;
 
-	dprintk(1, "%s vidq=0x%08lx\n", __func__, (unsigned long)vidq);
+	dprintk(1, "%s vidq=%p\n", __func__, vidq);
 	if (!list_empty(&vidq->active)) {
 		buf = list_entry(vidq->active.next, struct viu_buf, vb.queue);
 		dprintk(2, "restart_queue [%p/%d]: restart dma\n",
@@ -395,7 +401,7 @@ static void free_buffer(struct videobuf_queue *vq, struct viu_buf *buf)
 
 inline int buffer_activate(struct viu_dev *dev, struct viu_buf *buf)
 {
-	struct viu_reg *vr = dev->vr;
+	struct viu_reg __iomem *vr = dev->vr;
 	int bpp;
 
 	/* setup the DMA base address */
@@ -497,8 +503,7 @@ static void buffer_queue(struct videobuf_queue *vq, struct videobuf_buffer *vb)
 	struct viu_buf       *prev;
 
 	if (!list_empty(&vidq->queued)) {
-		dprintk(1, "adding vb queue=0x%08lx\n",
-				(unsigned long)&buf->vb.queue);
+		dprintk(1, "adding vb queue=%p\n", &buf->vb.queue);
 		dprintk(1, "vidq pointer 0x%p, queued 0x%p\n",
 				vidq, &vidq->queued);
 		dprintk(1, "dev %p, queued: self %p, next %p, head %p\n",
@@ -509,8 +514,7 @@ static void buffer_queue(struct videobuf_queue *vq, struct videobuf_buffer *vb)
 		dprintk(2, "[%p/%d] buffer_queue - append to queued\n",
 			buf, buf->vb.i);
 	} else if (list_empty(&vidq->active)) {
-		dprintk(1, "adding vb active=0x%08lx\n",
-				(unsigned long)&buf->vb.queue);
+		dprintk(1, "adding vb active=%p\n", &buf->vb.queue);
 		list_add_tail(&buf->vb.queue, &vidq->active);
 		buf->vb.state = VIDEOBUF_ACTIVE;
 		mod_timer(&vidq->timeout, jiffies+BUFFER_TIMEOUT);
@@ -519,8 +523,7 @@ static void buffer_queue(struct videobuf_queue *vq, struct videobuf_buffer *vb)
 
 		buffer_activate(dev, buf);
 	} else {
-		dprintk(1, "adding vb queue2=0x%08lx\n",
-				(unsigned long)&buf->vb.queue);
+		dprintk(1, "adding vb queue2=%p\n", &buf->vb.queue);
 		prev = list_entry(vidq->active.prev, struct viu_buf, vb.queue);
 		if (prev->vb.width  == buf->vb.width  &&
 		    prev->vb.height == buf->vb.height &&
@@ -703,10 +706,8 @@ static int verify_preview(struct viu_dev *dev, struct v4l2_window *win)
 	return 0;
 }
 
-inline void viu_activate_overlay(struct viu_reg *viu_reg)
+inline void viu_activate_overlay(struct viu_reg __iomem *vr)
 {
-	struct viu_reg *vr = viu_reg;
-
 	out_be32(&vr->field_base_addr, reg_val.field_base_addr);
 	out_be32(&vr->dma_inc, reg_val.dma_inc);
 	out_be32(&vr->picture_count, reg_val.picture_count);
@@ -749,7 +750,7 @@ static int viu_setup_preview(struct viu_dev *dev, struct viu_fh *fh)
 	reg_val.status_cfg |= DMA_ACT | INT_DMA_END_EN | INT_FIELD_EN;
 
 	/* setup the base address of the overlay buffer */
-	reg_val.field_base_addr = (u32)dev->ovbuf.base;
+	reg_val.field_base_addr = (u32)(long)dev->ovbuf.base;
 
 	return 0;
 }
@@ -802,7 +803,7 @@ static int vidioc_overlay(struct file *file, void *priv, unsigned int on)
 	return 0;
 }
 
-int vidioc_g_fbuf(struct file *file, void *priv, struct v4l2_framebuffer *arg)
+static int vidioc_g_fbuf(struct file *file, void *priv, struct v4l2_framebuffer *arg)
 {
 	struct viu_fh  *fh = priv;
 	struct viu_dev *dev = fh->dev;
@@ -813,7 +814,7 @@ int vidioc_g_fbuf(struct file *file, void *priv, struct v4l2_framebuffer *arg)
 	return 0;
 }
 
-int vidioc_s_fbuf(struct file *file, void *priv, const struct v4l2_framebuffer *arg)
+static int vidioc_s_fbuf(struct file *file, void *priv, const struct v4l2_framebuffer *arg)
 {
 	struct viu_fh  *fh = priv;
 	struct viu_dev *dev = fh->dev;
@@ -985,10 +986,8 @@ inline void viu_activate_next_buf(struct viu_dev *dev,
 	}
 }
 
-inline void viu_default_settings(struct viu_reg *viu_reg)
+inline void viu_default_settings(struct viu_reg __iomem *vr)
 {
-	struct viu_reg *vr = viu_reg;
-
 	out_be32(&vr->luminance, 0x9512A254);
 	out_be32(&vr->chroma_r, 0x03310000);
 	out_be32(&vr->chroma_g, 0x06600F38);
@@ -1001,7 +1000,7 @@ inline void viu_default_settings(struct viu_reg *viu_reg)
 
 static void viu_overlay_intr(struct viu_dev *dev, u32 status)
 {
-	struct viu_reg *vr = dev->vr;
+	struct viu_reg __iomem *vr = dev->vr;
 
 	if (status & INT_DMA_END_STATUS)
 		dev->dma_done = 1;
@@ -1032,7 +1031,7 @@ static void viu_overlay_intr(struct viu_dev *dev, u32 status)
 static void viu_capture_intr(struct viu_dev *dev, u32 status)
 {
 	struct viu_dmaqueue *vidq = &dev->vidq;
-	struct viu_reg *vr = dev->vr;
+	struct viu_reg __iomem *vr = dev->vr;
 	struct viu_buf *buf;
 	int field_num;
 	int need_two;
@@ -1104,7 +1103,7 @@ static void viu_capture_intr(struct viu_dev *dev, u32 status)
 static irqreturn_t viu_intr(int irq, void *dev_id)
 {
 	struct viu_dev *dev  = (struct viu_dev *)dev_id;
-	struct viu_reg *vr = dev->vr;
+	struct viu_reg __iomem *vr = dev->vr;
 	u32 status;
 	u32 error;
 
@@ -1169,7 +1168,7 @@ static int viu_open(struct file *file)
 	struct video_device *vdev = video_devdata(file);
 	struct viu_dev *dev = video_get_drvdata(vdev);
 	struct viu_fh *fh;
-	struct viu_reg *vr;
+	struct viu_reg __iomem *vr;
 	int minor = vdev->minor;
 	u32 status_cfg;
 
@@ -1210,9 +1209,7 @@ static int viu_open(struct file *file)
 	dev->crop_current.width  = fh->width;
 	dev->crop_current.height = fh->height;
 
-	dprintk(1, "Open: fh=0x%08lx, dev=0x%08lx, dev->vidq=0x%08lx\n",
-		(unsigned long)fh, (unsigned long)dev,
-		(unsigned long)&dev->vidq);
+	dprintk(1, "Open: fh=%p, dev=%p, dev->vidq=%p\n", fh, dev, &dev->vidq);
 	dprintk(1, "Open: list_empty queued=%d\n",
 		list_empty(&dev->vidq.queued));
 	dprintk(1, "Open: list_empty active=%d\n",
@@ -1305,7 +1302,7 @@ static int viu_release(struct file *file)
 	return 0;
 }
 
-void viu_reset(struct viu_reg *reg)
+static void viu_reset(struct viu_reg __iomem *reg)
 {
 	out_be32(&reg->status_cfg, 0);
 	out_be32(&reg->luminance, 0x9512a254);
@@ -1325,7 +1322,7 @@ static int viu_mmap(struct file *file, struct vm_area_struct *vma)
 	struct viu_dev *dev = fh->dev;
 	int ret;
 
-	dprintk(1, "mmap called, vma=0x%08lx\n", (unsigned long)vma);
+	dprintk(1, "mmap called, vma=%p\n", vma);
 
 	if (mutex_lock_interruptible(&dev->lock))
 		return -ERESTARTSYS;
@@ -1407,7 +1404,7 @@ static int viu_of_probe(struct platform_device *op)
 	}
 
 	viu_irq = irq_of_parse_and_map(op->dev.of_node, 0);
-	if (viu_irq == NO_IRQ) {
+	if (!viu_irq) {
 		dev_err(&op->dev, "Error while mapping the irq\n");
 		return -EINVAL;
 	}
diff --git a/drivers/media/platform/marvell-ccic/Kconfig b/drivers/media/platform/marvell-ccic/Kconfig
index 4bf5bd1e90d6..60dbe6d1fae9 100644
--- a/drivers/media/platform/marvell-ccic/Kconfig
+++ b/drivers/media/platform/marvell-ccic/Kconfig
@@ -1,7 +1,6 @@
 config VIDEO_CAFE_CCIC
 	tristate "Marvell 88ALP01 (Cafe) CMOS Camera Controller support"
 	depends on PCI && I2C && VIDEO_V4L2
-	depends on HAS_DMA
 	select VIDEO_OV7670
 	select VIDEOBUF2_VMALLOC
 	select VIDEOBUF2_DMA_CONTIG
@@ -13,8 +12,8 @@ config VIDEO_CAFE_CCIC
 
 config VIDEO_MMP_CAMERA
 	tristate "Marvell Armada 610 integrated camera controller support"
-	depends on ARCH_MMP && I2C && VIDEO_V4L2
-	depends on HAS_DMA && BROKEN
+	depends on I2C && VIDEO_V4L2
+	depends on ARCH_MMP || COMPILE_TEST
 	select VIDEO_OV7670
 	select I2C_GPIO
 	select VIDEOBUF2_DMA_SG
diff --git a/drivers/media/platform/marvell-ccic/mmp-driver.c b/drivers/media/platform/marvell-ccic/mmp-driver.c
index 816f4b6a7b8e..6d9f0abb2660 100644
--- a/drivers/media/platform/marvell-ccic/mmp-driver.c
+++ b/drivers/media/platform/marvell-ccic/mmp-driver.c
@@ -12,7 +12,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/i2c.h>
-#include <linux/i2c-gpio.h>
+#include <linux/platform_data/i2c-gpio.h>
 #include <linux/interrupt.h>
 #include <linux/spinlock.h>
 #include <linux/slab.h>
@@ -37,7 +37,7 @@ MODULE_LICENSE("GPL");
 static char *mcam_clks[] = {"CCICAXICLK", "CCICFUNCLK", "CCICPHYCLK"};
 
 struct mmp_camera {
-	void *power_regs;
+	void __iomem *power_regs;
 	struct platform_device *pdev;
 	struct mcam_camera mcam;
 	struct list_head devlist;
@@ -183,7 +183,7 @@ static void mmpcam_power_down(struct mcam_camera *mcam)
 	mcam_clk_disable(mcam);
 }
 
-void mcam_ctlr_reset(struct mcam_camera *mcam)
+static void mcam_ctlr_reset(struct mcam_camera *mcam)
 {
 	unsigned long val;
 	struct mmp_camera *cam = mcam_to_cam(mcam);
@@ -214,7 +214,7 @@ void mcam_ctlr_reset(struct mcam_camera *mcam)
  * CSI2_DPHY3 and CSI2_DPHY6 can be set with a default value
  * or be calculated dynamically
  */
-void mmpcam_calc_dphy(struct mcam_camera *mcam)
+static void mmpcam_calc_dphy(struct mcam_camera *mcam)
 {
 	struct mmp_camera *cam = mcam_to_cam(mcam);
 	struct mmp_camera_platform_data *pdata = cam->pdev->dev.platform_data;
diff --git a/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c b/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c
index af17aaa21f58..328e8f650d9b 100644
--- a/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c
+++ b/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c
@@ -1084,10 +1084,7 @@ static int mtk_jpeg_clk_init(struct mtk_jpeg_dev *jpeg)
 		return PTR_ERR(jpeg->clk_jdec);
 
 	jpeg->clk_jdec_smi = devm_clk_get(jpeg->dev, "jpgdec-smi");
-	if (IS_ERR(jpeg->clk_jdec_smi))
-		return PTR_ERR(jpeg->clk_jdec_smi);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(jpeg->clk_jdec_smi);
 }
 
 static int mtk_jpeg_probe(struct platform_device *pdev)
diff --git a/drivers/media/platform/omap/Kconfig b/drivers/media/platform/omap/Kconfig
index e8e2db181a7a..a414bcbb9b08 100644
--- a/drivers/media/platform/omap/Kconfig
+++ b/drivers/media/platform/omap/Kconfig
@@ -1,15 +1,16 @@
 config VIDEO_OMAP2_VOUT_VRFB
 	bool
+	default y
+	depends on VIDEO_OMAP2_VOUT && (OMAP2_VRFB || COMPILE_TEST)
 
 config VIDEO_OMAP2_VOUT
 	tristate "OMAP2/OMAP3 V4L2-Display driver"
 	depends on MMU
-	depends on ARCH_OMAP2 || ARCH_OMAP3
-	depends on FB_OMAP2
+	depends on FB_OMAP2 || COMPILE_TEST
+	depends on ARCH_OMAP2 || ARCH_OMAP3 || COMPILE_TEST
 	select VIDEOBUF_GEN
 	select VIDEOBUF_DMA_CONTIG
 	select OMAP2_VRFB if ARCH_OMAP2 || ARCH_OMAP3
-	select VIDEO_OMAP2_VOUT_VRFB if VIDEO_OMAP2_VOUT && OMAP2_VRFB
 	select FRAME_VECTOR
 	default n
 	---help---
diff --git a/drivers/media/platform/omap/omap_vout.c b/drivers/media/platform/omap/omap_vout.c
index a795a9fae899..5700b7818621 100644
--- a/drivers/media/platform/omap/omap_vout.c
+++ b/drivers/media/platform/omap/omap_vout.c
@@ -198,7 +198,7 @@ static int omap_vout_try_format(struct v4l2_pix_format *pix)
  * omap_vout_get_userptr: Convert user space virtual address to physical
  * address.
  */
-static int omap_vout_get_userptr(struct videobuf_buffer *vb, u32 virtp,
+static int omap_vout_get_userptr(struct videobuf_buffer *vb, long virtp,
 				 u32 *physp)
 {
 	struct frame_vector *vec;
@@ -702,19 +702,18 @@ static int omap_vout_buffer_setup(struct videobuf_queue *q, unsigned int *count,
 		virt_addr = omap_vout_alloc_buffer(vout->buffer_size,
 				&phy_addr);
 		if (!virt_addr) {
-			if (ovid->rotation_type == VOUT_ROT_NONE) {
+			if (ovid->rotation_type == VOUT_ROT_NONE)
 				break;
-			} else {
-				if (!is_rotation_enabled(vout))
-					break;
+
+			if (!is_rotation_enabled(vout))
+				break;
+
 			/* Free the VRFB buffers if no space for V4L2 buffers */
 			for (j = i; j < *count; j++) {
-				omap_vout_free_buffer(
-						vout->smsshado_virt_addr[j],
-						vout->smsshado_size);
+				omap_vout_free_buffer(vout->smsshado_virt_addr[j],
+						      vout->smsshado_size);
 				vout->smsshado_virt_addr[j] = 0;
 				vout->smsshado_phy_addr[j] = 0;
-				}
 			}
 		}
 		vout->buf_virt_addr[i] = virt_addr;
diff --git a/drivers/media/platform/omap/omap_vout_vrfb.c b/drivers/media/platform/omap/omap_vout_vrfb.c
index 1d8508237220..29e3f5da59c1 100644
--- a/drivers/media/platform/omap/omap_vout_vrfb.c
+++ b/drivers/media/platform/omap/omap_vout_vrfb.c
@@ -54,8 +54,8 @@ static int omap_vout_allocate_vrfb_buffers(struct omap_vout_device *vout,
 			*count = 0;
 			return -ENOMEM;
 		}
-		memset((void *) vout->smsshado_virt_addr[i], 0,
-				vout->smsshado_size);
+		memset((void *)(long)vout->smsshado_virt_addr[i], 0,
+		       vout->smsshado_size);
 	}
 	return 0;
 }
diff --git a/drivers/media/platform/omap3isp/isp.c b/drivers/media/platform/omap3isp/isp.c
index 8eb000e3d8fd..f22cf351e3ee 100644
--- a/drivers/media/platform/omap3isp/isp.c
+++ b/drivers/media/platform/omap3isp/isp.c
@@ -61,7 +61,9 @@
 #include <linux/sched.h>
 #include <linux/vmalloc.h>
 
+#ifdef CONFIG_ARM_DMA_USE_IOMMU
 #include <asm/dma-iommu.h>
+#endif
 
 #include <media/v4l2-common.h>
 #include <media/v4l2-fwnode.h>
@@ -284,13 +286,6 @@ static const struct clk_ops isp_xclk_ops = {
 
 static const char *isp_xclk_parent_name = "cam_mclk";
 
-static const struct clk_init_data isp_xclk_init_data = {
-	.name = "cam_xclk",
-	.ops = &isp_xclk_ops,
-	.parent_names = &isp_xclk_parent_name,
-	.num_parents = 1,
-};
-
 static struct clk *isp_xclk_src_get(struct of_phandle_args *clkspec, void *data)
 {
 	unsigned int idx = clkspec->args[0];
@@ -1945,12 +1940,16 @@ static int isp_initialize_modules(struct isp_device *isp)
 
 static void isp_detach_iommu(struct isp_device *isp)
 {
+#ifdef CONFIG_ARM_DMA_USE_IOMMU
+	arm_iommu_detach_device(isp->dev);
 	arm_iommu_release_mapping(isp->mapping);
 	isp->mapping = NULL;
+#endif
 }
 
 static int isp_attach_iommu(struct isp_device *isp)
 {
+#ifdef CONFIG_ARM_DMA_USE_IOMMU
 	struct dma_iommu_mapping *mapping;
 	int ret;
 
@@ -1961,8 +1960,7 @@ static int isp_attach_iommu(struct isp_device *isp)
 	mapping = arm_iommu_create_mapping(&platform_bus_type, SZ_1G, SZ_2G);
 	if (IS_ERR(mapping)) {
 		dev_err(isp->dev, "failed to create ARM IOMMU mapping\n");
-		ret = PTR_ERR(mapping);
-		goto error;
+		return PTR_ERR(mapping);
 	}
 
 	isp->mapping = mapping;
@@ -1977,8 +1975,12 @@ static int isp_attach_iommu(struct isp_device *isp)
 	return 0;
 
 error:
-	isp_detach_iommu(isp);
+	arm_iommu_release_mapping(isp->mapping);
+	isp->mapping = NULL;
 	return ret;
+#else
+	return -ENODEV;
+#endif
 }
 
 /*
diff --git a/drivers/media/platform/omap3isp/ispccdc.c b/drivers/media/platform/omap3isp/ispccdc.c
index b66276ab5765..77b73e27a274 100644
--- a/drivers/media/platform/omap3isp/ispccdc.c
+++ b/drivers/media/platform/omap3isp/ispccdc.c
@@ -735,7 +735,7 @@ static int ccdc_config(struct isp_ccdc_device *ccdc,
 				return -ENOMEM;
 
 			if (copy_from_user(fpc_new.addr,
-					   (__force void __user *)fpc.fpcaddr,
+					   (__force void __user *)(long)fpc.fpcaddr,
 					   size)) {
 				dma_free_coherent(isp->dev, size, fpc_new.addr,
 						  fpc_new.dma);
diff --git a/drivers/media/platform/omap3isp/isph3a_aewb.c b/drivers/media/platform/omap3isp/isph3a_aewb.c
index d44626f20ac6..3c82dea4d375 100644
--- a/drivers/media/platform/omap3isp/isph3a_aewb.c
+++ b/drivers/media/platform/omap3isp/isph3a_aewb.c
@@ -250,6 +250,8 @@ static long h3a_aewb_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
 		return omap3isp_stat_config(stat, arg);
 	case VIDIOC_OMAP3ISP_STAT_REQ:
 		return omap3isp_stat_request_statistics(stat, arg);
+	case VIDIOC_OMAP3ISP_STAT_REQ_TIME32:
+		return omap3isp_stat_request_statistics_time32(stat, arg);
 	case VIDIOC_OMAP3ISP_STAT_EN: {
 		unsigned long *en = arg;
 		return omap3isp_stat_enable(stat, !!*en);
diff --git a/drivers/media/platform/omap3isp/isph3a_af.c b/drivers/media/platform/omap3isp/isph3a_af.c
index 99bd6cc21d86..4da25c84f0c6 100644
--- a/drivers/media/platform/omap3isp/isph3a_af.c
+++ b/drivers/media/platform/omap3isp/isph3a_af.c
@@ -314,6 +314,8 @@ static long h3a_af_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
 		return omap3isp_stat_config(stat, arg);
 	case VIDIOC_OMAP3ISP_STAT_REQ:
 		return omap3isp_stat_request_statistics(stat, arg);
+	case VIDIOC_OMAP3ISP_STAT_REQ_TIME32:
+		return omap3isp_stat_request_statistics_time32(stat, arg);
 	case VIDIOC_OMAP3ISP_STAT_EN: {
 		int *en = arg;
 		return omap3isp_stat_enable(stat, !!*en);
diff --git a/drivers/media/platform/omap3isp/isphist.c b/drivers/media/platform/omap3isp/isphist.c
index a4ed5d140d48..d4be3d0e06f9 100644
--- a/drivers/media/platform/omap3isp/isphist.c
+++ b/drivers/media/platform/omap3isp/isphist.c
@@ -435,6 +435,8 @@ static long hist_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
 		return omap3isp_stat_config(stat, arg);
 	case VIDIOC_OMAP3ISP_STAT_REQ:
 		return omap3isp_stat_request_statistics(stat, arg);
+	case VIDIOC_OMAP3ISP_STAT_REQ_TIME32:
+		return omap3isp_stat_request_statistics_time32(stat, arg);
 	case VIDIOC_OMAP3ISP_STAT_EN: {
 		int *en = arg;
 		return omap3isp_stat_enable(stat, !!*en);
diff --git a/drivers/media/platform/omap3isp/isppreview.c b/drivers/media/platform/omap3isp/isppreview.c
index ac30a0f83780..3195f7c8b8b7 100644
--- a/drivers/media/platform/omap3isp/isppreview.c
+++ b/drivers/media/platform/omap3isp/isppreview.c
@@ -25,7 +25,7 @@
 #include "isppreview.h"
 
 /* Default values in Office Fluorescent Light for RGBtoRGB Blending */
-static struct omap3isp_prev_rgbtorgb flr_rgb2rgb = {
+static const struct omap3isp_prev_rgbtorgb flr_rgb2rgb = {
 	{	/* RGB-RGB Matrix */
 		{0x01E2, 0x0F30, 0x0FEE},
 		{0x0F9B, 0x01AC, 0x0FB9},
@@ -35,7 +35,7 @@ static struct omap3isp_prev_rgbtorgb flr_rgb2rgb = {
 };
 
 /* Default values in Office Fluorescent Light for RGB to YUV Conversion*/
-static struct omap3isp_prev_csc flr_prev_csc = {
+static const struct omap3isp_prev_csc flr_prev_csc = {
 	{	/* CSC Coef Matrix */
 		{66, 129, 25},
 		{-38, -75, 112},
@@ -890,7 +890,7 @@ static int preview_config(struct isp_prev_device *prev,
 		params = &prev->params.params[!!(active & bit)];
 
 		if (cfg->flag & bit) {
-			void __user *from = *(void * __user *)
+			void __user *from = *(void __user **)
 				((void *)cfg + attr->config_offset);
 			void *to = (void *)params + attr->param_offset;
 			size_t size = attr->param_size;
diff --git a/drivers/media/platform/omap3isp/ispstat.c b/drivers/media/platform/omap3isp/ispstat.c
index 47cbc7e3d825..47353fee26c3 100644
--- a/drivers/media/platform/omap3isp/ispstat.c
+++ b/drivers/media/platform/omap3isp/ispstat.c
@@ -17,6 +17,7 @@
 
 #include <linux/dma-mapping.h>
 #include <linux/slab.h>
+#include <linux/timekeeping.h>
 #include <linux/uaccess.h>
 
 #include "isp.h"
@@ -237,7 +238,7 @@ static int isp_stat_buf_queue(struct ispstat *stat)
 	if (!stat->active_buf)
 		return STAT_NO_BUF;
 
-	v4l2_get_timestamp(&stat->active_buf->ts);
+	ktime_get_ts64(&stat->active_buf->ts);
 
 	stat->active_buf->buf_size = stat->buf_size;
 	if (isp_stat_buf_check_magic(stat, stat->active_buf)) {
@@ -370,7 +371,7 @@ static int isp_stat_bufs_alloc_one(struct device *dev,
 	int ret;
 
 	buf->virt_addr = dma_alloc_coherent(dev, size, &buf->dma_addr,
-					    GFP_KERNEL | GFP_DMA);
+					    GFP_KERNEL);
 	if (!buf->virt_addr)
 		return -ENOMEM;
 
@@ -449,10 +450,8 @@ static int isp_stat_bufs_alloc(struct ispstat *stat, u32 size)
 		buf->empty = 1;
 
 		dev_dbg(stat->isp->dev,
-			"%s: buffer[%u] allocated. dma=0x%08lx virt=0x%08lx",
-			stat->subdev.name, i,
-			(unsigned long)buf->dma_addr,
-			(unsigned long)buf->virt_addr);
+			"%s: buffer[%u] allocated. dma=%pad virt=%p",
+			stat->subdev.name, i, &buf->dma_addr, buf->virt_addr);
 	}
 
 	return 0;
@@ -500,7 +499,8 @@ int omap3isp_stat_request_statistics(struct ispstat *stat,
 		return PTR_ERR(buf);
 	}
 
-	data->ts = buf->ts;
+	data->ts.tv_sec = buf->ts.tv_sec;
+	data->ts.tv_usec = buf->ts.tv_nsec / NSEC_PER_USEC;
 	data->config_counter = buf->config_counter;
 	data->frame_number = buf->frame_number;
 	data->buf_size = buf->buf_size;
@@ -512,6 +512,23 @@ int omap3isp_stat_request_statistics(struct ispstat *stat,
 	return 0;
 }
 
+int omap3isp_stat_request_statistics_time32(struct ispstat *stat,
+					struct omap3isp_stat_data_time32 *data)
+{
+	struct omap3isp_stat_data data64;
+	int ret;
+
+	ret = omap3isp_stat_request_statistics(stat, &data64);
+	if (ret)
+		return ret;
+
+	data->ts.tv_sec = data64.ts.tv_sec;
+	data->ts.tv_usec = data64.ts.tv_usec;
+	memcpy(&data->buf, &data64.buf, sizeof(*data) - sizeof(data->ts));
+
+	return 0;
+}
+
 /*
  * omap3isp_stat_config - Receives new statistic engine configuration.
  * @new_conf: Pointer to config structure.
@@ -527,12 +544,6 @@ int omap3isp_stat_config(struct ispstat *stat, void *new_conf)
 	struct ispstat_generic_config *user_cfg = new_conf;
 	u32 buf_size = user_cfg->buf_size;
 
-	if (!new_conf) {
-		dev_dbg(stat->isp->dev, "%s: configuration is NULL\n",
-			stat->subdev.name);
-		return -EINVAL;
-	}
-
 	mutex_lock(&stat->ioctl_lock);
 
 	dev_dbg(stat->isp->dev,
diff --git a/drivers/media/platform/omap3isp/ispstat.h b/drivers/media/platform/omap3isp/ispstat.h
index 6d9b0244f320..923b38cfc682 100644
--- a/drivers/media/platform/omap3isp/ispstat.h
+++ b/drivers/media/platform/omap3isp/ispstat.h
@@ -39,7 +39,7 @@ struct ispstat_buffer {
 	struct sg_table sgt;
 	void *virt_addr;
 	dma_addr_t dma_addr;
-	struct timeval ts;
+	struct timespec64 ts;
 	u32 buf_size;
 	u32 frame_number;
 	u16 config_counter;
@@ -130,6 +130,8 @@ struct ispstat_generic_config {
 int omap3isp_stat_config(struct ispstat *stat, void *new_conf);
 int omap3isp_stat_request_statistics(struct ispstat *stat,
 				     struct omap3isp_stat_data *data);
+int omap3isp_stat_request_statistics_time32(struct ispstat *stat,
+				     struct omap3isp_stat_data_time32 *data);
 int omap3isp_stat_init(struct ispstat *stat, const char *name,
 		       const struct v4l2_subdev_ops *sd_ops);
 void omap3isp_stat_cleanup(struct ispstat *stat);
diff --git a/drivers/media/platform/omap3isp/ispvideo.c b/drivers/media/platform/omap3isp/ispvideo.c
index a751c89a3ea8..9d228eac24ea 100644
--- a/drivers/media/platform/omap3isp/ispvideo.c
+++ b/drivers/media/platform/omap3isp/ispvideo.c
@@ -1403,7 +1403,7 @@ static int isp_video_mmap(struct file *file, struct vm_area_struct *vma)
 	return vb2_mmap(&vfh->queue, vma);
 }
 
-static struct v4l2_file_operations isp_video_fops = {
+static const struct v4l2_file_operations isp_video_fops = {
 	.owner = THIS_MODULE,
 	.unlocked_ioctl = video_ioctl2,
 	.open = isp_video_open,
diff --git a/drivers/media/platform/pxa_camera.c b/drivers/media/platform/pxa_camera.c
index c71a00736541..c792cb178a84 100644
--- a/drivers/media/platform/pxa_camera.c
+++ b/drivers/media/platform/pxa_camera.c
@@ -2040,6 +2040,9 @@ static int pxac_fops_camera_open(struct file *filp)
 	if (ret < 0)
 		goto out;
 
+	if (!v4l2_fh_is_singular_file(filp))
+		goto out;
+
 	ret = sensor_call(pcdev, core, s_power, 1);
 	if (ret)
 		v4l2_fh_release(filp);
@@ -2052,13 +2055,17 @@ static int pxac_fops_camera_release(struct file *filp)
 {
 	struct pxa_camera_dev *pcdev = video_drvdata(filp);
 	int ret;
-
-	ret = vb2_fop_release(filp);
-	if (ret < 0)
-		return ret;
+	bool fh_singular;
 
 	mutex_lock(&pcdev->mlock);
-	ret = sensor_call(pcdev, core, s_power, 0);
+
+	fh_singular = v4l2_fh_is_singular_file(filp);
+
+	ret = _vb2_fop_release(filp, NULL);
+
+	if (fh_singular)
+		ret = sensor_call(pcdev, core, s_power, 0);
+
 	mutex_unlock(&pcdev->mlock);
 
 	return ret;
diff --git a/drivers/media/platform/rcar-vin/Kconfig b/drivers/media/platform/rcar-vin/Kconfig
index af4c98b44d2e..baf4eafff6e3 100644
--- a/drivers/media/platform/rcar-vin/Kconfig
+++ b/drivers/media/platform/rcar-vin/Kconfig
@@ -1,12 +1,24 @@
+config VIDEO_RCAR_CSI2
+	tristate "R-Car MIPI CSI-2 Receiver"
+	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && OF
+	depends on ARCH_RENESAS || COMPILE_TEST
+	select V4L2_FWNODE
+	help
+	  Support for Renesas R-Car MIPI CSI-2 receiver.
+	  Supports R-Car Gen3 SoCs.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called rcar-csi2.
+
 config VIDEO_RCAR_VIN
 	tristate "R-Car Video Input (VIN) Driver"
-	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && OF && HAS_DMA && MEDIA_CONTROLLER
+	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && OF && MEDIA_CONTROLLER
 	depends on ARCH_RENESAS || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
 	---help---
 	  Support for Renesas R-Car Video Input (VIN) driver.
-	  Supports R-Car Gen2 SoCs.
+	  Supports R-Car Gen2 and Gen3 SoCs.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called rcar-vin.
diff --git a/drivers/media/platform/rcar-vin/Makefile b/drivers/media/platform/rcar-vin/Makefile
index 48c5632c21dc..5ab803d3e7c1 100644
--- a/drivers/media/platform/rcar-vin/Makefile
+++ b/drivers/media/platform/rcar-vin/Makefile
@@ -1,3 +1,4 @@
 rcar-vin-objs = rcar-core.o rcar-dma.o rcar-v4l2.o
 
+obj-$(CONFIG_VIDEO_RCAR_CSI2) += rcar-csi2.o
 obj-$(CONFIG_VIDEO_RCAR_VIN) += rcar-vin.o
diff --git a/drivers/media/platform/rcar-vin/rcar-core.c b/drivers/media/platform/rcar-vin/rcar-core.c
index f1fc7978d6d1..d3072e166a1c 100644
--- a/drivers/media/platform/rcar-vin/rcar-core.c
+++ b/drivers/media/platform/rcar-vin/rcar-core.c
@@ -20,12 +20,341 @@
 #include <linux/of_graph.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/sys_soc.h>
 
 #include <media/v4l2-async.h>
 #include <media/v4l2-fwnode.h>
+#include <media/v4l2-mc.h>
 
 #include "rcar-vin.h"
 
+/*
+ * The companion CSI-2 receiver driver (rcar-csi2) is known
+ * and we know it has one source pad (pad 0) and four sink
+ * pads (pad 1-4). So to translate a pad on the remote
+ * CSI-2 receiver to/from the VIN internal channel number simply
+ * subtract/add one from the pad/channel number.
+ */
+#define rvin_group_csi_pad_to_channel(pad) ((pad) - 1)
+#define rvin_group_csi_channel_to_pad(channel) ((channel) + 1)
+
+/*
+ * Not all VINs are created equal, master VINs control the
+ * routing for other VIN's. We can figure out which VIN is
+ * master by looking at a VINs id.
+ */
+#define rvin_group_id_to_master(vin) ((vin) < 4 ? 0 : 4)
+
+/* -----------------------------------------------------------------------------
+ * Media Controller link notification
+ */
+
+/* group lock should be held when calling this function. */
+static int rvin_group_entity_to_csi_id(struct rvin_group *group,
+				       struct media_entity *entity)
+{
+	struct v4l2_subdev *sd;
+	unsigned int i;
+
+	sd = media_entity_to_v4l2_subdev(entity);
+
+	for (i = 0; i < RVIN_CSI_MAX; i++)
+		if (group->csi[i].subdev == sd)
+			return i;
+
+	return -ENODEV;
+}
+
+static unsigned int rvin_group_get_mask(struct rvin_dev *vin,
+					enum rvin_csi_id csi_id,
+					unsigned char channel)
+{
+	const struct rvin_group_route *route;
+	unsigned int mask = 0;
+
+	for (route = vin->info->routes; route->mask; route++) {
+		if (route->vin == vin->id &&
+		    route->csi == csi_id &&
+		    route->channel == channel) {
+			vin_dbg(vin,
+				"Adding route: vin: %d csi: %d channel: %d\n",
+				route->vin, route->csi, route->channel);
+			mask |= route->mask;
+		}
+	}
+
+	return mask;
+}
+
+/*
+ * Link setup for the links between a VIN and a CSI-2 receiver is a bit
+ * complex. The reason for this is that the register controlling routing
+ * is not present in each VIN instance. There are special VINs which
+ * control routing for themselves and other VINs. There are not many
+ * different possible links combinations that can be enabled at the same
+ * time, therefor all already enabled links which are controlled by a
+ * master VIN need to be taken into account when making the decision
+ * if a new link can be enabled or not.
+ *
+ * 1. Find out which VIN the link the user tries to enable is connected to.
+ * 2. Lookup which master VIN controls the links for this VIN.
+ * 3. Start with a bitmask with all bits set.
+ * 4. For each previously enabled link from the master VIN bitwise AND its
+ *    route mask (see documentation for mask in struct rvin_group_route)
+ *    with the bitmask.
+ * 5. Bitwise AND the mask for the link the user tries to enable to the bitmask.
+ * 6. If the bitmask is not empty at this point the new link can be enabled
+ *    while keeping all previous links enabled. Update the CHSEL value of the
+ *    master VIN and inform the user that the link could be enabled.
+ *
+ * Please note that no link can be enabled if any VIN in the group is
+ * currently open.
+ */
+static int rvin_group_link_notify(struct media_link *link, u32 flags,
+				  unsigned int notification)
+{
+	struct rvin_group *group = container_of(link->graph_obj.mdev,
+						struct rvin_group, mdev);
+	unsigned int master_id, channel, mask_new, i;
+	unsigned int mask = ~0;
+	struct media_entity *entity;
+	struct video_device *vdev;
+	struct media_pad *csi_pad;
+	struct rvin_dev *vin = NULL;
+	int csi_id, ret;
+
+	ret = v4l2_pipeline_link_notify(link, flags, notification);
+	if (ret)
+		return ret;
+
+	/* Only care about link enablement for VIN nodes. */
+	if (!(flags & MEDIA_LNK_FL_ENABLED) ||
+	    !is_media_entity_v4l2_video_device(link->sink->entity))
+		return 0;
+
+	/* If any entity is in use don't allow link changes. */
+	media_device_for_each_entity(entity, &group->mdev)
+		if (entity->use_count)
+			return -EBUSY;
+
+	mutex_lock(&group->lock);
+
+	/* Find the master VIN that controls the routes. */
+	vdev = media_entity_to_video_device(link->sink->entity);
+	vin = container_of(vdev, struct rvin_dev, vdev);
+	master_id = rvin_group_id_to_master(vin->id);
+
+	if (WARN_ON(!group->vin[master_id])) {
+		ret = -ENODEV;
+		goto out;
+	}
+
+	/* Build a mask for already enabled links. */
+	for (i = master_id; i < master_id + 4; i++) {
+		if (!group->vin[i])
+			continue;
+
+		/* Get remote CSI-2, if any. */
+		csi_pad = media_entity_remote_pad(
+				&group->vin[i]->vdev.entity.pads[0]);
+		if (!csi_pad)
+			continue;
+
+		csi_id = rvin_group_entity_to_csi_id(group, csi_pad->entity);
+		channel = rvin_group_csi_pad_to_channel(csi_pad->index);
+
+		mask &= rvin_group_get_mask(group->vin[i], csi_id, channel);
+	}
+
+	/* Add the new link to the existing mask and check if it works. */
+	csi_id = rvin_group_entity_to_csi_id(group, link->source->entity);
+	channel = rvin_group_csi_pad_to_channel(link->source->index);
+	mask_new = mask & rvin_group_get_mask(vin, csi_id, channel);
+
+	vin_dbg(vin, "Try link change mask: 0x%x new: 0x%x\n", mask, mask_new);
+
+	if (!mask_new) {
+		ret = -EMLINK;
+		goto out;
+	}
+
+	/* New valid CHSEL found, set the new value. */
+	ret = rvin_set_channel_routing(group->vin[master_id], __ffs(mask_new));
+out:
+	mutex_unlock(&group->lock);
+
+	return ret;
+}
+
+static const struct media_device_ops rvin_media_ops = {
+	.link_notify = rvin_group_link_notify,
+};
+
+/* -----------------------------------------------------------------------------
+ * Gen3 CSI2 Group Allocator
+ */
+
+/* FIXME:  This should if we find a system that supports more
+ * than one group for the whole system be replaced with a linked
+ * list of groups. And eventually all of this should be replaced
+ * with a global device allocator API.
+ *
+ * But for now this works as on all supported systems there will
+ * be only one group for all instances.
+ */
+
+static DEFINE_MUTEX(rvin_group_lock);
+static struct rvin_group *rvin_group_data;
+
+static void rvin_group_cleanup(struct rvin_group *group)
+{
+	media_device_unregister(&group->mdev);
+	media_device_cleanup(&group->mdev);
+	mutex_destroy(&group->lock);
+}
+
+static int rvin_group_init(struct rvin_group *group, struct rvin_dev *vin)
+{
+	struct media_device *mdev = &group->mdev;
+	const struct of_device_id *match;
+	struct device_node *np;
+	int ret;
+
+	mutex_init(&group->lock);
+
+	/* Count number of VINs in the system */
+	group->count = 0;
+	for_each_matching_node(np, vin->dev->driver->of_match_table)
+		if (of_device_is_available(np))
+			group->count++;
+
+	vin_dbg(vin, "found %u enabled VIN's in DT", group->count);
+
+	mdev->dev = vin->dev;
+	mdev->ops = &rvin_media_ops;
+
+	match = of_match_node(vin->dev->driver->of_match_table,
+			      vin->dev->of_node);
+
+	strlcpy(mdev->driver_name, KBUILD_MODNAME, sizeof(mdev->driver_name));
+	strlcpy(mdev->model, match->compatible, sizeof(mdev->model));
+	snprintf(mdev->bus_info, sizeof(mdev->bus_info), "platform:%s",
+		 dev_name(mdev->dev));
+
+	media_device_init(mdev);
+
+	ret = media_device_register(&group->mdev);
+	if (ret)
+		rvin_group_cleanup(group);
+
+	return ret;
+}
+
+static void rvin_group_release(struct kref *kref)
+{
+	struct rvin_group *group =
+		container_of(kref, struct rvin_group, refcount);
+
+	mutex_lock(&rvin_group_lock);
+
+	rvin_group_data = NULL;
+
+	rvin_group_cleanup(group);
+
+	kfree(group);
+
+	mutex_unlock(&rvin_group_lock);
+}
+
+static int rvin_group_get(struct rvin_dev *vin)
+{
+	struct rvin_group *group;
+	u32 id;
+	int ret;
+
+	/* Make sure VIN id is present and sane */
+	ret = of_property_read_u32(vin->dev->of_node, "renesas,id", &id);
+	if (ret) {
+		vin_err(vin, "%pOF: No renesas,id property found\n",
+			vin->dev->of_node);
+		return -EINVAL;
+	}
+
+	if (id >= RCAR_VIN_NUM) {
+		vin_err(vin, "%pOF: Invalid renesas,id '%u'\n",
+			vin->dev->of_node, id);
+		return -EINVAL;
+	}
+
+	/* Join or create a VIN group */
+	mutex_lock(&rvin_group_lock);
+	if (rvin_group_data) {
+		group = rvin_group_data;
+		kref_get(&group->refcount);
+	} else {
+		group = kzalloc(sizeof(*group), GFP_KERNEL);
+		if (!group) {
+			ret = -ENOMEM;
+			goto err_group;
+		}
+
+		ret = rvin_group_init(group, vin);
+		if (ret) {
+			kfree(group);
+			vin_err(vin, "Failed to initialize group\n");
+			goto err_group;
+		}
+
+		kref_init(&group->refcount);
+
+		rvin_group_data = group;
+	}
+	mutex_unlock(&rvin_group_lock);
+
+	/* Add VIN to group */
+	mutex_lock(&group->lock);
+
+	if (group->vin[id]) {
+		vin_err(vin, "Duplicate renesas,id property value %u\n", id);
+		mutex_unlock(&group->lock);
+		kref_put(&group->refcount, rvin_group_release);
+		return -EINVAL;
+	}
+
+	group->vin[id] = vin;
+
+	vin->id = id;
+	vin->group = group;
+	vin->v4l2_dev.mdev = &group->mdev;
+
+	mutex_unlock(&group->lock);
+
+	return 0;
+err_group:
+	mutex_unlock(&rvin_group_lock);
+	return ret;
+}
+
+static void rvin_group_put(struct rvin_dev *vin)
+{
+	struct rvin_group *group = vin->group;
+
+	mutex_lock(&group->lock);
+
+	vin->group = NULL;
+	vin->v4l2_dev.mdev = NULL;
+
+	if (WARN_ON(group->vin[vin->id] != vin))
+		goto out;
+
+	group->vin[vin->id] = NULL;
+out:
+	mutex_unlock(&group->lock);
+
+	kref_put(&group->refcount, rvin_group_release);
+}
+
 /* -----------------------------------------------------------------------------
  * Async notifier
  */
@@ -46,30 +375,93 @@ static int rvin_find_pad(struct v4l2_subdev *sd, int direction)
 	return -EINVAL;
 }
 
-static bool rvin_mbus_supported(struct rvin_graph_entity *entity)
+/* -----------------------------------------------------------------------------
+ * Digital async notifier
+ */
+
+/* The vin lock should be held when calling the subdevice attach and detach */
+static int rvin_digital_subdevice_attach(struct rvin_dev *vin,
+					 struct v4l2_subdev *subdev)
 {
-	struct v4l2_subdev *sd = entity->subdev;
 	struct v4l2_subdev_mbus_code_enum code = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
+	int ret;
 
+	/* Find source and sink pad of remote subdevice */
+	ret = rvin_find_pad(subdev, MEDIA_PAD_FL_SOURCE);
+	if (ret < 0)
+		return ret;
+	vin->digital->source_pad = ret;
+
+	ret = rvin_find_pad(subdev, MEDIA_PAD_FL_SINK);
+	vin->digital->sink_pad = ret < 0 ? 0 : ret;
+
+	/* Find compatible subdevices mbus format */
+	vin->mbus_code = 0;
 	code.index = 0;
-	code.pad = entity->source_pad;
-	while (!v4l2_subdev_call(sd, pad, enum_mbus_code, NULL, &code)) {
+	code.pad = vin->digital->source_pad;
+	while (!vin->mbus_code &&
+	       !v4l2_subdev_call(subdev, pad, enum_mbus_code, NULL, &code)) {
 		code.index++;
 		switch (code.code) {
 		case MEDIA_BUS_FMT_YUYV8_1X16:
+		case MEDIA_BUS_FMT_UYVY8_1X16:
 		case MEDIA_BUS_FMT_UYVY8_2X8:
 		case MEDIA_BUS_FMT_UYVY10_2X10:
 		case MEDIA_BUS_FMT_RGB888_1X24:
-			entity->code = code.code;
-			return true;
+			vin->mbus_code = code.code;
+			vin_dbg(vin, "Found media bus format for %s: %d\n",
+				subdev->name, vin->mbus_code);
+			break;
 		default:
 			break;
 		}
 	}
 
-	return false;
+	if (!vin->mbus_code) {
+		vin_err(vin, "Unsupported media bus format for %s\n",
+			subdev->name);
+		return -EINVAL;
+	}
+
+	/* Read tvnorms */
+	ret = v4l2_subdev_call(subdev, video, g_tvnorms, &vin->vdev.tvnorms);
+	if (ret < 0 && ret != -ENOIOCTLCMD && ret != -ENODEV)
+		return ret;
+
+	/* Read standard */
+	vin->std = V4L2_STD_UNKNOWN;
+	ret = v4l2_subdev_call(subdev, video, g_std, &vin->std);
+	if (ret < 0 && ret != -ENOIOCTLCMD)
+		return ret;
+
+	/* Add the controls */
+	ret = v4l2_ctrl_handler_init(&vin->ctrl_handler, 16);
+	if (ret < 0)
+		return ret;
+
+	ret = v4l2_ctrl_add_handler(&vin->ctrl_handler, subdev->ctrl_handler,
+				    NULL);
+	if (ret < 0) {
+		v4l2_ctrl_handler_free(&vin->ctrl_handler);
+		return ret;
+	}
+
+	vin->vdev.ctrl_handler = &vin->ctrl_handler;
+
+	vin->digital->subdev = subdev;
+
+	return 0;
+}
+
+static void rvin_digital_subdevice_detach(struct rvin_dev *vin)
+{
+	rvin_v4l2_unregister(vin);
+	v4l2_ctrl_handler_free(&vin->ctrl_handler);
+
+	vin->vdev.ctrl_handler = NULL;
+	vin->digital->subdev = NULL;
 }
 
 static int rvin_digital_notify_complete(struct v4l2_async_notifier *notifier)
@@ -77,23 +469,13 @@ static int rvin_digital_notify_complete(struct v4l2_async_notifier *notifier)
 	struct rvin_dev *vin = notifier_to_vin(notifier);
 	int ret;
 
-	/* Verify subdevices mbus format */
-	if (!rvin_mbus_supported(vin->digital)) {
-		vin_err(vin, "Unsupported media bus format for %s\n",
-			vin->digital->subdev->name);
-		return -EINVAL;
-	}
-
-	vin_dbg(vin, "Found media bus format for %s: %d\n",
-		vin->digital->subdev->name, vin->digital->code);
-
 	ret = v4l2_device_register_subdev_nodes(&vin->v4l2_dev);
 	if (ret < 0) {
 		vin_err(vin, "Failed to register subdev nodes\n");
 		return ret;
 	}
 
-	return rvin_v4l2_probe(vin);
+	return rvin_v4l2_register(vin);
 }
 
 static void rvin_digital_notify_unbind(struct v4l2_async_notifier *notifier,
@@ -103,8 +485,10 @@ static void rvin_digital_notify_unbind(struct v4l2_async_notifier *notifier,
 	struct rvin_dev *vin = notifier_to_vin(notifier);
 
 	vin_dbg(vin, "unbind digital subdev %s\n", subdev->name);
-	rvin_v4l2_remove(vin);
-	vin->digital->subdev = NULL;
+
+	mutex_lock(&vin->lock);
+	rvin_digital_subdevice_detach(vin);
+	mutex_unlock(&vin->lock);
 }
 
 static int rvin_digital_notify_bound(struct v4l2_async_notifier *notifier,
@@ -114,19 +498,13 @@ static int rvin_digital_notify_bound(struct v4l2_async_notifier *notifier,
 	struct rvin_dev *vin = notifier_to_vin(notifier);
 	int ret;
 
-	v4l2_set_subdev_hostdata(subdev, vin);
-
-	/* Find source and sink pad of remote subdevice */
-
-	ret = rvin_find_pad(subdev, MEDIA_PAD_FL_SOURCE);
-	if (ret < 0)
+	mutex_lock(&vin->lock);
+	ret = rvin_digital_subdevice_attach(vin, subdev);
+	mutex_unlock(&vin->lock);
+	if (ret)
 		return ret;
-	vin->digital->source_pad = ret;
 
-	ret = rvin_find_pad(subdev, MEDIA_PAD_FL_SINK);
-	vin->digital->sink_pad = ret < 0 ? 0 : ret;
-
-	vin->digital->subdev = subdev;
+	v4l2_set_subdev_hostdata(subdev, vin);
 
 	vin_dbg(vin, "bound subdev %s source pad: %u sink pad: %u\n",
 		subdev->name, vin->digital->source_pad,
@@ -134,13 +512,13 @@ static int rvin_digital_notify_bound(struct v4l2_async_notifier *notifier,
 
 	return 0;
 }
+
 static const struct v4l2_async_notifier_operations rvin_digital_notify_ops = {
 	.bound = rvin_digital_notify_bound,
 	.unbind = rvin_digital_notify_unbind,
 	.complete = rvin_digital_notify_complete,
 };
 
-
 static int rvin_digital_parse_v4l2(struct device *dev,
 				   struct v4l2_fwnode_endpoint *vep,
 				   struct v4l2_async_subdev *asd)
@@ -152,16 +530,16 @@ static int rvin_digital_parse_v4l2(struct device *dev,
 	if (vep->base.port || vep->base.id)
 		return -ENOTCONN;
 
-	rvge->mbus_cfg.type = vep->bus_type;
+	vin->mbus_cfg.type = vep->bus_type;
 
-	switch (rvge->mbus_cfg.type) {
+	switch (vin->mbus_cfg.type) {
 	case V4L2_MBUS_PARALLEL:
 		vin_dbg(vin, "Found PARALLEL media bus\n");
-		rvge->mbus_cfg.flags = vep->bus.parallel.flags;
+		vin->mbus_cfg.flags = vep->bus.parallel.flags;
 		break;
 	case V4L2_MBUS_BT656:
 		vin_dbg(vin, "Found BT656 media bus\n");
-		rvge->mbus_cfg.flags = 0;
+		vin->mbus_cfg.flags = 0;
 		break;
 	default:
 		vin_err(vin, "Unknown media bus type\n");
@@ -199,25 +577,478 @@ static int rvin_digital_graph_init(struct rvin_dev *vin)
 	return 0;
 }
 
+/* -----------------------------------------------------------------------------
+ * Group async notifier
+ */
+
+static int rvin_group_notify_complete(struct v4l2_async_notifier *notifier)
+{
+	struct rvin_dev *vin = notifier_to_vin(notifier);
+	const struct rvin_group_route *route;
+	unsigned int i;
+	int ret;
+
+	ret = v4l2_device_register_subdev_nodes(&vin->v4l2_dev);
+	if (ret) {
+		vin_err(vin, "Failed to register subdev nodes\n");
+		return ret;
+	}
+
+	/* Register all video nodes for the group. */
+	for (i = 0; i < RCAR_VIN_NUM; i++) {
+		if (vin->group->vin[i]) {
+			ret = rvin_v4l2_register(vin->group->vin[i]);
+			if (ret)
+				return ret;
+		}
+	}
+
+	/* Create all media device links between VINs and CSI-2's. */
+	mutex_lock(&vin->group->lock);
+	for (route = vin->info->routes; route->mask; route++) {
+		struct media_pad *source_pad, *sink_pad;
+		struct media_entity *source, *sink;
+		unsigned int source_idx;
+
+		/* Check that VIN is part of the group. */
+		if (!vin->group->vin[route->vin])
+			continue;
+
+		/* Check that VIN' master is part of the group. */
+		if (!vin->group->vin[rvin_group_id_to_master(route->vin)])
+			continue;
+
+		/* Check that CSI-2 is part of the group. */
+		if (!vin->group->csi[route->csi].subdev)
+			continue;
+
+		source = &vin->group->csi[route->csi].subdev->entity;
+		source_idx = rvin_group_csi_channel_to_pad(route->channel);
+		source_pad = &source->pads[source_idx];
+
+		sink = &vin->group->vin[route->vin]->vdev.entity;
+		sink_pad = &sink->pads[0];
+
+		/* Skip if link already exists. */
+		if (media_entity_find_link(source_pad, sink_pad))
+			continue;
+
+		ret = media_create_pad_link(source, source_idx, sink, 0, 0);
+		if (ret) {
+			vin_err(vin, "Error adding link from %s to %s\n",
+				source->name, sink->name);
+			break;
+		}
+	}
+	mutex_unlock(&vin->group->lock);
+
+	return ret;
+}
+
+static void rvin_group_notify_unbind(struct v4l2_async_notifier *notifier,
+				     struct v4l2_subdev *subdev,
+				     struct v4l2_async_subdev *asd)
+{
+	struct rvin_dev *vin = notifier_to_vin(notifier);
+	unsigned int i;
+
+	for (i = 0; i < RCAR_VIN_NUM; i++)
+		if (vin->group->vin[i])
+			rvin_v4l2_unregister(vin->group->vin[i]);
+
+	mutex_lock(&vin->group->lock);
+
+	for (i = 0; i < RVIN_CSI_MAX; i++) {
+		if (vin->group->csi[i].fwnode != asd->match.fwnode)
+			continue;
+		vin->group->csi[i].subdev = NULL;
+		vin_dbg(vin, "Unbind CSI-2 %s from slot %u\n", subdev->name, i);
+		break;
+	}
+
+	mutex_unlock(&vin->group->lock);
+}
+
+static int rvin_group_notify_bound(struct v4l2_async_notifier *notifier,
+				   struct v4l2_subdev *subdev,
+				   struct v4l2_async_subdev *asd)
+{
+	struct rvin_dev *vin = notifier_to_vin(notifier);
+	unsigned int i;
+
+	mutex_lock(&vin->group->lock);
+
+	for (i = 0; i < RVIN_CSI_MAX; i++) {
+		if (vin->group->csi[i].fwnode != asd->match.fwnode)
+			continue;
+		vin->group->csi[i].subdev = subdev;
+		vin_dbg(vin, "Bound CSI-2 %s to slot %u\n", subdev->name, i);
+		break;
+	}
+
+	mutex_unlock(&vin->group->lock);
+
+	return 0;
+}
+
+static const struct v4l2_async_notifier_operations rvin_group_notify_ops = {
+	.bound = rvin_group_notify_bound,
+	.unbind = rvin_group_notify_unbind,
+	.complete = rvin_group_notify_complete,
+};
+
+static int rvin_mc_parse_of_endpoint(struct device *dev,
+				     struct v4l2_fwnode_endpoint *vep,
+				     struct v4l2_async_subdev *asd)
+{
+	struct rvin_dev *vin = dev_get_drvdata(dev);
+
+	if (vep->base.port != 1 || vep->base.id >= RVIN_CSI_MAX)
+		return -EINVAL;
+
+	if (!of_device_is_available(to_of_node(asd->match.fwnode))) {
+
+		vin_dbg(vin, "OF device %pOF disabled, ignoring\n",
+			to_of_node(asd->match.fwnode));
+		return -ENOTCONN;
+
+	}
+
+	if (vin->group->csi[vep->base.id].fwnode) {
+		vin_dbg(vin, "OF device %pOF already handled\n",
+			to_of_node(asd->match.fwnode));
+		return -ENOTCONN;
+	}
+
+	vin->group->csi[vep->base.id].fwnode = asd->match.fwnode;
+
+	vin_dbg(vin, "Add group OF device %pOF to slot %u\n",
+		to_of_node(asd->match.fwnode), vep->base.id);
+
+	return 0;
+}
+
+static int rvin_mc_parse_of_graph(struct rvin_dev *vin)
+{
+	unsigned int count = 0;
+	unsigned int i;
+	int ret;
+
+	mutex_lock(&vin->group->lock);
+
+	/* If there already is a notifier something has gone wrong, bail out. */
+	if (WARN_ON(vin->group->notifier)) {
+		mutex_unlock(&vin->group->lock);
+		return -EINVAL;
+	}
+
+	/* If not all VIN's are registered don't register the notifier. */
+	for (i = 0; i < RCAR_VIN_NUM; i++)
+		if (vin->group->vin[i])
+			count++;
+
+	if (vin->group->count != count) {
+		mutex_unlock(&vin->group->lock);
+		return 0;
+	}
+
+	/*
+	 * Have all VIN's look for subdevices. Some subdevices will overlap
+	 * but the parser function can handle it, so each subdevice will
+	 * only be registered once with the notifier.
+	 */
+
+	vin->group->notifier = &vin->notifier;
+
+	for (i = 0; i < RCAR_VIN_NUM; i++) {
+		if (!vin->group->vin[i])
+			continue;
+
+		ret = v4l2_async_notifier_parse_fwnode_endpoints_by_port(
+				vin->group->vin[i]->dev, vin->group->notifier,
+				sizeof(struct v4l2_async_subdev), 1,
+				rvin_mc_parse_of_endpoint);
+		if (ret) {
+			mutex_unlock(&vin->group->lock);
+			return ret;
+		}
+	}
+
+	mutex_unlock(&vin->group->lock);
+
+	vin->group->notifier->ops = &rvin_group_notify_ops;
+
+	ret = v4l2_async_notifier_register(&vin->v4l2_dev, &vin->notifier);
+	if (ret < 0) {
+		vin_err(vin, "Notifier registration failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int rvin_mc_init(struct rvin_dev *vin)
+{
+	int ret;
+
+	/* All our sources are CSI-2 */
+	vin->mbus_cfg.type = V4L2_MBUS_CSI2;
+	vin->mbus_cfg.flags = 0;
+
+	vin->pad.flags = MEDIA_PAD_FL_SINK;
+	ret = media_entity_pads_init(&vin->vdev.entity, 1, &vin->pad);
+	if (ret)
+		return ret;
+
+	ret = rvin_group_get(vin);
+	if (ret)
+		return ret;
+
+	ret = rvin_mc_parse_of_graph(vin);
+	if (ret)
+		rvin_group_put(vin);
+
+	return ret;
+}
+
 /* -----------------------------------------------------------------------------
  * Platform Device Driver
  */
 
+static const struct rvin_info rcar_info_h1 = {
+	.model = RCAR_H1,
+	.use_mc = false,
+	.max_width = 2048,
+	.max_height = 2048,
+};
+
+static const struct rvin_info rcar_info_m1 = {
+	.model = RCAR_M1,
+	.use_mc = false,
+	.max_width = 2048,
+	.max_height = 2048,
+};
+
+static const struct rvin_info rcar_info_gen2 = {
+	.model = RCAR_GEN2,
+	.use_mc = false,
+	.max_width = 2048,
+	.max_height = 2048,
+};
+
+static const struct rvin_group_route rcar_info_r8a7795_routes[] = {
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 0, .mask = BIT(0) | BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 0, .mask = BIT(1) | BIT(4) },
+	{ .csi = RVIN_CSI40, .channel = 1, .vin = 0, .mask = BIT(2) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 1, .mask = BIT(0) },
+	{ .csi = RVIN_CSI40, .channel = 1, .vin = 1, .mask = BIT(1) | BIT(3) },
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 1, .mask = BIT(2) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 1, .mask = BIT(4) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 2, .mask = BIT(0) },
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 2, .mask = BIT(1) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 2, .mask = BIT(2) },
+	{ .csi = RVIN_CSI40, .channel = 2, .vin = 2, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 2, .vin = 2, .mask = BIT(4) },
+	{ .csi = RVIN_CSI40, .channel = 1, .vin = 3, .mask = BIT(0) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 3, .mask = BIT(1) | BIT(2) },
+	{ .csi = RVIN_CSI40, .channel = 3, .vin = 3, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 3, .vin = 3, .mask = BIT(4) },
+	{ .csi = RVIN_CSI41, .channel = 0, .vin = 4, .mask = BIT(0) | BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 4, .mask = BIT(1) | BIT(4) },
+	{ .csi = RVIN_CSI41, .channel = 1, .vin = 4, .mask = BIT(2) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 5, .mask = BIT(0) },
+	{ .csi = RVIN_CSI41, .channel = 1, .vin = 5, .mask = BIT(1) | BIT(3) },
+	{ .csi = RVIN_CSI41, .channel = 0, .vin = 5, .mask = BIT(2) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 5, .mask = BIT(4) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 6, .mask = BIT(0) },
+	{ .csi = RVIN_CSI41, .channel = 0, .vin = 6, .mask = BIT(1) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 6, .mask = BIT(2) },
+	{ .csi = RVIN_CSI41, .channel = 2, .vin = 6, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 2, .vin = 6, .mask = BIT(4) },
+	{ .csi = RVIN_CSI41, .channel = 1, .vin = 7, .mask = BIT(0) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 7, .mask = BIT(1) | BIT(2) },
+	{ .csi = RVIN_CSI41, .channel = 3, .vin = 7, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 3, .vin = 7, .mask = BIT(4) },
+	{ /* Sentinel */ }
+};
+
+static const struct rvin_info rcar_info_r8a7795 = {
+	.model = RCAR_GEN3,
+	.use_mc = true,
+	.max_width = 4096,
+	.max_height = 4096,
+	.routes = rcar_info_r8a7795_routes,
+};
+
+static const struct rvin_group_route rcar_info_r8a7795es1_routes[] = {
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 0, .mask = BIT(0) | BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 0, .mask = BIT(1) | BIT(4) },
+	{ .csi = RVIN_CSI21, .channel = 0, .vin = 0, .mask = BIT(2) | BIT(5) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 1, .mask = BIT(0) },
+	{ .csi = RVIN_CSI21, .channel = 0, .vin = 1, .mask = BIT(1) },
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 1, .mask = BIT(2) },
+	{ .csi = RVIN_CSI40, .channel = 1, .vin = 1, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 1, .mask = BIT(4) },
+	{ .csi = RVIN_CSI21, .channel = 1, .vin = 1, .mask = BIT(5) },
+	{ .csi = RVIN_CSI21, .channel = 0, .vin = 2, .mask = BIT(0) },
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 2, .mask = BIT(1) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 2, .mask = BIT(2) },
+	{ .csi = RVIN_CSI40, .channel = 2, .vin = 2, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 2, .vin = 2, .mask = BIT(4) },
+	{ .csi = RVIN_CSI21, .channel = 2, .vin = 2, .mask = BIT(5) },
+	{ .csi = RVIN_CSI40, .channel = 1, .vin = 3, .mask = BIT(0) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 3, .mask = BIT(1) },
+	{ .csi = RVIN_CSI21, .channel = 1, .vin = 3, .mask = BIT(2) },
+	{ .csi = RVIN_CSI40, .channel = 3, .vin = 3, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 3, .vin = 3, .mask = BIT(4) },
+	{ .csi = RVIN_CSI21, .channel = 3, .vin = 3, .mask = BIT(5) },
+	{ .csi = RVIN_CSI41, .channel = 0, .vin = 4, .mask = BIT(0) | BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 4, .mask = BIT(1) | BIT(4) },
+	{ .csi = RVIN_CSI21, .channel = 0, .vin = 4, .mask = BIT(2) | BIT(5) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 5, .mask = BIT(0) },
+	{ .csi = RVIN_CSI21, .channel = 0, .vin = 5, .mask = BIT(1) },
+	{ .csi = RVIN_CSI41, .channel = 0, .vin = 5, .mask = BIT(2) },
+	{ .csi = RVIN_CSI41, .channel = 1, .vin = 5, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 5, .mask = BIT(4) },
+	{ .csi = RVIN_CSI21, .channel = 1, .vin = 5, .mask = BIT(5) },
+	{ .csi = RVIN_CSI21, .channel = 0, .vin = 6, .mask = BIT(0) },
+	{ .csi = RVIN_CSI41, .channel = 0, .vin = 6, .mask = BIT(1) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 6, .mask = BIT(2) },
+	{ .csi = RVIN_CSI41, .channel = 2, .vin = 6, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 2, .vin = 6, .mask = BIT(4) },
+	{ .csi = RVIN_CSI21, .channel = 2, .vin = 6, .mask = BIT(5) },
+	{ .csi = RVIN_CSI41, .channel = 1, .vin = 7, .mask = BIT(0) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 7, .mask = BIT(1) },
+	{ .csi = RVIN_CSI21, .channel = 1, .vin = 7, .mask = BIT(2) },
+	{ .csi = RVIN_CSI41, .channel = 3, .vin = 7, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 3, .vin = 7, .mask = BIT(4) },
+	{ .csi = RVIN_CSI21, .channel = 3, .vin = 7, .mask = BIT(5) },
+	{ /* Sentinel */ }
+};
+
+static const struct rvin_info rcar_info_r8a7795es1 = {
+	.model = RCAR_GEN3,
+	.use_mc = true,
+	.max_width = 4096,
+	.max_height = 4096,
+	.routes = rcar_info_r8a7795es1_routes,
+};
+
+static const struct rvin_group_route rcar_info_r8a7796_routes[] = {
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 0, .mask = BIT(0) | BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 0, .mask = BIT(1) | BIT(4) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 1, .mask = BIT(0) },
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 1, .mask = BIT(2) },
+	{ .csi = RVIN_CSI40, .channel = 1, .vin = 1, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 1, .mask = BIT(4) },
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 2, .mask = BIT(1) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 2, .mask = BIT(2) },
+	{ .csi = RVIN_CSI40, .channel = 2, .vin = 2, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 2, .vin = 2, .mask = BIT(4) },
+	{ .csi = RVIN_CSI40, .channel = 1, .vin = 3, .mask = BIT(0) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 3, .mask = BIT(1) },
+	{ .csi = RVIN_CSI40, .channel = 3, .vin = 3, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 3, .vin = 3, .mask = BIT(4) },
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 4, .mask = BIT(0) | BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 4, .mask = BIT(1) | BIT(4) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 5, .mask = BIT(0) },
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 5, .mask = BIT(2) },
+	{ .csi = RVIN_CSI40, .channel = 1, .vin = 5, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 5, .mask = BIT(4) },
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 6, .mask = BIT(1) },
+	{ .csi = RVIN_CSI20, .channel = 0, .vin = 6, .mask = BIT(2) },
+	{ .csi = RVIN_CSI40, .channel = 2, .vin = 6, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 2, .vin = 6, .mask = BIT(4) },
+	{ .csi = RVIN_CSI40, .channel = 1, .vin = 7, .mask = BIT(0) },
+	{ .csi = RVIN_CSI20, .channel = 1, .vin = 7, .mask = BIT(1) },
+	{ .csi = RVIN_CSI40, .channel = 3, .vin = 7, .mask = BIT(3) },
+	{ .csi = RVIN_CSI20, .channel = 3, .vin = 7, .mask = BIT(4) },
+	{ /* Sentinel */ }
+};
+
+static const struct rvin_info rcar_info_r8a7796 = {
+	.model = RCAR_GEN3,
+	.use_mc = true,
+	.max_width = 4096,
+	.max_height = 4096,
+	.routes = rcar_info_r8a7796_routes,
+};
+
+static const struct rvin_group_route _rcar_info_r8a77970_routes[] = {
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 0, .mask = BIT(0) | BIT(3) },
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 1, .mask = BIT(2) },
+	{ .csi = RVIN_CSI40, .channel = 1, .vin = 1, .mask = BIT(3) },
+	{ .csi = RVIN_CSI40, .channel = 0, .vin = 2, .mask = BIT(1) },
+	{ .csi = RVIN_CSI40, .channel = 2, .vin = 2, .mask = BIT(3) },
+	{ .csi = RVIN_CSI40, .channel = 1, .vin = 3, .mask = BIT(0) },
+	{ .csi = RVIN_CSI40, .channel = 3, .vin = 3, .mask = BIT(3) },
+	{ /* Sentinel */ }
+};
+
+static const struct rvin_info rcar_info_r8a77970 = {
+	.model = RCAR_GEN3,
+	.use_mc = true,
+	.max_width = 4096,
+	.max_height = 4096,
+	.routes = _rcar_info_r8a77970_routes,
+};
+
 static const struct of_device_id rvin_of_id_table[] = {
-	{ .compatible = "renesas,vin-r8a7794", .data = (void *)RCAR_GEN2 },
-	{ .compatible = "renesas,vin-r8a7793", .data = (void *)RCAR_GEN2 },
-	{ .compatible = "renesas,vin-r8a7791", .data = (void *)RCAR_GEN2 },
-	{ .compatible = "renesas,vin-r8a7790", .data = (void *)RCAR_GEN2 },
-	{ .compatible = "renesas,vin-r8a7779", .data = (void *)RCAR_H1 },
-	{ .compatible = "renesas,vin-r8a7778", .data = (void *)RCAR_M1 },
-	{ .compatible = "renesas,rcar-gen2-vin", .data = (void *)RCAR_GEN2 },
-	{ },
+	{
+		.compatible = "renesas,vin-r8a7778",
+		.data = &rcar_info_m1,
+	},
+	{
+		.compatible = "renesas,vin-r8a7779",
+		.data = &rcar_info_h1,
+	},
+	{
+		.compatible = "renesas,vin-r8a7790",
+		.data = &rcar_info_gen2,
+	},
+	{
+		.compatible = "renesas,vin-r8a7791",
+		.data = &rcar_info_gen2,
+	},
+	{
+		.compatible = "renesas,vin-r8a7793",
+		.data = &rcar_info_gen2,
+	},
+	{
+		.compatible = "renesas,vin-r8a7794",
+		.data = &rcar_info_gen2,
+	},
+	{
+		.compatible = "renesas,rcar-gen2-vin",
+		.data = &rcar_info_gen2,
+	},
+	{
+		.compatible = "renesas,vin-r8a7795",
+		.data = &rcar_info_r8a7795,
+	},
+	{
+		.compatible = "renesas,vin-r8a7796",
+		.data = &rcar_info_r8a7796,
+	},
+	{
+		.compatible = "renesas,vin-r8a77970",
+		.data = &rcar_info_r8a77970,
+	},
+	{ /* Sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, rvin_of_id_table);
 
+static const struct soc_device_attribute r8a7795es1[] = {
+	{
+		.soc_id = "r8a7795", .revision = "ES1.*",
+		.data = &rcar_info_r8a7795es1,
+	},
+	{ /* Sentinel */ }
+};
+
 static int rcar_vin_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *match;
+	const struct soc_device_attribute *attr;
 	struct rvin_dev *vin;
 	struct resource *mem;
 	int irq, ret;
@@ -226,12 +1057,16 @@ static int rcar_vin_probe(struct platform_device *pdev)
 	if (!vin)
 		return -ENOMEM;
 
-	match = of_match_device(of_match_ptr(rvin_of_id_table), &pdev->dev);
-	if (!match)
-		return -ENODEV;
-
 	vin->dev = &pdev->dev;
-	vin->chip = (enum chip_id)match->data;
+	vin->info = of_device_get_match_data(&pdev->dev);
+
+	/*
+	 * Special care is needed on r8a7795 ES1.x since it
+	 * uses different routing than r8a7795 ES2.0.
+	 */
+	attr = soc_device_match(r8a7795es1);
+	if (attr)
+		vin->info = attr->data;
 
 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (mem == NULL)
@@ -245,13 +1080,15 @@ static int rcar_vin_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
-	ret = rvin_dma_probe(vin, irq);
+	ret = rvin_dma_register(vin, irq);
 	if (ret)
 		return ret;
 
 	platform_set_drvdata(pdev, vin);
-
-	ret = rvin_digital_graph_init(vin);
+	if (vin->info->use_mc)
+		ret = rvin_mc_init(vin);
+	else
+		ret = rvin_digital_graph_init(vin);
 	if (ret < 0)
 		goto error;
 
@@ -260,7 +1097,7 @@ static int rcar_vin_probe(struct platform_device *pdev)
 
 	return 0;
 error:
-	rvin_dma_remove(vin);
+	rvin_dma_unregister(vin);
 	v4l2_async_notifier_cleanup(&vin->notifier);
 
 	return ret;
@@ -272,10 +1109,22 @@ static int rcar_vin_remove(struct platform_device *pdev)
 
 	pm_runtime_disable(&pdev->dev);
 
+	rvin_v4l2_unregister(vin);
+
 	v4l2_async_notifier_unregister(&vin->notifier);
 	v4l2_async_notifier_cleanup(&vin->notifier);
 
-	rvin_dma_remove(vin);
+	if (vin->info->use_mc) {
+		mutex_lock(&vin->group->lock);
+		if (vin->group->notifier == &vin->notifier)
+			vin->group->notifier = NULL;
+		mutex_unlock(&vin->group->lock);
+		rvin_group_put(vin);
+	} else {
+		v4l2_ctrl_handler_free(&vin->ctrl_handler);
+	}
+
+	rvin_dma_unregister(vin);
 
 	return 0;
 }
diff --git a/drivers/media/platform/rcar-vin/rcar-csi2.c b/drivers/media/platform/rcar-vin/rcar-csi2.c
new file mode 100644
index 000000000000..daef72d410a3
--- /dev/null
+++ b/drivers/media/platform/rcar-vin/rcar-csi2.c
@@ -0,0 +1,1084 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for Renesas R-Car MIPI CSI-2 Receiver
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ */
+
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/sys_soc.h>
+
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-mc.h>
+#include <media/v4l2-subdev.h>
+
+struct rcar_csi2;
+
+/* Register offsets and bits */
+
+/* Control Timing Select */
+#define TREF_REG			0x00
+#define TREF_TREF			BIT(0)
+
+/* Software Reset */
+#define SRST_REG			0x04
+#define SRST_SRST			BIT(0)
+
+/* PHY Operation Control */
+#define PHYCNT_REG			0x08
+#define PHYCNT_SHUTDOWNZ		BIT(17)
+#define PHYCNT_RSTZ			BIT(16)
+#define PHYCNT_ENABLECLK		BIT(4)
+#define PHYCNT_ENABLE_3			BIT(3)
+#define PHYCNT_ENABLE_2			BIT(2)
+#define PHYCNT_ENABLE_1			BIT(1)
+#define PHYCNT_ENABLE_0			BIT(0)
+
+/* Checksum Control */
+#define CHKSUM_REG			0x0c
+#define CHKSUM_ECC_EN			BIT(1)
+#define CHKSUM_CRC_EN			BIT(0)
+
+/*
+ * Channel Data Type Select
+ * VCDT[0-15]:  Channel 1 VCDT[16-31]:  Channel 2
+ * VCDT2[0-15]: Channel 3 VCDT2[16-31]: Channel 4
+ */
+#define VCDT_REG			0x10
+#define VCDT2_REG			0x14
+#define VCDT_VCDTN_EN			BIT(15)
+#define VCDT_SEL_VC(n)			(((n) & 0x3) << 8)
+#define VCDT_SEL_DTN_ON			BIT(6)
+#define VCDT_SEL_DT(n)			(((n) & 0x3f) << 0)
+
+/* Frame Data Type Select */
+#define FRDT_REG			0x18
+
+/* Field Detection Control */
+#define FLD_REG				0x1c
+#define FLD_FLD_NUM(n)			(((n) & 0xff) << 16)
+#define FLD_FLD_EN4			BIT(3)
+#define FLD_FLD_EN3			BIT(2)
+#define FLD_FLD_EN2			BIT(1)
+#define FLD_FLD_EN			BIT(0)
+
+/* Automatic Standby Control */
+#define ASTBY_REG			0x20
+
+/* Long Data Type Setting 0 */
+#define LNGDT0_REG			0x28
+
+/* Long Data Type Setting 1 */
+#define LNGDT1_REG			0x2c
+
+/* Interrupt Enable */
+#define INTEN_REG			0x30
+
+/* Interrupt Source Mask */
+#define INTCLOSE_REG			0x34
+
+/* Interrupt Status Monitor */
+#define INTSTATE_REG			0x38
+#define INTSTATE_INT_ULPS_START		BIT(7)
+#define INTSTATE_INT_ULPS_END		BIT(6)
+
+/* Interrupt Error Status Monitor */
+#define INTERRSTATE_REG			0x3c
+
+/* Short Packet Data */
+#define SHPDAT_REG			0x40
+
+/* Short Packet Count */
+#define SHPCNT_REG			0x44
+
+/* LINK Operation Control */
+#define LINKCNT_REG			0x48
+#define LINKCNT_MONITOR_EN		BIT(31)
+#define LINKCNT_REG_MONI_PACT_EN	BIT(25)
+#define LINKCNT_ICLK_NONSTOP		BIT(24)
+
+/* Lane Swap */
+#define LSWAP_REG			0x4c
+#define LSWAP_L3SEL(n)			(((n) & 0x3) << 6)
+#define LSWAP_L2SEL(n)			(((n) & 0x3) << 4)
+#define LSWAP_L1SEL(n)			(((n) & 0x3) << 2)
+#define LSWAP_L0SEL(n)			(((n) & 0x3) << 0)
+
+/* PHY Test Interface Write Register */
+#define PHTW_REG			0x50
+#define PHTW_DWEN			BIT(24)
+#define PHTW_TESTDIN_DATA(n)		(((n & 0xff)) << 16)
+#define PHTW_CWEN			BIT(8)
+#define PHTW_TESTDIN_CODE(n)		((n & 0xff))
+
+struct phtw_value {
+	u16 data;
+	u16 code;
+};
+
+struct rcsi2_mbps_reg {
+	u16 mbps;
+	u16 reg;
+};
+
+static const struct rcsi2_mbps_reg phtw_mbps_h3_v3h_m3n[] = {
+	{ .mbps =   80, .reg = 0x86 },
+	{ .mbps =   90, .reg = 0x86 },
+	{ .mbps =  100, .reg = 0x87 },
+	{ .mbps =  110, .reg = 0x87 },
+	{ .mbps =  120, .reg = 0x88 },
+	{ .mbps =  130, .reg = 0x88 },
+	{ .mbps =  140, .reg = 0x89 },
+	{ .mbps =  150, .reg = 0x89 },
+	{ .mbps =  160, .reg = 0x8a },
+	{ .mbps =  170, .reg = 0x8a },
+	{ .mbps =  180, .reg = 0x8b },
+	{ .mbps =  190, .reg = 0x8b },
+	{ .mbps =  205, .reg = 0x8c },
+	{ .mbps =  220, .reg = 0x8d },
+	{ .mbps =  235, .reg = 0x8e },
+	{ .mbps =  250, .reg = 0x8e },
+	{ /* sentinel */ },
+};
+
+static const struct rcsi2_mbps_reg phtw_mbps_v3m_e3[] = {
+	{ .mbps =   80, .reg = 0x00 },
+	{ .mbps =   90, .reg = 0x20 },
+	{ .mbps =  100, .reg = 0x40 },
+	{ .mbps =  110, .reg = 0x02 },
+	{ .mbps =  130, .reg = 0x22 },
+	{ .mbps =  140, .reg = 0x42 },
+	{ .mbps =  150, .reg = 0x04 },
+	{ .mbps =  170, .reg = 0x24 },
+	{ .mbps =  180, .reg = 0x44 },
+	{ .mbps =  200, .reg = 0x06 },
+	{ .mbps =  220, .reg = 0x26 },
+	{ .mbps =  240, .reg = 0x46 },
+	{ .mbps =  250, .reg = 0x08 },
+	{ .mbps =  270, .reg = 0x28 },
+	{ .mbps =  300, .reg = 0x0a },
+	{ .mbps =  330, .reg = 0x2a },
+	{ .mbps =  360, .reg = 0x4a },
+	{ .mbps =  400, .reg = 0x0c },
+	{ .mbps =  450, .reg = 0x2c },
+	{ .mbps =  500, .reg = 0x0e },
+	{ .mbps =  550, .reg = 0x2e },
+	{ .mbps =  600, .reg = 0x10 },
+	{ .mbps =  650, .reg = 0x30 },
+	{ .mbps =  700, .reg = 0x12 },
+	{ .mbps =  750, .reg = 0x32 },
+	{ .mbps =  800, .reg = 0x52 },
+	{ .mbps =  850, .reg = 0x72 },
+	{ .mbps =  900, .reg = 0x14 },
+	{ .mbps =  950, .reg = 0x34 },
+	{ .mbps = 1000, .reg = 0x54 },
+	{ .mbps = 1050, .reg = 0x74 },
+	{ .mbps = 1125, .reg = 0x16 },
+	{ /* sentinel */ },
+};
+
+/* PHY Test Interface Clear */
+#define PHTC_REG			0x58
+#define PHTC_TESTCLR			BIT(0)
+
+/* PHY Frequency Control */
+#define PHYPLL_REG			0x68
+#define PHYPLL_HSFREQRANGE(n)		((n) << 16)
+
+static const struct rcsi2_mbps_reg hsfreqrange_h3_v3h_m3n[] = {
+	{ .mbps =   80, .reg = 0x00 },
+	{ .mbps =   90, .reg = 0x10 },
+	{ .mbps =  100, .reg = 0x20 },
+	{ .mbps =  110, .reg = 0x30 },
+	{ .mbps =  120, .reg = 0x01 },
+	{ .mbps =  130, .reg = 0x11 },
+	{ .mbps =  140, .reg = 0x21 },
+	{ .mbps =  150, .reg = 0x31 },
+	{ .mbps =  160, .reg = 0x02 },
+	{ .mbps =  170, .reg = 0x12 },
+	{ .mbps =  180, .reg = 0x22 },
+	{ .mbps =  190, .reg = 0x32 },
+	{ .mbps =  205, .reg = 0x03 },
+	{ .mbps =  220, .reg = 0x13 },
+	{ .mbps =  235, .reg = 0x23 },
+	{ .mbps =  250, .reg = 0x33 },
+	{ .mbps =  275, .reg = 0x04 },
+	{ .mbps =  300, .reg = 0x14 },
+	{ .mbps =  325, .reg = 0x25 },
+	{ .mbps =  350, .reg = 0x35 },
+	{ .mbps =  400, .reg = 0x05 },
+	{ .mbps =  450, .reg = 0x16 },
+	{ .mbps =  500, .reg = 0x26 },
+	{ .mbps =  550, .reg = 0x37 },
+	{ .mbps =  600, .reg = 0x07 },
+	{ .mbps =  650, .reg = 0x18 },
+	{ .mbps =  700, .reg = 0x28 },
+	{ .mbps =  750, .reg = 0x39 },
+	{ .mbps =  800, .reg = 0x09 },
+	{ .mbps =  850, .reg = 0x19 },
+	{ .mbps =  900, .reg = 0x29 },
+	{ .mbps =  950, .reg = 0x3a },
+	{ .mbps = 1000, .reg = 0x0a },
+	{ .mbps = 1050, .reg = 0x1a },
+	{ .mbps = 1100, .reg = 0x2a },
+	{ .mbps = 1150, .reg = 0x3b },
+	{ .mbps = 1200, .reg = 0x0b },
+	{ .mbps = 1250, .reg = 0x1b },
+	{ .mbps = 1300, .reg = 0x2b },
+	{ .mbps = 1350, .reg = 0x3c },
+	{ .mbps = 1400, .reg = 0x0c },
+	{ .mbps = 1450, .reg = 0x1c },
+	{ .mbps = 1500, .reg = 0x2c },
+	{ /* sentinel */ },
+};
+
+static const struct rcsi2_mbps_reg hsfreqrange_m3w_h3es1[] = {
+	{ .mbps =   80,	.reg = 0x00 },
+	{ .mbps =   90,	.reg = 0x10 },
+	{ .mbps =  100,	.reg = 0x20 },
+	{ .mbps =  110,	.reg = 0x30 },
+	{ .mbps =  120,	.reg = 0x01 },
+	{ .mbps =  130,	.reg = 0x11 },
+	{ .mbps =  140,	.reg = 0x21 },
+	{ .mbps =  150,	.reg = 0x31 },
+	{ .mbps =  160,	.reg = 0x02 },
+	{ .mbps =  170,	.reg = 0x12 },
+	{ .mbps =  180,	.reg = 0x22 },
+	{ .mbps =  190,	.reg = 0x32 },
+	{ .mbps =  205,	.reg = 0x03 },
+	{ .mbps =  220,	.reg = 0x13 },
+	{ .mbps =  235,	.reg = 0x23 },
+	{ .mbps =  250,	.reg = 0x33 },
+	{ .mbps =  275,	.reg = 0x04 },
+	{ .mbps =  300,	.reg = 0x14 },
+	{ .mbps =  325,	.reg = 0x05 },
+	{ .mbps =  350,	.reg = 0x15 },
+	{ .mbps =  400,	.reg = 0x25 },
+	{ .mbps =  450,	.reg = 0x06 },
+	{ .mbps =  500,	.reg = 0x16 },
+	{ .mbps =  550,	.reg = 0x07 },
+	{ .mbps =  600,	.reg = 0x17 },
+	{ .mbps =  650,	.reg = 0x08 },
+	{ .mbps =  700,	.reg = 0x18 },
+	{ .mbps =  750,	.reg = 0x09 },
+	{ .mbps =  800,	.reg = 0x19 },
+	{ .mbps =  850,	.reg = 0x29 },
+	{ .mbps =  900,	.reg = 0x39 },
+	{ .mbps =  950,	.reg = 0x0a },
+	{ .mbps = 1000,	.reg = 0x1a },
+	{ .mbps = 1050,	.reg = 0x2a },
+	{ .mbps = 1100,	.reg = 0x3a },
+	{ .mbps = 1150,	.reg = 0x0b },
+	{ .mbps = 1200,	.reg = 0x1b },
+	{ .mbps = 1250,	.reg = 0x2b },
+	{ .mbps = 1300,	.reg = 0x3b },
+	{ .mbps = 1350,	.reg = 0x0c },
+	{ .mbps = 1400,	.reg = 0x1c },
+	{ .mbps = 1450,	.reg = 0x2c },
+	{ .mbps = 1500,	.reg = 0x3c },
+	{ /* sentinel */ },
+};
+
+/* PHY ESC Error Monitor */
+#define PHEERM_REG			0x74
+
+/* PHY Clock Lane Monitor */
+#define PHCLM_REG			0x78
+#define PHCLM_STOPSTATECKL		BIT(0)
+
+/* PHY Data Lane Monitor */
+#define PHDLM_REG			0x7c
+
+/* CSI0CLK Frequency Configuration Preset Register */
+#define CSI0CLKFCPR_REG			0x260
+#define CSI0CLKFREQRANGE(n)		((n & 0x3f) << 16)
+
+struct rcar_csi2_format {
+	u32 code;
+	unsigned int datatype;
+	unsigned int bpp;
+};
+
+static const struct rcar_csi2_format rcar_csi2_formats[] = {
+	{ .code = MEDIA_BUS_FMT_RGB888_1X24,	.datatype = 0x24, .bpp = 24 },
+	{ .code = MEDIA_BUS_FMT_UYVY8_1X16,	.datatype = 0x1e, .bpp = 16 },
+	{ .code = MEDIA_BUS_FMT_YUYV8_1X16,	.datatype = 0x1e, .bpp = 16 },
+	{ .code = MEDIA_BUS_FMT_UYVY8_2X8,	.datatype = 0x1e, .bpp = 16 },
+	{ .code = MEDIA_BUS_FMT_YUYV10_2X10,	.datatype = 0x1e, .bpp = 20 },
+};
+
+static const struct rcar_csi2_format *rcsi2_code_to_fmt(unsigned int code)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(rcar_csi2_formats); i++)
+		if (rcar_csi2_formats[i].code == code)
+			return &rcar_csi2_formats[i];
+
+	return NULL;
+}
+
+enum rcar_csi2_pads {
+	RCAR_CSI2_SINK,
+	RCAR_CSI2_SOURCE_VC0,
+	RCAR_CSI2_SOURCE_VC1,
+	RCAR_CSI2_SOURCE_VC2,
+	RCAR_CSI2_SOURCE_VC3,
+	NR_OF_RCAR_CSI2_PAD,
+};
+
+struct rcar_csi2_info {
+	int (*init_phtw)(struct rcar_csi2 *priv, unsigned int mbps);
+	const struct rcsi2_mbps_reg *hsfreqrange;
+	unsigned int csi0clkfreqrange;
+	bool clear_ulps;
+};
+
+struct rcar_csi2 {
+	struct device *dev;
+	void __iomem *base;
+	const struct rcar_csi2_info *info;
+
+	struct v4l2_subdev subdev;
+	struct media_pad pads[NR_OF_RCAR_CSI2_PAD];
+
+	struct v4l2_async_notifier notifier;
+	struct v4l2_async_subdev asd;
+	struct v4l2_subdev *remote;
+
+	struct v4l2_mbus_framefmt mf;
+
+	struct mutex lock;
+	int stream_count;
+
+	unsigned short lanes;
+	unsigned char lane_swap[4];
+};
+
+static inline struct rcar_csi2 *sd_to_csi2(struct v4l2_subdev *sd)
+{
+	return container_of(sd, struct rcar_csi2, subdev);
+}
+
+static inline struct rcar_csi2 *notifier_to_csi2(struct v4l2_async_notifier *n)
+{
+	return container_of(n, struct rcar_csi2, notifier);
+}
+
+static u32 rcsi2_read(struct rcar_csi2 *priv, unsigned int reg)
+{
+	return ioread32(priv->base + reg);
+}
+
+static void rcsi2_write(struct rcar_csi2 *priv, unsigned int reg, u32 data)
+{
+	iowrite32(data, priv->base + reg);
+}
+
+static void rcsi2_reset(struct rcar_csi2 *priv)
+{
+	rcsi2_write(priv, SRST_REG, SRST_SRST);
+	usleep_range(100, 150);
+	rcsi2_write(priv, SRST_REG, 0);
+}
+
+static int rcsi2_wait_phy_start(struct rcar_csi2 *priv)
+{
+	unsigned int timeout;
+
+	/* Wait for the clock and data lanes to enter LP-11 state. */
+	for (timeout = 0; timeout <= 20; timeout++) {
+		const u32 lane_mask = (1 << priv->lanes) - 1;
+
+		if ((rcsi2_read(priv, PHCLM_REG) & PHCLM_STOPSTATECKL)  &&
+		    (rcsi2_read(priv, PHDLM_REG) & lane_mask) == lane_mask)
+			return 0;
+
+		usleep_range(1000, 2000);
+	}
+
+	dev_err(priv->dev, "Timeout waiting for LP-11 state\n");
+
+	return -ETIMEDOUT;
+}
+
+static int rcsi2_set_phypll(struct rcar_csi2 *priv, unsigned int mbps)
+{
+	const struct rcsi2_mbps_reg *hsfreq;
+
+	for (hsfreq = priv->info->hsfreqrange; hsfreq->mbps != 0; hsfreq++)
+		if (hsfreq->mbps >= mbps)
+			break;
+
+	if (!hsfreq->mbps) {
+		dev_err(priv->dev, "Unsupported PHY speed (%u Mbps)", mbps);
+		return -ERANGE;
+	}
+
+	rcsi2_write(priv, PHYPLL_REG, PHYPLL_HSFREQRANGE(hsfreq->reg));
+
+	return 0;
+}
+
+static int rcsi2_calc_mbps(struct rcar_csi2 *priv, unsigned int bpp)
+{
+	struct v4l2_subdev *source;
+	struct v4l2_ctrl *ctrl;
+	u64 mbps;
+
+	if (!priv->remote)
+		return -ENODEV;
+
+	source = priv->remote;
+
+	/* Read the pixel rate control from remote. */
+	ctrl = v4l2_ctrl_find(source->ctrl_handler, V4L2_CID_PIXEL_RATE);
+	if (!ctrl) {
+		dev_err(priv->dev, "no pixel rate control in subdev %s\n",
+			source->name);
+		return -EINVAL;
+	}
+
+	/*
+	 * Calculate the phypll in mbps.
+	 * link_freq = (pixel_rate * bits_per_sample) / (2 * nr_of_lanes)
+	 * bps = link_freq * 2
+	 */
+	mbps = v4l2_ctrl_g_ctrl_int64(ctrl) * bpp;
+	do_div(mbps, priv->lanes * 1000000);
+
+	return mbps;
+}
+
+static int rcsi2_start(struct rcar_csi2 *priv)
+{
+	const struct rcar_csi2_format *format;
+	u32 phycnt, vcdt = 0, vcdt2 = 0;
+	unsigned int i;
+	int mbps, ret;
+
+	dev_dbg(priv->dev, "Input size (%ux%u%c)\n",
+		priv->mf.width, priv->mf.height,
+		priv->mf.field == V4L2_FIELD_NONE ? 'p' : 'i');
+
+	/* Code is validated in set_fmt. */
+	format = rcsi2_code_to_fmt(priv->mf.code);
+
+	/*
+	 * Enable all Virtual Channels.
+	 *
+	 * NOTE: It's not possible to get individual datatype for each
+	 *       source virtual channel. Once this is possible in V4L2
+	 *       it should be used here.
+	 */
+	for (i = 0; i < 4; i++) {
+		u32 vcdt_part;
+
+		vcdt_part = VCDT_SEL_VC(i) | VCDT_VCDTN_EN | VCDT_SEL_DTN_ON |
+			VCDT_SEL_DT(format->datatype);
+
+		/* Store in correct reg and offset. */
+		if (i < 2)
+			vcdt |= vcdt_part << ((i % 2) * 16);
+		else
+			vcdt2 |= vcdt_part << ((i % 2) * 16);
+	}
+
+	phycnt = PHYCNT_ENABLECLK;
+	phycnt |= (1 << priv->lanes) - 1;
+
+	mbps = rcsi2_calc_mbps(priv, format->bpp);
+	if (mbps < 0)
+		return mbps;
+
+	/* Init */
+	rcsi2_write(priv, TREF_REG, TREF_TREF);
+	rcsi2_reset(priv);
+	rcsi2_write(priv, PHTC_REG, 0);
+
+	/* Configure */
+	rcsi2_write(priv, FLD_REG, FLD_FLD_NUM(2) | FLD_FLD_EN4 |
+		    FLD_FLD_EN3 | FLD_FLD_EN2 | FLD_FLD_EN);
+	rcsi2_write(priv, VCDT_REG, vcdt);
+	rcsi2_write(priv, VCDT2_REG, vcdt2);
+	/* Lanes are zero indexed. */
+	rcsi2_write(priv, LSWAP_REG,
+		    LSWAP_L0SEL(priv->lane_swap[0] - 1) |
+		    LSWAP_L1SEL(priv->lane_swap[1] - 1) |
+		    LSWAP_L2SEL(priv->lane_swap[2] - 1) |
+		    LSWAP_L3SEL(priv->lane_swap[3] - 1));
+
+	/* Start */
+	if (priv->info->init_phtw) {
+		ret = priv->info->init_phtw(priv, mbps);
+		if (ret)
+			return ret;
+	}
+
+	if (priv->info->hsfreqrange) {
+		ret = rcsi2_set_phypll(priv, mbps);
+		if (ret)
+			return ret;
+	}
+
+	if (priv->info->csi0clkfreqrange)
+		rcsi2_write(priv, CSI0CLKFCPR_REG,
+			    CSI0CLKFREQRANGE(priv->info->csi0clkfreqrange));
+
+	rcsi2_write(priv, PHYCNT_REG, phycnt);
+	rcsi2_write(priv, LINKCNT_REG, LINKCNT_MONITOR_EN |
+		    LINKCNT_REG_MONI_PACT_EN | LINKCNT_ICLK_NONSTOP);
+	rcsi2_write(priv, PHYCNT_REG, phycnt | PHYCNT_SHUTDOWNZ);
+	rcsi2_write(priv, PHYCNT_REG, phycnt | PHYCNT_SHUTDOWNZ | PHYCNT_RSTZ);
+
+	ret = rcsi2_wait_phy_start(priv);
+	if (ret)
+		return ret;
+
+	/* Clear Ultra Low Power interrupt. */
+	if (priv->info->clear_ulps)
+		rcsi2_write(priv, INTSTATE_REG,
+			    INTSTATE_INT_ULPS_START |
+			    INTSTATE_INT_ULPS_END);
+	return 0;
+}
+
+static void rcsi2_stop(struct rcar_csi2 *priv)
+{
+	rcsi2_write(priv, PHYCNT_REG, 0);
+
+	rcsi2_reset(priv);
+
+	rcsi2_write(priv, PHTC_REG, PHTC_TESTCLR);
+}
+
+static int rcsi2_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct rcar_csi2 *priv = sd_to_csi2(sd);
+	struct v4l2_subdev *nextsd;
+	int ret = 0;
+
+	mutex_lock(&priv->lock);
+
+	if (!priv->remote) {
+		ret = -ENODEV;
+		goto out;
+	}
+
+	nextsd = priv->remote;
+
+	if (enable && priv->stream_count == 0) {
+		pm_runtime_get_sync(priv->dev);
+
+		ret = rcsi2_start(priv);
+		if (ret) {
+			pm_runtime_put(priv->dev);
+			goto out;
+		}
+
+		ret = v4l2_subdev_call(nextsd, video, s_stream, 1);
+		if (ret) {
+			rcsi2_stop(priv);
+			pm_runtime_put(priv->dev);
+			goto out;
+		}
+	} else if (!enable && priv->stream_count == 1) {
+		rcsi2_stop(priv);
+		v4l2_subdev_call(nextsd, video, s_stream, 0);
+		pm_runtime_put(priv->dev);
+	}
+
+	priv->stream_count += enable ? 1 : -1;
+out:
+	mutex_unlock(&priv->lock);
+
+	return ret;
+}
+
+static int rcsi2_set_pad_format(struct v4l2_subdev *sd,
+				struct v4l2_subdev_pad_config *cfg,
+				struct v4l2_subdev_format *format)
+{
+	struct rcar_csi2 *priv = sd_to_csi2(sd);
+	struct v4l2_mbus_framefmt *framefmt;
+
+	if (!rcsi2_code_to_fmt(format->format.code))
+		format->format.code = rcar_csi2_formats[0].code;
+
+	if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+		priv->mf = format->format;
+	} else {
+		framefmt = v4l2_subdev_get_try_format(sd, cfg, 0);
+		*framefmt = format->format;
+	}
+
+	return 0;
+}
+
+static int rcsi2_get_pad_format(struct v4l2_subdev *sd,
+				struct v4l2_subdev_pad_config *cfg,
+				struct v4l2_subdev_format *format)
+{
+	struct rcar_csi2 *priv = sd_to_csi2(sd);
+
+	if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE)
+		format->format = priv->mf;
+	else
+		format->format = *v4l2_subdev_get_try_format(sd, cfg, 0);
+
+	return 0;
+}
+
+static const struct v4l2_subdev_video_ops rcar_csi2_video_ops = {
+	.s_stream = rcsi2_s_stream,
+};
+
+static const struct v4l2_subdev_pad_ops rcar_csi2_pad_ops = {
+	.set_fmt = rcsi2_set_pad_format,
+	.get_fmt = rcsi2_get_pad_format,
+};
+
+static const struct v4l2_subdev_ops rcar_csi2_subdev_ops = {
+	.video	= &rcar_csi2_video_ops,
+	.pad	= &rcar_csi2_pad_ops,
+};
+
+/* -----------------------------------------------------------------------------
+ * Async handling and registration of subdevices and links.
+ */
+
+static int rcsi2_notify_bound(struct v4l2_async_notifier *notifier,
+			      struct v4l2_subdev *subdev,
+			      struct v4l2_async_subdev *asd)
+{
+	struct rcar_csi2 *priv = notifier_to_csi2(notifier);
+	int pad;
+
+	pad = media_entity_get_fwnode_pad(&subdev->entity, asd->match.fwnode,
+					  MEDIA_PAD_FL_SOURCE);
+	if (pad < 0) {
+		dev_err(priv->dev, "Failed to find pad for %s\n", subdev->name);
+		return pad;
+	}
+
+	priv->remote = subdev;
+
+	dev_dbg(priv->dev, "Bound %s pad: %d\n", subdev->name, pad);
+
+	return media_create_pad_link(&subdev->entity, pad,
+				     &priv->subdev.entity, 0,
+				     MEDIA_LNK_FL_ENABLED |
+				     MEDIA_LNK_FL_IMMUTABLE);
+}
+
+static void rcsi2_notify_unbind(struct v4l2_async_notifier *notifier,
+				struct v4l2_subdev *subdev,
+				struct v4l2_async_subdev *asd)
+{
+	struct rcar_csi2 *priv = notifier_to_csi2(notifier);
+
+	priv->remote = NULL;
+
+	dev_dbg(priv->dev, "Unbind %s\n", subdev->name);
+}
+
+static const struct v4l2_async_notifier_operations rcar_csi2_notify_ops = {
+	.bound = rcsi2_notify_bound,
+	.unbind = rcsi2_notify_unbind,
+};
+
+static int rcsi2_parse_v4l2(struct rcar_csi2 *priv,
+			    struct v4l2_fwnode_endpoint *vep)
+{
+	unsigned int i;
+
+	/* Only port 0 endpoint 0 is valid. */
+	if (vep->base.port || vep->base.id)
+		return -ENOTCONN;
+
+	if (vep->bus_type != V4L2_MBUS_CSI2) {
+		dev_err(priv->dev, "Unsupported bus: %u\n", vep->bus_type);
+		return -EINVAL;
+	}
+
+	priv->lanes = vep->bus.mipi_csi2.num_data_lanes;
+	if (priv->lanes != 1 && priv->lanes != 2 && priv->lanes != 4) {
+		dev_err(priv->dev, "Unsupported number of data-lanes: %u\n",
+			priv->lanes);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(priv->lane_swap); i++) {
+		priv->lane_swap[i] = i < priv->lanes ?
+			vep->bus.mipi_csi2.data_lanes[i] : i;
+
+		/* Check for valid lane number. */
+		if (priv->lane_swap[i] < 1 || priv->lane_swap[i] > 4) {
+			dev_err(priv->dev, "data-lanes must be in 1-4 range\n");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int rcsi2_parse_dt(struct rcar_csi2 *priv)
+{
+	struct device_node *ep;
+	struct v4l2_fwnode_endpoint v4l2_ep;
+	int ret;
+
+	ep = of_graph_get_endpoint_by_regs(priv->dev->of_node, 0, 0);
+	if (!ep) {
+		dev_err(priv->dev, "Not connected to subdevice\n");
+		return -EINVAL;
+	}
+
+	ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep), &v4l2_ep);
+	if (ret) {
+		dev_err(priv->dev, "Could not parse v4l2 endpoint\n");
+		of_node_put(ep);
+		return -EINVAL;
+	}
+
+	ret = rcsi2_parse_v4l2(priv, &v4l2_ep);
+	if (ret) {
+		of_node_put(ep);
+		return ret;
+	}
+
+	priv->asd.match.fwnode =
+		fwnode_graph_get_remote_endpoint(of_fwnode_handle(ep));
+	priv->asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
+
+	of_node_put(ep);
+
+	priv->notifier.subdevs = devm_kzalloc(priv->dev,
+					      sizeof(*priv->notifier.subdevs),
+					      GFP_KERNEL);
+	if (!priv->notifier.subdevs)
+		return -ENOMEM;
+
+	priv->notifier.num_subdevs = 1;
+	priv->notifier.subdevs[0] = &priv->asd;
+	priv->notifier.ops = &rcar_csi2_notify_ops;
+
+	dev_dbg(priv->dev, "Found '%pOF'\n",
+		to_of_node(priv->asd.match.fwnode));
+
+	return v4l2_async_subdev_notifier_register(&priv->subdev,
+						   &priv->notifier);
+}
+
+/* -----------------------------------------------------------------------------
+ * PHTW initialization sequences.
+ *
+ * NOTE: Magic values are from the datasheet and lack documentation.
+ */
+
+static int rcsi2_phtw_write(struct rcar_csi2 *priv, u16 data, u16 code)
+{
+	unsigned int timeout;
+
+	rcsi2_write(priv, PHTW_REG,
+		    PHTW_DWEN | PHTW_TESTDIN_DATA(data) |
+		    PHTW_CWEN | PHTW_TESTDIN_CODE(code));
+
+	/* Wait for DWEN and CWEN to be cleared by hardware. */
+	for (timeout = 0; timeout <= 20; timeout++) {
+		if (!(rcsi2_read(priv, PHTW_REG) & (PHTW_DWEN | PHTW_CWEN)))
+			return 0;
+
+		usleep_range(1000, 2000);
+	}
+
+	dev_err(priv->dev, "Timeout waiting for PHTW_DWEN and/or PHTW_CWEN\n");
+
+	return -ETIMEDOUT;
+}
+
+static int rcsi2_phtw_write_array(struct rcar_csi2 *priv,
+				  const struct phtw_value *values)
+{
+	const struct phtw_value *value;
+	int ret;
+
+	for (value = values; value->data || value->code; value++) {
+		ret = rcsi2_phtw_write(priv, value->data, value->code);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int rcsi2_phtw_write_mbps(struct rcar_csi2 *priv, unsigned int mbps,
+				 const struct rcsi2_mbps_reg *values, u16 code)
+{
+	const struct rcsi2_mbps_reg *value;
+
+	for (value = values; value->mbps; value++)
+		if (value->mbps >= mbps)
+			break;
+
+	if (!value->mbps) {
+		dev_err(priv->dev, "Unsupported PHY speed (%u Mbps)", mbps);
+		return -ERANGE;
+	}
+
+	return rcsi2_phtw_write(priv, value->reg, code);
+}
+
+static int rcsi2_init_phtw_h3_v3h_m3n(struct rcar_csi2 *priv, unsigned int mbps)
+{
+	static const struct phtw_value step1[] = {
+		{ .data = 0xcc, .code = 0xe2 },
+		{ .data = 0x01, .code = 0xe3 },
+		{ .data = 0x11, .code = 0xe4 },
+		{ .data = 0x01, .code = 0xe5 },
+		{ .data = 0x10, .code = 0x04 },
+		{ /* sentinel */ },
+	};
+
+	static const struct phtw_value step2[] = {
+		{ .data = 0x38, .code = 0x08 },
+		{ .data = 0x01, .code = 0x00 },
+		{ .data = 0x4b, .code = 0xac },
+		{ .data = 0x03, .code = 0x00 },
+		{ .data = 0x80, .code = 0x07 },
+		{ /* sentinel */ },
+	};
+
+	int ret;
+
+	ret = rcsi2_phtw_write_array(priv, step1);
+	if (ret)
+		return ret;
+
+	if (mbps <= 250) {
+		ret = rcsi2_phtw_write(priv, 0x39, 0x05);
+		if (ret)
+			return ret;
+
+		ret = rcsi2_phtw_write_mbps(priv, mbps, phtw_mbps_h3_v3h_m3n,
+					    0xf1);
+		if (ret)
+			return ret;
+	}
+
+	return rcsi2_phtw_write_array(priv, step2);
+}
+
+static int rcsi2_init_phtw_v3m_e3(struct rcar_csi2 *priv, unsigned int mbps)
+{
+	static const struct phtw_value step1[] = {
+		{ .data = 0xed, .code = 0x34 },
+		{ .data = 0xed, .code = 0x44 },
+		{ .data = 0xed, .code = 0x54 },
+		{ .data = 0xed, .code = 0x84 },
+		{ .data = 0xed, .code = 0x94 },
+		{ /* sentinel */ },
+	};
+
+	int ret;
+
+	ret = rcsi2_phtw_write_mbps(priv, mbps, phtw_mbps_v3m_e3, 0x44);
+	if (ret)
+		return ret;
+
+	return rcsi2_phtw_write_array(priv, step1);
+}
+
+/* -----------------------------------------------------------------------------
+ * Platform Device Driver.
+ */
+
+static const struct media_entity_operations rcar_csi2_entity_ops = {
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+static int rcsi2_probe_resources(struct rcar_csi2 *priv,
+				 struct platform_device *pdev)
+{
+	struct resource *res;
+	int irq;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(priv->base))
+		return PTR_ERR(priv->base);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	return 0;
+}
+
+static const struct rcar_csi2_info rcar_csi2_info_r8a7795 = {
+	.init_phtw = rcsi2_init_phtw_h3_v3h_m3n,
+	.hsfreqrange = hsfreqrange_h3_v3h_m3n,
+	.csi0clkfreqrange = 0x20,
+	.clear_ulps = true,
+};
+
+static const struct rcar_csi2_info rcar_csi2_info_r8a7795es1 = {
+	.hsfreqrange = hsfreqrange_m3w_h3es1,
+};
+
+static const struct rcar_csi2_info rcar_csi2_info_r8a7796 = {
+	.hsfreqrange = hsfreqrange_m3w_h3es1,
+};
+
+static const struct rcar_csi2_info rcar_csi2_info_r8a77965 = {
+	.init_phtw = rcsi2_init_phtw_h3_v3h_m3n,
+	.hsfreqrange = hsfreqrange_h3_v3h_m3n,
+	.csi0clkfreqrange = 0x20,
+	.clear_ulps = true,
+};
+
+static const struct rcar_csi2_info rcar_csi2_info_r8a77970 = {
+	.init_phtw = rcsi2_init_phtw_v3m_e3,
+};
+
+static const struct of_device_id rcar_csi2_of_table[] = {
+	{
+		.compatible = "renesas,r8a7795-csi2",
+		.data = &rcar_csi2_info_r8a7795,
+	},
+	{
+		.compatible = "renesas,r8a7796-csi2",
+		.data = &rcar_csi2_info_r8a7796,
+	},
+	{
+		.compatible = "renesas,r8a77965-csi2",
+		.data = &rcar_csi2_info_r8a77965,
+	},
+	{
+		.compatible = "renesas,r8a77970-csi2",
+		.data = &rcar_csi2_info_r8a77970,
+	},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, rcar_csi2_of_table);
+
+static const struct soc_device_attribute r8a7795es1[] = {
+	{
+		.soc_id = "r8a7795", .revision = "ES1.*",
+		.data = &rcar_csi2_info_r8a7795es1,
+	},
+	{ /* sentinel */ },
+};
+
+static int rcsi2_probe(struct platform_device *pdev)
+{
+	const struct soc_device_attribute *attr;
+	struct rcar_csi2 *priv;
+	unsigned int i;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->info = of_device_get_match_data(&pdev->dev);
+
+	/*
+	 * r8a7795 ES1.x behaves differently than the ES2.0+ but doesn't
+	 * have it's own compatible string.
+	 */
+	attr = soc_device_match(r8a7795es1);
+	if (attr)
+		priv->info = attr->data;
+
+	priv->dev = &pdev->dev;
+
+	mutex_init(&priv->lock);
+	priv->stream_count = 0;
+
+	ret = rcsi2_probe_resources(priv, pdev);
+	if (ret) {
+		dev_err(priv->dev, "Failed to get resources\n");
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, priv);
+
+	ret = rcsi2_parse_dt(priv);
+	if (ret)
+		return ret;
+
+	priv->subdev.owner = THIS_MODULE;
+	priv->subdev.dev = &pdev->dev;
+	v4l2_subdev_init(&priv->subdev, &rcar_csi2_subdev_ops);
+	v4l2_set_subdevdata(&priv->subdev, &pdev->dev);
+	snprintf(priv->subdev.name, V4L2_SUBDEV_NAME_SIZE, "%s %s",
+		 KBUILD_MODNAME, dev_name(&pdev->dev));
+	priv->subdev.flags = V4L2_SUBDEV_FL_HAS_DEVNODE;
+
+	priv->subdev.entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;
+	priv->subdev.entity.ops = &rcar_csi2_entity_ops;
+
+	priv->pads[RCAR_CSI2_SINK].flags = MEDIA_PAD_FL_SINK;
+	for (i = RCAR_CSI2_SOURCE_VC0; i < NR_OF_RCAR_CSI2_PAD; i++)
+		priv->pads[i].flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&priv->subdev.entity, NR_OF_RCAR_CSI2_PAD,
+				     priv->pads);
+	if (ret)
+		goto error;
+
+	pm_runtime_enable(&pdev->dev);
+
+	ret = v4l2_async_register_subdev(&priv->subdev);
+	if (ret < 0)
+		goto error;
+
+	dev_info(priv->dev, "%d lanes found\n", priv->lanes);
+
+	return 0;
+
+error:
+	v4l2_async_notifier_unregister(&priv->notifier);
+	v4l2_async_notifier_cleanup(&priv->notifier);
+
+	return ret;
+}
+
+static int rcsi2_remove(struct platform_device *pdev)
+{
+	struct rcar_csi2 *priv = platform_get_drvdata(pdev);
+
+	v4l2_async_notifier_unregister(&priv->notifier);
+	v4l2_async_notifier_cleanup(&priv->notifier);
+	v4l2_async_unregister_subdev(&priv->subdev);
+
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static struct platform_driver rcar_csi2_pdrv = {
+	.remove	= rcsi2_remove,
+	.probe	= rcsi2_probe,
+	.driver	= {
+		.name	= "rcar-csi2",
+		.of_match_table	= rcar_csi2_of_table,
+	},
+};
+
+module_platform_driver(rcar_csi2_pdrv);
+
+MODULE_AUTHOR("Niklas Söderlund <niklas.soderlund@ragnatech.se>");
+MODULE_DESCRIPTION("Renesas R-Car MIPI CSI-2 receiver driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/rcar-vin/rcar-dma.c b/drivers/media/platform/rcar-vin/rcar-dma.c
index 4a40e6ad1be7..ac07f99e3516 100644
--- a/drivers/media/platform/rcar-vin/rcar-dma.c
+++ b/drivers/media/platform/rcar-vin/rcar-dma.c
@@ -16,6 +16,7 @@
 
 #include <linux/delay.h>
 #include <linux/interrupt.h>
+#include <linux/pm_runtime.h>
 
 #include <media/videobuf2-dma-contig.h>
 
@@ -33,21 +34,23 @@
 #define VNELPRC_REG	0x10	/* Video n End Line Pre-Clip Register */
 #define VNSPPRC_REG	0x14	/* Video n Start Pixel Pre-Clip Register */
 #define VNEPPRC_REG	0x18	/* Video n End Pixel Pre-Clip Register */
-#define VNSLPOC_REG	0x1C	/* Video n Start Line Post-Clip Register */
-#define VNELPOC_REG	0x20	/* Video n End Line Post-Clip Register */
-#define VNSPPOC_REG	0x24	/* Video n Start Pixel Post-Clip Register */
-#define VNEPPOC_REG	0x28	/* Video n End Pixel Post-Clip Register */
 #define VNIS_REG	0x2C	/* Video n Image Stride Register */
 #define VNMB_REG(m)	(0x30 + ((m) << 2)) /* Video n Memory Base m Register */
 #define VNIE_REG	0x40	/* Video n Interrupt Enable Register */
 #define VNINTS_REG	0x44	/* Video n Interrupt Status Register */
 #define VNSI_REG	0x48	/* Video n Scanline Interrupt Register */
 #define VNMTC_REG	0x4C	/* Video n Memory Transfer Control Register */
-#define VNYS_REG	0x50	/* Video n Y Scale Register */
-#define VNXS_REG	0x54	/* Video n X Scale Register */
 #define VNDMR_REG	0x58	/* Video n Data Mode Register */
 #define VNDMR2_REG	0x5C	/* Video n Data Mode Register 2 */
 #define VNUVAOF_REG	0x60	/* Video n UV Address Offset Register */
+
+/* Register offsets specific for Gen2 */
+#define VNSLPOC_REG	0x1C	/* Video n Start Line Post-Clip Register */
+#define VNELPOC_REG	0x20	/* Video n End Line Post-Clip Register */
+#define VNSPPOC_REG	0x24	/* Video n Start Pixel Post-Clip Register */
+#define VNEPPOC_REG	0x28	/* Video n End Pixel Post-Clip Register */
+#define VNYS_REG	0x50	/* Video n Y Scale Register */
+#define VNXS_REG	0x54	/* Video n X Scale Register */
 #define VNC1A_REG	0x80	/* Video n Coefficient Set C1A Register */
 #define VNC1B_REG	0x84	/* Video n Coefficient Set C1B Register */
 #define VNC1C_REG	0x88	/* Video n Coefficient Set C1C Register */
@@ -73,9 +76,13 @@
 #define VNC8B_REG	0xF4	/* Video n Coefficient Set C8B Register */
 #define VNC8C_REG	0xF8	/* Video n Coefficient Set C8C Register */
 
+/* Register offsets specific for Gen3 */
+#define VNCSI_IFMD_REG		0x20 /* Video n CSI2 Interface Mode Register */
 
 /* Register bit fields for R-Car VIN */
 /* Video n Main Control Register bits */
+#define VNMC_DPINE		(1 << 27) /* Gen3 specific */
+#define VNMC_SCLE		(1 << 26) /* Gen3 specific */
 #define VNMC_FOC		(1 << 21)
 #define VNMC_YCAL		(1 << 19)
 #define VNMC_INF_YUV8_BT656	(0 << 16)
@@ -119,6 +126,12 @@
 #define VNDMR2_FTEV		(1 << 17)
 #define VNDMR2_VLV(n)		((n & 0xf) << 12)
 
+/* Video n CSI2 Interface Mode Register (Gen3) */
+#define VNCSI_IFMD_DES1		(1 << 26)
+#define VNCSI_IFMD_DES0		(1 << 25)
+#define VNCSI_IFMD_CSI_CHSEL(n) (((n) & 0xf) << 0)
+#define VNCSI_IFMD_CSI_CHSEL_MASK 0xf
+
 struct rvin_buffer {
 	struct vb2_v4l2_buffer vb;
 	struct list_head list;
@@ -138,267 +151,6 @@ static u32 rvin_read(struct rvin_dev *vin, u32 offset)
 	return ioread32(vin->base + offset);
 }
 
-static int rvin_setup(struct rvin_dev *vin)
-{
-	u32 vnmc, dmr, dmr2, interrupts;
-	v4l2_std_id std;
-	bool progressive = false, output_is_yuv = false, input_is_yuv = false;
-
-	switch (vin->format.field) {
-	case V4L2_FIELD_TOP:
-		vnmc = VNMC_IM_ODD;
-		break;
-	case V4L2_FIELD_BOTTOM:
-		vnmc = VNMC_IM_EVEN;
-		break;
-	case V4L2_FIELD_INTERLACED:
-		/* Default to TB */
-		vnmc = VNMC_IM_FULL;
-		/* Use BT if video standard can be read and is 60 Hz format */
-		if (!v4l2_subdev_call(vin_to_source(vin), video, g_std, &std)) {
-			if (std & V4L2_STD_525_60)
-				vnmc = VNMC_IM_FULL | VNMC_FOC;
-		}
-		break;
-	case V4L2_FIELD_INTERLACED_TB:
-		vnmc = VNMC_IM_FULL;
-		break;
-	case V4L2_FIELD_INTERLACED_BT:
-		vnmc = VNMC_IM_FULL | VNMC_FOC;
-		break;
-	case V4L2_FIELD_ALTERNATE:
-	case V4L2_FIELD_NONE:
-		vnmc = VNMC_IM_ODD_EVEN;
-		progressive = true;
-		break;
-	default:
-		vnmc = VNMC_IM_ODD;
-		break;
-	}
-
-	/*
-	 * Input interface
-	 */
-	switch (vin->digital->code) {
-	case MEDIA_BUS_FMT_YUYV8_1X16:
-		/* BT.601/BT.1358 16bit YCbCr422 */
-		vnmc |= VNMC_INF_YUV16;
-		input_is_yuv = true;
-		break;
-	case MEDIA_BUS_FMT_UYVY8_2X8:
-		/* BT.656 8bit YCbCr422 or BT.601 8bit YCbCr422 */
-		vnmc |= vin->digital->mbus_cfg.type == V4L2_MBUS_BT656 ?
-			VNMC_INF_YUV8_BT656 : VNMC_INF_YUV8_BT601;
-		input_is_yuv = true;
-		break;
-	case MEDIA_BUS_FMT_RGB888_1X24:
-		vnmc |= VNMC_INF_RGB888;
-		break;
-	case MEDIA_BUS_FMT_UYVY10_2X10:
-		/* BT.656 10bit YCbCr422 or BT.601 10bit YCbCr422 */
-		vnmc |= vin->digital->mbus_cfg.type == V4L2_MBUS_BT656 ?
-			VNMC_INF_YUV10_BT656 : VNMC_INF_YUV10_BT601;
-		input_is_yuv = true;
-		break;
-	default:
-		break;
-	}
-
-	/* Enable VSYNC Field Toogle mode after one VSYNC input */
-	dmr2 = VNDMR2_FTEV | VNDMR2_VLV(1);
-
-	/* Hsync Signal Polarity Select */
-	if (!(vin->digital->mbus_cfg.flags & V4L2_MBUS_HSYNC_ACTIVE_LOW))
-		dmr2 |= VNDMR2_HPS;
-
-	/* Vsync Signal Polarity Select */
-	if (!(vin->digital->mbus_cfg.flags & V4L2_MBUS_VSYNC_ACTIVE_LOW))
-		dmr2 |= VNDMR2_VPS;
-
-	/*
-	 * Output format
-	 */
-	switch (vin->format.pixelformat) {
-	case V4L2_PIX_FMT_NV16:
-		rvin_write(vin,
-			   ALIGN(vin->format.width * vin->format.height, 0x80),
-			   VNUVAOF_REG);
-		dmr = VNDMR_DTMD_YCSEP;
-		output_is_yuv = true;
-		break;
-	case V4L2_PIX_FMT_YUYV:
-		dmr = VNDMR_BPSM;
-		output_is_yuv = true;
-		break;
-	case V4L2_PIX_FMT_UYVY:
-		dmr = 0;
-		output_is_yuv = true;
-		break;
-	case V4L2_PIX_FMT_XRGB555:
-		dmr = VNDMR_DTMD_ARGB1555;
-		break;
-	case V4L2_PIX_FMT_RGB565:
-		dmr = 0;
-		break;
-	case V4L2_PIX_FMT_XBGR32:
-		/* Note: not supported on M1 */
-		dmr = VNDMR_EXRGB;
-		break;
-	default:
-		vin_err(vin, "Invalid pixelformat (0x%x)\n",
-			vin->format.pixelformat);
-		return -EINVAL;
-	}
-
-	/* Always update on field change */
-	vnmc |= VNMC_VUP;
-
-	/* If input and output use the same colorspace, use bypass mode */
-	if (input_is_yuv == output_is_yuv)
-		vnmc |= VNMC_BPS;
-
-	/* Progressive or interlaced mode */
-	interrupts = progressive ? VNIE_FIE : VNIE_EFE;
-
-	/* Ack interrupts */
-	rvin_write(vin, interrupts, VNINTS_REG);
-	/* Enable interrupts */
-	rvin_write(vin, interrupts, VNIE_REG);
-	/* Start capturing */
-	rvin_write(vin, dmr, VNDMR_REG);
-	rvin_write(vin, dmr2, VNDMR2_REG);
-
-	/* Enable module */
-	rvin_write(vin, vnmc | VNMC_ME, VNMC_REG);
-
-	return 0;
-}
-
-static void rvin_disable_interrupts(struct rvin_dev *vin)
-{
-	rvin_write(vin, 0, VNIE_REG);
-}
-
-static u32 rvin_get_interrupt_status(struct rvin_dev *vin)
-{
-	return rvin_read(vin, VNINTS_REG);
-}
-
-static void rvin_ack_interrupt(struct rvin_dev *vin)
-{
-	rvin_write(vin, rvin_read(vin, VNINTS_REG), VNINTS_REG);
-}
-
-static bool rvin_capture_active(struct rvin_dev *vin)
-{
-	return rvin_read(vin, VNMS_REG) & VNMS_CA;
-}
-
-static enum v4l2_field rvin_get_active_field(struct rvin_dev *vin, u32 vnms)
-{
-	if (vin->format.field == V4L2_FIELD_ALTERNATE) {
-		/* If FS is set it's a Even field */
-		if (vnms & VNMS_FS)
-			return V4L2_FIELD_BOTTOM;
-		return V4L2_FIELD_TOP;
-	}
-
-	return vin->format.field;
-}
-
-static void rvin_set_slot_addr(struct rvin_dev *vin, int slot, dma_addr_t addr)
-{
-	const struct rvin_video_format *fmt;
-	int offsetx, offsety;
-	dma_addr_t offset;
-
-	fmt = rvin_format_from_pixel(vin->format.pixelformat);
-
-	/*
-	 * There is no HW support for composition do the beast we can
-	 * by modifying the buffer offset
-	 */
-	offsetx = vin->compose.left * fmt->bpp;
-	offsety = vin->compose.top * vin->format.bytesperline;
-	offset = addr + offsetx + offsety;
-
-	/*
-	 * The address needs to be 128 bytes aligned. Driver should never accept
-	 * settings that do not satisfy this in the first place...
-	 */
-	if (WARN_ON((offsetx | offsety | offset) & HW_BUFFER_MASK))
-		return;
-
-	rvin_write(vin, offset, VNMB_REG(slot));
-}
-
-/*
- * Moves a buffer from the queue to the HW slot. If no buffer is
- * available use the scratch buffer. The scratch buffer is never
- * returned to userspace, its only function is to enable the capture
- * loop to keep running.
- */
-static void rvin_fill_hw_slot(struct rvin_dev *vin, int slot)
-{
-	struct rvin_buffer *buf;
-	struct vb2_v4l2_buffer *vbuf;
-	dma_addr_t phys_addr;
-
-	/* A already populated slot shall never be overwritten. */
-	if (WARN_ON(vin->queue_buf[slot] != NULL))
-		return;
-
-	vin_dbg(vin, "Filling HW slot: %d\n", slot);
-
-	if (list_empty(&vin->buf_list)) {
-		vin->queue_buf[slot] = NULL;
-		phys_addr = vin->scratch_phys;
-	} else {
-		/* Keep track of buffer we give to HW */
-		buf = list_entry(vin->buf_list.next, struct rvin_buffer, list);
-		vbuf = &buf->vb;
-		list_del_init(to_buf_list(vbuf));
-		vin->queue_buf[slot] = vbuf;
-
-		/* Setup DMA */
-		phys_addr = vb2_dma_contig_plane_dma_addr(&vbuf->vb2_buf, 0);
-	}
-
-	rvin_set_slot_addr(vin, slot, phys_addr);
-}
-
-static int rvin_capture_start(struct rvin_dev *vin)
-{
-	int slot, ret;
-
-	for (slot = 0; slot < HW_BUFFER_NUM; slot++)
-		rvin_fill_hw_slot(vin, slot);
-
-	rvin_crop_scale_comp(vin);
-
-	ret = rvin_setup(vin);
-	if (ret)
-		return ret;
-
-	vin_dbg(vin, "Starting to capture\n");
-
-	/* Continuous Frame Capture Mode */
-	rvin_write(vin, VNFC_C_FRAME, VNFC_REG);
-
-	vin->state = RUNNING;
-
-	return 0;
-}
-
-static void rvin_capture_stop(struct rvin_dev *vin)
-{
-	/* Set continuous & single transfer off */
-	rvin_write(vin, 0, VNFC_REG);
-
-	/* Disable module */
-	rvin_write(vin, rvin_read(vin, VNMC_REG) & ~VNMC_ME, VNMC_REG);
-}
-
 /* -----------------------------------------------------------------------------
  * Crop and Scaling Gen2
  */
@@ -736,122 +488,386 @@ static void rvin_set_coeff(struct rvin_dev *vin, unsigned short xs)
 			break;
 	}
 
-	/* Use previous value if its XS value is closer */
-	if (p_prev_set && p_set &&
-	    xs - p_prev_set->xs_value < p_set->xs_value - xs)
-		p_set = p_prev_set;
+	/* Use previous value if its XS value is closer */
+	if (p_prev_set && p_set &&
+	    xs - p_prev_set->xs_value < p_set->xs_value - xs)
+		p_set = p_prev_set;
+
+	/* Set coefficient registers */
+	rvin_write(vin, p_set->coeff_set[0], VNC1A_REG);
+	rvin_write(vin, p_set->coeff_set[1], VNC1B_REG);
+	rvin_write(vin, p_set->coeff_set[2], VNC1C_REG);
+
+	rvin_write(vin, p_set->coeff_set[3], VNC2A_REG);
+	rvin_write(vin, p_set->coeff_set[4], VNC2B_REG);
+	rvin_write(vin, p_set->coeff_set[5], VNC2C_REG);
+
+	rvin_write(vin, p_set->coeff_set[6], VNC3A_REG);
+	rvin_write(vin, p_set->coeff_set[7], VNC3B_REG);
+	rvin_write(vin, p_set->coeff_set[8], VNC3C_REG);
+
+	rvin_write(vin, p_set->coeff_set[9], VNC4A_REG);
+	rvin_write(vin, p_set->coeff_set[10], VNC4B_REG);
+	rvin_write(vin, p_set->coeff_set[11], VNC4C_REG);
+
+	rvin_write(vin, p_set->coeff_set[12], VNC5A_REG);
+	rvin_write(vin, p_set->coeff_set[13], VNC5B_REG);
+	rvin_write(vin, p_set->coeff_set[14], VNC5C_REG);
+
+	rvin_write(vin, p_set->coeff_set[15], VNC6A_REG);
+	rvin_write(vin, p_set->coeff_set[16], VNC6B_REG);
+	rvin_write(vin, p_set->coeff_set[17], VNC6C_REG);
+
+	rvin_write(vin, p_set->coeff_set[18], VNC7A_REG);
+	rvin_write(vin, p_set->coeff_set[19], VNC7B_REG);
+	rvin_write(vin, p_set->coeff_set[20], VNC7C_REG);
+
+	rvin_write(vin, p_set->coeff_set[21], VNC8A_REG);
+	rvin_write(vin, p_set->coeff_set[22], VNC8B_REG);
+	rvin_write(vin, p_set->coeff_set[23], VNC8C_REG);
+}
+
+static void rvin_crop_scale_comp_gen2(struct rvin_dev *vin)
+{
+	u32 xs, ys;
+
+	/* Set scaling coefficient */
+	ys = 0;
+	if (vin->crop.height != vin->compose.height)
+		ys = (4096 * vin->crop.height) / vin->compose.height;
+	rvin_write(vin, ys, VNYS_REG);
+
+	xs = 0;
+	if (vin->crop.width != vin->compose.width)
+		xs = (4096 * vin->crop.width) / vin->compose.width;
+
+	/* Horizontal upscaling is up to double size */
+	if (xs > 0 && xs < 2048)
+		xs = 2048;
+
+	rvin_write(vin, xs, VNXS_REG);
+
+	/* Horizontal upscaling is done out by scaling down from double size */
+	if (xs < 4096)
+		xs *= 2;
+
+	rvin_set_coeff(vin, xs);
+
+	/* Set Start/End Pixel/Line Post-Clip */
+	rvin_write(vin, 0, VNSPPOC_REG);
+	rvin_write(vin, 0, VNSLPOC_REG);
+	rvin_write(vin, vin->format.width - 1, VNEPPOC_REG);
+	switch (vin->format.field) {
+	case V4L2_FIELD_INTERLACED:
+	case V4L2_FIELD_INTERLACED_TB:
+	case V4L2_FIELD_INTERLACED_BT:
+		rvin_write(vin, vin->format.height / 2 - 1, VNELPOC_REG);
+		break;
+	default:
+		rvin_write(vin, vin->format.height - 1, VNELPOC_REG);
+		break;
+	}
+
+	vin_dbg(vin,
+		"Pre-Clip: %ux%u@%u:%u YS: %d XS: %d Post-Clip: %ux%u@%u:%u\n",
+		vin->crop.width, vin->crop.height, vin->crop.left,
+		vin->crop.top, ys, xs, vin->format.width, vin->format.height,
+		0, 0);
+}
+
+void rvin_crop_scale_comp(struct rvin_dev *vin)
+{
+	/* Set Start/End Pixel/Line Pre-Clip */
+	rvin_write(vin, vin->crop.left, VNSPPRC_REG);
+	rvin_write(vin, vin->crop.left + vin->crop.width - 1, VNEPPRC_REG);
+
+	switch (vin->format.field) {
+	case V4L2_FIELD_INTERLACED:
+	case V4L2_FIELD_INTERLACED_TB:
+	case V4L2_FIELD_INTERLACED_BT:
+		rvin_write(vin, vin->crop.top / 2, VNSLPRC_REG);
+		rvin_write(vin, (vin->crop.top + vin->crop.height) / 2 - 1,
+			   VNELPRC_REG);
+		break;
+	default:
+		rvin_write(vin, vin->crop.top, VNSLPRC_REG);
+		rvin_write(vin, vin->crop.top + vin->crop.height - 1,
+			   VNELPRC_REG);
+		break;
+	}
+
+	/* TODO: Add support for the UDS scaler. */
+	if (vin->info->model != RCAR_GEN3)
+		rvin_crop_scale_comp_gen2(vin);
+
+	if (vin->format.pixelformat == V4L2_PIX_FMT_NV16)
+		rvin_write(vin, ALIGN(vin->format.width, 0x20), VNIS_REG);
+	else
+		rvin_write(vin, ALIGN(vin->format.width, 0x10), VNIS_REG);
+}
+
+/* -----------------------------------------------------------------------------
+ * Hardware setup
+ */
+
+static int rvin_setup(struct rvin_dev *vin)
+{
+	u32 vnmc, dmr, dmr2, interrupts;
+	bool progressive = false, output_is_yuv = false, input_is_yuv = false;
+
+	switch (vin->format.field) {
+	case V4L2_FIELD_TOP:
+		vnmc = VNMC_IM_ODD;
+		break;
+	case V4L2_FIELD_BOTTOM:
+		vnmc = VNMC_IM_EVEN;
+		break;
+	case V4L2_FIELD_INTERLACED:
+		/* Default to TB */
+		vnmc = VNMC_IM_FULL;
+		/* Use BT if video standard can be read and is 60 Hz format */
+		if (!vin->info->use_mc && vin->std & V4L2_STD_525_60)
+			vnmc = VNMC_IM_FULL | VNMC_FOC;
+		break;
+	case V4L2_FIELD_INTERLACED_TB:
+		vnmc = VNMC_IM_FULL;
+		break;
+	case V4L2_FIELD_INTERLACED_BT:
+		vnmc = VNMC_IM_FULL | VNMC_FOC;
+		break;
+	case V4L2_FIELD_NONE:
+		vnmc = VNMC_IM_ODD_EVEN;
+		progressive = true;
+		break;
+	default:
+		vnmc = VNMC_IM_ODD;
+		break;
+	}
+
+	/*
+	 * Input interface
+	 */
+	switch (vin->mbus_code) {
+	case MEDIA_BUS_FMT_YUYV8_1X16:
+		/* BT.601/BT.1358 16bit YCbCr422 */
+		vnmc |= VNMC_INF_YUV16;
+		input_is_yuv = true;
+		break;
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		vnmc |= VNMC_INF_YUV16 | VNMC_YCAL;
+		input_is_yuv = true;
+		break;
+	case MEDIA_BUS_FMT_UYVY8_2X8:
+		/* BT.656 8bit YCbCr422 or BT.601 8bit YCbCr422 */
+		vnmc |= vin->mbus_cfg.type == V4L2_MBUS_BT656 ?
+			VNMC_INF_YUV8_BT656 : VNMC_INF_YUV8_BT601;
+		input_is_yuv = true;
+		break;
+	case MEDIA_BUS_FMT_RGB888_1X24:
+		vnmc |= VNMC_INF_RGB888;
+		break;
+	case MEDIA_BUS_FMT_UYVY10_2X10:
+		/* BT.656 10bit YCbCr422 or BT.601 10bit YCbCr422 */
+		vnmc |= vin->mbus_cfg.type == V4L2_MBUS_BT656 ?
+			VNMC_INF_YUV10_BT656 : VNMC_INF_YUV10_BT601;
+		input_is_yuv = true;
+		break;
+	default:
+		break;
+	}
+
+	/* Enable VSYNC Field Toogle mode after one VSYNC input */
+	if (vin->info->model == RCAR_GEN3)
+		dmr2 = VNDMR2_FTEV;
+	else
+		dmr2 = VNDMR2_FTEV | VNDMR2_VLV(1);
+
+	/* Hsync Signal Polarity Select */
+	if (!(vin->mbus_cfg.flags & V4L2_MBUS_HSYNC_ACTIVE_LOW))
+		dmr2 |= VNDMR2_HPS;
+
+	/* Vsync Signal Polarity Select */
+	if (!(vin->mbus_cfg.flags & V4L2_MBUS_VSYNC_ACTIVE_LOW))
+		dmr2 |= VNDMR2_VPS;
+
+	/*
+	 * Output format
+	 */
+	switch (vin->format.pixelformat) {
+	case V4L2_PIX_FMT_NV16:
+		rvin_write(vin,
+			   ALIGN(vin->format.width * vin->format.height, 0x80),
+			   VNUVAOF_REG);
+		dmr = VNDMR_DTMD_YCSEP;
+		output_is_yuv = true;
+		break;
+	case V4L2_PIX_FMT_YUYV:
+		dmr = VNDMR_BPSM;
+		output_is_yuv = true;
+		break;
+	case V4L2_PIX_FMT_UYVY:
+		dmr = 0;
+		output_is_yuv = true;
+		break;
+	case V4L2_PIX_FMT_XRGB555:
+		dmr = VNDMR_DTMD_ARGB1555;
+		break;
+	case V4L2_PIX_FMT_RGB565:
+		dmr = 0;
+		break;
+	case V4L2_PIX_FMT_XBGR32:
+		/* Note: not supported on M1 */
+		dmr = VNDMR_EXRGB;
+		break;
+	default:
+		vin_err(vin, "Invalid pixelformat (0x%x)\n",
+			vin->format.pixelformat);
+		return -EINVAL;
+	}
+
+	/* Always update on field change */
+	vnmc |= VNMC_VUP;
+
+	/* If input and output use the same colorspace, use bypass mode */
+	if (input_is_yuv == output_is_yuv)
+		vnmc |= VNMC_BPS;
+
+	if (vin->info->model == RCAR_GEN3) {
+		/* Select between CSI-2 and Digital input */
+		if (vin->mbus_cfg.type == V4L2_MBUS_CSI2)
+			vnmc &= ~VNMC_DPINE;
+		else
+			vnmc |= VNMC_DPINE;
+	}
+
+	/* Progressive or interlaced mode */
+	interrupts = progressive ? VNIE_FIE : VNIE_EFE;
+
+	/* Ack interrupts */
+	rvin_write(vin, interrupts, VNINTS_REG);
+	/* Enable interrupts */
+	rvin_write(vin, interrupts, VNIE_REG);
+	/* Start capturing */
+	rvin_write(vin, dmr, VNDMR_REG);
+	rvin_write(vin, dmr2, VNDMR2_REG);
+
+	/* Enable module */
+	rvin_write(vin, vnmc | VNMC_ME, VNMC_REG);
+
+	return 0;
+}
 
-	/* Set coefficient registers */
-	rvin_write(vin, p_set->coeff_set[0], VNC1A_REG);
-	rvin_write(vin, p_set->coeff_set[1], VNC1B_REG);
-	rvin_write(vin, p_set->coeff_set[2], VNC1C_REG);
+static void rvin_disable_interrupts(struct rvin_dev *vin)
+{
+	rvin_write(vin, 0, VNIE_REG);
+}
 
-	rvin_write(vin, p_set->coeff_set[3], VNC2A_REG);
-	rvin_write(vin, p_set->coeff_set[4], VNC2B_REG);
-	rvin_write(vin, p_set->coeff_set[5], VNC2C_REG);
+static u32 rvin_get_interrupt_status(struct rvin_dev *vin)
+{
+	return rvin_read(vin, VNINTS_REG);
+}
 
-	rvin_write(vin, p_set->coeff_set[6], VNC3A_REG);
-	rvin_write(vin, p_set->coeff_set[7], VNC3B_REG);
-	rvin_write(vin, p_set->coeff_set[8], VNC3C_REG);
+static void rvin_ack_interrupt(struct rvin_dev *vin)
+{
+	rvin_write(vin, rvin_read(vin, VNINTS_REG), VNINTS_REG);
+}
 
-	rvin_write(vin, p_set->coeff_set[9], VNC4A_REG);
-	rvin_write(vin, p_set->coeff_set[10], VNC4B_REG);
-	rvin_write(vin, p_set->coeff_set[11], VNC4C_REG);
+static bool rvin_capture_active(struct rvin_dev *vin)
+{
+	return rvin_read(vin, VNMS_REG) & VNMS_CA;
+}
 
-	rvin_write(vin, p_set->coeff_set[12], VNC5A_REG);
-	rvin_write(vin, p_set->coeff_set[13], VNC5B_REG);
-	rvin_write(vin, p_set->coeff_set[14], VNC5C_REG);
+static void rvin_set_slot_addr(struct rvin_dev *vin, int slot, dma_addr_t addr)
+{
+	const struct rvin_video_format *fmt;
+	int offsetx, offsety;
+	dma_addr_t offset;
 
-	rvin_write(vin, p_set->coeff_set[15], VNC6A_REG);
-	rvin_write(vin, p_set->coeff_set[16], VNC6B_REG);
-	rvin_write(vin, p_set->coeff_set[17], VNC6C_REG);
+	fmt = rvin_format_from_pixel(vin->format.pixelformat);
 
-	rvin_write(vin, p_set->coeff_set[18], VNC7A_REG);
-	rvin_write(vin, p_set->coeff_set[19], VNC7B_REG);
-	rvin_write(vin, p_set->coeff_set[20], VNC7C_REG);
+	/*
+	 * There is no HW support for composition do the beast we can
+	 * by modifying the buffer offset
+	 */
+	offsetx = vin->compose.left * fmt->bpp;
+	offsety = vin->compose.top * vin->format.bytesperline;
+	offset = addr + offsetx + offsety;
 
-	rvin_write(vin, p_set->coeff_set[21], VNC8A_REG);
-	rvin_write(vin, p_set->coeff_set[22], VNC8B_REG);
-	rvin_write(vin, p_set->coeff_set[23], VNC8C_REG);
+	/*
+	 * The address needs to be 128 bytes aligned. Driver should never accept
+	 * settings that do not satisfy this in the first place...
+	 */
+	if (WARN_ON((offsetx | offsety | offset) & HW_BUFFER_MASK))
+		return;
+
+	rvin_write(vin, offset, VNMB_REG(slot));
 }
 
-void rvin_crop_scale_comp(struct rvin_dev *vin)
+/*
+ * Moves a buffer from the queue to the HW slot. If no buffer is
+ * available use the scratch buffer. The scratch buffer is never
+ * returned to userspace, its only function is to enable the capture
+ * loop to keep running.
+ */
+static void rvin_fill_hw_slot(struct rvin_dev *vin, int slot)
 {
-	u32 xs, ys;
+	struct rvin_buffer *buf;
+	struct vb2_v4l2_buffer *vbuf;
+	dma_addr_t phys_addr;
 
-	/* Set Start/End Pixel/Line Pre-Clip */
-	rvin_write(vin, vin->crop.left, VNSPPRC_REG);
-	rvin_write(vin, vin->crop.left + vin->crop.width - 1, VNEPPRC_REG);
-	switch (vin->format.field) {
-	case V4L2_FIELD_INTERLACED:
-	case V4L2_FIELD_INTERLACED_TB:
-	case V4L2_FIELD_INTERLACED_BT:
-		rvin_write(vin, vin->crop.top / 2, VNSLPRC_REG);
-		rvin_write(vin, (vin->crop.top + vin->crop.height) / 2 - 1,
-			   VNELPRC_REG);
-		break;
-	default:
-		rvin_write(vin, vin->crop.top, VNSLPRC_REG);
-		rvin_write(vin, vin->crop.top + vin->crop.height - 1,
-			   VNELPRC_REG);
-		break;
+	/* A already populated slot shall never be overwritten. */
+	if (WARN_ON(vin->queue_buf[slot] != NULL))
+		return;
+
+	vin_dbg(vin, "Filling HW slot: %d\n", slot);
+
+	if (list_empty(&vin->buf_list)) {
+		vin->queue_buf[slot] = NULL;
+		phys_addr = vin->scratch_phys;
+	} else {
+		/* Keep track of buffer we give to HW */
+		buf = list_entry(vin->buf_list.next, struct rvin_buffer, list);
+		vbuf = &buf->vb;
+		list_del_init(to_buf_list(vbuf));
+		vin->queue_buf[slot] = vbuf;
+
+		/* Setup DMA */
+		phys_addr = vb2_dma_contig_plane_dma_addr(&vbuf->vb2_buf, 0);
 	}
 
-	/* Set scaling coefficient */
-	ys = 0;
-	if (vin->crop.height != vin->compose.height)
-		ys = (4096 * vin->crop.height) / vin->compose.height;
-	rvin_write(vin, ys, VNYS_REG);
+	rvin_set_slot_addr(vin, slot, phys_addr);
+}
 
-	xs = 0;
-	if (vin->crop.width != vin->compose.width)
-		xs = (4096 * vin->crop.width) / vin->compose.width;
+static int rvin_capture_start(struct rvin_dev *vin)
+{
+	int slot, ret;
 
-	/* Horizontal upscaling is up to double size */
-	if (xs > 0 && xs < 2048)
-		xs = 2048;
+	for (slot = 0; slot < HW_BUFFER_NUM; slot++)
+		rvin_fill_hw_slot(vin, slot);
 
-	rvin_write(vin, xs, VNXS_REG);
+	rvin_crop_scale_comp(vin);
 
-	/* Horizontal upscaling is done out by scaling down from double size */
-	if (xs < 4096)
-		xs *= 2;
+	ret = rvin_setup(vin);
+	if (ret)
+		return ret;
 
-	rvin_set_coeff(vin, xs);
+	vin_dbg(vin, "Starting to capture\n");
 
-	/* Set Start/End Pixel/Line Post-Clip */
-	rvin_write(vin, 0, VNSPPOC_REG);
-	rvin_write(vin, 0, VNSLPOC_REG);
-	rvin_write(vin, vin->format.width - 1, VNEPPOC_REG);
-	switch (vin->format.field) {
-	case V4L2_FIELD_INTERLACED:
-	case V4L2_FIELD_INTERLACED_TB:
-	case V4L2_FIELD_INTERLACED_BT:
-		rvin_write(vin, vin->format.height / 2 - 1, VNELPOC_REG);
-		break;
-	default:
-		rvin_write(vin, vin->format.height - 1, VNELPOC_REG);
-		break;
-	}
+	/* Continuous Frame Capture Mode */
+	rvin_write(vin, VNFC_C_FRAME, VNFC_REG);
 
-	if (vin->format.pixelformat == V4L2_PIX_FMT_NV16)
-		rvin_write(vin, ALIGN(vin->format.width, 0x20), VNIS_REG);
-	else
-		rvin_write(vin, ALIGN(vin->format.width, 0x10), VNIS_REG);
+	vin->state = RUNNING;
 
-	vin_dbg(vin,
-		"Pre-Clip: %ux%u@%u:%u YS: %d XS: %d Post-Clip: %ux%u@%u:%u\n",
-		vin->crop.width, vin->crop.height, vin->crop.left,
-		vin->crop.top, ys, xs, vin->format.width, vin->format.height,
-		0, 0);
+	return 0;
 }
 
-void rvin_scale_try(struct rvin_dev *vin, struct v4l2_pix_format *pix,
-		    u32 width, u32 height)
+static void rvin_capture_stop(struct rvin_dev *vin)
 {
-	/* All VIN channels on Gen2 have scalers */
-	pix->width = width;
-	pix->height = height;
+	/* Set continuous & single transfer off */
+	rvin_write(vin, 0, VNFC_REG);
+
+	/* Disable module */
+	rvin_write(vin, rvin_read(vin, VNMC_REG) & ~VNMC_ME, VNMC_REG);
 }
 
 /* -----------------------------------------------------------------------------
@@ -896,7 +912,7 @@ static irqreturn_t rvin_irq(int irq, void *data)
 
 	/* Capture frame */
 	if (vin->queue_buf[slot]) {
-		vin->queue_buf[slot]->field = rvin_get_active_field(vin, vnms);
+		vin->queue_buf[slot]->field = vin->format.field;
 		vin->queue_buf[slot]->sequence = vin->sequence;
 		vin->queue_buf[slot]->vb2_buf.timestamp = ktime_get_ns();
 		vb2_buffer_done(&vin->queue_buf[slot]->vb2_buf,
@@ -984,10 +1000,127 @@ static void rvin_buffer_queue(struct vb2_buffer *vb)
 	spin_unlock_irqrestore(&vin->qlock, flags);
 }
 
+static int rvin_mc_validate_format(struct rvin_dev *vin, struct v4l2_subdev *sd,
+				   struct media_pad *pad)
+{
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+
+	fmt.pad = pad->index;
+	if (v4l2_subdev_call(sd, pad, get_fmt, NULL, &fmt))
+		return -EPIPE;
+
+	switch (fmt.format.code) {
+	case MEDIA_BUS_FMT_YUYV8_1X16:
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+	case MEDIA_BUS_FMT_UYVY8_2X8:
+	case MEDIA_BUS_FMT_UYVY10_2X10:
+	case MEDIA_BUS_FMT_RGB888_1X24:
+		vin->mbus_code = fmt.format.code;
+		break;
+	default:
+		return -EPIPE;
+	}
+
+	switch (fmt.format.field) {
+	case V4L2_FIELD_TOP:
+	case V4L2_FIELD_BOTTOM:
+	case V4L2_FIELD_NONE:
+	case V4L2_FIELD_INTERLACED_TB:
+	case V4L2_FIELD_INTERLACED_BT:
+	case V4L2_FIELD_INTERLACED:
+	case V4L2_FIELD_SEQ_TB:
+	case V4L2_FIELD_SEQ_BT:
+		/* Supported natively */
+		break;
+	case V4L2_FIELD_ALTERNATE:
+		switch (vin->format.field) {
+		case V4L2_FIELD_TOP:
+		case V4L2_FIELD_BOTTOM:
+		case V4L2_FIELD_NONE:
+			break;
+		case V4L2_FIELD_INTERLACED_TB:
+		case V4L2_FIELD_INTERLACED_BT:
+		case V4L2_FIELD_INTERLACED:
+		case V4L2_FIELD_SEQ_TB:
+		case V4L2_FIELD_SEQ_BT:
+			/* Use VIN hardware to combine the two fields */
+			fmt.format.height *= 2;
+			break;
+		default:
+			return -EPIPE;
+		}
+		break;
+	default:
+		return -EPIPE;
+	}
+
+	if (fmt.format.width != vin->format.width ||
+	    fmt.format.height != vin->format.height ||
+	    fmt.format.code != vin->mbus_code)
+		return -EPIPE;
+
+	return 0;
+}
+
+static int rvin_set_stream(struct rvin_dev *vin, int on)
+{
+	struct media_pipeline *pipe;
+	struct media_device *mdev;
+	struct v4l2_subdev *sd;
+	struct media_pad *pad;
+	int ret;
+
+	/* No media controller used, simply pass operation to subdevice. */
+	if (!vin->info->use_mc) {
+		ret = v4l2_subdev_call(vin->digital->subdev, video, s_stream,
+				       on);
+
+		return ret == -ENOIOCTLCMD ? 0 : ret;
+	}
+
+	pad = media_entity_remote_pad(&vin->pad);
+	if (!pad)
+		return -EPIPE;
+
+	sd = media_entity_to_v4l2_subdev(pad->entity);
+
+	if (!on) {
+		media_pipeline_stop(&vin->vdev.entity);
+		return v4l2_subdev_call(sd, video, s_stream, 0);
+	}
+
+	ret = rvin_mc_validate_format(vin, sd, pad);
+	if (ret)
+		return ret;
+
+	/*
+	 * The graph lock needs to be taken to protect concurrent
+	 * starts of multiple VIN instances as they might share
+	 * a common subdevice down the line and then should use
+	 * the same pipe.
+	 */
+	mdev = vin->vdev.entity.graph_obj.mdev;
+	mutex_lock(&mdev->graph_mutex);
+	pipe = sd->entity.pipe ? sd->entity.pipe : &vin->vdev.pipe;
+	ret = __media_pipeline_start(&vin->vdev.entity, pipe);
+	mutex_unlock(&mdev->graph_mutex);
+	if (ret)
+		return ret;
+
+	ret = v4l2_subdev_call(sd, video, s_stream, 1);
+	if (ret == -ENOIOCTLCMD)
+		ret = 0;
+	if (ret)
+		media_pipeline_stop(&vin->vdev.entity);
+
+	return ret;
+}
+
 static int rvin_start_streaming(struct vb2_queue *vq, unsigned int count)
 {
 	struct rvin_dev *vin = vb2_get_drv_priv(vq);
-	struct v4l2_subdev *sd;
 	unsigned long flags;
 	int ret;
 
@@ -1002,8 +1135,13 @@ static int rvin_start_streaming(struct vb2_queue *vq, unsigned int count)
 		return -ENOMEM;
 	}
 
-	sd = vin_to_source(vin);
-	v4l2_subdev_call(sd, video, s_stream, 1);
+	ret = rvin_set_stream(vin, 1);
+	if (ret) {
+		spin_lock_irqsave(&vin->qlock, flags);
+		return_all_buffers(vin, VB2_BUF_STATE_QUEUED);
+		spin_unlock_irqrestore(&vin->qlock, flags);
+		goto out;
+	}
 
 	spin_lock_irqsave(&vin->qlock, flags);
 
@@ -1012,11 +1150,11 @@ static int rvin_start_streaming(struct vb2_queue *vq, unsigned int count)
 	ret = rvin_capture_start(vin);
 	if (ret) {
 		return_all_buffers(vin, VB2_BUF_STATE_QUEUED);
-		v4l2_subdev_call(sd, video, s_stream, 0);
+		rvin_set_stream(vin, 0);
 	}
 
 	spin_unlock_irqrestore(&vin->qlock, flags);
-
+out:
 	if (ret)
 		dma_free_coherent(vin->dev, vin->format.sizeimage, vin->scratch,
 				  vin->scratch_phys);
@@ -1027,7 +1165,6 @@ static int rvin_start_streaming(struct vb2_queue *vq, unsigned int count)
 static void rvin_stop_streaming(struct vb2_queue *vq)
 {
 	struct rvin_dev *vin = vb2_get_drv_priv(vq);
-	struct v4l2_subdev *sd;
 	unsigned long flags;
 	int retries = 0;
 
@@ -1066,8 +1203,7 @@ static void rvin_stop_streaming(struct vb2_queue *vq)
 
 	spin_unlock_irqrestore(&vin->qlock, flags);
 
-	sd = vin_to_source(vin);
-	v4l2_subdev_call(sd, video, s_stream, 0);
+	rvin_set_stream(vin, 0);
 
 	/* disable interrupts */
 	rvin_disable_interrupts(vin);
@@ -1087,14 +1223,14 @@ static const struct vb2_ops rvin_qops = {
 	.wait_finish		= vb2_ops_wait_finish,
 };
 
-void rvin_dma_remove(struct rvin_dev *vin)
+void rvin_dma_unregister(struct rvin_dev *vin)
 {
 	mutex_destroy(&vin->lock);
 
 	v4l2_device_unregister(&vin->v4l2_dev);
 }
 
-int rvin_dma_probe(struct rvin_dev *vin, int irq)
+int rvin_dma_register(struct rvin_dev *vin, int irq)
 {
 	struct vb2_queue *q = &vin->queue;
 	int i, ret;
@@ -1142,7 +1278,43 @@ int rvin_dma_probe(struct rvin_dev *vin, int irq)
 
 	return 0;
 error:
-	rvin_dma_remove(vin);
+	rvin_dma_unregister(vin);
+
+	return ret;
+}
+
+/* -----------------------------------------------------------------------------
+ * Gen3 CHSEL manipulation
+ */
+
+/*
+ * There is no need to have locking around changing the routing
+ * as it's only possible to do so when no VIN in the group is
+ * streaming so nothing can race with the VNMC register.
+ */
+int rvin_set_channel_routing(struct rvin_dev *vin, u8 chsel)
+{
+	u32 ifmd, vnmc;
+	int ret;
+
+	ret = pm_runtime_get_sync(vin->dev);
+	if (ret < 0)
+		return ret;
+
+	/* Make register writes take effect immediately. */
+	vnmc = rvin_read(vin, VNMC_REG);
+	rvin_write(vin, vnmc & ~VNMC_VUP, VNMC_REG);
+
+	ifmd = VNCSI_IFMD_DES1 | VNCSI_IFMD_DES0 | VNCSI_IFMD_CSI_CHSEL(chsel);
+
+	rvin_write(vin, ifmd, VNCSI_IFMD_REG);
+
+	vin_dbg(vin, "Set IFMD 0x%x\n", ifmd);
+
+	/* Restore VNMC. */
+	rvin_write(vin, vnmc, VNMC_REG);
+
+	pm_runtime_put(vin->dev);
 
 	return ret;
 }
diff --git a/drivers/media/platform/rcar-vin/rcar-v4l2.c b/drivers/media/platform/rcar-vin/rcar-v4l2.c
index b479b882da12..e78fba84d590 100644
--- a/drivers/media/platform/rcar-vin/rcar-v4l2.c
+++ b/drivers/media/platform/rcar-vin/rcar-v4l2.c
@@ -18,13 +18,16 @@
 
 #include <media/v4l2-event.h>
 #include <media/v4l2-ioctl.h>
+#include <media/v4l2-mc.h>
 #include <media/v4l2-rect.h>
 
 #include "rcar-vin.h"
 
 #define RVIN_DEFAULT_FORMAT	V4L2_PIX_FMT_YUYV
-#define RVIN_MAX_WIDTH		2048
-#define RVIN_MAX_HEIGHT		2048
+#define RVIN_DEFAULT_WIDTH	800
+#define RVIN_DEFAULT_HEIGHT	600
+#define RVIN_DEFAULT_FIELD	V4L2_FIELD_NONE
+#define RVIN_DEFAULT_COLORSPACE	V4L2_COLORSPACE_SRGB
 
 /* -----------------------------------------------------------------------------
  * Format Conversions
@@ -88,99 +91,111 @@ static u32 rvin_format_sizeimage(struct v4l2_pix_format *pix)
 	return pix->bytesperline * pix->height;
 }
 
-/* -----------------------------------------------------------------------------
- * V4L2
- */
-
-static void rvin_reset_crop_compose(struct rvin_dev *vin)
+static void rvin_format_align(struct rvin_dev *vin, struct v4l2_pix_format *pix)
 {
-	vin->crop.top = vin->crop.left = 0;
-	vin->crop.width = vin->source.width;
-	vin->crop.height = vin->source.height;
+	u32 walign;
 
-	vin->compose.top = vin->compose.left = 0;
-	vin->compose.width = vin->format.width;
-	vin->compose.height = vin->format.height;
+	if (!rvin_format_from_pixel(pix->pixelformat) ||
+	    (vin->info->model == RCAR_M1 &&
+	     pix->pixelformat == V4L2_PIX_FMT_XBGR32))
+		pix->pixelformat = RVIN_DEFAULT_FORMAT;
+
+	switch (pix->field) {
+	case V4L2_FIELD_TOP:
+	case V4L2_FIELD_BOTTOM:
+	case V4L2_FIELD_NONE:
+	case V4L2_FIELD_INTERLACED_TB:
+	case V4L2_FIELD_INTERLACED_BT:
+	case V4L2_FIELD_INTERLACED:
+		break;
+	case V4L2_FIELD_ALTERNATE:
+		/*
+		 * Driver does not (yet) support outputting ALTERNATE to a
+		 * userspace. It does support outputting INTERLACED so use
+		 * the VIN hardware to combine the two fields.
+		 */
+		pix->field = V4L2_FIELD_INTERLACED;
+		pix->height *= 2;
+		break;
+	default:
+		pix->field = RVIN_DEFAULT_FIELD;
+		break;
+	}
+
+	/* HW limit width to a multiple of 32 (2^5) for NV16 else 2 (2^1) */
+	walign = vin->format.pixelformat == V4L2_PIX_FMT_NV16 ? 5 : 1;
+
+	/* Limit to VIN capabilities */
+	v4l_bound_align_image(&pix->width, 2, vin->info->max_width, walign,
+			      &pix->height, 4, vin->info->max_height, 2, 0);
+
+	pix->bytesperline = rvin_format_bytesperline(pix);
+	pix->sizeimage = rvin_format_sizeimage(pix);
+
+	vin_dbg(vin, "Format %ux%u bpl: %u size: %u\n",
+		pix->width, pix->height, pix->bytesperline, pix->sizeimage);
 }
 
+/* -----------------------------------------------------------------------------
+ * V4L2
+ */
+
 static int rvin_reset_format(struct rvin_dev *vin)
 {
 	struct v4l2_subdev_format fmt = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.pad = vin->digital->source_pad,
 	};
-	struct v4l2_mbus_framefmt *mf = &fmt.format;
 	int ret;
 
-	fmt.pad = vin->digital->source_pad;
-
 	ret = v4l2_subdev_call(vin_to_source(vin), pad, get_fmt, NULL, &fmt);
 	if (ret)
 		return ret;
 
-	vin->format.width	= mf->width;
-	vin->format.height	= mf->height;
-	vin->format.colorspace	= mf->colorspace;
-	vin->format.field	= mf->field;
+	v4l2_fill_pix_format(&vin->format, &fmt.format);
 
-	/*
-	 * If the subdevice uses ALTERNATE field mode and G_STD is
-	 * implemented use the VIN HW to combine the two fields to
-	 * one INTERLACED frame. The ALTERNATE field mode can still
-	 * be requested in S_FMT and be respected, this is just the
-	 * default which is applied at probing or when S_STD is called.
-	 */
-	if (vin->format.field == V4L2_FIELD_ALTERNATE &&
-	    v4l2_subdev_has_op(vin_to_source(vin), video, g_std))
-		vin->format.field = V4L2_FIELD_INTERLACED;
+	rvin_format_align(vin, &vin->format);
 
-	switch (vin->format.field) {
-	case V4L2_FIELD_TOP:
-	case V4L2_FIELD_BOTTOM:
-	case V4L2_FIELD_ALTERNATE:
-		vin->format.height /= 2;
-		break;
-	case V4L2_FIELD_NONE:
-	case V4L2_FIELD_INTERLACED_TB:
-	case V4L2_FIELD_INTERLACED_BT:
-	case V4L2_FIELD_INTERLACED:
-		break;
-	default:
-		vin->format.field = V4L2_FIELD_NONE;
-		break;
-	}
-
-	rvin_reset_crop_compose(vin);
+	vin->source.top = 0;
+	vin->source.left = 0;
+	vin->source.width = vin->format.width;
+	vin->source.height = vin->format.height;
 
-	vin->format.bytesperline = rvin_format_bytesperline(&vin->format);
-	vin->format.sizeimage = rvin_format_sizeimage(&vin->format);
+	vin->crop = vin->source;
+	vin->compose = vin->source;
 
 	return 0;
 }
 
-static int __rvin_try_format_source(struct rvin_dev *vin,
-				    u32 which,
-				    struct v4l2_pix_format *pix,
-				    struct rvin_source_fmt *source)
+static int rvin_try_format(struct rvin_dev *vin, u32 which,
+			   struct v4l2_pix_format *pix,
+			   struct v4l2_rect *crop, struct v4l2_rect *compose)
 {
-	struct v4l2_subdev *sd;
+	struct v4l2_subdev *sd = vin_to_source(vin);
 	struct v4l2_subdev_pad_config *pad_cfg;
 	struct v4l2_subdev_format format = {
 		.which = which,
+		.pad = vin->digital->source_pad,
 	};
 	enum v4l2_field field;
+	u32 width, height;
 	int ret;
 
-	sd = vin_to_source(vin);
-
-	v4l2_fill_mbus_format(&format.format, pix, vin->digital->code);
-
 	pad_cfg = v4l2_subdev_alloc_pad_config(sd);
 	if (pad_cfg == NULL)
 		return -ENOMEM;
 
-	format.pad = vin->digital->source_pad;
+	if (!rvin_format_from_pixel(pix->pixelformat) ||
+	    (vin->info->model == RCAR_M1 &&
+	     pix->pixelformat == V4L2_PIX_FMT_XBGR32))
+		pix->pixelformat = RVIN_DEFAULT_FORMAT;
+
+	v4l2_fill_mbus_format(&format.format, pix, vin->mbus_code);
 
+	/* Allow the video device to override field and to scale */
 	field = pix->field;
+	width = pix->width;
+	height = pix->height;
 
 	ret = v4l2_subdev_call(sd, pad, set_fmt, pad_cfg, &format);
 	if (ret < 0 && ret != -ENOIOCTLCMD)
@@ -188,92 +203,36 @@ static int __rvin_try_format_source(struct rvin_dev *vin,
 
 	v4l2_fill_pix_format(pix, &format.format);
 
-	pix->field = field;
-
-	source->width = pix->width;
-	source->height = pix->height;
-
-	vin_dbg(vin, "Source resolution: %ux%u\n", source->width,
-		source->height);
+	if (crop) {
+		crop->top = 0;
+		crop->left = 0;
+		crop->width = pix->width;
+		crop->height = pix->height;
 
-done:
-	v4l2_subdev_free_pad_config(pad_cfg);
-	return ret;
-}
-
-static int __rvin_try_format(struct rvin_dev *vin,
-			     u32 which,
-			     struct v4l2_pix_format *pix,
-			     struct rvin_source_fmt *source)
-{
-	u32 rwidth, rheight, walign;
-	int ret;
-
-	/* Requested */
-	rwidth = pix->width;
-	rheight = pix->height;
-
-	/* Keep current field if no specific one is asked for */
-	if (pix->field == V4L2_FIELD_ANY)
-		pix->field = vin->format.field;
-
-	/* If requested format is not supported fallback to the default */
-	if (!rvin_format_from_pixel(pix->pixelformat)) {
-		vin_dbg(vin, "Format 0x%x not found, using default 0x%x\n",
-			pix->pixelformat, RVIN_DEFAULT_FORMAT);
-		pix->pixelformat = RVIN_DEFAULT_FORMAT;
+		/*
+		 * If source is ALTERNATE the driver will use the VIN hardware
+		 * to INTERLACE it. The crop height then needs to be doubled.
+		 */
+		if (pix->field == V4L2_FIELD_ALTERNATE)
+			crop->height *= 2;
 	}
 
-	/* Always recalculate */
-	pix->bytesperline = 0;
-	pix->sizeimage = 0;
+	if (field != V4L2_FIELD_ANY)
+		pix->field = field;
 
-	/* Limit to source capabilities */
-	ret = __rvin_try_format_source(vin, which, pix, source);
-	if (ret)
-		return ret;
+	pix->width = width;
+	pix->height = height;
 
-	switch (pix->field) {
-	case V4L2_FIELD_TOP:
-	case V4L2_FIELD_BOTTOM:
-	case V4L2_FIELD_ALTERNATE:
-		pix->height /= 2;
-		source->height /= 2;
-		break;
-	case V4L2_FIELD_NONE:
-	case V4L2_FIELD_INTERLACED_TB:
-	case V4L2_FIELD_INTERLACED_BT:
-	case V4L2_FIELD_INTERLACED:
-		break;
-	default:
-		pix->field = V4L2_FIELD_NONE;
-		break;
-	}
-
-	/* If source can't match format try if VIN can scale */
-	if (source->width != rwidth || source->height != rheight)
-		rvin_scale_try(vin, pix, rwidth, rheight);
-
-	/* HW limit width to a multiple of 32 (2^5) for NV16 else 2 (2^1) */
-	walign = vin->format.pixelformat == V4L2_PIX_FMT_NV16 ? 5 : 1;
+	rvin_format_align(vin, pix);
 
-	/* Limit to VIN capabilities */
-	v4l_bound_align_image(&pix->width, 2, RVIN_MAX_WIDTH, walign,
-			      &pix->height, 4, RVIN_MAX_HEIGHT, 2, 0);
-
-	pix->bytesperline = max_t(u32, pix->bytesperline,
-				  rvin_format_bytesperline(pix));
-	pix->sizeimage = max_t(u32, pix->sizeimage,
-			       rvin_format_sizeimage(pix));
-
-	if (vin->chip == RCAR_M1 && pix->pixelformat == V4L2_PIX_FMT_XBGR32) {
-		vin_err(vin, "pixel format XBGR32 not supported on M1\n");
-		return -EINVAL;
+	if (compose) {
+		compose->top = 0;
+		compose->left = 0;
+		compose->width = pix->width;
+		compose->height = pix->height;
 	}
-
-	vin_dbg(vin, "Requested %ux%u Got %ux%u bpl: %d size: %d\n",
-		rwidth, rheight, pix->width, pix->height,
-		pix->bytesperline, pix->sizeimage);
+done:
+	v4l2_subdev_free_pad_config(pad_cfg);
 
 	return 0;
 }
@@ -294,33 +253,30 @@ static int rvin_try_fmt_vid_cap(struct file *file, void *priv,
 				struct v4l2_format *f)
 {
 	struct rvin_dev *vin = video_drvdata(file);
-	struct rvin_source_fmt source;
 
-	return __rvin_try_format(vin, V4L2_SUBDEV_FORMAT_TRY, &f->fmt.pix,
-				 &source);
+	return rvin_try_format(vin, V4L2_SUBDEV_FORMAT_TRY, &f->fmt.pix, NULL,
+			       NULL);
 }
 
 static int rvin_s_fmt_vid_cap(struct file *file, void *priv,
 			      struct v4l2_format *f)
 {
 	struct rvin_dev *vin = video_drvdata(file);
-	struct rvin_source_fmt source;
+	struct v4l2_rect crop, compose;
 	int ret;
 
 	if (vb2_is_busy(&vin->queue))
 		return -EBUSY;
 
-	ret = __rvin_try_format(vin, V4L2_SUBDEV_FORMAT_ACTIVE, &f->fmt.pix,
-				&source);
+	ret = rvin_try_format(vin, V4L2_SUBDEV_FORMAT_ACTIVE, &f->fmt.pix,
+			      &crop, &compose);
 	if (ret)
 		return ret;
 
-	vin->source.width = source.width;
-	vin->source.height = source.height;
-
 	vin->format = f->fmt.pix;
-
-	rvin_reset_crop_compose(vin);
+	vin->crop = crop;
+	vin->compose = compose;
+	vin->source = crop;
 
 	return 0;
 }
@@ -405,8 +361,8 @@ static int rvin_s_selection(struct file *file, void *fh,
 		max_rect.height = vin->source.height;
 		v4l2_rect_map_inside(&r, &max_rect);
 
-		v4l_bound_align_image(&r.width, 2, vin->source.width, 1,
-				      &r.height, 4, vin->source.height, 2, 0);
+		v4l_bound_align_image(&r.width, 6, vin->source.width, 0,
+				      &r.height, 2, vin->source.height, 0, 0);
 
 		r.top  = clamp_t(s32, r.top, 0, vin->source.height - r.height);
 		r.left = clamp_t(s32, r.left, 0, vin->source.width - r.width);
@@ -523,6 +479,8 @@ static int rvin_s_std(struct file *file, void *priv, v4l2_std_id a)
 	if (ret < 0)
 		return ret;
 
+	vin->std = a;
+
 	/* Changing the standard will change the width/height */
 	return rvin_reset_format(vin);
 }
@@ -530,9 +488,13 @@ static int rvin_s_std(struct file *file, void *priv, v4l2_std_id a)
 static int rvin_g_std(struct file *file, void *priv, v4l2_std_id *a)
 {
 	struct rvin_dev *vin = video_drvdata(file);
-	struct v4l2_subdev *sd = vin_to_source(vin);
 
-	return v4l2_subdev_call(sd, video, g_std, a);
+	if (v4l2_subdev_has_op(vin_to_source(vin), pad, dv_timings_cap))
+		return -ENOIOCTLCMD;
+
+	*a = vin->std;
+
+	return 0;
 }
 
 static int rvin_subscribe_event(struct v4l2_fh *fh,
@@ -696,6 +658,97 @@ static const struct v4l2_ioctl_ops rvin_ioctl_ops = {
 	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
 };
 
+/* -----------------------------------------------------------------------------
+ * V4L2 Media Controller
+ */
+
+static void rvin_mc_try_format(struct rvin_dev *vin,
+			       struct v4l2_pix_format *pix)
+{
+	/*
+	 * The V4L2 specification clearly documents the colorspace fields
+	 * as being set by drivers for capture devices. Using the values
+	 * supplied by userspace thus wouldn't comply with the API. Until
+	 * the API is updated force fixed vaules.
+	 */
+	pix->colorspace = RVIN_DEFAULT_COLORSPACE;
+	pix->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(pix->colorspace);
+	pix->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(pix->colorspace);
+	pix->quantization = V4L2_MAP_QUANTIZATION_DEFAULT(true, pix->colorspace,
+							  pix->ycbcr_enc);
+
+	rvin_format_align(vin, pix);
+}
+
+static int rvin_mc_try_fmt_vid_cap(struct file *file, void *priv,
+				   struct v4l2_format *f)
+{
+	struct rvin_dev *vin = video_drvdata(file);
+
+	rvin_mc_try_format(vin, &f->fmt.pix);
+
+	return 0;
+}
+
+static int rvin_mc_s_fmt_vid_cap(struct file *file, void *priv,
+				 struct v4l2_format *f)
+{
+	struct rvin_dev *vin = video_drvdata(file);
+
+	if (vb2_is_busy(&vin->queue))
+		return -EBUSY;
+
+	rvin_mc_try_format(vin, &f->fmt.pix);
+
+	vin->format = f->fmt.pix;
+
+	vin->crop.top = 0;
+	vin->crop.left = 0;
+	vin->crop.width = vin->format.width;
+	vin->crop.height = vin->format.height;
+	vin->compose = vin->crop;
+
+	return 0;
+}
+
+static int rvin_mc_enum_input(struct file *file, void *priv,
+			      struct v4l2_input *i)
+{
+	if (i->index != 0)
+		return -EINVAL;
+
+	i->type = V4L2_INPUT_TYPE_CAMERA;
+	strlcpy(i->name, "Camera", sizeof(i->name));
+
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops rvin_mc_ioctl_ops = {
+	.vidioc_querycap		= rvin_querycap,
+	.vidioc_try_fmt_vid_cap		= rvin_mc_try_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap		= rvin_g_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap		= rvin_mc_s_fmt_vid_cap,
+	.vidioc_enum_fmt_vid_cap	= rvin_enum_fmt_vid_cap,
+
+	.vidioc_enum_input		= rvin_mc_enum_input,
+	.vidioc_g_input			= rvin_g_input,
+	.vidioc_s_input			= rvin_s_input,
+
+	.vidioc_reqbufs			= vb2_ioctl_reqbufs,
+	.vidioc_create_bufs		= vb2_ioctl_create_bufs,
+	.vidioc_querybuf		= vb2_ioctl_querybuf,
+	.vidioc_qbuf			= vb2_ioctl_qbuf,
+	.vidioc_dqbuf			= vb2_ioctl_dqbuf,
+	.vidioc_expbuf			= vb2_ioctl_expbuf,
+	.vidioc_prepare_buf		= vb2_ioctl_prepare_buf,
+	.vidioc_streamon		= vb2_ioctl_streamon,
+	.vidioc_streamoff		= vb2_ioctl_streamoff,
+
+	.vidioc_log_status		= v4l2_ctrl_log_status,
+	.vidioc_subscribe_event		= rvin_subscribe_event,
+	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
+};
+
 /* -----------------------------------------------------------------------------
  * File Operations
  */
@@ -839,14 +892,82 @@ static const struct v4l2_file_operations rvin_fops = {
 	.read		= vb2_fop_read,
 };
 
-void rvin_v4l2_remove(struct rvin_dev *vin)
+/* -----------------------------------------------------------------------------
+ * Media controller file operations
+ */
+
+static int rvin_mc_open(struct file *file)
 {
+	struct rvin_dev *vin = video_drvdata(file);
+	int ret;
+
+	ret = mutex_lock_interruptible(&vin->lock);
+	if (ret)
+		return ret;
+
+	ret = pm_runtime_get_sync(vin->dev);
+	if (ret < 0)
+		goto err_unlock;
+
+	ret = v4l2_pipeline_pm_use(&vin->vdev.entity, 1);
+	if (ret < 0)
+		goto err_pm;
+
+	file->private_data = vin;
+
+	ret = v4l2_fh_open(file);
+	if (ret)
+		goto err_v4l2pm;
+
+	mutex_unlock(&vin->lock);
+
+	return 0;
+err_v4l2pm:
+	v4l2_pipeline_pm_use(&vin->vdev.entity, 0);
+err_pm:
+	pm_runtime_put(vin->dev);
+err_unlock:
+	mutex_unlock(&vin->lock);
+
+	return ret;
+}
+
+static int rvin_mc_release(struct file *file)
+{
+	struct rvin_dev *vin = video_drvdata(file);
+	int ret;
+
+	mutex_lock(&vin->lock);
+
+	/* the release helper will cleanup any on-going streaming. */
+	ret = _vb2_fop_release(file, NULL);
+
+	v4l2_pipeline_pm_use(&vin->vdev.entity, 0);
+	pm_runtime_put(vin->dev);
+
+	mutex_unlock(&vin->lock);
+
+	return ret;
+}
+
+static const struct v4l2_file_operations rvin_mc_fops = {
+	.owner		= THIS_MODULE,
+	.unlocked_ioctl	= video_ioctl2,
+	.open		= rvin_mc_open,
+	.release	= rvin_mc_release,
+	.poll		= vb2_fop_poll,
+	.mmap		= vb2_fop_mmap,
+	.read		= vb2_fop_read,
+};
+
+void rvin_v4l2_unregister(struct rvin_dev *vin)
+{
+	if (!video_is_registered(&vin->vdev))
+		return;
+
 	v4l2_info(&vin->v4l2_dev, "Removing %s\n",
 		  video_device_node_name(&vin->vdev));
 
-	/* Checks internaly if handlers have been init or not */
-	v4l2_ctrl_handler_free(&vin->ctrl_handler);
-
 	/* Checks internaly if vdev have been init or not */
 	video_unregister_device(&vin->vdev);
 }
@@ -866,58 +987,39 @@ static void rvin_notify(struct v4l2_subdev *sd,
 	}
 }
 
-int rvin_v4l2_probe(struct rvin_dev *vin)
+int rvin_v4l2_register(struct rvin_dev *vin)
 {
 	struct video_device *vdev = &vin->vdev;
-	struct v4l2_subdev *sd = vin_to_source(vin);
 	int ret;
 
-	v4l2_set_subdev_hostdata(sd, vin);
-
 	vin->v4l2_dev.notify = rvin_notify;
 
-	ret = v4l2_subdev_call(sd, video, g_tvnorms, &vin->vdev.tvnorms);
-	if (ret < 0 && ret != -ENOIOCTLCMD && ret != -ENODEV)
-		return ret;
-
-	if (vin->vdev.tvnorms == 0) {
-		/* Disable the STD API if there are no tvnorms defined */
-		v4l2_disable_ioctl(&vin->vdev, VIDIOC_G_STD);
-		v4l2_disable_ioctl(&vin->vdev, VIDIOC_S_STD);
-		v4l2_disable_ioctl(&vin->vdev, VIDIOC_QUERYSTD);
-		v4l2_disable_ioctl(&vin->vdev, VIDIOC_ENUMSTD);
-	}
-
-	/* Add the controls */
-	/*
-	 * Currently the subdev with the largest number of controls (13) is
-	 * ov6550. So let's pick 16 as a hint for the control handler. Note
-	 * that this is a hint only: too large and you waste some memory, too
-	 * small and there is a (very) small performance hit when looking up
-	 * controls in the internal hash.
-	 */
-	ret = v4l2_ctrl_handler_init(&vin->ctrl_handler, 16);
-	if (ret < 0)
-		return ret;
-
-	ret = v4l2_ctrl_add_handler(&vin->ctrl_handler, sd->ctrl_handler, NULL);
-	if (ret < 0)
-		return ret;
-
 	/* video node */
-	vdev->fops = &rvin_fops;
 	vdev->v4l2_dev = &vin->v4l2_dev;
 	vdev->queue = &vin->queue;
-	strlcpy(vdev->name, KBUILD_MODNAME, sizeof(vdev->name));
+	snprintf(vdev->name, sizeof(vdev->name), "VIN%u output", vin->id);
 	vdev->release = video_device_release_empty;
-	vdev->ioctl_ops = &rvin_ioctl_ops;
 	vdev->lock = &vin->lock;
-	vdev->ctrl_handler = &vin->ctrl_handler;
 	vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING |
 		V4L2_CAP_READWRITE;
 
+	/* Set a default format */
 	vin->format.pixelformat	= RVIN_DEFAULT_FORMAT;
-	rvin_reset_format(vin);
+	vin->format.width = RVIN_DEFAULT_WIDTH;
+	vin->format.height = RVIN_DEFAULT_HEIGHT;
+	vin->format.field = RVIN_DEFAULT_FIELD;
+	vin->format.colorspace = RVIN_DEFAULT_COLORSPACE;
+
+	if (vin->info->use_mc) {
+		vdev->fops = &rvin_mc_fops;
+		vdev->ioctl_ops = &rvin_mc_ioctl_ops;
+	} else {
+		vdev->fops = &rvin_fops;
+		vdev->ioctl_ops = &rvin_ioctl_ops;
+		rvin_reset_format(vin);
+	}
+
+	rvin_format_align(vin, &vin->format);
 
 	ret = video_register_device(&vin->vdev, VFL_TYPE_GRABBER, -1);
 	if (ret) {
diff --git a/drivers/media/platform/rcar-vin/rcar-vin.h b/drivers/media/platform/rcar-vin/rcar-vin.h
index 95897127cc41..c2aef789b491 100644
--- a/drivers/media/platform/rcar-vin/rcar-vin.h
+++ b/drivers/media/platform/rcar-vin/rcar-vin.h
@@ -17,6 +17,8 @@
 #ifndef __RCAR_VIN__
 #define __RCAR_VIN__
 
+#include <linux/kref.h>
+
 #include <media/v4l2-async.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-dev.h>
@@ -29,10 +31,24 @@
 /* Address alignment mask for HW buffers */
 #define HW_BUFFER_MASK 0x7f
 
-enum chip_id {
+/* Max number on VIN instances that can be in a system */
+#define RCAR_VIN_NUM 8
+
+struct rvin_group;
+
+enum model_id {
 	RCAR_H1,
 	RCAR_M1,
 	RCAR_GEN2,
+	RCAR_GEN3,
+};
+
+enum rvin_csi_id {
+	RVIN_CSI20,
+	RVIN_CSI21,
+	RVIN_CSI40,
+	RVIN_CSI41,
+	RVIN_CSI_MAX,
 };
 
 /**
@@ -46,16 +62,6 @@ enum rvin_dma_state {
 	STOPPING,
 };
 
-/**
- * struct rvin_source_fmt - Source information
- * @width:	Width from source
- * @height:	Height from source
- */
-struct rvin_source_fmt {
-	u32 width;
-	u32 height;
-};
-
 /**
  * struct rvin_video_format - Data format stored in memory
  * @fourcc:	Pixelformat
@@ -70,8 +76,6 @@ struct rvin_video_format {
  * struct rvin_graph_entity - Video endpoint from async framework
  * @asd:	sub-device descriptor for async framework
  * @subdev:	subdevice matched using async framework
- * @code:	Media bus format from source
- * @mbus_cfg:	Media bus format from DT
  * @source_pad:	source pad of remote subdevice
  * @sink_pad:	sink pad of remote subdevice
  */
@@ -79,18 +83,64 @@ struct rvin_graph_entity {
 	struct v4l2_async_subdev asd;
 	struct v4l2_subdev *subdev;
 
-	u32 code;
-	struct v4l2_mbus_config mbus_cfg;
-
 	unsigned int source_pad;
 	unsigned int sink_pad;
 };
 
+/**
+ * struct rvin_group_route - describes a route from a channel of a
+ *	CSI-2 receiver to a VIN
+ *
+ * @csi:	CSI-2 receiver ID.
+ * @channel:	Output channel of the CSI-2 receiver.
+ * @vin:	VIN ID.
+ * @mask:	Bitmask of the different CHSEL register values that
+ *		allow for a route from @csi + @chan to @vin.
+ *
+ * .. note::
+ *	Each R-Car CSI-2 receiver has four output channels facing the VIN
+ *	devices, each channel can carry one CSI-2 Virtual Channel (VC).
+ *	There is no correlation between channel number and CSI-2 VC. It's
+ *	up to the CSI-2 receiver driver to configure which VC is output
+ *	on which channel, the VIN devices only care about output channels.
+ *
+ *	There are in some cases multiple CHSEL register settings which would
+ *	allow for the same route from @csi + @channel to @vin. For example
+ *	on R-Car H3 both the CHSEL values 0 and 3 allow for a route from
+ *	CSI40/VC0 to VIN0. All possible CHSEL values for a route need to be
+ *	recorded as a bitmask in @mask, in this example bit 0 and 3 should
+ *	be set.
+ */
+struct rvin_group_route {
+	enum rvin_csi_id csi;
+	unsigned int channel;
+	unsigned int vin;
+	unsigned int mask;
+};
+
+/**
+ * struct rvin_info - Information about the particular VIN implementation
+ * @model:		VIN model
+ * @use_mc:		use media controller instead of controlling subdevice
+ * @max_width:		max input width the VIN supports
+ * @max_height:		max input height the VIN supports
+ * @routes:		list of possible routes from the CSI-2 recivers to
+ *			all VINs. The list mush be NULL terminated.
+ */
+struct rvin_info {
+	enum model_id model;
+	bool use_mc;
+
+	unsigned int max_width;
+	unsigned int max_height;
+	const struct rvin_group_route *routes;
+};
+
 /**
  * struct rvin_dev - Renesas VIN device structure
  * @dev:		(OF) device
  * @base:		device I/O register space remapped to virtual memory
- * @chip:		type of VIN chip
+ * @info:		info about VIN instance
  *
  * @vdev:		V4L2 video device associated with VIN
  * @v4l2_dev:		V4L2 device
@@ -98,6 +148,10 @@ struct rvin_graph_entity {
  * @notifier:		V4L2 asynchronous subdevs notifier
  * @digital:		entity in the DT for local digital subdevice
  *
+ * @group:		Gen3 CSI group
+ * @id:			Gen3 group id for this VIN
+ * @pad:		media pad for the video device entity
+ *
  * @lock:		protects @queue
  * @queue:		vb2 buffers queue
  * @scratch:		cpu address for scratch buffer
@@ -110,16 +164,19 @@ struct rvin_graph_entity {
  * @sequence:		V4L2 buffers sequence number
  * @state:		keeps track of operation state
  *
- * @source:		active format from the video source
+ * @mbus_cfg:		media bus configuration from DT
+ * @mbus_code:		media bus format code
  * @format:		active V4L2 pixel format
  *
  * @crop:		active cropping
  * @compose:		active composing
+ * @source:		active size of the video source
+ * @std:		active video standard of the video source
  */
 struct rvin_dev {
 	struct device *dev;
 	void __iomem *base;
-	enum chip_id chip;
+	const struct rvin_info *info;
 
 	struct video_device vdev;
 	struct v4l2_device v4l2_dev;
@@ -127,6 +184,10 @@ struct rvin_dev {
 	struct v4l2_async_notifier notifier;
 	struct rvin_graph_entity *digital;
 
+	struct rvin_group *group;
+	unsigned int id;
+	struct media_pad pad;
+
 	struct mutex lock;
 	struct vb2_queue queue;
 	void *scratch;
@@ -138,11 +199,14 @@ struct rvin_dev {
 	unsigned int sequence;
 	enum rvin_dma_state state;
 
-	struct rvin_source_fmt source;
+	struct v4l2_mbus_config mbus_cfg;
+	u32 mbus_code;
 	struct v4l2_pix_format format;
 
 	struct v4l2_rect crop;
 	struct v4l2_rect compose;
+	struct v4l2_rect source;
+	v4l2_std_id std;
 };
 
 #define vin_to_source(vin)		((vin)->digital->subdev)
@@ -153,17 +217,47 @@ struct rvin_dev {
 #define vin_warn(d, fmt, arg...)	dev_warn(d->dev, fmt, ##arg)
 #define vin_err(d, fmt, arg...)		dev_err(d->dev, fmt, ##arg)
 
-int rvin_dma_probe(struct rvin_dev *vin, int irq);
-void rvin_dma_remove(struct rvin_dev *vin);
+/**
+ * struct rvin_group - VIN CSI2 group information
+ * @refcount:		number of VIN instances using the group
+ *
+ * @mdev:		media device which represents the group
+ *
+ * @lock:		protects the count, notifier, vin and csi members
+ * @count:		number of enabled VIN instances found in DT
+ * @notifier:		pointer to the notifier of a VIN which handles the
+ *			groups async sub-devices.
+ * @vin:		VIN instances which are part of the group
+ * @csi:		array of pairs of fwnode and subdev pointers
+ *			to all CSI-2 subdevices.
+ */
+struct rvin_group {
+	struct kref refcount;
+
+	struct media_device mdev;
 
-int rvin_v4l2_probe(struct rvin_dev *vin);
-void rvin_v4l2_remove(struct rvin_dev *vin);
+	struct mutex lock;
+	unsigned int count;
+	struct v4l2_async_notifier *notifier;
+	struct rvin_dev *vin[RCAR_VIN_NUM];
+
+	struct {
+		struct fwnode_handle *fwnode;
+		struct v4l2_subdev *subdev;
+	} csi[RVIN_CSI_MAX];
+};
+
+int rvin_dma_register(struct rvin_dev *vin, int irq);
+void rvin_dma_unregister(struct rvin_dev *vin);
+
+int rvin_v4l2_register(struct rvin_dev *vin);
+void rvin_v4l2_unregister(struct rvin_dev *vin);
 
 const struct rvin_video_format *rvin_format_from_pixel(u32 pixelformat);
 
 /* Cropping, composing and scaling */
-void rvin_scale_try(struct rvin_dev *vin, struct v4l2_pix_format *pix,
-		    u32 width, u32 height);
 void rvin_crop_scale_comp(struct rvin_dev *vin);
 
+int rvin_set_channel_routing(struct rvin_dev *vin, u8 chsel);
+
 #endif
diff --git a/drivers/media/platform/rcar_jpu.c b/drivers/media/platform/rcar_jpu.c
index f6092ae45912..8b44a849ab41 100644
--- a/drivers/media/platform/rcar_jpu.c
+++ b/drivers/media/platform/rcar_jpu.c
@@ -1280,7 +1280,7 @@ static int jpu_open(struct file *file)
 		/* ...issue software reset */
 		ret = jpu_reset(jpu);
 		if (ret)
-			goto device_prepare_rollback;
+			goto jpu_reset_rollback;
 	}
 
 	jpu->ref_count++;
@@ -1288,6 +1288,8 @@ static int jpu_open(struct file *file)
 	mutex_unlock(&jpu->mutex);
 	return 0;
 
+jpu_reset_rollback:
+	clk_disable_unprepare(jpu->clk);
 device_prepare_rollback:
 	mutex_unlock(&jpu->mutex);
 v4l_prepare_rollback:
diff --git a/drivers/media/platform/renesas-ceu.c b/drivers/media/platform/renesas-ceu.c
index 6599dba5ab84..fe4fe944592d 100644
--- a/drivers/media/platform/renesas-ceu.c
+++ b/drivers/media/platform/renesas-ceu.c
@@ -777,8 +777,15 @@ static int ceu_try_fmt(struct ceu_device *ceudev, struct v4l2_format *v4l2_fmt)
 	const struct ceu_fmt *ceu_fmt;
 	int ret;
 
+	/*
+	 * Set format on sensor sub device: bus format used to produce memory
+	 * format is selected at initialization time.
+	 */
 	struct v4l2_subdev_format sd_format = {
-		.which = V4L2_SUBDEV_FORMAT_TRY,
+		.which	= V4L2_SUBDEV_FORMAT_TRY,
+		.format	= {
+			.code = ceu_sd->mbus_fmt.mbus_code,
+		},
 	};
 
 	switch (pix->pixelformat) {
@@ -800,10 +807,6 @@ static int ceu_try_fmt(struct ceu_device *ceudev, struct v4l2_format *v4l2_fmt)
 	v4l_bound_align_image(&pix->width, 2, CEU_MAX_WIDTH, 4,
 			      &pix->height, 4, CEU_MAX_HEIGHT, 4, 0);
 
-	/*
-	 * Set format on sensor sub device: bus format used to produce memory
-	 * format is selected at initialization time.
-	 */
 	v4l2_fill_mbus_format_mplane(&sd_format.format, pix);
 	ret = v4l2_subdev_call(v4l2_sd, pad, set_fmt, &pad_cfg, &sd_format);
 	if (ret)
@@ -827,8 +830,15 @@ static int ceu_set_fmt(struct ceu_device *ceudev, struct v4l2_format *v4l2_fmt)
 	struct v4l2_subdev *v4l2_sd = ceu_sd->v4l2_sd;
 	int ret;
 
+	/*
+	 * Set format on sensor sub device: bus format used to produce memory
+	 * format is selected at initialization time.
+	 */
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.format	= {
+			.code = ceu_sd->mbus_fmt.mbus_code,
+		},
 	};
 
 	ret = ceu_try_fmt(ceudev, v4l2_fmt);
@@ -1365,7 +1375,7 @@ static int ceu_notify_complete(struct v4l2_async_notifier *notifier)
 		return ret;
 
 	/* Register the video device. */
-	strncpy(vdev->name, DRIVER_NAME, strlen(DRIVER_NAME));
+	strlcpy(vdev->name, DRIVER_NAME, sizeof(vdev->name));
 	vdev->v4l2_dev		= v4l2_dev;
 	vdev->lock		= &ceudev->mlock;
 	vdev->queue		= &ceudev->vb2_vq;
@@ -1545,6 +1555,7 @@ static const struct ceu_data ceu_data_sh4 = {
 #if IS_ENABLED(CONFIG_OF)
 static const struct of_device_id ceu_of_match[] = {
 	{ .compatible = "renesas,r7s72100-ceu", .data = &ceu_data_rz },
+	{ .compatible = "renesas,r8a7740-ceu", .data = &ceu_data_rz },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ceu_of_match);
diff --git a/drivers/media/platform/s5p-jpeg/jpeg-hw-exynos3250.c b/drivers/media/platform/s5p-jpeg/jpeg-hw-exynos3250.c
index 0974b9a7a584..0861842b2dfc 100644
--- a/drivers/media/platform/s5p-jpeg/jpeg-hw-exynos3250.c
+++ b/drivers/media/platform/s5p-jpeg/jpeg-hw-exynos3250.c
@@ -425,9 +425,9 @@ unsigned int exynos3250_jpeg_get_int_status(void __iomem *regs)
 }
 
 void exynos3250_jpeg_clear_int_status(void __iomem *regs,
-						unsigned int value)
+				      unsigned int value)
 {
-	return writel(value, regs + EXYNOS3250_JPGINTST);
+	writel(value, regs + EXYNOS3250_JPGINTST);
 }
 
 unsigned int exynos3250_jpeg_operating(void __iomem *regs)
diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c b/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c
index 5cf4d9921264..6a3cc4f86c5d 100644
--- a/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c
+++ b/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c
@@ -271,8 +271,8 @@ static int vidioc_querycap(struct file *file, void *priv,
 {
 	struct s5p_mfc_dev *dev = video_drvdata(file);
 
-	strncpy(cap->driver, S5P_MFC_NAME, sizeof(cap->driver) - 1);
-	strncpy(cap->card, dev->vfd_dec->name, sizeof(cap->card) - 1);
+	strlcpy(cap->driver, S5P_MFC_NAME, sizeof(cap->driver));
+	strlcpy(cap->card, dev->vfd_dec->name, sizeof(cap->card));
 	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
 		 dev_name(&dev->plat_dev->dev));
 	/*
diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc_enc.c b/drivers/media/platform/s5p-mfc/s5p_mfc_enc.c
index 5c0462ca9993..570f391f2cfd 100644
--- a/drivers/media/platform/s5p-mfc/s5p_mfc_enc.c
+++ b/drivers/media/platform/s5p-mfc/s5p_mfc_enc.c
@@ -1313,8 +1313,8 @@ static int vidioc_querycap(struct file *file, void *priv,
 {
 	struct s5p_mfc_dev *dev = video_drvdata(file);
 
-	strncpy(cap->driver, S5P_MFC_NAME, sizeof(cap->driver) - 1);
-	strncpy(cap->card, dev->vfd_enc->name, sizeof(cap->card) - 1);
+	strlcpy(cap->driver, S5P_MFC_NAME, sizeof(cap->driver));
+	strlcpy(cap->card, dev->vfd_enc->name, sizeof(cap->card));
 	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
 		 dev_name(&dev->plat_dev->dev));
 	/*
diff --git a/drivers/media/platform/soc_camera/Kconfig b/drivers/media/platform/soc_camera/Kconfig
index f5979c12ad61..669d116b8f09 100644
--- a/drivers/media/platform/soc_camera/Kconfig
+++ b/drivers/media/platform/soc_camera/Kconfig
@@ -18,9 +18,8 @@ config SOC_CAMERA_PLATFORM
 
 config VIDEO_SH_MOBILE_CEU
 	tristate "SuperH Mobile CEU Interface driver"
-	depends on VIDEO_DEV && SOC_CAMERA && HAS_DMA && HAVE_CLK
+	depends on VIDEO_DEV && SOC_CAMERA && HAVE_CLK
 	depends on ARCH_SHMOBILE || COMPILE_TEST
-	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select SOC_CAMERA_SCALE_CROP
 	---help---
diff --git a/drivers/media/platform/soc_camera/soc_camera_platform.c b/drivers/media/platform/soc_camera/soc_camera_platform.c
index cb4986b8f798..ce00e90d4e3c 100644
--- a/drivers/media/platform/soc_camera/soc_camera_platform.c
+++ b/drivers/media/platform/soc_camera/soc_camera_platform.c
@@ -159,7 +159,8 @@ static int soc_camera_platform_probe(struct platform_device *pdev)
 
 	v4l2_subdev_init(&priv->subdev, &platform_subdev_ops);
 	v4l2_set_subdevdata(&priv->subdev, p);
-	strncpy(priv->subdev.name, dev_name(&pdev->dev), V4L2_SUBDEV_NAME_SIZE);
+	strlcpy(priv->subdev.name, dev_name(&pdev->dev),
+		sizeof(priv->subdev.name));
 
 	return v4l2_device_register_subdev(&ici->v4l2_dev, &priv->subdev);
 }
diff --git a/drivers/media/platform/sti/bdisp/bdisp-hw.c b/drivers/media/platform/sti/bdisp/bdisp-hw.c
index a5eb592e12c0..26d9fa7aeb5f 100644
--- a/drivers/media/platform/sti/bdisp/bdisp-hw.c
+++ b/drivers/media/platform/sti/bdisp/bdisp-hw.c
@@ -455,7 +455,7 @@ int bdisp_hw_alloc_nodes(struct bdisp_ctx *ctx)
 
 	/* Allocate all the nodes within a single memory page */
 	base = dma_alloc_attrs(dev, node_size * MAX_NB_NODE, &paddr,
-			       GFP_KERNEL | GFP_DMA, DMA_ATTR_WRITE_COMBINE);
+			       GFP_KERNEL, DMA_ATTR_WRITE_COMBINE);
 	if (!base) {
 		dev_err(dev, "%s no mem\n", __func__);
 		return -ENOMEM;
diff --git a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
index bf4ca16db440..66b64096f5de 100644
--- a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
+++ b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
@@ -1297,6 +1297,10 @@ static int bdisp_probe(struct platform_device *pdev)
 	if (!bdisp)
 		return -ENOMEM;
 
+	ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (ret)
+		return ret;
+
 	bdisp->pdev = pdev;
 	bdisp->dev = dev;
 	platform_set_drvdata(pdev, bdisp);
diff --git a/drivers/media/platform/sti/c8sectpfe/Kconfig b/drivers/media/platform/sti/c8sectpfe/Kconfig
index 740190f8a3b6..7420a50572d3 100644
--- a/drivers/media/platform/sti/c8sectpfe/Kconfig
+++ b/drivers/media/platform/sti/c8sectpfe/Kconfig
@@ -1,6 +1,6 @@
 config DVB_C8SECTPFE
 	tristate "STMicroelectronics C8SECTPFE DVB support"
-	depends on PINCTRL && DVB_CORE && I2C && HAS_DMA
+	depends on PINCTRL && DVB_CORE && I2C
 	depends on ARCH_STI || ARCH_MULTIPLATFORM || COMPILE_TEST
 	select FW_LOADER
 	select DEBUG_FS
diff --git a/drivers/media/platform/sti/delta/delta-ipc.c b/drivers/media/platform/sti/delta/delta-ipc.c
index a4603d573c34..bd1bbbeedec3 100644
--- a/drivers/media/platform/sti/delta/delta-ipc.c
+++ b/drivers/media/platform/sti/delta/delta-ipc.c
@@ -175,8 +175,8 @@ int delta_ipc_open(struct delta_ctx *pctx, const char *name,
 	msg.ipc_buf_size = ipc_buf_size;
 	msg.ipc_buf_paddr = ctx->ipc_buf->paddr;
 
-	memcpy(msg.name, name, sizeof(msg.name));
-	msg.name[sizeof(msg.name) - 1] = 0;
+	memset(msg.name, 0, sizeof(msg.name));
+	strcpy(msg.name, name);
 
 	msg.param_size = param->size;
 	memcpy(ctx->ipc_buf->vaddr, param->data, msg.param_size);
diff --git a/drivers/media/platform/sti/hva/hva-mem.c b/drivers/media/platform/sti/hva/hva-mem.c
index caf50cd4bb77..68047b60b66c 100644
--- a/drivers/media/platform/sti/hva/hva-mem.c
+++ b/drivers/media/platform/sti/hva/hva-mem.c
@@ -22,7 +22,7 @@ int hva_mem_alloc(struct hva_ctx *ctx, u32 size, const char *name,
 		return -ENOMEM;
 	}
 
-	base = dma_alloc_attrs(dev, size, &paddr, GFP_KERNEL | GFP_DMA,
+	base = dma_alloc_attrs(dev, size, &paddr, GFP_KERNEL,
 			       DMA_ATTR_WRITE_COMBINE);
 	if (!base) {
 		dev_err(dev, "%s %s : dma_alloc_attrs failed for %s (size=%d)\n",
diff --git a/drivers/media/platform/sti/hva/hva-v4l2.c b/drivers/media/platform/sti/hva/hva-v4l2.c
index 2ab0b5cc5c22..15080cb00fa7 100644
--- a/drivers/media/platform/sti/hva/hva-v4l2.c
+++ b/drivers/media/platform/sti/hva/hva-v4l2.c
@@ -1355,6 +1355,10 @@ static int hva_probe(struct platform_device *pdev)
 		goto err;
 	}
 
+	ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (ret)
+		return ret;
+
 	hva->dev = dev;
 	hva->pdev = pdev;
 	platform_set_drvdata(pdev, hva);
diff --git a/drivers/media/platform/via-camera.c b/drivers/media/platform/via-camera.c
index e9a02639554b..c8bb82fe0b9d 100644
--- a/drivers/media/platform/via-camera.c
+++ b/drivers/media/platform/via-camera.c
@@ -27,7 +27,12 @@
 #include <linux/via-core.h>
 #include <linux/via-gpio.h>
 #include <linux/via_i2c.h>
+
+#ifdef CONFIG_X86
 #include <asm/olpc.h>
+#else
+#define machine_is_olpc(x) 0
+#endif
 
 #include "via-camera.h"
 
@@ -178,7 +183,7 @@ static int via_sensor_power_setup(struct via_camera *cam)
 
 	cam->power_gpio = viafb_gpio_lookup("VGPIO3");
 	cam->reset_gpio = viafb_gpio_lookup("VGPIO2");
-	if (cam->power_gpio < 0 || cam->reset_gpio < 0) {
+	if (!gpio_is_valid(cam->power_gpio) || !gpio_is_valid(cam->reset_gpio)) {
 		dev_err(&cam->platdev->dev, "Unable to find GPIO lines\n");
 		return -EINVAL;
 	}
diff --git a/drivers/media/platform/video-mux.c b/drivers/media/platform/video-mux.c
index ee89ad76bee2..1fb887293337 100644
--- a/drivers/media/platform/video-mux.c
+++ b/drivers/media/platform/video-mux.c
@@ -45,6 +45,7 @@ static int video_mux_link_setup(struct media_entity *entity,
 {
 	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(entity);
 	struct video_mux *vmux = v4l2_subdev_to_video_mux(sd);
+	u16 source_pad = entity->num_pads - 1;
 	int ret = 0;
 
 	/*
@@ -74,6 +75,9 @@ static int video_mux_link_setup(struct media_entity *entity,
 		if (ret < 0)
 			goto out;
 		vmux->active = local->index;
+
+		/* Propagate the active format to the source */
+		vmux->format_mbus[source_pad] = vmux->format_mbus[vmux->active];
 	} else {
 		if (vmux->active != local->index)
 			goto out;
@@ -162,14 +166,20 @@ static int video_mux_set_format(struct v4l2_subdev *sd,
 			    struct v4l2_subdev_format *sdformat)
 {
 	struct video_mux *vmux = v4l2_subdev_to_video_mux(sd);
-	struct v4l2_mbus_framefmt *mbusformat;
+	struct v4l2_mbus_framefmt *mbusformat, *source_mbusformat;
 	struct media_pad *pad = &vmux->pads[sdformat->pad];
+	u16 source_pad = sd->entity.num_pads - 1;
 
 	mbusformat = __video_mux_get_pad_format(sd, cfg, sdformat->pad,
 					    sdformat->which);
 	if (!mbusformat)
 		return -EINVAL;
 
+	source_mbusformat = __video_mux_get_pad_format(sd, cfg, source_pad,
+						       sdformat->which);
+	if (!source_mbusformat)
+		return -EINVAL;
+
 	mutex_lock(&vmux->lock);
 
 	/* Source pad mirrors active sink pad, no limitations on sink pads */
@@ -178,6 +188,10 @@ static int video_mux_set_format(struct v4l2_subdev *sd,
 
 	*mbusformat = sdformat->format;
 
+	/* Propagate the format from an active sink to source */
+	if ((pad->flags & MEDIA_PAD_FL_SINK) && (pad->index == vmux->active))
+		*source_mbusformat = sdformat->format;
+
 	mutex_unlock(&vmux->lock);
 
 	return 0;
diff --git a/drivers/media/platform/vivid/vivid-vid-common.c b/drivers/media/platform/vivid/vivid-vid-common.c
index e5914be0e12d..be531caa2cdf 100644
--- a/drivers/media/platform/vivid/vivid-vid-common.c
+++ b/drivers/media/platform/vivid/vivid-vid-common.c
@@ -860,7 +860,7 @@ int vidioc_g_edid(struct file *file, void *_fh,
 		return -ENODATA;
 	if (edid->start_block >= dev->edid_blocks)
 		return -EINVAL;
-	if (edid->start_block + edid->blocks > dev->edid_blocks)
+	if (edid->blocks > dev->edid_blocks - edid->start_block)
 		edid->blocks = dev->edid_blocks - edid->start_block;
 	if (adap)
 		cec_set_edid_phys_addr(dev->edid, dev->edid_blocks * 128, adap->phys_addr);
diff --git a/drivers/media/platform/vsp1/Makefile b/drivers/media/platform/vsp1/Makefile
index f5cd6f0491cb..4bb4dcbef7b5 100644
--- a/drivers/media/platform/vsp1/Makefile
+++ b/drivers/media/platform/vsp1/Makefile
@@ -3,8 +3,8 @@ vsp1-y					:= vsp1_drv.o vsp1_entity.o vsp1_pipe.o
 vsp1-y					+= vsp1_dl.o vsp1_drm.o vsp1_video.o
 vsp1-y					+= vsp1_rpf.o vsp1_rwpf.o vsp1_wpf.o
 vsp1-y					+= vsp1_clu.o vsp1_hsit.o vsp1_lut.o
-vsp1-y					+= vsp1_bru.o vsp1_sru.o vsp1_uds.o
+vsp1-y					+= vsp1_brx.o vsp1_sru.o vsp1_uds.o
 vsp1-y					+= vsp1_hgo.o vsp1_hgt.o vsp1_histo.o
-vsp1-y					+= vsp1_lif.o
+vsp1-y					+= vsp1_lif.o vsp1_uif.o
 
 obj-$(CONFIG_VIDEO_RENESAS_VSP1)	+= vsp1.o
diff --git a/drivers/media/platform/vsp1/vsp1.h b/drivers/media/platform/vsp1/vsp1.h
index 78ef838416b3..33f632331474 100644
--- a/drivers/media/platform/vsp1/vsp1.h
+++ b/drivers/media/platform/vsp1/vsp1.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1.h  --  R-Car VSP1 Driver
  *
  * Copyright (C) 2013-2014 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_H__
 #define __VSP1_H__
@@ -30,7 +26,7 @@ struct rcar_fcp_device;
 struct vsp1_drm;
 struct vsp1_entity;
 struct vsp1_platform_data;
-struct vsp1_bru;
+struct vsp1_brx;
 struct vsp1_clu;
 struct vsp1_hgo;
 struct vsp1_hgt;
@@ -40,10 +36,12 @@ struct vsp1_lut;
 struct vsp1_rwpf;
 struct vsp1_sru;
 struct vsp1_uds;
+struct vsp1_uif;
 
 #define VSP1_MAX_LIF		2
 #define VSP1_MAX_RPF		5
 #define VSP1_MAX_UDS		3
+#define VSP1_MAX_UIF		2
 #define VSP1_MAX_WPF		4
 
 #define VSP1_HAS_LUT		(1 << 1)
@@ -64,6 +62,7 @@ struct vsp1_device_info {
 	unsigned int lif_count;
 	unsigned int rpf_count;
 	unsigned int uds_count;
+	unsigned int uif_count;
 	unsigned int wpf_count;
 	unsigned int num_bru_inputs;
 	bool uapi;
@@ -78,8 +77,8 @@ struct vsp1_device {
 	struct rcar_fcp_device *fcp;
 	struct device *bus_master;
 
-	struct vsp1_bru *brs;
-	struct vsp1_bru *bru;
+	struct vsp1_brx *brs;
+	struct vsp1_brx *bru;
 	struct vsp1_clu *clu;
 	struct vsp1_hgo *hgo;
 	struct vsp1_hgt *hgt;
@@ -90,6 +89,7 @@ struct vsp1_device {
 	struct vsp1_rwpf *rpf[VSP1_MAX_RPF];
 	struct vsp1_sru *sru;
 	struct vsp1_uds *uds[VSP1_MAX_UDS];
+	struct vsp1_uif *uif[VSP1_MAX_UIF];
 	struct vsp1_rwpf *wpf[VSP1_MAX_WPF];
 
 	struct list_head entities;
diff --git a/drivers/media/platform/vsp1/vsp1_bru.c b/drivers/media/platform/vsp1/vsp1_bru.c
deleted file mode 100644
index e8fd2ae3b3eb..000000000000
--- a/drivers/media/platform/vsp1/vsp1_bru.c
+++ /dev/null
@@ -1,460 +0,0 @@
-/*
- * vsp1_bru.c  --  R-Car VSP1 Blend ROP Unit
- *
- * Copyright (C) 2013 Renesas Corporation
- *
- * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
-#include <linux/device.h>
-#include <linux/gfp.h>
-
-#include <media/v4l2-subdev.h>
-
-#include "vsp1.h"
-#include "vsp1_bru.h"
-#include "vsp1_dl.h"
-#include "vsp1_pipe.h"
-#include "vsp1_rwpf.h"
-#include "vsp1_video.h"
-
-#define BRU_MIN_SIZE				1U
-#define BRU_MAX_SIZE				8190U
-
-/* -----------------------------------------------------------------------------
- * Device Access
- */
-
-static inline void vsp1_bru_write(struct vsp1_bru *bru, struct vsp1_dl_list *dl,
-				  u32 reg, u32 data)
-{
-	vsp1_dl_list_write(dl, bru->base + reg, data);
-}
-
-/* -----------------------------------------------------------------------------
- * Controls
- */
-
-static int bru_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct vsp1_bru *bru =
-		container_of(ctrl->handler, struct vsp1_bru, ctrls);
-
-	switch (ctrl->id) {
-	case V4L2_CID_BG_COLOR:
-		bru->bgcolor = ctrl->val;
-		break;
-	}
-
-	return 0;
-}
-
-static const struct v4l2_ctrl_ops bru_ctrl_ops = {
-	.s_ctrl = bru_s_ctrl,
-};
-
-/* -----------------------------------------------------------------------------
- * V4L2 Subdevice Operations
- */
-
-/*
- * The BRU can't perform format conversion, all sink and source formats must be
- * identical. We pick the format on the first sink pad (pad 0) and propagate it
- * to all other pads.
- */
-
-static int bru_enum_mbus_code(struct v4l2_subdev *subdev,
-			      struct v4l2_subdev_pad_config *cfg,
-			      struct v4l2_subdev_mbus_code_enum *code)
-{
-	static const unsigned int codes[] = {
-		MEDIA_BUS_FMT_ARGB8888_1X32,
-		MEDIA_BUS_FMT_AYUV8_1X32,
-	};
-
-	return vsp1_subdev_enum_mbus_code(subdev, cfg, code, codes,
-					  ARRAY_SIZE(codes));
-}
-
-static int bru_enum_frame_size(struct v4l2_subdev *subdev,
-			       struct v4l2_subdev_pad_config *cfg,
-			       struct v4l2_subdev_frame_size_enum *fse)
-{
-	if (fse->index)
-		return -EINVAL;
-
-	if (fse->code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
-	    fse->code != MEDIA_BUS_FMT_AYUV8_1X32)
-		return -EINVAL;
-
-	fse->min_width = BRU_MIN_SIZE;
-	fse->max_width = BRU_MAX_SIZE;
-	fse->min_height = BRU_MIN_SIZE;
-	fse->max_height = BRU_MAX_SIZE;
-
-	return 0;
-}
-
-static struct v4l2_rect *bru_get_compose(struct vsp1_bru *bru,
-					 struct v4l2_subdev_pad_config *cfg,
-					 unsigned int pad)
-{
-	return v4l2_subdev_get_try_compose(&bru->entity.subdev, cfg, pad);
-}
-
-static void bru_try_format(struct vsp1_bru *bru,
-			   struct v4l2_subdev_pad_config *config,
-			   unsigned int pad, struct v4l2_mbus_framefmt *fmt)
-{
-	struct v4l2_mbus_framefmt *format;
-
-	switch (pad) {
-	case BRU_PAD_SINK(0):
-		/* Default to YUV if the requested format is not supported. */
-		if (fmt->code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
-		    fmt->code != MEDIA_BUS_FMT_AYUV8_1X32)
-			fmt->code = MEDIA_BUS_FMT_AYUV8_1X32;
-		break;
-
-	default:
-		/* The BRU can't perform format conversion. */
-		format = vsp1_entity_get_pad_format(&bru->entity, config,
-						    BRU_PAD_SINK(0));
-		fmt->code = format->code;
-		break;
-	}
-
-	fmt->width = clamp(fmt->width, BRU_MIN_SIZE, BRU_MAX_SIZE);
-	fmt->height = clamp(fmt->height, BRU_MIN_SIZE, BRU_MAX_SIZE);
-	fmt->field = V4L2_FIELD_NONE;
-	fmt->colorspace = V4L2_COLORSPACE_SRGB;
-}
-
-static int bru_set_format(struct v4l2_subdev *subdev,
-			  struct v4l2_subdev_pad_config *cfg,
-			  struct v4l2_subdev_format *fmt)
-{
-	struct vsp1_bru *bru = to_bru(subdev);
-	struct v4l2_subdev_pad_config *config;
-	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
-
-	mutex_lock(&bru->entity.lock);
-
-	config = vsp1_entity_get_pad_config(&bru->entity, cfg, fmt->which);
-	if (!config) {
-		ret = -EINVAL;
-		goto done;
-	}
-
-	bru_try_format(bru, config, fmt->pad, &fmt->format);
-
-	format = vsp1_entity_get_pad_format(&bru->entity, config, fmt->pad);
-	*format = fmt->format;
-
-	/* Reset the compose rectangle */
-	if (fmt->pad != bru->entity.source_pad) {
-		struct v4l2_rect *compose;
-
-		compose = bru_get_compose(bru, config, fmt->pad);
-		compose->left = 0;
-		compose->top = 0;
-		compose->width = format->width;
-		compose->height = format->height;
-	}
-
-	/* Propagate the format code to all pads */
-	if (fmt->pad == BRU_PAD_SINK(0)) {
-		unsigned int i;
-
-		for (i = 0; i <= bru->entity.source_pad; ++i) {
-			format = vsp1_entity_get_pad_format(&bru->entity,
-							    config, i);
-			format->code = fmt->format.code;
-		}
-	}
-
-done:
-	mutex_unlock(&bru->entity.lock);
-	return ret;
-}
-
-static int bru_get_selection(struct v4l2_subdev *subdev,
-			     struct v4l2_subdev_pad_config *cfg,
-			     struct v4l2_subdev_selection *sel)
-{
-	struct vsp1_bru *bru = to_bru(subdev);
-	struct v4l2_subdev_pad_config *config;
-
-	if (sel->pad == bru->entity.source_pad)
-		return -EINVAL;
-
-	switch (sel->target) {
-	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
-		sel->r.left = 0;
-		sel->r.top = 0;
-		sel->r.width = BRU_MAX_SIZE;
-		sel->r.height = BRU_MAX_SIZE;
-		return 0;
-
-	case V4L2_SEL_TGT_COMPOSE:
-		config = vsp1_entity_get_pad_config(&bru->entity, cfg,
-						    sel->which);
-		if (!config)
-			return -EINVAL;
-
-		mutex_lock(&bru->entity.lock);
-		sel->r = *bru_get_compose(bru, config, sel->pad);
-		mutex_unlock(&bru->entity.lock);
-		return 0;
-
-	default:
-		return -EINVAL;
-	}
-}
-
-static int bru_set_selection(struct v4l2_subdev *subdev,
-			     struct v4l2_subdev_pad_config *cfg,
-			     struct v4l2_subdev_selection *sel)
-{
-	struct vsp1_bru *bru = to_bru(subdev);
-	struct v4l2_subdev_pad_config *config;
-	struct v4l2_mbus_framefmt *format;
-	struct v4l2_rect *compose;
-	int ret = 0;
-
-	if (sel->pad == bru->entity.source_pad)
-		return -EINVAL;
-
-	if (sel->target != V4L2_SEL_TGT_COMPOSE)
-		return -EINVAL;
-
-	mutex_lock(&bru->entity.lock);
-
-	config = vsp1_entity_get_pad_config(&bru->entity, cfg, sel->which);
-	if (!config) {
-		ret = -EINVAL;
-		goto done;
-	}
-
-	/*
-	 * The compose rectangle top left corner must be inside the output
-	 * frame.
-	 */
-	format = vsp1_entity_get_pad_format(&bru->entity, config,
-					    bru->entity.source_pad);
-	sel->r.left = clamp_t(unsigned int, sel->r.left, 0, format->width - 1);
-	sel->r.top = clamp_t(unsigned int, sel->r.top, 0, format->height - 1);
-
-	/*
-	 * Scaling isn't supported, the compose rectangle size must be identical
-	 * to the sink format size.
-	 */
-	format = vsp1_entity_get_pad_format(&bru->entity, config, sel->pad);
-	sel->r.width = format->width;
-	sel->r.height = format->height;
-
-	compose = bru_get_compose(bru, config, sel->pad);
-	*compose = sel->r;
-
-done:
-	mutex_unlock(&bru->entity.lock);
-	return ret;
-}
-
-static const struct v4l2_subdev_pad_ops bru_pad_ops = {
-	.init_cfg = vsp1_entity_init_cfg,
-	.enum_mbus_code = bru_enum_mbus_code,
-	.enum_frame_size = bru_enum_frame_size,
-	.get_fmt = vsp1_subdev_get_pad_format,
-	.set_fmt = bru_set_format,
-	.get_selection = bru_get_selection,
-	.set_selection = bru_set_selection,
-};
-
-static const struct v4l2_subdev_ops bru_ops = {
-	.pad    = &bru_pad_ops,
-};
-
-/* -----------------------------------------------------------------------------
- * VSP1 Entity Operations
- */
-
-static void bru_configure(struct vsp1_entity *entity,
-			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl,
-			  enum vsp1_entity_params params)
-{
-	struct vsp1_bru *bru = to_bru(&entity->subdev);
-	struct v4l2_mbus_framefmt *format;
-	unsigned int flags;
-	unsigned int i;
-
-	if (params != VSP1_ENTITY_PARAMS_INIT)
-		return;
-
-	format = vsp1_entity_get_pad_format(&bru->entity, bru->entity.config,
-					    bru->entity.source_pad);
-
-	/*
-	 * The hardware is extremely flexible but we have no userspace API to
-	 * expose all the parameters, nor is it clear whether we would have use
-	 * cases for all the supported modes. Let's just harcode the parameters
-	 * to sane default values for now.
-	 */
-
-	/*
-	 * Disable dithering and enable color data normalization unless the
-	 * format at the pipeline output is premultiplied.
-	 */
-	flags = pipe->output ? pipe->output->format.flags : 0;
-	vsp1_bru_write(bru, dl, VI6_BRU_INCTRL,
-		       flags & V4L2_PIX_FMT_FLAG_PREMUL_ALPHA ?
-		       0 : VI6_BRU_INCTRL_NRM);
-
-	/*
-	 * Set the background position to cover the whole output image and
-	 * configure its color.
-	 */
-	vsp1_bru_write(bru, dl, VI6_BRU_VIRRPF_SIZE,
-		       (format->width << VI6_BRU_VIRRPF_SIZE_HSIZE_SHIFT) |
-		       (format->height << VI6_BRU_VIRRPF_SIZE_VSIZE_SHIFT));
-	vsp1_bru_write(bru, dl, VI6_BRU_VIRRPF_LOC, 0);
-
-	vsp1_bru_write(bru, dl, VI6_BRU_VIRRPF_COL, bru->bgcolor |
-		       (0xff << VI6_BRU_VIRRPF_COL_A_SHIFT));
-
-	/*
-	 * Route BRU input 1 as SRC input to the ROP unit and configure the ROP
-	 * unit with a NOP operation to make BRU input 1 available as the
-	 * Blend/ROP unit B SRC input. Only needed for BRU, the BRS has no ROP
-	 * unit.
-	 */
-	if (entity->type == VSP1_ENTITY_BRU)
-		vsp1_bru_write(bru, dl, VI6_BRU_ROP,
-			       VI6_BRU_ROP_DSTSEL_BRUIN(1) |
-			       VI6_BRU_ROP_CROP(VI6_ROP_NOP) |
-			       VI6_BRU_ROP_AROP(VI6_ROP_NOP));
-
-	for (i = 0; i < bru->entity.source_pad; ++i) {
-		bool premultiplied = false;
-		u32 ctrl = 0;
-
-		/*
-		 * Configure all Blend/ROP units corresponding to an enabled BRU
-		 * input for alpha blending. Blend/ROP units corresponding to
-		 * disabled BRU inputs are used in ROP NOP mode to ignore the
-		 * SRC input.
-		 */
-		if (bru->inputs[i].rpf) {
-			ctrl |= VI6_BRU_CTRL_RBC;
-
-			premultiplied = bru->inputs[i].rpf->format.flags
-				      & V4L2_PIX_FMT_FLAG_PREMUL_ALPHA;
-		} else {
-			ctrl |= VI6_BRU_CTRL_CROP(VI6_ROP_NOP)
-			     |  VI6_BRU_CTRL_AROP(VI6_ROP_NOP);
-		}
-
-		/*
-		 * Select the virtual RPF as the Blend/ROP unit A DST input to
-		 * serve as a background color.
-		 */
-		if (i == 0)
-			ctrl |= VI6_BRU_CTRL_DSTSEL_VRPF;
-
-		/*
-		 * Route inputs 0 to 3 as SRC inputs to Blend/ROP units A to D
-		 * in that order. In the BRU the Blend/ROP unit B SRC is
-		 * hardwired to the ROP unit output, the corresponding register
-		 * bits must be set to 0. The BRS has no ROP unit and doesn't
-		 * need any special processing.
-		 */
-		if (!(entity->type == VSP1_ENTITY_BRU && i == 1))
-			ctrl |= VI6_BRU_CTRL_SRCSEL_BRUIN(i);
-
-		vsp1_bru_write(bru, dl, VI6_BRU_CTRL(i), ctrl);
-
-		/*
-		 * Harcode the blending formula to
-		 *
-		 *	DSTc = DSTc * (1 - SRCa) + SRCc * SRCa
-		 *	DSTa = DSTa * (1 - SRCa) + SRCa
-		 *
-		 * when the SRC input isn't premultiplied, and to
-		 *
-		 *	DSTc = DSTc * (1 - SRCa) + SRCc
-		 *	DSTa = DSTa * (1 - SRCa) + SRCa
-		 *
-		 * otherwise.
-		 */
-		vsp1_bru_write(bru, dl, VI6_BRU_BLD(i),
-			       VI6_BRU_BLD_CCMDX_255_SRC_A |
-			       (premultiplied ? VI6_BRU_BLD_CCMDY_COEFY :
-						VI6_BRU_BLD_CCMDY_SRC_A) |
-			       VI6_BRU_BLD_ACMDX_255_SRC_A |
-			       VI6_BRU_BLD_ACMDY_COEFY |
-			       (0xff << VI6_BRU_BLD_COEFY_SHIFT));
-	}
-}
-
-static const struct vsp1_entity_operations bru_entity_ops = {
-	.configure = bru_configure,
-};
-
-/* -----------------------------------------------------------------------------
- * Initialization and Cleanup
- */
-
-struct vsp1_bru *vsp1_bru_create(struct vsp1_device *vsp1,
-				 enum vsp1_entity_type type)
-{
-	struct vsp1_bru *bru;
-	unsigned int num_pads;
-	const char *name;
-	int ret;
-
-	bru = devm_kzalloc(vsp1->dev, sizeof(*bru), GFP_KERNEL);
-	if (bru == NULL)
-		return ERR_PTR(-ENOMEM);
-
-	bru->base = type == VSP1_ENTITY_BRU ? VI6_BRU_BASE : VI6_BRS_BASE;
-	bru->entity.ops = &bru_entity_ops;
-	bru->entity.type = type;
-
-	if (type == VSP1_ENTITY_BRU) {
-		num_pads = vsp1->info->num_bru_inputs + 1;
-		name = "bru";
-	} else {
-		num_pads = 3;
-		name = "brs";
-	}
-
-	ret = vsp1_entity_init(vsp1, &bru->entity, name, num_pads, &bru_ops,
-			       MEDIA_ENT_F_PROC_VIDEO_COMPOSER);
-	if (ret < 0)
-		return ERR_PTR(ret);
-
-	/* Initialize the control handler. */
-	v4l2_ctrl_handler_init(&bru->ctrls, 1);
-	v4l2_ctrl_new_std(&bru->ctrls, &bru_ctrl_ops, V4L2_CID_BG_COLOR,
-			  0, 0xffffff, 1, 0);
-
-	bru->bgcolor = 0;
-
-	bru->entity.subdev.ctrl_handler = &bru->ctrls;
-
-	if (bru->ctrls.error) {
-		dev_err(vsp1->dev, "%s: failed to initialize controls\n", name);
-		ret = bru->ctrls.error;
-		vsp1_entity_destroy(&bru->entity);
-		return ERR_PTR(ret);
-	}
-
-	return bru;
-}
diff --git a/drivers/media/platform/vsp1/vsp1_bru.h b/drivers/media/platform/vsp1/vsp1_bru.h
deleted file mode 100644
index c98ed96d8de6..000000000000
--- a/drivers/media/platform/vsp1/vsp1_bru.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * vsp1_bru.h  --  R-Car VSP1 Blend ROP Unit
- *
- * Copyright (C) 2013 Renesas Corporation
- *
- * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-#ifndef __VSP1_BRU_H__
-#define __VSP1_BRU_H__
-
-#include <media/media-entity.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-subdev.h>
-
-#include "vsp1_entity.h"
-
-struct vsp1_device;
-struct vsp1_rwpf;
-
-#define BRU_PAD_SINK(n)				(n)
-
-struct vsp1_bru {
-	struct vsp1_entity entity;
-	unsigned int base;
-
-	struct v4l2_ctrl_handler ctrls;
-
-	struct {
-		struct vsp1_rwpf *rpf;
-	} inputs[VSP1_MAX_RPF];
-
-	u32 bgcolor;
-};
-
-static inline struct vsp1_bru *to_bru(struct v4l2_subdev *subdev)
-{
-	return container_of(subdev, struct vsp1_bru, entity.subdev);
-}
-
-struct vsp1_bru *vsp1_bru_create(struct vsp1_device *vsp1,
-				 enum vsp1_entity_type type);
-
-#endif /* __VSP1_BRU_H__ */
diff --git a/drivers/media/platform/vsp1/vsp1_brx.c b/drivers/media/platform/vsp1/vsp1_brx.c
new file mode 100644
index 000000000000..359917b5d842
--- /dev/null
+++ b/drivers/media/platform/vsp1/vsp1_brx.c
@@ -0,0 +1,452 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * vsp1_brx.c  --  R-Car VSP1 Blend ROP Unit (BRU and BRS)
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ */
+
+#include <linux/device.h>
+#include <linux/gfp.h>
+
+#include <media/v4l2-subdev.h>
+
+#include "vsp1.h"
+#include "vsp1_brx.h"
+#include "vsp1_dl.h"
+#include "vsp1_pipe.h"
+#include "vsp1_rwpf.h"
+#include "vsp1_video.h"
+
+#define BRX_MIN_SIZE				1U
+#define BRX_MAX_SIZE				8190U
+
+/* -----------------------------------------------------------------------------
+ * Device Access
+ */
+
+static inline void vsp1_brx_write(struct vsp1_brx *brx,
+				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
+{
+	vsp1_dl_body_write(dlb, brx->base + reg, data);
+}
+
+/* -----------------------------------------------------------------------------
+ * Controls
+ */
+
+static int brx_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct vsp1_brx *brx =
+		container_of(ctrl->handler, struct vsp1_brx, ctrls);
+
+	switch (ctrl->id) {
+	case V4L2_CID_BG_COLOR:
+		brx->bgcolor = ctrl->val;
+		break;
+	}
+
+	return 0;
+}
+
+static const struct v4l2_ctrl_ops brx_ctrl_ops = {
+	.s_ctrl = brx_s_ctrl,
+};
+
+/* -----------------------------------------------------------------------------
+ * V4L2 Subdevice Operations
+ */
+
+/*
+ * The BRx can't perform format conversion, all sink and source formats must be
+ * identical. We pick the format on the first sink pad (pad 0) and propagate it
+ * to all other pads.
+ */
+
+static int brx_enum_mbus_code(struct v4l2_subdev *subdev,
+			      struct v4l2_subdev_pad_config *cfg,
+			      struct v4l2_subdev_mbus_code_enum *code)
+{
+	static const unsigned int codes[] = {
+		MEDIA_BUS_FMT_ARGB8888_1X32,
+		MEDIA_BUS_FMT_AYUV8_1X32,
+	};
+
+	return vsp1_subdev_enum_mbus_code(subdev, cfg, code, codes,
+					  ARRAY_SIZE(codes));
+}
+
+static int brx_enum_frame_size(struct v4l2_subdev *subdev,
+			       struct v4l2_subdev_pad_config *cfg,
+			       struct v4l2_subdev_frame_size_enum *fse)
+{
+	if (fse->index)
+		return -EINVAL;
+
+	if (fse->code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
+	    fse->code != MEDIA_BUS_FMT_AYUV8_1X32)
+		return -EINVAL;
+
+	fse->min_width = BRX_MIN_SIZE;
+	fse->max_width = BRX_MAX_SIZE;
+	fse->min_height = BRX_MIN_SIZE;
+	fse->max_height = BRX_MAX_SIZE;
+
+	return 0;
+}
+
+static struct v4l2_rect *brx_get_compose(struct vsp1_brx *brx,
+					 struct v4l2_subdev_pad_config *cfg,
+					 unsigned int pad)
+{
+	return v4l2_subdev_get_try_compose(&brx->entity.subdev, cfg, pad);
+}
+
+static void brx_try_format(struct vsp1_brx *brx,
+			   struct v4l2_subdev_pad_config *config,
+			   unsigned int pad, struct v4l2_mbus_framefmt *fmt)
+{
+	struct v4l2_mbus_framefmt *format;
+
+	switch (pad) {
+	case BRX_PAD_SINK(0):
+		/* Default to YUV if the requested format is not supported. */
+		if (fmt->code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
+		    fmt->code != MEDIA_BUS_FMT_AYUV8_1X32)
+			fmt->code = MEDIA_BUS_FMT_AYUV8_1X32;
+		break;
+
+	default:
+		/* The BRx can't perform format conversion. */
+		format = vsp1_entity_get_pad_format(&brx->entity, config,
+						    BRX_PAD_SINK(0));
+		fmt->code = format->code;
+		break;
+	}
+
+	fmt->width = clamp(fmt->width, BRX_MIN_SIZE, BRX_MAX_SIZE);
+	fmt->height = clamp(fmt->height, BRX_MIN_SIZE, BRX_MAX_SIZE);
+	fmt->field = V4L2_FIELD_NONE;
+	fmt->colorspace = V4L2_COLORSPACE_SRGB;
+}
+
+static int brx_set_format(struct v4l2_subdev *subdev,
+			  struct v4l2_subdev_pad_config *cfg,
+			  struct v4l2_subdev_format *fmt)
+{
+	struct vsp1_brx *brx = to_brx(subdev);
+	struct v4l2_subdev_pad_config *config;
+	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
+
+	mutex_lock(&brx->entity.lock);
+
+	config = vsp1_entity_get_pad_config(&brx->entity, cfg, fmt->which);
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	brx_try_format(brx, config, fmt->pad, &fmt->format);
+
+	format = vsp1_entity_get_pad_format(&brx->entity, config, fmt->pad);
+	*format = fmt->format;
+
+	/* Reset the compose rectangle */
+	if (fmt->pad != brx->entity.source_pad) {
+		struct v4l2_rect *compose;
+
+		compose = brx_get_compose(brx, config, fmt->pad);
+		compose->left = 0;
+		compose->top = 0;
+		compose->width = format->width;
+		compose->height = format->height;
+	}
+
+	/* Propagate the format code to all pads */
+	if (fmt->pad == BRX_PAD_SINK(0)) {
+		unsigned int i;
+
+		for (i = 0; i <= brx->entity.source_pad; ++i) {
+			format = vsp1_entity_get_pad_format(&brx->entity,
+							    config, i);
+			format->code = fmt->format.code;
+		}
+	}
+
+done:
+	mutex_unlock(&brx->entity.lock);
+	return ret;
+}
+
+static int brx_get_selection(struct v4l2_subdev *subdev,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_selection *sel)
+{
+	struct vsp1_brx *brx = to_brx(subdev);
+	struct v4l2_subdev_pad_config *config;
+
+	if (sel->pad == brx->entity.source_pad)
+		return -EINVAL;
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = BRX_MAX_SIZE;
+		sel->r.height = BRX_MAX_SIZE;
+		return 0;
+
+	case V4L2_SEL_TGT_COMPOSE:
+		config = vsp1_entity_get_pad_config(&brx->entity, cfg,
+						    sel->which);
+		if (!config)
+			return -EINVAL;
+
+		mutex_lock(&brx->entity.lock);
+		sel->r = *brx_get_compose(brx, config, sel->pad);
+		mutex_unlock(&brx->entity.lock);
+		return 0;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int brx_set_selection(struct v4l2_subdev *subdev,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_selection *sel)
+{
+	struct vsp1_brx *brx = to_brx(subdev);
+	struct v4l2_subdev_pad_config *config;
+	struct v4l2_mbus_framefmt *format;
+	struct v4l2_rect *compose;
+	int ret = 0;
+
+	if (sel->pad == brx->entity.source_pad)
+		return -EINVAL;
+
+	if (sel->target != V4L2_SEL_TGT_COMPOSE)
+		return -EINVAL;
+
+	mutex_lock(&brx->entity.lock);
+
+	config = vsp1_entity_get_pad_config(&brx->entity, cfg, sel->which);
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	/*
+	 * The compose rectangle top left corner must be inside the output
+	 * frame.
+	 */
+	format = vsp1_entity_get_pad_format(&brx->entity, config,
+					    brx->entity.source_pad);
+	sel->r.left = clamp_t(unsigned int, sel->r.left, 0, format->width - 1);
+	sel->r.top = clamp_t(unsigned int, sel->r.top, 0, format->height - 1);
+
+	/*
+	 * Scaling isn't supported, the compose rectangle size must be identical
+	 * to the sink format size.
+	 */
+	format = vsp1_entity_get_pad_format(&brx->entity, config, sel->pad);
+	sel->r.width = format->width;
+	sel->r.height = format->height;
+
+	compose = brx_get_compose(brx, config, sel->pad);
+	*compose = sel->r;
+
+done:
+	mutex_unlock(&brx->entity.lock);
+	return ret;
+}
+
+static const struct v4l2_subdev_pad_ops brx_pad_ops = {
+	.init_cfg = vsp1_entity_init_cfg,
+	.enum_mbus_code = brx_enum_mbus_code,
+	.enum_frame_size = brx_enum_frame_size,
+	.get_fmt = vsp1_subdev_get_pad_format,
+	.set_fmt = brx_set_format,
+	.get_selection = brx_get_selection,
+	.set_selection = brx_set_selection,
+};
+
+static const struct v4l2_subdev_ops brx_ops = {
+	.pad    = &brx_pad_ops,
+};
+
+/* -----------------------------------------------------------------------------
+ * VSP1 Entity Operations
+ */
+
+static void brx_configure_stream(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_body *dlb)
+{
+	struct vsp1_brx *brx = to_brx(&entity->subdev);
+	struct v4l2_mbus_framefmt *format;
+	unsigned int flags;
+	unsigned int i;
+
+	format = vsp1_entity_get_pad_format(&brx->entity, brx->entity.config,
+					    brx->entity.source_pad);
+
+	/*
+	 * The hardware is extremely flexible but we have no userspace API to
+	 * expose all the parameters, nor is it clear whether we would have use
+	 * cases for all the supported modes. Let's just harcode the parameters
+	 * to sane default values for now.
+	 */
+
+	/*
+	 * Disable dithering and enable color data normalization unless the
+	 * format at the pipeline output is premultiplied.
+	 */
+	flags = pipe->output ? pipe->output->format.flags : 0;
+	vsp1_brx_write(brx, dlb, VI6_BRU_INCTRL,
+		       flags & V4L2_PIX_FMT_FLAG_PREMUL_ALPHA ?
+		       0 : VI6_BRU_INCTRL_NRM);
+
+	/*
+	 * Set the background position to cover the whole output image and
+	 * configure its color.
+	 */
+	vsp1_brx_write(brx, dlb, VI6_BRU_VIRRPF_SIZE,
+		       (format->width << VI6_BRU_VIRRPF_SIZE_HSIZE_SHIFT) |
+		       (format->height << VI6_BRU_VIRRPF_SIZE_VSIZE_SHIFT));
+	vsp1_brx_write(brx, dlb, VI6_BRU_VIRRPF_LOC, 0);
+
+	vsp1_brx_write(brx, dlb, VI6_BRU_VIRRPF_COL, brx->bgcolor |
+		       (0xff << VI6_BRU_VIRRPF_COL_A_SHIFT));
+
+	/*
+	 * Route BRU input 1 as SRC input to the ROP unit and configure the ROP
+	 * unit with a NOP operation to make BRU input 1 available as the
+	 * Blend/ROP unit B SRC input. Only needed for BRU, the BRS has no ROP
+	 * unit.
+	 */
+	if (entity->type == VSP1_ENTITY_BRU)
+		vsp1_brx_write(brx, dlb, VI6_BRU_ROP,
+			       VI6_BRU_ROP_DSTSEL_BRUIN(1) |
+			       VI6_BRU_ROP_CROP(VI6_ROP_NOP) |
+			       VI6_BRU_ROP_AROP(VI6_ROP_NOP));
+
+	for (i = 0; i < brx->entity.source_pad; ++i) {
+		bool premultiplied = false;
+		u32 ctrl = 0;
+
+		/*
+		 * Configure all Blend/ROP units corresponding to an enabled BRx
+		 * input for alpha blending. Blend/ROP units corresponding to
+		 * disabled BRx inputs are used in ROP NOP mode to ignore the
+		 * SRC input.
+		 */
+		if (brx->inputs[i].rpf) {
+			ctrl |= VI6_BRU_CTRL_RBC;
+
+			premultiplied = brx->inputs[i].rpf->format.flags
+				      & V4L2_PIX_FMT_FLAG_PREMUL_ALPHA;
+		} else {
+			ctrl |= VI6_BRU_CTRL_CROP(VI6_ROP_NOP)
+			     |  VI6_BRU_CTRL_AROP(VI6_ROP_NOP);
+		}
+
+		/*
+		 * Select the virtual RPF as the Blend/ROP unit A DST input to
+		 * serve as a background color.
+		 */
+		if (i == 0)
+			ctrl |= VI6_BRU_CTRL_DSTSEL_VRPF;
+
+		/*
+		 * Route inputs 0 to 3 as SRC inputs to Blend/ROP units A to D
+		 * in that order. In the BRU the Blend/ROP unit B SRC is
+		 * hardwired to the ROP unit output, the corresponding register
+		 * bits must be set to 0. The BRS has no ROP unit and doesn't
+		 * need any special processing.
+		 */
+		if (!(entity->type == VSP1_ENTITY_BRU && i == 1))
+			ctrl |= VI6_BRU_CTRL_SRCSEL_BRUIN(i);
+
+		vsp1_brx_write(brx, dlb, VI6_BRU_CTRL(i), ctrl);
+
+		/*
+		 * Harcode the blending formula to
+		 *
+		 *	DSTc = DSTc * (1 - SRCa) + SRCc * SRCa
+		 *	DSTa = DSTa * (1 - SRCa) + SRCa
+		 *
+		 * when the SRC input isn't premultiplied, and to
+		 *
+		 *	DSTc = DSTc * (1 - SRCa) + SRCc
+		 *	DSTa = DSTa * (1 - SRCa) + SRCa
+		 *
+		 * otherwise.
+		 */
+		vsp1_brx_write(brx, dlb, VI6_BRU_BLD(i),
+			       VI6_BRU_BLD_CCMDX_255_SRC_A |
+			       (premultiplied ? VI6_BRU_BLD_CCMDY_COEFY :
+						VI6_BRU_BLD_CCMDY_SRC_A) |
+			       VI6_BRU_BLD_ACMDX_255_SRC_A |
+			       VI6_BRU_BLD_ACMDY_COEFY |
+			       (0xff << VI6_BRU_BLD_COEFY_SHIFT));
+	}
+}
+
+static const struct vsp1_entity_operations brx_entity_ops = {
+	.configure_stream = brx_configure_stream,
+};
+
+/* -----------------------------------------------------------------------------
+ * Initialization and Cleanup
+ */
+
+struct vsp1_brx *vsp1_brx_create(struct vsp1_device *vsp1,
+				 enum vsp1_entity_type type)
+{
+	struct vsp1_brx *brx;
+	unsigned int num_pads;
+	const char *name;
+	int ret;
+
+	brx = devm_kzalloc(vsp1->dev, sizeof(*brx), GFP_KERNEL);
+	if (brx == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	brx->base = type == VSP1_ENTITY_BRU ? VI6_BRU_BASE : VI6_BRS_BASE;
+	brx->entity.ops = &brx_entity_ops;
+	brx->entity.type = type;
+
+	if (type == VSP1_ENTITY_BRU) {
+		num_pads = vsp1->info->num_bru_inputs + 1;
+		name = "bru";
+	} else {
+		num_pads = 3;
+		name = "brs";
+	}
+
+	ret = vsp1_entity_init(vsp1, &brx->entity, name, num_pads, &brx_ops,
+			       MEDIA_ENT_F_PROC_VIDEO_COMPOSER);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	/* Initialize the control handler. */
+	v4l2_ctrl_handler_init(&brx->ctrls, 1);
+	v4l2_ctrl_new_std(&brx->ctrls, &brx_ctrl_ops, V4L2_CID_BG_COLOR,
+			  0, 0xffffff, 1, 0);
+
+	brx->bgcolor = 0;
+
+	brx->entity.subdev.ctrl_handler = &brx->ctrls;
+
+	if (brx->ctrls.error) {
+		dev_err(vsp1->dev, "%s: failed to initialize controls\n", name);
+		ret = brx->ctrls.error;
+		vsp1_entity_destroy(&brx->entity);
+		return ERR_PTR(ret);
+	}
+
+	return brx;
+}
diff --git a/drivers/media/platform/vsp1/vsp1_brx.h b/drivers/media/platform/vsp1/vsp1_brx.h
new file mode 100644
index 000000000000..6abbb8c3343c
--- /dev/null
+++ b/drivers/media/platform/vsp1/vsp1_brx.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * vsp1_brx.h  --  R-Car VSP1 Blend ROP Unit (BRU and BRS)
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ */
+#ifndef __VSP1_BRX_H__
+#define __VSP1_BRX_H__
+
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-subdev.h>
+
+#include "vsp1_entity.h"
+
+struct vsp1_device;
+struct vsp1_rwpf;
+
+#define BRX_PAD_SINK(n)				(n)
+
+struct vsp1_brx {
+	struct vsp1_entity entity;
+	unsigned int base;
+
+	struct v4l2_ctrl_handler ctrls;
+
+	struct {
+		struct vsp1_rwpf *rpf;
+	} inputs[VSP1_MAX_RPF];
+
+	u32 bgcolor;
+};
+
+static inline struct vsp1_brx *to_brx(struct v4l2_subdev *subdev)
+{
+	return container_of(subdev, struct vsp1_brx, entity.subdev);
+}
+
+struct vsp1_brx *vsp1_brx_create(struct vsp1_device *vsp1,
+				 enum vsp1_entity_type type);
+
+#endif /* __VSP1_BRX_H__ */
diff --git a/drivers/media/platform/vsp1/vsp1_clu.c b/drivers/media/platform/vsp1/vsp1_clu.c
index f2fb26e5ab4e..942fc14c19d1 100644
--- a/drivers/media/platform/vsp1/vsp1_clu.c
+++ b/drivers/media/platform/vsp1/vsp1_clu.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_clu.c  --  R-Car VSP1 Cubic Look-Up Table
  *
  * Copyright (C) 2015-2016 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -23,14 +19,16 @@
 #define CLU_MIN_SIZE				4U
 #define CLU_MAX_SIZE				8190U
 
+#define CLU_SIZE				(17 * 17 * 17)
+
 /* -----------------------------------------------------------------------------
  * Device Access
  */
 
-static inline void vsp1_clu_write(struct vsp1_clu *clu, struct vsp1_dl_list *dl,
-				  u32 reg, u32 data)
+static inline void vsp1_clu_write(struct vsp1_clu *clu,
+				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg, data);
+	vsp1_dl_body_write(dlb, reg, data);
 }
 
 /* -----------------------------------------------------------------------------
@@ -47,19 +45,19 @@ static int clu_set_table(struct vsp1_clu *clu, struct v4l2_ctrl *ctrl)
 	struct vsp1_dl_body *dlb;
 	unsigned int i;
 
-	dlb = vsp1_dl_fragment_alloc(clu->entity.vsp1, 1 + 17 * 17 * 17);
+	dlb = vsp1_dl_body_get(clu->pool);
 	if (!dlb)
 		return -ENOMEM;
 
-	vsp1_dl_fragment_write(dlb, VI6_CLU_ADDR, 0);
-	for (i = 0; i < 17 * 17 * 17; ++i)
-		vsp1_dl_fragment_write(dlb, VI6_CLU_DATA, ctrl->p_new.p_u32[i]);
+	vsp1_dl_body_write(dlb, VI6_CLU_ADDR, 0);
+	for (i = 0; i < CLU_SIZE; ++i)
+		vsp1_dl_body_write(dlb, VI6_CLU_DATA, ctrl->p_new.p_u32[i]);
 
 	spin_lock_irq(&clu->lock);
 	swap(clu->clu, dlb);
 	spin_unlock_irq(&clu->lock);
 
-	vsp1_dl_fragment_free(dlb);
+	vsp1_dl_body_put(dlb);
 	return 0;
 }
 
@@ -118,18 +116,18 @@ static const struct v4l2_ctrl_config clu_mode_control = {
  * V4L2 Subdevice Pad Operations
  */
 
+static const unsigned int clu_codes[] = {
+	MEDIA_BUS_FMT_ARGB8888_1X32,
+	MEDIA_BUS_FMT_AHSV8888_1X32,
+	MEDIA_BUS_FMT_AYUV8_1X32,
+};
+
 static int clu_enum_mbus_code(struct v4l2_subdev *subdev,
 			      struct v4l2_subdev_pad_config *cfg,
 			      struct v4l2_subdev_mbus_code_enum *code)
 {
-	static const unsigned int codes[] = {
-		MEDIA_BUS_FMT_ARGB8888_1X32,
-		MEDIA_BUS_FMT_AHSV8888_1X32,
-		MEDIA_BUS_FMT_AYUV8_1X32,
-	};
-
-	return vsp1_subdev_enum_mbus_code(subdev, cfg, code, codes,
-					  ARRAY_SIZE(codes));
+	return vsp1_subdev_enum_mbus_code(subdev, cfg, code, clu_codes,
+					  ARRAY_SIZE(clu_codes));
 }
 
 static int clu_enum_frame_size(struct v4l2_subdev *subdev,
@@ -145,51 +143,10 @@ static int clu_set_format(struct v4l2_subdev *subdev,
 			  struct v4l2_subdev_pad_config *cfg,
 			  struct v4l2_subdev_format *fmt)
 {
-	struct vsp1_clu *clu = to_clu(subdev);
-	struct v4l2_subdev_pad_config *config;
-	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
-
-	mutex_lock(&clu->entity.lock);
-
-	config = vsp1_entity_get_pad_config(&clu->entity, cfg, fmt->which);
-	if (!config) {
-		ret = -EINVAL;
-		goto done;
-	}
-
-	/* Default to YUV if the requested format is not supported. */
-	if (fmt->format.code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
-	    fmt->format.code != MEDIA_BUS_FMT_AHSV8888_1X32 &&
-	    fmt->format.code != MEDIA_BUS_FMT_AYUV8_1X32)
-		fmt->format.code = MEDIA_BUS_FMT_AYUV8_1X32;
-
-	format = vsp1_entity_get_pad_format(&clu->entity, config, fmt->pad);
-
-	if (fmt->pad == CLU_PAD_SOURCE) {
-		/* The CLU output format can't be modified. */
-		fmt->format = *format;
-		goto done;
-	}
-
-	format->code = fmt->format.code;
-	format->width = clamp_t(unsigned int, fmt->format.width,
-				CLU_MIN_SIZE, CLU_MAX_SIZE);
-	format->height = clamp_t(unsigned int, fmt->format.height,
-				 CLU_MIN_SIZE, CLU_MAX_SIZE);
-	format->field = V4L2_FIELD_NONE;
-	format->colorspace = V4L2_COLORSPACE_SRGB;
-
-	fmt->format = *format;
-
-	/* Propagate the format to the source pad. */
-	format = vsp1_entity_get_pad_format(&clu->entity, config,
-					    CLU_PAD_SOURCE);
-	*format = fmt->format;
-
-done:
-	mutex_unlock(&clu->entity.lock);
-	return ret;
+	return vsp1_subdev_set_pad_format(subdev, cfg, fmt, clu_codes,
+					  ARRAY_SIZE(clu_codes),
+					  CLU_MIN_SIZE, CLU_MIN_SIZE,
+					  CLU_MAX_SIZE, CLU_MAX_SIZE);
 }
 
 /* -----------------------------------------------------------------------------
@@ -212,57 +169,65 @@ static const struct v4l2_subdev_ops clu_ops = {
  * VSP1 Entity Operations
  */
 
-static void clu_configure(struct vsp1_entity *entity,
-			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl,
-			  enum vsp1_entity_params params)
+static void clu_configure_stream(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_clu *clu = to_clu(&entity->subdev);
-	struct vsp1_dl_body *dlb;
+	struct v4l2_mbus_framefmt *format;
+
+	/*
+	 * The yuv_mode can't be changed during streaming. Cache it internally
+	 * for future runtime configuration calls.
+	 */
+	format = vsp1_entity_get_pad_format(&clu->entity,
+					    clu->entity.config,
+					    CLU_PAD_SINK);
+	clu->yuv_mode = format->code == MEDIA_BUS_FMT_AYUV8_1X32;
+}
+
+static void clu_configure_frame(struct vsp1_entity *entity,
+				struct vsp1_pipeline *pipe,
+				struct vsp1_dl_list *dl,
+				struct vsp1_dl_body *dlb)
+{
+	struct vsp1_clu *clu = to_clu(&entity->subdev);
+	struct vsp1_dl_body *clu_dlb;
 	unsigned long flags;
 	u32 ctrl = VI6_CLU_CTRL_AAI | VI6_CLU_CTRL_MVS | VI6_CLU_CTRL_EN;
 
-	switch (params) {
-	case VSP1_ENTITY_PARAMS_INIT: {
-		/*
-		 * The format can't be changed during streaming, only verify it
-		 * at setup time and store the information internally for future
-		 * runtime configuration calls.
-		 */
-		struct v4l2_mbus_framefmt *format;
-
-		format = vsp1_entity_get_pad_format(&clu->entity,
-						    clu->entity.config,
-						    CLU_PAD_SINK);
-		clu->yuv_mode = format->code == MEDIA_BUS_FMT_AYUV8_1X32;
-		break;
-	}
-
-	case VSP1_ENTITY_PARAMS_PARTITION:
-		break;
+	/* 2D mode can only be used with the YCbCr pixel encoding. */
+	if (clu->mode == V4L2_CID_VSP1_CLU_MODE_2D && clu->yuv_mode)
+		ctrl |= VI6_CLU_CTRL_AX1I_2D | VI6_CLU_CTRL_AX2I_2D
+		     |  VI6_CLU_CTRL_OS0_2D | VI6_CLU_CTRL_OS1_2D
+		     |  VI6_CLU_CTRL_OS2_2D | VI6_CLU_CTRL_M2D;
 
-	case VSP1_ENTITY_PARAMS_RUNTIME:
-		/* 2D mode can only be used with the YCbCr pixel encoding. */
-		if (clu->mode == V4L2_CID_VSP1_CLU_MODE_2D && clu->yuv_mode)
-			ctrl |= VI6_CLU_CTRL_AX1I_2D | VI6_CLU_CTRL_AX2I_2D
-			     |  VI6_CLU_CTRL_OS0_2D | VI6_CLU_CTRL_OS1_2D
-			     |  VI6_CLU_CTRL_OS2_2D | VI6_CLU_CTRL_M2D;
+	vsp1_clu_write(clu, dlb, VI6_CLU_CTRL, ctrl);
 
-		vsp1_clu_write(clu, dl, VI6_CLU_CTRL, ctrl);
+	spin_lock_irqsave(&clu->lock, flags);
+	clu_dlb = clu->clu;
+	clu->clu = NULL;
+	spin_unlock_irqrestore(&clu->lock, flags);
 
-		spin_lock_irqsave(&clu->lock, flags);
-		dlb = clu->clu;
-		clu->clu = NULL;
-		spin_unlock_irqrestore(&clu->lock, flags);
+	if (clu_dlb) {
+		vsp1_dl_list_add_body(dl, clu_dlb);
 
-		if (dlb)
-			vsp1_dl_list_add_fragment(dl, dlb);
-		break;
+		/* Release our local reference. */
+		vsp1_dl_body_put(clu_dlb);
 	}
 }
 
+static void clu_destroy(struct vsp1_entity *entity)
+{
+	struct vsp1_clu *clu = to_clu(&entity->subdev);
+
+	vsp1_dl_body_pool_destroy(clu->pool);
+}
+
 static const struct vsp1_entity_operations clu_entity_ops = {
-	.configure = clu_configure,
+	.configure_stream = clu_configure_stream,
+	.configure_frame = clu_configure_frame,
+	.destroy = clu_destroy,
 };
 
 /* -----------------------------------------------------------------------------
@@ -288,6 +253,17 @@ struct vsp1_clu *vsp1_clu_create(struct vsp1_device *vsp1)
 	if (ret < 0)
 		return ERR_PTR(ret);
 
+	/*
+	 * Pre-allocate a body pool, with 3 bodies allowing a userspace update
+	 * before the hardware has committed a previous set of tables, handling
+	 * both the queued and pending dl entries. One extra entry is added to
+	 * the CLU_SIZE to allow for the VI6_CLU_ADDR header.
+	 */
+	clu->pool = vsp1_dl_body_pool_create(clu->entity.vsp1, 3, CLU_SIZE + 1,
+					     0);
+	if (!clu->pool)
+		return ERR_PTR(-ENOMEM);
+
 	/* Initialize the control handler. */
 	v4l2_ctrl_handler_init(&clu->ctrls, 2);
 	v4l2_ctrl_new_custom(&clu->ctrls, &clu_table_control, NULL);
diff --git a/drivers/media/platform/vsp1/vsp1_clu.h b/drivers/media/platform/vsp1/vsp1_clu.h
index 036e0a2f1a42..cef2f44481ba 100644
--- a/drivers/media/platform/vsp1/vsp1_clu.h
+++ b/drivers/media/platform/vsp1/vsp1_clu.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_clu.h  --  R-Car VSP1 Cubic Look-Up Table
  *
  * Copyright (C) 2015 Renesas Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_CLU_H__
 #define __VSP1_CLU_H__
@@ -36,6 +32,7 @@ struct vsp1_clu {
 	spinlock_t lock;
 	unsigned int mode;
 	struct vsp1_dl_body *clu;
+	struct vsp1_dl_body_pool *pool;
 };
 
 static inline struct vsp1_clu *to_clu(struct v4l2_subdev *subdev)
diff --git a/drivers/media/platform/vsp1/vsp1_dl.c b/drivers/media/platform/vsp1/vsp1_dl.c
index 0b86ed01e85d..d9b9cdd8fbe2 100644
--- a/drivers/media/platform/vsp1/vsp1_dl.c
+++ b/drivers/media/platform/vsp1/vsp1_dl.c
@@ -1,19 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
- * vsp1_dl.h  --  R-Car VSP1 Display List
+ * vsp1_dl.c  --  R-Car VSP1 Display List
  *
  * Copyright (C) 2015 Renesas Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
 #include <linux/gfp.h>
+#include <linux/refcount.h>
 #include <linux/slab.h>
 #include <linux/workqueue.h>
 
@@ -45,14 +42,22 @@ struct vsp1_dl_entry {
 /**
  * struct vsp1_dl_body - Display list body
  * @list: entry in the display list list of bodies
+ * @free: entry in the pool free body list
+ * @pool: pool to which this body belongs
  * @vsp1: the VSP1 device
  * @entries: array of entries
  * @dma: DMA address of the entries
  * @size: size of the DMA memory in bytes
  * @num_entries: number of stored entries
+ * @max_entries: number of entries available
  */
 struct vsp1_dl_body {
 	struct list_head list;
+	struct list_head free;
+
+	refcount_t refcnt;
+
+	struct vsp1_dl_body_pool *pool;
 	struct vsp1_device *vsp1;
 
 	struct vsp1_dl_entry *entries;
@@ -60,6 +65,31 @@ struct vsp1_dl_body {
 	size_t size;
 
 	unsigned int num_entries;
+	unsigned int max_entries;
+};
+
+/**
+ * struct vsp1_dl_body_pool - display list body pool
+ * @dma: DMA address of the entries
+ * @size: size of the full DMA memory pool in bytes
+ * @mem: CPU memory pointer for the pool
+ * @bodies: Array of DLB structures for the pool
+ * @free: List of free DLB entries
+ * @lock: Protects the free list
+ * @vsp1: the VSP1 device
+ */
+struct vsp1_dl_body_pool {
+	/* DMA allocation */
+	dma_addr_t dma;
+	size_t size;
+	void *mem;
+
+	/* Body management */
+	struct vsp1_dl_body *bodies;
+	struct list_head free;
+	spinlock_t lock;
+
+	struct vsp1_device *vsp1;
 };
 
 /**
@@ -69,9 +99,10 @@ struct vsp1_dl_body {
  * @header: display list header, NULL for headerless lists
  * @dma: DMA address for the header
  * @body0: first display list body
- * @fragments: list of extra display list bodies
+ * @bodies: list of extra display list bodies
  * @has_chain: if true, indicates that there's a partition chain
  * @chain: entry in the display list partition chain
+ * @internal: whether the display list is used for internal purpose
  */
 struct vsp1_dl_list {
 	struct list_head list;
@@ -80,11 +111,13 @@ struct vsp1_dl_list {
 	struct vsp1_dl_header *header;
 	dma_addr_t dma;
 
-	struct vsp1_dl_body body0;
-	struct list_head fragments;
+	struct vsp1_dl_body *body0;
+	struct list_head bodies;
 
 	bool has_chain;
 	struct list_head chain;
+
+	bool internal;
 };
 
 enum vsp1_dl_mode {
@@ -98,13 +131,12 @@ enum vsp1_dl_mode {
  * @mode: display list operation mode (header or headerless)
  * @singleshot: execute the display list in single-shot mode
  * @vsp1: the VSP1 device
- * @lock: protects the free, active, queued, pending and gc_fragments lists
+ * @lock: protects the free, active, queued, and pending lists
  * @free: array of all free display lists
  * @active: list currently being processed (loaded) by hardware
  * @queued: list queued to the hardware (written to the DL registers)
  * @pending: list waiting to be queued to the hardware
- * @gc_work: fragments garbage collector work struct
- * @gc_fragments: array of display list fragments waiting to be freed
+ * @pool: body pool for the display list bodies
  */
 struct vsp1_dl_manager {
 	unsigned int index;
@@ -118,109 +150,164 @@ struct vsp1_dl_manager {
 	struct vsp1_dl_list *queued;
 	struct vsp1_dl_list *pending;
 
-	struct work_struct gc_work;
-	struct list_head gc_fragments;
+	struct vsp1_dl_body_pool *pool;
 };
 
 /* -----------------------------------------------------------------------------
  * Display List Body Management
  */
 
-/*
- * Initialize a display list body object and allocate DMA memory for the body
- * data. The display list body object is expected to have been initialized to
- * 0 when allocated.
+/**
+ * vsp1_dl_body_pool_create - Create a pool of bodies from a single allocation
+ * @vsp1: The VSP1 device
+ * @num_bodies: The number of bodies to allocate
+ * @num_entries: The maximum number of entries that a body can contain
+ * @extra_size: Extra allocation provided for the bodies
+ *
+ * Allocate a pool of display list bodies each with enough memory to contain the
+ * requested number of entries plus the @extra_size.
+ *
+ * Return a pointer to a pool on success or NULL if memory can't be allocated.
  */
-static int vsp1_dl_body_init(struct vsp1_device *vsp1,
-			     struct vsp1_dl_body *dlb, unsigned int num_entries,
-			     size_t extra_size)
+struct vsp1_dl_body_pool *
+vsp1_dl_body_pool_create(struct vsp1_device *vsp1, unsigned int num_bodies,
+			 unsigned int num_entries, size_t extra_size)
 {
-	size_t size = num_entries * sizeof(*dlb->entries) + extra_size;
+	struct vsp1_dl_body_pool *pool;
+	size_t dlb_size;
+	unsigned int i;
 
-	dlb->vsp1 = vsp1;
-	dlb->size = size;
+	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
+	if (!pool)
+		return NULL;
 
-	dlb->entries = dma_alloc_wc(vsp1->bus_master, dlb->size, &dlb->dma,
-				    GFP_KERNEL);
-	if (!dlb->entries)
-		return -ENOMEM;
+	pool->vsp1 = vsp1;
 
-	return 0;
+	/*
+	 * TODO: 'extra_size' is only used by vsp1_dlm_create(), to allocate
+	 * extra memory for the display list header. We need only one header per
+	 * display list, not per display list body, thus this allocation is
+	 * extraneous and should be reworked in the future.
+	 */
+	dlb_size = num_entries * sizeof(struct vsp1_dl_entry) + extra_size;
+	pool->size = dlb_size * num_bodies;
+
+	pool->bodies = kcalloc(num_bodies, sizeof(*pool->bodies), GFP_KERNEL);
+	if (!pool->bodies) {
+		kfree(pool);
+		return NULL;
+	}
+
+	pool->mem = dma_alloc_wc(vsp1->bus_master, pool->size, &pool->dma,
+				 GFP_KERNEL);
+	if (!pool->mem) {
+		kfree(pool->bodies);
+		kfree(pool);
+		return NULL;
+	}
+
+	spin_lock_init(&pool->lock);
+	INIT_LIST_HEAD(&pool->free);
+
+	for (i = 0; i < num_bodies; ++i) {
+		struct vsp1_dl_body *dlb = &pool->bodies[i];
+
+		dlb->pool = pool;
+		dlb->max_entries = num_entries;
+
+		dlb->dma = pool->dma + i * dlb_size;
+		dlb->entries = pool->mem + i * dlb_size;
+
+		list_add_tail(&dlb->free, &pool->free);
+	}
+
+	return pool;
 }
 
-/*
- * Cleanup a display list body and free allocated DMA memory allocated.
+/**
+ * vsp1_dl_body_pool_destroy - Release a body pool
+ * @pool: The body pool
+ *
+ * Release all components of a pool allocation.
  */
-static void vsp1_dl_body_cleanup(struct vsp1_dl_body *dlb)
+void vsp1_dl_body_pool_destroy(struct vsp1_dl_body_pool *pool)
 {
-	dma_free_wc(dlb->vsp1->bus_master, dlb->size, dlb->entries, dlb->dma);
+	if (!pool)
+		return;
+
+	if (pool->mem)
+		dma_free_wc(pool->vsp1->bus_master, pool->size, pool->mem,
+			    pool->dma);
+
+	kfree(pool->bodies);
+	kfree(pool);
 }
 
 /**
- * vsp1_dl_fragment_alloc - Allocate a display list fragment
- * @vsp1: The VSP1 device
- * @num_entries: The maximum number of entries that the fragment can contain
+ * vsp1_dl_body_get - Obtain a body from a pool
+ * @pool: The body pool
  *
- * Allocate a display list fragment with enough memory to contain the requested
- * number of entries.
+ * Obtain a body from the pool without blocking.
  *
- * Return a pointer to a fragment on success or NULL if memory can't be
- * allocated.
+ * Returns a display list body or NULL if there are none available.
  */
-struct vsp1_dl_body *vsp1_dl_fragment_alloc(struct vsp1_device *vsp1,
-					    unsigned int num_entries)
+struct vsp1_dl_body *vsp1_dl_body_get(struct vsp1_dl_body_pool *pool)
 {
-	struct vsp1_dl_body *dlb;
-	int ret;
+	struct vsp1_dl_body *dlb = NULL;
+	unsigned long flags;
 
-	dlb = kzalloc(sizeof(*dlb), GFP_KERNEL);
-	if (!dlb)
-		return NULL;
+	spin_lock_irqsave(&pool->lock, flags);
 
-	ret = vsp1_dl_body_init(vsp1, dlb, num_entries, 0);
-	if (ret < 0) {
-		kfree(dlb);
-		return NULL;
+	if (!list_empty(&pool->free)) {
+		dlb = list_first_entry(&pool->free, struct vsp1_dl_body, free);
+		list_del(&dlb->free);
+		refcount_set(&dlb->refcnt, 1);
 	}
 
+	spin_unlock_irqrestore(&pool->lock, flags);
+
 	return dlb;
 }
 
 /**
- * vsp1_dl_fragment_free - Free a display list fragment
- * @dlb: The fragment
- *
- * Free the given display list fragment and the associated DMA memory.
- *
- * Fragments must only be freed explicitly if they are not added to a display
- * list, as the display list will take ownership of them and free them
- * otherwise. Manual free typically happens at cleanup time for fragments that
- * have been allocated but not used.
+ * vsp1_dl_body_put - Return a body back to its pool
+ * @dlb: The display list body
  *
- * Passing a NULL pointer to this function is safe, in that case no operation
- * will be performed.
+ * Return a body back to the pool, and reset the num_entries to clear the list.
  */
-void vsp1_dl_fragment_free(struct vsp1_dl_body *dlb)
+void vsp1_dl_body_put(struct vsp1_dl_body *dlb)
 {
+	unsigned long flags;
+
 	if (!dlb)
 		return;
 
-	vsp1_dl_body_cleanup(dlb);
-	kfree(dlb);
+	if (!refcount_dec_and_test(&dlb->refcnt))
+		return;
+
+	dlb->num_entries = 0;
+
+	spin_lock_irqsave(&dlb->pool->lock, flags);
+	list_add_tail(&dlb->free, &dlb->pool->free);
+	spin_unlock_irqrestore(&dlb->pool->lock, flags);
 }
 
 /**
- * vsp1_dl_fragment_write - Write a register to a display list fragment
- * @dlb: The fragment
+ * vsp1_dl_body_write - Write a register to a display list body
+ * @dlb: The body
  * @reg: The register address
  * @data: The register value
  *
- * Write the given register and value to the display list fragment. The maximum
- * number of entries that can be written in a fragment is specified when the
- * fragment is allocated by vsp1_dl_fragment_alloc().
+ * Write the given register and value to the display list body. The maximum
+ * number of entries that can be written in a body is specified when the body is
+ * allocated by vsp1_dl_body_alloc().
  */
-void vsp1_dl_fragment_write(struct vsp1_dl_body *dlb, u32 reg, u32 data)
+void vsp1_dl_body_write(struct vsp1_dl_body *dlb, u32 reg, u32 data)
 {
+	if (WARN_ONCE(dlb->num_entries >= dlb->max_entries,
+		      "DLB size exceeded (max %u)", dlb->max_entries))
+		return;
+
 	dlb->entries[dlb->num_entries].addr = reg;
 	dlb->entries[dlb->num_entries].data = data;
 	dlb->num_entries++;
@@ -233,51 +320,47 @@ void vsp1_dl_fragment_write(struct vsp1_dl_body *dlb, u32 reg, u32 data)
 static struct vsp1_dl_list *vsp1_dl_list_alloc(struct vsp1_dl_manager *dlm)
 {
 	struct vsp1_dl_list *dl;
-	size_t header_size;
-	int ret;
 
 	dl = kzalloc(sizeof(*dl), GFP_KERNEL);
 	if (!dl)
 		return NULL;
 
-	INIT_LIST_HEAD(&dl->fragments);
+	INIT_LIST_HEAD(&dl->bodies);
 	dl->dlm = dlm;
 
-	/*
-	 * Initialize the display list body and allocate DMA memory for the body
-	 * and the optional header. Both are allocated together to avoid memory
-	 * fragmentation, with the header located right after the body in
-	 * memory.
-	 */
-	header_size = dlm->mode == VSP1_DL_MODE_HEADER
-		    ? ALIGN(sizeof(struct vsp1_dl_header), 8)
-		    : 0;
-
-	ret = vsp1_dl_body_init(dlm->vsp1, &dl->body0, VSP1_DL_NUM_ENTRIES,
-				header_size);
-	if (ret < 0) {
-		kfree(dl);
+	/* Get a default body for our list. */
+	dl->body0 = vsp1_dl_body_get(dlm->pool);
+	if (!dl->body0)
 		return NULL;
-	}
-
 	if (dlm->mode == VSP1_DL_MODE_HEADER) {
-		size_t header_offset = VSP1_DL_NUM_ENTRIES
-				     * sizeof(*dl->body0.entries);
+		size_t header_offset = dl->body0->max_entries
+				     * sizeof(*dl->body0->entries);
 
-		dl->header = ((void *)dl->body0.entries) + header_offset;
-		dl->dma = dl->body0.dma + header_offset;
+		dl->header = ((void *)dl->body0->entries) + header_offset;
+		dl->dma = dl->body0->dma + header_offset;
 
 		memset(dl->header, 0, sizeof(*dl->header));
-		dl->header->lists[0].addr = dl->body0.dma;
+		dl->header->lists[0].addr = dl->body0->dma;
 	}
 
 	return dl;
 }
 
+static void vsp1_dl_list_bodies_put(struct vsp1_dl_list *dl)
+{
+	struct vsp1_dl_body *dlb, *tmp;
+
+	list_for_each_entry_safe(dlb, tmp, &dl->bodies, list) {
+		list_del(&dlb->list);
+		vsp1_dl_body_put(dlb);
+	}
+}
+
 static void vsp1_dl_list_free(struct vsp1_dl_list *dl)
 {
-	vsp1_dl_body_cleanup(&dl->body0);
-	list_splice_init(&dl->fragments, &dl->dlm->gc_fragments);
+	vsp1_dl_body_put(dl->body0);
+	vsp1_dl_list_bodies_put(dl);
+
 	kfree(dl);
 }
 
@@ -331,18 +414,13 @@ static void __vsp1_dl_list_put(struct vsp1_dl_list *dl)
 
 	dl->has_chain = false;
 
+	vsp1_dl_list_bodies_put(dl);
+
 	/*
-	 * We can't free fragments here as DMA memory can only be freed in
-	 * interruptible context. Move all fragments to the display list
-	 * manager's list of fragments to be freed, they will be
-	 * garbage-collected by the work queue.
+	 * body0 is reused as as an optimisation as presently every display list
+	 * has at least one body, thus we reinitialise the entries list.
 	 */
-	if (!list_empty(&dl->fragments)) {
-		list_splice_init(&dl->fragments, &dl->dlm->gc_fragments);
-		schedule_work(&dl->dlm->gc_work);
-	}
-
-	dl->body0.num_entries = 0;
+	dl->body0->num_entries = 0;
 
 	list_add_tail(&dl->list, &dl->dlm->free);
 }
@@ -369,43 +447,46 @@ void vsp1_dl_list_put(struct vsp1_dl_list *dl)
 }
 
 /**
- * vsp1_dl_list_write - Write a register to the display list
+ * vsp1_dl_list_get_body0 - Obtain the default body for the display list
  * @dl: The display list
- * @reg: The register address
- * @data: The register value
  *
- * Write the given register and value to the display list. Up to 256 registers
- * can be written per display list.
+ * Obtain a pointer to the internal display list body allowing this to be passed
+ * directly to configure operations.
  */
-void vsp1_dl_list_write(struct vsp1_dl_list *dl, u32 reg, u32 data)
+struct vsp1_dl_body *vsp1_dl_list_get_body0(struct vsp1_dl_list *dl)
 {
-	vsp1_dl_fragment_write(&dl->body0, reg, data);
+	return dl->body0;
 }
 
 /**
- * vsp1_dl_list_add_fragment - Add a fragment to the display list
+ * vsp1_dl_list_add_body - Add a body to the display list
  * @dl: The display list
- * @dlb: The fragment
+ * @dlb: The body
+ *
+ * Add a display list body to a display list. Registers contained in bodies are
+ * processed after registers contained in the main display list, in the order in
+ * which bodies are added.
  *
- * Add a display list body as a fragment to a display list. Registers contained
- * in fragments are processed after registers contained in the main display
- * list, in the order in which fragments are added.
+ * Adding a body to a display list passes ownership of the body to the list. The
+ * caller retains its reference to the fragment when adding it to the display
+ * list, but is not allowed to add new entries to the body.
  *
- * Adding a fragment to a display list passes ownership of the fragment to the
- * list. The caller must not touch the fragment after this call, and must not
- * free it explicitly with vsp1_dl_fragment_free().
+ * The reference must be explicitly released by a call to vsp1_dl_body_put()
+ * when the body isn't needed anymore.
  *
- * Fragments are only usable for display lists in header mode. Attempt to
- * add a fragment to a header-less display list will return an error.
+ * Additional bodies are only usable for display lists in header mode.
+ * Attempting to add a body to a header-less display list will return an error.
  */
-int vsp1_dl_list_add_fragment(struct vsp1_dl_list *dl,
-			      struct vsp1_dl_body *dlb)
+int vsp1_dl_list_add_body(struct vsp1_dl_list *dl, struct vsp1_dl_body *dlb)
 {
 	/* Multi-body lists are only available in header mode. */
 	if (dl->dlm->mode != VSP1_DL_MODE_HEADER)
 		return -EINVAL;
 
-	list_add_tail(&dlb->list, &dl->fragments);
+	refcount_inc(&dlb->refcnt);
+
+	list_add_tail(&dlb->list, &dl->bodies);
+
 	return 0;
 }
 
@@ -451,10 +532,10 @@ static void vsp1_dl_list_fill_header(struct vsp1_dl_list *dl, bool is_last)
 	 * list was allocated.
 	 */
 
-	hdr->num_bytes = dl->body0.num_entries
+	hdr->num_bytes = dl->body0->num_entries
 		       * sizeof(*dl->header->lists);
 
-	list_for_each_entry(dlb, &dl->fragments, list) {
+	list_for_each_entry(dlb, &dl->bodies, list) {
 		num_lists++;
 		hdr++;
 
@@ -525,9 +606,9 @@ static void vsp1_dl_list_hw_enqueue(struct vsp1_dl_list *dl)
 		 * bit will be cleared by the hardware when the display list
 		 * processing starts.
 		 */
-		vsp1_write(vsp1, VI6_DL_HDR_ADDR(0), dl->body0.dma);
+		vsp1_write(vsp1, VI6_DL_HDR_ADDR(0), dl->body0->dma);
 		vsp1_write(vsp1, VI6_DL_BODY_SIZE, VI6_DL_BODY_SIZE_UPD |
-			   (dl->body0.num_entries * sizeof(*dl->header->lists)));
+			(dl->body0->num_entries * sizeof(*dl->header->lists)));
 	} else {
 		/*
 		 * In header mode, program the display list header address. If
@@ -550,8 +631,16 @@ static void vsp1_dl_list_commit_continuous(struct vsp1_dl_list *dl)
 	 * case we can't replace the queued list by the new one, as we could
 	 * race with the hardware. We thus mark the update as pending, it will
 	 * be queued up to the hardware by the frame end interrupt handler.
+	 *
+	 * If a display list is already pending we simply drop it as the new
+	 * display list is assumed to contain a more recent configuration. It is
+	 * an error if the already pending list has the internal flag set, as
+	 * there is then a process waiting for that list to complete. This
+	 * shouldn't happen as the waiting process should perform proper
+	 * locking, but warn just in case.
 	 */
 	if (vsp1_dl_list_hw_update_pending(dlm)) {
+		WARN_ON(dlm->pending && dlm->pending->internal);
 		__vsp1_dl_list_put(dlm->pending);
 		dlm->pending = dl;
 		return;
@@ -581,7 +670,7 @@ static void vsp1_dl_list_commit_singleshot(struct vsp1_dl_list *dl)
 	dlm->active = dl;
 }
 
-void vsp1_dl_list_commit(struct vsp1_dl_list *dl)
+void vsp1_dl_list_commit(struct vsp1_dl_list *dl, bool internal)
 {
 	struct vsp1_dl_manager *dlm = dl->dlm;
 	struct vsp1_dl_list *dl_child;
@@ -598,6 +687,8 @@ void vsp1_dl_list_commit(struct vsp1_dl_list *dl)
 		}
 	}
 
+	dl->internal = internal;
+
 	spin_lock_irqsave(&dlm->lock, flags);
 
 	if (dlm->singleshot)
@@ -616,14 +707,22 @@ void vsp1_dl_list_commit(struct vsp1_dl_list *dl)
  * vsp1_dlm_irq_frame_end - Display list handler for the frame end interrupt
  * @dlm: the display list manager
  *
- * Return true if the previous display list has completed at frame end, or false
- * if it has been delayed by one frame because the display list commit raced
- * with the frame end interrupt. The function always returns true in header mode
- * as display list processing is then not continuous and races never occur.
+ * Return a set of flags that indicates display list completion status.
+ *
+ * The VSP1_DL_FRAME_END_COMPLETED flag indicates that the previous display list
+ * has completed at frame end. If the flag is not returned display list
+ * completion has been delayed by one frame because the display list commit
+ * raced with the frame end interrupt. The function always returns with the flag
+ * set in header mode as display list processing is then not continuous and
+ * races never occur.
+ *
+ * The VSP1_DL_FRAME_END_INTERNAL flag indicates that the previous display list
+ * has completed and had been queued with the internal notification flag.
+ * Internal notification is only supported for continuous mode.
  */
-bool vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
+unsigned int vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 {
-	bool completed = false;
+	unsigned int flags = 0;
 
 	spin_lock(&dlm->lock);
 
@@ -634,7 +733,7 @@ bool vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 	if (dlm->singleshot) {
 		__vsp1_dl_list_put(dlm->active);
 		dlm->active = NULL;
-		completed = true;
+		flags |= VSP1_DL_FRAME_END_COMPLETED;
 		goto done;
 	}
 
@@ -652,10 +751,14 @@ bool vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 	 * frame end interrupt. The display list thus becomes active.
 	 */
 	if (dlm->queued) {
+		if (dlm->queued->internal)
+			flags |= VSP1_DL_FRAME_END_INTERNAL;
+		dlm->queued->internal = false;
+
 		__vsp1_dl_list_put(dlm->active);
 		dlm->active = dlm->queued;
 		dlm->queued = NULL;
-		completed = true;
+		flags |= VSP1_DL_FRAME_END_COMPLETED;
 	}
 
 	/*
@@ -672,7 +775,7 @@ bool vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 done:
 	spin_unlock(&dlm->lock);
 
-	return completed;
+	return flags;
 }
 
 /* Hardware Setup */
@@ -710,38 +813,9 @@ void vsp1_dlm_reset(struct vsp1_dl_manager *dlm)
 	dlm->pending = NULL;
 }
 
-/*
- * Free all fragments awaiting to be garbage-collected.
- *
- * This function must be called without the display list manager lock held.
- */
-static void vsp1_dlm_fragments_free(struct vsp1_dl_manager *dlm)
+struct vsp1_dl_body *vsp1_dlm_dl_body_get(struct vsp1_dl_manager *dlm)
 {
-	unsigned long flags;
-
-	spin_lock_irqsave(&dlm->lock, flags);
-
-	while (!list_empty(&dlm->gc_fragments)) {
-		struct vsp1_dl_body *dlb;
-
-		dlb = list_first_entry(&dlm->gc_fragments, struct vsp1_dl_body,
-				       list);
-		list_del(&dlb->list);
-
-		spin_unlock_irqrestore(&dlm->lock, flags);
-		vsp1_dl_fragment_free(dlb);
-		spin_lock_irqsave(&dlm->lock, flags);
-	}
-
-	spin_unlock_irqrestore(&dlm->lock, flags);
-}
-
-static void vsp1_dlm_garbage_collect(struct work_struct *work)
-{
-	struct vsp1_dl_manager *dlm =
-		container_of(work, struct vsp1_dl_manager, gc_work);
-
-	vsp1_dlm_fragments_free(dlm);
+	return vsp1_dl_body_get(dlm->pool);
 }
 
 struct vsp1_dl_manager *vsp1_dlm_create(struct vsp1_device *vsp1,
@@ -749,6 +823,7 @@ struct vsp1_dl_manager *vsp1_dlm_create(struct vsp1_device *vsp1,
 					unsigned int prealloc)
 {
 	struct vsp1_dl_manager *dlm;
+	size_t header_size;
 	unsigned int i;
 
 	dlm = devm_kzalloc(vsp1->dev, sizeof(*dlm), GFP_KERNEL);
@@ -763,8 +838,22 @@ struct vsp1_dl_manager *vsp1_dlm_create(struct vsp1_device *vsp1,
 
 	spin_lock_init(&dlm->lock);
 	INIT_LIST_HEAD(&dlm->free);
-	INIT_LIST_HEAD(&dlm->gc_fragments);
-	INIT_WORK(&dlm->gc_work, vsp1_dlm_garbage_collect);
+
+	/*
+	 * Initialize the display list body and allocate DMA memory for the body
+	 * and the optional header. Both are allocated together to avoid memory
+	 * fragmentation, with the header located right after the body in
+	 * memory. An extra body is allocated on top of the prealloc to account
+	 * for the cached body used by the vsp1_pipeline object.
+	 */
+	header_size = dlm->mode == VSP1_DL_MODE_HEADER
+		    ? ALIGN(sizeof(struct vsp1_dl_header), 8)
+		    : 0;
+
+	dlm->pool = vsp1_dl_body_pool_create(vsp1, prealloc + 1,
+					     VSP1_DL_NUM_ENTRIES, header_size);
+	if (!dlm->pool)
+		return NULL;
 
 	for (i = 0; i < prealloc; ++i) {
 		struct vsp1_dl_list *dl;
@@ -786,12 +875,10 @@ void vsp1_dlm_destroy(struct vsp1_dl_manager *dlm)
 	if (!dlm)
 		return;
 
-	cancel_work_sync(&dlm->gc_work);
-
 	list_for_each_entry_safe(dl, next, &dlm->free, list) {
 		list_del(&dl->list);
 		vsp1_dl_list_free(dl);
 	}
 
-	vsp1_dlm_fragments_free(dlm);
+	vsp1_dl_body_pool_destroy(dlm->pool);
 }
diff --git a/drivers/media/platform/vsp1/vsp1_dl.h b/drivers/media/platform/vsp1/vsp1_dl.h
index ee3508172f0a..7dba0469c92e 100644
--- a/drivers/media/platform/vsp1/vsp1_dl.h
+++ b/drivers/media/platform/vsp1/vsp1_dl.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_dl.h  --  R-Car VSP1 Display List
  *
  * Copyright (C) 2015 Renesas Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_DL_H__
 #define __VSP1_DL_H__
@@ -16,10 +12,14 @@
 #include <linux/types.h>
 
 struct vsp1_device;
-struct vsp1_dl_fragment;
+struct vsp1_dl_body;
+struct vsp1_dl_body_pool;
 struct vsp1_dl_list;
 struct vsp1_dl_manager;
 
+#define VSP1_DL_FRAME_END_COMPLETED		BIT(0)
+#define VSP1_DL_FRAME_END_INTERNAL		BIT(1)
+
 void vsp1_dlm_setup(struct vsp1_device *vsp1);
 
 struct vsp1_dl_manager *vsp1_dlm_create(struct vsp1_device *vsp1,
@@ -27,19 +27,23 @@ struct vsp1_dl_manager *vsp1_dlm_create(struct vsp1_device *vsp1,
 					unsigned int prealloc);
 void vsp1_dlm_destroy(struct vsp1_dl_manager *dlm);
 void vsp1_dlm_reset(struct vsp1_dl_manager *dlm);
-bool vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm);
+unsigned int vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm);
+struct vsp1_dl_body *vsp1_dlm_dl_body_get(struct vsp1_dl_manager *dlm);
 
 struct vsp1_dl_list *vsp1_dl_list_get(struct vsp1_dl_manager *dlm);
 void vsp1_dl_list_put(struct vsp1_dl_list *dl);
-void vsp1_dl_list_write(struct vsp1_dl_list *dl, u32 reg, u32 data);
-void vsp1_dl_list_commit(struct vsp1_dl_list *dl);
-
-struct vsp1_dl_body *vsp1_dl_fragment_alloc(struct vsp1_device *vsp1,
-					    unsigned int num_entries);
-void vsp1_dl_fragment_free(struct vsp1_dl_body *dlb);
-void vsp1_dl_fragment_write(struct vsp1_dl_body *dlb, u32 reg, u32 data);
-int vsp1_dl_list_add_fragment(struct vsp1_dl_list *dl,
-			      struct vsp1_dl_body *dlb);
+struct vsp1_dl_body *vsp1_dl_list_get_body0(struct vsp1_dl_list *dl);
+void vsp1_dl_list_commit(struct vsp1_dl_list *dl, bool internal);
+
+struct vsp1_dl_body_pool *
+vsp1_dl_body_pool_create(struct vsp1_device *vsp1, unsigned int num_bodies,
+			 unsigned int num_entries, size_t extra_size);
+void vsp1_dl_body_pool_destroy(struct vsp1_dl_body_pool *pool);
+struct vsp1_dl_body *vsp1_dl_body_get(struct vsp1_dl_body_pool *pool);
+void vsp1_dl_body_put(struct vsp1_dl_body *dlb);
+
+void vsp1_dl_body_write(struct vsp1_dl_body *dlb, u32 reg, u32 data);
+int vsp1_dl_list_add_body(struct vsp1_dl_list *dl, struct vsp1_dl_body *dlb);
 int vsp1_dl_list_add_chain(struct vsp1_dl_list *head, struct vsp1_dl_list *dl);
 
 #endif /* __VSP1_DL_H__ */
diff --git a/drivers/media/platform/vsp1/vsp1_drm.c b/drivers/media/platform/vsp1/vsp1_drm.c
index b8fee1834253..edb35a5c57ea 100644
--- a/drivers/media/platform/vsp1/vsp1_drm.c
+++ b/drivers/media/platform/vsp1/vsp1_drm.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
- * vsp1_drm.c  --  R-Car VSP1 DRM API
+ * vsp1_drm.c  --  R-Car VSP1 DRM/KMS Interface
  *
  * Copyright (C) 2015 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -20,26 +16,550 @@
 #include <media/vsp1.h>
 
 #include "vsp1.h"
-#include "vsp1_bru.h"
+#include "vsp1_brx.h"
 #include "vsp1_dl.h"
 #include "vsp1_drm.h"
 #include "vsp1_lif.h"
 #include "vsp1_pipe.h"
 #include "vsp1_rwpf.h"
+#include "vsp1_uif.h"
 
-#define BRU_NAME(e)	(e)->type == VSP1_ENTITY_BRU ? "BRU" : "BRS"
+#define BRX_NAME(e)	(e)->type == VSP1_ENTITY_BRU ? "BRU" : "BRS"
 
 /* -----------------------------------------------------------------------------
  * Interrupt Handling
  */
 
-static void vsp1_du_pipeline_frame_end(struct vsp1_pipeline *pipe,
-				       bool completed)
-{
-	struct vsp1_drm_pipeline *drm_pipe = to_vsp1_drm_pipeline(pipe);
+static void vsp1_du_pipeline_frame_end(struct vsp1_pipeline *pipe,
+				       unsigned int completion)
+{
+	struct vsp1_drm_pipeline *drm_pipe = to_vsp1_drm_pipeline(pipe);
+	bool complete = completion == VSP1_DL_FRAME_END_COMPLETED;
+
+	if (drm_pipe->du_complete) {
+		struct vsp1_entity *uif = drm_pipe->uif;
+		u32 crc;
+
+		crc = uif ? vsp1_uif_get_crc(to_uif(&uif->subdev)) : 0;
+		drm_pipe->du_complete(drm_pipe->du_private, complete, crc);
+	}
+
+	if (completion & VSP1_DL_FRAME_END_INTERNAL) {
+		drm_pipe->force_brx_release = false;
+		wake_up(&drm_pipe->wait_queue);
+	}
+}
+
+/* -----------------------------------------------------------------------------
+ * Pipeline Configuration
+ */
+
+/*
+ * Insert the UIF in the pipeline between the prev and next entities. If no UIF
+ * is available connect the two entities directly.
+ */
+static int vsp1_du_insert_uif(struct vsp1_device *vsp1,
+			      struct vsp1_pipeline *pipe,
+			      struct vsp1_entity *uif,
+			      struct vsp1_entity *prev, unsigned int prev_pad,
+			      struct vsp1_entity *next, unsigned int next_pad)
+{
+	struct v4l2_subdev_format format;
+	int ret;
+
+	if (!uif) {
+		/*
+		 * If there's no UIF to be inserted, connect the previous and
+		 * next entities directly.
+		 */
+		prev->sink = next;
+		prev->sink_pad = next_pad;
+		return 0;
+	}
+
+	prev->sink = uif;
+	prev->sink_pad = UIF_PAD_SINK;
+
+	memset(&format, 0, sizeof(format));
+	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	format.pad = prev_pad;
+
+	ret = v4l2_subdev_call(&prev->subdev, pad, get_fmt, NULL, &format);
+	if (ret < 0)
+		return ret;
+
+	format.pad = UIF_PAD_SINK;
+
+	ret = v4l2_subdev_call(&uif->subdev, pad, set_fmt, NULL, &format);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on UIF sink\n",
+		__func__, format.format.width, format.format.height,
+		format.format.code);
+
+	/*
+	 * The UIF doesn't mangle the format between its sink and source pads,
+	 * so there is no need to retrieve the format on its source pad.
+	 */
+
+	uif->sink = next;
+	uif->sink_pad = next_pad;
+
+	return 0;
+}
+
+/* Setup one RPF and the connected BRx sink pad. */
+static int vsp1_du_pipeline_setup_rpf(struct vsp1_device *vsp1,
+				      struct vsp1_pipeline *pipe,
+				      struct vsp1_rwpf *rpf,
+				      struct vsp1_entity *uif,
+				      unsigned int brx_input)
+{
+	struct v4l2_subdev_selection sel;
+	struct v4l2_subdev_format format;
+	const struct v4l2_rect *crop;
+	int ret;
+
+	/*
+	 * Configure the format on the RPF sink pad and propagate it up to the
+	 * BRx sink pad.
+	 */
+	crop = &vsp1->drm->inputs[rpf->entity.index].crop;
+
+	memset(&format, 0, sizeof(format));
+	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	format.pad = RWPF_PAD_SINK;
+	format.format.width = crop->width + crop->left;
+	format.format.height = crop->height + crop->top;
+	format.format.code = rpf->fmtinfo->mbus;
+	format.format.field = V4L2_FIELD_NONE;
+
+	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, set_fmt, NULL,
+			       &format);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(vsp1->dev,
+		"%s: set format %ux%u (%x) on RPF%u sink\n",
+		__func__, format.format.width, format.format.height,
+		format.format.code, rpf->entity.index);
+
+	memset(&sel, 0, sizeof(sel));
+	sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	sel.pad = RWPF_PAD_SINK;
+	sel.target = V4L2_SEL_TGT_CROP;
+	sel.r = *crop;
+
+	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, set_selection, NULL,
+			       &sel);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(vsp1->dev,
+		"%s: set selection (%u,%u)/%ux%u on RPF%u sink\n",
+		__func__, sel.r.left, sel.r.top, sel.r.width, sel.r.height,
+		rpf->entity.index);
+
+	/*
+	 * RPF source, hardcode the format to ARGB8888 to turn on format
+	 * conversion if needed.
+	 */
+	format.pad = RWPF_PAD_SOURCE;
+
+	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, get_fmt, NULL,
+			       &format);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(vsp1->dev,
+		"%s: got format %ux%u (%x) on RPF%u source\n",
+		__func__, format.format.width, format.format.height,
+		format.format.code, rpf->entity.index);
+
+	format.format.code = MEDIA_BUS_FMT_ARGB8888_1X32;
+
+	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, set_fmt, NULL,
+			       &format);
+	if (ret < 0)
+		return ret;
+
+	/* Insert and configure the UIF if available. */
+	ret = vsp1_du_insert_uif(vsp1, pipe, uif, &rpf->entity, RWPF_PAD_SOURCE,
+				 pipe->brx, brx_input);
+	if (ret < 0)
+		return ret;
+
+	/* BRx sink, propagate the format from the RPF source. */
+	format.pad = brx_input;
+
+	ret = v4l2_subdev_call(&pipe->brx->subdev, pad, set_fmt, NULL,
+			       &format);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on %s pad %u\n",
+		__func__, format.format.width, format.format.height,
+		format.format.code, BRX_NAME(pipe->brx), format.pad);
+
+	sel.pad = brx_input;
+	sel.target = V4L2_SEL_TGT_COMPOSE;
+	sel.r = vsp1->drm->inputs[rpf->entity.index].compose;
+
+	ret = v4l2_subdev_call(&pipe->brx->subdev, pad, set_selection, NULL,
+			       &sel);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(vsp1->dev, "%s: set selection (%u,%u)/%ux%u on %s pad %u\n",
+		__func__, sel.r.left, sel.r.top, sel.r.width, sel.r.height,
+		BRX_NAME(pipe->brx), sel.pad);
+
+	return 0;
+}
+
+/* Setup the BRx source pad. */
+static int vsp1_du_pipeline_setup_inputs(struct vsp1_device *vsp1,
+					 struct vsp1_pipeline *pipe);
+static void vsp1_du_pipeline_configure(struct vsp1_pipeline *pipe);
+
+static int vsp1_du_pipeline_setup_brx(struct vsp1_device *vsp1,
+				      struct vsp1_pipeline *pipe)
+{
+	struct vsp1_drm_pipeline *drm_pipe = to_vsp1_drm_pipeline(pipe);
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	struct vsp1_entity *brx;
+	int ret;
+
+	/*
+	 * Pick a BRx:
+	 * - If we need more than two inputs, use the BRU.
+	 * - Otherwise, if we are not forced to release our BRx, keep it.
+	 * - Else, use any free BRx (randomly starting with the BRU).
+	 */
+	if (pipe->num_inputs > 2)
+		brx = &vsp1->bru->entity;
+	else if (pipe->brx && !drm_pipe->force_brx_release)
+		brx = pipe->brx;
+	else if (!vsp1->bru->entity.pipe)
+		brx = &vsp1->bru->entity;
+	else
+		brx = &vsp1->brs->entity;
+
+	/* Switch BRx if needed. */
+	if (brx != pipe->brx) {
+		struct vsp1_entity *released_brx = NULL;
+
+		/* Release our BRx if we have one. */
+		if (pipe->brx) {
+			dev_dbg(vsp1->dev, "%s: pipe %u: releasing %s\n",
+				__func__, pipe->lif->index,
+				BRX_NAME(pipe->brx));
+
+			/*
+			 * The BRx might be acquired by the other pipeline in
+			 * the next step. We must thus remove it from the list
+			 * of entities for this pipeline. The other pipeline's
+			 * hardware configuration will reconfigure the BRx
+			 * routing.
+			 *
+			 * However, if the other pipeline doesn't acquire our
+			 * BRx, we need to keep it in the list, otherwise the
+			 * hardware configuration step won't disconnect it from
+			 * the pipeline. To solve this, store the released BRx
+			 * pointer to add it back to the list of entities later
+			 * if it isn't acquired by the other pipeline.
+			 */
+			released_brx = pipe->brx;
+
+			list_del(&pipe->brx->list_pipe);
+			pipe->brx->sink = NULL;
+			pipe->brx->pipe = NULL;
+			pipe->brx = NULL;
+		}
+
+		/*
+		 * If the BRx we need is in use, force the owner pipeline to
+		 * switch to the other BRx and wait until the switch completes.
+		 */
+		if (brx->pipe) {
+			struct vsp1_drm_pipeline *owner_pipe;
+
+			dev_dbg(vsp1->dev, "%s: pipe %u: waiting for %s\n",
+				__func__, pipe->lif->index, BRX_NAME(brx));
+
+			owner_pipe = to_vsp1_drm_pipeline(brx->pipe);
+			owner_pipe->force_brx_release = true;
+
+			vsp1_du_pipeline_setup_inputs(vsp1, &owner_pipe->pipe);
+			vsp1_du_pipeline_configure(&owner_pipe->pipe);
+
+			ret = wait_event_timeout(owner_pipe->wait_queue,
+						 !owner_pipe->force_brx_release,
+						 msecs_to_jiffies(500));
+			if (ret == 0)
+				dev_warn(vsp1->dev,
+					 "DRM pipeline %u reconfiguration timeout\n",
+					 owner_pipe->pipe.lif->index);
+		}
+
+		/*
+		 * If the BRx we have released previously hasn't been acquired
+		 * by the other pipeline, add it back to the entities list (with
+		 * the pipe pointer NULL) to let vsp1_du_pipeline_configure()
+		 * disconnect it from the hardware pipeline.
+		 */
+		if (released_brx && !released_brx->pipe)
+			list_add_tail(&released_brx->list_pipe,
+				      &pipe->entities);
+
+		/* Add the BRx to the pipeline. */
+		dev_dbg(vsp1->dev, "%s: pipe %u: acquired %s\n",
+			__func__, pipe->lif->index, BRX_NAME(brx));
+
+		pipe->brx = brx;
+		pipe->brx->pipe = pipe;
+		pipe->brx->sink = &pipe->output->entity;
+		pipe->brx->sink_pad = 0;
+
+		list_add_tail(&pipe->brx->list_pipe, &pipe->entities);
+	}
+
+	/*
+	 * Configure the format on the BRx source and verify that it matches the
+	 * requested format. We don't set the media bus code as it is configured
+	 * on the BRx sink pad 0 and propagated inside the entity, not on the
+	 * source pad.
+	 */
+	format.pad = pipe->brx->source_pad;
+	format.format.width = drm_pipe->width;
+	format.format.height = drm_pipe->height;
+	format.format.field = V4L2_FIELD_NONE;
+
+	ret = v4l2_subdev_call(&pipe->brx->subdev, pad, set_fmt, NULL,
+			       &format);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on %s pad %u\n",
+		__func__, format.format.width, format.format.height,
+		format.format.code, BRX_NAME(pipe->brx), pipe->brx->source_pad);
+
+	if (format.format.width != drm_pipe->width ||
+	    format.format.height != drm_pipe->height) {
+		dev_dbg(vsp1->dev, "%s: format mismatch\n", __func__);
+		return -EPIPE;
+	}
+
+	return 0;
+}
+
+static unsigned int rpf_zpos(struct vsp1_device *vsp1, struct vsp1_rwpf *rpf)
+{
+	return vsp1->drm->inputs[rpf->entity.index].zpos;
+}
+
+/* Setup the input side of the pipeline (RPFs and BRx). */
+static int vsp1_du_pipeline_setup_inputs(struct vsp1_device *vsp1,
+					struct vsp1_pipeline *pipe)
+{
+	struct vsp1_drm_pipeline *drm_pipe = to_vsp1_drm_pipeline(pipe);
+	struct vsp1_rwpf *inputs[VSP1_MAX_RPF] = { NULL, };
+	struct vsp1_entity *uif;
+	bool use_uif = false;
+	struct vsp1_brx *brx;
+	unsigned int i;
+	int ret;
+
+	/* Count the number of enabled inputs and sort them by Z-order. */
+	pipe->num_inputs = 0;
+
+	for (i = 0; i < vsp1->info->rpf_count; ++i) {
+		struct vsp1_rwpf *rpf = vsp1->rpf[i];
+		unsigned int j;
+
+		if (!pipe->inputs[i])
+			continue;
+
+		/* Insert the RPF in the sorted RPFs array. */
+		for (j = pipe->num_inputs++; j > 0; --j) {
+			if (rpf_zpos(vsp1, inputs[j-1]) <= rpf_zpos(vsp1, rpf))
+				break;
+			inputs[j] = inputs[j-1];
+		}
+
+		inputs[j] = rpf;
+	}
+
+	/*
+	 * Setup the BRx. This must be done before setting up the RPF input
+	 * pipelines as the BRx sink compose rectangles depend on the BRx source
+	 * format.
+	 */
+	ret = vsp1_du_pipeline_setup_brx(vsp1, pipe);
+	if (ret < 0) {
+		dev_err(vsp1->dev, "%s: failed to setup %s source\n", __func__,
+			BRX_NAME(pipe->brx));
+		return ret;
+	}
+
+	brx = to_brx(&pipe->brx->subdev);
+
+	/* Setup the RPF input pipeline for every enabled input. */
+	for (i = 0; i < pipe->brx->source_pad; ++i) {
+		struct vsp1_rwpf *rpf = inputs[i];
+
+		if (!rpf) {
+			brx->inputs[i].rpf = NULL;
+			continue;
+		}
+
+		if (!rpf->entity.pipe) {
+			rpf->entity.pipe = pipe;
+			list_add_tail(&rpf->entity.list_pipe, &pipe->entities);
+		}
+
+		brx->inputs[i].rpf = rpf;
+		rpf->brx_input = i;
+		rpf->entity.sink = pipe->brx;
+		rpf->entity.sink_pad = i;
+
+		dev_dbg(vsp1->dev, "%s: connecting RPF.%u to %s:%u\n",
+			__func__, rpf->entity.index, BRX_NAME(pipe->brx), i);
+
+		uif = drm_pipe->crc.source == VSP1_DU_CRC_PLANE &&
+		      drm_pipe->crc.index == i ? drm_pipe->uif : NULL;
+		if (uif)
+			use_uif = true;
+		ret = vsp1_du_pipeline_setup_rpf(vsp1, pipe, rpf, uif, i);
+		if (ret < 0) {
+			dev_err(vsp1->dev,
+				"%s: failed to setup RPF.%u\n",
+				__func__, rpf->entity.index);
+			return ret;
+		}
+	}
+
+	/* Insert and configure the UIF at the BRx output if available. */
+	uif = drm_pipe->crc.source == VSP1_DU_CRC_OUTPUT ? drm_pipe->uif : NULL;
+	if (uif)
+		use_uif = true;
+	ret = vsp1_du_insert_uif(vsp1, pipe, uif,
+				 pipe->brx, pipe->brx->source_pad,
+				 &pipe->output->entity, 0);
+	if (ret < 0)
+		dev_err(vsp1->dev, "%s: failed to setup UIF after %s\n",
+			__func__, BRX_NAME(pipe->brx));
+
+	/*
+	 * If the UIF is not in use schedule it for removal by setting its pipe
+	 * pointer to NULL, vsp1_du_pipeline_configure() will remove it from the
+	 * hardware pipeline and from the pipeline's list of entities. Otherwise
+	 * make sure it is present in the pipeline's list of entities if it
+	 * wasn't already.
+	 */
+	if (!use_uif) {
+		drm_pipe->uif->pipe = NULL;
+	} else if (!drm_pipe->uif->pipe) {
+		drm_pipe->uif->pipe = pipe;
+		list_add_tail(&drm_pipe->uif->list_pipe, &pipe->entities);
+	}
+
+	return 0;
+}
+
+/* Setup the output side of the pipeline (WPF and LIF). */
+static int vsp1_du_pipeline_setup_output(struct vsp1_device *vsp1,
+					 struct vsp1_pipeline *pipe)
+{
+	struct vsp1_drm_pipeline *drm_pipe = to_vsp1_drm_pipeline(pipe);
+	struct v4l2_subdev_format format = { 0, };
+	int ret;
+
+	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	format.pad = RWPF_PAD_SINK;
+	format.format.width = drm_pipe->width;
+	format.format.height = drm_pipe->height;
+	format.format.code = MEDIA_BUS_FMT_ARGB8888_1X32;
+	format.format.field = V4L2_FIELD_NONE;
+
+	ret = v4l2_subdev_call(&pipe->output->entity.subdev, pad, set_fmt, NULL,
+			       &format);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on WPF%u sink\n",
+		__func__, format.format.width, format.format.height,
+		format.format.code, pipe->output->entity.index);
+
+	format.pad = RWPF_PAD_SOURCE;
+	ret = v4l2_subdev_call(&pipe->output->entity.subdev, pad, get_fmt, NULL,
+			       &format);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(vsp1->dev, "%s: got format %ux%u (%x) on WPF%u source\n",
+		__func__, format.format.width, format.format.height,
+		format.format.code, pipe->output->entity.index);
+
+	format.pad = LIF_PAD_SINK;
+	ret = v4l2_subdev_call(&pipe->lif->subdev, pad, set_fmt, NULL,
+			       &format);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on LIF%u sink\n",
+		__func__, format.format.width, format.format.height,
+		format.format.code, pipe->lif->index);
+
+	/*
+	 * Verify that the format at the output of the pipeline matches the
+	 * requested frame size and media bus code.
+	 */
+	if (format.format.width != drm_pipe->width ||
+	    format.format.height != drm_pipe->height ||
+	    format.format.code != MEDIA_BUS_FMT_ARGB8888_1X32) {
+		dev_dbg(vsp1->dev, "%s: format mismatch on LIF%u\n", __func__,
+			pipe->lif->index);
+		return -EPIPE;
+	}
+
+	return 0;
+}
+
+/* Configure all entities in the pipeline. */
+static void vsp1_du_pipeline_configure(struct vsp1_pipeline *pipe)
+{
+	struct vsp1_drm_pipeline *drm_pipe = to_vsp1_drm_pipeline(pipe);
+	struct vsp1_entity *entity;
+	struct vsp1_entity *next;
+	struct vsp1_dl_list *dl;
+	struct vsp1_dl_body *dlb;
+
+	dl = vsp1_dl_list_get(pipe->output->dlm);
+	dlb = vsp1_dl_list_get_body0(dl);
+
+	list_for_each_entry_safe(entity, next, &pipe->entities, list_pipe) {
+		/* Disconnect unused entities from the pipeline. */
+		if (!entity->pipe) {
+			vsp1_dl_body_write(dlb, entity->route->reg,
+					   VI6_DPR_NODE_UNUSED);
+
+			entity->sink = NULL;
+			list_del(&entity->list_pipe);
+
+			continue;
+		}
+
+		vsp1_entity_route_setup(entity, pipe, dlb);
+		vsp1_entity_configure_stream(entity, pipe, dlb);
+		vsp1_entity_configure_frame(entity, pipe, dl, dlb);
+		vsp1_entity_configure_partition(entity, pipe, dl, dlb);
+	}
 
-	if (drm_pipe->du_complete)
-		drm_pipe->du_complete(drm_pipe->du_private, completed);
+	vsp1_dl_list_commit(dl, drm_pipe->force_brx_release);
 }
 
 /* -----------------------------------------------------------------------------
@@ -64,8 +584,8 @@ EXPORT_SYMBOL_GPL(vsp1_du_init);
  * @cfg: the LIF configuration
  *
  * Configure the output part of VSP DRM pipeline for the given frame @cfg.width
- * and @cfg.height. This sets up formats on the blend unit (BRU or BRS) source
- * pad, the WPF sink and source pads, and the LIF sink pad.
+ * and @cfg.height. This sets up formats on the BRx source pad, the WPF sink and
+ * source pads, and the LIF sink pad.
  *
  * The @pipe_index argument selects which DRM pipeline to setup. The number of
  * available pipelines depend on the VSP instance.
@@ -84,11 +604,6 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
 	struct vsp1_drm_pipeline *drm_pipe;
 	struct vsp1_pipeline *pipe;
-	struct vsp1_bru *bru;
-	struct vsp1_entity *entity;
-	struct vsp1_entity *next;
-	struct vsp1_dl_list *dl;
-	struct v4l2_subdev_format format;
 	unsigned long flags;
 	unsigned int i;
 	int ret;
@@ -98,9 +613,14 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 
 	drm_pipe = &vsp1->drm->pipe[pipe_index];
 	pipe = &drm_pipe->pipe;
-	bru = to_bru(&pipe->bru->subdev);
 
 	if (!cfg) {
+		struct vsp1_brx *brx;
+
+		mutex_lock(&vsp1->drm->lock);
+
+		brx = to_brx(&pipe->brx->subdev);
+
 		/*
 		 * NULL configuration means the CRTC is being disabled, stop
 		 * the pipeline and turn the light off.
@@ -109,8 +629,6 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 		if (ret == -ETIMEDOUT)
 			dev_err(vsp1->dev, "DRM pipeline stop timeout\n");
 
-		media_pipeline_stop(&pipe->output->entity.subdev.entity);
-
 		for (i = 0; i < ARRAY_SIZE(pipe->inputs); ++i) {
 			struct vsp1_rwpf *rpf = pipe->inputs[i];
 
@@ -118,19 +636,30 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 				continue;
 
 			/*
-			 * Remove the RPF from the pipe and the list of BRU
+			 * Remove the RPF from the pipe and the list of BRx
 			 * inputs.
 			 */
-			WARN_ON(list_empty(&rpf->entity.list_pipe));
-			list_del_init(&rpf->entity.list_pipe);
+			WARN_ON(!rpf->entity.pipe);
+			rpf->entity.pipe = NULL;
+			list_del(&rpf->entity.list_pipe);
 			pipe->inputs[i] = NULL;
 
-			bru->inputs[rpf->bru_input].rpf = NULL;
+			brx->inputs[rpf->brx_input].rpf = NULL;
 		}
 
 		drm_pipe->du_complete = NULL;
 		pipe->num_inputs = 0;
 
+		dev_dbg(vsp1->dev, "%s: pipe %u: releasing %s\n",
+			__func__, pipe->lif->index,
+			BRX_NAME(pipe->brx));
+
+		list_del(&pipe->brx->list_pipe);
+		pipe->brx->pipe = NULL;
+		pipe->brx = NULL;
+
+		mutex_unlock(&vsp1->drm->lock);
+
 		vsp1_dlm_reset(pipe->output->dlm);
 		vsp1_device_put(vsp1);
 
@@ -139,100 +668,27 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 		return 0;
 	}
 
+	drm_pipe->width = cfg->width;
+	drm_pipe->height = cfg->height;
+
 	dev_dbg(vsp1->dev, "%s: configuring LIF%u with format %ux%u\n",
 		__func__, pipe_index, cfg->width, cfg->height);
 
-	/*
-	 * Configure the format at the BRU sinks and propagate it through the
-	 * pipeline.
-	 */
-	memset(&format, 0, sizeof(format));
-	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-
-	for (i = 0; i < pipe->bru->source_pad; ++i) {
-		format.pad = i;
-
-		format.format.width = cfg->width;
-		format.format.height = cfg->height;
-		format.format.code = MEDIA_BUS_FMT_ARGB8888_1X32;
-		format.format.field = V4L2_FIELD_NONE;
-
-		ret = v4l2_subdev_call(&pipe->bru->subdev, pad,
-				       set_fmt, NULL, &format);
-		if (ret < 0)
-			return ret;
-
-		dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on %s pad %u\n",
-			__func__, format.format.width, format.format.height,
-			format.format.code, BRU_NAME(pipe->bru), i);
-	}
-
-	format.pad = pipe->bru->source_pad;
-	format.format.width = cfg->width;
-	format.format.height = cfg->height;
-	format.format.code = MEDIA_BUS_FMT_ARGB8888_1X32;
-	format.format.field = V4L2_FIELD_NONE;
-
-	ret = v4l2_subdev_call(&pipe->bru->subdev, pad, set_fmt, NULL,
-			       &format);
-	if (ret < 0)
-		return ret;
-
-	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on %s pad %u\n",
-		__func__, format.format.width, format.format.height,
-		format.format.code, BRU_NAME(pipe->bru), i);
-
-	format.pad = RWPF_PAD_SINK;
-	ret = v4l2_subdev_call(&pipe->output->entity.subdev, pad, set_fmt, NULL,
-			       &format);
-	if (ret < 0)
-		return ret;
-
-	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on WPF%u sink\n",
-		__func__, format.format.width, format.format.height,
-		format.format.code, pipe->output->entity.index);
+	mutex_lock(&vsp1->drm->lock);
 
-	format.pad = RWPF_PAD_SOURCE;
-	ret = v4l2_subdev_call(&pipe->output->entity.subdev, pad, get_fmt, NULL,
-			       &format);
+	/* Setup formats through the pipeline. */
+	ret = vsp1_du_pipeline_setup_inputs(vsp1, pipe);
 	if (ret < 0)
-		return ret;
-
-	dev_dbg(vsp1->dev, "%s: got format %ux%u (%x) on WPF%u source\n",
-		__func__, format.format.width, format.format.height,
-		format.format.code, pipe->output->entity.index);
+		goto unlock;
 
-	format.pad = LIF_PAD_SINK;
-	ret = v4l2_subdev_call(&pipe->lif->subdev, pad, set_fmt, NULL,
-			       &format);
+	ret = vsp1_du_pipeline_setup_output(vsp1, pipe);
 	if (ret < 0)
-		return ret;
-
-	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on LIF%u sink\n",
-		__func__, format.format.width, format.format.height,
-		format.format.code, pipe_index);
-
-	/*
-	 * Verify that the format at the output of the pipeline matches the
-	 * requested frame size and media bus code.
-	 */
-	if (format.format.width != cfg->width ||
-	    format.format.height != cfg->height ||
-	    format.format.code != MEDIA_BUS_FMT_ARGB8888_1X32) {
-		dev_dbg(vsp1->dev, "%s: format mismatch\n", __func__);
-		return -EPIPE;
-	}
+		goto unlock;
 
-	/*
-	 * Mark the pipeline as streaming and enable the VSP1. This will store
-	 * the pipeline pointer in all entities, which the s_stream handlers
-	 * will need. We don't start the entities themselves right at this point
-	 * as there's no plane configured yet, so we can't start processing
-	 * buffers.
-	 */
+	/* Enable the VSP1. */
 	ret = vsp1_device_get(vsp1);
 	if (ret < 0)
-		return ret;
+		goto unlock;
 
 	/*
 	 * Register a callback to allow us to notify the DRM driver of frame
@@ -241,35 +697,18 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 	drm_pipe->du_complete = cfg->callback;
 	drm_pipe->du_private = cfg->callback_data;
 
-	ret = media_pipeline_start(&pipe->output->entity.subdev.entity,
-					  &pipe->pipe);
-	if (ret < 0) {
-		dev_dbg(vsp1->dev, "%s: pipeline start failed\n", __func__);
-		vsp1_device_put(vsp1);
-		return ret;
-	}
-
 	/* Disable the display interrupts. */
 	vsp1_write(vsp1, VI6_DISP_IRQ_STA, 0);
 	vsp1_write(vsp1, VI6_DISP_IRQ_ENB, 0);
 
 	/* Configure all entities in the pipeline. */
-	dl = vsp1_dl_list_get(pipe->output->dlm);
+	vsp1_du_pipeline_configure(pipe);
 
-	list_for_each_entry_safe(entity, next, &pipe->entities, list_pipe) {
-		vsp1_entity_route_setup(entity, pipe, dl);
-
-		if (entity->ops->configure) {
-			entity->ops->configure(entity, pipe, dl,
-					       VSP1_ENTITY_PARAMS_INIT);
-			entity->ops->configure(entity, pipe, dl,
-					       VSP1_ENTITY_PARAMS_RUNTIME);
-			entity->ops->configure(entity, pipe, dl,
-					       VSP1_ENTITY_PARAMS_PARTITION);
-		}
-	}
+unlock:
+	mutex_unlock(&vsp1->drm->lock);
 
-	vsp1_dl_list_commit(dl);
+	if (ret < 0)
+		return ret;
 
 	/* Start the pipeline. */
 	spin_lock_irqsave(&pipe->irqlock, flags);
@@ -290,9 +729,8 @@ EXPORT_SYMBOL_GPL(vsp1_du_setup_lif);
 void vsp1_du_atomic_begin(struct device *dev, unsigned int pipe_index)
 {
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
-	struct vsp1_drm_pipeline *drm_pipe = &vsp1->drm->pipe[pipe_index];
 
-	drm_pipe->enabled = drm_pipe->pipe.num_inputs != 0;
+	mutex_lock(&vsp1->drm->lock);
 }
 EXPORT_SYMBOL_GPL(vsp1_du_atomic_begin);
 
@@ -345,10 +783,11 @@ int vsp1_du_atomic_update(struct device *dev, unsigned int pipe_index,
 			rpf_index);
 
 		/*
-		 * Remove the RPF from the pipe's inputs. The atomic flush
-		 * handler will disable the input and remove the entity from the
-		 * pipe's entities list.
+		 * Remove the RPF from the pipeline's inputs. Keep it in the
+		 * pipeline's entity list to let vsp1_du_pipeline_configure()
+		 * remove it from the hardware pipeline.
 		 */
+		rpf->entity.pipe = NULL;
 		drm_pipe->pipe.inputs[rpf_index] = NULL;
 		return 0;
 	}
@@ -392,214 +831,24 @@ int vsp1_du_atomic_update(struct device *dev, unsigned int pipe_index,
 }
 EXPORT_SYMBOL_GPL(vsp1_du_atomic_update);
 
-static int vsp1_du_setup_rpf_pipe(struct vsp1_device *vsp1,
-				  struct vsp1_pipeline *pipe,
-				  struct vsp1_rwpf *rpf, unsigned int bru_input)
-{
-	struct v4l2_subdev_selection sel;
-	struct v4l2_subdev_format format;
-	const struct v4l2_rect *crop;
-	int ret;
-
-	/*
-	 * Configure the format on the RPF sink pad and propagate it up to the
-	 * BRU sink pad.
-	 */
-	crop = &vsp1->drm->inputs[rpf->entity.index].crop;
-
-	memset(&format, 0, sizeof(format));
-	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-	format.pad = RWPF_PAD_SINK;
-	format.format.width = crop->width + crop->left;
-	format.format.height = crop->height + crop->top;
-	format.format.code = rpf->fmtinfo->mbus;
-	format.format.field = V4L2_FIELD_NONE;
-
-	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, set_fmt, NULL,
-			       &format);
-	if (ret < 0)
-		return ret;
-
-	dev_dbg(vsp1->dev,
-		"%s: set format %ux%u (%x) on RPF%u sink\n",
-		__func__, format.format.width, format.format.height,
-		format.format.code, rpf->entity.index);
-
-	memset(&sel, 0, sizeof(sel));
-	sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-	sel.pad = RWPF_PAD_SINK;
-	sel.target = V4L2_SEL_TGT_CROP;
-	sel.r = *crop;
-
-	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, set_selection, NULL,
-			       &sel);
-	if (ret < 0)
-		return ret;
-
-	dev_dbg(vsp1->dev,
-		"%s: set selection (%u,%u)/%ux%u on RPF%u sink\n",
-		__func__, sel.r.left, sel.r.top, sel.r.width, sel.r.height,
-		rpf->entity.index);
-
-	/*
-	 * RPF source, hardcode the format to ARGB8888 to turn on format
-	 * conversion if needed.
-	 */
-	format.pad = RWPF_PAD_SOURCE;
-
-	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, get_fmt, NULL,
-			       &format);
-	if (ret < 0)
-		return ret;
-
-	dev_dbg(vsp1->dev,
-		"%s: got format %ux%u (%x) on RPF%u source\n",
-		__func__, format.format.width, format.format.height,
-		format.format.code, rpf->entity.index);
-
-	format.format.code = MEDIA_BUS_FMT_ARGB8888_1X32;
-
-	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, set_fmt, NULL,
-			       &format);
-	if (ret < 0)
-		return ret;
-
-	/* BRU sink, propagate the format from the RPF source. */
-	format.pad = bru_input;
-
-	ret = v4l2_subdev_call(&pipe->bru->subdev, pad, set_fmt, NULL,
-			       &format);
-	if (ret < 0)
-		return ret;
-
-	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on %s pad %u\n",
-		__func__, format.format.width, format.format.height,
-		format.format.code, BRU_NAME(pipe->bru), format.pad);
-
-	sel.pad = bru_input;
-	sel.target = V4L2_SEL_TGT_COMPOSE;
-	sel.r = vsp1->drm->inputs[rpf->entity.index].compose;
-
-	ret = v4l2_subdev_call(&pipe->bru->subdev, pad, set_selection, NULL,
-			       &sel);
-	if (ret < 0)
-		return ret;
-
-	dev_dbg(vsp1->dev, "%s: set selection (%u,%u)/%ux%u on %s pad %u\n",
-		__func__, sel.r.left, sel.r.top, sel.r.width, sel.r.height,
-		BRU_NAME(pipe->bru), sel.pad);
-
-	return 0;
-}
-
-static unsigned int rpf_zpos(struct vsp1_device *vsp1, struct vsp1_rwpf *rpf)
-{
-	return vsp1->drm->inputs[rpf->entity.index].zpos;
-}
-
 /**
  * vsp1_du_atomic_flush - Commit an atomic update
  * @dev: the VSP device
  * @pipe_index: the DRM pipeline index
+ * @cfg: atomic pipe configuration
  */
-void vsp1_du_atomic_flush(struct device *dev, unsigned int pipe_index)
+void vsp1_du_atomic_flush(struct device *dev, unsigned int pipe_index,
+			  const struct vsp1_du_atomic_pipe_config *cfg)
 {
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
 	struct vsp1_drm_pipeline *drm_pipe = &vsp1->drm->pipe[pipe_index];
 	struct vsp1_pipeline *pipe = &drm_pipe->pipe;
-	struct vsp1_rwpf *inputs[VSP1_MAX_RPF] = { NULL, };
-	struct vsp1_bru *bru = to_bru(&pipe->bru->subdev);
-	struct vsp1_entity *entity;
-	struct vsp1_entity *next;
-	struct vsp1_dl_list *dl;
-	unsigned int i;
-	int ret;
-
-	/* Prepare the display list. */
-	dl = vsp1_dl_list_get(pipe->output->dlm);
-
-	/* Count the number of enabled inputs and sort them by Z-order. */
-	pipe->num_inputs = 0;
-
-	for (i = 0; i < vsp1->info->rpf_count; ++i) {
-		struct vsp1_rwpf *rpf = vsp1->rpf[i];
-		unsigned int j;
-
-		/*
-		 * Make sure we don't accept more inputs than the hardware can
-		 * handle. This is a temporary fix to avoid display stall, we
-		 * need to instead allocate the BRU or BRS to display pipelines
-		 * dynamically based on the number of planes they each use.
-		 */
-		if (pipe->num_inputs >= pipe->bru->source_pad)
-			pipe->inputs[i] = NULL;
-
-		if (!pipe->inputs[i])
-			continue;
-
-		/* Insert the RPF in the sorted RPFs array. */
-		for (j = pipe->num_inputs++; j > 0; --j) {
-			if (rpf_zpos(vsp1, inputs[j-1]) <= rpf_zpos(vsp1, rpf))
-				break;
-			inputs[j] = inputs[j-1];
-		}
-
-		inputs[j] = rpf;
-	}
-
-	/* Setup the RPF input pipeline for every enabled input. */
-	for (i = 0; i < pipe->bru->source_pad; ++i) {
-		struct vsp1_rwpf *rpf = inputs[i];
-
-		if (!rpf) {
-			bru->inputs[i].rpf = NULL;
-			continue;
-		}
-
-		if (list_empty(&rpf->entity.list_pipe))
-			list_add_tail(&rpf->entity.list_pipe, &pipe->entities);
 
-		bru->inputs[i].rpf = rpf;
-		rpf->bru_input = i;
-		rpf->entity.sink = pipe->bru;
-		rpf->entity.sink_pad = i;
-
-		dev_dbg(vsp1->dev, "%s: connecting RPF.%u to %s:%u\n",
-			__func__, rpf->entity.index, BRU_NAME(pipe->bru), i);
-
-		ret = vsp1_du_setup_rpf_pipe(vsp1, pipe, rpf, i);
-		if (ret < 0)
-			dev_err(vsp1->dev,
-				"%s: failed to setup RPF.%u\n",
-				__func__, rpf->entity.index);
-	}
-
-	/* Configure all entities in the pipeline. */
-	list_for_each_entry_safe(entity, next, &pipe->entities, list_pipe) {
-		/* Disconnect unused RPFs from the pipeline. */
-		if (entity->type == VSP1_ENTITY_RPF &&
-		    !pipe->inputs[entity->index]) {
-			vsp1_dl_list_write(dl, entity->route->reg,
-					   VI6_DPR_NODE_UNUSED);
-
-			list_del_init(&entity->list_pipe);
-
-			continue;
-		}
+	drm_pipe->crc = cfg->crc;
 
-		vsp1_entity_route_setup(entity, pipe, dl);
-
-		if (entity->ops->configure) {
-			entity->ops->configure(entity, pipe, dl,
-					       VSP1_ENTITY_PARAMS_INIT);
-			entity->ops->configure(entity, pipe, dl,
-					       VSP1_ENTITY_PARAMS_RUNTIME);
-			entity->ops->configure(entity, pipe, dl,
-					       VSP1_ENTITY_PARAMS_PARTITION);
-		}
-	}
-
-	vsp1_dl_list_commit(dl);
+	vsp1_du_pipeline_setup_inputs(vsp1, pipe);
+	vsp1_du_pipeline_configure(pipe);
+	mutex_unlock(&vsp1->drm->lock);
 }
 EXPORT_SYMBOL_GPL(vsp1_du_atomic_flush);
 
@@ -638,31 +887,40 @@ int vsp1_drm_init(struct vsp1_device *vsp1)
 	if (!vsp1->drm)
 		return -ENOMEM;
 
+	mutex_init(&vsp1->drm->lock);
+
 	/* Create one DRM pipeline per LIF. */
 	for (i = 0; i < vsp1->info->lif_count; ++i) {
 		struct vsp1_drm_pipeline *drm_pipe = &vsp1->drm->pipe[i];
 		struct vsp1_pipeline *pipe = &drm_pipe->pipe;
 
+		init_waitqueue_head(&drm_pipe->wait_queue);
+
 		vsp1_pipeline_init(pipe);
 
+		pipe->frame_end = vsp1_du_pipeline_frame_end;
+
 		/*
-		 * The DRM pipeline is static, add entities manually. The first
-		 * pipeline uses the BRU and the second pipeline the BRS.
+		 * The output side of the DRM pipeline is static, add the
+		 * corresponding entities manually.
 		 */
-		pipe->bru = i == 0 ? &vsp1->bru->entity : &vsp1->brs->entity;
-		pipe->lif = &vsp1->lif[i]->entity;
 		pipe->output = vsp1->wpf[i];
-		pipe->output->pipe = pipe;
-		pipe->frame_end = vsp1_du_pipeline_frame_end;
+		pipe->lif = &vsp1->lif[i]->entity;
 
-		pipe->bru->sink = &pipe->output->entity;
-		pipe->bru->sink_pad = 0;
+		pipe->output->entity.pipe = pipe;
 		pipe->output->entity.sink = pipe->lif;
 		pipe->output->entity.sink_pad = 0;
+		list_add_tail(&pipe->output->entity.list_pipe, &pipe->entities);
 
-		list_add_tail(&pipe->bru->list_pipe, &pipe->entities);
+		pipe->lif->pipe = pipe;
 		list_add_tail(&pipe->lif->list_pipe, &pipe->entities);
-		list_add_tail(&pipe->output->entity.list_pipe, &pipe->entities);
+
+		/*
+		 * CRC computation is initially disabled, don't add the UIF to
+		 * the pipeline.
+		 */
+		if (i < vsp1->info->uif_count)
+			drm_pipe->uif = &vsp1->uif[i]->entity;
 	}
 
 	/* Disable all RPFs initially. */
@@ -677,4 +935,5 @@ int vsp1_drm_init(struct vsp1_device *vsp1)
 
 void vsp1_drm_cleanup(struct vsp1_device *vsp1)
 {
+	mutex_destroy(&vsp1->drm->lock);
 }
diff --git a/drivers/media/platform/vsp1/vsp1_drm.h b/drivers/media/platform/vsp1/vsp1_drm.h
index 1cd9db785bf7..8dfd274a59e2 100644
--- a/drivers/media/platform/vsp1/vsp1_drm.h
+++ b/drivers/media/platform/vsp1/vsp1_drm.h
@@ -1,46 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_drm.h  --  R-Car VSP1 DRM/KMS Interface
  *
  * Copyright (C) 2015 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_DRM_H__
 #define __VSP1_DRM_H__
 
+#include <linux/mutex.h>
 #include <linux/videodev2.h>
+#include <linux/wait.h>
+
+#include <media/vsp1.h>
 
 #include "vsp1_pipe.h"
 
 /**
  * vsp1_drm_pipeline - State for the API exposed to the DRM driver
  * @pipe: the VSP1 pipeline used for display
- * @enabled: pipeline state at the beginning of an update
+ * @width: output display width
+ * @height: output display height
+ * @force_brx_release: when set, release the BRx during the next reconfiguration
+ * @wait_queue: wait queue to wait for BRx release completion
+ * @uif: UIF entity if available for the pipeline
+ * @crc: CRC computation configuration
  * @du_complete: frame completion callback for the DU driver (optional)
  * @du_private: data to be passed to the du_complete callback
  */
 struct vsp1_drm_pipeline {
 	struct vsp1_pipeline pipe;
-	bool enabled;
+
+	unsigned int width;
+	unsigned int height;
+
+	bool force_brx_release;
+	wait_queue_head_t wait_queue;
+
+	struct vsp1_entity *uif;
+	struct vsp1_du_crc_config crc;
 
 	/* Frame synchronisation */
-	void (*du_complete)(void *, bool);
+	void (*du_complete)(void *data, bool completed, u32 crc);
 	void *du_private;
 };
 
 /**
  * vsp1_drm - State for the API exposed to the DRM driver
  * @pipe: the VSP1 DRM pipeline used for display
+ * @lock: protects the BRU and BRS allocation
  * @inputs: source crop rectangle, destination compose rectangle and z-order
  *	position for every input (indexed by RPF index)
  */
 struct vsp1_drm {
 	struct vsp1_drm_pipeline pipe[VSP1_MAX_LIF];
+	struct mutex lock;
 
 	struct {
 		struct v4l2_rect crop;
diff --git a/drivers/media/platform/vsp1/vsp1_drv.c b/drivers/media/platform/vsp1/vsp1_drv.c
index eed9516e25e1..5d82f6ee56ea 100644
--- a/drivers/media/platform/vsp1/vsp1_drv.c
+++ b/drivers/media/platform/vsp1/vsp1_drv.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_drv.c  --  R-Car VSP1 Driver
  *
  * Copyright (C) 2013-2015 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/clk.h>
@@ -26,7 +22,7 @@
 #include <media/v4l2-subdev.h>
 
 #include "vsp1.h"
-#include "vsp1_bru.h"
+#include "vsp1_brx.h"
 #include "vsp1_clu.h"
 #include "vsp1_dl.h"
 #include "vsp1_drm.h"
@@ -39,6 +35,7 @@
 #include "vsp1_rwpf.h"
 #include "vsp1_sru.h"
 #include "vsp1_uds.h"
+#include "vsp1_uif.h"
 #include "vsp1_video.h"
 
 /* -----------------------------------------------------------------------------
@@ -63,7 +60,7 @@ static irqreturn_t vsp1_irq_handler(int irq, void *data)
 		vsp1_write(vsp1, VI6_WPF_IRQ_STA(i), ~status & mask);
 
 		if (status & VI6_WFP_IRQ_STA_DFE) {
-			vsp1_pipeline_frame_end(wpf->pipe);
+			vsp1_pipeline_frame_end(wpf->entity.pipe);
 			ret = IRQ_HANDLED;
 		}
 	}
@@ -269,7 +266,7 @@ static int vsp1_create_entities(struct vsp1_device *vsp1)
 
 	/* Instantiate all the entities. */
 	if (vsp1->info->features & VSP1_HAS_BRS) {
-		vsp1->brs = vsp1_bru_create(vsp1, VSP1_ENTITY_BRS);
+		vsp1->brs = vsp1_brx_create(vsp1, VSP1_ENTITY_BRS);
 		if (IS_ERR(vsp1->brs)) {
 			ret = PTR_ERR(vsp1->brs);
 			goto done;
@@ -279,7 +276,7 @@ static int vsp1_create_entities(struct vsp1_device *vsp1)
 	}
 
 	if (vsp1->info->features & VSP1_HAS_BRU) {
-		vsp1->bru = vsp1_bru_create(vsp1, VSP1_ENTITY_BRU);
+		vsp1->bru = vsp1_brx_create(vsp1, VSP1_ENTITY_BRU);
 		if (IS_ERR(vsp1->bru)) {
 			ret = PTR_ERR(vsp1->bru);
 			goto done;
@@ -413,6 +410,19 @@ static int vsp1_create_entities(struct vsp1_device *vsp1)
 		list_add_tail(&uds->entity.list_dev, &vsp1->entities);
 	}
 
+	for (i = 0; i < vsp1->info->uif_count; ++i) {
+		struct vsp1_uif *uif;
+
+		uif = vsp1_uif_create(vsp1, i);
+		if (IS_ERR(uif)) {
+			ret = PTR_ERR(uif);
+			goto done;
+		}
+
+		vsp1->uif[i] = uif;
+		list_add_tail(&uif->entity.list_dev, &vsp1->entities);
+	}
+
 	for (i = 0; i < vsp1->info->wpf_count; ++i) {
 		struct vsp1_rwpf *wpf;
 
@@ -517,6 +527,9 @@ static int vsp1_device_init(struct vsp1_device *vsp1)
 	for (i = 0; i < vsp1->info->uds_count; ++i)
 		vsp1_write(vsp1, VI6_DPR_UDS_ROUTE(i), VI6_DPR_NODE_UNUSED);
 
+	for (i = 0; i < vsp1->info->uif_count; ++i)
+		vsp1_write(vsp1, VI6_DPR_UIF_ROUTE(i), VI6_DPR_NODE_UNUSED);
+
 	vsp1_write(vsp1, VI6_DPR_SRU_ROUTE, VI6_DPR_NODE_UNUSED);
 	vsp1_write(vsp1, VI6_DPR_LUT_ROUTE, VI6_DPR_NODE_UNUSED);
 	vsp1_write(vsp1, VI6_DPR_CLU_ROUTE, VI6_DPR_NODE_UNUSED);
@@ -576,7 +589,7 @@ static int __maybe_unused vsp1_pm_suspend(struct device *dev)
 	 * restarted explicitly by the DU.
 	 */
 	if (!vsp1->drm)
-		vsp1_pipelines_suspend(vsp1);
+		vsp1_video_suspend(vsp1);
 
 	pm_runtime_force_suspend(vsp1->dev);
 
@@ -594,7 +607,7 @@ static int __maybe_unused vsp1_pm_resume(struct device *dev)
 	 * restarted explicitly by the DU.
 	 */
 	if (!vsp1->drm)
-		vsp1_pipelines_resume(vsp1);
+		vsp1_video_resume(vsp1);
 
 	return 0;
 }
@@ -744,6 +757,7 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.features = VSP1_HAS_BRU | VSP1_HAS_WPF_VFLIP,
 		.lif_count = 1,
 		.rpf_count = 5,
+		.uif_count = 1,
 		.wpf_count = 2,
 		.num_bru_inputs = 5,
 	}, {
@@ -753,6 +767,7 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.features = VSP1_HAS_BRS | VSP1_HAS_BRU,
 		.lif_count = 1,
 		.rpf_count = 5,
+		.uif_count = 1,
 		.wpf_count = 1,
 		.num_bru_inputs = 5,
 	}, {
@@ -762,6 +777,7 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.features = VSP1_HAS_BRS | VSP1_HAS_BRU,
 		.lif_count = 2,
 		.rpf_count = 5,
+		.uif_count = 2,
 		.wpf_count = 2,
 		.num_bru_inputs = 5,
 	},
diff --git a/drivers/media/platform/vsp1/vsp1_entity.c b/drivers/media/platform/vsp1/vsp1_entity.c
index 54de15095709..da276a85aa95 100644
--- a/drivers/media/platform/vsp1/vsp1_entity.c
+++ b/drivers/media/platform/vsp1/vsp1_entity.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_entity.c  --  R-Car VSP1 Base Entity
  *
  * Copyright (C) 2013-2014 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -26,7 +22,7 @@
 
 void vsp1_entity_route_setup(struct vsp1_entity *entity,
 			     struct vsp1_pipeline *pipe,
-			     struct vsp1_dl_list *dl)
+			     struct vsp1_dl_body *dlb)
 {
 	struct vsp1_entity *source;
 	u32 route;
@@ -42,7 +38,7 @@ void vsp1_entity_route_setup(struct vsp1_entity *entity,
 		smppt = (pipe->output->entity.index << VI6_DPR_SMPPT_TGW_SHIFT)
 		      | (source->route->output << VI6_DPR_SMPPT_PT_SHIFT);
 
-		vsp1_dl_list_write(dl, VI6_DPR_HGO_SMPPT, smppt);
+		vsp1_dl_body_write(dlb, VI6_DPR_HGO_SMPPT, smppt);
 		return;
 	} else if (entity->type == VSP1_ENTITY_HGT) {
 		u32 smppt;
@@ -55,7 +51,7 @@ void vsp1_entity_route_setup(struct vsp1_entity *entity,
 		smppt = (pipe->output->entity.index << VI6_DPR_SMPPT_TGW_SHIFT)
 		      | (source->route->output << VI6_DPR_SMPPT_PT_SHIFT);
 
-		vsp1_dl_list_write(dl, VI6_DPR_HGT_SMPPT, smppt);
+		vsp1_dl_body_write(dlb, VI6_DPR_HGT_SMPPT, smppt);
 		return;
 	}
 
@@ -70,7 +66,33 @@ void vsp1_entity_route_setup(struct vsp1_entity *entity,
 	 */
 	if (source->type == VSP1_ENTITY_BRS)
 		route |= VI6_DPR_ROUTE_BRSSEL;
-	vsp1_dl_list_write(dl, source->route->reg, route);
+	vsp1_dl_body_write(dlb, source->route->reg, route);
+}
+
+void vsp1_entity_configure_stream(struct vsp1_entity *entity,
+				  struct vsp1_pipeline *pipe,
+				  struct vsp1_dl_body *dlb)
+{
+	if (entity->ops->configure_stream)
+		entity->ops->configure_stream(entity, pipe, dlb);
+}
+
+void vsp1_entity_configure_frame(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
+				 struct vsp1_dl_body *dlb)
+{
+	if (entity->ops->configure_frame)
+		entity->ops->configure_frame(entity, pipe, dl, dlb);
+}
+
+void vsp1_entity_configure_partition(struct vsp1_entity *entity,
+				     struct vsp1_pipeline *pipe,
+				     struct vsp1_dl_list *dl,
+				     struct vsp1_dl_body *dlb)
+{
+	if (entity->ops->configure_partition)
+		entity->ops->configure_partition(entity, pipe, dl, dlb);
 }
 
 /* -----------------------------------------------------------------------------
@@ -311,6 +333,97 @@ int vsp1_subdev_enum_frame_size(struct v4l2_subdev *subdev,
 	return ret;
 }
 
+/*
+ * vsp1_subdev_set_pad_format - Subdev pad set_fmt handler
+ * @subdev: V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fmt: V4L2 subdev format
+ * @codes: Array of supported media bus codes
+ * @ncodes: Number of supported media bus codes
+ * @min_width: Minimum image width
+ * @min_height: Minimum image height
+ * @max_width: Maximum image width
+ * @max_height: Maximum image height
+ *
+ * This function implements the subdev set_fmt pad operation for entities that
+ * do not support scaling or cropping. It defaults to the first supplied media
+ * bus code if the requested code isn't supported, clamps the size to the
+ * supplied minimum and maximum, and propagates the sink pad format to the
+ * source pad.
+ */
+int vsp1_subdev_set_pad_format(struct v4l2_subdev *subdev,
+			       struct v4l2_subdev_pad_config *cfg,
+			       struct v4l2_subdev_format *fmt,
+			       const unsigned int *codes, unsigned int ncodes,
+			       unsigned int min_width, unsigned int min_height,
+			       unsigned int max_width, unsigned int max_height)
+{
+	struct vsp1_entity *entity = to_vsp1_entity(subdev);
+	struct v4l2_subdev_pad_config *config;
+	struct v4l2_mbus_framefmt *format;
+	struct v4l2_rect *selection;
+	unsigned int i;
+	int ret = 0;
+
+	mutex_lock(&entity->lock);
+
+	config = vsp1_entity_get_pad_config(entity, cfg, fmt->which);
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	format = vsp1_entity_get_pad_format(entity, config, fmt->pad);
+
+	if (fmt->pad == entity->source_pad) {
+		/* The output format can't be modified. */
+		fmt->format = *format;
+		goto done;
+	}
+
+	/*
+	 * Default to the first media bus code if the requested format is not
+	 * supported.
+	 */
+	for (i = 0; i < ncodes; ++i) {
+		if (fmt->format.code == codes[i])
+			break;
+	}
+
+	format->code = i < ncodes ? codes[i] : codes[0];
+	format->width = clamp_t(unsigned int, fmt->format.width,
+				min_width, max_width);
+	format->height = clamp_t(unsigned int, fmt->format.height,
+				 min_height, max_height);
+	format->field = V4L2_FIELD_NONE;
+	format->colorspace = V4L2_COLORSPACE_SRGB;
+
+	fmt->format = *format;
+
+	/* Propagate the format to the source pad. */
+	format = vsp1_entity_get_pad_format(entity, config, entity->source_pad);
+	*format = fmt->format;
+
+	/* Reset the crop and compose rectangles */
+	selection = vsp1_entity_get_pad_selection(entity, config, fmt->pad,
+						  V4L2_SEL_TGT_CROP);
+	selection->left = 0;
+	selection->top = 0;
+	selection->width = format->width;
+	selection->height = format->height;
+
+	selection = vsp1_entity_get_pad_selection(entity, config, fmt->pad,
+						  V4L2_SEL_TGT_COMPOSE);
+	selection->left = 0;
+	selection->top = 0;
+	selection->width = format->width;
+	selection->height = format->height;
+
+done:
+	mutex_unlock(&entity->lock);
+	return ret;
+}
+
 /* -----------------------------------------------------------------------------
  * Media Operations
  */
@@ -452,6 +565,10 @@ struct media_pad *vsp1_entity_remote_pad(struct media_pad *pad)
 	{ VSP1_ENTITY_UDS, idx, VI6_DPR_UDS_ROUTE(idx),			\
 	  { VI6_DPR_NODE_UDS(idx) }, VI6_DPR_NODE_UDS(idx) }
 
+#define VSP1_ENTITY_ROUTE_UIF(idx)					\
+	{ VSP1_ENTITY_UIF, idx, VI6_DPR_UIF_ROUTE(idx),			\
+	  { VI6_DPR_NODE_UIF(idx) }, VI6_DPR_NODE_UIF(idx) }
+
 #define VSP1_ENTITY_ROUTE_WPF(idx)					\
 	{ VSP1_ENTITY_WPF, idx, 0,					\
 	  { VI6_DPR_NODE_WPF(idx) }, VI6_DPR_NODE_WPF(idx) }
@@ -480,6 +597,8 @@ static const struct vsp1_route vsp1_routes[] = {
 	VSP1_ENTITY_ROUTE_UDS(0),
 	VSP1_ENTITY_ROUTE_UDS(1),
 	VSP1_ENTITY_ROUTE_UDS(2),
+	VSP1_ENTITY_ROUTE_UIF(0),	/* Named UIF4 in the documentation */
+	VSP1_ENTITY_ROUTE_UIF(1),	/* Named UIF5 in the documentation */
 	VSP1_ENTITY_ROUTE_WPF(0),
 	VSP1_ENTITY_ROUTE_WPF(1),
 	VSP1_ENTITY_ROUTE_WPF(2),
diff --git a/drivers/media/platform/vsp1/vsp1_entity.h b/drivers/media/platform/vsp1/vsp1_entity.h
index 408602ebeb97..97acb7795cf1 100644
--- a/drivers/media/platform/vsp1/vsp1_entity.h
+++ b/drivers/media/platform/vsp1/vsp1_entity.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_entity.h  --  R-Car VSP1 Base Entity
  *
  * Copyright (C) 2013-2014 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_ENTITY_H__
 #define __VSP1_ENTITY_H__
@@ -19,6 +15,7 @@
 #include <media/v4l2-subdev.h>
 
 struct vsp1_device;
+struct vsp1_dl_body;
 struct vsp1_dl_list;
 struct vsp1_pipeline;
 struct vsp1_partition;
@@ -37,21 +34,10 @@ enum vsp1_entity_type {
 	VSP1_ENTITY_RPF,
 	VSP1_ENTITY_SRU,
 	VSP1_ENTITY_UDS,
+	VSP1_ENTITY_UIF,
 	VSP1_ENTITY_WPF,
 };
 
-/**
- * enum vsp1_entity_params - Entity configuration parameters class
- * @VSP1_ENTITY_PARAMS_INIT - Initial parameters
- * @VSP1_ENTITY_PARAMS_PARTITION - Per-image partition parameters
- * @VSP1_ENTITY_PARAMS_RUNTIME - Runtime-configurable parameters
- */
-enum vsp1_entity_params {
-	VSP1_ENTITY_PARAMS_INIT,
-	VSP1_ENTITY_PARAMS_PARTITION,
-	VSP1_ENTITY_PARAMS_RUNTIME,
-};
-
 #define VSP1_ENTITY_MAX_INPUTS		5	/* For the BRU */
 
 /*
@@ -80,8 +66,10 @@ struct vsp1_route {
 /**
  * struct vsp1_entity_operations - Entity operations
  * @destroy:	Destroy the entity.
- * @configure:	Setup the hardware based on the entity state (pipeline, formats,
- *		selection rectangles, ...)
+ * @configure_stream:	Setup the hardware parameters for the stream which do
+ *			not vary between frames (pipeline, formats).
+ * @configure_frame:	Configure the runtime parameters for each frame.
+ * @configure_partition: Configure partition specific parameters.
  * @max_width:	Return the max supported width of data that the entity can
  *		process in a single operation.
  * @partition:	Process the partition construction based on this entity's
@@ -89,8 +77,14 @@ struct vsp1_route {
  */
 struct vsp1_entity_operations {
 	void (*destroy)(struct vsp1_entity *);
-	void (*configure)(struct vsp1_entity *, struct vsp1_pipeline *,
-			  struct vsp1_dl_list *, enum vsp1_entity_params);
+	void (*configure_stream)(struct vsp1_entity *, struct vsp1_pipeline *,
+				 struct vsp1_dl_body *);
+	void (*configure_frame)(struct vsp1_entity *, struct vsp1_pipeline *,
+				struct vsp1_dl_list *, struct vsp1_dl_body *);
+	void (*configure_partition)(struct vsp1_entity *,
+				    struct vsp1_pipeline *,
+				    struct vsp1_dl_list *,
+				    struct vsp1_dl_body *);
 	unsigned int (*max_width)(struct vsp1_entity *, struct vsp1_pipeline *);
 	void (*partition)(struct vsp1_entity *, struct vsp1_pipeline *,
 			  struct vsp1_partition *, unsigned int,
@@ -106,6 +100,8 @@ struct vsp1_entity {
 	unsigned int index;
 	const struct vsp1_route *route;
 
+	struct vsp1_pipeline *pipe;
+
 	struct list_head list_dev;
 	struct list_head list_pipe;
 
@@ -155,13 +151,33 @@ int vsp1_entity_init_cfg(struct v4l2_subdev *subdev,
 
 void vsp1_entity_route_setup(struct vsp1_entity *entity,
 			     struct vsp1_pipeline *pipe,
-			     struct vsp1_dl_list *dl);
+			     struct vsp1_dl_body *dlb);
+
+void vsp1_entity_configure_stream(struct vsp1_entity *entity,
+				  struct vsp1_pipeline *pipe,
+				  struct vsp1_dl_body *dlb);
+
+void vsp1_entity_configure_frame(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
+				 struct vsp1_dl_body *dlb);
+
+void vsp1_entity_configure_partition(struct vsp1_entity *entity,
+				     struct vsp1_pipeline *pipe,
+				     struct vsp1_dl_list *dl,
+				     struct vsp1_dl_body *dlb);
 
 struct media_pad *vsp1_entity_remote_pad(struct media_pad *pad);
 
 int vsp1_subdev_get_pad_format(struct v4l2_subdev *subdev,
 			       struct v4l2_subdev_pad_config *cfg,
 			       struct v4l2_subdev_format *fmt);
+int vsp1_subdev_set_pad_format(struct v4l2_subdev *subdev,
+			       struct v4l2_subdev_pad_config *cfg,
+			       struct v4l2_subdev_format *fmt,
+			       const unsigned int *codes, unsigned int ncodes,
+			       unsigned int min_width, unsigned int min_height,
+			       unsigned int max_width, unsigned int max_height);
 int vsp1_subdev_enum_mbus_code(struct v4l2_subdev *subdev,
 			       struct v4l2_subdev_pad_config *cfg,
 			       struct v4l2_subdev_mbus_code_enum *code,
diff --git a/drivers/media/platform/vsp1/vsp1_hgo.c b/drivers/media/platform/vsp1/vsp1_hgo.c
index 50309c053b78..827373c25351 100644
--- a/drivers/media/platform/vsp1/vsp1_hgo.c
+++ b/drivers/media/platform/vsp1/vsp1_hgo.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_hgo.c  --  R-Car VSP1 Histogram Generator 1D
  *
  * Copyright (C) 2016 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -32,10 +28,10 @@ static inline u32 vsp1_hgo_read(struct vsp1_hgo *hgo, u32 reg)
 	return vsp1_read(hgo->histo.entity.vsp1, reg);
 }
 
-static inline void vsp1_hgo_write(struct vsp1_hgo *hgo, struct vsp1_dl_list *dl,
-				  u32 reg, u32 data)
+static inline void vsp1_hgo_write(struct vsp1_hgo *hgo,
+				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg, data);
+	vsp1_dl_body_write(dlb, reg, data);
 }
 
 /* -----------------------------------------------------------------------------
@@ -133,10 +129,9 @@ static const struct v4l2_ctrl_config hgo_num_bins_control = {
  * VSP1 Entity Operations
  */
 
-static void hgo_configure(struct vsp1_entity *entity,
-			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl,
-			  enum vsp1_entity_params params)
+static void hgo_configure_stream(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_hgo *hgo = to_hgo(&entity->subdev);
 	struct v4l2_rect *compose;
@@ -144,21 +139,18 @@ static void hgo_configure(struct vsp1_entity *entity,
 	unsigned int hratio;
 	unsigned int vratio;
 
-	if (params != VSP1_ENTITY_PARAMS_INIT)
-		return;
-
 	crop = vsp1_entity_get_pad_selection(entity, entity->config,
 					     HISTO_PAD_SINK, V4L2_SEL_TGT_CROP);
 	compose = vsp1_entity_get_pad_selection(entity, entity->config,
 						HISTO_PAD_SINK,
 						V4L2_SEL_TGT_COMPOSE);
 
-	vsp1_hgo_write(hgo, dl, VI6_HGO_REGRST, VI6_HGO_REGRST_RCLEA);
+	vsp1_hgo_write(hgo, dlb, VI6_HGO_REGRST, VI6_HGO_REGRST_RCLEA);
 
-	vsp1_hgo_write(hgo, dl, VI6_HGO_OFFSET,
+	vsp1_hgo_write(hgo, dlb, VI6_HGO_OFFSET,
 		       (crop->left << VI6_HGO_OFFSET_HOFFSET_SHIFT) |
 		       (crop->top << VI6_HGO_OFFSET_VOFFSET_SHIFT));
-	vsp1_hgo_write(hgo, dl, VI6_HGO_SIZE,
+	vsp1_hgo_write(hgo, dlb, VI6_HGO_SIZE,
 		       (crop->width << VI6_HGO_SIZE_HSIZE_SHIFT) |
 		       (crop->height << VI6_HGO_SIZE_VSIZE_SHIFT));
 
@@ -170,7 +162,7 @@ static void hgo_configure(struct vsp1_entity *entity,
 
 	hratio = crop->width * 2 / compose->width / 3;
 	vratio = crop->height * 2 / compose->height / 3;
-	vsp1_hgo_write(hgo, dl, VI6_HGO_MODE,
+	vsp1_hgo_write(hgo, dlb, VI6_HGO_MODE,
 		       (hgo->num_bins == 256 ? VI6_HGO_MODE_STEP : 0) |
 		       (hgo->max_rgb ? VI6_HGO_MODE_MAXRGB : 0) |
 		       (hratio << VI6_HGO_MODE_HRATIO_SHIFT) |
@@ -178,7 +170,7 @@ static void hgo_configure(struct vsp1_entity *entity,
 }
 
 static const struct vsp1_entity_operations hgo_entity_ops = {
-	.configure = hgo_configure,
+	.configure_stream = hgo_configure_stream,
 	.destroy = vsp1_histogram_destroy,
 };
 
diff --git a/drivers/media/platform/vsp1/vsp1_hgo.h b/drivers/media/platform/vsp1/vsp1_hgo.h
index c6c0b7a80e0c..6b0c8580e1bf 100644
--- a/drivers/media/platform/vsp1/vsp1_hgo.h
+++ b/drivers/media/platform/vsp1/vsp1_hgo.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_hgo.h  --  R-Car VSP1 Histogram Generator 1D
  *
  * Copyright (C) 2016 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_HGO_H__
 #define __VSP1_HGO_H__
diff --git a/drivers/media/platform/vsp1/vsp1_hgt.c b/drivers/media/platform/vsp1/vsp1_hgt.c
index b5ce305e3e6f..bb6ce6fdd5f4 100644
--- a/drivers/media/platform/vsp1/vsp1_hgt.c
+++ b/drivers/media/platform/vsp1/vsp1_hgt.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_hgt.c  --  R-Car VSP1 Histogram Generator 2D
  *
  * Copyright (C) 2016 Renesas Electronics Corporation
  *
  * Contact: Niklas Söderlund (niklas.soderlund@ragnatech.se)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -32,10 +28,10 @@ static inline u32 vsp1_hgt_read(struct vsp1_hgt *hgt, u32 reg)
 	return vsp1_read(hgt->histo.entity.vsp1, reg);
 }
 
-static inline void vsp1_hgt_write(struct vsp1_hgt *hgt, struct vsp1_dl_list *dl,
-				  u32 reg, u32 data)
+static inline void vsp1_hgt_write(struct vsp1_hgt *hgt,
+				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg, data);
+	vsp1_dl_body_write(dlb, reg, data);
 }
 
 /* -----------------------------------------------------------------------------
@@ -129,10 +125,9 @@ static const struct v4l2_ctrl_config hgt_hue_areas = {
  * VSP1 Entity Operations
  */
 
-static void hgt_configure(struct vsp1_entity *entity,
-			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl,
-			  enum vsp1_entity_params params)
+static void hgt_configure_stream(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_hgt *hgt = to_hgt(&entity->subdev);
 	struct v4l2_rect *compose;
@@ -143,21 +138,18 @@ static void hgt_configure(struct vsp1_entity *entity,
 	u8 upper;
 	unsigned int i;
 
-	if (params != VSP1_ENTITY_PARAMS_INIT)
-		return;
-
 	crop = vsp1_entity_get_pad_selection(entity, entity->config,
 					     HISTO_PAD_SINK, V4L2_SEL_TGT_CROP);
 	compose = vsp1_entity_get_pad_selection(entity, entity->config,
 						HISTO_PAD_SINK,
 						V4L2_SEL_TGT_COMPOSE);
 
-	vsp1_hgt_write(hgt, dl, VI6_HGT_REGRST, VI6_HGT_REGRST_RCLEA);
+	vsp1_hgt_write(hgt, dlb, VI6_HGT_REGRST, VI6_HGT_REGRST_RCLEA);
 
-	vsp1_hgt_write(hgt, dl, VI6_HGT_OFFSET,
+	vsp1_hgt_write(hgt, dlb, VI6_HGT_OFFSET,
 		       (crop->left << VI6_HGT_OFFSET_HOFFSET_SHIFT) |
 		       (crop->top << VI6_HGT_OFFSET_VOFFSET_SHIFT));
-	vsp1_hgt_write(hgt, dl, VI6_HGT_SIZE,
+	vsp1_hgt_write(hgt, dlb, VI6_HGT_SIZE,
 		       (crop->width << VI6_HGT_SIZE_HSIZE_SHIFT) |
 		       (crop->height << VI6_HGT_SIZE_VSIZE_SHIFT));
 
@@ -165,7 +157,7 @@ static void hgt_configure(struct vsp1_entity *entity,
 	for (i = 0; i < HGT_NUM_HUE_AREAS; ++i) {
 		lower = hgt->hue_areas[i*2 + 0];
 		upper = hgt->hue_areas[i*2 + 1];
-		vsp1_hgt_write(hgt, dl, VI6_HGT_HUE_AREA(i),
+		vsp1_hgt_write(hgt, dlb, VI6_HGT_HUE_AREA(i),
 			       (lower << VI6_HGT_HUE_AREA_LOWER_SHIFT) |
 			       (upper << VI6_HGT_HUE_AREA_UPPER_SHIFT));
 	}
@@ -173,13 +165,13 @@ static void hgt_configure(struct vsp1_entity *entity,
 
 	hratio = crop->width * 2 / compose->width / 3;
 	vratio = crop->height * 2 / compose->height / 3;
-	vsp1_hgt_write(hgt, dl, VI6_HGT_MODE,
+	vsp1_hgt_write(hgt, dlb, VI6_HGT_MODE,
 		       (hratio << VI6_HGT_MODE_HRATIO_SHIFT) |
 		       (vratio << VI6_HGT_MODE_VRATIO_SHIFT));
 }
 
 static const struct vsp1_entity_operations hgt_entity_ops = {
-	.configure = hgt_configure,
+	.configure_stream = hgt_configure_stream,
 	.destroy = vsp1_histogram_destroy,
 };
 
diff --git a/drivers/media/platform/vsp1/vsp1_hgt.h b/drivers/media/platform/vsp1/vsp1_hgt.h
index 83f2e130942a..38ec237bdd2d 100644
--- a/drivers/media/platform/vsp1/vsp1_hgt.h
+++ b/drivers/media/platform/vsp1/vsp1_hgt.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_hgt.h  --  R-Car VSP1 Histogram Generator 2D
  *
  * Copyright (C) 2016 Renesas Electronics Corporation
  *
  * Contact: Niklas Söderlund (niklas.soderlund@ragnatech.se)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_HGT_H__
 #define __VSP1_HGT_H__
diff --git a/drivers/media/platform/vsp1/vsp1_histo.c b/drivers/media/platform/vsp1/vsp1_histo.c
index afab77cf4fa5..5e15c8ff88d9 100644
--- a/drivers/media/platform/vsp1/vsp1_histo.c
+++ b/drivers/media/platform/vsp1/vsp1_histo.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_histo.c  --  R-Car VSP1 Histogram API
  *
@@ -5,11 +6,6 @@
  * Copyright (C) 2016 Laurent Pinchart
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -61,7 +57,7 @@ void vsp1_histogram_buffer_complete(struct vsp1_histogram *histo,
 				    struct vsp1_histogram_buffer *buf,
 				    size_t size)
 {
-	struct vsp1_pipeline *pipe = histo->pipe;
+	struct vsp1_pipeline *pipe = histo->entity.pipe;
 	unsigned long flags;
 
 	/*
@@ -393,65 +389,14 @@ static int histo_set_format(struct v4l2_subdev *subdev,
 			    struct v4l2_subdev_format *fmt)
 {
 	struct vsp1_histogram *histo = subdev_to_histo(subdev);
-	struct v4l2_subdev_pad_config *config;
-	struct v4l2_mbus_framefmt *format;
-	struct v4l2_rect *selection;
-	unsigned int i;
-	int ret = 0;
 
 	if (fmt->pad != HISTO_PAD_SINK)
 		return histo_get_format(subdev, cfg, fmt);
 
-	mutex_lock(&histo->entity.lock);
-
-	config = vsp1_entity_get_pad_config(&histo->entity, cfg, fmt->which);
-	if (!config) {
-		ret = -EINVAL;
-		goto done;
-	}
-
-	/*
-	 * Default to the first format if the requested format is not
-	 * supported.
-	 */
-	for (i = 0; i < histo->num_formats; ++i) {
-		if (fmt->format.code == histo->formats[i])
-			break;
-	}
-	if (i == histo->num_formats)
-		fmt->format.code = histo->formats[0];
-
-	format = vsp1_entity_get_pad_format(&histo->entity, config, fmt->pad);
-
-	format->code = fmt->format.code;
-	format->width = clamp_t(unsigned int, fmt->format.width,
-				HISTO_MIN_SIZE, HISTO_MAX_SIZE);
-	format->height = clamp_t(unsigned int, fmt->format.height,
-				 HISTO_MIN_SIZE, HISTO_MAX_SIZE);
-	format->field = V4L2_FIELD_NONE;
-	format->colorspace = V4L2_COLORSPACE_SRGB;
-
-	fmt->format = *format;
-
-	/* Reset the crop and compose rectangles */
-	selection = vsp1_entity_get_pad_selection(&histo->entity, config,
-						  fmt->pad, V4L2_SEL_TGT_CROP);
-	selection->left = 0;
-	selection->top = 0;
-	selection->width = format->width;
-	selection->height = format->height;
-
-	selection = vsp1_entity_get_pad_selection(&histo->entity, config,
-						  fmt->pad,
-						  V4L2_SEL_TGT_COMPOSE);
-	selection->left = 0;
-	selection->top = 0;
-	selection->width = format->width;
-	selection->height = format->height;
-
-done:
-	mutex_unlock(&histo->entity.lock);
-	return ret;
+	return vsp1_subdev_set_pad_format(subdev, cfg, fmt,
+					  histo->formats, histo->num_formats,
+					  HISTO_MIN_SIZE, HISTO_MIN_SIZE,
+					  HISTO_MAX_SIZE, HISTO_MAX_SIZE);
 }
 
 static const struct v4l2_subdev_pad_ops histo_pad_ops = {
diff --git a/drivers/media/platform/vsp1/vsp1_histo.h b/drivers/media/platform/vsp1/vsp1_histo.h
index af2874f6031d..06f029846244 100644
--- a/drivers/media/platform/vsp1/vsp1_histo.h
+++ b/drivers/media/platform/vsp1/vsp1_histo.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_histo.h  --  R-Car VSP1 Histogram API
  *
@@ -5,11 +6,6 @@
  * Copyright (C) 2016 Laurent Pinchart
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_HISTO_H__
 #define __VSP1_HISTO_H__
@@ -25,7 +21,6 @@
 #include "vsp1_entity.h"
 
 struct vsp1_device;
-struct vsp1_pipeline;
 
 #define HISTO_PAD_SINK				0
 #define HISTO_PAD_SOURCE			1
@@ -37,8 +32,6 @@ struct vsp1_histogram_buffer {
 };
 
 struct vsp1_histogram {
-	struct vsp1_pipeline *pipe;
-
 	struct vsp1_entity entity;
 	struct video_device video;
 	struct media_pad pad;
diff --git a/drivers/media/platform/vsp1/vsp1_hsit.c b/drivers/media/platform/vsp1/vsp1_hsit.c
index 764d405345ee..39ab2e0c7c18 100644
--- a/drivers/media/platform/vsp1/vsp1_hsit.c
+++ b/drivers/media/platform/vsp1/vsp1_hsit.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_hsit.c  --  R-Car VSP1 Hue Saturation value (Inverse) Transform
  *
  * Copyright (C) 2013 Renesas Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -28,9 +24,9 @@
  */
 
 static inline void vsp1_hsit_write(struct vsp1_hsit *hsit,
-				   struct vsp1_dl_list *dl, u32 reg, u32 data)
+				   struct vsp1_dl_body *dlb, u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg, data);
+	vsp1_dl_body_write(dlb, reg, data);
 }
 
 /* -----------------------------------------------------------------------------
@@ -131,24 +127,20 @@ static const struct v4l2_subdev_ops hsit_ops = {
  * VSP1 Entity Operations
  */
 
-static void hsit_configure(struct vsp1_entity *entity,
-			   struct vsp1_pipeline *pipe,
-			   struct vsp1_dl_list *dl,
-			   enum vsp1_entity_params params)
+static void hsit_configure_stream(struct vsp1_entity *entity,
+				  struct vsp1_pipeline *pipe,
+				  struct vsp1_dl_body *dlb)
 {
 	struct vsp1_hsit *hsit = to_hsit(&entity->subdev);
 
-	if (params != VSP1_ENTITY_PARAMS_INIT)
-		return;
-
 	if (hsit->inverse)
-		vsp1_hsit_write(hsit, dl, VI6_HSI_CTRL, VI6_HSI_CTRL_EN);
+		vsp1_hsit_write(hsit, dlb, VI6_HSI_CTRL, VI6_HSI_CTRL_EN);
 	else
-		vsp1_hsit_write(hsit, dl, VI6_HST_CTRL, VI6_HST_CTRL_EN);
+		vsp1_hsit_write(hsit, dlb, VI6_HST_CTRL, VI6_HST_CTRL_EN);
 }
 
 static const struct vsp1_entity_operations hsit_entity_ops = {
-	.configure = hsit_configure,
+	.configure_stream = hsit_configure_stream,
 };
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/media/platform/vsp1/vsp1_hsit.h b/drivers/media/platform/vsp1/vsp1_hsit.h
index 82f1c8426900..a658b1aa49e7 100644
--- a/drivers/media/platform/vsp1/vsp1_hsit.h
+++ b/drivers/media/platform/vsp1/vsp1_hsit.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_hsit.h  --  R-Car VSP1 Hue Saturation value (Inverse) Transform
  *
  * Copyright (C) 2013 Renesas Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_HSIT_H__
 #define __VSP1_HSIT_H__
diff --git a/drivers/media/platform/vsp1/vsp1_lif.c b/drivers/media/platform/vsp1/vsp1_lif.c
index 704920753998..0cb63244b21a 100644
--- a/drivers/media/platform/vsp1/vsp1_lif.c
+++ b/drivers/media/platform/vsp1/vsp1_lif.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_lif.c  --  R-Car VSP1 LCD Controller Interface
  *
  * Copyright (C) 2013-2014 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -27,27 +23,28 @@
  * Device Access
  */
 
-static inline void vsp1_lif_write(struct vsp1_lif *lif, struct vsp1_dl_list *dl,
-				  u32 reg, u32 data)
+static inline void vsp1_lif_write(struct vsp1_lif *lif,
+				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg + lif->entity.index * VI6_LIF_OFFSET, data);
+	vsp1_dl_body_write(dlb, reg + lif->entity.index * VI6_LIF_OFFSET,
+			       data);
 }
 
 /* -----------------------------------------------------------------------------
  * V4L2 Subdevice Operations
  */
 
+static const unsigned int lif_codes[] = {
+	MEDIA_BUS_FMT_ARGB8888_1X32,
+	MEDIA_BUS_FMT_AYUV8_1X32,
+};
+
 static int lif_enum_mbus_code(struct v4l2_subdev *subdev,
 			      struct v4l2_subdev_pad_config *cfg,
 			      struct v4l2_subdev_mbus_code_enum *code)
 {
-	static const unsigned int codes[] = {
-		MEDIA_BUS_FMT_ARGB8888_1X32,
-		MEDIA_BUS_FMT_AYUV8_1X32,
-	};
-
-	return vsp1_subdev_enum_mbus_code(subdev, cfg, code, codes,
-					  ARRAY_SIZE(codes));
+	return vsp1_subdev_enum_mbus_code(subdev, cfg, code, lif_codes,
+					  ARRAY_SIZE(lif_codes));
 }
 
 static int lif_enum_frame_size(struct v4l2_subdev *subdev,
@@ -63,53 +60,10 @@ static int lif_set_format(struct v4l2_subdev *subdev,
 			  struct v4l2_subdev_pad_config *cfg,
 			  struct v4l2_subdev_format *fmt)
 {
-	struct vsp1_lif *lif = to_lif(subdev);
-	struct v4l2_subdev_pad_config *config;
-	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
-
-	mutex_lock(&lif->entity.lock);
-
-	config = vsp1_entity_get_pad_config(&lif->entity, cfg, fmt->which);
-	if (!config) {
-		ret = -EINVAL;
-		goto done;
-	}
-
-	/* Default to YUV if the requested format is not supported. */
-	if (fmt->format.code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
-	    fmt->format.code != MEDIA_BUS_FMT_AYUV8_1X32)
-		fmt->format.code = MEDIA_BUS_FMT_AYUV8_1X32;
-
-	format = vsp1_entity_get_pad_format(&lif->entity, config, fmt->pad);
-
-	if (fmt->pad == LIF_PAD_SOURCE) {
-		/*
-		 * The LIF source format is always identical to its sink
-		 * format.
-		 */
-		fmt->format = *format;
-		goto done;
-	}
-
-	format->code = fmt->format.code;
-	format->width = clamp_t(unsigned int, fmt->format.width,
-				LIF_MIN_SIZE, LIF_MAX_SIZE);
-	format->height = clamp_t(unsigned int, fmt->format.height,
-				 LIF_MIN_SIZE, LIF_MAX_SIZE);
-	format->field = V4L2_FIELD_NONE;
-	format->colorspace = V4L2_COLORSPACE_SRGB;
-
-	fmt->format = *format;
-
-	/* Propagate the format to the source pad. */
-	format = vsp1_entity_get_pad_format(&lif->entity, config,
-					    LIF_PAD_SOURCE);
-	*format = fmt->format;
-
-done:
-	mutex_unlock(&lif->entity.lock);
-	return ret;
+	return vsp1_subdev_set_pad_format(subdev, cfg, fmt, lif_codes,
+					  ARRAY_SIZE(lif_codes),
+					  LIF_MIN_SIZE, LIF_MIN_SIZE,
+					  LIF_MAX_SIZE, LIF_MAX_SIZE);
 }
 
 static const struct v4l2_subdev_pad_ops lif_pad_ops = {
@@ -128,10 +82,9 @@ static const struct v4l2_subdev_ops lif_ops = {
  * VSP1 Entity Operations
  */
 
-static void lif_configure(struct vsp1_entity *entity,
-			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl,
-			  enum vsp1_entity_params params)
+static void lif_configure_stream(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_body *dlb)
 {
 	const struct v4l2_mbus_framefmt *format;
 	struct vsp1_lif *lif = to_lif(&entity->subdev);
@@ -139,19 +92,16 @@ static void lif_configure(struct vsp1_entity *entity,
 	unsigned int obth = 400;
 	unsigned int lbth = 200;
 
-	if (params != VSP1_ENTITY_PARAMS_INIT)
-		return;
-
 	format = vsp1_entity_get_pad_format(&lif->entity, lif->entity.config,
 					    LIF_PAD_SOURCE);
 
 	obth = min(obth, (format->width + 1) / 2 * format->height - 4);
 
-	vsp1_lif_write(lif, dl, VI6_LIF_CSBTH,
+	vsp1_lif_write(lif, dlb, VI6_LIF_CSBTH,
 			(hbth << VI6_LIF_CSBTH_HBTH_SHIFT) |
 			(lbth << VI6_LIF_CSBTH_LBTH_SHIFT));
 
-	vsp1_lif_write(lif, dl, VI6_LIF_CTRL,
+	vsp1_lif_write(lif, dlb, VI6_LIF_CTRL,
 			(obth << VI6_LIF_CTRL_OBTH_SHIFT) |
 			(format->code == 0 ? VI6_LIF_CTRL_CFMT : 0) |
 			VI6_LIF_CTRL_REQSEL | VI6_LIF_CTRL_LIF_EN);
@@ -164,13 +114,13 @@ static void lif_configure(struct vsp1_entity *entity,
 	 */
 	if ((entity->vsp1->version & VI6_IP_VERSION_MASK) ==
 	    (VI6_IP_VERSION_MODEL_VSPD_V3 | VI6_IP_VERSION_SOC_V3M))
-		vsp1_lif_write(lif, dl, VI6_LIF_LBA,
+		vsp1_lif_write(lif, dlb, VI6_LIF_LBA,
 			       VI6_LIF_LBA_LBA0 |
 			       (1536 << VI6_LIF_LBA_LBA1_SHIFT));
 }
 
 static const struct vsp1_entity_operations lif_entity_ops = {
-	.configure = lif_configure,
+	.configure_stream = lif_configure_stream,
 };
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/media/platform/vsp1/vsp1_lif.h b/drivers/media/platform/vsp1/vsp1_lif.h
index 3417339379b1..71a4eda9c2b2 100644
--- a/drivers/media/platform/vsp1/vsp1_lif.h
+++ b/drivers/media/platform/vsp1/vsp1_lif.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_lif.h  --  R-Car VSP1 LCD Controller Interface
  *
  * Copyright (C) 2013-2014 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_LIF_H__
 #define __VSP1_LIF_H__
diff --git a/drivers/media/platform/vsp1/vsp1_lut.c b/drivers/media/platform/vsp1/vsp1_lut.c
index c67cc60db0db..64c48d9459b0 100644
--- a/drivers/media/platform/vsp1/vsp1_lut.c
+++ b/drivers/media/platform/vsp1/vsp1_lut.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_lut.c  --  R-Car VSP1 Look-Up Table
  *
  * Copyright (C) 2013 Renesas Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -23,14 +19,16 @@
 #define LUT_MIN_SIZE				4U
 #define LUT_MAX_SIZE				8190U
 
+#define LUT_SIZE				256
+
 /* -----------------------------------------------------------------------------
  * Device Access
  */
 
-static inline void vsp1_lut_write(struct vsp1_lut *lut, struct vsp1_dl_list *dl,
-				  u32 reg, u32 data)
+static inline void vsp1_lut_write(struct vsp1_lut *lut,
+				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg, data);
+	vsp1_dl_body_write(dlb, reg, data);
 }
 
 /* -----------------------------------------------------------------------------
@@ -44,19 +42,19 @@ static int lut_set_table(struct vsp1_lut *lut, struct v4l2_ctrl *ctrl)
 	struct vsp1_dl_body *dlb;
 	unsigned int i;
 
-	dlb = vsp1_dl_fragment_alloc(lut->entity.vsp1, 256);
+	dlb = vsp1_dl_body_get(lut->pool);
 	if (!dlb)
 		return -ENOMEM;
 
-	for (i = 0; i < 256; ++i)
-		vsp1_dl_fragment_write(dlb, VI6_LUT_TABLE + 4 * i,
+	for (i = 0; i < LUT_SIZE; ++i)
+		vsp1_dl_body_write(dlb, VI6_LUT_TABLE + 4 * i,
 				       ctrl->p_new.p_u32[i]);
 
 	spin_lock_irq(&lut->lock);
 	swap(lut->lut, dlb);
 	spin_unlock_irq(&lut->lock);
 
-	vsp1_dl_fragment_free(dlb);
+	vsp1_dl_body_put(dlb);
 	return 0;
 }
 
@@ -87,25 +85,25 @@ static const struct v4l2_ctrl_config lut_table_control = {
 	.max = 0x00ffffff,
 	.step = 1,
 	.def = 0,
-	.dims = { 256},
+	.dims = { LUT_SIZE },
 };
 
 /* -----------------------------------------------------------------------------
  * V4L2 Subdevice Pad Operations
  */
 
+static const unsigned int lut_codes[] = {
+	MEDIA_BUS_FMT_ARGB8888_1X32,
+	MEDIA_BUS_FMT_AHSV8888_1X32,
+	MEDIA_BUS_FMT_AYUV8_1X32,
+};
+
 static int lut_enum_mbus_code(struct v4l2_subdev *subdev,
 			      struct v4l2_subdev_pad_config *cfg,
 			      struct v4l2_subdev_mbus_code_enum *code)
 {
-	static const unsigned int codes[] = {
-		MEDIA_BUS_FMT_ARGB8888_1X32,
-		MEDIA_BUS_FMT_AHSV8888_1X32,
-		MEDIA_BUS_FMT_AYUV8_1X32,
-	};
-
-	return vsp1_subdev_enum_mbus_code(subdev, cfg, code, codes,
-					  ARRAY_SIZE(codes));
+	return vsp1_subdev_enum_mbus_code(subdev, cfg, code, lut_codes,
+					  ARRAY_SIZE(lut_codes));
 }
 
 static int lut_enum_frame_size(struct v4l2_subdev *subdev,
@@ -121,51 +119,10 @@ static int lut_set_format(struct v4l2_subdev *subdev,
 			  struct v4l2_subdev_pad_config *cfg,
 			  struct v4l2_subdev_format *fmt)
 {
-	struct vsp1_lut *lut = to_lut(subdev);
-	struct v4l2_subdev_pad_config *config;
-	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
-
-	mutex_lock(&lut->entity.lock);
-
-	config = vsp1_entity_get_pad_config(&lut->entity, cfg, fmt->which);
-	if (!config) {
-		ret = -EINVAL;
-		goto done;
-	}
-
-	/* Default to YUV if the requested format is not supported. */
-	if (fmt->format.code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
-	    fmt->format.code != MEDIA_BUS_FMT_AHSV8888_1X32 &&
-	    fmt->format.code != MEDIA_BUS_FMT_AYUV8_1X32)
-		fmt->format.code = MEDIA_BUS_FMT_AYUV8_1X32;
-
-	format = vsp1_entity_get_pad_format(&lut->entity, config, fmt->pad);
-
-	if (fmt->pad == LUT_PAD_SOURCE) {
-		/* The LUT output format can't be modified. */
-		fmt->format = *format;
-		goto done;
-	}
-
-	format->code = fmt->format.code;
-	format->width = clamp_t(unsigned int, fmt->format.width,
-				LUT_MIN_SIZE, LUT_MAX_SIZE);
-	format->height = clamp_t(unsigned int, fmt->format.height,
-				 LUT_MIN_SIZE, LUT_MAX_SIZE);
-	format->field = V4L2_FIELD_NONE;
-	format->colorspace = V4L2_COLORSPACE_SRGB;
-
-	fmt->format = *format;
-
-	/* Propagate the format to the source pad. */
-	format = vsp1_entity_get_pad_format(&lut->entity, config,
-					    LUT_PAD_SOURCE);
-	*format = fmt->format;
-
-done:
-	mutex_unlock(&lut->entity.lock);
-	return ret;
+	return vsp1_subdev_set_pad_format(subdev, cfg, fmt, lut_codes,
+					  ARRAY_SIZE(lut_codes),
+					  LUT_MIN_SIZE, LUT_MIN_SIZE,
+					  LUT_MAX_SIZE, LUT_MAX_SIZE);
 }
 
 /* -----------------------------------------------------------------------------
@@ -188,37 +145,48 @@ static const struct v4l2_subdev_ops lut_ops = {
  * VSP1 Entity Operations
  */
 
-static void lut_configure(struct vsp1_entity *entity,
-			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl,
-			  enum vsp1_entity_params params)
+static void lut_configure_stream(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_lut *lut = to_lut(&entity->subdev);
-	struct vsp1_dl_body *dlb;
-	unsigned long flags;
 
-	switch (params) {
-	case VSP1_ENTITY_PARAMS_INIT:
-		vsp1_lut_write(lut, dl, VI6_LUT_CTRL, VI6_LUT_CTRL_EN);
-		break;
+	vsp1_lut_write(lut, dlb, VI6_LUT_CTRL, VI6_LUT_CTRL_EN);
+}
 
-	case VSP1_ENTITY_PARAMS_PARTITION:
-		break;
+static void lut_configure_frame(struct vsp1_entity *entity,
+				struct vsp1_pipeline *pipe,
+				struct vsp1_dl_list *dl,
+				struct vsp1_dl_body *dlb)
+{
+	struct vsp1_lut *lut = to_lut(&entity->subdev);
+	struct vsp1_dl_body *lut_dlb;
+	unsigned long flags;
 
-	case VSP1_ENTITY_PARAMS_RUNTIME:
-		spin_lock_irqsave(&lut->lock, flags);
-		dlb = lut->lut;
-		lut->lut = NULL;
-		spin_unlock_irqrestore(&lut->lock, flags);
+	spin_lock_irqsave(&lut->lock, flags);
+	lut_dlb = lut->lut;
+	lut->lut = NULL;
+	spin_unlock_irqrestore(&lut->lock, flags);
 
-		if (dlb)
-			vsp1_dl_list_add_fragment(dl, dlb);
-		break;
+	if (lut_dlb) {
+		vsp1_dl_list_add_body(dl, lut_dlb);
+
+		/* Release our local reference. */
+		vsp1_dl_body_put(lut_dlb);
 	}
 }
 
+static void lut_destroy(struct vsp1_entity *entity)
+{
+	struct vsp1_lut *lut = to_lut(&entity->subdev);
+
+	vsp1_dl_body_pool_destroy(lut->pool);
+}
+
 static const struct vsp1_entity_operations lut_entity_ops = {
-	.configure = lut_configure,
+	.configure_stream = lut_configure_stream,
+	.configure_frame = lut_configure_frame,
+	.destroy = lut_destroy,
 };
 
 /* -----------------------------------------------------------------------------
@@ -244,6 +212,15 @@ struct vsp1_lut *vsp1_lut_create(struct vsp1_device *vsp1)
 	if (ret < 0)
 		return ERR_PTR(ret);
 
+	/*
+	 * Pre-allocate a body pool, with 3 bodies allowing a userspace update
+	 * before the hardware has committed a previous set of tables, handling
+	 * both the queued and pending dl entries.
+	 */
+	lut->pool = vsp1_dl_body_pool_create(vsp1, 3, LUT_SIZE, 0);
+	if (!lut->pool)
+		return ERR_PTR(-ENOMEM);
+
 	/* Initialize the control handler. */
 	v4l2_ctrl_handler_init(&lut->ctrls, 1);
 	v4l2_ctrl_new_custom(&lut->ctrls, &lut_table_control, NULL);
diff --git a/drivers/media/platform/vsp1/vsp1_lut.h b/drivers/media/platform/vsp1/vsp1_lut.h
index f8c4e8f0a79d..8cb0df1b7e27 100644
--- a/drivers/media/platform/vsp1/vsp1_lut.h
+++ b/drivers/media/platform/vsp1/vsp1_lut.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_lut.h  --  R-Car VSP1 Look-Up Table
  *
  * Copyright (C) 2013 Renesas Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_LUT_H__
 #define __VSP1_LUT_H__
@@ -33,6 +29,7 @@ struct vsp1_lut {
 
 	spinlock_t lock;
 	struct vsp1_dl_body *lut;
+	struct vsp1_dl_body_pool *pool;
 };
 
 static inline struct vsp1_lut *to_lut(struct v4l2_subdev *subdev)
diff --git a/drivers/media/platform/vsp1/vsp1_pipe.c b/drivers/media/platform/vsp1/vsp1_pipe.c
index 44944ac86d9b..54ff539ffea0 100644
--- a/drivers/media/platform/vsp1/vsp1_pipe.c
+++ b/drivers/media/platform/vsp1/vsp1_pipe.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_pipe.c  --  R-Car VSP1 Pipeline
  *
  * Copyright (C) 2013-2015 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/delay.h>
@@ -20,7 +16,7 @@
 #include <media/v4l2-subdev.h>
 
 #include "vsp1.h"
-#include "vsp1_bru.h"
+#include "vsp1_brx.h"
 #include "vsp1_dl.h"
 #include "vsp1_entity.h"
 #include "vsp1_hgo.h"
@@ -185,44 +181,29 @@ const struct vsp1_format_info *vsp1_get_format_info(struct vsp1_device *vsp1,
 
 void vsp1_pipeline_reset(struct vsp1_pipeline *pipe)
 {
+	struct vsp1_entity *entity;
 	unsigned int i;
 
-	if (pipe->bru) {
-		struct vsp1_bru *bru = to_bru(&pipe->bru->subdev);
-
-		for (i = 0; i < ARRAY_SIZE(bru->inputs); ++i)
-			bru->inputs[i].rpf = NULL;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(pipe->inputs); ++i) {
-		if (pipe->inputs[i]) {
-			pipe->inputs[i]->pipe = NULL;
-			pipe->inputs[i] = NULL;
-		}
-	}
+	if (pipe->brx) {
+		struct vsp1_brx *brx = to_brx(&pipe->brx->subdev);
 
-	if (pipe->output) {
-		pipe->output->pipe = NULL;
-		pipe->output = NULL;
+		for (i = 0; i < ARRAY_SIZE(brx->inputs); ++i)
+			brx->inputs[i].rpf = NULL;
 	}
 
-	if (pipe->hgo) {
-		struct vsp1_hgo *hgo = to_hgo(&pipe->hgo->subdev);
-
-		hgo->histo.pipe = NULL;
-	}
+	for (i = 0; i < ARRAY_SIZE(pipe->inputs); ++i)
+		pipe->inputs[i] = NULL;
 
-	if (pipe->hgt) {
-		struct vsp1_hgt *hgt = to_hgt(&pipe->hgt->subdev);
+	pipe->output = NULL;
 
-		hgt->histo.pipe = NULL;
-	}
+	list_for_each_entry(entity, &pipe->entities, list_pipe)
+		entity->pipe = NULL;
 
 	INIT_LIST_HEAD(&pipe->entities);
 	pipe->state = VSP1_PIPELINE_STOPPED;
 	pipe->buffers_ready = 0;
 	pipe->num_inputs = 0;
-	pipe->bru = NULL;
+	pipe->brx = NULL;
 	pipe->hgo = NULL;
 	pipe->hgt = NULL;
 	pipe->lif = NULL;
@@ -330,17 +311,17 @@ bool vsp1_pipeline_ready(struct vsp1_pipeline *pipe)
 
 void vsp1_pipeline_frame_end(struct vsp1_pipeline *pipe)
 {
-	bool completed;
+	unsigned int flags;
 
 	if (pipe == NULL)
 		return;
 
 	/*
 	 * If the DL commit raced with the frame end interrupt, the commit ends
-	 * up being postponed by one frame. @completed represents whether the
+	 * up being postponed by one frame. The returned flags tell whether the
 	 * active frame was finished or postponed.
 	 */
-	completed = vsp1_dlm_irq_frame_end(pipe->output->dlm);
+	flags = vsp1_dlm_irq_frame_end(pipe->output->dlm);
 
 	if (pipe->hgo)
 		vsp1_hgo_frame_end(pipe->hgo);
@@ -353,7 +334,7 @@ void vsp1_pipeline_frame_end(struct vsp1_pipeline *pipe)
 	 * frame_end to account for vblank events.
 	 */
 	if (pipe->frame_end)
-		pipe->frame_end(pipe, completed);
+		pipe->frame_end(pipe, flags);
 
 	pipe->sequence++;
 }
@@ -367,7 +348,7 @@ void vsp1_pipeline_frame_end(struct vsp1_pipeline *pipe)
  * from the input RPF alpha.
  */
 void vsp1_pipeline_propagate_alpha(struct vsp1_pipeline *pipe,
-				   struct vsp1_dl_list *dl, unsigned int alpha)
+				   struct vsp1_dl_body *dlb, unsigned int alpha)
 {
 	if (!pipe->uds)
 		return;
@@ -380,7 +361,7 @@ void vsp1_pipeline_propagate_alpha(struct vsp1_pipeline *pipe,
 	    pipe->uds_input->type == VSP1_ENTITY_BRS)
 		alpha = 255;
 
-	vsp1_uds_set_alpha(pipe->uds, dl, alpha);
+	vsp1_uds_set_alpha(pipe->uds, dlb, alpha);
 }
 
 /*
@@ -405,73 +386,3 @@ void vsp1_pipeline_propagate_partition(struct vsp1_pipeline *pipe,
 	}
 }
 
-void vsp1_pipelines_suspend(struct vsp1_device *vsp1)
-{
-	unsigned long flags;
-	unsigned int i;
-	int ret;
-
-	/*
-	 * To avoid increasing the system suspend time needlessly, loop over the
-	 * pipelines twice, first to set them all to the stopping state, and
-	 * then to wait for the stop to complete.
-	 */
-	for (i = 0; i < vsp1->info->wpf_count; ++i) {
-		struct vsp1_rwpf *wpf = vsp1->wpf[i];
-		struct vsp1_pipeline *pipe;
-
-		if (wpf == NULL)
-			continue;
-
-		pipe = wpf->pipe;
-		if (pipe == NULL)
-			continue;
-
-		spin_lock_irqsave(&pipe->irqlock, flags);
-		if (pipe->state == VSP1_PIPELINE_RUNNING)
-			pipe->state = VSP1_PIPELINE_STOPPING;
-		spin_unlock_irqrestore(&pipe->irqlock, flags);
-	}
-
-	for (i = 0; i < vsp1->info->wpf_count; ++i) {
-		struct vsp1_rwpf *wpf = vsp1->wpf[i];
-		struct vsp1_pipeline *pipe;
-
-		if (wpf == NULL)
-			continue;
-
-		pipe = wpf->pipe;
-		if (pipe == NULL)
-			continue;
-
-		ret = wait_event_timeout(pipe->wq, vsp1_pipeline_stopped(pipe),
-					 msecs_to_jiffies(500));
-		if (ret == 0)
-			dev_warn(vsp1->dev, "pipeline %u stop timeout\n",
-				 wpf->entity.index);
-	}
-}
-
-void vsp1_pipelines_resume(struct vsp1_device *vsp1)
-{
-	unsigned long flags;
-	unsigned int i;
-
-	/* Resume all running pipelines. */
-	for (i = 0; i < vsp1->info->wpf_count; ++i) {
-		struct vsp1_rwpf *wpf = vsp1->wpf[i];
-		struct vsp1_pipeline *pipe;
-
-		if (wpf == NULL)
-			continue;
-
-		pipe = wpf->pipe;
-		if (pipe == NULL)
-			continue;
-
-		spin_lock_irqsave(&pipe->irqlock, flags);
-		if (vsp1_pipeline_ready(pipe))
-			vsp1_pipeline_run(pipe);
-		spin_unlock_irqrestore(&pipe->irqlock, flags);
-	}
-}
diff --git a/drivers/media/platform/vsp1/vsp1_pipe.h b/drivers/media/platform/vsp1/vsp1_pipe.h
index dfff9b5685fe..743d8f0db45c 100644
--- a/drivers/media/platform/vsp1/vsp1_pipe.h
+++ b/drivers/media/platform/vsp1/vsp1_pipe.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_pipe.h  --  R-Car VSP1 Pipeline
  *
  * Copyright (C) 2013-2015 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_PIPE_H__
 #define __VSP1_PIPE_H__
@@ -99,14 +95,15 @@ struct vsp1_partition {
  * @num_inputs: number of RPFs
  * @inputs: array of RPFs in the pipeline (indexed by RPF index)
  * @output: WPF at the output of the pipeline
- * @bru: BRU entity, if present
+ * @brx: BRx entity, if present
  * @hgo: HGO entity, if present
  * @hgt: HGT entity, if present
  * @lif: LIF entity, if present
  * @uds: UDS entity, if present
  * @uds_input: entity at the input of the UDS, if the UDS is present
  * @entities: list of entities in the pipeline
- * @dl: display list associated with the pipeline
+ * @stream_config: cached stream configuration for video pipelines
+ * @configured: when false the @stream_config shall be written to the hardware
  * @partitions: The number of partitions used to process one frame
  * @partition: The current partition for configuration to process
  * @part_table: The pre-calculated partitions used by the pipeline
@@ -118,7 +115,7 @@ struct vsp1_pipeline {
 	enum vsp1_pipeline_state state;
 	wait_queue_head_t wq;
 
-	void (*frame_end)(struct vsp1_pipeline *pipe, bool completed);
+	void (*frame_end)(struct vsp1_pipeline *pipe, unsigned int completion);
 
 	struct mutex lock;
 	struct kref kref;
@@ -129,7 +126,7 @@ struct vsp1_pipeline {
 	unsigned int num_inputs;
 	struct vsp1_rwpf *inputs[VSP1_MAX_RPF];
 	struct vsp1_rwpf *output;
-	struct vsp1_entity *bru;
+	struct vsp1_entity *brx;
 	struct vsp1_entity *hgo;
 	struct vsp1_entity *hgt;
 	struct vsp1_entity *lif;
@@ -143,7 +140,8 @@ struct vsp1_pipeline {
 	 */
 	struct list_head entities;
 
-	struct vsp1_dl_list *dl;
+	struct vsp1_dl_body *stream_config;
+	bool configured;
 
 	unsigned int partitions;
 	struct vsp1_partition *partition;
@@ -161,16 +159,14 @@ bool vsp1_pipeline_ready(struct vsp1_pipeline *pipe);
 void vsp1_pipeline_frame_end(struct vsp1_pipeline *pipe);
 
 void vsp1_pipeline_propagate_alpha(struct vsp1_pipeline *pipe,
-				   struct vsp1_dl_list *dl, unsigned int alpha);
+				   struct vsp1_dl_body *dlb,
+				   unsigned int alpha);
 
 void vsp1_pipeline_propagate_partition(struct vsp1_pipeline *pipe,
 				       struct vsp1_partition *partition,
 				       unsigned int index,
 				       struct vsp1_partition_window *window);
 
-void vsp1_pipelines_suspend(struct vsp1_device *vsp1);
-void vsp1_pipelines_resume(struct vsp1_device *vsp1);
-
 const struct vsp1_format_info *vsp1_get_format_info(struct vsp1_device *vsp1,
 						    u32 fourcc);
 
diff --git a/drivers/media/platform/vsp1/vsp1_regs.h b/drivers/media/platform/vsp1/vsp1_regs.h
index dae0c1901297..0d249ff9f564 100644
--- a/drivers/media/platform/vsp1/vsp1_regs.h
+++ b/drivers/media/platform/vsp1/vsp1_regs.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * vsp1_regs.h  --  R-Car VSP1 Registers Definitions
  *
  * Copyright (C) 2013 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2
- * as published by the Free Software Foundation.
  */
 
 #ifndef __VSP1_REGS_H__
@@ -310,6 +307,44 @@
 #define VI6_WPF_WRBCK_CTRL		0x1034
 #define VI6_WPF_WRBCK_CTRL_WBMD		(1 << 0)
 
+/* -----------------------------------------------------------------------------
+ * UIF Control Registers
+ */
+
+#define VI6_UIF_OFFSET			0x100
+
+#define VI6_UIF_DISCOM_DOCMCR		0x1c00
+#define VI6_UIF_DISCOM_DOCMCR_CMPRU	(1 << 16)
+#define VI6_UIF_DISCOM_DOCMCR_CMPR	(1 << 0)
+
+#define VI6_UIF_DISCOM_DOCMSTR		0x1c04
+#define VI6_UIF_DISCOM_DOCMSTR_CMPPRE	(1 << 1)
+#define VI6_UIF_DISCOM_DOCMSTR_CMPST	(1 << 0)
+
+#define VI6_UIF_DISCOM_DOCMCLSTR	0x1c08
+#define VI6_UIF_DISCOM_DOCMCLSTR_CMPCLPRE	(1 << 1)
+#define VI6_UIF_DISCOM_DOCMCLSTR_CMPCLST	(1 << 0)
+
+#define VI6_UIF_DISCOM_DOCMIENR		0x1c0c
+#define VI6_UIF_DISCOM_DOCMIENR_CMPPREIEN	(1 << 1)
+#define VI6_UIF_DISCOM_DOCMIENR_CMPIEN		(1 << 0)
+
+#define VI6_UIF_DISCOM_DOCMMDR		0x1c10
+#define VI6_UIF_DISCOM_DOCMMDR_INTHRH(n)	((n) << 16)
+
+#define VI6_UIF_DISCOM_DOCMPMR		0x1c14
+#define VI6_UIF_DISCOM_DOCMPMR_CMPDFF(n)	((n) << 17)
+#define VI6_UIF_DISCOM_DOCMPMR_CMPDFA(n)	((n) << 8)
+#define VI6_UIF_DISCOM_DOCMPMR_CMPDAUF		(1 << 7)
+#define VI6_UIF_DISCOM_DOCMPMR_SEL(n)		((n) << 0)
+
+#define VI6_UIF_DISCOM_DOCMECRCR	0x1c18
+#define VI6_UIF_DISCOM_DOCMCCRCR	0x1c1c
+#define VI6_UIF_DISCOM_DOCMSPXR		0x1c20
+#define VI6_UIF_DISCOM_DOCMSPYR		0x1c24
+#define VI6_UIF_DISCOM_DOCMSZXR		0x1c28
+#define VI6_UIF_DISCOM_DOCMSZYR		0x1c2c
+
 /* -----------------------------------------------------------------------------
  * DPR Control Registers
  */
@@ -342,7 +377,10 @@
 #define VI6_DPR_SMPPT_PT_MASK		(0x3f << 0)
 #define VI6_DPR_SMPPT_PT_SHIFT		0
 
+#define VI6_DPR_UIF_ROUTE(n)		(0x2074 + (n) * 4)
+
 #define VI6_DPR_NODE_RPF(n)		(n)
+#define VI6_DPR_NODE_UIF(n)		(12 + (n))
 #define VI6_DPR_NODE_SRU		16
 #define VI6_DPR_NODE_UDS(n)		(17 + (n))
 #define VI6_DPR_NODE_LUT		22
diff --git a/drivers/media/platform/vsp1/vsp1_rpf.c b/drivers/media/platform/vsp1/vsp1_rpf.c
index fe0633da5a5f..69e5fe6e6b50 100644
--- a/drivers/media/platform/vsp1/vsp1_rpf.c
+++ b/drivers/media/platform/vsp1/vsp1_rpf.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_rpf.c  --  R-Car VSP1 Read Pixel Formatter
  *
  * Copyright (C) 2013-2014 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -29,9 +25,10 @@
  */
 
 static inline void vsp1_rpf_write(struct vsp1_rwpf *rpf,
-				  struct vsp1_dl_list *dl, u32 reg, u32 data)
+				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg + rpf->entity.index * VI6_RPF_OFFSET, data);
+	vsp1_dl_body_write(dlb, reg + rpf->entity.index * VI6_RPF_OFFSET,
+			       data);
 }
 
 /* -----------------------------------------------------------------------------
@@ -46,10 +43,9 @@ static const struct v4l2_subdev_ops rpf_ops = {
  * VSP1 Entity Operations
  */
 
-static void rpf_configure(struct vsp1_entity *entity,
-			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl,
-			  enum vsp1_entity_params params)
+static void rpf_configure_stream(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_rwpf *rpf = to_rwpf(&entity->subdev);
 	const struct vsp1_format_info *fmtinfo = rpf->fmtinfo;
@@ -61,80 +57,6 @@ static void rpf_configure(struct vsp1_entity *entity,
 	u32 pstride;
 	u32 infmt;
 
-	if (params == VSP1_ENTITY_PARAMS_RUNTIME) {
-		vsp1_rpf_write(rpf, dl, VI6_RPF_VRTCOL_SET,
-			       rpf->alpha << VI6_RPF_VRTCOL_SET_LAYA_SHIFT);
-		vsp1_rpf_write(rpf, dl, VI6_RPF_MULT_ALPHA, rpf->mult_alpha |
-			       (rpf->alpha << VI6_RPF_MULT_ALPHA_RATIO_SHIFT));
-
-		vsp1_pipeline_propagate_alpha(pipe, dl, rpf->alpha);
-		return;
-	}
-
-	if (params == VSP1_ENTITY_PARAMS_PARTITION) {
-		struct vsp1_device *vsp1 = rpf->entity.vsp1;
-		struct vsp1_rwpf_memory mem = rpf->mem;
-		struct v4l2_rect crop;
-
-		/*
-		 * Source size and crop offsets.
-		 *
-		 * The crop offsets correspond to the location of the crop
-		 * rectangle top left corner in the plane buffer. Only two
-		 * offsets are needed, as planes 2 and 3 always have identical
-		 * strides.
-		 */
-		crop = *vsp1_rwpf_get_crop(rpf, rpf->entity.config);
-
-		/*
-		 * Partition Algorithm Control
-		 *
-		 * The partition algorithm can split this frame into multiple
-		 * slices. We must scale our partition window based on the pipe
-		 * configuration to match the destination partition window.
-		 * To achieve this, we adjust our crop to provide a 'sub-crop'
-		 * matching the expected partition window. Only 'left' and
-		 * 'width' need to be adjusted.
-		 */
-		if (pipe->partitions > 1) {
-			crop.width = pipe->partition->rpf.width;
-			crop.left += pipe->partition->rpf.left;
-		}
-
-		vsp1_rpf_write(rpf, dl, VI6_RPF_SRC_BSIZE,
-			       (crop.width << VI6_RPF_SRC_BSIZE_BHSIZE_SHIFT) |
-			       (crop.height << VI6_RPF_SRC_BSIZE_BVSIZE_SHIFT));
-		vsp1_rpf_write(rpf, dl, VI6_RPF_SRC_ESIZE,
-			       (crop.width << VI6_RPF_SRC_ESIZE_EHSIZE_SHIFT) |
-			       (crop.height << VI6_RPF_SRC_ESIZE_EVSIZE_SHIFT));
-
-		mem.addr[0] += crop.top * format->plane_fmt[0].bytesperline
-			     + crop.left * fmtinfo->bpp[0] / 8;
-
-		if (format->num_planes > 1) {
-			unsigned int offset;
-
-			offset = crop.top * format->plane_fmt[1].bytesperline
-			       + crop.left / fmtinfo->hsub
-			       * fmtinfo->bpp[1] / 8;
-			mem.addr[1] += offset;
-			mem.addr[2] += offset;
-		}
-
-		/*
-		 * On Gen3 hardware the SPUVS bit has no effect on 3-planar
-		 * formats. Swap the U and V planes manually in that case.
-		 */
-		if (vsp1->info->gen == 3 && format->num_planes == 3 &&
-		    fmtinfo->swap_uv)
-			swap(mem.addr[1], mem.addr[2]);
-
-		vsp1_rpf_write(rpf, dl, VI6_RPF_SRCM_ADDR_Y, mem.addr[0]);
-		vsp1_rpf_write(rpf, dl, VI6_RPF_SRCM_ADDR_C0, mem.addr[1]);
-		vsp1_rpf_write(rpf, dl, VI6_RPF_SRCM_ADDR_C1, mem.addr[2]);
-		return;
-	}
-
 	/* Stride */
 	pstride = format->plane_fmt[0].bytesperline
 		<< VI6_RPF_SRCM_PSTRIDE_Y_SHIFT;
@@ -142,7 +64,7 @@ static void rpf_configure(struct vsp1_entity *entity,
 		pstride |= format->plane_fmt[1].bytesperline
 			<< VI6_RPF_SRCM_PSTRIDE_C_SHIFT;
 
-	vsp1_rpf_write(rpf, dl, VI6_RPF_SRCM_PSTRIDE, pstride);
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_SRCM_PSTRIDE, pstride);
 
 	/* Format */
 	sink_format = vsp1_entity_get_pad_format(&rpf->entity,
@@ -163,22 +85,22 @@ static void rpf_configure(struct vsp1_entity *entity,
 	if (sink_format->code != source_format->code)
 		infmt |= VI6_RPF_INFMT_CSC;
 
-	vsp1_rpf_write(rpf, dl, VI6_RPF_INFMT, infmt);
-	vsp1_rpf_write(rpf, dl, VI6_RPF_DSWAP, fmtinfo->swap);
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_INFMT, infmt);
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_DSWAP, fmtinfo->swap);
 
 	/* Output location */
-	if (pipe->bru) {
+	if (pipe->brx) {
 		const struct v4l2_rect *compose;
 
-		compose = vsp1_entity_get_pad_selection(pipe->bru,
-							pipe->bru->config,
-							rpf->bru_input,
+		compose = vsp1_entity_get_pad_selection(pipe->brx,
+							pipe->brx->config,
+							rpf->brx_input,
 							V4L2_SEL_TGT_COMPOSE);
 		left = compose->left;
 		top = compose->top;
 	}
 
-	vsp1_rpf_write(rpf, dl, VI6_RPF_LOC,
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_LOC,
 		       (left << VI6_RPF_LOC_HCOORD_SHIFT) |
 		       (top << VI6_RPF_LOC_VCOORD_SHIFT));
 
@@ -191,10 +113,10 @@ static void rpf_configure(struct vsp1_entity *entity,
 	 * alpha channel by a fixed global alpha value, and multiply the pixel
 	 * components to convert the input to premultiplied alpha.
 	 *
-	 * As alpha premultiplication is available in the BRU for both Gen2 and
+	 * As alpha premultiplication is available in the BRx for both Gen2 and
 	 * Gen3 we handle it there and use the Gen3 alpha multiplier for global
 	 * alpha multiplication only. This however prevents conversion to
-	 * premultiplied alpha if no BRU is present in the pipeline. If that use
+	 * premultiplied alpha if no BRx is present in the pipeline. If that use
 	 * case turns out to be useful we will revisit the implementation (for
 	 * Gen3 only).
 	 *
@@ -205,7 +127,7 @@ static void rpf_configure(struct vsp1_entity *entity,
 	 *
 	 * In all cases, disable color keying.
 	 */
-	vsp1_rpf_write(rpf, dl, VI6_RPF_ALPH_SEL, VI6_RPF_ALPH_SEL_AEXT_EXT |
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_ALPH_SEL, VI6_RPF_ALPH_SEL_AEXT_EXT |
 		       (fmtinfo->alpha ? VI6_RPF_ALPH_SEL_ASEL_PACKED
 				       : VI6_RPF_ALPH_SEL_ASEL_FIXED));
 
@@ -242,9 +164,94 @@ static void rpf_configure(struct vsp1_entity *entity,
 		rpf->mult_alpha = mult;
 	}
 
-	vsp1_rpf_write(rpf, dl, VI6_RPF_MSK_CTRL, 0);
-	vsp1_rpf_write(rpf, dl, VI6_RPF_CKEY_CTRL, 0);
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_MSK_CTRL, 0);
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_CKEY_CTRL, 0);
+
+}
+
+static void rpf_configure_frame(struct vsp1_entity *entity,
+				struct vsp1_pipeline *pipe,
+				struct vsp1_dl_list *dl,
+				struct vsp1_dl_body *dlb)
+{
+	struct vsp1_rwpf *rpf = to_rwpf(&entity->subdev);
+
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_VRTCOL_SET,
+		       rpf->alpha << VI6_RPF_VRTCOL_SET_LAYA_SHIFT);
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_MULT_ALPHA, rpf->mult_alpha |
+		       (rpf->alpha << VI6_RPF_MULT_ALPHA_RATIO_SHIFT));
+
+	vsp1_pipeline_propagate_alpha(pipe, dlb, rpf->alpha);
+}
+
+static void rpf_configure_partition(struct vsp1_entity *entity,
+				    struct vsp1_pipeline *pipe,
+				    struct vsp1_dl_list *dl,
+				    struct vsp1_dl_body *dlb)
+{
+	struct vsp1_rwpf *rpf = to_rwpf(&entity->subdev);
+	struct vsp1_rwpf_memory mem = rpf->mem;
+	struct vsp1_device *vsp1 = rpf->entity.vsp1;
+	const struct vsp1_format_info *fmtinfo = rpf->fmtinfo;
+	const struct v4l2_pix_format_mplane *format = &rpf->format;
+	struct v4l2_rect crop;
+
+	/*
+	 * Source size and crop offsets.
+	 *
+	 * The crop offsets correspond to the location of the crop
+	 * rectangle top left corner in the plane buffer. Only two
+	 * offsets are needed, as planes 2 and 3 always have identical
+	 * strides.
+	 */
+	crop = *vsp1_rwpf_get_crop(rpf, rpf->entity.config);
+
+	/*
+	 * Partition Algorithm Control
+	 *
+	 * The partition algorithm can split this frame into multiple
+	 * slices. We must scale our partition window based on the pipe
+	 * configuration to match the destination partition window.
+	 * To achieve this, we adjust our crop to provide a 'sub-crop'
+	 * matching the expected partition window. Only 'left' and
+	 * 'width' need to be adjusted.
+	 */
+	if (pipe->partitions > 1) {
+		crop.width = pipe->partition->rpf.width;
+		crop.left += pipe->partition->rpf.left;
+	}
+
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_SRC_BSIZE,
+		       (crop.width << VI6_RPF_SRC_BSIZE_BHSIZE_SHIFT) |
+		       (crop.height << VI6_RPF_SRC_BSIZE_BVSIZE_SHIFT));
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_SRC_ESIZE,
+		       (crop.width << VI6_RPF_SRC_ESIZE_EHSIZE_SHIFT) |
+		       (crop.height << VI6_RPF_SRC_ESIZE_EVSIZE_SHIFT));
+
+	mem.addr[0] += crop.top * format->plane_fmt[0].bytesperline
+		     + crop.left * fmtinfo->bpp[0] / 8;
+
+	if (format->num_planes > 1) {
+		unsigned int offset;
+
+		offset = crop.top * format->plane_fmt[1].bytesperline
+		       + crop.left / fmtinfo->hsub
+		       * fmtinfo->bpp[1] / 8;
+		mem.addr[1] += offset;
+		mem.addr[2] += offset;
+	}
+
+	/*
+	 * On Gen3 hardware the SPUVS bit has no effect on 3-planar
+	 * formats. Swap the U and V planes manually in that case.
+	 */
+	if (vsp1->info->gen == 3 && format->num_planes == 3 &&
+	    fmtinfo->swap_uv)
+		swap(mem.addr[1], mem.addr[2]);
 
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_SRCM_ADDR_Y, mem.addr[0]);
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_SRCM_ADDR_C0, mem.addr[1]);
+	vsp1_rpf_write(rpf, dlb, VI6_RPF_SRCM_ADDR_C1, mem.addr[2]);
 }
 
 static void rpf_partition(struct vsp1_entity *entity,
@@ -257,7 +264,9 @@ static void rpf_partition(struct vsp1_entity *entity,
 }
 
 static const struct vsp1_entity_operations rpf_entity_ops = {
-	.configure = rpf_configure,
+	.configure_stream = rpf_configure_stream,
+	.configure_frame = rpf_configure_frame,
+	.configure_partition = rpf_configure_partition,
 	.partition = rpf_partition,
 };
 
diff --git a/drivers/media/platform/vsp1/vsp1_rwpf.c b/drivers/media/platform/vsp1/vsp1_rwpf.c
index cfd8f1904fa6..049bdd958e56 100644
--- a/drivers/media/platform/vsp1/vsp1_rwpf.c
+++ b/drivers/media/platform/vsp1/vsp1_rwpf.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_rwpf.c  --  R-Car VSP1 Read and Write Pixel Formatters
  *
  * Copyright (C) 2013-2014 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <media/v4l2-subdev.h>
diff --git a/drivers/media/platform/vsp1/vsp1_rwpf.h b/drivers/media/platform/vsp1/vsp1_rwpf.h
index 58215a7ab631..70742ecf766f 100644
--- a/drivers/media/platform/vsp1/vsp1_rwpf.h
+++ b/drivers/media/platform/vsp1/vsp1_rwpf.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_rwpf.h  --  R-Car VSP1 Read and Write Pixel Formatters
  *
  * Copyright (C) 2013-2014 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_RWPF_H__
 #define __VSP1_RWPF_H__
@@ -27,7 +23,6 @@
 
 struct v4l2_ctrl;
 struct vsp1_dl_manager;
-struct vsp1_pipeline;
 struct vsp1_rwpf;
 struct vsp1_video;
 
@@ -39,7 +34,6 @@ struct vsp1_rwpf {
 	struct vsp1_entity entity;
 	struct v4l2_ctrl_handler ctrls;
 
-	struct vsp1_pipeline *pipe;
 	struct vsp1_video *video;
 
 	unsigned int max_width;
@@ -47,7 +41,7 @@ struct vsp1_rwpf {
 
 	struct v4l2_pix_format_mplane format;
 	const struct vsp1_format_info *fmtinfo;
-	unsigned int bru_input;
+	unsigned int brx_input;
 
 	unsigned int alpha;
 
diff --git a/drivers/media/platform/vsp1/vsp1_sru.c b/drivers/media/platform/vsp1/vsp1_sru.c
index 51e5691187c3..04e4e05af6ae 100644
--- a/drivers/media/platform/vsp1/vsp1_sru.c
+++ b/drivers/media/platform/vsp1/vsp1_sru.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_sru.c  --  R-Car VSP1 Super Resolution Unit
  *
  * Copyright (C) 2013 Renesas Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -28,10 +24,10 @@
  * Device Access
  */
 
-static inline void vsp1_sru_write(struct vsp1_sru *sru, struct vsp1_dl_list *dl,
-				  u32 reg, u32 data)
+static inline void vsp1_sru_write(struct vsp1_sru *sru,
+				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg, data);
+	vsp1_dl_body_write(dlb, reg, data);
 }
 
 /* -----------------------------------------------------------------------------
@@ -271,10 +267,9 @@ static const struct v4l2_subdev_ops sru_ops = {
  * VSP1 Entity Operations
  */
 
-static void sru_configure(struct vsp1_entity *entity,
-			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl,
-			  enum vsp1_entity_params params)
+static void sru_configure_stream(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_body *dlb)
 {
 	const struct vsp1_sru_param *param;
 	struct vsp1_sru *sru = to_sru(&entity->subdev);
@@ -282,9 +277,6 @@ static void sru_configure(struct vsp1_entity *entity,
 	struct v4l2_mbus_framefmt *output;
 	u32 ctrl0;
 
-	if (params != VSP1_ENTITY_PARAMS_INIT)
-		return;
-
 	input = vsp1_entity_get_pad_format(&sru->entity, sru->entity.config,
 					   SRU_PAD_SINK);
 	output = vsp1_entity_get_pad_format(&sru->entity, sru->entity.config,
@@ -303,9 +295,9 @@ static void sru_configure(struct vsp1_entity *entity,
 
 	ctrl0 |= param->ctrl0;
 
-	vsp1_sru_write(sru, dl, VI6_SRU_CTRL0, ctrl0);
-	vsp1_sru_write(sru, dl, VI6_SRU_CTRL1, VI6_SRU_CTRL1_PARAM5);
-	vsp1_sru_write(sru, dl, VI6_SRU_CTRL2, param->ctrl2);
+	vsp1_sru_write(sru, dlb, VI6_SRU_CTRL0, ctrl0);
+	vsp1_sru_write(sru, dlb, VI6_SRU_CTRL1, VI6_SRU_CTRL1_PARAM5);
+	vsp1_sru_write(sru, dlb, VI6_SRU_CTRL2, param->ctrl2);
 }
 
 static unsigned int sru_max_width(struct vsp1_entity *entity,
@@ -351,7 +343,7 @@ static void sru_partition(struct vsp1_entity *entity,
 }
 
 static const struct vsp1_entity_operations sru_entity_ops = {
-	.configure = sru_configure,
+	.configure_stream = sru_configure_stream,
 	.max_width = sru_max_width,
 	.partition = sru_partition,
 };
diff --git a/drivers/media/platform/vsp1/vsp1_sru.h b/drivers/media/platform/vsp1/vsp1_sru.h
index 85e241457af2..ddb00eadd1ea 100644
--- a/drivers/media/platform/vsp1/vsp1_sru.h
+++ b/drivers/media/platform/vsp1/vsp1_sru.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_sru.h  --  R-Car VSP1 Super Resolution Unit
  *
  * Copyright (C) 2013 Renesas Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_SRU_H__
 #define __VSP1_SRU_H__
diff --git a/drivers/media/platform/vsp1/vsp1_uds.c b/drivers/media/platform/vsp1/vsp1_uds.c
index 72f72a9d2152..c20c84b54936 100644
--- a/drivers/media/platform/vsp1/vsp1_uds.c
+++ b/drivers/media/platform/vsp1/vsp1_uds.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_uds.c  --  R-Car VSP1 Up and Down Scaler
  *
  * Copyright (C) 2013-2014 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -31,22 +27,22 @@
  * Device Access
  */
 
-static inline void vsp1_uds_write(struct vsp1_uds *uds, struct vsp1_dl_list *dl,
-				  u32 reg, u32 data)
+static inline void vsp1_uds_write(struct vsp1_uds *uds,
+				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg + uds->entity.index * VI6_UDS_OFFSET, data);
+	vsp1_dl_body_write(dlb, reg + uds->entity.index * VI6_UDS_OFFSET, data);
 }
 
 /* -----------------------------------------------------------------------------
  * Scaling Computation
  */
 
-void vsp1_uds_set_alpha(struct vsp1_entity *entity, struct vsp1_dl_list *dl,
+void vsp1_uds_set_alpha(struct vsp1_entity *entity, struct vsp1_dl_body *dlb,
 			unsigned int alpha)
 {
 	struct vsp1_uds *uds = to_uds(&entity->subdev);
 
-	vsp1_uds_write(uds, dl, VI6_UDS_ALPVAL,
+	vsp1_uds_write(uds, dlb, VI6_UDS_ALPVAL,
 		       alpha << VI6_UDS_ALPVAL_VAL0_SHIFT);
 }
 
@@ -259,10 +255,9 @@ static const struct v4l2_subdev_ops uds_ops = {
  * VSP1 Entity Operations
  */
 
-static void uds_configure(struct vsp1_entity *entity,
-			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl,
-			  enum vsp1_entity_params params)
+static void uds_configure_stream(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_uds *uds = to_uds(&entity->subdev);
 	const struct v4l2_mbus_framefmt *output;
@@ -276,27 +271,6 @@ static void uds_configure(struct vsp1_entity *entity,
 	output = vsp1_entity_get_pad_format(&uds->entity, uds->entity.config,
 					    UDS_PAD_SOURCE);
 
-	if (params == VSP1_ENTITY_PARAMS_PARTITION) {
-		struct vsp1_partition *partition = pipe->partition;
-
-		/* Input size clipping */
-		vsp1_uds_write(uds, dl, VI6_UDS_HSZCLIP, VI6_UDS_HSZCLIP_HCEN |
-			       (0 << VI6_UDS_HSZCLIP_HCL_OFST_SHIFT) |
-			       (partition->uds_sink.width
-					<< VI6_UDS_HSZCLIP_HCL_SIZE_SHIFT));
-
-		/* Output size clipping */
-		vsp1_uds_write(uds, dl, VI6_UDS_CLIP_SIZE,
-			       (partition->uds_source.width
-					<< VI6_UDS_CLIP_SIZE_HSIZE_SHIFT) |
-			       (output->height
-					<< VI6_UDS_CLIP_SIZE_VSIZE_SHIFT));
-		return;
-	}
-
-	if (params != VSP1_ENTITY_PARAMS_INIT)
-		return;
-
 	hscale = uds_compute_ratio(input->width, output->width);
 	vscale = uds_compute_ratio(input->height, output->height);
 
@@ -312,22 +286,48 @@ static void uds_configure(struct vsp1_entity *entity,
 	else
 		multitap = true;
 
-	vsp1_uds_write(uds, dl, VI6_UDS_CTRL,
+	vsp1_uds_write(uds, dlb, VI6_UDS_CTRL,
 		       (uds->scale_alpha ? VI6_UDS_CTRL_AON : 0) |
 		       (multitap ? VI6_UDS_CTRL_BC : 0));
 
-	vsp1_uds_write(uds, dl, VI6_UDS_PASS_BWIDTH,
+	vsp1_uds_write(uds, dlb, VI6_UDS_PASS_BWIDTH,
 		       (uds_passband_width(hscale)
 				<< VI6_UDS_PASS_BWIDTH_H_SHIFT) |
 		       (uds_passband_width(vscale)
 				<< VI6_UDS_PASS_BWIDTH_V_SHIFT));
 
 	/* Set the scaling ratios. */
-	vsp1_uds_write(uds, dl, VI6_UDS_SCALE,
+	vsp1_uds_write(uds, dlb, VI6_UDS_SCALE,
 		       (hscale << VI6_UDS_SCALE_HFRAC_SHIFT) |
 		       (vscale << VI6_UDS_SCALE_VFRAC_SHIFT));
 }
 
+static void uds_configure_partition(struct vsp1_entity *entity,
+				    struct vsp1_pipeline *pipe,
+				    struct vsp1_dl_list *dl,
+				    struct vsp1_dl_body *dlb)
+{
+	struct vsp1_uds *uds = to_uds(&entity->subdev);
+	struct vsp1_partition *partition = pipe->partition;
+	const struct v4l2_mbus_framefmt *output;
+
+	output = vsp1_entity_get_pad_format(&uds->entity, uds->entity.config,
+					    UDS_PAD_SOURCE);
+
+	/* Input size clipping */
+	vsp1_uds_write(uds, dlb, VI6_UDS_HSZCLIP, VI6_UDS_HSZCLIP_HCEN |
+		       (0 << VI6_UDS_HSZCLIP_HCL_OFST_SHIFT) |
+		       (partition->uds_sink.width
+				<< VI6_UDS_HSZCLIP_HCL_SIZE_SHIFT));
+
+	/* Output size clipping */
+	vsp1_uds_write(uds, dlb, VI6_UDS_CLIP_SIZE,
+		       (partition->uds_source.width
+				<< VI6_UDS_CLIP_SIZE_HSIZE_SHIFT) |
+		       (output->height
+				<< VI6_UDS_CLIP_SIZE_VSIZE_SHIFT));
+}
+
 static unsigned int uds_max_width(struct vsp1_entity *entity,
 				  struct vsp1_pipeline *pipe)
 {
@@ -384,7 +384,8 @@ static void uds_partition(struct vsp1_entity *entity,
 }
 
 static const struct vsp1_entity_operations uds_entity_ops = {
-	.configure = uds_configure,
+	.configure_stream = uds_configure_stream,
+	.configure_partition = uds_configure_partition,
 	.max_width = uds_max_width,
 	.partition = uds_partition,
 };
diff --git a/drivers/media/platform/vsp1/vsp1_uds.h b/drivers/media/platform/vsp1/vsp1_uds.h
index 7bf3cdcffc65..c34f95a666d2 100644
--- a/drivers/media/platform/vsp1/vsp1_uds.h
+++ b/drivers/media/platform/vsp1/vsp1_uds.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_uds.h  --  R-Car VSP1 Up and Down Scaler
  *
  * Copyright (C) 2013-2014 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_UDS_H__
 #define __VSP1_UDS_H__
@@ -35,7 +31,7 @@ static inline struct vsp1_uds *to_uds(struct v4l2_subdev *subdev)
 
 struct vsp1_uds *vsp1_uds_create(struct vsp1_device *vsp1, unsigned int index);
 
-void vsp1_uds_set_alpha(struct vsp1_entity *uds, struct vsp1_dl_list *dl,
+void vsp1_uds_set_alpha(struct vsp1_entity *uds, struct vsp1_dl_body *dlb,
 			unsigned int alpha);
 
 #endif /* __VSP1_UDS_H__ */
diff --git a/drivers/media/platform/vsp1/vsp1_uif.c b/drivers/media/platform/vsp1/vsp1_uif.c
new file mode 100644
index 000000000000..4b58d51df231
--- /dev/null
+++ b/drivers/media/platform/vsp1/vsp1_uif.c
@@ -0,0 +1,264 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * vsp1_uif.c  --  R-Car VSP1 User Logic Interface
+ *
+ * Copyright (C) 2017-2018 Laurent Pinchart
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ */
+
+#include <linux/device.h>
+#include <linux/gfp.h>
+#include <linux/sys_soc.h>
+
+#include <media/media-entity.h>
+#include <media/v4l2-subdev.h>
+
+#include "vsp1.h"
+#include "vsp1_dl.h"
+#include "vsp1_entity.h"
+#include "vsp1_uif.h"
+
+#define UIF_MIN_SIZE				4U
+#define UIF_MAX_SIZE				8190U
+
+/* -----------------------------------------------------------------------------
+ * Device Access
+ */
+
+static inline u32 vsp1_uif_read(struct vsp1_uif *uif, u32 reg)
+{
+	return vsp1_read(uif->entity.vsp1,
+			 uif->entity.index * VI6_UIF_OFFSET + reg);
+}
+
+static inline void vsp1_uif_write(struct vsp1_uif *uif,
+				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
+{
+	vsp1_dl_body_write(dlb, reg + uif->entity.index * VI6_UIF_OFFSET, data);
+}
+
+u32 vsp1_uif_get_crc(struct vsp1_uif *uif)
+{
+	return vsp1_uif_read(uif, VI6_UIF_DISCOM_DOCMCCRCR);
+}
+
+/* -----------------------------------------------------------------------------
+ * V4L2 Subdevice Pad Operations
+ */
+
+static const unsigned int uif_codes[] = {
+	MEDIA_BUS_FMT_ARGB8888_1X32,
+	MEDIA_BUS_FMT_AHSV8888_1X32,
+	MEDIA_BUS_FMT_AYUV8_1X32,
+};
+
+static int uif_enum_mbus_code(struct v4l2_subdev *subdev,
+			      struct v4l2_subdev_pad_config *cfg,
+			      struct v4l2_subdev_mbus_code_enum *code)
+{
+	return vsp1_subdev_enum_mbus_code(subdev, cfg, code, uif_codes,
+					  ARRAY_SIZE(uif_codes));
+}
+
+static int uif_enum_frame_size(struct v4l2_subdev *subdev,
+			       struct v4l2_subdev_pad_config *cfg,
+			       struct v4l2_subdev_frame_size_enum *fse)
+{
+	return vsp1_subdev_enum_frame_size(subdev, cfg, fse, UIF_MIN_SIZE,
+					   UIF_MIN_SIZE, UIF_MAX_SIZE,
+					   UIF_MAX_SIZE);
+}
+
+static int uif_set_format(struct v4l2_subdev *subdev,
+			    struct v4l2_subdev_pad_config *cfg,
+			    struct v4l2_subdev_format *fmt)
+{
+	return vsp1_subdev_set_pad_format(subdev, cfg, fmt, uif_codes,
+					  ARRAY_SIZE(uif_codes),
+					  UIF_MIN_SIZE, UIF_MIN_SIZE,
+					  UIF_MAX_SIZE, UIF_MAX_SIZE);
+}
+
+static int uif_get_selection(struct v4l2_subdev *subdev,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_selection *sel)
+{
+	struct vsp1_uif *uif = to_uif(subdev);
+	struct v4l2_subdev_pad_config *config;
+	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
+
+	if (sel->pad != UIF_PAD_SINK)
+		return -EINVAL;
+
+	mutex_lock(&uif->entity.lock);
+
+	config = vsp1_entity_get_pad_config(&uif->entity, cfg, sel->which);
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		format = vsp1_entity_get_pad_format(&uif->entity, config,
+						    UIF_PAD_SINK);
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = format->width;
+		sel->r.height = format->height;
+		break;
+
+	case V4L2_SEL_TGT_CROP:
+		sel->r = *vsp1_entity_get_pad_selection(&uif->entity, config,
+							sel->pad, sel->target);
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+done:
+	mutex_unlock(&uif->entity.lock);
+	return ret;
+}
+
+static int uif_set_selection(struct v4l2_subdev *subdev,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_selection *sel)
+{
+	struct vsp1_uif *uif = to_uif(subdev);
+	struct v4l2_subdev_pad_config *config;
+	struct v4l2_mbus_framefmt *format;
+	struct v4l2_rect *selection;
+	int ret = 0;
+
+	if (sel->pad != UIF_PAD_SINK ||
+	    sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
+
+	mutex_lock(&uif->entity.lock);
+
+	config = vsp1_entity_get_pad_config(&uif->entity, cfg, sel->which);
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	/* The crop rectangle must be inside the input frame. */
+	format = vsp1_entity_get_pad_format(&uif->entity, config, UIF_PAD_SINK);
+
+	sel->r.left = clamp_t(unsigned int, sel->r.left, 0, format->width - 1);
+	sel->r.top = clamp_t(unsigned int, sel->r.top, 0, format->height - 1);
+	sel->r.width = clamp_t(unsigned int, sel->r.width, UIF_MIN_SIZE,
+			       format->width - sel->r.left);
+	sel->r.height = clamp_t(unsigned int, sel->r.height, UIF_MIN_SIZE,
+				format->height - sel->r.top);
+
+	/* Store the crop rectangle. */
+	selection = vsp1_entity_get_pad_selection(&uif->entity, config,
+						  sel->pad, V4L2_SEL_TGT_CROP);
+	*selection = sel->r;
+
+done:
+	mutex_unlock(&uif->entity.lock);
+	return ret;
+}
+
+/* -----------------------------------------------------------------------------
+ * V4L2 Subdevice Operations
+ */
+
+static const struct v4l2_subdev_pad_ops uif_pad_ops = {
+	.init_cfg = vsp1_entity_init_cfg,
+	.enum_mbus_code = uif_enum_mbus_code,
+	.enum_frame_size = uif_enum_frame_size,
+	.get_fmt = vsp1_subdev_get_pad_format,
+	.set_fmt = uif_set_format,
+	.get_selection = uif_get_selection,
+	.set_selection = uif_set_selection,
+};
+
+static const struct v4l2_subdev_ops uif_ops = {
+	.pad    = &uif_pad_ops,
+};
+
+/* -----------------------------------------------------------------------------
+ * VSP1 Entity Operations
+ */
+
+static void uif_configure_stream(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_body *dlb)
+{
+	struct vsp1_uif *uif = to_uif(&entity->subdev);
+	const struct v4l2_rect *crop;
+	unsigned int left;
+	unsigned int width;
+
+	vsp1_uif_write(uif, dlb, VI6_UIF_DISCOM_DOCMPMR,
+		       VI6_UIF_DISCOM_DOCMPMR_SEL(9));
+
+	crop = vsp1_entity_get_pad_selection(entity, entity->config,
+					     UIF_PAD_SINK, V4L2_SEL_TGT_CROP);
+
+	left = crop->left;
+	width = crop->width;
+
+	/* On M3-W the horizontal coordinates are twice the register value. */
+	if (uif->m3w_quirk) {
+		left /= 2;
+		width /= 2;
+	}
+
+	vsp1_uif_write(uif, dlb, VI6_UIF_DISCOM_DOCMSPXR, left);
+	vsp1_uif_write(uif, dlb, VI6_UIF_DISCOM_DOCMSPYR, crop->top);
+	vsp1_uif_write(uif, dlb, VI6_UIF_DISCOM_DOCMSZXR, width);
+	vsp1_uif_write(uif, dlb, VI6_UIF_DISCOM_DOCMSZYR, crop->height);
+
+	vsp1_uif_write(uif, dlb, VI6_UIF_DISCOM_DOCMCR,
+		       VI6_UIF_DISCOM_DOCMCR_CMPR);
+}
+
+static const struct vsp1_entity_operations uif_entity_ops = {
+	.configure_stream = uif_configure_stream,
+};
+
+/* -----------------------------------------------------------------------------
+ * Initialization and Cleanup
+ */
+
+static const struct soc_device_attribute vsp1_r8a7796[] = {
+	{ .soc_id = "r8a7796" },
+	{ /* sentinel */ }
+};
+
+struct vsp1_uif *vsp1_uif_create(struct vsp1_device *vsp1, unsigned int index)
+{
+	struct vsp1_uif *uif;
+	char name[6];
+	int ret;
+
+	uif = devm_kzalloc(vsp1->dev, sizeof(*uif), GFP_KERNEL);
+	if (!uif)
+		return ERR_PTR(-ENOMEM);
+
+	if (soc_device_match(vsp1_r8a7796))
+		uif->m3w_quirk = true;
+
+	uif->entity.ops = &uif_entity_ops;
+	uif->entity.type = VSP1_ENTITY_UIF;
+	uif->entity.index = index;
+
+	/* The datasheet names the two UIF instances UIF4 and UIF5. */
+	sprintf(name, "uif.%u", index + 4);
+	ret = vsp1_entity_init(vsp1, &uif->entity, name, 2, &uif_ops,
+			       MEDIA_ENT_F_PROC_VIDEO_STATISTICS);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	return uif;
+}
diff --git a/drivers/media/platform/vsp1/vsp1_uif.h b/drivers/media/platform/vsp1/vsp1_uif.h
new file mode 100644
index 000000000000..c71ab5f6a6f8
--- /dev/null
+++ b/drivers/media/platform/vsp1/vsp1_uif.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * vsp1_uif.h  --  R-Car VSP1 User Logic Interface
+ *
+ * Copyright (C) 2017-2018 Laurent Pinchart
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ */
+#ifndef __VSP1_UIF_H__
+#define __VSP1_UIF_H__
+
+#include "vsp1_entity.h"
+
+struct vsp1_device;
+
+#define UIF_PAD_SINK				0
+#define UIF_PAD_SOURCE				1
+
+struct vsp1_uif {
+	struct vsp1_entity entity;
+	bool m3w_quirk;
+};
+
+static inline struct vsp1_uif *to_uif(struct v4l2_subdev *subdev)
+{
+	return container_of(subdev, struct vsp1_uif, entity.subdev);
+}
+
+struct vsp1_uif *vsp1_uif_create(struct vsp1_device *vsp1, unsigned int index);
+u32 vsp1_uif_get_crc(struct vsp1_uif *uif);
+
+#endif /* __VSP1_UIF_H__ */
diff --git a/drivers/media/platform/vsp1/vsp1_video.c b/drivers/media/platform/vsp1/vsp1_video.c
index c2d3b8f0f487..81d47a09d7bc 100644
--- a/drivers/media/platform/vsp1/vsp1_video.c
+++ b/drivers/media/platform/vsp1/vsp1_video.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_video.c  --  R-Car VSP1 Video Node
  *
  * Copyright (C) 2013-2015 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/list.h>
@@ -28,7 +24,7 @@
 #include <media/videobuf2-dma-contig.h>
 
 #include "vsp1.h"
-#include "vsp1_bru.h"
+#include "vsp1_brx.h"
 #include "vsp1_dl.h"
 #include "vsp1_entity.h"
 #include "vsp1_hgo.h"
@@ -324,7 +320,7 @@ static int vsp1_video_pipeline_setup_partitions(struct vsp1_pipeline *pipe)
 static struct vsp1_vb2_buffer *
 vsp1_video_complete_buffer(struct vsp1_video *video)
 {
-	struct vsp1_pipeline *pipe = video->rwpf->pipe;
+	struct vsp1_pipeline *pipe = video->rwpf->entity.pipe;
 	struct vsp1_vb2_buffer *next = NULL;
 	struct vsp1_vb2_buffer *done;
 	unsigned long flags;
@@ -382,69 +378,71 @@ static void vsp1_video_pipeline_run_partition(struct vsp1_pipeline *pipe,
 					      struct vsp1_dl_list *dl,
 					      unsigned int partition)
 {
+	struct vsp1_dl_body *dlb = vsp1_dl_list_get_body0(dl);
 	struct vsp1_entity *entity;
 
 	pipe->partition = &pipe->part_table[partition];
 
-	list_for_each_entry(entity, &pipe->entities, list_pipe) {
-		if (entity->ops->configure)
-			entity->ops->configure(entity, pipe, dl,
-					       VSP1_ENTITY_PARAMS_PARTITION);
-	}
+	list_for_each_entry(entity, &pipe->entities, list_pipe)
+		vsp1_entity_configure_partition(entity, pipe, dl, dlb);
 }
 
 static void vsp1_video_pipeline_run(struct vsp1_pipeline *pipe)
 {
 	struct vsp1_device *vsp1 = pipe->output->entity.vsp1;
 	struct vsp1_entity *entity;
+	struct vsp1_dl_body *dlb;
+	struct vsp1_dl_list *dl;
 	unsigned int partition;
 
-	if (!pipe->dl)
-		pipe->dl = vsp1_dl_list_get(pipe->output->dlm);
+	dl = vsp1_dl_list_get(pipe->output->dlm);
 
 	/*
-	 * Start with the runtime parameters as the configure operation can
-	 * compute/cache information needed when configuring partitions. This
-	 * is the case with flipping in the WPF.
+	 * If the VSP hardware isn't configured yet (which occurs either when
+	 * processing the first frame or after a system suspend/resume), add the
+	 * cached stream configuration to the display list to perform a full
+	 * initialisation.
 	 */
-	list_for_each_entry(entity, &pipe->entities, list_pipe) {
-		if (entity->ops->configure)
-			entity->ops->configure(entity, pipe, pipe->dl,
-					       VSP1_ENTITY_PARAMS_RUNTIME);
-	}
+	if (!pipe->configured)
+		vsp1_dl_list_add_body(dl, pipe->stream_config);
+
+	dlb = vsp1_dl_list_get_body0(dl);
 
-	/* Run the first partition */
-	vsp1_video_pipeline_run_partition(pipe, pipe->dl, 0);
+	list_for_each_entry(entity, &pipe->entities, list_pipe)
+		vsp1_entity_configure_frame(entity, pipe, dl, dlb);
 
-	/* Process consecutive partitions as necessary */
+	/* Run the first partition. */
+	vsp1_video_pipeline_run_partition(pipe, dl, 0);
+
+	/* Process consecutive partitions as necessary. */
 	for (partition = 1; partition < pipe->partitions; ++partition) {
-		struct vsp1_dl_list *dl;
+		struct vsp1_dl_list *dl_next;
 
-		dl = vsp1_dl_list_get(pipe->output->dlm);
+		dl_next = vsp1_dl_list_get(pipe->output->dlm);
 
 		/*
 		 * An incomplete chain will still function, but output only
 		 * the partitions that had a dl available. The frame end
 		 * interrupt will be marked on the last dl in the chain.
 		 */
-		if (!dl) {
+		if (!dl_next) {
 			dev_err(vsp1->dev, "Failed to obtain a dl list. Frame will be incomplete\n");
 			break;
 		}
 
-		vsp1_video_pipeline_run_partition(pipe, dl, partition);
-		vsp1_dl_list_add_chain(pipe->dl, dl);
+		vsp1_video_pipeline_run_partition(pipe, dl_next, partition);
+		vsp1_dl_list_add_chain(dl, dl_next);
 	}
 
 	/* Complete, and commit the head display list. */
-	vsp1_dl_list_commit(pipe->dl);
-	pipe->dl = NULL;
+	vsp1_dl_list_commit(dl, false);
+	pipe->configured = true;
 
 	vsp1_pipeline_run(pipe);
 }
 
 static void vsp1_video_pipeline_frame_end(struct vsp1_pipeline *pipe,
-					  bool completed)
+					  unsigned int completion)
 {
 	struct vsp1_device *vsp1 = pipe->output->entity.vsp1;
 	enum vsp1_pipeline_state state;
@@ -452,7 +450,7 @@ static void vsp1_video_pipeline_frame_end(struct vsp1_pipeline *pipe,
 	unsigned int i;
 
 	/* M2M Pipelines should never call here with an incomplete frame. */
-	WARN_ON_ONCE(!completed);
+	WARN_ON_ONCE(!(completion & VSP1_DL_FRAME_END_COMPLETED));
 
 	spin_lock_irqsave(&pipe->irqlock, flags);
 
@@ -488,7 +486,7 @@ static int vsp1_video_pipeline_build_branch(struct vsp1_pipeline *pipe,
 	struct media_entity_enum ent_enum;
 	struct vsp1_entity *entity;
 	struct media_pad *pad;
-	struct vsp1_bru *bru = NULL;
+	struct vsp1_brx *brx = NULL;
 	int ret;
 
 	ret = media_entity_enum_init(&ent_enum, &input->entity.vsp1->media_dev);
@@ -524,14 +522,14 @@ static int vsp1_video_pipeline_build_branch(struct vsp1_pipeline *pipe,
 		if (entity->type == VSP1_ENTITY_BRU ||
 		    entity->type == VSP1_ENTITY_BRS) {
 			/* BRU and BRS can't be chained. */
-			if (bru) {
+			if (brx) {
 				ret = -EPIPE;
 				goto out;
 			}
 
-			bru = to_bru(&entity->subdev);
-			bru->inputs[pad->index].rpf = input;
-			input->bru_input = pad->index;
+			brx = to_brx(&entity->subdev);
+			brx->inputs[pad->index].rpf = input;
+			input->brx_input = pad->index;
 		}
 
 		/* We've reached the WPF, we're done. */
@@ -553,7 +551,7 @@ static int vsp1_video_pipeline_build_branch(struct vsp1_pipeline *pipe,
 			}
 
 			pipe->uds = entity;
-			pipe->uds_input = bru ? &bru->entity : &input->entity;
+			pipe->uds_input = brx ? &brx->entity : &input->entity;
 		}
 
 		/* Follow the source link, ignoring any HGO or HGT. */
@@ -598,20 +596,19 @@ static int vsp1_video_pipeline_build(struct vsp1_pipeline *pipe,
 		subdev = media_entity_to_v4l2_subdev(entity);
 		e = to_vsp1_entity(subdev);
 		list_add_tail(&e->list_pipe, &pipe->entities);
+		e->pipe = pipe;
 
 		switch (e->type) {
 		case VSP1_ENTITY_RPF:
 			rwpf = to_rwpf(subdev);
 			pipe->inputs[rwpf->entity.index] = rwpf;
 			rwpf->video->pipe_index = ++pipe->num_inputs;
-			rwpf->pipe = pipe;
 			break;
 
 		case VSP1_ENTITY_WPF:
 			rwpf = to_rwpf(subdev);
 			pipe->output = rwpf;
 			rwpf->video->pipe_index = 0;
-			rwpf->pipe = pipe;
 			break;
 
 		case VSP1_ENTITY_LIF:
@@ -620,17 +617,15 @@ static int vsp1_video_pipeline_build(struct vsp1_pipeline *pipe,
 
 		case VSP1_ENTITY_BRU:
 		case VSP1_ENTITY_BRS:
-			pipe->bru = e;
+			pipe->brx = e;
 			break;
 
 		case VSP1_ENTITY_HGO:
 			pipe->hgo = e;
-			to_hgo(subdev)->histo.pipe = pipe;
 			break;
 
 		case VSP1_ENTITY_HGT:
 			pipe->hgt = e;
-			to_hgt(subdev)->histo.pipe = pipe;
 			break;
 
 		default:
@@ -682,7 +677,7 @@ static struct vsp1_pipeline *vsp1_video_pipeline_get(struct vsp1_video *video)
 	 * Otherwise allocate a new pipeline and initialize it, it will be freed
 	 * when the last reference is released.
 	 */
-	if (!video->rwpf->pipe) {
+	if (!video->rwpf->entity.pipe) {
 		pipe = kzalloc(sizeof(*pipe), GFP_KERNEL);
 		if (!pipe)
 			return ERR_PTR(-ENOMEM);
@@ -694,7 +689,7 @@ static struct vsp1_pipeline *vsp1_video_pipeline_get(struct vsp1_video *video)
 			return ERR_PTR(ret);
 		}
 	} else {
-		pipe = video->rwpf->pipe;
+		pipe = video->rwpf->entity.pipe;
 		kref_get(&pipe->kref);
 	}
 
@@ -777,7 +772,7 @@ static void vsp1_video_buffer_queue(struct vb2_buffer *vb)
 {
 	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
 	struct vsp1_video *video = vb2_get_drv_priv(vb->vb2_queue);
-	struct vsp1_pipeline *pipe = video->rwpf->pipe;
+	struct vsp1_pipeline *pipe = video->rwpf->entity.pipe;
 	struct vsp1_vb2_buffer *buf = to_vsp1_vb2_buffer(vbuf);
 	unsigned long flags;
 	bool empty;
@@ -812,11 +807,6 @@ static int vsp1_video_setup_pipeline(struct vsp1_pipeline *pipe)
 	if (ret < 0)
 		return ret;
 
-	/* Prepare the display list. */
-	pipe->dl = vsp1_dl_list_get(pipe->output->dlm);
-	if (!pipe->dl)
-		return -ENOMEM;
-
 	if (pipe->uds) {
 		struct vsp1_uds *uds = to_uds(&pipe->uds->subdev);
 
@@ -838,20 +828,25 @@ static int vsp1_video_setup_pipeline(struct vsp1_pipeline *pipe)
 		}
 	}
 
-	list_for_each_entry(entity, &pipe->entities, list_pipe) {
-		vsp1_entity_route_setup(entity, pipe, pipe->dl);
+	/*
+	 * Compute and cache the stream configuration into a body. The cached
+	 * body will be added to the display list by vsp1_video_pipeline_run()
+	 * whenever the pipeline needs to be fully reconfigured.
+	 */
+	pipe->stream_config = vsp1_dlm_dl_body_get(pipe->output->dlm);
+	if (!pipe->stream_config)
+		return -ENOMEM;
 
-		if (entity->ops->configure)
-			entity->ops->configure(entity, pipe, pipe->dl,
-					       VSP1_ENTITY_PARAMS_INIT);
+	list_for_each_entry(entity, &pipe->entities, list_pipe) {
+		vsp1_entity_route_setup(entity, pipe, pipe->stream_config);
+		vsp1_entity_configure_stream(entity, pipe, pipe->stream_config);
 	}
 
 	return 0;
 }
 
-static void vsp1_video_cleanup_pipeline(struct vsp1_pipeline *pipe)
+static void vsp1_video_release_buffers(struct vsp1_video *video)
 {
-	struct vsp1_video *video = pipe->output->video;
 	struct vsp1_vb2_buffer *buffer;
 	unsigned long flags;
 
@@ -861,18 +856,26 @@ static void vsp1_video_cleanup_pipeline(struct vsp1_pipeline *pipe)
 		vb2_buffer_done(&buffer->buf.vb2_buf, VB2_BUF_STATE_ERROR);
 	INIT_LIST_HEAD(&video->irqqueue);
 	spin_unlock_irqrestore(&video->irqlock, flags);
+}
+
+static void vsp1_video_cleanup_pipeline(struct vsp1_pipeline *pipe)
+{
+	lockdep_assert_held(&pipe->lock);
+
+	/* Release any cached configuration from our output video. */
+	vsp1_dl_body_put(pipe->stream_config);
+	pipe->stream_config = NULL;
+	pipe->configured = false;
 
 	/* Release our partition table allocation */
-	mutex_lock(&pipe->lock);
 	kfree(pipe->part_table);
 	pipe->part_table = NULL;
-	mutex_unlock(&pipe->lock);
 }
 
 static int vsp1_video_start_streaming(struct vb2_queue *vq, unsigned int count)
 {
 	struct vsp1_video *video = vb2_get_drv_priv(vq);
-	struct vsp1_pipeline *pipe = video->rwpf->pipe;
+	struct vsp1_pipeline *pipe = video->rwpf->entity.pipe;
 	bool start_pipeline = false;
 	unsigned long flags;
 	int ret;
@@ -881,8 +884,9 @@ static int vsp1_video_start_streaming(struct vb2_queue *vq, unsigned int count)
 	if (pipe->stream_count == pipe->num_inputs) {
 		ret = vsp1_video_setup_pipeline(pipe);
 		if (ret < 0) {
-			mutex_unlock(&pipe->lock);
+			vsp1_video_release_buffers(video);
 			vsp1_video_cleanup_pipeline(pipe);
+			mutex_unlock(&pipe->lock);
 			return ret;
 		}
 
@@ -913,7 +917,7 @@ static int vsp1_video_start_streaming(struct vb2_queue *vq, unsigned int count)
 static void vsp1_video_stop_streaming(struct vb2_queue *vq)
 {
 	struct vsp1_video *video = vb2_get_drv_priv(vq);
-	struct vsp1_pipeline *pipe = video->rwpf->pipe;
+	struct vsp1_pipeline *pipe = video->rwpf->entity.pipe;
 	unsigned long flags;
 	int ret;
 
@@ -932,13 +936,12 @@ static void vsp1_video_stop_streaming(struct vb2_queue *vq)
 		if (ret == -ETIMEDOUT)
 			dev_err(video->vsp1->dev, "pipeline stop timeout\n");
 
-		vsp1_dl_list_put(pipe->dl);
-		pipe->dl = NULL;
+		vsp1_video_cleanup_pipeline(pipe);
 	}
 	mutex_unlock(&pipe->lock);
 
 	media_pipeline_stop(&video->video.entity);
-	vsp1_video_cleanup_pipeline(pipe);
+	vsp1_video_release_buffers(video);
 	vsp1_video_pipeline_put(pipe);
 }
 
@@ -1172,6 +1175,87 @@ static const struct v4l2_file_operations vsp1_video_fops = {
 	.mmap = vb2_fop_mmap,
 };
 
+/* -----------------------------------------------------------------------------
+ * Suspend and Resume
+ */
+
+void vsp1_video_suspend(struct vsp1_device *vsp1)
+{
+	unsigned long flags;
+	unsigned int i;
+	int ret;
+
+	/*
+	 * To avoid increasing the system suspend time needlessly, loop over the
+	 * pipelines twice, first to set them all to the stopping state, and
+	 * then to wait for the stop to complete.
+	 */
+	for (i = 0; i < vsp1->info->wpf_count; ++i) {
+		struct vsp1_rwpf *wpf = vsp1->wpf[i];
+		struct vsp1_pipeline *pipe;
+
+		if (wpf == NULL)
+			continue;
+
+		pipe = wpf->entity.pipe;
+		if (pipe == NULL)
+			continue;
+
+		spin_lock_irqsave(&pipe->irqlock, flags);
+		if (pipe->state == VSP1_PIPELINE_RUNNING)
+			pipe->state = VSP1_PIPELINE_STOPPING;
+		spin_unlock_irqrestore(&pipe->irqlock, flags);
+	}
+
+	for (i = 0; i < vsp1->info->wpf_count; ++i) {
+		struct vsp1_rwpf *wpf = vsp1->wpf[i];
+		struct vsp1_pipeline *pipe;
+
+		if (wpf == NULL)
+			continue;
+
+		pipe = wpf->entity.pipe;
+		if (pipe == NULL)
+			continue;
+
+		ret = wait_event_timeout(pipe->wq, vsp1_pipeline_stopped(pipe),
+					 msecs_to_jiffies(500));
+		if (ret == 0)
+			dev_warn(vsp1->dev, "pipeline %u stop timeout\n",
+				 wpf->entity.index);
+	}
+}
+
+void vsp1_video_resume(struct vsp1_device *vsp1)
+{
+	unsigned long flags;
+	unsigned int i;
+
+	/* Resume all running pipelines. */
+	for (i = 0; i < vsp1->info->wpf_count; ++i) {
+		struct vsp1_rwpf *wpf = vsp1->wpf[i];
+		struct vsp1_pipeline *pipe;
+
+		if (wpf == NULL)
+			continue;
+
+		pipe = wpf->entity.pipe;
+		if (pipe == NULL)
+			continue;
+
+		/*
+		 * The hardware may have been reset during a suspend and will
+		 * need a full reconfiguration.
+		 */
+		pipe->configured = false;
+
+		spin_lock_irqsave(&pipe->irqlock, flags);
+		if (vsp1_pipeline_ready(pipe))
+			vsp1_video_pipeline_run(pipe);
+		spin_unlock_irqrestore(&pipe->irqlock, flags);
+	}
+}
+
 /* -----------------------------------------------------------------------------
  * Initialization and Cleanup
  */
diff --git a/drivers/media/platform/vsp1/vsp1_video.h b/drivers/media/platform/vsp1/vsp1_video.h
index 50ea7f02205f..f3cf5e2fdf5a 100644
--- a/drivers/media/platform/vsp1/vsp1_video.h
+++ b/drivers/media/platform/vsp1/vsp1_video.h
@@ -1,14 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * vsp1_video.h  --  R-Car VSP1 Video Node
  *
  * Copyright (C) 2013-2015 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #ifndef __VSP1_VIDEO_H__
 #define __VSP1_VIDEO_H__
@@ -55,6 +51,9 @@ static inline struct vsp1_video *to_vsp1_video(struct video_device *vdev)
 	return container_of(vdev, struct vsp1_video, video);
 }
 
+void vsp1_video_suspend(struct vsp1_device *vsp1);
+void vsp1_video_resume(struct vsp1_device *vsp1);
+
 struct vsp1_video *vsp1_video_create(struct vsp1_device *vsp1,
 				     struct vsp1_rwpf *rwpf);
 void vsp1_video_cleanup(struct vsp1_video *video);
diff --git a/drivers/media/platform/vsp1/vsp1_wpf.c b/drivers/media/platform/vsp1/vsp1_wpf.c
index 8bd6b2f1af15..23c8f706b3f2 100644
--- a/drivers/media/platform/vsp1/vsp1_wpf.c
+++ b/drivers/media/platform/vsp1/vsp1_wpf.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * vsp1_wpf.c  --  R-Car VSP1 Write Pixel Formatter
  *
  * Copyright (C) 2013-2014 Renesas Electronics Corporation
  *
  * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/device.h>
@@ -31,9 +27,9 @@
  */
 
 static inline void vsp1_wpf_write(struct vsp1_rwpf *wpf,
-				  struct vsp1_dl_list *dl, u32 reg, u32 data)
+				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg + wpf->entity.index * VI6_WPF_OFFSET, data);
+	vsp1_dl_body_write(dlb, reg + wpf->entity.index * VI6_WPF_OFFSET, data);
 }
 
 /* -----------------------------------------------------------------------------
@@ -236,10 +232,9 @@ static void vsp1_wpf_destroy(struct vsp1_entity *entity)
 	vsp1_dlm_destroy(wpf->dlm);
 }
 
-static void wpf_configure(struct vsp1_entity *entity,
-			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl,
-			  enum vsp1_entity_params params)
+static void wpf_configure_stream(struct vsp1_entity *entity,
+				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_rwpf *wpf = to_rwpf(&entity->subdev);
 	struct vsp1_device *vsp1 = wpf->entity.vsp1;
@@ -249,149 +244,12 @@ static void wpf_configure(struct vsp1_entity *entity,
 	u32 outfmt = 0;
 	u32 srcrpf = 0;
 
-	if (params == VSP1_ENTITY_PARAMS_RUNTIME) {
-		const unsigned int mask = BIT(WPF_CTRL_VFLIP)
-					| BIT(WPF_CTRL_HFLIP);
-		unsigned long flags;
-
-		spin_lock_irqsave(&wpf->flip.lock, flags);
-		wpf->flip.active = (wpf->flip.active & ~mask)
-				 | (wpf->flip.pending & mask);
-		spin_unlock_irqrestore(&wpf->flip.lock, flags);
-
-		outfmt = (wpf->alpha << VI6_WPF_OUTFMT_PDV_SHIFT) | wpf->outfmt;
-
-		if (wpf->flip.active & BIT(WPF_CTRL_VFLIP))
-			outfmt |= VI6_WPF_OUTFMT_FLP;
-		if (wpf->flip.active & BIT(WPF_CTRL_HFLIP))
-			outfmt |= VI6_WPF_OUTFMT_HFLP;
-
-		vsp1_wpf_write(wpf, dl, VI6_WPF_OUTFMT, outfmt);
-		return;
-	}
-
 	sink_format = vsp1_entity_get_pad_format(&wpf->entity,
 						 wpf->entity.config,
 						 RWPF_PAD_SINK);
 	source_format = vsp1_entity_get_pad_format(&wpf->entity,
 						   wpf->entity.config,
 						   RWPF_PAD_SOURCE);
-
-	if (params == VSP1_ENTITY_PARAMS_PARTITION) {
-		const struct v4l2_pix_format_mplane *format = &wpf->format;
-		const struct vsp1_format_info *fmtinfo = wpf->fmtinfo;
-		struct vsp1_rwpf_memory mem = wpf->mem;
-		unsigned int flip = wpf->flip.active;
-		unsigned int width = sink_format->width;
-		unsigned int height = sink_format->height;
-		unsigned int offset;
-
-		/*
-		 * Cropping. The partition algorithm can split the image into
-		 * multiple slices.
-		 */
-		if (pipe->partitions > 1)
-			width = pipe->partition->wpf.width;
-
-		vsp1_wpf_write(wpf, dl, VI6_WPF_HSZCLIP, VI6_WPF_SZCLIP_EN |
-			       (0 << VI6_WPF_SZCLIP_OFST_SHIFT) |
-			       (width << VI6_WPF_SZCLIP_SIZE_SHIFT));
-		vsp1_wpf_write(wpf, dl, VI6_WPF_VSZCLIP, VI6_WPF_SZCLIP_EN |
-			       (0 << VI6_WPF_SZCLIP_OFST_SHIFT) |
-			       (height << VI6_WPF_SZCLIP_SIZE_SHIFT));
-
-		if (pipe->lif)
-			return;
-
-		/*
-		 * Update the memory offsets based on flipping configuration.
-		 * The destination addresses point to the locations where the
-		 * VSP starts writing to memory, which can be any corner of the
-		 * image depending on the combination of flipping and rotation.
-		 */
-
-		/*
-		 * First take the partition left coordinate into account.
-		 * Compute the offset to order the partitions correctly on the
-		 * output based on whether flipping is enabled. Consider
-		 * horizontal flipping when rotation is disabled but vertical
-		 * flipping when rotation is enabled, as rotating the image
-		 * switches the horizontal and vertical directions. The offset
-		 * is applied horizontally or vertically accordingly.
-		 */
-		if (flip & BIT(WPF_CTRL_HFLIP) && !wpf->flip.rotate)
-			offset = format->width - pipe->partition->wpf.left
-				- pipe->partition->wpf.width;
-		else if (flip & BIT(WPF_CTRL_VFLIP) && wpf->flip.rotate)
-			offset = format->height - pipe->partition->wpf.left
-				- pipe->partition->wpf.width;
-		else
-			offset = pipe->partition->wpf.left;
-
-		for (i = 0; i < format->num_planes; ++i) {
-			unsigned int hsub = i > 0 ? fmtinfo->hsub : 1;
-			unsigned int vsub = i > 0 ? fmtinfo->vsub : 1;
-
-			if (wpf->flip.rotate)
-				mem.addr[i] += offset / vsub
-					     * format->plane_fmt[i].bytesperline;
-			else
-				mem.addr[i] += offset / hsub
-					     * fmtinfo->bpp[i] / 8;
-		}
-
-		if (flip & BIT(WPF_CTRL_VFLIP)) {
-			/*
-			 * When rotating the output (after rotation) image
-			 * height is equal to the partition width (before
-			 * rotation). Otherwise it is equal to the output
-			 * image height.
-			 */
-			if (wpf->flip.rotate)
-				height = pipe->partition->wpf.width;
-			else
-				height = format->height;
-
-			mem.addr[0] += (height - 1)
-				     * format->plane_fmt[0].bytesperline;
-
-			if (format->num_planes > 1) {
-				offset = (height / fmtinfo->vsub - 1)
-				       * format->plane_fmt[1].bytesperline;
-				mem.addr[1] += offset;
-				mem.addr[2] += offset;
-			}
-		}
-
-		if (wpf->flip.rotate && !(flip & BIT(WPF_CTRL_HFLIP))) {
-			unsigned int hoffset = max(0, (int)format->width - 16);
-
-			/*
-			 * Compute the output coordinate. The partition
-			 * horizontal (left) offset becomes a vertical offset.
-			 */
-			for (i = 0; i < format->num_planes; ++i) {
-				unsigned int hsub = i > 0 ? fmtinfo->hsub : 1;
-
-				mem.addr[i] += hoffset / hsub
-					     * fmtinfo->bpp[i] / 8;
-			}
-		}
-
-		/*
-		 * On Gen3 hardware the SPUVS bit has no effect on 3-planar
-		 * formats. Swap the U and V planes manually in that case.
-		 */
-		if (vsp1->info->gen == 3 && format->num_planes == 3 &&
-		    fmtinfo->swap_uv)
-			swap(mem.addr[1], mem.addr[2]);
-
-		vsp1_wpf_write(wpf, dl, VI6_WPF_DSTM_ADDR_Y, mem.addr[0]);
-		vsp1_wpf_write(wpf, dl, VI6_WPF_DSTM_ADDR_C0, mem.addr[1]);
-		vsp1_wpf_write(wpf, dl, VI6_WPF_DSTM_ADDR_C1, mem.addr[2]);
-		return;
-	}
-
 	/* Format */
 	if (!pipe->lif) {
 		const struct v4l2_pix_format_mplane *format = &wpf->format;
@@ -410,17 +268,17 @@ static void wpf_configure(struct vsp1_entity *entity,
 			outfmt |= VI6_WPF_OUTFMT_SPUVS;
 
 		/* Destination stride and byte swapping. */
-		vsp1_wpf_write(wpf, dl, VI6_WPF_DSTM_STRIDE_Y,
+		vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_STRIDE_Y,
 			       format->plane_fmt[0].bytesperline);
 		if (format->num_planes > 1)
-			vsp1_wpf_write(wpf, dl, VI6_WPF_DSTM_STRIDE_C,
+			vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_STRIDE_C,
 				       format->plane_fmt[1].bytesperline);
 
-		vsp1_wpf_write(wpf, dl, VI6_WPF_DSWAP, fmtinfo->swap);
+		vsp1_wpf_write(wpf, dlb, VI6_WPF_DSWAP, fmtinfo->swap);
 
 		if (vsp1->info->features & VSP1_HAS_WPF_HFLIP &&
 		    wpf->entity.index == 0)
-			vsp1_wpf_write(wpf, dl, VI6_WPF_ROT_CTRL,
+			vsp1_wpf_write(wpf, dlb, VI6_WPF_ROT_CTRL,
 				       VI6_WPF_ROT_CTRL_LN16 |
 				       (256 << VI6_WPF_ROT_CTRL_LMEM_WD_SHIFT));
 	}
@@ -430,13 +288,13 @@ static void wpf_configure(struct vsp1_entity *entity,
 
 	wpf->outfmt = outfmt;
 
-	vsp1_dl_list_write(dl, VI6_DPR_WPF_FPORCH(wpf->entity.index),
+	vsp1_dl_body_write(dlb, VI6_DPR_WPF_FPORCH(wpf->entity.index),
 			   VI6_DPR_WPF_FPORCH_FP_WPFN);
 
-	vsp1_dl_list_write(dl, VI6_WPF_WRBCK_CTRL, 0);
+	vsp1_dl_body_write(dlb, VI6_WPF_WRBCK_CTRL, 0);
 
 	/*
-	 * Sources. If the pipeline has a single input and BRU is not used,
+	 * Sources. If the pipeline has a single input and BRx is not used,
 	 * configure it as the master layer. Otherwise configure all
 	 * inputs as sub-layers and select the virtual RPF as the master
 	 * layer.
@@ -447,24 +305,180 @@ static void wpf_configure(struct vsp1_entity *entity,
 		if (!input)
 			continue;
 
-		srcrpf |= (!pipe->bru && pipe->num_inputs == 1)
+		srcrpf |= (!pipe->brx && pipe->num_inputs == 1)
 			? VI6_WPF_SRCRPF_RPF_ACT_MST(input->entity.index)
 			: VI6_WPF_SRCRPF_RPF_ACT_SUB(input->entity.index);
 	}
 
-	if (pipe->bru)
-		srcrpf |= pipe->bru->type == VSP1_ENTITY_BRU
+	if (pipe->brx)
+		srcrpf |= pipe->brx->type == VSP1_ENTITY_BRU
 			? VI6_WPF_SRCRPF_VIRACT_MST
 			: VI6_WPF_SRCRPF_VIRACT2_MST;
 
-	vsp1_wpf_write(wpf, dl, VI6_WPF_SRCRPF, srcrpf);
+	vsp1_wpf_write(wpf, dlb, VI6_WPF_SRCRPF, srcrpf);
 
 	/* Enable interrupts */
-	vsp1_dl_list_write(dl, VI6_WPF_IRQ_STA(wpf->entity.index), 0);
-	vsp1_dl_list_write(dl, VI6_WPF_IRQ_ENB(wpf->entity.index),
+	vsp1_dl_body_write(dlb, VI6_WPF_IRQ_STA(wpf->entity.index), 0);
+	vsp1_dl_body_write(dlb, VI6_WPF_IRQ_ENB(wpf->entity.index),
 			   VI6_WFP_IRQ_ENB_DFEE);
 }
 
+static void wpf_configure_frame(struct vsp1_entity *entity,
+				struct vsp1_pipeline *pipe,
+				struct vsp1_dl_list *dl,
+				struct vsp1_dl_body *dlb)
+{
+	const unsigned int mask = BIT(WPF_CTRL_VFLIP)
+				| BIT(WPF_CTRL_HFLIP);
+	struct vsp1_rwpf *wpf = to_rwpf(&entity->subdev);
+	unsigned long flags;
+	u32 outfmt;
+
+	spin_lock_irqsave(&wpf->flip.lock, flags);
+	wpf->flip.active = (wpf->flip.active & ~mask)
+			 | (wpf->flip.pending & mask);
+	spin_unlock_irqrestore(&wpf->flip.lock, flags);
+
+	outfmt = (wpf->alpha << VI6_WPF_OUTFMT_PDV_SHIFT) | wpf->outfmt;
+
+	if (wpf->flip.active & BIT(WPF_CTRL_VFLIP))
+		outfmt |= VI6_WPF_OUTFMT_FLP;
+	if (wpf->flip.active & BIT(WPF_CTRL_HFLIP))
+		outfmt |= VI6_WPF_OUTFMT_HFLP;
+
+	vsp1_wpf_write(wpf, dlb, VI6_WPF_OUTFMT, outfmt);
+}
+
+static void wpf_configure_partition(struct vsp1_entity *entity,
+				    struct vsp1_pipeline *pipe,
+				    struct vsp1_dl_list *dl,
+				    struct vsp1_dl_body *dlb)
+{
+	struct vsp1_rwpf *wpf = to_rwpf(&entity->subdev);
+	struct vsp1_device *vsp1 = wpf->entity.vsp1;
+	struct vsp1_rwpf_memory mem = wpf->mem;
+	const struct v4l2_mbus_framefmt *sink_format;
+	const struct v4l2_pix_format_mplane *format = &wpf->format;
+	const struct vsp1_format_info *fmtinfo = wpf->fmtinfo;
+	unsigned int width;
+	unsigned int height;
+	unsigned int offset;
+	unsigned int flip;
+	unsigned int i;
+
+	sink_format = vsp1_entity_get_pad_format(&wpf->entity,
+						 wpf->entity.config,
+						 RWPF_PAD_SINK);
+	width = sink_format->width;
+	height = sink_format->height;
+
+	/*
+	 * Cropping. The partition algorithm can split the image into
+	 * multiple slices.
+	 */
+	if (pipe->partitions > 1)
+		width = pipe->partition->wpf.width;
+
+	vsp1_wpf_write(wpf, dlb, VI6_WPF_HSZCLIP, VI6_WPF_SZCLIP_EN |
+		       (0 << VI6_WPF_SZCLIP_OFST_SHIFT) |
+		       (width << VI6_WPF_SZCLIP_SIZE_SHIFT));
+	vsp1_wpf_write(wpf, dlb, VI6_WPF_VSZCLIP, VI6_WPF_SZCLIP_EN |
+		       (0 << VI6_WPF_SZCLIP_OFST_SHIFT) |
+		       (height << VI6_WPF_SZCLIP_SIZE_SHIFT));
+
+	if (pipe->lif)
+		return;
+
+	/*
+	 * Update the memory offsets based on flipping configuration.
+	 * The destination addresses point to the locations where the
+	 * VSP starts writing to memory, which can be any corner of the
+	 * image depending on the combination of flipping and rotation.
+	 */
+
+	/*
+	 * First take the partition left coordinate into account.
+	 * Compute the offset to order the partitions correctly on the
+	 * output based on whether flipping is enabled. Consider
+	 * horizontal flipping when rotation is disabled but vertical
+	 * flipping when rotation is enabled, as rotating the image
+	 * switches the horizontal and vertical directions. The offset
+	 * is applied horizontally or vertically accordingly.
+	 */
+	flip = wpf->flip.active;
+
+	if (flip & BIT(WPF_CTRL_HFLIP) && !wpf->flip.rotate)
+		offset = format->width - pipe->partition->wpf.left
+			- pipe->partition->wpf.width;
+	else if (flip & BIT(WPF_CTRL_VFLIP) && wpf->flip.rotate)
+		offset = format->height - pipe->partition->wpf.left
+			- pipe->partition->wpf.width;
+	else
+		offset = pipe->partition->wpf.left;
+
+	for (i = 0; i < format->num_planes; ++i) {
+		unsigned int hsub = i > 0 ? fmtinfo->hsub : 1;
+		unsigned int vsub = i > 0 ? fmtinfo->vsub : 1;
+
+		if (wpf->flip.rotate)
+			mem.addr[i] += offset / vsub
+				     * format->plane_fmt[i].bytesperline;
+		else
+			mem.addr[i] += offset / hsub
+				     * fmtinfo->bpp[i] / 8;
+	}
+
+	if (flip & BIT(WPF_CTRL_VFLIP)) {
+		/*
+		 * When rotating the output (after rotation) image
+		 * height is equal to the partition width (before
+		 * rotation). Otherwise it is equal to the output
+		 * image height.
+		 */
+		if (wpf->flip.rotate)
+			height = pipe->partition->wpf.width;
+		else
+			height = format->height;
+
+		mem.addr[0] += (height - 1)
+			     * format->plane_fmt[0].bytesperline;
+
+		if (format->num_planes > 1) {
+			offset = (height / fmtinfo->vsub - 1)
+			       * format->plane_fmt[1].bytesperline;
+			mem.addr[1] += offset;
+			mem.addr[2] += offset;
+		}
+	}
+
+	if (wpf->flip.rotate && !(flip & BIT(WPF_CTRL_HFLIP))) {
+		unsigned int hoffset = max(0, (int)format->width - 16);
+
+		/*
+		 * Compute the output coordinate. The partition
+		 * horizontal (left) offset becomes a vertical offset.
+		 */
+		for (i = 0; i < format->num_planes; ++i) {
+			unsigned int hsub = i > 0 ? fmtinfo->hsub : 1;
+
+			mem.addr[i] += hoffset / hsub
+				     * fmtinfo->bpp[i] / 8;
+		}
+	}
+
+	/*
+	 * On Gen3 hardware the SPUVS bit has no effect on 3-planar
+	 * formats. Swap the U and V planes manually in that case.
+	 */
+	if (vsp1->info->gen == 3 && format->num_planes == 3 &&
+	    fmtinfo->swap_uv)
+		swap(mem.addr[1], mem.addr[2]);
+
+	vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_ADDR_Y, mem.addr[0]);
+	vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_ADDR_C0, mem.addr[1]);
+	vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_ADDR_C1, mem.addr[2]);
+}
+
 static unsigned int wpf_max_width(struct vsp1_entity *entity,
 				  struct vsp1_pipeline *pipe)
 {
@@ -484,7 +498,9 @@ static void wpf_partition(struct vsp1_entity *entity,
 
 static const struct vsp1_entity_operations wpf_entity_ops = {
 	.destroy = vsp1_wpf_destroy,
-	.configure = wpf_configure,
+	.configure_stream = wpf_configure_stream,
+	.configure_frame = wpf_configure_frame,
+	.configure_partition = wpf_configure_partition,
 	.max_width = wpf_max_width,
 	.partition = wpf_partition,
 };
diff --git a/drivers/media/platform/xilinx/xilinx-dma.c b/drivers/media/platform/xilinx/xilinx-dma.c
index 522cdfdd3345..d041f94be832 100644
--- a/drivers/media/platform/xilinx/xilinx-dma.c
+++ b/drivers/media/platform/xilinx/xilinx-dma.c
@@ -494,13 +494,15 @@ xvip_dma_querycap(struct file *file, void *fh, struct v4l2_capability *cap)
 	struct v4l2_fh *vfh = file->private_data;
 	struct xvip_dma *dma = to_xvip_dma(vfh->vdev);
 
-	cap->capabilities = V4L2_CAP_DEVICE_CAPS | V4L2_CAP_STREAMING
-			  | dma->xdev->v4l2_caps;
+	cap->device_caps = V4L2_CAP_STREAMING;
 
 	if (dma->queue.type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		cap->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING;
+		cap->device_caps |= V4L2_CAP_VIDEO_CAPTURE;
 	else
-		cap->device_caps = V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_STREAMING;
+		cap->device_caps |= V4L2_CAP_VIDEO_OUTPUT;
+
+	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS
+			  | dma->xdev->v4l2_caps;
 
 	strlcpy(cap->driver, "xilinx-vipp", sizeof(cap->driver));
 	strlcpy(cap->card, dma->video.name, sizeof(cap->card));
diff --git a/drivers/media/radio/Kconfig b/drivers/media/radio/Kconfig
index 192f36f2f4aa..8fa403c7149e 100644
--- a/drivers/media/radio/Kconfig
+++ b/drivers/media/radio/Kconfig
@@ -15,10 +15,6 @@ if RADIO_ADAPTERS && VIDEO_V4L2
 config RADIO_TEA575X
 	tristate
 
-config RADIO_SI470X
-	bool "Silicon Labs Si470x FM Radio Receiver support"
-	depends on VIDEO_V4L2
-
 source "drivers/media/radio/si470x/Kconfig"
 
 config RADIO_SI4713
@@ -235,7 +231,7 @@ source "drivers/media/radio/wl128x/Kconfig"
 
 menuconfig V4L_RADIO_ISA_DRIVERS
 	bool "ISA radio devices"
-	depends on ISA
+	depends on ISA || COMPILE_TEST
 	default n
 	---help---
 	  Say Y here to enable support for these ISA drivers.
@@ -243,12 +239,13 @@ menuconfig V4L_RADIO_ISA_DRIVERS
 if V4L_RADIO_ISA_DRIVERS
 
 config RADIO_ISA
-	depends on ISA
+	depends on ISA || COMPILE_TEST
 	tristate
 
 config RADIO_CADET
 	tristate "ADS Cadet AM/FM Tuner"
-	depends on ISA && VIDEO_V4L2
+	depends on ISA || COMPILE_TEST
+	depends on VIDEO_V4L2
 	---help---
 	  Choose Y here if you have one of these AM/FM radio cards, and then
 	  fill in the port address below.
@@ -258,8 +255,8 @@ config RADIO_CADET
 
 config RADIO_RTRACK
 	tristate "AIMSlab RadioTrack (aka RadioReveal) support"
-	depends on ISA && VIDEO_V4L2
-	select RADIO_ISA
+	depends on ISA || COMPILE_TEST
+	depends on VIDEO_V4L2
 	---help---
 	  Choose Y here if you have one of these FM radio cards, and then fill
 	  in the port address below.
@@ -289,7 +286,8 @@ config RADIO_RTRACK_PORT
 
 config RADIO_RTRACK2
 	tristate "AIMSlab RadioTrack II support"
-	depends on ISA && VIDEO_V4L2
+	depends on ISA || COMPILE_TEST
+	depends on VIDEO_V4L2
 	select RADIO_ISA
 	---help---
 	  Choose Y here if you have this FM radio card, and then fill in the
@@ -312,7 +310,8 @@ config RADIO_RTRACK2_PORT
 
 config RADIO_AZTECH
 	tristate "Aztech/Packard Bell Radio"
-	depends on ISA && VIDEO_V4L2
+	depends on ISA || COMPILE_TEST
+	depends on VIDEO_V4L2
 	select RADIO_ISA
 	---help---
 	  Choose Y here if you have one of these FM radio cards, and then fill
@@ -332,7 +331,8 @@ config RADIO_AZTECH_PORT
 
 config RADIO_GEMTEK
 	tristate "GemTek Radio card (or compatible) support"
-	depends on ISA && VIDEO_V4L2
+	depends on ISA || COMPILE_TEST
+	depends on VIDEO_V4L2
 	select RADIO_ISA
 	---help---
 	  Choose Y here if you have this FM radio card, and then fill in the
@@ -372,7 +372,8 @@ config RADIO_GEMTEK_PROBE
 
 config RADIO_MIROPCM20
 	tristate "miroSOUND PCM20 radio"
-	depends on ISA && ISA_DMA_API && VIDEO_V4L2 && SND
+	depends on ISA || COMPILE_TEST
+	depends on ISA_DMA_API && VIDEO_V4L2 && SND
 	select SND_ISA
 	select SND_MIRO
 	---help---
@@ -386,7 +387,8 @@ config RADIO_MIROPCM20
 
 config RADIO_SF16FMI
 	tristate "SF16-FMI/SF16-FMP/SF16-FMD Radio"
-	depends on ISA && VIDEO_V4L2
+	depends on ISA || COMPILE_TEST
+	depends on VIDEO_V4L2
 	---help---
 	  Choose Y here if you have one of these FM radio cards.
 
@@ -395,7 +397,8 @@ config RADIO_SF16FMI
 
 config RADIO_SF16FMR2
 	tristate "SF16-FMR2/SF16-FMD2 Radio"
-	depends on ISA && VIDEO_V4L2
+	depends on ISA || COMPILE_TEST
+	depends on VIDEO_V4L2
 	select RADIO_TEA575X
 	---help---
 	  Choose Y here if you have one of these FM radio cards.
@@ -405,7 +408,8 @@ config RADIO_SF16FMR2
 
 config RADIO_TERRATEC
 	tristate "TerraTec ActiveRadio ISA Standalone"
-	depends on ISA && VIDEO_V4L2
+	depends on ISA || COMPILE_TEST
+	depends on VIDEO_V4L2
 	select RADIO_ISA
 	---help---
 	  Choose Y here if you have this FM radio card.
@@ -419,7 +423,8 @@ config RADIO_TERRATEC
 
 config RADIO_TRUST
 	tristate "Trust FM radio card"
-	depends on ISA && VIDEO_V4L2
+	depends on ISA || COMPILE_TEST
+	depends on VIDEO_V4L2
 	select RADIO_ISA
 	help
 	  This is a driver for the Trust FM radio cards. Say Y if you have
@@ -442,7 +447,8 @@ config RADIO_TRUST_PORT
 
 config RADIO_TYPHOON
 	tristate "Typhoon Radio (a.k.a. EcoRadio)"
-	depends on ISA && VIDEO_V4L2
+	depends on ISA || COMPILE_TEST
+	depends on VIDEO_V4L2
 	select RADIO_ISA
 	---help---
 	  Choose Y here if you have one of these FM radio cards, and then fill
@@ -476,7 +482,8 @@ config RADIO_TYPHOON_MUTEFREQ
 
 config RADIO_ZOLTRIX
 	tristate "Zoltrix Radio"
-	depends on ISA && VIDEO_V4L2
+	depends on ISA || COMPILE_TEST
+	depends on VIDEO_V4L2
 	select RADIO_ISA
 	---help---
 	  Choose Y here if you have one of these FM radio cards, and then fill
diff --git a/drivers/media/radio/si470x/Kconfig b/drivers/media/radio/si470x/Kconfig
index a466654ee5c9..a21172e413a9 100644
--- a/drivers/media/radio/si470x/Kconfig
+++ b/drivers/media/radio/si470x/Kconfig
@@ -1,3 +1,17 @@
+config RADIO_SI470X
+        tristate "Silicon Labs Si470x FM Radio Receiver support"
+        depends on VIDEO_V4L2
+	---help---
+	  This is a driver for devices with the Silicon Labs SI470x
+	  chip (either via USB or I2C buses).
+
+	  Say Y here if you want to connect this type of radio to your
+	  computer's USB port or if it is used by some other driver
+	  via I2C bus.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called radio-si470x-common.
+
 config USB_SI470X
 	tristate "Silicon Labs Si470x FM Radio Receiver support with USB"
 	depends on USB && RADIO_SI470X
@@ -25,7 +39,7 @@ config USB_SI470X
 
 config I2C_SI470X
 	tristate "Silicon Labs Si470x FM Radio Receiver support with I2C"
-	depends on I2C && RADIO_SI470X && !USB_SI470X
+	depends on I2C && RADIO_SI470X
 	---help---
 	  This is a driver for I2C devices with the Silicon Labs SI470x
 	  chip.
diff --git a/drivers/media/radio/si470x/Makefile b/drivers/media/radio/si470x/Makefile
index 06964816cfd6..682b3146397e 100644
--- a/drivers/media/radio/si470x/Makefile
+++ b/drivers/media/radio/si470x/Makefile
@@ -2,8 +2,6 @@
 # Makefile for radios with Silicon Labs Si470x FM Radio Receivers
 #
 
-radio-usb-si470x-objs	:= radio-si470x-usb.o radio-si470x-common.o
-radio-i2c-si470x-objs	:= radio-si470x-i2c.o radio-si470x-common.o
-
-obj-$(CONFIG_USB_SI470X) += radio-usb-si470x.o
-obj-$(CONFIG_I2C_SI470X) += radio-i2c-si470x.o
+obj-$(CONFIG_RADIO_SI470X) += radio-si470x-common.o
+obj-$(CONFIG_USB_SI470X) += radio-si470x-usb.o
+obj-$(CONFIG_I2C_SI470X) += radio-si470x-i2c.o
diff --git a/drivers/media/radio/si470x/radio-si470x-common.c b/drivers/media/radio/si470x/radio-si470x-common.c
index b94d66e53d4e..c40e1753f34b 100644
--- a/drivers/media/radio/si470x/radio-si470x-common.c
+++ b/drivers/media/radio/si470x/radio-si470x-common.c
@@ -110,8 +110,6 @@
 /* kernel includes */
 #include "radio-si470x.h"
 
-
-
 /**************************************************************************
  * Module Parameters
  **************************************************************************/
@@ -195,7 +193,7 @@ static int si470x_set_band(struct si470x_device *radio, int band)
 	radio->band = band;
 	radio->registers[SYSCONFIG2] &= ~SYSCONFIG2_BAND;
 	radio->registers[SYSCONFIG2] |= radio->band << 6;
-	return si470x_set_register(radio, SYSCONFIG2);
+	return radio->set_register(radio, SYSCONFIG2);
 }
 
 /*
@@ -207,7 +205,7 @@ static int si470x_set_chan(struct si470x_device *radio, unsigned short chan)
 	unsigned long time_left;
 	bool timed_out = false;
 
-	retval = si470x_get_register(radio, POWERCFG);
+	retval = radio->get_register(radio, POWERCFG);
 	if (retval)
 		return retval;
 
@@ -219,7 +217,7 @@ static int si470x_set_chan(struct si470x_device *radio, unsigned short chan)
 	/* start tuning */
 	radio->registers[CHANNEL] &= ~CHANNEL_CHAN;
 	radio->registers[CHANNEL] |= CHANNEL_TUNE | chan;
-	retval = si470x_set_register(radio, CHANNEL);
+	retval = radio->set_register(radio, CHANNEL);
 	if (retval < 0)
 		goto done;
 
@@ -238,7 +236,7 @@ static int si470x_set_chan(struct si470x_device *radio, unsigned short chan)
 
 	/* stop tuning */
 	radio->registers[CHANNEL] &= ~CHANNEL_TUNE;
-	retval = si470x_set_register(radio, CHANNEL);
+	retval = radio->set_register(radio, CHANNEL);
 
 done:
 	return retval;
@@ -272,7 +270,7 @@ static int si470x_get_freq(struct si470x_device *radio, unsigned int *freq)
 	int chan, retval;
 
 	/* read channel */
-	retval = si470x_get_register(radio, READCHAN);
+	retval = radio->get_register(radio, READCHAN);
 	chan = radio->registers[READCHAN] & READCHAN_READCHAN;
 
 	/* Frequency (MHz) = Spacing (kHz) x Channel + Bottom of Band (MHz) */
@@ -296,6 +294,7 @@ int si470x_set_freq(struct si470x_device *radio, unsigned int freq)
 
 	return si470x_set_chan(radio, chan);
 }
+EXPORT_SYMBOL_GPL(si470x_set_freq);
 
 
 /*
@@ -343,7 +342,7 @@ static int si470x_set_seek(struct si470x_device *radio,
 		radio->registers[POWERCFG] |= POWERCFG_SEEKUP;
 	else
 		radio->registers[POWERCFG] &= ~POWERCFG_SEEKUP;
-	retval = si470x_set_register(radio, POWERCFG);
+	retval = radio->set_register(radio, POWERCFG);
 	if (retval < 0)
 		return retval;
 
@@ -362,7 +361,7 @@ static int si470x_set_seek(struct si470x_device *radio,
 
 	/* stop seeking */
 	radio->registers[POWERCFG] &= ~POWERCFG_SEEK;
-	retval = si470x_set_register(radio, POWERCFG);
+	retval = radio->set_register(radio, POWERCFG);
 
 	/* try again, if timed out */
 	if (retval == 0 && timed_out)
@@ -381,7 +380,7 @@ int si470x_start(struct si470x_device *radio)
 	/* powercfg */
 	radio->registers[POWERCFG] =
 		POWERCFG_DMUTE | POWERCFG_ENABLE | POWERCFG_RDSM;
-	retval = si470x_set_register(radio, POWERCFG);
+	retval = radio->set_register(radio, POWERCFG);
 	if (retval < 0)
 		goto done;
 
@@ -392,7 +391,7 @@ int si470x_start(struct si470x_device *radio)
 	radio->registers[SYSCONFIG1] |= SYSCONFIG1_GPIO2_INT;
 	if (de)
 		radio->registers[SYSCONFIG1] |= SYSCONFIG1_DE;
-	retval = si470x_set_register(radio, SYSCONFIG1);
+	retval = radio->set_register(radio, SYSCONFIG1);
 	if (retval < 0)
 		goto done;
 
@@ -402,7 +401,7 @@ int si470x_start(struct si470x_device *radio)
 		((radio->band << 6) & SYSCONFIG2_BAND) |/* BAND */
 		((space << 4) & SYSCONFIG2_SPACE) |	/* SPACE */
 		15;					/* VOLUME (max) */
-	retval = si470x_set_register(radio, SYSCONFIG2);
+	retval = radio->set_register(radio, SYSCONFIG2);
 	if (retval < 0)
 		goto done;
 
@@ -413,6 +412,7 @@ int si470x_start(struct si470x_device *radio)
 done:
 	return retval;
 }
+EXPORT_SYMBOL_GPL(si470x_start);
 
 
 /*
@@ -424,7 +424,7 @@ int si470x_stop(struct si470x_device *radio)
 
 	/* sysconfig 1 */
 	radio->registers[SYSCONFIG1] &= ~SYSCONFIG1_RDS;
-	retval = si470x_set_register(radio, SYSCONFIG1);
+	retval = radio->set_register(radio, SYSCONFIG1);
 	if (retval < 0)
 		goto done;
 
@@ -432,11 +432,12 @@ int si470x_stop(struct si470x_device *radio)
 	radio->registers[POWERCFG] &= ~POWERCFG_DMUTE;
 	/* POWERCFG_ENABLE has to automatically go low */
 	radio->registers[POWERCFG] |= POWERCFG_ENABLE |	POWERCFG_DISABLE;
-	retval = si470x_set_register(radio, POWERCFG);
+	retval = radio->set_register(radio, POWERCFG);
 
 done:
 	return retval;
 }
+EXPORT_SYMBOL_GPL(si470x_stop);
 
 
 /*
@@ -448,7 +449,7 @@ static int si470x_rds_on(struct si470x_device *radio)
 
 	/* sysconfig 1 */
 	radio->registers[SYSCONFIG1] |= SYSCONFIG1_RDS;
-	retval = si470x_set_register(radio, SYSCONFIG1);
+	retval = radio->set_register(radio, SYSCONFIG1);
 	if (retval < 0)
 		radio->registers[SYSCONFIG1] &= ~SYSCONFIG1_RDS;
 
@@ -542,6 +543,25 @@ static __poll_t si470x_fops_poll(struct file *file,
 }
 
 
+static int si470x_fops_open(struct file *file)
+{
+	struct si470x_device *radio = video_drvdata(file);
+
+	return radio->fops_open(file);
+}
+
+
+/*
+ * si470x_fops_release - file release
+ */
+static int si470x_fops_release(struct file *file)
+{
+	struct si470x_device *radio = video_drvdata(file);
+
+	return radio->fops_release(file);
+}
+
+
 /*
  * si470x_fops - file operations interface
  */
@@ -570,13 +590,13 @@ static int si470x_s_ctrl(struct v4l2_ctrl *ctrl)
 	case V4L2_CID_AUDIO_VOLUME:
 		radio->registers[SYSCONFIG2] &= ~SYSCONFIG2_VOLUME;
 		radio->registers[SYSCONFIG2] |= ctrl->val;
-		return si470x_set_register(radio, SYSCONFIG2);
+		return radio->set_register(radio, SYSCONFIG2);
 	case V4L2_CID_AUDIO_MUTE:
 		if (ctrl->val)
 			radio->registers[POWERCFG] &= ~POWERCFG_DMUTE;
 		else
 			radio->registers[POWERCFG] |= POWERCFG_DMUTE;
-		return si470x_set_register(radio, POWERCFG);
+		return radio->set_register(radio, POWERCFG);
 	default:
 		return -EINVAL;
 	}
@@ -596,7 +616,7 @@ static int si470x_vidioc_g_tuner(struct file *file, void *priv,
 		return -EINVAL;
 
 	if (!radio->status_rssi_auto_update) {
-		retval = si470x_get_register(radio, STATUSRSSI);
+		retval = radio->get_register(radio, STATUSRSSI);
 		if (retval < 0)
 			return retval;
 	}
@@ -665,7 +685,7 @@ static int si470x_vidioc_s_tuner(struct file *file, void *priv,
 		break;
 	}
 
-	return si470x_set_register(radio, POWERCFG);
+	return radio->set_register(radio, POWERCFG);
 }
 
 
@@ -742,6 +762,15 @@ static int si470x_vidioc_enum_freq_bands(struct file *file, void *priv,
 const struct v4l2_ctrl_ops si470x_ctrl_ops = {
 	.s_ctrl = si470x_s_ctrl,
 };
+EXPORT_SYMBOL_GPL(si470x_ctrl_ops);
+
+static int si470x_vidioc_querycap(struct file *file, void *priv,
+		struct v4l2_capability *capability)
+{
+	struct si470x_device *radio = video_drvdata(file);
+
+	return radio->vidioc_querycap(file, priv, capability);
+};
 
 /*
  * si470x_ioctl_ops - video device ioctl operations
@@ -768,3 +797,6 @@ const struct video_device si470x_viddev_template = {
 	.release		= video_device_release_empty,
 	.ioctl_ops		= &si470x_ioctl_ops,
 };
+EXPORT_SYMBOL_GPL(si470x_viddev_template);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/radio/si470x/radio-si470x-i2c.c b/drivers/media/radio/si470x/radio-si470x-i2c.c
index 41709b24b28f..e3b3ecd14a4d 100644
--- a/drivers/media/radio/si470x/radio-si470x-i2c.c
+++ b/drivers/media/radio/si470x/radio-si470x-i2c.c
@@ -89,9 +89,9 @@ MODULE_PARM_DESC(max_rds_errors, "RDS maximum block errors: *1*");
 /*
  * si470x_get_register - read register
  */
-int si470x_get_register(struct si470x_device *radio, int regnr)
+static int si470x_get_register(struct si470x_device *radio, int regnr)
 {
-	u16 buf[READ_REG_NUM];
+	__be16 buf[READ_REG_NUM];
 	struct i2c_msg msgs[1] = {
 		{
 			.addr = radio->client->addr,
@@ -113,10 +113,10 @@ int si470x_get_register(struct si470x_device *radio, int regnr)
 /*
  * si470x_set_register - write register
  */
-int si470x_set_register(struct si470x_device *radio, int regnr)
+static int si470x_set_register(struct si470x_device *radio, int regnr)
 {
 	int i;
-	u16 buf[WRITE_REG_NUM];
+	__be16 buf[WRITE_REG_NUM];
 	struct i2c_msg msgs[1] = {
 		{
 			.addr = radio->client->addr,
@@ -146,7 +146,7 @@ int si470x_set_register(struct si470x_device *radio, int regnr)
 static int si470x_get_all_registers(struct si470x_device *radio)
 {
 	int i;
-	u16 buf[READ_REG_NUM];
+	__be16 buf[READ_REG_NUM];
 	struct i2c_msg msgs[1] = {
 		{
 			.addr = radio->client->addr,
@@ -174,7 +174,7 @@ static int si470x_get_all_registers(struct si470x_device *radio)
 /*
  * si470x_fops_open - file open
  */
-int si470x_fops_open(struct file *file)
+static int si470x_fops_open(struct file *file)
 {
 	struct si470x_device *radio = video_drvdata(file);
 	int retval = v4l2_fh_open(file);
@@ -206,7 +206,7 @@ int si470x_fops_open(struct file *file)
 /*
  * si470x_fops_release - file release
  */
-int si470x_fops_release(struct file *file)
+static int si470x_fops_release(struct file *file)
 {
 	struct si470x_device *radio = video_drvdata(file);
 
@@ -226,8 +226,8 @@ int si470x_fops_release(struct file *file)
 /*
  * si470x_vidioc_querycap - query device capabilities
  */
-int si470x_vidioc_querycap(struct file *file, void *priv,
-		struct v4l2_capability *capability)
+static int si470x_vidioc_querycap(struct file *file, void *priv,
+				  struct v4l2_capability *capability)
 {
 	strlcpy(capability->driver, DRIVER_NAME, sizeof(capability->driver));
 	strlcpy(capability->card, DRIVER_CARD, sizeof(capability->card));
@@ -361,6 +361,12 @@ static int si470x_i2c_probe(struct i2c_client *client,
 	mutex_init(&radio->lock);
 	init_completion(&radio->completion);
 
+	radio->get_register = si470x_get_register;
+	radio->set_register = si470x_set_register;
+	radio->fops_open = si470x_fops_open;
+	radio->fops_release = si470x_fops_release;
+	radio->vidioc_querycap = si470x_vidioc_querycap;
+
 	retval = v4l2_device_register(&client->dev, &radio->v4l2_dev);
 	if (retval < 0) {
 		dev_err(&client->dev, "couldn't register v4l2_device\n");
diff --git a/drivers/media/radio/si470x/radio-si470x-usb.c b/drivers/media/radio/si470x/radio-si470x-usb.c
index 2277e850bb5e..313a95f195a2 100644
--- a/drivers/media/radio/si470x/radio-si470x-usb.c
+++ b/drivers/media/radio/si470x/radio-si470x-usb.c
@@ -250,7 +250,7 @@ static int si470x_set_report(struct si470x_device *radio, void *buf, int size)
 /*
  * si470x_get_register - read register
  */
-int si470x_get_register(struct si470x_device *radio, int regnr)
+static int si470x_get_register(struct si470x_device *radio, int regnr)
 {
 	int retval;
 
@@ -268,7 +268,7 @@ int si470x_get_register(struct si470x_device *radio, int regnr)
 /*
  * si470x_set_register - write register
  */
-int si470x_set_register(struct si470x_device *radio, int regnr)
+static int si470x_set_register(struct si470x_device *radio, int regnr)
 {
 	int retval;
 
@@ -482,12 +482,12 @@ static void si470x_int_in_callback(struct urb *urb)
 }
 
 
-int si470x_fops_open(struct file *file)
+static int si470x_fops_open(struct file *file)
 {
 	return v4l2_fh_open(file);
 }
 
-int si470x_fops_release(struct file *file)
+static int si470x_fops_release(struct file *file)
 {
 	return v4l2_fh_release(file);
 }
@@ -514,8 +514,8 @@ static void si470x_usb_release(struct v4l2_device *v4l2_dev)
 /*
  * si470x_vidioc_querycap - query device capabilities
  */
-int si470x_vidioc_querycap(struct file *file, void *priv,
-		struct v4l2_capability *capability)
+static int si470x_vidioc_querycap(struct file *file, void *priv,
+				  struct v4l2_capability *capability)
 {
 	struct si470x_device *radio = video_drvdata(file);
 
@@ -598,6 +598,12 @@ static int si470x_usb_driver_probe(struct usb_interface *intf,
 	mutex_init(&radio->lock);
 	init_completion(&radio->completion);
 
+	radio->get_register = si470x_get_register;
+	radio->set_register = si470x_set_register;
+	radio->fops_open = si470x_fops_open;
+	radio->fops_release = si470x_fops_release;
+	radio->vidioc_querycap = si470x_vidioc_querycap;
+
 	iface_desc = intf->cur_altsetting;
 
 	/* Set up interrupt endpoint information. */
diff --git a/drivers/media/radio/si470x/radio-si470x.h b/drivers/media/radio/si470x/radio-si470x.h
index 0202f8eb90c4..35fa0f3bbdd2 100644
--- a/drivers/media/radio/si470x/radio-si470x.h
+++ b/drivers/media/radio/si470x/radio-si470x.h
@@ -161,6 +161,15 @@ struct si470x_device {
 	struct completion completion;
 	bool status_rssi_auto_update;	/* Does RSSI get updated automatic? */
 
+	/* si470x ops */
+
+	int (*get_register)(struct si470x_device *radio, int regnr);
+	int (*set_register)(struct si470x_device *radio, int regnr);
+	int (*fops_open)(struct file *file);
+	int (*fops_release)(struct file *file);
+	int (*vidioc_querycap)(struct file *file, void *priv,
+			       struct v4l2_capability *capability);
+
 #if IS_ENABLED(CONFIG_USB_SI470X)
 	/* reference to USB and video device */
 	struct usb_device *usbdev;
@@ -213,13 +222,7 @@ struct si470x_device {
  **************************************************************************/
 extern const struct video_device si470x_viddev_template;
 extern const struct v4l2_ctrl_ops si470x_ctrl_ops;
-int si470x_get_register(struct si470x_device *radio, int regnr);
-int si470x_set_register(struct si470x_device *radio, int regnr);
 int si470x_disconnect_check(struct si470x_device *radio);
 int si470x_set_freq(struct si470x_device *radio, unsigned int freq);
 int si470x_start(struct si470x_device *radio);
 int si470x_stop(struct si470x_device *radio);
-int si470x_fops_open(struct file *file);
-int si470x_fops_release(struct file *file);
-int si470x_vidioc_querycap(struct file *file, void *priv,
-		struct v4l2_capability *capability);
diff --git a/drivers/media/rc/Kconfig b/drivers/media/rc/Kconfig
index eb2c3b6eca7f..1021c08a9ba4 100644
--- a/drivers/media/rc/Kconfig
+++ b/drivers/media/rc/Kconfig
@@ -25,6 +25,19 @@ config LIRC
 	   passes raw IR to and from userspace, which is needed for
 	   IR transmitting (aka "blasting") and for the lirc daemon.
 
+config BPF_LIRC_MODE2
+	bool "Support for eBPF programs attached to lirc devices"
+	depends on BPF_SYSCALL
+	depends on RC_CORE=y
+	depends on LIRC
+	help
+	   Allow attaching eBPF programs to a lirc device using the bpf(2)
+	   syscall command BPF_PROG_ATTACH. This is supported for raw IR
+	   receivers.
+
+	   These eBPF programs can be used to decode IR into scancodes, for
+	   IR protocols not supported by the kernel decoders.
+
 menuconfig RC_DECODERS
 	bool "Remote controller decoders"
 	depends on RC_CORE
@@ -149,7 +162,7 @@ config RC_ATI_REMOTE
 
 config IR_ENE
 	tristate "ENE eHome Receiver/Transceiver (pnp id: ENE0100/ENE02xxx)"
-	depends on PNP
+	depends on PNP || COMPILE_TEST
 	depends on RC_CORE
 	---help---
 	   Say Y here to enable support for integrated infrared receiver
@@ -210,7 +223,7 @@ config IR_MCEUSB
 
 config IR_ITE_CIR
 	tristate "ITE Tech Inc. IT8712/IT8512 Consumer Infrared Transceiver"
-	depends on PNP
+	depends on PNP || COMPILE_TEST
 	depends on RC_CORE
 	---help---
 	   Say Y here to enable support for integrated infrared receivers
@@ -223,7 +236,7 @@ config IR_ITE_CIR
 
 config IR_FINTEK
 	tristate "Fintek Consumer Infrared Transceiver"
-	depends on PNP
+	depends on PNP || COMPILE_TEST
 	depends on RC_CORE
 	---help---
 	   Say Y here to enable support for integrated infrared receiver
@@ -257,7 +270,7 @@ config IR_MTK
 
 config IR_NUVOTON
 	tristate "Nuvoton w836x7hg Consumer Infrared Transceiver"
-	depends on PNP
+	depends on PNP || COMPILE_TEST
 	depends on RC_CORE
 	---help---
 	   Say Y here to enable support for integrated infrared receiver
@@ -305,7 +318,7 @@ config IR_STREAMZAP
 
 config IR_WINBOND_CIR
 	tristate "Winbond IR remote control"
-	depends on X86 && PNP
+	depends on (X86 && PNP) || COMPILE_TEST
 	depends on RC_CORE
 	select NEW_LEDS
 	select LEDS_CLASS
diff --git a/drivers/media/rc/Makefile b/drivers/media/rc/Makefile
index 2e1c87066f6c..e0340d043fe8 100644
--- a/drivers/media/rc/Makefile
+++ b/drivers/media/rc/Makefile
@@ -5,6 +5,7 @@ obj-y += keymaps/
 obj-$(CONFIG_RC_CORE) += rc-core.o
 rc-core-y := rc-main.o rc-ir-raw.o
 rc-core-$(CONFIG_LIRC) += lirc_dev.o
+rc-core-$(CONFIG_BPF_LIRC_MODE2) += bpf-lirc.o
 obj-$(CONFIG_IR_NEC_DECODER) += ir-nec-decoder.o
 obj-$(CONFIG_IR_RC5_DECODER) += ir-rc5-decoder.o
 obj-$(CONFIG_IR_RC6_DECODER) += ir-rc6-decoder.o
diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c
new file mode 100644
index 000000000000..40826bba06b6
--- /dev/null
+++ b/drivers/media/rc/bpf-lirc.c
@@ -0,0 +1,313 @@
+// SPDX-License-Identifier: GPL-2.0
+// bpf-lirc.c - handles bpf
+//
+// Copyright (C) 2018 Sean Young <sean@mess.org>
+
+#include <linux/bpf.h>
+#include <linux/filter.h>
+#include <linux/bpf_lirc.h>
+#include "rc-core-priv.h"
+
+/*
+ * BPF interface for raw IR
+ */
+const struct bpf_prog_ops lirc_mode2_prog_ops = {
+};
+
+BPF_CALL_1(bpf_rc_repeat, u32*, sample)
+{
+	struct ir_raw_event_ctrl *ctrl;
+
+	ctrl = container_of(sample, struct ir_raw_event_ctrl, bpf_sample);
+
+	rc_repeat(ctrl->dev);
+
+	return 0;
+}
+
+static const struct bpf_func_proto rc_repeat_proto = {
+	.func	   = bpf_rc_repeat,
+	.gpl_only  = true, /* rc_repeat is EXPORT_SYMBOL_GPL */
+	.ret_type  = RET_INTEGER,
+	.arg1_type = ARG_PTR_TO_CTX,
+};
+
+/*
+ * Currently rc-core does not support 64-bit scancodes, but there are many
+ * known protocols with more than 32 bits. So, define the interface as u64
+ * as a future-proof.
+ */
+BPF_CALL_4(bpf_rc_keydown, u32*, sample, u32, protocol, u64, scancode,
+	   u32, toggle)
+{
+	struct ir_raw_event_ctrl *ctrl;
+
+	ctrl = container_of(sample, struct ir_raw_event_ctrl, bpf_sample);
+
+	rc_keydown(ctrl->dev, protocol, scancode, toggle != 0);
+
+	return 0;
+}
+
+static const struct bpf_func_proto rc_keydown_proto = {
+	.func	   = bpf_rc_keydown,
+	.gpl_only  = true, /* rc_keydown is EXPORT_SYMBOL_GPL */
+	.ret_type  = RET_INTEGER,
+	.arg1_type = ARG_PTR_TO_CTX,
+	.arg2_type = ARG_ANYTHING,
+	.arg3_type = ARG_ANYTHING,
+	.arg4_type = ARG_ANYTHING,
+};
+
+static const struct bpf_func_proto *
+lirc_mode2_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+{
+	switch (func_id) {
+	case BPF_FUNC_rc_repeat:
+		return &rc_repeat_proto;
+	case BPF_FUNC_rc_keydown:
+		return &rc_keydown_proto;
+	case BPF_FUNC_map_lookup_elem:
+		return &bpf_map_lookup_elem_proto;
+	case BPF_FUNC_map_update_elem:
+		return &bpf_map_update_elem_proto;
+	case BPF_FUNC_map_delete_elem:
+		return &bpf_map_delete_elem_proto;
+	case BPF_FUNC_ktime_get_ns:
+		return &bpf_ktime_get_ns_proto;
+	case BPF_FUNC_tail_call:
+		return &bpf_tail_call_proto;
+	case BPF_FUNC_get_prandom_u32:
+		return &bpf_get_prandom_u32_proto;
+	case BPF_FUNC_trace_printk:
+		if (capable(CAP_SYS_ADMIN))
+			return bpf_get_trace_printk_proto();
+		/* fall through */
+	default:
+		return NULL;
+	}
+}
+
+static bool lirc_mode2_is_valid_access(int off, int size,
+				       enum bpf_access_type type,
+				       const struct bpf_prog *prog,
+				       struct bpf_insn_access_aux *info)
+{
+	/* We have one field of u32 */
+	return type == BPF_READ && off == 0 && size == sizeof(u32);
+}
+
+const struct bpf_verifier_ops lirc_mode2_verifier_ops = {
+	.get_func_proto  = lirc_mode2_func_proto,
+	.is_valid_access = lirc_mode2_is_valid_access
+};
+
+#define BPF_MAX_PROGS 64
+
+static int lirc_bpf_attach(struct rc_dev *rcdev, struct bpf_prog *prog)
+{
+	struct bpf_prog_array __rcu *old_array;
+	struct bpf_prog_array *new_array;
+	struct ir_raw_event_ctrl *raw;
+	int ret;
+
+	if (rcdev->driver_type != RC_DRIVER_IR_RAW)
+		return -EINVAL;
+
+	ret = mutex_lock_interruptible(&ir_raw_handler_lock);
+	if (ret)
+		return ret;
+
+	raw = rcdev->raw;
+	if (!raw) {
+		ret = -ENODEV;
+		goto unlock;
+	}
+
+	if (raw->progs && bpf_prog_array_length(raw->progs) >= BPF_MAX_PROGS) {
+		ret = -E2BIG;
+		goto unlock;
+	}
+
+	old_array = raw->progs;
+	ret = bpf_prog_array_copy(old_array, NULL, prog, &new_array);
+	if (ret < 0)
+		goto unlock;
+
+	rcu_assign_pointer(raw->progs, new_array);
+	bpf_prog_array_free(old_array);
+
+unlock:
+	mutex_unlock(&ir_raw_handler_lock);
+	return ret;
+}
+
+static int lirc_bpf_detach(struct rc_dev *rcdev, struct bpf_prog *prog)
+{
+	struct bpf_prog_array __rcu *old_array;
+	struct bpf_prog_array *new_array;
+	struct ir_raw_event_ctrl *raw;
+	int ret;
+
+	if (rcdev->driver_type != RC_DRIVER_IR_RAW)
+		return -EINVAL;
+
+	ret = mutex_lock_interruptible(&ir_raw_handler_lock);
+	if (ret)
+		return ret;
+
+	raw = rcdev->raw;
+	if (!raw) {
+		ret = -ENODEV;
+		goto unlock;
+	}
+
+	old_array = raw->progs;
+	ret = bpf_prog_array_copy(old_array, prog, NULL, &new_array);
+	/*
+	 * Do not use bpf_prog_array_delete_safe() as we would end up
+	 * with a dummy entry in the array, and the we would free the
+	 * dummy in lirc_bpf_free()
+	 */
+	if (ret)
+		goto unlock;
+
+	rcu_assign_pointer(raw->progs, new_array);
+	bpf_prog_array_free(old_array);
+unlock:
+	mutex_unlock(&ir_raw_handler_lock);
+	return ret;
+}
+
+void lirc_bpf_run(struct rc_dev *rcdev, u32 sample)
+{
+	struct ir_raw_event_ctrl *raw = rcdev->raw;
+
+	raw->bpf_sample = sample;
+
+	if (raw->progs)
+		BPF_PROG_RUN_ARRAY(raw->progs, &raw->bpf_sample, BPF_PROG_RUN);
+}
+
+/*
+ * This should be called once the rc thread has been stopped, so there can be
+ * no concurrent bpf execution.
+ */
+void lirc_bpf_free(struct rc_dev *rcdev)
+{
+	struct bpf_prog **progs;
+
+	if (!rcdev->raw->progs)
+		return;
+
+	progs = rcu_dereference(rcdev->raw->progs)->progs;
+	while (*progs)
+		bpf_prog_put(*progs++);
+
+	bpf_prog_array_free(rcdev->raw->progs);
+}
+
+int lirc_prog_attach(const union bpf_attr *attr)
+{
+	struct bpf_prog *prog;
+	struct rc_dev *rcdev;
+	int ret;
+
+	if (attr->attach_flags)
+		return -EINVAL;
+
+	prog = bpf_prog_get_type(attr->attach_bpf_fd,
+				 BPF_PROG_TYPE_LIRC_MODE2);
+	if (IS_ERR(prog))
+		return PTR_ERR(prog);
+
+	rcdev = rc_dev_get_from_fd(attr->target_fd);
+	if (IS_ERR(rcdev)) {
+		bpf_prog_put(prog);
+		return PTR_ERR(rcdev);
+	}
+
+	ret = lirc_bpf_attach(rcdev, prog);
+	if (ret)
+		bpf_prog_put(prog);
+
+	put_device(&rcdev->dev);
+
+	return ret;
+}
+
+int lirc_prog_detach(const union bpf_attr *attr)
+{
+	struct bpf_prog *prog;
+	struct rc_dev *rcdev;
+	int ret;
+
+	if (attr->attach_flags)
+		return -EINVAL;
+
+	prog = bpf_prog_get_type(attr->attach_bpf_fd,
+				 BPF_PROG_TYPE_LIRC_MODE2);
+	if (IS_ERR(prog))
+		return PTR_ERR(prog);
+
+	rcdev = rc_dev_get_from_fd(attr->target_fd);
+	if (IS_ERR(rcdev)) {
+		bpf_prog_put(prog);
+		return PTR_ERR(rcdev);
+	}
+
+	ret = lirc_bpf_detach(rcdev, prog);
+
+	bpf_prog_put(prog);
+	put_device(&rcdev->dev);
+
+	return ret;
+}
+
+int lirc_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
+{
+	__u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);
+	struct bpf_prog_array __rcu *progs;
+	struct rc_dev *rcdev;
+	u32 cnt, flags = 0;
+	int ret;
+
+	if (attr->query.query_flags)
+		return -EINVAL;
+
+	rcdev = rc_dev_get_from_fd(attr->query.target_fd);
+	if (IS_ERR(rcdev))
+		return PTR_ERR(rcdev);
+
+	if (rcdev->driver_type != RC_DRIVER_IR_RAW) {
+		ret = -EINVAL;
+		goto put;
+	}
+
+	ret = mutex_lock_interruptible(&ir_raw_handler_lock);
+	if (ret)
+		goto put;
+
+	progs = rcdev->raw->progs;
+	cnt = progs ? bpf_prog_array_length(progs) : 0;
+
+	if (copy_to_user(&uattr->query.prog_cnt, &cnt, sizeof(cnt))) {
+		ret = -EFAULT;
+		goto unlock;
+	}
+
+	if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) {
+		ret = -EFAULT;
+		goto unlock;
+	}
+
+	if (attr->query.prog_cnt != 0 && prog_ids && cnt)
+		ret = bpf_prog_array_copy_to_user(progs, prog_ids, cnt);
+
+unlock:
+	mutex_unlock(&ir_raw_handler_lock);
+put:
+	put_device(&rcdev->dev);
+
+	return ret;
+}
diff --git a/drivers/media/rc/ir-imon-decoder.c b/drivers/media/rc/ir-imon-decoder.c
index a1ff06a26542..67c1b0c15aae 100644
--- a/drivers/media/rc/ir-imon-decoder.c
+++ b/drivers/media/rc/ir-imon-decoder.c
@@ -31,9 +31,69 @@ enum imon_state {
 	STATE_INACTIVE,
 	STATE_BIT_CHK,
 	STATE_BIT_START,
-	STATE_FINISHED
+	STATE_FINISHED,
+	STATE_ERROR,
 };
 
+static void ir_imon_decode_scancode(struct rc_dev *dev)
+{
+	struct imon_dec *imon = &dev->raw->imon;
+
+	/* Keyboard/Mouse toggle */
+	if (imon->bits == 0x299115b7)
+		imon->stick_keyboard = !imon->stick_keyboard;
+
+	if ((imon->bits & 0xfc0000ff) == 0x680000b7) {
+		int rel_x, rel_y;
+		u8 buf;
+
+		buf = imon->bits >> 16;
+		rel_x = (buf & 0x08) | (buf & 0x10) >> 2 |
+			(buf & 0x20) >> 4 | (buf & 0x40) >> 6;
+		if (imon->bits & 0x02000000)
+			rel_x |= ~0x0f;
+		buf = imon->bits >> 8;
+		rel_y = (buf & 0x08) | (buf & 0x10) >> 2 |
+			(buf & 0x20) >> 4 | (buf & 0x40) >> 6;
+		if (imon->bits & 0x01000000)
+			rel_y |= ~0x0f;
+
+		if (rel_x && rel_y && imon->stick_keyboard) {
+			if (abs(rel_y) > abs(rel_x))
+				imon->bits = rel_y > 0 ?
+					0x289515b7 : /* KEY_DOWN */
+					0x2aa515b7;  /* KEY_UP */
+			else
+				imon->bits = rel_x > 0 ?
+					0x2ba515b7 : /* KEY_RIGHT */
+					0x29a515b7;  /* KEY_LEFT */
+		}
+
+		if (!imon->stick_keyboard) {
+			struct lirc_scancode lsc = {
+				.scancode = imon->bits,
+				.rc_proto = RC_PROTO_IMON,
+			};
+
+			ir_lirc_scancode_event(dev, &lsc);
+
+			input_event(imon->idev, EV_MSC, MSC_SCAN, imon->bits);
+
+			input_report_rel(imon->idev, REL_X, rel_x);
+			input_report_rel(imon->idev, REL_Y, rel_y);
+
+			input_report_key(imon->idev, BTN_LEFT,
+					 (imon->bits & 0x00010000) != 0);
+			input_report_key(imon->idev, BTN_RIGHT,
+					 (imon->bits & 0x00040000) != 0);
+			input_sync(imon->idev);
+			return;
+		}
+	}
+
+	rc_keydown(dev, RC_PROTO_IMON, imon->bits, 0);
+}
+
 /**
  * ir_imon_decode() - Decode one iMON pulse or space
  * @dev:	the struct rc_dev descriptor of the device
@@ -56,6 +116,22 @@ static int ir_imon_decode(struct rc_dev *dev, struct ir_raw_event ev)
 		data->state, data->count, TO_US(ev.duration),
 		TO_STR(ev.pulse));
 
+	/*
+	 * Since iMON protocol is a series of bits, if at any point
+	 * we encounter an error, make sure that any remaining bits
+	 * aren't parsed as a scancode made up of less bits.
+	 *
+	 * Note that if the stick is held, then the remote repeats
+	 * the scancode with about 12ms between them. So, make sure
+	 * we have at least 10ms of space after an error. That way,
+	 * we're at a new scancode.
+	 */
+	if (data->state == STATE_ERROR) {
+		if (!ev.pulse && ev.duration > MS_TO_NS(10))
+			data->state = STATE_INACTIVE;
+		return 0;
+	}
+
 	for (;;) {
 		if (!geq_margin(ev.duration, IMON_UNIT, IMON_UNIT / 2))
 			return 0;
@@ -95,7 +171,7 @@ static int ir_imon_decode(struct rc_dev *dev, struct ir_raw_event ev)
 		case STATE_FINISHED:
 			if (ev.pulse)
 				goto err_out;
-			rc_keydown(dev, RC_PROTO_IMON, data->bits, 0);
+			ir_imon_decode_scancode(dev);
 			data->state = STATE_INACTIVE;
 			break;
 		}
@@ -107,7 +183,7 @@ static int ir_imon_decode(struct rc_dev *dev, struct ir_raw_event ev)
 		data->state, data->count, TO_US(ev.duration),
 		TO_STR(ev.pulse));
 
-	data->state = STATE_INACTIVE;
+	data->state = STATE_ERROR;
 
 	return -EINVAL;
 }
@@ -165,11 +241,65 @@ static int ir_imon_encode(enum rc_proto protocol, u32 scancode,
 	return e - events;
 }
 
+static int ir_imon_register(struct rc_dev *dev)
+{
+	struct input_dev *idev;
+	struct imon_dec *imon = &dev->raw->imon;
+	int ret;
+
+	idev = input_allocate_device();
+	if (!idev)
+		return -ENOMEM;
+
+	snprintf(imon->name, sizeof(imon->name),
+		 "iMON PAD Stick (%s)", dev->device_name);
+	idev->name = imon->name;
+	idev->phys = dev->input_phys;
+
+	/* Mouse bits */
+	set_bit(EV_REL, idev->evbit);
+	set_bit(EV_KEY, idev->evbit);
+	set_bit(REL_X, idev->relbit);
+	set_bit(REL_Y, idev->relbit);
+	set_bit(BTN_LEFT, idev->keybit);
+	set_bit(BTN_RIGHT, idev->keybit);
+
+	/* Report scancodes too */
+	set_bit(EV_MSC, idev->evbit);
+	set_bit(MSC_SCAN, idev->mscbit);
+
+	input_set_drvdata(idev, imon);
+
+	ret = input_register_device(idev);
+	if (ret < 0) {
+		input_free_device(idev);
+		return -EIO;
+	}
+
+	imon->idev = idev;
+	imon->stick_keyboard = false;
+
+	return 0;
+}
+
+static int ir_imon_unregister(struct rc_dev *dev)
+{
+	struct imon_dec *imon = &dev->raw->imon;
+
+	input_unregister_device(imon->idev);
+	imon->idev = NULL;
+
+	return 0;
+}
+
 static struct ir_raw_handler imon_handler = {
 	.protocols	= RC_PROTO_BIT_IMON,
 	.decode		= ir_imon_decode,
 	.encode		= ir_imon_encode,
 	.carrier	= 38000,
+	.raw_register	= ir_imon_register,
+	.raw_unregister	= ir_imon_unregister,
+	.min_timeout	= IMON_UNIT * IMON_BITS * 2,
 };
 
 static int __init ir_imon_decode_init(void)
diff --git a/drivers/media/rc/ir-jvc-decoder.c b/drivers/media/rc/ir-jvc-decoder.c
index 8cb68ae43282..5706cfe60027 100644
--- a/drivers/media/rc/ir-jvc-decoder.c
+++ b/drivers/media/rc/ir-jvc-decoder.c
@@ -213,6 +213,7 @@ static struct ir_raw_handler jvc_handler = {
 	.decode		= ir_jvc_decode,
 	.encode		= ir_jvc_encode,
 	.carrier	= 38000,
+	.min_timeout	= JVC_TRAILER_SPACE,
 };
 
 static int __init ir_jvc_decode_init(void)
diff --git a/drivers/media/rc/ir-mce_kbd-decoder.c b/drivers/media/rc/ir-mce_kbd-decoder.c
index c110984ca671..64ea42927669 100644
--- a/drivers/media/rc/ir-mce_kbd-decoder.c
+++ b/drivers/media/rc/ir-mce_kbd-decoder.c
@@ -119,17 +119,25 @@ static void mce_kbd_rx_timeout(struct timer_list *t)
 {
 	struct ir_raw_event_ctrl *raw = from_timer(raw, t, mce_kbd.rx_timeout);
 	unsigned char maskcode;
+	unsigned long flags;
 	int i;
 
 	dev_dbg(&raw->dev->dev, "timer callback clearing all keys\n");
 
-	for (i = 0; i < 7; i++) {
-		maskcode = kbd_keycodes[MCIR2_MASK_KEYS_START + i];
-		input_report_key(raw->mce_kbd.idev, maskcode, 0);
-	}
+	spin_lock_irqsave(&raw->mce_kbd.keylock, flags);
+
+	if (time_is_before_eq_jiffies(raw->mce_kbd.rx_timeout.expires)) {
+		for (i = 0; i < 7; i++) {
+			maskcode = kbd_keycodes[MCIR2_MASK_KEYS_START + i];
+			input_report_key(raw->mce_kbd.idev, maskcode, 0);
+		}
+
+		for (i = 0; i < MCIR2_MASK_KEYS_START; i++)
+			input_report_key(raw->mce_kbd.idev, kbd_keycodes[i], 0);
 
-	for (i = 0; i < MCIR2_MASK_KEYS_START; i++)
-		input_report_key(raw->mce_kbd.idev, kbd_keycodes[i], 0);
+		input_sync(raw->mce_kbd.idev);
+	}
+	spin_unlock_irqrestore(&raw->mce_kbd.keylock, flags);
 }
 
 static enum mce_kbd_mode mce_kbd_mode(struct mce_kbd_dec *data)
@@ -147,13 +155,14 @@ static enum mce_kbd_mode mce_kbd_mode(struct mce_kbd_dec *data)
 static void ir_mce_kbd_process_keyboard_data(struct rc_dev *dev, u32 scancode)
 {
 	struct mce_kbd_dec *data = &dev->raw->mce_kbd;
-	u8 keydata   = (scancode >> 8) & 0xff;
+	u8 keydata1  = (scancode >> 8) & 0xff;
+	u8 keydata2  = (scancode >> 16) & 0xff;
 	u8 shiftmask = scancode & 0xff;
-	unsigned char keycode, maskcode;
+	unsigned char maskcode;
 	int i, keystate;
 
-	dev_dbg(&dev->dev, "keyboard: keydata = 0x%02x, shiftmask = 0x%02x\n",
-		keydata, shiftmask);
+	dev_dbg(&dev->dev, "keyboard: keydata2 = 0x%02x, keydata1 = 0x%02x, shiftmask = 0x%02x\n",
+		keydata2, keydata1, shiftmask);
 
 	for (i = 0; i < 7; i++) {
 		maskcode = kbd_keycodes[MCIR2_MASK_KEYS_START + i];
@@ -164,10 +173,12 @@ static void ir_mce_kbd_process_keyboard_data(struct rc_dev *dev, u32 scancode)
 		input_report_key(data->idev, maskcode, keystate);
 	}
 
-	if (keydata) {
-		keycode = kbd_keycodes[keydata];
-		input_report_key(data->idev, keycode, 1);
-	} else {
+	if (keydata1)
+		input_report_key(data->idev, kbd_keycodes[keydata1], 1);
+	if (keydata2)
+		input_report_key(data->idev, kbd_keycodes[keydata2], 1);
+
+	if (!keydata1 && !keydata2) {
 		for (i = 0; i < MCIR2_MASK_KEYS_START; i++)
 			input_report_key(data->idev, kbd_keycodes[i], 0);
 	}
@@ -263,9 +274,6 @@ static int ir_mce_kbd_decode(struct rc_dev *dev, struct ir_raw_event ev)
 		return 0;
 
 	case STATE_HEADER_BIT_END:
-		if (!is_transition(&ev, &dev->raw->prev_ev))
-			break;
-
 		decrease_duration(&ev, MCIR2_BIT_END);
 
 		if (data->count != MCIR2_HEADER_NBITS) {
@@ -302,9 +310,6 @@ static int ir_mce_kbd_decode(struct rc_dev *dev, struct ir_raw_event ev)
 		return 0;
 
 	case STATE_BODY_BIT_END:
-		if (!is_transition(&ev, &dev->raw->prev_ev))
-			break;
-
 		if (data->count == data->wanted_bits)
 			data->state = STATE_FINISHED;
 		else
@@ -319,16 +324,20 @@ static int ir_mce_kbd_decode(struct rc_dev *dev, struct ir_raw_event ev)
 
 		switch (data->wanted_bits) {
 		case MCIR2_KEYBOARD_NBITS:
-			scancode = data->body & 0xffff;
+			scancode = data->body & 0xffffff;
 			dev_dbg(&dev->dev, "keyboard data 0x%08x\n",
 				data->body);
-			if (dev->timeout)
-				delay = usecs_to_jiffies(dev->timeout / 1000);
-			else
-				delay = msecs_to_jiffies(100);
-			mod_timer(&data->rx_timeout, jiffies + delay);
+			spin_lock(&data->keylock);
+			if (scancode) {
+				delay = nsecs_to_jiffies(dev->timeout) +
+					msecs_to_jiffies(100);
+				mod_timer(&data->rx_timeout, jiffies + delay);
+			} else {
+				del_timer(&data->rx_timeout);
+			}
 			/* Pass data to keyboard buffer parser */
 			ir_mce_kbd_process_keyboard_data(dev, scancode);
+			spin_unlock(&data->keylock);
 			lsc.rc_proto = RC_PROTO_MCIR2_KBD;
 			break;
 		case MCIR2_MOUSE_NBITS:
@@ -355,7 +364,6 @@ static int ir_mce_kbd_decode(struct rc_dev *dev, struct ir_raw_event ev)
 	dev_dbg(&dev->dev, "failed at state %i (%uus %s)\n",
 		data->state, TO_US(ev.duration), TO_STR(ev.pulse));
 	data->state = STATE_INACTIVE;
-	input_sync(data->idev);
 	return -EINVAL;
 }
 
@@ -394,6 +402,7 @@ static int ir_mce_kbd_register(struct rc_dev *dev)
 	set_bit(MSC_SCAN, idev->mscbit);
 
 	timer_setup(&mce_kbd->rx_timeout, mce_kbd_rx_timeout, 0);
+	spin_lock_init(&mce_kbd->keylock);
 
 	input_set_drvdata(idev, mce_kbd);
 
@@ -475,6 +484,7 @@ static struct ir_raw_handler mce_kbd_handler = {
 	.raw_register	= ir_mce_kbd_register,
 	.raw_unregister	= ir_mce_kbd_unregister,
 	.carrier	= 36000,
+	.min_timeout	= MCIR2_MAX_LEN + MCIR2_UNIT / 2,
 };
 
 static int __init ir_mce_kbd_decode_init(void)
diff --git a/drivers/media/rc/ir-nec-decoder.c b/drivers/media/rc/ir-nec-decoder.c
index 21647b809e6f..6a8973ae3684 100644
--- a/drivers/media/rc/ir-nec-decoder.c
+++ b/drivers/media/rc/ir-nec-decoder.c
@@ -253,6 +253,7 @@ static struct ir_raw_handler nec_handler = {
 	.decode		= ir_nec_decode,
 	.encode		= ir_nec_encode,
 	.carrier	= 38000,
+	.min_timeout	= NEC_TRAILER_SPACE,
 };
 
 static int __init ir_nec_decode_init(void)
diff --git a/drivers/media/rc/ir-rc5-decoder.c b/drivers/media/rc/ir-rc5-decoder.c
index 74d3b859c3a2..63624654a71e 100644
--- a/drivers/media/rc/ir-rc5-decoder.c
+++ b/drivers/media/rc/ir-rc5-decoder.c
@@ -88,9 +88,6 @@ static int ir_rc5_decode(struct rc_dev *dev, struct ir_raw_event ev)
 		return 0;
 
 	case STATE_BIT_END:
-		if (!is_transition(&ev, &dev->raw->prev_ev))
-			break;
-
 		if (data->count == CHECK_RC5X_NBITS)
 			data->state = STATE_CHECK_RC5X;
 		else
@@ -274,6 +271,7 @@ static struct ir_raw_handler rc5_handler = {
 	.decode		= ir_rc5_decode,
 	.encode		= ir_rc5_encode,
 	.carrier	= 36000,
+	.min_timeout	= RC5_TRAILER,
 };
 
 static int __init ir_rc5_decode_init(void)
diff --git a/drivers/media/rc/ir-rc6-decoder.c b/drivers/media/rc/ir-rc6-decoder.c
index 8314da32453f..68487ce9f79b 100644
--- a/drivers/media/rc/ir-rc6-decoder.c
+++ b/drivers/media/rc/ir-rc6-decoder.c
@@ -145,9 +145,6 @@ static int ir_rc6_decode(struct rc_dev *dev, struct ir_raw_event ev)
 		return 0;
 
 	case STATE_HEADER_BIT_END:
-		if (!is_transition(&ev, &dev->raw->prev_ev))
-			break;
-
 		if (data->count == RC6_HEADER_NBITS)
 			data->state = STATE_TOGGLE_START;
 		else
@@ -165,10 +162,6 @@ static int ir_rc6_decode(struct rc_dev *dev, struct ir_raw_event ev)
 		return 0;
 
 	case STATE_TOGGLE_END:
-		if (!is_transition(&ev, &dev->raw->prev_ev) ||
-		    !geq_margin(ev.duration, RC6_TOGGLE_END, RC6_UNIT / 2))
-			break;
-
 		if (!(data->header & RC6_STARTBIT_MASK)) {
 			dev_dbg(&dev->dev, "RC6 invalid start bit\n");
 			break;
@@ -210,9 +203,6 @@ static int ir_rc6_decode(struct rc_dev *dev, struct ir_raw_event ev)
 		break;
 
 	case STATE_BODY_BIT_END:
-		if (!is_transition(&ev, &dev->raw->prev_ev))
-			break;
-
 		if (data->count == data->wanted_bits)
 			data->state = STATE_FINISHED;
 		else
@@ -394,6 +384,7 @@ static struct ir_raw_handler rc6_handler = {
 	.decode		= ir_rc6_decode,
 	.encode		= ir_rc6_encode,
 	.carrier	= 36000,
+	.min_timeout	= RC6_SUFFIX_SPACE,
 };
 
 static int __init ir_rc6_decode_init(void)
diff --git a/drivers/media/rc/ir-rx51.c b/drivers/media/rc/ir-rx51.c
index 49265f02e772..8a93f7468622 100644
--- a/drivers/media/rc/ir-rx51.c
+++ b/drivers/media/rc/ir-rx51.c
@@ -22,7 +22,6 @@
 #include <linux/hrtimer.h>
 
 #include <media/rc-core.h>
-#include <linux/platform_data/media/ir-rx51.h>
 
 #define WBUF_LEN 256
 
@@ -31,7 +30,6 @@ struct ir_rx51 {
 	struct pwm_device *pwm;
 	struct hrtimer timer;
 	struct device	     *dev;
-	struct ir_rx51_platform_data *pdata;
 	wait_queue_head_t     wqueue;
 
 	unsigned int	freq;		/* carrier frequency */
@@ -130,10 +128,9 @@ static int ir_rx51_tx(struct rc_dev *dev, unsigned int *buffer,
 		ir_rx51->wbuf[count] = -1; /* Insert termination mark */
 
 	/*
-	 * Adjust latency requirements so the device doesn't go in too
-	 * deep sleep states
+	 * REVISIT: Adjust latency requirements so the device doesn't go in too
+	 * deep sleep states with pm_qos_add_request().
 	 */
-	ir_rx51->pdata->set_max_mpu_wakeup_lat(ir_rx51->dev, 50);
 
 	ir_rx51_on(ir_rx51);
 	ir_rx51->wbuf_index = 1;
@@ -146,8 +143,7 @@ static int ir_rx51_tx(struct rc_dev *dev, unsigned int *buffer,
 	 */
 	wait_event_interruptible(ir_rx51->wqueue, ir_rx51->wbuf_index < 0);
 
-	/* We can sleep again */
-	ir_rx51->pdata->set_max_mpu_wakeup_lat(ir_rx51->dev, -1);
+	/* REVISIT: Remove pm_qos constraint, we can sleep again */
 
 	return count;
 }
@@ -244,13 +240,6 @@ static int ir_rx51_probe(struct platform_device *dev)
 	struct pwm_device *pwm;
 	struct rc_dev *rcdev;
 
-	ir_rx51.pdata = dev->dev.platform_data;
-
-	if (!ir_rx51.pdata) {
-		dev_err(&dev->dev, "Platform Data is missing\n");
-		return -ENXIO;
-	}
-
 	pwm = pwm_get(&dev->dev, NULL);
 	if (IS_ERR(pwm)) {
 		int err = PTR_ERR(pwm);
diff --git a/drivers/media/rc/ir-sanyo-decoder.c b/drivers/media/rc/ir-sanyo-decoder.c
index 4efe6db5376a..dd6ee1e339d6 100644
--- a/drivers/media/rc/ir-sanyo-decoder.c
+++ b/drivers/media/rc/ir-sanyo-decoder.c
@@ -210,6 +210,7 @@ static struct ir_raw_handler sanyo_handler = {
 	.decode		= ir_sanyo_decode,
 	.encode		= ir_sanyo_encode,
 	.carrier	= 38000,
+	.min_timeout	= SANYO_TRAILER_SPACE,
 };
 
 static int __init ir_sanyo_decode_init(void)
diff --git a/drivers/media/rc/ir-sharp-decoder.c b/drivers/media/rc/ir-sharp-decoder.c
index 6a38c50566a4..f96e0c992eed 100644
--- a/drivers/media/rc/ir-sharp-decoder.c
+++ b/drivers/media/rc/ir-sharp-decoder.c
@@ -226,6 +226,7 @@ static struct ir_raw_handler sharp_handler = {
 	.decode		= ir_sharp_decode,
 	.encode		= ir_sharp_encode,
 	.carrier	= 38000,
+	.min_timeout	= SHARP_ECHO_SPACE + SHARP_ECHO_SPACE / 4,
 };
 
 static int __init ir_sharp_decode_init(void)
diff --git a/drivers/media/rc/ir-sony-decoder.c b/drivers/media/rc/ir-sony-decoder.c
index 6764ec9de646..5065c081238d 100644
--- a/drivers/media/rc/ir-sony-decoder.c
+++ b/drivers/media/rc/ir-sony-decoder.c
@@ -224,6 +224,7 @@ static struct ir_raw_handler sony_handler = {
 	.decode		= ir_sony_decode,
 	.encode		= ir_sony_encode,
 	.carrier	= 40000,
+	.min_timeout	= SONY_TRAILER_SPACE,
 };
 
 static int __init ir_sony_decode_init(void)
diff --git a/drivers/media/rc/ir-spi.c b/drivers/media/rc/ir-spi.c
index 7163d5ce2e64..66334e8d63ba 100644
--- a/drivers/media/rc/ir-spi.c
+++ b/drivers/media/rc/ir-spi.c
@@ -2,7 +2,7 @@
 // SPI driven IR LED device driver
 //
 // Copyright (c) 2016 Samsung Electronics Co., Ltd.
-// Copyright (c) Andi Shyti <andi.shyti@samsung.com>
+// Copyright (c) Andi Shyti <andi@etezian.org>
 
 #include <linux/delay.h>
 #include <linux/fs.h>
@@ -173,6 +173,6 @@ static struct spi_driver ir_spi_driver = {
 
 module_spi_driver(ir_spi_driver);
 
-MODULE_AUTHOR("Andi Shyti <andi.shyti@samsung.com>");
+MODULE_AUTHOR("Andi Shyti <andi@etezian.org>");
 MODULE_DESCRIPTION("SPI IR LED");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/rc/ir-xmp-decoder.c b/drivers/media/rc/ir-xmp-decoder.c
index 58b47af1a763..c965f51df1c1 100644
--- a/drivers/media/rc/ir-xmp-decoder.c
+++ b/drivers/media/rc/ir-xmp-decoder.c
@@ -199,6 +199,7 @@ static int ir_xmp_decode(struct rc_dev *dev, struct ir_raw_event ev)
 static struct ir_raw_handler xmp_handler = {
 	.protocols	= RC_PROTO_BIT_XMP,
 	.decode		= ir_xmp_decode,
+	.min_timeout	= XMP_TRAILER_SPACE,
 };
 
 static int __init ir_xmp_decode_init(void)
diff --git a/drivers/media/rc/ite-cir.c b/drivers/media/rc/ite-cir.c
index 65e104c7ddfc..de77d22c30a7 100644
--- a/drivers/media/rc/ite-cir.c
+++ b/drivers/media/rc/ite-cir.c
@@ -1561,9 +1561,11 @@ static int ite_probe(struct pnp_dev *pdev, const struct pnp_device_id
 	rdev->close = ite_close;
 	rdev->s_idle = ite_s_idle;
 	rdev->s_rx_carrier_range = ite_set_rx_carrier_range;
-	rdev->min_timeout = ITE_MIN_IDLE_TIMEOUT;
-	rdev->max_timeout = ITE_MAX_IDLE_TIMEOUT;
-	rdev->timeout = ITE_IDLE_TIMEOUT;
+	/* FIFO threshold is 17 bytes, so 17 * 8 samples minimum */
+	rdev->min_timeout = 17 * 8 * ITE_BAUDRATE_DIVISOR *
+			    itdev->params.sample_period;
+	rdev->timeout = IR_DEFAULT_TIMEOUT;
+	rdev->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
 	rdev->rx_resolution = ITE_BAUDRATE_DIVISOR *
 				itdev->params.sample_period;
 	rdev->tx_resolution = ITE_BAUDRATE_DIVISOR *
diff --git a/drivers/media/rc/ite-cir.h b/drivers/media/rc/ite-cir.h
index 0e8ebc880d1f..9cb24ac01350 100644
--- a/drivers/media/rc/ite-cir.h
+++ b/drivers/media/rc/ite-cir.h
@@ -154,13 +154,6 @@ struct ite_dev {
 /* default carrier freq for when demodulator is off (Hz) */
 #define ITE_DEFAULT_CARRIER_FREQ	38000
 
-/* default idling timeout in ns (0.2 seconds) */
-#define ITE_IDLE_TIMEOUT		200000000UL
-
-/* limit timeout values */
-#define ITE_MIN_IDLE_TIMEOUT		100000000UL
-#define ITE_MAX_IDLE_TIMEOUT		1000000000UL
-
 /* convert bits to us */
 #define ITE_BITS_TO_NS(bits, sample_period) \
 ((u32) ((bits) * ITE_BAUDRATE_DIVISOR * sample_period))
diff --git a/drivers/media/rc/lirc_dev.c b/drivers/media/rc/lirc_dev.c
index 24e9fbb80e81..f862f1b7f996 100644
--- a/drivers/media/rc/lirc_dev.c
+++ b/drivers/media/rc/lirc_dev.c
@@ -20,6 +20,7 @@
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/device.h>
+#include <linux/file.h>
 #include <linux/idr.h>
 #include <linux/poll.h>
 #include <linux/sched.h>
@@ -104,6 +105,12 @@ void ir_lirc_raw_event(struct rc_dev *dev, struct ir_raw_event ev)
 			TO_US(ev.duration), TO_STR(ev.pulse));
 	}
 
+	/*
+	 * bpf does not care about the gap generated above; that exists
+	 * for backwards compatibility
+	 */
+	lirc_bpf_run(dev, sample);
+
 	spin_lock_irqsave(&dev->lirc_fh_lock, flags);
 	list_for_each_entry(fh, &dev->lirc_fh, list) {
 		if (LIRC_IS_TIMEOUT(sample) && !fh->send_timeout_reports)
@@ -575,9 +582,16 @@ static long ir_lirc_ioctl(struct file *file, unsigned int cmd,
 		}
 		break;
 
-	case LIRC_SET_REC_TIMEOUT_REPORTS:
+	case LIRC_GET_REC_TIMEOUT:
 		if (!dev->timeout)
 			ret = -ENOTTY;
+		else
+			val = DIV_ROUND_UP(dev->timeout, 1000);
+		break;
+
+	case LIRC_SET_REC_TIMEOUT_REPORTS:
+		if (dev->driver_type != RC_DRIVER_IR_RAW)
+			ret = -ENOTTY;
 		else
 			fh->send_timeout_reports = !!val;
 		break;
@@ -735,6 +749,7 @@ static void lirc_release_device(struct device *ld)
 
 int ir_lirc_register(struct rc_dev *dev)
 {
+	const char *rx_type, *tx_type;
 	int err, minor;
 
 	minor = ida_simple_get(&lirc_ida, 0, RC_DEV_MAX, GFP_KERNEL);
@@ -759,8 +774,25 @@ int ir_lirc_register(struct rc_dev *dev)
 
 	get_device(&dev->dev);
 
-	dev_info(&dev->dev, "lirc_dev: driver %s registered at minor = %d",
-		 dev->driver_name, minor);
+	switch (dev->driver_type) {
+	case RC_DRIVER_SCANCODE:
+		rx_type = "scancode";
+		break;
+	case RC_DRIVER_IR_RAW:
+		rx_type = "raw IR";
+		break;
+	default:
+		rx_type = "no";
+		break;
+	}
+
+	if (dev->tx_ir)
+		tx_type = "raw IR";
+	else
+		tx_type = "no";
+
+	dev_info(&dev->dev, "lirc_dev: driver %s registered at minor = %d, %s receiver, %s transmitter",
+		 dev->driver_name, minor, rx_type, tx_type);
 
 	return 0;
 
@@ -816,4 +848,27 @@ void __exit lirc_dev_exit(void)
 	unregister_chrdev_region(lirc_base_dev, RC_DEV_MAX);
 }
 
+struct rc_dev *rc_dev_get_from_fd(int fd)
+{
+	struct fd f = fdget(fd);
+	struct lirc_fh *fh;
+	struct rc_dev *dev;
+
+	if (!f.file)
+		return ERR_PTR(-EBADF);
+
+	if (f.file->f_op != &lirc_fops) {
+		fdput(f);
+		return ERR_PTR(-EINVAL);
+	}
+
+	fh = f.file->private_data;
+	dev = fh->rc;
+
+	get_device(&dev->dev);
+	fdput(f);
+
+	return dev;
+}
+
 MODULE_ALIAS("lirc_dev");
diff --git a/drivers/media/rc/mceusb.c b/drivers/media/rc/mceusb.c
index 69ba57372c05..4c0c8008872a 100644
--- a/drivers/media/rc/mceusb.c
+++ b/drivers/media/rc/mceusb.c
@@ -181,6 +181,7 @@ enum mceusb_model_type {
 	MCE_GEN2 = 0,		/* Most boards */
 	MCE_GEN1,
 	MCE_GEN3,
+	MCE_GEN3_BROKEN_IRTIMEOUT,
 	MCE_GEN2_TX_INV,
 	MCE_GEN2_TX_INV_RX_GOOD,
 	POLARIS_EVK,
@@ -199,6 +200,7 @@ struct mceusb_model {
 	u32 mce_gen3:1;
 	u32 tx_mask_normal:1;
 	u32 no_tx:1;
+	u32 broken_irtimeout:1;
 	/*
 	 * 2nd IR receiver (short-range, wideband) for learning mode:
 	 *     0, absent 2nd receiver (rx2)
@@ -242,6 +244,12 @@ static const struct mceusb_model mceusb_model[] = {
 		.tx_mask_normal = 1,
 		.rx2 = 2,
 	},
+	[MCE_GEN3_BROKEN_IRTIMEOUT] = {
+		.mce_gen3 = 1,
+		.tx_mask_normal = 1,
+		.rx2 = 2,
+		.broken_irtimeout = 1
+	},
 	[POLARIS_EVK] = {
 		/*
 		 * In fact, the EVK is shipped without
@@ -352,7 +360,7 @@ static const struct usb_device_id mceusb_dev_table[] = {
 	  .driver_info = MCE_GEN2_TX_INV },
 	/* Topseed eHome Infrared Transceiver */
 	{ USB_DEVICE(VENDOR_TOPSEED, 0x0011),
-	  .driver_info = MCE_GEN3 },
+	  .driver_info = MCE_GEN3_BROKEN_IRTIMEOUT },
 	/* Ricavision internal Infrared Transceiver */
 	{ USB_DEVICE(VENDOR_RICAVISION, 0x0010) },
 	/* Itron ione Libra Q-11 */
@@ -564,6 +572,7 @@ static int mceusb_cmd_datasize(u8 cmd, u8 subcmd)
 			datasize = 1;
 			break;
 		}
+		break;
 	case MCE_CMD_PORT_IR:
 		switch (subcmd) {
 		case MCE_CMD_UNKNOWN:
@@ -982,6 +991,25 @@ static int mceusb_set_tx_carrier(struct rc_dev *dev, u32 carrier)
 	return 0;
 }
 
+static int mceusb_set_timeout(struct rc_dev *dev, unsigned int timeout)
+{
+	u8 cmdbuf[4] = { MCE_CMD_PORT_IR, MCE_CMD_SETIRTIMEOUT, 0, 0 };
+	struct mceusb_dev *ir = dev->priv;
+	unsigned int units;
+
+	units = DIV_ROUND_CLOSEST(timeout, US_TO_NS(MCE_TIME_UNIT));
+
+	cmdbuf[2] = units >> 8;
+	cmdbuf[3] = units;
+
+	mce_async_out(ir, cmdbuf, sizeof(cmdbuf));
+
+	/* get receiver timeout value */
+	mce_async_out(ir, GET_RX_TIMEOUT, sizeof(GET_RX_TIMEOUT));
+
+	return 0;
+}
+
 /*
  * Select or deselect the 2nd receiver port.
  * Second receiver is learning mode, wide-band, short-range receiver.
@@ -1150,6 +1178,11 @@ static void mceusb_process_ir_data(struct mceusb_dev *ir, int buf_len)
 			init_ir_raw_event(&rawir);
 			rawir.pulse = ((ir->buf_in[i] & MCE_PULSE_BIT) != 0);
 			rawir.duration = (ir->buf_in[i] & MCE_PULSE_MASK);
+			if (unlikely(!rawir.duration)) {
+				dev_warn(ir->dev, "nonsensical irdata %02x with duration 0",
+					 ir->buf_in[i]);
+				break;
+			}
 			if (rawir.pulse) {
 				ir->pulse_tunit += rawir.duration;
 				ir->pulse_count++;
@@ -1182,7 +1215,12 @@ static void mceusb_process_ir_data(struct mceusb_dev *ir, int buf_len)
 			if (ir->rem) {
 				ir->parser_state = PARSE_IRDATA;
 			} else {
-				ir_raw_event_reset(ir->rc);
+				init_ir_raw_event(&rawir);
+				rawir.timeout = 1;
+				rawir.duration = ir->rc->timeout;
+				if (ir_raw_event_store_with_filter(ir->rc,
+								   &rawir))
+					event = true;
 				ir->pulse_tunit = 0;
 				ir->pulse_count = 0;
 			}
@@ -1415,7 +1453,18 @@ static struct rc_dev *mceusb_init_rc_dev(struct mceusb_dev *ir)
 	rc->dev.parent = dev;
 	rc->priv = ir;
 	rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
+	rc->min_timeout = US_TO_NS(MCE_TIME_UNIT);
 	rc->timeout = MS_TO_NS(100);
+	if (!mceusb_model[ir->model].broken_irtimeout) {
+		rc->s_timeout = mceusb_set_timeout;
+		rc->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
+	} else {
+		/*
+		 * If we can't set the timeout using CMD_SETIRTIMEOUT, we can
+		 * rely on software timeouts for timeouts < 100ms.
+		 */
+		rc->max_timeout = rc->timeout;
+	}
 	if (!ir->flags.no_tx) {
 		rc->s_tx_mask = mceusb_set_tx_mask;
 		rc->s_tx_carrier = mceusb_set_tx_carrier;
diff --git a/drivers/media/rc/mtk-cir.c b/drivers/media/rc/mtk-cir.c
index e88eb64e8e69..e42efd9d382e 100644
--- a/drivers/media/rc/mtk-cir.c
+++ b/drivers/media/rc/mtk-cir.c
@@ -299,8 +299,6 @@ static int mtk_ir_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct device_node *dn = dev->of_node;
-	const struct of_device_id *of_id =
-		of_match_device(mtk_ir_match, &pdev->dev);
 	struct resource *res;
 	struct mtk_ir *ir;
 	u32 val;
@@ -312,7 +310,7 @@ static int mtk_ir_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	ir->dev = dev;
-	ir->data = of_id->data;
+	ir->data = of_device_get_match_data(dev);
 
 	ir->clk = devm_clk_get(dev, "clk");
 	if (IS_ERR(ir->clk)) {
diff --git a/drivers/media/rc/nuvoton-cir.c b/drivers/media/rc/nuvoton-cir.c
index 5e1d866a61a5..b8299c9a9744 100644
--- a/drivers/media/rc/nuvoton-cir.c
+++ b/drivers/media/rc/nuvoton-cir.c
@@ -535,6 +535,8 @@ static void nvt_set_cir_iren(struct nvt_dev *nvt)
 
 static void nvt_cir_regs_init(struct nvt_dev *nvt)
 {
+	nvt_enable_logical_dev(nvt, LOGICAL_DEV_CIR);
+
 	/* set sample limit count (PE interrupt raised when reached) */
 	nvt_cir_reg_write(nvt, CIR_RX_LIMIT_COUNT >> 8, CIR_SLCH);
 	nvt_cir_reg_write(nvt, CIR_RX_LIMIT_COUNT & 0xff, CIR_SLCL);
@@ -543,31 +545,17 @@ static void nvt_cir_regs_init(struct nvt_dev *nvt)
 	nvt_cir_reg_write(nvt, CIR_FIFOCON_TX_TRIGGER_LEV |
 			  CIR_FIFOCON_RX_TRIGGER_LEV, CIR_FIFOCON);
 
-	/*
-	 * Enable TX and RX, specify carrier on = low, off = high, and set
-	 * sample period (currently 50us)
-	 */
-	nvt_cir_reg_write(nvt,
-			  CIR_IRCON_TXEN | CIR_IRCON_RXEN |
-			  CIR_IRCON_RXINV | CIR_IRCON_SAMPLE_PERIOD_SEL,
-			  CIR_IRCON);
-
 	/* clear hardware rx and tx fifos */
 	nvt_clear_cir_fifo(nvt);
 	nvt_clear_tx_fifo(nvt);
 
-	/* clear any and all stray interrupts */
-	nvt_cir_reg_write(nvt, 0xff, CIR_IRSTS);
-
-	/* and finally, enable interrupts */
-	nvt_set_cir_iren(nvt);
-
-	/* enable the CIR logical device */
-	nvt_enable_logical_dev(nvt, LOGICAL_DEV_CIR);
+	nvt_disable_logical_dev(nvt, LOGICAL_DEV_CIR);
 }
 
 static void nvt_cir_wake_regs_init(struct nvt_dev *nvt)
 {
+	nvt_enable_logical_dev(nvt, LOGICAL_DEV_CIR_WAKE);
+
 	/*
 	 * Disable RX, set specific carrier on = low, off = high,
 	 * and sample period (currently 50us)
@@ -579,9 +567,6 @@ static void nvt_cir_wake_regs_init(struct nvt_dev *nvt)
 
 	/* clear any and all stray interrupts */
 	nvt_cir_wake_reg_write(nvt, 0xff, CIR_WAKE_IRSTS);
-
-	/* enable the CIR WAKE logical device */
-	nvt_enable_logical_dev(nvt, LOGICAL_DEV_CIR_WAKE);
 }
 
 static void nvt_enable_wake(struct nvt_dev *nvt)
@@ -892,6 +877,32 @@ static irqreturn_t nvt_cir_isr(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static void nvt_enable_cir(struct nvt_dev *nvt)
+{
+	unsigned long flags;
+
+	/* enable the CIR logical device */
+	nvt_enable_logical_dev(nvt, LOGICAL_DEV_CIR);
+
+	spin_lock_irqsave(&nvt->lock, flags);
+
+	/*
+	 * Enable TX and RX, specify carrier on = low, off = high, and set
+	 * sample period (currently 50us)
+	 */
+	nvt_cir_reg_write(nvt, CIR_IRCON_TXEN | CIR_IRCON_RXEN |
+			  CIR_IRCON_RXINV | CIR_IRCON_SAMPLE_PERIOD_SEL,
+			  CIR_IRCON);
+
+	/* clear all pending interrupts */
+	nvt_cir_reg_write(nvt, 0xff, CIR_IRSTS);
+
+	/* enable interrupts */
+	nvt_set_cir_iren(nvt);
+
+	spin_unlock_irqrestore(&nvt->lock, flags);
+}
+
 static void nvt_disable_cir(struct nvt_dev *nvt)
 {
 	unsigned long flags;
@@ -920,25 +931,8 @@ static void nvt_disable_cir(struct nvt_dev *nvt)
 static int nvt_open(struct rc_dev *dev)
 {
 	struct nvt_dev *nvt = dev->priv;
-	unsigned long flags;
-
-	spin_lock_irqsave(&nvt->lock, flags);
-
-	/* set function enable flags */
-	nvt_cir_reg_write(nvt, CIR_IRCON_TXEN | CIR_IRCON_RXEN |
-			  CIR_IRCON_RXINV | CIR_IRCON_SAMPLE_PERIOD_SEL,
-			  CIR_IRCON);
-
-	/* clear all pending interrupts */
-	nvt_cir_reg_write(nvt, 0xff, CIR_IRSTS);
-
-	/* enable interrupts */
-	nvt_set_cir_iren(nvt);
 
-	spin_unlock_irqrestore(&nvt->lock, flags);
-
-	/* enable the CIR logical device */
-	nvt_enable_logical_dev(nvt, LOGICAL_DEV_CIR);
+	nvt_enable_cir(nvt);
 
 	return 0;
 }
@@ -1093,19 +1087,13 @@ static void nvt_remove(struct pnp_dev *pdev)
 static int nvt_suspend(struct pnp_dev *pdev, pm_message_t state)
 {
 	struct nvt_dev *nvt = pnp_get_drvdata(pdev);
-	unsigned long flags;
 
 	nvt_dbg("%s called", __func__);
 
-	spin_lock_irqsave(&nvt->lock, flags);
-
-	/* disable all CIR interrupts */
-	nvt_cir_reg_write(nvt, 0, CIR_IREN);
-
-	spin_unlock_irqrestore(&nvt->lock, flags);
-
-	/* disable cir logical dev */
-	nvt_disable_logical_dev(nvt, LOGICAL_DEV_CIR);
+	mutex_lock(&nvt->rdev->lock);
+	if (nvt->rdev->users)
+		nvt_disable_cir(nvt);
+	mutex_unlock(&nvt->rdev->lock);
 
 	/* make sure wake is enabled */
 	nvt_enable_wake(nvt);
@@ -1122,6 +1110,11 @@ static int nvt_resume(struct pnp_dev *pdev)
 	nvt_cir_regs_init(nvt);
 	nvt_cir_wake_regs_init(nvt);
 
+	mutex_lock(&nvt->rdev->lock);
+	if (nvt->rdev->users)
+		nvt_enable_cir(nvt);
+	mutex_unlock(&nvt->rdev->lock);
+
 	return 0;
 }
 
diff --git a/drivers/media/rc/rc-core-priv.h b/drivers/media/rc/rc-core-priv.h
index e0e6a17460f6..e847bdad5c51 100644
--- a/drivers/media/rc/rc-core-priv.h
+++ b/drivers/media/rc/rc-core-priv.h
@@ -13,6 +13,7 @@
 #define	MAX_IR_EVENT_SIZE	512
 
 #include <linux/slab.h>
+#include <uapi/linux/bpf.h>
 #include <media/rc-core.h>
 
 /**
@@ -37,6 +38,7 @@ struct ir_raw_handler {
 	int (*encode)(enum rc_proto protocol, u32 scancode,
 		      struct ir_raw_event *events, unsigned int max);
 	u32 carrier;
+	u32 min_timeout;
 
 	/* These two should only be used by the mce kbd decoder */
 	int (*raw_register)(struct rc_dev *dev);
@@ -57,6 +59,11 @@ struct ir_raw_event_ctrl {
 	/* raw decoder state follows */
 	struct ir_raw_event prev_ev;
 	struct ir_raw_event this_ev;
+
+#ifdef CONFIG_BPF_LIRC_MODE2
+	u32				bpf_sample;
+	struct bpf_prog_array __rcu	*progs;
+#endif
 	struct nec_dec {
 		int state;
 		unsigned count;
@@ -104,6 +111,8 @@ struct ir_raw_event_ctrl {
 	} sharp;
 	struct mce_kbd_dec {
 		struct input_dev *idev;
+		/* locks key up timer */
+		spinlock_t keylock;
 		struct timer_list rx_timeout;
 		char name[64];
 		char phys[64];
@@ -123,9 +132,15 @@ struct ir_raw_event_ctrl {
 		int count;
 		int last_chk;
 		unsigned int bits;
+		bool stick_keyboard;
+		struct input_dev *idev;
+		char name[64];
 	} imon;
 };
 
+/* Mutex for locking raw IR processing and handler change */
+extern struct mutex ir_raw_handler_lock;
+
 /* macros for IR decoders */
 static inline bool geq_margin(unsigned d1, unsigned d2, unsigned margin)
 {
@@ -288,6 +303,7 @@ void ir_lirc_raw_event(struct rc_dev *dev, struct ir_raw_event ev);
 void ir_lirc_scancode_event(struct rc_dev *dev, struct lirc_scancode *lsc);
 int ir_lirc_register(struct rc_dev *dev);
 void ir_lirc_unregister(struct rc_dev *dev);
+struct rc_dev *rc_dev_get_from_fd(int fd);
 #else
 static inline int lirc_dev_init(void) { return 0; }
 static inline void lirc_dev_exit(void) {}
@@ -299,4 +315,15 @@ static inline int ir_lirc_register(struct rc_dev *dev) { return 0; }
 static inline void ir_lirc_unregister(struct rc_dev *dev) { }
 #endif
 
+/*
+ * bpf interface
+ */
+#ifdef CONFIG_BPF_LIRC_MODE2
+void lirc_bpf_free(struct rc_dev *dev);
+void lirc_bpf_run(struct rc_dev *dev, u32 sample);
+#else
+static inline void lirc_bpf_free(struct rc_dev *dev) { }
+static inline void lirc_bpf_run(struct rc_dev *dev, u32 sample) { }
+#endif
+
 #endif /* _RC_CORE_PRIV */
diff --git a/drivers/media/rc/rc-ir-raw.c b/drivers/media/rc/rc-ir-raw.c
index 374f83105a23..2e0066b1a31c 100644
--- a/drivers/media/rc/rc-ir-raw.c
+++ b/drivers/media/rc/rc-ir-raw.c
@@ -14,7 +14,7 @@
 static LIST_HEAD(ir_raw_client_list);
 
 /* Used to handle IR raw handler extensions */
-static DEFINE_MUTEX(ir_raw_handler_lock);
+DEFINE_MUTEX(ir_raw_handler_lock);
 static LIST_HEAD(ir_raw_handler_list);
 static atomic64_t available_protocols = ATOMIC64_INIT(0);
 
@@ -22,16 +22,27 @@ static int ir_raw_event_thread(void *data)
 {
 	struct ir_raw_event ev;
 	struct ir_raw_handler *handler;
-	struct ir_raw_event_ctrl *raw = (struct ir_raw_event_ctrl *)data;
+	struct ir_raw_event_ctrl *raw = data;
+	struct rc_dev *dev = raw->dev;
 
 	while (1) {
 		mutex_lock(&ir_raw_handler_lock);
 		while (kfifo_out(&raw->kfifo, &ev, 1)) {
+			if (is_timing_event(ev)) {
+				if (ev.duration == 0)
+					dev_err(&dev->dev, "nonsensical timing event of duration 0");
+				if (is_timing_event(raw->prev_ev) &&
+				    !is_transition(&ev, &raw->prev_ev))
+					dev_err(&dev->dev, "two consecutive events of type %s",
+						TO_STR(ev.pulse));
+				if (raw->prev_ev.reset && ev.pulse == 0)
+					dev_err(&dev->dev, "timing event after reset should be pulse");
+			}
 			list_for_each_entry(handler, &ir_raw_handler_list, list)
-				if (raw->dev->enabled_protocols &
+				if (dev->enabled_protocols &
 				    handler->protocols || !handler->protocols)
-					handler->decode(raw->dev, ev);
-			ir_lirc_raw_event(raw->dev, ev);
+					handler->decode(dev, ev);
+			ir_lirc_raw_event(dev, ev);
 			raw->prev_ev = ev;
 		}
 		mutex_unlock(&ir_raw_handler_lock);
@@ -233,7 +244,49 @@ ir_raw_get_allowed_protocols(void)
 
 static int change_protocol(struct rc_dev *dev, u64 *rc_proto)
 {
-	/* the caller will update dev->enabled_protocols */
+	struct ir_raw_handler *handler;
+	u32 timeout = 0;
+
+	mutex_lock(&ir_raw_handler_lock);
+	list_for_each_entry(handler, &ir_raw_handler_list, list) {
+		if (!(dev->enabled_protocols & handler->protocols) &&
+		    (*rc_proto & handler->protocols) && handler->raw_register)
+			handler->raw_register(dev);
+
+		if ((dev->enabled_protocols & handler->protocols) &&
+		    !(*rc_proto & handler->protocols) &&
+		    handler->raw_unregister)
+			handler->raw_unregister(dev);
+	}
+	mutex_unlock(&ir_raw_handler_lock);
+
+	if (!dev->max_timeout)
+		return 0;
+
+	mutex_lock(&ir_raw_handler_lock);
+	list_for_each_entry(handler, &ir_raw_handler_list, list) {
+		if (handler->protocols & *rc_proto) {
+			if (timeout < handler->min_timeout)
+				timeout = handler->min_timeout;
+		}
+	}
+	mutex_unlock(&ir_raw_handler_lock);
+
+	if (timeout == 0)
+		timeout = IR_DEFAULT_TIMEOUT;
+	else
+		timeout += MS_TO_NS(10);
+
+	if (timeout < dev->min_timeout)
+		timeout = dev->min_timeout;
+	else if (timeout > dev->max_timeout)
+		timeout = dev->max_timeout;
+
+	if (dev->s_timeout)
+		dev->s_timeout(dev, timeout);
+	else
+		dev->timeout = timeout;
+
 	return 0;
 }
 
@@ -569,6 +622,7 @@ int ir_raw_event_prepare(struct rc_dev *dev)
 
 	dev->raw->dev = dev;
 	dev->change_protocol = change_protocol;
+	dev->idle = true;
 	spin_lock_init(&dev->raw->edge_spinlock);
 	timer_setup(&dev->raw->edge_handle, ir_raw_edge_handle, 0);
 	INIT_KFIFO(dev->raw->kfifo);
@@ -578,7 +632,6 @@ int ir_raw_event_prepare(struct rc_dev *dev)
 
 int ir_raw_event_register(struct rc_dev *dev)
 {
-	struct ir_raw_handler *handler;
 	struct task_struct *thread;
 
 	thread = kthread_run(ir_raw_event_thread, dev->raw, "rc%u", dev->minor);
@@ -589,9 +642,6 @@ int ir_raw_event_register(struct rc_dev *dev)
 
 	mutex_lock(&ir_raw_handler_lock);
 	list_add_tail(&dev->raw->list, &ir_raw_client_list);
-	list_for_each_entry(handler, &ir_raw_handler_list, list)
-		if (handler->raw_register)
-			handler->raw_register(dev);
 	mutex_unlock(&ir_raw_handler_lock);
 
 	return 0;
@@ -619,11 +669,20 @@ void ir_raw_event_unregister(struct rc_dev *dev)
 	mutex_lock(&ir_raw_handler_lock);
 	list_del(&dev->raw->list);
 	list_for_each_entry(handler, &ir_raw_handler_list, list)
-		if (handler->raw_unregister)
+		if (handler->raw_unregister &&
+		    (handler->protocols & dev->enabled_protocols))
 			handler->raw_unregister(dev);
-	mutex_unlock(&ir_raw_handler_lock);
+
+	lirc_bpf_free(dev);
 
 	ir_raw_event_free(dev);
+
+	/*
+	 * A user can be calling bpf(BPF_PROG_{QUERY|ATTACH|DETACH}), so
+	 * ensure that the raw member is null on unlock; this is how
+	 * "device gone" is checked.
+	 */
+	mutex_unlock(&ir_raw_handler_lock);
 }
 
 /*
@@ -632,13 +691,8 @@ void ir_raw_event_unregister(struct rc_dev *dev)
 
 int ir_raw_handler_register(struct ir_raw_handler *ir_raw_handler)
 {
-	struct ir_raw_event_ctrl *raw;
-
 	mutex_lock(&ir_raw_handler_lock);
 	list_add_tail(&ir_raw_handler->list, &ir_raw_handler_list);
-	if (ir_raw_handler->raw_register)
-		list_for_each_entry(raw, &ir_raw_client_list, list)
-			ir_raw_handler->raw_register(raw->dev);
 	atomic64_or(ir_raw_handler->protocols, &available_protocols);
 	mutex_unlock(&ir_raw_handler_lock);
 
@@ -654,9 +708,10 @@ void ir_raw_handler_unregister(struct ir_raw_handler *ir_raw_handler)
 	mutex_lock(&ir_raw_handler_lock);
 	list_del(&ir_raw_handler->list);
 	list_for_each_entry(raw, &ir_raw_client_list, list) {
-		ir_raw_disable_protocols(raw->dev, protocols);
-		if (ir_raw_handler->raw_unregister)
+		if (ir_raw_handler->raw_unregister &&
+		    (raw->dev->enabled_protocols & protocols))
 			ir_raw_handler->raw_unregister(raw->dev);
+		ir_raw_disable_protocols(raw->dev, protocols);
 	}
 	atomic64_andnot(protocols, &available_protocols);
 	mutex_unlock(&ir_raw_handler_lock);
diff --git a/drivers/media/rc/rc-main.c b/drivers/media/rc/rc-main.c
index b67be33bd62f..2e222d9ee01f 100644
--- a/drivers/media/rc/rc-main.c
+++ b/drivers/media/rc/rc-main.c
@@ -26,50 +26,50 @@ static const struct {
 	unsigned int repeat_period;
 	unsigned int scancode_bits;
 } protocols[] = {
-	[RC_PROTO_UNKNOWN] = { .name = "unknown", .repeat_period = 250 },
-	[RC_PROTO_OTHER] = { .name = "other", .repeat_period = 250 },
+	[RC_PROTO_UNKNOWN] = { .name = "unknown", .repeat_period = 125 },
+	[RC_PROTO_OTHER] = { .name = "other", .repeat_period = 125 },
 	[RC_PROTO_RC5] = { .name = "rc-5",
-		.scancode_bits = 0x1f7f, .repeat_period = 250 },
+		.scancode_bits = 0x1f7f, .repeat_period = 114 },
 	[RC_PROTO_RC5X_20] = { .name = "rc-5x-20",
-		.scancode_bits = 0x1f7f3f, .repeat_period = 250 },
+		.scancode_bits = 0x1f7f3f, .repeat_period = 114 },
 	[RC_PROTO_RC5_SZ] = { .name = "rc-5-sz",
-		.scancode_bits = 0x2fff, .repeat_period = 250 },
+		.scancode_bits = 0x2fff, .repeat_period = 114 },
 	[RC_PROTO_JVC] = { .name = "jvc",
-		.scancode_bits = 0xffff, .repeat_period = 250 },
+		.scancode_bits = 0xffff, .repeat_period = 125 },
 	[RC_PROTO_SONY12] = { .name = "sony-12",
-		.scancode_bits = 0x1f007f, .repeat_period = 250 },
+		.scancode_bits = 0x1f007f, .repeat_period = 100 },
 	[RC_PROTO_SONY15] = { .name = "sony-15",
-		.scancode_bits = 0xff007f, .repeat_period = 250 },
+		.scancode_bits = 0xff007f, .repeat_period = 100 },
 	[RC_PROTO_SONY20] = { .name = "sony-20",
-		.scancode_bits = 0x1fff7f, .repeat_period = 250 },
+		.scancode_bits = 0x1fff7f, .repeat_period = 100 },
 	[RC_PROTO_NEC] = { .name = "nec",
-		.scancode_bits = 0xffff, .repeat_period = 250 },
+		.scancode_bits = 0xffff, .repeat_period = 110 },
 	[RC_PROTO_NECX] = { .name = "nec-x",
-		.scancode_bits = 0xffffff, .repeat_period = 250 },
+		.scancode_bits = 0xffffff, .repeat_period = 110 },
 	[RC_PROTO_NEC32] = { .name = "nec-32",
-		.scancode_bits = 0xffffffff, .repeat_period = 250 },
+		.scancode_bits = 0xffffffff, .repeat_period = 110 },
 	[RC_PROTO_SANYO] = { .name = "sanyo",
-		.scancode_bits = 0x1fffff, .repeat_period = 250 },
+		.scancode_bits = 0x1fffff, .repeat_period = 125 },
 	[RC_PROTO_MCIR2_KBD] = { .name = "mcir2-kbd",
-		.scancode_bits = 0xffff, .repeat_period = 250 },
+		.scancode_bits = 0xffffff, .repeat_period = 100 },
 	[RC_PROTO_MCIR2_MSE] = { .name = "mcir2-mse",
-		.scancode_bits = 0x1fffff, .repeat_period = 250 },
+		.scancode_bits = 0x1fffff, .repeat_period = 100 },
 	[RC_PROTO_RC6_0] = { .name = "rc-6-0",
-		.scancode_bits = 0xffff, .repeat_period = 250 },
+		.scancode_bits = 0xffff, .repeat_period = 114 },
 	[RC_PROTO_RC6_6A_20] = { .name = "rc-6-6a-20",
-		.scancode_bits = 0xfffff, .repeat_period = 250 },
+		.scancode_bits = 0xfffff, .repeat_period = 114 },
 	[RC_PROTO_RC6_6A_24] = { .name = "rc-6-6a-24",
-		.scancode_bits = 0xffffff, .repeat_period = 250 },
+		.scancode_bits = 0xffffff, .repeat_period = 114 },
 	[RC_PROTO_RC6_6A_32] = { .name = "rc-6-6a-32",
-		.scancode_bits = 0xffffffff, .repeat_period = 250 },
+		.scancode_bits = 0xffffffff, .repeat_period = 114 },
 	[RC_PROTO_RC6_MCE] = { .name = "rc-6-mce",
-		.scancode_bits = 0xffff7fff, .repeat_period = 250 },
+		.scancode_bits = 0xffff7fff, .repeat_period = 114 },
 	[RC_PROTO_SHARP] = { .name = "sharp",
-		.scancode_bits = 0x1fff, .repeat_period = 250 },
-	[RC_PROTO_XMP] = { .name = "xmp", .repeat_period = 250 },
-	[RC_PROTO_CEC] = { .name = "cec", .repeat_period = 550 },
+		.scancode_bits = 0x1fff, .repeat_period = 125 },
+	[RC_PROTO_XMP] = { .name = "xmp", .repeat_period = 125 },
+	[RC_PROTO_CEC] = { .name = "cec", .repeat_period = 0 },
 	[RC_PROTO_IMON] = { .name = "imon",
-		.scancode_bits = 0x7fffffff, .repeat_period = 250 },
+		.scancode_bits = 0x7fffffff, .repeat_period = 114 },
 };
 
 /* Used to keep track of known keymaps */
@@ -690,7 +690,8 @@ static void ir_timer_repeat(struct timer_list *t)
 void rc_repeat(struct rc_dev *dev)
 {
 	unsigned long flags;
-	unsigned int timeout = protocols[dev->last_protocol].repeat_period;
+	unsigned int timeout = nsecs_to_jiffies(dev->timeout) +
+		msecs_to_jiffies(protocols[dev->last_protocol].repeat_period);
 	struct lirc_scancode sc = {
 		.scancode = dev->last_scancode, .rc_proto = dev->last_protocol,
 		.keycode = dev->keypressed ? dev->last_keycode : KEY_RESERVED,
@@ -706,7 +707,7 @@ void rc_repeat(struct rc_dev *dev)
 	input_sync(dev->input_dev);
 
 	if (dev->keypressed) {
-		dev->keyup_jiffies = jiffies + msecs_to_jiffies(timeout);
+		dev->keyup_jiffies = jiffies + timeout;
 		mod_timer(&dev->timer_keyup, dev->keyup_jiffies);
 	}
 
@@ -801,7 +802,7 @@ void rc_keydown(struct rc_dev *dev, enum rc_proto protocol, u32 scancode,
 	ir_do_keydown(dev, protocol, scancode, keycode, toggle);
 
 	if (dev->keypressed) {
-		dev->keyup_jiffies = jiffies +
+		dev->keyup_jiffies = jiffies + nsecs_to_jiffies(dev->timeout) +
 			msecs_to_jiffies(protocols[protocol].repeat_period);
 		mod_timer(&dev->timer_keyup, dev->keyup_jiffies);
 	}
@@ -1241,6 +1242,9 @@ static ssize_t store_protocols(struct device *device,
 	if (rc < 0)
 		goto out;
 
+	if (dev->driver_type == RC_DRIVER_IR_RAW)
+		ir_raw_load_modules(&new_protocols);
+
 	rc = dev->change_protocol(dev, &new_protocols);
 	if (rc < 0) {
 		dev_dbg(&dev->dev, "Error setting protocols to 0x%llx\n",
@@ -1248,9 +1252,6 @@ static ssize_t store_protocols(struct device *device,
 		goto out;
 	}
 
-	if (dev->driver_type == RC_DRIVER_IR_RAW)
-		ir_raw_load_modules(&new_protocols);
-
 	if (new_protocols != old_protocols) {
 		*current_protocols = new_protocols;
 		dev_dbg(&dev->dev, "Protocols changed to 0x%llx\n",
@@ -1647,6 +1648,7 @@ struct rc_dev *rc_allocate_device(enum rc_driver_type type)
 		dev->input_dev->setkeycode = ir_setkeycode;
 		input_set_drvdata(dev->input_dev, dev);
 
+		dev->timeout = IR_DEFAULT_TIMEOUT;
 		timer_setup(&dev->timer_keyup, ir_timer_keyup, 0);
 		timer_setup(&dev->timer_repeat, ir_timer_repeat, 0);
 
@@ -1735,6 +1737,9 @@ static int rc_prepare_rx_device(struct rc_dev *dev)
 	if (dev->driver_type == RC_DRIVER_SCANCODE && !dev->change_protocol)
 		dev->enabled_protocols = dev->allowed_protocols;
 
+	if (dev->driver_type == RC_DRIVER_IR_RAW)
+		ir_raw_load_modules(&rc_proto);
+
 	if (dev->change_protocol) {
 		rc = dev->change_protocol(dev, &rc_proto);
 		if (rc < 0)
@@ -1742,9 +1747,6 @@ static int rc_prepare_rx_device(struct rc_dev *dev)
 		dev->enabled_protocols = rc_proto;
 	}
 
-	if (dev->driver_type == RC_DRIVER_IR_RAW)
-		ir_raw_load_modules(&rc_proto);
-
 	set_bit(EV_KEY, dev->input_dev->evbit);
 	set_bit(EV_REP, dev->input_dev->evbit);
 	set_bit(EV_MSC, dev->input_dev->evbit);
@@ -1860,6 +1862,8 @@ int rc_register_device(struct rc_dev *dev)
 		 dev->device_name ?: "Unspecified device", path ?: "N/A");
 	kfree(path);
 
+	dev->registered = true;
+
 	if (dev->driver_type != RC_DRIVER_IR_RAW_TX) {
 		rc = rc_setup_rx_device(dev);
 		if (rc)
@@ -1879,8 +1883,6 @@ int rc_register_device(struct rc_dev *dev)
 			goto out_lirc;
 	}
 
-	dev->registered = true;
-
 	dev_dbg(&dev->dev, "Registered rc%u (driver: %s)\n", dev->minor,
 		dev->driver_name ? dev->driver_name : "unknown");
 
diff --git a/drivers/media/rc/st_rc.c b/drivers/media/rc/st_rc.c
index d2efd7b2c3bc..c855b177103c 100644
--- a/drivers/media/rc/st_rc.c
+++ b/drivers/media/rc/st_rc.c
@@ -96,19 +96,24 @@ static void st_rc_send_lirc_timeout(struct rc_dev *rdev)
 
 static irqreturn_t st_rc_rx_interrupt(int irq, void *data)
 {
+	unsigned long timeout;
 	unsigned int symbol, mark = 0;
 	struct st_rc_device *dev = data;
 	int last_symbol = 0;
-	u32 status;
+	u32 status, int_status;
 	DEFINE_IR_RAW_EVENT(ev);
 
 	if (dev->irq_wake)
 		pm_wakeup_event(dev->dev, 0);
 
-	status  = readl(dev->rx_base + IRB_RX_STATUS);
+	/* FIXME: is 10ms good enough ? */
+	timeout = jiffies +  msecs_to_jiffies(10);
+	do {
+		status  = readl(dev->rx_base + IRB_RX_STATUS);
+		if (!(status & (IRB_FIFO_NOT_EMPTY | IRB_OVERFLOW)))
+			break;
 
-	while (status & (IRB_FIFO_NOT_EMPTY | IRB_OVERFLOW)) {
-		u32 int_status = readl(dev->rx_base + IRB_RX_INT_STATUS);
+		int_status = readl(dev->rx_base + IRB_RX_INT_STATUS);
 		if (unlikely(int_status & IRB_RX_OVERRUN_INT)) {
 			/* discard the entire collection in case of errors!  */
 			ir_raw_event_reset(dev->rdev);
@@ -148,8 +153,7 @@ static irqreturn_t st_rc_rx_interrupt(int irq, void *data)
 
 		}
 		last_symbol = 0;
-		status  = readl(dev->rx_base + IRB_RX_STATUS);
-	}
+	} while (time_is_after_jiffies(timeout));
 
 	writel(IRB_RX_INTS, dev->rx_base + IRB_RX_INT_CLEAR);
 
diff --git a/drivers/media/rc/winbond-cir.c b/drivers/media/rc/winbond-cir.c
index 0adf0991f5ab..851acba9b436 100644
--- a/drivers/media/rc/winbond-cir.c
+++ b/drivers/media/rc/winbond-cir.c
@@ -989,8 +989,7 @@ wbcir_init_hw(struct wbcir_data *data)
 
 	/* Clear RX state */
 	data->rxstate = WBCIR_RXSTATE_INACTIVE;
-	ir_raw_event_reset(data->dev);
-	ir_raw_event_set_idle(data->dev, true);
+	wbcir_idle_rx(data->dev, true);
 
 	/* Clear TX state */
 	if (data->txstate == WBCIR_TXSTATE_ACTIVE) {
@@ -1009,6 +1008,7 @@ wbcir_resume(struct pnp_dev *device)
 	struct wbcir_data *data = pnp_get_drvdata(device);
 
 	wbcir_init_hw(data);
+	ir_raw_event_reset(data->dev);
 	enable_irq(data->irq);
 	led_classdev_resume(&data->led);
 
diff --git a/drivers/media/spi/cxd2880-spi.c b/drivers/media/spi/cxd2880-spi.c
index 4df3bd312f48..11ce5101e19f 100644
--- a/drivers/media/spi/cxd2880-spi.c
+++ b/drivers/media/spi/cxd2880-spi.c
@@ -60,14 +60,13 @@ DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr);
 static int cxd2880_write_spi(struct spi_device *spi, u8 *data, u32 size)
 {
 	struct spi_message msg;
-	struct spi_transfer tx;
+	struct spi_transfer tx = {};
 
 	if (!spi || !data) {
 		pr_err("invalid arg\n");
 		return -EINVAL;
 	}
 
-	memset(&tx, 0, sizeof(tx));
 	tx.tx_buf = data;
 	tx.len = size;
 
@@ -88,7 +87,7 @@ static int cxd2880_write_reg(struct spi_device *spi,
 		pr_err("invalid arg\n");
 		return -EINVAL;
 	}
-	if (size > BURST_WRITE_MAX) {
+	if (size > BURST_WRITE_MAX || size > U8_MAX) {
 		pr_err("data size > WRITE_MAX\n");
 		return -EINVAL;
 	}
@@ -101,24 +100,14 @@ static int cxd2880_write_reg(struct spi_device *spi,
 	send_data[0] = 0x0e;
 	write_data_top = data;
 
-	while (size > 0) {
-		send_data[1] = sub_address;
-		if (size > 255)
-			send_data[2] = 255;
-		else
-			send_data[2] = (u8)size;
+	send_data[1] = sub_address;
+	send_data[2] = (u8)size;
 
-		memcpy(&send_data[3], write_data_top, send_data[2]);
+	memcpy(&send_data[3], write_data_top, send_data[2]);
 
-		ret = cxd2880_write_spi(spi, send_data, send_data[2] + 3);
-		if (ret) {
-			pr_err("write spi failed %d\n", ret);
-			break;
-		}
-		sub_address += send_data[2];
-		write_data_top += send_data[2];
-		size -= send_data[2];
-	}
+	ret = cxd2880_write_spi(spi, send_data, send_data[2] + 3);
+	if (ret)
+		pr_err("write spi failed %d\n", ret);
 
 	return ret;
 }
@@ -130,7 +119,7 @@ static int cxd2880_spi_read_ts(struct spi_device *spi,
 	int ret;
 	u8 data[3];
 	struct spi_message message;
-	struct spi_transfer transfer[2];
+	struct spi_transfer transfer[2] = {};
 
 	if (!spi || !read_data || !packet_num) {
 		pr_err("invalid arg\n");
@@ -146,7 +135,6 @@ static int cxd2880_spi_read_ts(struct spi_device *spi,
 	data[2] = packet_num;
 
 	spi_message_init(&message);
-	memset(transfer, 0, sizeof(transfer));
 
 	transfer[0].len = 3;
 	transfer[0].tx_buf = data;
@@ -383,7 +371,7 @@ static int cxd2880_start_feed(struct dvb_demux_feed *feed)
 			}
 		}
 		if (i == CXD2880_MAX_FILTER_SIZE) {
-			pr_err("PID filter is full. Assumed bug.\n");
+			pr_err("PID filter is full.\n");
 			return -EINVAL;
 		}
 		if (!dvb_spi->all_pid_feed_count)
diff --git a/drivers/media/tuners/Kconfig b/drivers/media/tuners/Kconfig
index 6687514df97f..147f3cd0bb95 100644
--- a/drivers/media/tuners/Kconfig
+++ b/drivers/media/tuners/Kconfig
@@ -284,4 +284,11 @@ config MEDIA_TUNER_QM1D1C0042
 	default m if !MEDIA_SUBDRV_AUTOSELECT
 	help
 	  Sharp QM1D1C0042 trellis coded 8PSK tuner driver.
+
+config MEDIA_TUNER_QM1D1B0004
+	tristate "Sharp QM1D1B0004 tuner"
+	depends on MEDIA_SUPPORT && I2C
+	default m if !MEDIA_SUBDRV_AUTOSELECT
+	help
+	  Sharp QM1D1B0004 ISDB-S tuner driver.
 endmenu
diff --git a/drivers/media/tuners/Makefile b/drivers/media/tuners/Makefile
index 0ff21f1c7eed..7b4f8423501e 100644
--- a/drivers/media/tuners/Makefile
+++ b/drivers/media/tuners/Makefile
@@ -41,6 +41,7 @@ obj-$(CONFIG_MEDIA_TUNER_IT913X) += it913x.o
 obj-$(CONFIG_MEDIA_TUNER_R820T) += r820t.o
 obj-$(CONFIG_MEDIA_TUNER_MXL301RF) += mxl301rf.o
 obj-$(CONFIG_MEDIA_TUNER_QM1D1C0042) += qm1d1c0042.o
+obj-$(CONFIG_MEDIA_TUNER_QM1D1B0004) += qm1d1b0004.o
 obj-$(CONFIG_MEDIA_TUNER_M88RS6000T) += m88rs6000t.o
 obj-$(CONFIG_MEDIA_TUNER_TDA18250) += tda18250.o
 
diff --git a/drivers/media/tuners/mxl301rf.c b/drivers/media/tuners/mxl301rf.c
index 1575a5db776a..57b0e4862aaf 100644
--- a/drivers/media/tuners/mxl301rf.c
+++ b/drivers/media/tuners/mxl301rf.c
@@ -1,17 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * MaxLinear MxL301RF OFDM tuner driver
  *
  * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /*
diff --git a/drivers/media/tuners/mxl301rf.h b/drivers/media/tuners/mxl301rf.h
index d32d4e8dc448..1d5bb10ba07a 100644
--- a/drivers/media/tuners/mxl301rf.h
+++ b/drivers/media/tuners/mxl301rf.h
@@ -1,17 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * MaxLinear MxL301RF OFDM tuner driver
  *
  * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #ifndef MXL301RF_H
diff --git a/drivers/media/tuners/qm1d1b0004.c b/drivers/media/tuners/qm1d1b0004.c
new file mode 100644
index 000000000000..b4495cc1626b
--- /dev/null
+++ b/drivers/media/tuners/qm1d1b0004.c
@@ -0,0 +1,266 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Sharp QM1D1B0004 satellite tuner
+ *
+ * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
+ *
+ * based on (former) drivers/media/pci/pt1/va1j5jf8007s.c.
+ */
+
+/*
+ * Note:
+ * Since the data-sheet of this tuner chip is not available,
+ * this driver lacks some tuner_ops and config options.
+ * In addition, the implementation might be dependent on the specific use
+ * in the FE module: VA1J5JF8007S and/or in the product: Earthsoft PT1/PT2.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <media/dvb_frontend.h>
+#include "qm1d1b0004.h"
+
+/*
+ * Tuner I/F (copied from the former va1j5jf8007s.c)
+ * b[0] I2C addr
+ * b[1] "0":1, BG:2, divider_quotient[7:3]:5
+ * b[2] divider_quotient[2:0]:3, divider_remainder:5
+ * b[3] "111":3, LPF[3:2]:2, TM:1, "0":1, REF:1
+ * b[4] BANDX, PSC:1, LPF[1:0]:2, DIV:1, "0":1
+ *
+ * PLL frequency step :=
+ *    REF == 0 -> PLL XTL frequency(4MHz) / 8
+ *    REF == 1 -> PLL XTL frequency(4MHz) / 4
+ *
+ * PreScaler :=
+ *    PSC == 0 -> x32
+ *    PSC == 1 -> x16
+ *
+ * divider_quotient := (frequency / PLL frequency step) / PreScaler
+ * divider_remainder := (frequency / PLL frequency step) % PreScaler
+ *
+ * LPF := LPF Frequency / 1000 / 2 - 2
+ * LPF Frequency @ baudrate=28.86Mbps = 30000
+ *
+ * band (1..9)
+ *   band 1 (freq <  986000) -> DIV:1, BANDX:5, PSC:1
+ *   band 2 (freq < 1072000) -> DIV:1, BANDX:6, PSC:1
+ *   band 3 (freq < 1154000) -> DIV:1, BANDX:7, PSC:0
+ *   band 4 (freq < 1291000) -> DIV:0, BANDX:1, PSC:0
+ *   band 5 (freq < 1447000) -> DIV:0, BANDX:2, PSC:0
+ *   band 6 (freq < 1615000) -> DIV:0, BANDX:3, PSC:0
+ *   band 7 (freq < 1791000) -> DIV:0, BANDX:4, PSC:0
+ *   band 8 (freq < 1972000) -> DIV:0, BANDX:5, PSC:0
+ *   band 9 (freq < 2150000) -> DIV:0, BANDX:6, PSC:0
+ */
+
+#define QM1D1B0004_PSC_MASK (1 << 4)
+
+#define QM1D1B0004_XTL_FREQ 4000
+#define QM1D1B0004_LPF_FALLBACK 30000
+
+#if 0 /* Currently unused */
+static const struct qm1d1b0004_config default_cfg = {
+	.lpf_freq = QM1D1B0004_CFG_LPF_DFLT,
+	.half_step = false,
+};
+#endif
+
+struct qm1d1b0004_state {
+	struct qm1d1b0004_config cfg;
+	struct i2c_client *i2c;
+};
+
+
+struct qm1d1b0004_cb_map {
+	u32 frequency;
+	u8 cb;
+};
+
+static const struct qm1d1b0004_cb_map cb_maps[] = {
+	{  986000, 0xb2 },
+	{ 1072000, 0xd2 },
+	{ 1154000, 0xe2 },
+	{ 1291000, 0x20 },
+	{ 1447000, 0x40 },
+	{ 1615000, 0x60 },
+	{ 1791000, 0x80 },
+	{ 1972000, 0xa0 },
+};
+
+static u8 lookup_cb(u32 frequency)
+{
+	int i;
+	const struct qm1d1b0004_cb_map *map;
+
+	for (i = 0; i < ARRAY_SIZE(cb_maps); i++) {
+		map = &cb_maps[i];
+		if (frequency < map->frequency)
+			return map->cb;
+	}
+	return 0xc0;
+}
+
+static int qm1d1b0004_set_params(struct dvb_frontend *fe)
+{
+	struct qm1d1b0004_state *state;
+	u32 frequency, pll, lpf_freq;
+	u16 word;
+	u8 buf[4], cb, lpf;
+	int ret;
+
+	state = fe->tuner_priv;
+	frequency = fe->dtv_property_cache.frequency;
+
+	pll = QM1D1B0004_XTL_FREQ / 4;
+	if (state->cfg.half_step)
+		pll /= 2;
+	word = DIV_ROUND_CLOSEST(frequency, pll);
+	cb = lookup_cb(frequency);
+	if (cb & QM1D1B0004_PSC_MASK)
+		word = (word << 1 & ~0x1f) | (word & 0x0f);
+
+	/* step.1: set frequency with BG:2, TM:0(4MHZ), LPF:4MHz */
+	buf[0] = 0x40 | word >> 8;
+	buf[1] = word;
+	/* inconsisnten with the above I/F doc. maybe the doc is wrong */
+	buf[2] = 0xe0 | state->cfg.half_step;
+	buf[3] = cb;
+	ret = i2c_master_send(state->i2c, buf, 4);
+	if (ret < 0)
+		return ret;
+
+	/* step.2: set TM:1 */
+	buf[0] = 0xe4 | state->cfg.half_step;
+	ret = i2c_master_send(state->i2c, buf, 1);
+	if (ret < 0)
+		return ret;
+	msleep(20);
+
+	/* step.3: set LPF */
+	lpf_freq = state->cfg.lpf_freq;
+	if (lpf_freq == QM1D1B0004_CFG_LPF_DFLT)
+		lpf_freq = fe->dtv_property_cache.symbol_rate / 1000;
+	if (lpf_freq == 0)
+		lpf_freq = QM1D1B0004_LPF_FALLBACK;
+	lpf = DIV_ROUND_UP(lpf_freq, 2000) - 2;
+	buf[0] = 0xe4 | ((lpf & 0x0c) << 1) | state->cfg.half_step;
+	buf[1] = cb | ((lpf & 0x03) << 2);
+	ret = i2c_master_send(state->i2c, buf, 2);
+	if (ret < 0)
+		return ret;
+
+	/* step.4: read PLL lock? */
+	buf[0] = 0;
+	ret = i2c_master_recv(state->i2c, buf, 1);
+	if (ret < 0)
+		return ret;
+	return 0;
+}
+
+
+static int qm1d1b0004_set_config(struct dvb_frontend *fe, void *priv_cfg)
+{
+	struct qm1d1b0004_state *state;
+
+	state = fe->tuner_priv;
+	memcpy(&state->cfg, priv_cfg, sizeof(state->cfg));
+	return 0;
+}
+
+
+static int qm1d1b0004_init(struct dvb_frontend *fe)
+{
+	struct qm1d1b0004_state *state;
+	u8 buf[2] = {0xf8, 0x04};
+
+	state = fe->tuner_priv;
+	if (state->cfg.half_step)
+		buf[0] |= 0x01;
+
+	return i2c_master_send(state->i2c, buf, 2);
+}
+
+
+static const struct dvb_tuner_ops qm1d1b0004_ops = {
+	.info = {
+		.name = "Sharp qm1d1b0004",
+
+		.frequency_min =  950000,
+		.frequency_max = 2150000,
+	},
+
+	.init = qm1d1b0004_init,
+
+	.set_params = qm1d1b0004_set_params,
+	.set_config = qm1d1b0004_set_config,
+};
+
+static int
+qm1d1b0004_probe(struct i2c_client *client, const struct i2c_device_id *id)
+{
+	struct dvb_frontend *fe;
+	struct qm1d1b0004_config *cfg;
+	struct qm1d1b0004_state *state;
+	int ret;
+
+	cfg = client->dev.platform_data;
+	fe = cfg->fe;
+	i2c_set_clientdata(client, fe);
+
+	fe->tuner_priv = kzalloc(sizeof(struct qm1d1b0004_state), GFP_KERNEL);
+	if (!fe->tuner_priv) {
+		ret = -ENOMEM;
+		goto err_mem;
+	}
+
+	memcpy(&fe->ops.tuner_ops, &qm1d1b0004_ops, sizeof(fe->ops.tuner_ops));
+
+	state = fe->tuner_priv;
+	state->i2c = client;
+	ret = qm1d1b0004_set_config(fe, cfg);
+	if (ret != 0)
+		goto err_priv;
+
+	dev_info(&client->dev, "Sharp QM1D1B0004 attached.\n");
+	return 0;
+
+err_priv:
+	kfree(fe->tuner_priv);
+err_mem:
+	fe->tuner_priv = NULL;
+	return ret;
+}
+
+static int qm1d1b0004_remove(struct i2c_client *client)
+{
+	struct dvb_frontend *fe;
+
+	fe = i2c_get_clientdata(client);
+	kfree(fe->tuner_priv);
+	fe->tuner_priv = NULL;
+	return 0;
+}
+
+
+static const struct i2c_device_id qm1d1b0004_id[] = {
+	{"qm1d1b0004", 0},
+	{}
+};
+
+MODULE_DEVICE_TABLE(i2c, qm1d1b0004_id);
+
+static struct i2c_driver qm1d1b0004_driver = {
+	.driver = {
+		.name = "qm1d1b0004",
+	},
+	.probe    = qm1d1b0004_probe,
+	.remove   = qm1d1b0004_remove,
+	.id_table = qm1d1b0004_id,
+};
+
+module_i2c_driver(qm1d1b0004_driver);
+
+MODULE_DESCRIPTION("Sharp QM1D1B0004");
+MODULE_AUTHOR("Akihiro Tsukada");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/tuners/qm1d1b0004.h b/drivers/media/tuners/qm1d1b0004.h
new file mode 100644
index 000000000000..7734ed109a22
--- /dev/null
+++ b/drivers/media/tuners/qm1d1b0004.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Sharp QM1D1B0004 satellite tuner
+ *
+ * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
+ */
+
+#ifndef QM1D1B0004_H
+#define QM1D1B0004_H
+
+#include <media/dvb_frontend.h>
+
+struct qm1d1b0004_config {
+	struct dvb_frontend *fe;
+
+	u32 lpf_freq;   /* LPF frequency[kHz]. Default: symbol rate */
+	bool half_step; /* use PLL frequency step of 500Hz istead of 1000Hz */
+};
+
+/* special values indicating to use the default in qm1d1b0004_config */
+#define QM1D1B0004_CFG_PLL_DFLT 0
+#define QM1D1B0004_CFG_LPF_DFLT 0
+
+#endif
diff --git a/drivers/media/tuners/qm1d1c0042.c b/drivers/media/tuners/qm1d1c0042.c
index 9af2a155cfca..642a065b9a07 100644
--- a/drivers/media/tuners/qm1d1c0042.c
+++ b/drivers/media/tuners/qm1d1c0042.c
@@ -1,17 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Sharp QM1D1C0042 8PSK tuner driver
  *
  * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 /*
diff --git a/drivers/media/tuners/qm1d1c0042.h b/drivers/media/tuners/qm1d1c0042.h
index 8331f8baa094..b22fa98bf56e 100644
--- a/drivers/media/tuners/qm1d1c0042.h
+++ b/drivers/media/tuners/qm1d1c0042.h
@@ -1,17 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Sharp QM1D1C0042 8PSK tuner driver
  *
  * Copyright (C) 2014 Akihiro Tsukada <tskd08@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #ifndef QM1D1C0042_H
diff --git a/drivers/media/usb/cx231xx/Kconfig b/drivers/media/usb/cx231xx/Kconfig
index 6276d9b2198b..0f13192634c7 100644
--- a/drivers/media/usb/cx231xx/Kconfig
+++ b/drivers/media/usb/cx231xx/Kconfig
@@ -1,13 +1,11 @@
 config VIDEO_CX231XX
 	tristate "Conexant cx231xx USB video capture support"
-	depends on VIDEO_DEV && I2C
+	depends on VIDEO_DEV && I2C && I2C_MUX
 	select VIDEO_TUNER
 	select VIDEO_TVEEPROM
-	depends on RC_CORE
 	select VIDEOBUF_VMALLOC
 	select VIDEO_CX25840
 	select VIDEO_CX2341X
-	select I2C_MUX
 
 	---help---
 	  This is a video4linux driver for Conexant 231xx USB based TV cards.
@@ -42,7 +40,6 @@ config VIDEO_CX231XX_ALSA
 config VIDEO_CX231XX_DVB
 	tristate "DVB/ATSC Support for Cx231xx based TV cards"
 	depends on VIDEO_CX231XX && DVB_CORE
-	select VIDEOBUF_DVB
 	select MEDIA_TUNER_XC5000 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_TDA18271 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_MB86A20S if MEDIA_SUBDRV_AUTOSELECT
diff --git a/drivers/media/usb/cx231xx/cx231xx-417.c b/drivers/media/usb/cx231xx/cx231xx-417.c
index b80e6857e2eb..2f3b0564d676 100644
--- a/drivers/media/usb/cx231xx/cx231xx-417.c
+++ b/drivers/media/usb/cx231xx/cx231xx-417.c
@@ -29,6 +29,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/firmware.h>
+#include <linux/slab.h>
 #include <linux/vmalloc.h>
 #include <media/v4l2-common.h>
 #include <media/v4l2-ioctl.h>
diff --git a/drivers/media/usb/cx231xx/cx231xx-cards.c b/drivers/media/usb/cx231xx/cx231xx-cards.c
index c76b2101193c..a431a998d58f 100644
--- a/drivers/media/usb/cx231xx/cx231xx-cards.c
+++ b/drivers/media/usb/cx231xx/cx231xx-cards.c
@@ -715,7 +715,7 @@ struct cx231xx_board cx231xx_boards[] = {
 		.tuner_i2c_master = I2C_1_MUX_3,
 		.demod_i2c_master = I2C_1_MUX_3,
 		.has_dvb = 1,
-		.demod_addr = 0x0e,
+		.demod_addr = 0x64, /* 0xc8 >> 1 */
 		.norm = V4L2_STD_PAL,
 
 		.input = {{
@@ -754,7 +754,7 @@ struct cx231xx_board cx231xx_boards[] = {
 		.tuner_i2c_master = I2C_1_MUX_3,
 		.demod_i2c_master = I2C_1_MUX_3,
 		.has_dvb = 1,
-		.demod_addr = 0x0e,
+		.demod_addr = 0x64, /* 0xc8 >> 1 */
 		.norm = V4L2_STD_PAL,
 
 		.input = {{
@@ -793,7 +793,7 @@ struct cx231xx_board cx231xx_boards[] = {
 		.tuner_i2c_master = I2C_1_MUX_3,
 		.demod_i2c_master = I2C_1_MUX_3,
 		.has_dvb = 1,
-		.demod_addr = 0x0e,
+		.demod_addr = 0x59, /* 0xb2 >> 1 */
 		.norm = V4L2_STD_NTSC,
 
 		.input = {{
@@ -1024,6 +1024,9 @@ struct usb_device_id cx231xx_id_table[] = {
 	 .driver_info = CX231XX_BOARD_CNXT_RDE_250},
 	{USB_DEVICE(0x0572, 0x58A0),
 	 .driver_info = CX231XX_BOARD_CNXT_RDU_250},
+	/* AverMedia DVD EZMaker 7 */
+	{USB_DEVICE(0x07ca, 0xc039),
+	 .driver_info = CX231XX_BOARD_CNXT_VIDEO_GRABBER},
 	{USB_DEVICE(0x2040, 0xb110),
 	 .driver_info = CX231XX_BOARD_HAUPPAUGE_USB2_FM_PAL},
 	{USB_DEVICE(0x2040, 0xb111),
diff --git a/drivers/media/usb/cx231xx/cx231xx-dvb.c b/drivers/media/usb/cx231xx/cx231xx-dvb.c
index 67ed66712d05..89357cb08b1a 100644
--- a/drivers/media/usb/cx231xx/cx231xx-dvb.c
+++ b/drivers/media/usb/cx231xx/cx231xx-dvb.c
@@ -23,8 +23,12 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 
+#include <media/dvbdev.h>
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_net.h>
+#include <media/dvb_frontend.h>
 #include <media/v4l2-common.h>
-#include <media/videobuf-vmalloc.h>
 #include <media/tuner.h>
 
 #include "xc5000.h"
@@ -156,10 +160,8 @@ static struct tda18271_config pv_tda18271_config = {
 };
 
 static struct lgdt3306a_config hauppauge_955q_lgdt3306a_config = {
-	.i2c_addr           = 0x59,
 	.qam_if_khz         = 4000,
 	.vsb_if_khz         = 3250,
-	.deny_i2c_rptr      = 1,
 	.spectral_inversion = 1,
 	.mpeg_mode          = LGDT3306A_MPEG_SERIAL,
 	.tpclk_edge         = LGDT3306A_TPCLK_RISING_EDGE,
@@ -600,7 +602,6 @@ static int register_dvb(struct cx231xx_dvb *dvb,
 
 static void unregister_dvb(struct cx231xx_dvb *dvb)
 {
-	struct i2c_client *client;
 	dvb_net_release(&dvb->net);
 	dvb->demux.dmx.remove_frontend(&dvb->demux.dmx, &dvb->fe_mem);
 	dvb->demux.dmx.remove_frontend(&dvb->demux.dmx, &dvb->fe_hw);
@@ -613,23 +614,15 @@ static void unregister_dvb(struct cx231xx_dvb *dvb)
 		dvb_frontend_detach(dvb->frontend[1]);
 	dvb_frontend_detach(dvb->frontend[0]);
 	dvb_unregister_adapter(&dvb->adapter);
+
 	/* remove I2C tuner */
-	client = dvb->i2c_client_tuner;
-	if (client) {
-		module_put(client->dev.driver->owner);
-		i2c_unregister_device(client);
-	}
-	/* remove I2C demod */
-	client = dvb->i2c_client_demod[1];
-	if (client) {
-		module_put(client->dev.driver->owner);
-		i2c_unregister_device(client);
-	}
-	client = dvb->i2c_client_demod[0];
-	if (client) {
-		module_put(client->dev.driver->owner);
-		i2c_unregister_device(client);
-	}
+	dvb_module_release(dvb->i2c_client_tuner);
+	dvb->i2c_client_tuner = NULL;
+	/* remove I2C demod(s) */
+	dvb_module_release(dvb->i2c_client_demod[1]);
+	dvb->i2c_client_demod[1] = NULL;
+	dvb_module_release(dvb->i2c_client_demod[0]);
+	dvb->i2c_client_demod[0] = NULL;
 }
 
 static int dvb_init(struct cx231xx *dev)
@@ -638,6 +631,8 @@ static int dvb_init(struct cx231xx *dev)
 	struct cx231xx_dvb *dvb;
 	struct i2c_adapter *tuner_i2c;
 	struct i2c_adapter *demod_i2c;
+	struct i2c_client *client;
+	struct i2c_adapter *adapter;
 
 	if (!dev->board.has_dvb) {
 		/* This device does not support the extension */
@@ -728,7 +723,7 @@ static int dvb_init(struct cx231xx *dev)
 		dvb->frontend[0]->callback = cx231xx_tuner_callback;
 
 		if (!dvb_attach(tda18271_attach, dev->dvb->frontend[0],
-			       0x60, tuner_i2c,
+			       dev->board.tuner_addr, tuner_i2c,
 			       &cnxt_rde253s_tunerconfig)) {
 			result = -EINVAL;
 			goto out_free;
@@ -752,7 +747,7 @@ static int dvb_init(struct cx231xx *dev)
 		dvb->frontend[0]->callback = cx231xx_tuner_callback;
 
 		if (!dvb_attach(tda18271_attach, dev->dvb->frontend[0],
-			       0x60, tuner_i2c,
+			       dev->board.tuner_addr, tuner_i2c,
 			       &cnxt_rde253s_tunerconfig)) {
 			result = -EINVAL;
 			goto out_free;
@@ -779,41 +774,27 @@ static int dvb_init(struct cx231xx *dev)
 		dvb->frontend[0]->callback = cx231xx_tuner_callback;
 
 		dvb_attach(tda18271_attach, dev->dvb->frontend[0],
-			   0x60, tuner_i2c,
+			   dev->board.tuner_addr, tuner_i2c,
 			   &hcw_tda18271_config);
 		break;
 
 	case CX231XX_BOARD_HAUPPAUGE_930C_HD_1113xx:
 	{
-		struct i2c_client *client;
-		struct i2c_board_info info;
-		struct si2165_platform_data si2165_pdata;
+		struct si2165_platform_data si2165_pdata = {};
 
 		/* attach demod */
-		memset(&si2165_pdata, 0, sizeof(si2165_pdata));
 		si2165_pdata.fe = &dev->dvb->frontend[0];
 		si2165_pdata.chip_mode = SI2165_MODE_PLL_XTAL;
 		si2165_pdata.ref_freq_hz = 16000000;
 
-		memset(&info, 0, sizeof(struct i2c_board_info));
-		strlcpy(info.type, "si2165", I2C_NAME_SIZE);
-		info.addr = 0x64;
-		info.platform_data = &si2165_pdata;
-		request_module(info.type);
-		client = i2c_new_device(demod_i2c, &info);
-		if (!client || !client->dev.driver || !dev->dvb->frontend[0]) {
-			dev_err(dev->dev,
-				"Failed to attach SI2165 front end\n");
-			result = -EINVAL;
-			goto out_free;
-		}
-
-		if (!try_module_get(client->dev.driver->owner)) {
-			i2c_unregister_device(client);
+		/* perform probe/init/attach */
+		client = dvb_module_probe("si2165", NULL, demod_i2c,
+						dev->board.demod_addr,
+						&si2165_pdata);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
 		dvb->i2c_client_demod[0] = client;
 
 		dev->dvb->frontend[0]->ops.i2c_gate_ctrl = NULL;
@@ -822,8 +803,7 @@ static int dvb_init(struct cx231xx *dev)
 		dvb->frontend[0]->callback = cx231xx_tuner_callback;
 
 		dvb_attach(tda18271_attach, dev->dvb->frontend[0],
-			0x60,
-			tuner_i2c,
+			dev->board.tuner_addr, tuner_i2c,
 			&hcw_tda18271_config);
 
 		dev->cx231xx_reset_analog_tuner = NULL;
@@ -831,74 +811,45 @@ static int dvb_init(struct cx231xx *dev)
 	}
 	case CX231XX_BOARD_HAUPPAUGE_930C_HD_1114xx:
 	{
-		struct i2c_client *client;
-		struct i2c_board_info info;
-		struct si2165_platform_data si2165_pdata;
-		struct si2157_config si2157_config;
+		struct si2165_platform_data si2165_pdata = {};
+		struct si2157_config si2157_config = {};
 
 		/* attach demod */
-		memset(&si2165_pdata, 0, sizeof(si2165_pdata));
 		si2165_pdata.fe = &dev->dvb->frontend[0];
 		si2165_pdata.chip_mode = SI2165_MODE_PLL_EXT;
 		si2165_pdata.ref_freq_hz = 24000000;
 
-		memset(&info, 0, sizeof(struct i2c_board_info));
-		strlcpy(info.type, "si2165", I2C_NAME_SIZE);
-		info.addr = 0x64;
-		info.platform_data = &si2165_pdata;
-		request_module(info.type);
-		client = i2c_new_device(demod_i2c, &info);
-		if (!client || !client->dev.driver || !dev->dvb->frontend[0]) {
-			dev_err(dev->dev,
-				"Failed to attach SI2165 front end\n");
-			result = -EINVAL;
-			goto out_free;
-		}
-
-		if (!try_module_get(client->dev.driver->owner)) {
-			i2c_unregister_device(client);
+		/* perform probe/init/attach */
+		client = dvb_module_probe("si2165", NULL, demod_i2c,
+						dev->board.demod_addr,
+						&si2165_pdata);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
 		dvb->i2c_client_demod[0] = client;
 
-		memset(&info, 0, sizeof(struct i2c_board_info));
-
 		dev->dvb->frontend[0]->ops.i2c_gate_ctrl = NULL;
 
 		/* define general-purpose callback pointer */
 		dvb->frontend[0]->callback = cx231xx_tuner_callback;
 
 		/* attach tuner */
-		memset(&si2157_config, 0, sizeof(si2157_config));
 		si2157_config.fe = dev->dvb->frontend[0];
 #ifdef CONFIG_MEDIA_CONTROLLER_DVB
 		si2157_config.mdev = dev->media_dev;
 #endif
 		si2157_config.if_port = 1;
 		si2157_config.inversion = true;
-		strlcpy(info.type, "si2157", I2C_NAME_SIZE);
-		info.addr = 0x60;
-		info.platform_data = &si2157_config;
-		request_module("si2157");
-
-		client = i2c_new_device(
-			tuner_i2c,
-			&info);
-		if (client == NULL || client->dev.driver == NULL) {
-			dvb_frontend_detach(dev->dvb->frontend[0]);
-			result = -ENODEV;
-			goto out_free;
-		}
 
-		if (!try_module_get(client->dev.driver->owner)) {
-			i2c_unregister_device(client);
-			dvb_frontend_detach(dev->dvb->frontend[0]);
+		/* perform probe/init/attach */
+		client = dvb_module_probe("si2157", NULL, tuner_i2c,
+						dev->board.tuner_addr,
+						&si2157_config);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
 		dev->cx231xx_reset_analog_tuner = NULL;
 
 		dev->dvb->i2c_client_tuner = client;
@@ -906,23 +857,22 @@ static int dvb_init(struct cx231xx *dev)
 	}
 	case CX231XX_BOARD_HAUPPAUGE_955Q:
 	{
-		struct i2c_client *client;
-		struct i2c_board_info info;
-		struct si2157_config si2157_config;
-
-		memset(&info, 0, sizeof(struct i2c_board_info));
+		struct si2157_config si2157_config = {};
+		struct lgdt3306a_config lgdt3306a_config = {};
 
-		dev->dvb->frontend[0] = dvb_attach(lgdt3306a_attach,
-			&hauppauge_955q_lgdt3306a_config,
-			demod_i2c
-			);
+		lgdt3306a_config = hauppauge_955q_lgdt3306a_config;
+		lgdt3306a_config.fe = &dev->dvb->frontend[0];
+		lgdt3306a_config.i2c_adapter = &adapter;
 
-		if (!dev->dvb->frontend[0]) {
-			dev_err(dev->dev,
-				"Failed to attach LGDT3306A frontend.\n");
-			result = -EINVAL;
+		/* perform probe/init/attach */
+		client = dvb_module_probe("lgdt3306a", NULL, demod_i2c,
+						dev->board.demod_addr,
+						&lgdt3306a_config);
+		if (!client) {
+			result = -ENODEV;
 			goto out_free;
 		}
+		dvb->i2c_client_demod[0] = client;
 
 		dev->dvb->frontend[0]->ops.i2c_gate_ctrl = NULL;
 
@@ -930,34 +880,21 @@ static int dvb_init(struct cx231xx *dev)
 		dvb->frontend[0]->callback = cx231xx_tuner_callback;
 
 		/* attach tuner */
-		memset(&si2157_config, 0, sizeof(si2157_config));
 		si2157_config.fe = dev->dvb->frontend[0];
 #ifdef CONFIG_MEDIA_CONTROLLER_DVB
 		si2157_config.mdev = dev->media_dev;
 #endif
 		si2157_config.if_port = 1;
 		si2157_config.inversion = true;
-		strlcpy(info.type, "si2157", I2C_NAME_SIZE);
-		info.addr = 0x60;
-		info.platform_data = &si2157_config;
-		request_module("si2157");
-
-		client = i2c_new_device(
-			tuner_i2c,
-			&info);
-		if (client == NULL || client->dev.driver == NULL) {
-			dvb_frontend_detach(dev->dvb->frontend[0]);
-			result = -ENODEV;
-			goto out_free;
-		}
 
-		if (!try_module_get(client->dev.driver->owner)) {
-			i2c_unregister_device(client);
-			dvb_frontend_detach(dev->dvb->frontend[0]);
+		/* perform probe/init/attach */
+		client = dvb_module_probe("si2157", NULL, tuner_i2c,
+						dev->board.tuner_addr,
+						&si2157_config);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
 		dev->cx231xx_reset_analog_tuner = NULL;
 
 		dev->dvb->i2c_client_tuner = client;
@@ -985,7 +922,7 @@ static int dvb_init(struct cx231xx *dev)
 		dvb->frontend[0]->callback = cx231xx_tuner_callback;
 
 		dvb_attach(tda18271_attach, dev->dvb->frontend[0],
-			   0x60, tuner_i2c,
+			   dev->board.tuner_addr, tuner_i2c,
 			   &pv_tda18271_config);
 		break;
 
@@ -993,9 +930,6 @@ static int dvb_init(struct cx231xx *dev)
 	{
 		struct si2157_config si2157_config = {};
 		struct si2168_config si2168_config = {};
-		struct i2c_board_info info = {};
-		struct i2c_client *client;
-		struct i2c_adapter *adapter;
 
 		/* attach demodulator chip */
 		si2168_config.ts_mode = SI2168_TS_SERIAL; /* from *.inf file */
@@ -1003,24 +937,14 @@ static int dvb_init(struct cx231xx *dev)
 		si2168_config.i2c_adapter = &adapter;
 		si2168_config.ts_clock_inv = true;
 
-		strlcpy(info.type, "si2168", sizeof(info.type));
-		info.addr = dev->board.demod_addr;
-		info.platform_data = &si2168_config;
-
-		request_module(info.type);
-		client = i2c_new_device(demod_i2c, &info);
-
-		if (client == NULL || client->dev.driver == NULL) {
+		/* perform probe/init/attach */
+		client = dvb_module_probe("si2168", NULL, demod_i2c,
+						dev->board.demod_addr,
+						&si2168_config);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
-		if (!try_module_get(client->dev.driver->owner)) {
-			i2c_unregister_device(client);
-			result = -ENODEV;
-			goto out_free;
-		}
-
 		dvb->i2c_client_demod[0] = client;
 
 		/* attach tuner chip */
@@ -1031,37 +955,20 @@ static int dvb_init(struct cx231xx *dev)
 		si2157_config.if_port = 1;
 		si2157_config.inversion = false;
 
-		memset(&info, 0, sizeof(info));
-		strlcpy(info.type, "si2157", sizeof(info.type));
-		info.addr = dev->board.tuner_addr;
-		info.platform_data = &si2157_config;
-
-		request_module(info.type);
-		client = i2c_new_device(tuner_i2c, &info);
-
-		if (client == NULL || client->dev.driver == NULL) {
-			module_put(dvb->i2c_client_demod[0]->dev.driver->owner);
-			i2c_unregister_device(dvb->i2c_client_demod[0]);
-			result = -ENODEV;
-			goto out_free;
-		}
-
-		if (!try_module_get(client->dev.driver->owner)) {
-			i2c_unregister_device(client);
-			module_put(dvb->i2c_client_demod[0]->dev.driver->owner);
-			i2c_unregister_device(dvb->i2c_client_demod[0]);
+		/* perform probe/init/attach */
+		client = dvb_module_probe("si2157", NULL, tuner_i2c,
+						dev->board.tuner_addr,
+						&si2157_config);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
 		dev->cx231xx_reset_analog_tuner = NULL;
 		dev->dvb->i2c_client_tuner = client;
 		break;
 	}
 	case CX231XX_BOARD_ASTROMETA_T2HYBRID:
 	{
-		struct i2c_client *client;
-		struct i2c_board_info info = {};
 		struct mn88473_config mn88473_config = {};
 
 		/* attach demodulator chip */
@@ -1069,24 +976,14 @@ static int dvb_init(struct cx231xx *dev)
 		mn88473_config.xtal = 25000000;
 		mn88473_config.fe = &dev->dvb->frontend[0];
 
-		strlcpy(info.type, "mn88473", sizeof(info.type));
-		info.addr = dev->board.demod_addr;
-		info.platform_data = &mn88473_config;
-
-		request_module(info.type);
-		client = i2c_new_device(demod_i2c, &info);
-
-		if (client == NULL || client->dev.driver == NULL) {
-			result = -ENODEV;
-			goto out_free;
-		}
-
-		if (!try_module_get(client->dev.driver->owner)) {
-			i2c_unregister_device(client);
+		/* perform probe/init/attach */
+		client = dvb_module_probe("mn88473", NULL, demod_i2c,
+						dev->board.demod_addr,
+						&mn88473_config);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
 		dvb->i2c_client_demod[0] = client;
 
 		/* define general-purpose callback pointer */
@@ -1100,9 +997,6 @@ static int dvb_init(struct cx231xx *dev)
 	}
 	case CX231XX_BOARD_HAUPPAUGE_935C:
 	{
-		struct i2c_client *client;
-		struct i2c_adapter *adapter;
-		struct i2c_board_info info = {};
 		struct si2157_config si2157_config = {};
 		struct si2168_config si2168_config = {};
 
@@ -1112,25 +1006,14 @@ static int dvb_init(struct cx231xx *dev)
 		si2168_config.i2c_adapter = &adapter;
 		si2168_config.ts_clock_inv = true;
 
-		strlcpy(info.type, "si2168", sizeof(info.type));
-		info.addr = dev->board.demod_addr;
-		info.platform_data = &si2168_config;
-
-		request_module(info.type);
-		client = i2c_new_device(demod_i2c, &info);
-		if (client == NULL || client->dev.driver == NULL) {
-			result = -ENODEV;
-			goto out_free;
-		}
-
-		if (!try_module_get(client->dev.driver->owner)) {
-			dev_err(dev->dev,
-				"Failed to attach %s frontend.\n", info.type);
-			i2c_unregister_device(client);
+		/* perform probe/init/attach */
+		client = dvb_module_probe("si2168", NULL, demod_i2c,
+						dev->board.demod_addr,
+						&si2168_config);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
 		dvb->i2c_client_demod[0] = client;
 		dev->dvb->frontend[0]->ops.i2c_gate_ctrl = NULL;
 
@@ -1145,40 +1028,21 @@ static int dvb_init(struct cx231xx *dev)
 		si2157_config.if_port = 1;
 		si2157_config.inversion = true;
 
-		memset(&info, 0, sizeof(struct i2c_board_info));
-		strlcpy(info.type, "si2157", I2C_NAME_SIZE);
-		info.addr = dev->board.tuner_addr;
-		info.platform_data = &si2157_config;
-		request_module("si2157");
-
-		client = i2c_new_device(adapter, &info);
-		if (client == NULL || client->dev.driver == NULL) {
-			module_put(dvb->i2c_client_demod[0]->dev.driver->owner);
-			i2c_unregister_device(dvb->i2c_client_demod[0]);
-			result = -ENODEV;
-			goto out_free;
-		}
-
-		if (!try_module_get(client->dev.driver->owner)) {
-			dev_err(dev->dev,
-				"Failed to obtain %s tuner.\n",	info.type);
-			i2c_unregister_device(client);
-			module_put(dvb->i2c_client_demod[0]->dev.driver->owner);
-			i2c_unregister_device(dvb->i2c_client_demod[0]);
+		/* perform probe/init/attach */
+		client = dvb_module_probe("si2157", NULL, tuner_i2c,
+						dev->board.tuner_addr,
+						&si2157_config);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
 		dev->cx231xx_reset_analog_tuner = NULL;
 		dev->dvb->i2c_client_tuner = client;
 		break;
 	}
 	case CX231XX_BOARD_HAUPPAUGE_975:
 	{
-		struct i2c_client *client;
-		struct i2c_adapter *adapter;
 		struct i2c_adapter *adapter2;
-		struct i2c_board_info info = {};
 		struct si2157_config si2157_config = {};
 		struct lgdt3306a_config lgdt3306a_config = {};
 		struct si2168_config si2168_config = {};
@@ -1187,27 +1051,15 @@ static int dvb_init(struct cx231xx *dev)
 		lgdt3306a_config = hauppauge_955q_lgdt3306a_config;
 		lgdt3306a_config.fe = &dev->dvb->frontend[0];
 		lgdt3306a_config.i2c_adapter = &adapter;
-		lgdt3306a_config.deny_i2c_rptr = 0;
-
-		strlcpy(info.type, "lgdt3306a", sizeof(info.type));
-		info.addr = dev->board.demod_addr;
-		info.platform_data = &lgdt3306a_config;
-
-		request_module(info.type);
-		client = i2c_new_device(demod_i2c, &info);
-		if (client == NULL || client->dev.driver == NULL) {
-			result = -ENODEV;
-			goto out_free;
-		}
 
-		if (!try_module_get(client->dev.driver->owner)) {
-			dev_err(dev->dev,
-				"Failed to attach %s frontend.\n", info.type);
-			i2c_unregister_device(client);
+		/* perform probe/init/attach */
+		client = dvb_module_probe("lgdt3306a", NULL, demod_i2c,
+						dev->board.demod_addr,
+						&lgdt3306a_config);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
 		dvb->i2c_client_demod[0] = client;
 
 		/* attach second demodulator chip */
@@ -1216,30 +1068,14 @@ static int dvb_init(struct cx231xx *dev)
 		si2168_config.i2c_adapter = &adapter2;
 		si2168_config.ts_clock_inv = true;
 
-		memset(&info, 0, sizeof(struct i2c_board_info));
-		strlcpy(info.type, "si2168", sizeof(info.type));
-		info.addr = dev->board.demod_addr2;
-		info.platform_data = &si2168_config;
-
-		request_module(info.type);
-		client = i2c_new_device(adapter, &info);
-		if (client == NULL || client->dev.driver == NULL) {
-			dev_err(dev->dev,
-				"Failed to attach %s frontend.\n", info.type);
-			module_put(dvb->i2c_client_demod[0]->dev.driver->owner);
-			i2c_unregister_device(dvb->i2c_client_demod[0]);
-			result = -ENODEV;
-			goto out_free;
-		}
-
-		if (!try_module_get(client->dev.driver->owner)) {
-			i2c_unregister_device(client);
-			module_put(dvb->i2c_client_demod[0]->dev.driver->owner);
-			i2c_unregister_device(dvb->i2c_client_demod[0]);
+		/* perform probe/init/attach */
+		client = dvb_module_probe("si2168", NULL, adapter,
+						dev->board.demod_addr2,
+						&si2168_config);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
 		dvb->i2c_client_demod[1] = client;
 		dvb->frontend[1]->id = 1;
 
@@ -1255,34 +1091,14 @@ static int dvb_init(struct cx231xx *dev)
 		si2157_config.if_port = 1;
 		si2157_config.inversion = true;
 
-		memset(&info, 0, sizeof(struct i2c_board_info));
-		strlcpy(info.type, "si2157", I2C_NAME_SIZE);
-		info.addr = dev->board.tuner_addr;
-		info.platform_data = &si2157_config;
-		request_module("si2157");
-
-		client = i2c_new_device(adapter, &info);
-		if (client == NULL || client->dev.driver == NULL) {
-			module_put(dvb->i2c_client_demod[1]->dev.driver->owner);
-			i2c_unregister_device(dvb->i2c_client_demod[1]);
-			module_put(dvb->i2c_client_demod[0]->dev.driver->owner);
-			i2c_unregister_device(dvb->i2c_client_demod[0]);
-			result = -ENODEV;
-			goto out_free;
-		}
-
-		if (!try_module_get(client->dev.driver->owner)) {
-			dev_err(dev->dev,
-				"Failed to obtain %s tuner.\n",	info.type);
-			i2c_unregister_device(client);
-			module_put(dvb->i2c_client_demod[1]->dev.driver->owner);
-			i2c_unregister_device(dvb->i2c_client_demod[1]);
-			module_put(dvb->i2c_client_demod[0]->dev.driver->owner);
-			i2c_unregister_device(dvb->i2c_client_demod[0]);
+		/* perform probe/init/attach */
+		client = dvb_module_probe("si2157", NULL, adapter,
+						dev->board.tuner_addr,
+						&si2157_config);
+		if (!client) {
 			result = -ENODEV;
 			goto out_free;
 		}
-
 		dev->cx231xx_reset_analog_tuner = NULL;
 		dvb->i2c_client_tuner = client;
 
@@ -1321,6 +1137,14 @@ static int dvb_init(struct cx231xx *dev)
 	return result;
 
 out_free:
+	/* remove I2C tuner */
+	dvb_module_release(dvb->i2c_client_tuner);
+	dvb->i2c_client_tuner = NULL;
+	/* remove I2C demod(s) */
+	dvb_module_release(dvb->i2c_client_demod[1]);
+	dvb->i2c_client_demod[1] = NULL;
+	dvb_module_release(dvb->i2c_client_demod[0]);
+	dvb->i2c_client_demod[0] = NULL;
 	kfree(dvb);
 	dev->dvb = NULL;
 	goto ret;
diff --git a/drivers/media/usb/cx231xx/cx231xx-pcb-cfg.c b/drivers/media/usb/cx231xx/cx231xx-pcb-cfg.c
index 5bc74149fcb9..746c34ab0ec8 100644
--- a/drivers/media/usb/cx231xx/cx231xx-pcb-cfg.c
+++ b/drivers/media/usb/cx231xx/cx231xx-pcb-cfg.c
@@ -769,7 +769,7 @@ int initialize_cx231xx(struct cx231xx *dev)
 			break;
 		default:
 			dev_err(dev->dev,
-				"bad senario!!!!!\nconfig_info=%x\n",
+				"bad scenario!!!!!\nconfig_info=%x\n",
 				config_info & SELFPOWER_MASK);
 			return -ENODEV;
 		}
diff --git a/drivers/media/usb/cx231xx/cx231xx.h b/drivers/media/usb/cx231xx/cx231xx.h
index 6ffa4bd96484..fa640bf20111 100644
--- a/drivers/media/usb/cx231xx/cx231xx.h
+++ b/drivers/media/usb/cx231xx/cx231xx.h
@@ -38,7 +38,6 @@
 #include <media/v4l2-fh.h>
 #include <media/rc-core.h>
 #include <media/i2c/ir-kbd-i2c.h>
-#include <media/videobuf-dvb.h>
 
 #include "cx231xx-reg.h"
 #include "cx231xx-pcb-cfg.h"
@@ -546,8 +545,6 @@ struct cx231xx_tsport {
 	int                        nr;
 	int                        sram_chno;
 
-	struct videobuf_dvb_frontends frontends;
-
 	/* dma queues */
 
 	u32                        ts_packet_size;
diff --git a/drivers/media/usb/dvb-usb-v2/dvbsky.c b/drivers/media/usb/dvb-usb-v2/dvbsky.c
index 43eb82884555..1aa88d94e57f 100644
--- a/drivers/media/usb/dvb-usb-v2/dvbsky.c
+++ b/drivers/media/usb/dvb-usb-v2/dvbsky.c
@@ -31,7 +31,6 @@ MODULE_PARM_DESC(disable_rc, "Disable inbuilt IR receiver.");
 DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr);
 
 struct dvbsky_state {
-	struct mutex stream_mutex;
 	u8 ibuf[DVBSKY_BUF_LEN];
 	u8 obuf[DVBSKY_BUF_LEN];
 	u8 last_lock;
@@ -68,18 +67,17 @@ static int dvbsky_usb_generic_rw(struct dvb_usb_device *d,
 
 static int dvbsky_stream_ctrl(struct dvb_usb_device *d, u8 onoff)
 {
-	struct dvbsky_state *state = d_to_priv(d);
 	int ret;
-	u8 obuf_pre[3] = { 0x37, 0, 0 };
-	u8 obuf_post[3] = { 0x36, 3, 0 };
+	static u8 obuf_pre[3] = { 0x37, 0, 0 };
+	static u8 obuf_post[3] = { 0x36, 3, 0 };
 
-	mutex_lock(&state->stream_mutex);
-	ret = dvbsky_usb_generic_rw(d, obuf_pre, 3, NULL, 0);
+	mutex_lock(&d->usb_mutex);
+	ret = dvb_usbv2_generic_rw_locked(d, obuf_pre, 3, NULL, 0);
 	if (!ret && onoff) {
 		msleep(20);
-		ret = dvbsky_usb_generic_rw(d, obuf_post, 3, NULL, 0);
+		ret = dvb_usbv2_generic_rw_locked(d, obuf_post, 3, NULL, 0);
 	}
-	mutex_unlock(&state->stream_mutex);
+	mutex_unlock(&d->usb_mutex);
 	return ret;
 }
 
@@ -290,61 +288,44 @@ static int dvbsky_usb_read_status(struct dvb_frontend *fe,
 	return ret;
 }
 
-static const struct m88ds3103_config dvbsky_s960_m88ds3103_config = {
-	.i2c_addr = 0x68,
-	.clock = 27000000,
-	.i2c_wr_max = 33,
-	.clock_out = 0,
-	.ts_mode = M88DS3103_TS_CI,
-	.ts_clk = 16000,
-	.ts_clk_pol = 0,
-	.agc = 0x99,
-	.lnb_hv_pol = 1,
-	.lnb_en_pol = 1,
-};
-
 static int dvbsky_s960_attach(struct dvb_usb_adapter *adap)
 {
 	struct dvbsky_state *state = adap_to_priv(adap);
 	struct dvb_usb_device *d = adap_to_d(adap);
-	int ret = 0;
-	/* demod I2C adapter */
-	struct i2c_adapter *i2c_adapter = NULL;
-	struct i2c_client *client;
-	struct i2c_board_info info;
+	struct i2c_adapter *i2c_adapter;
+	struct m88ds3103_platform_data m88ds3103_pdata = {};
 	struct ts2020_config ts2020_config = {};
-	memset(&info, 0, sizeof(struct i2c_board_info));
 
 	/* attach demod */
-	adap->fe[0] = dvb_attach(m88ds3103_attach,
-			&dvbsky_s960_m88ds3103_config,
-			&d->i2c_adap,
-			&i2c_adapter);
-	if (!adap->fe[0]) {
-		dev_err(&d->udev->dev, "dvbsky_s960_attach fail.\n");
-		ret = -ENODEV;
-		goto fail_attach;
-	}
+	m88ds3103_pdata.clk = 27000000;
+	m88ds3103_pdata.i2c_wr_max = 33;
+	m88ds3103_pdata.clk_out = 0;
+	m88ds3103_pdata.ts_mode = M88DS3103_TS_CI;
+	m88ds3103_pdata.ts_clk = 16000;
+	m88ds3103_pdata.ts_clk_pol = 0;
+	m88ds3103_pdata.agc = 0x99;
+	m88ds3103_pdata.lnb_hv_pol = 1,
+	m88ds3103_pdata.lnb_en_pol = 1,
+
+	state->i2c_client_demod = dvb_module_probe("m88ds3103", NULL,
+						   &d->i2c_adap,
+						   0x68, &m88ds3103_pdata);
+	if (!state->i2c_client_demod)
+		return -ENODEV;
+
+	adap->fe[0] = m88ds3103_pdata.get_dvb_frontend(state->i2c_client_demod);
+	i2c_adapter = m88ds3103_pdata.get_i2c_adapter(state->i2c_client_demod);
 
 	/* attach tuner */
 	ts2020_config.fe = adap->fe[0];
 	ts2020_config.get_agc_pwm = m88ds3103_get_agc_pwm;
-	strlcpy(info.type, "ts2020", I2C_NAME_SIZE);
-	info.addr = 0x60;
-	info.platform_data = &ts2020_config;
-	request_module("ts2020");
-	client = i2c_new_device(i2c_adapter, &info);
-	if (client == NULL || client->dev.driver == NULL) {
-		dvb_frontend_detach(adap->fe[0]);
-		ret = -ENODEV;
-		goto fail_attach;
-	}
 
-	if (!try_module_get(client->dev.driver->owner)) {
-		i2c_unregister_device(client);
-		dvb_frontend_detach(adap->fe[0]);
-		ret = -ENODEV;
-		goto fail_attach;
+	state->i2c_client_tuner = dvb_module_probe("ts2020", NULL,
+						   i2c_adapter,
+						   0x60, &ts2020_config);
+	if (!state->i2c_client_tuner) {
+		dvb_module_release(state->i2c_client_demod);
+		return -ENODEV;
 	}
 
 	/* delegate signal strength measurement to tuner */
@@ -359,10 +340,7 @@ static int dvbsky_s960_attach(struct dvb_usb_adapter *adap)
 	state->fe_set_voltage = adap->fe[0]->ops.set_voltage;
 	adap->fe[0]->ops.set_voltage = dvbsky_usb_set_voltage;
 
-	state->i2c_client_tuner = client;
-
-fail_attach:
-	return ret;
+	return 0;
 }
 
 static int dvbsky_usb_ci_set_voltage(struct dvb_frontend *fe,
@@ -412,80 +390,60 @@ static int dvbsky_ci_ctrl(void *priv, u8 read, int addr,
 	return ret;
 }
 
-static const struct m88ds3103_config dvbsky_s960c_m88ds3103_config = {
-	.i2c_addr = 0x68,
-	.clock = 27000000,
-	.i2c_wr_max = 33,
-	.clock_out = 0,
-	.ts_mode = M88DS3103_TS_CI,
-	.ts_clk = 10000,
-	.ts_clk_pol = 1,
-	.agc = 0x99,
-	.lnb_hv_pol = 0,
-	.lnb_en_pol = 1,
-};
-
 static int dvbsky_s960c_attach(struct dvb_usb_adapter *adap)
 {
 	struct dvbsky_state *state = adap_to_priv(adap);
 	struct dvb_usb_device *d = adap_to_d(adap);
-	int ret = 0;
-	/* demod I2C adapter */
-	struct i2c_adapter *i2c_adapter = NULL;
-	struct i2c_client *client_tuner, *client_ci;
-	struct i2c_board_info info;
-	struct sp2_config sp2_config;
+	struct i2c_adapter *i2c_adapter;
+	struct m88ds3103_platform_data m88ds3103_pdata = {};
 	struct ts2020_config ts2020_config = {};
-	memset(&info, 0, sizeof(struct i2c_board_info));
+	struct sp2_config sp2_config = {};
 
 	/* attach demod */
-	adap->fe[0] = dvb_attach(m88ds3103_attach,
-			&dvbsky_s960c_m88ds3103_config,
-			&d->i2c_adap,
-			&i2c_adapter);
-	if (!adap->fe[0]) {
-		dev_err(&d->udev->dev, "dvbsky_s960ci_attach fail.\n");
-		ret = -ENODEV;
-		goto fail_attach;
-	}
+	m88ds3103_pdata.clk = 27000000,
+	m88ds3103_pdata.i2c_wr_max = 33,
+	m88ds3103_pdata.clk_out = 0,
+	m88ds3103_pdata.ts_mode = M88DS3103_TS_CI,
+	m88ds3103_pdata.ts_clk = 10000,
+	m88ds3103_pdata.ts_clk_pol = 1,
+	m88ds3103_pdata.agc = 0x99,
+	m88ds3103_pdata.lnb_hv_pol = 0,
+	m88ds3103_pdata.lnb_en_pol = 1,
+
+	state->i2c_client_demod = dvb_module_probe("m88ds3103", NULL,
+						   &d->i2c_adap,
+						   0x68, &m88ds3103_pdata);
+	if (!state->i2c_client_demod)
+		return -ENODEV;
+
+	adap->fe[0] = m88ds3103_pdata.get_dvb_frontend(state->i2c_client_demod);
+	i2c_adapter = m88ds3103_pdata.get_i2c_adapter(state->i2c_client_demod);
 
 	/* attach tuner */
 	ts2020_config.fe = adap->fe[0];
 	ts2020_config.get_agc_pwm = m88ds3103_get_agc_pwm;
-	strlcpy(info.type, "ts2020", I2C_NAME_SIZE);
-	info.addr = 0x60;
-	info.platform_data = &ts2020_config;
-	request_module("ts2020");
-	client_tuner = i2c_new_device(i2c_adapter, &info);
-	if (client_tuner == NULL || client_tuner->dev.driver == NULL) {
-		ret = -ENODEV;
-		goto fail_tuner_device;
-	}
 
-	if (!try_module_get(client_tuner->dev.driver->owner)) {
-		ret = -ENODEV;
-		goto fail_tuner_module;
+	state->i2c_client_tuner = dvb_module_probe("ts2020", NULL,
+						   i2c_adapter,
+						   0x60, &ts2020_config);
+	if (!state->i2c_client_tuner) {
+		dvb_module_release(state->i2c_client_demod);
+		return -ENODEV;
 	}
 
 	/* attach ci controller */
-	memset(&sp2_config, 0, sizeof(sp2_config));
 	sp2_config.dvb_adap = &adap->dvb_adap;
 	sp2_config.priv = d;
 	sp2_config.ci_control = dvbsky_ci_ctrl;
-	memset(&info, 0, sizeof(struct i2c_board_info));
-	strlcpy(info.type, "sp2", I2C_NAME_SIZE);
-	info.addr = 0x40;
-	info.platform_data = &sp2_config;
-	request_module("sp2");
-	client_ci = i2c_new_device(&d->i2c_adap, &info);
-	if (client_ci == NULL || client_ci->dev.driver == NULL) {
-		ret = -ENODEV;
-		goto fail_ci_device;
-	}
 
-	if (!try_module_get(client_ci->dev.driver->owner)) {
-		ret = -ENODEV;
-		goto fail_ci_module;
+	state->i2c_client_ci = dvb_module_probe("sp2", NULL,
+						&d->i2c_adap,
+						0x40, &sp2_config);
+
+	if (!state->i2c_client_ci) {
+		dvb_module_release(state->i2c_client_tuner);
+		dvb_module_release(state->i2c_client_demod);
+		return -ENODEV;
 	}
 
 	/* delegate signal strength measurement to tuner */
@@ -500,165 +458,92 @@ static int dvbsky_s960c_attach(struct dvb_usb_adapter *adap)
 	state->fe_set_voltage = adap->fe[0]->ops.set_voltage;
 	adap->fe[0]->ops.set_voltage = dvbsky_usb_ci_set_voltage;
 
-	state->i2c_client_tuner = client_tuner;
-	state->i2c_client_ci = client_ci;
-	return ret;
-fail_ci_module:
-	i2c_unregister_device(client_ci);
-fail_ci_device:
-	module_put(client_tuner->dev.driver->owner);
-fail_tuner_module:
-	i2c_unregister_device(client_tuner);
-fail_tuner_device:
-	dvb_frontend_detach(adap->fe[0]);
-fail_attach:
-	return ret;
+	return 0;
 }
 
 static int dvbsky_t680c_attach(struct dvb_usb_adapter *adap)
 {
 	struct dvbsky_state *state = adap_to_priv(adap);
 	struct dvb_usb_device *d = adap_to_d(adap);
-	int ret = 0;
 	struct i2c_adapter *i2c_adapter;
-	struct i2c_client *client_demod, *client_tuner, *client_ci;
-	struct i2c_board_info info;
-	struct si2168_config si2168_config;
-	struct si2157_config si2157_config;
-	struct sp2_config sp2_config;
+	struct si2168_config si2168_config = {};
+	struct si2157_config si2157_config = {};
+	struct sp2_config sp2_config = {};
 
 	/* attach demod */
-	memset(&si2168_config, 0, sizeof(si2168_config));
 	si2168_config.i2c_adapter = &i2c_adapter;
 	si2168_config.fe = &adap->fe[0];
 	si2168_config.ts_mode = SI2168_TS_PARALLEL;
-	memset(&info, 0, sizeof(struct i2c_board_info));
-	strlcpy(info.type, "si2168", I2C_NAME_SIZE);
-	info.addr = 0x64;
-	info.platform_data = &si2168_config;
-
-	request_module(info.type);
-	client_demod = i2c_new_device(&d->i2c_adap, &info);
-	if (client_demod == NULL ||
-			client_demod->dev.driver == NULL)
-		goto fail_demod_device;
-	if (!try_module_get(client_demod->dev.driver->owner))
-		goto fail_demod_module;
+
+	state->i2c_client_demod = dvb_module_probe("si2168", NULL,
+						   &d->i2c_adap,
+						   0x64, &si2168_config);
+	if (!state->i2c_client_demod)
+		return -ENODEV;
 
 	/* attach tuner */
-	memset(&si2157_config, 0, sizeof(si2157_config));
 	si2157_config.fe = adap->fe[0];
 	si2157_config.if_port = 1;
-	memset(&info, 0, sizeof(struct i2c_board_info));
-	strlcpy(info.type, "si2157", I2C_NAME_SIZE);
-	info.addr = 0x60;
-	info.platform_data = &si2157_config;
-
-	request_module(info.type);
-	client_tuner = i2c_new_device(i2c_adapter, &info);
-	if (client_tuner == NULL ||
-			client_tuner->dev.driver == NULL)
-		goto fail_tuner_device;
-	if (!try_module_get(client_tuner->dev.driver->owner))
-		goto fail_tuner_module;
+
+	state->i2c_client_tuner = dvb_module_probe("si2157", NULL,
+						   i2c_adapter,
+						   0x60, &si2157_config);
+	if (!state->i2c_client_tuner) {
+		dvb_module_release(state->i2c_client_demod);
+		return -ENODEV;
+	}
 
 	/* attach ci controller */
-	memset(&sp2_config, 0, sizeof(sp2_config));
 	sp2_config.dvb_adap = &adap->dvb_adap;
 	sp2_config.priv = d;
 	sp2_config.ci_control = dvbsky_ci_ctrl;
-	memset(&info, 0, sizeof(struct i2c_board_info));
-	strlcpy(info.type, "sp2", I2C_NAME_SIZE);
-	info.addr = 0x40;
-	info.platform_data = &sp2_config;
 
-	request_module(info.type);
-	client_ci = i2c_new_device(&d->i2c_adap, &info);
+	state->i2c_client_ci = dvb_module_probe("sp2", NULL,
+						&d->i2c_adap,
+						0x40, &sp2_config);
 
-	if (client_ci == NULL || client_ci->dev.driver == NULL)
-		goto fail_ci_device;
-
-	if (!try_module_get(client_ci->dev.driver->owner))
-		goto fail_ci_module;
+	if (!state->i2c_client_ci) {
+		dvb_module_release(state->i2c_client_tuner);
+		dvb_module_release(state->i2c_client_demod);
+		return -ENODEV;
+	}
 
-	state->i2c_client_demod = client_demod;
-	state->i2c_client_tuner = client_tuner;
-	state->i2c_client_ci = client_ci;
-	return ret;
-fail_ci_module:
-	i2c_unregister_device(client_ci);
-fail_ci_device:
-	module_put(client_tuner->dev.driver->owner);
-fail_tuner_module:
-	i2c_unregister_device(client_tuner);
-fail_tuner_device:
-	module_put(client_demod->dev.driver->owner);
-fail_demod_module:
-	i2c_unregister_device(client_demod);
-fail_demod_device:
-	ret = -ENODEV;
-	return ret;
+	return 0;
 }
 
 static int dvbsky_t330_attach(struct dvb_usb_adapter *adap)
 {
 	struct dvbsky_state *state = adap_to_priv(adap);
 	struct dvb_usb_device *d = adap_to_d(adap);
-	int ret = 0;
 	struct i2c_adapter *i2c_adapter;
-	struct i2c_client *client_demod, *client_tuner;
-	struct i2c_board_info info;
-	struct si2168_config si2168_config;
-	struct si2157_config si2157_config;
+	struct si2168_config si2168_config = {};
+	struct si2157_config si2157_config = {};
 
 	/* attach demod */
-	memset(&si2168_config, 0, sizeof(si2168_config));
 	si2168_config.i2c_adapter = &i2c_adapter;
 	si2168_config.fe = &adap->fe[0];
 	si2168_config.ts_mode = SI2168_TS_PARALLEL;
 	si2168_config.ts_clock_gapped = true;
-	memset(&info, 0, sizeof(struct i2c_board_info));
-	strlcpy(info.type, "si2168", I2C_NAME_SIZE);
-	info.addr = 0x64;
-	info.platform_data = &si2168_config;
-
-	request_module(info.type);
-	client_demod = i2c_new_device(&d->i2c_adap, &info);
-	if (client_demod == NULL ||
-			client_demod->dev.driver == NULL)
-		goto fail_demod_device;
-	if (!try_module_get(client_demod->dev.driver->owner))
-		goto fail_demod_module;
+
+	state->i2c_client_demod = dvb_module_probe("si2168", NULL,
+						   &d->i2c_adap,
+						   0x64, &si2168_config);
+	if (!state->i2c_client_demod)
+		return -ENODEV;
 
 	/* attach tuner */
-	memset(&si2157_config, 0, sizeof(si2157_config));
 	si2157_config.fe = adap->fe[0];
 	si2157_config.if_port = 1;
-	memset(&info, 0, sizeof(struct i2c_board_info));
-	strlcpy(info.type, "si2157", I2C_NAME_SIZE);
-	info.addr = 0x60;
-	info.platform_data = &si2157_config;
-
-	request_module(info.type);
-	client_tuner = i2c_new_device(i2c_adapter, &info);
-	if (client_tuner == NULL ||
-			client_tuner->dev.driver == NULL)
-		goto fail_tuner_device;
-	if (!try_module_get(client_tuner->dev.driver->owner))
-		goto fail_tuner_module;
-
-	state->i2c_client_demod = client_demod;
-	state->i2c_client_tuner = client_tuner;
-	return ret;
-fail_tuner_module:
-	i2c_unregister_device(client_tuner);
-fail_tuner_device:
-	module_put(client_demod->dev.driver->owner);
-fail_demod_module:
-	i2c_unregister_device(client_demod);
-fail_demod_device:
-	ret = -ENODEV;
-	return ret;
+
+	state->i2c_client_tuner = dvb_module_probe("si2157", NULL,
+						   i2c_adapter,
+						   0x60, &si2157_config);
+	if (!state->i2c_client_tuner) {
+		dvb_module_release(state->i2c_client_demod);
+		return -ENODEV;
+	}
+
+	return 0;
 }
 
 static int dvbsky_mygica_t230c_attach(struct dvb_usb_adapter *adap)
@@ -666,57 +551,34 @@ static int dvbsky_mygica_t230c_attach(struct dvb_usb_adapter *adap)
 	struct dvbsky_state *state = adap_to_priv(adap);
 	struct dvb_usb_device *d = adap_to_d(adap);
 	struct i2c_adapter *i2c_adapter;
-	struct i2c_client *client_demod, *client_tuner;
-	struct i2c_board_info info;
-	struct si2168_config si2168_config;
-	struct si2157_config si2157_config;
+	struct si2168_config si2168_config = {};
+	struct si2157_config si2157_config = {};
 
 	/* attach demod */
-	memset(&si2168_config, 0, sizeof(si2168_config));
 	si2168_config.i2c_adapter = &i2c_adapter;
 	si2168_config.fe = &adap->fe[0];
 	si2168_config.ts_mode = SI2168_TS_PARALLEL;
 	si2168_config.ts_clock_inv = 1;
-	memset(&info, 0, sizeof(struct i2c_board_info));
-	strlcpy(info.type, "si2168", sizeof(info.type));
-	info.addr = 0x64;
-	info.platform_data = &si2168_config;
-
-	request_module("si2168");
-	client_demod = i2c_new_device(&d->i2c_adap, &info);
-	if (!client_demod || !client_demod->dev.driver)
-		goto fail_demod_device;
-	if (!try_module_get(client_demod->dev.driver->owner))
-		goto fail_demod_module;
+
+	state->i2c_client_demod = dvb_module_probe("si2168", NULL,
+						   &d->i2c_adap,
+						   0x64, &si2168_config);
+	if (!state->i2c_client_demod)
+		return -ENODEV;
 
 	/* attach tuner */
-	memset(&si2157_config, 0, sizeof(si2157_config));
 	si2157_config.fe = adap->fe[0];
 	si2157_config.if_port = 0;
-	memset(&info, 0, sizeof(struct i2c_board_info));
-	strlcpy(info.type, "si2141", sizeof(info.type));
-	info.addr = 0x60;
-	info.platform_data = &si2157_config;
-
-	request_module("si2157");
-	client_tuner = i2c_new_device(i2c_adapter, &info);
-	if (!client_tuner || !client_tuner->dev.driver)
-		goto fail_tuner_device;
-	if (!try_module_get(client_tuner->dev.driver->owner))
-		goto fail_tuner_module;
-
-	state->i2c_client_demod = client_demod;
-	state->i2c_client_tuner = client_tuner;
-	return 0;
 
-fail_tuner_module:
-	i2c_unregister_device(client_tuner);
-fail_tuner_device:
-	module_put(client_demod->dev.driver->owner);
-fail_demod_module:
-	i2c_unregister_device(client_demod);
-fail_demod_device:
-	return -ENODEV;
+	state->i2c_client_tuner = dvb_module_probe("si2157", "si2141",
+						   i2c_adapter,
+						   0x60, &si2157_config);
+	if (!state->i2c_client_tuner) {
+		dvb_module_release(state->i2c_client_demod);
+		return -ENODEV;
+	}
+
+	return 0;
 }
 
 
@@ -744,8 +606,6 @@ static int dvbsky_init(struct dvb_usb_device *d)
 	if (ret)
 		return ret;
 	*/
-	mutex_init(&state->stream_mutex);
-
 	state->last_lock = 0;
 
 	return 0;
@@ -754,26 +614,13 @@ static int dvbsky_init(struct dvb_usb_device *d)
 static void dvbsky_exit(struct dvb_usb_device *d)
 {
 	struct dvbsky_state *state = d_to_priv(d);
-	struct i2c_client *client;
+	struct dvb_usb_adapter *adap = &d->adapter[0];
 
-	client = state->i2c_client_tuner;
-	/* remove I2C tuner */
-	if (client) {
-		module_put(client->dev.driver->owner);
-		i2c_unregister_device(client);
-	}
-	client = state->i2c_client_demod;
-	/* remove I2C demod */
-	if (client) {
-		module_put(client->dev.driver->owner);
-		i2c_unregister_device(client);
-	}
-	client = state->i2c_client_ci;
-	/* remove I2C ci */
-	if (client) {
-		module_put(client->dev.driver->owner);
-		i2c_unregister_device(client);
-	}
+	dvb_module_release(state->i2c_client_tuner);
+	dvb_module_release(state->i2c_client_demod);
+	dvb_module_release(state->i2c_client_ci);
+
+	adap->fe[0] = NULL;
 }
 
 /* DVB USB Driver stuff */
diff --git a/drivers/media/usb/dvb-usb-v2/gl861.c b/drivers/media/usb/dvb-usb-v2/gl861.c
index b1b09c547861..4817dfd3e659 100644
--- a/drivers/media/usb/dvb-usb-v2/gl861.c
+++ b/drivers/media/usb/dvb-usb-v2/gl861.c
@@ -20,6 +20,8 @@ static int gl861_i2c_msg(struct dvb_usb_device *d, u8 addr,
 	u16 value = addr << (8 + 1);
 	int wo = (rbuf == NULL || rlen == 0); /* write-only */
 	u8 req, type;
+	u8 *buf;
+	int ret;
 
 	if (wo) {
 		req = GL861_REQ_I2C_WRITE;
@@ -42,11 +44,23 @@ static int gl861_i2c_msg(struct dvb_usb_device *d, u8 addr,
 				KBUILD_MODNAME, wlen);
 		return -EINVAL;
 	}
+	buf = NULL;
+	if (rlen > 0) {
+		buf = kmalloc(rlen, GFP_KERNEL);
+		if (!buf)
+			return -ENOMEM;
+	}
+	usleep_range(1000, 2000); /* avoid I2C errors */
+
+	ret = usb_control_msg(d->udev, usb_rcvctrlpipe(d->udev, 0), req, type,
+			      value, index, buf, rlen, 2000);
+	if (rlen > 0) {
+		if (ret > 0)
+			memcpy(rbuf, buf, rlen);
+		kfree(buf);
+	}
 
-	msleep(1); /* avoid I2C errors */
-
-	return usb_control_msg(d->udev, usb_rcvctrlpipe(d->udev, 0), req, type,
-			       value, index, rbuf, rlen, 2000);
+	return ret;
 }
 
 /* I2C */
diff --git a/drivers/media/usb/dvb-usb-v2/usb_urb.c b/drivers/media/usb/dvb-usb-v2/usb_urb.c
index dce2b97efce4..b0499f95ec45 100644
--- a/drivers/media/usb/dvb-usb-v2/usb_urb.c
+++ b/drivers/media/usb/dvb-usb-v2/usb_urb.c
@@ -155,8 +155,7 @@ static int usb_urb_alloc_bulk_urbs(struct usb_data_stream *stream)
 				stream->props.u.bulk.buffersize,
 				usb_urb_complete, stream);
 
-		stream->urb_list[i]->transfer_flags = URB_NO_TRANSFER_DMA_MAP;
-		stream->urb_list[i]->transfer_dma = stream->dma_addr[i];
+		stream->urb_list[i]->transfer_flags = URB_FREE_BUFFER;
 		stream->urbs_initialized++;
 	}
 	return 0;
@@ -187,13 +186,12 @@ static int usb_urb_alloc_isoc_urbs(struct usb_data_stream *stream)
 		urb->complete = usb_urb_complete;
 		urb->pipe = usb_rcvisocpipe(stream->udev,
 				stream->props.endpoint);
-		urb->transfer_flags = URB_ISO_ASAP | URB_NO_TRANSFER_DMA_MAP;
+		urb->transfer_flags = URB_ISO_ASAP | URB_FREE_BUFFER;
 		urb->interval = stream->props.u.isoc.interval;
 		urb->number_of_packets = stream->props.u.isoc.framesperurb;
 		urb->transfer_buffer_length = stream->props.u.isoc.framesize *
 				stream->props.u.isoc.framesperurb;
 		urb->transfer_buffer = stream->buf_list[i];
-		urb->transfer_dma = stream->dma_addr[i];
 
 		for (j = 0; j < stream->props.u.isoc.framesperurb; j++) {
 			urb->iso_frame_desc[j].offset = frame_offset;
@@ -212,11 +210,7 @@ static int usb_free_stream_buffers(struct usb_data_stream *stream)
 	if (stream->state & USB_STATE_URB_BUF) {
 		while (stream->buf_num) {
 			stream->buf_num--;
-			dev_dbg(&stream->udev->dev, "%s: free buf=%d\n",
-				__func__, stream->buf_num);
-			usb_free_coherent(stream->udev, stream->buf_size,
-					  stream->buf_list[stream->buf_num],
-					  stream->dma_addr[stream->buf_num]);
+			stream->buf_list[stream->buf_num] = NULL;
 		}
 	}
 
@@ -236,9 +230,7 @@ static int usb_alloc_stream_buffers(struct usb_data_stream *stream, int num,
 			__func__,  num * size);
 
 	for (stream->buf_num = 0; stream->buf_num < num; stream->buf_num++) {
-		stream->buf_list[stream->buf_num] = usb_alloc_coherent(
-				stream->udev, size, GFP_ATOMIC,
-				&stream->dma_addr[stream->buf_num]);
+		stream->buf_list[stream->buf_num] = kzalloc(size, GFP_ATOMIC);
 		if (!stream->buf_list[stream->buf_num]) {
 			dev_dbg(&stream->udev->dev, "%s: alloc buf=%d failed\n",
 					__func__, stream->buf_num);
@@ -250,7 +242,6 @@ static int usb_alloc_stream_buffers(struct usb_data_stream *stream, int num,
 				__func__, stream->buf_num,
 				stream->buf_list[stream->buf_num],
 				(long long)stream->dma_addr[stream->buf_num]);
-		memset(stream->buf_list[stream->buf_num], 0, size);
 		stream->state |= USB_STATE_URB_BUF;
 	}
 
diff --git a/drivers/media/usb/dvb-usb/cxusb.c b/drivers/media/usb/dvb-usb/cxusb.c
index 387a074ea6ec..b70d289dc738 100644
--- a/drivers/media/usb/dvb-usb/cxusb.c
+++ b/drivers/media/usb/dvb-usb/cxusb.c
@@ -540,12 +540,10 @@ static struct cx22702_config cxusb_cx22702_config = {
 };
 
 static struct lgdt330x_config cxusb_lgdt3303_config = {
-	.demod_address = 0x0e,
 	.demod_chip    = LGDT3303,
 };
 
 static struct lgdt330x_config cxusb_aver_lgdt3303_config = {
-	.demod_address       = 0x0e,
 	.demod_chip          = LGDT3303,
 	.clock_polarity_flip = 2,
 };
@@ -763,6 +761,7 @@ static int cxusb_lgdt3303_frontend_attach(struct dvb_usb_adapter *adap)
 
 	adap->fe_adap[0].fe = dvb_attach(lgdt330x_attach,
 					 &cxusb_lgdt3303_config,
+					 0x0e,
 					 &adap->dev->i2c_adap);
 	if ((adap->fe_adap[0].fe) != NULL)
 		return 0;
@@ -772,8 +771,10 @@ static int cxusb_lgdt3303_frontend_attach(struct dvb_usb_adapter *adap)
 
 static int cxusb_aver_lgdt3303_frontend_attach(struct dvb_usb_adapter *adap)
 {
-	adap->fe_adap[0].fe = dvb_attach(lgdt330x_attach, &cxusb_aver_lgdt3303_config,
-			      &adap->dev->i2c_adap);
+	adap->fe_adap[0].fe = dvb_attach(lgdt330x_attach,
+					 &cxusb_aver_lgdt3303_config,
+					 0x0e,
+					 &adap->dev->i2c_adap);
 	if (adap->fe_adap[0].fe != NULL)
 		return 0;
 
diff --git a/drivers/media/usb/dvb-usb/dvb-usb-dvb.c b/drivers/media/usb/dvb-usb/dvb-usb-dvb.c
index 3a66e732e0d8..8056053c9ab0 100644
--- a/drivers/media/usb/dvb-usb/dvb-usb-dvb.c
+++ b/drivers/media/usb/dvb-usb/dvb-usb-dvb.c
@@ -132,10 +132,14 @@ static void dvb_usb_media_device_unregister(struct dvb_usb_adapter *adap)
 	if (!adap->dvb_adap.mdev)
 		return;
 
+	mutex_lock(&adap->dvb_adap.mdev_lock);
+
 	media_device_unregister(adap->dvb_adap.mdev);
 	media_device_cleanup(adap->dvb_adap.mdev);
 	kfree(adap->dvb_adap.mdev);
 	adap->dvb_adap.mdev = NULL;
+
+	mutex_unlock(&adap->dvb_adap.mdev_lock);
 #endif
 }
 
diff --git a/drivers/media/usb/dvb-usb/gp8psk.c b/drivers/media/usb/dvb-usb/gp8psk.c
index 37f062225ed2..334b9fb98112 100644
--- a/drivers/media/usb/dvb-usb/gp8psk.c
+++ b/drivers/media/usb/dvb-usb/gp8psk.c
@@ -148,7 +148,7 @@ static int gp8psk_load_bcm4500fw(struct dvb_usb_device *d)
 	info("downloading bcm4500 firmware from file '%s'",bcm4500_firmware);
 
 	ptr = fw->data;
-	buf = kmalloc(64, GFP_KERNEL | GFP_DMA);
+	buf = kmalloc(64, GFP_KERNEL);
 	if (!buf) {
 		ret = -ENOMEM;
 		goto out_rel_fw;
diff --git a/drivers/media/usb/dvb-usb/usb-urb.c b/drivers/media/usb/dvb-usb/usb-urb.c
index 2804d2d0e83a..5e05963f4220 100644
--- a/drivers/media/usb/dvb-usb/usb-urb.c
+++ b/drivers/media/usb/dvb-usb/usb-urb.c
@@ -118,7 +118,7 @@ static int usb_allocate_stream_buffers(struct usb_data_stream *stream, int num,
 	for (stream->buf_num = 0; stream->buf_num < num; stream->buf_num++) {
 		deb_mem("allocating buffer %d\n",stream->buf_num);
 		if (( stream->buf_list[stream->buf_num] =
-					usb_alloc_coherent(stream->udev, size, GFP_ATOMIC,
+					usb_alloc_coherent(stream->udev, size, GFP_KERNEL,
 					&stream->dma_addr[stream->buf_num]) ) == NULL) {
 			deb_mem("not enough memory for urb-buffer allocation.\n");
 			usb_free_stream_buffers(stream);
@@ -145,7 +145,7 @@ static int usb_bulk_urb_init(struct usb_data_stream *stream)
 
 	/* allocate the URBs */
 	for (i = 0; i < stream->props.count; i++) {
-		stream->urb_list[i] = usb_alloc_urb(0, GFP_ATOMIC);
+		stream->urb_list[i] = usb_alloc_urb(0, GFP_KERNEL);
 		if (!stream->urb_list[i]) {
 			deb_mem("not enough memory for urb_alloc_urb!.\n");
 			for (j = 0; j < i; j++)
@@ -178,7 +178,7 @@ static int usb_isoc_urb_init(struct usb_data_stream *stream)
 		struct urb *urb;
 		int frame_offset = 0;
 
-		stream->urb_list[i] = usb_alloc_urb(stream->props.u.isoc.framesperurb, GFP_ATOMIC);
+		stream->urb_list[i] = usb_alloc_urb(stream->props.u.isoc.framesperurb, GFP_KERNEL);
 		if (!stream->urb_list[i]) {
 			deb_mem("not enough memory for urb_alloc_urb!\n");
 			for (j = 0; j < i; j++)
diff --git a/drivers/media/usb/em28xx/em28xx-cards.c b/drivers/media/usb/em28xx/em28xx-cards.c
index 7c3203d7044b..6c8438311d3b 100644
--- a/drivers/media/usb/em28xx/em28xx-cards.c
+++ b/drivers/media/usb/em28xx/em28xx-cards.c
@@ -87,6 +87,21 @@ static const struct em28xx_reg_seq default_digital[] = {
 	{	-1,		-1,	-1,		-1},
 };
 
+/* Board :Zolid Hybrid Tv Stick */
+static struct em28xx_reg_seq zolid_tuner[] = {
+	{EM2820_R08_GPIO_CTRL,		0xfd,		0xff,	100},
+	{EM2820_R08_GPIO_CTRL,		0xfe,		0xff,	100},
+	{		-1,					-1,			-1,		 -1},
+};
+
+static struct em28xx_reg_seq zolid_digital[] = {
+	{EM2820_R08_GPIO_CTRL,		0x6a,		0xff,	100},
+	{EM2820_R08_GPIO_CTRL,		0x7a,		0xff,	100},
+	{EM2880_R04_GPO,			0x04,		0xff,	100},
+	{EM2880_R04_GPO,			0x0c,		0xff,	100},
+	{	-1,						-1,			-1,		 -1},
+};
+
 /* Board Hauppauge WinTV HVR 900 analog */
 static const struct em28xx_reg_seq hauppauge_wintv_hvr_900_analog[] = {
 	{EM2820_R08_GPIO_CTRL,	0x2d,	~EM_GPIO_4,	10},
@@ -666,6 +681,16 @@ const struct em28xx_board em28xx_boards[] = {
 		.tuner_type    = TUNER_ABSENT,
 		.is_webcam     = 1,	/* To enable sensor probe */
 	},
+	[EM2882_BOARD_ZOLID_HYBRID_TV_STICK] = {
+		.name			= ":ZOLID HYBRID TV STICK",
+		.tuner_type		= TUNER_XC2028,
+		.tuner_gpio		= zolid_tuner,
+		.decoder		= EM28XX_TVP5150,
+		.xclk			= EM28XX_XCLK_FREQUENCY_12MHZ,
+		.mts_firmware	= 1,
+		.has_dvb		= 1,
+		.dvb_gpio		= zolid_digital,
+	},
 	[EM2750_BOARD_DLCW_130] = {
 		/* Beijing Huaqi Information Digital Technology Co., Ltd */
 		.name          = "Huaqi DLCW-130",
@@ -2493,7 +2518,7 @@ struct usb_device_id em28xx_id_table[] = {
 			.driver_info = EM2820_BOARD_UNKNOWN },
 	{ USB_DEVICE(0xeb1a, 0x2881),
 			.driver_info = EM2820_BOARD_UNKNOWN },
-	{ USB_DEVICE(0xeb1a, 0x2883),
+	{ USB_DEVICE(0xeb1a, 0x2883), /* used by :Zolid Hybrid Tv Stick */
 			.driver_info = EM2820_BOARD_UNKNOWN },
 	{ USB_DEVICE(0xeb1a, 0x2868),
 			.driver_info = EM2820_BOARD_UNKNOWN },
@@ -2663,6 +2688,8 @@ struct usb_device_id em28xx_id_table[] = {
 			.driver_info = EM28178_BOARD_PCTV_292E },
 	{ USB_DEVICE(0x2040, 0x8268), /* Hauppauge Retail WinTV-soloHD Bulk */
 			.driver_info = EM28178_BOARD_PCTV_292E },
+	{ USB_DEVICE(0x2040, 0x8268), /* Hauppauge WinTV-soloHD alt. PID */
+			.driver_info = EM28178_BOARD_PCTV_292E },
 	{ USB_DEVICE(0x0413, 0x6f07),
 			.driver_info = EM2861_BOARD_LEADTEK_VC100 },
 	{ USB_DEVICE(0xeb1a, 0x8179),
@@ -2688,6 +2715,7 @@ static const struct em28xx_hash_table em28xx_eeprom_hash[] = {
 	{0xb8846b20, EM2881_BOARD_PINNACLE_HYBRID_PRO, TUNER_XC2028},
 	{0x63f653bd, EM2870_BOARD_REDDO_DVB_C_USB_BOX, TUNER_ABSENT},
 	{0x4e913442, EM2882_BOARD_DIKOM_DK300, TUNER_XC2028},
+	{0x85dd871e, EM2882_BOARD_ZOLID_HYBRID_TV_STICK, TUNER_XC2028},
 };
 
 /* I2C devicelist hash table for devices with generic USB IDs */
@@ -2699,6 +2727,7 @@ static const struct em28xx_hash_table em28xx_i2c_hash[] = {
 	{0xc51200e3, EM2820_BOARD_GADMEI_TVR200, TUNER_LG_PAL_NEW_TAPC},
 	{0x4ba50080, EM2861_BOARD_GADMEI_UTV330PLUS, TUNER_TNF_5335MF},
 	{0x6b800080, EM2874_BOARD_LEADERSHIP_ISDBT, TUNER_ABSENT},
+	{0x27e10080, EM2882_BOARD_ZOLID_HYBRID_TV_STICK, TUNER_XC2028},
 };
 
 /* NOTE: introduce a separate hash table for devices with 16 bit eeproms */
@@ -3182,11 +3211,10 @@ void em28xx_setup_xc3028(struct em28xx *dev, struct xc2028_ctrl *ctl)
 	case EM2880_BOARD_EMPIRE_DUAL_TV:
 	case EM2880_BOARD_HAUPPAUGE_WINTV_HVR_900:
 	case EM2882_BOARD_TERRATEC_HYBRID_XS:
-		ctl->demod = XC3028_FE_ZARLINK456;
-		break;
 	case EM2880_BOARD_TERRATEC_HYBRID_XS:
 	case EM2880_BOARD_TERRATEC_HYBRID_XS_FR:
 	case EM2881_BOARD_PINNACLE_HYBRID_PRO:
+	case EM2882_BOARD_ZOLID_HYBRID_TV_STICK:
 		ctl->demod = XC3028_FE_ZARLINK456;
 		break;
 	case EM2880_BOARD_HAUPPAUGE_WINTV_HVR_900_R2:
@@ -3674,7 +3702,7 @@ static int em28xx_usb_probe(struct usb_interface *intf,
 
 	/* Don't register audio interfaces */
 	if (intf->altsetting[0].desc.bInterfaceClass == USB_CLASS_AUDIO) {
-		dev_err(&intf->dev,
+		dev_info(&intf->dev,
 			"audio device (%04x:%04x): interface %i, class %i\n",
 			le16_to_cpu(udev->descriptor.idVendor),
 			le16_to_cpu(udev->descriptor.idProduct),
@@ -3736,7 +3764,7 @@ static int em28xx_usb_probe(struct usb_interface *intf,
 		speed = "unknown";
 	}
 
-	dev_err(&intf->dev,
+	dev_info(&intf->dev,
 		"New device %s %s @ %s Mbps (%04x:%04x, interface %d, class %d)\n",
 		udev->manufacturer ? udev->manufacturer : "",
 		udev->product ? udev->product : "",
@@ -3771,7 +3799,7 @@ static int em28xx_usb_probe(struct usb_interface *intf,
 	dev->dev_next = NULL;
 
 	if (has_vendor_audio) {
-		dev_err(&intf->dev,
+		dev_info(&intf->dev,
 			"Audio interface %i found (Vendor Class)\n", ifnum);
 		dev->usb_audio_type = EM28XX_USB_AUDIO_VENDOR;
 	}
@@ -3790,12 +3818,12 @@ static int em28xx_usb_probe(struct usb_interface *intf,
 	}
 
 	if (has_video)
-		dev_err(&intf->dev, "Video interface %i found:%s%s\n",
+		dev_info(&intf->dev, "Video interface %i found:%s%s\n",
 			ifnum,
 			dev->analog_ep_bulk ? " bulk" : "",
 			dev->analog_ep_isoc ? " isoc" : "");
 	if (has_dvb)
-		dev_err(&intf->dev, "DVB interface %i found:%s%s\n",
+		dev_info(&intf->dev, "DVB interface %i found:%s%s\n",
 			ifnum,
 			dev->dvb_ep_bulk ? " bulk" : "",
 			dev->dvb_ep_isoc ? " isoc" : "");
@@ -3837,13 +3865,13 @@ static int em28xx_usb_probe(struct usb_interface *intf,
 	if (has_video) {
 		if (!dev->analog_ep_isoc || (try_bulk && dev->analog_ep_bulk))
 			dev->analog_xfer_bulk = 1;
-		dev_err(&intf->dev, "analog set to %s mode.\n",
+		dev_info(&intf->dev, "analog set to %s mode.\n",
 			dev->analog_xfer_bulk ? "bulk" : "isoc");
 	}
 	if (has_dvb) {
 		if (!dev->dvb_ep_isoc || (try_bulk && dev->dvb_ep_bulk))
 			dev->dvb_xfer_bulk = 1;
-		dev_err(&intf->dev, "dvb set to %s mode.\n",
+		dev_info(&intf->dev, "dvb set to %s mode.\n",
 			dev->dvb_xfer_bulk ? "bulk" : "isoc");
 	}
 
@@ -3957,7 +3985,7 @@ static void em28xx_usb_disconnect(struct usb_interface *intf)
 
 	dev->disconnected = 1;
 
-	dev_err(&dev->intf->dev, "Disconnecting %s\n", dev->name);
+	dev_info(&dev->intf->dev, "Disconnecting %s\n", dev->name);
 
 	flush_request_modules(dev);
 
diff --git a/drivers/media/usb/em28xx/em28xx-core.c b/drivers/media/usb/em28xx/em28xx-core.c
index f28995383090..f70845e7d8c6 100644
--- a/drivers/media/usb/em28xx/em28xx-core.c
+++ b/drivers/media/usb/em28xx/em28xx-core.c
@@ -1171,8 +1171,9 @@ int em28xx_resume_extension(struct em28xx *dev)
 	dev_info(&dev->intf->dev, "Resuming extensions\n");
 	mutex_lock(&em28xx_devlist_mutex);
 	list_for_each_entry(ops, &em28xx_extension_devlist, next) {
-		if (ops->resume)
-			ops->resume(dev);
+		if (!ops->resume)
+			continue;
+		ops->resume(dev);
 		if (dev->dev_next)
 			ops->resume(dev->dev_next);
 	}
diff --git a/drivers/media/usb/em28xx/em28xx-dvb.c b/drivers/media/usb/em28xx/em28xx-dvb.c
index 3f493e0b0716..b778d8a1983e 100644
--- a/drivers/media/usb/em28xx/em28xx-dvb.c
+++ b/drivers/media/usb/em28xx/em28xx-dvb.c
@@ -199,6 +199,7 @@ static int em28xx_start_streaming(struct em28xx_dvb *dvb)
 	int rc;
 	struct em28xx_i2c_bus *i2c_bus = dvb->adapter.priv;
 	struct em28xx *dev = i2c_bus->dev;
+	struct usb_device *udev = interface_to_usbdev(dev->intf);
 	int dvb_max_packet_size, packet_multiplier, dvb_alt;
 
 	if (dev->dvb_xfer_bulk) {
@@ -217,6 +218,7 @@ static int em28xx_start_streaming(struct em28xx_dvb *dvb)
 		dvb_alt = dev->dvb_alt_isoc;
 	}
 
+	usb_set_interface(udev, dev->ifnum, dvb_alt);
 	rc = em28xx_set_mode(dev, EM28XX_DIGITAL_MODE);
 	if (rc < 0)
 		return rc;
@@ -298,7 +300,6 @@ static int em28xx_dvb_bus_ctrl(struct dvb_frontend *fe, int acquire)
 /* ------------------------------------------------------------------ */
 
 static struct lgdt330x_config em2880_lgdt3303_dev = {
-	.demod_address = 0x0e,
 	.demod_chip = LGDT3303,
 };
 
@@ -1392,7 +1393,7 @@ static int em28174_dvb_init_hauppauge_wintv_dualhd_01595(struct em28xx *dev)
 
 	dvb->i2c_client_tuner = dvb_module_probe("si2157", NULL,
 						 adapter,
-						 0x60, &si2157_config);
+						 addr, &si2157_config);
 	if (!dvb->i2c_client_tuner) {
 		dvb_module_release(dvb->i2c_client_demod);
 		return -ENODEV;
@@ -1470,6 +1471,7 @@ static int em28xx_dvb_init(struct em28xx *dev)
 	case EM2880_BOARD_AMD_ATI_TV_WONDER_HD_600:
 		dvb->fe[0] = dvb_attach(lgdt330x_attach,
 					&em2880_lgdt3303_dev,
+					0x0e,
 					&dev->i2c_adap[dev->def_i2c_bus]);
 		if (em28xx_attach_xc3028(0x61, dev) < 0) {
 			result = -EINVAL;
@@ -1488,6 +1490,7 @@ static int em28xx_dvb_init(struct em28xx *dev)
 	case EM2880_BOARD_HAUPPAUGE_WINTV_HVR_900:
 	case EM2882_BOARD_TERRATEC_HYBRID_XS:
 	case EM2880_BOARD_EMPIRE_DUAL_TV:
+	case EM2882_BOARD_ZOLID_HYBRID_TV_STICK:
 		dvb->fe[0] = dvb_attach(zl10353_attach,
 					&em28xx_zl10353_xc3028_no_i2c_gate,
 					&dev->i2c_adap[dev->def_i2c_bus]);
@@ -1550,6 +1553,7 @@ static int em28xx_dvb_init(struct em28xx *dev)
 	case EM2882_BOARD_KWORLD_ATSC_315U:
 		dvb->fe[0] = dvb_attach(lgdt330x_attach,
 					&em2880_lgdt3303_dev,
+					0x0e,
 					&dev->i2c_adap[dev->def_i2c_bus]);
 		if (dvb->fe[0]) {
 			if (!dvb_attach(simple_tuner_attach, dvb->fe[0],
diff --git a/drivers/media/usb/em28xx/em28xx-v4l.h b/drivers/media/usb/em28xx/em28xx-v4l.h
index 1788dbf9024a..6216cdd182f3 100644
--- a/drivers/media/usb/em28xx/em28xx-v4l.h
+++ b/drivers/media/usb/em28xx/em28xx-v4l.h
@@ -3,7 +3,7 @@
  * em28xx-video.c - driver for Empia EM2800/EM2820/2840 USB
  *		    video capture devices
  *
- * Copyright (C) 2013-2014 Mauro Carvalho Chehab <m.chehab@samsung.com>
+ * Copyright (C) 2013-2014 Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
diff --git a/drivers/media/usb/em28xx/em28xx.h b/drivers/media/usb/em28xx/em28xx.h
index b0378e77ddff..953caac025f2 100644
--- a/drivers/media/usb/em28xx/em28xx.h
+++ b/drivers/media/usb/em28xx/em28xx.h
@@ -148,6 +148,7 @@
 #define EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_DVB  99
 #define EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_01595 100
 #define EM2884_BOARD_TERRATEC_H6		  101
+#define EM2882_BOARD_ZOLID_HYBRID_TV_STICK		102
 
 /* Limits minimum and default number of buffers */
 #define EM28XX_MIN_BUF 4
diff --git a/drivers/media/usb/go7007/go7007-fw.c b/drivers/media/usb/go7007/go7007-fw.c
index 60bf5f0644d1..87b4fc48ef09 100644
--- a/drivers/media/usb/go7007/go7007-fw.c
+++ b/drivers/media/usb/go7007/go7007-fw.c
@@ -1514,7 +1514,10 @@ static int do_special(struct go7007 *go, u16 type, __le16 *code, int space,
 		case V4L2_PIX_FMT_MPEG4:
 			return gen_mpeg4hdr_to_package(go, code, space,
 								framelen);
+		default:
+			break;
 		}
+		break;
 	case SPECIAL_BRC_CTRL:
 		return brctrl_to_package(go, code, space, framelen);
 	case SPECIAL_CONFIG:
diff --git a/drivers/media/usb/go7007/go7007-v4l2.c b/drivers/media/usb/go7007/go7007-v4l2.c
index 98cd57eaf36a..c55c82f70e54 100644
--- a/drivers/media/usb/go7007/go7007-v4l2.c
+++ b/drivers/media/usb/go7007/go7007-v4l2.c
@@ -634,7 +634,7 @@ static int vidioc_enum_input(struct file *file, void *priv,
 	if (inp->index >= go->board_info->num_inputs)
 		return -EINVAL;
 
-	strncpy(inp->name, go->board_info->inputs[inp->index].name,
+	strlcpy(inp->name, go->board_info->inputs[inp->index].name,
 			sizeof(inp->name));
 
 	/* If this board has a tuner, it will be the first input */
diff --git a/drivers/media/usb/gspca/Kconfig b/drivers/media/usb/gspca/Kconfig
index bc9a439745aa..d3b6665c342d 100644
--- a/drivers/media/usb/gspca/Kconfig
+++ b/drivers/media/usb/gspca/Kconfig
@@ -2,6 +2,7 @@ menuconfig USB_GSPCA
 	tristate "GSPCA based webcams"
 	depends on VIDEO_V4L2
 	depends on INPUT || INPUT=n
+	select VIDEOBUF2_VMALLOC
 	default m
 	---help---
 	  Say Y here if you want to enable selecting webcams based
diff --git a/drivers/media/usb/gspca/gspca.c b/drivers/media/usb/gspca/gspca.c
index d29773b8f696..57aa521e16b1 100644
--- a/drivers/media/usb/gspca/gspca.c
+++ b/drivers/media/usb/gspca/gspca.c
@@ -82,32 +82,6 @@ static void PDEBUG_MODE(struct gspca_dev *gspca_dev, int debug, char *txt,
 #define GSPCA_MEMORY_NO 0	/* V4L2_MEMORY_xxx starts from 1 */
 #define GSPCA_MEMORY_READ 7
 
-#define BUF_ALL_FLAGS (V4L2_BUF_FLAG_QUEUED | V4L2_BUF_FLAG_DONE)
-
-/*
- * VMA operations.
- */
-static void gspca_vm_open(struct vm_area_struct *vma)
-{
-	struct gspca_frame *frame = vma->vm_private_data;
-
-	frame->vma_use_count++;
-	frame->v4l2_buf.flags |= V4L2_BUF_FLAG_MAPPED;
-}
-
-static void gspca_vm_close(struct vm_area_struct *vma)
-{
-	struct gspca_frame *frame = vma->vm_private_data;
-
-	if (--frame->vma_use_count <= 0)
-		frame->v4l2_buf.flags &= ~V4L2_BUF_FLAG_MAPPED;
-}
-
-static const struct vm_operations_struct gspca_vm_ops = {
-	.open		= gspca_vm_open,
-	.close		= gspca_vm_close,
-};
-
 /*
  * Input and interrupt endpoint handling functions
  */
@@ -356,7 +330,7 @@ static void isoc_irq(struct urb *urb)
 	struct gspca_dev *gspca_dev = (struct gspca_dev *) urb->context;
 
 	gspca_dbg(gspca_dev, D_PACK, "isoc irq\n");
-	if (!gspca_dev->streaming)
+	if (!vb2_start_streaming_called(&gspca_dev->queue))
 		return;
 	fill_frame(gspca_dev, urb);
 }
@@ -370,7 +344,7 @@ static void bulk_irq(struct urb *urb)
 	int st;
 
 	gspca_dbg(gspca_dev, D_PACK, "bulk irq\n");
-	if (!gspca_dev->streaming)
+	if (!vb2_start_streaming_called(&gspca_dev->queue))
 		return;
 	switch (urb->status) {
 	case 0:
@@ -417,25 +391,24 @@ void gspca_frame_add(struct gspca_dev *gspca_dev,
 			const u8 *data,
 			int len)
 {
-	struct gspca_frame *frame;
-	int i, j;
+	struct gspca_buffer *buf;
+	unsigned long flags;
 
 	gspca_dbg(gspca_dev, D_PACK, "add t:%d l:%d\n",	packet_type, len);
 
-	if (packet_type == FIRST_PACKET) {
-		i = atomic_read(&gspca_dev->fr_i);
+	spin_lock_irqsave(&gspca_dev->qlock, flags);
+	buf = list_first_entry_or_null(&gspca_dev->buf_list,
+				       typeof(*buf), list);
+	spin_unlock_irqrestore(&gspca_dev->qlock, flags);
 
-		/* if there are no queued buffer, discard the whole frame */
-		if (i == atomic_read(&gspca_dev->fr_q)) {
+	if (packet_type == FIRST_PACKET) {
+		/* if there is no queued buffer, discard the whole frame */
+		if (!buf) {
 			gspca_dev->last_packet_type = DISCARD_PACKET;
 			gspca_dev->sequence++;
 			return;
 		}
-		j = gspca_dev->fr_queue[i];
-		frame = &gspca_dev->frame[j];
-		v4l2_get_timestamp(&frame->v4l2_buf.timestamp);
-		frame->v4l2_buf.sequence = gspca_dev->sequence++;
-		gspca_dev->image = frame->data;
+		gspca_dev->image = vb2_plane_vaddr(&buf->vb.vb2_buf, 0);
 		gspca_dev->image_len = 0;
 	} else {
 		switch (gspca_dev->last_packet_type) {
@@ -453,10 +426,10 @@ void gspca_frame_add(struct gspca_dev *gspca_dev,
 
 	/* append the packet to the frame buffer */
 	if (len > 0) {
-		if (gspca_dev->image_len + len > gspca_dev->frsz) {
+		if (gspca_dev->image_len + len > gspca_dev->pixfmt.sizeimage) {
 			gspca_err(gspca_dev, "frame overflow %d > %d\n",
 				  gspca_dev->image_len + len,
-				  gspca_dev->frsz);
+				  gspca_dev->pixfmt.sizeimage);
 			packet_type = DISCARD_PACKET;
 		} else {
 /* !! image is NULL only when last pkt is LAST or DISCARD
@@ -476,93 +449,43 @@ void gspca_frame_add(struct gspca_dev *gspca_dev,
 	 * next first packet, wake up the application and advance
 	 * in the queue */
 	if (packet_type == LAST_PACKET) {
-		i = atomic_read(&gspca_dev->fr_i);
-		j = gspca_dev->fr_queue[i];
-		frame = &gspca_dev->frame[j];
-		frame->v4l2_buf.bytesused = gspca_dev->image_len;
-		frame->v4l2_buf.flags = (frame->v4l2_buf.flags
-					 | V4L2_BUF_FLAG_DONE)
-					& ~V4L2_BUF_FLAG_QUEUED;
-		i = (i + 1) % GSPCA_MAX_FRAMES;
-		atomic_set(&gspca_dev->fr_i, i);
-		wake_up_interruptible(&gspca_dev->wq);	/* event = new frame */
+		spin_lock_irqsave(&gspca_dev->qlock, flags);
+		list_del(&buf->list);
+		spin_unlock_irqrestore(&gspca_dev->qlock, flags);
+		buf->vb.vb2_buf.timestamp = ktime_get_ns();
+		vb2_set_plane_payload(&buf->vb.vb2_buf, 0,
+				      gspca_dev->image_len);
+		buf->vb.sequence = gspca_dev->sequence++;
+		buf->vb.field = V4L2_FIELD_NONE;
 		gspca_dbg(gspca_dev, D_FRAM, "frame complete len:%d\n",
-			  frame->v4l2_buf.bytesused);
+			  gspca_dev->image_len);
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
 		gspca_dev->image = NULL;
 		gspca_dev->image_len = 0;
 	}
 }
 EXPORT_SYMBOL(gspca_frame_add);
 
-static int frame_alloc(struct gspca_dev *gspca_dev, struct file *file,
-			enum v4l2_memory memory, unsigned int count)
-{
-	struct gspca_frame *frame;
-	unsigned int frsz;
-	int i;
-
-	frsz = gspca_dev->pixfmt.sizeimage;
-	gspca_dbg(gspca_dev, D_STREAM, "frame alloc frsz: %d\n", frsz);
-	frsz = PAGE_ALIGN(frsz);
-	if (count >= GSPCA_MAX_FRAMES)
-		count = GSPCA_MAX_FRAMES - 1;
-	gspca_dev->frbuf = vmalloc_32(frsz * count);
-	if (!gspca_dev->frbuf) {
-		pr_err("frame alloc failed\n");
-		return -ENOMEM;
-	}
-	gspca_dev->capt_file = file;
-	gspca_dev->memory = memory;
-	gspca_dev->frsz = frsz;
-	gspca_dev->nframes = count;
-	for (i = 0; i < count; i++) {
-		frame = &gspca_dev->frame[i];
-		frame->v4l2_buf.index = i;
-		frame->v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-		frame->v4l2_buf.flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
-		frame->v4l2_buf.field = V4L2_FIELD_NONE;
-		frame->v4l2_buf.length = frsz;
-		frame->v4l2_buf.memory = memory;
-		frame->v4l2_buf.sequence = 0;
-		frame->data = gspca_dev->frbuf + i * frsz;
-		frame->v4l2_buf.m.offset = i * frsz;
-	}
-	atomic_set(&gspca_dev->fr_q, 0);
-	atomic_set(&gspca_dev->fr_i, 0);
-	gspca_dev->fr_o = 0;
-	return 0;
-}
-
-static void frame_free(struct gspca_dev *gspca_dev)
-{
-	int i;
-
-	gspca_dbg(gspca_dev, D_STREAM, "frame free\n");
-	if (gspca_dev->frbuf != NULL) {
-		vfree(gspca_dev->frbuf);
-		gspca_dev->frbuf = NULL;
-		for (i = 0; i < gspca_dev->nframes; i++)
-			gspca_dev->frame[i].data = NULL;
-	}
-	gspca_dev->nframes = 0;
-	gspca_dev->frsz = 0;
-	gspca_dev->capt_file = NULL;
-	gspca_dev->memory = GSPCA_MEMORY_NO;
-}
-
 static void destroy_urbs(struct gspca_dev *gspca_dev)
 {
 	struct urb *urb;
 	unsigned int i;
 
 	gspca_dbg(gspca_dev, D_STREAM, "kill transfer\n");
+
+	/* Killing all URBs guarantee that no URB completion
+	 * handler is running. Therefore, there shouldn't
+	 * be anyone trying to access gspca_dev->urb[i]
+	 */
+	for (i = 0; i < MAX_NURBS; i++)
+		usb_kill_urb(gspca_dev->urb[i]);
+
+	gspca_dbg(gspca_dev, D_STREAM, "releasing urbs\n");
 	for (i = 0; i < MAX_NURBS; i++) {
 		urb = gspca_dev->urb[i];
-		if (urb == NULL)
-			break;
-
+		if (!urb)
+			continue;
 		gspca_dev->urb[i] = NULL;
-		usb_kill_urb(urb);
 		usb_free_coherent(gspca_dev->dev,
 				  urb->transfer_buffer_length,
 				  urb->transfer_buffer,
@@ -583,22 +506,6 @@ static int gspca_set_alt0(struct gspca_dev *gspca_dev)
 	return ret;
 }
 
-/* Note: both the queue and the usb locks should be held when calling this */
-static void gspca_stream_off(struct gspca_dev *gspca_dev)
-{
-	gspca_dev->streaming = 0;
-	gspca_dev->usb_err = 0;
-	if (gspca_dev->sd_desc->stopN)
-		gspca_dev->sd_desc->stopN(gspca_dev);
-	destroy_urbs(gspca_dev);
-	gspca_input_destroy_urb(gspca_dev);
-	gspca_set_alt0(gspca_dev);
-	gspca_input_create_urb(gspca_dev);
-	if (gspca_dev->sd_desc->stop0)
-		gspca_dev->sd_desc->stop0(gspca_dev);
-	gspca_dbg(gspca_dev, D_STREAM, "stream off OK\n");
-}
-
 /*
  * look for an input transfer endpoint in an alternate setting.
  *
@@ -829,6 +736,23 @@ static int create_urbs(struct gspca_dev *gspca_dev,
 	return 0;
 }
 
+/* Note: both the queue and the usb locks should be held when calling this */
+static void gspca_stream_off(struct gspca_dev *gspca_dev)
+{
+	gspca_dev->streaming = false;
+	gspca_dev->usb_err = 0;
+	if (gspca_dev->sd_desc->stopN)
+		gspca_dev->sd_desc->stopN(gspca_dev);
+	destroy_urbs(gspca_dev);
+	gspca_input_destroy_urb(gspca_dev);
+	gspca_set_alt0(gspca_dev);
+	if (gspca_dev->present)
+		gspca_input_create_urb(gspca_dev);
+	if (gspca_dev->sd_desc->stop0)
+		gspca_dev->sd_desc->stop0(gspca_dev);
+	gspca_dbg(gspca_dev, D_STREAM, "stream off OK\n");
+}
+
 /*
  * start the USB transfer
  */
@@ -844,7 +768,6 @@ static int gspca_init_transfer(struct gspca_dev *gspca_dev)
 	gspca_dev->image = NULL;
 	gspca_dev->image_len = 0;
 	gspca_dev->last_packet_type = DISCARD_PACKET;
-	gspca_dev->sequence = 0;
 
 	gspca_dev->usb_err = 0;
 
@@ -924,8 +847,8 @@ static int gspca_init_transfer(struct gspca_dev *gspca_dev)
 			destroy_urbs(gspca_dev);
 			goto out;
 		}
-		gspca_dev->streaming = 1;
 		v4l2_ctrl_handler_setup(gspca_dev->vdev.ctrl_handler);
+		gspca_dev->streaming = true;
 
 		/* some bulk transfers are started by the subdriver */
 		if (gspca_dev->cam.bulk && gspca_dev->cam.bulk_nurbs == 0)
@@ -1165,11 +1088,9 @@ static int vidioc_try_fmt_vid_cap(struct file *file,
 			      struct v4l2_format *fmt)
 {
 	struct gspca_dev *gspca_dev = video_drvdata(file);
-	int ret;
 
-	ret = try_fmt_vid_cap(gspca_dev, fmt);
-	if (ret < 0)
-		return ret;
+	if (try_fmt_vid_cap(gspca_dev, fmt) < 0)
+		return -EINVAL;
 	return 0;
 }
 
@@ -1177,36 +1098,22 @@ static int vidioc_s_fmt_vid_cap(struct file *file, void *priv,
 			    struct v4l2_format *fmt)
 {
 	struct gspca_dev *gspca_dev = video_drvdata(file);
-	int ret;
-
-	if (mutex_lock_interruptible(&gspca_dev->queue_lock))
-		return -ERESTARTSYS;
+	int mode;
 
-	ret = try_fmt_vid_cap(gspca_dev, fmt);
-	if (ret < 0)
-		goto out;
+	if (vb2_is_busy(&gspca_dev->queue))
+		return -EBUSY;
 
-	if (gspca_dev->nframes != 0
-	    && fmt->fmt.pix.sizeimage > gspca_dev->frsz) {
-		ret = -EINVAL;
-		goto out;
-	}
+	mode = try_fmt_vid_cap(gspca_dev, fmt);
+	if (mode < 0)
+		return -EINVAL;
 
-	if (gspca_dev->streaming) {
-		ret = -EBUSY;
-		goto out;
-	}
-	gspca_dev->curr_mode = ret;
+	gspca_dev->curr_mode = mode;
 	if (gspca_dev->sd_desc->try_fmt)
 		/* subdriver try_fmt can modify format parameters */
 		gspca_dev->pixfmt = fmt->fmt.pix;
 	else
-		gspca_dev->pixfmt = gspca_dev->cam.cam_mode[ret];
-
-	ret = 0;
-out:
-	mutex_unlock(&gspca_dev->queue_lock);
-	return ret;
+		gspca_dev->pixfmt = gspca_dev->cam.cam_mode[mode];
+	return 0;
 }
 
 static int vidioc_enum_framesizes(struct file *file, void *priv,
@@ -1281,53 +1188,6 @@ static void gspca_release(struct v4l2_device *v4l2_device)
 	kfree(gspca_dev);
 }
 
-static int dev_open(struct file *file)
-{
-	struct gspca_dev *gspca_dev = video_drvdata(file);
-	int ret;
-
-	gspca_dbg(gspca_dev, D_STREAM, "[%s] open\n", current->comm);
-
-	/* protect the subdriver against rmmod */
-	if (!try_module_get(gspca_dev->module))
-		return -ENODEV;
-
-	ret = v4l2_fh_open(file);
-	if (ret)
-		module_put(gspca_dev->module);
-	return ret;
-}
-
-static int dev_close(struct file *file)
-{
-	struct gspca_dev *gspca_dev = video_drvdata(file);
-
-	gspca_dbg(gspca_dev, D_STREAM, "[%s] close\n", current->comm);
-
-	/* Needed for gspca_stream_off, always lock before queue_lock! */
-	if (mutex_lock_interruptible(&gspca_dev->usb_lock))
-		return -ERESTARTSYS;
-
-	if (mutex_lock_interruptible(&gspca_dev->queue_lock)) {
-		mutex_unlock(&gspca_dev->usb_lock);
-		return -ERESTARTSYS;
-	}
-
-	/* if the file did the capture, free the streaming resources */
-	if (gspca_dev->capt_file == file) {
-		if (gspca_dev->streaming)
-			gspca_stream_off(gspca_dev);
-		frame_free(gspca_dev);
-	}
-	module_put(gspca_dev->module);
-	mutex_unlock(&gspca_dev->queue_lock);
-	mutex_unlock(&gspca_dev->usb_lock);
-
-	gspca_dbg(gspca_dev, D_STREAM, "close done\n");
-
-	return v4l2_fh_release(file);
-}
-
 static int vidioc_querycap(struct file *file, void  *priv,
 			   struct v4l2_capability *cap)
 {
@@ -1377,167 +1237,9 @@ static int vidioc_s_input(struct file *file, void *priv, unsigned int i)
 {
 	if (i > 0)
 		return -EINVAL;
-	return (0);
-}
-
-static int vidioc_reqbufs(struct file *file, void *priv,
-			  struct v4l2_requestbuffers *rb)
-{
-	struct gspca_dev *gspca_dev = video_drvdata(file);
-	int i, ret = 0, streaming;
-
-	i = rb->memory;			/* (avoid compilation warning) */
-	switch (i) {
-	case GSPCA_MEMORY_READ:			/* (internal call) */
-	case V4L2_MEMORY_MMAP:
-	case V4L2_MEMORY_USERPTR:
-		break;
-	default:
-		return -EINVAL;
-	}
-	if (mutex_lock_interruptible(&gspca_dev->queue_lock))
-		return -ERESTARTSYS;
-
-	if (gspca_dev->memory != GSPCA_MEMORY_NO
-	    && gspca_dev->memory != GSPCA_MEMORY_READ
-	    && gspca_dev->memory != rb->memory) {
-		ret = -EBUSY;
-		goto out;
-	}
-
-	/* only one file may do the capture */
-	if (gspca_dev->capt_file != NULL
-	    && gspca_dev->capt_file != file) {
-		ret = -EBUSY;
-		goto out;
-	}
-
-	/* if allocated, the buffers must not be mapped */
-	for (i = 0; i < gspca_dev->nframes; i++) {
-		if (gspca_dev->frame[i].vma_use_count) {
-			ret = -EBUSY;
-			goto out;
-		}
-	}
-
-	/* stop streaming */
-	streaming = gspca_dev->streaming;
-	if (streaming) {
-		gspca_stream_off(gspca_dev);
-
-		/* Don't restart the stream when switching from read
-		 * to mmap mode */
-		if (gspca_dev->memory == GSPCA_MEMORY_READ)
-			streaming = 0;
-	}
-
-	/* free the previous allocated buffers, if any */
-	if (gspca_dev->nframes != 0)
-		frame_free(gspca_dev);
-	if (rb->count == 0)			/* unrequest */
-		goto out;
-	ret = frame_alloc(gspca_dev, file, rb->memory, rb->count);
-	if (ret == 0) {
-		rb->count = gspca_dev->nframes;
-		if (streaming)
-			ret = gspca_init_transfer(gspca_dev);
-	}
-out:
-	mutex_unlock(&gspca_dev->queue_lock);
-	gspca_dbg(gspca_dev, D_STREAM, "reqbufs st:%d c:%d\n", ret, rb->count);
-	return ret;
-}
-
-static int vidioc_querybuf(struct file *file, void *priv,
-			   struct v4l2_buffer *v4l2_buf)
-{
-	struct gspca_dev *gspca_dev = video_drvdata(file);
-	struct gspca_frame *frame;
-
-	if (v4l2_buf->index >= gspca_dev->nframes)
-		return -EINVAL;
-
-	frame = &gspca_dev->frame[v4l2_buf->index];
-	memcpy(v4l2_buf, &frame->v4l2_buf, sizeof *v4l2_buf);
 	return 0;
 }
 
-static int vidioc_streamon(struct file *file, void *priv,
-			   enum v4l2_buf_type buf_type)
-{
-	struct gspca_dev *gspca_dev = video_drvdata(file);
-	int ret;
-
-	if (buf_type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return -EINVAL;
-	if (mutex_lock_interruptible(&gspca_dev->queue_lock))
-		return -ERESTARTSYS;
-
-	/* check the capture file */
-	if (gspca_dev->capt_file != file) {
-		ret = -EBUSY;
-		goto out;
-	}
-
-	if (gspca_dev->nframes == 0
-	    || !(gspca_dev->frame[0].v4l2_buf.flags & V4L2_BUF_FLAG_QUEUED)) {
-		ret = -EINVAL;
-		goto out;
-	}
-	if (!gspca_dev->streaming) {
-		ret = gspca_init_transfer(gspca_dev);
-		if (ret < 0)
-			goto out;
-	}
-	PDEBUG_MODE(gspca_dev, D_STREAM, "stream on OK",
-		    gspca_dev->pixfmt.pixelformat,
-		    gspca_dev->pixfmt.width, gspca_dev->pixfmt.height);
-	ret = 0;
-out:
-	mutex_unlock(&gspca_dev->queue_lock);
-	return ret;
-}
-
-static int vidioc_streamoff(struct file *file, void *priv,
-				enum v4l2_buf_type buf_type)
-{
-	struct gspca_dev *gspca_dev = video_drvdata(file);
-	int i, ret;
-
-	if (buf_type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return -EINVAL;
-
-	if (mutex_lock_interruptible(&gspca_dev->queue_lock))
-		return -ERESTARTSYS;
-
-	if (!gspca_dev->streaming) {
-		ret = 0;
-		goto out;
-	}
-
-	/* check the capture file */
-	if (gspca_dev->capt_file != file) {
-		ret = -EBUSY;
-		goto out;
-	}
-
-	/* stop streaming */
-	gspca_stream_off(gspca_dev);
-	/* In case another thread is waiting in dqbuf */
-	wake_up_interruptible(&gspca_dev->wq);
-
-	/* empty the transfer queues */
-	for (i = 0; i < gspca_dev->nframes; i++)
-		gspca_dev->frame[i].v4l2_buf.flags &= ~BUF_ALL_FLAGS;
-	atomic_set(&gspca_dev->fr_q, 0);
-	atomic_set(&gspca_dev->fr_i, 0);
-	gspca_dev->fr_o = 0;
-	ret = 0;
-out:
-	mutex_unlock(&gspca_dev->queue_lock);
-	return ret;
-}
-
 static int vidioc_g_jpegcomp(struct file *file, void *priv,
 			struct v4l2_jpegcompression *jpegcomp)
 {
@@ -1561,449 +1263,167 @@ static int vidioc_g_parm(struct file *filp, void *priv,
 {
 	struct gspca_dev *gspca_dev = video_drvdata(filp);
 
-	parm->parm.capture.readbuffers = gspca_dev->nbufread;
+	parm->parm.capture.readbuffers = gspca_dev->queue.min_buffers_needed;
 
-	if (gspca_dev->sd_desc->get_streamparm) {
-		gspca_dev->usb_err = 0;
-		gspca_dev->sd_desc->get_streamparm(gspca_dev, parm);
-		return gspca_dev->usb_err;
-	}
-	return 0;
+	if (!gspca_dev->sd_desc->get_streamparm)
+		return 0;
+
+	parm->parm.capture.capability = V4L2_CAP_TIMEPERFRAME;
+	gspca_dev->usb_err = 0;
+	gspca_dev->sd_desc->get_streamparm(gspca_dev, parm);
+	return gspca_dev->usb_err;
 }
 
 static int vidioc_s_parm(struct file *filp, void *priv,
 			struct v4l2_streamparm *parm)
 {
 	struct gspca_dev *gspca_dev = video_drvdata(filp);
-	unsigned int n;
-
-	n = parm->parm.capture.readbuffers;
-	if (n == 0 || n >= GSPCA_MAX_FRAMES)
-		parm->parm.capture.readbuffers = gspca_dev->nbufread;
-	else
-		gspca_dev->nbufread = n;
-
-	if (gspca_dev->sd_desc->set_streamparm) {
-		gspca_dev->usb_err = 0;
-		gspca_dev->sd_desc->set_streamparm(gspca_dev, parm);
-		return gspca_dev->usb_err;
-	}
-
-	return 0;
-}
-
-static int dev_mmap(struct file *file, struct vm_area_struct *vma)
-{
-	struct gspca_dev *gspca_dev = video_drvdata(file);
-	struct gspca_frame *frame;
-	struct page *page;
-	unsigned long addr, start, size;
-	int i, ret;
-
-	start = vma->vm_start;
-	size = vma->vm_end - vma->vm_start;
-	gspca_dbg(gspca_dev, D_STREAM, "mmap start:%08x size:%d\n",
-		  (int) start, (int)size);
-
-	if (mutex_lock_interruptible(&gspca_dev->queue_lock))
-		return -ERESTARTSYS;
-	if (gspca_dev->capt_file != file) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	frame = NULL;
-	for (i = 0; i < gspca_dev->nframes; ++i) {
-		if (gspca_dev->frame[i].v4l2_buf.memory != V4L2_MEMORY_MMAP) {
-			gspca_dbg(gspca_dev, D_STREAM, "mmap bad memory type\n");
-			break;
-		}
-		if ((gspca_dev->frame[i].v4l2_buf.m.offset >> PAGE_SHIFT)
-						== vma->vm_pgoff) {
-			frame = &gspca_dev->frame[i];
-			break;
-		}
-	}
-	if (frame == NULL) {
-		gspca_dbg(gspca_dev, D_STREAM, "mmap no frame buffer found\n");
-		ret = -EINVAL;
-		goto out;
-	}
-	if (size != frame->v4l2_buf.length) {
-		gspca_dbg(gspca_dev, D_STREAM, "mmap bad size\n");
-		ret = -EINVAL;
-		goto out;
-	}
 
-	/*
-	 * - VM_IO marks the area as being a mmaped region for I/O to a
-	 *   device. It also prevents the region from being core dumped.
-	 */
-	vma->vm_flags |= VM_IO;
+	parm->parm.capture.readbuffers = gspca_dev->queue.min_buffers_needed;
 
-	addr = (unsigned long) frame->data;
-	while (size > 0) {
-		page = vmalloc_to_page((void *) addr);
-		ret = vm_insert_page(vma, start, page);
-		if (ret < 0)
-			goto out;
-		start += PAGE_SIZE;
-		addr += PAGE_SIZE;
-		size -= PAGE_SIZE;
+	if (!gspca_dev->sd_desc->set_streamparm) {
+		parm->parm.capture.capability = 0;
+		return 0;
 	}
 
-	vma->vm_ops = &gspca_vm_ops;
-	vma->vm_private_data = frame;
-	gspca_vm_open(vma);
-	ret = 0;
-out:
-	mutex_unlock(&gspca_dev->queue_lock);
-	return ret;
+	parm->parm.capture.capability = V4L2_CAP_TIMEPERFRAME;
+	gspca_dev->usb_err = 0;
+	gspca_dev->sd_desc->set_streamparm(gspca_dev, parm);
+	return gspca_dev->usb_err;
 }
 
-static int frame_ready_nolock(struct gspca_dev *gspca_dev, struct file *file,
-				enum v4l2_memory memory)
+static int gspca_queue_setup(struct vb2_queue *vq,
+			     unsigned int *nbuffers, unsigned int *nplanes,
+			     unsigned int sizes[], struct device *alloc_devs[])
 {
-	if (!gspca_dev->present)
-		return -ENODEV;
-	if (gspca_dev->capt_file != file || gspca_dev->memory != memory ||
-			!gspca_dev->streaming)
-		return -EINVAL;
+	struct gspca_dev *gspca_dev = vb2_get_drv_priv(vq);
 
-	/* check if a frame is ready */
-	return gspca_dev->fr_o != atomic_read(&gspca_dev->fr_i);
+	if (*nplanes)
+		return sizes[0] < gspca_dev->pixfmt.sizeimage ? -EINVAL : 0;
+	*nplanes = 1;
+	sizes[0] = gspca_dev->pixfmt.sizeimage;
+	return 0;
 }
 
-static int frame_ready(struct gspca_dev *gspca_dev, struct file *file,
-			enum v4l2_memory memory)
+static int gspca_buffer_prepare(struct vb2_buffer *vb)
 {
-	int ret;
+	struct gspca_dev *gspca_dev = vb2_get_drv_priv(vb->vb2_queue);
+	unsigned long size = gspca_dev->pixfmt.sizeimage;
 
-	if (mutex_lock_interruptible(&gspca_dev->queue_lock))
-		return -ERESTARTSYS;
-	ret = frame_ready_nolock(gspca_dev, file, memory);
-	mutex_unlock(&gspca_dev->queue_lock);
-	return ret;
+	if (vb2_plane_size(vb, 0) < size) {
+		gspca_err(gspca_dev, "buffer too small (%lu < %lu)\n",
+			 vb2_plane_size(vb, 0), size);
+		return -EINVAL;
+	}
+	return 0;
 }
 
-/*
- * dequeue a video buffer
- *
- * If nonblock_ing is false, block until a buffer is available.
- */
-static int vidioc_dqbuf(struct file *file, void *priv,
-			struct v4l2_buffer *v4l2_buf)
+static void gspca_buffer_finish(struct vb2_buffer *vb)
 {
-	struct gspca_dev *gspca_dev = video_drvdata(file);
-	struct gspca_frame *frame;
-	int i, j, ret;
-
-	gspca_dbg(gspca_dev, D_FRAM, "dqbuf\n");
-
-	if (mutex_lock_interruptible(&gspca_dev->queue_lock))
-		return -ERESTARTSYS;
+	struct gspca_dev *gspca_dev = vb2_get_drv_priv(vb->vb2_queue);
 
-	for (;;) {
-		ret = frame_ready_nolock(gspca_dev, file, v4l2_buf->memory);
-		if (ret < 0)
-			goto out;
-		if (ret > 0)
-			break;
-
-		mutex_unlock(&gspca_dev->queue_lock);
-
-		if (file->f_flags & O_NONBLOCK)
-			return -EAGAIN;
-
-		/* wait till a frame is ready */
-		ret = wait_event_interruptible_timeout(gspca_dev->wq,
-			frame_ready(gspca_dev, file, v4l2_buf->memory),
-			msecs_to_jiffies(3000));
-		if (ret < 0)
-			return ret;
-		if (ret == 0)
-			return -EIO;
-
-		if (mutex_lock_interruptible(&gspca_dev->queue_lock))
-			return -ERESTARTSYS;
-	}
-
-	i = gspca_dev->fr_o;
-	j = gspca_dev->fr_queue[i];
-	frame = &gspca_dev->frame[j];
-
-	gspca_dev->fr_o = (i + 1) % GSPCA_MAX_FRAMES;
-
-	frame->v4l2_buf.flags &= ~V4L2_BUF_FLAG_DONE;
-	memcpy(v4l2_buf, &frame->v4l2_buf, sizeof *v4l2_buf);
-	gspca_dbg(gspca_dev, D_FRAM, "dqbuf %d\n", j);
-	ret = 0;
-
-	if (gspca_dev->memory == V4L2_MEMORY_USERPTR) {
-		if (copy_to_user((__u8 __user *) frame->v4l2_buf.m.userptr,
-				 frame->data,
-				 frame->v4l2_buf.bytesused)) {
-			gspca_err(gspca_dev, "dqbuf cp to user failed\n");
-			ret = -EFAULT;
-		}
-	}
-out:
-	mutex_unlock(&gspca_dev->queue_lock);
-
-	if (ret == 0 && gspca_dev->sd_desc->dq_callback) {
-		mutex_lock(&gspca_dev->usb_lock);
-		gspca_dev->usb_err = 0;
-		if (gspca_dev->present)
-			gspca_dev->sd_desc->dq_callback(gspca_dev);
-		mutex_unlock(&gspca_dev->usb_lock);
-	}
+	if (!gspca_dev->sd_desc->dq_callback)
+		return;
 
-	return ret;
+	gspca_dev->usb_err = 0;
+	if (gspca_dev->present)
+		gspca_dev->sd_desc->dq_callback(gspca_dev);
 }
 
-/*
- * queue a video buffer
- *
- * Attempting to queue a buffer that has already been
- * queued will return -EINVAL.
- */
-static int vidioc_qbuf(struct file *file, void *priv,
-			struct v4l2_buffer *v4l2_buf)
+static void gspca_buffer_queue(struct vb2_buffer *vb)
 {
-	struct gspca_dev *gspca_dev = video_drvdata(file);
-	struct gspca_frame *frame;
-	int i, index, ret;
-
-	gspca_dbg(gspca_dev, D_FRAM, "qbuf %d\n", v4l2_buf->index);
-
-	if (mutex_lock_interruptible(&gspca_dev->queue_lock))
-		return -ERESTARTSYS;
-
-	index = v4l2_buf->index;
-	if ((unsigned) index >= gspca_dev->nframes) {
-		gspca_dbg(gspca_dev, D_FRAM,
-			  "qbuf idx %d >= %d\n", index, gspca_dev->nframes);
-		ret = -EINVAL;
-		goto out;
-	}
-	if (v4l2_buf->memory != gspca_dev->memory) {
-		gspca_dbg(gspca_dev, D_FRAM, "qbuf bad memory type\n");
-		ret = -EINVAL;
-		goto out;
-	}
-
-	frame = &gspca_dev->frame[index];
-	if (frame->v4l2_buf.flags & BUF_ALL_FLAGS) {
-		gspca_dbg(gspca_dev, D_FRAM, "qbuf bad state\n");
-		ret = -EINVAL;
-		goto out;
-	}
-
-	frame->v4l2_buf.flags |= V4L2_BUF_FLAG_QUEUED;
-
-	if (frame->v4l2_buf.memory == V4L2_MEMORY_USERPTR) {
-		frame->v4l2_buf.m.userptr = v4l2_buf->m.userptr;
-		frame->v4l2_buf.length = v4l2_buf->length;
-	}
-
-	/* put the buffer in the 'queued' queue */
-	i = atomic_read(&gspca_dev->fr_q);
-	gspca_dev->fr_queue[i] = index;
-	atomic_set(&gspca_dev->fr_q, (i + 1) % GSPCA_MAX_FRAMES);
+	struct gspca_dev *gspca_dev = vb2_get_drv_priv(vb->vb2_queue);
+	struct gspca_buffer *buf = to_gspca_buffer(vb);
+	unsigned long flags;
 
-	v4l2_buf->flags |= V4L2_BUF_FLAG_QUEUED;
-	v4l2_buf->flags &= ~V4L2_BUF_FLAG_DONE;
-	ret = 0;
-out:
-	mutex_unlock(&gspca_dev->queue_lock);
-	return ret;
+	spin_lock_irqsave(&gspca_dev->qlock, flags);
+	list_add_tail(&buf->list, &gspca_dev->buf_list);
+	spin_unlock_irqrestore(&gspca_dev->qlock, flags);
 }
 
-/*
- * allocate the resources for read()
- */
-static int read_alloc(struct gspca_dev *gspca_dev,
-			struct file *file)
+static void gspca_return_all_buffers(struct gspca_dev *gspca_dev,
+				     enum vb2_buffer_state state)
 {
-	struct v4l2_buffer v4l2_buf;
-	int i, ret;
+	struct gspca_buffer *buf, *node;
+	unsigned long flags;
 
-	gspca_dbg(gspca_dev, D_STREAM, "read alloc\n");
-
-	if (mutex_lock_interruptible(&gspca_dev->usb_lock))
-		return -ERESTARTSYS;
-
-	if (gspca_dev->nframes == 0) {
-		struct v4l2_requestbuffers rb;
-
-		memset(&rb, 0, sizeof rb);
-		rb.count = gspca_dev->nbufread;
-		rb.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-		rb.memory = GSPCA_MEMORY_READ;
-		ret = vidioc_reqbufs(file, gspca_dev, &rb);
-		if (ret != 0) {
-			gspca_dbg(gspca_dev, D_STREAM, "read reqbuf err %d\n",
-				  ret);
-			goto out;
-		}
-		memset(&v4l2_buf, 0, sizeof v4l2_buf);
-		v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-		v4l2_buf.memory = GSPCA_MEMORY_READ;
-		for (i = 0; i < gspca_dev->nbufread; i++) {
-			v4l2_buf.index = i;
-			ret = vidioc_qbuf(file, gspca_dev, &v4l2_buf);
-			if (ret != 0) {
-				gspca_dbg(gspca_dev, D_STREAM, "read qbuf err: %d\n",
-					  ret);
-				goto out;
-			}
-		}
+	spin_lock_irqsave(&gspca_dev->qlock, flags);
+	list_for_each_entry_safe(buf, node, &gspca_dev->buf_list, list) {
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
+		list_del(&buf->list);
 	}
-
-	/* start streaming */
-	ret = vidioc_streamon(file, gspca_dev, V4L2_BUF_TYPE_VIDEO_CAPTURE);
-	if (ret != 0)
-		gspca_dbg(gspca_dev, D_STREAM, "read streamon err %d\n", ret);
-out:
-	mutex_unlock(&gspca_dev->usb_lock);
-	return ret;
+	spin_unlock_irqrestore(&gspca_dev->qlock, flags);
 }
 
-static __poll_t dev_poll(struct file *file, poll_table *wait)
+static int gspca_start_streaming(struct vb2_queue *vq, unsigned int count)
 {
-	struct gspca_dev *gspca_dev = video_drvdata(file);
-	__poll_t req_events = poll_requested_events(wait);
-	__poll_t ret = 0;
-
-	gspca_dbg(gspca_dev, D_FRAM, "poll\n");
-
-	if (req_events & EPOLLPRI)
-		ret |= v4l2_ctrl_poll(file, wait);
-
-	if (req_events & (EPOLLIN | EPOLLRDNORM)) {
-		/* if reqbufs is not done, the user would use read() */
-		if (gspca_dev->memory == GSPCA_MEMORY_NO) {
-			if (read_alloc(gspca_dev, file) != 0) {
-				ret |= EPOLLERR;
-				goto out;
-			}
-		}
-
-		poll_wait(file, &gspca_dev->wq, wait);
-
-		/* check if an image has been received */
-		if (mutex_lock_interruptible(&gspca_dev->queue_lock) != 0) {
-			ret |= EPOLLERR;
-			goto out;
-		}
-		if (gspca_dev->fr_o != atomic_read(&gspca_dev->fr_i))
-			ret |= EPOLLIN | EPOLLRDNORM;
-		mutex_unlock(&gspca_dev->queue_lock);
-	}
+	struct gspca_dev *gspca_dev = vb2_get_drv_priv(vq);
+	int ret;
 
-out:
-	if (!gspca_dev->present)
-		ret |= EPOLLHUP;
+	gspca_dev->sequence = 0;
 
+	ret = gspca_init_transfer(gspca_dev);
+	if (ret)
+		gspca_return_all_buffers(gspca_dev, VB2_BUF_STATE_QUEUED);
 	return ret;
 }
 
-static ssize_t dev_read(struct file *file, char __user *data,
-		    size_t count, loff_t *ppos)
+static void gspca_stop_streaming(struct vb2_queue *vq)
 {
-	struct gspca_dev *gspca_dev = video_drvdata(file);
-	struct gspca_frame *frame;
-	struct v4l2_buffer v4l2_buf;
-	struct timeval timestamp;
-	int n, ret, ret2;
-
-	gspca_dbg(gspca_dev, D_FRAM, "read (%zd)\n", count);
-	if (gspca_dev->memory == GSPCA_MEMORY_NO) { /* first time ? */
-		ret = read_alloc(gspca_dev, file);
-		if (ret != 0)
-			return ret;
-	}
+	struct gspca_dev *gspca_dev = vb2_get_drv_priv(vq);
 
-	/* get a frame */
-	v4l2_get_timestamp(&timestamp);
-	timestamp.tv_sec--;
-	n = 2;
-	for (;;) {
-		memset(&v4l2_buf, 0, sizeof v4l2_buf);
-		v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-		v4l2_buf.memory = GSPCA_MEMORY_READ;
-		ret = vidioc_dqbuf(file, gspca_dev, &v4l2_buf);
-		if (ret != 0) {
-			gspca_dbg(gspca_dev, D_STREAM, "read dqbuf err %d\n",
-				  ret);
-			return ret;
-		}
-
-		/* if the process slept for more than 1 second,
-		 * get a newer frame */
-		frame = &gspca_dev->frame[v4l2_buf.index];
-		if (--n < 0)
-			break;			/* avoid infinite loop */
-		if (frame->v4l2_buf.timestamp.tv_sec >= timestamp.tv_sec)
-			break;
-		ret = vidioc_qbuf(file, gspca_dev, &v4l2_buf);
-		if (ret != 0) {
-			gspca_dbg(gspca_dev, D_STREAM, "read qbuf err %d\n",
-				  ret);
-			return ret;
-		}
-	}
+	gspca_stream_off(gspca_dev);
 
-	/* copy the frame */
-	if (count > frame->v4l2_buf.bytesused)
-		count = frame->v4l2_buf.bytesused;
-	ret = copy_to_user(data, frame->data, count);
-	if (ret != 0) {
-		gspca_err(gspca_dev, "read cp to user lack %d / %zd\n",
-			  ret, count);
-		ret = -EFAULT;
-		goto out;
-	}
-	ret = count;
-out:
-	/* in each case, requeue the buffer */
-	ret2 = vidioc_qbuf(file, gspca_dev, &v4l2_buf);
-	if (ret2 != 0)
-		return ret2;
-	return ret;
+	/* Release all active buffers */
+	gspca_return_all_buffers(gspca_dev, VB2_BUF_STATE_ERROR);
 }
 
+static const struct vb2_ops gspca_qops = {
+	.queue_setup		= gspca_queue_setup,
+	.buf_prepare		= gspca_buffer_prepare,
+	.buf_finish		= gspca_buffer_finish,
+	.buf_queue		= gspca_buffer_queue,
+	.start_streaming	= gspca_start_streaming,
+	.stop_streaming		= gspca_stop_streaming,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
+};
+
 static const struct v4l2_file_operations dev_fops = {
 	.owner = THIS_MODULE,
-	.open = dev_open,
-	.release = dev_close,
-	.read = dev_read,
-	.mmap = dev_mmap,
+	.open = v4l2_fh_open,
+	.release = vb2_fop_release,
 	.unlocked_ioctl = video_ioctl2,
-	.poll	= dev_poll,
+	.read = vb2_fop_read,
+	.mmap = vb2_fop_mmap,
+	.poll = vb2_fop_poll,
 };
 
 static const struct v4l2_ioctl_ops dev_ioctl_ops = {
 	.vidioc_querycap	= vidioc_querycap,
-	.vidioc_dqbuf		= vidioc_dqbuf,
-	.vidioc_qbuf		= vidioc_qbuf,
 	.vidioc_enum_fmt_vid_cap = vidioc_enum_fmt_vid_cap,
 	.vidioc_try_fmt_vid_cap	= vidioc_try_fmt_vid_cap,
 	.vidioc_g_fmt_vid_cap	= vidioc_g_fmt_vid_cap,
 	.vidioc_s_fmt_vid_cap	= vidioc_s_fmt_vid_cap,
-	.vidioc_streamon	= vidioc_streamon,
 	.vidioc_enum_input	= vidioc_enum_input,
 	.vidioc_g_input		= vidioc_g_input,
 	.vidioc_s_input		= vidioc_s_input,
-	.vidioc_reqbufs		= vidioc_reqbufs,
-	.vidioc_querybuf	= vidioc_querybuf,
-	.vidioc_streamoff	= vidioc_streamoff,
 	.vidioc_g_jpegcomp	= vidioc_g_jpegcomp,
 	.vidioc_s_jpegcomp	= vidioc_s_jpegcomp,
 	.vidioc_g_parm		= vidioc_g_parm,
 	.vidioc_s_parm		= vidioc_s_parm,
 	.vidioc_enum_framesizes = vidioc_enum_framesizes,
 	.vidioc_enum_frameintervals = vidioc_enum_frameintervals,
+
+	.vidioc_reqbufs		= vb2_ioctl_reqbufs,
+	.vidioc_create_bufs	= vb2_ioctl_create_bufs,
+	.vidioc_querybuf	= vb2_ioctl_querybuf,
+	.vidioc_qbuf		= vb2_ioctl_qbuf,
+	.vidioc_dqbuf		= vb2_ioctl_dqbuf,
+	.vidioc_expbuf		= vb2_ioctl_expbuf,
+	.vidioc_streamon	= vb2_ioctl_streamon,
+	.vidioc_streamoff	= vb2_ioctl_streamoff,
+
 #ifdef CONFIG_VIDEO_ADV_DEBUG
 	.vidioc_g_chip_info	= vidioc_g_chip_info,
 	.vidioc_g_register	= vidioc_g_register,
@@ -2034,6 +1454,7 @@ int gspca_dev_probe2(struct usb_interface *intf,
 {
 	struct gspca_dev *gspca_dev;
 	struct usb_device *dev = interface_to_usbdev(intf);
+	struct vb2_queue *q;
 	int ret;
 
 	pr_info("%s-" GSPCA_VERSION " probing %04x:%04x\n",
@@ -2078,20 +1499,37 @@ int gspca_dev_probe2(struct usb_interface *intf,
 	ret = v4l2_device_register(&intf->dev, &gspca_dev->v4l2_dev);
 	if (ret)
 		goto out;
+	gspca_dev->present = true;
 	gspca_dev->sd_desc = sd_desc;
-	gspca_dev->nbufread = 2;
 	gspca_dev->empty_packet = -1;	/* don't check the empty packets */
 	gspca_dev->vdev = gspca_template;
 	gspca_dev->vdev.v4l2_dev = &gspca_dev->v4l2_dev;
 	video_set_drvdata(&gspca_dev->vdev, gspca_dev);
 	gspca_dev->module = module;
-	gspca_dev->present = 1;
 
 	mutex_init(&gspca_dev->usb_lock);
 	gspca_dev->vdev.lock = &gspca_dev->usb_lock;
-	mutex_init(&gspca_dev->queue_lock);
 	init_waitqueue_head(&gspca_dev->wq);
 
+	/* Initialize the vb2 queue */
+	q = &gspca_dev->queue;
+	q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	q->io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF | VB2_READ;
+	q->drv_priv = gspca_dev;
+	q->buf_struct_size = sizeof(struct gspca_buffer);
+	q->ops = &gspca_qops;
+	q->mem_ops = &vb2_vmalloc_memops;
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->min_buffers_needed = 2;
+	q->lock = &gspca_dev->usb_lock;
+	ret = vb2_queue_init(q);
+	if (ret)
+		goto out;
+	gspca_dev->vdev.queue = q;
+
+	INIT_LIST_HEAD(&gspca_dev->buf_list);
+	spin_lock_init(&gspca_dev->qlock);
+
 	/* configure the subdriver and initialize the USB device */
 	ret = sd_desc->config(gspca_dev, id);
 	if (ret < 0)
@@ -2109,14 +1547,6 @@ int gspca_dev_probe2(struct usb_interface *intf,
 	if (ret)
 		goto out;
 
-	/*
-	 * Don't take usb_lock for these ioctls. This improves latency if
-	 * usb_lock is taken for a long time, e.g. when changing a control
-	 * value, and a new frame is ready to be dequeued.
-	 */
-	v4l2_disable_ioctl_locking(&gspca_dev->vdev, VIDIOC_DQBUF);
-	v4l2_disable_ioctl_locking(&gspca_dev->vdev, VIDIOC_QBUF);
-	v4l2_disable_ioctl_locking(&gspca_dev->vdev, VIDIOC_QUERYBUF);
 #ifdef CONFIG_VIDEO_ADV_DEBUG
 	if (!gspca_dev->sd_desc->get_register)
 		v4l2_disable_ioctl(&gspca_dev->vdev, VIDIOC_DBG_G_REGISTER);
@@ -2198,24 +1628,17 @@ void gspca_disconnect(struct usb_interface *intf)
 		  video_device_node_name(&gspca_dev->vdev));
 
 	mutex_lock(&gspca_dev->usb_lock);
+	gspca_dev->present = false;
 
-	gspca_dev->present = 0;
-	destroy_urbs(gspca_dev);
+	vb2_queue_error(&gspca_dev->queue);
 
 #if IS_ENABLED(CONFIG_INPUT)
-	gspca_input_destroy_urb(gspca_dev);
 	input_dev = gspca_dev->input_dev;
 	if (input_dev) {
 		gspca_dev->input_dev = NULL;
 		input_unregister_device(input_dev);
 	}
 #endif
-	/* Free subdriver's streaming resources / stop sd workqueue(s) */
-	if (gspca_dev->sd_desc->stop0 && gspca_dev->streaming)
-		gspca_dev->sd_desc->stop0(gspca_dev);
-	gspca_dev->streaming = 0;
-	gspca_dev->dev = NULL;
-	wake_up_interruptible(&gspca_dev->wq);
 
 	v4l2_device_disconnect(&gspca_dev->v4l2_dev);
 	video_unregister_device(&gspca_dev->vdev);
@@ -2234,7 +1657,7 @@ int gspca_suspend(struct usb_interface *intf, pm_message_t message)
 
 	gspca_input_destroy_urb(gspca_dev);
 
-	if (!gspca_dev->streaming)
+	if (!vb2_start_streaming_called(&gspca_dev->queue))
 		return 0;
 
 	mutex_lock(&gspca_dev->usb_lock);
@@ -2266,8 +1689,7 @@ int gspca_resume(struct usb_interface *intf)
 	 * only write to the device registers on s_ctrl when streaming ->
 	 * Clear streaming to avoid setting all ctrls twice.
 	 */
-	streaming = gspca_dev->streaming;
-	gspca_dev->streaming = 0;
+	streaming = vb2_start_streaming_called(&gspca_dev->queue);
 	if (streaming)
 		ret = gspca_init_transfer(gspca_dev);
 	else
diff --git a/drivers/media/usb/gspca/gspca.h b/drivers/media/usb/gspca/gspca.h
index 249cb38a542f..b0ced2e14006 100644
--- a/drivers/media/usb/gspca/gspca.h
+++ b/drivers/media/usb/gspca/gspca.h
@@ -9,6 +9,8 @@
 #include <media/v4l2-common.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/videobuf2-vmalloc.h>
 #include <linux/mutex.h>
 
 
@@ -138,19 +140,22 @@ enum gspca_packet_type {
 	LAST_PACKET
 };
 
-struct gspca_frame {
-	__u8 *data;			/* frame buffer */
-	int vma_use_count;
-	struct v4l2_buffer v4l2_buf;
+struct gspca_buffer {
+	struct vb2_v4l2_buffer vb;
+	struct list_head list;
 };
 
+static inline struct gspca_buffer *to_gspca_buffer(struct vb2_buffer *vb2)
+{
+	return container_of(vb2, struct gspca_buffer, vb.vb2_buf);
+}
+
 struct gspca_dev {
 	struct video_device vdev;	/* !! must be the first item */
 	struct module *module;		/* subdriver handling the device */
 	struct v4l2_device v4l2_dev;
 	struct usb_device *dev;
-	struct file *capt_file;		/* file doing video capture */
-					/* protected by queue_lock */
+
 #if IS_ENABLED(CONFIG_INPUT)
 	struct input_dev *input_dev;
 	char phys[64];			/* physical device path */
@@ -176,34 +181,29 @@ struct gspca_dev {
 	struct urb *int_urb;
 #endif
 
-	__u8 *frbuf;				/* buffer for nframes */
-	struct gspca_frame frame[GSPCA_MAX_FRAMES];
-	u8 *image;				/* image beeing filled */
-	__u32 frsz;				/* frame size */
+	u8 *image;				/* image being filled */
 	u32 image_len;				/* current length of image */
-	atomic_t fr_q;				/* next frame to queue */
-	atomic_t fr_i;				/* frame being filled */
-	signed char fr_queue[GSPCA_MAX_FRAMES];	/* frame queue */
-	char nframes;				/* number of frames */
-	u8 fr_o;				/* next frame to dequeue */
 	__u8 last_packet_type;
 	__s8 empty_packet;		/* if (-1) don't check empty packets */
-	__u8 streaming;			/* protected by both mutexes (*) */
+	bool streaming;
 
 	__u8 curr_mode;			/* current camera mode */
 	struct v4l2_pix_format pixfmt;	/* current mode parameters */
 	__u32 sequence;			/* frame sequence number */
 
+	struct vb2_queue queue;
+
+	spinlock_t qlock;
+	struct list_head buf_list;
+
 	wait_queue_head_t wq;		/* wait queue */
 	struct mutex usb_lock;		/* usb exchange protection */
-	struct mutex queue_lock;	/* ISOC queue protection */
 	int usb_err;			/* USB error - protected by usb_lock */
 	u16 pkt_size;			/* ISOC packet size */
 #ifdef CONFIG_PM
 	char frozen;			/* suspend - resume */
 #endif
-	char present;			/* device connected */
-	char nbufread;			/* number of buffers for read() */
+	bool present;
 	char memory;			/* memory type (V4L2_MEMORY_xxx) */
 	__u8 iface;			/* USB interface number */
 	__u8 alt;			/* USB alternate setting */
diff --git a/drivers/media/usb/gspca/jl2005bcd.c b/drivers/media/usb/gspca/jl2005bcd.c
index d668589598d6..c40245950553 100644
--- a/drivers/media/usb/gspca/jl2005bcd.c
+++ b/drivers/media/usb/gspca/jl2005bcd.c
@@ -321,7 +321,7 @@ static void jl2005c_dostream(struct work_struct *work)
 	int ret;
 	u8 *buffer;
 
-	buffer = kmalloc(JL2005C_MAX_TRANSFER, GFP_KERNEL | GFP_DMA);
+	buffer = kmalloc(JL2005C_MAX_TRANSFER, GFP_KERNEL);
 	if (!buffer) {
 		pr_err("Couldn't allocate USB buffer\n");
 		goto quit_stream;
diff --git a/drivers/media/usb/gspca/m5602/m5602_core.c b/drivers/media/usb/gspca/m5602/m5602_core.c
index b83ec4285a0b..30b7cf1feedd 100644
--- a/drivers/media/usb/gspca/m5602/m5602_core.c
+++ b/drivers/media/usb/gspca/m5602/m5602_core.c
@@ -342,7 +342,7 @@ static void m5602_urb_complete(struct gspca_dev *gspca_dev,
 		data += 4;
 		len -= 4;
 
-		if (cur_frame_len + len <= gspca_dev->frsz) {
+		if (cur_frame_len + len <= gspca_dev->pixfmt.sizeimage) {
 			gspca_dbg(gspca_dev, D_FRAM, "Continuing frame %d copying %d bytes\n",
 				  sd->frame_count, len);
 
@@ -351,7 +351,7 @@ static void m5602_urb_complete(struct gspca_dev *gspca_dev,
 		} else {
 			/* Add the remaining data up to frame size */
 			gspca_frame_add(gspca_dev, INTER_PACKET, data,
-				    gspca_dev->frsz - cur_frame_len);
+				gspca_dev->pixfmt.sizeimage - cur_frame_len);
 		}
 	}
 }
diff --git a/drivers/media/usb/gspca/ov534.c b/drivers/media/usb/gspca/ov534.c
index f293921a1f2b..d06dc0755b9a 100644
--- a/drivers/media/usb/gspca/ov534.c
+++ b/drivers/media/usb/gspca/ov534.c
@@ -1476,7 +1476,6 @@ static void sd_get_streamparm(struct gspca_dev *gspca_dev,
 	struct v4l2_fract *tpf = &cp->timeperframe;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	cp->capability |= V4L2_CAP_TIMEPERFRAME;
 	tpf->numerator = 1;
 	tpf->denominator = sd->frame_rate;
 }
diff --git a/drivers/media/usb/gspca/sq905.c b/drivers/media/usb/gspca/sq905.c
index cc8ff41b8ab3..ffea9c35b0a0 100644
--- a/drivers/media/usb/gspca/sq905.c
+++ b/drivers/media/usb/gspca/sq905.c
@@ -217,7 +217,7 @@ static void sq905_dostream(struct work_struct *work)
 	u8 *data;
 	u8 *buffer;
 
-	buffer = kmalloc(SQ905_MAX_TRANSFER, GFP_KERNEL | GFP_DMA);
+	buffer = kmalloc(SQ905_MAX_TRANSFER, GFP_KERNEL);
 	if (!buffer) {
 		pr_err("Couldn't allocate USB buffer\n");
 		goto quit_stream;
diff --git a/drivers/media/usb/gspca/sq905c.c b/drivers/media/usb/gspca/sq905c.c
index 5e1269eb7c50..274921c0bb46 100644
--- a/drivers/media/usb/gspca/sq905c.c
+++ b/drivers/media/usb/gspca/sq905c.c
@@ -138,7 +138,7 @@ static void sq905c_dostream(struct work_struct *work)
 	int ret;
 	u8 *buffer;
 
-	buffer = kmalloc(SQ905C_MAX_TRANSFER, GFP_KERNEL | GFP_DMA);
+	buffer = kmalloc(SQ905C_MAX_TRANSFER, GFP_KERNEL);
 	if (!buffer) {
 		pr_err("Couldn't allocate USB buffer\n");
 		goto quit_stream;
diff --git a/drivers/media/usb/gspca/topro.c b/drivers/media/usb/gspca/topro.c
index 82e2be14cad8..6f3ec0366a2f 100644
--- a/drivers/media/usb/gspca/topro.c
+++ b/drivers/media/usb/gspca/topro.c
@@ -4780,7 +4780,6 @@ static void sd_get_streamparm(struct gspca_dev *gspca_dev,
 	struct v4l2_fract *tpf = &cp->timeperframe;
 	int fr, i;
 
-	cp->capability |= V4L2_CAP_TIMEPERFRAME;
 	tpf->numerator = 1;
 	i = get_fr_idx(gspca_dev);
 	if (i & 0x80) {
diff --git a/drivers/media/usb/gspca/vc032x.c b/drivers/media/usb/gspca/vc032x.c
index 6b11597977c9..52d071659634 100644
--- a/drivers/media/usb/gspca/vc032x.c
+++ b/drivers/media/usb/gspca/vc032x.c
@@ -3642,7 +3642,7 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 		int size, l;
 
 		l = gspca_dev->image_len;
-		size = gspca_dev->frsz;
+		size = gspca_dev->pixfmt.sizeimage;
 		if (len > size - l)
 			len = size - l;
 	}
diff --git a/drivers/media/usb/gspca/vicam.c b/drivers/media/usb/gspca/vicam.c
index 554b90ef2200..8562bda0ef88 100644
--- a/drivers/media/usb/gspca/vicam.c
+++ b/drivers/media/usb/gspca/vicam.c
@@ -182,7 +182,7 @@ static void vicam_dostream(struct work_struct *work)
 
 	frame_sz = gspca_dev->cam.cam_mode[gspca_dev->curr_mode].sizeimage +
 		   HEADER_SIZE;
-	buffer = kmalloc(frame_sz, GFP_KERNEL | GFP_DMA);
+	buffer = kmalloc(frame_sz, GFP_KERNEL);
 	if (!buffer) {
 		pr_err("Couldn't allocate USB buffer\n");
 		goto exit;
diff --git a/drivers/media/usb/gspca/zc3xx.c b/drivers/media/usb/gspca/zc3xx.c
index 25b4dbe8e049..cf21991e3d99 100644
--- a/drivers/media/usb/gspca/zc3xx.c
+++ b/drivers/media/usb/gspca/zc3xx.c
@@ -3184,10 +3184,10 @@ static const struct usb_action ov7620_InitialScale[] = {	/* 320x240 */
 	{}
 };
 static const struct usb_action ov7620_50HZ[] = {
-	{0xaa, 0x13, 0x00a3},	/* 00,13,a3,aa */
 	{0xdd, 0x00, 0x0100},	/* 00,01,00,dd */
 	{0xaa, 0x2b, 0x0096},	/* 00,2b,96,aa */
-	{0xaa, 0x75, 0x008a},	/* 00,75,8a,aa */
+	/* enable 1/120s & 1/100s exposures for banding filter */
+	{0xaa, 0x75, 0x008e},
 	{0xaa, 0x2d, 0x0005},	/* 00,2d,05,aa */
 	{0xa0, 0x00, ZC3XX_R190_EXPOSURELIMITHIGH},	/* 01,90,00,cc */
 	{0xa0, 0x04, ZC3XX_R191_EXPOSURELIMITMID},	/* 01,91,04,cc */
@@ -3195,18 +3195,16 @@ static const struct usb_action ov7620_50HZ[] = {
 	{0xa0, 0x00, ZC3XX_R195_ANTIFLICKERHIGH},	/* 01,95,00,cc */
 	{0xa0, 0x00, ZC3XX_R196_ANTIFLICKERMID},	/* 01,96,00,cc */
 	{0xa0, 0x83, ZC3XX_R197_ANTIFLICKERLOW},	/* 01,97,83,cc */
-	{0xaa, 0x10, 0x0082},				/* 00,10,82,aa */
 	{0xaa, 0x76, 0x0003},				/* 00,76,03,aa */
 /*	{0xa0, 0x40, ZC3XX_R002_CLOCKSELECT},		 * 00,02,40,cc
 							 * if mode0 (640x480) */
 	{}
 };
 static const struct usb_action ov7620_60HZ[] = {
-	{0xaa, 0x13, 0x00a3},			/* 00,13,a3,aa */
-						/* (bug in zs211.inf) */
 	{0xdd, 0x00, 0x0100},			/* 00,01,00,dd */
 	{0xaa, 0x2b, 0x0000},			/* 00,2b,00,aa */
-	{0xaa, 0x75, 0x008a},			/* 00,75,8a,aa */
+	/* enable 1/120s & 1/100s exposures for banding filter */
+	{0xaa, 0x75, 0x008e},
 	{0xaa, 0x2d, 0x0005},			/* 00,2d,05,aa */
 	{0xa0, 0x00, ZC3XX_R190_EXPOSURELIMITHIGH}, /* 01,90,00,cc */
 	{0xa0, 0x04, ZC3XX_R191_EXPOSURELIMITMID}, /* 01,91,04,cc */
@@ -3214,7 +3212,6 @@ static const struct usb_action ov7620_60HZ[] = {
 	{0xa0, 0x00, ZC3XX_R195_ANTIFLICKERHIGH}, /* 01,95,00,cc */
 	{0xa0, 0x00, ZC3XX_R196_ANTIFLICKERMID}, /* 01,96,00,cc */
 	{0xa0, 0x83, ZC3XX_R197_ANTIFLICKERLOW}, /* 01,97,83,cc */
-	{0xaa, 0x10, 0x0020},			/* 00,10,20,aa */
 	{0xaa, 0x76, 0x0003},			/* 00,76,03,aa */
 /*	{0xa0, 0x40, ZC3XX_R002_CLOCKSELECT},	 * 00,02,40,cc
 						 * if mode0 (640x480) */
@@ -3224,11 +3221,10 @@ static const struct usb_action ov7620_60HZ[] = {
 	{}
 };
 static const struct usb_action ov7620_NoFliker[] = {
-	{0xaa, 0x13, 0x00a3},			/* 00,13,a3,aa */
-						/* (bug in zs211.inf) */
 	{0xdd, 0x00, 0x0100},			/* 00,01,00,dd */
 	{0xaa, 0x2b, 0x0000},			/* 00,2b,00,aa */
-	{0xaa, 0x75, 0x008e},			/* 00,75,8e,aa */
+	/* disable 1/120s & 1/100s exposures for banding filter */
+	{0xaa, 0x75, 0x008a},
 	{0xaa, 0x2d, 0x0001},			/* 00,2d,01,aa */
 	{0xa0, 0x00, ZC3XX_R190_EXPOSURELIMITHIGH}, /* 01,90,00,cc */
 	{0xa0, 0x04, ZC3XX_R191_EXPOSURELIMITMID}, /* 01,91,04,cc */
@@ -5778,16 +5774,34 @@ static void setcontrast(struct gspca_dev *gspca_dev,
 
 static s32 getexposure(struct gspca_dev *gspca_dev)
 {
-	return (i2c_read(gspca_dev, 0x25) << 9)
-		| (i2c_read(gspca_dev, 0x26) << 1)
-		| (i2c_read(gspca_dev, 0x27) >> 7);
+	struct sd *sd = (struct sd *) gspca_dev;
+
+	switch (sd->sensor) {
+	case SENSOR_HV7131R:
+		return (i2c_read(gspca_dev, 0x25) << 9)
+			| (i2c_read(gspca_dev, 0x26) << 1)
+			| (i2c_read(gspca_dev, 0x27) >> 7);
+	case SENSOR_OV7620:
+		return i2c_read(gspca_dev, 0x10);
+	default:
+		return -1;
+	}
 }
 
 static void setexposure(struct gspca_dev *gspca_dev, s32 val)
 {
-	i2c_write(gspca_dev, 0x25, val >> 9, 0x00);
-	i2c_write(gspca_dev, 0x26, val >> 1, 0x00);
-	i2c_write(gspca_dev, 0x27, val << 7, 0x00);
+	struct sd *sd = (struct sd *) gspca_dev;
+
+	switch (sd->sensor) {
+	case SENSOR_HV7131R:
+		i2c_write(gspca_dev, 0x25, val >> 9, 0x00);
+		i2c_write(gspca_dev, 0x26, val >> 1, 0x00);
+		i2c_write(gspca_dev, 0x27, val << 7, 0x00);
+		break;
+	case SENSOR_OV7620:
+		i2c_write(gspca_dev, 0x10, val, 0x00);
+		break;
+	}
 }
 
 static void setquality(struct gspca_dev *gspca_dev)
@@ -5918,7 +5932,12 @@ static void setlightfreq(struct gspca_dev *gspca_dev, s32 val)
 
 static void setautogain(struct gspca_dev *gspca_dev, s32 val)
 {
-	reg_w(gspca_dev, val ? 0x42 : 0x02, 0x0180);
+	struct sd *sd = (struct sd *) gspca_dev;
+
+	if (sd->sensor == SENSOR_OV7620)
+		i2c_write(gspca_dev, 0x13, val ? 0xa3 : 0x80, 0x00);
+	else
+		reg_w(gspca_dev, val ? 0x42 : 0x02, 0x0180);
 }
 
 /*
@@ -6439,6 +6458,9 @@ static int sd_init_controls(struct gspca_dev *gspca_dev)
 	if (sd->sensor == SENSOR_HV7131R)
 		sd->exposure = v4l2_ctrl_new_std(hdl, &zcxx_ctrl_ops,
 			V4L2_CID_EXPOSURE, 0x30d, 0x493e, 1, 0x927);
+	else if (sd->sensor == SENSOR_OV7620)
+		sd->exposure = v4l2_ctrl_new_std(hdl, &zcxx_ctrl_ops,
+			V4L2_CID_EXPOSURE, 0, 255, 1, 0x41);
 	sd->autogain = v4l2_ctrl_new_std(hdl, &zcxx_ctrl_ops,
 			V4L2_CID_AUTOGAIN, 0, 1, 1, 1);
 	if (sd->sensor != SENSOR_OV7630C)
@@ -6458,7 +6480,7 @@ static int sd_init_controls(struct gspca_dev *gspca_dev)
 		return hdl->error;
 	}
 	v4l2_ctrl_cluster(3, &sd->gamma);
-	if (sd->sensor == SENSOR_HV7131R)
+	if (sd->sensor == SENSOR_HV7131R || sd->sensor == SENSOR_OV7620)
 		v4l2_ctrl_auto_cluster(2, &sd->autogain, 0, true);
 	return 0;
 }
diff --git a/drivers/media/usb/hackrf/hackrf.c b/drivers/media/usb/hackrf/hackrf.c
index 7eb53517a82f..6d692fb3e8dd 100644
--- a/drivers/media/usb/hackrf/hackrf.c
+++ b/drivers/media/usb/hackrf/hackrf.c
@@ -909,18 +909,15 @@ static int hackrf_querycap(struct file *file, void *fh,
 
 	dev_dbg(&intf->dev, "\n");
 
+	cap->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_READWRITE;
 	if (vdev->vfl_dir == VFL_DIR_RX)
-		cap->device_caps = V4L2_CAP_SDR_CAPTURE | V4L2_CAP_TUNER |
-				   V4L2_CAP_STREAMING | V4L2_CAP_READWRITE;
-
+		cap->device_caps |= V4L2_CAP_SDR_CAPTURE | V4L2_CAP_TUNER;
 	else
-		cap->device_caps = V4L2_CAP_SDR_OUTPUT | V4L2_CAP_MODULATOR |
-				   V4L2_CAP_STREAMING | V4L2_CAP_READWRITE;
+		cap->device_caps |= V4L2_CAP_SDR_OUTPUT | V4L2_CAP_MODULATOR;
 
 	cap->capabilities = V4L2_CAP_SDR_CAPTURE | V4L2_CAP_TUNER |
 			    V4L2_CAP_SDR_OUTPUT | V4L2_CAP_MODULATOR |
-			    V4L2_CAP_STREAMING | V4L2_CAP_READWRITE |
-			    V4L2_CAP_DEVICE_CAPS;
+			    V4L2_CAP_DEVICE_CAPS | cap->device_caps;
 	strlcpy(cap->driver, KBUILD_MODNAME, sizeof(cap->driver));
 	strlcpy(cap->card, dev->rx_vdev.name, sizeof(cap->card));
 	usb_make_path(dev->udev, cap->bus_info, sizeof(cap->bus_info));
diff --git a/drivers/media/usb/hdpvr/hdpvr-i2c.c b/drivers/media/usb/hdpvr/hdpvr-i2c.c
index 4720d79b0282..c71ddefd2e58 100644
--- a/drivers/media/usb/hdpvr/hdpvr-i2c.c
+++ b/drivers/media/usb/hdpvr/hdpvr-i2c.c
@@ -173,7 +173,7 @@ static const struct i2c_algorithm hdpvr_algo = {
 };
 
 static const struct i2c_adapter hdpvr_i2c_adapter_template = {
-	.name   = "Hauppage HD PVR I2C",
+	.name   = "Hauppauge HD PVR I2C",
 	.owner  = THIS_MODULE,
 	.algo   = &hdpvr_algo,
 };
diff --git a/drivers/media/usb/hdpvr/hdpvr-video.c b/drivers/media/usb/hdpvr/hdpvr-video.c
index 77c3d331ff31..1b89c77bad66 100644
--- a/drivers/media/usb/hdpvr/hdpvr-video.c
+++ b/drivers/media/usb/hdpvr/hdpvr-video.c
@@ -873,7 +873,7 @@ static int vidioc_g_audio(struct file *file, void *private_data,
 
 	audio->index = dev->options.audio_input;
 	audio->capability = V4L2_AUDCAP_STEREO;
-	strncpy(audio->name, audio_iname[audio->index], sizeof(audio->name));
+	strlcpy(audio->name, audio_iname[audio->index], sizeof(audio->name));
 	audio->name[sizeof(audio->name) - 1] = '\0';
 	return 0;
 }
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-cx2584x-v4l.c b/drivers/media/usb/pvrusb2/pvrusb2-cx2584x-v4l.c
index 242b213b7599..d5bec0f69bec 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-cx2584x-v4l.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-cx2584x-v4l.c
@@ -23,7 +23,6 @@
 */
 
 #include "pvrusb2-cx2584x-v4l.h"
-#include "pvrusb2-video-v4l.h"
 
 
 #include "pvrusb2-hdw-internal.h"
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-devattr.c b/drivers/media/usb/pvrusb2/pvrusb2-devattr.c
index 71537097c13f..06de1c83f444 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-devattr.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-devattr.c
@@ -182,7 +182,6 @@ static const struct pvr2_device_desc pvr2_device_av400 = {
 
 #ifdef CONFIG_VIDEO_PVRUSB2_DVB
 static struct lgdt330x_config pvr2_lgdt3303_config = {
-	.demod_address       = 0x0e,
 	.demod_chip          = LGDT3303,
 	.clock_polarity_flip = 1,
 };
@@ -190,6 +189,7 @@ static struct lgdt330x_config pvr2_lgdt3303_config = {
 static int pvr2_lgdt3303_attach(struct pvr2_dvb_adapter *adap)
 {
 	adap->fe = dvb_attach(lgdt330x_attach, &pvr2_lgdt3303_config,
+			      0x0e,
 			      &adap->channel.hdw->i2c_adap);
 	if (adap->fe)
 		return 0;
@@ -243,13 +243,13 @@ static const struct pvr2_device_desc pvr2_device_onair_creator = {
 
 #ifdef CONFIG_VIDEO_PVRUSB2_DVB
 static struct lgdt330x_config pvr2_lgdt3302_config = {
-	.demod_address       = 0x0e,
 	.demod_chip          = LGDT3302,
 };
 
 static int pvr2_lgdt3302_attach(struct pvr2_dvb_adapter *adap)
 {
 	adap->fe = dvb_attach(lgdt330x_attach, &pvr2_lgdt3302_config,
+			      0x0e,
 			      &adap->channel.hdw->i2c_adap);
 	if (adap->fe)
 		return 0;
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c b/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
index 9fdc57c1658f..e53a80b589a1 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
@@ -159,9 +159,12 @@ static int pvr2_s_std(struct file *file, void *priv, v4l2_std_id std)
 {
 	struct pvr2_v4l2_fh *fh = file->private_data;
 	struct pvr2_hdw *hdw = fh->channel.mc_head->hdw;
+	int ret;
 
-	return pvr2_ctrl_set_value(
+	ret = pvr2_ctrl_set_value(
 		pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_STDCUR), std);
+	pvr2_hdw_commit_ctl(hdw);
+	return ret;
 }
 
 static int pvr2_querystd(struct file *file, void *priv, v4l2_std_id *std)
@@ -251,12 +254,15 @@ static int pvr2_s_input(struct file *file, void *priv, unsigned int inp)
 {
 	struct pvr2_v4l2_fh *fh = file->private_data;
 	struct pvr2_hdw *hdw = fh->channel.mc_head->hdw;
+	int ret;
 
 	if (inp >= fh->input_cnt)
 		return -EINVAL;
-	return pvr2_ctrl_set_value(
+	ret = pvr2_ctrl_set_value(
 			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_INPUT),
 			fh->input_map[inp]);
+	pvr2_hdw_commit_ctl(hdw);
+	return ret;
 }
 
 static int pvr2_enumaudio(struct file *file, void *priv, struct v4l2_audio *vin)
@@ -315,13 +321,16 @@ static int pvr2_s_tuner(struct file *file, void *priv, const struct v4l2_tuner *
 {
 	struct pvr2_v4l2_fh *fh = file->private_data;
 	struct pvr2_hdw *hdw = fh->channel.mc_head->hdw;
+	int ret;
 
 	if (vt->index != 0)
 		return -EINVAL;
 
-	return pvr2_ctrl_set_value(
+	ret = pvr2_ctrl_set_value(
 			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_AUDIOMODE),
 			vt->audmode);
+	pvr2_hdw_commit_ctl(hdw);
+	return ret;
 }
 
 static int pvr2_s_frequency(struct file *file, void *priv, const struct v4l2_frequency *vf)
@@ -353,8 +362,10 @@ static int pvr2_s_frequency(struct file *file, void *priv, const struct v4l2_fre
 		fv = (fv * 125) / 2;
 	else
 		fv = fv * 62500;
-	return pvr2_ctrl_set_value(
+	ret = pvr2_ctrl_set_value(
 			pvr2_hdw_get_ctrl_by_id(hdw,PVR2_CID_FREQUENCY),fv);
+	pvr2_hdw_commit_ctl(hdw);
+	return ret;
 }
 
 static int pvr2_g_frequency(struct file *file, void *priv, struct v4l2_frequency *vf)
@@ -470,6 +481,7 @@ static int pvr2_s_fmt_vid_cap(struct file *file, void *priv, struct v4l2_format
 	vcp = pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_VRES);
 	pvr2_ctrl_set_value(hcp, vf->fmt.pix.width);
 	pvr2_ctrl_set_value(vcp, vf->fmt.pix.height);
+	pvr2_hdw_commit_ctl(hdw);
 	return 0;
 }
 
@@ -597,9 +609,12 @@ static int pvr2_s_ctrl(struct file *file, void *priv, struct v4l2_control *vc)
 {
 	struct pvr2_v4l2_fh *fh = file->private_data;
 	struct pvr2_hdw *hdw = fh->channel.mc_head->hdw;
+	int ret;
 
-	return pvr2_ctrl_set_value(pvr2_hdw_get_ctrl_v4l(hdw, vc->id),
+	ret = pvr2_ctrl_set_value(pvr2_hdw_get_ctrl_v4l(hdw, vc->id),
 			vc->value);
+	pvr2_hdw_commit_ctl(hdw);
+	return ret;
 }
 
 static int pvr2_g_ext_ctrls(struct file *file, void *priv,
@@ -658,10 +673,12 @@ static int pvr2_s_ext_ctrls(struct file *file, void *priv,
 				ctrl->value);
 		if (ret) {
 			ctls->error_idx = idx;
-			return ret;
+			goto commit;
 		}
 	}
-	return 0;
+commit:
+	pvr2_hdw_commit_ctl(hdw);
+	return ret;
 }
 
 static int pvr2_try_ext_ctrls(struct file *file, void *priv,
@@ -764,23 +781,23 @@ static int pvr2_s_selection(struct file *file, void *priv,
 			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPL),
 			sel->r.left);
 	if (ret != 0)
-		return -EINVAL;
+		goto commit;
 	ret = pvr2_ctrl_set_value(
 			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPT),
 			sel->r.top);
 	if (ret != 0)
-		return -EINVAL;
+		goto commit;
 	ret = pvr2_ctrl_set_value(
 			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPW),
 			sel->r.width);
 	if (ret != 0)
-		return -EINVAL;
+		goto commit;
 	ret = pvr2_ctrl_set_value(
 			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPH),
 			sel->r.height);
-	if (ret != 0)
-		return -EINVAL;
-	return 0;
+commit:
+	pvr2_hdw_commit_ctl(hdw);
+	return ret;
 }
 
 static int pvr2_log_status(struct file *file, void *priv)
@@ -905,44 +922,6 @@ static void pvr2_v4l2_internal_check(struct pvr2_channel *chp)
 }
 
 
-static long pvr2_v4l2_ioctl(struct file *file,
-			   unsigned int cmd, unsigned long arg)
-{
-
-	struct pvr2_v4l2_fh *fh = file->private_data;
-	struct pvr2_hdw *hdw = fh->channel.mc_head->hdw;
-	long ret = -EINVAL;
-
-	if (pvrusb2_debug & PVR2_TRACE_V4LIOCTL)
-		v4l_printk_ioctl(pvr2_hdw_get_driver_name(hdw), cmd);
-
-	if (!pvr2_hdw_dev_ok(hdw)) {
-		pvr2_trace(PVR2_TRACE_ERROR_LEGS,
-			   "ioctl failed - bad or no context");
-		return -EFAULT;
-	}
-
-	ret = video_ioctl2(file, cmd, arg);
-
-	pvr2_hdw_commit_ctl(hdw);
-
-	if (ret < 0) {
-		if (pvrusb2_debug & PVR2_TRACE_V4LIOCTL) {
-			pvr2_trace(PVR2_TRACE_V4LIOCTL,
-				   "pvr2_v4l2_do_ioctl failure, ret=%ld command was:",
-ret);
-			v4l_printk_ioctl(pvr2_hdw_get_driver_name(hdw), cmd);
-		}
-	} else {
-		pvr2_trace(PVR2_TRACE_V4LIOCTL,
-			   "pvr2_v4l2_do_ioctl complete, ret=%ld (0x%lx)",
-			   ret, ret);
-	}
-	return ret;
-
-}
-
-
 static int pvr2_v4l2_release(struct file *file)
 {
 	struct pvr2_v4l2_fh *fhp = file->private_data;
@@ -1205,7 +1184,7 @@ static const struct v4l2_file_operations vdev_fops = {
 	.open       = pvr2_v4l2_open,
 	.release    = pvr2_v4l2_release,
 	.read       = pvr2_v4l2_read,
-	.unlocked_ioctl = pvr2_v4l2_ioctl,
+	.unlocked_ioctl = video_ioctl2,
 	.poll       = pvr2_v4l2_poll,
 };
 
diff --git a/drivers/media/usb/siano/smsusb.c b/drivers/media/usb/siano/smsusb.c
index 6d436e9e454f..be3634407f1f 100644
--- a/drivers/media/usb/siano/smsusb.c
+++ b/drivers/media/usb/siano/smsusb.c
@@ -455,7 +455,7 @@ static int smsusb_init_device(struct usb_interface *intf, int board_id)
 	mdev = siano_media_device_register(dev, board_id);
 
 	/* register in smscore */
-	rc = smscore_register_device(&params, &dev->coredev, mdev);
+	rc = smscore_register_device(&params, &dev->coredev, 0, mdev);
 	if (rc < 0) {
 		pr_err("smscore_register_device(...) failed, rc %d\n", rc);
 		smsusb_term_device(intf);
diff --git a/drivers/media/usb/stk1160/stk1160-core.c b/drivers/media/usb/stk1160/stk1160-core.c
index bea8bbbb84fb..72bd893c9659 100644
--- a/drivers/media/usb/stk1160/stk1160-core.c
+++ b/drivers/media/usb/stk1160/stk1160-core.c
@@ -167,6 +167,8 @@ static void stk1160_release(struct v4l2_device *v4l2_dev)
 
 	v4l2_ctrl_handler_free(&dev->ctrl_handler);
 	v4l2_device_unregister(&dev->v4l2_dev);
+	mutex_destroy(&dev->v4l_lock);
+	mutex_destroy(&dev->vb_queue_lock);
 	kfree(dev->alt_max_pkt_size);
 	kfree(dev);
 }
@@ -423,7 +425,7 @@ static void stk1160_disconnect(struct usb_interface *interface)
 
 	/*
 	 * This calls stk1160_release if it's the last reference.
-	 * therwise, release is posponed until there are no users left.
+	 * Otherwise, release is posponed until there are no users left.
 	 */
 	v4l2_device_put(&dev->v4l2_dev);
 }
diff --git a/drivers/media/usb/usbtv/usbtv-video.c b/drivers/media/usb/usbtv/usbtv-video.c
index 3668a04359e8..ce79df643c7e 100644
--- a/drivers/media/usb/usbtv/usbtv-video.c
+++ b/drivers/media/usb/usbtv/usbtv-video.c
@@ -54,7 +54,7 @@ static struct usbtv_norm_params norm_params[] = {
 		.cap_height = 480,
 	},
 	{
-		.norm = V4L2_STD_PAL,
+		.norm = V4L2_STD_625_50,
 		.cap_width = 720,
 		.cap_height = 576,
 	}
@@ -77,7 +77,7 @@ static int usbtv_configure_for_norm(struct usbtv *usbtv, v4l2_std_id norm)
 		usbtv->height = params->cap_height;
 		usbtv->n_chunks = usbtv->width * usbtv->height
 						/ 4 / USBTV_CHUNK;
-		usbtv->norm = params->norm;
+		usbtv->norm = norm;
 	} else
 		ret = -EINVAL;
 
@@ -121,52 +121,144 @@ static int usbtv_select_input(struct usbtv *usbtv, int input)
 	return ret;
 }
 
+static uint16_t usbtv_norm_to_16f_reg(v4l2_std_id norm)
+{
+	/* NTSC M/M-JP/M-KR */
+	if (norm & V4L2_STD_NTSC)
+		return 0x00b8;
+	/* PAL BG/DK/H/I */
+	if (norm & V4L2_STD_PAL)
+		return 0x00ee;
+	/* SECAM B/D/G/H/K/K1/L/Lc */
+	if (norm & V4L2_STD_SECAM)
+		return 0x00ff;
+	if (norm & V4L2_STD_NTSC_443)
+		return 0x00a8;
+	if (norm & (V4L2_STD_PAL_M | V4L2_STD_PAL_60))
+		return 0x00bc;
+	/* Fallback to automatic detection for other standards */
+	return 0x0000;
+}
+
 static int usbtv_select_norm(struct usbtv *usbtv, v4l2_std_id norm)
 {
 	int ret;
+	/* These are the series of register values used to configure the
+	 * decoder for a specific standard.
+	 * The first 21 register writes are copied from the
+	 * Settings\DecoderDefaults registry keys present in the Windows driver
+	 * .INF file, and control various image tuning parameters (color
+	 * correction, sharpness, ...).
+	 */
 	static const u16 pal[][2] = {
+		/* "AVPAL" tuning sequence from .INF file */
+		{ USBTV_BASE + 0x0003, 0x0004 },
 		{ USBTV_BASE + 0x001a, 0x0068 },
+		{ USBTV_BASE + 0x0100, 0x00d3 },
 		{ USBTV_BASE + 0x010e, 0x0072 },
 		{ USBTV_BASE + 0x010f, 0x00a2 },
 		{ USBTV_BASE + 0x0112, 0x00b0 },
+		{ USBTV_BASE + 0x0115, 0x0015 },
 		{ USBTV_BASE + 0x0117, 0x0001 },
 		{ USBTV_BASE + 0x0118, 0x002c },
 		{ USBTV_BASE + 0x012d, 0x0010 },
 		{ USBTV_BASE + 0x012f, 0x0020 },
+		{ USBTV_BASE + 0x0220, 0x002e },
+		{ USBTV_BASE + 0x0225, 0x0008 },
+		{ USBTV_BASE + 0x024e, 0x0002 },
 		{ USBTV_BASE + 0x024f, 0x0002 },
 		{ USBTV_BASE + 0x0254, 0x0059 },
 		{ USBTV_BASE + 0x025a, 0x0016 },
 		{ USBTV_BASE + 0x025b, 0x0035 },
 		{ USBTV_BASE + 0x0263, 0x0017 },
 		{ USBTV_BASE + 0x0266, 0x0016 },
-		{ USBTV_BASE + 0x0267, 0x0036 }
+		{ USBTV_BASE + 0x0267, 0x0036 },
+		/* End image tuning */
+		{ USBTV_BASE + 0x024e, 0x0002 },
+		{ USBTV_BASE + 0x024f, 0x0002 },
 	};
 
 	static const u16 ntsc[][2] = {
+		/* "AVNTSC" tuning sequence from .INF file */
+		{ USBTV_BASE + 0x0003, 0x0004 },
 		{ USBTV_BASE + 0x001a, 0x0079 },
+		{ USBTV_BASE + 0x0100, 0x00d3 },
 		{ USBTV_BASE + 0x010e, 0x0068 },
 		{ USBTV_BASE + 0x010f, 0x009c },
 		{ USBTV_BASE + 0x0112, 0x00f0 },
+		{ USBTV_BASE + 0x0115, 0x0015 },
 		{ USBTV_BASE + 0x0117, 0x0000 },
 		{ USBTV_BASE + 0x0118, 0x00fc },
 		{ USBTV_BASE + 0x012d, 0x0004 },
 		{ USBTV_BASE + 0x012f, 0x0008 },
+		{ USBTV_BASE + 0x0220, 0x002e },
+		{ USBTV_BASE + 0x0225, 0x0008 },
+		{ USBTV_BASE + 0x024e, 0x0002 },
 		{ USBTV_BASE + 0x024f, 0x0001 },
 		{ USBTV_BASE + 0x0254, 0x005f },
 		{ USBTV_BASE + 0x025a, 0x0012 },
 		{ USBTV_BASE + 0x025b, 0x0001 },
 		{ USBTV_BASE + 0x0263, 0x001c },
 		{ USBTV_BASE + 0x0266, 0x0011 },
-		{ USBTV_BASE + 0x0267, 0x0005 }
+		{ USBTV_BASE + 0x0267, 0x0005 },
+		/* End image tuning */
+		{ USBTV_BASE + 0x024e, 0x0002 },
+		{ USBTV_BASE + 0x024f, 0x0002 },
+	};
+
+	static const u16 secam[][2] = {
+		/* "AVSECAM" tuning sequence from .INF file */
+		{ USBTV_BASE + 0x0003, 0x0004 },
+		{ USBTV_BASE + 0x001a, 0x0073 },
+		{ USBTV_BASE + 0x0100, 0x00dc },
+		{ USBTV_BASE + 0x010e, 0x0072 },
+		{ USBTV_BASE + 0x010f, 0x00a2 },
+		{ USBTV_BASE + 0x0112, 0x0090 },
+		{ USBTV_BASE + 0x0115, 0x0035 },
+		{ USBTV_BASE + 0x0117, 0x0001 },
+		{ USBTV_BASE + 0x0118, 0x0030 },
+		{ USBTV_BASE + 0x012d, 0x0004 },
+		{ USBTV_BASE + 0x012f, 0x0008 },
+		{ USBTV_BASE + 0x0220, 0x002d },
+		{ USBTV_BASE + 0x0225, 0x0028 },
+		{ USBTV_BASE + 0x024e, 0x0008 },
+		{ USBTV_BASE + 0x024f, 0x0002 },
+		{ USBTV_BASE + 0x0254, 0x0069 },
+		{ USBTV_BASE + 0x025a, 0x0016 },
+		{ USBTV_BASE + 0x025b, 0x0035 },
+		{ USBTV_BASE + 0x0263, 0x0021 },
+		{ USBTV_BASE + 0x0266, 0x0016 },
+		{ USBTV_BASE + 0x0267, 0x0036 },
+		/* End image tuning */
+		{ USBTV_BASE + 0x024e, 0x0002 },
+		{ USBTV_BASE + 0x024f, 0x0002 },
 	};
 
 	ret = usbtv_configure_for_norm(usbtv, norm);
 
 	if (!ret) {
-		if (norm & V4L2_STD_525_60)
+		/* Masks for norms using a NTSC or PAL color encoding. */
+		static const v4l2_std_id ntsc_mask =
+			V4L2_STD_NTSC | V4L2_STD_NTSC_443;
+		static const v4l2_std_id pal_mask =
+			V4L2_STD_PAL | V4L2_STD_PAL_60 | V4L2_STD_PAL_M;
+
+		if (norm & ntsc_mask)
 			ret = usbtv_set_regs(usbtv, ntsc, ARRAY_SIZE(ntsc));
-		else if (norm & V4L2_STD_PAL)
+		else if (norm & pal_mask)
 			ret = usbtv_set_regs(usbtv, pal, ARRAY_SIZE(pal));
+		else if (norm & V4L2_STD_SECAM)
+			ret = usbtv_set_regs(usbtv, secam, ARRAY_SIZE(secam));
+		else
+			ret = -EINVAL;
+	}
+
+	if (!ret) {
+		/* Configure the decoder for the color standard */
+		const u16 cfg[][2] = {
+			{ USBTV_BASE + 0x016f, usbtv_norm_to_16f_reg(norm) }
+		};
+		ret = usbtv_set_regs(usbtv, cfg, ARRAY_SIZE(cfg));
 	}
 
 	return ret;
@@ -236,15 +328,6 @@ static int usbtv_setup_capture(struct usbtv *usbtv)
 		{ USBTV_BASE + 0x0158, 0x001f },
 		{ USBTV_BASE + 0x0159, 0x0006 },
 		{ USBTV_BASE + 0x015d, 0x0000 },
-
-		{ USBTV_BASE + 0x0003, 0x0004 },
-		{ USBTV_BASE + 0x0100, 0x00d3 },
-		{ USBTV_BASE + 0x0115, 0x0015 },
-		{ USBTV_BASE + 0x0220, 0x002e },
-		{ USBTV_BASE + 0x0225, 0x0008 },
-		{ USBTV_BASE + 0x024e, 0x0002 },
-		{ USBTV_BASE + 0x024e, 0x0002 },
-		{ USBTV_BASE + 0x024f, 0x0002 },
 	};
 
 	ret = usbtv_set_regs(usbtv, setup, ARRAY_SIZE(setup));
@@ -587,7 +670,7 @@ static int usbtv_s_std(struct file *file, void *priv, v4l2_std_id norm)
 	int ret = -EINVAL;
 	struct usbtv *usbtv = video_drvdata(file);
 
-	if ((norm & V4L2_STD_525_60) || (norm & V4L2_STD_PAL))
+	if (norm & USBTV_TV_STD)
 		ret = usbtv_select_norm(usbtv, norm);
 
 	return ret;
@@ -698,6 +781,8 @@ static const struct vb2_ops usbtv_vb2_ops = {
 	.buf_queue = usbtv_buf_queue,
 	.start_streaming = usbtv_start_streaming,
 	.stop_streaming = usbtv_stop_streaming,
+	.wait_prepare = vb2_ops_wait_prepare,
+	.wait_finish = vb2_ops_wait_finish,
 };
 
 static int usbtv_s_ctrl(struct v4l2_ctrl *ctrl)
diff --git a/drivers/media/usb/usbtv/usbtv.h b/drivers/media/usb/usbtv/usbtv.h
index 0231e449877e..77a368e90fd0 100644
--- a/drivers/media/usb/usbtv/usbtv.h
+++ b/drivers/media/usb/usbtv/usbtv.h
@@ -68,7 +68,7 @@
 #define USBTV_ODD(chunk)	((be32_to_cpu(chunk[0]) & 0x0000f000) >> 15)
 #define USBTV_CHUNK_NO(chunk)	(be32_to_cpu(chunk[0]) & 0x00000fff)
 
-#define USBTV_TV_STD  (V4L2_STD_525_60 | V4L2_STD_PAL)
+#define USBTV_TV_STD  (V4L2_STD_525_60 | V4L2_STD_PAL | V4L2_STD_SECAM)
 
 /* parameters for supported TV norms */
 struct usbtv_norm_params {
diff --git a/drivers/media/usb/usbvision/usbvision-core.c b/drivers/media/usb/usbvision/usbvision-core.c
index 3f87fbc80be2..7138c2b606cc 100644
--- a/drivers/media/usb/usbvision/usbvision-core.c
+++ b/drivers/media/usb/usbvision/usbvision-core.c
@@ -1857,7 +1857,7 @@ int usbvision_stream_interrupt(struct usb_usbvision *usbvision)
 
 static int usbvision_set_compress_params(struct usb_usbvision *usbvision)
 {
-	static const char proc[] = "usbvision_set_compresion_params: ";
+	static const char proc[] = "usbvision_set_compression_params: ";
 	int rc;
 	unsigned char *value = usbvision->ctrl_urb_buffer;
 
diff --git a/drivers/media/usb/uvc/uvc_ctrl.c b/drivers/media/usb/uvc/uvc_ctrl.c
index 102594ec3e97..a36b4fb949fa 100644
--- a/drivers/media/usb/uvc/uvc_ctrl.c
+++ b/drivers/media/usb/uvc/uvc_ctrl.c
@@ -1607,14 +1607,12 @@ static int uvc_ctrl_get_flags(struct uvc_device *dev,
 	ret = uvc_query_ctrl(dev, UVC_GET_INFO, ctrl->entity->id, dev->intfnum,
 			     info->selector, data, 1);
 	if (!ret)
-		info->flags = UVC_CTRL_FLAG_GET_MIN | UVC_CTRL_FLAG_GET_MAX
-			    | UVC_CTRL_FLAG_GET_RES | UVC_CTRL_FLAG_GET_DEF
-			    | (data[0] & UVC_CONTROL_CAP_GET ?
-			       UVC_CTRL_FLAG_GET_CUR : 0)
-			    | (data[0] & UVC_CONTROL_CAP_SET ?
-			       UVC_CTRL_FLAG_SET_CUR : 0)
-			    | (data[0] & UVC_CONTROL_CAP_AUTOUPDATE ?
-			       UVC_CTRL_FLAG_AUTO_UPDATE : 0);
+		info->flags |= (data[0] & UVC_CONTROL_CAP_GET ?
+				UVC_CTRL_FLAG_GET_CUR : 0)
+			    |  (data[0] & UVC_CONTROL_CAP_SET ?
+				UVC_CTRL_FLAG_SET_CUR : 0)
+			    |  (data[0] & UVC_CONTROL_CAP_AUTOUPDATE ?
+				UVC_CTRL_FLAG_AUTO_UPDATE : 0);
 
 	kfree(data);
 	return ret;
@@ -1689,6 +1687,9 @@ static int uvc_ctrl_fill_xu_info(struct uvc_device *dev,
 
 	info->size = le16_to_cpup((__le16 *)data);
 
+	info->flags = UVC_CTRL_FLAG_GET_MIN | UVC_CTRL_FLAG_GET_MAX
+		    | UVC_CTRL_FLAG_GET_RES | UVC_CTRL_FLAG_GET_DEF;
+
 	ret = uvc_ctrl_get_flags(dev, ctrl, info);
 	if (ret < 0) {
 		uvc_trace(UVC_TRACE_CONTROL,
diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c
index 2469b49b2b30..8e138201330f 100644
--- a/drivers/media/usb/uvc/uvc_driver.c
+++ b/drivers/media/usb/uvc/uvc_driver.c
@@ -1871,13 +1871,6 @@ static void uvc_unregister_video(struct uvc_device *dev)
 {
 	struct uvc_streaming *stream;
 
-	/* Unregistering all video devices might result in uvc_delete() being
-	 * called from inside the loop if there's no open file handle. To avoid
-	 * that, increment the refcount before iterating over the streams and
-	 * decrement it when done.
-	 */
-	kref_get(&dev->ref);
-
 	list_for_each_entry(stream, &dev->streams, list) {
 		if (!video_is_registered(&stream->vdev))
 			continue;
@@ -1887,8 +1880,6 @@ static void uvc_unregister_video(struct uvc_device *dev)
 
 		uvc_debugfs_cleanup_stream(stream);
 	}
-
-	kref_put(&dev->ref, uvc_delete);
 }
 
 int uvc_register_video_device(struct uvc_device *dev,
@@ -2184,6 +2175,7 @@ static int uvc_probe(struct usb_interface *intf,
 
 error:
 	uvc_unregister_video(dev);
+	kref_put(&dev->ref, uvc_delete);
 	return -ENODEV;
 }
 
@@ -2201,6 +2193,7 @@ static void uvc_disconnect(struct usb_interface *intf)
 		return;
 
 	uvc_unregister_video(dev);
+	kref_put(&dev->ref, uvc_delete);
 }
 
 static int uvc_suspend(struct usb_interface *intf, pm_message_t message)
diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c
index aa0082fe5833..b28c997a7ab0 100644
--- a/drivers/media/usb/uvc/uvc_video.c
+++ b/drivers/media/usb/uvc/uvc_video.c
@@ -163,14 +163,27 @@ static void uvc_fixup_video_ctrl(struct uvc_streaming *stream,
 	}
 }
 
+static size_t uvc_video_ctrl_size(struct uvc_streaming *stream)
+{
+	/*
+	 * Return the size of the video probe and commit controls, which depends
+	 * on the protocol version.
+	 */
+	if (stream->dev->uvc_version < 0x0110)
+		return 26;
+	else if (stream->dev->uvc_version < 0x0150)
+		return 34;
+	else
+		return 48;
+}
+
 static int uvc_get_video_ctrl(struct uvc_streaming *stream,
 	struct uvc_streaming_control *ctrl, int probe, u8 query)
 {
+	u16 size = uvc_video_ctrl_size(stream);
 	u8 *data;
-	u16 size;
 	int ret;
 
-	size = stream->dev->uvc_version >= 0x0110 ? 34 : 26;
 	if ((stream->dev->quirks & UVC_QUIRK_PROBE_DEF) &&
 			query == UVC_GET_DEF)
 		return -EIO;
@@ -225,7 +238,7 @@ static int uvc_get_video_ctrl(struct uvc_streaming *stream,
 	ctrl->dwMaxVideoFrameSize = get_unaligned_le32(&data[18]);
 	ctrl->dwMaxPayloadTransferSize = get_unaligned_le32(&data[22]);
 
-	if (size == 34) {
+	if (size >= 34) {
 		ctrl->dwClockFrequency = get_unaligned_le32(&data[26]);
 		ctrl->bmFramingInfo = data[30];
 		ctrl->bPreferedVersion = data[31];
@@ -254,11 +267,10 @@ static int uvc_get_video_ctrl(struct uvc_streaming *stream,
 static int uvc_set_video_ctrl(struct uvc_streaming *stream,
 	struct uvc_streaming_control *ctrl, int probe)
 {
+	u16 size = uvc_video_ctrl_size(stream);
 	u8 *data;
-	u16 size;
 	int ret;
 
-	size = stream->dev->uvc_version >= 0x0110 ? 34 : 26;
 	data = kzalloc(size, GFP_KERNEL);
 	if (data == NULL)
 		return -ENOMEM;
@@ -275,7 +287,7 @@ static int uvc_set_video_ctrl(struct uvc_streaming *stream,
 	put_unaligned_le32(ctrl->dwMaxVideoFrameSize, &data[18]);
 	put_unaligned_le32(ctrl->dwMaxPayloadTransferSize, &data[22]);
 
-	if (size == 34) {
+	if (size >= 34) {
 		put_unaligned_le32(ctrl->dwClockFrequency, &data[26]);
 		data[30] = ctrl->bmFramingInfo;
 		data[31] = ctrl->bPreferedVersion;
diff --git a/drivers/media/v4l2-core/Kconfig b/drivers/media/v4l2-core/Kconfig
index 8e37e7c5e0f7..b97090e85996 100644
--- a/drivers/media/v4l2-core/Kconfig
+++ b/drivers/media/v4l2-core/Kconfig
@@ -65,7 +65,6 @@ config VIDEOBUF_GEN
 
 config VIDEOBUF_DMA_SG
 	tristate
-	depends on HAS_DMA
 	select VIDEOBUF_GEN
 
 config VIDEOBUF_VMALLOC
@@ -73,10 +72,5 @@ config VIDEOBUF_VMALLOC
 	select VIDEOBUF_GEN
 
 config VIDEOBUF_DMA_CONTIG
-	tristate
-	depends on HAS_DMA
-	select VIDEOBUF_GEN
-
-config VIDEOBUF_DVB
 	tristate
 	select VIDEOBUF_GEN
diff --git a/drivers/media/v4l2-core/Makefile b/drivers/media/v4l2-core/Makefile
index 7df54582e956..9ee57e1efefe 100644
--- a/drivers/media/v4l2-core/Makefile
+++ b/drivers/media/v4l2-core/Makefile
@@ -31,7 +31,6 @@ obj-$(CONFIG_VIDEOBUF_GEN) += videobuf-core.o
 obj-$(CONFIG_VIDEOBUF_DMA_SG) += videobuf-dma-sg.o
 obj-$(CONFIG_VIDEOBUF_DMA_CONTIG) += videobuf-dma-contig.o
 obj-$(CONFIG_VIDEOBUF_VMALLOC) += videobuf-vmalloc.o
-obj-$(CONFIG_VIDEOBUF_DVB) += videobuf-dvb.o
 
 ccflags-y += -I$(srctree)/drivers/media/dvb-frontends
 ccflags-y += -I$(srctree)/drivers/media/tuners
diff --git a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
index 4312935f1dfc..6481212fda77 100644
--- a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
+++ b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
@@ -22,7 +22,18 @@
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-ioctl.h>
 
-/* Use the same argument order as copy_in_user */
+/**
+ * assign_in_user() - Copy from one __user var to another one
+ *
+ * @to: __user var where data will be stored
+ * @from: __user var where data will be retrieved.
+ *
+ * As this code very often needs to allocate userspace memory, it is easier
+ * to have a macro that will do both get_user() and put_user() at once.
+ *
+ * This function complements the macros defined at asm-generic/uaccess.h.
+ * It uses the same argument order as copy_in_user()
+ */
 #define assign_in_user(to, from)					\
 ({									\
 	typeof(*from) __assign_tmp;					\
@@ -30,6 +41,73 @@
 	get_user(__assign_tmp, from) || put_user(__assign_tmp, to);	\
 })
 
+/**
+ * get_user_cast() - Stores at a kernelspace local var the contents from a
+ *		pointer with userspace data that is not tagged with __user.
+ *
+ * @__x: var where data will be stored
+ * @__ptr: var where data will be retrieved.
+ *
+ * Sometimes we need to declare a pointer without __user because it
+ * comes from a pointer struct field that will be retrieved from userspace
+ * by the 64-bit native ioctl handler. This function ensures that the
+ * @__ptr will be cast to __user before calling get_user() in order to
+ * avoid warnings with static code analyzers like smatch.
+ */
+#define get_user_cast(__x, __ptr)					\
+({									\
+	get_user(__x, (typeof(*__ptr) __user *)(__ptr));		\
+})
+
+/**
+ * put_user_force() - Stores the contents of a kernelspace local var
+ *		      into a userspace pointer, removing any __user cast.
+ *
+ * @__x: var where data will be stored
+ * @__ptr: var where data will be retrieved.
+ *
+ * Sometimes we need to remove the __user attribute from some data,
+ * by passing the __force macro. This function ensures that the
+ * @__ptr will be cast with __force before calling put_user(), in order to
+ * avoid warnings with static code analyzers like smatch.
+ */
+#define put_user_force(__x, __ptr)					\
+({									\
+	put_user((typeof(*__x) __force *)(__x), __ptr);			\
+})
+
+/**
+ * assign_in_user_cast() - Copy from one __user var to another one
+ *
+ * @to: __user var where data will be stored
+ * @from: var where data will be retrieved that needs to be cast to __user.
+ *
+ * As this code very often needs to allocate userspace memory, it is easier
+ * to have a macro that will do both get_user_cast() and put_user() at once.
+ *
+ * This function should be used instead of assign_in_user() when the @from
+ * variable was not declared as __user. See get_user_cast() for more details.
+ *
+ * This function complements the macros defined at asm-generic/uaccess.h.
+ * It uses the same argument order as copy_in_user()
+ */
+#define assign_in_user_cast(to, from)					\
+({									\
+	typeof(*from) __assign_tmp;					\
+									\
+	get_user_cast(__assign_tmp, from) || put_user(__assign_tmp, to);\
+})
+
+/**
+ * native_ioctl - Ancillary function that calls the native 64 bits ioctl
+ * handler.
+ *
+ * @file: pointer to &struct file with the file handler
+ * @cmd: ioctl to be called
+ * @arg: arguments passed from/to the ioctl handler
+ *
+ * This function calls the native ioctl handler at v4l2-dev, e. g. v4l2_ioctl()
+ */
 static long native_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
 	long ret = -ENOIOCTLCMD;
@@ -41,6 +119,21 @@ static long native_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 }
 
 
+/*
+ * Per-ioctl data copy handlers.
+ *
+ * Those come in pairs, with a get_v4l2_foo() and a put_v4l2_foo() routine,
+ * where "v4l2_foo" is the name of the V4L2 struct.
+ *
+ * They basically get two __user pointers, one with a 32-bits struct that
+ * came from the userspace call and a 64-bits struct, also allocated as
+ * userspace, but filled internally by do_video_ioctl().
+ *
+ * For ioctls that have pointers inside it, the functions will also
+ * receive an ancillary buffer with extra space, used to pass extra
+ * data to the routine.
+ */
+
 struct v4l2_clip32 {
 	struct v4l2_rect        c;
 	compat_caddr_t		next;
@@ -56,8 +149,8 @@ struct v4l2_window32 {
 	__u8                    global_alpha;
 };
 
-static int get_v4l2_window32(struct v4l2_window __user *kp,
-			     struct v4l2_window32 __user *up,
+static int get_v4l2_window32(struct v4l2_window __user *p64,
+			     struct v4l2_window32 __user *p32,
 			     void __user *aux_buf, u32 aux_space)
 {
 	struct v4l2_clip32 __user *uclips;
@@ -65,26 +158,26 @@ static int get_v4l2_window32(struct v4l2_window __user *kp,
 	compat_caddr_t p;
 	u32 clipcount;
 
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
-	    copy_in_user(&kp->w, &up->w, sizeof(up->w)) ||
-	    assign_in_user(&kp->field, &up->field) ||
-	    assign_in_user(&kp->chromakey, &up->chromakey) ||
-	    assign_in_user(&kp->global_alpha, &up->global_alpha) ||
-	    get_user(clipcount, &up->clipcount) ||
-	    put_user(clipcount, &kp->clipcount))
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) ||
+	    copy_in_user(&p64->w, &p32->w, sizeof(p32->w)) ||
+	    assign_in_user(&p64->field, &p32->field) ||
+	    assign_in_user(&p64->chromakey, &p32->chromakey) ||
+	    assign_in_user(&p64->global_alpha, &p32->global_alpha) ||
+	    get_user(clipcount, &p32->clipcount) ||
+	    put_user(clipcount, &p64->clipcount))
 		return -EFAULT;
 	if (clipcount > 2048)
 		return -EINVAL;
 	if (!clipcount)
-		return put_user(NULL, &kp->clips);
+		return put_user(NULL, &p64->clips);
 
-	if (get_user(p, &up->clips))
+	if (get_user(p, &p32->clips))
 		return -EFAULT;
 	uclips = compat_ptr(p);
 	if (aux_space < clipcount * sizeof(*kclips))
 		return -EFAULT;
 	kclips = aux_buf;
-	if (put_user(kclips, &kp->clips))
+	if (put_user(kclips, &p64->clips))
 		return -EFAULT;
 
 	while (clipcount--) {
@@ -98,27 +191,27 @@ static int get_v4l2_window32(struct v4l2_window __user *kp,
 	return 0;
 }
 
-static int put_v4l2_window32(struct v4l2_window __user *kp,
-			     struct v4l2_window32 __user *up)
+static int put_v4l2_window32(struct v4l2_window __user *p64,
+			     struct v4l2_window32 __user *p32)
 {
 	struct v4l2_clip __user *kclips;
 	struct v4l2_clip32 __user *uclips;
 	compat_caddr_t p;
 	u32 clipcount;
 
-	if (copy_in_user(&up->w, &kp->w, sizeof(kp->w)) ||
-	    assign_in_user(&up->field, &kp->field) ||
-	    assign_in_user(&up->chromakey, &kp->chromakey) ||
-	    assign_in_user(&up->global_alpha, &kp->global_alpha) ||
-	    get_user(clipcount, &kp->clipcount) ||
-	    put_user(clipcount, &up->clipcount))
+	if (copy_in_user(&p32->w, &p64->w, sizeof(p64->w)) ||
+	    assign_in_user(&p32->field, &p64->field) ||
+	    assign_in_user(&p32->chromakey, &p64->chromakey) ||
+	    assign_in_user(&p32->global_alpha, &p64->global_alpha) ||
+	    get_user(clipcount, &p64->clipcount) ||
+	    put_user(clipcount, &p32->clipcount))
 		return -EFAULT;
 	if (!clipcount)
 		return 0;
 
-	if (get_user(kclips, &kp->clips))
+	if (get_user(kclips, &p64->clips))
 		return -EFAULT;
-	if (get_user(p, &up->clips))
+	if (get_user(p, &p32->clips))
 		return -EFAULT;
 	uclips = compat_ptr(p);
 	while (clipcount--) {
@@ -161,11 +254,11 @@ struct v4l2_create_buffers32 {
 	__u32			reserved[8];
 };
 
-static int __bufsize_v4l2_format(struct v4l2_format32 __user *up, u32 *size)
+static int __bufsize_v4l2_format(struct v4l2_format32 __user *p32, u32 *size)
 {
 	u32 type;
 
-	if (get_user(type, &up->type))
+	if (get_user(type, &p32->type))
 		return -EFAULT;
 
 	switch (type) {
@@ -173,7 +266,7 @@ static int __bufsize_v4l2_format(struct v4l2_format32 __user *up, u32 *size)
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY: {
 		u32 clipcount;
 
-		if (get_user(clipcount, &up->fmt.win.clipcount))
+		if (get_user(clipcount, &p32->fmt.win.clipcount))
 			return -EFAULT;
 		if (clipcount > 2048)
 			return -EINVAL;
@@ -186,141 +279,141 @@ static int __bufsize_v4l2_format(struct v4l2_format32 __user *up, u32 *size)
 	}
 }
 
-static int bufsize_v4l2_format(struct v4l2_format32 __user *up, u32 *size)
+static int bufsize_v4l2_format(struct v4l2_format32 __user *p32, u32 *size)
 {
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)))
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)))
 		return -EFAULT;
-	return __bufsize_v4l2_format(up, size);
+	return __bufsize_v4l2_format(p32, size);
 }
 
-static int __get_v4l2_format32(struct v4l2_format __user *kp,
-			       struct v4l2_format32 __user *up,
+static int __get_v4l2_format32(struct v4l2_format __user *p64,
+			       struct v4l2_format32 __user *p32,
 			       void __user *aux_buf, u32 aux_space)
 {
 	u32 type;
 
-	if (get_user(type, &up->type) || put_user(type, &kp->type))
+	if (get_user(type, &p32->type) || put_user(type, &p64->type))
 		return -EFAULT;
 
 	switch (type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
-		return copy_in_user(&kp->fmt.pix, &up->fmt.pix,
-				    sizeof(kp->fmt.pix)) ? -EFAULT : 0;
+		return copy_in_user(&p64->fmt.pix, &p32->fmt.pix,
+				    sizeof(p64->fmt.pix)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
-		return copy_in_user(&kp->fmt.pix_mp, &up->fmt.pix_mp,
-				    sizeof(kp->fmt.pix_mp)) ? -EFAULT : 0;
+		return copy_in_user(&p64->fmt.pix_mp, &p32->fmt.pix_mp,
+				    sizeof(p64->fmt.pix_mp)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY:
-		return get_v4l2_window32(&kp->fmt.win, &up->fmt.win,
+		return get_v4l2_window32(&p64->fmt.win, &p32->fmt.win,
 					 aux_buf, aux_space);
 	case V4L2_BUF_TYPE_VBI_CAPTURE:
 	case V4L2_BUF_TYPE_VBI_OUTPUT:
-		return copy_in_user(&kp->fmt.vbi, &up->fmt.vbi,
-				    sizeof(kp->fmt.vbi)) ? -EFAULT : 0;
+		return copy_in_user(&p64->fmt.vbi, &p32->fmt.vbi,
+				    sizeof(p64->fmt.vbi)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE:
 	case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT:
-		return copy_in_user(&kp->fmt.sliced, &up->fmt.sliced,
-				    sizeof(kp->fmt.sliced)) ? -EFAULT : 0;
+		return copy_in_user(&p64->fmt.sliced, &p32->fmt.sliced,
+				    sizeof(p64->fmt.sliced)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_SDR_CAPTURE:
 	case V4L2_BUF_TYPE_SDR_OUTPUT:
-		return copy_in_user(&kp->fmt.sdr, &up->fmt.sdr,
-				    sizeof(kp->fmt.sdr)) ? -EFAULT : 0;
+		return copy_in_user(&p64->fmt.sdr, &p32->fmt.sdr,
+				    sizeof(p64->fmt.sdr)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_META_CAPTURE:
-		return copy_in_user(&kp->fmt.meta, &up->fmt.meta,
-				    sizeof(kp->fmt.meta)) ? -EFAULT : 0;
+		return copy_in_user(&p64->fmt.meta, &p32->fmt.meta,
+				    sizeof(p64->fmt.meta)) ? -EFAULT : 0;
 	default:
 		return -EINVAL;
 	}
 }
 
-static int get_v4l2_format32(struct v4l2_format __user *kp,
-			     struct v4l2_format32 __user *up,
+static int get_v4l2_format32(struct v4l2_format __user *p64,
+			     struct v4l2_format32 __user *p32,
 			     void __user *aux_buf, u32 aux_space)
 {
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)))
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)))
 		return -EFAULT;
-	return __get_v4l2_format32(kp, up, aux_buf, aux_space);
+	return __get_v4l2_format32(p64, p32, aux_buf, aux_space);
 }
 
-static int bufsize_v4l2_create(struct v4l2_create_buffers32 __user *up,
+static int bufsize_v4l2_create(struct v4l2_create_buffers32 __user *p32,
 			       u32 *size)
 {
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)))
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)))
 		return -EFAULT;
-	return __bufsize_v4l2_format(&up->format, size);
+	return __bufsize_v4l2_format(&p32->format, size);
 }
 
-static int get_v4l2_create32(struct v4l2_create_buffers __user *kp,
-			     struct v4l2_create_buffers32 __user *up,
+static int get_v4l2_create32(struct v4l2_create_buffers __user *p64,
+			     struct v4l2_create_buffers32 __user *p32,
 			     void __user *aux_buf, u32 aux_space)
 {
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
-	    copy_in_user(kp, up,
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) ||
+	    copy_in_user(p64, p32,
 			 offsetof(struct v4l2_create_buffers32, format)))
 		return -EFAULT;
-	return __get_v4l2_format32(&kp->format, &up->format,
+	return __get_v4l2_format32(&p64->format, &p32->format,
 				   aux_buf, aux_space);
 }
 
-static int __put_v4l2_format32(struct v4l2_format __user *kp,
-			       struct v4l2_format32 __user *up)
+static int __put_v4l2_format32(struct v4l2_format __user *p64,
+			       struct v4l2_format32 __user *p32)
 {
 	u32 type;
 
-	if (get_user(type, &kp->type))
+	if (get_user(type, &p64->type))
 		return -EFAULT;
 
 	switch (type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
-		return copy_in_user(&up->fmt.pix, &kp->fmt.pix,
-				    sizeof(kp->fmt.pix)) ? -EFAULT : 0;
+		return copy_in_user(&p32->fmt.pix, &p64->fmt.pix,
+				    sizeof(p64->fmt.pix)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
-		return copy_in_user(&up->fmt.pix_mp, &kp->fmt.pix_mp,
-				    sizeof(kp->fmt.pix_mp)) ? -EFAULT : 0;
+		return copy_in_user(&p32->fmt.pix_mp, &p64->fmt.pix_mp,
+				    sizeof(p64->fmt.pix_mp)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY:
-		return put_v4l2_window32(&kp->fmt.win, &up->fmt.win);
+		return put_v4l2_window32(&p64->fmt.win, &p32->fmt.win);
 	case V4L2_BUF_TYPE_VBI_CAPTURE:
 	case V4L2_BUF_TYPE_VBI_OUTPUT:
-		return copy_in_user(&up->fmt.vbi, &kp->fmt.vbi,
-				    sizeof(kp->fmt.vbi)) ? -EFAULT : 0;
+		return copy_in_user(&p32->fmt.vbi, &p64->fmt.vbi,
+				    sizeof(p64->fmt.vbi)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE:
 	case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT:
-		return copy_in_user(&up->fmt.sliced, &kp->fmt.sliced,
-				    sizeof(kp->fmt.sliced)) ? -EFAULT : 0;
+		return copy_in_user(&p32->fmt.sliced, &p64->fmt.sliced,
+				    sizeof(p64->fmt.sliced)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_SDR_CAPTURE:
 	case V4L2_BUF_TYPE_SDR_OUTPUT:
-		return copy_in_user(&up->fmt.sdr, &kp->fmt.sdr,
-				    sizeof(kp->fmt.sdr)) ? -EFAULT : 0;
+		return copy_in_user(&p32->fmt.sdr, &p64->fmt.sdr,
+				    sizeof(p64->fmt.sdr)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_META_CAPTURE:
-		return copy_in_user(&up->fmt.meta, &kp->fmt.meta,
-				    sizeof(kp->fmt.meta)) ? -EFAULT : 0;
+		return copy_in_user(&p32->fmt.meta, &p64->fmt.meta,
+				    sizeof(p64->fmt.meta)) ? -EFAULT : 0;
 	default:
 		return -EINVAL;
 	}
 }
 
-static int put_v4l2_format32(struct v4l2_format __user *kp,
-			     struct v4l2_format32 __user *up)
+static int put_v4l2_format32(struct v4l2_format __user *p64,
+			     struct v4l2_format32 __user *p32)
 {
-	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)))
+	if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)))
 		return -EFAULT;
-	return __put_v4l2_format32(kp, up);
+	return __put_v4l2_format32(p64, p32);
 }
 
-static int put_v4l2_create32(struct v4l2_create_buffers __user *kp,
-			     struct v4l2_create_buffers32 __user *up)
+static int put_v4l2_create32(struct v4l2_create_buffers __user *p64,
+			     struct v4l2_create_buffers32 __user *p32)
 {
-	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
-	    copy_in_user(up, kp,
+	if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) ||
+	    copy_in_user(p32, p64,
 			 offsetof(struct v4l2_create_buffers32, format)) ||
-	    copy_in_user(up->reserved, kp->reserved, sizeof(kp->reserved)))
+	    copy_in_user(p32->reserved, p64->reserved, sizeof(p64->reserved)))
 		return -EFAULT;
-	return __put_v4l2_format32(&kp->format, &up->format);
+	return __put_v4l2_format32(&p64->format, &p32->format);
 }
 
 struct v4l2_standard32 {
@@ -332,27 +425,27 @@ struct v4l2_standard32 {
 	__u32		     reserved[4];
 };
 
-static int get_v4l2_standard32(struct v4l2_standard __user *kp,
-			       struct v4l2_standard32 __user *up)
+static int get_v4l2_standard32(struct v4l2_standard __user *p64,
+			       struct v4l2_standard32 __user *p32)
 {
 	/* other fields are not set by the user, nor used by the driver */
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
-	    assign_in_user(&kp->index, &up->index))
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) ||
+	    assign_in_user(&p64->index, &p32->index))
 		return -EFAULT;
 	return 0;
 }
 
-static int put_v4l2_standard32(struct v4l2_standard __user *kp,
-			       struct v4l2_standard32 __user *up)
+static int put_v4l2_standard32(struct v4l2_standard __user *p64,
+			       struct v4l2_standard32 __user *p32)
 {
-	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
-	    assign_in_user(&up->index, &kp->index) ||
-	    assign_in_user(&up->id, &kp->id) ||
-	    copy_in_user(up->name, kp->name, sizeof(up->name)) ||
-	    copy_in_user(&up->frameperiod, &kp->frameperiod,
-			 sizeof(up->frameperiod)) ||
-	    assign_in_user(&up->framelines, &kp->framelines) ||
-	    copy_in_user(up->reserved, kp->reserved, sizeof(up->reserved)))
+	if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) ||
+	    assign_in_user(&p32->index, &p64->index) ||
+	    assign_in_user(&p32->id, &p64->id) ||
+	    copy_in_user(p32->name, p64->name, sizeof(p32->name)) ||
+	    copy_in_user(&p32->frameperiod, &p64->frameperiod,
+			 sizeof(p32->frameperiod)) ||
+	    assign_in_user(&p32->framelines, &p64->framelines) ||
+	    copy_in_user(p32->reserved, p64->reserved, sizeof(p32->reserved)))
 		return -EFAULT;
 	return 0;
 }
@@ -392,31 +485,31 @@ struct v4l2_buffer32 {
 	__u32			reserved;
 };
 
-static int get_v4l2_plane32(struct v4l2_plane __user *up,
-			    struct v4l2_plane32 __user *up32,
+static int get_v4l2_plane32(struct v4l2_plane __user *p64,
+			    struct v4l2_plane32 __user *p32,
 			    enum v4l2_memory memory)
 {
 	compat_ulong_t p;
 
-	if (copy_in_user(up, up32, 2 * sizeof(__u32)) ||
-	    copy_in_user(&up->data_offset, &up32->data_offset,
-			 sizeof(up->data_offset)))
+	if (copy_in_user(p64, p32, 2 * sizeof(__u32)) ||
+	    copy_in_user(&p64->data_offset, &p32->data_offset,
+			 sizeof(p64->data_offset)))
 		return -EFAULT;
 
 	switch (memory) {
 	case V4L2_MEMORY_MMAP:
 	case V4L2_MEMORY_OVERLAY:
-		if (copy_in_user(&up->m.mem_offset, &up32->m.mem_offset,
-				 sizeof(up32->m.mem_offset)))
+		if (copy_in_user(&p64->m.mem_offset, &p32->m.mem_offset,
+				 sizeof(p32->m.mem_offset)))
 			return -EFAULT;
 		break;
 	case V4L2_MEMORY_USERPTR:
-		if (get_user(p, &up32->m.userptr) ||
-		    put_user((unsigned long)compat_ptr(p), &up->m.userptr))
+		if (get_user(p, &p32->m.userptr) ||
+		    put_user((unsigned long)compat_ptr(p), &p64->m.userptr))
 			return -EFAULT;
 		break;
 	case V4L2_MEMORY_DMABUF:
-		if (copy_in_user(&up->m.fd, &up32->m.fd, sizeof(up32->m.fd)))
+		if (copy_in_user(&p64->m.fd, &p32->m.fd, sizeof(p32->m.fd)))
 			return -EFAULT;
 		break;
 	}
@@ -424,32 +517,32 @@ static int get_v4l2_plane32(struct v4l2_plane __user *up,
 	return 0;
 }
 
-static int put_v4l2_plane32(struct v4l2_plane __user *up,
-			    struct v4l2_plane32 __user *up32,
+static int put_v4l2_plane32(struct v4l2_plane __user *p64,
+			    struct v4l2_plane32 __user *p32,
 			    enum v4l2_memory memory)
 {
 	unsigned long p;
 
-	if (copy_in_user(up32, up, 2 * sizeof(__u32)) ||
-	    copy_in_user(&up32->data_offset, &up->data_offset,
-			 sizeof(up->data_offset)))
+	if (copy_in_user(p32, p64, 2 * sizeof(__u32)) ||
+	    copy_in_user(&p32->data_offset, &p64->data_offset,
+			 sizeof(p64->data_offset)))
 		return -EFAULT;
 
 	switch (memory) {
 	case V4L2_MEMORY_MMAP:
 	case V4L2_MEMORY_OVERLAY:
-		if (copy_in_user(&up32->m.mem_offset, &up->m.mem_offset,
-				 sizeof(up->m.mem_offset)))
+		if (copy_in_user(&p32->m.mem_offset, &p64->m.mem_offset,
+				 sizeof(p64->m.mem_offset)))
 			return -EFAULT;
 		break;
 	case V4L2_MEMORY_USERPTR:
-		if (get_user(p, &up->m.userptr) ||
-		    put_user((compat_ulong_t)ptr_to_compat((__force void *)p),
-			     &up32->m.userptr))
+		if (get_user(p, &p64->m.userptr) ||
+		    put_user((compat_ulong_t)ptr_to_compat((void __user *)p),
+			     &p32->m.userptr))
 			return -EFAULT;
 		break;
 	case V4L2_MEMORY_DMABUF:
-		if (copy_in_user(&up32->m.fd, &up->m.fd, sizeof(up->m.fd)))
+		if (copy_in_user(&p32->m.fd, &p64->m.fd, sizeof(p64->m.fd)))
 			return -EFAULT;
 		break;
 	}
@@ -457,14 +550,14 @@ static int put_v4l2_plane32(struct v4l2_plane __user *up,
 	return 0;
 }
 
-static int bufsize_v4l2_buffer(struct v4l2_buffer32 __user *up, u32 *size)
+static int bufsize_v4l2_buffer(struct v4l2_buffer32 __user *p32, u32 *size)
 {
 	u32 type;
 	u32 length;
 
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
-	    get_user(type, &up->type) ||
-	    get_user(length, &up->length))
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) ||
+	    get_user(type, &p32->type) ||
+	    get_user(length, &p32->length))
 		return -EFAULT;
 
 	if (V4L2_TYPE_IS_MULTIPLANAR(type)) {
@@ -482,8 +575,8 @@ static int bufsize_v4l2_buffer(struct v4l2_buffer32 __user *up, u32 *size)
 	return 0;
 }
 
-static int get_v4l2_buffer32(struct v4l2_buffer __user *kp,
-			     struct v4l2_buffer32 __user *up,
+static int get_v4l2_buffer32(struct v4l2_buffer __user *p64,
+			     struct v4l2_buffer32 __user *p32,
 			     void __user *aux_buf, u32 aux_space)
 {
 	u32 type;
@@ -494,24 +587,24 @@ static int get_v4l2_buffer32(struct v4l2_buffer __user *kp,
 	compat_caddr_t p;
 	int ret;
 
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
-	    assign_in_user(&kp->index, &up->index) ||
-	    get_user(type, &up->type) ||
-	    put_user(type, &kp->type) ||
-	    assign_in_user(&kp->flags, &up->flags) ||
-	    get_user(memory, &up->memory) ||
-	    put_user(memory, &kp->memory) ||
-	    get_user(length, &up->length) ||
-	    put_user(length, &kp->length))
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) ||
+	    assign_in_user(&p64->index, &p32->index) ||
+	    get_user(type, &p32->type) ||
+	    put_user(type, &p64->type) ||
+	    assign_in_user(&p64->flags, &p32->flags) ||
+	    get_user(memory, &p32->memory) ||
+	    put_user(memory, &p64->memory) ||
+	    get_user(length, &p32->length) ||
+	    put_user(length, &p64->length))
 		return -EFAULT;
 
 	if (V4L2_TYPE_IS_OUTPUT(type))
-		if (assign_in_user(&kp->bytesused, &up->bytesused) ||
-		    assign_in_user(&kp->field, &up->field) ||
-		    assign_in_user(&kp->timestamp.tv_sec,
-				   &up->timestamp.tv_sec) ||
-		    assign_in_user(&kp->timestamp.tv_usec,
-				   &up->timestamp.tv_usec))
+		if (assign_in_user(&p64->bytesused, &p32->bytesused) ||
+		    assign_in_user(&p64->field, &p32->field) ||
+		    assign_in_user(&p64->timestamp.tv_sec,
+				   &p32->timestamp.tv_sec) ||
+		    assign_in_user(&p64->timestamp.tv_usec,
+				   &p32->timestamp.tv_usec))
 			return -EFAULT;
 
 	if (V4L2_TYPE_IS_MULTIPLANAR(type)) {
@@ -522,12 +615,12 @@ static int get_v4l2_buffer32(struct v4l2_buffer __user *kp,
 			 * num_planes == 0 is legal, e.g. when userspace doesn't
 			 * need planes array on DQBUF
 			 */
-			return put_user(NULL, &kp->m.planes);
+			return put_user(NULL, &p64->m.planes);
 		}
 		if (num_planes > VIDEO_MAX_PLANES)
 			return -EINVAL;
 
-		if (get_user(p, &up->m.planes))
+		if (get_user(p, &p32->m.planes))
 			return -EFAULT;
 
 		uplane32 = compat_ptr(p);
@@ -543,8 +636,7 @@ static int get_v4l2_buffer32(struct v4l2_buffer __user *kp,
 			return -EFAULT;
 
 		uplane = aux_buf;
-		if (put_user((__force struct v4l2_plane *)uplane,
-			     &kp->m.planes))
+		if (put_user_force(uplane, &p64->m.planes))
 			return -EFAULT;
 
 		while (num_planes--) {
@@ -558,20 +650,20 @@ static int get_v4l2_buffer32(struct v4l2_buffer __user *kp,
 		switch (memory) {
 		case V4L2_MEMORY_MMAP:
 		case V4L2_MEMORY_OVERLAY:
-			if (assign_in_user(&kp->m.offset, &up->m.offset))
+			if (assign_in_user(&p64->m.offset, &p32->m.offset))
 				return -EFAULT;
 			break;
 		case V4L2_MEMORY_USERPTR: {
 			compat_ulong_t userptr;
 
-			if (get_user(userptr, &up->m.userptr) ||
+			if (get_user(userptr, &p32->m.userptr) ||
 			    put_user((unsigned long)compat_ptr(userptr),
-				     &kp->m.userptr))
+				     &p64->m.userptr))
 				return -EFAULT;
 			break;
 		}
 		case V4L2_MEMORY_DMABUF:
-			if (assign_in_user(&kp->m.fd, &up->m.fd))
+			if (assign_in_user(&p64->m.fd, &p32->m.fd))
 				return -EFAULT;
 			break;
 		}
@@ -580,36 +672,36 @@ static int get_v4l2_buffer32(struct v4l2_buffer __user *kp,
 	return 0;
 }
 
-static int put_v4l2_buffer32(struct v4l2_buffer __user *kp,
-			     struct v4l2_buffer32 __user *up)
+static int put_v4l2_buffer32(struct v4l2_buffer __user *p64,
+			     struct v4l2_buffer32 __user *p32)
 {
 	u32 type;
 	u32 length;
 	enum v4l2_memory memory;
 	struct v4l2_plane32 __user *uplane32;
-	struct v4l2_plane __user *uplane;
+	struct v4l2_plane *uplane;
 	compat_caddr_t p;
 	int ret;
 
-	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
-	    assign_in_user(&up->index, &kp->index) ||
-	    get_user(type, &kp->type) ||
-	    put_user(type, &up->type) ||
-	    assign_in_user(&up->flags, &kp->flags) ||
-	    get_user(memory, &kp->memory) ||
-	    put_user(memory, &up->memory))
+	if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) ||
+	    assign_in_user(&p32->index, &p64->index) ||
+	    get_user(type, &p64->type) ||
+	    put_user(type, &p32->type) ||
+	    assign_in_user(&p32->flags, &p64->flags) ||
+	    get_user(memory, &p64->memory) ||
+	    put_user(memory, &p32->memory))
 		return -EFAULT;
 
-	if (assign_in_user(&up->bytesused, &kp->bytesused) ||
-	    assign_in_user(&up->field, &kp->field) ||
-	    assign_in_user(&up->timestamp.tv_sec, &kp->timestamp.tv_sec) ||
-	    assign_in_user(&up->timestamp.tv_usec, &kp->timestamp.tv_usec) ||
-	    copy_in_user(&up->timecode, &kp->timecode, sizeof(kp->timecode)) ||
-	    assign_in_user(&up->sequence, &kp->sequence) ||
-	    assign_in_user(&up->reserved2, &kp->reserved2) ||
-	    assign_in_user(&up->reserved, &kp->reserved) ||
-	    get_user(length, &kp->length) ||
-	    put_user(length, &up->length))
+	if (assign_in_user(&p32->bytesused, &p64->bytesused) ||
+	    assign_in_user(&p32->field, &p64->field) ||
+	    assign_in_user(&p32->timestamp.tv_sec, &p64->timestamp.tv_sec) ||
+	    assign_in_user(&p32->timestamp.tv_usec, &p64->timestamp.tv_usec) ||
+	    copy_in_user(&p32->timecode, &p64->timecode, sizeof(p64->timecode)) ||
+	    assign_in_user(&p32->sequence, &p64->sequence) ||
+	    assign_in_user(&p32->reserved2, &p64->reserved2) ||
+	    assign_in_user(&p32->reserved, &p64->reserved) ||
+	    get_user(length, &p64->length) ||
+	    put_user(length, &p32->length))
 		return -EFAULT;
 
 	if (V4L2_TYPE_IS_MULTIPLANAR(type)) {
@@ -617,15 +709,23 @@ static int put_v4l2_buffer32(struct v4l2_buffer __user *kp,
 
 		if (num_planes == 0)
 			return 0;
-
-		if (get_user(uplane, ((__force struct v4l2_plane __user **)&kp->m.planes)))
+		/* We need to define uplane without __user, even though
+		 * it does point to data in userspace here. The reason is
+		 * that v4l2-ioctl.c copies it from userspace to kernelspace,
+		 * so its definition in videodev2.h doesn't have a
+		 * __user markup. Defining uplane with __user causes
+		 * smatch warnings, so instead declare it without __user
+		 * and cast it as a userspace pointer to put_v4l2_plane32().
+		 */
+		if (get_user(uplane, &p64->m.planes))
 			return -EFAULT;
-		if (get_user(p, &up->m.planes))
+		if (get_user(p, &p32->m.planes))
 			return -EFAULT;
 		uplane32 = compat_ptr(p);
 
 		while (num_planes--) {
-			ret = put_v4l2_plane32(uplane, uplane32, memory);
+			ret = put_v4l2_plane32((void __user *)uplane,
+					       uplane32, memory);
 			if (ret)
 				return ret;
 			++uplane;
@@ -635,15 +735,15 @@ static int put_v4l2_buffer32(struct v4l2_buffer __user *kp,
 		switch (memory) {
 		case V4L2_MEMORY_MMAP:
 		case V4L2_MEMORY_OVERLAY:
-			if (assign_in_user(&up->m.offset, &kp->m.offset))
+			if (assign_in_user(&p32->m.offset, &p64->m.offset))
 				return -EFAULT;
 			break;
 		case V4L2_MEMORY_USERPTR:
-			if (assign_in_user(&up->m.userptr, &kp->m.userptr))
+			if (assign_in_user(&p32->m.userptr, &p64->m.userptr))
 				return -EFAULT;
 			break;
 		case V4L2_MEMORY_DMABUF:
-			if (assign_in_user(&up->m.fd, &kp->m.fd))
+			if (assign_in_user(&p32->m.fd, &p64->m.fd))
 				return -EFAULT;
 			break;
 		}
@@ -668,32 +768,32 @@ struct v4l2_framebuffer32 {
 	} fmt;
 };
 
-static int get_v4l2_framebuffer32(struct v4l2_framebuffer __user *kp,
-				  struct v4l2_framebuffer32 __user *up)
+static int get_v4l2_framebuffer32(struct v4l2_framebuffer __user *p64,
+				  struct v4l2_framebuffer32 __user *p32)
 {
 	compat_caddr_t tmp;
 
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
-	    get_user(tmp, &up->base) ||
-	    put_user((__force void *)compat_ptr(tmp), &kp->base) ||
-	    assign_in_user(&kp->capability, &up->capability) ||
-	    assign_in_user(&kp->flags, &up->flags) ||
-	    copy_in_user(&kp->fmt, &up->fmt, sizeof(kp->fmt)))
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) ||
+	    get_user(tmp, &p32->base) ||
+	    put_user_force(compat_ptr(tmp), &p64->base) ||
+	    assign_in_user(&p64->capability, &p32->capability) ||
+	    assign_in_user(&p64->flags, &p32->flags) ||
+	    copy_in_user(&p64->fmt, &p32->fmt, sizeof(p64->fmt)))
 		return -EFAULT;
 	return 0;
 }
 
-static int put_v4l2_framebuffer32(struct v4l2_framebuffer __user *kp,
-				  struct v4l2_framebuffer32 __user *up)
+static int put_v4l2_framebuffer32(struct v4l2_framebuffer __user *p64,
+				  struct v4l2_framebuffer32 __user *p32)
 {
 	void *base;
 
-	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
-	    get_user(base, &kp->base) ||
-	    put_user(ptr_to_compat(base), &up->base) ||
-	    assign_in_user(&up->capability, &kp->capability) ||
-	    assign_in_user(&up->flags, &kp->flags) ||
-	    copy_in_user(&up->fmt, &kp->fmt, sizeof(kp->fmt)))
+	if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) ||
+	    get_user(base, &p64->base) ||
+	    put_user(ptr_to_compat((void __user *)base), &p32->base) ||
+	    assign_in_user(&p32->capability, &p64->capability) ||
+	    assign_in_user(&p32->flags, &p64->flags) ||
+	    copy_in_user(&p32->fmt, &p64->fmt, sizeof(p64->fmt)))
 		return -EFAULT;
 	return 0;
 }
@@ -714,18 +814,18 @@ struct v4l2_input32 {
  * The 64-bit v4l2_input struct has extra padding at the end of the struct.
  * Otherwise it is identical to the 32-bit version.
  */
-static inline int get_v4l2_input32(struct v4l2_input __user *kp,
-				   struct v4l2_input32 __user *up)
+static inline int get_v4l2_input32(struct v4l2_input __user *p64,
+				   struct v4l2_input32 __user *p32)
 {
-	if (copy_in_user(kp, up, sizeof(*up)))
+	if (copy_in_user(p64, p32, sizeof(*p32)))
 		return -EFAULT;
 	return 0;
 }
 
-static inline int put_v4l2_input32(struct v4l2_input __user *kp,
-				   struct v4l2_input32 __user *up)
+static inline int put_v4l2_input32(struct v4l2_input __user *p64,
+				   struct v4l2_input32 __user *p32)
 {
-	if (copy_in_user(up, kp, sizeof(*up)))
+	if (copy_in_user(p32, p64, sizeof(*p32)))
 		return -EFAULT;
 	return 0;
 }
@@ -779,13 +879,13 @@ static inline bool ctrl_is_pointer(struct file *file, u32 id)
 		(qec.flags & V4L2_CTRL_FLAG_HAS_PAYLOAD);
 }
 
-static int bufsize_v4l2_ext_controls(struct v4l2_ext_controls32 __user *up,
+static int bufsize_v4l2_ext_controls(struct v4l2_ext_controls32 __user *p32,
 				     u32 *size)
 {
 	u32 count;
 
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
-	    get_user(count, &up->count))
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) ||
+	    get_user(count, &p32->count))
 		return -EFAULT;
 	if (count > V4L2_CID_MAX_CTRLS)
 		return -EINVAL;
@@ -794,8 +894,8 @@ static int bufsize_v4l2_ext_controls(struct v4l2_ext_controls32 __user *up,
 }
 
 static int get_v4l2_ext_controls32(struct file *file,
-				   struct v4l2_ext_controls __user *kp,
-				   struct v4l2_ext_controls32 __user *up,
+				   struct v4l2_ext_controls __user *p64,
+				   struct v4l2_ext_controls32 __user *p32,
 				   void __user *aux_buf, u32 aux_space)
 {
 	struct v4l2_ext_control32 __user *ucontrols;
@@ -804,19 +904,19 @@ static int get_v4l2_ext_controls32(struct file *file,
 	u32 n;
 	compat_caddr_t p;
 
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
-	    assign_in_user(&kp->which, &up->which) ||
-	    get_user(count, &up->count) ||
-	    put_user(count, &kp->count) ||
-	    assign_in_user(&kp->error_idx, &up->error_idx) ||
-	    copy_in_user(kp->reserved, up->reserved, sizeof(kp->reserved)))
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) ||
+	    assign_in_user(&p64->which, &p32->which) ||
+	    get_user(count, &p32->count) ||
+	    put_user(count, &p64->count) ||
+	    assign_in_user(&p64->error_idx, &p32->error_idx) ||
+	    copy_in_user(p64->reserved, p32->reserved, sizeof(p64->reserved)))
 		return -EFAULT;
 
 	if (count == 0)
-		return put_user(NULL, &kp->controls);
+		return put_user(NULL, &p64->controls);
 	if (count > V4L2_CID_MAX_CTRLS)
 		return -EINVAL;
-	if (get_user(p, &up->controls))
+	if (get_user(p, &p32->controls))
 		return -EFAULT;
 	ucontrols = compat_ptr(p);
 	if (!access_ok(VERIFY_READ, ucontrols, count * sizeof(*ucontrols)))
@@ -824,8 +924,7 @@ static int get_v4l2_ext_controls32(struct file *file,
 	if (aux_space < count * sizeof(*kcontrols))
 		return -EFAULT;
 	kcontrols = aux_buf;
-	if (put_user((__force struct v4l2_ext_control *)kcontrols,
-		     &kp->controls))
+	if (put_user_force(kcontrols, &p64->controls))
 		return -EFAULT;
 
 	for (n = 0; n < count; n++) {
@@ -853,27 +952,35 @@ static int get_v4l2_ext_controls32(struct file *file,
 }
 
 static int put_v4l2_ext_controls32(struct file *file,
-				   struct v4l2_ext_controls __user *kp,
-				   struct v4l2_ext_controls32 __user *up)
+				   struct v4l2_ext_controls __user *p64,
+				   struct v4l2_ext_controls32 __user *p32)
 {
 	struct v4l2_ext_control32 __user *ucontrols;
-	struct v4l2_ext_control __user *kcontrols;
+	struct v4l2_ext_control *kcontrols;
 	u32 count;
 	u32 n;
 	compat_caddr_t p;
 
-	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
-	    assign_in_user(&up->which, &kp->which) ||
-	    get_user(count, &kp->count) ||
-	    put_user(count, &up->count) ||
-	    assign_in_user(&up->error_idx, &kp->error_idx) ||
-	    copy_in_user(up->reserved, kp->reserved, sizeof(up->reserved)) ||
-	    get_user(kcontrols, &kp->controls))
+	/*
+	 * We need to define kcontrols without __user, even though it does
+	 * point to data in userspace here. The reason is that v4l2-ioctl.c
+	 * copies it from userspace to kernelspace, so its definition in
+	 * videodev2.h doesn't have a __user markup. Defining kcontrols
+	 * with __user causes smatch warnings, so instead declare it
+	 * without __user and cast it as a userspace pointer where needed.
+	 */
+	if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) ||
+	    assign_in_user(&p32->which, &p64->which) ||
+	    get_user(count, &p64->count) ||
+	    put_user(count, &p32->count) ||
+	    assign_in_user(&p32->error_idx, &p64->error_idx) ||
+	    copy_in_user(p32->reserved, p64->reserved, sizeof(p32->reserved)) ||
+	    get_user(kcontrols, &p64->controls))
 		return -EFAULT;
 
-	if (!count)
+	if (!count || count > (U32_MAX/sizeof(*ucontrols)))
 		return 0;
-	if (get_user(p, &up->controls))
+	if (get_user(p, &p32->controls))
 		return -EFAULT;
 	ucontrols = compat_ptr(p);
 	if (!access_ok(VERIFY_WRITE, ucontrols, count * sizeof(*ucontrols)))
@@ -883,10 +990,11 @@ static int put_v4l2_ext_controls32(struct file *file,
 		unsigned int size = sizeof(*ucontrols);
 		u32 id;
 
-		if (get_user(id, &kcontrols->id) ||
+		if (get_user_cast(id, &kcontrols->id) ||
 		    put_user(id, &ucontrols->id) ||
-		    assign_in_user(&ucontrols->size, &kcontrols->size) ||
-		    copy_in_user(&ucontrols->reserved2, &kcontrols->reserved2,
+		    assign_in_user_cast(&ucontrols->size, &kcontrols->size) ||
+		    copy_in_user(&ucontrols->reserved2,
+				 (void __user *)&kcontrols->reserved2,
 				 sizeof(ucontrols->reserved2)))
 			return -EFAULT;
 
@@ -898,7 +1006,8 @@ static int put_v4l2_ext_controls32(struct file *file,
 		if (ctrl_is_pointer(file, id))
 			size -= sizeof(ucontrols->value64);
 
-		if (copy_in_user(ucontrols, kcontrols, size))
+		if (copy_in_user(ucontrols,
+				 (void __user *)kcontrols, size))
 			return -EFAULT;
 
 		ucontrols++;
@@ -920,18 +1029,18 @@ struct v4l2_event32 {
 	__u32				reserved[8];
 };
 
-static int put_v4l2_event32(struct v4l2_event __user *kp,
-			    struct v4l2_event32 __user *up)
+static int put_v4l2_event32(struct v4l2_event __user *p64,
+			    struct v4l2_event32 __user *p32)
 {
-	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
-	    assign_in_user(&up->type, &kp->type) ||
-	    copy_in_user(&up->u, &kp->u, sizeof(kp->u)) ||
-	    assign_in_user(&up->pending, &kp->pending) ||
-	    assign_in_user(&up->sequence, &kp->sequence) ||
-	    assign_in_user(&up->timestamp.tv_sec, &kp->timestamp.tv_sec) ||
-	    assign_in_user(&up->timestamp.tv_nsec, &kp->timestamp.tv_nsec) ||
-	    assign_in_user(&up->id, &kp->id) ||
-	    copy_in_user(up->reserved, kp->reserved, sizeof(up->reserved)))
+	if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) ||
+	    assign_in_user(&p32->type, &p64->type) ||
+	    copy_in_user(&p32->u, &p64->u, sizeof(p64->u)) ||
+	    assign_in_user(&p32->pending, &p64->pending) ||
+	    assign_in_user(&p32->sequence, &p64->sequence) ||
+	    assign_in_user(&p32->timestamp.tv_sec, &p64->timestamp.tv_sec) ||
+	    assign_in_user(&p32->timestamp.tv_nsec, &p64->timestamp.tv_nsec) ||
+	    assign_in_user(&p32->id, &p64->id) ||
+	    copy_in_user(p32->reserved, p64->reserved, sizeof(p32->reserved)))
 		return -EFAULT;
 	return 0;
 }
@@ -944,38 +1053,45 @@ struct v4l2_edid32 {
 	compat_caddr_t edid;
 };
 
-static int get_v4l2_edid32(struct v4l2_edid __user *kp,
-			   struct v4l2_edid32 __user *up)
+static int get_v4l2_edid32(struct v4l2_edid __user *p64,
+			   struct v4l2_edid32 __user *p32)
 {
 	compat_uptr_t tmp;
 
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
-	    assign_in_user(&kp->pad, &up->pad) ||
-	    assign_in_user(&kp->start_block, &up->start_block) ||
-	    assign_in_user(&kp->blocks, &up->blocks) ||
-	    get_user(tmp, &up->edid) ||
-	    put_user(compat_ptr(tmp), &kp->edid) ||
-	    copy_in_user(kp->reserved, up->reserved, sizeof(kp->reserved)))
+	if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) ||
+	    assign_in_user(&p64->pad, &p32->pad) ||
+	    assign_in_user(&p64->start_block, &p32->start_block) ||
+	    assign_in_user_cast(&p64->blocks, &p32->blocks) ||
+	    get_user(tmp, &p32->edid) ||
+	    put_user_force(compat_ptr(tmp), &p64->edid) ||
+	    copy_in_user(p64->reserved, p32->reserved, sizeof(p64->reserved)))
 		return -EFAULT;
 	return 0;
 }
 
-static int put_v4l2_edid32(struct v4l2_edid __user *kp,
-			   struct v4l2_edid32 __user *up)
+static int put_v4l2_edid32(struct v4l2_edid __user *p64,
+			   struct v4l2_edid32 __user *p32)
 {
 	void *edid;
 
-	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
-	    assign_in_user(&up->pad, &kp->pad) ||
-	    assign_in_user(&up->start_block, &kp->start_block) ||
-	    assign_in_user(&up->blocks, &kp->blocks) ||
-	    get_user(edid, &kp->edid) ||
-	    put_user(ptr_to_compat(edid), &up->edid) ||
-	    copy_in_user(up->reserved, kp->reserved, sizeof(up->reserved)))
+	if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) ||
+	    assign_in_user(&p32->pad, &p64->pad) ||
+	    assign_in_user(&p32->start_block, &p64->start_block) ||
+	    assign_in_user(&p32->blocks, &p64->blocks) ||
+	    get_user(edid, &p64->edid) ||
+	    put_user(ptr_to_compat((void __user *)edid), &p32->edid) ||
+	    copy_in_user(p32->reserved, p64->reserved, sizeof(p32->reserved)))
 		return -EFAULT;
 	return 0;
 }
 
+/*
+ * List of ioctls that require 32-bits/64-bits conversion
+ *
+ * The V4L2 ioctls that aren't listed there don't have pointer arguments
+ * and the struct size is identical for both 32 and 64 bits versions, so
+ * they don't need translations.
+ */
 
 #define VIDIOC_G_FMT32		_IOWR('V',  4, struct v4l2_format32)
 #define VIDIOC_S_FMT32		_IOWR('V',  5, struct v4l2_format32)
@@ -1004,27 +1120,61 @@ static int put_v4l2_edid32(struct v4l2_edid __user *kp,
 #define VIDIOC_G_OUTPUT32	_IOR ('V', 46, s32)
 #define VIDIOC_S_OUTPUT32	_IOWR('V', 47, s32)
 
+/**
+ * alloc_userspace() - Allocates a 64-bits userspace pointer compatible
+ *	for calling the native 64-bits version of an ioctl.
+ *
+ * @size:	size of the structure itself to be allocated.
+ * @aux_space:	extra size needed to store "extra" data, e.g. space for
+ *		other __user data that is pointed to fields inside the
+ *		structure.
+ * @new_p64:	pointer to a pointer to be filled with the allocated struct.
+ *
+ * Return:
+ *
+ * if it can't allocate memory, either -ENOMEM or -EFAULT will be returned.
+ * Zero otherwise.
+ */
 static int alloc_userspace(unsigned int size, u32 aux_space,
-			   void __user **up_native)
+			   void __user **new_p64)
 {
-	*up_native = compat_alloc_user_space(size + aux_space);
-	if (!*up_native)
+	*new_p64 = compat_alloc_user_space(size + aux_space);
+	if (!*new_p64)
 		return -ENOMEM;
-	if (clear_user(*up_native, size))
+	if (clear_user(*new_p64, size))
 		return -EFAULT;
 	return 0;
 }
 
+/**
+ * do_video_ioctl() - Ancillary function with handles a compat32 ioctl call
+ *
+ * @file: pointer to &struct file with the file handler
+ * @cmd: ioctl to be called
+ * @arg: arguments passed from/to the ioctl handler
+ *
+ * This function is called when a 32 bits application calls a V4L2 ioctl
+ * and the Kernel is compiled with 64 bits.
+ *
+ * This function is called by v4l2_compat_ioctl32() when the function is
+ * not private to some specific driver.
+ *
+ * It converts a 32-bits struct into a 64 bits one, calls the native 64-bits
+ * ioctl handler and fills back the 32-bits struct with the results of the
+ * native call.
+ */
 static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
-	void __user *up = compat_ptr(arg);
-	void __user *up_native = NULL;
+	void __user *p32 = compat_ptr(arg);
+	void __user *new_p64 = NULL;
 	void __user *aux_buf;
 	u32 aux_space;
 	int compatible_arg = 1;
 	long err = 0;
 
-	/* First, convert the command. */
+	/*
+	 * 1. When struct size is different, converts the command.
+	 */
 	switch (cmd) {
 	case VIDIOC_G_FMT32: cmd = VIDIOC_G_FMT; break;
 	case VIDIOC_S_FMT32: cmd = VIDIOC_S_FMT; break;
@@ -1053,56 +1203,61 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar
 	case VIDIOC_S_EDID32: cmd = VIDIOC_S_EDID; break;
 	}
 
+	/*
+	 * 2. Allocates a 64-bits userspace pointer to store the
+	 * values of the ioctl and copy data from the 32-bits __user
+	 * argument into it.
+	 */
 	switch (cmd) {
 	case VIDIOC_OVERLAY:
 	case VIDIOC_STREAMON:
 	case VIDIOC_STREAMOFF:
 	case VIDIOC_S_INPUT:
 	case VIDIOC_S_OUTPUT:
-		err = alloc_userspace(sizeof(unsigned int), 0, &up_native);
-		if (!err && assign_in_user((unsigned int __user *)up_native,
-					   (compat_uint_t __user *)up))
+		err = alloc_userspace(sizeof(unsigned int), 0, &new_p64);
+		if (!err && assign_in_user((unsigned int __user *)new_p64,
+					   (compat_uint_t __user *)p32))
 			err = -EFAULT;
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_G_INPUT:
 	case VIDIOC_G_OUTPUT:
-		err = alloc_userspace(sizeof(unsigned int), 0, &up_native);
+		err = alloc_userspace(sizeof(unsigned int), 0, &new_p64);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_G_EDID:
 	case VIDIOC_S_EDID:
-		err = alloc_userspace(sizeof(struct v4l2_edid), 0, &up_native);
+		err = alloc_userspace(sizeof(struct v4l2_edid), 0, &new_p64);
 		if (!err)
-			err = get_v4l2_edid32(up_native, up);
+			err = get_v4l2_edid32(new_p64, p32);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_G_FMT:
 	case VIDIOC_S_FMT:
 	case VIDIOC_TRY_FMT:
-		err = bufsize_v4l2_format(up, &aux_space);
+		err = bufsize_v4l2_format(p32, &aux_space);
 		if (!err)
 			err = alloc_userspace(sizeof(struct v4l2_format),
-					      aux_space, &up_native);
+					      aux_space, &new_p64);
 		if (!err) {
-			aux_buf = up_native + sizeof(struct v4l2_format);
-			err = get_v4l2_format32(up_native, up,
+			aux_buf = new_p64 + sizeof(struct v4l2_format);
+			err = get_v4l2_format32(new_p64, p32,
 						aux_buf, aux_space);
 		}
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_CREATE_BUFS:
-		err = bufsize_v4l2_create(up, &aux_space);
+		err = bufsize_v4l2_create(p32, &aux_space);
 		if (!err)
 			err = alloc_userspace(sizeof(struct v4l2_create_buffers),
-					      aux_space, &up_native);
+					      aux_space, &new_p64);
 		if (!err) {
-			aux_buf = up_native + sizeof(struct v4l2_create_buffers);
-			err = get_v4l2_create32(up_native, up,
+			aux_buf = new_p64 + sizeof(struct v4l2_create_buffers);
+			err = get_v4l2_create32(new_p64, p32,
 						aux_buf, aux_space);
 		}
 		compatible_arg = 0;
@@ -1112,13 +1267,13 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar
 	case VIDIOC_QUERYBUF:
 	case VIDIOC_QBUF:
 	case VIDIOC_DQBUF:
-		err = bufsize_v4l2_buffer(up, &aux_space);
+		err = bufsize_v4l2_buffer(p32, &aux_space);
 		if (!err)
 			err = alloc_userspace(sizeof(struct v4l2_buffer),
-					      aux_space, &up_native);
+					      aux_space, &new_p64);
 		if (!err) {
-			aux_buf = up_native + sizeof(struct v4l2_buffer);
-			err = get_v4l2_buffer32(up_native, up,
+			aux_buf = new_p64 + sizeof(struct v4l2_buffer);
+			err = get_v4l2_buffer32(new_p64, p32,
 						aux_buf, aux_space);
 		}
 		compatible_arg = 0;
@@ -1126,133 +1281,165 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar
 
 	case VIDIOC_S_FBUF:
 		err = alloc_userspace(sizeof(struct v4l2_framebuffer), 0,
-				      &up_native);
+				      &new_p64);
 		if (!err)
-			err = get_v4l2_framebuffer32(up_native, up);
+			err = get_v4l2_framebuffer32(new_p64, p32);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_G_FBUF:
 		err = alloc_userspace(sizeof(struct v4l2_framebuffer), 0,
-				      &up_native);
+				      &new_p64);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_ENUMSTD:
 		err = alloc_userspace(sizeof(struct v4l2_standard), 0,
-				      &up_native);
+				      &new_p64);
 		if (!err)
-			err = get_v4l2_standard32(up_native, up);
+			err = get_v4l2_standard32(new_p64, p32);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_ENUMINPUT:
-		err = alloc_userspace(sizeof(struct v4l2_input), 0, &up_native);
+		err = alloc_userspace(sizeof(struct v4l2_input), 0, &new_p64);
 		if (!err)
-			err = get_v4l2_input32(up_native, up);
+			err = get_v4l2_input32(new_p64, p32);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_G_EXT_CTRLS:
 	case VIDIOC_S_EXT_CTRLS:
 	case VIDIOC_TRY_EXT_CTRLS:
-		err = bufsize_v4l2_ext_controls(up, &aux_space);
+		err = bufsize_v4l2_ext_controls(p32, &aux_space);
 		if (!err)
 			err = alloc_userspace(sizeof(struct v4l2_ext_controls),
-					      aux_space, &up_native);
+					      aux_space, &new_p64);
 		if (!err) {
-			aux_buf = up_native + sizeof(struct v4l2_ext_controls);
-			err = get_v4l2_ext_controls32(file, up_native, up,
+			aux_buf = new_p64 + sizeof(struct v4l2_ext_controls);
+			err = get_v4l2_ext_controls32(file, new_p64, p32,
 						      aux_buf, aux_space);
 		}
 		compatible_arg = 0;
 		break;
 	case VIDIOC_DQEVENT:
-		err = alloc_userspace(sizeof(struct v4l2_event), 0, &up_native);
+		err = alloc_userspace(sizeof(struct v4l2_event), 0, &new_p64);
 		compatible_arg = 0;
 		break;
 	}
 	if (err)
 		return err;
 
+	/*
+	 * 3. Calls the native 64-bits ioctl handler.
+	 *
+	 * For the functions where a conversion was not needed,
+	 * compatible_arg is true, and it will call it with the arguments
+	 * provided by userspace and stored at @p32 var.
+	 *
+	 * Otherwise, it will pass the newly allocated @new_p64 argument.
+	 */
 	if (compatible_arg)
-		err = native_ioctl(file, cmd, (unsigned long)up);
+		err = native_ioctl(file, cmd, (unsigned long)p32);
 	else
-		err = native_ioctl(file, cmd, (unsigned long)up_native);
+		err = native_ioctl(file, cmd, (unsigned long)new_p64);
 
 	if (err == -ENOTTY)
 		return err;
 
 	/*
-	 * Special case: even after an error we need to put the
-	 * results back for these ioctls since the error_idx will
-	 * contain information on which control failed.
+	 * 4. Special case: even after an error we need to put the
+	 * results back for some ioctls.
+	 *
+	 * In the case of EXT_CTRLS, the error_idx will contain information
+	 * on which control failed.
+	 *
+	 * In the case of S_EDID, the driver can return E2BIG and set
+	 * the blocks to maximum allowed value.
 	 */
 	switch (cmd) {
 	case VIDIOC_G_EXT_CTRLS:
 	case VIDIOC_S_EXT_CTRLS:
 	case VIDIOC_TRY_EXT_CTRLS:
-		if (put_v4l2_ext_controls32(file, up_native, up))
+		if (put_v4l2_ext_controls32(file, new_p64, p32))
 			err = -EFAULT;
 		break;
 	case VIDIOC_S_EDID:
-		if (put_v4l2_edid32(up_native, up))
+		if (put_v4l2_edid32(new_p64, p32))
 			err = -EFAULT;
 		break;
 	}
 	if (err)
 		return err;
 
+	/*
+	 * 5. Copy the data returned at the 64 bits userspace pointer to
+	 * the original 32 bits structure.
+	 */
 	switch (cmd) {
 	case VIDIOC_S_INPUT:
 	case VIDIOC_S_OUTPUT:
 	case VIDIOC_G_INPUT:
 	case VIDIOC_G_OUTPUT:
-		if (assign_in_user((compat_uint_t __user *)up,
-				   ((unsigned int __user *)up_native)))
+		if (assign_in_user((compat_uint_t __user *)p32,
+				   ((unsigned int __user *)new_p64)))
 			err = -EFAULT;
 		break;
 
 	case VIDIOC_G_FBUF:
-		err = put_v4l2_framebuffer32(up_native, up);
+		err = put_v4l2_framebuffer32(new_p64, p32);
 		break;
 
 	case VIDIOC_DQEVENT:
-		err = put_v4l2_event32(up_native, up);
+		err = put_v4l2_event32(new_p64, p32);
 		break;
 
 	case VIDIOC_G_EDID:
-		err = put_v4l2_edid32(up_native, up);
+		err = put_v4l2_edid32(new_p64, p32);
 		break;
 
 	case VIDIOC_G_FMT:
 	case VIDIOC_S_FMT:
 	case VIDIOC_TRY_FMT:
-		err = put_v4l2_format32(up_native, up);
+		err = put_v4l2_format32(new_p64, p32);
 		break;
 
 	case VIDIOC_CREATE_BUFS:
-		err = put_v4l2_create32(up_native, up);
+		err = put_v4l2_create32(new_p64, p32);
 		break;
 
 	case VIDIOC_PREPARE_BUF:
 	case VIDIOC_QUERYBUF:
 	case VIDIOC_QBUF:
 	case VIDIOC_DQBUF:
-		err = put_v4l2_buffer32(up_native, up);
+		err = put_v4l2_buffer32(new_p64, p32);
 		break;
 
 	case VIDIOC_ENUMSTD:
-		err = put_v4l2_standard32(up_native, up);
+		err = put_v4l2_standard32(new_p64, p32);
 		break;
 
 	case VIDIOC_ENUMINPUT:
-		err = put_v4l2_input32(up_native, up);
+		err = put_v4l2_input32(new_p64, p32);
 		break;
 	}
 	return err;
 }
 
+/**
+ * v4l2_compat_ioctl32() - Handles a compat32 ioctl call
+ *
+ * @file: pointer to &struct file with the file handler
+ * @cmd: ioctl to be called
+ * @arg: arguments passed from/to the ioctl handler
+ *
+ * This function is meant to be used as .compat_ioctl fops at v4l2-dev.c
+ * in order to deal with 32-bit calls on a 64-bits Kernel.
+ *
+ * This function calls do_video_ioctl() for non-private V4L2 ioctls.
+ * If the function is a private one it calls vdev->fops->compat_ioctl32
+ * instead.
+ */
 long v4l2_compat_ioctl32(struct file *file, unsigned int cmd, unsigned long arg)
 {
 	struct video_device *vdev = video_devdata(file);
diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c
index c080dcc75393..4ffd7d60a901 100644
--- a/drivers/media/v4l2-core/v4l2-dev.c
+++ b/drivers/media/v4l2-core/v4l2-dev.c
@@ -16,6 +16,8 @@
  *		- Added procfs support
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/module.h>
 #include <linux/types.h>
 #include <linux/kernel.h>
@@ -34,6 +36,12 @@
 #define VIDEO_NUM_DEVICES	256
 #define VIDEO_NAME              "video4linux"
 
+#define dprintk(fmt, arg...) do {					\
+		printk(KERN_DEBUG pr_fmt("%s: " fmt),			\
+		       __func__, ##arg);				\
+} while (0)
+
+
 /*
  *	sysfs stuff
  */
@@ -91,7 +99,7 @@ ATTRIBUTE_GROUPS(video_device);
 /*
  *	Active devices
  */
-static struct video_device *video_device[VIDEO_NUM_DEVICES];
+static struct video_device *video_devices[VIDEO_NUM_DEVICES];
 static DEFINE_MUTEX(videodev_lock);
 static DECLARE_BITMAP(devnode_nums[VFL_TYPE_MAX], VIDEO_NUM_DEVICES);
 
@@ -173,14 +181,14 @@ static void v4l2_device_release(struct device *cd)
 	struct v4l2_device *v4l2_dev = vdev->v4l2_dev;
 
 	mutex_lock(&videodev_lock);
-	if (WARN_ON(video_device[vdev->minor] != vdev)) {
+	if (WARN_ON(video_devices[vdev->minor] != vdev)) {
 		/* should not happen */
 		mutex_unlock(&videodev_lock);
 		return;
 	}
 
 	/* Free up this device for reuse */
-	video_device[vdev->minor] = NULL;
+	video_devices[vdev->minor] = NULL;
 
 	/* Delete the cdev on this minor as well */
 	cdev_del(vdev->cdev);
@@ -229,7 +237,7 @@ static struct class video_class = {
 
 struct video_device *video_devdata(struct file *file)
 {
-	return video_device[iminor(file_inode(file))];
+	return video_devices[iminor(file_inode(file))];
 }
 EXPORT_SYMBOL(video_devdata);
 
@@ -309,7 +317,7 @@ static ssize_t v4l2_read(struct file *filp, char __user *buf,
 		ret = vdev->fops->read(filp, buf, sz, off);
 	if ((vdev->dev_debug & V4L2_DEV_DEBUG_FOP) &&
 	    (vdev->dev_debug & V4L2_DEV_DEBUG_STREAMING))
-		printk(KERN_DEBUG "%s: read: %zd (%d)\n",
+		dprintk("%s: read: %zd (%d)\n",
 			video_device_node_name(vdev), sz, ret);
 	return ret;
 }
@@ -326,7 +334,7 @@ static ssize_t v4l2_write(struct file *filp, const char __user *buf,
 		ret = vdev->fops->write(filp, buf, sz, off);
 	if ((vdev->dev_debug & V4L2_DEV_DEBUG_FOP) &&
 	    (vdev->dev_debug & V4L2_DEV_DEBUG_STREAMING))
-		printk(KERN_DEBUG "%s: write: %zd (%d)\n",
+		dprintk("%s: write: %zd (%d)\n",
 			video_device_node_name(vdev), sz, ret);
 	return ret;
 }
@@ -341,7 +349,7 @@ static __poll_t v4l2_poll(struct file *filp, struct poll_table_struct *poll)
 	if (video_is_registered(vdev))
 		res = vdev->fops->poll(filp, poll);
 	if (vdev->dev_debug & V4L2_DEV_DEBUG_POLL)
-		printk(KERN_DEBUG "%s: poll: %08x\n",
+		dprintk("%s: poll: %08x\n",
 			video_device_node_name(vdev), res);
 	return res;
 }
@@ -352,14 +360,8 @@ static long v4l2_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	int ret = -ENODEV;
 
 	if (vdev->fops->unlocked_ioctl) {
-		struct mutex *lock = v4l2_ioctl_get_lock(vdev, cmd);
-
-		if (lock && mutex_lock_interruptible(lock))
-			return -ERESTARTSYS;
 		if (video_is_registered(vdev))
 			ret = vdev->fops->unlocked_ioctl(filp, cmd, arg);
-		if (lock)
-			mutex_unlock(lock);
 	} else
 		ret = -ENOTTY;
 
@@ -382,7 +384,7 @@ static unsigned long v4l2_get_unmapped_area(struct file *filp,
 		return -ENODEV;
 	ret = vdev->fops->get_unmapped_area(filp, addr, len, pgoff, flags);
 	if (vdev->dev_debug & V4L2_DEV_DEBUG_FOP)
-		printk(KERN_DEBUG "%s: get_unmapped_area (%d)\n",
+		dprintk("%s: get_unmapped_area (%d)\n",
 			video_device_node_name(vdev), ret);
 	return ret;
 }
@@ -398,7 +400,7 @@ static int v4l2_mmap(struct file *filp, struct vm_area_struct *vm)
 	if (video_is_registered(vdev))
 		ret = vdev->fops->mmap(filp, vm);
 	if (vdev->dev_debug & V4L2_DEV_DEBUG_FOP)
-		printk(KERN_DEBUG "%s: mmap (%d)\n",
+		dprintk("%s: mmap (%d)\n",
 			video_device_node_name(vdev), ret);
 	return ret;
 }
@@ -428,7 +430,7 @@ static int v4l2_open(struct inode *inode, struct file *filp)
 	}
 
 	if (vdev->dev_debug & V4L2_DEV_DEBUG_FOP)
-		printk(KERN_DEBUG "%s: open (%d)\n",
+		dprintk("%s: open (%d)\n",
 			video_device_node_name(vdev), ret);
 	/* decrease the refcount in case of an error */
 	if (ret)
@@ -445,7 +447,7 @@ static int v4l2_release(struct inode *inode, struct file *filp)
 	if (vdev->fops->release)
 		ret = vdev->fops->release(filp);
 	if (vdev->dev_debug & V4L2_DEV_DEBUG_FOP)
-		printk(KERN_DEBUG "%s: release\n",
+		dprintk("%s: release\n",
 			video_device_node_name(vdev));
 
 	/* decrease the refcount unconditionally since the release()
@@ -493,9 +495,9 @@ static int get_index(struct video_device *vdev)
 	bitmap_zero(used, VIDEO_NUM_DEVICES);
 
 	for (i = 0; i < VIDEO_NUM_DEVICES; i++) {
-		if (video_device[i] != NULL &&
-		    video_device[i]->v4l2_dev == vdev->v4l2_dev) {
-			set_bit(video_device[i]->index, used);
+		if (video_devices[i] != NULL &&
+		    video_devices[i]->v4l2_dev == vdev->v4l2_dev) {
+			set_bit(video_devices[i]->index, used);
 		}
 	}
 
@@ -786,8 +788,7 @@ static int video_register_media_controller(struct video_device *vdev, int type)
 		ret = media_device_register_entity(vdev->v4l2_dev->mdev,
 						   &vdev->entity);
 		if (ret < 0) {
-			printk(KERN_WARNING
-				"%s: media_device_register_entity failed\n",
+			pr_warn("%s: media_device_register_entity failed\n",
 				__func__);
 			return ret;
 		}
@@ -869,7 +870,7 @@ int __video_register_device(struct video_device *vdev,
 		name_base = "v4l-touch";
 		break;
 	default:
-		printk(KERN_ERR "%s called with unknown type: %d\n",
+		pr_err("%s called with unknown type: %d\n",
 		       __func__, type);
 		return -EINVAL;
 	}
@@ -918,7 +919,7 @@ int __video_register_device(struct video_device *vdev,
 	if (nr == minor_cnt)
 		nr = devnode_find(vdev, 0, minor_cnt);
 	if (nr == minor_cnt) {
-		printk(KERN_ERR "could not get a free device node number\n");
+		pr_err("could not get a free device node number\n");
 		mutex_unlock(&videodev_lock);
 		return -ENFILE;
 	}
@@ -929,11 +930,11 @@ int __video_register_device(struct video_device *vdev,
 	/* The device node number and minor numbers are independent, so
 	   we just find the first free minor number. */
 	for (i = 0; i < VIDEO_NUM_DEVICES; i++)
-		if (video_device[i] == NULL)
+		if (video_devices[i] == NULL)
 			break;
 	if (i == VIDEO_NUM_DEVICES) {
 		mutex_unlock(&videodev_lock);
-		printk(KERN_ERR "could not get a free minor\n");
+		pr_err("could not get a free minor\n");
 		return -ENFILE;
 	}
 #endif
@@ -941,14 +942,14 @@ int __video_register_device(struct video_device *vdev,
 	vdev->num = nr;
 
 	/* Should not happen since we thought this minor was free */
-	if (WARN_ON(video_device[vdev->minor])) {
+	if (WARN_ON(video_devices[vdev->minor])) {
 		mutex_unlock(&videodev_lock);
-		printk(KERN_ERR "video_device not empty!\n");
+		pr_err("video_device not empty!\n");
 		return -ENFILE;
 	}
 	devnode_set(vdev);
 	vdev->index = get_index(vdev);
-	video_device[vdev->minor] = vdev;
+	video_devices[vdev->minor] = vdev;
 	mutex_unlock(&videodev_lock);
 
 	if (vdev->ioctl_ops)
@@ -964,7 +965,7 @@ int __video_register_device(struct video_device *vdev,
 	vdev->cdev->owner = owner;
 	ret = cdev_add(vdev->cdev, MKDEV(VIDEO_MAJOR, vdev->minor), 1);
 	if (ret < 0) {
-		printk(KERN_ERR "%s: cdev_add failed\n", __func__);
+		pr_err("%s: cdev_add failed\n", __func__);
 		kfree(vdev->cdev);
 		vdev->cdev = NULL;
 		goto cleanup;
@@ -977,7 +978,7 @@ int __video_register_device(struct video_device *vdev,
 	dev_set_name(&vdev->dev, "%s%d", name_base, vdev->num);
 	ret = device_register(&vdev->dev);
 	if (ret < 0) {
-		printk(KERN_ERR "%s: device_register failed\n", __func__);
+		pr_err("%s: device_register failed\n", __func__);
 		goto cleanup;
 	}
 	/* Register the release callback that will be called when the last
@@ -985,7 +986,7 @@ int __video_register_device(struct video_device *vdev,
 	vdev->dev.release = v4l2_device_release;
 
 	if (nr != -1 && nr != vdev->num && warn_if_nr_in_use)
-		printk(KERN_WARNING "%s: requested %s%d, got %s\n", __func__,
+		pr_warn("%s: requested %s%d, got %s\n", __func__,
 			name_base, nr, video_device_node_name(vdev));
 
 	/* Increase v4l2_device refcount */
@@ -1003,7 +1004,7 @@ int __video_register_device(struct video_device *vdev,
 	mutex_lock(&videodev_lock);
 	if (vdev->cdev)
 		cdev_del(vdev->cdev);
-	video_device[vdev->minor] = NULL;
+	video_devices[vdev->minor] = NULL;
 	devnode_clear(vdev);
 	mutex_unlock(&videodev_lock);
 	/* Mark this video device as never having been registered. */
@@ -1043,10 +1044,10 @@ static int __init videodev_init(void)
 	dev_t dev = MKDEV(VIDEO_MAJOR, 0);
 	int ret;
 
-	printk(KERN_INFO "Linux video capture interface: v2.00\n");
+	pr_info("Linux video capture interface: v2.00\n");
 	ret = register_chrdev_region(dev, VIDEO_NUM_DEVICES, VIDEO_NAME);
 	if (ret < 0) {
-		printk(KERN_WARNING "videodev: unable to get major %d\n",
+		pr_warn("videodev: unable to get major %d\n",
 				VIDEO_MAJOR);
 		return ret;
 	}
@@ -1054,7 +1055,7 @@ static int __init videodev_init(void)
 	ret = class_register(&video_class);
 	if (ret < 0) {
 		unregister_chrdev_region(dev, VIDEO_NUM_DEVICES);
-		printk(KERN_WARNING "video_dev: class_register failed\n");
+		pr_warn("video_dev: class_register failed\n");
 		return -EIO;
 	}
 
diff --git a/drivers/media/v4l2-core/v4l2-fwnode.c b/drivers/media/v4l2-core/v4l2-fwnode.c
index d630640642ee..3f77aa318035 100644
--- a/drivers/media/v4l2-core/v4l2-fwnode.c
+++ b/drivers/media/v4l2-core/v4l2-fwnode.c
@@ -819,17 +819,25 @@ static int v4l2_fwnode_reference_parse_int_props(
 	unsigned int index;
 	int ret;
 
-	for (index = 0; !IS_ERR((fwnode = v4l2_fwnode_reference_get_int_prop(
-					 dev_fwnode(dev), prop, index, props,
-					 nprops))); index++)
+	index = 0;
+	do {
+		fwnode = v4l2_fwnode_reference_get_int_prop(dev_fwnode(dev),
+							    prop, index,
+							    props, nprops);
+		if (IS_ERR(fwnode)) {
+			/*
+			 * Note that right now both -ENODATA and -ENOENT may
+			 * signal out-of-bounds access. Return the error in
+			 * cases other than that.
+			 */
+			if (PTR_ERR(fwnode) != -ENOENT &&
+			    PTR_ERR(fwnode) != -ENODATA)
+				return PTR_ERR(fwnode);
+			break;
+		}
 		fwnode_handle_put(fwnode);
-
-	/*
-	 * Note that right now both -ENODATA and -ENOENT may signal
-	 * out-of-bounds access. Return the error in cases other than that.
-	 */
-	if (PTR_ERR(fwnode) != -ENOENT && PTR_ERR(fwnode) != -ENODATA)
-		return PTR_ERR(fwnode);
+		index++;
+	} while (1);
 
 	ret = v4l2_async_notifier_realloc(notifier,
 					  notifier->num_subdevs + index);
diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c
index de5d96dbe69e..965fd301f617 100644
--- a/drivers/media/v4l2-core/v4l2-ioctl.c
+++ b/drivers/media/v4l2-core/v4l2-ioctl.c
@@ -1952,7 +1952,22 @@ static int v4l_s_parm(const struct v4l2_ioctl_ops *ops,
 	struct v4l2_streamparm *p = arg;
 	int ret = check_fmt(file, p->type);
 
-	return ret ? ret : ops->vidioc_s_parm(file, fh, p);
+	if (ret)
+		return ret;
+
+	/* Note: extendedmode is never used in drivers */
+	if (V4L2_TYPE_IS_OUTPUT(p->type)) {
+		memset(p->parm.output.reserved, 0,
+		       sizeof(p->parm.output.reserved));
+		p->parm.output.extendedmode = 0;
+		p->parm.output.outputmode &= V4L2_MODE_HIGHQUALITY;
+	} else {
+		memset(p->parm.capture.reserved, 0,
+		       sizeof(p->parm.capture.reserved));
+		p->parm.capture.extendedmode = 0;
+		p->parm.capture.capturemode &= V4L2_MODE_HIGHQUALITY;
+	}
+	return ops->vidioc_s_parm(file, fh, p);
 }
 
 static int v4l_queryctrl(const struct v4l2_ioctl_ops *ops,
@@ -2489,11 +2504,8 @@ struct v4l2_ioctl_info {
 	unsigned int ioctl;
 	u32 flags;
 	const char * const name;
-	union {
-		u32 offset;
-		int (*func)(const struct v4l2_ioctl_ops *ops,
-				struct file *file, void *fh, void *p);
-	} u;
+	int (*func)(const struct v4l2_ioctl_ops *ops, struct file *file,
+		    void *fh, void *p);
 	void (*debug)(const void *arg, bool write_only);
 };
 
@@ -2501,137 +2513,160 @@ struct v4l2_ioctl_info {
 #define INFO_FL_PRIO		(1 << 0)
 /* This control can be valid if the filehandle passes a control handler. */
 #define INFO_FL_CTRL		(1 << 1)
-/* This is a standard ioctl, no need for special code */
-#define INFO_FL_STD		(1 << 2)
-/* This is ioctl has its own function */
-#define INFO_FL_FUNC		(1 << 3)
 /* Queuing ioctl */
-#define INFO_FL_QUEUE		(1 << 4)
+#define INFO_FL_QUEUE		(1 << 2)
 /* Always copy back result, even on error */
-#define INFO_FL_ALWAYS_COPY	(1 << 5)
+#define INFO_FL_ALWAYS_COPY	(1 << 3)
 /* Zero struct from after the field to the end */
 #define INFO_FL_CLEAR(v4l2_struct, field)			\
 	((offsetof(struct v4l2_struct, field) +			\
 	  sizeof(((struct v4l2_struct *)0)->field)) << 16)
 #define INFO_FL_CLEAR_MASK	(_IOC_SIZEMASK << 16)
 
-#define IOCTL_INFO_STD(_ioctl, _vidioc, _debug, _flags)			\
-	[_IOC_NR(_ioctl)] = {						\
-		.ioctl = _ioctl,					\
-		.flags = _flags | INFO_FL_STD,				\
-		.name = #_ioctl,					\
-		.u.offset = offsetof(struct v4l2_ioctl_ops, _vidioc),	\
-		.debug = _debug,					\
+#define DEFINE_V4L_STUB_FUNC(_vidioc)				\
+	static int v4l_stub_ ## _vidioc(			\
+			const struct v4l2_ioctl_ops *ops,	\
+			struct file *file, void *fh, void *p)	\
+	{							\
+		return ops->vidioc_ ## _vidioc(file, fh, p);	\
 	}
 
-#define IOCTL_INFO_FNC(_ioctl, _func, _debug, _flags)			\
-	[_IOC_NR(_ioctl)] = {						\
-		.ioctl = _ioctl,					\
-		.flags = _flags | INFO_FL_FUNC,				\
-		.name = #_ioctl,					\
-		.u.func = _func,					\
-		.debug = _debug,					\
+#define IOCTL_INFO(_ioctl, _func, _debug, _flags)		\
+	[_IOC_NR(_ioctl)] = {					\
+		.ioctl = _ioctl,				\
+		.flags = _flags,				\
+		.name = #_ioctl,				\
+		.func = _func,					\
+		.debug = _debug,				\
 	}
 
+DEFINE_V4L_STUB_FUNC(g_fbuf)
+DEFINE_V4L_STUB_FUNC(s_fbuf)
+DEFINE_V4L_STUB_FUNC(expbuf)
+DEFINE_V4L_STUB_FUNC(g_std)
+DEFINE_V4L_STUB_FUNC(g_audio)
+DEFINE_V4L_STUB_FUNC(s_audio)
+DEFINE_V4L_STUB_FUNC(g_input)
+DEFINE_V4L_STUB_FUNC(g_edid)
+DEFINE_V4L_STUB_FUNC(s_edid)
+DEFINE_V4L_STUB_FUNC(g_output)
+DEFINE_V4L_STUB_FUNC(g_audout)
+DEFINE_V4L_STUB_FUNC(s_audout)
+DEFINE_V4L_STUB_FUNC(g_jpegcomp)
+DEFINE_V4L_STUB_FUNC(s_jpegcomp)
+DEFINE_V4L_STUB_FUNC(enumaudio)
+DEFINE_V4L_STUB_FUNC(enumaudout)
+DEFINE_V4L_STUB_FUNC(enum_framesizes)
+DEFINE_V4L_STUB_FUNC(enum_frameintervals)
+DEFINE_V4L_STUB_FUNC(g_enc_index)
+DEFINE_V4L_STUB_FUNC(encoder_cmd)
+DEFINE_V4L_STUB_FUNC(try_encoder_cmd)
+DEFINE_V4L_STUB_FUNC(decoder_cmd)
+DEFINE_V4L_STUB_FUNC(try_decoder_cmd)
+DEFINE_V4L_STUB_FUNC(s_dv_timings)
+DEFINE_V4L_STUB_FUNC(g_dv_timings)
+DEFINE_V4L_STUB_FUNC(enum_dv_timings)
+DEFINE_V4L_STUB_FUNC(query_dv_timings)
+DEFINE_V4L_STUB_FUNC(dv_timings_cap)
+
 static struct v4l2_ioctl_info v4l2_ioctls[] = {
-	IOCTL_INFO_FNC(VIDIOC_QUERYCAP, v4l_querycap, v4l_print_querycap, 0),
-	IOCTL_INFO_FNC(VIDIOC_ENUM_FMT, v4l_enum_fmt, v4l_print_fmtdesc, INFO_FL_CLEAR(v4l2_fmtdesc, type)),
-	IOCTL_INFO_FNC(VIDIOC_G_FMT, v4l_g_fmt, v4l_print_format, 0),
-	IOCTL_INFO_FNC(VIDIOC_S_FMT, v4l_s_fmt, v4l_print_format, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_REQBUFS, v4l_reqbufs, v4l_print_requestbuffers, INFO_FL_PRIO | INFO_FL_QUEUE),
-	IOCTL_INFO_FNC(VIDIOC_QUERYBUF, v4l_querybuf, v4l_print_buffer, INFO_FL_QUEUE | INFO_FL_CLEAR(v4l2_buffer, length)),
-	IOCTL_INFO_STD(VIDIOC_G_FBUF, vidioc_g_fbuf, v4l_print_framebuffer, 0),
-	IOCTL_INFO_STD(VIDIOC_S_FBUF, vidioc_s_fbuf, v4l_print_framebuffer, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_OVERLAY, v4l_overlay, v4l_print_u32, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_QBUF, v4l_qbuf, v4l_print_buffer, INFO_FL_QUEUE),
-	IOCTL_INFO_STD(VIDIOC_EXPBUF, vidioc_expbuf, v4l_print_exportbuffer, INFO_FL_QUEUE | INFO_FL_CLEAR(v4l2_exportbuffer, flags)),
-	IOCTL_INFO_FNC(VIDIOC_DQBUF, v4l_dqbuf, v4l_print_buffer, INFO_FL_QUEUE),
-	IOCTL_INFO_FNC(VIDIOC_STREAMON, v4l_streamon, v4l_print_buftype, INFO_FL_PRIO | INFO_FL_QUEUE),
-	IOCTL_INFO_FNC(VIDIOC_STREAMOFF, v4l_streamoff, v4l_print_buftype, INFO_FL_PRIO | INFO_FL_QUEUE),
-	IOCTL_INFO_FNC(VIDIOC_G_PARM, v4l_g_parm, v4l_print_streamparm, INFO_FL_CLEAR(v4l2_streamparm, type)),
-	IOCTL_INFO_FNC(VIDIOC_S_PARM, v4l_s_parm, v4l_print_streamparm, INFO_FL_PRIO),
-	IOCTL_INFO_STD(VIDIOC_G_STD, vidioc_g_std, v4l_print_std, 0),
-	IOCTL_INFO_FNC(VIDIOC_S_STD, v4l_s_std, v4l_print_std, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_ENUMSTD, v4l_enumstd, v4l_print_standard, INFO_FL_CLEAR(v4l2_standard, index)),
-	IOCTL_INFO_FNC(VIDIOC_ENUMINPUT, v4l_enuminput, v4l_print_enuminput, INFO_FL_CLEAR(v4l2_input, index)),
-	IOCTL_INFO_FNC(VIDIOC_G_CTRL, v4l_g_ctrl, v4l_print_control, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_control, id)),
-	IOCTL_INFO_FNC(VIDIOC_S_CTRL, v4l_s_ctrl, v4l_print_control, INFO_FL_PRIO | INFO_FL_CTRL),
-	IOCTL_INFO_FNC(VIDIOC_G_TUNER, v4l_g_tuner, v4l_print_tuner, INFO_FL_CLEAR(v4l2_tuner, index)),
-	IOCTL_INFO_FNC(VIDIOC_S_TUNER, v4l_s_tuner, v4l_print_tuner, INFO_FL_PRIO),
-	IOCTL_INFO_STD(VIDIOC_G_AUDIO, vidioc_g_audio, v4l_print_audio, 0),
-	IOCTL_INFO_STD(VIDIOC_S_AUDIO, vidioc_s_audio, v4l_print_audio, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_QUERYCTRL, v4l_queryctrl, v4l_print_queryctrl, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_queryctrl, id)),
-	IOCTL_INFO_FNC(VIDIOC_QUERYMENU, v4l_querymenu, v4l_print_querymenu, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_querymenu, index)),
-	IOCTL_INFO_STD(VIDIOC_G_INPUT, vidioc_g_input, v4l_print_u32, 0),
-	IOCTL_INFO_FNC(VIDIOC_S_INPUT, v4l_s_input, v4l_print_u32, INFO_FL_PRIO),
-	IOCTL_INFO_STD(VIDIOC_G_EDID, vidioc_g_edid, v4l_print_edid, INFO_FL_ALWAYS_COPY),
-	IOCTL_INFO_STD(VIDIOC_S_EDID, vidioc_s_edid, v4l_print_edid, INFO_FL_PRIO | INFO_FL_ALWAYS_COPY),
-	IOCTL_INFO_STD(VIDIOC_G_OUTPUT, vidioc_g_output, v4l_print_u32, 0),
-	IOCTL_INFO_FNC(VIDIOC_S_OUTPUT, v4l_s_output, v4l_print_u32, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_ENUMOUTPUT, v4l_enumoutput, v4l_print_enumoutput, INFO_FL_CLEAR(v4l2_output, index)),
-	IOCTL_INFO_STD(VIDIOC_G_AUDOUT, vidioc_g_audout, v4l_print_audioout, 0),
-	IOCTL_INFO_STD(VIDIOC_S_AUDOUT, vidioc_s_audout, v4l_print_audioout, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_G_MODULATOR, v4l_g_modulator, v4l_print_modulator, INFO_FL_CLEAR(v4l2_modulator, index)),
-	IOCTL_INFO_FNC(VIDIOC_S_MODULATOR, v4l_s_modulator, v4l_print_modulator, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_G_FREQUENCY, v4l_g_frequency, v4l_print_frequency, INFO_FL_CLEAR(v4l2_frequency, tuner)),
-	IOCTL_INFO_FNC(VIDIOC_S_FREQUENCY, v4l_s_frequency, v4l_print_frequency, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_CROPCAP, v4l_cropcap, v4l_print_cropcap, INFO_FL_CLEAR(v4l2_cropcap, type)),
-	IOCTL_INFO_FNC(VIDIOC_G_CROP, v4l_g_crop, v4l_print_crop, INFO_FL_CLEAR(v4l2_crop, type)),
-	IOCTL_INFO_FNC(VIDIOC_S_CROP, v4l_s_crop, v4l_print_crop, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_G_SELECTION, v4l_g_selection, v4l_print_selection, INFO_FL_CLEAR(v4l2_selection, r)),
-	IOCTL_INFO_FNC(VIDIOC_S_SELECTION, v4l_s_selection, v4l_print_selection, INFO_FL_PRIO | INFO_FL_CLEAR(v4l2_selection, r)),
-	IOCTL_INFO_STD(VIDIOC_G_JPEGCOMP, vidioc_g_jpegcomp, v4l_print_jpegcompression, 0),
-	IOCTL_INFO_STD(VIDIOC_S_JPEGCOMP, vidioc_s_jpegcomp, v4l_print_jpegcompression, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_QUERYSTD, v4l_querystd, v4l_print_std, 0),
-	IOCTL_INFO_FNC(VIDIOC_TRY_FMT, v4l_try_fmt, v4l_print_format, 0),
-	IOCTL_INFO_STD(VIDIOC_ENUMAUDIO, vidioc_enumaudio, v4l_print_audio, INFO_FL_CLEAR(v4l2_audio, index)),
-	IOCTL_INFO_STD(VIDIOC_ENUMAUDOUT, vidioc_enumaudout, v4l_print_audioout, INFO_FL_CLEAR(v4l2_audioout, index)),
-	IOCTL_INFO_FNC(VIDIOC_G_PRIORITY, v4l_g_priority, v4l_print_u32, 0),
-	IOCTL_INFO_FNC(VIDIOC_S_PRIORITY, v4l_s_priority, v4l_print_u32, INFO_FL_PRIO),
-	IOCTL_INFO_FNC(VIDIOC_G_SLICED_VBI_CAP, v4l_g_sliced_vbi_cap, v4l_print_sliced_vbi_cap, INFO_FL_CLEAR(v4l2_sliced_vbi_cap, type)),
-	IOCTL_INFO_FNC(VIDIOC_LOG_STATUS, v4l_log_status, v4l_print_newline, 0),
-	IOCTL_INFO_FNC(VIDIOC_G_EXT_CTRLS, v4l_g_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL),
-	IOCTL_INFO_FNC(VIDIOC_S_EXT_CTRLS, v4l_s_ext_ctrls, v4l_print_ext_controls, INFO_FL_PRIO | INFO_FL_CTRL),
-	IOCTL_INFO_FNC(VIDIOC_TRY_EXT_CTRLS, v4l_try_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL),
-	IOCTL_INFO_STD(VIDIOC_ENUM_FRAMESIZES, vidioc_enum_framesizes, v4l_print_frmsizeenum, INFO_FL_CLEAR(v4l2_frmsizeenum, pixel_format)),
-	IOCTL_INFO_STD(VIDIOC_ENUM_FRAMEINTERVALS, vidioc_enum_frameintervals, v4l_print_frmivalenum, INFO_FL_CLEAR(v4l2_frmivalenum, height)),
-	IOCTL_INFO_STD(VIDIOC_G_ENC_INDEX, vidioc_g_enc_index, v4l_print_enc_idx, 0),
-	IOCTL_INFO_STD(VIDIOC_ENCODER_CMD, vidioc_encoder_cmd, v4l_print_encoder_cmd, INFO_FL_PRIO | INFO_FL_CLEAR(v4l2_encoder_cmd, flags)),
-	IOCTL_INFO_STD(VIDIOC_TRY_ENCODER_CMD, vidioc_try_encoder_cmd, v4l_print_encoder_cmd, INFO_FL_CLEAR(v4l2_encoder_cmd, flags)),
-	IOCTL_INFO_STD(VIDIOC_DECODER_CMD, vidioc_decoder_cmd, v4l_print_decoder_cmd, INFO_FL_PRIO),
-	IOCTL_INFO_STD(VIDIOC_TRY_DECODER_CMD, vidioc_try_decoder_cmd, v4l_print_decoder_cmd, 0),
-	IOCTL_INFO_FNC(VIDIOC_DBG_S_REGISTER, v4l_dbg_s_register, v4l_print_dbg_register, 0),
-	IOCTL_INFO_FNC(VIDIOC_DBG_G_REGISTER, v4l_dbg_g_register, v4l_print_dbg_register, 0),
-	IOCTL_INFO_FNC(VIDIOC_S_HW_FREQ_SEEK, v4l_s_hw_freq_seek, v4l_print_hw_freq_seek, INFO_FL_PRIO),
-	IOCTL_INFO_STD(VIDIOC_S_DV_TIMINGS, vidioc_s_dv_timings, v4l_print_dv_timings, INFO_FL_PRIO | INFO_FL_CLEAR(v4l2_dv_timings, bt.flags)),
-	IOCTL_INFO_STD(VIDIOC_G_DV_TIMINGS, vidioc_g_dv_timings, v4l_print_dv_timings, 0),
-	IOCTL_INFO_FNC(VIDIOC_DQEVENT, v4l_dqevent, v4l_print_event, 0),
-	IOCTL_INFO_FNC(VIDIOC_SUBSCRIBE_EVENT, v4l_subscribe_event, v4l_print_event_subscription, 0),
-	IOCTL_INFO_FNC(VIDIOC_UNSUBSCRIBE_EVENT, v4l_unsubscribe_event, v4l_print_event_subscription, 0),
-	IOCTL_INFO_FNC(VIDIOC_CREATE_BUFS, v4l_create_bufs, v4l_print_create_buffers, INFO_FL_PRIO | INFO_FL_QUEUE),
-	IOCTL_INFO_FNC(VIDIOC_PREPARE_BUF, v4l_prepare_buf, v4l_print_buffer, INFO_FL_QUEUE),
-	IOCTL_INFO_STD(VIDIOC_ENUM_DV_TIMINGS, vidioc_enum_dv_timings, v4l_print_enum_dv_timings, INFO_FL_CLEAR(v4l2_enum_dv_timings, pad)),
-	IOCTL_INFO_STD(VIDIOC_QUERY_DV_TIMINGS, vidioc_query_dv_timings, v4l_print_dv_timings, INFO_FL_ALWAYS_COPY),
-	IOCTL_INFO_STD(VIDIOC_DV_TIMINGS_CAP, vidioc_dv_timings_cap, v4l_print_dv_timings_cap, INFO_FL_CLEAR(v4l2_dv_timings_cap, pad)),
-	IOCTL_INFO_FNC(VIDIOC_ENUM_FREQ_BANDS, v4l_enum_freq_bands, v4l_print_freq_band, 0),
-	IOCTL_INFO_FNC(VIDIOC_DBG_G_CHIP_INFO, v4l_dbg_g_chip_info, v4l_print_dbg_chip_info, INFO_FL_CLEAR(v4l2_dbg_chip_info, match)),
-	IOCTL_INFO_FNC(VIDIOC_QUERY_EXT_CTRL, v4l_query_ext_ctrl, v4l_print_query_ext_ctrl, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_query_ext_ctrl, id)),
+	IOCTL_INFO(VIDIOC_QUERYCAP, v4l_querycap, v4l_print_querycap, 0),
+	IOCTL_INFO(VIDIOC_ENUM_FMT, v4l_enum_fmt, v4l_print_fmtdesc, INFO_FL_CLEAR(v4l2_fmtdesc, type)),
+	IOCTL_INFO(VIDIOC_G_FMT, v4l_g_fmt, v4l_print_format, 0),
+	IOCTL_INFO(VIDIOC_S_FMT, v4l_s_fmt, v4l_print_format, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_REQBUFS, v4l_reqbufs, v4l_print_requestbuffers, INFO_FL_PRIO | INFO_FL_QUEUE),
+	IOCTL_INFO(VIDIOC_QUERYBUF, v4l_querybuf, v4l_print_buffer, INFO_FL_QUEUE | INFO_FL_CLEAR(v4l2_buffer, length)),
+	IOCTL_INFO(VIDIOC_G_FBUF, v4l_stub_g_fbuf, v4l_print_framebuffer, 0),
+	IOCTL_INFO(VIDIOC_S_FBUF, v4l_stub_s_fbuf, v4l_print_framebuffer, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_OVERLAY, v4l_overlay, v4l_print_u32, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_QBUF, v4l_qbuf, v4l_print_buffer, INFO_FL_QUEUE),
+	IOCTL_INFO(VIDIOC_EXPBUF, v4l_stub_expbuf, v4l_print_exportbuffer, INFO_FL_QUEUE | INFO_FL_CLEAR(v4l2_exportbuffer, flags)),
+	IOCTL_INFO(VIDIOC_DQBUF, v4l_dqbuf, v4l_print_buffer, INFO_FL_QUEUE),
+	IOCTL_INFO(VIDIOC_STREAMON, v4l_streamon, v4l_print_buftype, INFO_FL_PRIO | INFO_FL_QUEUE),
+	IOCTL_INFO(VIDIOC_STREAMOFF, v4l_streamoff, v4l_print_buftype, INFO_FL_PRIO | INFO_FL_QUEUE),
+	IOCTL_INFO(VIDIOC_G_PARM, v4l_g_parm, v4l_print_streamparm, INFO_FL_CLEAR(v4l2_streamparm, type)),
+	IOCTL_INFO(VIDIOC_S_PARM, v4l_s_parm, v4l_print_streamparm, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_G_STD, v4l_stub_g_std, v4l_print_std, 0),
+	IOCTL_INFO(VIDIOC_S_STD, v4l_s_std, v4l_print_std, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_ENUMSTD, v4l_enumstd, v4l_print_standard, INFO_FL_CLEAR(v4l2_standard, index)),
+	IOCTL_INFO(VIDIOC_ENUMINPUT, v4l_enuminput, v4l_print_enuminput, INFO_FL_CLEAR(v4l2_input, index)),
+	IOCTL_INFO(VIDIOC_G_CTRL, v4l_g_ctrl, v4l_print_control, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_control, id)),
+	IOCTL_INFO(VIDIOC_S_CTRL, v4l_s_ctrl, v4l_print_control, INFO_FL_PRIO | INFO_FL_CTRL),
+	IOCTL_INFO(VIDIOC_G_TUNER, v4l_g_tuner, v4l_print_tuner, INFO_FL_CLEAR(v4l2_tuner, index)),
+	IOCTL_INFO(VIDIOC_S_TUNER, v4l_s_tuner, v4l_print_tuner, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_G_AUDIO, v4l_stub_g_audio, v4l_print_audio, 0),
+	IOCTL_INFO(VIDIOC_S_AUDIO, v4l_stub_s_audio, v4l_print_audio, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_QUERYCTRL, v4l_queryctrl, v4l_print_queryctrl, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_queryctrl, id)),
+	IOCTL_INFO(VIDIOC_QUERYMENU, v4l_querymenu, v4l_print_querymenu, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_querymenu, index)),
+	IOCTL_INFO(VIDIOC_G_INPUT, v4l_stub_g_input, v4l_print_u32, 0),
+	IOCTL_INFO(VIDIOC_S_INPUT, v4l_s_input, v4l_print_u32, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_G_EDID, v4l_stub_g_edid, v4l_print_edid, INFO_FL_ALWAYS_COPY),
+	IOCTL_INFO(VIDIOC_S_EDID, v4l_stub_s_edid, v4l_print_edid, INFO_FL_PRIO | INFO_FL_ALWAYS_COPY),
+	IOCTL_INFO(VIDIOC_G_OUTPUT, v4l_stub_g_output, v4l_print_u32, 0),
+	IOCTL_INFO(VIDIOC_S_OUTPUT, v4l_s_output, v4l_print_u32, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_ENUMOUTPUT, v4l_enumoutput, v4l_print_enumoutput, INFO_FL_CLEAR(v4l2_output, index)),
+	IOCTL_INFO(VIDIOC_G_AUDOUT, v4l_stub_g_audout, v4l_print_audioout, 0),
+	IOCTL_INFO(VIDIOC_S_AUDOUT, v4l_stub_s_audout, v4l_print_audioout, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_G_MODULATOR, v4l_g_modulator, v4l_print_modulator, INFO_FL_CLEAR(v4l2_modulator, index)),
+	IOCTL_INFO(VIDIOC_S_MODULATOR, v4l_s_modulator, v4l_print_modulator, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_G_FREQUENCY, v4l_g_frequency, v4l_print_frequency, INFO_FL_CLEAR(v4l2_frequency, tuner)),
+	IOCTL_INFO(VIDIOC_S_FREQUENCY, v4l_s_frequency, v4l_print_frequency, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_CROPCAP, v4l_cropcap, v4l_print_cropcap, INFO_FL_CLEAR(v4l2_cropcap, type)),
+	IOCTL_INFO(VIDIOC_G_CROP, v4l_g_crop, v4l_print_crop, INFO_FL_CLEAR(v4l2_crop, type)),
+	IOCTL_INFO(VIDIOC_S_CROP, v4l_s_crop, v4l_print_crop, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_G_SELECTION, v4l_g_selection, v4l_print_selection, INFO_FL_CLEAR(v4l2_selection, r)),
+	IOCTL_INFO(VIDIOC_S_SELECTION, v4l_s_selection, v4l_print_selection, INFO_FL_PRIO | INFO_FL_CLEAR(v4l2_selection, r)),
+	IOCTL_INFO(VIDIOC_G_JPEGCOMP, v4l_stub_g_jpegcomp, v4l_print_jpegcompression, 0),
+	IOCTL_INFO(VIDIOC_S_JPEGCOMP, v4l_stub_s_jpegcomp, v4l_print_jpegcompression, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_QUERYSTD, v4l_querystd, v4l_print_std, 0),
+	IOCTL_INFO(VIDIOC_TRY_FMT, v4l_try_fmt, v4l_print_format, 0),
+	IOCTL_INFO(VIDIOC_ENUMAUDIO, v4l_stub_enumaudio, v4l_print_audio, INFO_FL_CLEAR(v4l2_audio, index)),
+	IOCTL_INFO(VIDIOC_ENUMAUDOUT, v4l_stub_enumaudout, v4l_print_audioout, INFO_FL_CLEAR(v4l2_audioout, index)),
+	IOCTL_INFO(VIDIOC_G_PRIORITY, v4l_g_priority, v4l_print_u32, 0),
+	IOCTL_INFO(VIDIOC_S_PRIORITY, v4l_s_priority, v4l_print_u32, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_G_SLICED_VBI_CAP, v4l_g_sliced_vbi_cap, v4l_print_sliced_vbi_cap, INFO_FL_CLEAR(v4l2_sliced_vbi_cap, type)),
+	IOCTL_INFO(VIDIOC_LOG_STATUS, v4l_log_status, v4l_print_newline, 0),
+	IOCTL_INFO(VIDIOC_G_EXT_CTRLS, v4l_g_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL),
+	IOCTL_INFO(VIDIOC_S_EXT_CTRLS, v4l_s_ext_ctrls, v4l_print_ext_controls, INFO_FL_PRIO | INFO_FL_CTRL),
+	IOCTL_INFO(VIDIOC_TRY_EXT_CTRLS, v4l_try_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL),
+	IOCTL_INFO(VIDIOC_ENUM_FRAMESIZES, v4l_stub_enum_framesizes, v4l_print_frmsizeenum, INFO_FL_CLEAR(v4l2_frmsizeenum, pixel_format)),
+	IOCTL_INFO(VIDIOC_ENUM_FRAMEINTERVALS, v4l_stub_enum_frameintervals, v4l_print_frmivalenum, INFO_FL_CLEAR(v4l2_frmivalenum, height)),
+	IOCTL_INFO(VIDIOC_G_ENC_INDEX, v4l_stub_g_enc_index, v4l_print_enc_idx, 0),
+	IOCTL_INFO(VIDIOC_ENCODER_CMD, v4l_stub_encoder_cmd, v4l_print_encoder_cmd, INFO_FL_PRIO | INFO_FL_CLEAR(v4l2_encoder_cmd, flags)),
+	IOCTL_INFO(VIDIOC_TRY_ENCODER_CMD, v4l_stub_try_encoder_cmd, v4l_print_encoder_cmd, INFO_FL_CLEAR(v4l2_encoder_cmd, flags)),
+	IOCTL_INFO(VIDIOC_DECODER_CMD, v4l_stub_decoder_cmd, v4l_print_decoder_cmd, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_TRY_DECODER_CMD, v4l_stub_try_decoder_cmd, v4l_print_decoder_cmd, 0),
+	IOCTL_INFO(VIDIOC_DBG_S_REGISTER, v4l_dbg_s_register, v4l_print_dbg_register, 0),
+	IOCTL_INFO(VIDIOC_DBG_G_REGISTER, v4l_dbg_g_register, v4l_print_dbg_register, 0),
+	IOCTL_INFO(VIDIOC_S_HW_FREQ_SEEK, v4l_s_hw_freq_seek, v4l_print_hw_freq_seek, INFO_FL_PRIO),
+	IOCTL_INFO(VIDIOC_S_DV_TIMINGS, v4l_stub_s_dv_timings, v4l_print_dv_timings, INFO_FL_PRIO | INFO_FL_CLEAR(v4l2_dv_timings, bt.flags)),
+	IOCTL_INFO(VIDIOC_G_DV_TIMINGS, v4l_stub_g_dv_timings, v4l_print_dv_timings, 0),
+	IOCTL_INFO(VIDIOC_DQEVENT, v4l_dqevent, v4l_print_event, 0),
+	IOCTL_INFO(VIDIOC_SUBSCRIBE_EVENT, v4l_subscribe_event, v4l_print_event_subscription, 0),
+	IOCTL_INFO(VIDIOC_UNSUBSCRIBE_EVENT, v4l_unsubscribe_event, v4l_print_event_subscription, 0),
+	IOCTL_INFO(VIDIOC_CREATE_BUFS, v4l_create_bufs, v4l_print_create_buffers, INFO_FL_PRIO | INFO_FL_QUEUE),
+	IOCTL_INFO(VIDIOC_PREPARE_BUF, v4l_prepare_buf, v4l_print_buffer, INFO_FL_QUEUE),
+	IOCTL_INFO(VIDIOC_ENUM_DV_TIMINGS, v4l_stub_enum_dv_timings, v4l_print_enum_dv_timings, INFO_FL_CLEAR(v4l2_enum_dv_timings, pad)),
+	IOCTL_INFO(VIDIOC_QUERY_DV_TIMINGS, v4l_stub_query_dv_timings, v4l_print_dv_timings, INFO_FL_ALWAYS_COPY),
+	IOCTL_INFO(VIDIOC_DV_TIMINGS_CAP, v4l_stub_dv_timings_cap, v4l_print_dv_timings_cap, INFO_FL_CLEAR(v4l2_dv_timings_cap, pad)),
+	IOCTL_INFO(VIDIOC_ENUM_FREQ_BANDS, v4l_enum_freq_bands, v4l_print_freq_band, 0),
+	IOCTL_INFO(VIDIOC_DBG_G_CHIP_INFO, v4l_dbg_g_chip_info, v4l_print_dbg_chip_info, INFO_FL_CLEAR(v4l2_dbg_chip_info, match)),
+	IOCTL_INFO(VIDIOC_QUERY_EXT_CTRL, v4l_query_ext_ctrl, v4l_print_query_ext_ctrl, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_query_ext_ctrl, id)),
 };
 #define V4L2_IOCTLS ARRAY_SIZE(v4l2_ioctls)
 
-bool v4l2_is_known_ioctl(unsigned int cmd)
+static bool v4l2_is_known_ioctl(unsigned int cmd)
 {
 	if (_IOC_NR(cmd) >= V4L2_IOCTLS)
 		return false;
 	return v4l2_ioctls[_IOC_NR(cmd)].ioctl == cmd;
 }
 
-struct mutex *v4l2_ioctl_get_lock(struct video_device *vdev, unsigned cmd)
+static struct mutex *v4l2_ioctl_get_lock(struct video_device *vdev,
+					 unsigned int cmd)
 {
 	if (_IOC_NR(cmd) >= V4L2_IOCTLS)
 		return vdev->lock;
-	if (test_bit(_IOC_NR(cmd), vdev->disable_locking))
-		return NULL;
 	if (vdev->queue && vdev->queue->lock &&
 			(v4l2_ioctls[_IOC_NR(cmd)].flags & INFO_FL_QUEUE))
 		return vdev->queue->lock;
@@ -2679,6 +2714,7 @@ static long __video_do_ioctl(struct file *file,
 		unsigned int cmd, void *arg)
 {
 	struct video_device *vfd = video_devdata(file);
+	struct mutex *lock; /* ioctl serialization mutex */
 	const struct v4l2_ioctl_ops *ops = vfd->ioctl_ops;
 	bool write_only = false;
 	struct v4l2_ioctl_info default_info;
@@ -2697,6 +2733,16 @@ static long __video_do_ioctl(struct file *file,
 	if (test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags))
 		vfh = file->private_data;
 
+	lock = v4l2_ioctl_get_lock(vfd, cmd);
+
+	if (lock && mutex_lock_interruptible(lock))
+		return -ERESTARTSYS;
+
+	if (!video_is_registered(vfd)) {
+		ret = -ENODEV;
+		goto unlock;
+	}
+
 	if (v4l2_is_known_ioctl(cmd)) {
 		info = &v4l2_ioctls[_IOC_NR(cmd)];
 
@@ -2717,14 +2763,8 @@ static long __video_do_ioctl(struct file *file,
 	}
 
 	write_only = _IOC_DIR(cmd) == _IOC_WRITE;
-	if (info->flags & INFO_FL_STD) {
-		typedef int (*vidioc_op)(struct file *file, void *fh, void *p);
-		const void *p = vfd->ioctl_ops;
-		const vidioc_op *vidioc = p + info->u.offset;
-
-		ret = (*vidioc)(file, fh, arg);
-	} else if (info->flags & INFO_FL_FUNC) {
-		ret = info->u.func(ops, file, fh, arg);
+	if (info != &default_info) {
+		ret = info->func(ops, file, fh, arg);
 	} else if (!ops->vidioc_default) {
 		ret = -ENOTTY;
 	} else {
@@ -2752,6 +2792,9 @@ static long __video_do_ioctl(struct file *file,
 		}
 	}
 
+unlock:
+	if (lock)
+		mutex_unlock(lock);
 	return ret;
 }
 
@@ -2941,7 +2984,6 @@ video_usercopy(struct file *file, unsigned int cmd, unsigned long arg,
 	kvfree(mbuf);
 	return err;
 }
-EXPORT_SYMBOL(video_usercopy);
 
 long video_ioctl2(struct file *file,
 	       unsigned int cmd, unsigned long arg)
diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c
index f9eed938d348..6a7f7f75dfd7 100644
--- a/drivers/media/v4l2-core/v4l2-subdev.c
+++ b/drivers/media/v4l2-core/v4l2-subdev.c
@@ -502,10 +502,25 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg)
 	return 0;
 }
 
+static long subdev_do_ioctl_lock(struct file *file, unsigned int cmd, void *arg)
+{
+	struct video_device *vdev = video_devdata(file);
+	struct mutex *lock = vdev->lock;
+	long ret = -ENODEV;
+
+	if (lock && mutex_lock_interruptible(lock))
+		return -ERESTARTSYS;
+	if (video_is_registered(vdev))
+		ret = subdev_do_ioctl(file, cmd, arg);
+	if (lock)
+		mutex_unlock(lock);
+	return ret;
+}
+
 static long subdev_ioctl(struct file *file, unsigned int cmd,
 	unsigned long arg)
 {
-	return video_usercopy(file, cmd, arg, subdev_do_ioctl);
+	return video_usercopy(file, cmd, arg, subdev_do_ioctl_lock);
 }
 
 #ifdef CONFIG_COMPAT
diff --git a/drivers/media/v4l2-core/videobuf-dma-sg.c b/drivers/media/v4l2-core/videobuf-dma-sg.c
index 7770034aae28..2e5c346f9c30 100644
--- a/drivers/media/v4l2-core/videobuf-dma-sg.c
+++ b/drivers/media/v4l2-core/videobuf-dma-sg.c
@@ -334,7 +334,7 @@ int videobuf_dma_unmap(struct device *dev, struct videobuf_dmabuf *dma)
 	if (!dma->sglen)
 		return 0;
 
-	dma_unmap_sg(dev, dma->sglist, dma->sglen, dma->direction);
+	dma_unmap_sg(dev, dma->sglist, dma->nr_pages, dma->direction);
 
 	vfree(dma->sglist);
 	dma->sglist = NULL;
@@ -434,7 +434,7 @@ static void videobuf_vm_close(struct vm_area_struct *vma)
  * now ...).  Bounce buffers don't work very well for the data rates
  * video capture has.
  */
-static int videobuf_vm_fault(struct vm_fault *vmf)
+static vm_fault_t videobuf_vm_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct page *page;
@@ -581,7 +581,7 @@ static int __videobuf_sync(struct videobuf_queue *q,
 	MAGIC_CHECK(mem->dma.magic, MAGIC_DMABUF);
 
 	dma_sync_sg_for_cpu(q->dev, mem->dma.sglist,
-			    mem->dma.sglen, mem->dma.direction);
+			    mem->dma.nr_pages, mem->dma.direction);
 
 	return 0;
 }
diff --git a/drivers/media/v4l2-core/videobuf-dvb.c b/drivers/media/v4l2-core/videobuf-dvb.c
deleted file mode 100644
index b7efa4516d36..000000000000
--- a/drivers/media/v4l2-core/videobuf-dvb.c
+++ /dev/null
@@ -1,398 +0,0 @@
-/*
- *
- * some helper function for simple DVB cards which simply DMA the
- * complete transport stream and let the computer sort everything else
- * (i.e. we are using the software demux, ...).  Also uses the
- * video-buf to manage DMA buffers.
- *
- * (c) 2004 Gerd Knorr <kraxel@bytesex.org> [SUSE Labs]
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/device.h>
-#include <linux/fs.h>
-#include <linux/kthread.h>
-#include <linux/file.h>
-#include <linux/slab.h>
-
-#include <linux/freezer.h>
-
-#include <media/videobuf-core.h>
-#include <media/videobuf-dvb.h>
-
-/* ------------------------------------------------------------------ */
-
-MODULE_AUTHOR("Gerd Knorr <kraxel@bytesex.org> [SuSE Labs]");
-MODULE_LICENSE("GPL");
-
-static unsigned int debug;
-module_param(debug, int, 0644);
-MODULE_PARM_DESC(debug,"enable debug messages");
-
-#define dprintk(fmt, arg...)	if (debug)			\
-	printk(KERN_DEBUG "%s/dvb: " fmt, dvb->name , ## arg)
-
-/* ------------------------------------------------------------------ */
-
-static int videobuf_dvb_thread(void *data)
-{
-	struct videobuf_dvb *dvb = data;
-	struct videobuf_buffer *buf;
-	unsigned long flags;
-	void *outp;
-
-	dprintk("dvb thread started\n");
-	set_freezable();
-	videobuf_read_start(&dvb->dvbq);
-
-	for (;;) {
-		/* fetch next buffer */
-		buf = list_entry(dvb->dvbq.stream.next,
-				 struct videobuf_buffer, stream);
-		list_del(&buf->stream);
-		videobuf_waiton(&dvb->dvbq, buf, 0, 1);
-
-		/* no more feeds left or stop_feed() asked us to quit */
-		if (0 == dvb->nfeeds)
-			break;
-		if (kthread_should_stop())
-			break;
-		try_to_freeze();
-
-		/* feed buffer data to demux */
-		outp = videobuf_queue_to_vaddr(&dvb->dvbq, buf);
-
-		if (buf->state == VIDEOBUF_DONE)
-			dvb_dmx_swfilter(&dvb->demux, outp,
-					 buf->size);
-
-		/* requeue buffer */
-		list_add_tail(&buf->stream,&dvb->dvbq.stream);
-		spin_lock_irqsave(dvb->dvbq.irqlock,flags);
-		dvb->dvbq.ops->buf_queue(&dvb->dvbq,buf);
-		spin_unlock_irqrestore(dvb->dvbq.irqlock,flags);
-	}
-
-	videobuf_read_stop(&dvb->dvbq);
-	dprintk("dvb thread stopped\n");
-
-	/* Hmm, linux becomes *very* unhappy without this ... */
-	while (!kthread_should_stop()) {
-		set_current_state(TASK_INTERRUPTIBLE);
-		schedule();
-	}
-	return 0;
-}
-
-static int videobuf_dvb_start_feed(struct dvb_demux_feed *feed)
-{
-	struct dvb_demux *demux  = feed->demux;
-	struct videobuf_dvb *dvb = demux->priv;
-	int rc;
-
-	if (!demux->dmx.frontend)
-		return -EINVAL;
-
-	mutex_lock(&dvb->lock);
-	dvb->nfeeds++;
-	rc = dvb->nfeeds;
-
-	if (NULL != dvb->thread)
-		goto out;
-	dvb->thread = kthread_run(videobuf_dvb_thread,
-				  dvb, "%s dvb", dvb->name);
-	if (IS_ERR(dvb->thread)) {
-		rc = PTR_ERR(dvb->thread);
-		dvb->thread = NULL;
-	}
-
-out:
-	mutex_unlock(&dvb->lock);
-	return rc;
-}
-
-static int videobuf_dvb_stop_feed(struct dvb_demux_feed *feed)
-{
-	struct dvb_demux *demux  = feed->demux;
-	struct videobuf_dvb *dvb = demux->priv;
-	int err = 0;
-
-	mutex_lock(&dvb->lock);
-	dvb->nfeeds--;
-	if (0 == dvb->nfeeds  &&  NULL != dvb->thread) {
-		err = kthread_stop(dvb->thread);
-		dvb->thread = NULL;
-	}
-	mutex_unlock(&dvb->lock);
-	return err;
-}
-
-static int videobuf_dvb_register_adapter(struct videobuf_dvb_frontends *fe,
-			  struct module *module,
-			  void *adapter_priv,
-			  struct device *device,
-			  char *adapter_name,
-			  short *adapter_nr,
-			  int mfe_shared)
-{
-	int result;
-
-	mutex_init(&fe->lock);
-
-	/* register adapter */
-	result = dvb_register_adapter(&fe->adapter, adapter_name, module,
-		device, adapter_nr);
-	if (result < 0) {
-		printk(KERN_WARNING "%s: dvb_register_adapter failed (errno = %d)\n",
-		       adapter_name, result);
-	}
-	fe->adapter.priv = adapter_priv;
-	fe->adapter.mfe_shared = mfe_shared;
-
-	return result;
-}
-
-static int videobuf_dvb_register_frontend(struct dvb_adapter *adapter,
-	struct videobuf_dvb *dvb)
-{
-	int result;
-
-	/* register frontend */
-	result = dvb_register_frontend(adapter, dvb->frontend);
-	if (result < 0) {
-		printk(KERN_WARNING "%s: dvb_register_frontend failed (errno = %d)\n",
-		       dvb->name, result);
-		goto fail_frontend;
-	}
-
-	/* register demux stuff */
-	dvb->demux.dmx.capabilities =
-		DMX_TS_FILTERING | DMX_SECTION_FILTERING |
-		DMX_MEMORY_BASED_FILTERING;
-	dvb->demux.priv       = dvb;
-	dvb->demux.filternum  = 256;
-	dvb->demux.feednum    = 256;
-	dvb->demux.start_feed = videobuf_dvb_start_feed;
-	dvb->demux.stop_feed  = videobuf_dvb_stop_feed;
-	result = dvb_dmx_init(&dvb->demux);
-	if (result < 0) {
-		printk(KERN_WARNING "%s: dvb_dmx_init failed (errno = %d)\n",
-		       dvb->name, result);
-		goto fail_dmx;
-	}
-
-	dvb->dmxdev.filternum    = 256;
-	dvb->dmxdev.demux        = &dvb->demux.dmx;
-	dvb->dmxdev.capabilities = 0;
-	result = dvb_dmxdev_init(&dvb->dmxdev, adapter);
-
-	if (result < 0) {
-		printk(KERN_WARNING "%s: dvb_dmxdev_init failed (errno = %d)\n",
-		       dvb->name, result);
-		goto fail_dmxdev;
-	}
-
-	dvb->fe_hw.source = DMX_FRONTEND_0;
-	result = dvb->demux.dmx.add_frontend(&dvb->demux.dmx, &dvb->fe_hw);
-	if (result < 0) {
-		printk(KERN_WARNING "%s: add_frontend failed (DMX_FRONTEND_0, errno = %d)\n",
-		       dvb->name, result);
-		goto fail_fe_hw;
-	}
-
-	dvb->fe_mem.source = DMX_MEMORY_FE;
-	result = dvb->demux.dmx.add_frontend(&dvb->demux.dmx, &dvb->fe_mem);
-	if (result < 0) {
-		printk(KERN_WARNING "%s: add_frontend failed (DMX_MEMORY_FE, errno = %d)\n",
-		       dvb->name, result);
-		goto fail_fe_mem;
-	}
-
-	result = dvb->demux.dmx.connect_frontend(&dvb->demux.dmx, &dvb->fe_hw);
-	if (result < 0) {
-		printk(KERN_WARNING "%s: connect_frontend failed (errno = %d)\n",
-		       dvb->name, result);
-		goto fail_fe_conn;
-	}
-
-	/* register network adapter */
-	result = dvb_net_init(adapter, &dvb->net, &dvb->demux.dmx);
-	if (result < 0) {
-		printk(KERN_WARNING "%s: dvb_net_init failed (errno = %d)\n",
-		       dvb->name, result);
-		goto fail_fe_conn;
-	}
-	return 0;
-
-fail_fe_conn:
-	dvb->demux.dmx.remove_frontend(&dvb->demux.dmx, &dvb->fe_mem);
-fail_fe_mem:
-	dvb->demux.dmx.remove_frontend(&dvb->demux.dmx, &dvb->fe_hw);
-fail_fe_hw:
-	dvb_dmxdev_release(&dvb->dmxdev);
-fail_dmxdev:
-	dvb_dmx_release(&dvb->demux);
-fail_dmx:
-	dvb_unregister_frontend(dvb->frontend);
-fail_frontend:
-	dvb_frontend_detach(dvb->frontend);
-	dvb->frontend = NULL;
-
-	return result;
-}
-
-/* ------------------------------------------------------------------ */
-/* Register a single adapter and one or more frontends */
-int videobuf_dvb_register_bus(struct videobuf_dvb_frontends *f,
-			  struct module *module,
-			  void *adapter_priv,
-			  struct device *device,
-			  short *adapter_nr,
-			  int mfe_shared)
-{
-	struct list_head *list, *q;
-	struct videobuf_dvb_frontend *fe;
-	int res;
-
-	fe = videobuf_dvb_get_frontend(f, 1);
-	if (!fe) {
-		printk(KERN_WARNING "Unable to register the adapter which has no frontends\n");
-		return -EINVAL;
-	}
-
-	/* Bring up the adapter */
-	res = videobuf_dvb_register_adapter(f, module, adapter_priv, device,
-		fe->dvb.name, adapter_nr, mfe_shared);
-	if (res < 0) {
-		printk(KERN_WARNING "videobuf_dvb_register_adapter failed (errno = %d)\n", res);
-		return res;
-	}
-
-	/* Attach all of the frontends to the adapter */
-	mutex_lock(&f->lock);
-	list_for_each_safe(list, q, &f->felist) {
-		fe = list_entry(list, struct videobuf_dvb_frontend, felist);
-		res = videobuf_dvb_register_frontend(&f->adapter, &fe->dvb);
-		if (res < 0) {
-			printk(KERN_WARNING "%s: videobuf_dvb_register_frontend failed (errno = %d)\n",
-				fe->dvb.name, res);
-			goto err;
-		}
-	}
-	mutex_unlock(&f->lock);
-	return 0;
-
-err:
-	mutex_unlock(&f->lock);
-	videobuf_dvb_unregister_bus(f);
-	return res;
-}
-EXPORT_SYMBOL(videobuf_dvb_register_bus);
-
-void videobuf_dvb_unregister_bus(struct videobuf_dvb_frontends *f)
-{
-	videobuf_dvb_dealloc_frontends(f);
-
-	dvb_unregister_adapter(&f->adapter);
-}
-EXPORT_SYMBOL(videobuf_dvb_unregister_bus);
-
-struct videobuf_dvb_frontend *videobuf_dvb_get_frontend(
-	struct videobuf_dvb_frontends *f, int id)
-{
-	struct list_head *list, *q;
-	struct videobuf_dvb_frontend *fe, *ret = NULL;
-
-	mutex_lock(&f->lock);
-
-	list_for_each_safe(list, q, &f->felist) {
-		fe = list_entry(list, struct videobuf_dvb_frontend, felist);
-		if (fe->id == id) {
-			ret = fe;
-			break;
-		}
-	}
-
-	mutex_unlock(&f->lock);
-
-	return ret;
-}
-EXPORT_SYMBOL(videobuf_dvb_get_frontend);
-
-int videobuf_dvb_find_frontend(struct videobuf_dvb_frontends *f,
-	struct dvb_frontend *p)
-{
-	struct list_head *list, *q;
-	struct videobuf_dvb_frontend *fe = NULL;
-	int ret = 0;
-
-	mutex_lock(&f->lock);
-
-	list_for_each_safe(list, q, &f->felist) {
-		fe = list_entry(list, struct videobuf_dvb_frontend, felist);
-		if (fe->dvb.frontend == p) {
-			ret = fe->id;
-			break;
-		}
-	}
-
-	mutex_unlock(&f->lock);
-
-	return ret;
-}
-EXPORT_SYMBOL(videobuf_dvb_find_frontend);
-
-struct videobuf_dvb_frontend *videobuf_dvb_alloc_frontend(
-	struct videobuf_dvb_frontends *f, int id)
-{
-	struct videobuf_dvb_frontend *fe;
-
-	fe = kzalloc(sizeof(struct videobuf_dvb_frontend), GFP_KERNEL);
-	if (fe == NULL)
-		goto fail_alloc;
-
-	fe->id = id;
-	mutex_init(&fe->dvb.lock);
-
-	mutex_lock(&f->lock);
-	list_add_tail(&fe->felist, &f->felist);
-	mutex_unlock(&f->lock);
-
-fail_alloc:
-	return fe;
-}
-EXPORT_SYMBOL(videobuf_dvb_alloc_frontend);
-
-void videobuf_dvb_dealloc_frontends(struct videobuf_dvb_frontends *f)
-{
-	struct list_head *list, *q;
-	struct videobuf_dvb_frontend *fe;
-
-	mutex_lock(&f->lock);
-	list_for_each_safe(list, q, &f->felist) {
-		fe = list_entry(list, struct videobuf_dvb_frontend, felist);
-		if (fe->dvb.net.dvbdev) {
-			dvb_net_release(&fe->dvb.net);
-			fe->dvb.demux.dmx.remove_frontend(&fe->dvb.demux.dmx,
-				&fe->dvb.fe_mem);
-			fe->dvb.demux.dmx.remove_frontend(&fe->dvb.demux.dmx,
-				&fe->dvb.fe_hw);
-			dvb_dmxdev_release(&fe->dvb.dmxdev);
-			dvb_dmx_release(&fe->dvb.demux);
-			dvb_unregister_frontend(fe->dvb.frontend);
-		}
-		if (fe->dvb.frontend)
-			/* always allocated, may have been reset */
-			dvb_frontend_detach(fe->dvb.frontend);
-		list_del(list); /* remove list entry */
-		kfree(fe);	/* free frontend allocation */
-	}
-	mutex_unlock(&f->lock);
-}
-EXPORT_SYMBOL(videobuf_dvb_dealloc_frontends);
diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig
index 19a0e83f260d..8d731d6c3e54 100644
--- a/drivers/memory/Kconfig
+++ b/drivers/memory/Kconfig
@@ -104,16 +104,6 @@ config MVEBU_DEVBUS
 	  Armada 370 and Armada XP. This controller allows to handle flash
 	  devices such as NOR, NAND, SRAM, and FPGA.
 
-config TEGRA20_MC
-	bool "Tegra20 Memory Controller(MC) driver"
-	default y
-	depends on ARCH_TEGRA_2x_SOC
-	help
-	  This driver is for the Memory Controller(MC) module available
-	  in Tegra20 SoCs, mainly for a address translation fault
-	  analysis, especially for IOMMU/GART(Graphics Address
-	  Relocation Table) module.
-
 config FSL_CORENET_CF
 	tristate "Freescale CoreNet Error Reporting"
 	depends on FSL_SOC_BOOKE
diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile
index 66f55240830e..a01ab3e22f94 100644
--- a/drivers/memory/Makefile
+++ b/drivers/memory/Makefile
@@ -16,7 +16,6 @@ obj-$(CONFIG_OMAP_GPMC)		+= omap-gpmc.o
 obj-$(CONFIG_FSL_CORENET_CF)	+= fsl-corenet-cf.o
 obj-$(CONFIG_FSL_IFC)		+= fsl_ifc.o
 obj-$(CONFIG_MVEBU_DEVBUS)	+= mvebu-devbus.o
-obj-$(CONFIG_TEGRA20_MC)	+= tegra20-mc.o
 obj-$(CONFIG_JZ4780_NEMC)	+= jz4780-nemc.o
 obj-$(CONFIG_MTK_SMI)		+= mtk-smi.o
 obj-$(CONFIG_DA8XX_DDRCTL)	+= da8xx-ddrctl.o
diff --git a/drivers/memory/brcmstb_dpfe.c b/drivers/memory/brcmstb_dpfe.c
index e9c1485c32b9..04599eccd604 100644
--- a/drivers/memory/brcmstb_dpfe.c
+++ b/drivers/memory/brcmstb_dpfe.c
@@ -176,7 +176,6 @@ struct private_data {
 	void __iomem *dmem;
 	void __iomem *imem;
 	struct device *dev;
-	unsigned int index;
 	struct mutex lock;
 };
 
@@ -674,10 +673,8 @@ static int brcmstb_dpfe_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct private_data *priv;
-	struct device *dpfe_dev;
 	struct init_data init;
 	struct resource *res;
-	u32 index;
 	int ret;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
@@ -687,11 +684,6 @@ static int brcmstb_dpfe_probe(struct platform_device *pdev)
 	mutex_init(&priv->lock);
 	platform_set_drvdata(pdev, priv);
 
-	/* Cell index is optional; default to 0 if not present. */
-	ret = of_property_read_u32(dev->of_node, "cell-index", &index);
-	if (ret)
-		index = 0;
-
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dpfe-cpu");
 	priv->regs = devm_ioremap_resource(dev, res);
 	if (IS_ERR(priv->regs)) {
@@ -715,35 +707,20 @@ static int brcmstb_dpfe_probe(struct platform_device *pdev)
 
 	ret = brcmstb_dpfe_download_firmware(pdev, &init);
 	if (ret)
-		goto err;
-
-	dpfe_dev = devm_kzalloc(dev, sizeof(*dpfe_dev), GFP_KERNEL);
-	if (!dpfe_dev) {
-		ret = -ENOMEM;
-		goto err;
-	}
-
-	priv->dev = dpfe_dev;
-	priv->index = index;
+		return ret;
 
-	dpfe_dev->parent = dev;
-	dpfe_dev->groups = dpfe_groups;
-	dpfe_dev->of_node = dev->of_node;
-	dev_set_drvdata(dpfe_dev, priv);
-	dev_set_name(dpfe_dev, "dpfe%u", index);
+	ret = sysfs_create_groups(&pdev->dev.kobj, dpfe_groups);
+	if (!ret)
+		dev_info(dev, "registered.\n");
 
-	ret = device_register(dpfe_dev);
-	if (ret)
-		goto err;
+	return ret;
+}
 
-	dev_info(dev, "registered.\n");
+static int brcmstb_dpfe_remove(struct platform_device *pdev)
+{
+	sysfs_remove_groups(&pdev->dev.kobj, dpfe_groups);
 
 	return 0;
-
-err:
-	dev_err(dev, "failed to initialize -- error %d\n", ret);
-
-	return ret;
 }
 
 static const struct of_device_id brcmstb_dpfe_of_match[] = {
@@ -758,6 +735,7 @@ static struct platform_driver brcmstb_dpfe_driver = {
 		.of_match_table = brcmstb_dpfe_of_match,
 	},
 	.probe = brcmstb_dpfe_probe,
+	.remove	= brcmstb_dpfe_remove,
 	.resume = brcmstb_dpfe_resume,
 };
 
diff --git a/drivers/memory/omap-gpmc.c b/drivers/memory/omap-gpmc.c
index 90a66b3f7ae1..c215287e80cf 100644
--- a/drivers/memory/omap-gpmc.c
+++ b/drivers/memory/omap-gpmc.c
@@ -2060,8 +2060,8 @@ static int gpmc_probe_generic_child(struct platform_device *pdev,
 	 * timings.
 	 */
 	name = gpmc_cs_get_name(cs);
-	if (name && child->name && of_node_cmp(child->name, name) == 0)
-			goto no_timings;
+	if (name && of_node_cmp(child->name, name) == 0)
+		goto no_timings;
 
 	ret = gpmc_cs_request(cs, resource_size(&res), &base);
 	if (ret < 0) {
diff --git a/drivers/memory/tegra/Makefile b/drivers/memory/tegra/Makefile
index ce87a9470034..94ab16ba075b 100644
--- a/drivers/memory/tegra/Makefile
+++ b/drivers/memory/tegra/Makefile
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 tegra-mc-y := mc.o
 
+tegra-mc-$(CONFIG_ARCH_TEGRA_2x_SOC)  += tegra20.o
 tegra-mc-$(CONFIG_ARCH_TEGRA_3x_SOC)  += tegra30.o
 tegra-mc-$(CONFIG_ARCH_TEGRA_114_SOC) += tegra114.o
 tegra-mc-$(CONFIG_ARCH_TEGRA_124_SOC) += tegra124.o
diff --git a/drivers/memory/tegra/mc.c b/drivers/memory/tegra/mc.c
index a4803ac192bb..bb93cc53554e 100644
--- a/drivers/memory/tegra/mc.c
+++ b/drivers/memory/tegra/mc.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/clk.h>
+#include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -20,14 +21,6 @@
 #include "mc.h"
 
 #define MC_INTSTATUS 0x000
-#define  MC_INT_DECERR_MTS (1 << 16)
-#define  MC_INT_SECERR_SEC (1 << 13)
-#define  MC_INT_DECERR_VPR (1 << 12)
-#define  MC_INT_INVALID_APB_ASID_UPDATE (1 << 11)
-#define  MC_INT_INVALID_SMMU_PAGE (1 << 10)
-#define  MC_INT_ARBITRATION_EMEM (1 << 9)
-#define  MC_INT_SECURITY_VIOLATION (1 << 8)
-#define  MC_INT_DECERR_EMEM (1 << 6)
 
 #define MC_INTMASK 0x004
 
@@ -45,6 +38,9 @@
 
 #define MC_ERR_ADR 0x0c
 
+#define MC_DECERR_EMEM_OTHERS_STATUS	0x58
+#define MC_SECURITY_VIOLATION_STATUS	0x74
+
 #define MC_EMEM_ARB_CFG 0x90
 #define  MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(x)	(((x) & 0x1ff) << 0)
 #define  MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK	0x1ff
@@ -54,6 +50,9 @@
 #define MC_EMEM_ADR_CFG_EMEM_NUMDEV BIT(0)
 
 static const struct of_device_id tegra_mc_of_match[] = {
+#ifdef CONFIG_ARCH_TEGRA_2x_SOC
+	{ .compatible = "nvidia,tegra20-mc", .data = &tegra20_mc_soc },
+#endif
 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
 	{ .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
 #endif
@@ -73,6 +72,207 @@ static const struct of_device_id tegra_mc_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
 
+static int terga_mc_block_dma_common(struct tegra_mc *mc,
+				     const struct tegra_mc_reset *rst)
+{
+	unsigned long flags;
+	u32 value;
+
+	spin_lock_irqsave(&mc->lock, flags);
+
+	value = mc_readl(mc, rst->control) | BIT(rst->bit);
+	mc_writel(mc, value, rst->control);
+
+	spin_unlock_irqrestore(&mc->lock, flags);
+
+	return 0;
+}
+
+static bool terga_mc_dma_idling_common(struct tegra_mc *mc,
+				       const struct tegra_mc_reset *rst)
+{
+	return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
+}
+
+static int terga_mc_unblock_dma_common(struct tegra_mc *mc,
+				       const struct tegra_mc_reset *rst)
+{
+	unsigned long flags;
+	u32 value;
+
+	spin_lock_irqsave(&mc->lock, flags);
+
+	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
+	mc_writel(mc, value, rst->control);
+
+	spin_unlock_irqrestore(&mc->lock, flags);
+
+	return 0;
+}
+
+static int terga_mc_reset_status_common(struct tegra_mc *mc,
+					const struct tegra_mc_reset *rst)
+{
+	return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
+}
+
+const struct tegra_mc_reset_ops terga_mc_reset_ops_common = {
+	.block_dma = terga_mc_block_dma_common,
+	.dma_idling = terga_mc_dma_idling_common,
+	.unblock_dma = terga_mc_unblock_dma_common,
+	.reset_status = terga_mc_reset_status_common,
+};
+
+static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
+{
+	return container_of(rcdev, struct tegra_mc, reset);
+}
+
+static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
+							unsigned long id)
+{
+	unsigned int i;
+
+	for (i = 0; i < mc->soc->num_resets; i++)
+		if (mc->soc->resets[i].id == id)
+			return &mc->soc->resets[i];
+
+	return NULL;
+}
+
+static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
+				    unsigned long id)
+{
+	struct tegra_mc *mc = reset_to_mc(rcdev);
+	const struct tegra_mc_reset_ops *rst_ops;
+	const struct tegra_mc_reset *rst;
+	int retries = 500;
+	int err;
+
+	rst = tegra_mc_reset_find(mc, id);
+	if (!rst)
+		return -ENODEV;
+
+	rst_ops = mc->soc->reset_ops;
+	if (!rst_ops)
+		return -ENODEV;
+
+	if (rst_ops->block_dma) {
+		/* block clients DMA requests */
+		err = rst_ops->block_dma(mc, rst);
+		if (err) {
+			dev_err(mc->dev, "Failed to block %s DMA: %d\n",
+				rst->name, err);
+			return err;
+		}
+	}
+
+	if (rst_ops->dma_idling) {
+		/* wait for completion of the outstanding DMA requests */
+		while (!rst_ops->dma_idling(mc, rst)) {
+			if (!retries--) {
+				dev_err(mc->dev, "Failed to flush %s DMA\n",
+					rst->name);
+				return -EBUSY;
+			}
+
+			usleep_range(10, 100);
+		}
+	}
+
+	if (rst_ops->hotreset_assert) {
+		/* clear clients DMA requests sitting before arbitration */
+		err = rst_ops->hotreset_assert(mc, rst);
+		if (err) {
+			dev_err(mc->dev, "Failed to hot reset %s: %d\n",
+				rst->name, err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
+				      unsigned long id)
+{
+	struct tegra_mc *mc = reset_to_mc(rcdev);
+	const struct tegra_mc_reset_ops *rst_ops;
+	const struct tegra_mc_reset *rst;
+	int err;
+
+	rst = tegra_mc_reset_find(mc, id);
+	if (!rst)
+		return -ENODEV;
+
+	rst_ops = mc->soc->reset_ops;
+	if (!rst_ops)
+		return -ENODEV;
+
+	if (rst_ops->hotreset_deassert) {
+		/* take out client from hot reset */
+		err = rst_ops->hotreset_deassert(mc, rst);
+		if (err) {
+			dev_err(mc->dev, "Failed to deassert hot reset %s: %d\n",
+				rst->name, err);
+			return err;
+		}
+	}
+
+	if (rst_ops->unblock_dma) {
+		/* allow new DMA requests to proceed to arbitration */
+		err = rst_ops->unblock_dma(mc, rst);
+		if (err) {
+			dev_err(mc->dev, "Failed to unblock %s DMA : %d\n",
+				rst->name, err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
+				    unsigned long id)
+{
+	struct tegra_mc *mc = reset_to_mc(rcdev);
+	const struct tegra_mc_reset_ops *rst_ops;
+	const struct tegra_mc_reset *rst;
+
+	rst = tegra_mc_reset_find(mc, id);
+	if (!rst)
+		return -ENODEV;
+
+	rst_ops = mc->soc->reset_ops;
+	if (!rst_ops)
+		return -ENODEV;
+
+	return rst_ops->reset_status(mc, rst);
+}
+
+static const struct reset_control_ops tegra_mc_reset_ops = {
+	.assert = tegra_mc_hotreset_assert,
+	.deassert = tegra_mc_hotreset_deassert,
+	.status = tegra_mc_hotreset_status,
+};
+
+static int tegra_mc_reset_setup(struct tegra_mc *mc)
+{
+	int err;
+
+	mc->reset.ops = &tegra_mc_reset_ops;
+	mc->reset.owner = THIS_MODULE;
+	mc->reset.of_node = mc->dev->of_node;
+	mc->reset.of_reset_n_cells = 1;
+	mc->reset.nr_resets = mc->soc->num_resets;
+
+	err = reset_controller_register(&mc->reset);
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+
 static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
 {
 	unsigned long long tick;
@@ -229,6 +429,7 @@ static int tegra_mc_setup_timings(struct tegra_mc *mc)
 static const char *const status_names[32] = {
 	[ 1] = "External interrupt",
 	[ 6] = "EMEM address decode error",
+	[ 7] = "GART page fault",
 	[ 8] = "Security violation",
 	[ 9] = "EMEM arbitration error",
 	[10] = "Page fault",
@@ -248,12 +449,13 @@ static const char *const error_names[8] = {
 static irqreturn_t tegra_mc_irq(int irq, void *data)
 {
 	struct tegra_mc *mc = data;
-	unsigned long status, mask;
+	unsigned long status;
 	unsigned int bit;
 
 	/* mask all interrupts to avoid flooding */
-	status = mc_readl(mc, MC_INTSTATUS);
-	mask = mc_readl(mc, MC_INTMASK);
+	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
+	if (!status)
+		return IRQ_NONE;
 
 	for_each_set_bit(bit, &status, 32) {
 		const char *error = status_names[bit] ?: "unknown";
@@ -341,12 +543,78 @@ static irqreturn_t tegra_mc_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static __maybe_unused irqreturn_t tegra20_mc_irq(int irq, void *data)
+{
+	struct tegra_mc *mc = data;
+	unsigned long status;
+	unsigned int bit;
+
+	/* mask all interrupts to avoid flooding */
+	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
+	if (!status)
+		return IRQ_NONE;
+
+	for_each_set_bit(bit, &status, 32) {
+		const char *direction = "read", *secure = "";
+		const char *error = status_names[bit];
+		const char *client, *desc;
+		phys_addr_t addr;
+		u32 value, reg;
+		u8 id, type;
+
+		switch (BIT(bit)) {
+		case MC_INT_DECERR_EMEM:
+			reg = MC_DECERR_EMEM_OTHERS_STATUS;
+			value = mc_readl(mc, reg);
+
+			id = value & mc->soc->client_id_mask;
+			desc = error_names[2];
+
+			if (value & BIT(31))
+				direction = "write";
+			break;
+
+		case MC_INT_INVALID_GART_PAGE:
+			dev_err_ratelimited(mc->dev, "%s\n", error);
+			continue;
+
+		case MC_INT_SECURITY_VIOLATION:
+			reg = MC_SECURITY_VIOLATION_STATUS;
+			value = mc_readl(mc, reg);
+
+			id = value & mc->soc->client_id_mask;
+			type = (value & BIT(30)) ? 4 : 3;
+			desc = error_names[type];
+			secure = "secure ";
+
+			if (value & BIT(31))
+				direction = "write";
+			break;
+
+		default:
+			continue;
+		}
+
+		client = mc->soc->clients[id].name;
+		addr = mc_readl(mc, reg + sizeof(u32));
+
+		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
+				    client, secure, direction, &addr, error,
+				    desc);
+	}
+
+	/* clear interrupts */
+	mc_writel(mc, status, MC_INTSTATUS);
+
+	return IRQ_HANDLED;
+}
+
 static int tegra_mc_probe(struct platform_device *pdev)
 {
 	const struct of_device_id *match;
 	struct resource *res;
 	struct tegra_mc *mc;
-	u32 value;
+	void *isr;
 	int err;
 
 	match = of_match_node(tegra_mc_of_match, pdev->dev.of_node);
@@ -358,6 +626,7 @@ static int tegra_mc_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	platform_set_drvdata(pdev, mc);
+	spin_lock_init(&mc->lock);
 	mc->soc = match->data;
 	mc->dev = &pdev->dev;
 
@@ -369,18 +638,32 @@ static int tegra_mc_probe(struct platform_device *pdev)
 	if (IS_ERR(mc->regs))
 		return PTR_ERR(mc->regs);
 
-	mc->clk = devm_clk_get(&pdev->dev, "mc");
-	if (IS_ERR(mc->clk)) {
-		dev_err(&pdev->dev, "failed to get MC clock: %ld\n",
-			PTR_ERR(mc->clk));
-		return PTR_ERR(mc->clk);
-	}
+#ifdef CONFIG_ARCH_TEGRA_2x_SOC
+	if (mc->soc == &tegra20_mc_soc) {
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+		mc->regs2 = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(mc->regs2))
+			return PTR_ERR(mc->regs2);
 
-	err = tegra_mc_setup_latency_allowance(mc);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to setup latency allowance: %d\n",
-			err);
-		return err;
+		isr = tegra20_mc_irq;
+	} else
+#endif
+	{
+		mc->clk = devm_clk_get(&pdev->dev, "mc");
+		if (IS_ERR(mc->clk)) {
+			dev_err(&pdev->dev, "failed to get MC clock: %ld\n",
+				PTR_ERR(mc->clk));
+			return PTR_ERR(mc->clk);
+		}
+
+		err = tegra_mc_setup_latency_allowance(mc);
+		if (err < 0) {
+			dev_err(&pdev->dev, "failed to setup latency allowance: %d\n",
+				err);
+			return err;
+		}
+
+		isr = tegra_mc_irq;
 	}
 
 	err = tegra_mc_setup_timings(mc);
@@ -389,13 +672,11 @@ static int tegra_mc_probe(struct platform_device *pdev)
 		return err;
 	}
 
-	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU)) {
-		mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
-		if (IS_ERR(mc->smmu)) {
-			dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
-				PTR_ERR(mc->smmu));
-			return PTR_ERR(mc->smmu);
-		}
+	err = tegra_mc_reset_setup(mc);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to register reset controller: %d\n",
+			err);
+		return err;
 	}
 
 	mc->irq = platform_get_irq(pdev, 0);
@@ -404,7 +685,11 @@ static int tegra_mc_probe(struct platform_device *pdev)
 		return mc->irq;
 	}
 
-	err = devm_request_irq(&pdev->dev, mc->irq, tegra_mc_irq, IRQF_SHARED,
+	WARN(!mc->soc->client_id_mask, "Missing client ID mask for this SoC\n");
+
+	mc_writel(mc, mc->soc->intmask, MC_INTMASK);
+
+	err = devm_request_irq(&pdev->dev, mc->irq, isr, IRQF_SHARED,
 			       dev_name(&pdev->dev), mc);
 	if (err < 0) {
 		dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
@@ -412,13 +697,14 @@ static int tegra_mc_probe(struct platform_device *pdev)
 		return err;
 	}
 
-	WARN(!mc->soc->client_id_mask, "Missing client ID mask for this SoC\n");
-
-	value = MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
-		MC_INT_INVALID_APB_ASID_UPDATE | MC_INT_INVALID_SMMU_PAGE |
-		MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM;
-
-	mc_writel(mc, value, MC_INTMASK);
+	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU)) {
+		mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
+		if (IS_ERR(mc->smmu)) {
+			dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
+				PTR_ERR(mc->smmu));
+			return PTR_ERR(mc->smmu);
+		}
+	}
 
 	return 0;
 }
diff --git a/drivers/memory/tegra/mc.h b/drivers/memory/tegra/mc.h
index ddb16676c3af..01065f12ebeb 100644
--- a/drivers/memory/tegra/mc.h
+++ b/drivers/memory/tegra/mc.h
@@ -14,17 +14,39 @@
 
 #include <soc/tegra/mc.h>
 
+#define MC_INT_DECERR_MTS (1 << 16)
+#define MC_INT_SECERR_SEC (1 << 13)
+#define MC_INT_DECERR_VPR (1 << 12)
+#define MC_INT_INVALID_APB_ASID_UPDATE (1 << 11)
+#define MC_INT_INVALID_SMMU_PAGE (1 << 10)
+#define MC_INT_ARBITRATION_EMEM (1 << 9)
+#define MC_INT_SECURITY_VIOLATION (1 << 8)
+#define MC_INT_INVALID_GART_PAGE (1 << 7)
+#define MC_INT_DECERR_EMEM (1 << 6)
+
 static inline u32 mc_readl(struct tegra_mc *mc, unsigned long offset)
 {
+	if (mc->regs2 && offset >= 0x24)
+		return readl(mc->regs2 + offset - 0x3c);
+
 	return readl(mc->regs + offset);
 }
 
 static inline void mc_writel(struct tegra_mc *mc, u32 value,
 			     unsigned long offset)
 {
+	if (mc->regs2 && offset >= 0x24)
+		return writel(value, mc->regs2 + offset - 0x3c);
+
 	writel(value, mc->regs + offset);
 }
 
+extern const struct tegra_mc_reset_ops terga_mc_reset_ops_common;
+
+#ifdef CONFIG_ARCH_TEGRA_2x_SOC
+extern const struct tegra_mc_soc tegra20_mc_soc;
+#endif
+
 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
 extern const struct tegra_mc_soc tegra30_mc_soc;
 #endif
diff --git a/drivers/memory/tegra/tegra114.c b/drivers/memory/tegra/tegra114.c
index b20e6e3e208e..6560a5101322 100644
--- a/drivers/memory/tegra/tegra114.c
+++ b/drivers/memory/tegra/tegra114.c
@@ -938,6 +938,34 @@ static const struct tegra_smmu_soc tegra114_smmu_soc = {
 	.num_asids = 4,
 };
 
+#define TEGRA114_MC_RESET(_name, _control, _status, _bit)	\
+	{							\
+		.name = #_name,					\
+		.id = TEGRA114_MC_RESET_##_name,		\
+		.control = _control,				\
+		.status = _status,				\
+		.bit = _bit,					\
+	}
+
+static const struct tegra_mc_reset tegra114_mc_resets[] = {
+	TEGRA114_MC_RESET(AVPC,     0x200, 0x204,  1),
+	TEGRA114_MC_RESET(DC,       0x200, 0x204,  2),
+	TEGRA114_MC_RESET(DCB,      0x200, 0x204,  3),
+	TEGRA114_MC_RESET(EPP,      0x200, 0x204,  4),
+	TEGRA114_MC_RESET(2D,       0x200, 0x204,  5),
+	TEGRA114_MC_RESET(HC,       0x200, 0x204,  6),
+	TEGRA114_MC_RESET(HDA,      0x200, 0x204,  7),
+	TEGRA114_MC_RESET(ISP,      0x200, 0x204,  8),
+	TEGRA114_MC_RESET(MPCORE,   0x200, 0x204,  9),
+	TEGRA114_MC_RESET(MPCORELP, 0x200, 0x204, 10),
+	TEGRA114_MC_RESET(MPE,      0x200, 0x204, 11),
+	TEGRA114_MC_RESET(3D,       0x200, 0x204, 12),
+	TEGRA114_MC_RESET(3D2,      0x200, 0x204, 13),
+	TEGRA114_MC_RESET(PPCS,     0x200, 0x204, 14),
+	TEGRA114_MC_RESET(VDE,      0x200, 0x204, 16),
+	TEGRA114_MC_RESET(VI,       0x200, 0x204, 17),
+};
+
 const struct tegra_mc_soc tegra114_mc_soc = {
 	.clients = tegra114_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra114_mc_clients),
@@ -945,4 +973,9 @@ const struct tegra_mc_soc tegra114_mc_soc = {
 	.atom_size = 32,
 	.client_id_mask = 0x7f,
 	.smmu = &tegra114_smmu_soc,
+	.intmask = MC_INT_INVALID_SMMU_PAGE | MC_INT_SECURITY_VIOLATION |
+		   MC_INT_DECERR_EMEM,
+	.reset_ops = &terga_mc_reset_ops_common,
+	.resets = tegra114_mc_resets,
+	.num_resets = ARRAY_SIZE(tegra114_mc_resets),
 };
diff --git a/drivers/memory/tegra/tegra124.c b/drivers/memory/tegra/tegra124.c
index 8b6360eabb8a..b561a1fe7f46 100644
--- a/drivers/memory/tegra/tegra124.c
+++ b/drivers/memory/tegra/tegra124.c
@@ -1012,6 +1012,42 @@ static const struct tegra_smmu_group_soc tegra124_groups[] = {
 	},
 };
 
+#define TEGRA124_MC_RESET(_name, _control, _status, _bit)	\
+	{							\
+		.name = #_name,					\
+		.id = TEGRA124_MC_RESET_##_name,		\
+		.control = _control,				\
+		.status = _status,				\
+		.bit = _bit,					\
+	}
+
+static const struct tegra_mc_reset tegra124_mc_resets[] = {
+	TEGRA124_MC_RESET(AFI,       0x200, 0x204,  0),
+	TEGRA124_MC_RESET(AVPC,      0x200, 0x204,  1),
+	TEGRA124_MC_RESET(DC,        0x200, 0x204,  2),
+	TEGRA124_MC_RESET(DCB,       0x200, 0x204,  3),
+	TEGRA124_MC_RESET(HC,        0x200, 0x204,  6),
+	TEGRA124_MC_RESET(HDA,       0x200, 0x204,  7),
+	TEGRA124_MC_RESET(ISP2,      0x200, 0x204,  8),
+	TEGRA124_MC_RESET(MPCORE,    0x200, 0x204,  9),
+	TEGRA124_MC_RESET(MPCORELP,  0x200, 0x204, 10),
+	TEGRA124_MC_RESET(MSENC,     0x200, 0x204, 11),
+	TEGRA124_MC_RESET(PPCS,      0x200, 0x204, 14),
+	TEGRA124_MC_RESET(SATA,      0x200, 0x204, 15),
+	TEGRA124_MC_RESET(VDE,       0x200, 0x204, 16),
+	TEGRA124_MC_RESET(VI,        0x200, 0x204, 17),
+	TEGRA124_MC_RESET(VIC,       0x200, 0x204, 18),
+	TEGRA124_MC_RESET(XUSB_HOST, 0x200, 0x204, 19),
+	TEGRA124_MC_RESET(XUSB_DEV,  0x200, 0x204, 20),
+	TEGRA124_MC_RESET(TSEC,      0x200, 0x204, 21),
+	TEGRA124_MC_RESET(SDMMC1,    0x200, 0x204, 22),
+	TEGRA124_MC_RESET(SDMMC2,    0x200, 0x204, 23),
+	TEGRA124_MC_RESET(SDMMC3,    0x200, 0x204, 25),
+	TEGRA124_MC_RESET(SDMMC4,    0x970, 0x974,  0),
+	TEGRA124_MC_RESET(ISP2B,     0x970, 0x974,  1),
+	TEGRA124_MC_RESET(GPU,       0x970, 0x974,  2),
+};
+
 #ifdef CONFIG_ARCH_TEGRA_124_SOC
 static const struct tegra_smmu_soc tegra124_smmu_soc = {
 	.clients = tegra124_mc_clients,
@@ -1035,6 +1071,12 @@ const struct tegra_mc_soc tegra124_mc_soc = {
 	.smmu = &tegra124_smmu_soc,
 	.emem_regs = tegra124_mc_emem_regs,
 	.num_emem_regs = ARRAY_SIZE(tegra124_mc_emem_regs),
+	.intmask = MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
+		   MC_INT_INVALID_APB_ASID_UPDATE | MC_INT_INVALID_SMMU_PAGE |
+		   MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	.reset_ops = &terga_mc_reset_ops_common,
+	.resets = tegra124_mc_resets,
+	.num_resets = ARRAY_SIZE(tegra124_mc_resets),
 };
 #endif /* CONFIG_ARCH_TEGRA_124_SOC */
 
@@ -1059,5 +1101,11 @@ const struct tegra_mc_soc tegra132_mc_soc = {
 	.atom_size = 32,
 	.client_id_mask = 0x7f,
 	.smmu = &tegra132_smmu_soc,
+	.intmask = MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
+		   MC_INT_INVALID_APB_ASID_UPDATE | MC_INT_INVALID_SMMU_PAGE |
+		   MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	.reset_ops = &terga_mc_reset_ops_common,
+	.resets = tegra124_mc_resets,
+	.num_resets = ARRAY_SIZE(tegra124_mc_resets),
 };
 #endif /* CONFIG_ARCH_TEGRA_132_SOC */
diff --git a/drivers/memory/tegra/tegra20.c b/drivers/memory/tegra/tegra20.c
new file mode 100644
index 000000000000..7119e532471c
--- /dev/null
+++ b/drivers/memory/tegra/tegra20.c
@@ -0,0 +1,296 @@
+/*
+ * Copyright (C) 2012 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <dt-bindings/memory/tegra20-mc.h>
+
+#include "mc.h"
+
+static const struct tegra_mc_client tegra20_mc_clients[] = {
+	{
+		.id = 0x00,
+		.name = "display0a",
+	}, {
+		.id = 0x01,
+		.name = "display0ab",
+	}, {
+		.id = 0x02,
+		.name = "display0b",
+	}, {
+		.id = 0x03,
+		.name = "display0bb",
+	}, {
+		.id = 0x04,
+		.name = "display0c",
+	}, {
+		.id = 0x05,
+		.name = "display0cb",
+	}, {
+		.id = 0x06,
+		.name = "display1b",
+	}, {
+		.id = 0x07,
+		.name = "display1bb",
+	}, {
+		.id = 0x08,
+		.name = "eppup",
+	}, {
+		.id = 0x09,
+		.name = "g2pr",
+	}, {
+		.id = 0x0a,
+		.name = "g2sr",
+	}, {
+		.id = 0x0b,
+		.name = "mpeunifbr",
+	}, {
+		.id = 0x0c,
+		.name = "viruv",
+	}, {
+		.id = 0x0d,
+		.name = "avpcarm7r",
+	}, {
+		.id = 0x0e,
+		.name = "displayhc",
+	}, {
+		.id = 0x0f,
+		.name = "displayhcb",
+	}, {
+		.id = 0x10,
+		.name = "fdcdrd",
+	}, {
+		.id = 0x11,
+		.name = "g2dr",
+	}, {
+		.id = 0x12,
+		.name = "host1xdmar",
+	}, {
+		.id = 0x13,
+		.name = "host1xr",
+	}, {
+		.id = 0x14,
+		.name = "idxsrd",
+	}, {
+		.id = 0x15,
+		.name = "mpcorer",
+	}, {
+		.id = 0x16,
+		.name = "mpe_ipred",
+	}, {
+		.id = 0x17,
+		.name = "mpeamemrd",
+	}, {
+		.id = 0x18,
+		.name = "mpecsrd",
+	}, {
+		.id = 0x19,
+		.name = "ppcsahbdmar",
+	}, {
+		.id = 0x1a,
+		.name = "ppcsahbslvr",
+	}, {
+		.id = 0x1b,
+		.name = "texsrd",
+	}, {
+		.id = 0x1c,
+		.name = "vdebsevr",
+	}, {
+		.id = 0x1d,
+		.name = "vdember",
+	}, {
+		.id = 0x1e,
+		.name = "vdemcer",
+	}, {
+		.id = 0x1f,
+		.name = "vdetper",
+	}, {
+		.id = 0x20,
+		.name = "eppu",
+	}, {
+		.id = 0x21,
+		.name = "eppv",
+	}, {
+		.id = 0x22,
+		.name = "eppy",
+	}, {
+		.id = 0x23,
+		.name = "mpeunifbw",
+	}, {
+		.id = 0x24,
+		.name = "viwsb",
+	}, {
+		.id = 0x25,
+		.name = "viwu",
+	}, {
+		.id = 0x26,
+		.name = "viwv",
+	}, {
+		.id = 0x27,
+		.name = "viwy",
+	}, {
+		.id = 0x28,
+		.name = "g2dw",
+	}, {
+		.id = 0x29,
+		.name = "avpcarm7w",
+	}, {
+		.id = 0x2a,
+		.name = "fdcdwr",
+	}, {
+		.id = 0x2b,
+		.name = "host1xw",
+	}, {
+		.id = 0x2c,
+		.name = "ispw",
+	}, {
+		.id = 0x2d,
+		.name = "mpcorew",
+	}, {
+		.id = 0x2e,
+		.name = "mpecswr",
+	}, {
+		.id = 0x2f,
+		.name = "ppcsahbdmaw",
+	}, {
+		.id = 0x30,
+		.name = "ppcsahbslvw",
+	}, {
+		.id = 0x31,
+		.name = "vdebsevw",
+	}, {
+		.id = 0x32,
+		.name = "vdembew",
+	}, {
+		.id = 0x33,
+		.name = "vdetpmw",
+	},
+};
+
+#define TEGRA20_MC_RESET(_name, _control, _status, _reset, _bit)	\
+	{								\
+		.name = #_name,						\
+		.id = TEGRA20_MC_RESET_##_name,				\
+		.control = _control,					\
+		.status = _status,					\
+		.reset = _reset,					\
+		.bit = _bit,						\
+	}
+
+static const struct tegra_mc_reset tegra20_mc_resets[] = {
+	TEGRA20_MC_RESET(AVPC,   0x100, 0x140, 0x104,  0),
+	TEGRA20_MC_RESET(DC,     0x100, 0x144, 0x104,  1),
+	TEGRA20_MC_RESET(DCB,    0x100, 0x148, 0x104,  2),
+	TEGRA20_MC_RESET(EPP,    0x100, 0x14c, 0x104,  3),
+	TEGRA20_MC_RESET(2D,     0x100, 0x150, 0x104,  4),
+	TEGRA20_MC_RESET(HC,     0x100, 0x154, 0x104,  5),
+	TEGRA20_MC_RESET(ISP,    0x100, 0x158, 0x104,  6),
+	TEGRA20_MC_RESET(MPCORE, 0x100, 0x15c, 0x104,  7),
+	TEGRA20_MC_RESET(MPEA,   0x100, 0x160, 0x104,  8),
+	TEGRA20_MC_RESET(MPEB,   0x100, 0x164, 0x104,  9),
+	TEGRA20_MC_RESET(MPEC,   0x100, 0x168, 0x104, 10),
+	TEGRA20_MC_RESET(3D,     0x100, 0x16c, 0x104, 11),
+	TEGRA20_MC_RESET(PPCS,   0x100, 0x170, 0x104, 12),
+	TEGRA20_MC_RESET(VDE,    0x100, 0x174, 0x104, 13),
+	TEGRA20_MC_RESET(VI,     0x100, 0x178, 0x104, 14),
+};
+
+static int terga20_mc_hotreset_assert(struct tegra_mc *mc,
+				      const struct tegra_mc_reset *rst)
+{
+	unsigned long flags;
+	u32 value;
+
+	spin_lock_irqsave(&mc->lock, flags);
+
+	value = mc_readl(mc, rst->reset);
+	mc_writel(mc, value & ~BIT(rst->bit), rst->reset);
+
+	spin_unlock_irqrestore(&mc->lock, flags);
+
+	return 0;
+}
+
+static int terga20_mc_hotreset_deassert(struct tegra_mc *mc,
+					const struct tegra_mc_reset *rst)
+{
+	unsigned long flags;
+	u32 value;
+
+	spin_lock_irqsave(&mc->lock, flags);
+
+	value = mc_readl(mc, rst->reset);
+	mc_writel(mc, value | BIT(rst->bit), rst->reset);
+
+	spin_unlock_irqrestore(&mc->lock, flags);
+
+	return 0;
+}
+
+static int terga20_mc_block_dma(struct tegra_mc *mc,
+				const struct tegra_mc_reset *rst)
+{
+	unsigned long flags;
+	u32 value;
+
+	spin_lock_irqsave(&mc->lock, flags);
+
+	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
+	mc_writel(mc, value, rst->control);
+
+	spin_unlock_irqrestore(&mc->lock, flags);
+
+	return 0;
+}
+
+static bool terga20_mc_dma_idling(struct tegra_mc *mc,
+				  const struct tegra_mc_reset *rst)
+{
+	return mc_readl(mc, rst->status) == 0;
+}
+
+static int terga20_mc_reset_status(struct tegra_mc *mc,
+				   const struct tegra_mc_reset *rst)
+{
+	return (mc_readl(mc, rst->reset) & BIT(rst->bit)) == 0;
+}
+
+static int terga20_mc_unblock_dma(struct tegra_mc *mc,
+				  const struct tegra_mc_reset *rst)
+{
+	unsigned long flags;
+	u32 value;
+
+	spin_lock_irqsave(&mc->lock, flags);
+
+	value = mc_readl(mc, rst->control) | BIT(rst->bit);
+	mc_writel(mc, value, rst->control);
+
+	spin_unlock_irqrestore(&mc->lock, flags);
+
+	return 0;
+}
+
+const struct tegra_mc_reset_ops terga20_mc_reset_ops = {
+	.hotreset_assert = terga20_mc_hotreset_assert,
+	.hotreset_deassert = terga20_mc_hotreset_deassert,
+	.block_dma = terga20_mc_block_dma,
+	.dma_idling = terga20_mc_dma_idling,
+	.unblock_dma = terga20_mc_unblock_dma,
+	.reset_status = terga20_mc_reset_status,
+};
+
+const struct tegra_mc_soc tegra20_mc_soc = {
+	.clients = tegra20_mc_clients,
+	.num_clients = ARRAY_SIZE(tegra20_mc_clients),
+	.num_address_bits = 32,
+	.client_id_mask = 0x3f,
+	.intmask = MC_INT_SECURITY_VIOLATION | MC_INT_INVALID_GART_PAGE |
+		   MC_INT_DECERR_EMEM,
+	.reset_ops = &terga20_mc_reset_ops,
+	.resets = tegra20_mc_resets,
+	.num_resets = ARRAY_SIZE(tegra20_mc_resets),
+};
diff --git a/drivers/memory/tegra/tegra210.c b/drivers/memory/tegra/tegra210.c
index d398bcd3fc57..d00a77160407 100644
--- a/drivers/memory/tegra/tegra210.c
+++ b/drivers/memory/tegra/tegra210.c
@@ -6,11 +6,6 @@
  * published by the Free Software Foundation.
  */
 
-#include <linux/of.h>
-#include <linux/mm.h>
-
-#include <asm/cacheflush.h>
-
 #include <dt-bindings/memory/tegra210-mc.h>
 
 #include "mc.h"
@@ -1085,6 +1080,48 @@ static const struct tegra_smmu_soc tegra210_smmu_soc = {
 	.num_asids = 128,
 };
 
+#define TEGRA210_MC_RESET(_name, _control, _status, _bit)	\
+	{							\
+		.name = #_name,					\
+		.id = TEGRA210_MC_RESET_##_name,		\
+		.control = _control,				\
+		.status = _status,				\
+		.bit = _bit,					\
+	}
+
+static const struct tegra_mc_reset tegra210_mc_resets[] = {
+	TEGRA210_MC_RESET(AFI,       0x200, 0x204,  0),
+	TEGRA210_MC_RESET(AVPC,      0x200, 0x204,  1),
+	TEGRA210_MC_RESET(DC,        0x200, 0x204,  2),
+	TEGRA210_MC_RESET(DCB,       0x200, 0x204,  3),
+	TEGRA210_MC_RESET(HC,        0x200, 0x204,  6),
+	TEGRA210_MC_RESET(HDA,       0x200, 0x204,  7),
+	TEGRA210_MC_RESET(ISP2,      0x200, 0x204,  8),
+	TEGRA210_MC_RESET(MPCORE,    0x200, 0x204,  9),
+	TEGRA210_MC_RESET(NVENC,     0x200, 0x204, 11),
+	TEGRA210_MC_RESET(PPCS,      0x200, 0x204, 14),
+	TEGRA210_MC_RESET(SATA,      0x200, 0x204, 15),
+	TEGRA210_MC_RESET(VI,        0x200, 0x204, 17),
+	TEGRA210_MC_RESET(VIC,       0x200, 0x204, 18),
+	TEGRA210_MC_RESET(XUSB_HOST, 0x200, 0x204, 19),
+	TEGRA210_MC_RESET(XUSB_DEV,  0x200, 0x204, 20),
+	TEGRA210_MC_RESET(A9AVP,     0x200, 0x204, 21),
+	TEGRA210_MC_RESET(TSEC,      0x200, 0x204, 22),
+	TEGRA210_MC_RESET(SDMMC1,    0x200, 0x204, 29),
+	TEGRA210_MC_RESET(SDMMC2,    0x200, 0x204, 30),
+	TEGRA210_MC_RESET(SDMMC3,    0x200, 0x204, 31),
+	TEGRA210_MC_RESET(SDMMC4,    0x970, 0x974,  0),
+	TEGRA210_MC_RESET(ISP2B,     0x970, 0x974,  1),
+	TEGRA210_MC_RESET(GPU,       0x970, 0x974,  2),
+	TEGRA210_MC_RESET(NVDEC,     0x970, 0x974,  5),
+	TEGRA210_MC_RESET(APE,       0x970, 0x974,  6),
+	TEGRA210_MC_RESET(SE,        0x970, 0x974,  7),
+	TEGRA210_MC_RESET(NVJPG,     0x970, 0x974,  8),
+	TEGRA210_MC_RESET(AXIAP,     0x970, 0x974, 11),
+	TEGRA210_MC_RESET(ETR,       0x970, 0x974, 12),
+	TEGRA210_MC_RESET(TSECB,     0x970, 0x974, 13),
+};
+
 const struct tegra_mc_soc tegra210_mc_soc = {
 	.clients = tegra210_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra210_mc_clients),
@@ -1092,4 +1129,10 @@ const struct tegra_mc_soc tegra210_mc_soc = {
 	.atom_size = 64,
 	.client_id_mask = 0xff,
 	.smmu = &tegra210_smmu_soc,
+	.intmask = MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
+		   MC_INT_INVALID_APB_ASID_UPDATE | MC_INT_INVALID_SMMU_PAGE |
+		   MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	.reset_ops = &terga_mc_reset_ops_common,
+	.resets = tegra210_mc_resets,
+	.num_resets = ARRAY_SIZE(tegra210_mc_resets),
 };
diff --git a/drivers/memory/tegra/tegra30.c b/drivers/memory/tegra/tegra30.c
index d756c837f23e..bee5314ed404 100644
--- a/drivers/memory/tegra/tegra30.c
+++ b/drivers/memory/tegra/tegra30.c
@@ -960,6 +960,36 @@ static const struct tegra_smmu_soc tegra30_smmu_soc = {
 	.num_asids = 4,
 };
 
+#define TEGRA30_MC_RESET(_name, _control, _status, _bit)	\
+	{							\
+		.name = #_name,					\
+		.id = TEGRA30_MC_RESET_##_name,			\
+		.control = _control,				\
+		.status = _status,				\
+		.bit = _bit,					\
+	}
+
+static const struct tegra_mc_reset tegra30_mc_resets[] = {
+	TEGRA30_MC_RESET(AFI,      0x200, 0x204,  0),
+	TEGRA30_MC_RESET(AVPC,     0x200, 0x204,  1),
+	TEGRA30_MC_RESET(DC,       0x200, 0x204,  2),
+	TEGRA30_MC_RESET(DCB,      0x200, 0x204,  3),
+	TEGRA30_MC_RESET(EPP,      0x200, 0x204,  4),
+	TEGRA30_MC_RESET(2D,       0x200, 0x204,  5),
+	TEGRA30_MC_RESET(HC,       0x200, 0x204,  6),
+	TEGRA30_MC_RESET(HDA,      0x200, 0x204,  7),
+	TEGRA30_MC_RESET(ISP,      0x200, 0x204,  8),
+	TEGRA30_MC_RESET(MPCORE,   0x200, 0x204,  9),
+	TEGRA30_MC_RESET(MPCORELP, 0x200, 0x204, 10),
+	TEGRA30_MC_RESET(MPE,      0x200, 0x204, 11),
+	TEGRA30_MC_RESET(3D,       0x200, 0x204, 12),
+	TEGRA30_MC_RESET(3D2,      0x200, 0x204, 13),
+	TEGRA30_MC_RESET(PPCS,     0x200, 0x204, 14),
+	TEGRA30_MC_RESET(SATA,     0x200, 0x204, 15),
+	TEGRA30_MC_RESET(VDE,      0x200, 0x204, 16),
+	TEGRA30_MC_RESET(VI,       0x200, 0x204, 17),
+};
+
 const struct tegra_mc_soc tegra30_mc_soc = {
 	.clients = tegra30_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra30_mc_clients),
@@ -967,4 +997,9 @@ const struct tegra_mc_soc tegra30_mc_soc = {
 	.atom_size = 16,
 	.client_id_mask = 0x7f,
 	.smmu = &tegra30_smmu_soc,
+	.intmask = MC_INT_INVALID_SMMU_PAGE | MC_INT_SECURITY_VIOLATION |
+		   MC_INT_DECERR_EMEM,
+	.reset_ops = &terga_mc_reset_ops_common,
+	.resets = tegra30_mc_resets,
+	.num_resets = ARRAY_SIZE(tegra30_mc_resets),
 };
diff --git a/drivers/memory/tegra20-mc.c b/drivers/memory/tegra20-mc.c
deleted file mode 100644
index cc309a05289a..000000000000
--- a/drivers/memory/tegra20-mc.c
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Tegra20 Memory Controller
- *
- * Copyright (c) 2012, NVIDIA CORPORATION.  All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
- */
-
-#include <linux/err.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/ratelimit.h>
-#include <linux/platform_device.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-
-#define DRV_NAME "tegra20-mc"
-
-#define MC_INTSTATUS			0x0
-#define MC_INTMASK			0x4
-
-#define MC_INT_ERR_SHIFT		6
-#define MC_INT_ERR_MASK			(0x1f << MC_INT_ERR_SHIFT)
-#define MC_INT_DECERR_EMEM		BIT(MC_INT_ERR_SHIFT)
-#define MC_INT_INVALID_GART_PAGE	BIT(MC_INT_ERR_SHIFT + 1)
-#define MC_INT_SECURITY_VIOLATION	BIT(MC_INT_ERR_SHIFT + 2)
-#define MC_INT_ARBITRATION_EMEM		BIT(MC_INT_ERR_SHIFT + 3)
-
-#define MC_GART_ERROR_REQ		0x30
-#define MC_DECERR_EMEM_OTHERS_STATUS	0x58
-#define MC_SECURITY_VIOLATION_STATUS	0x74
-
-#define SECURITY_VIOLATION_TYPE		BIT(30)	/* 0=TRUSTZONE, 1=CARVEOUT */
-
-#define MC_CLIENT_ID_MASK		0x3f
-
-#define NUM_MC_REG_BANKS		2
-
-struct tegra20_mc {
-	void __iomem *regs[NUM_MC_REG_BANKS];
-	struct device *dev;
-};
-
-static inline u32 mc_readl(struct tegra20_mc *mc, u32 offs)
-{
-	u32 val = 0;
-
-	if (offs < 0x24)
-		val = readl(mc->regs[0] + offs);
-	else if (offs < 0x400)
-		val = readl(mc->regs[1] + offs - 0x3c);
-
-	return val;
-}
-
-static inline void mc_writel(struct tegra20_mc *mc, u32 val, u32 offs)
-{
-	if (offs < 0x24)
-		writel(val, mc->regs[0] + offs);
-	else if (offs < 0x400)
-		writel(val, mc->regs[1] + offs - 0x3c);
-}
-
-static const char * const tegra20_mc_client[] = {
-	"cbr_display0a",
-	"cbr_display0ab",
-	"cbr_display0b",
-	"cbr_display0bb",
-	"cbr_display0c",
-	"cbr_display0cb",
-	"cbr_display1b",
-	"cbr_display1bb",
-	"cbr_eppup",
-	"cbr_g2pr",
-	"cbr_g2sr",
-	"cbr_mpeunifbr",
-	"cbr_viruv",
-	"csr_avpcarm7r",
-	"csr_displayhc",
-	"csr_displayhcb",
-	"csr_fdcdrd",
-	"csr_g2dr",
-	"csr_host1xdmar",
-	"csr_host1xr",
-	"csr_idxsrd",
-	"csr_mpcorer",
-	"csr_mpe_ipred",
-	"csr_mpeamemrd",
-	"csr_mpecsrd",
-	"csr_ppcsahbdmar",
-	"csr_ppcsahbslvr",
-	"csr_texsrd",
-	"csr_vdebsevr",
-	"csr_vdember",
-	"csr_vdemcer",
-	"csr_vdetper",
-	"cbw_eppu",
-	"cbw_eppv",
-	"cbw_eppy",
-	"cbw_mpeunifbw",
-	"cbw_viwsb",
-	"cbw_viwu",
-	"cbw_viwv",
-	"cbw_viwy",
-	"ccw_g2dw",
-	"csw_avpcarm7w",
-	"csw_fdcdwr",
-	"csw_host1xw",
-	"csw_ispw",
-	"csw_mpcorew",
-	"csw_mpecswr",
-	"csw_ppcsahbdmaw",
-	"csw_ppcsahbslvw",
-	"csw_vdebsevw",
-	"csw_vdembew",
-	"csw_vdetpmw",
-};
-
-static void tegra20_mc_decode(struct tegra20_mc *mc, int n)
-{
-	u32 addr, req;
-	const char *client = "Unknown";
-	int idx, cid;
-	const struct reg_info {
-		u32 offset;
-		u32 write_bit;	/* 0=READ, 1=WRITE */
-		int cid_shift;
-		char *message;
-	} reg[] = {
-		{
-			.offset = MC_DECERR_EMEM_OTHERS_STATUS,
-			.write_bit = 31,
-			.message = "MC_DECERR",
-		},
-		{
-			.offset	= MC_GART_ERROR_REQ,
-			.cid_shift = 1,
-			.message = "MC_GART_ERR",
-
-		},
-		{
-			.offset = MC_SECURITY_VIOLATION_STATUS,
-			.write_bit = 31,
-			.message = "MC_SECURITY_ERR",
-		},
-	};
-
-	idx = n - MC_INT_ERR_SHIFT;
-	if ((idx < 0) || (idx >= ARRAY_SIZE(reg))) {
-		dev_err_ratelimited(mc->dev, "Unknown interrupt status %08lx\n",
-				    BIT(n));
-		return;
-	}
-
-	req = mc_readl(mc, reg[idx].offset);
-	cid = (req >> reg[idx].cid_shift) & MC_CLIENT_ID_MASK;
-	if (cid < ARRAY_SIZE(tegra20_mc_client))
-		client = tegra20_mc_client[cid];
-
-	addr = mc_readl(mc, reg[idx].offset + sizeof(u32));
-
-	dev_err_ratelimited(mc->dev, "%s (0x%08x): 0x%08x %s (%s %s)\n",
-			   reg[idx].message, req, addr, client,
-			   (req & BIT(reg[idx].write_bit)) ? "write" : "read",
-			   (reg[idx].offset == MC_SECURITY_VIOLATION_STATUS) ?
-			   ((req & SECURITY_VIOLATION_TYPE) ?
-			    "carveout" : "trustzone") : "");
-}
-
-static const struct of_device_id tegra20_mc_of_match[] = {
-	{ .compatible = "nvidia,tegra20-mc", },
-	{},
-};
-
-static irqreturn_t tegra20_mc_isr(int irq, void *data)
-{
-	u32 stat, mask, bit;
-	struct tegra20_mc *mc = data;
-
-	stat = mc_readl(mc, MC_INTSTATUS);
-	mask = mc_readl(mc, MC_INTMASK);
-	mask &= stat;
-	if (!mask)
-		return IRQ_NONE;
-	while ((bit = ffs(mask)) != 0) {
-		tegra20_mc_decode(mc, bit - 1);
-		mask &= ~BIT(bit - 1);
-	}
-
-	mc_writel(mc, stat, MC_INTSTATUS);
-	return IRQ_HANDLED;
-}
-
-static int tegra20_mc_probe(struct platform_device *pdev)
-{
-	struct resource *irq;
-	struct tegra20_mc *mc;
-	int i, err;
-	u32 intmask;
-
-	mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
-	if (!mc)
-		return -ENOMEM;
-	mc->dev = &pdev->dev;
-
-	for (i = 0; i < ARRAY_SIZE(mc->regs); i++) {
-		struct resource *res;
-
-		res = platform_get_resource(pdev, IORESOURCE_MEM, i);
-		mc->regs[i] = devm_ioremap_resource(&pdev->dev, res);
-		if (IS_ERR(mc->regs[i]))
-			return PTR_ERR(mc->regs[i]);
-	}
-
-	irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	if (!irq)
-		return -ENODEV;
-	err = devm_request_irq(&pdev->dev, irq->start, tegra20_mc_isr,
-			       IRQF_SHARED, dev_name(&pdev->dev), mc);
-	if (err)
-		return -ENODEV;
-
-	platform_set_drvdata(pdev, mc);
-
-	intmask = MC_INT_INVALID_GART_PAGE |
-		MC_INT_DECERR_EMEM | MC_INT_SECURITY_VIOLATION;
-	mc_writel(mc, intmask, MC_INTMASK);
-	return 0;
-}
-
-static struct platform_driver tegra20_mc_driver = {
-	.probe = tegra20_mc_probe,
-	.driver = {
-		.name = DRV_NAME,
-		.of_match_table = tegra20_mc_of_match,
-	},
-};
-module_platform_driver(tegra20_mc_driver);
-
-MODULE_AUTHOR("Hiroshi DOYU <hdoyu@nvidia.com>");
-MODULE_DESCRIPTION("Tegra20 MC driver");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("platform:" DRV_NAME);
diff --git a/drivers/memory/ti-aemif.c b/drivers/memory/ti-aemif.c
index 2744b1b91b57..31112f622b88 100644
--- a/drivers/memory/ti-aemif.c
+++ b/drivers/memory/ti-aemif.c
@@ -339,9 +339,6 @@ static int aemif_probe(struct platform_device *pdev)
 	struct aemif_platform_data *pdata;
 	struct of_dev_auxdata *dev_lookup;
 
-	if (np == NULL)
-		return 0;
-
 	aemif = devm_kzalloc(dev, sizeof(*aemif), GFP_KERNEL);
 	if (!aemif)
 		return -ENOMEM;
@@ -363,8 +360,10 @@ static int aemif_probe(struct platform_device *pdev)
 
 	aemif->clk_rate = clk_get_rate(aemif->clk) / MSEC_PER_SEC;
 
-	if (of_device_is_compatible(np, "ti,da850-aemif"))
+	if (np && of_device_is_compatible(np, "ti,da850-aemif"))
 		aemif->cs_offset = 2;
+	else if (pdata)
+		aemif->cs_offset = pdata->cs_offset;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	aemif->base = devm_ioremap_resource(dev, res);
@@ -373,15 +372,23 @@ static int aemif_probe(struct platform_device *pdev)
 		goto error;
 	}
 
-	/*
-	 * For every controller device node, there is a cs device node that
-	 * describe the bus configuration parameters. This functions iterate
-	 * over these nodes and update the cs data array.
-	 */
-	for_each_available_child_of_node(np, child_np) {
-		ret = of_aemif_parse_abus_config(pdev, child_np);
-		if (ret < 0)
-			goto error;
+	if (np) {
+		/*
+		 * For every controller device node, there is a cs device node
+		 * that describe the bus configuration parameters. This
+		 * functions iterate over these nodes and update the cs data
+		 * array.
+		 */
+		for_each_available_child_of_node(np, child_np) {
+			ret = of_aemif_parse_abus_config(pdev, child_np);
+			if (ret < 0)
+				goto error;
+		}
+	} else if (pdata && pdata->num_abus_data > 0) {
+		for (i = 0; i < pdata->num_abus_data; i++, aemif->num_cs++) {
+			aemif->cs_data[i].cs = pdata->abus_data[i].cs;
+			aemif_get_hw_params(pdev, i);
+		}
 	}
 
 	for (i = 0; i < aemif->num_cs; i++) {
@@ -394,14 +401,25 @@ static int aemif_probe(struct platform_device *pdev)
 	}
 
 	/*
-	 * Create a child devices explicitly from here to
-	 * guarantee that the child will be probed after the AEMIF timing
-	 * parameters are set.
+	 * Create a child devices explicitly from here to guarantee that the
+	 * child will be probed after the AEMIF timing parameters are set.
 	 */
-	for_each_available_child_of_node(np, child_np) {
-		ret = of_platform_populate(child_np, NULL, dev_lookup, dev);
-		if (ret < 0)
-			goto error;
+	if (np) {
+		for_each_available_child_of_node(np, child_np) {
+			ret = of_platform_populate(child_np, NULL,
+						   dev_lookup, dev);
+			if (ret < 0)
+				goto error;
+		}
+	} else {
+		for (i = 0; i < pdata->num_sub_devices; i++) {
+			pdata->sub_devices[i].dev.parent = dev;
+			ret = platform_device_register(&pdata->sub_devices[i]);
+			if (ret) {
+				dev_warn(dev, "Error register sub device %s\n",
+					 pdata->sub_devices[i].name);
+			}
+		}
 	}
 
 	return 0;
@@ -422,7 +440,7 @@ static struct platform_driver aemif_driver = {
 	.probe = aemif_probe,
 	.remove = aemif_remove,
 	.driver = {
-		.name = KBUILD_MODNAME,
+		.name = "ti-aemif",
 		.of_match_table = of_match_ptr(aemif_of_match),
 	},
 };
diff --git a/drivers/memstick/core/ms_block.c b/drivers/memstick/core/ms_block.c
index 57b13dfbd21e..a15181fa45f7 100644
--- a/drivers/memstick/core/ms_block.c
+++ b/drivers/memstick/core/ms_block.c
@@ -2094,14 +2094,9 @@ static const struct block_device_operations msb_bdops = {
 static int msb_init_disk(struct memstick_dev *card)
 {
 	struct msb_data *msb = memstick_get_drvdata(card);
-	struct memstick_host *host = card->host;
 	int rc;
-	u64 limit = BLK_BOUNCE_HIGH;
 	unsigned long capacity;
 
-	if (host->dev.dma_mask && *(host->dev.dma_mask))
-		limit = *(host->dev.dma_mask);
-
 	mutex_lock(&msb_disk_lock);
 	msb->disk_id = idr_alloc(&msb_disk_idr, card, 0, 256, GFP_KERNEL);
 	mutex_unlock(&msb_disk_lock);
@@ -2123,7 +2118,6 @@ static int msb_init_disk(struct memstick_dev *card)
 
 	msb->queue->queuedata = card;
 
-	blk_queue_bounce_limit(msb->queue, limit);
 	blk_queue_max_hw_sectors(msb->queue, MS_BLOCK_MAX_PAGES);
 	blk_queue_max_segments(msb->queue, MS_BLOCK_MAX_SEGS);
 	blk_queue_max_segment_size(msb->queue,
diff --git a/drivers/memstick/core/mspro_block.c b/drivers/memstick/core/mspro_block.c
index 8897962781bb..5ee932631fae 100644
--- a/drivers/memstick/core/mspro_block.c
+++ b/drivers/memstick/core/mspro_block.c
@@ -1170,17 +1170,12 @@ static int mspro_block_init_card(struct memstick_dev *card)
 static int mspro_block_init_disk(struct memstick_dev *card)
 {
 	struct mspro_block_data *msb = memstick_get_drvdata(card);
-	struct memstick_host *host = card->host;
 	struct mspro_devinfo *dev_info = NULL;
 	struct mspro_sys_info *sys_info = NULL;
 	struct mspro_sys_attr *s_attr = NULL;
 	int rc, disk_id;
-	u64 limit = BLK_BOUNCE_HIGH;
 	unsigned long capacity;
 
-	if (host->dev.dma_mask && *(host->dev.dma_mask))
-		limit = *(host->dev.dma_mask);
-
 	for (rc = 0; msb->attr_group.attrs[rc]; ++rc) {
 		s_attr = mspro_from_sysfs_attr(msb->attr_group.attrs[rc]);
 
@@ -1219,7 +1214,6 @@ static int mspro_block_init_disk(struct memstick_dev *card)
 
 	msb->queue->queuedata = card;
 
-	blk_queue_bounce_limit(msb->queue, limit);
 	blk_queue_max_hw_sectors(msb->queue, MSPRO_BLOCK_MAX_PAGES);
 	blk_queue_max_segments(msb->queue, MSPRO_BLOCK_MAX_SEGS);
 	blk_queue_max_segment_size(msb->queue,
diff --git a/drivers/message/fusion/lsi/mpi_cnfg.h b/drivers/message/fusion/lsi/mpi_cnfg.h
index 4e9c0ce94f27..059997f8ebce 100644
--- a/drivers/message/fusion/lsi/mpi_cnfg.h
+++ b/drivers/message/fusion/lsi/mpi_cnfg.h
@@ -1802,13 +1802,13 @@ typedef struct _CONFIG_PAGE_FC_PORT_0
 #define MPI_FCPORTPAGE0_SUPPORT_CLASS_2                 (0x00000002)
 #define MPI_FCPORTPAGE0_SUPPORT_CLASS_3                 (0x00000004)
 
-#define MPI_FCPORTPAGE0_SUPPORT_SPEED_UKNOWN            (0x00000000) /* (SNIA)HBA_PORTSPEED_UNKNOWN 0   Unknown - transceiver incapable of reporting */
+#define MPI_FCPORTPAGE0_SUPPORT_SPEED_UNKNOWN           (0x00000000) /* (SNIA)HBA_PORTSPEED_UNKNOWN 0   Unknown - transceiver incapable of reporting */
 #define MPI_FCPORTPAGE0_SUPPORT_1GBIT_SPEED             (0x00000001) /* (SNIA)HBA_PORTSPEED_1GBIT   1   1 GBit/sec */
 #define MPI_FCPORTPAGE0_SUPPORT_2GBIT_SPEED             (0x00000002) /* (SNIA)HBA_PORTSPEED_2GBIT   2   2 GBit/sec */
 #define MPI_FCPORTPAGE0_SUPPORT_10GBIT_SPEED            (0x00000004) /* (SNIA)HBA_PORTSPEED_10GBIT  4  10 GBit/sec */
 #define MPI_FCPORTPAGE0_SUPPORT_4GBIT_SPEED             (0x00000008) /* (SNIA)HBA_PORTSPEED_4GBIT   8   4 GBit/sec */
 
-#define MPI_FCPORTPAGE0_CURRENT_SPEED_UKNOWN            MPI_FCPORTPAGE0_SUPPORT_SPEED_UKNOWN
+#define MPI_FCPORTPAGE0_CURRENT_SPEED_UNKNOWN           MPI_FCPORTPAGE0_SUPPORT_SPEED_UNKNOWN
 #define MPI_FCPORTPAGE0_CURRENT_SPEED_1GBIT             MPI_FCPORTPAGE0_SUPPORT_1GBIT_SPEED
 #define MPI_FCPORTPAGE0_CURRENT_SPEED_2GBIT             MPI_FCPORTPAGE0_SUPPORT_2GBIT_SPEED
 #define MPI_FCPORTPAGE0_CURRENT_SPEED_10GBIT            MPI_FCPORTPAGE0_SUPPORT_10GBIT_SPEED
diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c
index 51eb1b027963..a625ac4e2872 100644
--- a/drivers/message/fusion/mptbase.c
+++ b/drivers/message/fusion/mptbase.c
@@ -197,9 +197,9 @@ static int	mpt_host_page_access_control(MPT_ADAPTER *ioc, u8 access_control_valu
 static int	mpt_host_page_alloc(MPT_ADAPTER *ioc, pIOCInit_t ioc_init);
 
 #ifdef CONFIG_PROC_FS
-static const struct file_operations mpt_summary_proc_fops;
-static const struct file_operations mpt_version_proc_fops;
-static const struct file_operations mpt_iocinfo_proc_fops;
+static int mpt_summary_proc_show(struct seq_file *m, void *v);
+static int mpt_version_proc_show(struct seq_file *m, void *v);
+static int mpt_iocinfo_proc_show(struct seq_file *m, void *v);
 #endif
 static void	mpt_get_fw_exp_ver(char *buf, MPT_ADAPTER *ioc);
 
@@ -2040,8 +2040,10 @@ mpt_attach(struct pci_dev *pdev, const struct pci_device_id *id)
 	 */
 	dent = proc_mkdir(ioc->name, mpt_proc_root_dir);
 	if (dent) {
-		proc_create_data("info", S_IRUGO, dent, &mpt_iocinfo_proc_fops, ioc);
-		proc_create_data("summary", S_IRUGO, dent, &mpt_summary_proc_fops, ioc);
+		proc_create_single_data("info", S_IRUGO, dent,
+				mpt_iocinfo_proc_show, ioc);
+		proc_create_single_data("summary", S_IRUGO, dent,
+				mpt_summary_proc_show, ioc);
 	}
 #endif
 
@@ -6606,8 +6608,10 @@ procmpt_create(void)
 	if (mpt_proc_root_dir == NULL)
 		return -ENOTDIR;
 
-	proc_create("summary", S_IRUGO, mpt_proc_root_dir, &mpt_summary_proc_fops);
-	proc_create("version", S_IRUGO, mpt_proc_root_dir, &mpt_version_proc_fops);
+	proc_create_single("summary", S_IRUGO, mpt_proc_root_dir,
+			mpt_summary_proc_show);
+	proc_create_single("version", S_IRUGO, mpt_proc_root_dir,
+			mpt_version_proc_show);
 	return 0;
 }
 
@@ -6646,19 +6650,6 @@ static int mpt_summary_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int mpt_summary_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, mpt_summary_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations mpt_summary_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= mpt_summary_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int mpt_version_proc_show(struct seq_file *m, void *v)
 {
 	u8	 cb_idx;
@@ -6701,19 +6692,6 @@ static int mpt_version_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int mpt_version_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, mpt_version_proc_show, NULL);
-}
-
-static const struct file_operations mpt_version_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= mpt_version_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int mpt_iocinfo_proc_show(struct seq_file *m, void *v)
 {
 	MPT_ADAPTER	*ioc = m->private;
@@ -6793,19 +6771,6 @@ static int mpt_iocinfo_proc_show(struct seq_file *m, void *v)
 
 	return 0;
 }
-
-static int mpt_iocinfo_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, mpt_iocinfo_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations mpt_iocinfo_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= mpt_iocinfo_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
 #endif		/* CONFIG_PROC_FS } */
 
 /*=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=*/
@@ -7635,7 +7600,7 @@ mpt_display_event_info(MPT_ADAPTER *ioc, EventNotificationReply_t *pEventReply)
 
 		snprintf(evStr, EVENT_DESCR_STR_SZ,
 		    "SAS Initiator Device Table Overflow: max initiators=%02d "
-		    "current initators=%02d",
+		    "current initiators=%02d",
 		    max_init, current_init);
 		break;
 	}
diff --git a/drivers/message/fusion/mptfc.c b/drivers/message/fusion/mptfc.c
index 6d461ca97150..06b175420be9 100644
--- a/drivers/message/fusion/mptfc.c
+++ b/drivers/message/fusion/mptfc.c
@@ -693,7 +693,7 @@ mptfc_display_port_link_speed(MPT_ADAPTER *ioc, int portnum, FCPortPage0_t *pp0d
 	state = pp0dest->PortState;
 
 	if (state != MPI_FCPORTPAGE0_PORTSTATE_OFFLINE &&
-	    new_speed != MPI_FCPORTPAGE0_CURRENT_SPEED_UKNOWN) {
+	    new_speed != MPI_FCPORTPAGE0_CURRENT_SPEED_UNKNOWN) {
 
 		old = old_speed == MPI_FCPORTPAGE0_CURRENT_SPEED_1GBIT ? "1 Gbps" :
 		       old_speed == MPI_FCPORTPAGE0_CURRENT_SPEED_2GBIT ? "2 Gbps" :
diff --git a/drivers/message/fusion/mptlan.c b/drivers/message/fusion/mptlan.c
index 55dd71bbdc2a..4cbed4d06aa7 100644
--- a/drivers/message/fusion/mptlan.c
+++ b/drivers/message/fusion/mptlan.c
@@ -670,7 +670,7 @@ mpt_lan_send_reply(struct net_device *dev, LANSendReply_t *pSendRep)
 }
 
 /*=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=*/
-static int
+static netdev_tx_t
 mpt_lan_sdu_send (struct sk_buff *skb, struct net_device *dev)
 {
 	struct mpt_lan_priv *priv = netdev_priv(dev);
diff --git a/drivers/message/fusion/mptsas.c b/drivers/message/fusion/mptsas.c
index 86503f60468f..76a66da33996 100644
--- a/drivers/message/fusion/mptsas.c
+++ b/drivers/message/fusion/mptsas.c
@@ -1929,7 +1929,7 @@ static enum blk_eh_timer_return mptsas_eh_timed_out(struct scsi_cmnd *sc)
 	MPT_SCSI_HOST *hd;
 	MPT_ADAPTER   *ioc;
 	VirtDevice    *vdevice;
-	enum blk_eh_timer_return rc = BLK_EH_NOT_HANDLED;
+	enum blk_eh_timer_return rc = BLK_EH_DONE;
 
 	hd = shost_priv(sc->device->host);
 	if (hd == NULL) {
@@ -4320,7 +4320,7 @@ mptsas_hotplug_work(MPT_ADAPTER *ioc, struct fw_event_work *fw_event,
 			if (ioc->raid_data.pIocPg2->RaidVolume[i].VolumeID ==
 			    hot_plug_info->id) {
 				printk(MYIOC_s_WARN_FMT "firmware bug: unable "
-				    "to add hidden disk - target_id matchs "
+				    "to add hidden disk - target_id matches "
 				    "volume_id\n", ioc->name);
 				mptsas_free_fw_event(ioc, fw_event);
 				return;
diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile
index d9d2cf0d32ef..e9fd20dba18d 100644
--- a/drivers/mfd/Makefile
+++ b/drivers/mfd/Makefile
@@ -13,7 +13,6 @@ obj-$(CONFIG_MFD_ASIC3)		+= asic3.o tmio_core.o
 obj-$(CONFIG_MFD_BCM590XX)	+= bcm590xx.o
 obj-$(CONFIG_MFD_BD9571MWV)	+= bd9571mwv.o
 cros_ec_core-objs		:= cros_ec.o
-cros_ec_core-$(CONFIG_ACPI)	+= cros_ec_acpi_gpe.o
 obj-$(CONFIG_MFD_CROS_EC)	+= cros_ec_core.o
 obj-$(CONFIG_MFD_CROS_EC_I2C)	+= cros_ec_i2c.o
 obj-$(CONFIG_MFD_CROS_EC_SPI)	+= cros_ec_spi.o
diff --git a/drivers/mfd/ab8500-debugfs.c b/drivers/mfd/ab8500-debugfs.c
index 8ba41073dd89..831a1ceb2ed2 100644
--- a/drivers/mfd/ab8500-debugfs.c
+++ b/drivers/mfd/ab8500-debugfs.c
@@ -2519,11 +2519,10 @@ static ssize_t ab8500_subscribe_write(struct file *file,
 	if (!dev_attr[irq_index])
 		return -ENOMEM;
 
-	event_name[irq_index] = kmalloc(count, GFP_KERNEL);
+	event_name[irq_index] = kasprintf(GFP_KERNEL, "%lu", user_val);
 	if (!event_name[irq_index])
 		return -ENOMEM;
 
-	sprintf(event_name[irq_index], "%lu", user_val);
 	dev_attr[irq_index]->show = show_irq;
 	dev_attr[irq_index]->store = NULL;
 	dev_attr[irq_index]->attr.name = event_name[irq_index];
diff --git a/drivers/mfd/abx500-core.c b/drivers/mfd/abx500-core.c
index 0d3846a4767c..f282d39a5917 100644
--- a/drivers/mfd/abx500-core.c
+++ b/drivers/mfd/abx500-core.c
@@ -37,15 +37,12 @@ int abx500_register_ops(struct device *dev, struct abx500_ops *ops)
 {
 	struct abx500_device_entry *dev_entry;
 
-	dev_entry = devm_kzalloc(dev,
-				 sizeof(struct abx500_device_entry),
-				 GFP_KERNEL);
-	if (!dev_entry) {
-		dev_err(dev, "register_ops kzalloc failed");
+	dev_entry = devm_kzalloc(dev, sizeof(*dev_entry), GFP_KERNEL);
+	if (!dev_entry)
 		return -ENOMEM;
-	}
+
 	dev_entry->dev = dev;
-	memcpy(&dev_entry->ops, ops, sizeof(struct abx500_ops));
+	memcpy(&dev_entry->ops, ops, sizeof(*ops));
 
 	list_add_tail(&dev_entry->list, &abx500_list);
 	return 0;
@@ -68,7 +65,7 @@ int abx500_set_register_interruptible(struct device *dev, u8 bank, u8 reg,
 	struct abx500_ops *ops;
 
 	lookup_ops(dev->parent, &ops);
-	if ((ops != NULL) && (ops->set_register != NULL))
+	if (ops && ops->set_register)
 		return ops->set_register(dev, bank, reg, value);
 	else
 		return -ENOTSUPP;
@@ -81,7 +78,7 @@ int abx500_get_register_interruptible(struct device *dev, u8 bank, u8 reg,
 	struct abx500_ops *ops;
 
 	lookup_ops(dev->parent, &ops);
-	if ((ops != NULL) && (ops->get_register != NULL))
+	if (ops && ops->get_register)
 		return ops->get_register(dev, bank, reg, value);
 	else
 		return -ENOTSUPP;
@@ -94,7 +91,7 @@ int abx500_get_register_page_interruptible(struct device *dev, u8 bank,
 	struct abx500_ops *ops;
 
 	lookup_ops(dev->parent, &ops);
-	if ((ops != NULL) && (ops->get_register_page != NULL))
+	if (ops && ops->get_register_page)
 		return ops->get_register_page(dev, bank,
 			first_reg, regvals, numregs);
 	else
@@ -108,7 +105,7 @@ int abx500_mask_and_set_register_interruptible(struct device *dev, u8 bank,
 	struct abx500_ops *ops;
 
 	lookup_ops(dev->parent, &ops);
-	if ((ops != NULL) && (ops->mask_and_set_register != NULL))
+	if (ops && ops->mask_and_set_register)
 		return ops->mask_and_set_register(dev, bank,
 			reg, bitmask, bitvalues);
 	else
@@ -121,7 +118,7 @@ int abx500_get_chip_id(struct device *dev)
 	struct abx500_ops *ops;
 
 	lookup_ops(dev->parent, &ops);
-	if ((ops != NULL) && (ops->get_chip_id != NULL))
+	if (ops && ops->get_chip_id)
 		return ops->get_chip_id(dev);
 	else
 		return -ENOTSUPP;
@@ -133,7 +130,7 @@ int abx500_event_registers_startup_state_get(struct device *dev, u8 *event)
 	struct abx500_ops *ops;
 
 	lookup_ops(dev->parent, &ops);
-	if ((ops != NULL) && (ops->event_registers_startup_state_get != NULL))
+	if (ops && ops->event_registers_startup_state_get)
 		return ops->event_registers_startup_state_get(dev, event);
 	else
 		return -ENOTSUPP;
@@ -145,7 +142,7 @@ int abx500_startup_irq_enabled(struct device *dev, unsigned int irq)
 	struct abx500_ops *ops;
 
 	lookup_ops(dev->parent, &ops);
-	if ((ops != NULL) && (ops->startup_irq_enabled != NULL))
+	if (ops && ops->startup_irq_enabled)
 		return ops->startup_irq_enabled(dev, irq);
 	else
 		return -ENOTSUPP;
diff --git a/drivers/mfd/arizona-core.c b/drivers/mfd/arizona-core.c
index 77875250abe5..83f1c5a516d9 100644
--- a/drivers/mfd/arizona-core.c
+++ b/drivers/mfd/arizona-core.c
@@ -13,13 +13,12 @@
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/err.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/mfd/core.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
-#include <linux/of_gpio.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
@@ -279,7 +278,7 @@ static int arizona_wait_for_boot(struct arizona *arizona)
 static inline void arizona_enable_reset(struct arizona *arizona)
 {
 	if (arizona->pdata.reset)
-		gpio_set_value_cansleep(arizona->pdata.reset, 0);
+		gpiod_set_raw_value_cansleep(arizona->pdata.reset, 0);
 }
 
 static void arizona_disable_reset(struct arizona *arizona)
@@ -295,7 +294,7 @@ static void arizona_disable_reset(struct arizona *arizona)
 			break;
 		}
 
-		gpio_set_value_cansleep(arizona->pdata.reset, 1);
+		gpiod_set_raw_value_cansleep(arizona->pdata.reset, 1);
 		usleep_range(1000, 5000);
 	}
 }
@@ -799,14 +798,27 @@ static int arizona_of_get_core_pdata(struct arizona *arizona)
 	struct arizona_pdata *pdata = &arizona->pdata;
 	int ret, i;
 
-	pdata->reset = of_get_named_gpio(arizona->dev->of_node, "wlf,reset", 0);
-	if (pdata->reset == -EPROBE_DEFER) {
-		return pdata->reset;
-	} else if (pdata->reset < 0) {
-		dev_err(arizona->dev, "Reset GPIO missing/malformed: %d\n",
-			pdata->reset);
+	/* Handle old non-standard DT binding */
+	pdata->reset = devm_gpiod_get_from_of_node(arizona->dev,
+						   arizona->dev->of_node,
+						   "wlf,reset", 0,
+						   GPIOD_OUT_LOW,
+						   "arizona /RESET");
+	if (IS_ERR(pdata->reset)) {
+		ret = PTR_ERR(pdata->reset);
 
-		pdata->reset = 0;
+		/*
+		 * Reset missing will be caught when other binding is read
+		 * but all other errors imply this binding is in use but has
+		 * encountered a problem so should be handled.
+		 */
+		if (ret == -EPROBE_DEFER)
+			return ret;
+		else if (ret != -ENOENT && ret != -ENOSYS)
+			dev_err(arizona->dev, "Reset GPIO malformed: %d\n",
+				ret);
+
+		pdata->reset = NULL;
 	}
 
 	ret = of_property_read_u32_array(arizona->dev->of_node,
@@ -1050,14 +1062,19 @@ int arizona_dev_init(struct arizona *arizona)
 		goto err_early;
 	}
 
-	if (arizona->pdata.reset) {
+	if (!arizona->pdata.reset) {
 		/* Start out with /RESET low to put the chip into reset */
-		ret = devm_gpio_request_one(arizona->dev, arizona->pdata.reset,
-					    GPIOF_DIR_OUT | GPIOF_INIT_LOW,
-					    "arizona /RESET");
-		if (ret != 0) {
-			dev_err(dev, "Failed to request /RESET: %d\n", ret);
-			goto err_dcvdd;
+		arizona->pdata.reset = devm_gpiod_get(arizona->dev, "reset",
+						      GPIOD_OUT_LOW);
+		if (IS_ERR(arizona->pdata.reset)) {
+			ret = PTR_ERR(arizona->pdata.reset);
+			if (ret == -EPROBE_DEFER)
+				goto err_dcvdd;
+
+			dev_err(arizona->dev,
+				"Reset GPIO missing/malformed: %d\n", ret);
+
+			arizona->pdata.reset = NULL;
 		}
 	}
 
diff --git a/drivers/mfd/atmel-smc.c b/drivers/mfd/atmel-smc.c
index 7d77948567d7..0adbd2e796fe 100644
--- a/drivers/mfd/atmel-smc.c
+++ b/drivers/mfd/atmel-smc.c
@@ -12,6 +12,7 @@
  */
 
 #include <linux/mfd/syscon/atmel-smc.h>
+#include <linux/string.h>
 
 /**
  * atmel_smc_cs_conf_init - initialize a SMC CS conf
diff --git a/drivers/mfd/axp20x.c b/drivers/mfd/axp20x.c
index e94c72c2faa2..9a2ef3d9b8f8 100644
--- a/drivers/mfd/axp20x.c
+++ b/drivers/mfd/axp20x.c
@@ -169,131 +169,61 @@ static const struct regmap_access_table axp806_volatile_table = {
 	.n_yes_ranges	= ARRAY_SIZE(axp806_volatile_ranges),
 };
 
-static struct resource axp152_pek_resources[] = {
+static const struct resource axp152_pek_resources[] = {
 	DEFINE_RES_IRQ_NAMED(AXP152_IRQ_PEK_RIS_EDGE, "PEK_DBR"),
 	DEFINE_RES_IRQ_NAMED(AXP152_IRQ_PEK_FAL_EDGE, "PEK_DBF"),
 };
 
-static struct resource axp20x_ac_power_supply_resources[] = {
+static const struct resource axp20x_ac_power_supply_resources[] = {
 	DEFINE_RES_IRQ_NAMED(AXP20X_IRQ_ACIN_PLUGIN, "ACIN_PLUGIN"),
 	DEFINE_RES_IRQ_NAMED(AXP20X_IRQ_ACIN_REMOVAL, "ACIN_REMOVAL"),
 	DEFINE_RES_IRQ_NAMED(AXP20X_IRQ_ACIN_OVER_V, "ACIN_OVER_V"),
 };
 
-static struct resource axp20x_pek_resources[] = {
-	{
-		.name	= "PEK_DBR",
-		.start	= AXP20X_IRQ_PEK_RIS_EDGE,
-		.end	= AXP20X_IRQ_PEK_RIS_EDGE,
-		.flags	= IORESOURCE_IRQ,
-	}, {
-		.name	= "PEK_DBF",
-		.start	= AXP20X_IRQ_PEK_FAL_EDGE,
-		.end	= AXP20X_IRQ_PEK_FAL_EDGE,
-		.flags	= IORESOURCE_IRQ,
-	},
+static const struct resource axp20x_pek_resources[] = {
+	DEFINE_RES_IRQ_NAMED(AXP20X_IRQ_PEK_RIS_EDGE, "PEK_DBR"),
+	DEFINE_RES_IRQ_NAMED(AXP20X_IRQ_PEK_FAL_EDGE, "PEK_DBF"),
 };
 
-static struct resource axp20x_usb_power_supply_resources[] = {
+static const struct resource axp20x_usb_power_supply_resources[] = {
 	DEFINE_RES_IRQ_NAMED(AXP20X_IRQ_VBUS_PLUGIN, "VBUS_PLUGIN"),
 	DEFINE_RES_IRQ_NAMED(AXP20X_IRQ_VBUS_REMOVAL, "VBUS_REMOVAL"),
 	DEFINE_RES_IRQ_NAMED(AXP20X_IRQ_VBUS_VALID, "VBUS_VALID"),
 	DEFINE_RES_IRQ_NAMED(AXP20X_IRQ_VBUS_NOT_VALID, "VBUS_NOT_VALID"),
 };
 
-static struct resource axp22x_usb_power_supply_resources[] = {
+static const struct resource axp22x_usb_power_supply_resources[] = {
 	DEFINE_RES_IRQ_NAMED(AXP22X_IRQ_VBUS_PLUGIN, "VBUS_PLUGIN"),
 	DEFINE_RES_IRQ_NAMED(AXP22X_IRQ_VBUS_REMOVAL, "VBUS_REMOVAL"),
 };
 
-static struct resource axp22x_pek_resources[] = {
-	{
-		.name   = "PEK_DBR",
-		.start  = AXP22X_IRQ_PEK_RIS_EDGE,
-		.end    = AXP22X_IRQ_PEK_RIS_EDGE,
-		.flags  = IORESOURCE_IRQ,
-	}, {
-		.name   = "PEK_DBF",
-		.start  = AXP22X_IRQ_PEK_FAL_EDGE,
-		.end    = AXP22X_IRQ_PEK_FAL_EDGE,
-		.flags  = IORESOURCE_IRQ,
-	},
+static const struct resource axp22x_pek_resources[] = {
+	DEFINE_RES_IRQ_NAMED(AXP22X_IRQ_PEK_RIS_EDGE, "PEK_DBR"),
+	DEFINE_RES_IRQ_NAMED(AXP22X_IRQ_PEK_FAL_EDGE, "PEK_DBF"),
 };
 
-static struct resource axp288_power_button_resources[] = {
-	{
-		.name	= "PEK_DBR",
-		.start	= AXP288_IRQ_POKP,
-		.end	= AXP288_IRQ_POKP,
-		.flags	= IORESOURCE_IRQ,
-	},
-	{
-		.name	= "PEK_DBF",
-		.start	= AXP288_IRQ_POKN,
-		.end	= AXP288_IRQ_POKN,
-		.flags	= IORESOURCE_IRQ,
-	},
+static const struct resource axp288_power_button_resources[] = {
+	DEFINE_RES_IRQ_NAMED(AXP288_IRQ_POKP, "PEK_DBR"),
+	DEFINE_RES_IRQ_NAMED(AXP288_IRQ_POKN, "PEK_DBF"),
 };
 
-static struct resource axp288_fuel_gauge_resources[] = {
-	{
-		.start = AXP288_IRQ_QWBTU,
-		.end   = AXP288_IRQ_QWBTU,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_WBTU,
-		.end   = AXP288_IRQ_WBTU,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_QWBTO,
-		.end   = AXP288_IRQ_QWBTO,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_WBTO,
-		.end   = AXP288_IRQ_WBTO,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_WL2,
-		.end   = AXP288_IRQ_WL2,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_WL1,
-		.end   = AXP288_IRQ_WL1,
-		.flags = IORESOURCE_IRQ,
-	},
+static const struct resource axp288_fuel_gauge_resources[] = {
+	DEFINE_RES_IRQ(AXP288_IRQ_QWBTU),
+	DEFINE_RES_IRQ(AXP288_IRQ_WBTU),
+	DEFINE_RES_IRQ(AXP288_IRQ_QWBTO),
+	DEFINE_RES_IRQ(AXP288_IRQ_WBTO),
+	DEFINE_RES_IRQ(AXP288_IRQ_WL2),
+	DEFINE_RES_IRQ(AXP288_IRQ_WL1),
 };
 
-static struct resource axp803_pek_resources[] = {
-	{
-		.name   = "PEK_DBR",
-		.start  = AXP803_IRQ_PEK_RIS_EDGE,
-		.end    = AXP803_IRQ_PEK_RIS_EDGE,
-		.flags  = IORESOURCE_IRQ,
-	}, {
-		.name   = "PEK_DBF",
-		.start  = AXP803_IRQ_PEK_FAL_EDGE,
-		.end    = AXP803_IRQ_PEK_FAL_EDGE,
-		.flags  = IORESOURCE_IRQ,
-	},
+static const struct resource axp803_pek_resources[] = {
+	DEFINE_RES_IRQ_NAMED(AXP803_IRQ_PEK_RIS_EDGE, "PEK_DBR"),
+	DEFINE_RES_IRQ_NAMED(AXP803_IRQ_PEK_FAL_EDGE, "PEK_DBF"),
 };
 
-static struct resource axp809_pek_resources[] = {
-	{
-		.name   = "PEK_DBR",
-		.start  = AXP809_IRQ_PEK_RIS_EDGE,
-		.end    = AXP809_IRQ_PEK_RIS_EDGE,
-		.flags  = IORESOURCE_IRQ,
-	}, {
-		.name   = "PEK_DBF",
-		.start  = AXP809_IRQ_PEK_FAL_EDGE,
-		.end    = AXP809_IRQ_PEK_FAL_EDGE,
-		.flags  = IORESOURCE_IRQ,
-	},
+static const struct resource axp809_pek_resources[] = {
+	DEFINE_RES_IRQ_NAMED(AXP809_IRQ_PEK_RIS_EDGE, "PEK_DBR"),
+	DEFINE_RES_IRQ_NAMED(AXP809_IRQ_PEK_FAL_EDGE, "PEK_DBF"),
 };
 
 static const struct regmap_config axp152_regmap_config = {
@@ -520,11 +450,11 @@ static const struct regmap_irq axp806_regmap_irqs[] = {
 	INIT_REGMAP_IRQ(AXP806, DCDCC_V_LOW,		0, 5),
 	INIT_REGMAP_IRQ(AXP806, DCDCD_V_LOW,		0, 6),
 	INIT_REGMAP_IRQ(AXP806, DCDCE_V_LOW,		0, 7),
-	INIT_REGMAP_IRQ(AXP806, PWROK_LONG,		1, 0),
-	INIT_REGMAP_IRQ(AXP806, PWROK_SHORT,		1, 1),
+	INIT_REGMAP_IRQ(AXP806, POK_LONG,		1, 0),
+	INIT_REGMAP_IRQ(AXP806, POK_SHORT,		1, 1),
 	INIT_REGMAP_IRQ(AXP806, WAKEUP,			1, 4),
-	INIT_REGMAP_IRQ(AXP806, PWROK_FALL,		1, 5),
-	INIT_REGMAP_IRQ(AXP806, PWROK_RISE,		1, 6),
+	INIT_REGMAP_IRQ(AXP806, POK_FALL,		1, 5),
+	INIT_REGMAP_IRQ(AXP806, POK_RISE,		1, 6),
 };
 
 static const struct regmap_irq axp809_regmap_irqs[] = {
@@ -648,7 +578,7 @@ static const struct regmap_irq_chip axp809_regmap_irq_chip = {
 	.num_regs		= 5,
 };
 
-static struct mfd_cell axp20x_cells[] = {
+static const struct mfd_cell axp20x_cells[] = {
 	{
 		.name		= "axp20x-gpio",
 		.of_compatible	= "x-powers,axp209-gpio",
@@ -660,6 +590,7 @@ static struct mfd_cell axp20x_cells[] = {
 		.name		= "axp20x-regulator",
 	}, {
 		.name		= "axp20x-adc",
+		.of_compatible	= "x-powers,axp209-adc",
 	}, {
 		.name		= "axp20x-battery-power-supply",
 		.of_compatible	= "x-powers,axp209-battery-power-supply",
@@ -676,7 +607,7 @@ static struct mfd_cell axp20x_cells[] = {
 	},
 };
 
-static struct mfd_cell axp221_cells[] = {
+static const struct mfd_cell axp221_cells[] = {
 	{
 		.name		= "axp221-pek",
 		.num_resources	= ARRAY_SIZE(axp22x_pek_resources),
@@ -684,7 +615,8 @@ static struct mfd_cell axp221_cells[] = {
 	}, {
 		.name		= "axp20x-regulator",
 	}, {
-		.name		= "axp22x-adc"
+		.name		= "axp22x-adc",
+		.of_compatible	= "x-powers,axp221-adc",
 	}, {
 		.name		= "axp20x-ac-power-supply",
 		.of_compatible	= "x-powers,axp221-ac-power-supply",
@@ -701,13 +633,14 @@ static struct mfd_cell axp221_cells[] = {
 	},
 };
 
-static struct mfd_cell axp223_cells[] = {
+static const struct mfd_cell axp223_cells[] = {
 	{
 		.name			= "axp221-pek",
 		.num_resources		= ARRAY_SIZE(axp22x_pek_resources),
 		.resources		= axp22x_pek_resources,
 	}, {
 		.name		= "axp22x-adc",
+		.of_compatible	= "x-powers,axp221-adc",
 	}, {
 		.name		= "axp20x-battery-power-supply",
 		.of_compatible	= "x-powers,axp221-battery-power-supply",
@@ -726,7 +659,7 @@ static struct mfd_cell axp223_cells[] = {
 	},
 };
 
-static struct mfd_cell axp152_cells[] = {
+static const struct mfd_cell axp152_cells[] = {
 	{
 		.name			= "axp20x-pek",
 		.num_resources		= ARRAY_SIZE(axp152_pek_resources),
@@ -734,87 +667,30 @@ static struct mfd_cell axp152_cells[] = {
 	},
 };
 
-static struct resource axp288_adc_resources[] = {
-	{
-		.name  = "GPADC",
-		.start = AXP288_IRQ_GPADC,
-		.end   = AXP288_IRQ_GPADC,
-		.flags = IORESOURCE_IRQ,
-	},
+static const struct resource axp288_adc_resources[] = {
+	DEFINE_RES_IRQ_NAMED(AXP288_IRQ_GPADC, "GPADC"),
 };
 
-static struct resource axp288_extcon_resources[] = {
-	{
-		.start = AXP288_IRQ_VBUS_FALL,
-		.end   = AXP288_IRQ_VBUS_FALL,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_VBUS_RISE,
-		.end   = AXP288_IRQ_VBUS_RISE,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_MV_CHNG,
-		.end   = AXP288_IRQ_MV_CHNG,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_BC_USB_CHNG,
-		.end   = AXP288_IRQ_BC_USB_CHNG,
-		.flags = IORESOURCE_IRQ,
-	},
+static const struct resource axp288_extcon_resources[] = {
+	DEFINE_RES_IRQ(AXP288_IRQ_VBUS_FALL),
+	DEFINE_RES_IRQ(AXP288_IRQ_VBUS_RISE),
+	DEFINE_RES_IRQ(AXP288_IRQ_MV_CHNG),
+	DEFINE_RES_IRQ(AXP288_IRQ_BC_USB_CHNG),
 };
 
-static struct resource axp288_charger_resources[] = {
-	{
-		.start = AXP288_IRQ_OV,
-		.end   = AXP288_IRQ_OV,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_DONE,
-		.end   = AXP288_IRQ_DONE,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_CHARGING,
-		.end   = AXP288_IRQ_CHARGING,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_SAFE_QUIT,
-		.end   = AXP288_IRQ_SAFE_QUIT,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_SAFE_ENTER,
-		.end   = AXP288_IRQ_SAFE_ENTER,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_QCBTU,
-		.end   = AXP288_IRQ_QCBTU,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_CBTU,
-		.end   = AXP288_IRQ_CBTU,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_QCBTO,
-		.end   = AXP288_IRQ_QCBTO,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.start = AXP288_IRQ_CBTO,
-		.end   = AXP288_IRQ_CBTO,
-		.flags = IORESOURCE_IRQ,
-	},
+static const struct resource axp288_charger_resources[] = {
+	DEFINE_RES_IRQ(AXP288_IRQ_OV),
+	DEFINE_RES_IRQ(AXP288_IRQ_DONE),
+	DEFINE_RES_IRQ(AXP288_IRQ_CHARGING),
+	DEFINE_RES_IRQ(AXP288_IRQ_SAFE_QUIT),
+	DEFINE_RES_IRQ(AXP288_IRQ_SAFE_ENTER),
+	DEFINE_RES_IRQ(AXP288_IRQ_QCBTU),
+	DEFINE_RES_IRQ(AXP288_IRQ_CBTU),
+	DEFINE_RES_IRQ(AXP288_IRQ_QCBTO),
+	DEFINE_RES_IRQ(AXP288_IRQ_CBTO),
 };
 
-static struct mfd_cell axp288_cells[] = {
+static const struct mfd_cell axp288_cells[] = {
 	{
 		.name = "axp288_adc",
 		.num_resources = ARRAY_SIZE(axp288_adc_resources),
@@ -845,7 +721,7 @@ static struct mfd_cell axp288_cells[] = {
 	},
 };
 
-static struct mfd_cell axp803_cells[] = {
+static const struct mfd_cell axp803_cells[] = {
 	{
 		.name			= "axp221-pek",
 		.num_resources		= ARRAY_SIZE(axp803_pek_resources),
@@ -854,14 +730,14 @@ static struct mfd_cell axp803_cells[] = {
 	{	.name			= "axp20x-regulator" },
 };
 
-static struct mfd_cell axp806_cells[] = {
+static const struct mfd_cell axp806_cells[] = {
 	{
 		.id			= 2,
 		.name			= "axp20x-regulator",
 	},
 };
 
-static struct mfd_cell axp809_cells[] = {
+static const struct mfd_cell axp809_cells[] = {
 	{
 		.name			= "axp221-pek",
 		.num_resources		= ARRAY_SIZE(axp809_pek_resources),
@@ -872,7 +748,7 @@ static struct mfd_cell axp809_cells[] = {
 	},
 };
 
-static struct mfd_cell axp813_cells[] = {
+static const struct mfd_cell axp813_cells[] = {
 	{
 		.name			= "axp221-pek",
 		.num_resources		= ARRAY_SIZE(axp803_pek_resources),
@@ -882,7 +758,13 @@ static struct mfd_cell axp813_cells[] = {
 	}, {
 		.name			= "axp20x-gpio",
 		.of_compatible		= "x-powers,axp813-gpio",
-	}
+	}, {
+		.name			= "axp813-adc",
+		.of_compatible		= "x-powers,axp813-adc",
+	}, {
+		.name		= "axp20x-battery-power-supply",
+		.of_compatible	= "x-powers,axp813-battery-power-supply",
+	},
 };
 
 static struct axp20x_dev *axp20x_pm_power_off;
diff --git a/drivers/mfd/bd9571mwv.c b/drivers/mfd/bd9571mwv.c
index 64e088dfe7b0..503979c81dae 100644
--- a/drivers/mfd/bd9571mwv.c
+++ b/drivers/mfd/bd9571mwv.c
@@ -29,6 +29,7 @@ static const struct mfd_cell bd9571mwv_cells[] = {
 
 static const struct regmap_range bd9571mwv_readable_yes_ranges[] = {
 	regmap_reg_range(BD9571MWV_VENDOR_CODE, BD9571MWV_PRODUCT_REVISION),
+	regmap_reg_range(BD9571MWV_BKUP_MODE_CNT, BD9571MWV_BKUP_MODE_CNT),
 	regmap_reg_range(BD9571MWV_AVS_SET_MONI, BD9571MWV_AVS_DVFS_VID(3)),
 	regmap_reg_range(BD9571MWV_VD18_VID, BD9571MWV_VD33_VID),
 	regmap_reg_range(BD9571MWV_DVFS_VINIT, BD9571MWV_DVFS_VINIT),
@@ -44,6 +45,7 @@ static const struct regmap_access_table bd9571mwv_readable_table = {
 };
 
 static const struct regmap_range bd9571mwv_writable_yes_ranges[] = {
+	regmap_reg_range(BD9571MWV_BKUP_MODE_CNT, BD9571MWV_BKUP_MODE_CNT),
 	regmap_reg_range(BD9571MWV_AVS_VD09_VID(0), BD9571MWV_AVS_VD09_VID(3)),
 	regmap_reg_range(BD9571MWV_DVFS_SETVID, BD9571MWV_DVFS_SETVID),
 	regmap_reg_range(BD9571MWV_GPIO_DIR, BD9571MWV_GPIO_OUT),
diff --git a/drivers/mfd/cros_ec.c b/drivers/mfd/cros_ec.c
index d61024141e2b..65a9757a6d21 100644
--- a/drivers/mfd/cros_ec.c
+++ b/drivers/mfd/cros_ec.c
@@ -112,12 +112,16 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 
 	mutex_init(&ec_dev->lock);
 
-	cros_ec_query_all(ec_dev);
+	err = cros_ec_query_all(ec_dev);
+	if (err) {
+		dev_err(dev, "Cannot identify the EC: error %d\n", err);
+		return err;
+	}
 
 	if (ec_dev->irq) {
-		err = request_threaded_irq(ec_dev->irq, NULL, ec_irq_thread,
-					   IRQF_TRIGGER_LOW | IRQF_ONESHOT,
-					   "chromeos-ec", ec_dev);
+		err = devm_request_threaded_irq(dev, ec_dev->irq, NULL,
+				ec_irq_thread, IRQF_TRIGGER_LOW | IRQF_ONESHOT,
+				"chromeos-ec", ec_dev);
 		if (err) {
 			dev_err(dev, "Failed to request IRQ %d: %d",
 				ec_dev->irq, err);
@@ -131,7 +135,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 		dev_err(dev,
 			"Failed to register Embedded Controller subdevice %d\n",
 			err);
-		goto fail_mfd;
+		return err;
 	}
 
 	if (ec_dev->max_passthru) {
@@ -149,7 +153,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 			dev_err(dev,
 				"Failed to register Power Delivery subdevice %d\n",
 				err);
-			goto fail_mfd;
+			return err;
 		}
 	}
 
@@ -158,7 +162,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 		if (err) {
 			mfd_remove_devices(dev);
 			dev_err(dev, "Failed to register sub-devices\n");
-			goto fail_mfd;
+			return err;
 		}
 	}
 
@@ -173,14 +177,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 
 	dev_info(dev, "Chrome EC device registered\n");
 
-	cros_ec_acpi_install_gpe_handler(dev);
-
 	return 0;
-
-fail_mfd:
-	if (ec_dev->irq)
-		free_irq(ec_dev->irq, ec_dev);
-	return err;
 }
 EXPORT_SYMBOL(cros_ec_register);
 
@@ -188,11 +185,6 @@ int cros_ec_remove(struct cros_ec_device *ec_dev)
 {
 	mfd_remove_devices(ec_dev->dev);
 
-	cros_ec_acpi_remove_gpe_handler();
-
-	if (ec_dev->irq)
-		free_irq(ec_dev->irq, ec_dev);
-
 	return 0;
 }
 EXPORT_SYMBOL(cros_ec_remove);
@@ -204,14 +196,9 @@ int cros_ec_suspend(struct cros_ec_device *ec_dev)
 	int ret;
 	u8 sleep_event;
 
-	if (!IS_ENABLED(CONFIG_ACPI) || pm_suspend_via_firmware()) {
-		sleep_event = HOST_SLEEP_EVENT_S3_SUSPEND;
-	} else {
-		sleep_event = HOST_SLEEP_EVENT_S0IX_SUSPEND;
-
-		/* Clearing the GPE status for any pending event */
-		cros_ec_acpi_clear_gpe();
-	}
+	sleep_event = (!IS_ENABLED(CONFIG_ACPI) || pm_suspend_via_firmware()) ?
+		      HOST_SLEEP_EVENT_S3_SUSPEND :
+		      HOST_SLEEP_EVENT_S0IX_SUSPEND;
 
 	ret = cros_ec_sleep_event(ec_dev, sleep_event);
 	if (ret < 0)
@@ -229,7 +216,7 @@ int cros_ec_suspend(struct cros_ec_device *ec_dev)
 }
 EXPORT_SYMBOL(cros_ec_suspend);
 
-static void cros_ec_drain_events(struct cros_ec_device *ec_dev)
+static void cros_ec_report_events_during_suspend(struct cros_ec_device *ec_dev)
 {
 	while (cros_ec_get_next_event(ec_dev, NULL) > 0)
 		blocking_notifier_call_chain(&ec_dev->event_notifier,
@@ -253,21 +240,16 @@ int cros_ec_resume(struct cros_ec_device *ec_dev)
 		dev_dbg(ec_dev->dev, "Error %d sending resume event to ec",
 			ret);
 
-	/*
-	 * In some cases, we need to distinguish between events that occur
-	 * during suspend if the EC is not a wake source. For example,
-	 * keypresses during suspend should be discarded if it does not wake
-	 * the system.
-	 *
-	 * If the EC is not a wake source, drain the event queue and mark them
-	 * as "queued during suspend".
-	 */
 	if (ec_dev->wake_enabled) {
 		disable_irq_wake(ec_dev->irq);
 		ec_dev->wake_enabled = 0;
-	} else {
-		cros_ec_drain_events(ec_dev);
 	}
+	/*
+	 * Let the mfd devices know about events that occur during
+	 * suspend. This way the clients know what to do with them.
+	 */
+	cros_ec_report_events_during_suspend(ec_dev);
+
 
 	return 0;
 }
diff --git a/drivers/mfd/cros_ec_acpi_gpe.c b/drivers/mfd/cros_ec_acpi_gpe.c
deleted file mode 100644
index 56d305dab2d4..000000000000
--- a/drivers/mfd/cros_ec_acpi_gpe.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * ChromeOS EC multi-function device
- *
- * Copyright (C) 2017 Google, Inc
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * The ChromeOS EC multi function device is used to mux all the requests
- * to the EC device for its multiple features: keyboard controller,
- * battery charging and regulator control, firmware update.
- */
-#include <linux/acpi.h>
-
-#define ACPI_LID_DEVICE      "LID0"
-
-static int ec_wake_gpe = -EINVAL;
-
-/*
- * This handler indicates to ACPI core that this GPE should stay enabled for
- * lid to work in suspend to idle path.
- */
-static u32 cros_ec_gpe_handler(acpi_handle gpe_device, u32 gpe_number,
-			       void *data)
-{
-	return ACPI_INTERRUPT_HANDLED | ACPI_REENABLE_GPE;
-}
-
-/*
- * Get ACPI GPE for LID0 device.
- */
-static int cros_ec_get_ec_wake_gpe(struct device *dev)
-{
-	struct acpi_device *cros_acpi_dev;
-	struct acpi_device *adev;
-	acpi_handle handle;
-	acpi_status status;
-	int ret;
-
-	cros_acpi_dev = ACPI_COMPANION(dev);
-
-	if (!cros_acpi_dev || !cros_acpi_dev->parent ||
-	   !cros_acpi_dev->parent->handle)
-		return -EINVAL;
-
-	status = acpi_get_handle(cros_acpi_dev->parent->handle, ACPI_LID_DEVICE,
-				 &handle);
-	if (ACPI_FAILURE(status))
-		return -EINVAL;
-
-	ret = acpi_bus_get_device(handle, &adev);
-	if (ret)
-		return ret;
-
-	return adev->wakeup.gpe_number;
-}
-
-int cros_ec_acpi_install_gpe_handler(struct device *dev)
-{
-	acpi_status status;
-
-	ec_wake_gpe = cros_ec_get_ec_wake_gpe(dev);
-
-	if (ec_wake_gpe < 0)
-		return ec_wake_gpe;
-
-	status = acpi_install_gpe_handler(NULL, ec_wake_gpe,
-					  ACPI_GPE_EDGE_TRIGGERED,
-					  &cros_ec_gpe_handler, NULL);
-	if (ACPI_FAILURE(status))
-		return -ENODEV;
-
-	dev_info(dev, "Initialized, GPE = 0x%x\n", ec_wake_gpe);
-
-	return 0;
-}
-
-void cros_ec_acpi_remove_gpe_handler(void)
-{
-	acpi_status status;
-
-	if (ec_wake_gpe < 0)
-		return;
-
-	status = acpi_remove_gpe_handler(NULL, ec_wake_gpe,
-						 &cros_ec_gpe_handler);
-	if (ACPI_FAILURE(status))
-		pr_err("failed to remove gpe handler\n");
-}
-
-void cros_ec_acpi_clear_gpe(void)
-{
-	if (ec_wake_gpe < 0)
-		return;
-
-	acpi_clear_gpe(NULL, ec_wake_gpe);
-}
diff --git a/drivers/mfd/cros_ec_dev.c b/drivers/mfd/cros_ec_dev.c
index eafd06f62a3a..1d6dc5c7a19d 100644
--- a/drivers/mfd/cros_ec_dev.c
+++ b/drivers/mfd/cros_ec_dev.c
@@ -383,6 +383,10 @@ static void cros_ec_sensors_register(struct cros_ec_dev *ec)
 	kfree(msg);
 }
 
+static const struct mfd_cell cros_ec_rtc_cells[] = {
+	{ .name = "cros-ec-rtc" }
+};
+
 static int ec_device_probe(struct platform_device *pdev)
 {
 	int retval = -ENOMEM;
@@ -422,6 +426,18 @@ static int ec_device_probe(struct platform_device *pdev)
 	if (cros_ec_check_features(ec, EC_FEATURE_MOTION_SENSE))
 		cros_ec_sensors_register(ec);
 
+	/* Check whether this EC instance has RTC host command support */
+	if (cros_ec_check_features(ec, EC_FEATURE_RTC)) {
+		retval = mfd_add_devices(ec->dev, PLATFORM_DEVID_AUTO,
+					 cros_ec_rtc_cells,
+					 ARRAY_SIZE(cros_ec_rtc_cells),
+					 NULL, 0, NULL);
+		if (retval)
+			dev_err(ec->dev,
+				"failed to add cros-ec-rtc device: %d\n",
+				retval);
+	}
+
 	/* Take control of the lightbar from the EC. */
 	lb_manual_suspend_ctrl(ec, 1);
 
@@ -456,9 +472,17 @@ static int ec_device_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static void ec_device_shutdown(struct platform_device *pdev)
+{
+	struct cros_ec_dev *ec = dev_get_drvdata(&pdev->dev);
+
+	/* Be sure to clear up debugfs delayed works */
+	cros_ec_debugfs_remove(ec);
+}
+
 static const struct platform_device_id cros_ec_id[] = {
 	{ DRV_NAME, 0 },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(platform, cros_ec_id);
 
@@ -466,6 +490,8 @@ static __maybe_unused int ec_device_suspend(struct device *dev)
 {
 	struct cros_ec_dev *ec = dev_get_drvdata(dev);
 
+	cros_ec_debugfs_suspend(ec);
+
 	lb_suspend(ec);
 
 	return 0;
@@ -475,6 +501,8 @@ static __maybe_unused int ec_device_resume(struct device *dev)
 {
 	struct cros_ec_dev *ec = dev_get_drvdata(dev);
 
+	cros_ec_debugfs_resume(ec);
+
 	lb_resume(ec);
 
 	return 0;
@@ -494,6 +522,7 @@ static struct platform_driver cros_ec_dev_driver = {
 	},
 	.probe = ec_device_probe,
 	.remove = ec_device_remove,
+	.shutdown = ec_device_shutdown,
 };
 
 static int __init cros_ec_dev_init(void)
diff --git a/drivers/mfd/cros_ec_i2c.c b/drivers/mfd/cros_ec_i2c.c
index 9f70de1e4c70..ef9b4763356f 100644
--- a/drivers/mfd/cros_ec_i2c.c
+++ b/drivers/mfd/cros_ec_i2c.c
@@ -13,6 +13,7 @@
  * GNU General Public License for more details.
  */
 
+#include <linux/acpi.h>
 #include <linux/delay.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -341,14 +342,17 @@ static int cros_ec_i2c_resume(struct device *dev)
 }
 #endif
 
-static SIMPLE_DEV_PM_OPS(cros_ec_i2c_pm_ops, cros_ec_i2c_suspend,
-			  cros_ec_i2c_resume);
+static const struct dev_pm_ops cros_ec_i2c_pm_ops = {
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(cros_ec_i2c_suspend, cros_ec_i2c_resume)
+};
 
+#ifdef CONFIG_OF
 static const struct of_device_id cros_ec_i2c_of_match[] = {
 	{ .compatible = "google,cros-ec-i2c", },
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, cros_ec_i2c_of_match);
+#endif
 
 static const struct i2c_device_id cros_ec_i2c_id[] = {
 	{ "cros-ec-i2c", 0 },
@@ -356,9 +360,18 @@ static const struct i2c_device_id cros_ec_i2c_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, cros_ec_i2c_id);
 
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id cros_ec_i2c_acpi_id[] = {
+	{ "GOOG0008", 0 },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(acpi, cros_ec_i2c_acpi_id);
+#endif
+
 static struct i2c_driver cros_ec_driver = {
 	.driver	= {
 		.name	= "cros-ec-i2c",
+		.acpi_match_table = ACPI_PTR(cros_ec_i2c_acpi_id),
 		.of_match_table = of_match_ptr(cros_ec_i2c_of_match),
 		.pm	= &cros_ec_i2c_pm_ops,
 	},
diff --git a/drivers/mfd/da9062-core.c b/drivers/mfd/da9062-core.c
index fe1811523e4a..9f6105906c09 100644
--- a/drivers/mfd/da9062-core.c
+++ b/drivers/mfd/da9062-core.c
@@ -365,186 +365,69 @@ static int da9062_get_device_type(struct da9062 *chip)
 }
 
 static const struct regmap_range da9061_aa_readable_ranges[] = {
-	{
-		.range_min = DA9062AA_PAGE_CON,
-		.range_max = DA9062AA_STATUS_B,
-	}, {
-		.range_min = DA9062AA_STATUS_D,
-		.range_max = DA9062AA_EVENT_C,
-	}, {
-		.range_min = DA9062AA_IRQ_MASK_A,
-		.range_max = DA9062AA_IRQ_MASK_C,
-	}, {
-		.range_min = DA9062AA_CONTROL_A,
-		.range_max = DA9062AA_GPIO_4,
-	}, {
-		.range_min = DA9062AA_GPIO_WKUP_MODE,
-		.range_max = DA9062AA_GPIO_OUT3_4,
-	}, {
-		.range_min = DA9062AA_BUCK1_CONT,
-		.range_max = DA9062AA_BUCK4_CONT,
-	}, {
-		.range_min = DA9062AA_BUCK3_CONT,
-		.range_max = DA9062AA_BUCK3_CONT,
-	}, {
-		.range_min = DA9062AA_LDO1_CONT,
-		.range_max = DA9062AA_LDO4_CONT,
-	}, {
-		.range_min = DA9062AA_DVC_1,
-		.range_max = DA9062AA_DVC_1,
-	}, {
-		.range_min = DA9062AA_SEQ,
-		.range_max = DA9062AA_ID_4_3,
-	}, {
-		.range_min = DA9062AA_ID_12_11,
-		.range_max = DA9062AA_ID_16_15,
-	}, {
-		.range_min = DA9062AA_ID_22_21,
-		.range_max = DA9062AA_ID_32_31,
-	}, {
-		.range_min = DA9062AA_SEQ_A,
-		.range_max = DA9062AA_WAIT,
-	}, {
-		.range_min = DA9062AA_RESET,
-		.range_max = DA9062AA_BUCK_ILIM_C,
-	}, {
-		.range_min = DA9062AA_BUCK1_CFG,
-		.range_max = DA9062AA_BUCK3_CFG,
-	}, {
-		.range_min = DA9062AA_VBUCK1_A,
-		.range_max = DA9062AA_VBUCK4_A,
-	}, {
-		.range_min = DA9062AA_VBUCK3_A,
-		.range_max = DA9062AA_VBUCK3_A,
-	}, {
-		.range_min = DA9062AA_VLDO1_A,
-		.range_max = DA9062AA_VLDO4_A,
-	}, {
-		.range_min = DA9062AA_VBUCK1_B,
-		.range_max = DA9062AA_VBUCK4_B,
-	}, {
-		.range_min = DA9062AA_VBUCK3_B,
-		.range_max = DA9062AA_VBUCK3_B,
-	}, {
-		.range_min = DA9062AA_VLDO1_B,
-		.range_max = DA9062AA_VLDO4_B,
-	}, {
-		.range_min = DA9062AA_INTERFACE,
-		.range_max = DA9062AA_CONFIG_E,
-	}, {
-		.range_min = DA9062AA_CONFIG_G,
-		.range_max = DA9062AA_CONFIG_K,
-	}, {
-		.range_min = DA9062AA_CONFIG_M,
-		.range_max = DA9062AA_CONFIG_M,
-	}, {
-		.range_min = DA9062AA_GP_ID_0,
-		.range_max = DA9062AA_GP_ID_19,
-	}, {
-		.range_min = DA9062AA_DEVICE_ID,
-		.range_max = DA9062AA_CONFIG_ID,
-	},
+	regmap_reg_range(DA9062AA_PAGE_CON, DA9062AA_STATUS_B),
+	regmap_reg_range(DA9062AA_STATUS_D, DA9062AA_EVENT_C),
+	regmap_reg_range(DA9062AA_IRQ_MASK_A, DA9062AA_IRQ_MASK_C),
+	regmap_reg_range(DA9062AA_CONTROL_A, DA9062AA_GPIO_4),
+	regmap_reg_range(DA9062AA_GPIO_WKUP_MODE, DA9062AA_GPIO_OUT3_4),
+	regmap_reg_range(DA9062AA_BUCK1_CONT, DA9062AA_BUCK4_CONT),
+	regmap_reg_range(DA9062AA_BUCK3_CONT, DA9062AA_BUCK3_CONT),
+	regmap_reg_range(DA9062AA_LDO1_CONT, DA9062AA_LDO4_CONT),
+	regmap_reg_range(DA9062AA_DVC_1, DA9062AA_DVC_1),
+	regmap_reg_range(DA9062AA_SEQ, DA9062AA_ID_4_3),
+	regmap_reg_range(DA9062AA_ID_12_11, DA9062AA_ID_16_15),
+	regmap_reg_range(DA9062AA_ID_22_21, DA9062AA_ID_32_31),
+	regmap_reg_range(DA9062AA_SEQ_A, DA9062AA_WAIT),
+	regmap_reg_range(DA9062AA_RESET, DA9062AA_BUCK_ILIM_C),
+	regmap_reg_range(DA9062AA_BUCK1_CFG, DA9062AA_BUCK3_CFG),
+	regmap_reg_range(DA9062AA_VBUCK1_A, DA9062AA_VBUCK4_A),
+	regmap_reg_range(DA9062AA_VBUCK3_A, DA9062AA_VBUCK3_A),
+	regmap_reg_range(DA9062AA_VLDO1_A, DA9062AA_VLDO4_A),
+	regmap_reg_range(DA9062AA_VBUCK1_B, DA9062AA_VBUCK4_B),
+	regmap_reg_range(DA9062AA_VBUCK3_B, DA9062AA_VBUCK3_B),
+	regmap_reg_range(DA9062AA_VLDO1_B, DA9062AA_VLDO4_B),
+	regmap_reg_range(DA9062AA_INTERFACE, DA9062AA_CONFIG_E),
+	regmap_reg_range(DA9062AA_CONFIG_G, DA9062AA_CONFIG_K),
+	regmap_reg_range(DA9062AA_CONFIG_M, DA9062AA_CONFIG_M),
+	regmap_reg_range(DA9062AA_GP_ID_0, DA9062AA_GP_ID_19),
+	regmap_reg_range(DA9062AA_DEVICE_ID, DA9062AA_CONFIG_ID),
 };
 
 static const struct regmap_range da9061_aa_writeable_ranges[] = {
-	{
-		.range_min = DA9062AA_PAGE_CON,
-		.range_max = DA9062AA_PAGE_CON,
-	}, {
-		.range_min = DA9062AA_FAULT_LOG,
-		.range_max = DA9062AA_EVENT_C,
-	}, {
-		.range_min = DA9062AA_IRQ_MASK_A,
-		.range_max = DA9062AA_IRQ_MASK_C,
-	}, {
-		.range_min = DA9062AA_CONTROL_A,
-		.range_max = DA9062AA_GPIO_4,
-	}, {
-		.range_min = DA9062AA_GPIO_WKUP_MODE,
-		.range_max = DA9062AA_GPIO_OUT3_4,
-	}, {
-		.range_min = DA9062AA_BUCK1_CONT,
-		.range_max = DA9062AA_BUCK4_CONT,
-	}, {
-		.range_min = DA9062AA_BUCK3_CONT,
-		.range_max = DA9062AA_BUCK3_CONT,
-	}, {
-		.range_min = DA9062AA_LDO1_CONT,
-		.range_max = DA9062AA_LDO4_CONT,
-	}, {
-		.range_min = DA9062AA_DVC_1,
-		.range_max = DA9062AA_DVC_1,
-	}, {
-		.range_min = DA9062AA_SEQ,
-		.range_max = DA9062AA_ID_4_3,
-	}, {
-		.range_min = DA9062AA_ID_12_11,
-		.range_max = DA9062AA_ID_16_15,
-	}, {
-		.range_min = DA9062AA_ID_22_21,
-		.range_max = DA9062AA_ID_32_31,
-	}, {
-		.range_min = DA9062AA_SEQ_A,
-		.range_max = DA9062AA_WAIT,
-	}, {
-		.range_min = DA9062AA_RESET,
-		.range_max = DA9062AA_BUCK_ILIM_C,
-	}, {
-		.range_min = DA9062AA_BUCK1_CFG,
-		.range_max = DA9062AA_BUCK3_CFG,
-	}, {
-		.range_min = DA9062AA_VBUCK1_A,
-		.range_max = DA9062AA_VBUCK4_A,
-	}, {
-		.range_min = DA9062AA_VBUCK3_A,
-		.range_max = DA9062AA_VBUCK3_A,
-	}, {
-		.range_min = DA9062AA_VLDO1_A,
-		.range_max = DA9062AA_VLDO4_A,
-	}, {
-		.range_min = DA9062AA_VBUCK1_B,
-		.range_max = DA9062AA_VBUCK4_B,
-	}, {
-		.range_min = DA9062AA_VBUCK3_B,
-		.range_max = DA9062AA_VBUCK3_B,
-	}, {
-		.range_min = DA9062AA_VLDO1_B,
-		.range_max = DA9062AA_VLDO4_B,
-	}, {
-		.range_min = DA9062AA_GP_ID_0,
-		.range_max = DA9062AA_GP_ID_19,
-	},
+	regmap_reg_range(DA9062AA_PAGE_CON, DA9062AA_PAGE_CON),
+	regmap_reg_range(DA9062AA_FAULT_LOG, DA9062AA_EVENT_C),
+	regmap_reg_range(DA9062AA_IRQ_MASK_A, DA9062AA_IRQ_MASK_C),
+	regmap_reg_range(DA9062AA_CONTROL_A, DA9062AA_GPIO_4),
+	regmap_reg_range(DA9062AA_GPIO_WKUP_MODE, DA9062AA_GPIO_OUT3_4),
+	regmap_reg_range(DA9062AA_BUCK1_CONT, DA9062AA_BUCK4_CONT),
+	regmap_reg_range(DA9062AA_BUCK3_CONT, DA9062AA_BUCK3_CONT),
+	regmap_reg_range(DA9062AA_LDO1_CONT, DA9062AA_LDO4_CONT),
+	regmap_reg_range(DA9062AA_DVC_1, DA9062AA_DVC_1),
+	regmap_reg_range(DA9062AA_SEQ, DA9062AA_ID_4_3),
+	regmap_reg_range(DA9062AA_ID_12_11, DA9062AA_ID_16_15),
+	regmap_reg_range(DA9062AA_ID_22_21, DA9062AA_ID_32_31),
+	regmap_reg_range(DA9062AA_SEQ_A, DA9062AA_WAIT),
+	regmap_reg_range(DA9062AA_RESET, DA9062AA_BUCK_ILIM_C),
+	regmap_reg_range(DA9062AA_BUCK1_CFG, DA9062AA_BUCK3_CFG),
+	regmap_reg_range(DA9062AA_VBUCK1_A, DA9062AA_VBUCK4_A),
+	regmap_reg_range(DA9062AA_VBUCK3_A, DA9062AA_VBUCK3_A),
+	regmap_reg_range(DA9062AA_VLDO1_A, DA9062AA_VLDO4_A),
+	regmap_reg_range(DA9062AA_VBUCK1_B, DA9062AA_VBUCK4_B),
+	regmap_reg_range(DA9062AA_VBUCK3_B, DA9062AA_VBUCK3_B),
+	regmap_reg_range(DA9062AA_VLDO1_B, DA9062AA_VLDO4_B),
+	regmap_reg_range(DA9062AA_GP_ID_0, DA9062AA_GP_ID_19),
 };
 
 static const struct regmap_range da9061_aa_volatile_ranges[] = {
-	{
-		.range_min = DA9062AA_PAGE_CON,
-		.range_max = DA9062AA_STATUS_B,
-	}, {
-		.range_min = DA9062AA_STATUS_D,
-		.range_max = DA9062AA_EVENT_C,
-	}, {
-		.range_min = DA9062AA_CONTROL_A,
-		.range_max = DA9062AA_CONTROL_B,
-	}, {
-		.range_min = DA9062AA_CONTROL_E,
-		.range_max = DA9062AA_CONTROL_F,
-	}, {
-		.range_min = DA9062AA_BUCK1_CONT,
-		.range_max = DA9062AA_BUCK4_CONT,
-	}, {
-		.range_min = DA9062AA_BUCK3_CONT,
-		.range_max = DA9062AA_BUCK3_CONT,
-	}, {
-		.range_min = DA9062AA_LDO1_CONT,
-		.range_max = DA9062AA_LDO4_CONT,
-	}, {
-		.range_min = DA9062AA_DVC_1,
-		.range_max = DA9062AA_DVC_1,
-	}, {
-		.range_min = DA9062AA_SEQ,
-		.range_max = DA9062AA_SEQ,
-	},
+	regmap_reg_range(DA9062AA_PAGE_CON, DA9062AA_STATUS_B),
+	regmap_reg_range(DA9062AA_STATUS_D, DA9062AA_EVENT_C),
+	regmap_reg_range(DA9062AA_CONTROL_A, DA9062AA_CONTROL_B),
+	regmap_reg_range(DA9062AA_CONTROL_E, DA9062AA_CONTROL_F),
+	regmap_reg_range(DA9062AA_BUCK1_CONT, DA9062AA_BUCK4_CONT),
+	regmap_reg_range(DA9062AA_BUCK3_CONT, DA9062AA_BUCK3_CONT),
+	regmap_reg_range(DA9062AA_LDO1_CONT, DA9062AA_LDO4_CONT),
+	regmap_reg_range(DA9062AA_DVC_1, DA9062AA_DVC_1),
+	regmap_reg_range(DA9062AA_SEQ, DA9062AA_SEQ),
 };
 
 static const struct regmap_access_table da9061_aa_readable_table = {
@@ -587,186 +470,69 @@ static struct regmap_config da9061_regmap_config = {
 };
 
 static const struct regmap_range da9062_aa_readable_ranges[] = {
-	{
-		.range_min = DA9062AA_PAGE_CON,
-		.range_max = DA9062AA_STATUS_B,
-	}, {
-		.range_min = DA9062AA_STATUS_D,
-		.range_max = DA9062AA_EVENT_C,
-	}, {
-		.range_min = DA9062AA_IRQ_MASK_A,
-		.range_max = DA9062AA_IRQ_MASK_C,
-	}, {
-		.range_min = DA9062AA_CONTROL_A,
-		.range_max = DA9062AA_GPIO_4,
-	}, {
-		.range_min = DA9062AA_GPIO_WKUP_MODE,
-		.range_max = DA9062AA_BUCK4_CONT,
-	}, {
-		.range_min = DA9062AA_BUCK3_CONT,
-		.range_max = DA9062AA_BUCK3_CONT,
-	}, {
-		.range_min = DA9062AA_LDO1_CONT,
-		.range_max = DA9062AA_LDO4_CONT,
-	}, {
-		.range_min = DA9062AA_DVC_1,
-		.range_max = DA9062AA_DVC_1,
-	}, {
-		.range_min = DA9062AA_COUNT_S,
-		.range_max = DA9062AA_SECOND_D,
-	}, {
-		.range_min = DA9062AA_SEQ,
-		.range_max = DA9062AA_ID_4_3,
-	}, {
-		.range_min = DA9062AA_ID_12_11,
-		.range_max = DA9062AA_ID_16_15,
-	}, {
-		.range_min = DA9062AA_ID_22_21,
-		.range_max = DA9062AA_ID_32_31,
-	}, {
-		.range_min = DA9062AA_SEQ_A,
-		.range_max = DA9062AA_BUCK3_CFG,
-	}, {
-		.range_min = DA9062AA_VBUCK2_A,
-		.range_max = DA9062AA_VBUCK4_A,
-	}, {
-		.range_min = DA9062AA_VBUCK3_A,
-		.range_max = DA9062AA_VBUCK3_A,
-	}, {
-		.range_min = DA9062AA_VLDO1_A,
-		.range_max = DA9062AA_VLDO4_A,
-	}, {
-		.range_min = DA9062AA_VBUCK2_B,
-		.range_max = DA9062AA_VBUCK4_B,
-	}, {
-		.range_min = DA9062AA_VBUCK3_B,
-		.range_max = DA9062AA_VBUCK3_B,
-	}, {
-		.range_min = DA9062AA_VLDO1_B,
-		.range_max = DA9062AA_VLDO4_B,
-	}, {
-		.range_min = DA9062AA_BBAT_CONT,
-		.range_max = DA9062AA_BBAT_CONT,
-	}, {
-		.range_min = DA9062AA_INTERFACE,
-		.range_max = DA9062AA_CONFIG_E,
-	}, {
-		.range_min = DA9062AA_CONFIG_G,
-		.range_max = DA9062AA_CONFIG_K,
-	}, {
-		.range_min = DA9062AA_CONFIG_M,
-		.range_max = DA9062AA_CONFIG_M,
-	}, {
-		.range_min = DA9062AA_TRIM_CLDR,
-		.range_max = DA9062AA_GP_ID_19,
-	}, {
-		.range_min = DA9062AA_DEVICE_ID,
-		.range_max = DA9062AA_CONFIG_ID,
-	},
+	regmap_reg_range(DA9062AA_PAGE_CON, DA9062AA_STATUS_B),
+	regmap_reg_range(DA9062AA_STATUS_D, DA9062AA_EVENT_C),
+	regmap_reg_range(DA9062AA_IRQ_MASK_A, DA9062AA_IRQ_MASK_C),
+	regmap_reg_range(DA9062AA_CONTROL_A, DA9062AA_GPIO_4),
+	regmap_reg_range(DA9062AA_GPIO_WKUP_MODE, DA9062AA_BUCK4_CONT),
+	regmap_reg_range(DA9062AA_BUCK3_CONT, DA9062AA_BUCK3_CONT),
+	regmap_reg_range(DA9062AA_LDO1_CONT, DA9062AA_LDO4_CONT),
+	regmap_reg_range(DA9062AA_DVC_1, DA9062AA_DVC_1),
+	regmap_reg_range(DA9062AA_COUNT_S, DA9062AA_SECOND_D),
+	regmap_reg_range(DA9062AA_SEQ, DA9062AA_ID_4_3),
+	regmap_reg_range(DA9062AA_ID_12_11, DA9062AA_ID_16_15),
+	regmap_reg_range(DA9062AA_ID_22_21, DA9062AA_ID_32_31),
+	regmap_reg_range(DA9062AA_SEQ_A, DA9062AA_BUCK3_CFG),
+	regmap_reg_range(DA9062AA_VBUCK2_A, DA9062AA_VBUCK4_A),
+	regmap_reg_range(DA9062AA_VBUCK3_A, DA9062AA_VBUCK3_A),
+	regmap_reg_range(DA9062AA_VLDO1_A, DA9062AA_VLDO4_A),
+	regmap_reg_range(DA9062AA_VBUCK2_B, DA9062AA_VBUCK4_B),
+	regmap_reg_range(DA9062AA_VBUCK3_B, DA9062AA_VBUCK3_B),
+	regmap_reg_range(DA9062AA_VLDO1_B, DA9062AA_VLDO4_B),
+	regmap_reg_range(DA9062AA_BBAT_CONT, DA9062AA_BBAT_CONT),
+	regmap_reg_range(DA9062AA_INTERFACE, DA9062AA_CONFIG_E),
+	regmap_reg_range(DA9062AA_CONFIG_G, DA9062AA_CONFIG_K),
+	regmap_reg_range(DA9062AA_CONFIG_M, DA9062AA_CONFIG_M),
+	regmap_reg_range(DA9062AA_TRIM_CLDR, DA9062AA_GP_ID_19),
+	regmap_reg_range(DA9062AA_DEVICE_ID, DA9062AA_CONFIG_ID),
 };
 
 static const struct regmap_range da9062_aa_writeable_ranges[] = {
-	{
-		.range_min = DA9062AA_PAGE_CON,
-		.range_max = DA9062AA_PAGE_CON,
-	}, {
-		.range_min = DA9062AA_FAULT_LOG,
-		.range_max = DA9062AA_EVENT_C,
-	}, {
-		.range_min = DA9062AA_IRQ_MASK_A,
-		.range_max = DA9062AA_IRQ_MASK_C,
-	}, {
-		.range_min = DA9062AA_CONTROL_A,
-		.range_max = DA9062AA_GPIO_4,
-	}, {
-		.range_min = DA9062AA_GPIO_WKUP_MODE,
-		.range_max = DA9062AA_BUCK4_CONT,
-	}, {
-		.range_min = DA9062AA_BUCK3_CONT,
-		.range_max = DA9062AA_BUCK3_CONT,
-	}, {
-		.range_min = DA9062AA_LDO1_CONT,
-		.range_max = DA9062AA_LDO4_CONT,
-	}, {
-		.range_min = DA9062AA_DVC_1,
-		.range_max = DA9062AA_DVC_1,
-	}, {
-		.range_min = DA9062AA_COUNT_S,
-		.range_max = DA9062AA_ALARM_Y,
-	}, {
-		.range_min = DA9062AA_SEQ,
-		.range_max = DA9062AA_ID_4_3,
-	}, {
-		.range_min = DA9062AA_ID_12_11,
-		.range_max = DA9062AA_ID_16_15,
-	}, {
-		.range_min = DA9062AA_ID_22_21,
-		.range_max = DA9062AA_ID_32_31,
-	}, {
-		.range_min = DA9062AA_SEQ_A,
-		.range_max = DA9062AA_BUCK3_CFG,
-	}, {
-		.range_min = DA9062AA_VBUCK2_A,
-		.range_max = DA9062AA_VBUCK4_A,
-	}, {
-		.range_min = DA9062AA_VBUCK3_A,
-		.range_max = DA9062AA_VBUCK3_A,
-	}, {
-		.range_min = DA9062AA_VLDO1_A,
-		.range_max = DA9062AA_VLDO4_A,
-	}, {
-		.range_min = DA9062AA_VBUCK2_B,
-		.range_max = DA9062AA_VBUCK4_B,
-	}, {
-		.range_min = DA9062AA_VBUCK3_B,
-		.range_max = DA9062AA_VBUCK3_B,
-	}, {
-		.range_min = DA9062AA_VLDO1_B,
-		.range_max = DA9062AA_VLDO4_B,
-	}, {
-		.range_min = DA9062AA_BBAT_CONT,
-		.range_max = DA9062AA_BBAT_CONT,
-	}, {
-		.range_min = DA9062AA_GP_ID_0,
-		.range_max = DA9062AA_GP_ID_19,
-	},
+	regmap_reg_range(DA9062AA_PAGE_CON, DA9062AA_PAGE_CON),
+	regmap_reg_range(DA9062AA_FAULT_LOG, DA9062AA_EVENT_C),
+	regmap_reg_range(DA9062AA_IRQ_MASK_A, DA9062AA_IRQ_MASK_C),
+	regmap_reg_range(DA9062AA_CONTROL_A, DA9062AA_GPIO_4),
+	regmap_reg_range(DA9062AA_GPIO_WKUP_MODE, DA9062AA_BUCK4_CONT),
+	regmap_reg_range(DA9062AA_BUCK3_CONT, DA9062AA_BUCK3_CONT),
+	regmap_reg_range(DA9062AA_LDO1_CONT, DA9062AA_LDO4_CONT),
+	regmap_reg_range(DA9062AA_DVC_1, DA9062AA_DVC_1),
+	regmap_reg_range(DA9062AA_COUNT_S, DA9062AA_ALARM_Y),
+	regmap_reg_range(DA9062AA_SEQ, DA9062AA_ID_4_3),
+	regmap_reg_range(DA9062AA_ID_12_11, DA9062AA_ID_16_15),
+	regmap_reg_range(DA9062AA_ID_22_21, DA9062AA_ID_32_31),
+	regmap_reg_range(DA9062AA_SEQ_A, DA9062AA_BUCK3_CFG),
+	regmap_reg_range(DA9062AA_VBUCK2_A, DA9062AA_VBUCK4_A),
+	regmap_reg_range(DA9062AA_VBUCK3_A, DA9062AA_VBUCK3_A),
+	regmap_reg_range(DA9062AA_VLDO1_A, DA9062AA_VLDO4_A),
+	regmap_reg_range(DA9062AA_VBUCK2_B, DA9062AA_VBUCK4_B),
+	regmap_reg_range(DA9062AA_VBUCK3_B, DA9062AA_VBUCK3_B),
+	regmap_reg_range(DA9062AA_VLDO1_B, DA9062AA_VLDO4_B),
+	regmap_reg_range(DA9062AA_BBAT_CONT, DA9062AA_BBAT_CONT),
+	regmap_reg_range(DA9062AA_GP_ID_0, DA9062AA_GP_ID_19),
 };
 
 static const struct regmap_range da9062_aa_volatile_ranges[] = {
-	{
-		.range_min = DA9062AA_PAGE_CON,
-		.range_max = DA9062AA_STATUS_B,
-	}, {
-		.range_min = DA9062AA_STATUS_D,
-		.range_max = DA9062AA_EVENT_C,
-	}, {
-		.range_min = DA9062AA_CONTROL_A,
-		.range_max = DA9062AA_CONTROL_B,
-	}, {
-		.range_min = DA9062AA_CONTROL_E,
-		.range_max = DA9062AA_CONTROL_F,
-	}, {
-		.range_min = DA9062AA_BUCK2_CONT,
-		.range_max = DA9062AA_BUCK4_CONT,
-	}, {
-		.range_min = DA9062AA_BUCK3_CONT,
-		.range_max = DA9062AA_BUCK3_CONT,
-	}, {
-		.range_min = DA9062AA_LDO1_CONT,
-		.range_max = DA9062AA_LDO4_CONT,
-	}, {
-		.range_min = DA9062AA_DVC_1,
-		.range_max = DA9062AA_DVC_1,
-	}, {
-		.range_min = DA9062AA_COUNT_S,
-		.range_max = DA9062AA_SECOND_D,
-	}, {
-		.range_min = DA9062AA_SEQ,
-		.range_max = DA9062AA_SEQ,
-	}, {
-		.range_min = DA9062AA_EN_32K,
-		.range_max = DA9062AA_EN_32K,
-	},
+	regmap_reg_range(DA9062AA_PAGE_CON, DA9062AA_STATUS_B),
+	regmap_reg_range(DA9062AA_STATUS_D, DA9062AA_EVENT_C),
+	regmap_reg_range(DA9062AA_CONTROL_A, DA9062AA_CONTROL_B),
+	regmap_reg_range(DA9062AA_CONTROL_E, DA9062AA_CONTROL_F),
+	regmap_reg_range(DA9062AA_BUCK2_CONT, DA9062AA_BUCK4_CONT),
+	regmap_reg_range(DA9062AA_BUCK3_CONT, DA9062AA_BUCK3_CONT),
+	regmap_reg_range(DA9062AA_LDO1_CONT, DA9062AA_LDO4_CONT),
+	regmap_reg_range(DA9062AA_DVC_1, DA9062AA_DVC_1),
+	regmap_reg_range(DA9062AA_COUNT_S, DA9062AA_SECOND_D),
+	regmap_reg_range(DA9062AA_SEQ, DA9062AA_SEQ),
+	regmap_reg_range(DA9062AA_EN_32K, DA9062AA_EN_32K),
 };
 
 static const struct regmap_access_table da9062_aa_readable_table = {
diff --git a/drivers/mfd/htc-i2cpld.c b/drivers/mfd/htc-i2cpld.c
index 3f9eee5f8fb9..4bf8b7781c77 100644
--- a/drivers/mfd/htc-i2cpld.c
+++ b/drivers/mfd/htc-i2cpld.c
@@ -479,10 +479,8 @@ static int htcpld_setup_chips(struct platform_device *pdev)
 	htcpld->nchips = pdata->num_chip;
 	htcpld->chip = devm_kzalloc(dev, sizeof(struct htcpld_chip) * htcpld->nchips,
 				    GFP_KERNEL);
-	if (!htcpld->chip) {
-		dev_warn(dev, "Unable to allocate memory for chips\n");
+	if (!htcpld->chip)
 		return -ENOMEM;
-	}
 
 	/* Add the chips as best we can */
 	for (i = 0; i < htcpld->nchips; i++) {
diff --git a/drivers/mfd/intel-lpss.c b/drivers/mfd/intel-lpss.c
index 9e545eb6e8b4..4bcf117a7ba8 100644
--- a/drivers/mfd/intel-lpss.c
+++ b/drivers/mfd/intel-lpss.c
@@ -275,11 +275,11 @@ static void intel_lpss_init_dev(const struct intel_lpss *lpss)
 
 	intel_lpss_deassert_reset(lpss);
 
+	intel_lpss_set_remap_addr(lpss);
+
 	if (!intel_lpss_has_idma(lpss))
 		return;
 
-	intel_lpss_set_remap_addr(lpss);
-
 	/* Make sure that SPI multiblock DMA transfers are re-enabled */
 	if (lpss->type == LPSS_DEV_SPI)
 		writel(value, lpss->priv + LPSS_PRIV_SSP_REG);
diff --git a/drivers/mfd/intel_quark_i2c_gpio.c b/drivers/mfd/intel_quark_i2c_gpio.c
index 90e35dec8648..5bddb84cfc1f 100644
--- a/drivers/mfd/intel_quark_i2c_gpio.c
+++ b/drivers/mfd/intel_quark_i2c_gpio.c
@@ -233,7 +233,8 @@ static int intel_quark_gpio_setup(struct pci_dev *pdev, struct mfd_cell *cell)
 	pdata->properties->idx		= 0;
 	pdata->properties->ngpio	= INTEL_QUARK_MFD_NGPIO;
 	pdata->properties->gpio_base	= INTEL_QUARK_MFD_GPIO_BASE;
-	pdata->properties->irq		= pdev->irq;
+	pdata->properties->irq[0]	= pdev->irq;
+	pdata->properties->has_irq	= true;
 	pdata->properties->irq_shared	= true;
 
 	cell->platform_data = pdata;
diff --git a/drivers/mfd/janz-cmodio.c b/drivers/mfd/janz-cmodio.c
index ec1f46a6be3a..317a47ad5bb7 100644
--- a/drivers/mfd/janz-cmodio.c
+++ b/drivers/mfd/janz-cmodio.c
@@ -183,10 +183,8 @@ static int cmodio_pci_probe(struct pci_dev *dev,
 	int ret;
 
 	priv = devm_kzalloc(&dev->dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv) {
-		dev_err(&dev->dev, "unable to allocate private data\n");
+	if (!priv)
 		return -ENOMEM;
-	}
 
 	pci_set_drvdata(dev, priv);
 	priv->pdev = dev;
diff --git a/drivers/mfd/jz4740-adc.c b/drivers/mfd/jz4740-adc.c
index 798e44306382..f4cd14294b61 100644
--- a/drivers/mfd/jz4740-adc.c
+++ b/drivers/mfd/jz4740-adc.c
@@ -212,10 +212,8 @@ static int jz4740_adc_probe(struct platform_device *pdev)
 	int irq_base;
 
 	adc = devm_kzalloc(&pdev->dev, sizeof(*adc), GFP_KERNEL);
-	if (!adc) {
-		dev_err(&pdev->dev, "Failed to allocate driver structure\n");
+	if (!adc)
 		return -ENOMEM;
-	}
 
 	adc->irq = platform_get_irq(pdev, 0);
 	if (adc->irq < 0) {
diff --git a/drivers/mfd/max8997.c b/drivers/mfd/max8997.c
index 2d6e2c392786..3f554c447521 100644
--- a/drivers/mfd/max8997.c
+++ b/drivers/mfd/max8997.c
@@ -148,10 +148,8 @@ static struct max8997_platform_data *max8997_i2c_parse_dt_pdata(
 	struct max8997_platform_data *pd;
 
 	pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);
-	if (!pd) {
-		dev_err(dev, "could not allocate memory for pdata\n");
+	if (!pd)
 		return ERR_PTR(-ENOMEM);
-	}
 
 	pd->ono = irq_of_parse_and_map(dev->of_node, 1);
 
diff --git a/drivers/mfd/mc13xxx-core.c b/drivers/mfd/mc13xxx-core.c
index d7f54e492aa6..c63e331738c1 100644
--- a/drivers/mfd/mc13xxx-core.c
+++ b/drivers/mfd/mc13xxx-core.c
@@ -279,8 +279,21 @@ int mc13xxx_adc_do_conversion(struct mc13xxx *mc13xxx, unsigned int mode,
 	adc0 = MC13XXX_ADC0_ADINC1 | MC13XXX_ADC0_ADINC2;
 	adc1 = MC13XXX_ADC1_ADEN | MC13XXX_ADC1_ADTRIGIGN | MC13XXX_ADC1_ASC;
 
-	if (channel > 7)
+	/*
+	 * Channels mapped through ADIN7:
+	 * 7  - General purpose ADIN7
+	 * 16 - UID
+	 * 17 - Die temperature
+	 */
+	if (channel > 7 && channel < 16) {
 		adc1 |= MC13XXX_ADC1_ADSEL;
+	} else if (channel == 16) {
+		adc0 |= MC13XXX_ADC0_ADIN7SEL_UID;
+		channel = 7;
+	} else if (channel == 17) {
+		adc0 |= MC13XXX_ADC0_ADIN7SEL_DIE;
+		channel = 7;
+	}
 
 	switch (mode) {
 	case MC13XXX_ADC_MODE_TS:
diff --git a/drivers/mfd/mt6397-core.c b/drivers/mfd/mt6397-core.c
index 04a601f6aebe..77b64bd64df3 100644
--- a/drivers/mfd/mt6397-core.c
+++ b/drivers/mfd/mt6397-core.c
@@ -43,6 +43,16 @@ static const struct resource mt6397_rtc_resources[] = {
 	},
 };
 
+static const struct resource mt6323_keys_resources[] = {
+	DEFINE_RES_IRQ(MT6323_IRQ_STATUS_PWRKEY),
+	DEFINE_RES_IRQ(MT6323_IRQ_STATUS_FCHRKEY),
+};
+
+static const struct resource mt6397_keys_resources[] = {
+	DEFINE_RES_IRQ(MT6397_IRQ_PWRKEY),
+	DEFINE_RES_IRQ(MT6397_IRQ_HOMEKEY),
+};
+
 static const struct mfd_cell mt6323_devs[] = {
 	{
 		.name = "mt6323-regulator",
@@ -50,6 +60,11 @@ static const struct mfd_cell mt6323_devs[] = {
 	}, {
 		.name = "mt6323-led",
 		.of_compatible = "mediatek,mt6323-led"
+	}, {
+		.name = "mtk-pmic-keys",
+		.num_resources = ARRAY_SIZE(mt6323_keys_resources),
+		.resources = mt6323_keys_resources,
+		.of_compatible = "mediatek,mt6323-keys"
 	},
 };
 
@@ -71,7 +86,12 @@ static const struct mfd_cell mt6397_devs[] = {
 	}, {
 		.name = "mt6397-pinctrl",
 		.of_compatible = "mediatek,mt6397-pinctrl",
-	},
+	}, {
+		.name = "mtk-pmic-keys",
+		.num_resources = ARRAY_SIZE(mt6397_keys_resources),
+		.resources = mt6397_keys_resources,
+		.of_compatible = "mediatek,mt6397-keys"
+	}
 };
 
 static void mt6397_irq_lock(struct irq_data *data)
@@ -289,7 +309,7 @@ static int mt6397_probe(struct platform_device *pdev)
 
 		ret = devm_mfd_add_devices(&pdev->dev, -1, mt6323_devs,
 					   ARRAY_SIZE(mt6323_devs), NULL,
-					   0, NULL);
+					   0, pmic->irq_domain);
 		break;
 
 	case MT6397_CID_CODE:
@@ -304,7 +324,7 @@ static int mt6397_probe(struct platform_device *pdev)
 
 		ret = devm_mfd_add_devices(&pdev->dev, -1, mt6397_devs,
 					   ARRAY_SIZE(mt6397_devs), NULL,
-					   0, NULL);
+					   0, pmic->irq_domain);
 		break;
 
 	default:
diff --git a/drivers/mfd/omap-usb-tll.c b/drivers/mfd/omap-usb-tll.c
index 44a5d66314c6..446713dbee27 100644
--- a/drivers/mfd/omap-usb-tll.c
+++ b/drivers/mfd/omap-usb-tll.c
@@ -108,9 +108,9 @@
 					 (x) != OMAP_EHCI_PORT_MODE_PHY)
 
 struct usbtll_omap {
-	int					nch;	/* num. of channels */
-	struct clk				**ch_clk;
-	void __iomem				*base;
+	void __iomem	*base;
+	int		nch;		/* num. of channels */
+	struct clk	*ch_clk[0];	/* must be the last member */
 };
 
 /*-------------------------------------------------------------------------*/
@@ -216,53 +216,49 @@ static int usbtll_omap_probe(struct platform_device *pdev)
 	struct device				*dev =  &pdev->dev;
 	struct resource				*res;
 	struct usbtll_omap			*tll;
-	int					ret = 0;
-	int					i, ver;
+	void __iomem				*base;
+	int					i, nch, ver;
 
 	dev_dbg(dev, "starting TI HSUSB TLL Controller\n");
 
-	tll = devm_kzalloc(dev, sizeof(struct usbtll_omap), GFP_KERNEL);
-	if (!tll) {
-		dev_err(dev, "Memory allocation failed\n");
-		return -ENOMEM;
-	}
-
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	tll->base = devm_ioremap_resource(dev, res);
-	if (IS_ERR(tll->base))
-		return PTR_ERR(tll->base);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
 
-	platform_set_drvdata(pdev, tll);
 	pm_runtime_enable(dev);
 	pm_runtime_get_sync(dev);
 
-	ver =  usbtll_read(tll->base, OMAP_USBTLL_REVISION);
+	ver = usbtll_read(base, OMAP_USBTLL_REVISION);
 	switch (ver) {
 	case OMAP_USBTLL_REV1:
 	case OMAP_USBTLL_REV4:
-		tll->nch = OMAP_TLL_CHANNEL_COUNT;
+		nch = OMAP_TLL_CHANNEL_COUNT;
 		break;
 	case OMAP_USBTLL_REV2:
 	case OMAP_USBTLL_REV3:
-		tll->nch = OMAP_REV2_TLL_CHANNEL_COUNT;
+		nch = OMAP_REV2_TLL_CHANNEL_COUNT;
 		break;
 	default:
-		tll->nch = OMAP_TLL_CHANNEL_COUNT;
-		dev_dbg(dev,
-		 "USB TLL Rev : 0x%x not recognized, assuming %d channels\n",
-			ver, tll->nch);
+		nch = OMAP_TLL_CHANNEL_COUNT;
+		dev_dbg(dev, "rev 0x%x not recognized, assuming %d channels\n",
+			ver, nch);
 		break;
 	}
 
-	tll->ch_clk = devm_kzalloc(dev, sizeof(struct clk *) * tll->nch,
-						GFP_KERNEL);
-	if (!tll->ch_clk) {
-		ret = -ENOMEM;
-		dev_err(dev, "Couldn't allocate memory for channel clocks\n");
-		goto err_clk_alloc;
+	tll = devm_kzalloc(dev, sizeof(*tll) + sizeof(tll->ch_clk[nch]),
+			   GFP_KERNEL);
+	if (!tll) {
+		pm_runtime_put_sync(dev);
+		pm_runtime_disable(dev);
+		return -ENOMEM;
 	}
 
-	for (i = 0; i < tll->nch; i++) {
+	tll->base = base;
+	tll->nch = nch;
+	platform_set_drvdata(pdev, tll);
+
+	for (i = 0; i < nch; i++) {
 		char clkname[] = "usb_tll_hs_usb_chx_clk";
 
 		snprintf(clkname, sizeof(clkname),
@@ -282,12 +278,6 @@ static int usbtll_omap_probe(struct platform_device *pdev)
 	spin_unlock(&tll_lock);
 
 	return 0;
-
-err_clk_alloc:
-	pm_runtime_put_sync(dev);
-	pm_runtime_disable(dev);
-
-	return ret;
 }
 
 /**
diff --git a/drivers/mfd/pcf50633-core.c b/drivers/mfd/pcf50633-core.c
index f952dff6765f..0d2a88d53eed 100644
--- a/drivers/mfd/pcf50633-core.c
+++ b/drivers/mfd/pcf50633-core.c
@@ -242,8 +242,10 @@ static int pcf50633_probe(struct i2c_client *client,
 
 	for (i = 0; i < PCF50633_NUM_REGULATORS; i++) {
 		pdev = platform_device_alloc("pcf50633-regulator", i);
-		if (!pdev)
-			return -ENOMEM;
+		if (!pdev) {
+			ret = -ENOMEM;
+			goto err2;
+		}
 
 		pdev->dev.parent = pcf->dev;
 		ret = platform_device_add_data(pdev, &pdata->reg_init_data[i],
@@ -269,6 +271,7 @@ static int pcf50633_probe(struct i2c_client *client,
 
 err:
 	platform_device_put(pdev);
+err2:
 	for (j = 0; j < i; j++)
 		platform_device_put(pcf->regulator_pdev[j]);
 
diff --git a/drivers/mfd/rave-sp.c b/drivers/mfd/rave-sp.c
index 5c858e784a89..36dcd98977d6 100644
--- a/drivers/mfd/rave-sp.c
+++ b/drivers/mfd/rave-sp.c
@@ -45,7 +45,9 @@
 #define RAVE_SP_DLE			0x10
 
 #define RAVE_SP_MAX_DATA_SIZE		64
-#define RAVE_SP_CHECKSUM_SIZE		2  /* Worst case scenario on RDU2 */
+#define RAVE_SP_CHECKSUM_8B2C		1
+#define RAVE_SP_CHECKSUM_CCITT		2
+#define RAVE_SP_CHECKSUM_SIZE		RAVE_SP_CHECKSUM_CCITT
 /*
  * We don't store STX, ETX and unescaped bytes, so Rx is only
  * DATA + CSUM
@@ -160,6 +162,8 @@ struct rave_sp_variant {
  * @variant:			Device variant specific information
  * @event_notifier_list:	Input event notification chain
  *
+ * @part_number_firmware:	Firmware version
+ * @part_number_bootloader:	Bootloader version
  */
 struct rave_sp {
 	struct serdev_device *serdev;
@@ -171,8 +175,40 @@ struct rave_sp {
 
 	const struct rave_sp_variant *variant;
 	struct blocking_notifier_head event_notifier_list;
+
+	const char *part_number_firmware;
+	const char *part_number_bootloader;
 };
 
+struct rave_sp_version {
+	u8     hardware;
+	__le16 major;
+	u8     minor;
+	u8     letter[2];
+} __packed;
+
+struct rave_sp_status {
+	struct rave_sp_version bootloader_version;
+	struct rave_sp_version firmware_version;
+	u16 rdu_eeprom_flag;
+	u16 dds_eeprom_flag;
+	u8  pic_flag;
+	u8  orientation;
+	u32 etc;
+	s16 temp[2];
+	u8  backlight_current[3];
+	u8  dip_switch;
+	u8  host_interrupt;
+	u16 voltage_28;
+	u8  i2c_device_status;
+	u8  power_status;
+	u8  general_status;
+	u8  deprecated1;
+	u8  power_led_status;
+	u8  deprecated2;
+	u8  periph_power_shutoff;
+} __packed;
+
 static bool rave_sp_id_is_event(u8 code)
 {
 	return (code & 0xF0) == RAVE_SP_EVNT_BASE;
@@ -275,8 +311,8 @@ static int rave_sp_write(struct rave_sp *sp, const u8 *data, u8 data_size)
 
 	length = dest - frame;
 
-	print_hex_dump(KERN_DEBUG, "rave-sp tx: ", DUMP_PREFIX_NONE,
-		       16, 1, frame, length, false);
+	print_hex_dump_debug("rave-sp tx: ", DUMP_PREFIX_NONE,
+			     16, 1, frame, length, false);
 
 	return serdev_device_write(sp->serdev, frame, length, HZ);
 }
@@ -415,10 +451,15 @@ static void rave_sp_receive_frame(struct rave_sp *sp,
 	const size_t payload_length  = length - checksum_length;
 	const u8 *crc_reported       = &data[payload_length];
 	struct device *dev           = &sp->serdev->dev;
-	u8 crc_calculated[checksum_length];
+	u8 crc_calculated[RAVE_SP_CHECKSUM_SIZE];
+
+	if (unlikely(checksum_length > sizeof(crc_calculated))) {
+		dev_warn(dev, "Checksum too long, dropping\n");
+		return;
+	}
 
-	print_hex_dump(KERN_DEBUG, "rave-sp rx: ", DUMP_PREFIX_NONE,
-		       16, 1, data, length, false);
+	print_hex_dump_debug("rave-sp rx: ", DUMP_PREFIX_NONE,
+			     16, 1, data, length, false);
 
 	if (unlikely(length <= checksum_length)) {
 		dev_warn(dev, "Dropping short frame\n");
@@ -512,8 +553,6 @@ static int rave_sp_receive_buf(struct serdev_device *serdev,
 			/* FALLTHROUGH */
 
 		case RAVE_SP_EXPECT_ESCAPED_DATA:
-			deframer->data[deframer->length++] = byte;
-
 			if (deframer->length == sizeof(deframer->data)) {
 				dev_warn(dev, "Bad frame: Too long\n");
 				/*
@@ -528,6 +567,8 @@ static int rave_sp_receive_buf(struct serdev_device *serdev,
 				goto reset_framer;
 			}
 
+			deframer->data[deframer->length++] = byte;
+
 			/*
 			 * We've extracted out special byte, now we
 			 * can go back to regular data collecting
@@ -609,6 +650,52 @@ static int rave_sp_default_cmd_translate(enum rave_sp_command command)
 	}
 }
 
+static const char *devm_rave_sp_version(struct device *dev,
+					struct rave_sp_version *version)
+{
+	/*
+	 * NOTE: The format string below uses %02d to display u16
+	 * intentionally for the sake of backwards compatibility with
+	 * legacy software.
+	 */
+	return devm_kasprintf(dev, GFP_KERNEL, "%02d%02d%02d.%c%c\n",
+			      version->hardware,
+			      le16_to_cpu(version->major),
+			      version->minor,
+			      version->letter[0],
+			      version->letter[1]);
+}
+
+static int rave_sp_get_status(struct rave_sp *sp)
+{
+	struct device *dev = &sp->serdev->dev;
+	u8 cmd[] = {
+		[0] = RAVE_SP_CMD_STATUS,
+		[1] = 0
+	};
+	struct rave_sp_status status;
+	const char *version;
+	int ret;
+
+	ret = rave_sp_exec(sp, cmd, sizeof(cmd), &status, sizeof(status));
+	if (ret)
+		return ret;
+
+	version = devm_rave_sp_version(dev, &status.firmware_version);
+	if (!version)
+		return -ENOMEM;
+
+	sp->part_number_firmware = version;
+
+	version = devm_rave_sp_version(dev, &status.bootloader_version);
+	if (!version)
+		return -ENOMEM;
+
+	sp->part_number_bootloader = version;
+
+	return 0;
+}
+
 static const struct rave_sp_checksum rave_sp_checksum_8b2c = {
 	.length     = 1,
 	.subroutine = csum_8b2c,
@@ -657,6 +744,7 @@ static const struct serdev_device_ops rave_sp_serdev_device_ops = {
 static int rave_sp_probe(struct serdev_device *serdev)
 {
 	struct device *dev = &serdev->dev;
+	const char *unknown = "unknown\n";
 	struct rave_sp *sp;
 	u32 baud;
 	int ret;
@@ -689,6 +777,20 @@ static int rave_sp_probe(struct serdev_device *serdev)
 
 	serdev_device_set_baudrate(serdev, baud);
 
+	ret = rave_sp_get_status(sp);
+	if (ret) {
+		dev_warn(dev, "Failed to get firmware status: %d\n", ret);
+		sp->part_number_firmware   = unknown;
+		sp->part_number_bootloader = unknown;
+	}
+
+	/*
+	 * Those strings already have a \n embedded, so there's no
+	 * need to have one in format string.
+	 */
+	dev_info(dev, "Firmware version: %s",   sp->part_number_firmware);
+	dev_info(dev, "Bootloader version: %s", sp->part_number_bootloader);
+
 	return devm_of_platform_populate(dev);
 }
 
diff --git a/drivers/mfd/rc5t583.c b/drivers/mfd/rc5t583.c
index d12243d5ecb8..fd46de02b715 100644
--- a/drivers/mfd/rc5t583.c
+++ b/drivers/mfd/rc5t583.c
@@ -258,11 +258,9 @@ static int rc5t583_i2c_probe(struct i2c_client *i2c,
 		return -EINVAL;
 	}
 
-	rc5t583 = devm_kzalloc(&i2c->dev, sizeof(struct rc5t583), GFP_KERNEL);
-	if (!rc5t583) {
-		dev_err(&i2c->dev, "Memory allocation failed\n");
+	rc5t583 = devm_kzalloc(&i2c->dev, sizeof(*rc5t583), GFP_KERNEL);
+	if (!rc5t583)
 		return -ENOMEM;
-	}
 
 	rc5t583->dev = &i2c->dev;
 	i2c_set_clientdata(i2c, rc5t583);
diff --git a/drivers/mfd/si476x-i2c.c b/drivers/mfd/si476x-i2c.c
index e6a3d999a376..2c5ec93333c3 100644
--- a/drivers/mfd/si476x-i2c.c
+++ b/drivers/mfd/si476x-i2c.c
@@ -697,11 +697,9 @@ static int si476x_core_probe(struct i2c_client *client,
 	int              cell_num;
 
 	core = devm_kzalloc(&client->dev, sizeof(*core), GFP_KERNEL);
-	if (!core) {
-		dev_err(&client->dev,
-			"failed to allocate 'struct si476x_core'\n");
+	if (!core)
 		return -ENOMEM;
-	}
+
 	core->client = client;
 
 	core->regmap = devm_regmap_init_si476x(core);
diff --git a/drivers/mfd/sm501.c b/drivers/mfd/sm501.c
index ad774161a22d..2a87b0d2f21f 100644
--- a/drivers/mfd/sm501.c
+++ b/drivers/mfd/sm501.c
@@ -19,7 +19,7 @@
 #include <linux/device.h>
 #include <linux/platform_device.h>
 #include <linux/pci.h>
-#include <linux/i2c-gpio.h>
+#include <linux/platform_data/i2c-gpio.h>
 #include <linux/gpio/machine.h>
 #include <linux/slab.h>
 
@@ -1050,13 +1050,13 @@ static int sm501_register_gpio(struct sm501_devdata *sm)
 	spin_lock_init(&gpio->lock);
 
 	gpio->regs_res = request_mem_region(iobase, 0x20, "sm501-gpio");
-	if (gpio->regs_res == NULL) {
+	if (!gpio->regs_res) {
 		dev_err(sm->dev, "gpio: failed to request region\n");
 		return -ENXIO;
 	}
 
 	gpio->regs = ioremap(iobase, 0x20);
-	if (gpio->regs == NULL) {
+	if (!gpio->regs) {
 		dev_err(sm->dev, "gpio: failed to remap registers\n");
 		ret = -ENXIO;
 		goto err_claimed;
@@ -1358,7 +1358,7 @@ static int sm501_init_dev(struct sm501_devdata *sm)
 			sm501_register_gpio(sm);
 	}
 
-	if (pdata && pdata->gpio_i2c != NULL && pdata->gpio_i2c_nr > 0) {
+	if (pdata && pdata->gpio_i2c && pdata->gpio_i2c_nr > 0) {
 		if (!sm501_gpio_isregistered(sm))
 			dev_err(sm->dev, "no gpio available for i2c gpio.\n");
 		else
@@ -1383,9 +1383,8 @@ static int sm501_plat_probe(struct platform_device *dev)
 	struct sm501_devdata *sm;
 	int ret;
 
-	sm = kzalloc(sizeof(struct sm501_devdata), GFP_KERNEL);
-	if (sm == NULL) {
-		dev_err(&dev->dev, "no memory for device data\n");
+	sm = kzalloc(sizeof(*sm), GFP_KERNEL);
+	if (!sm) {
 		ret = -ENOMEM;
 		goto err1;
 	}
@@ -1403,8 +1402,7 @@ static int sm501_plat_probe(struct platform_device *dev)
 
 	sm->io_res = platform_get_resource(dev, IORESOURCE_MEM, 1);
 	sm->mem_res = platform_get_resource(dev, IORESOURCE_MEM, 0);
-
-	if (sm->io_res == NULL || sm->mem_res == NULL) {
+	if (!sm->io_res || !sm->mem_res) {
 		dev_err(&dev->dev, "failed to get IO resource\n");
 		ret = -ENOENT;
 		goto err_res;
@@ -1412,8 +1410,7 @@ static int sm501_plat_probe(struct platform_device *dev)
 
 	sm->regs_claim = request_mem_region(sm->io_res->start,
 					    0x100, "sm501");
-
-	if (sm->regs_claim == NULL) {
+	if (!sm->regs_claim) {
 		dev_err(&dev->dev, "cannot claim registers\n");
 		ret = -EBUSY;
 		goto err_res;
@@ -1422,8 +1419,7 @@ static int sm501_plat_probe(struct platform_device *dev)
 	platform_set_drvdata(dev, sm);
 
 	sm->regs = ioremap(sm->io_res->start, resource_size(sm->io_res));
-
-	if (sm->regs == NULL) {
+	if (!sm->regs) {
 		dev_err(&dev->dev, "cannot remap registers\n");
 		ret = -EIO;
 		goto err_claim;
@@ -1449,7 +1445,7 @@ static void sm501_set_power(struct sm501_devdata *sm, int on)
 {
 	struct sm501_platdata *pd = sm->platdata;
 
-	if (pd == NULL)
+	if (!pd)
 		return;
 
 	if (pd->get_power) {
@@ -1573,9 +1569,8 @@ static int sm501_pci_probe(struct pci_dev *dev,
 	struct sm501_devdata *sm;
 	int err;
 
-	sm = kzalloc(sizeof(struct sm501_devdata), GFP_KERNEL);
-	if (sm == NULL) {
-		dev_err(&dev->dev, "no memory for device data\n");
+	sm = kzalloc(sizeof(*sm), GFP_KERNEL);
+	if (!sm) {
 		err = -ENOMEM;
 		goto err1;
 	}
@@ -1626,15 +1621,14 @@ static int sm501_pci_probe(struct pci_dev *dev,
 
 	sm->regs_claim = request_mem_region(sm->io_res->start,
 					    0x100, "sm501");
-	if (sm->regs_claim == NULL) {
+	if (!sm->regs_claim) {
 		dev_err(&dev->dev, "cannot claim registers\n");
 		err= -EBUSY;
 		goto err3;
 	}
 
 	sm->regs = pci_ioremap_bar(dev, 1);
-
-	if (sm->regs == NULL) {
+	if (!sm->regs) {
 		dev_err(&dev->dev, "cannot remap registers\n");
 		err = -EIO;
 		goto err4;
diff --git a/drivers/mfd/smsc-ece1099.c b/drivers/mfd/smsc-ece1099.c
index 93a8297de52a..57b792eb58fd 100644
--- a/drivers/mfd/smsc-ece1099.c
+++ b/drivers/mfd/smsc-ece1099.c
@@ -37,12 +37,9 @@ static int smsc_i2c_probe(struct i2c_client *i2c,
 	int devid, rev, venid_l, venid_h;
 	int ret;
 
-	smsc = devm_kzalloc(&i2c->dev, sizeof(struct smsc),
-				GFP_KERNEL);
-	if (!smsc) {
-		dev_err(&i2c->dev, "smsc mfd driver memory allocation failed\n");
+	smsc = devm_kzalloc(&i2c->dev, sizeof(*smsc), GFP_KERNEL);
+	if (!smsc)
 		return -ENOMEM;
-	}
 
 	smsc->regmap = devm_regmap_init_i2c(i2c, &smsc_regmap_config);
 	if (IS_ERR(smsc->regmap))
diff --git a/drivers/mfd/sprd-sc27xx-spi.c b/drivers/mfd/sprd-sc27xx-spi.c
index 56a4782f0569..3460ef07623c 100644
--- a/drivers/mfd/sprd-sc27xx-spi.c
+++ b/drivers/mfd/sprd-sc27xx-spi.c
@@ -111,6 +111,9 @@ static const struct mfd_cell sprd_pmic_devs[] = {
 	}, {
 		.name = "sc27xx-poweroff",
 		.of_compatible = "sprd,sc27xx-poweroff",
+	}, {
+		.name = "sc27xx-syscon",
+		.of_compatible = "sprd,sc27xx-syscon",
 	},
 };
 
diff --git a/drivers/mfd/stm32-timers.c b/drivers/mfd/stm32-timers.c
index 1d347e5dfa79..efcd4b980c94 100644
--- a/drivers/mfd/stm32-timers.c
+++ b/drivers/mfd/stm32-timers.c
@@ -4,16 +4,156 @@
  * Author: Benjamin Gaignard <benjamin.gaignard@st.com>
  */
 
+#include <linux/bitfield.h>
 #include <linux/mfd/stm32-timers.h>
 #include <linux/module.h>
 #include <linux/of_platform.h>
 #include <linux/reset.h>
 
+#define STM32_TIMERS_MAX_REGISTERS	0x3fc
+
+/* DIER register DMA enable bits */
+static const u32 stm32_timers_dier_dmaen[STM32_TIMERS_MAX_DMAS] = {
+	TIM_DIER_CC1DE,
+	TIM_DIER_CC2DE,
+	TIM_DIER_CC3DE,
+	TIM_DIER_CC4DE,
+	TIM_DIER_UIE,
+	TIM_DIER_TDE,
+	TIM_DIER_COMDE
+};
+
+static void stm32_timers_dma_done(void *p)
+{
+	struct stm32_timers_dma *dma = p;
+	struct dma_tx_state state;
+	enum dma_status status;
+
+	status = dmaengine_tx_status(dma->chan, dma->chan->cookie, &state);
+	if (status == DMA_COMPLETE)
+		complete(&dma->completion);
+}
+
+/**
+ * stm32_timers_dma_burst_read - Read from timers registers using DMA.
+ *
+ * Read from STM32 timers registers using DMA on a single event.
+ * @dev: reference to stm32_timers MFD device
+ * @buf: DMA'able destination buffer
+ * @id: stm32_timers_dmas event identifier (ch[1..4], up, trig or com)
+ * @reg: registers start offset for DMA to read from (like CCRx for capture)
+ * @num_reg: number of registers to read upon each DMA request, starting @reg.
+ * @bursts: number of bursts to read (e.g. like two for pwm period capture)
+ * @tmo_ms: timeout (milliseconds)
+ */
+int stm32_timers_dma_burst_read(struct device *dev, u32 *buf,
+				enum stm32_timers_dmas id, u32 reg,
+				unsigned int num_reg, unsigned int bursts,
+				unsigned long tmo_ms)
+{
+	struct stm32_timers *ddata = dev_get_drvdata(dev);
+	unsigned long timeout = msecs_to_jiffies(tmo_ms);
+	struct regmap *regmap = ddata->regmap;
+	struct stm32_timers_dma *dma = &ddata->dma;
+	size_t len = num_reg * bursts * sizeof(u32);
+	struct dma_async_tx_descriptor *desc;
+	struct dma_slave_config config;
+	dma_cookie_t cookie;
+	dma_addr_t dma_buf;
+	u32 dbl, dba;
+	long err;
+	int ret;
+
+	/* Sanity check */
+	if (id < STM32_TIMERS_DMA_CH1 || id >= STM32_TIMERS_MAX_DMAS)
+		return -EINVAL;
+
+	if (!num_reg || !bursts || reg > STM32_TIMERS_MAX_REGISTERS ||
+	    (reg + num_reg * sizeof(u32)) > STM32_TIMERS_MAX_REGISTERS)
+		return -EINVAL;
+
+	if (!dma->chans[id])
+		return -ENODEV;
+	mutex_lock(&dma->lock);
+
+	/* Select DMA channel in use */
+	dma->chan = dma->chans[id];
+	dma_buf = dma_map_single(dev, buf, len, DMA_FROM_DEVICE);
+	if (dma_mapping_error(dev, dma_buf)) {
+		ret = -ENOMEM;
+		goto unlock;
+	}
+
+	/* Prepare DMA read from timer registers, using DMA burst mode */
+	memset(&config, 0, sizeof(config));
+	config.src_addr = (dma_addr_t)dma->phys_base + TIM_DMAR;
+	config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	ret = dmaengine_slave_config(dma->chan, &config);
+	if (ret)
+		goto unmap;
+
+	desc = dmaengine_prep_slave_single(dma->chan, dma_buf, len,
+					   DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT);
+	if (!desc) {
+		ret = -EBUSY;
+		goto unmap;
+	}
+
+	desc->callback = stm32_timers_dma_done;
+	desc->callback_param = dma;
+	cookie = dmaengine_submit(desc);
+	ret = dma_submit_error(cookie);
+	if (ret)
+		goto dma_term;
+
+	reinit_completion(&dma->completion);
+	dma_async_issue_pending(dma->chan);
+
+	/* Setup and enable timer DMA burst mode */
+	dbl = FIELD_PREP(TIM_DCR_DBL, bursts - 1);
+	dba = FIELD_PREP(TIM_DCR_DBA, reg >> 2);
+	ret = regmap_write(regmap, TIM_DCR, dbl | dba);
+	if (ret)
+		goto dma_term;
+
+	/* Clear pending flags before enabling DMA request */
+	ret = regmap_write(regmap, TIM_SR, 0);
+	if (ret)
+		goto dcr_clr;
+
+	ret = regmap_update_bits(regmap, TIM_DIER, stm32_timers_dier_dmaen[id],
+				 stm32_timers_dier_dmaen[id]);
+	if (ret)
+		goto dcr_clr;
+
+	err = wait_for_completion_interruptible_timeout(&dma->completion,
+							timeout);
+	if (err == 0)
+		ret = -ETIMEDOUT;
+	else if (err < 0)
+		ret = err;
+
+	regmap_update_bits(regmap, TIM_DIER, stm32_timers_dier_dmaen[id], 0);
+	regmap_write(regmap, TIM_SR, 0);
+dcr_clr:
+	regmap_write(regmap, TIM_DCR, 0);
+dma_term:
+	dmaengine_terminate_all(dma->chan);
+unmap:
+	dma_unmap_single(dev, dma_buf, len, DMA_FROM_DEVICE);
+unlock:
+	dma->chan = NULL;
+	mutex_unlock(&dma->lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(stm32_timers_dma_burst_read);
+
 static const struct regmap_config stm32_timers_regmap_cfg = {
 	.reg_bits = 32,
 	.val_bits = 32,
 	.reg_stride = sizeof(u32),
-	.max_register = 0x3fc,
+	.max_register = STM32_TIMERS_MAX_REGISTERS,
 };
 
 static void stm32_timers_get_arr_size(struct stm32_timers *ddata)
@@ -27,12 +167,45 @@ static void stm32_timers_get_arr_size(struct stm32_timers *ddata)
 	regmap_write(ddata->regmap, TIM_ARR, 0x0);
 }
 
+static void stm32_timers_dma_probe(struct device *dev,
+				   struct stm32_timers *ddata)
+{
+	int i;
+	char name[4];
+
+	init_completion(&ddata->dma.completion);
+	mutex_init(&ddata->dma.lock);
+
+	/* Optional DMA support: get valid DMA channel(s) or NULL */
+	for (i = STM32_TIMERS_DMA_CH1; i <= STM32_TIMERS_DMA_CH4; i++) {
+		snprintf(name, ARRAY_SIZE(name), "ch%1d", i + 1);
+		ddata->dma.chans[i] = dma_request_slave_channel(dev, name);
+	}
+	ddata->dma.chans[STM32_TIMERS_DMA_UP] =
+		dma_request_slave_channel(dev, "up");
+	ddata->dma.chans[STM32_TIMERS_DMA_TRIG] =
+		dma_request_slave_channel(dev, "trig");
+	ddata->dma.chans[STM32_TIMERS_DMA_COM] =
+		dma_request_slave_channel(dev, "com");
+}
+
+static void stm32_timers_dma_remove(struct device *dev,
+				    struct stm32_timers *ddata)
+{
+	int i;
+
+	for (i = STM32_TIMERS_DMA_CH1; i < STM32_TIMERS_MAX_DMAS; i++)
+		if (ddata->dma.chans[i])
+			dma_release_channel(ddata->dma.chans[i]);
+}
+
 static int stm32_timers_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct stm32_timers *ddata;
 	struct resource *res;
 	void __iomem *mmio;
+	int ret;
 
 	ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL);
 	if (!ddata)
@@ -43,6 +216,9 @@ static int stm32_timers_probe(struct platform_device *pdev)
 	if (IS_ERR(mmio))
 		return PTR_ERR(mmio);
 
+	/* Timer physical addr for DMA */
+	ddata->dma.phys_base = res->start;
+
 	ddata->regmap = devm_regmap_init_mmio_clk(dev, "int", mmio,
 						  &stm32_timers_regmap_cfg);
 	if (IS_ERR(ddata->regmap))
@@ -54,9 +230,29 @@ static int stm32_timers_probe(struct platform_device *pdev)
 
 	stm32_timers_get_arr_size(ddata);
 
+	stm32_timers_dma_probe(dev, ddata);
+
 	platform_set_drvdata(pdev, ddata);
 
-	return devm_of_platform_populate(&pdev->dev);
+	ret = of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev);
+	if (ret)
+		stm32_timers_dma_remove(dev, ddata);
+
+	return ret;
+}
+
+static int stm32_timers_remove(struct platform_device *pdev)
+{
+	struct stm32_timers *ddata = platform_get_drvdata(pdev);
+
+	/*
+	 * Don't use devm_ here: enfore of_platform_depopulate() happens before
+	 * DMA are released, to avoid race on DMA.
+	 */
+	of_platform_depopulate(&pdev->dev);
+	stm32_timers_dma_remove(&pdev->dev, ddata);
+
+	return 0;
 }
 
 static const struct of_device_id stm32_timers_of_match[] = {
@@ -67,6 +263,7 @@ MODULE_DEVICE_TABLE(of, stm32_timers_of_match);
 
 static struct platform_driver stm32_timers_driver = {
 	.probe = stm32_timers_probe,
+	.remove = stm32_timers_remove,
 	.driver	= {
 		.name = "stm32-timers",
 		.of_match_table = stm32_timers_of_match,
diff --git a/drivers/mfd/syscon.c b/drivers/mfd/syscon.c
index 7eaa40bc703f..b6d05cd934e6 100644
--- a/drivers/mfd/syscon.c
+++ b/drivers/mfd/syscon.c
@@ -106,9 +106,11 @@ static struct syscon *of_syscon_register(struct device_node *np)
 		}
 	}
 
+	syscon_config.name = of_node_full_name(np);
 	syscon_config.reg_stride = reg_io_width;
 	syscon_config.val_bits = reg_io_width * 8;
 	syscon_config.max_register = resource_size(&res) - reg_io_width;
+	syscon_config.name = of_node_full_name(np);
 
 	regmap = regmap_init_mmio(NULL, base, &syscon_config);
 	if (IS_ERR(regmap)) {
diff --git a/drivers/mfd/ti_am335x_tscadc.c b/drivers/mfd/ti_am335x_tscadc.c
index 3cd958a31f36..47012c0899cd 100644
--- a/drivers/mfd/ti_am335x_tscadc.c
+++ b/drivers/mfd/ti_am335x_tscadc.c
@@ -169,10 +169,9 @@ static	int ti_tscadc_probe(struct platform_device *pdev)
 
 	/* Allocate memory for device */
 	tscadc = devm_kzalloc(&pdev->dev, sizeof(*tscadc), GFP_KERNEL);
-	if (!tscadc) {
-		dev_err(&pdev->dev, "failed to allocate memory.\n");
+	if (!tscadc)
 		return -ENOMEM;
-	}
+
 	tscadc->dev = &pdev->dev;
 
 	err = platform_get_irq(pdev, 0);
diff --git a/drivers/mfd/timberdale.c b/drivers/mfd/timberdale.c
index cd4a6d7d6750..33abf5a79f2b 100644
--- a/drivers/mfd/timberdale.c
+++ b/drivers/mfd/timberdale.c
@@ -30,8 +30,8 @@
 #include <linux/timb_gpio.h>
 
 #include <linux/i2c.h>
-#include <linux/i2c-ocores.h>
-#include <linux/i2c-xiic.h>
+#include <linux/platform_data/i2c-ocores.h>
+#include <linux/platform_data/i2c-xiic.h>
 
 #include <linux/spi/spi.h>
 #include <linux/spi/xilinx_spi.h>
diff --git a/drivers/mfd/tps65090.c b/drivers/mfd/tps65090.c
index d7ec318c40c3..f13e4cd06e89 100644
--- a/drivers/mfd/tps65090.c
+++ b/drivers/mfd/tps65090.c
@@ -192,10 +192,8 @@ static int tps65090_i2c_probe(struct i2c_client *client,
 		irq_base = pdata->irq_base;
 
 	tps65090 = devm_kzalloc(&client->dev, sizeof(*tps65090), GFP_KERNEL);
-	if (!tps65090) {
-		dev_err(&client->dev, "mem alloc for tps65090 failed\n");
+	if (!tps65090)
 		return -ENOMEM;
-	}
 
 	tps65090->dev = &client->dev;
 	i2c_set_clientdata(client, tps65090);
diff --git a/drivers/mfd/tps6586x.c b/drivers/mfd/tps6586x.c
index 5628a6b5b19b..b89379782741 100644
--- a/drivers/mfd/tps6586x.c
+++ b/drivers/mfd/tps6586x.c
@@ -423,10 +423,8 @@ static struct tps6586x_platform_data *tps6586x_parse_dt(struct i2c_client *clien
 	struct tps6586x_platform_data *pdata;
 
 	pdata = devm_kzalloc(&client->dev, sizeof(*pdata), GFP_KERNEL);
-	if (!pdata) {
-		dev_err(&client->dev, "Memory allocation failed\n");
+	if (!pdata)
 		return NULL;
-	}
 
 	pdata->num_subdevs = 0;
 	pdata->subdevs = NULL;
diff --git a/drivers/mfd/tps65910.c b/drivers/mfd/tps65910.c
index 8263605f6d2f..bf16cbe6fd88 100644
--- a/drivers/mfd/tps65910.c
+++ b/drivers/mfd/tps65910.c
@@ -229,7 +229,7 @@ static struct regmap_irq_chip tps65910_irq_chip = {
 static int tps65910_irq_init(struct tps65910 *tps65910, int irq,
 		    struct tps65910_platform_data *pdata)
 {
-	int ret = 0;
+	int ret;
 	static struct regmap_irq_chip *tps6591x_irqs_chip;
 
 	if (!irq) {
@@ -312,14 +312,14 @@ static int tps65910_ck32k_init(struct tps65910 *tps65910,
 static int tps65910_sleepinit(struct tps65910 *tps65910,
 		struct tps65910_board *pmic_pdata)
 {
-	struct device *dev = NULL;
-	int ret = 0;
-
-	dev = tps65910->dev;
+	struct device *dev;
+	int ret;
 
 	if (!pmic_pdata->en_dev_slp)
 		return 0;
 
+	dev = tps65910->dev;
+
 	/* enabling SLEEP device state */
 	ret = tps65910_reg_set_bits(tps65910, TPS65910_DEVCTRL,
 				DEVCTRL_DEV_SLP_MASK);
@@ -383,7 +383,7 @@ static struct tps65910_board *tps65910_parse_dt(struct i2c_client *client,
 	struct tps65910_board *board_info;
 	unsigned int prop;
 	const struct of_device_id *match;
-	int ret = 0;
+	int ret;
 
 	match = of_match_device(tps65910_of_match, &client->dev);
 	if (!match) {
@@ -395,10 +395,8 @@ static struct tps65910_board *tps65910_parse_dt(struct i2c_client *client,
 
 	board_info = devm_kzalloc(&client->dev, sizeof(*board_info),
 			GFP_KERNEL);
-	if (!board_info) {
-		dev_err(&client->dev, "Failed to allocate pdata\n");
+	if (!board_info)
 		return NULL;
-	}
 
 	ret = of_property_read_u32(np, "ti,vmbch-threshold", &prop);
 	if (!ret)
@@ -462,7 +460,7 @@ static int tps65910_i2c_probe(struct i2c_client *i2c,
 	struct tps65910_board *of_pmic_plat_data = NULL;
 	struct tps65910_platform_data *init_data;
 	unsigned long chip_id = id->driver_data;
-	int ret = 0;
+	int ret;
 
 	pmic_plat_data = dev_get_platdata(&i2c->dev);
 
diff --git a/drivers/mfd/tps65911-comparator.c b/drivers/mfd/tps65911-comparator.c
index c0789f81a1c5..33591767fb9b 100644
--- a/drivers/mfd/tps65911-comparator.c
+++ b/drivers/mfd/tps65911-comparator.c
@@ -22,9 +22,8 @@
 #include <linux/gpio.h>
 #include <linux/mfd/tps65910.h>
 
-#define COMP					0
-#define COMP1					1
-#define COMP2					2
+#define COMP1					0
+#define COMP2					1
 
 /* Comparator 1 voltage selection table in millivolts */
 static const u16 COMP_VSEL_TABLE[] = {
@@ -63,9 +62,6 @@ static int comp_threshold_set(struct tps65910 *tps65910, int id, int voltage)
 	int ret;
 	u8 index = 0, val;
 
-	if (id == COMP)
-		return 0;
-
 	while (curr_voltage < tps_comp.uV_max) {
 		curr_voltage = tps_comp.vsel_table[index];
 		if (curr_voltage >= voltage)
@@ -78,7 +74,7 @@ static int comp_threshold_set(struct tps65910 *tps65910, int id, int voltage)
 		return -EINVAL;
 
 	val = index << 1;
-	ret = tps65910->write(tps65910, tps_comp.reg, 1, &val);
+	ret = tps65910_reg_write(tps65910, tps_comp.reg, val);
 
 	return ret;
 }
@@ -86,13 +82,10 @@ static int comp_threshold_set(struct tps65910 *tps65910, int id, int voltage)
 static int comp_threshold_get(struct tps65910 *tps65910, int id)
 {
 	struct comparator tps_comp = tps_comparators[id];
+	unsigned int val;
 	int ret;
-	u8 val;
-
-	if (id == COMP)
-		return 0;
 
-	ret = tps65910->read(tps65910, tps_comp.reg, 1, &val);
+	ret = tps65910_reg_read(tps65910, tps_comp.reg, &val);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/mfd/tps68470.c b/drivers/mfd/tps68470.c
index 189efaea054c..a5981a79b29a 100644
--- a/drivers/mfd/tps68470.c
+++ b/drivers/mfd/tps68470.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * TPS68470 chip Parent driver
  *
@@ -8,15 +9,6 @@
  *	Tianshu Qiu <tian.shu.qiu@intel.com>
  *	Jian Xu Zheng <jian.xu.zheng@intel.com>
  *	Yuning Pu <yuning.pu@intel.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/acpi.h>
diff --git a/drivers/mfd/tps80031.c b/drivers/mfd/tps80031.c
index 0812df3b0d47..608c7f77830e 100644
--- a/drivers/mfd/tps80031.c
+++ b/drivers/mfd/tps80031.c
@@ -431,10 +431,8 @@ static int tps80031_probe(struct i2c_client *client,
 	}
 
 	tps80031 = devm_kzalloc(&client->dev, sizeof(*tps80031), GFP_KERNEL);
-	if (!tps80031) {
-		dev_err(&client->dev, "Malloc failed for tps80031\n");
+	if (!tps80031)
 		return -ENOMEM;
-	}
 
 	for (i = 0; i < TPS80031_NUM_SLAVES; i++) {
 		if (tps80031_slave_address[i] == client->addr)
diff --git a/drivers/mfd/twl-core.c b/drivers/mfd/twl-core.c
index d3133a371e27..c649344fd7f2 100644
--- a/drivers/mfd/twl-core.c
+++ b/drivers/mfd/twl-core.c
@@ -1177,7 +1177,7 @@ twl_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	twl_priv->ready = true;
 
 	/* setup clock framework */
-	clocks_init(&pdev->dev, pdata ? pdata->clock : NULL);
+	clocks_init(&client->dev, pdata ? pdata->clock : NULL);
 
 	/* read TWL IDCODE Register */
 	if (twl_class_is_4030()) {
diff --git a/drivers/mfd/twl6030-irq.c b/drivers/mfd/twl6030-irq.c
index e3ec8dfa9f1e..e939431ed10c 100644
--- a/drivers/mfd/twl6030-irq.c
+++ b/drivers/mfd/twl6030-irq.c
@@ -392,10 +392,8 @@ int twl6030_init_irq(struct device *dev, int irq_num)
 	nr_irqs = TWL6030_NR_IRQS;
 
 	twl6030_irq = devm_kzalloc(dev, sizeof(*twl6030_irq), GFP_KERNEL);
-	if (!twl6030_irq) {
-		dev_err(dev, "twl6030_irq: Memory allocation failed\n");
+	if (!twl6030_irq)
 		return -ENOMEM;
-	}
 
 	mask[0] = 0xFF;
 	mask[1] = 0xFF;
diff --git a/drivers/mfd/viperboard.c b/drivers/mfd/viperboard.c
index e6b3c70aeb22..e9f61262d583 100644
--- a/drivers/mfd/viperboard.c
+++ b/drivers/mfd/viperboard.c
@@ -59,10 +59,8 @@ static int vprbrd_probe(struct usb_interface *interface,
 
 	/* allocate memory for our device state and initialize it */
 	vb = kzalloc(sizeof(*vb), GFP_KERNEL);
-	if (vb == NULL) {
-		dev_err(&interface->dev, "Out of memory\n");
+	if (!vb)
 		return -ENOMEM;
-	}
 
 	mutex_init(&vb->lock);
 
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index 5d713008749b..3726eacdf65d 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -113,6 +113,20 @@ config IBM_ASM
 	  for information on the specific driver level and support statement
 	  for your IBM server.
 
+config IBMVMC
+	tristate "IBM Virtual Management Channel support"
+	depends on PPC_PSERIES
+	help
+	  This is the IBM POWER Virtual Management Channel
+
+	  This driver is to be used for the POWER Virtual
+	  Management Channel virtual adapter on the PowerVM
+	  platform. It provides both request/response and
+	  async message support through the /dev/ibmvmc node.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ibmvmc.
+
 config PHANTOM
 	tristate "Sensable PHANToM (PCI)"
 	depends on PCI
diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile
index 20be70c3f118..af22bbc3d00c 100644
--- a/drivers/misc/Makefile
+++ b/drivers/misc/Makefile
@@ -4,6 +4,7 @@
 #
 
 obj-$(CONFIG_IBM_ASM)		+= ibmasm/
+obj-$(CONFIG_IBMVMC)		+= ibmvmc.o
 obj-$(CONFIG_AD525X_DPOT)	+= ad525x_dpot.o
 obj-$(CONFIG_AD525X_DPOT_I2C)	+= ad525x_dpot-i2c.o
 obj-$(CONFIG_AD525X_DPOT_SPI)	+= ad525x_dpot-spi.o
diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
index 7ff315ad3692..c6ec872800a2 100644
--- a/drivers/misc/cxl/context.c
+++ b/drivers/misc/cxl/context.c
@@ -128,11 +128,12 @@ void cxl_context_set_mapping(struct cxl_context *ctx,
 	mutex_unlock(&ctx->mapping_lock);
 }
 
-static int cxl_mmap_fault(struct vm_fault *vmf)
+static vm_fault_t cxl_mmap_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct cxl_context *ctx = vma->vm_file->private_data;
 	u64 area, offset;
+	vm_fault_t ret;
 
 	offset = vmf->pgoff << PAGE_SHIFT;
 
@@ -169,11 +170,11 @@ static int cxl_mmap_fault(struct vm_fault *vmf)
 		return VM_FAULT_SIGBUS;
 	}
 
-	vm_insert_pfn(vma, vmf->address, (area + offset) >> PAGE_SHIFT);
+	ret = vmf_insert_pfn(vma, vmf->address, (area + offset) >> PAGE_SHIFT);
 
 	mutex_unlock(&ctx->status_mutex);
 
-	return VM_FAULT_NOPAGE;
+	return ret;
 }
 
 static const struct vm_operations_struct cxl_mmap_vmops = {
diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c
index 33053b0d1fdf..f5cc517d1131 100644
--- a/drivers/misc/eeprom/at24.c
+++ b/drivers/misc/eeprom/at24.c
@@ -532,6 +532,45 @@ static int at24_get_pdata(struct device *dev, struct at24_platform_data *pdata)
 	return 0;
 }
 
+static void at24_remove_dummy_clients(struct at24_data *at24)
+{
+	int i;
+
+	for (i = 1; i < at24->num_addresses; i++)
+		i2c_unregister_device(at24->client[i].client);
+}
+
+static int at24_make_dummy_client(struct at24_data *at24, unsigned int index,
+				  struct regmap_config *regmap_config)
+{
+	struct i2c_client *base_client, *dummy_client;
+	unsigned short int addr;
+	struct regmap *regmap;
+	struct device *dev;
+
+	base_client = at24->client[0].client;
+	dev = &base_client->dev;
+	addr = base_client->addr + index;
+
+	dummy_client = i2c_new_dummy(base_client->adapter,
+				     base_client->addr + index);
+	if (!dummy_client) {
+		dev_err(dev, "address 0x%02x unavailable\n", addr);
+		return -EADDRINUSE;
+	}
+
+	regmap = devm_regmap_init_i2c(dummy_client, regmap_config);
+	if (IS_ERR(regmap)) {
+		i2c_unregister_device(dummy_client);
+		return PTR_ERR(regmap);
+	}
+
+	at24->client[index].client = dummy_client;
+	at24->client[index].regmap = regmap;
+
+	return 0;
+}
+
 static unsigned int at24_get_offset_adj(u8 flags, unsigned int byte_len)
 {
 	if (flags & AT24_FLAG_MAC) {
@@ -637,20 +676,10 @@ static int at24_probe(struct i2c_client *client)
 
 	/* use dummy devices for multiple-address chips */
 	for (i = 1; i < num_addresses; i++) {
-		at24->client[i].client = i2c_new_dummy(client->adapter,
-						       client->addr + i);
-		if (!at24->client[i].client) {
-			dev_err(dev, "address 0x%02x unavailable\n",
-				client->addr + i);
-			err = -EADDRINUSE;
-			goto err_clients;
-		}
-		at24->client[i].regmap = devm_regmap_init_i2c(
-						at24->client[i].client,
-						&regmap_config);
-		if (IS_ERR(at24->client[i].regmap)) {
-			err = PTR_ERR(at24->client[i].regmap);
-			goto err_clients;
+		err = at24_make_dummy_client(at24, i, &regmap_config);
+		if (err) {
+			at24_remove_dummy_clients(at24);
+			return err;
 		}
 	}
 
@@ -685,7 +714,7 @@ static int at24_probe(struct i2c_client *client)
 	nvmem_config.word_size = 1;
 	nvmem_config.size = pdata.byte_len;
 
-	at24->nvmem = nvmem_register(&nvmem_config);
+	at24->nvmem = devm_nvmem_register(dev, &nvmem_config);
 	if (IS_ERR(at24->nvmem)) {
 		err = PTR_ERR(at24->nvmem);
 		goto err_clients;
@@ -702,10 +731,7 @@ static int at24_probe(struct i2c_client *client)
 	return 0;
 
 err_clients:
-	for (i = 1; i < num_addresses; i++)
-		if (at24->client[i].client)
-			i2c_unregister_device(at24->client[i].client);
-
+	at24_remove_dummy_clients(at24);
 	pm_runtime_disable(dev);
 
 	return err;
@@ -714,15 +740,10 @@ static int at24_probe(struct i2c_client *client)
 static int at24_remove(struct i2c_client *client)
 {
 	struct at24_data *at24;
-	int i;
 
 	at24 = i2c_get_clientdata(client);
 
-	nvmem_unregister(at24->nvmem);
-
-	for (i = 1; i < at24->num_addresses; i++)
-		i2c_unregister_device(at24->client[i].client);
-
+	at24_remove_dummy_clients(at24);
 	pm_runtime_disable(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
 
diff --git a/drivers/misc/ibmvmc.c b/drivers/misc/ibmvmc.c
new file mode 100644
index 000000000000..fb83d1375638
--- /dev/null
+++ b/drivers/misc/ibmvmc.c
@@ -0,0 +1,2418 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * IBM Power Systems Virtual Management Channel Support.
+ *
+ * Copyright (c) 2004, 2018 IBM Corp.
+ *   Dave Engebretsen engebret@us.ibm.com
+ *   Steven Royer seroyer@linux.vnet.ibm.com
+ *   Adam Reznechek adreznec@linux.vnet.ibm.com
+ *   Bryant G. Ly <bryantly@linux.vnet.ibm.com>
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/kthread.h>
+#include <linux/major.h>
+#include <linux/string.h>
+#include <linux/fcntl.h>
+#include <linux/slab.h>
+#include <linux/poll.h>
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/percpu.h>
+#include <linux/delay.h>
+#include <linux/uaccess.h>
+#include <linux/io.h>
+#include <linux/miscdevice.h>
+#include <linux/sched/signal.h>
+
+#include <asm/byteorder.h>
+#include <asm/irq.h>
+#include <asm/vio.h>
+
+#include "ibmvmc.h"
+
+#define IBMVMC_DRIVER_VERSION "1.0"
+
+/*
+ * Static global variables
+ */
+static DECLARE_WAIT_QUEUE_HEAD(ibmvmc_read_wait);
+
+static const char ibmvmc_driver_name[] = "ibmvmc";
+
+static struct ibmvmc_struct ibmvmc;
+static struct ibmvmc_hmc hmcs[MAX_HMCS];
+static struct crq_server_adapter ibmvmc_adapter;
+
+static int ibmvmc_max_buf_pool_size = DEFAULT_BUF_POOL_SIZE;
+static int ibmvmc_max_hmcs = DEFAULT_HMCS;
+static int ibmvmc_max_mtu = DEFAULT_MTU;
+
+static inline long h_copy_rdma(s64 length, u64 sliobn, u64 slioba,
+			       u64 dliobn, u64 dlioba)
+{
+	long rc = 0;
+
+	/* Ensure all writes to source memory are visible before hcall */
+	dma_wmb();
+	pr_debug("ibmvmc: h_copy_rdma(0x%llx, 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",
+		 length, sliobn, slioba, dliobn, dlioba);
+	rc = plpar_hcall_norets(H_COPY_RDMA, length, sliobn, slioba,
+				dliobn, dlioba);
+	pr_debug("ibmvmc: h_copy_rdma rc = 0x%lx\n", rc);
+
+	return rc;
+}
+
+static inline void h_free_crq(uint32_t unit_address)
+{
+	long rc = 0;
+
+	do {
+		if (H_IS_LONG_BUSY(rc))
+			msleep(get_longbusy_msecs(rc));
+
+		rc = plpar_hcall_norets(H_FREE_CRQ, unit_address);
+	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
+}
+
+/**
+ * h_request_vmc: - request a hypervisor virtual management channel device
+ * @vmc_index: drc index of the vmc device created
+ *
+ * Requests the hypervisor create a new virtual management channel device,
+ * allowing this partition to send hypervisor virtualization control
+ * commands.
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static inline long h_request_vmc(u32 *vmc_index)
+{
+	long rc = 0;
+	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
+
+	do {
+		if (H_IS_LONG_BUSY(rc))
+			msleep(get_longbusy_msecs(rc));
+
+		/* Call to request the VMC device from phyp */
+		rc = plpar_hcall(H_REQUEST_VMC, retbuf);
+		pr_debug("ibmvmc: %s rc = 0x%lx\n", __func__, rc);
+		*vmc_index = retbuf[0];
+	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
+
+	return rc;
+}
+
+/* routines for managing a command/response queue */
+/**
+ * ibmvmc_handle_event: - Interrupt handler for crq events
+ * @irq:        number of irq to handle, not used
+ * @dev_instance: crq_server_adapter that received interrupt
+ *
+ * Disables interrupts and schedules ibmvmc_task
+ *
+ * Always returns IRQ_HANDLED
+ */
+static irqreturn_t ibmvmc_handle_event(int irq, void *dev_instance)
+{
+	struct crq_server_adapter *adapter =
+		(struct crq_server_adapter *)dev_instance;
+
+	vio_disable_interrupts(to_vio_dev(adapter->dev));
+	tasklet_schedule(&adapter->work_task);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * ibmvmc_release_crq_queue - Release CRQ Queue
+ *
+ * @adapter:	crq_server_adapter struct
+ *
+ * Return:
+ *	0 - Success
+ *	Non-Zero - Failure
+ */
+static void ibmvmc_release_crq_queue(struct crq_server_adapter *adapter)
+{
+	struct vio_dev *vdev = to_vio_dev(adapter->dev);
+	struct crq_queue *queue = &adapter->queue;
+
+	free_irq(vdev->irq, (void *)adapter);
+	tasklet_kill(&adapter->work_task);
+
+	if (adapter->reset_task)
+		kthread_stop(adapter->reset_task);
+
+	h_free_crq(vdev->unit_address);
+	dma_unmap_single(adapter->dev,
+			 queue->msg_token,
+			 queue->size * sizeof(*queue->msgs), DMA_BIDIRECTIONAL);
+	free_page((unsigned long)queue->msgs);
+}
+
+/**
+ * ibmvmc_reset_crq_queue - Reset CRQ Queue
+ *
+ * @adapter:	crq_server_adapter struct
+ *
+ * This function calls h_free_crq and then calls H_REG_CRQ and does all the
+ * bookkeeping to get us back to where we can communicate.
+ *
+ * Return:
+ *	0 - Success
+ *	Non-Zero - Failure
+ */
+static int ibmvmc_reset_crq_queue(struct crq_server_adapter *adapter)
+{
+	struct vio_dev *vdev = to_vio_dev(adapter->dev);
+	struct crq_queue *queue = &adapter->queue;
+	int rc = 0;
+
+	/* Close the CRQ */
+	h_free_crq(vdev->unit_address);
+
+	/* Clean out the queue */
+	memset(queue->msgs, 0x00, PAGE_SIZE);
+	queue->cur = 0;
+
+	/* And re-open it again */
+	rc = plpar_hcall_norets(H_REG_CRQ,
+				vdev->unit_address,
+				queue->msg_token, PAGE_SIZE);
+	if (rc == 2)
+		/* Adapter is good, but other end is not ready */
+		dev_warn(adapter->dev, "Partner adapter not ready\n");
+	else if (rc != 0)
+		dev_err(adapter->dev, "couldn't register crq--rc 0x%x\n", rc);
+
+	return rc;
+}
+
+/**
+ * crq_queue_next_crq: - Returns the next entry in message queue
+ * @queue:      crq_queue to use
+ *
+ * Returns pointer to next entry in queue, or NULL if there are no new
+ * entried in the CRQ.
+ */
+static struct ibmvmc_crq_msg *crq_queue_next_crq(struct crq_queue *queue)
+{
+	struct ibmvmc_crq_msg *crq;
+	unsigned long flags;
+
+	spin_lock_irqsave(&queue->lock, flags);
+	crq = &queue->msgs[queue->cur];
+	if (crq->valid & 0x80) {
+		if (++queue->cur == queue->size)
+			queue->cur = 0;
+
+		/* Ensure the read of the valid bit occurs before reading any
+		 * other bits of the CRQ entry
+		 */
+		dma_rmb();
+	} else {
+		crq = NULL;
+	}
+
+	spin_unlock_irqrestore(&queue->lock, flags);
+
+	return crq;
+}
+
+/**
+ * ibmvmc_send_crq - Send CRQ
+ *
+ * @adapter:	crq_server_adapter struct
+ * @word1:	Word1 Data field
+ * @word2:	Word2 Data field
+ *
+ * Return:
+ *	0 - Success
+ *	Non-Zero - Failure
+ */
+static long ibmvmc_send_crq(struct crq_server_adapter *adapter,
+			    u64 word1, u64 word2)
+{
+	struct vio_dev *vdev = to_vio_dev(adapter->dev);
+	long rc = 0;
+
+	dev_dbg(adapter->dev, "(0x%x, 0x%016llx, 0x%016llx)\n",
+		vdev->unit_address, word1, word2);
+
+	/*
+	 * Ensure the command buffer is flushed to memory before handing it
+	 * over to the other side to prevent it from fetching any stale data.
+	 */
+	dma_wmb();
+	rc = plpar_hcall_norets(H_SEND_CRQ, vdev->unit_address, word1, word2);
+	dev_dbg(adapter->dev, "rc = 0x%lx\n", rc);
+
+	return rc;
+}
+
+/**
+ * alloc_dma_buffer - Create DMA Buffer
+ *
+ * @vdev:	vio_dev struct
+ * @size:	Size field
+ * @dma_handle:	DMA address field
+ *
+ * Allocates memory for the command queue and maps remote memory into an
+ * ioba.
+ *
+ * Returns a pointer to the buffer
+ */
+static void *alloc_dma_buffer(struct vio_dev *vdev, size_t size,
+			      dma_addr_t *dma_handle)
+{
+	/* allocate memory */
+	void *buffer = kzalloc(size, GFP_KERNEL);
+
+	if (!buffer) {
+		*dma_handle = 0;
+		return NULL;
+	}
+
+	/* DMA map */
+	*dma_handle = dma_map_single(&vdev->dev, buffer, size,
+				     DMA_BIDIRECTIONAL);
+
+	if (dma_mapping_error(&vdev->dev, *dma_handle)) {
+		*dma_handle = 0;
+		kzfree(buffer);
+		return NULL;
+	}
+
+	return buffer;
+}
+
+/**
+ * free_dma_buffer - Free DMA Buffer
+ *
+ * @vdev:	vio_dev struct
+ * @size:	Size field
+ * @vaddr:	Address field
+ * @dma_handle:	DMA address field
+ *
+ * Releases memory for a command queue and unmaps mapped remote memory.
+ */
+static void free_dma_buffer(struct vio_dev *vdev, size_t size, void *vaddr,
+			    dma_addr_t dma_handle)
+{
+	/* DMA unmap */
+	dma_unmap_single(&vdev->dev, dma_handle, size, DMA_BIDIRECTIONAL);
+
+	/* deallocate memory */
+	kzfree(vaddr);
+}
+
+/**
+ * ibmvmc_get_valid_hmc_buffer - Retrieve Valid HMC Buffer
+ *
+ * @hmc_index:	HMC Index Field
+ *
+ * Return:
+ *	Pointer to ibmvmc_buffer
+ */
+static struct ibmvmc_buffer *ibmvmc_get_valid_hmc_buffer(u8 hmc_index)
+{
+	struct ibmvmc_buffer *buffer;
+	struct ibmvmc_buffer *ret_buf = NULL;
+	unsigned long i;
+
+	if (hmc_index > ibmvmc.max_hmc_index)
+		return NULL;
+
+	buffer = hmcs[hmc_index].buffer;
+
+	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
+		if (buffer[i].valid && buffer[i].free &&
+		    buffer[i].owner == VMC_BUF_OWNER_ALPHA) {
+			buffer[i].free = 0;
+			ret_buf = &buffer[i];
+			break;
+		}
+	}
+
+	return ret_buf;
+}
+
+/**
+ * ibmvmc_get_free_hmc_buffer - Get Free HMC Buffer
+ *
+ * @adapter:	crq_server_adapter struct
+ * @hmc_index:	Hmc Index field
+ *
+ * Return:
+ *	Pointer to ibmvmc_buffer
+ */
+static struct ibmvmc_buffer *ibmvmc_get_free_hmc_buffer(struct crq_server_adapter *adapter,
+							u8 hmc_index)
+{
+	struct ibmvmc_buffer *buffer;
+	struct ibmvmc_buffer *ret_buf = NULL;
+	unsigned long i;
+
+	if (hmc_index > ibmvmc.max_hmc_index) {
+		dev_info(adapter->dev, "get_free_hmc_buffer: invalid hmc_index=0x%x\n",
+			 hmc_index);
+		return NULL;
+	}
+
+	buffer = hmcs[hmc_index].buffer;
+
+	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
+		if (buffer[i].free &&
+		    buffer[i].owner == VMC_BUF_OWNER_ALPHA) {
+			buffer[i].free = 0;
+			ret_buf = &buffer[i];
+			break;
+		}
+	}
+
+	return ret_buf;
+}
+
+/**
+ * ibmvmc_free_hmc_buffer - Free an HMC Buffer
+ *
+ * @hmc:	ibmvmc_hmc struct
+ * @buffer:	ibmvmc_buffer struct
+ *
+ */
+static void ibmvmc_free_hmc_buffer(struct ibmvmc_hmc *hmc,
+				   struct ibmvmc_buffer *buffer)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&hmc->lock, flags);
+	buffer->free = 1;
+	spin_unlock_irqrestore(&hmc->lock, flags);
+}
+
+/**
+ * ibmvmc_count_hmc_buffers - Count HMC Buffers
+ *
+ * @hmc_index:	HMC Index field
+ * @valid:	Valid number of buffers field
+ * @free:	Free number of buffers field
+ *
+ */
+static void ibmvmc_count_hmc_buffers(u8 hmc_index, unsigned int *valid,
+				     unsigned int *free)
+{
+	struct ibmvmc_buffer *buffer;
+	unsigned long i;
+	unsigned long flags;
+
+	if (hmc_index > ibmvmc.max_hmc_index)
+		return;
+
+	if (!valid || !free)
+		return;
+
+	*valid = 0; *free = 0;
+
+	buffer = hmcs[hmc_index].buffer;
+	spin_lock_irqsave(&hmcs[hmc_index].lock, flags);
+
+	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
+		if (buffer[i].valid) {
+			*valid = *valid + 1;
+			if (buffer[i].free)
+				*free = *free + 1;
+		}
+	}
+
+	spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);
+}
+
+/**
+ * ibmvmc_get_free_hmc - Get Free HMC
+ *
+ * Return:
+ *	Pointer to an available HMC Connection
+ *	Null otherwise
+ */
+static struct ibmvmc_hmc *ibmvmc_get_free_hmc(void)
+{
+	unsigned long i;
+	unsigned long flags;
+
+	/*
+	 * Find an available HMC connection.
+	 */
+	for (i = 0; i <= ibmvmc.max_hmc_index; i++) {
+		spin_lock_irqsave(&hmcs[i].lock, flags);
+		if (hmcs[i].state == ibmhmc_state_free) {
+			hmcs[i].index = i;
+			hmcs[i].state = ibmhmc_state_initial;
+			spin_unlock_irqrestore(&hmcs[i].lock, flags);
+			return &hmcs[i];
+		}
+		spin_unlock_irqrestore(&hmcs[i].lock, flags);
+	}
+
+	return NULL;
+}
+
+/**
+ * ibmvmc_return_hmc - Return an HMC Connection
+ *
+ * @hmc:		ibmvmc_hmc struct
+ * @release_readers:	Number of readers connected to session
+ *
+ * This function releases the HMC connections back into the pool.
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvmc_return_hmc(struct ibmvmc_hmc *hmc, bool release_readers)
+{
+	struct ibmvmc_buffer *buffer;
+	struct crq_server_adapter *adapter;
+	struct vio_dev *vdev;
+	unsigned long i;
+	unsigned long flags;
+
+	if (!hmc || !hmc->adapter)
+		return -EIO;
+
+	if (release_readers) {
+		if (hmc->file_session) {
+			struct ibmvmc_file_session *session = hmc->file_session;
+
+			session->valid = 0;
+			wake_up_interruptible(&ibmvmc_read_wait);
+		}
+	}
+
+	adapter = hmc->adapter;
+	vdev = to_vio_dev(adapter->dev);
+
+	spin_lock_irqsave(&hmc->lock, flags);
+	hmc->index = 0;
+	hmc->state = ibmhmc_state_free;
+	hmc->queue_head = 0;
+	hmc->queue_tail = 0;
+	buffer = hmc->buffer;
+	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
+		if (buffer[i].valid) {
+			free_dma_buffer(vdev,
+					ibmvmc.max_mtu,
+					buffer[i].real_addr_local,
+					buffer[i].dma_addr_local);
+			dev_dbg(adapter->dev, "Forgot buffer id 0x%lx\n", i);
+		}
+		memset(&buffer[i], 0, sizeof(struct ibmvmc_buffer));
+
+		hmc->queue_outbound_msgs[i] = VMC_INVALID_BUFFER_ID;
+	}
+
+	spin_unlock_irqrestore(&hmc->lock, flags);
+
+	return 0;
+}
+
+/**
+ * ibmvmc_send_open - Interface Open
+ * @buffer: Pointer to ibmvmc_buffer struct
+ * @hmc: Pointer to ibmvmc_hmc struct
+ *
+ * This command is sent by the management partition as the result of a
+ * management partition device request. It causes the hypervisor to
+ * prepare a set of data buffers for the management application connection
+ * indicated HMC idx. A unique HMC Idx would be used if multiple management
+ * applications running concurrently were desired. Before responding to this
+ * command, the hypervisor must provide the management partition with at
+ * least one of these new buffers via the Add Buffer. This indicates whether
+ * the messages are inbound or outbound from the hypervisor.
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvmc_send_open(struct ibmvmc_buffer *buffer,
+			    struct ibmvmc_hmc *hmc)
+{
+	struct ibmvmc_crq_msg crq_msg;
+	struct crq_server_adapter *adapter;
+	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
+	int rc = 0;
+
+	if (!hmc || !hmc->adapter)
+		return -EIO;
+
+	adapter = hmc->adapter;
+
+	dev_dbg(adapter->dev, "send_open: 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx\n",
+		(unsigned long)buffer->size, (unsigned long)adapter->liobn,
+		(unsigned long)buffer->dma_addr_local,
+		(unsigned long)adapter->riobn,
+		(unsigned long)buffer->dma_addr_remote);
+
+	rc = h_copy_rdma(buffer->size,
+			 adapter->liobn,
+			 buffer->dma_addr_local,
+			 adapter->riobn,
+			 buffer->dma_addr_remote);
+	if (rc) {
+		dev_err(adapter->dev, "Error: In send_open, h_copy_rdma rc 0x%x\n",
+			rc);
+		return -EIO;
+	}
+
+	hmc->state = ibmhmc_state_opening;
+
+	crq_msg.valid = 0x80;
+	crq_msg.type = VMC_MSG_OPEN;
+	crq_msg.status = 0;
+	crq_msg.var1.rsvd = 0;
+	crq_msg.hmc_session = hmc->session;
+	crq_msg.hmc_index = hmc->index;
+	crq_msg.var2.buffer_id = cpu_to_be16(buffer->id);
+	crq_msg.rsvd = 0;
+	crq_msg.var3.rsvd = 0;
+
+	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
+			be64_to_cpu(crq_as_u64[1]));
+
+	return rc;
+}
+
+/**
+ * ibmvmc_send_close - Interface Close
+ * @hmc: Pointer to ibmvmc_hmc struct
+ *
+ * This command is sent by the management partition to terminate a
+ * management application to hypervisor connection. When this command is
+ * sent, the management partition has quiesced all I/O operations to all
+ * buffers associated with this management application connection, and
+ * has freed any storage for these buffers.
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvmc_send_close(struct ibmvmc_hmc *hmc)
+{
+	struct ibmvmc_crq_msg crq_msg;
+	struct crq_server_adapter *adapter;
+	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
+	int rc = 0;
+
+	if (!hmc || !hmc->adapter)
+		return -EIO;
+
+	adapter = hmc->adapter;
+
+	dev_info(adapter->dev, "CRQ send: close\n");
+
+	crq_msg.valid = 0x80;
+	crq_msg.type = VMC_MSG_CLOSE;
+	crq_msg.status = 0;
+	crq_msg.var1.rsvd = 0;
+	crq_msg.hmc_session = hmc->session;
+	crq_msg.hmc_index = hmc->index;
+	crq_msg.var2.rsvd = 0;
+	crq_msg.rsvd = 0;
+	crq_msg.var3.rsvd = 0;
+
+	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
+			be64_to_cpu(crq_as_u64[1]));
+
+	return rc;
+}
+
+/**
+ * ibmvmc_send_capabilities - Send VMC Capabilities
+ *
+ * @adapter:	crq_server_adapter struct
+ *
+ * The capabilities message is an administrative message sent after the CRQ
+ * initialization sequence of messages and is used to exchange VMC capabilities
+ * between the management partition and the hypervisor. The management
+ * partition must send this message and the hypervisor must respond with VMC
+ * capabilities Response message before HMC interface message can begin. Any
+ * HMC interface messages received before the exchange of capabilities has
+ * complete are dropped.
+ *
+ * Return:
+ *	0 - Success
+ */
+static int ibmvmc_send_capabilities(struct crq_server_adapter *adapter)
+{
+	struct ibmvmc_admin_crq_msg crq_msg;
+	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
+
+	dev_dbg(adapter->dev, "ibmvmc: CRQ send: capabilities\n");
+	crq_msg.valid = 0x80;
+	crq_msg.type = VMC_MSG_CAP;
+	crq_msg.status = 0;
+	crq_msg.rsvd[0] = 0;
+	crq_msg.rsvd[1] = 0;
+	crq_msg.max_hmc = ibmvmc_max_hmcs;
+	crq_msg.max_mtu = cpu_to_be32(ibmvmc_max_mtu);
+	crq_msg.pool_size = cpu_to_be16(ibmvmc_max_buf_pool_size);
+	crq_msg.crq_size = cpu_to_be16(adapter->queue.size);
+	crq_msg.version = cpu_to_be16(IBMVMC_PROTOCOL_VERSION);
+
+	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
+			be64_to_cpu(crq_as_u64[1]));
+
+	ibmvmc.state = ibmvmc_state_capabilities;
+
+	return 0;
+}
+
+/**
+ * ibmvmc_send_add_buffer_resp - Add Buffer Response
+ *
+ * @adapter:	crq_server_adapter struct
+ * @status:	Status field
+ * @hmc_session: HMC Session field
+ * @hmc_index:	HMC Index field
+ * @buffer_id:	Buffer Id field
+ *
+ * This command is sent by the management partition to the hypervisor in
+ * response to the Add Buffer message. The Status field indicates the result of
+ * the command.
+ *
+ * Return:
+ *	0 - Success
+ */
+static int ibmvmc_send_add_buffer_resp(struct crq_server_adapter *adapter,
+				       u8 status, u8 hmc_session,
+				       u8 hmc_index, u16 buffer_id)
+{
+	struct ibmvmc_crq_msg crq_msg;
+	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
+
+	dev_dbg(adapter->dev, "CRQ send: add_buffer_resp\n");
+	crq_msg.valid = 0x80;
+	crq_msg.type = VMC_MSG_ADD_BUF_RESP;
+	crq_msg.status = status;
+	crq_msg.var1.rsvd = 0;
+	crq_msg.hmc_session = hmc_session;
+	crq_msg.hmc_index = hmc_index;
+	crq_msg.var2.buffer_id = cpu_to_be16(buffer_id);
+	crq_msg.rsvd = 0;
+	crq_msg.var3.rsvd = 0;
+
+	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
+			be64_to_cpu(crq_as_u64[1]));
+
+	return 0;
+}
+
+/**
+ * ibmvmc_send_rem_buffer_resp - Remove Buffer Response
+ *
+ * @adapter:	crq_server_adapter struct
+ * @status:	Status field
+ * @hmc_session: HMC Session field
+ * @hmc_index:	HMC Index field
+ * @buffer_id:	Buffer Id field
+ *
+ * This command is sent by the management partition to the hypervisor in
+ * response to the Remove Buffer message. The Buffer ID field indicates
+ * which buffer the management partition selected to remove. The Status
+ * field indicates the result of the command.
+ *
+ * Return:
+ *	0 - Success
+ */
+static int ibmvmc_send_rem_buffer_resp(struct crq_server_adapter *adapter,
+				       u8 status, u8 hmc_session,
+				       u8 hmc_index, u16 buffer_id)
+{
+	struct ibmvmc_crq_msg crq_msg;
+	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
+
+	dev_dbg(adapter->dev, "CRQ send: rem_buffer_resp\n");
+	crq_msg.valid = 0x80;
+	crq_msg.type = VMC_MSG_REM_BUF_RESP;
+	crq_msg.status = status;
+	crq_msg.var1.rsvd = 0;
+	crq_msg.hmc_session = hmc_session;
+	crq_msg.hmc_index = hmc_index;
+	crq_msg.var2.buffer_id = cpu_to_be16(buffer_id);
+	crq_msg.rsvd = 0;
+	crq_msg.var3.rsvd = 0;
+
+	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
+			be64_to_cpu(crq_as_u64[1]));
+
+	return 0;
+}
+
+/**
+ * ibmvmc_send_msg - Signal Message
+ *
+ * @adapter:	crq_server_adapter struct
+ * @buffer:	ibmvmc_buffer struct
+ * @hmc:	ibmvmc_hmc struct
+ * @msg_length:	message length field
+ *
+ * This command is sent between the management partition and the hypervisor
+ * in order to signal the arrival of an HMC protocol message. The command
+ * can be sent by both the management partition and the hypervisor. It is
+ * used for all traffic between the management application and the hypervisor,
+ * regardless of who initiated the communication.
+ *
+ * There is no response to this message.
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvmc_send_msg(struct crq_server_adapter *adapter,
+			   struct ibmvmc_buffer *buffer,
+			   struct ibmvmc_hmc *hmc, int msg_len)
+{
+	struct ibmvmc_crq_msg crq_msg;
+	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
+	int rc = 0;
+
+	dev_dbg(adapter->dev, "CRQ send: rdma to HV\n");
+	rc = h_copy_rdma(msg_len,
+			 adapter->liobn,
+			 buffer->dma_addr_local,
+			 adapter->riobn,
+			 buffer->dma_addr_remote);
+	if (rc) {
+		dev_err(adapter->dev, "Error in send_msg, h_copy_rdma rc 0x%x\n",
+			rc);
+		return rc;
+	}
+
+	crq_msg.valid = 0x80;
+	crq_msg.type = VMC_MSG_SIGNAL;
+	crq_msg.status = 0;
+	crq_msg.var1.rsvd = 0;
+	crq_msg.hmc_session = hmc->session;
+	crq_msg.hmc_index = hmc->index;
+	crq_msg.var2.buffer_id = cpu_to_be16(buffer->id);
+	crq_msg.var3.msg_len = cpu_to_be32(msg_len);
+	dev_dbg(adapter->dev, "CRQ send: msg to HV 0x%llx 0x%llx\n",
+		be64_to_cpu(crq_as_u64[0]), be64_to_cpu(crq_as_u64[1]));
+
+	buffer->owner = VMC_BUF_OWNER_HV;
+	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
+			be64_to_cpu(crq_as_u64[1]));
+
+	return rc;
+}
+
+/**
+ * ibmvmc_open - Open Session
+ *
+ * @inode:	inode struct
+ * @file:	file struct
+ *
+ * Return:
+ *	0 - Success
+ */
+static int ibmvmc_open(struct inode *inode, struct file *file)
+{
+	struct ibmvmc_file_session *session;
+	int rc = 0;
+
+	pr_debug("%s: inode = 0x%lx, file = 0x%lx, state = 0x%x\n", __func__,
+		 (unsigned long)inode, (unsigned long)file,
+		 ibmvmc.state);
+
+	session = kzalloc(sizeof(*session), GFP_KERNEL);
+	session->file = file;
+	file->private_data = session;
+
+	return rc;
+}
+
+/**
+ * ibmvmc_close - Close Session
+ *
+ * @inode:	inode struct
+ * @file:	file struct
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvmc_close(struct inode *inode, struct file *file)
+{
+	struct ibmvmc_file_session *session;
+	struct ibmvmc_hmc *hmc;
+	int rc = 0;
+	unsigned long flags;
+
+	pr_debug("%s: file = 0x%lx, state = 0x%x\n", __func__,
+		 (unsigned long)file, ibmvmc.state);
+
+	session = file->private_data;
+	if (!session)
+		return -EIO;
+
+	hmc = session->hmc;
+	if (hmc) {
+		if (!hmc->adapter)
+			return -EIO;
+
+		if (ibmvmc.state == ibmvmc_state_failed) {
+			dev_warn(hmc->adapter->dev, "close: state_failed\n");
+			return -EIO;
+		}
+
+		spin_lock_irqsave(&hmc->lock, flags);
+		if (hmc->state >= ibmhmc_state_opening) {
+			rc = ibmvmc_send_close(hmc);
+			if (rc)
+				dev_warn(hmc->adapter->dev, "close: send_close failed.\n");
+		}
+		spin_unlock_irqrestore(&hmc->lock, flags);
+	}
+
+	kzfree(session);
+
+	return rc;
+}
+
+/**
+ * ibmvmc_read - Read
+ *
+ * @file:	file struct
+ * @buf:	Character buffer
+ * @nbytes:	Size in bytes
+ * @ppos:	Offset
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static ssize_t ibmvmc_read(struct file *file, char *buf, size_t nbytes,
+			   loff_t *ppos)
+{
+	struct ibmvmc_file_session *session;
+	struct ibmvmc_hmc *hmc;
+	struct crq_server_adapter *adapter;
+	struct ibmvmc_buffer *buffer;
+	ssize_t n;
+	ssize_t retval = 0;
+	unsigned long flags;
+	DEFINE_WAIT(wait);
+
+	pr_debug("ibmvmc: read: file = 0x%lx, buf = 0x%lx, nbytes = 0x%lx\n",
+		 (unsigned long)file, (unsigned long)buf,
+		 (unsigned long)nbytes);
+
+	if (nbytes == 0)
+		return 0;
+
+	if (nbytes > ibmvmc.max_mtu) {
+		pr_warn("ibmvmc: read: nbytes invalid 0x%x\n",
+			(unsigned int)nbytes);
+		return -EINVAL;
+	}
+
+	session = file->private_data;
+	if (!session) {
+		pr_warn("ibmvmc: read: no session\n");
+		return -EIO;
+	}
+
+	hmc = session->hmc;
+	if (!hmc) {
+		pr_warn("ibmvmc: read: no hmc\n");
+		return -EIO;
+	}
+
+	adapter = hmc->adapter;
+	if (!adapter) {
+		pr_warn("ibmvmc: read: no adapter\n");
+		return -EIO;
+	}
+
+	do {
+		prepare_to_wait(&ibmvmc_read_wait, &wait, TASK_INTERRUPTIBLE);
+
+		spin_lock_irqsave(&hmc->lock, flags);
+		if (hmc->queue_tail != hmc->queue_head)
+			/* Data is available */
+			break;
+
+		spin_unlock_irqrestore(&hmc->lock, flags);
+
+		if (!session->valid) {
+			retval = -EBADFD;
+			goto out;
+		}
+		if (file->f_flags & O_NONBLOCK) {
+			retval = -EAGAIN;
+			goto out;
+		}
+
+		schedule();
+
+		if (signal_pending(current)) {
+			retval = -ERESTARTSYS;
+			goto out;
+		}
+	} while (1);
+
+	buffer = &(hmc->buffer[hmc->queue_outbound_msgs[hmc->queue_tail]]);
+	hmc->queue_tail++;
+	if (hmc->queue_tail == ibmvmc_max_buf_pool_size)
+		hmc->queue_tail = 0;
+	spin_unlock_irqrestore(&hmc->lock, flags);
+
+	nbytes = min_t(size_t, nbytes, buffer->msg_len);
+	n = copy_to_user((void *)buf, buffer->real_addr_local, nbytes);
+	dev_dbg(adapter->dev, "read: copy to user nbytes = 0x%lx.\n", nbytes);
+	ibmvmc_free_hmc_buffer(hmc, buffer);
+	retval = nbytes;
+
+	if (n) {
+		dev_warn(adapter->dev, "read: copy to user failed.\n");
+		retval = -EFAULT;
+	}
+
+ out:
+	finish_wait(&ibmvmc_read_wait, &wait);
+	dev_dbg(adapter->dev, "read: out %ld\n", retval);
+	return retval;
+}
+
+/**
+ * ibmvmc_poll - Poll
+ *
+ * @file:	file struct
+ * @wait:	Poll Table
+ *
+ * Return:
+ *	poll.h return values
+ */
+static unsigned int ibmvmc_poll(struct file *file, poll_table *wait)
+{
+	struct ibmvmc_file_session *session;
+	struct ibmvmc_hmc *hmc;
+	unsigned int mask = 0;
+
+	session = file->private_data;
+	if (!session)
+		return 0;
+
+	hmc = session->hmc;
+	if (!hmc)
+		return 0;
+
+	poll_wait(file, &ibmvmc_read_wait, wait);
+
+	if (hmc->queue_head != hmc->queue_tail)
+		mask |= POLLIN | POLLRDNORM;
+
+	return mask;
+}
+
+/**
+ * ibmvmc_write - Write
+ *
+ * @file:	file struct
+ * @buf:	Character buffer
+ * @count:	Count field
+ * @ppos:	Offset
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static ssize_t ibmvmc_write(struct file *file, const char *buffer,
+			    size_t count, loff_t *ppos)
+{
+	struct ibmvmc_buffer *vmc_buffer;
+	struct ibmvmc_file_session *session;
+	struct crq_server_adapter *adapter;
+	struct ibmvmc_hmc *hmc;
+	unsigned char *buf;
+	unsigned long flags;
+	size_t bytes;
+	const char *p = buffer;
+	size_t c = count;
+	int ret = 0;
+
+	session = file->private_data;
+	if (!session)
+		return -EIO;
+
+	hmc = session->hmc;
+	if (!hmc)
+		return -EIO;
+
+	spin_lock_irqsave(&hmc->lock, flags);
+	if (hmc->state == ibmhmc_state_free) {
+		/* HMC connection is not valid (possibly was reset under us). */
+		ret = -EIO;
+		goto out;
+	}
+
+	adapter = hmc->adapter;
+	if (!adapter) {
+		ret = -EIO;
+		goto out;
+	}
+
+	if (count > ibmvmc.max_mtu) {
+		dev_warn(adapter->dev, "invalid buffer size 0x%lx\n",
+			 (unsigned long)count);
+		ret = -EIO;
+		goto out;
+	}
+
+	/* Waiting for the open resp message to the ioctl(1) - retry */
+	if (hmc->state == ibmhmc_state_opening) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	/* Make sure the ioctl() was called & the open msg sent, and that
+	 * the HMC connection has not failed.
+	 */
+	if (hmc->state != ibmhmc_state_ready) {
+		ret = -EIO;
+		goto out;
+	}
+
+	vmc_buffer = ibmvmc_get_valid_hmc_buffer(hmc->index);
+	if (!vmc_buffer) {
+		/* No buffer available for the msg send, or we have not yet
+		 * completed the open/open_resp sequence.  Retry until this is
+		 * complete.
+		 */
+		ret = -EBUSY;
+		goto out;
+	}
+	if (!vmc_buffer->real_addr_local) {
+		dev_err(adapter->dev, "no buffer storage assigned\n");
+		ret = -EIO;
+		goto out;
+	}
+	buf = vmc_buffer->real_addr_local;
+
+	while (c > 0) {
+		bytes = min_t(size_t, c, vmc_buffer->size);
+
+		bytes -= copy_from_user(buf, p, bytes);
+		if (!bytes) {
+			ret = -EFAULT;
+			goto out;
+		}
+		c -= bytes;
+		p += bytes;
+	}
+	if (p == buffer)
+		goto out;
+
+	file->f_path.dentry->d_inode->i_mtime = current_time(file_inode(file));
+	mark_inode_dirty(file->f_path.dentry->d_inode);
+
+	dev_dbg(adapter->dev, "write: file = 0x%lx, count = 0x%lx\n",
+		(unsigned long)file, (unsigned long)count);
+
+	ibmvmc_send_msg(adapter, vmc_buffer, hmc, count);
+	ret = p - buffer;
+ out:
+	spin_unlock_irqrestore(&hmc->lock, flags);
+	return (ssize_t)(ret);
+}
+
+/**
+ * ibmvmc_setup_hmc - Setup the HMC
+ *
+ * @session:	ibmvmc_file_session struct
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static long ibmvmc_setup_hmc(struct ibmvmc_file_session *session)
+{
+	struct ibmvmc_hmc *hmc;
+	unsigned int valid, free, index;
+
+	if (ibmvmc.state == ibmvmc_state_failed) {
+		pr_warn("ibmvmc: Reserve HMC: state_failed\n");
+		return -EIO;
+	}
+
+	if (ibmvmc.state < ibmvmc_state_ready) {
+		pr_warn("ibmvmc: Reserve HMC: not state_ready\n");
+		return -EAGAIN;
+	}
+
+	/* Device is busy until capabilities have been exchanged and we
+	 * have a generic buffer for each possible HMC connection.
+	 */
+	for (index = 0; index <= ibmvmc.max_hmc_index; index++) {
+		valid = 0;
+		ibmvmc_count_hmc_buffers(index, &valid, &free);
+		if (valid == 0) {
+			pr_warn("ibmvmc: buffers not ready for index %d\n",
+				index);
+			return -ENOBUFS;
+		}
+	}
+
+	/* Get an hmc object, and transition to ibmhmc_state_initial */
+	hmc = ibmvmc_get_free_hmc();
+	if (!hmc) {
+		pr_warn("%s: free hmc not found\n", __func__);
+		return -EBUSY;
+	}
+
+	hmc->session = hmc->session + 1;
+	if (hmc->session == 0xff)
+		hmc->session = 1;
+
+	session->hmc = hmc;
+	hmc->adapter = &ibmvmc_adapter;
+	hmc->file_session = session;
+	session->valid = 1;
+
+	return 0;
+}
+
+/**
+ * ibmvmc_ioctl_sethmcid - IOCTL Set HMC ID
+ *
+ * @session:	ibmvmc_file_session struct
+ * @new_hmc_id:	HMC id field
+ *
+ * IOCTL command to setup the hmc id
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static long ibmvmc_ioctl_sethmcid(struct ibmvmc_file_session *session,
+				  unsigned char __user *new_hmc_id)
+{
+	struct ibmvmc_hmc *hmc;
+	struct ibmvmc_buffer *buffer;
+	size_t bytes;
+	char print_buffer[HMC_ID_LEN + 1];
+	unsigned long flags;
+	long rc = 0;
+
+	/* Reserve HMC session */
+	hmc = session->hmc;
+	if (!hmc) {
+		rc = ibmvmc_setup_hmc(session);
+		if (rc)
+			return rc;
+
+		hmc = session->hmc;
+		if (!hmc) {
+			pr_err("ibmvmc: setup_hmc success but no hmc\n");
+			return -EIO;
+		}
+	}
+
+	if (hmc->state != ibmhmc_state_initial) {
+		pr_warn("ibmvmc: sethmcid: invalid state to send open 0x%x\n",
+			hmc->state);
+		return -EIO;
+	}
+
+	bytes = copy_from_user(hmc->hmc_id, new_hmc_id, HMC_ID_LEN);
+	if (bytes)
+		return -EFAULT;
+
+	/* Send Open Session command */
+	spin_lock_irqsave(&hmc->lock, flags);
+	buffer = ibmvmc_get_valid_hmc_buffer(hmc->index);
+	spin_unlock_irqrestore(&hmc->lock, flags);
+
+	if (!buffer || !buffer->real_addr_local) {
+		pr_warn("ibmvmc: sethmcid: no buffer available\n");
+		return -EIO;
+	}
+
+	/* Make sure buffer is NULL terminated before trying to print it */
+	memset(print_buffer, 0, HMC_ID_LEN + 1);
+	strncpy(print_buffer, hmc->hmc_id, HMC_ID_LEN);
+	pr_info("ibmvmc: sethmcid: Set HMC ID: \"%s\"\n", print_buffer);
+
+	memcpy(buffer->real_addr_local, hmc->hmc_id, HMC_ID_LEN);
+	/* RDMA over ID, send open msg, change state to ibmhmc_state_opening */
+	rc = ibmvmc_send_open(buffer, hmc);
+
+	return rc;
+}
+
+/**
+ * ibmvmc_ioctl_query - IOCTL Query
+ *
+ * @session:	ibmvmc_file_session struct
+ * @ret_struct:	ibmvmc_query_struct
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static long ibmvmc_ioctl_query(struct ibmvmc_file_session *session,
+			       struct ibmvmc_query_struct __user *ret_struct)
+{
+	struct ibmvmc_query_struct query_struct;
+	size_t bytes;
+
+	memset(&query_struct, 0, sizeof(query_struct));
+	query_struct.have_vmc = (ibmvmc.state > ibmvmc_state_initial);
+	query_struct.state = ibmvmc.state;
+	query_struct.vmc_drc_index = ibmvmc.vmc_drc_index;
+
+	bytes = copy_to_user(ret_struct, &query_struct,
+			     sizeof(query_struct));
+	if (bytes)
+		return -EFAULT;
+
+	return 0;
+}
+
+/**
+ * ibmvmc_ioctl_requestvmc - IOCTL Request VMC
+ *
+ * @session:	ibmvmc_file_session struct
+ * @ret_vmc_index:	VMC Index
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static long ibmvmc_ioctl_requestvmc(struct ibmvmc_file_session *session,
+				    u32 __user *ret_vmc_index)
+{
+	/* TODO: (adreznec) Add locking to control multiple process access */
+	size_t bytes;
+	long rc;
+	u32 vmc_drc_index;
+
+	/* Call to request the VMC device from phyp*/
+	rc = h_request_vmc(&vmc_drc_index);
+	pr_debug("ibmvmc: requestvmc: H_REQUEST_VMC rc = 0x%lx\n", rc);
+
+	if (rc == H_SUCCESS) {
+		rc = 0;
+	} else if (rc == H_FUNCTION) {
+		pr_err("ibmvmc: requestvmc: h_request_vmc not supported\n");
+		return -EPERM;
+	} else if (rc == H_AUTHORITY) {
+		pr_err("ibmvmc: requestvmc: hypervisor denied vmc request\n");
+		return -EPERM;
+	} else if (rc == H_HARDWARE) {
+		pr_err("ibmvmc: requestvmc: hypervisor hardware fault\n");
+		return -EIO;
+	} else if (rc == H_RESOURCE) {
+		pr_err("ibmvmc: requestvmc: vmc resource unavailable\n");
+		return -ENODEV;
+	} else if (rc == H_NOT_AVAILABLE) {
+		pr_err("ibmvmc: requestvmc: system cannot be vmc managed\n");
+		return -EPERM;
+	} else if (rc == H_PARAMETER) {
+		pr_err("ibmvmc: requestvmc: invalid parameter\n");
+		return -EINVAL;
+	}
+
+	/* Success, set the vmc index in global struct */
+	ibmvmc.vmc_drc_index = vmc_drc_index;
+
+	bytes = copy_to_user(ret_vmc_index, &vmc_drc_index,
+			     sizeof(*ret_vmc_index));
+	if (bytes) {
+		pr_warn("ibmvmc: requestvmc: copy to user failed.\n");
+		return -EFAULT;
+	}
+	return rc;
+}
+
+/**
+ * ibmvmc_ioctl - IOCTL
+ *
+ * @session:	ibmvmc_file_session struct
+ * @cmd:	cmd field
+ * @arg:	Argument field
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static long ibmvmc_ioctl(struct file *file,
+			 unsigned int cmd, unsigned long arg)
+{
+	struct ibmvmc_file_session *session = file->private_data;
+
+	pr_debug("ibmvmc: ioctl file=0x%lx, cmd=0x%x, arg=0x%lx, ses=0x%lx\n",
+		 (unsigned long)file, cmd, arg,
+		 (unsigned long)session);
+
+	if (!session) {
+		pr_warn("ibmvmc: ioctl: no session\n");
+		return -EIO;
+	}
+
+	switch (cmd) {
+	case VMC_IOCTL_SETHMCID:
+		return ibmvmc_ioctl_sethmcid(session,
+			(unsigned char __user *)arg);
+	case VMC_IOCTL_QUERY:
+		return ibmvmc_ioctl_query(session,
+			(struct ibmvmc_query_struct __user *)arg);
+	case VMC_IOCTL_REQUESTVMC:
+		return ibmvmc_ioctl_requestvmc(session,
+			(unsigned int __user *)arg);
+	default:
+		pr_warn("ibmvmc: unknown ioctl 0x%x\n", cmd);
+		return -EINVAL;
+	}
+}
+
+static const struct file_operations ibmvmc_fops = {
+	.owner		= THIS_MODULE,
+	.read		= ibmvmc_read,
+	.write		= ibmvmc_write,
+	.poll		= ibmvmc_poll,
+	.unlocked_ioctl	= ibmvmc_ioctl,
+	.open           = ibmvmc_open,
+	.release        = ibmvmc_close,
+};
+
+/**
+ * ibmvmc_add_buffer - Add Buffer
+ *
+ * @adapter: crq_server_adapter struct
+ * @crq:	ibmvmc_crq_msg struct
+ *
+ * This message transfers a buffer from hypervisor ownership to management
+ * partition ownership. The LIOBA is obtained from the virtual TCE table
+ * associated with the hypervisor side of the VMC device, and points to a
+ * buffer of size MTU (as established in the capabilities exchange).
+ *
+ * Typical flow for ading buffers:
+ * 1. A new management application connection is opened by the management
+ *	partition.
+ * 2. The hypervisor assigns new buffers for the traffic associated with
+ *	that connection.
+ * 3. The hypervisor sends VMC Add Buffer messages to the management
+ *	partition, informing it of the new buffers.
+ * 4. The hypervisor sends an HMC protocol message (to the management
+ *	application) notifying it of the new buffers. This informs the
+ *	application that it has buffers available for sending HMC
+ *	commands.
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvmc_add_buffer(struct crq_server_adapter *adapter,
+			     struct ibmvmc_crq_msg *crq)
+{
+	struct ibmvmc_buffer *buffer;
+	u8 hmc_index;
+	u8 hmc_session;
+	u16 buffer_id;
+	unsigned long flags;
+	int rc = 0;
+
+	if (!crq)
+		return -1;
+
+	hmc_session = crq->hmc_session;
+	hmc_index = crq->hmc_index;
+	buffer_id = be16_to_cpu(crq->var2.buffer_id);
+
+	if (hmc_index > ibmvmc.max_hmc_index) {
+		dev_err(adapter->dev, "add_buffer: invalid hmc_index = 0x%x\n",
+			hmc_index);
+		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_HMC_INDEX,
+					    hmc_session, hmc_index, buffer_id);
+		return -1;
+	}
+
+	if (buffer_id >= ibmvmc.max_buffer_pool_size) {
+		dev_err(adapter->dev, "add_buffer: invalid buffer_id = 0x%x\n",
+			buffer_id);
+		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_BUFFER_ID,
+					    hmc_session, hmc_index, buffer_id);
+		return -1;
+	}
+
+	spin_lock_irqsave(&hmcs[hmc_index].lock, flags);
+	buffer = &hmcs[hmc_index].buffer[buffer_id];
+
+	if (buffer->real_addr_local || buffer->dma_addr_local) {
+		dev_warn(adapter->dev, "add_buffer: already allocated id = 0x%lx\n",
+			 (unsigned long)buffer_id);
+		spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);
+		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_BUFFER_ID,
+					    hmc_session, hmc_index, buffer_id);
+		return -1;
+	}
+
+	buffer->real_addr_local = alloc_dma_buffer(to_vio_dev(adapter->dev),
+						   ibmvmc.max_mtu,
+						   &buffer->dma_addr_local);
+
+	if (!buffer->real_addr_local) {
+		dev_err(adapter->dev, "add_buffer: alloc_dma_buffer failed.\n");
+		spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);
+		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INTERFACE_FAILURE,
+					    hmc_session, hmc_index, buffer_id);
+		return -1;
+	}
+
+	buffer->dma_addr_remote = be32_to_cpu(crq->var3.lioba);
+	buffer->size = ibmvmc.max_mtu;
+	buffer->owner = crq->var1.owner;
+	buffer->free = 1;
+	/* Must ensure valid==1 is observable only after all other fields are */
+	dma_wmb();
+	buffer->valid = 1;
+	buffer->id = buffer_id;
+
+	dev_dbg(adapter->dev, "add_buffer: successfully added a buffer:\n");
+	dev_dbg(adapter->dev, "   index: %d, session: %d, buffer: 0x%x, owner: %d\n",
+		hmc_index, hmc_session, buffer_id, buffer->owner);
+	dev_dbg(adapter->dev, "   local: 0x%x, remote: 0x%x\n",
+		(u32)buffer->dma_addr_local,
+		(u32)buffer->dma_addr_remote);
+	spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);
+
+	ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_SUCCESS, hmc_session,
+				    hmc_index, buffer_id);
+
+	return rc;
+}
+
+/**
+ * ibmvmc_rem_buffer - Remove Buffer
+ *
+ * @adapter: crq_server_adapter struct
+ * @crq:	ibmvmc_crq_msg struct
+ *
+ * This message requests an HMC buffer to be transferred from management
+ * partition ownership to hypervisor ownership. The management partition may
+ * not be able to satisfy the request at a particular point in time if all its
+ * buffers are in use. The management partition requires a depth of at least
+ * one inbound buffer to allow management application commands to flow to the
+ * hypervisor. It is, therefore, an interface error for the hypervisor to
+ * attempt to remove the management partition's last buffer.
+ *
+ * The hypervisor is expected to manage buffer usage with the management
+ * application directly and inform the management partition when buffers may be
+ * removed. The typical flow for removing buffers:
+ *
+ * 1. The management application no longer needs a communication path to a
+ *	particular hypervisor function. That function is closed.
+ * 2. The hypervisor and the management application quiesce all traffic to that
+ *	function. The hypervisor requests a reduction in buffer pool size.
+ * 3. The management application acknowledges the reduction in buffer pool size.
+ * 4. The hypervisor sends a Remove Buffer message to the management partition,
+ *	informing it of the reduction in buffers.
+ * 5. The management partition verifies it can remove the buffer. This is
+ *	possible if buffers have been quiesced.
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+/*
+ * The hypervisor requested that we pick an unused buffer, and return it.
+ * Before sending the buffer back, we free any storage associated with the
+ * buffer.
+ */
+static int ibmvmc_rem_buffer(struct crq_server_adapter *adapter,
+			     struct ibmvmc_crq_msg *crq)
+{
+	struct ibmvmc_buffer *buffer;
+	u8 hmc_index;
+	u8 hmc_session;
+	u16 buffer_id = 0;
+	unsigned long flags;
+	int rc = 0;
+
+	if (!crq)
+		return -1;
+
+	hmc_session = crq->hmc_session;
+	hmc_index = crq->hmc_index;
+
+	if (hmc_index > ibmvmc.max_hmc_index) {
+		dev_warn(adapter->dev, "rem_buffer: invalid hmc_index = 0x%x\n",
+			 hmc_index);
+		ibmvmc_send_rem_buffer_resp(adapter, VMC_MSG_INVALID_HMC_INDEX,
+					    hmc_session, hmc_index, buffer_id);
+		return -1;
+	}
+
+	spin_lock_irqsave(&hmcs[hmc_index].lock, flags);
+	buffer = ibmvmc_get_free_hmc_buffer(adapter, hmc_index);
+	if (!buffer) {
+		dev_info(adapter->dev, "rem_buffer: no buffer to remove\n");
+		spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);
+		ibmvmc_send_rem_buffer_resp(adapter, VMC_MSG_NO_BUFFER,
+					    hmc_session, hmc_index,
+					    VMC_INVALID_BUFFER_ID);
+		return -1;
+	}
+
+	buffer_id = buffer->id;
+
+	if (buffer->valid)
+		free_dma_buffer(to_vio_dev(adapter->dev),
+				ibmvmc.max_mtu,
+				buffer->real_addr_local,
+				buffer->dma_addr_local);
+
+	memset(buffer, 0, sizeof(struct ibmvmc_buffer));
+	spin_unlock_irqrestore(&hmcs[hmc_index].lock, flags);
+
+	dev_dbg(adapter->dev, "rem_buffer: removed buffer 0x%x.\n", buffer_id);
+	ibmvmc_send_rem_buffer_resp(adapter, VMC_MSG_SUCCESS, hmc_session,
+				    hmc_index, buffer_id);
+
+	return rc;
+}
+
+static int ibmvmc_recv_msg(struct crq_server_adapter *adapter,
+			   struct ibmvmc_crq_msg *crq)
+{
+	struct ibmvmc_buffer *buffer;
+	struct ibmvmc_hmc *hmc;
+	unsigned long msg_len;
+	u8 hmc_index;
+	u8 hmc_session;
+	u16 buffer_id;
+	unsigned long flags;
+	int rc = 0;
+
+	if (!crq)
+		return -1;
+
+	/* Hypervisor writes CRQs directly into our memory in big endian */
+	dev_dbg(adapter->dev, "Recv_msg: msg from HV 0x%016llx 0x%016llx\n",
+		be64_to_cpu(*((unsigned long *)crq)),
+		be64_to_cpu(*(((unsigned long *)crq) + 1)));
+
+	hmc_session = crq->hmc_session;
+	hmc_index = crq->hmc_index;
+	buffer_id = be16_to_cpu(crq->var2.buffer_id);
+	msg_len = be32_to_cpu(crq->var3.msg_len);
+
+	if (hmc_index > ibmvmc.max_hmc_index) {
+		dev_err(adapter->dev, "Recv_msg: invalid hmc_index = 0x%x\n",
+			hmc_index);
+		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_HMC_INDEX,
+					    hmc_session, hmc_index, buffer_id);
+		return -1;
+	}
+
+	if (buffer_id >= ibmvmc.max_buffer_pool_size) {
+		dev_err(adapter->dev, "Recv_msg: invalid buffer_id = 0x%x\n",
+			buffer_id);
+		ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_BUFFER_ID,
+					    hmc_session, hmc_index, buffer_id);
+		return -1;
+	}
+
+	hmc = &hmcs[hmc_index];
+	spin_lock_irqsave(&hmc->lock, flags);
+
+	if (hmc->state == ibmhmc_state_free) {
+		dev_err(adapter->dev, "Recv_msg: invalid hmc state = 0x%x\n",
+			hmc->state);
+		/* HMC connection is not valid (possibly was reset under us). */
+		spin_unlock_irqrestore(&hmc->lock, flags);
+		return -1;
+	}
+
+	buffer = &hmc->buffer[buffer_id];
+
+	if (buffer->valid == 0 || buffer->owner == VMC_BUF_OWNER_ALPHA) {
+		dev_err(adapter->dev, "Recv_msg: not valid, or not HV.  0x%x 0x%x\n",
+			buffer->valid, buffer->owner);
+		spin_unlock_irqrestore(&hmc->lock, flags);
+		return -1;
+	}
+
+	/* RDMA the data into the partition. */
+	rc = h_copy_rdma(msg_len,
+			 adapter->riobn,
+			 buffer->dma_addr_remote,
+			 adapter->liobn,
+			 buffer->dma_addr_local);
+
+	dev_dbg(adapter->dev, "Recv_msg: msg_len = 0x%x, buffer_id = 0x%x, queue_head = 0x%x, hmc_idx = 0x%x\n",
+		(unsigned int)msg_len, (unsigned int)buffer_id,
+		(unsigned int)hmc->queue_head, (unsigned int)hmc_index);
+	buffer->msg_len = msg_len;
+	buffer->free = 0;
+	buffer->owner = VMC_BUF_OWNER_ALPHA;
+
+	if (rc) {
+		dev_err(adapter->dev, "Failure in recv_msg: h_copy_rdma = 0x%x\n",
+			rc);
+		spin_unlock_irqrestore(&hmc->lock, flags);
+		return -1;
+	}
+
+	/* Must be locked because read operates on the same data */
+	hmc->queue_outbound_msgs[hmc->queue_head] = buffer_id;
+	hmc->queue_head++;
+	if (hmc->queue_head == ibmvmc_max_buf_pool_size)
+		hmc->queue_head = 0;
+
+	if (hmc->queue_head == hmc->queue_tail)
+		dev_err(adapter->dev, "outbound buffer queue wrapped.\n");
+
+	spin_unlock_irqrestore(&hmc->lock, flags);
+
+	wake_up_interruptible(&ibmvmc_read_wait);
+
+	return 0;
+}
+
+/**
+ * ibmvmc_process_capabilities - Process Capabilities
+ *
+ * @adapter:	crq_server_adapter struct
+ * @crqp:	ibmvmc_crq_msg struct
+ *
+ */
+static void ibmvmc_process_capabilities(struct crq_server_adapter *adapter,
+					struct ibmvmc_crq_msg *crqp)
+{
+	struct ibmvmc_admin_crq_msg *crq = (struct ibmvmc_admin_crq_msg *)crqp;
+
+	if ((be16_to_cpu(crq->version) >> 8) !=
+			(IBMVMC_PROTOCOL_VERSION >> 8)) {
+		dev_err(adapter->dev, "init failed, incompatible versions 0x%x 0x%x\n",
+			be16_to_cpu(crq->version),
+			IBMVMC_PROTOCOL_VERSION);
+		ibmvmc.state = ibmvmc_state_failed;
+		return;
+	}
+
+	ibmvmc.max_mtu = min_t(u32, ibmvmc_max_mtu, be32_to_cpu(crq->max_mtu));
+	ibmvmc.max_buffer_pool_size = min_t(u16, ibmvmc_max_buf_pool_size,
+					    be16_to_cpu(crq->pool_size));
+	ibmvmc.max_hmc_index = min_t(u8, ibmvmc_max_hmcs, crq->max_hmc) - 1;
+	ibmvmc.state = ibmvmc_state_ready;
+
+	dev_info(adapter->dev, "Capabilities: mtu=0x%x, pool_size=0x%x, max_hmc=0x%x\n",
+		 ibmvmc.max_mtu, ibmvmc.max_buffer_pool_size,
+		 ibmvmc.max_hmc_index);
+}
+
+/**
+ * ibmvmc_validate_hmc_session - Validate HMC Session
+ *
+ * @adapter:	crq_server_adapter struct
+ * @crq:	ibmvmc_crq_msg struct
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvmc_validate_hmc_session(struct crq_server_adapter *adapter,
+				       struct ibmvmc_crq_msg *crq)
+{
+	unsigned char hmc_index;
+
+	hmc_index = crq->hmc_index;
+
+	if (crq->hmc_session == 0)
+		return 0;
+
+	if (hmc_index > ibmvmc.max_hmc_index)
+		return -1;
+
+	if (hmcs[hmc_index].session != crq->hmc_session) {
+		dev_warn(adapter->dev, "Drop, bad session: expected 0x%x, recv 0x%x\n",
+			 hmcs[hmc_index].session, crq->hmc_session);
+		return -1;
+	}
+
+	return 0;
+}
+
+/**
+ * ibmvmc_reset - Reset
+ *
+ * @adapter:	crq_server_adapter struct
+ * @xport_event:	export_event field
+ *
+ * Closes all HMC sessions and conditionally schedules a CRQ reset.
+ * @xport_event: If true, the partner closed their CRQ; we don't need to reset.
+ *               If false, we need to schedule a CRQ reset.
+ */
+static void ibmvmc_reset(struct crq_server_adapter *adapter, bool xport_event)
+{
+	int i;
+
+	if (ibmvmc.state != ibmvmc_state_sched_reset) {
+		dev_info(adapter->dev, "*** Reset to initial state.\n");
+		for (i = 0; i < ibmvmc_max_hmcs; i++)
+			ibmvmc_return_hmc(&hmcs[i], xport_event);
+
+		if (xport_event) {
+			/* CRQ was closed by the partner.  We don't need to do
+			 * anything except set ourself to the correct state to
+			 * handle init msgs.
+			 */
+			ibmvmc.state = ibmvmc_state_crqinit;
+		} else {
+			/* The partner did not close their CRQ - instead, we're
+			 * closing the CRQ on our end. Need to schedule this
+			 * for process context, because CRQ reset may require a
+			 * sleep.
+			 *
+			 * Setting ibmvmc.state here immediately prevents
+			 * ibmvmc_open from completing until the reset
+			 * completes in process context.
+			 */
+			ibmvmc.state = ibmvmc_state_sched_reset;
+			dev_dbg(adapter->dev, "Device reset scheduled");
+			wake_up_interruptible(&adapter->reset_wait_queue);
+		}
+	}
+}
+
+/**
+ * ibmvmc_reset_task - Reset Task
+ *
+ * @data:	Data field
+ *
+ * Performs a CRQ reset of the VMC device in process context.
+ * NOTE: This function should not be called directly, use ibmvmc_reset.
+ */
+static int ibmvmc_reset_task(void *data)
+{
+	struct crq_server_adapter *adapter = data;
+	int rc;
+
+	set_user_nice(current, -20);
+
+	while (!kthread_should_stop()) {
+		wait_event_interruptible(adapter->reset_wait_queue,
+			(ibmvmc.state == ibmvmc_state_sched_reset) ||
+			kthread_should_stop());
+
+		if (kthread_should_stop())
+			break;
+
+		dev_dbg(adapter->dev, "CRQ resetting in process context");
+		tasklet_disable(&adapter->work_task);
+
+		rc = ibmvmc_reset_crq_queue(adapter);
+
+		if (rc != H_SUCCESS && rc != H_RESOURCE) {
+			dev_err(adapter->dev, "Error initializing CRQ.  rc = 0x%x\n",
+				rc);
+			ibmvmc.state = ibmvmc_state_failed;
+		} else {
+			ibmvmc.state = ibmvmc_state_crqinit;
+
+			if (ibmvmc_send_crq(adapter, 0xC001000000000000LL, 0)
+			    != 0 && rc != H_RESOURCE)
+				dev_warn(adapter->dev, "Failed to send initialize CRQ message\n");
+		}
+
+		vio_enable_interrupts(to_vio_dev(adapter->dev));
+		tasklet_enable(&adapter->work_task);
+	}
+
+	return 0;
+}
+
+/**
+ * ibmvmc_process_open_resp - Process Open Response
+ *
+ * @crq: ibmvmc_crq_msg struct
+ * @adapter:    crq_server_adapter struct
+ *
+ * This command is sent by the hypervisor in response to the Interface
+ * Open message. When this message is received, the indicated buffer is
+ * again available for management partition use.
+ */
+static void ibmvmc_process_open_resp(struct ibmvmc_crq_msg *crq,
+				     struct crq_server_adapter *adapter)
+{
+	unsigned char hmc_index;
+	unsigned short buffer_id;
+
+	hmc_index = crq->hmc_index;
+	if (hmc_index > ibmvmc.max_hmc_index) {
+		/* Why would PHYP give an index > max negotiated? */
+		ibmvmc_reset(adapter, false);
+		return;
+	}
+
+	if (crq->status) {
+		dev_warn(adapter->dev, "open_resp: failed - status 0x%x\n",
+			 crq->status);
+		ibmvmc_return_hmc(&hmcs[hmc_index], false);
+		return;
+	}
+
+	if (hmcs[hmc_index].state == ibmhmc_state_opening) {
+		buffer_id = be16_to_cpu(crq->var2.buffer_id);
+		if (buffer_id >= ibmvmc.max_buffer_pool_size) {
+			dev_err(adapter->dev, "open_resp: invalid buffer_id = 0x%x\n",
+				buffer_id);
+			hmcs[hmc_index].state = ibmhmc_state_failed;
+		} else {
+			ibmvmc_free_hmc_buffer(&hmcs[hmc_index],
+					       &hmcs[hmc_index].buffer[buffer_id]);
+			hmcs[hmc_index].state = ibmhmc_state_ready;
+			dev_dbg(adapter->dev, "open_resp: set hmc state = ready\n");
+		}
+	} else {
+		dev_warn(adapter->dev, "open_resp: invalid hmc state (0x%x)\n",
+			 hmcs[hmc_index].state);
+	}
+}
+
+/**
+ * ibmvmc_process_close_resp - Process Close Response
+ *
+ * @crq: ibmvmc_crq_msg struct
+ * @adapter:    crq_server_adapter struct
+ *
+ * This command is sent by the hypervisor in response to the managemant
+ * application Interface Close message.
+ *
+ * If the close fails, simply reset the entire driver as the state of the VMC
+ * must be in tough shape.
+ */
+static void ibmvmc_process_close_resp(struct ibmvmc_crq_msg *crq,
+				      struct crq_server_adapter *adapter)
+{
+	unsigned char hmc_index;
+
+	hmc_index = crq->hmc_index;
+	if (hmc_index > ibmvmc.max_hmc_index) {
+		ibmvmc_reset(adapter, false);
+		return;
+	}
+
+	if (crq->status) {
+		dev_warn(adapter->dev, "close_resp: failed - status 0x%x\n",
+			 crq->status);
+		ibmvmc_reset(adapter, false);
+		return;
+	}
+
+	ibmvmc_return_hmc(&hmcs[hmc_index], false);
+}
+
+/**
+ * ibmvmc_crq_process - Process CRQ
+ *
+ * @adapter:    crq_server_adapter struct
+ * @crq:	ibmvmc_crq_msg struct
+ *
+ * Process the CRQ message based upon the type of message received.
+ *
+ */
+static void ibmvmc_crq_process(struct crq_server_adapter *adapter,
+			       struct ibmvmc_crq_msg *crq)
+{
+	switch (crq->type) {
+	case VMC_MSG_CAP_RESP:
+		dev_dbg(adapter->dev, "CRQ recv: capabilities resp (0x%x)\n",
+			crq->type);
+		if (ibmvmc.state == ibmvmc_state_capabilities)
+			ibmvmc_process_capabilities(adapter, crq);
+		else
+			dev_warn(adapter->dev, "caps msg invalid in state 0x%x\n",
+				 ibmvmc.state);
+		break;
+	case VMC_MSG_OPEN_RESP:
+		dev_dbg(adapter->dev, "CRQ recv: open resp (0x%x)\n",
+			crq->type);
+		if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
+			ibmvmc_process_open_resp(crq, adapter);
+		break;
+	case VMC_MSG_ADD_BUF:
+		dev_dbg(adapter->dev, "CRQ recv: add buf (0x%x)\n",
+			crq->type);
+		if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
+			ibmvmc_add_buffer(adapter, crq);
+		break;
+	case VMC_MSG_REM_BUF:
+		dev_dbg(adapter->dev, "CRQ recv: rem buf (0x%x)\n",
+			crq->type);
+		if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
+			ibmvmc_rem_buffer(adapter, crq);
+		break;
+	case VMC_MSG_SIGNAL:
+		dev_dbg(adapter->dev, "CRQ recv: signal msg (0x%x)\n",
+			crq->type);
+		if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
+			ibmvmc_recv_msg(adapter, crq);
+		break;
+	case VMC_MSG_CLOSE_RESP:
+		dev_dbg(adapter->dev, "CRQ recv: close resp (0x%x)\n",
+			crq->type);
+		if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
+			ibmvmc_process_close_resp(crq, adapter);
+		break;
+	case VMC_MSG_CAP:
+	case VMC_MSG_OPEN:
+	case VMC_MSG_CLOSE:
+	case VMC_MSG_ADD_BUF_RESP:
+	case VMC_MSG_REM_BUF_RESP:
+		dev_warn(adapter->dev, "CRQ recv: unexpected msg (0x%x)\n",
+			 crq->type);
+		break;
+	default:
+		dev_warn(adapter->dev, "CRQ recv: unknown msg (0x%x)\n",
+			 crq->type);
+		break;
+	}
+}
+
+/**
+ * ibmvmc_handle_crq_init - Handle CRQ Init
+ *
+ * @crq:	ibmvmc_crq_msg struct
+ * @adapter:	crq_server_adapter struct
+ *
+ * Handle the type of crq initialization based on whether
+ * it is a message or a response.
+ *
+ */
+static void ibmvmc_handle_crq_init(struct ibmvmc_crq_msg *crq,
+				   struct crq_server_adapter *adapter)
+{
+	switch (crq->type) {
+	case 0x01:	/* Initialization message */
+		dev_dbg(adapter->dev, "CRQ recv: CRQ init msg - state 0x%x\n",
+			ibmvmc.state);
+		if (ibmvmc.state == ibmvmc_state_crqinit) {
+			/* Send back a response */
+			if (ibmvmc_send_crq(adapter, 0xC002000000000000,
+					    0) == 0)
+				ibmvmc_send_capabilities(adapter);
+			else
+				dev_err(adapter->dev, " Unable to send init rsp\n");
+		} else {
+			dev_err(adapter->dev, "Invalid state 0x%x mtu = 0x%x\n",
+				ibmvmc.state, ibmvmc.max_mtu);
+		}
+
+		break;
+	case 0x02:	/* Initialization response */
+		dev_dbg(adapter->dev, "CRQ recv: initialization resp msg - state 0x%x\n",
+			ibmvmc.state);
+		if (ibmvmc.state == ibmvmc_state_crqinit)
+			ibmvmc_send_capabilities(adapter);
+		break;
+	default:
+		dev_warn(adapter->dev, "Unknown crq message type 0x%lx\n",
+			 (unsigned long)crq->type);
+	}
+}
+
+/**
+ * ibmvmc_handle_crq - Handle CRQ
+ *
+ * @crq:	ibmvmc_crq_msg struct
+ * @adapter:	crq_server_adapter struct
+ *
+ * Read the command elements from the command queue and execute the
+ * requests based upon the type of crq message.
+ *
+ */
+static void ibmvmc_handle_crq(struct ibmvmc_crq_msg *crq,
+			      struct crq_server_adapter *adapter)
+{
+	switch (crq->valid) {
+	case 0xC0:		/* initialization */
+		ibmvmc_handle_crq_init(crq, adapter);
+		break;
+	case 0xFF:	/* Hypervisor telling us the connection is closed */
+		dev_warn(adapter->dev, "CRQ recv: virtual adapter failed - resetting.\n");
+		ibmvmc_reset(adapter, true);
+		break;
+	case 0x80:	/* real payload */
+		ibmvmc_crq_process(adapter, crq);
+		break;
+	default:
+		dev_warn(adapter->dev, "CRQ recv: unknown msg 0x%02x.\n",
+			 crq->valid);
+		break;
+	}
+}
+
+static void ibmvmc_task(unsigned long data)
+{
+	struct crq_server_adapter *adapter =
+		(struct crq_server_adapter *)data;
+	struct vio_dev *vdev = to_vio_dev(adapter->dev);
+	struct ibmvmc_crq_msg *crq;
+	int done = 0;
+
+	while (!done) {
+		/* Pull all the valid messages off the CRQ */
+		while ((crq = crq_queue_next_crq(&adapter->queue)) != NULL) {
+			ibmvmc_handle_crq(crq, adapter);
+			crq->valid = 0x00;
+			/* CRQ reset was requested, stop processing CRQs.
+			 * Interrupts will be re-enabled by the reset task.
+			 */
+			if (ibmvmc.state == ibmvmc_state_sched_reset)
+				return;
+		}
+
+		vio_enable_interrupts(vdev);
+		crq = crq_queue_next_crq(&adapter->queue);
+		if (crq) {
+			vio_disable_interrupts(vdev);
+			ibmvmc_handle_crq(crq, adapter);
+			crq->valid = 0x00;
+			/* CRQ reset was requested, stop processing CRQs.
+			 * Interrupts will be re-enabled by the reset task.
+			 */
+			if (ibmvmc.state == ibmvmc_state_sched_reset)
+				return;
+		} else {
+			done = 1;
+		}
+	}
+}
+
+/**
+ * ibmvmc_init_crq_queue - Init CRQ Queue
+ *
+ * @adapter:	crq_server_adapter struct
+ *
+ * Return:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvmc_init_crq_queue(struct crq_server_adapter *adapter)
+{
+	struct vio_dev *vdev = to_vio_dev(adapter->dev);
+	struct crq_queue *queue = &adapter->queue;
+	int rc = 0;
+	int retrc = 0;
+
+	queue->msgs = (struct ibmvmc_crq_msg *)get_zeroed_page(GFP_KERNEL);
+
+	if (!queue->msgs)
+		goto malloc_failed;
+
+	queue->size = PAGE_SIZE / sizeof(*queue->msgs);
+
+	queue->msg_token = dma_map_single(adapter->dev, queue->msgs,
+					  queue->size * sizeof(*queue->msgs),
+					  DMA_BIDIRECTIONAL);
+
+	if (dma_mapping_error(adapter->dev, queue->msg_token))
+		goto map_failed;
+
+	retrc = plpar_hcall_norets(H_REG_CRQ,
+				   vdev->unit_address,
+				   queue->msg_token, PAGE_SIZE);
+	retrc = rc;
+
+	if (rc == H_RESOURCE)
+		rc = ibmvmc_reset_crq_queue(adapter);
+
+	if (rc == 2) {
+		dev_warn(adapter->dev, "Partner adapter not ready\n");
+		retrc = 0;
+	} else if (rc != 0) {
+		dev_err(adapter->dev, "Error %d opening adapter\n", rc);
+		goto reg_crq_failed;
+	}
+
+	queue->cur = 0;
+	spin_lock_init(&queue->lock);
+
+	tasklet_init(&adapter->work_task, ibmvmc_task, (unsigned long)adapter);
+
+	if (request_irq(vdev->irq,
+			ibmvmc_handle_event,
+			0, "ibmvmc", (void *)adapter) != 0) {
+		dev_err(adapter->dev, "couldn't register irq 0x%x\n",
+			vdev->irq);
+		goto req_irq_failed;
+	}
+
+	rc = vio_enable_interrupts(vdev);
+	if (rc != 0) {
+		dev_err(adapter->dev, "Error %d enabling interrupts!!!\n", rc);
+		goto req_irq_failed;
+	}
+
+	return retrc;
+
+req_irq_failed:
+	/* Cannot have any work since we either never got our IRQ registered,
+	 * or never got interrupts enabled
+	 */
+	tasklet_kill(&adapter->work_task);
+	h_free_crq(vdev->unit_address);
+reg_crq_failed:
+	dma_unmap_single(adapter->dev,
+			 queue->msg_token,
+			 queue->size * sizeof(*queue->msgs), DMA_BIDIRECTIONAL);
+map_failed:
+	free_page((unsigned long)queue->msgs);
+malloc_failed:
+	return -ENOMEM;
+}
+
+/* Fill in the liobn and riobn fields on the adapter */
+static int read_dma_window(struct vio_dev *vdev,
+			   struct crq_server_adapter *adapter)
+{
+	const __be32 *dma_window;
+	const __be32 *prop;
+
+	/* TODO Using of_parse_dma_window would be better, but it doesn't give
+	 * a way to read multiple windows without already knowing the size of
+	 * a window or the number of windows
+	 */
+	dma_window =
+		(const __be32 *)vio_get_attribute(vdev, "ibm,my-dma-window",
+						NULL);
+	if (!dma_window) {
+		dev_warn(adapter->dev, "Couldn't find ibm,my-dma-window property\n");
+		return -1;
+	}
+
+	adapter->liobn = be32_to_cpu(*dma_window);
+	dma_window++;
+
+	prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-address-cells",
+						NULL);
+	if (!prop) {
+		dev_warn(adapter->dev, "Couldn't find ibm,#dma-address-cells property\n");
+		dma_window++;
+	} else {
+		dma_window += be32_to_cpu(*prop);
+	}
+
+	prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-size-cells",
+						NULL);
+	if (!prop) {
+		dev_warn(adapter->dev, "Couldn't find ibm,#dma-size-cells property\n");
+		dma_window++;
+	} else {
+		dma_window += be32_to_cpu(*prop);
+	}
+
+	/* dma_window should point to the second window now */
+	adapter->riobn = be32_to_cpu(*dma_window);
+
+	return 0;
+}
+
+static int ibmvmc_probe(struct vio_dev *vdev, const struct vio_device_id *id)
+{
+	struct crq_server_adapter *adapter = &ibmvmc_adapter;
+	int rc;
+
+	dev_set_drvdata(&vdev->dev, NULL);
+	memset(adapter, 0, sizeof(*adapter));
+	adapter->dev = &vdev->dev;
+
+	dev_info(adapter->dev, "Probe for UA 0x%x\n", vdev->unit_address);
+
+	rc = read_dma_window(vdev, adapter);
+	if (rc != 0) {
+		ibmvmc.state = ibmvmc_state_failed;
+		return -1;
+	}
+
+	dev_dbg(adapter->dev, "Probe: liobn 0x%x, riobn 0x%x\n",
+		adapter->liobn, adapter->riobn);
+
+	init_waitqueue_head(&adapter->reset_wait_queue);
+	adapter->reset_task = kthread_run(ibmvmc_reset_task, adapter, "ibmvmc");
+	if (IS_ERR(adapter->reset_task)) {
+		dev_err(adapter->dev, "Failed to start reset thread\n");
+		ibmvmc.state = ibmvmc_state_failed;
+		rc = PTR_ERR(adapter->reset_task);
+		adapter->reset_task = NULL;
+		return rc;
+	}
+
+	rc = ibmvmc_init_crq_queue(adapter);
+	if (rc != 0 && rc != H_RESOURCE) {
+		dev_err(adapter->dev, "Error initializing CRQ.  rc = 0x%x\n",
+			rc);
+		ibmvmc.state = ibmvmc_state_failed;
+		goto crq_failed;
+	}
+
+	ibmvmc.state = ibmvmc_state_crqinit;
+
+	/* Try to send an initialization message.  Note that this is allowed
+	 * to fail if the other end is not acive.  In that case we just wait
+	 * for the other side to initialize.
+	 */
+	if (ibmvmc_send_crq(adapter, 0xC001000000000000LL, 0) != 0 &&
+	    rc != H_RESOURCE)
+		dev_warn(adapter->dev, "Failed to send initialize CRQ message\n");
+
+	dev_set_drvdata(&vdev->dev, adapter);
+
+	return 0;
+
+crq_failed:
+	kthread_stop(adapter->reset_task);
+	adapter->reset_task = NULL;
+	return -EPERM;
+}
+
+static int ibmvmc_remove(struct vio_dev *vdev)
+{
+	struct crq_server_adapter *adapter = dev_get_drvdata(&vdev->dev);
+
+	dev_info(adapter->dev, "Entering remove for UA 0x%x\n",
+		 vdev->unit_address);
+	ibmvmc_release_crq_queue(adapter);
+
+	return 0;
+}
+
+static struct vio_device_id ibmvmc_device_table[] = {
+	{ "ibm,vmc", "IBM,vmc" },
+	{ "", "" }
+};
+MODULE_DEVICE_TABLE(vio, ibmvmc_device_table);
+
+static struct vio_driver ibmvmc_driver = {
+	.name        = ibmvmc_driver_name,
+	.id_table    = ibmvmc_device_table,
+	.probe       = ibmvmc_probe,
+	.remove      = ibmvmc_remove,
+};
+
+static void __init ibmvmc_scrub_module_parms(void)
+{
+	if (ibmvmc_max_mtu > MAX_MTU) {
+		pr_warn("ibmvmc: Max MTU reduced to %d\n", MAX_MTU);
+		ibmvmc_max_mtu = MAX_MTU;
+	} else if (ibmvmc_max_mtu < MIN_MTU) {
+		pr_warn("ibmvmc: Max MTU increased to %d\n", MIN_MTU);
+		ibmvmc_max_mtu = MIN_MTU;
+	}
+
+	if (ibmvmc_max_buf_pool_size > MAX_BUF_POOL_SIZE) {
+		pr_warn("ibmvmc: Max buffer pool size reduced to %d\n",
+			MAX_BUF_POOL_SIZE);
+		ibmvmc_max_buf_pool_size = MAX_BUF_POOL_SIZE;
+	} else if (ibmvmc_max_buf_pool_size < MIN_BUF_POOL_SIZE) {
+		pr_warn("ibmvmc: Max buffer pool size increased to %d\n",
+			MIN_BUF_POOL_SIZE);
+		ibmvmc_max_buf_pool_size = MIN_BUF_POOL_SIZE;
+	}
+
+	if (ibmvmc_max_hmcs > MAX_HMCS) {
+		pr_warn("ibmvmc: Max HMCs reduced to %d\n", MAX_HMCS);
+		ibmvmc_max_hmcs = MAX_HMCS;
+	} else if (ibmvmc_max_hmcs < MIN_HMCS) {
+		pr_warn("ibmvmc: Max HMCs increased to %d\n", MIN_HMCS);
+		ibmvmc_max_hmcs = MIN_HMCS;
+	}
+}
+
+static struct miscdevice ibmvmc_miscdev = {
+	.name = ibmvmc_driver_name,
+	.minor = MISC_DYNAMIC_MINOR,
+	.fops = &ibmvmc_fops,
+};
+
+static int __init ibmvmc_module_init(void)
+{
+	int rc, i, j;
+
+	ibmvmc.state = ibmvmc_state_initial;
+	pr_info("ibmvmc: version %s\n", IBMVMC_DRIVER_VERSION);
+
+	rc = misc_register(&ibmvmc_miscdev);
+	if (rc) {
+		pr_err("ibmvmc: misc registration failed\n");
+		goto misc_register_failed;
+	}
+	pr_info("ibmvmc: node %d:%d\n", MISC_MAJOR,
+		ibmvmc_miscdev.minor);
+
+	/* Initialize data structures */
+	memset(hmcs, 0, sizeof(struct ibmvmc_hmc) * MAX_HMCS);
+	for (i = 0; i < MAX_HMCS; i++) {
+		spin_lock_init(&hmcs[i].lock);
+		hmcs[i].state = ibmhmc_state_free;
+		for (j = 0; j < MAX_BUF_POOL_SIZE; j++)
+			hmcs[i].queue_outbound_msgs[j] = VMC_INVALID_BUFFER_ID;
+	}
+
+	/* Sanity check module parms */
+	ibmvmc_scrub_module_parms();
+
+	/*
+	 * Initialize some reasonable values.  Might be negotiated smaller
+	 * values during the capabilities exchange.
+	 */
+	ibmvmc.max_mtu = ibmvmc_max_mtu;
+	ibmvmc.max_buffer_pool_size = ibmvmc_max_buf_pool_size;
+	ibmvmc.max_hmc_index = ibmvmc_max_hmcs - 1;
+
+	rc = vio_register_driver(&ibmvmc_driver);
+
+	if (rc) {
+		pr_err("ibmvmc: rc %d from vio_register_driver\n", rc);
+		goto vio_reg_failed;
+	}
+
+	return 0;
+
+vio_reg_failed:
+	misc_deregister(&ibmvmc_miscdev);
+misc_register_failed:
+	return rc;
+}
+
+static void __exit ibmvmc_module_exit(void)
+{
+	pr_info("ibmvmc: module exit\n");
+	vio_unregister_driver(&ibmvmc_driver);
+	misc_deregister(&ibmvmc_miscdev);
+}
+
+module_init(ibmvmc_module_init);
+module_exit(ibmvmc_module_exit);
+
+module_param_named(buf_pool_size, ibmvmc_max_buf_pool_size,
+		   int, 0644);
+MODULE_PARM_DESC(buf_pool_size, "Buffer pool size");
+module_param_named(max_hmcs, ibmvmc_max_hmcs, int, 0644);
+MODULE_PARM_DESC(max_hmcs, "Max HMCs");
+module_param_named(max_mtu, ibmvmc_max_mtu, int, 0644);
+MODULE_PARM_DESC(max_mtu, "Max MTU");
+
+MODULE_AUTHOR("Steven Royer <seroyer@linux.vnet.ibm.com>");
+MODULE_DESCRIPTION("IBM VMC");
+MODULE_VERSION(IBMVMC_DRIVER_VERSION);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/misc/ibmvmc.h b/drivers/misc/ibmvmc.h
new file mode 100644
index 000000000000..e140ada8fe2c
--- /dev/null
+++ b/drivers/misc/ibmvmc.h
@@ -0,0 +1,209 @@
+/* SPDX-License-Identifier: GPL-2.0+
+ *
+ * linux/drivers/misc/ibmvmc.h
+ *
+ * IBM Power Systems Virtual Management Channel Support.
+ *
+ * Copyright (c) 2004, 2018 IBM Corp.
+ *   Dave Engebretsen engebret@us.ibm.com
+ *   Steven Royer seroyer@linux.vnet.ibm.com
+ *   Adam Reznechek adreznec@linux.vnet.ibm.com
+ *   Bryant G. Ly <bryantly@linux.vnet.ibm.com>
+ */
+#ifndef IBMVMC_H
+#define IBMVMC_H
+
+#include <linux/types.h>
+#include <linux/cdev.h>
+
+#include <asm/vio.h>
+
+#define IBMVMC_PROTOCOL_VERSION    0x0101
+
+#define MIN_BUF_POOL_SIZE 16
+#define MIN_HMCS          1
+#define MIN_MTU           4096
+#define MAX_BUF_POOL_SIZE 64
+#define MAX_HMCS          2
+#define MAX_MTU           (4 * 4096)
+#define DEFAULT_BUF_POOL_SIZE 32
+#define DEFAULT_HMCS          1
+#define DEFAULT_MTU           4096
+#define HMC_ID_LEN        32
+
+#define VMC_INVALID_BUFFER_ID 0xFFFF
+
+/* ioctl numbers */
+#define VMC_BASE	     0xCC
+#define VMC_IOCTL_SETHMCID   _IOW(VMC_BASE, 0x00, unsigned char *)
+#define VMC_IOCTL_QUERY      _IOR(VMC_BASE, 0x01, struct ibmvmc_query_struct)
+#define VMC_IOCTL_REQUESTVMC _IOR(VMC_BASE, 0x02, u32)
+
+#define VMC_MSG_CAP          0x01
+#define VMC_MSG_CAP_RESP     0x81
+#define VMC_MSG_OPEN         0x02
+#define VMC_MSG_OPEN_RESP    0x82
+#define VMC_MSG_CLOSE        0x03
+#define VMC_MSG_CLOSE_RESP   0x83
+#define VMC_MSG_ADD_BUF      0x04
+#define VMC_MSG_ADD_BUF_RESP 0x84
+#define VMC_MSG_REM_BUF      0x05
+#define VMC_MSG_REM_BUF_RESP 0x85
+#define VMC_MSG_SIGNAL       0x06
+
+#define VMC_MSG_SUCCESS 0
+#define VMC_MSG_INVALID_HMC_INDEX 1
+#define VMC_MSG_INVALID_BUFFER_ID 2
+#define VMC_MSG_CLOSED_HMC        3
+#define VMC_MSG_INTERFACE_FAILURE 4
+#define VMC_MSG_NO_BUFFER         5
+
+#define VMC_BUF_OWNER_ALPHA 0
+#define VMC_BUF_OWNER_HV    1
+
+enum ibmvmc_states {
+	ibmvmc_state_sched_reset  = -1,
+	ibmvmc_state_initial      = 0,
+	ibmvmc_state_crqinit      = 1,
+	ibmvmc_state_capabilities = 2,
+	ibmvmc_state_ready        = 3,
+	ibmvmc_state_failed       = 4,
+};
+
+enum ibmhmc_states {
+	/* HMC connection not established */
+	ibmhmc_state_free    = 0,
+
+	/* HMC connection established (open called) */
+	ibmhmc_state_initial = 1,
+
+	/* open msg sent to HV, due to ioctl(1) call */
+	ibmhmc_state_opening = 2,
+
+	/* HMC connection ready, open resp msg from HV */
+	ibmhmc_state_ready   = 3,
+
+	/* HMC connection failure */
+	ibmhmc_state_failed  = 4,
+};
+
+struct ibmvmc_buffer {
+	u8 valid;	/* 1 when DMA storage allocated to buffer          */
+	u8 free;	/* 1 when buffer available for the Alpha Partition */
+	u8 owner;
+	u16 id;
+	u32 size;
+	u32 msg_len;
+	dma_addr_t dma_addr_local;
+	dma_addr_t dma_addr_remote;
+	void *real_addr_local;
+};
+
+struct ibmvmc_admin_crq_msg {
+	u8 valid;	/* RPA Defined           */
+	u8 type;	/* ibmvmc msg type       */
+	u8 status;	/* Response msg status. Zero is success and on failure,
+			 * either 1 - General Failure, or 2 - Invalid Version is
+			 * returned.
+			 */
+	u8 rsvd[2];
+	u8 max_hmc;	/* Max # of independent HMC connections supported */
+	__be16 pool_size;	/* Maximum number of buffers supported per HMC
+				 * connection
+				 */
+	__be32 max_mtu;		/* Maximum message size supported (bytes) */
+	__be16 crq_size;	/* # of entries available in the CRQ for the
+				 * source partition. The target partition must
+				 * limit the number of outstanding messages to
+				 * one half or less.
+				 */
+	__be16 version;	/* Indicates the code level of the management partition
+			 * or the hypervisor with the high-order byte
+			 * indicating a major version and the low-order byte
+			 * indicating a minor version.
+			 */
+};
+
+struct ibmvmc_crq_msg {
+	u8 valid;     /* RPA Defined           */
+	u8 type;      /* ibmvmc msg type       */
+	u8 status;    /* Response msg status   */
+	union {
+		u8 rsvd;  /* Reserved              */
+		u8 owner;
+	} var1;
+	u8 hmc_session;	/* Session Identifier for the current VMC connection */
+	u8 hmc_index;	/* A unique HMC Idx would be used if multiple management
+			 * applications running concurrently were desired
+			 */
+	union {
+		__be16 rsvd;
+		__be16 buffer_id;
+	} var2;
+	__be32 rsvd;
+	union {
+		__be32 rsvd;
+		__be32 lioba;
+		__be32 msg_len;
+	} var3;
+};
+
+/* an RPA command/response transport queue */
+struct crq_queue {
+	struct ibmvmc_crq_msg *msgs;
+	int size, cur;
+	dma_addr_t msg_token;
+	spinlock_t lock;
+};
+
+/* VMC server adapter settings */
+struct crq_server_adapter {
+	struct device *dev;
+	struct crq_queue queue;
+	u32 liobn;
+	u32 riobn;
+	struct tasklet_struct work_task;
+	wait_queue_head_t reset_wait_queue;
+	struct task_struct *reset_task;
+};
+
+/* Driver wide settings */
+struct ibmvmc_struct {
+	u32 state;
+	u32 max_mtu;
+	u32 max_buffer_pool_size;
+	u32 max_hmc_index;
+	struct crq_server_adapter *adapter;
+	struct cdev cdev;
+	u32 vmc_drc_index;
+};
+
+struct ibmvmc_file_session;
+
+/* Connection specific settings */
+struct ibmvmc_hmc {
+	u8 session;
+	u8 index;
+	u32 state;
+	struct crq_server_adapter *adapter;
+	spinlock_t lock;
+	unsigned char hmc_id[HMC_ID_LEN];
+	struct ibmvmc_buffer buffer[MAX_BUF_POOL_SIZE];
+	unsigned short queue_outbound_msgs[MAX_BUF_POOL_SIZE];
+	int queue_head, queue_tail;
+	struct ibmvmc_file_session *file_session;
+};
+
+struct ibmvmc_file_session {
+	struct file *file;
+	struct ibmvmc_hmc *hmc;
+	bool valid;
+};
+
+struct ibmvmc_query_struct {
+	int have_vmc;
+	int state;
+	int vmc_drc_index;
+};
+
+#endif /* __IBMVMC_H */
diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c
index fe8897e64635..7b370466a227 100644
--- a/drivers/misc/pci_endpoint_test.c
+++ b/drivers/misc/pci_endpoint_test.c
@@ -203,7 +203,7 @@ static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test,
 	if (!val)
 		return false;
 
-	if (test->last_irq - pdev->irq == msi_num - 1)
+	if (pci_irq_vector(pdev, msi_num - 1) == test->last_irq)
 		return true;
 
 	return false;
@@ -233,7 +233,7 @@ static bool pci_endpoint_test_copy(struct pci_endpoint_test *test, size_t size)
 	orig_src_addr = dma_alloc_coherent(dev, size + alignment,
 					   &orig_src_phys_addr, GFP_KERNEL);
 	if (!orig_src_addr) {
-		dev_err(dev, "failed to allocate source buffer\n");
+		dev_err(dev, "Failed to allocate source buffer\n");
 		ret = false;
 		goto err;
 	}
@@ -259,7 +259,7 @@ static bool pci_endpoint_test_copy(struct pci_endpoint_test *test, size_t size)
 	orig_dst_addr = dma_alloc_coherent(dev, size + alignment,
 					   &orig_dst_phys_addr, GFP_KERNEL);
 	if (!orig_dst_addr) {
-		dev_err(dev, "failed to allocate destination address\n");
+		dev_err(dev, "Failed to allocate destination address\n");
 		ret = false;
 		goto err_orig_src_addr;
 	}
@@ -321,7 +321,7 @@ static bool pci_endpoint_test_write(struct pci_endpoint_test *test, size_t size)
 	orig_addr = dma_alloc_coherent(dev, size + alignment, &orig_phys_addr,
 				       GFP_KERNEL);
 	if (!orig_addr) {
-		dev_err(dev, "failed to allocate address\n");
+		dev_err(dev, "Failed to allocate address\n");
 		ret = false;
 		goto err;
 	}
@@ -382,7 +382,7 @@ static bool pci_endpoint_test_read(struct pci_endpoint_test *test, size_t size)
 	orig_addr = dma_alloc_coherent(dev, size + alignment, &orig_phys_addr,
 				       GFP_KERNEL);
 	if (!orig_addr) {
-		dev_err(dev, "failed to allocate destination address\n");
+		dev_err(dev, "Failed to allocate destination address\n");
 		ret = false;
 		goto err;
 	}
@@ -513,31 +513,31 @@ static int pci_endpoint_test_probe(struct pci_dev *pdev,
 	if (!no_msi) {
 		irq = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
 		if (irq < 0)
-			dev_err(dev, "failed to get MSI interrupts\n");
+			dev_err(dev, "Failed to get MSI interrupts\n");
 		test->num_irqs = irq;
 	}
 
 	err = devm_request_irq(dev, pdev->irq, pci_endpoint_test_irqhandler,
 			       IRQF_SHARED, DRV_MODULE_NAME, test);
 	if (err) {
-		dev_err(dev, "failed to request IRQ %d\n", pdev->irq);
+		dev_err(dev, "Failed to request IRQ %d\n", pdev->irq);
 		goto err_disable_msi;
 	}
 
 	for (i = 1; i < irq; i++) {
-		err = devm_request_irq(dev, pdev->irq + i,
+		err = devm_request_irq(dev, pci_irq_vector(pdev, i),
 				       pci_endpoint_test_irqhandler,
 				       IRQF_SHARED, DRV_MODULE_NAME, test);
 		if (err)
 			dev_err(dev, "failed to request IRQ %d for MSI %d\n",
-				pdev->irq + i, i + 1);
+				pci_irq_vector(pdev, i), i + 1);
 	}
 
 	for (bar = BAR_0; bar <= BAR_5; bar++) {
 		if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
 			base = pci_ioremap_bar(pdev, bar);
 			if (!base) {
-				dev_err(dev, "failed to read BAR%d\n", bar);
+				dev_err(dev, "Failed to read BAR%d\n", bar);
 				WARN_ON(bar == test_reg_bar);
 			}
 			test->bar[bar] = base;
@@ -557,7 +557,7 @@ static int pci_endpoint_test_probe(struct pci_dev *pdev,
 	id = ida_simple_get(&pci_endpoint_test_ida, 0, 0, GFP_KERNEL);
 	if (id < 0) {
 		err = id;
-		dev_err(dev, "unable to get id\n");
+		dev_err(dev, "Unable to get id\n");
 		goto err_iounmap;
 	}
 
@@ -573,7 +573,7 @@ static int pci_endpoint_test_probe(struct pci_dev *pdev,
 
 	err = misc_register(misc_device);
 	if (err) {
-		dev_err(dev, "failed to register device\n");
+		dev_err(dev, "Failed to register device\n");
 		goto err_kfree_name;
 	}
 
@@ -592,7 +592,7 @@ static int pci_endpoint_test_probe(struct pci_dev *pdev,
 	}
 
 	for (i = 0; i < irq; i++)
-		devm_free_irq(dev, pdev->irq + i, test);
+		devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), test);
 
 err_disable_msi:
 	pci_disable_msi(pdev);
@@ -625,7 +625,7 @@ static void pci_endpoint_test_remove(struct pci_dev *pdev)
 			pci_iounmap(pdev, test->bar[bar]);
 	}
 	for (i = 0; i < test->num_irqs; i++)
-		devm_free_irq(&pdev->dev, pdev->irq + i, test);
+		devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), test);
 	pci_disable_msi(pdev);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
@@ -634,6 +634,7 @@ static void pci_endpoint_test_remove(struct pci_dev *pdev)
 static const struct pci_device_id pci_endpoint_test_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_SYNOPSYS, 0xedda) },
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, pci_endpoint_test_tbl);
diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c
index 3641f1334cf0..ab174f28e3be 100644
--- a/drivers/misc/sgi-gru/grumain.c
+++ b/drivers/misc/sgi-gru/grumain.c
@@ -926,7 +926,7 @@ struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts)
  *
  * 	Note: gru segments alway mmaped on GRU_GSEG_PAGESIZE boundaries.
  */
-int gru_fault(struct vm_fault *vmf)
+vm_fault_t gru_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct gru_thread_state *gts;
diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c
index 4f7635922394..42ea2eccaee9 100644
--- a/drivers/misc/sgi-gru/gruprocfs.c
+++ b/drivers/misc/sgi-gru/gruprocfs.c
@@ -270,16 +270,6 @@ static int options_open(struct inode *inode, struct file *file)
 	return single_open(file, options_show, NULL);
 }
 
-static int cch_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &cch_seq_ops);
-}
-
-static int gru_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &gru_seq_ops);
-}
-
 /* *INDENT-OFF* */
 static const struct file_operations statistics_fops = {
 	.open 		= statistics_open,
@@ -305,73 +295,30 @@ static const struct file_operations options_fops = {
 	.release 	= single_release,
 };
 
-static const struct file_operations cch_fops = {
-	.open 		= cch_open,
-	.read 		= seq_read,
-	.llseek 	= seq_lseek,
-	.release 	= seq_release,
-};
-static const struct file_operations gru_fops = {
-	.open 		= gru_open,
-	.read 		= seq_read,
-	.llseek 	= seq_lseek,
-	.release 	= seq_release,
-};
-
-static struct proc_entry {
-	char *name;
-	umode_t mode;
-	const struct file_operations *fops;
-	struct proc_dir_entry *entry;
-} proc_files[] = {
-	{"statistics", 0644, &statistics_fops},
-	{"mcs_statistics", 0644, &mcs_statistics_fops},
-	{"debug_options", 0644, &options_fops},
-	{"cch_status", 0444, &cch_fops},
-	{"gru_status", 0444, &gru_fops},
-	{NULL}
-};
-/* *INDENT-ON* */
-
 static struct proc_dir_entry *proc_gru __read_mostly;
 
-static int create_proc_file(struct proc_entry *p)
-{
-	p->entry = proc_create(p->name, p->mode, proc_gru, p->fops);
-	if (!p->entry)
-		return -1;
-	return 0;
-}
-
-static void delete_proc_files(void)
-{
-	struct proc_entry *p;
-
-	if (proc_gru) {
-		for (p = proc_files; p->name; p++)
-			if (p->entry)
-				remove_proc_entry(p->name, proc_gru);
-		proc_remove(proc_gru);
-	}
-}
-
 int gru_proc_init(void)
 {
-	struct proc_entry *p;
-
 	proc_gru = proc_mkdir("sgi_uv/gru", NULL);
-
-	for (p = proc_files; p->name; p++)
-		if (create_proc_file(p))
-			goto err;
+	if (!proc_gru)
+		return -1;
+	if (!proc_create("statistics", 0644, proc_gru, &statistics_fops))
+		goto err;
+	if (!proc_create("mcs_statistics", 0644, proc_gru, &mcs_statistics_fops))
+		goto err;
+	if (!proc_create("debug_options", 0644, proc_gru, &options_fops))
+		goto err;
+	if (!proc_create_seq("cch_status", 0444, proc_gru, &cch_seq_ops))
+		goto err;
+	if (!proc_create_seq("gru_status", 0444, proc_gru, &gru_seq_ops))
+		goto err;
 	return 0;
-
 err:
-	delete_proc_files();
+	remove_proc_subtree("sgi_uv/gru", NULL);
 	return -1;
 }
 
 void gru_proc_exit(void)
 {
-	delete_proc_files();
+	remove_proc_subtree("sgi_uv/gru", NULL);
 }
diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h
index b5e308b50ed1..3e041b6f7a68 100644
--- a/drivers/misc/sgi-gru/grutables.h
+++ b/drivers/misc/sgi-gru/grutables.h
@@ -147,6 +147,7 @@
 #include <linux/mutex.h>
 #include <linux/wait.h>
 #include <linux/mmu_notifier.h>
+#include <linux/mm_types.h>
 #include "gru.h"
 #include "grulib.h"
 #include "gruhandles.h"
@@ -665,7 +666,7 @@ extern unsigned long gru_reserve_cb_resources(struct gru_state *gru,
 		int cbr_au_count, char *cbmap);
 extern unsigned long gru_reserve_ds_resources(struct gru_state *gru,
 		int dsr_au_count, char *dsmap);
-extern int gru_fault(struct vm_fault *vmf);
+extern vm_fault_t gru_fault(struct vm_fault *vmf);
 extern struct gru_mm_struct *gru_register_mmu_notifier(void);
 extern void gru_drop_mmu_notifier(struct gru_mm_struct *gms);
 
diff --git a/drivers/misc/sgi-xp/xpnet.c b/drivers/misc/sgi-xp/xpnet.c
index 0c26eaf5f62b..216d5c756236 100644
--- a/drivers/misc/sgi-xp/xpnet.c
+++ b/drivers/misc/sgi-xp/xpnet.c
@@ -407,7 +407,7 @@ xpnet_send(struct sk_buff *skb, struct xpnet_pending_msg *queued_msg,
  * destination partid.  If the destination partid octets are 0xffff,
  * this packet is to be broadcast to all connected partitions.
  */
-static int
+static netdev_tx_t
 xpnet_dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct xpnet_pending_msg *queued_msg;
diff --git a/drivers/misc/ti-st/st_kim.c b/drivers/misc/ti-st/st_kim.c
index b77aacafc3fc..5ec3f5a43718 100644
--- a/drivers/misc/ti-st/st_kim.c
+++ b/drivers/misc/ti-st/st_kim.c
@@ -735,7 +735,7 @@ static int kim_probe(struct platform_device *pdev)
 		st_kim_devices[0] = pdev;
 	}
 
-	kim_gdata = kzalloc(sizeof(struct kim_data_s), GFP_ATOMIC);
+	kim_gdata = kzalloc(sizeof(struct kim_data_s), GFP_KERNEL);
 	if (!kim_gdata) {
 		pr_err("no mem to allocate");
 		return -ENOMEM;
diff --git a/drivers/misc/tifm_7xx1.c b/drivers/misc/tifm_7xx1.c
index e5f108713dd8..9ac95b48ef92 100644
--- a/drivers/misc/tifm_7xx1.c
+++ b/drivers/misc/tifm_7xx1.c
@@ -239,9 +239,13 @@ static int tifm_7xx1_resume(struct pci_dev *dev)
 	unsigned long timeout;
 	unsigned int good_sockets = 0, bad_sockets = 0;
 	unsigned long flags;
-	unsigned char new_ids[fm->num_sockets];
+	/* Maximum number of entries is 4 */
+	unsigned char new_ids[4];
 	DECLARE_COMPLETION_ONSTACK(finish_resume);
 
+	if (WARN_ON(fm->num_sockets > ARRAY_SIZE(new_ids)))
+		return -ENXIO;
+
 	pci_set_power_state(dev, PCI_D0);
 	pci_restore_state(dev);
 	rc = pci_enable_device(dev);
diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c
index 38a7586b00cc..a0b9102c4c6e 100644
--- a/drivers/mmc/core/block.c
+++ b/drivers/mmc/core/block.c
@@ -244,7 +244,7 @@ static ssize_t power_ro_lock_store(struct device *dev,
 	mq = &md->queue;
 
 	/* Dispatch locking to the block layer */
-	req = blk_get_request(mq->queue, REQ_OP_DRV_OUT, __GFP_RECLAIM);
+	req = blk_get_request(mq->queue, REQ_OP_DRV_OUT, 0);
 	if (IS_ERR(req)) {
 		count = PTR_ERR(req);
 		goto out_put;
@@ -650,8 +650,7 @@ static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md,
 	 */
 	mq = &md->queue;
 	req = blk_get_request(mq->queue,
-		idata->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN,
-		__GFP_RECLAIM);
+		idata->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
 	if (IS_ERR(req)) {
 		err = PTR_ERR(req);
 		goto cmd_done;
@@ -721,8 +720,7 @@ static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md,
 	 */
 	mq = &md->queue;
 	req = blk_get_request(mq->queue,
-		idata[0]->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN,
-		__GFP_RECLAIM);
+		idata[0]->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
 	if (IS_ERR(req)) {
 		err = PTR_ERR(req);
 		goto cmd_err;
@@ -2353,7 +2351,8 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card,
 	set_disk_ro(md->disk, md->read_only || default_ro);
 	md->disk->flags = GENHD_FL_EXT_DEVT;
 	if (area_type & (MMC_BLK_DATA_AREA_RPMB | MMC_BLK_DATA_AREA_BOOT))
-		md->disk->flags |= GENHD_FL_NO_PART_SCAN;
+		md->disk->flags |= GENHD_FL_NO_PART_SCAN
+				   | GENHD_FL_SUPPRESS_PARTITION_INFO;
 
 	/*
 	 * As discussed on lkml, GENHD_FL_REMOVABLE should:
@@ -2750,7 +2749,7 @@ static int mmc_dbg_card_status_get(void *data, u64 *val)
 	int ret;
 
 	/* Ask the block layer about the card status */
-	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, 0);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS;
@@ -2786,7 +2785,7 @@ static int mmc_ext_csd_open(struct inode *inode, struct file *filp)
 		return -ENOMEM;
 
 	/* Ask the block layer for the EXT CSD */
-	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, 0);
 	if (IS_ERR(req)) {
 		err = PTR_ERR(req);
 		goto out_free;
@@ -2967,9 +2966,11 @@ static void mmc_blk_remove(struct mmc_card *card)
 	mmc_blk_remove_debugfs(card, md);
 	mmc_blk_remove_parts(card, md);
 	pm_runtime_get_sync(&card->dev);
-	mmc_claim_host(card->host);
-	mmc_blk_part_switch(card, md->part_type);
-	mmc_release_host(card->host);
+	if (md->part_curr != md->part_type) {
+		mmc_claim_host(card->host);
+		mmc_blk_part_switch(card, md->part_type);
+		mmc_release_host(card->host);
+	}
 	if (card->type != MMC_TYPE_SD_COMBO)
 		pm_runtime_disable(&card->dev);
 	pm_runtime_put_noidle(&card->dev);
diff --git a/drivers/mmc/core/card.h b/drivers/mmc/core/card.h
index 9c821eedd156..1170feb8f969 100644
--- a/drivers/mmc/core/card.h
+++ b/drivers/mmc/core/card.h
@@ -149,6 +149,12 @@ static inline void __maybe_unused remove_quirk(struct mmc_card *card, int data)
 	card->quirks &= ~data;
 }
 
+static inline void __maybe_unused add_limit_rate_quirk(struct mmc_card *card,
+						       int data)
+{
+	card->quirk_max_rate = data;
+}
+
 /*
  * Quirk add/remove for MMC products.
  */
diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c
index 121ce50b6d5e..281826d1fcca 100644
--- a/drivers/mmc/core/core.c
+++ b/drivers/mmc/core/core.c
@@ -50,9 +50,6 @@
 #include "sd_ops.h"
 #include "sdio_ops.h"
 
-/* If the device is not responding */
-#define MMC_CORE_TIMEOUT_MS	(10 * 60 * 1000) /* 10 minute timeout */
-
 /* The max erase timeout, used when host->max_busy_timeout isn't specified */
 #define MMC_ERASE_TIMEOUT_MS	(60 * 1000) /* 60 s */
 
@@ -1484,6 +1481,17 @@ int mmc_set_signal_voltage(struct mmc_host *host, int signal_voltage)
 
 }
 
+void mmc_set_initial_signal_voltage(struct mmc_host *host)
+{
+	/* Try to set signal voltage to 3.3V but fall back to 1.8v or 1.2v */
+	if (!mmc_set_signal_voltage(host, MMC_SIGNAL_VOLTAGE_330))
+		dev_dbg(mmc_dev(host), "Initial signal voltage of 3.3v\n");
+	else if (!mmc_set_signal_voltage(host, MMC_SIGNAL_VOLTAGE_180))
+		dev_dbg(mmc_dev(host), "Initial signal voltage of 1.8v\n");
+	else if (!mmc_set_signal_voltage(host, MMC_SIGNAL_VOLTAGE_120))
+		dev_dbg(mmc_dev(host), "Initial signal voltage of 1.2v\n");
+}
+
 int mmc_host_set_uhs_voltage(struct mmc_host *host)
 {
 	u32 clock;
@@ -1646,19 +1654,13 @@ void mmc_power_up(struct mmc_host *host, u32 ocr)
 	/* Set initial state and call mmc_set_ios */
 	mmc_set_initial_state(host);
 
-	/* Try to set signal voltage to 3.3V but fall back to 1.8v or 1.2v */
-	if (!mmc_set_signal_voltage(host, MMC_SIGNAL_VOLTAGE_330))
-		dev_dbg(mmc_dev(host), "Initial signal voltage of 3.3v\n");
-	else if (!mmc_set_signal_voltage(host, MMC_SIGNAL_VOLTAGE_180))
-		dev_dbg(mmc_dev(host), "Initial signal voltage of 1.8v\n");
-	else if (!mmc_set_signal_voltage(host, MMC_SIGNAL_VOLTAGE_120))
-		dev_dbg(mmc_dev(host), "Initial signal voltage of 1.2v\n");
+	mmc_set_initial_signal_voltage(host);
 
 	/*
 	 * This delay should be sufficient to allow the power supply
 	 * to reach the minimum voltage.
 	 */
-	mmc_delay(10);
+	mmc_delay(host->ios.power_delay_ms);
 
 	mmc_pwrseq_post_power_on(host);
 
@@ -1671,7 +1673,7 @@ void mmc_power_up(struct mmc_host *host, u32 ocr)
 	 * This delay must be at least 74 clock sizes, or 1 ms, or the
 	 * time required to reach a stable voltage.
 	 */
-	mmc_delay(10);
+	mmc_delay(host->ios.power_delay_ms);
 }
 
 void mmc_power_off(struct mmc_host *host)
@@ -1967,6 +1969,7 @@ static int mmc_do_erase(struct mmc_card *card, unsigned int from,
 	unsigned int qty = 0, busy_timeout = 0;
 	bool use_r1b_resp = false;
 	unsigned long timeout;
+	int loop_udelay=64, udelay_max=32768;
 	int err;
 
 	mmc_retune_hold(card->host);
@@ -2091,9 +2094,15 @@ static int mmc_do_erase(struct mmc_card *card, unsigned int from,
 			err =  -EIO;
 			goto out;
 		}
+		if ((cmd.resp[0] & R1_READY_FOR_DATA) &&
+		    R1_CURRENT_STATE(cmd.resp[0]) != R1_STATE_PRG)
+			break;
+
+		usleep_range(loop_udelay, loop_udelay*2);
+		if (loop_udelay < udelay_max)
+			loop_udelay *= 2;
+	} while (1);
 
-	} while (!(cmd.resp[0] & R1_READY_FOR_DATA) ||
-		 (R1_CURRENT_STATE(cmd.resp[0]) == R1_STATE_PRG));
 out:
 	mmc_retune_release(card->host);
 	return err;
@@ -2435,22 +2444,46 @@ int mmc_hw_reset(struct mmc_host *host)
 		return -EINVAL;
 
 	mmc_bus_get(host);
-	if (!host->bus_ops || host->bus_dead || !host->bus_ops->reset) {
+	if (!host->bus_ops || host->bus_dead || !host->bus_ops->hw_reset) {
 		mmc_bus_put(host);
 		return -EOPNOTSUPP;
 	}
 
-	ret = host->bus_ops->reset(host);
+	ret = host->bus_ops->hw_reset(host);
 	mmc_bus_put(host);
 
 	if (ret)
-		pr_warn("%s: tried to reset card, got error %d\n",
+		pr_warn("%s: tried to HW reset card, got error %d\n",
 			mmc_hostname(host), ret);
 
 	return ret;
 }
 EXPORT_SYMBOL(mmc_hw_reset);
 
+int mmc_sw_reset(struct mmc_host *host)
+{
+	int ret;
+
+	if (!host->card)
+		return -EINVAL;
+
+	mmc_bus_get(host);
+	if (!host->bus_ops || host->bus_dead || !host->bus_ops->sw_reset) {
+		mmc_bus_put(host);
+		return -EOPNOTSUPP;
+	}
+
+	ret = host->bus_ops->sw_reset(host);
+	mmc_bus_put(host);
+
+	if (ret)
+		pr_warn("%s: tried to SW reset card, got error %d\n",
+			mmc_hostname(host), ret);
+
+	return ret;
+}
+EXPORT_SYMBOL(mmc_sw_reset);
+
 static int mmc_rescan_try_freq(struct mmc_host *host, unsigned freq)
 {
 	host->f_init = freq;
diff --git a/drivers/mmc/core/core.h b/drivers/mmc/core/core.h
index d6303d69071b..9d8f09ac0821 100644
--- a/drivers/mmc/core/core.h
+++ b/drivers/mmc/core/core.h
@@ -32,7 +32,8 @@ struct mmc_bus_ops {
 	int (*power_restore)(struct mmc_host *);
 	int (*alive)(struct mmc_host *);
 	int (*shutdown)(struct mmc_host *);
-	int (*reset)(struct mmc_host *);
+	int (*hw_reset)(struct mmc_host *);
+	int (*sw_reset)(struct mmc_host *);
 };
 
 void mmc_attach_bus(struct mmc_host *host, const struct mmc_bus_ops *ops);
@@ -51,6 +52,7 @@ u32 mmc_select_voltage(struct mmc_host *host, u32 ocr);
 int mmc_set_uhs_voltage(struct mmc_host *host, u32 ocr);
 int mmc_host_set_uhs_voltage(struct mmc_host *host);
 int mmc_set_signal_voltage(struct mmc_host *host, int signal_voltage);
+void mmc_set_initial_signal_voltage(struct mmc_host *host);
 void mmc_set_timing(struct mmc_host *host, unsigned int timing);
 void mmc_set_driver_type(struct mmc_host *host, unsigned int drv_type);
 int mmc_select_drive_strength(struct mmc_card *card, unsigned int max_dtr,
diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c
index 64b03d6eaf18..abf9e884386c 100644
--- a/drivers/mmc/core/host.c
+++ b/drivers/mmc/core/host.c
@@ -143,9 +143,6 @@ int mmc_retune(struct mmc_host *host)
 			goto out;
 
 		return_to_hs400 = true;
-
-		if (host->ops->prepare_hs400_tuning)
-			host->ops->prepare_hs400_tuning(host, &host->ios);
 	}
 
 	err = mmc_execute_tuning(host->card);
@@ -179,7 +176,7 @@ static void mmc_retune_timer(struct timer_list *t)
 int mmc_of_parse(struct mmc_host *host)
 {
 	struct device *dev = host->parent;
-	u32 bus_width, drv_type;
+	u32 bus_width, drv_type, cd_debounce_delay_ms;
 	int ret;
 	bool cd_cap_invert, cd_gpio_invert = false;
 	bool ro_cap_invert, ro_gpio_invert = false;
@@ -230,11 +227,16 @@ int mmc_of_parse(struct mmc_host *host)
 	} else {
 		cd_cap_invert = device_property_read_bool(dev, "cd-inverted");
 
+		if (device_property_read_u32(dev, "cd-debounce-delay-ms",
+					     &cd_debounce_delay_ms))
+			cd_debounce_delay_ms = 200;
+
 		if (device_property_read_bool(dev, "broken-cd"))
 			host->caps |= MMC_CAP_NEEDS_POLL;
 
 		ret = mmc_gpiod_request_cd(host, "cd", 0, true,
-					   0, &cd_gpio_invert);
+					   cd_debounce_delay_ms,
+					   &cd_gpio_invert);
 		if (!ret)
 			dev_info(host->parent, "Got CD GPIO\n");
 		else if (ret != -ENOENT && ret != -ENOSYS)
@@ -338,6 +340,9 @@ int mmc_of_parse(struct mmc_host *host)
 		host->dsr_req = 0;
 	}
 
+	device_property_read_u32(dev, "post-power-on-delay-ms",
+				 &host->ios.power_delay_ms);
+
 	return mmc_pwrseq_alloc(host);
 }
 
@@ -403,6 +408,7 @@ struct mmc_host *mmc_alloc_host(int extra, struct device *dev)
 	host->max_blk_count = PAGE_SIZE / 512;
 
 	host->fixed_drv_type = -EINVAL;
+	host->ios.power_delay_ms = 10;
 
 	return host;
 }
diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c
index 6f8ebd6caa4c..4466f5de54d4 100644
--- a/drivers/mmc/core/mmc.c
+++ b/drivers/mmc/core/mmc.c
@@ -1282,6 +1282,10 @@ int mmc_hs400_to_hs200(struct mmc_card *card)
 
 	mmc_set_bus_speed(card);
 
+	/* Prepare tuning for HS400 mode. */
+	if (host->ops->prepare_hs400_tuning)
+		host->ops->prepare_hs400_tuning(host, &host->ios);
+
 	return 0;
 
 out_err:
@@ -1830,6 +1834,14 @@ static int mmc_init_card(struct mmc_host *host, u32 ocr,
 		}
 	}
 
+	if (host->caps2 & MMC_CAP2_AVOID_3_3V &&
+	    host->ios.signal_voltage == MMC_SIGNAL_VOLTAGE_330) {
+		pr_err("%s: Host failed to negotiate down from 3.3V\n",
+			mmc_hostname(host));
+		err = -EINVAL;
+		goto free_card;
+	}
+
 	if (!oldcard)
 		host->card = card;
 
@@ -2117,7 +2129,7 @@ static int mmc_can_reset(struct mmc_card *card)
 	return 1;
 }
 
-static int mmc_reset(struct mmc_host *host)
+static int _mmc_hw_reset(struct mmc_host *host)
 {
 	struct mmc_card *card = host->card;
 
@@ -2151,7 +2163,7 @@ static const struct mmc_bus_ops mmc_ops = {
 	.runtime_resume = mmc_runtime_resume,
 	.alive = mmc_alive,
 	.shutdown = mmc_shutdown,
-	.reset = mmc_reset,
+	.hw_reset = _mmc_hw_reset,
 };
 
 /*
diff --git a/drivers/mmc/core/pwrseq_simple.c b/drivers/mmc/core/pwrseq_simple.c
index 13ef162cf066..a8b9fee4d62a 100644
--- a/drivers/mmc/core/pwrseq_simple.c
+++ b/drivers/mmc/core/pwrseq_simple.c
@@ -40,14 +40,18 @@ static void mmc_pwrseq_simple_set_gpios_value(struct mmc_pwrseq_simple *pwrseq,
 	struct gpio_descs *reset_gpios = pwrseq->reset_gpios;
 
 	if (!IS_ERR(reset_gpios)) {
-		int i;
-		int values[reset_gpios->ndescs];
+		int i, *values;
+		int nvalues = reset_gpios->ndescs;
 
-		for (i = 0; i < reset_gpios->ndescs; i++)
+		values = kmalloc_array(nvalues, sizeof(int), GFP_KERNEL);
+		if (!values)
+			return;
+
+		for (i = 0; i < nvalues; i++)
 			values[i] = value;
 
-		gpiod_set_array_value_cansleep(
-			reset_gpios->ndescs, reset_gpios->desc, values);
+		gpiod_set_array_value_cansleep(nvalues, reset_gpios->desc, values);
+		kfree(values);
 	}
 }
 
diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c
index 56e9a803db21..648eb6743ed5 100644
--- a/drivers/mmc/core/queue.c
+++ b/drivers/mmc/core/queue.c
@@ -111,8 +111,9 @@ static enum blk_eh_timer_return mmc_cqe_timed_out(struct request *req)
 				__mmc_cqe_recovery_notifier(mq);
 			return BLK_EH_RESET_TIMER;
 		}
-		/* No timeout */
-		return BLK_EH_HANDLED;
+		/* No timeout (XXX: huh? comment doesn't make much sense) */
+		blk_mq_complete_request(req);
+		return BLK_EH_DONE;
 	default:
 		/* Timeout is handled by mmc core */
 		return BLK_EH_RESET_TIMER;
diff --git a/drivers/mmc/core/quirks.h b/drivers/mmc/core/quirks.h
index 5153577754f0..dd2f73af8f2c 100644
--- a/drivers/mmc/core/quirks.h
+++ b/drivers/mmc/core/quirks.h
@@ -132,6 +132,9 @@ static const struct mmc_fixup sdio_fixup_methods[] = {
 	SDIO_FIXUP(SDIO_VENDOR_ID_MARVELL, SDIO_DEVICE_ID_MARVELL_8797_F0,
 		   add_quirk, MMC_QUIRK_BROKEN_IRQ_POLLING),
 
+	SDIO_FIXUP(SDIO_VENDOR_ID_MARVELL, SDIO_DEVICE_ID_MARVELL_8887WLAN,
+		   add_limit_rate_quirk, 150000000),
+
 	END_FIXUP
 };
 
diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c
index baf3d5da4ccb..d0d9f90e7cdf 100644
--- a/drivers/mmc/core/sd.c
+++ b/drivers/mmc/core/sd.c
@@ -1058,6 +1058,14 @@ static int mmc_sd_init_card(struct mmc_host *host, u32 ocr,
 			mmc_set_bus_width(host, MMC_BUS_WIDTH_4);
 		}
 	}
+
+	if (host->caps2 & MMC_CAP2_AVOID_3_3V &&
+	    host->ios.signal_voltage == MMC_SIGNAL_VOLTAGE_330) {
+		pr_err("%s: Host failed to negotiate down from 3.3V\n",
+			mmc_hostname(host));
+		err = -EINVAL;
+		goto free_card;
+	}
 done:
 	host->card = card;
 	return 0;
@@ -1214,7 +1222,7 @@ static int mmc_sd_runtime_resume(struct mmc_host *host)
 	return 0;
 }
 
-static int mmc_sd_reset(struct mmc_host *host)
+static int mmc_sd_hw_reset(struct mmc_host *host)
 {
 	mmc_power_cycle(host, host->card->ocr);
 	return mmc_sd_init_card(host, host->card->ocr, host->card);
@@ -1229,7 +1237,7 @@ static const struct mmc_bus_ops mmc_sd_ops = {
 	.resume = mmc_sd_resume,
 	.alive = mmc_sd_alive,
 	.shutdown = mmc_sd_suspend,
-	.reset = mmc_sd_reset,
+	.hw_reset = mmc_sd_hw_reset,
 };
 
 /*
diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c
index c599a628a387..a86490dbca70 100644
--- a/drivers/mmc/core/sdio.c
+++ b/drivers/mmc/core/sdio.c
@@ -444,6 +444,7 @@ static int sdio_set_bus_speed_mode(struct mmc_card *card)
 	unsigned int bus_speed, timing;
 	int err;
 	unsigned char speed;
+	unsigned int max_rate;
 
 	/*
 	 * If the host doesn't support any of the UHS-I modes, fallback on
@@ -500,9 +501,12 @@ static int sdio_set_bus_speed_mode(struct mmc_card *card)
 	if (err)
 		return err;
 
+	max_rate = min_not_zero(card->quirk_max_rate,
+				card->sw_caps.uhs_max_dtr);
+
 	if (bus_speed) {
 		mmc_set_timing(card->host, timing);
-		mmc_set_clock(card->host, card->sw_caps.uhs_max_dtr);
+		mmc_set_clock(card->host, max_rate);
 	}
 
 	return 0;
@@ -788,6 +792,14 @@ static int mmc_sdio_init_card(struct mmc_host *host, u32 ocr,
 		if (err)
 			goto remove;
 	}
+
+	if (host->caps2 & MMC_CAP2_AVOID_3_3V &&
+	    host->ios.signal_voltage == MMC_SIGNAL_VOLTAGE_330) {
+		pr_err("%s: Host failed to negotiate down from 3.3V\n",
+			mmc_hostname(host));
+		err = -EINVAL;
+		goto remove;
+	}
 finish:
 	if (!oldcard)
 		host->card = card;
@@ -801,6 +813,22 @@ static int mmc_sdio_init_card(struct mmc_host *host, u32 ocr,
 	return err;
 }
 
+static int mmc_sdio_reinit_card(struct mmc_host *host, bool powered_resume)
+{
+	int ret;
+
+	sdio_reset(host);
+	mmc_go_idle(host);
+	mmc_send_if_cond(host, host->card->ocr);
+
+	ret = mmc_send_io_op_cond(host, 0, NULL);
+	if (ret)
+		return ret;
+
+	return mmc_sdio_init_card(host, host->card->ocr, host->card,
+				  powered_resume);
+}
+
 /*
  * Host is being removed. Free up the current card.
  */
@@ -948,14 +976,7 @@ static int mmc_sdio_resume(struct mmc_host *host)
 
 	/* No need to reinitialize powered-resumed nonremovable cards */
 	if (mmc_card_is_removable(host) || !mmc_card_keep_power(host)) {
-		sdio_reset(host);
-		mmc_go_idle(host);
-		mmc_send_if_cond(host, host->card->ocr);
-		err = mmc_send_io_op_cond(host, 0, NULL);
-		if (!err)
-			err = mmc_sdio_init_card(host, host->card->ocr,
-						 host->card,
-						 mmc_card_keep_power(host));
+		err = mmc_sdio_reinit_card(host, mmc_card_keep_power(host));
 	} else if (mmc_card_keep_power(host) && mmc_card_wake_sdio_irq(host)) {
 		/* We may have switched to 1-bit mode during suspend */
 		err = sdio_enable_4bit_bus(host->card);
@@ -978,8 +999,6 @@ static int mmc_sdio_power_restore(struct mmc_host *host)
 {
 	int ret;
 
-	mmc_claim_host(host);
-
 	/*
 	 * Reset the card by performing the same steps that are taken by
 	 * mmc_rescan_try_freq() and mmc_attach_sdio() during a "normal" probe.
@@ -997,20 +1016,12 @@ static int mmc_sdio_power_restore(struct mmc_host *host)
 	 *
 	 */
 
-	sdio_reset(host);
-	mmc_go_idle(host);
-	mmc_send_if_cond(host, host->card->ocr);
-
-	ret = mmc_send_io_op_cond(host, 0, NULL);
-	if (ret)
-		goto out;
+	mmc_claim_host(host);
 
-	ret = mmc_sdio_init_card(host, host->card->ocr, host->card,
-				mmc_card_keep_power(host));
+	ret = mmc_sdio_reinit_card(host, mmc_card_keep_power(host));
 	if (!ret && host->sdio_irqs)
 		mmc_signal_sdio_irq(host);
 
-out:
 	mmc_release_host(host);
 
 	return ret;
@@ -1039,12 +1050,24 @@ static int mmc_sdio_runtime_resume(struct mmc_host *host)
 	return ret;
 }
 
-static int mmc_sdio_reset(struct mmc_host *host)
+static int mmc_sdio_hw_reset(struct mmc_host *host)
 {
 	mmc_power_cycle(host, host->card->ocr);
 	return mmc_sdio_power_restore(host);
 }
 
+static int mmc_sdio_sw_reset(struct mmc_host *host)
+{
+	mmc_set_clock(host, host->f_init);
+	sdio_reset(host);
+	mmc_go_idle(host);
+
+	mmc_set_initial_state(host);
+	mmc_set_initial_signal_voltage(host);
+
+	return mmc_sdio_reinit_card(host, 0);
+}
+
 static const struct mmc_bus_ops mmc_sdio_ops = {
 	.remove = mmc_sdio_remove,
 	.detect = mmc_sdio_detect,
@@ -1055,7 +1078,8 @@ static const struct mmc_bus_ops mmc_sdio_ops = {
 	.runtime_resume = mmc_sdio_runtime_resume,
 	.power_restore = mmc_sdio_power_restore,
 	.alive = mmc_sdio_alive,
-	.reset = mmc_sdio_reset,
+	.hw_reset = mmc_sdio_hw_reset,
+	.sw_reset = mmc_sdio_sw_reset,
 };
 
 
diff --git a/drivers/mmc/core/sdio_bus.c b/drivers/mmc/core/sdio_bus.c
index 2b32b88949ba..b6d8203e46eb 100644
--- a/drivers/mmc/core/sdio_bus.c
+++ b/drivers/mmc/core/sdio_bus.c
@@ -139,7 +139,7 @@ static int sdio_bus_probe(struct device *dev)
 		return -ENODEV;
 
 	ret = dev_pm_domain_attach(dev, false);
-	if (ret == -EPROBE_DEFER)
+	if (ret)
 		return ret;
 
 	/* Unbound SDIO functions are always suspended.
diff --git a/drivers/mmc/core/sdio_uart.c b/drivers/mmc/core/sdio_uart.c
index d3c91f412b69..25e113001a3c 100644
--- a/drivers/mmc/core/sdio_uart.c
+++ b/drivers/mmc/core/sdio_uart.c
@@ -1008,19 +1008,6 @@ static int sdio_uart_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int sdio_uart_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, sdio_uart_proc_show, NULL);
-}
-
-static const struct file_operations sdio_uart_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= sdio_uart_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static const struct tty_port_operations sdio_uart_port_ops = {
 	.dtr_rts = uart_dtr_rts,
 	.carrier_raised = uart_carrier_raised,
@@ -1045,7 +1032,7 @@ static const struct tty_operations sdio_uart_ops = {
 	.tiocmset		= sdio_uart_tiocmset,
 	.install		= sdio_uart_install,
 	.cleanup		= sdio_uart_cleanup,
-	.proc_fops		= &sdio_uart_proc_fops,
+	.proc_show		= sdio_uart_proc_show,
 };
 
 static struct tty_driver *sdio_uart_tty_driver;
diff --git a/drivers/mmc/core/slot-gpio.c b/drivers/mmc/core/slot-gpio.c
index 31f7dbb15668..ef05e0039378 100644
--- a/drivers/mmc/core/slot-gpio.c
+++ b/drivers/mmc/core/slot-gpio.c
@@ -28,15 +28,17 @@ struct mmc_gpio {
 	irqreturn_t (*cd_gpio_isr)(int irq, void *dev_id);
 	char *ro_label;
 	char cd_label[0];
+	u32 cd_debounce_delay_ms;
 };
 
 static irqreturn_t mmc_gpio_cd_irqt(int irq, void *dev_id)
 {
 	/* Schedule a card detection after a debounce timeout */
 	struct mmc_host *host = dev_id;
+	struct mmc_gpio *ctx = host->slot.handler_priv;
 
 	host->trigger_card_event = true;
-	mmc_detect_change(host, msecs_to_jiffies(200));
+	mmc_detect_change(host, msecs_to_jiffies(ctx->cd_debounce_delay_ms));
 
 	return IRQ_HANDLED;
 }
@@ -49,6 +51,7 @@ int mmc_gpio_alloc(struct mmc_host *host)
 
 	if (ctx) {
 		ctx->ro_label = ctx->cd_label + len;
+		ctx->cd_debounce_delay_ms = 200;
 		snprintf(ctx->cd_label, len, "%s cd", dev_name(host->parent));
 		snprintf(ctx->ro_label, len, "%s ro", dev_name(host->parent));
 		host->slot.handler_priv = ctx;
@@ -76,15 +79,22 @@ EXPORT_SYMBOL(mmc_gpio_get_ro);
 int mmc_gpio_get_cd(struct mmc_host *host)
 {
 	struct mmc_gpio *ctx = host->slot.handler_priv;
+	int cansleep;
 
 	if (!ctx || !ctx->cd_gpio)
 		return -ENOSYS;
 
-	if (ctx->override_cd_active_level)
-		return !gpiod_get_raw_value_cansleep(ctx->cd_gpio) ^
-			!!(host->caps2 & MMC_CAP2_CD_ACTIVE_HIGH);
+	cansleep = gpiod_cansleep(ctx->cd_gpio);
+	if (ctx->override_cd_active_level) {
+		int value = cansleep ?
+				gpiod_get_raw_value_cansleep(ctx->cd_gpio) :
+				gpiod_get_raw_value(ctx->cd_gpio);
+		return !value ^ !!(host->caps2 & MMC_CAP2_CD_ACTIVE_HIGH);
+	}
 
-	return gpiod_get_value_cansleep(ctx->cd_gpio);
+	return cansleep ?
+		gpiod_get_value_cansleep(ctx->cd_gpio) :
+		gpiod_get_value(ctx->cd_gpio);
 }
 EXPORT_SYMBOL(mmc_gpio_get_cd);
 
@@ -261,7 +271,7 @@ int mmc_gpiod_request_cd(struct mmc_host *host, const char *con_id,
 	if (debounce) {
 		ret = gpiod_set_debounce(desc, debounce);
 		if (ret < 0)
-			return ret;
+			ctx->cd_debounce_delay_ms = debounce;
 	}
 
 	if (gpio_invert)
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index 9589f9c9046f..0581c199c996 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -345,11 +345,11 @@ config MMC_SDHCI_IPROC
 	  If unsure, say N.
 
 config MMC_MESON_GX
-	tristate "Amlogic S905/GX* SD/MMC Host Controller support"
+	tristate "Amlogic S905/GX*/AXG SD/MMC Host Controller support"
 	depends on ARCH_MESON && MMC
 	help
 	  This selects support for the Amlogic SD/MMC Host Controller
-	  found on the S905/GX* family of SoCs.  This controller is
+	  found on the S905/GX*/AXG family of SoCs.  This controller is
 	  MMC 5.1 compliant and supports SD, eMMC and SDIO interfaces.
 
 	  If you have a controller with this interface, say Y here.
@@ -358,7 +358,6 @@ config MMC_MESON_MX_SDIO
 	tristate "Amlogic Meson6/Meson8/Meson8b SD/MMC Host Controller support"
 	depends on ARCH_MESON || COMPILE_TEST
 	depends on COMMON_CLK
-	depends on HAS_DMA
 	depends on OF
 	help
 	  This selects support for the SD/MMC Host Controller on
@@ -401,7 +400,6 @@ config MMC_OMAP
 
 config MMC_OMAP_HS
 	tristate "TI OMAP High Speed Multimedia Card Interface support"
-	depends on HAS_DMA
 	depends on ARCH_OMAP2PLUS || ARCH_KEYSTONE || COMPILE_TEST
 	help
 	  This selects the TI OMAP High Speed Multimedia card Interface.
@@ -511,7 +509,6 @@ config MMC_DAVINCI
 
 config MMC_GOLDFISH
 	tristate "goldfish qemu Multimedia Card Interface support"
-	depends on HAS_DMA
 	depends on GOLDFISH || COMPILE_TEST
 	help
 	  This selects the Goldfish Multimedia card Interface emulation
@@ -605,7 +602,7 @@ config MMC_SDHI
 
 config MMC_SDHI_SYS_DMAC
 	tristate "DMA for SDHI SD/SDIO controllers using SYS-DMAC"
-	depends on MMC_SDHI && HAS_DMA
+	depends on MMC_SDHI
 	default MMC_SDHI if (SUPERH || ARM)
 	help
 	  This provides DMA support for SDHI SD/SDIO controllers
@@ -615,7 +612,7 @@ config MMC_SDHI_SYS_DMAC
 config MMC_SDHI_INTERNAL_DMAC
 	tristate "DMA for SDHI SD/SDIO controllers using on-chip bus mastering"
 	depends on ARM64 || COMPILE_TEST
-	depends on MMC_SDHI && HAS_DMA
+	depends on MMC_SDHI
 	default MMC_SDHI if ARM64
 	help
 	  This provides DMA support for SDHI SD/SDIO controllers
@@ -669,7 +666,6 @@ config MMC_CAVIUM_THUNDERX
 
 config MMC_DW
 	tristate "Synopsys DesignWare Memory Card Interface"
-	depends on HAS_DMA
 	depends on ARC || ARM || ARM64 || MIPS || COMPILE_TEST
 	help
 	  This selects support for the Synopsys DesignWare Mobile Storage IP
@@ -690,6 +686,15 @@ config MMC_DW_PLTFM
 
 	  If unsure, say Y.
 
+config MMC_DW_BLUEFIELD
+	tristate "BlueField specific extensions for Synopsys DW Memory Card Interface"
+	depends on MMC_DW
+	select MMC_DW_PLTFM
+	help
+	  This selects support for Mellanox BlueField SoC specific extensions to
+	  the Synopsys DesignWare Memory Card Interface driver. Select this
+	  option for platforms based on Mellanox BlueField SoC's.
+
 config MMC_DW_EXYNOS
 	tristate "Exynos specific extensions for Synopsys DW Memory Card Interface"
 	depends on MMC_DW
@@ -748,7 +753,6 @@ config MMC_DW_ZX
 
 config MMC_SH_MMCIF
 	tristate "SuperH Internal MMCIF support"
-	depends on HAS_DMA
 	depends on SUPERH || ARCH_RENESAS || COMPILE_TEST
 	help
 	  This selects the MMC Host Interface controller (MMCIF) found in various
@@ -756,11 +760,12 @@ config MMC_SH_MMCIF
 
 
 config MMC_JZ4740
-	tristate "JZ4740 SD/Multimedia Card Interface support"
-	depends on MACH_JZ4740
+	tristate "Ingenic JZ47xx SD/Multimedia Card Interface support"
+	depends on MACH_JZ4740 || MACH_JZ4780
 	help
-	  This selects support for the SD/MMC controller on Ingenic JZ4740
-	  SoCs.
+	  This selects support for the SD/MMC controller on Ingenic
+	  JZ4740, JZ4750, JZ4770 and JZ4780 SoCs.
+
 	  If you have a board based on such a SoC and with a SD/MMC slot,
 	  say Y or M here.
 
@@ -868,7 +873,6 @@ config MMC_TOSHIBA_PCI
 config MMC_BCM2835
 	tristate "Broadcom BCM2835 SDHOST MMC Controller support"
 	depends on ARCH_BCM2835 || COMPILE_TEST
-	depends on HAS_DMA
 	help
 	  This selects the BCM2835 SDHOST MMC controller. If you have
 	  a BCM2835 platform with SD or MMC devices, say Y or M here.
diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile
index 6aead24879b4..85dc1322c3de 100644
--- a/drivers/mmc/host/Makefile
+++ b/drivers/mmc/host/Makefile
@@ -49,6 +49,7 @@ thunderx-mmc-objs := cavium.o cavium-thunderx.o
 obj-$(CONFIG_MMC_CAVIUM_THUNDERX) += thunderx-mmc.o
 obj-$(CONFIG_MMC_DW)		+= dw_mmc.o
 obj-$(CONFIG_MMC_DW_PLTFM)	+= dw_mmc-pltfm.o
+obj-$(CONFIG_MMC_DW_BLUEFIELD)	+= dw_mmc-bluefield.o
 obj-$(CONFIG_MMC_DW_EXYNOS)	+= dw_mmc-exynos.o
 obj-$(CONFIG_MMC_DW_HI3798CV200) += dw_mmc-hi3798cv200.o
 obj-$(CONFIG_MMC_DW_K3)		+= dw_mmc-k3.o
diff --git a/drivers/mmc/host/android-goldfish.c b/drivers/mmc/host/android-goldfish.c
index 63d27589cd89..294de177632c 100644
--- a/drivers/mmc/host/android-goldfish.c
+++ b/drivers/mmc/host/android-goldfish.c
@@ -217,8 +217,8 @@ static void goldfish_mmc_xfer_done(struct goldfish_mmc_host *host,
 			 * We don't really have DMA, so we need
 			 * to copy from our platform driver buffer
 			 */
-			uint8_t *dest = (uint8_t *)sg_virt(data->sg);
-			memcpy(dest, host->virt_base, data->sg->length);
+			sg_copy_to_buffer(data->sg, 1, host->virt_base,
+					data->sg->length);
 		}
 		host->data->bytes_xfered += data->sg->length;
 		dma_unmap_sg(mmc_dev(host->mmc), data->sg, host->sg_len,
@@ -393,8 +393,8 @@ static void goldfish_mmc_prepare_data(struct goldfish_mmc_host *host,
 		 * We don't really have DMA, so we need to copy to our
 		 * platform driver buffer
 		 */
-		const uint8_t *src = (uint8_t *)sg_virt(data->sg);
-		memcpy(host->virt_base, src, data->sg->length);
+		sg_copy_from_buffer(data->sg, 1, host->virt_base,
+				data->sg->length);
 	}
 }
 
diff --git a/drivers/mmc/host/atmel-mci.c b/drivers/mmc/host/atmel-mci.c
index e55f3932d580..5aa2c9404e92 100644
--- a/drivers/mmc/host/atmel-mci.c
+++ b/drivers/mmc/host/atmel-mci.c
@@ -1967,7 +1967,6 @@ static void atmci_tasklet_func(unsigned long priv)
 static void atmci_read_data_pio(struct atmel_mci *host)
 {
 	struct scatterlist	*sg = host->sg;
-	void			*buf = sg_virt(sg);
 	unsigned int		offset = host->pio_offset;
 	struct mmc_data		*data = host->data;
 	u32			value;
@@ -1977,7 +1976,7 @@ static void atmci_read_data_pio(struct atmel_mci *host)
 	do {
 		value = atmci_readl(host, ATMCI_RDR);
 		if (likely(offset + 4 <= sg->length)) {
-			put_unaligned(value, (u32 *)(buf + offset));
+			sg_pcopy_to_buffer(sg, 1, &value, sizeof(u32), offset);
 
 			offset += 4;
 			nbytes += 4;
@@ -1990,11 +1989,11 @@ static void atmci_read_data_pio(struct atmel_mci *host)
 					goto done;
 
 				offset = 0;
-				buf = sg_virt(sg);
 			}
 		} else {
 			unsigned int remaining = sg->length - offset;
-			memcpy(buf + offset, &value, remaining);
+
+			sg_pcopy_to_buffer(sg, 1, &value, remaining, offset);
 			nbytes += remaining;
 
 			flush_dcache_page(sg_page(sg));
@@ -2004,8 +2003,8 @@ static void atmci_read_data_pio(struct atmel_mci *host)
 				goto done;
 
 			offset = 4 - remaining;
-			buf = sg_virt(sg);
-			memcpy(buf, (u8 *)&value + remaining, offset);
+			sg_pcopy_to_buffer(sg, 1, (u8 *)&value + remaining,
+					offset, 0);
 			nbytes += offset;
 		}
 
@@ -2035,7 +2034,6 @@ static void atmci_read_data_pio(struct atmel_mci *host)
 static void atmci_write_data_pio(struct atmel_mci *host)
 {
 	struct scatterlist	*sg = host->sg;
-	void			*buf = sg_virt(sg);
 	unsigned int		offset = host->pio_offset;
 	struct mmc_data		*data = host->data;
 	u32			value;
@@ -2044,7 +2042,7 @@ static void atmci_write_data_pio(struct atmel_mci *host)
 
 	do {
 		if (likely(offset + 4 <= sg->length)) {
-			value = get_unaligned((u32 *)(buf + offset));
+			sg_pcopy_from_buffer(sg, 1, &value, sizeof(u32), offset);
 			atmci_writel(host, ATMCI_TDR, value);
 
 			offset += 4;
@@ -2056,13 +2054,12 @@ static void atmci_write_data_pio(struct atmel_mci *host)
 					goto done;
 
 				offset = 0;
-				buf = sg_virt(sg);
 			}
 		} else {
 			unsigned int remaining = sg->length - offset;
 
 			value = 0;
-			memcpy(&value, buf + offset, remaining);
+			sg_pcopy_from_buffer(sg, 1, &value, remaining, offset);
 			nbytes += remaining;
 
 			host->sg = sg = sg_next(sg);
@@ -2073,8 +2070,8 @@ static void atmci_write_data_pio(struct atmel_mci *host)
 			}
 
 			offset = 4 - remaining;
-			buf = sg_virt(sg);
-			memcpy((u8 *)&value + remaining, buf, offset);
+			sg_pcopy_from_buffer(sg, 1, (u8 *)&value + remaining,
+					offset, 0);
 			atmci_writel(host, ATMCI_TDR, value);
 			nbytes += offset;
 		}
diff --git a/drivers/mmc/host/au1xmmc.c b/drivers/mmc/host/au1xmmc.c
index ed77fbfa4774..9b4be67330dd 100644
--- a/drivers/mmc/host/au1xmmc.c
+++ b/drivers/mmc/host/au1xmmc.c
@@ -40,6 +40,7 @@
 #include <linux/interrupt.h>
 #include <linux/dma-mapping.h>
 #include <linux/scatterlist.h>
+#include <linux/highmem.h>
 #include <linux/leds.h>
 #include <linux/mmc/host.h>
 #include <linux/slab.h>
@@ -405,7 +406,7 @@ static void au1xmmc_send_pio(struct au1xmmc_host *host)
 
 	/* This is the pointer to the data buffer */
 	sg = &data->sg[host->pio.index];
-	sg_ptr = sg_virt(sg) + host->pio.offset;
+	sg_ptr = kmap_atomic(sg_page(sg)) + sg->offset + host->pio.offset;
 
 	/* This is the space left inside the buffer */
 	sg_len = data->sg[host->pio.index].length - host->pio.offset;
@@ -421,11 +422,12 @@ static void au1xmmc_send_pio(struct au1xmmc_host *host)
 		if (!(status & SD_STATUS_TH))
 			break;
 
-		val = *sg_ptr++;
+		val = sg_ptr[count];
 
 		__raw_writel((unsigned long)val, HOST_TXPORT(host));
 		wmb(); /* drain writebuffer */
 	}
+	kunmap_atomic(sg_ptr);
 
 	host->pio.len -= count;
 	host->pio.offset += count;
@@ -462,7 +464,7 @@ static void au1xmmc_receive_pio(struct au1xmmc_host *host)
 
 	if (host->pio.index < host->dma.len) {
 		sg = &data->sg[host->pio.index];
-		sg_ptr = sg_virt(sg) + host->pio.offset;
+		sg_ptr = kmap_atomic(sg_page(sg)) + sg->offset + host->pio.offset;
 
 		/* This is the space left inside the buffer */
 		sg_len = sg_dma_len(&data->sg[host->pio.index]) - host->pio.offset;
@@ -501,8 +503,10 @@ static void au1xmmc_receive_pio(struct au1xmmc_host *host)
 		val = __raw_readl(HOST_RXPORT(host));
 
 		if (sg_ptr)
-			*sg_ptr++ = (unsigned char)(val & 0xFF);
+			sg_ptr[count] = (unsigned char)(val & 0xFF);
 	}
+	if (sg_ptr)
+		kunmap_atomic(sg_ptr);
 
 	host->pio.len -= count;
 	host->pio.offset += count;
diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c
index 8e363174f9d6..9e68c3645e22 100644
--- a/drivers/mmc/host/davinci_mmc.c
+++ b/drivers/mmc/host/davinci_mmc.c
@@ -1377,8 +1377,7 @@ static int __exit davinci_mmcsd_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM
 static int davinci_mmcsd_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct mmc_davinci_host *host = platform_get_drvdata(pdev);
+	struct mmc_davinci_host *host = dev_get_drvdata(dev);
 
 	writel(0, host->base + DAVINCI_MMCIM);
 	mmc_davinci_reset_ctrl(host, 1);
@@ -1389,8 +1388,7 @@ static int davinci_mmcsd_suspend(struct device *dev)
 
 static int davinci_mmcsd_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct mmc_davinci_host *host = platform_get_drvdata(pdev);
+	struct mmc_davinci_host *host = dev_get_drvdata(dev);
 
 	clk_enable(host->clk);
 	mmc_davinci_reset_ctrl(host, 0);
diff --git a/drivers/mmc/host/dw_mmc-bluefield.c b/drivers/mmc/host/dw_mmc-bluefield.c
new file mode 100644
index 000000000000..54c3fbb4a391
--- /dev/null
+++ b/drivers/mmc/host/dw_mmc-bluefield.c
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Mellanox Technologies.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/mmc/host.h>
+#include <linux/mmc/mmc.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include "dw_mmc.h"
+#include "dw_mmc-pltfm.h"
+
+#define UHS_REG_EXT_SAMPLE_MASK		GENMASK(22, 16)
+#define UHS_REG_EXT_DRIVE_MASK		GENMASK(29, 23)
+#define BLUEFIELD_UHS_REG_EXT_SAMPLE	2
+#define BLUEFIELD_UHS_REG_EXT_DRIVE	4
+
+static void dw_mci_bluefield_set_ios(struct dw_mci *host, struct mmc_ios *ios)
+{
+	u32 reg;
+
+	/* Update the Drive and Sample fields in register UHS_REG_EXT. */
+	reg = mci_readl(host, UHS_REG_EXT);
+	reg &= ~UHS_REG_EXT_SAMPLE_MASK;
+	reg |= FIELD_PREP(UHS_REG_EXT_SAMPLE_MASK,
+			  BLUEFIELD_UHS_REG_EXT_SAMPLE);
+	reg &= ~UHS_REG_EXT_DRIVE_MASK;
+	reg |= FIELD_PREP(UHS_REG_EXT_DRIVE_MASK, BLUEFIELD_UHS_REG_EXT_DRIVE);
+	mci_writel(host, UHS_REG_EXT, reg);
+}
+
+static const struct dw_mci_drv_data bluefield_drv_data = {
+	.set_ios		= dw_mci_bluefield_set_ios
+};
+
+static const struct of_device_id dw_mci_bluefield_match[] = {
+	{ .compatible = "mellanox,bluefield-dw-mshc",
+	  .data = &bluefield_drv_data },
+	{},
+};
+MODULE_DEVICE_TABLE(of, dw_mci_bluefield_match);
+
+static int dw_mci_bluefield_probe(struct platform_device *pdev)
+{
+	const struct dw_mci_drv_data *drv_data = NULL;
+	const struct of_device_id *match;
+
+	if (pdev->dev.of_node) {
+		match = of_match_node(dw_mci_bluefield_match,
+				      pdev->dev.of_node);
+		drv_data = match->data;
+	}
+
+	return dw_mci_pltfm_register(pdev, drv_data);
+}
+
+static struct platform_driver dw_mci_bluefield_pltfm_driver = {
+	.probe		= dw_mci_bluefield_probe,
+	.remove		= dw_mci_pltfm_remove,
+	.driver		= {
+		.name		= "dwmmc_bluefield",
+		.of_match_table	= dw_mci_bluefield_match,
+		.pm		= &dw_mci_pltfm_pmops,
+	},
+};
+
+module_platform_driver(dw_mci_bluefield_pltfm_driver);
+
+MODULE_DESCRIPTION("BlueField DW Multimedia Card driver");
+MODULE_AUTHOR("Mellanox Technologies");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mmc/host/dw_mmc-rockchip.c b/drivers/mmc/host/dw_mmc-rockchip.c
index 40d7de2eea12..8c86a800a8fd 100644
--- a/drivers/mmc/host/dw_mmc-rockchip.c
+++ b/drivers/mmc/host/dw_mmc-rockchip.c
@@ -44,9 +44,8 @@ static void dw_mci_rk3288_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 	 * bus_hz = cclkin / RK3288_CLKGEN_DIV
 	 * ios->clock = (div == 0) ? bus_hz : (bus_hz / (2 * div))
 	 *
-	 * Note: div can only be 0 or 1
-	 *       if DDR50 8bit mode(only emmc work in 8bit mode),
-	 *       div must be set 1
+	 * Note: div can only be 0 or 1, but div must be set to 1 for eMMC
+	 * DDR52 8-bit mode.
 	 */
 	if (ios->bus_width == MMC_BUS_WIDTH_8 &&
 	    ios->timing == MMC_TIMING_MMC_DDR52)
diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
index 29a1afa81f66..623f4d27fa01 100644
--- a/drivers/mmc/host/dw_mmc.c
+++ b/drivers/mmc/host/dw_mmc.c
@@ -1230,6 +1230,8 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 	if (host->state == STATE_WAITING_CMD11_DONE)
 		sdmmc_cmd_bits |= SDMMC_CMD_VOLT_SWITCH;
 
+	slot->mmc->actual_clock = 0;
+
 	if (!clock) {
 		mci_writel(host, CLKENA, 0);
 		mci_send_cmd(slot, sdmmc_cmd_bits, 0);
@@ -1288,6 +1290,8 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 
 		/* keep the last clock value that was requested from core */
 		slot->__clk_old = clock;
+		slot->mmc->actual_clock = div ? ((host->bus_hz / div) >> 1) :
+					  host->bus_hz;
 	}
 
 	host->current_speed = clock;
diff --git a/drivers/mmc/host/jz4740_mmc.c b/drivers/mmc/host/jz4740_mmc.c
index a0168e9e4fce..993386c9ea50 100644
--- a/drivers/mmc/host/jz4740_mmc.c
+++ b/drivers/mmc/host/jz4740_mmc.c
@@ -1,5 +1,7 @@
 /*
  *  Copyright (C) 2009-2010, Lars-Peter Clausen <lars@metafoo.de>
+ *  Copyright (C) 2013, Imagination Technologies
+ *
  *  JZ4740 SD/MMC controller driver
  *
  *  This program is free software; you can redistribute  it and/or modify it
@@ -13,24 +15,25 @@
  *
  */
 
-#include <linux/mmc/host.h>
-#include <linux/mmc/slot-gpio.h>
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
 #include <linux/err.h>
+#include <linux/gpio.h>
+#include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/irq.h>
-#include <linux/interrupt.h>
+#include <linux/mmc/host.h>
+#include <linux/mmc/slot-gpio.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
-#include <linux/delay.h>
 #include <linux/scatterlist.h>
-#include <linux/clk.h>
 
-#include <linux/bitops.h>
-#include <linux/gpio.h>
 #include <asm/cacheflush.h>
-#include <linux/dma-mapping.h>
-#include <linux/dmaengine.h>
 
 #include <asm/mach-jz4740/dma.h>
 #include <asm/mach-jz4740/jz4740_mmc.h>
@@ -51,6 +54,7 @@
 #define JZ_REG_MMC_RESP_FIFO	0x34
 #define JZ_REG_MMC_RXFIFO	0x38
 #define JZ_REG_MMC_TXFIFO	0x3C
+#define JZ_REG_MMC_DMAC		0x44
 
 #define JZ_MMC_STRPCL_EXIT_MULTIPLE BIT(7)
 #define JZ_MMC_STRPCL_EXIT_TRANSFER BIT(6)
@@ -104,9 +108,17 @@
 #define JZ_MMC_IRQ_PRG_DONE BIT(1)
 #define JZ_MMC_IRQ_DATA_TRAN_DONE BIT(0)
 
+#define JZ_MMC_DMAC_DMA_SEL BIT(1)
+#define JZ_MMC_DMAC_DMA_EN BIT(0)
 
 #define JZ_MMC_CLK_RATE 24000000
 
+enum jz4740_mmc_version {
+	JZ_MMC_JZ4740,
+	JZ_MMC_JZ4750,
+	JZ_MMC_JZ4780,
+};
+
 enum jz4740_mmc_state {
 	JZ4740_MMC_STATE_READ_RESPONSE,
 	JZ4740_MMC_STATE_TRANSFER_DATA,
@@ -125,6 +137,8 @@ struct jz4740_mmc_host {
 	struct jz4740_mmc_platform_data *pdata;
 	struct clk *clk;
 
+	enum jz4740_mmc_version version;
+
 	int irq;
 	int card_detect_irq;
 
@@ -137,7 +151,7 @@ struct jz4740_mmc_host {
 
 	uint32_t cmdat;
 
-	uint16_t irq_mask;
+	uint32_t irq_mask;
 
 	spinlock_t lock;
 
@@ -159,6 +173,32 @@ struct jz4740_mmc_host {
 #define JZ4740_MMC_FIFO_HALF_SIZE 8
 };
 
+static void jz4740_mmc_write_irq_mask(struct jz4740_mmc_host *host,
+				      uint32_t val)
+{
+	if (host->version >= JZ_MMC_JZ4750)
+		return writel(val, host->base + JZ_REG_MMC_IMASK);
+	else
+		return writew(val, host->base + JZ_REG_MMC_IMASK);
+}
+
+static void jz4740_mmc_write_irq_reg(struct jz4740_mmc_host *host,
+				     uint32_t val)
+{
+	if (host->version >= JZ_MMC_JZ4780)
+		return writel(val, host->base + JZ_REG_MMC_IREG);
+	else
+		return writew(val, host->base + JZ_REG_MMC_IREG);
+}
+
+static uint32_t jz4740_mmc_read_irq_reg(struct jz4740_mmc_host *host)
+{
+	if (host->version >= JZ_MMC_JZ4780)
+		return readl(host->base + JZ_REG_MMC_IREG);
+	else
+		return readw(host->base + JZ_REG_MMC_IREG);
+}
+
 /*----------------------------------------------------------------------------*/
 /* DMA infrastructure */
 
@@ -173,31 +213,23 @@ static void jz4740_mmc_release_dma_channels(struct jz4740_mmc_host *host)
 
 static int jz4740_mmc_acquire_dma_channels(struct jz4740_mmc_host *host)
 {
-	dma_cap_mask_t mask;
-
-	dma_cap_zero(mask);
-	dma_cap_set(DMA_SLAVE, mask);
-
-	host->dma_tx = dma_request_channel(mask, NULL, host);
-	if (!host->dma_tx) {
+	host->dma_tx = dma_request_chan(mmc_dev(host->mmc), "tx");
+	if (IS_ERR(host->dma_tx)) {
 		dev_err(mmc_dev(host->mmc), "Failed to get dma_tx channel\n");
-		return -ENODEV;
+		return PTR_ERR(host->dma_tx);
 	}
 
-	host->dma_rx = dma_request_channel(mask, NULL, host);
-	if (!host->dma_rx) {
+	host->dma_rx = dma_request_chan(mmc_dev(host->mmc), "rx");
+	if (IS_ERR(host->dma_rx)) {
 		dev_err(mmc_dev(host->mmc), "Failed to get dma_rx channel\n");
-		goto free_master_write;
+		dma_release_channel(host->dma_tx);
+		return PTR_ERR(host->dma_rx);
 	}
 
 	/* Initialize DMA pre request cookie */
 	host->next_data.cookie = 1;
 
 	return 0;
-
-free_master_write:
-	dma_release_channel(host->dma_tx);
-	return -ENODEV;
 }
 
 static inline struct dma_chan *jz4740_mmc_get_dma_chan(struct jz4740_mmc_host *host,
@@ -363,7 +395,7 @@ static void jz4740_mmc_set_irq_enabled(struct jz4740_mmc_host *host,
 	else
 		host->irq_mask |= irq;
 
-	writew(host->irq_mask, host->base + JZ_REG_MMC_IMASK);
+	jz4740_mmc_write_irq_mask(host, host->irq_mask);
 	spin_unlock_irqrestore(&host->lock, flags);
 }
 
@@ -415,10 +447,10 @@ static unsigned int jz4740_mmc_poll_irq(struct jz4740_mmc_host *host,
 	unsigned int irq)
 {
 	unsigned int timeout = 0x800;
-	uint16_t status;
+	uint32_t status;
 
 	do {
-		status = readw(host->base + JZ_REG_MMC_IREG);
+		status = jz4740_mmc_read_irq_reg(host);
 	} while (!(status & irq) && --timeout);
 
 	if (timeout == 0) {
@@ -518,7 +550,7 @@ static bool jz4740_mmc_read_data(struct jz4740_mmc_host *host,
 	void __iomem *fifo_addr = host->base + JZ_REG_MMC_RXFIFO;
 	uint32_t *buf;
 	uint32_t d;
-	uint16_t status;
+	uint32_t status;
 	size_t i, j;
 	unsigned int timeout;
 
@@ -654,8 +686,25 @@ static void jz4740_mmc_send_command(struct jz4740_mmc_host *host,
 		cmdat |= JZ_MMC_CMDAT_DATA_EN;
 		if (cmd->data->flags & MMC_DATA_WRITE)
 			cmdat |= JZ_MMC_CMDAT_WRITE;
-		if (host->use_dma)
-			cmdat |= JZ_MMC_CMDAT_DMA_EN;
+		if (host->use_dma) {
+			/*
+			 * The 4780's MMC controller has integrated DMA ability
+			 * in addition to being able to use the external DMA
+			 * controller. It moves DMA control bits to a separate
+			 * register. The DMA_SEL bit chooses the external
+			 * controller over the integrated one. Earlier SoCs
+			 * can only use the external controller, and have a
+			 * single DMA enable bit in CMDAT.
+			 */
+			if (host->version >= JZ_MMC_JZ4780) {
+				writel(JZ_MMC_DMAC_DMA_EN | JZ_MMC_DMAC_DMA_SEL,
+				       host->base + JZ_REG_MMC_DMAC);
+			} else {
+				cmdat |= JZ_MMC_CMDAT_DMA_EN;
+			}
+		} else if (host->version >= JZ_MMC_JZ4780) {
+			writel(0, host->base + JZ_REG_MMC_DMAC);
+		}
 
 		writew(cmd->data->blksz, host->base + JZ_REG_MMC_BLKLEN);
 		writew(cmd->data->blocks, host->base + JZ_REG_MMC_NOB);
@@ -736,7 +785,7 @@ static irqreturn_t jz_mmc_irq_worker(int irq, void *devid)
 			host->state = JZ4740_MMC_STATE_SEND_STOP;
 			break;
 		}
-		writew(JZ_MMC_IRQ_DATA_TRAN_DONE, host->base + JZ_REG_MMC_IREG);
+		jz4740_mmc_write_irq_reg(host, JZ_MMC_IRQ_DATA_TRAN_DONE);
 
 	case JZ4740_MMC_STATE_SEND_STOP:
 		if (!req->stop)
@@ -766,9 +815,10 @@ static irqreturn_t jz_mmc_irq(int irq, void *devid)
 {
 	struct jz4740_mmc_host *host = devid;
 	struct mmc_command *cmd = host->cmd;
-	uint16_t irq_reg, status, tmp;
+	uint32_t irq_reg, status, tmp;
 
-	irq_reg = readw(host->base + JZ_REG_MMC_IREG);
+	status = readl(host->base + JZ_REG_MMC_STATUS);
+	irq_reg = jz4740_mmc_read_irq_reg(host);
 
 	tmp = irq_reg;
 	irq_reg &= ~host->irq_mask;
@@ -777,10 +827,10 @@ static irqreturn_t jz_mmc_irq(int irq, void *devid)
 		JZ_MMC_IRQ_PRG_DONE | JZ_MMC_IRQ_DATA_TRAN_DONE);
 
 	if (tmp != irq_reg)
-		writew(tmp & ~irq_reg, host->base + JZ_REG_MMC_IREG);
+		jz4740_mmc_write_irq_reg(host, tmp & ~irq_reg);
 
 	if (irq_reg & JZ_MMC_IRQ_SDIO) {
-		writew(JZ_MMC_IRQ_SDIO, host->base + JZ_REG_MMC_IREG);
+		jz4740_mmc_write_irq_reg(host, JZ_MMC_IRQ_SDIO);
 		mmc_signal_sdio_irq(host->mmc);
 		irq_reg &= ~JZ_MMC_IRQ_SDIO;
 	}
@@ -789,8 +839,6 @@ static irqreturn_t jz_mmc_irq(int irq, void *devid)
 		if (test_and_clear_bit(0, &host->waiting)) {
 			del_timer(&host->timeout_timer);
 
-			status = readl(host->base + JZ_REG_MMC_STATUS);
-
 			if (status & JZ_MMC_STATUS_TIMEOUT_RES) {
 					cmd->error = -ETIMEDOUT;
 			} else if (status & JZ_MMC_STATUS_CRC_RES_ERR) {
@@ -803,7 +851,7 @@ static irqreturn_t jz_mmc_irq(int irq, void *devid)
 			}
 
 			jz4740_mmc_set_irq_enabled(host, irq_reg, false);
-			writew(irq_reg, host->base + JZ_REG_MMC_IREG);
+			jz4740_mmc_write_irq_reg(host, irq_reg);
 
 			return IRQ_WAKE_THREAD;
 		}
@@ -818,7 +866,7 @@ static int jz4740_mmc_set_clock_rate(struct jz4740_mmc_host *host, int rate)
 	int real_rate;
 
 	jz4740_mmc_clock_disable(host);
-	clk_set_rate(host->clk, JZ_MMC_CLK_RATE);
+	clk_set_rate(host->clk, host->mmc->f_max);
 
 	real_rate = clk_get_rate(host->clk);
 
@@ -837,9 +885,7 @@ static void jz4740_mmc_request(struct mmc_host *mmc, struct mmc_request *req)
 
 	host->req = req;
 
-	writew(0xffff, host->base + JZ_REG_MMC_IREG);
-
-	writew(JZ_MMC_IRQ_END_CMD_RES, host->base + JZ_REG_MMC_IREG);
+	jz4740_mmc_write_irq_reg(host, ~0);
 	jz4740_mmc_set_irq_enabled(host, JZ_MMC_IRQ_END_CMD_RES, true);
 
 	host->state = JZ4740_MMC_STATE_READ_RESPONSE;
@@ -857,7 +903,7 @@ static void jz4740_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 	switch (ios->power_mode) {
 	case MMC_POWER_UP:
 		jz4740_mmc_reset(host);
-		if (gpio_is_valid(host->pdata->gpio_power))
+		if (host->pdata && gpio_is_valid(host->pdata->gpio_power))
 			gpio_set_value(host->pdata->gpio_power,
 					!host->pdata->power_active_low);
 		host->cmdat |= JZ_MMC_CMDAT_INIT;
@@ -866,7 +912,7 @@ static void jz4740_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 	case MMC_POWER_ON:
 		break;
 	default:
-		if (gpio_is_valid(host->pdata->gpio_power))
+		if (host->pdata && gpio_is_valid(host->pdata->gpio_power))
 			gpio_set_value(host->pdata->gpio_power,
 					host->pdata->power_active_low);
 		clk_disable_unprepare(host->clk);
@@ -926,7 +972,7 @@ static int jz4740_mmc_request_gpio(struct device *dev, int gpio,
 static int jz4740_mmc_request_gpios(struct mmc_host *mmc,
 	struct platform_device *pdev)
 {
-	struct jz4740_mmc_platform_data *pdata = pdev->dev.platform_data;
+	struct jz4740_mmc_platform_data *pdata = dev_get_platdata(&pdev->dev);
 	int ret = 0;
 
 	if (!pdata)
@@ -955,7 +1001,7 @@ static int jz4740_mmc_request_gpios(struct mmc_host *mmc,
 
 static void jz4740_mmc_free_gpios(struct platform_device *pdev)
 {
-	struct jz4740_mmc_platform_data *pdata = pdev->dev.platform_data;
+	struct jz4740_mmc_platform_data *pdata = dev_get_platdata(&pdev->dev);
 
 	if (!pdata)
 		return;
@@ -964,14 +1010,22 @@ static void jz4740_mmc_free_gpios(struct platform_device *pdev)
 		gpio_free(pdata->gpio_power);
 }
 
+static const struct of_device_id jz4740_mmc_of_match[] = {
+	{ .compatible = "ingenic,jz4740-mmc", .data = (void *) JZ_MMC_JZ4740 },
+	{ .compatible = "ingenic,jz4780-mmc", .data = (void *) JZ_MMC_JZ4780 },
+	{},
+};
+MODULE_DEVICE_TABLE(of, jz4740_mmc_of_match);
+
 static int jz4740_mmc_probe(struct platform_device* pdev)
 {
 	int ret;
 	struct mmc_host *mmc;
 	struct jz4740_mmc_host *host;
+	const struct of_device_id *match;
 	struct jz4740_mmc_platform_data *pdata;
 
-	pdata = pdev->dev.platform_data;
+	pdata = dev_get_platdata(&pdev->dev);
 
 	mmc = mmc_alloc_host(sizeof(struct jz4740_mmc_host), &pdev->dev);
 	if (!mmc) {
@@ -982,6 +1036,27 @@ static int jz4740_mmc_probe(struct platform_device* pdev)
 	host = mmc_priv(mmc);
 	host->pdata = pdata;
 
+	match = of_match_device(jz4740_mmc_of_match, &pdev->dev);
+	if (match) {
+		host->version = (enum jz4740_mmc_version)match->data;
+		ret = mmc_of_parse(mmc);
+		if (ret) {
+			if (ret != -EPROBE_DEFER)
+				dev_err(&pdev->dev,
+					"could not parse of data: %d\n", ret);
+			goto err_free_host;
+		}
+	} else {
+		/* JZ4740 should be the only one using legacy probe */
+		host->version = JZ_MMC_JZ4740;
+		mmc->caps |= MMC_CAP_SDIO_IRQ;
+		if (!(pdata && pdata->data_1bit))
+			mmc->caps |= MMC_CAP_4_BIT_DATA;
+		ret = jz4740_mmc_request_gpios(mmc, pdev);
+		if (ret)
+			goto err_free_host;
+	}
+
 	host->irq = platform_get_irq(pdev, 0);
 	if (host->irq < 0) {
 		ret = host->irq;
@@ -1004,16 +1079,11 @@ static int jz4740_mmc_probe(struct platform_device* pdev)
 		goto err_free_host;
 	}
 
-	ret = jz4740_mmc_request_gpios(mmc, pdev);
-	if (ret)
-		goto err_release_dma;
-
 	mmc->ops = &jz4740_mmc_ops;
-	mmc->f_min = JZ_MMC_CLK_RATE / 128;
-	mmc->f_max = JZ_MMC_CLK_RATE;
+	if (!mmc->f_max)
+		mmc->f_max = JZ_MMC_CLK_RATE;
+	mmc->f_min = mmc->f_max / 128;
 	mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34;
-	mmc->caps = (pdata && pdata->data_1bit) ? 0 : MMC_CAP_4_BIT_DATA;
-	mmc->caps |= MMC_CAP_SDIO_IRQ;
 
 	mmc->max_blk_size = (1 << 10) - 1;
 	mmc->max_blk_count = (1 << 15) - 1;
@@ -1025,7 +1095,9 @@ static int jz4740_mmc_probe(struct platform_device* pdev)
 	host->mmc = mmc;
 	host->pdev = pdev;
 	spin_lock_init(&host->lock);
-	host->irq_mask = 0xffff;
+	host->irq_mask = ~0;
+
+	jz4740_mmc_reset(host);
 
 	ret = request_threaded_irq(host->irq, jz_mmc_irq, jz_mmc_irq_worker, 0,
 			dev_name(&pdev->dev), host);
@@ -1034,20 +1106,20 @@ static int jz4740_mmc_probe(struct platform_device* pdev)
 		goto err_free_gpios;
 	}
 
-	jz4740_mmc_reset(host);
 	jz4740_mmc_clock_disable(host);
 	timer_setup(&host->timeout_timer, jz4740_mmc_timeout, 0);
 
-	host->use_dma = true;
-	if (host->use_dma && jz4740_mmc_acquire_dma_channels(host) != 0)
-		host->use_dma = false;
+	ret = jz4740_mmc_acquire_dma_channels(host);
+	if (ret == -EPROBE_DEFER)
+		goto err_free_irq;
+	host->use_dma = !ret;
 
 	platform_set_drvdata(pdev, host);
 	ret = mmc_add_host(mmc);
 
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to add mmc host: %d\n", ret);
-		goto err_free_irq;
+		goto err_release_dma;
 	}
 	dev_info(&pdev->dev, "JZ SD/MMC card driver registered\n");
 
@@ -1057,13 +1129,13 @@ static int jz4740_mmc_probe(struct platform_device* pdev)
 
 	return 0;
 
+err_release_dma:
+	if (host->use_dma)
+		jz4740_mmc_release_dma_channels(host);
 err_free_irq:
 	free_irq(host->irq, host);
 err_free_gpios:
 	jz4740_mmc_free_gpios(pdev);
-err_release_dma:
-	if (host->use_dma)
-		jz4740_mmc_release_dma_channels(host);
 err_free_host:
 	mmc_free_host(mmc);
 
@@ -1116,6 +1188,7 @@ static struct platform_driver jz4740_mmc_driver = {
 	.remove = jz4740_mmc_remove,
 	.driver = {
 		.name = "jz4740-mmc",
+		.of_match_table = of_match_ptr(jz4740_mmc_of_match),
 		.pm = JZ4740_MMC_PM_OPS,
 	},
 };
diff --git a/drivers/mmc/host/meson-gx-mmc.c b/drivers/mmc/host/meson-gx-mmc.c
index 4f972b879fe6..c201c378537e 100644
--- a/drivers/mmc/host/meson-gx-mmc.c
+++ b/drivers/mmc/host/meson-gx-mmc.c
@@ -35,6 +35,7 @@
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/regulator/consumer.h>
+#include <linux/reset.h>
 #include <linux/interrupt.h>
 #include <linux/bitfield.h>
 #include <linux/pinctrl/consumer.h>
@@ -47,15 +48,29 @@
 #define   CLK_CORE_PHASE_MASK GENMASK(9, 8)
 #define   CLK_TX_PHASE_MASK GENMASK(11, 10)
 #define   CLK_RX_PHASE_MASK GENMASK(13, 12)
-#define   CLK_TX_DELAY_MASK GENMASK(19, 16)
-#define   CLK_RX_DELAY_MASK GENMASK(23, 20)
+#define   CLK_V2_TX_DELAY_MASK GENMASK(19, 16)
+#define   CLK_V2_RX_DELAY_MASK GENMASK(23, 20)
+#define   CLK_V2_ALWAYS_ON BIT(24)
+
+#define   CLK_V3_TX_DELAY_MASK GENMASK(21, 16)
+#define   CLK_V3_RX_DELAY_MASK GENMASK(27, 22)
+#define   CLK_V3_ALWAYS_ON BIT(28)
+
 #define   CLK_DELAY_STEP_PS 200
 #define   CLK_PHASE_STEP 30
 #define   CLK_PHASE_POINT_NUM (360 / CLK_PHASE_STEP)
-#define   CLK_ALWAYS_ON BIT(24)
+
+#define   CLK_TX_DELAY_MASK(h)		(h->data->tx_delay_mask)
+#define   CLK_RX_DELAY_MASK(h)		(h->data->rx_delay_mask)
+#define   CLK_ALWAYS_ON(h)		(h->data->always_on)
 
 #define SD_EMMC_DELAY 0x4
 #define SD_EMMC_ADJUST 0x8
+
+#define SD_EMMC_DELAY1 0x4
+#define SD_EMMC_DELAY2 0x8
+#define SD_EMMC_V3_ADJUST 0xc
+
 #define SD_EMMC_CALOUT 0x10
 #define SD_EMMC_START 0x40
 #define   START_DESC_INIT BIT(0)
@@ -122,6 +137,12 @@
 
 #define MUX_CLK_NUM_PARENTS 2
 
+struct meson_mmc_data {
+	unsigned int tx_delay_mask;
+	unsigned int rx_delay_mask;
+	unsigned int always_on;
+};
+
 struct sd_emmc_desc {
 	u32 cmd_cfg;
 	u32 cmd_arg;
@@ -131,6 +152,7 @@ struct sd_emmc_desc {
 
 struct meson_host {
 	struct	device		*dev;
+	struct	meson_mmc_data *data;
 	struct	mmc_host	*mmc;
 	struct	mmc_command	*cmd;
 
@@ -474,7 +496,7 @@ static int meson_mmc_clk_init(struct meson_host *host)
 
 	/* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
 	clk_reg = 0;
-	clk_reg |= CLK_ALWAYS_ON;
+	clk_reg |= CLK_ALWAYS_ON(host);
 	clk_reg |= CLK_DIV_MASK;
 	writel(clk_reg, host->regs + SD_EMMC_CLOCK);
 
@@ -574,7 +596,7 @@ static int meson_mmc_clk_init(struct meson_host *host)
 
 	tx->reg = host->regs + SD_EMMC_CLOCK;
 	tx->phase_mask = CLK_TX_PHASE_MASK;
-	tx->delay_mask = CLK_TX_DELAY_MASK;
+	tx->delay_mask = CLK_TX_DELAY_MASK(host);
 	tx->delay_step_ps = CLK_DELAY_STEP_PS;
 	tx->hw.init = &init;
 
@@ -597,7 +619,7 @@ static int meson_mmc_clk_init(struct meson_host *host)
 
 	rx->reg = host->regs + SD_EMMC_CLOCK;
 	rx->phase_mask = CLK_RX_PHASE_MASK;
-	rx->delay_mask = CLK_RX_DELAY_MASK;
+	rx->delay_mask = CLK_RX_DELAY_MASK(host);
 	rx->delay_step_ps = CLK_DELAY_STEP_PS;
 	rx->hw.init = &init;
 
@@ -1184,6 +1206,21 @@ static int meson_mmc_probe(struct platform_device *pdev)
 		goto free_host;
 	}
 
+	host->data = (struct meson_mmc_data *)
+		of_device_get_match_data(&pdev->dev);
+	if (!host->data) {
+		ret = -EINVAL;
+		goto free_host;
+	}
+
+	ret = device_reset_optional(&pdev->dev);
+	if (ret) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "device reset failed: %d\n", ret);
+
+		return ret;
+	}
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	host->regs = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(host->regs)) {
@@ -1315,11 +1352,24 @@ static int meson_mmc_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct meson_mmc_data meson_gx_data = {
+	.tx_delay_mask	= CLK_V2_TX_DELAY_MASK,
+	.rx_delay_mask	= CLK_V2_RX_DELAY_MASK,
+	.always_on	= CLK_V2_ALWAYS_ON,
+};
+
+static const struct meson_mmc_data meson_axg_data = {
+	.tx_delay_mask	= CLK_V3_TX_DELAY_MASK,
+	.rx_delay_mask	= CLK_V3_RX_DELAY_MASK,
+	.always_on	= CLK_V3_ALWAYS_ON,
+};
+
 static const struct of_device_id meson_mmc_of_match[] = {
-	{ .compatible = "amlogic,meson-gx-mmc", },
-	{ .compatible = "amlogic,meson-gxbb-mmc", },
-	{ .compatible = "amlogic,meson-gxl-mmc", },
-	{ .compatible = "amlogic,meson-gxm-mmc", },
+	{ .compatible = "amlogic,meson-gx-mmc",		.data = &meson_gx_data },
+	{ .compatible = "amlogic,meson-gxbb-mmc", 	.data = &meson_gx_data },
+	{ .compatible = "amlogic,meson-gxl-mmc",	.data = &meson_gx_data },
+	{ .compatible = "amlogic,meson-gxm-mmc",	.data = &meson_gx_data },
+	{ .compatible = "amlogic,meson-axg-mmc",	.data = &meson_axg_data },
 	{}
 };
 MODULE_DEVICE_TABLE(of, meson_mmc_of_match);
@@ -1335,6 +1385,6 @@ static struct platform_driver meson_mmc_driver = {
 
 module_platform_driver(meson_mmc_driver);
 
-MODULE_DESCRIPTION("Amlogic S905*/GX* SD/eMMC driver");
+MODULE_DESCRIPTION("Amlogic S905*/GX*/AXG SD/eMMC driver");
 MODULE_AUTHOR("Kevin Hilman <khilman@baylibre.com>");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/mmc/host/mmci.c b/drivers/mmc/host/mmci.c
index 70b0df8b9c78..f1849775e47e 100644
--- a/drivers/mmc/host/mmci.c
+++ b/drivers/mmc/host/mmci.c
@@ -1253,15 +1253,12 @@ static irqreturn_t mmci_pio_irq(int irq, void *dev_id)
 	struct sg_mapping_iter *sg_miter = &host->sg_miter;
 	struct variant_data *variant = host->variant;
 	void __iomem *base = host->base;
-	unsigned long flags;
 	u32 status;
 
 	status = readl(base + MMCISTATUS);
 
 	dev_dbg(mmc_dev(host->mmc), "irq1 (pio) %08x\n", status);
 
-	local_irq_save(flags);
-
 	do {
 		unsigned int remain, len;
 		char *buffer;
@@ -1301,8 +1298,6 @@ static irqreturn_t mmci_pio_irq(int irq, void *dev_id)
 
 	sg_miter_stop(sg_miter);
 
-	local_irq_restore(flags);
-
 	/*
 	 * If we have less than the fifo 'half-full' threshold to transfer,
 	 * trigger a PIO interrupt as soon as any data is available.
diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c
index cb274e822293..04841386b65d 100644
--- a/drivers/mmc/host/mtk-sd.c
+++ b/drivers/mmc/host/mtk-sd.c
@@ -19,6 +19,7 @@
 #include <linux/ioport.h>
 #include <linux/irq.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
 #include <linux/of_irq.h>
 #include <linux/of_gpio.h>
 #include <linux/pinctrl/consumer.h>
@@ -70,6 +71,7 @@
 #define SDC_ADV_CFG0     0x64
 #define EMMC_IOCON       0x7c
 #define SDC_ACMD_RESP    0x80
+#define DMA_SA_H4BIT     0x8c
 #define MSDC_DMA_SA      0x90
 #define MSDC_DMA_CTRL    0x98
 #define MSDC_DMA_CFG     0x9c
@@ -194,6 +196,9 @@
 /* SDC_ADV_CFG0 mask */
 #define SDC_RX_ENHANCE_EN	(0x1 << 20)	/* RW */
 
+/* DMA_SA_H4BIT mask */
+#define DMA_ADDR_HIGH_4BIT      (0xf << 0)      /* RW */
+
 /* MSDC_DMA_CTRL mask */
 #define MSDC_DMA_CTRL_START     (0x1 << 0)	/* W */
 #define MSDC_DMA_CTRL_STOP      (0x1 << 1)	/* W */
@@ -227,6 +232,7 @@
 
 #define MSDC_PATCH_BIT2_CFGRESP   (0x1 << 15)   /* RW */
 #define MSDC_PATCH_BIT2_CFGCRCSTS (0x1 << 28)   /* RW */
+#define MSDC_PB2_SUPPORT_64G      (0x1 << 1)    /* RW */
 #define MSDC_PB2_RESPWAIT         (0x3 << 2)    /* RW */
 #define MSDC_PB2_RESPSTSENSEL     (0x7 << 16)   /* RW */
 #define MSDC_PB2_CRCSTSENSEL      (0x7 << 29)   /* RW */
@@ -280,6 +286,8 @@ struct mt_gpdma_desc {
 #define GPDMA_DESC_BDP		(0x1 << 1)
 #define GPDMA_DESC_CHECKSUM	(0xff << 8) /* bit8 ~ bit15 */
 #define GPDMA_DESC_INT		(0x1 << 16)
+#define GPDMA_DESC_NEXT_H4	(0xf << 24)
+#define GPDMA_DESC_PTR_H4	(0xf << 28)
 	u32 next;
 	u32 ptr;
 	u32 gpd_data_len;
@@ -296,6 +304,8 @@ struct mt_bdma_desc {
 #define BDMA_DESC_CHECKSUM	(0xff << 8) /* bit8 ~ bit15 */
 #define BDMA_DESC_BLKPAD	(0x1 << 17)
 #define BDMA_DESC_DWPAD		(0x1 << 18)
+#define BDMA_DESC_NEXT_H4	(0xf << 24)
+#define BDMA_DESC_PTR_H4	(0xf << 28)
 	u32 next;
 	u32 ptr;
 	u32 bd_data_len;
@@ -334,6 +344,7 @@ struct mtk_mmc_compatible {
 	bool busy_check;
 	bool stop_clk_fix;
 	bool enhance_rx;
+	bool support_64g;
 };
 
 struct msdc_tune_para {
@@ -403,6 +414,7 @@ static const struct mtk_mmc_compatible mt8135_compat = {
 	.busy_check = false,
 	.stop_clk_fix = false,
 	.enhance_rx = false,
+	.support_64g = false,
 };
 
 static const struct mtk_mmc_compatible mt8173_compat = {
@@ -414,6 +426,7 @@ static const struct mtk_mmc_compatible mt8173_compat = {
 	.busy_check = false,
 	.stop_clk_fix = false,
 	.enhance_rx = false,
+	.support_64g = false,
 };
 
 static const struct mtk_mmc_compatible mt2701_compat = {
@@ -425,6 +438,7 @@ static const struct mtk_mmc_compatible mt2701_compat = {
 	.busy_check = false,
 	.stop_clk_fix = false,
 	.enhance_rx = false,
+	.support_64g = false,
 };
 
 static const struct mtk_mmc_compatible mt2712_compat = {
@@ -436,6 +450,7 @@ static const struct mtk_mmc_compatible mt2712_compat = {
 	.busy_check = true,
 	.stop_clk_fix = true,
 	.enhance_rx = true,
+	.support_64g = true,
 };
 
 static const struct mtk_mmc_compatible mt7622_compat = {
@@ -447,6 +462,7 @@ static const struct mtk_mmc_compatible mt7622_compat = {
 	.busy_check = true,
 	.stop_clk_fix = true,
 	.enhance_rx = true,
+	.support_64g = false,
 };
 
 static const struct of_device_id msdc_of_ids[] = {
@@ -556,7 +572,12 @@ static inline void msdc_dma_setup(struct msdc_host *host, struct msdc_dma *dma,
 		/* init bd */
 		bd[j].bd_info &= ~BDMA_DESC_BLKPAD;
 		bd[j].bd_info &= ~BDMA_DESC_DWPAD;
-		bd[j].ptr = (u32)dma_address;
+		bd[j].ptr = lower_32_bits(dma_address);
+		if (host->dev_comp->support_64g) {
+			bd[j].bd_info &= ~BDMA_DESC_PTR_H4;
+			bd[j].bd_info |= (upper_32_bits(dma_address) & 0xf)
+					 << 28;
+		}
 		bd[j].bd_data_len &= ~BDMA_DESC_BUFLEN;
 		bd[j].bd_data_len |= (dma_len & BDMA_DESC_BUFLEN);
 
@@ -575,7 +596,10 @@ static inline void msdc_dma_setup(struct msdc_host *host, struct msdc_dma *dma,
 	dma_ctrl &= ~(MSDC_DMA_CTRL_BRUSTSZ | MSDC_DMA_CTRL_MODE);
 	dma_ctrl |= (MSDC_BURST_64B << 12 | 1 << 8);
 	writel_relaxed(dma_ctrl, host->base + MSDC_DMA_CTRL);
-	writel((u32)dma->gpd_addr, host->base + MSDC_DMA_SA);
+	if (host->dev_comp->support_64g)
+		sdr_set_field(host->base + DMA_SA_H4BIT, DMA_ADDR_HIGH_4BIT,
+			      upper_32_bits(dma->gpd_addr) & 0xf);
+	writel(lower_32_bits(dma->gpd_addr), host->base + MSDC_DMA_SA);
 }
 
 static void msdc_prepare_data(struct msdc_host *host, struct mmc_request *mrq)
@@ -1366,6 +1390,9 @@ static void msdc_init_hw(struct msdc_host *host)
 			     MSDC_PATCH_BIT2_CFGCRCSTS);
 	}
 
+	if (host->dev_comp->support_64g)
+		sdr_set_bits(host->base + MSDC_PATCH_BIT2,
+			     MSDC_PB2_SUPPORT_64G);
 	if (host->dev_comp->data_tune) {
 		sdr_set_bits(host->base + tune_reg,
 			     MSDC_PAD_TUNE_RD_SEL | MSDC_PAD_TUNE_CMD_SEL);
@@ -1407,19 +1434,32 @@ static void msdc_init_gpd_bd(struct msdc_host *host, struct msdc_dma *dma)
 {
 	struct mt_gpdma_desc *gpd = dma->gpd;
 	struct mt_bdma_desc *bd = dma->bd;
+	dma_addr_t dma_addr;
 	int i;
 
 	memset(gpd, 0, sizeof(struct mt_gpdma_desc) * 2);
 
+	dma_addr = dma->gpd_addr + sizeof(struct mt_gpdma_desc);
 	gpd->gpd_info = GPDMA_DESC_BDP; /* hwo, cs, bd pointer */
-	gpd->ptr = (u32)dma->bd_addr; /* physical address */
 	/* gpd->next is must set for desc DMA
 	 * That's why must alloc 2 gpd structure.
 	 */
-	gpd->next = (u32)dma->gpd_addr + sizeof(struct mt_gpdma_desc);
+	gpd->next = lower_32_bits(dma_addr);
+	if (host->dev_comp->support_64g)
+		gpd->gpd_info |= (upper_32_bits(dma_addr) & 0xf) << 24;
+
+	dma_addr = dma->bd_addr;
+	gpd->ptr = lower_32_bits(dma->bd_addr); /* physical address */
+	if (host->dev_comp->support_64g)
+		gpd->gpd_info |= (upper_32_bits(dma_addr) & 0xf) << 28;
+
 	memset(bd, 0, sizeof(struct mt_bdma_desc) * MAX_BD_NUM);
-	for (i = 0; i < (MAX_BD_NUM - 1); i++)
-		bd[i].next = (u32)dma->bd_addr + sizeof(*bd) * (i + 1);
+	for (i = 0; i < (MAX_BD_NUM - 1); i++) {
+		dma_addr = dma->bd_addr + sizeof(*bd) * (i + 1);
+		bd[i].next = lower_32_bits(dma_addr);
+		if (host->dev_comp->support_64g)
+			bd[i].bd_info |= (upper_32_bits(dma_addr) & 0xf) << 24;
+	}
 }
 
 static void msdc_ops_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
@@ -1820,7 +1860,6 @@ static int msdc_drv_probe(struct platform_device *pdev)
 	struct mmc_host *mmc;
 	struct msdc_host *host;
 	struct resource *res;
-	const struct of_device_id *of_id;
 	int ret;
 
 	if (!pdev->dev.of_node) {
@@ -1828,9 +1867,6 @@ static int msdc_drv_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	of_id = of_match_node(msdc_of_ids, pdev->dev.of_node);
-	if (!of_id)
-		return -EINVAL;
 	/* Allocate MMC host for this device */
 	mmc = mmc_alloc_host(sizeof(struct msdc_host), &pdev->dev);
 	if (!mmc)
@@ -1899,7 +1935,7 @@ static int msdc_drv_probe(struct platform_device *pdev)
 	msdc_of_property_parse(pdev, host);
 
 	host->dev = &pdev->dev;
-	host->dev_comp = of_id->data;
+	host->dev_comp = of_device_get_match_data(&pdev->dev);
 	host->mmc = mmc;
 	host->src_clk_freq = clk_get_rate(host->src_clk);
 	/* Set host parameters to mmc */
@@ -1916,7 +1952,10 @@ static int msdc_drv_probe(struct platform_device *pdev)
 	mmc->max_blk_size = 2048;
 	mmc->max_req_size = 512 * 1024;
 	mmc->max_blk_count = mmc->max_req_size / 512;
-	host->dma_mask = DMA_BIT_MASK(32);
+	if (host->dev_comp->support_64g)
+		host->dma_mask = DMA_BIT_MASK(36);
+	else
+		host->dma_mask = DMA_BIT_MASK(32);
 	mmc_dev(mmc)->dma_mask = &host->dma_mask;
 
 	host->timeout_clks = 3 * 1048576;
diff --git a/drivers/mmc/host/mxcmmc.c b/drivers/mmc/host/mxcmmc.c
index 5ff8ef7223cc..75f781c11e89 100644
--- a/drivers/mmc/host/mxcmmc.c
+++ b/drivers/mmc/host/mxcmmc.c
@@ -21,6 +21,7 @@
 #include <linux/init.h>
 #include <linux/ioport.h>
 #include <linux/platform_device.h>
+#include <linux/highmem.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/blkdev.h>
@@ -291,8 +292,10 @@ static void mxcmci_swap_buffers(struct mmc_data *data)
 	struct scatterlist *sg;
 	int i;
 
-	for_each_sg(data->sg, sg, data->sg_len, i)
-		buffer_swap32(sg_virt(sg), sg->length);
+	for_each_sg(data->sg, sg, data->sg_len, i) {
+		void *buf = kmap_atomic(sg_page(sg) + sg->offset;
+		buffer_swap32(buf, sg->length);
+		kunmap_atomic(buf);
 }
 #else
 static inline void mxcmci_swap_buffers(struct mmc_data *data) {}
@@ -609,6 +612,7 @@ static int mxcmci_transfer_data(struct mxcmci_host *host)
 {
 	struct mmc_data *data = host->req->data;
 	struct scatterlist *sg;
+	void *buf;
 	int stat, i;
 
 	host->data = data;
@@ -616,14 +620,18 @@ static int mxcmci_transfer_data(struct mxcmci_host *host)
 
 	if (data->flags & MMC_DATA_READ) {
 		for_each_sg(data->sg, sg, data->sg_len, i) {
-			stat = mxcmci_pull(host, sg_virt(sg), sg->length);
+			buf = kmap_atomic(sg_page(sg) + sg->offset);
+			stat = mxcmci_pull(host, buf, sg->length);
+			kunmap(buf);
 			if (stat)
 				return stat;
 			host->datasize += sg->length;
 		}
 	} else {
 		for_each_sg(data->sg, sg, data->sg_len, i) {
-			stat = mxcmci_push(host, sg_virt(sg), sg->length);
+			buf = kmap_atomic(sg_page(sg) + sg->offset);
+			stat = mxcmci_push(host, buf, sg->length);
+			kunmap(buf);
 			if (stat)
 				return stat;
 			host->datasize += sg->length;
@@ -1206,7 +1214,8 @@ static int mxcmci_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static int __maybe_unused mxcmci_suspend(struct device *dev)
+#ifdef CONFIG_PM_SLEEP
+static int mxcmci_suspend(struct device *dev)
 {
 	struct mmc_host *mmc = dev_get_drvdata(dev);
 	struct mxcmci_host *host = mmc_priv(mmc);
@@ -1216,7 +1225,7 @@ static int __maybe_unused mxcmci_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused mxcmci_resume(struct device *dev)
+static int mxcmci_resume(struct device *dev)
 {
 	struct mmc_host *mmc = dev_get_drvdata(dev);
 	struct mxcmci_host *host = mmc_priv(mmc);
@@ -1232,6 +1241,7 @@ static int __maybe_unused mxcmci_resume(struct device *dev)
 
 	return ret;
 }
+#endif
 
 static SIMPLE_DEV_PM_OPS(mxcmci_pm_ops, mxcmci_suspend, mxcmci_resume);
 
diff --git a/drivers/mmc/host/renesas_sdhi_internal_dmac.c b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
index 6af946d16d24..b6edb7a695b5 100644
--- a/drivers/mmc/host/renesas_sdhi_internal_dmac.c
+++ b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
@@ -91,7 +91,7 @@ static const struct renesas_sdhi_of_data of_rcar_gen3_compatible = {
 	.scc_offset	= 0x1000,
 	.taps		= rcar_gen3_scc_taps,
 	.taps_num	= ARRAY_SIZE(rcar_gen3_scc_taps),
-	/* Gen3 SDHI DMAC can handle 0xffffffff blk count, but seg = 1 */
+	/* DMAC can handle 0xffffffff blk count but only 1 segment */
 	.max_blk_count	= 0xffffffff,
 	.max_segs	= 1,
 };
@@ -157,38 +157,34 @@ renesas_sdhi_internal_dmac_start_dma(struct tmio_mmc_host *host,
 {
 	struct scatterlist *sg = host->sg_ptr;
 	u32 dtran_mode = DTRAN_MODE_BUS_WID_TH | DTRAN_MODE_ADDR_MODE;
-	enum dma_data_direction dir;
-	int ret;
 
-	/* This DMAC cannot handle if sg_len is not 1 */
-	WARN_ON(host->sg_len > 1);
+	if (!dma_map_sg(&host->pdev->dev, sg, host->sg_len,
+			mmc_get_dma_dir(data)))
+		goto force_pio;
 
 	/* This DMAC cannot handle if buffer is not 8-bytes alignment */
-	if (!IS_ALIGNED(sg->offset, 8))
+	if (!IS_ALIGNED(sg_dma_address(sg), 8)) {
+		dma_unmap_sg(&host->pdev->dev, sg, host->sg_len,
+			     mmc_get_dma_dir(data));
 		goto force_pio;
+	}
 
 	if (data->flags & MMC_DATA_READ) {
 		dtran_mode |= DTRAN_MODE_CH_NUM_CH1;
-		dir = DMA_FROM_DEVICE;
 		if (test_bit(SDHI_INTERNAL_DMAC_ONE_RX_ONLY, &global_flags) &&
 		    test_and_set_bit(SDHI_INTERNAL_DMAC_RX_IN_USE, &global_flags))
 			goto force_pio;
 	} else {
 		dtran_mode |= DTRAN_MODE_CH_NUM_CH0;
-		dir = DMA_TO_DEVICE;
 	}
 
-	ret = dma_map_sg(&host->pdev->dev, sg, host->sg_len, dir);
-	if (ret == 0)
-		goto force_pio;
-
 	renesas_sdhi_internal_dmac_enable_dma(host, true);
 
 	/* set dma parameters */
 	renesas_sdhi_internal_dmac_dm_write(host, DM_CM_DTRAN_MODE,
 					    dtran_mode);
 	renesas_sdhi_internal_dmac_dm_write(host, DM_DTRAN_ADDR,
-					    sg->dma_address);
+					    sg_dma_address(sg));
 
 	return;
 
@@ -272,12 +268,17 @@ static const struct tmio_mmc_dma_ops renesas_sdhi_internal_dmac_dma_ops = {
  * implementation as others may use a different implementation.
  */
 static const struct soc_device_attribute gen3_soc_whitelist[] = {
+	/* specific ones */
 	{ .soc_id = "r8a7795", .revision = "ES1.*",
 	  .data = (void *)BIT(SDHI_INTERNAL_DMAC_ONE_RX_ONLY) },
-	{ .soc_id = "r8a7795", .revision = "ES2.0" },
 	{ .soc_id = "r8a7796", .revision = "ES1.0",
 	  .data = (void *)BIT(SDHI_INTERNAL_DMAC_ONE_RX_ONLY) },
-	{ .soc_id = "r8a77995", .revision = "ES1.0" },
+	/* generic ones */
+	{ .soc_id = "r8a7795" },
+	{ .soc_id = "r8a7796" },
+	{ .soc_id = "r8a77965" },
+	{ .soc_id = "r8a77980" },
+	{ .soc_id = "r8a77995" },
 	{ /* sentinel */ }
 };
 
diff --git a/drivers/mmc/host/rtsx_usb_sdmmc.c b/drivers/mmc/host/rtsx_usb_sdmmc.c
index 78422079ecfa..9a3ff22dd0fe 100644
--- a/drivers/mmc/host/rtsx_usb_sdmmc.c
+++ b/drivers/mmc/host/rtsx_usb_sdmmc.c
@@ -26,7 +26,6 @@
 #include <linux/mmc/host.h>
 #include <linux/mmc/mmc.h>
 #include <linux/mmc/sd.h>
-#include <linux/mmc/sdio.h>
 #include <linux/mmc/card.h>
 #include <linux/scatterlist.h>
 #include <linux/pm_runtime.h>
@@ -343,7 +342,7 @@ static void sd_send_cmd_get_rsp(struct rtsx_usb_sdmmc *host,
 	}
 
 	if (rsp_type == SD_RSP_TYPE_R1b)
-		timeout = 3000;
+		timeout = cmd->busy_timeout ? cmd->busy_timeout : 3000;
 
 	if (cmd->opcode == SD_SWITCH_VOLTAGE) {
 		err = rtsx_usb_write_register(ucr, SD_BUS_STAT,
@@ -839,17 +838,6 @@ static void sdmmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
 		goto finish_detect_card;
 	}
 
-	/*
-	 * Reject SDIO CMDs to speed up card identification
-	 * since unsupported
-	 */
-	if (cmd->opcode == SD_IO_SEND_OP_COND ||
-	    cmd->opcode == SD_IO_RW_DIRECT ||
-	    cmd->opcode == SD_IO_RW_EXTENDED) {
-		cmd->error = -EINVAL;
-		goto finish;
-	}
-
 	mutex_lock(&ucr->dev_mutex);
 
 	mutex_lock(&host->host_mutex);
@@ -1332,8 +1320,9 @@ static void rtsx_usb_init_host(struct rtsx_usb_sdmmc *host)
 	mmc->caps = MMC_CAP_4_BIT_DATA | MMC_CAP_SD_HIGHSPEED |
 		MMC_CAP_MMC_HIGHSPEED | MMC_CAP_BUS_WIDTH_TEST |
 		MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 | MMC_CAP_UHS_SDR50 |
-		MMC_CAP_NEEDS_POLL;
-	mmc->caps2 = MMC_CAP2_NO_PRESCAN_POWERUP | MMC_CAP2_FULL_PWR_CYCLE;
+		MMC_CAP_NEEDS_POLL | MMC_CAP_ERASE;
+	mmc->caps2 = MMC_CAP2_NO_PRESCAN_POWERUP | MMC_CAP2_FULL_PWR_CYCLE |
+		MMC_CAP2_NO_SDIO;
 
 	mmc->max_current_330 = 400;
 	mmc->max_current_180 = 800;
diff --git a/drivers/mmc/host/sdhci-bcm-kona.c b/drivers/mmc/host/sdhci-bcm-kona.c
index 11ca95c60bcf..bdbd4897c0f7 100644
--- a/drivers/mmc/host/sdhci-bcm-kona.c
+++ b/drivers/mmc/host/sdhci-bcm-kona.c
@@ -284,10 +284,8 @@ static int sdhci_bcm_kona_probe(struct platform_device *pdev)
 	sdhci_bcm_kona_sd_init(host);
 
 	ret = sdhci_add_host(host);
-	if (ret) {
-		dev_err(dev, "Failed sdhci_add_host\n");
+	if (ret)
 		goto err_reset;
-	}
 
 	/* if device is eMMC, emulate card insert right here */
 	if (!mmc_card_is_removable(host->mmc)) {
diff --git a/drivers/mmc/host/sdhci-cadence.c b/drivers/mmc/host/sdhci-cadence.c
index 0f589e26ee63..7a343b87b5e5 100644
--- a/drivers/mmc/host/sdhci-cadence.c
+++ b/drivers/mmc/host/sdhci-cadence.c
@@ -253,6 +253,7 @@ static int sdhci_cdns_set_tune_val(struct sdhci_host *host, unsigned int val)
 	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
 	void __iomem *reg = priv->hrs_addr + SDHCI_CDNS_HRS06;
 	u32 tmp;
+	int i, ret;
 
 	if (WARN_ON(!FIELD_FIT(SDHCI_CDNS_HRS06_TUNE, val)))
 		return -EINVAL;
@@ -260,11 +261,24 @@ static int sdhci_cdns_set_tune_val(struct sdhci_host *host, unsigned int val)
 	tmp = readl(reg);
 	tmp &= ~SDHCI_CDNS_HRS06_TUNE;
 	tmp |= FIELD_PREP(SDHCI_CDNS_HRS06_TUNE, val);
-	tmp |= SDHCI_CDNS_HRS06_TUNE_UP;
-	writel(tmp, reg);
 
-	return readl_poll_timeout(reg, tmp, !(tmp & SDHCI_CDNS_HRS06_TUNE_UP),
-				  0, 1);
+	/*
+	 * Workaround for IP errata:
+	 * The IP6116 SD/eMMC PHY design has a timing issue on receive data
+	 * path. Send tune request twice.
+	 */
+	for (i = 0; i < 2; i++) {
+		tmp |= SDHCI_CDNS_HRS06_TUNE_UP;
+		writel(tmp, reg);
+
+		ret = readl_poll_timeout(reg, tmp,
+					 !(tmp & SDHCI_CDNS_HRS06_TUNE_UP),
+					 0, 1);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
 static int sdhci_cdns_execute_tuning(struct mmc_host *mmc, u32 opcode)
diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c
index cd2b5f643a15..d6aef70d34fa 100644
--- a/drivers/mmc/host/sdhci-esdhc-imx.c
+++ b/drivers/mmc/host/sdhci-esdhc-imx.c
@@ -41,6 +41,12 @@
 #define  ESDHC_VENDOR_SPEC_FRC_SDCLK_ON	(1 << 8)
 #define ESDHC_WTMK_LVL			0x44
 #define  ESDHC_WTMK_DEFAULT_VAL		0x10401040
+#define  ESDHC_WTMK_LVL_RD_WML_MASK	0x000000FF
+#define  ESDHC_WTMK_LVL_RD_WML_SHIFT	0
+#define  ESDHC_WTMK_LVL_WR_WML_MASK	0x00FF0000
+#define  ESDHC_WTMK_LVL_WR_WML_SHIFT	16
+#define  ESDHC_WTMK_LVL_WML_VAL_DEF	64
+#define  ESDHC_WTMK_LVL_WML_VAL_MAX	128
 #define ESDHC_MIX_CTRL			0x48
 #define  ESDHC_MIX_CTRL_DDREN		(1 << 3)
 #define  ESDHC_MIX_CTRL_AC23EN		(1 << 7)
@@ -516,6 +522,7 @@ static void esdhc_writew_le(struct sdhci_host *host, u16 val, int reg)
 		}
 
 		if (esdhc_is_usdhc(imx_data)) {
+			u32 wml;
 			u32 m = readl(host->ioaddr + ESDHC_MIX_CTRL);
 			/* Swap AC23 bit */
 			if (val & SDHCI_TRNS_AUTO_CMD23) {
@@ -524,6 +531,21 @@ static void esdhc_writew_le(struct sdhci_host *host, u16 val, int reg)
 			}
 			m = val | (m & ~ESDHC_MIX_CTRL_SDHCI_MASK);
 			writel(m, host->ioaddr + ESDHC_MIX_CTRL);
+
+			/* Set watermark levels for PIO access to maximum value
+			 * (128 words) to accommodate full 512 bytes buffer.
+			 * For DMA access restore the levels to default value.
+			 */
+			m = readl(host->ioaddr + ESDHC_WTMK_LVL);
+			if (val & SDHCI_TRNS_DMA)
+				wml = ESDHC_WTMK_LVL_WML_VAL_DEF;
+			else
+				wml = ESDHC_WTMK_LVL_WML_VAL_MAX;
+			m &= ~(ESDHC_WTMK_LVL_RD_WML_MASK |
+			       ESDHC_WTMK_LVL_WR_WML_MASK);
+			m |= (wml << ESDHC_WTMK_LVL_RD_WML_SHIFT) |
+			     (wml << ESDHC_WTMK_LVL_WR_WML_SHIFT);
+			writel(m, host->ioaddr + ESDHC_WTMK_LVL);
 		} else {
 			/*
 			 * Postpone this write, we must do it together with a
diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c
index c283291db705..bb11916c58a1 100644
--- a/drivers/mmc/host/sdhci-msm.c
+++ b/drivers/mmc/host/sdhci-msm.c
@@ -21,6 +21,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/iopoll.h>
+#include <linux/regulator/consumer.h>
 
 #include "sdhci-pltfm.h"
 
@@ -77,10 +78,16 @@
 #define CORE_HC_MCLK_SEL_DFLT	(2 << 8)
 #define CORE_HC_MCLK_SEL_HS400	(3 << 8)
 #define CORE_HC_MCLK_SEL_MASK	(3 << 8)
+#define CORE_IO_PAD_PWR_SWITCH_EN	(1 << 15)
+#define CORE_IO_PAD_PWR_SWITCH  (1 << 16)
 #define CORE_HC_SELECT_IN_EN	BIT(18)
 #define CORE_HC_SELECT_IN_HS400	(6 << 19)
 #define CORE_HC_SELECT_IN_MASK	(7 << 19)
 
+#define CORE_3_0V_SUPPORT	(1 << 25)
+#define CORE_1_8V_SUPPORT	(1 << 26)
+#define CORE_VOLT_SUPPORT	(CORE_3_0V_SUPPORT | CORE_1_8V_SUPPORT)
+
 #define CORE_CSR_CDC_CTLR_CFG0		0x130
 #define CORE_SW_TRIG_FULL_CALIB		BIT(16)
 #define CORE_HW_AUTOCAL_ENA		BIT(17)
@@ -148,6 +155,7 @@ struct sdhci_msm_host {
 	u32 curr_io_level;
 	wait_queue_head_t pwr_irq_wait;
 	bool pwr_irq_flag;
+	u32 caps_0;
 };
 
 static unsigned int msm_get_clock_rate_for_bus_mode(struct sdhci_host *host,
@@ -1103,8 +1111,8 @@ static void sdhci_msm_handle_pwr_irq(struct sdhci_host *host, int irq)
 	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
 	u32 irq_status, irq_ack = 0;
 	int retry = 10;
-	int pwr_state = 0, io_level = 0;
-
+	u32 pwr_state = 0, io_level = 0;
+	u32 config;
 
 	irq_status = readl_relaxed(msm_host->core_mem + CORE_PWRCTL_STATUS);
 	irq_status &= INT_MASK;
@@ -1161,6 +1169,38 @@ static void sdhci_msm_handle_pwr_irq(struct sdhci_host *host, int irq)
 	 */
 	writel_relaxed(irq_ack, msm_host->core_mem + CORE_PWRCTL_CTL);
 
+	/*
+	 * If we don't have info regarding the voltage levels supported by
+	 * regulators, don't change the IO PAD PWR SWITCH.
+	 */
+	if (msm_host->caps_0 & CORE_VOLT_SUPPORT) {
+		u32 new_config;
+		/*
+		 * We should unset IO PAD PWR switch only if the register write
+		 * can set IO lines high and the regulator also switches to 3 V.
+		 * Else, we should keep the IO PAD PWR switch set.
+		 * This is applicable to certain targets where eMMC vccq supply
+		 * is only 1.8V. In such targets, even during REQ_IO_HIGH, the
+		 * IO PAD PWR switch must be kept set to reflect actual
+		 * regulator voltage. This way, during initialization of
+		 * controllers with only 1.8V, we will set the IO PAD bit
+		 * without waiting for a REQ_IO_LOW.
+		 */
+		config = readl_relaxed(host->ioaddr + CORE_VENDOR_SPEC);
+		new_config = config;
+
+		if ((io_level & REQ_IO_HIGH) &&
+				(msm_host->caps_0 & CORE_3_0V_SUPPORT))
+			new_config &= ~CORE_IO_PAD_PWR_SWITCH;
+		else if ((io_level & REQ_IO_LOW) ||
+				(msm_host->caps_0 & CORE_1_8V_SUPPORT))
+			new_config |= CORE_IO_PAD_PWR_SWITCH;
+
+		if (config ^ new_config)
+			writel_relaxed(new_config,
+					host->ioaddr + CORE_VENDOR_SPEC);
+	}
+
 	if (pwr_state)
 		msm_host->curr_pwr_state = pwr_state;
 	if (io_level)
@@ -1313,6 +1353,45 @@ static void sdhci_msm_writeb(struct sdhci_host *host, u8 val, int reg)
 		sdhci_msm_check_power_status(host, req_type);
 }
 
+static void sdhci_msm_set_regulator_caps(struct sdhci_msm_host *msm_host)
+{
+	struct mmc_host *mmc = msm_host->mmc;
+	struct regulator *supply = mmc->supply.vqmmc;
+	u32 caps = 0, config;
+	struct sdhci_host *host = mmc_priv(mmc);
+
+	if (!IS_ERR(mmc->supply.vqmmc)) {
+		if (regulator_is_supported_voltage(supply, 1700000, 1950000))
+			caps |= CORE_1_8V_SUPPORT;
+		if (regulator_is_supported_voltage(supply, 2700000, 3600000))
+			caps |= CORE_3_0V_SUPPORT;
+
+		if (!caps)
+			pr_warn("%s: 1.8/3V not supported for vqmmc\n",
+					mmc_hostname(mmc));
+	}
+
+	if (caps) {
+		/*
+		 * Set the PAD_PWR_SWITCH_EN bit so that the PAD_PWR_SWITCH
+		 * bit can be used as required later on.
+		 */
+		u32 io_level = msm_host->curr_io_level;
+
+		config = readl_relaxed(host->ioaddr + CORE_VENDOR_SPEC);
+		config |= CORE_IO_PAD_PWR_SWITCH_EN;
+
+		if ((io_level & REQ_IO_HIGH) && (caps &	CORE_3_0V_SUPPORT))
+			config &= ~CORE_IO_PAD_PWR_SWITCH;
+		else if ((io_level & REQ_IO_LOW) || (caps & CORE_1_8V_SUPPORT))
+			config |= CORE_IO_PAD_PWR_SWITCH;
+
+		writel_relaxed(config, host->ioaddr + CORE_VENDOR_SPEC);
+	}
+	msm_host->caps_0 |= caps;
+	pr_debug("%s: supported caps: 0x%08x\n", mmc_hostname(mmc), caps);
+}
+
 static const struct of_device_id sdhci_msm_dt_match[] = {
 	{ .compatible = "qcom,sdhci-msm-v4" },
 	{},
@@ -1530,6 +1609,7 @@ static int sdhci_msm_probe(struct platform_device *pdev)
 	ret = sdhci_add_host(host);
 	if (ret)
 		goto pm_runtime_disable;
+	sdhci_msm_set_regulator_caps(msm_host);
 
 	pm_runtime_mark_last_busy(&pdev->dev);
 	pm_runtime_put_autosuspend(&pdev->dev);
diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c
index c33a5f7393bd..e3332a522a5d 100644
--- a/drivers/mmc/host/sdhci-of-arasan.c
+++ b/drivers/mmc/host/sdhci-of-arasan.c
@@ -290,7 +290,8 @@ static const struct sdhci_pltfm_data sdhci_arasan_pdata = {
 	.ops = &sdhci_arasan_ops,
 	.quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
 	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
-			SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN,
+			SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN |
+			SDHCI_QUIRK2_STOP_WITH_TC,
 };
 
 static u32 sdhci_arasan_cqhci_irq(struct sdhci_host *host, u32 intmask)
@@ -359,8 +360,7 @@ static const struct sdhci_pltfm_data sdhci_arasan_cqe_pdata = {
  */
 static int sdhci_arasan_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct sdhci_host *host = platform_get_drvdata(pdev);
+	struct sdhci_host *host = dev_get_drvdata(dev);
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 	struct sdhci_arasan_data *sdhci_arasan = sdhci_pltfm_priv(pltfm_host);
 	int ret;
@@ -403,8 +403,7 @@ static int sdhci_arasan_suspend(struct device *dev)
  */
 static int sdhci_arasan_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct sdhci_host *host = platform_get_drvdata(pdev);
+	struct sdhci_host *host = dev_get_drvdata(dev);
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 	struct sdhci_arasan_data *sdhci_arasan = sdhci_pltfm_priv(pltfm_host);
 	int ret;
diff --git a/drivers/mmc/host/sdhci-omap.c b/drivers/mmc/host/sdhci-omap.c
index 1456abd5eeb9..f3a7c8ece4be 100644
--- a/drivers/mmc/host/sdhci-omap.c
+++ b/drivers/mmc/host/sdhci-omap.c
@@ -26,6 +26,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/regulator/consumer.h>
 #include <linux/pinctrl/consumer.h>
+#include <linux/sys_soc.h>
 
 #include "sdhci-pltfm.h"
 
@@ -35,6 +36,7 @@
 #define CON_DDR			BIT(19)
 #define CON_CLKEXTFREE		BIT(16)
 #define CON_PADEN		BIT(15)
+#define CON_CTPL		BIT(11)
 #define CON_INIT		BIT(1)
 #define CON_OD			BIT(0)
 
@@ -100,6 +102,7 @@ struct sdhci_omap_data {
 };
 
 struct sdhci_omap_host {
+	char			*version;
 	void __iomem		*base;
 	struct device		*dev;
 	struct	regulator	*pbias;
@@ -224,6 +227,23 @@ static void sdhci_omap_conf_bus_power(struct sdhci_omap_host *omap_host,
 	}
 }
 
+static void sdhci_omap_enable_sdio_irq(struct mmc_host *mmc, int enable)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_omap_host *omap_host = sdhci_pltfm_priv(pltfm_host);
+	u32 reg;
+
+	reg = sdhci_omap_readl(omap_host, SDHCI_OMAP_CON);
+	if (enable)
+		reg |= (CON_CTPL | CON_CLKEXTFREE);
+	else
+		reg &= ~(CON_CTPL | CON_CLKEXTFREE);
+	sdhci_omap_writel(omap_host, SDHCI_OMAP_CON, reg);
+
+	sdhci_enable_sdio_irq(mmc, enable);
+}
+
 static inline void sdhci_omap_set_dll(struct sdhci_omap_host *omap_host,
 				      int count)
 {
@@ -713,10 +733,15 @@ static const struct sdhci_pltfm_data sdhci_omap_pdata = {
 		  SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC,
 	.quirks2 = SDHCI_QUIRK2_ACMD23_BROKEN |
 		   SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
-		   SDHCI_QUIRK2_RSP_136_HAS_CRC,
+		   SDHCI_QUIRK2_RSP_136_HAS_CRC |
+		   SDHCI_QUIRK2_DISABLE_HW_TIMEOUT,
 	.ops = &sdhci_omap_ops,
 };
 
+static const struct sdhci_omap_data k2g_data = {
+	.offset = 0x200,
+};
+
 static const struct sdhci_omap_data dra7_data = {
 	.offset = 0x200,
 	.flags	= SDHCI_OMAP_REQUIRE_IODELAY,
@@ -724,6 +749,7 @@ static const struct sdhci_omap_data dra7_data = {
 
 static const struct of_device_id omap_sdhci_match[] = {
 	{ .compatible = "ti,dra7-sdhci", .data = &dra7_data },
+	{ .compatible = "ti,k2g-sdhci", .data = &k2g_data },
 	{},
 };
 MODULE_DEVICE_TABLE(of, omap_sdhci_match);
@@ -733,12 +759,21 @@ static struct pinctrl_state
 				  u32 *caps, u32 capmask)
 {
 	struct device *dev = omap_host->dev;
+	char *version = omap_host->version;
 	struct pinctrl_state *pinctrl_state = ERR_PTR(-ENODEV);
+	char str[20];
 
 	if (!(*caps & capmask))
 		goto ret;
 
-	pinctrl_state = pinctrl_lookup_state(omap_host->pinctrl, mode);
+	if (version) {
+		snprintf(str, 20, "%s-%s", mode, version);
+		pinctrl_state = pinctrl_lookup_state(omap_host->pinctrl, str);
+	}
+
+	if (IS_ERR(pinctrl_state))
+		pinctrl_state = pinctrl_lookup_state(omap_host->pinctrl, mode);
+
 	if (IS_ERR(pinctrl_state)) {
 		dev_err(dev, "no pinctrl state for %s mode", mode);
 		*caps &= ~capmask;
@@ -807,8 +842,15 @@ static int sdhci_omap_config_iodelay_pinctrl_state(struct sdhci_omap_host
 
 	state = sdhci_omap_iodelay_pinctrl_state(omap_host, "ddr_1_8v", caps,
 						 MMC_CAP_1_8V_DDR);
-	if (!IS_ERR(state))
+	if (!IS_ERR(state)) {
 		pinctrl_state[MMC_TIMING_MMC_DDR52] = state;
+	} else {
+		state = sdhci_omap_iodelay_pinctrl_state(omap_host, "ddr_3_3v",
+							 caps,
+							 MMC_CAP_3_3V_DDR);
+		if (!IS_ERR(state))
+			pinctrl_state[MMC_TIMING_MMC_DDR52] = state;
+	}
 
 	state = sdhci_omap_iodelay_pinctrl_state(omap_host, "hs", caps,
 						 MMC_CAP_SD_HIGHSPEED);
@@ -830,6 +872,16 @@ static int sdhci_omap_config_iodelay_pinctrl_state(struct sdhci_omap_host
 	return 0;
 }
 
+static const struct soc_device_attribute sdhci_omap_soc_devices[] = {
+	{
+		.machine = "DRA7[45]*",
+		.revision = "ES1.[01]",
+	},
+	{
+		/* sentinel */
+	}
+};
+
 static int sdhci_omap_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -841,6 +893,7 @@ static int sdhci_omap_probe(struct platform_device *pdev)
 	struct mmc_host *mmc;
 	const struct of_device_id *match;
 	struct sdhci_omap_data *data;
+	const struct soc_device_attribute *soc;
 
 	match = of_match_device(omap_sdhci_match, dev);
 	if (!match)
@@ -871,10 +924,22 @@ static int sdhci_omap_probe(struct platform_device *pdev)
 	host->ioaddr += offset;
 
 	mmc = host->mmc;
+	sdhci_get_of_property(pdev);
 	ret = mmc_of_parse(mmc);
 	if (ret)
 		goto err_pltfm_free;
 
+	soc = soc_device_match(sdhci_omap_soc_devices);
+	if (soc) {
+		omap_host->version = "rev11";
+		if (!strcmp(dev_name(dev), "4809c000.mmc"))
+			mmc->f_max = 96000000;
+		if (!strcmp(dev_name(dev), "480b4000.mmc"))
+			mmc->f_max = 48000000;
+		if (!strcmp(dev_name(dev), "480ad000.mmc"))
+			mmc->f_max = 48000000;
+	}
+
 	pltfm_host->clk = devm_clk_get(dev, "fck");
 	if (IS_ERR(pltfm_host->clk)) {
 		ret = PTR_ERR(pltfm_host->clk);
@@ -916,26 +981,31 @@ static int sdhci_omap_probe(struct platform_device *pdev)
 		goto err_put_sync;
 	}
 
-	ret = sdhci_omap_config_iodelay_pinctrl_state(omap_host);
-	if (ret)
-		goto err_put_sync;
-
 	host->mmc_host_ops.get_ro = mmc_gpio_get_ro;
 	host->mmc_host_ops.start_signal_voltage_switch =
 					sdhci_omap_start_signal_voltage_switch;
 	host->mmc_host_ops.set_ios = sdhci_omap_set_ios;
 	host->mmc_host_ops.card_busy = sdhci_omap_card_busy;
 	host->mmc_host_ops.execute_tuning = sdhci_omap_execute_tuning;
+	host->mmc_host_ops.enable_sdio_irq = sdhci_omap_enable_sdio_irq;
 
-	sdhci_read_caps(host);
-	host->caps |= SDHCI_CAN_DO_ADMA2;
-
-	ret = sdhci_add_host(host);
+	ret = sdhci_setup_host(host);
 	if (ret)
 		goto err_put_sync;
 
+	ret = sdhci_omap_config_iodelay_pinctrl_state(omap_host);
+	if (ret)
+		goto err_cleanup_host;
+
+	ret = __sdhci_add_host(host);
+	if (ret)
+		goto err_cleanup_host;
+
 	return 0;
 
+err_cleanup_host:
+	sdhci_cleanup_host(host);
+
 err_put_sync:
 	pm_runtime_put_sync(dev);
 
diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c
index 78c25ad35fd2..77dd3521daae 100644
--- a/drivers/mmc/host/sdhci-pci-core.c
+++ b/drivers/mmc/host/sdhci-pci-core.c
@@ -453,6 +453,7 @@ static const struct sdhci_pci_fixes sdhci_intel_pch_sdio = {
 enum {
 	INTEL_DSM_FNS		=  0,
 	INTEL_DSM_V18_SWITCH	=  3,
+	INTEL_DSM_V33_SWITCH	=  4,
 	INTEL_DSM_DRV_STRENGTH	=  9,
 	INTEL_DSM_D3_RETUNE	= 10,
 };
@@ -620,17 +621,37 @@ static void intel_hs400_enhanced_strobe(struct mmc_host *mmc,
 	sdhci_writel(host, val, INTEL_HS400_ES_REG);
 }
 
-static void sdhci_intel_voltage_switch(struct sdhci_host *host)
+static int intel_start_signal_voltage_switch(struct mmc_host *mmc,
+					     struct mmc_ios *ios)
 {
+	struct device *dev = mmc_dev(mmc);
+	struct sdhci_host *host = mmc_priv(mmc);
 	struct sdhci_pci_slot *slot = sdhci_priv(host);
 	struct intel_host *intel_host = sdhci_pci_priv(slot);
-	struct device *dev = &slot->chip->pdev->dev;
+	unsigned int fn;
 	u32 result = 0;
 	int err;
 
-	err = intel_dsm(intel_host, dev, INTEL_DSM_V18_SWITCH, &result);
-	pr_debug("%s: %s DSM error %d result %u\n",
-		 mmc_hostname(host->mmc), __func__, err, result);
+	err = sdhci_start_signal_voltage_switch(mmc, ios);
+	if (err)
+		return err;
+
+	switch (ios->signal_voltage) {
+	case MMC_SIGNAL_VOLTAGE_330:
+		fn = INTEL_DSM_V33_SWITCH;
+		break;
+	case MMC_SIGNAL_VOLTAGE_180:
+		fn = INTEL_DSM_V18_SWITCH;
+		break;
+	default:
+		return 0;
+	}
+
+	err = intel_dsm(intel_host, dev, fn, &result);
+	pr_debug("%s: %s DSM fn %u error %d result %u\n",
+		 mmc_hostname(mmc), __func__, fn, err, result);
+
+	return 0;
 }
 
 static const struct sdhci_ops sdhci_intel_byt_ops = {
@@ -641,7 +662,6 @@ static const struct sdhci_ops sdhci_intel_byt_ops = {
 	.reset			= sdhci_reset,
 	.set_uhs_signaling	= sdhci_set_uhs_signaling,
 	.hw_reset		= sdhci_pci_hw_reset,
-	.voltage_switch		= sdhci_intel_voltage_switch,
 };
 
 static const struct sdhci_ops sdhci_intel_glk_ops = {
@@ -652,7 +672,6 @@ static const struct sdhci_ops sdhci_intel_glk_ops = {
 	.reset			= sdhci_reset,
 	.set_uhs_signaling	= sdhci_set_uhs_signaling,
 	.hw_reset		= sdhci_pci_hw_reset,
-	.voltage_switch		= sdhci_intel_voltage_switch,
 	.irq			= sdhci_cqhci_irq,
 };
 
@@ -691,6 +710,7 @@ static void byt_probe_slot(struct sdhci_pci_slot *slot)
 	byt_read_dsm(slot);
 
 	ops->execute_tuning = intel_execute_tuning;
+	ops->start_signal_voltage_switch = intel_start_signal_voltage_switch;
 }
 
 static int byt_emmc_probe_slot(struct sdhci_pci_slot *slot)
@@ -832,6 +852,10 @@ static int byt_sd_probe_slot(struct sdhci_pci_slot *slot)
 	    slot->chip->pdev->device == PCI_DEVICE_ID_INTEL_GLK_SD)
 		slot->host->mmc_host_ops.get_cd = bxt_get_cd;
 
+	if (slot->chip->pdev->subsystem_vendor == PCI_VENDOR_ID_NI &&
+	    slot->chip->pdev->subsystem_device == PCI_SUBDEVICE_ID_NI_78E3)
+		slot->host->mmc->caps2 |= MMC_CAP2_AVOID_3_3V;
+
 	return 0;
 }
 
diff --git a/drivers/mmc/host/sdhci-pci.h b/drivers/mmc/host/sdhci-pci.h
index 5cbcdc448f98..db9cb54ef700 100644
--- a/drivers/mmc/host/sdhci-pci.h
+++ b/drivers/mmc/host/sdhci-pci.h
@@ -54,6 +54,7 @@
 #define PCI_DEVICE_ID_REALTEK_5250	0x5250
 
 #define PCI_SUBDEVICE_ID_NI_7884	0x7884
+#define PCI_SUBDEVICE_ID_NI_78E3	0x78e3
 
 #define PCI_VENDOR_ID_ARASAN		0x16e6
 #define PCI_DEVICE_ID_ARASAN_PHY_EMMC	0x0670
diff --git a/drivers/mmc/host/sdhci-pic32.c b/drivers/mmc/host/sdhci-pic32.c
index a6caa49ca25a..a11e6397d4ff 100644
--- a/drivers/mmc/host/sdhci-pic32.c
+++ b/drivers/mmc/host/sdhci-pic32.c
@@ -200,10 +200,8 @@ static int pic32_sdhci_probe(struct platform_device *pdev)
 	}
 
 	ret = sdhci_add_host(host);
-	if (ret) {
-		dev_err(&pdev->dev, "error adding host\n");
+	if (ret)
 		goto err_base_clk;
-	}
 
 	dev_info(&pdev->dev, "Successfully added sdhci host\n");
 	return 0;
diff --git a/drivers/mmc/host/sdhci-pxav2.c b/drivers/mmc/host/sdhci-pxav2.c
index 8986f9d9cf98..2c3827f54927 100644
--- a/drivers/mmc/host/sdhci-pxav2.c
+++ b/drivers/mmc/host/sdhci-pxav2.c
@@ -221,10 +221,8 @@ static int sdhci_pxav2_probe(struct platform_device *pdev)
 	host->ops = &pxav2_sdhci_ops;
 
 	ret = sdhci_add_host(host);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to add host\n");
+	if (ret)
 		goto disable_clk;
-	}
 
 	return 0;
 
diff --git a/drivers/mmc/host/sdhci-pxav3.c b/drivers/mmc/host/sdhci-pxav3.c
index a34434166ca7..b8e96f392428 100644
--- a/drivers/mmc/host/sdhci-pxav3.c
+++ b/drivers/mmc/host/sdhci-pxav3.c
@@ -472,10 +472,8 @@ static int sdhci_pxav3_probe(struct platform_device *pdev)
 	pm_suspend_ignore_children(&pdev->dev, 1);
 
 	ret = sdhci_add_host(host);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to add host\n");
+	if (ret)
 		goto err_add_host;
-	}
 
 	if (host->mmc->pm_caps & MMC_PM_WAKE_SDIO_IRQ)
 		device_init_wakeup(&pdev->dev, 1);
diff --git a/drivers/mmc/host/sdhci-s3c.c b/drivers/mmc/host/sdhci-s3c.c
index cda83ccb2702..9ef89d00970e 100644
--- a/drivers/mmc/host/sdhci-s3c.c
+++ b/drivers/mmc/host/sdhci-s3c.c
@@ -655,10 +655,8 @@ static int sdhci_s3c_probe(struct platform_device *pdev)
 		goto err_req_regs;
 
 	ret = sdhci_add_host(host);
-	if (ret) {
-		dev_err(dev, "sdhci_add_host() failed\n");
+	if (ret)
 		goto err_req_regs;
-	}
 
 #ifdef CONFIG_PM
 	if (pdata->cd_type != S3C_SDHCI_CD_INTERNAL)
diff --git a/drivers/mmc/host/sdhci-spear.c b/drivers/mmc/host/sdhci-spear.c
index 14511526a3a8..9247d51f2eed 100644
--- a/drivers/mmc/host/sdhci-spear.c
+++ b/drivers/mmc/host/sdhci-spear.c
@@ -126,10 +126,8 @@ static int sdhci_probe(struct platform_device *pdev)
 	}
 
 	ret = sdhci_add_host(host);
-	if (ret) {
-		dev_dbg(&pdev->dev, "error adding host\n");
+	if (ret)
 		goto disable_clk;
-	}
 
 	platform_set_drvdata(pdev, host);
 
diff --git a/drivers/mmc/host/sdhci-st.c b/drivers/mmc/host/sdhci-st.c
index c32daed0d418..8f95647195d9 100644
--- a/drivers/mmc/host/sdhci-st.c
+++ b/drivers/mmc/host/sdhci-st.c
@@ -422,10 +422,8 @@ static int sdhci_st_probe(struct platform_device *pdev)
 	st_mmcss_cconfig(np, host);
 
 	ret = sdhci_add_host(host);
-	if (ret) {
-		dev_err(&pdev->dev, "Failed sdhci_add_host\n");
+	if (ret)
 		goto err_out;
-	}
 
 	host_version = readw_relaxed((host->ioaddr + SDHCI_HOST_VERSION));
 
diff --git a/drivers/mmc/host/sdhci-tegra.c b/drivers/mmc/host/sdhci-tegra.c
index b877c13184c2..970d38f68939 100644
--- a/drivers/mmc/host/sdhci-tegra.c
+++ b/drivers/mmc/host/sdhci-tegra.c
@@ -231,7 +231,7 @@ static void tegra_sdhci_set_uhs_signaling(struct sdhci_host *host,
 	if (timing == MMC_TIMING_UHS_DDR50)
 		tegra_host->ddr_signaling = true;
 
-	return sdhci_set_uhs_signaling(host, timing);
+	sdhci_set_uhs_signaling(host, timing);
 }
 
 static unsigned int tegra_sdhci_get_max_clock(struct sdhci_host *host)
diff --git a/drivers/mmc/host/sdhci-xenon-phy.c b/drivers/mmc/host/sdhci-xenon-phy.c
index ec8794335241..a35804b203a7 100644
--- a/drivers/mmc/host/sdhci-xenon-phy.c
+++ b/drivers/mmc/host/sdhci-xenon-phy.c
@@ -814,15 +814,10 @@ static int xenon_add_phy(struct device_node *np, struct sdhci_host *host,
 {
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 	struct xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
-	int i, ret;
+	int ret;
 
-	for (i = 0; i < NR_PHY_TYPES; i++) {
-		if (!strcmp(phy_name, phy_types[i])) {
-			priv->phy_type = i;
-			break;
-		}
-	}
-	if (i == NR_PHY_TYPES) {
+	priv->phy_type = match_string(phy_types, NR_PHY_TYPES, phy_name);
+	if (priv->phy_type < 0) {
 		dev_err(mmc_dev(host->mmc),
 			"Unable to determine PHY name %s. Use default eMMC 5.1 PHY\n",
 			phy_name);
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index 2ededa7f43df..1c828e0e9905 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -709,29 +709,16 @@ static u32 sdhci_sdma_address(struct sdhci_host *host)
 		return sg_dma_address(host->data->sg);
 }
 
-static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd)
+static unsigned int sdhci_target_timeout(struct sdhci_host *host,
+					 struct mmc_command *cmd,
+					 struct mmc_data *data)
 {
-	u8 count;
-	struct mmc_data *data = cmd->data;
-	unsigned target_timeout, current_timeout;
-
-	/*
-	 * If the host controller provides us with an incorrect timeout
-	 * value, just skip the check and use 0xE.  The hardware may take
-	 * longer to time out, but that's much better than having a too-short
-	 * timeout value.
-	 */
-	if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)
-		return 0xE;
-
-	/* Unspecified timeout, assume max */
-	if (!data && !cmd->busy_timeout)
-		return 0xE;
+	unsigned int target_timeout;
 
 	/* timeout in us */
-	if (!data)
+	if (!data) {
 		target_timeout = cmd->busy_timeout * 1000;
-	else {
+	} else {
 		target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000);
 		if (host->clock && data->timeout_clks) {
 			unsigned long long val;
@@ -748,6 +735,67 @@ static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 		}
 	}
 
+	return target_timeout;
+}
+
+static void sdhci_calc_sw_timeout(struct sdhci_host *host,
+				  struct mmc_command *cmd)
+{
+	struct mmc_data *data = cmd->data;
+	struct mmc_host *mmc = host->mmc;
+	struct mmc_ios *ios = &mmc->ios;
+	unsigned char bus_width = 1 << ios->bus_width;
+	unsigned int blksz;
+	unsigned int freq;
+	u64 target_timeout;
+	u64 transfer_time;
+
+	target_timeout = sdhci_target_timeout(host, cmd, data);
+	target_timeout *= NSEC_PER_USEC;
+
+	if (data) {
+		blksz = data->blksz;
+		freq = host->mmc->actual_clock ? : host->clock;
+		transfer_time = (u64)blksz * NSEC_PER_SEC * (8 / bus_width);
+		do_div(transfer_time, freq);
+		/* multiply by '2' to account for any unknowns */
+		transfer_time = transfer_time * 2;
+		/* calculate timeout for the entire data */
+		host->data_timeout = data->blocks * target_timeout +
+				     transfer_time;
+	} else {
+		host->data_timeout = target_timeout;
+	}
+
+	if (host->data_timeout)
+		host->data_timeout += MMC_CMD_TRANSFER_TIME;
+}
+
+static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd,
+			     bool *too_big)
+{
+	u8 count;
+	struct mmc_data *data = cmd->data;
+	unsigned target_timeout, current_timeout;
+
+	*too_big = true;
+
+	/*
+	 * If the host controller provides us with an incorrect timeout
+	 * value, just skip the check and use 0xE.  The hardware may take
+	 * longer to time out, but that's much better than having a too-short
+	 * timeout value.
+	 */
+	if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)
+		return 0xE;
+
+	/* Unspecified timeout, assume max */
+	if (!data && !cmd->busy_timeout)
+		return 0xE;
+
+	/* timeout in us */
+	target_timeout = sdhci_target_timeout(host, cmd, data);
+
 	/*
 	 * Figure out needed cycles.
 	 * We do this in steps in order to fit inside a 32 bit int.
@@ -768,9 +816,12 @@ static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 	}
 
 	if (count >= 0xF) {
-		DBG("Too large timeout 0x%x requested for CMD%d!\n",
-		    count, cmd->opcode);
+		if (!(host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT))
+			DBG("Too large timeout 0x%x requested for CMD%d!\n",
+			    count, cmd->opcode);
 		count = 0xE;
+	} else {
+		*too_big = false;
 	}
 
 	return count;
@@ -790,6 +841,16 @@ static void sdhci_set_transfer_irqs(struct sdhci_host *host)
 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 }
 
+static void sdhci_set_data_timeout_irq(struct sdhci_host *host, bool enable)
+{
+	if (enable)
+		host->ier |= SDHCI_INT_DATA_TIMEOUT;
+	else
+		host->ier &= ~SDHCI_INT_DATA_TIMEOUT;
+	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
+	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
+}
+
 static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 {
 	u8 count;
@@ -797,7 +858,18 @@ static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 	if (host->ops->set_timeout) {
 		host->ops->set_timeout(host, cmd);
 	} else {
-		count = sdhci_calc_timeout(host, cmd);
+		bool too_big = false;
+
+		count = sdhci_calc_timeout(host, cmd, &too_big);
+
+		if (too_big &&
+		    host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT) {
+			sdhci_calc_sw_timeout(host, cmd);
+			sdhci_set_data_timeout_irq(host, false);
+		} else if (!(host->ier & SDHCI_INT_DATA_TIMEOUT)) {
+			sdhci_set_data_timeout_irq(host, true);
+		}
+
 		sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL);
 	}
 }
@@ -807,6 +879,8 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
 	u8 ctrl;
 	struct mmc_data *data = cmd->data;
 
+	host->data_timeout = 0;
+
 	if (sdhci_data_line_cmd(cmd))
 		sdhci_set_timeout(host, cmd);
 
@@ -1160,13 +1234,6 @@ void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)
 		mdelay(1);
 	}
 
-	timeout = jiffies;
-	if (!cmd->data && cmd->busy_timeout > 9000)
-		timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
-	else
-		timeout += 10 * HZ;
-	sdhci_mod_timer(host, cmd->mrq, timeout);
-
 	host->cmd = cmd;
 	if (sdhci_data_line_cmd(cmd)) {
 		WARN_ON(host->data_cmd);
@@ -1206,6 +1273,15 @@ void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)
 	    cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200)
 		flags |= SDHCI_CMD_DATA;
 
+	timeout = jiffies;
+	if (host->data_timeout)
+		timeout += nsecs_to_jiffies(host->data_timeout);
+	else if (!cmd->data && cmd->busy_timeout > 9000)
+		timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
+	else
+		timeout += 10 * HZ;
+	sdhci_mod_timer(host, cmd->mrq, timeout);
+
 	sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND);
 }
 EXPORT_SYMBOL_GPL(sdhci_send_command);
@@ -3616,6 +3692,10 @@ int sdhci_setup_host(struct sdhci_host *host)
 		mmc->max_busy_timeout /= host->timeout_clk;
 	}
 
+	if (host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT &&
+	    !host->ops->get_max_timeout_count)
+		mmc->max_busy_timeout = 0;
+
 	mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_ERASE | MMC_CAP_CMD23;
 	mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
 
@@ -3672,6 +3752,16 @@ int sdhci_setup_host(struct sdhci_host *host)
 	if (host->quirks2 & SDHCI_QUIRK2_NO_1_8_V) {
 		host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
 				 SDHCI_SUPPORT_DDR50);
+		/*
+		 * The SDHCI controller in a SoC might support HS200/HS400
+		 * (indicated using mmc-hs200-1_8v/mmc-hs400-1_8v dt property),
+		 * but if the board is modeled such that the IO lines are not
+		 * connected to 1.8v then HS200/HS400 cannot be supported.
+		 * Disable HS200/HS400 if the board does not have 1.8v connected
+		 * to the IO lines. (Applicable for other modes in 1.8v)
+		 */
+		mmc->caps2 &= ~(MMC_CAP2_HSX00_1_8V | MMC_CAP2_HS400_ES);
+		mmc->caps &= ~(MMC_CAP_1_8V_DDR | MMC_CAP_UHS);
 	}
 
 	/* Any UHS-I mode in caps implies SDR12 and SDR25 support. */
diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
index c95b0a4a7594..23966f887da6 100644
--- a/drivers/mmc/host/sdhci.h
+++ b/drivers/mmc/host/sdhci.h
@@ -332,6 +332,14 @@ struct sdhci_adma2_64_desc {
 /* Allow for a a command request and a data request at the same time */
 #define SDHCI_MAX_MRQS		2
 
+/*
+ * 48bit command and 136 bit response in 100KHz clock could take upto 2.48ms.
+ * However since the start time of the command, the time between
+ * command and response, and the time between response and start of data is
+ * not known, set the command transfer time to 10ms.
+ */
+#define MMC_CMD_TRANSFER_TIME	(10 * NSEC_PER_MSEC) /* max 10 ms */
+
 enum sdhci_cookie {
 	COOKIE_UNMAPPED,
 	COOKIE_PRE_MAPPED,	/* mapped by sdhci_pre_req() */
@@ -437,6 +445,11 @@ struct sdhci_host {
 #define SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN		(1<<15)
 /* Controller has CRC in 136 bit Command Response */
 #define SDHCI_QUIRK2_RSP_136_HAS_CRC			(1<<16)
+/*
+ * Disable HW timeout if the requested timeout is more than the maximum
+ * obtainable timeout.
+ */
+#define SDHCI_QUIRK2_DISABLE_HW_TIMEOUT			(1<<17)
 
 	int irq;		/* Device IRQ */
 	void __iomem *ioaddr;	/* Mapped address */
@@ -550,6 +563,8 @@ struct sdhci_host {
 	/* Host SDMA buffer boundary. */
 	u32			sdma_boundary;
 
+	u64			data_timeout;
+
 	unsigned long private[0] ____cacheline_aligned;
 };
 
diff --git a/drivers/mmc/host/sunxi-mmc.c b/drivers/mmc/host/sunxi-mmc.c
index 20cfb20418f3..e7472590f2ed 100644
--- a/drivers/mmc/host/sunxi-mmc.c
+++ b/drivers/mmc/host/sunxi-mmc.c
@@ -13,36 +13,34 @@
  * the License, or (at your option) any later version.
  */
 
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/io.h>
-#include <linux/device.h>
-#include <linux/interrupt.h>
-#include <linux/delay.h>
-#include <linux/err.h>
-
 #include <linux/clk.h>
 #include <linux/clk/sunxi-ng.h>
-#include <linux/gpio.h>
-#include <linux/platform_device.h>
-#include <linux/spinlock.h>
-#include <linux/scatterlist.h>
+#include <linux/delay.h>
+#include <linux/device.h>
 #include <linux/dma-mapping.h>
-#include <linux/slab.h>
-#include <linux/reset.h>
-#include <linux/regulator/consumer.h>
-
-#include <linux/of_address.h>
-#include <linux/of_gpio.h>
-#include <linux/of_platform.h>
-
+#include <linux/err.h>
+#include <linux/gpio.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mmc/card.h>
+#include <linux/mmc/core.h>
 #include <linux/mmc/host.h>
+#include <linux/mmc/mmc.h>
 #include <linux/mmc/sd.h>
 #include <linux/mmc/sdio.h>
-#include <linux/mmc/mmc.h>
-#include <linux/mmc/core.h>
-#include <linux/mmc/card.h>
 #include <linux/mmc/slot-gpio.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_gpio.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
 
 /* register offset definitions */
 #define SDXC_REG_GCTRL	(0x00) /* SMC Global Control Register */
@@ -322,10 +320,9 @@ static int sunxi_mmc_reset_host(struct sunxi_mmc_host *host)
 	return 0;
 }
 
-static int sunxi_mmc_init_host(struct mmc_host *mmc)
+static int sunxi_mmc_init_host(struct sunxi_mmc_host *host)
 {
 	u32 rval;
-	struct sunxi_mmc_host *host = mmc_priv(mmc);
 
 	if (sunxi_mmc_reset_host(host))
 		return -EIO;
@@ -859,17 +856,48 @@ static int sunxi_mmc_clk_set_rate(struct sunxi_mmc_host *host,
 	return 0;
 }
 
-static void sunxi_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
+static void sunxi_mmc_set_bus_width(struct sunxi_mmc_host *host,
+				   unsigned char width)
+{
+	switch (width) {
+	case MMC_BUS_WIDTH_1:
+		mmc_writel(host, REG_WIDTH, SDXC_WIDTH1);
+		break;
+	case MMC_BUS_WIDTH_4:
+		mmc_writel(host, REG_WIDTH, SDXC_WIDTH4);
+		break;
+	case MMC_BUS_WIDTH_8:
+		mmc_writel(host, REG_WIDTH, SDXC_WIDTH8);
+		break;
+	}
+}
+
+static void sunxi_mmc_set_clk(struct sunxi_mmc_host *host, struct mmc_ios *ios)
 {
-	struct sunxi_mmc_host *host = mmc_priv(mmc);
 	u32 rval;
 
-	/* Set the power state */
-	switch (ios->power_mode) {
-	case MMC_POWER_ON:
-		break;
+	/* set ddr mode */
+	rval = mmc_readl(host, REG_GCTRL);
+	if (ios->timing == MMC_TIMING_UHS_DDR50 ||
+	    ios->timing == MMC_TIMING_MMC_DDR52)
+		rval |= SDXC_DDR_MODE;
+	else
+		rval &= ~SDXC_DDR_MODE;
+	mmc_writel(host, REG_GCTRL, rval);
+
+	host->ferror = sunxi_mmc_clk_set_rate(host, ios);
+	/* Android code had a usleep_range(50000, 55000); here */
+}
 
+static void sunxi_mmc_card_power(struct sunxi_mmc_host *host,
+				 struct mmc_ios *ios)
+{
+	struct mmc_host *mmc = host->mmc;
+
+	switch (ios->power_mode) {
 	case MMC_POWER_UP:
+		dev_dbg(mmc_dev(mmc), "Powering card up\n");
+
 		if (!IS_ERR(mmc->supply.vmmc)) {
 			host->ferror = mmc_regulator_set_ocr(mmc,
 							     mmc->supply.vmmc,
@@ -887,53 +915,33 @@ static void sunxi_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 			}
 			host->vqmmc_enabled = true;
 		}
-
-		host->ferror = sunxi_mmc_init_host(mmc);
-		if (host->ferror)
-			return;
-
-		dev_dbg(mmc_dev(mmc), "power on!\n");
 		break;
 
 	case MMC_POWER_OFF:
-		dev_dbg(mmc_dev(mmc), "power off!\n");
-		sunxi_mmc_reset_host(host);
+		dev_dbg(mmc_dev(mmc), "Powering card off\n");
+
 		if (!IS_ERR(mmc->supply.vmmc))
 			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
 
 		if (!IS_ERR(mmc->supply.vqmmc) && host->vqmmc_enabled)
 			regulator_disable(mmc->supply.vqmmc);
+
 		host->vqmmc_enabled = false;
 		break;
-	}
 
-	/* set bus width */
-	switch (ios->bus_width) {
-	case MMC_BUS_WIDTH_1:
-		mmc_writel(host, REG_WIDTH, SDXC_WIDTH1);
-		break;
-	case MMC_BUS_WIDTH_4:
-		mmc_writel(host, REG_WIDTH, SDXC_WIDTH4);
-		break;
-	case MMC_BUS_WIDTH_8:
-		mmc_writel(host, REG_WIDTH, SDXC_WIDTH8);
+	default:
+		dev_dbg(mmc_dev(mmc), "Ignoring unknown card power state\n");
 		break;
 	}
+}
 
-	/* set ddr mode */
-	rval = mmc_readl(host, REG_GCTRL);
-	if (ios->timing == MMC_TIMING_UHS_DDR50 ||
-	    ios->timing == MMC_TIMING_MMC_DDR52)
-		rval |= SDXC_DDR_MODE;
-	else
-		rval &= ~SDXC_DDR_MODE;
-	mmc_writel(host, REG_GCTRL, rval);
+static void sunxi_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
+{
+	struct sunxi_mmc_host *host = mmc_priv(mmc);
 
-	/* set up clock */
-	if (ios->power_mode) {
-		host->ferror = sunxi_mmc_clk_set_rate(host, ios);
-		/* Android code had a usleep_range(50000, 55000); here */
-	}
+	sunxi_mmc_card_power(host, ios);
+	sunxi_mmc_set_bus_width(host, ios->bus_width);
+	sunxi_mmc_set_clk(host, ios);
 }
 
 static int sunxi_mmc_volt_switch(struct mmc_host *mmc, struct mmc_ios *ios)
@@ -955,6 +963,9 @@ static void sunxi_mmc_enable_sdio_irq(struct mmc_host *mmc, int enable)
 	unsigned long flags;
 	u32 imask;
 
+	if (enable)
+		pm_runtime_get_noresume(host->dev);
+
 	spin_lock_irqsave(&host->lock, flags);
 
 	imask = mmc_readl(host, REG_IMASK);
@@ -967,6 +978,9 @@ static void sunxi_mmc_enable_sdio_irq(struct mmc_host *mmc, int enable)
 	}
 	mmc_writel(host, REG_IMASK, imask);
 	spin_unlock_irqrestore(&host->lock, flags);
+
+	if (!enable)
+		pm_runtime_put_noidle(host->mmc->parent);
 }
 
 static void sunxi_mmc_hw_reset(struct mmc_host *mmc)
@@ -1380,6 +1394,15 @@ static int sunxi_mmc_probe(struct platform_device *pdev)
 	if (ret)
 		goto error_free_dma;
 
+	ret = sunxi_mmc_init_host(host);
+	if (ret)
+		goto error_free_dma;
+
+	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_set_autosuspend_delay(&pdev->dev, 50);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
+
 	ret = mmc_add_host(mmc);
 	if (ret)
 		goto error_free_dma;
@@ -1400,6 +1423,7 @@ static int sunxi_mmc_remove(struct platform_device *pdev)
 	struct sunxi_mmc_host *host = mmc_priv(mmc);
 
 	mmc_remove_host(mmc);
+	pm_runtime_force_suspend(&pdev->dev);
 	disable_irq(host->irq);
 	sunxi_mmc_disable(host);
 	dma_free_coherent(&pdev->dev, PAGE_SIZE, host->sg_cpu, host->sg_dma);
@@ -1408,10 +1432,47 @@ static int sunxi_mmc_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int sunxi_mmc_runtime_resume(struct device *dev)
+{
+	struct mmc_host	*mmc = dev_get_drvdata(dev);
+	struct sunxi_mmc_host *host = mmc_priv(mmc);
+	int ret;
+
+	ret = sunxi_mmc_enable(host);
+	if (ret)
+		return ret;
+
+	sunxi_mmc_init_host(host);
+	sunxi_mmc_set_bus_width(host, mmc->ios.bus_width);
+	sunxi_mmc_set_clk(host, &mmc->ios);
+
+	return 0;
+}
+
+static int sunxi_mmc_runtime_suspend(struct device *dev)
+{
+	struct mmc_host	*mmc = dev_get_drvdata(dev);
+	struct sunxi_mmc_host *host = mmc_priv(mmc);
+
+	sunxi_mmc_reset_host(host);
+	sunxi_mmc_disable(host);
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops sunxi_mmc_pm_ops = {
+	SET_RUNTIME_PM_OPS(sunxi_mmc_runtime_suspend,
+			   sunxi_mmc_runtime_resume,
+			   NULL)
+};
+
 static struct platform_driver sunxi_mmc_driver = {
 	.driver = {
 		.name	= "sunxi-mmc",
 		.of_match_table = of_match_ptr(sunxi_mmc_of_match),
+		.pm = &sunxi_mmc_pm_ops,
 	},
 	.probe		= sunxi_mmc_probe,
 	.remove		= sunxi_mmc_remove,
diff --git a/drivers/mmc/host/ushc.c b/drivers/mmc/host/ushc.c
index 81dac17064d7..b2b379b10dfa 100644
--- a/drivers/mmc/host/ushc.c
+++ b/drivers/mmc/host/ushc.c
@@ -300,8 +300,10 @@ static void ushc_request(struct mmc_host *mmc, struct mmc_request *req)
 			pipe = usb_sndbulkpipe(ushc->usb_dev, 2);
 
 		usb_fill_bulk_urb(ushc->data_urb, ushc->usb_dev, pipe,
-				  sg_virt(data->sg), data->sg->length,
+				  NULL, data->sg->length,
 				  data_callback, ushc);
+		ushc->data_urb->num_sgs = 1;
+		ushc->data_urb->sg = data->sg;
 		ret = usb_submit_urb(ushc->data_urb, GFP_ATOMIC);
 		if (ret < 0)
 			goto out;
diff --git a/drivers/mmc/host/wbsd.c b/drivers/mmc/host/wbsd.c
index f4233576153b..1e54bbf13d75 100644
--- a/drivers/mmc/host/wbsd.c
+++ b/drivers/mmc/host/wbsd.c
@@ -268,43 +268,29 @@ static inline int wbsd_next_sg(struct wbsd_host *host)
 	return host->num_sg;
 }
 
-static inline char *wbsd_sg_to_buffer(struct wbsd_host *host)
+static inline char *wbsd_map_sg(struct wbsd_host *host)
 {
-	return sg_virt(host->cur_sg);
+	return kmap_atomic(sg_page(host->cur_sg)) + host->cur_sg->offset;
 }
 
 static inline void wbsd_sg_to_dma(struct wbsd_host *host, struct mmc_data *data)
 {
-	unsigned int len, i;
-	struct scatterlist *sg;
-	char *dmabuf = host->dma_buffer;
-	char *sgbuf;
-
-	sg = data->sg;
-	len = data->sg_len;
-
-	for (i = 0; i < len; i++) {
-		sgbuf = sg_virt(&sg[i]);
-		memcpy(dmabuf, sgbuf, sg[i].length);
-		dmabuf += sg[i].length;
-	}
+	size_t len = 0;
+	int i;
+
+	for (i = 0; i < data->sg_len; i++)
+		len += data->sg[i].length;
+	sg_copy_to_buffer(data->sg, data->sg_len, host->dma_buffer, len);
 }
 
 static inline void wbsd_dma_to_sg(struct wbsd_host *host, struct mmc_data *data)
 {
-	unsigned int len, i;
-	struct scatterlist *sg;
-	char *dmabuf = host->dma_buffer;
-	char *sgbuf;
-
-	sg = data->sg;
-	len = data->sg_len;
-
-	for (i = 0; i < len; i++) {
-		sgbuf = sg_virt(&sg[i]);
-		memcpy(sgbuf, dmabuf, sg[i].length);
-		dmabuf += sg[i].length;
-	}
+	size_t len = 0;
+	int i;
+
+	for (i = 0; i < data->sg_len; i++)
+		len += data->sg[i].length;
+	sg_copy_from_buffer(data->sg, data->sg_len, host->dma_buffer, len);
 }
 
 /*
@@ -418,7 +404,7 @@ static void wbsd_empty_fifo(struct wbsd_host *host)
 {
 	struct mmc_data *data = host->mrq->cmd->data;
 	char *buffer;
-	int i, fsr, fifo;
+	int i, idx, fsr, fifo;
 
 	/*
 	 * Handle excessive data.
@@ -426,7 +412,8 @@ static void wbsd_empty_fifo(struct wbsd_host *host)
 	if (host->num_sg == 0)
 		return;
 
-	buffer = wbsd_sg_to_buffer(host) + host->offset;
+	buffer = wbsd_map_sg(host) + host->offset;
+	idx = 0;
 
 	/*
 	 * Drain the fifo. This has a tendency to loop longer
@@ -445,8 +432,7 @@ static void wbsd_empty_fifo(struct wbsd_host *host)
 			fifo = 1;
 
 		for (i = 0; i < fifo; i++) {
-			*buffer = inb(host->base + WBSD_DFR);
-			buffer++;
+			buffer[idx++] = inb(host->base + WBSD_DFR);
 			host->offset++;
 			host->remain--;
 
@@ -456,16 +442,19 @@ static void wbsd_empty_fifo(struct wbsd_host *host)
 			 * End of scatter list entry?
 			 */
 			if (host->remain == 0) {
+				kunmap_atomic(buffer);
 				/*
 				 * Get next entry. Check if last.
 				 */
 				if (!wbsd_next_sg(host))
 					return;
 
-				buffer = wbsd_sg_to_buffer(host);
+				buffer = wbsd_map_sg(host);
+				idx = 0;
 			}
 		}
 	}
+	kunmap_atomic(buffer);
 
 	/*
 	 * This is a very dirty hack to solve a
@@ -480,7 +469,7 @@ static void wbsd_fill_fifo(struct wbsd_host *host)
 {
 	struct mmc_data *data = host->mrq->cmd->data;
 	char *buffer;
-	int i, fsr, fifo;
+	int i, idx, fsr, fifo;
 
 	/*
 	 * Check that we aren't being called after the
@@ -489,7 +478,8 @@ static void wbsd_fill_fifo(struct wbsd_host *host)
 	if (host->num_sg == 0)
 		return;
 
-	buffer = wbsd_sg_to_buffer(host) + host->offset;
+	buffer = wbsd_map_sg(host) + host->offset;
+	idx = 0;
 
 	/*
 	 * Fill the fifo. This has a tendency to loop longer
@@ -508,8 +498,7 @@ static void wbsd_fill_fifo(struct wbsd_host *host)
 			fifo = 15;
 
 		for (i = 16; i > fifo; i--) {
-			outb(*buffer, host->base + WBSD_DFR);
-			buffer++;
+			outb(buffer[idx], host->base + WBSD_DFR);
 			host->offset++;
 			host->remain--;
 
@@ -519,16 +508,19 @@ static void wbsd_fill_fifo(struct wbsd_host *host)
 			 * End of scatter list entry?
 			 */
 			if (host->remain == 0) {
+				kunmap_atomic(buffer);
 				/*
 				 * Get next entry. Check if last.
 				 */
 				if (!wbsd_next_sg(host))
 					return;
 
-				buffer = wbsd_sg_to_buffer(host);
+				buffer = wbsd_map_sg(host);
+				idx = 0;
 			}
 		}
 	}
+	kunmap_atomic(buffer);
 
 	/*
 	 * The controller stops sending interrupts for
diff --git a/drivers/mmc/host/wmt-sdmmc.c b/drivers/mmc/host/wmt-sdmmc.c
index fd30ac7da5e5..3ba42f508014 100644
--- a/drivers/mmc/host/wmt-sdmmc.c
+++ b/drivers/mmc/host/wmt-sdmmc.c
@@ -928,8 +928,7 @@ static int wmt_mci_remove(struct platform_device *pdev)
 static int wmt_mci_suspend(struct device *dev)
 {
 	u32 reg_tmp;
-	struct platform_device *pdev = to_platform_device(dev);
-	struct mmc_host *mmc = platform_get_drvdata(pdev);
+	struct mmc_host *mmc = dev_get_drvdata(dev);
 	struct wmt_mci_priv *priv;
 
 	if (!mmc)
@@ -953,8 +952,7 @@ static int wmt_mci_suspend(struct device *dev)
 static int wmt_mci_resume(struct device *dev)
 {
 	u32 reg_tmp;
-	struct platform_device *pdev = to_platform_device(dev);
-	struct mmc_host *mmc = platform_get_drvdata(pdev);
+	struct mmc_host *mmc = dev_get_drvdata(dev);
 	struct wmt_mci_priv *priv;
 
 	if (mmc) {
diff --git a/drivers/mtd/bcm47xxpart.c b/drivers/mtd/bcm47xxpart.c
index fe2581d9d882..0f93d2239352 100644
--- a/drivers/mtd/bcm47xxpart.c
+++ b/drivers/mtd/bcm47xxpart.c
@@ -186,6 +186,8 @@ static int bcm47xxpart_parse(struct mtd_info *master,
 		/* TRX */
 		if (buf[0x000 / 4] == TRX_MAGIC) {
 			struct trx_header *trx;
+			uint32_t last_subpart;
+			uint32_t trx_size;
 
 			if (trx_num >= ARRAY_SIZE(trx_parts))
 				pr_warn("No enough space to store another TRX found at 0x%X\n",
@@ -195,11 +197,23 @@ static int bcm47xxpart_parse(struct mtd_info *master,
 			bcm47xxpart_add_part(&parts[curr_part++], "firmware",
 					     offset, 0);
 
-			/* Jump to the end of TRX */
+			/*
+			 * Try to find TRX size. The "length" field isn't fully
+			 * reliable as it could be decreased to make CRC32 cover
+			 * only part of TRX data. It's commonly used as checksum
+			 * can't cover e.g. ever-changing rootfs partition.
+			 * Use offsets as helpers for assuming min TRX size.
+			 */
 			trx = (struct trx_header *)buf;
-			offset = roundup(offset + trx->length, blocksize);
-			/* Next loop iteration will increase the offset */
-			offset -= blocksize;
+			last_subpart = max3(trx->offset[0], trx->offset[1],
+					    trx->offset[2]);
+			trx_size = max(trx->length, last_subpart + blocksize);
+
+			/*
+			 * Skip the TRX data. Decrease offset by block size as
+			 * the next loop iteration will increase it.
+			 */
+			offset += roundup(trx_size, blocksize) - blocksize;
 			continue;
 		}
 
@@ -290,9 +304,16 @@ static int bcm47xxpart_parse(struct mtd_info *master,
 	return curr_part;
 };
 
+static const struct of_device_id bcm47xxpart_of_match_table[] = {
+	{ .compatible = "brcm,bcm947xx-cfe-partitions" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, bcm47xxpart_of_match_table);
+
 static struct mtd_part_parser bcm47xxpart_mtd_parser = {
 	.parse_fn = bcm47xxpart_parse,
 	.name = "bcm47xxpart",
+	.of_match_table = bcm47xxpart_of_match_table,
 };
 module_mtd_part_parser(bcm47xxpart_mtd_parser);
 
diff --git a/drivers/mtd/chips/cfi_cmdset_0002.c b/drivers/mtd/chips/cfi_cmdset_0002.c
index 692902df2598..377dc122ed14 100644
--- a/drivers/mtd/chips/cfi_cmdset_0002.c
+++ b/drivers/mtd/chips/cfi_cmdset_0002.c
@@ -44,8 +44,8 @@
 
 #define MAX_WORD_RETRIES 3
 
-#define SST49LF004B	        0x0060
-#define SST49LF040B	        0x0050
+#define SST49LF004B		0x0060
+#define SST49LF040B		0x0050
 #define SST49LF008A		0x005a
 #define AT49BV6416		0x00d6
 
@@ -207,7 +207,7 @@ static void fixup_use_write_buffers(struct mtd_info *mtd)
 	struct map_info *map = mtd->priv;
 	struct cfi_private *cfi = map->fldrv_priv;
 	if (cfi->cfiq->BufWriteTimeoutTyp) {
-		pr_debug("Using buffer write method\n" );
+		pr_debug("Using buffer write method\n");
 		mtd->_write = cfi_amdstd_write_buffers;
 	}
 }
@@ -1563,7 +1563,7 @@ static int __xipram do_write_oneword(struct map_info *map, struct flchip *chip,
 	 * depending of the conditions.	 The ' + 1' is to avoid having a
 	 * timeout of 0 jiffies if HZ is smaller than 1000.
 	 */
-	unsigned long uWriteTimeout = ( HZ / 1000 ) + 1;
+	unsigned long uWriteTimeout = (HZ / 1000) + 1;
 	int ret = 0;
 	map_word oldd;
 	int retry_cnt = 0;
@@ -1578,7 +1578,7 @@ static int __xipram do_write_oneword(struct map_info *map, struct flchip *chip,
 	}
 
 	pr_debug("MTD %s(): WRITE 0x%.8lx(0x%.8lx)\n",
-	       __func__, adr, datum.x[0] );
+		 __func__, adr, datum.x[0]);
 
 	if (mode == FL_OTP_WRITE)
 		otp_enter(map, chip, adr, map_bankwidth(map));
@@ -1644,7 +1644,7 @@ static int __xipram do_write_oneword(struct map_info *map, struct flchip *chip,
 	/* Did we succeed? */
 	if (!chip_good(map, adr, datum)) {
 		/* reset on all failures. */
-		map_write( map, CMD(0xF0), chip->start );
+		map_write(map, CMD(0xF0), chip->start);
 		/* FIXME - should have reset delay before continuing */
 
 		if (++retry_cnt <= MAX_WORD_RETRIES)
@@ -1822,7 +1822,7 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip,
 	datum = map_word_load(map, buf);
 
 	pr_debug("MTD %s(): WRITE 0x%.8lx(0x%.8lx)\n",
-	       __func__, adr, datum.x[0] );
+		 __func__, adr, datum.x[0]);
 
 	XIP_INVAL_CACHED_RANGE(map, adr, len);
 	ENABLE_VPP(map);
@@ -2252,7 +2252,7 @@ static int __xipram do_erase_chip(struct map_info *map, struct flchip *chip)
 	}
 
 	pr_debug("MTD %s(): ERASE 0x%.8lx\n",
-	       __func__, chip->start );
+	       __func__, chip->start);
 
 	XIP_INVAL_CACHED_RANGE(map, adr, map->size);
 	ENABLE_VPP(map);
@@ -2299,7 +2299,7 @@ static int __xipram do_erase_chip(struct map_info *map, struct flchip *chip)
 
 		if (time_after(jiffies, timeo)) {
 			printk(KERN_WARNING "MTD %s(): software timeout\n",
-				__func__ );
+			       __func__);
 			break;
 		}
 
@@ -2309,7 +2309,7 @@ static int __xipram do_erase_chip(struct map_info *map, struct flchip *chip)
 	/* Did we succeed? */
 	if (!chip_good(map, adr, map_word_ff(map))) {
 		/* reset on all failures. */
-		map_write( map, CMD(0xF0), chip->start );
+		map_write(map, CMD(0xF0), chip->start);
 		/* FIXME - should have reset delay before continuing */
 
 		ret = -EIO;
@@ -2342,7 +2342,7 @@ static int __xipram do_erase_oneblock(struct map_info *map, struct flchip *chip,
 	}
 
 	pr_debug("MTD %s(): ERASE 0x%.8lx\n",
-	       __func__, adr );
+		 __func__, adr);
 
 	XIP_INVAL_CACHED_RANGE(map, adr, len);
 	ENABLE_VPP(map);
@@ -2392,7 +2392,7 @@ static int __xipram do_erase_oneblock(struct map_info *map, struct flchip *chip,
 		if (time_after(jiffies, timeo)) {
 			xip_enable(map, chip, adr);
 			printk(KERN_WARNING "MTD %s(): software timeout\n",
-				__func__ );
+			       __func__);
 			break;
 		}
 
@@ -2402,7 +2402,7 @@ static int __xipram do_erase_oneblock(struct map_info *map, struct flchip *chip,
 	/* Did we succeed? */
 	if (!chip_good(map, adr, map_word_ff(map))) {
 		/* reset on all failures. */
-		map_write( map, CMD(0xF0), chip->start );
+		map_write(map, CMD(0xF0), chip->start);
 		/* FIXME - should have reset delay before continuing */
 
 		ret = -EIO;
diff --git a/drivers/mtd/chips/cfi_probe.c b/drivers/mtd/chips/cfi_probe.c
index e8d0164498b0..cf426956454c 100644
--- a/drivers/mtd/chips/cfi_probe.c
+++ b/drivers/mtd/chips/cfi_probe.c
@@ -63,6 +63,30 @@ do { \
 
 #endif
 
+/*
+ * This fixup occurs immediately after reading the CFI structure and can affect
+ * the number of chips detected, unlike cfi_fixup, which occurs after an
+ * mtd_info structure has been created for the chip.
+ */
+struct cfi_early_fixup {
+	uint16_t mfr;
+	uint16_t id;
+	void (*fixup)(struct cfi_private *cfi);
+};
+
+static void cfi_early_fixup(struct cfi_private *cfi,
+			    const struct cfi_early_fixup *fixups)
+{
+	const struct cfi_early_fixup *f;
+
+	for (f = fixups; f->fixup; f++) {
+		if (((f->mfr == CFI_MFR_ANY) || (f->mfr == cfi->mfr)) &&
+		    ((f->id == CFI_ID_ANY) || (f->id == cfi->id))) {
+			f->fixup(cfi);
+		}
+	}
+}
+
 /* check for QRY.
    in: interleave,type,mode
    ret: table index, <0 for error
@@ -151,6 +175,22 @@ static int __xipram cfi_probe_chip(struct map_info *map, __u32 base,
 	return 1;
 }
 
+static void fixup_s70gl02gs_chips(struct cfi_private *cfi)
+{
+	/*
+	 * S70GL02GS flash reports a single 256 MiB chip, but is really made up
+	 * of two 128 MiB chips with 1024 sectors each.
+	 */
+	cfi->cfiq->DevSize = 27;
+	cfi->cfiq->EraseRegionInfo[0] = 0x20003ff;
+	pr_warn("Bad S70GL02GS CFI data; adjust to detect 2 chips\n");
+}
+
+static const struct cfi_early_fixup cfi_early_fixup_table[] = {
+	{ CFI_MFR_AMD, 0x4801, fixup_s70gl02gs_chips },
+	{ },
+};
+
 static int __xipram cfi_chip_setup(struct map_info *map,
 				   struct cfi_private *cfi)
 {
@@ -235,6 +275,8 @@ static int __xipram cfi_chip_setup(struct map_info *map,
 	cfi_qry_mode_off(base, map, cfi);
 	xip_allowed(base, map);
 
+	cfi_early_fixup(cfi, cfi_early_fixup_table);
+
 	printk(KERN_INFO "%s: Found %d x%d devices at 0x%x in %d-bit bank. Manufacturer ID %#08x Chip ID %#08x\n",
 	       map->name, cfi->interleave, cfi->device_type*8, base,
 	       map->bankwidth*8, cfi->mfr, cfi->id);
diff --git a/drivers/mtd/cmdlinepart.c b/drivers/mtd/cmdlinepart.c
index fbd5affc0acf..3ea44cff9b75 100644
--- a/drivers/mtd/cmdlinepart.c
+++ b/drivers/mtd/cmdlinepart.c
@@ -190,7 +190,10 @@ static struct mtd_partition * newpart(char *s,
 		extra_mem = (unsigned char *)(parts + *num_parts);
 	}
 
-	/* enter this partition (offset will be calculated later if it is zero at this point) */
+	/*
+	 * enter this partition (offset will be calculated later if it is
+	 * OFFSET_CONTINUOUS at this point)
+	 */
 	parts[this_part].size = size;
 	parts[this_part].offset = offset;
 	parts[this_part].mask_flags = mask_flags;
diff --git a/drivers/mtd/devices/Kconfig b/drivers/mtd/devices/Kconfig
index 6def5445e03e..57b02c4b3f63 100644
--- a/drivers/mtd/devices/Kconfig
+++ b/drivers/mtd/devices/Kconfig
@@ -81,6 +81,7 @@ config MTD_DATAFLASH_OTP
 config MTD_M25P80
 	tristate "Support most SPI Flash chips (AT26DF, M25P, W25X, ...)"
 	depends on SPI_MASTER && MTD_SPI_NOR
+	select SPI_MEM
 	help
 	  This enables access to most modern SPI flash chips, used for
 	  program and data storage.   Series supported include Atmel AT26DF,
diff --git a/drivers/mtd/devices/docg3.c b/drivers/mtd/devices/docg3.c
index c594fe5eac08..802d8f159e90 100644
--- a/drivers/mtd/devices/docg3.c
+++ b/drivers/mtd/devices/docg3.c
@@ -1470,8 +1470,7 @@ static struct docg3 *sysfs_dev2docg3(struct device *dev,
 				     struct device_attribute *attr)
 {
 	int floor;
-	struct platform_device *pdev = to_platform_device(dev);
-	struct mtd_info **docg3_floors = platform_get_drvdata(pdev);
+	struct mtd_info **docg3_floors = dev_get_drvdata(dev);
 
 	floor = attr->attr.name[1] - '0';
 	if (floor < 0 || floor >= DOC_MAX_NBFLOORS)
diff --git a/drivers/mtd/devices/m25p80.c b/drivers/mtd/devices/m25p80.c
index a4e18f6aaa33..e84563d2067f 100644
--- a/drivers/mtd/devices/m25p80.c
+++ b/drivers/mtd/devices/m25p80.c
@@ -24,12 +24,13 @@
 #include <linux/mtd/partitions.h>
 
 #include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
 #include <linux/spi/flash.h>
 #include <linux/mtd/spi-nor.h>
 
 #define	MAX_CMD_SIZE		6
 struct m25p {
-	struct spi_device	*spi;
+	struct spi_mem		*spimem;
 	struct spi_nor		spi_nor;
 	u8			command[MAX_CMD_SIZE];
 };
@@ -37,97 +38,68 @@ struct m25p {
 static int m25p80_read_reg(struct spi_nor *nor, u8 code, u8 *val, int len)
 {
 	struct m25p *flash = nor->priv;
-	struct spi_device *spi = flash->spi;
+	struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(code, 1),
+					  SPI_MEM_OP_NO_ADDR,
+					  SPI_MEM_OP_NO_DUMMY,
+					  SPI_MEM_OP_DATA_IN(len, val, 1));
 	int ret;
 
-	ret = spi_write_then_read(spi, &code, 1, val, len);
+	ret = spi_mem_exec_op(flash->spimem, &op);
 	if (ret < 0)
-		dev_err(&spi->dev, "error %d reading %x\n", ret, code);
+		dev_err(&flash->spimem->spi->dev, "error %d reading %x\n", ret,
+			code);
 
 	return ret;
 }
 
-static void m25p_addr2cmd(struct spi_nor *nor, unsigned int addr, u8 *cmd)
-{
-	/* opcode is in cmd[0] */
-	cmd[1] = addr >> (nor->addr_width * 8 -  8);
-	cmd[2] = addr >> (nor->addr_width * 8 - 16);
-	cmd[3] = addr >> (nor->addr_width * 8 - 24);
-	cmd[4] = addr >> (nor->addr_width * 8 - 32);
-}
-
-static int m25p_cmdsz(struct spi_nor *nor)
-{
-	return 1 + nor->addr_width;
-}
-
 static int m25p80_write_reg(struct spi_nor *nor, u8 opcode, u8 *buf, int len)
 {
 	struct m25p *flash = nor->priv;
-	struct spi_device *spi = flash->spi;
-
-	flash->command[0] = opcode;
-	if (buf)
-		memcpy(&flash->command[1], buf, len);
+	struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(opcode, 1),
+					  SPI_MEM_OP_NO_ADDR,
+					  SPI_MEM_OP_NO_DUMMY,
+					  SPI_MEM_OP_DATA_OUT(len, buf, 1));
 
-	return spi_write(spi, flash->command, len + 1);
+	return spi_mem_exec_op(flash->spimem, &op);
 }
 
 static ssize_t m25p80_write(struct spi_nor *nor, loff_t to, size_t len,
 			    const u_char *buf)
 {
 	struct m25p *flash = nor->priv;
-	struct spi_device *spi = flash->spi;
-	unsigned int inst_nbits, addr_nbits, data_nbits, data_idx;
-	struct spi_transfer t[3] = {};
-	struct spi_message m;
-	int cmd_sz = m25p_cmdsz(nor);
-	ssize_t ret;
+	struct spi_mem_op op =
+			SPI_MEM_OP(SPI_MEM_OP_CMD(nor->program_opcode, 1),
+				   SPI_MEM_OP_ADDR(nor->addr_width, to, 1),
+				   SPI_MEM_OP_DUMMY(0, 1),
+				   SPI_MEM_OP_DATA_OUT(len, buf, 1));
+	size_t remaining = len;
+	int ret;
 
 	/* get transfer protocols. */
-	inst_nbits = spi_nor_get_protocol_inst_nbits(nor->write_proto);
-	addr_nbits = spi_nor_get_protocol_addr_nbits(nor->write_proto);
-	data_nbits = spi_nor_get_protocol_data_nbits(nor->write_proto);
-
-	spi_message_init(&m);
+	op.cmd.buswidth = spi_nor_get_protocol_inst_nbits(nor->write_proto);
+	op.addr.buswidth = spi_nor_get_protocol_addr_nbits(nor->write_proto);
+	op.dummy.buswidth = op.addr.buswidth;
+	op.data.buswidth = spi_nor_get_protocol_data_nbits(nor->write_proto);
 
 	if (nor->program_opcode == SPINOR_OP_AAI_WP && nor->sst_write_second)
-		cmd_sz = 1;
-
-	flash->command[0] = nor->program_opcode;
-	m25p_addr2cmd(nor, to, flash->command);
+		op.addr.nbytes = 0;
 
-	t[0].tx_buf = flash->command;
-	t[0].tx_nbits = inst_nbits;
-	t[0].len = cmd_sz;
-	spi_message_add_tail(&t[0], &m);
-
-	/* split the op code and address bytes into two transfers if needed. */
-	data_idx = 1;
-	if (addr_nbits != inst_nbits) {
-		t[0].len = 1;
+	while (remaining) {
+		op.data.nbytes = remaining < UINT_MAX ? remaining : UINT_MAX;
+		ret = spi_mem_adjust_op_size(flash->spimem, &op);
+		if (ret)
+			return ret;
 
-		t[1].tx_buf = &flash->command[1];
-		t[1].tx_nbits = addr_nbits;
-		t[1].len = cmd_sz - 1;
-		spi_message_add_tail(&t[1], &m);
+		ret = spi_mem_exec_op(flash->spimem, &op);
+		if (ret)
+			return ret;
 
-		data_idx = 2;
+		op.addr.val += op.data.nbytes;
+		remaining -= op.data.nbytes;
+		op.data.buf.out += op.data.nbytes;
 	}
 
-	t[data_idx].tx_buf = buf;
-	t[data_idx].tx_nbits = data_nbits;
-	t[data_idx].len = len;
-	spi_message_add_tail(&t[data_idx], &m);
-
-	ret = spi_sync(spi, &m);
-	if (ret)
-		return ret;
-
-	ret = m.actual_length - cmd_sz;
-	if (ret < 0)
-		return -EIO;
-	return ret;
+	return len;
 }
 
 /*
@@ -138,92 +110,39 @@ static ssize_t m25p80_read(struct spi_nor *nor, loff_t from, size_t len,
 			   u_char *buf)
 {
 	struct m25p *flash = nor->priv;
-	struct spi_device *spi = flash->spi;
-	unsigned int inst_nbits, addr_nbits, data_nbits, data_idx;
-	struct spi_transfer t[3];
-	struct spi_message m;
-	unsigned int dummy = nor->read_dummy;
-	ssize_t ret;
-	int cmd_sz;
+	struct spi_mem_op op =
+			SPI_MEM_OP(SPI_MEM_OP_CMD(nor->read_opcode, 1),
+				   SPI_MEM_OP_ADDR(nor->addr_width, from, 1),
+				   SPI_MEM_OP_DUMMY(nor->read_dummy, 1),
+				   SPI_MEM_OP_DATA_IN(len, buf, 1));
+	size_t remaining = len;
+	int ret;
 
 	/* get transfer protocols. */
-	inst_nbits = spi_nor_get_protocol_inst_nbits(nor->read_proto);
-	addr_nbits = spi_nor_get_protocol_addr_nbits(nor->read_proto);
-	data_nbits = spi_nor_get_protocol_data_nbits(nor->read_proto);
+	op.cmd.buswidth = spi_nor_get_protocol_inst_nbits(nor->read_proto);
+	op.addr.buswidth = spi_nor_get_protocol_addr_nbits(nor->read_proto);
+	op.dummy.buswidth = op.addr.buswidth;
+	op.data.buswidth = spi_nor_get_protocol_data_nbits(nor->read_proto);
 
 	/* convert the dummy cycles to the number of bytes */
-	dummy = (dummy * addr_nbits) / 8;
-
-	if (spi_flash_read_supported(spi)) {
-		struct spi_flash_read_message msg;
-
-		memset(&msg, 0, sizeof(msg));
+	op.dummy.nbytes = (nor->read_dummy * op.dummy.buswidth) / 8;
 
-		msg.buf = buf;
-		msg.from = from;
-		msg.len = len;
-		msg.read_opcode = nor->read_opcode;
-		msg.addr_width = nor->addr_width;
-		msg.dummy_bytes = dummy;
-		msg.opcode_nbits = inst_nbits;
-		msg.addr_nbits = addr_nbits;
-		msg.data_nbits = data_nbits;
-
-		ret = spi_flash_read(spi, &msg);
-		if (ret < 0)
+	while (remaining) {
+		op.data.nbytes = remaining < UINT_MAX ? remaining : UINT_MAX;
+		ret = spi_mem_adjust_op_size(flash->spimem, &op);
+		if (ret)
 			return ret;
-		return msg.retlen;
-	}
 
-	spi_message_init(&m);
-	memset(t, 0, (sizeof t));
-
-	flash->command[0] = nor->read_opcode;
-	m25p_addr2cmd(nor, from, flash->command);
-
-	t[0].tx_buf = flash->command;
-	t[0].tx_nbits = inst_nbits;
-	t[0].len = m25p_cmdsz(nor) + dummy;
-	spi_message_add_tail(&t[0], &m);
-
-	/*
-	 * Set all dummy/mode cycle bits to avoid sending some manufacturer
-	 * specific pattern, which might make the memory enter its Continuous
-	 * Read mode by mistake.
-	 * Based on the different mode cycle bit patterns listed and described
-	 * in the JESD216B specification, the 0xff value works for all memories
-	 * and all manufacturers.
-	 */
-	cmd_sz = t[0].len;
-	memset(flash->command + cmd_sz - dummy, 0xff, dummy);
-
-	/* split the op code and address bytes into two transfers if needed. */
-	data_idx = 1;
-	if (addr_nbits != inst_nbits) {
-		t[0].len = 1;
-
-		t[1].tx_buf = &flash->command[1];
-		t[1].tx_nbits = addr_nbits;
-		t[1].len = cmd_sz - 1;
-		spi_message_add_tail(&t[1], &m);
+		ret = spi_mem_exec_op(flash->spimem, &op);
+		if (ret)
+			return ret;
 
-		data_idx = 2;
+		op.addr.val += op.data.nbytes;
+		remaining -= op.data.nbytes;
+		op.data.buf.in += op.data.nbytes;
 	}
 
-	t[data_idx].rx_buf = buf;
-	t[data_idx].rx_nbits = data_nbits;
-	t[data_idx].len = min3(len, spi_max_transfer_size(spi),
-			       spi_max_message_size(spi) - cmd_sz);
-	spi_message_add_tail(&t[data_idx], &m);
-
-	ret = spi_sync(spi, &m);
-	if (ret)
-		return ret;
-
-	ret = m.actual_length - cmd_sz;
-	if (ret < 0)
-		return -EIO;
-	return ret;
+	return len;
 }
 
 /*
@@ -231,8 +150,9 @@ static ssize_t m25p80_read(struct spi_nor *nor, loff_t from, size_t len,
  * matches what the READ command supports, at least until this driver
  * understands FAST_READ (for clocks over 25 MHz).
  */
-static int m25p_probe(struct spi_device *spi)
+static int m25p_probe(struct spi_mem *spimem)
 {
+	struct spi_device *spi = spimem->spi;
 	struct flash_platform_data	*data;
 	struct m25p *flash;
 	struct spi_nor *nor;
@@ -244,9 +164,9 @@ static int m25p_probe(struct spi_device *spi)
 	char *flash_name;
 	int ret;
 
-	data = dev_get_platdata(&spi->dev);
+	data = dev_get_platdata(&spimem->spi->dev);
 
-	flash = devm_kzalloc(&spi->dev, sizeof(*flash), GFP_KERNEL);
+	flash = devm_kzalloc(&spimem->spi->dev, sizeof(*flash), GFP_KERNEL);
 	if (!flash)
 		return -ENOMEM;
 
@@ -258,12 +178,12 @@ static int m25p_probe(struct spi_device *spi)
 	nor->write_reg = m25p80_write_reg;
 	nor->read_reg = m25p80_read_reg;
 
-	nor->dev = &spi->dev;
+	nor->dev = &spimem->spi->dev;
 	spi_nor_set_flash_node(nor, spi->dev.of_node);
 	nor->priv = flash;
 
-	spi_set_drvdata(spi, flash);
-	flash->spi = spi;
+	spi_mem_set_drvdata(spimem, flash);
+	flash->spimem = spimem;
 
 	if (spi->mode & SPI_RX_QUAD) {
 		hwcaps.mask |= SNOR_HWCAPS_READ_1_1_4;
@@ -303,9 +223,9 @@ static int m25p_probe(struct spi_device *spi)
 }
 
 
-static int m25p_remove(struct spi_device *spi)
+static int m25p_remove(struct spi_mem *spimem)
 {
-	struct m25p	*flash = spi_get_drvdata(spi);
+	struct m25p	*flash = spi_mem_get_drvdata(spimem);
 
 	spi_nor_restore(&flash->spi_nor);
 
@@ -313,9 +233,9 @@ static int m25p_remove(struct spi_device *spi)
 	return mtd_device_unregister(&flash->spi_nor.mtd);
 }
 
-static void m25p_shutdown(struct spi_device *spi)
+static void m25p_shutdown(struct spi_mem *spimem)
 {
-	struct m25p *flash = spi_get_drvdata(spi);
+	struct m25p *flash = spi_mem_get_drvdata(spimem);
 
 	spi_nor_restore(&flash->spi_nor);
 }
@@ -386,12 +306,14 @@ static const struct of_device_id m25p_of_table[] = {
 };
 MODULE_DEVICE_TABLE(of, m25p_of_table);
 
-static struct spi_driver m25p80_driver = {
-	.driver = {
-		.name	= "m25p80",
-		.of_match_table = m25p_of_table,
+static struct spi_mem_driver m25p80_driver = {
+	.spidrv = {
+		.driver = {
+			.name	= "m25p80",
+			.of_match_table = m25p_of_table,
+		},
+		.id_table	= m25p_ids,
 	},
-	.id_table	= m25p_ids,
 	.probe	= m25p_probe,
 	.remove	= m25p_remove,
 	.shutdown	= m25p_shutdown,
@@ -402,7 +324,7 @@ static struct spi_driver m25p80_driver = {
 	 */
 };
 
-module_spi_driver(m25p80_driver);
+module_spi_mem_driver(m25p80_driver);
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Mike Lavender");
diff --git a/drivers/mtd/devices/mtd_dataflash.c b/drivers/mtd/devices/mtd_dataflash.c
index aaaeaae01e1d..3a6f450d1093 100644
--- a/drivers/mtd/devices/mtd_dataflash.c
+++ b/drivers/mtd/devices/mtd_dataflash.c
@@ -140,7 +140,7 @@ static int dataflash_waitready(struct spi_device *spi)
 		if (status & (1 << 7))	/* RDY/nBSY */
 			return status;
 
-		msleep(3);
+		usleep_range(3000, 4000);
 	}
 }
 
diff --git a/drivers/mtd/inftlmount.c b/drivers/mtd/inftlmount.c
index aab4f68bd36f..2d598412972d 100644
--- a/drivers/mtd/inftlmount.c
+++ b/drivers/mtd/inftlmount.c
@@ -334,28 +334,37 @@ static int memcmpb(void *a, int c, int n)
 static int check_free_sectors(struct INFTLrecord *inftl, unsigned int address,
 	int len, int check_oob)
 {
-	u8 buf[SECTORSIZE + inftl->mbd.mtd->oobsize];
 	struct mtd_info *mtd = inftl->mbd.mtd;
 	size_t retlen;
-	int i;
+	int i, ret;
+	u8 *buf;
+
+	buf = kmalloc(SECTORSIZE + mtd->oobsize, GFP_KERNEL);
+	if (!buf)
+		return -1;
 
+	ret = -1;
 	for (i = 0; i < len; i += SECTORSIZE) {
 		if (mtd_read(mtd, address, SECTORSIZE, &retlen, buf))
-			return -1;
+			goto out;
 		if (memcmpb(buf, 0xff, SECTORSIZE) != 0)
-			return -1;
+			goto out;
 
 		if (check_oob) {
 			if(inftl_read_oob(mtd, address, mtd->oobsize,
 					  &retlen, &buf[SECTORSIZE]) < 0)
-				return -1;
+				goto out;
 			if (memcmpb(buf + SECTORSIZE, 0xff, mtd->oobsize) != 0)
-				return -1;
+				goto out;
 		}
 		address += SECTORSIZE;
 	}
 
-	return 0;
+	ret = 0;
+
+out:
+	kfree(buf);
+	return ret;
 }
 
 /*
diff --git a/drivers/mtd/maps/pismo.c b/drivers/mtd/maps/pismo.c
index dc6df9abea0b..c065d7995c0a 100644
--- a/drivers/mtd/maps/pismo.c
+++ b/drivers/mtd/maps/pismo.c
@@ -265,7 +265,6 @@ MODULE_DEVICE_TABLE(i2c, pismo_id);
 static struct i2c_driver pismo_driver = {
 	.driver	= {
 		.name	= "pismo",
-		.owner	= THIS_MODULE,
 	},
 	.probe		= pismo_probe,
 	.remove		= pismo_remove,
diff --git a/drivers/mtd/mtd_blkdevs.c b/drivers/mtd/mtd_blkdevs.c
index 16ae4ae8e8f9..29c0bfd74e8a 100644
--- a/drivers/mtd/mtd_blkdevs.c
+++ b/drivers/mtd/mtd_blkdevs.c
@@ -82,7 +82,6 @@ static blk_status_t do_blktrans_request(struct mtd_blktrans_ops *tr,
 
 	block = blk_rq_pos(req) << 9 >> tr->blkshift;
 	nsect = blk_rq_cur_bytes(req) >> tr->blkshift;
-	buf = bio_data(req->bio);
 
 	if (req_op(req) == REQ_OP_FLUSH) {
 		if (tr->flush(dev))
@@ -100,9 +99,14 @@ static blk_status_t do_blktrans_request(struct mtd_blktrans_ops *tr,
 			return BLK_STS_IOERR;
 		return BLK_STS_OK;
 	case REQ_OP_READ:
-		for (; nsect > 0; nsect--, block++, buf += tr->blksize)
-			if (tr->readsect(dev, block, buf))
+		buf = kmap(bio_page(req->bio)) + bio_offset(req->bio);
+		for (; nsect > 0; nsect--, block++, buf += tr->blksize) {
+			if (tr->readsect(dev, block, buf)) {
+				kunmap(bio_page(req->bio));
 				return BLK_STS_IOERR;
+			}
+		}
+		kunmap(bio_page(req->bio));
 		rq_flush_dcache_pages(req);
 		return BLK_STS_OK;
 	case REQ_OP_WRITE:
@@ -110,9 +114,14 @@ static blk_status_t do_blktrans_request(struct mtd_blktrans_ops *tr,
 			return BLK_STS_IOERR;
 
 		rq_flush_dcache_pages(req);
-		for (; nsect > 0; nsect--, block++, buf += tr->blksize)
-			if (tr->writesect(dev, block, buf))
+		buf = kmap(bio_page(req->bio)) + bio_offset(req->bio);
+		for (; nsect > 0; nsect--, block++, buf += tr->blksize) {
+			if (tr->writesect(dev, block, buf)) {
+				kunmap(bio_page(req->bio));
 				return BLK_STS_IOERR;
+			}
+		}
+		kunmap(bio_page(req->bio));
 		return BLK_STS_OK;
 	default:
 		return BLK_STS_IOERR;
@@ -418,7 +427,6 @@ int add_mtd_blktrans_dev(struct mtd_blktrans_dev *new)
 	new->rq->queuedata = new;
 	blk_queue_logical_block_size(new->rq, tr->blksize);
 
-	blk_queue_bounce_limit(new->rq, BLK_BOUNCE_HIGH);
 	blk_queue_flag_set(QUEUE_FLAG_NONROT, new->rq);
 	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, new->rq);
 
diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
index 807d17d863b3..42395df06be9 100644
--- a/drivers/mtd/mtdcore.c
+++ b/drivers/mtd/mtdcore.c
@@ -210,6 +210,15 @@ static ssize_t mtd_oobsize_show(struct device *dev,
 }
 static DEVICE_ATTR(oobsize, S_IRUGO, mtd_oobsize_show, NULL);
 
+static ssize_t mtd_oobavail_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct mtd_info *mtd = dev_get_drvdata(dev);
+
+	return snprintf(buf, PAGE_SIZE, "%u\n", mtd->oobavail);
+}
+static DEVICE_ATTR(oobavail, S_IRUGO, mtd_oobavail_show, NULL);
+
 static ssize_t mtd_numeraseregions_show(struct device *dev,
 		struct device_attribute *attr, char *buf)
 {
@@ -327,6 +336,7 @@ static struct attribute *mtd_attrs[] = {
 	&dev_attr_writesize.attr,
 	&dev_attr_subpagesize.attr,
 	&dev_attr_oobsize.attr,
+	&dev_attr_oobavail.attr,
 	&dev_attr_numeraseregions.attr,
 	&dev_attr_name.attr,
 	&dev_attr_ecc_strength.attr,
@@ -690,7 +700,6 @@ int mtd_device_parse_register(struct mtd_info *mtd, const char * const *types,
 			      const struct mtd_partition *parts,
 			      int nr_parts)
 {
-	struct mtd_partitions parsed = { };
 	int ret;
 
 	mtd_set_dev_defaults(mtd);
@@ -702,13 +711,10 @@ int mtd_device_parse_register(struct mtd_info *mtd, const char * const *types,
 	}
 
 	/* Prefer parsed partitions over driver-provided fallback */
-	ret = parse_mtd_partitions(mtd, types, &parsed, parser_data);
-	if (!ret && parsed.nr_parts) {
-		parts = parsed.parts;
-		nr_parts = parsed.nr_parts;
-	}
-
-	if (nr_parts)
+	ret = parse_mtd_partitions(mtd, types, parser_data);
+	if (ret > 0)
+		ret = 0;
+	else if (nr_parts)
 		ret = add_mtd_partitions(mtd, parts, nr_parts);
 	else if (!device_is_registered(&mtd->dev))
 		ret = add_mtd_device(mtd);
@@ -734,8 +740,6 @@ int mtd_device_parse_register(struct mtd_info *mtd, const char * const *types,
 	}
 
 out:
-	/* Cleanup any parsed partitions */
-	mtd_part_parser_cleanup(&parsed);
 	if (ret && device_is_registered(&mtd->dev))
 		del_mtd_device(mtd);
 
@@ -1829,18 +1833,6 @@ static int mtd_proc_show(struct seq_file *m, void *v)
 	mutex_unlock(&mtd_table_mutex);
 	return 0;
 }
-
-static int mtd_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, mtd_proc_show, NULL);
-}
-
-static const struct file_operations mtd_proc_ops = {
-	.open		= mtd_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
 #endif /* CONFIG_PROC_FS */
 
 /*====================================================================*/
@@ -1883,7 +1875,7 @@ static int __init init_mtd(void)
 		goto err_bdi;
 	}
 
-	proc_mtd = proc_create("mtd", 0, NULL, &mtd_proc_ops);
+	proc_mtd = proc_create_single("mtd", 0, NULL, mtd_proc_show);
 
 	ret = init_mtdchar();
 	if (ret)
diff --git a/drivers/mtd/mtdcore.h b/drivers/mtd/mtdcore.h
index 37accfd0400e..9887bda317cd 100644
--- a/drivers/mtd/mtdcore.h
+++ b/drivers/mtd/mtdcore.h
@@ -15,7 +15,6 @@ int del_mtd_partitions(struct mtd_info *);
 struct mtd_partitions;
 
 int parse_mtd_partitions(struct mtd_info *master, const char * const *types,
-			 struct mtd_partitions *pparts,
 			 struct mtd_part_parser_data *data);
 
 void mtd_part_parser_cleanup(struct mtd_partitions *parts);
diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c
index 023516a63276..f8d3a015cdad 100644
--- a/drivers/mtd/mtdpart.c
+++ b/drivers/mtd/mtdpart.c
@@ -335,20 +335,7 @@ static inline void free_partition(struct mtd_part *p)
  */
 static int mtd_parse_part(struct mtd_part *slave, const char *const *types)
 {
-	struct mtd_partitions parsed;
-	int err;
-
-	err = parse_mtd_partitions(&slave->mtd, types, &parsed, NULL);
-	if (err)
-		return err;
-	else if (!parsed.nr_parts)
-		return -ENOENT;
-
-	err = add_mtd_partitions(&slave->mtd, parsed.parts, parsed.nr_parts);
-
-	mtd_part_parser_cleanup(&parsed);
-
-	return err;
+	return parse_mtd_partitions(&slave->mtd, types, NULL);
 }
 
 static struct mtd_part *allocate_partition(struct mtd_info *parent,
@@ -933,30 +920,27 @@ static int mtd_part_of_parse(struct mtd_info *master,
 }
 
 /**
- * parse_mtd_partitions - parse MTD partitions
+ * parse_mtd_partitions - parse and register MTD partitions
+ *
  * @master: the master partition (describes whole MTD device)
  * @types: names of partition parsers to try or %NULL
- * @pparts: info about partitions found is returned here
  * @data: MTD partition parser-specific data
  *
- * This function tries to find partition on MTD device @master. It uses MTD
- * partition parsers, specified in @types. However, if @types is %NULL, then
- * the default list of parsers is used. The default list contains only the
+ * This function tries to find & register partitions on MTD device @master. It
+ * uses MTD partition parsers, specified in @types. However, if @types is %NULL,
+ * then the default list of parsers is used. The default list contains only the
  * "cmdlinepart" and "ofpart" parsers ATM.
  * Note: If there are more then one parser in @types, the kernel only takes the
  * partitions parsed out by the first parser.
  *
  * This function may return:
  * o a negative error code in case of failure
- * o zero otherwise, and @pparts will describe the partitions, number of
- *   partitions, and the parser which parsed them. Caller must release
- *   resources with mtd_part_parser_cleanup() when finished with the returned
- *   data.
+ * o number of found partitions otherwise
  */
 int parse_mtd_partitions(struct mtd_info *master, const char *const *types,
-			 struct mtd_partitions *pparts,
 			 struct mtd_part_parser_data *data)
 {
+	struct mtd_partitions pparts = { };
 	struct mtd_part_parser *parser;
 	int ret, err = 0;
 
@@ -970,7 +954,7 @@ int parse_mtd_partitions(struct mtd_info *master, const char *const *types,
 		 * handled in a separated function.
 		 */
 		if (!strcmp(*types, "ofpart")) {
-			ret = mtd_part_of_parse(master, pparts);
+			ret = mtd_part_of_parse(master, &pparts);
 		} else {
 			pr_debug("%s: parsing partitions %s\n", master->name,
 				 *types);
@@ -981,13 +965,17 @@ int parse_mtd_partitions(struct mtd_info *master, const char *const *types,
 				parser ? parser->name : NULL);
 			if (!parser)
 				continue;
-			ret = mtd_part_do_parse(parser, master, pparts, data);
+			ret = mtd_part_do_parse(parser, master, &pparts, data);
 			if (ret <= 0)
 				mtd_part_parser_put(parser);
 		}
 		/* Found partitions! */
-		if (ret > 0)
-			return 0;
+		if (ret > 0) {
+			err = add_mtd_partitions(master, pparts.parts,
+						 pparts.nr_parts);
+			mtd_part_parser_cleanup(&pparts);
+			return err ? err : pparts.nr_parts;
+		}
 		/*
 		 * Stash the first error we see; only report it if no parser
 		 * succeeds
diff --git a/drivers/mtd/nand/onenand/samsung.c b/drivers/mtd/nand/onenand/samsung.c
index 2e9d076e445a..4cce4c0311ca 100644
--- a/drivers/mtd/nand/onenand/samsung.c
+++ b/drivers/mtd/nand/onenand/samsung.c
@@ -958,8 +958,7 @@ static int s3c_onenand_remove(struct platform_device *pdev)
 
 static int s3c_pm_ops_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct mtd_info *mtd = platform_get_drvdata(pdev);
+	struct mtd_info *mtd = dev_get_drvdata(dev);
 	struct onenand_chip *this = mtd->priv;
 
 	this->wait(mtd, FL_PM_SUSPENDED);
@@ -968,8 +967,7 @@ static int s3c_pm_ops_suspend(struct device *dev)
 
 static  int s3c_pm_ops_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct mtd_info *mtd = platform_get_drvdata(pdev);
+	struct mtd_info *mtd = dev_get_drvdata(dev);
 	struct onenand_chip *this = mtd->priv;
 
 	this->unlock_all(mtd);
diff --git a/drivers/mtd/nand/raw/Kconfig b/drivers/mtd/nand/raw/Kconfig
index 19a2b283fbbe..6871ff0fd300 100644
--- a/drivers/mtd/nand/raw/Kconfig
+++ b/drivers/mtd/nand/raw/Kconfig
@@ -46,7 +46,7 @@ config MTD_NAND_DENALI
 config MTD_NAND_DENALI_PCI
         tristate "Support Denali NAND controller on Intel Moorestown"
 	select MTD_NAND_DENALI
-	depends on HAS_DMA && PCI
+	depends on PCI
         help
           Enable the driver for NAND flash on Intel Moorestown, using the
           Denali NAND controller core.
@@ -152,7 +152,6 @@ config MTD_NAND_S3C2410_CLKSTOP
 config MTD_NAND_TANGO
 	tristate "NAND Flash support for Tango chips"
 	depends on ARCH_TANGO || COMPILE_TEST
-	depends on HAS_DMA
 	help
 	  Enables the NAND Flash controller on Tango chips.
 
@@ -285,7 +284,7 @@ config MTD_NAND_MARVELL
 	tristate "NAND controller support on Marvell boards"
 	depends on PXA3xx || ARCH_MMP || PLAT_ORION || ARCH_MVEBU || \
 		   COMPILE_TEST
-	depends on HAS_IOMEM && HAS_DMA
+	depends on HAS_IOMEM
 	help
 	  This enables the NAND flash controller driver for Marvell boards,
 	  including:
@@ -447,7 +446,6 @@ config MTD_NAND_SH_FLCTL
 	tristate "Support for NAND on Renesas SuperH FLCTL"
 	depends on SUPERH || COMPILE_TEST
 	depends on HAS_IOMEM
-	depends on HAS_DMA
 	help
 	  Several Renesas SuperH CPU has FLCTL. This option enables support
 	  for NAND Flash using FLCTL.
@@ -515,7 +513,6 @@ config MTD_NAND_SUNXI
 config MTD_NAND_HISI504
 	tristate "Support for NAND controller on Hisilicon SoC Hip04"
 	depends on ARCH_HISI || COMPILE_TEST
-	depends on HAS_DMA
 	help
 	  Enables support for NAND controller on Hisilicon SoC Hip04.
 
@@ -529,7 +526,6 @@ config MTD_NAND_QCOM
 config MTD_NAND_MTK
 	tristate "Support for NAND controller on MTK SoCs"
 	depends on ARCH_MEDIATEK || COMPILE_TEST
-	depends on HAS_DMA
 	help
 	  Enables support for NAND controller on MTK SoCs.
 	  This controller is found on mt27xx, mt81xx, mt65xx SoCs.
diff --git a/drivers/mtd/nand/raw/cafe_nand.c b/drivers/mtd/nand/raw/cafe_nand.c
index d8c8c9d1e640..d721f489b38b 100644
--- a/drivers/mtd/nand/raw/cafe_nand.c
+++ b/drivers/mtd/nand/raw/cafe_nand.c
@@ -394,12 +394,13 @@ static int cafe_nand_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 
 		for (i=0; i<8; i+=2) {
 			uint32_t tmp = cafe_readl(cafe, NAND_ECC_SYN01 + (i*2));
-			syn[i] = cafe->rs->index_of[tmp & 0xfff];
-			syn[i+1] = cafe->rs->index_of[(tmp >> 16) & 0xfff];
+
+			syn[i] = cafe->rs->codec->index_of[tmp & 0xfff];
+			syn[i+1] = cafe->rs->codec->index_of[(tmp >> 16) & 0xfff];
 		}
 
 		n = decode_rs16(cafe->rs, NULL, NULL, 1367, syn, 0, pos, 0,
-		                pat);
+				pat);
 
 		for (i = 0; i < n; i++) {
 			int p = pos[i];
diff --git a/drivers/mtd/nand/raw/davinci_nand.c b/drivers/mtd/nand/raw/davinci_nand.c
index 0f09518d980f..cd12e5abafde 100644
--- a/drivers/mtd/nand/raw/davinci_nand.c
+++ b/drivers/mtd/nand/raw/davinci_nand.c
@@ -27,7 +27,6 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/err.h>
-#include <linux/clk.h>
 #include <linux/io.h>
 #include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
@@ -55,7 +54,6 @@ struct davinci_nand_info {
 	struct nand_chip	chip;
 
 	struct device		*dev;
-	struct clk		*clk;
 
 	bool			is_readmode;
 
@@ -547,7 +545,7 @@ static struct davinci_nand_pdata
 			return ERR_PTR(-ENOMEM);
 		if (!of_property_read_u32(pdev->dev.of_node,
 			"ti,davinci-chipselect", &prop))
-			pdev->id = prop;
+			pdata->core_chipsel = prop;
 		else
 			return ERR_PTR(-EINVAL);
 
@@ -629,7 +627,7 @@ static int nand_davinci_probe(struct platform_device *pdev)
 		return -ENODEV;
 
 	/* which external chipselect will we be managing? */
-	if (pdev->id < 0 || pdev->id > 3)
+	if (pdata->core_chipsel < 0 || pdata->core_chipsel > 3)
 		return -ENODEV;
 
 	info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL);
@@ -685,7 +683,7 @@ static int nand_davinci_probe(struct platform_device *pdev)
 	info->ioaddr		= (uint32_t __force) vaddr;
 
 	info->current_cs	= info->ioaddr;
-	info->core_chipsel	= pdev->id;
+	info->core_chipsel	= pdata->core_chipsel;
 	info->mask_chipsel	= pdata->mask_chipsel;
 
 	/* use nandboot-capable ALE/CLE masks by default */
@@ -703,22 +701,6 @@ static int nand_davinci_probe(struct platform_device *pdev)
 	/* Use board-specific ECC config */
 	info->chip.ecc.mode	= pdata->ecc_mode;
 
-	ret = -EINVAL;
-
-	info->clk = devm_clk_get(&pdev->dev, "aemif");
-	if (IS_ERR(info->clk)) {
-		ret = PTR_ERR(info->clk);
-		dev_dbg(&pdev->dev, "unable to get AEMIF clock, err %d\n", ret);
-		return ret;
-	}
-
-	ret = clk_prepare_enable(info->clk);
-	if (ret < 0) {
-		dev_dbg(&pdev->dev, "unable to enable AEMIF clock, err %d\n",
-			ret);
-		goto err_clk_enable;
-	}
-
 	spin_lock_irq(&davinci_nand_lock);
 
 	/* put CSxNAND into NAND mode */
@@ -732,7 +714,7 @@ static int nand_davinci_probe(struct platform_device *pdev)
 	ret = nand_scan_ident(mtd, pdata->mask_chipsel ? 2 : 1, NULL);
 	if (ret < 0) {
 		dev_dbg(&pdev->dev, "no NAND chip(s) found\n");
-		goto err;
+		return ret;
 	}
 
 	switch (info->chip.ecc.mode) {
@@ -838,9 +820,6 @@ static int nand_davinci_probe(struct platform_device *pdev)
 	nand_cleanup(&info->chip);
 
 err:
-	clk_disable_unprepare(info->clk);
-
-err_clk_enable:
 	spin_lock_irq(&davinci_nand_lock);
 	if (info->chip.ecc.mode == NAND_ECC_HW_SYNDROME)
 		ecc4_busy = false;
@@ -859,8 +838,6 @@ static int nand_davinci_remove(struct platform_device *pdev)
 
 	nand_release(nand_to_mtd(&info->chip));
 
-	clk_disable_unprepare(info->clk);
-
 	return 0;
 }
 
diff --git a/drivers/mtd/nand/raw/diskonchip.c b/drivers/mtd/nand/raw/diskonchip.c
index 86a258de0b75..8d10061abb4b 100644
--- a/drivers/mtd/nand/raw/diskonchip.c
+++ b/drivers/mtd/nand/raw/diskonchip.c
@@ -66,6 +66,7 @@ struct doc_priv {
 	int curchip;
 	int mh0_page;
 	int mh1_page;
+	struct rs_control *rs_decoder;
 	struct mtd_info *nextdoc;
 
 	/* Handle the last stage of initialization (BBT scan, partitioning) */
@@ -123,9 +124,6 @@ MODULE_PARM_DESC(doc_config_location, "Physical memory address at which to probe
 /* Number of symbols */
 #define NN 1023
 
-/* the Reed Solomon control structure */
-static struct rs_control *rs_decoder;
-
 /*
  * The HW decoder in the DoC ASIC's provides us a error syndrome,
  * which we must convert to a standard syndrome usable by the generic
@@ -140,6 +138,7 @@ static int doc_ecc_decode(struct rs_control *rs, uint8_t *data, uint8_t *ecc)
 	int i, j, nerr, errpos[8];
 	uint8_t parity;
 	uint16_t ds[4], s[5], tmp, errval[8], syn[4];
+	struct rs_codec *cd = rs->codec;
 
 	memset(syn, 0, sizeof(syn));
 	/* Convert the ecc bytes into words */
@@ -160,15 +159,15 @@ static int doc_ecc_decode(struct rs_control *rs, uint8_t *data, uint8_t *ecc)
 	for (j = 1; j < NROOTS; j++) {
 		if (ds[j] == 0)
 			continue;
-		tmp = rs->index_of[ds[j]];
+		tmp = cd->index_of[ds[j]];
 		for (i = 0; i < NROOTS; i++)
-			s[i] ^= rs->alpha_to[rs_modnn(rs, tmp + (FCR + i) * j)];
+			s[i] ^= cd->alpha_to[rs_modnn(cd, tmp + (FCR + i) * j)];
 	}
 
 	/* Calc syn[i] = s[i] / alpha^(v + i) */
 	for (i = 0; i < NROOTS; i++) {
 		if (s[i])
-			syn[i] = rs_modnn(rs, rs->index_of[s[i]] + (NN - FCR - i));
+			syn[i] = rs_modnn(cd, cd->index_of[s[i]] + (NN - FCR - i));
 	}
 	/* Call the decoder library */
 	nerr = decode_rs16(rs, NULL, NULL, 1019, syn, 0, errpos, 0, errval);
@@ -930,7 +929,7 @@ static int doc200x_correct_data(struct mtd_info *mtd, u_char *dat,
 				calc_ecc[i] = ReadDOC_(docptr, DoC_ECCSyndrome0 + i);
 		}
 
-		ret = doc_ecc_decode(rs_decoder, dat, calc_ecc);
+		ret = doc_ecc_decode(doc->rs_decoder, dat, calc_ecc);
 		if (ret > 0)
 			pr_err("doc200x_correct_data corrected %d errors\n",
 			       ret);
@@ -1421,10 +1420,10 @@ static inline int __init doc2001plus_init(struct mtd_info *mtd)
 
 static int __init doc_probe(unsigned long physadr)
 {
+	struct nand_chip *nand = NULL;
+	struct doc_priv *doc = NULL;
 	unsigned char ChipID;
 	struct mtd_info *mtd;
-	struct nand_chip *nand;
-	struct doc_priv *doc;
 	void __iomem *virtadr;
 	unsigned char save_control;
 	unsigned char tmp, tmpb, tmpc;
@@ -1481,12 +1480,12 @@ static int __init doc_probe(unsigned long physadr)
 		WriteDOC(tmp, virtadr, Mplus_DOCControl);
 		WriteDOC(~tmp, virtadr, Mplus_CtrlConfirm);
 
-		mdelay(1);
+		usleep_range(1000, 2000);
 		/* Enable the Millennium Plus ASIC */
 		tmp = DOC_MODE_NORMAL | DOC_MODE_MDWREN | DOC_MODE_RST_LAT | DOC_MODE_BDECT;
 		WriteDOC(tmp, virtadr, Mplus_DOCControl);
 		WriteDOC(~tmp, virtadr, Mplus_CtrlConfirm);
-		mdelay(1);
+		usleep_range(1000, 2000);
 
 		ChipID = ReadDOC(virtadr, ChipID);
 
@@ -1561,8 +1560,25 @@ static int __init doc_probe(unsigned long physadr)
 		goto fail;
 	}
 
+
+	/*
+	 * Allocate a RS codec instance
+	 *
+	 * Symbolsize is 10 (bits)
+	 * Primitve polynomial is x^10+x^3+1
+	 * First consecutive root is 510
+	 * Primitve element to generate roots = 1
+	 * Generator polinomial degree = 4
+	 */
+	doc = (struct doc_priv *) (nand + 1);
+	doc->rs_decoder = init_rs(10, 0x409, FCR, 1, NROOTS);
+	if (!doc->rs_decoder) {
+		pr_err("DiskOnChip: Could not create a RS codec\n");
+		ret = -ENOMEM;
+		goto fail;
+	}
+
 	mtd			= nand_to_mtd(nand);
-	doc			= (struct doc_priv *) (nand + 1);
 	nand->bbt_td		= (struct nand_bbt_descr *) (doc + 1);
 	nand->bbt_md		= nand->bbt_td + 1;
 
@@ -1612,7 +1628,6 @@ static int __init doc_probe(unsigned long physadr)
 		   haven't yet added it.  This is handled without incident by
 		   mtd_device_unregister, as far as I can tell. */
 		nand_release(mtd);
-		kfree(nand);
 		goto fail;
 	}
 
@@ -1625,6 +1640,9 @@ static int __init doc_probe(unsigned long physadr)
 	   actually a DiskOnChip.  */
 	WriteDOC(save_control, virtadr, DOCControl);
  fail:
+	if (doc)
+		free_rs(doc->rs_decoder);
+	kfree(nand);
 	iounmap(virtadr);
 
 error_ioremap:
@@ -1647,6 +1665,7 @@ static void release_nanddoc(void)
 		nand_release(mtd);
 		iounmap(doc->virtadr);
 		release_mem_region(doc->physadr, DOC_IOREMAP_LEN);
+		free_rs(doc->rs_decoder);
 		kfree(nand);
 	}
 }
@@ -1655,27 +1674,12 @@ static int __init init_nanddoc(void)
 {
 	int i, ret = 0;
 
-	/* We could create the decoder on demand, if memory is a concern.
-	 * This way we have it handy, if an error happens
-	 *
-	 * Symbolsize is 10 (bits)
-	 * Primitve polynomial is x^10+x^3+1
-	 * first consecutive root is 510
-	 * primitve element to generate roots = 1
-	 * generator polinomial degree = 4
-	 */
-	rs_decoder = init_rs(10, 0x409, FCR, 1, NROOTS);
-	if (!rs_decoder) {
-		pr_err("DiskOnChip: Could not create a RS decoder\n");
-		return -ENOMEM;
-	}
-
 	if (doc_config_location) {
 		pr_info("Using configured DiskOnChip probe address 0x%lx\n",
 			doc_config_location);
 		ret = doc_probe(doc_config_location);
 		if (ret < 0)
-			goto outerr;
+			return ret;
 	} else {
 		for (i = 0; (doc_locations[i] != 0xffffffff); i++) {
 			doc_probe(doc_locations[i]);
@@ -1686,11 +1690,7 @@ static int __init init_nanddoc(void)
 	if (!doclist) {
 		pr_info("No valid DiskOnChip devices found\n");
 		ret = -ENODEV;
-		goto outerr;
 	}
-	return 0;
- outerr:
-	free_rs(rs_decoder);
 	return ret;
 }
 
@@ -1698,11 +1698,6 @@ static void __exit cleanup_nanddoc(void)
 {
 	/* Cleanup the nand/DoC resources */
 	release_nanddoc();
-
-	/* Free the reed solomon resources */
-	if (rs_decoder) {
-		free_rs(rs_decoder);
-	}
 }
 
 module_init(init_nanddoc);
diff --git a/drivers/mtd/nand/raw/fsl_elbc_nand.c b/drivers/mtd/nand/raw/fsl_elbc_nand.c
index d28df991c73c..51f0b340bc0d 100644
--- a/drivers/mtd/nand/raw/fsl_elbc_nand.c
+++ b/drivers/mtd/nand/raw/fsl_elbc_nand.c
@@ -813,8 +813,6 @@ static int fsl_elbc_chip_remove(struct fsl_elbc_mtd *priv)
 	struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = priv->ctrl->nand;
 	struct mtd_info *mtd = nand_to_mtd(&priv->chip);
 
-	nand_release(mtd);
-
 	kfree(mtd->name);
 
 	if (priv->vbase)
@@ -926,15 +924,20 @@ static int fsl_elbc_nand_probe(struct platform_device *pdev)
 
 	/* First look for RedBoot table or partitions on the command
 	 * line, these take precedence over device tree information */
-	mtd_device_parse_register(mtd, part_probe_types, NULL,
-				  NULL, 0);
+	ret = mtd_device_parse_register(mtd, part_probe_types, NULL, NULL, 0);
+	if (ret)
+		goto cleanup_nand;
 
 	pr_info("eLBC NAND device at 0x%llx, bank %d\n",
 		(unsigned long long)res.start, priv->bank);
+
 	return 0;
 
+cleanup_nand:
+	nand_cleanup(&priv->chip);
 err:
 	fsl_elbc_chip_remove(priv);
+
 	return ret;
 }
 
@@ -942,7 +945,9 @@ static int fsl_elbc_nand_remove(struct platform_device *pdev)
 {
 	struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = fsl_lbc_ctrl_dev->nand;
 	struct fsl_elbc_mtd *priv = dev_get_drvdata(&pdev->dev);
+	struct mtd_info *mtd = nand_to_mtd(&priv->chip);
 
+	nand_release(mtd);
 	fsl_elbc_chip_remove(priv);
 
 	mutex_lock(&fsl_elbc_nand_mutex);
diff --git a/drivers/mtd/nand/raw/fsl_ifc_nand.c b/drivers/mtd/nand/raw/fsl_ifc_nand.c
index 61aae0224078..382b67e97174 100644
--- a/drivers/mtd/nand/raw/fsl_ifc_nand.c
+++ b/drivers/mtd/nand/raw/fsl_ifc_nand.c
@@ -342,9 +342,16 @@ static void fsl_ifc_cmdfunc(struct mtd_info *mtd, unsigned int command,
 
 	case NAND_CMD_READID:
 	case NAND_CMD_PARAM: {
+		/*
+		 * For READID, read 8 bytes that are currently used.
+		 * For PARAM, read all 3 copies of 256-bytes pages.
+		 */
+		int len = 8;
 		int timing = IFC_FIR_OP_RB;
-		if (command == NAND_CMD_PARAM)
+		if (command == NAND_CMD_PARAM) {
 			timing = IFC_FIR_OP_RBCD;
+			len = 256 * 3;
+		}
 
 		ifc_out32((IFC_FIR_OP_CW0 << IFC_NAND_FIR0_OP0_SHIFT) |
 			  (IFC_FIR_OP_UA  << IFC_NAND_FIR0_OP1_SHIFT) |
@@ -354,12 +361,8 @@ static void fsl_ifc_cmdfunc(struct mtd_info *mtd, unsigned int command,
 			  &ifc->ifc_nand.nand_fcr0);
 		ifc_out32(column, &ifc->ifc_nand.row3);
 
-		/*
-		 * although currently it's 8 bytes for READID, we always read
-		 * the maximum 256 bytes(for PARAM)
-		 */
-		ifc_out32(256, &ifc->ifc_nand.nand_fbcr);
-		ifc_nand_ctrl->read_bytes = 256;
+		ifc_out32(len, &ifc->ifc_nand.nand_fbcr);
+		ifc_nand_ctrl->read_bytes = len;
 
 		set_addr(mtd, 0, 0, 0);
 		fsl_ifc_run_command(mtd);
@@ -924,8 +927,6 @@ static int fsl_ifc_chip_remove(struct fsl_ifc_mtd *priv)
 {
 	struct mtd_info *mtd = nand_to_mtd(&priv->chip);
 
-	nand_release(mtd);
-
 	kfree(mtd->name);
 
 	if (priv->vbase)
@@ -1059,21 +1060,29 @@ static int fsl_ifc_nand_probe(struct platform_device *dev)
 
 	/* First look for RedBoot table or partitions on the command
 	 * line, these take precedence over device tree information */
-	mtd_device_parse_register(mtd, part_probe_types, NULL, NULL, 0);
+	ret = mtd_device_parse_register(mtd, part_probe_types, NULL, NULL, 0);
+	if (ret)
+		goto cleanup_nand;
 
 	dev_info(priv->dev, "IFC NAND device at 0x%llx, bank %d\n",
 		 (unsigned long long)res.start, priv->bank);
+
 	return 0;
 
+cleanup_nand:
+	nand_cleanup(&priv->chip);
 err:
 	fsl_ifc_chip_remove(priv);
+
 	return ret;
 }
 
 static int fsl_ifc_nand_remove(struct platform_device *dev)
 {
 	struct fsl_ifc_mtd *priv = dev_get_drvdata(&dev->dev);
+	struct mtd_info *mtd = nand_to_mtd(&priv->chip);
 
+	nand_release(mtd);
 	fsl_ifc_chip_remove(priv);
 
 	mutex_lock(&fsl_ifc_nand_mutex);
diff --git a/drivers/mtd/nand/raw/fsmc_nand.c b/drivers/mtd/nand/raw/fsmc_nand.c
index 28c48dcc514e..f4a5a317d4ae 100644
--- a/drivers/mtd/nand/raw/fsmc_nand.c
+++ b/drivers/mtd/nand/raw/fsmc_nand.c
@@ -1022,12 +1022,12 @@ static int __init fsmc_nand_probe(struct platform_device *pdev)
 		host->read_dma_chan = dma_request_channel(mask, filter, NULL);
 		if (!host->read_dma_chan) {
 			dev_err(&pdev->dev, "Unable to get read dma channel\n");
-			goto err_req_read_chnl;
+			goto disable_clk;
 		}
 		host->write_dma_chan = dma_request_channel(mask, filter, NULL);
 		if (!host->write_dma_chan) {
 			dev_err(&pdev->dev, "Unable to get write dma channel\n");
-			goto err_req_write_chnl;
+			goto release_dma_read_chan;
 		}
 	}
 
@@ -1050,7 +1050,7 @@ static int __init fsmc_nand_probe(struct platform_device *pdev)
 	ret = nand_scan_ident(mtd, 1, NULL);
 	if (ret) {
 		dev_err(&pdev->dev, "No NAND Device found!\n");
-		goto err_scan_ident;
+		goto release_dma_write_chan;
 	}
 
 	if (AMBA_REV_BITS(host->pid) >= 8) {
@@ -1065,7 +1065,7 @@ static int __init fsmc_nand_probe(struct platform_device *pdev)
 			dev_warn(&pdev->dev, "No oob scheme defined for oobsize %d\n",
 				 mtd->oobsize);
 			ret = -EINVAL;
-			goto err_probe;
+			goto release_dma_write_chan;
 		}
 
 		mtd_set_ooblayout(mtd, &fsmc_ecc4_ooblayout_ops);
@@ -1090,7 +1090,7 @@ static int __init fsmc_nand_probe(struct platform_device *pdev)
 
 		default:
 			dev_err(&pdev->dev, "Unsupported ECC mode!\n");
-			goto err_probe;
+			goto release_dma_write_chan;
 		}
 
 		/*
@@ -1110,7 +1110,7 @@ static int __init fsmc_nand_probe(struct platform_device *pdev)
 					 "No oob scheme defined for oobsize %d\n",
 					 mtd->oobsize);
 				ret = -EINVAL;
-				goto err_probe;
+				goto release_dma_write_chan;
 			}
 		}
 	}
@@ -1118,26 +1118,29 @@ static int __init fsmc_nand_probe(struct platform_device *pdev)
 	/* Second stage of scan to fill MTD data-structures */
 	ret = nand_scan_tail(mtd);
 	if (ret)
-		goto err_probe;
+		goto release_dma_write_chan;
 
 	mtd->name = "nand";
 	ret = mtd_device_register(mtd, NULL, 0);
 	if (ret)
-		goto err_probe;
+		goto cleanup_nand;
 
 	platform_set_drvdata(pdev, host);
 	dev_info(&pdev->dev, "FSMC NAND driver registration successful\n");
+
 	return 0;
 
-err_probe:
-err_scan_ident:
+cleanup_nand:
+	nand_cleanup(nand);
+release_dma_write_chan:
 	if (host->mode == USE_DMA_ACCESS)
 		dma_release_channel(host->write_dma_chan);
-err_req_write_chnl:
+release_dma_read_chan:
 	if (host->mode == USE_DMA_ACCESS)
 		dma_release_channel(host->read_dma_chan);
-err_req_read_chnl:
+disable_clk:
 	clk_disable_unprepare(host->clk);
+
 	return ret;
 }
 
diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-lib.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-lib.c
index e94556705dc7..83697b8df871 100644
--- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-lib.c
+++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-lib.c
@@ -258,8 +258,9 @@ int bch_set_geometry(struct gpmi_nand_data *this)
 	unsigned int gf_len;
 	int ret;
 
-	if (common_nfc_set_geometry(this))
-		return !0;
+	ret = common_nfc_set_geometry(this);
+	if (ret)
+		return ret;
 
 	block_count   = bch_geo->ecc_chunk_count - 1;
 	block_size    = bch_geo->ecc_chunk_size;
@@ -544,19 +545,13 @@ int gpmi_is_ready(struct gpmi_nand_data *this, unsigned chip)
 	return reg & mask;
 }
 
-static inline void set_dma_type(struct gpmi_nand_data *this,
-					enum dma_ops_type type)
-{
-	this->last_dma_type = this->dma_type;
-	this->dma_type = type;
-}
-
 int gpmi_send_command(struct gpmi_nand_data *this)
 {
 	struct dma_chan *channel = get_dma_chan(this);
 	struct dma_async_tx_descriptor *desc;
 	struct scatterlist *sgl;
 	int chip = this->current_chip;
+	int ret;
 	u32 pio[3];
 
 	/* [1] send out the PIO words */
@@ -586,15 +581,19 @@ int gpmi_send_command(struct gpmi_nand_data *this)
 		return -EINVAL;
 
 	/* [3] submit the DMA */
-	set_dma_type(this, DMA_FOR_COMMAND);
-	return start_dma_without_bch_irq(this, desc);
+	ret = start_dma_without_bch_irq(this, desc);
+
+	dma_unmap_sg(this->dev, sgl, 1, DMA_TO_DEVICE);
+
+	return ret;
 }
 
-int gpmi_send_data(struct gpmi_nand_data *this)
+int gpmi_send_data(struct gpmi_nand_data *this, const void *buf, int len)
 {
 	struct dma_async_tx_descriptor *desc;
 	struct dma_chan *channel = get_dma_chan(this);
 	int chip = this->current_chip;
+	int ret;
 	uint32_t command_mode;
 	uint32_t address;
 	u32 pio[2];
@@ -608,7 +607,7 @@ int gpmi_send_data(struct gpmi_nand_data *this)
 		| BF_GPMI_CTRL0_CS(chip, this)
 		| BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
 		| BF_GPMI_CTRL0_ADDRESS(address)
-		| BF_GPMI_CTRL0_XFER_COUNT(this->upper_len);
+		| BF_GPMI_CTRL0_XFER_COUNT(len);
 	pio[1] = 0;
 	desc = dmaengine_prep_slave_sg(channel, (struct scatterlist *)pio,
 					ARRAY_SIZE(pio), DMA_TRANS_NONE, 0);
@@ -616,7 +615,7 @@ int gpmi_send_data(struct gpmi_nand_data *this)
 		return -EINVAL;
 
 	/* [2] send DMA request */
-	prepare_data_dma(this, DMA_TO_DEVICE);
+	prepare_data_dma(this, buf, len, DMA_TO_DEVICE);
 	desc = dmaengine_prep_slave_sg(channel, &this->data_sgl,
 					1, DMA_MEM_TO_DEV,
 					DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
@@ -624,16 +623,21 @@ int gpmi_send_data(struct gpmi_nand_data *this)
 		return -EINVAL;
 
 	/* [3] submit the DMA */
-	set_dma_type(this, DMA_FOR_WRITE_DATA);
-	return start_dma_without_bch_irq(this, desc);
+	ret = start_dma_without_bch_irq(this, desc);
+
+	dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_TO_DEVICE);
+
+	return ret;
 }
 
-int gpmi_read_data(struct gpmi_nand_data *this)
+int gpmi_read_data(struct gpmi_nand_data *this, void *buf, int len)
 {
 	struct dma_async_tx_descriptor *desc;
 	struct dma_chan *channel = get_dma_chan(this);
 	int chip = this->current_chip;
+	int ret;
 	u32 pio[2];
+	bool direct;
 
 	/* [1] : send PIO */
 	pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__READ)
@@ -641,7 +645,7 @@ int gpmi_read_data(struct gpmi_nand_data *this)
 		| BF_GPMI_CTRL0_CS(chip, this)
 		| BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
 		| BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
-		| BF_GPMI_CTRL0_XFER_COUNT(this->upper_len);
+		| BF_GPMI_CTRL0_XFER_COUNT(len);
 	pio[1] = 0;
 	desc = dmaengine_prep_slave_sg(channel,
 					(struct scatterlist *)pio,
@@ -650,7 +654,7 @@ int gpmi_read_data(struct gpmi_nand_data *this)
 		return -EINVAL;
 
 	/* [2] : send DMA request */
-	prepare_data_dma(this, DMA_FROM_DEVICE);
+	direct = prepare_data_dma(this, buf, len, DMA_FROM_DEVICE);
 	desc = dmaengine_prep_slave_sg(channel, &this->data_sgl,
 					1, DMA_DEV_TO_MEM,
 					DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
@@ -658,8 +662,14 @@ int gpmi_read_data(struct gpmi_nand_data *this)
 		return -EINVAL;
 
 	/* [3] : submit the DMA */
-	set_dma_type(this, DMA_FOR_READ_DATA);
-	return start_dma_without_bch_irq(this, desc);
+
+	ret = start_dma_without_bch_irq(this, desc);
+
+	dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_FROM_DEVICE);
+	if (!direct)
+		memcpy(buf, this->data_buffer_dma, len);
+
+	return ret;
 }
 
 int gpmi_send_page(struct gpmi_nand_data *this,
@@ -703,7 +713,6 @@ int gpmi_send_page(struct gpmi_nand_data *this,
 	if (!desc)
 		return -EINVAL;
 
-	set_dma_type(this, DMA_FOR_WRITE_ECC_PAGE);
 	return start_dma_with_bch_irq(this, desc);
 }
 
@@ -785,7 +794,6 @@ int gpmi_read_page(struct gpmi_nand_data *this,
 		return -EINVAL;
 
 	/* [4] submit the DMA */
-	set_dma_type(this, DMA_FOR_READ_ECC_PAGE);
 	return start_dma_with_bch_irq(this, desc);
 }
 
diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
index c2597c8107a0..f6aa358a3452 100644
--- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
+++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
@@ -198,17 +198,16 @@ static inline bool gpmi_check_ecc(struct gpmi_nand_data *this)
  *
  * We may have available oob space in this case.
  */
-static int set_geometry_by_ecc_info(struct gpmi_nand_data *this)
+static int set_geometry_by_ecc_info(struct gpmi_nand_data *this,
+				    unsigned int ecc_strength,
+				    unsigned int ecc_step)
 {
 	struct bch_geometry *geo = &this->bch_geometry;
 	struct nand_chip *chip = &this->nand;
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	unsigned int block_mark_bit_offset;
 
-	if (!(chip->ecc_strength_ds > 0 && chip->ecc_step_ds > 0))
-		return -EINVAL;
-
-	switch (chip->ecc_step_ds) {
+	switch (ecc_step) {
 	case SZ_512:
 		geo->gf_len = 13;
 		break;
@@ -221,8 +220,8 @@ static int set_geometry_by_ecc_info(struct gpmi_nand_data *this)
 			chip->ecc_strength_ds, chip->ecc_step_ds);
 		return -EINVAL;
 	}
-	geo->ecc_chunk_size = chip->ecc_step_ds;
-	geo->ecc_strength = round_up(chip->ecc_strength_ds, 2);
+	geo->ecc_chunk_size = ecc_step;
+	geo->ecc_strength = round_up(ecc_strength, 2);
 	if (!gpmi_check_ecc(this))
 		return -EINVAL;
 
@@ -230,7 +229,7 @@ static int set_geometry_by_ecc_info(struct gpmi_nand_data *this)
 	if (geo->ecc_chunk_size < mtd->oobsize) {
 		dev_err(this->dev,
 			"unsupported nand chip. ecc size: %d, oob size : %d\n",
-			chip->ecc_step_ds, mtd->oobsize);
+			ecc_step, mtd->oobsize);
 		return -EINVAL;
 	}
 
@@ -423,9 +422,20 @@ static int legacy_set_geometry(struct gpmi_nand_data *this)
 
 int common_nfc_set_geometry(struct gpmi_nand_data *this)
 {
+	struct nand_chip *chip = &this->nand;
+
+	if (chip->ecc.strength > 0 && chip->ecc.size > 0)
+		return set_geometry_by_ecc_info(this, chip->ecc.strength,
+						chip->ecc.size);
+
 	if ((of_property_read_bool(this->dev->of_node, "fsl,use-minimum-ecc"))
-				|| legacy_set_geometry(this))
-		return set_geometry_by_ecc_info(this);
+				|| legacy_set_geometry(this)) {
+		if (!(chip->ecc_strength_ds > 0 && chip->ecc_step_ds > 0))
+			return -EINVAL;
+
+		return set_geometry_by_ecc_info(this, chip->ecc_strength_ds,
+						chip->ecc_step_ds);
+	}
 
 	return 0;
 }
@@ -437,33 +447,32 @@ struct dma_chan *get_dma_chan(struct gpmi_nand_data *this)
 }
 
 /* Can we use the upper's buffer directly for DMA? */
-void prepare_data_dma(struct gpmi_nand_data *this, enum dma_data_direction dr)
+bool prepare_data_dma(struct gpmi_nand_data *this, const void *buf, int len,
+		      enum dma_data_direction dr)
 {
 	struct scatterlist *sgl = &this->data_sgl;
 	int ret;
 
 	/* first try to map the upper buffer directly */
-	if (virt_addr_valid(this->upper_buf) &&
-		!object_is_on_stack(this->upper_buf)) {
-		sg_init_one(sgl, this->upper_buf, this->upper_len);
+	if (virt_addr_valid(buf) && !object_is_on_stack(buf)) {
+		sg_init_one(sgl, buf, len);
 		ret = dma_map_sg(this->dev, sgl, 1, dr);
 		if (ret == 0)
 			goto map_fail;
 
-		this->direct_dma_map_ok = true;
-		return;
+		return true;
 	}
 
 map_fail:
 	/* We have to use our own DMA buffer. */
-	sg_init_one(sgl, this->data_buffer_dma, this->upper_len);
+	sg_init_one(sgl, this->data_buffer_dma, len);
 
 	if (dr == DMA_TO_DEVICE)
-		memcpy(this->data_buffer_dma, this->upper_buf, this->upper_len);
+		memcpy(this->data_buffer_dma, buf, len);
 
 	dma_map_sg(this->dev, sgl, 1, dr);
 
-	this->direct_dma_map_ok = false;
+	return false;
 }
 
 /* This will be called after the DMA operation is finished. */
@@ -472,31 +481,6 @@ static void dma_irq_callback(void *param)
 	struct gpmi_nand_data *this = param;
 	struct completion *dma_c = &this->dma_done;
 
-	switch (this->dma_type) {
-	case DMA_FOR_COMMAND:
-		dma_unmap_sg(this->dev, &this->cmd_sgl, 1, DMA_TO_DEVICE);
-		break;
-
-	case DMA_FOR_READ_DATA:
-		dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_FROM_DEVICE);
-		if (this->direct_dma_map_ok == false)
-			memcpy(this->upper_buf, this->data_buffer_dma,
-				this->upper_len);
-		break;
-
-	case DMA_FOR_WRITE_DATA:
-		dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_TO_DEVICE);
-		break;
-
-	case DMA_FOR_READ_ECC_PAGE:
-	case DMA_FOR_WRITE_ECC_PAGE:
-		/* We have to wait the BCH interrupt to finish. */
-		break;
-
-	default:
-		dev_err(this->dev, "in wrong DMA operation.\n");
-	}
-
 	complete(dma_c);
 }
 
@@ -516,8 +500,7 @@ int start_dma_without_bch_irq(struct gpmi_nand_data *this,
 	/* Wait for the interrupt from the DMA block. */
 	timeout = wait_for_completion_timeout(dma_c, msecs_to_jiffies(1000));
 	if (!timeout) {
-		dev_err(this->dev, "DMA timeout, last DMA :%d\n",
-			this->last_dma_type);
+		dev_err(this->dev, "DMA timeout, last DMA\n");
 		gpmi_dump_info(this);
 		return -ETIMEDOUT;
 	}
@@ -546,8 +529,7 @@ int start_dma_with_bch_irq(struct gpmi_nand_data *this,
 	/* Wait for the interrupt from the BCH block. */
 	timeout = wait_for_completion_timeout(bch_c, msecs_to_jiffies(1000));
 	if (!timeout) {
-		dev_err(this->dev, "BCH timeout, last DMA :%d\n",
-			this->last_dma_type);
+		dev_err(this->dev, "BCH timeout\n");
 		gpmi_dump_info(this);
 		return -ETIMEDOUT;
 	}
@@ -695,56 +677,6 @@ static void release_resources(struct gpmi_nand_data *this)
 	release_dma_channels(this);
 }
 
-static int read_page_prepare(struct gpmi_nand_data *this,
-			void *destination, unsigned length,
-			void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
-			void **use_virt, dma_addr_t *use_phys)
-{
-	struct device *dev = this->dev;
-
-	if (virt_addr_valid(destination)) {
-		dma_addr_t dest_phys;
-
-		dest_phys = dma_map_single(dev, destination,
-						length, DMA_FROM_DEVICE);
-		if (dma_mapping_error(dev, dest_phys)) {
-			if (alt_size < length) {
-				dev_err(dev, "Alternate buffer is too small\n");
-				return -ENOMEM;
-			}
-			goto map_failed;
-		}
-		*use_virt = destination;
-		*use_phys = dest_phys;
-		this->direct_dma_map_ok = true;
-		return 0;
-	}
-
-map_failed:
-	*use_virt = alt_virt;
-	*use_phys = alt_phys;
-	this->direct_dma_map_ok = false;
-	return 0;
-}
-
-static inline void read_page_end(struct gpmi_nand_data *this,
-			void *destination, unsigned length,
-			void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
-			void *used_virt, dma_addr_t used_phys)
-{
-	if (this->direct_dma_map_ok)
-		dma_unmap_single(this->dev, used_phys, length, DMA_FROM_DEVICE);
-}
-
-static inline void read_page_swap_end(struct gpmi_nand_data *this,
-			void *destination, unsigned length,
-			void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
-			void *used_virt, dma_addr_t used_phys)
-{
-	if (!this->direct_dma_map_ok)
-		memcpy(destination, alt_virt, length);
-}
-
 static int send_page_prepare(struct gpmi_nand_data *this,
 			const void *source, unsigned length,
 			void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
@@ -946,10 +878,8 @@ static void gpmi_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
 
 	dev_dbg(this->dev, "len is %d\n", len);
-	this->upper_buf	= buf;
-	this->upper_len	= len;
 
-	gpmi_read_data(this);
+	gpmi_read_data(this, buf, len);
 }
 
 static void gpmi_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len)
@@ -958,10 +888,8 @@ static void gpmi_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len)
 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
 
 	dev_dbg(this->dev, "len is %d\n", len);
-	this->upper_buf	= (uint8_t *)buf;
-	this->upper_len	= len;
 
-	gpmi_send_data(this);
+	gpmi_send_data(this, buf, len);
 }
 
 static uint8_t gpmi_read_byte(struct mtd_info *mtd)
@@ -1031,44 +959,46 @@ static int gpmi_ecc_read_page_data(struct nand_chip *chip,
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	void          *payload_virt;
 	dma_addr_t    payload_phys;
-	void          *auxiliary_virt;
-	dma_addr_t    auxiliary_phys;
 	unsigned int  i;
 	unsigned char *status;
 	unsigned int  max_bitflips = 0;
 	int           ret;
+	bool          direct = false;
 
 	dev_dbg(this->dev, "page number is : %d\n", page);
-	ret = read_page_prepare(this, buf, nfc_geo->payload_size,
-					this->payload_virt, this->payload_phys,
-					nfc_geo->payload_size,
-					&payload_virt, &payload_phys);
-	if (ret) {
-		dev_err(this->dev, "Inadequate DMA buffer\n");
-		ret = -ENOMEM;
-		return ret;
+
+	payload_virt = this->payload_virt;
+	payload_phys = this->payload_phys;
+
+	if (virt_addr_valid(buf)) {
+		dma_addr_t dest_phys;
+
+		dest_phys = dma_map_single(this->dev, buf, nfc_geo->payload_size,
+					   DMA_FROM_DEVICE);
+		if (!dma_mapping_error(this->dev, dest_phys)) {
+			payload_virt = buf;
+			payload_phys = dest_phys;
+			direct = true;
+		}
 	}
-	auxiliary_virt = this->auxiliary_virt;
-	auxiliary_phys = this->auxiliary_phys;
 
 	/* go! */
-	ret = gpmi_read_page(this, payload_phys, auxiliary_phys);
-	read_page_end(this, buf, nfc_geo->payload_size,
-			this->payload_virt, this->payload_phys,
-			nfc_geo->payload_size,
-			payload_virt, payload_phys);
+	ret = gpmi_read_page(this, payload_phys, this->auxiliary_phys);
+
+	if (direct)
+		dma_unmap_single(this->dev, payload_phys, nfc_geo->payload_size,
+				 DMA_FROM_DEVICE);
+
 	if (ret) {
 		dev_err(this->dev, "Error in ECC-based read: %d\n", ret);
 		return ret;
 	}
 
 	/* Loop over status bytes, accumulating ECC status. */
-	status = auxiliary_virt + nfc_geo->auxiliary_status_offset;
+	status = this->auxiliary_virt + nfc_geo->auxiliary_status_offset;
 
-	read_page_swap_end(this, buf, nfc_geo->payload_size,
-			   this->payload_virt, this->payload_phys,
-			   nfc_geo->payload_size,
-			   payload_virt, payload_phys);
+	if (!direct)
+		memcpy(buf, this->payload_virt, nfc_geo->payload_size);
 
 	for (i = 0; i < nfc_geo->ecc_chunk_count; i++, status++) {
 		if ((*status == STATUS_GOOD) || (*status == STATUS_ERASED))
@@ -1123,7 +1053,7 @@ static int gpmi_ecc_read_page_data(struct nand_chip *chip,
 						buf + i * nfc_geo->ecc_chunk_size,
 						nfc_geo->ecc_chunk_size,
 						eccbuf, eccbytes,
-						auxiliary_virt,
+						this->auxiliary_virt,
 						nfc_geo->metadata_size,
 						nfc_geo->ecc_strength);
 			} else {
@@ -1151,7 +1081,7 @@ static int gpmi_ecc_read_page_data(struct nand_chip *chip,
 	}
 
 	/* handle the block mark swapping */
-	block_mark_swapping(this, buf, auxiliary_virt);
+	block_mark_swapping(this, buf, this->auxiliary_virt);
 
 	if (oob_required) {
 		/*
@@ -1165,7 +1095,7 @@ static int gpmi_ecc_read_page_data(struct nand_chip *chip,
 		 * the block mark.
 		 */
 		memset(chip->oob_poi, ~0, mtd->oobsize);
-		chip->oob_poi[0] = ((uint8_t *) auxiliary_virt)[0];
+		chip->oob_poi[0] = ((uint8_t *)this->auxiliary_virt)[0];
 	}
 
 	return max_bitflips;
diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.h b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.h
index 62fde59b995f..6aa10d6962d6 100644
--- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.h
+++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.h
@@ -77,15 +77,6 @@ struct boot_rom_geometry {
 	unsigned int  search_area_stride_exponent;
 };
 
-/* DMA operations types */
-enum dma_ops_type {
-	DMA_FOR_COMMAND = 1,
-	DMA_FOR_READ_DATA,
-	DMA_FOR_WRITE_DATA,
-	DMA_FOR_READ_ECC_PAGE,
-	DMA_FOR_WRITE_ECC_PAGE
-};
-
 enum gpmi_type {
 	IS_MX23,
 	IS_MX28,
@@ -150,13 +141,6 @@ struct gpmi_nand_data {
 	int			current_chip;
 	unsigned int		command_length;
 
-	/* passed from upper layer */
-	uint8_t			*upper_buf;
-	int			upper_len;
-
-	/* for DMA operations */
-	bool			direct_dma_map_ok;
-
 	struct scatterlist	cmd_sgl;
 	char			*cmd_buffer;
 
@@ -178,8 +162,6 @@ struct gpmi_nand_data {
 	/* DMA channels */
 #define DMA_CHANS		8
 	struct dma_chan		*dma_chans[DMA_CHANS];
-	enum dma_ops_type	last_dma_type;
-	enum dma_ops_type	dma_type;
 	struct completion	dma_done;
 
 	/* private */
@@ -189,7 +171,7 @@ struct gpmi_nand_data {
 /* Common Services */
 int common_nfc_set_geometry(struct gpmi_nand_data *);
 struct dma_chan *get_dma_chan(struct gpmi_nand_data *);
-void prepare_data_dma(struct gpmi_nand_data *,
+bool prepare_data_dma(struct gpmi_nand_data *, const void *buf, int len,
 		      enum dma_data_direction dr);
 int start_dma_without_bch_irq(struct gpmi_nand_data *,
 			      struct dma_async_tx_descriptor *);
@@ -208,8 +190,9 @@ int gpmi_disable_clk(struct gpmi_nand_data *this);
 int gpmi_setup_data_interface(struct mtd_info *mtd, int chipnr,
 			      const struct nand_data_interface *conf);
 void gpmi_nfc_apply_timings(struct gpmi_nand_data *this);
-int gpmi_read_data(struct gpmi_nand_data *);
-int gpmi_send_data(struct gpmi_nand_data *);
+int gpmi_read_data(struct gpmi_nand_data *, void *buf, int len);
+int gpmi_send_data(struct gpmi_nand_data *, const void *buf, int len);
+
 int gpmi_send_page(struct gpmi_nand_data *,
 		   dma_addr_t payload, dma_addr_t auxiliary);
 int gpmi_read_page(struct gpmi_nand_data *,
diff --git a/drivers/mtd/nand/raw/hisi504_nand.c b/drivers/mtd/nand/raw/hisi504_nand.c
index 27558a67fa41..a1e009c8e556 100644
--- a/drivers/mtd/nand/raw/hisi504_nand.c
+++ b/drivers/mtd/nand/raw/hisi504_nand.c
@@ -731,23 +731,19 @@ static int hisi_nfc_probe(struct platform_device *pdev)
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
 		dev_err(dev, "no IRQ resource defined\n");
-		ret = -ENXIO;
-		goto err_res;
+		return -ENXIO;
 	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	host->iobase = devm_ioremap_resource(dev, res);
-	if (IS_ERR(host->iobase)) {
-		ret = PTR_ERR(host->iobase);
-		goto err_res;
-	}
+	if (IS_ERR(host->iobase))
+		return PTR_ERR(host->iobase);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 	host->mmio = devm_ioremap_resource(dev, res);
 	if (IS_ERR(host->mmio)) {
-		ret = PTR_ERR(host->mmio);
 		dev_err(dev, "devm_ioremap_resource[1] fail\n");
-		goto err_res;
+		return PTR_ERR(host->mmio);
 	}
 
 	mtd->name		= "hisi_nand";
@@ -770,19 +766,17 @@ static int hisi_nfc_probe(struct platform_device *pdev)
 	ret = devm_request_irq(dev, irq, hinfc_irq_handle, 0x0, "nandc", host);
 	if (ret) {
 		dev_err(dev, "failed to request IRQ\n");
-		goto err_res;
+		return ret;
 	}
 
 	ret = nand_scan_ident(mtd, max_chips, NULL);
 	if (ret)
-		goto err_res;
+		return ret;
 
 	host->buffer = dmam_alloc_coherent(dev, mtd->writesize + mtd->oobsize,
 		&host->dma_buffer, GFP_KERNEL);
-	if (!host->buffer) {
-		ret = -ENOMEM;
-		goto err_res;
-	}
+	if (!host->buffer)
+		return -ENOMEM;
 
 	host->dma_oob = host->dma_buffer + mtd->writesize;
 	memset(host->buffer, 0xff, mtd->writesize + mtd->oobsize);
@@ -798,8 +792,7 @@ static int hisi_nfc_probe(struct platform_device *pdev)
 	 */
 	default:
 		dev_err(dev, "NON-2KB page size nand flash\n");
-		ret = -EINVAL;
-		goto err_res;
+		return -EINVAL;
 	}
 	hinfc_write(host, flag, HINFC504_CON);
 
@@ -809,21 +802,17 @@ static int hisi_nfc_probe(struct platform_device *pdev)
 	ret = nand_scan_tail(mtd);
 	if (ret) {
 		dev_err(dev, "nand_scan_tail failed: %d\n", ret);
-		goto err_res;
+		return ret;
 	}
 
 	ret = mtd_device_register(mtd, NULL, 0);
 	if (ret) {
 		dev_err(dev, "Err MTD partition=%d\n", ret);
-		goto err_mtd;
+		nand_cleanup(chip);
+		return ret;
 	}
 
 	return 0;
-
-err_mtd:
-	nand_release(mtd);
-err_res:
-	return ret;
 }
 
 static int hisi_nfc_remove(struct platform_device *pdev)
diff --git a/drivers/mtd/nand/raw/lpc32xx_mlc.c b/drivers/mtd/nand/raw/lpc32xx_mlc.c
index e357948a7505..052d123a8304 100644
--- a/drivers/mtd/nand/raw/lpc32xx_mlc.c
+++ b/drivers/mtd/nand/raw/lpc32xx_mlc.c
@@ -673,7 +673,7 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 	host->io_base = devm_ioremap_resource(&pdev->dev, rc);
 	if (IS_ERR(host->io_base))
 		return PTR_ERR(host->io_base);
-	
+
 	host->io_base_phy = rc->start;
 
 	nand_chip = &host->nand_chip;
@@ -706,11 +706,11 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 	if (IS_ERR(host->clk)) {
 		dev_err(&pdev->dev, "Clock initialization failure\n");
 		res = -ENOENT;
-		goto err_exit1;
+		goto free_gpio;
 	}
 	res = clk_prepare_enable(host->clk);
 	if (res)
-		goto err_put_clk;
+		goto put_clk;
 
 	nand_chip->cmd_ctrl = lpc32xx_nand_cmd_ctrl;
 	nand_chip->dev_ready = lpc32xx_nand_device_ready;
@@ -744,7 +744,7 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 		res = lpc32xx_dma_setup(host);
 		if (res) {
 			res = -EIO;
-			goto err_exit2;
+			goto unprepare_clk;
 		}
 	}
 
@@ -754,18 +754,18 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 	 */
 	res = nand_scan_ident(mtd, 1, NULL);
 	if (res)
-		goto err_exit3;
+		goto release_dma_chan;
 
 	host->dma_buf = devm_kzalloc(&pdev->dev, mtd->writesize, GFP_KERNEL);
 	if (!host->dma_buf) {
 		res = -ENOMEM;
-		goto err_exit3;
+		goto release_dma_chan;
 	}
 
 	host->dummy_buf = devm_kzalloc(&pdev->dev, mtd->writesize, GFP_KERNEL);
 	if (!host->dummy_buf) {
 		res = -ENOMEM;
-		goto err_exit3;
+		goto release_dma_chan;
 	}
 
 	nand_chip->ecc.mode = NAND_ECC_HW;
@@ -783,14 +783,14 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 	if (host->irq < 0) {
 		dev_err(&pdev->dev, "failed to get platform irq\n");
 		res = -EINVAL;
-		goto err_exit3;
+		goto release_dma_chan;
 	}
 
 	if (request_irq(host->irq, (irq_handler_t)&lpc3xxx_nand_irq,
 			IRQF_TRIGGER_HIGH, DRV_NAME, host)) {
 		dev_err(&pdev->dev, "Error requesting NAND IRQ\n");
 		res = -ENXIO;
-		goto err_exit3;
+		goto release_dma_chan;
 	}
 
 	/*
@@ -799,27 +799,29 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 	 */
 	res = nand_scan_tail(mtd);
 	if (res)
-		goto err_exit4;
+		goto free_irq;
 
 	mtd->name = DRV_NAME;
 
 	res = mtd_device_register(mtd, host->ncfg->parts,
 				  host->ncfg->num_parts);
-	if (!res)
-		return res;
+	if (res)
+		goto cleanup_nand;
 
-	nand_release(mtd);
+	return 0;
 
-err_exit4:
+cleanup_nand:
+	nand_cleanup(nand_chip);
+free_irq:
 	free_irq(host->irq, host);
-err_exit3:
+release_dma_chan:
 	if (use_dma)
 		dma_release_channel(host->dma_chan);
-err_exit2:
+unprepare_clk:
 	clk_disable_unprepare(host->clk);
-err_put_clk:
+put_clk:
 	clk_put(host->clk);
-err_exit1:
+free_gpio:
 	lpc32xx_wp_enable(host);
 	gpio_free(host->ncfg->wp_gpio);
 
diff --git a/drivers/mtd/nand/raw/lpc32xx_slc.c b/drivers/mtd/nand/raw/lpc32xx_slc.c
index 5f7cc6da0a7f..42820aa1abab 100644
--- a/drivers/mtd/nand/raw/lpc32xx_slc.c
+++ b/drivers/mtd/nand/raw/lpc32xx_slc.c
@@ -831,11 +831,11 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 	if (IS_ERR(host->clk)) {
 		dev_err(&pdev->dev, "Clock failure\n");
 		res = -ENOENT;
-		goto err_exit1;
+		goto enable_wp;
 	}
 	res = clk_prepare_enable(host->clk);
 	if (res)
-		goto err_exit1;
+		goto enable_wp;
 
 	/* Set NAND IO addresses and command/ready functions */
 	chip->IO_ADDR_R = SLC_DATA(host->io_base);
@@ -874,19 +874,19 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 				      GFP_KERNEL);
 	if (host->data_buf == NULL) {
 		res = -ENOMEM;
-		goto err_exit2;
+		goto unprepare_clk;
 	}
 
 	res = lpc32xx_nand_dma_setup(host);
 	if (res) {
 		res = -EIO;
-		goto err_exit2;
+		goto unprepare_clk;
 	}
 
 	/* Find NAND device */
 	res = nand_scan_ident(mtd, 1, NULL);
 	if (res)
-		goto err_exit3;
+		goto release_dma;
 
 	/* OOB and ECC CPU and DMA work areas */
 	host->ecc_buf = (uint32_t *)(host->data_buf + LPC32XX_DMA_DATA_SIZE);
@@ -920,21 +920,23 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 	 */
 	res = nand_scan_tail(mtd);
 	if (res)
-		goto err_exit3;
+		goto release_dma;
 
 	mtd->name = "nxp_lpc3220_slc";
 	res = mtd_device_register(mtd, host->ncfg->parts,
 				  host->ncfg->num_parts);
-	if (!res)
-		return res;
+	if (res)
+		goto cleanup_nand;
 
-	nand_release(mtd);
+	return 0;
 
-err_exit3:
+cleanup_nand:
+	nand_cleanup(chip);
+release_dma:
 	dma_release_channel(host->dma_chan);
-err_exit2:
+unprepare_clk:
 	clk_disable_unprepare(host->clk);
-err_exit1:
+enable_wp:
 	lpc32xx_wp_enable(host);
 
 	return res;
diff --git a/drivers/mtd/nand/raw/mtk_ecc.c b/drivers/mtd/nand/raw/mtk_ecc.c
index 40d86a861a70..6432bd70c3b3 100644
--- a/drivers/mtd/nand/raw/mtk_ecc.c
+++ b/drivers/mtd/nand/raw/mtk_ecc.c
@@ -500,7 +500,6 @@ static int mtk_ecc_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct mtk_ecc *ecc;
 	struct resource *res;
-	const struct of_device_id *of_ecc_id = NULL;
 	u32 max_eccdata_size;
 	int irq, ret;
 
@@ -508,11 +507,7 @@ static int mtk_ecc_probe(struct platform_device *pdev)
 	if (!ecc)
 		return -ENOMEM;
 
-	of_ecc_id = of_match_device(mtk_ecc_dt_match, &pdev->dev);
-	if (!of_ecc_id)
-		return -ENODEV;
-
-	ecc->caps = of_ecc_id->data;
+	ecc->caps = of_device_get_match_data(dev);
 
 	max_eccdata_size = ecc->caps->num_ecc_strength - 1;
 	max_eccdata_size = ecc->caps->ecc_strength[max_eccdata_size];
diff --git a/drivers/mtd/nand/raw/mtk_nand.c b/drivers/mtd/nand/raw/mtk_nand.c
index 6977da3a26aa..75c845adb050 100644
--- a/drivers/mtd/nand/raw/mtk_nand.c
+++ b/drivers/mtd/nand/raw/mtk_nand.c
@@ -1434,7 +1434,6 @@ static int mtk_nfc_probe(struct platform_device *pdev)
 	struct device_node *np = dev->of_node;
 	struct mtk_nfc *nfc;
 	struct resource *res;
-	const struct of_device_id *of_nfc_id = NULL;
 	int ret, irq;
 
 	nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
@@ -1452,6 +1451,7 @@ static int mtk_nfc_probe(struct platform_device *pdev)
 	else if (!nfc->ecc)
 		return -ENODEV;
 
+	nfc->caps = of_device_get_match_data(dev);
 	nfc->dev = dev;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -1498,14 +1498,6 @@ static int mtk_nfc_probe(struct platform_device *pdev)
 		goto clk_disable;
 	}
 
-	of_nfc_id = of_match_device(mtk_nfc_id_table, &pdev->dev);
-	if (!of_nfc_id) {
-		ret = -ENODEV;
-		goto clk_disable;
-	}
-
-	nfc->caps = of_nfc_id->data;
-
 	platform_set_drvdata(pdev, nfc);
 
 	ret = mtk_nfc_nand_chips_init(dev, nfc);
diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c
index f28c3a555861..10c4f9919850 100644
--- a/drivers/mtd/nand/raw/nand_base.c
+++ b/drivers/mtd/nand/raw/nand_base.c
@@ -2174,7 +2174,6 @@ static int nand_set_features_op(struct nand_chip *chip, u8 feature,
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	const u8 *params = data;
 	int i, ret;
-	u8 status;
 
 	if (chip->exec_op) {
 		const struct nand_sdr_timings *sdr =
@@ -2188,26 +2187,18 @@ static int nand_set_features_op(struct nand_chip *chip, u8 feature,
 		};
 		struct nand_operation op = NAND_OPERATION(instrs);
 
-		ret = nand_exec_op(chip, &op);
-		if (ret)
-			return ret;
-
-		ret = nand_status_op(chip, &status);
-		if (ret)
-			return ret;
-	} else {
-		chip->cmdfunc(mtd, NAND_CMD_SET_FEATURES, feature, -1);
-		for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
-			chip->write_byte(mtd, params[i]);
+		return nand_exec_op(chip, &op);
+	}
 
-		ret = chip->waitfunc(mtd, chip);
-		if (ret < 0)
-			return ret;
+	chip->cmdfunc(mtd, NAND_CMD_SET_FEATURES, feature, -1);
+	for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
+		chip->write_byte(mtd, params[i]);
 
-		status = ret;
-	}
+	ret = chip->waitfunc(mtd, chip);
+	if (ret < 0)
+		return ret;
 
-	if (status & NAND_STATUS_FAIL)
+	if (ret & NAND_STATUS_FAIL)
 		return -EIO;
 
 	return 0;
@@ -5091,6 +5082,37 @@ static int nand_flash_detect_ext_param_page(struct nand_chip *chip,
 	return ret;
 }
 
+/*
+ * Recover data with bit-wise majority
+ */
+static void nand_bit_wise_majority(const void **srcbufs,
+				   unsigned int nsrcbufs,
+				   void *dstbuf,
+				   unsigned int bufsize)
+{
+	int i, j, k;
+
+	for (i = 0; i < bufsize; i++) {
+		u8 val = 0;
+
+		for (j = 0; j < 8; j++) {
+			unsigned int cnt = 0;
+
+			for (k = 0; k < nsrcbufs; k++) {
+				const u8 *srcbuf = srcbufs[k];
+
+				if (srcbuf[i] & BIT(j))
+					cnt++;
+			}
+
+			if (cnt > nsrcbufs / 2)
+				val |= BIT(j);
+		}
+
+		((u8 *)dstbuf)[i] = val;
+	}
+}
+
 /*
  * Check if the NAND chip is ONFI compliant, returns 1 if it is, 0 otherwise.
  */
@@ -5107,7 +5129,7 @@ static int nand_flash_detect_onfi(struct nand_chip *chip)
 		return 0;
 
 	/* ONFI chip: allocate a buffer to hold its parameter page */
-	p = kzalloc(sizeof(*p), GFP_KERNEL);
+	p = kzalloc((sizeof(*p) * 3), GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
 
@@ -5118,21 +5140,32 @@ static int nand_flash_detect_onfi(struct nand_chip *chip)
 	}
 
 	for (i = 0; i < 3; i++) {
-		ret = nand_read_data_op(chip, p, sizeof(*p), true);
+		ret = nand_read_data_op(chip, &p[i], sizeof(*p), true);
 		if (ret) {
 			ret = 0;
 			goto free_onfi_param_page;
 		}
 
-		if (onfi_crc16(ONFI_CRC_BASE, (uint8_t *)p, 254) ==
+		if (onfi_crc16(ONFI_CRC_BASE, (u8 *)&p[i], 254) ==
 				le16_to_cpu(p->crc)) {
+			if (i)
+				memcpy(p, &p[i], sizeof(*p));
 			break;
 		}
 	}
 
 	if (i == 3) {
-		pr_err("Could not find valid ONFI parameter page; aborting\n");
-		goto free_onfi_param_page;
+		const void *srcbufs[3] = {p, p + 1, p + 2};
+
+		pr_warn("Could not find a valid ONFI parameter page, trying bit-wise majority to recover it\n");
+		nand_bit_wise_majority(srcbufs, ARRAY_SIZE(srcbufs), p,
+				       sizeof(*p));
+
+		if (onfi_crc16(ONFI_CRC_BASE, (u8 *)p, 254) !=
+				le16_to_cpu(p->crc)) {
+			pr_err("ONFI parameter recovery failed, aborting\n");
+			goto free_onfi_param_page;
+		}
 	}
 
 	/* Check version */
@@ -6635,24 +6668,26 @@ EXPORT_SYMBOL(nand_scan_tail);
 #endif
 
 /**
- * nand_scan - [NAND Interface] Scan for the NAND device
+ * nand_scan_with_ids - [NAND Interface] Scan for the NAND device
  * @mtd: MTD device structure
  * @maxchips: number of chips to scan for
+ * @ids: optional flash IDs table
  *
  * This fills out all the uninitialized function pointers with the defaults.
  * The flash ID is read and the mtd/chip structures are filled with the
  * appropriate values.
  */
-int nand_scan(struct mtd_info *mtd, int maxchips)
+int nand_scan_with_ids(struct mtd_info *mtd, int maxchips,
+		       struct nand_flash_dev *ids)
 {
 	int ret;
 
-	ret = nand_scan_ident(mtd, maxchips, NULL);
+	ret = nand_scan_ident(mtd, maxchips, ids);
 	if (!ret)
 		ret = nand_scan_tail(mtd);
 	return ret;
 }
-EXPORT_SYMBOL(nand_scan);
+EXPORT_SYMBOL(nand_scan_with_ids);
 
 /**
  * nand_cleanup - [NAND Interface] Free resources held by the NAND device
diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c
index aad42812a353..d831a141a196 100644
--- a/drivers/mtd/nand/raw/sunxi_nand.c
+++ b/drivers/mtd/nand/raw/sunxi_nand.c
@@ -165,49 +165,16 @@
 
 #define NFC_MAX_CS		7
 
-/*
- * Ready/Busy detection type: describes the Ready/Busy detection modes
- *
- * @RB_NONE:	no external detection available, rely on STATUS command
- *		and software timeouts
- * @RB_NATIVE:	use sunxi NAND controller Ready/Busy support. The Ready/Busy
- *		pin of the NAND flash chip must be connected to one of the
- *		native NAND R/B pins (those which can be muxed to the NAND
- *		Controller)
- * @RB_GPIO:	use a simple GPIO to handle Ready/Busy status. The Ready/Busy
- *		pin of the NAND flash chip must be connected to a GPIO capable
- *		pin.
- */
-enum sunxi_nand_rb_type {
-	RB_NONE,
-	RB_NATIVE,
-	RB_GPIO,
-};
-
-/*
- * Ready/Busy structure: stores information related to Ready/Busy detection
- *
- * @type:	the Ready/Busy detection mode
- * @info:	information related to the R/B detection mode. Either a gpio
- *		id or a native R/B id (those supported by the NAND controller).
- */
-struct sunxi_nand_rb {
-	enum sunxi_nand_rb_type type;
-	union {
-		int gpio;
-		int nativeid;
-	} info;
-};
-
 /*
  * Chip Select structure: stores information related to NAND Chip Select
  *
  * @cs:		the NAND CS id used to communicate with a NAND Chip
- * @rb:		the Ready/Busy description
+ * @rb:		the Ready/Busy pin ID. -1 means no R/B pin connected to the
+ *		NFC
  */
 struct sunxi_nand_chip_sel {
 	u8 cs;
-	struct sunxi_nand_rb rb;
+	s8 rb;
 };
 
 /*
@@ -440,30 +407,19 @@ static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
 	struct nand_chip *nand = mtd_to_nand(mtd);
 	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 	struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
-	struct sunxi_nand_rb *rb;
-	int ret;
+	u32 mask;
 
 	if (sunxi_nand->selected < 0)
 		return 0;
 
-	rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
-
-	switch (rb->type) {
-	case RB_NATIVE:
-		ret = !!(readl(nfc->regs + NFC_REG_ST) &
-			 NFC_RB_STATE(rb->info.nativeid));
-		break;
-	case RB_GPIO:
-		ret = gpio_get_value(rb->info.gpio);
-		break;
-	case RB_NONE:
-	default:
-		ret = 0;
+	if (sunxi_nand->sels[sunxi_nand->selected].rb < 0) {
 		dev_err(nfc->dev, "cannot check R/B NAND status!\n");
-		break;
+		return 0;
 	}
 
-	return ret;
+	mask = NFC_RB_STATE(sunxi_nand->sels[sunxi_nand->selected].rb);
+
+	return !!(readl(nfc->regs + NFC_REG_ST) & mask);
 }
 
 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
@@ -488,12 +444,11 @@ static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
 
 		ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
 		       NFC_PAGE_SHIFT(nand->page_shift);
-		if (sel->rb.type == RB_NONE) {
+		if (sel->rb < 0) {
 			nand->dev_ready = NULL;
 		} else {
 			nand->dev_ready = sunxi_nfc_dev_ready;
-			if (sel->rb.type == RB_NATIVE)
-				ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
+			ctl |= NFC_RB_SEL(sel->rb);
 		}
 
 		writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
@@ -1946,26 +1901,10 @@ static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
 		chip->sels[i].cs = tmp;
 
 		if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
-		    tmp < 2) {
-			chip->sels[i].rb.type = RB_NATIVE;
-			chip->sels[i].rb.info.nativeid = tmp;
-		} else {
-			ret = of_get_named_gpio(np, "rb-gpios", i);
-			if (ret >= 0) {
-				tmp = ret;
-				chip->sels[i].rb.type = RB_GPIO;
-				chip->sels[i].rb.info.gpio = tmp;
-				ret = devm_gpio_request(dev, tmp, "nand-rb");
-				if (ret)
-					return ret;
-
-				ret = gpio_direction_input(tmp);
-				if (ret)
-					return ret;
-			} else {
-				chip->sels[i].rb.type = RB_NONE;
-			}
-		}
+		    tmp < 2)
+			chip->sels[i].rb = tmp;
+		else
+			chip->sels[i].rb = -1;
 	}
 
 	nand = &chip->nand;
diff --git a/drivers/mtd/nftlmount.c b/drivers/mtd/nftlmount.c
index a6fbfa4e5799..6281da3dadac 100644
--- a/drivers/mtd/nftlmount.c
+++ b/drivers/mtd/nftlmount.c
@@ -272,28 +272,37 @@ static int memcmpb(void *a, int c, int n)
 static int check_free_sectors(struct NFTLrecord *nftl, unsigned int address, int len,
 			      int check_oob)
 {
-	u8 buf[SECTORSIZE + nftl->mbd.mtd->oobsize];
 	struct mtd_info *mtd = nftl->mbd.mtd;
 	size_t retlen;
-	int i;
+	int i, ret;
+	u8 *buf;
+
+	buf = kmalloc(SECTORSIZE + mtd->oobsize, GFP_KERNEL);
+	if (!buf)
+		return -1;
 
+	ret = -1;
 	for (i = 0; i < len; i += SECTORSIZE) {
 		if (mtd_read(mtd, address, SECTORSIZE, &retlen, buf))
-			return -1;
+			goto out;
 		if (memcmpb(buf, 0xff, SECTORSIZE) != 0)
-			return -1;
+			goto out;
 
 		if (check_oob) {
 			if(nftl_read_oob(mtd, address, mtd->oobsize,
 					 &retlen, &buf[SECTORSIZE]) < 0)
-				return -1;
+				goto out;
 			if (memcmpb(buf + SECTORSIZE, 0xff, mtd->oobsize) != 0)
-				return -1;
+				goto out;
 		}
 		address += SECTORSIZE;
 	}
 
-	return 0;
+	ret = 0;
+
+out:
+	kfree(buf);
+	return ret;
 }
 
 /* NFTL_format: format a Erase Unit by erasing ALL Erase Zones in the Erase Unit and
diff --git a/drivers/mtd/spi-nor/Kconfig b/drivers/mtd/spi-nor/Kconfig
index 89da88e59121..6cc9c929ff57 100644
--- a/drivers/mtd/spi-nor/Kconfig
+++ b/drivers/mtd/spi-nor/Kconfig
@@ -71,7 +71,7 @@ config SPI_FSL_QUADSPI
 config SPI_HISI_SFC
 	tristate "Hisilicon SPI-NOR Flash Controller(SFC)"
 	depends on ARCH_HISI || COMPILE_TEST
-	depends on HAS_IOMEM && HAS_DMA
+	depends on HAS_IOMEM
 	help
 	  This enables support for hisilicon SPI-NOR flash controller.
 
@@ -90,7 +90,7 @@ config SPI_INTEL_SPI
 	tristate
 
 config SPI_INTEL_SPI_PCI
-	tristate "Intel PCH/PCU SPI flash PCI driver"
+	tristate "Intel PCH/PCU SPI flash PCI driver (DANGEROUS)"
 	depends on X86 && PCI
 	select SPI_INTEL_SPI
 	help
@@ -106,7 +106,7 @@ config SPI_INTEL_SPI_PCI
 	  will be called intel-spi-pci.
 
 config SPI_INTEL_SPI_PLATFORM
-	tristate "Intel PCH/PCU SPI flash platform driver"
+	tristate "Intel PCH/PCU SPI flash platform driver (DANGEROUS)"
 	depends on X86
 	select SPI_INTEL_SPI
 	help
diff --git a/drivers/mtd/spi-nor/cadence-quadspi.c b/drivers/mtd/spi-nor/cadence-quadspi.c
index 5872f31eaa60..c3f7aaa5d18f 100644
--- a/drivers/mtd/spi-nor/cadence-quadspi.c
+++ b/drivers/mtd/spi-nor/cadence-quadspi.c
@@ -18,6 +18,8 @@
 #include <linux/clk.h>
 #include <linux/completion.h>
 #include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
 #include <linux/err.h>
 #include <linux/errno.h>
 #include <linux/interrupt.h>
@@ -73,6 +75,10 @@ struct cqspi_st {
 	struct completion	transfer_complete;
 	struct mutex		bus_mutex;
 
+	struct dma_chan		*rx_chan;
+	struct completion	rx_dma_complete;
+	dma_addr_t		mmap_phys_base;
+
 	int			current_cs;
 	int			current_page_size;
 	int			current_erase_size;
@@ -930,11 +936,75 @@ static ssize_t cqspi_write(struct spi_nor *nor, loff_t to,
 	return len;
 }
 
+static void cqspi_rx_dma_callback(void *param)
+{
+	struct cqspi_st *cqspi = param;
+
+	complete(&cqspi->rx_dma_complete);
+}
+
+static int cqspi_direct_read_execute(struct spi_nor *nor, u_char *buf,
+				     loff_t from, size_t len)
+{
+	struct cqspi_flash_pdata *f_pdata = nor->priv;
+	struct cqspi_st *cqspi = f_pdata->cqspi;
+	enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
+	dma_addr_t dma_src = (dma_addr_t)cqspi->mmap_phys_base + from;
+	int ret = 0;
+	struct dma_async_tx_descriptor *tx;
+	dma_cookie_t cookie;
+	dma_addr_t dma_dst;
+
+	if (!cqspi->rx_chan || !virt_addr_valid(buf)) {
+		memcpy_fromio(buf, cqspi->ahb_base + from, len);
+		return 0;
+	}
+
+	dma_dst = dma_map_single(nor->dev, buf, len, DMA_DEV_TO_MEM);
+	if (dma_mapping_error(nor->dev, dma_dst)) {
+		dev_err(nor->dev, "dma mapping failed\n");
+		return -ENOMEM;
+	}
+	tx = dmaengine_prep_dma_memcpy(cqspi->rx_chan, dma_dst, dma_src,
+				       len, flags);
+	if (!tx) {
+		dev_err(nor->dev, "device_prep_dma_memcpy error\n");
+		ret = -EIO;
+		goto err_unmap;
+	}
+
+	tx->callback = cqspi_rx_dma_callback;
+	tx->callback_param = cqspi;
+	cookie = tx->tx_submit(tx);
+	reinit_completion(&cqspi->rx_dma_complete);
+
+	ret = dma_submit_error(cookie);
+	if (ret) {
+		dev_err(nor->dev, "dma_submit_error %d\n", cookie);
+		ret = -EIO;
+		goto err_unmap;
+	}
+
+	dma_async_issue_pending(cqspi->rx_chan);
+	ret = wait_for_completion_timeout(&cqspi->rx_dma_complete,
+					  msecs_to_jiffies(len));
+	if (ret <= 0) {
+		dmaengine_terminate_sync(cqspi->rx_chan);
+		dev_err(nor->dev, "DMA wait_for_completion_timeout\n");
+		ret = -ETIMEDOUT;
+		goto err_unmap;
+	}
+
+err_unmap:
+	dma_unmap_single(nor->dev, dma_dst, len, DMA_DEV_TO_MEM);
+
+	return 0;
+}
+
 static ssize_t cqspi_read(struct spi_nor *nor, loff_t from,
 			  size_t len, u_char *buf)
 {
 	struct cqspi_flash_pdata *f_pdata = nor->priv;
-	struct cqspi_st *cqspi = f_pdata->cqspi;
 	int ret;
 
 	ret = cqspi_set_protocol(nor, 1);
@@ -946,7 +1016,7 @@ static ssize_t cqspi_read(struct spi_nor *nor, loff_t from,
 		return ret;
 
 	if (f_pdata->use_direct_mode)
-		memcpy_fromio(buf, cqspi->ahb_base + from, len);
+		ret = cqspi_direct_read_execute(nor, buf, from, len);
 	else
 		ret = cqspi_indirect_read_execute(nor, buf, from, len);
 	if (ret)
@@ -1115,6 +1185,21 @@ static void cqspi_controller_init(struct cqspi_st *cqspi)
 	cqspi_controller_enable(cqspi, 1);
 }
 
+static void cqspi_request_mmap_dma(struct cqspi_st *cqspi)
+{
+	dma_cap_mask_t mask;
+
+	dma_cap_zero(mask);
+	dma_cap_set(DMA_MEMCPY, mask);
+
+	cqspi->rx_chan = dma_request_chan_by_mask(&mask);
+	if (IS_ERR(cqspi->rx_chan)) {
+		dev_err(&cqspi->pdev->dev, "No Rx DMA available\n");
+		cqspi->rx_chan = NULL;
+	}
+	init_completion(&cqspi->rx_dma_complete);
+}
+
 static int cqspi_setup_flash(struct cqspi_st *cqspi, struct device_node *np)
 {
 	const struct spi_nor_hwcaps hwcaps = {
@@ -1192,6 +1277,9 @@ static int cqspi_setup_flash(struct cqspi_st *cqspi, struct device_node *np)
 			f_pdata->use_direct_mode = true;
 			dev_dbg(nor->dev, "using direct mode for %s\n",
 				mtd->name);
+
+			if (!cqspi->rx_chan)
+				cqspi_request_mmap_dma(cqspi);
 		}
 	}
 
@@ -1252,6 +1340,7 @@ static int cqspi_probe(struct platform_device *pdev)
 		dev_err(dev, "Cannot remap AHB address.\n");
 		return PTR_ERR(cqspi->ahb_base);
 	}
+	cqspi->mmap_phys_base = (dma_addr_t)res_ahb->start;
 	cqspi->ahb_size = resource_size(res_ahb);
 
 	init_completion(&cqspi->transfer_complete);
@@ -1322,6 +1411,9 @@ static int cqspi_remove(struct platform_device *pdev)
 
 	cqspi_controller_enable(cqspi, 0);
 
+	if (cqspi->rx_chan)
+		dma_release_channel(cqspi->rx_chan);
+
 	clk_disable_unprepare(cqspi->clk);
 
 	pm_runtime_put_sync(&pdev->dev);
diff --git a/drivers/mtd/spi-nor/fsl-quadspi.c b/drivers/mtd/spi-nor/fsl-quadspi.c
index 3e3c0bbc45c0..7d9620c7ff6c 100644
--- a/drivers/mtd/spi-nor/fsl-quadspi.c
+++ b/drivers/mtd/spi-nor/fsl-quadspi.c
@@ -214,6 +214,7 @@ enum fsl_qspi_devtype {
 	FSL_QUADSPI_IMX7D,
 	FSL_QUADSPI_IMX6UL,
 	FSL_QUADSPI_LS1021A,
+	FSL_QUADSPI_LS2080A,
 };
 
 struct fsl_qspi_devtype_data {
@@ -267,6 +268,15 @@ static struct fsl_qspi_devtype_data ls1021a_data = {
 	.driver_data = 0,
 };
 
+static const struct fsl_qspi_devtype_data ls2080a_data = {
+	.devtype = FSL_QUADSPI_LS2080A,
+	.rxfifo = 128,
+	.txfifo = 64,
+	.ahb_buf_size = 1024,
+	.driver_data = QUADSPI_QUIRK_TKT253890,
+};
+
+
 #define FSL_QSPI_MAX_CHIP	4
 struct fsl_qspi {
 	struct spi_nor nor[FSL_QSPI_MAX_CHIP];
@@ -661,7 +671,7 @@ static void fsl_qspi_set_map_addr(struct fsl_qspi *q)
  * causes the controller to clear the buffer, and use the sequence pointed
  * by the QUADSPI_BFGENCR[SEQID] to initiate a read from the flash.
  */
-static void fsl_qspi_init_abh_read(struct fsl_qspi *q)
+static void fsl_qspi_init_ahb_read(struct fsl_qspi *q)
 {
 	void __iomem *base = q->iobase;
 	int seqid;
@@ -795,7 +805,7 @@ static int fsl_qspi_nor_setup_last(struct fsl_qspi *q)
 	fsl_qspi_init_lut(q);
 
 	/* Init for AHB read */
-	fsl_qspi_init_abh_read(q);
+	fsl_qspi_init_ahb_read(q);
 
 	return 0;
 }
@@ -806,6 +816,7 @@ static const struct of_device_id fsl_qspi_dt_ids[] = {
 	{ .compatible = "fsl,imx7d-qspi", .data = &imx7d_data, },
 	{ .compatible = "fsl,imx6ul-qspi", .data = &imx6ul_data, },
 	{ .compatible = "fsl,ls1021a-qspi", .data = (void *)&ls1021a_data, },
+	{ .compatible = "fsl,ls2080a-qspi", .data = &ls2080a_data, },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, fsl_qspi_dt_ids);
diff --git a/drivers/mtd/spi-nor/hisi-sfc.c b/drivers/mtd/spi-nor/hisi-sfc.c
index 04f9fb5cd9b6..dea7b0c4b339 100644
--- a/drivers/mtd/spi-nor/hisi-sfc.c
+++ b/drivers/mtd/spi-nor/hisi-sfc.c
@@ -112,7 +112,7 @@ struct hifmc_host {
 	u32 num_chip;
 };
 
-static inline int wait_op_finish(struct hifmc_host *host)
+static inline int hisi_spi_nor_wait_op_finish(struct hifmc_host *host)
 {
 	u32 reg;
 
@@ -120,7 +120,7 @@ static inline int wait_op_finish(struct hifmc_host *host)
 		(reg & FMC_INT_OP_DONE), 0, FMC_WAIT_TIMEOUT);
 }
 
-static int get_if_type(enum spi_nor_protocol proto)
+static int hisi_spi_nor_get_if_type(enum spi_nor_protocol proto)
 {
 	enum hifmc_iftype if_type;
 
@@ -208,7 +208,7 @@ static int hisi_spi_nor_op_reg(struct spi_nor *nor,
 	reg = FMC_OP_CMD1_EN | FMC_OP_REG_OP_START | optype;
 	writel(reg, host->regbase + FMC_OP);
 
-	return wait_op_finish(host);
+	return hisi_spi_nor_wait_op_finish(host);
 }
 
 static int hisi_spi_nor_read_reg(struct spi_nor *nor, u8 opcode, u8 *buf,
@@ -259,9 +259,9 @@ static int hisi_spi_nor_dma_transfer(struct spi_nor *nor, loff_t start_off,
 
 	reg = OP_CFG_FM_CS(priv->chipselect);
 	if (op_type == FMC_OP_READ)
-		if_type = get_if_type(nor->read_proto);
+		if_type = hisi_spi_nor_get_if_type(nor->read_proto);
 	else
-		if_type = get_if_type(nor->write_proto);
+		if_type = hisi_spi_nor_get_if_type(nor->write_proto);
 	reg |= OP_CFG_MEM_IF_TYPE(if_type);
 	if (op_type == FMC_OP_READ)
 		reg |= OP_CFG_DUMMY_NUM(nor->read_dummy >> 3);
@@ -274,7 +274,7 @@ static int hisi_spi_nor_dma_transfer(struct spi_nor *nor, loff_t start_off,
 		: OP_CTRL_WR_OPCODE(nor->program_opcode);
 	writel(reg, host->regbase + FMC_OP_DMA);
 
-	return wait_op_finish(host);
+	return hisi_spi_nor_wait_op_finish(host);
 }
 
 static ssize_t hisi_spi_nor_read(struct spi_nor *nor, loff_t from, size_t len,
diff --git a/drivers/mtd/spi-nor/intel-spi.c b/drivers/mtd/spi-nor/intel-spi.c
index 699951523179..d2cbfc27826e 100644
--- a/drivers/mtd/spi-nor/intel-spi.c
+++ b/drivers/mtd/spi-nor/intel-spi.c
@@ -136,6 +136,7 @@
  * @swseq_reg: Use SW sequencer in register reads/writes
  * @swseq_erase: Use SW sequencer in erase operation
  * @erase_64k: 64k erase supported
+ * @atomic_preopcode: Holds preopcode when atomic sequence is requested
  * @opcodes: Opcodes which are supported. This are programmed by BIOS
  *           before it locks down the controller.
  */
@@ -153,6 +154,7 @@ struct intel_spi {
 	bool swseq_reg;
 	bool swseq_erase;
 	bool erase_64k;
+	u8 atomic_preopcode;
 	u8 opcodes[8];
 };
 
@@ -285,7 +287,7 @@ static int intel_spi_wait_hw_busy(struct intel_spi *ispi)
 	u32 val;
 
 	return readl_poll_timeout(ispi->base + HSFSTS_CTL, val,
-				  !(val & HSFSTS_CTL_SCIP), 0,
+				  !(val & HSFSTS_CTL_SCIP), 40,
 				  INTEL_SPI_TIMEOUT * 1000);
 }
 
@@ -294,7 +296,7 @@ static int intel_spi_wait_sw_busy(struct intel_spi *ispi)
 	u32 val;
 
 	return readl_poll_timeout(ispi->sregs + SSFSTS_CTL, val,
-				  !(val & SSFSTS_CTL_SCIP), 0,
+				  !(val & SSFSTS_CTL_SCIP), 40,
 				  INTEL_SPI_TIMEOUT * 1000);
 }
 
@@ -474,7 +476,7 @@ static int intel_spi_sw_cycle(struct intel_spi *ispi, u8 opcode, int len,
 			      int optype)
 {
 	u32 val = 0, status;
-	u16 preop;
+	u8 atomic_preopcode;
 	int ret;
 
 	ret = intel_spi_opcode_index(ispi, opcode, optype);
@@ -484,17 +486,42 @@ static int intel_spi_sw_cycle(struct intel_spi *ispi, u8 opcode, int len,
 	if (len > INTEL_SPI_FIFO_SZ)
 		return -EINVAL;
 
+	/*
+	 * Always clear it after each SW sequencer operation regardless
+	 * of whether it is successful or not.
+	 */
+	atomic_preopcode = ispi->atomic_preopcode;
+	ispi->atomic_preopcode = 0;
+
 	/* Only mark 'Data Cycle' bit when there is data to be transferred */
 	if (len > 0)
 		val = ((len - 1) << SSFSTS_CTL_DBC_SHIFT) | SSFSTS_CTL_DS;
 	val |= ret << SSFSTS_CTL_COP_SHIFT;
 	val |= SSFSTS_CTL_FCERR | SSFSTS_CTL_FDONE;
 	val |= SSFSTS_CTL_SCGO;
-	preop = readw(ispi->sregs + PREOP_OPTYPE);
-	if (preop) {
-		val |= SSFSTS_CTL_ACS;
-		if (preop >> 8)
-			val |= SSFSTS_CTL_SPOP;
+	if (atomic_preopcode) {
+		u16 preop;
+
+		switch (optype) {
+		case OPTYPE_WRITE_NO_ADDR:
+		case OPTYPE_WRITE_WITH_ADDR:
+			/* Pick matching preopcode for the atomic sequence */
+			preop = readw(ispi->sregs + PREOP_OPTYPE);
+			if ((preop & 0xff) == atomic_preopcode)
+				; /* Do nothing */
+			else if ((preop >> 8) == atomic_preopcode)
+				val |= SSFSTS_CTL_SPOP;
+			else
+				return -EINVAL;
+
+			/* Enable atomic sequence */
+			val |= SSFSTS_CTL_ACS;
+			break;
+
+		default:
+			return -EINVAL;
+		}
+
 	}
 	writel(val, ispi->sregs + SSFSTS_CTL);
 
@@ -538,13 +565,31 @@ static int intel_spi_write_reg(struct spi_nor *nor, u8 opcode, u8 *buf, int len)
 
 	/*
 	 * This is handled with atomic operation and preop code in Intel
-	 * controller so skip it here now. If the controller is not locked,
-	 * program the opcode to the PREOP register for later use.
+	 * controller so we only verify that it is available. If the
+	 * controller is not locked, program the opcode to the PREOP
+	 * register for later use.
+	 *
+	 * When hardware sequencer is used there is no need to program
+	 * any opcodes (it handles them automatically as part of a command).
 	 */
 	if (opcode == SPINOR_OP_WREN) {
-		if (!ispi->locked)
+		u16 preop;
+
+		if (!ispi->swseq_reg)
+			return 0;
+
+		preop = readw(ispi->sregs + PREOP_OPTYPE);
+		if ((preop & 0xff) != opcode && (preop >> 8) != opcode) {
+			if (ispi->locked)
+				return -EINVAL;
 			writel(opcode, ispi->sregs + PREOP_OPTYPE);
+		}
 
+		/*
+		 * This enables atomic sequence on next SW sycle. Will
+		 * be cleared after next operation.
+		 */
+		ispi->atomic_preopcode = opcode;
 		return 0;
 	}
 
@@ -569,6 +614,13 @@ static ssize_t intel_spi_read(struct spi_nor *nor, loff_t from, size_t len,
 	u32 val, status;
 	ssize_t ret;
 
+	/*
+	 * Atomic sequence is not expected with HW sequencer reads. Make
+	 * sure it is cleared regardless.
+	 */
+	if (WARN_ON_ONCE(ispi->atomic_preopcode))
+		ispi->atomic_preopcode = 0;
+
 	switch (nor->read_opcode) {
 	case SPINOR_OP_READ:
 	case SPINOR_OP_READ_FAST:
@@ -627,6 +679,9 @@ static ssize_t intel_spi_write(struct spi_nor *nor, loff_t to, size_t len,
 	u32 val, status;
 	ssize_t ret;
 
+	/* Not needed with HW sequencer write, make sure it is cleared */
+	ispi->atomic_preopcode = 0;
+
 	while (len > 0) {
 		block_size = min_t(size_t, len, INTEL_SPI_FIFO_SZ);
 
@@ -707,6 +762,9 @@ static int intel_spi_erase(struct spi_nor *nor, loff_t offs)
 		return 0;
 	}
 
+	/* Not needed with HW sequencer erase, make sure it is cleared */
+	ispi->atomic_preopcode = 0;
+
 	while (len > 0) {
 		writel(offs, ispi->base + FADDR);
 
diff --git a/drivers/mtd/spi-nor/spi-nor.c b/drivers/mtd/spi-nor/spi-nor.c
index 5bfa36e95f35..d9c368c44194 100644
--- a/drivers/mtd/spi-nor/spi-nor.c
+++ b/drivers/mtd/spi-nor/spi-nor.c
@@ -284,6 +284,20 @@ static inline int set_4byte(struct spi_nor *nor, const struct flash_info *info,
 		if (need_wren)
 			write_disable(nor);
 
+		if (!status && !enable &&
+		    JEDEC_MFR(info) == SNOR_MFR_WINBOND) {
+			/*
+			 * On Winbond W25Q256FV, leaving 4byte mode causes
+			 * the Extended Address Register to be set to 1, so all
+			 * 3-byte-address reads come from the second 16M.
+			 * We must clear the register to enable normal behavior.
+			 */
+			write_enable(nor);
+			nor->cmd_buf[0] = 0;
+			nor->write_reg(nor, SPINOR_OP_WREAR, nor->cmd_buf, 1);
+			write_disable(nor);
+		}
+
 		return status;
 	default:
 		/* Spansion style */
@@ -980,6 +994,7 @@ static const struct flash_info spi_nor_ids[] = {
 	{ "en25q32b",   INFO(0x1c3016, 0, 64 * 1024,   64, 0) },
 	{ "en25p64",    INFO(0x1c2017, 0, 64 * 1024,  128, 0) },
 	{ "en25q64",    INFO(0x1c3017, 0, 64 * 1024,  128, SECT_4K) },
+	{ "en25qh32",   INFO(0x1c7016, 0, 64 * 1024,   64, 0) },
 	{ "en25qh128",  INFO(0x1c7018, 0, 64 * 1024,  256, 0) },
 	{ "en25qh256",  INFO(0x1c7019, 0, 64 * 1024,  512, 0) },
 	{ "en25s64",	INFO(0x1c3817, 0, 64 * 1024,  128, SECT_4K) },
@@ -1049,6 +1064,14 @@ static const struct flash_info spi_nor_ids[] = {
 			SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
 	{ "is25lp128",  INFO(0x9d6018, 0, 64 * 1024, 256,
 			SECT_4K | SPI_NOR_DUAL_READ) },
+	{ "is25lp256",  INFO(0x9d6019, 0, 64 * 1024, 512,
+			SECT_4K | SPI_NOR_DUAL_READ) },
+	{ "is25wp032",  INFO(0x9d7016, 0, 64 * 1024,  64,
+			SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
+	{ "is25wp064",  INFO(0x9d7017, 0, 64 * 1024, 128,
+			SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
+	{ "is25wp128",  INFO(0x9d7018, 0, 64 * 1024, 256,
+			SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
 
 	/* Macronix */
 	{ "mx25l512e",   INFO(0xc22010, 0, 64 * 1024,   1, SECT_4K) },
@@ -1087,6 +1110,7 @@ static const struct flash_info spi_nor_ids[] = {
 	{ "n25q512ax3",  INFO(0x20ba20, 0, 64 * 1024, 1024, SECT_4K | USE_FSR | SPI_NOR_QUAD_READ) },
 	{ "n25q00",      INFO(0x20ba21, 0, 64 * 1024, 2048, SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | NO_CHIP_ERASE) },
 	{ "n25q00a",     INFO(0x20bb21, 0, 64 * 1024, 2048, SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | NO_CHIP_ERASE) },
+	{ "mt25qu02g",   INFO(0x20bb22, 0, 64 * 1024, 4096, SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | NO_CHIP_ERASE) },
 
 	/* PMC */
 	{ "pm25lv512",   INFO(0,        0, 32 * 1024,    2, SECT_4K_PMC) },
@@ -1198,6 +1222,11 @@ static const struct flash_info spi_nor_ids[] = {
 			SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
 			SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB)
 	},
+	{
+		"w25q32jv", INFO(0xef7016, 0, 64 * 1024,  64,
+			SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
+			SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB)
+	},
 	{ "w25x64", INFO(0xef3017, 0, 64 * 1024, 128, SECT_4K) },
 	{ "w25q64", INFO(0xef4017, 0, 64 * 1024, 128, SECT_4K) },
 	{
@@ -1230,6 +1259,10 @@ static const struct flash_info spi_nor_ids[] = {
 	{ "3S400AN", S3AN_INFO(0x1f2400, 256, 264) },
 	{ "3S700AN", S3AN_INFO(0x1f2500, 512, 264) },
 	{ "3S1400AN", S3AN_INFO(0x1f2600, 512, 528) },
+
+	/* XMC (Wuhan Xinxin Semiconductor Manufacturing Corp.) */
+	{ "XM25QH64A", INFO(0x207017, 0, 64 * 1024, 128, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
+	{ "XM25QH128A", INFO(0x207018, 0, 64 * 1024, 256, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
 	{ },
 };
 
diff --git a/drivers/mtd/spi-nor/stm32-quadspi.c b/drivers/mtd/spi-nor/stm32-quadspi.c
index b3c7f6addba7..72553506a00b 100644
--- a/drivers/mtd/spi-nor/stm32-quadspi.c
+++ b/drivers/mtd/spi-nor/stm32-quadspi.c
@@ -656,7 +656,7 @@ static int stm32_qspi_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	rstc = devm_reset_control_get(dev, NULL);
+	rstc = devm_reset_control_get_exclusive(dev, NULL);
 	if (!IS_ERR(rstc)) {
 		reset_control_assert(rstc);
 		udelay(2);
diff --git a/drivers/mux/adg792a.c b/drivers/mux/adg792a.c
index 6a8725cf3d71..e8fc2fc1ab09 100644
--- a/drivers/mux/adg792a.c
+++ b/drivers/mux/adg792a.c
@@ -58,8 +58,7 @@ static const struct mux_control_ops adg792a_ops = {
 	.set = adg792a_set,
 };
 
-static int adg792a_probe(struct i2c_client *i2c,
-			 const struct i2c_device_id *id)
+static int adg792a_probe(struct i2c_client *i2c)
 {
 	struct device *dev = &i2c->dev;
 	struct mux_chip *mux_chip;
@@ -144,7 +143,7 @@ static struct i2c_driver adg792a_driver = {
 		.name		= "adg792a",
 		.of_match_table = of_match_ptr(adg792a_of_match),
 	},
-	.probe		= adg792a_probe,
+	.probe_new	= adg792a_probe,
 	.id_table	= adg792a_id,
 };
 module_i2c_driver(adg792a_driver);
diff --git a/drivers/mux/core.c b/drivers/mux/core.c
index d1271c1ee23c..0f18ef79dc53 100644
--- a/drivers/mux/core.c
+++ b/drivers/mux/core.c
@@ -286,6 +286,9 @@ EXPORT_SYMBOL_GPL(devm_mux_chip_register);
  */
 unsigned int mux_control_states(struct mux_control *mux)
 {
+	if (!mux)
+		return 0;
+
 	return mux->states;
 }
 EXPORT_SYMBOL_GPL(mux_control_states);
@@ -335,6 +338,9 @@ int mux_control_select(struct mux_control *mux, unsigned int state)
 {
 	int ret;
 
+	if (!mux)
+		return 0;
+
 	ret = down_killable(&mux->lock);
 	if (ret < 0)
 		return ret;
@@ -367,6 +373,9 @@ int mux_control_try_select(struct mux_control *mux, unsigned int state)
 {
 	int ret;
 
+	if (!mux)
+		return 0;
+
 	if (down_trylock(&mux->lock))
 		return -EBUSY;
 
@@ -395,6 +404,9 @@ int mux_control_deselect(struct mux_control *mux)
 {
 	int ret = 0;
 
+	if (!mux)
+		return 0;
+
 	if (mux->idle_state != MUX_IDLE_AS_IS &&
 	    mux->idle_state != mux->cached_state)
 		ret = mux_control_set(mux, mux->idle_state);
@@ -420,14 +432,8 @@ static struct mux_chip *of_find_mux_chip_by_node(struct device_node *np)
 	return dev ? to_mux_chip(dev) : NULL;
 }
 
-/**
- * mux_control_get() - Get the mux-control for a device.
- * @dev: The device that needs a mux-control.
- * @mux_name: The name identifying the mux-control.
- *
- * Return: A pointer to the mux-control, or an ERR_PTR with a negative errno.
- */
-struct mux_control *mux_control_get(struct device *dev, const char *mux_name)
+static struct mux_control *
+__mux_control_get(struct device *dev, const char *mux_name, bool optional)
 {
 	struct device_node *np = dev->of_node;
 	struct of_phandle_args args;
@@ -439,16 +445,22 @@ struct mux_control *mux_control_get(struct device *dev, const char *mux_name)
 	if (mux_name) {
 		index = of_property_match_string(np, "mux-control-names",
 						 mux_name);
+		if ((index == -EINVAL || index == -ENODATA) && optional)
+			return NULL;
 		if (index < 0) {
 			dev_err(dev, "mux controller '%s' not found\n",
 				mux_name);
 			return ERR_PTR(index);
 		}
+		/* OF does point to a mux, so it's no longer optional */
+		optional = false;
 	}
 
 	ret = of_parse_phandle_with_args(np,
 					 "mux-controls", "#mux-control-cells",
 					 index, &args);
+	if (ret == -ENOENT && optional)
+		return NULL;
 	if (ret) {
 		dev_err(dev, "%pOF: failed to get mux-control %s(%i)\n",
 			np, mux_name ?: "", index);
@@ -481,8 +493,35 @@ struct mux_control *mux_control_get(struct device *dev, const char *mux_name)
 
 	return &mux_chip->mux[controller];
 }
+
+/**
+ * mux_control_get() - Get the mux-control for a device.
+ * @dev: The device that needs a mux-control.
+ * @mux_name: The name identifying the mux-control.
+ *
+ * Return: A pointer to the mux-control, or an ERR_PTR with a negative errno.
+ */
+struct mux_control *mux_control_get(struct device *dev, const char *mux_name)
+{
+	return __mux_control_get(dev, mux_name, false);
+}
 EXPORT_SYMBOL_GPL(mux_control_get);
 
+/**
+ * mux_control_get_optional() - Get the optional mux-control for a device.
+ * @dev: The device that needs a mux-control.
+ * @mux_name: The name identifying the mux-control.
+ *
+ * Return: NULL if no mux with the provided name is found, a pointer to
+ * the named mux-control or an ERR_PTR with a negative errno.
+ */
+struct mux_control *
+mux_control_get_optional(struct device *dev, const char *mux_name)
+{
+	return __mux_control_get(dev, mux_name, true);
+}
+EXPORT_SYMBOL_GPL(mux_control_get_optional);
+
 /**
  * mux_control_put() - Put away the mux-control for good.
  * @mux: The mux-control to put away.
@@ -491,6 +530,9 @@ EXPORT_SYMBOL_GPL(mux_control_get);
  */
 void mux_control_put(struct mux_control *mux)
 {
+	if (!mux)
+		return;
+
 	put_device(&mux->chip->dev);
 }
 EXPORT_SYMBOL_GPL(mux_control_put);
@@ -502,16 +544,8 @@ static void devm_mux_control_release(struct device *dev, void *res)
 	mux_control_put(mux);
 }
 
-/**
- * devm_mux_control_get() - Get the mux-control for a device, with resource
- *			    management.
- * @dev: The device that needs a mux-control.
- * @mux_name: The name identifying the mux-control.
- *
- * Return: Pointer to the mux-control, or an ERR_PTR with a negative errno.
- */
-struct mux_control *devm_mux_control_get(struct device *dev,
-					 const char *mux_name)
+static struct mux_control *
+__devm_mux_control_get(struct device *dev, const char *mux_name, bool optional)
 {
 	struct mux_control **ptr, *mux;
 
@@ -519,8 +553,8 @@ struct mux_control *devm_mux_control_get(struct device *dev,
 	if (!ptr)
 		return ERR_PTR(-ENOMEM);
 
-	mux = mux_control_get(dev, mux_name);
-	if (IS_ERR(mux)) {
+	mux = __mux_control_get(dev, mux_name, optional);
+	if (IS_ERR_OR_NULL(mux)) {
 		devres_free(ptr);
 		return mux;
 	}
@@ -530,8 +564,38 @@ struct mux_control *devm_mux_control_get(struct device *dev,
 
 	return mux;
 }
+
+/**
+ * devm_mux_control_get() - Get the mux-control for a device, with resource
+ *			    management.
+ * @dev: The device that needs a mux-control.
+ * @mux_name: The name identifying the mux-control.
+ *
+ * Return: Pointer to the mux-control, or an ERR_PTR with a negative errno.
+ */
+struct mux_control *
+devm_mux_control_get(struct device *dev, const char *mux_name)
+{
+	return __devm_mux_control_get(dev, mux_name, false);
+}
 EXPORT_SYMBOL_GPL(devm_mux_control_get);
 
+/**
+ * devm_mux_control_get_optional() - Get the optional mux-control for a device,
+ *				     with resource management.
+ * @dev: The device that needs a mux-control.
+ * @mux_name: The name identifying the mux-control.
+ *
+ * Return: NULL if no mux with the provided name is found, a pointer to
+ * the named mux-control or an ERR_PTR with a negative errno.
+ */
+struct mux_control *
+devm_mux_control_get_optional(struct device *dev, const char *mux_name)
+{
+	return __devm_mux_control_get(dev, mux_name, true);
+}
+EXPORT_SYMBOL_GPL(devm_mux_control_get_optional);
+
 /*
  * Using subsys_initcall instead of module_init here to try to ensure - for
  * the non-modular case - that the subsystem is initialized when mux consumers
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index 891846655000..d03775100f7d 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -198,6 +198,7 @@ config VXLAN
 config GENEVE
        tristate "Generic Network Virtualization Encapsulation"
        depends on INET && NET_UDP_TUNNEL
+       depends on IPV6 || !IPV6
        select NET_IP_TUNNEL
        select GRO_CELLS
        ---help---
@@ -331,6 +332,7 @@ config VETH
 config VIRTIO_NET
 	tristate "Virtio network driver"
 	depends on VIRTIO
+	select NET_FAILOVER
 	---help---
 	  This is the virtual network driver for virtio.  It can be used with
 	  QEMU based VMMs (like KVM or Xen).  Say Y or M.
@@ -509,4 +511,16 @@ config NETDEVSIM
 	  To compile this driver as a module, choose M here: the module
 	  will be called netdevsim.
 
+config NET_FAILOVER
+	tristate "Failover driver"
+	select FAILOVER
+	help
+	  This provides an automated failover mechanism via APIs to create
+	  and destroy a failover master netdev and manages a primary and
+	  standby slave netdevs that get registered via the generic failover
+	  infrastructure. This can be used by paravirtual drivers to enable
+	  an alternate low latency datapath. It alsoenables live migration of
+	  a VM with direct attached VF by failing over to the paravirtual
+	  datapath when the VF is unplugged.
+
 endif # NETDEVICES
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index 91e67e375dd4..21cde7e78621 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -78,3 +78,4 @@ obj-$(CONFIG_FUJITSU_ES) += fjes/
 thunderbolt-net-y += thunderbolt.o
 obj-$(CONFIG_THUNDERBOLT_NET) += thunderbolt-net.o
 obj-$(CONFIG_NETDEVSIM) += netdevsim/
+obj-$(CONFIG_NET_FAILOVER) += net_failover.o
diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c
index 5eb0df2e5464..e82108c917a6 100644
--- a/drivers/net/bonding/bond_alb.c
+++ b/drivers/net/bonding/bond_alb.c
@@ -40,11 +40,6 @@
 #include <net/bonding.h>
 #include <net/bond_alb.h>
 
-
-
-static const u8 mac_bcast[ETH_ALEN + 2] __long_aligned = {
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff
-};
 static const u8 mac_v6_allmcast[ETH_ALEN + 2] __long_aligned = {
 	0x33, 0x33, 0x00, 0x00, 0x00, 0x01
 };
@@ -420,8 +415,7 @@ static void rlb_clear_slave(struct bonding *bond, struct slave *slave)
 
 			if (assigned_slave) {
 				rx_hash_table[index].slave = assigned_slave;
-				if (!ether_addr_equal_64bits(rx_hash_table[index].mac_dst,
-							     mac_bcast)) {
+				if (is_valid_ether_addr(rx_hash_table[index].mac_dst)) {
 					bond_info->rx_hashtbl[index].ntt = 1;
 					bond_info->rx_ntt = 1;
 					/* A slave has been removed from the
@@ -524,7 +518,7 @@ static void rlb_req_update_slave_clients(struct bonding *bond, struct slave *sla
 		client_info = &(bond_info->rx_hashtbl[hash_index]);
 
 		if ((client_info->slave == slave) &&
-		    !ether_addr_equal_64bits(client_info->mac_dst, mac_bcast)) {
+		    is_valid_ether_addr(client_info->mac_dst)) {
 			client_info->ntt = 1;
 			ntt = 1;
 		}
@@ -565,7 +559,7 @@ static void rlb_req_update_subnet_clients(struct bonding *bond, __be32 src_ip)
 		if ((client_info->ip_src == src_ip) &&
 		    !ether_addr_equal_64bits(client_info->slave->dev->dev_addr,
 					     bond->dev->dev_addr) &&
-		    !ether_addr_equal_64bits(client_info->mac_dst, mac_bcast)) {
+		    is_valid_ether_addr(client_info->mac_dst)) {
 			client_info->ntt = 1;
 			bond_info->rx_ntt = 1;
 		}
@@ -593,7 +587,7 @@ static struct slave *rlb_choose_channel(struct sk_buff *skb, struct bonding *bon
 		if ((client_info->ip_src == arp->ip_src) &&
 		    (client_info->ip_dst == arp->ip_dst)) {
 			/* the entry is already assigned to this client */
-			if (!ether_addr_equal_64bits(arp->mac_dst, mac_bcast)) {
+			if (!is_broadcast_ether_addr(arp->mac_dst)) {
 				/* update mac address from arp */
 				ether_addr_copy(client_info->mac_dst, arp->mac_dst);
 			}
@@ -641,7 +635,7 @@ static struct slave *rlb_choose_channel(struct sk_buff *skb, struct bonding *bon
 		ether_addr_copy(client_info->mac_src, arp->mac_src);
 		client_info->slave = assigned_slave;
 
-		if (!ether_addr_equal_64bits(client_info->mac_dst, mac_bcast)) {
+		if (is_valid_ether_addr(client_info->mac_dst)) {
 			client_info->ntt = 1;
 			bond->alb_info.rx_ntt = 1;
 		} else {
@@ -733,8 +727,10 @@ static void rlb_rebalance(struct bonding *bond)
 		assigned_slave = __rlb_next_rx_slave(bond);
 		if (assigned_slave && (client_info->slave != assigned_slave)) {
 			client_info->slave = assigned_slave;
-			client_info->ntt = 1;
-			ntt = 1;
+			if (!is_zero_ether_addr(client_info->mac_dst)) {
+				client_info->ntt = 1;
+				ntt = 1;
+			}
 		}
 	}
 
@@ -1319,8 +1315,8 @@ void bond_alb_deinitialize(struct bonding *bond)
 		rlb_deinitialize(bond);
 }
 
-static int bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond,
-			    struct slave *tx_slave)
+static netdev_tx_t bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond,
+				    struct slave *tx_slave)
 {
 	struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond));
 	struct ethhdr *eth_data = eth_hdr(skb);
@@ -1354,7 +1350,7 @@ static int bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond,
 	return NETDEV_TX_OK;
 }
 
-int bond_tlb_xmit(struct sk_buff *skb, struct net_device *bond_dev)
+netdev_tx_t bond_tlb_xmit(struct sk_buff *skb, struct net_device *bond_dev)
 {
 	struct bonding *bond = netdev_priv(bond_dev);
 	struct ethhdr *eth_data;
@@ -1392,7 +1388,7 @@ int bond_tlb_xmit(struct sk_buff *skb, struct net_device *bond_dev)
 	return bond_do_alb_xmit(skb, bond, tx_slave);
 }
 
-int bond_alb_xmit(struct sk_buff *skb, struct net_device *bond_dev)
+netdev_tx_t bond_alb_xmit(struct sk_buff *skb, struct net_device *bond_dev)
 {
 	struct bonding *bond = netdev_priv(bond_dev);
 	struct ethhdr *eth_data;
@@ -1412,9 +1408,9 @@ int bond_alb_xmit(struct sk_buff *skb, struct net_device *bond_dev)
 	case ETH_P_IP: {
 		const struct iphdr *iph = ip_hdr(skb);
 
-		if (ether_addr_equal_64bits(eth_data->h_dest, mac_bcast) ||
-		    (iph->daddr == ip_bcast) ||
-		    (iph->protocol == IPPROTO_IGMP)) {
+		if (is_broadcast_ether_addr(eth_data->h_dest) ||
+		    iph->daddr == ip_bcast ||
+		    iph->protocol == IPPROTO_IGMP) {
 			do_tx_balance = false;
 			break;
 		}
@@ -1426,7 +1422,7 @@ int bond_alb_xmit(struct sk_buff *skb, struct net_device *bond_dev)
 		/* IPv6 doesn't really use broadcast mac address, but leave
 		 * that here just in case.
 		 */
-		if (ether_addr_equal_64bits(eth_data->h_dest, mac_bcast)) {
+		if (is_broadcast_ether_addr(eth_data->h_dest)) {
 			do_tx_balance = false;
 			break;
 		}
@@ -1482,8 +1478,24 @@ int bond_alb_xmit(struct sk_buff *skb, struct net_device *bond_dev)
 	}
 
 	if (do_tx_balance) {
-		hash_index = _simple_hash(hash_start, hash_size);
-		tx_slave = tlb_choose_channel(bond, hash_index, skb->len);
+		if (bond->params.tlb_dynamic_lb) {
+			hash_index = _simple_hash(hash_start, hash_size);
+			tx_slave = tlb_choose_channel(bond, hash_index, skb->len);
+		} else {
+			/*
+			 * do_tx_balance means we are free to select the tx_slave
+			 * So we do exactly what tlb would do for hash selection
+			 */
+
+			struct bond_up_slave *slaves;
+			unsigned int count;
+
+			slaves = rcu_dereference(bond->slave_arr);
+			count = slaves ? READ_ONCE(slaves->count) : 0;
+			if (likely(count))
+				tx_slave = slaves->arr[bond_xmit_hash(bond, skb) %
+						       count];
+		}
 	}
 
 	return bond_do_alb_xmit(skb, bond, tx_slave);
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 1f1e97b26f95..bd53a71f6b00 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -159,7 +159,7 @@ module_param(min_links, int, 0);
 MODULE_PARM_DESC(min_links, "Minimum number of available links before turning on carrier");
 
 module_param(xmit_hash_policy, charp, 0);
-MODULE_PARM_DESC(xmit_hash_policy, "balance-xor and 802.3ad hashing method; "
+MODULE_PARM_DESC(xmit_hash_policy, "balance-alb, balance-tlb, balance-xor, 802.3ad hashing method; "
 				   "0 for layer 2 (default), 1 for layer 3+4, "
 				   "2 for layer 2+3, 3 for encap layer 2+3, "
 				   "4 for encap layer 3+4");
@@ -247,7 +247,7 @@ void bond_dev_queue_xmit(struct bonding *bond, struct sk_buff *skb,
 
 	BUILD_BUG_ON(sizeof(skb->queue_mapping) !=
 		     sizeof(qdisc_skb_cb(skb)->slave_dev_queue_mapping));
-	skb->queue_mapping = qdisc_skb_cb(skb)->slave_dev_queue_mapping;
+	skb_set_queue_mapping(skb, qdisc_skb_cb(skb)->slave_dev_queue_mapping);
 
 	if (unlikely(netpoll_tx_running(bond->dev)))
 		bond_netpoll_send_skb(bond_get_slave_by_dev(bond, slave_dev), skb);
@@ -1107,7 +1107,8 @@ static void bond_compute_features(struct bonding *bond)
 
 done:
 	bond_dev->vlan_features = vlan_features;
-	bond_dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL;
+	bond_dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL |
+				    NETIF_F_GSO_UDP_L4;
 	bond_dev->gso_max_segs = gso_max_segs;
 	netif_set_gso_max_size(bond_dev, gso_max_size);
 
@@ -1217,12 +1218,37 @@ static enum netdev_lag_tx_type bond_lag_tx_type(struct bonding *bond)
 	}
 }
 
+static enum netdev_lag_hash bond_lag_hash_type(struct bonding *bond,
+					       enum netdev_lag_tx_type type)
+{
+	if (type != NETDEV_LAG_TX_TYPE_HASH)
+		return NETDEV_LAG_HASH_NONE;
+
+	switch (bond->params.xmit_policy) {
+	case BOND_XMIT_POLICY_LAYER2:
+		return NETDEV_LAG_HASH_L2;
+	case BOND_XMIT_POLICY_LAYER34:
+		return NETDEV_LAG_HASH_L34;
+	case BOND_XMIT_POLICY_LAYER23:
+		return NETDEV_LAG_HASH_L23;
+	case BOND_XMIT_POLICY_ENCAP23:
+		return NETDEV_LAG_HASH_E23;
+	case BOND_XMIT_POLICY_ENCAP34:
+		return NETDEV_LAG_HASH_E34;
+	default:
+		return NETDEV_LAG_HASH_UNKNOWN;
+	}
+}
+
 static int bond_master_upper_dev_link(struct bonding *bond, struct slave *slave,
 				      struct netlink_ext_ack *extack)
 {
 	struct netdev_lag_upper_info lag_upper_info;
+	enum netdev_lag_tx_type type;
 
-	lag_upper_info.tx_type = bond_lag_tx_type(bond);
+	type = bond_lag_tx_type(bond);
+	lag_upper_info.tx_type = type;
+	lag_upper_info.hash_type = bond_lag_hash_type(bond, type);
 
 	return netdev_master_upper_dev_link(slave->dev, bond->dev, slave,
 					    &lag_upper_info, extack);
@@ -1735,7 +1761,7 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev,
 		unblock_netpoll_tx();
 	}
 
-	if (bond_mode_uses_xmit_hash(bond))
+	if (bond_mode_can_use_xmit_hash(bond))
 		bond_update_slave_arr(bond, NULL);
 
 	bond->nest_level = dev_get_nest_level(bond_dev);
@@ -1870,7 +1896,7 @@ static int __bond_release_one(struct net_device *bond_dev,
 	if (BOND_MODE(bond) == BOND_MODE_8023AD)
 		bond_3ad_unbind_slave(slave);
 
-	if (bond_mode_uses_xmit_hash(bond))
+	if (bond_mode_can_use_xmit_hash(bond))
 		bond_update_slave_arr(bond, slave);
 
 	netdev_info(bond_dev, "Releasing %s interface %s\n",
@@ -2137,6 +2163,24 @@ static int bond_miimon_inspect(struct bonding *bond)
 	return commit;
 }
 
+static void bond_miimon_link_change(struct bonding *bond,
+				    struct slave *slave,
+				    char link)
+{
+	switch (BOND_MODE(bond)) {
+	case BOND_MODE_8023AD:
+		bond_3ad_handle_link_change(slave, link);
+		break;
+	case BOND_MODE_TLB:
+	case BOND_MODE_ALB:
+		bond_alb_handle_link_change(bond, slave, link);
+		break;
+	case BOND_MODE_XOR:
+		bond_update_slave_arr(bond, NULL);
+		break;
+	}
+}
+
 static void bond_miimon_commit(struct bonding *bond)
 {
 	struct list_head *iter;
@@ -2178,16 +2222,7 @@ static void bond_miimon_commit(struct bonding *bond)
 				    slave->speed == SPEED_UNKNOWN ? 0 : slave->speed,
 				    slave->duplex ? "full" : "half");
 
-			/* notify ad that the link status has changed */
-			if (BOND_MODE(bond) == BOND_MODE_8023AD)
-				bond_3ad_handle_link_change(slave, BOND_LINK_UP);
-
-			if (bond_is_lb(bond))
-				bond_alb_handle_link_change(bond, slave,
-							    BOND_LINK_UP);
-
-			if (BOND_MODE(bond) == BOND_MODE_XOR)
-				bond_update_slave_arr(bond, NULL);
+			bond_miimon_link_change(bond, slave, BOND_LINK_UP);
 
 			if (!bond->curr_active_slave || slave == primary)
 				goto do_failover;
@@ -2209,16 +2244,7 @@ static void bond_miimon_commit(struct bonding *bond)
 			netdev_info(bond->dev, "link status definitely down for interface %s, disabling it\n",
 				    slave->dev->name);
 
-			if (BOND_MODE(bond) == BOND_MODE_8023AD)
-				bond_3ad_handle_link_change(slave,
-							    BOND_LINK_DOWN);
-
-			if (bond_is_lb(bond))
-				bond_alb_handle_link_change(bond, slave,
-							    BOND_LINK_DOWN);
-
-			if (BOND_MODE(bond) == BOND_MODE_XOR)
-				bond_update_slave_arr(bond, NULL);
+			bond_miimon_link_change(bond, slave, BOND_LINK_DOWN);
 
 			if (slave == rcu_access_pointer(bond->curr_active_slave))
 				goto do_failover;
@@ -3102,7 +3128,7 @@ static int bond_slave_netdev_event(unsigned long event,
 		 * events. If these (miimon/arpmon) parameters are configured
 		 * then array gets refreshed twice and that should be fine!
 		 */
-		if (bond_mode_uses_xmit_hash(bond))
+		if (bond_mode_can_use_xmit_hash(bond))
 			bond_update_slave_arr(bond, NULL);
 		break;
 	case NETDEV_CHANGEMTU:
@@ -3322,7 +3348,7 @@ static int bond_open(struct net_device *bond_dev)
 		 */
 		if (bond_alb_initialize(bond, (BOND_MODE(bond) == BOND_MODE_ALB)))
 			return -ENOMEM;
-		if (bond->params.tlb_dynamic_lb)
+		if (bond->params.tlb_dynamic_lb || BOND_MODE(bond) == BOND_MODE_ALB)
 			queue_delayed_work(bond->wq, &bond->alb_work, 0);
 	}
 
@@ -3341,7 +3367,7 @@ static int bond_open(struct net_device *bond_dev)
 		bond_3ad_initiate_agg_selection(bond, 1);
 	}
 
-	if (bond_mode_uses_xmit_hash(bond))
+	if (bond_mode_can_use_xmit_hash(bond))
 		bond_update_slave_arr(bond, NULL);
 
 	return 0;
@@ -3807,7 +3833,8 @@ static u32 bond_rr_gen_slave_id(struct bonding *bond)
 	return slave_id;
 }
 
-static int bond_xmit_roundrobin(struct sk_buff *skb, struct net_device *bond_dev)
+static netdev_tx_t bond_xmit_roundrobin(struct sk_buff *skb,
+					struct net_device *bond_dev)
 {
 	struct bonding *bond = netdev_priv(bond_dev);
 	struct iphdr *iph = ip_hdr(skb);
@@ -3843,7 +3870,8 @@ static int bond_xmit_roundrobin(struct sk_buff *skb, struct net_device *bond_dev
 /* In active-backup mode, we know that bond->curr_active_slave is always valid if
  * the bond has a usable interface.
  */
-static int bond_xmit_activebackup(struct sk_buff *skb, struct net_device *bond_dev)
+static netdev_tx_t bond_xmit_activebackup(struct sk_buff *skb,
+					  struct net_device *bond_dev)
 {
 	struct bonding *bond = netdev_priv(bond_dev);
 	struct slave *slave;
@@ -3892,7 +3920,7 @@ static void bond_slave_arr_handler(struct work_struct *work)
  * to determine the slave interface -
  * (a) BOND_MODE_8023AD
  * (b) BOND_MODE_XOR
- * (c) BOND_MODE_TLB && tlb_dynamic_lb == 0
+ * (c) (BOND_MODE_TLB || BOND_MODE_ALB) && tlb_dynamic_lb == 0
  *
  * The caller is expected to hold RTNL only and NO other lock!
  */
@@ -3945,6 +3973,11 @@ int bond_update_slave_arr(struct bonding *bond, struct slave *skipslave)
 			continue;
 		if (skipslave == slave)
 			continue;
+
+		netdev_dbg(bond->dev,
+			   "Adding slave dev %s to tx hash array[%d]\n",
+			   slave->dev->name, new_arr->count);
+
 		new_arr->arr[new_arr->count++] = slave;
 	}
 
@@ -3981,7 +4014,8 @@ int bond_update_slave_arr(struct bonding *bond, struct slave *skipslave)
  * usable slave array is formed in the control path. The xmit function
  * just calculates hash and sends the packet out.
  */
-static int bond_3ad_xor_xmit(struct sk_buff *skb, struct net_device *dev)
+static netdev_tx_t bond_3ad_xor_xmit(struct sk_buff *skb,
+				     struct net_device *dev)
 {
 	struct bonding *bond = netdev_priv(dev);
 	struct slave *slave;
@@ -4001,7 +4035,8 @@ static int bond_3ad_xor_xmit(struct sk_buff *skb, struct net_device *dev)
 }
 
 /* in broadcast mode, we send everything to all usable interfaces. */
-static int bond_xmit_broadcast(struct sk_buff *skb, struct net_device *bond_dev)
+static netdev_tx_t bond_xmit_broadcast(struct sk_buff *skb,
+				       struct net_device *bond_dev)
 {
 	struct bonding *bond = netdev_priv(bond_dev);
 	struct slave *slave = NULL;
@@ -4038,12 +4073,12 @@ static inline int bond_slave_override(struct bonding *bond,
 	struct slave *slave = NULL;
 	struct list_head *iter;
 
-	if (!skb->queue_mapping)
+	if (!skb_rx_queue_recorded(skb))
 		return 1;
 
 	/* Find out if any slaves have the same mapping as this skb. */
 	bond_for_each_slave_rcu(bond, slave, iter) {
-		if (slave->queue_id == skb->queue_mapping) {
+		if (slave->queue_id == skb_get_queue_mapping(skb)) {
 			if (bond_slave_is_up(slave) &&
 			    slave->link == BOND_LINK_UP) {
 				bond_dev_queue_xmit(bond, skb, slave->dev);
@@ -4069,7 +4104,7 @@ static u16 bond_select_queue(struct net_device *dev, struct sk_buff *skb,
 	u16 txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : 0;
 
 	/* Save the original txq to restore before passing to the driver */
-	qdisc_skb_cb(skb)->slave_dev_queue_mapping = skb->queue_mapping;
+	qdisc_skb_cb(skb)->slave_dev_queue_mapping = skb_get_queue_mapping(skb);
 
 	if (unlikely(txq >= dev->real_num_tx_queues)) {
 		do {
@@ -4259,7 +4294,7 @@ void bond_setup(struct net_device *bond_dev)
 				NETIF_F_HW_VLAN_CTAG_RX |
 				NETIF_F_HW_VLAN_CTAG_FILTER;
 
-	bond_dev->hw_features |= NETIF_F_GSO_ENCAP_ALL;
+	bond_dev->hw_features |= NETIF_F_GSO_ENCAP_ALL | NETIF_F_GSO_UDP_L4;
 	bond_dev->features |= bond_dev->hw_features;
 }
 
@@ -4320,9 +4355,9 @@ static int bond_check_params(struct bond_params *params)
 	}
 
 	if (xmit_hash_policy) {
-		if ((bond_mode != BOND_MODE_XOR) &&
-		    (bond_mode != BOND_MODE_8023AD) &&
-		    (bond_mode != BOND_MODE_TLB)) {
+		if (bond_mode == BOND_MODE_ROUNDROBIN ||
+		    bond_mode == BOND_MODE_ACTIVEBACKUP ||
+		    bond_mode == BOND_MODE_BROADCAST) {
 			pr_info("xmit_hash_policy param is irrelevant in mode %s\n",
 				bond_mode_name(bond_mode));
 		} else {
diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c
index 58c705f24f96..8a945c9341d6 100644
--- a/drivers/net/bonding/bond_options.c
+++ b/drivers/net/bonding/bond_options.c
@@ -395,7 +395,7 @@ static const struct bond_option bond_opts[BOND_OPT_LAST] = {
 		.id = BOND_OPT_TLB_DYNAMIC_LB,
 		.name = "tlb_dynamic_lb",
 		.desc = "Enable dynamic flow shuffling",
-		.unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_TLB)),
+		.unsuppmodes = BOND_MODE_ALL_EX(BIT(BOND_MODE_TLB) | BIT(BOND_MODE_ALB)),
 		.values = bond_tlb_dynamic_lb_tbl,
 		.flags = BOND_OPTFLAG_IFDOWN,
 		.set = bond_option_tlb_dynamic_lb_set,
diff --git a/drivers/net/bonding/bond_procfs.c b/drivers/net/bonding/bond_procfs.c
index 01059f1a7bca..9f7d83e827c3 100644
--- a/drivers/net/bonding/bond_procfs.c
+++ b/drivers/net/bonding/bond_procfs.c
@@ -10,7 +10,7 @@
 static void *bond_info_seq_start(struct seq_file *seq, loff_t *pos)
 	__acquires(RCU)
 {
-	struct bonding *bond = seq->private;
+	struct bonding *bond = PDE_DATA(file_inode(seq->file));
 	struct list_head *iter;
 	struct slave *slave;
 	loff_t off = 0;
@@ -29,7 +29,7 @@ static void *bond_info_seq_start(struct seq_file *seq, loff_t *pos)
 
 static void *bond_info_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 {
-	struct bonding *bond = seq->private;
+	struct bonding *bond = PDE_DATA(file_inode(seq->file));
 	struct list_head *iter;
 	struct slave *slave;
 	bool found = false;
@@ -56,7 +56,7 @@ static void bond_info_seq_stop(struct seq_file *seq, void *v)
 
 static void bond_info_show_master(struct seq_file *seq)
 {
-	struct bonding *bond = seq->private;
+	struct bonding *bond = PDE_DATA(file_inode(seq->file));
 	const struct bond_opt_value *optval;
 	struct slave *curr, *primary;
 	int i;
@@ -167,7 +167,7 @@ static void bond_info_show_master(struct seq_file *seq)
 static void bond_info_show_slave(struct seq_file *seq,
 				 const struct slave *slave)
 {
-	struct bonding *bond = seq->private;
+	struct bonding *bond = PDE_DATA(file_inode(seq->file));
 
 	seq_printf(seq, "\nSlave Interface: %s\n", slave->dev->name);
 	seq_printf(seq, "MII Status: %s\n", bond_slave_link_status(slave->link));
@@ -257,38 +257,14 @@ static const struct seq_operations bond_info_seq_ops = {
 	.show  = bond_info_seq_show,
 };
 
-static int bond_info_open(struct inode *inode, struct file *file)
-{
-	struct seq_file *seq;
-	int res;
-
-	res = seq_open(file, &bond_info_seq_ops);
-	if (!res) {
-		/* recover the pointer buried in proc_dir_entry data */
-		seq = file->private_data;
-		seq->private = PDE_DATA(inode);
-	}
-
-	return res;
-}
-
-static const struct file_operations bond_info_fops = {
-	.owner   = THIS_MODULE,
-	.open    = bond_info_open,
-	.read    = seq_read,
-	.llseek  = seq_lseek,
-	.release = seq_release,
-};
-
 void bond_create_proc_entry(struct bonding *bond)
 {
 	struct net_device *bond_dev = bond->dev;
 	struct bond_net *bn = net_generic(dev_net(bond_dev), bond_net_id);
 
 	if (bn->proc_dir) {
-		bond->proc_entry = proc_create_data(bond_dev->name,
-						    0444, bn->proc_dir,
-						    &bond_info_fops, bond);
+		bond->proc_entry = proc_create_seq_data(bond_dev->name, 0444,
+				bn->proc_dir, &bond_info_seq_ops, bond);
 		if (bond->proc_entry == NULL)
 			netdev_warn(bond_dev, "Cannot create /proc/net/%s/%s\n",
 				    DRV_NAME, bond_dev->name);
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 78616787f2a3..9f561fe505cb 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -806,16 +806,39 @@ static unsigned int b53_get_mib_size(struct b53_device *dev)
 		return B53_MIBS_SIZE;
 }
 
-void b53_get_strings(struct dsa_switch *ds, int port, uint8_t *data)
+static struct phy_device *b53_get_phy_device(struct dsa_switch *ds, int port)
+{
+	/* These ports typically do not have built-in PHYs */
+	switch (port) {
+	case B53_CPU_PORT_25:
+	case 7:
+	case B53_CPU_PORT:
+		return NULL;
+	}
+
+	return mdiobus_get_phy(ds->slave_mii_bus, port);
+}
+
+void b53_get_strings(struct dsa_switch *ds, int port, u32 stringset,
+		     uint8_t *data)
 {
 	struct b53_device *dev = ds->priv;
 	const struct b53_mib_desc *mibs = b53_get_mib(dev);
 	unsigned int mib_size = b53_get_mib_size(dev);
+	struct phy_device *phydev;
 	unsigned int i;
 
-	for (i = 0; i < mib_size; i++)
-		strlcpy(data + i * ETH_GSTRING_LEN,
-			mibs[i].name, ETH_GSTRING_LEN);
+	if (stringset == ETH_SS_STATS) {
+		for (i = 0; i < mib_size; i++)
+			strlcpy(data + i * ETH_GSTRING_LEN,
+				mibs[i].name, ETH_GSTRING_LEN);
+	} else if (stringset == ETH_SS_PHY_STATS) {
+		phydev = b53_get_phy_device(ds, port);
+		if (!phydev)
+			return;
+
+		phy_ethtool_get_strings(phydev, data);
+	}
 }
 EXPORT_SYMBOL(b53_get_strings);
 
@@ -852,11 +875,34 @@ void b53_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *data)
 }
 EXPORT_SYMBOL(b53_get_ethtool_stats);
 
-int b53_get_sset_count(struct dsa_switch *ds, int port)
+void b53_get_ethtool_phy_stats(struct dsa_switch *ds, int port, uint64_t *data)
+{
+	struct phy_device *phydev;
+
+	phydev = b53_get_phy_device(ds, port);
+	if (!phydev)
+		return;
+
+	phy_ethtool_get_stats(phydev, NULL, data);
+}
+EXPORT_SYMBOL(b53_get_ethtool_phy_stats);
+
+int b53_get_sset_count(struct dsa_switch *ds, int port, int sset)
 {
 	struct b53_device *dev = ds->priv;
+	struct phy_device *phydev;
+
+	if (sset == ETH_SS_STATS) {
+		return b53_get_mib_size(dev);
+	} else if (sset == ETH_SS_PHY_STATS) {
+		phydev = b53_get_phy_device(ds, port);
+		if (!phydev)
+			return 0;
+
+		return phy_ethtool_get_sset_count(phydev);
+	}
 
-	return b53_get_mib_size(dev);
+	return 0;
 }
 EXPORT_SYMBOL(b53_get_sset_count);
 
@@ -1477,7 +1523,7 @@ void b53_br_fast_age(struct dsa_switch *ds, int port)
 }
 EXPORT_SYMBOL(b53_br_fast_age);
 
-static bool b53_can_enable_brcm_tags(struct dsa_switch *ds, int port)
+static bool b53_possible_cpu_port(struct dsa_switch *ds, int port)
 {
 	/* Broadcom switches will accept enabling Broadcom tags on the
 	 * following ports: 5, 7 and 8, any other port is not supported
@@ -1489,10 +1535,19 @@ static bool b53_can_enable_brcm_tags(struct dsa_switch *ds, int port)
 		return true;
 	}
 
-	dev_warn(ds->dev, "Port %d is not Broadcom tag capable\n", port);
 	return false;
 }
 
+static bool b53_can_enable_brcm_tags(struct dsa_switch *ds, int port)
+{
+	bool ret = b53_possible_cpu_port(ds, port);
+
+	if (!ret)
+		dev_warn(ds->dev, "Port %d is not Broadcom tag capable\n",
+			 port);
+	return ret;
+}
+
 enum dsa_tag_protocol b53_get_tag_protocol(struct dsa_switch *ds, int port)
 {
 	struct b53_device *dev = ds->priv;
@@ -1650,6 +1705,7 @@ static const struct dsa_switch_ops b53_switch_ops = {
 	.get_strings		= b53_get_strings,
 	.get_ethtool_stats	= b53_get_ethtool_stats,
 	.get_sset_count		= b53_get_sset_count,
+	.get_ethtool_phy_stats	= b53_get_ethtool_phy_stats,
 	.phy_read		= b53_phy_read16,
 	.phy_write		= b53_phy_write16,
 	.adjust_link		= b53_adjust_link,
@@ -1954,6 +2010,15 @@ static int b53_switch_init(struct b53_device *dev)
 	dev->num_ports = dev->cpu_port + 1;
 	dev->enabled_ports |= BIT(dev->cpu_port);
 
+	/* Include non standard CPU port built-in PHYs to be probed */
+	if (is539x(dev) || is531x5(dev)) {
+		for (i = 0; i < dev->num_ports; i++) {
+			if (!(dev->ds->phys_mii_mask & BIT(i)) &&
+			    !b53_possible_cpu_port(dev->ds, i))
+				dev->ds->phys_mii_mask |= BIT(i);
+		}
+	}
+
 	dev->ports = devm_kzalloc(dev->dev,
 				  sizeof(struct b53_port) * dev->num_ports,
 				  GFP_KERNEL);
diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h
index 1187ebd79287..cc284a514de9 100644
--- a/drivers/net/dsa/b53/b53_priv.h
+++ b/drivers/net/dsa/b53/b53_priv.h
@@ -286,9 +286,11 @@ static inline int b53_switch_get_reset_gpio(struct b53_device *dev)
 /* Exported functions towards other drivers */
 void b53_imp_vlan_setup(struct dsa_switch *ds, int cpu_port);
 int b53_configure_vlan(struct dsa_switch *ds);
-void b53_get_strings(struct dsa_switch *ds, int port, uint8_t *data);
+void b53_get_strings(struct dsa_switch *ds, int port, u32 stringset,
+		     uint8_t *data);
 void b53_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *data);
-int b53_get_sset_count(struct dsa_switch *ds, int port);
+int b53_get_sset_count(struct dsa_switch *ds, int port, int sset);
+void b53_get_ethtool_phy_stats(struct dsa_switch *ds, int port, uint64_t *data);
 int b53_br_join(struct dsa_switch *ds, int port, struct net_device *bridge);
 void b53_br_leave(struct dsa_switch *ds, int port, struct net_device *bridge);
 void b53_br_set_stp_state(struct dsa_switch *ds, int port, u8 state);
diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c
index 0378eded31f2..02e8982519ce 100644
--- a/drivers/net/dsa/bcm_sf2.c
+++ b/drivers/net/dsa/bcm_sf2.c
@@ -16,6 +16,7 @@
 #include <linux/platform_device.h>
 #include <linux/phy.h>
 #include <linux/phy_fixed.h>
+#include <linux/phylink.h>
 #include <linux/mii.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
@@ -306,7 +307,8 @@ static int bcm_sf2_sw_mdio_write(struct mii_bus *bus, int addr, int regnum,
 
 static irqreturn_t bcm_sf2_switch_0_isr(int irq, void *dev_id)
 {
-	struct bcm_sf2_priv *priv = dev_id;
+	struct dsa_switch *ds = dev_id;
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 
 	priv->irq0_stat = intrl2_0_readl(priv, INTRL2_CPU_STATUS) &
 				~priv->irq0_mask;
@@ -317,16 +319,21 @@ static irqreturn_t bcm_sf2_switch_0_isr(int irq, void *dev_id)
 
 static irqreturn_t bcm_sf2_switch_1_isr(int irq, void *dev_id)
 {
-	struct bcm_sf2_priv *priv = dev_id;
+	struct dsa_switch *ds = dev_id;
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 
 	priv->irq1_stat = intrl2_1_readl(priv, INTRL2_CPU_STATUS) &
 				~priv->irq1_mask;
 	intrl2_1_writel(priv, priv->irq1_stat, INTRL2_CPU_CLEAR);
 
-	if (priv->irq1_stat & P_LINK_UP_IRQ(P7_IRQ_OFF))
-		priv->port_sts[7].link = 1;
-	if (priv->irq1_stat & P_LINK_DOWN_IRQ(P7_IRQ_OFF))
-		priv->port_sts[7].link = 0;
+	if (priv->irq1_stat & P_LINK_UP_IRQ(P7_IRQ_OFF)) {
+		priv->port_sts[7].link = true;
+		dsa_port_phylink_mac_change(ds, 7, true);
+	}
+	if (priv->irq1_stat & P_LINK_DOWN_IRQ(P7_IRQ_OFF)) {
+		priv->port_sts[7].link = false;
+		dsa_port_phylink_mac_change(ds, 7, false);
+	}
 
 	return IRQ_HANDLED;
 }
@@ -443,12 +450,8 @@ static int bcm_sf2_mdio_register(struct dsa_switch *ds)
 	priv->slave_mii_bus->parent = ds->dev->parent;
 	priv->slave_mii_bus->phy_mask = ~priv->indir_phy_mask;
 
-	if (dn)
-		err = of_mdiobus_register(priv->slave_mii_bus, dn);
-	else
-		err = mdiobus_register(priv->slave_mii_bus);
-
-	if (err)
+	err = of_mdiobus_register(priv->slave_mii_bus, dn);
+	if (err && dn)
 		of_node_put(dn);
 
 	return err;
@@ -473,13 +476,56 @@ static u32 bcm_sf2_sw_get_phy_flags(struct dsa_switch *ds, int port)
 	return priv->hw_params.gphy_rev;
 }
 
-static void bcm_sf2_sw_adjust_link(struct dsa_switch *ds, int port,
-				   struct phy_device *phydev)
+static void bcm_sf2_sw_validate(struct dsa_switch *ds, int port,
+				unsigned long *supported,
+				struct phylink_link_state *state)
+{
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
+
+	if (!phy_interface_mode_is_rgmii(state->interface) &&
+	    state->interface != PHY_INTERFACE_MODE_MII &&
+	    state->interface != PHY_INTERFACE_MODE_REVMII &&
+	    state->interface != PHY_INTERFACE_MODE_GMII &&
+	    state->interface != PHY_INTERFACE_MODE_INTERNAL &&
+	    state->interface != PHY_INTERFACE_MODE_MOCA) {
+		bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
+		dev_err(ds->dev,
+			"Unsupported interface: %d\n", state->interface);
+		return;
+	}
+
+	/* Allow all the expected bits */
+	phylink_set(mask, Autoneg);
+	phylink_set_port_modes(mask);
+	phylink_set(mask, Pause);
+	phylink_set(mask, Asym_Pause);
+
+	/* With the exclusion of MII and Reverse MII, we support Gigabit,
+	 * including Half duplex
+	 */
+	if (state->interface != PHY_INTERFACE_MODE_MII &&
+	    state->interface != PHY_INTERFACE_MODE_REVMII) {
+		phylink_set(mask, 1000baseT_Full);
+		phylink_set(mask, 1000baseT_Half);
+	}
+
+	phylink_set(mask, 10baseT_Half);
+	phylink_set(mask, 10baseT_Full);
+	phylink_set(mask, 100baseT_Half);
+	phylink_set(mask, 100baseT_Full);
+
+	bitmap_and(supported, supported, mask,
+		   __ETHTOOL_LINK_MODE_MASK_NBITS);
+	bitmap_and(state->advertising, state->advertising, mask,
+		   __ETHTOOL_LINK_MODE_MASK_NBITS);
+}
+
+static void bcm_sf2_sw_mac_config(struct dsa_switch *ds, int port,
+				  unsigned int mode,
+				  const struct phylink_link_state *state)
 {
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
-	struct ethtool_eee *p = &priv->dev->ports[port].eee;
 	u32 id_mode_dis = 0, port_mode;
-	const char *str = NULL;
 	u32 reg, offset;
 
 	if (priv->type == BCM7445_DEVICE_ID)
@@ -487,62 +533,48 @@ static void bcm_sf2_sw_adjust_link(struct dsa_switch *ds, int port,
 	else
 		offset = CORE_STS_OVERRIDE_GMIIP2_PORT(port);
 
-	switch (phydev->interface) {
+	switch (state->interface) {
 	case PHY_INTERFACE_MODE_RGMII:
-		str = "RGMII (no delay)";
 		id_mode_dis = 1;
+		/* fallthrough */
 	case PHY_INTERFACE_MODE_RGMII_TXID:
-		if (!str)
-			str = "RGMII (TX delay)";
 		port_mode = EXT_GPHY;
 		break;
 	case PHY_INTERFACE_MODE_MII:
-		str = "MII";
 		port_mode = EXT_EPHY;
 		break;
 	case PHY_INTERFACE_MODE_REVMII:
-		str = "Reverse MII";
 		port_mode = EXT_REVMII;
 		break;
 	default:
-		/* All other PHYs: internal and MoCA */
+		/* all other PHYs: internal and MoCA */
 		goto force_link;
 	}
 
-	/* If the link is down, just disable the interface to conserve power */
-	if (!phydev->link) {
-		reg = reg_readl(priv, REG_RGMII_CNTRL_P(port));
-		reg &= ~RGMII_MODE_EN;
-		reg_writel(priv, reg, REG_RGMII_CNTRL_P(port));
-		goto force_link;
-	}
-
-	/* Clear id_mode_dis bit, and the existing port mode, but
-	 * make sure we enable the RGMII block for data to pass
+	/* Clear id_mode_dis bit, and the existing port mode, let
+	 * RGMII_MODE_EN bet set by mac_link_{up,down}
 	 */
 	reg = reg_readl(priv, REG_RGMII_CNTRL_P(port));
 	reg &= ~ID_MODE_DIS;
 	reg &= ~(PORT_MODE_MASK << PORT_MODE_SHIFT);
 	reg &= ~(RX_PAUSE_EN | TX_PAUSE_EN);
 
-	reg |= port_mode | RGMII_MODE_EN;
+	reg |= port_mode;
 	if (id_mode_dis)
 		reg |= ID_MODE_DIS;
 
-	if (phydev->pause) {
-		if (phydev->asym_pause)
+	if (state->pause & MLO_PAUSE_TXRX_MASK) {
+		if (state->pause & MLO_PAUSE_TX)
 			reg |= TX_PAUSE_EN;
 		reg |= RX_PAUSE_EN;
 	}
 
 	reg_writel(priv, reg, REG_RGMII_CNTRL_P(port));
 
-	pr_info("Port %d configured for %s\n", port, str);
-
 force_link:
 	/* Force link settings detected from the PHY */
 	reg = SW_OVERRIDE;
-	switch (phydev->speed) {
+	switch (state->speed) {
 	case SPEED_1000:
 		reg |= SPDSTS_1000 << SPEED_SHIFT;
 		break;
@@ -551,33 +583,61 @@ static void bcm_sf2_sw_adjust_link(struct dsa_switch *ds, int port,
 		break;
 	}
 
-	if (phydev->link)
+	if (state->link)
 		reg |= LINK_STS;
-	if (phydev->duplex == DUPLEX_FULL)
+	if (state->duplex == DUPLEX_FULL)
 		reg |= DUPLX_MODE;
 
 	core_writel(priv, reg, offset);
-
-	if (!phydev->is_pseudo_fixed_link)
-		p->eee_enabled = b53_eee_init(ds, port, phydev);
 }
 
-static void bcm_sf2_sw_fixed_link_update(struct dsa_switch *ds, int port,
-					 struct fixed_phy_status *status)
+static void bcm_sf2_sw_mac_link_set(struct dsa_switch *ds, int port,
+				    phy_interface_t interface, bool link)
 {
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
-	u32 duplex, pause, offset;
 	u32 reg;
 
-	if (priv->type == BCM7445_DEVICE_ID)
-		offset = CORE_STS_OVERRIDE_GMIIP_PORT(port);
+	if (!phy_interface_mode_is_rgmii(interface) &&
+	    interface != PHY_INTERFACE_MODE_MII &&
+	    interface != PHY_INTERFACE_MODE_REVMII)
+		return;
+
+	/* If the link is down, just disable the interface to conserve power */
+	reg = reg_readl(priv, REG_RGMII_CNTRL_P(port));
+	if (link)
+		reg |= RGMII_MODE_EN;
 	else
-		offset = CORE_STS_OVERRIDE_GMIIP2_PORT(port);
+		reg &= ~RGMII_MODE_EN;
+	reg_writel(priv, reg, REG_RGMII_CNTRL_P(port));
+}
 
-	duplex = core_readl(priv, CORE_DUPSTS);
-	pause = core_readl(priv, CORE_PAUSESTS);
+static void bcm_sf2_sw_mac_link_down(struct dsa_switch *ds, int port,
+				     unsigned int mode,
+				     phy_interface_t interface)
+{
+	bcm_sf2_sw_mac_link_set(ds, port, interface, false);
+}
 
-	status->link = 0;
+static void bcm_sf2_sw_mac_link_up(struct dsa_switch *ds, int port,
+				   unsigned int mode,
+				   phy_interface_t interface,
+				   struct phy_device *phydev)
+{
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
+	struct ethtool_eee *p = &priv->dev->ports[port].eee;
+
+	bcm_sf2_sw_mac_link_set(ds, port, interface, true);
+
+	if (mode == MLO_AN_PHY && phydev)
+		p->eee_enabled = b53_eee_init(ds, port, phydev);
+}
+
+static void bcm_sf2_sw_fixed_state(struct dsa_switch *ds, int port,
+				   struct phylink_link_state *status)
+{
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
+
+	status->link = false;
 
 	/* MoCA port is special as we do not get link status from CORE_LNKSTS,
 	 * which means that we need to force the link at the port override
@@ -596,28 +656,10 @@ static void bcm_sf2_sw_fixed_link_update(struct dsa_switch *ds, int port,
 		 */
 		if (!status->link)
 			netif_carrier_off(ds->ports[port].slave);
-		status->duplex = 1;
+		status->duplex = DUPLEX_FULL;
 	} else {
-		status->link = 1;
-		status->duplex = !!(duplex & (1 << port));
-	}
-
-	reg = core_readl(priv, offset);
-	reg |= SW_OVERRIDE;
-	if (status->link)
-		reg |= LINK_STS;
-	else
-		reg &= ~LINK_STS;
-	core_writel(priv, reg, offset);
-
-	if ((pause & (1 << port)) &&
-	    (pause & (1 << (port + PAUSESTS_TX_PAUSE_SHIFT)))) {
-		status->asym_pause = 1;
-		status->pause = 1;
+		status->link = true;
 	}
-
-	if (pause & (1 << port))
-		status->pause = 1;
 }
 
 static void bcm_sf2_enable_acb(struct dsa_switch *ds)
@@ -859,9 +901,13 @@ static const struct dsa_switch_ops bcm_sf2_ops = {
 	.get_strings		= b53_get_strings,
 	.get_ethtool_stats	= b53_get_ethtool_stats,
 	.get_sset_count		= b53_get_sset_count,
+	.get_ethtool_phy_stats	= b53_get_ethtool_phy_stats,
 	.get_phy_flags		= bcm_sf2_sw_get_phy_flags,
-	.adjust_link		= bcm_sf2_sw_adjust_link,
-	.fixed_link_update	= bcm_sf2_sw_fixed_link_update,
+	.phylink_validate	= bcm_sf2_sw_validate,
+	.phylink_mac_config	= bcm_sf2_sw_mac_config,
+	.phylink_mac_link_down	= bcm_sf2_sw_mac_link_down,
+	.phylink_mac_link_up	= bcm_sf2_sw_mac_link_up,
+	.phylink_fixed_state	= bcm_sf2_sw_fixed_state,
 	.suspend		= bcm_sf2_sw_suspend,
 	.resume			= bcm_sf2_sw_resume,
 	.get_wol		= bcm_sf2_sw_get_wol,
@@ -1064,14 +1110,14 @@ static int bcm_sf2_sw_probe(struct platform_device *pdev)
 	bcm_sf2_intr_disable(priv);
 
 	ret = devm_request_irq(&pdev->dev, priv->irq0, bcm_sf2_switch_0_isr, 0,
-			       "switch_0", priv);
+			       "switch_0", ds);
 	if (ret < 0) {
 		pr_err("failed to request switch_0 IRQ\n");
 		goto out_mdio;
 	}
 
 	ret = devm_request_irq(&pdev->dev, priv->irq1, bcm_sf2_switch_1_isr, 0,
-			       "switch_1", priv);
+			       "switch_1", ds);
 	if (ret < 0) {
 		pr_err("failed to request switch_1 IRQ\n");
 		goto out_mdio;
diff --git a/drivers/net/dsa/dsa_loop.c b/drivers/net/dsa/dsa_loop.c
index f77be9f85cb3..816f34d64736 100644
--- a/drivers/net/dsa/dsa_loop.c
+++ b/drivers/net/dsa/dsa_loop.c
@@ -67,7 +67,7 @@ static struct phy_device *phydevs[PHY_MAX_ADDR];
 static enum dsa_tag_protocol dsa_loop_get_protocol(struct dsa_switch *ds,
 						   int port)
 {
-	dev_dbg(ds->dev, "%s\n", __func__);
+	dev_dbg(ds->dev, "%s: port: %d\n", __func__, port);
 
 	return DSA_TAG_PROTO_NONE;
 }
@@ -86,16 +86,23 @@ static int dsa_loop_setup(struct dsa_switch *ds)
 	return 0;
 }
 
-static int dsa_loop_get_sset_count(struct dsa_switch *ds, int port)
+static int dsa_loop_get_sset_count(struct dsa_switch *ds, int port, int sset)
 {
+	if (sset != ETH_SS_STATS && sset != ETH_SS_PHY_STATS)
+		return 0;
+
 	return __DSA_LOOP_CNT_MAX;
 }
 
-static void dsa_loop_get_strings(struct dsa_switch *ds, int port, uint8_t *data)
+static void dsa_loop_get_strings(struct dsa_switch *ds, int port,
+				 u32 stringset, uint8_t *data)
 {
 	struct dsa_loop_priv *ps = ds->priv;
 	unsigned int i;
 
+	if (stringset != ETH_SS_STATS && stringset != ETH_SS_PHY_STATS)
+		return;
+
 	for (i = 0; i < __DSA_LOOP_CNT_MAX; i++)
 		memcpy(data + i * ETH_GSTRING_LEN,
 		       ps->ports[port].mib[i].name, ETH_GSTRING_LEN);
@@ -117,8 +124,6 @@ static int dsa_loop_phy_read(struct dsa_switch *ds, int port, int regnum)
 	struct mii_bus *bus = ps->bus;
 	int ret;
 
-	dev_dbg(ds->dev, "%s\n", __func__);
-
 	ret = mdiobus_read_nested(bus, ps->port_base + port, regnum);
 	if (ret < 0)
 		ps->ports[port].mib[DSA_LOOP_PHY_READ_ERR].val++;
@@ -135,8 +140,6 @@ static int dsa_loop_phy_write(struct dsa_switch *ds, int port,
 	struct mii_bus *bus = ps->bus;
 	int ret;
 
-	dev_dbg(ds->dev, "%s\n", __func__);
-
 	ret = mdiobus_write_nested(bus, ps->port_base + port, regnum, value);
 	if (ret < 0)
 		ps->ports[port].mib[DSA_LOOP_PHY_WRITE_ERR].val++;
@@ -149,7 +152,8 @@ static int dsa_loop_phy_write(struct dsa_switch *ds, int port,
 static int dsa_loop_port_bridge_join(struct dsa_switch *ds, int port,
 				     struct net_device *bridge)
 {
-	dev_dbg(ds->dev, "%s\n", __func__);
+	dev_dbg(ds->dev, "%s: port: %d, bridge: %s\n",
+		__func__, port, bridge->name);
 
 	return 0;
 }
@@ -157,19 +161,22 @@ static int dsa_loop_port_bridge_join(struct dsa_switch *ds, int port,
 static void dsa_loop_port_bridge_leave(struct dsa_switch *ds, int port,
 				       struct net_device *bridge)
 {
-	dev_dbg(ds->dev, "%s\n", __func__);
+	dev_dbg(ds->dev, "%s: port: %d, bridge: %s\n",
+		__func__, port, bridge->name);
 }
 
 static void dsa_loop_port_stp_state_set(struct dsa_switch *ds, int port,
 					u8 state)
 {
-	dev_dbg(ds->dev, "%s\n", __func__);
+	dev_dbg(ds->dev, "%s: port: %d, state: %d\n",
+		__func__, port, state);
 }
 
 static int dsa_loop_port_vlan_filtering(struct dsa_switch *ds, int port,
 					bool vlan_filtering)
 {
-	dev_dbg(ds->dev, "%s\n", __func__);
+	dev_dbg(ds->dev, "%s: port: %d, vlan_filtering: %d\n",
+		__func__, port, vlan_filtering);
 
 	return 0;
 }
@@ -181,7 +188,8 @@ dsa_loop_port_vlan_prepare(struct dsa_switch *ds, int port,
 	struct dsa_loop_priv *ps = ds->priv;
 	struct mii_bus *bus = ps->bus;
 
-	dev_dbg(ds->dev, "%s\n", __func__);
+	dev_dbg(ds->dev, "%s: port: %d, vlan: %d-%d",
+		__func__, port, vlan->vid_begin, vlan->vid_end);
 
 	/* Just do a sleeping operation to make lockdep checks effective */
 	mdiobus_read(bus, ps->port_base + port, MII_BMSR);
@@ -202,8 +210,6 @@ static void dsa_loop_port_vlan_add(struct dsa_switch *ds, int port,
 	struct dsa_loop_vlan *vl;
 	u16 vid;
 
-	dev_dbg(ds->dev, "%s\n", __func__);
-
 	/* Just do a sleeping operation to make lockdep checks effective */
 	mdiobus_read(bus, ps->port_base + port, MII_BMSR);
 
@@ -215,6 +221,9 @@ static void dsa_loop_port_vlan_add(struct dsa_switch *ds, int port,
 			vl->untagged |= BIT(port);
 		else
 			vl->untagged &= ~BIT(port);
+
+		dev_dbg(ds->dev, "%s: port: %d vlan: %d, %stagged, pvid: %d\n",
+			__func__, port, vid, untagged ? "un" : "", pvid);
 	}
 
 	if (pvid)
@@ -230,8 +239,6 @@ static int dsa_loop_port_vlan_del(struct dsa_switch *ds, int port,
 	struct dsa_loop_vlan *vl;
 	u16 vid, pvid = ps->pvid;
 
-	dev_dbg(ds->dev, "%s\n", __func__);
-
 	/* Just do a sleeping operation to make lockdep checks effective */
 	mdiobus_read(bus, ps->port_base + port, MII_BMSR);
 
@@ -244,6 +251,9 @@ static int dsa_loop_port_vlan_del(struct dsa_switch *ds, int port,
 
 		if (pvid == vid)
 			pvid = 1;
+
+		dev_dbg(ds->dev, "%s: port: %d vlan: %d, %stagged, pvid: %d\n",
+			__func__, port, vid, untagged ? "un" : "", pvid);
 	}
 	ps->pvid = pvid;
 
@@ -256,6 +266,7 @@ static const struct dsa_switch_ops dsa_loop_driver = {
 	.get_strings		= dsa_loop_get_strings,
 	.get_ethtool_stats	= dsa_loop_get_ethtool_stats,
 	.get_sset_count		= dsa_loop_get_sset_count,
+	.get_ethtool_phy_stats	= dsa_loop_get_ethtool_stats,
 	.phy_read		= dsa_loop_phy_read,
 	.phy_write		= dsa_loop_phy_write,
 	.port_bridge_join	= dsa_loop_port_bridge_join,
diff --git a/drivers/net/dsa/lan9303-core.c b/drivers/net/dsa/lan9303-core.c
index fefa454f3e56..b4f6e1a67dd9 100644
--- a/drivers/net/dsa/lan9303-core.c
+++ b/drivers/net/dsa/lan9303-core.c
@@ -977,10 +977,14 @@ static const struct lan9303_mib_desc lan9303_mib[] = {
 	{ .offset = LAN9303_MAC_TX_LATECOL_0, .name = "TxLateCol", },
 };
 
-static void lan9303_get_strings(struct dsa_switch *ds, int port, uint8_t *data)
+static void lan9303_get_strings(struct dsa_switch *ds, int port,
+				u32 stringset, uint8_t *data)
 {
 	unsigned int u;
 
+	if (stringset != ETH_SS_STATS)
+		return;
+
 	for (u = 0; u < ARRAY_SIZE(lan9303_mib); u++) {
 		strncpy(data + u * ETH_GSTRING_LEN, lan9303_mib[u].name,
 			ETH_GSTRING_LEN);
@@ -1007,8 +1011,11 @@ static void lan9303_get_ethtool_stats(struct dsa_switch *ds, int port,
 	}
 }
 
-static int lan9303_get_sset_count(struct dsa_switch *ds, int port)
+static int lan9303_get_sset_count(struct dsa_switch *ds, int port, int sset)
 {
+	if (sset != ETH_SS_STATS)
+		return 0;
+
 	return ARRAY_SIZE(lan9303_mib);
 }
 
diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c
index bcb3e6c734f2..7210c49b7922 100644
--- a/drivers/net/dsa/microchip/ksz_common.c
+++ b/drivers/net/dsa/microchip/ksz_common.c
@@ -439,15 +439,22 @@ static void ksz_disable_port(struct dsa_switch *ds, int port,
 	ksz_port_cfg(dev, port, REG_PORT_CTRL_0, PORT_MAC_LOOPBACK, true);
 }
 
-static int ksz_sset_count(struct dsa_switch *ds, int port)
+static int ksz_sset_count(struct dsa_switch *ds, int port, int sset)
 {
+	if (sset != ETH_SS_STATS)
+		return 0;
+
 	return TOTAL_SWITCH_COUNTER_NUM;
 }
 
-static void ksz_get_strings(struct dsa_switch *ds, int port, uint8_t *buf)
+static void ksz_get_strings(struct dsa_switch *ds, int port,
+			    u32 stringset, uint8_t *buf)
 {
 	int i;
 
+	if (stringset != ETH_SS_STATS)
+		return;
+
 	for (i = 0; i < TOTAL_SWITCH_COUNTER_NUM; i++) {
 		memcpy(buf + i * ETH_GSTRING_LEN, mib_names[i].string,
 		       ETH_GSTRING_LEN);
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 80a4dbc3a499..62e486652e62 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -573,10 +573,14 @@ static int mt7530_phy_write(struct dsa_switch *ds, int port, int regnum,
 }
 
 static void
-mt7530_get_strings(struct dsa_switch *ds, int port, uint8_t *data)
+mt7530_get_strings(struct dsa_switch *ds, int port, u32 stringset,
+		   uint8_t *data)
 {
 	int i;
 
+	if (stringset != ETH_SS_STATS)
+		return;
+
 	for (i = 0; i < ARRAY_SIZE(mt7530_mib); i++)
 		strncpy(data + i * ETH_GSTRING_LEN, mt7530_mib[i].name,
 			ETH_GSTRING_LEN);
@@ -604,8 +608,11 @@ mt7530_get_ethtool_stats(struct dsa_switch *ds, int port,
 }
 
 static int
-mt7530_get_sset_count(struct dsa_switch *ds, int port)
+mt7530_get_sset_count(struct dsa_switch *ds, int port, int sset)
 {
+	if (sset != ETH_SS_STATS)
+		return 0;
+
 	return ARRAY_SIZE(mt7530_mib);
 }
 
diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c
index 5b4374f21d76..12df00f593b7 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.c
+++ b/drivers/net/dsa/mv88e6xxx/chip.c
@@ -28,9 +28,11 @@
 #include <linux/of_device.h>
 #include <linux/of_irq.h>
 #include <linux/of_mdio.h>
+#include <linux/platform_data/mv88e6xxx.h>
 #include <linux/netdevice.h>
 #include <linux/gpio/consumer.h>
 #include <linux/phy.h>
+#include <linux/phylink.h>
 #include <net/dsa.h>
 
 #include "chip.h"
@@ -580,6 +582,83 @@ static void mv88e6xxx_adjust_link(struct dsa_switch *ds, int port,
 		dev_err(ds->dev, "p%d: failed to configure MAC\n", port);
 }
 
+static void mv88e6xxx_validate(struct dsa_switch *ds, int port,
+			       unsigned long *supported,
+			       struct phylink_link_state *state)
+{
+}
+
+static int mv88e6xxx_link_state(struct dsa_switch *ds, int port,
+				struct phylink_link_state *state)
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+	int err;
+
+	mutex_lock(&chip->reg_lock);
+	err = mv88e6xxx_port_link_state(chip, port, state);
+	mutex_unlock(&chip->reg_lock);
+
+	return err;
+}
+
+static void mv88e6xxx_mac_config(struct dsa_switch *ds, int port,
+				 unsigned int mode,
+				 const struct phylink_link_state *state)
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+	int speed, duplex, link, err;
+
+	if (mode == MLO_AN_PHY)
+		return;
+
+	if (mode == MLO_AN_FIXED) {
+		link = LINK_FORCED_UP;
+		speed = state->speed;
+		duplex = state->duplex;
+	} else {
+		speed = SPEED_UNFORCED;
+		duplex = DUPLEX_UNFORCED;
+		link = LINK_UNFORCED;
+	}
+
+	mutex_lock(&chip->reg_lock);
+	err = mv88e6xxx_port_setup_mac(chip, port, link, speed, duplex,
+				       state->interface);
+	mutex_unlock(&chip->reg_lock);
+
+	if (err && err != -EOPNOTSUPP)
+		dev_err(ds->dev, "p%d: failed to configure MAC\n", port);
+}
+
+static void mv88e6xxx_mac_link_force(struct dsa_switch *ds, int port, int link)
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+	int err;
+
+	mutex_lock(&chip->reg_lock);
+	err = chip->info->ops->port_set_link(chip, port, link);
+	mutex_unlock(&chip->reg_lock);
+
+	if (err)
+		dev_err(chip->dev, "p%d: failed to force MAC link\n", port);
+}
+
+static void mv88e6xxx_mac_link_down(struct dsa_switch *ds, int port,
+				    unsigned int mode,
+				    phy_interface_t interface)
+{
+	if (mode == MLO_AN_FIXED)
+		mv88e6xxx_mac_link_force(ds, port, LINK_FORCED_DOWN);
+}
+
+static void mv88e6xxx_mac_link_up(struct dsa_switch *ds, int port,
+				  unsigned int mode, phy_interface_t interface,
+				  struct phy_device *phydev)
+{
+	if (mode == MLO_AN_FIXED)
+		mv88e6xxx_mac_link_force(ds, port, LINK_FORCED_UP);
+}
+
 static int mv88e6xxx_stats_snapshot(struct mv88e6xxx_chip *chip, int port)
 {
 	if (!chip->info->ops->stats_snapshot)
@@ -665,13 +744,13 @@ static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_chip *chip,
 	case STATS_TYPE_PORT:
 		err = mv88e6xxx_port_read(chip, port, s->reg, &reg);
 		if (err)
-			return UINT64_MAX;
+			return U64_MAX;
 
 		low = reg;
 		if (s->size == 4) {
 			err = mv88e6xxx_port_read(chip, port, s->reg + 1, &reg);
 			if (err)
-				return UINT64_MAX;
+				return U64_MAX;
 			high = reg;
 		}
 		break;
@@ -685,7 +764,7 @@ static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_chip *chip,
 			mv88e6xxx_g1_stats_read(chip, reg + 1, &high);
 		break;
 	default:
-		return UINT64_MAX;
+		return U64_MAX;
 	}
 	value = (((u64)high) << 16) | low;
 	return value;
@@ -742,11 +821,14 @@ static void mv88e6xxx_atu_vtu_get_strings(uint8_t *data)
 }
 
 static void mv88e6xxx_get_strings(struct dsa_switch *ds, int port,
-				  uint8_t *data)
+				  u32 stringset, uint8_t *data)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
 	int count = 0;
 
+	if (stringset != ETH_SS_STATS)
+		return;
+
 	mutex_lock(&chip->reg_lock);
 
 	if (chip->info->ops->stats_get_strings)
@@ -789,12 +871,15 @@ static int mv88e6320_stats_get_sset_count(struct mv88e6xxx_chip *chip)
 					      STATS_TYPE_BANK1);
 }
 
-static int mv88e6xxx_get_sset_count(struct dsa_switch *ds, int port)
+static int mv88e6xxx_get_sset_count(struct dsa_switch *ds, int port, int sset)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
 	int serdes_count = 0;
 	int count = 0;
 
+	if (sset != ETH_SS_STATS)
+		return 0;
+
 	mutex_lock(&chip->reg_lock);
 	if (chip->info->ops->stats_get_sset_count)
 		count = chip->info->ops->stats_get_sset_count(chip);
@@ -911,14 +996,6 @@ static void mv88e6xxx_get_ethtool_stats(struct dsa_switch *ds, int port,
 
 }
 
-static int mv88e6xxx_stats_set_histogram(struct mv88e6xxx_chip *chip)
-{
-	if (chip->info->ops->stats_set_histogram)
-		return chip->info->ops->stats_set_histogram(chip);
-
-	return 0;
-}
-
 static int mv88e6xxx_get_regs_len(struct dsa_switch *ds, int port)
 {
 	return 32 * sizeof(u16);
@@ -1020,6 +1097,76 @@ static void mv88e6xxx_port_stp_state_set(struct dsa_switch *ds, int port,
 		dev_err(ds->dev, "p%d: failed to update state\n", port);
 }
 
+static int mv88e6xxx_pri_setup(struct mv88e6xxx_chip *chip)
+{
+	int err;
+
+	if (chip->info->ops->ieee_pri_map) {
+		err = chip->info->ops->ieee_pri_map(chip);
+		if (err)
+			return err;
+	}
+
+	if (chip->info->ops->ip_pri_map) {
+		err = chip->info->ops->ip_pri_map(chip);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int mv88e6xxx_devmap_setup(struct mv88e6xxx_chip *chip)
+{
+	int target, port;
+	int err;
+
+	if (!chip->info->global2_addr)
+		return 0;
+
+	/* Initialize the routing port to the 32 possible target devices */
+	for (target = 0; target < 32; target++) {
+		port = 0x1f;
+		if (target < DSA_MAX_SWITCHES)
+			if (chip->ds->rtable[target] != DSA_RTABLE_NONE)
+				port = chip->ds->rtable[target];
+
+		err = mv88e6xxx_g2_device_mapping_write(chip, target, port);
+		if (err)
+			return err;
+	}
+
+	if (chip->info->ops->set_cascade_port) {
+		port = MV88E6XXX_CASCADE_PORT_MULTIPLE;
+		err = chip->info->ops->set_cascade_port(chip, port);
+		if (err)
+			return err;
+	}
+
+	err = mv88e6xxx_g1_set_device_number(chip, chip->ds->index);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int mv88e6xxx_trunk_setup(struct mv88e6xxx_chip *chip)
+{
+	/* Clear all trunk masks and mapping */
+	if (chip->info->global2_addr)
+		return mv88e6xxx_g2_trunk_clear(chip);
+
+	return 0;
+}
+
+static int mv88e6xxx_rmu_setup(struct mv88e6xxx_chip *chip)
+{
+	if (chip->info->ops->rmu_disable)
+		return chip->info->ops->rmu_disable(chip);
+
+	return 0;
+}
+
 static int mv88e6xxx_pot_setup(struct mv88e6xxx_chip *chip)
 {
 	if (chip->info->ops->pot_clear)
@@ -2113,53 +2260,16 @@ static int mv88e6xxx_set_ageing_time(struct dsa_switch *ds,
 	return err;
 }
 
-static int mv88e6xxx_g1_setup(struct mv88e6xxx_chip *chip)
+static int mv88e6xxx_stats_setup(struct mv88e6xxx_chip *chip)
 {
-	struct dsa_switch *ds = chip->ds;
 	int err;
 
-	/* Disable remote management, and set the switch's DSA device number. */
-	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_CTL2,
-				 MV88E6XXX_G1_CTL2_MULTIPLE_CASCADE |
-				 (ds->index & 0x1f));
-	if (err)
-		return err;
-
-	/* Configure the IP ToS mapping registers. */
-	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_0, 0x0000);
-	if (err)
-		return err;
-	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_1, 0x0000);
-	if (err)
-		return err;
-	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_2, 0x5555);
-	if (err)
-		return err;
-	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_3, 0x5555);
-	if (err)
-		return err;
-	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_4, 0xaaaa);
-	if (err)
-		return err;
-	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_5, 0xaaaa);
-	if (err)
-		return err;
-	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_6, 0xffff);
-	if (err)
-		return err;
-	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_7, 0xffff);
-	if (err)
-		return err;
-
-	/* Configure the IEEE 802.1p priority mapping register. */
-	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IEEE_PRI, 0xfa41);
-	if (err)
-		return err;
-
 	/* Initialize the statistics unit */
-	err = mv88e6xxx_stats_set_histogram(chip);
-	if (err)
-		return err;
+	if (chip->info->ops->stats_set_histogram) {
+		err = chip->info->ops->stats_set_histogram(chip);
+		if (err)
+			return err;
+	}
 
 	return mv88e6xxx_g1_stats_clear(chip);
 }
@@ -2185,18 +2295,6 @@ static int mv88e6xxx_setup(struct dsa_switch *ds)
 			goto unlock;
 	}
 
-	/* Setup Switch Global 1 Registers */
-	err = mv88e6xxx_g1_setup(chip);
-	if (err)
-		goto unlock;
-
-	/* Setup Switch Global 2 Registers */
-	if (chip->info->global2_addr) {
-		err = mv88e6xxx_g2_setup(chip);
-		if (err)
-			goto unlock;
-	}
-
 	err = mv88e6xxx_irl_setup(chip);
 	if (err)
 		goto unlock;
@@ -2229,10 +2327,26 @@ static int mv88e6xxx_setup(struct dsa_switch *ds)
 	if (err)
 		goto unlock;
 
+	err = mv88e6xxx_rmu_setup(chip);
+	if (err)
+		goto unlock;
+
 	err = mv88e6xxx_rsvd2cpu_setup(chip);
 	if (err)
 		goto unlock;
 
+	err = mv88e6xxx_trunk_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_devmap_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_pri_setup(chip);
+	if (err)
+		goto unlock;
+
 	/* Setup PTP Hardware Clock and timestamping */
 	if (chip->info->ptp_support) {
 		err = mv88e6xxx_ptp_setup(chip);
@@ -2244,6 +2358,10 @@ static int mv88e6xxx_setup(struct dsa_switch *ds)
 			goto unlock;
 	}
 
+	err = mv88e6xxx_stats_setup(chip);
+	if (err)
+		goto unlock;
+
 unlock:
 	mutex_unlock(&chip->reg_lock);
 
@@ -2337,10 +2455,7 @@ static int mv88e6xxx_mdio_register(struct mv88e6xxx_chip *chip,
 			return err;
 	}
 
-	if (np)
-		err = of_mdiobus_register(bus, np);
-	else
-		err = mdiobus_register(bus);
+	err = of_mdiobus_register(bus, np);
 	if (err) {
 		dev_err(chip->dev, "Cannot register MDIO bus (%d)\n", err);
 		mv88e6xxx_g2_irq_mdio_free(chip, bus);
@@ -2460,6 +2575,8 @@ static int mv88e6xxx_set_eeprom(struct dsa_switch *ds,
 
 static const struct mv88e6xxx_ops mv88e6085_ops = {
 	/* MV88E6XXX_FAMILY_6097 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.set_switch_mac = mv88e6xxx_g1_set_switch_mac,
 	.phy_read = mv88e6185_phy_ppu_read,
@@ -2488,12 +2605,16 @@ static const struct mv88e6xxx_ops mv88e6085_ops = {
 	.ppu_enable = mv88e6185_g1_ppu_enable,
 	.ppu_disable = mv88e6185_g1_ppu_disable,
 	.reset = mv88e6185_g1_reset,
+	.rmu_disable = mv88e6085_g1_rmu_disable,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
+	.serdes_power = mv88e6341_serdes_power,
 };
 
 static const struct mv88e6xxx_ops mv88e6095_ops = {
 	/* MV88E6XXX_FAMILY_6095 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.set_switch_mac = mv88e6xxx_g1_set_switch_mac,
 	.phy_read = mv88e6185_phy_ppu_read,
 	.phy_write = mv88e6185_phy_ppu_write,
@@ -2518,6 +2639,8 @@ static const struct mv88e6xxx_ops mv88e6095_ops = {
 
 static const struct mv88e6xxx_ops mv88e6097_ops = {
 	/* MV88E6XXX_FAMILY_6097 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
 	.phy_read = mv88e6xxx_g2_smi_phy_read,
@@ -2545,12 +2668,15 @@ static const struct mv88e6xxx_ops mv88e6097_ops = {
 	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
 	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6085_g1_rmu_disable,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
 };
 
 static const struct mv88e6xxx_ops mv88e6123_ops = {
 	/* MV88E6XXX_FAMILY_6165 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
 	.phy_read = mv88e6xxx_g2_smi_phy_read,
@@ -2579,6 +2705,8 @@ static const struct mv88e6xxx_ops mv88e6123_ops = {
 
 static const struct mv88e6xxx_ops mv88e6131_ops = {
 	/* MV88E6XXX_FAMILY_6185 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.set_switch_mac = mv88e6xxx_g1_set_switch_mac,
 	.phy_read = mv88e6185_phy_ppu_read,
 	.phy_write = mv88e6185_phy_ppu_write,
@@ -2603,6 +2731,7 @@ static const struct mv88e6xxx_ops mv88e6131_ops = {
 	.watchdog_ops = &mv88e6097_watchdog_ops,
 	.mgmt_rsvd2cpu = mv88e6185_g2_mgmt_rsvd2cpu,
 	.ppu_enable = mv88e6185_g1_ppu_enable,
+	.set_cascade_port = mv88e6185_g1_set_cascade_port,
 	.ppu_disable = mv88e6185_g1_ppu_disable,
 	.reset = mv88e6185_g1_reset,
 	.vtu_getnext = mv88e6185_g1_vtu_getnext,
@@ -2611,6 +2740,8 @@ static const struct mv88e6xxx_ops mv88e6131_ops = {
 
 static const struct mv88e6xxx_ops mv88e6141_ops = {
 	/* MV88E6XXX_FAMILY_6341 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.get_eeprom = mv88e6xxx_g2_get_eeprom8,
 	.set_eeprom = mv88e6xxx_g2_set_eeprom8,
@@ -2648,6 +2779,8 @@ static const struct mv88e6xxx_ops mv88e6141_ops = {
 
 static const struct mv88e6xxx_ops mv88e6161_ops = {
 	/* MV88E6XXX_FAMILY_6165 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
 	.phy_read = mv88e6xxx_g2_smi_phy_read,
@@ -2681,6 +2814,8 @@ static const struct mv88e6xxx_ops mv88e6161_ops = {
 
 static const struct mv88e6xxx_ops mv88e6165_ops = {
 	/* MV88E6XXX_FAMILY_6165 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
 	.phy_read = mv88e6165_phy_read,
@@ -2707,6 +2842,8 @@ static const struct mv88e6xxx_ops mv88e6165_ops = {
 
 static const struct mv88e6xxx_ops mv88e6171_ops = {
 	/* MV88E6XXX_FAMILY_6351 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
 	.phy_read = mv88e6xxx_g2_smi_phy_read,
@@ -2741,6 +2878,8 @@ static const struct mv88e6xxx_ops mv88e6171_ops = {
 
 static const struct mv88e6xxx_ops mv88e6172_ops = {
 	/* MV88E6XXX_FAMILY_6352 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
 	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
@@ -2771,6 +2910,7 @@ static const struct mv88e6xxx_ops mv88e6172_ops = {
 	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
 	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6352_g1_rmu_disable,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
 	.serdes_power = mv88e6352_serdes_power,
@@ -2779,6 +2919,8 @@ static const struct mv88e6xxx_ops mv88e6172_ops = {
 
 static const struct mv88e6xxx_ops mv88e6175_ops = {
 	/* MV88E6XXX_FAMILY_6351 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
 	.phy_read = mv88e6xxx_g2_smi_phy_read,
@@ -2809,10 +2951,13 @@ static const struct mv88e6xxx_ops mv88e6175_ops = {
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
+	.serdes_power = mv88e6341_serdes_power,
 };
 
 static const struct mv88e6xxx_ops mv88e6176_ops = {
 	/* MV88E6XXX_FAMILY_6352 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
 	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
@@ -2843,6 +2988,7 @@ static const struct mv88e6xxx_ops mv88e6176_ops = {
 	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
 	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6352_g1_rmu_disable,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
 	.serdes_power = mv88e6352_serdes_power,
@@ -2851,6 +2997,8 @@ static const struct mv88e6xxx_ops mv88e6176_ops = {
 
 static const struct mv88e6xxx_ops mv88e6185_ops = {
 	/* MV88E6XXX_FAMILY_6185 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.set_switch_mac = mv88e6xxx_g1_set_switch_mac,
 	.phy_read = mv88e6185_phy_ppu_read,
 	.phy_write = mv88e6185_phy_ppu_write,
@@ -2870,6 +3018,7 @@ static const struct mv88e6xxx_ops mv88e6185_ops = {
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
 	.mgmt_rsvd2cpu = mv88e6185_g2_mgmt_rsvd2cpu,
+	.set_cascade_port = mv88e6185_g1_set_cascade_port,
 	.ppu_enable = mv88e6185_g1_ppu_enable,
 	.ppu_disable = mv88e6185_g1_ppu_disable,
 	.reset = mv88e6185_g1_reset,
@@ -2907,6 +3056,7 @@ static const struct mv88e6xxx_ops mv88e6190_ops = {
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
 	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6390_g1_rmu_disable,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
 	.serdes_power = mv88e6390_serdes_power,
@@ -2943,6 +3093,7 @@ static const struct mv88e6xxx_ops mv88e6190x_ops = {
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
 	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6390_g1_rmu_disable,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
 	.serdes_power = mv88e6390_serdes_power,
@@ -2979,6 +3130,7 @@ static const struct mv88e6xxx_ops mv88e6191_ops = {
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
 	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6390_g1_rmu_disable,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
 	.serdes_power = mv88e6390_serdes_power,
@@ -2986,6 +3138,8 @@ static const struct mv88e6xxx_ops mv88e6191_ops = {
 
 static const struct mv88e6xxx_ops mv88e6240_ops = {
 	/* MV88E6XXX_FAMILY_6352 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
 	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
@@ -3016,6 +3170,7 @@ static const struct mv88e6xxx_ops mv88e6240_ops = {
 	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
 	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6352_g1_rmu_disable,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
 	.serdes_power = mv88e6352_serdes_power,
@@ -3054,6 +3209,7 @@ static const struct mv88e6xxx_ops mv88e6290_ops = {
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
 	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6390_g1_rmu_disable,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
 	.serdes_power = mv88e6390_serdes_power,
@@ -3063,6 +3219,8 @@ static const struct mv88e6xxx_ops mv88e6290_ops = {
 
 static const struct mv88e6xxx_ops mv88e6320_ops = {
 	/* MV88E6XXX_FAMILY_6320 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
 	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
@@ -3099,6 +3257,8 @@ static const struct mv88e6xxx_ops mv88e6320_ops = {
 
 static const struct mv88e6xxx_ops mv88e6321_ops = {
 	/* MV88E6XXX_FAMILY_6320 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
 	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
@@ -3133,6 +3293,8 @@ static const struct mv88e6xxx_ops mv88e6321_ops = {
 
 static const struct mv88e6xxx_ops mv88e6341_ops = {
 	/* MV88E6XXX_FAMILY_6341 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.get_eeprom = mv88e6xxx_g2_get_eeprom8,
 	.set_eeprom = mv88e6xxx_g2_set_eeprom8,
@@ -3171,6 +3333,8 @@ static const struct mv88e6xxx_ops mv88e6341_ops = {
 
 static const struct mv88e6xxx_ops mv88e6350_ops = {
 	/* MV88E6XXX_FAMILY_6351 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
 	.phy_read = mv88e6xxx_g2_smi_phy_read,
@@ -3205,6 +3369,8 @@ static const struct mv88e6xxx_ops mv88e6350_ops = {
 
 static const struct mv88e6xxx_ops mv88e6351_ops = {
 	/* MV88E6XXX_FAMILY_6351 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
 	.phy_read = mv88e6xxx_g2_smi_phy_read,
@@ -3240,6 +3406,8 @@ static const struct mv88e6xxx_ops mv88e6351_ops = {
 
 static const struct mv88e6xxx_ops mv88e6352_ops = {
 	/* MV88E6XXX_FAMILY_6352 */
+	.ieee_pri_map = mv88e6085_g1_ieee_pri_map,
+	.ip_pri_map = mv88e6085_g1_ip_pri_map,
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
 	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
@@ -3270,6 +3438,7 @@ static const struct mv88e6xxx_ops mv88e6352_ops = {
 	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
 	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6352_g1_rmu_disable,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
 	.serdes_power = mv88e6352_serdes_power,
@@ -3313,6 +3482,7 @@ static const struct mv88e6xxx_ops mv88e6390_ops = {
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
 	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6390_g1_rmu_disable,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
 	.serdes_power = mv88e6390_serdes_power,
@@ -3353,6 +3523,7 @@ static const struct mv88e6xxx_ops mv88e6390x_ops = {
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
 	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6390_g1_rmu_disable,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
 	.serdes_power = mv88e6390_serdes_power,
@@ -4125,6 +4296,11 @@ static const struct dsa_switch_ops mv88e6xxx_switch_ops = {
 	.get_tag_protocol	= mv88e6xxx_get_tag_protocol,
 	.setup			= mv88e6xxx_setup,
 	.adjust_link		= mv88e6xxx_adjust_link,
+	.phylink_validate	= mv88e6xxx_validate,
+	.phylink_mac_link_state	= mv88e6xxx_link_state,
+	.phylink_mac_config	= mv88e6xxx_mac_config,
+	.phylink_mac_link_down	= mv88e6xxx_mac_link_down,
+	.phylink_mac_link_up	= mv88e6xxx_mac_link_up,
 	.get_strings		= mv88e6xxx_get_strings,
 	.get_ethtool_stats	= mv88e6xxx_get_ethtool_stats,
 	.get_sset_count		= mv88e6xxx_get_sset_count,
@@ -4175,6 +4351,7 @@ static int mv88e6xxx_register_switch(struct mv88e6xxx_chip *chip)
 		return -ENOMEM;
 
 	ds->priv = chip;
+	ds->dev = dev;
 	ds->ops = &mv88e6xxx_switch_ops;
 	ds->ageing_time_min = chip->info->age_time_coeff;
 	ds->ageing_time_max = chip->info->age_time_coeff * U8_MAX;
@@ -4189,42 +4366,82 @@ static void mv88e6xxx_unregister_switch(struct mv88e6xxx_chip *chip)
 	dsa_unregister_switch(chip->ds);
 }
 
+static const void *pdata_device_get_match_data(struct device *dev)
+{
+	const struct of_device_id *matches = dev->driver->of_match_table;
+	const struct dsa_mv88e6xxx_pdata *pdata = dev->platform_data;
+
+	for (; matches->name[0] || matches->type[0] || matches->compatible[0];
+	     matches++) {
+		if (!strcmp(pdata->compatible, matches->compatible))
+			return matches->data;
+	}
+	return NULL;
+}
+
 static int mv88e6xxx_probe(struct mdio_device *mdiodev)
 {
+	struct dsa_mv88e6xxx_pdata *pdata = mdiodev->dev.platform_data;
+	const struct mv88e6xxx_info *compat_info = NULL;
 	struct device *dev = &mdiodev->dev;
 	struct device_node *np = dev->of_node;
-	const struct mv88e6xxx_info *compat_info;
 	struct mv88e6xxx_chip *chip;
-	u32 eeprom_len;
+	int port;
 	int err;
 
-	compat_info = of_device_get_match_data(dev);
+	if (np)
+		compat_info = of_device_get_match_data(dev);
+
+	if (pdata) {
+		compat_info = pdata_device_get_match_data(dev);
+
+		if (!pdata->netdev)
+			return -EINVAL;
+
+		for (port = 0; port < DSA_MAX_PORTS; port++) {
+			if (!(pdata->enabled_ports & (1 << port)))
+				continue;
+			if (strcmp(pdata->cd.port_names[port], "cpu"))
+				continue;
+			pdata->cd.netdev[port] = &pdata->netdev->dev;
+			break;
+		}
+	}
+
 	if (!compat_info)
 		return -EINVAL;
 
 	chip = mv88e6xxx_alloc_chip(dev);
-	if (!chip)
-		return -ENOMEM;
+	if (!chip) {
+		err = -ENOMEM;
+		goto out;
+	}
 
 	chip->info = compat_info;
 
 	err = mv88e6xxx_smi_init(chip, mdiodev->bus, mdiodev->addr);
 	if (err)
-		return err;
+		goto out;
 
 	chip->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
-	if (IS_ERR(chip->reset))
-		return PTR_ERR(chip->reset);
+	if (IS_ERR(chip->reset)) {
+		err = PTR_ERR(chip->reset);
+		goto out;
+	}
 
 	err = mv88e6xxx_detect(chip);
 	if (err)
-		return err;
+		goto out;
 
 	mv88e6xxx_phy_init(chip);
 
-	if (chip->info->ops->get_eeprom &&
-	    !of_property_read_u32(np, "eeprom-length", &eeprom_len))
-		chip->eeprom_len = eeprom_len;
+	if (chip->info->ops->get_eeprom) {
+		if (np)
+			of_property_read_u32(np, "eeprom-length",
+					     &chip->eeprom_len);
+		else
+			chip->eeprom_len = pdata->eeprom_len;
+	}
 
 	mutex_lock(&chip->reg_lock);
 	err = mv88e6xxx_switch_reset(chip);
@@ -4293,6 +4510,9 @@ static int mv88e6xxx_probe(struct mdio_device *mdiodev)
 		mv88e6xxx_irq_poll_free(chip);
 	mutex_unlock(&chip->reg_lock);
 out:
+	if (pdata)
+		dev_put(pdata->netdev);
+
 	return err;
 }
 
diff --git a/drivers/net/dsa/mv88e6xxx/chip.h b/drivers/net/dsa/mv88e6xxx/chip.h
index 12b7f4649b25..8ac3fbb15352 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.h
+++ b/drivers/net/dsa/mv88e6xxx/chip.h
@@ -21,10 +21,6 @@
 #include <linux/timecounter.h>
 #include <net/dsa.h>
 
-#ifndef UINT64_MAX
-#define UINT64_MAX		(u64)(~((u64)0))
-#endif
-
 #define SMI_CMD			0x00
 #define SMI_CMD_BUSY		BIT(15)
 #define SMI_CMD_CLAUSE_22	BIT(12)
@@ -242,7 +238,7 @@ struct mv88e6xxx_chip {
 	struct gpio_desc *reset;
 
 	/* set to size of eeprom if supported by the switch */
-	int		eeprom_len;
+	u32 eeprom_len;
 
 	/* List of mdio busses */
 	struct list_head mdios;
@@ -298,6 +294,9 @@ struct mv88e6xxx_mdio_bus {
 };
 
 struct mv88e6xxx_ops {
+	int (*ieee_pri_map)(struct mv88e6xxx_chip *chip);
+	int (*ip_pri_map)(struct mv88e6xxx_chip *chip);
+
 	/* Ingress Rate Limit unit (IRL) operations */
 	int (*irl_init_all)(struct mv88e6xxx_chip *chip, int port);
 
@@ -406,6 +405,12 @@ struct mv88e6xxx_ops {
 			       uint64_t *data);
 	int (*set_cpu_port)(struct mv88e6xxx_chip *chip, int port);
 	int (*set_egress_port)(struct mv88e6xxx_chip *chip, int port);
+
+#define MV88E6XXX_CASCADE_PORT_NONE		0xe
+#define MV88E6XXX_CASCADE_PORT_MULTIPLE		0xf
+
+	int (*set_cascade_port)(struct mv88e6xxx_chip *chip, int port);
+
 	const struct mv88e6xxx_irq_ops *watchdog_ops;
 
 	int (*mgmt_rsvd2cpu)(struct mv88e6xxx_chip *chip);
@@ -431,6 +436,9 @@ struct mv88e6xxx_ops {
 
 	/* Interface to the AVB/PTP registers */
 	const struct mv88e6xxx_avb_ops *avb_ops;
+
+	/* Remote Management Unit operations */
+	int (*rmu_disable)(struct mv88e6xxx_chip *chip);
 };
 
 struct mv88e6xxx_irq_ops {
diff --git a/drivers/net/dsa/mv88e6xxx/global1.c b/drivers/net/dsa/mv88e6xxx/global1.c
index b43bd6476632..d721ccf7d8be 100644
--- a/drivers/net/dsa/mv88e6xxx/global1.c
+++ b/drivers/net/dsa/mv88e6xxx/global1.c
@@ -241,6 +241,64 @@ int mv88e6185_g1_ppu_disable(struct mv88e6xxx_chip *chip)
 	return mv88e6185_g1_wait_ppu_disabled(chip);
 }
 
+/* Offset 0x10: IP-PRI Mapping Register 0
+ * Offset 0x11: IP-PRI Mapping Register 1
+ * Offset 0x12: IP-PRI Mapping Register 2
+ * Offset 0x13: IP-PRI Mapping Register 3
+ * Offset 0x14: IP-PRI Mapping Register 4
+ * Offset 0x15: IP-PRI Mapping Register 5
+ * Offset 0x16: IP-PRI Mapping Register 6
+ * Offset 0x17: IP-PRI Mapping Register 7
+ */
+
+int mv88e6085_g1_ip_pri_map(struct mv88e6xxx_chip *chip)
+{
+	int err;
+
+	/* Reset the IP TOS/DiffServ/Traffic priorities to defaults */
+	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_0, 0x0000);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_1, 0x0000);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_2, 0x5555);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_3, 0x5555);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_4, 0xaaaa);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_5, 0xaaaa);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_6, 0xffff);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IP_PRI_7, 0xffff);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+/* Offset 0x18: IEEE-PRI Register */
+
+int mv88e6085_g1_ieee_pri_map(struct mv88e6xxx_chip *chip)
+{
+	/* Reset the IEEE Tag priorities to defaults */
+	return mv88e6xxx_g1_write(chip, MV88E6XXX_G1_IEEE_PRI, 0xfa41);
+}
+
 /* Offset 0x1a: Monitor Control */
 /* Offset 0x1a: Monitor & MGMT Control on some devices */
 
@@ -350,20 +408,59 @@ int mv88e6390_g1_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip)
 
 /* Offset 0x1c: Global Control 2 */
 
-int mv88e6390_g1_stats_set_histogram(struct mv88e6xxx_chip *chip)
+static int mv88e6xxx_g1_ctl2_mask(struct mv88e6xxx_chip *chip, u16 mask,
+				  u16 val)
 {
-	u16 val;
+	u16 reg;
 	int err;
 
-	err = mv88e6xxx_g1_read(chip, MV88E6XXX_G1_CTL2, &val);
+	err = mv88e6xxx_g1_read(chip, MV88E6XXX_G1_CTL2, &reg);
 	if (err)
 		return err;
 
-	val |= MV88E6XXX_G1_CTL2_HIST_RX_TX;
+	reg &= ~mask;
+	reg |= val & mask;
 
-	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_CTL2, val);
+	return mv88e6xxx_g1_write(chip, MV88E6XXX_G1_CTL2, reg);
+}
 
-	return err;
+int mv88e6185_g1_set_cascade_port(struct mv88e6xxx_chip *chip, int port)
+{
+	const u16 mask = MV88E6185_G1_CTL2_CASCADE_PORT_MASK;
+
+	return mv88e6xxx_g1_ctl2_mask(chip, mask, port << __bf_shf(mask));
+}
+
+int mv88e6085_g1_rmu_disable(struct mv88e6xxx_chip *chip)
+{
+	return mv88e6xxx_g1_ctl2_mask(chip, MV88E6085_G1_CTL2_P10RM |
+				      MV88E6085_G1_CTL2_RM_ENABLE, 0);
+}
+
+int mv88e6352_g1_rmu_disable(struct mv88e6xxx_chip *chip)
+{
+	return mv88e6xxx_g1_ctl2_mask(chip, MV88E6352_G1_CTL2_RMU_MODE_MASK,
+				      MV88E6352_G1_CTL2_RMU_MODE_DISABLED);
+}
+
+int mv88e6390_g1_rmu_disable(struct mv88e6xxx_chip *chip)
+{
+	return mv88e6xxx_g1_ctl2_mask(chip, MV88E6390_G1_CTL2_RMU_MODE_MASK,
+				      MV88E6390_G1_CTL2_RMU_MODE_DISABLED);
+}
+
+int mv88e6390_g1_stats_set_histogram(struct mv88e6xxx_chip *chip)
+{
+	return mv88e6xxx_g1_ctl2_mask(chip, MV88E6390_G1_CTL2_HIST_MODE_MASK,
+				      MV88E6390_G1_CTL2_HIST_MODE_RX |
+				      MV88E6390_G1_CTL2_HIST_MODE_TX);
+}
+
+int mv88e6xxx_g1_set_device_number(struct mv88e6xxx_chip *chip, int index)
+{
+	return mv88e6xxx_g1_ctl2_mask(chip,
+				      MV88E6XXX_G1_CTL2_DEVICE_NUMBER_MASK,
+				      index);
 }
 
 /* Offset 0x1d: Statistics Operation 2 */
diff --git a/drivers/net/dsa/mv88e6xxx/global1.h b/drivers/net/dsa/mv88e6xxx/global1.h
index 6aee7316fea6..7c791c1da4b9 100644
--- a/drivers/net/dsa/mv88e6xxx/global1.h
+++ b/drivers/net/dsa/mv88e6xxx/global1.h
@@ -201,11 +201,35 @@
 
 /* Offset 0x1C: Global Control 2 */
 #define MV88E6XXX_G1_CTL2			0x1c
-#define MV88E6XXX_G1_CTL2_NO_CASCADE		0xe000
-#define MV88E6XXX_G1_CTL2_MULTIPLE_CASCADE	0xf000
-#define MV88E6XXX_G1_CTL2_HIST_RX		0x0040
-#define MV88E6XXX_G1_CTL2_HIST_TX		0x0080
-#define MV88E6XXX_G1_CTL2_HIST_RX_TX		0x00c0
+#define MV88E6185_G1_CTL2_CASCADE_PORT_MASK	0xf000
+#define MV88E6185_G1_CTL2_CASCADE_PORT_NONE	0xe000
+#define MV88E6185_G1_CTL2_CASCADE_PORT_MULTI	0xf000
+#define MV88E6352_G1_CTL2_HEADER_TYPE_MASK	0xc000
+#define MV88E6352_G1_CTL2_HEADER_TYPE_ORIG	0x0000
+#define MV88E6352_G1_CTL2_HEADER_TYPE_MGMT	0x4000
+#define MV88E6390_G1_CTL2_HEADER_TYPE_LAG	0x8000
+#define MV88E6352_G1_CTL2_RMU_MODE_MASK		0x3000
+#define MV88E6352_G1_CTL2_RMU_MODE_DISABLED	0x0000
+#define MV88E6352_G1_CTL2_RMU_MODE_PORT_4	0x1000
+#define MV88E6352_G1_CTL2_RMU_MODE_PORT_5	0x2000
+#define MV88E6352_G1_CTL2_RMU_MODE_PORT_6	0x3000
+#define MV88E6085_G1_CTL2_DA_CHECK		0x4000
+#define MV88E6085_G1_CTL2_P10RM			0x2000
+#define MV88E6085_G1_CTL2_RM_ENABLE		0x1000
+#define MV88E6352_G1_CTL2_DA_CHECK		0x0800
+#define MV88E6390_G1_CTL2_RMU_MODE_MASK		0x0700
+#define MV88E6390_G1_CTL2_RMU_MODE_PORT_0	0x0000
+#define MV88E6390_G1_CTL2_RMU_MODE_PORT_1	0x0100
+#define MV88E6390_G1_CTL2_RMU_MODE_PORT_9	0x0200
+#define MV88E6390_G1_CTL2_RMU_MODE_PORT_10	0x0300
+#define MV88E6390_G1_CTL2_RMU_MODE_ALL_DSA	0x0600
+#define MV88E6390_G1_CTL2_RMU_MODE_DISABLED	0x0700
+#define MV88E6390_G1_CTL2_HIST_MODE_MASK	0x00c0
+#define MV88E6390_G1_CTL2_HIST_MODE_RX		0x0040
+#define MV88E6390_G1_CTL2_HIST_MODE_TX		0x0080
+#define MV88E6352_G1_CTL2_CTR_MODE_MASK		0x0060
+#define MV88E6390_G1_CTL2_CTR_MODE		0x0020
+#define MV88E6XXX_G1_CTL2_DEVICE_NUMBER_MASK	0x001f
 
 /* Offset 0x1D: Stats Operation Register */
 #define MV88E6XXX_G1_STATS_OP			0x1d
@@ -253,6 +277,17 @@ int mv88e6095_g1_set_cpu_port(struct mv88e6xxx_chip *chip, int port);
 int mv88e6390_g1_set_cpu_port(struct mv88e6xxx_chip *chip, int port);
 int mv88e6390_g1_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip);
 
+int mv88e6085_g1_ip_pri_map(struct mv88e6xxx_chip *chip);
+int mv88e6085_g1_ieee_pri_map(struct mv88e6xxx_chip *chip);
+
+int mv88e6185_g1_set_cascade_port(struct mv88e6xxx_chip *chip, int port);
+
+int mv88e6085_g1_rmu_disable(struct mv88e6xxx_chip *chip);
+int mv88e6352_g1_rmu_disable(struct mv88e6xxx_chip *chip);
+int mv88e6390_g1_rmu_disable(struct mv88e6xxx_chip *chip);
+
+int mv88e6xxx_g1_set_device_number(struct mv88e6xxx_chip *chip, int index);
+
 int mv88e6xxx_g1_atu_set_learn2all(struct mv88e6xxx_chip *chip, bool learn2all);
 int mv88e6xxx_g1_atu_set_age_time(struct mv88e6xxx_chip *chip,
 				  unsigned int msecs);
diff --git a/drivers/net/dsa/mv88e6xxx/global2.c b/drivers/net/dsa/mv88e6xxx/global2.c
index 8d22d66d84b7..91a3cb2452ac 100644
--- a/drivers/net/dsa/mv88e6xxx/global2.c
+++ b/drivers/net/dsa/mv88e6xxx/global2.c
@@ -119,37 +119,17 @@ int mv88e6352_g2_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip)
 
 /* Offset 0x06: Device Mapping Table register */
 
-static int mv88e6xxx_g2_device_mapping_write(struct mv88e6xxx_chip *chip,
-					     int target, int port)
+int mv88e6xxx_g2_device_mapping_write(struct mv88e6xxx_chip *chip, int target,
+				      int port)
 {
-	u16 val = (target << 8) | (port & 0xf);
+	u16 val = (target << 8) | (port & 0x1f);
+	/* Modern chips use 5 bits to define a device mapping port,
+	 * but bit 4 is reserved on older chips, so it is safe to use.
+	 */
 
 	return mv88e6xxx_g2_update(chip, MV88E6XXX_G2_DEVICE_MAPPING, val);
 }
 
-static int mv88e6xxx_g2_set_device_mapping(struct mv88e6xxx_chip *chip)
-{
-	int target, port;
-	int err;
-
-	/* Initialize the routing port to the 32 possible target devices */
-	for (target = 0; target < 32; ++target) {
-		port = 0xf;
-
-		if (target < DSA_MAX_SWITCHES) {
-			port = chip->ds->rtable[target];
-			if (port == DSA_RTABLE_NONE)
-				port = 0xf;
-		}
-
-		err = mv88e6xxx_g2_device_mapping_write(chip, target, port);
-		if (err)
-			break;
-	}
-
-	return err;
-}
-
 /* Offset 0x07: Trunk Mask Table register */
 
 static int mv88e6xxx_g2_trunk_mask_write(struct mv88e6xxx_chip *chip, int num,
@@ -174,7 +154,7 @@ static int mv88e6xxx_g2_trunk_mapping_write(struct mv88e6xxx_chip *chip, int id,
 	return mv88e6xxx_g2_update(chip, MV88E6XXX_G2_TRUNK_MAPPING, val);
 }
 
-static int mv88e6xxx_g2_clear_trunk(struct mv88e6xxx_chip *chip)
+int mv88e6xxx_g2_trunk_clear(struct mv88e6xxx_chip *chip)
 {
 	const u16 port_mask = BIT(mv88e6xxx_num_ports(chip)) - 1;
 	int i, err;
@@ -1067,9 +1047,6 @@ int mv88e6xxx_g2_irq_setup(struct mv88e6xxx_chip *chip)
 {
 	int err, irq, virq;
 
-	if (!chip->dev->of_node)
-		return -EINVAL;
-
 	chip->g2_irq.domain = irq_domain_add_simple(
 		chip->dev->of_node, 16, 0, &mv88e6xxx_g2_irq_domain_ops, chip);
 	if (!chip->g2_irq.domain)
@@ -1138,31 +1115,3 @@ void mv88e6xxx_g2_irq_mdio_free(struct mv88e6xxx_chip *chip,
 	for (phy = 0; phy < chip->info->num_internal_phys; phy++)
 		irq_dispose_mapping(bus->irq[phy]);
 }
-
-int mv88e6xxx_g2_setup(struct mv88e6xxx_chip *chip)
-{
-	u16 reg;
-	int err;
-
-	/* Ignore removed tag data on doubly tagged packets, disable
-	 * flow control messages, force flow control priority to the
-	 * highest, and send all special multicast frames to the CPU
-	 * port at the highest priority.
-	 */
-	reg = MV88E6XXX_G2_SWITCH_MGMT_FORCE_FLOW_CTL_PRI | (0x7 << 4);
-	err = mv88e6xxx_g2_write(chip, MV88E6XXX_G2_SWITCH_MGMT, reg);
-	if (err)
-		return err;
-
-	/* Program the DSA routing table. */
-	err = mv88e6xxx_g2_set_device_mapping(chip);
-	if (err)
-		return err;
-
-	/* Clear all trunk masks and mapping. */
-	err = mv88e6xxx_g2_clear_trunk(chip);
-	if (err)
-		return err;
-
-	return 0;
-}
diff --git a/drivers/net/dsa/mv88e6xxx/global2.h b/drivers/net/dsa/mv88e6xxx/global2.h
index 520ec70d32e8..37e8ce2c72a0 100644
--- a/drivers/net/dsa/mv88e6xxx/global2.h
+++ b/drivers/net/dsa/mv88e6xxx/global2.h
@@ -60,7 +60,8 @@
 #define MV88E6XXX_G2_DEVICE_MAPPING		0x06
 #define MV88E6XXX_G2_DEVICE_MAPPING_UPDATE	0x8000
 #define MV88E6XXX_G2_DEVICE_MAPPING_DEV_MASK	0x1f00
-#define MV88E6XXX_G2_DEVICE_MAPPING_PORT_MASK	0x000f
+#define MV88E6352_G2_DEVICE_MAPPING_PORT_MASK	0x000f
+#define MV88E6390_G2_DEVICE_MAPPING_PORT_MASK	0x001f
 
 /* Offset 0x07: Trunk Mask Table Register */
 #define MV88E6XXX_G2_TRUNK_MASK			0x07
@@ -313,7 +314,6 @@ int mv88e6xxx_g2_pvt_write(struct mv88e6xxx_chip *chip, int src_dev,
 			   int src_port, u16 data);
 int mv88e6xxx_g2_misc_4_bit_port(struct mv88e6xxx_chip *chip);
 
-int mv88e6xxx_g2_setup(struct mv88e6xxx_chip *chip);
 int mv88e6xxx_g2_irq_setup(struct mv88e6xxx_chip *chip);
 void mv88e6xxx_g2_irq_free(struct mv88e6xxx_chip *chip);
 
@@ -327,6 +327,11 @@ int mv88e6352_g2_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip);
 
 int mv88e6xxx_g2_pot_clear(struct mv88e6xxx_chip *chip);
 
+int mv88e6xxx_g2_trunk_clear(struct mv88e6xxx_chip *chip);
+
+int mv88e6xxx_g2_device_mapping_write(struct mv88e6xxx_chip *chip, int target,
+				      int port);
+
 extern const struct mv88e6xxx_irq_ops mv88e6097_watchdog_ops;
 extern const struct mv88e6xxx_irq_ops mv88e6390_watchdog_ops;
 
@@ -441,11 +446,6 @@ static inline int mv88e6xxx_g2_misc_4_bit_port(struct mv88e6xxx_chip *chip)
 	return -EOPNOTSUPP;
 }
 
-static inline int mv88e6xxx_g2_setup(struct mv88e6xxx_chip *chip)
-{
-	return -EOPNOTSUPP;
-}
-
 static inline int mv88e6xxx_g2_irq_setup(struct mv88e6xxx_chip *chip)
 {
 	return -EOPNOTSUPP;
@@ -495,6 +495,17 @@ static inline int mv88e6xxx_g2_scratch_gpio_set_smi(struct mv88e6xxx_chip *chip,
 	return -EOPNOTSUPP;
 }
 
+static inline int mv88e6xxx_g2_trunk_clear(struct mv88e6xxx_chip *chip)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int mv88e6xxx_g2_device_mapping_write(struct mv88e6xxx_chip *chip,
+						    int target, int port)
+{
+	return -EOPNOTSUPP;
+}
+
 #endif /* CONFIG_NET_DSA_MV88E6XXX_GLOBAL2 */
 
 #endif /* _MV88E6XXX_GLOBAL2_H */
diff --git a/drivers/net/dsa/mv88e6xxx/port.c b/drivers/net/dsa/mv88e6xxx/port.c
index 6315774d72b3..429d0ebcd5b1 100644
--- a/drivers/net/dsa/mv88e6xxx/port.c
+++ b/drivers/net/dsa/mv88e6xxx/port.c
@@ -15,6 +15,7 @@
 #include <linux/bitfield.h>
 #include <linux/if_bridge.h>
 #include <linux/phy.h>
+#include <linux/phylink.h>
 
 #include "chip.h"
 #include "port.h"
@@ -378,6 +379,44 @@ int mv88e6xxx_port_get_cmode(struct mv88e6xxx_chip *chip, int port, u8 *cmode)
 	return 0;
 }
 
+int mv88e6xxx_port_link_state(struct mv88e6xxx_chip *chip, int port,
+			      struct phylink_link_state *state)
+{
+	int err;
+	u16 reg;
+
+	err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_STS, &reg);
+	if (err)
+		return err;
+
+	switch (reg & MV88E6XXX_PORT_STS_SPEED_MASK) {
+	case MV88E6XXX_PORT_STS_SPEED_10:
+		state->speed = SPEED_10;
+		break;
+	case MV88E6XXX_PORT_STS_SPEED_100:
+		state->speed = SPEED_100;
+		break;
+	case MV88E6XXX_PORT_STS_SPEED_1000:
+		state->speed = SPEED_1000;
+		break;
+	case MV88E6XXX_PORT_STS_SPEED_10000:
+		if ((reg &MV88E6XXX_PORT_STS_CMODE_MASK) ==
+		    MV88E6XXX_PORT_STS_CMODE_2500BASEX)
+			state->speed = SPEED_2500;
+		else
+			state->speed = SPEED_10000;
+		break;
+	}
+
+	state->duplex = reg & MV88E6XXX_PORT_STS_DUPLEX ?
+			DUPLEX_FULL : DUPLEX_HALF;
+	state->link = !!(reg & MV88E6XXX_PORT_STS_LINK);
+	state->an_enabled = 1;
+	state->an_complete = state->link;
+
+	return 0;
+}
+
 /* Offset 0x02: Jamming Control
  *
  * Do not limit the period of time that this port can be paused for by
diff --git a/drivers/net/dsa/mv88e6xxx/port.h b/drivers/net/dsa/mv88e6xxx/port.h
index b16d5f0e6e9c..5e1db1b221ca 100644
--- a/drivers/net/dsa/mv88e6xxx/port.h
+++ b/drivers/net/dsa/mv88e6xxx/port.h
@@ -29,6 +29,7 @@
 #define MV88E6XXX_PORT_STS_SPEED_10		0x0000
 #define MV88E6XXX_PORT_STS_SPEED_100		0x0100
 #define MV88E6XXX_PORT_STS_SPEED_1000		0x0200
+#define MV88E6XXX_PORT_STS_SPEED_10000		0x0300
 #define MV88E6352_PORT_STS_EEE			0x0040
 #define MV88E6165_PORT_STS_AM_DIS		0x0040
 #define MV88E6185_PORT_STS_MGMII		0x0040
@@ -295,6 +296,8 @@ int mv88e6390_port_pause_limit(struct mv88e6xxx_chip *chip, int port, u8 in,
 int mv88e6390x_port_set_cmode(struct mv88e6xxx_chip *chip, int port,
 			      phy_interface_t mode);
 int mv88e6xxx_port_get_cmode(struct mv88e6xxx_chip *chip, int port, u8 *cmode);
+int mv88e6xxx_port_link_state(struct mv88e6xxx_chip *chip, int port,
+			      struct phylink_link_state *state);
 int mv88e6xxx_port_set_map_da(struct mv88e6xxx_chip *chip, int port);
 int mv88e6095_port_set_upstream_port(struct mv88e6xxx_chip *chip, int port,
 				     int upstream_port);
diff --git a/drivers/net/dsa/mv88e6xxx/serdes.c b/drivers/net/dsa/mv88e6xxx/serdes.c
index fb058fd35c0d..880b2cf0a530 100644
--- a/drivers/net/dsa/mv88e6xxx/serdes.c
+++ b/drivers/net/dsa/mv88e6xxx/serdes.c
@@ -326,3 +326,23 @@ int mv88e6390_serdes_power(struct mv88e6xxx_chip *chip, int port, bool on)
 
 	return 0;
 }
+
+int mv88e6341_serdes_power(struct mv88e6xxx_chip *chip, int port, bool on)
+{
+	int err;
+	u8 cmode;
+
+	if (port != 5)
+		return 0;
+
+	err = mv88e6xxx_port_get_cmode(chip, port, &cmode);
+	if (err)
+		return err;
+
+	if (cmode == MV88E6XXX_PORT_STS_CMODE_1000BASE_X ||
+	    cmode == MV88E6XXX_PORT_STS_CMODE_SGMII ||
+	    cmode == MV88E6XXX_PORT_STS_CMODE_2500BASEX)
+		return mv88e6390_serdes_sgmii(chip, MV88E6341_ADDR_SERDES, on);
+
+	return 0;
+}
diff --git a/drivers/net/dsa/mv88e6xxx/serdes.h b/drivers/net/dsa/mv88e6xxx/serdes.h
index 1897c01c6e19..b6e5fbd46b5e 100644
--- a/drivers/net/dsa/mv88e6xxx/serdes.h
+++ b/drivers/net/dsa/mv88e6xxx/serdes.h
@@ -19,6 +19,8 @@
 #define MV88E6352_ADDR_SERDES		0x0f
 #define MV88E6352_SERDES_PAGE_FIBER	0x01
 
+#define MV88E6341_ADDR_SERDES		0x15
+
 #define MV88E6390_PORT9_LANE0		0x09
 #define MV88E6390_PORT9_LANE1		0x12
 #define MV88E6390_PORT9_LANE2		0x13
@@ -42,6 +44,7 @@
 #define MV88E6390_SGMII_CONTROL_LOOPBACK	BIT(14)
 #define MV88E6390_SGMII_CONTROL_PDOWN		BIT(11)
 
+int mv88e6341_serdes_power(struct mv88e6xxx_chip *chip, int port, bool on);
 int mv88e6352_serdes_power(struct mv88e6xxx_chip *chip, int port, bool on);
 int mv88e6390_serdes_power(struct mv88e6xxx_chip *chip, int port, bool on);
 int mv88e6352_serdes_get_sset_count(struct mv88e6xxx_chip *chip, int port);
diff --git a/drivers/net/dsa/qca8k.c b/drivers/net/dsa/qca8k.c
index 600d5ad1fbde..cdcde7f8e0b2 100644
--- a/drivers/net/dsa/qca8k.c
+++ b/drivers/net/dsa/qca8k.c
@@ -1,17 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2009 Felix Fietkau <nbd@nbd.name>
  * Copyright (C) 2011-2012 Gabor Juhos <juhosg@openwrt.org>
  * Copyright (c) 2015, The Linux Foundation. All rights reserved.
  * Copyright (c) 2016 John Crispin <john@phrozen.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/module.h>
@@ -473,10 +465,10 @@ qca8k_set_pad_ctrl(struct qca8k_priv *priv, int port, int mode)
 static void
 qca8k_port_set_status(struct qca8k_priv *priv, int port, int enable)
 {
-	u32 mask = QCA8K_PORT_STATUS_TXMAC;
+	u32 mask = QCA8K_PORT_STATUS_TXMAC | QCA8K_PORT_STATUS_RXMAC;
 
 	/* Port 0 and 6 have no internal PHY */
-	if ((port > 0) && (port < 6))
+	if (port > 0 && port < 6)
 		mask |= QCA8K_PORT_STATUS_LINK_AUTO;
 
 	if (enable)
@@ -490,6 +482,7 @@ qca8k_setup(struct dsa_switch *ds)
 {
 	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
 	int ret, i, phy_mode = -1;
+	u32 mask;
 
 	/* Make sure that port 0 is the cpu port */
 	if (!dsa_is_cpu_port(ds, 0)) {
@@ -515,7 +508,10 @@ qca8k_setup(struct dsa_switch *ds)
 	if (ret < 0)
 		return ret;
 
-	/* Enable CPU Port */
+	/* Enable CPU Port, force it to maximum bandwidth and full-duplex */
+	mask = QCA8K_PORT_STATUS_SPEED_1000 | QCA8K_PORT_STATUS_TXFLOW |
+	       QCA8K_PORT_STATUS_RXFLOW | QCA8K_PORT_STATUS_DUPLEX;
+	qca8k_write(priv, QCA8K_REG_PORT_STATUS(QCA8K_CPU_PORT), mask);
 	qca8k_reg_set(priv, QCA8K_REG_GLOBAL_FW_CTRL0,
 		      QCA8K_GLOBAL_FW_CTRL0_CPU_PORT_EN);
 	qca8k_port_set_status(priv, QCA8K_CPU_PORT, 1);
@@ -583,6 +579,47 @@ qca8k_setup(struct dsa_switch *ds)
 	return 0;
 }
 
+static void
+qca8k_adjust_link(struct dsa_switch *ds, int port, struct phy_device *phy)
+{
+	struct qca8k_priv *priv = ds->priv;
+	u32 reg;
+
+	/* Force fixed-link setting for CPU port, skip others. */
+	if (!phy_is_pseudo_fixed_link(phy))
+		return;
+
+	/* Set port speed */
+	switch (phy->speed) {
+	case 10:
+		reg = QCA8K_PORT_STATUS_SPEED_10;
+		break;
+	case 100:
+		reg = QCA8K_PORT_STATUS_SPEED_100;
+		break;
+	case 1000:
+		reg = QCA8K_PORT_STATUS_SPEED_1000;
+		break;
+	default:
+		dev_dbg(priv->dev, "port%d link speed %dMbps not supported.\n",
+			port, phy->speed);
+		return;
+	}
+
+	/* Set duplex mode */
+	if (phy->duplex == DUPLEX_FULL)
+		reg |= QCA8K_PORT_STATUS_DUPLEX;
+
+	/* Force flow control */
+	if (dsa_is_cpu_port(ds, port))
+		reg |= QCA8K_PORT_STATUS_RXFLOW | QCA8K_PORT_STATUS_TXFLOW;
+
+	/* Force link down before changing MAC options */
+	qca8k_port_set_status(priv, port, 0);
+	qca8k_write(priv, QCA8K_REG_PORT_STATUS(port), reg);
+	qca8k_port_set_status(priv, port, 1);
+}
+
 static int
 qca8k_phy_read(struct dsa_switch *ds, int phy, int regnum)
 {
@@ -600,10 +637,13 @@ qca8k_phy_write(struct dsa_switch *ds, int phy, int regnum, u16 val)
 }
 
 static void
-qca8k_get_strings(struct dsa_switch *ds, int port, uint8_t *data)
+qca8k_get_strings(struct dsa_switch *ds, int port, u32 stringset, uint8_t *data)
 {
 	int i;
 
+	if (stringset != ETH_SS_STATS)
+		return;
+
 	for (i = 0; i < ARRAY_SIZE(ar8327_mib); i++)
 		strncpy(data + i * ETH_GSTRING_LEN, ar8327_mib[i].name,
 			ETH_GSTRING_LEN);
@@ -631,8 +671,11 @@ qca8k_get_ethtool_stats(struct dsa_switch *ds, int port,
 }
 
 static int
-qca8k_get_sset_count(struct dsa_switch *ds, int port)
+qca8k_get_sset_count(struct dsa_switch *ds, int port, int sset)
 {
+	if (sset != ETH_SS_STATS)
+		return 0;
+
 	return ARRAY_SIZE(ar8327_mib);
 }
 
@@ -831,6 +874,7 @@ qca8k_get_tag_protocol(struct dsa_switch *ds, int port)
 static const struct dsa_switch_ops qca8k_switch_ops = {
 	.get_tag_protocol	= qca8k_get_tag_protocol,
 	.setup			= qca8k_setup,
+	.adjust_link            = qca8k_adjust_link,
 	.get_strings		= qca8k_get_strings,
 	.phy_read		= qca8k_phy_read,
 	.phy_write		= qca8k_phy_write,
@@ -862,6 +906,7 @@ qca8k_sw_probe(struct mdio_device *mdiodev)
 		return -ENOMEM;
 
 	priv->bus = mdiodev->bus;
+	priv->dev = &mdiodev->dev;
 
 	/* read the switches ID register */
 	id = qca8k_read(priv, QCA8K_REG_MASK_CTRL);
@@ -933,6 +978,7 @@ static SIMPLE_DEV_PM_OPS(qca8k_pm_ops,
 			 qca8k_suspend, qca8k_resume);
 
 static const struct of_device_id qca8k_of_match[] = {
+	{ .compatible = "qca,qca8334" },
 	{ .compatible = "qca,qca8337" },
 	{ /* sentinel */ },
 };
diff --git a/drivers/net/dsa/qca8k.h b/drivers/net/dsa/qca8k.h
index 1cf8a920d4ff..613fe5c50236 100644
--- a/drivers/net/dsa/qca8k.h
+++ b/drivers/net/dsa/qca8k.h
@@ -51,8 +51,10 @@
 #define QCA8K_GOL_MAC_ADDR0				0x60
 #define QCA8K_GOL_MAC_ADDR1				0x64
 #define QCA8K_REG_PORT_STATUS(_i)			(0x07c + (_i) * 4)
-#define   QCA8K_PORT_STATUS_SPEED			GENMASK(2, 0)
-#define   QCA8K_PORT_STATUS_SPEED_S			0
+#define   QCA8K_PORT_STATUS_SPEED			GENMASK(1, 0)
+#define   QCA8K_PORT_STATUS_SPEED_10			0
+#define   QCA8K_PORT_STATUS_SPEED_100			0x1
+#define   QCA8K_PORT_STATUS_SPEED_1000			0x2
 #define   QCA8K_PORT_STATUS_TXMAC			BIT(2)
 #define   QCA8K_PORT_STATUS_RXMAC			BIT(3)
 #define   QCA8K_PORT_STATUS_TXFLOW			BIT(4)
@@ -165,6 +167,7 @@ struct qca8k_priv {
 	struct ar8xxx_port_status port_sts[QCA8K_NUM_PORTS];
 	struct dsa_switch *ds;
 	struct mutex reg_mutex;
+	struct device *dev;
 };
 
 struct qca8k_mib_desc {
diff --git a/drivers/net/ethernet/3com/3c59x.c b/drivers/net/ethernet/3com/3c59x.c
index 176861bd2252..5bc168314ea2 100644
--- a/drivers/net/ethernet/3com/3c59x.c
+++ b/drivers/net/ethernet/3com/3c59x.c
@@ -765,8 +765,9 @@ static netdev_tx_t boomerang_start_xmit(struct sk_buff *skb,
 					struct net_device *dev);
 static int vortex_rx(struct net_device *dev);
 static int boomerang_rx(struct net_device *dev);
-static irqreturn_t vortex_interrupt(int irq, void *dev_id);
-static irqreturn_t boomerang_interrupt(int irq, void *dev_id);
+static irqreturn_t vortex_boomerang_interrupt(int irq, void *dev_id);
+static irqreturn_t _vortex_interrupt(int irq, struct net_device *dev);
+static irqreturn_t _boomerang_interrupt(int irq, struct net_device *dev);
 static int vortex_close(struct net_device *dev);
 static void dump_tx_ring(struct net_device *dev);
 static void update_stats(void __iomem *ioaddr, struct net_device *dev);
@@ -838,11 +839,7 @@ MODULE_PARM_DESC(use_mmio, "3c59x: use memory-mapped PCI I/O resource (0-1)");
 #ifdef CONFIG_NET_POLL_CONTROLLER
 static void poll_vortex(struct net_device *dev)
 {
-	struct vortex_private *vp = netdev_priv(dev);
-	unsigned long flags;
-	local_irq_save(flags);
-	(vp->full_bus_master_rx ? boomerang_interrupt:vortex_interrupt)(dev->irq,dev);
-	local_irq_restore(flags);
+	vortex_boomerang_interrupt(dev->irq, dev);
 }
 #endif
 
@@ -1728,8 +1725,7 @@ vortex_open(struct net_device *dev)
 	dma_addr_t dma;
 
 	/* Use the now-standard shared IRQ implementation. */
-	if ((retval = request_irq(dev->irq, vp->full_bus_master_rx ?
-				boomerang_interrupt : vortex_interrupt, IRQF_SHARED, dev->name, dev))) {
+	if ((retval = request_irq(dev->irq, vortex_boomerang_interrupt, IRQF_SHARED, dev->name, dev))) {
 		pr_err("%s: Could not reserve IRQ %d\n", dev->name, dev->irq);
 		goto err;
 	}
@@ -1904,18 +1900,7 @@ static void vortex_tx_timeout(struct net_device *dev)
 		pr_err("%s: Interrupt posted but not delivered --"
 			   " IRQ blocked by another device?\n", dev->name);
 		/* Bad idea here.. but we might as well handle a few events. */
-		{
-			/*
-			 * Block interrupts because vortex_interrupt does a bare spin_lock()
-			 */
-			unsigned long flags;
-			local_irq_save(flags);
-			if (vp->full_bus_master_tx)
-				boomerang_interrupt(dev->irq, dev);
-			else
-				vortex_interrupt(dev->irq, dev);
-			local_irq_restore(flags);
-		}
+		vortex_boomerang_interrupt(dev->irq, dev);
 	}
 
 	if (vortex_debug > 0)
@@ -2266,9 +2251,8 @@ boomerang_start_xmit(struct sk_buff *skb, struct net_device *dev)
  */
 
 static irqreturn_t
-vortex_interrupt(int irq, void *dev_id)
+_vortex_interrupt(int irq, struct net_device *dev)
 {
-	struct net_device *dev = dev_id;
 	struct vortex_private *vp = netdev_priv(dev);
 	void __iomem *ioaddr;
 	int status;
@@ -2277,7 +2261,6 @@ vortex_interrupt(int irq, void *dev_id)
 	unsigned int bytes_compl = 0, pkts_compl = 0;
 
 	ioaddr = vp->ioaddr;
-	spin_lock(&vp->lock);
 
 	status = ioread16(ioaddr + EL3_STATUS);
 
@@ -2375,7 +2358,6 @@ vortex_interrupt(int irq, void *dev_id)
 		pr_debug("%s: exiting interrupt, status %4.4x.\n",
 			   dev->name, status);
 handler_exit:
-	spin_unlock(&vp->lock);
 	return IRQ_RETVAL(handled);
 }
 
@@ -2385,9 +2367,8 @@ vortex_interrupt(int irq, void *dev_id)
  */
 
 static irqreturn_t
-boomerang_interrupt(int irq, void *dev_id)
+_boomerang_interrupt(int irq, struct net_device *dev)
 {
-	struct net_device *dev = dev_id;
 	struct vortex_private *vp = netdev_priv(dev);
 	void __iomem *ioaddr;
 	int status;
@@ -2397,12 +2378,6 @@ boomerang_interrupt(int irq, void *dev_id)
 
 	ioaddr = vp->ioaddr;
 
-
-	/*
-	 * It seems dopey to put the spinlock this early, but we could race against vortex_tx_timeout
-	 * and boomerang_start_xmit
-	 */
-	spin_lock(&vp->lock);
 	vp->handling_irq = 1;
 
 	status = ioread16(ioaddr + EL3_STATUS);
@@ -2521,10 +2496,29 @@ boomerang_interrupt(int irq, void *dev_id)
 			   dev->name, status);
 handler_exit:
 	vp->handling_irq = 0;
-	spin_unlock(&vp->lock);
 	return IRQ_RETVAL(handled);
 }
 
+static irqreturn_t
+vortex_boomerang_interrupt(int irq, void *dev_id)
+{
+	struct net_device *dev = dev_id;
+	struct vortex_private *vp = netdev_priv(dev);
+	unsigned long flags;
+	irqreturn_t ret;
+
+	spin_lock_irqsave(&vp->lock, flags);
+
+	if (vp->full_bus_master_rx)
+		ret = _boomerang_interrupt(dev->irq, dev);
+	else
+		ret = _vortex_interrupt(dev->irq, dev);
+
+	spin_unlock_irqrestore(&vp->lock, flags);
+
+	return ret;
+}
+
 static int vortex_rx(struct net_device *dev)
 {
 	struct vortex_private *vp = netdev_priv(dev);
diff --git a/drivers/net/ethernet/8390/Kconfig b/drivers/net/ethernet/8390/Kconfig
index 9fee7c83ef9f..f2f0264c58ba 100644
--- a/drivers/net/ethernet/8390/Kconfig
+++ b/drivers/net/ethernet/8390/Kconfig
@@ -29,8 +29,8 @@ config PCMCIA_AXNET
 	  called axnet_cs.  If unsure, say N.
 
 config AX88796
-	tristate "ASIX AX88796 NE2000 clone support"
-	depends on (ARM || MIPS || SUPERH)
+	tristate "ASIX AX88796 NE2000 clone support" if !ZORRO
+	depends on (ARM || MIPS || SUPERH || ZORRO || COMPILE_TEST)
 	select CRC32
 	select PHYLIB
 	select MDIO_BITBANG
@@ -45,6 +45,19 @@ config AX88796_93CX6
 	---help---
 	  Select this if your platform comes with an external 93CX6 eeprom.
 
+config XSURF100
+	tristate "Amiga XSurf 100 AX88796/NE2000 clone support"
+	depends on ZORRO
+	select AX88796
+	select ASIX_PHY
+	help
+	  This driver is for the Individual Computers X-Surf 100 Ethernet
+	  card (based on the Asix AX88796 chip). If you have such a card,
+	  say Y. Otherwise, say N.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called xsurf100.
+
 config HYDRA
 	tristate "Hydra support"
 	depends on ZORRO
diff --git a/drivers/net/ethernet/8390/Makefile b/drivers/net/ethernet/8390/Makefile
index 1d650e66cc6e..85c83c566ec6 100644
--- a/drivers/net/ethernet/8390/Makefile
+++ b/drivers/net/ethernet/8390/Makefile
@@ -16,4 +16,5 @@ obj-$(CONFIG_PCMCIA_PCNET) += pcnet_cs.o 8390.o
 obj-$(CONFIG_STNIC) += stnic.o 8390.o
 obj-$(CONFIG_ULTRA) += smc-ultra.o 8390.o
 obj-$(CONFIG_WD80x3) += wd.o 8390.o
+obj-$(CONFIG_XSURF100) += xsurf100.o
 obj-$(CONFIG_ZORRO8390) += zorro8390.o
diff --git a/drivers/net/ethernet/8390/ax88796.c b/drivers/net/ethernet/8390/ax88796.c
index da61cf3cb3a9..2a0ddec1dd56 100644
--- a/drivers/net/ethernet/8390/ax88796.c
+++ b/drivers/net/ethernet/8390/ax88796.c
@@ -163,6 +163,21 @@ static void ax_reset_8390(struct net_device *dev)
 	ei_outb(ENISR_RESET, addr + EN0_ISR);	/* Ack intr. */
 }
 
+/* Wrapper for __ei_interrupt for platforms that have a platform-specific
+ * way to find out whether the interrupt request might be caused by
+ * the ax88796 chip.
+ */
+static irqreturn_t ax_ei_interrupt_filtered(int irq, void *dev_id)
+{
+	struct net_device *dev = dev_id;
+	struct ax_device *ax = to_ax_dev(dev);
+	struct platform_device *pdev = to_platform_device(dev->dev.parent);
+
+	if (!ax->plat->check_irq(pdev))
+		return IRQ_NONE;
+
+	return ax_ei_interrupt(irq, dev_id);
+}
 
 static void ax_get_8390_hdr(struct net_device *dev, struct e8390_pkt_hdr *hdr,
 			    int ring_page)
@@ -387,6 +402,90 @@ static void ax_phy_switch(struct net_device *dev, int on)
 	ei_outb(reg_gpoc, ei_local->mem + EI_SHIFT(0x17));
 }
 
+static void ax_bb_mdc(struct mdiobb_ctrl *ctrl, int level)
+{
+	struct ax_device *ax = container_of(ctrl, struct ax_device, bb_ctrl);
+
+	if (level)
+		ax->reg_memr |= AX_MEMR_MDC;
+	else
+		ax->reg_memr &= ~AX_MEMR_MDC;
+
+	ei_outb(ax->reg_memr, ax->addr_memr);
+}
+
+static void ax_bb_dir(struct mdiobb_ctrl *ctrl, int output)
+{
+	struct ax_device *ax = container_of(ctrl, struct ax_device, bb_ctrl);
+
+	if (output)
+		ax->reg_memr &= ~AX_MEMR_MDIR;
+	else
+		ax->reg_memr |= AX_MEMR_MDIR;
+
+	ei_outb(ax->reg_memr, ax->addr_memr);
+}
+
+static void ax_bb_set_data(struct mdiobb_ctrl *ctrl, int value)
+{
+	struct ax_device *ax = container_of(ctrl, struct ax_device, bb_ctrl);
+
+	if (value)
+		ax->reg_memr |= AX_MEMR_MDO;
+	else
+		ax->reg_memr &= ~AX_MEMR_MDO;
+
+	ei_outb(ax->reg_memr, ax->addr_memr);
+}
+
+static int ax_bb_get_data(struct mdiobb_ctrl *ctrl)
+{
+	struct ax_device *ax = container_of(ctrl, struct ax_device, bb_ctrl);
+	int reg_memr = ei_inb(ax->addr_memr);
+
+	return reg_memr & AX_MEMR_MDI ? 1 : 0;
+}
+
+static const struct mdiobb_ops bb_ops = {
+	.owner = THIS_MODULE,
+	.set_mdc = ax_bb_mdc,
+	.set_mdio_dir = ax_bb_dir,
+	.set_mdio_data = ax_bb_set_data,
+	.get_mdio_data = ax_bb_get_data,
+};
+
+static int ax_mii_init(struct net_device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev->dev.parent);
+	struct ei_device *ei_local = netdev_priv(dev);
+	struct ax_device *ax = to_ax_dev(dev);
+	int err;
+
+	ax->bb_ctrl.ops = &bb_ops;
+	ax->addr_memr = ei_local->mem + AX_MEMR;
+	ax->mii_bus = alloc_mdio_bitbang(&ax->bb_ctrl);
+	if (!ax->mii_bus) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	ax->mii_bus->name = "ax88796_mii_bus";
+	ax->mii_bus->parent = dev->dev.parent;
+	snprintf(ax->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",
+		 pdev->name, pdev->id);
+
+	err = mdiobus_register(ax->mii_bus);
+	if (err)
+		goto out_free_mdio_bitbang;
+
+	return 0;
+
+ out_free_mdio_bitbang:
+	free_mdio_bitbang(ax->mii_bus);
+ out:
+	return err;
+}
+
 static int ax_open(struct net_device *dev)
 {
 	struct ax_device *ax = to_ax_dev(dev);
@@ -394,8 +493,16 @@ static int ax_open(struct net_device *dev)
 
 	netdev_dbg(dev, "open\n");
 
-	ret = request_irq(dev->irq, ax_ei_interrupt, ax->irqflags,
-			  dev->name, dev);
+	ret = ax_mii_init(dev);
+	if (ret)
+		goto failed_mii;
+
+	if (ax->plat->check_irq)
+		ret = request_irq(dev->irq, ax_ei_interrupt_filtered,
+				  ax->irqflags, dev->name, dev);
+	else
+		ret = request_irq(dev->irq, ax_ei_interrupt, ax->irqflags,
+				  dev->name, dev);
 	if (ret)
 		goto failed_request_irq;
 
@@ -421,6 +528,10 @@ static int ax_open(struct net_device *dev)
 	ax_phy_switch(dev, 0);
 	free_irq(dev->irq, dev);
  failed_request_irq:
+	/* unregister mdiobus */
+	mdiobus_unregister(ax->mii_bus);
+	free_mdio_bitbang(ax->mii_bus);
+ failed_mii:
 	return ret;
 }
 
@@ -440,6 +551,9 @@ static int ax_close(struct net_device *dev)
 	phy_disconnect(dev->phydev);
 
 	free_irq(dev->irq, dev);
+
+	mdiobus_unregister(ax->mii_bus);
+	free_mdio_bitbang(ax->mii_bus);
 	return 0;
 }
 
@@ -539,92 +653,8 @@ static const struct net_device_ops ax_netdev_ops = {
 #endif
 };
 
-static void ax_bb_mdc(struct mdiobb_ctrl *ctrl, int level)
-{
-	struct ax_device *ax = container_of(ctrl, struct ax_device, bb_ctrl);
-
-	if (level)
-		ax->reg_memr |= AX_MEMR_MDC;
-	else
-		ax->reg_memr &= ~AX_MEMR_MDC;
-
-	ei_outb(ax->reg_memr, ax->addr_memr);
-}
-
-static void ax_bb_dir(struct mdiobb_ctrl *ctrl, int output)
-{
-	struct ax_device *ax = container_of(ctrl, struct ax_device, bb_ctrl);
-
-	if (output)
-		ax->reg_memr &= ~AX_MEMR_MDIR;
-	else
-		ax->reg_memr |= AX_MEMR_MDIR;
-
-	ei_outb(ax->reg_memr, ax->addr_memr);
-}
-
-static void ax_bb_set_data(struct mdiobb_ctrl *ctrl, int value)
-{
-	struct ax_device *ax = container_of(ctrl, struct ax_device, bb_ctrl);
-
-	if (value)
-		ax->reg_memr |= AX_MEMR_MDO;
-	else
-		ax->reg_memr &= ~AX_MEMR_MDO;
-
-	ei_outb(ax->reg_memr, ax->addr_memr);
-}
-
-static int ax_bb_get_data(struct mdiobb_ctrl *ctrl)
-{
-	struct ax_device *ax = container_of(ctrl, struct ax_device, bb_ctrl);
-	int reg_memr = ei_inb(ax->addr_memr);
-
-	return reg_memr & AX_MEMR_MDI ? 1 : 0;
-}
-
-static const struct mdiobb_ops bb_ops = {
-	.owner = THIS_MODULE,
-	.set_mdc = ax_bb_mdc,
-	.set_mdio_dir = ax_bb_dir,
-	.set_mdio_data = ax_bb_set_data,
-	.get_mdio_data = ax_bb_get_data,
-};
-
 /* setup code */
 
-static int ax_mii_init(struct net_device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev->dev.parent);
-	struct ei_device *ei_local = netdev_priv(dev);
-	struct ax_device *ax = to_ax_dev(dev);
-	int err;
-
-	ax->bb_ctrl.ops = &bb_ops;
-	ax->addr_memr = ei_local->mem + AX_MEMR;
-	ax->mii_bus = alloc_mdio_bitbang(&ax->bb_ctrl);
-	if (!ax->mii_bus) {
-		err = -ENOMEM;
-		goto out;
-	}
-
-	ax->mii_bus->name = "ax88796_mii_bus";
-	ax->mii_bus->parent = dev->dev.parent;
-	snprintf(ax->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",
-		pdev->name, pdev->id);
-
-	err = mdiobus_register(ax->mii_bus);
-	if (err)
-		goto out_free_mdio_bitbang;
-
-	return 0;
-
- out_free_mdio_bitbang:
-	free_mdio_bitbang(ax->mii_bus);
- out:
-	return err;
-}
-
 static void ax_initial_setup(struct net_device *dev, struct ei_device *ei_local)
 {
 	void __iomem *ioaddr = ei_local->mem;
@@ -669,10 +699,16 @@ static int ax_init_dev(struct net_device *dev)
 	if (ax->plat->flags & AXFLG_HAS_EEPROM) {
 		unsigned char SA_prom[32];
 
+		ei_outb(6, ioaddr + EN0_RCNTLO);
+		ei_outb(0, ioaddr + EN0_RCNTHI);
+		ei_outb(0, ioaddr + EN0_RSARLO);
+		ei_outb(0, ioaddr + EN0_RSARHI);
+		ei_outb(E8390_RREAD + E8390_START, ioaddr + NE_CMD);
 		for (i = 0; i < sizeof(SA_prom); i += 2) {
 			SA_prom[i] = ei_inb(ioaddr + NE_DATAPORT);
 			SA_prom[i + 1] = ei_inb(ioaddr + NE_DATAPORT);
 		}
+		ei_outb(ENISR_RDC, ioaddr + EN0_ISR);	/* Ack intr. */
 
 		if (ax->plat->wordlength == 2)
 			for (i = 0; i < 16; i++)
@@ -741,18 +777,20 @@ static int ax_init_dev(struct net_device *dev)
 #endif
 
 	ei_local->reset_8390 = &ax_reset_8390;
-	ei_local->block_input = &ax_block_input;
-	ei_local->block_output = &ax_block_output;
+	if (ax->plat->block_input)
+		ei_local->block_input = ax->plat->block_input;
+	else
+		ei_local->block_input = &ax_block_input;
+	if (ax->plat->block_output)
+		ei_local->block_output = ax->plat->block_output;
+	else
+		ei_local->block_output = &ax_block_output;
 	ei_local->get_8390_hdr = &ax_get_8390_hdr;
 	ei_local->priv = 0;
 
 	dev->netdev_ops = &ax_netdev_ops;
 	dev->ethtool_ops = &ax_ethtool_ops;
 
-	ret = ax_mii_init(dev);
-	if (ret)
-		goto err_out;
-
 	ax_NS8390_init(dev, 0);
 
 	ret = register_netdev(dev);
@@ -777,7 +815,6 @@ static int ax_remove(struct platform_device *pdev)
 	struct resource *mem;
 
 	unregister_netdev(dev);
-	free_irq(dev->irq, dev);
 
 	iounmap(ei_local->mem);
 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -789,6 +826,7 @@ static int ax_remove(struct platform_device *pdev)
 		release_mem_region(mem->start, resource_size(mem));
 	}
 
+	platform_set_drvdata(pdev, NULL);
 	free_netdev(dev);
 
 	return 0;
@@ -835,6 +873,9 @@ static int ax_probe(struct platform_device *pdev)
 	dev->irq = irq->start;
 	ax->irqflags = irq->flags & IRQF_TRIGGER_MASK;
 
+	if (irq->flags &  IORESOURCE_IRQ_SHAREABLE)
+		ax->irqflags |= IRQF_SHARED;
+
 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!mem) {
 		dev_err(&pdev->dev, "no MEM specified\n");
@@ -919,6 +960,7 @@ static int ax_probe(struct platform_device *pdev)
 	release_mem_region(mem->start, mem_size);
 
  exit_mem:
+	platform_set_drvdata(pdev, NULL);
 	free_netdev(dev);
 
 	return ret;
diff --git a/drivers/net/ethernet/8390/xsurf100.c b/drivers/net/ethernet/8390/xsurf100.c
new file mode 100644
index 000000000000..e2c963821ffe
--- /dev/null
+++ b/drivers/net/ethernet/8390/xsurf100.c
@@ -0,0 +1,382 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/platform_device.h>
+#include <linux/zorro.h>
+#include <net/ax88796.h>
+#include <asm/amigaints.h>
+
+#define ZORRO_PROD_INDIVIDUAL_COMPUTERS_X_SURF100 \
+		ZORRO_ID(INDIVIDUAL_COMPUTERS, 0x64, 0)
+
+#define XS100_IRQSTATUS_BASE 0x40
+#define XS100_8390_BASE 0x800
+
+/* Longword-access area. Translated to 2 16-bit access cycles by the
+ * X-Surf 100 FPGA
+ */
+#define XS100_8390_DATA32_BASE 0x8000
+#define XS100_8390_DATA32_SIZE 0x2000
+/* Sub-Areas for fast data register access; addresses relative to area begin */
+#define XS100_8390_DATA_READ32_BASE 0x0880
+#define XS100_8390_DATA_WRITE32_BASE 0x0C80
+#define XS100_8390_DATA_AREA_SIZE 0x80
+
+#define __NS8390_init ax_NS8390_init
+
+/* force unsigned long back to 'void __iomem *' */
+#define ax_convert_addr(_a) ((void __force __iomem *)(_a))
+
+#define ei_inb(_a) z_readb(ax_convert_addr(_a))
+#define ei_outb(_v, _a) z_writeb(_v, ax_convert_addr(_a))
+
+#define ei_inw(_a) z_readw(ax_convert_addr(_a))
+#define ei_outw(_v, _a) z_writew(_v, ax_convert_addr(_a))
+
+#define ei_inb_p(_a) ei_inb(_a)
+#define ei_outb_p(_v, _a) ei_outb(_v, _a)
+
+/* define EI_SHIFT() to take into account our register offsets */
+#define EI_SHIFT(x) (ei_local->reg_offset[(x)])
+
+/* Ensure we have our RCR base value */
+#define AX88796_PLATFORM
+
+static unsigned char version[] =
+		"ax88796.c: Copyright 2005,2007 Simtec Electronics\n";
+
+#include "lib8390.c"
+
+/* from ne.c */
+#define NE_CMD		EI_SHIFT(0x00)
+#define NE_RESET	EI_SHIFT(0x1f)
+#define NE_DATAPORT	EI_SHIFT(0x10)
+
+struct xsurf100_ax_plat_data {
+	struct ax_plat_data ax;
+	void __iomem *base_regs;
+	void __iomem *data_area;
+};
+
+static int is_xsurf100_network_irq(struct platform_device *pdev)
+{
+	struct xsurf100_ax_plat_data *xs100 = dev_get_platdata(&pdev->dev);
+
+	return (readw(xs100->base_regs + XS100_IRQSTATUS_BASE) & 0xaaaa) != 0;
+}
+
+/* These functions guarantee that the iomem is accessed with 32 bit
+ * cycles only. z_memcpy_fromio / z_memcpy_toio don't
+ */
+static void z_memcpy_fromio32(void *dst, const void __iomem *src, size_t bytes)
+{
+	while (bytes > 32) {
+		asm __volatile__
+		   ("movem.l (%0)+,%%d0-%%d7\n"
+		    "movem.l %%d0-%%d7,(%1)\n"
+		    "adda.l #32,%1" : "=a"(src), "=a"(dst)
+		    : "0"(src), "1"(dst) : "d0", "d1", "d2", "d3", "d4",
+					   "d5", "d6", "d7", "memory");
+		bytes -= 32;
+	}
+	while (bytes) {
+		*(uint32_t *)dst = z_readl(src);
+		src += 4;
+		dst += 4;
+		bytes -= 4;
+	}
+}
+
+static void z_memcpy_toio32(void __iomem *dst, const void *src, size_t bytes)
+{
+	while (bytes) {
+		z_writel(*(const uint32_t *)src, dst);
+		src += 4;
+		dst += 4;
+		bytes -= 4;
+	}
+}
+
+static void xs100_write(struct net_device *dev, const void *src,
+			unsigned int count)
+{
+	struct ei_device *ei_local = netdev_priv(dev);
+	struct platform_device *pdev = to_platform_device(dev->dev.parent);
+	struct xsurf100_ax_plat_data *xs100 = dev_get_platdata(&pdev->dev);
+
+	/* copy whole blocks */
+	while (count > XS100_8390_DATA_AREA_SIZE) {
+		z_memcpy_toio32(xs100->data_area +
+				XS100_8390_DATA_WRITE32_BASE, src,
+				XS100_8390_DATA_AREA_SIZE);
+		src += XS100_8390_DATA_AREA_SIZE;
+		count -= XS100_8390_DATA_AREA_SIZE;
+	}
+	/* copy whole dwords */
+	z_memcpy_toio32(xs100->data_area + XS100_8390_DATA_WRITE32_BASE,
+			src, count & ~3);
+	src += count & ~3;
+	if (count & 2) {
+		ei_outw(*(uint16_t *)src, ei_local->mem + NE_DATAPORT);
+		src += 2;
+	}
+	if (count & 1)
+		ei_outb(*(uint8_t *)src, ei_local->mem + NE_DATAPORT);
+}
+
+static void xs100_read(struct net_device *dev, void *dst, unsigned int count)
+{
+	struct ei_device *ei_local = netdev_priv(dev);
+	struct platform_device *pdev = to_platform_device(dev->dev.parent);
+	struct xsurf100_ax_plat_data *xs100 = dev_get_platdata(&pdev->dev);
+
+	/* copy whole blocks */
+	while (count > XS100_8390_DATA_AREA_SIZE) {
+		z_memcpy_fromio32(dst, xs100->data_area +
+				  XS100_8390_DATA_READ32_BASE,
+				  XS100_8390_DATA_AREA_SIZE);
+		dst += XS100_8390_DATA_AREA_SIZE;
+		count -= XS100_8390_DATA_AREA_SIZE;
+	}
+	/* copy whole dwords */
+	z_memcpy_fromio32(dst, xs100->data_area + XS100_8390_DATA_READ32_BASE,
+			  count & ~3);
+	dst += count & ~3;
+	if (count & 2) {
+		*(uint16_t *)dst = ei_inw(ei_local->mem + NE_DATAPORT);
+		dst += 2;
+	}
+	if (count & 1)
+		*(uint8_t *)dst = ei_inb(ei_local->mem + NE_DATAPORT);
+}
+
+/* Block input and output, similar to the Crynwr packet driver. If
+ * you are porting to a new ethercard, look at the packet driver
+ * source for hints. The NEx000 doesn't share the on-board packet
+ * memory -- you have to put the packet out through the "remote DMA"
+ * dataport using ei_outb.
+ */
+static void xs100_block_input(struct net_device *dev, int count,
+			      struct sk_buff *skb, int ring_offset)
+{
+	struct ei_device *ei_local = netdev_priv(dev);
+	void __iomem *nic_base = ei_local->mem;
+	char *buf = skb->data;
+
+	if (ei_local->dmaing) {
+		netdev_err(dev,
+			   "DMAing conflict in %s [DMAstat:%d][irqlock:%d]\n",
+			   __func__,
+			   ei_local->dmaing, ei_local->irqlock);
+		return;
+	}
+
+	ei_local->dmaing |= 0x01;
+
+	ei_outb(E8390_NODMA + E8390_PAGE0 + E8390_START, nic_base + NE_CMD);
+	ei_outb(count & 0xff, nic_base + EN0_RCNTLO);
+	ei_outb(count >> 8, nic_base + EN0_RCNTHI);
+	ei_outb(ring_offset & 0xff, nic_base + EN0_RSARLO);
+	ei_outb(ring_offset >> 8, nic_base + EN0_RSARHI);
+	ei_outb(E8390_RREAD + E8390_START, nic_base + NE_CMD);
+
+	xs100_read(dev, buf, count);
+
+	ei_local->dmaing &= ~1;
+}
+
+static void xs100_block_output(struct net_device *dev, int count,
+			       const unsigned char *buf, const int start_page)
+{
+	struct ei_device *ei_local = netdev_priv(dev);
+	void __iomem *nic_base = ei_local->mem;
+	unsigned long dma_start;
+
+	/* Round the count up for word writes. Do we need to do this?
+	 * What effect will an odd byte count have on the 8390?  I
+	 * should check someday.
+	 */
+	if (ei_local->word16 && (count & 0x01))
+		count++;
+
+	/* This *shouldn't* happen. If it does, it's the last thing
+	 * you'll see
+	 */
+	if (ei_local->dmaing) {
+		netdev_err(dev,
+			   "DMAing conflict in %s [DMAstat:%d][irqlock:%d]\n",
+			   __func__,
+			   ei_local->dmaing, ei_local->irqlock);
+		return;
+	}
+
+	ei_local->dmaing |= 0x01;
+	/* We should already be in page 0, but to be safe... */
+	ei_outb(E8390_PAGE0 + E8390_START + E8390_NODMA, nic_base + NE_CMD);
+
+	ei_outb(ENISR_RDC, nic_base + EN0_ISR);
+
+	/* Now the normal output. */
+	ei_outb(count & 0xff, nic_base + EN0_RCNTLO);
+	ei_outb(count >> 8, nic_base + EN0_RCNTHI);
+	ei_outb(0x00, nic_base + EN0_RSARLO);
+	ei_outb(start_page, nic_base + EN0_RSARHI);
+
+	ei_outb(E8390_RWRITE + E8390_START, nic_base + NE_CMD);
+
+	xs100_write(dev, buf, count);
+
+	dma_start = jiffies;
+
+	while ((ei_inb(nic_base + EN0_ISR) & ENISR_RDC) == 0) {
+		if (jiffies - dma_start > 2 * HZ / 100) {	/* 20ms */
+			netdev_warn(dev, "timeout waiting for Tx RDC.\n");
+			ei_local->reset_8390(dev);
+			ax_NS8390_init(dev, 1);
+			break;
+		}
+	}
+
+	ei_outb(ENISR_RDC, nic_base + EN0_ISR);	/* Ack intr. */
+	ei_local->dmaing &= ~0x01;
+}
+
+static int xsurf100_probe(struct zorro_dev *zdev,
+			  const struct zorro_device_id *ent)
+{
+	struct platform_device *pdev;
+	struct xsurf100_ax_plat_data ax88796_data;
+	struct resource res[2] = {
+		DEFINE_RES_NAMED(IRQ_AMIGA_PORTS, 1, NULL,
+				 IORESOURCE_IRQ | IORESOURCE_IRQ_SHAREABLE),
+		DEFINE_RES_MEM(zdev->resource.start + XS100_8390_BASE,
+			       4 * 0x20)
+	};
+	int reg;
+	/* This table is referenced in the device structure, so it must
+	 * outlive the scope of xsurf100_probe.
+	 */
+	static u32 reg_offsets[32];
+	int ret = 0;
+
+	/* X-Surf 100 control and 32 bit ring buffer data access areas.
+	 * These resources are not used by the ax88796 driver, so must
+	 * be requested here and passed via platform data.
+	 */
+
+	if (!request_mem_region(zdev->resource.start, 0x100, zdev->name)) {
+		dev_err(&zdev->dev, "cannot reserve X-Surf 100 control registers\n");
+		return -ENXIO;
+	}
+
+	if (!request_mem_region(zdev->resource.start +
+				XS100_8390_DATA32_BASE,
+				XS100_8390_DATA32_SIZE,
+				"X-Surf 100 32-bit data access")) {
+		dev_err(&zdev->dev, "cannot reserve 32-bit area\n");
+		ret = -ENXIO;
+		goto exit_req;
+	}
+
+	for (reg = 0; reg < 0x20; reg++)
+		reg_offsets[reg] = 4 * reg;
+
+	memset(&ax88796_data, 0, sizeof(ax88796_data));
+	ax88796_data.ax.flags = AXFLG_HAS_EEPROM;
+	ax88796_data.ax.wordlength = 2;
+	ax88796_data.ax.dcr_val = 0x48;
+	ax88796_data.ax.rcr_val = 0x40;
+	ax88796_data.ax.reg_offsets = reg_offsets;
+	ax88796_data.ax.check_irq = is_xsurf100_network_irq;
+	ax88796_data.base_regs = ioremap(zdev->resource.start, 0x100);
+
+	/* error handling for ioremap regs */
+	if (!ax88796_data.base_regs) {
+		dev_err(&zdev->dev, "Cannot ioremap area %pR (registers)\n",
+			&zdev->resource);
+
+		ret = -ENXIO;
+		goto exit_req2;
+	}
+
+	ax88796_data.data_area = ioremap(zdev->resource.start +
+			XS100_8390_DATA32_BASE, XS100_8390_DATA32_SIZE);
+
+	/* error handling for ioremap data */
+	if (!ax88796_data.data_area) {
+		dev_err(&zdev->dev,
+			"Cannot ioremap area %pR offset %x (32-bit access)\n",
+			&zdev->resource,  XS100_8390_DATA32_BASE);
+
+		ret = -ENXIO;
+		goto exit_mem;
+	}
+
+	ax88796_data.ax.block_output = xs100_block_output;
+	ax88796_data.ax.block_input = xs100_block_input;
+
+	pdev = platform_device_register_resndata(&zdev->dev, "ax88796",
+						 zdev->slotaddr, res, 2,
+						 &ax88796_data,
+						 sizeof(ax88796_data));
+
+	if (IS_ERR(pdev)) {
+		dev_err(&zdev->dev, "cannot register platform device\n");
+		ret = -ENXIO;
+		goto exit_mem2;
+	}
+
+	zorro_set_drvdata(zdev, pdev);
+
+	if (!ret)
+		return 0;
+
+ exit_mem2:
+	iounmap(ax88796_data.data_area);
+
+ exit_mem:
+	iounmap(ax88796_data.base_regs);
+
+ exit_req2:
+	release_mem_region(zdev->resource.start + XS100_8390_DATA32_BASE,
+			   XS100_8390_DATA32_SIZE);
+
+ exit_req:
+	release_mem_region(zdev->resource.start, 0x100);
+
+	return ret;
+}
+
+static void xsurf100_remove(struct zorro_dev *zdev)
+{
+	struct platform_device *pdev = zorro_get_drvdata(zdev);
+	struct xsurf100_ax_plat_data *xs100 = dev_get_platdata(&pdev->dev);
+
+	platform_device_unregister(pdev);
+
+	iounmap(xs100->base_regs);
+	release_mem_region(zdev->resource.start, 0x100);
+	iounmap(xs100->data_area);
+	release_mem_region(zdev->resource.start + XS100_8390_DATA32_BASE,
+			   XS100_8390_DATA32_SIZE);
+}
+
+static const struct zorro_device_id xsurf100_zorro_tbl[] = {
+	{ ZORRO_PROD_INDIVIDUAL_COMPUTERS_X_SURF100, },
+	{ 0 }
+};
+
+MODULE_DEVICE_TABLE(zorro, xsurf100_zorro_tbl);
+
+static struct zorro_driver xsurf100_driver = {
+	.name           = "xsurf100",
+	.id_table       = xsurf100_zorro_tbl,
+	.probe          = xsurf100_probe,
+	.remove         = xsurf100_remove,
+};
+
+module_driver(xsurf100_driver, zorro_register_driver, zorro_unregister_driver);
+
+MODULE_DESCRIPTION("X-Surf 100 driver");
+MODULE_AUTHOR("Michael Karcher <kernel@mkarcher.dialup.fu-berlin.de>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
index 603a5704dab8..af766fd61151 100644
--- a/drivers/net/ethernet/Kconfig
+++ b/drivers/net/ethernet/Kconfig
@@ -33,9 +33,9 @@ source "drivers/net/ethernet/aquantia/Kconfig"
 source "drivers/net/ethernet/arc/Kconfig"
 source "drivers/net/ethernet/atheros/Kconfig"
 source "drivers/net/ethernet/aurora/Kconfig"
-source "drivers/net/ethernet/cadence/Kconfig"
 source "drivers/net/ethernet/broadcom/Kconfig"
 source "drivers/net/ethernet/brocade/Kconfig"
+source "drivers/net/ethernet/cadence/Kconfig"
 source "drivers/net/ethernet/calxeda/Kconfig"
 source "drivers/net/ethernet/cavium/Kconfig"
 source "drivers/net/ethernet/chelsio/Kconfig"
@@ -72,16 +72,16 @@ source "drivers/net/ethernet/dec/Kconfig"
 source "drivers/net/ethernet/dlink/Kconfig"
 source "drivers/net/ethernet/emulex/Kconfig"
 source "drivers/net/ethernet/ezchip/Kconfig"
-source "drivers/net/ethernet/neterion/Kconfig"
 source "drivers/net/ethernet/faraday/Kconfig"
 source "drivers/net/ethernet/freescale/Kconfig"
 source "drivers/net/ethernet/fujitsu/Kconfig"
 source "drivers/net/ethernet/hisilicon/Kconfig"
 source "drivers/net/ethernet/hp/Kconfig"
 source "drivers/net/ethernet/huawei/Kconfig"
+source "drivers/net/ethernet/i825xx/Kconfig"
 source "drivers/net/ethernet/ibm/Kconfig"
 source "drivers/net/ethernet/intel/Kconfig"
-source "drivers/net/ethernet/i825xx/Kconfig"
+source "drivers/net/ethernet/neterion/Kconfig"
 source "drivers/net/ethernet/xscale/Kconfig"
 
 config JME
@@ -115,6 +115,7 @@ source "drivers/net/ethernet/mellanox/Kconfig"
 source "drivers/net/ethernet/micrel/Kconfig"
 source "drivers/net/ethernet/microchip/Kconfig"
 source "drivers/net/ethernet/moxa/Kconfig"
+source "drivers/net/ethernet/mscc/Kconfig"
 source "drivers/net/ethernet/myricom/Kconfig"
 
 config FEALNX
@@ -160,20 +161,21 @@ source "drivers/net/ethernet/packetengines/Kconfig"
 source "drivers/net/ethernet/pasemi/Kconfig"
 source "drivers/net/ethernet/qlogic/Kconfig"
 source "drivers/net/ethernet/qualcomm/Kconfig"
+source "drivers/net/ethernet/rdc/Kconfig"
 source "drivers/net/ethernet/realtek/Kconfig"
 source "drivers/net/ethernet/renesas/Kconfig"
-source "drivers/net/ethernet/rdc/Kconfig"
 source "drivers/net/ethernet/rocker/Kconfig"
 source "drivers/net/ethernet/samsung/Kconfig"
 source "drivers/net/ethernet/seeq/Kconfig"
-source "drivers/net/ethernet/silan/Kconfig"
-source "drivers/net/ethernet/sis/Kconfig"
 source "drivers/net/ethernet/sfc/Kconfig"
 source "drivers/net/ethernet/sgi/Kconfig"
+source "drivers/net/ethernet/silan/Kconfig"
+source "drivers/net/ethernet/sis/Kconfig"
 source "drivers/net/ethernet/smsc/Kconfig"
 source "drivers/net/ethernet/socionext/Kconfig"
 source "drivers/net/ethernet/stmicro/Kconfig"
 source "drivers/net/ethernet/sun/Kconfig"
+source "drivers/net/ethernet/synopsys/Kconfig"
 source "drivers/net/ethernet/tehuti/Kconfig"
 source "drivers/net/ethernet/ti/Kconfig"
 source "drivers/net/ethernet/toshiba/Kconfig"
@@ -182,6 +184,5 @@ source "drivers/net/ethernet/via/Kconfig"
 source "drivers/net/ethernet/wiznet/Kconfig"
 source "drivers/net/ethernet/xilinx/Kconfig"
 source "drivers/net/ethernet/xircom/Kconfig"
-source "drivers/net/ethernet/synopsys/Kconfig"
 
 endif # ETHERNET
diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile
index 2bfd2eea50bf..8fbfe9ce2fa5 100644
--- a/drivers/net/ethernet/Makefile
+++ b/drivers/net/ethernet/Makefile
@@ -55,6 +55,7 @@ obj-$(CONFIG_NET_VENDOR_MEDIATEK) += mediatek/
 obj-$(CONFIG_NET_VENDOR_MELLANOX) += mellanox/
 obj-$(CONFIG_NET_VENDOR_MICREL) += micrel/
 obj-$(CONFIG_NET_VENDOR_MICROCHIP) += microchip/
+obj-$(CONFIG_NET_VENDOR_MICROSEMI) += mscc/
 obj-$(CONFIG_NET_VENDOR_MOXART) += moxa/
 obj-$(CONFIG_NET_VENDOR_MYRI) += myricom/
 obj-$(CONFIG_FEALNX) += fealnx.o
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c
index a822e70c2af3..f2af87d70594 100644
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c
@@ -3385,32 +3385,6 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	return rc;
 }
 
-/*****************************************************************************/
-static int ena_sriov_configure(struct pci_dev *dev, int numvfs)
-{
-	int rc;
-
-	if (numvfs > 0) {
-		rc = pci_enable_sriov(dev, numvfs);
-		if (rc != 0) {
-			dev_err(&dev->dev,
-				"pci_enable_sriov failed to enable: %d vfs with the error: %d\n",
-				numvfs, rc);
-			return rc;
-		}
-
-		return numvfs;
-	}
-
-	if (numvfs == 0) {
-		pci_disable_sriov(dev);
-		return 0;
-	}
-
-	return -EINVAL;
-}
-
-/*****************************************************************************/
 /*****************************************************************************/
 
 /* ena_remove - Device Removal Routine
@@ -3526,7 +3500,7 @@ static struct pci_driver ena_pci_driver = {
 	.suspend    = ena_suspend,
 	.resume     = ena_resume,
 #endif
-	.sriov_configure = ena_sriov_configure,
+	.sriov_configure = pci_sriov_configure_simple,
 };
 
 static int __init ena_init(void)
diff --git a/drivers/net/ethernet/amd/amd8111e.c b/drivers/net/ethernet/amd/amd8111e.c
index c99e3e845ac0..a90080f12e67 100644
--- a/drivers/net/ethernet/amd/amd8111e.c
+++ b/drivers/net/ethernet/amd/amd8111e.c
@@ -1074,16 +1074,12 @@ static int amd8111e_calc_coalesce(struct net_device *dev)
 				amd8111e_set_coalesce(dev,TX_INTR_COAL);
 				coal_conf->tx_coal_type = MEDIUM_COALESCE;
 			}
-
-		}
-		else if(tx_pkt_size >= 1024){
-			if (tx_pkt_size >= 1024){
-				if(coal_conf->tx_coal_type !=  HIGH_COALESCE){
-					coal_conf->tx_timeout = 4;
-					coal_conf->tx_event_count = 8;
-					amd8111e_set_coalesce(dev,TX_INTR_COAL);
-					coal_conf->tx_coal_type = HIGH_COALESCE;
-				}
+		} else if (tx_pkt_size >= 1024) {
+			if (coal_conf->tx_coal_type != HIGH_COALESCE) {
+				coal_conf->tx_timeout = 4;
+				coal_conf->tx_event_count = 8;
+				amd8111e_set_coalesce(dev, TX_INTR_COAL);
+				coal_conf->tx_coal_type = HIGH_COALESCE;
 			}
 		}
 	}
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
index 7c204f05b418..24f1053b8785 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
@@ -1312,14 +1312,83 @@ int xgbe_powerup(struct net_device *netdev, unsigned int caller)
 	return 0;
 }
 
+static void xgbe_free_memory(struct xgbe_prv_data *pdata)
+{
+	struct xgbe_desc_if *desc_if = &pdata->desc_if;
+
+	/* Free the ring descriptors and buffers */
+	desc_if->free_ring_resources(pdata);
+
+	/* Free the channel and ring structures */
+	xgbe_free_channels(pdata);
+}
+
+static int xgbe_alloc_memory(struct xgbe_prv_data *pdata)
+{
+	struct xgbe_desc_if *desc_if = &pdata->desc_if;
+	struct net_device *netdev = pdata->netdev;
+	int ret;
+
+	if (pdata->new_tx_ring_count) {
+		pdata->tx_ring_count = pdata->new_tx_ring_count;
+		pdata->tx_q_count = pdata->tx_ring_count;
+		pdata->new_tx_ring_count = 0;
+	}
+
+	if (pdata->new_rx_ring_count) {
+		pdata->rx_ring_count = pdata->new_rx_ring_count;
+		pdata->new_rx_ring_count = 0;
+	}
+
+	/* Calculate the Rx buffer size before allocating rings */
+	pdata->rx_buf_size = xgbe_calc_rx_buf_size(netdev, netdev->mtu);
+
+	/* Allocate the channel and ring structures */
+	ret = xgbe_alloc_channels(pdata);
+	if (ret)
+		return ret;
+
+	/* Allocate the ring descriptors and buffers */
+	ret = desc_if->alloc_ring_resources(pdata);
+	if (ret)
+		goto err_channels;
+
+	/* Initialize the service and Tx timers */
+	xgbe_init_timers(pdata);
+
+	return 0;
+
+err_channels:
+	xgbe_free_memory(pdata);
+
+	return ret;
+}
+
 static int xgbe_start(struct xgbe_prv_data *pdata)
 {
 	struct xgbe_hw_if *hw_if = &pdata->hw_if;
 	struct xgbe_phy_if *phy_if = &pdata->phy_if;
 	struct net_device *netdev = pdata->netdev;
+	unsigned int i;
 	int ret;
 
-	DBGPR("-->xgbe_start\n");
+	/* Set the number of queues */
+	ret = netif_set_real_num_tx_queues(netdev, pdata->tx_ring_count);
+	if (ret) {
+		netdev_err(netdev, "error setting real tx queue count\n");
+		return ret;
+	}
+
+	ret = netif_set_real_num_rx_queues(netdev, pdata->rx_ring_count);
+	if (ret) {
+		netdev_err(netdev, "error setting real rx queue count\n");
+		return ret;
+	}
+
+	/* Set RSS lookup table data for programming */
+	for (i = 0; i < XGBE_RSS_MAX_TABLE_SIZE; i++)
+		XGMAC_SET_BITS(pdata->rss_table[i], MAC_RSSDR, DMCH,
+			       i % pdata->rx_ring_count);
 
 	ret = hw_if->init(pdata);
 	if (ret)
@@ -1347,8 +1416,6 @@ static int xgbe_start(struct xgbe_prv_data *pdata)
 
 	clear_bit(XGBE_STOPPED, &pdata->dev_state);
 
-	DBGPR("<--xgbe_start\n");
-
 	return 0;
 
 err_irqs:
@@ -1426,10 +1493,22 @@ static void xgbe_stopdev(struct work_struct *work)
 	netdev_alert(pdata->netdev, "device stopped\n");
 }
 
-static void xgbe_restart_dev(struct xgbe_prv_data *pdata)
+void xgbe_full_restart_dev(struct xgbe_prv_data *pdata)
 {
-	DBGPR("-->xgbe_restart_dev\n");
+	/* If not running, "restart" will happen on open */
+	if (!netif_running(pdata->netdev))
+		return;
+
+	xgbe_stop(pdata);
+
+	xgbe_free_memory(pdata);
+	xgbe_alloc_memory(pdata);
+
+	xgbe_start(pdata);
+}
 
+void xgbe_restart_dev(struct xgbe_prv_data *pdata)
+{
 	/* If not running, "restart" will happen on open */
 	if (!netif_running(pdata->netdev))
 		return;
@@ -1440,8 +1519,6 @@ static void xgbe_restart_dev(struct xgbe_prv_data *pdata)
 	xgbe_free_rx_data(pdata);
 
 	xgbe_start(pdata);
-
-	DBGPR("<--xgbe_restart_dev\n");
 }
 
 static void xgbe_restart(struct work_struct *work)
@@ -1827,11 +1904,8 @@ static void xgbe_packet_info(struct xgbe_prv_data *pdata,
 static int xgbe_open(struct net_device *netdev)
 {
 	struct xgbe_prv_data *pdata = netdev_priv(netdev);
-	struct xgbe_desc_if *desc_if = &pdata->desc_if;
 	int ret;
 
-	DBGPR("-->xgbe_open\n");
-
 	/* Create the various names based on netdev name */
 	snprintf(pdata->an_name, sizeof(pdata->an_name) - 1, "%s-pcs",
 		 netdev_name(netdev));
@@ -1876,43 +1950,25 @@ static int xgbe_open(struct net_device *netdev)
 		goto err_sysclk;
 	}
 
-	/* Calculate the Rx buffer size before allocating rings */
-	ret = xgbe_calc_rx_buf_size(netdev, netdev->mtu);
-	if (ret < 0)
-		goto err_ptpclk;
-	pdata->rx_buf_size = ret;
-
-	/* Allocate the channel and ring structures */
-	ret = xgbe_alloc_channels(pdata);
-	if (ret)
-		goto err_ptpclk;
-
-	/* Allocate the ring descriptors and buffers */
-	ret = desc_if->alloc_ring_resources(pdata);
-	if (ret)
-		goto err_channels;
-
 	INIT_WORK(&pdata->service_work, xgbe_service);
 	INIT_WORK(&pdata->restart_work, xgbe_restart);
 	INIT_WORK(&pdata->stopdev_work, xgbe_stopdev);
 	INIT_WORK(&pdata->tx_tstamp_work, xgbe_tx_tstamp);
-	xgbe_init_timers(pdata);
+
+	ret = xgbe_alloc_memory(pdata);
+	if (ret)
+		goto err_ptpclk;
 
 	ret = xgbe_start(pdata);
 	if (ret)
-		goto err_rings;
+		goto err_mem;
 
 	clear_bit(XGBE_DOWN, &pdata->dev_state);
 
-	DBGPR("<--xgbe_open\n");
-
 	return 0;
 
-err_rings:
-	desc_if->free_ring_resources(pdata);
-
-err_channels:
-	xgbe_free_channels(pdata);
+err_mem:
+	xgbe_free_memory(pdata);
 
 err_ptpclk:
 	clk_disable_unprepare(pdata->ptpclk);
@@ -1932,18 +1988,11 @@ static int xgbe_open(struct net_device *netdev)
 static int xgbe_close(struct net_device *netdev)
 {
 	struct xgbe_prv_data *pdata = netdev_priv(netdev);
-	struct xgbe_desc_if *desc_if = &pdata->desc_if;
-
-	DBGPR("-->xgbe_close\n");
 
 	/* Stop the device */
 	xgbe_stop(pdata);
 
-	/* Free the ring descriptors and buffers */
-	desc_if->free_ring_resources(pdata);
-
-	/* Free the channel and ring structures */
-	xgbe_free_channels(pdata);
+	xgbe_free_memory(pdata);
 
 	/* Disable the clocks */
 	clk_disable_unprepare(pdata->ptpclk);
@@ -1957,8 +2006,6 @@ static int xgbe_close(struct net_device *netdev)
 
 	set_bit(XGBE_DOWN, &pdata->dev_state);
 
-	DBGPR("<--xgbe_close\n");
-
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
index ff397bb25042..a880f10e3e70 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
@@ -626,6 +626,217 @@ static int xgbe_get_ts_info(struct net_device *netdev,
 	return 0;
 }
 
+static int xgbe_get_module_info(struct net_device *netdev,
+				struct ethtool_modinfo *modinfo)
+{
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+
+	return pdata->phy_if.module_info(pdata, modinfo);
+}
+
+static int xgbe_get_module_eeprom(struct net_device *netdev,
+				  struct ethtool_eeprom *eeprom, u8 *data)
+{
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+
+	return pdata->phy_if.module_eeprom(pdata, eeprom, data);
+}
+
+static void xgbe_get_ringparam(struct net_device *netdev,
+			       struct ethtool_ringparam *ringparam)
+{
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+
+	ringparam->rx_max_pending = XGBE_RX_DESC_CNT_MAX;
+	ringparam->tx_max_pending = XGBE_TX_DESC_CNT_MAX;
+	ringparam->rx_pending = pdata->rx_desc_count;
+	ringparam->tx_pending = pdata->tx_desc_count;
+}
+
+static int xgbe_set_ringparam(struct net_device *netdev,
+			      struct ethtool_ringparam *ringparam)
+{
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+	unsigned int rx, tx;
+
+	if (ringparam->rx_mini_pending || ringparam->rx_jumbo_pending) {
+		netdev_err(netdev, "unsupported ring parameter\n");
+		return -EINVAL;
+	}
+
+	if ((ringparam->rx_pending < XGBE_RX_DESC_CNT_MIN) ||
+	    (ringparam->rx_pending > XGBE_RX_DESC_CNT_MAX)) {
+		netdev_err(netdev,
+			   "rx ring parameter must be between %u and %u\n",
+			   XGBE_RX_DESC_CNT_MIN, XGBE_RX_DESC_CNT_MAX);
+		return -EINVAL;
+	}
+
+	if ((ringparam->tx_pending < XGBE_TX_DESC_CNT_MIN) ||
+	    (ringparam->tx_pending > XGBE_TX_DESC_CNT_MAX)) {
+		netdev_err(netdev,
+			   "tx ring parameter must be between %u and %u\n",
+			   XGBE_TX_DESC_CNT_MIN, XGBE_TX_DESC_CNT_MAX);
+		return -EINVAL;
+	}
+
+	rx = __rounddown_pow_of_two(ringparam->rx_pending);
+	if (rx != ringparam->rx_pending)
+		netdev_notice(netdev,
+			      "rx ring parameter rounded to power of two: %u\n",
+			      rx);
+
+	tx = __rounddown_pow_of_two(ringparam->tx_pending);
+	if (tx != ringparam->tx_pending)
+		netdev_notice(netdev,
+			      "tx ring parameter rounded to power of two: %u\n",
+			      tx);
+
+	if ((rx == pdata->rx_desc_count) &&
+	    (tx == pdata->tx_desc_count))
+		goto out;
+
+	pdata->rx_desc_count = rx;
+	pdata->tx_desc_count = tx;
+
+	xgbe_restart_dev(pdata);
+
+out:
+	return 0;
+}
+
+static void xgbe_get_channels(struct net_device *netdev,
+			      struct ethtool_channels *channels)
+{
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+	unsigned int rx, tx, combined;
+
+	/* Calculate maximums allowed:
+	 *   - Take into account the number of available IRQs
+	 *   - Do not take into account the number of online CPUs so that
+	 *     the user can over-subscribe if desired
+	 *   - Tx is additionally limited by the number of hardware queues
+	 */
+	rx = min(pdata->hw_feat.rx_ch_cnt, pdata->rx_max_channel_count);
+	rx = min(rx, pdata->channel_irq_count);
+	tx = min(pdata->hw_feat.tx_ch_cnt, pdata->tx_max_channel_count);
+	tx = min(tx, pdata->channel_irq_count);
+	tx = min(tx, pdata->tx_max_q_count);
+
+	combined = min(rx, tx);
+
+	channels->max_combined = combined;
+	channels->max_rx = rx ? rx - 1 : 0;
+	channels->max_tx = tx ? tx - 1 : 0;
+
+	/* Get current settings based on device state */
+	rx = pdata->new_rx_ring_count ? : pdata->rx_ring_count;
+	tx = pdata->new_tx_ring_count ? : pdata->tx_ring_count;
+
+	combined = min(rx, tx);
+	rx -= combined;
+	tx -= combined;
+
+	channels->combined_count = combined;
+	channels->rx_count = rx;
+	channels->tx_count = tx;
+}
+
+static void xgbe_print_set_channels_input(struct net_device *netdev,
+					  struct ethtool_channels *channels)
+{
+	netdev_err(netdev, "channel inputs: combined=%u, rx-only=%u, tx-only=%u\n",
+		   channels->combined_count, channels->rx_count,
+		   channels->tx_count);
+}
+
+static int xgbe_set_channels(struct net_device *netdev,
+			     struct ethtool_channels *channels)
+{
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+	unsigned int rx, rx_curr, tx, tx_curr, combined;
+
+	/* Calculate maximums allowed:
+	 *   - Take into account the number of available IRQs
+	 *   - Do not take into account the number of online CPUs so that
+	 *     the user can over-subscribe if desired
+	 *   - Tx is additionally limited by the number of hardware queues
+	 */
+	rx = min(pdata->hw_feat.rx_ch_cnt, pdata->rx_max_channel_count);
+	rx = min(rx, pdata->channel_irq_count);
+	tx = min(pdata->hw_feat.tx_ch_cnt, pdata->tx_max_channel_count);
+	tx = min(tx, pdata->tx_max_q_count);
+	tx = min(tx, pdata->channel_irq_count);
+
+	combined = min(rx, tx);
+
+	/* Should not be setting other count */
+	if (channels->other_count) {
+		netdev_err(netdev,
+			   "other channel count must be zero\n");
+		return -EINVAL;
+	}
+
+	/* Require at least one Combined (Rx and Tx) channel */
+	if (!channels->combined_count) {
+		netdev_err(netdev,
+			   "at least one combined Rx/Tx channel is required\n");
+		xgbe_print_set_channels_input(netdev, channels);
+		return -EINVAL;
+	}
+
+	/* Check combined channels */
+	if (channels->combined_count > combined) {
+		netdev_err(netdev,
+			   "combined channel count cannot exceed %u\n",
+			   combined);
+		xgbe_print_set_channels_input(netdev, channels);
+		return -EINVAL;
+	}
+
+	/* Can have some Rx-only or Tx-only channels, but not both */
+	if (channels->rx_count && channels->tx_count) {
+		netdev_err(netdev,
+			   "cannot specify both Rx-only and Tx-only channels\n");
+		xgbe_print_set_channels_input(netdev, channels);
+		return -EINVAL;
+	}
+
+	/* Check that we don't exceed the maximum number of channels */
+	if ((channels->combined_count + channels->rx_count) > rx) {
+		netdev_err(netdev,
+			   "total Rx channels (%u) requested exceeds maximum available (%u)\n",
+			   channels->combined_count + channels->rx_count, rx);
+		xgbe_print_set_channels_input(netdev, channels);
+		return -EINVAL;
+	}
+
+	if ((channels->combined_count + channels->tx_count) > tx) {
+		netdev_err(netdev,
+			   "total Tx channels (%u) requested exceeds maximum available (%u)\n",
+			   channels->combined_count + channels->tx_count, tx);
+		xgbe_print_set_channels_input(netdev, channels);
+		return -EINVAL;
+	}
+
+	rx = channels->combined_count + channels->rx_count;
+	tx = channels->combined_count + channels->tx_count;
+
+	rx_curr = pdata->new_rx_ring_count ? : pdata->rx_ring_count;
+	tx_curr = pdata->new_tx_ring_count ? : pdata->tx_ring_count;
+
+	if ((rx == rx_curr) && (tx == tx_curr))
+		goto out;
+
+	pdata->new_rx_ring_count = rx;
+	pdata->new_tx_ring_count = tx;
+
+	xgbe_full_restart_dev(pdata);
+
+out:
+	return 0;
+}
+
 static const struct ethtool_ops xgbe_ethtool_ops = {
 	.get_drvinfo = xgbe_get_drvinfo,
 	.get_msglevel = xgbe_get_msglevel,
@@ -646,6 +857,12 @@ static const struct ethtool_ops xgbe_ethtool_ops = {
 	.get_ts_info = xgbe_get_ts_info,
 	.get_link_ksettings = xgbe_get_link_ksettings,
 	.set_link_ksettings = xgbe_set_link_ksettings,
+	.get_module_info = xgbe_get_module_info,
+	.get_module_eeprom = xgbe_get_module_eeprom,
+	.get_ringparam = xgbe_get_ringparam,
+	.set_ringparam = xgbe_set_ringparam,
+	.get_channels = xgbe_get_channels,
+	.set_channels = xgbe_set_channels,
 };
 
 const struct ethtool_ops *xgbe_get_ethtool_ops(void)
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-main.c b/drivers/net/ethernet/amd/xgbe/xgbe-main.c
index 441d0973957b..b41f23679a08 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-main.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-main.c
@@ -265,7 +265,6 @@ int xgbe_config_netdev(struct xgbe_prv_data *pdata)
 {
 	struct net_device *netdev = pdata->netdev;
 	struct device *dev = pdata->dev;
-	unsigned int i;
 	int ret;
 
 	netdev->irq = pdata->dev_irq;
@@ -324,26 +323,9 @@ int xgbe_config_netdev(struct xgbe_prv_data *pdata)
 				pdata->tx_ring_count, pdata->rx_ring_count);
 	}
 
-	/* Set the number of queues */
-	ret = netif_set_real_num_tx_queues(netdev, pdata->tx_ring_count);
-	if (ret) {
-		dev_err(dev, "error setting real tx queue count\n");
-		return ret;
-	}
-
-	ret = netif_set_real_num_rx_queues(netdev, pdata->rx_ring_count);
-	if (ret) {
-		dev_err(dev, "error setting real rx queue count\n");
-		return ret;
-	}
-
-	/* Initialize RSS hash key and lookup table */
+	/* Initialize RSS hash key */
 	netdev_rss_key_fill(pdata->rss_key, sizeof(pdata->rss_key));
 
-	for (i = 0; i < XGBE_RSS_MAX_TABLE_SIZE; i++)
-		XGMAC_SET_BITS(pdata->rss_table[i], MAC_RSSDR, DMCH,
-			       i % pdata->rx_ring_count);
-
 	XGMAC_SET_BITS(pdata->rss_options, MAC_RSSCR, IP2TE, 1);
 	XGMAC_SET_BITS(pdata->rss_options, MAC_RSSCR, TCP4TE, 1);
 	XGMAC_SET_BITS(pdata->rss_options, MAC_RSSCR, UDP4TE, 1);
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
index 1b45cd73a258..4b5d625de8f0 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
@@ -126,6 +126,24 @@
 #include "xgbe.h"
 #include "xgbe-common.h"
 
+static int xgbe_phy_module_eeprom(struct xgbe_prv_data *pdata,
+				  struct ethtool_eeprom *eeprom, u8 *data)
+{
+	if (!pdata->phy_if.phy_impl.module_eeprom)
+		return -ENXIO;
+
+	return pdata->phy_if.phy_impl.module_eeprom(pdata, eeprom, data);
+}
+
+static int xgbe_phy_module_info(struct xgbe_prv_data *pdata,
+				struct ethtool_modinfo *modinfo)
+{
+	if (!pdata->phy_if.phy_impl.module_info)
+		return -ENXIO;
+
+	return pdata->phy_if.phy_impl.module_info(pdata, modinfo);
+}
+
 static void xgbe_an37_clear_interrupts(struct xgbe_prv_data *pdata)
 {
 	int reg;
@@ -198,31 +216,8 @@ static void xgbe_an_clear_interrupts_all(struct xgbe_prv_data *pdata)
 	xgbe_an37_clear_interrupts(pdata);
 }
 
-static void xgbe_an73_enable_kr_training(struct xgbe_prv_data *pdata)
-{
-	unsigned int reg;
-
-	reg = XMDIO_READ(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_PMD_CTRL);
-
-	reg |= XGBE_KR_TRAINING_ENABLE;
-	XMDIO_WRITE(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_PMD_CTRL, reg);
-}
-
-static void xgbe_an73_disable_kr_training(struct xgbe_prv_data *pdata)
-{
-	unsigned int reg;
-
-	reg = XMDIO_READ(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_PMD_CTRL);
-
-	reg &= ~XGBE_KR_TRAINING_ENABLE;
-	XMDIO_WRITE(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_PMD_CTRL, reg);
-}
-
 static void xgbe_kr_mode(struct xgbe_prv_data *pdata)
 {
-	/* Enable KR training */
-	xgbe_an73_enable_kr_training(pdata);
-
 	/* Set MAC to 10G speed */
 	pdata->hw_if.set_speed(pdata, SPEED_10000);
 
@@ -232,9 +227,6 @@ static void xgbe_kr_mode(struct xgbe_prv_data *pdata)
 
 static void xgbe_kx_2500_mode(struct xgbe_prv_data *pdata)
 {
-	/* Disable KR training */
-	xgbe_an73_disable_kr_training(pdata);
-
 	/* Set MAC to 2.5G speed */
 	pdata->hw_if.set_speed(pdata, SPEED_2500);
 
@@ -244,9 +236,6 @@ static void xgbe_kx_2500_mode(struct xgbe_prv_data *pdata)
 
 static void xgbe_kx_1000_mode(struct xgbe_prv_data *pdata)
 {
-	/* Disable KR training */
-	xgbe_an73_disable_kr_training(pdata);
-
 	/* Set MAC to 1G speed */
 	pdata->hw_if.set_speed(pdata, SPEED_1000);
 
@@ -260,9 +249,6 @@ static void xgbe_sfi_mode(struct xgbe_prv_data *pdata)
 	if (pdata->kr_redrv)
 		return xgbe_kr_mode(pdata);
 
-	/* Disable KR training */
-	xgbe_an73_disable_kr_training(pdata);
-
 	/* Set MAC to 10G speed */
 	pdata->hw_if.set_speed(pdata, SPEED_10000);
 
@@ -272,9 +258,6 @@ static void xgbe_sfi_mode(struct xgbe_prv_data *pdata)
 
 static void xgbe_x_mode(struct xgbe_prv_data *pdata)
 {
-	/* Disable KR training */
-	xgbe_an73_disable_kr_training(pdata);
-
 	/* Set MAC to 1G speed */
 	pdata->hw_if.set_speed(pdata, SPEED_1000);
 
@@ -284,9 +267,6 @@ static void xgbe_x_mode(struct xgbe_prv_data *pdata)
 
 static void xgbe_sgmii_1000_mode(struct xgbe_prv_data *pdata)
 {
-	/* Disable KR training */
-	xgbe_an73_disable_kr_training(pdata);
-
 	/* Set MAC to 1G speed */
 	pdata->hw_if.set_speed(pdata, SPEED_1000);
 
@@ -296,9 +276,6 @@ static void xgbe_sgmii_1000_mode(struct xgbe_prv_data *pdata)
 
 static void xgbe_sgmii_100_mode(struct xgbe_prv_data *pdata)
 {
-	/* Disable KR training */
-	xgbe_an73_disable_kr_training(pdata);
-
 	/* Set MAC to 1G speed */
 	pdata->hw_if.set_speed(pdata, SPEED_1000);
 
@@ -354,13 +331,15 @@ static void xgbe_switch_mode(struct xgbe_prv_data *pdata)
 	xgbe_change_mode(pdata, pdata->phy_if.phy_impl.switch_mode(pdata));
 }
 
-static void xgbe_set_mode(struct xgbe_prv_data *pdata,
+static bool xgbe_set_mode(struct xgbe_prv_data *pdata,
 			  enum xgbe_mode mode)
 {
 	if (mode == xgbe_cur_mode(pdata))
-		return;
+		return false;
 
 	xgbe_change_mode(pdata, mode);
+
+	return true;
 }
 
 static bool xgbe_use_mode(struct xgbe_prv_data *pdata,
@@ -407,6 +386,12 @@ static void xgbe_an73_set(struct xgbe_prv_data *pdata, bool enable,
 {
 	unsigned int reg;
 
+	/* Disable KR training for now */
+	reg = XMDIO_READ(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_PMD_CTRL);
+	reg &= ~XGBE_KR_TRAINING_ENABLE;
+	XMDIO_WRITE(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_PMD_CTRL, reg);
+
+	/* Update AN settings */
 	reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_CTRL1);
 	reg &= ~MDIO_AN_CTRL1_ENABLE;
 
@@ -504,21 +489,19 @@ static enum xgbe_an xgbe_an73_tx_training(struct xgbe_prv_data *pdata,
 	XMDIO_WRITE(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_FECCTRL, reg);
 
 	/* Start KR training */
-	reg = XMDIO_READ(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_PMD_CTRL);
-	if (reg & XGBE_KR_TRAINING_ENABLE) {
-		if (pdata->phy_if.phy_impl.kr_training_pre)
-			pdata->phy_if.phy_impl.kr_training_pre(pdata);
+	if (pdata->phy_if.phy_impl.kr_training_pre)
+		pdata->phy_if.phy_impl.kr_training_pre(pdata);
 
-		reg |= XGBE_KR_TRAINING_START;
-		XMDIO_WRITE(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_PMD_CTRL,
-			    reg);
+	reg = XMDIO_READ(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_PMD_CTRL);
+	reg |= XGBE_KR_TRAINING_ENABLE;
+	reg |= XGBE_KR_TRAINING_START;
+	XMDIO_WRITE(pdata, MDIO_MMD_PMAPMD, MDIO_PMA_10GBR_PMD_CTRL, reg);
 
-		netif_dbg(pdata, link, pdata->netdev,
-			  "KR training initiated\n");
+	netif_dbg(pdata, link, pdata->netdev,
+		  "KR training initiated\n");
 
-		if (pdata->phy_if.phy_impl.kr_training_post)
-			pdata->phy_if.phy_impl.kr_training_post(pdata);
-	}
+	if (pdata->phy_if.phy_impl.kr_training_post)
+		pdata->phy_if.phy_impl.kr_training_post(pdata);
 
 	return XGBE_AN_PAGE_RECEIVED;
 }
@@ -1197,21 +1180,23 @@ static int xgbe_phy_config_fixed(struct xgbe_prv_data *pdata)
 	return 0;
 }
 
-static int __xgbe_phy_config_aneg(struct xgbe_prv_data *pdata)
+static int __xgbe_phy_config_aneg(struct xgbe_prv_data *pdata, bool set_mode)
 {
 	int ret;
 
+	mutex_lock(&pdata->an_mutex);
+
 	set_bit(XGBE_LINK_INIT, &pdata->dev_state);
 	pdata->link_check = jiffies;
 
 	ret = pdata->phy_if.phy_impl.an_config(pdata);
 	if (ret)
-		return ret;
+		goto out;
 
 	if (pdata->phy.autoneg != AUTONEG_ENABLE) {
 		ret = xgbe_phy_config_fixed(pdata);
 		if (ret || !pdata->kr_redrv)
-			return ret;
+			goto out;
 
 		netif_dbg(pdata, link, pdata->netdev, "AN redriver support\n");
 	} else {
@@ -1221,24 +1206,27 @@ static int __xgbe_phy_config_aneg(struct xgbe_prv_data *pdata)
 	/* Disable auto-negotiation interrupt */
 	disable_irq(pdata->an_irq);
 
-	/* Start auto-negotiation in a supported mode */
-	if (xgbe_use_mode(pdata, XGBE_MODE_KR)) {
-		xgbe_set_mode(pdata, XGBE_MODE_KR);
-	} else if (xgbe_use_mode(pdata, XGBE_MODE_KX_2500)) {
-		xgbe_set_mode(pdata, XGBE_MODE_KX_2500);
-	} else if (xgbe_use_mode(pdata, XGBE_MODE_KX_1000)) {
-		xgbe_set_mode(pdata, XGBE_MODE_KX_1000);
-	} else if (xgbe_use_mode(pdata, XGBE_MODE_SFI)) {
-		xgbe_set_mode(pdata, XGBE_MODE_SFI);
-	} else if (xgbe_use_mode(pdata, XGBE_MODE_X)) {
-		xgbe_set_mode(pdata, XGBE_MODE_X);
-	} else if (xgbe_use_mode(pdata, XGBE_MODE_SGMII_1000)) {
-		xgbe_set_mode(pdata, XGBE_MODE_SGMII_1000);
-	} else if (xgbe_use_mode(pdata, XGBE_MODE_SGMII_100)) {
-		xgbe_set_mode(pdata, XGBE_MODE_SGMII_100);
-	} else {
-		enable_irq(pdata->an_irq);
-		return -EINVAL;
+	if (set_mode) {
+		/* Start auto-negotiation in a supported mode */
+		if (xgbe_use_mode(pdata, XGBE_MODE_KR)) {
+			xgbe_set_mode(pdata, XGBE_MODE_KR);
+		} else if (xgbe_use_mode(pdata, XGBE_MODE_KX_2500)) {
+			xgbe_set_mode(pdata, XGBE_MODE_KX_2500);
+		} else if (xgbe_use_mode(pdata, XGBE_MODE_KX_1000)) {
+			xgbe_set_mode(pdata, XGBE_MODE_KX_1000);
+		} else if (xgbe_use_mode(pdata, XGBE_MODE_SFI)) {
+			xgbe_set_mode(pdata, XGBE_MODE_SFI);
+		} else if (xgbe_use_mode(pdata, XGBE_MODE_X)) {
+			xgbe_set_mode(pdata, XGBE_MODE_X);
+		} else if (xgbe_use_mode(pdata, XGBE_MODE_SGMII_1000)) {
+			xgbe_set_mode(pdata, XGBE_MODE_SGMII_1000);
+		} else if (xgbe_use_mode(pdata, XGBE_MODE_SGMII_100)) {
+			xgbe_set_mode(pdata, XGBE_MODE_SGMII_100);
+		} else {
+			enable_irq(pdata->an_irq);
+			ret = -EINVAL;
+			goto out;
+		}
 	}
 
 	/* Disable and stop any in progress auto-negotiation */
@@ -1258,16 +1246,7 @@ static int __xgbe_phy_config_aneg(struct xgbe_prv_data *pdata)
 	xgbe_an_init(pdata);
 	xgbe_an_restart(pdata);
 
-	return 0;
-}
-
-static int xgbe_phy_config_aneg(struct xgbe_prv_data *pdata)
-{
-	int ret;
-
-	mutex_lock(&pdata->an_mutex);
-
-	ret = __xgbe_phy_config_aneg(pdata);
+out:
 	if (ret)
 		set_bit(XGBE_LINK_ERR, &pdata->dev_state);
 	else
@@ -1278,6 +1257,16 @@ static int xgbe_phy_config_aneg(struct xgbe_prv_data *pdata)
 	return ret;
 }
 
+static int xgbe_phy_config_aneg(struct xgbe_prv_data *pdata)
+{
+	return __xgbe_phy_config_aneg(pdata, true);
+}
+
+static int xgbe_phy_reconfig_aneg(struct xgbe_prv_data *pdata)
+{
+	return __xgbe_phy_config_aneg(pdata, false);
+}
+
 static bool xgbe_phy_aneg_done(struct xgbe_prv_data *pdata)
 {
 	return (pdata->an_result == XGBE_AN_COMPLETE);
@@ -1334,7 +1323,8 @@ static void xgbe_phy_status_result(struct xgbe_prv_data *pdata)
 
 	pdata->phy.duplex = DUPLEX_FULL;
 
-	xgbe_set_mode(pdata, mode);
+	if (xgbe_set_mode(pdata, mode) && pdata->an_again)
+		xgbe_phy_reconfig_aneg(pdata);
 }
 
 static void xgbe_phy_status(struct xgbe_prv_data *pdata)
@@ -1639,4 +1629,7 @@ void xgbe_init_function_ptrs_phy(struct xgbe_phy_if *phy_if)
 	phy_if->phy_valid_speed = xgbe_phy_valid_speed;
 
 	phy_if->an_isr          = xgbe_an_combined_isr;
+
+	phy_if->module_info     = xgbe_phy_module_info;
+	phy_if->module_eeprom   = xgbe_phy_module_eeprom;
 }
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
index 82d1f416ee2a..7b86240ecd5f 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
@@ -335,16 +335,33 @@ static int xgbe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	pdata->awcr = XGBE_DMA_PCI_AWCR;
 	pdata->awarcr = XGBE_DMA_PCI_AWARCR;
 
+	/* Read the port property registers */
+	pdata->pp0 = XP_IOREAD(pdata, XP_PROP_0);
+	pdata->pp1 = XP_IOREAD(pdata, XP_PROP_1);
+	pdata->pp2 = XP_IOREAD(pdata, XP_PROP_2);
+	pdata->pp3 = XP_IOREAD(pdata, XP_PROP_3);
+	pdata->pp4 = XP_IOREAD(pdata, XP_PROP_4);
+	if (netif_msg_probe(pdata)) {
+		dev_dbg(dev, "port property 0 = %#010x\n", pdata->pp0);
+		dev_dbg(dev, "port property 1 = %#010x\n", pdata->pp1);
+		dev_dbg(dev, "port property 2 = %#010x\n", pdata->pp2);
+		dev_dbg(dev, "port property 3 = %#010x\n", pdata->pp3);
+		dev_dbg(dev, "port property 4 = %#010x\n", pdata->pp4);
+	}
+
 	/* Set the maximum channels and queues */
-	reg = XP_IOREAD(pdata, XP_PROP_1);
-	pdata->tx_max_channel_count = XP_GET_BITS(reg, XP_PROP_1, MAX_TX_DMA);
-	pdata->rx_max_channel_count = XP_GET_BITS(reg, XP_PROP_1, MAX_RX_DMA);
-	pdata->tx_max_q_count = XP_GET_BITS(reg, XP_PROP_1, MAX_TX_QUEUES);
-	pdata->rx_max_q_count = XP_GET_BITS(reg, XP_PROP_1, MAX_RX_QUEUES);
+	pdata->tx_max_channel_count = XP_GET_BITS(pdata->pp1, XP_PROP_1,
+						  MAX_TX_DMA);
+	pdata->rx_max_channel_count = XP_GET_BITS(pdata->pp1, XP_PROP_1,
+						  MAX_RX_DMA);
+	pdata->tx_max_q_count = XP_GET_BITS(pdata->pp1, XP_PROP_1,
+					    MAX_TX_QUEUES);
+	pdata->rx_max_q_count = XP_GET_BITS(pdata->pp1, XP_PROP_1,
+					    MAX_RX_QUEUES);
 	if (netif_msg_probe(pdata)) {
 		dev_dbg(dev, "max tx/rx channel count = %u/%u\n",
 			pdata->tx_max_channel_count,
-			pdata->tx_max_channel_count);
+			pdata->rx_max_channel_count);
 		dev_dbg(dev, "max tx/rx hw queue count = %u/%u\n",
 			pdata->tx_max_q_count, pdata->rx_max_q_count);
 	}
@@ -353,12 +370,13 @@ static int xgbe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	xgbe_set_counts(pdata);
 
 	/* Set the maximum fifo amounts */
-	reg = XP_IOREAD(pdata, XP_PROP_2);
-	pdata->tx_max_fifo_size = XP_GET_BITS(reg, XP_PROP_2, TX_FIFO_SIZE);
+	pdata->tx_max_fifo_size = XP_GET_BITS(pdata->pp2, XP_PROP_2,
+					      TX_FIFO_SIZE);
 	pdata->tx_max_fifo_size *= 16384;
 	pdata->tx_max_fifo_size = min(pdata->tx_max_fifo_size,
 				      pdata->vdata->tx_max_fifo_size);
-	pdata->rx_max_fifo_size = XP_GET_BITS(reg, XP_PROP_2, RX_FIFO_SIZE);
+	pdata->rx_max_fifo_size = XP_GET_BITS(pdata->pp2, XP_PROP_2,
+					      RX_FIFO_SIZE);
 	pdata->rx_max_fifo_size *= 16384;
 	pdata->rx_max_fifo_size = min(pdata->rx_max_fifo_size,
 				      pdata->vdata->rx_max_fifo_size);
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
index aac884314000..3ceb4f95ca7c 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
@@ -119,6 +119,7 @@
 #include <linux/kmod.h>
 #include <linux/mdio.h>
 #include <linux/phy.h>
+#include <linux/ethtool.h>
 
 #include "xgbe.h"
 #include "xgbe-common.h"
@@ -270,6 +271,15 @@ struct xgbe_sfp_eeprom {
 	u8 vendor[32];
 };
 
+#define XGBE_SFP_DIAGS_SUPPORTED(_x)			\
+	((_x)->extd[XGBE_SFP_EXTD_SFF_8472] &&		\
+	 !((_x)->extd[XGBE_SFP_EXTD_DIAG] & XGBE_SFP_EXTD_DIAG_ADDR_CHANGE))
+
+#define XGBE_SFP_EEPROM_BASE_LEN	256
+#define XGBE_SFP_EEPROM_DIAG_LEN	256
+#define XGBE_SFP_EEPROM_MAX		(XGBE_SFP_EEPROM_BASE_LEN +	\
+					 XGBE_SFP_EEPROM_DIAG_LEN)
+
 #define XGBE_BEL_FUSE_VENDOR	"BEL-FUSE        "
 #define XGBE_BEL_FUSE_PARTNO	"1GBT-SFP06      "
 
@@ -327,8 +337,6 @@ struct xgbe_phy_data {
 
 	unsigned int mdio_addr;
 
-	unsigned int comm_owned;
-
 	/* SFP Support */
 	enum xgbe_sfp_comm sfp_comm;
 	unsigned int sfp_mux_address;
@@ -345,7 +353,6 @@ struct xgbe_phy_data {
 	unsigned int sfp_rx_los;
 	unsigned int sfp_tx_fault;
 	unsigned int sfp_mod_absent;
-	unsigned int sfp_diags;
 	unsigned int sfp_changed;
 	unsigned int sfp_phy_avail;
 	unsigned int sfp_cable_len;
@@ -382,12 +389,6 @@ static enum xgbe_an_mode xgbe_phy_an_mode(struct xgbe_prv_data *pdata);
 static int xgbe_phy_i2c_xfer(struct xgbe_prv_data *pdata,
 			     struct xgbe_i2c_op *i2c_op)
 {
-	struct xgbe_phy_data *phy_data = pdata->phy_data;
-
-	/* Be sure we own the bus */
-	if (WARN_ON(!phy_data->comm_owned))
-		return -EIO;
-
 	return pdata->i2c_if.i2c_xfer(pdata, i2c_op);
 }
 
@@ -549,10 +550,6 @@ static int xgbe_phy_sfp_get_mux(struct xgbe_prv_data *pdata)
 
 static void xgbe_phy_put_comm_ownership(struct xgbe_prv_data *pdata)
 {
-	struct xgbe_phy_data *phy_data = pdata->phy_data;
-
-	phy_data->comm_owned = 0;
-
 	mutex_unlock(&xgbe_phy_comm_lock);
 }
 
@@ -562,9 +559,6 @@ static int xgbe_phy_get_comm_ownership(struct xgbe_prv_data *pdata)
 	unsigned long timeout;
 	unsigned int mutex_id;
 
-	if (phy_data->comm_owned)
-		return 0;
-
 	/* The I2C and MDIO/GPIO bus is multiplexed between multiple devices,
 	 * the driver needs to take the software mutex and then the hardware
 	 * mutexes before being able to use the busses.
@@ -593,7 +587,6 @@ static int xgbe_phy_get_comm_ownership(struct xgbe_prv_data *pdata)
 		XP_IOWRITE(pdata, XP_I2C_MUTEX, mutex_id);
 		XP_IOWRITE(pdata, XP_MDIO_MUTEX, mutex_id);
 
-		phy_data->comm_owned = 1;
 		return 0;
 	}
 
@@ -867,6 +860,9 @@ static bool xgbe_phy_finisar_phy_quirks(struct xgbe_prv_data *pdata)
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
 	unsigned int phy_id = phy_data->phydev->phy_id;
 
+	if (phy_data->port_mode != XGBE_PORT_MODE_SFP)
+		return false;
+
 	if ((phy_id & 0xfffffff0) != 0x01ff0cc0)
 		return false;
 
@@ -892,8 +888,83 @@ static bool xgbe_phy_finisar_phy_quirks(struct xgbe_prv_data *pdata)
 	return true;
 }
 
+static bool xgbe_phy_belfuse_phy_quirks(struct xgbe_prv_data *pdata)
+{
+	struct xgbe_phy_data *phy_data = pdata->phy_data;
+	struct xgbe_sfp_eeprom *sfp_eeprom = &phy_data->sfp_eeprom;
+	unsigned int phy_id = phy_data->phydev->phy_id;
+	int reg;
+
+	if (phy_data->port_mode != XGBE_PORT_MODE_SFP)
+		return false;
+
+	if (memcmp(&sfp_eeprom->base[XGBE_SFP_BASE_VENDOR_NAME],
+		   XGBE_BEL_FUSE_VENDOR, XGBE_SFP_BASE_VENDOR_NAME_LEN))
+		return false;
+
+	/* For Bel-Fuse, use the extra AN flag */
+	pdata->an_again = 1;
+
+	if (memcmp(&sfp_eeprom->base[XGBE_SFP_BASE_VENDOR_PN],
+		   XGBE_BEL_FUSE_PARTNO, XGBE_SFP_BASE_VENDOR_PN_LEN))
+		return false;
+
+	if ((phy_id & 0xfffffff0) != 0x03625d10)
+		return false;
+
+	/* Disable RGMII mode */
+	phy_write(phy_data->phydev, 0x18, 0x7007);
+	reg = phy_read(phy_data->phydev, 0x18);
+	phy_write(phy_data->phydev, 0x18, reg & ~0x0080);
+
+	/* Enable fiber register bank */
+	phy_write(phy_data->phydev, 0x1c, 0x7c00);
+	reg = phy_read(phy_data->phydev, 0x1c);
+	reg &= 0x03ff;
+	reg &= ~0x0001;
+	phy_write(phy_data->phydev, 0x1c, 0x8000 | 0x7c00 | reg | 0x0001);
+
+	/* Power down SerDes */
+	reg = phy_read(phy_data->phydev, 0x00);
+	phy_write(phy_data->phydev, 0x00, reg | 0x00800);
+
+	/* Configure SGMII-to-Copper mode */
+	phy_write(phy_data->phydev, 0x1c, 0x7c00);
+	reg = phy_read(phy_data->phydev, 0x1c);
+	reg &= 0x03ff;
+	reg &= ~0x0006;
+	phy_write(phy_data->phydev, 0x1c, 0x8000 | 0x7c00 | reg | 0x0004);
+
+	/* Power up SerDes */
+	reg = phy_read(phy_data->phydev, 0x00);
+	phy_write(phy_data->phydev, 0x00, reg & ~0x00800);
+
+	/* Enable copper register bank */
+	phy_write(phy_data->phydev, 0x1c, 0x7c00);
+	reg = phy_read(phy_data->phydev, 0x1c);
+	reg &= 0x03ff;
+	reg &= ~0x0001;
+	phy_write(phy_data->phydev, 0x1c, 0x8000 | 0x7c00 | reg);
+
+	/* Power up SerDes */
+	reg = phy_read(phy_data->phydev, 0x00);
+	phy_write(phy_data->phydev, 0x00, reg & ~0x00800);
+
+	phy_data->phydev->supported = PHY_GBIT_FEATURES;
+	phy_data->phydev->supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
+	phy_data->phydev->advertising = phy_data->phydev->supported;
+
+	netif_dbg(pdata, drv, pdata->netdev,
+		  "BelFuse PHY quirk in place\n");
+
+	return true;
+}
+
 static void xgbe_phy_external_phy_quirks(struct xgbe_prv_data *pdata)
 {
+	if (xgbe_phy_belfuse_phy_quirks(pdata))
+		return;
+
 	if (xgbe_phy_finisar_phy_quirks(pdata))
 		return;
 }
@@ -910,6 +981,9 @@ static int xgbe_phy_find_phy_device(struct xgbe_prv_data *pdata)
 	if (phy_data->phydev)
 		return 0;
 
+	/* Clear the extra AN flag */
+	pdata->an_again = 0;
+
 	/* Check for the use of an external PHY */
 	if (phy_data->phydev_mode == XGBE_MDIO_MODE_NONE)
 		return 0;
@@ -1034,37 +1108,6 @@ static bool xgbe_phy_check_sfp_mod_absent(struct xgbe_phy_data *phy_data)
 	return false;
 }
 
-static bool xgbe_phy_belfuse_parse_quirks(struct xgbe_prv_data *pdata)
-{
-	struct xgbe_phy_data *phy_data = pdata->phy_data;
-	struct xgbe_sfp_eeprom *sfp_eeprom = &phy_data->sfp_eeprom;
-
-	if (memcmp(&sfp_eeprom->base[XGBE_SFP_BASE_VENDOR_NAME],
-		   XGBE_BEL_FUSE_VENDOR, XGBE_SFP_BASE_VENDOR_NAME_LEN))
-		return false;
-
-	if (!memcmp(&sfp_eeprom->base[XGBE_SFP_BASE_VENDOR_PN],
-		    XGBE_BEL_FUSE_PARTNO, XGBE_SFP_BASE_VENDOR_PN_LEN)) {
-		phy_data->sfp_base = XGBE_SFP_BASE_1000_SX;
-		phy_data->sfp_cable = XGBE_SFP_CABLE_ACTIVE;
-		phy_data->sfp_speed = XGBE_SFP_SPEED_1000;
-		if (phy_data->sfp_changed)
-			netif_dbg(pdata, drv, pdata->netdev,
-				  "Bel-Fuse SFP quirk in place\n");
-		return true;
-	}
-
-	return false;
-}
-
-static bool xgbe_phy_sfp_parse_quirks(struct xgbe_prv_data *pdata)
-{
-	if (xgbe_phy_belfuse_parse_quirks(pdata))
-		return true;
-
-	return false;
-}
-
 static void xgbe_phy_sfp_parse_eeprom(struct xgbe_prv_data *pdata)
 {
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
@@ -1083,9 +1126,6 @@ static void xgbe_phy_sfp_parse_eeprom(struct xgbe_prv_data *pdata)
 	phy_data->sfp_tx_fault = xgbe_phy_check_sfp_tx_fault(phy_data);
 	phy_data->sfp_rx_los = xgbe_phy_check_sfp_rx_los(phy_data);
 
-	if (xgbe_phy_sfp_parse_quirks(pdata))
-		return;
-
 	/* Assume ACTIVE cable unless told it is PASSIVE */
 	if (sfp_base[XGBE_SFP_BASE_CABLE] & XGBE_SFP_BASE_CABLE_PASSIVE) {
 		phy_data->sfp_cable = XGBE_SFP_CABLE_PASSIVE;
@@ -1227,13 +1267,6 @@ static int xgbe_phy_sfp_read_eeprom(struct xgbe_prv_data *pdata)
 
 		memcpy(&phy_data->sfp_eeprom, &sfp_eeprom, sizeof(sfp_eeprom));
 
-		if (sfp_eeprom.extd[XGBE_SFP_EXTD_SFF_8472]) {
-			u8 diag_type = sfp_eeprom.extd[XGBE_SFP_EXTD_DIAG];
-
-			if (!(diag_type & XGBE_SFP_EXTD_DIAG_ADDR_CHANGE))
-				phy_data->sfp_diags = 1;
-		}
-
 		xgbe_phy_free_phy_device(pdata);
 	} else {
 		phy_data->sfp_changed = 0;
@@ -1283,7 +1316,6 @@ static void xgbe_phy_sfp_reset(struct xgbe_phy_data *phy_data)
 	phy_data->sfp_rx_los = 0;
 	phy_data->sfp_tx_fault = 0;
 	phy_data->sfp_mod_absent = 1;
-	phy_data->sfp_diags = 0;
 	phy_data->sfp_base = XGBE_SFP_BASE_UNKNOWN;
 	phy_data->sfp_cable = XGBE_SFP_CABLE_UNKNOWN;
 	phy_data->sfp_speed = XGBE_SFP_SPEED_UNKNOWN;
@@ -1326,6 +1358,130 @@ static void xgbe_phy_sfp_detect(struct xgbe_prv_data *pdata)
 	xgbe_phy_put_comm_ownership(pdata);
 }
 
+static int xgbe_phy_module_eeprom(struct xgbe_prv_data *pdata,
+				  struct ethtool_eeprom *eeprom, u8 *data)
+{
+	struct xgbe_phy_data *phy_data = pdata->phy_data;
+	u8 eeprom_addr, eeprom_data[XGBE_SFP_EEPROM_MAX];
+	struct xgbe_sfp_eeprom *sfp_eeprom;
+	unsigned int i, j, rem;
+	int ret;
+
+	rem = eeprom->len;
+
+	if (!eeprom->len) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	if ((eeprom->offset + eeprom->len) > XGBE_SFP_EEPROM_MAX) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	if (phy_data->port_mode != XGBE_PORT_MODE_SFP) {
+		ret = -ENXIO;
+		goto done;
+	}
+
+	if (!netif_running(pdata->netdev)) {
+		ret = -EIO;
+		goto done;
+	}
+
+	if (phy_data->sfp_mod_absent) {
+		ret = -EIO;
+		goto done;
+	}
+
+	ret = xgbe_phy_get_comm_ownership(pdata);
+	if (ret) {
+		ret = -EIO;
+		goto done;
+	}
+
+	ret = xgbe_phy_sfp_get_mux(pdata);
+	if (ret) {
+		netdev_err(pdata->netdev, "I2C error setting SFP MUX\n");
+		ret = -EIO;
+		goto put_own;
+	}
+
+	/* Read the SFP serial ID eeprom */
+	eeprom_addr = 0;
+	ret = xgbe_phy_i2c_read(pdata, XGBE_SFP_SERIAL_ID_ADDRESS,
+				&eeprom_addr, sizeof(eeprom_addr),
+				eeprom_data, XGBE_SFP_EEPROM_BASE_LEN);
+	if (ret) {
+		netdev_err(pdata->netdev,
+			   "I2C error reading SFP EEPROM\n");
+		ret = -EIO;
+		goto put_mux;
+	}
+
+	sfp_eeprom = (struct xgbe_sfp_eeprom *)eeprom_data;
+
+	if (XGBE_SFP_DIAGS_SUPPORTED(sfp_eeprom)) {
+		/* Read the SFP diagnostic eeprom */
+		eeprom_addr = 0;
+		ret = xgbe_phy_i2c_read(pdata, XGBE_SFP_DIAG_INFO_ADDRESS,
+					&eeprom_addr, sizeof(eeprom_addr),
+					eeprom_data + XGBE_SFP_EEPROM_BASE_LEN,
+					XGBE_SFP_EEPROM_DIAG_LEN);
+		if (ret) {
+			netdev_err(pdata->netdev,
+				   "I2C error reading SFP DIAGS\n");
+			ret = -EIO;
+			goto put_mux;
+		}
+	}
+
+	for (i = 0, j = eeprom->offset; i < eeprom->len; i++, j++) {
+		if ((j >= XGBE_SFP_EEPROM_BASE_LEN) &&
+		    !XGBE_SFP_DIAGS_SUPPORTED(sfp_eeprom))
+			break;
+
+		data[i] = eeprom_data[j];
+		rem--;
+	}
+
+put_mux:
+	xgbe_phy_sfp_put_mux(pdata);
+
+put_own:
+	xgbe_phy_put_comm_ownership(pdata);
+
+done:
+	eeprom->len -= rem;
+
+	return ret;
+}
+
+static int xgbe_phy_module_info(struct xgbe_prv_data *pdata,
+				struct ethtool_modinfo *modinfo)
+{
+	struct xgbe_phy_data *phy_data = pdata->phy_data;
+
+	if (phy_data->port_mode != XGBE_PORT_MODE_SFP)
+		return -ENXIO;
+
+	if (!netif_running(pdata->netdev))
+		return -EIO;
+
+	if (phy_data->sfp_mod_absent)
+		return -EIO;
+
+	if (XGBE_SFP_DIAGS_SUPPORTED(&phy_data->sfp_eeprom)) {
+		modinfo->type = ETH_MODULE_SFF_8472;
+		modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN;
+	} else {
+		modinfo->type = ETH_MODULE_SFF_8079;
+		modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN;
+	}
+
+	return 0;
+}
+
 static void xgbe_phy_phydev_flowctrl(struct xgbe_prv_data *pdata)
 {
 	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
@@ -1611,6 +1767,10 @@ static void xgbe_phy_an_advertising(struct xgbe_prv_data *pdata,
 	XGBE_CLR_ADV(dlks, 1000baseKX_Full);
 	XGBE_CLR_ADV(dlks, 10000baseKR_Full);
 
+	/* Advertise FEC support is present */
+	if (pdata->fec_ability & MDIO_PMA_10GBR_FECABLE_ABLE)
+		XGBE_SET_ADV(dlks, 10000baseR_FEC);
+
 	switch (phy_data->port_mode) {
 	case XGBE_PORT_MODE_BACKPLANE:
 		XGBE_SET_ADV(dlks, 10000baseKR_Full);
@@ -2421,22 +2581,21 @@ static int xgbe_phy_link_status(struct xgbe_prv_data *pdata, int *an_restart)
 static void xgbe_phy_sfp_gpio_setup(struct xgbe_prv_data *pdata)
 {
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
-	unsigned int reg;
-
-	reg = XP_IOREAD(pdata, XP_PROP_3);
 
 	phy_data->sfp_gpio_address = XGBE_GPIO_ADDRESS_PCA9555 +
-				     XP_GET_BITS(reg, XP_PROP_3, GPIO_ADDR);
+				     XP_GET_BITS(pdata->pp3, XP_PROP_3,
+						 GPIO_ADDR);
 
-	phy_data->sfp_gpio_mask = XP_GET_BITS(reg, XP_PROP_3, GPIO_MASK);
+	phy_data->sfp_gpio_mask = XP_GET_BITS(pdata->pp3, XP_PROP_3,
+					      GPIO_MASK);
 
-	phy_data->sfp_gpio_rx_los = XP_GET_BITS(reg, XP_PROP_3,
+	phy_data->sfp_gpio_rx_los = XP_GET_BITS(pdata->pp3, XP_PROP_3,
 						GPIO_RX_LOS);
-	phy_data->sfp_gpio_tx_fault = XP_GET_BITS(reg, XP_PROP_3,
+	phy_data->sfp_gpio_tx_fault = XP_GET_BITS(pdata->pp3, XP_PROP_3,
 						  GPIO_TX_FAULT);
-	phy_data->sfp_gpio_mod_absent = XP_GET_BITS(reg, XP_PROP_3,
+	phy_data->sfp_gpio_mod_absent = XP_GET_BITS(pdata->pp3, XP_PROP_3,
 						    GPIO_MOD_ABS);
-	phy_data->sfp_gpio_rate_select = XP_GET_BITS(reg, XP_PROP_3,
+	phy_data->sfp_gpio_rate_select = XP_GET_BITS(pdata->pp3, XP_PROP_3,
 						     GPIO_RATE_SELECT);
 
 	if (netif_msg_probe(pdata)) {
@@ -2458,18 +2617,17 @@ static void xgbe_phy_sfp_gpio_setup(struct xgbe_prv_data *pdata)
 static void xgbe_phy_sfp_comm_setup(struct xgbe_prv_data *pdata)
 {
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
-	unsigned int reg, mux_addr_hi, mux_addr_lo;
+	unsigned int mux_addr_hi, mux_addr_lo;
 
-	reg = XP_IOREAD(pdata, XP_PROP_4);
-
-	mux_addr_hi = XP_GET_BITS(reg, XP_PROP_4, MUX_ADDR_HI);
-	mux_addr_lo = XP_GET_BITS(reg, XP_PROP_4, MUX_ADDR_LO);
+	mux_addr_hi = XP_GET_BITS(pdata->pp4, XP_PROP_4, MUX_ADDR_HI);
+	mux_addr_lo = XP_GET_BITS(pdata->pp4, XP_PROP_4, MUX_ADDR_LO);
 	if (mux_addr_lo == XGBE_SFP_DIRECT)
 		return;
 
 	phy_data->sfp_comm = XGBE_SFP_COMM_PCA9545;
 	phy_data->sfp_mux_address = (mux_addr_hi << 2) + mux_addr_lo;
-	phy_data->sfp_mux_channel = XP_GET_BITS(reg, XP_PROP_4, MUX_CHAN);
+	phy_data->sfp_mux_channel = XP_GET_BITS(pdata->pp4, XP_PROP_4,
+						MUX_CHAN);
 
 	if (netif_msg_probe(pdata)) {
 		dev_dbg(pdata->dev, "SFP: mux_address=%#x\n",
@@ -2592,13 +2750,11 @@ static bool xgbe_phy_redrv_error(struct xgbe_phy_data *phy_data)
 static int xgbe_phy_mdio_reset_setup(struct xgbe_prv_data *pdata)
 {
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
-	unsigned int reg;
 
 	if (phy_data->conn_type != XGBE_CONN_TYPE_MDIO)
 		return 0;
 
-	reg = XP_IOREAD(pdata, XP_PROP_3);
-	phy_data->mdio_reset = XP_GET_BITS(reg, XP_PROP_3, MDIO_RESET);
+	phy_data->mdio_reset = XP_GET_BITS(pdata->pp3, XP_PROP_3, MDIO_RESET);
 	switch (phy_data->mdio_reset) {
 	case XGBE_MDIO_RESET_NONE:
 	case XGBE_MDIO_RESET_I2C_GPIO:
@@ -2612,12 +2768,12 @@ static int xgbe_phy_mdio_reset_setup(struct xgbe_prv_data *pdata)
 
 	if (phy_data->mdio_reset == XGBE_MDIO_RESET_I2C_GPIO) {
 		phy_data->mdio_reset_addr = XGBE_GPIO_ADDRESS_PCA9555 +
-					    XP_GET_BITS(reg, XP_PROP_3,
+					    XP_GET_BITS(pdata->pp3, XP_PROP_3,
 							MDIO_RESET_I2C_ADDR);
-		phy_data->mdio_reset_gpio = XP_GET_BITS(reg, XP_PROP_3,
+		phy_data->mdio_reset_gpio = XP_GET_BITS(pdata->pp3, XP_PROP_3,
 							MDIO_RESET_I2C_GPIO);
 	} else if (phy_data->mdio_reset == XGBE_MDIO_RESET_INT_GPIO) {
-		phy_data->mdio_reset_gpio = XP_GET_BITS(reg, XP_PROP_3,
+		phy_data->mdio_reset_gpio = XP_GET_BITS(pdata->pp3, XP_PROP_3,
 							MDIO_RESET_INT_GPIO);
 	}
 
@@ -2707,12 +2863,9 @@ static bool xgbe_phy_conn_type_mismatch(struct xgbe_prv_data *pdata)
 
 static bool xgbe_phy_port_enabled(struct xgbe_prv_data *pdata)
 {
-	unsigned int reg;
-
-	reg = XP_IOREAD(pdata, XP_PROP_0);
-	if (!XP_GET_BITS(reg, XP_PROP_0, PORT_SPEEDS))
+	if (!XP_GET_BITS(pdata->pp0, XP_PROP_0, PORT_SPEEDS))
 		return false;
-	if (!XP_GET_BITS(reg, XP_PROP_0, CONN_TYPE))
+	if (!XP_GET_BITS(pdata->pp0, XP_PROP_0, CONN_TYPE))
 		return false;
 
 	return true;
@@ -2921,7 +3074,6 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	struct xgbe_phy_data *phy_data;
 	struct mii_bus *mii;
-	unsigned int reg;
 	int ret;
 
 	/* Check if enabled */
@@ -2940,12 +3092,11 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 		return -ENOMEM;
 	pdata->phy_data = phy_data;
 
-	reg = XP_IOREAD(pdata, XP_PROP_0);
-	phy_data->port_mode = XP_GET_BITS(reg, XP_PROP_0, PORT_MODE);
-	phy_data->port_id = XP_GET_BITS(reg, XP_PROP_0, PORT_ID);
-	phy_data->port_speeds = XP_GET_BITS(reg, XP_PROP_0, PORT_SPEEDS);
-	phy_data->conn_type = XP_GET_BITS(reg, XP_PROP_0, CONN_TYPE);
-	phy_data->mdio_addr = XP_GET_BITS(reg, XP_PROP_0, MDIO_ADDR);
+	phy_data->port_mode = XP_GET_BITS(pdata->pp0, XP_PROP_0, PORT_MODE);
+	phy_data->port_id = XP_GET_BITS(pdata->pp0, XP_PROP_0, PORT_ID);
+	phy_data->port_speeds = XP_GET_BITS(pdata->pp0, XP_PROP_0, PORT_SPEEDS);
+	phy_data->conn_type = XP_GET_BITS(pdata->pp0, XP_PROP_0, CONN_TYPE);
+	phy_data->mdio_addr = XP_GET_BITS(pdata->pp0, XP_PROP_0, MDIO_ADDR);
 	if (netif_msg_probe(pdata)) {
 		dev_dbg(pdata->dev, "port mode=%u\n", phy_data->port_mode);
 		dev_dbg(pdata->dev, "port id=%u\n", phy_data->port_id);
@@ -2954,12 +3105,11 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 		dev_dbg(pdata->dev, "mdio addr=%u\n", phy_data->mdio_addr);
 	}
 
-	reg = XP_IOREAD(pdata, XP_PROP_4);
-	phy_data->redrv = XP_GET_BITS(reg, XP_PROP_4, REDRV_PRESENT);
-	phy_data->redrv_if = XP_GET_BITS(reg, XP_PROP_4, REDRV_IF);
-	phy_data->redrv_addr = XP_GET_BITS(reg, XP_PROP_4, REDRV_ADDR);
-	phy_data->redrv_lane = XP_GET_BITS(reg, XP_PROP_4, REDRV_LANE);
-	phy_data->redrv_model = XP_GET_BITS(reg, XP_PROP_4, REDRV_MODEL);
+	phy_data->redrv = XP_GET_BITS(pdata->pp4, XP_PROP_4, REDRV_PRESENT);
+	phy_data->redrv_if = XP_GET_BITS(pdata->pp4, XP_PROP_4, REDRV_IF);
+	phy_data->redrv_addr = XP_GET_BITS(pdata->pp4, XP_PROP_4, REDRV_ADDR);
+	phy_data->redrv_lane = XP_GET_BITS(pdata->pp4, XP_PROP_4, REDRV_LANE);
+	phy_data->redrv_model = XP_GET_BITS(pdata->pp4, XP_PROP_4, REDRV_MODEL);
 	if (phy_data->redrv && netif_msg_probe(pdata)) {
 		dev_dbg(pdata->dev, "redrv present\n");
 		dev_dbg(pdata->dev, "redrv i/f=%u\n", phy_data->redrv_if);
@@ -3231,4 +3381,7 @@ void xgbe_init_function_ptrs_phy_v2(struct xgbe_phy_if *phy_if)
 
 	phy_impl->kr_training_pre	= xgbe_phy_kr_training_pre;
 	phy_impl->kr_training_post	= xgbe_phy_kr_training_post;
+
+	phy_impl->module_info		= xgbe_phy_module_info;
+	phy_impl->module_eeprom		= xgbe_phy_module_eeprom;
 }
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h
index 95d4b56448c6..47bcbcf58048 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe.h
+++ b/drivers/net/ethernet/amd/xgbe/xgbe.h
@@ -144,6 +144,11 @@
 #define XGBE_TX_DESC_MAX_PROC	(XGBE_TX_DESC_CNT >> 1)
 #define XGBE_RX_DESC_CNT	512
 
+#define XGBE_TX_DESC_CNT_MIN	64
+#define XGBE_TX_DESC_CNT_MAX	4096
+#define XGBE_RX_DESC_CNT_MIN	64
+#define XGBE_RX_DESC_CNT_MAX	4096
+
 #define XGBE_TX_MAX_BUF_SIZE	(0x3fff & ~(64 - 1))
 
 /* Descriptors required for maximum contiguous TSO/GSO packet */
@@ -835,6 +840,7 @@ struct xgbe_hw_if {
  *   Optional routines:
  *     an_pre, an_post
  *     kr_training_pre, kr_training_post
+ *     module_info, module_eeprom
  */
 struct xgbe_phy_impl_if {
 	/* Perform Setup/teardown actions */
@@ -883,6 +889,12 @@ struct xgbe_phy_impl_if {
 	/* Pre/Post KR training enablement support */
 	void (*kr_training_pre)(struct xgbe_prv_data *);
 	void (*kr_training_post)(struct xgbe_prv_data *);
+
+	/* SFP module related info */
+	int (*module_info)(struct xgbe_prv_data *pdata,
+			   struct ethtool_modinfo *modinfo);
+	int (*module_eeprom)(struct xgbe_prv_data *pdata,
+			     struct ethtool_eeprom *eeprom, u8 *data);
 };
 
 struct xgbe_phy_if {
@@ -905,6 +917,12 @@ struct xgbe_phy_if {
 	/* For single interrupt support */
 	irqreturn_t (*an_isr)(struct xgbe_prv_data *);
 
+	/* For ethtool PHY support */
+	int (*module_info)(struct xgbe_prv_data *pdata,
+			   struct ethtool_modinfo *modinfo);
+	int (*module_eeprom)(struct xgbe_prv_data *pdata,
+			     struct ethtool_eeprom *eeprom, u8 *data);
+
 	/* PHY implementation specific services */
 	struct xgbe_phy_impl_if phy_impl;
 };
@@ -1027,6 +1045,13 @@ struct xgbe_prv_data {
 	void __iomem *xprop_regs;	/* XGBE property registers */
 	void __iomem *xi2c_regs;	/* XGBE I2C CSRs */
 
+	/* Port property registers */
+	unsigned int pp0;
+	unsigned int pp1;
+	unsigned int pp2;
+	unsigned int pp3;
+	unsigned int pp4;
+
 	/* Overall device lock */
 	spinlock_t lock;
 
@@ -1097,6 +1122,9 @@ struct xgbe_prv_data {
 	unsigned int rx_ring_count;
 	unsigned int rx_desc_count;
 
+	unsigned int new_tx_ring_count;
+	unsigned int new_rx_ring_count;
+
 	unsigned int tx_max_q_count;
 	unsigned int rx_max_q_count;
 	unsigned int tx_q_count;
@@ -1233,6 +1261,7 @@ struct xgbe_prv_data {
 	enum xgbe_rx kr_state;
 	enum xgbe_rx kx_state;
 	struct work_struct an_work;
+	unsigned int an_again;
 	unsigned int an_supported;
 	unsigned int parallel_detect;
 	unsigned int fec_ability;
@@ -1310,6 +1339,8 @@ int xgbe_powerup(struct net_device *, unsigned int);
 int xgbe_powerdown(struct net_device *, unsigned int);
 void xgbe_init_rx_coalesce(struct xgbe_prv_data *);
 void xgbe_init_tx_coalesce(struct xgbe_prv_data *);
+void xgbe_restart_dev(struct xgbe_prv_data *pdata);
+void xgbe_full_restart_dev(struct xgbe_prv_data *pdata);
 
 #ifdef CONFIG_DEBUG_FS
 void xgbe_debugfs_init(struct xgbe_prv_data *);
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c
index f33b25fbca63..d5fca2e5a9bc 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.c
+++ b/drivers/net/ethernet/broadcom/bcmsysport.c
@@ -654,7 +654,7 @@ static int bcm_sysport_set_coalesce(struct net_device *dev,
 	pkts = priv->rx_max_coalesced_frames;
 
 	if (ec->use_adaptive_rx_coalesce && !priv->dim.use_dim) {
-		moder = net_dim_get_def_profile(priv->dim.dim.mode);
+		moder = net_dim_get_def_rx_moderation(priv->dim.dim.mode);
 		usecs = moder.usec;
 		pkts = moder.pkts;
 	}
@@ -1064,7 +1064,7 @@ static void bcm_sysport_dim_work(struct work_struct *work)
 	struct bcm_sysport_priv *priv =
 			container_of(ndim, struct bcm_sysport_priv, dim);
 	struct net_dim_cq_moder cur_profile =
-				net_dim_get_profile(dim->mode, dim->profile_ix);
+			net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
 
 	bcm_sysport_set_rx_coalesce(priv, cur_profile.usec, cur_profile.pkts);
 	dim->state = NET_DIM_START_MEASURE;
@@ -1437,7 +1437,7 @@ static void bcm_sysport_init_rx_coalesce(struct bcm_sysport_priv *priv)
 
 	/* If DIM was enabled, re-apply default parameters */
 	if (dim->use_dim) {
-		moder = net_dim_get_def_profile(dim->dim.mode);
+		moder = net_dim_get_def_rx_moderation(dim->dim.mode);
 		usecs = moder.usec;
 		pkts = moder.pkts;
 	}
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
index 95871576ab92..8cd73ff5debc 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
@@ -4962,8 +4962,13 @@ void bnx2x_tx_timeout(struct net_device *dev)
 {
 	struct bnx2x *bp = netdev_priv(dev);
 
-#ifdef BNX2X_STOP_ON_ERROR
+	/* We want the information of the dump logged,
+	 * but calling bnx2x_panic() would kill all chances of recovery.
+	 */
 	if (!bp->panic)
+#ifndef BNX2X_STOP_ON_ERROR
+		bnx2x_panic_dump(bp, false);
+#else
 		bnx2x_panic();
 #endif
 
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
index c766ae23bc74..5b1ed240bf18 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
@@ -13922,8 +13922,6 @@ static int bnx2x_init_one(struct pci_dev *pdev,
 {
 	struct net_device *dev = NULL;
 	struct bnx2x *bp;
-	enum pcie_link_width pcie_width;
-	enum pci_bus_speed pcie_speed;
 	int rc, max_non_def_sbs;
 	int rx_count, tx_count, rss_count, doorbell_size;
 	int max_cos_est;
@@ -14091,21 +14089,12 @@ static int bnx2x_init_one(struct pci_dev *pdev,
 		dev_addr_add(bp->dev, bp->fip_mac, NETDEV_HW_ADDR_T_SAN);
 		rtnl_unlock();
 	}
-	if (pcie_get_minimum_link(bp->pdev, &pcie_speed, &pcie_width) ||
-	    pcie_speed == PCI_SPEED_UNKNOWN ||
-	    pcie_width == PCIE_LNK_WIDTH_UNKNOWN)
-		BNX2X_DEV_INFO("Failed to determine PCI Express Bandwidth\n");
-	else
-		BNX2X_DEV_INFO(
-		       "%s (%c%d) PCI-E x%d %s found at mem %lx, IRQ %d, node addr %pM\n",
-		       board_info[ent->driver_data].name,
-		       (CHIP_REV(bp) >> 12) + 'A', (CHIP_METAL(bp) >> 4),
-		       pcie_width,
-		       pcie_speed == PCIE_SPEED_2_5GT ? "2.5GHz" :
-		       pcie_speed == PCIE_SPEED_5_0GT ? "5.0GHz" :
-		       pcie_speed == PCIE_SPEED_8_0GT ? "8.0GHz" :
-		       "Unknown",
-		       dev->base_addr, bp->pdev->irq, dev->dev_addr);
+	BNX2X_DEV_INFO(
+	       "%s (%c%d) PCI-E found at mem %lx, IRQ %d, node addr %pM\n",
+	       board_info[ent->driver_data].name,
+	       (CHIP_REV(bp) >> 12) + 'A', (CHIP_METAL(bp) >> 4),
+	       dev->base_addr, bp->pdev->irq, dev->dev_addr);
+	pcie_print_link_status(bp->pdev);
 
 	bnx2x_register_phc(bp);
 
diff --git a/drivers/net/ethernet/broadcom/bnxt/Makefile b/drivers/net/ethernet/broadcom/bnxt/Makefile
index 7c560d545c03..5a779b19d149 100644
--- a/drivers/net/ethernet/broadcom/bnxt/Makefile
+++ b/drivers/net/ethernet/broadcom/bnxt/Makefile
@@ -2,3 +2,4 @@ obj-$(CONFIG_BNXT) += bnxt_en.o
 
 bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o bnxt_dim.o
 bnxt_en-$(CONFIG_BNXT_FLOWER_OFFLOAD) += bnxt_tc.o
+bnxt_en-$(CONFIG_DEBUG_FS) += bnxt_debugfs.o
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index f83769d8047b..176fc9f4d7de 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -62,6 +62,7 @@
 #include "bnxt_vfr.h"
 #include "bnxt_tc.h"
 #include "bnxt_devlink.h"
+#include "bnxt_debugfs.h"
 
 #define BNXT_TX_TIMEOUT		(5 * HZ)
 
@@ -2383,6 +2384,7 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp)
 	for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
 		struct bnxt_ring_struct *ring;
+		u8 qidx;
 
 		ring = &txr->tx_ring_struct;
 
@@ -2411,7 +2413,8 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp)
 
 			memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
 		}
-		ring->queue_id = bp->q_info[j].queue_id;
+		qidx = bp->tc_to_qidx[j];
+		ring->queue_id = bp->q_info[qidx].queue_id;
 		if (i < bp->tx_nr_rings_xdp)
 			continue;
 		if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
@@ -3493,15 +3496,29 @@ static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
 
 	if (!timeout)
 		timeout = DFLT_HWRM_CMD_TIMEOUT;
+	/* convert timeout to usec */
+	timeout *= 1000;
 
 	i = 0;
-	tmo_count = timeout * 40;
+	/* Short timeout for the first few iterations:
+	 * number of loops = number of loops for short timeout +
+	 * number of loops for standard timeout.
+	 */
+	tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
+	timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
+	tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
 	resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET;
 	if (intr_process) {
 		/* Wait until hwrm response cmpl interrupt is processed */
 		while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID &&
 		       i++ < tmo_count) {
-			usleep_range(25, 40);
+			/* on first few passes, just barely sleep */
+			if (i < HWRM_SHORT_TIMEOUT_COUNTER)
+				usleep_range(HWRM_SHORT_MIN_TIMEOUT,
+					     HWRM_SHORT_MAX_TIMEOUT);
+			else
+				usleep_range(HWRM_MIN_TIMEOUT,
+					     HWRM_MAX_TIMEOUT);
 		}
 
 		if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) {
@@ -3513,25 +3530,34 @@ static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
 		      HWRM_RESP_LEN_SFT;
 		valid = bp->hwrm_cmd_resp_addr + len - 1;
 	} else {
+		int j;
+
 		/* Check if response len is updated */
 		for (i = 0; i < tmo_count; i++) {
 			len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
 			      HWRM_RESP_LEN_SFT;
 			if (len)
 				break;
-			usleep_range(25, 40);
+			/* on first few passes, just barely sleep */
+			if (i < DFLT_HWRM_CMD_TIMEOUT)
+				usleep_range(HWRM_SHORT_MIN_TIMEOUT,
+					     HWRM_SHORT_MAX_TIMEOUT);
+			else
+				usleep_range(HWRM_MIN_TIMEOUT,
+					     HWRM_MAX_TIMEOUT);
 		}
 
 		if (i >= tmo_count) {
 			netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
-				   timeout, le16_to_cpu(req->req_type),
+				   HWRM_TOTAL_TIMEOUT(i),
+				   le16_to_cpu(req->req_type),
 				   le16_to_cpu(req->seq_id), len);
 			return -1;
 		}
 
 		/* Last byte of resp contains valid bit */
 		valid = bp->hwrm_cmd_resp_addr + len - 1;
-		for (i = 0; i < 5; i++) {
+		for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
 			/* make sure we read from updated DMA memory */
 			dma_rmb();
 			if (*valid)
@@ -3539,9 +3565,10 @@ static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
 			udelay(1);
 		}
 
-		if (i >= 5) {
+		if (j >= HWRM_VALID_BIT_DELAY_USEC) {
 			netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
-				   timeout, le16_to_cpu(req->req_type),
+				   HWRM_TOTAL_TIMEOUT(i),
+				   le16_to_cpu(req->req_type),
 				   le16_to_cpu(req->seq_id), len, *valid);
 			return -1;
 		}
@@ -4334,26 +4361,9 @@ static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
 	mutex_unlock(&bp->hwrm_cmd_lock);
 
 	if (rc || err) {
-		switch (ring_type) {
-		case RING_FREE_REQ_RING_TYPE_L2_CMPL:
-			netdev_err(bp->dev, "hwrm_ring_alloc cp failed. rc:%x err:%x\n",
-				   rc, err);
-			return -1;
-
-		case RING_FREE_REQ_RING_TYPE_RX:
-			netdev_err(bp->dev, "hwrm_ring_alloc rx failed. rc:%x err:%x\n",
-				   rc, err);
-			return -1;
-
-		case RING_FREE_REQ_RING_TYPE_TX:
-			netdev_err(bp->dev, "hwrm_ring_alloc tx failed. rc:%x err:%x\n",
-				   rc, err);
-			return -1;
-
-		default:
-			netdev_err(bp->dev, "Invalid ring\n");
-			return -1;
-		}
+		netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
+			   ring_type, rc, err);
+		return -EIO;
 	}
 	ring->fw_ring_id = ring_id;
 	return rc;
@@ -4477,23 +4487,9 @@ static int hwrm_ring_free_send_msg(struct bnxt *bp,
 	mutex_unlock(&bp->hwrm_cmd_lock);
 
 	if (rc || error_code) {
-		switch (ring_type) {
-		case RING_FREE_REQ_RING_TYPE_L2_CMPL:
-			netdev_err(bp->dev, "hwrm_ring_free cp failed. rc:%d\n",
-				   rc);
-			return rc;
-		case RING_FREE_REQ_RING_TYPE_RX:
-			netdev_err(bp->dev, "hwrm_ring_free rx failed. rc:%d\n",
-				   rc);
-			return rc;
-		case RING_FREE_REQ_RING_TYPE_TX:
-			netdev_err(bp->dev, "hwrm_ring_free tx failed. rc:%d\n",
-				   rc);
-			return rc;
-		default:
-			netdev_err(bp->dev, "Invalid ring\n");
-			return -1;
-		}
+		netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
+			   ring_type, rc, error_code);
+		return -EIO;
 	}
 	return 0;
 }
@@ -4721,6 +4717,10 @@ bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
 
 	__bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
 				     cp_rings, vnics);
+	req.enables |= cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS |
+				   FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS);
+	req.num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
+	req.num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
 	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 	if (rc)
 		return -ENOMEM;
@@ -5309,6 +5309,7 @@ static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
 	for (i = 0; i < bp->max_tc; i++) {
 		bp->q_info[i].queue_id = *qptr++;
 		bp->q_info[i].queue_profile = *qptr++;
+		bp->tc_to_qidx[i] = i;
 	}
 
 qportcfg_exit:
@@ -5376,7 +5377,8 @@ int bnxt_hwrm_fw_set_time(struct bnxt *bp)
 	struct tm tm;
 	time64_t now = ktime_get_real_seconds();
 
-	if (bp->hwrm_spec_code < 0x10400)
+	if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
+	    bp->hwrm_spec_code < 0x10400)
 		return -EOPNOTSUPP;
 
 	time64_to_tm(now, 0, &tm);
@@ -5958,6 +5960,9 @@ static int bnxt_init_msix(struct bnxt *bp)
 	if (total_vecs > max)
 		total_vecs = max;
 
+	if (!total_vecs)
+		return 0;
+
 	msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
 	if (!msix_ent)
 		return -ENOMEM;
@@ -6457,6 +6462,9 @@ static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
 	}
 	mutex_unlock(&bp->hwrm_cmd_lock);
 
+	if (!BNXT_SINGLE_PF(bp))
+		return 0;
+
 	diff = link_info->support_auto_speeds ^ link_info->advertising;
 	if ((link_info->support_auto_speeds | diff) !=
 	    link_info->support_auto_speeds) {
@@ -6843,6 +6851,8 @@ static void bnxt_preset_reg_win(struct bnxt *bp)
 	}
 }
 
+static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
+
 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
 {
 	int rc = 0;
@@ -6850,6 +6860,12 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
 	bnxt_preset_reg_win(bp);
 	netif_carrier_off(bp->dev);
 	if (irq_re_init) {
+		/* Reserve rings now if none were reserved at driver probe. */
+		rc = bnxt_init_dflt_ring_mode(bp);
+		if (rc) {
+			netdev_err(bp->dev, "Failed to reserve default rings at open\n");
+			return rc;
+		}
 		rc = bnxt_reserve_rings(bp);
 		if (rc)
 			return rc;
@@ -6877,6 +6893,7 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
 	}
 
 	bnxt_enable_napi(bp);
+	bnxt_debug_dev_init(bp);
 
 	rc = bnxt_init_nic(bp, irq_re_init);
 	if (rc) {
@@ -6909,6 +6926,7 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
 	return 0;
 
 open_err:
+	bnxt_debug_dev_exit(bp);
 	bnxt_disable_napi(bp);
 	bnxt_del_napi(bp);
 
@@ -7002,6 +7020,7 @@ static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
 
 	/* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
 
+	bnxt_debug_dev_exit(bp);
 	bnxt_disable_napi(bp);
 	del_timer_sync(&bp->timer);
 	bnxt_free_skbs(bp);
@@ -7279,6 +7298,25 @@ static int bnxt_cfg_rx_mode(struct bnxt *bp)
 	return rc;
 }
 
+static bool bnxt_can_reserve_rings(struct bnxt *bp)
+{
+#ifdef CONFIG_BNXT_SRIOV
+	if ((bp->flags & BNXT_FLAG_NEW_RM) && BNXT_VF(bp)) {
+		struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
+
+		/* No minimum rings were provisioned by the PF.  Don't
+		 * reserve rings by default when device is down.
+		 */
+		if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
+			return true;
+
+		if (!netif_running(bp->dev))
+			return false;
+	}
+#endif
+	return true;
+}
+
 /* If the chip and firmware supports RFS */
 static bool bnxt_rfs_supported(struct bnxt *bp)
 {
@@ -7295,7 +7333,7 @@ static bool bnxt_rfs_capable(struct bnxt *bp)
 #ifdef CONFIG_RFS_ACCEL
 	int vnics, max_vnics, max_rss_ctxs;
 
-	if (!(bp->flags & BNXT_FLAG_MSIX_CAP))
+	if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
 		return false;
 
 	vnics = 1 + bp->rx_nr_rings;
@@ -7729,7 +7767,7 @@ static void bnxt_init_dflt_coal(struct bnxt *bp)
 	coal->coal_bufs = 30;
 	coal->coal_ticks_irq = 1;
 	coal->coal_bufs_irq = 2;
-	coal->idle_thresh = 25;
+	coal->idle_thresh = 50;
 	coal->bufs_per_record = 2;
 	coal->budget = 64;		/* NAPI budget */
 
@@ -8529,6 +8567,9 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
 {
 	int dflt_rings, max_rx_rings, max_tx_rings, rc;
 
+	if (!bnxt_can_reserve_rings(bp))
+		return 0;
+
 	if (sh)
 		bp->flags |= BNXT_FLAG_SHARED_RINGS;
 	dflt_rings = netif_get_num_default_rss_queues();
@@ -8574,6 +8615,29 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
 	return rc;
 }
 
+static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
+{
+	int rc;
+
+	if (bp->tx_nr_rings)
+		return 0;
+
+	rc = bnxt_set_dflt_rings(bp, true);
+	if (rc) {
+		netdev_err(bp->dev, "Not enough rings available.\n");
+		return rc;
+	}
+	rc = bnxt_init_int_mode(bp);
+	if (rc)
+		return rc;
+	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
+	if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
+		bp->flags |= BNXT_FLAG_RFS;
+		bp->dev->features |= NETIF_F_NTUPLE;
+	}
+	return 0;
+}
+
 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
 {
 	int rc;
@@ -8614,29 +8678,13 @@ static int bnxt_init_mac_addr(struct bnxt *bp)
 			memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
 		} else {
 			eth_hw_addr_random(bp->dev);
-			rc = bnxt_approve_mac(bp, bp->dev->dev_addr);
 		}
+		rc = bnxt_approve_mac(bp, bp->dev->dev_addr);
 #endif
 	}
 	return rc;
 }
 
-static void bnxt_parse_log_pcie_link(struct bnxt *bp)
-{
-	enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
-	enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
-
-	if (pcie_get_minimum_link(pci_physfn(bp->pdev), &speed, &width) ||
-	    speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
-		netdev_info(bp->dev, "Failed to determine PCIe Link Info\n");
-	else
-		netdev_info(bp->dev, "PCIe: Speed %s Width x%d\n",
-			    speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
-			    speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
-			    speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
-			    "Unknown", width);
-}
-
 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	static int version_printed;
@@ -8851,8 +8899,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
 		    board_info[ent->driver_data].name,
 		    (long)pci_resource_start(pdev, 0), dev->dev_addr);
-
-	bnxt_parse_log_pcie_link(bp);
+	pcie_print_link_status(pdev);
 
 	return 0;
 
@@ -9078,6 +9125,7 @@ static struct pci_driver bnxt_pci_driver = {
 
 static int __init bnxt_init(void)
 {
+	bnxt_debug_init();
 	return pci_register_driver(&bnxt_pci_driver);
 }
 
@@ -9086,6 +9134,7 @@ static void __exit bnxt_exit(void)
 	pci_unregister_driver(&bnxt_pci_driver);
 	if (bnxt_pf_wq)
 		destroy_workqueue(bnxt_pf_wq);
+	bnxt_debug_exit();
 }
 
 module_init(bnxt_init);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
index 3d55d3b56865..9b14eb610b9f 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
@@ -532,6 +532,19 @@ struct rx_tpa_end_cmp_ext {
 #define BNXT_HWRM_REQ_MAX_SIZE		128
 #define BNXT_HWRM_REQS_PER_PAGE		(BNXT_PAGE_SIZE /	\
 					 BNXT_HWRM_REQ_MAX_SIZE)
+#define HWRM_SHORT_MIN_TIMEOUT		3
+#define HWRM_SHORT_MAX_TIMEOUT		10
+#define HWRM_SHORT_TIMEOUT_COUNTER	5
+
+#define HWRM_MIN_TIMEOUT		25
+#define HWRM_MAX_TIMEOUT		40
+
+#define HWRM_TOTAL_TIMEOUT(n)	(((n) <= HWRM_SHORT_TIMEOUT_COUNTER) ?	\
+	((n) * HWRM_SHORT_MIN_TIMEOUT) :				\
+	(HWRM_SHORT_TIMEOUT_COUNTER * HWRM_SHORT_MIN_TIMEOUT +		\
+	 ((n) - HWRM_SHORT_TIMEOUT_COUNTER) * HWRM_MIN_TIMEOUT))
+
+#define HWRM_VALID_BIT_DELAY_USEC	20
 
 #define BNXT_RX_EVENT	1
 #define BNXT_AGG_EVENT	2
@@ -1242,6 +1255,7 @@ struct bnxt {
 	u8			max_tc;
 	u8			max_lltc;	/* lossless TCs */
 	struct bnxt_queue_info	q_info[BNXT_MAX_QUEUE];
+	u8			tc_to_qidx[BNXT_MAX_QUEUE];
 
 	unsigned int		current_interval;
 #define BNXT_TIMER_INTERVAL	HZ
@@ -1384,6 +1398,8 @@ struct bnxt {
 	u16			*cfa_code_map; /* cfa_code -> vf_idx map */
 	u8			switch_id[8];
 	struct bnxt_tc_info	*tc_info;
+	struct dentry		*debugfs_pdev;
+	struct dentry		*debugfs_dim;
 };
 
 #define BNXT_RX_STATS_OFFSET(counter)			\
@@ -1398,8 +1414,7 @@ struct bnxt {
 
 #define I2C_DEV_ADDR_A0				0xa0
 #define I2C_DEV_ADDR_A2				0xa2
-#define SFP_EEPROM_SFF_8472_COMP_ADDR		0x5e
-#define SFP_EEPROM_SFF_8472_COMP_SIZE		1
+#define SFF_DIAG_SUPPORT_OFFSET			0x5c
 #define SFF_MODULE_ID_SFP			0x3
 #define SFF_MODULE_ID_QSFP			0xc
 #define SFF_MODULE_ID_QSFP_PLUS			0xd
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c
index 3c746f2d9ed8..d5bc72cecde3 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c
@@ -21,6 +21,21 @@
 #include "bnxt_dcb.h"
 
 #ifdef CONFIG_BNXT_DCB
+static int bnxt_queue_to_tc(struct bnxt *bp, u8 queue_id)
+{
+	int i, j;
+
+	for (i = 0; i < bp->max_tc; i++) {
+		if (bp->q_info[i].queue_id == queue_id) {
+			for (j = 0; j < bp->max_tc; j++) {
+				if (bp->tc_to_qidx[j] == i)
+					return j;
+			}
+		}
+	}
+	return -EINVAL;
+}
+
 static int bnxt_hwrm_queue_pri2cos_cfg(struct bnxt *bp, struct ieee_ets *ets)
 {
 	struct hwrm_queue_pri2cos_cfg_input req = {0};
@@ -33,10 +48,13 @@ static int bnxt_hwrm_queue_pri2cos_cfg(struct bnxt *bp, struct ieee_ets *ets)
 
 	pri2cos = &req.pri0_cos_queue_id;
 	for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) {
+		u8 qidx;
+
 		req.enables |= cpu_to_le32(
 			QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI0_COS_QUEUE_ID << i);
 
-		pri2cos[i] = bp->q_info[ets->prio_tc[i]].queue_id;
+		qidx = bp->tc_to_qidx[ets->prio_tc[i]];
+		pri2cos[i] = bp->q_info[qidx].queue_id;
 	}
 	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 	return rc;
@@ -55,17 +73,15 @@ static int bnxt_hwrm_queue_pri2cos_qcfg(struct bnxt *bp, struct ieee_ets *ets)
 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 	if (!rc) {
 		u8 *pri2cos = &resp->pri0_cos_queue_id;
-		int i, j;
+		int i;
 
 		for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) {
 			u8 queue_id = pri2cos[i];
+			int tc;
 
-			for (j = 0; j < bp->max_tc; j++) {
-				if (bp->q_info[j].queue_id == queue_id) {
-					ets->prio_tc[i] = j;
-					break;
-				}
-			}
+			tc = bnxt_queue_to_tc(bp, queue_id);
+			if (tc >= 0)
+				ets->prio_tc[i] = tc;
 		}
 	}
 	mutex_unlock(&bp->hwrm_cmd_lock);
@@ -81,13 +97,15 @@ static int bnxt_hwrm_queue_cos2bw_cfg(struct bnxt *bp, struct ieee_ets *ets,
 	void *data;
 
 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_COS2BW_CFG, -1, -1);
-	data = &req.unused_0;
-	for (i = 0; i < max_tc; i++, data += sizeof(cos2bw) - 4) {
+	for (i = 0; i < max_tc; i++) {
+		u8 qidx;
+
 		req.enables |= cpu_to_le32(
 			QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID0_VALID << i);
 
 		memset(&cos2bw, 0, sizeof(cos2bw));
-		cos2bw.queue_id = bp->q_info[i].queue_id;
+		qidx = bp->tc_to_qidx[i];
+		cos2bw.queue_id = bp->q_info[qidx].queue_id;
 		if (ets->tc_tsa[i] == IEEE_8021QAZ_TSA_STRICT) {
 			cos2bw.tsa =
 				QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_SP;
@@ -103,8 +121,9 @@ static int bnxt_hwrm_queue_cos2bw_cfg(struct bnxt *bp, struct ieee_ets *ets,
 				cpu_to_le32((ets->tc_tx_bw[i] * 100) |
 					    BW_VALUE_UNIT_PERCENT1_100);
 		}
+		data = &req.unused_0 + qidx * (sizeof(cos2bw) - 4);
 		memcpy(data, &cos2bw.queue_id, sizeof(cos2bw) - 4);
-		if (i == 0) {
+		if (qidx == 0) {
 			req.queue_id0 = cos2bw.queue_id;
 			req.unused_0 = 0;
 		}
@@ -132,66 +151,81 @@ static int bnxt_hwrm_queue_cos2bw_qcfg(struct bnxt *bp, struct ieee_ets *ets)
 
 	data = &resp->queue_id0 + offsetof(struct bnxt_cos2bw_cfg, queue_id);
 	for (i = 0; i < bp->max_tc; i++, data += sizeof(cos2bw) - 4) {
-		int j;
+		int tc;
 
 		memcpy(&cos2bw.queue_id, data, sizeof(cos2bw) - 4);
 		if (i == 0)
 			cos2bw.queue_id = resp->queue_id0;
 
-		for (j = 0; j < bp->max_tc; j++) {
-			if (bp->q_info[j].queue_id != cos2bw.queue_id)
-				continue;
-			if (cos2bw.tsa ==
-			    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_SP) {
-				ets->tc_tsa[j] = IEEE_8021QAZ_TSA_STRICT;
-			} else {
-				ets->tc_tsa[j] = IEEE_8021QAZ_TSA_ETS;
-				ets->tc_tx_bw[j] = cos2bw.bw_weight;
-			}
+		tc = bnxt_queue_to_tc(bp, cos2bw.queue_id);
+		if (tc < 0)
+			continue;
+
+		if (cos2bw.tsa ==
+		    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_SP) {
+			ets->tc_tsa[tc] = IEEE_8021QAZ_TSA_STRICT;
+		} else {
+			ets->tc_tsa[tc] = IEEE_8021QAZ_TSA_ETS;
+			ets->tc_tx_bw[tc] = cos2bw.bw_weight;
 		}
 	}
 	mutex_unlock(&bp->hwrm_cmd_lock);
 	return 0;
 }
 
-static int bnxt_hwrm_queue_cfg(struct bnxt *bp, unsigned int lltc_mask)
+static int bnxt_queue_remap(struct bnxt *bp, unsigned int lltc_mask)
 {
-	struct hwrm_queue_cfg_input req = {0};
-	int i;
+	unsigned long qmap = 0;
+	int max = bp->max_tc;
+	int i, j, rc;
 
-	if (netif_running(bp->dev))
-		bnxt_tx_disable(bp);
+	/* Assign lossless TCs first */
+	for (i = 0, j = 0; i < max; ) {
+		if (lltc_mask & (1 << i)) {
+			if (BNXT_LLQ(bp->q_info[j].queue_profile)) {
+				bp->tc_to_qidx[i] = j;
+				__set_bit(j, &qmap);
+				i++;
+			}
+			j++;
+			continue;
+		}
+		i++;
+	}
 
-	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_CFG, -1, -1);
-	req.flags = cpu_to_le32(QUEUE_CFG_REQ_FLAGS_PATH_BIDIR);
-	req.enables = cpu_to_le32(QUEUE_CFG_REQ_ENABLES_SERVICE_PROFILE);
+	for (i = 0, j = 0; i < max; i++) {
+		if (lltc_mask & (1 << i))
+			continue;
+		j = find_next_zero_bit(&qmap, max, j);
+		bp->tc_to_qidx[i] = j;
+		__set_bit(j, &qmap);
+		j++;
+	}
 
-	/* Configure lossless queues to lossy first */
-	req.service_profile = QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSY;
-	for (i = 0; i < bp->max_tc; i++) {
-		if (BNXT_LLQ(bp->q_info[i].queue_profile)) {
-			req.queue_id = cpu_to_le32(bp->q_info[i].queue_id);
-			hwrm_send_message(bp, &req, sizeof(req),
-					  HWRM_CMD_TIMEOUT);
-			bp->q_info[i].queue_profile =
-				QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSY;
+	if (netif_running(bp->dev)) {
+		bnxt_close_nic(bp, false, false);
+		rc = bnxt_open_nic(bp, false, false);
+		if (rc) {
+			netdev_warn(bp->dev, "failed to open NIC, rc = %d\n", rc);
+			return rc;
 		}
 	}
-
-	/* Now configure desired queues to lossless */
-	req.service_profile = QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSLESS;
-	for (i = 0; i < bp->max_tc; i++) {
-		if (lltc_mask & (1 << i)) {
-			req.queue_id = cpu_to_le32(bp->q_info[i].queue_id);
-			hwrm_send_message(bp, &req, sizeof(req),
-					  HWRM_CMD_TIMEOUT);
-			bp->q_info[i].queue_profile =
-				QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSLESS;
+	if (bp->ieee_ets) {
+		int tc = netdev_get_num_tc(bp->dev);
+
+		if (!tc)
+			tc = 1;
+		rc = bnxt_hwrm_queue_cos2bw_cfg(bp, bp->ieee_ets, tc);
+		if (rc) {
+			netdev_warn(bp->dev, "failed to config BW, rc = %d\n", rc);
+			return rc;
+		}
+		rc = bnxt_hwrm_queue_pri2cos_cfg(bp, bp->ieee_ets);
+		if (rc) {
+			netdev_warn(bp->dev, "failed to config prio, rc = %d\n", rc);
+			return rc;
 		}
 	}
-	if (netif_running(bp->dev))
-		bnxt_tx_enable(bp);
-
 	return 0;
 }
 
@@ -201,7 +235,7 @@ static int bnxt_hwrm_queue_pfc_cfg(struct bnxt *bp, struct ieee_pfc *pfc)
 	struct ieee_ets *my_ets = bp->ieee_ets;
 	unsigned int tc_mask = 0, pri_mask = 0;
 	u8 i, pri, lltc_count = 0;
-	bool need_q_recfg = false;
+	bool need_q_remap = false;
 	int rc;
 
 	if (!my_ets)
@@ -221,21 +255,25 @@ static int bnxt_hwrm_queue_pfc_cfg(struct bnxt *bp, struct ieee_pfc *pfc)
 	if (lltc_count > bp->max_lltc)
 		return -EINVAL;
 
-	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_PFCENABLE_CFG, -1, -1);
-	req.flags = cpu_to_le32(pri_mask);
-	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
-	if (rc)
-		return rc;
-
 	for (i = 0; i < bp->max_tc; i++) {
 		if (tc_mask & (1 << i)) {
-			if (!BNXT_LLQ(bp->q_info[i].queue_profile))
-				need_q_recfg = true;
+			u8 qidx = bp->tc_to_qidx[i];
+
+			if (!BNXT_LLQ(bp->q_info[qidx].queue_profile)) {
+				need_q_remap = true;
+				break;
+			}
 		}
 	}
 
-	if (need_q_recfg)
-		rc = bnxt_hwrm_queue_cfg(bp, tc_mask);
+	if (need_q_remap)
+		rc = bnxt_queue_remap(bp, tc_mask);
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_PFCENABLE_CFG, -1, -1);
+	req.flags = cpu_to_le32(pri_mask);
+	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (rc)
+		return rc;
 
 	return rc;
 }
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c
new file mode 100644
index 000000000000..94e208e9789f
--- /dev/null
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c
@@ -0,0 +1,124 @@
+/* Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2017-2018 Broadcom Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/debugfs.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include "bnxt_hsi.h"
+#include <linux/net_dim.h>
+#include "bnxt.h"
+#include "bnxt_debugfs.h"
+
+static struct dentry *bnxt_debug_mnt;
+
+static ssize_t debugfs_dim_read(struct file *filep,
+				char __user *buffer,
+				size_t count, loff_t *ppos)
+{
+	struct net_dim *dim = filep->private_data;
+	int len;
+	char *buf;
+
+	if (*ppos)
+		return 0;
+	if (!dim)
+		return -ENODEV;
+	buf = kasprintf(GFP_KERNEL,
+			"state = %d\n" \
+			"profile_ix = %d\n" \
+			"mode = %d\n" \
+			"tune_state = %d\n" \
+			"steps_right = %d\n" \
+			"steps_left = %d\n" \
+			"tired = %d\n",
+			dim->state,
+			dim->profile_ix,
+			dim->mode,
+			dim->tune_state,
+			dim->steps_right,
+			dim->steps_left,
+			dim->tired);
+	if (!buf)
+		return -ENOMEM;
+	if (count < strlen(buf)) {
+		kfree(buf);
+		return -ENOSPC;
+	}
+	len = simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
+	kfree(buf);
+	return len;
+}
+
+static const struct file_operations debugfs_dim_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.read = debugfs_dim_read,
+};
+
+static struct dentry *debugfs_dim_ring_init(struct net_dim *dim, int ring_idx,
+					    struct dentry *dd)
+{
+	static char qname[16];
+
+	snprintf(qname, 10, "%d", ring_idx);
+	return debugfs_create_file(qname, 0600, dd,
+				   dim, &debugfs_dim_fops);
+}
+
+void bnxt_debug_dev_init(struct bnxt *bp)
+{
+	const char *pname = pci_name(bp->pdev);
+	struct dentry *pdevf;
+	int i;
+
+	bp->debugfs_pdev = debugfs_create_dir(pname, bnxt_debug_mnt);
+	if (bp->debugfs_pdev) {
+		pdevf = debugfs_create_dir("dim", bp->debugfs_pdev);
+		if (!pdevf) {
+			pr_err("failed to create debugfs entry %s/dim\n",
+			       pname);
+			return;
+		}
+		bp->debugfs_dim = pdevf;
+		/* create files for each rx ring */
+		for (i = 0; i < bp->cp_nr_rings; i++) {
+			struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
+
+			if (cpr && bp->bnapi[i]->rx_ring) {
+				pdevf = debugfs_dim_ring_init(&cpr->dim, i,
+							      bp->debugfs_dim);
+				if (!pdevf)
+					pr_err("failed to create debugfs entry %s/dim/%d\n",
+					       pname, i);
+			}
+		}
+	} else {
+		pr_err("failed to create debugfs entry %s\n", pname);
+	}
+}
+
+void bnxt_debug_dev_exit(struct bnxt *bp)
+{
+	if (bp) {
+		debugfs_remove_recursive(bp->debugfs_pdev);
+		bp->debugfs_pdev = NULL;
+	}
+}
+
+void bnxt_debug_init(void)
+{
+	bnxt_debug_mnt = debugfs_create_dir("bnxt_en", NULL);
+	if (!bnxt_debug_mnt)
+		pr_err("failed to init bnxt_en debugfs\n");
+}
+
+void bnxt_debug_exit(void)
+{
+	debugfs_remove_recursive(bnxt_debug_mnt);
+}
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.h
new file mode 100644
index 000000000000..d0bb4887acd0
--- /dev/null
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.h
@@ -0,0 +1,23 @@
+/* Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2017-2018 Broadcom Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#include "bnxt_hsi.h"
+#include "bnxt.h"
+
+#ifdef CONFIG_DEBUG_FS
+void bnxt_debug_init(void);
+void bnxt_debug_exit(void);
+void bnxt_debug_dev_init(struct bnxt *bp);
+void bnxt_debug_dev_exit(struct bnxt *bp);
+#else
+static inline void bnxt_debug_init(void) {}
+static inline void bnxt_debug_exit(void) {}
+static inline void bnxt_debug_dev_init(struct bnxt *bp) {}
+static inline void bnxt_debug_dev_exit(struct bnxt *bp) {}
+#endif
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c
index 408dd190331e..afa97c8bb081 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c
@@ -21,11 +21,11 @@ void bnxt_dim_work(struct work_struct *work)
 	struct bnxt_napi *bnapi = container_of(cpr,
 					       struct bnxt_napi,
 					       cp_ring);
-	struct net_dim_cq_moder cur_profile = net_dim_get_profile(dim->mode,
-								  dim->profile_ix);
+	struct net_dim_cq_moder cur_moder =
+		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
 
-	cpr->rx_ring_coal.coal_ticks = cur_profile.usec;
-	cpr->rx_ring_coal.coal_bufs = cur_profile.pkts;
+	cpr->rx_ring_coal.coal_ticks = cur_moder.usec;
+	cpr->rx_ring_coal.coal_bufs = cur_moder.pkts;
 
 	bnxt_hwrm_set_ring_coal(bnapi->bp, bnapi);
 	dim->state = NET_DIM_START_MEASURE;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
index 8ba14ae00e8f..7270c8b0cef3 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
@@ -140,6 +140,19 @@ static int bnxt_set_coalesce(struct net_device *dev,
 #define BNXT_RX_STATS_EXT_ENTRY(counter)	\
 	{ BNXT_RX_STATS_EXT_OFFSET(counter), __stringify(counter) }
 
+enum {
+	RX_TOTAL_DISCARDS,
+	TX_TOTAL_DISCARDS,
+};
+
+static struct {
+	u64			counter;
+	char			string[ETH_GSTRING_LEN];
+} bnxt_sw_func_stats[] = {
+	{0, "rx_total_discard_pkts"},
+	{0, "tx_total_discard_pkts"},
+};
+
 static const struct {
 	long offset;
 	char string[ETH_GSTRING_LEN];
@@ -237,6 +250,7 @@ static const struct {
 	BNXT_RX_STATS_EXT_ENTRY(resume_roce_pause_events),
 };
 
+#define BNXT_NUM_SW_FUNC_STATS	ARRAY_SIZE(bnxt_sw_func_stats)
 #define BNXT_NUM_PORT_STATS ARRAY_SIZE(bnxt_port_stats_arr)
 #define BNXT_NUM_PORT_STATS_EXT ARRAY_SIZE(bnxt_port_stats_ext_arr)
 
@@ -244,6 +258,8 @@ static int bnxt_get_num_stats(struct bnxt *bp)
 {
 	int num_stats = BNXT_NUM_STATS * bp->cp_nr_rings;
 
+	num_stats += BNXT_NUM_SW_FUNC_STATS;
+
 	if (bp->flags & BNXT_FLAG_PORT_STATS)
 		num_stats += BNXT_NUM_PORT_STATS;
 
@@ -279,6 +295,9 @@ static void bnxt_get_ethtool_stats(struct net_device *dev,
 	if (!bp->bnapi)
 		return;
 
+	for (i = 0; i < BNXT_NUM_SW_FUNC_STATS; i++)
+		bnxt_sw_func_stats[i].counter = 0;
+
 	for (i = 0; i < bp->cp_nr_rings; i++) {
 		struct bnxt_napi *bnapi = bp->bnapi[i];
 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
@@ -288,7 +307,16 @@ static void bnxt_get_ethtool_stats(struct net_device *dev,
 		for (k = 0; k < stat_fields; j++, k++)
 			buf[j] = le64_to_cpu(hw_stats[k]);
 		buf[j++] = cpr->rx_l4_csum_errors;
+
+		bnxt_sw_func_stats[RX_TOTAL_DISCARDS].counter +=
+			le64_to_cpu(cpr->hw_stats->rx_discard_pkts);
+		bnxt_sw_func_stats[TX_TOTAL_DISCARDS].counter +=
+			le64_to_cpu(cpr->hw_stats->tx_discard_pkts);
 	}
+
+	for (i = 0; i < BNXT_NUM_SW_FUNC_STATS; i++, j++)
+		buf[j] = bnxt_sw_func_stats[i].counter;
+
 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
 		__le64 *port_stats = (__le64 *)bp->hw_rx_port_stats;
 
@@ -359,6 +387,11 @@ static void bnxt_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
 			sprintf(buf, "[%d]: rx_l4_csum_errors", i);
 			buf += ETH_GSTRING_LEN;
 		}
+		for (i = 0; i < BNXT_NUM_SW_FUNC_STATS; i++) {
+			strcpy(buf, bnxt_sw_func_stats[i].string);
+			buf += ETH_GSTRING_LEN;
+		}
+
 		if (bp->flags & BNXT_FLAG_PORT_STATS) {
 			for (i = 0; i < BNXT_NUM_PORT_STATS; i++) {
 				strcpy(buf, bnxt_port_stats_arr[i].string);
@@ -551,6 +584,8 @@ static int bnxt_set_channels(struct net_device *dev,
 			 * to renable
 			 */
 		}
+	} else {
+		rc = bnxt_reserve_rings(bp);
 	}
 
 	return rc;
@@ -1785,6 +1820,11 @@ static int nvm_get_dir_info(struct net_device *dev, u32 *entries, u32 *length)
 
 static int bnxt_get_eeprom_len(struct net_device *dev)
 {
+	struct bnxt *bp = netdev_priv(dev);
+
+	if (BNXT_VF(bp))
+		return 0;
+
 	/* The -1 return value allows the entire 32-bit range of offsets to be
 	 * passed via the ethtool command-line utility.
 	 */
@@ -2144,9 +2184,8 @@ static int bnxt_read_sfp_module_eeprom_info(struct bnxt *bp, u16 i2c_addr,
 static int bnxt_get_module_info(struct net_device *dev,
 				struct ethtool_modinfo *modinfo)
 {
+	u8 data[SFF_DIAG_SUPPORT_OFFSET + 1];
 	struct bnxt *bp = netdev_priv(dev);
-	struct hwrm_port_phy_i2c_read_input req = {0};
-	struct hwrm_port_phy_i2c_read_output *output = bp->hwrm_cmd_resp_addr;
 	int rc;
 
 	/* No point in going further if phy status indicates
@@ -2161,21 +2200,19 @@ static int bnxt_get_module_info(struct net_device *dev,
 	if (bp->hwrm_spec_code < 0x10202)
 		return -EOPNOTSUPP;
 
-	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_I2C_READ, -1, -1);
-	req.i2c_slave_addr = I2C_DEV_ADDR_A0;
-	req.page_number = 0;
-	req.page_offset = cpu_to_le16(SFP_EEPROM_SFF_8472_COMP_ADDR);
-	req.data_length = SFP_EEPROM_SFF_8472_COMP_SIZE;
-	req.port_id = cpu_to_le16(bp->pf.port_id);
-	mutex_lock(&bp->hwrm_cmd_lock);
-	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	rc = bnxt_read_sfp_module_eeprom_info(bp, I2C_DEV_ADDR_A0, 0, 0,
+					      SFF_DIAG_SUPPORT_OFFSET + 1,
+					      data);
 	if (!rc) {
-		u32 module_id = le32_to_cpu(output->data[0]);
+		u8 module_id = data[0];
+		u8 diag_supported = data[SFF_DIAG_SUPPORT_OFFSET];
 
 		switch (module_id) {
 		case SFF_MODULE_ID_SFP:
 			modinfo->type = ETH_MODULE_SFF_8472;
 			modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN;
+			if (!diag_supported)
+				modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN;
 			break;
 		case SFF_MODULE_ID_QSFP:
 		case SFF_MODULE_ID_QSFP_PLUS:
@@ -2191,7 +2228,6 @@ static int bnxt_get_module_info(struct net_device *dev,
 			break;
 		}
 	}
-	mutex_unlock(&bp->hwrm_cmd_lock);
 	return rc;
 }
 
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
index f952963d594e..a64910892c25 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
@@ -462,13 +462,13 @@ static int bnxt_hwrm_func_vf_resc_cfg(struct bnxt *bp, int num_vfs)
 	vf_vnics = hw_resc->max_vnics - bp->nr_vnics;
 	vf_vnics = min_t(u16, vf_vnics, vf_rx_rings);
 
-	req.min_rsscos_ctx = cpu_to_le16(1);
-	req.max_rsscos_ctx = cpu_to_le16(1);
+	req.min_rsscos_ctx = cpu_to_le16(BNXT_VF_MIN_RSS_CTX);
+	req.max_rsscos_ctx = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
 	if (pf->vf_resv_strategy == BNXT_VF_RESV_STRATEGY_MINIMAL) {
 		req.min_cmpl_rings = cpu_to_le16(1);
 		req.min_tx_rings = cpu_to_le16(1);
 		req.min_rx_rings = cpu_to_le16(1);
-		req.min_l2_ctxs = cpu_to_le16(1);
+		req.min_l2_ctxs = cpu_to_le16(BNXT_VF_MIN_L2_CTX);
 		req.min_vnics = cpu_to_le16(1);
 		req.min_stat_ctx = cpu_to_le16(1);
 		req.min_hw_ring_grps = cpu_to_le16(1);
@@ -483,7 +483,7 @@ static int bnxt_hwrm_func_vf_resc_cfg(struct bnxt *bp, int num_vfs)
 		req.min_cmpl_rings = cpu_to_le16(vf_cp_rings);
 		req.min_tx_rings = cpu_to_le16(vf_tx_rings);
 		req.min_rx_rings = cpu_to_le16(vf_rx_rings);
-		req.min_l2_ctxs = cpu_to_le16(4);
+		req.min_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
 		req.min_vnics = cpu_to_le16(vf_vnics);
 		req.min_stat_ctx = cpu_to_le16(vf_stat_ctx);
 		req.min_hw_ring_grps = cpu_to_le16(vf_ring_grps);
@@ -491,7 +491,7 @@ static int bnxt_hwrm_func_vf_resc_cfg(struct bnxt *bp, int num_vfs)
 	req.max_cmpl_rings = cpu_to_le16(vf_cp_rings);
 	req.max_tx_rings = cpu_to_le16(vf_tx_rings);
 	req.max_rx_rings = cpu_to_le16(vf_rx_rings);
-	req.max_l2_ctxs = cpu_to_le16(4);
+	req.max_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
 	req.max_vnics = cpu_to_le16(vf_vnics);
 	req.max_stat_ctx = cpu_to_le16(vf_stat_ctx);
 	req.max_hw_ring_grps = cpu_to_le16(vf_ring_grps);
@@ -809,6 +809,9 @@ static int bnxt_hwrm_fwd_resp(struct bnxt *bp, struct bnxt_vf_info *vf,
 	struct hwrm_fwd_resp_input req = {0};
 	struct hwrm_fwd_resp_output *resp = bp->hwrm_cmd_resp_addr;
 
+	if (BNXT_FWD_RESP_SIZE_ERR(msg_size))
+		return -EINVAL;
+
 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FWD_RESP, -1, -1);
 
 	/* Set the new target id */
@@ -845,6 +848,9 @@ static int bnxt_hwrm_fwd_err_resp(struct bnxt *bp, struct bnxt_vf_info *vf,
 	struct hwrm_reject_fwd_resp_input req = {0};
 	struct hwrm_reject_fwd_resp_output *resp = bp->hwrm_cmd_resp_addr;
 
+	if (BNXT_REJ_FWD_RESP_SIZE_ERR(msg_size))
+		return -EINVAL;
+
 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_REJECT_FWD_RESP, -1, -1);
 	/* Set the new target id */
 	req.target_id = cpu_to_le16(vf->fw_fid);
@@ -877,6 +883,9 @@ static int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, struct bnxt_vf_info *vf,
 	struct hwrm_exec_fwd_resp_input req = {0};
 	struct hwrm_exec_fwd_resp_output *resp = bp->hwrm_cmd_resp_addr;
 
+	if (BNXT_EXEC_FWD_RESP_SIZE_ERR(msg_size))
+		return -EINVAL;
+
 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_EXEC_FWD_RESP, -1, -1);
 	/* Set the new target id */
 	req.target_id = cpu_to_le16(vf->fw_fid);
@@ -914,7 +923,8 @@ static int bnxt_vf_configure_mac(struct bnxt *bp, struct bnxt_vf_info *vf)
 	if (req->enables & cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_DFLT_MAC_ADDR)) {
 		if (is_valid_ether_addr(req->dflt_mac_addr) &&
 		    ((vf->flags & BNXT_VF_TRUST) ||
-		     (!is_valid_ether_addr(vf->mac_addr)))) {
+		     !is_valid_ether_addr(vf->mac_addr) ||
+		     ether_addr_equal(req->dflt_mac_addr, vf->mac_addr))) {
 			ether_addr_copy(vf->vf_mac_addr, req->dflt_mac_addr);
 			return bnxt_hwrm_exec_fwd_resp(bp, vf, msg_size);
 		}
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.h
index d10f6f6c7860..e9b20cd19881 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.h
@@ -11,6 +11,23 @@
 #ifndef BNXT_SRIOV_H
 #define BNXT_SRIOV_H
 
+#define BNXT_FWD_RESP_SIZE_ERR(n)					\
+	((offsetof(struct hwrm_fwd_resp_input, encap_resp) + n) >	\
+	 sizeof(struct hwrm_fwd_resp_input))
+
+#define BNXT_EXEC_FWD_RESP_SIZE_ERR(n)					\
+	((offsetof(struct hwrm_exec_fwd_resp_input, encap_request) + n) >\
+	 offsetof(struct hwrm_exec_fwd_resp_input, encap_resp_target_id))
+
+#define BNXT_REJ_FWD_RESP_SIZE_ERR(n)					\
+	((offsetof(struct hwrm_reject_fwd_resp_input, encap_request) + n) >\
+	 offsetof(struct hwrm_reject_fwd_resp_input, encap_resp_target_id))
+
+#define BNXT_VF_MIN_RSS_CTX	1
+#define BNXT_VF_MAX_RSS_CTX	1
+#define BNXT_VF_MIN_L2_CTX	1
+#define BNXT_VF_MAX_L2_CTX	4
+
 int bnxt_get_vf_config(struct net_device *, int, struct ifla_vf_info *);
 int bnxt_set_vf_mac(struct net_device *, int, u8 *);
 int bnxt_set_vf_vlan(struct net_device *, int, u16, u8, __be16);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
index 1389ab5e05df..1f0e872d0667 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
@@ -113,10 +113,10 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
 	if (tx_avail != bp->tx_ring_size)
 		*event &= ~BNXT_RX_EVENT;
 
+	*len = xdp.data_end - xdp.data;
 	if (orig_data != xdp.data) {
 		offset = xdp.data - xdp.data_hard_start;
 		*data_ptr = xdp.data_hard_start + offset;
-		*len = xdp.data_end - xdp.data;
 	}
 	switch (act) {
 	case XDP_PASS:
diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
index 0445f2c0c629..20c1681bb1af 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c
+++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
@@ -652,7 +652,7 @@ static void bcmgenet_set_ring_rx_coalesce(struct bcmgenet_rx_ring *ring,
 	pkts = ring->rx_max_coalesced_frames;
 
 	if (ec->use_adaptive_rx_coalesce && !ring->dim.use_dim) {
-		moder = net_dim_get_def_profile(ring->dim.dim.mode);
+		moder = net_dim_get_def_rx_moderation(ring->dim.dim.mode);
 		usecs = moder.usec;
 		pkts = moder.pkts;
 	}
@@ -1925,7 +1925,7 @@ static void bcmgenet_dim_work(struct work_struct *work)
 	struct bcmgenet_rx_ring *ring =
 			container_of(ndim, struct bcmgenet_rx_ring, dim);
 	struct net_dim_cq_moder cur_profile =
-			net_dim_get_profile(dim->mode, dim->profile_ix);
+			net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
 
 	bcmgenet_set_rx_coalesce(ring, cur_profile.usec, cur_profile.pkts);
 	dim->state = NET_DIM_START_MEASURE;
@@ -2102,7 +2102,7 @@ static void bcmgenet_init_rx_coalesce(struct bcmgenet_rx_ring *ring)
 
 	/* If DIM was enabled, re-apply default parameters */
 	if (dim->use_dim) {
-		moder = net_dim_get_def_profile(dim->dim.mode);
+		moder = net_dim_get_def_rx_moderation(dim->dim.mode);
 		usecs = moder.usec;
 		pkts = moder.pkts;
 	}
diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c
index b4c9268100bb..3e93df5d4e3b 100644
--- a/drivers/net/ethernet/cadence/macb_main.c
+++ b/drivers/net/ethernet/cadence/macb_main.c
@@ -591,16 +591,10 @@ static int macb_mii_init(struct macb *bp)
 	dev_set_drvdata(&bp->dev->dev, bp->mii_bus);
 
 	np = bp->pdev->dev.of_node;
+	if (pdata)
+		bp->mii_bus->phy_mask = pdata->phy_mask;
 
-	if (np) {
-		err = of_mdiobus_register(bp->mii_bus, np);
-	} else {
-		if (pdata)
-			bp->mii_bus->phy_mask = pdata->phy_mask;
-
-		err = mdiobus_register(bp->mii_bus);
-	}
-
+	err = of_mdiobus_register(bp->mii_bus, np);
 	if (err)
 		goto err_out_free_mdiobus;
 
diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c
index e8b290473ee2..929d485a3a2f 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c
+++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c
@@ -1245,7 +1245,7 @@ static void cn23xx_setup_reg_address(struct octeon_device *oct)
 	    CN23XX_SLI_MAC_PF_INT_ENB64(oct->pcie_port, oct->pf_num);
 }
 
-static int cn23xx_sriov_config(struct octeon_device *oct)
+int cn23xx_sriov_config(struct octeon_device *oct)
 {
 	struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
 	u32 max_rings, total_rings, max_vfs, rings_per_vf;
@@ -1269,8 +1269,8 @@ static int cn23xx_sriov_config(struct octeon_device *oct)
 		break;
 	}
 
-	if (max_rings <= num_present_cpus())
-		num_pf_rings = 1;
+	if (oct->sriov_info.num_pf_rings)
+		num_pf_rings = oct->sriov_info.num_pf_rings;
 	else
 		num_pf_rings = num_present_cpus();
 
@@ -1497,3 +1497,57 @@ void cn23xx_tell_vf_its_macaddr_changed(struct octeon_device *oct, int vfidx,
 		octeon_mbox_write(oct, &mbox_cmd);
 	}
 }
+
+static void
+cn23xx_get_vf_stats_callback(struct octeon_device *oct,
+			     struct octeon_mbox_cmd *cmd, void *arg)
+{
+	struct oct_vf_stats_ctx *ctx = arg;
+
+	memcpy(ctx->stats, cmd->data, sizeof(struct oct_vf_stats));
+	atomic_set(&ctx->status, 1);
+}
+
+int cn23xx_get_vf_stats(struct octeon_device *oct, int vfidx,
+			struct oct_vf_stats *stats)
+{
+	u32 timeout = HZ; // 1sec
+	struct octeon_mbox_cmd mbox_cmd;
+	struct oct_vf_stats_ctx ctx;
+	u32 count = 0, ret;
+
+	if (!(oct->sriov_info.vf_drv_loaded_mask & (1ULL << vfidx)))
+		return -1;
+
+	if (sizeof(struct oct_vf_stats) > sizeof(mbox_cmd.data))
+		return -1;
+
+	mbox_cmd.msg.u64 = 0;
+	mbox_cmd.msg.s.type = OCTEON_MBOX_REQUEST;
+	mbox_cmd.msg.s.resp_needed = 1;
+	mbox_cmd.msg.s.cmd = OCTEON_GET_VF_STATS;
+	mbox_cmd.msg.s.len = 1;
+	mbox_cmd.q_no = vfidx * oct->sriov_info.rings_per_vf;
+	mbox_cmd.recv_len = 0;
+	mbox_cmd.recv_status = 0;
+	mbox_cmd.fn = (octeon_mbox_callback_t)cn23xx_get_vf_stats_callback;
+	ctx.stats = stats;
+	atomic_set(&ctx.status, 0);
+	mbox_cmd.fn_arg = (void *)&ctx;
+	memset(mbox_cmd.data, 0, sizeof(mbox_cmd.data));
+	octeon_mbox_write(oct, &mbox_cmd);
+
+	do {
+		schedule_timeout_uninterruptible(1);
+	} while ((atomic_read(&ctx.status) == 0) && (count++ < timeout));
+
+	ret = atomic_read(&ctx.status);
+	if (ret == 0) {
+		octeon_mbox_cancel(oct, 0);
+		dev_err(&oct->pci_dev->dev, "Unable to get stats from VF-%d, timedout\n",
+			vfidx);
+		return -1;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.h b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.h
index 2aba5247b6d8..e6f31d0d5c0b 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.h
+++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.h
@@ -43,6 +43,15 @@ struct octeon_cn23xx_pf {
 
 #define CN23XX_SLI_DEF_BP			0x40
 
+struct oct_vf_stats {
+	u64 rx_packets;
+	u64 tx_packets;
+	u64 rx_bytes;
+	u64 tx_bytes;
+	u64 broadcast;
+	u64 multicast;
+};
+
 int setup_cn23xx_octeon_pf_device(struct octeon_device *oct);
 
 int validate_cn23xx_pf_config_info(struct octeon_device *oct,
@@ -52,8 +61,13 @@ u32 cn23xx_pf_get_oq_ticks(struct octeon_device *oct, u32 time_intr_in_us);
 
 void cn23xx_dump_pf_initialized_regs(struct octeon_device *oct);
 
+int cn23xx_sriov_config(struct octeon_device *oct);
+
 int cn23xx_fw_loaded(struct octeon_device *oct);
 
 void cn23xx_tell_vf_its_macaddr_changed(struct octeon_device *oct, int vfidx,
 					u8 *mac);
+
+int cn23xx_get_vf_stats(struct octeon_device *oct, int ifidx,
+			struct oct_vf_stats *stats);
 #endif
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_core.c b/drivers/net/ethernet/cavium/liquidio/lio_core.c
index 2a94eee943b2..8093c5eafea2 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_core.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_core.c
@@ -29,6 +29,162 @@
 /* OOM task polling interval */
 #define LIO_OOM_POLL_INTERVAL_MS 250
 
+#define OCTNIC_MAX_SG  MAX_SKB_FRAGS
+
+/**
+ * \brief Callback for getting interface configuration
+ * @param status status of request
+ * @param buf pointer to resp structure
+ */
+void lio_if_cfg_callback(struct octeon_device *oct,
+			 u32 status __attribute__((unused)), void *buf)
+{
+	struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
+	struct liquidio_if_cfg_context *ctx;
+	struct liquidio_if_cfg_resp *resp;
+
+	resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
+	ctx = (struct liquidio_if_cfg_context *)sc->ctxptr;
+
+	oct = lio_get_device(ctx->octeon_id);
+	if (resp->status)
+		dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n",
+			CVM_CAST64(resp->status));
+	WRITE_ONCE(ctx->cond, 1);
+
+	snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s",
+		 resp->cfg_info.liquidio_firmware_version);
+
+	/* This barrier is required to be sure that the response has been
+	 * written fully before waking up the handler
+	 */
+	wmb();
+
+	wake_up_interruptible(&ctx->wc);
+}
+
+/**
+ * \brief Delete gather lists
+ * @param lio per-network private data
+ */
+void lio_delete_glists(struct lio *lio)
+{
+	struct octnic_gather *g;
+	int i;
+
+	kfree(lio->glist_lock);
+	lio->glist_lock = NULL;
+
+	if (!lio->glist)
+		return;
+
+	for (i = 0; i < lio->oct_dev->num_iqs; i++) {
+		do {
+			g = (struct octnic_gather *)
+			    lio_list_delete_head(&lio->glist[i]);
+			kfree(g);
+		} while (g);
+
+		if (lio->glists_virt_base && lio->glists_virt_base[i] &&
+		    lio->glists_dma_base && lio->glists_dma_base[i]) {
+			lio_dma_free(lio->oct_dev,
+				     lio->glist_entry_size * lio->tx_qsize,
+				     lio->glists_virt_base[i],
+				     lio->glists_dma_base[i]);
+		}
+	}
+
+	kfree(lio->glists_virt_base);
+	lio->glists_virt_base = NULL;
+
+	kfree(lio->glists_dma_base);
+	lio->glists_dma_base = NULL;
+
+	kfree(lio->glist);
+	lio->glist = NULL;
+}
+
+/**
+ * \brief Setup gather lists
+ * @param lio per-network private data
+ */
+int lio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
+{
+	struct octnic_gather *g;
+	int i, j;
+
+	lio->glist_lock =
+	    kcalloc(num_iqs, sizeof(*lio->glist_lock), GFP_KERNEL);
+	if (!lio->glist_lock)
+		return -ENOMEM;
+
+	lio->glist =
+	    kcalloc(num_iqs, sizeof(*lio->glist), GFP_KERNEL);
+	if (!lio->glist) {
+		kfree(lio->glist_lock);
+		lio->glist_lock = NULL;
+		return -ENOMEM;
+	}
+
+	lio->glist_entry_size =
+		ROUNDUP8((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * OCT_SG_ENTRY_SIZE);
+
+	/* allocate memory to store virtual and dma base address of
+	 * per glist consistent memory
+	 */
+	lio->glists_virt_base = kcalloc(num_iqs, sizeof(*lio->glists_virt_base),
+					GFP_KERNEL);
+	lio->glists_dma_base = kcalloc(num_iqs, sizeof(*lio->glists_dma_base),
+				       GFP_KERNEL);
+
+	if (!lio->glists_virt_base || !lio->glists_dma_base) {
+		lio_delete_glists(lio);
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < num_iqs; i++) {
+		int numa_node = dev_to_node(&oct->pci_dev->dev);
+
+		spin_lock_init(&lio->glist_lock[i]);
+
+		INIT_LIST_HEAD(&lio->glist[i]);
+
+		lio->glists_virt_base[i] =
+			lio_dma_alloc(oct,
+				      lio->glist_entry_size * lio->tx_qsize,
+				      &lio->glists_dma_base[i]);
+
+		if (!lio->glists_virt_base[i]) {
+			lio_delete_glists(lio);
+			return -ENOMEM;
+		}
+
+		for (j = 0; j < lio->tx_qsize; j++) {
+			g = kzalloc_node(sizeof(*g), GFP_KERNEL,
+					 numa_node);
+			if (!g)
+				g = kzalloc(sizeof(*g), GFP_KERNEL);
+			if (!g)
+				break;
+
+			g->sg = lio->glists_virt_base[i] +
+				(j * lio->glist_entry_size);
+
+			g->sg_dma_ptr = lio->glists_dma_base[i] +
+					(j * lio->glist_entry_size);
+
+			list_add_tail(&g->list, &lio->glist[i]);
+		}
+
+		if (j != lio->tx_qsize) {
+			lio_delete_glists(lio);
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
 int liquidio_set_feature(struct net_device *netdev, int cmd, u16 param1)
 {
 	struct lio *lio = GET_LIO(netdev);
@@ -880,8 +1036,8 @@ int octeon_setup_interrupt(struct octeon_device *oct, u32 num_ioqs)
 	int num_ioq_vectors;
 	int irqret, err;
 
-	oct->num_msix_irqs = num_ioqs;
 	if (oct->msix_on) {
+		oct->num_msix_irqs = num_ioqs;
 		if (OCTEON_CN23XX_PF(oct)) {
 			num_interrupts = MAX_IOQ_INTERRUPTS_PER_PF + 1;
 
@@ -1169,3 +1325,355 @@ int lio_wait_for_clean_oq(struct octeon_device *oct)
 
 	return pending_pkts;
 }
+
+static void
+octnet_nic_stats_callback(struct octeon_device *oct_dev,
+			  u32 status, void *ptr)
+{
+	struct octeon_soft_command *sc = (struct octeon_soft_command *)ptr;
+	struct oct_nic_stats_resp *resp =
+	    (struct oct_nic_stats_resp *)sc->virtrptr;
+	struct oct_nic_stats_ctrl *ctrl =
+	    (struct oct_nic_stats_ctrl *)sc->ctxptr;
+	struct nic_rx_stats *rsp_rstats = &resp->stats.fromwire;
+	struct nic_tx_stats *rsp_tstats = &resp->stats.fromhost;
+	struct nic_rx_stats *rstats = &oct_dev->link_stats.fromwire;
+	struct nic_tx_stats *tstats = &oct_dev->link_stats.fromhost;
+
+	if (status != OCTEON_REQUEST_TIMEOUT && !resp->status) {
+		octeon_swap_8B_data((u64 *)&resp->stats,
+				    (sizeof(struct oct_link_stats)) >> 3);
+
+		/* RX link-level stats */
+		rstats->total_rcvd = rsp_rstats->total_rcvd;
+		rstats->bytes_rcvd = rsp_rstats->bytes_rcvd;
+		rstats->total_bcst = rsp_rstats->total_bcst;
+		rstats->total_mcst = rsp_rstats->total_mcst;
+		rstats->runts      = rsp_rstats->runts;
+		rstats->ctl_rcvd   = rsp_rstats->ctl_rcvd;
+		/* Accounts for over/under-run of buffers */
+		rstats->fifo_err  = rsp_rstats->fifo_err;
+		rstats->dmac_drop = rsp_rstats->dmac_drop;
+		rstats->fcs_err   = rsp_rstats->fcs_err;
+		rstats->jabber_err = rsp_rstats->jabber_err;
+		rstats->l2_err    = rsp_rstats->l2_err;
+		rstats->frame_err = rsp_rstats->frame_err;
+		rstats->red_drops = rsp_rstats->red_drops;
+
+		/* RX firmware stats */
+		rstats->fw_total_rcvd = rsp_rstats->fw_total_rcvd;
+		rstats->fw_total_fwd = rsp_rstats->fw_total_fwd;
+		rstats->fw_total_mcast = rsp_rstats->fw_total_mcast;
+		rstats->fw_total_bcast = rsp_rstats->fw_total_bcast;
+		rstats->fw_err_pko = rsp_rstats->fw_err_pko;
+		rstats->fw_err_link = rsp_rstats->fw_err_link;
+		rstats->fw_err_drop = rsp_rstats->fw_err_drop;
+		rstats->fw_rx_vxlan = rsp_rstats->fw_rx_vxlan;
+		rstats->fw_rx_vxlan_err = rsp_rstats->fw_rx_vxlan_err;
+
+		/* Number of packets that are LROed      */
+		rstats->fw_lro_pkts = rsp_rstats->fw_lro_pkts;
+		/* Number of octets that are LROed       */
+		rstats->fw_lro_octs = rsp_rstats->fw_lro_octs;
+		/* Number of LRO packets formed          */
+		rstats->fw_total_lro = rsp_rstats->fw_total_lro;
+		/* Number of times lRO of packet aborted */
+		rstats->fw_lro_aborts = rsp_rstats->fw_lro_aborts;
+		rstats->fw_lro_aborts_port = rsp_rstats->fw_lro_aborts_port;
+		rstats->fw_lro_aborts_seq = rsp_rstats->fw_lro_aborts_seq;
+		rstats->fw_lro_aborts_tsval = rsp_rstats->fw_lro_aborts_tsval;
+		rstats->fw_lro_aborts_timer = rsp_rstats->fw_lro_aborts_timer;
+		/* intrmod: packet forward rate */
+		rstats->fwd_rate = rsp_rstats->fwd_rate;
+
+		/* TX link-level stats */
+		tstats->total_pkts_sent = rsp_tstats->total_pkts_sent;
+		tstats->total_bytes_sent = rsp_tstats->total_bytes_sent;
+		tstats->mcast_pkts_sent = rsp_tstats->mcast_pkts_sent;
+		tstats->bcast_pkts_sent = rsp_tstats->bcast_pkts_sent;
+		tstats->ctl_sent = rsp_tstats->ctl_sent;
+		/* Packets sent after one collision*/
+		tstats->one_collision_sent = rsp_tstats->one_collision_sent;
+		/* Packets sent after multiple collision*/
+		tstats->multi_collision_sent = rsp_tstats->multi_collision_sent;
+		/* Packets not sent due to max collisions */
+		tstats->max_collision_fail = rsp_tstats->max_collision_fail;
+		/* Packets not sent due to max deferrals */
+		tstats->max_deferral_fail = rsp_tstats->max_deferral_fail;
+		/* Accounts for over/under-run of buffers */
+		tstats->fifo_err = rsp_tstats->fifo_err;
+		tstats->runts = rsp_tstats->runts;
+		/* Total number of collisions detected */
+		tstats->total_collisions = rsp_tstats->total_collisions;
+
+		/* firmware stats */
+		tstats->fw_total_sent = rsp_tstats->fw_total_sent;
+		tstats->fw_total_fwd = rsp_tstats->fw_total_fwd;
+		tstats->fw_total_mcast_sent = rsp_tstats->fw_total_mcast_sent;
+		tstats->fw_total_bcast_sent = rsp_tstats->fw_total_bcast_sent;
+		tstats->fw_err_pko = rsp_tstats->fw_err_pko;
+		tstats->fw_err_pki = rsp_tstats->fw_err_pki;
+		tstats->fw_err_link = rsp_tstats->fw_err_link;
+		tstats->fw_err_drop = rsp_tstats->fw_err_drop;
+		tstats->fw_tso = rsp_tstats->fw_tso;
+		tstats->fw_tso_fwd = rsp_tstats->fw_tso_fwd;
+		tstats->fw_err_tso = rsp_tstats->fw_err_tso;
+		tstats->fw_tx_vxlan = rsp_tstats->fw_tx_vxlan;
+
+		resp->status = 1;
+	} else {
+		resp->status = -1;
+	}
+	complete(&ctrl->complete);
+}
+
+int octnet_get_link_stats(struct net_device *netdev)
+{
+	struct lio *lio = GET_LIO(netdev);
+	struct octeon_device *oct_dev = lio->oct_dev;
+	struct octeon_soft_command *sc;
+	struct oct_nic_stats_ctrl *ctrl;
+	struct oct_nic_stats_resp *resp;
+	int retval;
+
+	/* Alloc soft command */
+	sc = (struct octeon_soft_command *)
+		octeon_alloc_soft_command(oct_dev,
+					  0,
+					  sizeof(struct oct_nic_stats_resp),
+					  sizeof(struct octnic_ctrl_pkt));
+
+	if (!sc)
+		return -ENOMEM;
+
+	resp = (struct oct_nic_stats_resp *)sc->virtrptr;
+	memset(resp, 0, sizeof(struct oct_nic_stats_resp));
+
+	ctrl = (struct oct_nic_stats_ctrl *)sc->ctxptr;
+	memset(ctrl, 0, sizeof(struct oct_nic_stats_ctrl));
+	ctrl->netdev = netdev;
+	init_completion(&ctrl->complete);
+
+	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
+
+	octeon_prepare_soft_command(oct_dev, sc, OPCODE_NIC,
+				    OPCODE_NIC_PORT_STATS, 0, 0, 0);
+
+	sc->callback = octnet_nic_stats_callback;
+	sc->callback_arg = sc;
+	sc->wait_time = 500;	/*in milli seconds*/
+
+	retval = octeon_send_soft_command(oct_dev, sc);
+	if (retval == IQ_SEND_FAILED) {
+		octeon_free_soft_command(oct_dev, sc);
+		return -EINVAL;
+	}
+
+	wait_for_completion_timeout(&ctrl->complete, msecs_to_jiffies(1000));
+
+	if (resp->status != 1) {
+		octeon_free_soft_command(oct_dev, sc);
+
+		return -EINVAL;
+	}
+
+	octeon_free_soft_command(oct_dev, sc);
+
+	return 0;
+}
+
+static void liquidio_nic_seapi_ctl_callback(struct octeon_device *oct,
+					    u32 status,
+					    void *buf)
+{
+	struct liquidio_nic_seapi_ctl_context *ctx;
+	struct octeon_soft_command *sc = buf;
+
+	ctx = sc->ctxptr;
+
+	oct = lio_get_device(ctx->octeon_id);
+	if (status) {
+		dev_err(&oct->pci_dev->dev, "%s: instruction failed. Status: %llx\n",
+			__func__,
+			CVM_CAST64(status));
+	}
+	ctx->status = status;
+	complete(&ctx->complete);
+}
+
+int liquidio_set_speed(struct lio *lio, int speed)
+{
+	struct liquidio_nic_seapi_ctl_context *ctx;
+	struct octeon_device *oct = lio->oct_dev;
+	struct oct_nic_seapi_resp *resp;
+	struct octeon_soft_command *sc;
+	union octnet_cmd *ncmd;
+	u32 ctx_size;
+	int retval;
+	u32 var;
+
+	if (oct->speed_setting == speed)
+		return 0;
+
+	if (!OCTEON_CN23XX_PF(oct)) {
+		dev_err(&oct->pci_dev->dev, "%s: SET SPEED only for PF\n",
+			__func__);
+		return -EOPNOTSUPP;
+	}
+
+	ctx_size = sizeof(struct liquidio_nic_seapi_ctl_context);
+	sc = octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
+				       sizeof(struct oct_nic_seapi_resp),
+				       ctx_size);
+	if (!sc)
+		return -ENOMEM;
+
+	ncmd = sc->virtdptr;
+	ctx  = sc->ctxptr;
+	resp = sc->virtrptr;
+	memset(resp, 0, sizeof(struct oct_nic_seapi_resp));
+
+	ctx->octeon_id = lio_get_device_id(oct);
+	ctx->status = 0;
+	init_completion(&ctx->complete);
+
+	ncmd->u64 = 0;
+	ncmd->s.cmd = SEAPI_CMD_SPEED_SET;
+	ncmd->s.param1 = speed;
+
+	octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
+
+	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
+
+	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
+				    OPCODE_NIC_UBOOT_CTL, 0, 0, 0);
+
+	sc->callback = liquidio_nic_seapi_ctl_callback;
+	sc->callback_arg = sc;
+	sc->wait_time = 5000;
+
+	retval = octeon_send_soft_command(oct, sc);
+	if (retval == IQ_SEND_FAILED) {
+		dev_info(&oct->pci_dev->dev, "Failed to send soft command\n");
+		retval = -EBUSY;
+	} else {
+		/* Wait for response or timeout */
+		if (wait_for_completion_timeout(&ctx->complete,
+						msecs_to_jiffies(10000)) == 0) {
+			dev_err(&oct->pci_dev->dev, "%s: sc timeout\n",
+				__func__);
+			octeon_free_soft_command(oct, sc);
+			return -EINTR;
+		}
+
+		retval = resp->status;
+
+		if (retval) {
+			dev_err(&oct->pci_dev->dev, "%s failed, retval=%d\n",
+				__func__, retval);
+			octeon_free_soft_command(oct, sc);
+			return -EIO;
+		}
+
+		var = be32_to_cpu((__force __be32)resp->speed);
+		if (var != speed) {
+			dev_err(&oct->pci_dev->dev,
+				"%s: setting failed speed= %x, expect %x\n",
+				__func__, var, speed);
+		}
+
+		oct->speed_setting = var;
+	}
+
+	octeon_free_soft_command(oct, sc);
+
+	return retval;
+}
+
+int liquidio_get_speed(struct lio *lio)
+{
+	struct liquidio_nic_seapi_ctl_context *ctx;
+	struct octeon_device *oct = lio->oct_dev;
+	struct oct_nic_seapi_resp *resp;
+	struct octeon_soft_command *sc;
+	union octnet_cmd *ncmd;
+	u32 ctx_size;
+	int retval;
+
+	ctx_size = sizeof(struct liquidio_nic_seapi_ctl_context);
+	sc = octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
+				       sizeof(struct oct_nic_seapi_resp),
+				       ctx_size);
+	if (!sc)
+		return -ENOMEM;
+
+	ncmd = sc->virtdptr;
+	ctx  = sc->ctxptr;
+	resp = sc->virtrptr;
+	memset(resp, 0, sizeof(struct oct_nic_seapi_resp));
+
+	ctx->octeon_id = lio_get_device_id(oct);
+	ctx->status = 0;
+	init_completion(&ctx->complete);
+
+	ncmd->u64 = 0;
+	ncmd->s.cmd = SEAPI_CMD_SPEED_GET;
+
+	octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
+
+	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
+
+	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
+				    OPCODE_NIC_UBOOT_CTL, 0, 0, 0);
+
+	sc->callback = liquidio_nic_seapi_ctl_callback;
+	sc->callback_arg = sc;
+	sc->wait_time = 5000;
+
+	retval = octeon_send_soft_command(oct, sc);
+	if (retval == IQ_SEND_FAILED) {
+		dev_info(&oct->pci_dev->dev, "Failed to send soft command\n");
+		oct->no_speed_setting = 1;
+		oct->speed_setting = 25;
+
+		retval = -EBUSY;
+	} else {
+		if (wait_for_completion_timeout(&ctx->complete,
+						msecs_to_jiffies(10000)) == 0) {
+			dev_err(&oct->pci_dev->dev, "%s: sc timeout\n",
+				__func__);
+
+			oct->speed_setting = 25;
+			oct->no_speed_setting = 1;
+
+			octeon_free_soft_command(oct, sc);
+
+			return -EINTR;
+		}
+		retval = resp->status;
+		if (retval) {
+			dev_err(&oct->pci_dev->dev,
+				"%s failed retval=%d\n", __func__, retval);
+			oct->no_speed_setting = 1;
+			oct->speed_setting = 25;
+			octeon_free_soft_command(oct, sc);
+			retval = -EIO;
+		} else {
+			u32 var;
+
+			var = be32_to_cpu((__force __be32)resp->speed);
+			oct->speed_setting = var;
+			if (var == 0xffff) {
+				oct->no_speed_setting = 1;
+				/* unable to access boot variables
+				 * get the default value based on the NIC type
+				 */
+				oct->speed_setting = 25;
+			}
+		}
+	}
+
+	octeon_free_soft_command(oct, sc);
+
+	return retval;
+}
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
index 550ac29682a5..06f7449c569d 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
@@ -32,7 +32,6 @@
 #include "cn23xx_vf_device.h"
 
 static int lio_reset_queues(struct net_device *netdev, uint32_t num_qs);
-static int octnet_get_link_stats(struct net_device *netdev);
 
 struct oct_intrmod_context {
 	int octeon_id;
@@ -96,11 +95,9 @@ static const char oct_stats_strings[][ETH_GSTRING_LEN] = {
 	"tx_packets",
 	"rx_bytes",
 	"tx_bytes",
-	"rx_errors",	/*jabber_err+l2_err+frame_err */
-	"tx_errors",	/*fw_err_pko+fw_err_link+fw_err_drop */
-	"rx_dropped",   /*st->fromwire.total_rcvd - st->fromwire.fw_total_rcvd +
-			 *st->fromwire.dmac_drop + st->fromwire.fw_err_drop
-			 */
+	"rx_errors",
+	"tx_errors",
+	"rx_dropped",
 	"tx_dropped",
 
 	"tx_total_sent",
@@ -115,14 +112,17 @@ static const char oct_stats_strings[][ETH_GSTRING_LEN] = {
 	"tx_tso_err",
 	"tx_vxlan",
 
+	"tx_mcast",
+	"tx_bcast",
+
 	"mac_tx_total_pkts",
 	"mac_tx_total_bytes",
 	"mac_tx_mcast_pkts",
 	"mac_tx_bcast_pkts",
-	"mac_tx_ctl_packets",	/*oct->link_stats.fromhost.ctl_sent */
+	"mac_tx_ctl_packets",
 	"mac_tx_total_collisions",
 	"mac_tx_one_collision",
-	"mac_tx_multi_collison",
+	"mac_tx_multi_collision",
 	"mac_tx_max_collision_fail",
 	"mac_tx_max_deferal_fail",
 	"mac_tx_fifo_err",
@@ -130,6 +130,8 @@ static const char oct_stats_strings[][ETH_GSTRING_LEN] = {
 
 	"rx_total_rcvd",
 	"rx_total_fwd",
+	"rx_mcast",
+	"rx_bcast",
 	"rx_jabber_err",
 	"rx_l2_err",
 	"rx_frame_err",
@@ -170,17 +172,21 @@ static const char oct_vf_stats_strings[][ETH_GSTRING_LEN] = {
 	"tx_packets",
 	"rx_bytes",
 	"tx_bytes",
-	"rx_errors", /* jabber_err + l2_err+frame_err */
-	"tx_errors", /* fw_err_pko + fw_err_link+fw_err_drop */
-	"rx_dropped", /* total_rcvd - fw_total_rcvd + dmac_drop + fw_err_drop */
+	"rx_errors",
+	"tx_errors",
+	"rx_dropped",
 	"tx_dropped",
+	"rx_mcast",
+	"tx_mcast",
+	"rx_bcast",
+	"tx_bcast",
 	"link_state_changes",
 };
 
 /* statistics of host tx queue */
 static const char oct_iq_stats_strings[][ETH_GSTRING_LEN] = {
-	"packets",		/*oct->instr_queue[iq_no]->stats.tx_done*/
-	"bytes",		/*oct->instr_queue[iq_no]->stats.tx_tot_bytes*/
+	"packets",
+	"bytes",
 	"dropped",
 	"iq_busy",
 	"sgentry_sent",
@@ -197,13 +203,9 @@ static const char oct_iq_stats_strings[][ETH_GSTRING_LEN] = {
 
 /* statistics of host rx queue */
 static const char oct_droq_stats_strings[][ETH_GSTRING_LEN] = {
-	"packets",		/*oct->droq[oq_no]->stats.rx_pkts_received */
-	"bytes",		/*oct->droq[oq_no]->stats.rx_bytes_received */
-	"dropped",		/*oct->droq[oq_no]->stats.rx_dropped+
-				 *oct->droq[oq_no]->stats.dropped_nodispatch+
-				 *oct->droq[oq_no]->stats.dropped_toomany+
-				 *oct->droq[oq_no]->stats.dropped_nomem
-				 */
+	"packets",
+	"bytes",
+	"dropped",
 	"dropped_nomem",
 	"dropped_toomany",
 	"fw_dropped",
@@ -228,46 +230,147 @@ static int lio_get_link_ksettings(struct net_device *netdev,
 	struct lio *lio = GET_LIO(netdev);
 	struct octeon_device *oct = lio->oct_dev;
 	struct oct_link_info *linfo;
-	u32 supported = 0, advertising = 0;
 
 	linfo = &lio->linfo;
 
+	ethtool_link_ksettings_zero_link_mode(ecmd, supported);
+	ethtool_link_ksettings_zero_link_mode(ecmd, advertising);
+
 	switch (linfo->link.s.phy_type) {
 	case LIO_PHY_PORT_TP:
 		ecmd->base.port = PORT_TP;
-		supported = (SUPPORTED_10000baseT_Full |
-			     SUPPORTED_TP | SUPPORTED_Pause);
-		advertising = (ADVERTISED_10000baseT_Full | ADVERTISED_Pause);
 		ecmd->base.autoneg = AUTONEG_DISABLE;
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, TP);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, Pause);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     10000baseT_Full);
+
+		ethtool_link_ksettings_add_link_mode(ecmd, advertising, Pause);
+		ethtool_link_ksettings_add_link_mode(ecmd, advertising,
+						     10000baseT_Full);
+
 		break;
 
 	case LIO_PHY_PORT_FIBRE:
-		ecmd->base.port = PORT_FIBRE;
-
-		if (linfo->link.s.speed == SPEED_10000) {
-			supported = SUPPORTED_10000baseT_Full;
-			advertising = ADVERTISED_10000baseT_Full;
+		if (linfo->link.s.if_mode == INTERFACE_MODE_XAUI ||
+		    linfo->link.s.if_mode == INTERFACE_MODE_RXAUI ||
+		    linfo->link.s.if_mode == INTERFACE_MODE_XLAUI ||
+		    linfo->link.s.if_mode == INTERFACE_MODE_XFI) {
+			dev_dbg(&oct->pci_dev->dev, "ecmd->base.transceiver is XCVR_EXTERNAL\n");
+		} else {
+			dev_err(&oct->pci_dev->dev, "Unknown link interface mode: %d\n",
+				linfo->link.s.if_mode);
 		}
 
-		supported |= SUPPORTED_FIBRE | SUPPORTED_Pause;
-		advertising |= ADVERTISED_Pause;
+		ecmd->base.port = PORT_FIBRE;
 		ecmd->base.autoneg = AUTONEG_DISABLE;
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
+
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, Pause);
+		ethtool_link_ksettings_add_link_mode(ecmd, advertising, Pause);
+		if (oct->subsystem_id == OCTEON_CN2350_25GB_SUBSYS_ID ||
+		    oct->subsystem_id == OCTEON_CN2360_25GB_SUBSYS_ID) {
+			if (OCTEON_CN23XX_PF(oct)) {
+				ethtool_link_ksettings_add_link_mode
+					(ecmd, supported, 25000baseSR_Full);
+				ethtool_link_ksettings_add_link_mode
+					(ecmd, supported, 25000baseKR_Full);
+				ethtool_link_ksettings_add_link_mode
+					(ecmd, supported, 25000baseCR_Full);
+
+				if (oct->no_speed_setting == 0)  {
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, supported,
+						 10000baseSR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, supported,
+						 10000baseKR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, supported,
+						 10000baseCR_Full);
+				}
+
+				if (oct->no_speed_setting == 0)
+					liquidio_get_speed(lio);
+				else
+					oct->speed_setting = 25;
+
+				if (oct->speed_setting == 10) {
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 10000baseSR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 10000baseKR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 10000baseCR_Full);
+				}
+				if (oct->speed_setting == 25) {
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 25000baseSR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 25000baseKR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 25000baseCR_Full);
+				}
+			} else { /* VF */
+				if (linfo->link.s.speed == 10000) {
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, supported,
+						 10000baseSR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, supported,
+						 10000baseKR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, supported,
+						 10000baseCR_Full);
+
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 10000baseSR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 10000baseKR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 10000baseCR_Full);
+				}
+
+				if (linfo->link.s.speed == 25000) {
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, supported,
+						 25000baseSR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, supported,
+						 25000baseKR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, supported,
+						 25000baseCR_Full);
+
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 25000baseSR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 25000baseKR_Full);
+					ethtool_link_ksettings_add_link_mode
+						(ecmd, advertising,
+						 25000baseCR_Full);
+				}
+			}
+		} else {
+			ethtool_link_ksettings_add_link_mode(ecmd, supported,
+							     10000baseT_Full);
+			ethtool_link_ksettings_add_link_mode(ecmd, advertising,
+							     10000baseT_Full);
+		}
 		break;
 	}
 
-	if (linfo->link.s.if_mode == INTERFACE_MODE_XAUI ||
-	    linfo->link.s.if_mode == INTERFACE_MODE_RXAUI ||
-	    linfo->link.s.if_mode == INTERFACE_MODE_XLAUI ||
-	    linfo->link.s.if_mode == INTERFACE_MODE_XFI) {
-		ethtool_convert_legacy_u32_to_link_mode(
-			ecmd->link_modes.supported, supported);
-		ethtool_convert_legacy_u32_to_link_mode(
-			ecmd->link_modes.advertising, advertising);
-	} else {
-		dev_err(&oct->pci_dev->dev, "Unknown link interface reported %d\n",
-			linfo->link.s.if_mode);
-	}
-
 	if (linfo->link.s.link_up) {
 		ecmd->base.speed = linfo->link.s.speed;
 		ecmd->base.duplex = linfo->link.s.duplex;
@@ -279,6 +382,51 @@ static int lio_get_link_ksettings(struct net_device *netdev,
 	return 0;
 }
 
+static int lio_set_link_ksettings(struct net_device *netdev,
+				  const struct ethtool_link_ksettings *ecmd)
+{
+	const int speed = ecmd->base.speed;
+	struct lio *lio = GET_LIO(netdev);
+	struct oct_link_info *linfo;
+	struct octeon_device *oct;
+	u32 is25G = 0;
+
+	oct = lio->oct_dev;
+
+	linfo = &lio->linfo;
+
+	if (oct->subsystem_id == OCTEON_CN2350_25GB_SUBSYS_ID ||
+	    oct->subsystem_id == OCTEON_CN2360_25GB_SUBSYS_ID) {
+		is25G = 1;
+	} else {
+		return -EOPNOTSUPP;
+	}
+
+	if (oct->no_speed_setting) {
+		dev_err(&oct->pci_dev->dev, "%s: Changing speed is not supported\n",
+			__func__);
+		return -EOPNOTSUPP;
+	}
+
+	if ((ecmd->base.duplex != DUPLEX_UNKNOWN &&
+	     ecmd->base.duplex != linfo->link.s.duplex) ||
+	     ecmd->base.autoneg != AUTONEG_DISABLE ||
+	    (ecmd->base.speed != 10000 && ecmd->base.speed != 25000 &&
+	     ecmd->base.speed != SPEED_UNKNOWN))
+		return -EOPNOTSUPP;
+
+	if ((oct->speed_boot == speed / 1000) &&
+	    oct->speed_boot == oct->speed_setting)
+		return 0;
+
+	liquidio_set_speed(lio, speed / 1000);
+
+	dev_dbg(&oct->pci_dev->dev, "Port speed is set to %dG\n",
+		oct->speed_setting);
+
+	return 0;
+}
+
 static void
 lio_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
 {
@@ -359,7 +507,14 @@ lio_ethtool_get_channels(struct net_device *dev,
 		rx_count = CFG_GET_NUM_RXQS_NIC_IF(conf6x, lio->ifidx);
 		tx_count = CFG_GET_NUM_TXQS_NIC_IF(conf6x, lio->ifidx);
 	} else if (OCTEON_CN23XX_PF(oct)) {
-		max_combined = lio->linfo.num_txpciq;
+		if (oct->sriov_info.sriov_enabled) {
+			max_combined = lio->linfo.num_txpciq;
+		} else {
+			struct octeon_config *conf23_pf =
+				CHIP_CONF(oct, cn23xx_pf);
+
+			max_combined = CFG_GET_IQ_MAX_Q(conf23_pf);
+		}
 		combined_count = oct->num_iqs;
 	} else if (OCTEON_CN23XX_VF(oct)) {
 		u64 reg_val = 0ULL;
@@ -423,9 +578,15 @@ lio_irq_reallocate_irqs(struct octeon_device *oct, uint32_t num_ioqs)
 
 	kfree(oct->irq_name_storage);
 	oct->irq_name_storage = NULL;
+
+	if (octeon_allocate_ioq_vector(oct, num_ioqs)) {
+		dev_err(&oct->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
+		return -1;
+	}
+
 	if (octeon_setup_interrupt(oct, num_ioqs)) {
 		dev_info(&oct->pci_dev->dev, "Setup interrupt failed\n");
-		return 1;
+		return -1;
 	}
 
 	/* Enable Octeon device interrupts */
@@ -455,7 +616,16 @@ lio_ethtool_set_channels(struct net_device *dev,
 	combined_count = channel->combined_count;
 
 	if (OCTEON_CN23XX_PF(oct)) {
-		max_combined = channel->max_combined;
+		if (oct->sriov_info.sriov_enabled) {
+			max_combined = lio->linfo.num_txpciq;
+		} else {
+			struct octeon_config *conf23_pf =
+				CHIP_CONF(oct,
+					  cn23xx_pf);
+
+			max_combined =
+				CFG_GET_IQ_MAX_Q(conf23_pf);
+		}
 	} else if (OCTEON_CN23XX_VF(oct)) {
 		u64 reg_val = 0ULL;
 		u64 ctrl = CN23XX_VF_SLI_IQ_PKT_CONTROL64(0);
@@ -483,7 +653,6 @@ lio_ethtool_set_channels(struct net_device *dev,
 	if (lio_reset_queues(dev, combined_count))
 		return -EINVAL;
 
-	lio_irq_reallocate_irqs(oct, combined_count);
 	if (stopped)
 		dev->netdev_ops->ndo_open(dev);
 
@@ -822,12 +991,120 @@ lio_ethtool_get_ringparam(struct net_device *netdev,
 	ering->rx_jumbo_max_pending = 0;
 }
 
+static int lio_23xx_reconfigure_queue_count(struct lio *lio)
+{
+	struct octeon_device *oct = lio->oct_dev;
+	struct liquidio_if_cfg_context *ctx;
+	u32 resp_size, ctx_size, data_size;
+	struct liquidio_if_cfg_resp *resp;
+	struct octeon_soft_command *sc;
+	union oct_nic_if_cfg if_cfg;
+	struct lio_version *vdata;
+	u32 ifidx_or_pfnum;
+	int retval;
+	int j;
+
+	resp_size = sizeof(struct liquidio_if_cfg_resp);
+	ctx_size = sizeof(struct liquidio_if_cfg_context);
+	data_size = sizeof(struct lio_version);
+	sc = (struct octeon_soft_command *)
+		octeon_alloc_soft_command(oct, data_size,
+					  resp_size, ctx_size);
+	if (!sc) {
+		dev_err(&oct->pci_dev->dev, "%s: Failed to allocate soft command\n",
+			__func__);
+		return -1;
+	}
+
+	resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
+	ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
+	vdata = (struct lio_version *)sc->virtdptr;
+
+	vdata->major = (__force u16)cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
+	vdata->minor = (__force u16)cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
+	vdata->micro = (__force u16)cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
+
+	ifidx_or_pfnum = oct->pf_num;
+	WRITE_ONCE(ctx->cond, 0);
+	ctx->octeon_id = lio_get_device_id(oct);
+	init_waitqueue_head(&ctx->wc);
+
+	if_cfg.u64 = 0;
+	if_cfg.s.num_iqueues = oct->sriov_info.num_pf_rings;
+	if_cfg.s.num_oqueues = oct->sriov_info.num_pf_rings;
+	if_cfg.s.base_queue = oct->sriov_info.pf_srn;
+	if_cfg.s.gmx_port_id = oct->pf_num;
+
+	sc->iq_no = 0;
+	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
+				    OPCODE_NIC_QCOUNT_UPDATE, 0,
+				    if_cfg.u64, 0);
+	sc->callback = lio_if_cfg_callback;
+	sc->callback_arg = sc;
+	sc->wait_time = LIO_IFCFG_WAIT_TIME;
+
+	retval = octeon_send_soft_command(oct, sc);
+	if (retval == IQ_SEND_FAILED) {
+		dev_err(&oct->pci_dev->dev,
+			"iq/oq config failed status: %x\n",
+			retval);
+		goto qcount_update_fail;
+	}
+
+	if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR) {
+		dev_err(&oct->pci_dev->dev, "Wait interrupted\n");
+		return -1;
+	}
+
+	retval = resp->status;
+	if (retval) {
+		dev_err(&oct->pci_dev->dev, "iq/oq config failed\n");
+		goto qcount_update_fail;
+	}
+
+	octeon_swap_8B_data((u64 *)(&resp->cfg_info),
+			    (sizeof(struct liquidio_if_cfg_info)) >> 3);
+
+	lio->ifidx = ifidx_or_pfnum;
+	lio->linfo.num_rxpciq = hweight64(resp->cfg_info.iqmask);
+	lio->linfo.num_txpciq = hweight64(resp->cfg_info.iqmask);
+	for (j = 0; j < lio->linfo.num_rxpciq; j++) {
+		lio->linfo.rxpciq[j].u64 =
+			resp->cfg_info.linfo.rxpciq[j].u64;
+	}
+
+	for (j = 0; j < lio->linfo.num_txpciq; j++) {
+		lio->linfo.txpciq[j].u64 =
+			resp->cfg_info.linfo.txpciq[j].u64;
+	}
+
+	lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
+	lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
+	lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
+	lio->txq = lio->linfo.txpciq[0].s.q_no;
+	lio->rxq = lio->linfo.rxpciq[0].s.q_no;
+
+	octeon_free_soft_command(oct, sc);
+	dev_info(&oct->pci_dev->dev, "Queue count updated to %d\n",
+		 lio->linfo.num_rxpciq);
+
+	return 0;
+
+qcount_update_fail:
+	octeon_free_soft_command(oct, sc);
+
+	return -1;
+}
+
 static int lio_reset_queues(struct net_device *netdev, uint32_t num_qs)
 {
 	struct lio *lio = GET_LIO(netdev);
 	struct octeon_device *oct = lio->oct_dev;
+	int i, queue_count_update = 0;
 	struct napi_struct *napi, *n;
-	int i, update = 0;
+	int ret;
+
+	schedule_timeout_uninterruptible(msecs_to_jiffies(100));
 
 	if (wait_for_pending_requests(oct))
 		dev_err(&oct->pci_dev->dev, "There were pending requests\n");
@@ -836,7 +1113,7 @@ static int lio_reset_queues(struct net_device *netdev, uint32_t num_qs)
 		dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
 
 	if (octeon_set_io_queues_off(oct)) {
-		dev_err(&oct->pci_dev->dev, "setting io queues off failed\n");
+		dev_err(&oct->pci_dev->dev, "Setting io queues off failed\n");
 		return -1;
 	}
 
@@ -849,9 +1126,40 @@ static int lio_reset_queues(struct net_device *netdev, uint32_t num_qs)
 		netif_napi_del(napi);
 
 	if (num_qs != oct->num_iqs) {
-		netif_set_real_num_rx_queues(netdev, num_qs);
-		netif_set_real_num_tx_queues(netdev, num_qs);
-		update = 1;
+		ret = netif_set_real_num_rx_queues(netdev, num_qs);
+		if (ret) {
+			dev_err(&oct->pci_dev->dev,
+				"Setting real number rx failed\n");
+			return ret;
+		}
+
+		ret = netif_set_real_num_tx_queues(netdev, num_qs);
+		if (ret) {
+			dev_err(&oct->pci_dev->dev,
+				"Setting real number tx failed\n");
+			return ret;
+		}
+
+		/* The value of queue_count_update decides whether it is the
+		 * queue count or the descriptor count that is being
+		 * re-configured.
+		 */
+		queue_count_update = 1;
+	}
+
+	/* Re-configuration of queues can happen in two scenarios, SRIOV enabled
+	 * and SRIOV disabled. Few things like recreating queue zero, resetting
+	 * glists and IRQs are required for both. For the latter, some more
+	 * steps like updating sriov_info for the octeon device need to be done.
+	 */
+	if (queue_count_update) {
+		lio_delete_glists(lio);
+
+		/* Delete mbox for PF which is SRIOV disabled because sriov_info
+		 * will be now changed.
+		 */
+		if ((OCTEON_CN23XX_PF(oct)) && !oct->sriov_info.sriov_enabled)
+			oct->fn_list.free_mbox(oct);
 	}
 
 	for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
@@ -866,24 +1174,91 @@ static int lio_reset_queues(struct net_device *netdev, uint32_t num_qs)
 		octeon_delete_instr_queue(oct, i);
 	}
 
+	if (queue_count_update) {
+		/* For PF re-configure sriov related information */
+		if ((OCTEON_CN23XX_PF(oct)) &&
+		    !oct->sriov_info.sriov_enabled) {
+			oct->sriov_info.num_pf_rings = num_qs;
+			if (cn23xx_sriov_config(oct)) {
+				dev_err(&oct->pci_dev->dev,
+					"Queue reset aborted: SRIOV config failed\n");
+				return -1;
+			}
+
+			num_qs = oct->sriov_info.num_pf_rings;
+		}
+	}
+
 	if (oct->fn_list.setup_device_regs(oct)) {
 		dev_err(&oct->pci_dev->dev, "Failed to configure device registers\n");
 		return -1;
 	}
 
-	if (liquidio_setup_io_queues(oct, 0, num_qs, num_qs)) {
-		dev_err(&oct->pci_dev->dev, "IO queues initialization failed\n");
-		return -1;
+	/* The following are needed in case of queue count re-configuration and
+	 * not for descriptor count re-configuration.
+	 */
+	if (queue_count_update) {
+		if (octeon_setup_instr_queues(oct))
+			return -1;
+
+		if (octeon_setup_output_queues(oct))
+			return -1;
+
+		/* Recreating mbox for PF that is SRIOV disabled */
+		if (OCTEON_CN23XX_PF(oct) && !oct->sriov_info.sriov_enabled) {
+			if (oct->fn_list.setup_mbox(oct)) {
+				dev_err(&oct->pci_dev->dev, "Mailbox setup failed\n");
+				return -1;
+			}
+		}
+
+		/* Deleting and recreating IRQs whether the interface is SRIOV
+		 * enabled or disabled.
+		 */
+		if (lio_irq_reallocate_irqs(oct, num_qs)) {
+			dev_err(&oct->pci_dev->dev, "IRQs could not be allocated\n");
+			return -1;
+		}
+
+		/* Enable the input and output queues for this Octeon device */
+		if (oct->fn_list.enable_io_queues(oct)) {
+			dev_err(&oct->pci_dev->dev, "Failed to enable input/output queues\n");
+			return -1;
+		}
+
+		for (i = 0; i < oct->num_oqs; i++)
+			writel(oct->droq[i]->max_count,
+			       oct->droq[i]->pkts_credit_reg);
+
+		/* Informing firmware about the new queue count. It is required
+		 * for firmware to allocate more number of queues than those at
+		 * load time.
+		 */
+		if (OCTEON_CN23XX_PF(oct) && !oct->sriov_info.sriov_enabled) {
+			if (lio_23xx_reconfigure_queue_count(lio))
+				return -1;
+		}
 	}
 
-	/* Enable the input and output queues for this Octeon device */
-	if (oct->fn_list.enable_io_queues(oct)) {
-		dev_err(&oct->pci_dev->dev, "Failed to enable input/output queues");
+	/* Once firmware is aware of the new value, queues can be recreated */
+	if (liquidio_setup_io_queues(oct, 0, num_qs, num_qs)) {
+		dev_err(&oct->pci_dev->dev, "I/O queues creation failed\n");
 		return -1;
 	}
 
-	if (update && lio_send_queue_count_update(netdev, num_qs))
-		return -1;
+	if (queue_count_update) {
+		if (lio_setup_glists(oct, lio, num_qs)) {
+			dev_err(&oct->pci_dev->dev, "Gather list allocation failed\n");
+			return -1;
+		}
+
+		/* Send firmware the information about new number of queues
+		 * if the interface is a VF or a PF that is SRIOV enabled.
+		 */
+		if (oct->sriov_info.sriov_enabled || OCTEON_CN23XX_VF(oct))
+			if (lio_send_queue_count_update(netdev, num_qs))
+				return -1;
+	}
 
 	return 0;
 }
@@ -928,7 +1303,7 @@ static int lio_ethtool_set_ringparam(struct net_device *netdev,
 		CFG_SET_NUM_RX_DESCS_NIC_IF(octeon_get_conf(oct), lio->ifidx,
 					    rx_count);
 
-	if (lio_reset_queues(netdev, lio->linfo.num_txpciq))
+	if (lio_reset_queues(netdev, oct->num_iqs))
 		goto err_lio_reset_queues;
 
 	if (stopped)
@@ -1063,33 +1438,48 @@ lio_get_ethtool_stats(struct net_device *netdev,
 {
 	struct lio *lio = GET_LIO(netdev);
 	struct octeon_device *oct_dev = lio->oct_dev;
-	struct net_device_stats *netstats = &netdev->stats;
+	struct rtnl_link_stats64 lstats;
 	int i = 0, j;
 
 	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
 		return;
 
-	netdev->netdev_ops->ndo_get_stats(netdev);
-	octnet_get_link_stats(netdev);
-
+	netdev->netdev_ops->ndo_get_stats64(netdev, &lstats);
 	/*sum of oct->droq[oq_no]->stats->rx_pkts_received */
-	data[i++] = CVM_CAST64(netstats->rx_packets);
+	data[i++] = lstats.rx_packets;
 	/*sum of oct->instr_queue[iq_no]->stats.tx_done */
-	data[i++] = CVM_CAST64(netstats->tx_packets);
+	data[i++] = lstats.tx_packets;
 	/*sum of oct->droq[oq_no]->stats->rx_bytes_received */
-	data[i++] = CVM_CAST64(netstats->rx_bytes);
+	data[i++] = lstats.rx_bytes;
 	/*sum of oct->instr_queue[iq_no]->stats.tx_tot_bytes */
-	data[i++] = CVM_CAST64(netstats->tx_bytes);
-	data[i++] = CVM_CAST64(netstats->rx_errors);
-	data[i++] = CVM_CAST64(netstats->tx_errors);
+	data[i++] = lstats.tx_bytes;
+	data[i++] = lstats.rx_errors +
+			oct_dev->link_stats.fromwire.fcs_err +
+			oct_dev->link_stats.fromwire.jabber_err +
+			oct_dev->link_stats.fromwire.l2_err +
+			oct_dev->link_stats.fromwire.frame_err;
+	data[i++] = lstats.tx_errors;
 	/*sum of oct->droq[oq_no]->stats->rx_dropped +
 	 *oct->droq[oq_no]->stats->dropped_nodispatch +
 	 *oct->droq[oq_no]->stats->dropped_toomany +
 	 *oct->droq[oq_no]->stats->dropped_nomem
 	 */
-	data[i++] = CVM_CAST64(netstats->rx_dropped);
+	data[i++] = lstats.rx_dropped +
+			oct_dev->link_stats.fromwire.fifo_err +
+			oct_dev->link_stats.fromwire.dmac_drop +
+			oct_dev->link_stats.fromwire.red_drops +
+			oct_dev->link_stats.fromwire.fw_err_pko +
+			oct_dev->link_stats.fromwire.fw_err_link +
+			oct_dev->link_stats.fromwire.fw_err_drop;
 	/*sum of oct->instr_queue[iq_no]->stats.tx_dropped */
-	data[i++] = CVM_CAST64(netstats->tx_dropped);
+	data[i++] = lstats.tx_dropped +
+			oct_dev->link_stats.fromhost.max_collision_fail +
+			oct_dev->link_stats.fromhost.max_deferral_fail +
+			oct_dev->link_stats.fromhost.total_collisions +
+			oct_dev->link_stats.fromhost.fw_err_pko +
+			oct_dev->link_stats.fromhost.fw_err_link +
+			oct_dev->link_stats.fromhost.fw_err_drop +
+			oct_dev->link_stats.fromhost.fw_err_pki;
 
 	/* firmware tx stats */
 	/*per_core_stats[cvmx_get_core_num()].link_stats[mdata->from_ifidx].
@@ -1124,6 +1514,10 @@ lio_get_ethtool_stats(struct net_device *netdev,
 	 */
 	data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_tx_vxlan);
 
+	/* Multicast packets sent by this port */
+	data[i++] = oct_dev->link_stats.fromhost.fw_total_mcast_sent;
+	data[i++] = oct_dev->link_stats.fromhost.fw_total_bcast_sent;
+
 	/* mac tx statistics */
 	/*CVMX_BGXX_CMRX_TX_STAT5 */
 	data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.total_pkts_sent);
@@ -1160,6 +1554,9 @@ lio_get_ethtool_stats(struct net_device *netdev,
 	 *fw_total_fwd
 	 */
 	data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.fw_total_fwd);
+	/* Multicast packets received on this port */
+	data[i++] = oct_dev->link_stats.fromwire.fw_total_mcast;
+	data[i++] = oct_dev->link_stats.fromwire.fw_total_bcast;
 	/*per_core_stats[core_id].link_stats[ifidx].fromwire.jabber_err */
 	data[i++] = CVM_CAST64(oct_dev->link_stats.fromwire.jabber_err);
 	/*per_core_stats[core_id].link_stats[ifidx].fromwire.l2_err */
@@ -1328,7 +1725,7 @@ static void lio_vf_get_ethtool_stats(struct net_device *netdev,
 				     __attribute__((unused)),
 				     u64 *data)
 {
-	struct net_device_stats *netstats = &netdev->stats;
+	struct rtnl_link_stats64 lstats;
 	struct lio *lio = GET_LIO(netdev);
 	struct octeon_device *oct_dev = lio->oct_dev;
 	int i = 0, j, vj;
@@ -1336,25 +1733,31 @@ static void lio_vf_get_ethtool_stats(struct net_device *netdev,
 	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
 		return;
 
-	netdev->netdev_ops->ndo_get_stats(netdev);
+	netdev->netdev_ops->ndo_get_stats64(netdev, &lstats);
 	/* sum of oct->droq[oq_no]->stats->rx_pkts_received */
-	data[i++] = CVM_CAST64(netstats->rx_packets);
+	data[i++] = lstats.rx_packets;
 	/* sum of oct->instr_queue[iq_no]->stats.tx_done */
-	data[i++] = CVM_CAST64(netstats->tx_packets);
+	data[i++] = lstats.tx_packets;
 	/* sum of oct->droq[oq_no]->stats->rx_bytes_received */
-	data[i++] = CVM_CAST64(netstats->rx_bytes);
+	data[i++] = lstats.rx_bytes;
 	/* sum of oct->instr_queue[iq_no]->stats.tx_tot_bytes */
-	data[i++] = CVM_CAST64(netstats->tx_bytes);
-	data[i++] = CVM_CAST64(netstats->rx_errors);
-	data[i++] = CVM_CAST64(netstats->tx_errors);
+	data[i++] = lstats.tx_bytes;
+	data[i++] = lstats.rx_errors;
+	data[i++] = lstats.tx_errors;
 	 /* sum of oct->droq[oq_no]->stats->rx_dropped +
 	  * oct->droq[oq_no]->stats->dropped_nodispatch +
 	  * oct->droq[oq_no]->stats->dropped_toomany +
 	  * oct->droq[oq_no]->stats->dropped_nomem
 	  */
-	data[i++] = CVM_CAST64(netstats->rx_dropped);
+	data[i++] = lstats.rx_dropped;
 	/* sum of oct->instr_queue[iq_no]->stats.tx_dropped */
-	data[i++] = CVM_CAST64(netstats->tx_dropped);
+	data[i++] = lstats.tx_dropped;
+
+	data[i++] = oct_dev->link_stats.fromwire.fw_total_mcast;
+	data[i++] = oct_dev->link_stats.fromhost.fw_total_mcast_sent;
+	data[i++] = oct_dev->link_stats.fromwire.fw_total_bcast;
+	data[i++] = oct_dev->link_stats.fromhost.fw_total_bcast_sent;
+
 	/* lio->link_changes */
 	data[i++] = CVM_CAST64(lio->link_changes);
 
@@ -1765,161 +2168,6 @@ static int octnet_set_intrmod_cfg(struct lio *lio,
 	return -EINTR;
 }
 
-static void
-octnet_nic_stats_callback(struct octeon_device *oct_dev,
-			  u32 status, void *ptr)
-{
-	struct octeon_soft_command *sc = (struct octeon_soft_command *)ptr;
-	struct oct_nic_stats_resp *resp =
-	    (struct oct_nic_stats_resp *)sc->virtrptr;
-	struct oct_nic_stats_ctrl *ctrl =
-	    (struct oct_nic_stats_ctrl *)sc->ctxptr;
-	struct nic_rx_stats *rsp_rstats = &resp->stats.fromwire;
-	struct nic_tx_stats *rsp_tstats = &resp->stats.fromhost;
-
-	struct nic_rx_stats *rstats = &oct_dev->link_stats.fromwire;
-	struct nic_tx_stats *tstats = &oct_dev->link_stats.fromhost;
-
-	if ((status != OCTEON_REQUEST_TIMEOUT) && !resp->status) {
-		octeon_swap_8B_data((u64 *)&resp->stats,
-				    (sizeof(struct oct_link_stats)) >> 3);
-
-		/* RX link-level stats */
-		rstats->total_rcvd = rsp_rstats->total_rcvd;
-		rstats->bytes_rcvd = rsp_rstats->bytes_rcvd;
-		rstats->total_bcst = rsp_rstats->total_bcst;
-		rstats->total_mcst = rsp_rstats->total_mcst;
-		rstats->runts      = rsp_rstats->runts;
-		rstats->ctl_rcvd   = rsp_rstats->ctl_rcvd;
-		/* Accounts for over/under-run of buffers */
-		rstats->fifo_err  = rsp_rstats->fifo_err;
-		rstats->dmac_drop = rsp_rstats->dmac_drop;
-		rstats->fcs_err   = rsp_rstats->fcs_err;
-		rstats->jabber_err = rsp_rstats->jabber_err;
-		rstats->l2_err    = rsp_rstats->l2_err;
-		rstats->frame_err = rsp_rstats->frame_err;
-
-		/* RX firmware stats */
-		rstats->fw_total_rcvd = rsp_rstats->fw_total_rcvd;
-		rstats->fw_total_fwd = rsp_rstats->fw_total_fwd;
-		rstats->fw_err_pko = rsp_rstats->fw_err_pko;
-		rstats->fw_err_link = rsp_rstats->fw_err_link;
-		rstats->fw_err_drop = rsp_rstats->fw_err_drop;
-		rstats->fw_rx_vxlan = rsp_rstats->fw_rx_vxlan;
-		rstats->fw_rx_vxlan_err = rsp_rstats->fw_rx_vxlan_err;
-
-		/* Number of packets that are LROed      */
-		rstats->fw_lro_pkts = rsp_rstats->fw_lro_pkts;
-		/* Number of octets that are LROed       */
-		rstats->fw_lro_octs = rsp_rstats->fw_lro_octs;
-		/* Number of LRO packets formed          */
-		rstats->fw_total_lro = rsp_rstats->fw_total_lro;
-		/* Number of times lRO of packet aborted */
-		rstats->fw_lro_aborts = rsp_rstats->fw_lro_aborts;
-		rstats->fw_lro_aborts_port = rsp_rstats->fw_lro_aborts_port;
-		rstats->fw_lro_aborts_seq = rsp_rstats->fw_lro_aborts_seq;
-		rstats->fw_lro_aborts_tsval = rsp_rstats->fw_lro_aborts_tsval;
-		rstats->fw_lro_aborts_timer = rsp_rstats->fw_lro_aborts_timer;
-		/* intrmod: packet forward rate */
-		rstats->fwd_rate = rsp_rstats->fwd_rate;
-
-		/* TX link-level stats */
-		tstats->total_pkts_sent = rsp_tstats->total_pkts_sent;
-		tstats->total_bytes_sent = rsp_tstats->total_bytes_sent;
-		tstats->mcast_pkts_sent = rsp_tstats->mcast_pkts_sent;
-		tstats->bcast_pkts_sent = rsp_tstats->bcast_pkts_sent;
-		tstats->ctl_sent = rsp_tstats->ctl_sent;
-		/* Packets sent after one collision*/
-		tstats->one_collision_sent = rsp_tstats->one_collision_sent;
-		/* Packets sent after multiple collision*/
-		tstats->multi_collision_sent = rsp_tstats->multi_collision_sent;
-		/* Packets not sent due to max collisions */
-		tstats->max_collision_fail = rsp_tstats->max_collision_fail;
-		/* Packets not sent due to max deferrals */
-		tstats->max_deferral_fail = rsp_tstats->max_deferral_fail;
-		/* Accounts for over/under-run of buffers */
-		tstats->fifo_err = rsp_tstats->fifo_err;
-		tstats->runts = rsp_tstats->runts;
-		/* Total number of collisions detected */
-		tstats->total_collisions = rsp_tstats->total_collisions;
-
-		/* firmware stats */
-		tstats->fw_total_sent = rsp_tstats->fw_total_sent;
-		tstats->fw_total_fwd = rsp_tstats->fw_total_fwd;
-		tstats->fw_err_pko = rsp_tstats->fw_err_pko;
-		tstats->fw_err_pki = rsp_tstats->fw_err_pki;
-		tstats->fw_err_link = rsp_tstats->fw_err_link;
-		tstats->fw_err_drop = rsp_tstats->fw_err_drop;
-		tstats->fw_tso = rsp_tstats->fw_tso;
-		tstats->fw_tso_fwd = rsp_tstats->fw_tso_fwd;
-		tstats->fw_err_tso = rsp_tstats->fw_err_tso;
-		tstats->fw_tx_vxlan = rsp_tstats->fw_tx_vxlan;
-
-		resp->status = 1;
-	} else {
-		resp->status = -1;
-	}
-	complete(&ctrl->complete);
-}
-
-/*  Configure interrupt moderation parameters */
-static int octnet_get_link_stats(struct net_device *netdev)
-{
-	struct lio *lio = GET_LIO(netdev);
-	struct octeon_device *oct_dev = lio->oct_dev;
-
-	struct octeon_soft_command *sc;
-	struct oct_nic_stats_ctrl *ctrl;
-	struct oct_nic_stats_resp *resp;
-
-	int retval;
-
-	/* Alloc soft command */
-	sc = (struct octeon_soft_command *)
-		octeon_alloc_soft_command(oct_dev,
-					  0,
-					  sizeof(struct oct_nic_stats_resp),
-					  sizeof(struct octnic_ctrl_pkt));
-
-	if (!sc)
-		return -ENOMEM;
-
-	resp = (struct oct_nic_stats_resp *)sc->virtrptr;
-	memset(resp, 0, sizeof(struct oct_nic_stats_resp));
-
-	ctrl = (struct oct_nic_stats_ctrl *)sc->ctxptr;
-	memset(ctrl, 0, sizeof(struct oct_nic_stats_ctrl));
-	ctrl->netdev = netdev;
-	init_completion(&ctrl->complete);
-
-	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
-
-	octeon_prepare_soft_command(oct_dev, sc, OPCODE_NIC,
-				    OPCODE_NIC_PORT_STATS, 0, 0, 0);
-
-	sc->callback = octnet_nic_stats_callback;
-	sc->callback_arg = sc;
-	sc->wait_time = 500;	/*in milli seconds*/
-
-	retval = octeon_send_soft_command(oct_dev, sc);
-	if (retval == IQ_SEND_FAILED) {
-		octeon_free_soft_command(oct_dev, sc);
-		return -EINVAL;
-	}
-
-	wait_for_completion_timeout(&ctrl->complete, msecs_to_jiffies(1000));
-
-	if (resp->status != 1) {
-		octeon_free_soft_command(oct_dev, sc);
-
-		return -EINVAL;
-	}
-
-	octeon_free_soft_command(oct_dev, sc);
-
-	return 0;
-}
-
 static int lio_get_intr_coalesce(struct net_device *netdev,
 				 struct ethtool_coalesce *intr_coal)
 {
@@ -2864,6 +3112,7 @@ static int lio_set_priv_flags(struct net_device *netdev, u32 flags)
 
 static const struct ethtool_ops lio_ethtool_ops = {
 	.get_link_ksettings	= lio_get_link_ksettings,
+	.set_link_ksettings	= lio_set_link_ksettings,
 	.get_link		= ethtool_op_get_link,
 	.get_drvinfo		= lio_get_drvinfo,
 	.get_ringparam		= lio_ethtool_get_ringparam,
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c
index 603a144d3d9c..e500528ad751 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c
@@ -138,33 +138,10 @@ union tx_info {
  * by this structure in the NIC module.
  */
 
-#define OCTNIC_MAX_SG  (MAX_SKB_FRAGS)
-
 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
 #define OCTNIC_GSO_MAX_SIZE                                                    \
 	(CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
 
-/** Structure of a node in list of gather components maintained by
- * NIC driver for each network device.
- */
-struct octnic_gather {
-	/** List manipulation. Next and prev pointers. */
-	struct list_head list;
-
-	/** Size of the gather component at sg in bytes. */
-	int sg_size;
-
-	/** Number of bytes that sg was adjusted to make it 8B-aligned. */
-	int adjust;
-
-	/** Gather component that can accommodate max sized fragment list
-	 *  received from the IP layer.
-	 */
-	struct octeon_sg_entry *sg;
-
-	dma_addr_t sg_dma_ptr;
-};
-
 struct handshake {
 	struct completion init;
 	struct completion started;
@@ -520,7 +497,7 @@ static void liquidio_deinit_pci(void)
  */
 static inline int check_txq_status(struct lio *lio)
 {
-	int numqs = lio->netdev->num_tx_queues;
+	int numqs = lio->netdev->real_num_tx_queues;
 	int ret_val = 0;
 	int q, iq;
 
@@ -541,148 +518,6 @@ static inline int check_txq_status(struct lio *lio)
 	return ret_val;
 }
 
-/**
- * Remove the node at the head of the list. The list would be empty at
- * the end of this call if there are no more nodes in the list.
- */
-static inline struct list_head *list_delete_head(struct list_head *root)
-{
-	struct list_head *node;
-
-	if ((root->prev == root) && (root->next == root))
-		node = NULL;
-	else
-		node = root->next;
-
-	if (node)
-		list_del(node);
-
-	return node;
-}
-
-/**
- * \brief Delete gather lists
- * @param lio per-network private data
- */
-static void delete_glists(struct lio *lio)
-{
-	struct octnic_gather *g;
-	int i;
-
-	kfree(lio->glist_lock);
-	lio->glist_lock = NULL;
-
-	if (!lio->glist)
-		return;
-
-	for (i = 0; i < lio->linfo.num_txpciq; i++) {
-		do {
-			g = (struct octnic_gather *)
-				list_delete_head(&lio->glist[i]);
-			if (g)
-				kfree(g);
-		} while (g);
-
-		if (lio->glists_virt_base && lio->glists_virt_base[i] &&
-		    lio->glists_dma_base && lio->glists_dma_base[i]) {
-			lio_dma_free(lio->oct_dev,
-				     lio->glist_entry_size * lio->tx_qsize,
-				     lio->glists_virt_base[i],
-				     lio->glists_dma_base[i]);
-		}
-	}
-
-	kfree(lio->glists_virt_base);
-	lio->glists_virt_base = NULL;
-
-	kfree(lio->glists_dma_base);
-	lio->glists_dma_base = NULL;
-
-	kfree(lio->glist);
-	lio->glist = NULL;
-}
-
-/**
- * \brief Setup gather lists
- * @param lio per-network private data
- */
-static int setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
-{
-	int i, j;
-	struct octnic_gather *g;
-
-	lio->glist_lock = kcalloc(num_iqs, sizeof(*lio->glist_lock),
-				  GFP_KERNEL);
-	if (!lio->glist_lock)
-		return -ENOMEM;
-
-	lio->glist = kcalloc(num_iqs, sizeof(*lio->glist),
-			     GFP_KERNEL);
-	if (!lio->glist) {
-		kfree(lio->glist_lock);
-		lio->glist_lock = NULL;
-		return -ENOMEM;
-	}
-
-	lio->glist_entry_size =
-		ROUNDUP8((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * OCT_SG_ENTRY_SIZE);
-
-	/* allocate memory to store virtual and dma base address of
-	 * per glist consistent memory
-	 */
-	lio->glists_virt_base = kcalloc(num_iqs, sizeof(*lio->glists_virt_base),
-					GFP_KERNEL);
-	lio->glists_dma_base = kcalloc(num_iqs, sizeof(*lio->glists_dma_base),
-				       GFP_KERNEL);
-
-	if (!lio->glists_virt_base || !lio->glists_dma_base) {
-		delete_glists(lio);
-		return -ENOMEM;
-	}
-
-	for (i = 0; i < num_iqs; i++) {
-		int numa_node = dev_to_node(&oct->pci_dev->dev);
-
-		spin_lock_init(&lio->glist_lock[i]);
-
-		INIT_LIST_HEAD(&lio->glist[i]);
-
-		lio->glists_virt_base[i] =
-			lio_dma_alloc(oct,
-				      lio->glist_entry_size * lio->tx_qsize,
-				      &lio->glists_dma_base[i]);
-
-		if (!lio->glists_virt_base[i]) {
-			delete_glists(lio);
-			return -ENOMEM;
-		}
-
-		for (j = 0; j < lio->tx_qsize; j++) {
-			g = kzalloc_node(sizeof(*g), GFP_KERNEL,
-					 numa_node);
-			if (!g)
-				g = kzalloc(sizeof(*g), GFP_KERNEL);
-			if (!g)
-				break;
-
-			g->sg = lio->glists_virt_base[i] +
-				(j * lio->glist_entry_size);
-
-			g->sg_dma_ptr = lio->glists_dma_base[i] +
-					(j * lio->glist_entry_size);
-
-			list_add_tail(&g->list, &lio->glist[i]);
-		}
-
-		if (j != lio->tx_qsize) {
-			delete_glists(lio);
-			return -ENOMEM;
-		}
-	}
-
-	return 0;
-}
-
 /**
  * \brief Print link information
  * @param netdev network device
@@ -1077,6 +912,9 @@ liquidio_probe(struct pci_dev *pdev,
 	/* set linux specific device pointer */
 	oct_dev->pci_dev = (void *)pdev;
 
+	oct_dev->subsystem_id = pdev->subsystem_vendor |
+		(pdev->subsystem_device << 16);
+
 	hs = &handshake[oct_dev->octeon_id];
 	init_completion(&hs->init);
 	init_completion(&hs->started);
@@ -1471,7 +1309,7 @@ static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
 
 	cleanup_rx_oom_poll_fn(netdev);
 
-	delete_glists(lio);
+	lio_delete_glists(lio);
 
 	free_netdev(netdev);
 
@@ -1686,7 +1524,7 @@ static void free_netsgbuf(void *buf)
 		i++;
 	}
 
-	iq = skb_iq(lio, skb);
+	iq = skb_iq(lio->oct_dev, skb);
 	spin_lock(&lio->glist_lock[iq]);
 	list_add_tail(&g->list, &lio->glist[iq]);
 	spin_unlock(&lio->glist_lock[iq]);
@@ -1729,7 +1567,7 @@ static void free_netsgbuf_with_resp(void *buf)
 		i++;
 	}
 
-	iq = skb_iq(lio, skb);
+	iq = skb_iq(lio->oct_dev, skb);
 
 	spin_lock(&lio->glist_lock[iq]);
 	list_add_tail(&g->list, &lio->glist[iq]);
@@ -1941,39 +1779,6 @@ static int load_firmware(struct octeon_device *oct)
 	return ret;
 }
 
-/**
- * \brief Callback for getting interface configuration
- * @param status status of request
- * @param buf pointer to resp structure
- */
-static void if_cfg_callback(struct octeon_device *oct,
-			    u32 status __attribute__((unused)),
-			    void *buf)
-{
-	struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
-	struct liquidio_if_cfg_resp *resp;
-	struct liquidio_if_cfg_context *ctx;
-
-	resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
-	ctx = (struct liquidio_if_cfg_context *)sc->ctxptr;
-
-	oct = lio_get_device(ctx->octeon_id);
-	if (resp->status)
-		dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: 0x%llx (0x%08x)\n",
-			CVM_CAST64(resp->status), status);
-	WRITE_ONCE(ctx->cond, 1);
-
-	snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s",
-		 resp->cfg_info.liquidio_firmware_version);
-
-	/* This barrier is required to be sure that the response has been
-	 * written fully before waking up the handler
-	 */
-	wmb();
-
-	wake_up_interruptible(&ctx->wc);
-}
-
 /**
  * \brief Poll routine for checking transmit queue status
  * @param work work_struct data structure
@@ -2049,11 +1854,6 @@ static int liquidio_open(struct net_device *netdev)
 
 	ifstate_set(lio, LIO_IFSTATE_RUNNING);
 
-	/* Ready for link status updates */
-	lio->intf_open = 1;
-
-	netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
-
 	if (OCTEON_CN23XX_PF(oct)) {
 		if (!oct->msix_on)
 			if (setup_tx_poll_fn(netdev))
@@ -2063,7 +1863,12 @@ static int liquidio_open(struct net_device *netdev)
 			return -1;
 	}
 
-	start_txqs(netdev);
+	netif_tx_start_all_queues(netdev);
+
+	/* Ready for link status updates */
+	lio->intf_open = 1;
+
+	netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
 
 	/* tell Octeon to start forwarding packets to host */
 	send_rx_ctrl_cmd(lio, 1);
@@ -2086,11 +1891,15 @@ static int liquidio_stop(struct net_device *netdev)
 
 	ifstate_reset(lio, LIO_IFSTATE_RUNNING);
 
-	netif_tx_disable(netdev);
+	/* Stop any link updates */
+	lio->intf_open = 0;
+
+	stop_txqs(netdev);
 
 	/* Inform that netif carrier is down */
 	netif_carrier_off(netdev);
-	lio->intf_open = 0;
+	netif_tx_disable(netdev);
+
 	lio->linfo.link.s.link_up = 0;
 	lio->link_changes++;
 
@@ -2252,14 +2061,11 @@ static int liquidio_set_mac(struct net_device *netdev, void *p)
 	return 0;
 }
 
-/**
- * \brief Net device get_stats
- * @param netdev network device
- */
-static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
+static void
+liquidio_get_stats64(struct net_device *netdev,
+		     struct rtnl_link_stats64 *lstats)
 {
 	struct lio *lio = GET_LIO(netdev);
-	struct net_device_stats *stats = &netdev->stats;
 	struct octeon_device *oct;
 	u64 pkts = 0, drop = 0, bytes = 0;
 	struct oct_droq_stats *oq_stats;
@@ -2269,7 +2075,7 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
 	oct = lio->oct_dev;
 
 	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
-		return stats;
+		return;
 
 	for (i = 0; i < oct->num_iqs; i++) {
 		iq_no = lio->linfo.txpciq[i].s.q_no;
@@ -2279,9 +2085,9 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
 		bytes += iq_stats->tx_tot_bytes;
 	}
 
-	stats->tx_packets = pkts;
-	stats->tx_bytes = bytes;
-	stats->tx_dropped = drop;
+	lstats->tx_packets = pkts;
+	lstats->tx_bytes = bytes;
+	lstats->tx_dropped = drop;
 
 	pkts = 0;
 	drop = 0;
@@ -2298,11 +2104,34 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
 		bytes += oq_stats->rx_bytes_received;
 	}
 
-	stats->rx_bytes = bytes;
-	stats->rx_packets = pkts;
-	stats->rx_dropped = drop;
-
-	return stats;
+	lstats->rx_bytes = bytes;
+	lstats->rx_packets = pkts;
+	lstats->rx_dropped = drop;
+
+	octnet_get_link_stats(netdev);
+	lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;
+	lstats->collisions = oct->link_stats.fromhost.total_collisions;
+
+	/* detailed rx_errors: */
+	lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;
+	/* recved pkt with crc error    */
+	lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;
+	/* recv'd frame alignment error */
+	lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;
+	/* recv'r fifo overrun */
+	lstats->rx_fifo_errors = oct->link_stats.fromwire.fifo_err;
+
+	lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors +
+		lstats->rx_frame_errors + lstats->rx_fifo_errors;
+
+	/* detailed tx_errors */
+	lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;
+	lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;
+	lstats->tx_fifo_errors = oct->link_stats.fromhost.fifo_err;
+
+	lstats->tx_errors = lstats->tx_aborted_errors +
+		lstats->tx_carrier_errors +
+		lstats->tx_fifo_errors;
 }
 
 /**
@@ -2510,7 +2339,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 	lio = GET_LIO(netdev);
 	oct = lio->oct_dev;
 
-	q_idx = skb_iq(lio, skb);
+	q_idx = skb_iq(oct, skb);
 	tag = q_idx;
 	iq_no = lio->linfo.txpciq[q_idx].s.q_no;
 
@@ -2585,6 +2414,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 		if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
 				__func__);
+			stats->tx_dmamap_fail++;
 			return NETDEV_TX_BUSY;
 		}
 
@@ -2602,7 +2432,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 
 		spin_lock(&lio->glist_lock[q_idx]);
 		g = (struct octnic_gather *)
-			list_delete_head(&lio->glist[q_idx]);
+			lio_list_delete_head(&lio->glist[q_idx]);
 		spin_unlock(&lio->glist_lock[q_idx]);
 
 		if (!g) {
@@ -2624,6 +2454,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 		if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
 				__func__);
+			stats->tx_dmamap_fail++;
 			return NETDEV_TX_BUSY;
 		}
 		add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
@@ -3324,11 +3155,36 @@ static const struct switchdev_ops lio_pf_switchdev_ops = {
 	.switchdev_port_attr_get = lio_pf_switchdev_attr_get,
 };
 
+static int liquidio_get_vf_stats(struct net_device *netdev, int vfidx,
+				 struct ifla_vf_stats *vf_stats)
+{
+	struct lio *lio = GET_LIO(netdev);
+	struct octeon_device *oct = lio->oct_dev;
+	struct oct_vf_stats stats;
+	int ret;
+
+	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
+		return -EINVAL;
+
+	memset(&stats, 0, sizeof(struct oct_vf_stats));
+	ret = cn23xx_get_vf_stats(oct, vfidx, &stats);
+	if (!ret) {
+		vf_stats->rx_packets = stats.rx_packets;
+		vf_stats->tx_packets = stats.tx_packets;
+		vf_stats->rx_bytes = stats.rx_bytes;
+		vf_stats->tx_bytes = stats.tx_bytes;
+		vf_stats->broadcast = stats.broadcast;
+		vf_stats->multicast = stats.multicast;
+	}
+
+	return ret;
+}
+
 static const struct net_device_ops lionetdevops = {
 	.ndo_open		= liquidio_open,
 	.ndo_stop		= liquidio_stop,
 	.ndo_start_xmit		= liquidio_xmit,
-	.ndo_get_stats		= liquidio_get_stats,
+	.ndo_get_stats64	= liquidio_get_stats64,
 	.ndo_set_mac_address	= liquidio_set_mac,
 	.ndo_set_rx_mode	= liquidio_set_mcast_list,
 	.ndo_tx_timeout		= liquidio_tx_timeout,
@@ -3346,6 +3202,7 @@ static const struct net_device_ops lionetdevops = {
 	.ndo_get_vf_config	= liquidio_get_vf_config,
 	.ndo_set_vf_trust	= liquidio_set_vf_trust,
 	.ndo_set_vf_link_state  = liquidio_set_vf_link_state,
+	.ndo_get_vf_stats	= liquidio_get_vf_stats,
 };
 
 /** \brief Entry point for the liquidio module
@@ -3448,6 +3305,7 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 	struct liquidio_if_cfg_resp *resp;
 	struct octdev_props *props;
 	int retval, num_iqueues, num_oqueues;
+	int max_num_queues = 0;
 	union oct_nic_if_cfg if_cfg;
 	unsigned int base_queue;
 	unsigned int gmx_port_id;
@@ -3528,9 +3386,9 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 					    OPCODE_NIC_IF_CFG, 0,
 					    if_cfg.u64, 0);
 
-		sc->callback = if_cfg_callback;
+		sc->callback = lio_if_cfg_callback;
 		sc->callback_arg = sc;
-		sc->wait_time = 3000;
+		sc->wait_time = LIO_IFCFG_WAIT_TIME;
 
 		retval = octeon_send_soft_command(octeon_dev, sc);
 		if (retval == IQ_SEND_FAILED) {
@@ -3584,11 +3442,20 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 				resp->cfg_info.oqmask);
 			goto setup_nic_dev_fail;
 		}
+
+		if (OCTEON_CN6XXX(octeon_dev)) {
+			max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
+								    cn6xxx));
+		} else if (OCTEON_CN23XX_PF(octeon_dev)) {
+			max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
+								    cn23xx_pf));
+		}
+
 		dev_dbg(&octeon_dev->pci_dev->dev,
-			"interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
+			"interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d max_num_queues: %d\n",
 			i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
-			num_iqueues, num_oqueues);
-		netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues);
+			num_iqueues, num_oqueues, max_num_queues);
+		netdev = alloc_etherdev_mq(LIO_SIZE, max_num_queues);
 
 		if (!netdev) {
 			dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
@@ -3603,6 +3470,20 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 		netdev->netdev_ops = &lionetdevops;
 		SWITCHDEV_SET_OPS(netdev, &lio_pf_switchdev_ops);
 
+		retval = netif_set_real_num_rx_queues(netdev, num_oqueues);
+		if (retval) {
+			dev_err(&octeon_dev->pci_dev->dev,
+				"setting real number rx failed\n");
+			goto setup_nic_dev_fail;
+		}
+
+		retval = netif_set_real_num_tx_queues(netdev, num_iqueues);
+		if (retval) {
+			dev_err(&octeon_dev->pci_dev->dev,
+				"setting real number tx failed\n");
+			goto setup_nic_dev_fail;
+		}
+
 		lio = GET_LIO(netdev);
 
 		memset(lio, 0, sizeof(struct lio));
@@ -3724,7 +3605,7 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 		lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
 		lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
 
-		if (setup_glists(octeon_dev, lio, num_iqueues)) {
+		if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
 			dev_err(&octeon_dev->pci_dev->dev,
 				"Gather list allocation failed\n");
 			goto setup_nic_dev_fail;
@@ -3786,6 +3667,23 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 			"NIC ifidx:%d Setup successful\n", i);
 
 		octeon_free_soft_command(octeon_dev, sc);
+
+		if (octeon_dev->subsystem_id ==
+			OCTEON_CN2350_25GB_SUBSYS_ID ||
+		    octeon_dev->subsystem_id ==
+			OCTEON_CN2360_25GB_SUBSYS_ID) {
+			liquidio_get_speed(lio);
+
+			if (octeon_dev->speed_setting == 0) {
+				octeon_dev->speed_setting = 25;
+				octeon_dev->no_speed_setting = 1;
+			}
+		} else {
+			octeon_dev->no_speed_setting = 1;
+			octeon_dev->speed_setting = 10;
+		}
+		octeon_dev->speed_boot = octeon_dev->speed_setting;
+
 	}
 
 	devlink = devlink_alloc(&liquidio_devlink_ops,
@@ -4223,7 +4121,9 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 		}
 		atomic_set(&octeon_dev->status, OCT_DEV_MBOX_SETUP_DONE);
 
-		if (octeon_allocate_ioq_vector(octeon_dev)) {
+		if (octeon_allocate_ioq_vector
+				(octeon_dev,
+				 octeon_dev->sriov_info.num_pf_rings)) {
 			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
 			return 1;
 		}
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
index f92dfa411de6..7fa0212873ac 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
@@ -69,30 +69,10 @@ union tx_info {
 	} s;
 };
 
-#define OCTNIC_MAX_SG  (MAX_SKB_FRAGS)
-
 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
 #define OCTNIC_GSO_MAX_SIZE \
 		(CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
 
-struct octnic_gather {
-	/* List manipulation. Next and prev pointers. */
-	struct list_head list;
-
-	/* Size of the gather component at sg in bytes. */
-	int sg_size;
-
-	/* Number of bytes that sg was adjusted to make it 8B-aligned. */
-	int adjust;
-
-	/* Gather component that can accommodate max sized fragment list
-	 * received from the IP layer.
-	 */
-	struct octeon_sg_entry *sg;
-
-	dma_addr_t sg_dma_ptr;
-};
-
 static int
 liquidio_vf_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
 static void liquidio_vf_remove(struct pci_dev *pdev);
@@ -284,142 +264,6 @@ static struct pci_driver liquidio_vf_pci_driver = {
 	.err_handler	= &liquidio_vf_err_handler,    /* For AER */
 };
 
-/**
- * Remove the node at the head of the list. The list would be empty at
- * the end of this call if there are no more nodes in the list.
- */
-static struct list_head *list_delete_head(struct list_head *root)
-{
-	struct list_head *node;
-
-	if ((root->prev == root) && (root->next == root))
-		node = NULL;
-	else
-		node = root->next;
-
-	if (node)
-		list_del(node);
-
-	return node;
-}
-
-/**
- * \brief Delete gather lists
- * @param lio per-network private data
- */
-static void delete_glists(struct lio *lio)
-{
-	struct octnic_gather *g;
-	int i;
-
-	kfree(lio->glist_lock);
-	lio->glist_lock = NULL;
-
-	if (!lio->glist)
-		return;
-
-	for (i = 0; i < lio->linfo.num_txpciq; i++) {
-		do {
-			g = (struct octnic_gather *)
-			    list_delete_head(&lio->glist[i]);
-			kfree(g);
-		} while (g);
-
-		if (lio->glists_virt_base && lio->glists_virt_base[i] &&
-		    lio->glists_dma_base && lio->glists_dma_base[i]) {
-			lio_dma_free(lio->oct_dev,
-				     lio->glist_entry_size * lio->tx_qsize,
-				     lio->glists_virt_base[i],
-				     lio->glists_dma_base[i]);
-		}
-	}
-
-	kfree(lio->glists_virt_base);
-	lio->glists_virt_base = NULL;
-
-	kfree(lio->glists_dma_base);
-	lio->glists_dma_base = NULL;
-
-	kfree(lio->glist);
-	lio->glist = NULL;
-}
-
-/**
- * \brief Setup gather lists
- * @param lio per-network private data
- */
-static int setup_glists(struct lio *lio, int num_iqs)
-{
-	struct octnic_gather *g;
-	int i, j;
-
-	lio->glist_lock =
-	    kzalloc(sizeof(*lio->glist_lock) * num_iqs, GFP_KERNEL);
-	if (!lio->glist_lock)
-		return -ENOMEM;
-
-	lio->glist =
-	    kzalloc(sizeof(*lio->glist) * num_iqs, GFP_KERNEL);
-	if (!lio->glist) {
-		kfree(lio->glist_lock);
-		lio->glist_lock = NULL;
-		return -ENOMEM;
-	}
-
-	lio->glist_entry_size =
-		ROUNDUP8((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * OCT_SG_ENTRY_SIZE);
-
-	/* allocate memory to store virtual and dma base address of
-	 * per glist consistent memory
-	 */
-	lio->glists_virt_base = kcalloc(num_iqs, sizeof(*lio->glists_virt_base),
-					GFP_KERNEL);
-	lio->glists_dma_base = kcalloc(num_iqs, sizeof(*lio->glists_dma_base),
-				       GFP_KERNEL);
-
-	if (!lio->glists_virt_base || !lio->glists_dma_base) {
-		delete_glists(lio);
-		return -ENOMEM;
-	}
-
-	for (i = 0; i < num_iqs; i++) {
-		spin_lock_init(&lio->glist_lock[i]);
-
-		INIT_LIST_HEAD(&lio->glist[i]);
-
-		lio->glists_virt_base[i] =
-			lio_dma_alloc(lio->oct_dev,
-				      lio->glist_entry_size * lio->tx_qsize,
-				      &lio->glists_dma_base[i]);
-
-		if (!lio->glists_virt_base[i]) {
-			delete_glists(lio);
-			return -ENOMEM;
-		}
-
-		for (j = 0; j < lio->tx_qsize; j++) {
-			g = kzalloc(sizeof(*g), GFP_KERNEL);
-			if (!g)
-				break;
-
-			g->sg = lio->glists_virt_base[i] +
-				(j * lio->glist_entry_size);
-
-			g->sg_dma_ptr = lio->glists_dma_base[i] +
-					(j * lio->glist_entry_size);
-
-			list_add_tail(&g->list, &lio->glist[i]);
-		}
-
-		if (j != lio->tx_qsize) {
-			delete_glists(lio);
-			return -ENOMEM;
-		}
-	}
-
-	return 0;
-}
-
 /**
  * \brief Print link information
  * @param netdev network device
@@ -567,6 +411,9 @@ liquidio_vf_probe(struct pci_dev *pdev,
 	/* set linux specific device pointer */
 	oct_dev->pci_dev = pdev;
 
+	oct_dev->subsystem_id = pdev->subsystem_vendor |
+		(pdev->subsystem_device << 16);
+
 	if (octeon_device_init(oct_dev)) {
 		liquidio_vf_remove(pdev);
 		return -ENOMEM;
@@ -856,7 +703,7 @@ static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
 
 	cleanup_link_status_change_wq(netdev);
 
-	delete_glists(lio);
+	lio_delete_glists(lio);
 
 	free_netdev(netdev);
 
@@ -1005,7 +852,7 @@ static void free_netsgbuf(void *buf)
 		i++;
 	}
 
-	iq = skb_iq(lio, skb);
+	iq = skb_iq(lio->oct_dev, skb);
 
 	spin_lock(&lio->glist_lock[iq]);
 	list_add_tail(&g->list, &lio->glist[iq]);
@@ -1049,7 +896,7 @@ static void free_netsgbuf_with_resp(void *buf)
 		i++;
 	}
 
-	iq = skb_iq(lio, skb);
+	iq = skb_iq(lio->oct_dev, skb);
 
 	spin_lock(&lio->glist_lock[iq]);
 	list_add_tail(&g->list, &lio->glist[iq]);
@@ -1058,38 +905,6 @@ static void free_netsgbuf_with_resp(void *buf)
 	/* Don't free the skb yet */
 }
 
-/**
- * \brief Callback for getting interface configuration
- * @param status status of request
- * @param buf pointer to resp structure
- */
-static void if_cfg_callback(struct octeon_device *oct,
-			    u32 status __attribute__((unused)), void *buf)
-{
-	struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
-	struct liquidio_if_cfg_context *ctx;
-	struct liquidio_if_cfg_resp *resp;
-
-	resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
-	ctx = (struct liquidio_if_cfg_context *)sc->ctxptr;
-
-	oct = lio_get_device(ctx->octeon_id);
-	if (resp->status)
-		dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n",
-			CVM_CAST64(resp->status));
-	WRITE_ONCE(ctx->cond, 1);
-
-	snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s",
-		 resp->cfg_info.liquidio_firmware_version);
-
-	/* This barrier is required to be sure that the response has been
-	 * written fully before waking up the handler
-	 */
-	wmb();
-
-	wake_up_interruptible(&ctx->wc);
-}
-
 /**
  * \brief Net device open for LiquidIO
  * @param netdev network device
@@ -1336,24 +1151,21 @@ static int liquidio_set_mac(struct net_device *netdev, void *p)
 	return 0;
 }
 
-/**
- * \brief Net device get_stats
- * @param netdev network device
- */
-static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
+static void
+liquidio_get_stats64(struct net_device *netdev,
+		     struct rtnl_link_stats64 *lstats)
 {
 	struct lio *lio = GET_LIO(netdev);
-	struct net_device_stats *stats = &netdev->stats;
+	struct octeon_device *oct;
 	u64 pkts = 0, drop = 0, bytes = 0;
 	struct oct_droq_stats *oq_stats;
 	struct oct_iq_stats *iq_stats;
-	struct octeon_device *oct;
 	int i, iq_no, oq_no;
 
 	oct = lio->oct_dev;
 
 	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
-		return stats;
+		return;
 
 	for (i = 0; i < oct->num_iqs; i++) {
 		iq_no = lio->linfo.txpciq[i].s.q_no;
@@ -1363,9 +1175,9 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
 		bytes += iq_stats->tx_tot_bytes;
 	}
 
-	stats->tx_packets = pkts;
-	stats->tx_bytes = bytes;
-	stats->tx_dropped = drop;
+	lstats->tx_packets = pkts;
+	lstats->tx_bytes = bytes;
+	lstats->tx_dropped = drop;
 
 	pkts = 0;
 	drop = 0;
@@ -1382,11 +1194,29 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
 		bytes += oq_stats->rx_bytes_received;
 	}
 
-	stats->rx_bytes = bytes;
-	stats->rx_packets = pkts;
-	stats->rx_dropped = drop;
+	lstats->rx_bytes = bytes;
+	lstats->rx_packets = pkts;
+	lstats->rx_dropped = drop;
+
+	octnet_get_link_stats(netdev);
+	lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;
+
+	/* detailed rx_errors: */
+	lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;
+	/* recved pkt with crc error */
+	lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;
+	/* recv'd frame alignment error */
+	lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;
 
-	return stats;
+	lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors +
+			    lstats->rx_frame_errors;
+
+	/* detailed tx_errors */
+	lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;
+	lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;
+
+	lstats->tx_errors = lstats->tx_aborted_errors +
+		lstats->tx_carrier_errors;
 }
 
 /**
@@ -1580,7 +1410,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 	lio = GET_LIO(netdev);
 	oct = lio->oct_dev;
 
-	q_idx = skb_iq(lio, skb);
+	q_idx = skb_iq(lio->oct_dev, skb);
 	tag = q_idx;
 	iq_no = lio->linfo.txpciq[q_idx].s.q_no;
 
@@ -1661,8 +1491,8 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 		int i, frags;
 
 		spin_lock(&lio->glist_lock[q_idx]);
-		g = (struct octnic_gather *)list_delete_head(
-		    &lio->glist[q_idx]);
+		g = (struct octnic_gather *)
+			lio_list_delete_head(&lio->glist[q_idx]);
 		spin_unlock(&lio->glist_lock[q_idx]);
 
 		if (!g) {
@@ -2034,7 +1864,7 @@ static const struct net_device_ops lionetdevops = {
 	.ndo_open		= liquidio_open,
 	.ndo_stop		= liquidio_stop,
 	.ndo_start_xmit		= liquidio_xmit,
-	.ndo_get_stats		= liquidio_get_stats,
+	.ndo_get_stats64	= liquidio_get_stats64,
 	.ndo_set_mac_address	= liquidio_set_mac,
 	.ndo_set_rx_mode	= liquidio_set_mcast_list,
 	.ndo_tx_timeout		= liquidio_tx_timeout,
@@ -2156,7 +1986,7 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 					    OPCODE_NIC_IF_CFG, 0, if_cfg.u64,
 					    0);
 
-		sc->callback = if_cfg_callback;
+		sc->callback = lio_if_cfg_callback;
 		sc->callback_arg = sc;
 		sc->wait_time = 5000;
 
@@ -2273,6 +2103,7 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 		netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
 
 		netdev->hw_features = lio->dev_capability;
+		netdev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
 
 		/* MTU range: 68 - 16000 */
 		netdev->min_mtu = LIO_MIN_MTU_SIZE;
@@ -2321,7 +2152,7 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 		lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
 		lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
 
-		if (setup_glists(lio, num_iqueues)) {
+		if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
 			dev_err(&octeon_dev->pci_dev->dev,
 				"Gather list allocation failed\n");
 			goto setup_nic_dev_fail;
@@ -2371,6 +2202,8 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 			"NIC ifidx:%d Setup successful\n", i);
 
 		octeon_free_soft_command(octeon_dev, sc);
+
+		octeon_dev->no_speed_setting = 1;
 	}
 
 	return 0;
@@ -2512,7 +2345,7 @@ static int octeon_device_init(struct octeon_device *oct)
 	}
 	atomic_set(&oct->status, OCT_DEV_MBOX_SETUP_DONE);
 
-	if (octeon_allocate_ioq_vector(oct)) {
+	if (octeon_allocate_ioq_vector(oct, oct->sriov_info.rings_per_vf)) {
 		dev_err(&oct->pci_dev->dev, "ioq vector allocation failed\n");
 		return 1;
 	}
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_vf_rep.c b/drivers/net/ethernet/cavium/liquidio/lio_vf_rep.c
index 2adafa366d3f..ddd7431579f4 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_vf_rep.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_vf_rep.c
@@ -201,13 +201,14 @@ lio_vf_rep_get_stats64(struct net_device *dev,
 {
 	struct lio_vf_rep_desc *vf_rep = netdev_priv(dev);
 
-	stats64->tx_packets = vf_rep->stats.tx_packets;
-	stats64->tx_bytes   = vf_rep->stats.tx_bytes;
-	stats64->tx_dropped = vf_rep->stats.tx_dropped;
-
-	stats64->rx_packets = vf_rep->stats.rx_packets;
-	stats64->rx_bytes   = vf_rep->stats.rx_bytes;
-	stats64->rx_dropped = vf_rep->stats.rx_dropped;
+	/* Swap tx and rx stats as VF rep is a switch port */
+	stats64->tx_packets = vf_rep->stats.rx_packets;
+	stats64->tx_bytes   = vf_rep->stats.rx_bytes;
+	stats64->tx_dropped = vf_rep->stats.rx_dropped;
+
+	stats64->rx_packets = vf_rep->stats.tx_packets;
+	stats64->rx_bytes   = vf_rep->stats.tx_bytes;
+	stats64->rx_dropped = vf_rep->stats.tx_dropped;
 }
 
 static int
diff --git a/drivers/net/ethernet/cavium/liquidio/liquidio_common.h b/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
index 75eea83c7cc6..690424b6781a 100644
--- a/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
+++ b/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
@@ -28,7 +28,7 @@
 #define LIQUIDIO_PACKAGE ""
 #define LIQUIDIO_BASE_MAJOR_VERSION 1
 #define LIQUIDIO_BASE_MINOR_VERSION 7
-#define LIQUIDIO_BASE_MICRO_VERSION 0
+#define LIQUIDIO_BASE_MICRO_VERSION 2
 #define LIQUIDIO_BASE_VERSION   __stringify(LIQUIDIO_BASE_MAJOR_VERSION) "." \
 				__stringify(LIQUIDIO_BASE_MINOR_VERSION)
 #define LIQUIDIO_MICRO_VERSION  "." __stringify(LIQUIDIO_BASE_MICRO_VERSION)
@@ -84,6 +84,7 @@ enum octeon_tag_type {
 #define OPCODE_NIC_IF_CFG              0x09
 #define OPCODE_NIC_VF_DRV_NOTICE       0x0A
 #define OPCODE_NIC_INTRMOD_PARAMS      0x0B
+#define OPCODE_NIC_QCOUNT_UPDATE       0x12
 #define OPCODE_NIC_SET_TRUSTED_VF	0x13
 #define OPCODE_NIC_SYNC_OCTEON_TIME	0x14
 #define VF_DRV_LOADED                  1
@@ -92,6 +93,7 @@ enum octeon_tag_type {
 
 #define OPCODE_NIC_VF_REP_PKT          0x15
 #define OPCODE_NIC_VF_REP_CMD          0x16
+#define OPCODE_NIC_UBOOT_CTL           0x17
 
 #define CORE_DRV_TEST_SCATTER_OP    0xFFF5
 
@@ -248,6 +250,9 @@ static inline void add_sg_size(struct octeon_sg_entry *sg_entry,
 #define   OCTNET_CMD_VLAN_FILTER_ENABLE 0x1
 #define   OCTNET_CMD_VLAN_FILTER_DISABLE 0x0
 
+#define   SEAPI_CMD_SPEED_SET           0x2
+#define   SEAPI_CMD_SPEED_GET           0x3
+
 #define   LIO_CMD_WAIT_TM 100
 
 /* RX(packets coming from wire) Checksum verification flags */
@@ -779,23 +784,32 @@ struct liquidio_if_cfg_info {
 /** Stats for each NIC port in RX direction. */
 struct nic_rx_stats {
 	/* link-level stats */
-	u64 total_rcvd;
-	u64 bytes_rcvd;
-	u64 total_bcst;
-	u64 total_mcst;
-	u64 runts;
-	u64 ctl_rcvd;
-	u64 fifo_err;      /* Accounts for over/under-run of buffers */
-	u64 dmac_drop;
-	u64 fcs_err;
-	u64 jabber_err;
-	u64 l2_err;
-	u64 frame_err;
+	u64 total_rcvd;		/* Received packets */
+	u64 bytes_rcvd;		/* Octets of received packets */
+	u64 total_bcst;		/* Number of non-dropped L2 broadcast packets */
+	u64 total_mcst;		/* Number of non-dropped L2 multicast packets */
+	u64 runts;		/* Packets shorter than allowed */
+	u64 ctl_rcvd;		/* Received PAUSE packets */
+	u64 fifo_err;		/* Packets dropped due to RX FIFO full */
+	u64 dmac_drop;		/* Packets dropped by the DMAC filter */
+	u64 fcs_err;		/* Sum of fragment, overrun, and FCS errors */
+	u64 jabber_err;		/* Packets larger than allowed */
+	u64 l2_err;		/* Sum of DMA, parity, PCAM access, no memory,
+				 * buffer overflow, malformed L2 header or
+				 * length, oversize errors
+				 **/
+	u64 frame_err;		/* Sum of IPv4 and L4 checksum errors */
+	u64 red_drops;		/* Packets dropped by RED due to buffer
+				 * exhaustion
+				 **/
 
 	/* firmware stats */
 	u64 fw_total_rcvd;
 	u64 fw_total_fwd;
 	u64 fw_total_fwd_bytes;
+	u64 fw_total_mcast;
+	u64 fw_total_bcast;
+
 	u64 fw_err_pko;
 	u64 fw_err_link;
 	u64 fw_err_drop;
@@ -806,11 +820,11 @@ struct nic_rx_stats {
 	u64 fw_lro_pkts;   /* Number of packets that are LROed      */
 	u64 fw_lro_octs;   /* Number of octets that are LROed       */
 	u64 fw_total_lro;  /* Number of LRO packets formed          */
-	u64 fw_lro_aborts; /* Number of times lRO of packet aborted */
+	u64 fw_lro_aborts; /* Number of times LRO of packet aborted */
 	u64 fw_lro_aborts_port;
 	u64 fw_lro_aborts_seq;
 	u64 fw_lro_aborts_tsval;
-	u64 fw_lro_aborts_timer;
+	u64 fw_lro_aborts_timer;	/* Timer setting error */
 	/* intrmod: packet forward rate */
 	u64 fwd_rate;
 };
@@ -818,23 +832,42 @@ struct nic_rx_stats {
 /** Stats for each NIC port in RX direction. */
 struct nic_tx_stats {
 	/* link-level stats */
-	u64 total_pkts_sent;
-	u64 total_bytes_sent;
-	u64 mcast_pkts_sent;
-	u64 bcast_pkts_sent;
-	u64 ctl_sent;
-	u64 one_collision_sent;   /* Packets sent after one collision*/
-	u64 multi_collision_sent; /* Packets sent after multiple collision*/
-	u64 max_collision_fail;   /* Packets not sent due to max collisions */
-	u64 max_deferral_fail;   /* Packets not sent due to max deferrals */
-	u64 fifo_err;       /* Accounts for over/under-run of buffers */
-	u64 runts;
-	u64 total_collisions; /* Total number of collisions detected */
+	u64 total_pkts_sent;		/* Total frames sent on the interface */
+	u64 total_bytes_sent;		/* Total octets sent on the interface */
+	u64 mcast_pkts_sent;		/* Packets sent to the multicast DMAC */
+	u64 bcast_pkts_sent;		/* Packets sent to a broadcast DMAC */
+	u64 ctl_sent;			/* Control/PAUSE packets sent */
+	u64 one_collision_sent;		/* Packets sent that experienced a
+					 * single collision before successful
+					 * transmission
+					 **/
+	u64 multi_collision_sent;	/* Packets sent that experienced
+					 * multiple collisions before successful
+					 * transmission
+					 **/
+	u64 max_collision_fail;		/* Packets dropped due to excessive
+					 * collisions
+					 **/
+	u64 max_deferral_fail;		/* Packets not sent due to max
+					 * deferrals
+					 **/
+	u64 fifo_err;			/* Packets sent that experienced a
+					 * transmit underflow and were
+					 * truncated
+					 **/
+	u64 runts;			/* Packets sent with an octet count
+					 * lessthan 64
+					 **/
+	u64 total_collisions;		/* Packets dropped due to excessive
+					 * collisions
+					 **/
 
 	/* firmware stats */
 	u64 fw_total_sent;
 	u64 fw_total_fwd;
 	u64 fw_total_fwd_bytes;
+	u64 fw_total_mcast_sent;
+	u64 fw_total_bcast_sent;
 	u64 fw_err_pko;
 	u64 fw_err_link;
 	u64 fw_err_drop;
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_device.c b/drivers/net/ethernet/cavium/liquidio/octeon_device.c
index f38abf626412..f878a552fef3 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_device.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_device.c
@@ -824,23 +824,18 @@ int octeon_deregister_device(struct octeon_device *oct)
 }
 
 int
-octeon_allocate_ioq_vector(struct octeon_device  *oct)
+octeon_allocate_ioq_vector(struct octeon_device *oct, u32 num_ioqs)
 {
-	int i, num_ioqs = 0;
 	struct octeon_ioq_vector *ioq_vector;
 	int cpu_num;
 	int size;
-
-	if (OCTEON_CN23XX_PF(oct))
-		num_ioqs = oct->sriov_info.num_pf_rings;
-	else if (OCTEON_CN23XX_VF(oct))
-		num_ioqs = oct->sriov_info.rings_per_vf;
+	int i;
 
 	size = sizeof(struct octeon_ioq_vector) * num_ioqs;
 
 	oct->ioq_vector = vzalloc(size);
 	if (!oct->ioq_vector)
-		return 1;
+		return -1;
 	for (i = 0; i < num_ioqs; i++) {
 		ioq_vector		= &oct->ioq_vector[i];
 		ioq_vector->oct_dev	= oct;
@@ -856,6 +851,7 @@ octeon_allocate_ioq_vector(struct octeon_device  *oct)
 		else
 			ioq_vector->ioq_num	= i;
 	}
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_device.h b/drivers/net/ethernet/cavium/liquidio/octeon_device.h
index 91937cc5c1d7..94a4ed88d618 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_device.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_device.h
@@ -43,6 +43,13 @@
 #define  OCTEON_CN23XX_REV_1_1        0x01
 #define  OCTEON_CN23XX_REV_2_0        0x80
 
+/**SubsystemId for the chips */
+#define	 OCTEON_CN2350_10GB_SUBSYS_ID_1	0X3177d
+#define	 OCTEON_CN2350_10GB_SUBSYS_ID_2	0X4177d
+#define	 OCTEON_CN2360_10GB_SUBSYS_ID	0X5177d
+#define	 OCTEON_CN2350_25GB_SUBSYS_ID	0X7177d
+#define	 OCTEON_CN2360_25GB_SUBSYS_ID	0X6177d
+
 /** Endian-swap modes supported by Octeon. */
 enum octeon_pci_swap_mode {
 	OCTEON_PCI_PASSTHROUGH = 0,
@@ -430,6 +437,8 @@ struct octeon_device {
 
 	u16 rev_id;
 
+	u32 subsystem_id;
+
 	u16 pf_num;
 
 	u16 vf_num;
@@ -584,6 +593,11 @@ struct octeon_device {
 	struct lio_vf_rep_list vf_rep_list;
 	struct devlink *devlink;
 	enum devlink_eswitch_mode eswitch_mode;
+
+	/* for 25G NIC speed change */
+	u8  speed_boot;
+	u8  speed_setting;
+	u8  no_speed_setting;
 };
 
 #define  OCT_DRV_ONLINE 1
@@ -867,7 +881,7 @@ void *oct_get_config_info(struct octeon_device *oct, u16 card_type);
 struct octeon_config *octeon_get_conf(struct octeon_device *oct);
 
 void octeon_free_ioq_vector(struct octeon_device *oct);
-int octeon_allocate_ioq_vector(struct octeon_device  *oct);
+int octeon_allocate_ioq_vector(struct octeon_device  *oct, u32 num_ioqs);
 void lio_enable_irq(struct octeon_droq *droq, struct octeon_instr_queue *iq);
 
 /* LiquidIO driver pivate flags */
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
index 81c987682941..5fed7b63223e 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
@@ -62,8 +62,8 @@ struct oct_iq_stats {
 	u64 tx_tot_bytes;/**< Total count of bytes sento to network. */
 	u64 tx_gso;  /* count of tso */
 	u64 tx_vxlan; /* tunnel */
-	u64 tx_dmamap_fail;
-	u64 tx_restart;
+	u64 tx_dmamap_fail; /* Number of times dma mapping failed */
+	u64 tx_restart; /* Number of times this queue restarted */
 };
 
 #define OCT_IQ_STATS_SIZE   (sizeof(struct oct_iq_stats))
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c b/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c
index 28e74ee23ff8..021d99cd1665 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c
@@ -24,6 +24,7 @@
 #include "octeon_device.h"
 #include "octeon_main.h"
 #include "octeon_mailbox.h"
+#include "cn23xx_pf_device.h"
 
 /**
  * octeon_mbox_read:
@@ -205,6 +206,26 @@ int octeon_mbox_write(struct octeon_device *oct,
 	return ret;
 }
 
+static void get_vf_stats(struct octeon_device *oct,
+			 struct oct_vf_stats *stats)
+{
+	int i;
+
+	for (i = 0; i < oct->num_iqs; i++) {
+		if (!oct->instr_queue[i])
+			continue;
+		stats->tx_packets += oct->instr_queue[i]->stats.tx_done;
+		stats->tx_bytes += oct->instr_queue[i]->stats.tx_tot_bytes;
+	}
+
+	for (i = 0; i < oct->num_oqs; i++) {
+		if (!oct->droq[i])
+			continue;
+		stats->rx_packets += oct->droq[i]->stats.rx_pkts_received;
+		stats->rx_bytes += oct->droq[i]->stats.rx_bytes_received;
+	}
+}
+
 /**
  * octeon_mbox_process_cmd:
  * @mbox: Pointer mailbox
@@ -250,6 +271,15 @@ static int octeon_mbox_process_cmd(struct octeon_mbox *mbox,
 						     mbox_cmd->msg.s.params);
 		break;
 
+	case OCTEON_GET_VF_STATS:
+		dev_dbg(&oct->pci_dev->dev, "Got VF stats request. Sending data back\n");
+		mbox_cmd->msg.s.type = OCTEON_MBOX_RESPONSE;
+		mbox_cmd->msg.s.resp_needed = 1;
+		mbox_cmd->msg.s.len = 1 +
+			sizeof(struct oct_vf_stats) / sizeof(u64);
+		get_vf_stats(oct, (struct oct_vf_stats *)mbox_cmd->data);
+		octeon_mbox_write(oct, mbox_cmd);
+		break;
 	default:
 		break;
 	}
@@ -322,3 +352,25 @@ int octeon_mbox_process_message(struct octeon_mbox *mbox)
 
 	return 0;
 }
+
+int octeon_mbox_cancel(struct octeon_device *oct, int q_no)
+{
+	struct octeon_mbox *mbox = oct->mbox[q_no];
+	struct octeon_mbox_cmd *mbox_cmd;
+	unsigned long flags = 0;
+
+	spin_lock_irqsave(&mbox->lock, flags);
+	mbox_cmd = &mbox->mbox_resp;
+
+	if (!(mbox->state & OCTEON_MBOX_STATE_RESPONSE_PENDING)) {
+		spin_unlock_irqrestore(&mbox->lock, flags);
+		return 1;
+	}
+
+	mbox->state = OCTEON_MBOX_STATE_IDLE;
+	memset(mbox_cmd, 0, sizeof(*mbox_cmd));
+	writeq(OCTEON_PFVFSIG, mbox->mbox_read_reg);
+	spin_unlock_irqrestore(&mbox->lock, flags);
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.h b/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.h
index 1def22afeff1..d92bd7e16477 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_mailbox.h
@@ -25,6 +25,7 @@
 #define OCTEON_VF_ACTIVE		0x1
 #define OCTEON_VF_FLR_REQUEST		0x2
 #define OCTEON_PF_CHANGED_VF_MACADDR	0x4
+#define OCTEON_GET_VF_STATS		0x8
 
 /*Macro for Read acknowldgement*/
 #define OCTEON_PFVFACK			0xffffffffffffffffULL
@@ -107,9 +108,15 @@ struct octeon_mbox {
 
 };
 
+struct oct_vf_stats_ctx {
+	atomic_t status;
+	struct oct_vf_stats *stats;
+};
+
 int octeon_mbox_read(struct octeon_mbox *mbox);
 int octeon_mbox_write(struct octeon_device *oct,
 		      struct octeon_mbox_cmd *mbox_cmd);
 int octeon_mbox_process_message(struct octeon_mbox *mbox);
+int octeon_mbox_cancel(struct octeon_device *oct, int q_no);
 
 #endif
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_network.h b/drivers/net/ethernet/cavium/liquidio/octeon_network.h
index 4069710796a8..d7a3916fe877 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_network.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_network.h
@@ -47,6 +47,29 @@ struct liquidio_if_cfg_resp {
 	u64 status;
 };
 
+#define LIO_IFCFG_WAIT_TIME    3000 /* In milli seconds */
+
+/* Structure of a node in list of gather components maintained by
+ * NIC driver for each network device.
+ */
+struct octnic_gather {
+	/* List manipulation. Next and prev pointers. */
+	struct list_head list;
+
+	/* Size of the gather component at sg in bytes. */
+	int sg_size;
+
+	/* Number of bytes that sg was adjusted to make it 8B-aligned. */
+	int adjust;
+
+	/* Gather component that can accommodate max sized fragment list
+	 * received from the IP layer.
+	 */
+	struct octeon_sg_entry *sg;
+
+	dma_addr_t sg_dma_ptr;
+};
+
 struct oct_nic_stats_resp {
 	u64     rh;
 	struct oct_link_stats stats;
@@ -58,6 +81,18 @@ struct oct_nic_stats_ctrl {
 	struct net_device *netdev;
 };
 
+struct oct_nic_seapi_resp {
+	u64 rh;
+	u32 speed;
+	u64 status;
+};
+
+struct liquidio_nic_seapi_ctl_context {
+	int octeon_id;
+	u32 status;
+	struct completion complete;
+};
+
 /** LiquidIO per-interface network private data */
 struct lio {
 	/** State of the interface. Rx/Tx happens only in the RUNNING state.  */
@@ -157,7 +192,7 @@ struct lio {
 #define LIO_SIZE         (sizeof(struct lio))
 #define GET_LIO(netdev)  ((struct lio *)netdev_priv(netdev))
 
-#define LIO_MAX_CORES                12
+#define LIO_MAX_CORES                16
 
 /**
  * \brief Enable or disable feature
@@ -190,6 +225,8 @@ irqreturn_t liquidio_msix_intr_handler(int irq __attribute__((unused)),
 
 int octeon_setup_interrupt(struct octeon_device *oct, u32 num_ioqs);
 
+int octnet_get_link_stats(struct net_device *netdev);
+
 int lio_wait_for_clean_oq(struct octeon_device *oct);
 /**
  * \brief Register ethtool operations
@@ -197,6 +234,17 @@ int lio_wait_for_clean_oq(struct octeon_device *oct);
  */
 void liquidio_set_ethtool_ops(struct net_device *netdev);
 
+void lio_if_cfg_callback(struct octeon_device *oct,
+			 u32 status __attribute__((unused)),
+			 void *buf);
+
+void lio_delete_glists(struct lio *lio);
+
+int lio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_qs);
+
+int liquidio_get_speed(struct lio *lio);
+int liquidio_set_speed(struct lio *lio, int speed);
+
 /**
  * \brief Net device change_mtu
  * @param netdev network device
@@ -515,7 +563,7 @@ static inline void stop_txqs(struct net_device *netdev)
 {
 	int i;
 
-	for (i = 0; i < netdev->num_tx_queues; i++)
+	for (i = 0; i < netdev->real_num_tx_queues; i++)
 		netif_stop_subqueue(netdev, i);
 }
 
@@ -528,7 +576,7 @@ static inline void wake_txqs(struct net_device *netdev)
 	struct lio *lio = GET_LIO(netdev);
 	int i, qno;
 
-	for (i = 0; i < netdev->num_tx_queues; i++) {
+	for (i = 0; i < netdev->real_num_tx_queues; i++) {
 		qno = lio->linfo.txpciq[i % lio->oct_dev->num_iqs].s.q_no;
 
 		if (__netif_subqueue_stopped(netdev, i)) {
@@ -549,14 +597,33 @@ static inline void start_txqs(struct net_device *netdev)
 	int i;
 
 	if (lio->linfo.link.s.link_up) {
-		for (i = 0; i < netdev->num_tx_queues; i++)
+		for (i = 0; i < netdev->real_num_tx_queues; i++)
 			netif_start_subqueue(netdev, i);
 	}
 }
 
-static inline int skb_iq(struct lio *lio, struct sk_buff *skb)
+static inline int skb_iq(struct octeon_device *oct, struct sk_buff *skb)
 {
-	return skb->queue_mapping % lio->linfo.num_txpciq;
+	return skb->queue_mapping % oct->num_iqs;
+}
+
+/**
+ * Remove the node at the head of the list. The list would be empty at
+ * the end of this call if there are no more nodes in the list.
+ */
+static inline struct list_head *lio_list_delete_head(struct list_head *root)
+{
+	struct list_head *node;
+
+	if (root->prev == root && root->next == root)
+		node = NULL;
+	else
+		node = root->next;
+
+	if (node)
+		list_del(node);
+
+	return node;
 }
 
 #endif
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c
index 707db3304396..7135db45927e 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c
@@ -538,9 +538,9 @@ static inline bool nicvf_xdp_rx(struct nicvf *nic, struct bpf_prog *prog,
 	action = bpf_prog_run_xdp(prog, &xdp);
 	rcu_read_unlock();
 
+	len = xdp.data_end - xdp.data;
 	/* Check if XDP program has changed headers */
 	if (orig_data != xdp.data) {
-		len = xdp.data_end - xdp.data;
 		offset = orig_data - xdp.data;
 		dma_addr -= offset;
 	}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h
index dc25066c59a1..3c5057868ab3 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h
@@ -62,6 +62,18 @@ struct cudbg_hw_sched {
 	u32 map;
 };
 
+#define SGE_QBASE_DATA_REG_NUM 4
+
+struct sge_qbase_reg_field {
+	u32 reg_addr;
+	u32 reg_data[SGE_QBASE_DATA_REG_NUM];
+	/* Max supported PFs */
+	u32 pf_data_value[PCIE_FW_MASTER_M + 1][SGE_QBASE_DATA_REG_NUM];
+	/* Max supported VFs */
+	u32 vf_data_value[T6_VF_M + 1][SGE_QBASE_DATA_REG_NUM];
+	u32 vfcount; /* Actual number of max vfs in current configuration */
+};
+
 struct ireg_field {
 	u32 ireg_addr;
 	u32 ireg_data;
@@ -235,6 +247,9 @@ struct cudbg_vpd_data {
 };
 
 #define CUDBG_MAX_TCAM_TID 0x800
+#define CUDBG_T6_CLIP 1536
+#define CUDBG_MAX_TID_COMP_EN 6144
+#define CUDBG_MAX_TID_COMP_DIS 3072
 
 enum cudbg_le_entry_types {
 	LE_ET_UNKNOWN = 0,
@@ -354,6 +369,11 @@ static const u32 t5_sge_dbg_index_array[2][IREG_NUM_ELEM] = {
 	{0x10cc, 0x10d4, 0x0, 16},
 };
 
+static const u32 t6_sge_qbase_index_array[] = {
+	/* 1 addr reg SGE_QBASE_INDEX and 4 data reg SGE_QBASE_MAP[0-3] */
+	0x1250, 0x1240, 0x1244, 0x1248, 0x124c,
+};
+
 static const u32 t5_pcie_pdbg_array[][IREG_NUM_ELEM] = {
 	{0x5a04, 0x5a0c, 0x00, 0x20}, /* t5_pcie_pdbg_regs_00_to_20 */
 	{0x5a04, 0x5a0c, 0x21, 0x20}, /* t5_pcie_pdbg_regs_21_to_40 */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h
index 8568a51f6414..215fe6260fd7 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h
@@ -24,6 +24,7 @@
 #define CUDBG_STATUS_NOT_IMPLEMENTED -28
 #define CUDBG_SYSTEM_ERROR -29
 #define CUDBG_STATUS_CCLK_NOT_DEFINED -32
+#define CUDBG_STATUS_PARTIAL_DATA -41
 
 #define CUDBG_MAJOR_VERSION 1
 #define CUDBG_MINOR_VERSION 14
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c
index 9da6f57901a9..0afcfe99bff3 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c
@@ -1339,16 +1339,39 @@ int cudbg_collect_tp_indirect(struct cudbg_init *pdbg_init,
 	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
+static void cudbg_read_sge_qbase_indirect_reg(struct adapter *padap,
+					      struct sge_qbase_reg_field *qbase,
+					      u32 func, bool is_pf)
+{
+	u32 *buff, i;
+
+	if (is_pf) {
+		buff = qbase->pf_data_value[func];
+	} else {
+		buff = qbase->vf_data_value[func];
+		/* In SGE_QBASE_INDEX,
+		 * Entries 0->7 are PF0->7, Entries 8->263 are VFID0->256.
+		 */
+		func += 8;
+	}
+
+	t4_write_reg(padap, qbase->reg_addr, func);
+	for (i = 0; i < SGE_QBASE_DATA_REG_NUM; i++, buff++)
+		*buff = t4_read_reg(padap, qbase->reg_data[i]);
+}
+
 int cudbg_collect_sge_indirect(struct cudbg_init *pdbg_init,
 			       struct cudbg_buffer *dbg_buff,
 			       struct cudbg_error *cudbg_err)
 {
 	struct adapter *padap = pdbg_init->adap;
 	struct cudbg_buffer temp_buff = { 0 };
+	struct sge_qbase_reg_field *sge_qbase;
 	struct ireg_buf *ch_sge_dbg;
 	int i, rc;
 
-	rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(*ch_sge_dbg) * 2,
+	rc = cudbg_get_buff(pdbg_init, dbg_buff,
+			    sizeof(*ch_sge_dbg) * 2 + sizeof(*sge_qbase),
 			    &temp_buff);
 	if (rc)
 		return rc;
@@ -1370,6 +1393,28 @@ int cudbg_collect_sge_indirect(struct cudbg_init *pdbg_init,
 				 sge_pio->ireg_local_offset);
 		ch_sge_dbg++;
 	}
+
+	if (CHELSIO_CHIP_VERSION(padap->params.chip) > CHELSIO_T5) {
+		sge_qbase = (struct sge_qbase_reg_field *)ch_sge_dbg;
+		/* 1 addr reg SGE_QBASE_INDEX and 4 data reg
+		 * SGE_QBASE_MAP[0-3]
+		 */
+		sge_qbase->reg_addr = t6_sge_qbase_index_array[0];
+		for (i = 0; i < SGE_QBASE_DATA_REG_NUM; i++)
+			sge_qbase->reg_data[i] =
+				t6_sge_qbase_index_array[i + 1];
+
+		for (i = 0; i <= PCIE_FW_MASTER_M; i++)
+			cudbg_read_sge_qbase_indirect_reg(padap, sge_qbase,
+							  i, true);
+
+		for (i = 0; i < padap->params.arch.vfcount; i++)
+			cudbg_read_sge_qbase_indirect_reg(padap, sge_qbase,
+							  i, false);
+
+		sge_qbase->vfcount = padap->params.arch.vfcount;
+	}
+
 	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
@@ -2366,8 +2411,11 @@ void cudbg_fill_le_tcam_info(struct adapter *padap,
 	value = t4_read_reg(padap, LE_DB_ROUTING_TABLE_INDEX_A);
 	tcam_region->routing_start = value;
 
-	/*Get clip table index */
-	value = t4_read_reg(padap, LE_DB_CLIP_TABLE_INDEX_A);
+	/* Get clip table index. For T6 there is separate CLIP TCAM */
+	if (is_t6(padap->params.chip))
+		value = t4_read_reg(padap, LE_DB_CLCAM_TID_BASE_A);
+	else
+		value = t4_read_reg(padap, LE_DB_CLIP_TABLE_INDEX_A);
 	tcam_region->clip_start = value;
 
 	/* Get filter table index */
@@ -2392,8 +2440,16 @@ void cudbg_fill_le_tcam_info(struct adapter *padap,
 					       tcam_region->tid_hash_base;
 		}
 	} else { /* hash not enabled */
-		tcam_region->max_tid = CUDBG_MAX_TCAM_TID;
+		if (is_t6(padap->params.chip))
+			tcam_region->max_tid = (value & ASLIPCOMPEN_F) ?
+					       CUDBG_MAX_TID_COMP_EN :
+					       CUDBG_MAX_TID_COMP_DIS;
+		else
+			tcam_region->max_tid = CUDBG_MAX_TCAM_TID;
 	}
+
+	if (is_t6(padap->params.chip))
+		tcam_region->max_tid += CUDBG_T6_CLIP;
 }
 
 int cudbg_collect_le_tcam(struct cudbg_init *pdbg_init,
@@ -2423,18 +2479,31 @@ int cudbg_collect_le_tcam(struct cudbg_init *pdbg_init,
 	for (i = 0; i < tcam_region.max_tid; ) {
 		rc = cudbg_read_tid(pdbg_init, i, tid_data);
 		if (rc) {
-			cudbg_err->sys_err = rc;
-			cudbg_put_buff(pdbg_init, &temp_buff);
-			return rc;
+			cudbg_err->sys_warn = CUDBG_STATUS_PARTIAL_DATA;
+			/* Update tcam header and exit */
+			tcam_region.max_tid = i;
+			memcpy(temp_buff.data, &tcam_region,
+			       sizeof(struct cudbg_tcam));
+			goto out;
 		}
 
-		/* ipv6 takes two tids */
-		cudbg_is_ipv6_entry(tid_data, tcam_region) ? i += 2 : i++;
+		if (cudbg_is_ipv6_entry(tid_data, tcam_region)) {
+			/* T6 CLIP TCAM: ipv6 takes 4 entries */
+			if (is_t6(padap->params.chip) &&
+			    i >= tcam_region.clip_start &&
+			    i < tcam_region.clip_start + CUDBG_T6_CLIP)
+				i += 4;
+			else /* Main TCAM: ipv6 takes two tids */
+				i += 2;
+		} else {
+			i++;
+		}
 
 		tid_data++;
 		bytes += sizeof(struct cudbg_tid_data);
 	}
 
+out:
 	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index 688f95440af2..0dbe2d9e22d6 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -50,6 +50,7 @@
 #include <linux/net_tstamp.h>
 #include <linux/ptp_clock_kernel.h>
 #include <linux/ptp_classify.h>
+#include <linux/crash_dump.h>
 #include <asm/io.h>
 #include "t4_chip_type.h"
 #include "cxgb4_uld.h"
@@ -490,6 +491,9 @@ struct link_config {
 
 	unsigned char  link_ok;          /* link up? */
 	unsigned char  link_down_rc;     /* link down reason */
+
+	bool new_module;		 /* ->OS Transceiver Module inserted */
+	bool redo_l1cfg;		 /* ->CC redo current "sticky" L1 CFG */
 };
 
 #define FW_LEN16(fw_struct) FW_CMD_LEN16_V(sizeof(fw_struct) / 16)
@@ -964,6 +968,9 @@ struct adapter {
 	struct hma_data hma;
 
 	struct srq_data *srq;
+
+	/* Dump buffer for collecting logs in kdump kernel */
+	struct vmcoredd_data vmcoredd;
 };
 
 /* Support for "sched-class" command to allow a TX Scheduling Class to be
@@ -1034,6 +1041,7 @@ struct ch_sched_queue {
 #define VF_BITWIDTH 8
 #define IVLAN_BITWIDTH 16
 #define OVLAN_BITWIDTH 16
+#define ENCAP_VNI_BITWIDTH 24
 
 /* Filter matching rules.  These consist of a set of ingress packet field
  * (value, mask) tuples.  The associated ingress packet field matches the
@@ -1064,6 +1072,7 @@ struct ch_filter_tuple {
 	uint32_t ivlan_vld:1;                   /* inner VLAN valid */
 	uint32_t ovlan_vld:1;                   /* outer VLAN valid */
 	uint32_t pfvf_vld:1;                    /* PF/VF valid */
+	uint32_t encap_vld:1;			/* Encapsulation valid */
 	uint32_t macidx:MACIDX_BITWIDTH;        /* exact match MAC index */
 	uint32_t fcoe:FCOE_BITWIDTH;            /* FCoE packet */
 	uint32_t iport:IPORT_BITWIDTH;          /* ingress port */
@@ -1074,6 +1083,7 @@ struct ch_filter_tuple {
 	uint32_t vf:VF_BITWIDTH;                /* PCI-E VF ID */
 	uint32_t ivlan:IVLAN_BITWIDTH;          /* inner VLAN */
 	uint32_t ovlan:OVLAN_BITWIDTH;          /* outer VLAN */
+	uint32_t vni:ENCAP_VNI_BITWIDTH;	/* VNI of tunnel */
 
 	/* Uncompressed header matching field rules.  These are always
 	 * available for field rules.
@@ -1317,7 +1327,7 @@ static inline unsigned int qtimer_val(const struct adapter *adap,
 extern char cxgb4_driver_name[];
 extern const char cxgb4_driver_version[];
 
-void t4_os_portmod_changed(const struct adapter *adap, int port_id);
+void t4_os_portmod_changed(struct adapter *adap, int port_id);
 void t4_os_link_changed(struct adapter *adap, int port_id, int link_stat);
 
 void t4_free_sge_resources(struct adapter *adap);
@@ -1498,8 +1508,25 @@ void t4_intr_disable(struct adapter *adapter);
 int t4_slow_intr_handler(struct adapter *adapter);
 
 int t4_wait_dev_ready(void __iomem *regs);
-int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port,
-		  struct link_config *lc);
+
+int t4_link_l1cfg_core(struct adapter *adap, unsigned int mbox,
+		       unsigned int port, struct link_config *lc,
+		       bool sleep_ok, int timeout);
+
+static inline int t4_link_l1cfg(struct adapter *adapter, unsigned int mbox,
+				unsigned int port, struct link_config *lc)
+{
+	return t4_link_l1cfg_core(adapter, mbox, port, lc,
+				  true, FW_CMD_MAX_TIMEOUT);
+}
+
+static inline int t4_link_l1cfg_ns(struct adapter *adapter, unsigned int mbox,
+				   unsigned int port, struct link_config *lc)
+{
+	return t4_link_l1cfg_core(adapter, mbox, port, lc,
+				  false, FW_CMD_MAX_TIMEOUT);
+}
+
 int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port);
 
 u32 t4_read_pcie_cfg4(struct adapter *adap, int reg);
@@ -1690,6 +1717,12 @@ int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
 int t4_free_raw_mac_filt(struct adapter *adap, unsigned int viid,
 			 const u8 *addr, const u8 *mask, unsigned int idx,
 			 u8 lookup_type, u8 port_id, bool sleep_ok);
+int t4_free_encap_mac_filt(struct adapter *adap, unsigned int viid, int idx,
+			   bool sleep_ok);
+int t4_alloc_encap_mac_filt(struct adapter *adap, unsigned int viid,
+			    const u8 *addr, const u8 *mask, unsigned int vni,
+			    unsigned int vni_mask, u8 dip_hit, u8 lookup_type,
+			    bool sleep_ok);
 int t4_alloc_raw_mac_filt(struct adapter *adap, unsigned int viid,
 			  const u8 *addr, const u8 *mask, unsigned int idx,
 			  u8 lookup_type, u8 port_id, bool sleep_ok);
@@ -1705,6 +1738,9 @@ int t4_set_addr_hash(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		     bool ucast, u64 vec, bool sleep_ok);
 int t4_enable_vi_params(struct adapter *adap, unsigned int mbox,
 			unsigned int viid, bool rx_en, bool tx_en, bool dcb_en);
+int t4_enable_pi_params(struct adapter *adap, unsigned int mbox,
+			struct port_info *pi,
+			bool rx_en, bool tx_en, bool dcb_en);
 int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		 bool rx_en, bool tx_en);
 int t4_identify_port(struct adapter *adap, unsigned int mbox, unsigned int viid,
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c
index 143686c60234..8d751efcb90e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c
@@ -214,7 +214,8 @@ static u32 cxgb4_get_entity_length(struct adapter *adap, u32 entity)
 		len = sizeof(struct ireg_buf) * n;
 		break;
 	case CUDBG_SGE_INDIRECT:
-		len = sizeof(struct ireg_buf) * 2;
+		len = sizeof(struct ireg_buf) * 2 +
+		      sizeof(struct sge_qbase_reg_field);
 		break;
 	case CUDBG_ULPRX_LA:
 		len = sizeof(struct cudbg_ulprx_la);
@@ -488,3 +489,28 @@ void cxgb4_init_ethtool_dump(struct adapter *adapter)
 	adapter->eth_dump.version = adapter->params.fw_vers;
 	adapter->eth_dump.len = 0;
 }
+
+static int cxgb4_cudbg_vmcoredd_collect(struct vmcoredd_data *data, void *buf)
+{
+	struct adapter *adap = container_of(data, struct adapter, vmcoredd);
+	u32 len = data->size;
+
+	return cxgb4_cudbg_collect(adap, buf, &len, CXGB4_ETH_DUMP_ALL);
+}
+
+int cxgb4_cudbg_vmcore_add_dump(struct adapter *adap)
+{
+	struct vmcoredd_data *data = &adap->vmcoredd;
+	u32 len;
+
+	len = sizeof(struct cudbg_hdr) +
+	      sizeof(struct cudbg_entity_hdr) * CUDBG_MAX_ENTITY;
+	len += CUDBG_DUMP_BUFF_SIZE;
+
+	data->size = len;
+	snprintf(data->dump_name, sizeof(data->dump_name), "%s_%s",
+		 cxgb4_driver_name, adap->name);
+	data->vmcoredd_callback = cxgb4_cudbg_vmcoredd_collect;
+
+	return vmcore_add_device_dump(data);
+}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h
index ce1ac9a1c878..ef59ba1ed968 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h
@@ -41,8 +41,11 @@ enum CXGB4_ETHTOOL_DUMP_FLAGS {
 	CXGB4_ETH_DUMP_HW = (1 << 1), /* various FW and HW dumps */
 };
 
+#define CXGB4_ETH_DUMP_ALL (CXGB4_ETH_DUMP_MEM | CXGB4_ETH_DUMP_HW)
+
 u32 cxgb4_get_dump_length(struct adapter *adap, u32 flag);
 int cxgb4_cudbg_collect(struct adapter *adap, void *buf, u32 *buf_size,
 			u32 flag);
 void cxgb4_init_ethtool_dump(struct adapter *adapter);
+int cxgb4_cudbg_vmcore_add_dump(struct adapter *adap);
 #endif /* __CXGB4_CUDBG_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
index 59d04d73c672..f7eef93ffc87 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
@@ -800,24 +800,20 @@ static int set_link_ksettings(struct net_device *dev,
 	if (base->duplex != DUPLEX_FULL)
 		return -EINVAL;
 
-	if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
-		/* PHY offers a single speed.  See if that's what's
-		 * being requested.
-		 */
-		if (base->autoneg == AUTONEG_DISABLE &&
-		    (lc->pcaps & speed_to_fw_caps(base->speed)))
-			return 0;
-		return -EINVAL;
-	}
-
 	old_lc = *lc;
-	if (base->autoneg == AUTONEG_DISABLE) {
+	if (!(lc->pcaps & FW_PORT_CAP32_ANEG) ||
+	    base->autoneg == AUTONEG_DISABLE) {
 		fw_caps = speed_to_fw_caps(base->speed);
 
-		if (!(lc->pcaps & fw_caps))
+		/* Must only specify a single speed which must be supported
+		 * as part of the Physical Port Capabilities.
+		 */
+		if ((fw_caps & (fw_caps - 1)) != 0 ||
+		    !(lc->pcaps & fw_caps))
 			return -EINVAL;
+
 		lc->speed_caps = fw_caps;
-		lc->acaps = 0;
+		lc->acaps = fw_caps;
 	} else {
 		fw_caps =
 			 lmm_to_fw_caps(link_ksettings->link_modes.advertising);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c
index b76447baccaf..00fc5f1afb1d 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c
@@ -64,8 +64,7 @@ static int set_tcb_field(struct adapter *adap, struct filter_entry *f,
 	if (!skb)
 		return -ENOMEM;
 
-	req = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*req));
-	memset(req, 0, sizeof(*req));
+	req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
 	INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, ftid);
 	req->reply_ctrl = htons(REPLY_CHAN_V(0) |
 				QUEUENO_V(adap->sge.fw_evtq.abs_id) |
@@ -266,6 +265,8 @@ static int validate_filter(struct net_device *dev,
 			fs->mask.pfvf_vld) ||
 	    unsupported(fconf, VNIC_ID_F, fs->val.ovlan_vld,
 			fs->mask.ovlan_vld) ||
+	    unsupported(fconf, VNIC_ID_F, fs->val.encap_vld,
+			fs->mask.encap_vld) ||
 	    unsupported(fconf, VLAN_F, fs->val.ivlan_vld, fs->mask.ivlan_vld))
 		return -EOPNOTSUPP;
 
@@ -276,8 +277,12 @@ static int validate_filter(struct net_device *dev,
 	 * carries that overlap, we need to translate any PF/VF
 	 * specification into that internal format below.
 	 */
-	if (is_field_set(fs->val.pfvf_vld, fs->mask.pfvf_vld) &&
-	    is_field_set(fs->val.ovlan_vld, fs->mask.ovlan_vld))
+	if ((is_field_set(fs->val.pfvf_vld, fs->mask.pfvf_vld) &&
+	     is_field_set(fs->val.ovlan_vld, fs->mask.ovlan_vld)) ||
+	    (is_field_set(fs->val.pfvf_vld, fs->mask.pfvf_vld) &&
+	     is_field_set(fs->val.encap_vld, fs->mask.encap_vld)) ||
+	    (is_field_set(fs->val.ovlan_vld, fs->mask.ovlan_vld) &&
+	     is_field_set(fs->val.encap_vld, fs->mask.encap_vld)))
 		return -EOPNOTSUPP;
 	if (unsupported(iconf, VNIC_F, fs->val.pfvf_vld, fs->mask.pfvf_vld) ||
 	    (is_field_set(fs->val.ovlan_vld, fs->mask.ovlan_vld) &&
@@ -307,6 +312,9 @@ static int validate_filter(struct net_device *dev,
 	     fs->newvlan == VLAN_REWRITE))
 		return -EOPNOTSUPP;
 
+	if (fs->val.encap_vld &&
+	    CHELSIO_CHIP_VERSION(adapter->params.chip) < CHELSIO_T6)
+		return -EOPNOTSUPP;
 	return 0;
 }
 
@@ -706,6 +714,8 @@ int delete_filter(struct adapter *adapter, unsigned int fidx)
  */
 void clear_filter(struct adapter *adap, struct filter_entry *f)
 {
+	struct port_info *pi = netdev_priv(f->dev);
+
 	/* If the new or old filter have loopback rewriteing rules then we'll
 	 * need to free any existing L2T, SMT, CLIP entries of filter
 	 * rule.
@@ -716,6 +726,12 @@ void clear_filter(struct adapter *adap, struct filter_entry *f)
 	if (f->smt)
 		cxgb4_smt_release(f->smt);
 
+	if (f->fs.val.encap_vld && f->fs.val.ovlan_vld)
+		if (atomic_dec_and_test(&adap->mps_encap[f->fs.val.ovlan &
+							 0x1ff].refcnt))
+			t4_free_encap_mac_filt(adap, pi->viid,
+					       f->fs.val.ovlan & 0x1ff, 0);
+
 	if ((f->fs.hash || is_t6(adap->params.chip)) && f->fs.type)
 		cxgb4_clip_release(f->dev, (const u32 *)&f->fs.val.lip, 1);
 
@@ -841,6 +857,10 @@ bool is_filter_exact_match(struct adapter *adap,
 	if (!is_hashfilter(adap))
 		return false;
 
+	 /* Keep tunnel VNI match disabled for hash-filters for now */
+	if (fs->mask.encap_vld)
+		return false;
+
 	if (fs->type) {
 		if (is_inaddr_any(fs->val.fip, AF_INET6) ||
 		    !is_addr_all_mask(fs->mask.fip, AF_INET6))
@@ -934,8 +954,12 @@ static u64 hash_filter_ntuple(struct ch_filter_specification *fs,
 		ntuple |= (u64)(fs->val.tos) << tp->tos_shift;
 
 	if (tp->vnic_shift >= 0) {
-		if ((adap->params.tp.ingress_config & VNIC_F) &&
-		    fs->mask.pfvf_vld)
+		if ((adap->params.tp.ingress_config & USE_ENC_IDX_F) &&
+		    fs->mask.encap_vld)
+			ntuple |= (u64)((fs->val.encap_vld << 16) |
+					(fs->val.ovlan)) << tp->vnic_shift;
+		else if ((adap->params.tp.ingress_config & VNIC_F) &&
+			 fs->mask.pfvf_vld)
 			ntuple |= (u64)((fs->val.pfvf_vld << 16) |
 					(fs->val.pf << 13) |
 					(fs->val.vf)) << tp->vnic_shift;
@@ -1049,6 +1073,7 @@ static int cxgb4_set_hash_filter(struct net_device *dev,
 				 struct filter_ctx *ctx)
 {
 	struct adapter *adapter = netdev2adap(dev);
+	struct port_info *pi = netdev_priv(dev);
 	struct tid_info *t = &adapter->tids;
 	struct filter_entry *f;
 	struct sk_buff *skb;
@@ -1115,13 +1140,34 @@ static int cxgb4_set_hash_filter(struct net_device *dev,
 		f->fs.mask.ovlan = (fs->mask.pf << 13) | fs->mask.vf;
 		f->fs.val.ovlan_vld = fs->val.pfvf_vld;
 		f->fs.mask.ovlan_vld = fs->mask.pfvf_vld;
+	} else if (iconf & USE_ENC_IDX_F) {
+		if (f->fs.val.encap_vld) {
+			struct port_info *pi = netdev_priv(f->dev);
+			u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 };
+
+			/* allocate MPS TCAM entry */
+			ret = t4_alloc_encap_mac_filt(adapter, pi->viid,
+						      match_all_mac,
+						      match_all_mac,
+						      f->fs.val.vni,
+						      f->fs.mask.vni,
+						      0, 1, 1);
+			if (ret < 0)
+				goto free_atid;
+
+			atomic_inc(&adapter->mps_encap[ret].refcnt);
+			f->fs.val.ovlan = ret;
+			f->fs.mask.ovlan = 0xffff;
+			f->fs.val.ovlan_vld = 1;
+			f->fs.mask.ovlan_vld = 1;
+		}
 	}
 
 	size = sizeof(struct cpl_t6_act_open_req);
 	if (f->fs.type) {
 		ret = cxgb4_clip_get(f->dev, (const u32 *)&f->fs.val.lip, 1);
 		if (ret)
-			goto free_atid;
+			goto free_mps;
 
 		skb = alloc_skb(size, GFP_KERNEL);
 		if (!skb) {
@@ -1136,7 +1182,7 @@ static int cxgb4_set_hash_filter(struct net_device *dev,
 		skb = alloc_skb(size, GFP_KERNEL);
 		if (!skb) {
 			ret = -ENOMEM;
-			goto free_atid;
+			goto free_mps;
 		}
 
 		mk_act_open_req(f, skb,
@@ -1152,6 +1198,10 @@ static int cxgb4_set_hash_filter(struct net_device *dev,
 free_clip:
 	cxgb4_clip_release(f->dev, (const u32 *)&f->fs.val.lip, 1);
 
+free_mps:
+	if (f->fs.val.encap_vld && f->fs.val.ovlan_vld)
+		t4_free_encap_mac_filt(adapter, pi->viid, f->fs.val.ovlan, 1);
+
 free_atid:
 	cxgb4_free_atid(t, atid);
 
@@ -1333,6 +1383,27 @@ int __cxgb4_set_filter(struct net_device *dev, int filter_id,
 		f->fs.mask.ovlan = (fs->mask.pf << 13) | fs->mask.vf;
 		f->fs.val.ovlan_vld = fs->val.pfvf_vld;
 		f->fs.mask.ovlan_vld = fs->mask.pfvf_vld;
+	} else if (iconf & USE_ENC_IDX_F) {
+		if (f->fs.val.encap_vld) {
+			struct port_info *pi = netdev_priv(f->dev);
+			u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 };
+
+			/* allocate MPS TCAM entry */
+			ret = t4_alloc_encap_mac_filt(adapter, pi->viid,
+						      match_all_mac,
+						      match_all_mac,
+						      f->fs.val.vni,
+						      f->fs.mask.vni,
+						      0, 1, 1);
+			if (ret < 0)
+				goto free_clip;
+
+			atomic_inc(&adapter->mps_encap[ret].refcnt);
+			f->fs.val.ovlan = ret;
+			f->fs.mask.ovlan = 0x1ff;
+			f->fs.val.ovlan_vld = 1;
+			f->fs.mask.ovlan_vld = 1;
+		}
 	}
 
 	/* Attempt to set the filter.  If we don't succeed, we clear
@@ -1349,6 +1420,13 @@ int __cxgb4_set_filter(struct net_device *dev, int filter_id,
 	}
 
 	return ret;
+
+free_clip:
+	if (is_t6(adapter->params.chip) && f->fs.type)
+		cxgb4_clip_release(f->dev, (const u32 *)&f->fs.val.lip, 1);
+	cxgb4_clear_ftid(&adapter->tids, filter_id,
+			 fs->type ? PF_INET6 : PF_INET, chip_ver);
+	return ret;
 }
 
 static int cxgb4_del_hash_filter(struct net_device *dev, int filter_id,
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 005283c7cdfe..35cb3ae4f7b6 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -301,14 +301,14 @@ void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 	}
 }
 
-void t4_os_portmod_changed(const struct adapter *adap, int port_id)
+void t4_os_portmod_changed(struct adapter *adap, int port_id)
 {
 	static const char *mod_str[] = {
 		NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
 	};
 
-	const struct net_device *dev = adap->port[port_id];
-	const struct port_info *pi = netdev_priv(dev);
+	struct net_device *dev = adap->port[port_id];
+	struct port_info *pi = netdev_priv(dev);
 
 	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 		netdev_info(dev, "port module unplugged\n");
@@ -325,6 +325,11 @@ void t4_os_portmod_changed(const struct adapter *adap, int port_id)
 	else
 		netdev_info(dev, "%s: unknown module type %d inserted\n",
 			    dev->name, pi->mod_type);
+
+	/* If the interface is running, then we'll need any "sticky" Link
+	 * Parameters redone with a new Transceiver Module.
+	 */
+	pi->link_cfg.redo_l1cfg = netif_running(dev);
 }
 
 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
@@ -460,7 +465,7 @@ static int link_start(struct net_device *dev)
 				    &pi->link_cfg);
 	if (ret == 0) {
 		local_bh_disable();
-		ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true,
+		ret = t4_enable_pi_params(pi->adapter, mb, pi, true,
 					  true, CXGB4_DCB_ENABLED);
 		local_bh_enable();
 	}
@@ -2339,7 +2344,8 @@ static int cxgb_close(struct net_device *dev)
 
 	netif_tx_stop_all_queues(dev);
 	netif_carrier_off(dev);
-	ret = t4_enable_vi(adapter, adapter->pf, pi->viid, false, false);
+	ret = t4_enable_pi_params(adapter, adapter->pf, pi,
+				  false, false, false);
 #ifdef CONFIG_CHELSIO_T4_DCB
 	cxgb4_dcb_reset(dev);
 	dcb_tx_queue_prio_enable(dev, false);
@@ -2886,13 +2892,13 @@ static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
 	}
 
 	/* Convert from Mbps to Kbps */
-	req_rate = rate << 10;
+	req_rate = rate * 1000;
 
 	/* Max rate is 100 Gbps */
-	if (req_rate >= SCHED_MAX_RATE_KBPS) {
+	if (req_rate > SCHED_MAX_RATE_KBPS) {
 		dev_err(adap->pdev_dev,
 			"Invalid rate %u Mbps, Max rate is %u Mbps\n",
-			rate, SCHED_MAX_RATE_KBPS >> 10);
+			rate, SCHED_MAX_RATE_KBPS / 1000);
 		return -ERANGE;
 	}
 
@@ -3081,7 +3087,7 @@ static void cxgb_del_udp_tunnel(struct net_device *netdev,
 					   match_all_mac, match_all_mac,
 					   adapter->rawf_start +
 					    pi->port_id,
-					   1, pi->port_id, true);
+					   1, pi->port_id, false);
 		if (ret < 0) {
 			netdev_info(netdev, "Failed to free mac filter entry, for port %d\n",
 				    i);
@@ -3169,7 +3175,7 @@ static void cxgb_add_udp_tunnel(struct net_device *netdev,
 					    match_all_mac,
 					    adapter->rawf_start +
 					    pi->port_id,
-					    1, pi->port_id, true);
+					    1, pi->port_id, false);
 		if (ret < 0) {
 			netdev_info(netdev, "Failed to allocate a mac filter entry, not adding port %d\n",
 				    be16_to_cpu(ti->port));
@@ -4135,6 +4141,10 @@ static int adap_init0(struct adapter *adap)
 		 * card
 		 */
 		card_fw = kvzalloc(sizeof(*card_fw), GFP_KERNEL);
+		if (!card_fw) {
+			ret = -ENOMEM;
+			goto bye;
+		}
 
 		/* Get FW from from /lib/firmware/ */
 		ret = request_firmware(&fw, fw_info->fw_mod_name,
@@ -4276,6 +4286,20 @@ static int adap_init0(struct adapter *adap)
 	adap->tids.nftids = val[4] - val[3] + 1;
 	adap->sge.ingr_start = val[5];
 
+	if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) {
+		/* Read the raw mps entries. In T6, the last 2 tcam entries
+		 * are reserved for raw mac addresses (rawf = 2, one per port).
+		 */
+		params[0] = FW_PARAM_PFVF(RAWF_START);
+		params[1] = FW_PARAM_PFVF(RAWF_END);
+		ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
+				      params, val);
+		if (ret == 0) {
+			adap->rawf_start = val[0];
+			adap->rawf_cnt = val[1] - val[0] + 1;
+		}
+	}
+
 	/* qids (ingress/egress) returned from firmware can be anywhere
 	 * in the range from EQ(IQFLINT)_START to EQ(IQFLINT)_END.
 	 * Hence driver needs to allocate memory for this range to
@@ -5042,79 +5066,6 @@ static int init_rss(struct adapter *adap)
 	return 0;
 }
 
-static int cxgb4_get_pcie_dev_link_caps(struct adapter *adap,
-					enum pci_bus_speed *speed,
-					enum pcie_link_width *width)
-{
-	u32 lnkcap1, lnkcap2;
-	int err1, err2;
-
-#define  PCIE_MLW_CAP_SHIFT 4   /* start of MLW mask in link capabilities */
-
-	*speed = PCI_SPEED_UNKNOWN;
-	*width = PCIE_LNK_WIDTH_UNKNOWN;
-
-	err1 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP,
-					  &lnkcap1);
-	err2 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP2,
-					  &lnkcap2);
-	if (!err2 && lnkcap2) { /* PCIe r3.0-compliant */
-		if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB)
-			*speed = PCIE_SPEED_8_0GT;
-		else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB)
-			*speed = PCIE_SPEED_5_0GT;
-		else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB)
-			*speed = PCIE_SPEED_2_5GT;
-	}
-	if (!err1) {
-		*width = (lnkcap1 & PCI_EXP_LNKCAP_MLW) >> PCIE_MLW_CAP_SHIFT;
-		if (!lnkcap2) { /* pre-r3.0 */
-			if (lnkcap1 & PCI_EXP_LNKCAP_SLS_5_0GB)
-				*speed = PCIE_SPEED_5_0GT;
-			else if (lnkcap1 & PCI_EXP_LNKCAP_SLS_2_5GB)
-				*speed = PCIE_SPEED_2_5GT;
-		}
-	}
-
-	if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN)
-		return err1 ? err1 : err2 ? err2 : -EINVAL;
-	return 0;
-}
-
-static void cxgb4_check_pcie_caps(struct adapter *adap)
-{
-	enum pcie_link_width width, width_cap;
-	enum pci_bus_speed speed, speed_cap;
-
-#define PCIE_SPEED_STR(speed) \
-	(speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : \
-	 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : \
-	 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : \
-	 "Unknown")
-
-	if (cxgb4_get_pcie_dev_link_caps(adap, &speed_cap, &width_cap)) {
-		dev_warn(adap->pdev_dev,
-			 "Unable to determine PCIe device BW capabilities\n");
-		return;
-	}
-
-	if (pcie_get_minimum_link(adap->pdev, &speed, &width) ||
-	    speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) {
-		dev_warn(adap->pdev_dev,
-			 "Unable to determine PCI Express bandwidth.\n");
-		return;
-	}
-
-	dev_info(adap->pdev_dev, "PCIe link speed is %s, device supports %s\n",
-		 PCIE_SPEED_STR(speed), PCIE_SPEED_STR(speed_cap));
-	dev_info(adap->pdev_dev, "PCIe link width is x%d, device supports x%d\n",
-		 width, width_cap);
-	if (speed < speed_cap || width < width_cap)
-		dev_info(adap->pdev_dev,
-			 "A slot with more lanes and/or higher speed is "
-			 "suggested for optimal performance.\n");
-}
-
 /* Dump basic information about the adapter */
 static void print_adapter_info(struct adapter *adapter)
 {
@@ -5181,6 +5132,7 @@ static void free_some_resources(struct adapter *adapter)
 {
 	unsigned int i;
 
+	kvfree(adapter->mps_encap);
 	kvfree(adapter->smt);
 	kvfree(adapter->l2t);
 	kvfree(adapter->srq);
@@ -5216,14 +5168,11 @@ static void free_some_resources(struct adapter *adapter)
 		   NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
 #define SEGMENT_SIZE 128
 
-static int get_chip_type(struct pci_dev *pdev, u32 pl_rev)
+static int t4_get_chip_type(struct adapter *adap, int ver)
 {
-	u16 device_id;
-
-	/* Retrieve adapter's device ID */
-	pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id);
+	u32 pl_rev = REV_G(t4_read_reg(adap, PL_REV_A));
 
-	switch (device_id >> 12) {
+	switch (ver) {
 	case CHELSIO_T4:
 		return CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev);
 	case CHELSIO_T5:
@@ -5231,8 +5180,7 @@ static int get_chip_type(struct pci_dev *pdev, u32 pl_rev)
 	case CHELSIO_T6:
 		return CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev);
 	default:
-		dev_err(&pdev->dev, "Device %d is not supported\n",
-			device_id);
+		break;
 	}
 	return -EINVAL;
 }
@@ -5261,13 +5209,9 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
 	u32 pcie_fw;
 
 	pcie_fw = readl(adap->regs + PCIE_FW_A);
-	/* Check if cxgb4 is the MASTER and fw is initialized */
-	if (num_vfs &&
-	    (!(pcie_fw & PCIE_FW_INIT_F) ||
-	    !(pcie_fw & PCIE_FW_MASTER_VLD_F) ||
-	    PCIE_FW_MASTER_G(pcie_fw) != CXGB4_UNIFIED_PF)) {
-		dev_warn(&pdev->dev,
-			 "cxgb4 driver needs to be MASTER to support SRIOV\n");
+	/* Check if fw is initialized */
+	if (!(pcie_fw & PCIE_FW_INIT_F)) {
+		dev_warn(&pdev->dev, "Device not initialized\n");
 		return -EOPNOTSUPP;
 	}
 
@@ -5406,15 +5350,18 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
 
 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
-	int func, i, err, s_qpp, qpp, num_seg;
+	struct net_device *netdev;
+	struct adapter *adapter;
+	static int adap_idx = 1;
+	int s_qpp, qpp, num_seg;
 	struct port_info *pi;
 	bool highdma = false;
-	struct adapter *adapter = NULL;
-	struct net_device *netdev;
-	void __iomem *regs;
-	u32 whoami, pl_rev;
 	enum chip_type chip;
-	static int adap_idx = 1;
+	void __iomem *regs;
+	int func, chip_ver;
+	u16 device_id;
+	int i, err;
+	u32 whoami;
 
 	printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
 
@@ -5450,11 +5397,17 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto out_free_adapter;
 
 	/* We control everything through one PF */
-	whoami = readl(regs + PL_WHOAMI_A);
-	pl_rev = REV_G(readl(regs + PL_REV_A));
-	chip = get_chip_type(pdev, pl_rev);
-	func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ?
-		SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami);
+	whoami = t4_read_reg(adapter, PL_WHOAMI_A);
+	pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id);
+	chip = t4_get_chip_type(adapter, CHELSIO_PCI_ID_VER(device_id));
+	if (chip < 0) {
+		dev_err(&pdev->dev, "Device %d is not supported\n", device_id);
+		err = chip;
+		goto out_free_adapter;
+	}
+	chip_ver = CHELSIO_CHIP_VERSION(chip);
+	func = chip_ver <= CHELSIO_T5 ?
+	       SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami);
 
 	adapter->pdev = pdev;
 	adapter->pdev_dev = &pdev->dev;
@@ -5543,6 +5496,16 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (err)
 		goto out_free_adapter;
 
+	if (is_kdump_kernel()) {
+		/* Collect hardware state and append to /proc/vmcore */
+		err = cxgb4_cudbg_vmcore_add_dump(adapter);
+		if (err) {
+			dev_warn(adapter->pdev_dev,
+				 "Fail collecting vmcore device dump, err: %d. Continuing\n",
+				 err);
+			err = 0;
+		}
+	}
 
 	if (!is_t4(adapter->params.chip)) {
 		s_qpp = (QUEUESPERPAGEPF0_S +
@@ -5610,8 +5573,15 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 			NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
 			NETIF_F_HW_TC;
 
-		if (CHELSIO_CHIP_VERSION(chip) > CHELSIO_T5)
+		if (chip_ver > CHELSIO_T5) {
+			netdev->hw_enc_features |= NETIF_F_IP_CSUM |
+						   NETIF_F_IPV6_CSUM |
+						   NETIF_F_RXCSUM |
+						   NETIF_F_GSO_UDP_TUNNEL |
+						   NETIF_F_TSO | NETIF_F_TSO6;
+
 			netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
+		}
 
 		if (highdma)
 			netdev->hw_features |= NETIF_F_HIGHDMA;
@@ -5676,8 +5646,14 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		adapter->params.offload = 0;
 	}
 
+	adapter->mps_encap = kvzalloc(sizeof(struct mps_encap_entry) *
+					  adapter->params.arch.mps_tcam_size,
+				      GFP_KERNEL);
+	if (!adapter->mps_encap)
+		dev_warn(&pdev->dev, "could not allocate MPS Encap entries, continuing\n");
+
 #if IS_ENABLED(CONFIG_IPV6)
-	if ((CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) &&
+	if (chip_ver <= CHELSIO_T5 &&
 	    (!(t4_read_reg(adapter, LE_DB_CONFIG_A) & ASLIPCOMPEN_F))) {
 		/* CLIP functionality is not present in hardware,
 		 * hence disable all offload features
@@ -5749,7 +5725,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 
 	/* check for PCI Express bandwidth capabiltites */
-	cxgb4_check_pcie_caps(adapter);
+	pcie_print_link_status(pdev);
 
 	err = init_rss(adapter);
 	if (err)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c
index 36563364bae7..3ddd2c4acf68 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c
@@ -194,6 +194,23 @@ static void cxgb4_process_flow_match(struct net_device *dev,
 		fs->mask.tos = mask->tos;
 	}
 
+	if (dissector_uses_key(cls->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
+		struct flow_dissector_key_keyid *key, *mask;
+
+		key = skb_flow_dissector_target(cls->dissector,
+						FLOW_DISSECTOR_KEY_ENC_KEYID,
+						cls->key);
+		mask = skb_flow_dissector_target(cls->dissector,
+						 FLOW_DISSECTOR_KEY_ENC_KEYID,
+						 cls->mask);
+		fs->val.vni = be32_to_cpu(key->keyid);
+		fs->mask.vni = be32_to_cpu(mask->keyid);
+		if (fs->mask.vni) {
+			fs->val.encap_vld = 1;
+			fs->mask.encap_vld = 1;
+		}
+	}
+
 	if (dissector_uses_key(cls->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
 		struct flow_dissector_key_vlan *key, *mask;
 		u16 vlan_tci, vlan_tci_mask;
@@ -247,6 +264,7 @@ static int cxgb4_validate_flow_match(struct net_device *dev,
 	      BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 	      BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 	      BIT(FLOW_DISSECTOR_KEY_PORTS) |
+	      BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
 	      BIT(FLOW_DISSECTOR_KEY_VLAN) |
 	      BIT(FLOW_DISSECTOR_KEY_IP))) {
 		netdev_warn(dev, "Unsupported key used: 0x%x\n",
diff --git a/drivers/net/ethernet/chelsio/cxgb4/l2t.c b/drivers/net/ethernet/chelsio/cxgb4/l2t.c
index 1817a0307d26..77c2c538b1fd 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/l2t.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/l2t.c
@@ -491,7 +491,7 @@ u64 cxgb4_select_ntuple(struct net_device *dev,
 	if (tp->protocol_shift >= 0)
 		ntuple |= (u64)IPPROTO_TCP << tp->protocol_shift;
 
-	if (tp->vnic_shift >= 0) {
+	if (tp->vnic_shift >= 0 && (tp->ingress_config & VNIC_F)) {
 		u32 viid = cxgb4_port_viid(dev);
 		u32 vf = FW_VIID_VIN_G(viid);
 		u32 pf = FW_VIID_PFN_G(viid);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c
index 1a28df137e1f..276f22357f81 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c
@@ -1072,12 +1072,27 @@ static void *inline_tx_skb_header(const struct sk_buff *skb,
 static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb)
 {
 	int csum_type;
-	const struct iphdr *iph = ip_hdr(skb);
+	bool inner_hdr_csum = false;
+	u16 proto, ver;
 
-	if (iph->version == 4) {
-		if (iph->protocol == IPPROTO_TCP)
+	if (skb->encapsulation &&
+	    (CHELSIO_CHIP_VERSION(chip) > CHELSIO_T5))
+		inner_hdr_csum = true;
+
+	if (inner_hdr_csum) {
+		ver = inner_ip_hdr(skb)->version;
+		proto = (ver == 4) ? inner_ip_hdr(skb)->protocol :
+			inner_ipv6_hdr(skb)->nexthdr;
+	} else {
+		ver = ip_hdr(skb)->version;
+		proto = (ver == 4) ? ip_hdr(skb)->protocol :
+			ipv6_hdr(skb)->nexthdr;
+	}
+
+	if (ver == 4) {
+		if (proto == IPPROTO_TCP)
 			csum_type = TX_CSUM_TCPIP;
-		else if (iph->protocol == IPPROTO_UDP)
+		else if (proto == IPPROTO_UDP)
 			csum_type = TX_CSUM_UDPIP;
 		else {
 nocsum:			/*
@@ -1090,19 +1105,29 @@ static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb)
 		/*
 		 * this doesn't work with extension headers
 		 */
-		const struct ipv6hdr *ip6h = (const struct ipv6hdr *)iph;
-
-		if (ip6h->nexthdr == IPPROTO_TCP)
+		if (proto == IPPROTO_TCP)
 			csum_type = TX_CSUM_TCPIP6;
-		else if (ip6h->nexthdr == IPPROTO_UDP)
+		else if (proto == IPPROTO_UDP)
 			csum_type = TX_CSUM_UDPIP6;
 		else
 			goto nocsum;
 	}
 
 	if (likely(csum_type >= TX_CSUM_TCPIP)) {
-		u64 hdr_len = TXPKT_IPHDR_LEN_V(skb_network_header_len(skb));
-		int eth_hdr_len = skb_network_offset(skb) - ETH_HLEN;
+		int eth_hdr_len, l4_len;
+		u64 hdr_len;
+
+		if (inner_hdr_csum) {
+			/* This allows checksum offload for all encapsulated
+			 * packets like GRE etc..
+			 */
+			l4_len = skb_inner_network_header_len(skb);
+			eth_hdr_len = skb_inner_network_offset(skb) - ETH_HLEN;
+		} else {
+			l4_len = skb_network_header_len(skb);
+			eth_hdr_len = skb_network_offset(skb) - ETH_HLEN;
+		}
+		hdr_len = TXPKT_IPHDR_LEN_V(l4_len);
 
 		if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5)
 			hdr_len |= TXPKT_ETHHDR_LEN_V(eth_hdr_len);
@@ -1273,7 +1298,7 @@ static inline void t6_fill_tnl_lso(struct sk_buff *skb,
 netdev_tx_t t4_eth_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	u32 wr_mid, ctrl0, op;
-	u64 cntrl, *end;
+	u64 cntrl, *end, *sgl;
 	int qidx, credits;
 	unsigned int flits, ndesc;
 	struct adapter *adap;
@@ -1386,8 +1411,9 @@ out_free:	dev_kfree_skb_any(skb);
 	end = (u64 *)wr + flits;
 
 	len = immediate ? skb->len : 0;
+	len += sizeof(*cpl);
 	if (ssi->gso_size) {
-		struct cpl_tx_pkt_lso *lso = (void *)wr;
+		struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
 		bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
 		int l3hdr_len = skb_network_header_len(skb);
 		int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
@@ -1417,20 +1443,19 @@ out_free:	dev_kfree_skb_any(skb);
 			if (skb->ip_summed == CHECKSUM_PARTIAL)
 				cntrl = hwcsum(adap->params.chip, skb);
 		} else {
-			lso->c.lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
-					  LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F |
-					  LSO_IPV6_V(v6) |
-					  LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
-					  LSO_IPHDR_LEN_V(l3hdr_len / 4) |
-					  LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
-			lso->c.ipid_ofst = htons(0);
-			lso->c.mss = htons(ssi->gso_size);
-			lso->c.seqno_offset = htonl(0);
+			lso->lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
+					LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F |
+					LSO_IPV6_V(v6) |
+					LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
+					LSO_IPHDR_LEN_V(l3hdr_len / 4) |
+					LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
+			lso->ipid_ofst = htons(0);
+			lso->mss = htons(ssi->gso_size);
+			lso->seqno_offset = htonl(0);
 			if (is_t4(adap->params.chip))
-				lso->c.len = htonl(skb->len);
+				lso->len = htonl(skb->len);
 			else
-				lso->c.len =
-					htonl(LSO_T5_XFER_SIZE_V(skb->len));
+				lso->len = htonl(LSO_T5_XFER_SIZE_V(skb->len));
 			cpl = (void *)(lso + 1);
 
 			if (CHELSIO_CHIP_VERSION(adap->params.chip)
@@ -1443,10 +1468,22 @@ out_free:	dev_kfree_skb_any(skb);
 				 TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
 				 TXPKT_IPHDR_LEN_V(l3hdr_len);
 		}
+		sgl = (u64 *)(cpl + 1); /* sgl start here */
+		if (unlikely((u8 *)sgl >= (u8 *)q->q.stat)) {
+			/* If current position is already at the end of the
+			 * txq, reset the current to point to start of the queue
+			 * and update the end ptr as well.
+			 */
+			if (sgl == (u64 *)q->q.stat) {
+				int left = (u8 *)end - (u8 *)q->q.stat;
+
+				end = (void *)q->q.desc + left;
+				sgl = (void *)q->q.desc;
+			}
+		}
 		q->tso++;
 		q->tx_cso += ssi->gso_segs;
 	} else {
-		len += sizeof(*cpl);
 		if (ptp_enabled)
 			op = FW_PTP_TX_PKT_WR;
 		else
@@ -1454,6 +1491,7 @@ out_free:	dev_kfree_skb_any(skb);
 		wr->op_immdlen = htonl(FW_WR_OP_V(op) |
 				       FW_WR_IMMDLEN_V(len));
 		cpl = (void *)(wr + 1);
+		sgl = (u64 *)(cpl + 1);
 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
 			cntrl = hwcsum(adap->params.chip, skb) |
 				TXPKT_IPCSUM_DIS_F;
@@ -1487,20 +1525,19 @@ out_free:	dev_kfree_skb_any(skb);
 	cpl->ctrl1 = cpu_to_be64(cntrl);
 
 	if (immediate) {
-		cxgb4_inline_tx_skb(skb, &q->q, cpl + 1);
+		cxgb4_inline_tx_skb(skb, &q->q, sgl);
 		dev_consume_skb_any(skb);
 	} else {
 		int last_desc;
 
-		cxgb4_write_sgl(skb, &q->q, (struct ulptx_sgl *)(cpl + 1),
-				end, 0, addr);
+		cxgb4_write_sgl(skb, &q->q, (void *)sgl, end, 0, addr);
 		skb_orphan(skb);
 
 		last_desc = q->q.pidx + ndesc - 1;
 		if (last_desc >= q->q.size)
 			last_desc -= q->q.size;
 		q->q.sdesc[last_desc].skb = skb;
-		q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)(cpl + 1);
+		q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)sgl;
 	}
 
 	txq_advance(&q->q, ndesc);
@@ -2259,7 +2296,7 @@ static void cxgb4_sgetim_to_hwtstamp(struct adapter *adap,
 }
 
 static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
-		   const struct cpl_rx_pkt *pkt)
+		   const struct cpl_rx_pkt *pkt, unsigned long tnl_hdr_len)
 {
 	struct adapter *adapter = rxq->rspq.adap;
 	struct sge *s = &adapter->sge;
@@ -2275,6 +2312,8 @@ static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
 	}
 
 	copy_frags(skb, gl, s->pktshift);
+	if (tnl_hdr_len)
+		skb->csum_level = 1;
 	skb->len = gl->tot_len - s->pktshift;
 	skb->data_len = skb->len;
 	skb->truesize += skb->data_len;
@@ -2406,7 +2445,7 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
 	struct sge *s = &q->adap->sge;
 	int cpl_trace_pkt = is_t4(q->adap->params.chip) ?
 			    CPL_TRACE_PKT : CPL_TRACE_PKT_T5;
-	u16 err_vec;
+	u16 err_vec, tnl_hdr_len = 0;
 	struct port_info *pi;
 	int ret = 0;
 
@@ -2415,16 +2454,19 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
 
 	pkt = (const struct cpl_rx_pkt *)rsp;
 	/* Compressed error vector is enabled for T6 only */
-	if (q->adap->params.tp.rx_pkt_encap)
+	if (q->adap->params.tp.rx_pkt_encap) {
 		err_vec = T6_COMPR_RXERR_VEC_G(be16_to_cpu(pkt->err_vec));
-	else
+		tnl_hdr_len = T6_RX_TNLHDR_LEN_G(ntohs(pkt->err_vec));
+	} else {
 		err_vec = be16_to_cpu(pkt->err_vec);
+	}
 
 	csum_ok = pkt->csum_calc && !err_vec &&
 		  (q->netdev->features & NETIF_F_RXCSUM);
-	if ((pkt->l2info & htonl(RXF_TCP_F)) &&
+	if (((pkt->l2info & htonl(RXF_TCP_F)) ||
+	     tnl_hdr_len) &&
 	    (q->netdev->features & NETIF_F_GRO) && csum_ok && !pkt->ip_frag) {
-		do_gro(rxq, si, pkt);
+		do_gro(rxq, si, pkt, tnl_hdr_len);
 		return 0;
 	}
 
@@ -2471,7 +2513,13 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
 		} else if (pkt->l2info & htonl(RXF_IP_F)) {
 			__sum16 c = (__force __sum16)pkt->csum;
 			skb->csum = csum_unfold(c);
-			skb->ip_summed = CHECKSUM_COMPLETE;
+
+			if (tnl_hdr_len) {
+				skb->ip_summed = CHECKSUM_UNNECESSARY;
+				skb->csum_level = 1;
+			} else {
+				skb->ip_summed = CHECKSUM_COMPLETE;
+			}
 			rxq->stats.rx_cso++;
 		}
 	} else {
diff --git a/drivers/net/ethernet/chelsio/cxgb4/srq.c b/drivers/net/ethernet/chelsio/cxgb4/srq.c
index 6228a5708307..82b70a565e24 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/srq.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/srq.c
@@ -84,8 +84,7 @@ int cxgb4_get_srq_entry(struct net_device *dev,
 	if (!skb)
 		return -ENOMEM;
 	req = (struct cpl_srq_table_req *)
-		__skb_put(skb, sizeof(*req));
-	memset(req, 0, sizeof(*req));
+		__skb_put_zero(skb, sizeof(*req));
 	INIT_TP_WR(req, 0);
 	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SRQ_TABLE_REQ,
 					      TID_TID_V(srq_idx) |
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_chip_type.h b/drivers/net/ethernet/chelsio/cxgb4/t4_chip_type.h
index 54b718111e3f..721c77577ec5 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_chip_type.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_chip_type.h
@@ -34,6 +34,8 @@
 #ifndef __T4_CHIP_TYPE_H__
 #define __T4_CHIP_TYPE_H__
 
+#define CHELSIO_PCI_ID_VER(__DeviceID)  ((__DeviceID) >> 12)
+
 #define CHELSIO_T4		0x4
 #define CHELSIO_T5		0x5
 #define CHELSIO_T6		0x6
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
index 7cb3ef466cc7..39da7e3c804b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
@@ -3941,8 +3941,8 @@ static fw_port_cap32_t fwcaps16_to_caps32(fw_port_cap16_t caps16)
 	CAP16_TO_CAP32(FC_RX);
 	CAP16_TO_CAP32(FC_TX);
 	CAP16_TO_CAP32(ANEG);
-	CAP16_TO_CAP32(MDIX);
 	CAP16_TO_CAP32(MDIAUTO);
+	CAP16_TO_CAP32(MDISTRAIGHT);
 	CAP16_TO_CAP32(FEC_RS);
 	CAP16_TO_CAP32(FEC_BASER_RS);
 	CAP16_TO_CAP32(802_3_PAUSE);
@@ -3982,8 +3982,8 @@ static fw_port_cap16_t fwcaps32_to_caps16(fw_port_cap32_t caps32)
 	CAP32_TO_CAP16(802_3_PAUSE);
 	CAP32_TO_CAP16(802_3_ASM_DIR);
 	CAP32_TO_CAP16(ANEG);
-	CAP32_TO_CAP16(MDIX);
 	CAP32_TO_CAP16(MDIAUTO);
+	CAP32_TO_CAP16(MDISTRAIGHT);
 	CAP32_TO_CAP16(FEC_RS);
 	CAP32_TO_CAP16(FEC_BASER_RS);
 
@@ -4058,14 +4058,17 @@ static inline fw_port_cap32_t cc_to_fwcap_fec(enum cc_fec cc_fec)
  *	- If auto-negotiation is off set the MAC to the proper speed/duplex/FC,
  *	  otherwise do it later based on the outcome of auto-negotiation.
  */
-int t4_link_l1cfg(struct adapter *adapter, unsigned int mbox,
-		  unsigned int port, struct link_config *lc)
+int t4_link_l1cfg_core(struct adapter *adapter, unsigned int mbox,
+		       unsigned int port, struct link_config *lc,
+		       bool sleep_ok, int timeout)
 {
 	unsigned int fw_caps = adapter->params.fw_caps_support;
-	struct fw_port_cmd cmd;
-	unsigned int fw_mdi = FW_PORT_CAP32_MDI_V(FW_PORT_CAP32_MDI_AUTO);
 	fw_port_cap32_t fw_fc, cc_fec, fw_fec, rcap;
+	struct fw_port_cmd cmd;
+	unsigned int fw_mdi;
+	int ret;
 
+	fw_mdi = (FW_PORT_CAP32_MDI_V(FW_PORT_CAP32_MDI_AUTO) & lc->pcaps);
 	/* Convert driver coding of Pause Frame Flow Control settings into the
 	 * Firmware's API.
 	 */
@@ -4087,7 +4090,7 @@ int t4_link_l1cfg(struct adapter *adapter, unsigned int mbox,
 	/* Figure out what our Requested Port Capabilities are going to be.
 	 */
 	if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
-		rcap = (lc->pcaps & ADVERT_MASK) | fw_fc | fw_fec;
+		rcap = lc->acaps | fw_fc | fw_fec;
 		lc->fc = lc->requested_fc & ~PAUSE_AUTONEG;
 		lc->fec = cc_fec;
 	} else if (lc->autoneg == AUTONEG_DISABLE) {
@@ -4098,6 +4101,13 @@ int t4_link_l1cfg(struct adapter *adapter, unsigned int mbox,
 		rcap = lc->acaps | fw_fc | fw_fec | fw_mdi;
 	}
 
+	if (rcap & ~lc->pcaps) {
+		dev_err(adapter->pdev_dev,
+			"Requested Port Capabilities %#x exceed Physical Port Capabilities %#x\n",
+			rcap, lc->pcaps);
+		return -EINVAL;
+	}
+
 	/* And send that on to the Firmware ...
 	 */
 	memset(&cmd, 0, sizeof(cmd));
@@ -4108,12 +4118,21 @@ int t4_link_l1cfg(struct adapter *adapter, unsigned int mbox,
 		cpu_to_be32(FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16
 						 ? FW_PORT_ACTION_L1_CFG
 						 : FW_PORT_ACTION_L1_CFG32) |
-			    FW_LEN16(cmd));
+						 FW_LEN16(cmd));
 	if (fw_caps == FW_CAPS16)
 		cmd.u.l1cfg.rcap = cpu_to_be32(fwcaps32_to_caps16(rcap));
 	else
 		cmd.u.l1cfg32.rcap32 = cpu_to_be32(rcap);
-	return t4_wr_mbox(adapter, mbox, &cmd, sizeof(cmd), NULL);
+
+	ret = t4_wr_mbox_meat_timeout(adapter, mbox, &cmd, sizeof(cmd), NULL,
+				      sleep_ok, timeout);
+	if (ret) {
+		dev_err(adapter->pdev_dev,
+			"Requested Port Capabilities %#x rejected, error %d\n",
+			rcap, -ret);
+		return ret;
+	}
+	return ret;
 }
 
 /**
@@ -7512,6 +7531,43 @@ int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
 	return t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok);
 }
 
+/**
+ *      t4_free_encap_mac_filt - frees MPS entry at given index
+ *      @adap: the adapter
+ *      @viid: the VI id
+ *      @idx: index of MPS entry to be freed
+ *      @sleep_ok: call is allowed to sleep
+ *
+ *      Frees the MPS entry at supplied index
+ *
+ *      Returns a negative error number or zero on success
+ */
+int t4_free_encap_mac_filt(struct adapter *adap, unsigned int viid,
+			   int idx, bool sleep_ok)
+{
+	struct fw_vi_mac_exact *p;
+	u8 addr[] = {0, 0, 0, 0, 0, 0};
+	struct fw_vi_mac_cmd c;
+	int ret = 0;
+	u32 exact;
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
+				   FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
+				   FW_CMD_EXEC_V(0) |
+				   FW_VI_MAC_CMD_VIID_V(viid));
+	exact = FW_VI_MAC_CMD_ENTRY_TYPE_V(FW_VI_MAC_TYPE_EXACTMAC);
+	c.freemacs_to_len16 = cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(0) |
+					  exact |
+					  FW_CMD_LEN16_V(1));
+	p = c.u.exact;
+	p->valid_to_idx = cpu_to_be16(FW_VI_MAC_CMD_VALID_F |
+				      FW_VI_MAC_CMD_IDX_V(idx));
+	memcpy(p->macaddr, addr, sizeof(p->macaddr));
+	ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok);
+	return ret;
+}
+
 /**
  *	t4_free_raw_mac_filt - Frees a raw mac entry in mps tcam
  *	@adap: the adapter
@@ -7562,6 +7618,55 @@ int t4_free_raw_mac_filt(struct adapter *adap, unsigned int viid,
 	return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok);
 }
 
+/**
+ *      t4_alloc_encap_mac_filt - Adds a mac entry in mps tcam with VNI support
+ *      @adap: the adapter
+ *      @viid: the VI id
+ *      @mac: the MAC address
+ *      @mask: the mask
+ *      @vni: the VNI id for the tunnel protocol
+ *      @vni_mask: mask for the VNI id
+ *      @dip_hit: to enable DIP match for the MPS entry
+ *      @lookup_type: MAC address for inner (1) or outer (0) header
+ *      @sleep_ok: call is allowed to sleep
+ *
+ *      Allocates an MPS entry with specified MAC address and VNI value.
+ *
+ *      Returns a negative error number or the allocated index for this mac.
+ */
+int t4_alloc_encap_mac_filt(struct adapter *adap, unsigned int viid,
+			    const u8 *addr, const u8 *mask, unsigned int vni,
+			    unsigned int vni_mask, u8 dip_hit, u8 lookup_type,
+			    bool sleep_ok)
+{
+	struct fw_vi_mac_cmd c;
+	struct fw_vi_mac_vni *p = c.u.exact_vni;
+	int ret = 0;
+	u32 val;
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
+				   FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
+				   FW_VI_MAC_CMD_VIID_V(viid));
+	val = FW_CMD_LEN16_V(1) |
+	      FW_VI_MAC_CMD_ENTRY_TYPE_V(FW_VI_MAC_TYPE_EXACTMAC_VNI);
+	c.freemacs_to_len16 = cpu_to_be32(val);
+	p->valid_to_idx = cpu_to_be16(FW_VI_MAC_CMD_VALID_F |
+				      FW_VI_MAC_CMD_IDX_V(FW_VI_MAC_ADD_MAC));
+	memcpy(p->macaddr, addr, sizeof(p->macaddr));
+	memcpy(p->macaddr_mask, mask, sizeof(p->macaddr_mask));
+
+	p->lookup_type_to_vni =
+		cpu_to_be32(FW_VI_MAC_CMD_VNI_V(vni) |
+			    FW_VI_MAC_CMD_DIP_HIT_V(dip_hit) |
+			    FW_VI_MAC_CMD_LOOKUP_TYPE_V(lookup_type));
+	p->vni_mask_pkd = cpu_to_be32(FW_VI_MAC_CMD_VNI_MASK_V(vni_mask));
+	ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok);
+	if (ret == 0)
+		ret = FW_VI_MAC_CMD_IDX_G(be16_to_cpu(p->valid_to_idx));
+	return ret;
+}
+
 /**
  *	t4_alloc_raw_mac_filt - Adds a mac entry in mps tcam
  *	@adap: the adapter
@@ -7908,6 +8013,34 @@ int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid,
 	return t4_enable_vi_params(adap, mbox, viid, rx_en, tx_en, 0);
 }
 
+/**
+ *	t4_enable_pi_params - enable/disable a Port's Virtual Interface
+ *      @adap: the adapter
+ *      @mbox: mailbox to use for the FW command
+ *      @pi: the Port Information structure
+ *      @rx_en: 1=enable Rx, 0=disable Rx
+ *      @tx_en: 1=enable Tx, 0=disable Tx
+ *      @dcb_en: 1=enable delivery of Data Center Bridging messages.
+ *
+ *      Enables/disables a Port's Virtual Interface.  Note that setting DCB
+ *	Enable only makes sense when enabling a Virtual Interface ...
+ *	If the Virtual Interface enable/disable operation is successful,
+ *	we notify the OS-specific code of a potential Link Status change
+ *	via the OS Contract API t4_os_link_changed().
+ */
+int t4_enable_pi_params(struct adapter *adap, unsigned int mbox,
+			struct port_info *pi,
+			bool rx_en, bool tx_en, bool dcb_en)
+{
+	int ret = t4_enable_vi_params(adap, mbox, pi->viid,
+				      rx_en, tx_en, dcb_en);
+	if (ret)
+		return ret;
+	t4_os_link_changed(adap, pi->port_id,
+			   rx_en && tx_en && pi->link_cfg.link_ok);
+	return 0;
+}
+
 /**
  *	t4_identify_port - identify a VI's port by blinking its LED
  *	@adap: the adapter
@@ -8249,6 +8382,9 @@ void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl)
 	fc = fwcap_to_cc_pause(linkattr);
 	speed = fwcap_to_speed(linkattr);
 
+	lc->new_module = false;
+	lc->redo_l1cfg = false;
+
 	if (mod_type != pi->mod_type) {
 		/* With the newer SFP28 and QSFP28 Transceiver Module Types,
 		 * various fundamental Port Capabilities which used to be
@@ -8283,6 +8419,8 @@ void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl)
 		pi->port_type = port_type;
 
 		pi->mod_type = mod_type;
+
+		lc->new_module = t4_is_inserted_mod_type(mod_type);
 		t4_os_portmod_changed(adapter, pi->port_id);
 	}
 
@@ -8301,7 +8439,9 @@ void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl)
 		lc->lpacaps = lpacaps;
 		lc->acaps = acaps & ADVERT_MASK;
 
-		if (lc->acaps & FW_PORT_CAP32_ANEG) {
+		if (!(lc->acaps & FW_PORT_CAP32_ANEG)) {
+			lc->autoneg = AUTONEG_DISABLE;
+		} else if (lc->acaps & FW_PORT_CAP32_ANEG) {
 			lc->autoneg = AUTONEG_ENABLE;
 		} else {
 			/* When Autoneg is disabled, user needs to set
@@ -8315,6 +8455,26 @@ void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl)
 
 		t4_os_link_changed(adapter, pi->port_id, link_ok);
 	}
+
+	if (lc->new_module && lc->redo_l1cfg) {
+		struct link_config old_lc;
+		int ret;
+
+		/* Save the current L1 Configuration and restore it if an
+		 * error occurs.  We probably should fix the l1_cfg*()
+		 * routines not to change the link_config when an error
+		 * occurs ...
+		 */
+		old_lc = *lc;
+		ret = t4_link_l1cfg_ns(adapter, adapter->mbox, pi->lport, lc);
+		if (ret) {
+			*lc = old_lc;
+			dev_warn(adapter->pdev_dev,
+				 "Attempt to update new Transceiver Module settings failed\n");
+		}
+	}
+	lc->new_module = false;
+	lc->redo_l1cfg = false;
 }
 
 /**
@@ -8486,6 +8646,13 @@ static void init_link_config(struct link_config *lc, fw_port_cap32_t pcaps,
 	lc->requested_fec = FEC_AUTO;
 	lc->fec = fwcap_to_cc_fec(lc->def_acaps);
 
+	/* If the Port is capable of Auto-Negtotiation, initialize it as
+	 * "enabled" and copy over all of the Physical Port Capabilities
+	 * to the Advertised Port Capabilities.  Otherwise mark it as
+	 * Auto-Negotiate disabled and select the highest supported speed
+	 * for the link.  Note parallel structure in t4_link_l1cfg_core()
+	 * and t4_handle_get_port_info().
+	 */
 	if (lc->pcaps & FW_PORT_CAP32_ANEG) {
 		lc->acaps = lc->pcaps & ADVERT_MASK;
 		lc->autoneg = AUTONEG_ENABLE;
@@ -8493,6 +8660,7 @@ static void init_link_config(struct link_config *lc, fw_port_cap32_t pcaps,
 	} else {
 		lc->acaps = 0;
 		lc->autoneg = AUTONEG_DISABLE;
+		lc->speed_caps = fwcap_to_fwspeed(acaps);
 	}
 }
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
index fe2029e993a2..09e38f0733bd 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
@@ -1233,6 +1233,11 @@ struct cpl_rx_pkt {
 #define T6_COMPR_RXERR_SUM_V(x) ((x) << T6_COMPR_RXERR_SUM_S)
 #define T6_COMPR_RXERR_SUM_F    T6_COMPR_RXERR_SUM_V(1U)
 
+#define T6_RX_TNLHDR_LEN_S    8
+#define T6_RX_TNLHDR_LEN_M    0xFF
+#define T6_RX_TNLHDR_LEN_V(x) ((x) << T6_RX_TNLHDR_LEN_S)
+#define T6_RX_TNLHDR_LEN_G(x) (((x) >> T6_RX_TNLHDR_LEN_S) & T6_RX_TNLHDR_LEN_M)
+
 struct cpl_trace_pkt {
 	u8 opcode;
 	u8 intf;
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h b/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h
index 51b18035d691..c7f8d0441278 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h
@@ -145,6 +145,9 @@ CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN
 	CH_PCI_ID_TABLE_FENTRY(0x5016),	/* T580-OCP-SO */
 	CH_PCI_ID_TABLE_FENTRY(0x5017),	/* T520-OCP-SO */
 	CH_PCI_ID_TABLE_FENTRY(0x5018),	/* T540-BT */
+	CH_PCI_ID_TABLE_FENTRY(0x5019),	/* T540-LP-BT */
+	CH_PCI_ID_TABLE_FENTRY(0x501a),	/* T540-SO-BT */
+	CH_PCI_ID_TABLE_FENTRY(0x501b),	/* T540-SO-CR */
 	CH_PCI_ID_TABLE_FENTRY(0x5080),	/* Custom T540-cr */
 	CH_PCI_ID_TABLE_FENTRY(0x5081),	/* Custom T540-LL-cr */
 	CH_PCI_ID_TABLE_FENTRY(0x5082),	/* Custom T504-cr */
@@ -184,6 +187,7 @@ CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN
 	CH_PCI_ID_TABLE_FENTRY(0x50aa), /* Custom T580-CR */
 	CH_PCI_ID_TABLE_FENTRY(0x50ab), /* Custom T520-CR */
 	CH_PCI_ID_TABLE_FENTRY(0x50ac), /* Custom T540-BT */
+	CH_PCI_ID_TABLE_FENTRY(0x50ad), /* Custom T520-CR */
 
 	/* T6 adapters:
 	 */
@@ -208,6 +212,8 @@ CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN
 	CH_PCI_ID_TABLE_FENTRY(0x6085), /* Custom T6240-SO */
 	CH_PCI_ID_TABLE_FENTRY(0x6086), /* Custom T6225-SO-CR */
 	CH_PCI_ID_TABLE_FENTRY(0x6087), /* Custom T6225-CR */
+	CH_PCI_ID_TABLE_FENTRY(0x6088), /* Custom T62100-CR */
+	CH_PCI_ID_TABLE_FENTRY(0x6089), /* Custom T62100-KR */
 CH_PCI_DEVICE_ID_TABLE_DEFINE_END;
 
 #endif /* __T4_PCI_ID_TBL_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h b/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h
index 276fdf214b75..6b55aa2eb2a5 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h
@@ -1598,6 +1598,10 @@
 #define VNIC_V(x) ((x) << VNIC_S)
 #define VNIC_F    VNIC_V(1U)
 
+#define USE_ENC_IDX_S		13
+#define USE_ENC_IDX_V(x)	((x) << USE_ENC_IDX_S)
+#define USE_ENC_IDX_F		USE_ENC_IDX_V(1U)
+
 #define CSUM_HAS_PSEUDO_HDR_S    10
 #define CSUM_HAS_PSEUDO_HDR_V(x) ((x) << CSUM_HAS_PSEUDO_HDR_S)
 #define CSUM_HAS_PSEUDO_HDR_F    CSUM_HAS_PSEUDO_HDR_V(1U)
@@ -2995,6 +2999,7 @@
 #define LE_DB_HASH_TID_BASE_A 0x19c30
 #define LE_DB_HASH_TBL_BASE_ADDR_A 0x19c30
 #define LE_DB_INT_CAUSE_A 0x19c3c
+#define LE_DB_CLCAM_TID_BASE_A 0x19df4
 #define LE_DB_TID_HASHBASE_A 0x19df8
 #define T6_LE_DB_HASH_TID_BASE_A 0x19df8
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
index e3d4751f21ac..2d91480a5a0e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
@@ -1305,6 +1305,8 @@ enum fw_params_param_pfvf {
 	FW_PARAMS_PARAM_PFVF_HPFILTER_END = 0x33,
 	FW_PARAMS_PARAM_PFVF_TLS_START = 0x34,
 	FW_PARAMS_PARAM_PFVF_TLS_END = 0x35,
+	FW_PARAMS_PARAM_PFVF_RAWF_START = 0x36,
+	FW_PARAMS_PARAM_PFVF_RAWF_END = 0x37,
 	FW_PARAMS_PARAM_PFVF_NCRYPTO_LOOKASIDE = 0x39,
 	FW_PARAMS_PARAM_PFVF_PORT_CAPS32 = 0x3A,
 };
@@ -2156,6 +2158,14 @@ struct fw_vi_mac_cmd {
 			__be64 data0m_pkd;
 			__be32 data1m[2];
 		} raw;
+		struct fw_vi_mac_vni {
+			__be16 valid_to_idx;
+			__u8 macaddr[6];
+			__be16 r7;
+			__u8 macaddr_mask[6];
+			__be32 lookup_type_to_vni;
+			__be32 vni_mask_pkd;
+		} exact_vni[2];
 	} u;
 };
 
@@ -2203,6 +2213,32 @@ struct fw_vi_mac_cmd {
 #define FW_VI_MAC_CMD_RAW_IDX_G(x)      \
 	(((x) >> FW_VI_MAC_CMD_RAW_IDX_S) & FW_VI_MAC_CMD_RAW_IDX_M)
 
+#define FW_VI_MAC_CMD_LOOKUP_TYPE_S	31
+#define FW_VI_MAC_CMD_LOOKUP_TYPE_M	0x1
+#define FW_VI_MAC_CMD_LOOKUP_TYPE_V(x)	((x) << FW_VI_MAC_CMD_LOOKUP_TYPE_S)
+#define FW_VI_MAC_CMD_LOOKUP_TYPE_G(x)	\
+	(((x) >> FW_VI_MAC_CMD_LOOKUP_TYPE_S) & FW_VI_MAC_CMD_LOOKUP_TYPE_M)
+#define FW_VI_MAC_CMD_LOOKUP_TYPE_F	FW_VI_MAC_CMD_LOOKUP_TYPE_V(1U)
+
+#define FW_VI_MAC_CMD_DIP_HIT_S		30
+#define FW_VI_MAC_CMD_DIP_HIT_M		0x1
+#define FW_VI_MAC_CMD_DIP_HIT_V(x)	((x) << FW_VI_MAC_CMD_DIP_HIT_S)
+#define FW_VI_MAC_CMD_DIP_HIT_G(x)	\
+	(((x) >> FW_VI_MAC_CMD_DIP_HIT_S) & FW_VI_MAC_CMD_DIP_HIT_M)
+#define FW_VI_MAC_CMD_DIP_HIT_F		FW_VI_MAC_CMD_DIP_HIT_V(1U)
+
+#define FW_VI_MAC_CMD_VNI_S		0
+#define FW_VI_MAC_CMD_VNI_M		0xffffff
+#define FW_VI_MAC_CMD_VNI_V(x)		((x) << FW_VI_MAC_CMD_VNI_S)
+#define FW_VI_MAC_CMD_VNI_G(x)		\
+	(((x) >> FW_VI_MAC_CMD_VNI_S) & FW_VI_MAC_CMD_VNI_M)
+
+#define FW_VI_MAC_CMD_VNI_MASK_S	0
+#define FW_VI_MAC_CMD_VNI_MASK_M	0xffffff
+#define FW_VI_MAC_CMD_VNI_MASK_V(x)	((x) << FW_VI_MAC_CMD_VNI_MASK_S)
+#define FW_VI_MAC_CMD_VNI_MASK_G(x)	\
+	(((x) >> FW_VI_MAC_CMD_VNI_MASK_S) & FW_VI_MAC_CMD_VNI_MASK_M)
+
 #define FW_RXMODE_MTU_NO_CHG	65535
 
 struct fw_vi_rxmode_cmd {
@@ -2435,8 +2471,8 @@ enum fw_port_cap {
 	FW_PORT_CAP_FC_RX		= 0x0040,
 	FW_PORT_CAP_FC_TX		= 0x0080,
 	FW_PORT_CAP_ANEG		= 0x0100,
-	FW_PORT_CAP_MDIX		= 0x0200,
-	FW_PORT_CAP_MDIAUTO		= 0x0400,
+	FW_PORT_CAP_MDIAUTO		= 0x0200,
+	FW_PORT_CAP_MDISTRAIGHT		= 0x0400,
 	FW_PORT_CAP_FEC_RS		= 0x0800,
 	FW_PORT_CAP_FEC_BASER_RS	= 0x1000,
 	FW_PORT_CAP_FEC_RESERVED	= 0x2000,
@@ -2479,8 +2515,8 @@ enum fw_port_mdi {
 #define	FW_PORT_CAP32_802_3_PAUSE	0x00040000UL
 #define	FW_PORT_CAP32_802_3_ASM_DIR	0x00080000UL
 #define	FW_PORT_CAP32_ANEG		0x00100000UL
-#define	FW_PORT_CAP32_MDIX		0x00200000UL
-#define	FW_PORT_CAP32_MDIAUTO		0x00400000UL
+#define	FW_PORT_CAP32_MDIAUTO		0x00200000UL
+#define	FW_PORT_CAP32_MDISTRAIGHT	0x00400000UL
 #define	FW_PORT_CAP32_FEC_RS		0x00800000UL
 #define	FW_PORT_CAP32_FEC_BASER_RS	0x01000000UL
 #define	FW_PORT_CAP32_FEC_RESERVED1	0x02000000UL
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_version.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_version.h
index 123e2c1b65f5..4eb15ceddca3 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_version.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_version.h
@@ -36,8 +36,8 @@
 #define __T4FW_VERSION_H__
 
 #define T4FW_VERSION_MAJOR 0x01
-#define T4FW_VERSION_MINOR 0x10
-#define T4FW_VERSION_MICRO 0x3F
+#define T4FW_VERSION_MINOR 0x13
+#define T4FW_VERSION_MICRO 0x01
 #define T4FW_VERSION_BUILD 0x00
 
 #define T4FW_MIN_VERSION_MAJOR 0x01
@@ -45,8 +45,8 @@
 #define T4FW_MIN_VERSION_MICRO 0x00
 
 #define T5FW_VERSION_MAJOR 0x01
-#define T5FW_VERSION_MINOR 0x10
-#define T5FW_VERSION_MICRO 0x3F
+#define T5FW_VERSION_MINOR 0x13
+#define T5FW_VERSION_MICRO 0x01
 #define T5FW_VERSION_BUILD 0x00
 
 #define T5FW_MIN_VERSION_MAJOR 0x00
@@ -54,8 +54,8 @@
 #define T5FW_MIN_VERSION_MICRO 0x00
 
 #define T6FW_VERSION_MAJOR 0x01
-#define T6FW_VERSION_MINOR 0x10
-#define T6FW_VERSION_MICRO 0x3F
+#define T6FW_VERSION_MINOR 0x13
+#define T6FW_VERSION_MICRO 0x01
 #define T6FW_VERSION_BUILD 0x00
 
 #define T6FW_MIN_VERSION_MAJOR 0x00
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
index 9a81b52307a9..ff84791a0ff8 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
@@ -274,7 +274,7 @@ static int link_start(struct net_device *dev)
 	 * is enabled on a port.
 	 */
 	if (ret == 0)
-		ret = t4vf_enable_vi(pi->adapter, pi->viid, true, true);
+		ret = t4vf_enable_pi(pi->adapter, pi, true, true);
 
 	/* The Virtual Interfaces are connected to an internal switch on the
 	 * chip which allows VIs attached to the same port to talk to each
@@ -822,8 +822,7 @@ static int cxgb4vf_stop(struct net_device *dev)
 
 	netif_tx_stop_all_queues(dev);
 	netif_carrier_off(dev);
-	t4vf_enable_vi(adapter, pi->viid, false, false);
-	pi->link_cfg.link_ok = 0;
+	t4vf_enable_pi(adapter, pi, false, false);
 
 	clear_bit(pi->port_id, &adapter->open_device_map);
 	if (adapter->open_device_map == 0)
@@ -1419,6 +1418,22 @@ static int cxgb4vf_get_link_ksettings(struct net_device *dev,
 		base->duplex = DUPLEX_UNKNOWN;
 	}
 
+	if (pi->link_cfg.fc & PAUSE_RX) {
+		if (pi->link_cfg.fc & PAUSE_TX) {
+			ethtool_link_ksettings_add_link_mode(link_ksettings,
+							     advertising,
+							     Pause);
+		} else {
+			ethtool_link_ksettings_add_link_mode(link_ksettings,
+							     advertising,
+							     Asym_Pause);
+		}
+	} else if (pi->link_cfg.fc & PAUSE_TX) {
+		ethtool_link_ksettings_add_link_mode(link_ksettings,
+						     advertising,
+						     Asym_Pause);
+	}
+
 	base->autoneg = pi->link_cfg.autoneg;
 	if (pi->link_cfg.pcaps & FW_PORT_CAP32_ANEG)
 		ethtool_link_ksettings_add_link_mode(link_ksettings,
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
index 712e8f0c71b4..ccca67cf4487 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
@@ -391,7 +391,10 @@ int t4vf_config_rss_range(struct adapter *, unsigned int, int, int,
 
 int t4vf_alloc_vi(struct adapter *, int);
 int t4vf_free_vi(struct adapter *, int);
-int t4vf_enable_vi(struct adapter *, unsigned int, bool, bool);
+int t4vf_enable_vi(struct adapter *adapter, unsigned int viid, bool rx_en,
+		   bool tx_en);
+int t4vf_enable_pi(struct adapter *adapter, struct port_info *pi, bool rx_en,
+		   bool tx_en);
 int t4vf_identify_port(struct adapter *, unsigned int, unsigned int);
 
 int t4vf_set_rxmode(struct adapter *, unsigned int, int, int, int, int, int,
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
index 798695bf8678..5b8c08cf523f 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
@@ -341,8 +341,8 @@ static fw_port_cap32_t fwcaps16_to_caps32(fw_port_cap16_t caps16)
 	CAP16_TO_CAP32(FC_RX);
 	CAP16_TO_CAP32(FC_TX);
 	CAP16_TO_CAP32(ANEG);
-	CAP16_TO_CAP32(MDIX);
 	CAP16_TO_CAP32(MDIAUTO);
+	CAP16_TO_CAP32(MDISTRAIGHT);
 	CAP16_TO_CAP32(FEC_RS);
 	CAP16_TO_CAP32(FEC_BASER_RS);
 	CAP16_TO_CAP32(802_3_PAUSE);
@@ -405,6 +405,36 @@ static unsigned int fwcap_to_speed(fw_port_cap32_t caps)
 	return 0;
 }
 
+/**
+ *      fwcap_to_fwspeed - return highest speed in Port Capabilities
+ *      @acaps: advertised Port Capabilities
+ *
+ *      Get the highest speed for the port from the advertised Port
+ *      Capabilities.  It will be either the highest speed from the list of
+ *      speeds or whatever user has set using ethtool.
+ */
+static fw_port_cap32_t fwcap_to_fwspeed(fw_port_cap32_t acaps)
+{
+	#define TEST_SPEED_RETURN(__caps_speed) \
+		do { \
+			if (acaps & FW_PORT_CAP32_SPEED_##__caps_speed) \
+				return FW_PORT_CAP32_SPEED_##__caps_speed; \
+		} while (0)
+
+	TEST_SPEED_RETURN(400G);
+	TEST_SPEED_RETURN(200G);
+	TEST_SPEED_RETURN(100G);
+	TEST_SPEED_RETURN(50G);
+	TEST_SPEED_RETURN(40G);
+	TEST_SPEED_RETURN(25G);
+	TEST_SPEED_RETURN(10G);
+	TEST_SPEED_RETURN(1G);
+	TEST_SPEED_RETURN(100M);
+
+	#undef TEST_SPEED_RETURN
+	return 0;
+}
+
 /*
  *	init_link_config - initialize a link's SW state
  *	@lc: structure holding the link state
@@ -431,6 +461,13 @@ static void init_link_config(struct link_config *lc,
 	lc->requested_fec = FEC_AUTO;
 	lc->fec = lc->auto_fec;
 
+	/* If the Port is capable of Auto-Negtotiation, initialize it as
+	 * "enabled" and copy over all of the Physical Port Capabilities
+	 * to the Advertised Port Capabilities.  Otherwise mark it as
+	 * Auto-Negotiate disabled and select the highest supported speed
+	 * for the link.  Note parallel structure in t4_link_l1cfg_core()
+	 * and t4_handle_get_port_info().
+	 */
 	if (lc->pcaps & FW_PORT_CAP32_ANEG) {
 		lc->acaps = acaps & ADVERT_MASK;
 		lc->autoneg = AUTONEG_ENABLE;
@@ -438,6 +475,7 @@ static void init_link_config(struct link_config *lc,
 	} else {
 		lc->acaps = 0;
 		lc->autoneg = AUTONEG_DISABLE;
+		lc->speed_caps = fwcap_to_fwspeed(acaps);
 	}
 }
 
@@ -1361,6 +1399,30 @@ int t4vf_enable_vi(struct adapter *adapter, unsigned int viid,
 	return t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), NULL);
 }
 
+/**
+ *	t4vf_enable_pi - enable/disable a Port's virtual interface
+ *	@adapter: the adapter
+ *	@pi: the Port Information structure
+ *	@rx_en: 1=enable Rx, 0=disable Rx
+ *	@tx_en: 1=enable Tx, 0=disable Tx
+ *
+ *	Enables/disables a Port's virtual interface.  If the Virtual
+ *	Interface enable/disable operation is successful, we notify the
+ *	OS-specific code of a potential Link Status change via the OS Contract
+ *	API t4vf_os_link_changed().
+ */
+int t4vf_enable_pi(struct adapter *adapter, struct port_info *pi,
+		   bool rx_en, bool tx_en)
+{
+	int ret = t4vf_enable_vi(adapter, pi->viid, rx_en, tx_en);
+
+	if (ret)
+		return ret;
+	t4vf_os_link_changed(adapter, pi->pidx,
+			     rx_en && tx_en && pi->link_cfg.link_ok);
+	return 0;
+}
+
 /**
  *	t4vf_identify_port - identify a VI's port by blinking its LED
  *	@adapter: the adapter
@@ -1955,7 +2017,14 @@ static void t4vf_handle_get_port_info(struct port_info *pi,
 		lc->lpacaps = lpacaps;
 		lc->acaps = acaps & ADVERT_MASK;
 
-		if (lc->acaps & FW_PORT_CAP32_ANEG) {
+		/* If we're not physically capable of Auto-Negotiation, note
+		 * this as Auto-Negotiation disabled.  Otherwise, we track
+		 * what Auto-Negotiation settings we have.  Note parallel
+		 * structure in init_link_config().
+		 */
+		if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
+			lc->autoneg = AUTONEG_DISABLE;
+		} else if (lc->acaps & FW_PORT_CAP32_ANEG) {
 			lc->autoneg = AUTONEG_ENABLE;
 		} else {
 			/* When Autoneg is disabled, user needs to set
diff --git a/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.h b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.h
index 4b5aacc09cab..240ba9d4c399 100644
--- a/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.h
+++ b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.h
@@ -90,8 +90,7 @@ cxgb_mk_tid_release(struct sk_buff *skb, u32 len, u32 tid, u16 chan)
 {
 	struct cpl_tid_release *req;
 
-	req = __skb_put(skb, len);
-	memset(req, 0, len);
+	req = __skb_put_zero(skb, len);
 
 	INIT_TP_WR(req, tid);
 	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
@@ -104,8 +103,7 @@ cxgb_mk_close_con_req(struct sk_buff *skb, u32 len, u32 tid, u16 chan,
 {
 	struct cpl_close_con_req *req;
 
-	req = __skb_put(skb, len);
-	memset(req, 0, len);
+	req = __skb_put_zero(skb, len);
 
 	INIT_TP_WR(req, tid);
 	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
@@ -119,8 +117,7 @@ cxgb_mk_abort_req(struct sk_buff *skb, u32 len, u32 tid, u16 chan,
 {
 	struct cpl_abort_req *req;
 
-	req = __skb_put(skb, len);
-	memset(req, 0, len);
+	req = __skb_put_zero(skb, len);
 
 	INIT_TP_WR(req, tid);
 	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
@@ -134,8 +131,7 @@ cxgb_mk_abort_rpl(struct sk_buff *skb, u32 len, u32 tid, u16 chan)
 {
 	struct cpl_abort_rpl *rpl;
 
-	rpl = __skb_put(skb, len);
-	memset(rpl, 0, len);
+	rpl = __skb_put_zero(skb, len);
 
 	INIT_TP_WR(rpl, tid);
 	OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
@@ -149,8 +145,7 @@ cxgb_mk_rx_data_ack(struct sk_buff *skb, u32 len, u32 tid, u16 chan,
 {
 	struct cpl_rx_data_ack *req;
 
-	req = __skb_put(skb, len);
-	memset(req, 0, len);
+	req = __skb_put_zero(skb, len);
 
 	INIT_TP_WR(req, tid);
 	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, tid));
diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c
index c697e79e491e..8f755009ff38 100644
--- a/drivers/net/ethernet/emulex/benet/be_main.c
+++ b/drivers/net/ethernet/emulex/benet/be_main.c
@@ -3309,7 +3309,9 @@ void be_detect_error(struct be_adapter *adapter)
 				if ((val & POST_STAGE_FAT_LOG_START)
 				     != POST_STAGE_FAT_LOG_START &&
 				    (val & POST_STAGE_ARMFW_UE)
-				     != POST_STAGE_ARMFW_UE)
+				     != POST_STAGE_ARMFW_UE &&
+				    (val & POST_STAGE_RECOVERABLE_ERR)
+				     != POST_STAGE_RECOVERABLE_ERR)
 					return;
 			}
 
diff --git a/drivers/net/ethernet/ethoc.c b/drivers/net/ethernet/ethoc.c
index 8bb0db990c8f..00a57273b753 100644
--- a/drivers/net/ethernet/ethoc.c
+++ b/drivers/net/ethernet/ethoc.c
@@ -1246,8 +1246,7 @@ static int ethoc_probe(struct platform_device *pdev)
 	mdiobus_unregister(priv->mdio);
 	mdiobus_free(priv->mdio);
 free2:
-	if (priv->clk)
-		clk_disable_unprepare(priv->clk);
+	clk_disable_unprepare(priv->clk);
 free:
 	free_netdev(netdev);
 out:
@@ -1271,8 +1270,7 @@ static int ethoc_remove(struct platform_device *pdev)
 			mdiobus_unregister(priv->mdio);
 			mdiobus_free(priv->mdio);
 		}
-		if (priv->clk)
-			clk_disable_unprepare(priv->clk);
+		clk_disable_unprepare(priv->clk);
 		unregister_netdev(netdev);
 		free_netdev(netdev);
 	}
diff --git a/drivers/net/ethernet/freescale/Kconfig b/drivers/net/ethernet/freescale/Kconfig
index 6e490fd2345d..a580a3dcbe59 100644
--- a/drivers/net/ethernet/freescale/Kconfig
+++ b/drivers/net/ethernet/freescale/Kconfig
@@ -22,7 +22,7 @@ if NET_VENDOR_FREESCALE
 config FEC
 	tristate "FEC ethernet controller (of ColdFire and some i.MX CPUs)"
 	depends on (M523x || M527x || M5272 || M528x || M520x || M532x || \
-		   ARCH_MXC || SOC_IMX28)
+		   ARCH_MXC || SOC_IMX28 || COMPILE_TEST)
 	default ARCH_MXC || SOC_IMX28 if ARM
 	select PHYLIB
 	imply PTP_1588_CLOCK
diff --git a/drivers/net/ethernet/freescale/Makefile b/drivers/net/ethernet/freescale/Makefile
index ed8ad0fefbda..0914a3ea4405 100644
--- a/drivers/net/ethernet/freescale/Makefile
+++ b/drivers/net/ethernet/freescale/Makefile
@@ -14,7 +14,6 @@ obj-$(CONFIG_FS_ENET) += fs_enet/
 obj-$(CONFIG_FSL_PQ_MDIO) += fsl_pq_mdio.o
 obj-$(CONFIG_FSL_XGMAC_MDIO) += xgmac_mdio.o
 obj-$(CONFIG_GIANFAR) += gianfar_driver.o
-obj-$(CONFIG_PTP_1588_CLOCK_GIANFAR) += gianfar_ptp.o
 gianfar_driver-objs := gianfar.o \
 		gianfar_ethtool.o
 obj-$(CONFIG_UCC_GETH) += ucc_geth_driver.o
diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h
index e7381f8ef89d..4778b663653e 100644
--- a/drivers/net/ethernet/freescale/fec.h
+++ b/drivers/net/ethernet/freescale/fec.h
@@ -21,7 +21,7 @@
 
 #if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
     defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \
-    defined(CONFIG_ARM64)
+    defined(CONFIG_ARM64) || defined(CONFIG_COMPILE_TEST)
 /*
  *	Just figures, Motorola would have to change the offsets for
  *	registers in the same peripheral device on different models
diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c
index 9d3eed46830d..c729665107f5 100644
--- a/drivers/net/ethernet/freescale/fec_main.c
+++ b/drivers/net/ethernet/freescale/fec_main.c
@@ -2053,13 +2053,9 @@ static int fec_enet_mii_init(struct platform_device *pdev)
 	fep->mii_bus->parent = &pdev->dev;
 
 	node = of_get_child_by_name(pdev->dev.of_node, "mdio");
-	if (node) {
-		err = of_mdiobus_register(fep->mii_bus, node);
+	err = of_mdiobus_register(fep->mii_bus, node);
+	if (node)
 		of_node_put(node);
-	} else {
-		err = mdiobus_register(fep->mii_bus);
-	}
-
 	if (err)
 		goto err_out_free_mdiobus;
 
@@ -2112,7 +2108,7 @@ static int fec_enet_get_regs_len(struct net_device *ndev)
 /* List of registers that can be safety be read to dump them with ethtool */
 #if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
 	defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \
-	defined(CONFIG_ARM64)
+	defined(CONFIG_ARM64) || defined(CONFIG_COMPILE_TEST)
 static u32 fec_enet_register_offset[] = {
 	FEC_IEVENT, FEC_IMASK, FEC_R_DES_ACTIVE_0, FEC_X_DES_ACTIVE_0,
 	FEC_ECNTRL, FEC_MII_DATA, FEC_MII_SPEED, FEC_MIB_CTRLSTAT, FEC_R_CNTRL,
@@ -3518,7 +3514,7 @@ fec_probe(struct platform_device *pdev)
 		goto failed_init;
 
 	for (i = 0; i < irq_cnt; i++) {
-		sprintf(irq_name, "int%d", i);
+		snprintf(irq_name, sizeof(irq_name), "int%d", i);
 		irq = platform_get_irq_byname(pdev, irq_name);
 		if (irq < 0)
 			irq = platform_get_irq(pdev, i);
diff --git a/drivers/net/ethernet/freescale/fec_ptp.c b/drivers/net/ethernet/freescale/fec_ptp.c
index 43d973215040..36c2d7d6ee1b 100644
--- a/drivers/net/ethernet/freescale/fec_ptp.c
+++ b/drivers/net/ethernet/freescale/fec_ptp.c
@@ -454,12 +454,6 @@ static int fec_ptp_enable(struct ptp_clock_info *ptp,
 	return -EOPNOTSUPP;
 }
 
-/**
- * fec_ptp_hwtstamp_ioctl - control hardware time stamping
- * @ndev: pointer to net_device
- * @ifreq: ioctl data
- * @cmd: particular ioctl requested
- */
 int fec_ptp_set(struct net_device *ndev, struct ifreq *ifr)
 {
 	struct fec_enet_private *fep = netdev_priv(ndev);
diff --git a/drivers/net/ethernet/freescale/fman/fman_port.c b/drivers/net/ethernet/freescale/fman/fman_port.c
index 6552d68ea6e1..ce6e24c74978 100644
--- a/drivers/net/ethernet/freescale/fman/fman_port.c
+++ b/drivers/net/ethernet/freescale/fman/fman_port.c
@@ -1391,12 +1391,10 @@ int fman_port_config(struct fman_port *port, struct fman_port_params *params)
 		/* FM_WRONG_RESET_VALUES_ERRATA_FMAN_A005127 Errata
 		 * workaround
 		 */
-		if (port->rev_info.major >= 6) {
-			u32 reg;
+		u32 reg;
 
-			reg = 0x00001013;
-			iowrite32be(reg, &port->bmi_regs->tx.fmbm_tfp);
-		}
+		reg = 0x00001013;
+		iowrite32be(reg, &port->bmi_regs->tx.fmbm_tfp);
 	}
 
 	return 0;
diff --git a/drivers/net/ethernet/freescale/gianfar.h b/drivers/net/ethernet/freescale/gianfar.h
index 5aa814799d70..8e42c0246611 100644
--- a/drivers/net/ethernet/freescale/gianfar.h
+++ b/drivers/net/ethernet/freescale/gianfar.h
@@ -1372,7 +1372,4 @@ struct filer_table {
 	struct gfar_filer_entry fe[MAX_FILER_CACHE_IDX + 20];
 };
 
-/* The gianfar_ptp module will set this variable */
-extern int gfar_phc_index;
-
 #endif /* __GIANFAR_H */
diff --git a/drivers/net/ethernet/freescale/gianfar_ethtool.c b/drivers/net/ethernet/freescale/gianfar_ethtool.c
index a93e0199c369..8cb98cae0a6f 100644
--- a/drivers/net/ethernet/freescale/gianfar_ethtool.c
+++ b/drivers/net/ethernet/freescale/gianfar_ethtool.c
@@ -41,6 +41,8 @@
 #include <linux/phy.h>
 #include <linux/sort.h>
 #include <linux/if_vlan.h>
+#include <linux/of_platform.h>
+#include <linux/fsl/ptp_qoriq.h>
 
 #include "gianfar.h"
 
@@ -1509,24 +1511,35 @@ static int gfar_get_nfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
 	return ret;
 }
 
-int gfar_phc_index = -1;
-EXPORT_SYMBOL(gfar_phc_index);
-
 static int gfar_get_ts_info(struct net_device *dev,
 			    struct ethtool_ts_info *info)
 {
 	struct gfar_private *priv = netdev_priv(dev);
+	struct platform_device *ptp_dev;
+	struct device_node *ptp_node;
+	struct qoriq_ptp *ptp = NULL;
+
+	info->phc_index = -1;
 
 	if (!(priv->device_flags & FSL_GIANFAR_DEV_HAS_TIMER)) {
 		info->so_timestamping = SOF_TIMESTAMPING_RX_SOFTWARE |
 					SOF_TIMESTAMPING_SOFTWARE;
-		info->phc_index = -1;
 		return 0;
 	}
+
+	ptp_node = of_find_compatible_node(NULL, NULL, "fsl,etsec-ptp");
+	if (ptp_node) {
+		ptp_dev = of_find_device_by_node(ptp_node);
+		if (ptp_dev)
+			ptp = platform_get_drvdata(ptp_dev);
+	}
+
+	if (ptp)
+		info->phc_index = ptp->phc_index;
+
 	info->so_timestamping = SOF_TIMESTAMPING_TX_HARDWARE |
 				SOF_TIMESTAMPING_RX_HARDWARE |
 				SOF_TIMESTAMPING_RAW_HARDWARE;
-	info->phc_index = gfar_phc_index;
 	info->tx_types = (1 << HWTSTAMP_TX_OFF) |
 			 (1 << HWTSTAMP_TX_ON);
 	info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) |
diff --git a/drivers/net/ethernet/freescale/gianfar_ptp.c b/drivers/net/ethernet/freescale/gianfar_ptp.c
deleted file mode 100644
index 9f8d4f8e57e3..000000000000
--- a/drivers/net/ethernet/freescale/gianfar_ptp.c
+++ /dev/null
@@ -1,572 +0,0 @@
-/*
- * PTP 1588 clock using the eTSEC
- *
- * Copyright (C) 2010 OMICRON electronics GmbH
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/device.h>
-#include <linux/hrtimer.h>
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_platform.h>
-#include <linux/timex.h>
-#include <linux/io.h>
-
-#include <linux/ptp_clock_kernel.h>
-
-#include "gianfar.h"
-
-/*
- * gianfar ptp registers
- * Generated by regen.tcl on Thu May 13 01:38:57 PM CEST 2010
- */
-struct gianfar_ptp_registers {
-	u32 tmr_ctrl;     /* Timer control register */
-	u32 tmr_tevent;   /* Timestamp event register */
-	u32 tmr_temask;   /* Timer event mask register */
-	u32 tmr_pevent;   /* Timestamp event register */
-	u32 tmr_pemask;   /* Timer event mask register */
-	u32 tmr_stat;     /* Timestamp status register */
-	u32 tmr_cnt_h;    /* Timer counter high register */
-	u32 tmr_cnt_l;    /* Timer counter low register */
-	u32 tmr_add;      /* Timer drift compensation addend register */
-	u32 tmr_acc;      /* Timer accumulator register */
-	u32 tmr_prsc;     /* Timer prescale */
-	u8  res1[4];
-	u32 tmroff_h;     /* Timer offset high */
-	u32 tmroff_l;     /* Timer offset low */
-	u8  res2[8];
-	u32 tmr_alarm1_h; /* Timer alarm 1 high register */
-	u32 tmr_alarm1_l; /* Timer alarm 1 high register */
-	u32 tmr_alarm2_h; /* Timer alarm 2 high register */
-	u32 tmr_alarm2_l; /* Timer alarm 2 high register */
-	u8  res3[48];
-	u32 tmr_fiper1;   /* Timer fixed period interval */
-	u32 tmr_fiper2;   /* Timer fixed period interval */
-	u32 tmr_fiper3;   /* Timer fixed period interval */
-	u8  res4[20];
-	u32 tmr_etts1_h;  /* Timestamp of general purpose external trigger */
-	u32 tmr_etts1_l;  /* Timestamp of general purpose external trigger */
-	u32 tmr_etts2_h;  /* Timestamp of general purpose external trigger */
-	u32 tmr_etts2_l;  /* Timestamp of general purpose external trigger */
-};
-
-/* Bit definitions for the TMR_CTRL register */
-#define ALM1P                 (1<<31) /* Alarm1 output polarity */
-#define ALM2P                 (1<<30) /* Alarm2 output polarity */
-#define FIPERST               (1<<28) /* FIPER start indication */
-#define PP1L                  (1<<27) /* Fiper1 pulse loopback mode enabled. */
-#define PP2L                  (1<<26) /* Fiper2 pulse loopback mode enabled. */
-#define TCLK_PERIOD_SHIFT     (16) /* 1588 timer reference clock period. */
-#define TCLK_PERIOD_MASK      (0x3ff)
-#define RTPE                  (1<<15) /* Record Tx Timestamp to PAL Enable. */
-#define FRD                   (1<<14) /* FIPER Realignment Disable */
-#define ESFDP                 (1<<11) /* External Tx/Rx SFD Polarity. */
-#define ESFDE                 (1<<10) /* External Tx/Rx SFD Enable. */
-#define ETEP2                 (1<<9) /* External trigger 2 edge polarity */
-#define ETEP1                 (1<<8) /* External trigger 1 edge polarity */
-#define COPH                  (1<<7) /* Generated clock output phase. */
-#define CIPH                  (1<<6) /* External oscillator input clock phase */
-#define TMSR                  (1<<5) /* Timer soft reset. */
-#define BYP                   (1<<3) /* Bypass drift compensated clock */
-#define TE                    (1<<2) /* 1588 timer enable. */
-#define CKSEL_SHIFT           (0)    /* 1588 Timer reference clock source */
-#define CKSEL_MASK            (0x3)
-
-/* Bit definitions for the TMR_TEVENT register */
-#define ETS2                  (1<<25) /* External trigger 2 timestamp sampled */
-#define ETS1                  (1<<24) /* External trigger 1 timestamp sampled */
-#define ALM2                  (1<<17) /* Current time = alarm time register 2 */
-#define ALM1                  (1<<16) /* Current time = alarm time register 1 */
-#define PP1                   (1<<7)  /* periodic pulse generated on FIPER1 */
-#define PP2                   (1<<6)  /* periodic pulse generated on FIPER2 */
-#define PP3                   (1<<5)  /* periodic pulse generated on FIPER3 */
-
-/* Bit definitions for the TMR_TEMASK register */
-#define ETS2EN                (1<<25) /* External trigger 2 timestamp enable */
-#define ETS1EN                (1<<24) /* External trigger 1 timestamp enable */
-#define ALM2EN                (1<<17) /* Timer ALM2 event enable */
-#define ALM1EN                (1<<16) /* Timer ALM1 event enable */
-#define PP1EN                 (1<<7) /* Periodic pulse event 1 enable */
-#define PP2EN                 (1<<6) /* Periodic pulse event 2 enable */
-
-/* Bit definitions for the TMR_PEVENT register */
-#define TXP2                  (1<<9) /* PTP transmitted timestamp im TXTS2 */
-#define TXP1                  (1<<8) /* PTP transmitted timestamp in TXTS1 */
-#define RXP                   (1<<0) /* PTP frame has been received */
-
-/* Bit definitions for the TMR_PEMASK register */
-#define TXP2EN                (1<<9) /* Transmit PTP packet event 2 enable */
-#define TXP1EN                (1<<8) /* Transmit PTP packet event 1 enable */
-#define RXPEN                 (1<<0) /* Receive PTP packet event enable */
-
-/* Bit definitions for the TMR_STAT register */
-#define STAT_VEC_SHIFT        (0) /* Timer general purpose status vector */
-#define STAT_VEC_MASK         (0x3f)
-
-/* Bit definitions for the TMR_PRSC register */
-#define PRSC_OCK_SHIFT        (0) /* Output clock division/prescale factor. */
-#define PRSC_OCK_MASK         (0xffff)
-
-
-#define DRIVER		"gianfar_ptp"
-#define DEFAULT_CKSEL	1
-#define N_EXT_TS	2
-#define REG_SIZE	sizeof(struct gianfar_ptp_registers)
-
-struct etsects {
-	struct gianfar_ptp_registers __iomem *regs;
-	spinlock_t lock; /* protects regs */
-	struct ptp_clock *clock;
-	struct ptp_clock_info caps;
-	struct resource *rsrc;
-	int irq;
-	u64 alarm_interval; /* for periodic alarm */
-	u64 alarm_value;
-	u32 tclk_period;  /* nanoseconds */
-	u32 tmr_prsc;
-	u32 tmr_add;
-	u32 cksel;
-	u32 tmr_fiper1;
-	u32 tmr_fiper2;
-};
-
-/*
- * Register access functions
- */
-
-/* Caller must hold etsects->lock. */
-static u64 tmr_cnt_read(struct etsects *etsects)
-{
-	u64 ns;
-	u32 lo, hi;
-
-	lo = gfar_read(&etsects->regs->tmr_cnt_l);
-	hi = gfar_read(&etsects->regs->tmr_cnt_h);
-	ns = ((u64) hi) << 32;
-	ns |= lo;
-	return ns;
-}
-
-/* Caller must hold etsects->lock. */
-static void tmr_cnt_write(struct etsects *etsects, u64 ns)
-{
-	u32 hi = ns >> 32;
-	u32 lo = ns & 0xffffffff;
-
-	gfar_write(&etsects->regs->tmr_cnt_l, lo);
-	gfar_write(&etsects->regs->tmr_cnt_h, hi);
-}
-
-/* Caller must hold etsects->lock. */
-static void set_alarm(struct etsects *etsects)
-{
-	u64 ns;
-	u32 lo, hi;
-
-	ns = tmr_cnt_read(etsects) + 1500000000ULL;
-	ns = div_u64(ns, 1000000000UL) * 1000000000ULL;
-	ns -= etsects->tclk_period;
-	hi = ns >> 32;
-	lo = ns & 0xffffffff;
-	gfar_write(&etsects->regs->tmr_alarm1_l, lo);
-	gfar_write(&etsects->regs->tmr_alarm1_h, hi);
-}
-
-/* Caller must hold etsects->lock. */
-static void set_fipers(struct etsects *etsects)
-{
-	set_alarm(etsects);
-	gfar_write(&etsects->regs->tmr_fiper1, etsects->tmr_fiper1);
-	gfar_write(&etsects->regs->tmr_fiper2, etsects->tmr_fiper2);
-}
-
-/*
- * Interrupt service routine
- */
-
-static irqreturn_t isr(int irq, void *priv)
-{
-	struct etsects *etsects = priv;
-	struct ptp_clock_event event;
-	u64 ns;
-	u32 ack = 0, lo, hi, mask, val;
-
-	val = gfar_read(&etsects->regs->tmr_tevent);
-
-	if (val & ETS1) {
-		ack |= ETS1;
-		hi = gfar_read(&etsects->regs->tmr_etts1_h);
-		lo = gfar_read(&etsects->regs->tmr_etts1_l);
-		event.type = PTP_CLOCK_EXTTS;
-		event.index = 0;
-		event.timestamp = ((u64) hi) << 32;
-		event.timestamp |= lo;
-		ptp_clock_event(etsects->clock, &event);
-	}
-
-	if (val & ETS2) {
-		ack |= ETS2;
-		hi = gfar_read(&etsects->regs->tmr_etts2_h);
-		lo = gfar_read(&etsects->regs->tmr_etts2_l);
-		event.type = PTP_CLOCK_EXTTS;
-		event.index = 1;
-		event.timestamp = ((u64) hi) << 32;
-		event.timestamp |= lo;
-		ptp_clock_event(etsects->clock, &event);
-	}
-
-	if (val & ALM2) {
-		ack |= ALM2;
-		if (etsects->alarm_value) {
-			event.type = PTP_CLOCK_ALARM;
-			event.index = 0;
-			event.timestamp = etsects->alarm_value;
-			ptp_clock_event(etsects->clock, &event);
-		}
-		if (etsects->alarm_interval) {
-			ns = etsects->alarm_value + etsects->alarm_interval;
-			hi = ns >> 32;
-			lo = ns & 0xffffffff;
-			spin_lock(&etsects->lock);
-			gfar_write(&etsects->regs->tmr_alarm2_l, lo);
-			gfar_write(&etsects->regs->tmr_alarm2_h, hi);
-			spin_unlock(&etsects->lock);
-			etsects->alarm_value = ns;
-		} else {
-			gfar_write(&etsects->regs->tmr_tevent, ALM2);
-			spin_lock(&etsects->lock);
-			mask = gfar_read(&etsects->regs->tmr_temask);
-			mask &= ~ALM2EN;
-			gfar_write(&etsects->regs->tmr_temask, mask);
-			spin_unlock(&etsects->lock);
-			etsects->alarm_value = 0;
-			etsects->alarm_interval = 0;
-		}
-	}
-
-	if (val & PP1) {
-		ack |= PP1;
-		event.type = PTP_CLOCK_PPS;
-		ptp_clock_event(etsects->clock, &event);
-	}
-
-	if (ack) {
-		gfar_write(&etsects->regs->tmr_tevent, ack);
-		return IRQ_HANDLED;
-	} else
-		return IRQ_NONE;
-}
-
-/*
- * PTP clock operations
- */
-
-static int ptp_gianfar_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
-{
-	u64 adj, diff;
-	u32 tmr_add;
-	int neg_adj = 0;
-	struct etsects *etsects = container_of(ptp, struct etsects, caps);
-
-	if (scaled_ppm < 0) {
-		neg_adj = 1;
-		scaled_ppm = -scaled_ppm;
-	}
-	tmr_add = etsects->tmr_add;
-	adj = tmr_add;
-
-	/* calculate diff as adj*(scaled_ppm/65536)/1000000
-	 * and round() to the nearest integer
-	 */
-	adj *= scaled_ppm;
-	diff = div_u64(adj, 8000000);
-	diff = (diff >> 13) + ((diff >> 12) & 1);
-
-	tmr_add = neg_adj ? tmr_add - diff : tmr_add + diff;
-
-	gfar_write(&etsects->regs->tmr_add, tmr_add);
-
-	return 0;
-}
-
-static int ptp_gianfar_adjtime(struct ptp_clock_info *ptp, s64 delta)
-{
-	s64 now;
-	unsigned long flags;
-	struct etsects *etsects = container_of(ptp, struct etsects, caps);
-
-	spin_lock_irqsave(&etsects->lock, flags);
-
-	now = tmr_cnt_read(etsects);
-	now += delta;
-	tmr_cnt_write(etsects, now);
-	set_fipers(etsects);
-
-	spin_unlock_irqrestore(&etsects->lock, flags);
-
-	return 0;
-}
-
-static int ptp_gianfar_gettime(struct ptp_clock_info *ptp,
-			       struct timespec64 *ts)
-{
-	u64 ns;
-	unsigned long flags;
-	struct etsects *etsects = container_of(ptp, struct etsects, caps);
-
-	spin_lock_irqsave(&etsects->lock, flags);
-
-	ns = tmr_cnt_read(etsects);
-
-	spin_unlock_irqrestore(&etsects->lock, flags);
-
-	*ts = ns_to_timespec64(ns);
-
-	return 0;
-}
-
-static int ptp_gianfar_settime(struct ptp_clock_info *ptp,
-			       const struct timespec64 *ts)
-{
-	u64 ns;
-	unsigned long flags;
-	struct etsects *etsects = container_of(ptp, struct etsects, caps);
-
-	ns = timespec64_to_ns(ts);
-
-	spin_lock_irqsave(&etsects->lock, flags);
-
-	tmr_cnt_write(etsects, ns);
-	set_fipers(etsects);
-
-	spin_unlock_irqrestore(&etsects->lock, flags);
-
-	return 0;
-}
-
-static int ptp_gianfar_enable(struct ptp_clock_info *ptp,
-			      struct ptp_clock_request *rq, int on)
-{
-	struct etsects *etsects = container_of(ptp, struct etsects, caps);
-	unsigned long flags;
-	u32 bit, mask;
-
-	switch (rq->type) {
-	case PTP_CLK_REQ_EXTTS:
-		switch (rq->extts.index) {
-		case 0:
-			bit = ETS1EN;
-			break;
-		case 1:
-			bit = ETS2EN;
-			break;
-		default:
-			return -EINVAL;
-		}
-		spin_lock_irqsave(&etsects->lock, flags);
-		mask = gfar_read(&etsects->regs->tmr_temask);
-		if (on)
-			mask |= bit;
-		else
-			mask &= ~bit;
-		gfar_write(&etsects->regs->tmr_temask, mask);
-		spin_unlock_irqrestore(&etsects->lock, flags);
-		return 0;
-
-	case PTP_CLK_REQ_PPS:
-		spin_lock_irqsave(&etsects->lock, flags);
-		mask = gfar_read(&etsects->regs->tmr_temask);
-		if (on)
-			mask |= PP1EN;
-		else
-			mask &= ~PP1EN;
-		gfar_write(&etsects->regs->tmr_temask, mask);
-		spin_unlock_irqrestore(&etsects->lock, flags);
-		return 0;
-
-	default:
-		break;
-	}
-
-	return -EOPNOTSUPP;
-}
-
-static const struct ptp_clock_info ptp_gianfar_caps = {
-	.owner		= THIS_MODULE,
-	.name		= "gianfar clock",
-	.max_adj	= 512000,
-	.n_alarm	= 0,
-	.n_ext_ts	= N_EXT_TS,
-	.n_per_out	= 0,
-	.n_pins		= 0,
-	.pps		= 1,
-	.adjfine	= ptp_gianfar_adjfine,
-	.adjtime	= ptp_gianfar_adjtime,
-	.gettime64	= ptp_gianfar_gettime,
-	.settime64	= ptp_gianfar_settime,
-	.enable		= ptp_gianfar_enable,
-};
-
-static int gianfar_ptp_probe(struct platform_device *dev)
-{
-	struct device_node *node = dev->dev.of_node;
-	struct etsects *etsects;
-	struct timespec64 now;
-	int err = -ENOMEM;
-	u32 tmr_ctrl;
-	unsigned long flags;
-
-	etsects = kzalloc(sizeof(*etsects), GFP_KERNEL);
-	if (!etsects)
-		goto no_memory;
-
-	err = -ENODEV;
-
-	etsects->caps = ptp_gianfar_caps;
-
-	if (of_property_read_u32(node, "fsl,cksel", &etsects->cksel))
-		etsects->cksel = DEFAULT_CKSEL;
-
-	if (of_property_read_u32(node,
-				 "fsl,tclk-period", &etsects->tclk_period) ||
-	    of_property_read_u32(node,
-				 "fsl,tmr-prsc", &etsects->tmr_prsc) ||
-	    of_property_read_u32(node,
-				 "fsl,tmr-add", &etsects->tmr_add) ||
-	    of_property_read_u32(node,
-				 "fsl,tmr-fiper1", &etsects->tmr_fiper1) ||
-	    of_property_read_u32(node,
-				 "fsl,tmr-fiper2", &etsects->tmr_fiper2) ||
-	    of_property_read_u32(node,
-				 "fsl,max-adj", &etsects->caps.max_adj)) {
-		pr_err("device tree node missing required elements\n");
-		goto no_node;
-	}
-
-	etsects->irq = platform_get_irq(dev, 0);
-
-	if (etsects->irq < 0) {
-		pr_err("irq not in device tree\n");
-		goto no_node;
-	}
-	if (request_irq(etsects->irq, isr, 0, DRIVER, etsects)) {
-		pr_err("request_irq failed\n");
-		goto no_node;
-	}
-
-	etsects->rsrc = platform_get_resource(dev, IORESOURCE_MEM, 0);
-	if (!etsects->rsrc) {
-		pr_err("no resource\n");
-		goto no_resource;
-	}
-	if (request_resource(&iomem_resource, etsects->rsrc)) {
-		pr_err("resource busy\n");
-		goto no_resource;
-	}
-
-	spin_lock_init(&etsects->lock);
-
-	etsects->regs = ioremap(etsects->rsrc->start,
-				resource_size(etsects->rsrc));
-	if (!etsects->regs) {
-		pr_err("ioremap ptp registers failed\n");
-		goto no_ioremap;
-	}
-	getnstimeofday64(&now);
-	ptp_gianfar_settime(&etsects->caps, &now);
-
-	tmr_ctrl =
-	  (etsects->tclk_period & TCLK_PERIOD_MASK) << TCLK_PERIOD_SHIFT |
-	  (etsects->cksel & CKSEL_MASK) << CKSEL_SHIFT;
-
-	spin_lock_irqsave(&etsects->lock, flags);
-
-	gfar_write(&etsects->regs->tmr_ctrl,   tmr_ctrl);
-	gfar_write(&etsects->regs->tmr_add,    etsects->tmr_add);
-	gfar_write(&etsects->regs->tmr_prsc,   etsects->tmr_prsc);
-	gfar_write(&etsects->regs->tmr_fiper1, etsects->tmr_fiper1);
-	gfar_write(&etsects->regs->tmr_fiper2, etsects->tmr_fiper2);
-	set_alarm(etsects);
-	gfar_write(&etsects->regs->tmr_ctrl,   tmr_ctrl|FIPERST|RTPE|TE|FRD);
-
-	spin_unlock_irqrestore(&etsects->lock, flags);
-
-	etsects->clock = ptp_clock_register(&etsects->caps, &dev->dev);
-	if (IS_ERR(etsects->clock)) {
-		err = PTR_ERR(etsects->clock);
-		goto no_clock;
-	}
-	gfar_phc_index = ptp_clock_index(etsects->clock);
-
-	platform_set_drvdata(dev, etsects);
-
-	return 0;
-
-no_clock:
-	iounmap(etsects->regs);
-no_ioremap:
-	release_resource(etsects->rsrc);
-no_resource:
-	free_irq(etsects->irq, etsects);
-no_node:
-	kfree(etsects);
-no_memory:
-	return err;
-}
-
-static int gianfar_ptp_remove(struct platform_device *dev)
-{
-	struct etsects *etsects = platform_get_drvdata(dev);
-
-	gfar_write(&etsects->regs->tmr_temask, 0);
-	gfar_write(&etsects->regs->tmr_ctrl,   0);
-
-	gfar_phc_index = -1;
-	ptp_clock_unregister(etsects->clock);
-	iounmap(etsects->regs);
-	release_resource(etsects->rsrc);
-	free_irq(etsects->irq, etsects);
-	kfree(etsects);
-
-	return 0;
-}
-
-static const struct of_device_id match_table[] = {
-	{ .compatible = "fsl,etsec-ptp" },
-	{},
-};
-MODULE_DEVICE_TABLE(of, match_table);
-
-static struct platform_driver gianfar_ptp_driver = {
-	.driver = {
-		.name		= "gianfar_ptp",
-		.of_match_table	= match_table,
-	},
-	.probe       = gianfar_ptp_probe,
-	.remove      = gianfar_ptp_remove,
-};
-
-module_platform_driver(gianfar_ptp_driver);
-
-MODULE_AUTHOR("Richard Cochran <richardcochran@gmail.com>");
-MODULE_DESCRIPTION("PTP clock using the eTSEC");
-MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.c b/drivers/net/ethernet/hisilicon/hns3/hnae3.c
index 02145f2de820..9d79dad2c6aa 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hnae3.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.c
@@ -36,6 +36,49 @@ static bool hnae3_client_match(enum hnae3_client_type client_type,
 	return false;
 }
 
+static void hnae3_set_client_init_flag(struct hnae3_client *client,
+				       struct hnae3_ae_dev *ae_dev, int inited)
+{
+	switch (client->type) {
+	case HNAE3_CLIENT_KNIC:
+		hnae_set_bit(ae_dev->flag, HNAE3_KNIC_CLIENT_INITED_B, inited);
+		break;
+	case HNAE3_CLIENT_UNIC:
+		hnae_set_bit(ae_dev->flag, HNAE3_UNIC_CLIENT_INITED_B, inited);
+		break;
+	case HNAE3_CLIENT_ROCE:
+		hnae_set_bit(ae_dev->flag, HNAE3_ROCE_CLIENT_INITED_B, inited);
+		break;
+	default:
+		break;
+	}
+}
+
+static int hnae3_get_client_init_flag(struct hnae3_client *client,
+				       struct hnae3_ae_dev *ae_dev)
+{
+	int inited = 0;
+
+	switch (client->type) {
+	case HNAE3_CLIENT_KNIC:
+		inited = hnae_get_bit(ae_dev->flag,
+				       HNAE3_KNIC_CLIENT_INITED_B);
+		break;
+	case HNAE3_CLIENT_UNIC:
+		inited = hnae_get_bit(ae_dev->flag,
+				       HNAE3_UNIC_CLIENT_INITED_B);
+		break;
+	case HNAE3_CLIENT_ROCE:
+		inited = hnae_get_bit(ae_dev->flag,
+				      HNAE3_ROCE_CLIENT_INITED_B);
+		break;
+	default:
+		break;
+	}
+
+	return inited;
+}
+
 static int hnae3_match_n_instantiate(struct hnae3_client *client,
 				     struct hnae3_ae_dev *ae_dev, bool is_reg)
 {
@@ -50,13 +93,22 @@ static int hnae3_match_n_instantiate(struct hnae3_client *client,
 	/* now, (un-)instantiate client by calling lower layer */
 	if (is_reg) {
 		ret = ae_dev->ops->init_client_instance(client, ae_dev);
-		if (ret)
+		if (ret) {
 			dev_err(&ae_dev->pdev->dev,
 				"fail to instantiate client\n");
-		return ret;
+			return ret;
+		}
+
+		hnae3_set_client_init_flag(client, ae_dev, 1);
+		return 0;
+	}
+
+	if (hnae3_get_client_init_flag(client, ae_dev)) {
+		ae_dev->ops->uninit_client_instance(client, ae_dev);
+
+		hnae3_set_client_init_flag(client, ae_dev, 0);
 	}
 
-	ae_dev->ops->uninit_client_instance(client, ae_dev);
 	return 0;
 }
 
@@ -89,7 +141,7 @@ int hnae3_register_client(struct hnae3_client *client)
 exit:
 	mutex_unlock(&hnae3_common_lock);
 
-	return ret;
+	return 0;
 }
 EXPORT_SYMBOL(hnae3_register_client);
 
@@ -112,7 +164,7 @@ EXPORT_SYMBOL(hnae3_unregister_client);
  * @ae_algo: AE algorithm
  * NOTE: the duplicated name will not be checked
  */
-int hnae3_register_ae_algo(struct hnae3_ae_algo *ae_algo)
+void hnae3_register_ae_algo(struct hnae3_ae_algo *ae_algo)
 {
 	const struct pci_device_id *id;
 	struct hnae3_ae_dev *ae_dev;
@@ -151,8 +203,6 @@ int hnae3_register_ae_algo(struct hnae3_ae_algo *ae_algo)
 	}
 
 	mutex_unlock(&hnae3_common_lock);
-
-	return ret;
 }
 EXPORT_SYMBOL(hnae3_register_ae_algo);
 
@@ -168,6 +218,9 @@ void hnae3_unregister_ae_algo(struct hnae3_ae_algo *ae_algo)
 	mutex_lock(&hnae3_common_lock);
 	/* Check if there are matched ae_dev */
 	list_for_each_entry(ae_dev, &hnae3_ae_dev_list, node) {
+		if (!hnae_get_bit(ae_dev->flag, HNAE3_DEV_INITED_B))
+			continue;
+
 		id = pci_match_id(ae_algo->pdev_id_table, ae_dev->pdev);
 		if (!id)
 			continue;
@@ -191,22 +244,14 @@ EXPORT_SYMBOL(hnae3_unregister_ae_algo);
  * @ae_dev: the AE device
  * NOTE: the duplicated name will not be checked
  */
-int hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev)
+void hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev)
 {
 	const struct pci_device_id *id;
 	struct hnae3_ae_algo *ae_algo;
 	struct hnae3_client *client;
-	int ret = 0, lock_acquired;
+	int ret = 0;
 
-	/* we can get deadlocked if SRIOV is being enabled in context to probe
-	 * and probe gets called again in same context. This can happen when
-	 * pci_enable_sriov() is called to create VFs from PF probes context.
-	 * Therefore, for simplicity uniformly defering further probing in all
-	 * cases where we detect contention.
-	 */
-	lock_acquired = mutex_trylock(&hnae3_common_lock);
-	if (!lock_acquired)
-		return -EPROBE_DEFER;
+	mutex_lock(&hnae3_common_lock);
 
 	list_add_tail(&ae_dev->node, &hnae3_ae_dev_list);
 
@@ -220,7 +265,6 @@ int hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev)
 
 		if (!ae_dev->ops) {
 			dev_err(&ae_dev->pdev->dev, "ae_dev ops are null\n");
-			ret = -EOPNOTSUPP;
 			goto out_err;
 		}
 
@@ -247,8 +291,6 @@ int hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev)
 
 out_err:
 	mutex_unlock(&hnae3_common_lock);
-
-	return ret;
 }
 EXPORT_SYMBOL(hnae3_register_ae_dev);
 
@@ -264,6 +306,9 @@ void hnae3_unregister_ae_dev(struct hnae3_ae_dev *ae_dev)
 	mutex_lock(&hnae3_common_lock);
 	/* Check if there are matched ae_algo */
 	list_for_each_entry(ae_algo, &hnae3_ae_algo_list, node) {
+		if (!hnae_get_bit(ae_dev->flag, HNAE3_DEV_INITED_B))
+			continue;
+
 		id = pci_match_id(ae_algo->pdev_id_table, ae_dev->pdev);
 		if (!id)
 			continue;
@@ -283,3 +328,4 @@ EXPORT_SYMBOL(hnae3_unregister_ae_dev);
 MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("HNAE3(Hisilicon Network Acceleration Engine) Framework");
+MODULE_VERSION(HNAE3_MOD_VERSION);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
index 37ec1b3286c6..f250c592a218 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
@@ -36,6 +36,8 @@
 #include <linux/pci.h>
 #include <linux/types.h>
 
+#define HNAE3_MOD_VERSION "1.0"
+
 /* Device IDs */
 #define HNAE3_DEV_ID_GE				0xA220
 #define HNAE3_DEV_ID_25GE			0xA221
@@ -52,6 +54,9 @@
 #define HNAE3_DEV_INITED_B			0x0
 #define HNAE3_DEV_SUPPORT_ROCE_B		0x1
 #define HNAE3_DEV_SUPPORT_DCB_B			0x2
+#define HNAE3_KNIC_CLIENT_INITED_B		0x3
+#define HNAE3_UNIC_CLIENT_INITED_B		0x4
+#define HNAE3_ROCE_CLIENT_INITED_B		0x5
 
 #define HNAE3_DEV_SUPPORT_ROCE_DCB_BITS (BIT(HNAE3_DEV_SUPPORT_DCB_B) |\
 		BIT(HNAE3_DEV_SUPPORT_ROCE_B))
@@ -273,10 +278,6 @@ struct hnae3_ae_dev {
  *   Map rings to vector
  * unmap_ring_from_vector()
  *   Unmap rings from vector
- * add_tunnel_udp()
- *   Add tunnel information to hardware
- * del_tunnel_udp()
- *   Delete tunnel information from hardware
  * reset_queue()
  *   Reset queue
  * get_fw_version()
@@ -388,9 +389,6 @@ struct hnae3_ae_ops {
 				      int vector_num,
 				      struct hnae3_ring_chain_node *vr_chain);
 
-	int (*add_tunnel_udp)(struct hnae3_handle *handle, u16 port_num);
-	int (*del_tunnel_udp)(struct hnae3_handle *handle, u16 port_num);
-
 	void (*reset_queue)(struct hnae3_handle *handle, u16 queue_id);
 	u32 (*get_fw_version)(struct hnae3_handle *handle);
 	void (*get_mdix_mode)(struct hnae3_handle *handle,
@@ -521,11 +519,11 @@ struct hnae3_handle {
 #define hnae_get_bit(origin, shift) \
 	hnae_get_field((origin), (0x1 << (shift)), (shift))
 
-int hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev);
+void hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev);
 void hnae3_unregister_ae_dev(struct hnae3_ae_dev *ae_dev);
 
 void hnae3_unregister_ae_algo(struct hnae3_ae_algo *ae_algo);
-int hnae3_register_ae_algo(struct hnae3_ae_algo *ae_algo);
+void hnae3_register_ae_algo(struct hnae3_ae_algo *ae_algo);
 
 void hnae3_unregister_client(struct hnae3_client *client);
 int hnae3_register_client(struct hnae3_client *client);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
index 8c55965a66ac..05290129793f 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
@@ -25,6 +25,9 @@
 #include "hnae3.h"
 #include "hns3_enet.h"
 
+static void hns3_clear_all_ring(struct hnae3_handle *h);
+static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h);
+
 static const char hns3_driver_name[] = "hns3";
 const char hns3_driver_version[] = VERMAGIC_STRING;
 static const char hns3_driver_string[] =
@@ -273,6 +276,10 @@ static int hns3_nic_net_up(struct net_device *netdev)
 	int i, j;
 	int ret;
 
+	ret = hns3_nic_reset_all_ring(h);
+	if (ret)
+		return ret;
+
 	/* get irq resource for all vectors */
 	ret = hns3_nic_init_irq(priv);
 	if (ret) {
@@ -333,17 +340,19 @@ static void hns3_nic_net_down(struct net_device *netdev)
 	if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
 		return;
 
+	/* disable vectors */
+	for (i = 0; i < priv->vector_num; i++)
+		hns3_vector_disable(&priv->tqp_vector[i]);
+
 	/* stop ae_dev */
 	ops = priv->ae_handle->ae_algo->ops;
 	if (ops->stop)
 		ops->stop(priv->ae_handle);
 
-	/* disable vectors */
-	for (i = 0; i < priv->vector_num; i++)
-		hns3_vector_disable(&priv->tqp_vector[i]);
-
 	/* free irq resources */
 	hns3_nic_uninit_irq(priv);
+
+	hns3_clear_all_ring(priv->ae_handle);
 }
 
 static int hns3_nic_net_stop(struct net_device *netdev)
@@ -502,7 +511,7 @@ static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
 
 	/* find outer header point */
 	l3.hdr = skb_network_header(skb);
-	l4_hdr = skb_inner_transport_header(skb);
+	l4_hdr = skb_transport_header(skb);
 
 	if (skb->protocol == htons(ETH_P_IPV6)) {
 		exthdr = l3.hdr + sizeof(*l3.v6);
@@ -1244,93 +1253,6 @@ static void hns3_nic_get_stats64(struct net_device *netdev,
 	stats->tx_compressed = netdev->stats.tx_compressed;
 }
 
-static void hns3_add_tunnel_port(struct net_device *netdev, u16 port,
-				 enum hns3_udp_tnl_type type)
-{
-	struct hns3_nic_priv *priv = netdev_priv(netdev);
-	struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
-	struct hnae3_handle *h = priv->ae_handle;
-
-	if (udp_tnl->used && udp_tnl->dst_port == port) {
-		udp_tnl->used++;
-		return;
-	}
-
-	if (udp_tnl->used) {
-		netdev_warn(netdev,
-			    "UDP tunnel [%d], port [%d] offload\n", type, port);
-		return;
-	}
-
-	udp_tnl->dst_port = port;
-	udp_tnl->used = 1;
-	/* TBD send command to hardware to add port */
-	if (h->ae_algo->ops->add_tunnel_udp)
-		h->ae_algo->ops->add_tunnel_udp(h, port);
-}
-
-static void hns3_del_tunnel_port(struct net_device *netdev, u16 port,
-				 enum hns3_udp_tnl_type type)
-{
-	struct hns3_nic_priv *priv = netdev_priv(netdev);
-	struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
-	struct hnae3_handle *h = priv->ae_handle;
-
-	if (!udp_tnl->used || udp_tnl->dst_port != port) {
-		netdev_warn(netdev,
-			    "Invalid UDP tunnel port %d\n", port);
-		return;
-	}
-
-	udp_tnl->used--;
-	if (udp_tnl->used)
-		return;
-
-	udp_tnl->dst_port = 0;
-	/* TBD send command to hardware to del port  */
-	if (h->ae_algo->ops->del_tunnel_udp)
-		h->ae_algo->ops->del_tunnel_udp(h, port);
-}
-
-/* hns3_nic_udp_tunnel_add - Get notifiacetion about UDP tunnel ports
- * @netdev: This physical ports's netdev
- * @ti: Tunnel information
- */
-static void hns3_nic_udp_tunnel_add(struct net_device *netdev,
-				    struct udp_tunnel_info *ti)
-{
-	u16 port_n = ntohs(ti->port);
-
-	switch (ti->type) {
-	case UDP_TUNNEL_TYPE_VXLAN:
-		hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
-		break;
-	case UDP_TUNNEL_TYPE_GENEVE:
-		hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
-		break;
-	default:
-		netdev_err(netdev, "unsupported tunnel type %d\n", ti->type);
-		break;
-	}
-}
-
-static void hns3_nic_udp_tunnel_del(struct net_device *netdev,
-				    struct udp_tunnel_info *ti)
-{
-	u16 port_n = ntohs(ti->port);
-
-	switch (ti->type) {
-	case UDP_TUNNEL_TYPE_VXLAN:
-		hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
-		break;
-	case UDP_TUNNEL_TYPE_GENEVE:
-		hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
-		break;
-	default:
-		break;
-	}
-}
-
 static int hns3_setup_tc(struct net_device *netdev, void *type_data)
 {
 	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
@@ -1569,13 +1491,50 @@ static const struct net_device_ops hns3_nic_netdev_ops = {
 	.ndo_get_stats64	= hns3_nic_get_stats64,
 	.ndo_setup_tc		= hns3_nic_setup_tc,
 	.ndo_set_rx_mode	= hns3_nic_set_rx_mode,
-	.ndo_udp_tunnel_add	= hns3_nic_udp_tunnel_add,
-	.ndo_udp_tunnel_del	= hns3_nic_udp_tunnel_del,
 	.ndo_vlan_rx_add_vid	= hns3_vlan_rx_add_vid,
 	.ndo_vlan_rx_kill_vid	= hns3_vlan_rx_kill_vid,
 	.ndo_set_vf_vlan	= hns3_ndo_set_vf_vlan,
 };
 
+static bool hns3_is_phys_func(struct pci_dev *pdev)
+{
+	u32 dev_id = pdev->device;
+
+	switch (dev_id) {
+	case HNAE3_DEV_ID_GE:
+	case HNAE3_DEV_ID_25GE:
+	case HNAE3_DEV_ID_25GE_RDMA:
+	case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
+	case HNAE3_DEV_ID_50GE_RDMA:
+	case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
+	case HNAE3_DEV_ID_100G_RDMA_MACSEC:
+		return true;
+	case HNAE3_DEV_ID_100G_VF:
+	case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
+		return false;
+	default:
+		dev_warn(&pdev->dev, "un-recognized pci device-id %d",
+			 dev_id);
+	}
+
+	return false;
+}
+
+static void hns3_disable_sriov(struct pci_dev *pdev)
+{
+	/* If our VFs are assigned we cannot shut down SR-IOV
+	 * without causing issues, so just leave the hardware
+	 * available but disabled
+	 */
+	if (pci_vfs_assigned(pdev)) {
+		dev_warn(&pdev->dev,
+			 "disabling driver while VFs are assigned\n");
+		return;
+	}
+
+	pci_disable_sriov(pdev);
+}
+
 /* hns3_probe - Device initialization routine
  * @pdev: PCI device information struct
  * @ent: entry in hns3_pci_tbl
@@ -1603,7 +1562,9 @@ static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	ae_dev->dev_type = HNAE3_DEV_KNIC;
 	pci_set_drvdata(pdev, ae_dev);
 
-	return hnae3_register_ae_dev(ae_dev);
+	hnae3_register_ae_dev(ae_dev);
+
+	return 0;
 }
 
 /* hns3_remove - Device removal routine
@@ -1613,21 +1574,56 @@ static void hns3_remove(struct pci_dev *pdev)
 {
 	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
 
+	if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
+		hns3_disable_sriov(pdev);
+
 	hnae3_unregister_ae_dev(ae_dev);
 }
 
+/**
+ * hns3_pci_sriov_configure
+ * @pdev: pointer to a pci_dev structure
+ * @num_vfs: number of VFs to allocate
+ *
+ * Enable or change the number of VFs. Called when the user updates the number
+ * of VFs in sysfs.
+ **/
+static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
+{
+	int ret;
+
+	if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
+		dev_warn(&pdev->dev, "Can not config SRIOV\n");
+		return -EINVAL;
+	}
+
+	if (num_vfs) {
+		ret = pci_enable_sriov(pdev, num_vfs);
+		if (ret)
+			dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
+		else
+			return num_vfs;
+	} else if (!pci_vfs_assigned(pdev)) {
+		pci_disable_sriov(pdev);
+	} else {
+		dev_warn(&pdev->dev,
+			 "Unable to free VFs because some are assigned to VMs.\n");
+	}
+
+	return 0;
+}
+
 static struct pci_driver hns3_driver = {
 	.name     = hns3_driver_name,
 	.id_table = hns3_pci_tbl,
 	.probe    = hns3_probe,
 	.remove   = hns3_remove,
+	.sriov_configure = hns3_pci_sriov_configure,
 };
 
 /* set default feature to hns3 */
 static void hns3_set_default_feature(struct net_device *netdev)
 {
-	struct hnae3_handle *h = hns3_get_handle(netdev);
-
 	netdev->priv_flags |= IFF_UNICAST_FLT;
 
 	netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
@@ -1656,15 +1652,11 @@ static void hns3_set_default_feature(struct net_device *netdev)
 		NETIF_F_GSO_UDP_TUNNEL_CSUM;
 
 	netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
-		NETIF_F_HW_VLAN_CTAG_TX |
+		NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
 		NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
 		NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
 		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
 		NETIF_F_GSO_UDP_TUNNEL_CSUM;
-
-	if (!(h->flags & HNAE3_SUPPORT_VF))
-		netdev->hw_features |=
-			NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_RX;
 }
 
 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
@@ -1836,6 +1828,7 @@ static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
 	hns3_unmap_buffer(ring, &ring->desc_cb[i]);
 	ring->desc_cb[i] = *res_cb;
 	ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
+	ring->desc[i].rx.bd_base_info = 0;
 }
 
 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
@@ -1843,6 +1836,7 @@ static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
 	ring->desc_cb[i].reuse_flag = 0;
 	ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
 		+ ring->desc_cb[i].page_offset);
+	ring->desc[i].rx.bd_base_info = 0;
 }
 
 static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
@@ -1971,106 +1965,6 @@ hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
 	writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
 }
 
-/* hns3_nic_get_headlen - determine size of header for LRO/GRO
- * @data: pointer to the start of the headers
- * @max: total length of section to find headers in
- *
- * This function is meant to determine the length of headers that will
- * be recognized by hardware for LRO, GRO, and RSC offloads.  The main
- * motivation of doing this is to only perform one pull for IPv4 TCP
- * packets so that we can do basic things like calculating the gso_size
- * based on the average data per packet.
- */
-static unsigned int hns3_nic_get_headlen(unsigned char *data, u32 flag,
-					 unsigned int max_size)
-{
-	unsigned char *network;
-	u8 hlen;
-
-	/* This should never happen, but better safe than sorry */
-	if (max_size < ETH_HLEN)
-		return max_size;
-
-	/* Initialize network frame pointer */
-	network = data;
-
-	/* Set first protocol and move network header forward */
-	network += ETH_HLEN;
-
-	/* Handle any vlan tag if present */
-	if (hnae_get_field(flag, HNS3_RXD_VLAN_M, HNS3_RXD_VLAN_S)
-		== HNS3_RX_FLAG_VLAN_PRESENT) {
-		if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN))
-			return max_size;
-
-		network += VLAN_HLEN;
-	}
-
-	/* Handle L3 protocols */
-	if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
-		== HNS3_RX_FLAG_L3ID_IPV4) {
-		if ((typeof(max_size))(network - data) >
-		    (max_size - sizeof(struct iphdr)))
-			return max_size;
-
-		/* Access ihl as a u8 to avoid unaligned access on ia64 */
-		hlen = (network[0] & 0x0F) << 2;
-
-		/* Verify hlen meets minimum size requirements */
-		if (hlen < sizeof(struct iphdr))
-			return network - data;
-
-		/* Record next protocol if header is present */
-	} else if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
-		== HNS3_RX_FLAG_L3ID_IPV6) {
-		if ((typeof(max_size))(network - data) >
-		    (max_size - sizeof(struct ipv6hdr)))
-			return max_size;
-
-		/* Record next protocol */
-		hlen = sizeof(struct ipv6hdr);
-	} else {
-		return network - data;
-	}
-
-	/* Relocate pointer to start of L4 header */
-	network += hlen;
-
-	/* Finally sort out TCP/UDP */
-	if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
-		== HNS3_RX_FLAG_L4ID_TCP) {
-		if ((typeof(max_size))(network - data) >
-		    (max_size - sizeof(struct tcphdr)))
-			return max_size;
-
-		/* Access doff as a u8 to avoid unaligned access on ia64 */
-		hlen = (network[12] & 0xF0) >> 2;
-
-		/* Verify hlen meets minimum size requirements */
-		if (hlen < sizeof(struct tcphdr))
-			return network - data;
-
-		network += hlen;
-	} else if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
-		== HNS3_RX_FLAG_L4ID_UDP) {
-		if ((typeof(max_size))(network - data) >
-		    (max_size - sizeof(struct udphdr)))
-			return max_size;
-
-		network += sizeof(struct udphdr);
-	}
-
-	/* If everything has gone correctly network should be the
-	 * data section of the packet and will be the end of the header.
-	 * If not then it probably represents the end of the last recognized
-	 * header.
-	 */
-	if ((typeof(max_size))(network - data) < max_size)
-		return network - data;
-	else
-		return max_size;
-}
-
 static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
 				struct hns3_enet_ring *ring, int pull_len,
 				struct hns3_desc_cb *desc_cb)
@@ -2183,6 +2077,39 @@ static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
 	napi_gro_receive(&ring->tqp_vector->napi, skb);
 }
 
+static u16 hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
+			       struct hns3_desc *desc, u32 l234info)
+{
+	struct pci_dev *pdev = ring->tqp->handle->pdev;
+	u16 vlan_tag;
+
+	if (pdev->revision == 0x20) {
+		vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
+		if (!(vlan_tag & VLAN_VID_MASK))
+			vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
+
+		return vlan_tag;
+	}
+
+#define HNS3_STRP_OUTER_VLAN	0x1
+#define HNS3_STRP_INNER_VLAN	0x2
+
+	switch (hnae_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
+			       HNS3_RXD_STRP_TAGP_S)) {
+	case HNS3_STRP_OUTER_VLAN:
+		vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
+		break;
+	case HNS3_STRP_INNER_VLAN:
+		vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
+		break;
+	default:
+		vlan_tag = 0;
+		break;
+	}
+
+	return vlan_tag;
+}
+
 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
 			     struct sk_buff **out_skb, int *out_bnum)
 {
@@ -2202,9 +2129,8 @@ static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
 
 	prefetch(desc);
 
-	length = le16_to_cpu(desc->rx.pkt_len);
+	length = le16_to_cpu(desc->rx.size);
 	bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
-	l234info = le32_to_cpu(desc->rx.l234_info);
 
 	/* Check valid BD */
 	if (!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))
@@ -2238,22 +2164,6 @@ static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
 
 	prefetchw(skb->data);
 
-	/* Based on hw strategy, the tag offloaded will be stored at
-	 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
-	 * in one layer tag case.
-	 */
-	if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
-		u16 vlan_tag;
-
-		vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
-		if (!(vlan_tag & VLAN_VID_MASK))
-			vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
-		if (vlan_tag & VLAN_VID_MASK)
-			__vlan_hwaccel_put_tag(skb,
-					       htons(ETH_P_8021Q),
-					       vlan_tag);
-	}
-
 	bnum = 1;
 	if (length <= HNS3_RX_HEAD_SIZE) {
 		memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
@@ -2270,8 +2180,8 @@ static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
 		ring->stats.seg_pkt_cnt++;
 		u64_stats_update_end(&ring->syncp);
 
-		pull_len = hns3_nic_get_headlen(va, l234info,
-						HNS3_RX_HEAD_SIZE);
+		pull_len = eth_get_headlen(va, HNS3_RX_HEAD_SIZE);
+
 		memcpy(__skb_put(skb, pull_len), va,
 		       ALIGN(pull_len, sizeof(long)));
 
@@ -2290,6 +2200,22 @@ static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
 
 	*out_bnum = bnum;
 
+	l234info = le32_to_cpu(desc->rx.l234_info);
+
+	/* Based on hw strategy, the tag offloaded will be stored at
+	 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
+	 * in one layer tag case.
+	 */
+	if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
+		u16 vlan_tag;
+
+		vlan_tag = hns3_parse_vlan_tag(ring, desc, l234info);
+		if (vlan_tag & VLAN_VID_MASK)
+			__vlan_hwaccel_put_tag(skb,
+					       htons(ETH_P_8021Q),
+					       vlan_tag);
+	}
+
 	if (unlikely(!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
 		netdev_err(netdev, "no valid bd,%016llx,%016llx\n",
 			   ((u64 *)desc)[0], ((u64 *)desc)[1]);
@@ -3022,8 +2948,6 @@ int hns3_init_all_ring(struct hns3_nic_priv *priv)
 			goto out_when_alloc_ring_memory;
 		}
 
-		hns3_init_ring_hw(priv->ring_data[i].ring);
-
 		u64_stats_init(&priv->ring_data[i].ring->syncp);
 	}
 
@@ -3052,13 +2976,13 @@ int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
 }
 
 /* Set mac addr if it is configured. or leave it to the AE driver */
-static void hns3_init_mac_addr(struct net_device *netdev)
+static void hns3_init_mac_addr(struct net_device *netdev, bool init)
 {
 	struct hns3_nic_priv *priv = netdev_priv(netdev);
 	struct hnae3_handle *h = priv->ae_handle;
 	u8 mac_addr_temp[ETH_ALEN];
 
-	if (h->ae_algo->ops->get_mac_addr) {
+	if (h->ae_algo->ops->get_mac_addr && init) {
 		h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
 		ether_addr_copy(netdev->dev_addr, mac_addr_temp);
 	}
@@ -3112,7 +3036,7 @@ static int hns3_client_init(struct hnae3_handle *handle)
 	handle->kinfo.netdev = netdev;
 	handle->priv = (void *)priv;
 
-	hns3_init_mac_addr(netdev);
+	hns3_init_mac_addr(netdev, true);
 
 	hns3_set_default_feature(netdev);
 
@@ -3185,6 +3109,8 @@ static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
 	if (netdev->reg_state != NETREG_UNINITIALIZED)
 		unregister_netdev(netdev);
 
+	hns3_force_clear_all_rx_ring(handle);
+
 	ret = hns3_nic_uninit_vector_data(priv);
 	if (ret)
 		netdev_err(netdev, "uninit vector error\n");
@@ -3298,9 +3224,76 @@ static void hns3_recover_hw_addr(struct net_device *ndev)
 		hns3_nic_mc_sync(ndev, ha->addr);
 }
 
-static void hns3_drop_skb_data(struct hns3_enet_ring *ring, struct sk_buff *skb)
+static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
 {
-	dev_kfree_skb_any(skb);
+	while (ring->next_to_clean != ring->next_to_use) {
+		ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
+		hns3_free_buffer_detach(ring, ring->next_to_clean);
+		ring_ptr_move_fw(ring, next_to_clean);
+	}
+}
+
+static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
+{
+	struct hns3_desc_cb res_cbs;
+	int ret;
+
+	while (ring->next_to_use != ring->next_to_clean) {
+		/* When a buffer is not reused, it's memory has been
+		 * freed in hns3_handle_rx_bd or will be freed by
+		 * stack, so we need to replace the buffer here.
+		 */
+		if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
+			ret = hns3_reserve_buffer_map(ring, &res_cbs);
+			if (ret) {
+				u64_stats_update_begin(&ring->syncp);
+				ring->stats.sw_err_cnt++;
+				u64_stats_update_end(&ring->syncp);
+				/* if alloc new buffer fail, exit directly
+				 * and reclear in up flow.
+				 */
+				netdev_warn(ring->tqp->handle->kinfo.netdev,
+					    "reserve buffer map failed, ret = %d\n",
+					    ret);
+				return ret;
+			}
+			hns3_replace_buffer(ring, ring->next_to_use,
+					    &res_cbs);
+		}
+		ring_ptr_move_fw(ring, next_to_use);
+	}
+
+	return 0;
+}
+
+static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
+{
+	while (ring->next_to_use != ring->next_to_clean) {
+		/* When a buffer is not reused, it's memory has been
+		 * freed in hns3_handle_rx_bd or will be freed by
+		 * stack, so only need to unmap the buffer here.
+		 */
+		if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
+			hns3_unmap_buffer(ring,
+					  &ring->desc_cb[ring->next_to_use]);
+			ring->desc_cb[ring->next_to_use].dma = 0;
+		}
+
+		ring_ptr_move_fw(ring, next_to_use);
+	}
+}
+
+static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h)
+{
+	struct net_device *ndev = h->kinfo.netdev;
+	struct hns3_nic_priv *priv = netdev_priv(ndev);
+	struct hns3_enet_ring *ring;
+	u32 i;
+
+	for (i = 0; i < h->kinfo.num_tqps; i++) {
+		ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
+		hns3_force_clear_rx_ring(ring);
+	}
 }
 
 static void hns3_clear_all_ring(struct hnae3_handle *h)
@@ -3314,14 +3307,55 @@ static void hns3_clear_all_ring(struct hnae3_handle *h)
 		struct hns3_enet_ring *ring;
 
 		ring = priv->ring_data[i].ring;
-		hns3_clean_tx_ring(ring, ring->desc_num);
+		hns3_clear_tx_ring(ring);
 		dev_queue = netdev_get_tx_queue(ndev,
 						priv->ring_data[i].queue_index);
 		netdev_tx_reset_queue(dev_queue);
 
 		ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
-		hns3_clean_rx_ring(ring, ring->desc_num, hns3_drop_skb_data);
+		/* Continue to clear other rings even if clearing some
+		 * rings failed.
+		 */
+		hns3_clear_rx_ring(ring);
+	}
+}
+
+int hns3_nic_reset_all_ring(struct hnae3_handle *h)
+{
+	struct net_device *ndev = h->kinfo.netdev;
+	struct hns3_nic_priv *priv = netdev_priv(ndev);
+	struct hns3_enet_ring *rx_ring;
+	int i, j;
+	int ret;
+
+	for (i = 0; i < h->kinfo.num_tqps; i++) {
+		h->ae_algo->ops->reset_queue(h, i);
+		hns3_init_ring_hw(priv->ring_data[i].ring);
+
+		/* We need to clear tx ring here because self test will
+		 * use the ring and will not run down before up
+		 */
+		hns3_clear_tx_ring(priv->ring_data[i].ring);
+		priv->ring_data[i].ring->next_to_clean = 0;
+		priv->ring_data[i].ring->next_to_use = 0;
+
+		rx_ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
+		hns3_init_ring_hw(rx_ring);
+		ret = hns3_clear_rx_ring(rx_ring);
+		if (ret)
+			return ret;
+
+		/* We can not know the hardware head and tail when this
+		 * function is called in reset flow, so we reuse all desc.
+		 */
+		for (j = 0; j < rx_ring->desc_num; j++)
+			hns3_reuse_buffer(rx_ring, j);
+
+		rx_ring->next_to_clean = 0;
+		rx_ring->next_to_use = 0;
 	}
+
+	return 0;
 }
 
 static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
@@ -3359,7 +3393,7 @@ static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
 	struct hns3_nic_priv *priv = netdev_priv(netdev);
 	int ret;
 
-	hns3_init_mac_addr(netdev);
+	hns3_init_mac_addr(netdev, false);
 	hns3_nic_set_rx_mode(netdev);
 	hns3_recover_hw_addr(netdev);
 
@@ -3393,7 +3427,7 @@ static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
 	struct hns3_nic_priv *priv = netdev_priv(netdev);
 	int ret;
 
-	hns3_clear_all_ring(handle);
+	hns3_force_clear_all_rx_ring(handle);
 
 	ret = hns3_nic_uninit_vector_data(priv);
 	if (ret) {
@@ -3529,8 +3563,6 @@ int hns3_set_channels(struct net_device *netdev,
 	if (if_running)
 		hns3_nic_net_stop(netdev);
 
-	hns3_clear_all_ring(h);
-
 	ret = hns3_nic_uninit_vector_data(priv);
 	if (ret) {
 		dev_err(&netdev->dev,
@@ -3600,6 +3632,8 @@ static int __init hns3_init_module(void)
 
 	client.ops = &client_ops;
 
+	INIT_LIST_HEAD(&client.node);
+
 	ret = hnae3_register_client(&client);
 	if (ret)
 		return ret;
@@ -3627,3 +3661,4 @@ MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
 MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("pci:hns-nic");
+MODULE_VERSION(HNS3_MOD_VERSION);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
index 98cdbd3a1163..3b083d5ae9ce 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
@@ -14,6 +14,8 @@
 
 #include "hnae3.h"
 
+#define HNS3_MOD_VERSION "1.0"
+
 extern const char hns3_driver_version[];
 
 enum hns3_nic_state {
@@ -102,6 +104,9 @@ enum hns3_nic_state {
 #define HNS3_RXD_L4ID_S				8
 #define HNS3_RXD_L4ID_M				(0xf << HNS3_RXD_L4ID_S)
 #define HNS3_RXD_FRAG_B				12
+#define HNS3_RXD_STRP_TAGP_S			13
+#define HNS3_RXD_STRP_TAGP_M			(0x3 << HNS3_RXD_STRP_TAGP_S)
+
 #define HNS3_RXD_L2E_B				16
 #define HNS3_RXD_L3E_B				17
 #define HNS3_RXD_L4E_B				18
@@ -620,6 +625,7 @@ int hns3_set_channels(struct net_device *netdev,
 bool hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget);
 int hns3_init_all_ring(struct hns3_nic_priv *priv);
 int hns3_uninit_all_ring(struct hns3_nic_priv *priv);
+int hns3_nic_reset_all_ring(struct hnae3_handle *h);
 netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev);
 int hns3_clean_rx_ring(
 		struct hns3_enet_ring *ring, int budget,
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
index eb3c34f3cf87..8f8cc2413656 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
@@ -74,7 +74,7 @@ struct hns3_link_mode_mapping {
 	u32 ethtool_link_mode;
 };
 
-static int hns3_lp_setup(struct net_device *ndev, enum hnae3_loop loop)
+static int hns3_lp_setup(struct net_device *ndev, enum hnae3_loop loop, bool en)
 {
 	struct hnae3_handle *h = hns3_get_handle(ndev);
 	int ret;
@@ -85,11 +85,7 @@ static int hns3_lp_setup(struct net_device *ndev, enum hnae3_loop loop)
 
 	switch (loop) {
 	case HNAE3_MAC_INTER_LOOP_MAC:
-		ret = h->ae_algo->ops->set_loopback(h, loop, true);
-		break;
-	case HNAE3_MAC_LOOP_NONE:
-		ret = h->ae_algo->ops->set_loopback(h,
-			HNAE3_MAC_INTER_LOOP_MAC, false);
+		ret = h->ae_algo->ops->set_loopback(h, loop, en);
 		break;
 	default:
 		ret = -ENOTSUPP;
@@ -99,10 +95,7 @@ static int hns3_lp_setup(struct net_device *ndev, enum hnae3_loop loop)
 	if (ret)
 		return ret;
 
-	if (loop == HNAE3_MAC_LOOP_NONE)
-		h->ae_algo->ops->set_promisc_mode(h, ndev->flags & IFF_PROMISC);
-	else
-		h->ae_algo->ops->set_promisc_mode(h, 1);
+	h->ae_algo->ops->set_promisc_mode(h, en);
 
 	return ret;
 }
@@ -115,6 +108,10 @@ static int hns3_lp_up(struct net_device *ndev, enum hnae3_loop loop_mode)
 	if (!h->ae_algo->ops->start)
 		return -EOPNOTSUPP;
 
+	ret = hns3_nic_reset_all_ring(h);
+	if (ret)
+		return ret;
+
 	ret = h->ae_algo->ops->start(h);
 	if (ret) {
 		netdev_err(ndev,
@@ -122,13 +119,13 @@ static int hns3_lp_up(struct net_device *ndev, enum hnae3_loop loop_mode)
 		return ret;
 	}
 
-	ret = hns3_lp_setup(ndev, loop_mode);
+	ret = hns3_lp_setup(ndev, loop_mode, true);
 	usleep_range(10000, 20000);
 
 	return ret;
 }
 
-static int hns3_lp_down(struct net_device *ndev)
+static int hns3_lp_down(struct net_device *ndev, enum hnae3_loop loop_mode)
 {
 	struct hnae3_handle *h = hns3_get_handle(ndev);
 	int ret;
@@ -136,7 +133,7 @@ static int hns3_lp_down(struct net_device *ndev)
 	if (!h->ae_algo->ops->stop)
 		return -EOPNOTSUPP;
 
-	ret = hns3_lp_setup(ndev, HNAE3_MAC_LOOP_NONE);
+	ret = hns3_lp_setup(ndev, loop_mode, false);
 	if (ret) {
 		netdev_err(ndev, "lb_setup return error: %d\n", ret);
 		return ret;
@@ -332,7 +329,7 @@ static void hns3_self_test(struct net_device *ndev,
 		data[test_index] = hns3_lp_up(ndev, loop_type);
 		if (!data[test_index]) {
 			data[test_index] = hns3_lp_run_test(ndev, loop_type);
-			hns3_lp_down(ndev);
+			hns3_lp_down(ndev, loop_type);
 		}
 
 		if (data[test_index])
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c
index ff13d1876d9e..c36d64710fa6 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c
@@ -31,6 +31,17 @@ static int hclge_ring_space(struct hclge_cmq_ring *ring)
 	return ring->desc_num - used - 1;
 }
 
+static int is_valid_csq_clean_head(struct hclge_cmq_ring *ring, int h)
+{
+	int u = ring->next_to_use;
+	int c = ring->next_to_clean;
+
+	if (unlikely(h >= ring->desc_num))
+		return 0;
+
+	return u > c ? (h > c && h <= u) : (h > c || h <= u);
+}
+
 static int hclge_alloc_cmd_desc(struct hclge_cmq_ring *ring)
 {
 	int size  = ring->desc_num * sizeof(struct hclge_desc);
@@ -141,6 +152,7 @@ static void hclge_cmd_init_regs(struct hclge_hw *hw)
 
 static int hclge_cmd_csq_clean(struct hclge_hw *hw)
 {
+	struct hclge_dev *hdev = (struct hclge_dev *)hw->back;
 	struct hclge_cmq_ring *csq = &hw->cmq.csq;
 	u16 ntc = csq->next_to_clean;
 	struct hclge_desc *desc;
@@ -149,6 +161,13 @@ static int hclge_cmd_csq_clean(struct hclge_hw *hw)
 
 	desc = &csq->desc[ntc];
 	head = hclge_read_dev(hw, HCLGE_NIC_CSQ_HEAD_REG);
+	rmb(); /* Make sure head is ready before touch any data */
+
+	if (!is_valid_csq_clean_head(csq, head)) {
+		dev_warn(&hdev->pdev->dev, "wrong head (%d, %d-%d)\n", head,
+			   csq->next_to_use, csq->next_to_clean);
+		return 0;
+	}
 
 	while (head != ntc) {
 		memset(desc, 0, sizeof(*desc));
@@ -171,7 +190,11 @@ static int hclge_cmd_csq_done(struct hclge_hw *hw)
 
 static bool hclge_is_special_opcode(u16 opcode)
 {
-	u16 spec_opcode[3] = {0x0030, 0x0031, 0x0032};
+	/* these commands have several descriptors,
+	 * and use the first one to save opcode and return value
+	 */
+	u16 spec_opcode[3] = {HCLGE_OPC_STATS_64_BIT,
+		HCLGE_OPC_STATS_32_BIT, HCLGE_OPC_STATS_MAC};
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(spec_opcode); i++) {
@@ -362,9 +385,9 @@ int hclge_cmd_init(struct hclge_dev *hdev)
 
 static void hclge_destroy_queue(struct hclge_cmq_ring *ring)
 {
-	spin_lock_bh(&ring->lock);
+	spin_lock(&ring->lock);
 	hclge_free_cmd_desc(ring);
-	spin_unlock_bh(&ring->lock);
+	spin_unlock(&ring->lock);
 }
 
 void hclge_destroy_cmd_queue(struct hclge_hw *hw)
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
index ee3cbac6dfaa..d9aaa76c76eb 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
@@ -115,7 +115,6 @@ enum hclge_opcode_type {
 	HCLGE_OPC_QUERY_LINK_STATUS	= 0x0307,
 	HCLGE_OPC_CONFIG_MAX_FRM_SIZE	= 0x0308,
 	HCLGE_OPC_CONFIG_SPEED_DUP	= 0x0309,
-	HCLGE_OPC_STATS_MAC_TRAFFIC	= 0x0314,
 	/* MACSEC command */
 
 	/* PFC/Pause CMD*/
@@ -484,6 +483,8 @@ struct hclge_promisc_param {
 	u8 enable;
 };
 
+#define HCLGE_PROMISC_TX_EN_B	BIT(4)
+#define HCLGE_PROMISC_RX_EN_B	BIT(5)
 #define HCLGE_PROMISC_EN_B	1
 #define HCLGE_PROMISC_EN_ALL	0x7
 #define HCLGE_PROMISC_EN_UC	0x1
@@ -704,11 +705,14 @@ struct hclge_vlan_filter_vf_cfg_cmd {
 	u8  vf_bitmap[16];
 };
 
-#define HCLGE_ACCEPT_TAG_B		0
-#define HCLGE_ACCEPT_UNTAG_B		1
+#define HCLGE_ACCEPT_TAG1_B		0
+#define HCLGE_ACCEPT_UNTAG1_B		1
 #define HCLGE_PORT_INS_TAG1_EN_B	2
 #define HCLGE_PORT_INS_TAG2_EN_B	3
 #define HCLGE_CFG_NIC_ROCE_SEL_B	4
+#define HCLGE_ACCEPT_TAG2_B		5
+#define HCLGE_ACCEPT_UNTAG2_B		6
+
 struct hclge_vport_vtag_tx_cfg_cmd {
 	u8 vport_vlan_cfg;
 	u8 vf_offset;
@@ -813,21 +817,13 @@ struct hclge_reset_cmd {
 #define HCLGE_NIC_CMQ_DESC_NUM		1024
 #define HCLGE_NIC_CMQ_DESC_NUM_S	3
 
-#define HCLGE_LED_PORT_SPEED_STATE_S	0
-#define HCLGE_LED_PORT_SPEED_STATE_M	GENMASK(5, 0)
-#define HCLGE_LED_ACTIVITY_STATE_S	0
-#define HCLGE_LED_ACTIVITY_STATE_M	GENMASK(1, 0)
-#define HCLGE_LED_LINK_STATE_S		0
-#define HCLGE_LED_LINK_STATE_M		GENMASK(1, 0)
 #define HCLGE_LED_LOCATE_STATE_S	0
 #define HCLGE_LED_LOCATE_STATE_M	GENMASK(1, 0)
 
 struct hclge_set_led_state_cmd {
-	u8 port_speed_led_config;
-	u8 link_led_config;
-	u8 activity_led_config;
+	u8 rsv1[3];
 	u8 locate_led_config;
-	u8 rsv[20];
+	u8 rsv2[20];
 };
 
 int hclge_cmd_init(struct hclge_dev *hdev);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
index 2066dd734444..69166858a6dc 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
@@ -39,7 +39,6 @@ static int hclge_set_mta_filter_mode(struct hclge_dev *hdev,
 static int hclge_set_mtu(struct hnae3_handle *handle, int new_mtu);
 static int hclge_init_vlan_config(struct hclge_dev *hdev);
 static int hclge_reset_ae_dev(struct hnae3_ae_dev *ae_dev);
-static int hclge_update_led_status(struct hclge_dev *hdev);
 
 static struct hnae3_ae_algo ae_algo;
 
@@ -304,8 +303,6 @@ static const struct hclge_comm_stats_str g_mac_stats_string[] = {
 		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_2048_4095_oct_pkt_num)},
 	{"mac_tx_4096_8191_oct_pkt_num",
 		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_4096_8191_oct_pkt_num)},
-	{"mac_tx_8192_12287_oct_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_8192_12287_oct_pkt_num)},
 	{"mac_tx_8192_9216_oct_pkt_num",
 		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_8192_9216_oct_pkt_num)},
 	{"mac_tx_9217_12287_oct_pkt_num",
@@ -356,8 +353,6 @@ static const struct hclge_comm_stats_str g_mac_stats_string[] = {
 		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_2048_4095_oct_pkt_num)},
 	{"mac_rx_4096_8191_oct_pkt_num",
 		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_4096_8191_oct_pkt_num)},
-	{"mac_rx_8192_12287_oct_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_8192_12287_oct_pkt_num)},
 	{"mac_rx_8192_9216_oct_pkt_num",
 		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_8192_9216_oct_pkt_num)},
 	{"mac_rx_9217_12287_oct_pkt_num",
@@ -508,38 +503,6 @@ static int hclge_32_bit_update_stats(struct hclge_dev *hdev)
 	return 0;
 }
 
-static int hclge_mac_get_traffic_stats(struct hclge_dev *hdev)
-{
-	struct hclge_mac_stats *mac_stats = &hdev->hw_stats.mac_stats;
-	struct hclge_desc desc;
-	__le64 *desc_data;
-	int ret;
-
-	/* for fiber port, need to query the total rx/tx packets statstics,
-	 * used for data transferring checking.
-	 */
-	if (hdev->hw.mac.media_type != HNAE3_MEDIA_TYPE_FIBER)
-		return 0;
-
-	if (test_bit(HCLGE_STATE_STATISTICS_UPDATING, &hdev->state))
-		return 0;
-
-	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_STATS_MAC_TRAFFIC, true);
-	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
-	if (ret) {
-		dev_err(&hdev->pdev->dev,
-			"Get MAC total pkt stats fail, ret = %d\n", ret);
-
-		return ret;
-	}
-
-	desc_data = (__le64 *)(&desc.data[0]);
-	mac_stats->mac_tx_total_pkt_num += le64_to_cpu(*desc_data++);
-	mac_stats->mac_rx_total_pkt_num += le64_to_cpu(*desc_data);
-
-	return 0;
-}
-
 static int hclge_mac_update_stats(struct hclge_dev *hdev)
 {
 #define HCLGE_MAC_CMD_NUM 21
@@ -1459,8 +1422,11 @@ static int hclge_alloc_vport(struct hclge_dev *hdev)
 	/* We need to alloc a vport for main NIC of PF */
 	num_vport = hdev->num_vmdq_vport + hdev->num_req_vfs + 1;
 
-	if (hdev->num_tqps < num_vport)
-		num_vport = hdev->num_tqps;
+	if (hdev->num_tqps < num_vport) {
+		dev_err(&hdev->pdev->dev, "tqps(%d) is less than vports(%d)",
+			hdev->num_tqps, num_vport);
+		return -EINVAL;
+	}
 
 	/* Alloc the same number of TQPs for every vport */
 	tqp_per_vport = hdev->num_tqps / num_vport;
@@ -1474,21 +1440,8 @@ static int hclge_alloc_vport(struct hclge_dev *hdev)
 	hdev->vport = vport;
 	hdev->num_alloc_vport = num_vport;
 
-#ifdef CONFIG_PCI_IOV
-	/* Enable SRIOV */
-	if (hdev->num_req_vfs) {
-		dev_info(&pdev->dev, "active VFs(%d) found, enabling SRIOV\n",
-			 hdev->num_req_vfs);
-		ret = pci_enable_sriov(hdev->pdev, hdev->num_req_vfs);
-		if (ret) {
-			hdev->num_alloc_vfs = 0;
-			dev_err(&pdev->dev, "SRIOV enable failed %d\n",
-				ret);
-			return ret;
-		}
-	}
-	hdev->num_alloc_vfs = hdev->num_req_vfs;
-#endif
+	if (IS_ENABLED(CONFIG_PCI_IOV))
+		hdev->num_alloc_vfs = hdev->num_req_vfs;
 
 	for (i = 0; i < num_vport; i++) {
 		vport->back = hdev;
@@ -2930,38 +2883,16 @@ static void hclge_service_task(struct work_struct *work)
 	struct hclge_dev *hdev =
 		container_of(work, struct hclge_dev, service_task);
 
-	/* The total rx/tx packets statstics are wanted to be updated
-	 * per second. Both hclge_update_stats_for_all() and
-	 * hclge_mac_get_traffic_stats() can do it.
-	 */
 	if (hdev->hw_stats.stats_timer >= HCLGE_STATS_TIMER_INTERVAL) {
 		hclge_update_stats_for_all(hdev);
 		hdev->hw_stats.stats_timer = 0;
-	} else {
-		hclge_mac_get_traffic_stats(hdev);
 	}
 
 	hclge_update_speed_duplex(hdev);
 	hclge_update_link_status(hdev);
-	hclge_update_led_status(hdev);
 	hclge_service_complete(hdev);
 }
 
-static void hclge_disable_sriov(struct hclge_dev *hdev)
-{
-	/* If our VFs are assigned we cannot shut down SR-IOV
-	 * without causing issues, so just leave the hardware
-	 * available but disabled
-	 */
-	if (pci_vfs_assigned(hdev->pdev)) {
-		dev_warn(&hdev->pdev->dev,
-			 "disabling driver while VFs are assigned\n");
-		return;
-	}
-
-	pci_disable_sriov(hdev->pdev);
-}
-
 struct hclge_vport *hclge_get_vport(struct hnae3_handle *handle)
 {
 	/* VF handle has no client */
@@ -3615,7 +3546,14 @@ int hclge_cmd_set_promisc_mode(struct hclge_dev *hdev,
 
 	req = (struct hclge_promisc_cfg_cmd *)desc.data;
 	req->vf_id = param->vf_id;
-	req->flag = (param->enable << HCLGE_PROMISC_EN_B);
+
+	/* HCLGE_PROMISC_TX_EN_B and HCLGE_PROMISC_RX_EN_B are not supported on
+	 * pdev revision(0x20), new revision support them. The
+	 * value of this two fields will not return error when driver
+	 * send command to fireware in revision(0x20).
+	 */
+	req->flag = (param->enable << HCLGE_PROMISC_EN_B) |
+		HCLGE_PROMISC_TX_EN_B | HCLGE_PROMISC_RX_EN_B;
 
 	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
 	if (ret) {
@@ -3683,48 +3621,50 @@ static void hclge_cfg_mac_mode(struct hclge_dev *hdev, bool enable)
 			"mac enable fail, ret =%d.\n", ret);
 }
 
-static int hclge_set_loopback(struct hnae3_handle *handle,
-			      enum hnae3_loop loop_mode, bool en)
+static int hclge_set_mac_loopback(struct hclge_dev *hdev, bool en)
 {
-	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_config_mac_mode_cmd *req;
-	struct hclge_dev *hdev = vport->back;
 	struct hclge_desc desc;
 	u32 loop_en;
 	int ret;
 
-	switch (loop_mode) {
-	case HNAE3_MAC_INTER_LOOP_MAC:
-		req = (struct hclge_config_mac_mode_cmd *)&desc.data[0];
-		/* 1 Read out the MAC mode config at first */
-		hclge_cmd_setup_basic_desc(&desc,
-					   HCLGE_OPC_CONFIG_MAC_MODE,
-					   true);
-		ret = hclge_cmd_send(&hdev->hw, &desc, 1);
-		if (ret) {
-			dev_err(&hdev->pdev->dev,
-				"mac loopback get fail, ret =%d.\n",
-				ret);
-			return ret;
-		}
+	req = (struct hclge_config_mac_mode_cmd *)&desc.data[0];
+	/* 1 Read out the MAC mode config at first */
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CONFIG_MAC_MODE, true);
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"mac loopback get fail, ret =%d.\n", ret);
+		return ret;
+	}
 
-		/* 2 Then setup the loopback flag */
-		loop_en = le32_to_cpu(req->txrx_pad_fcs_loop_en);
-		if (en)
-			hnae_set_bit(loop_en, HCLGE_MAC_APP_LP_B, 1);
-		else
-			hnae_set_bit(loop_en, HCLGE_MAC_APP_LP_B, 0);
+	/* 2 Then setup the loopback flag */
+	loop_en = le32_to_cpu(req->txrx_pad_fcs_loop_en);
+	hnae_set_bit(loop_en, HCLGE_MAC_APP_LP_B, en ? 1 : 0);
 
-		req->txrx_pad_fcs_loop_en = cpu_to_le32(loop_en);
+	req->txrx_pad_fcs_loop_en = cpu_to_le32(loop_en);
 
-		/* 3 Config mac work mode with loopback flag
-		 * and its original configure parameters
-		 */
-		hclge_cmd_reuse_desc(&desc, false);
-		ret = hclge_cmd_send(&hdev->hw, &desc, 1);
-		if (ret)
-			dev_err(&hdev->pdev->dev,
-				"mac loopback set fail, ret =%d.\n", ret);
+	/* 3 Config mac work mode with loopback flag
+	 * and its original configure parameters
+	 */
+	hclge_cmd_reuse_desc(&desc, false);
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret)
+		dev_err(&hdev->pdev->dev,
+			"mac loopback set fail, ret =%d.\n", ret);
+	return ret;
+}
+
+static int hclge_set_loopback(struct hnae3_handle *handle,
+			      enum hnae3_loop loop_mode, bool en)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	int ret;
+
+	switch (loop_mode) {
+	case HNAE3_MAC_INTER_LOOP_MAC:
+		ret = hclge_set_mac_loopback(hdev, en);
 		break;
 	default:
 		ret = -ENOTSUPP;
@@ -3783,6 +3723,7 @@ static int hclge_ae_start(struct hnae3_handle *handle)
 	hclge_cfg_mac_mode(hdev, true);
 	clear_bit(HCLGE_STATE_DOWN, &hdev->state);
 	mod_timer(&hdev->service_timer, jiffies + HZ);
+	hdev->hw.mac.link = 0;
 
 	/* reset tqp stats */
 	hclge_reset_tqp_stats(handle);
@@ -3819,6 +3760,8 @@ static void hclge_ae_stop(struct hnae3_handle *handle)
 
 	/* reset tqp stats */
 	hclge_reset_tqp_stats(handle);
+	del_timer_sync(&hdev->service_timer);
+	cancel_work_sync(&hdev->service_task);
 	hclge_update_link_status(hdev);
 }
 
@@ -4540,8 +4483,9 @@ static void hclge_enable_vlan_filter(struct hnae3_handle *handle, bool enable)
 	hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_VF, enable);
 }
 
-int hclge_set_vf_vlan_common(struct hclge_dev *hdev, int vfid,
-			     bool is_kill, u16 vlan, u8 qos, __be16 proto)
+static int hclge_set_vf_vlan_common(struct hclge_dev *hdev, int vfid,
+				    bool is_kill, u16 vlan, u8 qos,
+				    __be16 proto)
 {
 #define HCLGE_MAX_VF_BYTES  16
 	struct hclge_vlan_filter_vf_cfg_cmd *req0;
@@ -4599,12 +4543,9 @@ int hclge_set_vf_vlan_common(struct hclge_dev *hdev, int vfid,
 	return -EIO;
 }
 
-static int hclge_set_port_vlan_filter(struct hnae3_handle *handle,
-				      __be16 proto, u16 vlan_id,
-				      bool is_kill)
+static int hclge_set_port_vlan_filter(struct hclge_dev *hdev, __be16 proto,
+				      u16 vlan_id, bool is_kill)
 {
-	struct hclge_vport *vport = hclge_get_vport(handle);
-	struct hclge_dev *hdev = vport->back;
 	struct hclge_vlan_filter_pf_cfg_cmd *req;
 	struct hclge_desc desc;
 	u8 vlan_offset_byte_val;
@@ -4624,22 +4565,66 @@ static int hclge_set_port_vlan_filter(struct hnae3_handle *handle,
 	req->vlan_offset_bitmap[vlan_offset_byte] = vlan_offset_byte_val;
 
 	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret)
+		dev_err(&hdev->pdev->dev,
+			"port vlan command, send fail, ret =%d.\n", ret);
+	return ret;
+}
+
+static int hclge_set_vlan_filter_hw(struct hclge_dev *hdev, __be16 proto,
+				    u16 vport_id, u16 vlan_id, u8 qos,
+				    bool is_kill)
+{
+	u16 vport_idx, vport_num = 0;
+	int ret;
+
+	ret = hclge_set_vf_vlan_common(hdev, vport_id, is_kill, vlan_id,
+				       0, proto);
 	if (ret) {
 		dev_err(&hdev->pdev->dev,
-			"port vlan command, send fail, ret =%d.\n",
-			ret);
+			"Set %d vport vlan filter config fail, ret =%d.\n",
+			vport_id, ret);
 		return ret;
 	}
 
-	ret = hclge_set_vf_vlan_common(hdev, 0, is_kill, vlan_id, 0, proto);
-	if (ret) {
+	/* vlan 0 may be added twice when 8021q module is enabled */
+	if (!is_kill && !vlan_id &&
+	    test_bit(vport_id, hdev->vlan_table[vlan_id]))
+		return 0;
+
+	if (!is_kill && test_and_set_bit(vport_id, hdev->vlan_table[vlan_id])) {
 		dev_err(&hdev->pdev->dev,
-			"Set pf vlan filter config fail, ret =%d.\n",
-			ret);
-		return -EIO;
+			"Add port vlan failed, vport %d is already in vlan %d\n",
+			vport_id, vlan_id);
+		return -EINVAL;
 	}
 
-	return 0;
+	if (is_kill &&
+	    !test_and_clear_bit(vport_id, hdev->vlan_table[vlan_id])) {
+		dev_err(&hdev->pdev->dev,
+			"Delete port vlan failed, vport %d is not in vlan %d\n",
+			vport_id, vlan_id);
+		return -EINVAL;
+	}
+
+	for_each_set_bit(vport_idx, hdev->vlan_table[vlan_id], VLAN_N_VID)
+		vport_num++;
+
+	if ((is_kill && vport_num == 0) || (!is_kill && vport_num == 1))
+		ret = hclge_set_port_vlan_filter(hdev, proto, vlan_id,
+						 is_kill);
+
+	return ret;
+}
+
+int hclge_set_vlan_filter(struct hnae3_handle *handle, __be16 proto,
+			  u16 vlan_id, bool is_kill)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	return hclge_set_vlan_filter_hw(hdev, proto, vport->vport_id, vlan_id,
+					0, is_kill);
 }
 
 static int hclge_set_vf_vlan_filter(struct hnae3_handle *handle, int vfid,
@@ -4653,7 +4638,7 @@ static int hclge_set_vf_vlan_filter(struct hnae3_handle *handle, int vfid,
 	if (proto != htons(ETH_P_8021Q))
 		return -EPROTONOSUPPORT;
 
-	return hclge_set_vf_vlan_common(hdev, vfid, false, vlan, qos, proto);
+	return hclge_set_vlan_filter_hw(hdev, proto, vfid, vlan, qos, false);
 }
 
 static int hclge_set_vlan_tx_offload_cfg(struct hclge_vport *vport)
@@ -4669,10 +4654,14 @@ static int hclge_set_vlan_tx_offload_cfg(struct hclge_vport *vport)
 	req = (struct hclge_vport_vtag_tx_cfg_cmd *)desc.data;
 	req->def_vlan_tag1 = cpu_to_le16(vcfg->default_tag1);
 	req->def_vlan_tag2 = cpu_to_le16(vcfg->default_tag2);
-	hnae_set_bit(req->vport_vlan_cfg, HCLGE_ACCEPT_TAG_B,
-		     vcfg->accept_tag ? 1 : 0);
-	hnae_set_bit(req->vport_vlan_cfg, HCLGE_ACCEPT_UNTAG_B,
-		     vcfg->accept_untag ? 1 : 0);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_ACCEPT_TAG1_B,
+			vcfg->accept_tag1 ? 1 : 0);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_ACCEPT_UNTAG1_B,
+			vcfg->accept_untag1 ? 1 : 0);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_ACCEPT_TAG2_B,
+			vcfg->accept_tag2 ? 1 : 0);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_ACCEPT_UNTAG2_B,
+			vcfg->accept_untag2 ? 1 : 0);
 	hnae_set_bit(req->vport_vlan_cfg, HCLGE_PORT_INS_TAG1_EN_B,
 		     vcfg->insert_tag1_en ? 1 : 0);
 	hnae_set_bit(req->vport_vlan_cfg, HCLGE_PORT_INS_TAG2_EN_B,
@@ -4796,8 +4785,18 @@ static int hclge_init_vlan_config(struct hclge_dev *hdev)
 
 	for (i = 0; i < hdev->num_alloc_vport; i++) {
 		vport = &hdev->vport[i];
-		vport->txvlan_cfg.accept_tag = true;
-		vport->txvlan_cfg.accept_untag = true;
+		vport->txvlan_cfg.accept_tag1 = true;
+		vport->txvlan_cfg.accept_untag1 = true;
+
+		/* accept_tag2 and accept_untag2 are not supported on
+		 * pdev revision(0x20), new revision support them. The
+		 * value of this two fields will not return error when driver
+		 * send command to fireware in revision(0x20).
+		 * This two fields can not configured by user.
+		 */
+		vport->txvlan_cfg.accept_tag2 = true;
+		vport->txvlan_cfg.accept_untag2 = true;
+
 		vport->txvlan_cfg.insert_tag1_en = false;
 		vport->txvlan_cfg.insert_tag2_en = false;
 		vport->txvlan_cfg.default_tag1 = 0;
@@ -4818,10 +4817,10 @@ static int hclge_init_vlan_config(struct hclge_dev *hdev)
 	}
 
 	handle = &hdev->vport[0].nic;
-	return hclge_set_port_vlan_filter(handle, htons(ETH_P_8021Q), 0, false);
+	return hclge_set_vlan_filter(handle, htons(ETH_P_8021Q), 0, false);
 }
 
-static int hclge_en_hw_strip_rxvtag(struct hnae3_handle *handle, bool enable)
+int hclge_en_hw_strip_rxvtag(struct hnae3_handle *handle, bool enable)
 {
 	struct hclge_vport *vport = hclge_get_vport(handle);
 
@@ -5166,12 +5165,6 @@ static int hclge_set_pauseparam(struct hnae3_handle *handle, u32 auto_neg,
 	struct phy_device *phydev = hdev->hw.mac.phydev;
 	u32 fc_autoneg;
 
-	/* Only support flow control negotiation for netdev with
-	 * phy attached for now.
-	 */
-	if (!phydev)
-		return -EOPNOTSUPP;
-
 	fc_autoneg = hclge_get_autoneg(handle);
 	if (auto_neg != fc_autoneg) {
 		dev_info(&hdev->pdev->dev,
@@ -5190,6 +5183,12 @@ static int hclge_set_pauseparam(struct hnae3_handle *handle, u32 auto_neg,
 	if (!fc_autoneg)
 		return hclge_cfg_pauseparam(hdev, rx_en, tx_en);
 
+	/* Only support flow control negotiation for netdev with
+	 * phy attached for now.
+	 */
+	if (!phydev)
+		return -EOPNOTSUPP;
+
 	return phy_start_aneg(phydev);
 }
 
@@ -5282,7 +5281,7 @@ static int hclge_init_client_instance(struct hnae3_client *client,
 			vport->nic.client = client;
 			ret = client->ops->init_instance(&vport->nic);
 			if (ret)
-				goto err;
+				return ret;
 
 			if (hdev->roce_client &&
 			    hnae3_dev_roce_supported(hdev)) {
@@ -5290,11 +5289,11 @@ static int hclge_init_client_instance(struct hnae3_client *client,
 
 				ret = hclge_init_roce_base_info(vport);
 				if (ret)
-					goto err;
+					return ret;
 
 				ret = rc->ops->init_instance(&vport->roce);
 				if (ret)
-					goto err;
+					return ret;
 			}
 
 			break;
@@ -5304,7 +5303,7 @@ static int hclge_init_client_instance(struct hnae3_client *client,
 
 			ret = client->ops->init_instance(&vport->nic);
 			if (ret)
-				goto err;
+				return ret;
 
 			break;
 		case HNAE3_CLIENT_ROCE:
@@ -5316,18 +5315,16 @@ static int hclge_init_client_instance(struct hnae3_client *client,
 			if (hdev->roce_client && hdev->nic_client) {
 				ret = hclge_init_roce_base_info(vport);
 				if (ret)
-					goto err;
+					return ret;
 
 				ret = client->ops->init_instance(&vport->roce);
 				if (ret)
-					goto err;
+					return ret;
 			}
 		}
 	}
 
 	return 0;
-err:
-	return ret;
 }
 
 static void hclge_uninit_client_instance(struct hnae3_client *client,
@@ -5364,7 +5361,7 @@ static int hclge_pci_init(struct hclge_dev *hdev)
 	ret = pci_enable_device(pdev);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to enable PCI device\n");
-		goto err_no_drvdata;
+		return ret;
 	}
 
 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
@@ -5402,8 +5399,6 @@ static int hclge_pci_init(struct hclge_dev *hdev)
 	pci_release_regions(pdev);
 err_disable_device:
 	pci_disable_device(pdev);
-err_no_drvdata:
-	pci_set_drvdata(pdev, NULL);
 
 	return ret;
 }
@@ -5412,6 +5407,7 @@ static void hclge_pci_uninit(struct hclge_dev *hdev)
 {
 	struct pci_dev *pdev = hdev->pdev;
 
+	pcim_iounmap(pdev, hdev->hw.io_base);
 	pci_free_irq_vectors(pdev);
 	pci_clear_master(pdev);
 	pci_release_mem_regions(pdev);
@@ -5427,7 +5423,7 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev)
 	hdev = devm_kzalloc(&pdev->dev, sizeof(*hdev), GFP_KERNEL);
 	if (!hdev) {
 		ret = -ENOMEM;
-		goto err_hclge_dev;
+		goto out;
 	}
 
 	hdev->pdev = pdev;
@@ -5440,38 +5436,38 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev)
 	ret = hclge_pci_init(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "PCI init failed\n");
-		goto err_pci_init;
+		goto out;
 	}
 
 	/* Firmware command queue initialize */
 	ret = hclge_cmd_queue_init(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "Cmd queue init failed, ret = %d.\n", ret);
-		return ret;
+		goto err_pci_uninit;
 	}
 
 	/* Firmware command initialize */
 	ret = hclge_cmd_init(hdev);
 	if (ret)
-		goto err_cmd_init;
+		goto err_cmd_uninit;
 
 	ret = hclge_get_cap(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "get hw capability error, ret = %d.\n",
 			ret);
-		return ret;
+		goto err_cmd_uninit;
 	}
 
 	ret = hclge_configure(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "Configure dev error, ret = %d.\n", ret);
-		return ret;
+		goto err_cmd_uninit;
 	}
 
 	ret = hclge_init_msi(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "Init MSI/MSI-X error, ret = %d.\n", ret);
-		return ret;
+		goto err_cmd_uninit;
 	}
 
 	ret = hclge_misc_irq_init(hdev);
@@ -5479,69 +5475,71 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev)
 		dev_err(&pdev->dev,
 			"Misc IRQ(vector0) init error, ret = %d.\n",
 			ret);
-		return ret;
+		goto err_msi_uninit;
 	}
 
 	ret = hclge_alloc_tqps(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "Allocate TQPs error, ret = %d.\n", ret);
-		return ret;
+		goto err_msi_irq_uninit;
 	}
 
 	ret = hclge_alloc_vport(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "Allocate vport error, ret = %d.\n", ret);
-		return ret;
+		goto err_msi_irq_uninit;
 	}
 
 	ret = hclge_map_tqp(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "Map tqp error, ret = %d.\n", ret);
-		return ret;
+		goto err_msi_irq_uninit;
 	}
 
-	ret = hclge_mac_mdio_config(hdev);
-	if (ret) {
-		dev_warn(&hdev->pdev->dev,
-			 "mdio config fail ret=%d\n", ret);
-		return ret;
+	if (hdev->hw.mac.media_type == HNAE3_MEDIA_TYPE_COPPER) {
+		ret = hclge_mac_mdio_config(hdev);
+		if (ret) {
+			dev_err(&hdev->pdev->dev,
+				"mdio config fail ret=%d\n", ret);
+			goto err_msi_irq_uninit;
+		}
 	}
 
 	ret = hclge_mac_init(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "Mac init error, ret = %d\n", ret);
-		return ret;
+		goto err_mdiobus_unreg;
 	}
 
 	ret = hclge_config_tso(hdev, HCLGE_TSO_MSS_MIN, HCLGE_TSO_MSS_MAX);
 	if (ret) {
 		dev_err(&pdev->dev, "Enable tso fail, ret =%d\n", ret);
-		return ret;
+		goto err_mdiobus_unreg;
 	}
 
 	ret = hclge_init_vlan_config(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "VLAN init fail, ret =%d\n", ret);
-		return  ret;
+		goto err_mdiobus_unreg;
 	}
 
 	ret = hclge_tm_schd_init(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "tm schd init fail, ret =%d\n", ret);
-		return ret;
+		goto err_mdiobus_unreg;
 	}
 
 	hclge_rss_init_cfg(hdev);
 	ret = hclge_rss_init_hw(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "Rss init fail, ret =%d\n", ret);
-		return ret;
+		goto err_mdiobus_unreg;
 	}
 
 	ret = init_mgr_tbl(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "manager table init fail, ret =%d\n", ret);
-		return ret;
+		goto err_mdiobus_unreg;
 	}
 
 	hclge_dcb_ops_set(hdev);
@@ -5564,11 +5562,21 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev)
 	pr_info("%s driver initialization finished.\n", HCLGE_DRIVER_NAME);
 	return 0;
 
-err_cmd_init:
+err_mdiobus_unreg:
+	if (hdev->hw.mac.phydev)
+		mdiobus_unregister(hdev->hw.mac.mdio_bus);
+err_msi_irq_uninit:
+	hclge_misc_irq_uninit(hdev);
+err_msi_uninit:
+	pci_free_irq_vectors(pdev);
+err_cmd_uninit:
+	hclge_destroy_cmd_queue(&hdev->hw);
+err_pci_uninit:
+	pcim_iounmap(pdev, hdev->hw.io_base);
+	pci_clear_master(pdev);
 	pci_release_regions(pdev);
-err_pci_init:
-	pci_set_drvdata(pdev, NULL);
-err_hclge_dev:
+	pci_disable_device(pdev);
+out:
 	return ret;
 }
 
@@ -5586,6 +5594,7 @@ static int hclge_reset_ae_dev(struct hnae3_ae_dev *ae_dev)
 	set_bit(HCLGE_STATE_DOWN, &hdev->state);
 
 	hclge_stats_clear(hdev);
+	memset(hdev->vlan_table, 0, sizeof(hdev->vlan_table));
 
 	ret = hclge_cmd_init(hdev);
 	if (ret) {
@@ -5658,9 +5667,6 @@ static void hclge_uninit_ae_dev(struct hnae3_ae_dev *ae_dev)
 
 	set_bit(HCLGE_STATE_DOWN, &hdev->state);
 
-	if (IS_ENABLED(CONFIG_PCI_IOV))
-		hclge_disable_sriov(hdev);
-
 	if (hdev->service_timer.function)
 		del_timer_sync(&hdev->service_timer);
 	if (hdev->service_task.func)
@@ -5985,9 +5991,7 @@ static void hclge_get_regs(struct hnae3_handle *handle, u32 *version,
 			"Get 64 bit register failed, ret = %d.\n", ret);
 }
 
-static int hclge_set_led_status_sfp(struct hclge_dev *hdev, u8 speed_led_status,
-				    u8 act_led_status, u8 link_led_status,
-				    u8 locate_led_status)
+static int hclge_set_led_status(struct hclge_dev *hdev, u8 locate_led_status)
 {
 	struct hclge_set_led_state_cmd *req;
 	struct hclge_desc desc;
@@ -5996,12 +6000,6 @@ static int hclge_set_led_status_sfp(struct hclge_dev *hdev, u8 speed_led_status,
 	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_LED_STATUS_CFG, false);
 
 	req = (struct hclge_set_led_state_cmd *)desc.data;
-	hnae_set_field(req->port_speed_led_config, HCLGE_LED_PORT_SPEED_STATE_M,
-		       HCLGE_LED_PORT_SPEED_STATE_S, speed_led_status);
-	hnae_set_field(req->link_led_config, HCLGE_LED_ACTIVITY_STATE_M,
-		       HCLGE_LED_ACTIVITY_STATE_S, act_led_status);
-	hnae_set_field(req->activity_led_config, HCLGE_LED_LINK_STATE_M,
-		       HCLGE_LED_LINK_STATE_S, link_led_status);
 	hnae_set_field(req->locate_led_config, HCLGE_LED_LOCATE_STATE_M,
 		       HCLGE_LED_LOCATE_STATE_S, locate_led_status);
 
@@ -6022,105 +6020,17 @@ enum hclge_led_status {
 static int hclge_set_led_id(struct hnae3_handle *handle,
 			    enum ethtool_phys_id_state status)
 {
-#define BLINK_FREQUENCY		2
 	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_dev *hdev = vport->back;
-	struct phy_device *phydev = hdev->hw.mac.phydev;
-	int ret = 0;
-
-	if (phydev || hdev->hw.mac.media_type != HNAE3_MEDIA_TYPE_FIBER)
-		return -EOPNOTSUPP;
 
 	switch (status) {
 	case ETHTOOL_ID_ACTIVE:
-		ret = hclge_set_led_status_sfp(hdev,
-					       HCLGE_LED_NO_CHANGE,
-					       HCLGE_LED_NO_CHANGE,
-					       HCLGE_LED_NO_CHANGE,
-					       HCLGE_LED_ON);
-		break;
+		return hclge_set_led_status(hdev, HCLGE_LED_ON);
 	case ETHTOOL_ID_INACTIVE:
-		ret = hclge_set_led_status_sfp(hdev,
-					       HCLGE_LED_NO_CHANGE,
-					       HCLGE_LED_NO_CHANGE,
-					       HCLGE_LED_NO_CHANGE,
-					       HCLGE_LED_OFF);
-		break;
+		return hclge_set_led_status(hdev, HCLGE_LED_OFF);
 	default:
-		ret = -EINVAL;
-		break;
-	}
-
-	return ret;
-}
-
-enum hclge_led_port_speed {
-	HCLGE_SPEED_LED_FOR_1G,
-	HCLGE_SPEED_LED_FOR_10G,
-	HCLGE_SPEED_LED_FOR_25G,
-	HCLGE_SPEED_LED_FOR_40G,
-	HCLGE_SPEED_LED_FOR_50G,
-	HCLGE_SPEED_LED_FOR_100G,
-};
-
-static u8 hclge_led_get_speed_status(u32 speed)
-{
-	u8 speed_led;
-
-	switch (speed) {
-	case HCLGE_MAC_SPEED_1G:
-		speed_led = HCLGE_SPEED_LED_FOR_1G;
-		break;
-	case HCLGE_MAC_SPEED_10G:
-		speed_led = HCLGE_SPEED_LED_FOR_10G;
-		break;
-	case HCLGE_MAC_SPEED_25G:
-		speed_led = HCLGE_SPEED_LED_FOR_25G;
-		break;
-	case HCLGE_MAC_SPEED_40G:
-		speed_led = HCLGE_SPEED_LED_FOR_40G;
-		break;
-	case HCLGE_MAC_SPEED_50G:
-		speed_led = HCLGE_SPEED_LED_FOR_50G;
-		break;
-	case HCLGE_MAC_SPEED_100G:
-		speed_led = HCLGE_SPEED_LED_FOR_100G;
-		break;
-	default:
-		speed_led = HCLGE_LED_NO_CHANGE;
+		return -EINVAL;
 	}
-
-	return speed_led;
-}
-
-static int hclge_update_led_status(struct hclge_dev *hdev)
-{
-	u8 port_speed_status, link_status, activity_status;
-	u64 rx_pkts, tx_pkts;
-
-	if (hdev->hw.mac.media_type != HNAE3_MEDIA_TYPE_FIBER)
-		return 0;
-
-	port_speed_status = hclge_led_get_speed_status(hdev->hw.mac.speed);
-
-	rx_pkts = hdev->hw_stats.mac_stats.mac_rx_total_pkt_num;
-	tx_pkts = hdev->hw_stats.mac_stats.mac_tx_total_pkt_num;
-	if (rx_pkts != hdev->rx_pkts_for_led ||
-	    tx_pkts != hdev->tx_pkts_for_led)
-		activity_status = HCLGE_LED_ON;
-	else
-		activity_status = HCLGE_LED_OFF;
-	hdev->rx_pkts_for_led = rx_pkts;
-	hdev->tx_pkts_for_led = tx_pkts;
-
-	if (hdev->hw.mac.link)
-		link_status = HCLGE_LED_ON;
-	else
-		link_status = HCLGE_LED_OFF;
-
-	return hclge_set_led_status_sfp(hdev, port_speed_status,
-					activity_status, link_status,
-					HCLGE_LED_NO_CHANGE);
 }
 
 static void hclge_get_link_mode(struct hnae3_handle *handle,
@@ -6203,7 +6113,7 @@ static const struct hnae3_ae_ops hclge_ops = {
 	.get_fw_version = hclge_get_fw_version,
 	.get_mdix_mode = hclge_get_mdix_mode,
 	.enable_vlan_filter = hclge_enable_vlan_filter,
-	.set_vlan_filter = hclge_set_port_vlan_filter,
+	.set_vlan_filter = hclge_set_vlan_filter,
 	.set_vf_vlan_filter = hclge_set_vf_vlan_filter,
 	.enable_hw_strip_rxvtag = hclge_en_hw_strip_rxvtag,
 	.reset_event = hclge_reset_event,
@@ -6228,7 +6138,9 @@ static int hclge_init(void)
 {
 	pr_info("%s is initializing\n", HCLGE_NAME);
 
-	return hnae3_register_ae_algo(&ae_algo);
+	hnae3_register_ae_algo(&ae_algo);
+
+	return 0;
 }
 
 static void hclge_exit(void)
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
index 0f4157e71282..7fcabdeb4ce9 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
@@ -12,10 +12,12 @@
 #include <linux/fs.h>
 #include <linux/types.h>
 #include <linux/phy.h>
+#include <linux/if_vlan.h>
+
 #include "hclge_cmd.h"
 #include "hnae3.h"
 
-#define HCLGE_MOD_VERSION "v1.0"
+#define HCLGE_MOD_VERSION "1.0"
 #define HCLGE_DRIVER_NAME "hclge"
 
 #define HCLGE_INVALID_VPORT 0xffff
@@ -406,9 +408,9 @@ struct hclge_mac_stats {
 	u64 mac_tx_1519_2047_oct_pkt_num;
 	u64 mac_tx_2048_4095_oct_pkt_num;
 	u64 mac_tx_4096_8191_oct_pkt_num;
-	u64 mac_tx_8192_12287_oct_pkt_num; /* valid for GE MAC only */
-	u64 mac_tx_8192_9216_oct_pkt_num; /* valid for LGE & CGE MAC only */
-	u64 mac_tx_9217_12287_oct_pkt_num; /* valid for LGE & CGE MAC */
+	u64 rsv0;
+	u64 mac_tx_8192_9216_oct_pkt_num;
+	u64 mac_tx_9217_12287_oct_pkt_num;
 	u64 mac_tx_12288_16383_oct_pkt_num;
 	u64 mac_tx_1519_max_good_oct_pkt_num;
 	u64 mac_tx_1519_max_bad_oct_pkt_num;
@@ -433,9 +435,9 @@ struct hclge_mac_stats {
 	u64 mac_rx_1519_2047_oct_pkt_num;
 	u64 mac_rx_2048_4095_oct_pkt_num;
 	u64 mac_rx_4096_8191_oct_pkt_num;
-	u64 mac_rx_8192_12287_oct_pkt_num;/* valid for GE MAC only */
-	u64 mac_rx_8192_9216_oct_pkt_num; /* valid for LGE & CGE MAC only */
-	u64 mac_rx_9217_12287_oct_pkt_num; /* valid for LGE & CGE MAC only */
+	u64 rsv1;
+	u64 mac_rx_8192_9216_oct_pkt_num;
+	u64 mac_rx_9217_12287_oct_pkt_num;
 	u64 mac_rx_12288_16383_oct_pkt_num;
 	u64 mac_rx_1519_max_good_oct_pkt_num;
 	u64 mac_rx_1519_max_bad_oct_pkt_num;
@@ -471,6 +473,7 @@ struct hclge_vlan_type_cfg {
 	u16 tx_in_vlan_type;
 };
 
+#define HCLGE_VPORT_NUM 256
 struct hclge_dev {
 	struct pci_dev *pdev;
 	struct hnae3_ae_dev *ae_dev;
@@ -560,14 +563,15 @@ struct hclge_dev {
 
 	struct hclge_vlan_type_cfg vlan_type_cfg;
 
-	u64 rx_pkts_for_led;
-	u64 tx_pkts_for_led;
+	unsigned long vlan_table[VLAN_N_VID][BITS_TO_LONGS(HCLGE_VPORT_NUM)];
 };
 
 /* VPort level vlan tag configuration for TX direction */
 struct hclge_tx_vtag_cfg {
-	bool accept_tag;	/* Whether accept tagged packet from host */
-	bool accept_untag;	/* Whether accept untagged packet from host */
+	bool accept_tag1;	/* Whether accept tag1 packet from host */
+	bool accept_untag1;	/* Whether accept untag1 packet from host */
+	bool accept_tag2;
+	bool accept_untag2;
 	bool insert_tag1_en;	/* Whether insert inner vlan tag */
 	bool insert_tag2_en;	/* Whether insert outer vlan tag */
 	u16  default_tag1;	/* The default inner vlan tag to insert */
@@ -646,8 +650,9 @@ static inline int hclge_get_queue_id(struct hnae3_queue *queue)
 }
 
 int hclge_cfg_mac_speed_dup(struct hclge_dev *hdev, int speed, u8 duplex);
-int hclge_set_vf_vlan_common(struct hclge_dev *vport, int vfid,
-			     bool is_kill, u16 vlan, u8 qos, __be16 proto);
+int hclge_set_vlan_filter(struct hnae3_handle *handle, __be16 proto,
+			  u16 vlan_id, bool is_kill);
+int hclge_en_hw_strip_rxvtag(struct hnae3_handle *handle, bool enable);
 
 int hclge_buffer_alloc(struct hclge_dev *hdev);
 int hclge_rss_init_hw(struct hclge_dev *hdev);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
index a6f7ffa9c259..31f3d9a43d8d 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
@@ -264,19 +264,23 @@ static int hclge_set_vf_vlan_cfg(struct hclge_vport *vport,
 				 struct hclge_mbx_vf_to_pf_cmd *mbx_req,
 				 bool gen_resp)
 {
-	struct hclge_dev *hdev = vport->back;
 	int status = 0;
 
 	if (mbx_req->msg[1] == HCLGE_MBX_VLAN_FILTER) {
+		struct hnae3_handle *handle = &vport->nic;
 		u16 vlan, proto;
 		bool is_kill;
 
 		is_kill = !!mbx_req->msg[2];
 		memcpy(&vlan, &mbx_req->msg[3], sizeof(vlan));
 		memcpy(&proto, &mbx_req->msg[5], sizeof(proto));
-		status = hclge_set_vf_vlan_common(hdev, vport->vport_id,
-						  is_kill, vlan, 0,
-						  cpu_to_be16(proto));
+		status = hclge_set_vlan_filter(handle, cpu_to_be16(proto),
+					       vlan, is_kill);
+	} else if (mbx_req->msg[1] == HCLGE_MBX_VLAN_RX_OFF_CFG) {
+		struct hnae3_handle *handle = &vport->nic;
+		bool en = mbx_req->msg[2] ? true : false;
+
+		status = hclge_en_hw_strip_rxvtag(handle, en);
 	}
 
 	if (gen_resp)
@@ -378,6 +382,13 @@ static void hclge_reset_vf(struct hclge_vport *vport,
 	hclge_func_reset_cmd(hdev, mbx_req->mbx_src_vfid);
 }
 
+static bool hclge_cmd_crq_empty(struct hclge_hw *hw)
+{
+	u32 tail = hclge_read_dev(hw, HCLGE_NIC_CRQ_TAIL_REG);
+
+	return tail == hw->cmq.crq.next_to_use;
+}
+
 void hclge_mbx_handler(struct hclge_dev *hdev)
 {
 	struct hclge_cmq_ring *crq = &hdev->hw.cmq.crq;
@@ -386,12 +397,23 @@ void hclge_mbx_handler(struct hclge_dev *hdev)
 	struct hclge_desc *desc;
 	int ret, flag;
 
-	flag = le16_to_cpu(crq->desc[crq->next_to_use].flag);
 	/* handle all the mailbox requests in the queue */
-	while (hnae_get_bit(flag, HCLGE_CMDQ_RX_OUTVLD_B)) {
+	while (!hclge_cmd_crq_empty(&hdev->hw)) {
 		desc = &crq->desc[crq->next_to_use];
 		req = (struct hclge_mbx_vf_to_pf_cmd *)desc->data;
 
+		flag = le16_to_cpu(crq->desc[crq->next_to_use].flag);
+		if (unlikely(!hnae_get_bit(flag, HCLGE_CMDQ_RX_OUTVLD_B))) {
+			dev_warn(&hdev->pdev->dev,
+				 "dropped invalid mailbox message, code = %d\n",
+				 req->msg[0]);
+
+			/* dropping/not processing this invalid message */
+			crq->desc[crq->next_to_use].flag = 0;
+			hclge_mbx_ring_ptr_move_crq(crq);
+			continue;
+		}
+
 		vport = &hdev->vport[req->mbx_src_vfid];
 
 		switch (req->msg[0]) {
@@ -466,7 +488,6 @@ void hclge_mbx_handler(struct hclge_dev *hdev)
 		}
 		crq->desc[crq->next_to_use].flag = 0;
 		hclge_mbx_ring_ptr_move_crq(crq);
-		flag = le16_to_cpu(crq->desc[crq->next_to_use].flag);
 	}
 
 	/* Write back CMDQ_RQ header pointer, M7 need this pointer */
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c
index 682c2d6618e7..9f7932e423b5 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c
@@ -140,8 +140,11 @@ int hclge_mac_mdio_config(struct hclge_dev *hdev)
 	struct mii_bus *mdio_bus;
 	int ret;
 
-	if (hdev->hw.mac.phy_addr >= PHY_MAX_ADDR)
-		return 0;
+	if (hdev->hw.mac.phy_addr >= PHY_MAX_ADDR) {
+		dev_err(&hdev->pdev->dev, "phy_addr(%d) is too large.\n",
+			hdev->hw.mac.phy_addr);
+		return -EINVAL;
+	}
 
 	mdio_bus = devm_mdiobus_alloc(&hdev->pdev->dev);
 	if (!mdio_bus)
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
index 885f25cd7be4..262c125f8137 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
@@ -134,11 +134,8 @@ static int hclge_pfc_stats_get(struct hclge_dev *hdev,
 	}
 
 	ret = hclge_cmd_send(&hdev->hw, desc, HCLGE_TM_PFC_PKT_GET_CMD_NUM);
-	if (ret) {
-		dev_err(&hdev->pdev->dev,
-			"Get pfc pause stats fail, ret = %d.\n", ret);
+	if (ret)
 		return ret;
-	}
 
 	for (i = 0; i < HCLGE_TM_PFC_PKT_GET_CMD_NUM; i++) {
 		struct hclge_pfc_stats_cmd *pfc_stats =
@@ -503,7 +500,8 @@ static int hclge_tm_qs_schd_mode_cfg(struct hclge_dev *hdev, u16 qs_id, u8 mode)
 	return hclge_cmd_send(&hdev->hw, &desc, 1);
 }
 
-static int hclge_tm_qs_bp_cfg(struct hclge_dev *hdev, u8 tc)
+static int hclge_tm_qs_bp_cfg(struct hclge_dev *hdev, u8 tc, u8 grp_id,
+			      u32 bit_map)
 {
 	struct hclge_bp_to_qs_map_cmd *bp_to_qs_map_cmd;
 	struct hclge_desc desc;
@@ -514,9 +512,8 @@ static int hclge_tm_qs_bp_cfg(struct hclge_dev *hdev, u8 tc)
 	bp_to_qs_map_cmd = (struct hclge_bp_to_qs_map_cmd *)desc.data;
 
 	bp_to_qs_map_cmd->tc_id = tc;
-
-	/* Qset and tc is one by one mapping */
-	bp_to_qs_map_cmd->qs_bit_map = cpu_to_le32(1 << tc);
+	bp_to_qs_map_cmd->qs_group_id = grp_id;
+	bp_to_qs_map_cmd->qs_bit_map = cpu_to_le32(bit_map);
 
 	return hclge_cmd_send(&hdev->hw, &desc, 1);
 }
@@ -1170,6 +1167,41 @@ static int hclge_pfc_setup_hw(struct hclge_dev *hdev)
 				      hdev->tm_info.hw_pfc_map);
 }
 
+/* Each Tc has a 1024 queue sets to backpress, it divides to
+ * 32 group, each group contains 32 queue sets, which can be
+ * represented by u32 bitmap.
+ */
+static int hclge_bp_setup_hw(struct hclge_dev *hdev, u8 tc)
+{
+	struct hclge_vport *vport = hdev->vport;
+	u32 i, k, qs_bitmap;
+	int ret;
+
+	for (i = 0; i < HCLGE_BP_GRP_NUM; i++) {
+		qs_bitmap = 0;
+
+		for (k = 0; k < hdev->num_alloc_vport; k++) {
+			u16 qs_id = vport->qs_offset + tc;
+			u8 grp, sub_grp;
+
+			grp = hnae_get_field(qs_id, HCLGE_BP_GRP_ID_M,
+					     HCLGE_BP_GRP_ID_S);
+			sub_grp = hnae_get_field(qs_id, HCLGE_BP_SUB_GRP_ID_M,
+						 HCLGE_BP_SUB_GRP_ID_S);
+			if (i == grp)
+				qs_bitmap |= (1 << sub_grp);
+
+			vport++;
+		}
+
+		ret = hclge_tm_qs_bp_cfg(hdev, tc, i, qs_bitmap);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static int hclge_mac_pause_setup_hw(struct hclge_dev *hdev)
 {
 	bool tx_en, rx_en;
@@ -1221,7 +1253,7 @@ int hclge_pause_setup_hw(struct hclge_dev *hdev)
 		dev_warn(&hdev->pdev->dev, "set pfc pause failed:%d\n", ret);
 
 	for (i = 0; i < hdev->tm_info.num_tc; i++) {
-		ret = hclge_tm_qs_bp_cfg(hdev, i);
+		ret = hclge_bp_setup_hw(hdev, i);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
index 2dbe177581e9..c2b6e8a6700f 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
@@ -89,6 +89,11 @@ struct hclge_pg_shapping_cmd {
 	__le32 pg_shapping_para;
 };
 
+#define HCLGE_BP_GRP_NUM		32
+#define HCLGE_BP_SUB_GRP_ID_S		0
+#define HCLGE_BP_SUB_GRP_ID_M		GENMASK(4, 0)
+#define HCLGE_BP_GRP_ID_S		5
+#define HCLGE_BP_GRP_ID_M		GENMASK(9, 5)
 struct hclge_bp_to_qs_map_cmd {
 	u8 tc_id;
 	u8 rsvd[2];
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
index 2b8426412cc9..266cdcba506e 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
@@ -830,6 +830,17 @@ static int hclgevf_set_vlan_filter(struct hnae3_handle *handle,
 				    HCLGEVF_VLAN_MBX_MSG_LEN, false, NULL, 0);
 }
 
+static int hclgevf_en_hw_strip_rxvtag(struct hnae3_handle *handle, bool enable)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	u8 msg_data;
+
+	msg_data = enable ? 1 : 0;
+	return hclgevf_send_mbx_msg(hdev, HCLGE_MBX_SET_VLAN,
+				    HCLGE_MBX_VLAN_RX_OFF_CFG, &msg_data,
+				    1, false, NULL, 0);
+}
+
 static void hclgevf_reset_tqp(struct hnae3_handle *handle, u16 queue_id)
 {
 	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
@@ -1489,10 +1500,12 @@ static int hclgevf_init_instance(struct hclgevf_dev *hdev,
 			return ret;
 		break;
 	case HNAE3_CLIENT_ROCE:
-		hdev->roce_client = client;
-		hdev->roce.client = client;
+		if (hnae3_dev_roce_supported(hdev)) {
+			hdev->roce_client = client;
+			hdev->roce.client = client;
+		}
 
-		if (hdev->roce_client && hnae3_dev_roce_supported(hdev)) {
+		if (hdev->roce_client && hdev->nic_client) {
 			ret = hclgevf_init_roce_base_info(hdev);
 			if (ret)
 				return ret;
@@ -1552,7 +1565,7 @@ static int hclgevf_pci_init(struct hclgevf_dev *hdev)
 	ret = pci_enable_device(pdev);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to enable PCI device\n");
-		goto err_no_drvdata;
+		return ret;
 	}
 
 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
@@ -1584,8 +1597,7 @@ static int hclgevf_pci_init(struct hclgevf_dev *hdev)
 	pci_release_regions(pdev);
 err_disable_device:
 	pci_disable_device(pdev);
-err_no_drvdata:
-	pci_set_drvdata(pdev, NULL);
+
 	return ret;
 }
 
@@ -1597,7 +1609,6 @@ static void hclgevf_pci_uninit(struct hclgevf_dev *hdev)
 	pci_clear_master(pdev);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
-	pci_set_drvdata(pdev, NULL);
 }
 
 static int hclgevf_init_hdev(struct hclgevf_dev *hdev)
@@ -1625,6 +1636,10 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev)
 
 	hclgevf_state_init(hdev);
 
+	ret = hclgevf_cmd_init(hdev);
+	if (ret)
+		goto err_cmd_init;
+
 	ret = hclgevf_misc_irq_init(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "failed(%d) to init Misc IRQ(vector0)\n",
@@ -1632,10 +1647,6 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev)
 		goto err_misc_irq_init;
 	}
 
-	ret = hclgevf_cmd_init(hdev);
-	if (ret)
-		goto err_cmd_init;
-
 	ret = hclgevf_configure(hdev);
 	if (ret) {
 		dev_err(&pdev->dev, "failed(%d) to fetch configuration\n", ret);
@@ -1683,10 +1694,10 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev)
 	return 0;
 
 err_config:
-	hclgevf_cmd_uninit(hdev);
-err_cmd_init:
 	hclgevf_misc_irq_uninit(hdev);
 err_misc_irq_init:
+	hclgevf_cmd_uninit(hdev);
+err_cmd_init:
 	hclgevf_state_uninit(hdev);
 	hclgevf_uninit_msi(hdev);
 err_irq_init:
@@ -1696,9 +1707,9 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev)
 
 static void hclgevf_uninit_hdev(struct hclgevf_dev *hdev)
 {
-	hclgevf_cmd_uninit(hdev);
-	hclgevf_misc_irq_uninit(hdev);
 	hclgevf_state_uninit(hdev);
+	hclgevf_misc_irq_uninit(hdev);
+	hclgevf_cmd_uninit(hdev);
 	hclgevf_uninit_msi(hdev);
 	hclgevf_pci_uninit(hdev);
 }
@@ -1825,6 +1836,7 @@ static const struct hnae3_ae_ops hclgevf_ops = {
 	.get_tc_size = hclgevf_get_tc_size,
 	.get_fw_version = hclgevf_get_fw_version,
 	.set_vlan_filter = hclgevf_set_vlan_filter,
+	.enable_hw_strip_rxvtag = hclgevf_en_hw_strip_rxvtag,
 	.reset_event = hclgevf_reset_event,
 	.get_channels = hclgevf_get_channels,
 	.get_tqps_and_rss_info = hclgevf_get_tqps_and_rss_info,
@@ -1842,7 +1854,9 @@ static int hclgevf_init(void)
 {
 	pr_info("%s is initializing\n", HCLGEVF_NAME);
 
-	return hnae3_register_ae_algo(&ae_algovf);
+	hnae3_register_ae_algo(&ae_algovf);
+
+	return 0;
 }
 
 static void hclgevf_exit(void)
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h
index a477a7c36bbd..9763e742e6fb 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h
@@ -9,7 +9,7 @@
 #include "hclgevf_cmd.h"
 #include "hnae3.h"
 
-#define HCLGEVF_MOD_VERSION "v1.0"
+#define HCLGEVF_MOD_VERSION "1.0"
 #define HCLGEVF_DRIVER_NAME "hclgevf"
 
 #define HCLGEVF_ROCEE_VECTOR_NUM	0
diff --git a/drivers/net/ethernet/huawei/hinic/Kconfig b/drivers/net/ethernet/huawei/hinic/Kconfig
index 08db24954f7e..e4e8b24c1a5d 100644
--- a/drivers/net/ethernet/huawei/hinic/Kconfig
+++ b/drivers/net/ethernet/huawei/hinic/Kconfig
@@ -4,7 +4,7 @@
 
 config HINIC
 	tristate "Huawei Intelligent PCIE Network Interface Card"
-	depends on (PCI_MSI && X86)
+	depends on (PCI_MSI && (X86 || ARM64))
 	---help---
 	  This driver supports HiNIC PCIE Ethernet cards.
 	  To compile this driver as part of the kernel, choose Y here.
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_main.c b/drivers/net/ethernet/huawei/hinic/hinic_main.c
index eb53bd93065e..5b122728dcb4 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_main.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_main.c
@@ -51,7 +51,9 @@ static unsigned int rx_weight = 64;
 module_param(rx_weight, uint, 0644);
 MODULE_PARM_DESC(rx_weight, "Number Rx packets for NAPI budget (default=64)");
 
-#define PCI_DEVICE_ID_HI1822_PF         0x1822
+#define HINIC_DEV_ID_QUAD_PORT_25GE     0x1822
+#define HINIC_DEV_ID_DUAL_PORT_25GE     0x0200
+#define HINIC_DEV_ID_DUAL_PORT_100GE    0x0201
 
 #define HINIC_WQ_NAME                   "hinic_dev"
 
@@ -1097,7 +1099,9 @@ static void hinic_remove(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id hinic_pci_table[] = {
-	{ PCI_VDEVICE(HUAWEI, PCI_DEVICE_ID_HI1822_PF), 0},
+	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_QUAD_PORT_25GE), 0},
+	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_DUAL_PORT_25GE), 0},
+	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_DUAL_PORT_100GE), 0},
 	{ 0, 0}
 };
 MODULE_DEVICE_TABLE(pci, hinic_pci_table);
diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
index 5ec1185808e5..d0e196bff081 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.c
+++ b/drivers/net/ethernet/ibm/ibmvnic.c
@@ -109,13 +109,14 @@ static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *,
 					struct ibmvnic_sub_crq_queue *);
 static int ibmvnic_poll(struct napi_struct *napi, int data);
 static void send_map_query(struct ibmvnic_adapter *adapter);
-static void send_request_map(struct ibmvnic_adapter *, dma_addr_t, __be32, u8);
-static void send_request_unmap(struct ibmvnic_adapter *, u8);
+static int send_request_map(struct ibmvnic_adapter *, dma_addr_t, __be32, u8);
+static int send_request_unmap(struct ibmvnic_adapter *, u8);
 static int send_login(struct ibmvnic_adapter *adapter);
 static void send_cap_queries(struct ibmvnic_adapter *adapter);
 static int init_sub_crqs(struct ibmvnic_adapter *);
 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter);
 static int ibmvnic_init(struct ibmvnic_adapter *);
+static int ibmvnic_reset_init(struct ibmvnic_adapter *);
 static void release_crq_queue(struct ibmvnic_adapter *);
 static int __ibmvnic_set_mac(struct net_device *netdev, struct sockaddr *p);
 static int init_crq_queue(struct ibmvnic_adapter *adapter);
@@ -172,6 +173,7 @@ static int alloc_long_term_buff(struct ibmvnic_adapter *adapter,
 				struct ibmvnic_long_term_buff *ltb, int size)
 {
 	struct device *dev = &adapter->vdev->dev;
+	int rc;
 
 	ltb->size = size;
 	ltb->buff = dma_alloc_coherent(dev, ltb->size, &ltb->addr,
@@ -185,8 +187,12 @@ static int alloc_long_term_buff(struct ibmvnic_adapter *adapter,
 	adapter->map_id++;
 
 	init_completion(&adapter->fw_done);
-	send_request_map(adapter, ltb->addr,
-			 ltb->size, ltb->map_id);
+	rc = send_request_map(adapter, ltb->addr,
+			      ltb->size, ltb->map_id);
+	if (rc) {
+		dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
+		return rc;
+	}
 	wait_for_completion(&adapter->fw_done);
 
 	if (adapter->fw_done_rc) {
@@ -215,10 +221,14 @@ static void free_long_term_buff(struct ibmvnic_adapter *adapter,
 static int reset_long_term_buff(struct ibmvnic_adapter *adapter,
 				struct ibmvnic_long_term_buff *ltb)
 {
+	int rc;
+
 	memset(ltb->buff, 0, ltb->size);
 
 	init_completion(&adapter->fw_done);
-	send_request_map(adapter, ltb->addr, ltb->size, ltb->map_id);
+	rc = send_request_map(adapter, ltb->addr, ltb->size, ltb->map_id);
+	if (rc)
+		return rc;
 	wait_for_completion(&adapter->fw_done);
 
 	if (adapter->fw_done_rc) {
@@ -789,6 +799,7 @@ static void release_napi(struct ibmvnic_adapter *adapter)
 	kfree(adapter->napi);
 	adapter->napi = NULL;
 	adapter->num_active_rx_napi = 0;
+	adapter->napi_enabled = false;
 }
 
 static int ibmvnic_login(struct net_device *netdev)
@@ -924,6 +935,10 @@ static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state)
 			/* Partuial success, delay and re-send */
 			mdelay(1000);
 			resend = true;
+		} else if (adapter->init_done_rc) {
+			netdev_warn(netdev, "Unable to set link state, rc=%d\n",
+				    adapter->init_done_rc);
+			return adapter->init_done_rc;
 		}
 	} while (resend);
 
@@ -956,6 +971,7 @@ static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter)
 	struct device *dev = &adapter->vdev->dev;
 	union ibmvnic_crq crq;
 	int len = 0;
+	int rc;
 
 	if (adapter->vpd->buff)
 		len = adapter->vpd->len;
@@ -963,7 +979,9 @@ static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter)
 	init_completion(&adapter->fw_done);
 	crq.get_vpd_size.first = IBMVNIC_CRQ_CMD;
 	crq.get_vpd_size.cmd = GET_VPD_SIZE;
-	ibmvnic_send_crq(adapter, &crq);
+	rc = ibmvnic_send_crq(adapter, &crq);
+	if (rc)
+		return rc;
 	wait_for_completion(&adapter->fw_done);
 
 	if (!adapter->vpd->len)
@@ -996,7 +1014,12 @@ static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter)
 	crq.get_vpd.cmd = GET_VPD;
 	crq.get_vpd.ioba = cpu_to_be32(adapter->vpd->dma_addr);
 	crq.get_vpd.len = cpu_to_be32((u32)adapter->vpd->len);
-	ibmvnic_send_crq(adapter, &crq);
+	rc = ibmvnic_send_crq(adapter, &crq);
+	if (rc) {
+		kfree(adapter->vpd->buff);
+		adapter->vpd->buff = NULL;
+		return rc;
+	}
 	wait_for_completion(&adapter->fw_done);
 
 	return 0;
@@ -1695,6 +1718,7 @@ static int __ibmvnic_set_mac(struct net_device *netdev, struct sockaddr *p)
 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
 	struct sockaddr *addr = p;
 	union ibmvnic_crq crq;
+	int rc;
 
 	if (!is_valid_ether_addr(addr->sa_data))
 		return -EADDRNOTAVAIL;
@@ -1705,7 +1729,9 @@ static int __ibmvnic_set_mac(struct net_device *netdev, struct sockaddr *p)
 	ether_addr_copy(&crq.change_mac_addr.mac_addr[0], addr->sa_data);
 
 	init_completion(&adapter->fw_done);
-	ibmvnic_send_crq(adapter, &crq);
+	rc = ibmvnic_send_crq(adapter, &crq);
+	if (rc)
+		return rc;
 	wait_for_completion(&adapter->fw_done);
 	/* netdev->dev_addr is changed in handle_change_mac_rsp function */
 	return adapter->fw_done_rc ? -EIO : 0;
@@ -1787,7 +1813,7 @@ static int do_reset(struct ibmvnic_adapter *adapter,
 			return rc;
 		}
 
-		rc = ibmvnic_init(adapter);
+		rc = ibmvnic_reset_init(adapter);
 		if (rc)
 			return IBMVNIC_INIT_FAILED;
 
@@ -1857,6 +1883,85 @@ static int do_reset(struct ibmvnic_adapter *adapter,
 	return 0;
 }
 
+static int do_hard_reset(struct ibmvnic_adapter *adapter,
+			 struct ibmvnic_rwi *rwi, u32 reset_state)
+{
+	struct net_device *netdev = adapter->netdev;
+	int rc;
+
+	netdev_dbg(adapter->netdev, "Hard resetting driver (%d)\n",
+		   rwi->reset_reason);
+
+	netif_carrier_off(netdev);
+	adapter->reset_reason = rwi->reset_reason;
+
+	ibmvnic_cleanup(netdev);
+	release_resources(adapter);
+	release_sub_crqs(adapter, 0);
+	release_crq_queue(adapter);
+
+	/* remove the closed state so when we call open it appears
+	 * we are coming from the probed state.
+	 */
+	adapter->state = VNIC_PROBED;
+
+	rc = init_crq_queue(adapter);
+	if (rc) {
+		netdev_err(adapter->netdev,
+			   "Couldn't initialize crq. rc=%d\n", rc);
+		return rc;
+	}
+
+	rc = ibmvnic_init(adapter);
+	if (rc)
+		return rc;
+
+	/* If the adapter was in PROBE state prior to the reset,
+	 * exit here.
+	 */
+	if (reset_state == VNIC_PROBED)
+		return 0;
+
+	rc = ibmvnic_login(netdev);
+	if (rc) {
+		adapter->state = VNIC_PROBED;
+		return 0;
+	}
+	/* netif_set_real_num_xx_queues needs to take rtnl lock here
+	 * unless wait_for_reset is set, in which case the rtnl lock
+	 * has already been taken before initializing the reset
+	 */
+	if (!adapter->wait_for_reset) {
+		rtnl_lock();
+		rc = init_resources(adapter);
+		rtnl_unlock();
+	} else {
+		rc = init_resources(adapter);
+	}
+	if (rc)
+		return rc;
+
+	ibmvnic_disable_irqs(adapter);
+	adapter->state = VNIC_CLOSED;
+
+	if (reset_state == VNIC_CLOSED)
+		return 0;
+
+	rc = __ibmvnic_open(netdev);
+	if (rc) {
+		if (list_empty(&adapter->rwi_list))
+			adapter->state = VNIC_CLOSED;
+		else
+			adapter->state = reset_state;
+
+		return 0;
+	}
+
+	netif_carrier_on(netdev);
+
+	return 0;
+}
+
 static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter)
 {
 	struct ibmvnic_rwi *rwi;
@@ -1898,14 +2003,19 @@ static void __ibmvnic_reset(struct work_struct *work)
 	netdev = adapter->netdev;
 
 	mutex_lock(&adapter->reset_lock);
-	adapter->resetting = true;
 	reset_state = adapter->state;
 
 	rwi = get_next_rwi(adapter);
 	while (rwi) {
-		rc = do_reset(adapter, rwi, reset_state);
+		if (adapter->force_reset_recovery) {
+			adapter->force_reset_recovery = false;
+			rc = do_hard_reset(adapter, rwi, reset_state);
+		} else {
+			rc = do_reset(adapter, rwi, reset_state);
+		}
 		kfree(rwi);
-		if (rc && rc != IBMVNIC_INIT_FAILED)
+		if (rc && rc != IBMVNIC_INIT_FAILED &&
+		    !adapter->force_reset_recovery)
 			break;
 
 		rwi = get_next_rwi(adapter);
@@ -1931,9 +2041,9 @@ static void __ibmvnic_reset(struct work_struct *work)
 static int ibmvnic_reset(struct ibmvnic_adapter *adapter,
 			 enum ibmvnic_reset_reason reason)
 {
+	struct list_head *entry, *tmp_entry;
 	struct ibmvnic_rwi *rwi, *tmp;
 	struct net_device *netdev = adapter->netdev;
-	struct list_head *entry;
 	int ret;
 
 	if (adapter->state == VNIC_REMOVING ||
@@ -1969,11 +2079,17 @@ static int ibmvnic_reset(struct ibmvnic_adapter *adapter,
 		ret = ENOMEM;
 		goto err;
 	}
-
+	/* if we just received a transport event,
+	 * flush reset queue and process this reset
+	 */
+	if (adapter->force_reset_recovery && !list_empty(&adapter->rwi_list)) {
+		list_for_each_safe(entry, tmp_entry, &adapter->rwi_list)
+			list_del(entry);
+	}
 	rwi->reset_reason = reason;
 	list_add_tail(&rwi->list, &adapter->rwi_list);
 	mutex_unlock(&adapter->rwi_lock);
-
+	adapter->resetting = true;
 	netdev_dbg(adapter->netdev, "Scheduling reset (reason %d)\n", reason);
 	schedule_work(&adapter->ibmvnic_reset);
 
@@ -2369,6 +2485,7 @@ static void ibmvnic_get_ethtool_stats(struct net_device *dev,
 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
 	union ibmvnic_crq crq;
 	int i, j;
+	int rc;
 
 	memset(&crq, 0, sizeof(crq));
 	crq.request_statistics.first = IBMVNIC_CRQ_CMD;
@@ -2379,7 +2496,9 @@ static void ibmvnic_get_ethtool_stats(struct net_device *dev,
 
 	/* Wait for data to be written */
 	init_completion(&adapter->stats_done);
-	ibmvnic_send_crq(adapter, &crq);
+	rc = ibmvnic_send_crq(adapter, &crq);
+	if (rc)
+		return;
 	wait_for_completion(&adapter->stats_done);
 
 	for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++)
@@ -3154,6 +3273,12 @@ static int ibmvnic_send_crq(struct ibmvnic_adapter *adapter,
 		   (unsigned long int)cpu_to_be64(u64_crq[0]),
 		   (unsigned long int)cpu_to_be64(u64_crq[1]));
 
+	if (!adapter->crq.active &&
+	    crq->generic.first != IBMVNIC_CRQ_INIT_CMD) {
+		dev_warn(dev, "Invalid request detected while CRQ is inactive, possible device state change during reset\n");
+		return -EINVAL;
+	}
+
 	/* Make sure the hypervisor sees the complete request */
 	mb();
 
@@ -3378,8 +3503,8 @@ static int send_login(struct ibmvnic_adapter *adapter)
 	return -1;
 }
 
-static void send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr,
-			     u32 len, u8 map_id)
+static int send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr,
+			    u32 len, u8 map_id)
 {
 	union ibmvnic_crq crq;
 
@@ -3389,10 +3514,10 @@ static void send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr,
 	crq.request_map.map_id = map_id;
 	crq.request_map.ioba = cpu_to_be32(addr);
 	crq.request_map.len = cpu_to_be32(len);
-	ibmvnic_send_crq(adapter, &crq);
+	return ibmvnic_send_crq(adapter, &crq);
 }
 
-static void send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id)
+static int send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id)
 {
 	union ibmvnic_crq crq;
 
@@ -3400,7 +3525,7 @@ static void send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id)
 	crq.request_unmap.first = IBMVNIC_CRQ_CMD;
 	crq.request_unmap.cmd = REQUEST_UNMAP;
 	crq.request_unmap.map_id = map_id;
-	ibmvnic_send_crq(adapter, &crq);
+	return ibmvnic_send_crq(adapter, &crq);
 }
 
 static void send_map_query(struct ibmvnic_adapter *adapter)
@@ -4227,11 +4352,15 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq,
 			dev_info(dev, "Partner initialized\n");
 			adapter->from_passive_init = true;
 			adapter->failover_pending = false;
-			complete(&adapter->init_done);
+			if (!completion_done(&adapter->init_done)) {
+				complete(&adapter->init_done);
+				adapter->init_done_rc = -EIO;
+			}
 			ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);
 			break;
 		case IBMVNIC_CRQ_INIT_COMPLETE:
 			dev_info(dev, "Partner initialization complete\n");
+			adapter->crq.active = true;
 			send_version_xchg(adapter);
 			break;
 		default:
@@ -4240,6 +4369,9 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq,
 		return;
 	case IBMVNIC_CRQ_XPORT_EVENT:
 		netif_carrier_off(netdev);
+		adapter->crq.active = false;
+		if (adapter->resetting)
+			adapter->force_reset_recovery = true;
 		if (gen_crq->cmd == IBMVNIC_PARTITION_MIGRATED) {
 			dev_info(dev, "Migrated, re-enabling adapter\n");
 			ibmvnic_reset(adapter, VNIC_RESET_MOBILITY);
@@ -4427,6 +4559,7 @@ static int ibmvnic_reset_crq(struct ibmvnic_adapter *adapter)
 	/* Clean out the queue */
 	memset(crq->msgs, 0, PAGE_SIZE);
 	crq->cur = 0;
+	crq->active = false;
 
 	/* And re-open it again */
 	rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address,
@@ -4461,6 +4594,7 @@ static void release_crq_queue(struct ibmvnic_adapter *adapter)
 			 DMA_BIDIRECTIONAL);
 	free_page((unsigned long)crq->msgs);
 	crq->msgs = NULL;
+	crq->active = false;
 }
 
 static int init_crq_queue(struct ibmvnic_adapter *adapter)
@@ -4538,7 +4672,7 @@ static int init_crq_queue(struct ibmvnic_adapter *adapter)
 	return retrc;
 }
 
-static int ibmvnic_init(struct ibmvnic_adapter *adapter)
+static int ibmvnic_reset_init(struct ibmvnic_adapter *adapter)
 {
 	struct device *dev = &adapter->vdev->dev;
 	unsigned long timeout = msecs_to_jiffies(30000);
@@ -4597,6 +4731,49 @@ static int ibmvnic_init(struct ibmvnic_adapter *adapter)
 	return rc;
 }
 
+static int ibmvnic_init(struct ibmvnic_adapter *adapter)
+{
+	struct device *dev = &adapter->vdev->dev;
+	unsigned long timeout = msecs_to_jiffies(30000);
+	int rc;
+
+	adapter->from_passive_init = false;
+
+	init_completion(&adapter->init_done);
+	adapter->init_done_rc = 0;
+	ibmvnic_send_crq_init(adapter);
+	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {
+		dev_err(dev, "Initialization sequence timed out\n");
+		return -1;
+	}
+
+	if (adapter->init_done_rc) {
+		release_crq_queue(adapter);
+		return adapter->init_done_rc;
+	}
+
+	if (adapter->from_passive_init) {
+		adapter->state = VNIC_OPEN;
+		adapter->from_passive_init = false;
+		return -1;
+	}
+
+	rc = init_sub_crqs(adapter);
+	if (rc) {
+		dev_err(dev, "Initialization of sub crqs failed\n");
+		release_crq_queue(adapter);
+		return rc;
+	}
+
+	rc = init_sub_crq_irqs(adapter);
+	if (rc) {
+		dev_err(dev, "Failed to initialize sub crq irqs\n");
+		release_crq_queue(adapter);
+	}
+
+	return rc;
+}
+
 static struct device_attribute dev_attr_failover;
 
 static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h
index 22391e8805f6..f9fb780102ac 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.h
+++ b/drivers/net/ethernet/ibm/ibmvnic.h
@@ -865,6 +865,7 @@ struct ibmvnic_crq_queue {
 	int size, cur;
 	dma_addr_t msg_token;
 	spinlock_t lock;
+	bool active;
 };
 
 union sub_crq {
@@ -1108,6 +1109,7 @@ struct ibmvnic_adapter {
 
 	bool mac_change_pending;
 	bool failover_pending;
+	bool force_reset_recovery;
 
 	struct ibmvnic_tunables desired;
 	struct ibmvnic_tunables fallback;
diff --git a/drivers/net/ethernet/intel/e100.c b/drivers/net/ethernet/intel/e100.c
index 41ad56edfb96..27d5f27163d2 100644
--- a/drivers/net/ethernet/intel/e100.c
+++ b/drivers/net/ethernet/intel/e100.c
@@ -1,31 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-
-  Intel PRO/100 Linux driver
-  Copyright(c) 1999 - 2006 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2006 Intel Corporation. */
 
 /*
  *	e100.c: Intel(R) PRO/100 ethernet driver
diff --git a/drivers/net/ethernet/intel/e1000/Makefile b/drivers/net/ethernet/intel/e1000/Makefile
index c7caadd3c8af..314c52d44b7c 100644
--- a/drivers/net/ethernet/intel/e1000/Makefile
+++ b/drivers/net/ethernet/intel/e1000/Makefile
@@ -1,31 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-################################################################################
-#
-# Intel PRO/1000 Linux driver
 # Copyright(c) 1999 - 2006 Intel Corporation.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms and conditions of the GNU General Public License,
-# version 2, as published by the Free Software Foundation.
-#
-# This program is distributed in the hope it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# The full GNU General Public License is included in this distribution in
-# the file called "COPYING".
-#
-# Contact Information:
-# Linux NICS <linux.nics@intel.com>
-# e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-# Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-#
-################################################################################
 
 #
 # Makefile for the Intel(R) PRO/1000 ethernet driver
diff --git a/drivers/net/ethernet/intel/e1000/e1000.h b/drivers/net/ethernet/intel/e1000/e1000.h
index 3a0feea2df54..c40729b2c184 100644
--- a/drivers/net/ethernet/intel/e1000/e1000.h
+++ b/drivers/net/ethernet/intel/e1000/e1000.h
@@ -1,32 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel PRO/1000 Linux driver
-  Copyright(c) 1999 - 2006 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
-
+/* Copyright(c) 1999 - 2006 Intel Corporation. */
 
 /* Linux PRO/1000 Ethernet Driver main header file */
 
diff --git a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c
index 3e80ca170dd7..5d365a986bb0 100644
--- a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c
+++ b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c
@@ -1,26 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- * Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2006 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 1999 - 2006 Intel Corporation. */
 
 /* ethtool support for e1000 */
 
diff --git a/drivers/net/ethernet/intel/e1000/e1000_hw.c b/drivers/net/ethernet/intel/e1000/e1000_hw.c
index 6e7e923d57bf..48428d6a00be 100644
--- a/drivers/net/ethernet/intel/e1000/e1000_hw.c
+++ b/drivers/net/ethernet/intel/e1000/e1000_hw.c
@@ -1,31 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-*
-  Intel PRO/1000 Linux driver
-  Copyright(c) 1999 - 2006 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
- */
+/* Copyright(c) 1999 - 2006 Intel Corporation. */
 
 /* e1000_hw.c
  * Shared functions for accessing and configuring the MAC
diff --git a/drivers/net/ethernet/intel/e1000/e1000_hw.h b/drivers/net/ethernet/intel/e1000/e1000_hw.h
index f09c569ec19b..b57a04954ccf 100644
--- a/drivers/net/ethernet/intel/e1000/e1000_hw.h
+++ b/drivers/net/ethernet/intel/e1000/e1000_hw.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel PRO/1000 Linux driver
-  Copyright(c) 1999 - 2006 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2006 Intel Corporation. */
 
 /* e1000_hw.h
  * Structures, enums, and macros for the MAC
diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c
index d5eb19b86a0a..2110d5f2da19 100644
--- a/drivers/net/ethernet/intel/e1000/e1000_main.c
+++ b/drivers/net/ethernet/intel/e1000/e1000_main.c
@@ -1,31 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-
-  Intel PRO/1000 Linux driver
-  Copyright(c) 1999 - 2006 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2006 Intel Corporation. */
 
 #include "e1000.h"
 #include <net/ip6_checksum.h>
diff --git a/drivers/net/ethernet/intel/e1000/e1000_osdep.h b/drivers/net/ethernet/intel/e1000/e1000_osdep.h
index ae0559b8b011..e966bb290797 100644
--- a/drivers/net/ethernet/intel/e1000/e1000_osdep.h
+++ b/drivers/net/ethernet/intel/e1000/e1000_osdep.h
@@ -1,32 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel PRO/1000 Linux driver
-  Copyright(c) 1999 - 2006 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
-
+/* Copyright(c) 1999 - 2006 Intel Corporation. */
 
 /* glue for the OS independent part of e1000
  * includes register access macros
diff --git a/drivers/net/ethernet/intel/e1000/e1000_param.c b/drivers/net/ethernet/intel/e1000/e1000_param.c
index 345f23927bcc..d3f29ffe1e47 100644
--- a/drivers/net/ethernet/intel/e1000/e1000_param.c
+++ b/drivers/net/ethernet/intel/e1000/e1000_param.c
@@ -1,31 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-
-  Intel PRO/1000 Linux driver
-  Copyright(c) 1999 - 2006 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2006 Intel Corporation. */
 
 #include "e1000.h"
 
diff --git a/drivers/net/ethernet/intel/e1000e/80003es2lan.c b/drivers/net/ethernet/intel/e1000e/80003es2lan.c
index 953e99df420c..257bd59bc9c6 100644
--- a/drivers/net/ethernet/intel/e1000e/80003es2lan.c
+++ b/drivers/net/ethernet/intel/e1000e/80003es2lan.c
@@ -1,24 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 /* 80003ES2LAN Gigabit Ethernet Controller (Copper)
  * 80003ES2LAN Gigabit Ethernet Controller (Serdes)
diff --git a/drivers/net/ethernet/intel/e1000e/80003es2lan.h b/drivers/net/ethernet/intel/e1000e/80003es2lan.h
index ee6d1256fda4..aa9d639c6cbb 100644
--- a/drivers/net/ethernet/intel/e1000e/80003es2lan.h
+++ b/drivers/net/ethernet/intel/e1000e/80003es2lan.h
@@ -1,24 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000E_80003ES2LAN_H_
 #define _E1000E_80003ES2LAN_H_
diff --git a/drivers/net/ethernet/intel/e1000e/82571.c b/drivers/net/ethernet/intel/e1000e/82571.c
index 924f2c8dfa6c..b9309302c29e 100644
--- a/drivers/net/ethernet/intel/e1000e/82571.c
+++ b/drivers/net/ethernet/intel/e1000e/82571.c
@@ -1,24 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 /* 82571EB Gigabit Ethernet Controller
  * 82571EB Gigabit Ethernet Controller (Copper)
diff --git a/drivers/net/ethernet/intel/e1000e/82571.h b/drivers/net/ethernet/intel/e1000e/82571.h
index 9a24c645f726..834c238d02db 100644
--- a/drivers/net/ethernet/intel/e1000e/82571.h
+++ b/drivers/net/ethernet/intel/e1000e/82571.h
@@ -1,24 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000E_82571_H_
 #define _E1000E_82571_H_
diff --git a/drivers/net/ethernet/intel/e1000e/Makefile b/drivers/net/ethernet/intel/e1000e/Makefile
index 24e391a4ac68..44e58b6e7660 100644
--- a/drivers/net/ethernet/intel/e1000e/Makefile
+++ b/drivers/net/ethernet/intel/e1000e/Makefile
@@ -1,30 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-################################################################################
-#
-# Intel PRO/1000 Linux driver
-# Copyright(c) 1999 - 2014 Intel Corporation.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms and conditions of the GNU General Public License,
-# version 2, as published by the Free Software Foundation.
-#
-# This program is distributed in the hope it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
-#
-# The full GNU General Public License is included in this distribution in
-# the file called "COPYING".
-#
-# Contact Information:
-# Linux NICS <linux.nics@intel.com>
-# e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-# Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-#
-################################################################################
+# Copyright(c) 1999 - 2018 Intel Corporation.
 
 #
 # Makefile for the Intel(R) PRO/1000 ethernet driver
diff --git a/drivers/net/ethernet/intel/e1000e/defines.h b/drivers/net/ethernet/intel/e1000e/defines.h
index 22883015a695..fd550dee4982 100644
--- a/drivers/net/ethernet/intel/e1000e/defines.h
+++ b/drivers/net/ethernet/intel/e1000e/defines.h
@@ -1,24 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000_DEFINES_H_
 #define _E1000_DEFINES_H_
diff --git a/drivers/net/ethernet/intel/e1000e/e1000.h b/drivers/net/ethernet/intel/e1000e/e1000.h
index da88555ba1fd..c760dc72c520 100644
--- a/drivers/net/ethernet/intel/e1000e/e1000.h
+++ b/drivers/net/ethernet/intel/e1000e/e1000.h
@@ -1,24 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 /* Linux PRO/1000 Ethernet Driver main header file */
 
diff --git a/drivers/net/ethernet/intel/e1000e/ethtool.c b/drivers/net/ethernet/intel/e1000e/ethtool.c
index 64dc0c11147f..e084cb734eb1 100644
--- a/drivers/net/ethernet/intel/e1000e/ethtool.c
+++ b/drivers/net/ethernet/intel/e1000e/ethtool.c
@@ -1,24 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 /* ethtool support for e1000 */
 
diff --git a/drivers/net/ethernet/intel/e1000e/hw.h b/drivers/net/ethernet/intel/e1000e/hw.h
index 21802396bed6..eff75bd8a8f0 100644
--- a/drivers/net/ethernet/intel/e1000e/hw.h
+++ b/drivers/net/ethernet/intel/e1000e/hw.h
@@ -1,24 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000_HW_H_
 #define _E1000_HW_H_
diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c
index 1551d6ce5341..cdae0efde8e6 100644
--- a/drivers/net/ethernet/intel/e1000e/ich8lan.c
+++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c
@@ -1,24 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 /* 82562G 10/100 Network Connection
  * 82562G-2 10/100 Network Connection
diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.h b/drivers/net/ethernet/intel/e1000e/ich8lan.h
index 3c4f82c21084..eb09c755fa17 100644
--- a/drivers/net/ethernet/intel/e1000e/ich8lan.h
+++ b/drivers/net/ethernet/intel/e1000e/ich8lan.h
@@ -1,24 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000E_ICH8LAN_H_
 #define _E1000E_ICH8LAN_H_
diff --git a/drivers/net/ethernet/intel/e1000e/mac.c b/drivers/net/ethernet/intel/e1000e/mac.c
index b293464a9f27..4abd55d646c5 100644
--- a/drivers/net/ethernet/intel/e1000e/mac.c
+++ b/drivers/net/ethernet/intel/e1000e/mac.c
@@ -1,24 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "e1000.h"
 
diff --git a/drivers/net/ethernet/intel/e1000e/mac.h b/drivers/net/ethernet/intel/e1000e/mac.h
index cb0abf6c76a5..6ab261119801 100644
--- a/drivers/net/ethernet/intel/e1000e/mac.h
+++ b/drivers/net/ethernet/intel/e1000e/mac.h
@@ -1,24 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000E_MAC_H_
 #define _E1000E_MAC_H_
diff --git a/drivers/net/ethernet/intel/e1000e/manage.c b/drivers/net/ethernet/intel/e1000e/manage.c
index e027660aeb92..c4c9b20bc51f 100644
--- a/drivers/net/ethernet/intel/e1000e/manage.c
+++ b/drivers/net/ethernet/intel/e1000e/manage.c
@@ -1,24 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "e1000.h"
 
diff --git a/drivers/net/ethernet/intel/e1000e/manage.h b/drivers/net/ethernet/intel/e1000e/manage.h
index 3268f2e58593..d868aad806d4 100644
--- a/drivers/net/ethernet/intel/e1000e/manage.h
+++ b/drivers/net/ethernet/intel/e1000e/manage.h
@@ -1,24 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000E_MANAGE_H_
 #define _E1000E_MANAGE_H_
diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c
index ec4a9759a6f2..d3fef7fefea8 100644
--- a/drivers/net/ethernet/intel/e1000e/netdev.c
+++ b/drivers/net/ethernet/intel/e1000e/netdev.c
@@ -1,24 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
diff --git a/drivers/net/ethernet/intel/e1000e/nvm.c b/drivers/net/ethernet/intel/e1000e/nvm.c
index 68949bb41b7b..937f9af22d26 100644
--- a/drivers/net/ethernet/intel/e1000e/nvm.c
+++ b/drivers/net/ethernet/intel/e1000e/nvm.c
@@ -1,24 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "e1000.h"
 
diff --git a/drivers/net/ethernet/intel/e1000e/nvm.h b/drivers/net/ethernet/intel/e1000e/nvm.h
index 8e082028be7d..6a30dfea4117 100644
--- a/drivers/net/ethernet/intel/e1000e/nvm.h
+++ b/drivers/net/ethernet/intel/e1000e/nvm.h
@@ -1,24 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000E_NVM_H_
 #define _E1000E_NVM_H_
diff --git a/drivers/net/ethernet/intel/e1000e/param.c b/drivers/net/ethernet/intel/e1000e/param.c
index 2def33eba9e6..098369fd3e65 100644
--- a/drivers/net/ethernet/intel/e1000e/param.c
+++ b/drivers/net/ethernet/intel/e1000e/param.c
@@ -1,24 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include <linux/netdevice.h>
 #include <linux/module.h>
diff --git a/drivers/net/ethernet/intel/e1000e/phy.c b/drivers/net/ethernet/intel/e1000e/phy.c
index b8226ed0e338..42233019255a 100644
--- a/drivers/net/ethernet/intel/e1000e/phy.c
+++ b/drivers/net/ethernet/intel/e1000e/phy.c
@@ -1,24 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "e1000.h"
 
diff --git a/drivers/net/ethernet/intel/e1000e/phy.h b/drivers/net/ethernet/intel/e1000e/phy.h
index d4180b5e9196..c48777d09523 100644
--- a/drivers/net/ethernet/intel/e1000e/phy.h
+++ b/drivers/net/ethernet/intel/e1000e/phy.h
@@ -1,24 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000E_PHY_H_
 #define _E1000E_PHY_H_
diff --git a/drivers/net/ethernet/intel/e1000e/ptp.c b/drivers/net/ethernet/intel/e1000e/ptp.c
index f941e5085f44..37c76945ad9b 100644
--- a/drivers/net/ethernet/intel/e1000e/ptp.c
+++ b/drivers/net/ethernet/intel/e1000e/ptp.c
@@ -1,24 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 /* PTP 1588 Hardware Clock (PHC)
  * Derived from PTP Hardware Clock driver for Intel 82576 and 82580 (igb)
diff --git a/drivers/net/ethernet/intel/e1000e/regs.h b/drivers/net/ethernet/intel/e1000e/regs.h
index 16afc3c2a986..47f5ca793970 100644
--- a/drivers/net/ethernet/intel/e1000e/regs.h
+++ b/drivers/net/ethernet/intel/e1000e/regs.h
@@ -1,24 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel PRO/1000 Linux driver
- * Copyright(c) 1999 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000E_REGS_H_
 #define _E1000E_REGS_H_
diff --git a/drivers/net/ethernet/intel/fm10k/Makefile b/drivers/net/ethernet/intel/fm10k/Makefile
index 93277cb99cb7..26a9746ccb14 100644
--- a/drivers/net/ethernet/intel/fm10k/Makefile
+++ b/drivers/net/ethernet/intel/fm10k/Makefile
@@ -1,26 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-################################################################################
-#
-# Intel(R) Ethernet Switch Host Interface Driver
-# Copyright(c) 2013 - 2016 Intel Corporation.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms and conditions of the GNU General Public License,
-# version 2, as published by the Free Software Foundation.
-#
-# This program is distributed in the hope it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# The full GNU General Public License is included in this distribution in
-# the file called "COPYING".
-#
-# Contact Information:
-# e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-# Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-#
-################################################################################
+# Copyright(c) 2013 - 2018 Intel Corporation.
 
 #
 # Makefile for the Intel(R) Ethernet Switch Host Interface Driver
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k.h b/drivers/net/ethernet/intel/fm10k/fm10k.h
index a9cdf763c59d..a903a0ba45e1 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k.h
+++ b/drivers/net/ethernet/intel/fm10k/fm10k.h
@@ -1,23 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _FM10K_H_
 #define _FM10K_H_
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_common.c b/drivers/net/ethernet/intel/fm10k/fm10k_common.c
index e303d88720ef..f51a63fca513 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_common.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_common.c
@@ -1,23 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2018 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "fm10k_common.h"
 
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_common.h b/drivers/net/ethernet/intel/fm10k/fm10k_common.h
index 2bdb24d2ca9d..4c48fb73b3e7 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_common.h
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_common.h
@@ -1,23 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _FM10K_COMMON_H_
 #define _FM10K_COMMON_H_
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_dcbnl.c b/drivers/net/ethernet/intel/fm10k/fm10k_dcbnl.c
index c4f733452ef2..20768ac7f17e 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_dcbnl.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_dcbnl.c
@@ -1,23 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "fm10k.h"
 
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_debugfs.c b/drivers/net/ethernet/intel/fm10k/fm10k_debugfs.c
index 43e8d839831f..dca104121c05 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_debugfs.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_debugfs.c
@@ -1,23 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "fm10k.h"
 
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c
index 28b6b4e56487..7657daa27298 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c
@@ -1,40 +1,32 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include <linux/vmalloc.h>
 
 #include "fm10k.h"
 
 struct fm10k_stats {
+	/* The stat_string is expected to be a format string formatted using
+	 * vsnprintf by fm10k_add_stat_strings. Every member of a stats array
+	 * should use the same format specifiers as they will be formatted
+	 * using the same variadic arguments.
+	 */
 	char stat_string[ETH_GSTRING_LEN];
 	int sizeof_stat;
 	int stat_offset;
 };
 
-#define FM10K_NETDEV_STAT(_net_stat) { \
-	.stat_string = #_net_stat, \
-	.sizeof_stat = FIELD_SIZEOF(struct net_device_stats, _net_stat), \
-	.stat_offset = offsetof(struct net_device_stats, _net_stat) \
+#define FM10K_STAT_FIELDS(_type, _name, _stat) { \
+	.stat_string = _name, \
+	.sizeof_stat = FIELD_SIZEOF(_type, _stat), \
+	.stat_offset = offsetof(_type, _stat) \
 }
 
+/* netdevice statistics */
+#define FM10K_NETDEV_STAT(_net_stat) \
+	FM10K_STAT_FIELDS(struct net_device_stats, __stringify(_net_stat), \
+			  _net_stat)
+
 static const struct fm10k_stats fm10k_gstrings_net_stats[] = {
 	FM10K_NETDEV_STAT(tx_packets),
 	FM10K_NETDEV_STAT(tx_bytes),
@@ -52,11 +44,9 @@ static const struct fm10k_stats fm10k_gstrings_net_stats[] = {
 
 #define FM10K_NETDEV_STATS_LEN	ARRAY_SIZE(fm10k_gstrings_net_stats)
 
-#define FM10K_STAT(_name, _stat) { \
-	.stat_string = _name, \
-	.sizeof_stat = FIELD_SIZEOF(struct fm10k_intfc, _stat), \
-	.stat_offset = offsetof(struct fm10k_intfc, _stat) \
-}
+/* General interface statistics */
+#define FM10K_STAT(_name, _stat) \
+	FM10K_STAT_FIELDS(struct fm10k_intfc, _name, _stat)
 
 static const struct fm10k_stats fm10k_gstrings_global_stats[] = {
 	FM10K_STAT("tx_restart_queue", restart_queue),
@@ -93,11 +83,9 @@ static const struct fm10k_stats fm10k_gstrings_pf_stats[] = {
 	FM10K_STAT("nodesc_drop", stats.nodesc_drop.count),
 };
 
-#define FM10K_MBX_STAT(_name, _stat) { \
-	.stat_string = _name, \
-	.sizeof_stat = FIELD_SIZEOF(struct fm10k_mbx_info, _stat), \
-	.stat_offset = offsetof(struct fm10k_mbx_info, _stat) \
-}
+/* mailbox statistics */
+#define FM10K_MBX_STAT(_name, _stat) \
+	FM10K_STAT_FIELDS(struct fm10k_mbx_info, _name, _stat)
 
 static const struct fm10k_stats fm10k_gstrings_mbx_stats[] = {
 	FM10K_MBX_STAT("mbx_tx_busy", tx_busy),
@@ -111,15 +99,13 @@ static const struct fm10k_stats fm10k_gstrings_mbx_stats[] = {
 	FM10K_MBX_STAT("mbx_rx_mbmem_pushed", rx_mbmem_pushed),
 };
 
-#define FM10K_QUEUE_STAT(_name, _stat) { \
-	.stat_string = _name, \
-	.sizeof_stat = FIELD_SIZEOF(struct fm10k_ring, _stat), \
-	.stat_offset = offsetof(struct fm10k_ring, _stat) \
-}
+/* per-queue ring statistics */
+#define FM10K_QUEUE_STAT(_name, _stat) \
+	FM10K_STAT_FIELDS(struct fm10k_ring, _name, _stat)
 
 static const struct fm10k_stats fm10k_gstrings_queue_stats[] = {
-	FM10K_QUEUE_STAT("packets", stats.packets),
-	FM10K_QUEUE_STAT("bytes", stats.bytes),
+	FM10K_QUEUE_STAT("%s_queue_%u_packets", stats.packets),
+	FM10K_QUEUE_STAT("%s_queue_%u_bytes", stats.bytes),
 };
 
 #define FM10K_GLOBAL_STATS_LEN ARRAY_SIZE(fm10k_gstrings_global_stats)
@@ -149,49 +135,44 @@ enum {
 static const char fm10k_prv_flags[FM10K_PRV_FLAG_LEN][ETH_GSTRING_LEN] = {
 };
 
-static void fm10k_add_stat_strings(u8 **p, const char *prefix,
-				   const struct fm10k_stats stats[],
-				   const unsigned int size)
+static void __fm10k_add_stat_strings(u8 **p, const struct fm10k_stats stats[],
+				     const unsigned int size, ...)
 {
 	unsigned int i;
 
 	for (i = 0; i < size; i++) {
-		snprintf(*p, ETH_GSTRING_LEN, "%s%s",
-			 prefix, stats[i].stat_string);
+		va_list args;
+
+		va_start(args, size);
+		vsnprintf(*p, ETH_GSTRING_LEN, stats[i].stat_string, args);
 		*p += ETH_GSTRING_LEN;
+		va_end(args);
 	}
 }
 
+#define fm10k_add_stat_strings(p, stats, ...) \
+	__fm10k_add_stat_strings(p, stats, ARRAY_SIZE(stats), ## __VA_ARGS__)
+
 static void fm10k_get_stat_strings(struct net_device *dev, u8 *data)
 {
 	struct fm10k_intfc *interface = netdev_priv(dev);
 	unsigned int i;
 
-	fm10k_add_stat_strings(&data, "", fm10k_gstrings_net_stats,
-			       FM10K_NETDEV_STATS_LEN);
+	fm10k_add_stat_strings(&data, fm10k_gstrings_net_stats);
 
-	fm10k_add_stat_strings(&data, "", fm10k_gstrings_global_stats,
-			       FM10K_GLOBAL_STATS_LEN);
+	fm10k_add_stat_strings(&data, fm10k_gstrings_global_stats);
 
-	fm10k_add_stat_strings(&data, "", fm10k_gstrings_mbx_stats,
-			       FM10K_MBX_STATS_LEN);
+	fm10k_add_stat_strings(&data, fm10k_gstrings_mbx_stats);
 
 	if (interface->hw.mac.type != fm10k_mac_vf)
-		fm10k_add_stat_strings(&data, "", fm10k_gstrings_pf_stats,
-				       FM10K_PF_STATS_LEN);
+		fm10k_add_stat_strings(&data, fm10k_gstrings_pf_stats);
 
 	for (i = 0; i < interface->hw.mac.max_queues; i++) {
-		char prefix[ETH_GSTRING_LEN];
-
-		snprintf(prefix, ETH_GSTRING_LEN, "tx_queue_%u_", i);
-		fm10k_add_stat_strings(&data, prefix,
-				       fm10k_gstrings_queue_stats,
-				       FM10K_QUEUE_STATS_LEN);
+		fm10k_add_stat_strings(&data, fm10k_gstrings_queue_stats,
+				       "tx", i);
 
-		snprintf(prefix, ETH_GSTRING_LEN, "rx_queue_%u_", i);
-		fm10k_add_stat_strings(&data, prefix,
-				       fm10k_gstrings_queue_stats,
-				       FM10K_QUEUE_STATS_LEN);
+		fm10k_add_stat_strings(&data, fm10k_gstrings_queue_stats,
+				       "rx", i);
 	}
 }
 
@@ -236,9 +217,9 @@ static int fm10k_get_sset_count(struct net_device *dev, int sset)
 	}
 }
 
-static void fm10k_add_ethtool_stats(u64 **data, void *pointer,
-				    const struct fm10k_stats stats[],
-				    const unsigned int size)
+static void __fm10k_add_ethtool_stats(u64 **data, void *pointer,
+				      const struct fm10k_stats stats[],
+				      const unsigned int size)
 {
 	unsigned int i;
 	char *p;
@@ -267,11 +248,16 @@ static void fm10k_add_ethtool_stats(u64 **data, void *pointer,
 			*((*data)++) = *(u8 *)p;
 			break;
 		default:
+			WARN_ONCE(1, "unexpected stat size for %s",
+				  stats[i].stat_string);
 			*((*data)++) = 0;
 		}
 	}
 }
 
+#define fm10k_add_ethtool_stats(data, pointer, stats) \
+	__fm10k_add_ethtool_stats(data, pointer, stats, ARRAY_SIZE(stats))
+
 static void fm10k_get_ethtool_stats(struct net_device *netdev,
 				    struct ethtool_stats __always_unused *stats,
 				    u64 *data)
@@ -282,20 +268,16 @@ static void fm10k_get_ethtool_stats(struct net_device *netdev,
 
 	fm10k_update_stats(interface);
 
-	fm10k_add_ethtool_stats(&data, net_stats, fm10k_gstrings_net_stats,
-				FM10K_NETDEV_STATS_LEN);
+	fm10k_add_ethtool_stats(&data, net_stats, fm10k_gstrings_net_stats);
 
-	fm10k_add_ethtool_stats(&data, interface, fm10k_gstrings_global_stats,
-				FM10K_GLOBAL_STATS_LEN);
+	fm10k_add_ethtool_stats(&data, interface, fm10k_gstrings_global_stats);
 
 	fm10k_add_ethtool_stats(&data, &interface->hw.mbx,
-				fm10k_gstrings_mbx_stats,
-				FM10K_MBX_STATS_LEN);
+				fm10k_gstrings_mbx_stats);
 
 	if (interface->hw.mac.type != fm10k_mac_vf) {
 		fm10k_add_ethtool_stats(&data, interface,
-					fm10k_gstrings_pf_stats,
-					FM10K_PF_STATS_LEN);
+					fm10k_gstrings_pf_stats);
 	}
 
 	for (i = 0; i < interface->hw.mac.max_queues; i++) {
@@ -303,13 +285,11 @@ static void fm10k_get_ethtool_stats(struct net_device *netdev,
 
 		ring = interface->tx_ring[i];
 		fm10k_add_ethtool_stats(&data, ring,
-					fm10k_gstrings_queue_stats,
-					FM10K_QUEUE_STATS_LEN);
+					fm10k_gstrings_queue_stats);
 
 		ring = interface->rx_ring[i];
 		fm10k_add_ethtool_stats(&data, ring,
-					fm10k_gstrings_queue_stats,
-					FM10K_QUEUE_STATS_LEN);
+					fm10k_gstrings_queue_stats);
 	}
 }
 
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_iov.c b/drivers/net/ethernet/intel/fm10k/fm10k_iov.c
index 30395f5e5e87..e707d717012f 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_iov.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_iov.c
@@ -1,23 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "fm10k.h"
 #include "fm10k_vf.h"
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_main.c b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
index df8607097e4a..3f536541f45f 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_main.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
@@ -1,23 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include <linux/types.h>
 #include <linux/module.h>
@@ -445,15 +427,14 @@ static void fm10k_type_trans(struct fm10k_ring *rx_ring,
 			l2_accel = NULL;
 	}
 
-	skb->protocol = eth_type_trans(skb, dev);
-
 	/* Record Rx queue, or update macvlan statistics */
 	if (!l2_accel)
 		skb_record_rx_queue(skb, rx_ring->queue_index);
 	else
 		macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true,
-				 (skb->pkt_type == PACKET_BROADCAST) ||
-				 (skb->pkt_type == PACKET_MULTICAST));
+				 false);
+
+	skb->protocol = eth_type_trans(skb, dev);
 }
 
 /**
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_mbx.c b/drivers/net/ethernet/intel/fm10k/fm10k_mbx.c
index c01bf30a0c9e..21021fe4f1c3 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_mbx.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_mbx.c
@@ -1,23 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "fm10k_common.h"
 
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_mbx.h b/drivers/net/ethernet/intel/fm10k/fm10k_mbx.h
index 007e1dfa9b7a..56d1abff04e2 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_mbx.h
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_mbx.h
@@ -1,23 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _FM10K_MBX_H_
 #define _FM10K_MBX_H_
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
index 45793491d4ba..929f538d28bc 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
@@ -1,27 +1,10 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2018 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "fm10k.h"
 #include <linux/vmalloc.h>
 #include <net/udp_tunnel.h>
+#include <linux/if_macvlan.h>
 
 /**
  * fm10k_setup_tx_resources - allocate Tx resources (Descriptors)
@@ -924,7 +907,9 @@ static int fm10k_mc_vlan_unsync(struct net_device *netdev,
 static int fm10k_update_vid(struct net_device *netdev, u16 vid, bool set)
 {
 	struct fm10k_intfc *interface = netdev_priv(netdev);
+	struct fm10k_l2_accel *l2_accel = interface->l2_accel;
 	struct fm10k_hw *hw = &interface->hw;
+	u16 glort;
 	s32 err;
 	int i;
 
@@ -992,6 +977,22 @@ static int fm10k_update_vid(struct net_device *netdev, u16 vid, bool set)
 	if (err)
 		goto err_out;
 
+	/* Update L2 accelerated macvlan addresses */
+	if (l2_accel) {
+		for (i = 0; i < l2_accel->size; i++) {
+			struct net_device *sdev = l2_accel->macvlan[i];
+
+			if (!sdev)
+				continue;
+
+			glort = l2_accel->dglort + 1 + i;
+
+			fm10k_queue_mac_request(interface, glort,
+						sdev->dev_addr,
+						vid, set);
+		}
+	}
+
 	/* set VLAN ID prior to syncing/unsyncing the VLAN */
 	interface->vid = vid + (set ? VLAN_N_VID : 0);
 
@@ -1231,6 +1232,22 @@ void fm10k_restore_rx_state(struct fm10k_intfc *interface)
 
 		fm10k_queue_mac_request(interface, glort,
 					hw->mac.addr, vid, true);
+
+		/* synchronize macvlan addresses */
+		if (l2_accel) {
+			for (i = 0; i < l2_accel->size; i++) {
+				struct net_device *sdev = l2_accel->macvlan[i];
+
+				if (!sdev)
+					continue;
+
+				glort = l2_accel->dglort + 1 + i;
+
+				fm10k_queue_mac_request(interface, glort,
+							sdev->dev_addr,
+							vid, true);
+			}
+		}
 	}
 
 	/* update xcast mode before synchronizing addresses if host's mailbox
@@ -1254,7 +1271,7 @@ void fm10k_restore_rx_state(struct fm10k_intfc *interface)
 			glort = l2_accel->dglort + 1 + i;
 
 			hw->mac.ops.update_xcast_mode(hw, glort,
-						      FM10K_XCAST_MODE_MULTI);
+						      FM10K_XCAST_MODE_NONE);
 			fm10k_queue_mac_request(interface, glort,
 						sdev->dev_addr,
 						hw->mac.default_vid, true);
@@ -1447,7 +1464,14 @@ static void *fm10k_dfwd_add_station(struct net_device *dev,
 	struct fm10k_dglort_cfg dglort = { 0 };
 	struct fm10k_hw *hw = &interface->hw;
 	int size = 0, i;
-	u16 glort;
+	u16 vid, glort;
+
+	/* The hardware supported by fm10k only filters on the destination MAC
+	 * address. In order to avoid issues we only support offloading modes
+	 * where the hardware can actually provide the functionality.
+	 */
+	if (!macvlan_supports_dest_filter(sdev))
+		return ERR_PTR(-EMEDIUMTYPE);
 
 	/* allocate l2 accel structure if it is not available */
 	if (!l2_accel) {
@@ -1513,12 +1537,18 @@ static void *fm10k_dfwd_add_station(struct net_device *dev,
 
 	glort = l2_accel->dglort + 1 + i;
 
-	if (fm10k_host_mbx_ready(interface)) {
+	if (fm10k_host_mbx_ready(interface))
 		hw->mac.ops.update_xcast_mode(hw, glort,
-					      FM10K_XCAST_MODE_MULTI);
+					      FM10K_XCAST_MODE_NONE);
+
+	fm10k_queue_mac_request(interface, glort, sdev->dev_addr,
+				hw->mac.default_vid, true);
+
+	for (vid = fm10k_find_next_vlan(interface, 0);
+	     vid < VLAN_N_VID;
+	     vid = fm10k_find_next_vlan(interface, vid))
 		fm10k_queue_mac_request(interface, glort, sdev->dev_addr,
-					hw->mac.default_vid, true);
-	}
+					vid, true);
 
 	fm10k_mbx_unlock(interface);
 
@@ -1532,8 +1562,8 @@ static void fm10k_dfwd_del_station(struct net_device *dev, void *priv)
 	struct fm10k_dglort_cfg dglort = { 0 };
 	struct fm10k_hw *hw = &interface->hw;
 	struct net_device *sdev = priv;
+	u16 vid, glort;
 	int i;
-	u16 glort;
 
 	if (!l2_accel)
 		return;
@@ -1553,12 +1583,18 @@ static void fm10k_dfwd_del_station(struct net_device *dev, void *priv)
 
 	glort = l2_accel->dglort + 1 + i;
 
-	if (fm10k_host_mbx_ready(interface)) {
+	if (fm10k_host_mbx_ready(interface))
 		hw->mac.ops.update_xcast_mode(hw, glort,
 					      FM10K_XCAST_MODE_NONE);
+
+	fm10k_queue_mac_request(interface, glort, sdev->dev_addr,
+				hw->mac.default_vid, false);
+
+	for (vid = fm10k_find_next_vlan(interface, 0);
+	     vid < VLAN_N_VID;
+	     vid = fm10k_find_next_vlan(interface, vid))
 		fm10k_queue_mac_request(interface, glort, sdev->dev_addr,
-					hw->mac.default_vid, false);
-	}
+					vid, false);
 
 	fm10k_mbx_unlock(interface);
 
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
index c4a2b688b38b..15071e4adb98 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
@@ -1,23 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2018 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include <linux/module.h>
 #include <linux/interrupt.h>
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c
index 7ba54c534f8c..8f0a99b6a537 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c
@@ -1,23 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2018 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "fm10k_pf.h"
 #include "fm10k_vf.h"
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pf.h b/drivers/net/ethernet/intel/fm10k/fm10k_pf.h
index ae81f9a16602..8e814df709d2 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_pf.h
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_pf.h
@@ -1,23 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _FM10K_PF_H_
 #define _FM10K_PF_H_
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_tlv.c b/drivers/net/ethernet/intel/fm10k/fm10k_tlv.c
index 725ecb7abccd..2a7a40bf2b1c 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_tlv.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_tlv.c
@@ -1,23 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2018 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "fm10k_tlv.h"
 
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_tlv.h b/drivers/net/ethernet/intel/fm10k/fm10k_tlv.h
index 5d2ee759507e..160bc5b78f99 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_tlv.h
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_tlv.h
@@ -1,23 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _FM10K_TLV_H_
 #define _FM10K_TLV_H_
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_type.h b/drivers/net/ethernet/intel/fm10k/fm10k_type.h
index dd23af11e2c1..3e608e493f9d 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_type.h
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_type.h
@@ -1,23 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _FM10K_TYPE_H_
 #define _FM10K_TYPE_H_
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_vf.c b/drivers/net/ethernet/intel/fm10k/fm10k_vf.c
index f06913630b39..a8519c1f0406 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_vf.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_vf.c
@@ -1,23 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "fm10k_vf.h"
 
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_vf.h b/drivers/net/ethernet/intel/fm10k/fm10k_vf.h
index 66a66b73a2f1..787d0d570a28 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_vf.h
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_vf.h
@@ -1,23 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Ethernet Switch Host Interface Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _FM10K_VF_H_
 #define _FM10K_VF_H_
diff --git a/drivers/net/ethernet/intel/i40e/Makefile b/drivers/net/ethernet/intel/i40e/Makefile
index 75437768a07c..14397e7e9925 100644
--- a/drivers/net/ethernet/intel/i40e/Makefile
+++ b/drivers/net/ethernet/intel/i40e/Makefile
@@ -1,29 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-################################################################################
-#
-# Intel Ethernet Controller XL710 Family Linux Driver
-# Copyright(c) 2013 - 2015 Intel Corporation.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms and conditions of the GNU General Public License,
-# version 2, as published by the Free Software Foundation.
-#
-# This program is distributed in the hope it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# The full GNU General Public License is included in this distribution in
-# the file called "COPYING".
-#
-# Contact Information:
-# e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-# Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-#
-################################################################################
+# Copyright(c) 2013 - 2018 Intel Corporation.
 
 #
 # Makefile for the Intel(R) Ethernet Connection XL710 (i40e.ko) driver
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
index a44139c1de80..7a80652e2500 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_H_
 #define _I40E_H_
@@ -334,10 +310,12 @@ struct i40e_tc_configuration {
 	struct i40e_tc_info tc_info[I40E_MAX_TRAFFIC_CLASS];
 };
 
+#define I40E_UDP_PORT_INDEX_UNUSED	255
 struct i40e_udp_port_config {
 	/* AdminQ command interface expects port number in Host byte order */
 	u16 port;
 	u8 type;
+	u8 filter_index;
 };
 
 /* macros related to FLX_PIT */
@@ -608,7 +586,7 @@ struct i40e_pf {
 	unsigned long ptp_tx_start;
 	struct hwtstamp_config tstamp_config;
 	struct mutex tmreg_lock; /* Used to protect the SYSTIME registers. */
-	u64 ptp_base_adj;
+	u32 ptp_adj_mult;
 	u32 tx_hwtstamp_timeouts;
 	u32 tx_hwtstamp_skipped;
 	u32 rx_hwtstamp_cleared;
@@ -1009,6 +987,9 @@ void i40e_service_event_schedule(struct i40e_pf *pf);
 void i40e_notify_client_of_vf_msg(struct i40e_vsi *vsi, u32 vf_id,
 				  u8 *msg, u16 len);
 
+int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, bool is_xdp,
+			   bool enable);
+int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable);
 int i40e_vsi_start_rings(struct i40e_vsi *vsi);
 void i40e_vsi_stop_rings(struct i40e_vsi *vsi);
 void i40e_vsi_stop_rings_no_wait(struct  i40e_vsi *vsi);
diff --git a/drivers/net/ethernet/intel/i40e/i40e_adminq.c b/drivers/net/ethernet/intel/i40e/i40e_adminq.c
index 843fc7781ef8..ddbea79d18e5 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_adminq.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_adminq.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40e_status.h"
 #include "i40e_type.h"
diff --git a/drivers/net/ethernet/intel/i40e/i40e_adminq.h b/drivers/net/ethernet/intel/i40e/i40e_adminq.h
index 0a8749ee9fd3..edec3df78971 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_adminq.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_adminq.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_ADMINQ_H_
 #define _I40E_ADMINQ_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h b/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h
index 0244923edeb8..7d888e05f96f 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_ADMINQ_CMD_H_
 #define _I40E_ADMINQ_CMD_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_alloc.h b/drivers/net/ethernet/intel/i40e/i40e_alloc.h
index abed0c52e782..cb8689222c8b 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_alloc.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_alloc.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_ALLOC_H_
 #define _I40E_ALLOC_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_client.c b/drivers/net/ethernet/intel/i40e/i40e_client.c
index d8ce4999864f..5f3b8b9ff511 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_client.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_client.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include <linux/list.h>
 #include <linux/errno.h>
@@ -64,7 +40,7 @@ static struct i40e_ops i40e_lan_ops = {
 /**
  * i40e_client_get_params - Get the params that can change at runtime
  * @vsi: the VSI with the message
- * @param: clinet param struct
+ * @params: client param struct
  *
  **/
 static
@@ -590,7 +566,7 @@ static int i40e_client_virtchnl_send(struct i40e_info *ldev,
  * i40e_client_setup_qvlist
  * @ldev: pointer to L2 context.
  * @client: Client pointer.
- * @qv_info: queue and vector list
+ * @qvlist_info: queue and vector list
  *
  * Return 0 on success or < 0 on error
  **/
@@ -665,7 +641,7 @@ static int i40e_client_setup_qvlist(struct i40e_info *ldev,
  * i40e_client_request_reset
  * @ldev: pointer to L2 context.
  * @client: Client pointer.
- * @level: reset level
+ * @reset_level: reset level
  **/
 static void i40e_client_request_reset(struct i40e_info *ldev,
 				      struct i40e_client *client,
diff --git a/drivers/net/ethernet/intel/i40e/i40e_client.h b/drivers/net/ethernet/intel/i40e/i40e_client.h
index 9d464d40bc17..72994baf4941 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_client.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_client.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_CLIENT_H_
 #define _I40E_CLIENT_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c
index c0a3dae8a2db..eb2d1530d331 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_common.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_common.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40e_type.h"
 #include "i40e_adminq.h"
@@ -1695,6 +1671,8 @@ enum i40e_status_code i40e_aq_set_phy_config(struct i40e_hw *hw,
 /**
  * i40e_set_fc
  * @hw: pointer to the hw struct
+ * @aq_failures: buffer to return AdminQ failure information
+ * @atomic_restart: whether to enable atomic link restart
  *
  * Set the requested flow control mode using set_phy_config.
  **/
@@ -2831,8 +2809,8 @@ i40e_status i40e_aq_remove_macvlan(struct i40e_hw *hw, u16 seid,
  * @mr_list: list of mirrored VSI SEIDs or VLAN IDs
  * @cmd_details: pointer to command details structure or NULL
  * @rule_id: Rule ID returned from FW
- * @rule_used: Number of rules used in internal switch
- * @rule_free: Number of rules free in internal switch
+ * @rules_used: Number of rules used in internal switch
+ * @rules_free: Number of rules free in internal switch
  *
  * Add/Delete a mirror rule to a specific switch. Mirror rules are supported for
  * VEBs/VEPA elements only
@@ -2892,8 +2870,8 @@ static i40e_status i40e_mirrorrule_op(struct i40e_hw *hw,
  * @mr_list: list of mirrored VSI SEIDs or VLAN IDs
  * @cmd_details: pointer to command details structure or NULL
  * @rule_id: Rule ID returned from FW
- * @rule_used: Number of rules used in internal switch
- * @rule_free: Number of rules free in internal switch
+ * @rules_used: Number of rules used in internal switch
+ * @rules_free: Number of rules free in internal switch
  *
  * Add mirror rule. Mirror rules are supported for VEBs or VEPA elements only
  **/
@@ -2923,8 +2901,8 @@ i40e_status i40e_aq_add_mirrorrule(struct i40e_hw *hw, u16 sw_seid,
  *		add_mirrorrule.
  * @mr_list: list of mirrored VLAN IDs to be removed
  * @cmd_details: pointer to command details structure or NULL
- * @rule_used: Number of rules used in internal switch
- * @rule_free: Number of rules free in internal switch
+ * @rules_used: Number of rules used in internal switch
+ * @rules_free: Number of rules free in internal switch
  *
  * Delete a mirror rule. Mirror rules are supported for VEBs/VEPA elements only
  **/
@@ -3672,6 +3650,8 @@ i40e_status i40e_aq_stop_lldp(struct i40e_hw *hw, bool shutdown_agent,
 /**
  * i40e_aq_start_lldp
  * @hw: pointer to the hw struct
+ * @buff: buffer for result
+ * @buff_size: buffer size
  * @cmd_details: pointer to command details structure or NULL
  *
  * Start the embedded LLDP Agent on all ports.
@@ -3752,7 +3732,6 @@ i40e_status i40e_aq_get_cee_dcb_config(struct i40e_hw *hw,
  * i40e_aq_add_udp_tunnel
  * @hw: pointer to the hw struct
  * @udp_port: the UDP port to add in Host byte order
- * @header_len: length of the tunneling header length in DWords
  * @protocol_index: protocol index type
  * @filter_index: pointer to filter index
  * @cmd_details: pointer to command details structure or NULL
@@ -3971,6 +3950,7 @@ i40e_status i40e_aq_config_vsi_tc_bw(struct i40e_hw *hw,
  * @hw: pointer to the hw struct
  * @seid: seid of the switching component connected to Physical Port
  * @ets_data: Buffer holding ETS parameters
+ * @opcode: Tx scheduler AQ command opcode
  * @cmd_details: pointer to command details structure or NULL
  **/
 i40e_status i40e_aq_config_switch_comp_ets(struct i40e_hw *hw,
@@ -4314,10 +4294,10 @@ i40e_status i40e_aq_add_rem_control_packet_filter(struct i40e_hw *hw,
  * @hw: pointer to the hw struct
  * @seid: VSI seid to add ethertype filter from
  **/
-#define I40E_FLOW_CONTROL_ETHTYPE 0x8808
 void i40e_add_filter_to_drop_tx_flow_control_frames(struct i40e_hw *hw,
 						    u16 seid)
 {
+#define I40E_FLOW_CONTROL_ETHTYPE 0x8808
 	u16 flag = I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC |
 		   I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP |
 		   I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TX;
@@ -4448,6 +4428,7 @@ void i40e_set_pci_config_data(struct i40e_hw *hw, u16 link_status)
  * @ret_buff_size: actual buffer size returned
  * @ret_next_table: next block to read
  * @ret_next_index: next index to read
+ * @cmd_details: pointer to command details structure or NULL
  *
  * Dump internal FW/HW data for debug purposes.
  *
@@ -4574,7 +4555,7 @@ i40e_status i40e_aq_configure_partition_bw(struct i40e_hw *hw,
  * i40e_read_phy_register_clause22
  * @hw: pointer to the HW structure
  * @reg: register address in the page
- * @phy_adr: PHY address on MDIO interface
+ * @phy_addr: PHY address on MDIO interface
  * @value: PHY register value
  *
  * Reads specified PHY register value
@@ -4619,7 +4600,7 @@ i40e_status i40e_read_phy_register_clause22(struct i40e_hw *hw,
  * i40e_write_phy_register_clause22
  * @hw: pointer to the HW structure
  * @reg: register address in the page
- * @phy_adr: PHY address on MDIO interface
+ * @phy_addr: PHY address on MDIO interface
  * @value: PHY register value
  *
  * Writes specified PHY register value
@@ -4660,7 +4641,7 @@ i40e_status i40e_write_phy_register_clause22(struct i40e_hw *hw,
  * @hw: pointer to the HW structure
  * @page: registers page number
  * @reg: register address in the page
- * @phy_adr: PHY address on MDIO interface
+ * @phy_addr: PHY address on MDIO interface
  * @value: PHY register value
  *
  * Reads specified PHY register value
@@ -4734,7 +4715,7 @@ i40e_status i40e_read_phy_register_clause45(struct i40e_hw *hw,
  * @hw: pointer to the HW structure
  * @page: registers page number
  * @reg: register address in the page
- * @phy_adr: PHY address on MDIO interface
+ * @phy_addr: PHY address on MDIO interface
  * @value: PHY register value
  *
  * Writes value to specified PHY register
@@ -4801,7 +4782,7 @@ i40e_status i40e_write_phy_register_clause45(struct i40e_hw *hw,
  * @hw: pointer to the HW structure
  * @page: registers page number
  * @reg: register address in the page
- * @phy_adr: PHY address on MDIO interface
+ * @phy_addr: PHY address on MDIO interface
  * @value: PHY register value
  *
  * Writes value to specified PHY register
@@ -4837,7 +4818,7 @@ i40e_status i40e_write_phy_register(struct i40e_hw *hw,
  * @hw: pointer to the HW structure
  * @page: registers page number
  * @reg: register address in the page
- * @phy_adr: PHY address on MDIO interface
+ * @phy_addr: PHY address on MDIO interface
  * @value: PHY register value
  *
  * Reads specified PHY register value
@@ -4872,7 +4853,6 @@ i40e_status i40e_read_phy_register(struct i40e_hw *hw,
  * i40e_get_phy_address
  * @hw: pointer to the HW structure
  * @dev_num: PHY port num that address we want
- * @phy_addr: Returned PHY address
  *
  * Gets PHY address for current port
  **/
@@ -5082,7 +5062,9 @@ i40e_status i40e_led_get_phy(struct i40e_hw *hw, u16 *led_addr,
  * i40e_led_set_phy
  * @hw: pointer to the HW structure
  * @on: true or false
+ * @led_addr: address of led register to use
  * @mode: original val plus bit for set or ignore
+ *
  * Set led's on or off when controlled by the PHY
  *
  **/
@@ -5371,6 +5353,7 @@ i40e_status_code i40e_aq_write_ddp(struct i40e_hw *hw, void *buff,
  * @hw: pointer to the hw struct
  * @buff: command buffer (size in bytes = buff_size)
  * @buff_size: buffer size in bytes
+ * @flags: AdminQ command flags
  * @cmd_details: pointer to command details structure or NULL
  **/
 enum
diff --git a/drivers/net/ethernet/intel/i40e/i40e_dcb.c b/drivers/net/ethernet/intel/i40e/i40e_dcb.c
index 9fec728dc4b9..56bff8faf371 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_dcb.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_dcb.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40e_adminq.h"
 #include "i40e_prototype.h"
@@ -944,6 +920,70 @@ i40e_status i40e_init_dcb(struct i40e_hw *hw)
 	return ret;
 }
 
+/**
+ * _i40e_read_lldp_cfg - generic read of LLDP Configuration data from NVM
+ * @hw: pointer to the HW structure
+ * @lldp_cfg: pointer to hold lldp configuration variables
+ * @module: address of the module pointer
+ * @word_offset: offset of LLDP configuration
+ *
+ * Reads the LLDP configuration data from NVM using passed addresses
+ **/
+static i40e_status _i40e_read_lldp_cfg(struct i40e_hw *hw,
+				       struct i40e_lldp_variables *lldp_cfg,
+				       u8 module, u32 word_offset)
+{
+	u32 address, offset = (2 * word_offset);
+	i40e_status ret;
+	__le16 raw_mem;
+	u16 mem;
+
+	ret = i40e_acquire_nvm(hw, I40E_RESOURCE_READ);
+	if (ret)
+		return ret;
+
+	ret = i40e_aq_read_nvm(hw, 0x0, module * 2, sizeof(raw_mem), &raw_mem,
+			       true, NULL);
+	i40e_release_nvm(hw);
+	if (ret)
+		return ret;
+
+	mem = le16_to_cpu(raw_mem);
+	/* Check if this pointer needs to be read in word size or 4K sector
+	 * units.
+	 */
+	if (mem & I40E_PTR_TYPE)
+		address = (0x7FFF & mem) * 4096;
+	else
+		address = (0x7FFF & mem) * 2;
+
+	ret = i40e_acquire_nvm(hw, I40E_RESOURCE_READ);
+	if (ret)
+		goto err_lldp_cfg;
+
+	ret = i40e_aq_read_nvm(hw, module, offset, sizeof(raw_mem), &raw_mem,
+			       true, NULL);
+	i40e_release_nvm(hw);
+	if (ret)
+		return ret;
+
+	mem = le16_to_cpu(raw_mem);
+	offset = mem + word_offset;
+	offset *= 2;
+
+	ret = i40e_acquire_nvm(hw, I40E_RESOURCE_READ);
+	if (ret)
+		goto err_lldp_cfg;
+
+	ret = i40e_aq_read_nvm(hw, 0, address + offset,
+			       sizeof(struct i40e_lldp_variables), lldp_cfg,
+			       true, NULL);
+	i40e_release_nvm(hw);
+
+err_lldp_cfg:
+	return ret;
+}
+
 /**
  * i40e_read_lldp_cfg - read LLDP Configuration data from NVM
  * @hw: pointer to the HW structure
@@ -955,21 +995,34 @@ i40e_status i40e_read_lldp_cfg(struct i40e_hw *hw,
 			       struct i40e_lldp_variables *lldp_cfg)
 {
 	i40e_status ret = 0;
-	u32 offset = (2 * I40E_NVM_LLDP_CFG_PTR);
+	u32 mem;
 
 	if (!lldp_cfg)
 		return I40E_ERR_PARAM;
 
 	ret = i40e_acquire_nvm(hw, I40E_RESOURCE_READ);
 	if (ret)
-		goto err_lldp_cfg;
+		return ret;
 
-	ret = i40e_aq_read_nvm(hw, I40E_SR_EMP_MODULE_PTR, offset,
-			       sizeof(struct i40e_lldp_variables),
-			       (u8 *)lldp_cfg,
-			       true, NULL);
+	ret = i40e_aq_read_nvm(hw, I40E_SR_NVM_CONTROL_WORD, 0, sizeof(mem),
+			       &mem, true, NULL);
 	i40e_release_nvm(hw);
+	if (ret)
+		return ret;
+
+	/* Read a bit that holds information whether we are running flat or
+	 * structured NVM image. Flat image has LLDP configuration in shadow
+	 * ram, so there is a need to pass different addresses for both cases.
+	 */
+	if (mem & I40E_SR_NVM_MAP_STRUCTURE_TYPE) {
+		/* Flat NVM case */
+		ret = _i40e_read_lldp_cfg(hw, lldp_cfg, I40E_SR_EMP_MODULE_PTR,
+					  I40E_SR_LLDP_CFG_PTR);
+	} else {
+		/* Good old structured NVM image */
+		ret = _i40e_read_lldp_cfg(hw, lldp_cfg, I40E_EMP_MODULE_PTR,
+					  I40E_NVM_LLDP_CFG_PTR);
+	}
 
-err_lldp_cfg:
 	return ret;
 }
diff --git a/drivers/net/ethernet/intel/i40e/i40e_dcb.h b/drivers/net/ethernet/intel/i40e/i40e_dcb.h
index 4f806386cb22..2b748a60a843 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_dcb.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_dcb.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_DCB_H_
 #define _I40E_DCB_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_dcb_nl.c b/drivers/net/ethernet/intel/i40e/i40e_dcb_nl.c
index 502818e3da78..9deae9a35423 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_dcb_nl.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_dcb_nl.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifdef CONFIG_I40E_DCB
 #include "i40e.h"
@@ -47,7 +23,7 @@ static void i40e_get_pfc_delay(struct i40e_hw *hw, u16 *delay)
 
 /**
  * i40e_dcbnl_ieee_getets - retrieve local IEEE ETS configuration
- * @netdev: the corresponding netdev
+ * @dev: the corresponding netdev
  * @ets: structure to hold the ETS information
  *
  * Returns local IEEE ETS configuration
@@ -86,8 +62,8 @@ static int i40e_dcbnl_ieee_getets(struct net_device *dev,
 
 /**
  * i40e_dcbnl_ieee_getpfc - retrieve local IEEE PFC configuration
- * @netdev: the corresponding netdev
- * @ets: structure to hold the PFC information
+ * @dev: the corresponding netdev
+ * @pfc: structure to hold the PFC information
  *
  * Returns local IEEE PFC configuration
  **/
@@ -119,7 +95,7 @@ static int i40e_dcbnl_ieee_getpfc(struct net_device *dev,
 
 /**
  * i40e_dcbnl_getdcbx - retrieve current DCBx capability
- * @netdev: the corresponding netdev
+ * @dev: the corresponding netdev
  *
  * Returns DCBx capability features
  **/
@@ -132,7 +108,8 @@ static u8 i40e_dcbnl_getdcbx(struct net_device *dev)
 
 /**
  * i40e_dcbnl_get_perm_hw_addr - MAC address used by DCBx
- * @netdev: the corresponding netdev
+ * @dev: the corresponding netdev
+ * @perm_addr: buffer to store the MAC address
  *
  * Returns the SAN MAC address used for LLDP exchange
  **/
diff --git a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
index d494dcaf18d0..56b911a5dd8b 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifdef CONFIG_DEBUG_FS
 
@@ -36,8 +12,8 @@ static struct dentry *i40e_dbg_root;
 
 /**
  * i40e_dbg_find_vsi - searches for the vsi with the given seid
- * @pf - the PF structure to search for the vsi
- * @seid - seid of the vsi it is searching for
+ * @pf: the PF structure to search for the vsi
+ * @seid: seid of the vsi it is searching for
  **/
 static struct i40e_vsi *i40e_dbg_find_vsi(struct i40e_pf *pf, int seid)
 {
@@ -55,8 +31,8 @@ static struct i40e_vsi *i40e_dbg_find_vsi(struct i40e_pf *pf, int seid)
 
 /**
  * i40e_dbg_find_veb - searches for the veb with the given seid
- * @pf - the PF structure to search for the veb
- * @seid - seid of the veb it is searching for
+ * @pf: the PF structure to search for the veb
+ * @seid: seid of the veb it is searching for
  **/
 static struct i40e_veb *i40e_dbg_find_veb(struct i40e_pf *pf, int seid)
 {
diff --git a/drivers/net/ethernet/intel/i40e/i40e_devids.h b/drivers/net/ethernet/intel/i40e/i40e_devids.h
index ad6a66ccb576..334b05ff685a 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_devids.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_devids.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_DEVIDS_H_
 #define _I40E_DEVIDS_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_diag.c b/drivers/net/ethernet/intel/i40e/i40e_diag.c
index df3e60470f8b..ef4d3762bf37 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_diag.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_diag.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40e_diag.h"
 #include "i40e_prototype.h"
diff --git a/drivers/net/ethernet/intel/i40e/i40e_diag.h b/drivers/net/ethernet/intel/i40e/i40e_diag.h
index be8341763475..c3340f320a18 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_diag.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_diag.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_DIAG_H_
 #define _I40E_DIAG_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
index b974482ff630..6947a2a571cb 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 /* ethtool support for i40e */
 
@@ -43,13 +19,13 @@ struct i40e_stats {
 }
 
 #define I40E_NETDEV_STAT(_net_stat) \
-		I40E_STAT(struct rtnl_link_stats64, #_net_stat, _net_stat)
+	I40E_STAT(struct rtnl_link_stats64, #_net_stat, _net_stat)
 #define I40E_PF_STAT(_name, _stat) \
-		I40E_STAT(struct i40e_pf, _name, _stat)
+	I40E_STAT(struct i40e_pf, _name, _stat)
 #define I40E_VSI_STAT(_name, _stat) \
-		I40E_STAT(struct i40e_vsi, _name, _stat)
+	I40E_STAT(struct i40e_vsi, _name, _stat)
 #define I40E_VEB_STAT(_name, _stat) \
-		I40E_STAT(struct i40e_veb, _name, _stat)
+	I40E_STAT(struct i40e_veb, _name, _stat)
 
 static const struct i40e_stats i40e_gstrings_net_stats[] = {
 	I40E_NETDEV_STAT(rx_packets),
@@ -66,18 +42,18 @@ static const struct i40e_stats i40e_gstrings_net_stats[] = {
 };
 
 static const struct i40e_stats i40e_gstrings_veb_stats[] = {
-	I40E_VEB_STAT("rx_bytes", stats.rx_bytes),
-	I40E_VEB_STAT("tx_bytes", stats.tx_bytes),
-	I40E_VEB_STAT("rx_unicast", stats.rx_unicast),
-	I40E_VEB_STAT("tx_unicast", stats.tx_unicast),
-	I40E_VEB_STAT("rx_multicast", stats.rx_multicast),
-	I40E_VEB_STAT("tx_multicast", stats.tx_multicast),
-	I40E_VEB_STAT("rx_broadcast", stats.rx_broadcast),
-	I40E_VEB_STAT("tx_broadcast", stats.tx_broadcast),
-	I40E_VEB_STAT("rx_discards", stats.rx_discards),
-	I40E_VEB_STAT("tx_discards", stats.tx_discards),
-	I40E_VEB_STAT("tx_errors", stats.tx_errors),
-	I40E_VEB_STAT("rx_unknown_protocol", stats.rx_unknown_protocol),
+	I40E_VEB_STAT("veb.rx_bytes", stats.rx_bytes),
+	I40E_VEB_STAT("veb.tx_bytes", stats.tx_bytes),
+	I40E_VEB_STAT("veb.rx_unicast", stats.rx_unicast),
+	I40E_VEB_STAT("veb.tx_unicast", stats.tx_unicast),
+	I40E_VEB_STAT("veb.rx_multicast", stats.rx_multicast),
+	I40E_VEB_STAT("veb.tx_multicast", stats.tx_multicast),
+	I40E_VEB_STAT("veb.rx_broadcast", stats.rx_broadcast),
+	I40E_VEB_STAT("veb.tx_broadcast", stats.tx_broadcast),
+	I40E_VEB_STAT("veb.rx_discards", stats.rx_discards),
+	I40E_VEB_STAT("veb.tx_discards", stats.tx_discards),
+	I40E_VEB_STAT("veb.tx_errors", stats.tx_errors),
+	I40E_VEB_STAT("veb.rx_unknown_protocol", stats.rx_unknown_protocol),
 };
 
 static const struct i40e_stats i40e_gstrings_misc_stats[] = {
@@ -90,6 +66,7 @@ static const struct i40e_stats i40e_gstrings_misc_stats[] = {
 	I40E_VSI_STAT("rx_unknown_protocol", eth_stats.rx_unknown_protocol),
 	I40E_VSI_STAT("tx_linearize", tx_linearize),
 	I40E_VSI_STAT("tx_force_wb", tx_force_wb),
+	I40E_VSI_STAT("tx_busy", tx_busy),
 	I40E_VSI_STAT("rx_alloc_fail", rx_buf_failed),
 	I40E_VSI_STAT("rx_pg_alloc_fail", rx_page_failed),
 };
@@ -105,76 +82,77 @@ static const struct i40e_stats i40e_gstrings_misc_stats[] = {
  * is queried on the base PF netdev, not on the VMDq or FCoE netdev.
  */
 static const struct i40e_stats i40e_gstrings_stats[] = {
-	I40E_PF_STAT("rx_bytes", stats.eth.rx_bytes),
-	I40E_PF_STAT("tx_bytes", stats.eth.tx_bytes),
-	I40E_PF_STAT("rx_unicast", stats.eth.rx_unicast),
-	I40E_PF_STAT("tx_unicast", stats.eth.tx_unicast),
-	I40E_PF_STAT("rx_multicast", stats.eth.rx_multicast),
-	I40E_PF_STAT("tx_multicast", stats.eth.tx_multicast),
-	I40E_PF_STAT("rx_broadcast", stats.eth.rx_broadcast),
-	I40E_PF_STAT("tx_broadcast", stats.eth.tx_broadcast),
-	I40E_PF_STAT("tx_errors", stats.eth.tx_errors),
-	I40E_PF_STAT("rx_dropped", stats.eth.rx_discards),
-	I40E_PF_STAT("tx_dropped_link_down", stats.tx_dropped_link_down),
-	I40E_PF_STAT("rx_crc_errors", stats.crc_errors),
-	I40E_PF_STAT("illegal_bytes", stats.illegal_bytes),
-	I40E_PF_STAT("mac_local_faults", stats.mac_local_faults),
-	I40E_PF_STAT("mac_remote_faults", stats.mac_remote_faults),
-	I40E_PF_STAT("tx_timeout", tx_timeout_count),
-	I40E_PF_STAT("rx_csum_bad", hw_csum_rx_error),
-	I40E_PF_STAT("rx_length_errors", stats.rx_length_errors),
-	I40E_PF_STAT("link_xon_rx", stats.link_xon_rx),
-	I40E_PF_STAT("link_xoff_rx", stats.link_xoff_rx),
-	I40E_PF_STAT("link_xon_tx", stats.link_xon_tx),
-	I40E_PF_STAT("link_xoff_tx", stats.link_xoff_tx),
-	I40E_PF_STAT("priority_xon_rx", stats.priority_xon_rx),
-	I40E_PF_STAT("priority_xoff_rx", stats.priority_xoff_rx),
-	I40E_PF_STAT("priority_xon_tx", stats.priority_xon_tx),
-	I40E_PF_STAT("priority_xoff_tx", stats.priority_xoff_tx),
-	I40E_PF_STAT("rx_size_64", stats.rx_size_64),
-	I40E_PF_STAT("rx_size_127", stats.rx_size_127),
-	I40E_PF_STAT("rx_size_255", stats.rx_size_255),
-	I40E_PF_STAT("rx_size_511", stats.rx_size_511),
-	I40E_PF_STAT("rx_size_1023", stats.rx_size_1023),
-	I40E_PF_STAT("rx_size_1522", stats.rx_size_1522),
-	I40E_PF_STAT("rx_size_big", stats.rx_size_big),
-	I40E_PF_STAT("tx_size_64", stats.tx_size_64),
-	I40E_PF_STAT("tx_size_127", stats.tx_size_127),
-	I40E_PF_STAT("tx_size_255", stats.tx_size_255),
-	I40E_PF_STAT("tx_size_511", stats.tx_size_511),
-	I40E_PF_STAT("tx_size_1023", stats.tx_size_1023),
-	I40E_PF_STAT("tx_size_1522", stats.tx_size_1522),
-	I40E_PF_STAT("tx_size_big", stats.tx_size_big),
-	I40E_PF_STAT("rx_undersize", stats.rx_undersize),
-	I40E_PF_STAT("rx_fragments", stats.rx_fragments),
-	I40E_PF_STAT("rx_oversize", stats.rx_oversize),
-	I40E_PF_STAT("rx_jabber", stats.rx_jabber),
-	I40E_PF_STAT("VF_admin_queue_requests", vf_aq_requests),
-	I40E_PF_STAT("arq_overflows", arq_overflows),
-	I40E_PF_STAT("rx_hwtstamp_cleared", rx_hwtstamp_cleared),
-	I40E_PF_STAT("tx_hwtstamp_skipped", tx_hwtstamp_skipped),
-	I40E_PF_STAT("fdir_flush_cnt", fd_flush_cnt),
-	I40E_PF_STAT("fdir_atr_match", stats.fd_atr_match),
-	I40E_PF_STAT("fdir_atr_tunnel_match", stats.fd_atr_tunnel_match),
-	I40E_PF_STAT("fdir_atr_status", stats.fd_atr_status),
-	I40E_PF_STAT("fdir_sb_match", stats.fd_sb_match),
-	I40E_PF_STAT("fdir_sb_status", stats.fd_sb_status),
+	I40E_PF_STAT("port.rx_bytes", stats.eth.rx_bytes),
+	I40E_PF_STAT("port.tx_bytes", stats.eth.tx_bytes),
+	I40E_PF_STAT("port.rx_unicast", stats.eth.rx_unicast),
+	I40E_PF_STAT("port.tx_unicast", stats.eth.tx_unicast),
+	I40E_PF_STAT("port.rx_multicast", stats.eth.rx_multicast),
+	I40E_PF_STAT("port.tx_multicast", stats.eth.tx_multicast),
+	I40E_PF_STAT("port.rx_broadcast", stats.eth.rx_broadcast),
+	I40E_PF_STAT("port.tx_broadcast", stats.eth.tx_broadcast),
+	I40E_PF_STAT("port.tx_errors", stats.eth.tx_errors),
+	I40E_PF_STAT("port.rx_dropped", stats.eth.rx_discards),
+	I40E_PF_STAT("port.tx_dropped_link_down", stats.tx_dropped_link_down),
+	I40E_PF_STAT("port.rx_crc_errors", stats.crc_errors),
+	I40E_PF_STAT("port.illegal_bytes", stats.illegal_bytes),
+	I40E_PF_STAT("port.mac_local_faults", stats.mac_local_faults),
+	I40E_PF_STAT("port.mac_remote_faults", stats.mac_remote_faults),
+	I40E_PF_STAT("port.tx_timeout", tx_timeout_count),
+	I40E_PF_STAT("port.rx_csum_bad", hw_csum_rx_error),
+	I40E_PF_STAT("port.rx_length_errors", stats.rx_length_errors),
+	I40E_PF_STAT("port.link_xon_rx", stats.link_xon_rx),
+	I40E_PF_STAT("port.link_xoff_rx", stats.link_xoff_rx),
+	I40E_PF_STAT("port.link_xon_tx", stats.link_xon_tx),
+	I40E_PF_STAT("port.link_xoff_tx", stats.link_xoff_tx),
+	I40E_PF_STAT("port.rx_size_64", stats.rx_size_64),
+	I40E_PF_STAT("port.rx_size_127", stats.rx_size_127),
+	I40E_PF_STAT("port.rx_size_255", stats.rx_size_255),
+	I40E_PF_STAT("port.rx_size_511", stats.rx_size_511),
+	I40E_PF_STAT("port.rx_size_1023", stats.rx_size_1023),
+	I40E_PF_STAT("port.rx_size_1522", stats.rx_size_1522),
+	I40E_PF_STAT("port.rx_size_big", stats.rx_size_big),
+	I40E_PF_STAT("port.tx_size_64", stats.tx_size_64),
+	I40E_PF_STAT("port.tx_size_127", stats.tx_size_127),
+	I40E_PF_STAT("port.tx_size_255", stats.tx_size_255),
+	I40E_PF_STAT("port.tx_size_511", stats.tx_size_511),
+	I40E_PF_STAT("port.tx_size_1023", stats.tx_size_1023),
+	I40E_PF_STAT("port.tx_size_1522", stats.tx_size_1522),
+	I40E_PF_STAT("port.tx_size_big", stats.tx_size_big),
+	I40E_PF_STAT("port.rx_undersize", stats.rx_undersize),
+	I40E_PF_STAT("port.rx_fragments", stats.rx_fragments),
+	I40E_PF_STAT("port.rx_oversize", stats.rx_oversize),
+	I40E_PF_STAT("port.rx_jabber", stats.rx_jabber),
+	I40E_PF_STAT("port.VF_admin_queue_requests", vf_aq_requests),
+	I40E_PF_STAT("port.arq_overflows", arq_overflows),
+	I40E_PF_STAT("port.tx_hwtstamp_timeouts", tx_hwtstamp_timeouts),
+	I40E_PF_STAT("port.rx_hwtstamp_cleared", rx_hwtstamp_cleared),
+	I40E_PF_STAT("port.tx_hwtstamp_skipped", tx_hwtstamp_skipped),
+	I40E_PF_STAT("port.fdir_flush_cnt", fd_flush_cnt),
+	I40E_PF_STAT("port.fdir_atr_match", stats.fd_atr_match),
+	I40E_PF_STAT("port.fdir_atr_tunnel_match", stats.fd_atr_tunnel_match),
+	I40E_PF_STAT("port.fdir_atr_status", stats.fd_atr_status),
+	I40E_PF_STAT("port.fdir_sb_match", stats.fd_sb_match),
+	I40E_PF_STAT("port.fdir_sb_status", stats.fd_sb_status),
 
 	/* LPI stats */
-	I40E_PF_STAT("tx_lpi_status", stats.tx_lpi_status),
-	I40E_PF_STAT("rx_lpi_status", stats.rx_lpi_status),
-	I40E_PF_STAT("tx_lpi_count", stats.tx_lpi_count),
-	I40E_PF_STAT("rx_lpi_count", stats.rx_lpi_count),
+	I40E_PF_STAT("port.tx_lpi_status", stats.tx_lpi_status),
+	I40E_PF_STAT("port.rx_lpi_status", stats.rx_lpi_status),
+	I40E_PF_STAT("port.tx_lpi_count", stats.tx_lpi_count),
+	I40E_PF_STAT("port.rx_lpi_count", stats.rx_lpi_count),
 };
 
-#define I40E_QUEUE_STATS_LEN(n) \
-	(((struct i40e_netdev_priv *)netdev_priv((n)))->vsi->num_queue_pairs \
+/* We use num_tx_queues here as a proxy for the maximum number of queues
+ * available because we always allocate queues symmetrically.
+ */
+#define I40E_MAX_NUM_QUEUES(n) ((n)->num_tx_queues)
+#define I40E_QUEUE_STATS_LEN(n)                                              \
+	   (I40E_MAX_NUM_QUEUES(n)                                           \
 	    * 2 /* Tx and Rx together */                                     \
 	    * (sizeof(struct i40e_queue_stats) / sizeof(u64)))
 #define I40E_GLOBAL_STATS_LEN	ARRAY_SIZE(i40e_gstrings_stats)
-#define I40E_NETDEV_STATS_LEN   ARRAY_SIZE(i40e_gstrings_net_stats)
+#define I40E_NETDEV_STATS_LEN	ARRAY_SIZE(i40e_gstrings_net_stats)
 #define I40E_MISC_STATS_LEN	ARRAY_SIZE(i40e_gstrings_misc_stats)
-#define I40E_VSI_STATS_LEN(n)   (I40E_NETDEV_STATS_LEN + \
+#define I40E_VSI_STATS_LEN(n)	(I40E_NETDEV_STATS_LEN + \
 				 I40E_MISC_STATS_LEN + \
 				 I40E_QUEUE_STATS_LEN((n)))
 #define I40E_PFC_STATS_LEN ( \
@@ -977,7 +955,9 @@ static int i40e_set_link_ksettings(struct net_device *netdev,
 	    ethtool_link_ksettings_test_link_mode(ks, advertising,
 						  10000baseCR_Full) ||
 	    ethtool_link_ksettings_test_link_mode(ks, advertising,
-						  10000baseSR_Full))
+						  10000baseSR_Full) ||
+	    ethtool_link_ksettings_test_link_mode(ks, advertising,
+						  10000baseLR_Full))
 		config.link_speed |= I40E_LINK_SPEED_10GB;
 	if (ethtool_link_ksettings_test_link_mode(ks, advertising,
 						  20000baseKR2_Full))
@@ -1079,6 +1059,9 @@ static int i40e_nway_reset(struct net_device *netdev)
 
 /**
  * i40e_get_pauseparam -  Get Flow Control status
+ * @netdev: netdevice structure
+ * @pause: buffer to return pause parameters
+ *
  * Return tx/rx-pause status
  **/
 static void i40e_get_pauseparam(struct net_device *netdev,
@@ -1677,6 +1660,32 @@ static int i40e_set_ringparam(struct net_device *netdev,
 	return err;
 }
 
+/**
+ * i40e_get_stats_count - return the stats count for a device
+ * @netdev: the netdev to return the count for
+ *
+ * Returns the total number of statistics for this netdev. Note that even
+ * though this is a function, it is required that the count for a specific
+ * netdev must never change. Basing the count on static values such as the
+ * maximum number of queues or the device type is ok. However, the API for
+ * obtaining stats is *not* safe against changes based on non-static
+ * values such as the *current* number of queues, or runtime flags.
+ *
+ * If a statistic is not always enabled, return it as part of the count
+ * anyways, always return its string, and report its value as zero.
+ **/
+static int i40e_get_stats_count(struct net_device *netdev)
+{
+	struct i40e_netdev_priv *np = netdev_priv(netdev);
+	struct i40e_vsi *vsi = np->vsi;
+	struct i40e_pf *pf = vsi->back;
+
+	if (vsi == pf->vsi[pf->lan_vsi] && pf->hw.partition_id == 1)
+		return I40E_PF_STATS_LEN(netdev) + I40E_VEB_STATS_TOTAL;
+	else
+		return I40E_VSI_STATS_LEN(netdev);
+}
+
 static int i40e_get_sset_count(struct net_device *netdev, int sset)
 {
 	struct i40e_netdev_priv *np = netdev_priv(netdev);
@@ -1687,16 +1696,7 @@ static int i40e_get_sset_count(struct net_device *netdev, int sset)
 	case ETH_SS_TEST:
 		return I40E_TEST_LEN;
 	case ETH_SS_STATS:
-		if (vsi == pf->vsi[pf->lan_vsi] && pf->hw.partition_id == 1) {
-			int len = I40E_PF_STATS_LEN(netdev);
-
-			if ((pf->lan_veb != I40E_NO_VEB) &&
-			    (pf->flags & I40E_FLAG_VEB_STATS_ENABLED))
-				len += I40E_VEB_STATS_TOTAL;
-			return len;
-		} else {
-			return I40E_VSI_STATS_LEN(netdev);
-		}
+		return i40e_get_stats_count(netdev);
 	case ETH_SS_PRIV_FLAGS:
 		return I40E_PRIV_FLAGS_STR_LEN +
 			(pf->hw.pf_id == 0 ? I40E_GL_PRIV_FLAGS_STR_LEN : 0);
@@ -1705,6 +1705,20 @@ static int i40e_get_sset_count(struct net_device *netdev, int sset)
 	}
 }
 
+/**
+ * i40e_get_ethtool_stats - copy stat values into supplied buffer
+ * @netdev: the netdev to collect stats for
+ * @stats: ethtool stats command structure
+ * @data: ethtool supplied buffer
+ *
+ * Copy the stats values for this netdev into the buffer. Expects data to be
+ * pre-allocated to the size returned by i40e_get_stats_count.. Note that all
+ * statistics must be copied in a static order, and the count must not change
+ * for a given netdev. See i40e_get_stats_count for more details.
+ *
+ * If a statistic is not currently valid (such as a disabled queue), this
+ * function reports its value as zero.
+ **/
 static void i40e_get_ethtool_stats(struct net_device *netdev,
 				   struct ethtool_stats *stats, u64 *data)
 {
@@ -1712,47 +1726,54 @@ static void i40e_get_ethtool_stats(struct net_device *netdev,
 	struct i40e_ring *tx_ring, *rx_ring;
 	struct i40e_vsi *vsi = np->vsi;
 	struct i40e_pf *pf = vsi->back;
-	unsigned int j;
-	int i = 0;
+	unsigned int i;
 	char *p;
 	struct rtnl_link_stats64 *net_stats = i40e_get_vsi_stats_struct(vsi);
 	unsigned int start;
 
 	i40e_update_stats(vsi);
 
-	for (j = 0; j < I40E_NETDEV_STATS_LEN; j++) {
-		p = (char *)net_stats + i40e_gstrings_net_stats[j].stat_offset;
-		data[i++] = (i40e_gstrings_net_stats[j].sizeof_stat ==
+	for (i = 0; i < I40E_NETDEV_STATS_LEN; i++) {
+		p = (char *)net_stats + i40e_gstrings_net_stats[i].stat_offset;
+		*(data++) = (i40e_gstrings_net_stats[i].sizeof_stat ==
 			sizeof(u64)) ? *(u64 *)p : *(u32 *)p;
 	}
-	for (j = 0; j < I40E_MISC_STATS_LEN; j++) {
-		p = (char *)vsi + i40e_gstrings_misc_stats[j].stat_offset;
-		data[i++] = (i40e_gstrings_misc_stats[j].sizeof_stat ==
+	for (i = 0; i < I40E_MISC_STATS_LEN; i++) {
+		p = (char *)vsi + i40e_gstrings_misc_stats[i].stat_offset;
+		*(data++) = (i40e_gstrings_misc_stats[i].sizeof_stat ==
 			    sizeof(u64)) ? *(u64 *)p : *(u32 *)p;
 	}
 	rcu_read_lock();
-	for (j = 0; j < vsi->num_queue_pairs; j++) {
-		tx_ring = READ_ONCE(vsi->tx_rings[j]);
+	for (i = 0; i < I40E_MAX_NUM_QUEUES(netdev) ; i++) {
+		tx_ring = READ_ONCE(vsi->tx_rings[i]);
 
-		if (!tx_ring)
+		if (!tx_ring) {
+			/* Bump the stat counter to skip these stats, and make
+			 * sure the memory is zero'd
+			 */
+			*(data++) = 0;
+			*(data++) = 0;
+			*(data++) = 0;
+			*(data++) = 0;
 			continue;
+		}
 
 		/* process Tx ring statistics */
 		do {
 			start = u64_stats_fetch_begin_irq(&tx_ring->syncp);
-			data[i] = tx_ring->stats.packets;
-			data[i + 1] = tx_ring->stats.bytes;
+			data[0] = tx_ring->stats.packets;
+			data[1] = tx_ring->stats.bytes;
 		} while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));
-		i += 2;
+		data += 2;
 
 		/* Rx ring is the 2nd half of the queue pair */
 		rx_ring = &tx_ring[1];
 		do {
 			start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
-			data[i] = rx_ring->stats.packets;
-			data[i + 1] = rx_ring->stats.bytes;
+			data[0] = rx_ring->stats.packets;
+			data[1] = rx_ring->stats.bytes;
 		} while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
-		i += 2;
+		data += 2;
 	}
 	rcu_read_unlock();
 	if (vsi != pf->vsi[pf->lan_vsi] || pf->hw.partition_id != 1)
@@ -1762,38 +1783,131 @@ static void i40e_get_ethtool_stats(struct net_device *netdev,
 	    (pf->flags & I40E_FLAG_VEB_STATS_ENABLED)) {
 		struct i40e_veb *veb = pf->veb[pf->lan_veb];
 
-		for (j = 0; j < I40E_VEB_STATS_LEN; j++) {
+		for (i = 0; i < I40E_VEB_STATS_LEN; i++) {
 			p = (char *)veb;
-			p += i40e_gstrings_veb_stats[j].stat_offset;
-			data[i++] = (i40e_gstrings_veb_stats[j].sizeof_stat ==
+			p += i40e_gstrings_veb_stats[i].stat_offset;
+			*(data++) = (i40e_gstrings_veb_stats[i].sizeof_stat ==
 				     sizeof(u64)) ? *(u64 *)p : *(u32 *)p;
 		}
-		for (j = 0; j < I40E_MAX_TRAFFIC_CLASS; j++) {
-			data[i++] = veb->tc_stats.tc_tx_packets[j];
-			data[i++] = veb->tc_stats.tc_tx_bytes[j];
-			data[i++] = veb->tc_stats.tc_rx_packets[j];
-			data[i++] = veb->tc_stats.tc_rx_bytes[j];
+		for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
+			*(data++) = veb->tc_stats.tc_tx_packets[i];
+			*(data++) = veb->tc_stats.tc_tx_bytes[i];
+			*(data++) = veb->tc_stats.tc_rx_packets[i];
+			*(data++) = veb->tc_stats.tc_rx_bytes[i];
 		}
+	} else {
+		data += I40E_VEB_STATS_TOTAL;
 	}
-	for (j = 0; j < I40E_GLOBAL_STATS_LEN; j++) {
-		p = (char *)pf + i40e_gstrings_stats[j].stat_offset;
-		data[i++] = (i40e_gstrings_stats[j].sizeof_stat ==
+	for (i = 0; i < I40E_GLOBAL_STATS_LEN; i++) {
+		p = (char *)pf + i40e_gstrings_stats[i].stat_offset;
+		*(data++) = (i40e_gstrings_stats[i].sizeof_stat ==
 			     sizeof(u64)) ? *(u64 *)p : *(u32 *)p;
 	}
-	for (j = 0; j < I40E_MAX_USER_PRIORITY; j++) {
-		data[i++] = pf->stats.priority_xon_tx[j];
-		data[i++] = pf->stats.priority_xoff_tx[j];
+	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
+		*(data++) = pf->stats.priority_xon_tx[i];
+		*(data++) = pf->stats.priority_xoff_tx[i];
 	}
-	for (j = 0; j < I40E_MAX_USER_PRIORITY; j++) {
-		data[i++] = pf->stats.priority_xon_rx[j];
-		data[i++] = pf->stats.priority_xoff_rx[j];
+	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
+		*(data++) = pf->stats.priority_xon_rx[i];
+		*(data++) = pf->stats.priority_xoff_rx[i];
 	}
-	for (j = 0; j < I40E_MAX_USER_PRIORITY; j++)
-		data[i++] = pf->stats.priority_xon_2_xoff[j];
+	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
+		*(data++) = pf->stats.priority_xon_2_xoff[i];
 }
 
-static void i40e_get_strings(struct net_device *netdev, u32 stringset,
-			     u8 *data)
+/**
+ * i40e_get_stat_strings - copy stat strings into supplied buffer
+ * @netdev: the netdev to collect strings for
+ * @data: supplied buffer to copy strings into
+ *
+ * Copy the strings related to stats for this netdev. Expects data to be
+ * pre-allocated with the size reported by i40e_get_stats_count. Note that the
+ * strings must be copied in a static order and the total count must not
+ * change for a given netdev. See i40e_get_stats_count for more details.
+ **/
+static void i40e_get_stat_strings(struct net_device *netdev, u8 *data)
+{
+	struct i40e_netdev_priv *np = netdev_priv(netdev);
+	struct i40e_vsi *vsi = np->vsi;
+	struct i40e_pf *pf = vsi->back;
+	unsigned int i;
+	u8 *p = data;
+
+	for (i = 0; i < I40E_NETDEV_STATS_LEN; i++) {
+		snprintf(data, ETH_GSTRING_LEN, "%s",
+			 i40e_gstrings_net_stats[i].stat_string);
+		data += ETH_GSTRING_LEN;
+	}
+	for (i = 0; i < I40E_MISC_STATS_LEN; i++) {
+		snprintf(data, ETH_GSTRING_LEN, "%s",
+			 i40e_gstrings_misc_stats[i].stat_string);
+		data += ETH_GSTRING_LEN;
+	}
+	for (i = 0; i < I40E_MAX_NUM_QUEUES(netdev); i++) {
+		snprintf(data, ETH_GSTRING_LEN, "tx-%u.tx_packets", i);
+		data += ETH_GSTRING_LEN;
+		snprintf(data, ETH_GSTRING_LEN, "tx-%u.tx_bytes", i);
+		data += ETH_GSTRING_LEN;
+		snprintf(data, ETH_GSTRING_LEN, "rx-%u.rx_packets", i);
+		data += ETH_GSTRING_LEN;
+		snprintf(data, ETH_GSTRING_LEN, "rx-%u.rx_bytes", i);
+		data += ETH_GSTRING_LEN;
+	}
+	if (vsi != pf->vsi[pf->lan_vsi] || pf->hw.partition_id != 1)
+		return;
+
+	for (i = 0; i < I40E_VEB_STATS_LEN; i++) {
+		snprintf(data, ETH_GSTRING_LEN, "%s",
+			 i40e_gstrings_veb_stats[i].stat_string);
+		data += ETH_GSTRING_LEN;
+	}
+	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
+		snprintf(data, ETH_GSTRING_LEN,
+			 "veb.tc_%u_tx_packets", i);
+		data += ETH_GSTRING_LEN;
+		snprintf(data, ETH_GSTRING_LEN,
+			 "veb.tc_%u_tx_bytes", i);
+		data += ETH_GSTRING_LEN;
+		snprintf(data, ETH_GSTRING_LEN,
+			 "veb.tc_%u_rx_packets", i);
+		data += ETH_GSTRING_LEN;
+		snprintf(data, ETH_GSTRING_LEN,
+			 "veb.tc_%u_rx_bytes", i);
+		data += ETH_GSTRING_LEN;
+	}
+
+	for (i = 0; i < I40E_GLOBAL_STATS_LEN; i++) {
+		snprintf(data, ETH_GSTRING_LEN, "%s",
+			 i40e_gstrings_stats[i].stat_string);
+		data += ETH_GSTRING_LEN;
+	}
+	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
+		snprintf(data, ETH_GSTRING_LEN,
+			 "port.tx_priority_%u_xon", i);
+		data += ETH_GSTRING_LEN;
+		snprintf(data, ETH_GSTRING_LEN,
+			 "port.tx_priority_%u_xoff", i);
+		data += ETH_GSTRING_LEN;
+	}
+	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
+		snprintf(data, ETH_GSTRING_LEN,
+			 "port.rx_priority_%u_xon", i);
+		data += ETH_GSTRING_LEN;
+		snprintf(data, ETH_GSTRING_LEN,
+			 "port.rx_priority_%u_xoff", i);
+		data += ETH_GSTRING_LEN;
+	}
+	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
+		snprintf(data, ETH_GSTRING_LEN,
+			 "port.rx_priority_%u_xon_2_xoff", i);
+		data += ETH_GSTRING_LEN;
+	}
+
+	WARN_ONCE(p - data != i40e_get_stats_count(netdev) * ETH_GSTRING_LEN,
+		  "stat strings count mismatch!");
+}
+
+static void i40e_get_priv_flag_strings(struct net_device *netdev, u8 *data)
 {
 	struct i40e_netdev_priv *np = netdev_priv(netdev);
 	struct i40e_vsi *vsi = np->vsi;
@@ -1801,98 +1915,33 @@ static void i40e_get_strings(struct net_device *netdev, u32 stringset,
 	char *p = (char *)data;
 	unsigned int i;
 
+	for (i = 0; i < I40E_PRIV_FLAGS_STR_LEN; i++) {
+		snprintf(p, ETH_GSTRING_LEN, "%s",
+			 i40e_gstrings_priv_flags[i].flag_string);
+		p += ETH_GSTRING_LEN;
+	}
+	if (pf->hw.pf_id != 0)
+		return;
+	for (i = 0; i < I40E_GL_PRIV_FLAGS_STR_LEN; i++) {
+		snprintf(p, ETH_GSTRING_LEN, "%s",
+			 i40e_gl_gstrings_priv_flags[i].flag_string);
+		p += ETH_GSTRING_LEN;
+	}
+}
+
+static void i40e_get_strings(struct net_device *netdev, u32 stringset,
+			     u8 *data)
+{
 	switch (stringset) {
 	case ETH_SS_TEST:
 		memcpy(data, i40e_gstrings_test,
 		       I40E_TEST_LEN * ETH_GSTRING_LEN);
 		break;
 	case ETH_SS_STATS:
-		for (i = 0; i < I40E_NETDEV_STATS_LEN; i++) {
-			snprintf(p, ETH_GSTRING_LEN, "%s",
-				 i40e_gstrings_net_stats[i].stat_string);
-			p += ETH_GSTRING_LEN;
-		}
-		for (i = 0; i < I40E_MISC_STATS_LEN; i++) {
-			snprintf(p, ETH_GSTRING_LEN, "%s",
-				 i40e_gstrings_misc_stats[i].stat_string);
-			p += ETH_GSTRING_LEN;
-		}
-		for (i = 0; i < vsi->num_queue_pairs; i++) {
-			snprintf(p, ETH_GSTRING_LEN, "tx-%d.tx_packets", i);
-			p += ETH_GSTRING_LEN;
-			snprintf(p, ETH_GSTRING_LEN, "tx-%d.tx_bytes", i);
-			p += ETH_GSTRING_LEN;
-			snprintf(p, ETH_GSTRING_LEN, "rx-%d.rx_packets", i);
-			p += ETH_GSTRING_LEN;
-			snprintf(p, ETH_GSTRING_LEN, "rx-%d.rx_bytes", i);
-			p += ETH_GSTRING_LEN;
-		}
-		if (vsi != pf->vsi[pf->lan_vsi] || pf->hw.partition_id != 1)
-			return;
-
-		if ((pf->lan_veb != I40E_NO_VEB) &&
-		    (pf->flags & I40E_FLAG_VEB_STATS_ENABLED)) {
-			for (i = 0; i < I40E_VEB_STATS_LEN; i++) {
-				snprintf(p, ETH_GSTRING_LEN, "veb.%s",
-					i40e_gstrings_veb_stats[i].stat_string);
-				p += ETH_GSTRING_LEN;
-			}
-			for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
-				snprintf(p, ETH_GSTRING_LEN,
-					 "veb.tc_%d_tx_packets", i);
-				p += ETH_GSTRING_LEN;
-				snprintf(p, ETH_GSTRING_LEN,
-					 "veb.tc_%d_tx_bytes", i);
-				p += ETH_GSTRING_LEN;
-				snprintf(p, ETH_GSTRING_LEN,
-					 "veb.tc_%d_rx_packets", i);
-				p += ETH_GSTRING_LEN;
-				snprintf(p, ETH_GSTRING_LEN,
-					 "veb.tc_%d_rx_bytes", i);
-				p += ETH_GSTRING_LEN;
-			}
-		}
-		for (i = 0; i < I40E_GLOBAL_STATS_LEN; i++) {
-			snprintf(p, ETH_GSTRING_LEN, "port.%s",
-				 i40e_gstrings_stats[i].stat_string);
-			p += ETH_GSTRING_LEN;
-		}
-		for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
-			snprintf(p, ETH_GSTRING_LEN,
-				 "port.tx_priority_%d_xon", i);
-			p += ETH_GSTRING_LEN;
-			snprintf(p, ETH_GSTRING_LEN,
-				 "port.tx_priority_%d_xoff", i);
-			p += ETH_GSTRING_LEN;
-		}
-		for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
-			snprintf(p, ETH_GSTRING_LEN,
-				 "port.rx_priority_%d_xon", i);
-			p += ETH_GSTRING_LEN;
-			snprintf(p, ETH_GSTRING_LEN,
-				 "port.rx_priority_%d_xoff", i);
-			p += ETH_GSTRING_LEN;
-		}
-		for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
-			snprintf(p, ETH_GSTRING_LEN,
-				 "port.rx_priority_%d_xon_2_xoff", i);
-			p += ETH_GSTRING_LEN;
-		}
-		/* BUG_ON(p - data != I40E_STATS_LEN * ETH_GSTRING_LEN); */
+		i40e_get_stat_strings(netdev, data);
 		break;
 	case ETH_SS_PRIV_FLAGS:
-		for (i = 0; i < I40E_PRIV_FLAGS_STR_LEN; i++) {
-			snprintf(p, ETH_GSTRING_LEN, "%s",
-				 i40e_gstrings_priv_flags[i].flag_string);
-			p += ETH_GSTRING_LEN;
-		}
-		if (pf->hw.pf_id != 0)
-			break;
-		for (i = 0; i < I40E_GL_PRIV_FLAGS_STR_LEN; i++) {
-			snprintf(p, ETH_GSTRING_LEN, "%s",
-				 i40e_gl_gstrings_priv_flags[i].flag_string);
-			p += ETH_GSTRING_LEN;
-		}
+		i40e_get_priv_flag_strings(netdev, data);
 		break;
 	default:
 		break;
@@ -2550,7 +2599,7 @@ static int i40e_get_rss_hash_opts(struct i40e_pf *pf, struct ethtool_rxnfc *cmd)
 /**
  * i40e_check_mask - Check whether a mask field is set
  * @mask: the full mask value
- * @field; mask of the field to check
+ * @field: mask of the field to check
  *
  * If the given mask is fully set, return positive value. If the mask for the
  * field is fully unset, return zero. Otherwise return a negative error code.
@@ -2621,6 +2670,7 @@ static int i40e_parse_rx_flow_user_data(struct ethtool_rx_flow_spec *fsp,
 /**
  * i40e_fill_rx_flow_user_data - Fill in user-defined data field
  * @fsp: pointer to rx_flow specification
+ * @data: pointer to return userdef data
  *
  * Reads the userdef data structure and properly fills in the user defined
  * fields of the rx_flow_spec.
@@ -2799,6 +2849,7 @@ static int i40e_get_ethtool_fdir_entry(struct i40e_pf *pf,
  * i40e_get_rxnfc - command to get RX flow classification rules
  * @netdev: network interface device structure
  * @cmd: ethtool rxnfc command
+ * @rule_locs: pointer to store rule data
  *
  * Returns Success if the command is supported.
  **/
@@ -2840,7 +2891,7 @@ static int i40e_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd,
 /**
  * i40e_get_rss_hash_bits - Read RSS Hash bits from register
  * @nfc: pointer to user request
- * @i_setc bits currently set
+ * @i_setc: bits currently set
  *
  * Returns value of bits to be set per user request
  **/
@@ -2885,7 +2936,7 @@ static u64 i40e_get_rss_hash_bits(struct ethtool_rxnfc *nfc, u64 i_setc)
 /**
  * i40e_set_rss_hash_opt - Enable/Disable flow types for RSS hash
  * @pf: pointer to the physical function struct
- * @cmd: ethtool rxnfc command
+ * @nfc: ethtool rxnfc command
  *
  * Returns Success if the flow input set is supported.
  **/
@@ -3284,7 +3335,7 @@ static int i40e_add_flex_offset(struct list_head *flex_pit_list,
  * __i40e_reprogram_flex_pit - Re-program specific FLX_PIT table
  * @pf: Pointer to the PF structure
  * @flex_pit_list: list of flexible src offsets in use
- * #flex_pit_start: index to first entry for this section of the table
+ * @flex_pit_start: index to first entry for this section of the table
  *
  * In order to handle flexible data, the hardware uses a table of values
  * called the FLX_PIT table. This table is used to indicate which sections of
@@ -3398,7 +3449,7 @@ static void i40e_reprogram_flex_pit(struct i40e_pf *pf)
 
 /**
  * i40e_flow_str - Converts a flow_type into a human readable string
- * @flow_type: the flow type from a flow specification
+ * @fsp: the flow specification
  *
  * Currently only flow types we support are included here, and the string
  * value attempts to match what ethtool would use to configure this flow type.
@@ -4103,7 +4154,7 @@ static unsigned int i40e_max_channels(struct i40e_vsi *vsi)
 
 /**
  * i40e_get_channels - Get the current channels enabled and max supported etc.
- * @netdev: network interface device structure
+ * @dev: network interface device structure
  * @ch: ethtool channels structure
  *
  * We don't support separate tx and rx queues as channels. The other count
@@ -4112,7 +4163,7 @@ static unsigned int i40e_max_channels(struct i40e_vsi *vsi)
  * q_vectors since we support a lot more queue pairs than q_vectors.
  **/
 static void i40e_get_channels(struct net_device *dev,
-			       struct ethtool_channels *ch)
+			      struct ethtool_channels *ch)
 {
 	struct i40e_netdev_priv *np = netdev_priv(dev);
 	struct i40e_vsi *vsi = np->vsi;
@@ -4131,14 +4182,14 @@ static void i40e_get_channels(struct net_device *dev,
 
 /**
  * i40e_set_channels - Set the new channels count.
- * @netdev: network interface device structure
+ * @dev: network interface device structure
  * @ch: ethtool channels structure
  *
  * The new channels count may not be the same as requested by the user
  * since it gets rounded down to a power of 2 value.
  **/
 static int i40e_set_channels(struct net_device *dev,
-			      struct ethtool_channels *ch)
+			     struct ethtool_channels *ch)
 {
 	const u8 drop = I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET;
 	struct i40e_netdev_priv *np = netdev_priv(dev);
@@ -4273,6 +4324,7 @@ static int i40e_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key,
  * @netdev: network interface device structure
  * @indir: indirection table
  * @key: hash key
+ * @hfunc: hash function to use
  *
  * Returns -EINVAL if the table specifies an invalid queue id, otherwise
  * returns 0 after programming the table.
diff --git a/drivers/net/ethernet/intel/i40e/i40e_hmc.c b/drivers/net/ethernet/intel/i40e/i40e_hmc.c
index 6d4b590f851b..19ce93d7fd0a 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_hmc.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_hmc.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40e_osdep.h"
 #include "i40e_register.h"
@@ -198,7 +174,6 @@ i40e_status i40e_add_pd_table_entry(struct i40e_hw *hw,
  * @hw: pointer to our HW structure
  * @hmc_info: pointer to the HMC configuration information structure
  * @idx: the page index
- * @is_pf: distinguishes a VF from a PF
  *
  * This function:
  *	1. Marks the entry in pd tabe (for paged address mode) or in sd table
diff --git a/drivers/net/ethernet/intel/i40e/i40e_hmc.h b/drivers/net/ethernet/intel/i40e/i40e_hmc.h
index 7b5fd33d70ae..1c78de838857 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_hmc.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_hmc.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_HMC_H_
 #define _I40E_HMC_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.c b/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.c
index cd40dc487b38..994011c38fb4 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40e_osdep.h"
 #include "i40e_register.h"
diff --git a/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.h b/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.h
index 79e1396735d9..c46a2c449e60 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_lan_hmc.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_LAN_HMC_H_
 #define _I40E_LAN_HMC_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 16229998fb1e..b5daa5c9c7de 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include <linux/etherdevice.h>
 #include <linux/of_net.h>
@@ -278,8 +254,8 @@ static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 
 /**
  * i40e_find_vsi_from_id - searches for the vsi with the given id
- * @pf - the pf structure to search for the vsi
- * @id - id of the vsi it is searching for
+ * @pf: the pf structure to search for the vsi
+ * @id: id of the vsi it is searching for
  **/
 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 {
@@ -435,6 +411,7 @@ static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
 /**
  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
  * @netdev: network interface device structure
+ * @stats: data structure to store statistics
  *
  * Returns the address of the device statistics structure.
  * The statistics are actually updated from the service task.
@@ -2027,7 +2004,7 @@ struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
  * from firmware
  * @count: Number of filters added
  * @add_list: return data from fw
- * @head: pointer to first filter in current batch
+ * @add_head: pointer to first filter in current batch
  *
  * MAC filter entries from list were slated to be added to device. Returns
  * number of successful filters. Note that 0 does NOT mean success!
@@ -2134,6 +2111,7 @@ void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
 /**
  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
  * @vsi: pointer to the VSI
+ * @vsi_name: the VSI name
  * @f: filter data
  *
  * This function sets or clears the promiscuous broadcast flags for VLAN
@@ -2840,6 +2818,7 @@ void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
 /**
  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
  * @netdev: network interface to be adjusted
+ * @proto: unused protocol value
  * @vid: vlan id to be added
  *
  * net_device_ops implementation for adding vlan ids
@@ -2861,9 +2840,27 @@ static int i40e_vlan_rx_add_vid(struct net_device *netdev,
 	return ret;
 }
 
+/**
+ * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
+ * @netdev: network interface to be adjusted
+ * @proto: unused protocol value
+ * @vid: vlan id to be added
+ **/
+static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
+				    __always_unused __be16 proto, u16 vid)
+{
+	struct i40e_netdev_priv *np = netdev_priv(netdev);
+	struct i40e_vsi *vsi = np->vsi;
+
+	if (vid >= VLAN_N_VID)
+		return;
+	set_bit(vid, vsi->active_vlans);
+}
+
 /**
  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
  * @netdev: network interface to be adjusted
+ * @proto: unused protocol value
  * @vid: vlan id to be removed
  *
  * net_device_ops implementation for removing vlan ids
@@ -2902,8 +2899,8 @@ static void i40e_restore_vlan(struct i40e_vsi *vsi)
 		i40e_vlan_stripping_disable(vsi);
 
 	for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
-		i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
-				     vid);
+		i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
+					vid);
 }
 
 /**
@@ -3485,7 +3482,7 @@ static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
 
 /**
  * i40e_enable_misc_int_causes - enable the non-queue interrupts
- * @hw: ptr to the hardware info
+ * @pf: pointer to private device data structure
  **/
 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
 {
@@ -4255,8 +4252,8 @@ static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
  * @is_xdp: true if the queue is used for XDP
  * @enable: start or stop the queue
  **/
-static int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
-				  bool is_xdp, bool enable)
+int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
+			   bool is_xdp, bool enable)
 {
 	int ret;
 
@@ -4301,7 +4298,6 @@ static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
 		if (ret)
 			break;
 	}
-
 	return ret;
 }
 
@@ -4340,9 +4336,9 @@ static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
  * @pf_q: the PF queue to configure
  * @enable: start or stop the queue
  *
- * This function enables or disables a single queue. Note that any delay
- * required after the operation is expected to be handled by the caller of
- * this function.
+ * This function enables or disables a single queue. Note that
+ * any delay required after the operation is expected to be
+ * handled by the caller of this function.
  **/
 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
 {
@@ -4371,6 +4367,30 @@ static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
 	wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
 }
 
+/**
+ * i40e_control_wait_rx_q
+ * @pf: the PF structure
+ * @pf_q: queue being configured
+ * @enable: start or stop the rings
+ *
+ * This function enables or disables a single queue along with waiting
+ * for the change to finish. The caller of this function should handle
+ * the delays needed in the case of disabling queues.
+ **/
+int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
+{
+	int ret = 0;
+
+	i40e_control_rx_q(pf, pf_q, enable);
+
+	/* wait for the change to finish */
+	ret = i40e_pf_rxq_wait(pf, pf_q, enable);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
 /**
  * i40e_vsi_control_rx - Start or stop a VSI's rings
  * @vsi: the VSI being configured
@@ -4383,10 +4403,7 @@ static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
 
 	pf_q = vsi->base_queue;
 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
-		i40e_control_rx_q(pf, pf_q, enable);
-
-		/* wait for the change to finish */
-		ret = i40e_pf_rxq_wait(pf, pf_q, enable);
+		ret = i40e_control_wait_rx_q(pf, pf_q, enable);
 		if (ret) {
 			dev_info(&pf->pdev->dev,
 				 "VSI seid %d Rx ring %d %sable timeout\n",
@@ -5096,7 +5113,7 @@ static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
  * @vsi: the VSI being configured
  * @enabled_tc: TC bitmap
- * @bw_credits: BW shared credits per TC
+ * @bw_share: BW shared credits per TC
  *
  * Returns 0 on success, negative value on failure
  **/
@@ -6353,6 +6370,7 @@ static int i40e_init_pf_dcb(struct i40e_pf *pf)
 /**
  * i40e_print_link_message - print link up or down
  * @vsi: the VSI for which link needs a message
+ * @isup: true of link is up, false otherwise
  */
 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
 {
@@ -7212,8 +7230,7 @@ static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
 			if (mask->dst == cpu_to_be32(0xffffffff)) {
 				field_flags |= I40E_CLOUD_FIELD_IIP;
 			} else {
-				mask->dst = be32_to_cpu(mask->dst);
-				dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4\n",
+				dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
 					&mask->dst);
 				return I40E_ERR_CONFIG;
 			}
@@ -7223,8 +7240,7 @@ static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
 			if (mask->src == cpu_to_be32(0xffffffff)) {
 				field_flags |= I40E_CLOUD_FIELD_IIP;
 			} else {
-				mask->src = be32_to_cpu(mask->src);
-				dev_err(&pf->pdev->dev, "Bad ip src mask %pI4\n",
+				dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
 					&mask->src);
 				return I40E_ERR_CONFIG;
 			}
@@ -9691,9 +9707,9 @@ static void i40e_handle_mdd_event(struct i40e_pf *pf)
 	i40e_flush(hw);
 }
 
-static const char *i40e_tunnel_name(struct i40e_udp_port_config *port)
+static const char *i40e_tunnel_name(u8 type)
 {
-	switch (port->type) {
+	switch (type) {
 	case UDP_TUNNEL_TYPE_VXLAN:
 		return "vxlan";
 	case UDP_TUNNEL_TYPE_GENEVE:
@@ -9727,37 +9743,68 @@ static void i40e_sync_udp_filters(struct i40e_pf *pf)
 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
-	i40e_status ret;
+	u8 filter_index, type;
 	u16 port;
 	int i;
 
 	if (!test_and_clear_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state))
 		return;
 
+	/* acquire RTNL to maintain state of flags and port requests */
+	rtnl_lock();
+
 	for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
 		if (pf->pending_udp_bitmap & BIT_ULL(i)) {
+			struct i40e_udp_port_config *udp_port;
+			i40e_status ret = 0;
+
+			udp_port = &pf->udp_ports[i];
 			pf->pending_udp_bitmap &= ~BIT_ULL(i);
-			port = pf->udp_ports[i].port;
+
+			port = READ_ONCE(udp_port->port);
+			type = READ_ONCE(udp_port->type);
+			filter_index = READ_ONCE(udp_port->filter_index);
+
+			/* release RTNL while we wait on AQ command */
+			rtnl_unlock();
+
 			if (port)
 				ret = i40e_aq_add_udp_tunnel(hw, port,
-							pf->udp_ports[i].type,
-							NULL, NULL);
-			else
-				ret = i40e_aq_del_udp_tunnel(hw, i, NULL);
+							     type,
+							     &filter_index,
+							     NULL);
+			else if (filter_index != I40E_UDP_PORT_INDEX_UNUSED)
+				ret = i40e_aq_del_udp_tunnel(hw, filter_index,
+							     NULL);
+
+			/* reacquire RTNL so we can update filter_index */
+			rtnl_lock();
 
 			if (ret) {
 				dev_info(&pf->pdev->dev,
 					 "%s %s port %d, index %d failed, err %s aq_err %s\n",
-					 i40e_tunnel_name(&pf->udp_ports[i]),
+					 i40e_tunnel_name(type),
 					 port ? "add" : "delete",
-					 port, i,
+					 port,
+					 filter_index,
 					 i40e_stat_str(&pf->hw, ret),
 					 i40e_aq_str(&pf->hw,
 						     pf->hw.aq.asq_last_status));
-				pf->udp_ports[i].port = 0;
+				if (port) {
+					/* failed to add, just reset port,
+					 * drop pending bit for any deletion
+					 */
+					udp_port->port = 0;
+					pf->pending_udp_bitmap &= ~BIT_ULL(i);
+				}
+			} else if (port) {
+				/* record filter index on success */
+				udp_port->filter_index = filter_index;
 			}
 		}
 	}
+
+	rtnl_unlock();
 }
 
 /**
@@ -10004,7 +10051,7 @@ static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
 
 /**
  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
- * @type: VSI pointer
+ * @vsi: VSI pointer
  * @free_qvectors: a bool to specify if q_vectors need to be freed.
  *
  * On error: returns error code (negative)
@@ -10279,21 +10326,28 @@ static int i40e_init_msix(struct i40e_pf *pf)
 
 	/* any vectors left over go for VMDq support */
 	if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
-		int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps;
-		int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted);
-
 		if (!vectors_left) {
 			pf->num_vmdq_msix = 0;
 			pf->num_vmdq_qps = 0;
 		} else {
+			int vmdq_vecs_wanted =
+				pf->num_vmdq_vsis * pf->num_vmdq_qps;
+			int vmdq_vecs =
+				min_t(int, vectors_left, vmdq_vecs_wanted);
+
 			/* if we're short on vectors for what's desired, we limit
 			 * the queues per vmdq.  If this is still more than are
 			 * available, the user will need to change the number of
 			 * queues/vectors used by the PF later with the ethtool
 			 * channels command
 			 */
-			if (vmdq_vecs < vmdq_vecs_wanted)
+			if (vectors_left < vmdq_vecs_wanted) {
 				pf->num_vmdq_qps = 1;
+				vmdq_vecs_wanted = pf->num_vmdq_vsis;
+				vmdq_vecs = min_t(int,
+						  vectors_left,
+						  vmdq_vecs_wanted);
+			}
 			pf->num_vmdq_msix = pf->num_vmdq_qps;
 
 			v_budget += vmdq_vecs;
@@ -10800,7 +10854,7 @@ int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
  * @vsi: Pointer to VSI structure
  * @seed: Buffer to store the keys
  * @lut: Buffer to store the lookup table entries
- * lut_size: Size of buffer to store the lookup table entries
+ * @lut_size: Size of buffer to store the lookup table entries
  *
  * Returns 0 on success, negative on failure
  */
@@ -11374,6 +11428,11 @@ static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port)
 	u8 i;
 
 	for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
+		/* Do not report ports with pending deletions as
+		 * being available.
+		 */
+		if (!port && (pf->pending_udp_bitmap & BIT_ULL(i)))
+			continue;
 		if (pf->udp_ports[i].port == port)
 			return i;
 	}
@@ -11428,6 +11487,7 @@ static void i40e_udp_tunnel_add(struct net_device *netdev,
 
 	/* New port: add it and mark its index in the bitmap */
 	pf->udp_ports[next_idx].port = port;
+	pf->udp_ports[next_idx].filter_index = I40E_UDP_PORT_INDEX_UNUSED;
 	pf->pending_udp_bitmap |= BIT_ULL(next_idx);
 	set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
 }
@@ -11469,7 +11529,12 @@ static void i40e_udp_tunnel_del(struct net_device *netdev,
 	 * and make it pending
 	 */
 	pf->udp_ports[idx].port = 0;
-	pf->pending_udp_bitmap |= BIT_ULL(idx);
+
+	/* Toggle pending bit instead of setting it. This way if we are
+	 * deleting a port that has yet to be added we just clear the pending
+	 * bit and don't have to worry about it.
+	 */
+	pf->pending_udp_bitmap ^= BIT_ULL(idx);
 	set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
 
 	return;
@@ -11500,6 +11565,7 @@ static int i40e_get_phys_port_id(struct net_device *netdev,
  * @tb: pointer to array of nladdr (unused)
  * @dev: the net device pointer
  * @addr: the MAC address entry being added
+ * @vid: VLAN ID
  * @flags: instructions from stack about fdb operation
  */
 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
@@ -11545,6 +11611,7 @@ static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
  * @dev: the netdev being configured
  * @nlh: RTNL message
+ * @flags: bridge flags
  *
  * Inserts a new hardware bridge if not already created and
  * enables the bridging mode requested (VEB or VEPA). If the
@@ -14118,6 +14185,7 @@ static void i40e_remove(struct pci_dev *pdev)
 /**
  * i40e_pci_error_detected - warning that something funky happened in PCI land
  * @pdev: PCI device information struct
+ * @error: the type of PCI error
  *
  * Called to warn that something happened and the error handling steps
  * are in progress.  Allows the driver to quiesce things, be ready for
diff --git a/drivers/net/ethernet/intel/i40e/i40e_nvm.c b/drivers/net/ethernet/intel/i40e/i40e_nvm.c
index ba9687c03795..0299e5bbb902 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_nvm.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_nvm.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40e_prototype.h"
 
@@ -1173,6 +1149,7 @@ void i40e_nvmupd_clear_wait_state(struct i40e_hw *hw)
  * i40e_nvmupd_check_wait_event - handle NVM update operation events
  * @hw: pointer to the hardware structure
  * @opcode: the event that just happened
+ * @desc: AdminQ descriptor
  **/
 void i40e_nvmupd_check_wait_event(struct i40e_hw *hw, u16 opcode,
 				  struct i40e_aq_desc *desc)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_osdep.h b/drivers/net/ethernet/intel/i40e/i40e_osdep.h
index 9c3c3b0d3ac4..a07574bff550 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_osdep.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_osdep.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_OSDEP_H_
 #define _I40E_OSDEP_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_prototype.h b/drivers/net/ethernet/intel/i40e/i40e_prototype.h
index 2ec24188d6e2..3170655cdeb9 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_prototype.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_prototype.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_PROTOTYPE_H_
 #define _I40E_PROTOTYPE_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
index 5b47dd1f75a5..35f2866b38c6 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40e.h"
 #include <linux/ptp_classify.h>
@@ -40,9 +16,9 @@
  * At 1Gb link, the period is multiplied by 20. (32ns)
  * 1588 functionality is not supported at 100Mbps.
  */
-#define I40E_PTP_40GB_INCVAL 0x0199999999ULL
-#define I40E_PTP_10GB_INCVAL 0x0333333333ULL
-#define I40E_PTP_1GB_INCVAL  0x2000000000ULL
+#define I40E_PTP_40GB_INCVAL		0x0199999999ULL
+#define I40E_PTP_10GB_INCVAL_MULT	2
+#define I40E_PTP_1GB_INCVAL_MULT	20
 
 #define I40E_PRTTSYN_CTL1_TSYNTYPE_V1  BIT(I40E_PRTTSYN_CTL1_TSYNTYPE_SHIFT)
 #define I40E_PRTTSYN_CTL1_TSYNTYPE_V2  (2 << \
@@ -130,17 +106,24 @@ static int i40e_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
 		ppb = -ppb;
 	}
 
-	smp_mb(); /* Force any pending update before accessing. */
-	adj = READ_ONCE(pf->ptp_base_adj);
-
-	freq = adj;
+	freq = I40E_PTP_40GB_INCVAL;
 	freq *= ppb;
 	diff = div_u64(freq, 1000000000ULL);
 
 	if (neg_adj)
-		adj -= diff;
+		adj = I40E_PTP_40GB_INCVAL - diff;
 	else
-		adj += diff;
+		adj = I40E_PTP_40GB_INCVAL + diff;
+
+	/* At some link speeds, the base incval is so large that directly
+	 * multiplying by ppb would result in arithmetic overflow even when
+	 * using a u64. Avoid this by instead calculating the new incval
+	 * always in terms of the 40GbE clock rate and then multiplying by the
+	 * link speed factor afterwards. This does result in slightly lower
+	 * precision at lower link speeds, but it is fairly minor.
+	 */
+	smp_mb(); /* Force any pending update before accessing. */
+	adj *= READ_ONCE(pf->ptp_adj_mult);
 
 	wr32(hw, I40E_PRTTSYN_INC_L, adj & 0xFFFFFFFF);
 	wr32(hw, I40E_PRTTSYN_INC_H, adj >> 32);
@@ -334,10 +317,12 @@ void i40e_ptp_rx_hang(struct i40e_pf *pf)
  * This watchdog task is run periodically to make sure that we clear the Tx
  * timestamp logic if we don't obtain a timestamp in a reasonable amount of
  * time. It is unexpected in the normal case but if it occurs it results in
- * permanently prevent timestamps of future packets
+ * permanently preventing timestamps of future packets.
  **/
 void i40e_ptp_tx_hang(struct i40e_pf *pf)
 {
+	struct sk_buff *skb;
+
 	if (!(pf->flags & I40E_FLAG_PTP) || !pf->ptp_tx)
 		return;
 
@@ -350,9 +335,12 @@ void i40e_ptp_tx_hang(struct i40e_pf *pf)
 	 * within a second it is reasonable to assume that we never will.
 	 */
 	if (time_is_before_jiffies(pf->ptp_tx_start + HZ)) {
-		dev_kfree_skb_any(pf->ptp_tx_skb);
+		skb = pf->ptp_tx_skb;
 		pf->ptp_tx_skb = NULL;
 		clear_bit_unlock(__I40E_PTP_TX_IN_PROGRESS, pf->state);
+
+		/* Free the skb after we clear the bitlock */
+		dev_kfree_skb_any(skb);
 		pf->tx_hwtstamp_timeouts++;
 	}
 }
@@ -462,6 +450,7 @@ void i40e_ptp_set_increment(struct i40e_pf *pf)
 	struct i40e_link_status *hw_link_info;
 	struct i40e_hw *hw = &pf->hw;
 	u64 incval;
+	u32 mult;
 
 	hw_link_info = &hw->phy.link_info;
 
@@ -469,10 +458,10 @@ void i40e_ptp_set_increment(struct i40e_pf *pf)
 
 	switch (hw_link_info->link_speed) {
 	case I40E_LINK_SPEED_10GB:
-		incval = I40E_PTP_10GB_INCVAL;
+		mult = I40E_PTP_10GB_INCVAL_MULT;
 		break;
 	case I40E_LINK_SPEED_1GB:
-		incval = I40E_PTP_1GB_INCVAL;
+		mult = I40E_PTP_1GB_INCVAL_MULT;
 		break;
 	case I40E_LINK_SPEED_100MB:
 	{
@@ -483,15 +472,20 @@ void i40e_ptp_set_increment(struct i40e_pf *pf)
 				 "1588 functionality is not supported at 100 Mbps. Stopping the PHC.\n");
 			warn_once++;
 		}
-		incval = 0;
+		mult = 0;
 		break;
 	}
 	case I40E_LINK_SPEED_40GB:
 	default:
-		incval = I40E_PTP_40GB_INCVAL;
+		mult = 1;
 		break;
 	}
 
+	/* The increment value is calculated by taking the base 40GbE incvalue
+	 * and multiplying it by a factor based on the link speed.
+	 */
+	incval = I40E_PTP_40GB_INCVAL * mult;
+
 	/* Write the new increment value into the increment register. The
 	 * hardware will not update the clock until both registers have been
 	 * written.
@@ -500,14 +494,14 @@ void i40e_ptp_set_increment(struct i40e_pf *pf)
 	wr32(hw, I40E_PRTTSYN_INC_H, incval >> 32);
 
 	/* Update the base adjustement value. */
-	WRITE_ONCE(pf->ptp_base_adj, incval);
+	WRITE_ONCE(pf->ptp_adj_mult, mult);
 	smp_mb(); /* Force the above update. */
 }
 
 /**
  * i40e_ptp_get_ts_config - ioctl interface to read the HW timestamping
  * @pf: Board private structure
- * @ifreq: ioctl data
+ * @ifr: ioctl data
  *
  * Obtain the current hardware timestamping settigs as requested. To do this,
  * keep a shadow copy of the timestamp settings rather than attempting to
@@ -651,7 +645,7 @@ static int i40e_ptp_set_timestamp_mode(struct i40e_pf *pf,
 /**
  * i40e_ptp_set_ts_config - ioctl interface to control the HW timestamping
  * @pf: Board private structure
- * @ifreq: ioctl data
+ * @ifr: ioctl data
  *
  * Respond to the user filter requests and make the appropriate hardware
  * changes here. The XL710 cannot support splitting of the Tx/Rx timestamping
@@ -805,9 +799,11 @@ void i40e_ptp_stop(struct i40e_pf *pf)
 	pf->ptp_rx = false;
 
 	if (pf->ptp_tx_skb) {
-		dev_kfree_skb_any(pf->ptp_tx_skb);
+		struct sk_buff *skb = pf->ptp_tx_skb;
+
 		pf->ptp_tx_skb = NULL;
 		clear_bit_unlock(__I40E_PTP_TX_IN_PROGRESS, pf->state);
+		dev_kfree_skb_any(skb);
 	}
 
 	if (pf->ptp_clock) {
diff --git a/drivers/net/ethernet/intel/i40e/i40e_register.h b/drivers/net/ethernet/intel/i40e/i40e_register.h
index b3e206e49cc2..52e3680c57f8 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_register.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_register.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_REGISTER_H_
 #define _I40E_REGISTER_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_status.h b/drivers/net/ethernet/intel/i40e/i40e_status.h
index 10c86f63dc52..77be0702d07c 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_status.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_status.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_STATUS_H_
 #define _I40E_STATUS_H_
diff --git a/drivers/net/ethernet/intel/i40e/i40e_trace.h b/drivers/net/ethernet/intel/i40e/i40e_trace.h
index 410ba13bcf21..424f02077e2e 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_trace.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_trace.h
@@ -1,26 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel(R) 40-10 Gigabit Ethernet Connection Network Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 /* Modeled on trace-events-sample.h */
 
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index f174c72480ab..9b698c5acd05 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include <linux/prefetch.h>
 #include <net/busy_poll.h>
@@ -495,7 +471,7 @@ static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi,
 /**
  * i40e_add_del_fdir - Build raw packets to add/del fdir filter
  * @vsi: pointer to the targeted VSI
- * @cmd: command to get or set RX flow classification rules
+ * @input: filter to add or delete
  * @add: true adds a filter, false removes it
  *
  **/
@@ -638,7 +614,7 @@ static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
 		if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
 			kfree(tx_buffer->raw_buf);
 		else if (ring_is_xdp(ring))
-			page_frag_free(tx_buffer->raw_buf);
+			xdp_return_frame(tx_buffer->xdpf);
 		else
 			dev_kfree_skb_any(tx_buffer->skb);
 		if (dma_unmap_len(tx_buffer, len))
@@ -713,7 +689,7 @@ void i40e_free_tx_resources(struct i40e_ring *tx_ring)
 
 /**
  * i40e_get_tx_pending - how many tx descriptors not processed
- * @tx_ring: the ring of descriptors
+ * @ring: the ring of descriptors
  * @in_sw: use SW variables
  *
  * Since there is no access to the ring head register
@@ -841,7 +817,7 @@ static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
 
 		/* free the skb/XDP data */
 		if (ring_is_xdp(tx_ring))
-			page_frag_free(tx_buf->raw_buf);
+			xdp_return_frame(tx_buf->xdpf);
 		else
 			napi_consume_skb(tx_buf->skb, napi_budget);
 
@@ -1795,6 +1771,8 @@ static inline int i40e_ptype_to_htype(u8 ptype)
  * i40e_rx_hash - set the hash value in the skb
  * @ring: descriptor ring
  * @rx_desc: specific descriptor
+ * @skb: skb currently being received and modified
+ * @rx_ptype: Rx packet type
  **/
 static inline void i40e_rx_hash(struct i40e_ring *ring,
 				union i40e_rx_desc *rx_desc,
@@ -2203,9 +2181,20 @@ static bool i40e_is_non_eop(struct i40e_ring *rx_ring,
 #define I40E_XDP_CONSUMED 1
 #define I40E_XDP_TX 2
 
-static int i40e_xmit_xdp_ring(struct xdp_buff *xdp,
+static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf,
 			      struct i40e_ring *xdp_ring);
 
+static int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp,
+				 struct i40e_ring *xdp_ring)
+{
+	struct xdp_frame *xdpf = convert_to_xdp_frame(xdp);
+
+	if (unlikely(!xdpf))
+		return I40E_XDP_CONSUMED;
+
+	return i40e_xmit_xdp_ring(xdpf, xdp_ring);
+}
+
 /**
  * i40e_run_xdp - run an XDP program
  * @rx_ring: Rx ring being processed
@@ -2225,13 +2214,15 @@ static struct sk_buff *i40e_run_xdp(struct i40e_ring *rx_ring,
 	if (!xdp_prog)
 		goto xdp_out;
 
+	prefetchw(xdp->data_hard_start); /* xdp_frame write */
+
 	act = bpf_prog_run_xdp(xdp_prog, xdp);
 	switch (act) {
 	case XDP_PASS:
 		break;
 	case XDP_TX:
 		xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index];
-		result = i40e_xmit_xdp_ring(xdp, xdp_ring);
+		result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring);
 		break;
 	case XDP_REDIRECT:
 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
@@ -3478,13 +3469,13 @@ static inline int i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
  * @xdp: data to transmit
  * @xdp_ring: XDP Tx ring
  **/
-static int i40e_xmit_xdp_ring(struct xdp_buff *xdp,
+static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf,
 			      struct i40e_ring *xdp_ring)
 {
-	u32 size = xdp->data_end - xdp->data;
 	u16 i = xdp_ring->next_to_use;
 	struct i40e_tx_buffer *tx_bi;
 	struct i40e_tx_desc *tx_desc;
+	u32 size = xdpf->len;
 	dma_addr_t dma;
 
 	if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) {
@@ -3492,14 +3483,14 @@ static int i40e_xmit_xdp_ring(struct xdp_buff *xdp,
 		return I40E_XDP_CONSUMED;
 	}
 
-	dma = dma_map_single(xdp_ring->dev, xdp->data, size, DMA_TO_DEVICE);
+	dma = dma_map_single(xdp_ring->dev, xdpf->data, size, DMA_TO_DEVICE);
 	if (dma_mapping_error(xdp_ring->dev, dma))
 		return I40E_XDP_CONSUMED;
 
 	tx_bi = &xdp_ring->tx_bi[i];
 	tx_bi->bytecount = size;
 	tx_bi->gso_segs = 1;
-	tx_bi->raw_buf = xdp->data;
+	tx_bi->xdpf = xdpf;
 
 	/* record length, and DMA address */
 	dma_unmap_len_set(tx_bi, len, size);
@@ -3673,14 +3664,19 @@ netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
  * @dev: netdev
  * @xdp: XDP buffer
  *
- * Returns Zero if sent, else an error code
+ * Returns number of frames successfully sent. Frames that fail are
+ * free'ed via XDP return API.
+ *
+ * For error cases, a negative errno code is returned and no-frames
+ * are transmitted (caller must handle freeing frames).
  **/
-int i40e_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
+int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames)
 {
 	struct i40e_netdev_priv *np = netdev_priv(dev);
 	unsigned int queue_index = smp_processor_id();
 	struct i40e_vsi *vsi = np->vsi;
-	int err;
+	int drops = 0;
+	int i;
 
 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
 		return -ENETDOWN;
@@ -3688,11 +3684,18 @@ int i40e_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
 	if (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs)
 		return -ENXIO;
 
-	err = i40e_xmit_xdp_ring(xdp, vsi->xdp_rings[queue_index]);
-	if (err != I40E_XDP_TX)
-		return -ENOSPC;
+	for (i = 0; i < n; i++) {
+		struct xdp_frame *xdpf = frames[i];
+		int err;
 
-	return 0;
+		err = i40e_xmit_xdp_ring(xdpf, vsi->xdp_rings[queue_index]);
+		if (err != I40E_XDP_TX) {
+			xdp_return_frame_rx_napi(xdpf);
+			drops++;
+		}
+	}
+
+	return n - drops;
 }
 
 /**
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
index 3043483ec426..eb8804b3d7b6 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_TXRX_H_
 #define _I40E_TXRX_H_
@@ -306,6 +282,7 @@ static inline unsigned int i40e_txd_use_count(unsigned int size)
 struct i40e_tx_buffer {
 	struct i40e_tx_desc *next_to_watch;
 	union {
+		struct xdp_frame *xdpf;
 		struct sk_buff *skb;
 		void *raw_buf;
 	};
@@ -510,7 +487,7 @@ u32 i40e_get_tx_pending(struct i40e_ring *ring, bool in_sw);
 void i40e_detect_recover_hung(struct i40e_vsi *vsi);
 int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size);
 bool __i40e_chk_linearize(struct sk_buff *skb);
-int i40e_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp);
+int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames);
 void i40e_xdp_flush(struct net_device *dev);
 
 /**
diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h
index bfb80092b352..7df969c59855 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_type.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_type.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_TYPE_H_
 #define _I40E_TYPE_H_
@@ -1318,7 +1294,8 @@ struct i40e_hw_port_stats {
 
 /* Checksum and Shadow RAM pointers */
 #define I40E_SR_NVM_CONTROL_WORD		0x00
-#define I40E_SR_EMP_MODULE_PTR			0x0F
+#define I40E_EMP_MODULE_PTR			0x0F
+#define I40E_SR_EMP_MODULE_PTR			0x48
 #define I40E_SR_PBA_FLAGS			0x15
 #define I40E_SR_PBA_BLOCK_PTR			0x16
 #define I40E_SR_BOOT_CONFIG_PTR			0x17
@@ -1337,6 +1314,8 @@ struct i40e_hw_port_stats {
 #define I40E_SR_PCIE_ALT_MODULE_MAX_SIZE	1024
 #define I40E_SR_CONTROL_WORD_1_SHIFT		0x06
 #define I40E_SR_CONTROL_WORD_1_MASK	(0x03 << I40E_SR_CONTROL_WORD_1_SHIFT)
+#define I40E_SR_CONTROL_WORD_1_NVM_BANK_VALID	BIT(5)
+#define I40E_SR_NVM_MAP_STRUCTURE_TYPE		BIT(12)
 #define I40E_PTR_TYPE				BIT(15)
 #define I40E_SR_OCP_CFG_WORD0			0x2B
 #define I40E_SR_OCP_ENABLED			BIT(15)
@@ -1454,7 +1433,8 @@ enum i40e_reset_type {
 };
 
 /* IEEE 802.1AB LLDP Agent Variables from NVM */
-#define I40E_NVM_LLDP_CFG_PTR		0xD
+#define I40E_NVM_LLDP_CFG_PTR	0x06
+#define I40E_SR_LLDP_CFG_PTR	0x31
 struct i40e_lldp_variables {
 	u16 length;
 	u16 adminstatus;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
index 35173cbe80f7..c6d24eaede18 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40e.h"
 
@@ -32,8 +8,8 @@
 /**
  * i40e_vc_vf_broadcast
  * @pf: pointer to the PF structure
- * @opcode: operation code
- * @retval: return value
+ * @v_opcode: operation code
+ * @v_retval: return value
  * @msg: pointer to the msg buffer
  * @msglen: msg length
  *
@@ -1663,6 +1639,7 @@ static int i40e_vc_send_resp_to_vf(struct i40e_vf *vf,
 /**
  * i40e_vc_get_version_msg
  * @vf: pointer to the VF info
+ * @msg: pointer to the msg buffer
  *
  * called from the VF to request the API version used by the PF
  **/
@@ -1706,7 +1683,6 @@ static void i40e_del_qch(struct i40e_vf *vf)
  * i40e_vc_get_vf_resources_msg
  * @vf: pointer to the VF info
  * @msg: pointer to the msg buffer
- * @msglen: msg length
  *
  * called from the VF to request its resources
  **/
@@ -1830,8 +1806,6 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg)
 /**
  * i40e_vc_reset_vf_msg
  * @vf: pointer to the VF info
- * @msg: pointer to the msg buffer
- * @msglen: msg length
  *
  * called from the VF to reset itself,
  * unlike other virtchnl messages, PF driver
@@ -2179,6 +2153,51 @@ static int i40e_vc_config_irq_map_msg(struct i40e_vf *vf, u8 *msg, u16 msglen)
 				       aq_ret);
 }
 
+/**
+ * i40e_ctrl_vf_tx_rings
+ * @vsi: the SRIOV VSI being configured
+ * @q_map: bit map of the queues to be enabled
+ * @enable: start or stop the queue
+ **/
+static int i40e_ctrl_vf_tx_rings(struct i40e_vsi *vsi, unsigned long q_map,
+				 bool enable)
+{
+	struct i40e_pf *pf = vsi->back;
+	int ret = 0;
+	u16 q_id;
+
+	for_each_set_bit(q_id, &q_map, I40E_MAX_VF_QUEUES) {
+		ret = i40e_control_wait_tx_q(vsi->seid, pf,
+					     vsi->base_queue + q_id,
+					     false /*is xdp*/, enable);
+		if (ret)
+			break;
+	}
+	return ret;
+}
+
+/**
+ * i40e_ctrl_vf_rx_rings
+ * @vsi: the SRIOV VSI being configured
+ * @q_map: bit map of the queues to be enabled
+ * @enable: start or stop the queue
+ **/
+static int i40e_ctrl_vf_rx_rings(struct i40e_vsi *vsi, unsigned long q_map,
+				 bool enable)
+{
+	struct i40e_pf *pf = vsi->back;
+	int ret = 0;
+	u16 q_id;
+
+	for_each_set_bit(q_id, &q_map, I40E_MAX_VF_QUEUES) {
+		ret = i40e_control_wait_rx_q(pf, vsi->base_queue + q_id,
+					     enable);
+		if (ret)
+			break;
+	}
+	return ret;
+}
+
 /**
  * i40e_vc_enable_queues_msg
  * @vf: pointer to the VF info
@@ -2211,8 +2230,17 @@ static int i40e_vc_enable_queues_msg(struct i40e_vf *vf, u8 *msg, u16 msglen)
 		goto error_param;
 	}
 
-	if (i40e_vsi_start_rings(pf->vsi[vf->lan_vsi_idx]))
+	/* Use the queue bit map sent by the VF */
+	if (i40e_ctrl_vf_rx_rings(pf->vsi[vf->lan_vsi_idx], vqs->rx_queues,
+				  true)) {
 		aq_ret = I40E_ERR_TIMEOUT;
+		goto error_param;
+	}
+	if (i40e_ctrl_vf_tx_rings(pf->vsi[vf->lan_vsi_idx], vqs->tx_queues,
+				  true)) {
+		aq_ret = I40E_ERR_TIMEOUT;
+		goto error_param;
+	}
 
 	/* need to start the rings for additional ADq VSI's as well */
 	if (vf->adq_enabled) {
@@ -2260,8 +2288,17 @@ static int i40e_vc_disable_queues_msg(struct i40e_vf *vf, u8 *msg, u16 msglen)
 		goto error_param;
 	}
 
-	i40e_vsi_stop_rings(pf->vsi[vf->lan_vsi_idx]);
-
+	/* Use the queue bit map sent by the VF */
+	if (i40e_ctrl_vf_tx_rings(pf->vsi[vf->lan_vsi_idx], vqs->tx_queues,
+				  false)) {
+		aq_ret = I40E_ERR_TIMEOUT;
+		goto error_param;
+	}
+	if (i40e_ctrl_vf_rx_rings(pf->vsi[vf->lan_vsi_idx], vqs->rx_queues,
+				  false)) {
+		aq_ret = I40E_ERR_TIMEOUT;
+		goto error_param;
+	}
 error_param:
 	/* send the response to the VF */
 	return i40e_vc_send_resp_to_vf(vf, VIRTCHNL_OP_DISABLE_QUEUES,
@@ -3556,15 +3593,16 @@ static int i40e_vc_del_qch_msg(struct i40e_vf *vf, u8 *msg)
  * i40e_vc_process_vf_msg
  * @pf: pointer to the PF structure
  * @vf_id: source VF id
+ * @v_opcode: operation code
+ * @v_retval: unused return value code
  * @msg: pointer to the msg buffer
  * @msglen: msg length
- * @msghndl: msg handle
  *
  * called from the common aeq/arq handler to
  * process request from VF
  **/
 int i40e_vc_process_vf_msg(struct i40e_pf *pf, s16 vf_id, u32 v_opcode,
-			   u32 v_retval, u8 *msg, u16 msglen)
+			   u32 __always_unused v_retval, u8 *msg, u16 msglen)
 {
 	struct i40e_hw *hw = &pf->hw;
 	int local_vf_id = vf_id - (s16)hw->func_caps.vf_base_id;
@@ -4015,7 +4053,8 @@ int i40e_ndo_set_vf_port_vlan(struct net_device *netdev, int vf_id,
  * i40e_ndo_set_vf_bw
  * @netdev: network interface device structure
  * @vf_id: VF identifier
- * @tx_rate: Tx rate
+ * @min_tx_rate: Minimum Tx rate
+ * @max_tx_rate: Maximum Tx rate
  *
  * configure VF Tx rate
  **/
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h
index 57f727bb9e36..bf67d62e2b5f 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Driver
- * Copyright(c) 2013 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_VIRTCHNL_PF_H_
 #define _I40E_VIRTCHNL_PF_H_
diff --git a/drivers/net/ethernet/intel/i40evf/Makefile b/drivers/net/ethernet/intel/i40evf/Makefile
index 1e89c5487676..3c5c6e962280 100644
--- a/drivers/net/ethernet/intel/i40evf/Makefile
+++ b/drivers/net/ethernet/intel/i40evf/Makefile
@@ -1,29 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-################################################################################
-#
-# Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
-# Copyright(c) 2013 - 2014 Intel Corporation.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms and conditions of the GNU General Public License,
-# version 2, as published by the Free Software Foundation.
-#
-# This program is distributed in the hope it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# The full GNU General Public License is included in this distribution in
-# the file called "COPYING".
-#
-# Contact Information:
-# e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-# Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-#
-################################################################################
+# Copyright(c) 2013 - 2018 Intel Corporation.
 
 #
 ## Makefile for the Intel(R) 40GbE VF driver
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_adminq.c b/drivers/net/ethernet/intel/i40evf/i40e_adminq.c
index 6fd677efa9da..c355120dfdfd 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_adminq.c
+++ b/drivers/net/ethernet/intel/i40evf/i40e_adminq.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40e_status.h"
 #include "i40e_type.h"
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_adminq.h b/drivers/net/ethernet/intel/i40evf/i40e_adminq.h
index a7137c165256..1f264b9b6805 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_adminq.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_adminq.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_ADMINQ_H_
 #define _I40E_ADMINQ_H_
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h b/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h
index 439e71882049..aa81e87cd471 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_ADMINQ_CMD_H_
 #define _I40E_ADMINQ_CMD_H_
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_alloc.h b/drivers/net/ethernet/intel/i40evf/i40e_alloc.h
index 7e0fddd8af36..cb8689222c8b 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_alloc.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_alloc.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_ALLOC_H_
 #define _I40E_ALLOC_H_
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_common.c b/drivers/net/ethernet/intel/i40evf/i40e_common.c
index 67140cdbcd7a..9cef54971312 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_common.c
+++ b/drivers/net/ethernet/intel/i40evf/i40e_common.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40e_type.h"
 #include "i40e_adminq.h"
@@ -1255,6 +1231,7 @@ i40e_status_code i40evf_aq_write_ddp(struct i40e_hw *hw, void *buff,
  * @hw: pointer to the hw struct
  * @buff: command buffer (size in bytes = buff_size)
  * @buff_size: buffer size in bytes
+ * @flags: AdminQ command flags
  * @cmd_details: pointer to command details structure or NULL
  **/
 enum
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_devids.h b/drivers/net/ethernet/intel/i40evf/i40e_devids.h
index 352dd3f3eb6a..f300bf271824 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_devids.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_devids.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_DEVIDS_H_
 #define _I40E_DEVIDS_H_
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_hmc.h b/drivers/net/ethernet/intel/i40evf/i40e_hmc.h
index 7432596164f4..1c78de838857 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_hmc.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_hmc.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_HMC_H_
 #define _I40E_HMC_H_
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_lan_hmc.h b/drivers/net/ethernet/intel/i40evf/i40e_lan_hmc.h
index ddac0e4908d3..82b00f70a632 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_lan_hmc.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_lan_hmc.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_LAN_HMC_H_
 #define _I40E_LAN_HMC_H_
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_osdep.h b/drivers/net/ethernet/intel/i40evf/i40e_osdep.h
index 8668ad6c1a65..3ddddb46455b 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_osdep.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_osdep.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_OSDEP_H_
 #define _I40E_OSDEP_H_
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_prototype.h b/drivers/net/ethernet/intel/i40evf/i40e_prototype.h
index 72501bd0f1a9..a358f4b9d5aa 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_prototype.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_prototype.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_PROTOTYPE_H_
 #define _I40E_PROTOTYPE_H_
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_register.h b/drivers/net/ethernet/intel/i40evf/i40e_register.h
index c9c935659758..49e1f57d99cc 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_register.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_register.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_REGISTER_H_
 #define _I40E_REGISTER_H_
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_status.h b/drivers/net/ethernet/intel/i40evf/i40e_status.h
index 0d7993ecb99a..77be0702d07c 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_status.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_status.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_STATUS_H_
 #define _I40E_STATUS_H_
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_trace.h b/drivers/net/ethernet/intel/i40evf/i40e_trace.h
index ece01dd12a3c..d7a4e68820a8 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_trace.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_trace.h
@@ -1,26 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel(R) 40-10 Gigabit Ethernet Virtual Function Driver
- * Copyright(c) 2013 - 2017 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 /* Modeled on trace-events-sample.h */
 
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_txrx.c b/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
index 12bd937861e7..a9730711e257 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include <linux/prefetch.h>
 #include <net/busy_poll.h>
@@ -129,7 +105,7 @@ void i40evf_free_tx_resources(struct i40e_ring *tx_ring)
 
 /**
  * i40evf_get_tx_pending - how many Tx descriptors not processed
- * @tx_ring: the ring of descriptors
+ * @ring: the ring of descriptors
  * @in_sw: is tx_pending being checked in SW or HW
  *
  * Since there is no access to the ring head register
@@ -1070,6 +1046,8 @@ static inline int i40e_ptype_to_htype(u8 ptype)
  * i40e_rx_hash - set the hash value in the skb
  * @ring: descriptor ring
  * @rx_desc: specific descriptor
+ * @skb: skb currently being received and modified
+ * @rx_ptype: Rx packet type
  **/
 static inline void i40e_rx_hash(struct i40e_ring *ring,
 				union i40e_rx_desc *rx_desc,
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_txrx.h b/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
index 5790897eae2e..3b5a63b3236e 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_TXRX_H_
 #define _I40E_TXRX_H_
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_type.h b/drivers/net/ethernet/intel/i40evf/i40e_type.h
index 449de4b0058e..094387db3c11 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_type.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_type.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40E_TYPE_H_
 #define _I40E_TYPE_H_
@@ -1257,7 +1233,8 @@ struct i40e_hw_port_stats {
 
 /* Checksum and Shadow RAM pointers */
 #define I40E_SR_NVM_CONTROL_WORD		0x00
-#define I40E_SR_EMP_MODULE_PTR			0x0F
+#define I40E_EMP_MODULE_PTR			0x0F
+#define I40E_SR_EMP_MODULE_PTR			0x48
 #define I40E_NVM_OEM_VER_OFF			0x83
 #define I40E_SR_NVM_DEV_STARTER_VERSION		0x18
 #define I40E_SR_NVM_WAKE_ON_LAN			0x19
@@ -1273,6 +1250,9 @@ struct i40e_hw_port_stats {
 #define I40E_SR_PCIE_ALT_MODULE_MAX_SIZE	1024
 #define I40E_SR_CONTROL_WORD_1_SHIFT		0x06
 #define I40E_SR_CONTROL_WORD_1_MASK	(0x03 << I40E_SR_CONTROL_WORD_1_SHIFT)
+#define I40E_SR_CONTROL_WORD_1_NVM_BANK_VALID	BIT(5)
+#define I40E_SR_NVM_MAP_STRUCTURE_TYPE		BIT(12)
+#define I40E_PTR_TYPE				BIT(15)
 
 /* Shadow RAM related */
 #define I40E_SR_SECTOR_SIZE_IN_WORDS	0x800
@@ -1386,6 +1366,10 @@ enum i40e_reset_type {
 	I40E_RESET_EMPR		= 3,
 };
 
+/* IEEE 802.1AB LLDP Agent Variables from NVM */
+#define I40E_NVM_LLDP_CFG_PTR	0x06
+#define I40E_SR_LLDP_CFG_PTR	0x31
+
 /* RSS Hash Table Size */
 #define I40E_PFQF_CTL_0_HASHLUTSIZE_512	0x00010000
 
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf.h b/drivers/net/ethernet/intel/i40evf/i40evf.h
index 3a7a1e77bf39..96e537a35000 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf.h
+++ b/drivers/net/ethernet/intel/i40evf/i40evf.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #ifndef _I40EVF_H_
 #define _I40EVF_H_
@@ -105,7 +81,6 @@ struct i40e_vsi {
 #define I40E_TX_DESC(R, i) (&(((struct i40e_tx_desc *)((R)->desc))[i]))
 #define I40E_TX_CTXTDESC(R, i) \
 	(&(((struct i40e_tx_context_desc *)((R)->desc))[i]))
-#define MAX_QUEUES 16
 #define I40EVF_MAX_REQ_QUEUES 4
 
 #define I40EVF_HKEY_ARRAY_SIZE ((I40E_VFQF_HKEY_MAX_INDEX + 1) * 4)
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_client.c b/drivers/net/ethernet/intel/i40evf/i40evf_client.c
index da60ce12b33d..3cc9d60d0d72 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_client.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_client.c
@@ -1,4 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
 #include <linux/list.h>
 #include <linux/errno.h>
 
@@ -176,7 +178,6 @@ void i40evf_notify_client_close(struct i40e_vsi *vsi, bool reset)
 /**
  * i40evf_client_add_instance - add a client instance to the instance list
  * @adapter: pointer to the board struct
- * @client: pointer to a client struct in the client list.
  *
  * Returns cinst ptr on success, NULL on failure
  **/
@@ -234,7 +235,6 @@ i40evf_client_add_instance(struct i40evf_adapter *adapter)
 /**
  * i40evf_client_del_instance - removes a client instance from the list
  * @adapter: pointer to the board struct
- * @client: pointer to the client struct
  *
  **/
 static
@@ -438,7 +438,7 @@ static u32 i40evf_client_virtchnl_send(struct i40e_info *ldev,
  * i40evf_client_setup_qvlist - send a message to the PF to setup iwarp qv map
  * @ldev: pointer to L2 context.
  * @client: Client pointer.
- * @qv_info: queue and vector list
+ * @qvlist_info: queue and vector list
  *
  * Return 0 on success or < 0 on error
  **/
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_client.h b/drivers/net/ethernet/intel/i40evf/i40evf_client.h
index 15a10da5bd4a..5585f362048a 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_client.h
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_client.h
@@ -1,6 +1,8 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _I40E_CLIENT_H_
-#define _I40E_CLIENT_H_
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _I40EVF_CLIENT_H_
+#define _I40EVF_CLIENT_H_
 
 #define I40EVF_CLIENT_STR_LENGTH 10
 
@@ -164,4 +166,4 @@ struct i40e_client {
 /* used by clients */
 int i40evf_register_client(struct i40e_client *client);
 int i40evf_unregister_client(struct i40e_client *client);
-#endif /* _I40E_CLIENT_H_ */
+#endif /* _I40EVF_CLIENT_H_ */
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c b/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
index dc4cde274fb8..69efe0aec76a 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 /* ethtool support for i40evf */
 #include "i40evf.h"
@@ -226,7 +202,7 @@ static void i40evf_get_strings(struct net_device *netdev, u32 sset, u8 *data)
 
 /**
  * i40evf_get_priv_flags - report device private flags
- * @dev: network interface device structure
+ * @netdev: network interface device structure
  *
  * The get string set count and the string set should be matched for each
  * flag returned.  Add new strings for each flag to the i40e_gstrings_priv_flags
@@ -253,7 +229,7 @@ static u32 i40evf_get_priv_flags(struct net_device *netdev)
 
 /**
  * i40evf_set_priv_flags - set private flags
- * @dev: network interface device structure
+ * @netdev: network interface device structure
  * @flags: bit flags to be set
  **/
 static int i40evf_set_priv_flags(struct net_device *netdev, u32 flags)
@@ -627,6 +603,7 @@ static int i40evf_set_per_queue_coalesce(struct net_device *netdev,
  * i40evf_get_rxnfc - command to get RX flow classification rules
  * @netdev: network interface device structure
  * @cmd: ethtool rxnfc command
+ * @rule_locs: pointer to store rule locations
  *
  * Returns Success if the command is supported.
  **/
@@ -746,6 +723,7 @@ static u32 i40evf_get_rxfh_indir_size(struct net_device *netdev)
  * @netdev: network interface device structure
  * @indir: indirection table
  * @key: hash key
+ * @hfunc: hash function in use
  *
  * Reads the indirection table directly from the hardware. Always returns 0.
  **/
@@ -774,6 +752,7 @@ static int i40evf_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key,
  * @netdev: network interface device structure
  * @indir: indirection table
  * @key: hash key
+ * @hfunc: hash function to use
  *
  * Returns -EINVAL if the table specifies an inavlid queue id, otherwise
  * returns 0 after programming the table.
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_main.c b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
index 5f71532be7f1..a7b87f935411 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_main.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40evf.h"
 #include "i40e_prototype.h"
@@ -473,6 +449,7 @@ static void i40evf_irq_affinity_release(struct kref *ref) {}
 /**
  * i40evf_request_traffic_irqs - Initialize MSI-X interrupts
  * @adapter: board private structure
+ * @basename: device basename
  *
  * Allocates MSI-X vectors for tx and rx handling, and requests
  * interrupts from the kernel.
@@ -705,7 +682,7 @@ i40evf_vlan_filter *i40evf_add_vlan(struct i40evf_adapter *adapter, u16 vlan)
 
 	f = i40evf_find_vlan(adapter, vlan);
 	if (!f) {
-		f = kzalloc(sizeof(*f), GFP_ATOMIC);
+		f = kzalloc(sizeof(*f), GFP_KERNEL);
 		if (!f)
 			goto clearout;
 
@@ -745,6 +722,7 @@ static void i40evf_del_vlan(struct i40evf_adapter *adapter, u16 vlan)
 /**
  * i40evf_vlan_rx_add_vid - Add a VLAN filter to a device
  * @netdev: network device struct
+ * @proto: unused protocol data
  * @vid: VLAN tag
  **/
 static int i40evf_vlan_rx_add_vid(struct net_device *netdev,
@@ -762,6 +740,7 @@ static int i40evf_vlan_rx_add_vid(struct net_device *netdev,
 /**
  * i40evf_vlan_rx_kill_vid - Remove a VLAN filter from a device
  * @netdev: network device struct
+ * @proto: unused protocol data
  * @vid: VLAN tag
  **/
 static int i40evf_vlan_rx_kill_vid(struct net_device *netdev,
@@ -1946,7 +1925,8 @@ static void i40evf_reset_task(struct work_struct *work)
 	 * ndo_open() returning, so we can't assume it means all our open
 	 * tasks have finished, since we're not holding the rtnl_lock here.
 	 */
-	running = (adapter->state == __I40EVF_RUNNING);
+	running = ((adapter->state == __I40EVF_RUNNING) ||
+		   (adapter->state == __I40EVF_RESETTING));
 
 	if (running) {
 		netif_carrier_off(netdev);
@@ -2352,7 +2332,7 @@ static int i40evf_validate_ch_config(struct i40evf_adapter *adapter,
 		total_max_rate += tx_rate;
 		num_qps += mqprio_qopt->qopt.count[i];
 	}
-	if (num_qps > MAX_QUEUES)
+	if (num_qps > I40EVF_MAX_REQ_QUEUES)
 		return -EINVAL;
 
 	ret = i40evf_validate_tx_bandwidth(adapter, total_max_rate);
@@ -3160,7 +3140,7 @@ static int i40evf_set_features(struct net_device *netdev,
 /**
  * i40evf_features_check - Validate encapsulated packet conforms to limits
  * @skb: skb buff
- * @netdev: This physical port's netdev
+ * @dev: This physical port's netdev
  * @features: Offload features that the stack believes apply
  **/
 static netdev_features_t i40evf_features_check(struct sk_buff *skb,
@@ -3378,6 +3358,24 @@ int i40evf_process_config(struct i40evf_adapter *adapter)
 	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
 		netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
 
+	/* Do not turn on offloads when they are requested to be turned off.
+	 * TSO needs minimum 576 bytes to work correctly.
+	 */
+	if (netdev->wanted_features) {
+		if (!(netdev->wanted_features & NETIF_F_TSO) ||
+		    netdev->mtu < 576)
+			netdev->features &= ~NETIF_F_TSO;
+		if (!(netdev->wanted_features & NETIF_F_TSO6) ||
+		    netdev->mtu < 576)
+			netdev->features &= ~NETIF_F_TSO6;
+		if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
+			netdev->features &= ~NETIF_F_TSO_ECN;
+		if (!(netdev->wanted_features & NETIF_F_GRO))
+			netdev->features &= ~NETIF_F_GRO;
+		if (!(netdev->wanted_features & NETIF_F_GSO))
+			netdev->features &= ~NETIF_F_GSO;
+	}
+
 	adapter->vsi.id = adapter->vsi_res->vsi_id;
 
 	adapter->vsi.back = adapter;
@@ -3692,7 +3690,8 @@ static int i40evf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	pci_set_master(pdev);
 
-	netdev = alloc_etherdev_mq(sizeof(struct i40evf_adapter), MAX_QUEUES);
+	netdev = alloc_etherdev_mq(sizeof(struct i40evf_adapter),
+				   I40EVF_MAX_REQ_QUEUES);
 	if (!netdev) {
 		err = -ENOMEM;
 		goto err_alloc_etherdev;
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
index 26a59890532f..565677de5ba3 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
- *
- * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
- * Copyright(c) 2013 - 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
 
 #include "i40evf.h"
 #include "i40e_prototype.h"
@@ -179,8 +155,7 @@ int i40evf_send_vf_config_msg(struct i40evf_adapter *adapter)
 
 /**
  * i40evf_get_vf_config
- * @hw: pointer to the hardware structure
- * @len: length of buffer
+ * @adapter: private adapter structure
  *
  * Get VF configuration from PF and populate hw structure. Must be called after
  * admin queue is initialized. Busy waits until response is received from PF,
@@ -423,8 +398,6 @@ int i40evf_request_queues(struct i40evf_adapter *adapter, int num)
 /**
  * i40evf_add_ether_addrs
  * @adapter: adapter structure
- * @addrs: the MAC address filters to add (contiguous)
- * @count: number of filters
  *
  * Request that the PF add one or more addresses to our filters.
  **/
@@ -497,8 +470,6 @@ void i40evf_add_ether_addrs(struct i40evf_adapter *adapter)
 /**
  * i40evf_del_ether_addrs
  * @adapter: adapter structure
- * @addrs: the MAC address filters to remove (contiguous)
- * @count: number of filtes
  *
  * Request that the PF remove one or more addresses from our filters.
  **/
@@ -571,8 +542,6 @@ void i40evf_del_ether_addrs(struct i40evf_adapter *adapter)
 /**
  * i40evf_add_vlans
  * @adapter: adapter structure
- * @vlans: the VLANs to add
- * @count: number of VLANs
  *
  * Request that the PF add one or more VLAN filters to our VSI.
  **/
@@ -643,8 +612,6 @@ void i40evf_add_vlans(struct i40evf_adapter *adapter)
 /**
  * i40evf_del_vlans
  * @adapter: adapter structure
- * @vlans: the VLANs to remove
- * @count: number of VLANs
  *
  * Request that the PF remove one or more VLAN filters from our VSI.
  **/
diff --git a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h
index 7dc5f045e969..7541ec2270b3 100644
--- a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h
+++ b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h
@@ -1049,7 +1049,9 @@ struct ice_aqc_set_event_mask {
  * NVM Update commands (indirect 0x0703)
  */
 struct ice_aqc_nvm {
-	u8	cmd_flags;
+	__le16 offset_low;
+	u8 offset_high;
+	u8 cmd_flags;
 #define ICE_AQC_NVM_LAST_CMD		BIT(0)
 #define ICE_AQC_NVM_PCIR_REQ		BIT(0)	/* Used by NVM Update reply */
 #define ICE_AQC_NVM_PRESERVATION_S	1
@@ -1058,12 +1060,11 @@ struct ice_aqc_nvm {
 #define ICE_AQC_NVM_PRESERVE_ALL	BIT(1)
 #define ICE_AQC_NVM_PRESERVE_SELECTED	(3 << CSR_AQ_NVM_PRESERVATION_S)
 #define ICE_AQC_NVM_FLASH_ONLY		BIT(7)
-	u8	module_typeid;
-	__le16	length;
+	__le16 module_typeid;
+	__le16 length;
 #define ICE_AQC_NVM_ERASE_LEN	0xFFFF
-	__le32	offset;
-	__le32	addr_high;
-	__le32	addr_low;
+	__le32 addr_high;
+	__le32 addr_low;
 };
 
 /* Get/Set RSS key (indirect 0x0B04/0x0B02) */
diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.c b/drivers/net/ethernet/intel/ice/ice_nvm.c
index fa7a69ac92b0..92da0a626ce0 100644
--- a/drivers/net/ethernet/intel/ice/ice_nvm.c
+++ b/drivers/net/ethernet/intel/ice/ice_nvm.c
@@ -16,7 +16,7 @@
  * Read the NVM using the admin queue commands (0x0701)
  */
 static enum ice_status
-ice_aq_read_nvm(struct ice_hw *hw, u8 module_typeid, u32 offset, u16 length,
+ice_aq_read_nvm(struct ice_hw *hw, u16 module_typeid, u32 offset, u16 length,
 		void *data, bool last_command, struct ice_sq_cd *cd)
 {
 	struct ice_aq_desc desc;
@@ -33,8 +33,9 @@ ice_aq_read_nvm(struct ice_hw *hw, u8 module_typeid, u32 offset, u16 length,
 	/* If this is the last command in a series, set the proper flag. */
 	if (last_command)
 		cmd->cmd_flags |= ICE_AQC_NVM_LAST_CMD;
-	cmd->module_typeid = module_typeid;
-	cmd->offset = cpu_to_le32(offset);
+	cmd->module_typeid = cpu_to_le16(module_typeid);
+	cmd->offset_low = cpu_to_le16(offset & 0xFFFF);
+	cmd->offset_high = (offset >> 16) & 0xFF;
 	cmd->length = cpu_to_le16(length);
 
 	return ice_aq_send_cmd(hw, &desc, data, length, cd);
diff --git a/drivers/net/ethernet/intel/igb/Makefile b/drivers/net/ethernet/intel/igb/Makefile
index c48583e98ac1..394c1e0656b9 100644
--- a/drivers/net/ethernet/intel/igb/Makefile
+++ b/drivers/net/ethernet/intel/igb/Makefile
@@ -1,31 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-################################################################################
-#
-# Intel 82575 PCI-Express Ethernet Linux driver
-# Copyright(c) 1999 - 2014 Intel Corporation.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms and conditions of the GNU General Public License,
-# version 2, as published by the Free Software Foundation.
-#
-# This program is distributed in the hope it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, see <http://www.gnu.org/licenses/>.
-#
-# The full GNU General Public License is included in this distribution in
-# the file called "COPYING".
-#
-# Contact Information:
-# Linux NICS <linux.nics@intel.com>
-# e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-# Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-#
-################################################################################
-
+# Copyright(c) 1999 - 2018 Intel Corporation.
 #
 # Makefile for the Intel(R) 82575 PCI-Express ethernet driver
 #
diff --git a/drivers/net/ethernet/intel/igb/e1000_82575.c b/drivers/net/ethernet/intel/igb/e1000_82575.c
index dd9b6cac220d..b13b42e5a1d9 100644
--- a/drivers/net/ethernet/intel/igb/e1000_82575.c
+++ b/drivers/net/ethernet/intel/igb/e1000_82575.c
@@ -1,26 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 /* e1000_82575
  * e1000_82576
diff --git a/drivers/net/ethernet/intel/igb/e1000_82575.h b/drivers/net/ethernet/intel/igb/e1000_82575.h
index e53ebe97d709..6ad775b1a4c5 100644
--- a/drivers/net/ethernet/intel/igb/e1000_82575.h
+++ b/drivers/net/ethernet/intel/igb/e1000_82575.h
@@ -1,26 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #ifndef _E1000_82575_H_
 #define _E1000_82575_H_
diff --git a/drivers/net/ethernet/intel/igb/e1000_defines.h b/drivers/net/ethernet/intel/igb/e1000_defines.h
index 98534f765e0e..252440a418dc 100644
--- a/drivers/net/ethernet/intel/igb/e1000_defines.h
+++ b/drivers/net/ethernet/intel/igb/e1000_defines.h
@@ -1,26 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #ifndef _E1000_DEFINES_H_
 #define _E1000_DEFINES_H_
@@ -491,6 +470,8 @@
  * manageability enabled, allowing us room for 15 multicast addresses.
  */
 #define E1000_RAH_AV  0x80000000        /* Receive descriptor valid */
+#define E1000_RAH_ASEL_SRC_ADDR 0x00010000
+#define E1000_RAH_QSEL_ENABLE 0x10000000
 #define E1000_RAL_MAC_ADDR_LEN 4
 #define E1000_RAH_MAC_ADDR_LEN 2
 #define E1000_RAH_POOL_MASK 0x03FC0000
diff --git a/drivers/net/ethernet/intel/igb/e1000_hw.h b/drivers/net/ethernet/intel/igb/e1000_hw.h
index ff835e1e853d..5d87957b2627 100644
--- a/drivers/net/ethernet/intel/igb/e1000_hw.h
+++ b/drivers/net/ethernet/intel/igb/e1000_hw.h
@@ -1,25 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #ifndef _E1000_HW_H_
 #define _E1000_HW_H_
diff --git a/drivers/net/ethernet/intel/igb/e1000_i210.c b/drivers/net/ethernet/intel/igb/e1000_i210.c
index 6f548247e6d8..c54ebedca6da 100644
--- a/drivers/net/ethernet/intel/igb/e1000_i210.c
+++ b/drivers/net/ethernet/intel/igb/e1000_i210.c
@@ -1,26 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 /* e1000_i210
  * e1000_i211
diff --git a/drivers/net/ethernet/intel/igb/e1000_i210.h b/drivers/net/ethernet/intel/igb/e1000_i210.h
index 56f015ccb206..5c437fdc49ee 100644
--- a/drivers/net/ethernet/intel/igb/e1000_i210.h
+++ b/drivers/net/ethernet/intel/igb/e1000_i210.h
@@ -1,26 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #ifndef _E1000_I210_H_
 #define _E1000_I210_H_
diff --git a/drivers/net/ethernet/intel/igb/e1000_mac.c b/drivers/net/ethernet/intel/igb/e1000_mac.c
index 298afa0d9159..79ee0a747260 100644
--- a/drivers/net/ethernet/intel/igb/e1000_mac.c
+++ b/drivers/net/ethernet/intel/igb/e1000_mac.c
@@ -1,26 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #include <linux/if_ether.h>
 #include <linux/delay.h>
diff --git a/drivers/net/ethernet/intel/igb/e1000_mac.h b/drivers/net/ethernet/intel/igb/e1000_mac.h
index 04d80c765aee..6e110f28f922 100644
--- a/drivers/net/ethernet/intel/igb/e1000_mac.h
+++ b/drivers/net/ethernet/intel/igb/e1000_mac.h
@@ -1,26 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #ifndef _E1000_MAC_H_
 #define _E1000_MAC_H_
diff --git a/drivers/net/ethernet/intel/igb/e1000_mbx.c b/drivers/net/ethernet/intel/igb/e1000_mbx.c
index ef42f1689b3b..46debd991bfe 100644
--- a/drivers/net/ethernet/intel/igb/e1000_mbx.c
+++ b/drivers/net/ethernet/intel/igb/e1000_mbx.c
@@ -1,26 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #include "e1000_mbx.h"
 
diff --git a/drivers/net/ethernet/intel/igb/e1000_mbx.h b/drivers/net/ethernet/intel/igb/e1000_mbx.h
index 4f0ecd28354d..178e60ec71d4 100644
--- a/drivers/net/ethernet/intel/igb/e1000_mbx.h
+++ b/drivers/net/ethernet/intel/igb/e1000_mbx.h
@@ -1,26 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #ifndef _E1000_MBX_H_
 #define _E1000_MBX_H_
diff --git a/drivers/net/ethernet/intel/igb/e1000_nvm.c b/drivers/net/ethernet/intel/igb/e1000_nvm.c
index e4596f151cd4..09f4dcb09632 100644
--- a/drivers/net/ethernet/intel/igb/e1000_nvm.c
+++ b/drivers/net/ethernet/intel/igb/e1000_nvm.c
@@ -1,25 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #include <linux/if_ether.h>
 #include <linux/delay.h>
diff --git a/drivers/net/ethernet/intel/igb/e1000_nvm.h b/drivers/net/ethernet/intel/igb/e1000_nvm.h
index dde68cd54a53..091cddf4ada8 100644
--- a/drivers/net/ethernet/intel/igb/e1000_nvm.h
+++ b/drivers/net/ethernet/intel/igb/e1000_nvm.h
@@ -1,26 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #ifndef _E1000_NVM_H_
 #define _E1000_NVM_H_
diff --git a/drivers/net/ethernet/intel/igb/e1000_phy.c b/drivers/net/ethernet/intel/igb/e1000_phy.c
index 4ec61243da82..2be0e762ec69 100644
--- a/drivers/net/ethernet/intel/igb/e1000_phy.c
+++ b/drivers/net/ethernet/intel/igb/e1000_phy.c
@@ -1,26 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2015 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #include <linux/if_ether.h>
 #include <linux/delay.h>
diff --git a/drivers/net/ethernet/intel/igb/e1000_phy.h b/drivers/net/ethernet/intel/igb/e1000_phy.h
index 856d2cda0643..5894e4b1d0a8 100644
--- a/drivers/net/ethernet/intel/igb/e1000_phy.h
+++ b/drivers/net/ethernet/intel/igb/e1000_phy.h
@@ -1,26 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #ifndef _E1000_PHY_H_
 #define _E1000_PHY_H_
diff --git a/drivers/net/ethernet/intel/igb/e1000_regs.h b/drivers/net/ethernet/intel/igb/e1000_regs.h
index e8fa8c6530e0..0ad737d2f289 100644
--- a/drivers/net/ethernet/intel/igb/e1000_regs.h
+++ b/drivers/net/ethernet/intel/igb/e1000_regs.h
@@ -1,26 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #ifndef _E1000_REGS_H_
 #define _E1000_REGS_H_
diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h
index 8dbc399b345e..9643b5b3d444 100644
--- a/drivers/net/ethernet/intel/igb/igb.h
+++ b/drivers/net/ethernet/intel/igb/igb.h
@@ -1,26 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 /* Linux PRO/1000 Ethernet Driver main header file */
 
@@ -442,6 +421,8 @@ struct hwmon_buff {
 enum igb_filter_match_flags {
 	IGB_FILTER_FLAG_ETHER_TYPE = 0x1,
 	IGB_FILTER_FLAG_VLAN_TCI   = 0x2,
+	IGB_FILTER_FLAG_SRC_MAC_ADDR   = 0x4,
+	IGB_FILTER_FLAG_DST_MAC_ADDR   = 0x8,
 };
 
 #define IGB_MAX_RXNFC_FILTERS 16
@@ -456,11 +437,14 @@ struct igb_nfc_input {
 	u8 match_flags;
 	__be16 etype;
 	__be16 vlan_tci;
+	u8 src_addr[ETH_ALEN];
+	u8 dst_addr[ETH_ALEN];
 };
 
 struct igb_nfc_filter {
 	struct hlist_node nfc_node;
 	struct igb_nfc_input filter;
+	unsigned long cookie;
 	u16 etype_reg_index;
 	u16 sw_idx;
 	u16 action;
@@ -474,6 +458,8 @@ struct igb_mac_addr {
 
 #define IGB_MAC_STATE_DEFAULT	0x1
 #define IGB_MAC_STATE_IN_USE	0x2
+#define IGB_MAC_STATE_SRC_ADDR	0x4
+#define IGB_MAC_STATE_QUEUE_STEERING 0x8
 
 /* board specific private data structure */
 struct igb_adapter {
@@ -598,6 +584,7 @@ struct igb_adapter {
 
 	/* RX network flow classification support */
 	struct hlist_head nfc_filter_list;
+	struct hlist_head cls_flower_list;
 	unsigned int nfc_filter_count;
 	/* lock for RX network flow classification filter */
 	spinlock_t nfc_lock;
@@ -739,4 +726,9 @@ int igb_add_filter(struct igb_adapter *adapter,
 int igb_erase_filter(struct igb_adapter *adapter,
 		     struct igb_nfc_filter *input);
 
+int igb_add_mac_steering_filter(struct igb_adapter *adapter,
+				const u8 *addr, u8 queue, u8 flags);
+int igb_del_mac_steering_filter(struct igb_adapter *adapter,
+				const u8 *addr, u8 queue, u8 flags);
+
 #endif /* _IGB_H_ */
diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c
index e77ba0d5866d..2d798499d35e 100644
--- a/drivers/net/ethernet/intel/igb/igb_ethtool.c
+++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c
@@ -1,26 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 /* ethtool support for igb */
 
@@ -2495,6 +2474,23 @@ static int igb_get_ethtool_nfc_entry(struct igb_adapter *adapter,
 			fsp->h_ext.vlan_tci = rule->filter.vlan_tci;
 			fsp->m_ext.vlan_tci = htons(VLAN_PRIO_MASK);
 		}
+		if (rule->filter.match_flags & IGB_FILTER_FLAG_DST_MAC_ADDR) {
+			ether_addr_copy(fsp->h_u.ether_spec.h_dest,
+					rule->filter.dst_addr);
+			/* As we only support matching by the full
+			 * mask, return the mask to userspace
+			 */
+			eth_broadcast_addr(fsp->m_u.ether_spec.h_dest);
+		}
+		if (rule->filter.match_flags & IGB_FILTER_FLAG_SRC_MAC_ADDR) {
+			ether_addr_copy(fsp->h_u.ether_spec.h_source,
+					rule->filter.src_addr);
+			/* As we only support matching by the full
+			 * mask, return the mask to userspace
+			 */
+			eth_broadcast_addr(fsp->m_u.ether_spec.h_source);
+		}
+
 		return 0;
 	}
 	return -EINVAL;
@@ -2768,14 +2764,41 @@ static int igb_rxnfc_write_vlan_prio_filter(struct igb_adapter *adapter,
 
 int igb_add_filter(struct igb_adapter *adapter, struct igb_nfc_filter *input)
 {
+	struct e1000_hw *hw = &adapter->hw;
 	int err = -EINVAL;
 
+	if (hw->mac.type == e1000_i210 &&
+	    !(input->filter.match_flags & ~IGB_FILTER_FLAG_SRC_MAC_ADDR)) {
+		dev_err(&adapter->pdev->dev,
+			"i210 doesn't support flow classification rules specifying only source addresses.\n");
+		return -EOPNOTSUPP;
+	}
+
 	if (input->filter.match_flags & IGB_FILTER_FLAG_ETHER_TYPE) {
 		err = igb_rxnfc_write_etype_filter(adapter, input);
 		if (err)
 			return err;
 	}
 
+	if (input->filter.match_flags & IGB_FILTER_FLAG_DST_MAC_ADDR) {
+		err = igb_add_mac_steering_filter(adapter,
+						  input->filter.dst_addr,
+						  input->action, 0);
+		err = min_t(int, err, 0);
+		if (err)
+			return err;
+	}
+
+	if (input->filter.match_flags & IGB_FILTER_FLAG_SRC_MAC_ADDR) {
+		err = igb_add_mac_steering_filter(adapter,
+						  input->filter.src_addr,
+						  input->action,
+						  IGB_MAC_STATE_SRC_ADDR);
+		err = min_t(int, err, 0);
+		if (err)
+			return err;
+	}
+
 	if (input->filter.match_flags & IGB_FILTER_FLAG_VLAN_TCI)
 		err = igb_rxnfc_write_vlan_prio_filter(adapter, input);
 
@@ -2824,6 +2847,15 @@ int igb_erase_filter(struct igb_adapter *adapter, struct igb_nfc_filter *input)
 		igb_clear_vlan_prio_filter(adapter,
 					   ntohs(input->filter.vlan_tci));
 
+	if (input->filter.match_flags & IGB_FILTER_FLAG_SRC_MAC_ADDR)
+		igb_del_mac_steering_filter(adapter, input->filter.src_addr,
+					    input->action,
+					    IGB_MAC_STATE_SRC_ADDR);
+
+	if (input->filter.match_flags & IGB_FILTER_FLAG_DST_MAC_ADDR)
+		igb_del_mac_steering_filter(adapter, input->filter.dst_addr,
+					    input->action, 0);
+
 	return 0;
 }
 
@@ -2865,7 +2897,7 @@ static int igb_update_ethtool_nfc_entry(struct igb_adapter *adapter,
 
 	/* add filter to the list */
 	if (parent)
-		hlist_add_behind(&parent->nfc_node, &input->nfc_node);
+		hlist_add_behind(&input->nfc_node, &parent->nfc_node);
 	else
 		hlist_add_head(&input->nfc_node, &adapter->nfc_filter_list);
 
@@ -2905,10 +2937,6 @@ static int igb_add_ethtool_nfc_entry(struct igb_adapter *adapter,
 	if ((fsp->flow_type & ~FLOW_EXT) != ETHER_FLOW)
 		return -EINVAL;
 
-	if (fsp->m_u.ether_spec.h_proto != ETHER_TYPE_FULL_MASK &&
-	    fsp->m_ext.vlan_tci != htons(VLAN_PRIO_MASK))
-		return -EINVAL;
-
 	input = kzalloc(sizeof(*input), GFP_KERNEL);
 	if (!input)
 		return -ENOMEM;
@@ -2918,6 +2946,20 @@ static int igb_add_ethtool_nfc_entry(struct igb_adapter *adapter,
 		input->filter.match_flags = IGB_FILTER_FLAG_ETHER_TYPE;
 	}
 
+	/* Only support matching addresses by the full mask */
+	if (is_broadcast_ether_addr(fsp->m_u.ether_spec.h_source)) {
+		input->filter.match_flags |= IGB_FILTER_FLAG_SRC_MAC_ADDR;
+		ether_addr_copy(input->filter.src_addr,
+				fsp->h_u.ether_spec.h_source);
+	}
+
+	/* Only support matching addresses by the full mask */
+	if (is_broadcast_ether_addr(fsp->m_u.ether_spec.h_dest)) {
+		input->filter.match_flags |= IGB_FILTER_FLAG_DST_MAC_ADDR;
+		ether_addr_copy(input->filter.dst_addr,
+				fsp->h_u.ether_spec.h_dest);
+	}
+
 	if ((fsp->flow_type & FLOW_EXT) && fsp->m_ext.vlan_tci) {
 		if (fsp->m_ext.vlan_tci != htons(VLAN_PRIO_MASK)) {
 			err = -EINVAL;
diff --git a/drivers/net/ethernet/intel/igb/igb_hwmon.c b/drivers/net/ethernet/intel/igb/igb_hwmon.c
index bebe43b3a836..3b83747b2700 100644
--- a/drivers/net/ethernet/intel/igb/igb_hwmon.c
+++ b/drivers/net/ethernet/intel/igb/igb_hwmon.c
@@ -1,26 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #include "igb.h"
 #include "e1000_82575.h"
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index cce7ada89255..78574c06635b 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -1,26 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/* Intel(R) Gigabit Ethernet Linux driver
- * Copyright(c) 2007-2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- */
+/* Copyright(c) 2007 - 2018 Intel Corporation. */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
@@ -36,6 +15,7 @@
 #include <net/checksum.h>
 #include <net/ip6_checksum.h>
 #include <net/pkt_sched.h>
+#include <net/pkt_cls.h>
 #include <linux/net_tstamp.h>
 #include <linux/mii.h>
 #include <linux/ethtool.h>
@@ -2513,6 +2493,250 @@ static int igb_offload_cbs(struct igb_adapter *adapter,
 	return 0;
 }
 
+#define ETHER_TYPE_FULL_MASK ((__force __be16)~0)
+#define VLAN_PRIO_FULL_MASK (0x07)
+
+static int igb_parse_cls_flower(struct igb_adapter *adapter,
+				struct tc_cls_flower_offload *f,
+				int traffic_class,
+				struct igb_nfc_filter *input)
+{
+	struct netlink_ext_ack *extack = f->common.extack;
+
+	if (f->dissector->used_keys &
+	    ~(BIT(FLOW_DISSECTOR_KEY_BASIC) |
+	      BIT(FLOW_DISSECTOR_KEY_CONTROL) |
+	      BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
+	      BIT(FLOW_DISSECTOR_KEY_VLAN))) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Unsupported key used, only BASIC, CONTROL, ETH_ADDRS and VLAN are supported");
+		return -EOPNOTSUPP;
+	}
+
+	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
+		struct flow_dissector_key_eth_addrs *key, *mask;
+
+		key = skb_flow_dissector_target(f->dissector,
+						FLOW_DISSECTOR_KEY_ETH_ADDRS,
+						f->key);
+		mask = skb_flow_dissector_target(f->dissector,
+						 FLOW_DISSECTOR_KEY_ETH_ADDRS,
+						 f->mask);
+
+		if (!is_zero_ether_addr(mask->dst)) {
+			if (!is_broadcast_ether_addr(mask->dst)) {
+				NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for destination MAC address");
+				return -EINVAL;
+			}
+
+			input->filter.match_flags |=
+				IGB_FILTER_FLAG_DST_MAC_ADDR;
+			ether_addr_copy(input->filter.dst_addr, key->dst);
+		}
+
+		if (!is_zero_ether_addr(mask->src)) {
+			if (!is_broadcast_ether_addr(mask->src)) {
+				NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for source MAC address");
+				return -EINVAL;
+			}
+
+			input->filter.match_flags |=
+				IGB_FILTER_FLAG_SRC_MAC_ADDR;
+			ether_addr_copy(input->filter.src_addr, key->src);
+		}
+	}
+
+	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
+		struct flow_dissector_key_basic *key, *mask;
+
+		key = skb_flow_dissector_target(f->dissector,
+						FLOW_DISSECTOR_KEY_BASIC,
+						f->key);
+		mask = skb_flow_dissector_target(f->dissector,
+						 FLOW_DISSECTOR_KEY_BASIC,
+						 f->mask);
+
+		if (mask->n_proto) {
+			if (mask->n_proto != ETHER_TYPE_FULL_MASK) {
+				NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for EtherType filter");
+				return -EINVAL;
+			}
+
+			input->filter.match_flags |= IGB_FILTER_FLAG_ETHER_TYPE;
+			input->filter.etype = key->n_proto;
+		}
+	}
+
+	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
+		struct flow_dissector_key_vlan *key, *mask;
+
+		key = skb_flow_dissector_target(f->dissector,
+						FLOW_DISSECTOR_KEY_VLAN,
+						f->key);
+		mask = skb_flow_dissector_target(f->dissector,
+						 FLOW_DISSECTOR_KEY_VLAN,
+						 f->mask);
+
+		if (mask->vlan_priority) {
+			if (mask->vlan_priority != VLAN_PRIO_FULL_MASK) {
+				NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for VLAN priority");
+				return -EINVAL;
+			}
+
+			input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI;
+			input->filter.vlan_tci = key->vlan_priority;
+		}
+	}
+
+	input->action = traffic_class;
+	input->cookie = f->cookie;
+
+	return 0;
+}
+
+static int igb_configure_clsflower(struct igb_adapter *adapter,
+				   struct tc_cls_flower_offload *cls_flower)
+{
+	struct netlink_ext_ack *extack = cls_flower->common.extack;
+	struct igb_nfc_filter *filter, *f;
+	int err, tc;
+
+	tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
+	if (tc < 0) {
+		NL_SET_ERR_MSG_MOD(extack, "Invalid traffic class");
+		return -EINVAL;
+	}
+
+	filter = kzalloc(sizeof(*filter), GFP_KERNEL);
+	if (!filter)
+		return -ENOMEM;
+
+	err = igb_parse_cls_flower(adapter, cls_flower, tc, filter);
+	if (err < 0)
+		goto err_parse;
+
+	spin_lock(&adapter->nfc_lock);
+
+	hlist_for_each_entry(f, &adapter->nfc_filter_list, nfc_node) {
+		if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
+			err = -EEXIST;
+			NL_SET_ERR_MSG_MOD(extack,
+					   "This filter is already set in ethtool");
+			goto err_locked;
+		}
+	}
+
+	hlist_for_each_entry(f, &adapter->cls_flower_list, nfc_node) {
+		if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
+			err = -EEXIST;
+			NL_SET_ERR_MSG_MOD(extack,
+					   "This filter is already set in cls_flower");
+			goto err_locked;
+		}
+	}
+
+	err = igb_add_filter(adapter, filter);
+	if (err < 0) {
+		NL_SET_ERR_MSG_MOD(extack, "Could not add filter to the adapter");
+		goto err_locked;
+	}
+
+	hlist_add_head(&filter->nfc_node, &adapter->cls_flower_list);
+
+	spin_unlock(&adapter->nfc_lock);
+
+	return 0;
+
+err_locked:
+	spin_unlock(&adapter->nfc_lock);
+
+err_parse:
+	kfree(filter);
+
+	return err;
+}
+
+static int igb_delete_clsflower(struct igb_adapter *adapter,
+				struct tc_cls_flower_offload *cls_flower)
+{
+	struct igb_nfc_filter *filter;
+	int err;
+
+	spin_lock(&adapter->nfc_lock);
+
+	hlist_for_each_entry(filter, &adapter->cls_flower_list, nfc_node)
+		if (filter->cookie == cls_flower->cookie)
+			break;
+
+	if (!filter) {
+		err = -ENOENT;
+		goto out;
+	}
+
+	err = igb_erase_filter(adapter, filter);
+	if (err < 0)
+		goto out;
+
+	hlist_del(&filter->nfc_node);
+	kfree(filter);
+
+out:
+	spin_unlock(&adapter->nfc_lock);
+
+	return err;
+}
+
+static int igb_setup_tc_cls_flower(struct igb_adapter *adapter,
+				   struct tc_cls_flower_offload *cls_flower)
+{
+	switch (cls_flower->command) {
+	case TC_CLSFLOWER_REPLACE:
+		return igb_configure_clsflower(adapter, cls_flower);
+	case TC_CLSFLOWER_DESTROY:
+		return igb_delete_clsflower(adapter, cls_flower);
+	case TC_CLSFLOWER_STATS:
+		return -EOPNOTSUPP;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int igb_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
+				 void *cb_priv)
+{
+	struct igb_adapter *adapter = cb_priv;
+
+	if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
+		return -EOPNOTSUPP;
+
+	switch (type) {
+	case TC_SETUP_CLSFLOWER:
+		return igb_setup_tc_cls_flower(adapter, type_data);
+
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int igb_setup_tc_block(struct igb_adapter *adapter,
+			      struct tc_block_offload *f)
+{
+	if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
+		return -EOPNOTSUPP;
+
+	switch (f->command) {
+	case TC_BLOCK_BIND:
+		return tcf_block_cb_register(f->block, igb_setup_tc_block_cb,
+					     adapter, adapter);
+	case TC_BLOCK_UNBIND:
+		tcf_block_cb_unregister(f->block, igb_setup_tc_block_cb,
+					adapter);
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
 			void *type_data)
 {
@@ -2521,6 +2745,8 @@ static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
 	switch (type) {
 	case TC_SETUP_QDISC_CBS:
 		return igb_offload_cbs(adapter, type_data);
+	case TC_SETUP_BLOCK:
+		return igb_setup_tc_block(adapter, type_data);
 
 	default:
 		return -EOPNOTSUPP;
@@ -2822,6 +3048,9 @@ static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (hw->mac.type >= e1000_82576)
 		netdev->features |= NETIF_F_SCTP_CRC;
 
+	if (hw->mac.type >= e1000_i350)
+		netdev->features |= NETIF_F_HW_TC;
+
 #define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
 				  NETIF_F_GSO_GRE_CSUM | \
 				  NETIF_F_GSO_IPXIP4 | \
@@ -6856,8 +7085,35 @@ static void igb_set_default_mac_filter(struct igb_adapter *adapter)
 	igb_rar_set_index(adapter, 0);
 }
 
-static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr,
-			      const u8 queue)
+/* If the filter to be added and an already existing filter express
+ * the same address and address type, it should be possible to only
+ * override the other configurations, for example the queue to steer
+ * traffic.
+ */
+static bool igb_mac_entry_can_be_used(const struct igb_mac_addr *entry,
+				      const u8 *addr, const u8 flags)
+{
+	if (!(entry->state & IGB_MAC_STATE_IN_USE))
+		return true;
+
+	if ((entry->state & IGB_MAC_STATE_SRC_ADDR) !=
+	    (flags & IGB_MAC_STATE_SRC_ADDR))
+		return false;
+
+	if (!ether_addr_equal(addr, entry->addr))
+		return false;
+
+	return true;
+}
+
+/* Add a MAC filter for 'addr' directing matching traffic to 'queue',
+ * 'flags' is used to indicate what kind of match is made, match is by
+ * default for the destination address, if matching by source address
+ * is desired the flag IGB_MAC_STATE_SRC_ADDR can be used.
+ */
+static int igb_add_mac_filter_flags(struct igb_adapter *adapter,
+				    const u8 *addr, const u8 queue,
+				    const u8 flags)
 {
 	struct e1000_hw *hw = &adapter->hw;
 	int rar_entries = hw->mac.rar_entry_count -
@@ -6872,12 +7128,13 @@ static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr,
 	 * addresses.
 	 */
 	for (i = 0; i < rar_entries; i++) {
-		if (adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE)
+		if (!igb_mac_entry_can_be_used(&adapter->mac_table[i],
+					       addr, flags))
 			continue;
 
 		ether_addr_copy(adapter->mac_table[i].addr, addr);
 		adapter->mac_table[i].queue = queue;
-		adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE;
+		adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE | flags;
 
 		igb_rar_set_index(adapter, i);
 		return i;
@@ -6886,8 +7143,21 @@ static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr,
 	return -ENOSPC;
 }
 
-static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr,
+static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr,
 			      const u8 queue)
+{
+	return igb_add_mac_filter_flags(adapter, addr, queue, 0);
+}
+
+/* Remove a MAC filter for 'addr' directing matching traffic to
+ * 'queue', 'flags' is used to indicate what kind of match need to be
+ * removed, match is by default for the destination address, if
+ * matching by source address is to be removed the flag
+ * IGB_MAC_STATE_SRC_ADDR can be used.
+ */
+static int igb_del_mac_filter_flags(struct igb_adapter *adapter,
+				    const u8 *addr, const u8 queue,
+				    const u8 flags)
 {
 	struct e1000_hw *hw = &adapter->hw;
 	int rar_entries = hw->mac.rar_entry_count -
@@ -6904,14 +7174,26 @@ static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr,
 	for (i = 0; i < rar_entries; i++) {
 		if (!(adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE))
 			continue;
+		if ((adapter->mac_table[i].state & flags) != flags)
+			continue;
 		if (adapter->mac_table[i].queue != queue)
 			continue;
 		if (!ether_addr_equal(adapter->mac_table[i].addr, addr))
 			continue;
 
-		adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE;
-		memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
-		adapter->mac_table[i].queue = 0;
+		/* When a filter for the default address is "deleted",
+		 * we return it to its initial configuration
+		 */
+		if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) {
+			adapter->mac_table[i].state =
+				IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
+			adapter->mac_table[i].queue =
+				adapter->vfs_allocated_count;
+		} else {
+			adapter->mac_table[i].state = 0;
+			adapter->mac_table[i].queue = 0;
+			memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
+		}
 
 		igb_rar_set_index(adapter, i);
 		return 0;
@@ -6920,6 +7202,34 @@ static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr,
 	return -ENOENT;
 }
 
+static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr,
+			      const u8 queue)
+{
+	return igb_del_mac_filter_flags(adapter, addr, queue, 0);
+}
+
+int igb_add_mac_steering_filter(struct igb_adapter *adapter,
+				const u8 *addr, u8 queue, u8 flags)
+{
+	struct e1000_hw *hw = &adapter->hw;
+
+	/* In theory, this should be supported on 82575 as well, but
+	 * that part wasn't easily accessible during development.
+	 */
+	if (hw->mac.type != e1000_i210)
+		return -EOPNOTSUPP;
+
+	return igb_add_mac_filter_flags(adapter, addr, queue,
+					IGB_MAC_STATE_QUEUE_STEERING | flags);
+}
+
+int igb_del_mac_steering_filter(struct igb_adapter *adapter,
+				const u8 *addr, u8 queue, u8 flags)
+{
+	return igb_del_mac_filter_flags(adapter, addr, queue,
+					IGB_MAC_STATE_QUEUE_STEERING | flags);
+}
+
 static int igb_uc_sync(struct net_device *netdev, const unsigned char *addr)
 {
 	struct igb_adapter *adapter = netdev_priv(netdev);
@@ -8763,12 +9073,24 @@ static void igb_rar_set_index(struct igb_adapter *adapter, u32 index)
 		if (is_valid_ether_addr(addr))
 			rar_high |= E1000_RAH_AV;
 
-		if (hw->mac.type == e1000_82575)
+		if (adapter->mac_table[index].state & IGB_MAC_STATE_SRC_ADDR)
+			rar_high |= E1000_RAH_ASEL_SRC_ADDR;
+
+		switch (hw->mac.type) {
+		case e1000_82575:
+		case e1000_i210:
+			if (adapter->mac_table[index].state &
+			    IGB_MAC_STATE_QUEUE_STEERING)
+				rar_high |= E1000_RAH_QSEL_ENABLE;
+
 			rar_high |= E1000_RAH_POOL_1 *
 				    adapter->mac_table[index].queue;
-		else
+			break;
+		default:
 			rar_high |= E1000_RAH_POOL_1 <<
 				    adapter->mac_table[index].queue;
+			break;
+		}
 	}
 
 	wr32(E1000_RAL(index), rar_low);
@@ -9206,6 +9528,9 @@ static void igb_nfc_filter_exit(struct igb_adapter *adapter)
 	hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
 		igb_erase_filter(adapter, rule);
 
+	hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node)
+		igb_erase_filter(adapter, rule);
+
 	spin_unlock(&adapter->nfc_lock);
 }
 
diff --git a/drivers/net/ethernet/intel/igb/igb_ptp.c b/drivers/net/ethernet/intel/igb/igb_ptp.c
index 7454b9895a65..9f4d700e09df 100644
--- a/drivers/net/ethernet/intel/igb/igb_ptp.c
+++ b/drivers/net/ethernet/intel/igb/igb_ptp.c
@@ -1,21 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+
-/* PTP Hardware Clock (PHC) driver for the Intel 82576 and 82580
- *
- * Copyright (C) 2011 Richard Cochran <richardcochran@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, see <http://www.gnu.org/licenses/>.
- */
+/* Copyright (C) 2011 Richard Cochran <richardcochran@gmail.com> */
+
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/pci.h>
diff --git a/drivers/net/ethernet/intel/igbvf/Makefile b/drivers/net/ethernet/intel/igbvf/Makefile
index efe29dae384a..afd3e36eae75 100644
--- a/drivers/net/ethernet/intel/igbvf/Makefile
+++ b/drivers/net/ethernet/intel/igbvf/Makefile
@@ -1,31 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-################################################################################
-#
-# Intel(R) 82576 Virtual Function Linux driver
-# Copyright(c) 2009 - 2012 Intel Corporation.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms and conditions of the GNU General Public License,
-# version 2, as published by the Free Software Foundation.
-#
-# This program is distributed in the hope it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# The full GNU General Public License is included in this distribution in
-# the file called "COPYING".
-#
-# Contact Information:
-# e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-# Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-#
-################################################################################
-
+# Copyright(c) 2009 - 2018 Intel Corporation.
 #
 # Makefile for the Intel(R) 82576 VF ethernet driver
 #
diff --git a/drivers/net/ethernet/intel/igbvf/defines.h b/drivers/net/ethernet/intel/igbvf/defines.h
index 04bcfec0641b..4437f832412d 100644
--- a/drivers/net/ethernet/intel/igbvf/defines.h
+++ b/drivers/net/ethernet/intel/igbvf/defines.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel(R) 82576 Virtual Function Linux driver
-  Copyright(c) 1999 - 2012 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000_DEFINES_H_
 #define _E1000_DEFINES_H_
diff --git a/drivers/net/ethernet/intel/igbvf/ethtool.c b/drivers/net/ethernet/intel/igbvf/ethtool.c
index ca39e3cccaeb..3ae358b35227 100644
--- a/drivers/net/ethernet/intel/igbvf/ethtool.c
+++ b/drivers/net/ethernet/intel/igbvf/ethtool.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-
-  Intel(R) 82576 Virtual Function Linux driver
-  Copyright(c) 2009 - 2012 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 2009 - 2018 Intel Corporation. */
 
 /* ethtool support for igbvf */
 
diff --git a/drivers/net/ethernet/intel/igbvf/igbvf.h b/drivers/net/ethernet/intel/igbvf/igbvf.h
index f5bf248e22eb..eee26a3be90b 100644
--- a/drivers/net/ethernet/intel/igbvf/igbvf.h
+++ b/drivers/net/ethernet/intel/igbvf/igbvf.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel(R) 82576 Virtual Function Linux driver
-  Copyright(c) 2009 - 2012 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 2009 - 2018 Intel Corporation. */
 
 /* Linux PRO/1000 Ethernet Driver main header file */
 
diff --git a/drivers/net/ethernet/intel/igbvf/mbx.c b/drivers/net/ethernet/intel/igbvf/mbx.c
index 9195884096f8..163e5838f7c2 100644
--- a/drivers/net/ethernet/intel/igbvf/mbx.c
+++ b/drivers/net/ethernet/intel/igbvf/mbx.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-
-  Intel(R) 82576 Virtual Function Linux driver
-  Copyright(c) 2009 - 2012 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 2009 - 2018 Intel Corporation. */
 
 #include "mbx.h"
 
diff --git a/drivers/net/ethernet/intel/igbvf/mbx.h b/drivers/net/ethernet/intel/igbvf/mbx.h
index 479b062fe9ee..e5b31818d565 100644
--- a/drivers/net/ethernet/intel/igbvf/mbx.h
+++ b/drivers/net/ethernet/intel/igbvf/mbx.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel(R) 82576 Virtual Function Linux driver
-  Copyright(c) 1999 - 2012 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _E1000_MBX_H_
 #define _E1000_MBX_H_
diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c
index e2b7502f1953..f818f060e5a7 100644
--- a/drivers/net/ethernet/intel/igbvf/netdev.c
+++ b/drivers/net/ethernet/intel/igbvf/netdev.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-
-  Intel(R) 82576 Virtual Function Linux driver
-  Copyright(c) 2009 - 2012 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 2009 - 2018 Intel Corporation. */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
diff --git a/drivers/net/ethernet/intel/igbvf/regs.h b/drivers/net/ethernet/intel/igbvf/regs.h
index 614e52409f11..625a309a3355 100644
--- a/drivers/net/ethernet/intel/igbvf/regs.h
+++ b/drivers/net/ethernet/intel/igbvf/regs.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel(R) 82576 Virtual Function Linux driver
-  Copyright(c) 2009 - 2012 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 2009 - 2018 Intel Corporation. */
 
 #ifndef _E1000_REGS_H_
 #define _E1000_REGS_H_
diff --git a/drivers/net/ethernet/intel/igbvf/vf.c b/drivers/net/ethernet/intel/igbvf/vf.c
index bfe8d8297b2e..b8ba3f94c363 100644
--- a/drivers/net/ethernet/intel/igbvf/vf.c
+++ b/drivers/net/ethernet/intel/igbvf/vf.c
@@ -1,29 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-
-  Intel(R) 82576 Virtual Function Linux driver
-  Copyright(c) 2009 - 2012 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 2009 - 2018 Intel Corporation. */
 
 #include "vf.h"
 
diff --git a/drivers/net/ethernet/intel/igbvf/vf.h b/drivers/net/ethernet/intel/igbvf/vf.h
index 193b50026246..c71b0d7dbcee 100644
--- a/drivers/net/ethernet/intel/igbvf/vf.h
+++ b/drivers/net/ethernet/intel/igbvf/vf.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel(R) 82576 Virtual Function Linux driver
-  Copyright(c) 2009 - 2012 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 2009 - 2018 Intel Corporation. */
 
 #ifndef _E1000_VF_H_
 #define _E1000_VF_H_
diff --git a/drivers/net/ethernet/intel/ixgb/Makefile b/drivers/net/ethernet/intel/ixgb/Makefile
index 1b42dd554dd2..2433e9300a33 100644
--- a/drivers/net/ethernet/intel/ixgb/Makefile
+++ b/drivers/net/ethernet/intel/ixgb/Makefile
@@ -1,32 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-################################################################################
-#
-# Intel PRO/10GbE Linux driver
 # Copyright(c) 1999 - 2008 Intel Corporation.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms and conditions of the GNU General Public License,
-# version 2, as published by the Free Software Foundation.
-#
-# This program is distributed in the hope it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# The full GNU General Public License is included in this distribution in
-# the file called "COPYING".
-#
-# Contact Information:
-# Linux NICS <linux.nics@intel.com>
-# e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-# Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-#
-################################################################################
-
 #
 # Makefile for the Intel(R) PRO/10GbE ethernet driver
 #
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb.h b/drivers/net/ethernet/intel/ixgb/ixgb.h
index 92022841755f..e85271b68410 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb.h
+++ b/drivers/net/ethernet/intel/ixgb/ixgb.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel PRO/10GbE Linux driver
-  Copyright(c) 1999 - 2008 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2008 Intel Corporation. */
 
 #ifndef _IXGB_H_
 #define _IXGB_H_
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ee.c b/drivers/net/ethernet/intel/ixgb/ixgb_ee.c
index eca216b9b859..129286fc1634 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb_ee.c
+++ b/drivers/net/ethernet/intel/ixgb/ixgb_ee.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel PRO/10GbE Linux driver
-  Copyright(c) 1999 - 2008 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2008 Intel Corporation. */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ee.h b/drivers/net/ethernet/intel/ixgb/ixgb_ee.h
index 475297a810fe..3ee0a09e5d0a 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb_ee.h
+++ b/drivers/net/ethernet/intel/ixgb/ixgb_ee.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel PRO/10GbE Linux driver
-  Copyright(c) 1999 - 2008 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2008 Intel Corporation. */
 
 #ifndef _IXGB_EE_H_
 #define _IXGB_EE_H_
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c b/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c
index d10a0d242dda..43744bf0fc1c 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c
+++ b/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel PRO/10GbE Linux driver
-  Copyright(c) 1999 - 2008 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2008 Intel Corporation. */
 
 /* ethtool support for ixgb */
 
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_hw.c b/drivers/net/ethernet/intel/ixgb/ixgb_hw.c
index bf9a220f71fb..cbaa933ef30d 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb_hw.c
+++ b/drivers/net/ethernet/intel/ixgb/ixgb_hw.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel PRO/10GbE Linux driver
-  Copyright(c) 1999 - 2008 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2008 Intel Corporation. */
 
 /* ixgb_hw.c
  * Shared functions for accessing and configuring the adapter
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_hw.h b/drivers/net/ethernet/intel/ixgb/ixgb_hw.h
index 19f36d87ef61..6064583095da 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb_hw.h
+++ b/drivers/net/ethernet/intel/ixgb/ixgb_hw.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel PRO/10GbE Linux driver
-  Copyright(c) 1999 - 2008 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2008 Intel Corporation. */
 
 #ifndef _IXGB_HW_H_
 #define _IXGB_HW_H_
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ids.h b/drivers/net/ethernet/intel/ixgb/ixgb_ids.h
index 24e849902d60..9695b8215f01 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb_ids.h
+++ b/drivers/net/ethernet/intel/ixgb/ixgb_ids.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel PRO/10GbE Linux driver
-  Copyright(c) 1999 - 2008 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2008 Intel Corporation. */
 
 #ifndef _IXGB_IDS_H_
 #define _IXGB_IDS_H_
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_main.c b/drivers/net/ethernet/intel/ixgb/ixgb_main.c
index 2353c383f0a7..62f2173bc20e 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb_main.c
+++ b/drivers/net/ethernet/intel/ixgb/ixgb_main.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel PRO/10GbE Linux driver
-  Copyright(c) 1999 - 2008 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2008 Intel Corporation. */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_osdep.h b/drivers/net/ethernet/intel/ixgb/ixgb_osdep.h
index b1710379192e..7bd54efa698d 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb_osdep.h
+++ b/drivers/net/ethernet/intel/ixgb/ixgb_osdep.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel PRO/10GbE Linux driver
-  Copyright(c) 1999 - 2008 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2008 Intel Corporation. */
 
 /* glue for the OS independent part of ixgb
  * includes register access macros
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_param.c b/drivers/net/ethernet/intel/ixgb/ixgb_param.c
index 04a60640ddda..f0cadd532c53 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb_param.c
+++ b/drivers/net/ethernet/intel/ixgb/ixgb_param.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel PRO/10GbE Linux driver
-  Copyright(c) 1999 - 2008 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2008 Intel Corporation. */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
diff --git a/drivers/net/ethernet/intel/ixgbe/Makefile b/drivers/net/ethernet/intel/ixgbe/Makefile
index 4cd96c88cb5d..5414685189ce 100644
--- a/drivers/net/ethernet/intel/ixgbe/Makefile
+++ b/drivers/net/ethernet/intel/ixgbe/Makefile
@@ -1,32 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-################################################################################
-#
-# Intel 10 Gigabit PCI Express Linux driver
-# Copyright(c) 1999 - 2013 Intel Corporation.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms and conditions of the GNU General Public License,
-# version 2, as published by the Free Software Foundation.
-#
-# This program is distributed in the hope it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# The full GNU General Public License is included in this distribution in
-# the file called "COPYING".
-#
-# Contact Information:
-# Linux NICS <linux.nics@intel.com>
-# e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-# Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-#
-################################################################################
-
+# Copyright(c) 1999 - 2018 Intel Corporation.
 #
 # Makefile for the Intel(R) 10GbE PCI Express ethernet driver
 #
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
index 4f08c712e58e..fc534e91c6b2 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBE_H_
 #define _IXGBE_H_
@@ -241,8 +215,7 @@ struct ixgbe_tx_buffer {
 	unsigned long time_stamp;
 	union {
 		struct sk_buff *skb;
-		/* XDP uses address ptr on irq_clean */
-		void *data;
+		struct xdp_frame *xdpf;
 	};
 	unsigned int bytecount;
 	unsigned short gso_segs;
@@ -306,7 +279,6 @@ enum ixgbe_ring_state_t {
 struct ixgbe_fwd_adapter {
 	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
 	struct net_device *netdev;
-	struct ixgbe_adapter *real_adapter;
 	unsigned int tx_base_queue;
 	unsigned int rx_base_queue;
 	int pool;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_82598.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_82598.c
index cb0fe5fedb33..eee277c1bedf 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_82598.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_82598.c
@@ -1,31 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include <linux/pci.h>
 #include <linux/delay.h>
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c
index 66a74f4651e8..1e49716f52bc 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c
@@ -1,31 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include <linux/pci.h>
 #include <linux/delay.h>
@@ -1462,7 +1436,8 @@ void ixgbe_atr_compute_perfect_hash_82599(union ixgbe_atr_input *input,
 {
 
 	u32 hi_hash_dword, lo_hash_dword, flow_vm_vlan;
-	u32 bucket_hash = 0, hi_dword = 0;
+	u32 bucket_hash = 0;
+	__be32 hi_dword = 0;
 	int i;
 
 	/* Apply masks to input data */
@@ -1501,7 +1476,7 @@ void ixgbe_atr_compute_perfect_hash_82599(union ixgbe_atr_input *input,
 	 * Limit hash to 13 bits since max bucket count is 8K.
 	 * Store result at the end of the input stream.
 	 */
-	input->formatted.bkt_hash = bucket_hash & 0x1FFF;
+	input->formatted.bkt_hash = (__force __be16)(bucket_hash & 0x1FFF);
 }
 
 /**
@@ -1610,7 +1585,7 @@ s32 ixgbe_fdir_set_input_mask_82599(struct ixgbe_hw *hw,
 		return IXGBE_ERR_CONFIG;
 	}
 
-	switch (input_mask->formatted.flex_bytes & 0xFFFF) {
+	switch ((__force u16)input_mask->formatted.flex_bytes & 0xFFFF) {
 	case 0x0000:
 		/* Mask Flex Bytes */
 		fdirm |= IXGBE_FDIRM_FLEX;
@@ -1680,13 +1655,13 @@ s32 ixgbe_fdir_write_perfect_filter_82599(struct ixgbe_hw *hw,
 	IXGBE_WRITE_REG(hw, IXGBE_FDIRPORT, fdirport);
 
 	/* record vlan (little-endian) and flex_bytes(big-endian) */
-	fdirvlan = IXGBE_STORE_AS_BE16(input->formatted.flex_bytes);
+	fdirvlan = IXGBE_STORE_AS_BE16((__force u16)input->formatted.flex_bytes);
 	fdirvlan <<= IXGBE_FDIRVLAN_FLEX_SHIFT;
 	fdirvlan |= ntohs(input->formatted.vlan_id);
 	IXGBE_WRITE_REG(hw, IXGBE_FDIRVLAN, fdirvlan);
 
 	/* configure FDIRHASH register */
-	fdirhash = input->formatted.bkt_hash;
+	fdirhash = (__force u32)input->formatted.bkt_hash;
 	fdirhash |= soft_id << IXGBE_FDIRHASH_SIG_SW_INDEX_SHIFT;
 	IXGBE_WRITE_REG(hw, IXGBE_FDIRHASH, fdirhash);
 
@@ -1724,7 +1699,7 @@ s32 ixgbe_fdir_erase_perfect_filter_82599(struct ixgbe_hw *hw,
 	s32 err;
 
 	/* configure FDIRHASH register */
-	fdirhash = input->formatted.bkt_hash;
+	fdirhash = (__force u32)input->formatted.bkt_hash;
 	fdirhash |= soft_id << IXGBE_FDIRHASH_SIG_SW_INDEX_SHIFT;
 	IXGBE_WRITE_REG(hw, IXGBE_FDIRHASH, fdirhash);
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
index 633be93f3dbb..3f5c350716bb 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
@@ -1,31 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include <linux/pci.h>
 #include <linux/delay.h>
@@ -3652,7 +3626,7 @@ s32 ixgbe_hic_unlocked(struct ixgbe_hw *hw, u32 *buffer, u32 length,
 	 */
 	for (i = 0; i < dword_len; i++)
 		IXGBE_WRITE_REG_ARRAY(hw, IXGBE_FLEX_MNG,
-				      i, cpu_to_le32(buffer[i]));
+				      i, (__force u32)cpu_to_le32(buffer[i]));
 
 	/* Setting this bit tells the ARC that a new command is pending. */
 	IXGBE_WRITE_REG(hw, IXGBE_HICR, hicr | IXGBE_HICR_C);
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.h
index 2b311382167a..4b531e8ae38a 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBE_COMMON_H_
 #define _IXGBE_COMMON_H_
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.c
index aaea8282bfd2..d26cea5b43bd 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.c
@@ -1,31 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
-
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "ixgbe.h"
 #include "ixgbe_type.h"
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.h
index 73b6362d4327..60cd5863bf5e 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2013 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _DCB_CONFIG_H_
 #define _DCB_CONFIG_H_
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.c
index 085130626330..379ae747cdce 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.c
@@ -1,31 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2013 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "ixgbe.h"
 #include "ixgbe_type.h"
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.h
index 7edce607f901..fdca41abb44c 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2013 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _DCB_82598_CONFIG_H_
 #define _DCB_82598_CONFIG_H_
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.c
index 1eed6811e914..7948849840a5 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2013 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "ixgbe.h"
 #include "ixgbe_type.h"
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.h
index fa030f0abc18..c6f084883cab 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2013 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _DCB_82599_CONFIG_H_
 #define _DCB_82599_CONFIG_H_
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c
index b33f3f87e4b1..c00332d2e02a 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2014 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "ixgbe.h"
 #include <linux/dcbnl.h>
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c
index ad54080488ee..55fe8114fe99 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c
@@ -1,30 +1,6 @@
-/*******************************************************************************
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2013 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
 #include <linux/debugfs.h>
 #include <linux/module.h>
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
index c0e6ab42e0e1..bdd179c29ea4 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 /* ethtool support for ixgbe */
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c
index 7a09a40e4472..94b3165ff543 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2014 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "ixgbe.h"
 #include <linux/if_ether.h>
@@ -465,7 +440,7 @@ int ixgbe_fcoe_ddp(struct ixgbe_adapter *adapter,
 	case cpu_to_le32(IXGBE_RXDADV_STAT_FCSTAT_FCPRSP):
 		dma_unmap_sg(&adapter->pdev->dev, ddp->sgl,
 			     ddp->sgc, DMA_FROM_DEVICE);
-		ddp->err = ddp_err;
+		ddp->err = (__force u32)ddp_err;
 		ddp->sgl = NULL;
 		ddp->sgc = 0;
 		/* fall through */
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.h
index cf1919901514..724f5382329f 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2013 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBE_FCOE_H
 #define _IXGBE_FCOE_H
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c
index cead23e3db0c..99b170f1efd1 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c
@@ -1,29 +1,5 @@
-/*******************************************************************************
- *
- * Intel 10 Gigabit PCI Express Linux driver
- * Copyright(c) 2017 Oracle and/or its affiliates. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * Linux NICS <linux.nics@intel.com>
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 Oracle and/or its affiliates. All rights reserved. */
 
 #include "ixgbe.h"
 #include <net/xfrm.h>
@@ -43,8 +19,9 @@ static void ixgbe_ipsec_set_tx_sa(struct ixgbe_hw *hw, u16 idx,
 	int i;
 
 	for (i = 0; i < 4; i++)
-		IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(i), cpu_to_be32(key[3 - i]));
-	IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, cpu_to_be32(salt));
+		IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(i),
+				(__force u32)cpu_to_be32(key[3 - i]));
+	IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, (__force u32)cpu_to_be32(salt));
 	IXGBE_WRITE_FLUSH(hw);
 
 	reg = IXGBE_READ_REG(hw, IXGBE_IPSTXIDX);
@@ -93,7 +70,8 @@ static void ixgbe_ipsec_set_rx_sa(struct ixgbe_hw *hw, u16 idx, __be32 spi,
 	int i;
 
 	/* store the SPI (in bigendian) and IPidx */
-	IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, cpu_to_le32(spi));
+	IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI,
+			(__force u32)cpu_to_le32((__force u32)spi));
 	IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, ip_idx);
 	IXGBE_WRITE_FLUSH(hw);
 
@@ -101,8 +79,9 @@ static void ixgbe_ipsec_set_rx_sa(struct ixgbe_hw *hw, u16 idx, __be32 spi,
 
 	/* store the key, salt, and mode */
 	for (i = 0; i < 4; i++)
-		IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(i), cpu_to_be32(key[3 - i]));
-	IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, cpu_to_be32(salt));
+		IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(i),
+				(__force u32)cpu_to_be32(key[3 - i]));
+	IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, (__force u32)cpu_to_be32(salt));
 	IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, mode);
 	IXGBE_WRITE_FLUSH(hw);
 
@@ -121,7 +100,8 @@ static void ixgbe_ipsec_set_rx_ip(struct ixgbe_hw *hw, u16 idx, __be32 addr[])
 
 	/* store the ip address */
 	for (i = 0; i < 4; i++)
-		IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(i), cpu_to_le32(addr[i]));
+		IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(i),
+				(__force u32)cpu_to_le32((__force u32)addr[i]));
 	IXGBE_WRITE_FLUSH(hw);
 
 	ixgbe_ipsec_set_rx_item(hw, idx, ips_rx_ip_tbl);
@@ -391,7 +371,8 @@ static struct xfrm_state *ixgbe_ipsec_find_rx_state(struct ixgbe_ipsec *ipsec,
 	struct xfrm_state *ret = NULL;
 
 	rcu_read_lock();
-	hash_for_each_possible_rcu(ipsec->rx_sa_list, rsa, hlist, spi)
+	hash_for_each_possible_rcu(ipsec->rx_sa_list, rsa, hlist,
+				   (__force u32)spi) {
 		if (spi == rsa->xs->id.spi &&
 		    ((ip4 && *daddr == rsa->xs->id.daddr.a4) ||
 		      (!ip4 && !memcmp(daddr, &rsa->xs->id.daddr.a6,
@@ -401,6 +382,7 @@ static struct xfrm_state *ixgbe_ipsec_find_rx_state(struct ixgbe_ipsec *ipsec,
 			xfrm_state_hold(ret);
 			break;
 		}
+	}
 	rcu_read_unlock();
 	return ret;
 }
@@ -593,7 +575,7 @@ static int ixgbe_ipsec_add_sa(struct xfrm_state *xs)
 
 		/* hash the new entry for faster search in Rx path */
 		hash_add_rcu(ipsec->rx_sa_list, &ipsec->rx_tbl[sa_idx].hlist,
-			     rsa.xs->id.spi);
+			     (__force u64)rsa.xs->id.spi);
 	} else {
 		struct tx_sa tsa;
 
@@ -677,7 +659,8 @@ static void ixgbe_ipsec_del_sa(struct xfrm_state *xs)
 			if (!ipsec->ip_tbl[ipi].ref_cnt) {
 				memset(&ipsec->ip_tbl[ipi], 0,
 				       sizeof(struct rx_ip_sa));
-				ixgbe_ipsec_set_rx_ip(hw, ipi, zerobuf);
+				ixgbe_ipsec_set_rx_ip(hw, ipi,
+						      (__force __be32 *)zerobuf);
 			}
 		}
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.h
index 4f099f516645..9ef7faadda69 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.h
@@ -1,30 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 2017 Oracle and/or its affiliates. All rights reserved.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program.  If not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 2017 Oracle and/or its affiliates. All rights reserved. */
 
 #ifndef _IXGBE_IPSEC_H_
 #define _IXGBE_IPSEC_H_
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
index ed4cbe94c355..893a9206e718 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "ixgbe.h"
 #include "ixgbe_sriov.h"
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index afadba99f7b8..bc299cfcd56d 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include <linux/types.h>
 #include <linux/module.h>
@@ -270,9 +245,6 @@ static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter,
 				     int expected_gts)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
-	int max_gts = 0;
-	enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
-	enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
 	struct pci_dev *pdev;
 
 	/* Some devices are not connected over PCIe and thus do not negotiate
@@ -288,49 +260,7 @@ static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter,
 	else
 		pdev = adapter->pdev;
 
-	if (pcie_get_minimum_link(pdev, &speed, &width) ||
-	    speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) {
-		e_dev_warn("Unable to determine PCI Express bandwidth.\n");
-		return;
-	}
-
-	switch (speed) {
-	case PCIE_SPEED_2_5GT:
-		/* 8b/10b encoding reduces max throughput by 20% */
-		max_gts = 2 * width;
-		break;
-	case PCIE_SPEED_5_0GT:
-		/* 8b/10b encoding reduces max throughput by 20% */
-		max_gts = 4 * width;
-		break;
-	case PCIE_SPEED_8_0GT:
-		/* 128b/130b encoding reduces throughput by less than 2% */
-		max_gts = 8 * width;
-		break;
-	default:
-		e_dev_warn("Unable to determine PCI Express bandwidth.\n");
-		return;
-	}
-
-	e_dev_info("PCI Express bandwidth of %dGT/s available\n",
-		   max_gts);
-	e_dev_info("(Speed:%s, Width: x%d, Encoding Loss:%s)\n",
-		   (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
-		    speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
-		    speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
-		    "Unknown"),
-		   width,
-		   (speed == PCIE_SPEED_2_5GT ? "20%" :
-		    speed == PCIE_SPEED_5_0GT ? "20%" :
-		    speed == PCIE_SPEED_8_0GT ? "<2%" :
-		    "Unknown"));
-
-	if (max_gts < expected_gts) {
-		e_dev_warn("This is not sufficient for optimal performance of this card.\n");
-		e_dev_warn("For optimal performance, at least %dGT/s of bandwidth is required.\n",
-			expected_gts);
-		e_dev_warn("A slot with more lanes and/or higher speed is suggested.\n");
-	}
+	pcie_print_link_status(pdev);
 }
 
 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
@@ -752,8 +682,8 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter)
 					ring_desc = "";
 				pr_info("T [0x%03X]    %016llX %016llX %016llX %08X %p %016llX %p%s",
 					i,
-					le64_to_cpu(u0->a),
-					le64_to_cpu(u0->b),
+					le64_to_cpu((__force __le64)u0->a),
+					le64_to_cpu((__force __le64)u0->b),
 					(u64)dma_unmap_addr(tx_buffer, dma),
 					dma_unmap_len(tx_buffer, len),
 					tx_buffer->next_to_watch,
@@ -864,15 +794,15 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter)
 				/* Descriptor Done */
 				pr_info("RWB[0x%03X]     %016llX %016llX ---------------- %p%s\n",
 					i,
-					le64_to_cpu(u0->a),
-					le64_to_cpu(u0->b),
+					le64_to_cpu((__force __le64)u0->a),
+					le64_to_cpu((__force __le64)u0->b),
 					rx_buffer_info->skb,
 					ring_desc);
 			} else {
 				pr_info("R  [0x%03X]     %016llX %016llX %016llX %p%s\n",
 					i,
-					le64_to_cpu(u0->a),
-					le64_to_cpu(u0->b),
+					le64_to_cpu((__force __le64)u0->a),
+					le64_to_cpu((__force __le64)u0->b),
 					(u64)rx_buffer_info->dma,
 					rx_buffer_info->skb,
 					ring_desc);
@@ -1216,7 +1146,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
 
 		/* free the skb */
 		if (ring_is_xdp(tx_ring))
-			page_frag_free(tx_buffer->data);
+			xdp_return_frame(tx_buffer->xdpf);
 		else
 			napi_consume_skb(tx_buffer->skb, napi_budget);
 
@@ -1768,15 +1698,14 @@ static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
 	if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP))
 		ixgbe_ipsec_rx(rx_ring, rx_desc, skb);
 
-	skb->protocol = eth_type_trans(skb, dev);
-
 	/* record Rx queue, or update MACVLAN statistics */
 	if (netif_is_ixgbe(dev))
 		skb_record_rx_queue(skb, rx_ring->queue_index);
 	else
 		macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true,
-				 (skb->pkt_type == PACKET_BROADCAST) ||
-				 (skb->pkt_type == PACKET_MULTICAST));
+				 false);
+
+	skb->protocol = eth_type_trans(skb, dev);
 }
 
 static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
@@ -2262,7 +2191,7 @@ static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring,
 #define IXGBE_XDP_TX 2
 
 static int ixgbe_xmit_xdp_ring(struct ixgbe_adapter *adapter,
-			       struct xdp_buff *xdp);
+			       struct xdp_frame *xdpf);
 
 static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,
 				     struct ixgbe_ring *rx_ring,
@@ -2270,6 +2199,7 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,
 {
 	int err, result = IXGBE_XDP_PASS;
 	struct bpf_prog *xdp_prog;
+	struct xdp_frame *xdpf;
 	u32 act;
 
 	rcu_read_lock();
@@ -2278,12 +2208,19 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,
 	if (!xdp_prog)
 		goto xdp_out;
 
+	prefetchw(xdp->data_hard_start); /* xdp_frame write */
+
 	act = bpf_prog_run_xdp(xdp_prog, xdp);
 	switch (act) {
 	case XDP_PASS:
 		break;
 	case XDP_TX:
-		result = ixgbe_xmit_xdp_ring(adapter, xdp);
+		xdpf = convert_to_xdp_frame(xdp);
+		if (unlikely(!xdpf)) {
+			result = IXGBE_XDP_CONSUMED;
+			break;
+		}
+		result = ixgbe_xmit_xdp_ring(adapter, xdpf);
 		break;
 	case XDP_REDIRECT:
 		err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog);
@@ -4211,7 +4148,8 @@ static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
-	u32 reg_offset, vf_shift;
+	u16 pool = adapter->num_rx_pools;
+	u32 reg_offset, vf_shift, vmolr;
 	u32 gcr_ext, vmdctl;
 	int i;
 
@@ -4225,6 +4163,13 @@ static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
 	vmdctl |= IXGBE_VT_CTL_REPLEN;
 	IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
 
+	/* accept untagged packets until a vlan tag is
+	 * specifically set for the VMDQ queue/pool
+	 */
+	vmolr = IXGBE_VMOLR_AUPE;
+	while (pool--)
+		IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(pool)), vmolr);
+
 	vf_shift = VMDQ_P(0) % 32;
 	reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0;
 
@@ -4892,36 +4837,6 @@ int ixgbe_del_mac_filter(struct ixgbe_adapter *adapter,
 	return -ENOMEM;
 }
 
-/**
- * ixgbe_write_uc_addr_list - write unicast addresses to RAR table
- * @netdev: network interface device structure
- * @vfn: pool to associate with unicast addresses
- *
- * Writes unicast address list to the RAR table.
- * Returns: -ENOMEM on failure/insufficient address space
- *                0 on no addresses written
- *                X on writing X addresses to the RAR table
- **/
-static int ixgbe_write_uc_addr_list(struct net_device *netdev, int vfn)
-{
-	struct ixgbe_adapter *adapter = netdev_priv(netdev);
-	int count = 0;
-
-	/* return ENOMEM indicating insufficient memory for addresses */
-	if (netdev_uc_count(netdev) > ixgbe_available_rars(adapter, vfn))
-		return -ENOMEM;
-
-	if (!netdev_uc_empty(netdev)) {
-		struct netdev_hw_addr *ha;
-		netdev_for_each_uc_addr(ha, netdev) {
-			ixgbe_del_mac_filter(adapter, ha->addr, vfn);
-			ixgbe_add_mac_filter(adapter, ha->addr, vfn);
-			count++;
-		}
-	}
-	return count;
-}
-
 static int ixgbe_uc_sync(struct net_device *netdev, const unsigned char *addr)
 {
 	struct ixgbe_adapter *adapter = netdev_priv(netdev);
@@ -5301,29 +5216,6 @@ static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
 	spin_unlock(&adapter->fdir_perfect_lock);
 }
 
-static void ixgbe_macvlan_set_rx_mode(struct net_device *dev, unsigned int pool,
-				      struct ixgbe_adapter *adapter)
-{
-	struct ixgbe_hw *hw = &adapter->hw;
-	u32 vmolr;
-
-	/* No unicast promiscuous support for VMDQ devices. */
-	vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
-	vmolr |= (IXGBE_VMOLR_ROMPE | IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE);
-
-	/* clear the affected bit */
-	vmolr &= ~IXGBE_VMOLR_MPE;
-
-	if (dev->flags & IFF_ALLMULTI) {
-		vmolr |= IXGBE_VMOLR_MPE;
-	} else {
-		vmolr |= IXGBE_VMOLR_ROMPE;
-		hw->mac.ops.update_mc_addr_list(hw, dev);
-	}
-	ixgbe_write_uc_addr_list(adapter->netdev, pool);
-	IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
-}
-
 /**
  * ixgbe_clean_rx_ring - Free Rx Buffers per Queue
  * @rx_ring: ring to free buffers from
@@ -5376,21 +5268,17 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
 	rx_ring->next_to_use = 0;
 }
 
-static int ixgbe_fwd_ring_up(struct net_device *vdev,
+static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter,
 			     struct ixgbe_fwd_adapter *accel)
 {
-	struct ixgbe_adapter *adapter = accel->real_adapter;
+	struct net_device *vdev = accel->netdev;
 	int i, baseq, err;
 
-	if (!test_bit(accel->pool, adapter->fwd_bitmask))
-		return 0;
-
 	baseq = accel->pool * adapter->num_rx_queues_per_pool;
 	netdev_dbg(vdev, "pool %i:%i queues %i:%i\n",
 		   accel->pool, adapter->num_rx_pools,
 		   baseq, baseq + adapter->num_rx_queues_per_pool);
 
-	accel->netdev = vdev;
 	accel->rx_base_queue = baseq;
 	accel->tx_base_queue = baseq;
 
@@ -5407,26 +5295,36 @@ static int ixgbe_fwd_ring_up(struct net_device *vdev,
 	 */
 	err = ixgbe_add_mac_filter(adapter, vdev->dev_addr,
 				   VMDQ_P(accel->pool));
-	if (err >= 0) {
-		ixgbe_macvlan_set_rx_mode(vdev, accel->pool, adapter);
+	if (err >= 0)
 		return 0;
-	}
+
+	/* if we cannot add the MAC rule then disable the offload */
+	macvlan_release_l2fw_offload(vdev);
 
 	for (i = 0; i < adapter->num_rx_queues_per_pool; i++)
 		adapter->rx_ring[baseq + i]->netdev = NULL;
 
+	netdev_err(vdev, "L2FW offload disabled due to L2 filter error\n");
+
+	clear_bit(accel->pool, adapter->fwd_bitmask);
+	kfree(accel);
+
 	return err;
 }
 
-static int ixgbe_upper_dev_walk(struct net_device *upper, void *data)
+static int ixgbe_macvlan_up(struct net_device *vdev, void *data)
 {
-	if (netif_is_macvlan(upper)) {
-		struct macvlan_dev *dfwd = netdev_priv(upper);
-		struct ixgbe_fwd_adapter *vadapter = dfwd->fwd_priv;
+	struct ixgbe_adapter *adapter = data;
+	struct ixgbe_fwd_adapter *accel;
 
-		if (dfwd->fwd_priv)
-			ixgbe_fwd_ring_up(upper, vadapter);
-	}
+	if (!netif_is_macvlan(vdev))
+		return 0;
+
+	accel = macvlan_accel_priv(vdev);
+	if (!accel)
+		return 0;
+
+	ixgbe_fwd_ring_up(adapter, accel);
 
 	return 0;
 }
@@ -5434,7 +5332,7 @@ static int ixgbe_upper_dev_walk(struct net_device *upper, void *data)
 static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter)
 {
 	netdev_walk_all_upper_dev_rcu(adapter->netdev,
-				      ixgbe_upper_dev_walk, NULL);
+				      ixgbe_macvlan_up, adapter);
 }
 
 static void ixgbe_configure(struct ixgbe_adapter *adapter)
@@ -5797,7 +5695,7 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
 
 		/* Free all the Tx ring sk_buffs */
 		if (ring_is_xdp(tx_ring))
-			page_frag_free(tx_buffer->data);
+			xdp_return_frame(tx_buffer->xdpf);
 		else
 			dev_kfree_skb_any(tx_buffer->skb);
 
@@ -6370,7 +6268,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter,
 	struct device *dev = rx_ring->dev;
 	int orig_node = dev_to_node(dev);
 	int ring_node = -1;
-	int size;
+	int size, err;
 
 	size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
 
@@ -6407,6 +6305,13 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter,
 			     rx_ring->queue_index) < 0)
 		goto err;
 
+	err = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq,
+					 MEM_TYPE_PAGE_SHARED, NULL);
+	if (err) {
+		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
+		goto err;
+	}
+
 	rx_ring->xdp_prog = adapter->xdp_prog;
 
 	return 0;
@@ -7801,7 +7706,7 @@ static int ixgbe_tso(struct ixgbe_ring *tx_ring,
 
 	/* remove payload length from inner checksum */
 	paylen = skb->len - l4_offset;
-	csum_replace_by_diff(&l4.tcp->check, htonl(paylen));
+	csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
 
 	/* update gso size and bytecount with header size */
 	first->gso_segs = skb_shinfo(skb)->gso_segs;
@@ -8336,7 +8241,7 @@ static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb,
 }
 
 static int ixgbe_xmit_xdp_ring(struct ixgbe_adapter *adapter,
-			       struct xdp_buff *xdp)
+			       struct xdp_frame *xdpf)
 {
 	struct ixgbe_ring *ring = adapter->xdp_ring[smp_processor_id()];
 	struct ixgbe_tx_buffer *tx_buffer;
@@ -8345,12 +8250,12 @@ static int ixgbe_xmit_xdp_ring(struct ixgbe_adapter *adapter,
 	dma_addr_t dma;
 	u16 i;
 
-	len = xdp->data_end - xdp->data;
+	len = xdpf->len;
 
 	if (unlikely(!ixgbe_desc_unused(ring)))
 		return IXGBE_XDP_CONSUMED;
 
-	dma = dma_map_single(ring->dev, xdp->data, len, DMA_TO_DEVICE);
+	dma = dma_map_single(ring->dev, xdpf->data, len, DMA_TO_DEVICE);
 	if (dma_mapping_error(ring->dev, dma))
 		return IXGBE_XDP_CONSUMED;
 
@@ -8365,7 +8270,8 @@ static int ixgbe_xmit_xdp_ring(struct ixgbe_adapter *adapter,
 
 	dma_unmap_len_set(tx_buffer, len, len);
 	dma_unmap_addr_set(tx_buffer, dma, dma);
-	tx_buffer->data = xdp->data;
+	tx_buffer->xdpf = xdpf;
+
 	tx_desc->read.buffer_addr = cpu_to_le64(dma);
 
 	/* put descriptor type bits */
@@ -8827,6 +8733,49 @@ static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter)
 }
 
 #endif /* CONFIG_IXGBE_DCB */
+static int ixgbe_reassign_macvlan_pool(struct net_device *vdev, void *data)
+{
+	struct ixgbe_adapter *adapter = data;
+	struct ixgbe_fwd_adapter *accel;
+	int pool;
+
+	/* we only care about macvlans... */
+	if (!netif_is_macvlan(vdev))
+		return 0;
+
+	/* that have hardware offload enabled... */
+	accel = macvlan_accel_priv(vdev);
+	if (!accel)
+		return 0;
+
+	/* If we can relocate to a different bit do so */
+	pool = find_first_zero_bit(adapter->fwd_bitmask, adapter->num_rx_pools);
+	if (pool < adapter->num_rx_pools) {
+		set_bit(pool, adapter->fwd_bitmask);
+		accel->pool = pool;
+		return 0;
+	}
+
+	/* if we cannot find a free pool then disable the offload */
+	netdev_err(vdev, "L2FW offload disabled due to lack of queue resources\n");
+	macvlan_release_l2fw_offload(vdev);
+	kfree(accel);
+
+	return 0;
+}
+
+static void ixgbe_defrag_macvlan_pools(struct net_device *dev)
+{
+	struct ixgbe_adapter *adapter = netdev_priv(dev);
+
+	/* flush any stale bits out of the fwd bitmask */
+	bitmap_clear(adapter->fwd_bitmask, 1, 63);
+
+	/* walk through upper devices reassigning pools */
+	netdev_walk_all_upper_dev_rcu(dev, ixgbe_reassign_macvlan_pool,
+				      adapter);
+}
+
 /**
  * ixgbe_setup_tc - configure net_device for multiple traffic classes
  *
@@ -8894,6 +8843,8 @@ int ixgbe_setup_tc(struct net_device *dev, u8 tc)
 #endif /* CONFIG_IXGBE_DCB */
 	ixgbe_init_interrupt_scheme(adapter);
 
+	ixgbe_defrag_macvlan_pools(dev);
+
 	if (netif_running(dev))
 		return ixgbe_open(dev);
 
@@ -8998,13 +8949,12 @@ struct upper_walk_data {
 static int get_macvlan_queue(struct net_device *upper, void *_data)
 {
 	if (netif_is_macvlan(upper)) {
-		struct macvlan_dev *dfwd = netdev_priv(upper);
-		struct ixgbe_fwd_adapter *vadapter = dfwd->fwd_priv;
+		struct ixgbe_fwd_adapter *vadapter = macvlan_accel_priv(upper);
 		struct upper_walk_data *data = _data;
 		struct ixgbe_adapter *adapter = data->adapter;
 		int ifindex = data->ifindex;
 
-		if (vadapter && vadapter->netdev->ifindex == ifindex) {
+		if (vadapter && upper->ifindex == ifindex) {
 			data->queue = adapter->rx_ring[vadapter->rx_base_queue]->reg_idx;
 			data->action = data->queue;
 			return 1;
@@ -9109,7 +9059,8 @@ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input,
 
 		for (j = 0; field_ptr[j].val; j++) {
 			if (field_ptr[j].off == off) {
-				field_ptr[j].val(input, mask, val, m);
+				field_ptr[j].val(input, mask, (__force u32)val,
+						 (__force u32)m);
 				input->filter.formatted.flow_type |=
 					field_ptr[j].type;
 				found_entry = true;
@@ -9118,8 +9069,10 @@ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input,
 		}
 		if (nexthdr) {
 			if (nexthdr->off == cls->knode.sel->keys[i].off &&
-			    nexthdr->val == cls->knode.sel->keys[i].val &&
-			    nexthdr->mask == cls->knode.sel->keys[i].mask)
+			    nexthdr->val ==
+			    (__force u32)cls->knode.sel->keys[i].val &&
+			    nexthdr->mask ==
+			    (__force u32)cls->knode.sel->keys[i].mask)
 				found_jump_field = true;
 			else
 				continue;
@@ -9223,7 +9176,8 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter,
 		for (i = 0; nexthdr[i].jump; i++) {
 			if (nexthdr[i].o != cls->knode.sel->offoff ||
 			    nexthdr[i].s != cls->knode.sel->offshift ||
-			    nexthdr[i].m != cls->knode.sel->offmask)
+			    nexthdr[i].m !=
+			    (__force u32)cls->knode.sel->offmask)
 				return err;
 
 			jump = kzalloc(sizeof(*jump), GFP_KERNEL);
@@ -9444,6 +9398,22 @@ static netdev_features_t ixgbe_fix_features(struct net_device *netdev,
 	return features;
 }
 
+static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter)
+{
+	int rss = min_t(int, ixgbe_max_rss_indices(adapter),
+			num_online_cpus());
+
+	/* go back to full RSS if we're not running SR-IOV */
+	if (!adapter->ring_feature[RING_F_VMDQ].offset)
+		adapter->flags &= ~(IXGBE_FLAG_VMDQ_ENABLED |
+				    IXGBE_FLAG_SRIOV_ENABLED);
+
+	adapter->ring_feature[RING_F_RSS].limit = rss;
+	adapter->ring_feature[RING_F_VMDQ].limit = 1;
+
+	ixgbe_setup_tc(adapter->netdev, adapter->hw_tcs);
+}
+
 static int ixgbe_set_features(struct net_device *netdev,
 			      netdev_features_t features)
 {
@@ -9524,7 +9494,9 @@ static int ixgbe_set_features(struct net_device *netdev,
 		}
 	}
 
-	if (need_reset)
+	if ((changed & NETIF_F_HW_L2FW_DOFFLOAD) && adapter->num_rx_pools > 1)
+		ixgbe_reset_l2fw_offload(adapter);
+	else if (need_reset)
 		ixgbe_do_reset(netdev);
 	else if (changed & (NETIF_F_HW_VLAN_CTAG_RX |
 			    NETIF_F_HW_VLAN_CTAG_FILTER))
@@ -9787,71 +9759,98 @@ static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
 
 static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev)
 {
-	struct ixgbe_fwd_adapter *fwd_adapter = NULL;
 	struct ixgbe_adapter *adapter = netdev_priv(pdev);
-	int used_pools = adapter->num_vfs + adapter->num_rx_pools;
+	struct ixgbe_fwd_adapter *accel;
 	int tcs = adapter->hw_tcs ? : 1;
-	unsigned int limit;
 	int pool, err;
 
-	/* Hardware has a limited number of available pools. Each VF, and the
-	 * PF require a pool. Check to ensure we don't attempt to use more
-	 * then the available number of pools.
+	/* The hardware supported by ixgbe only filters on the destination MAC
+	 * address. In order to avoid issues we only support offloading modes
+	 * where the hardware can actually provide the functionality.
 	 */
-	if (used_pools >= IXGBE_MAX_VF_FUNCTIONS)
-		return ERR_PTR(-EINVAL);
+	if (!macvlan_supports_dest_filter(vdev))
+		return ERR_PTR(-EMEDIUMTYPE);
 
-	if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
-	      adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) ||
-	    (adapter->num_rx_pools > IXGBE_MAX_MACVLANS))
-		return ERR_PTR(-EBUSY);
+	pool = find_first_zero_bit(adapter->fwd_bitmask, adapter->num_rx_pools);
+	if (pool == adapter->num_rx_pools) {
+		u16 used_pools = adapter->num_vfs + adapter->num_rx_pools;
+		u16 reserved_pools;
+
+		if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
+		     adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) ||
+		    adapter->num_rx_pools > IXGBE_MAX_MACVLANS)
+			return ERR_PTR(-EBUSY);
+
+		/* Hardware has a limited number of available pools. Each VF,
+		 * and the PF require a pool. Check to ensure we don't
+		 * attempt to use more then the available number of pools.
+		 */
+		if (used_pools >= IXGBE_MAX_VF_FUNCTIONS)
+			return ERR_PTR(-EBUSY);
 
-	fwd_adapter = kzalloc(sizeof(*fwd_adapter), GFP_KERNEL);
-	if (!fwd_adapter)
-		return ERR_PTR(-ENOMEM);
+		/* Enable VMDq flag so device will be set in VM mode */
+		adapter->flags |= IXGBE_FLAG_VMDQ_ENABLED |
+				  IXGBE_FLAG_SRIOV_ENABLED;
 
-	pool = find_first_zero_bit(adapter->fwd_bitmask, adapter->num_rx_pools);
-	set_bit(pool, adapter->fwd_bitmask);
-	limit = find_last_bit(adapter->fwd_bitmask, adapter->num_rx_pools + 1);
+		/* Try to reserve as many queues per pool as possible,
+		 * we start with the configurations that support 4 queues
+		 * per pools, followed by 2, and then by just 1 per pool.
+		 */
+		if (used_pools < 32 && adapter->num_rx_pools < 16)
+			reserved_pools = min_t(u16,
+					       32 - used_pools,
+					       16 - adapter->num_rx_pools);
+		else if (adapter->num_rx_pools < 32)
+			reserved_pools = min_t(u16,
+					       64 - used_pools,
+					       32 - adapter->num_rx_pools);
+		else
+			reserved_pools = 64 - used_pools;
 
-	/* Enable VMDq flag so device will be set in VM mode */
-	adapter->flags |= IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_SRIOV_ENABLED;
-	adapter->ring_feature[RING_F_VMDQ].limit = limit + 1;
 
-	fwd_adapter->pool = pool;
-	fwd_adapter->real_adapter = adapter;
+		if (!reserved_pools)
+			return ERR_PTR(-EBUSY);
 
-	/* Force reinit of ring allocation with VMDQ enabled */
-	err = ixgbe_setup_tc(pdev, adapter->hw_tcs);
+		adapter->ring_feature[RING_F_VMDQ].limit += reserved_pools;
 
-	if (!err && netif_running(pdev))
-		err = ixgbe_fwd_ring_up(vdev, fwd_adapter);
+		/* Force reinit of ring allocation with VMDQ enabled */
+		err = ixgbe_setup_tc(pdev, adapter->hw_tcs);
+		if (err)
+			return ERR_PTR(err);
 
-	if (!err)
-		return fwd_adapter;
+		if (pool >= adapter->num_rx_pools)
+			return ERR_PTR(-ENOMEM);
+	}
 
-	/* unwind counter and free adapter struct */
-	netdev_info(pdev,
-		    "%s: dfwd hardware acceleration failed\n", vdev->name);
-	clear_bit(pool, adapter->fwd_bitmask);
-	kfree(fwd_adapter);
-	return ERR_PTR(err);
+	accel = kzalloc(sizeof(*accel), GFP_KERNEL);
+	if (!accel)
+		return ERR_PTR(-ENOMEM);
+
+	set_bit(pool, adapter->fwd_bitmask);
+	accel->pool = pool;
+	accel->netdev = vdev;
+
+	if (!netif_running(pdev))
+		return accel;
+
+	err = ixgbe_fwd_ring_up(adapter, accel);
+	if (err)
+		return ERR_PTR(err);
+
+	return accel;
 }
 
 static void ixgbe_fwd_del(struct net_device *pdev, void *priv)
 {
 	struct ixgbe_fwd_adapter *accel = priv;
-	struct ixgbe_adapter *adapter = accel->real_adapter;
+	struct ixgbe_adapter *adapter = netdev_priv(pdev);
 	unsigned int rxbase = accel->rx_base_queue;
-	unsigned int limit, i;
+	unsigned int i;
 
 	/* delete unicast filter associated with offloaded interface */
 	ixgbe_del_mac_filter(adapter, accel->netdev->dev_addr,
 			     VMDQ_P(accel->pool));
 
-	/* disable ability to receive packets for this pool */
-	IXGBE_WRITE_REG(&adapter->hw, IXGBE_VMOLR(accel->pool), 0);
-
 	/* Allow remaining Rx packets to get flushed out of the
 	 * Rx FIFO before we drop the netdev for the ring.
 	 */
@@ -9870,25 +9869,6 @@ static void ixgbe_fwd_del(struct net_device *pdev, void *priv)
 	}
 
 	clear_bit(accel->pool, adapter->fwd_bitmask);
-	limit = find_last_bit(adapter->fwd_bitmask, adapter->num_rx_pools);
-	adapter->ring_feature[RING_F_VMDQ].limit = limit + 1;
-
-	/* go back to full RSS if we're done with our VMQs */
-	if (adapter->ring_feature[RING_F_VMDQ].limit == 1) {
-		int rss = min_t(int, ixgbe_max_rss_indices(adapter),
-				num_online_cpus());
-
-		adapter->flags &= ~IXGBE_FLAG_VMDQ_ENABLED;
-		adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
-		adapter->ring_feature[RING_F_RSS].limit = rss;
-	}
-
-	ixgbe_setup_tc(pdev, adapter->hw_tcs);
-	netdev_dbg(pdev, "pool %i:%i queues %i:%i\n",
-		   accel->pool, adapter->num_rx_pools,
-		   accel->rx_base_queue,
-		   accel->rx_base_queue +
-		   adapter->num_rx_queues_per_pool);
 	kfree(accel);
 }
 
@@ -9970,7 +9950,8 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog)
 		}
 	} else {
 		for (i = 0; i < adapter->num_rx_queues; i++)
-			xchg(&adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog);
+			(void)xchg(&adapter->rx_ring[i]->xdp_prog,
+			    adapter->xdp_prog);
 	}
 
 	if (old_prog)
@@ -9996,11 +9977,13 @@ static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 	}
 }
 
-static int ixgbe_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
+static int ixgbe_xdp_xmit(struct net_device *dev, int n,
+			  struct xdp_frame **frames)
 {
 	struct ixgbe_adapter *adapter = netdev_priv(dev);
 	struct ixgbe_ring *ring;
-	int err;
+	int drops = 0;
+	int i;
 
 	if (unlikely(test_bit(__IXGBE_DOWN, &adapter->state)))
 		return -ENETDOWN;
@@ -10012,11 +9995,18 @@ static int ixgbe_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
 	if (unlikely(!ring))
 		return -ENXIO;
 
-	err = ixgbe_xmit_xdp_ring(adapter, xdp);
-	if (err != IXGBE_XDP_TX)
-		return -ENOSPC;
+	for (i = 0; i < n; i++) {
+		struct xdp_frame *xdpf = frames[i];
+		int err;
 
-	return 0;
+		err = ixgbe_xmit_xdp_ring(adapter, xdpf);
+		if (err != IXGBE_XDP_TX) {
+			xdp_return_frame_rx_napi(xdpf);
+			drops++;
+		}
+	}
+
+	return n - drops;
 }
 
 static void ixgbe_xdp_flush(struct net_device *dev)
@@ -10909,14 +10899,14 @@ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
 	rtnl_lock();
 	netif_device_detach(netdev);
 
+	if (netif_running(netdev))
+		ixgbe_close_suspend(adapter);
+
 	if (state == pci_channel_io_perm_failure) {
 		rtnl_unlock();
 		return PCI_ERS_RESULT_DISCONNECT;
 	}
 
-	if (netif_running(netdev))
-		ixgbe_close_suspend(adapter);
-
 	if (!test_and_set_bit(__IXGBE_DISABLED, &adapter->state))
 		pci_disable_device(pdev);
 	rtnl_unlock();
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_mbx.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_mbx.c
index a0cb84381cd0..5679293e53f7 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_mbx.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_mbx.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include <linux/pci.h>
 #include <linux/delay.h>
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_mbx.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_mbx.h
index c4628b663590..e085b6520dac 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_mbx.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_mbx.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBE_MBX_H_
 #define _IXGBE_MBX_H_
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_model.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_model.h
index 72446644f9fa..1e6cf220f543 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_model.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_model.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel 10 Gigabit PCI Express Linux drive
- * Copyright(c) 2016 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * Contact Information:
- * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBE_MODEL_H_
 #define _IXGBE_MODEL_H_
@@ -53,8 +29,8 @@ static inline int ixgbe_mat_prgm_sip(struct ixgbe_fdir_filter *input,
 				     union ixgbe_atr_input *mask,
 				     u32 val, u32 m)
 {
-	input->filter.formatted.src_ip[0] = val;
-	mask->formatted.src_ip[0] = m;
+	input->filter.formatted.src_ip[0] = (__force __be32)val;
+	mask->formatted.src_ip[0] = (__force __be32)m;
 	return 0;
 }
 
@@ -62,8 +38,8 @@ static inline int ixgbe_mat_prgm_dip(struct ixgbe_fdir_filter *input,
 				     union ixgbe_atr_input *mask,
 				     u32 val, u32 m)
 {
-	input->filter.formatted.dst_ip[0] = val;
-	mask->formatted.dst_ip[0] = m;
+	input->filter.formatted.dst_ip[0] = (__force __be32)val;
+	mask->formatted.dst_ip[0] = (__force __be32)m;
 	return 0;
 }
 
@@ -79,10 +55,10 @@ static inline int ixgbe_mat_prgm_ports(struct ixgbe_fdir_filter *input,
 				       union ixgbe_atr_input *mask,
 				       u32 val, u32 m)
 {
-	input->filter.formatted.src_port = val & 0xffff;
-	mask->formatted.src_port = m & 0xffff;
-	input->filter.formatted.dst_port = val >> 16;
-	mask->formatted.dst_port = m >> 16;
+	input->filter.formatted.src_port = (__force __be16)(val & 0xffff);
+	mask->formatted.src_port = (__force __be16)(m & 0xffff);
+	input->filter.formatted.dst_port = (__force __be16)(val >> 16);
+	mask->formatted.dst_port = (__force __be16)(m >> 16);
 
 	return 0;
 };
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c
index 91bde90f9265..919a7af84b42 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2014 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include <linux/pci.h>
 #include <linux/delay.h>
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.h
index d6a7e77348c5..64e44e01c973 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBE_PHY_H_
 #define _IXGBE_PHY_H_
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c
index f6cc9166082a..b3e0d8bb5cbd 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c
@@ -1,30 +1,6 @@
-/*******************************************************************************
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
 #include "ixgbe.h"
 #include <linux/ptp_classify.h>
 #include <linux/clocksource.h>
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
index 008aa073a679..6f59933cdff7 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2015 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include <linux/types.h>
 #include <linux/module.h>
@@ -266,7 +241,7 @@ int ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
 #endif
 
 	/* Disable VMDq flag so device will be set in VM mode */
-	if (adapter->ring_feature[RING_F_VMDQ].limit == 1) {
+	if (bitmap_weight(adapter->fwd_bitmask, adapter->num_rx_pools) == 1) {
 		adapter->flags &= ~IXGBE_FLAG_VMDQ_ENABLED;
 		adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
 		rss = min_t(int, ixgbe_max_rss_indices(adapter),
@@ -312,7 +287,8 @@ static int ixgbe_pci_sriov_enable(struct pci_dev *dev, int num_vfs)
 	 * other values out of range.
 	 */
 	num_tc = adapter->hw_tcs;
-	num_rx_pools = adapter->num_rx_pools;
+	num_rx_pools = bitmap_weight(adapter->fwd_bitmask,
+				     adapter->num_rx_pools);
 	limit = (num_tc > 4) ? IXGBE_MAX_VFS_8TC :
 		(num_tc > 1) ? IXGBE_MAX_VFS_4TC : IXGBE_MAX_VFS_1TC;
 
@@ -878,14 +854,11 @@ static int ixgbe_vf_reset_msg(struct ixgbe_adapter *adapter, u32 vf)
 
 	/* reply to reset with ack and vf mac address */
 	msgbuf[0] = IXGBE_VF_RESET;
-	if (!is_zero_ether_addr(vf_mac)) {
+	if (!is_zero_ether_addr(vf_mac) && adapter->vfinfo[vf].pf_set_mac) {
 		msgbuf[0] |= IXGBE_VT_MSGTYPE_ACK;
 		memcpy(addr, vf_mac, ETH_ALEN);
 	} else {
 		msgbuf[0] |= IXGBE_VT_MSGTYPE_NACK;
-		dev_warn(&adapter->pdev->dev,
-			 "VF %d has no MAC address assigned, you may have to assign one manually\n",
-			 vf);
 	}
 
 	/*
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
index e30d1f07e891..3ec21923c89c 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2013 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBE_SRIOV_H_
 #define _IXGBE_SRIOV_H_
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sysfs.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sysfs.c
index 24766e125592..204844288c16 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sysfs.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sysfs.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2013 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "ixgbe.h"
 #include "ixgbe_common.h"
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
index 2daa81e6e9b2..e8ed37749ab1 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
@@ -1,31 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBE_TYPE_H_
 #define _IXGBE_TYPE_H_
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c
index b8c5fd2a2115..de563cfd294d 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c
@@ -1,30 +1,5 @@
-/*******************************************************************************
-
-  Intel 10 Gigabit PCI Express Linux driver
-  Copyright(c) 1999 - 2016 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  Linux NICS <linux.nics@intel.com>
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include <linux/pci.h>
 #include <linux/delay.h>
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.h
index 182d640e9f7a..e246c0d2a427 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.h
@@ -1,27 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
- *
- * Intel 10 Gigabit PCI Express Linux driver
- *  Copyright(c) 1999 - 2014 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify it
- *  under the terms and conditions of the GNU General Public License,
- *  version 2, as published by the Free Software Foundation.
- *
- *  This program is distributed in the hope it will be useful, but WITHOUT
- *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- *  more details.
- *
- *  The full GNU General Public License is included in this distribution in
- *  the file called "COPYING".
- *
- *  Contact Information:
- *  Linux NICS <linux.nics@intel.com>
- *  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- *  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- *****************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "ixgbe_type.h"
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
index 9592f3e3e42e..a8148c7126e5 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
@@ -1,26 +1,6 @@
-/*******************************************************************************
- *
- *  Intel 10 Gigabit PCI Express Linux driver
- *  Copyright(c) 1999 - 2016 Intel Corporation.
- *
- *  This program is free software; you can redistribute it and/or modify it
- *  under the terms and conditions of the GNU General Public License,
- *  version 2, as published by the Free Software Foundation.
- *
- *  This program is distributed in the hope it will be useful, but WITHOUT
- *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- *  more details.
- *
- *  The full GNU General Public License is included in this distribution in
- *  the file called "COPYING".
- *
- *  Contact Information:
- *  Linux NICS <linux.nics@intel.com>
- *  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
- *  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- ******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
+
 #include "ixgbe_x540.h"
 #include "ixgbe_type.h"
 #include "ixgbe_common.h"
@@ -898,8 +878,9 @@ static s32 ixgbe_read_ee_hostif_buffer_X550(struct ixgbe_hw *hw,
 		buffer.hdr.req.checksum = FW_DEFAULT_CHECKSUM;
 
 		/* convert offset from words to bytes */
-		buffer.address = cpu_to_be32((offset + current_word) * 2);
-		buffer.length = cpu_to_be16(words_to_read * 2);
+		buffer.address = (__force u32)cpu_to_be32((offset +
+							   current_word) * 2);
+		buffer.length = (__force u16)cpu_to_be16(words_to_read * 2);
 		buffer.pad2 = 0;
 		buffer.pad3 = 0;
 
@@ -1109,9 +1090,9 @@ static s32 ixgbe_read_ee_hostif_X550(struct ixgbe_hw *hw, u16 offset, u16 *data)
 	buffer.hdr.req.checksum = FW_DEFAULT_CHECKSUM;
 
 	/* convert offset from words to bytes */
-	buffer.address = cpu_to_be32(offset * 2);
+	buffer.address = (__force u32)cpu_to_be32(offset * 2);
 	/* one word */
-	buffer.length = cpu_to_be16(sizeof(u16));
+	buffer.length = (__force u16)cpu_to_be16(sizeof(u16));
 
 	status = hw->mac.ops.acquire_swfw_sync(hw, mask);
 	if (status)
diff --git a/drivers/net/ethernet/intel/ixgbevf/Makefile b/drivers/net/ethernet/intel/ixgbevf/Makefile
index bb47814cfa90..aba1e6a37a6a 100644
--- a/drivers/net/ethernet/intel/ixgbevf/Makefile
+++ b/drivers/net/ethernet/intel/ixgbevf/Makefile
@@ -1,31 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-################################################################################
-#
-# Intel 82599 Virtual Function driver
-# Copyright(c) 1999 - 2012 Intel Corporation.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms and conditions of the GNU General Public License,
-# version 2, as published by the Free Software Foundation.
-#
-# This program is distributed in the hope it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# The full GNU General Public License is included in this distribution in
-# the file called "COPYING".
-#
-# Contact Information:
-# e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-# Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-#
-################################################################################
-
+# Copyright(c) 1999 - 2018 Intel Corporation.
 #
 # Makefile for the Intel(R) 82599 VF ethernet driver
 #
diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h
index 71c828842b11..700d8eb2f6f8 100644
--- a/drivers/net/ethernet/intel/ixgbevf/defines.h
+++ b/drivers/net/ethernet/intel/ixgbevf/defines.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 82599 Virtual Function driver
-  Copyright(c) 1999 - 2015 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBEVF_DEFINES_H_
 #define _IXGBEVF_DEFINES_H_
diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c
index 8e7d6c6f5c92..e7813d76527c 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c
@@ -1,28 +1,5 @@
-/*******************************************************************************
-
-  Intel 82599 Virtual Function driver
-  Copyright(c) 1999 - 2018 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 /* ethtool support for ixgbevf */
 
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
index 447ce1d5e0e3..70c75681495f 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 82599 Virtual Function driver
-  Copyright(c) 1999 - 2018 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBEVF_H_
 #define _IXGBEVF_H_
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index 850f8af95e49..083041129539 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -1,28 +1,5 @@
-/*******************************************************************************
-
-  Intel 82599 Virtual Function driver
-  Copyright(c) 1999 - 2018 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 /******************************************************************************
  Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
@@ -4187,6 +4164,7 @@ static int ixgbevf_set_mac(struct net_device *netdev, void *p)
 		return -EPERM;
 
 	ether_addr_copy(hw->mac.addr, addr->sa_data);
+	ether_addr_copy(hw->mac.perm_addr, addr->sa_data);
 	ether_addr_copy(netdev->dev_addr, addr->sa_data);
 
 	return 0;
@@ -4770,14 +4748,14 @@ static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
 	rtnl_lock();
 	netif_device_detach(netdev);
 
+	if (netif_running(netdev))
+		ixgbevf_close_suspend(adapter);
+
 	if (state == pci_channel_io_perm_failure) {
 		rtnl_unlock();
 		return PCI_ERS_RESULT_DISCONNECT;
 	}
 
-	if (netif_running(netdev))
-		ixgbevf_close_suspend(adapter);
-
 	if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state))
 		pci_disable_device(pdev);
 	rtnl_unlock();
diff --git a/drivers/net/ethernet/intel/ixgbevf/mbx.c b/drivers/net/ethernet/intel/ixgbevf/mbx.c
index 2819abc454c7..6bc1953263b9 100644
--- a/drivers/net/ethernet/intel/ixgbevf/mbx.c
+++ b/drivers/net/ethernet/intel/ixgbevf/mbx.c
@@ -1,28 +1,5 @@
-/*******************************************************************************
-
-  Intel 82599 Virtual Function driver
-  Copyright(c) 1999 - 2015 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "mbx.h"
 #include "ixgbevf.h"
diff --git a/drivers/net/ethernet/intel/ixgbevf/mbx.h b/drivers/net/ethernet/intel/ixgbevf/mbx.h
index 5ec947fe3d09..bfd9ae150808 100644
--- a/drivers/net/ethernet/intel/ixgbevf/mbx.h
+++ b/drivers/net/ethernet/intel/ixgbevf/mbx.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 82599 Virtual Function driver
-  Copyright(c) 1999 - 2015 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBE_MBX_H_
 #define _IXGBE_MBX_H_
diff --git a/drivers/net/ethernet/intel/ixgbevf/regs.h b/drivers/net/ethernet/intel/ixgbevf/regs.h
index 278f73980501..68d16ae5b65a 100644
--- a/drivers/net/ethernet/intel/ixgbevf/regs.h
+++ b/drivers/net/ethernet/intel/ixgbevf/regs.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 82599 Virtual Function driver
-  Copyright(c) 1999 - 2015 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef _IXGBEVF_REGS_H_
 #define _IXGBEVF_REGS_H_
diff --git a/drivers/net/ethernet/intel/ixgbevf/vf.c b/drivers/net/ethernet/intel/ixgbevf/vf.c
index 38d3a327c1bc..bf0577e819e1 100644
--- a/drivers/net/ethernet/intel/ixgbevf/vf.c
+++ b/drivers/net/ethernet/intel/ixgbevf/vf.c
@@ -1,28 +1,5 @@
-/*******************************************************************************
-
-  Intel 82599 Virtual Function driver
-  Copyright(c) 1999 - 2015 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #include "vf.h"
 #include "ixgbevf.h"
diff --git a/drivers/net/ethernet/intel/ixgbevf/vf.h b/drivers/net/ethernet/intel/ixgbevf/vf.h
index 194fbdaa4519..d1e9e306653b 100644
--- a/drivers/net/ethernet/intel/ixgbevf/vf.h
+++ b/drivers/net/ethernet/intel/ixgbevf/vf.h
@@ -1,29 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*******************************************************************************
-
-  Intel 82599 Virtual Function driver
-  Copyright(c) 1999 - 2015 Intel Corporation.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms and conditions of the GNU General Public License,
-  version 2, as published by the Free Software Foundation.
-
-  This program is distributed in the hope it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  You should have received a copy of the GNU General Public License along with
-  this program; if not, see <http://www.gnu.org/licenses/>.
-
-  The full GNU General Public License is included in this distribution in
-  the file called "COPYING".
-
-  Contact Information:
-  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-*******************************************************************************/
+/* Copyright(c) 1999 - 2018 Intel Corporation. */
 
 #ifndef __IXGBE_VF_H__
 #define __IXGBE_VF_H__
diff --git a/drivers/net/ethernet/marvell/Kconfig b/drivers/net/ethernet/marvell/Kconfig
index ebe5c9148935..cc2f7701e71e 100644
--- a/drivers/net/ethernet/marvell/Kconfig
+++ b/drivers/net/ethernet/marvell/Kconfig
@@ -86,6 +86,7 @@ config MVPP2
 	depends on ARCH_MVEBU || COMPILE_TEST
 	depends on HAS_DMA
 	select MVMDIO
+	select PHYLINK
 	---help---
 	  This driver supports the network interface units in the
 	  Marvell ARMADA 375, 7K and 8K SoCs.
diff --git a/drivers/net/ethernet/marvell/mvmdio.c b/drivers/net/ethernet/marvell/mvmdio.c
index 0495487f7b42..c5dac6bd2be4 100644
--- a/drivers/net/ethernet/marvell/mvmdio.c
+++ b/drivers/net/ethernet/marvell/mvmdio.c
@@ -348,10 +348,7 @@ static int orion_mdio_probe(struct platform_device *pdev)
 		goto out_mdio;
 	}
 
-	if (pdev->dev.of_node)
-		ret = of_mdiobus_register(bus, pdev->dev.of_node);
-	else
-		ret = mdiobus_register(bus);
+	ret = of_mdiobus_register(bus, pdev->dev.of_node);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Cannot register MDIO bus (%d)\n", ret);
 		goto out_mdio;
diff --git a/drivers/net/ethernet/marvell/mvpp2.c b/drivers/net/ethernet/marvell/mvpp2.c
index 6f410235987c..6847cd431aa0 100644
--- a/drivers/net/ethernet/marvell/mvpp2.c
+++ b/drivers/net/ethernet/marvell/mvpp2.c
@@ -29,6 +29,7 @@
 #include <linux/of_address.h>
 #include <linux/of_device.h>
 #include <linux/phy.h>
+#include <linux/phylink.h>
 #include <linux/phy/phy.h>
 #include <linux/clk.h>
 #include <linux/hrtimer.h>
@@ -359,15 +360,23 @@
 #define     MVPP2_GMAC_FORCE_LINK_PASS		BIT(1)
 #define     MVPP2_GMAC_IN_BAND_AUTONEG		BIT(2)
 #define     MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS	BIT(3)
+#define     MVPP2_GMAC_IN_BAND_RESTART_AN	BIT(4)
 #define     MVPP2_GMAC_CONFIG_MII_SPEED	BIT(5)
 #define     MVPP2_GMAC_CONFIG_GMII_SPEED	BIT(6)
 #define     MVPP2_GMAC_AN_SPEED_EN		BIT(7)
 #define     MVPP2_GMAC_FC_ADV_EN		BIT(9)
+#define     MVPP2_GMAC_FC_ADV_ASM_EN		BIT(10)
 #define     MVPP2_GMAC_FLOW_CTRL_AUTONEG	BIT(11)
 #define     MVPP2_GMAC_CONFIG_FULL_DUPLEX	BIT(12)
 #define     MVPP2_GMAC_AN_DUPLEX_EN		BIT(13)
 #define MVPP2_GMAC_STATUS0			0x10
 #define     MVPP2_GMAC_STATUS0_LINK_UP		BIT(0)
+#define     MVPP2_GMAC_STATUS0_GMII_SPEED	BIT(1)
+#define     MVPP2_GMAC_STATUS0_MII_SPEED	BIT(2)
+#define     MVPP2_GMAC_STATUS0_FULL_DUPLEX	BIT(3)
+#define     MVPP2_GMAC_STATUS0_RX_PAUSE		BIT(6)
+#define     MVPP2_GMAC_STATUS0_TX_PAUSE		BIT(7)
+#define     MVPP2_GMAC_STATUS0_AN_COMPLETE	BIT(11)
 #define MVPP2_GMAC_PORT_FIFO_CFG_1_REG		0x1c
 #define     MVPP2_GMAC_TX_FIFO_MIN_TH_OFFS	6
 #define     MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK	0x1fc0
@@ -379,6 +388,8 @@
 #define     MVPP22_GMAC_INT_MASK_LINK_STAT	BIT(1)
 #define MVPP22_GMAC_CTRL_4_REG			0x90
 #define     MVPP22_CTRL4_EXT_PIN_GMII_SEL	BIT(0)
+#define     MVPP22_CTRL4_RX_FC_EN		BIT(3)
+#define     MVPP22_CTRL4_TX_FC_EN		BIT(4)
 #define     MVPP22_CTRL4_DP_CLK_SEL		BIT(5)
 #define     MVPP22_CTRL4_SYNC_BYPASS_DIS	BIT(6)
 #define     MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE	BIT(7)
@@ -392,6 +403,7 @@
 #define     MVPP22_XLG_CTRL0_PORT_EN		BIT(0)
 #define     MVPP22_XLG_CTRL0_MAC_RESET_DIS	BIT(1)
 #define     MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN	BIT(7)
+#define     MVPP22_XLG_CTRL0_TX_FLOW_CTRL_EN	BIT(8)
 #define     MVPP22_XLG_CTRL0_MIB_CNT_DIS	BIT(14)
 #define MVPP22_XLG_CTRL1_REG			0x104
 #define     MVPP22_XLG_CTRL1_FRAMESIZELIMIT_OFFS	0
@@ -413,6 +425,7 @@
 #define     MVPP22_XLG_CTRL4_FWD_FC		BIT(5)
 #define     MVPP22_XLG_CTRL4_FWD_PFC		BIT(6)
 #define     MVPP22_XLG_CTRL4_MACMODSELECT_GMAC	BIT(12)
+#define     MVPP22_XLG_CTRL4_EN_IDLE_CHECK	BIT(14)
 
 /* SMI registers. PPv2.2 only, relative to priv->iface_base. */
 #define MVPP22_SMI_MISC_CFG_REG			0x1204
@@ -1017,6 +1030,9 @@ struct mvpp2_port {
 	/* Firmware node associated to the port */
 	struct fwnode_handle *fwnode;
 
+	/* Is a PHY always connected to the port */
+	bool has_phy;
+
 	/* Per-port registers' base address */
 	void __iomem *base;
 	void __iomem *stats_base;
@@ -1044,12 +1060,11 @@ struct mvpp2_port {
 	struct mutex gather_stats_lock;
 	struct delayed_work stats_work;
 
+	struct device_node *of_node;
+
 	phy_interface_t phy_interface;
-	struct device_node *phy_node;
+	struct phylink *phylink;
 	struct phy *comphy;
-	unsigned int link;
-	unsigned int duplex;
-	unsigned int speed;
 
 	struct mvpp2_bm_pool *pool_long;
 	struct mvpp2_bm_pool *pool_short;
@@ -1338,6 +1353,12 @@ struct mvpp2_bm_pool {
 	 (addr) < (txq_pcpu)->tso_headers_dma + \
 	 (txq_pcpu)->size * TSO_HEADER_SIZE)
 
+/* The prototype is added here to be used in start_dev when using ACPI. This
+ * will be removed once phylink is used for all modes (dt+ACPI).
+ */
+static void mvpp2_mac_config(struct net_device *dev, unsigned int mode,
+			     const struct phylink_link_state *state);
+
 /* Queue modes */
 #define MVPP2_QDIST_SINGLE_MODE	0
 #define MVPP2_QDIST_MULTI_MODE	1
@@ -1735,7 +1756,6 @@ static void mvpp2_prs_tcam_ai_update(struct mvpp2_prs_entry *pe,
 	int i, ai_idx = MVPP2_PRS_TCAM_AI_BYTE;
 
 	for (i = 0; i < MVPP2_PRS_AI_BITS; i++) {
-
 		if (!(enable & BIT(i)))
 			continue;
 
@@ -1819,7 +1839,6 @@ static void mvpp2_prs_sram_ai_update(struct mvpp2_prs_entry *pe,
 	int ai_off = MVPP2_PRS_SRAM_AI_OFFS;
 
 	for (i = 0; i < MVPP2_PRS_SRAM_AI_CTRL_BITS; i++) {
-
 		if (!(mask & BIT(i)))
 			continue;
 
@@ -2109,6 +2128,9 @@ static void mvpp2_prs_dsa_tag_set(struct mvpp2 *priv, int port, bool add,
 				mvpp2_prs_sram_ai_update(&pe, 0,
 							MVPP2_PRS_SRAM_AI_MASK);
 
+			/* Set result info bits to 'single vlan' */
+			mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_VLAN_SINGLE,
+						 MVPP2_PRS_RI_VLAN_MASK);
 			/* If packet is tagged continue check vid filtering */
 			mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_VID);
 		} else {
@@ -4849,6 +4871,8 @@ static int mvpp22_gop_init(struct mvpp2_port *port)
 		mvpp22_gop_init_rgmii(port);
 		break;
 	case PHY_INTERFACE_MODE_SGMII:
+	case PHY_INTERFACE_MODE_1000BASEX:
+	case PHY_INTERFACE_MODE_2500BASEX:
 		mvpp22_gop_init_sgmii(port);
 		break;
 	case PHY_INTERFACE_MODE_10GKR:
@@ -4886,7 +4910,9 @@ static void mvpp22_gop_unmask_irq(struct mvpp2_port *port)
 	u32 val;
 
 	if (phy_interface_mode_is_rgmii(port->phy_interface) ||
-	    port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+	    port->phy_interface == PHY_INTERFACE_MODE_SGMII ||
+	    port->phy_interface == PHY_INTERFACE_MODE_1000BASEX ||
+	    port->phy_interface == PHY_INTERFACE_MODE_2500BASEX) {
 		/* Enable the GMAC link status irq for this port */
 		val = readl(port->base + MVPP22_GMAC_INT_SUM_MASK);
 		val |= MVPP22_GMAC_INT_SUM_MASK_LINK_STAT;
@@ -4911,12 +4937,14 @@ static void mvpp22_gop_mask_irq(struct mvpp2_port *port)
 	if (port->gop_id == 0) {
 		val = readl(port->base + MVPP22_XLG_EXT_INT_MASK);
 		val &= ~(MVPP22_XLG_EXT_INT_MASK_XLG |
-		         MVPP22_XLG_EXT_INT_MASK_GIG);
+			 MVPP22_XLG_EXT_INT_MASK_GIG);
 		writel(val, port->base + MVPP22_XLG_EXT_INT_MASK);
 	}
 
 	if (phy_interface_mode_is_rgmii(port->phy_interface) ||
-	    port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+	    port->phy_interface == PHY_INTERFACE_MODE_SGMII ||
+	    port->phy_interface == PHY_INTERFACE_MODE_1000BASEX ||
+	    port->phy_interface == PHY_INTERFACE_MODE_2500BASEX) {
 		val = readl(port->base + MVPP22_GMAC_INT_SUM_MASK);
 		val &= ~MVPP22_GMAC_INT_SUM_MASK_LINK_STAT;
 		writel(val, port->base + MVPP22_GMAC_INT_SUM_MASK);
@@ -4928,7 +4956,9 @@ static void mvpp22_gop_setup_irq(struct mvpp2_port *port)
 	u32 val;
 
 	if (phy_interface_mode_is_rgmii(port->phy_interface) ||
-	    port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+	    port->phy_interface == PHY_INTERFACE_MODE_SGMII ||
+	    port->phy_interface == PHY_INTERFACE_MODE_1000BASEX ||
+	    port->phy_interface == PHY_INTERFACE_MODE_2500BASEX) {
 		val = readl(port->base + MVPP22_GMAC_INT_MASK);
 		val |= MVPP22_GMAC_INT_MASK_LINK_STAT;
 		writel(val, port->base + MVPP22_GMAC_INT_MASK);
@@ -4943,6 +4973,16 @@ static void mvpp22_gop_setup_irq(struct mvpp2_port *port)
 	mvpp22_gop_unmask_irq(port);
 }
 
+/* Sets the PHY mode of the COMPHY (which configures the serdes lanes).
+ *
+ * The PHY mode used by the PPv2 driver comes from the network subsystem, while
+ * the one given to the COMPHY comes from the generic PHY subsystem. Hence they
+ * differ.
+ *
+ * The COMPHY configures the serdes lanes regardless of the actual use of the
+ * lanes by the physical layer. This is why configurations like
+ * "PPv2 (2500BaseX) - COMPHY (2500SGMII)" are valid.
+ */
 static int mvpp22_comphy_init(struct mvpp2_port *port)
 {
 	enum phy_mode mode;
@@ -4953,8 +4993,12 @@ static int mvpp22_comphy_init(struct mvpp2_port *port)
 
 	switch (port->phy_interface) {
 	case PHY_INTERFACE_MODE_SGMII:
+	case PHY_INTERFACE_MODE_1000BASEX:
 		mode = PHY_MODE_SGMII;
 		break;
+	case PHY_INTERFACE_MODE_2500BASEX:
+		mode = PHY_MODE_2500SGMII;
+		break;
 	case PHY_INTERFACE_MODE_10GKR:
 		mode = PHY_MODE_10GKR;
 		break;
@@ -4969,133 +5013,6 @@ static int mvpp22_comphy_init(struct mvpp2_port *port)
 	return phy_power_on(port->comphy);
 }
 
-static void mvpp2_port_mii_gmac_configure_mode(struct mvpp2_port *port)
-{
-	u32 val;
-
-	if (port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
-		val = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
-		val |= MVPP22_CTRL4_SYNC_BYPASS_DIS | MVPP22_CTRL4_DP_CLK_SEL |
-		       MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
-		val &= ~MVPP22_CTRL4_EXT_PIN_GMII_SEL;
-		writel(val, port->base + MVPP22_GMAC_CTRL_4_REG);
-	} else if (phy_interface_mode_is_rgmii(port->phy_interface)) {
-		val = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
-		val |= MVPP22_CTRL4_EXT_PIN_GMII_SEL |
-		       MVPP22_CTRL4_SYNC_BYPASS_DIS |
-		       MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
-		val &= ~MVPP22_CTRL4_DP_CLK_SEL;
-		writel(val, port->base + MVPP22_GMAC_CTRL_4_REG);
-	}
-
-	/* The port is connected to a copper PHY */
-	val = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
-	val &= ~MVPP2_GMAC_PORT_TYPE_MASK;
-	writel(val, port->base + MVPP2_GMAC_CTRL_0_REG);
-
-	val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-	val |= MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS |
-	       MVPP2_GMAC_AN_SPEED_EN | MVPP2_GMAC_FLOW_CTRL_AUTONEG |
-	       MVPP2_GMAC_AN_DUPLEX_EN;
-	if (port->phy_interface == PHY_INTERFACE_MODE_SGMII)
-		val |= MVPP2_GMAC_IN_BAND_AUTONEG;
-	writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-}
-
-static void mvpp2_port_mii_gmac_configure(struct mvpp2_port *port)
-{
-	u32 val;
-
-	/* Force link down */
-	val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-	val &= ~MVPP2_GMAC_FORCE_LINK_PASS;
-	val |= MVPP2_GMAC_FORCE_LINK_DOWN;
-	writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-
-	/* Set the GMAC in a reset state */
-	val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
-	val |= MVPP2_GMAC_PORT_RESET_MASK;
-	writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
-
-	/* Configure the PCS and in-band AN */
-	val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
-	if (port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
-	        val |= MVPP2_GMAC_INBAND_AN_MASK | MVPP2_GMAC_PCS_ENABLE_MASK;
-	} else if (phy_interface_mode_is_rgmii(port->phy_interface)) {
-		val &= ~MVPP2_GMAC_PCS_ENABLE_MASK;
-	}
-	writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
-
-	mvpp2_port_mii_gmac_configure_mode(port);
-
-	/* Unset the GMAC reset state */
-	val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
-	val &= ~MVPP2_GMAC_PORT_RESET_MASK;
-	writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
-
-	/* Stop forcing link down */
-	val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-	val &= ~MVPP2_GMAC_FORCE_LINK_DOWN;
-	writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-}
-
-static void mvpp2_port_mii_xlg_configure(struct mvpp2_port *port)
-{
-	u32 val;
-
-	if (port->gop_id != 0)
-		return;
-
-	val = readl(port->base + MVPP22_XLG_CTRL0_REG);
-	val |= MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN;
-	writel(val, port->base + MVPP22_XLG_CTRL0_REG);
-
-	val = readl(port->base + MVPP22_XLG_CTRL4_REG);
-	val &= ~MVPP22_XLG_CTRL4_MACMODSELECT_GMAC;
-	val |= MVPP22_XLG_CTRL4_FWD_FC | MVPP22_XLG_CTRL4_FWD_PFC;
-	writel(val, port->base + MVPP22_XLG_CTRL4_REG);
-}
-
-static void mvpp22_port_mii_set(struct mvpp2_port *port)
-{
-	u32 val;
-
-	/* Only GOP port 0 has an XLG MAC */
-	if (port->gop_id == 0) {
-		val = readl(port->base + MVPP22_XLG_CTRL3_REG);
-		val &= ~MVPP22_XLG_CTRL3_MACMODESELECT_MASK;
-
-		if (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
-		    port->phy_interface == PHY_INTERFACE_MODE_10GKR)
-			val |= MVPP22_XLG_CTRL3_MACMODESELECT_10G;
-		else
-			val |= MVPP22_XLG_CTRL3_MACMODESELECT_GMAC;
-
-		writel(val, port->base + MVPP22_XLG_CTRL3_REG);
-	}
-}
-
-static void mvpp2_port_mii_set(struct mvpp2_port *port)
-{
-	if (port->priv->hw_version == MVPP22)
-		mvpp22_port_mii_set(port);
-
-	if (phy_interface_mode_is_rgmii(port->phy_interface) ||
-	    port->phy_interface == PHY_INTERFACE_MODE_SGMII)
-		mvpp2_port_mii_gmac_configure(port);
-	else if (port->phy_interface == PHY_INTERFACE_MODE_10GKR)
-		mvpp2_port_mii_xlg_configure(port);
-}
-
-static void mvpp2_port_fc_adv_enable(struct mvpp2_port *port)
-{
-	u32 val;
-
-	val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-	val |= MVPP2_GMAC_FC_ADV_EN;
-	writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-}
-
 static void mvpp2_port_enable(struct mvpp2_port *port)
 {
 	u32 val;
@@ -5126,8 +5043,11 @@ static void mvpp2_port_disable(struct mvpp2_port *port)
 	    (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
 	     port->phy_interface == PHY_INTERFACE_MODE_10GKR)) {
 		val = readl(port->base + MVPP22_XLG_CTRL0_REG);
-		val &= ~(MVPP22_XLG_CTRL0_PORT_EN |
-			 MVPP22_XLG_CTRL0_MAC_RESET_DIS);
+		val &= ~MVPP22_XLG_CTRL0_PORT_EN;
+		writel(val, port->base + MVPP22_XLG_CTRL0_REG);
+
+		/* Disable & reset should be done separately */
+		val &= ~MVPP22_XLG_CTRL0_MAC_RESET_DIS;
 		writel(val, port->base + MVPP22_XLG_CTRL0_REG);
 	} else {
 		val = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
@@ -5147,18 +5067,21 @@ static void mvpp2_port_periodic_xon_disable(struct mvpp2_port *port)
 }
 
 /* Configure loopback port */
-static void mvpp2_port_loopback_set(struct mvpp2_port *port)
+static void mvpp2_port_loopback_set(struct mvpp2_port *port,
+				    const struct phylink_link_state *state)
 {
 	u32 val;
 
 	val = readl(port->base + MVPP2_GMAC_CTRL_1_REG);
 
-	if (port->speed == 1000)
+	if (state->speed == 1000)
 		val |= MVPP2_GMAC_GMII_LB_EN_MASK;
 	else
 		val &= ~MVPP2_GMAC_GMII_LB_EN_MASK;
 
-	if (port->phy_interface == PHY_INTERFACE_MODE_SGMII)
+	if (port->phy_interface == PHY_INTERFACE_MODE_SGMII ||
+	    port->phy_interface == PHY_INTERFACE_MODE_1000BASEX ||
+	    port->phy_interface == PHY_INTERFACE_MODE_2500BASEX)
 		val |= MVPP2_GMAC_PCS_LB_EN_MASK;
 	else
 		val &= ~MVPP2_GMAC_PCS_LB_EN_MASK;
@@ -5331,10 +5254,6 @@ static void mvpp2_defaults_set(struct mvpp2_port *port)
 	int tx_port_num, val, queue, ptxq, lrxq;
 
 	if (port->priv->hw_version == MVPP21) {
-		/* Configure port to loopback if needed */
-		if (port->flags & MVPP2_F_LOOPBACK)
-			mvpp2_port_loopback_set(port);
-
 		/* Update TX FIFO MIN Threshold */
 		val = readl(port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG);
 		val &= ~MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK;
@@ -5552,7 +5471,6 @@ static void mvpp2_aggr_txq_pend_desc_add(struct mvpp2_port *port, int pending)
 			   MVPP2_AGGR_TXQ_UPDATE_REG, pending);
 }
 
-
 /* Check if there are enough free descriptors in aggregated txq.
  * If not, update the number of occupied descriptors and repeat the check.
  *
@@ -5569,11 +5487,10 @@ static int mvpp2_aggr_desc_num_check(struct mvpp2 *priv,
 					     MVPP2_AGGR_TXQ_STATUS_REG(cpu));
 
 		aggr_txq->count = val & MVPP2_AGGR_TXQ_PENDING_MASK;
-	}
-
-	if ((aggr_txq->count + num) > MVPP2_AGGR_TXQ_SIZE)
-		return -ENOMEM;
 
+		if ((aggr_txq->count + num) > MVPP2_AGGR_TXQ_SIZE)
+			return -ENOMEM;
+	}
 	return 0;
 }
 
@@ -5633,7 +5550,7 @@ static int mvpp2_txq_reserved_desc_num_proc(struct mvpp2 *priv,
 
 	txq_pcpu->reserved_num += mvpp2_txq_alloc_reserved_desc(priv, txq, req);
 
-	/* OK, the descriptor cound has been updated: check again. */
+	/* OK, the descriptor could have been updated: check again. */
 	if (txq_pcpu->reserved_num < num)
 		return -ENOMEM;
 	return 0;
@@ -6115,7 +6032,7 @@ static int mvpp2_txq_init(struct mvpp2_port *port,
 	/* Calculate base address in prefetch buffer. We reserve 16 descriptors
 	 * for each existing TXQ.
 	 * TCONTS for PON port must be continuous from 0 to MVPP2_MAX_TCONT
-	 * GBE ports assumed to be continious from 0 to MVPP2_MAX_PORTS
+	 * GBE ports assumed to be continuous from 0 to MVPP2_MAX_PORTS
 	 */
 	desc_per_txq = 16;
 	desc = (port->id * MVPP2_MAX_TXQ * desc_per_txq) +
@@ -6372,7 +6289,9 @@ static irqreturn_t mvpp2_link_status_isr(int irq, void *dev_id)
 				link = true;
 		}
 	} else if (phy_interface_mode_is_rgmii(port->phy_interface) ||
-		   port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+		   port->phy_interface == PHY_INTERFACE_MODE_SGMII ||
+		   port->phy_interface == PHY_INTERFACE_MODE_1000BASEX ||
+		   port->phy_interface == PHY_INTERFACE_MODE_2500BASEX) {
 		val = readl(port->base + MVPP22_GMAC_INT_STAT);
 		if (val & MVPP22_GMAC_INT_STAT_LINK) {
 			event = true;
@@ -6382,6 +6301,11 @@ static irqreturn_t mvpp2_link_status_isr(int irq, void *dev_id)
 		}
 	}
 
+	if (port->phylink) {
+		phylink_mac_change(port->phylink, link);
+		goto handled;
+	}
+
 	if (!netif_running(dev) || !event)
 		goto handled;
 
@@ -6406,111 +6330,6 @@ static irqreturn_t mvpp2_link_status_isr(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static void mvpp2_gmac_set_autoneg(struct mvpp2_port *port,
-				   struct phy_device *phydev)
-{
-	u32 val;
-
-	if (port->phy_interface != PHY_INTERFACE_MODE_RGMII &&
-	    port->phy_interface != PHY_INTERFACE_MODE_RGMII_ID &&
-	    port->phy_interface != PHY_INTERFACE_MODE_RGMII_RXID &&
-	    port->phy_interface != PHY_INTERFACE_MODE_RGMII_TXID &&
-	    port->phy_interface != PHY_INTERFACE_MODE_SGMII)
-		return;
-
-	val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-	val &= ~(MVPP2_GMAC_CONFIG_MII_SPEED |
-		 MVPP2_GMAC_CONFIG_GMII_SPEED |
-		 MVPP2_GMAC_CONFIG_FULL_DUPLEX |
-		 MVPP2_GMAC_AN_SPEED_EN |
-		 MVPP2_GMAC_AN_DUPLEX_EN);
-
-	if (phydev->duplex)
-		val |= MVPP2_GMAC_CONFIG_FULL_DUPLEX;
-
-	if (phydev->speed == SPEED_1000)
-		val |= MVPP2_GMAC_CONFIG_GMII_SPEED;
-	else if (phydev->speed == SPEED_100)
-		val |= MVPP2_GMAC_CONFIG_MII_SPEED;
-
-	writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-}
-
-/* Adjust link */
-static void mvpp2_link_event(struct net_device *dev)
-{
-	struct mvpp2_port *port = netdev_priv(dev);
-	struct phy_device *phydev = dev->phydev;
-	bool link_reconfigured = false;
-	u32 val;
-
-	if (phydev->link) {
-		if (port->phy_interface != phydev->interface && port->comphy) {
-	                /* disable current port for reconfiguration */
-	                mvpp2_interrupts_disable(port);
-	                netif_carrier_off(port->dev);
-	                mvpp2_port_disable(port);
-			phy_power_off(port->comphy);
-
-	                /* comphy reconfiguration */
-	                port->phy_interface = phydev->interface;
-	                mvpp22_comphy_init(port);
-
-	                /* gop/mac reconfiguration */
-	                mvpp22_gop_init(port);
-	                mvpp2_port_mii_set(port);
-
-	                link_reconfigured = true;
-		}
-
-		if ((port->speed != phydev->speed) ||
-		    (port->duplex != phydev->duplex)) {
-			mvpp2_gmac_set_autoneg(port, phydev);
-
-			port->duplex = phydev->duplex;
-			port->speed  = phydev->speed;
-		}
-	}
-
-	if (phydev->link != port->link || link_reconfigured) {
-		port->link = phydev->link;
-
-		if (phydev->link) {
-			if (port->phy_interface == PHY_INTERFACE_MODE_RGMII ||
-			    port->phy_interface == PHY_INTERFACE_MODE_RGMII_ID ||
-			    port->phy_interface == PHY_INTERFACE_MODE_RGMII_RXID ||
-			    port->phy_interface == PHY_INTERFACE_MODE_RGMII_TXID ||
-			    port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
-				val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-				val |= (MVPP2_GMAC_FORCE_LINK_PASS |
-					MVPP2_GMAC_FORCE_LINK_DOWN);
-				writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-			}
-
-			mvpp2_interrupts_enable(port);
-			mvpp2_port_enable(port);
-
-			mvpp2_egress_enable(port);
-			mvpp2_ingress_enable(port);
-			netif_carrier_on(dev);
-			netif_tx_wake_all_queues(dev);
-		} else {
-			port->duplex = -1;
-			port->speed = 0;
-
-			netif_tx_stop_all_queues(dev);
-			netif_carrier_off(dev);
-			mvpp2_ingress_disable(port);
-			mvpp2_egress_disable(port);
-
-			mvpp2_port_disable(port);
-			mvpp2_interrupts_disable(port);
-		}
-
-		phy_print_status(phydev);
-	}
-}
-
 static void mvpp2_timer_set(struct mvpp2_port_pcpu *port_pcpu)
 {
 	ktime_t interval;
@@ -6562,21 +6381,23 @@ static void mvpp2_rx_error(struct mvpp2_port *port,
 {
 	u32 status = mvpp2_rxdesc_status_get(port, rx_desc);
 	size_t sz = mvpp2_rxdesc_size_get(port, rx_desc);
+	char *err_str = NULL;
 
 	switch (status & MVPP2_RXD_ERR_CODE_MASK) {
 	case MVPP2_RXD_ERR_CRC:
-		netdev_err(port->dev, "bad rx status %08x (crc error), size=%zu\n",
-			   status, sz);
+		err_str = "crc";
 		break;
 	case MVPP2_RXD_ERR_OVERRUN:
-		netdev_err(port->dev, "bad rx status %08x (overrun error), size=%zu\n",
-			   status, sz);
+		err_str = "overrun";
 		break;
 	case MVPP2_RXD_ERR_RESOURCE:
-		netdev_err(port->dev, "bad rx status %08x (resource error), size=%zu\n",
-			   status, sz);
+		err_str = "resource";
 		break;
 	}
+	if (err_str && net_ratelimit())
+		netdev_err(port->dev,
+			   "bad rx status %08x (%s error), size=%zu\n",
+			   status, err_str, sz);
 }
 
 /* Handle RX checksum offload */
@@ -6781,8 +6602,7 @@ static int mvpp2_tx_frag_process(struct mvpp2_port *port, struct sk_buff *skb,
 		mvpp2_txdesc_size_set(port, tx_desc, frag->size);
 
 		buf_dma_addr = dma_map_single(port->dev->dev.parent, addr,
-					       frag->size,
-					       DMA_TO_DEVICE);
+					      frag->size, DMA_TO_DEVICE);
 		if (dma_mapping_error(port->dev->dev.parent, buf_dma_addr)) {
 			mvpp2_txq_desc_put(txq);
 			goto cleanup;
@@ -7118,11 +6938,29 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
 	return rx_done;
 }
 
-/* Set hw internals when starting port */
-static void mvpp2_start_dev(struct mvpp2_port *port)
+static void mvpp22_mode_reconfigure(struct mvpp2_port *port)
 {
-	struct net_device *ndev = port->dev;
-	int i;
+	u32 ctrl3;
+
+	/* comphy reconfiguration */
+	mvpp22_comphy_init(port);
+
+	/* gop reconfiguration */
+	mvpp22_gop_init(port);
+
+	/* Only GOP port 0 has an XLG MAC */
+	if (port->gop_id == 0) {
+		ctrl3 = readl(port->base + MVPP22_XLG_CTRL3_REG);
+		ctrl3 &= ~MVPP22_XLG_CTRL3_MACMODESELECT_MASK;
+
+		if (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
+		    port->phy_interface == PHY_INTERFACE_MODE_10GKR)
+			ctrl3 |= MVPP22_XLG_CTRL3_MACMODESELECT_10G;
+		else
+			ctrl3 |= MVPP22_XLG_CTRL3_MACMODESELECT_GMAC;
+
+		writel(ctrl3, port->base + MVPP22_XLG_CTRL3_REG);
+	}
 
 	if (port->gop_id == 0 &&
 	    (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
@@ -7130,6 +6968,12 @@ static void mvpp2_start_dev(struct mvpp2_port *port)
 		mvpp2_xlg_max_rx_size_set(port);
 	else
 		mvpp2_gmac_max_rx_size_set(port);
+}
+
+/* Set hw internals when starting port */
+static void mvpp2_start_dev(struct mvpp2_port *port)
+{
+	int i;
 
 	mvpp2_txp_max_tx_size_set(port);
 
@@ -7139,42 +6983,39 @@ static void mvpp2_start_dev(struct mvpp2_port *port)
 	/* Enable interrupts on all CPUs */
 	mvpp2_interrupts_enable(port);
 
-	if (port->priv->hw_version == MVPP22) {
-		mvpp22_comphy_init(port);
-		mvpp22_gop_init(port);
+	if (port->priv->hw_version == MVPP22)
+		mvpp22_mode_reconfigure(port);
+
+	if (port->phylink) {
+		phylink_start(port->phylink);
+	} else {
+		/* Phylink isn't used as of now for ACPI, so the MAC has to be
+		 * configured manually when the interface is started. This will
+		 * be removed as soon as the phylink ACPI support lands in.
+		 */
+		struct phylink_link_state state = {
+			.interface = port->phy_interface,
+			.link = 1,
+		};
+		mvpp2_mac_config(port->dev, MLO_AN_INBAND, &state);
 	}
 
-	mvpp2_port_mii_set(port);
-	mvpp2_port_enable(port);
-	if (ndev->phydev)
-		phy_start(ndev->phydev);
 	netif_tx_start_all_queues(port->dev);
 }
 
 /* Set hw internals when stopping port */
 static void mvpp2_stop_dev(struct mvpp2_port *port)
 {
-	struct net_device *ndev = port->dev;
 	int i;
 
-	/* Stop new packets from arriving to RXQs */
-	mvpp2_ingress_disable(port);
-
-	mdelay(10);
-
 	/* Disable interrupts on all CPUs */
 	mvpp2_interrupts_disable(port);
 
 	for (i = 0; i < port->nqvecs; i++)
 		napi_disable(&port->qvecs[i].napi);
 
-	netif_carrier_off(port->dev);
-	netif_tx_stop_all_queues(port->dev);
-
-	mvpp2_egress_disable(port);
-	mvpp2_port_disable(port);
-	if (ndev->phydev)
-		phy_stop(ndev->phydev);
+	if (port->phylink)
+		phylink_stop(port->phylink);
 	phy_power_off(port->comphy);
 }
 
@@ -7233,40 +7074,6 @@ static void mvpp21_get_mac_address(struct mvpp2_port *port, unsigned char *addr)
 	addr[5] = (mac_addr_l >> MVPP2_GMAC_SA_LOW_OFFS) & 0xFF;
 }
 
-static int mvpp2_phy_connect(struct mvpp2_port *port)
-{
-	struct phy_device *phy_dev;
-
-	/* No PHY is attached */
-	if (!port->phy_node)
-		return 0;
-
-	phy_dev = of_phy_connect(port->dev, port->phy_node, mvpp2_link_event, 0,
-				 port->phy_interface);
-	if (!phy_dev) {
-		netdev_err(port->dev, "cannot connect to phy\n");
-		return -ENODEV;
-	}
-	phy_dev->supported &= PHY_GBIT_FEATURES;
-	phy_dev->advertising = phy_dev->supported;
-
-	port->link    = 0;
-	port->duplex  = 0;
-	port->speed   = 0;
-
-	return 0;
-}
-
-static void mvpp2_phy_disconnect(struct mvpp2_port *port)
-{
-	struct net_device *ndev = port->dev;
-
-	if (!ndev->phydev)
-		return;
-
-	phy_disconnect(ndev->phydev);
-}
-
 static int mvpp2_irqs_init(struct mvpp2_port *port)
 {
 	int err, i;
@@ -7350,6 +7157,7 @@ static int mvpp2_open(struct net_device *dev)
 	struct mvpp2 *priv = port->priv;
 	unsigned char mac_bcast[ETH_ALEN] = {
 			0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+	bool valid = false;
 	int err;
 
 	err = mvpp2_prs_mac_da_accept(port, mac_bcast, true);
@@ -7392,7 +7200,19 @@ static int mvpp2_open(struct net_device *dev)
 		goto err_cleanup_txqs;
 	}
 
-	if (priv->hw_version == MVPP22 && !port->phy_node && port->link_irq) {
+	/* Phylink isn't supported yet in ACPI mode */
+	if (port->of_node) {
+		err = phylink_of_phy_connect(port->phylink, port->of_node, 0);
+		if (err) {
+			netdev_err(port->dev, "could not attach PHY (%d)\n",
+				   err);
+			goto err_free_irq;
+		}
+
+		valid = true;
+	}
+
+	if (priv->hw_version == MVPP22 && port->link_irq && !port->phylink) {
 		err = request_irq(port->link_irq, mvpp2_link_status_isr, 0,
 				  dev->name, port);
 		if (err) {
@@ -7402,14 +7222,20 @@ static int mvpp2_open(struct net_device *dev)
 		}
 
 		mvpp22_gop_setup_irq(port);
-	}
 
-	/* In default link is down */
-	netif_carrier_off(port->dev);
+		/* In default link is down */
+		netif_carrier_off(port->dev);
 
-	err = mvpp2_phy_connect(port);
-	if (err < 0)
-		goto err_free_link_irq;
+		valid = true;
+	} else {
+		port->link_irq = 0;
+	}
+
+	if (!valid) {
+		netdev_err(port->dev,
+			   "invalid configuration: no dt or link IRQ");
+		goto err_free_irq;
+	}
 
 	/* Unmask interrupts on all CPUs */
 	on_each_cpu(mvpp2_interrupts_unmask, port, 1);
@@ -7426,9 +7252,6 @@ static int mvpp2_open(struct net_device *dev)
 
 	return 0;
 
-err_free_link_irq:
-	if (priv->hw_version == MVPP22 && !port->phy_node && port->link_irq)
-		free_irq(port->link_irq, port);
 err_free_irq:
 	mvpp2_irqs_deinit(port);
 err_cleanup_txqs:
@@ -7442,17 +7265,17 @@ static int mvpp2_stop(struct net_device *dev)
 {
 	struct mvpp2_port *port = netdev_priv(dev);
 	struct mvpp2_port_pcpu *port_pcpu;
-	struct mvpp2 *priv = port->priv;
 	int cpu;
 
 	mvpp2_stop_dev(port);
-	mvpp2_phy_disconnect(port);
 
 	/* Mask interrupts on all CPUs */
 	on_each_cpu(mvpp2_interrupts_mask, port, 1);
 	mvpp2_shared_interrupt_mask_unmask(port, true);
 
-	if (priv->hw_version == MVPP22 && !port->phy_node && port->link_irq)
+	if (port->phylink)
+		phylink_disconnect_phy(port->phylink);
+	if (port->link_irq)
 		free_irq(port->link_irq, port);
 
 	mvpp2_irqs_deinit(port);
@@ -7535,42 +7358,18 @@ static void mvpp2_set_rx_mode(struct net_device *dev)
 
 static int mvpp2_set_mac_address(struct net_device *dev, void *p)
 {
-	struct mvpp2_port *port = netdev_priv(dev);
 	const struct sockaddr *addr = p;
 	int err;
 
-	if (!is_valid_ether_addr(addr->sa_data)) {
-		err = -EADDRNOTAVAIL;
-		goto log_error;
-	}
-
-	if (!netif_running(dev)) {
-		err = mvpp2_prs_update_mac_da(dev, addr->sa_data);
-		if (!err)
-			return 0;
-		/* Reconfigure parser to accept the original MAC address */
-		err = mvpp2_prs_update_mac_da(dev, dev->dev_addr);
-		if (err)
-			goto log_error;
-	}
-
-	mvpp2_stop_dev(port);
+	if (!is_valid_ether_addr(addr->sa_data))
+		return -EADDRNOTAVAIL;
 
 	err = mvpp2_prs_update_mac_da(dev, addr->sa_data);
-	if (!err)
-		goto out_start;
-
-	/* Reconfigure parser accept the original MAC address */
-	err = mvpp2_prs_update_mac_da(dev, dev->dev_addr);
-	if (err)
-		goto log_error;
-out_start:
-	mvpp2_start_dev(port);
-	mvpp2_egress_enable(port);
-	mvpp2_ingress_enable(port);
-	return 0;
-log_error:
-	netdev_err(dev, "failed to change MAC address\n");
+	if (err) {
+		/* Reconfigure parser accept the original MAC address */
+		mvpp2_prs_update_mac_da(dev, dev->dev_addr);
+		netdev_err(dev, "failed to change MAC address\n");
+	}
 	return err;
 }
 
@@ -7658,16 +7457,12 @@ mvpp2_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
 
 static int mvpp2_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 {
-	int ret;
+	struct mvpp2_port *port = netdev_priv(dev);
 
-	if (!dev->phydev)
+	if (!port->phylink)
 		return -ENOTSUPP;
 
-	ret = phy_mii_ioctl(dev->phydev, ifr, cmd);
-	if (!ret)
-		mvpp2_link_event(dev);
-
-	return ret;
+	return phylink_mii_ioctl(port->phylink, ifr, cmd);
 }
 
 static int mvpp2_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
@@ -7714,6 +7509,16 @@ static int mvpp2_set_features(struct net_device *dev,
 
 /* Ethtool methods */
 
+static int mvpp2_ethtool_nway_reset(struct net_device *dev)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+
+	if (!port->phylink)
+		return -ENOTSUPP;
+
+	return phylink_ethtool_nway_reset(port->phylink);
+}
+
 /* Set interrupt coalescing for ethtools */
 static int mvpp2_ethtool_set_coalesce(struct net_device *dev,
 				      struct ethtool_coalesce *c)
@@ -7842,6 +7647,50 @@ static int mvpp2_ethtool_set_ringparam(struct net_device *dev,
 	return err;
 }
 
+static void mvpp2_ethtool_get_pause_param(struct net_device *dev,
+					  struct ethtool_pauseparam *pause)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+
+	if (!port->phylink)
+		return;
+
+	phylink_ethtool_get_pauseparam(port->phylink, pause);
+}
+
+static int mvpp2_ethtool_set_pause_param(struct net_device *dev,
+					 struct ethtool_pauseparam *pause)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+
+	if (!port->phylink)
+		return -ENOTSUPP;
+
+	return phylink_ethtool_set_pauseparam(port->phylink, pause);
+}
+
+static int mvpp2_ethtool_get_link_ksettings(struct net_device *dev,
+					    struct ethtool_link_ksettings *cmd)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+
+	if (!port->phylink)
+		return -ENOTSUPP;
+
+	return phylink_ethtool_ksettings_get(port->phylink, cmd);
+}
+
+static int mvpp2_ethtool_set_link_ksettings(struct net_device *dev,
+					    const struct ethtool_link_ksettings *cmd)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+
+	if (!port->phylink)
+		return -ENOTSUPP;
+
+	return phylink_ethtool_ksettings_set(port->phylink, cmd);
+}
+
 /* Device ops */
 
 static const struct net_device_ops mvpp2_netdev_ops = {
@@ -7859,18 +7708,20 @@ static const struct net_device_ops mvpp2_netdev_ops = {
 };
 
 static const struct ethtool_ops mvpp2_eth_tool_ops = {
-	.nway_reset	= phy_ethtool_nway_reset,
-	.get_link	= ethtool_op_get_link,
-	.set_coalesce	= mvpp2_ethtool_set_coalesce,
-	.get_coalesce	= mvpp2_ethtool_get_coalesce,
-	.get_drvinfo	= mvpp2_ethtool_get_drvinfo,
-	.get_ringparam	= mvpp2_ethtool_get_ringparam,
-	.set_ringparam	= mvpp2_ethtool_set_ringparam,
-	.get_strings	= mvpp2_ethtool_get_strings,
-	.get_ethtool_stats = mvpp2_ethtool_get_stats,
-	.get_sset_count	= mvpp2_ethtool_get_sset_count,
-	.get_link_ksettings = phy_ethtool_get_link_ksettings,
-	.set_link_ksettings = phy_ethtool_set_link_ksettings,
+	.nway_reset		= mvpp2_ethtool_nway_reset,
+	.get_link		= ethtool_op_get_link,
+	.set_coalesce		= mvpp2_ethtool_set_coalesce,
+	.get_coalesce		= mvpp2_ethtool_get_coalesce,
+	.get_drvinfo		= mvpp2_ethtool_get_drvinfo,
+	.get_ringparam		= mvpp2_ethtool_get_ringparam,
+	.set_ringparam		= mvpp2_ethtool_set_ringparam,
+	.get_strings		= mvpp2_ethtool_get_strings,
+	.get_ethtool_stats	= mvpp2_ethtool_get_stats,
+	.get_sset_count		= mvpp2_ethtool_get_sset_count,
+	.get_pauseparam		= mvpp2_ethtool_get_pause_param,
+	.set_pauseparam		= mvpp2_ethtool_set_pause_param,
+	.get_link_ksettings	= mvpp2_ethtool_get_link_ksettings,
+	.set_link_ksettings	= mvpp2_ethtool_set_link_ksettings,
 };
 
 /* Used for PPv2.1, or PPv2.2 with the old Device Tree binding that
@@ -8172,18 +8023,361 @@ static void mvpp2_port_copy_mac_addr(struct net_device *dev, struct mvpp2 *priv,
 	eth_hw_addr_random(dev);
 }
 
+static void mvpp2_phylink_validate(struct net_device *dev,
+				   unsigned long *supported,
+				   struct phylink_link_state *state)
+{
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
+
+	phylink_set(mask, Autoneg);
+	phylink_set_port_modes(mask);
+	phylink_set(mask, Pause);
+	phylink_set(mask, Asym_Pause);
+
+	switch (state->interface) {
+	case PHY_INTERFACE_MODE_10GKR:
+		phylink_set(mask, 10000baseCR_Full);
+		phylink_set(mask, 10000baseSR_Full);
+		phylink_set(mask, 10000baseLR_Full);
+		phylink_set(mask, 10000baseLRM_Full);
+		phylink_set(mask, 10000baseER_Full);
+		phylink_set(mask, 10000baseKR_Full);
+		/* Fall-through */
+	default:
+		phylink_set(mask, 10baseT_Half);
+		phylink_set(mask, 10baseT_Full);
+		phylink_set(mask, 100baseT_Half);
+		phylink_set(mask, 100baseT_Full);
+		phylink_set(mask, 10000baseT_Full);
+		/* Fall-through */
+	case PHY_INTERFACE_MODE_1000BASEX:
+	case PHY_INTERFACE_MODE_2500BASEX:
+		phylink_set(mask, 1000baseT_Full);
+		phylink_set(mask, 1000baseX_Full);
+		phylink_set(mask, 2500baseX_Full);
+	}
+
+	bitmap_and(supported, supported, mask, __ETHTOOL_LINK_MODE_MASK_NBITS);
+	bitmap_and(state->advertising, state->advertising, mask,
+		   __ETHTOOL_LINK_MODE_MASK_NBITS);
+}
+
+static void mvpp22_xlg_link_state(struct mvpp2_port *port,
+				  struct phylink_link_state *state)
+{
+	u32 val;
+
+	state->speed = SPEED_10000;
+	state->duplex = 1;
+	state->an_complete = 1;
+
+	val = readl(port->base + MVPP22_XLG_STATUS);
+	state->link = !!(val & MVPP22_XLG_STATUS_LINK_UP);
+
+	state->pause = 0;
+	val = readl(port->base + MVPP22_XLG_CTRL0_REG);
+	if (val & MVPP22_XLG_CTRL0_TX_FLOW_CTRL_EN)
+		state->pause |= MLO_PAUSE_TX;
+	if (val & MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN)
+		state->pause |= MLO_PAUSE_RX;
+}
+
+static void mvpp2_gmac_link_state(struct mvpp2_port *port,
+				  struct phylink_link_state *state)
+{
+	u32 val;
+
+	val = readl(port->base + MVPP2_GMAC_STATUS0);
+
+	state->an_complete = !!(val & MVPP2_GMAC_STATUS0_AN_COMPLETE);
+	state->link = !!(val & MVPP2_GMAC_STATUS0_LINK_UP);
+	state->duplex = !!(val & MVPP2_GMAC_STATUS0_FULL_DUPLEX);
+
+	switch (port->phy_interface) {
+	case PHY_INTERFACE_MODE_1000BASEX:
+		state->speed = SPEED_1000;
+		break;
+	case PHY_INTERFACE_MODE_2500BASEX:
+		state->speed = SPEED_2500;
+		break;
+	default:
+		if (val & MVPP2_GMAC_STATUS0_GMII_SPEED)
+			state->speed = SPEED_1000;
+		else if (val & MVPP2_GMAC_STATUS0_MII_SPEED)
+			state->speed = SPEED_100;
+		else
+			state->speed = SPEED_10;
+	}
+
+	state->pause = 0;
+	if (val & MVPP2_GMAC_STATUS0_RX_PAUSE)
+		state->pause |= MLO_PAUSE_RX;
+	if (val & MVPP2_GMAC_STATUS0_TX_PAUSE)
+		state->pause |= MLO_PAUSE_TX;
+}
+
+static int mvpp2_phylink_mac_link_state(struct net_device *dev,
+					struct phylink_link_state *state)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+
+	if (port->priv->hw_version == MVPP22 && port->gop_id == 0) {
+		u32 mode = readl(port->base + MVPP22_XLG_CTRL3_REG);
+		mode &= MVPP22_XLG_CTRL3_MACMODESELECT_MASK;
+
+		if (mode == MVPP22_XLG_CTRL3_MACMODESELECT_10G) {
+			mvpp22_xlg_link_state(port, state);
+			return 1;
+		}
+	}
+
+	mvpp2_gmac_link_state(port, state);
+	return 1;
+}
+
+static void mvpp2_mac_an_restart(struct net_device *dev)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+	u32 val;
+
+	if (port->phy_interface != PHY_INTERFACE_MODE_SGMII)
+		return;
+
+	val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+	/* The RESTART_AN bit is cleared by the h/w after restarting the AN
+	 * process.
+	 */
+	val |= MVPP2_GMAC_IN_BAND_RESTART_AN | MVPP2_GMAC_IN_BAND_AUTONEG;
+	writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+}
+
+static void mvpp2_xlg_config(struct mvpp2_port *port, unsigned int mode,
+			     const struct phylink_link_state *state)
+{
+	u32 ctrl0, ctrl4;
+
+	ctrl0 = readl(port->base + MVPP22_XLG_CTRL0_REG);
+	ctrl4 = readl(port->base + MVPP22_XLG_CTRL4_REG);
+
+	if (state->pause & MLO_PAUSE_TX)
+		ctrl0 |= MVPP22_XLG_CTRL0_TX_FLOW_CTRL_EN;
+	if (state->pause & MLO_PAUSE_RX)
+		ctrl0 |= MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN;
+
+	ctrl4 &= ~MVPP22_XLG_CTRL4_MACMODSELECT_GMAC;
+	ctrl4 |= MVPP22_XLG_CTRL4_FWD_FC | MVPP22_XLG_CTRL4_FWD_PFC |
+		 MVPP22_XLG_CTRL4_EN_IDLE_CHECK;
+
+	writel(ctrl0, port->base + MVPP22_XLG_CTRL0_REG);
+	writel(ctrl4, port->base + MVPP22_XLG_CTRL4_REG);
+}
+
+static void mvpp2_gmac_config(struct mvpp2_port *port, unsigned int mode,
+			      const struct phylink_link_state *state)
+{
+	u32 an, ctrl0, ctrl2, ctrl4;
+
+	an = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+	ctrl0 = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
+	ctrl2 = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
+	ctrl4 = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
+
+	/* Force link down */
+	an &= ~MVPP2_GMAC_FORCE_LINK_PASS;
+	an |= MVPP2_GMAC_FORCE_LINK_DOWN;
+	writel(an, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+
+	/* Set the GMAC in a reset state */
+	ctrl2 |= MVPP2_GMAC_PORT_RESET_MASK;
+	writel(ctrl2, port->base + MVPP2_GMAC_CTRL_2_REG);
+
+	an &= ~(MVPP2_GMAC_CONFIG_MII_SPEED | MVPP2_GMAC_CONFIG_GMII_SPEED |
+		MVPP2_GMAC_AN_SPEED_EN | MVPP2_GMAC_FC_ADV_EN |
+		MVPP2_GMAC_FC_ADV_ASM_EN | MVPP2_GMAC_FLOW_CTRL_AUTONEG |
+		MVPP2_GMAC_CONFIG_FULL_DUPLEX | MVPP2_GMAC_AN_DUPLEX_EN |
+		MVPP2_GMAC_FORCE_LINK_DOWN);
+	ctrl0 &= ~MVPP2_GMAC_PORT_TYPE_MASK;
+	ctrl2 &= ~(MVPP2_GMAC_PORT_RESET_MASK | MVPP2_GMAC_PCS_ENABLE_MASK);
+
+	if (state->interface == PHY_INTERFACE_MODE_1000BASEX ||
+	    state->interface == PHY_INTERFACE_MODE_2500BASEX) {
+		/* 1000BaseX and 2500BaseX ports cannot negotiate speed nor can
+		 * they negotiate duplex: they are always operating with a fixed
+		 * speed of 1000/2500Mbps in full duplex, so force 1000/2500
+		 * speed and full duplex here.
+		 */
+		ctrl0 |= MVPP2_GMAC_PORT_TYPE_MASK;
+		an |= MVPP2_GMAC_CONFIG_GMII_SPEED |
+		      MVPP2_GMAC_CONFIG_FULL_DUPLEX;
+	} else if (!phy_interface_mode_is_rgmii(state->interface)) {
+		an |= MVPP2_GMAC_AN_SPEED_EN | MVPP2_GMAC_FLOW_CTRL_AUTONEG;
+	}
+
+	if (state->duplex)
+		an |= MVPP2_GMAC_CONFIG_FULL_DUPLEX;
+	if (phylink_test(state->advertising, Pause))
+		an |= MVPP2_GMAC_FC_ADV_EN;
+	if (phylink_test(state->advertising, Asym_Pause))
+		an |= MVPP2_GMAC_FC_ADV_ASM_EN;
+
+	if (state->interface == PHY_INTERFACE_MODE_SGMII ||
+	    state->interface == PHY_INTERFACE_MODE_1000BASEX ||
+	    state->interface == PHY_INTERFACE_MODE_2500BASEX) {
+		an |= MVPP2_GMAC_IN_BAND_AUTONEG;
+		ctrl2 |= MVPP2_GMAC_INBAND_AN_MASK | MVPP2_GMAC_PCS_ENABLE_MASK;
+
+		ctrl4 &= ~(MVPP22_CTRL4_EXT_PIN_GMII_SEL |
+			   MVPP22_CTRL4_RX_FC_EN | MVPP22_CTRL4_TX_FC_EN);
+		ctrl4 |= MVPP22_CTRL4_SYNC_BYPASS_DIS |
+			 MVPP22_CTRL4_DP_CLK_SEL |
+			 MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
+
+		if (state->pause & MLO_PAUSE_TX)
+			ctrl4 |= MVPP22_CTRL4_TX_FC_EN;
+		if (state->pause & MLO_PAUSE_RX)
+			ctrl4 |= MVPP22_CTRL4_RX_FC_EN;
+	} else if (phy_interface_mode_is_rgmii(state->interface)) {
+		an |= MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS;
+
+		if (state->speed == SPEED_1000)
+			an |= MVPP2_GMAC_CONFIG_GMII_SPEED;
+		else if (state->speed == SPEED_100)
+			an |= MVPP2_GMAC_CONFIG_MII_SPEED;
+
+		ctrl4 &= ~MVPP22_CTRL4_DP_CLK_SEL;
+		ctrl4 |= MVPP22_CTRL4_EXT_PIN_GMII_SEL |
+			 MVPP22_CTRL4_SYNC_BYPASS_DIS |
+			 MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
+	}
+
+	writel(ctrl0, port->base + MVPP2_GMAC_CTRL_0_REG);
+	writel(ctrl2, port->base + MVPP2_GMAC_CTRL_2_REG);
+	writel(ctrl4, port->base + MVPP22_GMAC_CTRL_4_REG);
+	writel(an, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+}
+
+static void mvpp2_mac_config(struct net_device *dev, unsigned int mode,
+			     const struct phylink_link_state *state)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+
+	/* Check for invalid configuration */
+	if (state->interface == PHY_INTERFACE_MODE_10GKR && port->gop_id != 0) {
+		netdev_err(dev, "Invalid mode on %s\n", dev->name);
+		return;
+	}
+
+	netif_tx_stop_all_queues(port->dev);
+	if (!port->has_phy)
+		netif_carrier_off(port->dev);
+
+	/* Make sure the port is disabled when reconfiguring the mode */
+	mvpp2_port_disable(port);
+
+	if (port->priv->hw_version == MVPP22 &&
+	    port->phy_interface != state->interface) {
+		port->phy_interface = state->interface;
+
+		/* Reconfigure the serdes lanes */
+		phy_power_off(port->comphy);
+		mvpp22_mode_reconfigure(port);
+	}
+
+	/* mac (re)configuration */
+	if (state->interface == PHY_INTERFACE_MODE_10GKR)
+		mvpp2_xlg_config(port, mode, state);
+	else if (phy_interface_mode_is_rgmii(state->interface) ||
+		 state->interface == PHY_INTERFACE_MODE_SGMII ||
+		 state->interface == PHY_INTERFACE_MODE_1000BASEX ||
+		 state->interface == PHY_INTERFACE_MODE_2500BASEX)
+		mvpp2_gmac_config(port, mode, state);
+
+	if (port->priv->hw_version == MVPP21 && port->flags & MVPP2_F_LOOPBACK)
+		mvpp2_port_loopback_set(port, state);
+
+	/* If the port already was up, make sure it's still in the same state */
+	if (state->link || !port->has_phy) {
+		mvpp2_port_enable(port);
+
+		mvpp2_egress_enable(port);
+		mvpp2_ingress_enable(port);
+		if (!port->has_phy)
+			netif_carrier_on(dev);
+		netif_tx_wake_all_queues(dev);
+	}
+}
+
+static void mvpp2_mac_link_up(struct net_device *dev, unsigned int mode,
+			      phy_interface_t interface, struct phy_device *phy)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+	u32 val;
+
+	if (!phylink_autoneg_inband(mode) &&
+	    interface != PHY_INTERFACE_MODE_10GKR) {
+		val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+		val &= ~MVPP2_GMAC_FORCE_LINK_DOWN;
+		if (phy_interface_mode_is_rgmii(interface))
+			val |= MVPP2_GMAC_FORCE_LINK_PASS;
+		writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+	}
+
+	mvpp2_port_enable(port);
+
+	mvpp2_egress_enable(port);
+	mvpp2_ingress_enable(port);
+	netif_tx_wake_all_queues(dev);
+}
+
+static void mvpp2_mac_link_down(struct net_device *dev, unsigned int mode,
+				phy_interface_t interface)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+	u32 val;
+
+	if (!phylink_autoneg_inband(mode) &&
+	    interface != PHY_INTERFACE_MODE_10GKR) {
+		val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+		val &= ~MVPP2_GMAC_FORCE_LINK_PASS;
+		val |= MVPP2_GMAC_FORCE_LINK_DOWN;
+		writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+	}
+
+	netif_tx_stop_all_queues(dev);
+	mvpp2_egress_disable(port);
+	mvpp2_ingress_disable(port);
+
+	/* When using link interrupts to notify phylink of a MAC state change,
+	 * we do not want the port to be disabled (we want to receive further
+	 * interrupts, to be notified when the port will have a link later).
+	 */
+	if (!port->has_phy)
+		return;
+
+	mvpp2_port_disable(port);
+}
+
+static const struct phylink_mac_ops mvpp2_phylink_ops = {
+	.validate = mvpp2_phylink_validate,
+	.mac_link_state = mvpp2_phylink_mac_link_state,
+	.mac_an_restart = mvpp2_mac_an_restart,
+	.mac_config = mvpp2_mac_config,
+	.mac_link_up = mvpp2_mac_link_up,
+	.mac_link_down = mvpp2_mac_link_down,
+};
+
 /* Ports initialization */
 static int mvpp2_port_probe(struct platform_device *pdev,
 			    struct fwnode_handle *port_fwnode,
 			    struct mvpp2 *priv)
 {
-	struct device_node *phy_node;
 	struct phy *comphy = NULL;
 	struct mvpp2_port *port;
 	struct mvpp2_port_pcpu *port_pcpu;
 	struct device_node *port_node = to_of_node(port_fwnode);
 	struct net_device *dev;
 	struct resource *res;
+	struct phylink *phylink;
 	char *mac_from = "";
 	unsigned int ntxqs, nrxqs;
 	bool has_tx_irqs;
@@ -8212,11 +8406,6 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 	if (!dev)
 		return -ENOMEM;
 
-	if (port_node)
-		phy_node = of_parse_phandle(port_node, "phy", 0);
-	else
-		phy_node = NULL;
-
 	phy_mode = fwnode_get_phy_mode(port_fwnode);
 	if (phy_mode < 0) {
 		dev_err(&pdev->dev, "incorrect phy mode\n");
@@ -8249,6 +8438,7 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 	port = netdev_priv(dev);
 	port->dev = dev;
 	port->fwnode = port_fwnode;
+	port->has_phy = !!of_find_property(port_node, "phy", NULL);
 	port->ntxqs = ntxqs;
 	port->nrxqs = nrxqs;
 	port->priv = priv;
@@ -8279,7 +8469,7 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 	else
 		port->first_rxq = port->id * priv->max_port_rxqs;
 
-	port->phy_node = phy_node;
+	port->of_node = port_node;
 	port->phy_interface = phy_mode;
 	port->comphy = comphy;
 
@@ -8340,9 +8530,6 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 
 	mvpp2_port_periodic_xon_disable(port);
 
-	if (priv->hw_version == MVPP21)
-		mvpp2_port_fc_adv_enable(port);
-
 	mvpp2_port_reset(port);
 
 	port->pcpu = alloc_percpu(struct mvpp2_port_pcpu);
@@ -8386,10 +8573,23 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 	/* 9704 == 9728 - 20 and rounding to 8 */
 	dev->max_mtu = MVPP2_BM_JUMBO_PKT_SIZE;
 
+	/* Phylink isn't used w/ ACPI as of now */
+	if (port_node) {
+		phylink = phylink_create(dev, port_fwnode, phy_mode,
+					 &mvpp2_phylink_ops);
+		if (IS_ERR(phylink)) {
+			err = PTR_ERR(phylink);
+			goto err_free_port_pcpu;
+		}
+		port->phylink = phylink;
+	} else {
+		port->phylink = NULL;
+	}
+
 	err = register_netdev(dev);
 	if (err < 0) {
 		dev_err(&pdev->dev, "failed to register netdev\n");
-		goto err_free_port_pcpu;
+		goto err_phylink;
 	}
 	netdev_info(dev, "Using %s mac address %pM\n", mac_from, dev->dev_addr);
 
@@ -8397,6 +8597,9 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 
 	return 0;
 
+err_phylink:
+	if (port->phylink)
+		phylink_destroy(port->phylink);
 err_free_port_pcpu:
 	free_percpu(port->pcpu);
 err_free_txq_pcpu:
@@ -8410,7 +8613,6 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 err_deinit_qvecs:
 	mvpp2_queue_vectors_deinit(port);
 err_free_netdev:
-	of_node_put(phy_node);
 	free_netdev(dev);
 	return err;
 }
@@ -8421,7 +8623,8 @@ static void mvpp2_port_remove(struct mvpp2_port *port)
 	int i;
 
 	unregister_netdev(port->dev);
-	of_node_put(port->phy_node);
+	if (port->phylink)
+		phylink_destroy(port->phylink);
 	free_percpu(port->pcpu);
 	free_percpu(port->stats);
 	for (i = 0; i < port->ntxqs; i++)
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index e0b72bf428f7..d8ebf0a05e0c 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -2503,7 +2503,6 @@ static int mtk_probe(struct platform_device *pdev)
 {
 	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	struct device_node *mac_np;
-	const struct of_device_id *match;
 	struct mtk_eth *eth;
 	int err;
 	int i;
@@ -2512,8 +2511,7 @@ static int mtk_probe(struct platform_device *pdev)
 	if (!eth)
 		return -ENOMEM;
 
-	match = of_match_device(of_mtk_match, &pdev->dev);
-	eth->soc = (struct mtk_soc_data *)match->data;
+	eth->soc = of_device_get_match_data(&pdev->dev);
 
 	eth->dev = &pdev->dev;
 	eth->base = devm_ioremap_resource(&pdev->dev, res);
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index 5c613c6663da..9f54ccbddea7 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -593,30 +593,25 @@ static int get_fixed_ipv4_csum(__wsum hw_checksum, struct sk_buff *skb,
 }
 
 #if IS_ENABLED(CONFIG_IPV6)
-/* In IPv6 packets, besides subtracting the pseudo header checksum,
- * we also compute/add the IP header checksum which
- * is not added by the HW.
+/* In IPv6 packets, hw_checksum lacks 6 bytes from IPv6 header:
+ * 4 first bytes : priority, version, flow_lbl
+ * and 2 additional bytes : nexthdr, hop_limit.
  */
 static int get_fixed_ipv6_csum(__wsum hw_checksum, struct sk_buff *skb,
 			       struct ipv6hdr *ipv6h)
 {
 	__u8 nexthdr = ipv6h->nexthdr;
-	__wsum csum_pseudo_hdr = 0;
+	__wsum temp;
 
 	if (unlikely(nexthdr == IPPROTO_FRAGMENT ||
 		     nexthdr == IPPROTO_HOPOPTS ||
 		     nexthdr == IPPROTO_SCTP))
 		return -1;
-	hw_checksum = csum_add(hw_checksum, (__force __wsum)htons(nexthdr));
 
-	csum_pseudo_hdr = csum_partial(&ipv6h->saddr,
-				       sizeof(ipv6h->saddr) + sizeof(ipv6h->daddr), 0);
-	csum_pseudo_hdr = csum_add(csum_pseudo_hdr, (__force __wsum)ipv6h->payload_len);
-	csum_pseudo_hdr = csum_add(csum_pseudo_hdr,
-				   (__force __wsum)htons(nexthdr));
-
-	skb->csum = csum_sub(hw_checksum, csum_pseudo_hdr);
-	skb->csum = csum_add(skb->csum, csum_partial(ipv6h, sizeof(struct ipv6hdr), 0));
+	/* priority, version, flow_lbl */
+	temp = csum_add(hw_checksum, *(__wsum *)ipv6h);
+	/* nexthdr and hop_limit */
+	skb->csum = csum_add(temp, (__force __wsum)*(__be16 *)&ipv6h->nexthdr);
 	return 0;
 }
 #endif
@@ -775,8 +770,8 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
 
 			act = bpf_prog_run_xdp(xdp_prog, &xdp);
 
+			length = xdp.data_end - xdp.data;
 			if (xdp.data != orig_data) {
-				length = xdp.data_end - xdp.data;
 				frags[0].page_offset = xdp.data -
 					xdp.data_hard_start;
 				va = xdp.data;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
index 6b6853773848..0227786308af 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
@@ -694,7 +694,7 @@ u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb,
 	u16 rings_p_up = priv->num_tx_rings_p_up;
 
 	if (netdev_get_num_tc(dev))
-		return skb_tx_hash(dev, skb);
+		return fallback(dev, skb);
 
 	return fallback(dev, skb) % rings_p_up;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c
index de6b3d416148..46dcbfbe4c5e 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.c
@@ -165,6 +165,7 @@ static void dump_dev_cap_flags2(struct mlx4_dev *dev, u64 flags)
 		[36] = "QinQ VST mode support",
 		[37] = "sl to vl mapping table change event support",
 		[38] = "user MAC support",
+		[39] = "Report driver version to FW support",
 	};
 	int i;
 
@@ -1038,6 +1039,8 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_ETH_BACKPL_AN_REP;
 	if (field32 & (1 << 7))
 		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT;
+	if (field32 & (1 << 8))
+		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_DRIVER_VERSION_TO_FW;
 	MLX4_GET(field32, outbox, QUERY_DEV_CAP_DIAG_RPRT_PER_PORT);
 	if (field32 & (1 << 17))
 		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_DIAG_PER_PORT;
@@ -1860,6 +1863,8 @@ int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param)
 #define  INIT_HCA_UC_STEERING_OFFSET	 (INIT_HCA_MCAST_OFFSET + 0x18)
 #define	 INIT_HCA_LOG_MC_TABLE_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x1b)
 #define  INIT_HCA_DEVICE_MANAGED_FLOW_STEERING_EN	0x6
+#define  INIT_HCA_DRIVER_VERSION_OFFSET   0x140
+#define  INIT_HCA_DRIVER_VERSION_SZ       0x40
 #define  INIT_HCA_FS_PARAM_OFFSET         0x1d0
 #define  INIT_HCA_FS_BASE_OFFSET          (INIT_HCA_FS_PARAM_OFFSET + 0x00)
 #define  INIT_HCA_FS_LOG_ENTRY_SZ_OFFSET  (INIT_HCA_FS_PARAM_OFFSET + 0x12)
@@ -1950,6 +1955,13 @@ int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param)
 	if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT)
 		*(inbox + INIT_HCA_RECOVERABLE_ERROR_EVENT_OFFSET / 4) |= cpu_to_be32(1 << 31);
 
+	if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_DRIVER_VERSION_TO_FW) {
+		u8 *dst = (u8 *)(inbox + INIT_HCA_DRIVER_VERSION_OFFSET / 4);
+
+		strncpy(dst, DRV_NAME_FOR_FW, INIT_HCA_DRIVER_VERSION_SZ - 1);
+		mlx4_dbg(dev, "Reporting Driver Version to FW: %s\n", dst);
+	}
+
 	/* QPC/EEC/CQC/EQC/RDMARC attributes */
 
 	MLX4_PUT(inbox, param->qpc_base,      INIT_HCA_QPC_BASE_OFFSET);
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 60172a38c4a4..0a30d81aab3b 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -73,7 +73,7 @@ MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
 
 static int msi_x = 1;
 module_param(msi_x, int, 0444);
-MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");
+MODULE_PARM_DESC(msi_x, "0 - don't use MSI-X, 1 - use MSI-X, >1 - limit number of MSI-X irqs to msi_x");
 
 #else /* CONFIG_PCI_MSI */
 
@@ -2815,6 +2815,9 @@ static void mlx4_enable_msi_x(struct mlx4_dev *dev)
 				dev->caps.num_eqs - dev->caps.reserved_eqs,
 				MAX_MSIX);
 
+		if (msi_x > 1)
+			nreq = min_t(int, nreq, msi_x);
+
 		entries = kcalloc(nreq, sizeof(*entries), GFP_KERNEL);
 		if (!entries)
 			goto no_msi;
@@ -4127,17 +4130,68 @@ static const struct pci_error_handlers mlx4_err_handler = {
 	.resume		= mlx4_pci_resume,
 };
 
+static int mlx4_suspend(struct pci_dev *pdev, pm_message_t state)
+{
+	struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
+	struct mlx4_dev	*dev = persist->dev;
+
+	mlx4_err(dev, "suspend was called\n");
+	mutex_lock(&persist->interface_state_mutex);
+	if (persist->interface_state & MLX4_INTERFACE_STATE_UP)
+		mlx4_unload_one(pdev);
+	mutex_unlock(&persist->interface_state_mutex);
+
+	return 0;
+}
+
+static int mlx4_resume(struct pci_dev *pdev)
+{
+	struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
+	struct mlx4_dev	*dev = persist->dev;
+	struct mlx4_priv *priv = mlx4_priv(dev);
+	int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0};
+	int total_vfs;
+	int ret = 0;
+
+	mlx4_err(dev, "resume was called\n");
+	total_vfs = dev->persist->num_vfs;
+	memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs));
+
+	mutex_lock(&persist->interface_state_mutex);
+	if (!(persist->interface_state & MLX4_INTERFACE_STATE_UP)) {
+		ret = mlx4_load_one(pdev, priv->pci_dev_data, total_vfs,
+				    nvfs, priv, 1);
+		if (!ret) {
+			ret = restore_current_port_types(dev,
+					dev->persist->curr_port_type,
+					dev->persist->curr_port_poss_type);
+			if (ret)
+				mlx4_err(dev, "resume: could not restore original port types (%d)\n", ret);
+		}
+	}
+	mutex_unlock(&persist->interface_state_mutex);
+
+	return ret;
+}
+
 static struct pci_driver mlx4_driver = {
 	.name		= DRV_NAME,
 	.id_table	= mlx4_pci_table,
 	.probe		= mlx4_init_one,
 	.shutdown	= mlx4_shutdown,
 	.remove		= mlx4_remove_one,
+	.suspend	= mlx4_suspend,
+	.resume		= mlx4_resume,
 	.err_handler    = &mlx4_err_handler,
 };
 
 static int __init mlx4_verify_params(void)
 {
+	if (msi_x < 0) {
+		pr_warn("mlx4_core: bad msi_x: %d\n", msi_x);
+		return -1;
+	}
+
 	if ((log_num_mac < 0) || (log_num_mac > 7)) {
 		pr_warn("mlx4_core: bad num_mac: %d\n", log_num_mac);
 		return -1;
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4.h b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
index c68da1986e51..cb9e923e8399 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
@@ -55,8 +55,8 @@
 #include "fw_qos.h"
 
 #define DRV_NAME	"mlx4_core"
-#define PFX		DRV_NAME ": "
 #define DRV_VERSION	"4.0-0"
+#define DRV_NAME_FOR_FW		"Linux," DRV_NAME "," DRV_VERSION
 
 #define MLX4_FS_UDP_UC_EN		(1 << 1)
 #define MLX4_FS_TCP_UC_EN		(1 << 2)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
index c032319f1cb9..2545296a0c08 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
@@ -30,6 +30,7 @@ config MLX5_CORE_EN
 	bool "Mellanox Technologies ConnectX-4 Ethernet support"
 	depends on NETDEVICES && ETHERNET && INET && PCI && MLX5_CORE
 	depends on IPV6=y || IPV6=n || MLX5_CORE=m
+	select PAGE_POOL
 	default n
 	---help---
 	  Ethernet support in Mellanox Technologies ConnectX-4 NIC.
@@ -85,3 +86,15 @@ config MLX5_EN_IPSEC
 	  Build support for IPsec cryptography-offload accelaration in the NIC.
 	  Note: Support for hardware with this capability needs to be selected
 	  for this option to become available.
+
+config MLX5_EN_TLS
+	bool "TLS cryptography-offload accelaration"
+	depends on MLX5_CORE_EN
+	depends on TLS_DEVICE
+	depends on TLS=y || MLX5_CORE=m
+	depends on MLX5_ACCEL
+	default n
+	---help---
+	  Build support for TLS cryptography-offload accelaration in the NIC.
+	  Note: Support for hardware with this capability needs to be selected
+	  for this option to become available.
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index c805769d92a9..9efbf193ad5a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -8,24 +8,26 @@ mlx5_core-y :=	main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
 		fs_counters.o rl.o lag.o dev.o wq.o lib/gid.o lib/clock.o \
 		diag/fs_tracepoint.o
 
-mlx5_core-$(CONFIG_MLX5_ACCEL) += accel/ipsec.o
+mlx5_core-$(CONFIG_MLX5_ACCEL) += accel/ipsec.o accel/tls.o
 
 mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o \
-		fpga/ipsec.o
+		fpga/ipsec.o fpga/tls.o
 
 mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \
 		en_tx.o en_rx.o en_dim.o en_txrx.o en_stats.o vxlan.o \
-		en_arfs.o en_fs_ethtool.o en_selftest.o
+		en_arfs.o en_fs_ethtool.o en_selftest.o en/port.o
 
 mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o
 
 mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o en_rep.o en_tc.o
 
-mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) +=  en_dcbnl.o
+mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) +=  en_dcbnl.o en/port_buffer.o
 
 mlx5_core-$(CONFIG_MLX5_CORE_IPOIB) += ipoib/ipoib.o ipoib/ethtool.o ipoib/ipoib_vlan.o
 
 mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \
 		en_accel/ipsec_stats.o
 
+mlx5_core-$(CONFIG_MLX5_EN_TLS) +=  en_accel/tls.o en_accel/tls_rxtx.o en_accel/tls_stats.o
+
 CFLAGS_tracepoint.o := -I$(src)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.c b/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.c
new file mode 100644
index 000000000000..77ac19f38cbe
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/mlx5/device.h>
+
+#include "accel/tls.h"
+#include "mlx5_core.h"
+#include "fpga/tls.h"
+
+int mlx5_accel_tls_add_tx_flow(struct mlx5_core_dev *mdev, void *flow,
+			       struct tls_crypto_info *crypto_info,
+			       u32 start_offload_tcp_sn, u32 *p_swid)
+{
+	return mlx5_fpga_tls_add_tx_flow(mdev, flow, crypto_info,
+					 start_offload_tcp_sn, p_swid);
+}
+
+void mlx5_accel_tls_del_tx_flow(struct mlx5_core_dev *mdev, u32 swid)
+{
+	mlx5_fpga_tls_del_tx_flow(mdev, swid, GFP_KERNEL);
+}
+
+bool mlx5_accel_is_tls_device(struct mlx5_core_dev *mdev)
+{
+	return mlx5_fpga_is_tls_device(mdev);
+}
+
+u32 mlx5_accel_tls_device_caps(struct mlx5_core_dev *mdev)
+{
+	return mlx5_fpga_tls_device_caps(mdev);
+}
+
+int mlx5_accel_tls_init(struct mlx5_core_dev *mdev)
+{
+	return mlx5_fpga_tls_init(mdev);
+}
+
+void mlx5_accel_tls_cleanup(struct mlx5_core_dev *mdev)
+{
+	mlx5_fpga_tls_cleanup(mdev);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h b/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h
new file mode 100644
index 000000000000..6f9c9f446ecc
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef __MLX5_ACCEL_TLS_H__
+#define __MLX5_ACCEL_TLS_H__
+
+#include <linux/mlx5/driver.h>
+#include <linux/tls.h>
+
+#ifdef CONFIG_MLX5_ACCEL
+
+enum {
+	MLX5_ACCEL_TLS_TX = BIT(0),
+	MLX5_ACCEL_TLS_RX = BIT(1),
+	MLX5_ACCEL_TLS_V12 = BIT(2),
+	MLX5_ACCEL_TLS_V13 = BIT(3),
+	MLX5_ACCEL_TLS_LRO = BIT(4),
+	MLX5_ACCEL_TLS_IPV6 = BIT(5),
+	MLX5_ACCEL_TLS_AES_GCM128 = BIT(30),
+	MLX5_ACCEL_TLS_AES_GCM256 = BIT(31),
+};
+
+struct mlx5_ifc_tls_flow_bits {
+	u8         src_port[0x10];
+	u8         dst_port[0x10];
+	union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits src_ipv4_src_ipv6;
+	union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits dst_ipv4_dst_ipv6;
+	u8         ipv6[0x1];
+	u8         direction_sx[0x1];
+	u8         reserved_at_2[0x1e];
+};
+
+int mlx5_accel_tls_add_tx_flow(struct mlx5_core_dev *mdev, void *flow,
+			       struct tls_crypto_info *crypto_info,
+			       u32 start_offload_tcp_sn, u32 *p_swid);
+void mlx5_accel_tls_del_tx_flow(struct mlx5_core_dev *mdev, u32 swid);
+bool mlx5_accel_is_tls_device(struct mlx5_core_dev *mdev);
+u32 mlx5_accel_tls_device_caps(struct mlx5_core_dev *mdev);
+int mlx5_accel_tls_init(struct mlx5_core_dev *mdev);
+void mlx5_accel_tls_cleanup(struct mlx5_core_dev *mdev);
+
+#else
+
+static inline int
+mlx5_accel_tls_add_tx_flow(struct mlx5_core_dev *mdev, void *flow,
+			   struct tls_crypto_info *crypto_info,
+			   u32 start_offload_tcp_sn, u32 *p_swid) { return 0; }
+static inline void mlx5_accel_tls_del_tx_flow(struct mlx5_core_dev *mdev, u32 swid) { }
+static inline bool mlx5_accel_is_tls_device(struct mlx5_core_dev *mdev) { return false; }
+static inline u32 mlx5_accel_tls_device_caps(struct mlx5_core_dev *mdev) { return 0; }
+static inline int mlx5_accel_tls_init(struct mlx5_core_dev *mdev) { return 0; }
+static inline void mlx5_accel_tls_cleanup(struct mlx5_core_dev *mdev) { }
+
+#endif
+
+#endif	/* __MLX5_ACCEL_TLS_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
index 21cd1703a862..487388aed98f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
@@ -135,6 +135,14 @@ static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
 	return cmd->cmd_buf + (idx << cmd->log_stride);
 }
 
+static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg)
+{
+	int size = msg->len;
+	int blen = size - min_t(int, sizeof(msg->first.data), size);
+
+	return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE);
+}
+
 static u8 xor8_buf(void *buf, size_t offset, int len)
 {
 	u8 *ptr = buf;
@@ -174,10 +182,7 @@ static void calc_block_sig(struct mlx5_cmd_prot_block *block)
 static void calc_chain_sig(struct mlx5_cmd_msg *msg)
 {
 	struct mlx5_cmd_mailbox *next = msg->next;
-	int size = msg->len;
-	int blen = size - min_t(int, sizeof(msg->first.data), size);
-	int n = (blen + MLX5_CMD_DATA_BLOCK_SIZE - 1)
-		/ MLX5_CMD_DATA_BLOCK_SIZE;
+	int n = mlx5_calc_cmd_blocks(msg);
 	int i = 0;
 
 	for (i = 0; i < n && next; i++)  {
@@ -220,12 +225,9 @@ static void free_cmd(struct mlx5_cmd_work_ent *ent)
 static int verify_signature(struct mlx5_cmd_work_ent *ent)
 {
 	struct mlx5_cmd_mailbox *next = ent->out->next;
+	int n = mlx5_calc_cmd_blocks(ent->out);
 	int err;
 	u8 sig;
-	int size = ent->out->len;
-	int blen = size - min_t(int, sizeof(ent->out->first.data), size);
-	int n = (blen + MLX5_CMD_DATA_BLOCK_SIZE - 1)
-		/ MLX5_CMD_DATA_BLOCK_SIZE;
 	int i = 0;
 
 	sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay));
@@ -720,9 +722,11 @@ static void dump_command(struct mlx5_core_dev *dev,
 	struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
 	u16 op = MLX5_GET(mbox_in, ent->lay->in, opcode);
 	struct mlx5_cmd_mailbox *next = msg->next;
+	int n = mlx5_calc_cmd_blocks(msg);
 	int data_only;
 	u32 offset = 0;
 	int dump_len;
+	int i;
 
 	data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA));
 
@@ -749,7 +753,7 @@ static void dump_command(struct mlx5_core_dev *dev,
 		offset += sizeof(*ent->lay);
 	}
 
-	while (next && offset < msg->len) {
+	for (i = 0; i < n && next; i++)  {
 		if (data_only) {
 			dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset);
 			dump_buf(next->buf, dump_len, 1, offset);
@@ -1137,7 +1141,6 @@ static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
 	struct mlx5_cmd_mailbox *tmp, *head = NULL;
 	struct mlx5_cmd_prot_block *block;
 	struct mlx5_cmd_msg *msg;
-	int blen;
 	int err;
 	int n;
 	int i;
@@ -1146,8 +1149,8 @@ static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
 	if (!msg)
 		return ERR_PTR(-ENOMEM);
 
-	blen = size - min_t(int, sizeof(msg->first.data), size);
-	n = (blen + MLX5_CMD_DATA_BLOCK_SIZE - 1) / MLX5_CMD_DATA_BLOCK_SIZE;
+	msg->len = size;
+	n = mlx5_calc_cmd_blocks(msg);
 
 	for (i = 0; i < n; i++) {
 		tmp = alloc_cmd_box(dev, flags);
@@ -1165,7 +1168,6 @@ static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
 		head = tmp;
 	}
 	msg->next = head;
-	msg->len = size;
 	return msg;
 
 err_alloc:
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
index d93ff567b40d..b3820a34e773 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
@@ -235,7 +235,7 @@ const char *parse_fs_dst(struct trace_seq *p,
 
 	switch (dst->type) {
 	case MLX5_FLOW_DESTINATION_TYPE_VPORT:
-		trace_seq_printf(p, "vport=%u\n", dst->vport_num);
+		trace_seq_printf(p, "vport=%u\n", dst->vport.num);
 		break;
 	case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
 		trace_seq_printf(p, "ft=%p\n", dst->ft);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 30cad07be2b5..1c04df043e07 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -53,6 +53,11 @@
 #include "mlx5_core.h"
 #include "en_stats.h"
 
+struct page_pool;
+
+#define MLX5E_METADATA_ETHER_TYPE (0x8CE4)
+#define MLX5E_METADATA_ETHER_LEN 8
+
 #define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v)
 
 #define MLX5E_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN)
@@ -60,6 +65,7 @@
 #define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu))
 #define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu))
 
+#define MLX5E_MAX_PRIORITY      8
 #define MLX5E_MAX_DSCP          64
 #define MLX5E_MAX_NUM_TC	8
 
@@ -177,6 +183,7 @@ static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
 struct mlx5e_tx_wqe {
 	struct mlx5_wqe_ctrl_seg ctrl;
 	struct mlx5_wqe_eth_seg  eth;
+	struct mlx5_wqe_data_seg data[0];
 };
 
 struct mlx5e_rx_wqe {
@@ -239,6 +246,7 @@ struct mlx5e_params {
 	bool vlan_strip_disable;
 	bool scatter_fcs_en;
 	bool rx_dim_enabled;
+	bool tx_dim_enabled;
 	u32 lro_timeout;
 	u32 pflags;
 	struct bpf_prog *xdp_prog;
@@ -269,6 +277,11 @@ struct mlx5e_dcbx {
 	/* The only setting that cannot be read from FW */
 	u8                         tc_tsa[IEEE_8021QAZ_MAX_TCS];
 	u8                         cap;
+
+	/* Buffer configuration */
+	bool                       manual_buffer;
+	u32                        cable_len;
+	u32                        xoff;
 };
 
 struct mlx5e_dcbx_dp {
@@ -282,8 +295,6 @@ enum {
 	MLX5E_RQ_STATE_AM,
 };
 
-#define MLX5E_TEST_BIT(state, nr) (state & BIT(nr))
-
 struct mlx5e_cq {
 	/* data path - accessed per cqe */
 	struct mlx5_cqwq           wq;
@@ -303,7 +314,7 @@ struct mlx5e_cq {
 
 	/* control */
 	struct mlx5_core_dev      *mdev;
-	struct mlx5_frag_wq_ctrl   wq_ctrl;
+	struct mlx5_wq_ctrl        wq_ctrl;
 } ____cacheline_aligned_in_smp;
 
 struct mlx5e_tx_wqe_info {
@@ -328,6 +339,8 @@ enum {
 	MLX5E_SQ_STATE_ENABLED,
 	MLX5E_SQ_STATE_RECOVERING,
 	MLX5E_SQ_STATE_IPSEC,
+	MLX5E_SQ_STATE_AM,
+	MLX5E_SQ_STATE_TLS,
 };
 
 struct mlx5e_sq_wqe_info {
@@ -340,11 +353,11 @@ struct mlx5e_txqsq {
 	/* dirtied @completion */
 	u16                        cc;
 	u32                        dma_fifo_cc;
+	struct net_dim             dim; /* Adaptive Moderation */
 
 	/* dirtied @xmit */
 	u16                        pc ____cacheline_aligned_in_smp;
 	u32                        dma_fifo_pc;
-	struct mlx5e_sq_stats      stats;
 
 	struct mlx5e_cq            cq;
 
@@ -357,11 +370,11 @@ struct mlx5e_txqsq {
 	/* read only */
 	struct mlx5_wq_cyc         wq;
 	u32                        dma_fifo_mask;
+	struct mlx5e_sq_stats     *stats;
 	void __iomem              *uar_map;
 	struct netdev_queue       *txq;
 	u32                        sqn;
 	u8                         min_inline_mode;
-	u16                        edge;
 	struct device             *pdev;
 	__be32                     mkey_be;
 	unsigned long              state;
@@ -392,6 +405,7 @@ struct mlx5e_xdpsq {
 	struct {
 		struct mlx5e_dma_info     *di;
 		bool                       doorbell;
+		bool                       redirect_flush;
 	} db;
 
 	/* read only */
@@ -425,7 +439,6 @@ struct mlx5e_icosq {
 	struct mlx5_wq_cyc         wq;
 	void __iomem              *uar_map;
 	u32                        sqn;
-	u16                        edge;
 	unsigned long              state;
 
 	/* control path */
@@ -436,7 +449,7 @@ struct mlx5e_icosq {
 static inline bool
 mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n)
 {
-	return (((wq->sz_m1 & (cc - pc)) >= n) || (cc == pc));
+	return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc);
 }
 
 struct mlx5e_dma_info {
@@ -513,7 +526,7 @@ struct mlx5e_rq {
 	struct mlx5e_channel  *channel;
 	struct device         *pdev;
 	struct net_device     *netdev;
-	struct mlx5e_rq_stats  stats;
+	struct mlx5e_rq_stats *stats;
 	struct mlx5e_cq        cq;
 	struct mlx5e_page_cache page_cache;
 	struct hwtstamp_config *tstamp;
@@ -533,6 +546,7 @@ struct mlx5e_rq {
 	unsigned int           hw_mtu;
 	struct mlx5e_xdpsq     xdpsq;
 	DECLARE_BITMAP(flags, 8);
+	struct page_pool      *page_pool;
 
 	/* control */
 	struct mlx5_wq_ctrl    wq_ctrl;
@@ -560,7 +574,7 @@ struct mlx5e_channel {
 
 	/* data path - accessed per napi poll */
 	struct irq_desc *irq_desc;
-	struct mlx5e_ch_stats      stats;
+	struct mlx5e_ch_stats     *stats;
 
 	/* control */
 	struct mlx5e_priv         *priv;
@@ -576,6 +590,12 @@ struct mlx5e_channels {
 	struct mlx5e_params    params;
 };
 
+struct mlx5e_channel_stats {
+	struct mlx5e_ch_stats ch;
+	struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];
+	struct mlx5e_rq_stats rq;
+} ____cacheline_aligned_in_smp;
+
 enum mlx5e_traffic_types {
 	MLX5E_TT_IPV4_TCP,
 	MLX5E_TT_IPV6_TCP,
@@ -625,7 +645,6 @@ struct mlx5e_flow_table {
 struct mlx5e_tc_table {
 	struct mlx5_flow_table		*t;
 
-	struct rhashtable_params        ht_params;
 	struct rhashtable               ht;
 
 	DECLARE_HASHTABLE(mod_hdr_tbl, 8);
@@ -758,6 +777,8 @@ struct mlx5e_priv {
 	struct mutex               state_lock; /* Protects Interface state */
 	struct mlx5e_rq            drop_rq;
 
+	rwlock_t                   stats_lock; /* Protects channels SW stats updates */
+	bool                       channels_active;
 	struct mlx5e_channels      channels;
 	u32                        tisn[MLX5E_MAX_NUM_TC];
 	struct mlx5e_rqt           indir_rqt;
@@ -778,6 +799,8 @@ struct mlx5e_priv {
 	struct mlx5_core_dev      *mdev;
 	struct net_device         *netdev;
 	struct mlx5e_stats         stats;
+	struct mlx5e_channel_stats channel_stats[MLX5E_MAX_NUM_CHANNELS];
+	u8                         max_opened_tc;
 	struct hwtstamp_config     tstamp;
 	u16                        q_counter;
 	u16                        drop_rq_q_counter;
@@ -790,6 +813,9 @@ struct mlx5e_priv {
 #ifdef CONFIG_MLX5_EN_IPSEC
 	struct mlx5e_ipsec        *ipsec;
 #endif
+#ifdef CONFIG_MLX5_EN_TLS
+	struct mlx5e_tls          *tls;
+#endif
 };
 
 struct mlx5e_profile {
@@ -820,6 +846,8 @@ void mlx5e_build_ptys2ethtool_map(void);
 u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
 		       void *accel_priv, select_queue_fallback_t fallback);
 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev);
+netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
+			  struct mlx5e_tx_wqe *wqe, u16 pi);
 
 void mlx5e_completion_event(struct mlx5_core_cq *mcq);
 void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
@@ -919,8 +947,6 @@ void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv);
 
 void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
 				   int num_channels);
-int mlx5e_get_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed);
-
 void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params,
 				 u8 cq_period_mode);
 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params,
@@ -935,10 +961,21 @@ static inline bool mlx5e_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev)
 		MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.inner_ip_version));
 }
 
+static inline void mlx5e_sq_fetch_wqe(struct mlx5e_txqsq *sq,
+				      struct mlx5e_tx_wqe **wqe,
+				      u16 *pi)
+{
+	struct mlx5_wq_cyc *wq = &sq->wq;
+
+	*pi  = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
+	*wqe = mlx5_wq_cyc_get_wqe(wq, *pi);
+	memset(*wqe, 0, sizeof(**wqe));
+}
+
 static inline
 struct mlx5e_tx_wqe *mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
 {
-	u16                         pi   = *pc & wq->sz_m1;
+	u16                         pi   = mlx5_wq_cyc_ctr2ix(wq, *pc);
 	struct mlx5e_tx_wqe        *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
 	struct mlx5_wqe_ctrl_seg   *cseg = &wqe->ctrl;
 
@@ -1092,9 +1129,6 @@ int mlx5e_ethtool_get_ts_info(struct mlx5e_priv *priv,
 int mlx5e_ethtool_flash_device(struct mlx5e_priv *priv,
 			       struct ethtool_flash *flash);
 
-int mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
-			    void *cb_priv);
-
 /* mlx5e generic netdev management API */
 struct net_device*
 mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile,
@@ -1107,4 +1141,5 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
 			    u16 max_channels, u16 mtu);
 u8 mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev);
 void mlx5e_rx_dim_work(struct work_struct *work);
+void mlx5e_tx_dim_work(struct work_struct *work);
 #endif /* __MLX5_EN_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/en/Makefile
new file mode 100644
index 000000000000..d8e17110f25d
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/Makefile
@@ -0,0 +1 @@
+subdir-ccflags-y += -I$(src)/..
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port.c b/drivers/net/ethernet/mellanox/mlx5/core/en/port.c
new file mode 100644
index 000000000000..24e3b564964f
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port.c
@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "port.h"
+
+/* speed in units of 1Mb */
+static const u32 mlx5e_link_speed[MLX5E_LINK_MODES_NUMBER] = {
+	[MLX5E_1000BASE_CX_SGMII] = 1000,
+	[MLX5E_1000BASE_KX]       = 1000,
+	[MLX5E_10GBASE_CX4]       = 10000,
+	[MLX5E_10GBASE_KX4]       = 10000,
+	[MLX5E_10GBASE_KR]        = 10000,
+	[MLX5E_20GBASE_KR2]       = 20000,
+	[MLX5E_40GBASE_CR4]       = 40000,
+	[MLX5E_40GBASE_KR4]       = 40000,
+	[MLX5E_56GBASE_R4]        = 56000,
+	[MLX5E_10GBASE_CR]        = 10000,
+	[MLX5E_10GBASE_SR]        = 10000,
+	[MLX5E_10GBASE_ER]        = 10000,
+	[MLX5E_40GBASE_SR4]       = 40000,
+	[MLX5E_40GBASE_LR4]       = 40000,
+	[MLX5E_50GBASE_SR2]       = 50000,
+	[MLX5E_100GBASE_CR4]      = 100000,
+	[MLX5E_100GBASE_SR4]      = 100000,
+	[MLX5E_100GBASE_KR4]      = 100000,
+	[MLX5E_100GBASE_LR4]      = 100000,
+	[MLX5E_100BASE_TX]        = 100,
+	[MLX5E_1000BASE_T]        = 1000,
+	[MLX5E_10GBASE_T]         = 10000,
+	[MLX5E_25GBASE_CR]        = 25000,
+	[MLX5E_25GBASE_KR]        = 25000,
+	[MLX5E_25GBASE_SR]        = 25000,
+	[MLX5E_50GBASE_CR2]       = 50000,
+	[MLX5E_50GBASE_KR2]       = 50000,
+};
+
+u32 mlx5e_port_ptys2speed(u32 eth_proto_oper)
+{
+	unsigned long temp = eth_proto_oper;
+	u32 speed = 0;
+	int i;
+
+	i = find_first_bit(&temp, MLX5E_LINK_MODES_NUMBER);
+	if (i < MLX5E_LINK_MODES_NUMBER)
+		speed = mlx5e_link_speed[i];
+
+	return speed;
+}
+
+int mlx5e_port_linkspeed(struct mlx5_core_dev *mdev, u32 *speed)
+{
+	u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {};
+	u32 eth_proto_oper;
+	int err;
+
+	err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, 1);
+	if (err)
+		return err;
+
+	eth_proto_oper = MLX5_GET(ptys_reg, out, eth_proto_oper);
+	*speed = mlx5e_port_ptys2speed(eth_proto_oper);
+	if (!(*speed)) {
+		mlx5_core_warn(mdev, "cannot get port speed\n");
+		err = -EINVAL;
+	}
+
+	return err;
+}
+
+int mlx5e_port_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed)
+{
+	u32 max_speed = 0;
+	u32 proto_cap;
+	int err;
+	int i;
+
+	err = mlx5_query_port_proto_cap(mdev, &proto_cap, MLX5_PTYS_EN);
+	if (err)
+		return err;
+
+	for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i)
+		if (proto_cap & MLX5E_PROT_MASK(i))
+			max_speed = max(max_speed, mlx5e_link_speed[i]);
+
+	*speed = max_speed;
+	return 0;
+}
+
+u32 mlx5e_port_speed2linkmodes(u32 speed)
+{
+	u32 link_modes = 0;
+	int i;
+
+	for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
+		if (mlx5e_link_speed[i] == speed)
+			link_modes |= MLX5E_PROT_MASK(i);
+	}
+
+	return link_modes;
+}
+
+int mlx5e_port_query_pbmc(struct mlx5_core_dev *mdev, void *out)
+{
+	int sz = MLX5_ST_SZ_BYTES(pbmc_reg);
+	void *in;
+	int err;
+
+	in = kzalloc(sz, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(pbmc_reg, in, local_port, 1);
+	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PBMC, 0, 0);
+
+	kfree(in);
+	return err;
+}
+
+int mlx5e_port_set_pbmc(struct mlx5_core_dev *mdev, void *in)
+{
+	int sz = MLX5_ST_SZ_BYTES(pbmc_reg);
+	void *out;
+	int err;
+
+	out = kzalloc(sz, GFP_KERNEL);
+	if (!out)
+		return -ENOMEM;
+
+	MLX5_SET(pbmc_reg, in, local_port, 1);
+	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PBMC, 0, 1);
+
+	kfree(out);
+	return err;
+}
+
+/* buffer[i]: buffer that priority i mapped to */
+int mlx5e_port_query_priority2buffer(struct mlx5_core_dev *mdev, u8 *buffer)
+{
+	int sz = MLX5_ST_SZ_BYTES(pptb_reg);
+	u32 prio_x_buff;
+	void *out;
+	void *in;
+	int prio;
+	int err;
+
+	in = kzalloc(sz, GFP_KERNEL);
+	out = kzalloc(sz, GFP_KERNEL);
+	if (!in || !out) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	MLX5_SET(pptb_reg, in, local_port, 1);
+	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPTB, 0, 0);
+	if (err)
+		goto out;
+
+	prio_x_buff = MLX5_GET(pptb_reg, out, prio_x_buff);
+	for (prio = 0; prio < 8; prio++) {
+		buffer[prio] = (u8)(prio_x_buff >> (4 * prio)) & 0xF;
+		mlx5_core_dbg(mdev, "prio %d, buffer %d\n", prio, buffer[prio]);
+	}
+out:
+	kfree(in);
+	kfree(out);
+	return err;
+}
+
+int mlx5e_port_set_priority2buffer(struct mlx5_core_dev *mdev, u8 *buffer)
+{
+	int sz = MLX5_ST_SZ_BYTES(pptb_reg);
+	u32 prio_x_buff;
+	void *out;
+	void *in;
+	int prio;
+	int err;
+
+	in = kzalloc(sz, GFP_KERNEL);
+	out = kzalloc(sz, GFP_KERNEL);
+	if (!in || !out) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	/* First query the pptb register */
+	MLX5_SET(pptb_reg, in, local_port, 1);
+	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPTB, 0, 0);
+	if (err)
+		goto out;
+
+	memcpy(in, out, sz);
+	MLX5_SET(pptb_reg, in, local_port, 1);
+
+	/* Update the pm and prio_x_buff */
+	MLX5_SET(pptb_reg, in, pm, 0xFF);
+
+	prio_x_buff = 0;
+	for (prio = 0; prio < 8; prio++)
+		prio_x_buff |= (buffer[prio] << (4 * prio));
+	MLX5_SET(pptb_reg, in, prio_x_buff, prio_x_buff);
+
+	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPTB, 0, 1);
+
+out:
+	kfree(in);
+	kfree(out);
+	return err;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port.h b/drivers/net/ethernet/mellanox/mlx5/core/en/port.h
new file mode 100644
index 000000000000..f8cbd8194179
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __MLX5E_EN_PORT_H
+#define __MLX5E_EN_PORT_H
+
+#include <linux/mlx5/driver.h>
+#include "en.h"
+
+u32 mlx5e_port_ptys2speed(u32 eth_proto_oper);
+int mlx5e_port_linkspeed(struct mlx5_core_dev *mdev, u32 *speed);
+int mlx5e_port_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed);
+u32 mlx5e_port_speed2linkmodes(u32 speed);
+
+int mlx5e_port_query_pbmc(struct mlx5_core_dev *mdev, void *out);
+int mlx5e_port_set_pbmc(struct mlx5_core_dev *mdev, void *in);
+int mlx5e_port_query_priority2buffer(struct mlx5_core_dev *mdev, u8 *buffer);
+int mlx5e_port_set_priority2buffer(struct mlx5_core_dev *mdev, u8 *buffer);
+#endif
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c
new file mode 100644
index 000000000000..c047da8752da
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c
@@ -0,0 +1,327 @@
+/*
+ * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "port_buffer.h"
+
+int mlx5e_port_query_buffer(struct mlx5e_priv *priv,
+			    struct mlx5e_port_buffer *port_buffer)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	int sz = MLX5_ST_SZ_BYTES(pbmc_reg);
+	u32 total_used = 0;
+	void *buffer;
+	void *out;
+	int err;
+	int i;
+
+	out = kzalloc(sz, GFP_KERNEL);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5e_port_query_pbmc(mdev, out);
+	if (err)
+		goto out;
+
+	for (i = 0; i < MLX5E_MAX_BUFFER; i++) {
+		buffer = MLX5_ADDR_OF(pbmc_reg, out, buffer[i]);
+		port_buffer->buffer[i].lossy =
+			MLX5_GET(bufferx_reg, buffer, lossy);
+		port_buffer->buffer[i].epsb =
+			MLX5_GET(bufferx_reg, buffer, epsb);
+		port_buffer->buffer[i].size =
+			MLX5_GET(bufferx_reg, buffer, size) << MLX5E_BUFFER_CELL_SHIFT;
+		port_buffer->buffer[i].xon =
+			MLX5_GET(bufferx_reg, buffer, xon_threshold) << MLX5E_BUFFER_CELL_SHIFT;
+		port_buffer->buffer[i].xoff =
+			MLX5_GET(bufferx_reg, buffer, xoff_threshold) << MLX5E_BUFFER_CELL_SHIFT;
+		total_used += port_buffer->buffer[i].size;
+
+		mlx5e_dbg(HW, priv, "buffer %d: size=%d, xon=%d, xoff=%d, epsb=%d, lossy=%d\n", i,
+			  port_buffer->buffer[i].size,
+			  port_buffer->buffer[i].xon,
+			  port_buffer->buffer[i].xoff,
+			  port_buffer->buffer[i].epsb,
+			  port_buffer->buffer[i].lossy);
+	}
+
+	port_buffer->port_buffer_size =
+		MLX5_GET(pbmc_reg, out, port_buffer_size) << MLX5E_BUFFER_CELL_SHIFT;
+	port_buffer->spare_buffer_size =
+		port_buffer->port_buffer_size - total_used;
+
+	mlx5e_dbg(HW, priv, "total buffer size=%d, spare buffer size=%d\n",
+		  port_buffer->port_buffer_size,
+		  port_buffer->spare_buffer_size);
+out:
+	kfree(out);
+	return err;
+}
+
+static int port_set_buffer(struct mlx5e_priv *priv,
+			   struct mlx5e_port_buffer *port_buffer)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	int sz = MLX5_ST_SZ_BYTES(pbmc_reg);
+	void *buffer;
+	void *in;
+	int err;
+	int i;
+
+	in = kzalloc(sz, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	err = mlx5e_port_query_pbmc(mdev, in);
+	if (err)
+		goto out;
+
+	for (i = 0; i < MLX5E_MAX_BUFFER; i++) {
+		buffer = MLX5_ADDR_OF(pbmc_reg, in, buffer[i]);
+
+		MLX5_SET(bufferx_reg, buffer, size,
+			 port_buffer->buffer[i].size >> MLX5E_BUFFER_CELL_SHIFT);
+		MLX5_SET(bufferx_reg, buffer, lossy,
+			 port_buffer->buffer[i].lossy);
+		MLX5_SET(bufferx_reg, buffer, xoff_threshold,
+			 port_buffer->buffer[i].xoff >> MLX5E_BUFFER_CELL_SHIFT);
+		MLX5_SET(bufferx_reg, buffer, xon_threshold,
+			 port_buffer->buffer[i].xon >> MLX5E_BUFFER_CELL_SHIFT);
+	}
+
+	err = mlx5e_port_set_pbmc(mdev, in);
+out:
+	kfree(in);
+	return err;
+}
+
+/* xoff = ((301+2.16 * len [m]) * speed [Gbps] + 2.72 MTU [B]) */
+static u32 calculate_xoff(struct mlx5e_priv *priv, unsigned int mtu)
+{
+	u32 speed;
+	u32 xoff;
+	int err;
+
+	err = mlx5e_port_linkspeed(priv->mdev, &speed);
+	if (err)
+		return 0;
+
+	xoff = (301 + 216 * priv->dcbx.cable_len / 100) * speed / 1000 + 272 * mtu / 100;
+
+	mlx5e_dbg(HW, priv, "%s: xoff=%d\n", __func__, xoff);
+	return xoff;
+}
+
+static int update_xoff_threshold(struct mlx5e_port_buffer *port_buffer,
+				 u32 xoff, unsigned int mtu)
+{
+	int i;
+
+	for (i = 0; i < MLX5E_MAX_BUFFER; i++) {
+		if (port_buffer->buffer[i].lossy) {
+			port_buffer->buffer[i].xoff = 0;
+			port_buffer->buffer[i].xon  = 0;
+			continue;
+		}
+
+		if (port_buffer->buffer[i].size <
+		    (xoff + mtu + (1 << MLX5E_BUFFER_CELL_SHIFT)))
+			return -ENOMEM;
+
+		port_buffer->buffer[i].xoff = port_buffer->buffer[i].size - xoff;
+		port_buffer->buffer[i].xon  = port_buffer->buffer[i].xoff - mtu;
+	}
+
+	return 0;
+}
+
+/**
+ * update_buffer_lossy()
+ *   mtu: device's MTU
+ *   pfc_en: <input> current pfc configuration
+ *   buffer: <input> current prio to buffer mapping
+ *   xoff:   <input> xoff value
+ *   port_buffer: <output> port receive buffer configuration
+ *   change: <output>
+ *
+ *   Update buffer configuration based on pfc configuraiton and priority
+ *   to buffer mapping.
+ *   Buffer's lossy bit is changed to:
+ *     lossless if there is at least one PFC enabled priority mapped to this buffer
+ *     lossy if all priorities mapped to this buffer are PFC disabled
+ *
+ *   Return:
+ *     Return 0 if no error.
+ *     Set change to true if buffer configuration is modified.
+ */
+static int update_buffer_lossy(unsigned int mtu,
+			       u8 pfc_en, u8 *buffer, u32 xoff,
+			       struct mlx5e_port_buffer *port_buffer,
+			       bool *change)
+{
+	bool changed = false;
+	u8 lossy_count;
+	u8 prio_count;
+	u8 lossy;
+	int prio;
+	int err;
+	int i;
+
+	for (i = 0; i < MLX5E_MAX_BUFFER; i++) {
+		prio_count = 0;
+		lossy_count = 0;
+
+		for (prio = 0; prio < MLX5E_MAX_PRIORITY; prio++) {
+			if (buffer[prio] != i)
+				continue;
+
+			prio_count++;
+			lossy_count += !(pfc_en & (1 << prio));
+		}
+
+		if (lossy_count == prio_count)
+			lossy = 1;
+		else /* lossy_count < prio_count */
+			lossy = 0;
+
+		if (lossy != port_buffer->buffer[i].lossy) {
+			port_buffer->buffer[i].lossy = lossy;
+			changed = true;
+		}
+	}
+
+	if (changed) {
+		err = update_xoff_threshold(port_buffer, xoff, mtu);
+		if (err)
+			return err;
+
+		*change = true;
+	}
+
+	return 0;
+}
+
+int mlx5e_port_manual_buffer_config(struct mlx5e_priv *priv,
+				    u32 change, unsigned int mtu,
+				    struct ieee_pfc *pfc,
+				    u32 *buffer_size,
+				    u8 *prio2buffer)
+{
+	struct mlx5e_port_buffer port_buffer;
+	u32 xoff = calculate_xoff(priv, mtu);
+	bool update_prio2buffer = false;
+	u8 buffer[MLX5E_MAX_PRIORITY];
+	bool update_buffer = false;
+	u32 total_used = 0;
+	u8 curr_pfc_en;
+	int err;
+	int i;
+
+	mlx5e_dbg(HW, priv, "%s: change=%x\n", __func__, change);
+
+	err = mlx5e_port_query_buffer(priv, &port_buffer);
+	if (err)
+		return err;
+
+	if (change & MLX5E_PORT_BUFFER_CABLE_LEN) {
+		update_buffer = true;
+		err = update_xoff_threshold(&port_buffer, xoff, mtu);
+		if (err)
+			return err;
+	}
+
+	if (change & MLX5E_PORT_BUFFER_PFC) {
+		err = mlx5e_port_query_priority2buffer(priv->mdev, buffer);
+		if (err)
+			return err;
+
+		err = update_buffer_lossy(mtu, pfc->pfc_en, buffer, xoff,
+					  &port_buffer, &update_buffer);
+		if (err)
+			return err;
+	}
+
+	if (change & MLX5E_PORT_BUFFER_PRIO2BUFFER) {
+		update_prio2buffer = true;
+		err = mlx5_query_port_pfc(priv->mdev, &curr_pfc_en, NULL);
+		if (err)
+			return err;
+
+		err = update_buffer_lossy(mtu, curr_pfc_en, prio2buffer, xoff,
+					  &port_buffer, &update_buffer);
+		if (err)
+			return err;
+	}
+
+	if (change & MLX5E_PORT_BUFFER_SIZE) {
+		for (i = 0; i < MLX5E_MAX_BUFFER; i++) {
+			mlx5e_dbg(HW, priv, "%s: buffer[%d]=%d\n", __func__, i, buffer_size[i]);
+			if (!port_buffer.buffer[i].lossy && !buffer_size[i]) {
+				mlx5e_dbg(HW, priv, "%s: lossless buffer[%d] size cannot be zero\n",
+					  __func__, i);
+				return -EINVAL;
+			}
+
+			port_buffer.buffer[i].size = buffer_size[i];
+			total_used += buffer_size[i];
+		}
+
+		mlx5e_dbg(HW, priv, "%s: total buffer requested=%d\n", __func__, total_used);
+
+		if (total_used > port_buffer.port_buffer_size)
+			return -EINVAL;
+
+		update_buffer = true;
+		err = update_xoff_threshold(&port_buffer, xoff, mtu);
+		if (err)
+			return err;
+	}
+
+	/* Need to update buffer configuration if xoff value is changed */
+	if (!update_buffer && xoff != priv->dcbx.xoff) {
+		update_buffer = true;
+		err = update_xoff_threshold(&port_buffer, xoff, mtu);
+		if (err)
+			return err;
+	}
+	priv->dcbx.xoff = xoff;
+
+	/* Apply the settings */
+	if (update_buffer) {
+		err = port_set_buffer(priv, &port_buffer);
+		if (err)
+			return err;
+	}
+
+	if (update_prio2buffer)
+		err = mlx5e_port_set_priority2buffer(priv->mdev, prio2buffer);
+
+	return err;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.h b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.h
new file mode 100644
index 000000000000..34f55b81a0de
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef __MLX5_EN_PORT_BUFFER_H__
+#define __MLX5_EN_PORT_BUFFER_H__
+
+#include "en.h"
+#include "port.h"
+
+#define MLX5E_MAX_BUFFER 8
+#define MLX5E_BUFFER_CELL_SHIFT 7
+#define MLX5E_DEFAULT_CABLE_LEN 7 /* 7 meters */
+
+#define MLX5_BUFFER_SUPPORTED(mdev) (MLX5_CAP_GEN(mdev, pcam_reg) && \
+				     MLX5_CAP_PCAM_REG(mdev, pbmc) && \
+				     MLX5_CAP_PCAM_REG(mdev, pptb))
+
+enum {
+	MLX5E_PORT_BUFFER_CABLE_LEN   = BIT(0),
+	MLX5E_PORT_BUFFER_PFC         = BIT(1),
+	MLX5E_PORT_BUFFER_PRIO2BUFFER = BIT(2),
+	MLX5E_PORT_BUFFER_SIZE        = BIT(3),
+};
+
+struct mlx5e_bufferx_reg {
+	u8   lossy;
+	u8   epsb;
+	u32  size;
+	u32  xoff;
+	u32  xon;
+};
+
+struct mlx5e_port_buffer {
+	u32                       port_buffer_size;
+	u32                       spare_buffer_size;
+	struct mlx5e_bufferx_reg  buffer[MLX5E_MAX_BUFFER];
+};
+
+int mlx5e_port_manual_buffer_config(struct mlx5e_priv *priv,
+				    u32 change, unsigned int mtu,
+				    struct ieee_pfc *pfc,
+				    u32 *buffer_size,
+				    u8 *prio2buffer);
+
+int mlx5e_port_query_buffer(struct mlx5e_priv *priv,
+			    struct mlx5e_port_buffer *port_buffer);
+#endif
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h
new file mode 100644
index 000000000000..f20074dbef32
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef __MLX5E_EN_ACCEL_H__
+#define __MLX5E_EN_ACCEL_H__
+
+#ifdef CONFIG_MLX5_ACCEL
+
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+#include "en_accel/ipsec_rxtx.h"
+#include "en_accel/tls_rxtx.h"
+#include "en.h"
+
+static inline struct sk_buff *mlx5e_accel_handle_tx(struct sk_buff *skb,
+						    struct mlx5e_txqsq *sq,
+						    struct net_device *dev,
+						    struct mlx5e_tx_wqe **wqe,
+						    u16 *pi)
+{
+#ifdef CONFIG_MLX5_EN_TLS
+	if (test_bit(MLX5E_SQ_STATE_TLS, &sq->state)) {
+		skb = mlx5e_tls_handle_tx_skb(dev, sq, skb, wqe, pi);
+		if (unlikely(!skb))
+			return NULL;
+	}
+#endif
+
+#ifdef CONFIG_MLX5_EN_IPSEC
+	if (test_bit(MLX5E_SQ_STATE_IPSEC, &sq->state)) {
+		skb = mlx5e_ipsec_handle_tx_skb(dev, *wqe, skb);
+		if (unlikely(!skb))
+			return NULL;
+	}
+#endif
+
+	return skb;
+}
+
+#endif /* CONFIG_MLX5_ACCEL */
+
+#endif /* __MLX5E_EN_ACCEL_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
index 1198fc1eba4c..93bf10e6508c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
@@ -45,9 +45,6 @@
 #define MLX5E_IPSEC_SADB_RX_BITS 10
 #define MLX5E_IPSEC_ESN_SCOPE_MID 0x80000000L
 
-#define MLX5E_METADATA_ETHER_TYPE (0x8CE4)
-#define MLX5E_METADATA_ETHER_LEN 8
-
 struct mlx5e_priv;
 
 struct mlx5e_ipsec_sw_stats {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c
new file mode 100644
index 000000000000..d167845271c3
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c
@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/netdevice.h>
+#include <net/ipv6.h>
+#include "en_accel/tls.h"
+#include "accel/tls.h"
+
+static void mlx5e_tls_set_ipv4_flow(void *flow, struct sock *sk)
+{
+	struct inet_sock *inet = inet_sk(sk);
+
+	MLX5_SET(tls_flow, flow, ipv6, 0);
+	memcpy(MLX5_ADDR_OF(tls_flow, flow, dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
+	       &inet->inet_daddr, MLX5_FLD_SZ_BYTES(ipv4_layout, ipv4));
+	memcpy(MLX5_ADDR_OF(tls_flow, flow, src_ipv4_src_ipv6.ipv4_layout.ipv4),
+	       &inet->inet_rcv_saddr, MLX5_FLD_SZ_BYTES(ipv4_layout, ipv4));
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+static void mlx5e_tls_set_ipv6_flow(void *flow, struct sock *sk)
+{
+	struct ipv6_pinfo *np = inet6_sk(sk);
+
+	MLX5_SET(tls_flow, flow, ipv6, 1);
+	memcpy(MLX5_ADDR_OF(tls_flow, flow, dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
+	       &sk->sk_v6_daddr, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6));
+	memcpy(MLX5_ADDR_OF(tls_flow, flow, src_ipv4_src_ipv6.ipv6_layout.ipv6),
+	       &np->saddr, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6));
+}
+#endif
+
+static void mlx5e_tls_set_flow_tcp_ports(void *flow, struct sock *sk)
+{
+	struct inet_sock *inet = inet_sk(sk);
+
+	memcpy(MLX5_ADDR_OF(tls_flow, flow, src_port), &inet->inet_sport,
+	       MLX5_FLD_SZ_BYTES(tls_flow, src_port));
+	memcpy(MLX5_ADDR_OF(tls_flow, flow, dst_port), &inet->inet_dport,
+	       MLX5_FLD_SZ_BYTES(tls_flow, dst_port));
+}
+
+static int mlx5e_tls_set_flow(void *flow, struct sock *sk, u32 caps)
+{
+	switch (sk->sk_family) {
+	case AF_INET:
+		mlx5e_tls_set_ipv4_flow(flow, sk);
+		break;
+#if IS_ENABLED(CONFIG_IPV6)
+	case AF_INET6:
+		if (!sk->sk_ipv6only &&
+		    ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
+			mlx5e_tls_set_ipv4_flow(flow, sk);
+			break;
+		}
+		if (!(caps & MLX5_ACCEL_TLS_IPV6))
+			goto error_out;
+
+		mlx5e_tls_set_ipv6_flow(flow, sk);
+		break;
+#endif
+	default:
+		goto error_out;
+	}
+
+	mlx5e_tls_set_flow_tcp_ports(flow, sk);
+	return 0;
+error_out:
+	return -EINVAL;
+}
+
+static int mlx5e_tls_add(struct net_device *netdev, struct sock *sk,
+			 enum tls_offload_ctx_dir direction,
+			 struct tls_crypto_info *crypto_info,
+			 u32 start_offload_tcp_sn)
+{
+	struct mlx5e_priv *priv = netdev_priv(netdev);
+	struct tls_context *tls_ctx = tls_get_ctx(sk);
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 caps = mlx5_accel_tls_device_caps(mdev);
+	int ret = -ENOMEM;
+	void *flow;
+
+	if (direction != TLS_OFFLOAD_CTX_DIR_TX)
+		return -EINVAL;
+
+	flow = kzalloc(MLX5_ST_SZ_BYTES(tls_flow), GFP_KERNEL);
+	if (!flow)
+		return ret;
+
+	ret = mlx5e_tls_set_flow(flow, sk, caps);
+	if (ret)
+		goto free_flow;
+
+	if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
+		struct mlx5e_tls_offload_context *tx_ctx =
+		    mlx5e_get_tls_tx_context(tls_ctx);
+		u32 swid;
+
+		ret = mlx5_accel_tls_add_tx_flow(mdev, flow, crypto_info,
+						 start_offload_tcp_sn, &swid);
+		if (ret < 0)
+			goto free_flow;
+
+		tx_ctx->swid = htonl(swid);
+		tx_ctx->expected_seq = start_offload_tcp_sn;
+	}
+
+	return 0;
+free_flow:
+	kfree(flow);
+	return ret;
+}
+
+static void mlx5e_tls_del(struct net_device *netdev,
+			  struct tls_context *tls_ctx,
+			  enum tls_offload_ctx_dir direction)
+{
+	struct mlx5e_priv *priv = netdev_priv(netdev);
+
+	if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
+		u32 swid = ntohl(mlx5e_get_tls_tx_context(tls_ctx)->swid);
+
+		mlx5_accel_tls_del_tx_flow(priv->mdev, swid);
+	} else {
+		netdev_err(netdev, "unsupported direction %d\n", direction);
+	}
+}
+
+static const struct tlsdev_ops mlx5e_tls_ops = {
+	.tls_dev_add = mlx5e_tls_add,
+	.tls_dev_del = mlx5e_tls_del,
+};
+
+void mlx5e_tls_build_netdev(struct mlx5e_priv *priv)
+{
+	struct net_device *netdev = priv->netdev;
+
+	if (!mlx5_accel_is_tls_device(priv->mdev))
+		return;
+
+	netdev->features |= NETIF_F_HW_TLS_TX;
+	netdev->hw_features |= NETIF_F_HW_TLS_TX;
+	netdev->tlsdev_ops = &mlx5e_tls_ops;
+}
+
+int mlx5e_tls_init(struct mlx5e_priv *priv)
+{
+	struct mlx5e_tls *tls = kzalloc(sizeof(*tls), GFP_KERNEL);
+
+	if (!tls)
+		return -ENOMEM;
+
+	priv->tls = tls;
+	return 0;
+}
+
+void mlx5e_tls_cleanup(struct mlx5e_priv *priv)
+{
+	struct mlx5e_tls *tls = priv->tls;
+
+	if (!tls)
+		return;
+
+	kfree(tls);
+	priv->tls = NULL;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h
new file mode 100644
index 000000000000..b6162178f621
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+#ifndef __MLX5E_TLS_H__
+#define __MLX5E_TLS_H__
+
+#ifdef CONFIG_MLX5_EN_TLS
+
+#include <net/tls.h>
+#include "en.h"
+
+struct mlx5e_tls_sw_stats {
+	atomic64_t tx_tls_drop_metadata;
+	atomic64_t tx_tls_drop_resync_alloc;
+	atomic64_t tx_tls_drop_no_sync_data;
+	atomic64_t tx_tls_drop_bypass_required;
+};
+
+struct mlx5e_tls {
+	struct mlx5e_tls_sw_stats sw_stats;
+};
+
+struct mlx5e_tls_offload_context {
+	struct tls_offload_context base;
+	u32 expected_seq;
+	__be32 swid;
+};
+
+static inline struct mlx5e_tls_offload_context *
+mlx5e_get_tls_tx_context(struct tls_context *tls_ctx)
+{
+	BUILD_BUG_ON(sizeof(struct mlx5e_tls_offload_context) >
+		     TLS_OFFLOAD_CONTEXT_SIZE);
+	return container_of(tls_offload_ctx(tls_ctx),
+			    struct mlx5e_tls_offload_context,
+			    base);
+}
+
+void mlx5e_tls_build_netdev(struct mlx5e_priv *priv);
+int mlx5e_tls_init(struct mlx5e_priv *priv);
+void mlx5e_tls_cleanup(struct mlx5e_priv *priv);
+
+int mlx5e_tls_get_count(struct mlx5e_priv *priv);
+int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data);
+int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data);
+
+#else
+
+static inline void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) { }
+static inline int mlx5e_tls_init(struct mlx5e_priv *priv) { return 0; }
+static inline void mlx5e_tls_cleanup(struct mlx5e_priv *priv) { }
+static inline int mlx5e_tls_get_count(struct mlx5e_priv *priv) { return 0; }
+static inline int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data) { return 0; }
+static inline int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data) { return 0; }
+
+#endif
+
+#endif /* __MLX5E_TLS_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c
new file mode 100644
index 000000000000..15aef71d1957
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c
@@ -0,0 +1,278 @@
+/*
+ * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include "en_accel/tls.h"
+#include "en_accel/tls_rxtx.h"
+
+#define SYNDROME_OFFLOAD_REQUIRED 32
+#define SYNDROME_SYNC 33
+
+struct sync_info {
+	u64 rcd_sn;
+	s32 sync_len;
+	int nr_frags;
+	skb_frag_t frags[MAX_SKB_FRAGS];
+};
+
+struct mlx5e_tls_metadata {
+	/* One byte of syndrome followed by 3 bytes of swid */
+	__be32 syndrome_swid;
+	__be16 first_seq;
+	/* packet type ID field	*/
+	__be16 ethertype;
+} __packed;
+
+static int mlx5e_tls_add_metadata(struct sk_buff *skb, __be32 swid)
+{
+	struct mlx5e_tls_metadata *pet;
+	struct ethhdr *eth;
+
+	if (skb_cow_head(skb, sizeof(struct mlx5e_tls_metadata)))
+		return -ENOMEM;
+
+	eth = (struct ethhdr *)skb_push(skb, sizeof(struct mlx5e_tls_metadata));
+	skb->mac_header -= sizeof(struct mlx5e_tls_metadata);
+	pet = (struct mlx5e_tls_metadata *)(eth + 1);
+
+	memmove(skb->data, skb->data + sizeof(struct mlx5e_tls_metadata),
+		2 * ETH_ALEN);
+
+	eth->h_proto = cpu_to_be16(MLX5E_METADATA_ETHER_TYPE);
+	pet->syndrome_swid = htonl(SYNDROME_OFFLOAD_REQUIRED << 24) | swid;
+
+	return 0;
+}
+
+static int mlx5e_tls_get_sync_data(struct mlx5e_tls_offload_context *context,
+				   u32 tcp_seq, struct sync_info *info)
+{
+	int remaining, i = 0, ret = -EINVAL;
+	struct tls_record_info *record;
+	unsigned long flags;
+	s32 sync_size;
+
+	spin_lock_irqsave(&context->base.lock, flags);
+	record = tls_get_record(&context->base, tcp_seq, &info->rcd_sn);
+
+	if (unlikely(!record))
+		goto out;
+
+	sync_size = tcp_seq - tls_record_start_seq(record);
+	info->sync_len = sync_size;
+	if (unlikely(sync_size < 0)) {
+		if (tls_record_is_start_marker(record))
+			goto done;
+
+		goto out;
+	}
+
+	remaining = sync_size;
+	while (remaining > 0) {
+		info->frags[i] = record->frags[i];
+		__skb_frag_ref(&info->frags[i]);
+		remaining -= skb_frag_size(&info->frags[i]);
+
+		if (remaining < 0)
+			skb_frag_size_add(&info->frags[i], remaining);
+
+		i++;
+	}
+	info->nr_frags = i;
+done:
+	ret = 0;
+out:
+	spin_unlock_irqrestore(&context->base.lock, flags);
+	return ret;
+}
+
+static void mlx5e_tls_complete_sync_skb(struct sk_buff *skb,
+					struct sk_buff *nskb, u32 tcp_seq,
+					int headln, __be64 rcd_sn)
+{
+	struct mlx5e_tls_metadata *pet;
+	u8 syndrome = SYNDROME_SYNC;
+	struct iphdr *iph;
+	struct tcphdr *th;
+	int data_len, mss;
+
+	nskb->dev = skb->dev;
+	skb_reset_mac_header(nskb);
+	skb_set_network_header(nskb, skb_network_offset(skb));
+	skb_set_transport_header(nskb, skb_transport_offset(skb));
+	memcpy(nskb->data, skb->data, headln);
+	memcpy(nskb->data + headln, &rcd_sn, sizeof(rcd_sn));
+
+	iph = ip_hdr(nskb);
+	iph->tot_len = htons(nskb->len - skb_network_offset(nskb));
+	th = tcp_hdr(nskb);
+	data_len = nskb->len - headln;
+	tcp_seq -= data_len;
+	th->seq = htonl(tcp_seq);
+
+	mss = nskb->dev->mtu - (headln - skb_network_offset(nskb));
+	skb_shinfo(nskb)->gso_size = 0;
+	if (data_len > mss) {
+		skb_shinfo(nskb)->gso_size = mss;
+		skb_shinfo(nskb)->gso_segs = DIV_ROUND_UP(data_len, mss);
+	}
+	skb_shinfo(nskb)->gso_type = skb_shinfo(skb)->gso_type;
+
+	pet = (struct mlx5e_tls_metadata *)(nskb->data + sizeof(struct ethhdr));
+	memcpy(pet, &syndrome, sizeof(syndrome));
+	pet->first_seq = htons(tcp_seq);
+
+	/* MLX5 devices don't care about the checksum partial start, offset
+	 * and pseudo header
+	 */
+	nskb->ip_summed = CHECKSUM_PARTIAL;
+
+	nskb->xmit_more = 1;
+	nskb->queue_mapping = skb->queue_mapping;
+}
+
+static struct sk_buff *
+mlx5e_tls_handle_ooo(struct mlx5e_tls_offload_context *context,
+		     struct mlx5e_txqsq *sq, struct sk_buff *skb,
+		     struct mlx5e_tx_wqe **wqe,
+		     u16 *pi,
+		     struct mlx5e_tls *tls)
+{
+	u32 tcp_seq = ntohl(tcp_hdr(skb)->seq);
+	struct sync_info info;
+	struct sk_buff *nskb;
+	int linear_len = 0;
+	int headln;
+	int i;
+
+	sq->stats->tls_ooo++;
+
+	if (mlx5e_tls_get_sync_data(context, tcp_seq, &info)) {
+		/* We might get here if a retransmission reaches the driver
+		 * after the relevant record is acked.
+		 * It should be safe to drop the packet in this case
+		 */
+		atomic64_inc(&tls->sw_stats.tx_tls_drop_no_sync_data);
+		goto err_out;
+	}
+
+	if (unlikely(info.sync_len < 0)) {
+		u32 payload;
+
+		headln = skb_transport_offset(skb) + tcp_hdrlen(skb);
+		payload = skb->len - headln;
+		if (likely(payload <= -info.sync_len))
+			/* SKB payload doesn't require offload
+			 */
+			return skb;
+
+		atomic64_inc(&tls->sw_stats.tx_tls_drop_bypass_required);
+		goto err_out;
+	}
+
+	if (unlikely(mlx5e_tls_add_metadata(skb, context->swid))) {
+		atomic64_inc(&tls->sw_stats.tx_tls_drop_metadata);
+		goto err_out;
+	}
+
+	headln = skb_transport_offset(skb) + tcp_hdrlen(skb);
+	linear_len += headln + sizeof(info.rcd_sn);
+	nskb = alloc_skb(linear_len, GFP_ATOMIC);
+	if (unlikely(!nskb)) {
+		atomic64_inc(&tls->sw_stats.tx_tls_drop_resync_alloc);
+		goto err_out;
+	}
+
+	context->expected_seq = tcp_seq + skb->len - headln;
+	skb_put(nskb, linear_len);
+	for (i = 0; i < info.nr_frags; i++)
+		skb_shinfo(nskb)->frags[i] = info.frags[i];
+
+	skb_shinfo(nskb)->nr_frags = info.nr_frags;
+	nskb->data_len = info.sync_len;
+	nskb->len += info.sync_len;
+	sq->stats->tls_resync_bytes += nskb->len;
+	mlx5e_tls_complete_sync_skb(skb, nskb, tcp_seq, headln,
+				    cpu_to_be64(info.rcd_sn));
+	mlx5e_sq_xmit(sq, nskb, *wqe, *pi);
+	mlx5e_sq_fetch_wqe(sq, wqe, pi);
+	return skb;
+
+err_out:
+	dev_kfree_skb_any(skb);
+	return NULL;
+}
+
+struct sk_buff *mlx5e_tls_handle_tx_skb(struct net_device *netdev,
+					struct mlx5e_txqsq *sq,
+					struct sk_buff *skb,
+					struct mlx5e_tx_wqe **wqe,
+					u16 *pi)
+{
+	struct mlx5e_priv *priv = netdev_priv(netdev);
+	struct mlx5e_tls_offload_context *context;
+	struct tls_context *tls_ctx;
+	u32 expected_seq;
+	int datalen;
+	u32 skb_seq;
+
+	if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk))
+		goto out;
+
+	datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb));
+	if (!datalen)
+		goto out;
+
+	tls_ctx = tls_get_ctx(skb->sk);
+	if (unlikely(tls_ctx->netdev != netdev))
+		goto out;
+
+	skb_seq = ntohl(tcp_hdr(skb)->seq);
+	context = mlx5e_get_tls_tx_context(tls_ctx);
+	expected_seq = context->expected_seq;
+
+	if (unlikely(expected_seq != skb_seq)) {
+		skb = mlx5e_tls_handle_ooo(context, sq, skb, wqe, pi, priv->tls);
+		goto out;
+	}
+
+	if (unlikely(mlx5e_tls_add_metadata(skb, context->swid))) {
+		atomic64_inc(&priv->tls->sw_stats.tx_tls_drop_metadata);
+		dev_kfree_skb_any(skb);
+		skb = NULL;
+		goto out;
+	}
+
+	context->expected_seq = skb_seq + datalen;
+out:
+	return skb;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h
new file mode 100644
index 000000000000..405dfd302225
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef __MLX5E_TLS_RXTX_H__
+#define __MLX5E_TLS_RXTX_H__
+
+#ifdef CONFIG_MLX5_EN_TLS
+
+#include <linux/skbuff.h>
+#include "en.h"
+
+struct sk_buff *mlx5e_tls_handle_tx_skb(struct net_device *netdev,
+					struct mlx5e_txqsq *sq,
+					struct sk_buff *skb,
+					struct mlx5e_tx_wqe **wqe,
+					u16 *pi);
+
+#endif /* CONFIG_MLX5_EN_TLS */
+
+#endif /* __MLX5E_TLS_RXTX_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_stats.c
new file mode 100644
index 000000000000..01468ec27446
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_stats.c
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/ethtool.h>
+#include <net/sock.h>
+
+#include "en.h"
+#include "accel/tls.h"
+#include "fpga/sdk.h"
+#include "en_accel/tls.h"
+
+static const struct counter_desc mlx5e_tls_sw_stats_desc[] = {
+	{ MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_drop_metadata) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_drop_resync_alloc) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_drop_no_sync_data) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_drop_bypass_required) },
+};
+
+#define MLX5E_READ_CTR_ATOMIC64(ptr, dsc, i) \
+	atomic64_read((atomic64_t *)((char *)(ptr) + (dsc)[i].offset))
+
+#define NUM_TLS_SW_COUNTERS ARRAY_SIZE(mlx5e_tls_sw_stats_desc)
+
+int mlx5e_tls_get_count(struct mlx5e_priv *priv)
+{
+	if (!priv->tls)
+		return 0;
+
+	return NUM_TLS_SW_COUNTERS;
+}
+
+int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data)
+{
+	unsigned int i, idx = 0;
+
+	if (!priv->tls)
+		return 0;
+
+	for (i = 0; i < NUM_TLS_SW_COUNTERS; i++)
+		strcpy(data + (idx++) * ETH_GSTRING_LEN,
+		       mlx5e_tls_sw_stats_desc[i].format);
+
+	return NUM_TLS_SW_COUNTERS;
+}
+
+int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data)
+{
+	int i, idx = 0;
+
+	if (!priv->tls)
+		return 0;
+
+	for (i = 0; i < NUM_TLS_SW_COUNTERS; i++)
+		data[idx++] =
+		    MLX5E_READ_CTR_ATOMIC64(&priv->tls->sw_stats,
+					    mlx5e_tls_sw_stats_desc, i);
+
+	return NUM_TLS_SW_COUNTERS;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c
index 610d485c4b03..f64b5e78519b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c
@@ -565,7 +565,7 @@ static void arfs_modify_rule_rq(struct mlx5e_priv *priv,
 	err =  mlx5_modify_rule_destination(rule, &dst, NULL);
 	if (err)
 		netdev_warn(priv->netdev,
-			    "Failed to modfiy aRFS rule destination to rq=%d\n", rxq);
+			    "Failed to modify aRFS rule destination to rq=%d\n", rxq);
 }
 
 static void arfs_handle_work(struct work_struct *work)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
index c641d5656b2d..0a52f31fef37 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
@@ -32,8 +32,8 @@
 #include <linux/device.h>
 #include <linux/netdevice.h>
 #include "en.h"
-
-#define MLX5E_MAX_PRIORITY 8
+#include "en/port.h"
+#include "en/port_buffer.h"
 
 #define MLX5E_100MB (100000)
 #define MLX5E_1GB   (1000000)
@@ -41,6 +41,9 @@
 #define MLX5E_CEE_STATE_UP    1
 #define MLX5E_CEE_STATE_DOWN  0
 
+/* Max supported cable length is 1000 meters */
+#define MLX5E_MAX_CABLE_LENGTH 1000
+
 enum {
 	MLX5E_VENDOR_TC_GROUP_NUM = 7,
 	MLX5E_LOWEST_PRIO_GROUP   = 0,
@@ -338,6 +341,9 @@ static int mlx5e_dcbnl_ieee_getpfc(struct net_device *dev,
 		pfc->indications[i] = PPORT_PER_PRIO_GET(pstats, i, rx_pause);
 	}
 
+	if (MLX5_BUFFER_SUPPORTED(mdev))
+		pfc->delay = priv->dcbx.cable_len;
+
 	return mlx5_query_port_pfc(mdev, &pfc->pfc_en, NULL);
 }
 
@@ -346,16 +352,39 @@ static int mlx5e_dcbnl_ieee_setpfc(struct net_device *dev,
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 old_cable_len = priv->dcbx.cable_len;
+	struct ieee_pfc pfc_new;
+	u32 changed = 0;
 	u8 curr_pfc_en;
-	int ret;
+	int ret = 0;
 
+	/* pfc_en */
 	mlx5_query_port_pfc(mdev, &curr_pfc_en, NULL);
+	if (pfc->pfc_en != curr_pfc_en) {
+		ret = mlx5_set_port_pfc(mdev, pfc->pfc_en, pfc->pfc_en);
+		if (ret)
+			return ret;
+		mlx5_toggle_port_link(mdev);
+		changed |= MLX5E_PORT_BUFFER_PFC;
+	}
 
-	if (pfc->pfc_en == curr_pfc_en)
-		return 0;
+	if (pfc->delay &&
+	    pfc->delay < MLX5E_MAX_CABLE_LENGTH &&
+	    pfc->delay != priv->dcbx.cable_len) {
+		priv->dcbx.cable_len = pfc->delay;
+		changed |= MLX5E_PORT_BUFFER_CABLE_LEN;
+	}
 
-	ret = mlx5_set_port_pfc(mdev, pfc->pfc_en, pfc->pfc_en);
-	mlx5_toggle_port_link(mdev);
+	if (MLX5_BUFFER_SUPPORTED(mdev)) {
+		pfc_new.pfc_en = (changed & MLX5E_PORT_BUFFER_PFC) ? pfc->pfc_en : curr_pfc_en;
+		if (priv->dcbx.manual_buffer)
+			ret = mlx5e_port_manual_buffer_config(priv, changed,
+							      dev->mtu, &pfc_new,
+							      NULL, NULL);
+
+		if (ret && (changed & MLX5E_PORT_BUFFER_CABLE_LEN))
+			priv->dcbx.cable_len = old_cable_len;
+	}
 
 	if (!ret) {
 		mlx5e_dbg(HW, priv,
@@ -873,6 +902,90 @@ static void mlx5e_dcbnl_setpfcstate(struct net_device *netdev, u8 state)
 	cee_cfg->pfc_enable = state;
 }
 
+static int mlx5e_dcbnl_getbuffer(struct net_device *dev,
+				 struct dcbnl_buffer *dcb_buffer)
+{
+	struct mlx5e_priv *priv = netdev_priv(dev);
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5e_port_buffer port_buffer;
+	u8 buffer[MLX5E_MAX_PRIORITY];
+	int i, err;
+
+	if (!MLX5_BUFFER_SUPPORTED(mdev))
+		return -EOPNOTSUPP;
+
+	err = mlx5e_port_query_priority2buffer(mdev, buffer);
+	if (err)
+		return err;
+
+	for (i = 0; i < MLX5E_MAX_PRIORITY; i++)
+		dcb_buffer->prio2buffer[i] = buffer[i];
+
+	err = mlx5e_port_query_buffer(priv, &port_buffer);
+	if (err)
+		return err;
+
+	for (i = 0; i < MLX5E_MAX_BUFFER; i++)
+		dcb_buffer->buffer_size[i] = port_buffer.buffer[i].size;
+	dcb_buffer->total_size = port_buffer.port_buffer_size;
+
+	return 0;
+}
+
+static int mlx5e_dcbnl_setbuffer(struct net_device *dev,
+				 struct dcbnl_buffer *dcb_buffer)
+{
+	struct mlx5e_priv *priv = netdev_priv(dev);
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5e_port_buffer port_buffer;
+	u8 old_prio2buffer[MLX5E_MAX_PRIORITY];
+	u32 *buffer_size = NULL;
+	u8 *prio2buffer = NULL;
+	u32 changed = 0;
+	int i, err;
+
+	if (!MLX5_BUFFER_SUPPORTED(mdev))
+		return -EOPNOTSUPP;
+
+	for (i = 0; i < DCBX_MAX_BUFFERS; i++)
+		mlx5_core_dbg(mdev, "buffer[%d]=%d\n", i, dcb_buffer->buffer_size[i]);
+
+	for (i = 0; i < MLX5E_MAX_PRIORITY; i++)
+		mlx5_core_dbg(mdev, "priority %d buffer%d\n", i, dcb_buffer->prio2buffer[i]);
+
+	err = mlx5e_port_query_priority2buffer(mdev, old_prio2buffer);
+	if (err)
+		return err;
+
+	for (i = 0; i < MLX5E_MAX_PRIORITY; i++) {
+		if (dcb_buffer->prio2buffer[i] != old_prio2buffer[i]) {
+			changed |= MLX5E_PORT_BUFFER_PRIO2BUFFER;
+			prio2buffer = dcb_buffer->prio2buffer;
+			break;
+		}
+	}
+
+	err = mlx5e_port_query_buffer(priv, &port_buffer);
+	if (err)
+		return err;
+
+	for (i = 0; i < MLX5E_MAX_BUFFER; i++) {
+		if (port_buffer.buffer[i].size != dcb_buffer->buffer_size[i]) {
+			changed |= MLX5E_PORT_BUFFER_SIZE;
+			buffer_size = dcb_buffer->buffer_size;
+			break;
+		}
+	}
+
+	if (!changed)
+		return 0;
+
+	priv->dcbx.manual_buffer = true;
+	err = mlx5e_port_manual_buffer_config(priv, changed, dev->mtu, NULL,
+					      buffer_size, prio2buffer);
+	return err;
+}
+
 const struct dcbnl_rtnl_ops mlx5e_dcbnl_ops = {
 	.ieee_getets	= mlx5e_dcbnl_ieee_getets,
 	.ieee_setets	= mlx5e_dcbnl_ieee_setets,
@@ -884,6 +997,8 @@ const struct dcbnl_rtnl_ops mlx5e_dcbnl_ops = {
 	.ieee_delapp    = mlx5e_dcbnl_ieee_delapp,
 	.getdcbx	= mlx5e_dcbnl_getdcbx,
 	.setdcbx	= mlx5e_dcbnl_setdcbx,
+	.dcbnl_getbuffer = mlx5e_dcbnl_getbuffer,
+	.dcbnl_setbuffer = mlx5e_dcbnl_setbuffer,
 
 /* CEE interfaces */
 	.setall         = mlx5e_dcbnl_setall,
@@ -1091,5 +1206,8 @@ void mlx5e_dcbnl_initialize(struct mlx5e_priv *priv)
 	if (priv->dcbx.mode == MLX5E_DCBX_PARAM_VER_OPER_HOST)
 		priv->dcbx.cap |= DCB_CAP_DCBX_HOST;
 
+	priv->dcbx.manual_buffer = false;
+	priv->dcbx.cable_len = MLX5E_DEFAULT_CABLE_LEN;
+
 	mlx5e_ets_init(priv);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c
index 602851ab5b14..d67adf70a97b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c
@@ -33,16 +33,30 @@
 #include <linux/net_dim.h>
 #include "en.h"
 
+static void
+mlx5e_complete_dim_work(struct net_dim *dim, struct net_dim_cq_moder moder,
+			struct mlx5_core_dev *mdev, struct mlx5_core_cq *mcq)
+{
+	mlx5_core_modify_cq_moderation(mdev, mcq, moder.usec, moder.pkts);
+	dim->state = NET_DIM_START_MEASURE;
+}
+
 void mlx5e_rx_dim_work(struct work_struct *work)
 {
-	struct net_dim *dim = container_of(work, struct net_dim,
-					   work);
+	struct net_dim *dim = container_of(work, struct net_dim, work);
 	struct mlx5e_rq *rq = container_of(dim, struct mlx5e_rq, dim);
-	struct net_dim_cq_moder cur_profile = net_dim_get_profile(dim->mode,
-								  dim->profile_ix);
+	struct net_dim_cq_moder cur_moder =
+		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
 
-	mlx5_core_modify_cq_moderation(rq->mdev, &rq->cq.mcq,
-				       cur_profile.usec, cur_profile.pkts);
+	mlx5e_complete_dim_work(dim, cur_moder, rq->mdev, &rq->cq.mcq);
+}
 
-	dim->state = NET_DIM_START_MEASURE;
+void mlx5e_tx_dim_work(struct work_struct *work)
+{
+	struct net_dim *dim = container_of(work, struct net_dim, work);
+	struct mlx5e_txqsq *sq = container_of(dim, struct mlx5e_txqsq, dim);
+	struct net_dim_cq_moder cur_moder =
+		net_dim_get_tx_moderation(dim->mode, dim->profile_ix);
+
+	mlx5e_complete_dim_work(dim, cur_moder, sq->cq.mdev, &sq->cq.mcq);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index 37fd0245b6c1..42bd256e680d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -31,6 +31,7 @@
  */
 
 #include "en.h"
+#include "en/port.h"
 
 void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
 			       struct ethtool_drvinfo *drvinfo)
@@ -59,18 +60,16 @@ static void mlx5e_get_drvinfo(struct net_device *dev,
 struct ptys2ethtool_config {
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(supported);
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(advertised);
-	u32 speed;
 };
 
 static struct ptys2ethtool_config ptys2ethtool_table[MLX5E_LINK_MODES_NUMBER];
 
-#define MLX5_BUILD_PTYS2ETHTOOL_CONFIG(reg_, speed_, ...)               \
+#define MLX5_BUILD_PTYS2ETHTOOL_CONFIG(reg_, ...)                       \
 	({                                                              \
 		struct ptys2ethtool_config *cfg;                        \
 		const unsigned int modes[] = { __VA_ARGS__ };           \
 		unsigned int i;                                         \
 		cfg = &ptys2ethtool_table[reg_];                        \
-		cfg->speed = speed_;                                    \
 		bitmap_zero(cfg->supported,                             \
 			    __ETHTOOL_LINK_MODE_MASK_NBITS);            \
 		bitmap_zero(cfg->advertised,                            \
@@ -83,55 +82,55 @@ static struct ptys2ethtool_config ptys2ethtool_table[MLX5E_LINK_MODES_NUMBER];
 
 void mlx5e_build_ptys2ethtool_map(void)
 {
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_1000BASE_CX_SGMII, SPEED_1000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_1000BASE_CX_SGMII,
 				       ETHTOOL_LINK_MODE_1000baseKX_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_1000BASE_KX, SPEED_1000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_1000BASE_KX,
 				       ETHTOOL_LINK_MODE_1000baseKX_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_CX4, SPEED_10000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_CX4,
 				       ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_KX4, SPEED_10000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_KX4,
 				       ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_KR, SPEED_10000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_KR,
 				       ETHTOOL_LINK_MODE_10000baseKR_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_20GBASE_KR2, SPEED_20000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_20GBASE_KR2,
 				       ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_CR4, SPEED_40000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_CR4,
 				       ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_KR4, SPEED_40000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_KR4,
 				       ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_56GBASE_R4, SPEED_56000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_56GBASE_R4,
 				       ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_CR, SPEED_10000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_CR,
 				       ETHTOOL_LINK_MODE_10000baseKR_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_SR, SPEED_10000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_SR,
 				       ETHTOOL_LINK_MODE_10000baseKR_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_ER, SPEED_10000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_ER,
 				       ETHTOOL_LINK_MODE_10000baseKR_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_SR4, SPEED_40000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_SR4,
 				       ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_LR4, SPEED_40000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_40GBASE_LR4,
 				       ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_50GBASE_SR2, SPEED_50000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_50GBASE_SR2,
 				       ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_CR4, SPEED_100000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_CR4,
 				       ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_SR4, SPEED_100000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_SR4,
 				       ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_KR4, SPEED_100000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_KR4,
 				       ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_LR4, SPEED_100000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GBASE_LR4,
 				       ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_T, SPEED_10000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_10GBASE_T,
 				       ETHTOOL_LINK_MODE_10000baseT_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_25GBASE_CR, SPEED_25000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_25GBASE_CR,
 				       ETHTOOL_LINK_MODE_25000baseCR_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_25GBASE_KR, SPEED_25000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_25GBASE_KR,
 				       ETHTOOL_LINK_MODE_25000baseKR_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_25GBASE_SR, SPEED_25000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_25GBASE_SR,
 				       ETHTOOL_LINK_MODE_25000baseSR_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_50GBASE_CR2, SPEED_50000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_50GBASE_CR2,
 				       ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT);
-	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_50GBASE_KR2, SPEED_50000,
+	MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_50GBASE_KR2,
 				       ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT);
 }
 
@@ -389,14 +388,20 @@ static int mlx5e_set_channels(struct net_device *dev,
 int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv,
 			       struct ethtool_coalesce *coal)
 {
+	struct net_dim_cq_moder *rx_moder, *tx_moder;
+
 	if (!MLX5_CAP_GEN(priv->mdev, cq_moderation))
 		return -EOPNOTSUPP;
 
-	coal->rx_coalesce_usecs       = priv->channels.params.rx_cq_moderation.usec;
-	coal->rx_max_coalesced_frames = priv->channels.params.rx_cq_moderation.pkts;
-	coal->tx_coalesce_usecs       = priv->channels.params.tx_cq_moderation.usec;
-	coal->tx_max_coalesced_frames = priv->channels.params.tx_cq_moderation.pkts;
-	coal->use_adaptive_rx_coalesce = priv->channels.params.rx_dim_enabled;
+	rx_moder = &priv->channels.params.rx_cq_moderation;
+	coal->rx_coalesce_usecs		= rx_moder->usec;
+	coal->rx_max_coalesced_frames	= rx_moder->pkts;
+	coal->use_adaptive_rx_coalesce	= priv->channels.params.rx_dim_enabled;
+
+	tx_moder = &priv->channels.params.tx_cq_moderation;
+	coal->tx_coalesce_usecs		= tx_moder->usec;
+	coal->tx_max_coalesced_frames	= tx_moder->pkts;
+	coal->use_adaptive_tx_coalesce	= priv->channels.params.tx_dim_enabled;
 
 	return 0;
 }
@@ -438,6 +443,7 @@ mlx5e_set_priv_channels_coalesce(struct mlx5e_priv *priv, struct ethtool_coalesc
 int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv,
 			       struct ethtool_coalesce *coal)
 {
+	struct net_dim_cq_moder *rx_moder, *tx_moder;
 	struct mlx5_core_dev *mdev = priv->mdev;
 	struct mlx5e_channels new_channels = {};
 	int err = 0;
@@ -463,11 +469,15 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv,
 	mutex_lock(&priv->state_lock);
 	new_channels.params = priv->channels.params;
 
-	new_channels.params.tx_cq_moderation.usec = coal->tx_coalesce_usecs;
-	new_channels.params.tx_cq_moderation.pkts = coal->tx_max_coalesced_frames;
-	new_channels.params.rx_cq_moderation.usec = coal->rx_coalesce_usecs;
-	new_channels.params.rx_cq_moderation.pkts = coal->rx_max_coalesced_frames;
-	new_channels.params.rx_dim_enabled        = !!coal->use_adaptive_rx_coalesce;
+	rx_moder          = &new_channels.params.rx_cq_moderation;
+	rx_moder->usec    = coal->rx_coalesce_usecs;
+	rx_moder->pkts    = coal->rx_max_coalesced_frames;
+	new_channels.params.rx_dim_enabled = !!coal->use_adaptive_rx_coalesce;
+
+	tx_moder          = &new_channels.params.tx_cq_moderation;
+	tx_moder->usec    = coal->tx_coalesce_usecs;
+	tx_moder->pkts    = coal->tx_max_coalesced_frames;
+	new_channels.params.tx_dim_enabled = !!coal->use_adaptive_tx_coalesce;
 
 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
 		priv->channels.params = new_channels.params;
@@ -475,7 +485,9 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv,
 	}
 	/* we are opened */
 
-	reset = !!coal->use_adaptive_rx_coalesce != priv->channels.params.rx_dim_enabled;
+	reset = (!!coal->use_adaptive_rx_coalesce != priv->channels.params.rx_dim_enabled) ||
+		(!!coal->use_adaptive_tx_coalesce != priv->channels.params.tx_dim_enabled);
+
 	if (!reset) {
 		mlx5e_set_priv_channels_coalesce(priv, coal);
 		priv->channels.params = new_channels.params;
@@ -604,43 +616,24 @@ static void ptys2ethtool_supported_advertised_port(struct ethtool_link_ksettings
 	}
 }
 
-int mlx5e_get_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed)
-{
-	u32 max_speed = 0;
-	u32 proto_cap;
-	int err;
-	int i;
-
-	err = mlx5_query_port_proto_cap(mdev, &proto_cap, MLX5_PTYS_EN);
-	if (err)
-		return err;
-
-	for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i)
-		if (proto_cap & MLX5E_PROT_MASK(i))
-			max_speed = max(max_speed, ptys2ethtool_table[i].speed);
-
-	*speed = max_speed;
-	return 0;
-}
-
 static void get_speed_duplex(struct net_device *netdev,
 			     u32 eth_proto_oper,
 			     struct ethtool_link_ksettings *link_ksettings)
 {
-	int i;
 	u32 speed = SPEED_UNKNOWN;
 	u8 duplex = DUPLEX_UNKNOWN;
 
 	if (!netif_carrier_ok(netdev))
 		goto out;
 
-	for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
-		if (eth_proto_oper & MLX5E_PROT_MASK(i)) {
-			speed = ptys2ethtool_table[i].speed;
-			duplex = DUPLEX_FULL;
-			break;
-		}
+	speed = mlx5e_port_ptys2speed(eth_proto_oper);
+	if (!speed) {
+		speed = SPEED_UNKNOWN;
+		goto out;
 	}
+
+	duplex = DUPLEX_FULL;
+
 out:
 	link_ksettings->base.speed = speed;
 	link_ksettings->base.duplex = duplex;
@@ -798,18 +791,6 @@ static u32 mlx5e_ethtool2ptys_adver_link(const unsigned long *link_modes)
 	return ptys_modes;
 }
 
-static u32 mlx5e_ethtool2ptys_speed_link(u32 speed)
-{
-	u32 i, speed_links = 0;
-
-	for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
-		if (ptys2ethtool_table[i].speed == speed)
-			speed_links |= MLX5E_PROT_MASK(i);
-	}
-
-	return speed_links;
-}
-
 static int mlx5e_set_link_ksettings(struct net_device *netdev,
 				    const struct ethtool_link_ksettings *link_ksettings)
 {
@@ -829,7 +810,7 @@ static int mlx5e_set_link_ksettings(struct net_device *netdev,
 
 	link_modes = link_ksettings->base.autoneg == AUTONEG_ENABLE ?
 		mlx5e_ethtool2ptys_adver_link(link_ksettings->link_modes.advertising) :
-		mlx5e_ethtool2ptys_speed_link(speed);
+		mlx5e_port_speed2linkmodes(speed);
 
 	err = mlx5_query_port_proto_cap(mdev, &eth_proto_cap, MLX5_PTYS_EN);
 	if (err) {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
index f64dda2bed31..76cc10e44080 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
@@ -277,7 +277,6 @@ static void mlx5e_del_vlan_rule(struct mlx5e_priv *priv,
 		}
 		break;
 	case MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID:
-		mlx5e_vport_context_update_vlans(priv);
 		if (priv->fs.vlan.active_cvlans_rule[vid]) {
 			mlx5_del_flow_rules(priv->fs.vlan.active_cvlans_rule[vid]);
 			priv->fs.vlan.active_cvlans_rule[vid] = NULL;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index b29c1d93f058..9b19863b059d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -35,14 +35,18 @@
 #include <linux/mlx5/fs.h>
 #include <net/vxlan.h>
 #include <linux/bpf.h>
+#include <net/page_pool.h>
 #include "eswitch.h"
 #include "en.h"
 #include "en_tc.h"
 #include "en_rep.h"
 #include "en_accel/ipsec.h"
 #include "en_accel/ipsec_rxtx.h"
+#include "en_accel/tls.h"
 #include "accel/ipsec.h"
+#include "accel/tls.h"
 #include "vxlan.h"
+#include "en/port.h"
 
 struct mlx5e_rq_param {
 	u32			rqc[MLX5_ST_SZ_DW(rqc)];
@@ -389,10 +393,11 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 			  struct mlx5e_rq_param *rqp,
 			  struct mlx5e_rq *rq)
 {
+	struct page_pool_params pp_params = { 0 };
 	struct mlx5_core_dev *mdev = c->mdev;
 	void *rqc = rqp->rqc;
 	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
-	u32 byte_count;
+	u32 byte_count, pool_size;
 	int npages;
 	int wq_sz;
 	int err;
@@ -418,6 +423,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 	rq->ix      = c->ix;
 	rq->mdev    = mdev;
 	rq->hw_mtu  = MLX5E_SW2HW_MTU(params, params->sw_mtu);
+	rq->stats   = &c->priv->channel_stats[c->ix].rq;
 
 	rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;
 	if (IS_ERR(rq->xdp_prog)) {
@@ -432,9 +438,12 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 
 	rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
 	rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params);
+	pool_size = 1 << params->log_rq_mtu_frames;
 
 	switch (rq->wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
+
+		pool_size = MLX5_MPWRQ_PAGES_PER_WQE << mlx5e_mpwqe_get_log_rq_size(params);
 		rq->post_wqes = mlx5e_post_rx_mpwqes;
 		rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
 
@@ -512,6 +521,32 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 		rq->mkey_be = c->mkey_be;
 	}
 
+	/* Create a page_pool and register it with rxq */
+	pp_params.order     = rq->buff.page_order;
+	pp_params.flags     = 0; /* No-internal DMA mapping in page_pool */
+	pp_params.pool_size = pool_size;
+	pp_params.nid       = cpu_to_node(c->cpu);
+	pp_params.dev       = c->pdev;
+	pp_params.dma_dir   = rq->buff.map_dir;
+
+	/* page_pool can be used even when there is no rq->xdp_prog,
+	 * given page_pool does not handle DMA mapping there is no
+	 * required state to clear. And page_pool gracefully handle
+	 * elevated refcnt.
+	 */
+	rq->page_pool = page_pool_create(&pp_params);
+	if (IS_ERR(rq->page_pool)) {
+		if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
+			kfree(rq->wqe.frag_info);
+		err = PTR_ERR(rq->page_pool);
+		rq->page_pool = NULL;
+		goto err_rq_wq_destroy;
+	}
+	err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
+					 MEM_TYPE_PAGE_POOL, rq->page_pool);
+	if (err)
+		goto err_rq_wq_destroy;
+
 	for (i = 0; i < wq_sz; i++) {
 		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
 
@@ -548,6 +583,8 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 	if (rq->xdp_prog)
 		bpf_prog_put(rq->xdp_prog);
 	xdp_rxq_info_unreg(&rq->xdp_rxq);
+	if (rq->page_pool)
+		page_pool_destroy(rq->page_pool);
 	mlx5_wq_destroy(&rq->wq_ctrl);
 
 	return err;
@@ -561,6 +598,8 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
 		bpf_prog_put(rq->xdp_prog);
 
 	xdp_rxq_info_unreg(&rq->xdp_rxq);
+	if (rq->page_pool)
+		page_pool_destroy(rq->page_pool);
 
 	switch (rq->wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
@@ -608,8 +647,8 @@ static int mlx5e_create_rq(struct mlx5e_rq *rq,
 						MLX5_ADAPTER_PAGE_SHIFT);
 	MLX5_SET64(wq, wq,  dbr_addr,		rq->wq_ctrl.db.dma);
 
-	mlx5_fill_page_array(&rq->wq_ctrl.buf,
-			     (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
+	mlx5_fill_page_frag_array(&rq->wq_ctrl.buf,
+				  (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 
 	err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
 
@@ -710,23 +749,24 @@ static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
 	mlx5_core_destroy_rq(rq->mdev, rq->rqn);
 }
 
-static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
+static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
 {
-	unsigned long exp_time = jiffies + msecs_to_jiffies(20000);
+	unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time);
 	struct mlx5e_channel *c = rq->channel;
 
 	struct mlx5_wq_ll *wq = &rq->wq;
 	u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5_wq_ll_get_size(wq));
 
-	while (time_before(jiffies, exp_time)) {
+	do {
 		if (wq->cur_sz >= min_wqes)
 			return 0;
 
 		msleep(20);
-	}
+	} while (time_before(jiffies, exp_time));
+
+	netdev_warn(c->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
+		    c->ix, rq->rqn, wq->cur_sz, min_wqes);
 
-	netdev_warn(c->netdev, "Failed to get min RX wqes on RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
-		    rq->rqn, wq->cur_sz, min_wqes);
 	return -ETIMEDOUT;
 }
 
@@ -782,7 +822,7 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
 		goto err_destroy_rq;
 
 	if (params->rx_dim_enabled)
-		c->rq.state |= BIT(MLX5E_RQ_STATE_AM);
+		__set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
 
 	return 0;
 
@@ -797,13 +837,15 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
 static void mlx5e_activate_rq(struct mlx5e_rq *rq)
 {
 	struct mlx5e_icosq *sq = &rq->channel->icosq;
-	u16 pi = sq->pc & sq->wq.sz_m1;
+	struct mlx5_wq_cyc *wq = &sq->wq;
 	struct mlx5e_tx_wqe *nopwqe;
 
+	u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
+
 	set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
 	sq->db.ico_wqe[pi].opcode     = MLX5_OPCODE_NOP;
-	nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
-	mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
+	nopwqe = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
+	mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
 }
 
 static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
@@ -846,6 +888,7 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
 {
 	void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
 	struct mlx5_core_dev *mdev = c->mdev;
+	struct mlx5_wq_cyc *wq = &sq->wq;
 	int err;
 
 	sq->pdev      = c->pdev;
@@ -855,10 +898,10 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
 	sq->min_inline_mode = params->tx_min_inline_mode;
 
 	param->wq.db_numa_node = cpu_to_node(c->cpu);
-	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);
+	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
 	if (err)
 		return err;
-	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
+	wq->db = &wq->db[MLX5_SND_DBR];
 
 	err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));
 	if (err)
@@ -901,23 +944,22 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
 {
 	void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
 	struct mlx5_core_dev *mdev = c->mdev;
+	struct mlx5_wq_cyc *wq = &sq->wq;
 	int err;
 
 	sq->channel   = c;
 	sq->uar_map   = mdev->mlx5e_res.bfreg.map;
 
 	param->wq.db_numa_node = cpu_to_node(c->cpu);
-	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);
+	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
 	if (err)
 		return err;
-	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
+	wq->db = &wq->db[MLX5_SND_DBR];
 
 	err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));
 	if (err)
 		goto err_sq_wq_destroy;
 
-	sq->edge = (sq->wq.sz_m1 + 1) - MLX5E_ICOSQ_MAX_WQEBBS;
-
 	return 0;
 
 err_sq_wq_destroy:
@@ -962,10 +1004,12 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
 			     int txq_ix,
 			     struct mlx5e_params *params,
 			     struct mlx5e_sq_param *param,
-			     struct mlx5e_txqsq *sq)
+			     struct mlx5e_txqsq *sq,
+			     int tc)
 {
 	void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
 	struct mlx5_core_dev *mdev = c->mdev;
+	struct mlx5_wq_cyc *wq = &sq->wq;
 	int err;
 
 	sq->pdev      = c->pdev;
@@ -976,21 +1020,25 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
 	sq->txq_ix    = txq_ix;
 	sq->uar_map   = mdev->mlx5e_res.bfreg.map;
 	sq->min_inline_mode = params->tx_min_inline_mode;
+	sq->stats     = &c->priv->channel_stats[c->ix].sq[tc];
 	INIT_WORK(&sq->recover.recover_work, mlx5e_sq_recover);
 	if (MLX5_IPSEC_DEV(c->priv->mdev))
 		set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
+	if (mlx5_accel_is_tls_device(c->priv->mdev))
+		set_bit(MLX5E_SQ_STATE_TLS, &sq->state);
 
 	param->wq.db_numa_node = cpu_to_node(c->cpu);
-	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);
+	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
 	if (err)
 		return err;
-	sq->wq.db    = &sq->wq.db[MLX5_SND_DBR];
+	wq->db    = &wq->db[MLX5_SND_DBR];
 
 	err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));
 	if (err)
 		goto err_sq_wq_destroy;
 
-	sq->edge = (sq->wq.sz_m1 + 1) - MLX5_SEND_WQE_MAX_WQEBBS;
+	INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work);
+	sq->dim.mode = params->tx_cq_moderation.cq_period_mode;
 
 	return 0;
 
@@ -1051,7 +1099,8 @@ static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
 					  MLX5_ADAPTER_PAGE_SHIFT);
 	MLX5_SET64(wq, wq, dbr_addr,      csp->wq_ctrl->db.dma);
 
-	mlx5_fill_page_array(&csp->wq_ctrl->buf, (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
+	mlx5_fill_page_frag_array(&csp->wq_ctrl->buf,
+				  (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 
 	err = mlx5_core_create_sq(mdev, in, inlen, sqn);
 
@@ -1130,13 +1179,14 @@ static int mlx5e_open_txqsq(struct mlx5e_channel *c,
 			    int txq_ix,
 			    struct mlx5e_params *params,
 			    struct mlx5e_sq_param *param,
-			    struct mlx5e_txqsq *sq)
+			    struct mlx5e_txqsq *sq,
+			    int tc)
 {
 	struct mlx5e_create_sq_param csp = {};
 	u32 tx_rate;
 	int err;
 
-	err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq);
+	err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc);
 	if (err)
 		return err;
 
@@ -1153,6 +1203,9 @@ static int mlx5e_open_txqsq(struct mlx5e_channel *c,
 	if (tx_rate)
 		mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
 
+	if (params->tx_dim_enabled)
+		sq->state |= BIT(MLX5E_SQ_STATE_AM);
+
 	return 0;
 
 err_free_txqsq:
@@ -1191,6 +1244,7 @@ static inline void netif_tx_disable_queue(struct netdev_queue *txq)
 static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
 {
 	struct mlx5e_channel *c = sq->channel;
+	struct mlx5_wq_cyc *wq = &sq->wq;
 
 	clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
 	/* prevent netif_tx_wake_queue */
@@ -1199,12 +1253,13 @@ static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
 	netif_tx_disable_queue(sq->txq);
 
 	/* last doorbell out, godspeed .. */
-	if (mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1)) {
+	if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) {
+		u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
 		struct mlx5e_tx_wqe *nop;
 
-		sq->db.wqe_info[(sq->pc & sq->wq.sz_m1)].skb = NULL;
-		nop = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
-		mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nop->ctrl);
+		sq->db.wqe_info[pi].skb = NULL;
+		nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
+		mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl);
 	}
 }
 
@@ -1319,7 +1374,7 @@ static void mlx5e_sq_recover(struct work_struct *work)
 		return;
 
 	mlx5e_reset_txqsq_cc_pc(sq);
-	sq->stats.recover++;
+	sq->stats->recover++;
 	recover->last_recover = jiffies;
 	mlx5e_activate_txqsq(sq);
 }
@@ -1488,7 +1543,7 @@ static int mlx5e_alloc_cq(struct mlx5e_channel *c,
 
 static void mlx5e_free_cq(struct mlx5e_cq *cq)
 {
-	mlx5_cqwq_destroy(&cq->wq_ctrl);
+	mlx5_wq_destroy(&cq->wq_ctrl);
 }
 
 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
@@ -1504,7 +1559,7 @@ static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
 	int err;
 
 	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
-		sizeof(u64) * cq->wq_ctrl.frag_buf.npages;
+		sizeof(u64) * cq->wq_ctrl.buf.npages;
 	in = kvzalloc(inlen, GFP_KERNEL);
 	if (!in)
 		return -ENOMEM;
@@ -1513,7 +1568,7 @@ static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
 
 	memcpy(cqc, param->cqc, sizeof(param->cqc));
 
-	mlx5_fill_page_frag_array(&cq->wq_ctrl.frag_buf,
+	mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
 				  (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
 
 	mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
@@ -1521,7 +1576,7 @@ static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
 	MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
 	MLX5_SET(cqc,   cqc, c_eqn,         eqn);
 	MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
-	MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.frag_buf.page_shift -
+	MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
 					    MLX5_ADAPTER_PAGE_SHIFT);
 	MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
 
@@ -1614,14 +1669,14 @@ static int mlx5e_open_sqs(struct mlx5e_channel *c,
 			  struct mlx5e_params *params,
 			  struct mlx5e_channel_param *cparam)
 {
-	int err;
-	int tc;
+	struct mlx5e_priv *priv = c->priv;
+	int err, tc, max_nch = priv->profile->max_nch(priv->mdev);
 
 	for (tc = 0; tc < params->num_tc; tc++) {
-		int txq_ix = c->ix + tc * params->num_channels;
+		int txq_ix = c->ix + tc * max_nch;
 
 		err = mlx5e_open_txqsq(c, c->priv->tisn[tc], txq_ix,
-				       params, &cparam->sq, &c->sq[tc]);
+				       params, &cparam->sq, &c->sq[tc], tc);
 		if (err)
 			goto err_close_sqs;
 	}
@@ -1751,6 +1806,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
 	c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
 	c->num_tc   = params->num_tc;
 	c->xdp      = !!params->xdp_prog;
+	c->stats    = &priv->channel_stats[ix].ch;
 
 	mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq);
 	c->irq_desc = irq_to_desc(irq);
@@ -1896,7 +1952,6 @@ static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
 	MLX5_SET(rqc, rqc, scatter_fcs,    params->scatter_fcs_en);
 
 	param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev);
-	param->wq.linear = 1;
 }
 
 static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv,
@@ -2084,13 +2139,11 @@ static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
 	int err = 0;
 	int i;
 
-	for (i = 0; i < chs->num; i++) {
-		err = mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq);
-		if (err)
-			break;
-	}
+	for (i = 0; i < chs->num; i++)
+		err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq,
+						  err ? 0 : 20000);
 
-	return err;
+	return err ? -ETIMEDOUT : 0;
 }
 
 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
@@ -2586,7 +2639,7 @@ static void mlx5e_build_channels_tx_maps(struct mlx5e_priv *priv)
 	struct mlx5e_txqsq *sq;
 	int i, tc;
 
-	for (i = 0; i < priv->channels.num; i++)
+	for (i = 0; i < priv->profile->max_nch(priv->mdev); i++)
 		for (tc = 0; tc < priv->profile->max_tc; tc++)
 			priv->channel_tc2txq[i][tc] = i + tc * priv->channels.num;
 
@@ -2610,6 +2663,9 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
 
 	mlx5e_build_channels_tx_maps(priv);
 	mlx5e_activate_channels(&priv->channels);
+	write_lock(&priv->stats_lock);
+	priv->channels_active = true;
+	write_unlock(&priv->stats_lock);
 	netif_tx_start_all_queues(priv->netdev);
 
 	if (MLX5_VPORT_MANAGER(priv->mdev))
@@ -2631,6 +2687,9 @@ void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
 	 */
 	netif_tx_stop_all_queues(priv->netdev);
 	netif_tx_disable(priv->netdev);
+	write_lock(&priv->stats_lock);
+	priv->channels_active = false;
+	write_unlock(&priv->stats_lock);
 	mlx5e_deactivate_channels(&priv->channels);
 }
 
@@ -3085,6 +3144,8 @@ static int mlx5e_setup_tc_mqprio(struct net_device *netdev,
 	if (err)
 		goto out;
 
+	priv->max_opened_tc = max_t(u8, priv->max_opened_tc,
+				    new_channels.params.num_tc);
 	mlx5e_switch_priv_channels(priv, &new_channels, NULL);
 out:
 	mutex_unlock(&priv->state_lock);
@@ -3093,22 +3154,23 @@ static int mlx5e_setup_tc_mqprio(struct net_device *netdev,
 
 #ifdef CONFIG_MLX5_ESWITCH
 static int mlx5e_setup_tc_cls_flower(struct mlx5e_priv *priv,
-				     struct tc_cls_flower_offload *cls_flower)
+				     struct tc_cls_flower_offload *cls_flower,
+				     int flags)
 {
 	switch (cls_flower->command) {
 	case TC_CLSFLOWER_REPLACE:
-		return mlx5e_configure_flower(priv, cls_flower);
+		return mlx5e_configure_flower(priv, cls_flower, flags);
 	case TC_CLSFLOWER_DESTROY:
-		return mlx5e_delete_flower(priv, cls_flower);
+		return mlx5e_delete_flower(priv, cls_flower, flags);
 	case TC_CLSFLOWER_STATS:
-		return mlx5e_stats_flower(priv, cls_flower);
+		return mlx5e_stats_flower(priv, cls_flower, flags);
 	default:
 		return -EOPNOTSUPP;
 	}
 }
 
-int mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
-			    void *cb_priv)
+static int mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
+				   void *cb_priv)
 {
 	struct mlx5e_priv *priv = cb_priv;
 
@@ -3117,7 +3179,7 @@ int mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
 
 	switch (type) {
 	case TC_SETUP_CLSFLOWER:
-		return mlx5e_setup_tc_cls_flower(priv, type_data);
+		return mlx5e_setup_tc_cls_flower(priv, type_data, MLX5E_TC_INGRESS);
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -3174,6 +3236,7 @@ mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
 		stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
 		stats->tx_bytes   = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
 	} else {
+		mlx5e_grp_sw_update_stats(priv);
 		stats->rx_packets = sstats->rx_packets;
 		stats->rx_bytes   = sstats->rx_bytes;
 		stats->tx_packets = sstats->tx_packets;
@@ -3770,7 +3833,7 @@ static bool mlx5e_tx_timeout_eq_recover(struct net_device *dev,
 		return false;
 
 	netdev_err(dev, "Recover %d eqes on EQ 0x%x\n", eqe_count, eq->eqn);
-	sq->channel->stats.eq_rearm++;
+	sq->channel->stats->eq_rearm++;
 	return true;
 }
 
@@ -4038,7 +4101,7 @@ static bool slow_pci_heuristic(struct mlx5_core_dev *mdev)
 	u32 link_speed = 0;
 	u32 pci_bw = 0;
 
-	mlx5e_get_max_linkspeed(mdev, &link_speed);
+	mlx5e_port_max_linkspeed(mdev, &link_speed);
 	pci_bw = pcie_bandwidth_available(mdev->pdev, NULL, NULL, NULL);
 	mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n",
 			   link_speed, pci_bw);
@@ -4049,18 +4112,48 @@ static bool slow_pci_heuristic(struct mlx5_core_dev *mdev)
 		link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw;
 }
 
-void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
+static struct net_dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode)
 {
-	params->tx_cq_moderation.cq_period_mode = cq_period_mode;
+	struct net_dim_cq_moder moder;
 
-	params->tx_cq_moderation.pkts =
-		MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
-	params->tx_cq_moderation.usec =
-		MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
+	moder.cq_period_mode = cq_period_mode;
+	moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
+	moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
+	if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
+		moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE;
+
+	return moder;
+}
 
+static struct net_dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode)
+{
+	struct net_dim_cq_moder moder;
+
+	moder.cq_period_mode = cq_period_mode;
+	moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
+	moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
 	if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
-		params->tx_cq_moderation.usec =
-			MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE;
+		moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
+
+	return moder;
+}
+
+static u8 mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode)
+{
+	return cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE ?
+		NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE :
+		NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
+}
+
+void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
+{
+	if (params->tx_dim_enabled) {
+		u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
+
+		params->tx_cq_moderation = net_dim_get_def_tx_moderation(dim_period_mode);
+	} else {
+		params->tx_cq_moderation = mlx5e_get_def_tx_moderation(cq_period_mode);
+	}
 
 	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_TX_CQE_BASED_MODER,
 			params->tx_cq_moderation.cq_period_mode ==
@@ -4069,28 +4162,12 @@ void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
 
 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
 {
-	params->rx_cq_moderation.cq_period_mode = cq_period_mode;
-
-	params->rx_cq_moderation.pkts =
-		MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
-	params->rx_cq_moderation.usec =
-		MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
-
-	if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
-		params->rx_cq_moderation.usec =
-			MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
-
 	if (params->rx_dim_enabled) {
-		switch (cq_period_mode) {
-		case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
-			params->rx_cq_moderation =
-				net_dim_get_def_profile(NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE);
-			break;
-		case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
-		default:
-			params->rx_cq_moderation =
-				net_dim_get_def_profile(NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE);
-		}
+		u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
+
+		params->rx_cq_moderation = net_dim_get_def_rx_moderation(dim_period_mode);
+	} else {
+		params->rx_cq_moderation = mlx5e_get_def_rx_moderation(cq_period_mode);
 	}
 
 	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER,
@@ -4154,6 +4231,7 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
 			MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
 			MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
 	params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
+	params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
 	mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode);
 	mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
 
@@ -4179,11 +4257,13 @@ static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
 	priv->profile     = profile;
 	priv->ppriv       = ppriv;
 	priv->msglevel    = MLX5E_MSG_LEVEL;
+	priv->max_opened_tc = 1;
 
 	mlx5e_build_nic_params(mdev, &priv->channels.params,
 			       profile->max_nch(mdev), netdev->mtu);
 
 	mutex_init(&priv->state_lock);
+	rwlock_init(&priv->stats_lock);
 
 	INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
 	INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
@@ -4320,6 +4400,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
 #endif
 
 	mlx5e_ipsec_build_netdev(priv);
+	mlx5e_tls_build_netdev(priv);
 }
 
 static void mlx5e_create_q_counters(struct mlx5e_priv *priv)
@@ -4361,12 +4442,16 @@ static void mlx5e_nic_init(struct mlx5_core_dev *mdev,
 	err = mlx5e_ipsec_init(priv);
 	if (err)
 		mlx5_core_err(mdev, "IPSec initialization failed, %d\n", err);
+	err = mlx5e_tls_init(priv);
+	if (err)
+		mlx5_core_err(mdev, "TLS initialization failed, %d\n", err);
 	mlx5e_build_nic_netdev(netdev);
 	mlx5e_vxlan_init(priv);
 }
 
 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
 {
+	mlx5e_tls_cleanup(priv);
 	mlx5e_ipsec_cleanup(priv);
 	mlx5e_vxlan_cleanup(priv);
 }
@@ -4398,7 +4483,7 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
 		goto err_destroy_direct_tirs;
 	}
 
-	err = mlx5e_tc_init(priv);
+	err = mlx5e_tc_nic_init(priv);
 	if (err)
 		goto err_destroy_flow_steering;
 
@@ -4419,7 +4504,7 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
 
 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
 {
-	mlx5e_tc_cleanup(priv);
+	mlx5e_tc_nic_cleanup(priv);
 	mlx5e_destroy_flow_steering(priv);
 	mlx5e_destroy_direct_tirs(priv);
 	mlx5e_destroy_indirect_tirs(priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 876c3e4c6193..de6364125f0f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -66,18 +66,36 @@ static const struct counter_desc sw_rep_stats_desc[] = {
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_bytes) },
 };
 
-#define NUM_VPORT_REP_COUNTERS	ARRAY_SIZE(sw_rep_stats_desc)
+struct vport_stats {
+	u64 vport_rx_packets;
+	u64 vport_tx_packets;
+	u64 vport_rx_bytes;
+	u64 vport_tx_bytes;
+};
+
+static const struct counter_desc vport_rep_stats_desc[] = {
+	{ MLX5E_DECLARE_STAT(struct vport_stats, vport_rx_packets) },
+	{ MLX5E_DECLARE_STAT(struct vport_stats, vport_rx_bytes) },
+	{ MLX5E_DECLARE_STAT(struct vport_stats, vport_tx_packets) },
+	{ MLX5E_DECLARE_STAT(struct vport_stats, vport_tx_bytes) },
+};
+
+#define NUM_VPORT_REP_SW_COUNTERS ARRAY_SIZE(sw_rep_stats_desc)
+#define NUM_VPORT_REP_HW_COUNTERS ARRAY_SIZE(vport_rep_stats_desc)
 
 static void mlx5e_rep_get_strings(struct net_device *dev,
 				  u32 stringset, uint8_t *data)
 {
-	int i;
+	int i, j;
 
 	switch (stringset) {
 	case ETH_SS_STATS:
-		for (i = 0; i < NUM_VPORT_REP_COUNTERS; i++)
+		for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++)
 			strcpy(data + (i * ETH_GSTRING_LEN),
 			       sw_rep_stats_desc[i].format);
+		for (j = 0; j < NUM_VPORT_REP_HW_COUNTERS; j++, i++)
+			strcpy(data + (i * ETH_GSTRING_LEN),
+			       vport_rep_stats_desc[j].format);
 		break;
 	}
 }
@@ -112,35 +130,35 @@ static void mlx5e_rep_update_sw_counters(struct mlx5e_priv *priv)
 	struct mlx5e_sq_stats *sq_stats;
 	int i, j;
 
+	read_lock(&priv->stats_lock);
+	if (!priv->channels_active)
+	        goto out;
+
 	memset(s, 0, sizeof(*s));
 	for (i = 0; i < priv->channels.num; i++) {
 		struct mlx5e_channel *c = priv->channels.c[i];
 
-		rq_stats = &c->rq.stats;
+		rq_stats = c->rq.stats;
 
 		s->rx_packets	+= rq_stats->packets;
 		s->rx_bytes	+= rq_stats->bytes;
 
 		for (j = 0; j < priv->channels.params.num_tc; j++) {
-			sq_stats = &c->sq[j].stats;
+			sq_stats = c->sq[j].stats;
 
 			s->tx_packets		+= sq_stats->packets;
 			s->tx_bytes		+= sq_stats->bytes;
 		}
 	}
-}
-
-static void mlx5e_rep_update_stats(struct mlx5e_priv *priv)
-{
-	mlx5e_rep_update_sw_counters(priv);
-	mlx5e_rep_update_hw_counters(priv);
+out:
+	read_unlock(&priv->stats_lock);
 }
 
 static void mlx5e_rep_get_ethtool_stats(struct net_device *dev,
 					struct ethtool_stats *stats, u64 *data)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
-	int i;
+	int i, j;
 
 	if (!data)
 		return;
@@ -148,18 +166,23 @@ static void mlx5e_rep_get_ethtool_stats(struct net_device *dev,
 	mutex_lock(&priv->state_lock);
 	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 		mlx5e_rep_update_sw_counters(priv);
+	mlx5e_rep_update_hw_counters(priv);
 	mutex_unlock(&priv->state_lock);
 
-	for (i = 0; i < NUM_VPORT_REP_COUNTERS; i++)
+	for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++)
 		data[i] = MLX5E_READ_CTR64_CPU(&priv->stats.sw,
 					       sw_rep_stats_desc, i);
+
+	for (j = 0; j < NUM_VPORT_REP_HW_COUNTERS; j++, i++)
+		data[i] = MLX5E_READ_CTR64_CPU(&priv->stats.vf_vport,
+					       vport_rep_stats_desc, j);
 }
 
 static int mlx5e_rep_get_sset_count(struct net_device *dev, int sset)
 {
 	switch (sset) {
 	case ETH_SS_STATS:
-		return NUM_VPORT_REP_COUNTERS;
+		return NUM_VPORT_REP_SW_COUNTERS + NUM_VPORT_REP_HW_COUNTERS;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -681,8 +704,8 @@ static int mlx5e_rep_open(struct net_device *dev)
 		goto unlock;
 
 	if (!mlx5_modify_vport_admin_state(priv->mdev,
-			MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
-			rep->vport, MLX5_ESW_VPORT_ADMIN_STATE_UP))
+					   MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
+					   rep->vport, MLX5_ESW_VPORT_ADMIN_STATE_UP))
 		netif_carrier_on(dev);
 
 unlock:
@@ -699,8 +722,8 @@ static int mlx5e_rep_close(struct net_device *dev)
 
 	mutex_lock(&priv->state_lock);
 	mlx5_modify_vport_admin_state(priv->mdev,
-			MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
-			rep->vport, MLX5_ESW_VPORT_ADMIN_STATE_DOWN);
+				      MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
+				      rep->vport, MLX5_ESW_VPORT_ADMIN_STATE_DOWN);
 	ret = mlx5e_close_locked(dev);
 	mutex_unlock(&priv->state_lock);
 	return ret;
@@ -723,15 +746,31 @@ static int mlx5e_rep_get_phys_port_name(struct net_device *dev,
 
 static int
 mlx5e_rep_setup_tc_cls_flower(struct mlx5e_priv *priv,
-			      struct tc_cls_flower_offload *cls_flower)
+			      struct tc_cls_flower_offload *cls_flower, int flags)
 {
 	switch (cls_flower->command) {
 	case TC_CLSFLOWER_REPLACE:
-		return mlx5e_configure_flower(priv, cls_flower);
+		return mlx5e_configure_flower(priv, cls_flower, flags);
 	case TC_CLSFLOWER_DESTROY:
-		return mlx5e_delete_flower(priv, cls_flower);
+		return mlx5e_delete_flower(priv, cls_flower, flags);
 	case TC_CLSFLOWER_STATS:
-		return mlx5e_stats_flower(priv, cls_flower);
+		return mlx5e_stats_flower(priv, cls_flower, flags);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int mlx5e_rep_setup_tc_cb_egdev(enum tc_setup_type type, void *type_data,
+				       void *cb_priv)
+{
+	struct mlx5e_priv *priv = cb_priv;
+
+	if (!tc_cls_can_offload_and_chain0(priv->netdev, type_data))
+		return -EOPNOTSUPP;
+
+	switch (type) {
+	case TC_SETUP_CLSFLOWER:
+		return mlx5e_rep_setup_tc_cls_flower(priv, type_data, MLX5E_TC_EGRESS);
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -747,7 +786,7 @@ static int mlx5e_rep_setup_tc_cb(enum tc_setup_type type, void *type_data,
 
 	switch (type) {
 	case TC_SETUP_CLSFLOWER:
-		return mlx5e_rep_setup_tc_cls_flower(priv, type_data);
+		return mlx5e_rep_setup_tc_cls_flower(priv, type_data, MLX5E_TC_INGRESS);
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -832,6 +871,8 @@ mlx5e_get_sw_stats64(const struct net_device *dev,
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	struct mlx5e_sw_stats *sstats = &priv->stats.sw;
 
+	mlx5e_rep_update_sw_counters(priv);
+
 	stats->rx_packets = sstats->rx_packets;
 	stats->rx_bytes   = sstats->rx_bytes;
 	stats->tx_packets = sstats->tx_packets;
@@ -965,14 +1006,8 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv)
 	}
 	rpriv->vport_rx_rule = flow_rule;
 
-	err = mlx5e_tc_init(priv);
-	if (err)
-		goto err_del_flow_rule;
-
 	return 0;
 
-err_del_flow_rule:
-	mlx5_del_flow_rules(rpriv->vport_rx_rule);
 err_destroy_direct_tirs:
 	mlx5e_destroy_direct_tirs(priv);
 err_destroy_direct_rqts:
@@ -984,7 +1019,6 @@ static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv)
 {
 	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 
-	mlx5e_tc_cleanup(priv);
 	mlx5_del_flow_rules(rpriv->vport_rx_rule);
 	mlx5e_destroy_direct_tirs(priv);
 	mlx5e_destroy_direct_rqts(priv);
@@ -1014,7 +1048,7 @@ static const struct mlx5e_profile mlx5e_rep_profile = {
 	.cleanup_rx		= mlx5e_cleanup_rep_rx,
 	.init_tx		= mlx5e_init_rep_tx,
 	.cleanup_tx		= mlx5e_cleanup_nic_tx,
-	.update_stats           = mlx5e_rep_update_stats,
+	.update_stats           = mlx5e_rep_update_hw_counters,
 	.max_nch		= mlx5e_get_rep_max_num_channels,
 	.update_carrier		= NULL,
 	.rx_handlers.handle_rx_cqe       = mlx5e_handle_rx_cqe_rep,
@@ -1042,8 +1076,15 @@ mlx5e_nic_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
 	if (err)
 		goto err_remove_sqs;
 
+	/* init shared tc flow table */
+	err = mlx5e_tc_esw_init(&rpriv->tc_ht);
+	if (err)
+		goto  err_neigh_cleanup;
+
 	return 0;
 
+err_neigh_cleanup:
+	mlx5e_rep_neigh_cleanup(rpriv);
 err_remove_sqs:
 	mlx5e_remove_sqs_fwd_rules(priv);
 	return err;
@@ -1058,9 +1099,8 @@ mlx5e_nic_rep_unload(struct mlx5_eswitch_rep *rep)
 	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 		mlx5e_remove_sqs_fwd_rules(priv);
 
-	/* clean (and re-init) existing uplink offloaded TC rules */
-	mlx5e_tc_cleanup(priv);
-	mlx5e_tc_init(priv);
+	/* clean uplink offloaded TC rules, delete shared tc flow table */
+	mlx5e_tc_esw_cleanup(&rpriv->tc_ht);
 
 	mlx5e_rep_neigh_cleanup(rpriv);
 }
@@ -1107,7 +1147,7 @@ mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
 
 	uplink_rpriv = mlx5_eswitch_get_uplink_priv(dev->priv.eswitch, REP_ETH);
 	upriv = netdev_priv(uplink_rpriv->netdev);
-	err = tc_setup_cb_egdev_register(netdev, mlx5e_setup_tc_block_cb,
+	err = tc_setup_cb_egdev_register(netdev, mlx5e_rep_setup_tc_cb_egdev,
 					 upriv);
 	if (err)
 		goto err_neigh_cleanup;
@@ -1122,7 +1162,7 @@ mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
 	return 0;
 
 err_egdev_cleanup:
-	tc_setup_cb_egdev_unregister(netdev, mlx5e_setup_tc_block_cb,
+	tc_setup_cb_egdev_unregister(netdev, mlx5e_rep_setup_tc_cb_egdev,
 				     upriv);
 
 err_neigh_cleanup:
@@ -1151,7 +1191,7 @@ mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep)
 	uplink_rpriv = mlx5_eswitch_get_uplink_priv(priv->mdev->priv.eswitch,
 						    REP_ETH);
 	upriv = netdev_priv(uplink_rpriv->netdev);
-	tc_setup_cb_egdev_unregister(netdev, mlx5e_setup_tc_block_cb,
+	tc_setup_cb_egdev_unregister(netdev, mlx5e_rep_setup_tc_cb_egdev,
 				     upriv);
 	mlx5e_rep_neigh_cleanup(rpriv);
 	mlx5e_detach_netdev(priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
index b9b481f2833a..844d32d5c29f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
@@ -59,6 +59,7 @@ struct mlx5e_rep_priv {
 	struct net_device      *netdev;
 	struct mlx5_flow_handle *vport_rx_rule;
 	struct list_head       vport_sqs_list;
+	struct rhashtable      tc_ht; /* valid for uplink rep */
 };
 
 static inline
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 1ff0b0e93804..5551e4e22ad0 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -37,6 +37,7 @@
 #include <linux/bpf_trace.h>
 #include <net/busy_poll.h>
 #include <net/ip6_checksum.h>
+#include <net/page_pool.h>
 #include "en.h"
 #include "en_tc.h"
 #include "eswitch.h"
@@ -53,7 +54,7 @@ static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
 static inline void mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cqcc,
 				       void *data)
 {
-	u32 ci = cqcc & cq->wq.fbc.sz_m1;
+	u32 ci = mlx5_cqwq_ctr2ix(&cq->wq, cqcc);
 
 	memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, ci), sizeof(struct mlx5_cqe64));
 }
@@ -64,7 +65,7 @@ static inline void mlx5e_read_title_slot(struct mlx5e_rq *rq,
 	mlx5e_read_cqe_slot(cq, cqcc, &cq->title);
 	cq->decmprs_left        = be32_to_cpu(cq->title.byte_cnt);
 	cq->decmprs_wqe_counter = be16_to_cpu(cq->title.wqe_counter);
-	rq->stats.cqe_compress_blks++;
+	rq->stats->cqe_compress_blks++;
 }
 
 static inline void mlx5e_read_mini_arr_slot(struct mlx5e_cq *cq, u32 cqcc)
@@ -75,10 +76,11 @@ static inline void mlx5e_read_mini_arr_slot(struct mlx5e_cq *cq, u32 cqcc)
 
 static inline void mlx5e_cqes_update_owner(struct mlx5e_cq *cq, u32 cqcc, int n)
 {
-	struct mlx5_frag_buf_ctrl *fbc = &cq->wq.fbc;
-	u8 op_own = (cqcc >> fbc->log_sz) & 1;
-	u32 wq_sz = 1 << fbc->log_sz;
-	u32 ci = cqcc & fbc->sz_m1;
+	struct mlx5_cqwq *wq = &cq->wq;
+
+	u8  op_own = mlx5_cqwq_get_ctr_wrap_cnt(wq, cqcc) & 1;
+	u32 ci     = mlx5_cqwq_ctr2ix(wq, cqcc);
+	u32 wq_sz  = mlx5_cqwq_get_size(wq);
 	u32 ci_top = min_t(u32, wq_sz, ci + n);
 
 	for (; ci < ci_top; ci++, n--) {
@@ -111,7 +113,7 @@ static inline void mlx5e_decompress_cqe(struct mlx5e_rq *rq,
 			mpwrq_get_cqe_consumed_strides(&cq->title);
 	else
 		cq->decmprs_wqe_counter =
-			(cq->decmprs_wqe_counter + 1) & rq->wq.sz_m1;
+			mlx5_wq_ll_ctr2ix(&rq->wq, cq->decmprs_wqe_counter + 1);
 }
 
 static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq *rq,
@@ -144,7 +146,7 @@ static inline u32 mlx5e_decompress_cqes_cont(struct mlx5e_rq *rq,
 	mlx5e_cqes_update_owner(cq, cq->wq.cc, cqcc - cq->wq.cc);
 	cq->wq.cc = cqcc;
 	cq->decmprs_left -= cqe_count;
-	rq->stats.cqe_compress_pkts += cqe_count;
+	rq->stats->cqe_compress_pkts += cqe_count;
 
 	return cqe_count;
 }
@@ -174,14 +176,15 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
 {
 	struct mlx5e_page_cache *cache = &rq->page_cache;
 	u32 tail_next = (cache->tail + 1) & (MLX5E_CACHE_SIZE - 1);
+	struct mlx5e_rq_stats *stats = rq->stats;
 
 	if (tail_next == cache->head) {
-		rq->stats.cache_full++;
+		stats->cache_full++;
 		return false;
 	}
 
 	if (unlikely(mlx5e_page_is_reserved(dma_info->page))) {
-		rq->stats.cache_waive++;
+		stats->cache_waive++;
 		return false;
 	}
 
@@ -194,20 +197,21 @@ static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq,
 				      struct mlx5e_dma_info *dma_info)
 {
 	struct mlx5e_page_cache *cache = &rq->page_cache;
+	struct mlx5e_rq_stats *stats = rq->stats;
 
 	if (unlikely(cache->head == cache->tail)) {
-		rq->stats.cache_empty++;
+		stats->cache_empty++;
 		return false;
 	}
 
 	if (page_ref_count(cache->page_cache[cache->head].page) != 1) {
-		rq->stats.cache_busy++;
+		stats->cache_busy++;
 		return false;
 	}
 
 	*dma_info = cache->page_cache[cache->head];
 	cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1);
-	rq->stats.cache_reuse++;
+	stats->cache_reuse++;
 
 	dma_sync_single_for_device(rq->pdev, dma_info->addr,
 				   RQ_PAGE_SIZE(rq),
@@ -221,7 +225,7 @@ static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq,
 	if (mlx5e_rx_cache_get(rq, dma_info))
 		return 0;
 
-	dma_info->page = dev_alloc_pages(rq->buff.page_order);
+	dma_info->page = page_pool_dev_alloc_pages(rq->page_pool);
 	if (unlikely(!dma_info->page))
 		return -ENOMEM;
 
@@ -236,15 +240,26 @@ static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq,
 	return 0;
 }
 
+static void mlx5e_page_dma_unmap(struct mlx5e_rq *rq,
+					struct mlx5e_dma_info *dma_info)
+{
+	dma_unmap_page(rq->pdev, dma_info->addr, RQ_PAGE_SIZE(rq),
+		       rq->buff.map_dir);
+}
+
 void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,
 			bool recycle)
 {
-	if (likely(recycle) && mlx5e_rx_cache_put(rq, dma_info))
-		return;
+	if (likely(recycle)) {
+		if (mlx5e_rx_cache_put(rq, dma_info))
+			return;
 
-	dma_unmap_page(rq->pdev, dma_info->addr, RQ_PAGE_SIZE(rq),
-		       rq->buff.map_dir);
-	put_page(dma_info->page);
+		mlx5e_page_dma_unmap(rq, dma_info);
+		page_pool_recycle_direct(rq->page_pool, dma_info->page);
+	} else {
+		mlx5e_page_dma_unmap(rq, dma_info);
+		put_page(dma_info->page);
+	}
 }
 
 static inline bool mlx5e_page_reuse(struct mlx5e_rq *rq,
@@ -281,7 +296,7 @@ static inline void mlx5e_free_rx_wqe_reuse(struct mlx5e_rq *rq,
 					   struct mlx5e_wqe_frag_info *wi)
 {
 	if (mlx5e_page_reuse(rq, wi)) {
-		rq->stats.page_reuse++;
+		rq->stats->page_reuse++;
 		return;
 	}
 
@@ -370,6 +385,22 @@ static inline u16 mlx5e_icosq_wrap_cnt(struct mlx5e_icosq *sq)
 	return sq->pc >> MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
 }
 
+static inline void mlx5e_fill_icosq_frag_edge(struct mlx5e_icosq *sq,
+					      struct mlx5_wq_cyc *wq,
+					      u16 pi, u16 frag_pi)
+{
+	struct mlx5e_sq_wqe_info *edge_wi, *wi = &sq->db.ico_wqe[pi];
+	u8 nnops = mlx5_wq_cyc_get_frag_size(wq) - frag_pi;
+
+	edge_wi = wi + nnops;
+
+	/* fill sq frag edge with nops to avoid wqe wrapping two pages */
+	for (; wi < edge_wi; wi++) {
+		wi->opcode = MLX5_OPCODE_NOP;
+		mlx5e_post_nop(wq, sq->sqn, &sq->pc);
+	}
+}
+
 static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 {
 	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
@@ -378,14 +409,16 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 	struct mlx5_wq_cyc *wq = &sq->wq;
 	struct mlx5e_umr_wqe *umr_wqe;
 	u16 xlt_offset = ix << (MLX5E_LOG_ALIGNED_MPWQE_PPW - 1);
+	u16 pi, frag_pi;
 	int err;
-	u16 pi;
 	int i;
 
-	/* fill sq edge with nops to avoid wqe wrap around */
-	while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {
-		sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
-		mlx5e_post_nop(wq, sq->sqn, &sq->pc);
+	pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
+	frag_pi = mlx5_wq_cyc_ctr2fragix(wq, sq->pc);
+
+	if (unlikely(frag_pi + MLX5E_UMR_WQEBBS > mlx5_wq_cyc_get_frag_size(wq))) {
+		mlx5e_fill_icosq_frag_edge(sq, wq, pi, frag_pi);
+		pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
 	}
 
 	umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
@@ -421,7 +454,7 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 		dma_info--;
 		mlx5e_page_release(rq, dma_info, true);
 	}
-	rq->stats.buff_alloc_err++;
+	rq->stats->buff_alloc_err++;
 
 	return err;
 }
@@ -438,7 +471,7 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
 	struct mlx5_wq_ll *wq = &rq->wq;
 	int err;
 
-	if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
+	if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
 		return false;
 
 	if (mlx5_wq_ll_is_full(wq))
@@ -449,7 +482,7 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
 
 		err = mlx5e_alloc_rx_wqe(rq, wqe, wq->head);
 		if (unlikely(err)) {
-			rq->stats.buff_alloc_err++;
+			rq->stats->buff_alloc_err++;
 			break;
 		}
 
@@ -470,7 +503,7 @@ static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq,
 					     struct mlx5_cqe64 *cqe)
 {
 	struct mlx5_wq_cyc *wq = &sq->wq;
-	u16 ci = be16_to_cpu(cqe->wqe_counter) & wq->sz_m1;
+	u16 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
 	struct mlx5e_sq_wqe_info *icowi = &sq->db.ico_wqe[ci];
 
 	mlx5_cqwq_pop(&cq->wq);
@@ -496,7 +529,7 @@ static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq)
 	struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
 	struct mlx5_cqe64 *cqe;
 
-	if (unlikely(!MLX5E_TEST_BIT(sq->state, MLX5E_SQ_STATE_ENABLED)))
+	if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
 		return;
 
 	cqe = mlx5_cqwq_get_cqe(&cq->wq);
@@ -513,7 +546,7 @@ bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
 {
 	struct mlx5_wq_ll *wq = &rq->wq;
 
-	if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
+	if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
 		return false;
 
 	mlx5e_poll_ico_cq(&rq->channel->icosq.cq, rq);
@@ -660,6 +693,7 @@ static inline void mlx5e_handle_csum(struct net_device *netdev,
 				     struct sk_buff *skb,
 				     bool   lro)
 {
+	struct mlx5e_rq_stats *stats = rq->stats;
 	int network_depth = 0;
 
 	if (unlikely(!(netdev->features & NETIF_F_RXCSUM)))
@@ -667,7 +701,7 @@ static inline void mlx5e_handle_csum(struct net_device *netdev,
 
 	if (lro) {
 		skb->ip_summed = CHECKSUM_UNNECESSARY;
-		rq->stats.csum_unnecessary++;
+		stats->csum_unnecessary++;
 		return;
 	}
 
@@ -685,7 +719,7 @@ static inline void mlx5e_handle_csum(struct net_device *netdev,
 		if (unlikely(netdev->features & NETIF_F_RXFCS))
 			skb->csum = csum_add(skb->csum,
 					     (__force __wsum)mlx5e_get_fcs(skb));
-		rq->stats.csum_complete++;
+		stats->csum_complete++;
 		return;
 	}
 
@@ -695,15 +729,15 @@ static inline void mlx5e_handle_csum(struct net_device *netdev,
 		if (cqe_is_tunneled(cqe)) {
 			skb->csum_level = 1;
 			skb->encapsulation = 1;
-			rq->stats.csum_unnecessary_inner++;
+			stats->csum_unnecessary_inner++;
 			return;
 		}
-		rq->stats.csum_unnecessary++;
+		stats->csum_unnecessary++;
 		return;
 	}
 csum_none:
 	skb->ip_summed = CHECKSUM_NONE;
-	rq->stats.csum_none++;
+	stats->csum_none++;
 }
 
 static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
@@ -711,20 +745,20 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
 				      struct mlx5e_rq *rq,
 				      struct sk_buff *skb)
 {
+	u8 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
+	struct mlx5e_rq_stats *stats = rq->stats;
 	struct net_device *netdev = rq->netdev;
-	int lro_num_seg;
 
 	skb->mac_len = ETH_HLEN;
-	lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
 	if (lro_num_seg > 1) {
 		mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt);
 		skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt, lro_num_seg);
 		/* Subtract one since we already counted this as one
 		 * "regular" packet in mlx5e_complete_rx_cqe()
 		 */
-		rq->stats.packets += lro_num_seg - 1;
-		rq->stats.lro_packets++;
-		rq->stats.lro_bytes += cqe_bcnt;
+		stats->packets += lro_num_seg - 1;
+		stats->lro_packets++;
+		stats->lro_bytes += cqe_bcnt;
 	}
 
 	if (unlikely(mlx5e_rx_hw_stamp(rq->tstamp)))
@@ -739,7 +773,7 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
 	if (cqe_has_vlan(cqe)) {
 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 				       be16_to_cpu(cqe->vlan_info));
-		rq->stats.removed_vlan_packets++;
+		stats->removed_vlan_packets++;
 	}
 
 	skb->mark = be32_to_cpu(cqe->sop_drop_qpn) & MLX5E_TC_FLOW_ID_MASK;
@@ -753,8 +787,10 @@ static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq,
 					 u32 cqe_bcnt,
 					 struct sk_buff *skb)
 {
-	rq->stats.packets++;
-	rq->stats.bytes += cqe_bcnt;
+	struct mlx5e_rq_stats *stats = rq->stats;
+
+	stats->packets++;
+	stats->bytes += cqe_bcnt;
 	mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb);
 }
 
@@ -762,7 +798,7 @@ static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq)
 {
 	struct mlx5_wq_cyc *wq = &sq->wq;
 	struct mlx5e_tx_wqe *wqe;
-	u16 pi = (sq->pc - 1) & wq->sz_m1; /* last pi */
+	u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc - 1); /* last pi */
 
 	wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
 
@@ -775,7 +811,7 @@ static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
 {
 	struct mlx5e_xdpsq       *sq   = &rq->xdpsq;
 	struct mlx5_wq_cyc       *wq   = &sq->wq;
-	u16                       pi   = sq->pc & wq->sz_m1;
+	u16                       pi   = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
 	struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
 
 	struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
@@ -786,10 +822,12 @@ static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
 	dma_addr_t dma_addr  = di->addr + data_offset;
 	unsigned int dma_len = xdp->data_end - xdp->data;
 
+	struct mlx5e_rq_stats *stats = rq->stats;
+
 	prefetchw(wqe);
 
 	if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || rq->hw_mtu < dma_len)) {
-		rq->stats.xdp_drop++;
+		stats->xdp_drop++;
 		return false;
 	}
 
@@ -799,7 +837,7 @@ static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
 			mlx5e_xmit_xdp_doorbell(sq);
 			sq->db.doorbell = false;
 		}
-		rq->stats.xdp_tx_full++;
+		stats->xdp_tx_full++;
 		return false;
 	}
 
@@ -833,18 +871,19 @@ static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
 
 	sq->db.doorbell = true;
 
-	rq->stats.xdp_tx++;
+	stats->xdp_tx++;
 	return true;
 }
 
 /* returns true if packet was consumed by xdp */
-static inline int mlx5e_xdp_handle(struct mlx5e_rq *rq,
-				   struct mlx5e_dma_info *di,
-				   void *va, u16 *rx_headroom, u32 *len)
+static inline bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
+				    struct mlx5e_dma_info *di,
+				    void *va, u16 *rx_headroom, u32 *len)
 {
-	const struct bpf_prog *prog = READ_ONCE(rq->xdp_prog);
+	struct bpf_prog *prog = READ_ONCE(rq->xdp_prog);
 	struct xdp_buff xdp;
 	u32 act;
+	int err;
 
 	if (!prog)
 		return false;
@@ -865,12 +904,21 @@ static inline int mlx5e_xdp_handle(struct mlx5e_rq *rq,
 		if (unlikely(!mlx5e_xmit_xdp_frame(rq, di, &xdp)))
 			trace_xdp_exception(rq->netdev, prog, act);
 		return true;
+	case XDP_REDIRECT:
+		/* When XDP enabled then page-refcnt==1 here */
+		err = xdp_do_redirect(rq->netdev, &xdp, prog);
+		if (!err) {
+			__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
+			rq->xdpsq.db.redirect_flush = true;
+			mlx5e_page_dma_unmap(rq, di);
+		}
+		return true;
 	default:
 		bpf_warn_invalid_xdp_action(act);
 	case XDP_ABORTED:
 		trace_xdp_exception(rq->netdev, prog, act);
 	case XDP_DROP:
-		rq->stats.xdp_drop++;
+		rq->stats->xdp_drop++;
 		return true;
 	}
 }
@@ -883,7 +931,7 @@ struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
 	struct sk_buff *skb = build_skb(va, frag_size);
 
 	if (unlikely(!skb)) {
-		rq->stats.buff_alloc_err++;
+		rq->stats->buff_alloc_err++;
 		return NULL;
 	}
 
@@ -910,11 +958,12 @@ struct sk_buff *skb_from_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
 
 	dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset,
 				      frag_size, DMA_FROM_DEVICE);
+	prefetchw(va); /* xdp_frame data area */
 	prefetch(data);
 	wi->offset += frag_size;
 
 	if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
-		rq->stats.wqe_err++;
+		rq->stats->wqe_err++;
 		return NULL;
 	}
 
@@ -1031,7 +1080,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
 	skb = napi_alloc_skb(rq->cq.napi,
 			     ALIGN(MLX5_MPWRQ_SMALL_PACKET_THRESHOLD, sizeof(long)));
 	if (unlikely(!skb)) {
-		rq->stats.buff_alloc_err++;
+		rq->stats->buff_alloc_err++;
 		return NULL;
 	}
 
@@ -1117,12 +1166,12 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 	wi->consumed_strides += cstrides;
 
 	if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
-		rq->stats.wqe_err++;
+		rq->stats->wqe_err++;
 		goto mpwrq_cqe_out;
 	}
 
 	if (unlikely(mpwrq_is_filler_cqe(cqe))) {
-		rq->stats.mpwqe_filler++;
+		rq->stats->mpwqe_filler++;
 		goto mpwrq_cqe_out;
 	}
 
@@ -1152,7 +1201,7 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
 	struct mlx5_cqe64 *cqe;
 	int work_done = 0;
 
-	if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
+	if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
 		return 0;
 
 	if (cq->decmprs_left)
@@ -1182,6 +1231,11 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
 		xdpsq->db.doorbell = false;
 	}
 
+	if (xdpsq->db.redirect_flush) {
+		xdp_do_flush_map();
+		xdpsq->db.redirect_flush = false;
+	}
+
 	mlx5_cqwq_update_db_record(&cq->wq);
 
 	/* ensure cq space is freed before enabling more cqes */
@@ -1200,7 +1254,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
 
 	sq = container_of(cq, struct mlx5e_xdpsq, cq);
 
-	if (unlikely(!MLX5E_TEST_BIT(sq->state, MLX5E_SQ_STATE_ENABLED)))
+	if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
 		return false;
 
 	cqe = mlx5_cqwq_get_cqe(&cq->wq);
@@ -1229,7 +1283,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
 
 			last_wqe = (sqcc == wqe_counter);
 
-			ci = sqcc & sq->wq.sz_m1;
+			ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
 			di = &sq->db.di[ci];
 
 			sqcc++;
@@ -1254,7 +1308,7 @@ void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)
 	u16 ci;
 
 	while (sq->cc != sq->pc) {
-		ci = sq->cc & sq->wq.sz_m1;
+		ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);
 		di = &sq->db.di[ci];
 		sq->cc++;
 
@@ -1272,6 +1326,7 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq,
 					 u32 cqe_bcnt,
 					 struct sk_buff *skb)
 {
+	struct mlx5e_rq_stats *stats = rq->stats;
 	struct hwtstamp_config *tstamp;
 	struct net_device *netdev;
 	struct mlx5e_priv *priv;
@@ -1333,9 +1388,9 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq,
 
 	skb->dev = netdev;
 
-	rq->stats.csum_complete++;
-	rq->stats.packets++;
-	rq->stats.bytes += cqe_bcnt;
+	stats->csum_complete++;
+	stats->packets++;
+	stats->bytes += cqe_bcnt;
 }
 
 void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
index b08c94422907..776b4d68e156 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
@@ -32,6 +32,7 @@
 
 #include "en.h"
 #include "en_accel/ipsec.h"
+#include "en_accel/tls.h"
 
 static const struct counter_desc sw_stats_desc[] = {
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_packets) },
@@ -43,6 +44,12 @@ static const struct counter_desc sw_stats_desc[] = {
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tso_inner_packets) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tso_inner_bytes) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_added_vlan_packets) },
+
+#ifdef CONFIG_MLX5_EN_TLS
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_ooo) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_resync_bytes) },
+#endif
+
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_lro_packets) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_lro_bytes) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_removed_vlan_packets) },
@@ -74,7 +81,6 @@ static const struct counter_desc sw_stats_desc[] = {
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_busy) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_waive) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_eq_rearm) },
-	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, link_down_events_phy) },
 };
 
 #define NUM_SW_COUNTERS			ARRAY_SIZE(sw_stats_desc)
@@ -102,20 +108,22 @@ static int mlx5e_grp_sw_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
 	return idx;
 }
 
-static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
+void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
 {
 	struct mlx5e_sw_stats temp, *s = &temp;
-	struct mlx5e_rq_stats *rq_stats;
-	struct mlx5e_sq_stats *sq_stats;
-	struct mlx5e_ch_stats *ch_stats;
-	int i, j;
+	int i;
 
 	memset(s, 0, sizeof(*s));
-	for (i = 0; i < priv->channels.num; i++) {
-		struct mlx5e_channel *c = priv->channels.c[i];
+	read_lock(&priv->stats_lock);
+	if (!priv->channels_active)
+		goto out;
 
-		rq_stats = &c->rq.stats;
-		ch_stats = &c->stats;
+	for (i = 0; i < priv->profile->max_nch(priv->mdev); i++) {
+		struct mlx5e_channel_stats *channel_stats =
+			&priv->channel_stats[i];
+		struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
+		struct mlx5e_ch_stats *ch_stats = &channel_stats->ch;
+		int j;
 
 		s->rx_packets	+= rq_stats->packets;
 		s->rx_bytes	+= rq_stats->bytes;
@@ -142,8 +150,8 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
 		s->rx_cache_waive += rq_stats->cache_waive;
 		s->ch_eq_rearm += ch_stats->eq_rearm;
 
-		for (j = 0; j < priv->channels.params.num_tc; j++) {
-			sq_stats = &c->sq[j].stats;
+		for (j = 0; j < priv->max_opened_tc; j++) {
+			struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
 
 			s->tx_packets		+= sq_stats->packets;
 			s->tx_bytes		+= sq_stats->bytes;
@@ -161,13 +169,16 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
 			s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
 			s->tx_csum_none		+= sq_stats->csum_none;
 			s->tx_csum_partial	+= sq_stats->csum_partial;
+#ifdef CONFIG_MLX5_EN_TLS
+			s->tx_tls_ooo		+= sq_stats->tls_ooo;
+			s->tx_tls_resync_bytes	+= sq_stats->tls_resync_bytes;
+#endif
 		}
 	}
 
-	s->link_down_events_phy = MLX5_GET(ppcnt_reg,
-				priv->stats.pport.phy_counters,
-				counter_set.phys_layer_cntrs.link_down_events);
 	memcpy(&priv->stats.sw, s, sizeof(*s));
+out:
+	read_unlock(&priv->stats_lock);
 }
 
 static const struct counter_desc q_stats_desc[] = {
@@ -569,12 +580,13 @@ static const struct counter_desc pport_phy_statistical_stats_desc[] = {
 	{ "rx_corrected_bits_phy", PPORT_PHY_STATISTICAL_OFF(phy_corrected_bits) },
 };
 
-#define NUM_PPORT_PHY_COUNTERS		ARRAY_SIZE(pport_phy_statistical_stats_desc)
+#define NUM_PPORT_PHY_STATISTICAL_COUNTERS ARRAY_SIZE(pport_phy_statistical_stats_desc)
 
 static int mlx5e_grp_phy_get_num_stats(struct mlx5e_priv *priv)
 {
+	/* "1" for link_down_events special counter */
 	return MLX5_CAP_PCAM_FEATURE((priv)->mdev, ppcnt_statistical_group) ?
-		NUM_PPORT_PHY_COUNTERS : 0;
+		NUM_PPORT_PHY_STATISTICAL_COUNTERS + 1 : 1;
 }
 
 static int mlx5e_grp_phy_fill_strings(struct mlx5e_priv *priv, u8 *data,
@@ -582,10 +594,14 @@ static int mlx5e_grp_phy_fill_strings(struct mlx5e_priv *priv, u8 *data,
 {
 	int i;
 
-	if (MLX5_CAP_PCAM_FEATURE((priv)->mdev, ppcnt_statistical_group))
-		for (i = 0; i < NUM_PPORT_PHY_COUNTERS; i++)
-			strcpy(data + (idx++) * ETH_GSTRING_LEN,
-			       pport_phy_statistical_stats_desc[i].format);
+	strcpy(data + (idx++) * ETH_GSTRING_LEN, "link_down_events_phy");
+
+	if (!MLX5_CAP_PCAM_FEATURE((priv)->mdev, ppcnt_statistical_group))
+		return idx;
+
+	for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++)
+		strcpy(data + (idx++) * ETH_GSTRING_LEN,
+		       pport_phy_statistical_stats_desc[i].format);
 	return idx;
 }
 
@@ -593,11 +609,17 @@ static int mlx5e_grp_phy_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
 {
 	int i;
 
-	if (MLX5_CAP_PCAM_FEATURE((priv)->mdev, ppcnt_statistical_group))
-		for (i = 0; i < NUM_PPORT_PHY_COUNTERS; i++)
-			data[idx++] =
-				MLX5E_READ_CTR64_BE(&priv->stats.pport.phy_statistical_counters,
-						    pport_phy_statistical_stats_desc, i);
+	/* link_down_events_phy has special handling since it is not stored in __be64 format */
+	data[idx++] = MLX5_GET(ppcnt_reg, priv->stats.pport.phy_counters,
+			       counter_set.phys_layer_cntrs.link_down_events);
+
+	if (!MLX5_CAP_PCAM_FEATURE((priv)->mdev, ppcnt_statistical_group))
+		return idx;
+
+	for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++)
+		data[idx++] =
+			MLX5E_READ_CTR64_BE(&priv->stats.pport.phy_statistical_counters,
+					    pport_phy_statistical_stats_desc, i);
 	return idx;
 }
 
@@ -1065,6 +1087,22 @@ static void mlx5e_grp_ipsec_update_stats(struct mlx5e_priv *priv)
 	mlx5e_ipsec_update_stats(priv);
 }
 
+static int mlx5e_grp_tls_get_num_stats(struct mlx5e_priv *priv)
+{
+	return mlx5e_tls_get_count(priv);
+}
+
+static int mlx5e_grp_tls_fill_strings(struct mlx5e_priv *priv, u8 *data,
+				      int idx)
+{
+	return idx + mlx5e_tls_get_strings(priv, data + idx * ETH_GSTRING_LEN);
+}
+
+static int mlx5e_grp_tls_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
+{
+	return idx + mlx5e_tls_get_stats(priv, data + idx);
+}
+
 static const struct counter_desc rq_stats_desc[] = {
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, packets) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, bytes) },
@@ -1121,30 +1159,37 @@ static const struct counter_desc ch_stats_desc[] = {
 
 static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv)
 {
-	return (NUM_RQ_STATS * priv->channels.num) +
-		(NUM_CH_STATS * priv->channels.num) +
-		(NUM_SQ_STATS * priv->channels.num * priv->channels.params.num_tc);
+	int max_nch = priv->profile->max_nch(priv->mdev);
+
+	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
+		return 0;
+
+	return (NUM_RQ_STATS * max_nch) +
+	       (NUM_CH_STATS * max_nch) +
+	       (NUM_SQ_STATS * max_nch * priv->max_opened_tc);
 }
 
 static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv *priv, u8 *data,
 					   int idx)
 {
+	int max_nch = priv->profile->max_nch(priv->mdev);
 	int i, j, tc;
 
 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
 		return idx;
 
-	for (i = 0; i < priv->channels.num; i++)
+	for (i = 0; i < max_nch; i++)
 		for (j = 0; j < NUM_CH_STATS; j++)
 			sprintf(data + (idx++) * ETH_GSTRING_LEN,
 				ch_stats_desc[j].format, i);
 
-	for (i = 0; i < priv->channels.num; i++)
+	for (i = 0; i < max_nch; i++)
 		for (j = 0; j < NUM_RQ_STATS; j++)
 			sprintf(data + (idx++) * ETH_GSTRING_LEN, rq_stats_desc[j].format, i);
 
-	for (tc = 0; tc < priv->channels.params.num_tc; tc++)
-		for (i = 0; i < priv->channels.num; i++)
+	/* priv->channel_tc2txq[i][tc] is valid only when device is open */
+	for (tc = 0; tc < priv->max_opened_tc; tc++)
+		for (i = 0; i < max_nch; i++)
 			for (j = 0; j < NUM_SQ_STATS; j++)
 				sprintf(data + (idx++) * ETH_GSTRING_LEN,
 					sq_stats_desc[j].format,
@@ -1156,29 +1201,29 @@ static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv *priv, u8 *data,
 static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv *priv, u64 *data,
 					 int idx)
 {
-	struct mlx5e_channels *channels = &priv->channels;
+	int max_nch = priv->profile->max_nch(priv->mdev);
 	int i, j, tc;
 
 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
 		return idx;
 
-	for (i = 0; i < channels->num; i++)
+	for (i = 0; i < max_nch; i++)
 		for (j = 0; j < NUM_CH_STATS; j++)
 			data[idx++] =
-				MLX5E_READ_CTR64_CPU(&channels->c[i]->stats,
+				MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].ch,
 						     ch_stats_desc, j);
 
-	for (i = 0; i < channels->num; i++)
+	for (i = 0; i < max_nch; i++)
 		for (j = 0; j < NUM_RQ_STATS; j++)
 			data[idx++] =
-				MLX5E_READ_CTR64_CPU(&channels->c[i]->rq.stats,
+				MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq,
 						     rq_stats_desc, j);
 
-	for (tc = 0; tc < priv->channels.params.num_tc; tc++)
-		for (i = 0; i < channels->num; i++)
+	for (tc = 0; tc < priv->max_opened_tc; tc++)
+		for (i = 0; i < max_nch; i++)
 			for (j = 0; j < NUM_SQ_STATS; j++)
 				data[idx++] =
-					MLX5E_READ_CTR64_CPU(&channels->c[i]->sq[tc].stats,
+					MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].sq[tc],
 							     sq_stats_desc, j);
 
 	return idx;
@@ -1190,7 +1235,6 @@ const struct mlx5e_stats_grp mlx5e_stats_grps[] = {
 		.get_num_stats = mlx5e_grp_sw_get_num_stats,
 		.fill_strings = mlx5e_grp_sw_fill_strings,
 		.fill_stats = mlx5e_grp_sw_fill_stats,
-		.update_stats_mask = MLX5E_NDO_UPDATE_STATS,
 		.update_stats = mlx5e_grp_sw_update_stats,
 	},
 	{
@@ -1267,6 +1311,11 @@ const struct mlx5e_stats_grp mlx5e_stats_grps[] = {
 		.fill_stats = mlx5e_grp_ipsec_fill_stats,
 		.update_stats = mlx5e_grp_ipsec_update_stats,
 	},
+	{
+		.get_num_stats = mlx5e_grp_tls_get_num_stats,
+		.fill_strings = mlx5e_grp_tls_fill_strings,
+		.fill_stats = mlx5e_grp_tls_fill_stats,
+	},
 	{
 		.get_num_stats = mlx5e_grp_channels_get_num_stats,
 		.fill_strings = mlx5e_grp_channels_fill_strings,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
index 53111a2df587..390c7afa5188 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
@@ -93,8 +93,10 @@ struct mlx5e_sw_stats {
 	u64 rx_cache_waive;
 	u64 ch_eq_rearm;
 
-	/* Special handling counters */
-	u64 link_down_events_phy;
+#ifdef CONFIG_MLX5_EN_TLS
+	u64 tx_tls_ooo;
+	u64 tx_tls_resync_bytes;
+#endif
 };
 
 struct mlx5e_qcounter_stats {
@@ -194,6 +196,10 @@ struct mlx5e_sq_stats {
 	u64 csum_partial_inner;
 	u64 added_vlan_packets;
 	u64 nop;
+#ifdef CONFIG_MLX5_EN_TLS
+	u64 tls_ooo;
+	u64 tls_resync_bytes;
+#endif
 	/* less likely accessed in data path */
 	u64 csum_none;
 	u64 stopped;
@@ -233,4 +239,6 @@ struct mlx5e_stats_grp {
 extern const struct mlx5e_stats_grp mlx5e_stats_grps[];
 extern const int mlx5e_num_stats_grps;
 
+void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv);
+
 #endif /* __MLX5_EN_STATS_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index b94276db3ce9..9372d914abe5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -52,28 +52,37 @@
 #include "eswitch.h"
 #include "vxlan.h"
 #include "fs_core.h"
+#include "en/port.h"
 
 struct mlx5_nic_flow_attr {
 	u32 action;
 	u32 flow_tag;
 	u32 mod_hdr_id;
 	u32 hairpin_tirn;
+	u8 match_level;
 	struct mlx5_flow_table	*hairpin_ft;
 };
 
+#define MLX5E_TC_FLOW_BASE (MLX5E_TC_LAST_EXPORTED_BIT + 1)
+
 enum {
-	MLX5E_TC_FLOW_ESWITCH	= BIT(0),
-	MLX5E_TC_FLOW_NIC	= BIT(1),
-	MLX5E_TC_FLOW_OFFLOADED	= BIT(2),
-	MLX5E_TC_FLOW_HAIRPIN	= BIT(3),
-	MLX5E_TC_FLOW_HAIRPIN_RSS = BIT(4),
+	MLX5E_TC_FLOW_INGRESS	= MLX5E_TC_INGRESS,
+	MLX5E_TC_FLOW_EGRESS	= MLX5E_TC_EGRESS,
+	MLX5E_TC_FLOW_ESWITCH	= BIT(MLX5E_TC_FLOW_BASE),
+	MLX5E_TC_FLOW_NIC	= BIT(MLX5E_TC_FLOW_BASE + 1),
+	MLX5E_TC_FLOW_OFFLOADED	= BIT(MLX5E_TC_FLOW_BASE + 2),
+	MLX5E_TC_FLOW_HAIRPIN	= BIT(MLX5E_TC_FLOW_BASE + 3),
+	MLX5E_TC_FLOW_HAIRPIN_RSS = BIT(MLX5E_TC_FLOW_BASE + 4),
 };
 
+#define MLX5E_TC_MAX_SPLITS 1
+
 struct mlx5e_tc_flow {
 	struct rhash_head	node;
+	struct mlx5e_priv	*priv;
 	u64			cookie;
 	u8			flags;
-	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_handle *rule[MLX5E_TC_MAX_SPLITS + 1];
 	struct list_head	encap;   /* flows sharing the same encap ID */
 	struct list_head	mod_hdr; /* flows sharing the same mod hdr ID */
 	struct list_head	hairpin; /* flows sharing the same hairpin */
@@ -97,7 +106,7 @@ enum {
 };
 
 #define MLX5E_TC_TABLE_NUM_GROUPS 4
-#define MLX5E_TC_TABLE_MAX_GROUP_SIZE (1 << 16)
+#define MLX5E_TC_TABLE_MAX_GROUP_SIZE BIT(16)
 
 struct mlx5e_hairpin {
 	struct mlx5_hairpin *pair;
@@ -607,7 +616,7 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
 
 	params.q_counter = priv->q_counter;
 	/* set hairpin pair per each 50Gbs share of the link */
-	mlx5e_get_max_linkspeed(priv->mdev, &link_speed);
+	mlx5e_port_max_linkspeed(priv->mdev, &link_speed);
 	link_speed = max_t(u32, link_speed, 50000);
 	link_speed64 = link_speed;
 	do_div(link_speed64, 50000);
@@ -753,7 +762,9 @@ mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
 		table_created = true;
 	}
 
-	parse_attr->spec.match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	if (attr->match_level != MLX5_MATCH_NONE)
+		parse_attr->spec.match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+
 	rule = mlx5_add_flow_rules(priv->fs.tc.t, &parse_attr->spec,
 				   &flow_act, dest, dest_ix);
 
@@ -785,11 +796,11 @@ static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv,
 	struct mlx5_nic_flow_attr *attr = flow->nic_attr;
 	struct mlx5_fc *counter = NULL;
 
-	counter = mlx5_flow_rule_counter(flow->rule);
-	mlx5_del_flow_rules(flow->rule);
+	counter = mlx5_flow_rule_counter(flow->rule[0]);
+	mlx5_del_flow_rules(flow->rule[0]);
 	mlx5_fc_destroy(priv->mdev, counter);
 
-	if (!mlx5e_tc_num_filters(priv) && (priv->fs.tc.t)) {
+	if (!mlx5e_tc_num_filters(priv) && priv->fs.tc.t) {
 		mlx5_destroy_flow_table(priv->fs.tc.t);
 		priv->fs.tc.t = NULL;
 	}
@@ -835,7 +846,8 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv,
 		}
 		out_priv = netdev_priv(encap_dev);
 		rpriv = out_priv->ppriv;
-		attr->out_rep = rpriv->rep;
+		attr->out_rep[attr->out_count] = rpriv->rep;
+		attr->out_mdev[attr->out_count++] = out_priv->mdev;
 	}
 
 	err = mlx5_eswitch_add_vlan_action(esw, attr);
@@ -860,9 +872,18 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv,
 		rule = mlx5_eswitch_add_offloaded_rule(esw, &parse_attr->spec, attr);
 		if (IS_ERR(rule))
 			goto err_add_rule;
+
+		if (attr->mirror_count) {
+			flow->rule[1] = mlx5_eswitch_add_fwd_rule(esw, &parse_attr->spec, attr);
+			if (IS_ERR(flow->rule[1]))
+				goto err_fwd_rule;
+		}
 	}
 	return rule;
 
+err_fwd_rule:
+	mlx5_eswitch_del_offloaded_rule(esw, rule, attr);
+	rule = flow->rule[1];
 err_add_rule:
 	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
 		mlx5e_detach_mod_hdr(priv, flow);
@@ -883,7 +904,9 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv,
 
 	if (flow->flags & MLX5E_TC_FLOW_OFFLOADED) {
 		flow->flags &= ~MLX5E_TC_FLOW_OFFLOADED;
-		mlx5_eswitch_del_offloaded_rule(esw, flow->rule, attr);
+		if (attr->mirror_count)
+			mlx5_eswitch_del_offloaded_rule(esw, flow->rule[1], attr);
+		mlx5_eswitch_del_offloaded_rule(esw, flow->rule[0], attr);
 	}
 
 	mlx5_eswitch_del_vlan_action(esw, attr);
@@ -919,13 +942,25 @@ void mlx5e_tc_encap_flows_add(struct mlx5e_priv *priv,
 	list_for_each_entry(flow, &e->flows, encap) {
 		esw_attr = flow->esw_attr;
 		esw_attr->encap_id = e->encap_id;
-		flow->rule = mlx5_eswitch_add_offloaded_rule(esw, &esw_attr->parse_attr->spec, esw_attr);
-		if (IS_ERR(flow->rule)) {
-			err = PTR_ERR(flow->rule);
+		flow->rule[0] = mlx5_eswitch_add_offloaded_rule(esw, &esw_attr->parse_attr->spec, esw_attr);
+		if (IS_ERR(flow->rule[0])) {
+			err = PTR_ERR(flow->rule[0]);
 			mlx5_core_warn(priv->mdev, "Failed to update cached encapsulation flow, %d\n",
 				       err);
 			continue;
 		}
+
+		if (esw_attr->mirror_count) {
+			flow->rule[1] = mlx5_eswitch_add_fwd_rule(esw, &esw_attr->parse_attr->spec, esw_attr);
+			if (IS_ERR(flow->rule[1])) {
+				mlx5_eswitch_del_offloaded_rule(esw, flow->rule[0], esw_attr);
+				err = PTR_ERR(flow->rule[1]);
+				mlx5_core_warn(priv->mdev, "Failed to update cached mirror flow, %d\n",
+					       err);
+				continue;
+			}
+		}
+
 		flow->flags |= MLX5E_TC_FLOW_OFFLOADED;
 	}
 }
@@ -938,8 +973,12 @@ void mlx5e_tc_encap_flows_del(struct mlx5e_priv *priv,
 
 	list_for_each_entry(flow, &e->flows, encap) {
 		if (flow->flags & MLX5E_TC_FLOW_OFFLOADED) {
+			struct mlx5_esw_flow_attr *attr = flow->esw_attr;
+
 			flow->flags &= ~MLX5E_TC_FLOW_OFFLOADED;
-			mlx5_eswitch_del_offloaded_rule(esw, flow->rule, flow->esw_attr);
+			if (attr->mirror_count)
+				mlx5_eswitch_del_offloaded_rule(esw, flow->rule[1], attr);
+			mlx5_eswitch_del_offloaded_rule(esw, flow->rule[0], attr);
 		}
 	}
 
@@ -974,7 +1013,7 @@ void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe)
 			continue;
 		list_for_each_entry(flow, &e->flows, encap) {
 			if (flow->flags & MLX5E_TC_FLOW_OFFLOADED) {
-				counter = mlx5_flow_rule_counter(flow->rule);
+				counter = mlx5_flow_rule_counter(flow->rule[0]);
 				mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse);
 				if (time_after((unsigned long)lastuse, nhe->reported_lastuse)) {
 					neigh_used = true;
@@ -982,6 +1021,8 @@ void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe)
 				}
 			}
 		}
+		if (neigh_used)
+			break;
 	}
 
 	if (neigh_used) {
@@ -1190,7 +1231,7 @@ static int parse_tunnel_attr(struct mlx5e_priv *priv,
 static int __parse_cls_flower(struct mlx5e_priv *priv,
 			      struct mlx5_flow_spec *spec,
 			      struct tc_cls_flower_offload *f,
-			      u8 *min_inline)
+			      u8 *match_level)
 {
 	void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 				       outer_headers);
@@ -1199,7 +1240,7 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
 	u16 addr_type = 0;
 	u8 ip_proto = 0;
 
-	*min_inline = MLX5_INLINE_MODE_L2;
+	*match_level = MLX5_MATCH_NONE;
 
 	if (f->dissector->used_keys &
 	    ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
@@ -1249,58 +1290,6 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
 					 inner_headers);
 	}
 
-	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
-		struct flow_dissector_key_control *key =
-			skb_flow_dissector_target(f->dissector,
-						  FLOW_DISSECTOR_KEY_CONTROL,
-						  f->key);
-
-		struct flow_dissector_key_control *mask =
-			skb_flow_dissector_target(f->dissector,
-						  FLOW_DISSECTOR_KEY_CONTROL,
-						  f->mask);
-		addr_type = key->addr_type;
-
-		/* the HW doesn't support frag first/later */
-		if (mask->flags & FLOW_DIS_FIRST_FRAG)
-			return -EOPNOTSUPP;
-
-		if (mask->flags & FLOW_DIS_IS_FRAGMENT) {
-			MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
-			MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
-				 key->flags & FLOW_DIS_IS_FRAGMENT);
-
-			/* the HW doesn't need L3 inline to match on frag=no */
-			if (key->flags & FLOW_DIS_IS_FRAGMENT)
-				*min_inline = MLX5_INLINE_MODE_IP;
-		}
-	}
-
-	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
-		struct flow_dissector_key_basic *key =
-			skb_flow_dissector_target(f->dissector,
-						  FLOW_DISSECTOR_KEY_BASIC,
-						  f->key);
-		struct flow_dissector_key_basic *mask =
-			skb_flow_dissector_target(f->dissector,
-						  FLOW_DISSECTOR_KEY_BASIC,
-						  f->mask);
-		ip_proto = key->ip_proto;
-
-		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype,
-			 ntohs(mask->n_proto));
-		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
-			 ntohs(key->n_proto));
-
-		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
-			 mask->ip_proto);
-		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
-			 key->ip_proto);
-
-		if (mask->ip_proto)
-			*min_inline = MLX5_INLINE_MODE_IP;
-	}
-
 	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 		struct flow_dissector_key_eth_addrs *key =
 			skb_flow_dissector_target(f->dissector,
@@ -1324,6 +1313,9 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
 		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 					     smac_47_16),
 				key->src);
+
+		if (!is_zero_ether_addr(mask->src) || !is_zero_ether_addr(mask->dst))
+			*match_level = MLX5_MATCH_L2;
 	}
 
 	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
@@ -1344,9 +1336,79 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
 
 			MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_prio, mask->vlan_priority);
 			MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, key->vlan_priority);
+
+			*match_level = MLX5_MATCH_L2;
+		}
+	}
+
+	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
+		struct flow_dissector_key_basic *key =
+			skb_flow_dissector_target(f->dissector,
+						  FLOW_DISSECTOR_KEY_BASIC,
+						  f->key);
+		struct flow_dissector_key_basic *mask =
+			skb_flow_dissector_target(f->dissector,
+						  FLOW_DISSECTOR_KEY_BASIC,
+						  f->mask);
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype,
+			 ntohs(mask->n_proto));
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
+			 ntohs(key->n_proto));
+
+		if (mask->n_proto)
+			*match_level = MLX5_MATCH_L2;
+	}
+
+	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
+		struct flow_dissector_key_control *key =
+			skb_flow_dissector_target(f->dissector,
+						  FLOW_DISSECTOR_KEY_CONTROL,
+						  f->key);
+
+		struct flow_dissector_key_control *mask =
+			skb_flow_dissector_target(f->dissector,
+						  FLOW_DISSECTOR_KEY_CONTROL,
+						  f->mask);
+		addr_type = key->addr_type;
+
+		/* the HW doesn't support frag first/later */
+		if (mask->flags & FLOW_DIS_FIRST_FRAG)
+			return -EOPNOTSUPP;
+
+		if (mask->flags & FLOW_DIS_IS_FRAGMENT) {
+			MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
+			MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
+				 key->flags & FLOW_DIS_IS_FRAGMENT);
+
+			/* the HW doesn't need L3 inline to match on frag=no */
+			if (!(key->flags & FLOW_DIS_IS_FRAGMENT))
+				*match_level = MLX5_INLINE_MODE_L2;
+	/* ***  L2 attributes parsing up to here *** */
+			else
+				*match_level = MLX5_INLINE_MODE_IP;
 		}
 	}
 
+	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
+		struct flow_dissector_key_basic *key =
+			skb_flow_dissector_target(f->dissector,
+						  FLOW_DISSECTOR_KEY_BASIC,
+						  f->key);
+		struct flow_dissector_key_basic *mask =
+			skb_flow_dissector_target(f->dissector,
+						  FLOW_DISSECTOR_KEY_BASIC,
+						  f->mask);
+		ip_proto = key->ip_proto;
+
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
+			 mask->ip_proto);
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
+			 key->ip_proto);
+
+		if (mask->ip_proto)
+			*match_level = MLX5_MATCH_L3;
+	}
+
 	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 		struct flow_dissector_key_ipv4_addrs *key =
 			skb_flow_dissector_target(f->dissector,
@@ -1371,7 +1433,7 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
 		       &key->dst, sizeof(key->dst));
 
 		if (mask->src || mask->dst)
-			*min_inline = MLX5_INLINE_MODE_IP;
+			*match_level = MLX5_MATCH_L3;
 	}
 
 	if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
@@ -1400,7 +1462,7 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
 
 		if (ipv6_addr_type(&mask->src) != IPV6_ADDR_ANY ||
 		    ipv6_addr_type(&mask->dst) != IPV6_ADDR_ANY)
-			*min_inline = MLX5_INLINE_MODE_IP;
+			*match_level = MLX5_MATCH_L3;
 	}
 
 	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_IP)) {
@@ -1428,9 +1490,11 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
 			return -EOPNOTSUPP;
 
 		if (mask->tos || mask->ttl)
-			*min_inline = MLX5_INLINE_MODE_IP;
+			*match_level = MLX5_MATCH_L3;
 	}
 
+	/* ***  L3 attributes parsing up to here *** */
+
 	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) {
 		struct flow_dissector_key_ports *key =
 			skb_flow_dissector_target(f->dissector,
@@ -1471,7 +1535,7 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
 		}
 
 		if (mask->src || mask->dst)
-			*min_inline = MLX5_INLINE_MODE_TCP_UDP;
+			*match_level = MLX5_MATCH_L4;
 	}
 
 	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_TCP)) {
@@ -1490,7 +1554,7 @@ static int __parse_cls_flower(struct mlx5e_priv *priv,
 			 ntohs(key->flags));
 
 		if (mask->flags)
-			*min_inline = MLX5_INLINE_MODE_TCP_UDP;
+			*match_level = MLX5_MATCH_L4;
 	}
 
 	return 0;
@@ -1505,23 +1569,28 @@ static int parse_cls_flower(struct mlx5e_priv *priv,
 	struct mlx5_eswitch *esw = dev->priv.eswitch;
 	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 	struct mlx5_eswitch_rep *rep;
-	u8 min_inline;
+	u8 match_level;
 	int err;
 
-	err = __parse_cls_flower(priv, spec, f, &min_inline);
+	err = __parse_cls_flower(priv, spec, f, &match_level);
 
 	if (!err && (flow->flags & MLX5E_TC_FLOW_ESWITCH)) {
 		rep = rpriv->rep;
 		if (rep->vport != FDB_UPLINK_VPORT &&
 		    (esw->offloads.inline_mode != MLX5_INLINE_MODE_NONE &&
-		    esw->offloads.inline_mode < min_inline)) {
+		    esw->offloads.inline_mode < match_level)) {
 			netdev_warn(priv->netdev,
 				    "Flow is not offloaded due to min inline setting, required %d actual %d\n",
-				    min_inline, esw->offloads.inline_mode);
+				    match_level, esw->offloads.inline_mode);
 			return -EOPNOTSUPP;
 		}
 	}
 
+	if (flow->flags & MLX5E_TC_FLOW_ESWITCH)
+		flow->esw_attr->match_level = match_level;
+	else
+		flow->nic_attr->match_level = match_level;
+
 	return err;
 }
 
@@ -1577,7 +1646,6 @@ struct mlx5_fields {
 		{MLX5_ACTION_IN_FIELD_OUT_ ## fw_field, size, offsetof(struct pedit_headers, field) + (off)}
 
 static struct mlx5_fields fields[] = {
-	OFFLOAD(DMAC_47_16, 4, eth.h_dest[0], 0),
 	OFFLOAD(DMAC_47_16, 4, eth.h_dest[0], 0),
 	OFFLOAD(DMAC_15_0,  2, eth.h_dest[4], 0),
 	OFFLOAD(SMAC_47_16, 4, eth.h_source[0], 0),
@@ -1764,12 +1832,12 @@ static int parse_tc_pedit_action(struct mlx5e_priv *priv,
 		err = -EOPNOTSUPP; /* can't be all optimistic */
 
 		if (htype == TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK) {
-			printk(KERN_WARNING "mlx5: legacy pedit isn't offloaded\n");
+			netdev_warn(priv->netdev, "legacy pedit isn't offloaded\n");
 			goto out_err;
 		}
 
 		if (cmd != TCA_PEDIT_KEY_EX_CMD_SET && cmd != TCA_PEDIT_KEY_EX_CMD_ADD) {
-			printk(KERN_WARNING "mlx5: pedit cmd %d isn't offloaded\n", cmd);
+			netdev_warn(priv->netdev, "pedit cmd %d isn't offloaded\n", cmd);
 			goto out_err;
 		}
 
@@ -1793,8 +1861,7 @@ static int parse_tc_pedit_action(struct mlx5e_priv *priv,
 	for (cmd = 0; cmd < __PEDIT_CMD_MAX; cmd++) {
 		cmd_masks = &masks[cmd];
 		if (memcmp(cmd_masks, &zero_masks, sizeof(zero_masks))) {
-			printk(KERN_WARNING "mlx5: attempt to offload an unsupported field (cmd %d)\n",
-			       cmd);
+			netdev_warn(priv->netdev, "attempt to offload an unsupported field (cmd %d)\n", cmd);
 			print_hex_dump(KERN_WARNING, "mask: ", DUMP_PREFIX_ADDRESS,
 				       16, 1, cmd_masks, sizeof(zero_masks), true);
 			err = -EOPNOTSUPP;
@@ -1917,21 +1984,21 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 	struct mlx5_nic_flow_attr *attr = flow->nic_attr;
 	const struct tc_action *a;
 	LIST_HEAD(actions);
+	u32 action = 0;
 	int err;
 
 	if (!tcf_exts_has_actions(exts))
 		return -EINVAL;
 
 	attr->flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
-	attr->action = 0;
 
 	tcf_exts_to_list(exts, &actions);
 	list_for_each_entry(a, &actions, list) {
 		if (is_tcf_gact_shot(a)) {
-			attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+			action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
 			if (MLX5_CAP_FLOWTABLE(priv->mdev,
 					       flow_table_properties_nic_receive.flow_counter))
-				attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
+				action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
 			continue;
 		}
 
@@ -1941,13 +2008,13 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 			if (err)
 				return err;
 
-			attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
-					MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+			action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
+				  MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 			continue;
 		}
 
 		if (is_tcf_csum(a)) {
-			if (csum_offload_supported(priv, attr->action,
+			if (csum_offload_supported(priv, action,
 						   tcf_csum_update_flags(a)))
 				continue;
 
@@ -1961,8 +2028,8 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 			    same_hw_devs(priv, netdev_priv(peer_dev))) {
 				parse_attr->mirred_ifindex = peer_dev->ifindex;
 				flow->flags |= MLX5E_TC_FLOW_HAIRPIN;
-				attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
-						MLX5_FLOW_CONTEXT_ACTION_COUNT;
+				action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
+					  MLX5_FLOW_CONTEXT_ACTION_COUNT;
 			} else {
 				netdev_warn(priv->netdev, "device %s not on same HW, can't offload\n",
 					    peer_dev->name);
@@ -1981,13 +2048,14 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 			}
 
 			attr->flow_tag = mark;
-			attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+			action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 			continue;
 		}
 
 		return -EINVAL;
 	}
 
+	attr->action = action;
 	if (!actions_match_supported(priv, exts, parse_attr, flow))
 		return -EOPNOTSUPP;
 
@@ -2044,6 +2112,20 @@ static int mlx5e_route_lookup_ipv4(struct mlx5e_priv *priv,
 	return 0;
 }
 
+static bool is_merged_eswitch_dev(struct mlx5e_priv *priv,
+				  struct net_device *peer_netdev)
+{
+	struct mlx5e_priv *peer_priv;
+
+	peer_priv = netdev_priv(peer_netdev);
+
+	return (MLX5_CAP_ESW(priv->mdev, merged_eswitch) &&
+		(priv->netdev->netdev_ops == peer_netdev->netdev_ops) &&
+		same_hw_devs(priv, peer_priv) &&
+		MLX5_VPORT_MANAGER(peer_priv->mdev) &&
+		(peer_priv->mdev->priv.eswitch->mode == SRIOV_OFFLOADS));
+}
+
 static int mlx5e_route_lookup_ipv6(struct mlx5e_priv *priv,
 				   struct net_device *mirred_dev,
 				   struct net_device **out_dev,
@@ -2459,60 +2541,71 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 	const struct tc_action *a;
 	LIST_HEAD(actions);
 	bool encap = false;
-	int err = 0;
+	u32 action = 0;
 
 	if (!tcf_exts_has_actions(exts))
 		return -EINVAL;
 
-	memset(attr, 0, sizeof(*attr));
 	attr->in_rep = rpriv->rep;
+	attr->in_mdev = priv->mdev;
 
 	tcf_exts_to_list(exts, &actions);
 	list_for_each_entry(a, &actions, list) {
 		if (is_tcf_gact_shot(a)) {
-			attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP |
-					MLX5_FLOW_CONTEXT_ACTION_COUNT;
+			action |= MLX5_FLOW_CONTEXT_ACTION_DROP |
+				  MLX5_FLOW_CONTEXT_ACTION_COUNT;
 			continue;
 		}
 
 		if (is_tcf_pedit(a)) {
+			int err;
+
 			err = parse_tc_pedit_action(priv, a, MLX5_FLOW_NAMESPACE_FDB,
 						    parse_attr);
 			if (err)
 				return err;
 
-			attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+			action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+			attr->mirror_count = attr->out_count;
 			continue;
 		}
 
 		if (is_tcf_csum(a)) {
-			if (csum_offload_supported(priv, attr->action,
+			if (csum_offload_supported(priv, action,
 						   tcf_csum_update_flags(a)))
 				continue;
 
 			return -EOPNOTSUPP;
 		}
 
-		if (is_tcf_mirred_egress_redirect(a)) {
-			struct net_device *out_dev;
+		if (is_tcf_mirred_egress_redirect(a) || is_tcf_mirred_egress_mirror(a)) {
 			struct mlx5e_priv *out_priv;
+			struct net_device *out_dev;
 
 			out_dev = tcf_mirred_dev(a);
 
+			if (attr->out_count >= MLX5_MAX_FLOW_FWD_VPORTS) {
+				pr_err("can't support more than %d output ports, can't offload forwarding\n",
+				       attr->out_count);
+				return -EOPNOTSUPP;
+			}
+
 			if (switchdev_port_same_parent_id(priv->netdev,
-							  out_dev)) {
-				attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
-					MLX5_FLOW_CONTEXT_ACTION_COUNT;
+							  out_dev) ||
+			    is_merged_eswitch_dev(priv, out_dev)) {
+				action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
+					  MLX5_FLOW_CONTEXT_ACTION_COUNT;
 				out_priv = netdev_priv(out_dev);
 				rpriv = out_priv->ppriv;
-				attr->out_rep = rpriv->rep;
+				attr->out_rep[attr->out_count] = rpriv->rep;
+				attr->out_mdev[attr->out_count++] = out_priv->mdev;
 			} else if (encap) {
 				parse_attr->mirred_ifindex = out_dev->ifindex;
 				parse_attr->tun_info = *info;
 				attr->parse_attr = parse_attr;
-				attr->action |= MLX5_FLOW_CONTEXT_ACTION_ENCAP |
-					MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
-					MLX5_FLOW_CONTEXT_ACTION_COUNT;
+				action |= MLX5_FLOW_CONTEXT_ACTION_ENCAP |
+					  MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
+					  MLX5_FLOW_CONTEXT_ACTION_COUNT;
 				/* attr->out_rep is resolved when we handle encap */
 			} else {
 				pr_err("devices %s %s not on same switch HW, can't offload forwarding\n",
@@ -2528,14 +2621,15 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 				encap = true;
 			else
 				return -EOPNOTSUPP;
+			attr->mirror_count = attr->out_count;
 			continue;
 		}
 
 		if (is_tcf_vlan(a)) {
 			if (tcf_vlan_action(a) == TCA_VLAN_ACT_POP) {
-				attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP;
+				action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP;
 			} else if (tcf_vlan_action(a) == TCA_VLAN_ACT_PUSH) {
-				attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
+				action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
 				attr->vlan_vid = tcf_vlan_push_vid(a);
 				if (mlx5_eswitch_vlan_actions_supported(priv->mdev)) {
 					attr->vlan_prio = tcf_vlan_push_prio(a);
@@ -2549,38 +2643,84 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 			} else { /* action is TCA_VLAN_ACT_MODIFY */
 				return -EOPNOTSUPP;
 			}
+			attr->mirror_count = attr->out_count;
 			continue;
 		}
 
 		if (is_tcf_tunnel_release(a)) {
-			attr->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
+			action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
 			continue;
 		}
 
 		return -EINVAL;
 	}
 
+	attr->action = action;
 	if (!actions_match_supported(priv, exts, parse_attr, flow))
 		return -EOPNOTSUPP;
 
-	return err;
+	if (attr->out_count > 1 && !mlx5_esw_has_fwd_fdb(priv->mdev)) {
+		netdev_warn_once(priv->netdev, "current firmware doesn't support split rule for port mirroring\n");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static void get_flags(int flags, u8 *flow_flags)
+{
+	u8 __flow_flags = 0;
+
+	if (flags & MLX5E_TC_INGRESS)
+		__flow_flags |= MLX5E_TC_FLOW_INGRESS;
+	if (flags & MLX5E_TC_EGRESS)
+		__flow_flags |= MLX5E_TC_FLOW_EGRESS;
+
+	*flow_flags = __flow_flags;
+}
+
+static const struct rhashtable_params tc_ht_params = {
+	.head_offset = offsetof(struct mlx5e_tc_flow, node),
+	.key_offset = offsetof(struct mlx5e_tc_flow, cookie),
+	.key_len = sizeof(((struct mlx5e_tc_flow *)0)->cookie),
+	.automatic_shrinking = true,
+};
+
+static struct rhashtable *get_tc_ht(struct mlx5e_priv *priv)
+{
+	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
+	struct mlx5e_rep_priv *uplink_rpriv;
+
+	if (MLX5_VPORT_MANAGER(priv->mdev) && esw->mode == SRIOV_OFFLOADS) {
+		uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
+		return &uplink_rpriv->tc_ht;
+	} else
+		return &priv->fs.tc.ht;
 }
 
 int mlx5e_configure_flower(struct mlx5e_priv *priv,
-			   struct tc_cls_flower_offload *f)
+			   struct tc_cls_flower_offload *f, int flags)
 {
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	struct mlx5e_tc_flow_parse_attr *parse_attr;
-	struct mlx5e_tc_table *tc = &priv->fs.tc;
+	struct rhashtable *tc_ht = get_tc_ht(priv);
 	struct mlx5e_tc_flow *flow;
 	int attr_size, err = 0;
 	u8 flow_flags = 0;
 
+	get_flags(flags, &flow_flags);
+
+	flow = rhashtable_lookup_fast(tc_ht, &f->cookie, tc_ht_params);
+	if (flow) {
+		netdev_warn_once(priv->netdev, "flow cookie %lx already exists, ignoring\n", f->cookie);
+		return 0;
+	}
+
 	if (esw && esw->mode == SRIOV_OFFLOADS) {
-		flow_flags = MLX5E_TC_FLOW_ESWITCH;
+		flow_flags |= MLX5E_TC_FLOW_ESWITCH;
 		attr_size  = sizeof(struct mlx5_esw_flow_attr);
 	} else {
-		flow_flags = MLX5E_TC_FLOW_NIC;
+		flow_flags |= MLX5E_TC_FLOW_NIC;
 		attr_size  = sizeof(struct mlx5_nic_flow_attr);
 	}
 
@@ -2593,6 +2733,7 @@ int mlx5e_configure_flower(struct mlx5e_priv *priv,
 
 	flow->cookie = f->cookie;
 	flow->flags = flow_flags;
+	flow->priv = priv;
 
 	err = parse_cls_flower(priv, flow, &parse_attr->spec, f);
 	if (err < 0)
@@ -2602,16 +2743,16 @@ int mlx5e_configure_flower(struct mlx5e_priv *priv,
 		err = parse_tc_fdb_actions(priv, f->exts, parse_attr, flow);
 		if (err < 0)
 			goto err_free;
-		flow->rule = mlx5e_tc_add_fdb_flow(priv, parse_attr, flow);
+		flow->rule[0] = mlx5e_tc_add_fdb_flow(priv, parse_attr, flow);
 	} else {
 		err = parse_tc_nic_actions(priv, f->exts, parse_attr, flow);
 		if (err < 0)
 			goto err_free;
-		flow->rule = mlx5e_tc_add_nic_flow(priv, parse_attr, flow);
+		flow->rule[0] = mlx5e_tc_add_nic_flow(priv, parse_attr, flow);
 	}
 
-	if (IS_ERR(flow->rule)) {
-		err = PTR_ERR(flow->rule);
+	if (IS_ERR(flow->rule[0])) {
+		err = PTR_ERR(flow->rule[0]);
 		if (err != -EAGAIN)
 			goto err_free;
 	}
@@ -2623,8 +2764,7 @@ int mlx5e_configure_flower(struct mlx5e_priv *priv,
 	    !(flow->esw_attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP))
 		kvfree(parse_attr);
 
-	err = rhashtable_insert_fast(&tc->ht, &flow->node,
-				     tc->ht_params);
+	err = rhashtable_insert_fast(tc_ht, &flow->node, tc_ht_params);
 	if (err) {
 		mlx5e_tc_del_flow(priv, flow);
 		kfree(flow);
@@ -2638,18 +2778,28 @@ int mlx5e_configure_flower(struct mlx5e_priv *priv,
 	return err;
 }
 
+#define DIRECTION_MASK (MLX5E_TC_INGRESS | MLX5E_TC_EGRESS)
+#define FLOW_DIRECTION_MASK (MLX5E_TC_FLOW_INGRESS | MLX5E_TC_FLOW_EGRESS)
+
+static bool same_flow_direction(struct mlx5e_tc_flow *flow, int flags)
+{
+	if ((flow->flags & FLOW_DIRECTION_MASK) == (flags & DIRECTION_MASK))
+		return true;
+
+	return false;
+}
+
 int mlx5e_delete_flower(struct mlx5e_priv *priv,
-			struct tc_cls_flower_offload *f)
+			struct tc_cls_flower_offload *f, int flags)
 {
+	struct rhashtable *tc_ht = get_tc_ht(priv);
 	struct mlx5e_tc_flow *flow;
-	struct mlx5e_tc_table *tc = &priv->fs.tc;
 
-	flow = rhashtable_lookup_fast(&tc->ht, &f->cookie,
-				      tc->ht_params);
-	if (!flow)
+	flow = rhashtable_lookup_fast(tc_ht, &f->cookie, tc_ht_params);
+	if (!flow || !same_flow_direction(flow, flags))
 		return -EINVAL;
 
-	rhashtable_remove_fast(&tc->ht, &flow->node, tc->ht_params);
+	rhashtable_remove_fast(tc_ht, &flow->node, tc_ht_params);
 
 	mlx5e_tc_del_flow(priv, flow);
 
@@ -2659,24 +2809,23 @@ int mlx5e_delete_flower(struct mlx5e_priv *priv,
 }
 
 int mlx5e_stats_flower(struct mlx5e_priv *priv,
-		       struct tc_cls_flower_offload *f)
+		       struct tc_cls_flower_offload *f, int flags)
 {
-	struct mlx5e_tc_table *tc = &priv->fs.tc;
+	struct rhashtable *tc_ht = get_tc_ht(priv);
 	struct mlx5e_tc_flow *flow;
 	struct mlx5_fc *counter;
 	u64 bytes;
 	u64 packets;
 	u64 lastuse;
 
-	flow = rhashtable_lookup_fast(&tc->ht, &f->cookie,
-				      tc->ht_params);
-	if (!flow)
+	flow = rhashtable_lookup_fast(tc_ht, &f->cookie, tc_ht_params);
+	if (!flow || !same_flow_direction(flow, flags))
 		return -EINVAL;
 
 	if (!(flow->flags & MLX5E_TC_FLOW_OFFLOADED))
 		return 0;
 
-	counter = mlx5_flow_rule_counter(flow->rule);
+	counter = mlx5_flow_rule_counter(flow->rule[0]);
 	if (!counter)
 		return 0;
 
@@ -2687,41 +2836,43 @@ int mlx5e_stats_flower(struct mlx5e_priv *priv,
 	return 0;
 }
 
-static const struct rhashtable_params mlx5e_tc_flow_ht_params = {
-	.head_offset = offsetof(struct mlx5e_tc_flow, node),
-	.key_offset = offsetof(struct mlx5e_tc_flow, cookie),
-	.key_len = sizeof(((struct mlx5e_tc_flow *)0)->cookie),
-	.automatic_shrinking = true,
-};
-
-int mlx5e_tc_init(struct mlx5e_priv *priv)
+int mlx5e_tc_nic_init(struct mlx5e_priv *priv)
 {
 	struct mlx5e_tc_table *tc = &priv->fs.tc;
 
 	hash_init(tc->mod_hdr_tbl);
 	hash_init(tc->hairpin_tbl);
 
-	tc->ht_params = mlx5e_tc_flow_ht_params;
-	return rhashtable_init(&tc->ht, &tc->ht_params);
+	return rhashtable_init(&tc->ht, &tc_ht_params);
 }
 
 static void _mlx5e_tc_del_flow(void *ptr, void *arg)
 {
 	struct mlx5e_tc_flow *flow = ptr;
-	struct mlx5e_priv *priv = arg;
+	struct mlx5e_priv *priv = flow->priv;
 
 	mlx5e_tc_del_flow(priv, flow);
 	kfree(flow);
 }
 
-void mlx5e_tc_cleanup(struct mlx5e_priv *priv)
+void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv)
 {
 	struct mlx5e_tc_table *tc = &priv->fs.tc;
 
-	rhashtable_free_and_destroy(&tc->ht, _mlx5e_tc_del_flow, priv);
+	rhashtable_free_and_destroy(&tc->ht, _mlx5e_tc_del_flow, NULL);
 
 	if (!IS_ERR_OR_NULL(tc->t)) {
 		mlx5_destroy_flow_table(tc->t);
 		tc->t = NULL;
 	}
 }
+
+int mlx5e_tc_esw_init(struct rhashtable *tc_ht)
+{
+	return rhashtable_init(tc_ht, &tc_ht_params);
+}
+
+void mlx5e_tc_esw_cleanup(struct rhashtable *tc_ht)
+{
+	rhashtable_free_and_destroy(tc_ht, _mlx5e_tc_del_flow, NULL);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
index c14c263a739b..59e52b845beb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
@@ -38,16 +38,26 @@
 #define MLX5E_TC_FLOW_ID_MASK 0x0000ffff
 
 #ifdef CONFIG_MLX5_ESWITCH
-int mlx5e_tc_init(struct mlx5e_priv *priv);
-void mlx5e_tc_cleanup(struct mlx5e_priv *priv);
+
+enum {
+	MLX5E_TC_INGRESS = BIT(0),
+	MLX5E_TC_EGRESS  = BIT(1),
+	MLX5E_TC_LAST_EXPORTED_BIT = 1,
+};
+
+int mlx5e_tc_nic_init(struct mlx5e_priv *priv);
+void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv);
+
+int mlx5e_tc_esw_init(struct rhashtable *tc_ht);
+void mlx5e_tc_esw_cleanup(struct rhashtable *tc_ht);
 
 int mlx5e_configure_flower(struct mlx5e_priv *priv,
-			   struct tc_cls_flower_offload *f);
+			   struct tc_cls_flower_offload *f, int flags);
 int mlx5e_delete_flower(struct mlx5e_priv *priv,
-			struct tc_cls_flower_offload *f);
+			struct tc_cls_flower_offload *f, int flags);
 
 int mlx5e_stats_flower(struct mlx5e_priv *priv,
-		       struct tc_cls_flower_offload *f);
+		       struct tc_cls_flower_offload *f, int flags);
 
 struct mlx5e_encap_entry;
 void mlx5e_tc_encap_flows_add(struct mlx5e_priv *priv,
@@ -64,8 +74,8 @@ static inline int mlx5e_tc_num_filters(struct mlx5e_priv *priv)
 }
 
 #else /* CONFIG_MLX5_ESWITCH */
-static inline int  mlx5e_tc_init(struct mlx5e_priv *priv) { return 0; }
-static inline void mlx5e_tc_cleanup(struct mlx5e_priv *priv) {}
+static inline int  mlx5e_tc_nic_init(struct mlx5e_priv *priv) { return 0; }
+static inline void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv) {}
 static inline int  mlx5e_tc_num_filters(struct mlx5e_priv *priv) { return 0; }
 #endif
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
index 5532aa3675c7..aafd75257fd0 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
@@ -35,12 +35,21 @@
 #include <net/dsfield.h>
 #include "en.h"
 #include "ipoib/ipoib.h"
-#include "en_accel/ipsec_rxtx.h"
+#include "en_accel/en_accel.h"
 #include "lib/clock.h"
 
 #define MLX5E_SQ_NOPS_ROOM  MLX5_SEND_WQE_MAX_WQEBBS
+
+#ifndef CONFIG_MLX5_EN_TLS
 #define MLX5E_SQ_STOP_ROOM (MLX5_SEND_WQE_MAX_WQEBBS +\
 			    MLX5E_SQ_NOPS_ROOM)
+#else
+/* TLS offload requires MLX5E_SQ_STOP_ROOM to have
+ * enough room for a resync SKB, a normal SKB and a NOP
+ */
+#define MLX5E_SQ_STOP_ROOM (2 * MLX5_SEND_WQE_MAX_WQEBBS +\
+			    MLX5E_SQ_NOPS_ROOM)
+#endif
 
 static inline void mlx5e_tx_dma_unmap(struct device *pdev,
 				      struct mlx5e_sq_dma *dma)
@@ -211,34 +220,31 @@ mlx5e_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb, struct
 		if (skb->encapsulation) {
 			eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM |
 					  MLX5_ETH_WQE_L4_INNER_CSUM;
-			sq->stats.csum_partial_inner++;
+			sq->stats->csum_partial_inner++;
 		} else {
 			eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM;
-			sq->stats.csum_partial++;
+			sq->stats->csum_partial++;
 		}
 	} else
-		sq->stats.csum_none++;
+		sq->stats->csum_none++;
 }
 
 static inline u16
-mlx5e_txwqe_build_eseg_gso(struct mlx5e_txqsq *sq, struct sk_buff *skb,
-			   struct mlx5_wqe_eth_seg *eseg, unsigned int *num_bytes)
+mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb)
 {
+	struct mlx5e_sq_stats *stats = sq->stats;
 	u16 ihs;
 
-	eseg->mss    = cpu_to_be16(skb_shinfo(skb)->gso_size);
-
 	if (skb->encapsulation) {
 		ihs = skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb);
-		sq->stats.tso_inner_packets++;
-		sq->stats.tso_inner_bytes += skb->len - ihs;
+		stats->tso_inner_packets++;
+		stats->tso_inner_bytes += skb->len - ihs;
 	} else {
 		ihs = skb_transport_offset(skb) + tcp_hdrlen(skb);
-		sq->stats.tso_packets++;
-		sq->stats.tso_bytes += skb->len - ihs;
+		stats->tso_packets++;
+		stats->tso_bytes += skb->len - ihs;
 	}
 
-	*num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs;
 	return ihs;
 }
 
@@ -291,17 +297,34 @@ mlx5e_txwqe_build_dsegs(struct mlx5e_txqsq *sq, struct sk_buff *skb,
 	return -ENOMEM;
 }
 
+static inline void mlx5e_fill_sq_frag_edge(struct mlx5e_txqsq *sq,
+					   struct mlx5_wq_cyc *wq,
+					   u16 pi, u16 frag_pi)
+{
+	struct mlx5e_tx_wqe_info *edge_wi, *wi = &sq->db.wqe_info[pi];
+	u8 nnops = mlx5_wq_cyc_get_frag_size(wq) - frag_pi;
+
+	edge_wi = wi + nnops;
+
+	/* fill sq frag edge with nops to avoid wqe wrapping two pages */
+	for (; wi < edge_wi; wi++) {
+		wi->skb        = NULL;
+		wi->num_wqebbs = 1;
+		mlx5e_post_nop(wq, sq->sqn, &sq->pc);
+	}
+	sq->stats->nop += nnops;
+}
+
 static inline void
 mlx5e_txwqe_complete(struct mlx5e_txqsq *sq, struct sk_buff *skb,
-		     u8 opcode, u16 ds_cnt, u32 num_bytes, u8 num_dma,
+		     u8 opcode, u16 ds_cnt, u8 num_wqebbs, u32 num_bytes, u8 num_dma,
 		     struct mlx5e_tx_wqe_info *wi, struct mlx5_wqe_ctrl_seg *cseg)
 {
 	struct mlx5_wq_cyc *wq = &sq->wq;
-	u16 pi;
 
 	wi->num_bytes = num_bytes;
 	wi->num_dma = num_dma;
-	wi->num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
+	wi->num_wqebbs = num_wqebbs;
 	wi->skb = skb;
 
 	cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
@@ -315,84 +338,111 @@ mlx5e_txwqe_complete(struct mlx5e_txqsq *sq, struct sk_buff *skb,
 	sq->pc += wi->num_wqebbs;
 	if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, MLX5E_SQ_STOP_ROOM))) {
 		netif_tx_stop_queue(sq->txq);
-		sq->stats.stopped++;
+		sq->stats->stopped++;
 	}
 
 	if (!skb->xmit_more || netif_xmit_stopped(sq->txq))
 		mlx5e_notify_hw(wq, sq->pc, sq->uar_map, cseg);
-
-	/* fill sq edge with nops to avoid wqe wrap around */
-	while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {
-		sq->db.wqe_info[pi].skb = NULL;
-		mlx5e_post_nop(wq, sq->sqn, &sq->pc);
-		sq->stats.nop++;
-	}
 }
 
-static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
-				 struct mlx5e_tx_wqe *wqe, u16 pi)
-{
-	struct mlx5e_tx_wqe_info *wi   = &sq->db.wqe_info[pi];
+#define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start))
 
-	struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
-	struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
+netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
+			  struct mlx5e_tx_wqe *wqe, u16 pi)
+{
+	struct mlx5_wq_cyc *wq = &sq->wq;
+	struct mlx5_wqe_ctrl_seg *cseg;
+	struct mlx5_wqe_eth_seg  *eseg;
+	struct mlx5_wqe_data_seg *dseg;
+	struct mlx5e_tx_wqe_info *wi;
 
+	struct mlx5e_sq_stats *stats = sq->stats;
 	unsigned char *skb_data = skb->data;
 	unsigned int skb_len = skb->len;
-	u8  opcode = MLX5_OPCODE_SEND;
-	unsigned int num_bytes;
+	u16 ds_cnt, ds_cnt_inl = 0;
+	u16 headlen, ihs, frag_pi;
+	u8 num_wqebbs, opcode;
+	u32 num_bytes;
 	int num_dma;
-	u16 headlen;
-	u16 ds_cnt;
-	u16 ihs;
-
-	mlx5e_txwqe_build_eseg_csum(sq, skb, eseg);
+	__be16 mss;
 
+	/* Calc ihs and ds cnt, no writes to wqe yet */
+	ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
 	if (skb_is_gso(skb)) {
-		opcode = MLX5_OPCODE_LSO;
-		ihs = mlx5e_txwqe_build_eseg_gso(sq, skb, eseg, &num_bytes);
-		sq->stats.packets += skb_shinfo(skb)->gso_segs;
+		opcode    = MLX5_OPCODE_LSO;
+		mss       = cpu_to_be16(skb_shinfo(skb)->gso_size);
+		ihs       = mlx5e_tx_get_gso_ihs(sq, skb);
+		num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs;
+		stats->packets += skb_shinfo(skb)->gso_segs;
 	} else {
-		ihs = mlx5e_calc_min_inline(sq->min_inline_mode, skb);
+		opcode    = MLX5_OPCODE_SEND;
+		mss       = 0;
+		ihs       = mlx5e_calc_min_inline(sq->min_inline_mode, skb);
 		num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN);
-		sq->stats.packets++;
+		stats->packets++;
 	}
-	sq->stats.bytes += num_bytes;
-	sq->stats.xmit_more += skb->xmit_more;
 
-	ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
+	stats->bytes     += num_bytes;
+	stats->xmit_more += skb->xmit_more;
+
+	headlen = skb_len - ihs - skb->data_len;
+	ds_cnt += !!headlen;
+	ds_cnt += skb_shinfo(skb)->nr_frags;
+
+	if (ihs) {
+		ihs += !!skb_vlan_tag_present(skb) * VLAN_HLEN;
+
+		ds_cnt_inl = DIV_ROUND_UP(ihs - INL_HDR_START_SZ, MLX5_SEND_WQE_DS);
+		ds_cnt += ds_cnt_inl;
+	}
+
+	num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
+	frag_pi = mlx5_wq_cyc_ctr2fragix(wq, sq->pc);
+	if (unlikely(frag_pi + num_wqebbs > mlx5_wq_cyc_get_frag_size(wq))) {
+		mlx5e_fill_sq_frag_edge(sq, wq, pi, frag_pi);
+		mlx5e_sq_fetch_wqe(sq, &wqe, &pi);
+	}
+
+	/* fill wqe */
+	wi   = &sq->db.wqe_info[pi];
+	cseg = &wqe->ctrl;
+	eseg = &wqe->eth;
+	dseg =  wqe->data;
+
+	mlx5e_txwqe_build_eseg_csum(sq, skb, eseg);
+
+	eseg->mss = mss;
+
 	if (ihs) {
 		if (skb_vlan_tag_present(skb)) {
-			mlx5e_insert_vlan(eseg->inline_hdr.start, skb, ihs, &skb_data, &skb_len);
-			ihs += VLAN_HLEN;
-			sq->stats.added_vlan_packets++;
+			mlx5e_insert_vlan(eseg->inline_hdr.start, skb,
+					  ihs - VLAN_HLEN, &skb_data, &skb_len);
+			stats->added_vlan_packets++;
 		} else {
 			memcpy(eseg->inline_hdr.start, skb_data, ihs);
 			mlx5e_tx_skb_pull_inline(&skb_data, &skb_len, ihs);
 		}
 		eseg->inline_hdr.sz = cpu_to_be16(ihs);
-		ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr.start), MLX5_SEND_WQE_DS);
+		dseg += ds_cnt_inl;
 	} else if (skb_vlan_tag_present(skb)) {
 		eseg->insert.type = cpu_to_be16(MLX5_ETH_WQE_INSERT_VLAN);
 		if (skb->vlan_proto == cpu_to_be16(ETH_P_8021AD))
 			eseg->insert.type |= cpu_to_be16(MLX5_ETH_WQE_SVLAN);
 		eseg->insert.vlan_tci = cpu_to_be16(skb_vlan_tag_get(skb));
-		sq->stats.added_vlan_packets++;
+		stats->added_vlan_packets++;
 	}
 
-	headlen = skb_len - skb->data_len;
-	num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb_data, headlen,
-					  (struct mlx5_wqe_data_seg *)cseg + ds_cnt);
+	num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb_data, headlen, dseg);
 	if (unlikely(num_dma < 0))
 		goto err_drop;
 
-	mlx5e_txwqe_complete(sq, skb, opcode, ds_cnt + num_dma,
-			     num_bytes, num_dma, wi, cseg);
+	mlx5e_txwqe_complete(sq, skb, opcode, ds_cnt, num_wqebbs, num_bytes,
+			     num_dma, wi, cseg);
 
 	return NETDEV_TX_OK;
 
 err_drop:
-	sq->stats.dropped++;
+	stats->dropped++;
 	dev_kfree_skb_any(skb);
 
 	return NETDEV_TX_OK;
@@ -401,21 +451,19 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
-	struct mlx5e_txqsq *sq = priv->txq2sq[skb_get_queue_mapping(skb)];
-	struct mlx5_wq_cyc *wq = &sq->wq;
-	u16 pi = sq->pc & wq->sz_m1;
-	struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
+	struct mlx5e_tx_wqe *wqe;
+	struct mlx5e_txqsq *sq;
+	u16 pi;
 
-	memset(wqe, 0, sizeof(*wqe));
+	sq = priv->txq2sq[skb_get_queue_mapping(skb)];
+	mlx5e_sq_fetch_wqe(sq, &wqe, &pi);
 
-#ifdef CONFIG_MLX5_EN_IPSEC
-	if (sq->state & BIT(MLX5E_SQ_STATE_IPSEC)) {
-		skb = mlx5e_ipsec_handle_tx_skb(dev, wqe, skb);
-		if (unlikely(!skb))
-			return NETDEV_TX_OK;
-	}
+#ifdef CONFIG_MLX5_ACCEL
+	/* might send skbs and update wqe and pi */
+	skb = mlx5e_accel_handle_tx(skb, sq, dev, &wqe, &pi);
+	if (unlikely(!skb))
+		return NETDEV_TX_OK;
 #endif
-
 	return mlx5e_sq_xmit(sq, skb, wqe, pi);
 }
 
@@ -443,7 +491,7 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 
 	sq = container_of(cq, struct mlx5e_txqsq, cq);
 
-	if (unlikely(!MLX5E_TEST_BIT(sq->state, MLX5E_SQ_STATE_ENABLED)))
+	if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
 		return false;
 
 	cqe = mlx5_cqwq_get_cqe(&cq->wq);
@@ -478,7 +526,7 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 				queue_work(cq->channel->priv->wq,
 					   &sq->recover.recover_work);
 			}
-			sq->stats.cqe_err++;
+			sq->stats->cqe_err++;
 		}
 
 		do {
@@ -489,7 +537,7 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 
 			last_wqe = (sqcc == wqe_counter);
 
-			ci = sqcc & sq->wq.sz_m1;
+			ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
 			wi = &sq->db.wqe_info[ci];
 			skb = wi->skb;
 
@@ -538,7 +586,7 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 				   MLX5E_SQ_STOP_ROOM) &&
 	    !test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) {
 		netif_tx_wake_queue(sq->txq);
-		sq->stats.wake++;
+		sq->stats->wake++;
 	}
 
 	return (i == MLX5E_TX_CQ_POLL_BUDGET);
@@ -552,7 +600,7 @@ void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq)
 	int i;
 
 	while (sq->cc != sq->pc) {
-		ci = sq->cc & sq->wq.sz_m1;
+		ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);
 		wi = &sq->db.wqe_info[ci];
 		skb = wi->skb;
 
@@ -574,18 +622,6 @@ void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq)
 }
 
 #ifdef CONFIG_MLX5_CORE_IPOIB
-
-struct mlx5_wqe_eth_pad {
-	u8 rsvd0[16];
-};
-
-struct mlx5i_tx_wqe {
-	struct mlx5_wqe_ctrl_seg     ctrl;
-	struct mlx5_wqe_datagram_seg datagram;
-	struct mlx5_wqe_eth_pad      pad;
-	struct mlx5_wqe_eth_seg      eth;
-};
-
 static inline void
 mlx5i_txwqe_build_datagram(struct mlx5_av *av, u32 dqpn, u32 dqkey,
 			   struct mlx5_wqe_datagram_seg *dseg)
@@ -598,67 +634,94 @@ mlx5i_txwqe_build_datagram(struct mlx5_av *av, u32 dqpn, u32 dqkey,
 netdev_tx_t mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
 			  struct mlx5_av *av, u32 dqpn, u32 dqkey)
 {
-	struct mlx5_wq_cyc       *wq   = &sq->wq;
-	u16                       pi   = sq->pc & wq->sz_m1;
-	struct mlx5i_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
-	struct mlx5e_tx_wqe_info *wi   = &sq->db.wqe_info[pi];
+	struct mlx5_wq_cyc *wq = &sq->wq;
+	struct mlx5i_tx_wqe *wqe;
 
-	struct mlx5_wqe_ctrl_seg     *cseg = &wqe->ctrl;
-	struct mlx5_wqe_datagram_seg *datagram = &wqe->datagram;
-	struct mlx5_wqe_eth_seg      *eseg = &wqe->eth;
+	struct mlx5_wqe_datagram_seg *datagram;
+	struct mlx5_wqe_ctrl_seg *cseg;
+	struct mlx5_wqe_eth_seg  *eseg;
+	struct mlx5_wqe_data_seg *dseg;
+	struct mlx5e_tx_wqe_info *wi;
 
+	struct mlx5e_sq_stats *stats = sq->stats;
 	unsigned char *skb_data = skb->data;
 	unsigned int skb_len = skb->len;
-	u8  opcode = MLX5_OPCODE_SEND;
-	unsigned int num_bytes;
+	u16 headlen, ihs, pi, frag_pi;
+	u16 ds_cnt, ds_cnt_inl = 0;
+	u8 num_wqebbs, opcode;
+	u32 num_bytes;
 	int num_dma;
-	u16 headlen;
-	u16 ds_cnt;
-	u16 ihs;
-
-	memset(wqe, 0, sizeof(*wqe));
+	__be16 mss;
 
-	mlx5i_txwqe_build_datagram(av, dqpn, dqkey, datagram);
-
-	mlx5e_txwqe_build_eseg_csum(sq, skb, eseg);
+	mlx5i_sq_fetch_wqe(sq, &wqe, &pi);
 
+	/* Calc ihs and ds cnt, no writes to wqe yet */
+	ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
 	if (skb_is_gso(skb)) {
-		opcode = MLX5_OPCODE_LSO;
-		ihs = mlx5e_txwqe_build_eseg_gso(sq, skb, eseg, &num_bytes);
-		sq->stats.packets += skb_shinfo(skb)->gso_segs;
+		opcode    = MLX5_OPCODE_LSO;
+		mss       = cpu_to_be16(skb_shinfo(skb)->gso_size);
+		ihs       = mlx5e_tx_get_gso_ihs(sq, skb);
+		num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs;
+		stats->packets += skb_shinfo(skb)->gso_segs;
 	} else {
-		ihs = mlx5e_calc_min_inline(sq->min_inline_mode, skb);
+		opcode    = MLX5_OPCODE_SEND;
+		mss       = 0;
+		ihs       = mlx5e_calc_min_inline(sq->min_inline_mode, skb);
 		num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN);
-		sq->stats.packets++;
+		stats->packets++;
 	}
 
-	sq->stats.bytes += num_bytes;
-	sq->stats.xmit_more += skb->xmit_more;
+	stats->bytes     += num_bytes;
+	stats->xmit_more += skb->xmit_more;
+
+	headlen = skb_len - ihs - skb->data_len;
+	ds_cnt += !!headlen;
+	ds_cnt += skb_shinfo(skb)->nr_frags;
+
+	if (ihs) {
+		ds_cnt_inl = DIV_ROUND_UP(ihs - INL_HDR_START_SZ, MLX5_SEND_WQE_DS);
+		ds_cnt += ds_cnt_inl;
+	}
+
+	num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
+	frag_pi = mlx5_wq_cyc_ctr2fragix(wq, sq->pc);
+	if (unlikely(frag_pi + num_wqebbs > mlx5_wq_cyc_get_frag_size(wq))) {
+		mlx5e_fill_sq_frag_edge(sq, wq, pi, frag_pi);
+		mlx5i_sq_fetch_wqe(sq, &wqe, &pi);
+	}
+
+	/* fill wqe */
+	wi       = &sq->db.wqe_info[pi];
+	cseg     = &wqe->ctrl;
+	datagram = &wqe->datagram;
+	eseg     = &wqe->eth;
+	dseg     =  wqe->data;
+
+	mlx5i_txwqe_build_datagram(av, dqpn, dqkey, datagram);
+
+	mlx5e_txwqe_build_eseg_csum(sq, skb, eseg);
+
+	eseg->mss = mss;
 
-	ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
 	if (ihs) {
 		memcpy(eseg->inline_hdr.start, skb_data, ihs);
-		mlx5e_tx_skb_pull_inline(&skb_data, &skb_len, ihs);
 		eseg->inline_hdr.sz = cpu_to_be16(ihs);
-		ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr.start), MLX5_SEND_WQE_DS);
+		dseg += ds_cnt_inl;
 	}
 
-	headlen = skb_len - skb->data_len;
-	num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb_data, headlen,
-					  (struct mlx5_wqe_data_seg *)cseg + ds_cnt);
+	num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb_data, headlen, dseg);
 	if (unlikely(num_dma < 0))
 		goto err_drop;
 
-	mlx5e_txwqe_complete(sq, skb, opcode, ds_cnt + num_dma,
-			     num_bytes, num_dma, wi, cseg);
+	mlx5e_txwqe_complete(sq, skb, opcode, ds_cnt, num_wqebbs, num_bytes,
+			     num_dma, wi, cseg);
 
 	return NETDEV_TX_OK;
 
 err_drop:
-	sq->stats.dropped++;
+	stats->dropped++;
 	dev_kfree_skb_any(skb);
 
 	return NETDEV_TX_OK;
 }
-
 #endif
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index f292bb346985..1b17f682693b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -44,6 +44,32 @@ static inline bool mlx5e_channel_no_affinity_change(struct mlx5e_channel *c)
 	return cpumask_test_cpu(current_cpu, aff);
 }
 
+static void mlx5e_handle_tx_dim(struct mlx5e_txqsq *sq)
+{
+	struct mlx5e_sq_stats *stats = sq->stats;
+	struct net_dim_sample dim_sample;
+
+	if (unlikely(!test_bit(MLX5E_SQ_STATE_AM, &sq->state)))
+		return;
+
+	net_dim_sample(sq->cq.event_ctr, stats->packets, stats->bytes,
+		       &dim_sample);
+	net_dim(&sq->dim, dim_sample);
+}
+
+static void mlx5e_handle_rx_dim(struct mlx5e_rq *rq)
+{
+	struct mlx5e_rq_stats *stats = rq->stats;
+	struct net_dim_sample dim_sample;
+
+	if (unlikely(!test_bit(MLX5E_RQ_STATE_AM, &rq->state)))
+		return;
+
+	net_dim_sample(rq->cq.event_ctr, stats->packets, stats->bytes,
+		       &dim_sample);
+	net_dim(&rq->dim, dim_sample);
+}
+
 int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 {
 	struct mlx5e_channel *c = container_of(napi, struct mlx5e_channel,
@@ -75,18 +101,13 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 	if (unlikely(!napi_complete_done(napi, work_done)))
 		return work_done;
 
-	for (i = 0; i < c->num_tc; i++)
+	for (i = 0; i < c->num_tc; i++) {
+		mlx5e_handle_tx_dim(&c->sq[i]);
 		mlx5e_cq_arm(&c->sq[i].cq);
-
-	if (MLX5E_TEST_BIT(c->rq.state, MLX5E_RQ_STATE_AM)) {
-		struct net_dim_sample dim_sample;
-		net_dim_sample(c->rq.cq.event_ctr,
-			       c->rq.stats.packets,
-			       c->rq.stats.bytes,
-			       &dim_sample);
-		net_dim(&c->rq.dim, dim_sample);
 	}
 
+	mlx5e_handle_rx_dim(&c->rq);
+
 	mlx5e_cq_arm(&c->rq.cq);
 	mlx5e_cq_arm(&c->icosq.cq);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index 1352d13eedb3..6cab1dd66d1b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -192,7 +192,7 @@ __esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u32 vport, bool rx_rule,
 	}
 
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
-	dest.vport_num = vport;
+	dest.vport.num = vport;
 
 	esw_debug(esw->dev,
 		  "\tFDB add rule dmac_v(%pM) dmac_c(%pM) -> vport(%d)\n",
@@ -200,7 +200,7 @@ __esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u32 vport, bool rx_rule,
 	spec->match_criteria_enable = match_header;
 	flow_act.action =  MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 	flow_rule =
-		mlx5_add_flow_rules(esw->fdb_table.fdb, spec,
+		mlx5_add_flow_rules(esw->fdb_table.legacy.fdb, spec,
 				    &flow_act, &dest, 1);
 	if (IS_ERR(flow_rule)) {
 		esw_warn(esw->dev,
@@ -282,7 +282,7 @@ static int esw_create_legacy_fdb_table(struct mlx5_eswitch *esw, int nvports)
 		esw_warn(dev, "Failed to create FDB Table err %d\n", err);
 		goto out;
 	}
-	esw->fdb_table.fdb = fdb;
+	esw->fdb_table.legacy.fdb = fdb;
 
 	/* Addresses group : Full match unicast/multicast addresses */
 	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
@@ -343,9 +343,9 @@ static int esw_create_legacy_fdb_table(struct mlx5_eswitch *esw, int nvports)
 			mlx5_destroy_flow_group(esw->fdb_table.legacy.addr_grp);
 			esw->fdb_table.legacy.addr_grp = NULL;
 		}
-		if (!IS_ERR_OR_NULL(esw->fdb_table.fdb)) {
-			mlx5_destroy_flow_table(esw->fdb_table.fdb);
-			esw->fdb_table.fdb = NULL;
+		if (!IS_ERR_OR_NULL(esw->fdb_table.legacy.fdb)) {
+			mlx5_destroy_flow_table(esw->fdb_table.legacy.fdb);
+			esw->fdb_table.legacy.fdb = NULL;
 		}
 	}
 
@@ -355,15 +355,15 @@ static int esw_create_legacy_fdb_table(struct mlx5_eswitch *esw, int nvports)
 
 static void esw_destroy_legacy_fdb_table(struct mlx5_eswitch *esw)
 {
-	if (!esw->fdb_table.fdb)
+	if (!esw->fdb_table.legacy.fdb)
 		return;
 
 	esw_debug(esw->dev, "Destroy FDB Table\n");
 	mlx5_destroy_flow_group(esw->fdb_table.legacy.promisc_grp);
 	mlx5_destroy_flow_group(esw->fdb_table.legacy.allmulti_grp);
 	mlx5_destroy_flow_group(esw->fdb_table.legacy.addr_grp);
-	mlx5_destroy_flow_table(esw->fdb_table.fdb);
-	esw->fdb_table.fdb = NULL;
+	mlx5_destroy_flow_table(esw->fdb_table.legacy.fdb);
+	esw->fdb_table.legacy.fdb = NULL;
 	esw->fdb_table.legacy.addr_grp = NULL;
 	esw->fdb_table.legacy.allmulti_grp = NULL;
 	esw->fdb_table.legacy.promisc_grp = NULL;
@@ -396,7 +396,7 @@ static int esw_add_uc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 
 fdb_add:
 	/* SRIOV is enabled: Forward UC MAC to vport */
-	if (esw->fdb_table.fdb && esw->mode == SRIOV_LEGACY)
+	if (esw->fdb_table.legacy.fdb && esw->mode == SRIOV_LEGACY)
 		vaddr->flow_rule = esw_fdb_set_vport_rule(esw, mac, vport);
 
 	esw_debug(esw->dev, "\tADDED UC MAC: vport[%d] %pM fr(%p)\n",
@@ -486,7 +486,7 @@ static int esw_add_mc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 	u8 *mac = vaddr->node.addr;
 	u32 vport = vaddr->vport;
 
-	if (!esw->fdb_table.fdb)
+	if (!esw->fdb_table.legacy.fdb)
 		return 0;
 
 	esw_mc = l2addr_hash_find(hash, mac, struct esw_mc_addr);
@@ -526,7 +526,7 @@ static int esw_del_mc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 	u8 *mac = vaddr->node.addr;
 	u32 vport = vaddr->vport;
 
-	if (!esw->fdb_table.fdb)
+	if (!esw->fdb_table.legacy.fdb)
 		return 0;
 
 	esw_mc = l2addr_hash_find(hash, mac, struct esw_mc_addr);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index 4cd773fa55e3..b174da2884c5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -55,6 +55,9 @@
 #define MLX5_RATE_TO_BW_SHARE(rate, divider, limit) \
 	min_t(u32, max_t(u32, (rate) / (divider), MLX5_MIN_BW_SHARE), limit)
 
+#define mlx5_esw_has_fwd_fdb(dev) \
+	MLX5_CAP_ESW_FLOWTABLE(dev, fdb_multi_path_to_table)
+
 struct vport_ingress {
 	struct mlx5_flow_table *acl;
 	struct mlx5_flow_group *allow_untagged_spoofchk_grp;
@@ -117,16 +120,18 @@ struct mlx5_vport {
 };
 
 struct mlx5_eswitch_fdb {
-	void *fdb;
 	union {
 		struct legacy_fdb {
+			struct mlx5_flow_table *fdb;
 			struct mlx5_flow_group *addr_grp;
 			struct mlx5_flow_group *allmulti_grp;
 			struct mlx5_flow_group *promisc_grp;
 		} legacy;
 
 		struct offloads_fdb {
-			struct mlx5_flow_table *fdb;
+			struct mlx5_flow_table *fast_fdb;
+			struct mlx5_flow_table *fwd_fdb;
+			struct mlx5_flow_table *slow_fdb;
 			struct mlx5_flow_group *send_to_vport_grp;
 			struct mlx5_flow_group *miss_grp;
 			struct mlx5_flow_handle *miss_rule_uni;
@@ -214,6 +219,10 @@ struct mlx5_flow_handle *
 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
 				struct mlx5_flow_spec *spec,
 				struct mlx5_esw_flow_attr *attr);
+struct mlx5_flow_handle *
+mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
+			  struct mlx5_flow_spec *spec,
+			  struct mlx5_esw_flow_attr *attr);
 void
 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
 				struct mlx5_flow_handle *rule,
@@ -227,9 +236,24 @@ enum {
 	SET_VLAN_INSERT	= BIT(1)
 };
 
+enum mlx5_flow_match_level {
+	MLX5_MATCH_NONE	= MLX5_INLINE_MODE_NONE,
+	MLX5_MATCH_L2	= MLX5_INLINE_MODE_L2,
+	MLX5_MATCH_L3	= MLX5_INLINE_MODE_IP,
+	MLX5_MATCH_L4	= MLX5_INLINE_MODE_TCP_UDP,
+};
+
+/* current maximum for flow based vport multicasting */
+#define MLX5_MAX_FLOW_FWD_VPORTS 2
+
 struct mlx5_esw_flow_attr {
 	struct mlx5_eswitch_rep *in_rep;
-	struct mlx5_eswitch_rep *out_rep;
+	struct mlx5_eswitch_rep *out_rep[MLX5_MAX_FLOW_FWD_VPORTS];
+	struct mlx5_core_dev	*out_mdev[MLX5_MAX_FLOW_FWD_VPORTS];
+	struct mlx5_core_dev	*in_mdev;
+
+	int mirror_count;
+	int out_count;
 
 	int	action;
 	__be16	vlan_proto;
@@ -238,6 +262,7 @@ struct mlx5_esw_flow_attr {
 	bool	vlan_handled;
 	u32	encap_id;
 	u32	mod_hdr_id;
+	u8	match_level;
 	struct mlx5e_tc_flow_parse_attr *parse_attr;
 };
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index 35e256eb2f6e..cecd201f0b73 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -48,16 +48,22 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
 				struct mlx5_flow_spec *spec,
 				struct mlx5_esw_flow_attr *attr)
 {
-	struct mlx5_flow_destination dest[2] = {};
+	struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {};
 	struct mlx5_flow_act flow_act = {0};
+	struct mlx5_flow_table *ft = NULL;
 	struct mlx5_fc *counter = NULL;
 	struct mlx5_flow_handle *rule;
+	int j, i = 0;
 	void *misc;
-	int i = 0;
 
 	if (esw->mode != SRIOV_OFFLOADS)
 		return ERR_PTR(-EOPNOTSUPP);
 
+	if (attr->mirror_count)
+		ft = esw->fdb_table.offloads.fwd_fdb;
+	else
+		ft = esw->fdb_table.offloads.fast_fdb;
+
 	flow_act.action = attr->action;
 	/* if per flow vlan pop/push is emulated, don't set that into the firmware */
 	if (!mlx5_eswitch_vlan_actions_supported(esw->dev))
@@ -70,9 +76,14 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
 	}
 
 	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
-		dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
-		dest[i].vport_num = attr->out_rep->vport;
-		i++;
+		for (j = attr->mirror_count; j < attr->out_count; j++) {
+			dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
+			dest[i].vport.num = attr->out_rep[j]->vport;
+			dest[i].vport.vhca_id =
+				MLX5_CAP_GEN(attr->out_mdev[j], vhca_id);
+			dest[i].vport.vhca_id_valid = !!MLX5_CAP_ESW(esw->dev, merged_eswitch);
+			i++;
+		}
 	}
 	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
 		counter = mlx5_fc_create(esw->dev, true);
@@ -88,11 +99,23 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
 	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
 	MLX5_SET(fte_match_set_misc, misc, source_port, attr->in_rep->vport);
 
+	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
+		MLX5_SET(fte_match_set_misc, misc,
+			 source_eswitch_owner_vhca_id,
+			 MLX5_CAP_GEN(attr->in_mdev, vhca_id));
+
 	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
 	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
+	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
+		MLX5_SET_TO_ONES(fte_match_set_misc, misc,
+				 source_eswitch_owner_vhca_id);
+
+	if (attr->match_level == MLX5_MATCH_NONE)
+		spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
+	else
+		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS |
+					      MLX5_MATCH_MISC_PARAMETERS;
 
-	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS |
-				      MLX5_MATCH_MISC_PARAMETERS;
 	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
 		spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
 
@@ -102,8 +125,7 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
 	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_ENCAP)
 		flow_act.encap_id = attr->encap_id;
 
-	rule = mlx5_add_flow_rules((struct mlx5_flow_table *)esw->fdb_table.fdb,
-				   spec, &flow_act, dest, i);
+	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, i);
 	if (IS_ERR(rule))
 		goto err_add_rule;
 	else
@@ -117,6 +139,57 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
 	return rule;
 }
 
+struct mlx5_flow_handle *
+mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
+			  struct mlx5_flow_spec *spec,
+			  struct mlx5_esw_flow_attr *attr)
+{
+	struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {};
+	struct mlx5_flow_act flow_act = {0};
+	struct mlx5_flow_handle *rule;
+	void *misc;
+	int i;
+
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+	for (i = 0; i < attr->mirror_count; i++) {
+		dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
+		dest[i].vport.num = attr->out_rep[i]->vport;
+		dest[i].vport.vhca_id =
+			MLX5_CAP_GEN(attr->out_mdev[i], vhca_id);
+		dest[i].vport.vhca_id_valid = !!MLX5_CAP_ESW(esw->dev, merged_eswitch);
+	}
+	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest[i].ft = esw->fdb_table.offloads.fwd_fdb,
+	i++;
+
+	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
+	MLX5_SET(fte_match_set_misc, misc, source_port, attr->in_rep->vport);
+
+	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
+		MLX5_SET(fte_match_set_misc, misc,
+			 source_eswitch_owner_vhca_id,
+			 MLX5_CAP_GEN(attr->in_mdev, vhca_id));
+
+	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
+	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
+	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
+		MLX5_SET_TO_ONES(fte_match_set_misc, misc,
+				 source_eswitch_owner_vhca_id);
+
+	if (attr->match_level == MLX5_MATCH_NONE)
+		spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
+	else
+		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS |
+					      MLX5_MATCH_MISC_PARAMETERS;
+
+	rule = mlx5_add_flow_rules(esw->fdb_table.offloads.fast_fdb, spec, &flow_act, dest, i);
+
+	if (!IS_ERR(rule))
+		esw->offloads.num_flows++;
+
+	return rule;
+}
+
 void
 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
 				struct mlx5_flow_handle *rule,
@@ -156,7 +229,7 @@ esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop)
 	struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL;
 
 	in_rep  = attr->in_rep;
-	out_rep = attr->out_rep;
+	out_rep = attr->out_rep[0];
 
 	if (push)
 		vport = in_rep;
@@ -177,7 +250,7 @@ static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr,
 		goto out_notsupp;
 
 	in_rep  = attr->in_rep;
-	out_rep = attr->out_rep;
+	out_rep = attr->out_rep[0];
 
 	if (push && in_rep->vport == FDB_UPLINK_VPORT)
 		goto out_notsupp;
@@ -228,7 +301,7 @@ int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw,
 
 	if (!push && !pop && fwd) {
 		/* tracks VF --> wire rules without vlan push action */
-		if (attr->out_rep->vport == FDB_UPLINK_VPORT) {
+		if (attr->out_rep[0]->vport == FDB_UPLINK_VPORT) {
 			vport->vlan_refcount++;
 			attr->vlan_handled = true;
 		}
@@ -288,7 +361,7 @@ int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw,
 
 	if (!push && !pop && fwd) {
 		/* tracks VF --> wire rules without vlan push action */
-		if (attr->out_rep->vport == FDB_UPLINK_VPORT)
+		if (attr->out_rep[0]->vport == FDB_UPLINK_VPORT)
 			vport->vlan_refcount--;
 
 		return 0;
@@ -343,10 +416,10 @@ mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, int vport, u32 sqn
 
 	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
-	dest.vport_num = vport;
+	dest.vport.num = vport;
 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 
-	flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.fdb, spec,
+	flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, spec,
 					&flow_act, &dest, 1);
 	if (IS_ERR(flow_rule))
 		esw_warn(esw->dev, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule));
@@ -387,10 +460,10 @@ static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
 	dmac_c[0] = 0x01;
 
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
-	dest.vport_num = 0;
+	dest.vport.num = 0;
 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 
-	flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.fdb, spec,
+	flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, spec,
 					&flow_act, &dest, 1);
 	if (IS_ERR(flow_rule)) {
 		err = PTR_ERR(flow_rule);
@@ -405,7 +478,7 @@ static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
 	dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
 			      outer_headers.dmac_47_16);
 	dmac_v[0] = 0x01;
-	flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.fdb, spec,
+	flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, spec,
 					&flow_act, &dest, 1);
 	if (IS_ERR(flow_rule)) {
 		err = PTR_ERR(flow_rule);
@@ -437,7 +510,7 @@ static int esw_create_offloads_fast_fdb_table(struct mlx5_eswitch *esw)
 	if (!root_ns) {
 		esw_warn(dev, "Failed to get FDB flow namespace\n");
 		err = -EOPNOTSUPP;
-		goto out;
+		goto out_namespace;
 	}
 
 	esw_debug(dev, "Create offloads FDB table, min (max esw size(2^%d), max counters(%d)*groups(%d))\n",
@@ -447,6 +520,9 @@ static int esw_create_offloads_fast_fdb_table(struct mlx5_eswitch *esw)
 	esw_size = min_t(int, max_flow_counter * ESW_OFFLOADS_NUM_GROUPS,
 			 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size));
 
+	if (mlx5_esw_has_fwd_fdb(dev))
+		esw_size >>= 1;
+
 	if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
 		flags |= MLX5_FLOW_TABLE_TUNNEL_EN;
 
@@ -457,17 +533,37 @@ static int esw_create_offloads_fast_fdb_table(struct mlx5_eswitch *esw)
 	if (IS_ERR(fdb)) {
 		err = PTR_ERR(fdb);
 		esw_warn(dev, "Failed to create Fast path FDB Table err %d\n", err);
-		goto out;
+		goto out_namespace;
 	}
-	esw->fdb_table.fdb = fdb;
+	esw->fdb_table.offloads.fast_fdb = fdb;
 
-out:
+	if (!mlx5_esw_has_fwd_fdb(dev))
+		goto out_namespace;
+
+	fdb = mlx5_create_auto_grouped_flow_table(root_ns, FDB_FAST_PATH,
+						  esw_size,
+						  ESW_OFFLOADS_NUM_GROUPS, 1,
+						  flags);
+	if (IS_ERR(fdb)) {
+		err = PTR_ERR(fdb);
+		esw_warn(dev, "Failed to create fwd table err %d\n", err);
+		goto out_ft;
+	}
+	esw->fdb_table.offloads.fwd_fdb = fdb;
+
+	return err;
+
+out_ft:
+	mlx5_destroy_flow_table(esw->fdb_table.offloads.fast_fdb);
+out_namespace:
 	return err;
 }
 
 static void esw_destroy_offloads_fast_fdb_table(struct mlx5_eswitch *esw)
 {
-	mlx5_destroy_flow_table(esw->fdb_table.fdb);
+	if (mlx5_esw_has_fwd_fdb(esw->dev))
+		mlx5_destroy_flow_table(esw->fdb_table.offloads.fwd_fdb);
+	mlx5_destroy_flow_table(esw->fdb_table.offloads.fast_fdb);
 }
 
 #define MAX_PF_SQ 256
@@ -513,7 +609,7 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports)
 		esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
 		goto slow_fdb_err;
 	}
-	esw->fdb_table.offloads.fdb = fdb;
+	esw->fdb_table.offloads.slow_fdb = fdb;
 
 	/* create send-to-vport group */
 	memset(flow_group_in, 0, inlen);
@@ -569,9 +665,9 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports)
 miss_err:
 	mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
 send_vport_err:
-	mlx5_destroy_flow_table(esw->fdb_table.offloads.fdb);
+	mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
 slow_fdb_err:
-	mlx5_destroy_flow_table(esw->fdb_table.fdb);
+	esw_destroy_offloads_fast_fdb_table(esw);
 fast_fdb_err:
 ns_err:
 	kvfree(flow_group_in);
@@ -580,7 +676,7 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports)
 
 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
 {
-	if (!esw->fdb_table.fdb)
+	if (!esw->fdb_table.offloads.fast_fdb)
 		return;
 
 	esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
@@ -589,7 +685,7 @@ static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
 	mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
 	mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
 
-	mlx5_destroy_flow_table(esw->fdb_table.offloads.fdb);
+	mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
 	esw_destroy_offloads_fast_fdb_table(esw);
 }
 
@@ -663,7 +759,7 @@ static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
 
 	esw->offloads.vport_rx_group = g;
 out:
-	kfree(flow_group_in);
+	kvfree(flow_group_in);
 	return err;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
index de7fe087d6fe..4e5a5cf25f17 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
@@ -454,7 +454,7 @@ static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size)
 	}
 
 	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
-		sizeof(u64) * conn->cq.wq_ctrl.frag_buf.npages;
+		sizeof(u64) * conn->cq.wq_ctrl.buf.npages;
 	in = kvzalloc(inlen, GFP_KERNEL);
 	if (!in) {
 		err = -ENOMEM;
@@ -469,12 +469,12 @@ static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size)
 	MLX5_SET(cqc, cqc, log_cq_size, ilog2(cq_size));
 	MLX5_SET(cqc, cqc, c_eqn, eqn);
 	MLX5_SET(cqc, cqc, uar_page, fdev->conn_res.uar->index);
-	MLX5_SET(cqc, cqc, log_page_size, conn->cq.wq_ctrl.frag_buf.page_shift -
+	MLX5_SET(cqc, cqc, log_page_size, conn->cq.wq_ctrl.buf.page_shift -
 			   MLX5_ADAPTER_PAGE_SHIFT);
 	MLX5_SET64(cqc, cqc, dbr_addr, conn->cq.wq_ctrl.db.dma);
 
 	pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas);
-	mlx5_fill_page_frag_array(&conn->cq.wq_ctrl.frag_buf, pas);
+	mlx5_fill_page_frag_array(&conn->cq.wq_ctrl.buf, pas);
 
 	err = mlx5_core_create_cq(mdev, &conn->cq.mcq, in, inlen);
 	kvfree(in);
@@ -500,7 +500,7 @@ static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size)
 	goto out;
 
 err_cqwq:
-	mlx5_cqwq_destroy(&conn->cq.wq_ctrl);
+	mlx5_wq_destroy(&conn->cq.wq_ctrl);
 out:
 	return err;
 }
@@ -510,7 +510,7 @@ static void mlx5_fpga_conn_destroy_cq(struct mlx5_fpga_conn *conn)
 	tasklet_disable(&conn->cq.tasklet);
 	tasklet_kill(&conn->cq.tasklet);
 	mlx5_core_destroy_cq(conn->fdev->mdev, &conn->cq.mcq);
-	mlx5_cqwq_destroy(&conn->cq.wq_ctrl);
+	mlx5_wq_destroy(&conn->cq.wq_ctrl);
 }
 
 static int mlx5_fpga_conn_create_wq(struct mlx5_fpga_conn *conn, void *qpc)
@@ -591,8 +591,8 @@ static int mlx5_fpga_conn_create_qp(struct mlx5_fpga_conn *conn,
 	if (MLX5_CAP_GEN(mdev, cqe_version) == 1)
 		MLX5_SET(qpc, qpc, user_index, 0xFFFFFF);
 
-	mlx5_fill_page_array(&conn->qp.wq_ctrl.buf,
-			     (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas));
+	mlx5_fill_page_frag_array(&conn->qp.wq_ctrl.buf,
+				  (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas));
 
 	err = mlx5_core_create_qp(mdev, &conn->qp.mqp, in, inlen);
 	if (err)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.h b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.h
index 44bd9eccc711..634ae10e287b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.h
@@ -54,7 +54,7 @@ struct mlx5_fpga_conn {
 	/* CQ */
 	struct {
 		struct mlx5_cqwq wq;
-		struct mlx5_frag_wq_ctrl wq_ctrl;
+		struct mlx5_wq_ctrl wq_ctrl;
 		struct mlx5_core_cq mcq;
 		struct tasklet_struct tasklet;
 	} cq;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.h b/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.h
index 82405ed84725..3e2355c8df3f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.h
@@ -53,6 +53,7 @@ struct mlx5_fpga_device {
 	} conn_res;
 
 	struct mlx5_fpga_ipsec *ipsec;
+	struct mlx5_fpga_tls *tls;
 };
 
 #define mlx5_fpga_dbg(__adev, format, ...) \
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c
index fad8c2e3804e..a0433b48e833 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c
@@ -43,9 +43,6 @@
 #include "fpga/sdk.h"
 #include "fpga/core.h"
 
-#define SBU_QP_QUEUE_SIZE 8
-#define MLX5_FPGA_IPSEC_CMD_TIMEOUT_MSEC	(60 * 1000)
-
 enum mlx5_fpga_ipsec_cmd_status {
 	MLX5_FPGA_IPSEC_CMD_PENDING,
 	MLX5_FPGA_IPSEC_CMD_SEND_FAIL,
@@ -256,7 +253,7 @@ static int mlx5_fpga_ipsec_cmd_wait(void *ctx)
 {
 	struct mlx5_fpga_ipsec_cmd_context *context = ctx;
 	unsigned long timeout =
-		msecs_to_jiffies(MLX5_FPGA_IPSEC_CMD_TIMEOUT_MSEC);
+		msecs_to_jiffies(MLX5_FPGA_CMD_TIMEOUT_MSEC);
 	int res;
 
 	res = wait_for_completion_timeout(&context->complete, timeout);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/sdk.h b/drivers/net/ethernet/mellanox/mlx5/core/fpga/sdk.h
index baa537e54a49..a0573cc2fc9b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/sdk.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/sdk.h
@@ -41,6 +41,8 @@
  * DOC: Innova SDK
  * This header defines the in-kernel API for Innova FPGA client drivers.
  */
+#define SBU_QP_QUEUE_SIZE 8
+#define MLX5_FPGA_CMD_TIMEOUT_MSEC (60 * 1000)
 
 enum mlx5_fpga_access_type {
 	MLX5_FPGA_ACCESS_TYPE_I2C = 0x0,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.c
new file mode 100644
index 000000000000..21048013826c
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.c
@@ -0,0 +1,562 @@
+/*
+ * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/mlx5/device.h>
+#include "fpga/tls.h"
+#include "fpga/cmd.h"
+#include "fpga/sdk.h"
+#include "fpga/core.h"
+#include "accel/tls.h"
+
+struct mlx5_fpga_tls_command_context;
+
+typedef void (*mlx5_fpga_tls_command_complete)
+	(struct mlx5_fpga_conn *conn, struct mlx5_fpga_device *fdev,
+	 struct mlx5_fpga_tls_command_context *ctx,
+	 struct mlx5_fpga_dma_buf *resp);
+
+struct mlx5_fpga_tls_command_context {
+	struct list_head list;
+	/* There is no guarantee on the order between the TX completion
+	 * and the command response.
+	 * The TX completion is going to touch cmd->buf even in
+	 * the case of successful transmission.
+	 * So instead of requiring separate allocations for cmd
+	 * and cmd->buf we've decided to use a reference counter
+	 */
+	refcount_t ref;
+	struct mlx5_fpga_dma_buf buf;
+	mlx5_fpga_tls_command_complete complete;
+};
+
+static void
+mlx5_fpga_tls_put_command_ctx(struct mlx5_fpga_tls_command_context *ctx)
+{
+	if (refcount_dec_and_test(&ctx->ref))
+		kfree(ctx);
+}
+
+static void mlx5_fpga_tls_cmd_complete(struct mlx5_fpga_device *fdev,
+				       struct mlx5_fpga_dma_buf *resp)
+{
+	struct mlx5_fpga_conn *conn = fdev->tls->conn;
+	struct mlx5_fpga_tls_command_context *ctx;
+	struct mlx5_fpga_tls *tls = fdev->tls;
+	unsigned long flags;
+
+	spin_lock_irqsave(&tls->pending_cmds_lock, flags);
+	ctx = list_first_entry(&tls->pending_cmds,
+			       struct mlx5_fpga_tls_command_context, list);
+	list_del(&ctx->list);
+	spin_unlock_irqrestore(&tls->pending_cmds_lock, flags);
+	ctx->complete(conn, fdev, ctx, resp);
+}
+
+static void mlx5_fpga_cmd_send_complete(struct mlx5_fpga_conn *conn,
+					struct mlx5_fpga_device *fdev,
+					struct mlx5_fpga_dma_buf *buf,
+					u8 status)
+{
+	struct mlx5_fpga_tls_command_context *ctx =
+	    container_of(buf, struct mlx5_fpga_tls_command_context, buf);
+
+	mlx5_fpga_tls_put_command_ctx(ctx);
+
+	if (unlikely(status))
+		mlx5_fpga_tls_cmd_complete(fdev, NULL);
+}
+
+static void mlx5_fpga_tls_cmd_send(struct mlx5_fpga_device *fdev,
+				   struct mlx5_fpga_tls_command_context *cmd,
+				   mlx5_fpga_tls_command_complete complete)
+{
+	struct mlx5_fpga_tls *tls = fdev->tls;
+	unsigned long flags;
+	int ret;
+
+	refcount_set(&cmd->ref, 2);
+	cmd->complete = complete;
+	cmd->buf.complete = mlx5_fpga_cmd_send_complete;
+
+	spin_lock_irqsave(&tls->pending_cmds_lock, flags);
+	/* mlx5_fpga_sbu_conn_sendmsg is called under pending_cmds_lock
+	 * to make sure commands are inserted to the tls->pending_cmds list
+	 * and the command QP in the same order.
+	 */
+	ret = mlx5_fpga_sbu_conn_sendmsg(tls->conn, &cmd->buf);
+	if (likely(!ret))
+		list_add_tail(&cmd->list, &tls->pending_cmds);
+	else
+		complete(tls->conn, fdev, cmd, NULL);
+	spin_unlock_irqrestore(&tls->pending_cmds_lock, flags);
+}
+
+/* Start of context identifiers range (inclusive) */
+#define SWID_START	0
+/* End of context identifiers range (exclusive) */
+#define SWID_END	BIT(24)
+
+static int mlx5_fpga_tls_alloc_swid(struct idr *idr, spinlock_t *idr_spinlock,
+				    void *ptr)
+{
+	int ret;
+
+	/* TLS metadata format is 1 byte for syndrome followed
+	 * by 3 bytes of swid (software ID)
+	 * swid must not exceed 3 bytes.
+	 * See tls_rxtx.c:insert_pet() for details
+	 */
+	BUILD_BUG_ON((SWID_END - 1) & 0xFF000000);
+
+	idr_preload(GFP_KERNEL);
+	spin_lock_irq(idr_spinlock);
+	ret = idr_alloc(idr, ptr, SWID_START, SWID_END, GFP_ATOMIC);
+	spin_unlock_irq(idr_spinlock);
+	idr_preload_end();
+
+	return ret;
+}
+
+static void mlx5_fpga_tls_release_swid(struct idr *idr,
+				       spinlock_t *idr_spinlock, u32 swid)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(idr_spinlock, flags);
+	idr_remove(idr, swid);
+	spin_unlock_irqrestore(idr_spinlock, flags);
+}
+
+struct mlx5_teardown_stream_context {
+	struct mlx5_fpga_tls_command_context cmd;
+	u32 swid;
+};
+
+static void
+mlx5_fpga_tls_teardown_completion(struct mlx5_fpga_conn *conn,
+				  struct mlx5_fpga_device *fdev,
+				  struct mlx5_fpga_tls_command_context *cmd,
+				  struct mlx5_fpga_dma_buf *resp)
+{
+	struct mlx5_teardown_stream_context *ctx =
+		    container_of(cmd, struct mlx5_teardown_stream_context, cmd);
+
+	if (resp) {
+		u32 syndrome = MLX5_GET(tls_resp, resp->sg[0].data, syndrome);
+
+		if (syndrome)
+			mlx5_fpga_err(fdev,
+				      "Teardown stream failed with syndrome = %d",
+				      syndrome);
+		else
+			mlx5_fpga_tls_release_swid(&fdev->tls->tx_idr,
+						   &fdev->tls->idr_spinlock,
+						   ctx->swid);
+	}
+	mlx5_fpga_tls_put_command_ctx(cmd);
+}
+
+static void mlx5_fpga_tls_flow_to_cmd(void *flow, void *cmd)
+{
+	memcpy(MLX5_ADDR_OF(tls_cmd, cmd, src_port), flow,
+	       MLX5_BYTE_OFF(tls_flow, ipv6));
+
+	MLX5_SET(tls_cmd, cmd, ipv6, MLX5_GET(tls_flow, flow, ipv6));
+	MLX5_SET(tls_cmd, cmd, direction_sx,
+		 MLX5_GET(tls_flow, flow, direction_sx));
+}
+
+void mlx5_fpga_tls_send_teardown_cmd(struct mlx5_core_dev *mdev, void *flow,
+				     u32 swid, gfp_t flags)
+{
+	struct mlx5_teardown_stream_context *ctx;
+	struct mlx5_fpga_dma_buf *buf;
+	void *cmd;
+
+	ctx = kzalloc(sizeof(*ctx) + MLX5_TLS_COMMAND_SIZE, flags);
+	if (!ctx)
+		return;
+
+	buf = &ctx->cmd.buf;
+	cmd = (ctx + 1);
+	MLX5_SET(tls_cmd, cmd, command_type, CMD_TEARDOWN_STREAM);
+	MLX5_SET(tls_cmd, cmd, swid, swid);
+
+	mlx5_fpga_tls_flow_to_cmd(flow, cmd);
+	kfree(flow);
+
+	buf->sg[0].data = cmd;
+	buf->sg[0].size = MLX5_TLS_COMMAND_SIZE;
+
+	ctx->swid = swid;
+	mlx5_fpga_tls_cmd_send(mdev->fpga, &ctx->cmd,
+			       mlx5_fpga_tls_teardown_completion);
+}
+
+void mlx5_fpga_tls_del_tx_flow(struct mlx5_core_dev *mdev, u32 swid,
+			       gfp_t flags)
+{
+	struct mlx5_fpga_tls *tls = mdev->fpga->tls;
+	void *flow;
+
+	rcu_read_lock();
+	flow = idr_find(&tls->tx_idr, swid);
+	rcu_read_unlock();
+
+	if (!flow) {
+		mlx5_fpga_err(mdev->fpga, "No flow information for swid %u\n",
+			      swid);
+		return;
+	}
+
+	mlx5_fpga_tls_send_teardown_cmd(mdev, flow, swid, flags);
+}
+
+enum mlx5_fpga_setup_stream_status {
+	MLX5_FPGA_CMD_PENDING,
+	MLX5_FPGA_CMD_SEND_FAILED,
+	MLX5_FPGA_CMD_RESPONSE_RECEIVED,
+	MLX5_FPGA_CMD_ABANDONED,
+};
+
+struct mlx5_setup_stream_context {
+	struct mlx5_fpga_tls_command_context cmd;
+	atomic_t status;
+	u32 syndrome;
+	struct completion comp;
+};
+
+static void
+mlx5_fpga_tls_setup_completion(struct mlx5_fpga_conn *conn,
+			       struct mlx5_fpga_device *fdev,
+			       struct mlx5_fpga_tls_command_context *cmd,
+			       struct mlx5_fpga_dma_buf *resp)
+{
+	struct mlx5_setup_stream_context *ctx =
+	    container_of(cmd, struct mlx5_setup_stream_context, cmd);
+	int status = MLX5_FPGA_CMD_SEND_FAILED;
+	void *tls_cmd = ctx + 1;
+
+	/* If we failed to send to command resp == NULL */
+	if (resp) {
+		ctx->syndrome = MLX5_GET(tls_resp, resp->sg[0].data, syndrome);
+		status = MLX5_FPGA_CMD_RESPONSE_RECEIVED;
+	}
+
+	status = atomic_xchg_release(&ctx->status, status);
+	if (likely(status != MLX5_FPGA_CMD_ABANDONED)) {
+		complete(&ctx->comp);
+		return;
+	}
+
+	mlx5_fpga_err(fdev, "Command was abandoned, syndrome = %u\n",
+		      ctx->syndrome);
+
+	if (!ctx->syndrome) {
+		/* The process was killed while waiting for the context to be
+		 * added, and the add completed successfully.
+		 * We need to destroy the HW context, and we can't can't reuse
+		 * the command context because we might not have received
+		 * the tx completion yet.
+		 */
+		mlx5_fpga_tls_del_tx_flow(fdev->mdev,
+					  MLX5_GET(tls_cmd, tls_cmd, swid),
+					  GFP_ATOMIC);
+	}
+
+	mlx5_fpga_tls_put_command_ctx(cmd);
+}
+
+static int mlx5_fpga_tls_setup_stream_cmd(struct mlx5_core_dev *mdev,
+					  struct mlx5_setup_stream_context *ctx)
+{
+	struct mlx5_fpga_dma_buf *buf;
+	void *cmd = ctx + 1;
+	int status, ret = 0;
+
+	buf = &ctx->cmd.buf;
+	buf->sg[0].data = cmd;
+	buf->sg[0].size = MLX5_TLS_COMMAND_SIZE;
+	MLX5_SET(tls_cmd, cmd, command_type, CMD_SETUP_STREAM);
+
+	init_completion(&ctx->comp);
+	atomic_set(&ctx->status, MLX5_FPGA_CMD_PENDING);
+	ctx->syndrome = -1;
+
+	mlx5_fpga_tls_cmd_send(mdev->fpga, &ctx->cmd,
+			       mlx5_fpga_tls_setup_completion);
+	wait_for_completion_killable(&ctx->comp);
+
+	status = atomic_xchg_acquire(&ctx->status, MLX5_FPGA_CMD_ABANDONED);
+	if (unlikely(status == MLX5_FPGA_CMD_PENDING))
+	/* ctx is going to be released in mlx5_fpga_tls_setup_completion */
+		return -EINTR;
+
+	if (unlikely(ctx->syndrome))
+		ret = -ENOMEM;
+
+	mlx5_fpga_tls_put_command_ctx(&ctx->cmd);
+	return ret;
+}
+
+static void mlx5_fpga_tls_hw_qp_recv_cb(void *cb_arg,
+					struct mlx5_fpga_dma_buf *buf)
+{
+	struct mlx5_fpga_device *fdev = (struct mlx5_fpga_device *)cb_arg;
+
+	mlx5_fpga_tls_cmd_complete(fdev, buf);
+}
+
+bool mlx5_fpga_is_tls_device(struct mlx5_core_dev *mdev)
+{
+	if (!mdev->fpga || !MLX5_CAP_GEN(mdev, fpga))
+		return false;
+
+	if (MLX5_CAP_FPGA(mdev, ieee_vendor_id) !=
+	    MLX5_FPGA_CAP_SANDBOX_VENDOR_ID_MLNX)
+		return false;
+
+	if (MLX5_CAP_FPGA(mdev, sandbox_product_id) !=
+	    MLX5_FPGA_CAP_SANDBOX_PRODUCT_ID_TLS)
+		return false;
+
+	if (MLX5_CAP_FPGA(mdev, sandbox_product_version) != 0)
+		return false;
+
+	return true;
+}
+
+static int mlx5_fpga_tls_get_caps(struct mlx5_fpga_device *fdev,
+				  u32 *p_caps)
+{
+	int err, cap_size = MLX5_ST_SZ_BYTES(tls_extended_cap);
+	u32 caps = 0;
+	void *buf;
+
+	buf = kzalloc(cap_size, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	err = mlx5_fpga_get_sbu_caps(fdev, cap_size, buf);
+	if (err)
+		goto out;
+
+	if (MLX5_GET(tls_extended_cap, buf, tx))
+		caps |= MLX5_ACCEL_TLS_TX;
+	if (MLX5_GET(tls_extended_cap, buf, rx))
+		caps |= MLX5_ACCEL_TLS_RX;
+	if (MLX5_GET(tls_extended_cap, buf, tls_v12))
+		caps |= MLX5_ACCEL_TLS_V12;
+	if (MLX5_GET(tls_extended_cap, buf, tls_v13))
+		caps |= MLX5_ACCEL_TLS_V13;
+	if (MLX5_GET(tls_extended_cap, buf, lro))
+		caps |= MLX5_ACCEL_TLS_LRO;
+	if (MLX5_GET(tls_extended_cap, buf, ipv6))
+		caps |= MLX5_ACCEL_TLS_IPV6;
+
+	if (MLX5_GET(tls_extended_cap, buf, aes_gcm_128))
+		caps |= MLX5_ACCEL_TLS_AES_GCM128;
+	if (MLX5_GET(tls_extended_cap, buf, aes_gcm_256))
+		caps |= MLX5_ACCEL_TLS_AES_GCM256;
+
+	*p_caps = caps;
+	err = 0;
+out:
+	kfree(buf);
+	return err;
+}
+
+int mlx5_fpga_tls_init(struct mlx5_core_dev *mdev)
+{
+	struct mlx5_fpga_device *fdev = mdev->fpga;
+	struct mlx5_fpga_conn_attr init_attr = {0};
+	struct mlx5_fpga_conn *conn;
+	struct mlx5_fpga_tls *tls;
+	int err = 0;
+
+	if (!mlx5_fpga_is_tls_device(mdev) || !fdev)
+		return 0;
+
+	tls = kzalloc(sizeof(*tls), GFP_KERNEL);
+	if (!tls)
+		return -ENOMEM;
+
+	err = mlx5_fpga_tls_get_caps(fdev, &tls->caps);
+	if (err)
+		goto error;
+
+	if (!(tls->caps & (MLX5_ACCEL_TLS_TX | MLX5_ACCEL_TLS_V12 |
+				 MLX5_ACCEL_TLS_AES_GCM128))) {
+		err = -ENOTSUPP;
+		goto error;
+	}
+
+	init_attr.rx_size = SBU_QP_QUEUE_SIZE;
+	init_attr.tx_size = SBU_QP_QUEUE_SIZE;
+	init_attr.recv_cb = mlx5_fpga_tls_hw_qp_recv_cb;
+	init_attr.cb_arg = fdev;
+	conn = mlx5_fpga_sbu_conn_create(fdev, &init_attr);
+	if (IS_ERR(conn)) {
+		err = PTR_ERR(conn);
+		mlx5_fpga_err(fdev, "Error creating TLS command connection %d\n",
+			      err);
+		goto error;
+	}
+
+	tls->conn = conn;
+	spin_lock_init(&tls->pending_cmds_lock);
+	INIT_LIST_HEAD(&tls->pending_cmds);
+
+	idr_init(&tls->tx_idr);
+	spin_lock_init(&tls->idr_spinlock);
+	fdev->tls = tls;
+	return 0;
+
+error:
+	kfree(tls);
+	return err;
+}
+
+void mlx5_fpga_tls_cleanup(struct mlx5_core_dev *mdev)
+{
+	struct mlx5_fpga_device *fdev = mdev->fpga;
+
+	if (!fdev || !fdev->tls)
+		return;
+
+	mlx5_fpga_sbu_conn_destroy(fdev->tls->conn);
+	kfree(fdev->tls);
+	fdev->tls = NULL;
+}
+
+static void mlx5_fpga_tls_set_aes_gcm128_ctx(void *cmd,
+					     struct tls_crypto_info *info,
+					     __be64 *rcd_sn)
+{
+	struct tls12_crypto_info_aes_gcm_128 *crypto_info =
+	    (struct tls12_crypto_info_aes_gcm_128 *)info;
+
+	memcpy(MLX5_ADDR_OF(tls_cmd, cmd, tls_rcd_sn), crypto_info->rec_seq,
+	       TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE);
+
+	memcpy(MLX5_ADDR_OF(tls_cmd, cmd, tls_implicit_iv),
+	       crypto_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
+	memcpy(MLX5_ADDR_OF(tls_cmd, cmd, encryption_key),
+	       crypto_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
+
+	/* in AES-GCM 128 we need to write the key twice */
+	memcpy(MLX5_ADDR_OF(tls_cmd, cmd, encryption_key) +
+		   TLS_CIPHER_AES_GCM_128_KEY_SIZE,
+	       crypto_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
+
+	MLX5_SET(tls_cmd, cmd, alg, MLX5_TLS_ALG_AES_GCM_128);
+}
+
+static int mlx5_fpga_tls_set_key_material(void *cmd, u32 caps,
+					  struct tls_crypto_info *crypto_info)
+{
+	__be64 rcd_sn;
+
+	switch (crypto_info->cipher_type) {
+	case TLS_CIPHER_AES_GCM_128:
+		if (!(caps & MLX5_ACCEL_TLS_AES_GCM128))
+			return -EINVAL;
+		mlx5_fpga_tls_set_aes_gcm128_ctx(cmd, crypto_info, &rcd_sn);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int mlx5_fpga_tls_add_flow(struct mlx5_core_dev *mdev, void *flow,
+				  struct tls_crypto_info *crypto_info, u32 swid,
+				  u32 tcp_sn)
+{
+	u32 caps = mlx5_fpga_tls_device_caps(mdev);
+	struct mlx5_setup_stream_context *ctx;
+	int ret = -ENOMEM;
+	size_t cmd_size;
+	void *cmd;
+
+	cmd_size = MLX5_TLS_COMMAND_SIZE + sizeof(*ctx);
+	ctx = kzalloc(cmd_size, GFP_KERNEL);
+	if (!ctx)
+		goto out;
+
+	cmd = ctx + 1;
+	ret = mlx5_fpga_tls_set_key_material(cmd, caps, crypto_info);
+	if (ret)
+		goto free_ctx;
+
+	mlx5_fpga_tls_flow_to_cmd(flow, cmd);
+
+	MLX5_SET(tls_cmd, cmd, swid, swid);
+	MLX5_SET(tls_cmd, cmd, tcp_sn, tcp_sn);
+
+	return mlx5_fpga_tls_setup_stream_cmd(mdev, ctx);
+
+free_ctx:
+	kfree(ctx);
+out:
+	return ret;
+}
+
+int mlx5_fpga_tls_add_tx_flow(struct mlx5_core_dev *mdev, void *flow,
+			      struct tls_crypto_info *crypto_info,
+			      u32 start_offload_tcp_sn, u32 *p_swid)
+{
+	struct mlx5_fpga_tls *tls = mdev->fpga->tls;
+	int ret = -ENOMEM;
+	u32 swid;
+
+	ret = mlx5_fpga_tls_alloc_swid(&tls->tx_idr, &tls->idr_spinlock, flow);
+	if (ret < 0)
+		return ret;
+
+	swid = ret;
+	MLX5_SET(tls_flow, flow, direction_sx, 1);
+
+	ret = mlx5_fpga_tls_add_flow(mdev, flow, crypto_info, swid,
+				     start_offload_tcp_sn);
+	if (ret && ret != -EINTR)
+		goto free_swid;
+
+	*p_swid = swid;
+	return 0;
+free_swid:
+	mlx5_fpga_tls_release_swid(&tls->tx_idr, &tls->idr_spinlock, swid);
+
+	return ret;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.h b/drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.h
new file mode 100644
index 000000000000..800a214e4e49
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef __MLX5_FPGA_TLS_H__
+#define __MLX5_FPGA_TLS_H__
+
+#include <linux/mlx5/driver.h>
+
+#include <net/tls.h>
+#include "fpga/core.h"
+
+struct mlx5_fpga_tls {
+	struct list_head pending_cmds;
+	spinlock_t pending_cmds_lock; /* Protects pending_cmds */
+	u32 caps;
+	struct mlx5_fpga_conn *conn;
+
+	struct idr tx_idr;
+	spinlock_t idr_spinlock; /* protects the IDR */
+};
+
+int mlx5_fpga_tls_add_tx_flow(struct mlx5_core_dev *mdev, void *flow,
+			      struct tls_crypto_info *crypto_info,
+			      u32 start_offload_tcp_sn, u32 *p_swid);
+
+void mlx5_fpga_tls_del_tx_flow(struct mlx5_core_dev *mdev, u32 swid,
+			       gfp_t flags);
+
+bool mlx5_fpga_is_tls_device(struct mlx5_core_dev *mdev);
+int mlx5_fpga_tls_init(struct mlx5_core_dev *mdev);
+void mlx5_fpga_tls_cleanup(struct mlx5_core_dev *mdev);
+
+static inline u32 mlx5_fpga_tls_device_caps(struct mlx5_core_dev *mdev)
+{
+	return mdev->fpga->tls->caps;
+}
+
+#endif /* __MLX5_FPGA_TLS_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
index ef5afd7c9325..5a00deff5457 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
@@ -372,6 +372,15 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
 			if (dst->dest_attr.type ==
 			    MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) {
 				id = dst->dest_attr.ft->id;
+			} else if (dst->dest_attr.type ==
+				   MLX5_FLOW_DESTINATION_TYPE_VPORT) {
+				id = dst->dest_attr.vport.num;
+				MLX5_SET(dest_format_struct, in_dests,
+					 destination_eswitch_owner_vhca_id_valid,
+					 dst->dest_attr.vport.vhca_id_valid);
+				MLX5_SET(dest_format_struct, in_dests,
+					 destination_eswitch_owner_vhca_id,
+					 dst->dest_attr.vport.vhca_id);
 			} else {
 				id = dst->dest_attr.tir_num;
 			}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
index c39c1692e674..362c0660dfab 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
@@ -325,7 +325,8 @@ static bool check_valid_mask(u8 match_criteria_enable, const u32 *match_criteria
 	if (match_criteria_enable & ~(
 		(1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS)   |
 		(1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) |
-		(1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS)))
+		(1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) |
+		(1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_2)))
 		return false;
 
 	if (!(match_criteria_enable &
@@ -361,6 +362,17 @@ static bool check_valid_mask(u8 match_criteria_enable, const u32 *match_criteria
 			return false;
 	}
 
+	if (!(match_criteria_enable &
+	      1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_2)) {
+		char *fg_type_mask = MLX5_ADDR_OF(fte_match_param,
+						  match_criteria, misc_parameters_2);
+
+		if (fg_type_mask[0] ||
+		    memcmp(fg_type_mask, fg_type_mask + 1,
+			   MLX5_ST_SZ_BYTES(fte_match_set_misc2) - 1))
+			return false;
+	}
+
 	return check_last_reserved(match_criteria);
 }
 
@@ -1374,6 +1386,8 @@ static int create_auto_flow_group(struct mlx5_flow_table *ft,
 	struct mlx5_core_dev *dev = get_dev(&ft->node);
 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 	void *match_criteria_addr;
+	u8 src_esw_owner_mask_on;
+	void *misc;
 	int err;
 	u32 *in;
 
@@ -1386,6 +1400,14 @@ static int create_auto_flow_group(struct mlx5_flow_table *ft,
 	MLX5_SET(create_flow_group_in, in, start_flow_index, fg->start_index);
 	MLX5_SET(create_flow_group_in, in, end_flow_index,   fg->start_index +
 		 fg->max_ftes - 1);
+
+	misc = MLX5_ADDR_OF(fte_match_param, fg->mask.match_criteria,
+			    misc_parameters);
+	src_esw_owner_mask_on = !!MLX5_GET(fte_match_set_misc, misc,
+					 source_eswitch_owner_vhca_id);
+	MLX5_SET(create_flow_group_in, in,
+		 source_eswitch_owner_vhca_id_valid, src_esw_owner_mask_on);
+
 	match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in,
 					   in, match_criteria);
 	memcpy(match_criteria_addr, fg->mask.match_criteria,
@@ -1406,7 +1428,7 @@ static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
 {
 	if (d1->type == d2->type) {
 		if ((d1->type == MLX5_FLOW_DESTINATION_TYPE_VPORT &&
-		     d1->vport_num == d2->vport_num) ||
+		     d1->vport.num == d2->vport.num) ||
 		    (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE &&
 		     d1->ft == d2->ft) ||
 		    (d1->type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
@@ -2485,7 +2507,7 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering)
 	if (!steering->fdb_root_ns)
 		return -ENOMEM;
 
-	prio = fs_create_prio(&steering->fdb_root_ns->ns, 0, 1);
+	prio = fs_create_prio(&steering->fdb_root_ns->ns, 0, 2);
 	if (IS_ERR(prio))
 		goto out_err;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
index e26d3e9d5f9f..b6da322a8016 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
@@ -159,7 +159,7 @@ struct mlx5_ft_underlay_qp {
 	u32 qpn;
 };
 
-#define MLX5_FTE_MATCH_PARAM_RESERVED	reserved_at_600
+#define MLX5_FTE_MATCH_PARAM_RESERVED	reserved_at_800
 /* Calculate the fte_match_param length and without the reserved length.
  * Make sure the reserved field is the last.
  */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h
index 6d9053bcbe95..08eac92fc26c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h
@@ -33,6 +33,8 @@
 #ifndef __MLX5E_IPOB_H__
 #define __MLX5E_IPOB_H__
 
+#ifdef CONFIG_MLX5_CORE_IPOIB
+
 #include <linux/mlx5/fs.h>
 #include "en.h"
 
@@ -93,8 +95,32 @@ const struct mlx5e_profile *mlx5i_pkey_get_profile(void);
 /* Extract mlx5e_priv from IPoIB netdev */
 #define mlx5i_epriv(netdev) ((void *)(((struct mlx5i_priv *)netdev_priv(netdev))->mlx5e_priv))
 
+struct mlx5_wqe_eth_pad {
+	u8 rsvd0[16];
+};
+
+struct mlx5i_tx_wqe {
+	struct mlx5_wqe_ctrl_seg     ctrl;
+	struct mlx5_wqe_datagram_seg datagram;
+	struct mlx5_wqe_eth_pad      pad;
+	struct mlx5_wqe_eth_seg      eth;
+	struct mlx5_wqe_data_seg     data[0];
+};
+
+static inline void mlx5i_sq_fetch_wqe(struct mlx5e_txqsq *sq,
+				      struct mlx5i_tx_wqe **wqe,
+				      u16 *pi)
+{
+	struct mlx5_wq_cyc *wq = &sq->wq;
+
+	*pi  = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
+	*wqe = mlx5_wq_cyc_get_wqe(wq, *pi);
+	memset(*wqe, 0, sizeof(**wqe));
+}
+
 netdev_tx_t mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
 			  struct mlx5_av *av, u32 dqpn, u32 dqkey);
 void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
 
+#endif /* CONFIG_MLX5_CORE_IPOIB */
 #endif /* __MLX5E_IPOB_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index e2c465b0b3f8..615005e63819 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -60,6 +60,7 @@
 #include "fpga/core.h"
 #include "fpga/ipsec.h"
 #include "accel/ipsec.h"
+#include "accel/tls.h"
 #include "lib/clock.h"
 
 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
@@ -1190,6 +1191,12 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 		goto err_ipsec_start;
 	}
 
+	err = mlx5_accel_tls_init(dev);
+	if (err) {
+		dev_err(&pdev->dev, "TLS device start failed %d\n", err);
+		goto err_tls_start;
+	}
+
 	err = mlx5_init_fs(dev);
 	if (err) {
 		dev_err(&pdev->dev, "Failed to init flow steering\n");
@@ -1231,6 +1238,9 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 	mlx5_cleanup_fs(dev);
 
 err_fs:
+	mlx5_accel_tls_cleanup(dev);
+
+err_tls_start:
 	mlx5_accel_ipsec_cleanup(dev);
 
 err_ipsec_start:
@@ -1306,6 +1316,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 	mlx5_sriov_detach(dev);
 	mlx5_cleanup_fs(dev);
 	mlx5_accel_ipsec_cleanup(dev);
+	mlx5_accel_tls_cleanup(dev);
 	mlx5_fpga_device_stop(dev);
 	mlx5_irq_clear_affinity_hints(dev);
 	free_comp_eqs(dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mr.c b/drivers/net/ethernet/mellanox/mlx5/core/mr.c
index b9736f505bdf..f4f02f775c93 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mr.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mr.c
@@ -123,8 +123,8 @@ int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev,
 	deleted_mkey = radix_tree_delete(&table->tree, mlx5_base_mkey(mkey->key));
 	write_unlock_irqrestore(&table->lock, flags);
 	if (!deleted_mkey) {
-		mlx5_core_warn(dev, "failed radix tree delete of mkey 0x%x\n",
-			       mlx5_base_mkey(mkey->key));
+		mlx5_core_dbg(dev, "failed radix tree delete of mkey 0x%x\n",
+			      mlx5_base_mkey(mkey->key));
 		return -ENOENT;
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/qp.c b/drivers/net/ethernet/mellanox/mlx5/core/qp.c
index 02d6c5b5d502..4ca07bfb6b14 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/qp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/qp.c
@@ -407,21 +407,21 @@ static int modify_qp_mbox_alloc(struct mlx5_core_dev *dev, u16 opcode, int qpn,
 	case MLX5_CMD_OP_RST2INIT_QP:
 		if (MBOX_ALLOC(mbox, rst2init_qp))
 			return -ENOMEM;
-		 MOD_QP_IN_SET_QPC(rst2init_qp, mbox->in, opcode, qpn,
-				   opt_param_mask, qpc);
-		 break;
+		MOD_QP_IN_SET_QPC(rst2init_qp, mbox->in, opcode, qpn,
+				  opt_param_mask, qpc);
+		break;
 	case MLX5_CMD_OP_INIT2RTR_QP:
 		if (MBOX_ALLOC(mbox, init2rtr_qp))
 			return -ENOMEM;
-		 MOD_QP_IN_SET_QPC(init2rtr_qp, mbox->in, opcode, qpn,
-				   opt_param_mask, qpc);
-		 break;
+		MOD_QP_IN_SET_QPC(init2rtr_qp, mbox->in, opcode, qpn,
+				  opt_param_mask, qpc);
+		break;
 	case MLX5_CMD_OP_RTR2RTS_QP:
 		if (MBOX_ALLOC(mbox, rtr2rts_qp))
 			return -ENOMEM;
-		 MOD_QP_IN_SET_QPC(rtr2rts_qp, mbox->in, opcode, qpn,
-				   opt_param_mask, qpc);
-		 break;
+		MOD_QP_IN_SET_QPC(rtr2rts_qp, mbox->in, opcode, qpn,
+				  opt_param_mask, qpc);
+		break;
 	case MLX5_CMD_OP_RTS2RTS_QP:
 		if (MBOX_ALLOC(mbox, rts2rts_qp))
 			return -ENOMEM;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c
index 177e076b8d17..719cecb182c6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c
@@ -511,7 +511,7 @@ int mlx5_query_nic_vport_system_image_guid(struct mlx5_core_dev *mdev,
 	*system_image_guid = MLX5_GET64(query_nic_vport_context_out, out,
 					nic_vport_context.system_image_guid);
 
-	kfree(out);
+	kvfree(out);
 
 	return 0;
 }
@@ -531,7 +531,7 @@ int mlx5_query_nic_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid)
 	*node_guid = MLX5_GET64(query_nic_vport_context_out, out,
 				nic_vport_context.node_guid);
 
-	kfree(out);
+	kvfree(out);
 
 	return 0;
 }
@@ -587,7 +587,7 @@ int mlx5_query_nic_vport_qkey_viol_cntr(struct mlx5_core_dev *mdev,
 	*qkey_viol_cntr = MLX5_GET(query_nic_vport_context_out, out,
 				   nic_vport_context.qkey_violation_counter);
 
-	kfree(out);
+	kvfree(out);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wq.c b/drivers/net/ethernet/mellanox/mlx5/core/wq.c
index ea66448ba365..5b8b35392025 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/wq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/wq.c
@@ -36,7 +36,12 @@
 
 u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq)
 {
-	return (u32)wq->sz_m1 + 1;
+	return (u32)wq->fbc.sz_m1 + 1;
+}
+
+u32 mlx5_wq_cyc_get_frag_size(struct mlx5_wq_cyc *wq)
+{
+	return (u32)wq->fbc.frag_sz_m1 + 1;
 }
 
 u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq)
@@ -46,12 +51,12 @@ u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq)
 
 u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq)
 {
-	return (u32)wq->sz_m1 + 1;
+	return (u32)wq->fbc.sz_m1 + 1;
 }
 
 static u32 mlx5_wq_cyc_get_byte_size(struct mlx5_wq_cyc *wq)
 {
-	return mlx5_wq_cyc_get_size(wq) << wq->log_stride;
+	return mlx5_wq_cyc_get_size(wq) << wq->fbc.log_stride;
 }
 
 static u32 mlx5_wq_qp_get_byte_size(struct mlx5_wq_qp *wq)
@@ -67,17 +72,19 @@ static u32 mlx5_cqwq_get_byte_size(struct mlx5_cqwq *wq)
 
 static u32 mlx5_wq_ll_get_byte_size(struct mlx5_wq_ll *wq)
 {
-	return mlx5_wq_ll_get_size(wq) << wq->log_stride;
+	return mlx5_wq_ll_get_size(wq) << wq->fbc.log_stride;
 }
 
 int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 		       void *wqc, struct mlx5_wq_cyc *wq,
 		       struct mlx5_wq_ctrl *wq_ctrl)
 {
+	struct mlx5_frag_buf_ctrl *fbc = &wq->fbc;
 	int err;
 
-	wq->log_stride = MLX5_GET(wq, wqc, log_wq_stride);
-	wq->sz_m1 = (1 << MLX5_GET(wq, wqc, log_wq_sz)) - 1;
+	mlx5_fill_fbc(MLX5_GET(wq, wqc, log_wq_stride),
+		      MLX5_GET(wq, wqc, log_wq_sz),
+		      fbc);
 
 	err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
 	if (err) {
@@ -85,14 +92,14 @@ int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 		return err;
 	}
 
-	err = mlx5_buf_alloc_node(mdev, mlx5_wq_cyc_get_byte_size(wq),
-				  &wq_ctrl->buf, param->buf_numa_node);
+	err = mlx5_frag_buf_alloc_node(mdev, mlx5_wq_cyc_get_byte_size(wq),
+				       &wq_ctrl->buf, param->buf_numa_node);
 	if (err) {
-		mlx5_core_warn(mdev, "mlx5_buf_alloc_node() failed, %d\n", err);
+		mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n", err);
 		goto err_db_free;
 	}
 
-	wq->buf = wq_ctrl->buf.frags->buf;
+	fbc->frag_buf = wq_ctrl->buf;
 	wq->db  = wq_ctrl->db.db;
 
 	wq_ctrl->mdev = mdev;
@@ -105,17 +112,35 @@ int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 	return err;
 }
 
+static void mlx5e_qp_set_frag_buf(struct mlx5_frag_buf *buf,
+				  struct mlx5_wq_qp *qp)
+{
+	struct mlx5_frag_buf *rqb, *sqb;
+
+	rqb = &qp->rq.fbc.frag_buf;
+	*rqb = *buf;
+	rqb->size   = mlx5_wq_cyc_get_byte_size(&qp->rq);
+	rqb->npages = 1 << get_order(rqb->size);
+
+	sqb = &qp->sq.fbc.frag_buf;
+	*sqb = *buf;
+	sqb->size   = mlx5_wq_cyc_get_byte_size(&qp->rq);
+	sqb->npages = 1 << get_order(sqb->size);
+	sqb->frags += rqb->npages; /* first part is for the rq */
+}
+
 int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 		      void *qpc, struct mlx5_wq_qp *wq,
 		      struct mlx5_wq_ctrl *wq_ctrl)
 {
 	int err;
 
-	wq->rq.log_stride = MLX5_GET(qpc, qpc, log_rq_stride) + 4;
-	wq->rq.sz_m1 = (1 << MLX5_GET(qpc, qpc, log_rq_size)) - 1;
-
-	wq->sq.log_stride = ilog2(MLX5_SEND_WQE_BB);
-	wq->sq.sz_m1 = (1 << MLX5_GET(qpc, qpc, log_sq_size)) - 1;
+	mlx5_fill_fbc(MLX5_GET(qpc, qpc, log_rq_stride) + 4,
+		      MLX5_GET(qpc, qpc, log_rq_size),
+		      &wq->rq.fbc);
+	mlx5_fill_fbc(ilog2(MLX5_SEND_WQE_BB),
+		      MLX5_GET(qpc, qpc, log_sq_size),
+		      &wq->sq.fbc);
 
 	err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
 	if (err) {
@@ -123,15 +148,15 @@ int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 		return err;
 	}
 
-	err = mlx5_buf_alloc_node(mdev, mlx5_wq_qp_get_byte_size(wq),
-				  &wq_ctrl->buf, param->buf_numa_node);
+	err = mlx5_frag_buf_alloc_node(mdev, mlx5_wq_qp_get_byte_size(wq),
+				       &wq_ctrl->buf, param->buf_numa_node);
 	if (err) {
-		mlx5_core_warn(mdev, "mlx5_buf_alloc_node() failed, %d\n", err);
+		mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n", err);
 		goto err_db_free;
 	}
 
-	wq->rq.buf = wq_ctrl->buf.frags->buf;
-	wq->sq.buf = wq->rq.buf + mlx5_wq_cyc_get_byte_size(&wq->rq);
+	mlx5e_qp_set_frag_buf(&wq_ctrl->buf, wq);
+
 	wq->rq.db  = &wq_ctrl->db.db[MLX5_RCV_DBR];
 	wq->sq.db  = &wq_ctrl->db.db[MLX5_SND_DBR];
 
@@ -147,7 +172,7 @@ int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 
 int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 		     void *cqc, struct mlx5_cqwq *wq,
-		     struct mlx5_frag_wq_ctrl *wq_ctrl)
+		     struct mlx5_wq_ctrl *wq_ctrl)
 {
 	int err;
 
@@ -160,7 +185,7 @@ int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 	}
 
 	err = mlx5_frag_buf_alloc_node(mdev, mlx5_cqwq_get_byte_size(wq),
-				       &wq_ctrl->frag_buf,
+				       &wq_ctrl->buf,
 				       param->buf_numa_node);
 	if (err) {
 		mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n",
@@ -168,7 +193,7 @@ int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 		goto err_db_free;
 	}
 
-	wq->fbc.frag_buf = wq_ctrl->frag_buf;
+	wq->fbc.frag_buf = wq_ctrl->buf;
 	wq->db  = wq_ctrl->db.db;
 
 	wq_ctrl->mdev = mdev;
@@ -185,12 +210,14 @@ int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 		      void *wqc, struct mlx5_wq_ll *wq,
 		      struct mlx5_wq_ctrl *wq_ctrl)
 {
+	struct mlx5_frag_buf_ctrl *fbc = &wq->fbc;
 	struct mlx5_wqe_srq_next_seg *next_seg;
 	int err;
 	int i;
 
-	wq->log_stride = MLX5_GET(wq, wqc, log_wq_stride);
-	wq->sz_m1 = (1 << MLX5_GET(wq, wqc, log_wq_sz)) - 1;
+	mlx5_fill_fbc(MLX5_GET(wq, wqc, log_wq_stride),
+		      MLX5_GET(wq, wqc, log_wq_sz),
+		      fbc);
 
 	err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
 	if (err) {
@@ -198,17 +225,17 @@ int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 		return err;
 	}
 
-	err = mlx5_buf_alloc_node(mdev, mlx5_wq_ll_get_byte_size(wq),
-				  &wq_ctrl->buf, param->buf_numa_node);
+	err = mlx5_frag_buf_alloc_node(mdev, mlx5_wq_ll_get_byte_size(wq),
+				       &wq_ctrl->buf, param->buf_numa_node);
 	if (err) {
-		mlx5_core_warn(mdev, "mlx5_buf_alloc_node() failed, %d\n", err);
+		mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n", err);
 		goto err_db_free;
 	}
 
-	wq->buf = wq_ctrl->buf.frags->buf;
+	wq->fbc.frag_buf = wq_ctrl->buf;
 	wq->db  = wq_ctrl->db.db;
 
-	for (i = 0; i < wq->sz_m1; i++) {
+	for (i = 0; i < fbc->sz_m1; i++) {
 		next_seg = mlx5_wq_ll_get_wqe(wq, i);
 		next_seg->next_wqe_index = cpu_to_be16(i + 1);
 	}
@@ -227,12 +254,7 @@ int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 
 void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl)
 {
-	mlx5_buf_free(wq_ctrl->mdev, &wq_ctrl->buf);
+	mlx5_frag_buf_free(wq_ctrl->mdev, &wq_ctrl->buf);
 	mlx5_db_free(wq_ctrl->mdev, &wq_ctrl->db);
 }
 
-void mlx5_cqwq_destroy(struct mlx5_frag_wq_ctrl *wq_ctrl)
-{
-	mlx5_frag_buf_free(wq_ctrl->mdev, &wq_ctrl->frag_buf);
-	mlx5_db_free(wq_ctrl->mdev, &wq_ctrl->db);
-}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wq.h b/drivers/net/ethernet/mellanox/mlx5/core/wq.h
index fca90b94596d..b9d7c01fc7cb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/wq.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/wq.h
@@ -38,7 +38,6 @@
 #include <linux/mlx5/qp.h>
 
 struct mlx5_wq_param {
-	int		linear;
 	int		buf_numa_node;
 	int		db_numa_node;
 };
@@ -49,17 +48,9 @@ struct mlx5_wq_ctrl {
 	struct mlx5_db		db;
 };
 
-struct mlx5_frag_wq_ctrl {
-	struct mlx5_core_dev	*mdev;
-	struct mlx5_frag_buf	frag_buf;
-	struct mlx5_db		db;
-};
-
 struct mlx5_wq_cyc {
-	void			*buf;
+	struct mlx5_frag_buf_ctrl fbc;
 	__be32			*db;
-	u16			sz_m1;
-	u8			log_stride;
 };
 
 struct mlx5_wq_qp {
@@ -74,20 +65,19 @@ struct mlx5_cqwq {
 };
 
 struct mlx5_wq_ll {
-	void			*buf;
+	struct mlx5_frag_buf_ctrl fbc;
 	__be32			*db;
 	__be16			*tail_next;
-	u16			sz_m1;
 	u16			head;
 	u16			wqe_ctr;
 	u16			cur_sz;
-	u8			log_stride;
 };
 
 int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 		       void *wqc, struct mlx5_wq_cyc *wq,
 		       struct mlx5_wq_ctrl *wq_ctrl);
 u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq);
+u32 mlx5_wq_cyc_get_frag_size(struct mlx5_wq_cyc *wq);
 
 int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 		      void *qpc, struct mlx5_wq_qp *wq,
@@ -95,7 +85,7 @@ int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 
 int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 		     void *cqc, struct mlx5_cqwq *wq,
-		     struct mlx5_frag_wq_ctrl *wq_ctrl);
+		     struct mlx5_wq_ctrl *wq_ctrl);
 u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq);
 
 int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
@@ -104,16 +94,20 @@ int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
 u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq);
 
 void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl);
-void mlx5_cqwq_destroy(struct mlx5_frag_wq_ctrl *wq_ctrl);
 
 static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr)
 {
-	return ctr & wq->sz_m1;
+	return ctr & wq->fbc.sz_m1;
+}
+
+static inline u16 mlx5_wq_cyc_ctr2fragix(struct mlx5_wq_cyc *wq, u16 ctr)
+{
+	return ctr & wq->fbc.frag_sz_m1;
 }
 
 static inline void *mlx5_wq_cyc_get_wqe(struct mlx5_wq_cyc *wq, u16 ix)
 {
-	return wq->buf + (ix << wq->log_stride);
+	return mlx5_frag_buf_get_wqe(&wq->fbc, ix);
 }
 
 static inline int mlx5_wq_cyc_cc_bigger(u16 cc1, u16 cc2)
@@ -124,9 +118,14 @@ static inline int mlx5_wq_cyc_cc_bigger(u16 cc1, u16 cc2)
 	return !equal && !smaller;
 }
 
+static inline u32 mlx5_cqwq_ctr2ix(struct mlx5_cqwq *wq, u32 ctr)
+{
+	return ctr & wq->fbc.sz_m1;
+}
+
 static inline u32 mlx5_cqwq_get_ci(struct mlx5_cqwq *wq)
 {
-	return wq->cc & wq->fbc.sz_m1;
+	return mlx5_cqwq_ctr2ix(wq, wq->cc);
 }
 
 static inline void *mlx5_cqwq_get_wqe(struct mlx5_cqwq *wq, u32 ix)
@@ -134,9 +133,14 @@ static inline void *mlx5_cqwq_get_wqe(struct mlx5_cqwq *wq, u32 ix)
 	return mlx5_frag_buf_get_wqe(&wq->fbc, ix);
 }
 
+static inline u32 mlx5_cqwq_get_ctr_wrap_cnt(struct mlx5_cqwq *wq, u32 ctr)
+{
+	return ctr >> wq->fbc.log_sz;
+}
+
 static inline u32 mlx5_cqwq_get_wrap_cnt(struct mlx5_cqwq *wq)
 {
-	return wq->cc >> wq->fbc.log_sz;
+	return mlx5_cqwq_get_ctr_wrap_cnt(wq, wq->cc);
 }
 
 static inline void mlx5_cqwq_pop(struct mlx5_cqwq *wq)
@@ -167,7 +171,7 @@ static inline struct mlx5_cqe64 *mlx5_cqwq_get_cqe(struct mlx5_cqwq *wq)
 
 static inline int mlx5_wq_ll_is_full(struct mlx5_wq_ll *wq)
 {
-	return wq->cur_sz == wq->sz_m1;
+	return wq->cur_sz == wq->fbc.sz_m1;
 }
 
 static inline int mlx5_wq_ll_is_empty(struct mlx5_wq_ll *wq)
@@ -175,9 +179,14 @@ static inline int mlx5_wq_ll_is_empty(struct mlx5_wq_ll *wq)
 	return !wq->cur_sz;
 }
 
+static inline u16 mlx5_wq_ll_ctr2ix(struct mlx5_wq_ll *wq, u16 ctr)
+{
+	return ctr & wq->fbc.sz_m1;
+}
+
 static inline void *mlx5_wq_ll_get_wqe(struct mlx5_wq_ll *wq, u16 ix)
 {
-	return wq->buf + (ix << wq->log_stride);
+	return mlx5_frag_buf_get_wqe(&wq->fbc, ix);
 }
 
 static inline void mlx5_wq_ll_push(struct mlx5_wq_ll *wq, u16 head_next)
diff --git a/drivers/net/ethernet/mellanox/mlxsw/cmd.h b/drivers/net/ethernet/mellanox/mlxsw/cmd.h
index 479511cf79bc..2bc48054b685 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/cmd.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/cmd.h
@@ -58,7 +58,7 @@ static inline void mlxsw_cmd_mbox_zero(char *mbox)
 struct mlxsw_core;
 
 int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod,
-		   u32 in_mod, bool out_mbox_direct,
+		   u32 in_mod, bool out_mbox_direct, bool reset_ok,
 		   char *in_mbox, size_t in_mbox_size,
 		   char *out_mbox, size_t out_mbox_size);
 
@@ -67,7 +67,7 @@ static inline int mlxsw_cmd_exec_in(struct mlxsw_core *mlxsw_core, u16 opcode,
 				    size_t in_mbox_size)
 {
 	return mlxsw_cmd_exec(mlxsw_core, opcode, opcode_mod, in_mod, false,
-			      in_mbox, in_mbox_size, NULL, 0);
+			      false, in_mbox, in_mbox_size, NULL, 0);
 }
 
 static inline int mlxsw_cmd_exec_out(struct mlxsw_core *mlxsw_core, u16 opcode,
@@ -76,7 +76,7 @@ static inline int mlxsw_cmd_exec_out(struct mlxsw_core *mlxsw_core, u16 opcode,
 				     char *out_mbox, size_t out_mbox_size)
 {
 	return mlxsw_cmd_exec(mlxsw_core, opcode, opcode_mod, in_mod,
-			      out_mbox_direct, NULL, 0,
+			      out_mbox_direct, false, NULL, 0,
 			      out_mbox, out_mbox_size);
 }
 
@@ -84,7 +84,7 @@ static inline int mlxsw_cmd_exec_none(struct mlxsw_core *mlxsw_core, u16 opcode,
 				      u8 opcode_mod, u32 in_mod)
 {
 	return mlxsw_cmd_exec(mlxsw_core, opcode, opcode_mod, in_mod, false,
-			      NULL, 0, NULL, 0);
+			      false, NULL, 0, NULL, 0);
 }
 
 enum mlxsw_cmd_opcode {
@@ -179,6 +179,8 @@ enum mlxsw_cmd_status {
 	MLXSW_CMD_STATUS_BAD_INDEX	= 0x0A,
 	/* NVMEM checksum/CRC failed. */
 	MLXSW_CMD_STATUS_BAD_NVMEM	= 0x0B,
+	/* Device is currently running reset */
+	MLXSW_CMD_STATUS_RUNNING_RESET	= 0x26,
 	/* Bad management packet (silently discarded). */
 	MLXSW_CMD_STATUS_BAD_PKT	= 0x30,
 };
@@ -208,6 +210,8 @@ static inline const char *mlxsw_cmd_status_str(u8 status)
 		return "BAD_INDEX";
 	case MLXSW_CMD_STATUS_BAD_NVMEM:
 		return "BAD_NVMEM";
+	case MLXSW_CMD_STATUS_RUNNING_RESET:
+		return "RUNNING_RESET";
 	case MLXSW_CMD_STATUS_BAD_PKT:
 		return "BAD_PKT";
 	default:
@@ -424,10 +428,15 @@ MLXSW_ITEM32(cmd_mbox, query_aq_cap, log_max_rdq_sz, 0x04, 24, 8);
 MLXSW_ITEM32(cmd_mbox, query_aq_cap, max_num_rdqs, 0x04, 0, 8);
 
 /* cmd_mbox_query_aq_cap_log_max_cq_sz
- * Log (base 2) of max CQEs allowed on CQ.
+ * Log (base 2) of the Maximum CQEs allowed in a CQ for CQEv0 and CQEv1.
  */
 MLXSW_ITEM32(cmd_mbox, query_aq_cap, log_max_cq_sz, 0x08, 24, 8);
 
+/* cmd_mbox_query_aq_cap_log_max_cqv2_sz
+ * Log (base 2) of the Maximum CQEs allowed in a CQ for CQEv2.
+ */
+MLXSW_ITEM32(cmd_mbox, query_aq_cap, log_max_cqv2_sz, 0x08, 16, 8);
+
 /* cmd_mbox_query_aq_cap_max_num_cqs
  * Maximum number of CQs.
  */
@@ -662,6 +671,12 @@ MLXSW_ITEM32(cmd_mbox, config_profile, set_kvd_hash_single_size, 0x0C, 25, 1);
  */
 MLXSW_ITEM32(cmd_mbox, config_profile, set_kvd_hash_double_size, 0x0C, 26, 1);
 
+/* cmd_mbox_config_set_cqe_version
+ * Capability bit. Setting a bit to 1 configures the profile
+ * according to the mailbox contents.
+ */
+MLXSW_ITEM32(cmd_mbox, config_profile, set_cqe_version, 0x08, 0, 1);
+
 /* cmd_mbox_config_profile_max_vepa_channels
  * Maximum number of VEPA channels per port (0 through 16)
  * 0 - multi-channel VEPA is disabled
@@ -841,6 +856,14 @@ MLXSW_ITEM32_INDEXED(cmd_mbox, config_profile, swid_config_type,
 MLXSW_ITEM32_INDEXED(cmd_mbox, config_profile, swid_config_properties,
 		     0x60, 0, 8, 0x08, 0x00, false);
 
+/* cmd_mbox_config_profile_cqe_version
+ * CQE version:
+ * 0: CQE version is 0
+ * 1: CQE version is either 1 or 2
+ * CQE ver 1 or 2 is configured by Completion Queue Context field cqe_ver.
+ */
+MLXSW_ITEM32(cmd_mbox, config_profile, cqe_version, 0xB0, 0, 8);
+
 /* ACCESS_REG - Access EMAD Supported Register
  * ----------------------------------
  * OpMod == 0 (N/A), INMmod == 0 (N/A)
@@ -850,10 +873,12 @@ MLXSW_ITEM32_INDEXED(cmd_mbox, config_profile, swid_config_properties,
  */
 
 static inline int mlxsw_cmd_access_reg(struct mlxsw_core *mlxsw_core,
+				       bool reset_ok,
 				       char *in_mbox, char *out_mbox)
 {
 	return mlxsw_cmd_exec(mlxsw_core, MLXSW_CMD_OPCODE_ACCESS_REG,
-			      0, 0, false, in_mbox, MLXSW_CMD_MBOX_SIZE,
+			      0, 0, false, reset_ok,
+			      in_mbox, MLXSW_CMD_MBOX_SIZE,
 			      out_mbox, MLXSW_CMD_MBOX_SIZE);
 }
 
@@ -1032,11 +1057,15 @@ static inline int mlxsw_cmd_sw2hw_cq(struct mlxsw_core *mlxsw_core,
 				 0, cq_number, in_mbox, MLXSW_CMD_MBOX_SIZE);
 }
 
-/* cmd_mbox_sw2hw_cq_cv
+enum mlxsw_cmd_mbox_sw2hw_cq_cqe_ver {
+	MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_1,
+	MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_2,
+};
+
+/* cmd_mbox_sw2hw_cq_cqe_ver
  * CQE Version.
- * 0 - CQE Version 0, 1 - CQE Version 1
  */
-MLXSW_ITEM32(cmd_mbox, sw2hw_cq, cv, 0x00, 28, 4);
+MLXSW_ITEM32(cmd_mbox, sw2hw_cq, cqe_ver, 0x00, 28, 4);
 
 /* cmd_mbox_sw2hw_cq_c_eqn
  * Event Queue this CQ reports completion events to.
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c
index e13ac3b8dff7..8a766fe28fa0 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.c
@@ -966,14 +966,12 @@ mlxsw_devlink_sb_occ_tc_port_bind_get(struct devlink_port *devlink_port,
 static int mlxsw_devlink_core_bus_device_reload(struct devlink *devlink)
 {
 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
-	const struct mlxsw_bus *mlxsw_bus = mlxsw_core->bus;
 	int err;
 
-	if (!mlxsw_bus->reset)
+	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_RESET))
 		return -EOPNOTSUPP;
 
 	mlxsw_core_bus_device_unregister(mlxsw_core, true);
-	mlxsw_bus->reset(mlxsw_core->bus_priv);
 	err = mlxsw_core_bus_device_register(mlxsw_core->bus_info,
 					     mlxsw_core->bus,
 					     mlxsw_core->bus_priv, true,
@@ -1480,6 +1478,7 @@ static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core,
 {
 	enum mlxsw_emad_op_tlv_status status;
 	int err, n_retry;
+	bool reset_ok;
 	char *in_mbox, *out_mbox, *tmp;
 
 	dev_dbg(mlxsw_core->bus_info->dev, "Reg cmd access (reg_id=%x(%s),type=%s)\n",
@@ -1501,9 +1500,16 @@ static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core,
 	tmp = in_mbox + MLXSW_EMAD_OP_TLV_LEN * sizeof(u32);
 	mlxsw_emad_pack_reg_tlv(tmp, reg, payload);
 
+	/* There is a special treatment needed for MRSR (reset) register.
+	 * The command interface will return error after the command
+	 * is executed, so tell the lower layer to expect it
+	 * and cope accordingly.
+	 */
+	reset_ok = reg->id == MLXSW_REG_MRSR_ID;
+
 	n_retry = 0;
 retry:
-	err = mlxsw_cmd_access_reg(mlxsw_core, in_mbox, out_mbox);
+	err = mlxsw_cmd_access_reg(mlxsw_core, reset_ok, in_mbox, out_mbox);
 	if (!err) {
 		err = mlxsw_emad_process_status(out_mbox, &status);
 		if (err) {
@@ -1714,15 +1720,16 @@ EXPORT_SYMBOL(mlxsw_core_port_fini);
 
 void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port,
 			     void *port_driver_priv, struct net_device *dev,
-			     bool split, u32 split_group)
+			     u32 port_number, bool split,
+			     u32 split_port_subnumber)
 {
 	struct mlxsw_core_port *mlxsw_core_port =
 					&mlxsw_core->ports[local_port];
 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
 
 	mlxsw_core_port->port_driver_priv = port_driver_priv;
-	if (split)
-		devlink_port_split_set(devlink_port, split_group);
+	devlink_port_attrs_set(devlink_port, DEVLINK_PORT_FLAVOUR_PHYSICAL,
+			       port_number, split, split_port_subnumber);
 	devlink_port_type_eth_set(devlink_port, dev);
 }
 EXPORT_SYMBOL(mlxsw_core_port_eth_set);
@@ -1762,6 +1769,17 @@ enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core,
 }
 EXPORT_SYMBOL(mlxsw_core_port_type_get);
 
+int mlxsw_core_port_get_phys_port_name(struct mlxsw_core *mlxsw_core,
+				       u8 local_port, char *name, size_t len)
+{
+	struct mlxsw_core_port *mlxsw_core_port =
+					&mlxsw_core->ports[local_port];
+	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
+
+	return devlink_port_get_phys_port_name(devlink_port, name, len);
+}
+EXPORT_SYMBOL(mlxsw_core_port_get_phys_port_name);
+
 static void mlxsw_core_buf_dump_dbg(struct mlxsw_core *mlxsw_core,
 				    const char *buf, size_t size)
 {
@@ -1781,7 +1799,7 @@ static void mlxsw_core_buf_dump_dbg(struct mlxsw_core *mlxsw_core,
 }
 
 int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod,
-		   u32 in_mod, bool out_mbox_direct,
+		   u32 in_mod, bool out_mbox_direct, bool reset_ok,
 		   char *in_mbox, size_t in_mbox_size,
 		   char *out_mbox, size_t out_mbox_size)
 {
@@ -1804,7 +1822,15 @@ int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod,
 					in_mbox, in_mbox_size,
 					out_mbox, out_mbox_size, &status);
 
-	if (err == -EIO && status != MLXSW_CMD_STATUS_OK) {
+	if (!err && out_mbox) {
+		dev_dbg(mlxsw_core->bus_info->dev, "Output mailbox:\n");
+		mlxsw_core_buf_dump_dbg(mlxsw_core, out_mbox, out_mbox_size);
+	}
+
+	if (reset_ok && err == -EIO &&
+	    status == MLXSW_CMD_STATUS_RUNNING_RESET) {
+		err = 0;
+	} else if (err == -EIO && status != MLXSW_CMD_STATUS_OK) {
 		dev_err(mlxsw_core->bus_info->dev, "Cmd exec failed (opcode=%x(%s),opcode_mod=%x,in_mod=%x,status=%x(%s))\n",
 			opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
 			in_mod, status, mlxsw_cmd_status_str(status));
@@ -1814,10 +1840,6 @@ int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod,
 			in_mod);
 	}
 
-	if (!err && out_mbox) {
-		dev_dbg(mlxsw_core->bus_info->dev, "Output mailbox:\n");
-		mlxsw_core_buf_dump_dbg(mlxsw_core, out_mbox, out_mbox_size);
-	}
 	return err;
 }
 EXPORT_SYMBOL(mlxsw_cmd_exec);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h
index 092d39399f3c..4a8d4c7f89d9 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.h
@@ -201,13 +201,16 @@ int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port);
 void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port);
 void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port,
 			     void *port_driver_priv, struct net_device *dev,
-			     bool split, u32 split_group);
+			     u32 port_number, bool split,
+			     u32 split_port_subnumber);
 void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u8 local_port,
 			    void *port_driver_priv);
 void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u8 local_port,
 			   void *port_driver_priv);
 enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core,
 						u8 local_port);
+int mlxsw_core_port_get_phys_port_name(struct mlxsw_core *mlxsw_core,
+				       u8 local_port, char *name, size_t len);
 
 int mlxsw_core_schedule_dw(struct delayed_work *dwork, unsigned long delay);
 bool mlxsw_core_schedule_work(struct work_struct *work);
@@ -334,6 +337,7 @@ u64 mlxsw_core_res_get(struct mlxsw_core *mlxsw_core,
 	mlxsw_core_res_get(mlxsw_core, MLXSW_RES_ID_##short_res_id)
 
 #define MLXSW_BUS_F_TXRX	BIT(0)
+#define MLXSW_BUS_F_RESET	BIT(1)
 
 struct mlxsw_bus {
 	const char *kind;
@@ -341,7 +345,6 @@ struct mlxsw_bus {
 		    const struct mlxsw_config_profile *profile,
 		    struct mlxsw_res *res);
 	void (*fini)(void *bus_priv);
-	void (*reset)(void *bus_priv);
 	bool (*skb_transmit_busy)(void *bus_priv,
 				  const struct mlxsw_tx_info *tx_info);
 	int (*skb_transmit)(void *bus_priv, struct sk_buff *skb,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c
index 3a9381977d6d..fc4557245ff4 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/pci.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c
@@ -117,6 +117,7 @@ struct mlxsw_pci_queue {
 		struct {
 			u32 comp_sdq_count;
 			u32 comp_rdq_count;
+			enum mlxsw_pci_cqe_v v;
 		} cq;
 		struct {
 			u32 ev_cmd_count;
@@ -155,6 +156,8 @@ struct mlxsw_pci {
 	} cmd;
 	struct mlxsw_bus_info bus_info;
 	const struct pci_device_id *id;
+	enum mlxsw_pci_cqe_v max_cqe_ver; /* Maximal supported CQE version */
+	u8 num_sdq_cqs; /* Number of CQs used for SDQs */
 };
 
 static void mlxsw_pci_queue_tasklet_schedule(struct mlxsw_pci_queue *q)
@@ -202,24 +205,6 @@ static bool mlxsw_pci_elem_hw_owned(struct mlxsw_pci_queue *q, bool owner_bit)
 	return owner_bit != !!(q->consumer_counter & q->count);
 }
 
-static char *
-mlxsw_pci_queue_sw_elem_get(struct mlxsw_pci_queue *q,
-			    u32 (*get_elem_owner_func)(const char *))
-{
-	struct mlxsw_pci_queue_elem_info *elem_info;
-	char *elem;
-	bool owner_bit;
-
-	elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
-	elem = elem_info->elem;
-	owner_bit = get_elem_owner_func(elem);
-	if (mlxsw_pci_elem_hw_owned(q, owner_bit))
-		return NULL;
-	q->consumer_counter++;
-	rmb(); /* make sure we read owned bit before the rest of elem */
-	return elem;
-}
-
 static struct mlxsw_pci_queue_type_group *
 mlxsw_pci_queue_type_group_get(struct mlxsw_pci *mlxsw_pci,
 			       enum mlxsw_pci_queue_type q_type)
@@ -494,6 +479,17 @@ static void mlxsw_pci_rdq_fini(struct mlxsw_pci *mlxsw_pci,
 	}
 }
 
+static void mlxsw_pci_cq_pre_init(struct mlxsw_pci *mlxsw_pci,
+				  struct mlxsw_pci_queue *q)
+{
+	q->u.cq.v = mlxsw_pci->max_cqe_ver;
+
+	/* For SDQ it is pointless to use CQEv2, so use CQEv1 instead */
+	if (q->u.cq.v == MLXSW_PCI_CQE_V2 &&
+	    q->num < mlxsw_pci->num_sdq_cqs)
+		q->u.cq.v = MLXSW_PCI_CQE_V1;
+}
+
 static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
 			     struct mlxsw_pci_queue *q)
 {
@@ -505,10 +501,16 @@ static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
 	for (i = 0; i < q->count; i++) {
 		char *elem = mlxsw_pci_queue_elem_get(q, i);
 
-		mlxsw_pci_cqe_owner_set(elem, 1);
+		mlxsw_pci_cqe_owner_set(q->u.cq.v, elem, 1);
 	}
 
-	mlxsw_cmd_mbox_sw2hw_cq_cv_set(mbox, 0); /* CQE ver 0 */
+	if (q->u.cq.v == MLXSW_PCI_CQE_V1)
+		mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox,
+				MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_1);
+	else if (q->u.cq.v == MLXSW_PCI_CQE_V2)
+		mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox,
+				MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_2);
+
 	mlxsw_cmd_mbox_sw2hw_cq_c_eqn_set(mbox, MLXSW_PCI_EQ_COMP_NUM);
 	mlxsw_cmd_mbox_sw2hw_cq_st_set(mbox, 0);
 	mlxsw_cmd_mbox_sw2hw_cq_log_cq_size_set(mbox, ilog2(q->count));
@@ -559,7 +561,7 @@ static void mlxsw_pci_cqe_sdq_handle(struct mlxsw_pci *mlxsw_pci,
 static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci,
 				     struct mlxsw_pci_queue *q,
 				     u16 consumer_counter_limit,
-				     char *cqe)
+				     enum mlxsw_pci_cqe_v cqe_v, char *cqe)
 {
 	struct pci_dev *pdev = mlxsw_pci->pdev;
 	struct mlxsw_pci_queue_elem_info *elem_info;
@@ -579,10 +581,11 @@ static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci,
 	if (q->consumer_counter++ != consumer_counter_limit)
 		dev_dbg_ratelimited(&pdev->dev, "Consumer counter does not match limit in RDQ\n");
 
-	if (mlxsw_pci_cqe_lag_get(cqe)) {
+	if (mlxsw_pci_cqe_lag_get(cqe_v, cqe)) {
 		rx_info.is_lag = true;
-		rx_info.u.lag_id = mlxsw_pci_cqe_lag_id_get(cqe);
-		rx_info.lag_port_index = mlxsw_pci_cqe_lag_port_index_get(cqe);
+		rx_info.u.lag_id = mlxsw_pci_cqe_lag_id_get(cqe_v, cqe);
+		rx_info.lag_port_index =
+			mlxsw_pci_cqe_lag_subport_get(cqe_v, cqe);
 	} else {
 		rx_info.is_lag = false;
 		rx_info.u.sys_port = mlxsw_pci_cqe_system_port_get(cqe);
@@ -591,7 +594,7 @@ static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci,
 	rx_info.trap_id = mlxsw_pci_cqe_trap_id_get(cqe);
 
 	byte_count = mlxsw_pci_cqe_byte_count_get(cqe);
-	if (mlxsw_pci_cqe_crc_get(cqe))
+	if (mlxsw_pci_cqe_crc_get(cqe_v, cqe))
 		byte_count -= ETH_FCS_LEN;
 	skb_put(skb, byte_count);
 	mlxsw_core_skb_receive(mlxsw_pci->core, skb, &rx_info);
@@ -608,7 +611,18 @@ static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci,
 
 static char *mlxsw_pci_cq_sw_cqe_get(struct mlxsw_pci_queue *q)
 {
-	return mlxsw_pci_queue_sw_elem_get(q, mlxsw_pci_cqe_owner_get);
+	struct mlxsw_pci_queue_elem_info *elem_info;
+	char *elem;
+	bool owner_bit;
+
+	elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
+	elem = elem_info->elem;
+	owner_bit = mlxsw_pci_cqe_owner_get(q->u.cq.v, elem);
+	if (mlxsw_pci_elem_hw_owned(q, owner_bit))
+		return NULL;
+	q->consumer_counter++;
+	rmb(); /* make sure we read owned bit before the rest of elem */
+	return elem;
 }
 
 static void mlxsw_pci_cq_tasklet(unsigned long data)
@@ -621,8 +635,8 @@ static void mlxsw_pci_cq_tasklet(unsigned long data)
 
 	while ((cqe = mlxsw_pci_cq_sw_cqe_get(q))) {
 		u16 wqe_counter = mlxsw_pci_cqe_wqe_counter_get(cqe);
-		u8 sendq = mlxsw_pci_cqe_sr_get(cqe);
-		u8 dqn = mlxsw_pci_cqe_dqn_get(cqe);
+		u8 sendq = mlxsw_pci_cqe_sr_get(q->u.cq.v, cqe);
+		u8 dqn = mlxsw_pci_cqe_dqn_get(q->u.cq.v, cqe);
 
 		if (sendq) {
 			struct mlxsw_pci_queue *sdq;
@@ -636,7 +650,7 @@ static void mlxsw_pci_cq_tasklet(unsigned long data)
 
 			rdq = mlxsw_pci_rdq_get(mlxsw_pci, dqn);
 			mlxsw_pci_cqe_rdq_handle(mlxsw_pci, rdq,
-						 wqe_counter, cqe);
+						 wqe_counter, q->u.cq.v, cqe);
 			q->u.cq.comp_rdq_count++;
 		}
 		if (++items == credits)
@@ -648,6 +662,18 @@ static void mlxsw_pci_cq_tasklet(unsigned long data)
 	}
 }
 
+static u16 mlxsw_pci_cq_elem_count(const struct mlxsw_pci_queue *q)
+{
+	return q->u.cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_COUNT :
+					       MLXSW_PCI_CQE01_COUNT;
+}
+
+static u8 mlxsw_pci_cq_elem_size(const struct mlxsw_pci_queue *q)
+{
+	return q->u.cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_SIZE :
+					       MLXSW_PCI_CQE01_SIZE;
+}
+
 static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
 			     struct mlxsw_pci_queue *q)
 {
@@ -696,7 +722,18 @@ static void mlxsw_pci_eq_cmd_event(struct mlxsw_pci *mlxsw_pci, char *eqe)
 
 static char *mlxsw_pci_eq_sw_eqe_get(struct mlxsw_pci_queue *q)
 {
-	return mlxsw_pci_queue_sw_elem_get(q, mlxsw_pci_eqe_owner_get);
+	struct mlxsw_pci_queue_elem_info *elem_info;
+	char *elem;
+	bool owner_bit;
+
+	elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
+	elem = elem_info->elem;
+	owner_bit = mlxsw_pci_eqe_owner_get(elem);
+	if (mlxsw_pci_elem_hw_owned(q, owner_bit))
+		return NULL;
+	q->consumer_counter++;
+	rmb(); /* make sure we read owned bit before the rest of elem */
+	return elem;
 }
 
 static void mlxsw_pci_eq_tasklet(unsigned long data)
@@ -749,11 +786,15 @@ static void mlxsw_pci_eq_tasklet(unsigned long data)
 struct mlxsw_pci_queue_ops {
 	const char *name;
 	enum mlxsw_pci_queue_type type;
+	void (*pre_init)(struct mlxsw_pci *mlxsw_pci,
+			 struct mlxsw_pci_queue *q);
 	int (*init)(struct mlxsw_pci *mlxsw_pci, char *mbox,
 		    struct mlxsw_pci_queue *q);
 	void (*fini)(struct mlxsw_pci *mlxsw_pci,
 		     struct mlxsw_pci_queue *q);
 	void (*tasklet)(unsigned long data);
+	u16 (*elem_count_f)(const struct mlxsw_pci_queue *q);
+	u8 (*elem_size_f)(const struct mlxsw_pci_queue *q);
 	u16 elem_count;
 	u8 elem_size;
 };
@@ -776,11 +817,12 @@ static const struct mlxsw_pci_queue_ops mlxsw_pci_rdq_ops = {
 
 static const struct mlxsw_pci_queue_ops mlxsw_pci_cq_ops = {
 	.type		= MLXSW_PCI_QUEUE_TYPE_CQ,
+	.pre_init	= mlxsw_pci_cq_pre_init,
 	.init		= mlxsw_pci_cq_init,
 	.fini		= mlxsw_pci_cq_fini,
 	.tasklet	= mlxsw_pci_cq_tasklet,
-	.elem_count	= MLXSW_PCI_CQE_COUNT,
-	.elem_size	= MLXSW_PCI_CQE_SIZE
+	.elem_count_f	= mlxsw_pci_cq_elem_count,
+	.elem_size_f	= mlxsw_pci_cq_elem_size
 };
 
 static const struct mlxsw_pci_queue_ops mlxsw_pci_eq_ops = {
@@ -800,10 +842,15 @@ static int mlxsw_pci_queue_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
 	int i;
 	int err;
 
-	spin_lock_init(&q->lock);
 	q->num = q_num;
-	q->count = q_ops->elem_count;
-	q->elem_size = q_ops->elem_size;
+	if (q_ops->pre_init)
+		q_ops->pre_init(mlxsw_pci, q);
+
+	spin_lock_init(&q->lock);
+	q->count = q_ops->elem_count_f ? q_ops->elem_count_f(q) :
+					 q_ops->elem_count;
+	q->elem_size = q_ops->elem_size_f ? q_ops->elem_size_f(q) :
+					    q_ops->elem_size;
 	q->type = q_ops->type;
 	q->pci = mlxsw_pci;
 
@@ -832,7 +879,7 @@ static int mlxsw_pci_queue_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
 
 		elem_info = mlxsw_pci_queue_elem_info_get(q, i);
 		elem_info->elem =
-			__mlxsw_pci_queue_elem_get(q, q_ops->elem_size, i);
+			__mlxsw_pci_queue_elem_get(q, q->elem_size, i);
 	}
 
 	mlxsw_cmd_mbox_zero(mbox);
@@ -912,6 +959,7 @@ static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox)
 	u8 rdq_log2sz;
 	u8 num_cqs;
 	u8 cq_log2sz;
+	u8 cqv2_log2sz;
 	u8 num_eqs;
 	u8 eq_log2sz;
 	int err;
@@ -927,6 +975,7 @@ static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox)
 	rdq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_rdq_sz_get(mbox);
 	num_cqs = mlxsw_cmd_mbox_query_aq_cap_max_num_cqs_get(mbox);
 	cq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_cq_sz_get(mbox);
+	cqv2_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_cqv2_sz_get(mbox);
 	num_eqs = mlxsw_cmd_mbox_query_aq_cap_max_num_eqs_get(mbox);
 	eq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_eq_sz_get(mbox);
 
@@ -938,12 +987,16 @@ static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox)
 
 	if ((1 << sdq_log2sz != MLXSW_PCI_WQE_COUNT) ||
 	    (1 << rdq_log2sz != MLXSW_PCI_WQE_COUNT) ||
-	    (1 << cq_log2sz != MLXSW_PCI_CQE_COUNT) ||
+	    (1 << cq_log2sz != MLXSW_PCI_CQE01_COUNT) ||
+	    (mlxsw_pci->max_cqe_ver == MLXSW_PCI_CQE_V2 &&
+	     (1 << cqv2_log2sz != MLXSW_PCI_CQE2_COUNT)) ||
 	    (1 << eq_log2sz != MLXSW_PCI_EQE_COUNT)) {
 		dev_err(&pdev->dev, "Unsupported number of async queue descriptors\n");
 		return -EINVAL;
 	}
 
+	mlxsw_pci->num_sdq_cqs = num_sdqs;
+
 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_eq_ops,
 					 num_eqs);
 	if (err) {
@@ -1184,6 +1237,11 @@ static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,
 		mlxsw_pci_config_profile_swid_config(mlxsw_pci, mbox, i,
 						     &profile->swid_config[i]);
 
+	if (mlxsw_pci->max_cqe_ver > MLXSW_PCI_CQE_V0) {
+		mlxsw_cmd_mbox_config_profile_set_cqe_version_set(mbox, 1);
+		mlxsw_cmd_mbox_config_profile_cqe_version_set(mbox, 1);
+	}
+
 	return mlxsw_cmd_config_profile_set(mlxsw_pci->core, mbox);
 }
 
@@ -1313,6 +1371,51 @@ static void mlxsw_pci_mbox_free(struct mlxsw_pci *mlxsw_pci,
 			    mbox->mapaddr);
 }
 
+static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci,
+			      const struct pci_device_id *id)
+{
+	unsigned long end;
+	char mrsr_pl[MLXSW_REG_MRSR_LEN];
+	int err;
+
+	mlxsw_reg_mrsr_pack(mrsr_pl);
+	err = mlxsw_reg_write(mlxsw_pci->core, MLXSW_REG(mrsr), mrsr_pl);
+	if (err)
+		return err;
+	if (id->device == PCI_DEVICE_ID_MELLANOX_SWITCHX2) {
+		msleep(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS);
+		return 0;
+	}
+
+	/* We must wait for the HW to become responsive once again. */
+	msleep(MLXSW_PCI_SW_RESET_WAIT_MSECS);
+
+	end = jiffies + msecs_to_jiffies(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS);
+	do {
+		u32 val = mlxsw_pci_read32(mlxsw_pci, FW_READY);
+
+		if ((val & MLXSW_PCI_FW_READY_MASK) == MLXSW_PCI_FW_READY_MAGIC)
+			break;
+		cond_resched();
+	} while (time_before(jiffies, end));
+	return 0;
+}
+
+static int mlxsw_pci_alloc_irq_vectors(struct mlxsw_pci *mlxsw_pci)
+{
+	int err;
+
+	err = pci_alloc_irq_vectors(mlxsw_pci->pdev, 1, 1, PCI_IRQ_MSIX);
+	if (err < 0)
+		dev_err(&mlxsw_pci->pdev->dev, "MSI-X init failed\n");
+	return err;
+}
+
+static void mlxsw_pci_free_irq_vectors(struct mlxsw_pci *mlxsw_pci)
+{
+	pci_free_irq_vectors(mlxsw_pci->pdev);
+}
+
 static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
 			  const struct mlxsw_config_profile *profile,
 			  struct mlxsw_res *res)
@@ -1340,6 +1443,16 @@ static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
 	if (err)
 		goto err_out_mbox_alloc;
 
+	err = mlxsw_pci_sw_reset(mlxsw_pci, mlxsw_pci->id);
+	if (err)
+		goto err_sw_reset;
+
+	err = mlxsw_pci_alloc_irq_vectors(mlxsw_pci);
+	if (err < 0) {
+		dev_err(&pdev->dev, "MSI-X init failed\n");
+		goto err_alloc_irq;
+	}
+
 	err = mlxsw_cmd_query_fw(mlxsw_core, mbox);
 	if (err)
 		goto err_query_fw;
@@ -1378,6 +1491,21 @@ static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
 	if (err)
 		goto err_query_resources;
 
+	if (MLXSW_CORE_RES_VALID(mlxsw_core, CQE_V2) &&
+	    MLXSW_CORE_RES_GET(mlxsw_core, CQE_V2))
+		mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V2;
+	else if (MLXSW_CORE_RES_VALID(mlxsw_core, CQE_V1) &&
+		 MLXSW_CORE_RES_GET(mlxsw_core, CQE_V1))
+		mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V1;
+	else if ((MLXSW_CORE_RES_VALID(mlxsw_core, CQE_V0) &&
+		  MLXSW_CORE_RES_GET(mlxsw_core, CQE_V0)) ||
+		 !MLXSW_CORE_RES_VALID(mlxsw_core, CQE_V0)) {
+		mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V0;
+	} else {
+		dev_err(&pdev->dev, "Invalid supported CQE version combination reported\n");
+		goto err_cqe_v_check;
+	}
+
 	err = mlxsw_pci_config_profile(mlxsw_pci, mbox, profile, res);
 	if (err)
 		goto err_config_profile;
@@ -1400,6 +1528,7 @@ static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
 	mlxsw_pci_aqs_fini(mlxsw_pci);
 err_aqs_init:
 err_config_profile:
+err_cqe_v_check:
 err_query_resources:
 err_boardinfo:
 	mlxsw_pci_fw_area_fini(mlxsw_pci);
@@ -1407,6 +1536,9 @@ static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
 err_doorbell_page_bar:
 err_iface_rev:
 err_query_fw:
+	mlxsw_pci_free_irq_vectors(mlxsw_pci);
+err_alloc_irq:
+err_sw_reset:
 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);
 err_out_mbox_alloc:
 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);
@@ -1422,6 +1554,7 @@ static void mlxsw_pci_fini(void *bus_priv)
 	free_irq(pci_irq_vector(mlxsw_pci->pdev, 0), mlxsw_pci);
 	mlxsw_pci_aqs_fini(mlxsw_pci);
 	mlxsw_pci_fw_area_fini(mlxsw_pci);
+	mlxsw_pci_free_irq_vectors(mlxsw_pci);
 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);
 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);
 }
@@ -1603,58 +1736,6 @@ static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod,
 	return err;
 }
 
-static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci,
-			      const struct pci_device_id *id)
-{
-	unsigned long end;
-
-	mlxsw_pci_write32(mlxsw_pci, SW_RESET, MLXSW_PCI_SW_RESET_RST_BIT);
-	if (id->device == PCI_DEVICE_ID_MELLANOX_SWITCHX2) {
-		msleep(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS);
-		return 0;
-	}
-
-	/* Reset needs to be written before we read control register, and
-	 * we must wait for the HW to become responsive once again
-	 */
-	wmb();
-	msleep(MLXSW_PCI_SW_RESET_WAIT_MSECS);
-
-	end = jiffies + msecs_to_jiffies(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS);
-	do {
-		u32 val = mlxsw_pci_read32(mlxsw_pci, FW_READY);
-
-		if ((val & MLXSW_PCI_FW_READY_MASK) == MLXSW_PCI_FW_READY_MAGIC)
-			break;
-		cond_resched();
-	} while (time_before(jiffies, end));
-	return 0;
-}
-
-static void mlxsw_pci_free_irq_vectors(struct mlxsw_pci *mlxsw_pci)
-{
-	pci_free_irq_vectors(mlxsw_pci->pdev);
-}
-
-static int mlxsw_pci_alloc_irq_vectors(struct mlxsw_pci *mlxsw_pci)
-{
-	int err;
-
-	err = pci_alloc_irq_vectors(mlxsw_pci->pdev, 1, 1, PCI_IRQ_MSIX);
-	if (err < 0)
-		dev_err(&mlxsw_pci->pdev->dev, "MSI-X init failed\n");
-	return err;
-}
-
-static void mlxsw_pci_reset(void *bus_priv)
-{
-	struct mlxsw_pci *mlxsw_pci = bus_priv;
-
-	mlxsw_pci_free_irq_vectors(mlxsw_pci);
-	mlxsw_pci_sw_reset(mlxsw_pci, mlxsw_pci->id);
-	mlxsw_pci_alloc_irq_vectors(mlxsw_pci);
-}
-
 static const struct mlxsw_bus mlxsw_pci_bus = {
 	.kind			= "pci",
 	.init			= mlxsw_pci_init,
@@ -1662,8 +1743,7 @@ static const struct mlxsw_bus mlxsw_pci_bus = {
 	.skb_transmit_busy	= mlxsw_pci_skb_transmit_busy,
 	.skb_transmit		= mlxsw_pci_skb_transmit,
 	.cmd_exec		= mlxsw_pci_cmd_exec,
-	.features		= MLXSW_BUS_F_TXRX,
-	.reset			= mlxsw_pci_reset,
+	.features		= MLXSW_BUS_F_TXRX | MLXSW_BUS_F_RESET,
 };
 
 static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
@@ -1721,18 +1801,6 @@ static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	mlxsw_pci->pdev = pdev;
 	pci_set_drvdata(pdev, mlxsw_pci);
 
-	err = mlxsw_pci_sw_reset(mlxsw_pci, id);
-	if (err) {
-		dev_err(&pdev->dev, "Software reset failed\n");
-		goto err_sw_reset;
-	}
-
-	err = mlxsw_pci_alloc_irq_vectors(mlxsw_pci);
-	if (err < 0) {
-		dev_err(&pdev->dev, "MSI-X init failed\n");
-		goto err_msix_init;
-	}
-
 	mlxsw_pci->bus_info.device_kind = driver_name;
 	mlxsw_pci->bus_info.device_name = pci_name(mlxsw_pci->pdev);
 	mlxsw_pci->bus_info.dev = &pdev->dev;
@@ -1749,9 +1817,6 @@ static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	return 0;
 
 err_bus_device_register:
-	mlxsw_pci_free_irq_vectors(mlxsw_pci);
-err_msix_init:
-err_sw_reset:
 	iounmap(mlxsw_pci->hw_addr);
 err_ioremap:
 err_pci_resource_len_check:
@@ -1769,7 +1834,6 @@ static void mlxsw_pci_remove(struct pci_dev *pdev)
 	struct mlxsw_pci *mlxsw_pci = pci_get_drvdata(pdev);
 
 	mlxsw_core_bus_device_unregister(mlxsw_pci->core, false);
-	mlxsw_pci_free_irq_vectors(mlxsw_pci);
 	iounmap(mlxsw_pci->hw_addr);
 	pci_release_regions(mlxsw_pci->pdev);
 	pci_disable_device(mlxsw_pci->pdev);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h
index fb082ad21b00..963155f6a17a 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h
@@ -82,10 +82,12 @@
 #define MLXSW_PCI_AQ_PAGES	8
 #define MLXSW_PCI_AQ_SIZE	(MLXSW_PCI_PAGE_SIZE * MLXSW_PCI_AQ_PAGES)
 #define MLXSW_PCI_WQE_SIZE	32 /* 32 bytes per element */
-#define MLXSW_PCI_CQE_SIZE	16 /* 16 bytes per element */
+#define MLXSW_PCI_CQE01_SIZE	16 /* 16 bytes per element */
+#define MLXSW_PCI_CQE2_SIZE	32 /* 32 bytes per element */
 #define MLXSW_PCI_EQE_SIZE	16 /* 16 bytes per element */
 #define MLXSW_PCI_WQE_COUNT	(MLXSW_PCI_AQ_SIZE / MLXSW_PCI_WQE_SIZE)
-#define MLXSW_PCI_CQE_COUNT	(MLXSW_PCI_AQ_SIZE / MLXSW_PCI_CQE_SIZE)
+#define MLXSW_PCI_CQE01_COUNT	(MLXSW_PCI_AQ_SIZE / MLXSW_PCI_CQE01_SIZE)
+#define MLXSW_PCI_CQE2_COUNT	(MLXSW_PCI_AQ_SIZE / MLXSW_PCI_CQE2_SIZE)
 #define MLXSW_PCI_EQE_COUNT	(MLXSW_PCI_AQ_SIZE / MLXSW_PCI_EQE_SIZE)
 #define MLXSW_PCI_EQE_UPDATE_COUNT	0x80
 
@@ -126,10 +128,48 @@ MLXSW_ITEM16_INDEXED(pci, wqe, byte_count, 0x02, 0, 14, 0x02, 0x00, false);
  */
 MLXSW_ITEM64_INDEXED(pci, wqe, address, 0x08, 0, 64, 0x8, 0x0, false);
 
+enum mlxsw_pci_cqe_v {
+	MLXSW_PCI_CQE_V0,
+	MLXSW_PCI_CQE_V1,
+	MLXSW_PCI_CQE_V2,
+};
+
+#define mlxsw_pci_cqe_item_helpers(name, v0, v1, v2)				\
+static inline u32 mlxsw_pci_cqe_##name##_get(enum mlxsw_pci_cqe_v v, char *cqe)	\
+{										\
+	switch (v) {								\
+	default:								\
+	case MLXSW_PCI_CQE_V0:							\
+		return mlxsw_pci_cqe##v0##_##name##_get(cqe);			\
+	case MLXSW_PCI_CQE_V1:							\
+		return mlxsw_pci_cqe##v1##_##name##_get(cqe);			\
+	case MLXSW_PCI_CQE_V2:							\
+		return mlxsw_pci_cqe##v2##_##name##_get(cqe);			\
+	}									\
+}										\
+static inline void mlxsw_pci_cqe_##name##_set(enum mlxsw_pci_cqe_v v,		\
+					      char *cqe, u32 val)		\
+{										\
+	switch (v) {								\
+	default:								\
+	case MLXSW_PCI_CQE_V0:							\
+		mlxsw_pci_cqe##v0##_##name##_set(cqe, val);			\
+		break;								\
+	case MLXSW_PCI_CQE_V1:							\
+		mlxsw_pci_cqe##v1##_##name##_set(cqe, val);			\
+		break;								\
+	case MLXSW_PCI_CQE_V2:							\
+		mlxsw_pci_cqe##v2##_##name##_set(cqe, val);			\
+		break;								\
+	}									\
+}
+
 /* pci_cqe_lag
  * Packet arrives from a port which is a LAG
  */
-MLXSW_ITEM32(pci, cqe, lag, 0x00, 23, 1);
+MLXSW_ITEM32(pci, cqe0, lag, 0x00, 23, 1);
+MLXSW_ITEM32(pci, cqe12, lag, 0x00, 24, 1);
+mlxsw_pci_cqe_item_helpers(lag, 0, 12, 12);
 
 /* pci_cqe_system_port/lag_id
  * When lag=0: System port on which the packet was received
@@ -138,8 +178,12 @@ MLXSW_ITEM32(pci, cqe, lag, 0x00, 23, 1);
  * bits [3:0] sub_port on which the packet was received
  */
 MLXSW_ITEM32(pci, cqe, system_port, 0x00, 0, 16);
-MLXSW_ITEM32(pci, cqe, lag_id, 0x00, 4, 12);
-MLXSW_ITEM32(pci, cqe, lag_port_index, 0x00, 0, 4);
+MLXSW_ITEM32(pci, cqe0, lag_id, 0x00, 4, 12);
+MLXSW_ITEM32(pci, cqe12, lag_id, 0x00, 0, 16);
+mlxsw_pci_cqe_item_helpers(lag_id, 0, 12, 12);
+MLXSW_ITEM32(pci, cqe0, lag_subport, 0x00, 0, 4);
+MLXSW_ITEM32(pci, cqe12, lag_subport, 0x00, 16, 8);
+mlxsw_pci_cqe_item_helpers(lag_subport, 0, 12, 12);
 
 /* pci_cqe_wqe_counter
  * WQE count of the WQEs completed on the associated dqn
@@ -162,28 +206,38 @@ MLXSW_ITEM32(pci, cqe, trap_id, 0x08, 0, 9);
  * Length include CRC. Indicates the length field includes
  * the packet's CRC.
  */
-MLXSW_ITEM32(pci, cqe, crc, 0x0C, 8, 1);
+MLXSW_ITEM32(pci, cqe0, crc, 0x0C, 8, 1);
+MLXSW_ITEM32(pci, cqe12, crc, 0x0C, 9, 1);
+mlxsw_pci_cqe_item_helpers(crc, 0, 12, 12);
 
 /* pci_cqe_e
  * CQE with Error.
  */
-MLXSW_ITEM32(pci, cqe, e, 0x0C, 7, 1);
+MLXSW_ITEM32(pci, cqe0, e, 0x0C, 7, 1);
+MLXSW_ITEM32(pci, cqe12, e, 0x00, 27, 1);
+mlxsw_pci_cqe_item_helpers(e, 0, 12, 12);
 
 /* pci_cqe_sr
  * 1 - Send Queue
  * 0 - Receive Queue
  */
-MLXSW_ITEM32(pci, cqe, sr, 0x0C, 6, 1);
+MLXSW_ITEM32(pci, cqe0, sr, 0x0C, 6, 1);
+MLXSW_ITEM32(pci, cqe12, sr, 0x00, 26, 1);
+mlxsw_pci_cqe_item_helpers(sr, 0, 12, 12);
 
 /* pci_cqe_dqn
  * Descriptor Queue (DQ) Number.
  */
-MLXSW_ITEM32(pci, cqe, dqn, 0x0C, 1, 5);
+MLXSW_ITEM32(pci, cqe0, dqn, 0x0C, 1, 5);
+MLXSW_ITEM32(pci, cqe12, dqn, 0x0C, 1, 6);
+mlxsw_pci_cqe_item_helpers(dqn, 0, 12, 12);
 
 /* pci_cqe_owner
  * Ownership bit.
  */
-MLXSW_ITEM32(pci, cqe, owner, 0x0C, 0, 1);
+MLXSW_ITEM32(pci, cqe01, owner, 0x0C, 0, 1);
+MLXSW_ITEM32(pci, cqe2, owner, 0x1C, 0, 1);
+mlxsw_pci_cqe_item_helpers(owner, 01, 01, 2);
 
 /* pci_eqe_event_type
  * Event type.
diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h
index 6218231e379e..1877d9f8a11a 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/reg.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h
@@ -6833,6 +6833,12 @@ enum mlxsw_reg_mpat_span_type {
 	 */
 	MLXSW_REG_MPAT_SPAN_TYPE_LOCAL_ETH = 0x0,
 
+	/* Remote SPAN Ethernet VLAN.
+	 * The packet is forwarded to the monitoring port on the monitoring
+	 * VLAN.
+	 */
+	MLXSW_REG_MPAT_SPAN_TYPE_REMOTE_ETH = 0x1,
+
 	/* Encapsulated Remote SPAN Ethernet L3 GRE.
 	 * The packet is encapsulated with GRE header.
 	 */
@@ -7028,6 +7034,30 @@ static inline void mlxsw_reg_mpar_pack(char *payload, u8 local_port,
 	mlxsw_reg_mpar_pa_id_set(payload, pa_id);
 }
 
+/* MRSR - Management Reset and Shutdown Register
+ * ---------------------------------------------
+ * MRSR register is used to reset or shutdown the switch or
+ * the entire system (when applicable).
+ */
+#define MLXSW_REG_MRSR_ID 0x9023
+#define MLXSW_REG_MRSR_LEN 0x08
+
+MLXSW_REG_DEFINE(mrsr, MLXSW_REG_MRSR_ID, MLXSW_REG_MRSR_LEN);
+
+/* reg_mrsr_command
+ * Reset/shutdown command
+ * 0 - do nothing
+ * 1 - software reset
+ * Access: WO
+ */
+MLXSW_ITEM32(reg, mrsr, command, 0x00, 0, 4);
+
+static inline void mlxsw_reg_mrsr_pack(char *payload)
+{
+	MLXSW_REG_ZERO(mrsr, payload);
+	mlxsw_reg_mrsr_command_set(payload, 1);
+}
+
 /* MLCR - Management LED Control Register
  * --------------------------------------
  * Controls the system LEDs.
@@ -7892,6 +7922,7 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = {
 	MLXSW_REG(mcia),
 	MLXSW_REG(mpat),
 	MLXSW_REG(mpar),
+	MLXSW_REG(mrsr),
 	MLXSW_REG(mlcr),
 	MLXSW_REG(mpsc),
 	MLXSW_REG(mcqi),
diff --git a/drivers/net/ethernet/mellanox/mlxsw/resources.h b/drivers/net/ethernet/mellanox/mlxsw/resources.h
index 087aad52c195..fd9299ccec72 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/resources.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/resources.h
@@ -43,6 +43,9 @@ enum mlxsw_res_id {
 	MLXSW_RES_ID_KVD_SINGLE_MIN_SIZE,
 	MLXSW_RES_ID_KVD_DOUBLE_MIN_SIZE,
 	MLXSW_RES_ID_MAX_TRAP_GROUPS,
+	MLXSW_RES_ID_CQE_V0,
+	MLXSW_RES_ID_CQE_V1,
+	MLXSW_RES_ID_CQE_V2,
 	MLXSW_RES_ID_COUNTER_POOL_SIZE,
 	MLXSW_RES_ID_MAX_SPAN,
 	MLXSW_RES_ID_COUNTER_SIZE_PACKETS_BYTES,
@@ -81,6 +84,9 @@ static u16 mlxsw_res_ids[] = {
 	[MLXSW_RES_ID_KVD_SINGLE_MIN_SIZE] = 0x1002,
 	[MLXSW_RES_ID_KVD_DOUBLE_MIN_SIZE] = 0x1003,
 	[MLXSW_RES_ID_MAX_TRAP_GROUPS] = 0x2201,
+	[MLXSW_RES_ID_CQE_V0] = 0x2210,
+	[MLXSW_RES_ID_CQE_V1] = 0x2211,
+	[MLXSW_RES_ID_CQE_V2] = 0x2212,
 	[MLXSW_RES_ID_COUNTER_POOL_SIZE] = 0x2410,
 	[MLXSW_RES_ID_MAX_SPAN] = 0x2420,
 	[MLXSW_RES_ID_COUNTER_SIZE_PACKETS_BYTES] = 0x2443,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
index ca38a30fbe91..fc39f22e5c70 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
@@ -441,29 +441,29 @@ static void mlxsw_sp_txhdr_construct(struct sk_buff *skb,
 	mlxsw_tx_hdr_type_set(txhdr, MLXSW_TXHDR_TYPE_CONTROL);
 }
 
-int mlxsw_sp_port_vid_stp_set(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid,
-			      u8 state)
+enum mlxsw_reg_spms_state mlxsw_sp_stp_spms_state(u8 state)
 {
-	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
-	enum mlxsw_reg_spms_state spms_state;
-	char *spms_pl;
-	int err;
-
 	switch (state) {
 	case BR_STATE_FORWARDING:
-		spms_state = MLXSW_REG_SPMS_STATE_FORWARDING;
-		break;
+		return MLXSW_REG_SPMS_STATE_FORWARDING;
 	case BR_STATE_LEARNING:
-		spms_state = MLXSW_REG_SPMS_STATE_LEARNING;
-		break;
+		return MLXSW_REG_SPMS_STATE_LEARNING;
 	case BR_STATE_LISTENING: /* fall-through */
 	case BR_STATE_DISABLED: /* fall-through */
 	case BR_STATE_BLOCKING:
-		spms_state = MLXSW_REG_SPMS_STATE_DISCARDING;
-		break;
+		return MLXSW_REG_SPMS_STATE_DISCARDING;
 	default:
 		BUG();
 	}
+}
+
+int mlxsw_sp_port_vid_stp_set(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid,
+			      u8 state)
+{
+	enum mlxsw_reg_spms_state spms_state = mlxsw_sp_stp_spms_state(state);
+	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
+	char *spms_pl;
+	int err;
 
 	spms_pl = kmalloc(MLXSW_REG_SPMS_LEN, GFP_KERNEL);
 	if (!spms_pl)
@@ -1238,21 +1238,10 @@ static int mlxsw_sp_port_get_phys_port_name(struct net_device *dev, char *name,
 					    size_t len)
 {
 	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
-	u8 module = mlxsw_sp_port->mapping.module;
-	u8 width = mlxsw_sp_port->mapping.width;
-	u8 lane = mlxsw_sp_port->mapping.lane;
-	int err;
-
-	if (!mlxsw_sp_port->split)
-		err = snprintf(name, len, "p%d", module + 1);
-	else
-		err = snprintf(name, len, "p%ds%d", module + 1,
-			       lane / width);
-
-	if (err >= len)
-		return -EINVAL;
 
-	return 0;
+	return mlxsw_core_port_get_phys_port_name(mlxsw_sp_port->mlxsw_sp->core,
+						  mlxsw_sp_port->local_port,
+						  name, len);
 }
 
 static struct mlxsw_sp_port_mall_tc_entry *
@@ -2927,8 +2916,8 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 	}
 
 	mlxsw_core_port_eth_set(mlxsw_sp->core, mlxsw_sp_port->local_port,
-				mlxsw_sp_port, dev, mlxsw_sp_port->split,
-				module);
+				mlxsw_sp_port, dev, module + 1,
+				mlxsw_sp_port->split, lane / width);
 	mlxsw_core_schedule_dw(&mlxsw_sp_port->periodic_hw_stats.update_dw, 0);
 	return 0;
 
@@ -3666,6 +3655,15 @@ static int mlxsw_sp_init(struct mlxsw_core *mlxsw_core,
 		goto err_lag_init;
 	}
 
+	/* Initialize SPAN before router and switchdev, so that those components
+	 * can call mlxsw_sp_span_respin().
+	 */
+	err = mlxsw_sp_span_init(mlxsw_sp);
+	if (err) {
+		dev_err(mlxsw_sp->bus_info->dev, "Failed to init span system\n");
+		goto err_span_init;
+	}
+
 	err = mlxsw_sp_switchdev_init(mlxsw_sp);
 	if (err) {
 		dev_err(mlxsw_sp->bus_info->dev, "Failed to initialize switchdev\n");
@@ -3684,15 +3682,6 @@ static int mlxsw_sp_init(struct mlxsw_core *mlxsw_core,
 		goto err_afa_init;
 	}
 
-	err = mlxsw_sp_span_init(mlxsw_sp);
-	if (err) {
-		dev_err(mlxsw_sp->bus_info->dev, "Failed to init span system\n");
-		goto err_span_init;
-	}
-
-	/* Initialize router after SPAN is initialized, so that the FIB and
-	 * neighbor event handlers can issue SPAN respin.
-	 */
 	err = mlxsw_sp_router_init(mlxsw_sp);
 	if (err) {
 		dev_err(mlxsw_sp->bus_info->dev, "Failed to initialize router\n");
@@ -3739,14 +3728,14 @@ static int mlxsw_sp_init(struct mlxsw_core *mlxsw_core,
 err_netdev_notifier:
 	mlxsw_sp_router_fini(mlxsw_sp);
 err_router_init:
-	mlxsw_sp_span_fini(mlxsw_sp);
-err_span_init:
 	mlxsw_sp_afa_fini(mlxsw_sp);
 err_afa_init:
 	mlxsw_sp_counter_pool_fini(mlxsw_sp);
 err_counter_pool_init:
 	mlxsw_sp_switchdev_fini(mlxsw_sp);
 err_switchdev_init:
+	mlxsw_sp_span_fini(mlxsw_sp);
+err_span_init:
 	mlxsw_sp_lag_fini(mlxsw_sp);
 err_lag_init:
 	mlxsw_sp_buffers_fini(mlxsw_sp);
@@ -3768,10 +3757,10 @@ static void mlxsw_sp_fini(struct mlxsw_core *mlxsw_core)
 	mlxsw_sp_acl_fini(mlxsw_sp);
 	unregister_netdevice_notifier(&mlxsw_sp->netdevice_nb);
 	mlxsw_sp_router_fini(mlxsw_sp);
-	mlxsw_sp_span_fini(mlxsw_sp);
 	mlxsw_sp_afa_fini(mlxsw_sp);
 	mlxsw_sp_counter_pool_fini(mlxsw_sp);
 	mlxsw_sp_switchdev_fini(mlxsw_sp);
+	mlxsw_sp_span_fini(mlxsw_sp);
 	mlxsw_sp_lag_fini(mlxsw_sp);
 	mlxsw_sp_buffers_fini(mlxsw_sp);
 	mlxsw_sp_traps_fini(mlxsw_sp);
@@ -4433,6 +4422,11 @@ static int mlxsw_sp_netdevice_port_upper_event(struct net_device *lower_dev,
 			NL_SET_ERR_MSG_MOD(extack, "Can not put a VLAN on an OVS port");
 			return -EINVAL;
 		}
+		if (is_vlan_dev(upper_dev) &&
+		    vlan_dev_vlan_id(upper_dev) == 1) {
+			NL_SET_ERR_MSG_MOD(extack, "Creating a VLAN device with VID 1 is unsupported: VLAN 1 carries untagged traffic");
+			return -EINVAL;
+		}
 		break;
 	case NETDEV_CHANGEUPPER:
 		upper_dev = info->upper_dev;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
index 804d4d2c8031..4a519d8edec8 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
@@ -364,6 +364,7 @@ int __mlxsw_sp_port_headroom_set(struct mlxsw_sp_port *mlxsw_sp_port, int mtu,
 int mlxsw_sp_port_ets_maxrate_set(struct mlxsw_sp_port *mlxsw_sp_port,
 				  enum mlxsw_reg_qeec_hr hr, u8 index,
 				  u8 next_index, u32 maxrate);
+enum mlxsw_reg_spms_state mlxsw_sp_stp_spms_state(u8 stp_state);
 int mlxsw_sp_port_vid_stp_set(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid,
 			      u8 state);
 int mlxsw_sp_port_vp_mode_set(struct mlxsw_sp_port *mlxsw_sp_port, bool enable);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
index 1904c0323d39..77b2adb29341 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
@@ -442,7 +442,7 @@ struct mlxsw_sp_fib6_entry {
 
 struct mlxsw_sp_rt6 {
 	struct list_head list;
-	struct rt6_info *rt;
+	struct fib6_info *rt;
 };
 
 struct mlxsw_sp_lpm_tree {
@@ -2770,9 +2770,9 @@ mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
 		struct in6_addr *gw;
 		int ifindex, weight;
 
-		ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
-		weight = mlxsw_sp_rt6->rt->rt6i_nh_weight;
-		gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
+		ifindex = mlxsw_sp_rt6->rt->fib6_nh.nh_dev->ifindex;
+		weight = mlxsw_sp_rt6->rt->fib6_nh.nh_weight;
+		gw = &mlxsw_sp_rt6->rt->fib6_nh.nh_gw;
 		if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
 							 weight))
 			return false;
@@ -2838,7 +2838,7 @@ mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
 	struct net_device *dev;
 
 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
-		dev = mlxsw_sp_rt6->rt->dst.dev;
+		dev = mlxsw_sp_rt6->rt->fib6_nh.nh_dev;
 		val ^= dev->ifindex;
 	}
 
@@ -3834,11 +3834,11 @@ mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
 
 	for (i = 0; i < nh_grp->count; i++) {
 		struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
-		struct rt6_info *rt = mlxsw_sp_rt6->rt;
+		struct fib6_info *rt = mlxsw_sp_rt6->rt;
 
-		if (nh->rif && nh->rif->dev == rt->dst.dev &&
+		if (nh->rif && nh->rif->dev == rt->fib6_nh.nh_dev &&
 		    ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
-				    &rt->rt6i_gateway))
+				    &rt->fib6_nh.nh_gw))
 			return nh;
 		continue;
 	}
@@ -3895,7 +3895,7 @@ mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
 
 	if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
 		list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
-				 list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
+				 list)->rt->fib6_nh.nh_flags |= RTNH_F_OFFLOAD;
 		return;
 	}
 
@@ -3905,9 +3905,9 @@ mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
 
 		nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
 		if (nh && nh->offloaded)
-			mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
+			mlxsw_sp_rt6->rt->fib6_nh.nh_flags |= RTNH_F_OFFLOAD;
 		else
-			mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
+			mlxsw_sp_rt6->rt->fib6_nh.nh_flags &= ~RTNH_F_OFFLOAD;
 	}
 }
 
@@ -3920,9 +3920,9 @@ mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
 	fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
 				  common);
 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
-		struct rt6_info *rt = mlxsw_sp_rt6->rt;
+		struct fib6_info *rt = mlxsw_sp_rt6->rt;
 
-		rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
+		rt->fib6_nh.nh_flags &= ~RTNH_F_OFFLOAD;
 	}
 }
 
@@ -4699,29 +4699,29 @@ static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
 }
 
-static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
+static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
 {
 	/* Packets with link-local destination IP arriving to the router
 	 * are trapped to the CPU, so no need to program specific routes
 	 * for them.
 	 */
-	if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
+	if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_LINKLOCAL)
 		return true;
 
 	/* Multicast routes aren't supported, so ignore them. Neighbour
 	 * Discovery packets are specifically trapped.
 	 */
-	if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
+	if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
 		return true;
 
 	/* Cloned routes are irrelevant in the forwarding path. */
-	if (rt->rt6i_flags & RTF_CACHE)
+	if (rt->fib6_flags & RTF_CACHE)
 		return true;
 
 	return false;
 }
 
-static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
+static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
 {
 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
 
@@ -4734,18 +4734,18 @@ static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
 	 * memory.
 	 */
 	mlxsw_sp_rt6->rt = rt;
-	rt6_hold(rt);
+	fib6_info_hold(rt);
 
 	return mlxsw_sp_rt6;
 }
 
 #if IS_ENABLED(CONFIG_IPV6)
-static void mlxsw_sp_rt6_release(struct rt6_info *rt)
+static void mlxsw_sp_rt6_release(struct fib6_info *rt)
 {
-	rt6_release(rt);
+	fib6_info_release(rt);
 }
 #else
-static void mlxsw_sp_rt6_release(struct rt6_info *rt)
+static void mlxsw_sp_rt6_release(struct fib6_info *rt)
 {
 }
 #endif
@@ -4756,13 +4756,13 @@ static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
 	kfree(mlxsw_sp_rt6);
 }
 
-static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
+static bool mlxsw_sp_fib6_rt_can_mp(const struct fib6_info *rt)
 {
 	/* RTF_CACHE routes are ignored */
-	return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
+	return (rt->fib6_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
 }
 
-static struct rt6_info *
+static struct fib6_info *
 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
 {
 	return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
@@ -4771,7 +4771,7 @@ mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
 
 static struct mlxsw_sp_fib6_entry *
 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
-				 const struct rt6_info *nrt, bool replace)
+				 const struct fib6_info *nrt, bool replace)
 {
 	struct mlxsw_sp_fib6_entry *fib6_entry;
 
@@ -4779,21 +4779,21 @@ mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
 		return NULL;
 
 	list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
-		struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
+		struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
 
 		/* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
 		 * virtual router.
 		 */
-		if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
+		if (rt->fib6_table->tb6_id > nrt->fib6_table->tb6_id)
 			continue;
-		if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
+		if (rt->fib6_table->tb6_id != nrt->fib6_table->tb6_id)
 			break;
-		if (rt->rt6i_metric < nrt->rt6i_metric)
+		if (rt->fib6_metric < nrt->fib6_metric)
 			continue;
-		if (rt->rt6i_metric == nrt->rt6i_metric &&
+		if (rt->fib6_metric == nrt->fib6_metric &&
 		    mlxsw_sp_fib6_rt_can_mp(rt))
 			return fib6_entry;
-		if (rt->rt6i_metric > nrt->rt6i_metric)
+		if (rt->fib6_metric > nrt->fib6_metric)
 			break;
 	}
 
@@ -4802,7 +4802,7 @@ mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
 
 static struct mlxsw_sp_rt6 *
 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
-			    const struct rt6_info *rt)
+			    const struct fib6_info *rt)
 {
 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
 
@@ -4815,21 +4815,21 @@ mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
 }
 
 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
-					const struct rt6_info *rt,
+					const struct fib6_info *rt,
 					enum mlxsw_sp_ipip_type *ret)
 {
-	return rt->dst.dev &&
-	       mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->dst.dev, ret);
+	return rt->fib6_nh.nh_dev &&
+	       mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh.nh_dev, ret);
 }
 
 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp,
 				       struct mlxsw_sp_nexthop_group *nh_grp,
 				       struct mlxsw_sp_nexthop *nh,
-				       const struct rt6_info *rt)
+				       const struct fib6_info *rt)
 {
 	const struct mlxsw_sp_ipip_ops *ipip_ops;
 	struct mlxsw_sp_ipip_entry *ipip_entry;
-	struct net_device *dev = rt->dst.dev;
+	struct net_device *dev = rt->fib6_nh.nh_dev;
 	struct mlxsw_sp_rif *rif;
 	int err;
 
@@ -4870,13 +4870,13 @@ static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp,
 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
 				  struct mlxsw_sp_nexthop_group *nh_grp,
 				  struct mlxsw_sp_nexthop *nh,
-				  const struct rt6_info *rt)
+				  const struct fib6_info *rt)
 {
-	struct net_device *dev = rt->dst.dev;
+	struct net_device *dev = rt->fib6_nh.nh_dev;
 
 	nh->nh_grp = nh_grp;
-	nh->nh_weight = rt->rt6i_nh_weight;
-	memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
+	nh->nh_weight = rt->fib6_nh.nh_weight;
+	memcpy(&nh->gw_addr, &rt->fib6_nh.nh_gw, sizeof(nh->gw_addr));
 	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
 
 	list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
@@ -4897,9 +4897,9 @@ static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
 }
 
 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
-				    const struct rt6_info *rt)
+				    const struct fib6_info *rt)
 {
-	return rt->rt6i_flags & RTF_GATEWAY ||
+	return rt->fib6_flags & RTF_GATEWAY ||
 	       mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
 }
 
@@ -4928,7 +4928,7 @@ mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
 	nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
 	nh_grp->count = fib6_entry->nrt6;
 	for (i = 0; i < nh_grp->count; i++) {
-		struct rt6_info *rt = mlxsw_sp_rt6->rt;
+		struct fib6_info *rt = mlxsw_sp_rt6->rt;
 
 		nh = &nh_grp->nexthops[i];
 		err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
@@ -5040,7 +5040,7 @@ mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
 static int
 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
 				struct mlxsw_sp_fib6_entry *fib6_entry,
-				struct rt6_info *rt)
+				struct fib6_info *rt)
 {
 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
 	int err;
@@ -5068,7 +5068,7 @@ mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
 static void
 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
 				struct mlxsw_sp_fib6_entry *fib6_entry,
-				struct rt6_info *rt)
+				struct fib6_info *rt)
 {
 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
 
@@ -5084,7 +5084,7 @@ mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
 
 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
 					 struct mlxsw_sp_fib_entry *fib_entry,
-					 const struct rt6_info *rt)
+					 const struct fib6_info *rt)
 {
 	/* Packets hitting RTF_REJECT routes need to be discarded by the
 	 * stack. We can rely on their destination device not having a
@@ -5092,9 +5092,9 @@ static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
 	 * local, which will cause them to be trapped with a lower
 	 * priority than packets that need to be locally received.
 	 */
-	if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST))
+	if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
-	else if (rt->rt6i_flags & RTF_REJECT)
+	else if (rt->fib6_flags & RTF_REJECT)
 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
 	else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
@@ -5118,7 +5118,7 @@ mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
 static struct mlxsw_sp_fib6_entry *
 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
 			   struct mlxsw_sp_fib_node *fib_node,
-			   struct rt6_info *rt)
+			   struct fib6_info *rt)
 {
 	struct mlxsw_sp_fib6_entry *fib6_entry;
 	struct mlxsw_sp_fib_entry *fib_entry;
@@ -5168,25 +5168,25 @@ static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 
 static struct mlxsw_sp_fib6_entry *
 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
-			      const struct rt6_info *nrt, bool replace)
+			      const struct fib6_info *nrt, bool replace)
 {
 	struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
 
 	list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
-		struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
+		struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
 
-		if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
+		if (rt->fib6_table->tb6_id > nrt->fib6_table->tb6_id)
 			continue;
-		if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
+		if (rt->fib6_table->tb6_id != nrt->fib6_table->tb6_id)
 			break;
-		if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
+		if (replace && rt->fib6_metric == nrt->fib6_metric) {
 			if (mlxsw_sp_fib6_rt_can_mp(rt) ==
 			    mlxsw_sp_fib6_rt_can_mp(nrt))
 				return fib6_entry;
 			if (mlxsw_sp_fib6_rt_can_mp(nrt))
 				fallback = fallback ?: fib6_entry;
 		}
-		if (rt->rt6i_metric > nrt->rt6i_metric)
+		if (rt->fib6_metric > nrt->fib6_metric)
 			return fallback ?: fib6_entry;
 	}
 
@@ -5198,7 +5198,7 @@ mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
 			       bool replace)
 {
 	struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
-	struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
+	struct fib6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
 	struct mlxsw_sp_fib6_entry *fib6_entry;
 
 	fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
@@ -5213,9 +5213,9 @@ mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
 		struct mlxsw_sp_fib6_entry *last;
 
 		list_for_each_entry(last, &fib_node->entry_list, common.list) {
-			struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
+			struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(last);
 
-			if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
+			if (nrt->fib6_table->tb6_id > rt->fib6_table->tb6_id)
 				break;
 			fib6_entry = last;
 		}
@@ -5268,29 +5268,29 @@ mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
 
 static struct mlxsw_sp_fib6_entry *
 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
-			   const struct rt6_info *rt)
+			   const struct fib6_info *rt)
 {
 	struct mlxsw_sp_fib6_entry *fib6_entry;
 	struct mlxsw_sp_fib_node *fib_node;
 	struct mlxsw_sp_fib *fib;
 	struct mlxsw_sp_vr *vr;
 
-	vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
+	vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
 	if (!vr)
 		return NULL;
 	fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
 
-	fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
-					    sizeof(rt->rt6i_dst.addr),
-					    rt->rt6i_dst.plen);
+	fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
+					    sizeof(rt->fib6_dst.addr),
+					    rt->fib6_dst.plen);
 	if (!fib_node)
 		return NULL;
 
 	list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
-		struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
+		struct fib6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
 
-		if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
-		    rt->rt6i_metric == iter_rt->rt6i_metric &&
+		if (rt->fib6_table->tb6_id == iter_rt->fib6_table->tb6_id &&
+		    rt->fib6_metric == iter_rt->fib6_metric &&
 		    mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
 			return fib6_entry;
 	}
@@ -5316,7 +5316,7 @@ static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
 }
 
 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
-				    struct rt6_info *rt, bool replace)
+				    struct fib6_info *rt, bool replace)
 {
 	struct mlxsw_sp_fib6_entry *fib6_entry;
 	struct mlxsw_sp_fib_node *fib_node;
@@ -5325,16 +5325,16 @@ static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
 	if (mlxsw_sp->router->aborted)
 		return 0;
 
-	if (rt->rt6i_src.plen)
+	if (rt->fib6_src.plen)
 		return -EINVAL;
 
 	if (mlxsw_sp_fib6_rt_should_ignore(rt))
 		return 0;
 
-	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
-					 &rt->rt6i_dst.addr,
-					 sizeof(rt->rt6i_dst.addr),
-					 rt->rt6i_dst.plen,
+	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
+					 &rt->fib6_dst.addr,
+					 sizeof(rt->fib6_dst.addr),
+					 rt->fib6_dst.plen,
 					 MLXSW_SP_L3_PROTO_IPV6);
 	if (IS_ERR(fib_node))
 		return PTR_ERR(fib_node);
@@ -5373,7 +5373,7 @@ static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
 }
 
 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
-				     struct rt6_info *rt)
+				     struct fib6_info *rt)
 {
 	struct mlxsw_sp_fib6_entry *fib6_entry;
 	struct mlxsw_sp_fib_node *fib_node;
@@ -5725,6 +5725,7 @@ static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
 
 	switch (fib_work->event) {
 	case FIB_EVENT_ENTRY_REPLACE: /* fall through */
+	case FIB_EVENT_ENTRY_APPEND: /* fall through */
 	case FIB_EVENT_ENTRY_ADD:
 		replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
 		err = mlxsw_sp_router_fib6_add(mlxsw_sp,
@@ -5831,12 +5832,13 @@ static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
 
 	switch (fib_work->event) {
 	case FIB_EVENT_ENTRY_REPLACE: /* fall through */
+	case FIB_EVENT_ENTRY_APPEND: /* fall through */
 	case FIB_EVENT_ENTRY_ADD: /* fall through */
 	case FIB_EVENT_ENTRY_DEL:
 		fen6_info = container_of(info, struct fib6_entry_notifier_info,
 					 info);
 		fib_work->fen6_info = *fen6_info;
-		rt6_hold(fib_work->fen6_info.rt);
+		fib6_info_hold(fib_work->fen6_info.rt);
 		break;
 	}
 }
@@ -5882,24 +5884,24 @@ static int mlxsw_sp_router_fib_rule_event(unsigned long event,
 	switch (info->family) {
 	case AF_INET:
 		if (!fib4_rule_default(rule) && !rule->l3mdev)
-			err = -1;
+			err = -EOPNOTSUPP;
 		break;
 	case AF_INET6:
 		if (!fib6_rule_default(rule) && !rule->l3mdev)
-			err = -1;
+			err = -EOPNOTSUPP;
 		break;
 	case RTNL_FAMILY_IPMR:
 		if (!ipmr_rule_default(rule) && !rule->l3mdev)
-			err = -1;
+			err = -EOPNOTSUPP;
 		break;
 	case RTNL_FAMILY_IP6MR:
 		if (!ip6mr_rule_default(rule) && !rule->l3mdev)
-			err = -1;
+			err = -EOPNOTSUPP;
 		break;
 	}
 
 	if (err < 0)
-		NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported. Aborting offload");
+		NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
 
 	return err;
 }
@@ -5926,8 +5928,15 @@ static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
 	case FIB_EVENT_RULE_DEL:
 		err = mlxsw_sp_router_fib_rule_event(event, info,
 						     router->mlxsw_sp);
-		if (!err)
-			return NOTIFY_DONE;
+		if (!err || info->extack)
+			return notifier_from_errno(err);
+		break;
+	case FIB_EVENT_ENTRY_ADD:
+		if (router->aborted) {
+			NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route");
+			return notifier_from_errno(-EINVAL);
+		}
+		break;
 	}
 
 	fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c
index 65a77708ff61..da3f7f527360 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c
@@ -32,6 +32,7 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <linux/if_bridge.h>
 #include <linux/list.h>
 #include <net/arp.h>
 #include <net/gre.h>
@@ -39,8 +40,9 @@
 #include <net/ip6_tunnel.h>
 
 #include "spectrum.h"
-#include "spectrum_span.h"
 #include "spectrum_ipip.h"
+#include "spectrum_span.h"
+#include "spectrum_switchdev.h"
 
 int mlxsw_sp_span_init(struct mlxsw_sp *mlxsw_sp)
 {
@@ -135,14 +137,14 @@ struct mlxsw_sp_span_entry_ops mlxsw_sp_span_entry_ops_phys = {
 
 static int mlxsw_sp_span_dmac(struct neigh_table *tbl,
 			      const void *pkey,
-			      struct net_device *l3edev,
+			      struct net_device *dev,
 			      unsigned char dmac[ETH_ALEN])
 {
-	struct neighbour *neigh = neigh_lookup(tbl, pkey, l3edev);
+	struct neighbour *neigh = neigh_lookup(tbl, pkey, dev);
 	int err = 0;
 
 	if (!neigh) {
-		neigh = neigh_create(tbl, pkey, l3edev);
+		neigh = neigh_create(tbl, pkey, dev);
 		if (IS_ERR(neigh))
 			return PTR_ERR(neigh);
 	}
@@ -167,8 +169,97 @@ mlxsw_sp_span_entry_unoffloadable(struct mlxsw_sp_span_parms *sparmsp)
 	return 0;
 }
 
+static struct net_device *
+mlxsw_sp_span_entry_bridge_8021q(const struct net_device *br_dev,
+				 unsigned char *dmac,
+				 u16 *p_vid)
+{
+	struct bridge_vlan_info vinfo;
+	struct net_device *edev;
+	u16 vid = *p_vid;
+
+	if (!vid && WARN_ON(br_vlan_get_pvid(br_dev, &vid)))
+		return NULL;
+	if (!vid ||
+	    br_vlan_get_info(br_dev, vid, &vinfo) ||
+	    !(vinfo.flags & BRIDGE_VLAN_INFO_BRENTRY))
+		return NULL;
+
+	edev = br_fdb_find_port(br_dev, dmac, vid);
+	if (!edev)
+		return NULL;
+
+	if (br_vlan_get_info(edev, vid, &vinfo))
+		return NULL;
+	if (!(vinfo.flags & BRIDGE_VLAN_INFO_UNTAGGED))
+		*p_vid = vid;
+	return edev;
+}
+
+static struct net_device *
+mlxsw_sp_span_entry_bridge_8021d(const struct net_device *br_dev,
+				 unsigned char *dmac)
+{
+	return br_fdb_find_port(br_dev, dmac, 0);
+}
+
+static struct net_device *
+mlxsw_sp_span_entry_bridge(const struct net_device *br_dev,
+			   unsigned char dmac[ETH_ALEN],
+			   u16 *p_vid)
+{
+	struct mlxsw_sp_bridge_port *bridge_port;
+	enum mlxsw_reg_spms_state spms_state;
+	struct net_device *dev = NULL;
+	struct mlxsw_sp_port *port;
+	u8 stp_state;
+
+	if (br_vlan_enabled(br_dev))
+		dev = mlxsw_sp_span_entry_bridge_8021q(br_dev, dmac, p_vid);
+	else if (!*p_vid)
+		dev = mlxsw_sp_span_entry_bridge_8021d(br_dev, dmac);
+	if (!dev)
+		return NULL;
+
+	port = mlxsw_sp_port_dev_lower_find(dev);
+	if (!port)
+		return NULL;
+
+	bridge_port = mlxsw_sp_bridge_port_find(port->mlxsw_sp->bridge, dev);
+	if (!bridge_port)
+		return NULL;
+
+	stp_state = mlxsw_sp_bridge_port_stp_state(bridge_port);
+	spms_state = mlxsw_sp_stp_spms_state(stp_state);
+	if (spms_state != MLXSW_REG_SPMS_STATE_FORWARDING)
+		return NULL;
+
+	return dev;
+}
+
+static struct net_device *
+mlxsw_sp_span_entry_vlan(const struct net_device *vlan_dev,
+			 u16 *p_vid)
+{
+	*p_vid = vlan_dev_vlan_id(vlan_dev);
+	return vlan_dev_real_dev(vlan_dev);
+}
+
+static struct net_device *
+mlxsw_sp_span_entry_lag(struct net_device *lag_dev)
+{
+	struct net_device *dev;
+	struct list_head *iter;
+
+	netdev_for_each_lower_dev(lag_dev, dev, iter)
+		if ((dev->flags & IFF_UP) && mlxsw_sp_port_dev_check(dev))
+			return dev;
+
+	return NULL;
+}
+
 static __maybe_unused int
-mlxsw_sp_span_entry_tunnel_parms_common(struct net_device *l3edev,
+mlxsw_sp_span_entry_tunnel_parms_common(struct net_device *edev,
 					union mlxsw_sp_l3addr saddr,
 					union mlxsw_sp_l3addr daddr,
 					union mlxsw_sp_l3addr gw,
@@ -177,21 +268,51 @@ mlxsw_sp_span_entry_tunnel_parms_common(struct net_device *l3edev,
 					struct mlxsw_sp_span_parms *sparmsp)
 {
 	unsigned char dmac[ETH_ALEN];
+	u16 vid = 0;
 
 	if (mlxsw_sp_l3addr_is_zero(gw))
 		gw = daddr;
 
-	if (!l3edev || !mlxsw_sp_port_dev_check(l3edev) ||
-	    mlxsw_sp_span_dmac(tbl, &gw, l3edev, dmac))
-		return mlxsw_sp_span_entry_unoffloadable(sparmsp);
+	if (!edev || mlxsw_sp_span_dmac(tbl, &gw, edev, dmac))
+		goto unoffloadable;
+
+	if (is_vlan_dev(edev))
+		edev = mlxsw_sp_span_entry_vlan(edev, &vid);
 
-	sparmsp->dest_port = netdev_priv(l3edev);
+	if (netif_is_bridge_master(edev)) {
+		edev = mlxsw_sp_span_entry_bridge(edev, dmac, &vid);
+		if (!edev)
+			goto unoffloadable;
+	}
+
+	if (is_vlan_dev(edev)) {
+		if (vid || !(edev->flags & IFF_UP))
+			goto unoffloadable;
+		edev = mlxsw_sp_span_entry_vlan(edev, &vid);
+	}
+
+	if (netif_is_lag_master(edev)) {
+		if (!(edev->flags & IFF_UP))
+			goto unoffloadable;
+		edev = mlxsw_sp_span_entry_lag(edev);
+		if (!edev)
+			goto unoffloadable;
+	}
+
+	if (!mlxsw_sp_port_dev_check(edev))
+		goto unoffloadable;
+
+	sparmsp->dest_port = netdev_priv(edev);
 	sparmsp->ttl = ttl;
 	memcpy(sparmsp->dmac, dmac, ETH_ALEN);
-	memcpy(sparmsp->smac, l3edev->dev_addr, ETH_ALEN);
+	memcpy(sparmsp->smac, edev->dev_addr, ETH_ALEN);
 	sparmsp->saddr = saddr;
 	sparmsp->daddr = daddr;
+	sparmsp->vid = vid;
 	return 0;
+
+unoffloadable:
+	return mlxsw_sp_span_entry_unoffloadable(sparmsp);
 }
 
 #if IS_ENABLED(CONFIG_NET_IPGRE)
@@ -268,9 +389,10 @@ mlxsw_sp_span_entry_gretap4_configure(struct mlxsw_sp_span_entry *span_entry,
 	/* Create a new port analayzer entry for local_port. */
 	mlxsw_reg_mpat_pack(mpat_pl, pa_id, local_port, true,
 			    MLXSW_REG_MPAT_SPAN_TYPE_REMOTE_ETH_L3);
+	mlxsw_reg_mpat_eth_rspan_pack(mpat_pl, sparms.vid);
 	mlxsw_reg_mpat_eth_rspan_l2_pack(mpat_pl,
 				    MLXSW_REG_MPAT_ETH_RSPAN_VERSION_NO_HEADER,
-				    sparms.dmac, false);
+				    sparms.dmac, !!sparms.vid);
 	mlxsw_reg_mpat_eth_rspan_l3_ipv4_pack(mpat_pl,
 					      sparms.ttl, sparms.smac,
 					      be32_to_cpu(sparms.saddr.addr4),
@@ -368,9 +490,10 @@ mlxsw_sp_span_entry_gretap6_configure(struct mlxsw_sp_span_entry *span_entry,
 	/* Create a new port analayzer entry for local_port. */
 	mlxsw_reg_mpat_pack(mpat_pl, pa_id, local_port, true,
 			    MLXSW_REG_MPAT_SPAN_TYPE_REMOTE_ETH_L3);
+	mlxsw_reg_mpat_eth_rspan_pack(mpat_pl, sparms.vid);
 	mlxsw_reg_mpat_eth_rspan_l2_pack(mpat_pl,
 				    MLXSW_REG_MPAT_ETH_RSPAN_VERSION_NO_HEADER,
-				    sparms.dmac, false);
+				    sparms.dmac, !!sparms.vid);
 	mlxsw_reg_mpat_eth_rspan_l3_ipv6_pack(mpat_pl, sparms.ttl, sparms.smac,
 					      sparms.saddr.addr6,
 					      sparms.daddr.addr6);
@@ -394,6 +517,61 @@ struct mlxsw_sp_span_entry_ops mlxsw_sp_span_entry_ops_gretap6 = {
 };
 #endif
 
+static bool
+mlxsw_sp_span_vlan_can_handle(const struct net_device *dev)
+{
+	return is_vlan_dev(dev) &&
+	       mlxsw_sp_port_dev_check(vlan_dev_real_dev(dev));
+}
+
+static int
+mlxsw_sp_span_entry_vlan_parms(const struct net_device *to_dev,
+			       struct mlxsw_sp_span_parms *sparmsp)
+{
+	struct net_device *real_dev;
+	u16 vid;
+
+	if (!(to_dev->flags & IFF_UP))
+		return mlxsw_sp_span_entry_unoffloadable(sparmsp);
+
+	real_dev = mlxsw_sp_span_entry_vlan(to_dev, &vid);
+	sparmsp->dest_port = netdev_priv(real_dev);
+	sparmsp->vid = vid;
+	return 0;
+}
+
+static int
+mlxsw_sp_span_entry_vlan_configure(struct mlxsw_sp_span_entry *span_entry,
+				   struct mlxsw_sp_span_parms sparms)
+{
+	struct mlxsw_sp_port *dest_port = sparms.dest_port;
+	struct mlxsw_sp *mlxsw_sp = dest_port->mlxsw_sp;
+	u8 local_port = dest_port->local_port;
+	char mpat_pl[MLXSW_REG_MPAT_LEN];
+	int pa_id = span_entry->id;
+
+	mlxsw_reg_mpat_pack(mpat_pl, pa_id, local_port, true,
+			    MLXSW_REG_MPAT_SPAN_TYPE_REMOTE_ETH);
+	mlxsw_reg_mpat_eth_rspan_pack(mpat_pl, sparms.vid);
+
+	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mpat), mpat_pl);
+}
+
+static void
+mlxsw_sp_span_entry_vlan_deconfigure(struct mlxsw_sp_span_entry *span_entry)
+{
+	mlxsw_sp_span_entry_deconfigure_common(span_entry,
+					MLXSW_REG_MPAT_SPAN_TYPE_REMOTE_ETH);
+}
+
+static const
+struct mlxsw_sp_span_entry_ops mlxsw_sp_span_entry_ops_vlan = {
+	.can_handle = mlxsw_sp_span_vlan_can_handle,
+	.parms = mlxsw_sp_span_entry_vlan_parms,
+	.configure = mlxsw_sp_span_entry_vlan_configure,
+	.deconfigure = mlxsw_sp_span_entry_vlan_deconfigure,
+};
+
 static const
 struct mlxsw_sp_span_entry_ops *const mlxsw_sp_span_entry_types[] = {
 	&mlxsw_sp_span_entry_ops_phys,
@@ -403,6 +581,7 @@ struct mlxsw_sp_span_entry_ops *const mlxsw_sp_span_entry_types[] = {
 #if IS_ENABLED(CONFIG_IPV6_GRE)
 	&mlxsw_sp_span_entry_ops_gretap6,
 #endif
+	&mlxsw_sp_span_entry_ops_vlan,
 };
 
 static int
@@ -766,7 +945,7 @@ int mlxsw_sp_span_mirror_add(struct mlxsw_sp_port *from,
 
 	span_entry = mlxsw_sp_span_entry_get(mlxsw_sp, to_dev, ops, sparms);
 	if (!span_entry)
-		return -ENOENT;
+		return -ENOBUFS;
 
 	netdev_dbg(from->dev, "Adding inspected port to SPAN entry %d\n",
 		   span_entry->id);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.h
index 4b87ec20e658..14a6de904db1 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.h
@@ -63,6 +63,7 @@ struct mlxsw_sp_span_parms {
 	unsigned char smac[ETH_ALEN];
 	union mlxsw_sp_l3addr daddr;
 	union mlxsw_sp_l3addr saddr;
+	u16 vid;
 };
 
 struct mlxsw_sp_span_entry_ops;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
index 4ed01182a82c..8c9cf8ee9398 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
@@ -49,7 +49,9 @@
 #include <linux/netlink.h>
 #include <net/switchdev.h>
 
+#include "spectrum_span.h"
 #include "spectrum_router.h"
+#include "spectrum_switchdev.h"
 #include "spectrum.h"
 #include "core.h"
 #include "reg.h"
@@ -239,7 +241,7 @@ __mlxsw_sp_bridge_port_find(const struct mlxsw_sp_bridge_device *bridge_device,
 	return NULL;
 }
 
-static struct mlxsw_sp_bridge_port *
+struct mlxsw_sp_bridge_port *
 mlxsw_sp_bridge_port_find(struct mlxsw_sp_bridge *bridge,
 			  struct net_device *brport_dev)
 {
@@ -922,6 +924,9 @@ static int mlxsw_sp_port_attr_set(struct net_device *dev,
 		break;
 	}
 
+	if (switchdev_trans_ph_commit(trans))
+		mlxsw_sp_span_respin(mlxsw_sp_port->mlxsw_sp);
+
 	return err;
 }
 
@@ -1646,18 +1651,57 @@ mlxsw_sp_port_mrouter_update_mdb(struct mlxsw_sp_port *mlxsw_sp_port,
 	}
 }
 
+struct mlxsw_sp_span_respin_work {
+	struct work_struct work;
+	struct mlxsw_sp *mlxsw_sp;
+};
+
+static void mlxsw_sp_span_respin_work(struct work_struct *work)
+{
+	struct mlxsw_sp_span_respin_work *respin_work =
+		container_of(work, struct mlxsw_sp_span_respin_work, work);
+
+	rtnl_lock();
+	mlxsw_sp_span_respin(respin_work->mlxsw_sp);
+	rtnl_unlock();
+	kfree(respin_work);
+}
+
+static void mlxsw_sp_span_respin_schedule(struct mlxsw_sp *mlxsw_sp)
+{
+	struct mlxsw_sp_span_respin_work *respin_work;
+
+	respin_work = kzalloc(sizeof(*respin_work), GFP_ATOMIC);
+	if (!respin_work)
+		return;
+
+	INIT_WORK(&respin_work->work, mlxsw_sp_span_respin_work);
+	respin_work->mlxsw_sp = mlxsw_sp;
+
+	mlxsw_core_schedule_work(&respin_work->work);
+}
+
 static int mlxsw_sp_port_obj_add(struct net_device *dev,
 				 const struct switchdev_obj *obj,
 				 struct switchdev_trans *trans)
 {
 	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
+	const struct switchdev_obj_port_vlan *vlan;
 	int err = 0;
 
 	switch (obj->id) {
 	case SWITCHDEV_OBJ_ID_PORT_VLAN:
-		err = mlxsw_sp_port_vlans_add(mlxsw_sp_port,
-					      SWITCHDEV_OBJ_PORT_VLAN(obj),
-					      trans);
+		vlan = SWITCHDEV_OBJ_PORT_VLAN(obj);
+		err = mlxsw_sp_port_vlans_add(mlxsw_sp_port, vlan, trans);
+
+		if (switchdev_trans_ph_commit(trans)) {
+			/* The event is emitted before the changes are actually
+			 * applied to the bridge. Therefore schedule the respin
+			 * call for later, so that the respin logic sees the
+			 * updated bridge state.
+			 */
+			mlxsw_sp_span_respin_schedule(mlxsw_sp_port->mlxsw_sp);
+		}
 		break;
 	case SWITCHDEV_OBJ_ID_PORT_MDB:
 		err = mlxsw_sp_port_mdb_add(mlxsw_sp_port,
@@ -1806,6 +1850,8 @@ static int mlxsw_sp_port_obj_del(struct net_device *dev,
 		break;
 	}
 
+	mlxsw_sp_span_respin(mlxsw_sp_port->mlxsw_sp);
+
 	return err;
 }
 
@@ -2222,6 +2268,8 @@ static void mlxsw_sp_switchdev_event_work(struct work_struct *work)
 	switch (switchdev_work->event) {
 	case SWITCHDEV_FDB_ADD_TO_DEVICE:
 		fdb_info = &switchdev_work->fdb_info;
+		if (!fdb_info->added_by_user)
+			break;
 		err = mlxsw_sp_port_fdb_set(mlxsw_sp_port, fdb_info, true);
 		if (err)
 			break;
@@ -2231,10 +2279,20 @@ static void mlxsw_sp_switchdev_event_work(struct work_struct *work)
 		break;
 	case SWITCHDEV_FDB_DEL_TO_DEVICE:
 		fdb_info = &switchdev_work->fdb_info;
+		if (!fdb_info->added_by_user)
+			break;
 		mlxsw_sp_port_fdb_set(mlxsw_sp_port, fdb_info, false);
 		break;
+	case SWITCHDEV_FDB_ADD_TO_BRIDGE: /* fall through */
+	case SWITCHDEV_FDB_DEL_TO_BRIDGE:
+		/* These events are only used to potentially update an existing
+		 * SPAN mirror.
+		 */
+		break;
 	}
 
+	mlxsw_sp_span_respin(mlxsw_sp_port->mlxsw_sp);
+
 out:
 	rtnl_unlock();
 	kfree(switchdev_work->fdb_info.addr);
@@ -2263,7 +2321,9 @@ static int mlxsw_sp_switchdev_event(struct notifier_block *unused,
 
 	switch (event) {
 	case SWITCHDEV_FDB_ADD_TO_DEVICE: /* fall through */
-	case SWITCHDEV_FDB_DEL_TO_DEVICE:
+	case SWITCHDEV_FDB_DEL_TO_DEVICE: /* fall through */
+	case SWITCHDEV_FDB_ADD_TO_BRIDGE: /* fall through */
+	case SWITCHDEV_FDB_DEL_TO_BRIDGE:
 		memcpy(&switchdev_work->fdb_info, ptr,
 		       sizeof(switchdev_work->fdb_info));
 		switchdev_work->fdb_info.addr = kzalloc(ETH_ALEN, GFP_ATOMIC);
@@ -2295,6 +2355,12 @@ static struct notifier_block mlxsw_sp_switchdev_notifier = {
 	.notifier_call = mlxsw_sp_switchdev_event,
 };
 
+u8
+mlxsw_sp_bridge_port_stp_state(struct mlxsw_sp_bridge_port *bridge_port)
+{
+	return bridge_port->stp_state;
+}
+
 static int mlxsw_sp_fdb_init(struct mlxsw_sp *mlxsw_sp)
 {
 	struct mlxsw_sp_bridge *bridge = mlxsw_sp->bridge;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.h
new file mode 100644
index 000000000000..bc44d5effc28
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.h
@@ -0,0 +1,43 @@
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
+ * drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.h
+ * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/netdevice.h>
+
+struct mlxsw_sp_bridge;
+struct mlxsw_sp_bridge_port;
+
+struct mlxsw_sp_bridge_port *
+mlxsw_sp_bridge_port_find(struct mlxsw_sp_bridge *bridge,
+			  struct net_device *brport_dev);
+
+u8 mlxsw_sp_bridge_port_stp_state(struct mlxsw_sp_bridge_port *bridge_port);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c
index a655c5850aa6..3922c1cfe5f5 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c
@@ -417,13 +417,10 @@ static int mlxsw_sx_port_get_phys_port_name(struct net_device *dev, char *name,
 					    size_t len)
 {
 	struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev);
-	int err;
-
-	err = snprintf(name, len, "p%d", mlxsw_sx_port->mapping.module + 1);
-	if (err >= len)
-		return -EINVAL;
 
-	return 0;
+	return mlxsw_core_port_get_phys_port_name(mlxsw_sx_port->mlxsw_sx->core,
+						  mlxsw_sx_port->local_port,
+						  name, len);
 }
 
 static const struct net_device_ops mlxsw_sx_port_netdev_ops = {
@@ -1149,7 +1146,7 @@ static int __mlxsw_sx_port_eth_create(struct mlxsw_sx *mlxsw_sx, u8 local_port,
 	}
 
 	mlxsw_core_port_eth_set(mlxsw_sx->core, mlxsw_sx_port->local_port,
-				mlxsw_sx_port, dev, false, 0);
+				mlxsw_sx_port, dev, module + 1, false, 0);
 	mlxsw_sx->ports[local_port] = mlxsw_sx_port;
 	return 0;
 
diff --git a/drivers/net/ethernet/mscc/Kconfig b/drivers/net/ethernet/mscc/Kconfig
new file mode 100644
index 000000000000..36c84625d54e
--- /dev/null
+++ b/drivers/net/ethernet/mscc/Kconfig
@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: (GPL-2.0 OR MIT)
+config NET_VENDOR_MICROSEMI
+	bool "Microsemi devices"
+	default y
+	help
+	  If you have a network (Ethernet) card belonging to this class, say Y.
+
+	  Note that the answer to this question doesn't directly affect the
+	  kernel: saying N will just cause the configurator to skip all
+	  the questions about Microsemi devices.
+
+if NET_VENDOR_MICROSEMI
+
+config MSCC_OCELOT_SWITCH
+	tristate "Ocelot switch driver"
+	depends on NET_SWITCHDEV
+	depends on HAS_IOMEM
+	select PHYLIB
+	select REGMAP_MMIO
+	help
+	  This driver supports the Ocelot network switch device.
+
+config MSCC_OCELOT_SWITCH_OCELOT
+	tristate "Ocelot switch driver on Ocelot"
+	depends on MSCC_OCELOT_SWITCH
+	help
+	  This driver supports the Ocelot network switch device as present on
+	  the Ocelot SoCs.
+
+endif # NET_VENDOR_MICROSEMI
diff --git a/drivers/net/ethernet/mscc/Makefile b/drivers/net/ethernet/mscc/Makefile
new file mode 100644
index 000000000000..cb52a3b128ae
--- /dev/null
+++ b/drivers/net/ethernet/mscc/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: (GPL-2.0 OR MIT)
+obj-$(CONFIG_MSCC_OCELOT_SWITCH) += mscc_ocelot_common.o
+mscc_ocelot_common-y := ocelot.o ocelot_io.o
+mscc_ocelot_common-y += ocelot_regs.o
+obj-$(CONFIG_MSCC_OCELOT_SWITCH_OCELOT) += ocelot_board.o
diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c
new file mode 100644
index 000000000000..c8c74aa548d9
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot.c
@@ -0,0 +1,1333 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/if_bridge.h>
+#include <linux/if_ether.h>
+#include <linux/if_vlan.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/phy.h>
+#include <linux/skbuff.h>
+#include <net/arp.h>
+#include <net/netevent.h>
+#include <net/rtnetlink.h>
+#include <net/switchdev.h>
+
+#include "ocelot.h"
+
+/* MAC table entry types.
+ * ENTRYTYPE_NORMAL is subject to aging.
+ * ENTRYTYPE_LOCKED is not subject to aging.
+ * ENTRYTYPE_MACv4 is not subject to aging. For IPv4 multicast.
+ * ENTRYTYPE_MACv6 is not subject to aging. For IPv6 multicast.
+ */
+enum macaccess_entry_type {
+	ENTRYTYPE_NORMAL = 0,
+	ENTRYTYPE_LOCKED,
+	ENTRYTYPE_MACv4,
+	ENTRYTYPE_MACv6,
+};
+
+struct ocelot_mact_entry {
+	u8 mac[ETH_ALEN];
+	u16 vid;
+	enum macaccess_entry_type type;
+};
+
+static inline int ocelot_mact_wait_for_completion(struct ocelot *ocelot)
+{
+	unsigned int val, timeout = 10;
+
+	/* Wait for the issued mac table command to be completed, or timeout.
+	 * When the command read from  ANA_TABLES_MACACCESS is
+	 * MACACCESS_CMD_IDLE, the issued command completed successfully.
+	 */
+	do {
+		val = ocelot_read(ocelot, ANA_TABLES_MACACCESS);
+		val &= ANA_TABLES_MACACCESS_MAC_TABLE_CMD_M;
+	} while (val != MACACCESS_CMD_IDLE && timeout--);
+
+	if (!timeout)
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+static void ocelot_mact_select(struct ocelot *ocelot,
+			       const unsigned char mac[ETH_ALEN],
+			       unsigned int vid)
+{
+	u32 macl = 0, mach = 0;
+
+	/* Set the MAC address to handle and the vlan associated in a format
+	 * understood by the hardware.
+	 */
+	mach |= vid    << 16;
+	mach |= mac[0] << 8;
+	mach |= mac[1] << 0;
+	macl |= mac[2] << 24;
+	macl |= mac[3] << 16;
+	macl |= mac[4] << 8;
+	macl |= mac[5] << 0;
+
+	ocelot_write(ocelot, macl, ANA_TABLES_MACLDATA);
+	ocelot_write(ocelot, mach, ANA_TABLES_MACHDATA);
+
+}
+
+static int ocelot_mact_learn(struct ocelot *ocelot, int port,
+			     const unsigned char mac[ETH_ALEN],
+			     unsigned int vid,
+			     enum macaccess_entry_type type)
+{
+	ocelot_mact_select(ocelot, mac, vid);
+
+	/* Issue a write command */
+	ocelot_write(ocelot, ANA_TABLES_MACACCESS_VALID |
+			     ANA_TABLES_MACACCESS_DEST_IDX(port) |
+			     ANA_TABLES_MACACCESS_ENTRYTYPE(type) |
+			     ANA_TABLES_MACACCESS_MAC_TABLE_CMD(MACACCESS_CMD_LEARN),
+			     ANA_TABLES_MACACCESS);
+
+	return ocelot_mact_wait_for_completion(ocelot);
+}
+
+static int ocelot_mact_forget(struct ocelot *ocelot,
+			      const unsigned char mac[ETH_ALEN],
+			      unsigned int vid)
+{
+	ocelot_mact_select(ocelot, mac, vid);
+
+	/* Issue a forget command */
+	ocelot_write(ocelot,
+		     ANA_TABLES_MACACCESS_MAC_TABLE_CMD(MACACCESS_CMD_FORGET),
+		     ANA_TABLES_MACACCESS);
+
+	return ocelot_mact_wait_for_completion(ocelot);
+}
+
+static void ocelot_mact_init(struct ocelot *ocelot)
+{
+	/* Configure the learning mode entries attributes:
+	 * - Do not copy the frame to the CPU extraction queues.
+	 * - Use the vlan and mac_cpoy for dmac lookup.
+	 */
+	ocelot_rmw(ocelot, 0,
+		   ANA_AGENCTRL_LEARN_CPU_COPY | ANA_AGENCTRL_IGNORE_DMAC_FLAGS
+		   | ANA_AGENCTRL_LEARN_FWD_KILL
+		   | ANA_AGENCTRL_LEARN_IGNORE_VLAN,
+		   ANA_AGENCTRL);
+
+	/* Clear the MAC table */
+	ocelot_write(ocelot, MACACCESS_CMD_INIT, ANA_TABLES_MACACCESS);
+}
+
+static inline int ocelot_vlant_wait_for_completion(struct ocelot *ocelot)
+{
+	unsigned int val, timeout = 10;
+
+	/* Wait for the issued mac table command to be completed, or timeout.
+	 * When the command read from ANA_TABLES_MACACCESS is
+	 * MACACCESS_CMD_IDLE, the issued command completed successfully.
+	 */
+	do {
+		val = ocelot_read(ocelot, ANA_TABLES_VLANACCESS);
+		val &= ANA_TABLES_VLANACCESS_VLAN_TBL_CMD_M;
+	} while (val != ANA_TABLES_VLANACCESS_CMD_IDLE && timeout--);
+
+	if (!timeout)
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+static void ocelot_vlan_init(struct ocelot *ocelot)
+{
+	/* Clear VLAN table, by default all ports are members of all VLANs */
+	ocelot_write(ocelot, ANA_TABLES_VLANACCESS_CMD_INIT,
+		     ANA_TABLES_VLANACCESS);
+	ocelot_vlant_wait_for_completion(ocelot);
+}
+
+/* Watermark encode
+ * Bit 8:   Unit; 0:1, 1:16
+ * Bit 7-0: Value to be multiplied with unit
+ */
+static u16 ocelot_wm_enc(u16 value)
+{
+	if (value >= BIT(8))
+		return BIT(8) | (value / 16);
+
+	return value;
+}
+
+static void ocelot_port_adjust_link(struct net_device *dev)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+	u8 p = port->chip_port;
+	int speed, atop_wm, mode = 0;
+
+	switch (dev->phydev->speed) {
+	case SPEED_10:
+		speed = OCELOT_SPEED_10;
+		break;
+	case SPEED_100:
+		speed = OCELOT_SPEED_100;
+		break;
+	case SPEED_1000:
+		speed = OCELOT_SPEED_1000;
+		mode = DEV_MAC_MODE_CFG_GIGA_MODE_ENA;
+		break;
+	case SPEED_2500:
+		speed = OCELOT_SPEED_2500;
+		mode = DEV_MAC_MODE_CFG_GIGA_MODE_ENA;
+		break;
+	default:
+		netdev_err(dev, "Unsupported PHY speed: %d\n",
+			   dev->phydev->speed);
+		return;
+	}
+
+	phy_print_status(dev->phydev);
+
+	if (!dev->phydev->link)
+		return;
+
+	/* Only full duplex supported for now */
+	ocelot_port_writel(port, DEV_MAC_MODE_CFG_FDX_ENA |
+			   mode, DEV_MAC_MODE_CFG);
+
+	/* Set MAC IFG Gaps
+	 * FDX: TX_IFG = 5, RX_IFG1 = RX_IFG2 = 0
+	 * !FDX: TX_IFG = 5, RX_IFG1 = RX_IFG2 = 5
+	 */
+	ocelot_port_writel(port, DEV_MAC_IFG_CFG_TX_IFG(5), DEV_MAC_IFG_CFG);
+
+	/* Load seed (0) and set MAC HDX late collision  */
+	ocelot_port_writel(port, DEV_MAC_HDX_CFG_LATE_COL_POS(67) |
+			   DEV_MAC_HDX_CFG_SEED_LOAD,
+			   DEV_MAC_HDX_CFG);
+	mdelay(1);
+	ocelot_port_writel(port, DEV_MAC_HDX_CFG_LATE_COL_POS(67),
+			   DEV_MAC_HDX_CFG);
+
+	/* Disable HDX fast control */
+	ocelot_port_writel(port, DEV_PORT_MISC_HDX_FAST_DIS, DEV_PORT_MISC);
+
+	/* SGMII only for now */
+	ocelot_port_writel(port, PCS1G_MODE_CFG_SGMII_MODE_ENA, PCS1G_MODE_CFG);
+	ocelot_port_writel(port, PCS1G_SD_CFG_SD_SEL, PCS1G_SD_CFG);
+
+	/* Enable PCS */
+	ocelot_port_writel(port, PCS1G_CFG_PCS_ENA, PCS1G_CFG);
+
+	/* No aneg on SGMII */
+	ocelot_port_writel(port, 0, PCS1G_ANEG_CFG);
+
+	/* No loopback */
+	ocelot_port_writel(port, 0, PCS1G_LB_CFG);
+
+	/* Set Max Length and maximum tags allowed */
+	ocelot_port_writel(port, VLAN_ETH_FRAME_LEN, DEV_MAC_MAXLEN_CFG);
+	ocelot_port_writel(port, DEV_MAC_TAGS_CFG_TAG_ID(ETH_P_8021AD) |
+			   DEV_MAC_TAGS_CFG_VLAN_AWR_ENA |
+			   DEV_MAC_TAGS_CFG_VLAN_LEN_AWR_ENA,
+			   DEV_MAC_TAGS_CFG);
+
+	/* Enable MAC module */
+	ocelot_port_writel(port, DEV_MAC_ENA_CFG_RX_ENA |
+			   DEV_MAC_ENA_CFG_TX_ENA, DEV_MAC_ENA_CFG);
+
+	/* Take MAC, Port, Phy (intern) and PCS (SGMII/Serdes) clock out of
+	 * reset */
+	ocelot_port_writel(port, DEV_CLOCK_CFG_LINK_SPEED(speed),
+			   DEV_CLOCK_CFG);
+
+	/* Set SMAC of Pause frame (00:00:00:00:00:00) */
+	ocelot_port_writel(port, 0, DEV_MAC_FC_MAC_HIGH_CFG);
+	ocelot_port_writel(port, 0, DEV_MAC_FC_MAC_LOW_CFG);
+
+	/* No PFC */
+	ocelot_write_gix(ocelot, ANA_PFC_PFC_CFG_FC_LINK_SPEED(speed),
+			 ANA_PFC_PFC_CFG, p);
+
+	/* Set Pause WM hysteresis
+	 * 152 = 6 * VLAN_ETH_FRAME_LEN / OCELOT_BUFFER_CELL_SZ
+	 * 101 = 4 * VLAN_ETH_FRAME_LEN / OCELOT_BUFFER_CELL_SZ
+	 */
+	ocelot_write_rix(ocelot, SYS_PAUSE_CFG_PAUSE_ENA |
+			 SYS_PAUSE_CFG_PAUSE_STOP(101) |
+			 SYS_PAUSE_CFG_PAUSE_START(152), SYS_PAUSE_CFG, p);
+
+	/* Core: Enable port for frame transfer */
+	ocelot_write_rix(ocelot, QSYS_SWITCH_PORT_MODE_INGRESS_DROP_MODE |
+			 QSYS_SWITCH_PORT_MODE_SCH_NEXT_CFG(1) |
+			 QSYS_SWITCH_PORT_MODE_PORT_ENA,
+			 QSYS_SWITCH_PORT_MODE, p);
+
+	/* Flow control */
+	ocelot_write_rix(ocelot, SYS_MAC_FC_CFG_PAUSE_VAL_CFG(0xffff) |
+			 SYS_MAC_FC_CFG_RX_FC_ENA | SYS_MAC_FC_CFG_TX_FC_ENA |
+			 SYS_MAC_FC_CFG_ZERO_PAUSE_ENA |
+			 SYS_MAC_FC_CFG_FC_LATENCY_CFG(0x7) |
+			 SYS_MAC_FC_CFG_FC_LINK_SPEED(speed),
+			 SYS_MAC_FC_CFG, p);
+	ocelot_write_rix(ocelot, 0, ANA_POL_FLOWC, p);
+
+	/* Tail dropping watermark */
+	atop_wm = (ocelot->shared_queue_sz - 9 * VLAN_ETH_FRAME_LEN) / OCELOT_BUFFER_CELL_SZ;
+	ocelot_write_rix(ocelot, ocelot_wm_enc(9 * VLAN_ETH_FRAME_LEN),
+			 SYS_ATOP, p);
+	ocelot_write(ocelot, ocelot_wm_enc(atop_wm), SYS_ATOP_TOT_CFG);
+}
+
+static int ocelot_port_open(struct net_device *dev)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+	int err;
+
+	/* Enable receiving frames on the port, and activate auto-learning of
+	 * MAC addresses.
+	 */
+	ocelot_write_gix(ocelot, ANA_PORT_PORT_CFG_LEARNAUTO |
+			 ANA_PORT_PORT_CFG_RECV_ENA |
+			 ANA_PORT_PORT_CFG_PORTID_VAL(port->chip_port),
+			 ANA_PORT_PORT_CFG, port->chip_port);
+
+	err = phy_connect_direct(dev, port->phy, &ocelot_port_adjust_link,
+				 PHY_INTERFACE_MODE_NA);
+	if (err) {
+		netdev_err(dev, "Could not attach to PHY\n");
+		return err;
+	}
+
+	dev->phydev = port->phy;
+
+	phy_attached_info(port->phy);
+	phy_start(port->phy);
+	return 0;
+}
+
+static int ocelot_port_stop(struct net_device *dev)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+
+	phy_disconnect(port->phy);
+
+	dev->phydev = NULL;
+
+	ocelot_port_writel(port, 0, DEV_MAC_ENA_CFG);
+	ocelot_rmw_rix(port->ocelot, 0, QSYS_SWITCH_PORT_MODE_PORT_ENA,
+			 QSYS_SWITCH_PORT_MODE, port->chip_port);
+	return 0;
+}
+
+/* Generate the IFH for frame injection
+ *
+ * The IFH is a 128bit-value
+ * bit 127: bypass the analyzer processing
+ * bit 56-67: destination mask
+ * bit 28-29: pop_cnt: 3 disables all rewriting of the frame
+ * bit 20-27: cpu extraction queue mask
+ * bit 16: tag type 0: C-tag, 1: S-tag
+ * bit 0-11: VID
+ */
+static int ocelot_gen_ifh(u32 *ifh, struct frame_info *info)
+{
+	ifh[0] = IFH_INJ_BYPASS;
+	ifh[1] = (0xff00 & info->port) >> 8;
+	ifh[2] = (0xff & info->port) << 24;
+	ifh[3] = IFH_INJ_POP_CNT_DISABLE | (info->cpuq << 20) |
+		 (info->tag_type << 16) | info->vid;
+
+	return 0;
+}
+
+static int ocelot_port_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+	u32 val, ifh[IFH_LEN];
+	struct frame_info info = {};
+	u8 grp = 0; /* Send everything on CPU group 0 */
+	unsigned int i, count, last;
+
+	val = ocelot_read(ocelot, QS_INJ_STATUS);
+	if (!(val & QS_INJ_STATUS_FIFO_RDY(BIT(grp))) ||
+	    (val & QS_INJ_STATUS_WMARK_REACHED(BIT(grp))))
+		return NETDEV_TX_BUSY;
+
+	ocelot_write_rix(ocelot, QS_INJ_CTRL_GAP_SIZE(1) |
+			 QS_INJ_CTRL_SOF, QS_INJ_CTRL, grp);
+
+	info.port = BIT(port->chip_port);
+	info.cpuq = 0xff;
+	ocelot_gen_ifh(ifh, &info);
+
+	for (i = 0; i < IFH_LEN; i++)
+		ocelot_write_rix(ocelot, ifh[i], QS_INJ_WR, grp);
+
+	count = (skb->len + 3) / 4;
+	last = skb->len % 4;
+	for (i = 0; i < count; i++) {
+		ocelot_write_rix(ocelot, ((u32 *)skb->data)[i], QS_INJ_WR, grp);
+	}
+
+	/* Add padding */
+	while (i < (OCELOT_BUFFER_CELL_SZ / 4)) {
+		ocelot_write_rix(ocelot, 0, QS_INJ_WR, grp);
+		i++;
+	}
+
+	/* Indicate EOF and valid bytes in last word */
+	ocelot_write_rix(ocelot, QS_INJ_CTRL_GAP_SIZE(1) |
+			 QS_INJ_CTRL_VLD_BYTES(skb->len < OCELOT_BUFFER_CELL_SZ ? 0 : last) |
+			 QS_INJ_CTRL_EOF,
+			 QS_INJ_CTRL, grp);
+
+	/* Add dummy CRC */
+	ocelot_write_rix(ocelot, 0, QS_INJ_WR, grp);
+	skb_tx_timestamp(skb);
+
+	dev->stats.tx_packets++;
+	dev->stats.tx_bytes += skb->len;
+	dev_kfree_skb_any(skb);
+
+	return NETDEV_TX_OK;
+}
+
+static void ocelot_mact_mc_reset(struct ocelot_port *port)
+{
+	struct ocelot *ocelot = port->ocelot;
+	struct netdev_hw_addr *ha, *n;
+
+	/* Free and forget all the MAC addresses stored in the port private mc
+	 * list. These are mc addresses that were previously added by calling
+	 * ocelot_mact_mc_add().
+	 */
+	list_for_each_entry_safe(ha, n, &port->mc, list) {
+		ocelot_mact_forget(ocelot, ha->addr, port->pvid);
+		list_del(&ha->list);
+		kfree(ha);
+	}
+}
+
+static int ocelot_mact_mc_add(struct ocelot_port *port,
+			      struct netdev_hw_addr *hw_addr)
+{
+	struct ocelot *ocelot = port->ocelot;
+	struct netdev_hw_addr *ha = kzalloc(sizeof(*ha), GFP_KERNEL);
+
+	if (!ha)
+		return -ENOMEM;
+
+	memcpy(ha, hw_addr, sizeof(*ha));
+	list_add_tail(&ha->list, &port->mc);
+
+	ocelot_mact_learn(ocelot, PGID_CPU, ha->addr, port->pvid,
+			  ENTRYTYPE_LOCKED);
+
+	return 0;
+}
+
+static void ocelot_set_rx_mode(struct net_device *dev)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+	struct netdev_hw_addr *ha;
+	int i;
+	u32 val;
+
+	/* This doesn't handle promiscuous mode because the bridge core is
+	 * setting IFF_PROMISC on all slave interfaces and all frames would be
+	 * forwarded to the CPU port.
+	 */
+	val = GENMASK(ocelot->num_phys_ports - 1, 0);
+	for (i = ocelot->num_phys_ports + 1; i < PGID_CPU; i++)
+		ocelot_write_rix(ocelot, val, ANA_PGID_PGID, i);
+
+	/* Handle the device multicast addresses. First remove all the
+	 * previously installed addresses and then add the latest ones to the
+	 * mac table.
+	 */
+	ocelot_mact_mc_reset(port);
+	netdev_for_each_mc_addr(ha, dev)
+		ocelot_mact_mc_add(port, ha);
+}
+
+static int ocelot_port_get_phys_port_name(struct net_device *dev,
+					  char *buf, size_t len)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	int ret;
+
+	ret = snprintf(buf, len, "p%d", port->chip_port);
+	if (ret >= len)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int ocelot_port_set_mac_address(struct net_device *dev, void *p)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+	const struct sockaddr *addr = p;
+
+	/* Learn the new net device MAC address in the mac table. */
+	ocelot_mact_learn(ocelot, PGID_CPU, addr->sa_data, port->pvid,
+			  ENTRYTYPE_LOCKED);
+	/* Then forget the previous one. */
+	ocelot_mact_forget(ocelot, dev->dev_addr, port->pvid);
+
+	ether_addr_copy(dev->dev_addr, addr->sa_data);
+	return 0;
+}
+
+static void ocelot_get_stats64(struct net_device *dev,
+			       struct rtnl_link_stats64 *stats)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+
+	/* Configure the port to read the stats from */
+	ocelot_write(ocelot, SYS_STAT_CFG_STAT_VIEW(port->chip_port),
+		     SYS_STAT_CFG);
+
+	/* Get Rx stats */
+	stats->rx_bytes = ocelot_read(ocelot, SYS_COUNT_RX_OCTETS);
+	stats->rx_packets = ocelot_read(ocelot, SYS_COUNT_RX_SHORTS) +
+			    ocelot_read(ocelot, SYS_COUNT_RX_FRAGMENTS) +
+			    ocelot_read(ocelot, SYS_COUNT_RX_JABBERS) +
+			    ocelot_read(ocelot, SYS_COUNT_RX_LONGS) +
+			    ocelot_read(ocelot, SYS_COUNT_RX_64) +
+			    ocelot_read(ocelot, SYS_COUNT_RX_65_127) +
+			    ocelot_read(ocelot, SYS_COUNT_RX_128_255) +
+			    ocelot_read(ocelot, SYS_COUNT_RX_256_1023) +
+			    ocelot_read(ocelot, SYS_COUNT_RX_1024_1526) +
+			    ocelot_read(ocelot, SYS_COUNT_RX_1527_MAX);
+	stats->multicast = ocelot_read(ocelot, SYS_COUNT_RX_MULTICAST);
+	stats->rx_dropped = dev->stats.rx_dropped;
+
+	/* Get Tx stats */
+	stats->tx_bytes = ocelot_read(ocelot, SYS_COUNT_TX_OCTETS);
+	stats->tx_packets = ocelot_read(ocelot, SYS_COUNT_TX_64) +
+			    ocelot_read(ocelot, SYS_COUNT_TX_65_127) +
+			    ocelot_read(ocelot, SYS_COUNT_TX_128_511) +
+			    ocelot_read(ocelot, SYS_COUNT_TX_512_1023) +
+			    ocelot_read(ocelot, SYS_COUNT_TX_1024_1526) +
+			    ocelot_read(ocelot, SYS_COUNT_TX_1527_MAX);
+	stats->tx_dropped = ocelot_read(ocelot, SYS_COUNT_TX_DROPS) +
+			    ocelot_read(ocelot, SYS_COUNT_TX_AGING);
+	stats->collisions = ocelot_read(ocelot, SYS_COUNT_TX_COLLISION);
+}
+
+static int ocelot_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
+			  struct net_device *dev, const unsigned char *addr,
+			  u16 vid, u16 flags)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+
+	return ocelot_mact_learn(ocelot, port->chip_port, addr, vid,
+				 ENTRYTYPE_NORMAL);
+}
+
+static int ocelot_fdb_del(struct ndmsg *ndm, struct nlattr *tb[],
+			  struct net_device *dev,
+			  const unsigned char *addr, u16 vid)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+
+	return ocelot_mact_forget(ocelot, addr, vid);
+}
+
+struct ocelot_dump_ctx {
+	struct net_device *dev;
+	struct sk_buff *skb;
+	struct netlink_callback *cb;
+	int idx;
+};
+
+static int ocelot_fdb_do_dump(struct ocelot_mact_entry *entry,
+			      struct ocelot_dump_ctx *dump)
+{
+	u32 portid = NETLINK_CB(dump->cb->skb).portid;
+	u32 seq = dump->cb->nlh->nlmsg_seq;
+	struct nlmsghdr *nlh;
+	struct ndmsg *ndm;
+
+	if (dump->idx < dump->cb->args[2])
+		goto skip;
+
+	nlh = nlmsg_put(dump->skb, portid, seq, RTM_NEWNEIGH,
+			sizeof(*ndm), NLM_F_MULTI);
+	if (!nlh)
+		return -EMSGSIZE;
+
+	ndm = nlmsg_data(nlh);
+	ndm->ndm_family  = AF_BRIDGE;
+	ndm->ndm_pad1    = 0;
+	ndm->ndm_pad2    = 0;
+	ndm->ndm_flags   = NTF_SELF;
+	ndm->ndm_type    = 0;
+	ndm->ndm_ifindex = dump->dev->ifindex;
+	ndm->ndm_state   = NUD_REACHABLE;
+
+	if (nla_put(dump->skb, NDA_LLADDR, ETH_ALEN, entry->mac))
+		goto nla_put_failure;
+
+	if (entry->vid && nla_put_u16(dump->skb, NDA_VLAN, entry->vid))
+		goto nla_put_failure;
+
+	nlmsg_end(dump->skb, nlh);
+
+skip:
+	dump->idx++;
+	return 0;
+
+nla_put_failure:
+	nlmsg_cancel(dump->skb, nlh);
+	return -EMSGSIZE;
+}
+
+static inline int ocelot_mact_read(struct ocelot_port *port, int row, int col,
+				   struct ocelot_mact_entry *entry)
+{
+	struct ocelot *ocelot = port->ocelot;
+	char mac[ETH_ALEN];
+	u32 val, dst, macl, mach;
+
+	/* Set row and column to read from */
+	ocelot_field_write(ocelot, ANA_TABLES_MACTINDX_M_INDEX, row);
+	ocelot_field_write(ocelot, ANA_TABLES_MACTINDX_BUCKET, col);
+
+	/* Issue a read command */
+	ocelot_write(ocelot,
+		     ANA_TABLES_MACACCESS_MAC_TABLE_CMD(MACACCESS_CMD_READ),
+		     ANA_TABLES_MACACCESS);
+
+	if (ocelot_mact_wait_for_completion(ocelot))
+		return -ETIMEDOUT;
+
+	/* Read the entry flags */
+	val = ocelot_read(ocelot, ANA_TABLES_MACACCESS);
+	if (!(val & ANA_TABLES_MACACCESS_VALID))
+		return -EINVAL;
+
+	/* If the entry read has another port configured as its destination,
+	 * do not report it.
+	 */
+	dst = (val & ANA_TABLES_MACACCESS_DEST_IDX_M) >> 3;
+	if (dst != port->chip_port)
+		return -EINVAL;
+
+	/* Get the entry's MAC address and VLAN id */
+	macl = ocelot_read(ocelot, ANA_TABLES_MACLDATA);
+	mach = ocelot_read(ocelot, ANA_TABLES_MACHDATA);
+
+	mac[0] = (mach >> 8)  & 0xff;
+	mac[1] = (mach >> 0)  & 0xff;
+	mac[2] = (macl >> 24) & 0xff;
+	mac[3] = (macl >> 16) & 0xff;
+	mac[4] = (macl >> 8)  & 0xff;
+	mac[5] = (macl >> 0)  & 0xff;
+
+	entry->vid = (mach >> 16) & 0xfff;
+	ether_addr_copy(entry->mac, mac);
+
+	return 0;
+}
+
+static int ocelot_fdb_dump(struct sk_buff *skb, struct netlink_callback *cb,
+			   struct net_device *dev,
+			   struct net_device *filter_dev, int *idx)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	int i, j, ret = 0;
+	struct ocelot_dump_ctx dump = {
+		.dev = dev,
+		.skb = skb,
+		.cb = cb,
+		.idx = *idx,
+	};
+
+	struct ocelot_mact_entry entry;
+
+	/* Loop through all the mac tables entries. There are 1024 rows of 4
+	 * entries.
+	 */
+	for (i = 0; i < 1024; i++) {
+		for (j = 0; j < 4; j++) {
+			ret = ocelot_mact_read(port, i, j, &entry);
+			/* If the entry is invalid (wrong port, invalid...),
+			 * skip it.
+			 */
+			if (ret == -EINVAL)
+				continue;
+			else if (ret)
+				goto end;
+
+			ret = ocelot_fdb_do_dump(&entry, &dump);
+			if (ret)
+				goto end;
+		}
+	}
+
+end:
+	*idx = dump.idx;
+	return ret;
+}
+
+static const struct net_device_ops ocelot_port_netdev_ops = {
+	.ndo_open			= ocelot_port_open,
+	.ndo_stop			= ocelot_port_stop,
+	.ndo_start_xmit			= ocelot_port_xmit,
+	.ndo_set_rx_mode		= ocelot_set_rx_mode,
+	.ndo_get_phys_port_name		= ocelot_port_get_phys_port_name,
+	.ndo_set_mac_address		= ocelot_port_set_mac_address,
+	.ndo_get_stats64		= ocelot_get_stats64,
+	.ndo_fdb_add			= ocelot_fdb_add,
+	.ndo_fdb_del			= ocelot_fdb_del,
+	.ndo_fdb_dump			= ocelot_fdb_dump,
+};
+
+static void ocelot_get_strings(struct net_device *netdev, u32 sset, u8 *data)
+{
+	struct ocelot_port *port = netdev_priv(netdev);
+	struct ocelot *ocelot = port->ocelot;
+	int i;
+
+	if (sset != ETH_SS_STATS)
+		return;
+
+	for (i = 0; i < ocelot->num_stats; i++)
+		memcpy(data + i * ETH_GSTRING_LEN, ocelot->stats_layout[i].name,
+		       ETH_GSTRING_LEN);
+}
+
+static void ocelot_check_stats(struct work_struct *work)
+{
+	struct delayed_work *del_work = to_delayed_work(work);
+	struct ocelot *ocelot = container_of(del_work, struct ocelot, stats_work);
+	int i, j;
+
+	mutex_lock(&ocelot->stats_lock);
+
+	for (i = 0; i < ocelot->num_phys_ports; i++) {
+		/* Configure the port to read the stats from */
+		ocelot_write(ocelot, SYS_STAT_CFG_STAT_VIEW(i), SYS_STAT_CFG);
+
+		for (j = 0; j < ocelot->num_stats; j++) {
+			u32 val;
+			unsigned int idx = i * ocelot->num_stats + j;
+
+			val = ocelot_read_rix(ocelot, SYS_COUNT_RX_OCTETS,
+					      ocelot->stats_layout[j].offset);
+
+			if (val < (ocelot->stats[idx] & U32_MAX))
+				ocelot->stats[idx] += (u64)1 << 32;
+
+			ocelot->stats[idx] = (ocelot->stats[idx] &
+					      ~(u64)U32_MAX) + val;
+		}
+	}
+
+	cancel_delayed_work(&ocelot->stats_work);
+	queue_delayed_work(ocelot->stats_queue, &ocelot->stats_work,
+			   OCELOT_STATS_CHECK_DELAY);
+
+	mutex_unlock(&ocelot->stats_lock);
+}
+
+static void ocelot_get_ethtool_stats(struct net_device *dev,
+				     struct ethtool_stats *stats, u64 *data)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+	int i;
+
+	/* check and update now */
+	ocelot_check_stats(&ocelot->stats_work.work);
+
+	/* Copy all counters */
+	for (i = 0; i < ocelot->num_stats; i++)
+		*data++ = ocelot->stats[port->chip_port * ocelot->num_stats + i];
+}
+
+static int ocelot_get_sset_count(struct net_device *dev, int sset)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+
+	if (sset != ETH_SS_STATS)
+		return -EOPNOTSUPP;
+	return ocelot->num_stats;
+}
+
+static const struct ethtool_ops ocelot_ethtool_ops = {
+	.get_strings		= ocelot_get_strings,
+	.get_ethtool_stats	= ocelot_get_ethtool_stats,
+	.get_sset_count		= ocelot_get_sset_count,
+};
+
+static int ocelot_port_attr_get(struct net_device *dev,
+				struct switchdev_attr *attr)
+{
+	struct ocelot_port *ocelot_port = netdev_priv(dev);
+	struct ocelot *ocelot = ocelot_port->ocelot;
+
+	switch (attr->id) {
+	case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
+		attr->u.ppid.id_len = sizeof(ocelot->base_mac);
+		memcpy(&attr->u.ppid.id, &ocelot->base_mac,
+		       attr->u.ppid.id_len);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int ocelot_port_attr_stp_state_set(struct ocelot_port *ocelot_port,
+					  struct switchdev_trans *trans,
+					  u8 state)
+{
+	struct ocelot *ocelot = ocelot_port->ocelot;
+	u32 port_cfg;
+	int port, i;
+
+	if (switchdev_trans_ph_prepare(trans))
+		return 0;
+
+	if (!(BIT(ocelot_port->chip_port) & ocelot->bridge_mask))
+		return 0;
+
+	port_cfg = ocelot_read_gix(ocelot, ANA_PORT_PORT_CFG,
+				   ocelot_port->chip_port);
+
+	switch (state) {
+	case BR_STATE_FORWARDING:
+		ocelot->bridge_fwd_mask |= BIT(ocelot_port->chip_port);
+		/* Fallthrough */
+	case BR_STATE_LEARNING:
+		port_cfg |= ANA_PORT_PORT_CFG_LEARN_ENA;
+		break;
+
+	default:
+		port_cfg &= ~ANA_PORT_PORT_CFG_LEARN_ENA;
+		ocelot->bridge_fwd_mask &= ~BIT(ocelot_port->chip_port);
+		break;
+	}
+
+	ocelot_write_gix(ocelot, port_cfg, ANA_PORT_PORT_CFG,
+			 ocelot_port->chip_port);
+
+	/* Apply FWD mask. The loop is needed to add/remove the current port as
+	 * a source for the other ports.
+	 */
+	for (port = 0; port < ocelot->num_phys_ports; port++) {
+		if (ocelot->bridge_fwd_mask & BIT(port)) {
+			unsigned long mask = ocelot->bridge_fwd_mask & ~BIT(port);
+
+			for (i = 0; i < ocelot->num_phys_ports; i++) {
+				unsigned long bond_mask = ocelot->lags[i];
+
+				if (!bond_mask)
+					continue;
+
+				if (bond_mask & BIT(port)) {
+					mask &= ~bond_mask;
+					break;
+				}
+			}
+
+			ocelot_write_rix(ocelot,
+					 BIT(ocelot->num_phys_ports) | mask,
+					 ANA_PGID_PGID, PGID_SRC + port);
+		} else {
+			/* Only the CPU port, this is compatible with link
+			 * aggregation.
+			 */
+			ocelot_write_rix(ocelot,
+					 BIT(ocelot->num_phys_ports),
+					 ANA_PGID_PGID, PGID_SRC + port);
+		}
+	}
+
+	return 0;
+}
+
+static void ocelot_port_attr_ageing_set(struct ocelot_port *ocelot_port,
+					unsigned long ageing_clock_t)
+{
+	struct ocelot *ocelot = ocelot_port->ocelot;
+	unsigned long ageing_jiffies = clock_t_to_jiffies(ageing_clock_t);
+	u32 ageing_time = jiffies_to_msecs(ageing_jiffies) / 1000;
+
+	ocelot_write(ocelot, ANA_AUTOAGE_AGE_PERIOD(ageing_time / 2),
+		     ANA_AUTOAGE);
+}
+
+static void ocelot_port_attr_mc_set(struct ocelot_port *port, bool mc)
+{
+	struct ocelot *ocelot = port->ocelot;
+	u32 val = ocelot_read_gix(ocelot, ANA_PORT_CPU_FWD_CFG,
+				  port->chip_port);
+
+	if (mc)
+		val |= ANA_PORT_CPU_FWD_CFG_CPU_IGMP_REDIR_ENA |
+		       ANA_PORT_CPU_FWD_CFG_CPU_MLD_REDIR_ENA |
+		       ANA_PORT_CPU_FWD_CFG_CPU_IPMC_CTRL_COPY_ENA;
+	else
+		val &= ~(ANA_PORT_CPU_FWD_CFG_CPU_IGMP_REDIR_ENA |
+			 ANA_PORT_CPU_FWD_CFG_CPU_MLD_REDIR_ENA |
+			 ANA_PORT_CPU_FWD_CFG_CPU_IPMC_CTRL_COPY_ENA);
+
+	ocelot_write_gix(ocelot, val, ANA_PORT_CPU_FWD_CFG, port->chip_port);
+}
+
+static int ocelot_port_attr_set(struct net_device *dev,
+				const struct switchdev_attr *attr,
+				struct switchdev_trans *trans)
+{
+	struct ocelot_port *ocelot_port = netdev_priv(dev);
+	int err = 0;
+
+	switch (attr->id) {
+	case SWITCHDEV_ATTR_ID_PORT_STP_STATE:
+		ocelot_port_attr_stp_state_set(ocelot_port, trans,
+					       attr->u.stp_state);
+		break;
+	case SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME:
+		ocelot_port_attr_ageing_set(ocelot_port, attr->u.ageing_time);
+		break;
+	case SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED:
+		ocelot_port_attr_mc_set(ocelot_port, !attr->u.mc_disabled);
+		break;
+	default:
+		err = -EOPNOTSUPP;
+		break;
+	}
+
+	return err;
+}
+
+static struct ocelot_multicast *ocelot_multicast_get(struct ocelot *ocelot,
+						     const unsigned char *addr,
+						     u16 vid)
+{
+	struct ocelot_multicast *mc;
+
+	list_for_each_entry(mc, &ocelot->multicast, list) {
+		if (ether_addr_equal(mc->addr, addr) && mc->vid == vid)
+			return mc;
+	}
+
+	return NULL;
+}
+
+static int ocelot_port_obj_add_mdb(struct net_device *dev,
+				   const struct switchdev_obj_port_mdb *mdb,
+				   struct switchdev_trans *trans)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+	struct ocelot_multicast *mc;
+	unsigned char addr[ETH_ALEN];
+	u16 vid = mdb->vid;
+	bool new = false;
+
+	if (!vid)
+		vid = 1;
+
+	mc = ocelot_multicast_get(ocelot, mdb->addr, vid);
+	if (!mc) {
+		mc = devm_kzalloc(ocelot->dev, sizeof(*mc), GFP_KERNEL);
+		if (!mc)
+			return -ENOMEM;
+
+		memcpy(mc->addr, mdb->addr, ETH_ALEN);
+		mc->vid = vid;
+
+		list_add_tail(&mc->list, &ocelot->multicast);
+		new = true;
+	}
+
+	memcpy(addr, mc->addr, ETH_ALEN);
+	addr[0] = 0;
+
+	if (!new) {
+		addr[2] = mc->ports << 0;
+		addr[1] = mc->ports << 8;
+		ocelot_mact_forget(ocelot, addr, vid);
+	}
+
+	mc->ports |= BIT(port->chip_port);
+	addr[2] = mc->ports << 0;
+	addr[1] = mc->ports << 8;
+
+	return ocelot_mact_learn(ocelot, 0, addr, vid, ENTRYTYPE_MACv4);
+}
+
+static int ocelot_port_obj_del_mdb(struct net_device *dev,
+				   const struct switchdev_obj_port_mdb *mdb)
+{
+	struct ocelot_port *port = netdev_priv(dev);
+	struct ocelot *ocelot = port->ocelot;
+	struct ocelot_multicast *mc;
+	unsigned char addr[ETH_ALEN];
+	u16 vid = mdb->vid;
+
+	if (!vid)
+		vid = 1;
+
+	mc = ocelot_multicast_get(ocelot, mdb->addr, vid);
+	if (!mc)
+		return -ENOENT;
+
+	memcpy(addr, mc->addr, ETH_ALEN);
+	addr[2] = mc->ports << 0;
+	addr[1] = mc->ports << 8;
+	addr[0] = 0;
+	ocelot_mact_forget(ocelot, addr, vid);
+
+	mc->ports &= ~BIT(port->chip_port);
+	if (!mc->ports) {
+		list_del(&mc->list);
+		devm_kfree(ocelot->dev, mc);
+		return 0;
+	}
+
+	addr[2] = mc->ports << 0;
+	addr[1] = mc->ports << 8;
+
+	return ocelot_mact_learn(ocelot, 0, addr, vid, ENTRYTYPE_MACv4);
+}
+
+static int ocelot_port_obj_add(struct net_device *dev,
+			       const struct switchdev_obj *obj,
+			       struct switchdev_trans *trans)
+{
+	int ret = 0;
+
+	switch (obj->id) {
+	case SWITCHDEV_OBJ_ID_PORT_MDB:
+		ret = ocelot_port_obj_add_mdb(dev, SWITCHDEV_OBJ_PORT_MDB(obj),
+					      trans);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return ret;
+}
+
+static int ocelot_port_obj_del(struct net_device *dev,
+			       const struct switchdev_obj *obj)
+{
+	int ret = 0;
+
+	switch (obj->id) {
+	case SWITCHDEV_OBJ_ID_PORT_MDB:
+		ret = ocelot_port_obj_del_mdb(dev, SWITCHDEV_OBJ_PORT_MDB(obj));
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return ret;
+}
+
+static const struct switchdev_ops ocelot_port_switchdev_ops = {
+	.switchdev_port_attr_get	= ocelot_port_attr_get,
+	.switchdev_port_attr_set	= ocelot_port_attr_set,
+	.switchdev_port_obj_add		= ocelot_port_obj_add,
+	.switchdev_port_obj_del		= ocelot_port_obj_del,
+};
+
+static int ocelot_port_bridge_join(struct ocelot_port *ocelot_port,
+				   struct net_device *bridge)
+{
+	struct ocelot *ocelot = ocelot_port->ocelot;
+
+	if (!ocelot->bridge_mask) {
+		ocelot->hw_bridge_dev = bridge;
+	} else {
+		if (ocelot->hw_bridge_dev != bridge)
+			/* This is adding the port to a second bridge, this is
+			 * unsupported */
+			return -ENODEV;
+	}
+
+	ocelot->bridge_mask |= BIT(ocelot_port->chip_port);
+
+	return 0;
+}
+
+static void ocelot_port_bridge_leave(struct ocelot_port *ocelot_port,
+				     struct net_device *bridge)
+{
+	struct ocelot *ocelot = ocelot_port->ocelot;
+
+	ocelot->bridge_mask &= ~BIT(ocelot_port->chip_port);
+
+	if (!ocelot->bridge_mask)
+		ocelot->hw_bridge_dev = NULL;
+}
+
+/* Checks if the net_device instance given to us originate from our driver. */
+static bool ocelot_netdevice_dev_check(const struct net_device *dev)
+{
+	return dev->netdev_ops == &ocelot_port_netdev_ops;
+}
+
+static int ocelot_netdevice_port_event(struct net_device *dev,
+				       unsigned long event,
+				       struct netdev_notifier_changeupper_info *info)
+{
+	struct ocelot_port *ocelot_port = netdev_priv(dev);
+	int err = 0;
+
+	if (!ocelot_netdevice_dev_check(dev))
+		return 0;
+
+	switch (event) {
+	case NETDEV_CHANGEUPPER:
+		if (netif_is_bridge_master(info->upper_dev)) {
+			if (info->linking)
+				err = ocelot_port_bridge_join(ocelot_port,
+							      info->upper_dev);
+			else
+				ocelot_port_bridge_leave(ocelot_port,
+							 info->upper_dev);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return err;
+}
+
+static int ocelot_netdevice_event(struct notifier_block *unused,
+				  unsigned long event, void *ptr)
+{
+	struct netdev_notifier_changeupper_info *info = ptr;
+	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
+	int ret;
+
+	if (netif_is_lag_master(dev)) {
+		struct net_device *slave;
+		struct list_head *iter;
+
+		netdev_for_each_lower_dev(dev, slave, iter) {
+			ret = ocelot_netdevice_port_event(slave, event, info);
+			if (ret)
+				goto notify;
+		}
+	} else {
+		ret = ocelot_netdevice_port_event(dev, event, info);
+	}
+
+notify:
+	return notifier_from_errno(ret);
+}
+
+struct notifier_block ocelot_netdevice_nb __read_mostly = {
+	.notifier_call = ocelot_netdevice_event,
+};
+EXPORT_SYMBOL(ocelot_netdevice_nb);
+
+int ocelot_probe_port(struct ocelot *ocelot, u8 port,
+		      void __iomem *regs,
+		      struct phy_device *phy)
+{
+	struct ocelot_port *ocelot_port;
+	struct net_device *dev;
+	int err;
+
+	dev = alloc_etherdev(sizeof(struct ocelot_port));
+	if (!dev)
+		return -ENOMEM;
+	SET_NETDEV_DEV(dev, ocelot->dev);
+	ocelot_port = netdev_priv(dev);
+	ocelot_port->dev = dev;
+	ocelot_port->ocelot = ocelot;
+	ocelot_port->regs = regs;
+	ocelot_port->chip_port = port;
+	ocelot_port->phy = phy;
+	INIT_LIST_HEAD(&ocelot_port->mc);
+	ocelot->ports[port] = ocelot_port;
+
+	dev->netdev_ops = &ocelot_port_netdev_ops;
+	dev->ethtool_ops = &ocelot_ethtool_ops;
+	dev->switchdev_ops = &ocelot_port_switchdev_ops;
+
+	memcpy(dev->dev_addr, ocelot->base_mac, ETH_ALEN);
+	dev->dev_addr[ETH_ALEN - 1] += port;
+	ocelot_mact_learn(ocelot, PGID_CPU, dev->dev_addr, ocelot_port->pvid,
+			  ENTRYTYPE_LOCKED);
+
+	err = register_netdev(dev);
+	if (err) {
+		dev_err(ocelot->dev, "register_netdev failed\n");
+		goto err_register_netdev;
+	}
+
+	return 0;
+
+err_register_netdev:
+	free_netdev(dev);
+	return err;
+}
+EXPORT_SYMBOL(ocelot_probe_port);
+
+int ocelot_init(struct ocelot *ocelot)
+{
+	u32 port;
+	int i, cpu = ocelot->num_phys_ports;
+	char queue_name[32];
+
+	ocelot->stats = devm_kcalloc(ocelot->dev,
+				     ocelot->num_phys_ports * ocelot->num_stats,
+				     sizeof(u64), GFP_KERNEL);
+	if (!ocelot->stats)
+		return -ENOMEM;
+
+	mutex_init(&ocelot->stats_lock);
+	snprintf(queue_name, sizeof(queue_name), "%s-stats",
+		 dev_name(ocelot->dev));
+	ocelot->stats_queue = create_singlethread_workqueue(queue_name);
+	if (!ocelot->stats_queue)
+		return -ENOMEM;
+
+	ocelot_mact_init(ocelot);
+	ocelot_vlan_init(ocelot);
+
+	for (port = 0; port < ocelot->num_phys_ports; port++) {
+		/* Clear all counters (5 groups) */
+		ocelot_write(ocelot, SYS_STAT_CFG_STAT_VIEW(port) |
+				     SYS_STAT_CFG_STAT_CLEAR_SHOT(0x7f),
+			     SYS_STAT_CFG);
+	}
+
+	/* Only use S-Tag */
+	ocelot_write(ocelot, ETH_P_8021AD, SYS_VLAN_ETYPE_CFG);
+
+	/* Aggregation mode */
+	ocelot_write(ocelot, ANA_AGGR_CFG_AC_SMAC_ENA |
+			     ANA_AGGR_CFG_AC_DMAC_ENA |
+			     ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA |
+			     ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA, ANA_AGGR_CFG);
+
+	/* Set MAC age time to default value. The entry is aged after
+	 * 2*AGE_PERIOD
+	 */
+	ocelot_write(ocelot,
+		     ANA_AUTOAGE_AGE_PERIOD(BR_DEFAULT_AGEING_TIME / 2 / HZ),
+		     ANA_AUTOAGE);
+
+	/* Disable learning for frames discarded by VLAN ingress filtering */
+	regmap_field_write(ocelot->regfields[ANA_ADVLEARN_VLAN_CHK], 1);
+
+	/* Setup frame ageing - fixed value "2 sec" - in 6.5 us units */
+	ocelot_write(ocelot, SYS_FRM_AGING_AGE_TX_ENA |
+		     SYS_FRM_AGING_MAX_AGE(307692), SYS_FRM_AGING);
+
+	/* Setup flooding PGIDs */
+	ocelot_write_rix(ocelot, ANA_FLOODING_FLD_MULTICAST(PGID_MC) |
+			 ANA_FLOODING_FLD_BROADCAST(PGID_MC) |
+			 ANA_FLOODING_FLD_UNICAST(PGID_UC),
+			 ANA_FLOODING, 0);
+	ocelot_write(ocelot, ANA_FLOODING_IPMC_FLD_MC6_DATA(PGID_MCIPV6) |
+		     ANA_FLOODING_IPMC_FLD_MC6_CTRL(PGID_MC) |
+		     ANA_FLOODING_IPMC_FLD_MC4_DATA(PGID_MCIPV4) |
+		     ANA_FLOODING_IPMC_FLD_MC4_CTRL(PGID_MC),
+		     ANA_FLOODING_IPMC);
+
+	for (port = 0; port < ocelot->num_phys_ports; port++) {
+		/* Transmit the frame to the local port. */
+		ocelot_write_rix(ocelot, BIT(port), ANA_PGID_PGID, port);
+		/* Do not forward BPDU frames to the front ports. */
+		ocelot_write_gix(ocelot,
+				 ANA_PORT_CPU_FWD_BPDU_CFG_BPDU_REDIR_ENA(0xffff),
+				 ANA_PORT_CPU_FWD_BPDU_CFG,
+				 port);
+		/* Ensure bridging is disabled */
+		ocelot_write_rix(ocelot, 0, ANA_PGID_PGID, PGID_SRC + port);
+	}
+
+	/* Configure and enable the CPU port. */
+	ocelot_write_rix(ocelot, 0, ANA_PGID_PGID, cpu);
+	ocelot_write_rix(ocelot, BIT(cpu), ANA_PGID_PGID, PGID_CPU);
+	ocelot_write_gix(ocelot, ANA_PORT_PORT_CFG_RECV_ENA |
+			 ANA_PORT_PORT_CFG_PORTID_VAL(cpu),
+			 ANA_PORT_PORT_CFG, cpu);
+
+	/* Allow broadcast MAC frames. */
+	for (i = ocelot->num_phys_ports + 1; i < PGID_CPU; i++) {
+		u32 val = ANA_PGID_PGID_PGID(GENMASK(ocelot->num_phys_ports - 1, 0));
+
+		ocelot_write_rix(ocelot, val, ANA_PGID_PGID, i);
+	}
+	ocelot_write_rix(ocelot,
+			 ANA_PGID_PGID_PGID(GENMASK(ocelot->num_phys_ports, 0)),
+			 ANA_PGID_PGID, PGID_MC);
+	ocelot_write_rix(ocelot, 0, ANA_PGID_PGID, PGID_MCIPV4);
+	ocelot_write_rix(ocelot, 0, ANA_PGID_PGID, PGID_MCIPV6);
+
+	/* CPU port Injection/Extraction configuration */
+	ocelot_write_rix(ocelot, QSYS_SWITCH_PORT_MODE_INGRESS_DROP_MODE |
+			 QSYS_SWITCH_PORT_MODE_SCH_NEXT_CFG(1) |
+			 QSYS_SWITCH_PORT_MODE_PORT_ENA,
+			 QSYS_SWITCH_PORT_MODE, cpu);
+	ocelot_write_rix(ocelot, SYS_PORT_MODE_INCL_XTR_HDR(1) |
+			 SYS_PORT_MODE_INCL_INJ_HDR(1), SYS_PORT_MODE, cpu);
+	/* Allow manual injection via DEVCPU_QS registers, and byte swap these
+	 * registers endianness.
+	 */
+	ocelot_write_rix(ocelot, QS_INJ_GRP_CFG_BYTE_SWAP |
+			 QS_INJ_GRP_CFG_MODE(1), QS_INJ_GRP_CFG, 0);
+	ocelot_write_rix(ocelot, QS_XTR_GRP_CFG_BYTE_SWAP |
+			 QS_XTR_GRP_CFG_MODE(1), QS_XTR_GRP_CFG, 0);
+	ocelot_write(ocelot, ANA_CPUQ_CFG_CPUQ_MIRROR(2) |
+		     ANA_CPUQ_CFG_CPUQ_LRN(2) |
+		     ANA_CPUQ_CFG_CPUQ_MAC_COPY(2) |
+		     ANA_CPUQ_CFG_CPUQ_SRC_COPY(2) |
+		     ANA_CPUQ_CFG_CPUQ_LOCKED_PORTMOVE(2) |
+		     ANA_CPUQ_CFG_CPUQ_ALLBRIDGE(6) |
+		     ANA_CPUQ_CFG_CPUQ_IPMC_CTRL(6) |
+		     ANA_CPUQ_CFG_CPUQ_IGMP(6) |
+		     ANA_CPUQ_CFG_CPUQ_MLD(6), ANA_CPUQ_CFG);
+	for (i = 0; i < 16; i++)
+		ocelot_write_rix(ocelot, ANA_CPUQ_8021_CFG_CPUQ_GARP_VAL(6) |
+				 ANA_CPUQ_8021_CFG_CPUQ_BPDU_VAL(6),
+				 ANA_CPUQ_8021_CFG, i);
+
+	INIT_DELAYED_WORK(&ocelot->stats_work, ocelot_check_stats);
+	queue_delayed_work(ocelot->stats_queue, &ocelot->stats_work,
+			   OCELOT_STATS_CHECK_DELAY);
+	return 0;
+}
+EXPORT_SYMBOL(ocelot_init);
+
+void ocelot_deinit(struct ocelot *ocelot)
+{
+	destroy_workqueue(ocelot->stats_queue);
+	mutex_destroy(&ocelot->stats_lock);
+}
+EXPORT_SYMBOL(ocelot_deinit);
+
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/net/ethernet/mscc/ocelot.h b/drivers/net/ethernet/mscc/ocelot.h
new file mode 100644
index 000000000000..097bd12a10d4
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot.h
@@ -0,0 +1,572 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+
+#ifndef _MSCC_OCELOT_H_
+#define _MSCC_OCELOT_H_
+
+#include <linux/bitops.h>
+#include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include "ocelot_ana.h"
+#include "ocelot_dev.h"
+#include "ocelot_hsio.h"
+#include "ocelot_qsys.h"
+#include "ocelot_rew.h"
+#include "ocelot_sys.h"
+#include "ocelot_qs.h"
+
+#define PGID_AGGR    64
+#define PGID_SRC     80
+
+/* Reserved PGIDs */
+#define PGID_CPU     (PGID_AGGR - 5)
+#define PGID_UC      (PGID_AGGR - 4)
+#define PGID_MC      (PGID_AGGR - 3)
+#define PGID_MCIPV4  (PGID_AGGR - 2)
+#define PGID_MCIPV6  (PGID_AGGR - 1)
+
+#define OCELOT_BUFFER_CELL_SZ 60
+
+#define OCELOT_STATS_CHECK_DELAY (2 * HZ)
+
+#define IFH_LEN 4
+
+struct frame_info {
+	u32 len;
+	u16 port;
+	u16 vid;
+	u8 cpuq;
+	u8 tag_type;
+};
+
+#define IFH_INJ_BYPASS	BIT(31)
+#define IFH_INJ_POP_CNT_DISABLE (3 << 28)
+
+#define IFH_TAG_TYPE_C 0
+#define IFH_TAG_TYPE_S 1
+
+#define OCELOT_SPEED_2500 0
+#define OCELOT_SPEED_1000 1
+#define OCELOT_SPEED_100  2
+#define OCELOT_SPEED_10   3
+
+#define TARGET_OFFSET 24
+#define REG_MASK GENMASK(TARGET_OFFSET - 1, 0)
+#define REG(reg, offset) [reg & REG_MASK] = offset
+
+enum ocelot_target {
+	ANA = 1,
+	QS,
+	QSYS,
+	REW,
+	SYS,
+	HSIO,
+	TARGET_MAX,
+};
+
+enum ocelot_reg {
+	ANA_ADVLEARN = ANA << TARGET_OFFSET,
+	ANA_VLANMASK,
+	ANA_PORT_B_DOMAIN,
+	ANA_ANAGEFIL,
+	ANA_ANEVENTS,
+	ANA_STORMLIMIT_BURST,
+	ANA_STORMLIMIT_CFG,
+	ANA_ISOLATED_PORTS,
+	ANA_COMMUNITY_PORTS,
+	ANA_AUTOAGE,
+	ANA_MACTOPTIONS,
+	ANA_LEARNDISC,
+	ANA_AGENCTRL,
+	ANA_MIRRORPORTS,
+	ANA_EMIRRORPORTS,
+	ANA_FLOODING,
+	ANA_FLOODING_IPMC,
+	ANA_SFLOW_CFG,
+	ANA_PORT_MODE,
+	ANA_CUT_THRU_CFG,
+	ANA_PGID_PGID,
+	ANA_TABLES_ANMOVED,
+	ANA_TABLES_MACHDATA,
+	ANA_TABLES_MACLDATA,
+	ANA_TABLES_STREAMDATA,
+	ANA_TABLES_MACACCESS,
+	ANA_TABLES_MACTINDX,
+	ANA_TABLES_VLANACCESS,
+	ANA_TABLES_VLANTIDX,
+	ANA_TABLES_ISDXACCESS,
+	ANA_TABLES_ISDXTIDX,
+	ANA_TABLES_ENTRYLIM,
+	ANA_TABLES_PTP_ID_HIGH,
+	ANA_TABLES_PTP_ID_LOW,
+	ANA_TABLES_STREAMACCESS,
+	ANA_TABLES_STREAMTIDX,
+	ANA_TABLES_SEQ_HISTORY,
+	ANA_TABLES_SEQ_MASK,
+	ANA_TABLES_SFID_MASK,
+	ANA_TABLES_SFIDACCESS,
+	ANA_TABLES_SFIDTIDX,
+	ANA_MSTI_STATE,
+	ANA_OAM_UPM_LM_CNT,
+	ANA_SG_ACCESS_CTRL,
+	ANA_SG_CONFIG_REG_1,
+	ANA_SG_CONFIG_REG_2,
+	ANA_SG_CONFIG_REG_3,
+	ANA_SG_CONFIG_REG_4,
+	ANA_SG_CONFIG_REG_5,
+	ANA_SG_GCL_GS_CONFIG,
+	ANA_SG_GCL_TI_CONFIG,
+	ANA_SG_STATUS_REG_1,
+	ANA_SG_STATUS_REG_2,
+	ANA_SG_STATUS_REG_3,
+	ANA_PORT_VLAN_CFG,
+	ANA_PORT_DROP_CFG,
+	ANA_PORT_QOS_CFG,
+	ANA_PORT_VCAP_CFG,
+	ANA_PORT_VCAP_S1_KEY_CFG,
+	ANA_PORT_VCAP_S2_CFG,
+	ANA_PORT_PCP_DEI_MAP,
+	ANA_PORT_CPU_FWD_CFG,
+	ANA_PORT_CPU_FWD_BPDU_CFG,
+	ANA_PORT_CPU_FWD_GARP_CFG,
+	ANA_PORT_CPU_FWD_CCM_CFG,
+	ANA_PORT_PORT_CFG,
+	ANA_PORT_POL_CFG,
+	ANA_PORT_PTP_CFG,
+	ANA_PORT_PTP_DLY1_CFG,
+	ANA_PORT_PTP_DLY2_CFG,
+	ANA_PORT_SFID_CFG,
+	ANA_PFC_PFC_CFG,
+	ANA_PFC_PFC_TIMER,
+	ANA_IPT_OAM_MEP_CFG,
+	ANA_IPT_IPT,
+	ANA_PPT_PPT,
+	ANA_FID_MAP_FID_MAP,
+	ANA_AGGR_CFG,
+	ANA_CPUQ_CFG,
+	ANA_CPUQ_CFG2,
+	ANA_CPUQ_8021_CFG,
+	ANA_DSCP_CFG,
+	ANA_DSCP_REWR_CFG,
+	ANA_VCAP_RNG_TYPE_CFG,
+	ANA_VCAP_RNG_VAL_CFG,
+	ANA_VRAP_CFG,
+	ANA_VRAP_HDR_DATA,
+	ANA_VRAP_HDR_MASK,
+	ANA_DISCARD_CFG,
+	ANA_FID_CFG,
+	ANA_POL_PIR_CFG,
+	ANA_POL_CIR_CFG,
+	ANA_POL_MODE_CFG,
+	ANA_POL_PIR_STATE,
+	ANA_POL_CIR_STATE,
+	ANA_POL_STATE,
+	ANA_POL_FLOWC,
+	ANA_POL_HYST,
+	ANA_POL_MISC_CFG,
+	QS_XTR_GRP_CFG = QS << TARGET_OFFSET,
+	QS_XTR_RD,
+	QS_XTR_FRM_PRUNING,
+	QS_XTR_FLUSH,
+	QS_XTR_DATA_PRESENT,
+	QS_XTR_CFG,
+	QS_INJ_GRP_CFG,
+	QS_INJ_WR,
+	QS_INJ_CTRL,
+	QS_INJ_STATUS,
+	QS_INJ_ERR,
+	QS_INH_DBG,
+	QSYS_PORT_MODE = QSYS << TARGET_OFFSET,
+	QSYS_SWITCH_PORT_MODE,
+	QSYS_STAT_CNT_CFG,
+	QSYS_EEE_CFG,
+	QSYS_EEE_THRES,
+	QSYS_IGR_NO_SHARING,
+	QSYS_EGR_NO_SHARING,
+	QSYS_SW_STATUS,
+	QSYS_EXT_CPU_CFG,
+	QSYS_PAD_CFG,
+	QSYS_CPU_GROUP_MAP,
+	QSYS_QMAP,
+	QSYS_ISDX_SGRP,
+	QSYS_TIMED_FRAME_ENTRY,
+	QSYS_TFRM_MISC,
+	QSYS_TFRM_PORT_DLY,
+	QSYS_TFRM_TIMER_CFG_1,
+	QSYS_TFRM_TIMER_CFG_2,
+	QSYS_TFRM_TIMER_CFG_3,
+	QSYS_TFRM_TIMER_CFG_4,
+	QSYS_TFRM_TIMER_CFG_5,
+	QSYS_TFRM_TIMER_CFG_6,
+	QSYS_TFRM_TIMER_CFG_7,
+	QSYS_TFRM_TIMER_CFG_8,
+	QSYS_RED_PROFILE,
+	QSYS_RES_QOS_MODE,
+	QSYS_RES_CFG,
+	QSYS_RES_STAT,
+	QSYS_EGR_DROP_MODE,
+	QSYS_EQ_CTRL,
+	QSYS_EVENTS_CORE,
+	QSYS_QMAXSDU_CFG_0,
+	QSYS_QMAXSDU_CFG_1,
+	QSYS_QMAXSDU_CFG_2,
+	QSYS_QMAXSDU_CFG_3,
+	QSYS_QMAXSDU_CFG_4,
+	QSYS_QMAXSDU_CFG_5,
+	QSYS_QMAXSDU_CFG_6,
+	QSYS_QMAXSDU_CFG_7,
+	QSYS_PREEMPTION_CFG,
+	QSYS_CIR_CFG,
+	QSYS_EIR_CFG,
+	QSYS_SE_CFG,
+	QSYS_SE_DWRR_CFG,
+	QSYS_SE_CONNECT,
+	QSYS_SE_DLB_SENSE,
+	QSYS_CIR_STATE,
+	QSYS_EIR_STATE,
+	QSYS_SE_STATE,
+	QSYS_HSCH_MISC_CFG,
+	QSYS_TAG_CONFIG,
+	QSYS_TAS_PARAM_CFG_CTRL,
+	QSYS_PORT_MAX_SDU,
+	QSYS_PARAM_CFG_REG_1,
+	QSYS_PARAM_CFG_REG_2,
+	QSYS_PARAM_CFG_REG_3,
+	QSYS_PARAM_CFG_REG_4,
+	QSYS_PARAM_CFG_REG_5,
+	QSYS_GCL_CFG_REG_1,
+	QSYS_GCL_CFG_REG_2,
+	QSYS_PARAM_STATUS_REG_1,
+	QSYS_PARAM_STATUS_REG_2,
+	QSYS_PARAM_STATUS_REG_3,
+	QSYS_PARAM_STATUS_REG_4,
+	QSYS_PARAM_STATUS_REG_5,
+	QSYS_PARAM_STATUS_REG_6,
+	QSYS_PARAM_STATUS_REG_7,
+	QSYS_PARAM_STATUS_REG_8,
+	QSYS_PARAM_STATUS_REG_9,
+	QSYS_GCL_STATUS_REG_1,
+	QSYS_GCL_STATUS_REG_2,
+	REW_PORT_VLAN_CFG = REW << TARGET_OFFSET,
+	REW_TAG_CFG,
+	REW_PORT_CFG,
+	REW_DSCP_CFG,
+	REW_PCP_DEI_QOS_MAP_CFG,
+	REW_PTP_CFG,
+	REW_PTP_DLY1_CFG,
+	REW_RED_TAG_CFG,
+	REW_DSCP_REMAP_DP1_CFG,
+	REW_DSCP_REMAP_CFG,
+	REW_STAT_CFG,
+	REW_REW_STICKY,
+	REW_PPT,
+	SYS_COUNT_RX_OCTETS = SYS << TARGET_OFFSET,
+	SYS_COUNT_RX_UNICAST,
+	SYS_COUNT_RX_MULTICAST,
+	SYS_COUNT_RX_BROADCAST,
+	SYS_COUNT_RX_SHORTS,
+	SYS_COUNT_RX_FRAGMENTS,
+	SYS_COUNT_RX_JABBERS,
+	SYS_COUNT_RX_CRC_ALIGN_ERRS,
+	SYS_COUNT_RX_SYM_ERRS,
+	SYS_COUNT_RX_64,
+	SYS_COUNT_RX_65_127,
+	SYS_COUNT_RX_128_255,
+	SYS_COUNT_RX_256_1023,
+	SYS_COUNT_RX_1024_1526,
+	SYS_COUNT_RX_1527_MAX,
+	SYS_COUNT_RX_PAUSE,
+	SYS_COUNT_RX_CONTROL,
+	SYS_COUNT_RX_LONGS,
+	SYS_COUNT_RX_CLASSIFIED_DROPS,
+	SYS_COUNT_TX_OCTETS,
+	SYS_COUNT_TX_UNICAST,
+	SYS_COUNT_TX_MULTICAST,
+	SYS_COUNT_TX_BROADCAST,
+	SYS_COUNT_TX_COLLISION,
+	SYS_COUNT_TX_DROPS,
+	SYS_COUNT_TX_PAUSE,
+	SYS_COUNT_TX_64,
+	SYS_COUNT_TX_65_127,
+	SYS_COUNT_TX_128_511,
+	SYS_COUNT_TX_512_1023,
+	SYS_COUNT_TX_1024_1526,
+	SYS_COUNT_TX_1527_MAX,
+	SYS_COUNT_TX_AGING,
+	SYS_RESET_CFG,
+	SYS_SR_ETYPE_CFG,
+	SYS_VLAN_ETYPE_CFG,
+	SYS_PORT_MODE,
+	SYS_FRONT_PORT_MODE,
+	SYS_FRM_AGING,
+	SYS_STAT_CFG,
+	SYS_SW_STATUS,
+	SYS_MISC_CFG,
+	SYS_REW_MAC_HIGH_CFG,
+	SYS_REW_MAC_LOW_CFG,
+	SYS_TIMESTAMP_OFFSET,
+	SYS_CMID,
+	SYS_PAUSE_CFG,
+	SYS_PAUSE_TOT_CFG,
+	SYS_ATOP,
+	SYS_ATOP_TOT_CFG,
+	SYS_MAC_FC_CFG,
+	SYS_MMGT,
+	SYS_MMGT_FAST,
+	SYS_EVENTS_DIF,
+	SYS_EVENTS_CORE,
+	SYS_CNT,
+	SYS_PTP_STATUS,
+	SYS_PTP_TXSTAMP,
+	SYS_PTP_NXT,
+	SYS_PTP_CFG,
+	SYS_RAM_INIT,
+	SYS_CM_ADDR,
+	SYS_CM_DATA_WR,
+	SYS_CM_DATA_RD,
+	SYS_CM_OP,
+	SYS_CM_DATA,
+	HSIO_PLL5G_CFG0 = HSIO << TARGET_OFFSET,
+	HSIO_PLL5G_CFG1,
+	HSIO_PLL5G_CFG2,
+	HSIO_PLL5G_CFG3,
+	HSIO_PLL5G_CFG4,
+	HSIO_PLL5G_CFG5,
+	HSIO_PLL5G_CFG6,
+	HSIO_PLL5G_STATUS0,
+	HSIO_PLL5G_STATUS1,
+	HSIO_PLL5G_BIST_CFG0,
+	HSIO_PLL5G_BIST_CFG1,
+	HSIO_PLL5G_BIST_CFG2,
+	HSIO_PLL5G_BIST_STAT0,
+	HSIO_PLL5G_BIST_STAT1,
+	HSIO_RCOMP_CFG0,
+	HSIO_RCOMP_STATUS,
+	HSIO_SYNC_ETH_CFG,
+	HSIO_SYNC_ETH_PLL_CFG,
+	HSIO_S1G_DES_CFG,
+	HSIO_S1G_IB_CFG,
+	HSIO_S1G_OB_CFG,
+	HSIO_S1G_SER_CFG,
+	HSIO_S1G_COMMON_CFG,
+	HSIO_S1G_PLL_CFG,
+	HSIO_S1G_PLL_STATUS,
+	HSIO_S1G_DFT_CFG0,
+	HSIO_S1G_DFT_CFG1,
+	HSIO_S1G_DFT_CFG2,
+	HSIO_S1G_TP_CFG,
+	HSIO_S1G_RC_PLL_BIST_CFG,
+	HSIO_S1G_MISC_CFG,
+	HSIO_S1G_DFT_STATUS,
+	HSIO_S1G_MISC_STATUS,
+	HSIO_MCB_S1G_ADDR_CFG,
+	HSIO_S6G_DIG_CFG,
+	HSIO_S6G_DFT_CFG0,
+	HSIO_S6G_DFT_CFG1,
+	HSIO_S6G_DFT_CFG2,
+	HSIO_S6G_TP_CFG0,
+	HSIO_S6G_TP_CFG1,
+	HSIO_S6G_RC_PLL_BIST_CFG,
+	HSIO_S6G_MISC_CFG,
+	HSIO_S6G_OB_ANEG_CFG,
+	HSIO_S6G_DFT_STATUS,
+	HSIO_S6G_ERR_CNT,
+	HSIO_S6G_MISC_STATUS,
+	HSIO_S6G_DES_CFG,
+	HSIO_S6G_IB_CFG,
+	HSIO_S6G_IB_CFG1,
+	HSIO_S6G_IB_CFG2,
+	HSIO_S6G_IB_CFG3,
+	HSIO_S6G_IB_CFG4,
+	HSIO_S6G_IB_CFG5,
+	HSIO_S6G_OB_CFG,
+	HSIO_S6G_OB_CFG1,
+	HSIO_S6G_SER_CFG,
+	HSIO_S6G_COMMON_CFG,
+	HSIO_S6G_PLL_CFG,
+	HSIO_S6G_ACJTAG_CFG,
+	HSIO_S6G_GP_CFG,
+	HSIO_S6G_IB_STATUS0,
+	HSIO_S6G_IB_STATUS1,
+	HSIO_S6G_ACJTAG_STATUS,
+	HSIO_S6G_PLL_STATUS,
+	HSIO_S6G_REVID,
+	HSIO_MCB_S6G_ADDR_CFG,
+	HSIO_HW_CFG,
+	HSIO_HW_QSGMII_CFG,
+	HSIO_HW_QSGMII_STAT,
+	HSIO_CLK_CFG,
+	HSIO_TEMP_SENSOR_CTRL,
+	HSIO_TEMP_SENSOR_CFG,
+	HSIO_TEMP_SENSOR_STAT,
+};
+
+enum ocelot_regfield {
+	ANA_ADVLEARN_VLAN_CHK,
+	ANA_ADVLEARN_LEARN_MIRROR,
+	ANA_ANEVENTS_FLOOD_DISCARD,
+	ANA_ANEVENTS_MSTI_DROP,
+	ANA_ANEVENTS_ACLKILL,
+	ANA_ANEVENTS_ACLUSED,
+	ANA_ANEVENTS_AUTOAGE,
+	ANA_ANEVENTS_VS2TTL1,
+	ANA_ANEVENTS_STORM_DROP,
+	ANA_ANEVENTS_LEARN_DROP,
+	ANA_ANEVENTS_AGED_ENTRY,
+	ANA_ANEVENTS_CPU_LEARN_FAILED,
+	ANA_ANEVENTS_AUTO_LEARN_FAILED,
+	ANA_ANEVENTS_LEARN_REMOVE,
+	ANA_ANEVENTS_AUTO_LEARNED,
+	ANA_ANEVENTS_AUTO_MOVED,
+	ANA_ANEVENTS_DROPPED,
+	ANA_ANEVENTS_CLASSIFIED_DROP,
+	ANA_ANEVENTS_CLASSIFIED_COPY,
+	ANA_ANEVENTS_VLAN_DISCARD,
+	ANA_ANEVENTS_FWD_DISCARD,
+	ANA_ANEVENTS_MULTICAST_FLOOD,
+	ANA_ANEVENTS_UNICAST_FLOOD,
+	ANA_ANEVENTS_DEST_KNOWN,
+	ANA_ANEVENTS_BUCKET3_MATCH,
+	ANA_ANEVENTS_BUCKET2_MATCH,
+	ANA_ANEVENTS_BUCKET1_MATCH,
+	ANA_ANEVENTS_BUCKET0_MATCH,
+	ANA_ANEVENTS_CPU_OPERATION,
+	ANA_ANEVENTS_DMAC_LOOKUP,
+	ANA_ANEVENTS_SMAC_LOOKUP,
+	ANA_ANEVENTS_SEQ_GEN_ERR_0,
+	ANA_ANEVENTS_SEQ_GEN_ERR_1,
+	ANA_TABLES_MACACCESS_B_DOM,
+	ANA_TABLES_MACTINDX_BUCKET,
+	ANA_TABLES_MACTINDX_M_INDEX,
+	QSYS_TIMED_FRAME_ENTRY_TFRM_VLD,
+	QSYS_TIMED_FRAME_ENTRY_TFRM_FP,
+	QSYS_TIMED_FRAME_ENTRY_TFRM_PORTNO,
+	QSYS_TIMED_FRAME_ENTRY_TFRM_TM_SEL,
+	QSYS_TIMED_FRAME_ENTRY_TFRM_TM_T,
+	SYS_RESET_CFG_CORE_ENA,
+	SYS_RESET_CFG_MEM_ENA,
+	SYS_RESET_CFG_MEM_INIT,
+	REGFIELD_MAX
+};
+
+struct ocelot_multicast {
+	struct list_head list;
+	unsigned char addr[ETH_ALEN];
+	u16 vid;
+	u16 ports;
+};
+
+struct ocelot_port;
+
+struct ocelot_stat_layout {
+	u32 offset;
+	char name[ETH_GSTRING_LEN];
+};
+
+struct ocelot {
+	struct device *dev;
+
+	struct regmap *targets[TARGET_MAX];
+	struct regmap_field *regfields[REGFIELD_MAX];
+	const u32 *const *map;
+	const struct ocelot_stat_layout *stats_layout;
+	unsigned int num_stats;
+
+	u8 base_mac[ETH_ALEN];
+
+	struct net_device *hw_bridge_dev;
+	u16 bridge_mask;
+	u16 bridge_fwd_mask;
+
+	struct workqueue_struct *ocelot_owq;
+
+	int shared_queue_sz;
+
+	u8 num_phys_ports;
+	u8 num_cpu_ports;
+	struct ocelot_port **ports;
+
+	u16 lags[16];
+
+	/* Keep track of the vlan port masks */
+	u32 vlan_mask[VLAN_N_VID];
+
+	struct list_head multicast;
+
+	/* Workqueue to check statistics for overflow with its lock */
+	struct mutex stats_lock;
+	u64 *stats;
+	struct delayed_work stats_work;
+	struct workqueue_struct *stats_queue;
+};
+
+struct ocelot_port {
+	struct net_device *dev;
+	struct ocelot *ocelot;
+	struct phy_device *phy;
+	void __iomem *regs;
+	u8 chip_port;
+	/* Keep a track of the mc addresses added to the mac table, so that they
+	 * can be removed when needed.
+	 */
+	struct list_head mc;
+
+	/* Ingress default VLAN (pvid) */
+	u16 pvid;
+
+	/* Egress default VLAN (vid) */
+	u16 vid;
+
+	u8 vlan_aware;
+
+	u64 *stats;
+};
+
+u32 __ocelot_read_ix(struct ocelot *ocelot, u32 reg, u32 offset);
+#define ocelot_read_ix(ocelot, reg, gi, ri) __ocelot_read_ix(ocelot, reg, reg##_GSZ * (gi) + reg##_RSZ * (ri))
+#define ocelot_read_gix(ocelot, reg, gi) __ocelot_read_ix(ocelot, reg, reg##_GSZ * (gi))
+#define ocelot_read_rix(ocelot, reg, ri) __ocelot_read_ix(ocelot, reg, reg##_RSZ * (ri))
+#define ocelot_read(ocelot, reg) __ocelot_read_ix(ocelot, reg, 0)
+
+void __ocelot_write_ix(struct ocelot *ocelot, u32 val, u32 reg, u32 offset);
+#define ocelot_write_ix(ocelot, val, reg, gi, ri) __ocelot_write_ix(ocelot, val, reg, reg##_GSZ * (gi) + reg##_RSZ * (ri))
+#define ocelot_write_gix(ocelot, val, reg, gi) __ocelot_write_ix(ocelot, val, reg, reg##_GSZ * (gi))
+#define ocelot_write_rix(ocelot, val, reg, ri) __ocelot_write_ix(ocelot, val, reg, reg##_RSZ * (ri))
+#define ocelot_write(ocelot, val, reg) __ocelot_write_ix(ocelot, val, reg, 0)
+
+void __ocelot_rmw_ix(struct ocelot *ocelot, u32 val, u32 reg, u32 mask,
+		     u32 offset);
+#define ocelot_rmw_ix(ocelot, val, m, reg, gi, ri) __ocelot_rmw_ix(ocelot, val, m, reg, reg##_GSZ * (gi) + reg##_RSZ * (ri))
+#define ocelot_rmw_gix(ocelot, val, m, reg, gi) __ocelot_rmw_ix(ocelot, val, m, reg, reg##_GSZ * (gi))
+#define ocelot_rmw_rix(ocelot, val, m, reg, ri) __ocelot_rmw_ix(ocelot, val, m, reg, reg##_RSZ * (ri))
+#define ocelot_rmw(ocelot, val, m, reg) __ocelot_rmw_ix(ocelot, val, m, reg, 0)
+
+u32 ocelot_port_readl(struct ocelot_port *port, u32 reg);
+void ocelot_port_writel(struct ocelot_port *port, u32 val, u32 reg);
+
+int ocelot_regfields_init(struct ocelot *ocelot,
+			  const struct reg_field *const regfields);
+struct regmap *ocelot_io_platform_init(struct ocelot *ocelot,
+				       struct platform_device *pdev,
+				       const char *name);
+
+#define ocelot_field_write(ocelot, reg, val) regmap_field_write((ocelot)->regfields[(reg)], (val))
+#define ocelot_field_read(ocelot, reg, val) regmap_field_read((ocelot)->regfields[(reg)], (val))
+
+int ocelot_init(struct ocelot *ocelot);
+void ocelot_deinit(struct ocelot *ocelot);
+int ocelot_chip_init(struct ocelot *ocelot);
+int ocelot_probe_port(struct ocelot *ocelot, u8 port,
+		      void __iomem *regs,
+		      struct phy_device *phy);
+
+extern struct notifier_block ocelot_netdevice_nb;
+
+#endif
diff --git a/drivers/net/ethernet/mscc/ocelot_ana.h b/drivers/net/ethernet/mscc/ocelot_ana.h
new file mode 100644
index 000000000000..841c6ec22b64
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_ana.h
@@ -0,0 +1,625 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+
+#ifndef _MSCC_OCELOT_ANA_H_
+#define _MSCC_OCELOT_ANA_H_
+
+#define ANA_ANAGEFIL_B_DOM_EN                             BIT(22)
+#define ANA_ANAGEFIL_B_DOM_VAL                            BIT(21)
+#define ANA_ANAGEFIL_AGE_LOCKED                           BIT(20)
+#define ANA_ANAGEFIL_PID_EN                               BIT(19)
+#define ANA_ANAGEFIL_PID_VAL(x)                           (((x) << 14) & GENMASK(18, 14))
+#define ANA_ANAGEFIL_PID_VAL_M                            GENMASK(18, 14)
+#define ANA_ANAGEFIL_PID_VAL_X(x)                         (((x) & GENMASK(18, 14)) >> 14)
+#define ANA_ANAGEFIL_VID_EN                               BIT(13)
+#define ANA_ANAGEFIL_VID_VAL(x)                           ((x) & GENMASK(12, 0))
+#define ANA_ANAGEFIL_VID_VAL_M                            GENMASK(12, 0)
+
+#define ANA_STORMLIMIT_CFG_RSZ                            0x4
+
+#define ANA_STORMLIMIT_CFG_STORM_RATE(x)                  (((x) << 3) & GENMASK(6, 3))
+#define ANA_STORMLIMIT_CFG_STORM_RATE_M                   GENMASK(6, 3)
+#define ANA_STORMLIMIT_CFG_STORM_RATE_X(x)                (((x) & GENMASK(6, 3)) >> 3)
+#define ANA_STORMLIMIT_CFG_STORM_UNIT                     BIT(2)
+#define ANA_STORMLIMIT_CFG_STORM_MODE(x)                  ((x) & GENMASK(1, 0))
+#define ANA_STORMLIMIT_CFG_STORM_MODE_M                   GENMASK(1, 0)
+
+#define ANA_AUTOAGE_AGE_FAST                              BIT(21)
+#define ANA_AUTOAGE_AGE_PERIOD(x)                         (((x) << 1) & GENMASK(20, 1))
+#define ANA_AUTOAGE_AGE_PERIOD_M                          GENMASK(20, 1)
+#define ANA_AUTOAGE_AGE_PERIOD_X(x)                       (((x) & GENMASK(20, 1)) >> 1)
+#define ANA_AUTOAGE_AUTOAGE_LOCKED                        BIT(0)
+
+#define ANA_MACTOPTIONS_REDUCED_TABLE                     BIT(1)
+#define ANA_MACTOPTIONS_SHADOW                            BIT(0)
+
+#define ANA_AGENCTRL_FID_MASK(x)                          (((x) << 12) & GENMASK(23, 12))
+#define ANA_AGENCTRL_FID_MASK_M                           GENMASK(23, 12)
+#define ANA_AGENCTRL_FID_MASK_X(x)                        (((x) & GENMASK(23, 12)) >> 12)
+#define ANA_AGENCTRL_IGNORE_DMAC_FLAGS                    BIT(11)
+#define ANA_AGENCTRL_IGNORE_SMAC_FLAGS                    BIT(10)
+#define ANA_AGENCTRL_FLOOD_SPECIAL                        BIT(9)
+#define ANA_AGENCTRL_FLOOD_IGNORE_VLAN                    BIT(8)
+#define ANA_AGENCTRL_MIRROR_CPU                           BIT(7)
+#define ANA_AGENCTRL_LEARN_CPU_COPY                       BIT(6)
+#define ANA_AGENCTRL_LEARN_FWD_KILL                       BIT(5)
+#define ANA_AGENCTRL_LEARN_IGNORE_VLAN                    BIT(4)
+#define ANA_AGENCTRL_CPU_CPU_KILL_ENA                     BIT(3)
+#define ANA_AGENCTRL_GREEN_COUNT_MODE                     BIT(2)
+#define ANA_AGENCTRL_YELLOW_COUNT_MODE                    BIT(1)
+#define ANA_AGENCTRL_RED_COUNT_MODE                       BIT(0)
+
+#define ANA_FLOODING_RSZ                                  0x4
+
+#define ANA_FLOODING_FLD_UNICAST(x)                       (((x) << 12) & GENMASK(17, 12))
+#define ANA_FLOODING_FLD_UNICAST_M                        GENMASK(17, 12)
+#define ANA_FLOODING_FLD_UNICAST_X(x)                     (((x) & GENMASK(17, 12)) >> 12)
+#define ANA_FLOODING_FLD_BROADCAST(x)                     (((x) << 6) & GENMASK(11, 6))
+#define ANA_FLOODING_FLD_BROADCAST_M                      GENMASK(11, 6)
+#define ANA_FLOODING_FLD_BROADCAST_X(x)                   (((x) & GENMASK(11, 6)) >> 6)
+#define ANA_FLOODING_FLD_MULTICAST(x)                     ((x) & GENMASK(5, 0))
+#define ANA_FLOODING_FLD_MULTICAST_M                      GENMASK(5, 0)
+
+#define ANA_FLOODING_IPMC_FLD_MC4_CTRL(x)                 (((x) << 18) & GENMASK(23, 18))
+#define ANA_FLOODING_IPMC_FLD_MC4_CTRL_M                  GENMASK(23, 18)
+#define ANA_FLOODING_IPMC_FLD_MC4_CTRL_X(x)               (((x) & GENMASK(23, 18)) >> 18)
+#define ANA_FLOODING_IPMC_FLD_MC4_DATA(x)                 (((x) << 12) & GENMASK(17, 12))
+#define ANA_FLOODING_IPMC_FLD_MC4_DATA_M                  GENMASK(17, 12)
+#define ANA_FLOODING_IPMC_FLD_MC4_DATA_X(x)               (((x) & GENMASK(17, 12)) >> 12)
+#define ANA_FLOODING_IPMC_FLD_MC6_CTRL(x)                 (((x) << 6) & GENMASK(11, 6))
+#define ANA_FLOODING_IPMC_FLD_MC6_CTRL_M                  GENMASK(11, 6)
+#define ANA_FLOODING_IPMC_FLD_MC6_CTRL_X(x)               (((x) & GENMASK(11, 6)) >> 6)
+#define ANA_FLOODING_IPMC_FLD_MC6_DATA(x)                 ((x) & GENMASK(5, 0))
+#define ANA_FLOODING_IPMC_FLD_MC6_DATA_M                  GENMASK(5, 0)
+
+#define ANA_SFLOW_CFG_RSZ                                 0x4
+
+#define ANA_SFLOW_CFG_SF_RATE(x)                          (((x) << 2) & GENMASK(13, 2))
+#define ANA_SFLOW_CFG_SF_RATE_M                           GENMASK(13, 2)
+#define ANA_SFLOW_CFG_SF_RATE_X(x)                        (((x) & GENMASK(13, 2)) >> 2)
+#define ANA_SFLOW_CFG_SF_SAMPLE_RX                        BIT(1)
+#define ANA_SFLOW_CFG_SF_SAMPLE_TX                        BIT(0)
+
+#define ANA_PORT_MODE_RSZ                                 0x4
+
+#define ANA_PORT_MODE_REDTAG_PARSE_CFG                    BIT(3)
+#define ANA_PORT_MODE_VLAN_PARSE_CFG(x)                   (((x) << 1) & GENMASK(2, 1))
+#define ANA_PORT_MODE_VLAN_PARSE_CFG_M                    GENMASK(2, 1)
+#define ANA_PORT_MODE_VLAN_PARSE_CFG_X(x)                 (((x) & GENMASK(2, 1)) >> 1)
+#define ANA_PORT_MODE_L3_PARSE_CFG                        BIT(0)
+
+#define ANA_CUT_THRU_CFG_RSZ                              0x4
+
+#define ANA_PGID_PGID_RSZ                                 0x4
+
+#define ANA_PGID_PGID_PGID(x)                             ((x) & GENMASK(11, 0))
+#define ANA_PGID_PGID_PGID_M                              GENMASK(11, 0)
+#define ANA_PGID_PGID_CPUQ_DST_PGID(x)                    (((x) << 27) & GENMASK(29, 27))
+#define ANA_PGID_PGID_CPUQ_DST_PGID_M                     GENMASK(29, 27)
+#define ANA_PGID_PGID_CPUQ_DST_PGID_X(x)                  (((x) & GENMASK(29, 27)) >> 27)
+
+#define ANA_TABLES_MACHDATA_VID(x)                        (((x) << 16) & GENMASK(28, 16))
+#define ANA_TABLES_MACHDATA_VID_M                         GENMASK(28, 16)
+#define ANA_TABLES_MACHDATA_VID_X(x)                      (((x) & GENMASK(28, 16)) >> 16)
+#define ANA_TABLES_MACHDATA_MACHDATA(x)                   ((x) & GENMASK(15, 0))
+#define ANA_TABLES_MACHDATA_MACHDATA_M                    GENMASK(15, 0)
+
+#define ANA_TABLES_STREAMDATA_SSID_VALID                  BIT(16)
+#define ANA_TABLES_STREAMDATA_SSID(x)                     (((x) << 9) & GENMASK(15, 9))
+#define ANA_TABLES_STREAMDATA_SSID_M                      GENMASK(15, 9)
+#define ANA_TABLES_STREAMDATA_SSID_X(x)                   (((x) & GENMASK(15, 9)) >> 9)
+#define ANA_TABLES_STREAMDATA_SFID_VALID                  BIT(8)
+#define ANA_TABLES_STREAMDATA_SFID(x)                     ((x) & GENMASK(7, 0))
+#define ANA_TABLES_STREAMDATA_SFID_M                      GENMASK(7, 0)
+
+#define ANA_TABLES_MACACCESS_MAC_CPU_COPY                 BIT(15)
+#define ANA_TABLES_MACACCESS_SRC_KILL                     BIT(14)
+#define ANA_TABLES_MACACCESS_IGNORE_VLAN                  BIT(13)
+#define ANA_TABLES_MACACCESS_AGED_FLAG                    BIT(12)
+#define ANA_TABLES_MACACCESS_VALID                        BIT(11)
+#define ANA_TABLES_MACACCESS_ENTRYTYPE(x)                 (((x) << 9) & GENMASK(10, 9))
+#define ANA_TABLES_MACACCESS_ENTRYTYPE_M                  GENMASK(10, 9)
+#define ANA_TABLES_MACACCESS_ENTRYTYPE_X(x)               (((x) & GENMASK(10, 9)) >> 9)
+#define ANA_TABLES_MACACCESS_DEST_IDX(x)                  (((x) << 3) & GENMASK(8, 3))
+#define ANA_TABLES_MACACCESS_DEST_IDX_M                   GENMASK(8, 3)
+#define ANA_TABLES_MACACCESS_DEST_IDX_X(x)                (((x) & GENMASK(8, 3)) >> 3)
+#define ANA_TABLES_MACACCESS_MAC_TABLE_CMD(x)             ((x) & GENMASK(2, 0))
+#define ANA_TABLES_MACACCESS_MAC_TABLE_CMD_M              GENMASK(2, 0)
+#define MACACCESS_CMD_IDLE                     0
+#define MACACCESS_CMD_LEARN                    1
+#define MACACCESS_CMD_FORGET                   2
+#define MACACCESS_CMD_AGE                      3
+#define MACACCESS_CMD_GET_NEXT                 4
+#define MACACCESS_CMD_INIT                     5
+#define MACACCESS_CMD_READ                     6
+#define MACACCESS_CMD_WRITE                    7
+
+#define ANA_TABLES_VLANACCESS_VLAN_PORT_MASK(x)           (((x) << 2) & GENMASK(13, 2))
+#define ANA_TABLES_VLANACCESS_VLAN_PORT_MASK_M            GENMASK(13, 2)
+#define ANA_TABLES_VLANACCESS_VLAN_PORT_MASK_X(x)         (((x) & GENMASK(13, 2)) >> 2)
+#define ANA_TABLES_VLANACCESS_VLAN_TBL_CMD(x)             ((x) & GENMASK(1, 0))
+#define ANA_TABLES_VLANACCESS_VLAN_TBL_CMD_M              GENMASK(1, 0)
+#define ANA_TABLES_VLANACCESS_CMD_IDLE                    0x0
+#define ANA_TABLES_VLANACCESS_CMD_WRITE                   0x2
+#define ANA_TABLES_VLANACCESS_CMD_INIT                    0x3
+
+#define ANA_TABLES_VLANTIDX_VLAN_SEC_FWD_ENA              BIT(17)
+#define ANA_TABLES_VLANTIDX_VLAN_FLOOD_DIS                BIT(16)
+#define ANA_TABLES_VLANTIDX_VLAN_PRIV_VLAN                BIT(15)
+#define ANA_TABLES_VLANTIDX_VLAN_LEARN_DISABLED           BIT(14)
+#define ANA_TABLES_VLANTIDX_VLAN_MIRROR                   BIT(13)
+#define ANA_TABLES_VLANTIDX_VLAN_SRC_CHK                  BIT(12)
+#define ANA_TABLES_VLANTIDX_V_INDEX(x)                    ((x) & GENMASK(11, 0))
+#define ANA_TABLES_VLANTIDX_V_INDEX_M                     GENMASK(11, 0)
+
+#define ANA_TABLES_ISDXACCESS_ISDX_PORT_MASK(x)           (((x) << 2) & GENMASK(8, 2))
+#define ANA_TABLES_ISDXACCESS_ISDX_PORT_MASK_M            GENMASK(8, 2)
+#define ANA_TABLES_ISDXACCESS_ISDX_PORT_MASK_X(x)         (((x) & GENMASK(8, 2)) >> 2)
+#define ANA_TABLES_ISDXACCESS_ISDX_TBL_CMD(x)             ((x) & GENMASK(1, 0))
+#define ANA_TABLES_ISDXACCESS_ISDX_TBL_CMD_M              GENMASK(1, 0)
+
+#define ANA_TABLES_ISDXTIDX_ISDX_SDLBI(x)                 (((x) << 21) & GENMASK(28, 21))
+#define ANA_TABLES_ISDXTIDX_ISDX_SDLBI_M                  GENMASK(28, 21)
+#define ANA_TABLES_ISDXTIDX_ISDX_SDLBI_X(x)               (((x) & GENMASK(28, 21)) >> 21)
+#define ANA_TABLES_ISDXTIDX_ISDX_MSTI(x)                  (((x) << 15) & GENMASK(20, 15))
+#define ANA_TABLES_ISDXTIDX_ISDX_MSTI_M                   GENMASK(20, 15)
+#define ANA_TABLES_ISDXTIDX_ISDX_MSTI_X(x)                (((x) & GENMASK(20, 15)) >> 15)
+#define ANA_TABLES_ISDXTIDX_ISDX_ES0_KEY_ENA              BIT(14)
+#define ANA_TABLES_ISDXTIDX_ISDX_FORCE_ENA                BIT(10)
+#define ANA_TABLES_ISDXTIDX_ISDX_INDEX(x)                 ((x) & GENMASK(7, 0))
+#define ANA_TABLES_ISDXTIDX_ISDX_INDEX_M                  GENMASK(7, 0)
+
+#define ANA_TABLES_ENTRYLIM_RSZ                           0x4
+
+#define ANA_TABLES_ENTRYLIM_ENTRYLIM(x)                   (((x) << 14) & GENMASK(17, 14))
+#define ANA_TABLES_ENTRYLIM_ENTRYLIM_M                    GENMASK(17, 14)
+#define ANA_TABLES_ENTRYLIM_ENTRYLIM_X(x)                 (((x) & GENMASK(17, 14)) >> 14)
+#define ANA_TABLES_ENTRYLIM_ENTRYSTAT(x)                  ((x) & GENMASK(13, 0))
+#define ANA_TABLES_ENTRYLIM_ENTRYSTAT_M                   GENMASK(13, 0)
+
+#define ANA_TABLES_STREAMACCESS_GEN_REC_SEQ_NUM(x)        (((x) << 4) & GENMASK(31, 4))
+#define ANA_TABLES_STREAMACCESS_GEN_REC_SEQ_NUM_M         GENMASK(31, 4)
+#define ANA_TABLES_STREAMACCESS_GEN_REC_SEQ_NUM_X(x)      (((x) & GENMASK(31, 4)) >> 4)
+#define ANA_TABLES_STREAMACCESS_SEQ_GEN_REC_ENA           BIT(3)
+#define ANA_TABLES_STREAMACCESS_GEN_REC_TYPE              BIT(2)
+#define ANA_TABLES_STREAMACCESS_STREAM_TBL_CMD(x)         ((x) & GENMASK(1, 0))
+#define ANA_TABLES_STREAMACCESS_STREAM_TBL_CMD_M          GENMASK(1, 0)
+
+#define ANA_TABLES_STREAMTIDX_SEQ_GEN_ERR_STATUS(x)       (((x) << 30) & GENMASK(31, 30))
+#define ANA_TABLES_STREAMTIDX_SEQ_GEN_ERR_STATUS_M        GENMASK(31, 30)
+#define ANA_TABLES_STREAMTIDX_SEQ_GEN_ERR_STATUS_X(x)     (((x) & GENMASK(31, 30)) >> 30)
+#define ANA_TABLES_STREAMTIDX_S_INDEX(x)                  (((x) << 16) & GENMASK(22, 16))
+#define ANA_TABLES_STREAMTIDX_S_INDEX_M                   GENMASK(22, 16)
+#define ANA_TABLES_STREAMTIDX_S_INDEX_X(x)                (((x) & GENMASK(22, 16)) >> 16)
+#define ANA_TABLES_STREAMTIDX_FORCE_SF_BEHAVIOUR          BIT(14)
+#define ANA_TABLES_STREAMTIDX_SEQ_HISTORY_LEN(x)          (((x) << 8) & GENMASK(13, 8))
+#define ANA_TABLES_STREAMTIDX_SEQ_HISTORY_LEN_M           GENMASK(13, 8)
+#define ANA_TABLES_STREAMTIDX_SEQ_HISTORY_LEN_X(x)        (((x) & GENMASK(13, 8)) >> 8)
+#define ANA_TABLES_STREAMTIDX_RESET_ON_ROGUE              BIT(7)
+#define ANA_TABLES_STREAMTIDX_REDTAG_POP                  BIT(6)
+#define ANA_TABLES_STREAMTIDX_STREAM_SPLIT                BIT(5)
+#define ANA_TABLES_STREAMTIDX_SEQ_SPACE_LOG2(x)           ((x) & GENMASK(4, 0))
+#define ANA_TABLES_STREAMTIDX_SEQ_SPACE_LOG2_M            GENMASK(4, 0)
+
+#define ANA_TABLES_SEQ_MASK_SPLIT_MASK(x)                 (((x) << 16) & GENMASK(22, 16))
+#define ANA_TABLES_SEQ_MASK_SPLIT_MASK_M                  GENMASK(22, 16)
+#define ANA_TABLES_SEQ_MASK_SPLIT_MASK_X(x)               (((x) & GENMASK(22, 16)) >> 16)
+#define ANA_TABLES_SEQ_MASK_INPUT_PORT_MASK(x)            ((x) & GENMASK(6, 0))
+#define ANA_TABLES_SEQ_MASK_INPUT_PORT_MASK_M             GENMASK(6, 0)
+
+#define ANA_TABLES_SFID_MASK_IGR_PORT_MASK(x)             (((x) << 1) & GENMASK(7, 1))
+#define ANA_TABLES_SFID_MASK_IGR_PORT_MASK_M              GENMASK(7, 1)
+#define ANA_TABLES_SFID_MASK_IGR_PORT_MASK_X(x)           (((x) & GENMASK(7, 1)) >> 1)
+#define ANA_TABLES_SFID_MASK_IGR_SRCPORT_MATCH_ENA        BIT(0)
+
+#define ANA_TABLES_SFIDACCESS_IGR_PRIO_MATCH_ENA          BIT(22)
+#define ANA_TABLES_SFIDACCESS_IGR_PRIO(x)                 (((x) << 19) & GENMASK(21, 19))
+#define ANA_TABLES_SFIDACCESS_IGR_PRIO_M                  GENMASK(21, 19)
+#define ANA_TABLES_SFIDACCESS_IGR_PRIO_X(x)               (((x) & GENMASK(21, 19)) >> 19)
+#define ANA_TABLES_SFIDACCESS_FORCE_BLOCK                 BIT(18)
+#define ANA_TABLES_SFIDACCESS_MAX_SDU_LEN(x)              (((x) << 2) & GENMASK(17, 2))
+#define ANA_TABLES_SFIDACCESS_MAX_SDU_LEN_M               GENMASK(17, 2)
+#define ANA_TABLES_SFIDACCESS_MAX_SDU_LEN_X(x)            (((x) & GENMASK(17, 2)) >> 2)
+#define ANA_TABLES_SFIDACCESS_SFID_TBL_CMD(x)             ((x) & GENMASK(1, 0))
+#define ANA_TABLES_SFIDACCESS_SFID_TBL_CMD_M              GENMASK(1, 0)
+
+#define ANA_TABLES_SFIDTIDX_SGID_VALID                    BIT(26)
+#define ANA_TABLES_SFIDTIDX_SGID(x)                       (((x) << 18) & GENMASK(25, 18))
+#define ANA_TABLES_SFIDTIDX_SGID_M                        GENMASK(25, 18)
+#define ANA_TABLES_SFIDTIDX_SGID_X(x)                     (((x) & GENMASK(25, 18)) >> 18)
+#define ANA_TABLES_SFIDTIDX_POL_ENA                       BIT(17)
+#define ANA_TABLES_SFIDTIDX_POL_IDX(x)                    (((x) << 8) & GENMASK(16, 8))
+#define ANA_TABLES_SFIDTIDX_POL_IDX_M                     GENMASK(16, 8)
+#define ANA_TABLES_SFIDTIDX_POL_IDX_X(x)                  (((x) & GENMASK(16, 8)) >> 8)
+#define ANA_TABLES_SFIDTIDX_SFID_INDEX(x)                 ((x) & GENMASK(7, 0))
+#define ANA_TABLES_SFIDTIDX_SFID_INDEX_M                  GENMASK(7, 0)
+
+#define ANA_MSTI_STATE_RSZ                                0x4
+
+#define ANA_OAM_UPM_LM_CNT_RSZ                            0x4
+
+#define ANA_SG_ACCESS_CTRL_SGID(x)                        ((x) & GENMASK(7, 0))
+#define ANA_SG_ACCESS_CTRL_SGID_M                         GENMASK(7, 0)
+#define ANA_SG_ACCESS_CTRL_CONFIG_CHANGE                  BIT(28)
+
+#define ANA_SG_CONFIG_REG_3_BASE_TIME_SEC_MSB(x)          ((x) & GENMASK(15, 0))
+#define ANA_SG_CONFIG_REG_3_BASE_TIME_SEC_MSB_M           GENMASK(15, 0)
+#define ANA_SG_CONFIG_REG_3_LIST_LENGTH(x)                (((x) << 16) & GENMASK(18, 16))
+#define ANA_SG_CONFIG_REG_3_LIST_LENGTH_M                 GENMASK(18, 16)
+#define ANA_SG_CONFIG_REG_3_LIST_LENGTH_X(x)              (((x) & GENMASK(18, 16)) >> 16)
+#define ANA_SG_CONFIG_REG_3_GATE_ENABLE                   BIT(20)
+#define ANA_SG_CONFIG_REG_3_INIT_IPS(x)                   (((x) << 24) & GENMASK(27, 24))
+#define ANA_SG_CONFIG_REG_3_INIT_IPS_M                    GENMASK(27, 24)
+#define ANA_SG_CONFIG_REG_3_INIT_IPS_X(x)                 (((x) & GENMASK(27, 24)) >> 24)
+#define ANA_SG_CONFIG_REG_3_INIT_GATE_STATE               BIT(28)
+
+#define ANA_SG_GCL_GS_CONFIG_RSZ                          0x4
+
+#define ANA_SG_GCL_GS_CONFIG_IPS(x)                       ((x) & GENMASK(3, 0))
+#define ANA_SG_GCL_GS_CONFIG_IPS_M                        GENMASK(3, 0)
+#define ANA_SG_GCL_GS_CONFIG_GATE_STATE                   BIT(4)
+
+#define ANA_SG_GCL_TI_CONFIG_RSZ                          0x4
+
+#define ANA_SG_STATUS_REG_3_CFG_CHG_TIME_SEC_MSB(x)       ((x) & GENMASK(15, 0))
+#define ANA_SG_STATUS_REG_3_CFG_CHG_TIME_SEC_MSB_M        GENMASK(15, 0)
+#define ANA_SG_STATUS_REG_3_GATE_STATE                    BIT(16)
+#define ANA_SG_STATUS_REG_3_IPS(x)                        (((x) << 20) & GENMASK(23, 20))
+#define ANA_SG_STATUS_REG_3_IPS_M                         GENMASK(23, 20)
+#define ANA_SG_STATUS_REG_3_IPS_X(x)                      (((x) & GENMASK(23, 20)) >> 20)
+#define ANA_SG_STATUS_REG_3_CONFIG_PENDING                BIT(24)
+
+#define ANA_PORT_VLAN_CFG_GSZ                             0x100
+
+#define ANA_PORT_VLAN_CFG_VLAN_VID_AS_ISDX                BIT(21)
+#define ANA_PORT_VLAN_CFG_VLAN_AWARE_ENA                  BIT(20)
+#define ANA_PORT_VLAN_CFG_VLAN_POP_CNT(x)                 (((x) << 18) & GENMASK(19, 18))
+#define ANA_PORT_VLAN_CFG_VLAN_POP_CNT_M                  GENMASK(19, 18)
+#define ANA_PORT_VLAN_CFG_VLAN_POP_CNT_X(x)               (((x) & GENMASK(19, 18)) >> 18)
+#define ANA_PORT_VLAN_CFG_VLAN_INNER_TAG_ENA              BIT(17)
+#define ANA_PORT_VLAN_CFG_VLAN_TAG_TYPE                   BIT(16)
+#define ANA_PORT_VLAN_CFG_VLAN_DEI                        BIT(15)
+#define ANA_PORT_VLAN_CFG_VLAN_PCP(x)                     (((x) << 12) & GENMASK(14, 12))
+#define ANA_PORT_VLAN_CFG_VLAN_PCP_M                      GENMASK(14, 12)
+#define ANA_PORT_VLAN_CFG_VLAN_PCP_X(x)                   (((x) & GENMASK(14, 12)) >> 12)
+#define ANA_PORT_VLAN_CFG_VLAN_VID(x)                     ((x) & GENMASK(11, 0))
+#define ANA_PORT_VLAN_CFG_VLAN_VID_M                      GENMASK(11, 0)
+
+#define ANA_PORT_DROP_CFG_GSZ                             0x100
+
+#define ANA_PORT_DROP_CFG_DROP_UNTAGGED_ENA               BIT(6)
+#define ANA_PORT_DROP_CFG_DROP_S_TAGGED_ENA               BIT(5)
+#define ANA_PORT_DROP_CFG_DROP_C_TAGGED_ENA               BIT(4)
+#define ANA_PORT_DROP_CFG_DROP_PRIO_S_TAGGED_ENA          BIT(3)
+#define ANA_PORT_DROP_CFG_DROP_PRIO_C_TAGGED_ENA          BIT(2)
+#define ANA_PORT_DROP_CFG_DROP_NULL_MAC_ENA               BIT(1)
+#define ANA_PORT_DROP_CFG_DROP_MC_SMAC_ENA                BIT(0)
+
+#define ANA_PORT_QOS_CFG_GSZ                              0x100
+
+#define ANA_PORT_QOS_CFG_DP_DEFAULT_VAL                   BIT(8)
+#define ANA_PORT_QOS_CFG_QOS_DEFAULT_VAL(x)               (((x) << 5) & GENMASK(7, 5))
+#define ANA_PORT_QOS_CFG_QOS_DEFAULT_VAL_M                GENMASK(7, 5)
+#define ANA_PORT_QOS_CFG_QOS_DEFAULT_VAL_X(x)             (((x) & GENMASK(7, 5)) >> 5)
+#define ANA_PORT_QOS_CFG_QOS_DSCP_ENA                     BIT(4)
+#define ANA_PORT_QOS_CFG_QOS_PCP_ENA                      BIT(3)
+#define ANA_PORT_QOS_CFG_DSCP_TRANSLATE_ENA               BIT(2)
+#define ANA_PORT_QOS_CFG_DSCP_REWR_CFG(x)                 ((x) & GENMASK(1, 0))
+#define ANA_PORT_QOS_CFG_DSCP_REWR_CFG_M                  GENMASK(1, 0)
+
+#define ANA_PORT_VCAP_CFG_GSZ                             0x100
+
+#define ANA_PORT_VCAP_CFG_S1_ENA                          BIT(14)
+#define ANA_PORT_VCAP_CFG_S1_DMAC_DIP_ENA(x)              (((x) << 11) & GENMASK(13, 11))
+#define ANA_PORT_VCAP_CFG_S1_DMAC_DIP_ENA_M               GENMASK(13, 11)
+#define ANA_PORT_VCAP_CFG_S1_DMAC_DIP_ENA_X(x)            (((x) & GENMASK(13, 11)) >> 11)
+#define ANA_PORT_VCAP_CFG_S1_VLAN_INNER_TAG_ENA(x)        (((x) << 8) & GENMASK(10, 8))
+#define ANA_PORT_VCAP_CFG_S1_VLAN_INNER_TAG_ENA_M         GENMASK(10, 8)
+#define ANA_PORT_VCAP_CFG_S1_VLAN_INNER_TAG_ENA_X(x)      (((x) & GENMASK(10, 8)) >> 8)
+#define ANA_PORT_VCAP_CFG_PAG_VAL(x)                      ((x) & GENMASK(7, 0))
+#define ANA_PORT_VCAP_CFG_PAG_VAL_M                       GENMASK(7, 0)
+
+#define ANA_PORT_VCAP_S1_KEY_CFG_GSZ                      0x100
+#define ANA_PORT_VCAP_S1_KEY_CFG_RSZ                      0x4
+
+#define ANA_PORT_VCAP_S1_KEY_CFG_S1_KEY_IP6_CFG(x)        (((x) << 4) & GENMASK(6, 4))
+#define ANA_PORT_VCAP_S1_KEY_CFG_S1_KEY_IP6_CFG_M         GENMASK(6, 4)
+#define ANA_PORT_VCAP_S1_KEY_CFG_S1_KEY_IP6_CFG_X(x)      (((x) & GENMASK(6, 4)) >> 4)
+#define ANA_PORT_VCAP_S1_KEY_CFG_S1_KEY_IP4_CFG(x)        (((x) << 2) & GENMASK(3, 2))
+#define ANA_PORT_VCAP_S1_KEY_CFG_S1_KEY_IP4_CFG_M         GENMASK(3, 2)
+#define ANA_PORT_VCAP_S1_KEY_CFG_S1_KEY_IP4_CFG_X(x)      (((x) & GENMASK(3, 2)) >> 2)
+#define ANA_PORT_VCAP_S1_KEY_CFG_S1_KEY_OTHER_CFG(x)      ((x) & GENMASK(1, 0))
+#define ANA_PORT_VCAP_S1_KEY_CFG_S1_KEY_OTHER_CFG_M       GENMASK(1, 0)
+
+#define ANA_PORT_VCAP_S2_CFG_GSZ                          0x100
+
+#define ANA_PORT_VCAP_S2_CFG_S2_UDP_PAYLOAD_ENA(x)        (((x) << 17) & GENMASK(18, 17))
+#define ANA_PORT_VCAP_S2_CFG_S2_UDP_PAYLOAD_ENA_M         GENMASK(18, 17)
+#define ANA_PORT_VCAP_S2_CFG_S2_UDP_PAYLOAD_ENA_X(x)      (((x) & GENMASK(18, 17)) >> 17)
+#define ANA_PORT_VCAP_S2_CFG_S2_ETYPE_PAYLOAD_ENA(x)      (((x) << 15) & GENMASK(16, 15))
+#define ANA_PORT_VCAP_S2_CFG_S2_ETYPE_PAYLOAD_ENA_M       GENMASK(16, 15)
+#define ANA_PORT_VCAP_S2_CFG_S2_ETYPE_PAYLOAD_ENA_X(x)    (((x) & GENMASK(16, 15)) >> 15)
+#define ANA_PORT_VCAP_S2_CFG_S2_ENA                       BIT(14)
+#define ANA_PORT_VCAP_S2_CFG_S2_SNAP_DIS(x)               (((x) << 12) & GENMASK(13, 12))
+#define ANA_PORT_VCAP_S2_CFG_S2_SNAP_DIS_M                GENMASK(13, 12)
+#define ANA_PORT_VCAP_S2_CFG_S2_SNAP_DIS_X(x)             (((x) & GENMASK(13, 12)) >> 12)
+#define ANA_PORT_VCAP_S2_CFG_S2_ARP_DIS(x)                (((x) << 10) & GENMASK(11, 10))
+#define ANA_PORT_VCAP_S2_CFG_S2_ARP_DIS_M                 GENMASK(11, 10)
+#define ANA_PORT_VCAP_S2_CFG_S2_ARP_DIS_X(x)              (((x) & GENMASK(11, 10)) >> 10)
+#define ANA_PORT_VCAP_S2_CFG_S2_IP_TCPUDP_DIS(x)          (((x) << 8) & GENMASK(9, 8))
+#define ANA_PORT_VCAP_S2_CFG_S2_IP_TCPUDP_DIS_M           GENMASK(9, 8)
+#define ANA_PORT_VCAP_S2_CFG_S2_IP_TCPUDP_DIS_X(x)        (((x) & GENMASK(9, 8)) >> 8)
+#define ANA_PORT_VCAP_S2_CFG_S2_IP_OTHER_DIS(x)           (((x) << 6) & GENMASK(7, 6))
+#define ANA_PORT_VCAP_S2_CFG_S2_IP_OTHER_DIS_M            GENMASK(7, 6)
+#define ANA_PORT_VCAP_S2_CFG_S2_IP_OTHER_DIS_X(x)         (((x) & GENMASK(7, 6)) >> 6)
+#define ANA_PORT_VCAP_S2_CFG_S2_IP6_CFG(x)                (((x) << 2) & GENMASK(5, 2))
+#define ANA_PORT_VCAP_S2_CFG_S2_IP6_CFG_M                 GENMASK(5, 2)
+#define ANA_PORT_VCAP_S2_CFG_S2_IP6_CFG_X(x)              (((x) & GENMASK(5, 2)) >> 2)
+#define ANA_PORT_VCAP_S2_CFG_S2_OAM_DIS(x)                ((x) & GENMASK(1, 0))
+#define ANA_PORT_VCAP_S2_CFG_S2_OAM_DIS_M                 GENMASK(1, 0)
+
+#define ANA_PORT_PCP_DEI_MAP_GSZ                          0x100
+#define ANA_PORT_PCP_DEI_MAP_RSZ                          0x4
+
+#define ANA_PORT_PCP_DEI_MAP_DP_PCP_DEI_VAL               BIT(3)
+#define ANA_PORT_PCP_DEI_MAP_QOS_PCP_DEI_VAL(x)           ((x) & GENMASK(2, 0))
+#define ANA_PORT_PCP_DEI_MAP_QOS_PCP_DEI_VAL_M            GENMASK(2, 0)
+
+#define ANA_PORT_CPU_FWD_CFG_GSZ                          0x100
+
+#define ANA_PORT_CPU_FWD_CFG_CPU_VRAP_REDIR_ENA           BIT(7)
+#define ANA_PORT_CPU_FWD_CFG_CPU_MLD_REDIR_ENA            BIT(6)
+#define ANA_PORT_CPU_FWD_CFG_CPU_IGMP_REDIR_ENA           BIT(5)
+#define ANA_PORT_CPU_FWD_CFG_CPU_IPMC_CTRL_COPY_ENA       BIT(4)
+#define ANA_PORT_CPU_FWD_CFG_CPU_SRC_COPY_ENA             BIT(3)
+#define ANA_PORT_CPU_FWD_CFG_CPU_ALLBRIDGE_DROP_ENA       BIT(2)
+#define ANA_PORT_CPU_FWD_CFG_CPU_ALLBRIDGE_REDIR_ENA      BIT(1)
+#define ANA_PORT_CPU_FWD_CFG_CPU_OAM_ENA                  BIT(0)
+
+#define ANA_PORT_CPU_FWD_BPDU_CFG_GSZ                     0x100
+
+#define ANA_PORT_CPU_FWD_BPDU_CFG_BPDU_DROP_ENA(x)        (((x) << 16) & GENMASK(31, 16))
+#define ANA_PORT_CPU_FWD_BPDU_CFG_BPDU_DROP_ENA_M         GENMASK(31, 16)
+#define ANA_PORT_CPU_FWD_BPDU_CFG_BPDU_DROP_ENA_X(x)      (((x) & GENMASK(31, 16)) >> 16)
+#define ANA_PORT_CPU_FWD_BPDU_CFG_BPDU_REDIR_ENA(x)       ((x) & GENMASK(15, 0))
+#define ANA_PORT_CPU_FWD_BPDU_CFG_BPDU_REDIR_ENA_M        GENMASK(15, 0)
+
+#define ANA_PORT_CPU_FWD_GARP_CFG_GSZ                     0x100
+
+#define ANA_PORT_CPU_FWD_GARP_CFG_GARP_DROP_ENA(x)        (((x) << 16) & GENMASK(31, 16))
+#define ANA_PORT_CPU_FWD_GARP_CFG_GARP_DROP_ENA_M         GENMASK(31, 16)
+#define ANA_PORT_CPU_FWD_GARP_CFG_GARP_DROP_ENA_X(x)      (((x) & GENMASK(31, 16)) >> 16)
+#define ANA_PORT_CPU_FWD_GARP_CFG_GARP_REDIR_ENA(x)       ((x) & GENMASK(15, 0))
+#define ANA_PORT_CPU_FWD_GARP_CFG_GARP_REDIR_ENA_M        GENMASK(15, 0)
+
+#define ANA_PORT_CPU_FWD_CCM_CFG_GSZ                      0x100
+
+#define ANA_PORT_CPU_FWD_CCM_CFG_CCM_DROP_ENA(x)          (((x) << 16) & GENMASK(31, 16))
+#define ANA_PORT_CPU_FWD_CCM_CFG_CCM_DROP_ENA_M           GENMASK(31, 16)
+#define ANA_PORT_CPU_FWD_CCM_CFG_CCM_DROP_ENA_X(x)        (((x) & GENMASK(31, 16)) >> 16)
+#define ANA_PORT_CPU_FWD_CCM_CFG_CCM_REDIR_ENA(x)         ((x) & GENMASK(15, 0))
+#define ANA_PORT_CPU_FWD_CCM_CFG_CCM_REDIR_ENA_M          GENMASK(15, 0)
+
+#define ANA_PORT_PORT_CFG_GSZ                             0x100
+
+#define ANA_PORT_PORT_CFG_SRC_MIRROR_ENA                  BIT(15)
+#define ANA_PORT_PORT_CFG_LIMIT_DROP                      BIT(14)
+#define ANA_PORT_PORT_CFG_LIMIT_CPU                       BIT(13)
+#define ANA_PORT_PORT_CFG_LOCKED_PORTMOVE_DROP            BIT(12)
+#define ANA_PORT_PORT_CFG_LOCKED_PORTMOVE_CPU             BIT(11)
+#define ANA_PORT_PORT_CFG_LEARNDROP                       BIT(10)
+#define ANA_PORT_PORT_CFG_LEARNCPU                        BIT(9)
+#define ANA_PORT_PORT_CFG_LEARNAUTO                       BIT(8)
+#define ANA_PORT_PORT_CFG_LEARN_ENA                       BIT(7)
+#define ANA_PORT_PORT_CFG_RECV_ENA                        BIT(6)
+#define ANA_PORT_PORT_CFG_PORTID_VAL(x)                   (((x) << 2) & GENMASK(5, 2))
+#define ANA_PORT_PORT_CFG_PORTID_VAL_M                    GENMASK(5, 2)
+#define ANA_PORT_PORT_CFG_PORTID_VAL_X(x)                 (((x) & GENMASK(5, 2)) >> 2)
+#define ANA_PORT_PORT_CFG_USE_B_DOM_TBL                   BIT(1)
+#define ANA_PORT_PORT_CFG_LSR_MODE                        BIT(0)
+
+#define ANA_PORT_POL_CFG_GSZ                              0x100
+
+#define ANA_PORT_POL_CFG_POL_CPU_REDIR_8021               BIT(19)
+#define ANA_PORT_POL_CFG_POL_CPU_REDIR_IP                 BIT(18)
+#define ANA_PORT_POL_CFG_PORT_POL_ENA                     BIT(17)
+#define ANA_PORT_POL_CFG_QUEUE_POL_ENA(x)                 (((x) << 9) & GENMASK(16, 9))
+#define ANA_PORT_POL_CFG_QUEUE_POL_ENA_M                  GENMASK(16, 9)
+#define ANA_PORT_POL_CFG_QUEUE_POL_ENA_X(x)               (((x) & GENMASK(16, 9)) >> 9)
+#define ANA_PORT_POL_CFG_POL_ORDER(x)                     ((x) & GENMASK(8, 0))
+#define ANA_PORT_POL_CFG_POL_ORDER_M                      GENMASK(8, 0)
+
+#define ANA_PORT_PTP_CFG_GSZ                              0x100
+
+#define ANA_PORT_PTP_CFG_PTP_BACKPLANE_MODE               BIT(0)
+
+#define ANA_PORT_PTP_DLY1_CFG_GSZ                         0x100
+
+#define ANA_PORT_PTP_DLY2_CFG_GSZ                         0x100
+
+#define ANA_PORT_SFID_CFG_GSZ                             0x100
+#define ANA_PORT_SFID_CFG_RSZ                             0x4
+
+#define ANA_PORT_SFID_CFG_SFID_VALID                      BIT(8)
+#define ANA_PORT_SFID_CFG_SFID(x)                         ((x) & GENMASK(7, 0))
+#define ANA_PORT_SFID_CFG_SFID_M                          GENMASK(7, 0)
+
+#define ANA_PFC_PFC_CFG_GSZ                               0x40
+
+#define ANA_PFC_PFC_CFG_RX_PFC_ENA(x)                     (((x) << 2) & GENMASK(9, 2))
+#define ANA_PFC_PFC_CFG_RX_PFC_ENA_M                      GENMASK(9, 2)
+#define ANA_PFC_PFC_CFG_RX_PFC_ENA_X(x)                   (((x) & GENMASK(9, 2)) >> 2)
+#define ANA_PFC_PFC_CFG_FC_LINK_SPEED(x)                  ((x) & GENMASK(1, 0))
+#define ANA_PFC_PFC_CFG_FC_LINK_SPEED_M                   GENMASK(1, 0)
+
+#define ANA_PFC_PFC_TIMER_GSZ                             0x40
+#define ANA_PFC_PFC_TIMER_RSZ                             0x4
+
+#define ANA_IPT_OAM_MEP_CFG_GSZ                           0x8
+
+#define ANA_IPT_OAM_MEP_CFG_MEP_IDX_P(x)                  (((x) << 6) & GENMASK(10, 6))
+#define ANA_IPT_OAM_MEP_CFG_MEP_IDX_P_M                   GENMASK(10, 6)
+#define ANA_IPT_OAM_MEP_CFG_MEP_IDX_P_X(x)                (((x) & GENMASK(10, 6)) >> 6)
+#define ANA_IPT_OAM_MEP_CFG_MEP_IDX(x)                    (((x) << 1) & GENMASK(5, 1))
+#define ANA_IPT_OAM_MEP_CFG_MEP_IDX_M                     GENMASK(5, 1)
+#define ANA_IPT_OAM_MEP_CFG_MEP_IDX_X(x)                  (((x) & GENMASK(5, 1)) >> 1)
+#define ANA_IPT_OAM_MEP_CFG_MEP_IDX_ENA                   BIT(0)
+
+#define ANA_IPT_IPT_GSZ                                   0x8
+
+#define ANA_IPT_IPT_IPT_CFG(x)                            (((x) << 15) & GENMASK(16, 15))
+#define ANA_IPT_IPT_IPT_CFG_M                             GENMASK(16, 15)
+#define ANA_IPT_IPT_IPT_CFG_X(x)                          (((x) & GENMASK(16, 15)) >> 15)
+#define ANA_IPT_IPT_ISDX_P(x)                             (((x) << 7) & GENMASK(14, 7))
+#define ANA_IPT_IPT_ISDX_P_M                              GENMASK(14, 7)
+#define ANA_IPT_IPT_ISDX_P_X(x)                           (((x) & GENMASK(14, 7)) >> 7)
+#define ANA_IPT_IPT_PPT_IDX(x)                            ((x) & GENMASK(6, 0))
+#define ANA_IPT_IPT_PPT_IDX_M                             GENMASK(6, 0)
+
+#define ANA_PPT_PPT_RSZ                                   0x4
+
+#define ANA_FID_MAP_FID_MAP_RSZ                           0x4
+
+#define ANA_FID_MAP_FID_MAP_FID_C_VAL(x)                  (((x) << 6) & GENMASK(11, 6))
+#define ANA_FID_MAP_FID_MAP_FID_C_VAL_M                   GENMASK(11, 6)
+#define ANA_FID_MAP_FID_MAP_FID_C_VAL_X(x)                (((x) & GENMASK(11, 6)) >> 6)
+#define ANA_FID_MAP_FID_MAP_FID_B_VAL(x)                  ((x) & GENMASK(5, 0))
+#define ANA_FID_MAP_FID_MAP_FID_B_VAL_M                   GENMASK(5, 0)
+
+#define ANA_AGGR_CFG_AC_RND_ENA                           BIT(7)
+#define ANA_AGGR_CFG_AC_DMAC_ENA                          BIT(6)
+#define ANA_AGGR_CFG_AC_SMAC_ENA                          BIT(5)
+#define ANA_AGGR_CFG_AC_IP6_FLOW_LBL_ENA                  BIT(4)
+#define ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA                    BIT(3)
+#define ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA                    BIT(2)
+#define ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA                    BIT(1)
+#define ANA_AGGR_CFG_AC_ISDX_ENA                          BIT(0)
+
+#define ANA_CPUQ_CFG_CPUQ_MLD(x)                          (((x) << 27) & GENMASK(29, 27))
+#define ANA_CPUQ_CFG_CPUQ_MLD_M                           GENMASK(29, 27)
+#define ANA_CPUQ_CFG_CPUQ_MLD_X(x)                        (((x) & GENMASK(29, 27)) >> 27)
+#define ANA_CPUQ_CFG_CPUQ_IGMP(x)                         (((x) << 24) & GENMASK(26, 24))
+#define ANA_CPUQ_CFG_CPUQ_IGMP_M                          GENMASK(26, 24)
+#define ANA_CPUQ_CFG_CPUQ_IGMP_X(x)                       (((x) & GENMASK(26, 24)) >> 24)
+#define ANA_CPUQ_CFG_CPUQ_IPMC_CTRL(x)                    (((x) << 21) & GENMASK(23, 21))
+#define ANA_CPUQ_CFG_CPUQ_IPMC_CTRL_M                     GENMASK(23, 21)
+#define ANA_CPUQ_CFG_CPUQ_IPMC_CTRL_X(x)                  (((x) & GENMASK(23, 21)) >> 21)
+#define ANA_CPUQ_CFG_CPUQ_ALLBRIDGE(x)                    (((x) << 18) & GENMASK(20, 18))
+#define ANA_CPUQ_CFG_CPUQ_ALLBRIDGE_M                     GENMASK(20, 18)
+#define ANA_CPUQ_CFG_CPUQ_ALLBRIDGE_X(x)                  (((x) & GENMASK(20, 18)) >> 18)
+#define ANA_CPUQ_CFG_CPUQ_LOCKED_PORTMOVE(x)              (((x) << 15) & GENMASK(17, 15))
+#define ANA_CPUQ_CFG_CPUQ_LOCKED_PORTMOVE_M               GENMASK(17, 15)
+#define ANA_CPUQ_CFG_CPUQ_LOCKED_PORTMOVE_X(x)            (((x) & GENMASK(17, 15)) >> 15)
+#define ANA_CPUQ_CFG_CPUQ_SRC_COPY(x)                     (((x) << 12) & GENMASK(14, 12))
+#define ANA_CPUQ_CFG_CPUQ_SRC_COPY_M                      GENMASK(14, 12)
+#define ANA_CPUQ_CFG_CPUQ_SRC_COPY_X(x)                   (((x) & GENMASK(14, 12)) >> 12)
+#define ANA_CPUQ_CFG_CPUQ_MAC_COPY(x)                     (((x) << 9) & GENMASK(11, 9))
+#define ANA_CPUQ_CFG_CPUQ_MAC_COPY_M                      GENMASK(11, 9)
+#define ANA_CPUQ_CFG_CPUQ_MAC_COPY_X(x)                   (((x) & GENMASK(11, 9)) >> 9)
+#define ANA_CPUQ_CFG_CPUQ_LRN(x)                          (((x) << 6) & GENMASK(8, 6))
+#define ANA_CPUQ_CFG_CPUQ_LRN_M                           GENMASK(8, 6)
+#define ANA_CPUQ_CFG_CPUQ_LRN_X(x)                        (((x) & GENMASK(8, 6)) >> 6)
+#define ANA_CPUQ_CFG_CPUQ_MIRROR(x)                       (((x) << 3) & GENMASK(5, 3))
+#define ANA_CPUQ_CFG_CPUQ_MIRROR_M                        GENMASK(5, 3)
+#define ANA_CPUQ_CFG_CPUQ_MIRROR_X(x)                     (((x) & GENMASK(5, 3)) >> 3)
+#define ANA_CPUQ_CFG_CPUQ_SFLOW(x)                        ((x) & GENMASK(2, 0))
+#define ANA_CPUQ_CFG_CPUQ_SFLOW_M                         GENMASK(2, 0)
+
+#define ANA_CPUQ_8021_CFG_RSZ                             0x4
+
+#define ANA_CPUQ_8021_CFG_CPUQ_BPDU_VAL(x)                (((x) << 6) & GENMASK(8, 6))
+#define ANA_CPUQ_8021_CFG_CPUQ_BPDU_VAL_M                 GENMASK(8, 6)
+#define ANA_CPUQ_8021_CFG_CPUQ_BPDU_VAL_X(x)              (((x) & GENMASK(8, 6)) >> 6)
+#define ANA_CPUQ_8021_CFG_CPUQ_GARP_VAL(x)                (((x) << 3) & GENMASK(5, 3))
+#define ANA_CPUQ_8021_CFG_CPUQ_GARP_VAL_M                 GENMASK(5, 3)
+#define ANA_CPUQ_8021_CFG_CPUQ_GARP_VAL_X(x)              (((x) & GENMASK(5, 3)) >> 3)
+#define ANA_CPUQ_8021_CFG_CPUQ_CCM_VAL(x)                 ((x) & GENMASK(2, 0))
+#define ANA_CPUQ_8021_CFG_CPUQ_CCM_VAL_M                  GENMASK(2, 0)
+
+#define ANA_DSCP_CFG_RSZ                                  0x4
+
+#define ANA_DSCP_CFG_DP_DSCP_VAL                          BIT(11)
+#define ANA_DSCP_CFG_QOS_DSCP_VAL(x)                      (((x) << 8) & GENMASK(10, 8))
+#define ANA_DSCP_CFG_QOS_DSCP_VAL_M                       GENMASK(10, 8)
+#define ANA_DSCP_CFG_QOS_DSCP_VAL_X(x)                    (((x) & GENMASK(10, 8)) >> 8)
+#define ANA_DSCP_CFG_DSCP_TRANSLATE_VAL(x)                (((x) << 2) & GENMASK(7, 2))
+#define ANA_DSCP_CFG_DSCP_TRANSLATE_VAL_M                 GENMASK(7, 2)
+#define ANA_DSCP_CFG_DSCP_TRANSLATE_VAL_X(x)              (((x) & GENMASK(7, 2)) >> 2)
+#define ANA_DSCP_CFG_DSCP_TRUST_ENA                       BIT(1)
+#define ANA_DSCP_CFG_DSCP_REWR_ENA                        BIT(0)
+
+#define ANA_DSCP_REWR_CFG_RSZ                             0x4
+
+#define ANA_VCAP_RNG_TYPE_CFG_RSZ                         0x4
+
+#define ANA_VCAP_RNG_VAL_CFG_RSZ                          0x4
+
+#define ANA_VCAP_RNG_VAL_CFG_VCAP_RNG_MIN_VAL(x)          (((x) << 16) & GENMASK(31, 16))
+#define ANA_VCAP_RNG_VAL_CFG_VCAP_RNG_MIN_VAL_M           GENMASK(31, 16)
+#define ANA_VCAP_RNG_VAL_CFG_VCAP_RNG_MIN_VAL_X(x)        (((x) & GENMASK(31, 16)) >> 16)
+#define ANA_VCAP_RNG_VAL_CFG_VCAP_RNG_MAX_VAL(x)          ((x) & GENMASK(15, 0))
+#define ANA_VCAP_RNG_VAL_CFG_VCAP_RNG_MAX_VAL_M           GENMASK(15, 0)
+
+#define ANA_VRAP_CFG_VRAP_VLAN_AWARE_ENA                  BIT(12)
+#define ANA_VRAP_CFG_VRAP_VID(x)                          ((x) & GENMASK(11, 0))
+#define ANA_VRAP_CFG_VRAP_VID_M                           GENMASK(11, 0)
+
+#define ANA_DISCARD_CFG_DROP_TAGGING_ISDX0                BIT(3)
+#define ANA_DISCARD_CFG_DROP_CTRLPROT_ISDX0               BIT(2)
+#define ANA_DISCARD_CFG_DROP_TAGGING_S2_ENA               BIT(1)
+#define ANA_DISCARD_CFG_DROP_CTRLPROT_S2_ENA              BIT(0)
+
+#define ANA_FID_CFG_VID_MC_ENA                            BIT(0)
+
+#define ANA_POL_PIR_CFG_GSZ                               0x20
+
+#define ANA_POL_PIR_CFG_PIR_RATE(x)                       (((x) << 6) & GENMASK(20, 6))
+#define ANA_POL_PIR_CFG_PIR_RATE_M                        GENMASK(20, 6)
+#define ANA_POL_PIR_CFG_PIR_RATE_X(x)                     (((x) & GENMASK(20, 6)) >> 6)
+#define ANA_POL_PIR_CFG_PIR_BURST(x)                      ((x) & GENMASK(5, 0))
+#define ANA_POL_PIR_CFG_PIR_BURST_M                       GENMASK(5, 0)
+
+#define ANA_POL_CIR_CFG_GSZ                               0x20
+
+#define ANA_POL_CIR_CFG_CIR_RATE(x)                       (((x) << 6) & GENMASK(20, 6))
+#define ANA_POL_CIR_CFG_CIR_RATE_M                        GENMASK(20, 6)
+#define ANA_POL_CIR_CFG_CIR_RATE_X(x)                     (((x) & GENMASK(20, 6)) >> 6)
+#define ANA_POL_CIR_CFG_CIR_BURST(x)                      ((x) & GENMASK(5, 0))
+#define ANA_POL_CIR_CFG_CIR_BURST_M                       GENMASK(5, 0)
+
+#define ANA_POL_MODE_CFG_GSZ                              0x20
+
+#define ANA_POL_MODE_CFG_IPG_SIZE(x)                      (((x) << 5) & GENMASK(9, 5))
+#define ANA_POL_MODE_CFG_IPG_SIZE_M                       GENMASK(9, 5)
+#define ANA_POL_MODE_CFG_IPG_SIZE_X(x)                    (((x) & GENMASK(9, 5)) >> 5)
+#define ANA_POL_MODE_CFG_FRM_MODE(x)                      (((x) << 3) & GENMASK(4, 3))
+#define ANA_POL_MODE_CFG_FRM_MODE_M                       GENMASK(4, 3)
+#define ANA_POL_MODE_CFG_FRM_MODE_X(x)                    (((x) & GENMASK(4, 3)) >> 3)
+#define ANA_POL_MODE_CFG_DLB_COUPLED                      BIT(2)
+#define ANA_POL_MODE_CFG_CIR_ENA                          BIT(1)
+#define ANA_POL_MODE_CFG_OVERSHOOT_ENA                    BIT(0)
+
+#define ANA_POL_PIR_STATE_GSZ                             0x20
+
+#define ANA_POL_CIR_STATE_GSZ                             0x20
+
+#define ANA_POL_STATE_GSZ                                 0x20
+
+#define ANA_POL_FLOWC_RSZ                                 0x4
+
+#define ANA_POL_FLOWC_POL_FLOWC                           BIT(0)
+
+#define ANA_POL_HYST_POL_FC_HYST(x)                       (((x) << 4) & GENMASK(9, 4))
+#define ANA_POL_HYST_POL_FC_HYST_M                        GENMASK(9, 4)
+#define ANA_POL_HYST_POL_FC_HYST_X(x)                     (((x) & GENMASK(9, 4)) >> 4)
+#define ANA_POL_HYST_POL_STOP_HYST(x)                     ((x) & GENMASK(3, 0))
+#define ANA_POL_HYST_POL_STOP_HYST_M                      GENMASK(3, 0)
+
+#define ANA_POL_MISC_CFG_POL_CLOSE_ALL                    BIT(1)
+#define ANA_POL_MISC_CFG_POL_LEAK_DIS                     BIT(0)
+
+#endif
diff --git a/drivers/net/ethernet/mscc/ocelot_board.c b/drivers/net/ethernet/mscc/ocelot_board.c
new file mode 100644
index 000000000000..18df7d934e81
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_board.c
@@ -0,0 +1,316 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/of_mdio.h>
+#include <linux/of_platform.h>
+#include <linux/skbuff.h>
+
+#include "ocelot.h"
+
+static int ocelot_parse_ifh(u32 *ifh, struct frame_info *info)
+{
+	int i;
+	u8 llen, wlen;
+
+	/* The IFH is in network order, switch to CPU order */
+	for (i = 0; i < IFH_LEN; i++)
+		ifh[i] = ntohl((__force __be32)ifh[i]);
+
+	wlen = (ifh[1] >> 7) & 0xff;
+	llen = (ifh[1] >> 15) & 0x3f;
+	info->len = OCELOT_BUFFER_CELL_SZ * wlen + llen - 80;
+
+	info->port = (ifh[2] & GENMASK(14, 11)) >> 11;
+
+	info->cpuq = (ifh[3] & GENMASK(27, 20)) >> 20;
+	info->tag_type = (ifh[3] & GENMASK(16, 16)) >> 16;
+	info->vid = ifh[3] & GENMASK(11, 0);
+
+	return 0;
+}
+
+static int ocelot_rx_frame_word(struct ocelot *ocelot, u8 grp, bool ifh,
+				u32 *rval)
+{
+	u32 val;
+	u32 bytes_valid;
+
+	val = ocelot_read_rix(ocelot, QS_XTR_RD, grp);
+	if (val == XTR_NOT_READY) {
+		if (ifh)
+			return -EIO;
+
+		do {
+			val = ocelot_read_rix(ocelot, QS_XTR_RD, grp);
+		} while (val == XTR_NOT_READY);
+	}
+
+	switch (val) {
+	case XTR_ABORT:
+		return -EIO;
+	case XTR_EOF_0:
+	case XTR_EOF_1:
+	case XTR_EOF_2:
+	case XTR_EOF_3:
+	case XTR_PRUNED:
+		bytes_valid = XTR_VALID_BYTES(val);
+		val = ocelot_read_rix(ocelot, QS_XTR_RD, grp);
+		if (val == XTR_ESCAPE)
+			*rval = ocelot_read_rix(ocelot, QS_XTR_RD, grp);
+		else
+			*rval = val;
+
+		return bytes_valid;
+	case XTR_ESCAPE:
+		*rval = ocelot_read_rix(ocelot, QS_XTR_RD, grp);
+
+		return 4;
+	default:
+		*rval = val;
+
+		return 4;
+	}
+}
+
+static irqreturn_t ocelot_xtr_irq_handler(int irq, void *arg)
+{
+	struct ocelot *ocelot = arg;
+	int i = 0, grp = 0;
+	int err = 0;
+
+	if (!(ocelot_read(ocelot, QS_XTR_DATA_PRESENT) & BIT(grp)))
+		return IRQ_NONE;
+
+	do {
+		struct sk_buff *skb;
+		struct net_device *dev;
+		u32 *buf;
+		int sz, len;
+		u32 ifh[4];
+		u32 val;
+		struct frame_info info;
+
+		for (i = 0; i < IFH_LEN; i++) {
+			err = ocelot_rx_frame_word(ocelot, grp, true, &ifh[i]);
+			if (err != 4)
+				break;
+		}
+
+		if (err != 4)
+			break;
+
+		ocelot_parse_ifh(ifh, &info);
+
+		dev = ocelot->ports[info.port]->dev;
+
+		skb = netdev_alloc_skb(dev, info.len);
+
+		if (unlikely(!skb)) {
+			netdev_err(dev, "Unable to allocate sk_buff\n");
+			err = -ENOMEM;
+			break;
+		}
+		buf = (u32 *)skb_put(skb, info.len);
+
+		len = 0;
+		do {
+			sz = ocelot_rx_frame_word(ocelot, grp, false, &val);
+			*buf++ = val;
+			len += sz;
+		} while ((sz == 4) && (len < info.len));
+
+		if (sz < 0) {
+			err = sz;
+			break;
+		}
+
+		/* Everything we see on an interface that is in the HW bridge
+		 * has already been forwarded.
+		 */
+		if (ocelot->bridge_mask & BIT(info.port))
+			skb->offload_fwd_mark = 1;
+
+		skb->protocol = eth_type_trans(skb, dev);
+		netif_rx(skb);
+		dev->stats.rx_bytes += len;
+		dev->stats.rx_packets++;
+	} while (ocelot_read(ocelot, QS_XTR_DATA_PRESENT) & BIT(grp));
+
+	if (err)
+		while (ocelot_read(ocelot, QS_XTR_DATA_PRESENT) & BIT(grp))
+			ocelot_read_rix(ocelot, QS_XTR_RD, grp);
+
+	return IRQ_HANDLED;
+}
+
+static const struct of_device_id mscc_ocelot_match[] = {
+	{ .compatible = "mscc,vsc7514-switch" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mscc_ocelot_match);
+
+static int mscc_ocelot_probe(struct platform_device *pdev)
+{
+	int err, irq;
+	unsigned int i;
+	struct device_node *np = pdev->dev.of_node;
+	struct device_node *ports, *portnp;
+	struct ocelot *ocelot;
+	u32 val;
+
+	struct {
+		enum ocelot_target id;
+		char *name;
+	} res[] = {
+		{ SYS, "sys" },
+		{ REW, "rew" },
+		{ QSYS, "qsys" },
+		{ ANA, "ana" },
+		{ QS, "qs" },
+		{ HSIO, "hsio" },
+	};
+
+	if (!np && !pdev->dev.platform_data)
+		return -ENODEV;
+
+	ocelot = devm_kzalloc(&pdev->dev, sizeof(*ocelot), GFP_KERNEL);
+	if (!ocelot)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ocelot);
+	ocelot->dev = &pdev->dev;
+
+	for (i = 0; i < ARRAY_SIZE(res); i++) {
+		struct regmap *target;
+
+		target = ocelot_io_platform_init(ocelot, pdev, res[i].name);
+		if (IS_ERR(target))
+			return PTR_ERR(target);
+
+		ocelot->targets[res[i].id] = target;
+	}
+
+	err = ocelot_chip_init(ocelot);
+	if (err)
+		return err;
+
+	irq = platform_get_irq_byname(pdev, "xtr");
+	if (irq < 0)
+		return -ENODEV;
+
+	err = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+					ocelot_xtr_irq_handler, IRQF_ONESHOT,
+					"frame extraction", ocelot);
+	if (err)
+		return err;
+
+	regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_INIT], 1);
+	regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_ENA], 1);
+
+	do {
+		msleep(1);
+		regmap_field_read(ocelot->regfields[SYS_RESET_CFG_MEM_INIT],
+				  &val);
+	} while (val);
+
+	regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_ENA], 1);
+	regmap_field_write(ocelot->regfields[SYS_RESET_CFG_CORE_ENA], 1);
+
+	ocelot->num_cpu_ports = 1; /* 1 port on the switch, two groups */
+
+	ports = of_get_child_by_name(np, "ethernet-ports");
+	if (!ports) {
+		dev_err(&pdev->dev, "no ethernet-ports child node found\n");
+		return -ENODEV;
+	}
+
+	ocelot->num_phys_ports = of_get_child_count(ports);
+
+	ocelot->ports = devm_kcalloc(&pdev->dev, ocelot->num_phys_ports,
+				     sizeof(struct ocelot_port *), GFP_KERNEL);
+
+	INIT_LIST_HEAD(&ocelot->multicast);
+	ocelot_init(ocelot);
+
+	ocelot_rmw(ocelot, HSIO_HW_CFG_DEV1G_4_MODE |
+		     HSIO_HW_CFG_DEV1G_6_MODE |
+		     HSIO_HW_CFG_DEV1G_9_MODE,
+		     HSIO_HW_CFG_DEV1G_4_MODE |
+		     HSIO_HW_CFG_DEV1G_6_MODE |
+		     HSIO_HW_CFG_DEV1G_9_MODE,
+		     HSIO_HW_CFG);
+
+	for_each_available_child_of_node(ports, portnp) {
+		struct device_node *phy_node;
+		struct phy_device *phy;
+		struct resource *res;
+		void __iomem *regs;
+		char res_name[8];
+		u32 port;
+
+		if (of_property_read_u32(portnp, "reg", &port))
+			continue;
+
+		snprintf(res_name, sizeof(res_name), "port%d", port);
+
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   res_name);
+		regs = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(regs))
+			continue;
+
+		phy_node = of_parse_phandle(portnp, "phy-handle", 0);
+		if (!phy_node)
+			continue;
+
+		phy = of_phy_find_device(phy_node);
+		if (!phy)
+			continue;
+
+		err = ocelot_probe_port(ocelot, port, regs, phy);
+		if (err) {
+			dev_err(&pdev->dev, "failed to probe ports\n");
+			goto err_probe_ports;
+		}
+	}
+
+	register_netdevice_notifier(&ocelot_netdevice_nb);
+
+	dev_info(&pdev->dev, "Ocelot switch probed\n");
+
+	return 0;
+
+err_probe_ports:
+	return err;
+}
+
+static int mscc_ocelot_remove(struct platform_device *pdev)
+{
+	struct ocelot *ocelot = platform_get_drvdata(pdev);
+
+	ocelot_deinit(ocelot);
+	unregister_netdevice_notifier(&ocelot_netdevice_nb);
+
+	return 0;
+}
+
+static struct platform_driver mscc_ocelot_driver = {
+	.probe = mscc_ocelot_probe,
+	.remove = mscc_ocelot_remove,
+	.driver = {
+		.name = "ocelot-switch",
+		.of_match_table = mscc_ocelot_match,
+	},
+};
+
+module_platform_driver(mscc_ocelot_driver);
+
+MODULE_DESCRIPTION("Microsemi Ocelot switch driver");
+MODULE_AUTHOR("Alexandre Belloni <alexandre.belloni@bootlin.com>");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/net/ethernet/mscc/ocelot_dev.h b/drivers/net/ethernet/mscc/ocelot_dev.h
new file mode 100644
index 000000000000..0a50d53bbd3f
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_dev.h
@@ -0,0 +1,275 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+
+#ifndef _MSCC_OCELOT_DEV_H_
+#define _MSCC_OCELOT_DEV_H_
+
+#define DEV_CLOCK_CFG                                     0x0
+
+#define DEV_CLOCK_CFG_MAC_TX_RST                          BIT(7)
+#define DEV_CLOCK_CFG_MAC_RX_RST                          BIT(6)
+#define DEV_CLOCK_CFG_PCS_TX_RST                          BIT(5)
+#define DEV_CLOCK_CFG_PCS_RX_RST                          BIT(4)
+#define DEV_CLOCK_CFG_PORT_RST                            BIT(3)
+#define DEV_CLOCK_CFG_PHY_RST                             BIT(2)
+#define DEV_CLOCK_CFG_LINK_SPEED(x)                       ((x) & GENMASK(1, 0))
+#define DEV_CLOCK_CFG_LINK_SPEED_M                        GENMASK(1, 0)
+
+#define DEV_PORT_MISC                                     0x4
+
+#define DEV_PORT_MISC_FWD_ERROR_ENA                       BIT(4)
+#define DEV_PORT_MISC_FWD_PAUSE_ENA                       BIT(3)
+#define DEV_PORT_MISC_FWD_CTRL_ENA                        BIT(2)
+#define DEV_PORT_MISC_DEV_LOOP_ENA                        BIT(1)
+#define DEV_PORT_MISC_HDX_FAST_DIS                        BIT(0)
+
+#define DEV_EVENTS                                        0x8
+
+#define DEV_EEE_CFG                                       0xc
+
+#define DEV_EEE_CFG_EEE_ENA                               BIT(22)
+#define DEV_EEE_CFG_EEE_TIMER_AGE(x)                      (((x) << 15) & GENMASK(21, 15))
+#define DEV_EEE_CFG_EEE_TIMER_AGE_M                       GENMASK(21, 15)
+#define DEV_EEE_CFG_EEE_TIMER_AGE_X(x)                    (((x) & GENMASK(21, 15)) >> 15)
+#define DEV_EEE_CFG_EEE_TIMER_WAKEUP(x)                   (((x) << 8) & GENMASK(14, 8))
+#define DEV_EEE_CFG_EEE_TIMER_WAKEUP_M                    GENMASK(14, 8)
+#define DEV_EEE_CFG_EEE_TIMER_WAKEUP_X(x)                 (((x) & GENMASK(14, 8)) >> 8)
+#define DEV_EEE_CFG_EEE_TIMER_HOLDOFF(x)                  (((x) << 1) & GENMASK(7, 1))
+#define DEV_EEE_CFG_EEE_TIMER_HOLDOFF_M                   GENMASK(7, 1)
+#define DEV_EEE_CFG_EEE_TIMER_HOLDOFF_X(x)                (((x) & GENMASK(7, 1)) >> 1)
+#define DEV_EEE_CFG_PORT_LPI                              BIT(0)
+
+#define DEV_RX_PATH_DELAY                                 0x10
+
+#define DEV_TX_PATH_DELAY                                 0x14
+
+#define DEV_PTP_PREDICT_CFG                               0x18
+
+#define DEV_PTP_PREDICT_CFG_PTP_PHY_PREDICT_CFG(x)        (((x) << 4) & GENMASK(11, 4))
+#define DEV_PTP_PREDICT_CFG_PTP_PHY_PREDICT_CFG_M         GENMASK(11, 4)
+#define DEV_PTP_PREDICT_CFG_PTP_PHY_PREDICT_CFG_X(x)      (((x) & GENMASK(11, 4)) >> 4)
+#define DEV_PTP_PREDICT_CFG_PTP_PHASE_PREDICT_CFG(x)      ((x) & GENMASK(3, 0))
+#define DEV_PTP_PREDICT_CFG_PTP_PHASE_PREDICT_CFG_M       GENMASK(3, 0)
+
+#define DEV_MAC_ENA_CFG                                   0x1c
+
+#define DEV_MAC_ENA_CFG_RX_ENA                            BIT(4)
+#define DEV_MAC_ENA_CFG_TX_ENA                            BIT(0)
+
+#define DEV_MAC_MODE_CFG                                  0x20
+
+#define DEV_MAC_MODE_CFG_FC_WORD_SYNC_ENA                 BIT(8)
+#define DEV_MAC_MODE_CFG_GIGA_MODE_ENA                    BIT(4)
+#define DEV_MAC_MODE_CFG_FDX_ENA                          BIT(0)
+
+#define DEV_MAC_MAXLEN_CFG                                0x24
+
+#define DEV_MAC_TAGS_CFG                                  0x28
+
+#define DEV_MAC_TAGS_CFG_TAG_ID(x)                        (((x) << 16) & GENMASK(31, 16))
+#define DEV_MAC_TAGS_CFG_TAG_ID_M                         GENMASK(31, 16)
+#define DEV_MAC_TAGS_CFG_TAG_ID_X(x)                      (((x) & GENMASK(31, 16)) >> 16)
+#define DEV_MAC_TAGS_CFG_VLAN_LEN_AWR_ENA                 BIT(2)
+#define DEV_MAC_TAGS_CFG_PB_ENA                           BIT(1)
+#define DEV_MAC_TAGS_CFG_VLAN_AWR_ENA                     BIT(0)
+
+#define DEV_MAC_ADV_CHK_CFG                               0x2c
+
+#define DEV_MAC_ADV_CHK_CFG_LEN_DROP_ENA                  BIT(0)
+
+#define DEV_MAC_IFG_CFG                                   0x30
+
+#define DEV_MAC_IFG_CFG_RESTORE_OLD_IPG_CHECK             BIT(17)
+#define DEV_MAC_IFG_CFG_REDUCED_TX_IFG                    BIT(16)
+#define DEV_MAC_IFG_CFG_TX_IFG(x)                         (((x) << 8) & GENMASK(12, 8))
+#define DEV_MAC_IFG_CFG_TX_IFG_M                          GENMASK(12, 8)
+#define DEV_MAC_IFG_CFG_TX_IFG_X(x)                       (((x) & GENMASK(12, 8)) >> 8)
+#define DEV_MAC_IFG_CFG_RX_IFG2(x)                        (((x) << 4) & GENMASK(7, 4))
+#define DEV_MAC_IFG_CFG_RX_IFG2_M                         GENMASK(7, 4)
+#define DEV_MAC_IFG_CFG_RX_IFG2_X(x)                      (((x) & GENMASK(7, 4)) >> 4)
+#define DEV_MAC_IFG_CFG_RX_IFG1(x)                        ((x) & GENMASK(3, 0))
+#define DEV_MAC_IFG_CFG_RX_IFG1_M                         GENMASK(3, 0)
+
+#define DEV_MAC_HDX_CFG                                   0x34
+
+#define DEV_MAC_HDX_CFG_BYPASS_COL_SYNC                   BIT(26)
+#define DEV_MAC_HDX_CFG_OB_ENA                            BIT(25)
+#define DEV_MAC_HDX_CFG_WEXC_DIS                          BIT(24)
+#define DEV_MAC_HDX_CFG_SEED(x)                           (((x) << 16) & GENMASK(23, 16))
+#define DEV_MAC_HDX_CFG_SEED_M                            GENMASK(23, 16)
+#define DEV_MAC_HDX_CFG_SEED_X(x)                         (((x) & GENMASK(23, 16)) >> 16)
+#define DEV_MAC_HDX_CFG_SEED_LOAD                         BIT(12)
+#define DEV_MAC_HDX_CFG_RETRY_AFTER_EXC_COL_ENA           BIT(8)
+#define DEV_MAC_HDX_CFG_LATE_COL_POS(x)                   ((x) & GENMASK(6, 0))
+#define DEV_MAC_HDX_CFG_LATE_COL_POS_M                    GENMASK(6, 0)
+
+#define DEV_MAC_DBG_CFG                                   0x38
+
+#define DEV_MAC_DBG_CFG_TBI_MODE                          BIT(4)
+#define DEV_MAC_DBG_CFG_IFG_CRS_EXT_CHK_ENA               BIT(0)
+
+#define DEV_MAC_FC_MAC_LOW_CFG                            0x3c
+
+#define DEV_MAC_FC_MAC_HIGH_CFG                           0x40
+
+#define DEV_MAC_STICKY                                    0x44
+
+#define DEV_MAC_STICKY_RX_IPG_SHRINK_STICKY               BIT(9)
+#define DEV_MAC_STICKY_RX_PREAM_SHRINK_STICKY             BIT(8)
+#define DEV_MAC_STICKY_RX_CARRIER_EXT_STICKY              BIT(7)
+#define DEV_MAC_STICKY_RX_CARRIER_EXT_ERR_STICKY          BIT(6)
+#define DEV_MAC_STICKY_RX_JUNK_STICKY                     BIT(5)
+#define DEV_MAC_STICKY_TX_RETRANSMIT_STICKY               BIT(4)
+#define DEV_MAC_STICKY_TX_JAM_STICKY                      BIT(3)
+#define DEV_MAC_STICKY_TX_FIFO_OFLW_STICKY                BIT(2)
+#define DEV_MAC_STICKY_TX_FRM_LEN_OVR_STICKY              BIT(1)
+#define DEV_MAC_STICKY_TX_ABORT_STICKY                    BIT(0)
+
+#define PCS1G_CFG                                         0x48
+
+#define PCS1G_CFG_LINK_STATUS_TYPE                        BIT(4)
+#define PCS1G_CFG_AN_LINK_CTRL_ENA                        BIT(1)
+#define PCS1G_CFG_PCS_ENA                                 BIT(0)
+
+#define PCS1G_MODE_CFG                                    0x4c
+
+#define PCS1G_MODE_CFG_UNIDIR_MODE_ENA                    BIT(4)
+#define PCS1G_MODE_CFG_SGMII_MODE_ENA                     BIT(0)
+
+#define PCS1G_SD_CFG                                      0x50
+
+#define PCS1G_SD_CFG_SD_SEL                               BIT(8)
+#define PCS1G_SD_CFG_SD_POL                               BIT(4)
+#define PCS1G_SD_CFG_SD_ENA                               BIT(0)
+
+#define PCS1G_ANEG_CFG                                    0x54
+
+#define PCS1G_ANEG_CFG_ADV_ABILITY(x)                     (((x) << 16) & GENMASK(31, 16))
+#define PCS1G_ANEG_CFG_ADV_ABILITY_M                      GENMASK(31, 16)
+#define PCS1G_ANEG_CFG_ADV_ABILITY_X(x)                   (((x) & GENMASK(31, 16)) >> 16)
+#define PCS1G_ANEG_CFG_SW_RESOLVE_ENA                     BIT(8)
+#define PCS1G_ANEG_CFG_ANEG_RESTART_ONE_SHOT              BIT(1)
+#define PCS1G_ANEG_CFG_ANEG_ENA                           BIT(0)
+
+#define PCS1G_ANEG_NP_CFG                                 0x58
+
+#define PCS1G_ANEG_NP_CFG_NP_TX(x)                        (((x) << 16) & GENMASK(31, 16))
+#define PCS1G_ANEG_NP_CFG_NP_TX_M                         GENMASK(31, 16)
+#define PCS1G_ANEG_NP_CFG_NP_TX_X(x)                      (((x) & GENMASK(31, 16)) >> 16)
+#define PCS1G_ANEG_NP_CFG_NP_LOADED_ONE_SHOT              BIT(0)
+
+#define PCS1G_LB_CFG                                      0x5c
+
+#define PCS1G_LB_CFG_RA_ENA                               BIT(4)
+#define PCS1G_LB_CFG_GMII_PHY_LB_ENA                      BIT(1)
+#define PCS1G_LB_CFG_TBI_HOST_LB_ENA                      BIT(0)
+
+#define PCS1G_DBG_CFG                                     0x60
+
+#define PCS1G_DBG_CFG_UDLT                                BIT(0)
+
+#define PCS1G_CDET_CFG                                    0x64
+
+#define PCS1G_CDET_CFG_CDET_ENA                           BIT(0)
+
+#define PCS1G_ANEG_STATUS                                 0x68
+
+#define PCS1G_ANEG_STATUS_LP_ADV_ABILITY(x)               (((x) << 16) & GENMASK(31, 16))
+#define PCS1G_ANEG_STATUS_LP_ADV_ABILITY_M                GENMASK(31, 16)
+#define PCS1G_ANEG_STATUS_LP_ADV_ABILITY_X(x)             (((x) & GENMASK(31, 16)) >> 16)
+#define PCS1G_ANEG_STATUS_PR                              BIT(4)
+#define PCS1G_ANEG_STATUS_PAGE_RX_STICKY                  BIT(3)
+#define PCS1G_ANEG_STATUS_ANEG_COMPLETE                   BIT(0)
+
+#define PCS1G_ANEG_NP_STATUS                              0x6c
+
+#define PCS1G_LINK_STATUS                                 0x70
+
+#define PCS1G_LINK_STATUS_DELAY_VAR(x)                    (((x) << 12) & GENMASK(15, 12))
+#define PCS1G_LINK_STATUS_DELAY_VAR_M                     GENMASK(15, 12)
+#define PCS1G_LINK_STATUS_DELAY_VAR_X(x)                  (((x) & GENMASK(15, 12)) >> 12)
+#define PCS1G_LINK_STATUS_SIGNAL_DETECT                   BIT(8)
+#define PCS1G_LINK_STATUS_LINK_STATUS                     BIT(4)
+#define PCS1G_LINK_STATUS_SYNC_STATUS                     BIT(0)
+
+#define PCS1G_LINK_DOWN_CNT                               0x74
+
+#define PCS1G_STICKY                                      0x78
+
+#define PCS1G_STICKY_LINK_DOWN_STICKY                     BIT(4)
+#define PCS1G_STICKY_OUT_OF_SYNC_STICKY                   BIT(0)
+
+#define PCS1G_DEBUG_STATUS                                0x7c
+
+#define PCS1G_LPI_CFG                                     0x80
+
+#define PCS1G_LPI_CFG_QSGMII_MS_SEL                       BIT(20)
+#define PCS1G_LPI_CFG_RX_LPI_OUT_DIS                      BIT(17)
+#define PCS1G_LPI_CFG_LPI_TESTMODE                        BIT(16)
+#define PCS1G_LPI_CFG_LPI_RX_WTIM(x)                      (((x) << 4) & GENMASK(5, 4))
+#define PCS1G_LPI_CFG_LPI_RX_WTIM_M                       GENMASK(5, 4)
+#define PCS1G_LPI_CFG_LPI_RX_WTIM_X(x)                    (((x) & GENMASK(5, 4)) >> 4)
+#define PCS1G_LPI_CFG_TX_ASSERT_LPIDLE                    BIT(0)
+
+#define PCS1G_LPI_WAKE_ERROR_CNT                          0x84
+
+#define PCS1G_LPI_STATUS                                  0x88
+
+#define PCS1G_LPI_STATUS_RX_LPI_FAIL                      BIT(16)
+#define PCS1G_LPI_STATUS_RX_LPI_EVENT_STICKY              BIT(12)
+#define PCS1G_LPI_STATUS_RX_QUIET                         BIT(9)
+#define PCS1G_LPI_STATUS_RX_LPI_MODE                      BIT(8)
+#define PCS1G_LPI_STATUS_TX_LPI_EVENT_STICKY              BIT(4)
+#define PCS1G_LPI_STATUS_TX_QUIET                         BIT(1)
+#define PCS1G_LPI_STATUS_TX_LPI_MODE                      BIT(0)
+
+#define PCS1G_TSTPAT_MODE_CFG                             0x8c
+
+#define PCS1G_TSTPAT_STATUS                               0x90
+
+#define PCS1G_TSTPAT_STATUS_JTP_ERR_CNT(x)                (((x) << 8) & GENMASK(15, 8))
+#define PCS1G_TSTPAT_STATUS_JTP_ERR_CNT_M                 GENMASK(15, 8)
+#define PCS1G_TSTPAT_STATUS_JTP_ERR_CNT_X(x)              (((x) & GENMASK(15, 8)) >> 8)
+#define PCS1G_TSTPAT_STATUS_JTP_ERR                       BIT(4)
+#define PCS1G_TSTPAT_STATUS_JTP_LOCK                      BIT(0)
+
+#define DEV_PCS_FX100_CFG                                 0x94
+
+#define DEV_PCS_FX100_CFG_SD_SEL                          BIT(26)
+#define DEV_PCS_FX100_CFG_SD_POL                          BIT(25)
+#define DEV_PCS_FX100_CFG_SD_ENA                          BIT(24)
+#define DEV_PCS_FX100_CFG_LOOPBACK_ENA                    BIT(20)
+#define DEV_PCS_FX100_CFG_SWAP_MII_ENA                    BIT(16)
+#define DEV_PCS_FX100_CFG_RXBITSEL(x)                     (((x) << 12) & GENMASK(15, 12))
+#define DEV_PCS_FX100_CFG_RXBITSEL_M                      GENMASK(15, 12)
+#define DEV_PCS_FX100_CFG_RXBITSEL_X(x)                   (((x) & GENMASK(15, 12)) >> 12)
+#define DEV_PCS_FX100_CFG_SIGDET_CFG(x)                   (((x) << 9) & GENMASK(10, 9))
+#define DEV_PCS_FX100_CFG_SIGDET_CFG_M                    GENMASK(10, 9)
+#define DEV_PCS_FX100_CFG_SIGDET_CFG_X(x)                 (((x) & GENMASK(10, 9)) >> 9)
+#define DEV_PCS_FX100_CFG_LINKHYST_TM_ENA                 BIT(8)
+#define DEV_PCS_FX100_CFG_LINKHYSTTIMER(x)                (((x) << 4) & GENMASK(7, 4))
+#define DEV_PCS_FX100_CFG_LINKHYSTTIMER_M                 GENMASK(7, 4)
+#define DEV_PCS_FX100_CFG_LINKHYSTTIMER_X(x)              (((x) & GENMASK(7, 4)) >> 4)
+#define DEV_PCS_FX100_CFG_UNIDIR_MODE_ENA                 BIT(3)
+#define DEV_PCS_FX100_CFG_FEFCHK_ENA                      BIT(2)
+#define DEV_PCS_FX100_CFG_FEFGEN_ENA                      BIT(1)
+#define DEV_PCS_FX100_CFG_PCS_ENA                         BIT(0)
+
+#define DEV_PCS_FX100_STATUS                              0x98
+
+#define DEV_PCS_FX100_STATUS_EDGE_POS_PTP(x)              (((x) << 8) & GENMASK(11, 8))
+#define DEV_PCS_FX100_STATUS_EDGE_POS_PTP_M               GENMASK(11, 8)
+#define DEV_PCS_FX100_STATUS_EDGE_POS_PTP_X(x)            (((x) & GENMASK(11, 8)) >> 8)
+#define DEV_PCS_FX100_STATUS_PCS_ERROR_STICKY             BIT(7)
+#define DEV_PCS_FX100_STATUS_FEF_FOUND_STICKY             BIT(6)
+#define DEV_PCS_FX100_STATUS_SSD_ERROR_STICKY             BIT(5)
+#define DEV_PCS_FX100_STATUS_SYNC_LOST_STICKY             BIT(4)
+#define DEV_PCS_FX100_STATUS_FEF_STATUS                   BIT(2)
+#define DEV_PCS_FX100_STATUS_SIGNAL_DETECT                BIT(1)
+#define DEV_PCS_FX100_STATUS_SYNC_STATUS                  BIT(0)
+
+#endif
diff --git a/drivers/net/ethernet/mscc/ocelot_dev_gmii.h b/drivers/net/ethernet/mscc/ocelot_dev_gmii.h
new file mode 100644
index 000000000000..6aa40ea223a2
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_dev_gmii.h
@@ -0,0 +1,154 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+
+#ifndef _MSCC_OCELOT_DEV_GMII_H_
+#define _MSCC_OCELOT_DEV_GMII_H_
+
+#define DEV_GMII_PORT_MODE_CLOCK_CFG                      0x0
+
+#define DEV_GMII_PORT_MODE_CLOCK_CFG_MAC_TX_RST           BIT(5)
+#define DEV_GMII_PORT_MODE_CLOCK_CFG_MAC_RX_RST           BIT(4)
+#define DEV_GMII_PORT_MODE_CLOCK_CFG_PORT_RST             BIT(3)
+#define DEV_GMII_PORT_MODE_CLOCK_CFG_PHY_RST              BIT(2)
+#define DEV_GMII_PORT_MODE_CLOCK_CFG_LINK_SPEED(x)        ((x) & GENMASK(1, 0))
+#define DEV_GMII_PORT_MODE_CLOCK_CFG_LINK_SPEED_M         GENMASK(1, 0)
+
+#define DEV_GMII_PORT_MODE_PORT_MISC                      0x4
+
+#define DEV_GMII_PORT_MODE_PORT_MISC_MPLS_RX_ENA          BIT(5)
+#define DEV_GMII_PORT_MODE_PORT_MISC_FWD_ERROR_ENA        BIT(4)
+#define DEV_GMII_PORT_MODE_PORT_MISC_FWD_PAUSE_ENA        BIT(3)
+#define DEV_GMII_PORT_MODE_PORT_MISC_FWD_CTRL_ENA         BIT(2)
+#define DEV_GMII_PORT_MODE_PORT_MISC_GMII_LOOP_ENA        BIT(1)
+#define DEV_GMII_PORT_MODE_PORT_MISC_DEV_LOOP_ENA         BIT(0)
+
+#define DEV_GMII_PORT_MODE_EVENTS                         0x8
+
+#define DEV_GMII_PORT_MODE_EEE_CFG                        0xc
+
+#define DEV_GMII_PORT_MODE_EEE_CFG_EEE_ENA                BIT(22)
+#define DEV_GMII_PORT_MODE_EEE_CFG_EEE_TIMER_AGE(x)       (((x) << 15) & GENMASK(21, 15))
+#define DEV_GMII_PORT_MODE_EEE_CFG_EEE_TIMER_AGE_M        GENMASK(21, 15)
+#define DEV_GMII_PORT_MODE_EEE_CFG_EEE_TIMER_AGE_X(x)     (((x) & GENMASK(21, 15)) >> 15)
+#define DEV_GMII_PORT_MODE_EEE_CFG_EEE_TIMER_WAKEUP(x)    (((x) << 8) & GENMASK(14, 8))
+#define DEV_GMII_PORT_MODE_EEE_CFG_EEE_TIMER_WAKEUP_M     GENMASK(14, 8)
+#define DEV_GMII_PORT_MODE_EEE_CFG_EEE_TIMER_WAKEUP_X(x)  (((x) & GENMASK(14, 8)) >> 8)
+#define DEV_GMII_PORT_MODE_EEE_CFG_EEE_TIMER_HOLDOFF(x)   (((x) << 1) & GENMASK(7, 1))
+#define DEV_GMII_PORT_MODE_EEE_CFG_EEE_TIMER_HOLDOFF_M    GENMASK(7, 1)
+#define DEV_GMII_PORT_MODE_EEE_CFG_EEE_TIMER_HOLDOFF_X(x) (((x) & GENMASK(7, 1)) >> 1)
+#define DEV_GMII_PORT_MODE_EEE_CFG_PORT_LPI               BIT(0)
+
+#define DEV_GMII_PORT_MODE_RX_PATH_DELAY                  0x10
+
+#define DEV_GMII_PORT_MODE_TX_PATH_DELAY                  0x14
+
+#define DEV_GMII_PORT_MODE_PTP_PREDICT_CFG                0x18
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_ENA_CFG               0x1c
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_ENA_CFG_RX_ENA        BIT(4)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_ENA_CFG_TX_ENA        BIT(0)
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_MODE_CFG              0x20
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_MODE_CFG_FC_WORD_SYNC_ENA BIT(8)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_MODE_CFG_GIGA_MODE_ENA BIT(4)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_MODE_CFG_FDX_ENA      BIT(0)
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_MAXLEN_CFG            0x24
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_TAGS_CFG              0x28
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_TAGS_CFG_TAG_ID(x)    (((x) << 16) & GENMASK(31, 16))
+#define DEV_GMII_MAC_CFG_STATUS_MAC_TAGS_CFG_TAG_ID_M     GENMASK(31, 16)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_TAGS_CFG_TAG_ID_X(x)  (((x) & GENMASK(31, 16)) >> 16)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_TAGS_CFG_PB_ENA       BIT(1)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_TAGS_CFG_VLAN_AWR_ENA BIT(0)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_TAGS_CFG_VLAN_LEN_AWR_ENA BIT(2)
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_ADV_CHK_CFG           0x2c
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_ADV_CHK_CFG_LEN_DROP_ENA BIT(0)
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_IFG_CFG               0x30
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_IFG_CFG_RESTORE_OLD_IPG_CHECK BIT(17)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_IFG_CFG_REDUCED_TX_IFG BIT(16)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_IFG_CFG_TX_IFG(x)     (((x) << 8) & GENMASK(12, 8))
+#define DEV_GMII_MAC_CFG_STATUS_MAC_IFG_CFG_TX_IFG_M      GENMASK(12, 8)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_IFG_CFG_TX_IFG_X(x)   (((x) & GENMASK(12, 8)) >> 8)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_IFG_CFG_RX_IFG2(x)    (((x) << 4) & GENMASK(7, 4))
+#define DEV_GMII_MAC_CFG_STATUS_MAC_IFG_CFG_RX_IFG2_M     GENMASK(7, 4)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_IFG_CFG_RX_IFG2_X(x)  (((x) & GENMASK(7, 4)) >> 4)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_IFG_CFG_RX_IFG1(x)    ((x) & GENMASK(3, 0))
+#define DEV_GMII_MAC_CFG_STATUS_MAC_IFG_CFG_RX_IFG1_M     GENMASK(3, 0)
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_HDX_CFG               0x34
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_HDX_CFG_BYPASS_COL_SYNC BIT(26)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_HDX_CFG_OB_ENA        BIT(25)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_HDX_CFG_WEXC_DIS      BIT(24)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_HDX_CFG_SEED(x)       (((x) << 16) & GENMASK(23, 16))
+#define DEV_GMII_MAC_CFG_STATUS_MAC_HDX_CFG_SEED_M        GENMASK(23, 16)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_HDX_CFG_SEED_X(x)     (((x) & GENMASK(23, 16)) >> 16)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_HDX_CFG_SEED_LOAD     BIT(12)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_HDX_CFG_RETRY_AFTER_EXC_COL_ENA BIT(8)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_HDX_CFG_LATE_COL_POS(x) ((x) & GENMASK(6, 0))
+#define DEV_GMII_MAC_CFG_STATUS_MAC_HDX_CFG_LATE_COL_POS_M GENMASK(6, 0)
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_DBG_CFG               0x38
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_DBG_CFG_TBI_MODE      BIT(4)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_DBG_CFG_IFG_CRS_EXT_CHK_ENA BIT(0)
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_FC_MAC_LOW_CFG        0x3c
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_FC_MAC_HIGH_CFG       0x40
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_STICKY                0x44
+
+#define DEV_GMII_MAC_CFG_STATUS_MAC_STICKY_RX_IPG_SHRINK_STICKY BIT(9)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_STICKY_RX_PREAM_SHRINK_STICKY BIT(8)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_STICKY_RX_CARRIER_EXT_STICKY BIT(7)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_STICKY_RX_CARRIER_EXT_ERR_STICKY BIT(6)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_STICKY_RX_JUNK_STICKY BIT(5)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_STICKY_TX_RETRANSMIT_STICKY BIT(4)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_STICKY_TX_JAM_STICKY  BIT(3)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_STICKY_TX_FIFO_OFLW_STICKY BIT(2)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_STICKY_TX_FRM_LEN_OVR_STICKY BIT(1)
+#define DEV_GMII_MAC_CFG_STATUS_MAC_STICKY_TX_ABORT_STICKY BIT(0)
+
+#define DEV_GMII_MM_CONFIG_ENABLE_CONFIG                  0x48
+
+#define DEV_GMII_MM_CONFIG_ENABLE_CONFIG_MM_RX_ENA        BIT(0)
+#define DEV_GMII_MM_CONFIG_ENABLE_CONFIG_MM_TX_ENA        BIT(4)
+#define DEV_GMII_MM_CONFIG_ENABLE_CONFIG_KEEP_S_AFTER_D   BIT(8)
+
+#define DEV_GMII_MM_CONFIG_VERIF_CONFIG                   0x4c
+
+#define DEV_GMII_MM_CONFIG_VERIF_CONFIG_PRM_VERIFY_DIS    BIT(0)
+#define DEV_GMII_MM_CONFIG_VERIF_CONFIG_PRM_VERIFY_TIME(x) (((x) << 4) & GENMASK(11, 4))
+#define DEV_GMII_MM_CONFIG_VERIF_CONFIG_PRM_VERIFY_TIME_M GENMASK(11, 4)
+#define DEV_GMII_MM_CONFIG_VERIF_CONFIG_PRM_VERIFY_TIME_X(x) (((x) & GENMASK(11, 4)) >> 4)
+#define DEV_GMII_MM_CONFIG_VERIF_CONFIG_VERIF_TIMER_UNITS(x) (((x) << 12) & GENMASK(13, 12))
+#define DEV_GMII_MM_CONFIG_VERIF_CONFIG_VERIF_TIMER_UNITS_M GENMASK(13, 12)
+#define DEV_GMII_MM_CONFIG_VERIF_CONFIG_VERIF_TIMER_UNITS_X(x) (((x) & GENMASK(13, 12)) >> 12)
+
+#define DEV_GMII_MM_STATISTICS_MM_STATUS                  0x50
+
+#define DEV_GMII_MM_STATISTICS_MM_STATUS_PRMPT_ACTIVE_STATUS BIT(0)
+#define DEV_GMII_MM_STATISTICS_MM_STATUS_PRMPT_ACTIVE_STICKY BIT(4)
+#define DEV_GMII_MM_STATISTICS_MM_STATUS_PRMPT_VERIFY_STATE(x) (((x) << 8) & GENMASK(10, 8))
+#define DEV_GMII_MM_STATISTICS_MM_STATUS_PRMPT_VERIFY_STATE_M GENMASK(10, 8)
+#define DEV_GMII_MM_STATISTICS_MM_STATUS_PRMPT_VERIFY_STATE_X(x) (((x) & GENMASK(10, 8)) >> 8)
+#define DEV_GMII_MM_STATISTICS_MM_STATUS_UNEXP_RX_PFRM_STICKY BIT(12)
+#define DEV_GMII_MM_STATISTICS_MM_STATUS_UNEXP_TX_PFRM_STICKY BIT(16)
+#define DEV_GMII_MM_STATISTICS_MM_STATUS_MM_RX_FRAME_STATUS BIT(20)
+#define DEV_GMII_MM_STATISTICS_MM_STATUS_MM_TX_FRAME_STATUS BIT(24)
+#define DEV_GMII_MM_STATISTICS_MM_STATUS_MM_TX_PRMPT_STATUS BIT(28)
+
+#endif
diff --git a/drivers/net/ethernet/mscc/ocelot_hsio.h b/drivers/net/ethernet/mscc/ocelot_hsio.h
new file mode 100644
index 000000000000..d93ddec3931b
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_hsio.h
@@ -0,0 +1,785 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+
+#ifndef _MSCC_OCELOT_HSIO_H_
+#define _MSCC_OCELOT_HSIO_H_
+
+#define HSIO_PLL5G_CFG0_ENA_ROT                           BIT(31)
+#define HSIO_PLL5G_CFG0_ENA_LANE                          BIT(30)
+#define HSIO_PLL5G_CFG0_ENA_CLKTREE                       BIT(29)
+#define HSIO_PLL5G_CFG0_DIV4                              BIT(28)
+#define HSIO_PLL5G_CFG0_ENA_LOCK_FINE                     BIT(27)
+#define HSIO_PLL5G_CFG0_SELBGV820(x)                      (((x) << 23) & GENMASK(26, 23))
+#define HSIO_PLL5G_CFG0_SELBGV820_M                       GENMASK(26, 23)
+#define HSIO_PLL5G_CFG0_SELBGV820_X(x)                    (((x) & GENMASK(26, 23)) >> 23)
+#define HSIO_PLL5G_CFG0_LOOP_BW_RES(x)                    (((x) << 18) & GENMASK(22, 18))
+#define HSIO_PLL5G_CFG0_LOOP_BW_RES_M                     GENMASK(22, 18)
+#define HSIO_PLL5G_CFG0_LOOP_BW_RES_X(x)                  (((x) & GENMASK(22, 18)) >> 18)
+#define HSIO_PLL5G_CFG0_SELCPI(x)                         (((x) << 16) & GENMASK(17, 16))
+#define HSIO_PLL5G_CFG0_SELCPI_M                          GENMASK(17, 16)
+#define HSIO_PLL5G_CFG0_SELCPI_X(x)                       (((x) & GENMASK(17, 16)) >> 16)
+#define HSIO_PLL5G_CFG0_ENA_VCO_CONTRH                    BIT(15)
+#define HSIO_PLL5G_CFG0_ENA_CP1                           BIT(14)
+#define HSIO_PLL5G_CFG0_ENA_VCO_BUF                       BIT(13)
+#define HSIO_PLL5G_CFG0_ENA_BIAS                          BIT(12)
+#define HSIO_PLL5G_CFG0_CPU_CLK_DIV(x)                    (((x) << 6) & GENMASK(11, 6))
+#define HSIO_PLL5G_CFG0_CPU_CLK_DIV_M                     GENMASK(11, 6)
+#define HSIO_PLL5G_CFG0_CPU_CLK_DIV_X(x)                  (((x) & GENMASK(11, 6)) >> 6)
+#define HSIO_PLL5G_CFG0_CORE_CLK_DIV(x)                   ((x) & GENMASK(5, 0))
+#define HSIO_PLL5G_CFG0_CORE_CLK_DIV_M                    GENMASK(5, 0)
+
+#define HSIO_PLL5G_CFG1_ENA_DIRECT                        BIT(18)
+#define HSIO_PLL5G_CFG1_ROT_SPEED                         BIT(17)
+#define HSIO_PLL5G_CFG1_ROT_DIR                           BIT(16)
+#define HSIO_PLL5G_CFG1_READBACK_DATA_SEL                 BIT(15)
+#define HSIO_PLL5G_CFG1_RC_ENABLE                         BIT(14)
+#define HSIO_PLL5G_CFG1_RC_CTRL_DATA(x)                   (((x) << 6) & GENMASK(13, 6))
+#define HSIO_PLL5G_CFG1_RC_CTRL_DATA_M                    GENMASK(13, 6)
+#define HSIO_PLL5G_CFG1_RC_CTRL_DATA_X(x)                 (((x) & GENMASK(13, 6)) >> 6)
+#define HSIO_PLL5G_CFG1_QUARTER_RATE                      BIT(5)
+#define HSIO_PLL5G_CFG1_PWD_TX                            BIT(4)
+#define HSIO_PLL5G_CFG1_PWD_RX                            BIT(3)
+#define HSIO_PLL5G_CFG1_OUT_OF_RANGE_RECAL_ENA            BIT(2)
+#define HSIO_PLL5G_CFG1_HALF_RATE                         BIT(1)
+#define HSIO_PLL5G_CFG1_FORCE_SET_ENA                     BIT(0)
+
+#define HSIO_PLL5G_CFG2_ENA_TEST_MODE                     BIT(30)
+#define HSIO_PLL5G_CFG2_ENA_PFD_IN_FLIP                   BIT(29)
+#define HSIO_PLL5G_CFG2_ENA_VCO_NREF_TESTOUT              BIT(28)
+#define HSIO_PLL5G_CFG2_ENA_FBTESTOUT                     BIT(27)
+#define HSIO_PLL5G_CFG2_ENA_RCPLL                         BIT(26)
+#define HSIO_PLL5G_CFG2_ENA_CP2                           BIT(25)
+#define HSIO_PLL5G_CFG2_ENA_CLK_BYPASS1                   BIT(24)
+#define HSIO_PLL5G_CFG2_AMPC_SEL(x)                       (((x) << 16) & GENMASK(23, 16))
+#define HSIO_PLL5G_CFG2_AMPC_SEL_M                        GENMASK(23, 16)
+#define HSIO_PLL5G_CFG2_AMPC_SEL_X(x)                     (((x) & GENMASK(23, 16)) >> 16)
+#define HSIO_PLL5G_CFG2_ENA_CLK_BYPASS                    BIT(15)
+#define HSIO_PLL5G_CFG2_PWD_AMPCTRL_N                     BIT(14)
+#define HSIO_PLL5G_CFG2_ENA_AMPCTRL                       BIT(13)
+#define HSIO_PLL5G_CFG2_ENA_AMP_CTRL_FORCE                BIT(12)
+#define HSIO_PLL5G_CFG2_FRC_FSM_POR                       BIT(11)
+#define HSIO_PLL5G_CFG2_DISABLE_FSM_POR                   BIT(10)
+#define HSIO_PLL5G_CFG2_GAIN_TEST(x)                      (((x) << 5) & GENMASK(9, 5))
+#define HSIO_PLL5G_CFG2_GAIN_TEST_M                       GENMASK(9, 5)
+#define HSIO_PLL5G_CFG2_GAIN_TEST_X(x)                    (((x) & GENMASK(9, 5)) >> 5)
+#define HSIO_PLL5G_CFG2_EN_RESET_OVERRUN                  BIT(4)
+#define HSIO_PLL5G_CFG2_EN_RESET_LIM_DET                  BIT(3)
+#define HSIO_PLL5G_CFG2_EN_RESET_FRQ_DET                  BIT(2)
+#define HSIO_PLL5G_CFG2_DISABLE_FSM                       BIT(1)
+#define HSIO_PLL5G_CFG2_ENA_GAIN_TEST                     BIT(0)
+
+#define HSIO_PLL5G_CFG3_TEST_ANA_OUT_SEL(x)               (((x) << 22) & GENMASK(23, 22))
+#define HSIO_PLL5G_CFG3_TEST_ANA_OUT_SEL_M                GENMASK(23, 22)
+#define HSIO_PLL5G_CFG3_TEST_ANA_OUT_SEL_X(x)             (((x) & GENMASK(23, 22)) >> 22)
+#define HSIO_PLL5G_CFG3_TESTOUT_SEL(x)                    (((x) << 19) & GENMASK(21, 19))
+#define HSIO_PLL5G_CFG3_TESTOUT_SEL_M                     GENMASK(21, 19)
+#define HSIO_PLL5G_CFG3_TESTOUT_SEL_X(x)                  (((x) & GENMASK(21, 19)) >> 19)
+#define HSIO_PLL5G_CFG3_ENA_ANA_TEST_OUT                  BIT(18)
+#define HSIO_PLL5G_CFG3_ENA_TEST_OUT                      BIT(17)
+#define HSIO_PLL5G_CFG3_SEL_FBDCLK                        BIT(16)
+#define HSIO_PLL5G_CFG3_SEL_CML_CMOS_PFD                  BIT(15)
+#define HSIO_PLL5G_CFG3_RST_FB_N                          BIT(14)
+#define HSIO_PLL5G_CFG3_FORCE_VCO_CONTRH                  BIT(13)
+#define HSIO_PLL5G_CFG3_FORCE_LO                          BIT(12)
+#define HSIO_PLL5G_CFG3_FORCE_HI                          BIT(11)
+#define HSIO_PLL5G_CFG3_FORCE_ENA                         BIT(10)
+#define HSIO_PLL5G_CFG3_FORCE_CP                          BIT(9)
+#define HSIO_PLL5G_CFG3_FBDIVSEL_TST_ENA                  BIT(8)
+#define HSIO_PLL5G_CFG3_FBDIVSEL(x)                       ((x) & GENMASK(7, 0))
+#define HSIO_PLL5G_CFG3_FBDIVSEL_M                        GENMASK(7, 0)
+
+#define HSIO_PLL5G_CFG4_IB_BIAS_CTRL(x)                   (((x) << 16) & GENMASK(23, 16))
+#define HSIO_PLL5G_CFG4_IB_BIAS_CTRL_M                    GENMASK(23, 16)
+#define HSIO_PLL5G_CFG4_IB_BIAS_CTRL_X(x)                 (((x) & GENMASK(23, 16)) >> 16)
+#define HSIO_PLL5G_CFG4_IB_CTRL(x)                        ((x) & GENMASK(15, 0))
+#define HSIO_PLL5G_CFG4_IB_CTRL_M                         GENMASK(15, 0)
+
+#define HSIO_PLL5G_CFG5_OB_BIAS_CTRL(x)                   (((x) << 16) & GENMASK(23, 16))
+#define HSIO_PLL5G_CFG5_OB_BIAS_CTRL_M                    GENMASK(23, 16)
+#define HSIO_PLL5G_CFG5_OB_BIAS_CTRL_X(x)                 (((x) & GENMASK(23, 16)) >> 16)
+#define HSIO_PLL5G_CFG5_OB_CTRL(x)                        ((x) & GENMASK(15, 0))
+#define HSIO_PLL5G_CFG5_OB_CTRL_M                         GENMASK(15, 0)
+
+#define HSIO_PLL5G_CFG6_REFCLK_SEL_SRC                    BIT(23)
+#define HSIO_PLL5G_CFG6_REFCLK_SEL(x)                     (((x) << 20) & GENMASK(22, 20))
+#define HSIO_PLL5G_CFG6_REFCLK_SEL_M                      GENMASK(22, 20)
+#define HSIO_PLL5G_CFG6_REFCLK_SEL_X(x)                   (((x) & GENMASK(22, 20)) >> 20)
+#define HSIO_PLL5G_CFG6_REFCLK_SRC                        BIT(19)
+#define HSIO_PLL5G_CFG6_POR_DEL_SEL(x)                    (((x) << 16) & GENMASK(17, 16))
+#define HSIO_PLL5G_CFG6_POR_DEL_SEL_M                     GENMASK(17, 16)
+#define HSIO_PLL5G_CFG6_POR_DEL_SEL_X(x)                  (((x) & GENMASK(17, 16)) >> 16)
+#define HSIO_PLL5G_CFG6_DIV125REF_SEL(x)                  (((x) << 8) & GENMASK(15, 8))
+#define HSIO_PLL5G_CFG6_DIV125REF_SEL_M                   GENMASK(15, 8)
+#define HSIO_PLL5G_CFG6_DIV125REF_SEL_X(x)                (((x) & GENMASK(15, 8)) >> 8)
+#define HSIO_PLL5G_CFG6_ENA_REFCLKC2                      BIT(7)
+#define HSIO_PLL5G_CFG6_ENA_FBCLKC2                       BIT(6)
+#define HSIO_PLL5G_CFG6_DDR_CLK_DIV(x)                    ((x) & GENMASK(5, 0))
+#define HSIO_PLL5G_CFG6_DDR_CLK_DIV_M                     GENMASK(5, 0)
+
+#define HSIO_PLL5G_STATUS0_RANGE_LIM                      BIT(12)
+#define HSIO_PLL5G_STATUS0_OUT_OF_RANGE_ERR               BIT(11)
+#define HSIO_PLL5G_STATUS0_CALIBRATION_ERR                BIT(10)
+#define HSIO_PLL5G_STATUS0_CALIBRATION_DONE               BIT(9)
+#define HSIO_PLL5G_STATUS0_READBACK_DATA(x)               (((x) << 1) & GENMASK(8, 1))
+#define HSIO_PLL5G_STATUS0_READBACK_DATA_M                GENMASK(8, 1)
+#define HSIO_PLL5G_STATUS0_READBACK_DATA_X(x)             (((x) & GENMASK(8, 1)) >> 1)
+#define HSIO_PLL5G_STATUS0_LOCK_STATUS                    BIT(0)
+
+#define HSIO_PLL5G_STATUS1_SIG_DEL(x)                     (((x) << 21) & GENMASK(28, 21))
+#define HSIO_PLL5G_STATUS1_SIG_DEL_M                      GENMASK(28, 21)
+#define HSIO_PLL5G_STATUS1_SIG_DEL_X(x)                   (((x) & GENMASK(28, 21)) >> 21)
+#define HSIO_PLL5G_STATUS1_GAIN_STAT(x)                   (((x) << 16) & GENMASK(20, 16))
+#define HSIO_PLL5G_STATUS1_GAIN_STAT_M                    GENMASK(20, 16)
+#define HSIO_PLL5G_STATUS1_GAIN_STAT_X(x)                 (((x) & GENMASK(20, 16)) >> 16)
+#define HSIO_PLL5G_STATUS1_FBCNT_DIF(x)                   (((x) << 4) & GENMASK(13, 4))
+#define HSIO_PLL5G_STATUS1_FBCNT_DIF_M                    GENMASK(13, 4)
+#define HSIO_PLL5G_STATUS1_FBCNT_DIF_X(x)                 (((x) & GENMASK(13, 4)) >> 4)
+#define HSIO_PLL5G_STATUS1_FSM_STAT(x)                    (((x) << 1) & GENMASK(3, 1))
+#define HSIO_PLL5G_STATUS1_FSM_STAT_M                     GENMASK(3, 1)
+#define HSIO_PLL5G_STATUS1_FSM_STAT_X(x)                  (((x) & GENMASK(3, 1)) >> 1)
+#define HSIO_PLL5G_STATUS1_FSM_LOCK                       BIT(0)
+
+#define HSIO_PLL5G_BIST_CFG0_PLLB_START_BIST              BIT(31)
+#define HSIO_PLL5G_BIST_CFG0_PLLB_MEAS_MODE               BIT(30)
+#define HSIO_PLL5G_BIST_CFG0_PLLB_LOCK_REPEAT(x)          (((x) << 20) & GENMASK(23, 20))
+#define HSIO_PLL5G_BIST_CFG0_PLLB_LOCK_REPEAT_M           GENMASK(23, 20)
+#define HSIO_PLL5G_BIST_CFG0_PLLB_LOCK_REPEAT_X(x)        (((x) & GENMASK(23, 20)) >> 20)
+#define HSIO_PLL5G_BIST_CFG0_PLLB_LOCK_UNCERT(x)          (((x) << 16) & GENMASK(19, 16))
+#define HSIO_PLL5G_BIST_CFG0_PLLB_LOCK_UNCERT_M           GENMASK(19, 16)
+#define HSIO_PLL5G_BIST_CFG0_PLLB_LOCK_UNCERT_X(x)        (((x) & GENMASK(19, 16)) >> 16)
+#define HSIO_PLL5G_BIST_CFG0_PLLB_DIV_FACTOR_PRE(x)       ((x) & GENMASK(15, 0))
+#define HSIO_PLL5G_BIST_CFG0_PLLB_DIV_FACTOR_PRE_M        GENMASK(15, 0)
+
+#define HSIO_PLL5G_BIST_STAT0_PLLB_FSM_STAT(x)            (((x) << 4) & GENMASK(7, 4))
+#define HSIO_PLL5G_BIST_STAT0_PLLB_FSM_STAT_M             GENMASK(7, 4)
+#define HSIO_PLL5G_BIST_STAT0_PLLB_FSM_STAT_X(x)          (((x) & GENMASK(7, 4)) >> 4)
+#define HSIO_PLL5G_BIST_STAT0_PLLB_BUSY                   BIT(2)
+#define HSIO_PLL5G_BIST_STAT0_PLLB_DONE_N                 BIT(1)
+#define HSIO_PLL5G_BIST_STAT0_PLLB_FAIL                   BIT(0)
+
+#define HSIO_PLL5G_BIST_STAT1_PLLB_CNT_OUT(x)             (((x) << 16) & GENMASK(31, 16))
+#define HSIO_PLL5G_BIST_STAT1_PLLB_CNT_OUT_M              GENMASK(31, 16)
+#define HSIO_PLL5G_BIST_STAT1_PLLB_CNT_OUT_X(x)           (((x) & GENMASK(31, 16)) >> 16)
+#define HSIO_PLL5G_BIST_STAT1_PLLB_CNT_REF_DIFF(x)        ((x) & GENMASK(15, 0))
+#define HSIO_PLL5G_BIST_STAT1_PLLB_CNT_REF_DIFF_M         GENMASK(15, 0)
+
+#define HSIO_RCOMP_CFG0_PWD_ENA                           BIT(13)
+#define HSIO_RCOMP_CFG0_RUN_CAL                           BIT(12)
+#define HSIO_RCOMP_CFG0_SPEED_SEL(x)                      (((x) << 10) & GENMASK(11, 10))
+#define HSIO_RCOMP_CFG0_SPEED_SEL_M                       GENMASK(11, 10)
+#define HSIO_RCOMP_CFG0_SPEED_SEL_X(x)                    (((x) & GENMASK(11, 10)) >> 10)
+#define HSIO_RCOMP_CFG0_MODE_SEL(x)                       (((x) << 8) & GENMASK(9, 8))
+#define HSIO_RCOMP_CFG0_MODE_SEL_M                        GENMASK(9, 8)
+#define HSIO_RCOMP_CFG0_MODE_SEL_X(x)                     (((x) & GENMASK(9, 8)) >> 8)
+#define HSIO_RCOMP_CFG0_FORCE_ENA                         BIT(4)
+#define HSIO_RCOMP_CFG0_RCOMP_VAL(x)                      ((x) & GENMASK(3, 0))
+#define HSIO_RCOMP_CFG0_RCOMP_VAL_M                       GENMASK(3, 0)
+
+#define HSIO_RCOMP_STATUS_BUSY                            BIT(12)
+#define HSIO_RCOMP_STATUS_DELTA_ALERT                     BIT(7)
+#define HSIO_RCOMP_STATUS_RCOMP(x)                        ((x) & GENMASK(3, 0))
+#define HSIO_RCOMP_STATUS_RCOMP_M                         GENMASK(3, 0)
+
+#define HSIO_SYNC_ETH_CFG_RSZ                             0x4
+
+#define HSIO_SYNC_ETH_CFG_SEL_RECO_CLK_SRC(x)             (((x) << 4) & GENMASK(7, 4))
+#define HSIO_SYNC_ETH_CFG_SEL_RECO_CLK_SRC_M              GENMASK(7, 4)
+#define HSIO_SYNC_ETH_CFG_SEL_RECO_CLK_SRC_X(x)           (((x) & GENMASK(7, 4)) >> 4)
+#define HSIO_SYNC_ETH_CFG_SEL_RECO_CLK_DIV(x)             (((x) << 1) & GENMASK(3, 1))
+#define HSIO_SYNC_ETH_CFG_SEL_RECO_CLK_DIV_M              GENMASK(3, 1)
+#define HSIO_SYNC_ETH_CFG_SEL_RECO_CLK_DIV_X(x)           (((x) & GENMASK(3, 1)) >> 1)
+#define HSIO_SYNC_ETH_CFG_RECO_CLK_ENA                    BIT(0)
+
+#define HSIO_SYNC_ETH_PLL_CFG_PLL_AUTO_SQUELCH_ENA        BIT(0)
+
+#define HSIO_S1G_DES_CFG_DES_PHS_CTRL(x)                  (((x) << 13) & GENMASK(16, 13))
+#define HSIO_S1G_DES_CFG_DES_PHS_CTRL_M                   GENMASK(16, 13)
+#define HSIO_S1G_DES_CFG_DES_PHS_CTRL_X(x)                (((x) & GENMASK(16, 13)) >> 13)
+#define HSIO_S1G_DES_CFG_DES_CPMD_SEL(x)                  (((x) << 11) & GENMASK(12, 11))
+#define HSIO_S1G_DES_CFG_DES_CPMD_SEL_M                   GENMASK(12, 11)
+#define HSIO_S1G_DES_CFG_DES_CPMD_SEL_X(x)                (((x) & GENMASK(12, 11)) >> 11)
+#define HSIO_S1G_DES_CFG_DES_MBTR_CTRL(x)                 (((x) << 8) & GENMASK(10, 8))
+#define HSIO_S1G_DES_CFG_DES_MBTR_CTRL_M                  GENMASK(10, 8)
+#define HSIO_S1G_DES_CFG_DES_MBTR_CTRL_X(x)               (((x) & GENMASK(10, 8)) >> 8)
+#define HSIO_S1G_DES_CFG_DES_BW_ANA(x)                    (((x) << 5) & GENMASK(7, 5))
+#define HSIO_S1G_DES_CFG_DES_BW_ANA_M                     GENMASK(7, 5)
+#define HSIO_S1G_DES_CFG_DES_BW_ANA_X(x)                  (((x) & GENMASK(7, 5)) >> 5)
+#define HSIO_S1G_DES_CFG_DES_SWAP_ANA                     BIT(4)
+#define HSIO_S1G_DES_CFG_DES_BW_HYST(x)                   (((x) << 1) & GENMASK(3, 1))
+#define HSIO_S1G_DES_CFG_DES_BW_HYST_M                    GENMASK(3, 1)
+#define HSIO_S1G_DES_CFG_DES_BW_HYST_X(x)                 (((x) & GENMASK(3, 1)) >> 1)
+#define HSIO_S1G_DES_CFG_DES_SWAP_HYST                    BIT(0)
+
+#define HSIO_S1G_IB_CFG_IB_FX100_ENA                      BIT(27)
+#define HSIO_S1G_IB_CFG_ACJTAG_HYST(x)                    (((x) << 24) & GENMASK(26, 24))
+#define HSIO_S1G_IB_CFG_ACJTAG_HYST_M                     GENMASK(26, 24)
+#define HSIO_S1G_IB_CFG_ACJTAG_HYST_X(x)                  (((x) & GENMASK(26, 24)) >> 24)
+#define HSIO_S1G_IB_CFG_IB_DET_LEV(x)                     (((x) << 19) & GENMASK(21, 19))
+#define HSIO_S1G_IB_CFG_IB_DET_LEV_M                      GENMASK(21, 19)
+#define HSIO_S1G_IB_CFG_IB_DET_LEV_X(x)                   (((x) & GENMASK(21, 19)) >> 19)
+#define HSIO_S1G_IB_CFG_IB_HYST_LEV                       BIT(14)
+#define HSIO_S1G_IB_CFG_IB_ENA_CMV_TERM                   BIT(13)
+#define HSIO_S1G_IB_CFG_IB_ENA_DC_COUPLING                BIT(12)
+#define HSIO_S1G_IB_CFG_IB_ENA_DETLEV                     BIT(11)
+#define HSIO_S1G_IB_CFG_IB_ENA_HYST                       BIT(10)
+#define HSIO_S1G_IB_CFG_IB_ENA_OFFSET_COMP                BIT(9)
+#define HSIO_S1G_IB_CFG_IB_EQ_GAIN(x)                     (((x) << 6) & GENMASK(8, 6))
+#define HSIO_S1G_IB_CFG_IB_EQ_GAIN_M                      GENMASK(8, 6)
+#define HSIO_S1G_IB_CFG_IB_EQ_GAIN_X(x)                   (((x) & GENMASK(8, 6)) >> 6)
+#define HSIO_S1G_IB_CFG_IB_SEL_CORNER_FREQ(x)             (((x) << 4) & GENMASK(5, 4))
+#define HSIO_S1G_IB_CFG_IB_SEL_CORNER_FREQ_M              GENMASK(5, 4)
+#define HSIO_S1G_IB_CFG_IB_SEL_CORNER_FREQ_X(x)           (((x) & GENMASK(5, 4)) >> 4)
+#define HSIO_S1G_IB_CFG_IB_RESISTOR_CTRL(x)               ((x) & GENMASK(3, 0))
+#define HSIO_S1G_IB_CFG_IB_RESISTOR_CTRL_M                GENMASK(3, 0)
+
+#define HSIO_S1G_OB_CFG_OB_SLP(x)                         (((x) << 17) & GENMASK(18, 17))
+#define HSIO_S1G_OB_CFG_OB_SLP_M                          GENMASK(18, 17)
+#define HSIO_S1G_OB_CFG_OB_SLP_X(x)                       (((x) & GENMASK(18, 17)) >> 17)
+#define HSIO_S1G_OB_CFG_OB_AMP_CTRL(x)                    (((x) << 13) & GENMASK(16, 13))
+#define HSIO_S1G_OB_CFG_OB_AMP_CTRL_M                     GENMASK(16, 13)
+#define HSIO_S1G_OB_CFG_OB_AMP_CTRL_X(x)                  (((x) & GENMASK(16, 13)) >> 13)
+#define HSIO_S1G_OB_CFG_OB_CMM_BIAS_CTRL(x)               (((x) << 10) & GENMASK(12, 10))
+#define HSIO_S1G_OB_CFG_OB_CMM_BIAS_CTRL_M                GENMASK(12, 10)
+#define HSIO_S1G_OB_CFG_OB_CMM_BIAS_CTRL_X(x)             (((x) & GENMASK(12, 10)) >> 10)
+#define HSIO_S1G_OB_CFG_OB_DIS_VCM_CTRL                   BIT(9)
+#define HSIO_S1G_OB_CFG_OB_EN_MEAS_VREG                   BIT(8)
+#define HSIO_S1G_OB_CFG_OB_VCM_CTRL(x)                    (((x) << 4) & GENMASK(7, 4))
+#define HSIO_S1G_OB_CFG_OB_VCM_CTRL_M                     GENMASK(7, 4)
+#define HSIO_S1G_OB_CFG_OB_VCM_CTRL_X(x)                  (((x) & GENMASK(7, 4)) >> 4)
+#define HSIO_S1G_OB_CFG_OB_RESISTOR_CTRL(x)               ((x) & GENMASK(3, 0))
+#define HSIO_S1G_OB_CFG_OB_RESISTOR_CTRL_M                GENMASK(3, 0)
+
+#define HSIO_S1G_SER_CFG_SER_IDLE                         BIT(9)
+#define HSIO_S1G_SER_CFG_SER_DEEMPH                       BIT(8)
+#define HSIO_S1G_SER_CFG_SER_CPMD_SEL                     BIT(7)
+#define HSIO_S1G_SER_CFG_SER_SWAP_CPMD                    BIT(6)
+#define HSIO_S1G_SER_CFG_SER_ALISEL(x)                    (((x) << 4) & GENMASK(5, 4))
+#define HSIO_S1G_SER_CFG_SER_ALISEL_M                     GENMASK(5, 4)
+#define HSIO_S1G_SER_CFG_SER_ALISEL_X(x)                  (((x) & GENMASK(5, 4)) >> 4)
+#define HSIO_S1G_SER_CFG_SER_ENHYS                        BIT(3)
+#define HSIO_S1G_SER_CFG_SER_BIG_WIN                      BIT(2)
+#define HSIO_S1G_SER_CFG_SER_EN_WIN                       BIT(1)
+#define HSIO_S1G_SER_CFG_SER_ENALI                        BIT(0)
+
+#define HSIO_S1G_COMMON_CFG_SYS_RST                       BIT(31)
+#define HSIO_S1G_COMMON_CFG_SE_AUTO_SQUELCH_ENA           BIT(21)
+#define HSIO_S1G_COMMON_CFG_ENA_LANE                      BIT(18)
+#define HSIO_S1G_COMMON_CFG_PWD_RX                        BIT(17)
+#define HSIO_S1G_COMMON_CFG_PWD_TX                        BIT(16)
+#define HSIO_S1G_COMMON_CFG_LANE_CTRL(x)                  (((x) << 13) & GENMASK(15, 13))
+#define HSIO_S1G_COMMON_CFG_LANE_CTRL_M                   GENMASK(15, 13)
+#define HSIO_S1G_COMMON_CFG_LANE_CTRL_X(x)                (((x) & GENMASK(15, 13)) >> 13)
+#define HSIO_S1G_COMMON_CFG_ENA_DIRECT                    BIT(12)
+#define HSIO_S1G_COMMON_CFG_ENA_ELOOP                     BIT(11)
+#define HSIO_S1G_COMMON_CFG_ENA_FLOOP                     BIT(10)
+#define HSIO_S1G_COMMON_CFG_ENA_ILOOP                     BIT(9)
+#define HSIO_S1G_COMMON_CFG_ENA_PLOOP                     BIT(8)
+#define HSIO_S1G_COMMON_CFG_HRATE                         BIT(7)
+#define HSIO_S1G_COMMON_CFG_IF_MODE                       BIT(0)
+
+#define HSIO_S1G_PLL_CFG_PLL_ENA_FB_DIV2                  BIT(22)
+#define HSIO_S1G_PLL_CFG_PLL_ENA_RC_DIV2                  BIT(21)
+#define HSIO_S1G_PLL_CFG_PLL_FSM_CTRL_DATA(x)             (((x) << 8) & GENMASK(15, 8))
+#define HSIO_S1G_PLL_CFG_PLL_FSM_CTRL_DATA_M              GENMASK(15, 8)
+#define HSIO_S1G_PLL_CFG_PLL_FSM_CTRL_DATA_X(x)           (((x) & GENMASK(15, 8)) >> 8)
+#define HSIO_S1G_PLL_CFG_PLL_FSM_ENA                      BIT(7)
+#define HSIO_S1G_PLL_CFG_PLL_FSM_FORCE_SET_ENA            BIT(6)
+#define HSIO_S1G_PLL_CFG_PLL_FSM_OOR_RECAL_ENA            BIT(5)
+#define HSIO_S1G_PLL_CFG_PLL_RB_DATA_SEL                  BIT(3)
+
+#define HSIO_S1G_PLL_STATUS_PLL_CAL_NOT_DONE              BIT(12)
+#define HSIO_S1G_PLL_STATUS_PLL_CAL_ERR                   BIT(11)
+#define HSIO_S1G_PLL_STATUS_PLL_OUT_OF_RANGE_ERR          BIT(10)
+#define HSIO_S1G_PLL_STATUS_PLL_RB_DATA(x)                ((x) & GENMASK(7, 0))
+#define HSIO_S1G_PLL_STATUS_PLL_RB_DATA_M                 GENMASK(7, 0)
+
+#define HSIO_S1G_DFT_CFG0_LAZYBIT                         BIT(31)
+#define HSIO_S1G_DFT_CFG0_INV_DIS                         BIT(23)
+#define HSIO_S1G_DFT_CFG0_PRBS_SEL(x)                     (((x) << 20) & GENMASK(21, 20))
+#define HSIO_S1G_DFT_CFG0_PRBS_SEL_M                      GENMASK(21, 20)
+#define HSIO_S1G_DFT_CFG0_PRBS_SEL_X(x)                   (((x) & GENMASK(21, 20)) >> 20)
+#define HSIO_S1G_DFT_CFG0_TEST_MODE(x)                    (((x) << 16) & GENMASK(18, 16))
+#define HSIO_S1G_DFT_CFG0_TEST_MODE_M                     GENMASK(18, 16)
+#define HSIO_S1G_DFT_CFG0_TEST_MODE_X(x)                  (((x) & GENMASK(18, 16)) >> 16)
+#define HSIO_S1G_DFT_CFG0_RX_PHS_CORR_DIS                 BIT(4)
+#define HSIO_S1G_DFT_CFG0_RX_PDSENS_ENA                   BIT(3)
+#define HSIO_S1G_DFT_CFG0_RX_DFT_ENA                      BIT(2)
+#define HSIO_S1G_DFT_CFG0_TX_DFT_ENA                      BIT(0)
+
+#define HSIO_S1G_DFT_CFG1_TX_JITTER_AMPL(x)               (((x) << 8) & GENMASK(17, 8))
+#define HSIO_S1G_DFT_CFG1_TX_JITTER_AMPL_M                GENMASK(17, 8)
+#define HSIO_S1G_DFT_CFG1_TX_JITTER_AMPL_X(x)             (((x) & GENMASK(17, 8)) >> 8)
+#define HSIO_S1G_DFT_CFG1_TX_STEP_FREQ(x)                 (((x) << 4) & GENMASK(7, 4))
+#define HSIO_S1G_DFT_CFG1_TX_STEP_FREQ_M                  GENMASK(7, 4)
+#define HSIO_S1G_DFT_CFG1_TX_STEP_FREQ_X(x)               (((x) & GENMASK(7, 4)) >> 4)
+#define HSIO_S1G_DFT_CFG1_TX_JI_ENA                       BIT(3)
+#define HSIO_S1G_DFT_CFG1_TX_WAVEFORM_SEL                 BIT(2)
+#define HSIO_S1G_DFT_CFG1_TX_FREQOFF_DIR                  BIT(1)
+#define HSIO_S1G_DFT_CFG1_TX_FREQOFF_ENA                  BIT(0)
+
+#define HSIO_S1G_DFT_CFG2_RX_JITTER_AMPL(x)               (((x) << 8) & GENMASK(17, 8))
+#define HSIO_S1G_DFT_CFG2_RX_JITTER_AMPL_M                GENMASK(17, 8)
+#define HSIO_S1G_DFT_CFG2_RX_JITTER_AMPL_X(x)             (((x) & GENMASK(17, 8)) >> 8)
+#define HSIO_S1G_DFT_CFG2_RX_STEP_FREQ(x)                 (((x) << 4) & GENMASK(7, 4))
+#define HSIO_S1G_DFT_CFG2_RX_STEP_FREQ_M                  GENMASK(7, 4)
+#define HSIO_S1G_DFT_CFG2_RX_STEP_FREQ_X(x)               (((x) & GENMASK(7, 4)) >> 4)
+#define HSIO_S1G_DFT_CFG2_RX_JI_ENA                       BIT(3)
+#define HSIO_S1G_DFT_CFG2_RX_WAVEFORM_SEL                 BIT(2)
+#define HSIO_S1G_DFT_CFG2_RX_FREQOFF_DIR                  BIT(1)
+#define HSIO_S1G_DFT_CFG2_RX_FREQOFF_ENA                  BIT(0)
+
+#define HSIO_S1G_RC_PLL_BIST_CFG_PLL_BIST_ENA             BIT(20)
+#define HSIO_S1G_RC_PLL_BIST_CFG_PLL_BIST_FBS_HIGH(x)     (((x) << 16) & GENMASK(17, 16))
+#define HSIO_S1G_RC_PLL_BIST_CFG_PLL_BIST_FBS_HIGH_M      GENMASK(17, 16)
+#define HSIO_S1G_RC_PLL_BIST_CFG_PLL_BIST_FBS_HIGH_X(x)   (((x) & GENMASK(17, 16)) >> 16)
+#define HSIO_S1G_RC_PLL_BIST_CFG_PLL_BIST_HIGH(x)         (((x) << 8) & GENMASK(15, 8))
+#define HSIO_S1G_RC_PLL_BIST_CFG_PLL_BIST_HIGH_M          GENMASK(15, 8)
+#define HSIO_S1G_RC_PLL_BIST_CFG_PLL_BIST_HIGH_X(x)       (((x) & GENMASK(15, 8)) >> 8)
+#define HSIO_S1G_RC_PLL_BIST_CFG_PLL_BIST_LOW(x)          ((x) & GENMASK(7, 0))
+#define HSIO_S1G_RC_PLL_BIST_CFG_PLL_BIST_LOW_M           GENMASK(7, 0)
+
+#define HSIO_S1G_MISC_CFG_DES_100FX_KICK_MODE(x)          (((x) << 11) & GENMASK(12, 11))
+#define HSIO_S1G_MISC_CFG_DES_100FX_KICK_MODE_M           GENMASK(12, 11)
+#define HSIO_S1G_MISC_CFG_DES_100FX_KICK_MODE_X(x)        (((x) & GENMASK(12, 11)) >> 11)
+#define HSIO_S1G_MISC_CFG_DES_100FX_CPMD_SWAP             BIT(10)
+#define HSIO_S1G_MISC_CFG_DES_100FX_CPMD_MODE             BIT(9)
+#define HSIO_S1G_MISC_CFG_DES_100FX_CPMD_ENA              BIT(8)
+#define HSIO_S1G_MISC_CFG_RX_LPI_MODE_ENA                 BIT(5)
+#define HSIO_S1G_MISC_CFG_TX_LPI_MODE_ENA                 BIT(4)
+#define HSIO_S1G_MISC_CFG_RX_DATA_INV_ENA                 BIT(3)
+#define HSIO_S1G_MISC_CFG_TX_DATA_INV_ENA                 BIT(2)
+#define HSIO_S1G_MISC_CFG_LANE_RST                        BIT(0)
+
+#define HSIO_S1G_DFT_STATUS_PLL_BIST_NOT_DONE             BIT(7)
+#define HSIO_S1G_DFT_STATUS_PLL_BIST_FAILED               BIT(6)
+#define HSIO_S1G_DFT_STATUS_PLL_BIST_TIMEOUT_ERR          BIT(5)
+#define HSIO_S1G_DFT_STATUS_BIST_ACTIVE                   BIT(3)
+#define HSIO_S1G_DFT_STATUS_BIST_NOSYNC                   BIT(2)
+#define HSIO_S1G_DFT_STATUS_BIST_COMPLETE_N               BIT(1)
+#define HSIO_S1G_DFT_STATUS_BIST_ERROR                    BIT(0)
+
+#define HSIO_S1G_MISC_STATUS_DES_100FX_PHASE_SEL          BIT(0)
+
+#define HSIO_MCB_S1G_ADDR_CFG_SERDES1G_WR_ONE_SHOT        BIT(31)
+#define HSIO_MCB_S1G_ADDR_CFG_SERDES1G_RD_ONE_SHOT        BIT(30)
+#define HSIO_MCB_S1G_ADDR_CFG_SERDES1G_ADDR(x)            ((x) & GENMASK(8, 0))
+#define HSIO_MCB_S1G_ADDR_CFG_SERDES1G_ADDR_M             GENMASK(8, 0)
+
+#define HSIO_S6G_DIG_CFG_GP(x)                            (((x) << 16) & GENMASK(18, 16))
+#define HSIO_S6G_DIG_CFG_GP_M                             GENMASK(18, 16)
+#define HSIO_S6G_DIG_CFG_GP_X(x)                          (((x) & GENMASK(18, 16)) >> 16)
+#define HSIO_S6G_DIG_CFG_TX_BIT_DOUBLING_MODE_ENA         BIT(7)
+#define HSIO_S6G_DIG_CFG_SIGDET_TESTMODE                  BIT(6)
+#define HSIO_S6G_DIG_CFG_SIGDET_AST(x)                    (((x) << 3) & GENMASK(5, 3))
+#define HSIO_S6G_DIG_CFG_SIGDET_AST_M                     GENMASK(5, 3)
+#define HSIO_S6G_DIG_CFG_SIGDET_AST_X(x)                  (((x) & GENMASK(5, 3)) >> 3)
+#define HSIO_S6G_DIG_CFG_SIGDET_DST(x)                    ((x) & GENMASK(2, 0))
+#define HSIO_S6G_DIG_CFG_SIGDET_DST_M                     GENMASK(2, 0)
+
+#define HSIO_S6G_DFT_CFG0_LAZYBIT                         BIT(31)
+#define HSIO_S6G_DFT_CFG0_INV_DIS                         BIT(23)
+#define HSIO_S6G_DFT_CFG0_PRBS_SEL(x)                     (((x) << 20) & GENMASK(21, 20))
+#define HSIO_S6G_DFT_CFG0_PRBS_SEL_M                      GENMASK(21, 20)
+#define HSIO_S6G_DFT_CFG0_PRBS_SEL_X(x)                   (((x) & GENMASK(21, 20)) >> 20)
+#define HSIO_S6G_DFT_CFG0_TEST_MODE(x)                    (((x) << 16) & GENMASK(18, 16))
+#define HSIO_S6G_DFT_CFG0_TEST_MODE_M                     GENMASK(18, 16)
+#define HSIO_S6G_DFT_CFG0_TEST_MODE_X(x)                  (((x) & GENMASK(18, 16)) >> 16)
+#define HSIO_S6G_DFT_CFG0_RX_PHS_CORR_DIS                 BIT(4)
+#define HSIO_S6G_DFT_CFG0_RX_PDSENS_ENA                   BIT(3)
+#define HSIO_S6G_DFT_CFG0_RX_DFT_ENA                      BIT(2)
+#define HSIO_S6G_DFT_CFG0_TX_DFT_ENA                      BIT(0)
+
+#define HSIO_S6G_DFT_CFG1_TX_JITTER_AMPL(x)               (((x) << 8) & GENMASK(17, 8))
+#define HSIO_S6G_DFT_CFG1_TX_JITTER_AMPL_M                GENMASK(17, 8)
+#define HSIO_S6G_DFT_CFG1_TX_JITTER_AMPL_X(x)             (((x) & GENMASK(17, 8)) >> 8)
+#define HSIO_S6G_DFT_CFG1_TX_STEP_FREQ(x)                 (((x) << 4) & GENMASK(7, 4))
+#define HSIO_S6G_DFT_CFG1_TX_STEP_FREQ_M                  GENMASK(7, 4)
+#define HSIO_S6G_DFT_CFG1_TX_STEP_FREQ_X(x)               (((x) & GENMASK(7, 4)) >> 4)
+#define HSIO_S6G_DFT_CFG1_TX_JI_ENA                       BIT(3)
+#define HSIO_S6G_DFT_CFG1_TX_WAVEFORM_SEL                 BIT(2)
+#define HSIO_S6G_DFT_CFG1_TX_FREQOFF_DIR                  BIT(1)
+#define HSIO_S6G_DFT_CFG1_TX_FREQOFF_ENA                  BIT(0)
+
+#define HSIO_S6G_DFT_CFG2_RX_JITTER_AMPL(x)               (((x) << 8) & GENMASK(17, 8))
+#define HSIO_S6G_DFT_CFG2_RX_JITTER_AMPL_M                GENMASK(17, 8)
+#define HSIO_S6G_DFT_CFG2_RX_JITTER_AMPL_X(x)             (((x) & GENMASK(17, 8)) >> 8)
+#define HSIO_S6G_DFT_CFG2_RX_STEP_FREQ(x)                 (((x) << 4) & GENMASK(7, 4))
+#define HSIO_S6G_DFT_CFG2_RX_STEP_FREQ_M                  GENMASK(7, 4)
+#define HSIO_S6G_DFT_CFG2_RX_STEP_FREQ_X(x)               (((x) & GENMASK(7, 4)) >> 4)
+#define HSIO_S6G_DFT_CFG2_RX_JI_ENA                       BIT(3)
+#define HSIO_S6G_DFT_CFG2_RX_WAVEFORM_SEL                 BIT(2)
+#define HSIO_S6G_DFT_CFG2_RX_FREQOFF_DIR                  BIT(1)
+#define HSIO_S6G_DFT_CFG2_RX_FREQOFF_ENA                  BIT(0)
+
+#define HSIO_S6G_RC_PLL_BIST_CFG_PLL_BIST_ENA             BIT(20)
+#define HSIO_S6G_RC_PLL_BIST_CFG_PLL_BIST_FBS_HIGH(x)     (((x) << 16) & GENMASK(19, 16))
+#define HSIO_S6G_RC_PLL_BIST_CFG_PLL_BIST_FBS_HIGH_M      GENMASK(19, 16)
+#define HSIO_S6G_RC_PLL_BIST_CFG_PLL_BIST_FBS_HIGH_X(x)   (((x) & GENMASK(19, 16)) >> 16)
+#define HSIO_S6G_RC_PLL_BIST_CFG_PLL_BIST_HIGH(x)         (((x) << 8) & GENMASK(15, 8))
+#define HSIO_S6G_RC_PLL_BIST_CFG_PLL_BIST_HIGH_M          GENMASK(15, 8)
+#define HSIO_S6G_RC_PLL_BIST_CFG_PLL_BIST_HIGH_X(x)       (((x) & GENMASK(15, 8)) >> 8)
+#define HSIO_S6G_RC_PLL_BIST_CFG_PLL_BIST_LOW(x)          ((x) & GENMASK(7, 0))
+#define HSIO_S6G_RC_PLL_BIST_CFG_PLL_BIST_LOW_M           GENMASK(7, 0)
+
+#define HSIO_S6G_MISC_CFG_SEL_RECO_CLK(x)                 (((x) << 13) & GENMASK(14, 13))
+#define HSIO_S6G_MISC_CFG_SEL_RECO_CLK_M                  GENMASK(14, 13)
+#define HSIO_S6G_MISC_CFG_SEL_RECO_CLK_X(x)               (((x) & GENMASK(14, 13)) >> 13)
+#define HSIO_S6G_MISC_CFG_DES_100FX_KICK_MODE(x)          (((x) << 11) & GENMASK(12, 11))
+#define HSIO_S6G_MISC_CFG_DES_100FX_KICK_MODE_M           GENMASK(12, 11)
+#define HSIO_S6G_MISC_CFG_DES_100FX_KICK_MODE_X(x)        (((x) & GENMASK(12, 11)) >> 11)
+#define HSIO_S6G_MISC_CFG_DES_100FX_CPMD_SWAP             BIT(10)
+#define HSIO_S6G_MISC_CFG_DES_100FX_CPMD_MODE             BIT(9)
+#define HSIO_S6G_MISC_CFG_DES_100FX_CPMD_ENA              BIT(8)
+#define HSIO_S6G_MISC_CFG_RX_BUS_FLIP_ENA                 BIT(7)
+#define HSIO_S6G_MISC_CFG_TX_BUS_FLIP_ENA                 BIT(6)
+#define HSIO_S6G_MISC_CFG_RX_LPI_MODE_ENA                 BIT(5)
+#define HSIO_S6G_MISC_CFG_TX_LPI_MODE_ENA                 BIT(4)
+#define HSIO_S6G_MISC_CFG_RX_DATA_INV_ENA                 BIT(3)
+#define HSIO_S6G_MISC_CFG_TX_DATA_INV_ENA                 BIT(2)
+#define HSIO_S6G_MISC_CFG_LANE_RST                        BIT(0)
+
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_POST0(x)               (((x) << 23) & GENMASK(28, 23))
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_POST0_M                GENMASK(28, 23)
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_POST0_X(x)             (((x) & GENMASK(28, 23)) >> 23)
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_POST1(x)               (((x) << 18) & GENMASK(22, 18))
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_POST1_M                GENMASK(22, 18)
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_POST1_X(x)             (((x) & GENMASK(22, 18)) >> 18)
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_PREC(x)                (((x) << 13) & GENMASK(17, 13))
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_PREC_M                 GENMASK(17, 13)
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_PREC_X(x)              (((x) & GENMASK(17, 13)) >> 13)
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_ENA_CAS(x)             (((x) << 6) & GENMASK(8, 6))
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_ENA_CAS_M              GENMASK(8, 6)
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_ENA_CAS_X(x)           (((x) & GENMASK(8, 6)) >> 6)
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_LEV(x)                 ((x) & GENMASK(5, 0))
+#define HSIO_S6G_OB_ANEG_CFG_AN_OB_LEV_M                  GENMASK(5, 0)
+
+#define HSIO_S6G_DFT_STATUS_PRBS_SYNC_STAT                BIT(8)
+#define HSIO_S6G_DFT_STATUS_PLL_BIST_NOT_DONE             BIT(7)
+#define HSIO_S6G_DFT_STATUS_PLL_BIST_FAILED               BIT(6)
+#define HSIO_S6G_DFT_STATUS_PLL_BIST_TIMEOUT_ERR          BIT(5)
+#define HSIO_S6G_DFT_STATUS_BIST_ACTIVE                   BIT(3)
+#define HSIO_S6G_DFT_STATUS_BIST_NOSYNC                   BIT(2)
+#define HSIO_S6G_DFT_STATUS_BIST_COMPLETE_N               BIT(1)
+#define HSIO_S6G_DFT_STATUS_BIST_ERROR                    BIT(0)
+
+#define HSIO_S6G_MISC_STATUS_DES_100FX_PHASE_SEL          BIT(0)
+
+#define HSIO_S6G_DES_CFG_DES_PHS_CTRL(x)                  (((x) << 13) & GENMASK(16, 13))
+#define HSIO_S6G_DES_CFG_DES_PHS_CTRL_M                   GENMASK(16, 13)
+#define HSIO_S6G_DES_CFG_DES_PHS_CTRL_X(x)                (((x) & GENMASK(16, 13)) >> 13)
+#define HSIO_S6G_DES_CFG_DES_MBTR_CTRL(x)                 (((x) << 10) & GENMASK(12, 10))
+#define HSIO_S6G_DES_CFG_DES_MBTR_CTRL_M                  GENMASK(12, 10)
+#define HSIO_S6G_DES_CFG_DES_MBTR_CTRL_X(x)               (((x) & GENMASK(12, 10)) >> 10)
+#define HSIO_S6G_DES_CFG_DES_CPMD_SEL(x)                  (((x) << 8) & GENMASK(9, 8))
+#define HSIO_S6G_DES_CFG_DES_CPMD_SEL_M                   GENMASK(9, 8)
+#define HSIO_S6G_DES_CFG_DES_CPMD_SEL_X(x)                (((x) & GENMASK(9, 8)) >> 8)
+#define HSIO_S6G_DES_CFG_DES_BW_HYST(x)                   (((x) << 5) & GENMASK(7, 5))
+#define HSIO_S6G_DES_CFG_DES_BW_HYST_M                    GENMASK(7, 5)
+#define HSIO_S6G_DES_CFG_DES_BW_HYST_X(x)                 (((x) & GENMASK(7, 5)) >> 5)
+#define HSIO_S6G_DES_CFG_DES_SWAP_HYST                    BIT(4)
+#define HSIO_S6G_DES_CFG_DES_BW_ANA(x)                    (((x) << 1) & GENMASK(3, 1))
+#define HSIO_S6G_DES_CFG_DES_BW_ANA_M                     GENMASK(3, 1)
+#define HSIO_S6G_DES_CFG_DES_BW_ANA_X(x)                  (((x) & GENMASK(3, 1)) >> 1)
+#define HSIO_S6G_DES_CFG_DES_SWAP_ANA                     BIT(0)
+
+#define HSIO_S6G_IB_CFG_IB_SOFSI(x)                       (((x) << 29) & GENMASK(30, 29))
+#define HSIO_S6G_IB_CFG_IB_SOFSI_M                        GENMASK(30, 29)
+#define HSIO_S6G_IB_CFG_IB_SOFSI_X(x)                     (((x) & GENMASK(30, 29)) >> 29)
+#define HSIO_S6G_IB_CFG_IB_VBULK_SEL                      BIT(28)
+#define HSIO_S6G_IB_CFG_IB_RTRM_ADJ(x)                    (((x) << 24) & GENMASK(27, 24))
+#define HSIO_S6G_IB_CFG_IB_RTRM_ADJ_M                     GENMASK(27, 24)
+#define HSIO_S6G_IB_CFG_IB_RTRM_ADJ_X(x)                  (((x) & GENMASK(27, 24)) >> 24)
+#define HSIO_S6G_IB_CFG_IB_ICML_ADJ(x)                    (((x) << 20) & GENMASK(23, 20))
+#define HSIO_S6G_IB_CFG_IB_ICML_ADJ_M                     GENMASK(23, 20)
+#define HSIO_S6G_IB_CFG_IB_ICML_ADJ_X(x)                  (((x) & GENMASK(23, 20)) >> 20)
+#define HSIO_S6G_IB_CFG_IB_TERM_MODE_SEL(x)               (((x) << 18) & GENMASK(19, 18))
+#define HSIO_S6G_IB_CFG_IB_TERM_MODE_SEL_M                GENMASK(19, 18)
+#define HSIO_S6G_IB_CFG_IB_TERM_MODE_SEL_X(x)             (((x) & GENMASK(19, 18)) >> 18)
+#define HSIO_S6G_IB_CFG_IB_SIG_DET_CLK_SEL(x)             (((x) << 15) & GENMASK(17, 15))
+#define HSIO_S6G_IB_CFG_IB_SIG_DET_CLK_SEL_M              GENMASK(17, 15)
+#define HSIO_S6G_IB_CFG_IB_SIG_DET_CLK_SEL_X(x)           (((x) & GENMASK(17, 15)) >> 15)
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_HP(x)              (((x) << 13) & GENMASK(14, 13))
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_HP_M               GENMASK(14, 13)
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_HP_X(x)            (((x) & GENMASK(14, 13)) >> 13)
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_MID(x)             (((x) << 11) & GENMASK(12, 11))
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_MID_M              GENMASK(12, 11)
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_MID_X(x)           (((x) & GENMASK(12, 11)) >> 11)
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_LP(x)              (((x) << 9) & GENMASK(10, 9))
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_LP_M               GENMASK(10, 9)
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_LP_X(x)            (((x) & GENMASK(10, 9)) >> 9)
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_OFFSET(x)          (((x) << 7) & GENMASK(8, 7))
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_OFFSET_M           GENMASK(8, 7)
+#define HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_OFFSET_X(x)        (((x) & GENMASK(8, 7)) >> 7)
+#define HSIO_S6G_IB_CFG_IB_ANA_TEST_ENA                   BIT(6)
+#define HSIO_S6G_IB_CFG_IB_SIG_DET_ENA                    BIT(5)
+#define HSIO_S6G_IB_CFG_IB_CONCUR                         BIT(4)
+#define HSIO_S6G_IB_CFG_IB_CAL_ENA                        BIT(3)
+#define HSIO_S6G_IB_CFG_IB_SAM_ENA                        BIT(2)
+#define HSIO_S6G_IB_CFG_IB_EQZ_ENA                        BIT(1)
+#define HSIO_S6G_IB_CFG_IB_REG_ENA                        BIT(0)
+
+#define HSIO_S6G_IB_CFG1_IB_TJTAG(x)                      (((x) << 17) & GENMASK(21, 17))
+#define HSIO_S6G_IB_CFG1_IB_TJTAG_M                       GENMASK(21, 17)
+#define HSIO_S6G_IB_CFG1_IB_TJTAG_X(x)                    (((x) & GENMASK(21, 17)) >> 17)
+#define HSIO_S6G_IB_CFG1_IB_TSDET(x)                      (((x) << 12) & GENMASK(16, 12))
+#define HSIO_S6G_IB_CFG1_IB_TSDET_M                       GENMASK(16, 12)
+#define HSIO_S6G_IB_CFG1_IB_TSDET_X(x)                    (((x) & GENMASK(16, 12)) >> 12)
+#define HSIO_S6G_IB_CFG1_IB_SCALY(x)                      (((x) << 8) & GENMASK(11, 8))
+#define HSIO_S6G_IB_CFG1_IB_SCALY_M                       GENMASK(11, 8)
+#define HSIO_S6G_IB_CFG1_IB_SCALY_X(x)                    (((x) & GENMASK(11, 8)) >> 8)
+#define HSIO_S6G_IB_CFG1_IB_FILT_HP                       BIT(7)
+#define HSIO_S6G_IB_CFG1_IB_FILT_MID                      BIT(6)
+#define HSIO_S6G_IB_CFG1_IB_FILT_LP                       BIT(5)
+#define HSIO_S6G_IB_CFG1_IB_FILT_OFFSET                   BIT(4)
+#define HSIO_S6G_IB_CFG1_IB_FRC_HP                        BIT(3)
+#define HSIO_S6G_IB_CFG1_IB_FRC_MID                       BIT(2)
+#define HSIO_S6G_IB_CFG1_IB_FRC_LP                        BIT(1)
+#define HSIO_S6G_IB_CFG1_IB_FRC_OFFSET                    BIT(0)
+
+#define HSIO_S6G_IB_CFG2_IB_TINFV(x)                      (((x) << 27) & GENMASK(29, 27))
+#define HSIO_S6G_IB_CFG2_IB_TINFV_M                       GENMASK(29, 27)
+#define HSIO_S6G_IB_CFG2_IB_TINFV_X(x)                    (((x) & GENMASK(29, 27)) >> 27)
+#define HSIO_S6G_IB_CFG2_IB_OINFI(x)                      (((x) << 22) & GENMASK(26, 22))
+#define HSIO_S6G_IB_CFG2_IB_OINFI_M                       GENMASK(26, 22)
+#define HSIO_S6G_IB_CFG2_IB_OINFI_X(x)                    (((x) & GENMASK(26, 22)) >> 22)
+#define HSIO_S6G_IB_CFG2_IB_TAUX(x)                       (((x) << 19) & GENMASK(21, 19))
+#define HSIO_S6G_IB_CFG2_IB_TAUX_M                        GENMASK(21, 19)
+#define HSIO_S6G_IB_CFG2_IB_TAUX_X(x)                     (((x) & GENMASK(21, 19)) >> 19)
+#define HSIO_S6G_IB_CFG2_IB_OINFS(x)                      (((x) << 16) & GENMASK(18, 16))
+#define HSIO_S6G_IB_CFG2_IB_OINFS_M                       GENMASK(18, 16)
+#define HSIO_S6G_IB_CFG2_IB_OINFS_X(x)                    (((x) & GENMASK(18, 16)) >> 16)
+#define HSIO_S6G_IB_CFG2_IB_OCALS(x)                      (((x) << 10) & GENMASK(15, 10))
+#define HSIO_S6G_IB_CFG2_IB_OCALS_M                       GENMASK(15, 10)
+#define HSIO_S6G_IB_CFG2_IB_OCALS_X(x)                    (((x) & GENMASK(15, 10)) >> 10)
+#define HSIO_S6G_IB_CFG2_IB_TCALV(x)                      (((x) << 5) & GENMASK(9, 5))
+#define HSIO_S6G_IB_CFG2_IB_TCALV_M                       GENMASK(9, 5)
+#define HSIO_S6G_IB_CFG2_IB_TCALV_X(x)                    (((x) & GENMASK(9, 5)) >> 5)
+#define HSIO_S6G_IB_CFG2_IB_UMAX(x)                       (((x) << 3) & GENMASK(4, 3))
+#define HSIO_S6G_IB_CFG2_IB_UMAX_M                        GENMASK(4, 3)
+#define HSIO_S6G_IB_CFG2_IB_UMAX_X(x)                     (((x) & GENMASK(4, 3)) >> 3)
+#define HSIO_S6G_IB_CFG2_IB_UREG(x)                       ((x) & GENMASK(2, 0))
+#define HSIO_S6G_IB_CFG2_IB_UREG_M                        GENMASK(2, 0)
+
+#define HSIO_S6G_IB_CFG3_IB_INI_HP(x)                     (((x) << 18) & GENMASK(23, 18))
+#define HSIO_S6G_IB_CFG3_IB_INI_HP_M                      GENMASK(23, 18)
+#define HSIO_S6G_IB_CFG3_IB_INI_HP_X(x)                   (((x) & GENMASK(23, 18)) >> 18)
+#define HSIO_S6G_IB_CFG3_IB_INI_MID(x)                    (((x) << 12) & GENMASK(17, 12))
+#define HSIO_S6G_IB_CFG3_IB_INI_MID_M                     GENMASK(17, 12)
+#define HSIO_S6G_IB_CFG3_IB_INI_MID_X(x)                  (((x) & GENMASK(17, 12)) >> 12)
+#define HSIO_S6G_IB_CFG3_IB_INI_LP(x)                     (((x) << 6) & GENMASK(11, 6))
+#define HSIO_S6G_IB_CFG3_IB_INI_LP_M                      GENMASK(11, 6)
+#define HSIO_S6G_IB_CFG3_IB_INI_LP_X(x)                   (((x) & GENMASK(11, 6)) >> 6)
+#define HSIO_S6G_IB_CFG3_IB_INI_OFFSET(x)                 ((x) & GENMASK(5, 0))
+#define HSIO_S6G_IB_CFG3_IB_INI_OFFSET_M                  GENMASK(5, 0)
+
+#define HSIO_S6G_IB_CFG4_IB_MAX_HP(x)                     (((x) << 18) & GENMASK(23, 18))
+#define HSIO_S6G_IB_CFG4_IB_MAX_HP_M                      GENMASK(23, 18)
+#define HSIO_S6G_IB_CFG4_IB_MAX_HP_X(x)                   (((x) & GENMASK(23, 18)) >> 18)
+#define HSIO_S6G_IB_CFG4_IB_MAX_MID(x)                    (((x) << 12) & GENMASK(17, 12))
+#define HSIO_S6G_IB_CFG4_IB_MAX_MID_M                     GENMASK(17, 12)
+#define HSIO_S6G_IB_CFG4_IB_MAX_MID_X(x)                  (((x) & GENMASK(17, 12)) >> 12)
+#define HSIO_S6G_IB_CFG4_IB_MAX_LP(x)                     (((x) << 6) & GENMASK(11, 6))
+#define HSIO_S6G_IB_CFG4_IB_MAX_LP_M                      GENMASK(11, 6)
+#define HSIO_S6G_IB_CFG4_IB_MAX_LP_X(x)                   (((x) & GENMASK(11, 6)) >> 6)
+#define HSIO_S6G_IB_CFG4_IB_MAX_OFFSET(x)                 ((x) & GENMASK(5, 0))
+#define HSIO_S6G_IB_CFG4_IB_MAX_OFFSET_M                  GENMASK(5, 0)
+
+#define HSIO_S6G_IB_CFG5_IB_MIN_HP(x)                     (((x) << 18) & GENMASK(23, 18))
+#define HSIO_S6G_IB_CFG5_IB_MIN_HP_M                      GENMASK(23, 18)
+#define HSIO_S6G_IB_CFG5_IB_MIN_HP_X(x)                   (((x) & GENMASK(23, 18)) >> 18)
+#define HSIO_S6G_IB_CFG5_IB_MIN_MID(x)                    (((x) << 12) & GENMASK(17, 12))
+#define HSIO_S6G_IB_CFG5_IB_MIN_MID_M                     GENMASK(17, 12)
+#define HSIO_S6G_IB_CFG5_IB_MIN_MID_X(x)                  (((x) & GENMASK(17, 12)) >> 12)
+#define HSIO_S6G_IB_CFG5_IB_MIN_LP(x)                     (((x) << 6) & GENMASK(11, 6))
+#define HSIO_S6G_IB_CFG5_IB_MIN_LP_M                      GENMASK(11, 6)
+#define HSIO_S6G_IB_CFG5_IB_MIN_LP_X(x)                   (((x) & GENMASK(11, 6)) >> 6)
+#define HSIO_S6G_IB_CFG5_IB_MIN_OFFSET(x)                 ((x) & GENMASK(5, 0))
+#define HSIO_S6G_IB_CFG5_IB_MIN_OFFSET_M                  GENMASK(5, 0)
+
+#define HSIO_S6G_OB_CFG_OB_IDLE                           BIT(31)
+#define HSIO_S6G_OB_CFG_OB_ENA1V_MODE                     BIT(30)
+#define HSIO_S6G_OB_CFG_OB_POL                            BIT(29)
+#define HSIO_S6G_OB_CFG_OB_POST0(x)                       (((x) << 23) & GENMASK(28, 23))
+#define HSIO_S6G_OB_CFG_OB_POST0_M                        GENMASK(28, 23)
+#define HSIO_S6G_OB_CFG_OB_POST0_X(x)                     (((x) & GENMASK(28, 23)) >> 23)
+#define HSIO_S6G_OB_CFG_OB_PREC(x)                        (((x) << 18) & GENMASK(22, 18))
+#define HSIO_S6G_OB_CFG_OB_PREC_M                         GENMASK(22, 18)
+#define HSIO_S6G_OB_CFG_OB_PREC_X(x)                      (((x) & GENMASK(22, 18)) >> 18)
+#define HSIO_S6G_OB_CFG_OB_R_ADJ_MUX                      BIT(17)
+#define HSIO_S6G_OB_CFG_OB_R_ADJ_PDR                      BIT(16)
+#define HSIO_S6G_OB_CFG_OB_POST1(x)                       (((x) << 11) & GENMASK(15, 11))
+#define HSIO_S6G_OB_CFG_OB_POST1_M                        GENMASK(15, 11)
+#define HSIO_S6G_OB_CFG_OB_POST1_X(x)                     (((x) & GENMASK(15, 11)) >> 11)
+#define HSIO_S6G_OB_CFG_OB_R_COR                          BIT(10)
+#define HSIO_S6G_OB_CFG_OB_SEL_RCTRL                      BIT(9)
+#define HSIO_S6G_OB_CFG_OB_SR_H                           BIT(8)
+#define HSIO_S6G_OB_CFG_OB_SR(x)                          (((x) << 4) & GENMASK(7, 4))
+#define HSIO_S6G_OB_CFG_OB_SR_M                           GENMASK(7, 4)
+#define HSIO_S6G_OB_CFG_OB_SR_X(x)                        (((x) & GENMASK(7, 4)) >> 4)
+#define HSIO_S6G_OB_CFG_OB_RESISTOR_CTRL(x)               ((x) & GENMASK(3, 0))
+#define HSIO_S6G_OB_CFG_OB_RESISTOR_CTRL_M                GENMASK(3, 0)
+
+#define HSIO_S6G_OB_CFG1_OB_ENA_CAS(x)                    (((x) << 6) & GENMASK(8, 6))
+#define HSIO_S6G_OB_CFG1_OB_ENA_CAS_M                     GENMASK(8, 6)
+#define HSIO_S6G_OB_CFG1_OB_ENA_CAS_X(x)                  (((x) & GENMASK(8, 6)) >> 6)
+#define HSIO_S6G_OB_CFG1_OB_LEV(x)                        ((x) & GENMASK(5, 0))
+#define HSIO_S6G_OB_CFG1_OB_LEV_M                         GENMASK(5, 0)
+
+#define HSIO_S6G_SER_CFG_SER_4TAP_ENA                     BIT(8)
+#define HSIO_S6G_SER_CFG_SER_CPMD_SEL                     BIT(7)
+#define HSIO_S6G_SER_CFG_SER_SWAP_CPMD                    BIT(6)
+#define HSIO_S6G_SER_CFG_SER_ALISEL(x)                    (((x) << 4) & GENMASK(5, 4))
+#define HSIO_S6G_SER_CFG_SER_ALISEL_M                     GENMASK(5, 4)
+#define HSIO_S6G_SER_CFG_SER_ALISEL_X(x)                  (((x) & GENMASK(5, 4)) >> 4)
+#define HSIO_S6G_SER_CFG_SER_ENHYS                        BIT(3)
+#define HSIO_S6G_SER_CFG_SER_BIG_WIN                      BIT(2)
+#define HSIO_S6G_SER_CFG_SER_EN_WIN                       BIT(1)
+#define HSIO_S6G_SER_CFG_SER_ENALI                        BIT(0)
+
+#define HSIO_S6G_COMMON_CFG_SYS_RST                       BIT(17)
+#define HSIO_S6G_COMMON_CFG_SE_DIV2_ENA                   BIT(16)
+#define HSIO_S6G_COMMON_CFG_SE_AUTO_SQUELCH_ENA           BIT(15)
+#define HSIO_S6G_COMMON_CFG_ENA_LANE                      BIT(14)
+#define HSIO_S6G_COMMON_CFG_PWD_RX                        BIT(13)
+#define HSIO_S6G_COMMON_CFG_PWD_TX                        BIT(12)
+#define HSIO_S6G_COMMON_CFG_LANE_CTRL(x)                  (((x) << 9) & GENMASK(11, 9))
+#define HSIO_S6G_COMMON_CFG_LANE_CTRL_M                   GENMASK(11, 9)
+#define HSIO_S6G_COMMON_CFG_LANE_CTRL_X(x)                (((x) & GENMASK(11, 9)) >> 9)
+#define HSIO_S6G_COMMON_CFG_ENA_DIRECT                    BIT(8)
+#define HSIO_S6G_COMMON_CFG_ENA_ELOOP                     BIT(7)
+#define HSIO_S6G_COMMON_CFG_ENA_FLOOP                     BIT(6)
+#define HSIO_S6G_COMMON_CFG_ENA_ILOOP                     BIT(5)
+#define HSIO_S6G_COMMON_CFG_ENA_PLOOP                     BIT(4)
+#define HSIO_S6G_COMMON_CFG_HRATE                         BIT(3)
+#define HSIO_S6G_COMMON_CFG_QRATE                         BIT(2)
+#define HSIO_S6G_COMMON_CFG_IF_MODE(x)                    ((x) & GENMASK(1, 0))
+#define HSIO_S6G_COMMON_CFG_IF_MODE_M                     GENMASK(1, 0)
+
+#define HSIO_S6G_PLL_CFG_PLL_ENA_OFFS(x)                  (((x) << 16) & GENMASK(17, 16))
+#define HSIO_S6G_PLL_CFG_PLL_ENA_OFFS_M                   GENMASK(17, 16)
+#define HSIO_S6G_PLL_CFG_PLL_ENA_OFFS_X(x)                (((x) & GENMASK(17, 16)) >> 16)
+#define HSIO_S6G_PLL_CFG_PLL_DIV4                         BIT(15)
+#define HSIO_S6G_PLL_CFG_PLL_ENA_ROT                      BIT(14)
+#define HSIO_S6G_PLL_CFG_PLL_FSM_CTRL_DATA(x)             (((x) << 6) & GENMASK(13, 6))
+#define HSIO_S6G_PLL_CFG_PLL_FSM_CTRL_DATA_M              GENMASK(13, 6)
+#define HSIO_S6G_PLL_CFG_PLL_FSM_CTRL_DATA_X(x)           (((x) & GENMASK(13, 6)) >> 6)
+#define HSIO_S6G_PLL_CFG_PLL_FSM_ENA                      BIT(5)
+#define HSIO_S6G_PLL_CFG_PLL_FSM_FORCE_SET_ENA            BIT(4)
+#define HSIO_S6G_PLL_CFG_PLL_FSM_OOR_RECAL_ENA            BIT(3)
+#define HSIO_S6G_PLL_CFG_PLL_RB_DATA_SEL                  BIT(2)
+#define HSIO_S6G_PLL_CFG_PLL_ROT_DIR                      BIT(1)
+#define HSIO_S6G_PLL_CFG_PLL_ROT_FRQ                      BIT(0)
+
+#define HSIO_S6G_ACJTAG_CFG_ACJTAG_INIT_DATA_N            BIT(5)
+#define HSIO_S6G_ACJTAG_CFG_ACJTAG_INIT_DATA_P            BIT(4)
+#define HSIO_S6G_ACJTAG_CFG_ACJTAG_INIT_CLK               BIT(3)
+#define HSIO_S6G_ACJTAG_CFG_OB_DIRECT                     BIT(2)
+#define HSIO_S6G_ACJTAG_CFG_ACJTAG_ENA                    BIT(1)
+#define HSIO_S6G_ACJTAG_CFG_JTAG_CTRL_ENA                 BIT(0)
+
+#define HSIO_S6G_GP_CFG_GP_MSB(x)                         (((x) << 16) & GENMASK(31, 16))
+#define HSIO_S6G_GP_CFG_GP_MSB_M                          GENMASK(31, 16)
+#define HSIO_S6G_GP_CFG_GP_MSB_X(x)                       (((x) & GENMASK(31, 16)) >> 16)
+#define HSIO_S6G_GP_CFG_GP_LSB(x)                         ((x) & GENMASK(15, 0))
+#define HSIO_S6G_GP_CFG_GP_LSB_M                          GENMASK(15, 0)
+
+#define HSIO_S6G_IB_STATUS0_IB_CAL_DONE                   BIT(8)
+#define HSIO_S6G_IB_STATUS0_IB_HP_GAIN_ACT                BIT(7)
+#define HSIO_S6G_IB_STATUS0_IB_MID_GAIN_ACT               BIT(6)
+#define HSIO_S6G_IB_STATUS0_IB_LP_GAIN_ACT                BIT(5)
+#define HSIO_S6G_IB_STATUS0_IB_OFFSET_ACT                 BIT(4)
+#define HSIO_S6G_IB_STATUS0_IB_OFFSET_VLD                 BIT(3)
+#define HSIO_S6G_IB_STATUS0_IB_OFFSET_ERR                 BIT(2)
+#define HSIO_S6G_IB_STATUS0_IB_OFFSDIR                    BIT(1)
+#define HSIO_S6G_IB_STATUS0_IB_SIG_DET                    BIT(0)
+
+#define HSIO_S6G_IB_STATUS1_IB_HP_GAIN_STAT(x)            (((x) << 18) & GENMASK(23, 18))
+#define HSIO_S6G_IB_STATUS1_IB_HP_GAIN_STAT_M             GENMASK(23, 18)
+#define HSIO_S6G_IB_STATUS1_IB_HP_GAIN_STAT_X(x)          (((x) & GENMASK(23, 18)) >> 18)
+#define HSIO_S6G_IB_STATUS1_IB_MID_GAIN_STAT(x)           (((x) << 12) & GENMASK(17, 12))
+#define HSIO_S6G_IB_STATUS1_IB_MID_GAIN_STAT_M            GENMASK(17, 12)
+#define HSIO_S6G_IB_STATUS1_IB_MID_GAIN_STAT_X(x)         (((x) & GENMASK(17, 12)) >> 12)
+#define HSIO_S6G_IB_STATUS1_IB_LP_GAIN_STAT(x)            (((x) << 6) & GENMASK(11, 6))
+#define HSIO_S6G_IB_STATUS1_IB_LP_GAIN_STAT_M             GENMASK(11, 6)
+#define HSIO_S6G_IB_STATUS1_IB_LP_GAIN_STAT_X(x)          (((x) & GENMASK(11, 6)) >> 6)
+#define HSIO_S6G_IB_STATUS1_IB_OFFSET_STAT(x)             ((x) & GENMASK(5, 0))
+#define HSIO_S6G_IB_STATUS1_IB_OFFSET_STAT_M              GENMASK(5, 0)
+
+#define HSIO_S6G_ACJTAG_STATUS_ACJTAG_CAPT_DATA_N         BIT(2)
+#define HSIO_S6G_ACJTAG_STATUS_ACJTAG_CAPT_DATA_P         BIT(1)
+#define HSIO_S6G_ACJTAG_STATUS_IB_DIRECT                  BIT(0)
+
+#define HSIO_S6G_PLL_STATUS_PLL_CAL_NOT_DONE              BIT(10)
+#define HSIO_S6G_PLL_STATUS_PLL_CAL_ERR                   BIT(9)
+#define HSIO_S6G_PLL_STATUS_PLL_OUT_OF_RANGE_ERR          BIT(8)
+#define HSIO_S6G_PLL_STATUS_PLL_RB_DATA(x)                ((x) & GENMASK(7, 0))
+#define HSIO_S6G_PLL_STATUS_PLL_RB_DATA_M                 GENMASK(7, 0)
+
+#define HSIO_S6G_REVID_SERDES_REV(x)                      (((x) << 26) & GENMASK(31, 26))
+#define HSIO_S6G_REVID_SERDES_REV_M                       GENMASK(31, 26)
+#define HSIO_S6G_REVID_SERDES_REV_X(x)                    (((x) & GENMASK(31, 26)) >> 26)
+#define HSIO_S6G_REVID_RCPLL_REV(x)                       (((x) << 21) & GENMASK(25, 21))
+#define HSIO_S6G_REVID_RCPLL_REV_M                        GENMASK(25, 21)
+#define HSIO_S6G_REVID_RCPLL_REV_X(x)                     (((x) & GENMASK(25, 21)) >> 21)
+#define HSIO_S6G_REVID_SER_REV(x)                         (((x) << 16) & GENMASK(20, 16))
+#define HSIO_S6G_REVID_SER_REV_M                          GENMASK(20, 16)
+#define HSIO_S6G_REVID_SER_REV_X(x)                       (((x) & GENMASK(20, 16)) >> 16)
+#define HSIO_S6G_REVID_DES_REV(x)                         (((x) << 10) & GENMASK(15, 10))
+#define HSIO_S6G_REVID_DES_REV_M                          GENMASK(15, 10)
+#define HSIO_S6G_REVID_DES_REV_X(x)                       (((x) & GENMASK(15, 10)) >> 10)
+#define HSIO_S6G_REVID_OB_REV(x)                          (((x) << 5) & GENMASK(9, 5))
+#define HSIO_S6G_REVID_OB_REV_M                           GENMASK(9, 5)
+#define HSIO_S6G_REVID_OB_REV_X(x)                        (((x) & GENMASK(9, 5)) >> 5)
+#define HSIO_S6G_REVID_IB_REV(x)                          ((x) & GENMASK(4, 0))
+#define HSIO_S6G_REVID_IB_REV_M                           GENMASK(4, 0)
+
+#define HSIO_MCB_S6G_ADDR_CFG_SERDES6G_WR_ONE_SHOT        BIT(31)
+#define HSIO_MCB_S6G_ADDR_CFG_SERDES6G_RD_ONE_SHOT        BIT(30)
+#define HSIO_MCB_S6G_ADDR_CFG_SERDES6G_ADDR(x)            ((x) & GENMASK(24, 0))
+#define HSIO_MCB_S6G_ADDR_CFG_SERDES6G_ADDR_M             GENMASK(24, 0)
+
+#define HSIO_HW_CFG_DEV2G5_10_MODE                        BIT(6)
+#define HSIO_HW_CFG_DEV1G_9_MODE                          BIT(5)
+#define HSIO_HW_CFG_DEV1G_6_MODE                          BIT(4)
+#define HSIO_HW_CFG_DEV1G_5_MODE                          BIT(3)
+#define HSIO_HW_CFG_DEV1G_4_MODE                          BIT(2)
+#define HSIO_HW_CFG_PCIE_ENA                              BIT(1)
+#define HSIO_HW_CFG_QSGMII_ENA                            BIT(0)
+
+#define HSIO_HW_QSGMII_CFG_SHYST_DIS                      BIT(3)
+#define HSIO_HW_QSGMII_CFG_E_DET_ENA                      BIT(2)
+#define HSIO_HW_QSGMII_CFG_USE_I1_ENA                     BIT(1)
+#define HSIO_HW_QSGMII_CFG_FLIP_LANES                     BIT(0)
+
+#define HSIO_HW_QSGMII_STAT_DELAY_VAR_X200PS(x)           (((x) << 1) & GENMASK(6, 1))
+#define HSIO_HW_QSGMII_STAT_DELAY_VAR_X200PS_M            GENMASK(6, 1)
+#define HSIO_HW_QSGMII_STAT_DELAY_VAR_X200PS_X(x)         (((x) & GENMASK(6, 1)) >> 1)
+#define HSIO_HW_QSGMII_STAT_SYNC                          BIT(0)
+
+#define HSIO_CLK_CFG_CLKDIV_PHY(x)                        (((x) << 1) & GENMASK(8, 1))
+#define HSIO_CLK_CFG_CLKDIV_PHY_M                         GENMASK(8, 1)
+#define HSIO_CLK_CFG_CLKDIV_PHY_X(x)                      (((x) & GENMASK(8, 1)) >> 1)
+#define HSIO_CLK_CFG_CLKDIV_PHY_DIS                       BIT(0)
+
+#define HSIO_TEMP_SENSOR_CTRL_FORCE_TEMP_RD               BIT(5)
+#define HSIO_TEMP_SENSOR_CTRL_FORCE_RUN                   BIT(4)
+#define HSIO_TEMP_SENSOR_CTRL_FORCE_NO_RST                BIT(3)
+#define HSIO_TEMP_SENSOR_CTRL_FORCE_POWER_UP              BIT(2)
+#define HSIO_TEMP_SENSOR_CTRL_FORCE_CLK                   BIT(1)
+#define HSIO_TEMP_SENSOR_CTRL_SAMPLE_ENA                  BIT(0)
+
+#define HSIO_TEMP_SENSOR_CFG_RUN_WID(x)                   (((x) << 8) & GENMASK(15, 8))
+#define HSIO_TEMP_SENSOR_CFG_RUN_WID_M                    GENMASK(15, 8)
+#define HSIO_TEMP_SENSOR_CFG_RUN_WID_X(x)                 (((x) & GENMASK(15, 8)) >> 8)
+#define HSIO_TEMP_SENSOR_CFG_SAMPLE_PER(x)                ((x) & GENMASK(7, 0))
+#define HSIO_TEMP_SENSOR_CFG_SAMPLE_PER_M                 GENMASK(7, 0)
+
+#define HSIO_TEMP_SENSOR_STAT_TEMP_VALID                  BIT(8)
+#define HSIO_TEMP_SENSOR_STAT_TEMP(x)                     ((x) & GENMASK(7, 0))
+#define HSIO_TEMP_SENSOR_STAT_TEMP_M                      GENMASK(7, 0)
+
+#endif
diff --git a/drivers/net/ethernet/mscc/ocelot_io.c b/drivers/net/ethernet/mscc/ocelot_io.c
new file mode 100644
index 000000000000..c6db8ad31fdf
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_io.c
@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/platform_device.h>
+
+#include "ocelot.h"
+
+u32 __ocelot_read_ix(struct ocelot *ocelot, u32 reg, u32 offset)
+{
+	u16 target = reg >> TARGET_OFFSET;
+	u32 val;
+
+	WARN_ON(!target);
+
+	regmap_read(ocelot->targets[target],
+		    ocelot->map[target][reg & REG_MASK] + offset, &val);
+	return val;
+}
+EXPORT_SYMBOL(__ocelot_read_ix);
+
+void __ocelot_write_ix(struct ocelot *ocelot, u32 val, u32 reg, u32 offset)
+{
+	u16 target = reg >> TARGET_OFFSET;
+
+	WARN_ON(!target);
+
+	regmap_write(ocelot->targets[target],
+		     ocelot->map[target][reg & REG_MASK] + offset, val);
+}
+EXPORT_SYMBOL(__ocelot_write_ix);
+
+void __ocelot_rmw_ix(struct ocelot *ocelot, u32 val, u32 mask, u32 reg,
+		     u32 offset)
+{
+	u16 target = reg >> TARGET_OFFSET;
+
+	WARN_ON(!target);
+
+	regmap_update_bits(ocelot->targets[target],
+			   ocelot->map[target][reg & REG_MASK] + offset,
+			   mask, val);
+}
+EXPORT_SYMBOL(__ocelot_rmw_ix);
+
+u32 ocelot_port_readl(struct ocelot_port *port, u32 reg)
+{
+	return readl(port->regs + reg);
+}
+EXPORT_SYMBOL(ocelot_port_readl);
+
+void ocelot_port_writel(struct ocelot_port *port, u32 val, u32 reg)
+{
+	writel(val, port->regs + reg);
+}
+EXPORT_SYMBOL(ocelot_port_writel);
+
+int ocelot_regfields_init(struct ocelot *ocelot,
+			  const struct reg_field *const regfields)
+{
+	unsigned int i;
+	u16 target;
+
+	for (i = 0; i < REGFIELD_MAX; i++) {
+		struct reg_field regfield = {};
+		u32 reg = regfields[i].reg;
+
+		if (!reg)
+			continue;
+
+		target = regfields[i].reg >> TARGET_OFFSET;
+
+		regfield.reg = ocelot->map[target][reg & REG_MASK];
+		regfield.lsb = regfields[i].lsb;
+		regfield.msb = regfields[i].msb;
+
+		ocelot->regfields[i] =
+		devm_regmap_field_alloc(ocelot->dev,
+					ocelot->targets[target],
+					regfield);
+
+		if (IS_ERR(ocelot->regfields[i]))
+			return PTR_ERR(ocelot->regfields[i]);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(ocelot_regfields_init);
+
+static struct regmap_config ocelot_regmap_config = {
+	.reg_bits	= 32,
+	.val_bits	= 32,
+	.reg_stride	= 4,
+};
+
+struct regmap *ocelot_io_platform_init(struct ocelot *ocelot,
+				       struct platform_device *pdev,
+				       const char *name)
+{
+	struct resource *res;
+	void __iomem *regs;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
+	regs = devm_ioremap_resource(ocelot->dev, res);
+	if (IS_ERR(regs))
+		return ERR_CAST(regs);
+
+	ocelot_regmap_config.name = name;
+	return devm_regmap_init_mmio(ocelot->dev, regs,
+				     &ocelot_regmap_config);
+}
+EXPORT_SYMBOL(ocelot_io_platform_init);
diff --git a/drivers/net/ethernet/mscc/ocelot_qs.h b/drivers/net/ethernet/mscc/ocelot_qs.h
new file mode 100644
index 000000000000..d18ae726c01d
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_qs.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+
+#ifndef _MSCC_OCELOT_QS_H_
+#define _MSCC_OCELOT_QS_H_
+
+/* TODO handle BE */
+#define XTR_EOF_0          0x00000080U
+#define XTR_EOF_1          0x01000080U
+#define XTR_EOF_2          0x02000080U
+#define XTR_EOF_3          0x03000080U
+#define XTR_PRUNED         0x04000080U
+#define XTR_ABORT          0x05000080U
+#define XTR_ESCAPE         0x06000080U
+#define XTR_NOT_READY      0x07000080U
+#define XTR_VALID_BYTES(x) (4 - (((x) >> 24) & 3))
+
+#define QS_XTR_GRP_CFG_RSZ                                0x4
+
+#define QS_XTR_GRP_CFG_MODE(x)                            (((x) << 2) & GENMASK(3, 2))
+#define QS_XTR_GRP_CFG_MODE_M                             GENMASK(3, 2)
+#define QS_XTR_GRP_CFG_MODE_X(x)                          (((x) & GENMASK(3, 2)) >> 2)
+#define QS_XTR_GRP_CFG_STATUS_WORD_POS                    BIT(1)
+#define QS_XTR_GRP_CFG_BYTE_SWAP                          BIT(0)
+
+#define QS_XTR_RD_RSZ                                     0x4
+
+#define QS_XTR_FRM_PRUNING_RSZ                            0x4
+
+#define QS_XTR_CFG_DP_WM(x)                               (((x) << 5) & GENMASK(7, 5))
+#define QS_XTR_CFG_DP_WM_M                                GENMASK(7, 5)
+#define QS_XTR_CFG_DP_WM_X(x)                             (((x) & GENMASK(7, 5)) >> 5)
+#define QS_XTR_CFG_SCH_WM(x)                              (((x) << 2) & GENMASK(4, 2))
+#define QS_XTR_CFG_SCH_WM_M                               GENMASK(4, 2)
+#define QS_XTR_CFG_SCH_WM_X(x)                            (((x) & GENMASK(4, 2)) >> 2)
+#define QS_XTR_CFG_OFLW_ERR_STICKY(x)                     ((x) & GENMASK(1, 0))
+#define QS_XTR_CFG_OFLW_ERR_STICKY_M                      GENMASK(1, 0)
+
+#define QS_INJ_GRP_CFG_RSZ                                0x4
+
+#define QS_INJ_GRP_CFG_MODE(x)                            (((x) << 2) & GENMASK(3, 2))
+#define QS_INJ_GRP_CFG_MODE_M                             GENMASK(3, 2)
+#define QS_INJ_GRP_CFG_MODE_X(x)                          (((x) & GENMASK(3, 2)) >> 2)
+#define QS_INJ_GRP_CFG_BYTE_SWAP                          BIT(0)
+
+#define QS_INJ_WR_RSZ                                     0x4
+
+#define QS_INJ_CTRL_RSZ                                   0x4
+
+#define QS_INJ_CTRL_GAP_SIZE(x)                           (((x) << 21) & GENMASK(24, 21))
+#define QS_INJ_CTRL_GAP_SIZE_M                            GENMASK(24, 21)
+#define QS_INJ_CTRL_GAP_SIZE_X(x)                         (((x) & GENMASK(24, 21)) >> 21)
+#define QS_INJ_CTRL_ABORT                                 BIT(20)
+#define QS_INJ_CTRL_EOF                                   BIT(19)
+#define QS_INJ_CTRL_SOF                                   BIT(18)
+#define QS_INJ_CTRL_VLD_BYTES(x)                          (((x) << 16) & GENMASK(17, 16))
+#define QS_INJ_CTRL_VLD_BYTES_M                           GENMASK(17, 16)
+#define QS_INJ_CTRL_VLD_BYTES_X(x)                        (((x) & GENMASK(17, 16)) >> 16)
+
+#define QS_INJ_STATUS_WMARK_REACHED(x)                    (((x) << 4) & GENMASK(5, 4))
+#define QS_INJ_STATUS_WMARK_REACHED_M                     GENMASK(5, 4)
+#define QS_INJ_STATUS_WMARK_REACHED_X(x)                  (((x) & GENMASK(5, 4)) >> 4)
+#define QS_INJ_STATUS_FIFO_RDY(x)                         (((x) << 2) & GENMASK(3, 2))
+#define QS_INJ_STATUS_FIFO_RDY_M                          GENMASK(3, 2)
+#define QS_INJ_STATUS_FIFO_RDY_X(x)                       (((x) & GENMASK(3, 2)) >> 2)
+#define QS_INJ_STATUS_INJ_IN_PROGRESS(x)                  ((x) & GENMASK(1, 0))
+#define QS_INJ_STATUS_INJ_IN_PROGRESS_M                   GENMASK(1, 0)
+
+#define QS_INJ_ERR_RSZ                                    0x4
+
+#define QS_INJ_ERR_ABORT_ERR_STICKY                       BIT(1)
+#define QS_INJ_ERR_WR_ERR_STICKY                          BIT(0)
+
+#endif
diff --git a/drivers/net/ethernet/mscc/ocelot_qsys.h b/drivers/net/ethernet/mscc/ocelot_qsys.h
new file mode 100644
index 000000000000..d8c63aa761be
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_qsys.h
@@ -0,0 +1,270 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+
+#ifndef _MSCC_OCELOT_QSYS_H_
+#define _MSCC_OCELOT_QSYS_H_
+
+#define QSYS_PORT_MODE_RSZ                                0x4
+
+#define QSYS_PORT_MODE_DEQUEUE_DIS                        BIT(1)
+#define QSYS_PORT_MODE_DEQUEUE_LATE                       BIT(0)
+
+#define QSYS_SWITCH_PORT_MODE_RSZ                         0x4
+
+#define QSYS_SWITCH_PORT_MODE_PORT_ENA                    BIT(14)
+#define QSYS_SWITCH_PORT_MODE_SCH_NEXT_CFG(x)             (((x) << 11) & GENMASK(13, 11))
+#define QSYS_SWITCH_PORT_MODE_SCH_NEXT_CFG_M              GENMASK(13, 11)
+#define QSYS_SWITCH_PORT_MODE_SCH_NEXT_CFG_X(x)           (((x) & GENMASK(13, 11)) >> 11)
+#define QSYS_SWITCH_PORT_MODE_YEL_RSRVD                   BIT(10)
+#define QSYS_SWITCH_PORT_MODE_INGRESS_DROP_MODE           BIT(9)
+#define QSYS_SWITCH_PORT_MODE_TX_PFC_ENA(x)               (((x) << 1) & GENMASK(8, 1))
+#define QSYS_SWITCH_PORT_MODE_TX_PFC_ENA_M                GENMASK(8, 1)
+#define QSYS_SWITCH_PORT_MODE_TX_PFC_ENA_X(x)             (((x) & GENMASK(8, 1)) >> 1)
+#define QSYS_SWITCH_PORT_MODE_TX_PFC_MODE                 BIT(0)
+
+#define QSYS_STAT_CNT_CFG_TX_GREEN_CNT_MODE               BIT(5)
+#define QSYS_STAT_CNT_CFG_TX_YELLOW_CNT_MODE              BIT(4)
+#define QSYS_STAT_CNT_CFG_DROP_GREEN_CNT_MODE             BIT(3)
+#define QSYS_STAT_CNT_CFG_DROP_YELLOW_CNT_MODE            BIT(2)
+#define QSYS_STAT_CNT_CFG_DROP_COUNT_ONCE                 BIT(1)
+#define QSYS_STAT_CNT_CFG_DROP_COUNT_EGRESS               BIT(0)
+
+#define QSYS_EEE_CFG_RSZ                                  0x4
+
+#define QSYS_EEE_THRES_EEE_HIGH_BYTES(x)                  (((x) << 8) & GENMASK(15, 8))
+#define QSYS_EEE_THRES_EEE_HIGH_BYTES_M                   GENMASK(15, 8)
+#define QSYS_EEE_THRES_EEE_HIGH_BYTES_X(x)                (((x) & GENMASK(15, 8)) >> 8)
+#define QSYS_EEE_THRES_EEE_HIGH_FRAMES(x)                 ((x) & GENMASK(7, 0))
+#define QSYS_EEE_THRES_EEE_HIGH_FRAMES_M                  GENMASK(7, 0)
+
+#define QSYS_SW_STATUS_RSZ                                0x4
+
+#define QSYS_EXT_CPU_CFG_EXT_CPU_PORT(x)                  (((x) << 8) & GENMASK(12, 8))
+#define QSYS_EXT_CPU_CFG_EXT_CPU_PORT_M                   GENMASK(12, 8)
+#define QSYS_EXT_CPU_CFG_EXT_CPU_PORT_X(x)                (((x) & GENMASK(12, 8)) >> 8)
+#define QSYS_EXT_CPU_CFG_EXT_CPUQ_MSK(x)                  ((x) & GENMASK(7, 0))
+#define QSYS_EXT_CPU_CFG_EXT_CPUQ_MSK_M                   GENMASK(7, 0)
+
+#define QSYS_QMAP_GSZ                                     0x4
+
+#define QSYS_QMAP_SE_BASE(x)                              (((x) << 5) & GENMASK(12, 5))
+#define QSYS_QMAP_SE_BASE_M                               GENMASK(12, 5)
+#define QSYS_QMAP_SE_BASE_X(x)                            (((x) & GENMASK(12, 5)) >> 5)
+#define QSYS_QMAP_SE_IDX_SEL(x)                           (((x) << 2) & GENMASK(4, 2))
+#define QSYS_QMAP_SE_IDX_SEL_M                            GENMASK(4, 2)
+#define QSYS_QMAP_SE_IDX_SEL_X(x)                         (((x) & GENMASK(4, 2)) >> 2)
+#define QSYS_QMAP_SE_INP_SEL(x)                           ((x) & GENMASK(1, 0))
+#define QSYS_QMAP_SE_INP_SEL_M                            GENMASK(1, 0)
+
+#define QSYS_ISDX_SGRP_GSZ                                0x4
+
+#define QSYS_TIMED_FRAME_ENTRY_GSZ                        0x4
+
+#define QSYS_TFRM_MISC_TIMED_CANCEL_SLOT(x)               (((x) << 9) & GENMASK(18, 9))
+#define QSYS_TFRM_MISC_TIMED_CANCEL_SLOT_M                GENMASK(18, 9)
+#define QSYS_TFRM_MISC_TIMED_CANCEL_SLOT_X(x)             (((x) & GENMASK(18, 9)) >> 9)
+#define QSYS_TFRM_MISC_TIMED_CANCEL_1SHOT                 BIT(8)
+#define QSYS_TFRM_MISC_TIMED_SLOT_MODE_MC                 BIT(7)
+#define QSYS_TFRM_MISC_TIMED_ENTRY_FAST_CNT(x)            ((x) & GENMASK(6, 0))
+#define QSYS_TFRM_MISC_TIMED_ENTRY_FAST_CNT_M             GENMASK(6, 0)
+
+#define QSYS_RED_PROFILE_RSZ                              0x4
+
+#define QSYS_RED_PROFILE_WM_RED_LOW(x)                    (((x) << 8) & GENMASK(15, 8))
+#define QSYS_RED_PROFILE_WM_RED_LOW_M                     GENMASK(15, 8)
+#define QSYS_RED_PROFILE_WM_RED_LOW_X(x)                  (((x) & GENMASK(15, 8)) >> 8)
+#define QSYS_RED_PROFILE_WM_RED_HIGH(x)                   ((x) & GENMASK(7, 0))
+#define QSYS_RED_PROFILE_WM_RED_HIGH_M                    GENMASK(7, 0)
+
+#define QSYS_RES_CFG_GSZ                                  0x8
+
+#define QSYS_RES_STAT_GSZ                                 0x8
+
+#define QSYS_RES_STAT_INUSE(x)                            (((x) << 12) & GENMASK(23, 12))
+#define QSYS_RES_STAT_INUSE_M                             GENMASK(23, 12)
+#define QSYS_RES_STAT_INUSE_X(x)                          (((x) & GENMASK(23, 12)) >> 12)
+#define QSYS_RES_STAT_MAXUSE(x)                           ((x) & GENMASK(11, 0))
+#define QSYS_RES_STAT_MAXUSE_M                            GENMASK(11, 0)
+
+#define QSYS_EVENTS_CORE_EV_FDC(x)                        (((x) << 2) & GENMASK(4, 2))
+#define QSYS_EVENTS_CORE_EV_FDC_M                         GENMASK(4, 2)
+#define QSYS_EVENTS_CORE_EV_FDC_X(x)                      (((x) & GENMASK(4, 2)) >> 2)
+#define QSYS_EVENTS_CORE_EV_FRD(x)                        ((x) & GENMASK(1, 0))
+#define QSYS_EVENTS_CORE_EV_FRD_M                         GENMASK(1, 0)
+
+#define QSYS_QMAXSDU_CFG_0_RSZ                            0x4
+
+#define QSYS_QMAXSDU_CFG_1_RSZ                            0x4
+
+#define QSYS_QMAXSDU_CFG_2_RSZ                            0x4
+
+#define QSYS_QMAXSDU_CFG_3_RSZ                            0x4
+
+#define QSYS_QMAXSDU_CFG_4_RSZ                            0x4
+
+#define QSYS_QMAXSDU_CFG_5_RSZ                            0x4
+
+#define QSYS_QMAXSDU_CFG_6_RSZ                            0x4
+
+#define QSYS_QMAXSDU_CFG_7_RSZ                            0x4
+
+#define QSYS_PREEMPTION_CFG_RSZ                           0x4
+
+#define QSYS_PREEMPTION_CFG_P_QUEUES(x)                   ((x) & GENMASK(7, 0))
+#define QSYS_PREEMPTION_CFG_P_QUEUES_M                    GENMASK(7, 0)
+#define QSYS_PREEMPTION_CFG_MM_ADD_FRAG_SIZE(x)           (((x) << 8) & GENMASK(9, 8))
+#define QSYS_PREEMPTION_CFG_MM_ADD_FRAG_SIZE_M            GENMASK(9, 8)
+#define QSYS_PREEMPTION_CFG_MM_ADD_FRAG_SIZE_X(x)         (((x) & GENMASK(9, 8)) >> 8)
+#define QSYS_PREEMPTION_CFG_STRICT_IPG(x)                 (((x) << 12) & GENMASK(13, 12))
+#define QSYS_PREEMPTION_CFG_STRICT_IPG_M                  GENMASK(13, 12)
+#define QSYS_PREEMPTION_CFG_STRICT_IPG_X(x)               (((x) & GENMASK(13, 12)) >> 12)
+#define QSYS_PREEMPTION_CFG_HOLD_ADVANCE(x)               (((x) << 16) & GENMASK(31, 16))
+#define QSYS_PREEMPTION_CFG_HOLD_ADVANCE_M                GENMASK(31, 16)
+#define QSYS_PREEMPTION_CFG_HOLD_ADVANCE_X(x)             (((x) & GENMASK(31, 16)) >> 16)
+
+#define QSYS_CIR_CFG_GSZ                                  0x80
+
+#define QSYS_CIR_CFG_CIR_RATE(x)                          (((x) << 6) & GENMASK(20, 6))
+#define QSYS_CIR_CFG_CIR_RATE_M                           GENMASK(20, 6)
+#define QSYS_CIR_CFG_CIR_RATE_X(x)                        (((x) & GENMASK(20, 6)) >> 6)
+#define QSYS_CIR_CFG_CIR_BURST(x)                         ((x) & GENMASK(5, 0))
+#define QSYS_CIR_CFG_CIR_BURST_M                          GENMASK(5, 0)
+
+#define QSYS_EIR_CFG_GSZ                                  0x80
+
+#define QSYS_EIR_CFG_EIR_RATE(x)                          (((x) << 7) & GENMASK(21, 7))
+#define QSYS_EIR_CFG_EIR_RATE_M                           GENMASK(21, 7)
+#define QSYS_EIR_CFG_EIR_RATE_X(x)                        (((x) & GENMASK(21, 7)) >> 7)
+#define QSYS_EIR_CFG_EIR_BURST(x)                         (((x) << 1) & GENMASK(6, 1))
+#define QSYS_EIR_CFG_EIR_BURST_M                          GENMASK(6, 1)
+#define QSYS_EIR_CFG_EIR_BURST_X(x)                       (((x) & GENMASK(6, 1)) >> 1)
+#define QSYS_EIR_CFG_EIR_MARK_ENA                         BIT(0)
+
+#define QSYS_SE_CFG_GSZ                                   0x80
+
+#define QSYS_SE_CFG_SE_DWRR_CNT(x)                        (((x) << 6) & GENMASK(9, 6))
+#define QSYS_SE_CFG_SE_DWRR_CNT_M                         GENMASK(9, 6)
+#define QSYS_SE_CFG_SE_DWRR_CNT_X(x)                      (((x) & GENMASK(9, 6)) >> 6)
+#define QSYS_SE_CFG_SE_RR_ENA                             BIT(5)
+#define QSYS_SE_CFG_SE_AVB_ENA                            BIT(4)
+#define QSYS_SE_CFG_SE_FRM_MODE(x)                        (((x) << 2) & GENMASK(3, 2))
+#define QSYS_SE_CFG_SE_FRM_MODE_M                         GENMASK(3, 2)
+#define QSYS_SE_CFG_SE_FRM_MODE_X(x)                      (((x) & GENMASK(3, 2)) >> 2)
+#define QSYS_SE_CFG_SE_EXC_ENA                            BIT(1)
+#define QSYS_SE_CFG_SE_EXC_FWD                            BIT(0)
+
+#define QSYS_SE_DWRR_CFG_GSZ                              0x80
+#define QSYS_SE_DWRR_CFG_RSZ                              0x4
+
+#define QSYS_SE_CONNECT_GSZ                               0x80
+
+#define QSYS_SE_CONNECT_SE_OUTP_IDX(x)                    (((x) << 17) & GENMASK(24, 17))
+#define QSYS_SE_CONNECT_SE_OUTP_IDX_M                     GENMASK(24, 17)
+#define QSYS_SE_CONNECT_SE_OUTP_IDX_X(x)                  (((x) & GENMASK(24, 17)) >> 17)
+#define QSYS_SE_CONNECT_SE_INP_IDX(x)                     (((x) << 9) & GENMASK(16, 9))
+#define QSYS_SE_CONNECT_SE_INP_IDX_M                      GENMASK(16, 9)
+#define QSYS_SE_CONNECT_SE_INP_IDX_X(x)                   (((x) & GENMASK(16, 9)) >> 9)
+#define QSYS_SE_CONNECT_SE_OUTP_CON(x)                    (((x) << 5) & GENMASK(8, 5))
+#define QSYS_SE_CONNECT_SE_OUTP_CON_M                     GENMASK(8, 5)
+#define QSYS_SE_CONNECT_SE_OUTP_CON_X(x)                  (((x) & GENMASK(8, 5)) >> 5)
+#define QSYS_SE_CONNECT_SE_INP_CNT(x)                     (((x) << 1) & GENMASK(4, 1))
+#define QSYS_SE_CONNECT_SE_INP_CNT_M                      GENMASK(4, 1)
+#define QSYS_SE_CONNECT_SE_INP_CNT_X(x)                   (((x) & GENMASK(4, 1)) >> 1)
+#define QSYS_SE_CONNECT_SE_TERMINAL                       BIT(0)
+
+#define QSYS_SE_DLB_SENSE_GSZ                             0x80
+
+#define QSYS_SE_DLB_SENSE_SE_DLB_PRIO(x)                  (((x) << 11) & GENMASK(13, 11))
+#define QSYS_SE_DLB_SENSE_SE_DLB_PRIO_M                   GENMASK(13, 11)
+#define QSYS_SE_DLB_SENSE_SE_DLB_PRIO_X(x)                (((x) & GENMASK(13, 11)) >> 11)
+#define QSYS_SE_DLB_SENSE_SE_DLB_SPORT(x)                 (((x) << 7) & GENMASK(10, 7))
+#define QSYS_SE_DLB_SENSE_SE_DLB_SPORT_M                  GENMASK(10, 7)
+#define QSYS_SE_DLB_SENSE_SE_DLB_SPORT_X(x)               (((x) & GENMASK(10, 7)) >> 7)
+#define QSYS_SE_DLB_SENSE_SE_DLB_DPORT(x)                 (((x) << 3) & GENMASK(6, 3))
+#define QSYS_SE_DLB_SENSE_SE_DLB_DPORT_M                  GENMASK(6, 3)
+#define QSYS_SE_DLB_SENSE_SE_DLB_DPORT_X(x)               (((x) & GENMASK(6, 3)) >> 3)
+#define QSYS_SE_DLB_SENSE_SE_DLB_PRIO_ENA                 BIT(2)
+#define QSYS_SE_DLB_SENSE_SE_DLB_SPORT_ENA                BIT(1)
+#define QSYS_SE_DLB_SENSE_SE_DLB_DPORT_ENA                BIT(0)
+
+#define QSYS_CIR_STATE_GSZ                                0x80
+
+#define QSYS_CIR_STATE_CIR_LVL(x)                         (((x) << 4) & GENMASK(25, 4))
+#define QSYS_CIR_STATE_CIR_LVL_M                          GENMASK(25, 4)
+#define QSYS_CIR_STATE_CIR_LVL_X(x)                       (((x) & GENMASK(25, 4)) >> 4)
+#define QSYS_CIR_STATE_SHP_TIME(x)                        ((x) & GENMASK(3, 0))
+#define QSYS_CIR_STATE_SHP_TIME_M                         GENMASK(3, 0)
+
+#define QSYS_EIR_STATE_GSZ                                0x80
+
+#define QSYS_SE_STATE_GSZ                                 0x80
+
+#define QSYS_SE_STATE_SE_OUTP_LVL(x)                      (((x) << 1) & GENMASK(2, 1))
+#define QSYS_SE_STATE_SE_OUTP_LVL_M                       GENMASK(2, 1)
+#define QSYS_SE_STATE_SE_OUTP_LVL_X(x)                    (((x) & GENMASK(2, 1)) >> 1)
+#define QSYS_SE_STATE_SE_WAS_YEL                          BIT(0)
+
+#define QSYS_HSCH_MISC_CFG_SE_CONNECT_VLD                 BIT(8)
+#define QSYS_HSCH_MISC_CFG_FRM_ADJ(x)                     (((x) << 3) & GENMASK(7, 3))
+#define QSYS_HSCH_MISC_CFG_FRM_ADJ_M                      GENMASK(7, 3)
+#define QSYS_HSCH_MISC_CFG_FRM_ADJ_X(x)                   (((x) & GENMASK(7, 3)) >> 3)
+#define QSYS_HSCH_MISC_CFG_LEAK_DIS                       BIT(2)
+#define QSYS_HSCH_MISC_CFG_QSHP_EXC_ENA                   BIT(1)
+#define QSYS_HSCH_MISC_CFG_PFC_BYP_UPD                    BIT(0)
+
+#define QSYS_TAG_CONFIG_RSZ                               0x4
+
+#define QSYS_TAG_CONFIG_ENABLE                            BIT(0)
+#define QSYS_TAG_CONFIG_LINK_SPEED(x)                     (((x) << 4) & GENMASK(5, 4))
+#define QSYS_TAG_CONFIG_LINK_SPEED_M                      GENMASK(5, 4)
+#define QSYS_TAG_CONFIG_LINK_SPEED_X(x)                   (((x) & GENMASK(5, 4)) >> 4)
+#define QSYS_TAG_CONFIG_INIT_GATE_STATE(x)                (((x) << 8) & GENMASK(15, 8))
+#define QSYS_TAG_CONFIG_INIT_GATE_STATE_M                 GENMASK(15, 8)
+#define QSYS_TAG_CONFIG_INIT_GATE_STATE_X(x)              (((x) & GENMASK(15, 8)) >> 8)
+#define QSYS_TAG_CONFIG_SCH_TRAFFIC_QUEUES(x)             (((x) << 16) & GENMASK(23, 16))
+#define QSYS_TAG_CONFIG_SCH_TRAFFIC_QUEUES_M              GENMASK(23, 16)
+#define QSYS_TAG_CONFIG_SCH_TRAFFIC_QUEUES_X(x)           (((x) & GENMASK(23, 16)) >> 16)
+
+#define QSYS_TAS_PARAM_CFG_CTRL_PORT_NUM(x)               ((x) & GENMASK(7, 0))
+#define QSYS_TAS_PARAM_CFG_CTRL_PORT_NUM_M                GENMASK(7, 0)
+#define QSYS_TAS_PARAM_CFG_CTRL_ALWAYS_GUARD_BAND_SCH_Q   BIT(8)
+#define QSYS_TAS_PARAM_CFG_CTRL_CONFIG_CHANGE             BIT(16)
+
+#define QSYS_PORT_MAX_SDU_RSZ                             0x4
+
+#define QSYS_PARAM_CFG_REG_3_BASE_TIME_SEC_MSB(x)         ((x) & GENMASK(15, 0))
+#define QSYS_PARAM_CFG_REG_3_BASE_TIME_SEC_MSB_M          GENMASK(15, 0)
+#define QSYS_PARAM_CFG_REG_3_LIST_LENGTH(x)               (((x) << 16) & GENMASK(31, 16))
+#define QSYS_PARAM_CFG_REG_3_LIST_LENGTH_M                GENMASK(31, 16)
+#define QSYS_PARAM_CFG_REG_3_LIST_LENGTH_X(x)             (((x) & GENMASK(31, 16)) >> 16)
+
+#define QSYS_GCL_CFG_REG_1_GCL_ENTRY_NUM(x)               ((x) & GENMASK(5, 0))
+#define QSYS_GCL_CFG_REG_1_GCL_ENTRY_NUM_M                GENMASK(5, 0)
+#define QSYS_GCL_CFG_REG_1_GATE_STATE(x)                  (((x) << 8) & GENMASK(15, 8))
+#define QSYS_GCL_CFG_REG_1_GATE_STATE_M                   GENMASK(15, 8)
+#define QSYS_GCL_CFG_REG_1_GATE_STATE_X(x)                (((x) & GENMASK(15, 8)) >> 8)
+
+#define QSYS_PARAM_STATUS_REG_3_BASE_TIME_SEC_MSB(x)      ((x) & GENMASK(15, 0))
+#define QSYS_PARAM_STATUS_REG_3_BASE_TIME_SEC_MSB_M       GENMASK(15, 0)
+#define QSYS_PARAM_STATUS_REG_3_LIST_LENGTH(x)            (((x) << 16) & GENMASK(31, 16))
+#define QSYS_PARAM_STATUS_REG_3_LIST_LENGTH_M             GENMASK(31, 16)
+#define QSYS_PARAM_STATUS_REG_3_LIST_LENGTH_X(x)          (((x) & GENMASK(31, 16)) >> 16)
+
+#define QSYS_PARAM_STATUS_REG_8_CFG_CHG_TIME_SEC_MSB(x)   ((x) & GENMASK(15, 0))
+#define QSYS_PARAM_STATUS_REG_8_CFG_CHG_TIME_SEC_MSB_M    GENMASK(15, 0)
+#define QSYS_PARAM_STATUS_REG_8_OPER_GATE_STATE(x)        (((x) << 16) & GENMASK(23, 16))
+#define QSYS_PARAM_STATUS_REG_8_OPER_GATE_STATE_M         GENMASK(23, 16)
+#define QSYS_PARAM_STATUS_REG_8_OPER_GATE_STATE_X(x)      (((x) & GENMASK(23, 16)) >> 16)
+#define QSYS_PARAM_STATUS_REG_8_CONFIG_PENDING            BIT(24)
+
+#define QSYS_GCL_STATUS_REG_1_GCL_ENTRY_NUM(x)            ((x) & GENMASK(5, 0))
+#define QSYS_GCL_STATUS_REG_1_GCL_ENTRY_NUM_M             GENMASK(5, 0)
+#define QSYS_GCL_STATUS_REG_1_GATE_STATE(x)               (((x) << 8) & GENMASK(15, 8))
+#define QSYS_GCL_STATUS_REG_1_GATE_STATE_M                GENMASK(15, 8)
+#define QSYS_GCL_STATUS_REG_1_GATE_STATE_X(x)             (((x) & GENMASK(15, 8)) >> 8)
+
+#endif
diff --git a/drivers/net/ethernet/mscc/ocelot_regs.c b/drivers/net/ethernet/mscc/ocelot_regs.c
new file mode 100644
index 000000000000..e334b406c40c
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_regs.c
@@ -0,0 +1,497 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+#include "ocelot.h"
+
+static const u32 ocelot_ana_regmap[] = {
+	REG(ANA_ADVLEARN,                  0x009000),
+	REG(ANA_VLANMASK,                  0x009004),
+	REG(ANA_PORT_B_DOMAIN,             0x009008),
+	REG(ANA_ANAGEFIL,                  0x00900c),
+	REG(ANA_ANEVENTS,                  0x009010),
+	REG(ANA_STORMLIMIT_BURST,          0x009014),
+	REG(ANA_STORMLIMIT_CFG,            0x009018),
+	REG(ANA_ISOLATED_PORTS,            0x009028),
+	REG(ANA_COMMUNITY_PORTS,           0x00902c),
+	REG(ANA_AUTOAGE,                   0x009030),
+	REG(ANA_MACTOPTIONS,               0x009034),
+	REG(ANA_LEARNDISC,                 0x009038),
+	REG(ANA_AGENCTRL,                  0x00903c),
+	REG(ANA_MIRRORPORTS,               0x009040),
+	REG(ANA_EMIRRORPORTS,              0x009044),
+	REG(ANA_FLOODING,                  0x009048),
+	REG(ANA_FLOODING_IPMC,             0x00904c),
+	REG(ANA_SFLOW_CFG,                 0x009050),
+	REG(ANA_PORT_MODE,                 0x009080),
+	REG(ANA_PGID_PGID,                 0x008c00),
+	REG(ANA_TABLES_ANMOVED,            0x008b30),
+	REG(ANA_TABLES_MACHDATA,           0x008b34),
+	REG(ANA_TABLES_MACLDATA,           0x008b38),
+	REG(ANA_TABLES_MACACCESS,          0x008b3c),
+	REG(ANA_TABLES_MACTINDX,           0x008b40),
+	REG(ANA_TABLES_VLANACCESS,         0x008b44),
+	REG(ANA_TABLES_VLANTIDX,           0x008b48),
+	REG(ANA_TABLES_ISDXACCESS,         0x008b4c),
+	REG(ANA_TABLES_ISDXTIDX,           0x008b50),
+	REG(ANA_TABLES_ENTRYLIM,           0x008b00),
+	REG(ANA_TABLES_PTP_ID_HIGH,        0x008b54),
+	REG(ANA_TABLES_PTP_ID_LOW,         0x008b58),
+	REG(ANA_MSTI_STATE,                0x008e00),
+	REG(ANA_PORT_VLAN_CFG,             0x007000),
+	REG(ANA_PORT_DROP_CFG,             0x007004),
+	REG(ANA_PORT_QOS_CFG,              0x007008),
+	REG(ANA_PORT_VCAP_CFG,             0x00700c),
+	REG(ANA_PORT_VCAP_S1_KEY_CFG,      0x007010),
+	REG(ANA_PORT_VCAP_S2_CFG,          0x00701c),
+	REG(ANA_PORT_PCP_DEI_MAP,          0x007020),
+	REG(ANA_PORT_CPU_FWD_CFG,          0x007060),
+	REG(ANA_PORT_CPU_FWD_BPDU_CFG,     0x007064),
+	REG(ANA_PORT_CPU_FWD_GARP_CFG,     0x007068),
+	REG(ANA_PORT_CPU_FWD_CCM_CFG,      0x00706c),
+	REG(ANA_PORT_PORT_CFG,             0x007070),
+	REG(ANA_PORT_POL_CFG,              0x007074),
+	REG(ANA_PORT_PTP_CFG,              0x007078),
+	REG(ANA_PORT_PTP_DLY1_CFG,         0x00707c),
+	REG(ANA_OAM_UPM_LM_CNT,            0x007c00),
+	REG(ANA_PORT_PTP_DLY2_CFG,         0x007080),
+	REG(ANA_PFC_PFC_CFG,               0x008800),
+	REG(ANA_PFC_PFC_TIMER,             0x008804),
+	REG(ANA_IPT_OAM_MEP_CFG,           0x008000),
+	REG(ANA_IPT_IPT,                   0x008004),
+	REG(ANA_PPT_PPT,                   0x008ac0),
+	REG(ANA_FID_MAP_FID_MAP,           0x000000),
+	REG(ANA_AGGR_CFG,                  0x0090b4),
+	REG(ANA_CPUQ_CFG,                  0x0090b8),
+	REG(ANA_CPUQ_CFG2,                 0x0090bc),
+	REG(ANA_CPUQ_8021_CFG,             0x0090c0),
+	REG(ANA_DSCP_CFG,                  0x009100),
+	REG(ANA_DSCP_REWR_CFG,             0x009200),
+	REG(ANA_VCAP_RNG_TYPE_CFG,         0x009240),
+	REG(ANA_VCAP_RNG_VAL_CFG,          0x009260),
+	REG(ANA_VRAP_CFG,                  0x009280),
+	REG(ANA_VRAP_HDR_DATA,             0x009284),
+	REG(ANA_VRAP_HDR_MASK,             0x009288),
+	REG(ANA_DISCARD_CFG,               0x00928c),
+	REG(ANA_FID_CFG,                   0x009290),
+	REG(ANA_POL_PIR_CFG,               0x004000),
+	REG(ANA_POL_CIR_CFG,               0x004004),
+	REG(ANA_POL_MODE_CFG,              0x004008),
+	REG(ANA_POL_PIR_STATE,             0x00400c),
+	REG(ANA_POL_CIR_STATE,             0x004010),
+	REG(ANA_POL_STATE,                 0x004014),
+	REG(ANA_POL_FLOWC,                 0x008b80),
+	REG(ANA_POL_HYST,                  0x008bec),
+	REG(ANA_POL_MISC_CFG,              0x008bf0),
+};
+
+static const u32 ocelot_qs_regmap[] = {
+	REG(QS_XTR_GRP_CFG,                0x000000),
+	REG(QS_XTR_RD,                     0x000008),
+	REG(QS_XTR_FRM_PRUNING,            0x000010),
+	REG(QS_XTR_FLUSH,                  0x000018),
+	REG(QS_XTR_DATA_PRESENT,           0x00001c),
+	REG(QS_XTR_CFG,                    0x000020),
+	REG(QS_INJ_GRP_CFG,                0x000024),
+	REG(QS_INJ_WR,                     0x00002c),
+	REG(QS_INJ_CTRL,                   0x000034),
+	REG(QS_INJ_STATUS,                 0x00003c),
+	REG(QS_INJ_ERR,                    0x000040),
+	REG(QS_INH_DBG,                    0x000048),
+};
+
+static const u32 ocelot_hsio_regmap[] = {
+	REG(HSIO_PLL5G_CFG0,               0x000000),
+	REG(HSIO_PLL5G_CFG1,               0x000004),
+	REG(HSIO_PLL5G_CFG2,               0x000008),
+	REG(HSIO_PLL5G_CFG3,               0x00000c),
+	REG(HSIO_PLL5G_CFG4,               0x000010),
+	REG(HSIO_PLL5G_CFG5,               0x000014),
+	REG(HSIO_PLL5G_CFG6,               0x000018),
+	REG(HSIO_PLL5G_STATUS0,            0x00001c),
+	REG(HSIO_PLL5G_STATUS1,            0x000020),
+	REG(HSIO_PLL5G_BIST_CFG0,          0x000024),
+	REG(HSIO_PLL5G_BIST_CFG1,          0x000028),
+	REG(HSIO_PLL5G_BIST_CFG2,          0x00002c),
+	REG(HSIO_PLL5G_BIST_STAT0,         0x000030),
+	REG(HSIO_PLL5G_BIST_STAT1,         0x000034),
+	REG(HSIO_RCOMP_CFG0,               0x000038),
+	REG(HSIO_RCOMP_STATUS,             0x00003c),
+	REG(HSIO_SYNC_ETH_CFG,             0x000040),
+	REG(HSIO_SYNC_ETH_PLL_CFG,         0x000048),
+	REG(HSIO_S1G_DES_CFG,              0x00004c),
+	REG(HSIO_S1G_IB_CFG,               0x000050),
+	REG(HSIO_S1G_OB_CFG,               0x000054),
+	REG(HSIO_S1G_SER_CFG,              0x000058),
+	REG(HSIO_S1G_COMMON_CFG,           0x00005c),
+	REG(HSIO_S1G_PLL_CFG,              0x000060),
+	REG(HSIO_S1G_PLL_STATUS,           0x000064),
+	REG(HSIO_S1G_DFT_CFG0,             0x000068),
+	REG(HSIO_S1G_DFT_CFG1,             0x00006c),
+	REG(HSIO_S1G_DFT_CFG2,             0x000070),
+	REG(HSIO_S1G_TP_CFG,               0x000074),
+	REG(HSIO_S1G_RC_PLL_BIST_CFG,      0x000078),
+	REG(HSIO_S1G_MISC_CFG,             0x00007c),
+	REG(HSIO_S1G_DFT_STATUS,           0x000080),
+	REG(HSIO_S1G_MISC_STATUS,          0x000084),
+	REG(HSIO_MCB_S1G_ADDR_CFG,         0x000088),
+	REG(HSIO_S6G_DIG_CFG,              0x00008c),
+	REG(HSIO_S6G_DFT_CFG0,             0x000090),
+	REG(HSIO_S6G_DFT_CFG1,             0x000094),
+	REG(HSIO_S6G_DFT_CFG2,             0x000098),
+	REG(HSIO_S6G_TP_CFG0,              0x00009c),
+	REG(HSIO_S6G_TP_CFG1,              0x0000a0),
+	REG(HSIO_S6G_RC_PLL_BIST_CFG,      0x0000a4),
+	REG(HSIO_S6G_MISC_CFG,             0x0000a8),
+	REG(HSIO_S6G_OB_ANEG_CFG,          0x0000ac),
+	REG(HSIO_S6G_DFT_STATUS,           0x0000b0),
+	REG(HSIO_S6G_ERR_CNT,              0x0000b4),
+	REG(HSIO_S6G_MISC_STATUS,          0x0000b8),
+	REG(HSIO_S6G_DES_CFG,              0x0000bc),
+	REG(HSIO_S6G_IB_CFG,               0x0000c0),
+	REG(HSIO_S6G_IB_CFG1,              0x0000c4),
+	REG(HSIO_S6G_IB_CFG2,              0x0000c8),
+	REG(HSIO_S6G_IB_CFG3,              0x0000cc),
+	REG(HSIO_S6G_IB_CFG4,              0x0000d0),
+	REG(HSIO_S6G_IB_CFG5,              0x0000d4),
+	REG(HSIO_S6G_OB_CFG,               0x0000d8),
+	REG(HSIO_S6G_OB_CFG1,              0x0000dc),
+	REG(HSIO_S6G_SER_CFG,              0x0000e0),
+	REG(HSIO_S6G_COMMON_CFG,           0x0000e4),
+	REG(HSIO_S6G_PLL_CFG,              0x0000e8),
+	REG(HSIO_S6G_ACJTAG_CFG,           0x0000ec),
+	REG(HSIO_S6G_GP_CFG,               0x0000f0),
+	REG(HSIO_S6G_IB_STATUS0,           0x0000f4),
+	REG(HSIO_S6G_IB_STATUS1,           0x0000f8),
+	REG(HSIO_S6G_ACJTAG_STATUS,        0x0000fc),
+	REG(HSIO_S6G_PLL_STATUS,           0x000100),
+	REG(HSIO_S6G_REVID,                0x000104),
+	REG(HSIO_MCB_S6G_ADDR_CFG,         0x000108),
+	REG(HSIO_HW_CFG,                   0x00010c),
+	REG(HSIO_HW_QSGMII_CFG,            0x000110),
+	REG(HSIO_HW_QSGMII_STAT,           0x000114),
+	REG(HSIO_CLK_CFG,                  0x000118),
+	REG(HSIO_TEMP_SENSOR_CTRL,         0x00011c),
+	REG(HSIO_TEMP_SENSOR_CFG,          0x000120),
+	REG(HSIO_TEMP_SENSOR_STAT,         0x000124),
+};
+
+static const u32 ocelot_qsys_regmap[] = {
+	REG(QSYS_PORT_MODE,                0x011200),
+	REG(QSYS_SWITCH_PORT_MODE,         0x011234),
+	REG(QSYS_STAT_CNT_CFG,             0x011264),
+	REG(QSYS_EEE_CFG,                  0x011268),
+	REG(QSYS_EEE_THRES,                0x011294),
+	REG(QSYS_IGR_NO_SHARING,           0x011298),
+	REG(QSYS_EGR_NO_SHARING,           0x01129c),
+	REG(QSYS_SW_STATUS,                0x0112a0),
+	REG(QSYS_EXT_CPU_CFG,              0x0112d0),
+	REG(QSYS_PAD_CFG,                  0x0112d4),
+	REG(QSYS_CPU_GROUP_MAP,            0x0112d8),
+	REG(QSYS_QMAP,                     0x0112dc),
+	REG(QSYS_ISDX_SGRP,                0x011400),
+	REG(QSYS_TIMED_FRAME_ENTRY,        0x014000),
+	REG(QSYS_TFRM_MISC,                0x011310),
+	REG(QSYS_TFRM_PORT_DLY,            0x011314),
+	REG(QSYS_TFRM_TIMER_CFG_1,         0x011318),
+	REG(QSYS_TFRM_TIMER_CFG_2,         0x01131c),
+	REG(QSYS_TFRM_TIMER_CFG_3,         0x011320),
+	REG(QSYS_TFRM_TIMER_CFG_4,         0x011324),
+	REG(QSYS_TFRM_TIMER_CFG_5,         0x011328),
+	REG(QSYS_TFRM_TIMER_CFG_6,         0x01132c),
+	REG(QSYS_TFRM_TIMER_CFG_7,         0x011330),
+	REG(QSYS_TFRM_TIMER_CFG_8,         0x011334),
+	REG(QSYS_RED_PROFILE,              0x011338),
+	REG(QSYS_RES_QOS_MODE,             0x011378),
+	REG(QSYS_RES_CFG,                  0x012000),
+	REG(QSYS_RES_STAT,                 0x012004),
+	REG(QSYS_EGR_DROP_MODE,            0x01137c),
+	REG(QSYS_EQ_CTRL,                  0x011380),
+	REG(QSYS_EVENTS_CORE,              0x011384),
+	REG(QSYS_CIR_CFG,                  0x000000),
+	REG(QSYS_EIR_CFG,                  0x000004),
+	REG(QSYS_SE_CFG,                   0x000008),
+	REG(QSYS_SE_DWRR_CFG,              0x00000c),
+	REG(QSYS_SE_CONNECT,               0x00003c),
+	REG(QSYS_SE_DLB_SENSE,             0x000040),
+	REG(QSYS_CIR_STATE,                0x000044),
+	REG(QSYS_EIR_STATE,                0x000048),
+	REG(QSYS_SE_STATE,                 0x00004c),
+	REG(QSYS_HSCH_MISC_CFG,            0x011388),
+};
+
+static const u32 ocelot_rew_regmap[] = {
+	REG(REW_PORT_VLAN_CFG,             0x000000),
+	REG(REW_TAG_CFG,                   0x000004),
+	REG(REW_PORT_CFG,                  0x000008),
+	REG(REW_DSCP_CFG,                  0x00000c),
+	REG(REW_PCP_DEI_QOS_MAP_CFG,       0x000010),
+	REG(REW_PTP_CFG,                   0x000050),
+	REG(REW_PTP_DLY1_CFG,              0x000054),
+	REG(REW_DSCP_REMAP_DP1_CFG,        0x000690),
+	REG(REW_DSCP_REMAP_CFG,            0x000790),
+	REG(REW_STAT_CFG,                  0x000890),
+	REG(REW_PPT,                       0x000680),
+};
+
+static const u32 ocelot_sys_regmap[] = {
+	REG(SYS_COUNT_RX_OCTETS,           0x000000),
+	REG(SYS_COUNT_RX_UNICAST,          0x000004),
+	REG(SYS_COUNT_RX_MULTICAST,        0x000008),
+	REG(SYS_COUNT_RX_BROADCAST,        0x00000c),
+	REG(SYS_COUNT_RX_SHORTS,           0x000010),
+	REG(SYS_COUNT_RX_FRAGMENTS,        0x000014),
+	REG(SYS_COUNT_RX_JABBERS,          0x000018),
+	REG(SYS_COUNT_RX_CRC_ALIGN_ERRS,   0x00001c),
+	REG(SYS_COUNT_RX_SYM_ERRS,         0x000020),
+	REG(SYS_COUNT_RX_64,               0x000024),
+	REG(SYS_COUNT_RX_65_127,           0x000028),
+	REG(SYS_COUNT_RX_128_255,          0x00002c),
+	REG(SYS_COUNT_RX_256_1023,         0x000030),
+	REG(SYS_COUNT_RX_1024_1526,        0x000034),
+	REG(SYS_COUNT_RX_1527_MAX,         0x000038),
+	REG(SYS_COUNT_RX_PAUSE,            0x00003c),
+	REG(SYS_COUNT_RX_CONTROL,          0x000040),
+	REG(SYS_COUNT_RX_LONGS,            0x000044),
+	REG(SYS_COUNT_RX_CLASSIFIED_DROPS, 0x000048),
+	REG(SYS_COUNT_TX_OCTETS,           0x000100),
+	REG(SYS_COUNT_TX_UNICAST,          0x000104),
+	REG(SYS_COUNT_TX_MULTICAST,        0x000108),
+	REG(SYS_COUNT_TX_BROADCAST,        0x00010c),
+	REG(SYS_COUNT_TX_COLLISION,        0x000110),
+	REG(SYS_COUNT_TX_DROPS,            0x000114),
+	REG(SYS_COUNT_TX_PAUSE,            0x000118),
+	REG(SYS_COUNT_TX_64,               0x00011c),
+	REG(SYS_COUNT_TX_65_127,           0x000120),
+	REG(SYS_COUNT_TX_128_511,          0x000124),
+	REG(SYS_COUNT_TX_512_1023,         0x000128),
+	REG(SYS_COUNT_TX_1024_1526,        0x00012c),
+	REG(SYS_COUNT_TX_1527_MAX,         0x000130),
+	REG(SYS_COUNT_TX_AGING,            0x000170),
+	REG(SYS_RESET_CFG,                 0x000508),
+	REG(SYS_CMID,                      0x00050c),
+	REG(SYS_VLAN_ETYPE_CFG,            0x000510),
+	REG(SYS_PORT_MODE,                 0x000514),
+	REG(SYS_FRONT_PORT_MODE,           0x000548),
+	REG(SYS_FRM_AGING,                 0x000574),
+	REG(SYS_STAT_CFG,                  0x000578),
+	REG(SYS_SW_STATUS,                 0x00057c),
+	REG(SYS_MISC_CFG,                  0x0005ac),
+	REG(SYS_REW_MAC_HIGH_CFG,          0x0005b0),
+	REG(SYS_REW_MAC_LOW_CFG,           0x0005dc),
+	REG(SYS_CM_ADDR,                   0x000500),
+	REG(SYS_CM_DATA,                   0x000504),
+	REG(SYS_PAUSE_CFG,                 0x000608),
+	REG(SYS_PAUSE_TOT_CFG,             0x000638),
+	REG(SYS_ATOP,                      0x00063c),
+	REG(SYS_ATOP_TOT_CFG,              0x00066c),
+	REG(SYS_MAC_FC_CFG,                0x000670),
+	REG(SYS_MMGT,                      0x00069c),
+	REG(SYS_MMGT_FAST,                 0x0006a0),
+	REG(SYS_EVENTS_DIF,                0x0006a4),
+	REG(SYS_EVENTS_CORE,               0x0006b4),
+	REG(SYS_CNT,                       0x000000),
+	REG(SYS_PTP_STATUS,                0x0006b8),
+	REG(SYS_PTP_TXSTAMP,               0x0006bc),
+	REG(SYS_PTP_NXT,                   0x0006c0),
+	REG(SYS_PTP_CFG,                   0x0006c4),
+};
+
+static const u32 *ocelot_regmap[] = {
+	[ANA] = ocelot_ana_regmap,
+	[QS] = ocelot_qs_regmap,
+	[HSIO] = ocelot_hsio_regmap,
+	[QSYS] = ocelot_qsys_regmap,
+	[REW] = ocelot_rew_regmap,
+	[SYS] = ocelot_sys_regmap,
+};
+
+static const struct reg_field ocelot_regfields[] = {
+	[ANA_ADVLEARN_VLAN_CHK] = REG_FIELD(ANA_ADVLEARN, 11, 11),
+	[ANA_ADVLEARN_LEARN_MIRROR] = REG_FIELD(ANA_ADVLEARN, 0, 10),
+	[ANA_ANEVENTS_MSTI_DROP] = REG_FIELD(ANA_ANEVENTS, 27, 27),
+	[ANA_ANEVENTS_ACLKILL] = REG_FIELD(ANA_ANEVENTS, 26, 26),
+	[ANA_ANEVENTS_ACLUSED] = REG_FIELD(ANA_ANEVENTS, 25, 25),
+	[ANA_ANEVENTS_AUTOAGE] = REG_FIELD(ANA_ANEVENTS, 24, 24),
+	[ANA_ANEVENTS_VS2TTL1] = REG_FIELD(ANA_ANEVENTS, 23, 23),
+	[ANA_ANEVENTS_STORM_DROP] = REG_FIELD(ANA_ANEVENTS, 22, 22),
+	[ANA_ANEVENTS_LEARN_DROP] = REG_FIELD(ANA_ANEVENTS, 21, 21),
+	[ANA_ANEVENTS_AGED_ENTRY] = REG_FIELD(ANA_ANEVENTS, 20, 20),
+	[ANA_ANEVENTS_CPU_LEARN_FAILED] = REG_FIELD(ANA_ANEVENTS, 19, 19),
+	[ANA_ANEVENTS_AUTO_LEARN_FAILED] = REG_FIELD(ANA_ANEVENTS, 18, 18),
+	[ANA_ANEVENTS_LEARN_REMOVE] = REG_FIELD(ANA_ANEVENTS, 17, 17),
+	[ANA_ANEVENTS_AUTO_LEARNED] = REG_FIELD(ANA_ANEVENTS, 16, 16),
+	[ANA_ANEVENTS_AUTO_MOVED] = REG_FIELD(ANA_ANEVENTS, 15, 15),
+	[ANA_ANEVENTS_DROPPED] = REG_FIELD(ANA_ANEVENTS, 14, 14),
+	[ANA_ANEVENTS_CLASSIFIED_DROP] = REG_FIELD(ANA_ANEVENTS, 13, 13),
+	[ANA_ANEVENTS_CLASSIFIED_COPY] = REG_FIELD(ANA_ANEVENTS, 12, 12),
+	[ANA_ANEVENTS_VLAN_DISCARD] = REG_FIELD(ANA_ANEVENTS, 11, 11),
+	[ANA_ANEVENTS_FWD_DISCARD] = REG_FIELD(ANA_ANEVENTS, 10, 10),
+	[ANA_ANEVENTS_MULTICAST_FLOOD] = REG_FIELD(ANA_ANEVENTS, 9, 9),
+	[ANA_ANEVENTS_UNICAST_FLOOD] = REG_FIELD(ANA_ANEVENTS, 8, 8),
+	[ANA_ANEVENTS_DEST_KNOWN] = REG_FIELD(ANA_ANEVENTS, 7, 7),
+	[ANA_ANEVENTS_BUCKET3_MATCH] = REG_FIELD(ANA_ANEVENTS, 6, 6),
+	[ANA_ANEVENTS_BUCKET2_MATCH] = REG_FIELD(ANA_ANEVENTS, 5, 5),
+	[ANA_ANEVENTS_BUCKET1_MATCH] = REG_FIELD(ANA_ANEVENTS, 4, 4),
+	[ANA_ANEVENTS_BUCKET0_MATCH] = REG_FIELD(ANA_ANEVENTS, 3, 3),
+	[ANA_ANEVENTS_CPU_OPERATION] = REG_FIELD(ANA_ANEVENTS, 2, 2),
+	[ANA_ANEVENTS_DMAC_LOOKUP] = REG_FIELD(ANA_ANEVENTS, 1, 1),
+	[ANA_ANEVENTS_SMAC_LOOKUP] = REG_FIELD(ANA_ANEVENTS, 0, 0),
+	[ANA_TABLES_MACACCESS_B_DOM] = REG_FIELD(ANA_TABLES_MACACCESS, 18, 18),
+	[ANA_TABLES_MACTINDX_BUCKET] = REG_FIELD(ANA_TABLES_MACTINDX, 10, 11),
+	[ANA_TABLES_MACTINDX_M_INDEX] = REG_FIELD(ANA_TABLES_MACTINDX, 0, 9),
+	[QSYS_TIMED_FRAME_ENTRY_TFRM_VLD] = REG_FIELD(QSYS_TIMED_FRAME_ENTRY, 20, 20),
+	[QSYS_TIMED_FRAME_ENTRY_TFRM_FP] = REG_FIELD(QSYS_TIMED_FRAME_ENTRY, 8, 19),
+	[QSYS_TIMED_FRAME_ENTRY_TFRM_PORTNO] = REG_FIELD(QSYS_TIMED_FRAME_ENTRY, 4, 7),
+	[QSYS_TIMED_FRAME_ENTRY_TFRM_TM_SEL] = REG_FIELD(QSYS_TIMED_FRAME_ENTRY, 1, 3),
+	[QSYS_TIMED_FRAME_ENTRY_TFRM_TM_T] = REG_FIELD(QSYS_TIMED_FRAME_ENTRY, 0, 0),
+	[SYS_RESET_CFG_CORE_ENA] = REG_FIELD(SYS_RESET_CFG, 2, 2),
+	[SYS_RESET_CFG_MEM_ENA] = REG_FIELD(SYS_RESET_CFG, 1, 1),
+	[SYS_RESET_CFG_MEM_INIT] = REG_FIELD(SYS_RESET_CFG, 0, 0),
+};
+
+static const struct ocelot_stat_layout ocelot_stats_layout[] = {
+	{ .name = "rx_octets", .offset = 0x00, },
+	{ .name = "rx_unicast", .offset = 0x01, },
+	{ .name = "rx_multicast", .offset = 0x02, },
+	{ .name = "rx_broadcast", .offset = 0x03, },
+	{ .name = "rx_shorts", .offset = 0x04, },
+	{ .name = "rx_fragments", .offset = 0x05, },
+	{ .name = "rx_jabbers", .offset = 0x06, },
+	{ .name = "rx_crc_align_errs", .offset = 0x07, },
+	{ .name = "rx_sym_errs", .offset = 0x08, },
+	{ .name = "rx_frames_below_65_octets", .offset = 0x09, },
+	{ .name = "rx_frames_65_to_127_octets", .offset = 0x0A, },
+	{ .name = "rx_frames_128_to_255_octets", .offset = 0x0B, },
+	{ .name = "rx_frames_256_to_511_octets", .offset = 0x0C, },
+	{ .name = "rx_frames_512_to_1023_octets", .offset = 0x0D, },
+	{ .name = "rx_frames_1024_to_1526_octets", .offset = 0x0E, },
+	{ .name = "rx_frames_over_1526_octets", .offset = 0x0F, },
+	{ .name = "rx_pause", .offset = 0x10, },
+	{ .name = "rx_control", .offset = 0x11, },
+	{ .name = "rx_longs", .offset = 0x12, },
+	{ .name = "rx_classified_drops", .offset = 0x13, },
+	{ .name = "rx_red_prio_0", .offset = 0x14, },
+	{ .name = "rx_red_prio_1", .offset = 0x15, },
+	{ .name = "rx_red_prio_2", .offset = 0x16, },
+	{ .name = "rx_red_prio_3", .offset = 0x17, },
+	{ .name = "rx_red_prio_4", .offset = 0x18, },
+	{ .name = "rx_red_prio_5", .offset = 0x19, },
+	{ .name = "rx_red_prio_6", .offset = 0x1A, },
+	{ .name = "rx_red_prio_7", .offset = 0x1B, },
+	{ .name = "rx_yellow_prio_0", .offset = 0x1C, },
+	{ .name = "rx_yellow_prio_1", .offset = 0x1D, },
+	{ .name = "rx_yellow_prio_2", .offset = 0x1E, },
+	{ .name = "rx_yellow_prio_3", .offset = 0x1F, },
+	{ .name = "rx_yellow_prio_4", .offset = 0x20, },
+	{ .name = "rx_yellow_prio_5", .offset = 0x21, },
+	{ .name = "rx_yellow_prio_6", .offset = 0x22, },
+	{ .name = "rx_yellow_prio_7", .offset = 0x23, },
+	{ .name = "rx_green_prio_0", .offset = 0x24, },
+	{ .name = "rx_green_prio_1", .offset = 0x25, },
+	{ .name = "rx_green_prio_2", .offset = 0x26, },
+	{ .name = "rx_green_prio_3", .offset = 0x27, },
+	{ .name = "rx_green_prio_4", .offset = 0x28, },
+	{ .name = "rx_green_prio_5", .offset = 0x29, },
+	{ .name = "rx_green_prio_6", .offset = 0x2A, },
+	{ .name = "rx_green_prio_7", .offset = 0x2B, },
+	{ .name = "tx_octets", .offset = 0x40, },
+	{ .name = "tx_unicast", .offset = 0x41, },
+	{ .name = "tx_multicast", .offset = 0x42, },
+	{ .name = "tx_broadcast", .offset = 0x43, },
+	{ .name = "tx_collision", .offset = 0x44, },
+	{ .name = "tx_drops", .offset = 0x45, },
+	{ .name = "tx_pause", .offset = 0x46, },
+	{ .name = "tx_frames_below_65_octets", .offset = 0x47, },
+	{ .name = "tx_frames_65_to_127_octets", .offset = 0x48, },
+	{ .name = "tx_frames_128_255_octets", .offset = 0x49, },
+	{ .name = "tx_frames_256_511_octets", .offset = 0x4A, },
+	{ .name = "tx_frames_512_1023_octets", .offset = 0x4B, },
+	{ .name = "tx_frames_1024_1526_octets", .offset = 0x4C, },
+	{ .name = "tx_frames_over_1526_octets", .offset = 0x4D, },
+	{ .name = "tx_yellow_prio_0", .offset = 0x4E, },
+	{ .name = "tx_yellow_prio_1", .offset = 0x4F, },
+	{ .name = "tx_yellow_prio_2", .offset = 0x50, },
+	{ .name = "tx_yellow_prio_3", .offset = 0x51, },
+	{ .name = "tx_yellow_prio_4", .offset = 0x52, },
+	{ .name = "tx_yellow_prio_5", .offset = 0x53, },
+	{ .name = "tx_yellow_prio_6", .offset = 0x54, },
+	{ .name = "tx_yellow_prio_7", .offset = 0x55, },
+	{ .name = "tx_green_prio_0", .offset = 0x56, },
+	{ .name = "tx_green_prio_1", .offset = 0x57, },
+	{ .name = "tx_green_prio_2", .offset = 0x58, },
+	{ .name = "tx_green_prio_3", .offset = 0x59, },
+	{ .name = "tx_green_prio_4", .offset = 0x5A, },
+	{ .name = "tx_green_prio_5", .offset = 0x5B, },
+	{ .name = "tx_green_prio_6", .offset = 0x5C, },
+	{ .name = "tx_green_prio_7", .offset = 0x5D, },
+	{ .name = "tx_aged", .offset = 0x5E, },
+	{ .name = "drop_local", .offset = 0x80, },
+	{ .name = "drop_tail", .offset = 0x81, },
+	{ .name = "drop_yellow_prio_0", .offset = 0x82, },
+	{ .name = "drop_yellow_prio_1", .offset = 0x83, },
+	{ .name = "drop_yellow_prio_2", .offset = 0x84, },
+	{ .name = "drop_yellow_prio_3", .offset = 0x85, },
+	{ .name = "drop_yellow_prio_4", .offset = 0x86, },
+	{ .name = "drop_yellow_prio_5", .offset = 0x87, },
+	{ .name = "drop_yellow_prio_6", .offset = 0x88, },
+	{ .name = "drop_yellow_prio_7", .offset = 0x89, },
+	{ .name = "drop_green_prio_0", .offset = 0x8A, },
+	{ .name = "drop_green_prio_1", .offset = 0x8B, },
+	{ .name = "drop_green_prio_2", .offset = 0x8C, },
+	{ .name = "drop_green_prio_3", .offset = 0x8D, },
+	{ .name = "drop_green_prio_4", .offset = 0x8E, },
+	{ .name = "drop_green_prio_5", .offset = 0x8F, },
+	{ .name = "drop_green_prio_6", .offset = 0x90, },
+	{ .name = "drop_green_prio_7", .offset = 0x91, },
+};
+
+static void ocelot_pll5_init(struct ocelot *ocelot)
+{
+	/* Configure PLL5. This will need a proper CCF driver
+	 * The values are coming from the VTSS API for Ocelot
+	 */
+	ocelot_write(ocelot, HSIO_PLL5G_CFG4_IB_CTRL(0x7600) |
+		     HSIO_PLL5G_CFG4_IB_BIAS_CTRL(0x8), HSIO_PLL5G_CFG4);
+	ocelot_write(ocelot, HSIO_PLL5G_CFG0_CORE_CLK_DIV(0x11) |
+		     HSIO_PLL5G_CFG0_CPU_CLK_DIV(2) |
+		     HSIO_PLL5G_CFG0_ENA_BIAS |
+		     HSIO_PLL5G_CFG0_ENA_VCO_BUF |
+		     HSIO_PLL5G_CFG0_ENA_CP1 |
+		     HSIO_PLL5G_CFG0_SELCPI(2) |
+		     HSIO_PLL5G_CFG0_LOOP_BW_RES(0xe) |
+		     HSIO_PLL5G_CFG0_SELBGV820(4) |
+		     HSIO_PLL5G_CFG0_DIV4 |
+		     HSIO_PLL5G_CFG0_ENA_CLKTREE |
+		     HSIO_PLL5G_CFG0_ENA_LANE, HSIO_PLL5G_CFG0);
+	ocelot_write(ocelot, HSIO_PLL5G_CFG2_EN_RESET_FRQ_DET |
+		     HSIO_PLL5G_CFG2_EN_RESET_OVERRUN |
+		     HSIO_PLL5G_CFG2_GAIN_TEST(0x8) |
+		     HSIO_PLL5G_CFG2_ENA_AMPCTRL |
+		     HSIO_PLL5G_CFG2_PWD_AMPCTRL_N |
+		     HSIO_PLL5G_CFG2_AMPC_SEL(0x10), HSIO_PLL5G_CFG2);
+}
+
+int ocelot_chip_init(struct ocelot *ocelot)
+{
+	int ret;
+
+	ocelot->map = ocelot_regmap;
+	ocelot->stats_layout = ocelot_stats_layout;
+	ocelot->num_stats = ARRAY_SIZE(ocelot_stats_layout);
+	ocelot->shared_queue_sz = 224 * 1024;
+
+	ret = ocelot_regfields_init(ocelot, ocelot_regfields);
+	if (ret)
+		return ret;
+
+	ocelot_pll5_init(ocelot);
+
+	eth_random_addr(ocelot->base_mac);
+	ocelot->base_mac[5] &= 0xf0;
+
+	return 0;
+}
+EXPORT_SYMBOL(ocelot_chip_init);
diff --git a/drivers/net/ethernet/mscc/ocelot_rew.h b/drivers/net/ethernet/mscc/ocelot_rew.h
new file mode 100644
index 000000000000..210914b7e20f
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_rew.h
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+
+#ifndef _MSCC_OCELOT_REW_H_
+#define _MSCC_OCELOT_REW_H_
+
+#define REW_PORT_VLAN_CFG_GSZ                             0x80
+
+#define REW_PORT_VLAN_CFG_PORT_TPID(x)                    (((x) << 16) & GENMASK(31, 16))
+#define REW_PORT_VLAN_CFG_PORT_TPID_M                     GENMASK(31, 16)
+#define REW_PORT_VLAN_CFG_PORT_TPID_X(x)                  (((x) & GENMASK(31, 16)) >> 16)
+#define REW_PORT_VLAN_CFG_PORT_DEI                        BIT(15)
+#define REW_PORT_VLAN_CFG_PORT_PCP(x)                     (((x) << 12) & GENMASK(14, 12))
+#define REW_PORT_VLAN_CFG_PORT_PCP_M                      GENMASK(14, 12)
+#define REW_PORT_VLAN_CFG_PORT_PCP_X(x)                   (((x) & GENMASK(14, 12)) >> 12)
+#define REW_PORT_VLAN_CFG_PORT_VID(x)                     ((x) & GENMASK(11, 0))
+#define REW_PORT_VLAN_CFG_PORT_VID_M                      GENMASK(11, 0)
+
+#define REW_TAG_CFG_GSZ                                   0x80
+
+#define REW_TAG_CFG_TAG_CFG(x)                            (((x) << 7) & GENMASK(8, 7))
+#define REW_TAG_CFG_TAG_CFG_M                             GENMASK(8, 7)
+#define REW_TAG_CFG_TAG_CFG_X(x)                          (((x) & GENMASK(8, 7)) >> 7)
+#define REW_TAG_CFG_TAG_TPID_CFG(x)                       (((x) << 5) & GENMASK(6, 5))
+#define REW_TAG_CFG_TAG_TPID_CFG_M                        GENMASK(6, 5)
+#define REW_TAG_CFG_TAG_TPID_CFG_X(x)                     (((x) & GENMASK(6, 5)) >> 5)
+#define REW_TAG_CFG_TAG_VID_CFG                           BIT(4)
+#define REW_TAG_CFG_TAG_PCP_CFG(x)                        (((x) << 2) & GENMASK(3, 2))
+#define REW_TAG_CFG_TAG_PCP_CFG_M                         GENMASK(3, 2)
+#define REW_TAG_CFG_TAG_PCP_CFG_X(x)                      (((x) & GENMASK(3, 2)) >> 2)
+#define REW_TAG_CFG_TAG_DEI_CFG(x)                        ((x) & GENMASK(1, 0))
+#define REW_TAG_CFG_TAG_DEI_CFG_M                         GENMASK(1, 0)
+
+#define REW_PORT_CFG_GSZ                                  0x80
+
+#define REW_PORT_CFG_ES0_EN                               BIT(5)
+#define REW_PORT_CFG_FCS_UPDATE_NONCPU_CFG(x)             (((x) << 3) & GENMASK(4, 3))
+#define REW_PORT_CFG_FCS_UPDATE_NONCPU_CFG_M              GENMASK(4, 3)
+#define REW_PORT_CFG_FCS_UPDATE_NONCPU_CFG_X(x)           (((x) & GENMASK(4, 3)) >> 3)
+#define REW_PORT_CFG_FCS_UPDATE_CPU_ENA                   BIT(2)
+#define REW_PORT_CFG_FLUSH_ENA                            BIT(1)
+#define REW_PORT_CFG_AGE_DIS                              BIT(0)
+
+#define REW_DSCP_CFG_GSZ                                  0x80
+
+#define REW_PCP_DEI_QOS_MAP_CFG_GSZ                       0x80
+#define REW_PCP_DEI_QOS_MAP_CFG_RSZ                       0x4
+
+#define REW_PCP_DEI_QOS_MAP_CFG_DEI_QOS_VAL               BIT(3)
+#define REW_PCP_DEI_QOS_MAP_CFG_PCP_QOS_VAL(x)            ((x) & GENMASK(2, 0))
+#define REW_PCP_DEI_QOS_MAP_CFG_PCP_QOS_VAL_M             GENMASK(2, 0)
+
+#define REW_PTP_CFG_GSZ                                   0x80
+
+#define REW_PTP_CFG_PTP_BACKPLANE_MODE                    BIT(7)
+#define REW_PTP_CFG_GP_CFG_UNUSED(x)                      (((x) << 3) & GENMASK(6, 3))
+#define REW_PTP_CFG_GP_CFG_UNUSED_M                       GENMASK(6, 3)
+#define REW_PTP_CFG_GP_CFG_UNUSED_X(x)                    (((x) & GENMASK(6, 3)) >> 3)
+#define REW_PTP_CFG_PTP_1STEP_DIS                         BIT(2)
+#define REW_PTP_CFG_PTP_2STEP_DIS                         BIT(1)
+#define REW_PTP_CFG_PTP_UDP_KEEP                          BIT(0)
+
+#define REW_PTP_DLY1_CFG_GSZ                              0x80
+
+#define REW_RED_TAG_CFG_GSZ                               0x80
+
+#define REW_RED_TAG_CFG_RED_TAG_CFG                       BIT(0)
+
+#define REW_DSCP_REMAP_DP1_CFG_RSZ                        0x4
+
+#define REW_DSCP_REMAP_CFG_RSZ                            0x4
+
+#define REW_REW_STICKY_ES0_TAGB_PUSH_FAILED               BIT(0)
+
+#define REW_PPT_RSZ                                       0x4
+
+#endif
diff --git a/drivers/net/ethernet/mscc/ocelot_sys.h b/drivers/net/ethernet/mscc/ocelot_sys.h
new file mode 100644
index 000000000000..16f91e172bcb
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_sys.h
@@ -0,0 +1,144 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Microsemi Ocelot Switch driver
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+
+#ifndef _MSCC_OCELOT_SYS_H_
+#define _MSCC_OCELOT_SYS_H_
+
+#define SYS_COUNT_RX_OCTETS_RSZ                           0x4
+
+#define SYS_COUNT_TX_OCTETS_RSZ                           0x4
+
+#define SYS_PORT_MODE_RSZ                                 0x4
+
+#define SYS_PORT_MODE_DATA_WO_TS(x)                       (((x) << 5) & GENMASK(6, 5))
+#define SYS_PORT_MODE_DATA_WO_TS_M                        GENMASK(6, 5)
+#define SYS_PORT_MODE_DATA_WO_TS_X(x)                     (((x) & GENMASK(6, 5)) >> 5)
+#define SYS_PORT_MODE_INCL_INJ_HDR(x)                     (((x) << 3) & GENMASK(4, 3))
+#define SYS_PORT_MODE_INCL_INJ_HDR_M                      GENMASK(4, 3)
+#define SYS_PORT_MODE_INCL_INJ_HDR_X(x)                   (((x) & GENMASK(4, 3)) >> 3)
+#define SYS_PORT_MODE_INCL_XTR_HDR(x)                     (((x) << 1) & GENMASK(2, 1))
+#define SYS_PORT_MODE_INCL_XTR_HDR_M                      GENMASK(2, 1)
+#define SYS_PORT_MODE_INCL_XTR_HDR_X(x)                   (((x) & GENMASK(2, 1)) >> 1)
+#define SYS_PORT_MODE_INJ_HDR_ERR                         BIT(0)
+
+#define SYS_FRONT_PORT_MODE_RSZ                           0x4
+
+#define SYS_FRONT_PORT_MODE_HDX_MODE                      BIT(0)
+
+#define SYS_FRM_AGING_AGE_TX_ENA                          BIT(20)
+#define SYS_FRM_AGING_MAX_AGE(x)                          ((x) & GENMASK(19, 0))
+#define SYS_FRM_AGING_MAX_AGE_M                           GENMASK(19, 0)
+
+#define SYS_STAT_CFG_STAT_CLEAR_SHOT(x)                   (((x) << 10) & GENMASK(16, 10))
+#define SYS_STAT_CFG_STAT_CLEAR_SHOT_M                    GENMASK(16, 10)
+#define SYS_STAT_CFG_STAT_CLEAR_SHOT_X(x)                 (((x) & GENMASK(16, 10)) >> 10)
+#define SYS_STAT_CFG_STAT_VIEW(x)                         ((x) & GENMASK(9, 0))
+#define SYS_STAT_CFG_STAT_VIEW_M                          GENMASK(9, 0)
+
+#define SYS_SW_STATUS_RSZ                                 0x4
+
+#define SYS_SW_STATUS_PORT_RX_PAUSED                      BIT(0)
+
+#define SYS_MISC_CFG_PTP_RSRV_CLR                         BIT(1)
+#define SYS_MISC_CFG_PTP_DIS_NEG_RO                       BIT(0)
+
+#define SYS_REW_MAC_HIGH_CFG_RSZ                          0x4
+
+#define SYS_REW_MAC_LOW_CFG_RSZ                           0x4
+
+#define SYS_TIMESTAMP_OFFSET_ETH_TYPE_CFG(x)              (((x) << 6) & GENMASK(21, 6))
+#define SYS_TIMESTAMP_OFFSET_ETH_TYPE_CFG_M               GENMASK(21, 6)
+#define SYS_TIMESTAMP_OFFSET_ETH_TYPE_CFG_X(x)            (((x) & GENMASK(21, 6)) >> 6)
+#define SYS_TIMESTAMP_OFFSET_TIMESTAMP_OFFSET(x)          ((x) & GENMASK(5, 0))
+#define SYS_TIMESTAMP_OFFSET_TIMESTAMP_OFFSET_M           GENMASK(5, 0)
+
+#define SYS_PAUSE_CFG_RSZ                                 0x4
+
+#define SYS_PAUSE_CFG_PAUSE_START(x)                      (((x) << 10) & GENMASK(18, 10))
+#define SYS_PAUSE_CFG_PAUSE_START_M                       GENMASK(18, 10)
+#define SYS_PAUSE_CFG_PAUSE_START_X(x)                    (((x) & GENMASK(18, 10)) >> 10)
+#define SYS_PAUSE_CFG_PAUSE_STOP(x)                       (((x) << 1) & GENMASK(9, 1))
+#define SYS_PAUSE_CFG_PAUSE_STOP_M                        GENMASK(9, 1)
+#define SYS_PAUSE_CFG_PAUSE_STOP_X(x)                     (((x) & GENMASK(9, 1)) >> 1)
+#define SYS_PAUSE_CFG_PAUSE_ENA                           BIT(0)
+
+#define SYS_PAUSE_TOT_CFG_PAUSE_TOT_START(x)              (((x) << 9) & GENMASK(17, 9))
+#define SYS_PAUSE_TOT_CFG_PAUSE_TOT_START_M               GENMASK(17, 9)
+#define SYS_PAUSE_TOT_CFG_PAUSE_TOT_START_X(x)            (((x) & GENMASK(17, 9)) >> 9)
+#define SYS_PAUSE_TOT_CFG_PAUSE_TOT_STOP(x)               ((x) & GENMASK(8, 0))
+#define SYS_PAUSE_TOT_CFG_PAUSE_TOT_STOP_M                GENMASK(8, 0)
+
+#define SYS_ATOP_RSZ                                      0x4
+
+#define SYS_MAC_FC_CFG_RSZ                                0x4
+
+#define SYS_MAC_FC_CFG_FC_LINK_SPEED(x)                   (((x) << 26) & GENMASK(27, 26))
+#define SYS_MAC_FC_CFG_FC_LINK_SPEED_M                    GENMASK(27, 26)
+#define SYS_MAC_FC_CFG_FC_LINK_SPEED_X(x)                 (((x) & GENMASK(27, 26)) >> 26)
+#define SYS_MAC_FC_CFG_FC_LATENCY_CFG(x)                  (((x) << 20) & GENMASK(25, 20))
+#define SYS_MAC_FC_CFG_FC_LATENCY_CFG_M                   GENMASK(25, 20)
+#define SYS_MAC_FC_CFG_FC_LATENCY_CFG_X(x)                (((x) & GENMASK(25, 20)) >> 20)
+#define SYS_MAC_FC_CFG_ZERO_PAUSE_ENA                     BIT(18)
+#define SYS_MAC_FC_CFG_TX_FC_ENA                          BIT(17)
+#define SYS_MAC_FC_CFG_RX_FC_ENA                          BIT(16)
+#define SYS_MAC_FC_CFG_PAUSE_VAL_CFG(x)                   ((x) & GENMASK(15, 0))
+#define SYS_MAC_FC_CFG_PAUSE_VAL_CFG_M                    GENMASK(15, 0)
+
+#define SYS_MMGT_RELCNT(x)                                (((x) << 16) & GENMASK(31, 16))
+#define SYS_MMGT_RELCNT_M                                 GENMASK(31, 16)
+#define SYS_MMGT_RELCNT_X(x)                              (((x) & GENMASK(31, 16)) >> 16)
+#define SYS_MMGT_FREECNT(x)                               ((x) & GENMASK(15, 0))
+#define SYS_MMGT_FREECNT_M                                GENMASK(15, 0)
+
+#define SYS_MMGT_FAST_FREEVLD(x)                          (((x) << 4) & GENMASK(7, 4))
+#define SYS_MMGT_FAST_FREEVLD_M                           GENMASK(7, 4)
+#define SYS_MMGT_FAST_FREEVLD_X(x)                        (((x) & GENMASK(7, 4)) >> 4)
+#define SYS_MMGT_FAST_RELVLD(x)                           ((x) & GENMASK(3, 0))
+#define SYS_MMGT_FAST_RELVLD_M                            GENMASK(3, 0)
+
+#define SYS_EVENTS_DIF_RSZ                                0x4
+
+#define SYS_EVENTS_DIF_EV_DRX(x)                          (((x) << 6) & GENMASK(8, 6))
+#define SYS_EVENTS_DIF_EV_DRX_M                           GENMASK(8, 6)
+#define SYS_EVENTS_DIF_EV_DRX_X(x)                        (((x) & GENMASK(8, 6)) >> 6)
+#define SYS_EVENTS_DIF_EV_DTX(x)                          ((x) & GENMASK(5, 0))
+#define SYS_EVENTS_DIF_EV_DTX_M                           GENMASK(5, 0)
+
+#define SYS_EVENTS_CORE_EV_FWR                            BIT(2)
+#define SYS_EVENTS_CORE_EV_ANA(x)                         ((x) & GENMASK(1, 0))
+#define SYS_EVENTS_CORE_EV_ANA_M                          GENMASK(1, 0)
+
+#define SYS_CNT_GSZ                                       0x4
+
+#define SYS_PTP_STATUS_PTP_TXSTAMP_OAM                    BIT(29)
+#define SYS_PTP_STATUS_PTP_OVFL                           BIT(28)
+#define SYS_PTP_STATUS_PTP_MESS_VLD                       BIT(27)
+#define SYS_PTP_STATUS_PTP_MESS_ID(x)                     (((x) << 21) & GENMASK(26, 21))
+#define SYS_PTP_STATUS_PTP_MESS_ID_M                      GENMASK(26, 21)
+#define SYS_PTP_STATUS_PTP_MESS_ID_X(x)                   (((x) & GENMASK(26, 21)) >> 21)
+#define SYS_PTP_STATUS_PTP_MESS_TXPORT(x)                 (((x) << 16) & GENMASK(20, 16))
+#define SYS_PTP_STATUS_PTP_MESS_TXPORT_M                  GENMASK(20, 16)
+#define SYS_PTP_STATUS_PTP_MESS_TXPORT_X(x)               (((x) & GENMASK(20, 16)) >> 16)
+#define SYS_PTP_STATUS_PTP_MESS_SEQ_ID(x)                 ((x) & GENMASK(15, 0))
+#define SYS_PTP_STATUS_PTP_MESS_SEQ_ID_M                  GENMASK(15, 0)
+
+#define SYS_PTP_TXSTAMP_PTP_TXSTAMP(x)                    ((x) & GENMASK(29, 0))
+#define SYS_PTP_TXSTAMP_PTP_TXSTAMP_M                     GENMASK(29, 0)
+#define SYS_PTP_TXSTAMP_PTP_TXSTAMP_SEC                   BIT(31)
+
+#define SYS_PTP_NXT_PTP_NXT                               BIT(0)
+
+#define SYS_PTP_CFG_PTP_STAMP_WID(x)                      (((x) << 2) & GENMASK(7, 2))
+#define SYS_PTP_CFG_PTP_STAMP_WID_M                       GENMASK(7, 2)
+#define SYS_PTP_CFG_PTP_STAMP_WID_X(x)                    (((x) & GENMASK(7, 2)) >> 2)
+#define SYS_PTP_CFG_PTP_CF_ROLL_MODE(x)                   ((x) & GENMASK(1, 0))
+#define SYS_PTP_CFG_PTP_CF_ROLL_MODE_M                    GENMASK(1, 0)
+
+#define SYS_RAM_INIT_RAM_INIT                             BIT(1)
+#define SYS_RAM_INIT_RAM_CFG_HOOK                         BIT(0)
+
+#endif
diff --git a/drivers/net/ethernet/neterion/vxge/vxge-config.c b/drivers/net/ethernet/neterion/vxge/vxge-config.c
index 6223930a8155..c60da9e8bf14 100644
--- a/drivers/net/ethernet/neterion/vxge/vxge-config.c
+++ b/drivers/net/ethernet/neterion/vxge/vxge-config.c
@@ -693,7 +693,7 @@ __vxge_hw_device_is_privilaged(u32 host_type, u32 func_id)
 		VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)
 		return VXGE_HW_OK;
 	else
-		return VXGE_HW_ERR_PRIVILAGED_OPEARATION;
+		return VXGE_HW_ERR_PRIVILEGED_OPERATION;
 }
 
 /*
@@ -1920,7 +1920,7 @@ enum vxge_hw_status vxge_hw_device_getpause_data(struct __vxge_hw_device *hldev,
 	}
 
 	if (!(hldev->access_rights & VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
-		status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
+		status = VXGE_HW_ERR_PRIVILEGED_OPERATION;
 		goto exit;
 	}
 
@@ -3153,7 +3153,7 @@ vxge_hw_mgmt_reg_read(struct __vxge_hw_device *hldev,
 	case vxge_hw_mgmt_reg_type_mrpcim:
 		if (!(hldev->access_rights &
 			VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
-			status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
+			status = VXGE_HW_ERR_PRIVILEGED_OPERATION;
 			break;
 		}
 		if (offset > sizeof(struct vxge_hw_mrpcim_reg) - 8) {
@@ -3165,7 +3165,7 @@ vxge_hw_mgmt_reg_read(struct __vxge_hw_device *hldev,
 	case vxge_hw_mgmt_reg_type_srpcim:
 		if (!(hldev->access_rights &
 			VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM)) {
-			status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
+			status = VXGE_HW_ERR_PRIVILEGED_OPERATION;
 			break;
 		}
 		if (index > VXGE_HW_TITAN_SRPCIM_REG_SPACES - 1) {
@@ -3279,7 +3279,7 @@ vxge_hw_mgmt_reg_write(struct __vxge_hw_device *hldev,
 	case vxge_hw_mgmt_reg_type_mrpcim:
 		if (!(hldev->access_rights &
 			VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
-			status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
+			status = VXGE_HW_ERR_PRIVILEGED_OPERATION;
 			break;
 		}
 		if (offset > sizeof(struct vxge_hw_mrpcim_reg) - 8) {
@@ -3291,7 +3291,7 @@ vxge_hw_mgmt_reg_write(struct __vxge_hw_device *hldev,
 	case vxge_hw_mgmt_reg_type_srpcim:
 		if (!(hldev->access_rights &
 			VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM)) {
-			status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
+			status = VXGE_HW_ERR_PRIVILEGED_OPERATION;
 			break;
 		}
 		if (index > VXGE_HW_TITAN_SRPCIM_REG_SPACES - 1) {
diff --git a/drivers/net/ethernet/neterion/vxge/vxge-config.h b/drivers/net/ethernet/neterion/vxge/vxge-config.h
index cfa970417f81..d743a37a3cee 100644
--- a/drivers/net/ethernet/neterion/vxge/vxge-config.h
+++ b/drivers/net/ethernet/neterion/vxge/vxge-config.h
@@ -127,7 +127,7 @@ enum vxge_hw_status {
 	VXGE_HW_ERR_INVALID_TCODE 		  = VXGE_HW_BASE_ERR + 14,
 	VXGE_HW_ERR_INVALID_BLOCK_SIZE		  = VXGE_HW_BASE_ERR + 15,
 	VXGE_HW_ERR_INVALID_STATE		  = VXGE_HW_BASE_ERR + 16,
-	VXGE_HW_ERR_PRIVILAGED_OPEARATION	  = VXGE_HW_BASE_ERR + 17,
+	VXGE_HW_ERR_PRIVILEGED_OPERATION	  = VXGE_HW_BASE_ERR + 17,
 	VXGE_HW_ERR_INVALID_PORT 		  = VXGE_HW_BASE_ERR + 18,
 	VXGE_HW_ERR_FIFO		 	  = VXGE_HW_BASE_ERR + 19,
 	VXGE_HW_ERR_VPATH			  = VXGE_HW_BASE_ERR + 20,
diff --git a/drivers/net/ethernet/neterion/vxge/vxge-ethtool.c b/drivers/net/ethernet/neterion/vxge/vxge-ethtool.c
index 0452848d1316..03c3d1230c17 100644
--- a/drivers/net/ethernet/neterion/vxge/vxge-ethtool.c
+++ b/drivers/net/ethernet/neterion/vxge/vxge-ethtool.c
@@ -276,7 +276,7 @@ static void vxge_get_ethtool_stats(struct net_device *dev,
 	*ptr++ = 0;
 	status = vxge_hw_device_xmac_stats_get(hldev, xmac_stats);
 	if (status != VXGE_HW_OK) {
-		if (status != VXGE_HW_ERR_PRIVILAGED_OPEARATION) {
+		if (status != VXGE_HW_ERR_PRIVILEGED_OPERATION) {
 			vxge_debug_init(VXGE_ERR,
 				"%s : %d Failure in getting xmac stats",
 				__func__, __LINE__);
diff --git a/drivers/net/ethernet/neterion/vxge/vxge-main.c b/drivers/net/ethernet/neterion/vxge/vxge-main.c
index b2299f2b2155..a8918bb7c802 100644
--- a/drivers/net/ethernet/neterion/vxge/vxge-main.c
+++ b/drivers/net/ethernet/neterion/vxge/vxge-main.c
@@ -3484,11 +3484,11 @@ static int vxge_device_register(struct __vxge_hw_device *hldev,
 				0,
 				&stat);
 
-	if (status == VXGE_HW_ERR_PRIVILAGED_OPEARATION)
+	if (status == VXGE_HW_ERR_PRIVILEGED_OPERATION)
 		vxge_debug_init(
 			vxge_hw_device_trace_level_get(hldev),
 			"%s: device stats clear returns"
-			"VXGE_HW_ERR_PRIVILAGED_OPEARATION", ndev->name);
+			"VXGE_HW_ERR_PRIVILEGED_OPERATION", ndev->name);
 
 	vxge_debug_entryexit(vxge_hw_device_trace_level_get(hldev),
 		"%s: %s:%d  Exiting...",
diff --git a/drivers/net/ethernet/netronome/Kconfig b/drivers/net/ethernet/netronome/Kconfig
index ae0c46ba7546..66f15b05b65e 100644
--- a/drivers/net/ethernet/netronome/Kconfig
+++ b/drivers/net/ethernet/netronome/Kconfig
@@ -36,6 +36,19 @@ config NFP_APP_FLOWER
 	  either directly, with Open vSwitch, or any other way.  Note that
 	  TC Flower offload requires specific FW to work.
 
+config NFP_APP_ABM_NIC
+	bool "NFP4000/NFP6000 Advanced buffer management NIC support"
+	depends on NFP
+	depends on NET_SWITCHDEV
+	default y
+	help
+	  Enable driver support for Advanced buffer management NIC on NFP.
+	  ABM NIC allows advanced configuration of queuing and scheduling
+	  of packets, including ECN marking. Say Y, if you are planning to
+	  use one of the NFP4000 and NFP6000 platforms which support this
+	  functionality.
+	  Code will be built into the nfp.ko driver.
+
 config NFP_DEBUG
 	bool "Debug support for Netronome(R) NFP4000/NFP6000 NIC drivers"
 	depends on NFP
diff --git a/drivers/net/ethernet/netronome/nfp/Makefile b/drivers/net/ethernet/netronome/nfp/Makefile
index d5866d708dfa..4afb10375397 100644
--- a/drivers/net/ethernet/netronome/nfp/Makefile
+++ b/drivers/net/ethernet/netronome/nfp/Makefile
@@ -30,12 +30,14 @@ nfp-objs := \
 	    nfp_net_sriov.o \
 	    nfp_netvf_main.o \
 	    nfp_port.o \
+	    nfp_shared_buf.o \
 	    nic/main.o
 
 ifeq ($(CONFIG_NFP_APP_FLOWER),y)
 nfp-objs += \
 	    flower/action.o \
 	    flower/cmsg.o \
+	    flower/lag_conf.o \
 	    flower/main.o \
 	    flower/match.o \
 	    flower/metadata.o \
@@ -52,4 +54,10 @@ nfp-objs += \
 	    bpf/jit.o
 endif
 
+ifeq ($(CONFIG_NFP_APP_ABM_NIC),y)
+nfp-objs += \
+	    abm/ctrl.o \
+	    abm/main.o
+endif
+
 nfp-$(CONFIG_NFP_DEBUG) += nfp_net_debugfs.o
diff --git a/drivers/net/ethernet/netronome/nfp/abm/ctrl.c b/drivers/net/ethernet/netronome/nfp/abm/ctrl.c
new file mode 100644
index 000000000000..b157ccd8c80f
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/abm/ctrl.c
@@ -0,0 +1,333 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+/*
+ * Copyright (C) 2018 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+
+#include "../nfpcore/nfp_cpp.h"
+#include "../nfpcore/nfp_nffw.h"
+#include "../nfp_app.h"
+#include "../nfp_abi.h"
+#include "../nfp_main.h"
+#include "../nfp_net.h"
+#include "main.h"
+
+#define NFP_QLVL_SYM_NAME	"_abi_nfd_out_q_lvls_%u"
+#define NFP_QLVL_STRIDE		16
+#define NFP_QLVL_BLOG_BYTES	0
+#define NFP_QLVL_BLOG_PKTS	4
+#define NFP_QLVL_THRS		8
+
+#define NFP_QMSTAT_SYM_NAME	"_abi_nfdqm%u_stats"
+#define NFP_QMSTAT_STRIDE	32
+#define NFP_QMSTAT_NON_STO	0
+#define NFP_QMSTAT_STO		8
+#define NFP_QMSTAT_DROP		16
+#define NFP_QMSTAT_ECN		24
+
+static unsigned long long
+nfp_abm_q_lvl_thrs(struct nfp_abm_link *alink, unsigned int queue)
+{
+	return alink->abm->q_lvls->addr +
+		(alink->queue_base + queue) * NFP_QLVL_STRIDE + NFP_QLVL_THRS;
+}
+
+static int
+nfp_abm_ctrl_stat(struct nfp_abm_link *alink, const struct nfp_rtsym *sym,
+		  unsigned int stride, unsigned int offset, unsigned int i,
+		  bool is_u64, u64 *res)
+{
+	struct nfp_cpp *cpp = alink->abm->app->cpp;
+	u32 val32, mur;
+	u64 val, addr;
+	int err;
+
+	mur = NFP_CPP_ATOMIC_RD(sym->target, sym->domain);
+
+	addr = sym->addr + (alink->queue_base + i) * stride + offset;
+	if (is_u64)
+		err = nfp_cpp_readq(cpp, mur, addr, &val);
+	else
+		err = nfp_cpp_readl(cpp, mur, addr, &val32);
+	if (err) {
+		nfp_err(cpp,
+			"RED offload reading stat failed on vNIC %d queue %d\n",
+			alink->id, i);
+		return err;
+	}
+
+	*res = is_u64 ? val : val32;
+	return 0;
+}
+
+static int
+nfp_abm_ctrl_stat_all(struct nfp_abm_link *alink, const struct nfp_rtsym *sym,
+		      unsigned int stride, unsigned int offset, bool is_u64,
+		      u64 *res)
+{
+	u64 val, sum = 0;
+	unsigned int i;
+	int err;
+
+	for (i = 0; i < alink->vnic->max_rx_rings; i++) {
+		err = nfp_abm_ctrl_stat(alink, sym, stride, offset, i,
+					is_u64, &val);
+		if (err)
+			return err;
+		sum += val;
+	}
+
+	*res = sum;
+	return 0;
+}
+
+int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int i, u32 val)
+{
+	struct nfp_cpp *cpp = alink->abm->app->cpp;
+	u32 muw;
+	int err;
+
+	muw = NFP_CPP_ATOMIC_WR(alink->abm->q_lvls->target,
+				alink->abm->q_lvls->domain);
+
+	err = nfp_cpp_writel(cpp, muw, nfp_abm_q_lvl_thrs(alink, i), val);
+	if (err) {
+		nfp_err(cpp, "RED offload setting level failed on vNIC %d queue %d\n",
+			alink->id, i);
+		return err;
+	}
+
+	return 0;
+}
+
+int nfp_abm_ctrl_set_all_q_lvls(struct nfp_abm_link *alink, u32 val)
+{
+	int i, err;
+
+	for (i = 0; i < alink->vnic->max_rx_rings; i++) {
+		err = nfp_abm_ctrl_set_q_lvl(alink, i, val);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int i)
+{
+	u64 val;
+
+	if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, NFP_QMSTAT_STRIDE,
+			      NFP_QMSTAT_NON_STO, i, true, &val))
+		return 0;
+	return val;
+}
+
+u64 nfp_abm_ctrl_stat_sto(struct nfp_abm_link *alink, unsigned int i)
+{
+	u64 val;
+
+	if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, NFP_QMSTAT_STRIDE,
+			      NFP_QMSTAT_STO, i, true, &val))
+		return 0;
+	return val;
+}
+
+int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int i,
+			      struct nfp_alink_stats *stats)
+{
+	int err;
+
+	stats->tx_pkts = nn_readq(alink->vnic, NFP_NET_CFG_RXR_STATS(i));
+	stats->tx_bytes = nn_readq(alink->vnic, NFP_NET_CFG_RXR_STATS(i) + 8);
+
+	err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls,
+				NFP_QLVL_STRIDE, NFP_QLVL_BLOG_BYTES,
+				i, false, &stats->backlog_bytes);
+	if (err)
+		return err;
+
+	err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls,
+				NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS,
+				i, false, &stats->backlog_pkts);
+	if (err)
+		return err;
+
+	err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
+				NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
+				i, true, &stats->drops);
+	if (err)
+		return err;
+
+	return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
+				 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
+				 i, true, &stats->overlimits);
+}
+
+int nfp_abm_ctrl_read_stats(struct nfp_abm_link *alink,
+			    struct nfp_alink_stats *stats)
+{
+	u64 pkts = 0, bytes = 0;
+	int i, err;
+
+	for (i = 0; i < alink->vnic->max_rx_rings; i++) {
+		pkts += nn_readq(alink->vnic, NFP_NET_CFG_RXR_STATS(i));
+		bytes += nn_readq(alink->vnic, NFP_NET_CFG_RXR_STATS(i) + 8);
+	}
+	stats->tx_pkts = pkts;
+	stats->tx_bytes = bytes;
+
+	err = nfp_abm_ctrl_stat_all(alink, alink->abm->q_lvls,
+				    NFP_QLVL_STRIDE, NFP_QLVL_BLOG_BYTES,
+				    false, &stats->backlog_bytes);
+	if (err)
+		return err;
+
+	err = nfp_abm_ctrl_stat_all(alink, alink->abm->q_lvls,
+				    NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS,
+				    false, &stats->backlog_pkts);
+	if (err)
+		return err;
+
+	err = nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats,
+				    NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
+				    true, &stats->drops);
+	if (err)
+		return err;
+
+	return nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats,
+				     NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
+				     true, &stats->overlimits);
+}
+
+int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink, unsigned int i,
+			       struct nfp_alink_xstats *xstats)
+{
+	int err;
+
+	err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
+				NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
+				i, true, &xstats->pdrop);
+	if (err)
+		return err;
+
+	return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
+				 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
+				 i, true, &xstats->ecn_marked);
+}
+
+int nfp_abm_ctrl_read_xstats(struct nfp_abm_link *alink,
+			     struct nfp_alink_xstats *xstats)
+{
+	int err;
+
+	err = nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats,
+				    NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
+				    true, &xstats->pdrop);
+	if (err)
+		return err;
+
+	return nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats,
+				     NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
+				     true, &xstats->ecn_marked);
+}
+
+int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm)
+{
+	return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_ENABLE,
+			    NULL, 0, NULL, 0);
+}
+
+int nfp_abm_ctrl_qm_disable(struct nfp_abm *abm)
+{
+	return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_DISABLE,
+			    NULL, 0, NULL, 0);
+}
+
+void nfp_abm_ctrl_read_params(struct nfp_abm_link *alink)
+{
+	alink->queue_base = nn_readl(alink->vnic, NFP_NET_CFG_START_RXQ);
+	alink->queue_base /= alink->vnic->stride_rx;
+}
+
+static const struct nfp_rtsym *
+nfp_abm_ctrl_find_rtsym(struct nfp_pf *pf, const char *name, unsigned int size)
+{
+	const struct nfp_rtsym *sym;
+
+	sym = nfp_rtsym_lookup(pf->rtbl, name);
+	if (!sym) {
+		nfp_err(pf->cpp, "Symbol '%s' not found\n", name);
+		return ERR_PTR(-ENOENT);
+	}
+	if (sym->size != size) {
+		nfp_err(pf->cpp,
+			"Symbol '%s' wrong size: expected %u got %llu\n",
+			name, size, sym->size);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return sym;
+}
+
+static const struct nfp_rtsym *
+nfp_abm_ctrl_find_q_rtsym(struct nfp_pf *pf, const char *name,
+			  unsigned int size)
+{
+	return nfp_abm_ctrl_find_rtsym(pf, name, size * NFP_NET_MAX_RX_RINGS);
+}
+
+int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm)
+{
+	struct nfp_pf *pf = abm->app->pf;
+	const struct nfp_rtsym *sym;
+	unsigned int pf_id;
+	char pf_symbol[64];
+
+	pf_id =	nfp_cppcore_pcie_unit(pf->cpp);
+	abm->pf_id = pf_id;
+
+	snprintf(pf_symbol, sizeof(pf_symbol), NFP_QLVL_SYM_NAME, pf_id);
+	sym = nfp_abm_ctrl_find_q_rtsym(pf, pf_symbol, NFP_QLVL_STRIDE);
+	if (IS_ERR(sym))
+		return PTR_ERR(sym);
+	abm->q_lvls = sym;
+
+	snprintf(pf_symbol, sizeof(pf_symbol), NFP_QMSTAT_SYM_NAME, pf_id);
+	sym = nfp_abm_ctrl_find_q_rtsym(pf, pf_symbol, NFP_QMSTAT_STRIDE);
+	if (IS_ERR(sym))
+		return PTR_ERR(sym);
+	abm->qm_stats = sym;
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/netronome/nfp/abm/main.c b/drivers/net/ethernet/netronome/nfp/abm/main.c
new file mode 100644
index 000000000000..1561c2724c26
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/abm/main.c
@@ -0,0 +1,765 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+/*
+ * Copyright (C) 2018 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/etherdevice.h>
+#include <linux/lockdep.h>
+#include <linux/netdevice.h>
+#include <linux/rcupdate.h>
+#include <linux/slab.h>
+#include <net/pkt_cls.h>
+#include <net/pkt_sched.h>
+#include <net/red.h>
+
+#include "../nfpcore/nfp.h"
+#include "../nfpcore/nfp_cpp.h"
+#include "../nfpcore/nfp_nsp.h"
+#include "../nfp_app.h"
+#include "../nfp_main.h"
+#include "../nfp_net.h"
+#include "../nfp_net_repr.h"
+#include "../nfp_port.h"
+#include "main.h"
+
+static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id)
+{
+	return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) |
+	       FIELD_PREP(NFP_ABM_PORTID_ID, id);
+}
+
+static int
+__nfp_abm_reset_root(struct net_device *netdev, struct nfp_abm_link *alink,
+		     u32 handle, unsigned int qs, u32 init_val)
+{
+	struct nfp_port *port = nfp_port_from_netdev(netdev);
+	int ret;
+
+	ret = nfp_abm_ctrl_set_all_q_lvls(alink, init_val);
+	memset(alink->qdiscs, 0, sizeof(*alink->qdiscs) * alink->num_qdiscs);
+
+	alink->parent = handle;
+	alink->num_qdiscs = qs;
+	port->tc_offload_cnt = qs;
+
+	return ret;
+}
+
+static void
+nfp_abm_reset_root(struct net_device *netdev, struct nfp_abm_link *alink,
+		   u32 handle, unsigned int qs)
+{
+	__nfp_abm_reset_root(netdev, alink, handle, qs, ~0);
+}
+
+static int
+nfp_abm_red_find(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt)
+{
+	unsigned int i = TC_H_MIN(opt->parent) - 1;
+
+	if (opt->parent == TC_H_ROOT)
+		i = 0;
+	else if (TC_H_MAJ(alink->parent) == TC_H_MAJ(opt->parent))
+		i = TC_H_MIN(opt->parent) - 1;
+	else
+		return -EOPNOTSUPP;
+
+	if (i >= alink->num_qdiscs || opt->handle != alink->qdiscs[i].handle)
+		return -EOPNOTSUPP;
+
+	return i;
+}
+
+static void
+nfp_abm_red_destroy(struct net_device *netdev, struct nfp_abm_link *alink,
+		    u32 handle)
+{
+	unsigned int i;
+
+	for (i = 0; i < alink->num_qdiscs; i++)
+		if (handle == alink->qdiscs[i].handle)
+			break;
+	if (i == alink->num_qdiscs)
+		return;
+
+	if (alink->parent == TC_H_ROOT) {
+		nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 0);
+	} else {
+		nfp_abm_ctrl_set_q_lvl(alink, i, ~0);
+		memset(&alink->qdiscs[i], 0, sizeof(*alink->qdiscs));
+	}
+}
+
+static int
+nfp_abm_red_replace(struct net_device *netdev, struct nfp_abm_link *alink,
+		    struct tc_red_qopt_offload *opt)
+{
+	bool existing;
+	int i, err;
+
+	i = nfp_abm_red_find(alink, opt);
+	existing = i >= 0;
+
+	if (opt->set.min != opt->set.max || !opt->set.is_ecn) {
+		nfp_warn(alink->abm->app->cpp,
+			 "RED offload failed - unsupported parameters\n");
+		err = -EINVAL;
+		goto err_destroy;
+	}
+
+	if (existing) {
+		if (alink->parent == TC_H_ROOT)
+			err = nfp_abm_ctrl_set_all_q_lvls(alink, opt->set.min);
+		else
+			err = nfp_abm_ctrl_set_q_lvl(alink, i, opt->set.min);
+		if (err)
+			goto err_destroy;
+		return 0;
+	}
+
+	if (opt->parent == TC_H_ROOT) {
+		i = 0;
+		err = __nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 1,
+					   opt->set.min);
+	} else if (TC_H_MAJ(alink->parent) == TC_H_MAJ(opt->parent)) {
+		i = TC_H_MIN(opt->parent) - 1;
+		err = nfp_abm_ctrl_set_q_lvl(alink, i, opt->set.min);
+	} else {
+		return -EINVAL;
+	}
+	/* Set the handle to try full clean up, in case IO failed */
+	alink->qdiscs[i].handle = opt->handle;
+	if (err)
+		goto err_destroy;
+
+	if (opt->parent == TC_H_ROOT)
+		err = nfp_abm_ctrl_read_stats(alink, &alink->qdiscs[i].stats);
+	else
+		err = nfp_abm_ctrl_read_q_stats(alink, i,
+						&alink->qdiscs[i].stats);
+	if (err)
+		goto err_destroy;
+
+	if (opt->parent == TC_H_ROOT)
+		err = nfp_abm_ctrl_read_xstats(alink,
+					       &alink->qdiscs[i].xstats);
+	else
+		err = nfp_abm_ctrl_read_q_xstats(alink, i,
+						 &alink->qdiscs[i].xstats);
+	if (err)
+		goto err_destroy;
+
+	alink->qdiscs[i].stats.backlog_pkts = 0;
+	alink->qdiscs[i].stats.backlog_bytes = 0;
+
+	return 0;
+err_destroy:
+	/* If the qdisc keeps on living, but we can't offload undo changes */
+	if (existing) {
+		opt->set.qstats->qlen -= alink->qdiscs[i].stats.backlog_pkts;
+		opt->set.qstats->backlog -=
+			alink->qdiscs[i].stats.backlog_bytes;
+	}
+	nfp_abm_red_destroy(netdev, alink, opt->handle);
+
+	return err;
+}
+
+static void
+nfp_abm_update_stats(struct nfp_alink_stats *new, struct nfp_alink_stats *old,
+		     struct tc_qopt_offload_stats *stats)
+{
+	_bstats_update(stats->bstats, new->tx_bytes - old->tx_bytes,
+		       new->tx_pkts - old->tx_pkts);
+	stats->qstats->qlen += new->backlog_pkts - old->backlog_pkts;
+	stats->qstats->backlog += new->backlog_bytes - old->backlog_bytes;
+	stats->qstats->overlimits += new->overlimits - old->overlimits;
+	stats->qstats->drops += new->drops - old->drops;
+}
+
+static int
+nfp_abm_red_stats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt)
+{
+	struct nfp_alink_stats *prev_stats;
+	struct nfp_alink_stats stats;
+	int i, err;
+
+	i = nfp_abm_red_find(alink, opt);
+	if (i < 0)
+		return i;
+	prev_stats = &alink->qdiscs[i].stats;
+
+	if (alink->parent == TC_H_ROOT)
+		err = nfp_abm_ctrl_read_stats(alink, &stats);
+	else
+		err = nfp_abm_ctrl_read_q_stats(alink, i, &stats);
+	if (err)
+		return err;
+
+	nfp_abm_update_stats(&stats, prev_stats, &opt->stats);
+
+	*prev_stats = stats;
+
+	return 0;
+}
+
+static int
+nfp_abm_red_xstats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt)
+{
+	struct nfp_alink_xstats *prev_xstats;
+	struct nfp_alink_xstats xstats;
+	int i, err;
+
+	i = nfp_abm_red_find(alink, opt);
+	if (i < 0)
+		return i;
+	prev_xstats = &alink->qdiscs[i].xstats;
+
+	if (alink->parent == TC_H_ROOT)
+		err = nfp_abm_ctrl_read_xstats(alink, &xstats);
+	else
+		err = nfp_abm_ctrl_read_q_xstats(alink, i, &xstats);
+	if (err)
+		return err;
+
+	opt->xstats->forced_mark += xstats.ecn_marked - prev_xstats->ecn_marked;
+	opt->xstats->pdrop += xstats.pdrop - prev_xstats->pdrop;
+
+	*prev_xstats = xstats;
+
+	return 0;
+}
+
+static int
+nfp_abm_setup_tc_red(struct net_device *netdev, struct nfp_abm_link *alink,
+		     struct tc_red_qopt_offload *opt)
+{
+	switch (opt->command) {
+	case TC_RED_REPLACE:
+		return nfp_abm_red_replace(netdev, alink, opt);
+	case TC_RED_DESTROY:
+		nfp_abm_red_destroy(netdev, alink, opt->handle);
+		return 0;
+	case TC_RED_STATS:
+		return nfp_abm_red_stats(alink, opt);
+	case TC_RED_XSTATS:
+		return nfp_abm_red_xstats(alink, opt);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int
+nfp_abm_mq_stats(struct nfp_abm_link *alink, struct tc_mq_qopt_offload *opt)
+{
+	struct nfp_alink_stats stats;
+	unsigned int i;
+	int err;
+
+	for (i = 0; i < alink->num_qdiscs; i++) {
+		if (alink->qdiscs[i].handle == TC_H_UNSPEC)
+			continue;
+
+		err = nfp_abm_ctrl_read_q_stats(alink, i, &stats);
+		if (err)
+			return err;
+
+		nfp_abm_update_stats(&stats, &alink->qdiscs[i].stats,
+				     &opt->stats);
+	}
+
+	return 0;
+}
+
+static int
+nfp_abm_setup_tc_mq(struct net_device *netdev, struct nfp_abm_link *alink,
+		    struct tc_mq_qopt_offload *opt)
+{
+	switch (opt->command) {
+	case TC_MQ_CREATE:
+		nfp_abm_reset_root(netdev, alink, opt->handle,
+				   alink->total_queues);
+		return 0;
+	case TC_MQ_DESTROY:
+		if (opt->handle == alink->parent)
+			nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 0);
+		return 0;
+	case TC_MQ_STATS:
+		return nfp_abm_mq_stats(alink, opt);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int
+nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev,
+		 enum tc_setup_type type, void *type_data)
+{
+	struct nfp_repr *repr = netdev_priv(netdev);
+	struct nfp_port *port;
+
+	port = nfp_port_from_netdev(netdev);
+	if (!port || port->type != NFP_PORT_PF_PORT)
+		return -EOPNOTSUPP;
+
+	switch (type) {
+	case TC_SETUP_QDISC_MQ:
+		return nfp_abm_setup_tc_mq(netdev, repr->app_priv, type_data);
+	case TC_SETUP_QDISC_RED:
+		return nfp_abm_setup_tc_red(netdev, repr->app_priv, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static struct net_device *nfp_abm_repr_get(struct nfp_app *app, u32 port_id)
+{
+	enum nfp_repr_type rtype;
+	struct nfp_reprs *reprs;
+	u8 port;
+
+	rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id);
+	port = FIELD_GET(NFP_ABM_PORTID_ID, port_id);
+
+	reprs = rcu_dereference(app->reprs[rtype]);
+	if (!reprs)
+		return NULL;
+
+	if (port >= reprs->num_reprs)
+		return NULL;
+
+	return rcu_dereference(reprs->reprs[port]);
+}
+
+static int
+nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink,
+		   enum nfp_port_type ptype)
+{
+	struct net_device *netdev;
+	enum nfp_repr_type rtype;
+	struct nfp_reprs *reprs;
+	struct nfp_repr *repr;
+	struct nfp_port *port;
+	unsigned int txqs;
+	int err;
+
+	if (ptype == NFP_PORT_PHYS_PORT) {
+		rtype = NFP_REPR_TYPE_PHYS_PORT;
+		txqs = 1;
+	} else {
+		rtype = NFP_REPR_TYPE_PF;
+		txqs = alink->vnic->max_rx_rings;
+	}
+
+	netdev = nfp_repr_alloc_mqs(app, txqs, 1);
+	if (!netdev)
+		return -ENOMEM;
+	repr = netdev_priv(netdev);
+	repr->app_priv = alink;
+
+	port = nfp_port_alloc(app, ptype, netdev);
+	if (IS_ERR(port)) {
+		err = PTR_ERR(port);
+		goto err_free_repr;
+	}
+
+	if (ptype == NFP_PORT_PHYS_PORT) {
+		port->eth_forced = true;
+		err = nfp_port_init_phy_port(app->pf, app, port, alink->id);
+		if (err)
+			goto err_free_port;
+	} else {
+		port->pf_id = alink->abm->pf_id;
+		port->pf_split = app->pf->max_data_vnics > 1;
+		port->pf_split_id = alink->id;
+		port->vnic = alink->vnic->dp.ctrl_bar;
+	}
+
+	SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev);
+	eth_hw_addr_random(netdev);
+
+	err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id),
+			    port, alink->vnic->dp.netdev);
+	if (err)
+		goto err_free_port;
+
+	reprs = nfp_reprs_get_locked(app, rtype);
+	WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr");
+	rcu_assign_pointer(reprs->reprs[alink->id], netdev);
+
+	nfp_info(app->cpp, "%s Port %d Representor(%s) created\n",
+		 ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys",
+		 alink->id, netdev->name);
+
+	return 0;
+
+err_free_port:
+	nfp_port_free(port);
+err_free_repr:
+	nfp_repr_free(netdev);
+	return err;
+}
+
+static void
+nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink,
+		  enum nfp_repr_type rtype)
+{
+	struct net_device *netdev;
+	struct nfp_reprs *reprs;
+
+	reprs = nfp_reprs_get_locked(app, rtype);
+	netdev = nfp_repr_get_locked(app, reprs, alink->id);
+	if (!netdev)
+		return;
+	rcu_assign_pointer(reprs->reprs[alink->id], NULL);
+	synchronize_rcu();
+	/* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */
+	nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev));
+}
+
+static void
+nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink)
+{
+	nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF);
+	nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT);
+}
+
+static void nfp_abm_kill_reprs_all(struct nfp_abm *abm)
+{
+	struct nfp_pf *pf = abm->app->pf;
+	struct nfp_net *nn;
+
+	list_for_each_entry(nn, &pf->vnics, vnic_list)
+		nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv);
+}
+
+static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app)
+{
+	struct nfp_abm *abm = app->priv;
+
+	return abm->eswitch_mode;
+}
+
+static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm)
+{
+	nfp_abm_kill_reprs_all(abm);
+	nfp_abm_ctrl_qm_disable(abm);
+
+	abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
+	return 0;
+}
+
+static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm)
+{
+	if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY)
+		WARN_ON(nfp_abm_eswitch_set_legacy(abm));
+}
+
+static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm)
+{
+	struct nfp_app *app = abm->app;
+	struct nfp_pf *pf = app->pf;
+	struct nfp_net *nn;
+	int err;
+
+	err = nfp_abm_ctrl_qm_enable(abm);
+	if (err)
+		return err;
+
+	list_for_each_entry(nn, &pf->vnics, vnic_list) {
+		struct nfp_abm_link *alink = nn->app_priv;
+
+		err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT);
+		if (err)
+			goto err_kill_all_reprs;
+
+		err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT);
+		if (err)
+			goto err_kill_all_reprs;
+	}
+
+	abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
+	return 0;
+
+err_kill_all_reprs:
+	nfp_abm_kill_reprs_all(abm);
+	nfp_abm_ctrl_qm_disable(abm);
+	return err;
+}
+
+static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode)
+{
+	struct nfp_abm *abm = app->priv;
+
+	if (abm->eswitch_mode == mode)
+		return 0;
+
+	switch (mode) {
+	case DEVLINK_ESWITCH_MODE_LEGACY:
+		return nfp_abm_eswitch_set_legacy(abm);
+	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
+		return nfp_abm_eswitch_set_switchdev(abm);
+	default:
+		return -EINVAL;
+	}
+}
+
+static void
+nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn,
+		     unsigned int id)
+{
+	struct nfp_eth_table_port *eth_port = &pf->eth_tbl->ports[id];
+	u8 mac_addr[ETH_ALEN];
+	const char *mac_str;
+	char name[32];
+
+	if (id > pf->eth_tbl->count) {
+		nfp_warn(pf->cpp, "No entry for persistent MAC address\n");
+		eth_hw_addr_random(nn->dp.netdev);
+		return;
+	}
+
+	snprintf(name, sizeof(name), "eth%u.mac.pf%u",
+		 eth_port->eth_index, abm->pf_id);
+
+	mac_str = nfp_hwinfo_lookup(pf->hwinfo, name);
+	if (!mac_str) {
+		nfp_warn(pf->cpp, "Can't lookup persistent MAC address (%s)\n",
+			 name);
+		eth_hw_addr_random(nn->dp.netdev);
+		return;
+	}
+
+	if (sscanf(mac_str, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx",
+		   &mac_addr[0], &mac_addr[1], &mac_addr[2],
+		   &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) {
+		nfp_warn(pf->cpp, "Can't parse persistent MAC address (%s)\n",
+			 mac_str);
+		eth_hw_addr_random(nn->dp.netdev);
+		return;
+	}
+
+	ether_addr_copy(nn->dp.netdev->dev_addr, mac_addr);
+	ether_addr_copy(nn->dp.netdev->perm_addr, mac_addr);
+}
+
+static int
+nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
+{
+	struct nfp_eth_table_port *eth_port = &app->pf->eth_tbl->ports[id];
+	struct nfp_abm *abm = app->priv;
+	struct nfp_abm_link *alink;
+	int err;
+
+	alink = kzalloc(sizeof(*alink), GFP_KERNEL);
+	if (!alink)
+		return -ENOMEM;
+	nn->app_priv = alink;
+	alink->abm = abm;
+	alink->vnic = nn;
+	alink->id = id;
+	alink->parent = TC_H_ROOT;
+	alink->total_queues = alink->vnic->max_rx_rings;
+	alink->qdiscs = kvzalloc(sizeof(*alink->qdiscs) * alink->total_queues,
+				 GFP_KERNEL);
+	if (!alink->qdiscs) {
+		err = -ENOMEM;
+		goto err_free_alink;
+	}
+
+	/* This is a multi-host app, make sure MAC/PHY is up, but don't
+	 * make the MAC/PHY state follow the state of any of the ports.
+	 */
+	err = nfp_eth_set_configured(app->cpp, eth_port->index, true);
+	if (err < 0)
+		goto err_free_qdiscs;
+
+	netif_keep_dst(nn->dp.netdev);
+
+	nfp_abm_vnic_set_mac(app->pf, abm, nn, id);
+	nfp_abm_ctrl_read_params(alink);
+
+	return 0;
+
+err_free_qdiscs:
+	kvfree(alink->qdiscs);
+err_free_alink:
+	kfree(alink);
+	return err;
+}
+
+static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn)
+{
+	struct nfp_abm_link *alink = nn->app_priv;
+
+	nfp_abm_kill_reprs(alink->abm, alink);
+	kvfree(alink->qdiscs);
+	kfree(alink);
+}
+
+static u64 *
+nfp_abm_port_get_stats(struct nfp_app *app, struct nfp_port *port, u64 *data)
+{
+	struct nfp_repr *repr = netdev_priv(port->netdev);
+	struct nfp_abm_link *alink;
+	unsigned int i;
+
+	if (port->type != NFP_PORT_PF_PORT)
+		return data;
+	alink = repr->app_priv;
+	for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) {
+		*data++ = nfp_abm_ctrl_stat_non_sto(alink, i);
+		*data++ = nfp_abm_ctrl_stat_sto(alink, i);
+	}
+	return data;
+}
+
+static int
+nfp_abm_port_get_stats_count(struct nfp_app *app, struct nfp_port *port)
+{
+	struct nfp_repr *repr = netdev_priv(port->netdev);
+	struct nfp_abm_link *alink;
+
+	if (port->type != NFP_PORT_PF_PORT)
+		return 0;
+	alink = repr->app_priv;
+	return alink->vnic->dp.num_r_vecs * 2;
+}
+
+static u8 *
+nfp_abm_port_get_stats_strings(struct nfp_app *app, struct nfp_port *port,
+			       u8 *data)
+{
+	struct nfp_repr *repr = netdev_priv(port->netdev);
+	struct nfp_abm_link *alink;
+	unsigned int i;
+
+	if (port->type != NFP_PORT_PF_PORT)
+		return data;
+	alink = repr->app_priv;
+	for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) {
+		data = nfp_pr_et(data, "q%u_no_wait", i);
+		data = nfp_pr_et(data, "q%u_delayed", i);
+	}
+	return data;
+}
+
+static int nfp_abm_init(struct nfp_app *app)
+{
+	struct nfp_pf *pf = app->pf;
+	struct nfp_reprs *reprs;
+	struct nfp_abm *abm;
+	int err;
+
+	if (!pf->eth_tbl) {
+		nfp_err(pf->cpp, "ABM NIC requires ETH table\n");
+		return -EINVAL;
+	}
+	if (pf->max_data_vnics != pf->eth_tbl->count) {
+		nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
+			pf->max_data_vnics, pf->eth_tbl->count);
+		return -EINVAL;
+	}
+	if (!pf->mac_stats_bar) {
+		nfp_warn(app->cpp, "ABM NIC requires mac_stats symbol\n");
+		return -EINVAL;
+	}
+
+	abm = kzalloc(sizeof(*abm), GFP_KERNEL);
+	if (!abm)
+		return -ENOMEM;
+	app->priv = abm;
+	abm->app = app;
+
+	err = nfp_abm_ctrl_find_addrs(abm);
+	if (err)
+		goto err_free_abm;
+
+	/* We start in legacy mode, make sure advanced queuing is disabled */
+	err = nfp_abm_ctrl_qm_disable(abm);
+	if (err)
+		goto err_free_abm;
+
+	err = -ENOMEM;
+	reprs = nfp_reprs_alloc(pf->max_data_vnics);
+	if (!reprs)
+		goto err_free_abm;
+	RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs);
+
+	reprs = nfp_reprs_alloc(pf->max_data_vnics);
+	if (!reprs)
+		goto err_free_phys;
+	RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs);
+
+	return 0;
+
+err_free_phys:
+	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
+err_free_abm:
+	kfree(abm);
+	app->priv = NULL;
+	return err;
+}
+
+static void nfp_abm_clean(struct nfp_app *app)
+{
+	struct nfp_abm *abm = app->priv;
+
+	nfp_abm_eswitch_clean_up(abm);
+	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF);
+	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
+	kfree(abm);
+	app->priv = NULL;
+}
+
+const struct nfp_app_type app_abm = {
+	.id		= NFP_APP_ACTIVE_BUFFER_MGMT_NIC,
+	.name		= "abm",
+
+	.init		= nfp_abm_init,
+	.clean		= nfp_abm_clean,
+
+	.vnic_alloc	= nfp_abm_vnic_alloc,
+	.vnic_free	= nfp_abm_vnic_free,
+
+	.port_get_stats		= nfp_abm_port_get_stats,
+	.port_get_stats_count	= nfp_abm_port_get_stats_count,
+	.port_get_stats_strings	= nfp_abm_port_get_stats_strings,
+
+	.setup_tc	= nfp_abm_setup_tc,
+
+	.eswitch_mode_get	= nfp_abm_eswitch_mode_get,
+	.eswitch_mode_set	= nfp_abm_eswitch_mode_set,
+
+	.repr_get	= nfp_abm_repr_get,
+};
diff --git a/drivers/net/ethernet/netronome/nfp/abm/main.h b/drivers/net/ethernet/netronome/nfp/abm/main.h
new file mode 100644
index 000000000000..934a70835473
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/abm/main.h
@@ -0,0 +1,142 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
+/*
+ * Copyright (C) 2018 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __NFP_ABM_H__
+#define __NFP_ABM_H__ 1
+
+#include <net/devlink.h>
+
+struct nfp_app;
+struct nfp_net;
+
+#define NFP_ABM_PORTID_TYPE	GENMASK(23, 16)
+#define NFP_ABM_PORTID_ID	GENMASK(7, 0)
+
+/**
+ * struct nfp_abm - ABM NIC app structure
+ * @app:	back pointer to nfp_app
+ * @pf_id:	ID of our PF link
+ * @eswitch_mode:	devlink eswitch mode, advanced functions only visible
+ *			in switchdev mode
+ * @q_lvls:	queue level control area
+ * @qm_stats:	queue statistics symbol
+ */
+struct nfp_abm {
+	struct nfp_app *app;
+	unsigned int pf_id;
+	enum devlink_eswitch_mode eswitch_mode;
+	const struct nfp_rtsym *q_lvls;
+	const struct nfp_rtsym *qm_stats;
+};
+
+/**
+ * struct nfp_alink_stats - ABM NIC statistics
+ * @tx_pkts:		number of TXed packets
+ * @tx_bytes:		number of TXed bytes
+ * @backlog_pkts:	momentary backlog length (packets)
+ * @backlog_bytes:	momentary backlog length (bytes)
+ * @overlimits:		number of ECN marked TXed packets (accumulative)
+ * @drops:		number of tail-dropped packets (accumulative)
+ */
+struct nfp_alink_stats {
+	u64 tx_pkts;
+	u64 tx_bytes;
+	u64 backlog_pkts;
+	u64 backlog_bytes;
+	u64 overlimits;
+	u64 drops;
+};
+
+/**
+ * struct nfp_alink_xstats - extended ABM NIC statistics
+ * @ecn_marked:		number of ECN marked TXed packets
+ * @pdrop:		number of hard drops due to queue limit
+ */
+struct nfp_alink_xstats {
+	u64 ecn_marked;
+	u64 pdrop;
+};
+
+/**
+ * struct nfp_red_qdisc - representation of single RED Qdisc
+ * @handle:	handle of currently offloaded RED Qdisc
+ * @stats:	statistics from last refresh
+ * @xstats:	base of extended statistics
+ */
+struct nfp_red_qdisc {
+	u32 handle;
+	struct nfp_alink_stats stats;
+	struct nfp_alink_xstats xstats;
+};
+
+/**
+ * struct nfp_abm_link - port tuple of a ABM NIC
+ * @abm:	back pointer to nfp_abm
+ * @vnic:	data vNIC
+ * @id:		id of the data vNIC
+ * @queue_base:	id of base to host queue within PCIe (not QC idx)
+ * @total_queues:	number of PF queues
+ * @parent:	handle of expected parent, i.e. handle of MQ, or TC_H_ROOT
+ * @num_qdiscs:	number of currently used qdiscs
+ * @qdiscs:	array of qdiscs
+ */
+struct nfp_abm_link {
+	struct nfp_abm *abm;
+	struct nfp_net *vnic;
+	unsigned int id;
+	unsigned int queue_base;
+	unsigned int total_queues;
+	u32 parent;
+	unsigned int num_qdiscs;
+	struct nfp_red_qdisc *qdiscs;
+};
+
+void nfp_abm_ctrl_read_params(struct nfp_abm_link *alink);
+int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm);
+int nfp_abm_ctrl_set_all_q_lvls(struct nfp_abm_link *alink, u32 val);
+int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int i,
+			   u32 val);
+int nfp_abm_ctrl_read_stats(struct nfp_abm_link *alink,
+			    struct nfp_alink_stats *stats);
+int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int i,
+			      struct nfp_alink_stats *stats);
+int nfp_abm_ctrl_read_xstats(struct nfp_abm_link *alink,
+			     struct nfp_alink_xstats *xstats);
+int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink, unsigned int i,
+			       struct nfp_alink_xstats *xstats);
+u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int i);
+u64 nfp_abm_ctrl_stat_sto(struct nfp_abm_link *alink, unsigned int i);
+int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm);
+int nfp_abm_ctrl_qm_disable(struct nfp_abm *abm);
+#endif
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/cmsg.c b/drivers/net/ethernet/netronome/nfp/bpf/cmsg.c
index 7e298148ca26..cb87fccb9f6a 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/cmsg.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/cmsg.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2017 Netronome Systems, Inc.
+ * Copyright (C) 2017-2018 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -102,6 +102,15 @@ nfp_bpf_cmsg_map_req_alloc(struct nfp_app_bpf *bpf, unsigned int n)
 	return nfp_bpf_cmsg_alloc(bpf, size);
 }
 
+static u8 nfp_bpf_cmsg_get_type(struct sk_buff *skb)
+{
+	struct cmsg_hdr *hdr;
+
+	hdr = (struct cmsg_hdr *)skb->data;
+
+	return hdr->type;
+}
+
 static unsigned int nfp_bpf_cmsg_get_tag(struct sk_buff *skb)
 {
 	struct cmsg_hdr *hdr;
@@ -431,6 +440,11 @@ void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb)
 		goto err_free;
 	}
 
+	if (nfp_bpf_cmsg_get_type(skb) == CMSG_TYPE_BPF_EVENT) {
+		nfp_bpf_event_output(bpf, skb);
+		return;
+	}
+
 	nfp_ctrl_lock(bpf->app->ctrl);
 
 	tag = nfp_bpf_cmsg_get_tag(skb);
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/fw.h b/drivers/net/ethernet/netronome/nfp/bpf/fw.h
index 39639ac28b01..4c7972e3db63 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/fw.h
+++ b/drivers/net/ethernet/netronome/nfp/bpf/fw.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2017 Netronome Systems, Inc.
+ * Copyright (C) 2017-2018 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -37,11 +37,20 @@
 #include <linux/bitops.h>
 #include <linux/types.h>
 
+/* Kernel's enum bpf_reg_type is not uABI so people may change it breaking
+ * our FW ABI.  In that case we will do translation in the driver.
+ */
+#define NFP_BPF_SCALAR_VALUE		1
+#define NFP_BPF_MAP_VALUE		4
+#define NFP_BPF_STACK			6
+#define NFP_BPF_PACKET_DATA		8
+
 enum bpf_cap_tlv_type {
 	NFP_BPF_CAP_TYPE_FUNC		= 1,
 	NFP_BPF_CAP_TYPE_ADJUST_HEAD	= 2,
 	NFP_BPF_CAP_TYPE_MAPS		= 3,
 	NFP_BPF_CAP_TYPE_RANDOM		= 4,
+	NFP_BPF_CAP_TYPE_QUEUE_SELECT	= 5,
 };
 
 struct nfp_bpf_cap_tlv_func {
@@ -81,6 +90,7 @@ enum nfp_bpf_cmsg_type {
 	CMSG_TYPE_MAP_DELETE	= 5,
 	CMSG_TYPE_MAP_GETNEXT	= 6,
 	CMSG_TYPE_MAP_GETFIRST	= 7,
+	CMSG_TYPE_BPF_EVENT	= 8,
 	__CMSG_TYPE_MAP_MAX,
 };
 
@@ -155,4 +165,13 @@ struct cmsg_reply_map_op {
 	__be32 resv;
 	struct cmsg_key_value_pair elem[0];
 };
+
+struct cmsg_bpf_event {
+	struct cmsg_hdr hdr;
+	__be32 cpu_id;
+	__be64 map_ptr;
+	__be32 data_size;
+	__be32 pkt_size;
+	u8 data[0];
+};
 #endif
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
index 29b4e5f8c102..8a92088df0d7 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016-2017 Netronome Systems, Inc.
+ * Copyright (C) 2016-2018 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -42,6 +42,7 @@
 
 #include "main.h"
 #include "../nfp_asm.h"
+#include "../nfp_net_ctrl.h"
 
 /* --- NFP prog --- */
 /* Foreach "multiple" entries macros provide pos and next<n> pointers.
@@ -210,6 +211,60 @@ emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer)
 	emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL);
 }
 
+static void
+__emit_br_bit(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 addr, u8 defer,
+	      bool set, bool src_lmextn)
+{
+	u16 addr_lo, addr_hi;
+	u64 insn;
+
+	addr_lo = addr & (OP_BR_BIT_ADDR_LO >> __bf_shf(OP_BR_BIT_ADDR_LO));
+	addr_hi = addr != addr_lo;
+
+	insn = OP_BR_BIT_BASE |
+		FIELD_PREP(OP_BR_BIT_A_SRC, areg) |
+		FIELD_PREP(OP_BR_BIT_B_SRC, breg) |
+		FIELD_PREP(OP_BR_BIT_BV, set) |
+		FIELD_PREP(OP_BR_BIT_DEFBR, defer) |
+		FIELD_PREP(OP_BR_BIT_ADDR_LO, addr_lo) |
+		FIELD_PREP(OP_BR_BIT_ADDR_HI, addr_hi) |
+		FIELD_PREP(OP_BR_BIT_SRC_LMEXTN, src_lmextn);
+
+	nfp_prog_push(nfp_prog, insn);
+}
+
+static void
+emit_br_bit_relo(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr,
+		 u8 defer, bool set, enum nfp_relo_type relo)
+{
+	struct nfp_insn_re_regs reg;
+	int err;
+
+	/* NOTE: The bit to test is specified as an rotation amount, such that
+	 *	 the bit to test will be placed on the MSB of the result when
+	 *	 doing a rotate right. For bit X, we need right rotate X + 1.
+	 */
+	bit += 1;
+
+	err = swreg_to_restricted(reg_none(), src, reg_imm(bit), &reg, false);
+	if (err) {
+		nfp_prog->error = err;
+		return;
+	}
+
+	__emit_br_bit(nfp_prog, reg.areg, reg.breg, addr, defer, set,
+		      reg.src_lmextn);
+
+	nfp_prog->prog[nfp_prog->prog_len - 1] |=
+		FIELD_PREP(OP_RELO_TYPE, relo);
+}
+
+static void
+emit_br_bset(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, u8 defer)
+{
+	emit_br_bit_relo(nfp_prog, src, bit, addr, defer, true, RELO_BR_REL);
+}
+
 static void
 __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi,
 	     enum immed_width width, bool invert,
@@ -308,6 +363,19 @@ emit_shf(struct nfp_prog *nfp_prog, swreg dst,
 		   reg.dst_lmextn, reg.src_lmextn);
 }
 
+static void
+emit_shf_indir(struct nfp_prog *nfp_prog, swreg dst,
+	       swreg lreg, enum shf_op op, swreg rreg, enum shf_sc sc)
+{
+	if (sc == SHF_SC_R_ROT) {
+		pr_err("indirect shift is not allowed on rotation\n");
+		nfp_prog->error = -EFAULT;
+		return;
+	}
+
+	emit_shf(nfp_prog, dst, lreg, op, rreg, sc, 0);
+}
+
 static void
 __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
 	   u16 areg, enum alu_op op, u16 breg, bool swap, bool wr_both,
@@ -1214,45 +1282,83 @@ wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 	return 0;
 }
 
-static int
-wrp_cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
-	    enum br_mask br_mask, bool swap)
+static const struct jmp_code_map {
+	enum br_mask br_mask;
+	bool swap;
+} jmp_code_map[] = {
+	[BPF_JGT >> 4]	= { BR_BLO, true },
+	[BPF_JGE >> 4]	= { BR_BHS, false },
+	[BPF_JLT >> 4]	= { BR_BLO, false },
+	[BPF_JLE >> 4]	= { BR_BHS, true },
+	[BPF_JSGT >> 4]	= { BR_BLT, true },
+	[BPF_JSGE >> 4]	= { BR_BGE, false },
+	[BPF_JSLT >> 4]	= { BR_BLT, false },
+	[BPF_JSLE >> 4]	= { BR_BGE, true },
+};
+
+static const struct jmp_code_map *nfp_jmp_code_get(struct nfp_insn_meta *meta)
+{
+	unsigned int op;
+
+	op = BPF_OP(meta->insn.code) >> 4;
+	/* br_mask of 0 is BR_BEQ which we don't use in jump code table */
+	if (WARN_ONCE(op >= ARRAY_SIZE(jmp_code_map) ||
+		      !jmp_code_map[op].br_mask,
+		      "no code found for jump instruction"))
+		return NULL;
+
+	return &jmp_code_map[op];
+}
+
+static int cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
 	const struct bpf_insn *insn = &meta->insn;
 	u64 imm = insn->imm; /* sign extend */
+	const struct jmp_code_map *code;
+	enum alu_op alu_op, carry_op;
 	u8 reg = insn->dst_reg * 2;
 	swreg tmp_reg;
 
+	code = nfp_jmp_code_get(meta);
+	if (!code)
+		return -EINVAL;
+
+	alu_op = meta->jump_neg_op ? ALU_OP_ADD : ALU_OP_SUB;
+	carry_op = meta->jump_neg_op ? ALU_OP_ADD_C : ALU_OP_SUB_C;
+
 	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
-	if (!swap)
-		emit_alu(nfp_prog, reg_none(), reg_a(reg), ALU_OP_SUB, tmp_reg);
+	if (!code->swap)
+		emit_alu(nfp_prog, reg_none(), reg_a(reg), alu_op, tmp_reg);
 	else
-		emit_alu(nfp_prog, reg_none(), tmp_reg, ALU_OP_SUB, reg_a(reg));
+		emit_alu(nfp_prog, reg_none(), tmp_reg, alu_op, reg_a(reg));
 
 	tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
-	if (!swap)
+	if (!code->swap)
 		emit_alu(nfp_prog, reg_none(),
-			 reg_a(reg + 1), ALU_OP_SUB_C, tmp_reg);
+			 reg_a(reg + 1), carry_op, tmp_reg);
 	else
 		emit_alu(nfp_prog, reg_none(),
-			 tmp_reg, ALU_OP_SUB_C, reg_a(reg + 1));
+			 tmp_reg, carry_op, reg_a(reg + 1));
 
-	emit_br(nfp_prog, br_mask, insn->off, 0);
+	emit_br(nfp_prog, code->br_mask, insn->off, 0);
 
 	return 0;
 }
 
-static int
-wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
-	    enum br_mask br_mask, bool swap)
+static int cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
 	const struct bpf_insn *insn = &meta->insn;
+	const struct jmp_code_map *code;
 	u8 areg, breg;
 
+	code = nfp_jmp_code_get(meta);
+	if (!code)
+		return -EINVAL;
+
 	areg = insn->dst_reg * 2;
 	breg = insn->src_reg * 2;
 
-	if (swap) {
+	if (code->swap) {
 		areg ^= breg;
 		breg ^= areg;
 		areg ^= breg;
@@ -1261,7 +1367,7 @@ wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 	emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg));
 	emit_alu(nfp_prog, reg_none(),
 		 reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1));
-	emit_br(nfp_prog, br_mask, insn->off, 0);
+	emit_br(nfp_prog, code->br_mask, insn->off, 0);
 
 	return 0;
 }
@@ -1357,15 +1463,9 @@ static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 static int
 map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
-	struct bpf_offloaded_map *offmap;
-	struct nfp_bpf_map *nfp_map;
 	bool load_lm_ptr;
 	u32 ret_tgt;
 	s64 lm_off;
-	swreg tid;
-
-	offmap = (struct bpf_offloaded_map *)meta->arg1.map_ptr;
-	nfp_map = offmap->dev_priv;
 
 	/* We only have to reload LM0 if the key is not at start of stack */
 	lm_off = nfp_prog->stack_depth;
@@ -1378,17 +1478,12 @@ map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	if (meta->func_id == BPF_FUNC_map_update_elem)
 		emit_csr_wr(nfp_prog, reg_b(3 * 2), NFP_CSR_ACT_LM_ADDR2);
 
-	/* Load map ID into a register, it should actually fit as an immediate
-	 * but in case it doesn't deal with it here, not in the delay slots.
-	 */
-	tid = ur_load_imm_any(nfp_prog, nfp_map->tid, imm_a(nfp_prog));
-
 	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id,
 		     2, RELO_BR_HELPER);
 	ret_tgt = nfp_prog_current_offset(nfp_prog) + 2;
 
 	/* Load map ID into A0 */
-	wrp_mov(nfp_prog, reg_a(0), tid);
+	wrp_mov(nfp_prog, reg_a(0), reg_a(2));
 
 	/* Load the return address into B0 */
 	wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL);
@@ -1400,7 +1495,7 @@ map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	if (!load_lm_ptr)
 		return 0;
 
-	emit_csr_wr(nfp_prog, stack_reg(nfp_prog),  NFP_CSR_ACT_LM_ADDR0);
+	emit_csr_wr(nfp_prog, stack_reg(nfp_prog), NFP_CSR_ACT_LM_ADDR0);
 	wrp_nops(nfp_prog, 3);
 
 	return 0;
@@ -1418,6 +1513,63 @@ nfp_get_prandom_u32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	return 0;
 }
 
+static int
+nfp_perf_event_output(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	swreg ptr_type;
+	u32 ret_tgt;
+
+	ptr_type = ur_load_imm_any(nfp_prog, meta->arg1.type, imm_a(nfp_prog));
+
+	ret_tgt = nfp_prog_current_offset(nfp_prog) + 3;
+
+	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id,
+		     2, RELO_BR_HELPER);
+
+	/* Load ptr type into A1 */
+	wrp_mov(nfp_prog, reg_a(1), ptr_type);
+
+	/* Load the return address into B0 */
+	wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL);
+
+	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+nfp_queue_select(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	u32 jmp_tgt;
+
+	jmp_tgt = nfp_prog_current_offset(nfp_prog) + 5;
+
+	/* Make sure the queue id fits into FW field */
+	emit_alu(nfp_prog, reg_none(), reg_a(meta->insn.src_reg * 2),
+		 ALU_OP_AND_NOT_B, reg_imm(0xff));
+	emit_br(nfp_prog, BR_BEQ, jmp_tgt, 2);
+
+	/* Set the 'queue selected' bit and the queue value */
+	emit_shf(nfp_prog, pv_qsel_set(nfp_prog),
+		 pv_qsel_set(nfp_prog), SHF_OP_OR, reg_imm(1),
+		 SHF_SC_L_SHF, PKT_VEL_QSEL_SET_BIT);
+	emit_ld_field(nfp_prog,
+		      pv_qsel_val(nfp_prog), 0x1, reg_b(meta->insn.src_reg * 2),
+		      SHF_SC_NONE, 0);
+	/* Delay slots end here, we will jump over next instruction if queue
+	 * value fits into the field.
+	 */
+	emit_ld_field(nfp_prog,
+		      pv_qsel_val(nfp_prog), 0x1, reg_imm(NFP_NET_RXR_MAX),
+		      SHF_SC_NONE, 0);
+
+	if (!nfp_prog_confirm_current_offset(nfp_prog, jmp_tgt))
+		return -EINVAL;
+
+	return 0;
+}
+
 /* --- Callbacks --- */
 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
@@ -1544,26 +1696,142 @@ static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	return 0;
 }
 
+/* Pseudo code:
+ *   if shift_amt >= 32
+ *     dst_high = dst_low << shift_amt[4:0]
+ *     dst_low = 0;
+ *   else
+ *     dst_high = (dst_high, dst_low) >> (32 - shift_amt)
+ *     dst_low = dst_low << shift_amt
+ *
+ * The indirect shift will use the same logic at runtime.
+ */
+static int __shl_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
+{
+	if (shift_amt < 32) {
+		emit_shf(nfp_prog, reg_both(dst + 1), reg_a(dst + 1),
+			 SHF_OP_NONE, reg_b(dst), SHF_SC_R_DSHF,
+			 32 - shift_amt);
+		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
+			 reg_b(dst), SHF_SC_L_SHF, shift_amt);
+	} else if (shift_amt == 32) {
+		wrp_reg_mov(nfp_prog, dst + 1, dst);
+		wrp_immed(nfp_prog, reg_both(dst), 0);
+	} else if (shift_amt > 32) {
+		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
+			 reg_b(dst), SHF_SC_L_SHF, shift_amt - 32);
+		wrp_immed(nfp_prog, reg_both(dst), 0);
+	}
+
+	return 0;
+}
+
 static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
 	const struct bpf_insn *insn = &meta->insn;
 	u8 dst = insn->dst_reg * 2;
 
-	if (insn->imm < 32) {
-		emit_shf(nfp_prog, reg_both(dst + 1),
-			 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst),
-			 SHF_SC_R_DSHF, 32 - insn->imm);
-		emit_shf(nfp_prog, reg_both(dst),
-			 reg_none(), SHF_OP_NONE, reg_b(dst),
-			 SHF_SC_L_SHF, insn->imm);
-	} else if (insn->imm == 32) {
-		wrp_reg_mov(nfp_prog, dst + 1, dst);
-		wrp_immed(nfp_prog, reg_both(dst), 0);
-	} else if (insn->imm > 32) {
-		emit_shf(nfp_prog, reg_both(dst + 1),
-			 reg_none(), SHF_OP_NONE, reg_b(dst),
-			 SHF_SC_L_SHF, insn->imm - 32);
-		wrp_immed(nfp_prog, reg_both(dst), 0);
+	return __shl_imm64(nfp_prog, dst, insn->imm);
+}
+
+static void shl_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	emit_alu(nfp_prog, imm_both(nfp_prog), reg_imm(32), ALU_OP_SUB,
+		 reg_b(src));
+	emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR, reg_imm(0));
+	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), SHF_OP_NONE,
+		       reg_b(dst), SHF_SC_R_DSHF);
+}
+
+/* NOTE: for indirect left shift, HIGH part should be calculated first. */
+static void shl_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
+	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
+		       reg_b(dst), SHF_SC_L_SHF);
+}
+
+static void shl_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	shl_reg64_lt32_high(nfp_prog, dst, src);
+	shl_reg64_lt32_low(nfp_prog, dst, src);
+}
+
+static void shl_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
+	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
+		       reg_b(dst), SHF_SC_L_SHF);
+	wrp_immed(nfp_prog, reg_both(dst), 0);
+}
+
+static int shl_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u64 umin, umax;
+	u8 dst, src;
+
+	dst = insn->dst_reg * 2;
+	umin = meta->umin;
+	umax = meta->umax;
+	if (umin == umax)
+		return __shl_imm64(nfp_prog, dst, umin);
+
+	src = insn->src_reg * 2;
+	if (umax < 32) {
+		shl_reg64_lt32(nfp_prog, dst, src);
+	} else if (umin >= 32) {
+		shl_reg64_ge32(nfp_prog, dst, src);
+	} else {
+		/* Generate different instruction sequences depending on runtime
+		 * value of shift amount.
+		 */
+		u16 label_ge32, label_end;
+
+		label_ge32 = nfp_prog_current_offset(nfp_prog) + 7;
+		emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
+
+		shl_reg64_lt32_high(nfp_prog, dst, src);
+		label_end = nfp_prog_current_offset(nfp_prog) + 6;
+		emit_br(nfp_prog, BR_UNC, label_end, 2);
+		/* shl_reg64_lt32_low packed in delay slot. */
+		shl_reg64_lt32_low(nfp_prog, dst, src);
+
+		if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
+			return -EINVAL;
+		shl_reg64_ge32(nfp_prog, dst, src);
+
+		if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* Pseudo code:
+ *   if shift_amt >= 32
+ *     dst_high = 0;
+ *     dst_low = dst_high >> shift_amt[4:0]
+ *   else
+ *     dst_high = dst_high >> shift_amt
+ *     dst_low = (dst_high, dst_low) >> shift_amt
+ *
+ * The indirect shift will use the same logic at runtime.
+ */
+static int __shr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
+{
+	if (shift_amt < 32) {
+		emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
+			 reg_b(dst), SHF_SC_R_DSHF, shift_amt);
+		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
+			 reg_b(dst + 1), SHF_SC_R_SHF, shift_amt);
+	} else if (shift_amt == 32) {
+		wrp_reg_mov(nfp_prog, dst, dst + 1);
+		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+	} else if (shift_amt > 32) {
+		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
+			 reg_b(dst + 1), SHF_SC_R_SHF, shift_amt - 32);
+		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
 	}
 
 	return 0;
@@ -1574,21 +1842,186 @@ static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	const struct bpf_insn *insn = &meta->insn;
 	u8 dst = insn->dst_reg * 2;
 
-	if (insn->imm < 32) {
-		emit_shf(nfp_prog, reg_both(dst),
-			 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst),
-			 SHF_SC_R_DSHF, insn->imm);
-		emit_shf(nfp_prog, reg_both(dst + 1),
-			 reg_none(), SHF_OP_NONE, reg_b(dst + 1),
-			 SHF_SC_R_SHF, insn->imm);
-	} else if (insn->imm == 32) {
+	return __shr_imm64(nfp_prog, dst, insn->imm);
+}
+
+/* NOTE: for indirect right shift, LOW part should be calculated first. */
+static void shr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
+	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
+		       reg_b(dst + 1), SHF_SC_R_SHF);
+}
+
+static void shr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
+	emit_shf_indir(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
+		       reg_b(dst), SHF_SC_R_DSHF);
+}
+
+static void shr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	shr_reg64_lt32_low(nfp_prog, dst, src);
+	shr_reg64_lt32_high(nfp_prog, dst, src);
+}
+
+static void shr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
+	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
+		       reg_b(dst + 1), SHF_SC_R_SHF);
+	wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+}
+
+static int shr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u64 umin, umax;
+	u8 dst, src;
+
+	dst = insn->dst_reg * 2;
+	umin = meta->umin;
+	umax = meta->umax;
+	if (umin == umax)
+		return __shr_imm64(nfp_prog, dst, umin);
+
+	src = insn->src_reg * 2;
+	if (umax < 32) {
+		shr_reg64_lt32(nfp_prog, dst, src);
+	} else if (umin >= 32) {
+		shr_reg64_ge32(nfp_prog, dst, src);
+	} else {
+		/* Generate different instruction sequences depending on runtime
+		 * value of shift amount.
+		 */
+		u16 label_ge32, label_end;
+
+		label_ge32 = nfp_prog_current_offset(nfp_prog) + 6;
+		emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
+		shr_reg64_lt32_low(nfp_prog, dst, src);
+		label_end = nfp_prog_current_offset(nfp_prog) + 6;
+		emit_br(nfp_prog, BR_UNC, label_end, 2);
+		/* shr_reg64_lt32_high packed in delay slot. */
+		shr_reg64_lt32_high(nfp_prog, dst, src);
+
+		if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
+			return -EINVAL;
+		shr_reg64_ge32(nfp_prog, dst, src);
+
+		if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* Code logic is the same as __shr_imm64 except ashr requires signedness bit
+ * told through PREV_ALU result.
+ */
+static int __ashr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
+{
+	if (shift_amt < 32) {
+		emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
+			 reg_b(dst), SHF_SC_R_DSHF, shift_amt);
+		/* Set signedness bit. */
+		emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR,
+			 reg_imm(0));
+		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
+			 reg_b(dst + 1), SHF_SC_R_SHF, shift_amt);
+	} else if (shift_amt == 32) {
+		/* NOTE: this also helps setting signedness bit. */
 		wrp_reg_mov(nfp_prog, dst, dst + 1);
-		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
-	} else if (insn->imm > 32) {
-		emit_shf(nfp_prog, reg_both(dst),
-			 reg_none(), SHF_OP_NONE, reg_b(dst + 1),
-			 SHF_SC_R_SHF, insn->imm - 32);
-		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
+			 reg_b(dst + 1), SHF_SC_R_SHF, 31);
+	} else if (shift_amt > 32) {
+		emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR,
+			 reg_imm(0));
+		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
+			 reg_b(dst + 1), SHF_SC_R_SHF, shift_amt - 32);
+		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
+			 reg_b(dst + 1), SHF_SC_R_SHF, 31);
+	}
+
+	return 0;
+}
+
+static int ashr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u8 dst = insn->dst_reg * 2;
+
+	return __ashr_imm64(nfp_prog, dst, insn->imm);
+}
+
+static void ashr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	/* NOTE: the first insn will set both indirect shift amount (source A)
+	 * and signedness bit (MSB of result).
+	 */
+	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1));
+	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
+		       reg_b(dst + 1), SHF_SC_R_SHF);
+}
+
+static void ashr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	/* NOTE: it is the same as logic shift because we don't need to shift in
+	 * signedness bit when the shift amount is less than 32.
+	 */
+	return shr_reg64_lt32_low(nfp_prog, dst, src);
+}
+
+static void ashr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	ashr_reg64_lt32_low(nfp_prog, dst, src);
+	ashr_reg64_lt32_high(nfp_prog, dst, src);
+}
+
+static void ashr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
+{
+	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1));
+	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
+		       reg_b(dst + 1), SHF_SC_R_SHF);
+	emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
+		 reg_b(dst + 1), SHF_SC_R_SHF, 31);
+}
+
+/* Like ashr_imm64, but need to use indirect shift. */
+static int ashr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u64 umin, umax;
+	u8 dst, src;
+
+	dst = insn->dst_reg * 2;
+	umin = meta->umin;
+	umax = meta->umax;
+	if (umin == umax)
+		return __ashr_imm64(nfp_prog, dst, umin);
+
+	src = insn->src_reg * 2;
+	if (umax < 32) {
+		ashr_reg64_lt32(nfp_prog, dst, src);
+	} else if (umin >= 32) {
+		ashr_reg64_ge32(nfp_prog, dst, src);
+	} else {
+		u16 label_ge32, label_end;
+
+		label_ge32 = nfp_prog_current_offset(nfp_prog) + 6;
+		emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
+		ashr_reg64_lt32_low(nfp_prog, dst, src);
+		label_end = nfp_prog_current_offset(nfp_prog) + 6;
+		emit_br(nfp_prog, BR_UNC, label_end, 2);
+		/* ashr_reg64_lt32_high packed in delay slot. */
+		ashr_reg64_lt32_high(nfp_prog, dst, src);
+
+		if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
+			return -EINVAL;
+		ashr_reg64_ge32(nfp_prog, dst, src);
+
+		if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
+			return -EINVAL;
 	}
 
 	return 0;
@@ -2108,6 +2541,17 @@ mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 			    false, wrp_lmem_store);
 }
 
+static int mem_stx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	switch (meta->insn.off) {
+	case offsetof(struct xdp_md, rx_queue_index):
+		return nfp_queue_select(nfp_prog, meta);
+	}
+
+	WARN_ON_ONCE(1); /* verifier should have rejected bad accesses */
+	return -EOPNOTSUPP;
+}
+
 static int
 mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 	unsigned int size)
@@ -2134,6 +2578,9 @@ static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 
 static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
+	if (meta->ptr.type == PTR_TO_CTX)
+		if (nfp_prog->type == BPF_PROG_TYPE_XDP)
+			return mem_stx_xdp(nfp_prog, meta);
 	return mem_stx(nfp_prog, meta, 4);
 }
 
@@ -2283,46 +2730,6 @@ static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	return 0;
 }
 
-static int jgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_imm(nfp_prog, meta, BR_BLO, true);
-}
-
-static int jge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_imm(nfp_prog, meta, BR_BHS, false);
-}
-
-static int jlt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_imm(nfp_prog, meta, BR_BLO, false);
-}
-
-static int jle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true);
-}
-
-static int jsgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_imm(nfp_prog, meta, BR_BLT, true);
-}
-
-static int jsge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_imm(nfp_prog, meta, BR_BGE, false);
-}
-
-static int jslt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_imm(nfp_prog, meta, BR_BLT, false);
-}
-
-static int jsle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_imm(nfp_prog, meta, BR_BGE, true);
-}
-
 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
 	const struct bpf_insn *insn = &meta->insn;
@@ -2392,46 +2799,6 @@ static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	return 0;
 }
 
-static int jgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_reg(nfp_prog, meta, BR_BLO, true);
-}
-
-static int jge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_reg(nfp_prog, meta, BR_BHS, false);
-}
-
-static int jlt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_reg(nfp_prog, meta, BR_BLO, false);
-}
-
-static int jle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true);
-}
-
-static int jsgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_reg(nfp_prog, meta, BR_BLT, true);
-}
-
-static int jsge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_reg(nfp_prog, meta, BR_BGE, false);
-}
-
-static int jslt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_reg(nfp_prog, meta, BR_BLT, false);
-}
-
-static int jsle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return wrp_cmp_reg(nfp_prog, meta, BR_BGE, true);
-}
-
 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
 	return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE);
@@ -2453,6 +2820,8 @@ static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 		return map_call_stack_common(nfp_prog, meta);
 	case BPF_FUNC_get_prandom_u32:
 		return nfp_get_prandom_u32(nfp_prog, meta);
+	case BPF_FUNC_perf_event_output:
+		return nfp_perf_event_output(nfp_prog, meta);
 	default:
 		WARN_ONCE(1, "verifier allowed unsupported function\n");
 		return -EOPNOTSUPP;
@@ -2480,8 +2849,12 @@ static const instr_cb_t instr_cb[256] = {
 	[BPF_ALU64 | BPF_SUB | BPF_X] =	sub_reg64,
 	[BPF_ALU64 | BPF_SUB | BPF_K] =	sub_imm64,
 	[BPF_ALU64 | BPF_NEG] =		neg_reg64,
+	[BPF_ALU64 | BPF_LSH | BPF_X] =	shl_reg64,
 	[BPF_ALU64 | BPF_LSH | BPF_K] =	shl_imm64,
+	[BPF_ALU64 | BPF_RSH | BPF_X] =	shr_reg64,
 	[BPF_ALU64 | BPF_RSH | BPF_K] =	shr_imm64,
+	[BPF_ALU64 | BPF_ARSH | BPF_X] = ashr_reg64,
+	[BPF_ALU64 | BPF_ARSH | BPF_K] = ashr_imm64,
 	[BPF_ALU | BPF_MOV | BPF_X] =	mov_reg,
 	[BPF_ALU | BPF_MOV | BPF_K] =	mov_imm,
 	[BPF_ALU | BPF_XOR | BPF_X] =	xor_reg,
@@ -2520,25 +2893,25 @@ static const instr_cb_t instr_cb[256] = {
 	[BPF_ST | BPF_MEM | BPF_DW] =	mem_st8,
 	[BPF_JMP | BPF_JA | BPF_K] =	jump,
 	[BPF_JMP | BPF_JEQ | BPF_K] =	jeq_imm,
-	[BPF_JMP | BPF_JGT | BPF_K] =	jgt_imm,
-	[BPF_JMP | BPF_JGE | BPF_K] =	jge_imm,
-	[BPF_JMP | BPF_JLT | BPF_K] =	jlt_imm,
-	[BPF_JMP | BPF_JLE | BPF_K] =	jle_imm,
-	[BPF_JMP | BPF_JSGT | BPF_K] =  jsgt_imm,
-	[BPF_JMP | BPF_JSGE | BPF_K] =  jsge_imm,
-	[BPF_JMP | BPF_JSLT | BPF_K] =  jslt_imm,
-	[BPF_JMP | BPF_JSLE | BPF_K] =  jsle_imm,
+	[BPF_JMP | BPF_JGT | BPF_K] =	cmp_imm,
+	[BPF_JMP | BPF_JGE | BPF_K] =	cmp_imm,
+	[BPF_JMP | BPF_JLT | BPF_K] =	cmp_imm,
+	[BPF_JMP | BPF_JLE | BPF_K] =	cmp_imm,
+	[BPF_JMP | BPF_JSGT | BPF_K] =  cmp_imm,
+	[BPF_JMP | BPF_JSGE | BPF_K] =  cmp_imm,
+	[BPF_JMP | BPF_JSLT | BPF_K] =  cmp_imm,
+	[BPF_JMP | BPF_JSLE | BPF_K] =  cmp_imm,
 	[BPF_JMP | BPF_JSET | BPF_K] =	jset_imm,
 	[BPF_JMP | BPF_JNE | BPF_K] =	jne_imm,
 	[BPF_JMP | BPF_JEQ | BPF_X] =	jeq_reg,
-	[BPF_JMP | BPF_JGT | BPF_X] =	jgt_reg,
-	[BPF_JMP | BPF_JGE | BPF_X] =	jge_reg,
-	[BPF_JMP | BPF_JLT | BPF_X] =	jlt_reg,
-	[BPF_JMP | BPF_JLE | BPF_X] =	jle_reg,
-	[BPF_JMP | BPF_JSGT | BPF_X] =  jsgt_reg,
-	[BPF_JMP | BPF_JSGE | BPF_X] =  jsge_reg,
-	[BPF_JMP | BPF_JSLT | BPF_X] =  jslt_reg,
-	[BPF_JMP | BPF_JSLE | BPF_X] =  jsle_reg,
+	[BPF_JMP | BPF_JGT | BPF_X] =	cmp_reg,
+	[BPF_JMP | BPF_JGE | BPF_X] =	cmp_reg,
+	[BPF_JMP | BPF_JLT | BPF_X] =	cmp_reg,
+	[BPF_JMP | BPF_JLE | BPF_X] =	cmp_reg,
+	[BPF_JMP | BPF_JSGT | BPF_X] =  cmp_reg,
+	[BPF_JMP | BPF_JSGE | BPF_X] =  cmp_reg,
+	[BPF_JMP | BPF_JSLT | BPF_X] =  cmp_reg,
+	[BPF_JMP | BPF_JSLE | BPF_X] =  cmp_reg,
 	[BPF_JMP | BPF_JSET | BPF_X] =	jset_reg,
 	[BPF_JMP | BPF_JNE | BPF_X] =	jne_reg,
 	[BPF_JMP | BPF_CALL] =		call,
@@ -2777,6 +3150,54 @@ static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog)
 	}
 }
 
+/* abs(insn.imm) will fit better into unrestricted reg immediate -
+ * convert add/sub of a negative number into a sub/add of a positive one.
+ */
+static void nfp_bpf_opt_neg_add_sub(struct nfp_prog *nfp_prog)
+{
+	struct nfp_insn_meta *meta;
+
+	list_for_each_entry(meta, &nfp_prog->insns, l) {
+		struct bpf_insn insn = meta->insn;
+
+		if (meta->skip)
+			continue;
+
+		if (BPF_CLASS(insn.code) != BPF_ALU &&
+		    BPF_CLASS(insn.code) != BPF_ALU64 &&
+		    BPF_CLASS(insn.code) != BPF_JMP)
+			continue;
+		if (BPF_SRC(insn.code) != BPF_K)
+			continue;
+		if (insn.imm >= 0)
+			continue;
+
+		if (BPF_CLASS(insn.code) == BPF_JMP) {
+			switch (BPF_OP(insn.code)) {
+			case BPF_JGE:
+			case BPF_JSGE:
+			case BPF_JLT:
+			case BPF_JSLT:
+				meta->jump_neg_op = true;
+				break;
+			default:
+				continue;
+			}
+		} else {
+			if (BPF_OP(insn.code) == BPF_ADD)
+				insn.code = BPF_CLASS(insn.code) | BPF_SUB;
+			else if (BPF_OP(insn.code) == BPF_SUB)
+				insn.code = BPF_CLASS(insn.code) | BPF_ADD;
+			else
+				continue;
+
+			meta->insn.code = insn.code | BPF_K;
+		}
+
+		meta->insn.imm = -insn.imm;
+	}
+}
+
 /* Remove masking after load since our load guarantees this is not needed */
 static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog)
 {
@@ -3212,6 +3633,7 @@ static int nfp_bpf_optimize(struct nfp_prog *nfp_prog)
 {
 	nfp_bpf_opt_reg_init(nfp_prog);
 
+	nfp_bpf_opt_neg_add_sub(nfp_prog);
 	nfp_bpf_opt_ld_mask(nfp_prog);
 	nfp_bpf_opt_ld_shift(nfp_prog);
 	nfp_bpf_opt_ldst_gather(nfp_prog);
@@ -3220,6 +3642,33 @@ static int nfp_bpf_optimize(struct nfp_prog *nfp_prog)
 	return 0;
 }
 
+static int nfp_bpf_replace_map_ptrs(struct nfp_prog *nfp_prog)
+{
+	struct nfp_insn_meta *meta1, *meta2;
+	struct nfp_bpf_map *nfp_map;
+	struct bpf_map *map;
+
+	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
+		if (meta1->skip || meta2->skip)
+			continue;
+
+		if (meta1->insn.code != (BPF_LD | BPF_IMM | BPF_DW) ||
+		    meta1->insn.src_reg != BPF_PSEUDO_MAP_FD)
+			continue;
+
+		map = (void *)(unsigned long)((u32)meta1->insn.imm |
+					      (u64)meta2->insn.imm << 32);
+		if (bpf_map_offload_neutral(map))
+			continue;
+		nfp_map = map_to_offmap(map)->dev_priv;
+
+		meta1->insn.imm = nfp_map->tid;
+		meta2->insn.imm = 0;
+	}
+
+	return 0;
+}
+
 static int nfp_bpf_ustore_calc(u64 *prog, unsigned int len)
 {
 	__le64 *ustore = (__force __le64 *)prog;
@@ -3256,6 +3705,10 @@ int nfp_bpf_jit(struct nfp_prog *nfp_prog)
 {
 	int ret;
 
+	ret = nfp_bpf_replace_map_ptrs(nfp_prog);
+	if (ret)
+		return ret;
+
 	ret = nfp_bpf_optimize(nfp_prog);
 	if (ret)
 		return ret;
@@ -3346,6 +3799,9 @@ void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv)
 			case BPF_FUNC_map_delete_elem:
 				val = nfp_prog->bpf->helpers.map_delete;
 				break;
+			case BPF_FUNC_perf_event_output:
+				val = nfp_prog->bpf->helpers.perf_event_output;
+				break;
 			default:
 				pr_err("relocation of unknown helper %d\n",
 				       val);
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.c b/drivers/net/ethernet/netronome/nfp/bpf/main.c
index 35fb31f682af..fcdfb8e7fdea 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/main.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/main.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2017 Netronome Systems, Inc.
+ * Copyright (C) 2017-2018 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -43,6 +43,14 @@
 #include "fw.h"
 #include "main.h"
 
+const struct rhashtable_params nfp_bpf_maps_neutral_params = {
+	.nelem_hint		= 4,
+	.key_len		= FIELD_SIZEOF(struct nfp_bpf_neutral_map, ptr),
+	.key_offset		= offsetof(struct nfp_bpf_neutral_map, ptr),
+	.head_offset		= offsetof(struct nfp_bpf_neutral_map, l),
+	.automatic_shrinking	= true,
+};
+
 static bool nfp_net_ebpf_capable(struct nfp_net *nn)
 {
 #ifdef __LITTLE_ENDIAN
@@ -290,6 +298,9 @@ nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
 	case BPF_FUNC_map_delete_elem:
 		bpf->helpers.map_delete = readl(&cap->func_addr);
 		break;
+	case BPF_FUNC_perf_event_output:
+		bpf->helpers.perf_event_output = readl(&cap->func_addr);
+		break;
 	}
 
 	return 0;
@@ -323,6 +334,13 @@ nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value,
 	return 0;
 }
 
+static int
+nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
+{
+	bpf->queue_select = true;
+	return 0;
+}
+
 static int nfp_bpf_parse_capabilities(struct nfp_app *app)
 {
 	struct nfp_cpp *cpp = app->pf->cpp;
@@ -365,6 +383,10 @@ static int nfp_bpf_parse_capabilities(struct nfp_app *app)
 			if (nfp_bpf_parse_cap_random(app->priv, value, length))
 				goto err_release_free;
 			break;
+		case NFP_BPF_CAP_TYPE_QUEUE_SELECT:
+			if (nfp_bpf_parse_cap_qsel(app->priv, value, length))
+				goto err_release_free;
+			break;
 		default:
 			nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
 			break;
@@ -401,17 +423,28 @@ static int nfp_bpf_init(struct nfp_app *app)
 	init_waitqueue_head(&bpf->cmsg_wq);
 	INIT_LIST_HEAD(&bpf->map_list);
 
-	err = nfp_bpf_parse_capabilities(app);
+	err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params);
 	if (err)
 		goto err_free_bpf;
 
+	err = nfp_bpf_parse_capabilities(app);
+	if (err)
+		goto err_free_neutral_maps;
+
 	return 0;
 
+err_free_neutral_maps:
+	rhashtable_destroy(&bpf->maps_neutral);
 err_free_bpf:
 	kfree(bpf);
 	return err;
 }
 
+static void nfp_check_rhashtable_empty(void *ptr, void *arg)
+{
+	WARN_ON_ONCE(1);
+}
+
 static void nfp_bpf_clean(struct nfp_app *app)
 {
 	struct nfp_app_bpf *bpf = app->priv;
@@ -419,6 +452,8 @@ static void nfp_bpf_clean(struct nfp_app *app)
 	WARN_ON(!skb_queue_empty(&bpf->cmsg_replies));
 	WARN_ON(!list_empty(&bpf->map_list));
 	WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
+	rhashtable_free_and_destroy(&bpf->maps_neutral,
+				    nfp_check_rhashtable_empty, NULL);
 	kfree(bpf);
 }
 
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.h b/drivers/net/ethernet/netronome/nfp/bpf/main.h
index 4981c8944ca3..654fe7823e5e 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/main.h
+++ b/drivers/net/ethernet/netronome/nfp/bpf/main.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016-2017 Netronome Systems, Inc.
+ * Copyright (C) 2016-2018 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -39,6 +39,7 @@
 #include <linux/bpf_verifier.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
+#include <linux/rhashtable.h>
 #include <linux/skbuff.h>
 #include <linux/types.h>
 #include <linux/wait.h>
@@ -81,10 +82,16 @@ enum static_regs {
 enum pkt_vec {
 	PKT_VEC_PKT_LEN		= 0,
 	PKT_VEC_PKT_PTR		= 2,
+	PKT_VEC_QSEL_SET	= 4,
+	PKT_VEC_QSEL_VAL	= 6,
 };
 
+#define PKT_VEL_QSEL_SET_BIT	4
+
 #define pv_len(np)	reg_lm(1, PKT_VEC_PKT_LEN)
 #define pv_ctm_ptr(np)	reg_lm(1, PKT_VEC_PKT_PTR)
+#define pv_qsel_set(np)	reg_lm(1, PKT_VEC_QSEL_SET)
+#define pv_qsel_val(np)	reg_lm(1, PKT_VEC_QSEL_VAL)
 
 #define stack_reg(np)	reg_a(STATIC_REG_STACK)
 #define stack_imm(np)	imm_b(np)
@@ -114,6 +121,8 @@ enum pkt_vec {
  * @maps_in_use:	number of currently offloaded maps
  * @map_elems_in_use:	number of elements allocated to offloaded maps
  *
+ * @maps_neutral:	hash table of offload-neutral maps (on pointer)
+ *
  * @adjust_head:	adjust head capability
  * @adjust_head.flags:		extra flags for adjust head
  * @adjust_head.off_min:	minimal packet offset within buffer required
@@ -133,8 +142,10 @@ enum pkt_vec {
  * @helpers.map_lookup:		map lookup helper address
  * @helpers.map_update:		map update helper address
  * @helpers.map_delete:		map delete helper address
+ * @helpers.perf_event_output:	output perf event to a ring buffer
  *
  * @pseudo_random:	FW initialized the pseudo-random machinery (CSRs)
+ * @queue_select:	BPF can set the RX queue ID in packet vector
  */
 struct nfp_app_bpf {
 	struct nfp_app *app;
@@ -150,6 +161,8 @@ struct nfp_app_bpf {
 	unsigned int maps_in_use;
 	unsigned int map_elems_in_use;
 
+	struct rhashtable maps_neutral;
+
 	struct nfp_bpf_cap_adjust_head {
 		u32 flags;
 		int off_min;
@@ -171,9 +184,11 @@ struct nfp_app_bpf {
 		u32 map_lookup;
 		u32 map_update;
 		u32 map_delete;
+		u32 perf_event_output;
 	} helpers;
 
 	bool pseudo_random;
+	bool queue_select;
 };
 
 enum nfp_bpf_map_use {
@@ -199,6 +214,14 @@ struct nfp_bpf_map {
 	enum nfp_bpf_map_use use_map[];
 };
 
+struct nfp_bpf_neutral_map {
+	struct rhash_head l;
+	struct bpf_map *ptr;
+	u32 count;
+};
+
+extern const struct rhashtable_params nfp_bpf_maps_neutral_params;
+
 struct nfp_prog;
 struct nfp_insn_meta;
 typedef int (*instr_cb_t)(struct nfp_prog *, struct nfp_insn_meta *);
@@ -236,9 +259,12 @@ struct nfp_bpf_reg_state {
  * @xadd_over_16bit: 16bit immediate is not guaranteed
  * @xadd_maybe_16bit: 16bit immediate is possible
  * @jmp_dst: destination info for jump instructions
+ * @jump_neg_op: jump instruction has inverted immediate, use ADD instead of SUB
  * @func_id: function id for call instructions
  * @arg1: arg1 for call instructions
  * @arg2: arg2 for call instructions
+ * @umin: copy of core verifier umin_value.
+ * @umax: copy of core verifier umax_value.
  * @off: index of first generated machine instruction (in nfp_prog.prog)
  * @n: eBPF instruction number
  * @flags: eBPF instruction extra optimization flags
@@ -264,13 +290,23 @@ struct nfp_insn_meta {
 			bool xadd_maybe_16bit;
 		};
 		/* jump */
-		struct nfp_insn_meta *jmp_dst;
+		struct {
+			struct nfp_insn_meta *jmp_dst;
+			bool jump_neg_op;
+		};
 		/* function calls */
 		struct {
 			u32 func_id;
 			struct bpf_reg_state arg1;
 			struct nfp_bpf_reg_state arg2;
 		};
+		/* We are interested in range info for some operands,
+		 * for example, the shift amount.
+		 */
+		struct {
+			u64 umin;
+			u64 umax;
+		};
 	};
 	unsigned int off;
 	unsigned short n;
@@ -348,6 +384,25 @@ static inline bool is_mbpf_xadd(const struct nfp_insn_meta *meta)
 	return (meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_XADD);
 }
 
+static inline bool is_mbpf_indir_shift(const struct nfp_insn_meta *meta)
+{
+	u8 code = meta->insn.code;
+	bool is_alu, is_shift;
+	u8 opclass, opcode;
+
+	opclass = BPF_CLASS(code);
+	is_alu = opclass == BPF_ALU64 || opclass == BPF_ALU;
+	if (!is_alu)
+		return false;
+
+	opcode = BPF_OP(code);
+	is_shift = opcode == BPF_LSH || opcode == BPF_RSH || opcode == BPF_ARSH;
+	if (!is_shift)
+		return false;
+
+	return BPF_SRC(code) == BPF_X;
+}
+
 /**
  * struct nfp_prog - nfp BPF program
  * @bpf: backpointer to the bpf app priv structure
@@ -363,6 +418,8 @@ static inline bool is_mbpf_xadd(const struct nfp_insn_meta *meta)
  * @error: error code if something went wrong
  * @stack_depth: max stack depth from the verifier
  * @adjust_head_location: if program has single adjust head call - the insn no.
+ * @map_records_cnt: the number of map pointers recorded for this prog
+ * @map_records: the map record pointers from bpf->maps_neutral
  * @insns: list of BPF instruction wrappers (struct nfp_insn_meta)
  */
 struct nfp_prog {
@@ -386,6 +443,9 @@ struct nfp_prog {
 	unsigned int stack_depth;
 	unsigned int adjust_head_location;
 
+	unsigned int map_records_cnt;
+	struct nfp_bpf_neutral_map **map_records;
+
 	struct list_head insns;
 };
 
@@ -436,5 +496,7 @@ int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap,
 int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap,
 			       void *key, void *next_key);
 
+int nfp_bpf_event_output(struct nfp_app_bpf *bpf, struct sk_buff *skb);
+
 void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb);
 #endif
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/offload.c b/drivers/net/ethernet/netronome/nfp/bpf/offload.c
index 42d98792bd25..7eae4c0266f8 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/offload.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/offload.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016-2017 Netronome Systems, Inc.
+ * Copyright (C) 2016-2018 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -56,6 +56,126 @@
 #include "../nfp_net_ctrl.h"
 #include "../nfp_net.h"
 
+static int
+nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
+		   struct bpf_map *map)
+{
+	struct nfp_bpf_neutral_map *record;
+	int err;
+
+	/* Map record paths are entered via ndo, update side is protected. */
+	ASSERT_RTNL();
+
+	/* Reuse path - other offloaded program is already tracking this map. */
+	record = rhashtable_lookup_fast(&bpf->maps_neutral, &map,
+					nfp_bpf_maps_neutral_params);
+	if (record) {
+		nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
+		record->count++;
+		return 0;
+	}
+
+	/* Grab a single ref to the map for our record.  The prog destroy ndo
+	 * happens after free_used_maps().
+	 */
+	map = bpf_map_inc(map, false);
+	if (IS_ERR(map))
+		return PTR_ERR(map);
+
+	record = kmalloc(sizeof(*record), GFP_KERNEL);
+	if (!record) {
+		err = -ENOMEM;
+		goto err_map_put;
+	}
+
+	record->ptr = map;
+	record->count = 1;
+
+	err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l,
+				     nfp_bpf_maps_neutral_params);
+	if (err)
+		goto err_free_rec;
+
+	nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
+
+	return 0;
+
+err_free_rec:
+	kfree(record);
+err_map_put:
+	bpf_map_put(map);
+	return err;
+}
+
+static void
+nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog)
+{
+	bool freed = false;
+	int i;
+
+	ASSERT_RTNL();
+
+	for (i = 0; i < nfp_prog->map_records_cnt; i++) {
+		if (--nfp_prog->map_records[i]->count) {
+			nfp_prog->map_records[i] = NULL;
+			continue;
+		}
+
+		WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral,
+					       &nfp_prog->map_records[i]->l,
+					       nfp_bpf_maps_neutral_params));
+		freed = true;
+	}
+
+	if (freed) {
+		synchronize_rcu();
+
+		for (i = 0; i < nfp_prog->map_records_cnt; i++)
+			if (nfp_prog->map_records[i]) {
+				bpf_map_put(nfp_prog->map_records[i]->ptr);
+				kfree(nfp_prog->map_records[i]);
+			}
+	}
+
+	kfree(nfp_prog->map_records);
+	nfp_prog->map_records = NULL;
+	nfp_prog->map_records_cnt = 0;
+}
+
+static int
+nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
+		    struct bpf_prog *prog)
+{
+	int i, cnt, err;
+
+	/* Quickly count the maps we will have to remember */
+	cnt = 0;
+	for (i = 0; i < prog->aux->used_map_cnt; i++)
+		if (bpf_map_offload_neutral(prog->aux->used_maps[i]))
+			cnt++;
+	if (!cnt)
+		return 0;
+
+	nfp_prog->map_records = kmalloc_array(cnt,
+					      sizeof(nfp_prog->map_records[0]),
+					      GFP_KERNEL);
+	if (!nfp_prog->map_records)
+		return -ENOMEM;
+
+	for (i = 0; i < prog->aux->used_map_cnt; i++)
+		if (bpf_map_offload_neutral(prog->aux->used_maps[i])) {
+			err = nfp_map_ptr_record(bpf, nfp_prog,
+						 prog->aux->used_maps[i]);
+			if (err) {
+				nfp_map_ptrs_forget(bpf, nfp_prog);
+				return err;
+			}
+		}
+	WARN_ON(cnt != nfp_prog->map_records_cnt);
+
+	return 0;
+}
+
 static int
 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
 		 unsigned int cnt)
@@ -70,6 +190,8 @@ nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
 
 		meta->insn = prog[i];
 		meta->n = i;
+		if (is_mbpf_indir_shift(meta))
+			meta->umin = U64_MAX;
 
 		list_add_tail(&meta->l, &nfp_prog->insns);
 	}
@@ -151,7 +273,7 @@ static int nfp_bpf_translate(struct nfp_net *nn, struct bpf_prog *prog)
 	prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64);
 	prog->aux->offload->jited_image = nfp_prog->prog;
 
-	return 0;
+	return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog);
 }
 
 static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog)
@@ -159,6 +281,7 @@ static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog)
 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
 
 	kvfree(nfp_prog->prog);
+	nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog);
 	nfp_prog_free(nfp_prog);
 
 	return 0;
@@ -320,6 +443,53 @@ int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
 	}
 }
 
+static unsigned long
+nfp_bpf_perf_event_copy(void *dst, const void *src,
+			unsigned long off, unsigned long len)
+{
+	memcpy(dst, src + off, len);
+	return 0;
+}
+
+int nfp_bpf_event_output(struct nfp_app_bpf *bpf, struct sk_buff *skb)
+{
+	struct cmsg_bpf_event *cbe = (void *)skb->data;
+	u32 pkt_size, data_size;
+	struct bpf_map *map;
+
+	if (skb->len < sizeof(struct cmsg_bpf_event))
+		goto err_drop;
+
+	pkt_size = be32_to_cpu(cbe->pkt_size);
+	data_size = be32_to_cpu(cbe->data_size);
+	map = (void *)(unsigned long)be64_to_cpu(cbe->map_ptr);
+
+	if (skb->len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size)
+		goto err_drop;
+	if (cbe->hdr.ver != CMSG_MAP_ABI_VERSION)
+		goto err_drop;
+
+	rcu_read_lock();
+	if (!rhashtable_lookup_fast(&bpf->maps_neutral, &map,
+				    nfp_bpf_maps_neutral_params)) {
+		rcu_read_unlock();
+		pr_warn("perf event: dest map pointer %px not recognized, dropping event\n",
+			map);
+		goto err_drop;
+	}
+
+	bpf_event_output(map, be32_to_cpu(cbe->cpu_id),
+			 &cbe->data[round_up(pkt_size, 4)], data_size,
+			 cbe->data, pkt_size, nfp_bpf_perf_event_copy);
+	rcu_read_unlock();
+
+	dev_consume_skb_any(skb);
+	return 0;
+err_drop:
+	dev_kfree_skb_any(skb);
+	return -EINVAL;
+}
+
 static int
 nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog,
 		 struct netlink_ext_ack *extack)
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
index 06ad53ce4ad9..4bfeba7b21b2 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016-2017 Netronome Systems, Inc.
+ * Copyright (C) 2016-2018 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -36,6 +36,8 @@
 #include <linux/kernel.h>
 #include <linux/pkt_cls.h>
 
+#include "../nfp_app.h"
+#include "../nfp_main.h"
 #include "fw.h"
 #include "main.h"
 
@@ -149,15 +151,6 @@ nfp_bpf_map_call_ok(const char *fname, struct bpf_verifier_env *env,
 		return false;
 	}
 
-	/* Rest of the checks is only if we re-parse the same insn */
-	if (!meta->func_id)
-		return true;
-
-	if (meta->arg1.map_ptr != reg1->map_ptr) {
-		pr_vlog(env, "%s: called for different map\n", fname);
-		return false;
-	}
-
 	return true;
 }
 
@@ -216,6 +209,71 @@ nfp_bpf_check_call(struct nfp_prog *nfp_prog, struct bpf_verifier_env *env,
 		pr_vlog(env, "bpf_get_prandom_u32(): FW doesn't support random number generation\n");
 		return -EOPNOTSUPP;
 
+	case BPF_FUNC_perf_event_output:
+		BUILD_BUG_ON(NFP_BPF_SCALAR_VALUE != SCALAR_VALUE ||
+			     NFP_BPF_MAP_VALUE != PTR_TO_MAP_VALUE ||
+			     NFP_BPF_STACK != PTR_TO_STACK ||
+			     NFP_BPF_PACKET_DATA != PTR_TO_PACKET);
+
+		if (!bpf->helpers.perf_event_output) {
+			pr_vlog(env, "event_output: not supported by FW\n");
+			return -EOPNOTSUPP;
+		}
+
+		/* Force current CPU to make sure we can report the event
+		 * wherever we get the control message from FW.
+		 */
+		if (reg3->var_off.mask & BPF_F_INDEX_MASK ||
+		    (reg3->var_off.value & BPF_F_INDEX_MASK) !=
+		    BPF_F_CURRENT_CPU) {
+			char tn_buf[48];
+
+			tnum_strn(tn_buf, sizeof(tn_buf), reg3->var_off);
+			pr_vlog(env, "event_output: must use BPF_F_CURRENT_CPU, var_off: %s\n",
+				tn_buf);
+			return -EOPNOTSUPP;
+		}
+
+		/* Save space in meta, we don't care about arguments other
+		 * than 4th meta, shove it into arg1.
+		 */
+		reg1 = cur_regs(env) + BPF_REG_4;
+
+		if (reg1->type != SCALAR_VALUE /* NULL ptr */ &&
+		    reg1->type != PTR_TO_STACK &&
+		    reg1->type != PTR_TO_MAP_VALUE &&
+		    reg1->type != PTR_TO_PACKET) {
+			pr_vlog(env, "event_output: unsupported ptr type: %d\n",
+				reg1->type);
+			return -EOPNOTSUPP;
+		}
+
+		if (reg1->type == PTR_TO_STACK &&
+		    !nfp_bpf_stack_arg_ok("event_output", env, reg1, NULL))
+			return -EOPNOTSUPP;
+
+		/* Warn user that on offload NFP may return success even if map
+		 * is not going to accept the event, since the event output is
+		 * fully async and device won't know the state of the map.
+		 * There is also FW limitation on the event length.
+		 *
+		 * Lost events will not show up on the perf ring, driver
+		 * won't see them at all.  Events may also get reordered.
+		 */
+		dev_warn_once(&nfp_prog->bpf->app->pf->pdev->dev,
+			      "bpf: note: return codes and behavior of bpf_event_output() helper differs for offloaded programs!\n");
+		pr_vlog(env, "warning: return codes and behavior of event_output helper differ for offload!\n");
+
+		if (!meta->func_id)
+			break;
+
+		if (reg1->type != meta->arg1.type) {
+			pr_vlog(env, "event_output: ptr type changed: %d %d\n",
+				meta->arg1.type, reg1->type);
+			return -EINVAL;
+		}
+		break;
+
 	default:
 		pr_vlog(env, "unsupported function id: %d\n", func_id);
 		return -EOPNOTSUPP;
@@ -409,6 +467,30 @@ nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 	return 0;
 }
 
+static int
+nfp_bpf_check_store(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+		    struct bpf_verifier_env *env)
+{
+	const struct bpf_reg_state *reg = cur_regs(env) + meta->insn.dst_reg;
+
+	if (reg->type == PTR_TO_CTX) {
+		if (nfp_prog->type == BPF_PROG_TYPE_XDP) {
+			/* XDP ctx accesses must be 4B in size */
+			switch (meta->insn.off) {
+			case offsetof(struct xdp_md, rx_queue_index):
+				if (nfp_prog->bpf->queue_select)
+					goto exit_check_ptr;
+				pr_vlog(env, "queue selection not supported by FW\n");
+				return -EOPNOTSUPP;
+			}
+		}
+		pr_vlog(env, "unsupported store to context field\n");
+		return -EOPNOTSUPP;
+	}
+exit_check_ptr:
+	return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg);
+}
+
 static int
 nfp_bpf_check_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 		   struct bpf_verifier_env *env)
@@ -464,11 +546,19 @@ nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx)
 		return nfp_bpf_check_ptr(nfp_prog, meta, env,
 					 meta->insn.src_reg);
 	if (is_mbpf_store(meta))
-		return nfp_bpf_check_ptr(nfp_prog, meta, env,
-					 meta->insn.dst_reg);
+		return nfp_bpf_check_store(nfp_prog, meta, env);
+
 	if (is_mbpf_xadd(meta))
 		return nfp_bpf_check_xadd(nfp_prog, meta, env);
 
+	if (is_mbpf_indir_shift(meta)) {
+		const struct bpf_reg_state *sreg =
+			cur_regs(env) + meta->insn.src_reg;
+
+		meta->umin = min(meta->umin, sreg->umin_value);
+		meta->umax = max(meta->umax, sreg->umax_value);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/netronome/nfp/flower/action.c b/drivers/net/ethernet/netronome/nfp/flower/action.c
index 80df9a5d4217..4a6d2db75071 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/action.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/action.c
@@ -72,6 +72,42 @@ nfp_fl_push_vlan(struct nfp_fl_push_vlan *push_vlan,
 	push_vlan->vlan_tci = cpu_to_be16(tmp_push_vlan_tci);
 }
 
+static int
+nfp_fl_pre_lag(struct nfp_app *app, const struct tc_action *action,
+	       struct nfp_fl_payload *nfp_flow, int act_len)
+{
+	size_t act_size = sizeof(struct nfp_fl_pre_lag);
+	struct nfp_fl_pre_lag *pre_lag;
+	struct net_device *out_dev;
+	int err;
+
+	out_dev = tcf_mirred_dev(action);
+	if (!out_dev || !netif_is_lag_master(out_dev))
+		return 0;
+
+	if (act_len + act_size > NFP_FL_MAX_A_SIZ)
+		return -EOPNOTSUPP;
+
+	/* Pre_lag action must be first on action list.
+	 * If other actions already exist they need pushed forward.
+	 */
+	if (act_len)
+		memmove(nfp_flow->action_data + act_size,
+			nfp_flow->action_data, act_len);
+
+	pre_lag = (struct nfp_fl_pre_lag *)nfp_flow->action_data;
+	err = nfp_flower_lag_populate_pre_action(app, out_dev, pre_lag);
+	if (err)
+		return err;
+
+	pre_lag->head.jump_id = NFP_FL_ACTION_OPCODE_PRE_LAG;
+	pre_lag->head.len_lw = act_size >> NFP_FL_LW_SIZ;
+
+	nfp_flow->meta.shortcut = cpu_to_be32(NFP_FL_SC_ACT_NULL);
+
+	return act_size;
+}
+
 static bool nfp_fl_netdev_is_tunnel_type(struct net_device *out_dev,
 					 enum nfp_flower_tun_type tun_type)
 {
@@ -88,12 +124,13 @@ static bool nfp_fl_netdev_is_tunnel_type(struct net_device *out_dev,
 }
 
 static int
-nfp_fl_output(struct nfp_fl_output *output, const struct tc_action *action,
-	      struct nfp_fl_payload *nfp_flow, bool last,
-	      struct net_device *in_dev, enum nfp_flower_tun_type tun_type,
-	      int *tun_out_cnt)
+nfp_fl_output(struct nfp_app *app, struct nfp_fl_output *output,
+	      const struct tc_action *action, struct nfp_fl_payload *nfp_flow,
+	      bool last, struct net_device *in_dev,
+	      enum nfp_flower_tun_type tun_type, int *tun_out_cnt)
 {
 	size_t act_size = sizeof(struct nfp_fl_output);
+	struct nfp_flower_priv *priv = app->priv;
 	struct net_device *out_dev;
 	u16 tmp_flags;
 
@@ -118,6 +155,15 @@ nfp_fl_output(struct nfp_fl_output *output, const struct tc_action *action,
 		output->flags = cpu_to_be16(tmp_flags |
 					    NFP_FL_OUT_FLAGS_USE_TUN);
 		output->port = cpu_to_be32(NFP_FL_PORT_TYPE_TUN | tun_type);
+	} else if (netif_is_lag_master(out_dev) &&
+		   priv->flower_ext_feats & NFP_FL_FEATS_LAG) {
+		int gid;
+
+		output->flags = cpu_to_be16(tmp_flags);
+		gid = nfp_flower_lag_get_output_id(app, out_dev);
+		if (gid < 0)
+			return gid;
+		output->port = cpu_to_be32(NFP_FL_LAG_OUT | gid);
 	} else {
 		/* Set action output parameters. */
 		output->flags = cpu_to_be16(tmp_flags);
@@ -164,7 +210,7 @@ static struct nfp_fl_pre_tunnel *nfp_fl_pre_tunnel(char *act_data, int act_len)
 	struct nfp_fl_pre_tunnel *pre_tun_act;
 
 	/* Pre_tunnel action must be first on action list.
-	 * If other actions already exist they need pushed forward.
+	 * If other actions already exist they need to be pushed forward.
 	 */
 	if (act_len)
 		memmove(act_data + act_size, act_data, act_len);
@@ -443,42 +489,73 @@ nfp_fl_pedit(const struct tc_action *action, char *nfp_action, int *a_len)
 }
 
 static int
-nfp_flower_loop_action(const struct tc_action *a,
+nfp_flower_output_action(struct nfp_app *app, const struct tc_action *a,
+			 struct nfp_fl_payload *nfp_fl, int *a_len,
+			 struct net_device *netdev, bool last,
+			 enum nfp_flower_tun_type *tun_type, int *tun_out_cnt,
+			 int *out_cnt)
+{
+	struct nfp_flower_priv *priv = app->priv;
+	struct nfp_fl_output *output;
+	int err, prelag_size;
+
+	if (*a_len + sizeof(struct nfp_fl_output) > NFP_FL_MAX_A_SIZ)
+		return -EOPNOTSUPP;
+
+	output = (struct nfp_fl_output *)&nfp_fl->action_data[*a_len];
+	err = nfp_fl_output(app, output, a, nfp_fl, last, netdev, *tun_type,
+			    tun_out_cnt);
+	if (err)
+		return err;
+
+	*a_len += sizeof(struct nfp_fl_output);
+
+	if (priv->flower_ext_feats & NFP_FL_FEATS_LAG) {
+		/* nfp_fl_pre_lag returns -err or size of prelag action added.
+		 * This will be 0 if it is not egressing to a lag dev.
+		 */
+		prelag_size = nfp_fl_pre_lag(app, a, nfp_fl, *a_len);
+		if (prelag_size < 0)
+			return prelag_size;
+		else if (prelag_size > 0 && (!last || *out_cnt))
+			return -EOPNOTSUPP;
+
+		*a_len += prelag_size;
+	}
+	(*out_cnt)++;
+
+	return 0;
+}
+
+static int
+nfp_flower_loop_action(struct nfp_app *app, const struct tc_action *a,
 		       struct nfp_fl_payload *nfp_fl, int *a_len,
 		       struct net_device *netdev,
-		       enum nfp_flower_tun_type *tun_type, int *tun_out_cnt)
+		       enum nfp_flower_tun_type *tun_type, int *tun_out_cnt,
+		       int *out_cnt)
 {
 	struct nfp_fl_set_ipv4_udp_tun *set_tun;
 	struct nfp_fl_pre_tunnel *pre_tun;
 	struct nfp_fl_push_vlan *psh_v;
 	struct nfp_fl_pop_vlan *pop_v;
-	struct nfp_fl_output *output;
 	int err;
 
 	if (is_tcf_gact_shot(a)) {
 		nfp_fl->meta.shortcut = cpu_to_be32(NFP_FL_SC_ACT_DROP);
 	} else if (is_tcf_mirred_egress_redirect(a)) {
-		if (*a_len + sizeof(struct nfp_fl_output) > NFP_FL_MAX_A_SIZ)
-			return -EOPNOTSUPP;
-
-		output = (struct nfp_fl_output *)&nfp_fl->action_data[*a_len];
-		err = nfp_fl_output(output, a, nfp_fl, true, netdev, *tun_type,
-				    tun_out_cnt);
+		err = nfp_flower_output_action(app, a, nfp_fl, a_len, netdev,
+					       true, tun_type, tun_out_cnt,
+					       out_cnt);
 		if (err)
 			return err;
 
-		*a_len += sizeof(struct nfp_fl_output);
 	} else if (is_tcf_mirred_egress_mirror(a)) {
-		if (*a_len + sizeof(struct nfp_fl_output) > NFP_FL_MAX_A_SIZ)
-			return -EOPNOTSUPP;
-
-		output = (struct nfp_fl_output *)&nfp_fl->action_data[*a_len];
-		err = nfp_fl_output(output, a, nfp_fl, false, netdev, *tun_type,
-				    tun_out_cnt);
+		err = nfp_flower_output_action(app, a, nfp_fl, a_len, netdev,
+					       false, tun_type, tun_out_cnt,
+					       out_cnt);
 		if (err)
 			return err;
 
-		*a_len += sizeof(struct nfp_fl_output);
 	} else if (is_tcf_vlan(a) && tcf_vlan_action(a) == TCA_VLAN_ACT_POP) {
 		if (*a_len + sizeof(struct nfp_fl_pop_vlan) > NFP_FL_MAX_A_SIZ)
 			return -EOPNOTSUPP;
@@ -535,11 +612,12 @@ nfp_flower_loop_action(const struct tc_action *a,
 	return 0;
 }
 
-int nfp_flower_compile_action(struct tc_cls_flower_offload *flow,
+int nfp_flower_compile_action(struct nfp_app *app,
+			      struct tc_cls_flower_offload *flow,
 			      struct net_device *netdev,
 			      struct nfp_fl_payload *nfp_flow)
 {
-	int act_len, act_cnt, err, tun_out_cnt;
+	int act_len, act_cnt, err, tun_out_cnt, out_cnt;
 	enum nfp_flower_tun_type tun_type;
 	const struct tc_action *a;
 	LIST_HEAD(actions);
@@ -550,11 +628,12 @@ int nfp_flower_compile_action(struct tc_cls_flower_offload *flow,
 	act_len = 0;
 	act_cnt = 0;
 	tun_out_cnt = 0;
+	out_cnt = 0;
 
 	tcf_exts_to_list(flow->exts, &actions);
 	list_for_each_entry(a, &actions, list) {
-		err = nfp_flower_loop_action(a, nfp_flow, &act_len, netdev,
-					     &tun_type, &tun_out_cnt);
+		err = nfp_flower_loop_action(app, a, nfp_flow, &act_len, netdev,
+					     &tun_type, &tun_out_cnt, &out_cnt);
 		if (err)
 			return err;
 		act_cnt++;
diff --git a/drivers/net/ethernet/netronome/nfp/flower/cmsg.c b/drivers/net/ethernet/netronome/nfp/flower/cmsg.c
index 577659f332e4..cb8565222621 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/cmsg.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/cmsg.c
@@ -239,8 +239,10 @@ nfp_flower_cmsg_portreify_rx(struct nfp_app *app, struct sk_buff *skb)
 static void
 nfp_flower_cmsg_process_one_rx(struct nfp_app *app, struct sk_buff *skb)
 {
+	struct nfp_flower_priv *app_priv = app->priv;
 	struct nfp_flower_cmsg_hdr *cmsg_hdr;
 	enum nfp_flower_cmsg_type_port type;
+	bool skb_stored = false;
 
 	cmsg_hdr = nfp_flower_cmsg_get_hdr(skb);
 
@@ -258,13 +260,20 @@ nfp_flower_cmsg_process_one_rx(struct nfp_app *app, struct sk_buff *skb)
 	case NFP_FLOWER_CMSG_TYPE_ACTIVE_TUNS:
 		nfp_tunnel_keep_alive(app, skb);
 		break;
+	case NFP_FLOWER_CMSG_TYPE_LAG_CONFIG:
+		if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG) {
+			skb_stored = nfp_flower_lag_unprocessed_msg(app, skb);
+			break;
+		}
+		/* fall through */
 	default:
 		nfp_flower_cmsg_warn(app, "Cannot handle invalid repr control type %u\n",
 				     type);
 		goto out;
 	}
 
-	dev_consume_skb_any(skb);
+	if (!skb_stored)
+		dev_consume_skb_any(skb);
 	return;
 out:
 	dev_kfree_skb_any(skb);
diff --git a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h
index bee4367a2c38..4a7f3510a296 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h
+++ b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h
@@ -92,6 +92,7 @@
 #define NFP_FL_ACTION_OPCODE_SET_IPV6_DST	12
 #define NFP_FL_ACTION_OPCODE_SET_UDP		14
 #define NFP_FL_ACTION_OPCODE_SET_TCP		15
+#define NFP_FL_ACTION_OPCODE_PRE_LAG		16
 #define NFP_FL_ACTION_OPCODE_PRE_TUNNEL		17
 #define NFP_FL_ACTION_OPCODE_NUM		32
 
@@ -103,6 +104,9 @@
 #define NFP_FL_PUSH_VLAN_CFI		BIT(12)
 #define NFP_FL_PUSH_VLAN_VID		GENMASK(11, 0)
 
+/* LAG ports */
+#define NFP_FL_LAG_OUT			0xC0DE0000
+
 /* Tunnel ports */
 #define NFP_FL_PORT_TYPE_TUN		0x50000000
 #define NFP_FL_IPV4_TUNNEL_TYPE		GENMASK(7, 4)
@@ -177,6 +181,15 @@ struct nfp_fl_pop_vlan {
 	__be16 reserved;
 };
 
+struct nfp_fl_pre_lag {
+	struct nfp_fl_act_head head;
+	__be16 group_id;
+	u8 lag_version[3];
+	u8 instance;
+};
+
+#define NFP_FL_PRE_LAG_VER_OFF	8
+
 struct nfp_fl_pre_tunnel {
 	struct nfp_fl_act_head head;
 	__be16 reserved;
@@ -366,6 +379,7 @@ struct nfp_flower_cmsg_hdr {
 enum nfp_flower_cmsg_type_port {
 	NFP_FLOWER_CMSG_TYPE_FLOW_ADD =		0,
 	NFP_FLOWER_CMSG_TYPE_FLOW_DEL =		2,
+	NFP_FLOWER_CMSG_TYPE_LAG_CONFIG =	4,
 	NFP_FLOWER_CMSG_TYPE_PORT_REIFY =	6,
 	NFP_FLOWER_CMSG_TYPE_MAC_REPR =		7,
 	NFP_FLOWER_CMSG_TYPE_PORT_MOD =		8,
diff --git a/drivers/net/ethernet/netronome/nfp/flower/lag_conf.c b/drivers/net/ethernet/netronome/nfp/flower/lag_conf.c
new file mode 100644
index 000000000000..0c4c957717ea
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/flower/lag_conf.c
@@ -0,0 +1,726 @@
+/*
+ * Copyright (C) 2018 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "main.h"
+
+/* LAG group config flags. */
+#define NFP_FL_LAG_LAST			BIT(1)
+#define NFP_FL_LAG_FIRST		BIT(2)
+#define NFP_FL_LAG_DATA			BIT(3)
+#define NFP_FL_LAG_XON			BIT(4)
+#define NFP_FL_LAG_SYNC			BIT(5)
+#define NFP_FL_LAG_SWITCH		BIT(6)
+#define NFP_FL_LAG_RESET		BIT(7)
+
+/* LAG port state flags. */
+#define NFP_PORT_LAG_LINK_UP		BIT(0)
+#define NFP_PORT_LAG_TX_ENABLED		BIT(1)
+#define NFP_PORT_LAG_CHANGED		BIT(2)
+
+enum nfp_fl_lag_batch {
+	NFP_FL_LAG_BATCH_FIRST,
+	NFP_FL_LAG_BATCH_MEMBER,
+	NFP_FL_LAG_BATCH_FINISHED
+};
+
+/**
+ * struct nfp_flower_cmsg_lag_config - control message payload for LAG config
+ * @ctrl_flags:	Configuration flags
+ * @reserved:	Reserved for future use
+ * @ttl:	Time to live of packet - host always sets to 0xff
+ * @pkt_number:	Config message packet number - increment for each message
+ * @batch_ver:	Batch version of messages - increment for each batch of messages
+ * @group_id:	Group ID applicable
+ * @group_inst:	Group instance number - increment when group is reused
+ * @members:	Array of 32-bit words listing all active group members
+ */
+struct nfp_flower_cmsg_lag_config {
+	u8 ctrl_flags;
+	u8 reserved[2];
+	u8 ttl;
+	__be32 pkt_number;
+	__be32 batch_ver;
+	__be32 group_id;
+	__be32 group_inst;
+	__be32 members[];
+};
+
+/**
+ * struct nfp_fl_lag_group - list entry for each LAG group
+ * @group_id:		Assigned group ID for host/kernel sync
+ * @group_inst:		Group instance in case of ID reuse
+ * @list:		List entry
+ * @master_ndev:	Group master Netdev
+ * @dirty:		Marked if the group needs synced to HW
+ * @offloaded:		Marked if the group is currently offloaded to NIC
+ * @to_remove:		Marked if the group should be removed from NIC
+ * @to_destroy:		Marked if the group should be removed from driver
+ * @slave_cnt:		Number of slaves in group
+ */
+struct nfp_fl_lag_group {
+	unsigned int group_id;
+	u8 group_inst;
+	struct list_head list;
+	struct net_device *master_ndev;
+	bool dirty;
+	bool offloaded;
+	bool to_remove;
+	bool to_destroy;
+	unsigned int slave_cnt;
+};
+
+#define NFP_FL_LAG_PKT_NUMBER_MASK	GENMASK(30, 0)
+#define NFP_FL_LAG_VERSION_MASK		GENMASK(22, 0)
+#define NFP_FL_LAG_HOST_TTL		0xff
+
+/* Use this ID with zero members to ack a batch config */
+#define NFP_FL_LAG_SYNC_ID		0
+#define NFP_FL_LAG_GROUP_MIN		1 /* ID 0 reserved */
+#define NFP_FL_LAG_GROUP_MAX		32 /* IDs 1 to 31 are valid */
+
+/* wait for more config */
+#define NFP_FL_LAG_DELAY		(msecs_to_jiffies(2))
+
+#define NFP_FL_LAG_RETRANS_LIMIT	100 /* max retrans cmsgs to store */
+
+static unsigned int nfp_fl_get_next_pkt_number(struct nfp_fl_lag *lag)
+{
+	lag->pkt_num++;
+	lag->pkt_num &= NFP_FL_LAG_PKT_NUMBER_MASK;
+
+	return lag->pkt_num;
+}
+
+static void nfp_fl_increment_version(struct nfp_fl_lag *lag)
+{
+	/* LSB is not considered by firmware so add 2 for each increment. */
+	lag->batch_ver += 2;
+	lag->batch_ver &= NFP_FL_LAG_VERSION_MASK;
+
+	/* Zero is reserved by firmware. */
+	if (!lag->batch_ver)
+		lag->batch_ver += 2;
+}
+
+static struct nfp_fl_lag_group *
+nfp_fl_lag_group_create(struct nfp_fl_lag *lag, struct net_device *master)
+{
+	struct nfp_fl_lag_group *group;
+	struct nfp_flower_priv *priv;
+	int id;
+
+	priv = container_of(lag, struct nfp_flower_priv, nfp_lag);
+
+	id = ida_simple_get(&lag->ida_handle, NFP_FL_LAG_GROUP_MIN,
+			    NFP_FL_LAG_GROUP_MAX, GFP_KERNEL);
+	if (id < 0) {
+		nfp_flower_cmsg_warn(priv->app,
+				     "No more bonding groups available\n");
+		return ERR_PTR(id);
+	}
+
+	group = kmalloc(sizeof(*group), GFP_KERNEL);
+	if (!group) {
+		ida_simple_remove(&lag->ida_handle, id);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	group->group_id = id;
+	group->master_ndev = master;
+	group->dirty = true;
+	group->offloaded = false;
+	group->to_remove = false;
+	group->to_destroy = false;
+	group->slave_cnt = 0;
+	group->group_inst = ++lag->global_inst;
+	list_add_tail(&group->list, &lag->group_list);
+
+	return group;
+}
+
+static struct nfp_fl_lag_group *
+nfp_fl_lag_find_group_for_master_with_lag(struct nfp_fl_lag *lag,
+					  struct net_device *master)
+{
+	struct nfp_fl_lag_group *entry;
+
+	if (!master)
+		return NULL;
+
+	list_for_each_entry(entry, &lag->group_list, list)
+		if (entry->master_ndev == master)
+			return entry;
+
+	return NULL;
+}
+
+int nfp_flower_lag_populate_pre_action(struct nfp_app *app,
+				       struct net_device *master,
+				       struct nfp_fl_pre_lag *pre_act)
+{
+	struct nfp_flower_priv *priv = app->priv;
+	struct nfp_fl_lag_group *group = NULL;
+	__be32 temp_vers;
+
+	mutex_lock(&priv->nfp_lag.lock);
+	group = nfp_fl_lag_find_group_for_master_with_lag(&priv->nfp_lag,
+							  master);
+	if (!group) {
+		mutex_unlock(&priv->nfp_lag.lock);
+		return -ENOENT;
+	}
+
+	pre_act->group_id = cpu_to_be16(group->group_id);
+	temp_vers = cpu_to_be32(priv->nfp_lag.batch_ver <<
+				NFP_FL_PRE_LAG_VER_OFF);
+	memcpy(pre_act->lag_version, &temp_vers, 3);
+	pre_act->instance = group->group_inst;
+	mutex_unlock(&priv->nfp_lag.lock);
+
+	return 0;
+}
+
+int nfp_flower_lag_get_output_id(struct nfp_app *app, struct net_device *master)
+{
+	struct nfp_flower_priv *priv = app->priv;
+	struct nfp_fl_lag_group *group = NULL;
+	int group_id = -ENOENT;
+
+	mutex_lock(&priv->nfp_lag.lock);
+	group = nfp_fl_lag_find_group_for_master_with_lag(&priv->nfp_lag,
+							  master);
+	if (group)
+		group_id = group->group_id;
+	mutex_unlock(&priv->nfp_lag.lock);
+
+	return group_id;
+}
+
+static int
+nfp_fl_lag_config_group(struct nfp_fl_lag *lag, struct nfp_fl_lag_group *group,
+			struct net_device **active_members,
+			unsigned int member_cnt, enum nfp_fl_lag_batch *batch)
+{
+	struct nfp_flower_cmsg_lag_config *cmsg_payload;
+	struct nfp_flower_priv *priv;
+	unsigned long int flags;
+	unsigned int size, i;
+	struct sk_buff *skb;
+
+	priv = container_of(lag, struct nfp_flower_priv, nfp_lag);
+	size = sizeof(*cmsg_payload) + sizeof(__be32) * member_cnt;
+	skb = nfp_flower_cmsg_alloc(priv->app, size,
+				    NFP_FLOWER_CMSG_TYPE_LAG_CONFIG,
+				    GFP_KERNEL);
+	if (!skb)
+		return -ENOMEM;
+
+	cmsg_payload = nfp_flower_cmsg_get_data(skb);
+	flags = 0;
+
+	/* Increment batch version for each new batch of config messages. */
+	if (*batch == NFP_FL_LAG_BATCH_FIRST) {
+		flags |= NFP_FL_LAG_FIRST;
+		nfp_fl_increment_version(lag);
+		*batch = NFP_FL_LAG_BATCH_MEMBER;
+	}
+
+	/* If it is a reset msg then it is also the end of the batch. */
+	if (lag->rst_cfg) {
+		flags |= NFP_FL_LAG_RESET;
+		*batch = NFP_FL_LAG_BATCH_FINISHED;
+	}
+
+	/* To signal the end of a batch, both the switch and last flags are set
+	 * and the the reserved SYNC group ID is used.
+	 */
+	if (*batch == NFP_FL_LAG_BATCH_FINISHED) {
+		flags |= NFP_FL_LAG_SWITCH | NFP_FL_LAG_LAST;
+		lag->rst_cfg = false;
+		cmsg_payload->group_id = cpu_to_be32(NFP_FL_LAG_SYNC_ID);
+		cmsg_payload->group_inst = 0;
+	} else {
+		cmsg_payload->group_id = cpu_to_be32(group->group_id);
+		cmsg_payload->group_inst = cpu_to_be32(group->group_inst);
+	}
+
+	cmsg_payload->reserved[0] = 0;
+	cmsg_payload->reserved[1] = 0;
+	cmsg_payload->ttl = NFP_FL_LAG_HOST_TTL;
+	cmsg_payload->ctrl_flags = flags;
+	cmsg_payload->batch_ver = cpu_to_be32(lag->batch_ver);
+	cmsg_payload->pkt_number = cpu_to_be32(nfp_fl_get_next_pkt_number(lag));
+
+	for (i = 0; i < member_cnt; i++)
+		cmsg_payload->members[i] =
+			cpu_to_be32(nfp_repr_get_port_id(active_members[i]));
+
+	nfp_ctrl_tx(priv->app->ctrl, skb);
+	return 0;
+}
+
+static void nfp_fl_lag_do_work(struct work_struct *work)
+{
+	enum nfp_fl_lag_batch batch = NFP_FL_LAG_BATCH_FIRST;
+	struct nfp_fl_lag_group *entry, *storage;
+	struct delayed_work *delayed_work;
+	struct nfp_flower_priv *priv;
+	struct nfp_fl_lag *lag;
+	int err;
+
+	delayed_work = to_delayed_work(work);
+	lag = container_of(delayed_work, struct nfp_fl_lag, work);
+	priv = container_of(lag, struct nfp_flower_priv, nfp_lag);
+
+	mutex_lock(&lag->lock);
+	list_for_each_entry_safe(entry, storage, &lag->group_list, list) {
+		struct net_device *iter_netdev, **acti_netdevs;
+		struct nfp_flower_repr_priv *repr_priv;
+		int active_count = 0, slaves = 0;
+		struct nfp_repr *repr;
+		unsigned long *flags;
+
+		if (entry->to_remove) {
+			/* Active count of 0 deletes group on hw. */
+			err = nfp_fl_lag_config_group(lag, entry, NULL, 0,
+						      &batch);
+			if (!err) {
+				entry->to_remove = false;
+				entry->offloaded = false;
+			} else {
+				nfp_flower_cmsg_warn(priv->app,
+						     "group delete failed\n");
+				schedule_delayed_work(&lag->work,
+						      NFP_FL_LAG_DELAY);
+				continue;
+			}
+
+			if (entry->to_destroy) {
+				ida_simple_remove(&lag->ida_handle,
+						  entry->group_id);
+				list_del(&entry->list);
+				kfree(entry);
+			}
+			continue;
+		}
+
+		acti_netdevs = kmalloc_array(entry->slave_cnt,
+					     sizeof(*acti_netdevs), GFP_KERNEL);
+
+		/* Include sanity check in the loop. It may be that a bond has
+		 * changed between processing the last notification and the
+		 * work queue triggering. If the number of slaves has changed
+		 * or it now contains netdevs that cannot be offloaded, ignore
+		 * the group until pending notifications are processed.
+		 */
+		rcu_read_lock();
+		for_each_netdev_in_bond_rcu(entry->master_ndev, iter_netdev) {
+			if (!nfp_netdev_is_nfp_repr(iter_netdev)) {
+				slaves = 0;
+				break;
+			}
+
+			repr = netdev_priv(iter_netdev);
+
+			if (repr->app != priv->app) {
+				slaves = 0;
+				break;
+			}
+
+			slaves++;
+			if (slaves > entry->slave_cnt)
+				break;
+
+			/* Check the ports for state changes. */
+			repr_priv = repr->app_priv;
+			flags = &repr_priv->lag_port_flags;
+
+			if (*flags & NFP_PORT_LAG_CHANGED) {
+				*flags &= ~NFP_PORT_LAG_CHANGED;
+				entry->dirty = true;
+			}
+
+			if ((*flags & NFP_PORT_LAG_TX_ENABLED) &&
+			    (*flags & NFP_PORT_LAG_LINK_UP))
+				acti_netdevs[active_count++] = iter_netdev;
+		}
+		rcu_read_unlock();
+
+		if (slaves != entry->slave_cnt || !entry->dirty) {
+			kfree(acti_netdevs);
+			continue;
+		}
+
+		err = nfp_fl_lag_config_group(lag, entry, acti_netdevs,
+					      active_count, &batch);
+		if (!err) {
+			entry->offloaded = true;
+			entry->dirty = false;
+		} else {
+			nfp_flower_cmsg_warn(priv->app,
+					     "group offload failed\n");
+			schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY);
+		}
+
+		kfree(acti_netdevs);
+	}
+
+	/* End the config batch if at least one packet has been batched. */
+	if (batch == NFP_FL_LAG_BATCH_MEMBER) {
+		batch = NFP_FL_LAG_BATCH_FINISHED;
+		err = nfp_fl_lag_config_group(lag, NULL, NULL, 0, &batch);
+		if (err)
+			nfp_flower_cmsg_warn(priv->app,
+					     "group batch end cmsg failed\n");
+	}
+
+	mutex_unlock(&lag->lock);
+}
+
+static int
+nfp_fl_lag_put_unprocessed(struct nfp_fl_lag *lag, struct sk_buff *skb)
+{
+	struct nfp_flower_cmsg_lag_config *cmsg_payload;
+
+	cmsg_payload = nfp_flower_cmsg_get_data(skb);
+	if (be32_to_cpu(cmsg_payload->group_id) >= NFP_FL_LAG_GROUP_MAX)
+		return -EINVAL;
+
+	/* Drop cmsg retrans if storage limit is exceeded to prevent
+	 * overloading. If the fw notices that expected messages have not been
+	 * received in a given time block, it will request a full resync.
+	 */
+	if (skb_queue_len(&lag->retrans_skbs) >= NFP_FL_LAG_RETRANS_LIMIT)
+		return -ENOSPC;
+
+	__skb_queue_tail(&lag->retrans_skbs, skb);
+
+	return 0;
+}
+
+static void nfp_fl_send_unprocessed(struct nfp_fl_lag *lag)
+{
+	struct nfp_flower_priv *priv;
+	struct sk_buff *skb;
+
+	priv = container_of(lag, struct nfp_flower_priv, nfp_lag);
+
+	while ((skb = __skb_dequeue(&lag->retrans_skbs)))
+		nfp_ctrl_tx(priv->app->ctrl, skb);
+}
+
+bool nfp_flower_lag_unprocessed_msg(struct nfp_app *app, struct sk_buff *skb)
+{
+	struct nfp_flower_cmsg_lag_config *cmsg_payload;
+	struct nfp_flower_priv *priv = app->priv;
+	struct nfp_fl_lag_group *group_entry;
+	unsigned long int flags;
+	bool store_skb = false;
+	int err;
+
+	cmsg_payload = nfp_flower_cmsg_get_data(skb);
+	flags = cmsg_payload->ctrl_flags;
+
+	/* Note the intentional fall through below. If DATA and XON are both
+	 * set, the message will stored and sent again with the rest of the
+	 * unprocessed messages list.
+	 */
+
+	/* Store */
+	if (flags & NFP_FL_LAG_DATA)
+		if (!nfp_fl_lag_put_unprocessed(&priv->nfp_lag, skb))
+			store_skb = true;
+
+	/* Send stored */
+	if (flags & NFP_FL_LAG_XON)
+		nfp_fl_send_unprocessed(&priv->nfp_lag);
+
+	/* Resend all */
+	if (flags & NFP_FL_LAG_SYNC) {
+		/* To resend all config:
+		 * 1) Clear all unprocessed messages
+		 * 2) Mark all groups dirty
+		 * 3) Reset NFP group config
+		 * 4) Schedule a LAG config update
+		 */
+
+		__skb_queue_purge(&priv->nfp_lag.retrans_skbs);
+
+		mutex_lock(&priv->nfp_lag.lock);
+		list_for_each_entry(group_entry, &priv->nfp_lag.group_list,
+				    list)
+			group_entry->dirty = true;
+
+		err = nfp_flower_lag_reset(&priv->nfp_lag);
+		if (err)
+			nfp_flower_cmsg_warn(priv->app,
+					     "mem err in group reset msg\n");
+		mutex_unlock(&priv->nfp_lag.lock);
+
+		schedule_delayed_work(&priv->nfp_lag.work, 0);
+	}
+
+	return store_skb;
+}
+
+static void
+nfp_fl_lag_schedule_group_remove(struct nfp_fl_lag *lag,
+				 struct nfp_fl_lag_group *group)
+{
+	group->to_remove = true;
+
+	schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY);
+}
+
+static int
+nfp_fl_lag_schedule_group_delete(struct nfp_fl_lag *lag,
+				 struct net_device *master)
+{
+	struct nfp_fl_lag_group *group;
+
+	mutex_lock(&lag->lock);
+	group = nfp_fl_lag_find_group_for_master_with_lag(lag, master);
+	if (!group) {
+		mutex_unlock(&lag->lock);
+		return -ENOENT;
+	}
+
+	group->to_remove = true;
+	group->to_destroy = true;
+	mutex_unlock(&lag->lock);
+
+	schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY);
+	return 0;
+}
+
+static int
+nfp_fl_lag_changeupper_event(struct nfp_fl_lag *lag,
+			     struct netdev_notifier_changeupper_info *info)
+{
+	struct net_device *upper = info->upper_dev, *iter_netdev;
+	struct netdev_lag_upper_info *lag_upper_info;
+	struct nfp_fl_lag_group *group;
+	struct nfp_flower_priv *priv;
+	unsigned int slave_count = 0;
+	bool can_offload = true;
+	struct nfp_repr *repr;
+
+	if (!netif_is_lag_master(upper))
+		return 0;
+
+	priv = container_of(lag, struct nfp_flower_priv, nfp_lag);
+
+	rcu_read_lock();
+	for_each_netdev_in_bond_rcu(upper, iter_netdev) {
+		if (!nfp_netdev_is_nfp_repr(iter_netdev)) {
+			can_offload = false;
+			break;
+		}
+		repr = netdev_priv(iter_netdev);
+
+		/* Ensure all ports are created by the same app/on same card. */
+		if (repr->app != priv->app) {
+			can_offload = false;
+			break;
+		}
+
+		slave_count++;
+	}
+	rcu_read_unlock();
+
+	lag_upper_info = info->upper_info;
+
+	/* Firmware supports active/backup and L3/L4 hash bonds. */
+	if (lag_upper_info &&
+	    lag_upper_info->tx_type != NETDEV_LAG_TX_TYPE_ACTIVEBACKUP &&
+	    (lag_upper_info->tx_type != NETDEV_LAG_TX_TYPE_HASH ||
+	    (lag_upper_info->hash_type != NETDEV_LAG_HASH_L34 &&
+	    lag_upper_info->hash_type != NETDEV_LAG_HASH_E34))) {
+		can_offload = false;
+		nfp_flower_cmsg_warn(priv->app,
+				     "Unable to offload tx_type %u hash %u\n",
+				     lag_upper_info->tx_type,
+				     lag_upper_info->hash_type);
+	}
+
+	mutex_lock(&lag->lock);
+	group = nfp_fl_lag_find_group_for_master_with_lag(lag, upper);
+
+	if (slave_count == 0 || !can_offload) {
+		/* Cannot offload the group - remove if previously offloaded. */
+		if (group && group->offloaded)
+			nfp_fl_lag_schedule_group_remove(lag, group);
+
+		mutex_unlock(&lag->lock);
+		return 0;
+	}
+
+	if (!group) {
+		group = nfp_fl_lag_group_create(lag, upper);
+		if (IS_ERR(group)) {
+			mutex_unlock(&lag->lock);
+			return PTR_ERR(group);
+		}
+	}
+
+	group->dirty = true;
+	group->slave_cnt = slave_count;
+
+	/* Group may have been on queue for removal but is now offfloable. */
+	group->to_remove = false;
+	mutex_unlock(&lag->lock);
+
+	schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY);
+	return 0;
+}
+
+static int
+nfp_fl_lag_changels_event(struct nfp_fl_lag *lag, struct net_device *netdev,
+			  struct netdev_notifier_changelowerstate_info *info)
+{
+	struct netdev_lag_lower_state_info *lag_lower_info;
+	struct nfp_flower_repr_priv *repr_priv;
+	struct nfp_flower_priv *priv;
+	struct nfp_repr *repr;
+	unsigned long *flags;
+
+	if (!netif_is_lag_port(netdev) || !nfp_netdev_is_nfp_repr(netdev))
+		return 0;
+
+	lag_lower_info = info->lower_state_info;
+	if (!lag_lower_info)
+		return 0;
+
+	priv = container_of(lag, struct nfp_flower_priv, nfp_lag);
+	repr = netdev_priv(netdev);
+
+	/* Verify that the repr is associated with this app. */
+	if (repr->app != priv->app)
+		return 0;
+
+	repr_priv = repr->app_priv;
+	flags = &repr_priv->lag_port_flags;
+
+	mutex_lock(&lag->lock);
+	if (lag_lower_info->link_up)
+		*flags |= NFP_PORT_LAG_LINK_UP;
+	else
+		*flags &= ~NFP_PORT_LAG_LINK_UP;
+
+	if (lag_lower_info->tx_enabled)
+		*flags |= NFP_PORT_LAG_TX_ENABLED;
+	else
+		*flags &= ~NFP_PORT_LAG_TX_ENABLED;
+
+	*flags |= NFP_PORT_LAG_CHANGED;
+	mutex_unlock(&lag->lock);
+
+	schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY);
+	return 0;
+}
+
+static int
+nfp_fl_lag_netdev_event(struct notifier_block *nb, unsigned long event,
+			void *ptr)
+{
+	struct net_device *netdev;
+	struct nfp_fl_lag *lag;
+	int err;
+
+	netdev = netdev_notifier_info_to_dev(ptr);
+	lag = container_of(nb, struct nfp_fl_lag, lag_nb);
+
+	switch (event) {
+	case NETDEV_CHANGEUPPER:
+		err = nfp_fl_lag_changeupper_event(lag, ptr);
+		if (err)
+			return NOTIFY_BAD;
+		return NOTIFY_OK;
+	case NETDEV_CHANGELOWERSTATE:
+		err = nfp_fl_lag_changels_event(lag, netdev, ptr);
+		if (err)
+			return NOTIFY_BAD;
+		return NOTIFY_OK;
+	case NETDEV_UNREGISTER:
+		if (netif_is_bond_master(netdev)) {
+			err = nfp_fl_lag_schedule_group_delete(lag, netdev);
+			if (err)
+				return NOTIFY_BAD;
+			return NOTIFY_OK;
+		}
+	}
+
+	return NOTIFY_DONE;
+}
+
+int nfp_flower_lag_reset(struct nfp_fl_lag *lag)
+{
+	enum nfp_fl_lag_batch batch = NFP_FL_LAG_BATCH_FIRST;
+
+	lag->rst_cfg = true;
+	return nfp_fl_lag_config_group(lag, NULL, NULL, 0, &batch);
+}
+
+void nfp_flower_lag_init(struct nfp_fl_lag *lag)
+{
+	INIT_DELAYED_WORK(&lag->work, nfp_fl_lag_do_work);
+	INIT_LIST_HEAD(&lag->group_list);
+	mutex_init(&lag->lock);
+	ida_init(&lag->ida_handle);
+
+	__skb_queue_head_init(&lag->retrans_skbs);
+
+	/* 0 is a reserved batch version so increment to first valid value. */
+	nfp_fl_increment_version(lag);
+
+	lag->lag_nb.notifier_call = nfp_fl_lag_netdev_event;
+}
+
+void nfp_flower_lag_cleanup(struct nfp_fl_lag *lag)
+{
+	struct nfp_fl_lag_group *entry, *storage;
+
+	cancel_delayed_work_sync(&lag->work);
+
+	__skb_queue_purge(&lag->retrans_skbs);
+
+	/* Remove all groups. */
+	mutex_lock(&lag->lock);
+	list_for_each_entry_safe(entry, storage, &lag->group_list, list) {
+		list_del(&entry->list);
+		kfree(entry);
+	}
+	mutex_unlock(&lag->lock);
+	mutex_destroy(&lag->lock);
+	ida_destroy(&lag->ida_handle);
+}
diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.c b/drivers/net/ethernet/netronome/nfp/flower/main.c
index 84e3b9f5abb1..19cfa162ac65 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/main.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/main.c
@@ -185,6 +185,10 @@ nfp_flower_repr_netdev_init(struct nfp_app *app, struct net_device *netdev)
 static void
 nfp_flower_repr_netdev_clean(struct nfp_app *app, struct net_device *netdev)
 {
+	struct nfp_repr *repr = netdev_priv(netdev);
+
+	kfree(repr->app_priv);
+
 	tc_setup_cb_egdev_unregister(netdev, nfp_flower_setup_tc_egress_cb,
 				     netdev_priv(netdev));
 }
@@ -225,7 +229,9 @@ nfp_flower_spawn_vnic_reprs(struct nfp_app *app,
 	u8 nfp_pcie = nfp_cppcore_pcie_unit(app->pf->cpp);
 	struct nfp_flower_priv *priv = app->priv;
 	atomic_t *replies = &priv->reify_replies;
+	struct nfp_flower_repr_priv *repr_priv;
 	enum nfp_port_type port_type;
+	struct nfp_repr *nfp_repr;
 	struct nfp_reprs *reprs;
 	int i, err, reify_cnt;
 	const u8 queue = 0;
@@ -247,12 +253,25 @@ nfp_flower_spawn_vnic_reprs(struct nfp_app *app,
 			err = -ENOMEM;
 			goto err_reprs_clean;
 		}
-		RCU_INIT_POINTER(reprs->reprs[i], repr);
+
+		repr_priv = kzalloc(sizeof(*repr_priv), GFP_KERNEL);
+		if (!repr_priv) {
+			err = -ENOMEM;
+			goto err_reprs_clean;
+		}
+
+		nfp_repr = netdev_priv(repr);
+		nfp_repr->app_priv = repr_priv;
 
 		/* For now we only support 1 PF */
 		WARN_ON(repr_type == NFP_REPR_TYPE_PF && i);
 
 		port = nfp_port_alloc(app, port_type, repr);
+		if (IS_ERR(port)) {
+			err = PTR_ERR(port);
+			nfp_repr_free(repr);
+			goto err_reprs_clean;
+		}
 		if (repr_type == NFP_REPR_TYPE_PF) {
 			port->pf_id = i;
 			port->vnic = priv->nn->dp.ctrl_bar;
@@ -271,9 +290,11 @@ nfp_flower_spawn_vnic_reprs(struct nfp_app *app,
 				    port_id, port, priv->nn->dp.netdev);
 		if (err) {
 			nfp_port_free(port);
+			nfp_repr_free(repr);
 			goto err_reprs_clean;
 		}
 
+		RCU_INIT_POINTER(reprs->reprs[i], repr);
 		nfp_info(app->cpp, "%s%d Representor(%s) created\n",
 			 repr_type == NFP_REPR_TYPE_PF ? "PF" : "VF", i,
 			 repr->name);
@@ -318,6 +339,8 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 {
 	struct nfp_eth_table *eth_tbl = app->pf->eth_tbl;
 	atomic_t *replies = &priv->reify_replies;
+	struct nfp_flower_repr_priv *repr_priv;
+	struct nfp_repr *nfp_repr;
 	struct sk_buff *ctrl_skb;
 	struct nfp_reprs *reprs;
 	int err, reify_cnt;
@@ -344,16 +367,26 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 			err = -ENOMEM;
 			goto err_reprs_clean;
 		}
-		RCU_INIT_POINTER(reprs->reprs[phys_port], repr);
+
+		repr_priv = kzalloc(sizeof(*repr_priv), GFP_KERNEL);
+		if (!repr_priv) {
+			err = -ENOMEM;
+			goto err_reprs_clean;
+		}
+
+		nfp_repr = netdev_priv(repr);
+		nfp_repr->app_priv = repr_priv;
 
 		port = nfp_port_alloc(app, NFP_PORT_PHYS_PORT, repr);
 		if (IS_ERR(port)) {
 			err = PTR_ERR(port);
+			nfp_repr_free(repr);
 			goto err_reprs_clean;
 		}
 		err = nfp_port_init_phy_port(app->pf, app, port, i);
 		if (err) {
 			nfp_port_free(port);
+			nfp_repr_free(repr);
 			goto err_reprs_clean;
 		}
 
@@ -365,6 +398,7 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 				    cmsg_port_id, port, priv->nn->dp.netdev);
 		if (err) {
 			nfp_port_free(port);
+			nfp_repr_free(repr);
 			goto err_reprs_clean;
 		}
 
@@ -373,6 +407,7 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 					     eth_tbl->ports[i].base,
 					     phys_port);
 
+		RCU_INIT_POINTER(reprs->reprs[phys_port], repr);
 		nfp_info(app->cpp, "Phys Port %d Representor(%s) created\n",
 			 phys_port, repr->name);
 	}
@@ -537,8 +572,22 @@ static int nfp_flower_init(struct nfp_app *app)
 	else
 		app_priv->flower_ext_feats = features;
 
+	/* Tell the firmware that the driver supports lag. */
+	err = nfp_rtsym_write_le(app->pf->rtbl,
+				 "_abi_flower_balance_sync_enable", 1);
+	if (!err) {
+		app_priv->flower_ext_feats |= NFP_FL_FEATS_LAG;
+		nfp_flower_lag_init(&app_priv->nfp_lag);
+	} else if (err == -ENOENT) {
+		nfp_warn(app->cpp, "LAG not supported by FW.\n");
+	} else {
+		goto err_cleanup_metadata;
+	}
+
 	return 0;
 
+err_cleanup_metadata:
+	nfp_flower_metadata_cleanup(app);
 err_free_app_priv:
 	vfree(app->priv);
 	return err;
@@ -552,6 +601,9 @@ static void nfp_flower_clean(struct nfp_app *app)
 	skb_queue_purge(&app_priv->cmsg_skbs_low);
 	flush_work(&app_priv->cmsg_work);
 
+	if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG)
+		nfp_flower_lag_cleanup(&app_priv->nfp_lag);
+
 	nfp_flower_metadata_cleanup(app);
 	vfree(app->priv);
 	app->priv = NULL;
@@ -618,11 +670,29 @@ nfp_flower_repr_change_mtu(struct nfp_app *app, struct net_device *netdev,
 
 static int nfp_flower_start(struct nfp_app *app)
 {
+	struct nfp_flower_priv *app_priv = app->priv;
+	int err;
+
+	if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG) {
+		err = nfp_flower_lag_reset(&app_priv->nfp_lag);
+		if (err)
+			return err;
+
+		err = register_netdevice_notifier(&app_priv->nfp_lag.lag_nb);
+		if (err)
+			return err;
+	}
+
 	return nfp_tunnel_config_start(app);
 }
 
 static void nfp_flower_stop(struct nfp_app *app)
 {
+	struct nfp_flower_priv *app_priv = app->priv;
+
+	if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG)
+		unregister_netdevice_notifier(&app_priv->nfp_lag.lag_nb);
+
 	nfp_tunnel_config_stop(app);
 }
 
diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.h b/drivers/net/ethernet/netronome/nfp/flower/main.h
index c67e1b54c614..bbe5764d26cb 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/main.h
+++ b/drivers/net/ethernet/netronome/nfp/flower/main.h
@@ -43,10 +43,13 @@
 #include <net/pkt_cls.h>
 #include <net/tcp.h>
 #include <linux/workqueue.h>
+#include <linux/idr.h>
 
+struct nfp_fl_pre_lag;
 struct net_device;
 struct nfp_app;
 
+#define NFP_FL_STATS_CTX_DONT_CARE	cpu_to_be32(0xffffffff)
 #define NFP_FL_STATS_ENTRY_RS		BIT(20)
 #define NFP_FL_STATS_ELEM_RS		4
 #define NFP_FL_REPEATED_HASH_MAX	BIT(17)
@@ -66,6 +69,7 @@ struct nfp_app;
 /* Extra features bitmap. */
 #define NFP_FL_FEATS_GENEVE		BIT(0)
 #define NFP_FL_NBI_MTU_SETTING		BIT(1)
+#define NFP_FL_FEATS_LAG		BIT(31)
 
 struct nfp_fl_mask_id {
 	struct circ_buf mask_id_free_list;
@@ -95,6 +99,33 @@ struct nfp_mtu_conf {
 	spinlock_t lock;
 };
 
+/**
+ * struct nfp_fl_lag - Flower APP priv data for link aggregation
+ * @lag_nb:		Notifier to track master/slave events
+ * @work:		Work queue for writing configs to the HW
+ * @lock:		Lock to protect lag_group_list
+ * @group_list:		List of all master/slave groups offloaded
+ * @ida_handle:		IDA to handle group ids
+ * @pkt_num:		Incremented for each config packet sent
+ * @batch_ver:		Incremented for each batch of config packets
+ * @global_inst:	Instance allocator for groups
+ * @rst_cfg:		Marker to reset HW LAG config
+ * @retrans_skbs:	Cmsgs that could not be processed by HW and require
+ *			retransmission
+ */
+struct nfp_fl_lag {
+	struct notifier_block lag_nb;
+	struct delayed_work work;
+	struct mutex lock;
+	struct list_head group_list;
+	struct ida ida_handle;
+	unsigned int pkt_num;
+	unsigned int batch_ver;
+	u8 global_inst;
+	bool rst_cfg;
+	struct sk_buff_head retrans_skbs;
+};
+
 /**
  * struct nfp_flower_priv - Flower APP per-vNIC priv data
  * @app:		Back pointer to app
@@ -127,6 +158,7 @@ struct nfp_mtu_conf {
  *			from firmware for repr reify
  * @reify_wait_queue:	wait queue for repr reify response counting
  * @mtu_conf:		Configuration of repr MTU value
+ * @nfp_lag:		Link aggregation data block
  */
 struct nfp_flower_priv {
 	struct nfp_app *app;
@@ -156,6 +188,15 @@ struct nfp_flower_priv {
 	atomic_t reify_replies;
 	wait_queue_head_t reify_wait_queue;
 	struct nfp_mtu_conf mtu_conf;
+	struct nfp_fl_lag nfp_lag;
+};
+
+/**
+ * struct nfp_flower_repr_priv - Flower APP per-repr priv data
+ * @lag_port_flags:	Extended port flags to record lag state of repr
+ */
+struct nfp_flower_repr_priv {
+	unsigned long lag_port_flags;
 };
 
 struct nfp_fl_key_ls {
@@ -189,9 +230,11 @@ struct nfp_fl_payload {
 	spinlock_t lock; /* lock stats */
 	struct nfp_fl_stats stats;
 	__be32 nfp_tun_ipv4_addr;
+	struct net_device *ingress_dev;
 	char *unmasked_data;
 	char *mask_data;
 	char *action_data;
+	bool ingress_offload;
 };
 
 struct nfp_fl_stats_frame {
@@ -211,17 +254,20 @@ int nfp_flower_compile_flow_match(struct tc_cls_flower_offload *flow,
 				  struct net_device *netdev,
 				  struct nfp_fl_payload *nfp_flow,
 				  enum nfp_flower_tun_type tun_type);
-int nfp_flower_compile_action(struct tc_cls_flower_offload *flow,
+int nfp_flower_compile_action(struct nfp_app *app,
+			      struct tc_cls_flower_offload *flow,
 			      struct net_device *netdev,
 			      struct nfp_fl_payload *nfp_flow);
 int nfp_compile_flow_metadata(struct nfp_app *app,
 			      struct tc_cls_flower_offload *flow,
-			      struct nfp_fl_payload *nfp_flow);
+			      struct nfp_fl_payload *nfp_flow,
+			      struct net_device *netdev);
 int nfp_modify_flow_metadata(struct nfp_app *app,
 			     struct nfp_fl_payload *nfp_flow);
 
 struct nfp_fl_payload *
-nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie);
+nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie,
+			   struct net_device *netdev, __be32 host_ctx);
 struct nfp_fl_payload *
 nfp_flower_remove_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie);
 
@@ -236,5 +282,14 @@ void nfp_tunnel_request_route(struct nfp_app *app, struct sk_buff *skb);
 void nfp_tunnel_keep_alive(struct nfp_app *app, struct sk_buff *skb);
 int nfp_flower_setup_tc_egress_cb(enum tc_setup_type type, void *type_data,
 				  void *cb_priv);
+void nfp_flower_lag_init(struct nfp_fl_lag *lag);
+void nfp_flower_lag_cleanup(struct nfp_fl_lag *lag);
+int nfp_flower_lag_reset(struct nfp_fl_lag *lag);
+bool nfp_flower_lag_unprocessed_msg(struct nfp_app *app, struct sk_buff *skb);
+int nfp_flower_lag_populate_pre_action(struct nfp_app *app,
+				       struct net_device *master,
+				       struct nfp_fl_pre_lag *pre_act);
+int nfp_flower_lag_get_output_id(struct nfp_app *app,
+				 struct net_device *master);
 
 #endif
diff --git a/drivers/net/ethernet/netronome/nfp/flower/metadata.c b/drivers/net/ethernet/netronome/nfp/flower/metadata.c
index db977cf8e933..21668aa435e8 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/metadata.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/metadata.c
@@ -99,14 +99,18 @@ static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id)
 
 /* Must be called with either RTNL or rcu_read_lock */
 struct nfp_fl_payload *
-nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie)
+nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie,
+			   struct net_device *netdev, __be32 host_ctx)
 {
 	struct nfp_flower_priv *priv = app->priv;
 	struct nfp_fl_payload *flower_entry;
 
 	hash_for_each_possible_rcu(priv->flow_table, flower_entry, link,
 				   tc_flower_cookie)
-		if (flower_entry->tc_flower_cookie == tc_flower_cookie)
+		if (flower_entry->tc_flower_cookie == tc_flower_cookie &&
+		    (!netdev || flower_entry->ingress_dev == netdev) &&
+		    (host_ctx == NFP_FL_STATS_CTX_DONT_CARE ||
+		     flower_entry->meta.host_ctx_id == host_ctx))
 			return flower_entry;
 
 	return NULL;
@@ -121,13 +125,11 @@ nfp_flower_update_stats(struct nfp_app *app, struct nfp_fl_stats_frame *stats)
 	flower_cookie = be64_to_cpu(stats->stats_cookie);
 
 	rcu_read_lock();
-	nfp_flow = nfp_flower_search_fl_table(app, flower_cookie);
+	nfp_flow = nfp_flower_search_fl_table(app, flower_cookie, NULL,
+					      stats->stats_con_id);
 	if (!nfp_flow)
 		goto exit_rcu_unlock;
 
-	if (nfp_flow->meta.host_ctx_id != stats->stats_con_id)
-		goto exit_rcu_unlock;
-
 	spin_lock(&nfp_flow->lock);
 	nfp_flow->stats.pkts += be32_to_cpu(stats->pkt_count);
 	nfp_flow->stats.bytes += be64_to_cpu(stats->byte_count);
@@ -317,7 +319,8 @@ nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len,
 
 int nfp_compile_flow_metadata(struct nfp_app *app,
 			      struct tc_cls_flower_offload *flow,
-			      struct nfp_fl_payload *nfp_flow)
+			      struct nfp_fl_payload *nfp_flow,
+			      struct net_device *netdev)
 {
 	struct nfp_flower_priv *priv = app->priv;
 	struct nfp_fl_payload *check_entry;
@@ -348,7 +351,8 @@ int nfp_compile_flow_metadata(struct nfp_app *app,
 	nfp_flow->stats.bytes = 0;
 	nfp_flow->stats.used = jiffies;
 
-	check_entry = nfp_flower_search_fl_table(app, flow->cookie);
+	check_entry = nfp_flower_search_fl_table(app, flow->cookie, netdev,
+						 NFP_FL_STATS_CTX_DONT_CARE);
 	if (check_entry) {
 		if (nfp_release_stats_entry(app, stats_cxt))
 			return -EINVAL;
diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c
index 114d2ab02a38..c42e64f32333 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/offload.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c
@@ -345,7 +345,7 @@ nfp_flower_calculate_key_layers(struct nfp_app *app,
 }
 
 static struct nfp_fl_payload *
-nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer)
+nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer, bool egress)
 {
 	struct nfp_fl_payload *flow_pay;
 
@@ -371,6 +371,8 @@ nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer)
 	flow_pay->meta.flags = 0;
 	spin_lock_init(&flow_pay->lock);
 
+	flow_pay->ingress_offload = !egress;
+
 	return flow_pay;
 
 err_free_mask:
@@ -402,8 +404,20 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev,
 	struct nfp_flower_priv *priv = app->priv;
 	struct nfp_fl_payload *flow_pay;
 	struct nfp_fl_key_ls *key_layer;
+	struct net_device *ingr_dev;
 	int err;
 
+	ingr_dev = egress ? NULL : netdev;
+	flow_pay = nfp_flower_search_fl_table(app, flow->cookie, ingr_dev,
+					      NFP_FL_STATS_CTX_DONT_CARE);
+	if (flow_pay) {
+		/* Ignore as duplicate if it has been added by different cb. */
+		if (flow_pay->ingress_offload && egress)
+			return 0;
+		else
+			return -EOPNOTSUPP;
+	}
+
 	key_layer = kmalloc(sizeof(*key_layer), GFP_KERNEL);
 	if (!key_layer)
 		return -ENOMEM;
@@ -413,22 +427,25 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev,
 	if (err)
 		goto err_free_key_ls;
 
-	flow_pay = nfp_flower_allocate_new(key_layer);
+	flow_pay = nfp_flower_allocate_new(key_layer, egress);
 	if (!flow_pay) {
 		err = -ENOMEM;
 		goto err_free_key_ls;
 	}
 
+	flow_pay->ingress_dev = egress ? NULL : netdev;
+
 	err = nfp_flower_compile_flow_match(flow, key_layer, netdev, flow_pay,
 					    tun_type);
 	if (err)
 		goto err_destroy_flow;
 
-	err = nfp_flower_compile_action(flow, netdev, flow_pay);
+	err = nfp_flower_compile_action(app, flow, netdev, flow_pay);
 	if (err)
 		goto err_destroy_flow;
 
-	err = nfp_compile_flow_metadata(app, flow, flow_pay);
+	err = nfp_compile_flow_metadata(app, flow, flow_pay,
+					flow_pay->ingress_dev);
 	if (err)
 		goto err_destroy_flow;
 
@@ -462,6 +479,7 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev,
  * @app:	Pointer to the APP handle
  * @netdev:	netdev structure.
  * @flow:	TC flower classifier offload structure
+ * @egress:	Netdev is the egress dev.
  *
  * Removes a flow from the repeated hash structure and clears the
  * action payload.
@@ -470,15 +488,18 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev,
  */
 static int
 nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev,
-		       struct tc_cls_flower_offload *flow)
+		       struct tc_cls_flower_offload *flow, bool egress)
 {
 	struct nfp_port *port = nfp_port_from_netdev(netdev);
 	struct nfp_fl_payload *nfp_flow;
+	struct net_device *ingr_dev;
 	int err;
 
-	nfp_flow = nfp_flower_search_fl_table(app, flow->cookie);
+	ingr_dev = egress ? NULL : netdev;
+	nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, ingr_dev,
+					      NFP_FL_STATS_CTX_DONT_CARE);
 	if (!nfp_flow)
-		return -ENOENT;
+		return egress ? 0 : -ENOENT;
 
 	err = nfp_modify_flow_metadata(app, nfp_flow);
 	if (err)
@@ -505,7 +526,9 @@ nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev,
 /**
  * nfp_flower_get_stats() - Populates flow stats obtained from hardware.
  * @app:	Pointer to the APP handle
+ * @netdev:	Netdev structure.
  * @flow:	TC flower classifier offload structure
+ * @egress:	Netdev is the egress dev.
  *
  * Populates a flow statistics structure which which corresponds to a
  * specific flow.
@@ -513,14 +536,21 @@ nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev,
  * Return: negative value on error, 0 if stats populated successfully.
  */
 static int
-nfp_flower_get_stats(struct nfp_app *app, struct tc_cls_flower_offload *flow)
+nfp_flower_get_stats(struct nfp_app *app, struct net_device *netdev,
+		     struct tc_cls_flower_offload *flow, bool egress)
 {
 	struct nfp_fl_payload *nfp_flow;
+	struct net_device *ingr_dev;
 
-	nfp_flow = nfp_flower_search_fl_table(app, flow->cookie);
+	ingr_dev = egress ? NULL : netdev;
+	nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, ingr_dev,
+					      NFP_FL_STATS_CTX_DONT_CARE);
 	if (!nfp_flow)
 		return -EINVAL;
 
+	if (nfp_flow->ingress_offload && egress)
+		return 0;
+
 	spin_lock_bh(&nfp_flow->lock);
 	tcf_exts_stats_update(flow->exts, nfp_flow->stats.bytes,
 			      nfp_flow->stats.pkts, nfp_flow->stats.used);
@@ -543,9 +573,9 @@ nfp_flower_repr_offload(struct nfp_app *app, struct net_device *netdev,
 	case TC_CLSFLOWER_REPLACE:
 		return nfp_flower_add_offload(app, netdev, flower, egress);
 	case TC_CLSFLOWER_DESTROY:
-		return nfp_flower_del_offload(app, netdev, flower);
+		return nfp_flower_del_offload(app, netdev, flower, egress);
 	case TC_CLSFLOWER_STATS:
-		return nfp_flower_get_stats(app, flower);
+		return nfp_flower_get_stats(app, netdev, flower, egress);
 	}
 
 	return -EOPNOTSUPP;
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_abi.h b/drivers/net/ethernet/netronome/nfp/nfp_abi.h
new file mode 100644
index 000000000000..8b56c27931bf
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfp_abi.h
@@ -0,0 +1,143 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
+/*
+ * Copyright (C) 2018 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __NFP_ABI__
+#define __NFP_ABI__ 1
+
+#include <linux/types.h>
+
+#define NFP_MBOX_SYM_NAME		"_abi_nfd_pf%u_mbox"
+#define NFP_MBOX_SYM_MIN_SIZE		16 /* When no data needed */
+
+#define NFP_MBOX_CMD		0x00
+#define NFP_MBOX_RET		0x04
+#define NFP_MBOX_DATA_LEN	0x08
+#define NFP_MBOX_RESERVED	0x0c
+#define NFP_MBOX_DATA		0x10
+
+/**
+ * enum nfp_mbox_cmd - PF mailbox commands
+ *
+ * @NFP_MBOX_NO_CMD:	null command
+ * Used to indicate previous command has finished.
+ *
+ * @NFP_MBOX_POOL_GET:	get shared buffer pool info/config
+ * Input  - struct nfp_shared_buf_pool_id
+ * Output - struct nfp_shared_buf_pool_info_get
+ *
+ * @NFP_MBOX_POOL_SET:	set shared buffer pool info/config
+ * Input  - struct nfp_shared_buf_pool_info_set
+ * Output - None
+ *
+ * @NFP_MBOX_PCIE_ABM_ENABLE:	enable PCIe-side advanced buffer management
+ * Enable advanced buffer management of the PCIe block.  If ABM is disabled
+ * PCIe block maintains a very short queue of buffers and does tail drop.
+ * ABM allows more advanced buffering and priority control.
+ * Input  - None
+ * Output - None
+ *
+ * @NFP_MBOX_PCIE_ABM_DISABLE:	disable PCIe-side advanced buffer management
+ * Input  - None
+ * Output - None
+ */
+enum nfp_mbox_cmd {
+	NFP_MBOX_NO_CMD			= 0x00,
+
+	NFP_MBOX_POOL_GET		= 0x01,
+	NFP_MBOX_POOL_SET		= 0x02,
+
+	NFP_MBOX_PCIE_ABM_ENABLE	= 0x03,
+	NFP_MBOX_PCIE_ABM_DISABLE	= 0x04,
+};
+
+#define NFP_SHARED_BUF_COUNT_SYM_NAME	"_abi_nfd_pf%u_sb_cnt"
+#define NFP_SHARED_BUF_TABLE_SYM_NAME	"_abi_nfd_pf%u_sb_tbl"
+
+/**
+ * struct nfp_shared_buf - NFP shared buffer description
+ * @id:				numerical user-visible id of the shared buffer
+ * @size:			size in bytes of the buffer
+ * @ingress_pools_count:	number of ingress pools
+ * @egress_pools_count:		number of egress pools
+ * @ingress_tc_count:		number of ingress trafic classes
+ * @egress_tc_count:		number of egress trafic classes
+ * @pool_size_unit:		pool size may be in credits, each credit is
+ *				@pool_size_unit bytes
+ */
+struct nfp_shared_buf {
+	__le32 id;
+	__le32 size;
+	__le16 ingress_pools_count;
+	__le16 egress_pools_count;
+	__le16 ingress_tc_count;
+	__le16 egress_tc_count;
+
+	__le32 pool_size_unit;
+};
+
+/**
+ * struct nfp_shared_buf_pool_id - shared buffer pool identification
+ * @shared_buf:		shared buffer id
+ * @pool:		pool index
+ */
+struct nfp_shared_buf_pool_id {
+	__le32 shared_buf;
+	__le32 pool;
+};
+
+/**
+ * struct nfp_shared_buf_pool_info_get - struct devlink_sb_pool_info mirror
+ * @pool_type:		one of enum devlink_sb_pool_type
+ * @size:		pool size in units of SB's @pool_size_unit
+ * @threshold_type:	one of enum devlink_sb_threshold_type
+ */
+struct nfp_shared_buf_pool_info_get {
+	__le32 pool_type;
+	__le32 size;
+	__le32 threshold_type;
+};
+
+/**
+ * struct nfp_shared_buf_pool_info_set - packed args of sb_pool_set
+ * @id:			pool identification info
+ * @size:		pool size in units of SB's @pool_size_unit
+ * @threshold_type:	one of enum devlink_sb_threshold_type
+ */
+struct nfp_shared_buf_pool_info_set {
+	struct nfp_shared_buf_pool_id id;
+	__le32 size;
+	__le32 threshold_type;
+};
+
+#endif
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.c b/drivers/net/ethernet/netronome/nfp/nfp_app.c
index 6aedef0ad433..f28b244f4ee7 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_app.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_app.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2017 Netronome Systems, Inc.
+ * Copyright (C) 2017-2018 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -43,6 +43,7 @@
 #include "nfp_main.h"
 #include "nfp_net.h"
 #include "nfp_net_repr.h"
+#include "nfp_port.h"
 
 static const struct nfp_app_type *apps[] = {
 	[NFP_APP_CORE_NIC]	= &app_nic,
@@ -54,6 +55,9 @@ static const struct nfp_app_type *apps[] = {
 #ifdef CONFIG_NFP_APP_FLOWER
 	[NFP_APP_FLOWER_NIC]	= &app_flower,
 #endif
+#ifdef CONFIG_NFP_APP_ABM_NIC
+	[NFP_APP_ACTIVE_BUFFER_MGMT_NIC] = &app_abm,
+#endif
 };
 
 struct nfp_app *nfp_app_from_netdev(struct net_device *netdev)
@@ -82,6 +86,27 @@ const char *nfp_app_mip_name(struct nfp_app *app)
 	return nfp_mip_name(app->pf->mip);
 }
 
+u64 *nfp_app_port_get_stats(struct nfp_port *port, u64 *data)
+{
+	if (!port || !port->app || !port->app->type->port_get_stats)
+		return data;
+	return port->app->type->port_get_stats(port->app, port, data);
+}
+
+int nfp_app_port_get_stats_count(struct nfp_port *port)
+{
+	if (!port || !port->app || !port->app->type->port_get_stats_count)
+		return 0;
+	return port->app->type->port_get_stats_count(port->app, port);
+}
+
+u8 *nfp_app_port_get_stats_strings(struct nfp_port *port, u8 *data)
+{
+	if (!port || !port->app || !port->app->type->port_get_stats_strings)
+		return data;
+	return port->app->type->port_get_stats_strings(port->app, port, data);
+}
+
 struct sk_buff *
 nfp_app_ctrl_msg_alloc(struct nfp_app *app, unsigned int size, gfp_t priority)
 {
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.h b/drivers/net/ethernet/netronome/nfp/nfp_app.h
index 2d9cb2528fc7..ee74caacb015 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_app.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_app.h
@@ -57,11 +57,13 @@ enum nfp_app_id {
 	NFP_APP_CORE_NIC	= 0x1,
 	NFP_APP_BPF_NIC		= 0x2,
 	NFP_APP_FLOWER_NIC	= 0x3,
+	NFP_APP_ACTIVE_BUFFER_MGMT_NIC = 0x4,
 };
 
 extern const struct nfp_app_type app_nic;
 extern const struct nfp_app_type app_bpf;
 extern const struct nfp_app_type app_flower;
+extern const struct nfp_app_type app_abm;
 
 /**
  * struct nfp_app_type - application definition
@@ -88,6 +90,9 @@ extern const struct nfp_app_type app_flower;
  * @repr_stop:	representor netdev stop callback
  * @check_mtu:	MTU change request on a netdev (verify it is valid)
  * @repr_change_mtu:	MTU change request on repr (make and verify change)
+ * @port_get_stats:		get extra ethtool statistics for a port
+ * @port_get_stats_count:	get count of extra statistics for a port
+ * @port_get_stats_strings:	get strings for extra statistics
  * @start:	start application logic
  * @stop:	stop application logic
  * @ctrl_msg_rx:    control message handler
@@ -95,6 +100,7 @@ extern const struct nfp_app_type app_flower;
  * @bpf:	BPF ndo offload-related calls
  * @xdp_offload:    offload an XDP program
  * @eswitch_mode_get:    get SR-IOV eswitch mode
+ * @eswitch_mode_set:    set SR-IOV eswitch mode (under pf->lock)
  * @sriov_enable: app-specific sriov initialisation
  * @sriov_disable: app-specific sriov clean-up
  * @repr_get:	get representor netdev
@@ -129,6 +135,12 @@ struct nfp_app_type {
 	int (*repr_change_mtu)(struct nfp_app *app, struct net_device *netdev,
 			       int new_mtu);
 
+	u64 *(*port_get_stats)(struct nfp_app *app,
+			       struct nfp_port *port, u64 *data);
+	int (*port_get_stats_count)(struct nfp_app *app, struct nfp_port *port);
+	u8 *(*port_get_stats_strings)(struct nfp_app *app,
+				      struct nfp_port *port, u8 *data);
+
 	int (*start)(struct nfp_app *app);
 	void (*stop)(struct nfp_app *app);
 
@@ -146,6 +158,7 @@ struct nfp_app_type {
 	void (*sriov_disable)(struct nfp_app *app);
 
 	enum devlink_eswitch_mode (*eswitch_mode_get)(struct nfp_app *app);
+	int (*eswitch_mode_set)(struct nfp_app *app, u16 mode);
 	struct net_device *(*repr_get)(struct nfp_app *app, u32 id);
 };
 
@@ -370,6 +383,13 @@ static inline int nfp_app_eswitch_mode_get(struct nfp_app *app, u16 *mode)
 	return 0;
 }
 
+static inline int nfp_app_eswitch_mode_set(struct nfp_app *app, u16 mode)
+{
+	if (!app->type->eswitch_mode_set)
+		return -EOPNOTSUPP;
+	return app->type->eswitch_mode_set(app, mode);
+}
+
 static inline int nfp_app_sriov_enable(struct nfp_app *app, int num_vfs)
 {
 	if (!app || !app->type->sriov_enable)
@@ -393,6 +413,10 @@ static inline struct net_device *nfp_app_repr_get(struct nfp_app *app, u32 id)
 
 struct nfp_app *nfp_app_from_netdev(struct net_device *netdev);
 
+u64 *nfp_app_port_get_stats(struct nfp_port *port, u64 *data);
+int nfp_app_port_get_stats_count(struct nfp_port *port);
+u8 *nfp_app_port_get_stats_strings(struct nfp_port *port, u8 *data);
+
 struct nfp_reprs *
 nfp_reprs_get_locked(struct nfp_app *app, enum nfp_repr_type type);
 struct nfp_reprs *
@@ -410,5 +434,7 @@ void nfp_app_free(struct nfp_app *app);
 
 int nfp_app_nic_vnic_alloc(struct nfp_app *app, struct nfp_net *nn,
 			   unsigned int id);
+int nfp_app_nic_vnic_init_phy_port(struct nfp_pf *pf, struct nfp_app *app,
+				   struct nfp_net *nn, unsigned int id);
 
 #endif
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c b/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c
index b9618c37403f..e2dfe4f168bb 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c
@@ -38,9 +38,8 @@
 #include "nfp_net.h"
 #include "nfp_port.h"
 
-static int
-nfp_app_nic_vnic_init_phy_port(struct nfp_pf *pf, struct nfp_app *app,
-			       struct nfp_net *nn, unsigned int id)
+int nfp_app_nic_vnic_init_phy_port(struct nfp_pf *pf, struct nfp_app *app,
+				   struct nfp_net *nn, unsigned int id)
 {
 	int err;
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_asm.h b/drivers/net/ethernet/netronome/nfp/nfp_asm.h
index 5f2b2f24f4fa..f6677bc9875a 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_asm.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_asm.h
@@ -72,8 +72,21 @@
 #define OP_BR_ADDR_LO		0x007ffc00000ULL
 #define OP_BR_ADDR_HI		0x10000000000ULL
 
-#define nfp_is_br(_insn)				\
-	(((_insn) & OP_BR_BASE_MASK) == OP_BR_BASE)
+#define OP_BR_BIT_BASE		0x0d000000000ULL
+#define OP_BR_BIT_BASE_MASK	0x0f800080300ULL
+#define OP_BR_BIT_A_SRC		0x000000000ffULL
+#define OP_BR_BIT_B_SRC		0x0000003fc00ULL
+#define OP_BR_BIT_BV		0x00000040000ULL
+#define OP_BR_BIT_SRC_LMEXTN	0x40000000000ULL
+#define OP_BR_BIT_DEFBR		OP_BR_DEFBR
+#define OP_BR_BIT_ADDR_LO	OP_BR_ADDR_LO
+#define OP_BR_BIT_ADDR_HI	OP_BR_ADDR_HI
+
+static inline bool nfp_is_br(u64 insn)
+{
+	return (insn & OP_BR_BASE_MASK) == OP_BR_BASE ||
+	       (insn & OP_BR_BIT_BASE_MASK) == OP_BR_BIT_BASE;
+}
 
 enum br_mask {
 	BR_BEQ = 0x00,
@@ -161,6 +174,7 @@ enum shf_op {
 	SHF_OP_NONE = 0,
 	SHF_OP_AND = 2,
 	SHF_OP_OR = 5,
+	SHF_OP_ASHR = 6,
 };
 
 enum shf_sc {
@@ -183,16 +197,18 @@ enum shf_sc {
 #define OP_ALU_DST_LMEXTN	0x80000000000ULL
 
 enum alu_op {
-	ALU_OP_NONE	= 0x00,
-	ALU_OP_ADD	= 0x01,
-	ALU_OP_NOT	= 0x04,
-	ALU_OP_ADD_2B	= 0x05,
-	ALU_OP_AND	= 0x08,
-	ALU_OP_SUB_C	= 0x0d,
-	ALU_OP_ADD_C	= 0x11,
-	ALU_OP_OR	= 0x14,
-	ALU_OP_SUB	= 0x15,
-	ALU_OP_XOR	= 0x18,
+	ALU_OP_NONE		= 0x00,
+	ALU_OP_ADD		= 0x01,
+	ALU_OP_NOT		= 0x04,
+	ALU_OP_ADD_2B		= 0x05,
+	ALU_OP_AND		= 0x08,
+	ALU_OP_AND_NOT_A	= 0x0c,
+	ALU_OP_SUB_C		= 0x0d,
+	ALU_OP_AND_NOT_B	= 0x10,
+	ALU_OP_ADD_C		= 0x11,
+	ALU_OP_OR		= 0x14,
+	ALU_OP_SUB		= 0x15,
+	ALU_OP_XOR		= 0x18,
 };
 
 enum alu_dst_ab {
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_devlink.c b/drivers/net/ethernet/netronome/nfp/nfp_devlink.c
index eb0fc614673d..71c2edd83031 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_devlink.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_devlink.c
@@ -149,6 +149,26 @@ nfp_devlink_port_unsplit(struct devlink *devlink, unsigned int port_index)
 	return ret;
 }
 
+static int
+nfp_devlink_sb_pool_get(struct devlink *devlink, unsigned int sb_index,
+			u16 pool_index, struct devlink_sb_pool_info *pool_info)
+{
+	struct nfp_pf *pf = devlink_priv(devlink);
+
+	return nfp_shared_buf_pool_get(pf, sb_index, pool_index, pool_info);
+}
+
+static int
+nfp_devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index,
+			u16 pool_index,
+			u32 size, enum devlink_sb_threshold_type threshold_type)
+{
+	struct nfp_pf *pf = devlink_priv(devlink);
+
+	return nfp_shared_buf_pool_set(pf, sb_index, pool_index,
+				       size, threshold_type);
+}
+
 static int nfp_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
 {
 	struct nfp_pf *pf = devlink_priv(devlink);
@@ -156,10 +176,25 @@ static int nfp_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
 	return nfp_app_eswitch_mode_get(pf->app, mode);
 }
 
+static int nfp_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode)
+{
+	struct nfp_pf *pf = devlink_priv(devlink);
+	int ret;
+
+	mutex_lock(&pf->lock);
+	ret = nfp_app_eswitch_mode_set(pf->app, mode);
+	mutex_unlock(&pf->lock);
+
+	return ret;
+}
+
 const struct devlink_ops nfp_devlink_ops = {
 	.port_split		= nfp_devlink_port_split,
 	.port_unsplit		= nfp_devlink_port_unsplit,
+	.sb_pool_get		= nfp_devlink_sb_pool_get,
+	.sb_pool_set		= nfp_devlink_sb_pool_set,
 	.eswitch_mode_get	= nfp_devlink_eswitch_mode_get,
+	.eswitch_mode_set	= nfp_devlink_eswitch_mode_set,
 };
 
 int nfp_devlink_port_register(struct nfp_app *app, struct nfp_port *port)
@@ -175,8 +210,9 @@ int nfp_devlink_port_register(struct nfp_app *app, struct nfp_port *port)
 		return ret;
 
 	devlink_port_type_eth_set(&port->dl_port, port->netdev);
-	if (eth_port.is_split)
-		devlink_port_split_set(&port->dl_port, eth_port.label_port);
+	devlink_port_attrs_set(&port->dl_port, DEVLINK_PORT_FLAVOUR_PHYSICAL,
+			       eth_port.label_port, eth_port.is_split,
+			       eth_port.label_subport);
 
 	devlink = priv_to_devlink(app->pf);
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.c b/drivers/net/ethernet/netronome/nfp/nfp_main.c
index c4b1f344b4da..46b76d5a726c 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_main.c
@@ -55,6 +55,7 @@
 
 #include "nfpcore/nfp6000_pcie.h"
 
+#include "nfp_abi.h"
 #include "nfp_app.h"
 #include "nfp_main.h"
 #include "nfp_net.h"
@@ -75,6 +76,122 @@ static const struct pci_device_id nfp_pci_device_ids[] = {
 };
 MODULE_DEVICE_TABLE(pci, nfp_pci_device_ids);
 
+int nfp_pf_rtsym_read_optional(struct nfp_pf *pf, const char *format,
+			       unsigned int default_val)
+{
+	char name[256];
+	int err = 0;
+	u64 val;
+
+	snprintf(name, sizeof(name), format, nfp_cppcore_pcie_unit(pf->cpp));
+
+	val = nfp_rtsym_read_le(pf->rtbl, name, &err);
+	if (err) {
+		if (err == -ENOENT)
+			return default_val;
+		nfp_err(pf->cpp, "Unable to read symbol %s\n", name);
+		return err;
+	}
+
+	return val;
+}
+
+u8 __iomem *
+nfp_pf_map_rtsym(struct nfp_pf *pf, const char *name, const char *sym_fmt,
+		 unsigned int min_size, struct nfp_cpp_area **area)
+{
+	char pf_symbol[256];
+
+	snprintf(pf_symbol, sizeof(pf_symbol), sym_fmt,
+		 nfp_cppcore_pcie_unit(pf->cpp));
+
+	return nfp_rtsym_map(pf->rtbl, pf_symbol, name, min_size, area);
+}
+
+/* Callers should hold the devlink instance lock */
+int nfp_mbox_cmd(struct nfp_pf *pf, u32 cmd, void *in_data, u64 in_length,
+		 void *out_data, u64 out_length)
+{
+	unsigned long long addr;
+	unsigned long err_at;
+	u64 max_data_sz;
+	u32 val = 0;
+	u32 cpp_id;
+	int n, err;
+
+	if (!pf->mbox)
+		return -EOPNOTSUPP;
+
+	cpp_id = NFP_CPP_ISLAND_ID(pf->mbox->target, NFP_CPP_ACTION_RW, 0,
+				   pf->mbox->domain);
+	addr = pf->mbox->addr;
+	max_data_sz = pf->mbox->size - NFP_MBOX_SYM_MIN_SIZE;
+
+	/* Check if cmd field is clear */
+	err = nfp_cpp_readl(pf->cpp, cpp_id, addr + NFP_MBOX_CMD, &val);
+	if (err || val) {
+		nfp_warn(pf->cpp, "failed to issue command (%u): %u, err: %d\n",
+			 cmd, val, err);
+		return err ?: -EBUSY;
+	}
+
+	in_length = min(in_length, max_data_sz);
+	n = nfp_cpp_write(pf->cpp, cpp_id, addr + NFP_MBOX_DATA,
+			  in_data, in_length);
+	if (n != in_length)
+		return -EIO;
+	/* Write data_len and wipe reserved */
+	err = nfp_cpp_writeq(pf->cpp, cpp_id, addr + NFP_MBOX_DATA_LEN,
+			     in_length);
+	if (err)
+		return err;
+
+	/* Read back for ordering */
+	err = nfp_cpp_readl(pf->cpp, cpp_id, addr + NFP_MBOX_DATA_LEN, &val);
+	if (err)
+		return err;
+
+	/* Write cmd and wipe return value */
+	err = nfp_cpp_writeq(pf->cpp, cpp_id, addr + NFP_MBOX_CMD, cmd);
+	if (err)
+		return err;
+
+	err_at = jiffies + 5 * HZ;
+	while (true) {
+		/* Wait for command to go to 0 (NFP_MBOX_NO_CMD) */
+		err = nfp_cpp_readl(pf->cpp, cpp_id, addr + NFP_MBOX_CMD, &val);
+		if (err)
+			return err;
+		if (!val)
+			break;
+
+		if (time_is_before_eq_jiffies(err_at))
+			return -ETIMEDOUT;
+
+		msleep(5);
+	}
+
+	/* Copy output if any (could be error info, do it before reading ret) */
+	err = nfp_cpp_readl(pf->cpp, cpp_id, addr + NFP_MBOX_DATA_LEN, &val);
+	if (err)
+		return err;
+
+	out_length = min_t(u32, val, min(out_length, max_data_sz));
+	n = nfp_cpp_read(pf->cpp, cpp_id, addr + NFP_MBOX_DATA,
+			 out_data, out_length);
+	if (n != out_length)
+		return -EIO;
+
+	/* Check if there is an error */
+	err = nfp_cpp_readl(pf->cpp, cpp_id, addr + NFP_MBOX_RET, &val);
+	if (err)
+		return err;
+	if (val)
+		return -val;
+
+	return out_length;
+}
+
 static bool nfp_board_ready(struct nfp_pf *pf)
 {
 	const char *cp;
@@ -436,6 +553,25 @@ static void nfp_fw_unload(struct nfp_pf *pf)
 	nfp_nsp_close(nsp);
 }
 
+static int nfp_pf_find_rtsyms(struct nfp_pf *pf)
+{
+	char pf_symbol[256];
+	unsigned int pf_id;
+
+	pf_id = nfp_cppcore_pcie_unit(pf->cpp);
+
+	/* Optional per-PCI PF mailbox */
+	snprintf(pf_symbol, sizeof(pf_symbol), NFP_MBOX_SYM_NAME, pf_id);
+	pf->mbox = nfp_rtsym_lookup(pf->rtbl, pf_symbol);
+	if (pf->mbox && pf->mbox->size < NFP_MBOX_SYM_MIN_SIZE) {
+		nfp_err(pf->cpp, "PF mailbox symbol too small: %llu < %d\n",
+			pf->mbox->size, NFP_MBOX_SYM_MIN_SIZE);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int nfp_pci_probe(struct pci_dev *pdev,
 			 const struct pci_device_id *pci_id)
 {
@@ -486,6 +622,10 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 		goto err_disable_msix;
 	}
 
+	err = nfp_resource_table_init(pf->cpp);
+	if (err)
+		goto err_cpp_free;
+
 	pf->hwinfo = nfp_hwinfo_read(pf->cpp);
 
 	dev_info(&pdev->dev, "Assembly: %s%s%s-%s CPLD: %s\n",
@@ -506,6 +646,10 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 	pf->mip = nfp_mip_open(pf->cpp);
 	pf->rtbl = __nfp_rtsym_table_read(pf->cpp, pf->mip);
 
+	err = nfp_pf_find_rtsyms(pf);
+	if (err)
+		goto err_fw_unload;
+
 	pf->dump_flag = NFP_DUMP_NSP_DIAG;
 	pf->dumpspec = nfp_net_dump_load_dumpspec(pf->cpp, pf->rtbl);
 
@@ -548,6 +692,7 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 	vfree(pf->dumpspec);
 err_hwinfo_free:
 	kfree(pf->hwinfo);
+err_cpp_free:
 	nfp_cpp_free(pf->cpp);
 err_disable_msix:
 	destroy_workqueue(pf->wq);
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.h b/drivers/net/ethernet/netronome/nfp/nfp_main.h
index 42211083b51f..595b3dc280e3 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_main.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_main.h
@@ -46,10 +46,10 @@
 #include <linux/mutex.h>
 #include <linux/pci.h>
 #include <linux/workqueue.h>
+#include <net/devlink.h>
 
 struct dentry;
 struct device;
-struct devlink_ops;
 struct pci_dev;
 
 struct nfp_cpp;
@@ -60,7 +60,9 @@ struct nfp_mip;
 struct nfp_net;
 struct nfp_nsp_identify;
 struct nfp_port;
+struct nfp_rtsym;
 struct nfp_rtsym_table;
+struct nfp_shared_buf;
 
 /**
  * struct nfp_dumpspec - NFP FW dump specification structure
@@ -87,6 +89,7 @@ struct nfp_dumpspec {
  * @vf_cfg_mem:		Pointer to mapped VF configuration area
  * @vfcfg_tbl2_area:	Pointer to the CPP area for the VF config table
  * @vfcfg_tbl2:		Pointer to mapped VF config table
+ * @mbox:		RTSym of per-PCI PF mailbox (under devlink lock)
  * @irq_entries:	Array of MSI-X entries for all vNICs
  * @limit_vfs:		Number of VFs supported by firmware (~0 for PCI limit)
  * @num_vfs:		Number of SR-IOV VFs enabled
@@ -108,6 +111,8 @@ struct nfp_dumpspec {
  * @ports:		Linked list of port structures (struct nfp_port)
  * @wq:			Workqueue for running works which need to grab @lock
  * @port_refresh_work:	Work entry for taking netdevs out
+ * @shared_bufs:	Array of shared buffer structures if FW has any SBs
+ * @num_shared_bufs:	Number of elements in @shared_bufs
  * @lock:		Protects all fields which may change after probe
  */
 struct nfp_pf {
@@ -127,6 +132,8 @@ struct nfp_pf {
 	struct nfp_cpp_area *vfcfg_tbl2_area;
 	u8 __iomem *vfcfg_tbl2;
 
+	const struct nfp_rtsym *mbox;
+
 	struct msix_entry *irq_entries;
 
 	unsigned int limit_vfs;
@@ -158,6 +165,9 @@ struct nfp_pf {
 	struct workqueue_struct *wq;
 	struct work_struct port_refresh_work;
 
+	struct nfp_shared_buf *shared_bufs;
+	unsigned int num_shared_bufs;
+
 	struct mutex lock;
 };
 
@@ -177,6 +187,14 @@ nfp_net_get_mac_addr(struct nfp_pf *pf, struct net_device *netdev,
 
 bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb);
 
+int nfp_pf_rtsym_read_optional(struct nfp_pf *pf, const char *format,
+			       unsigned int default_val);
+u8 __iomem *
+nfp_pf_map_rtsym(struct nfp_pf *pf, const char *name, const char *sym_fmt,
+		 unsigned int min_size, struct nfp_cpp_area **area);
+int nfp_mbox_cmd(struct nfp_pf *pf, u32 cmd, void *in_data, u64 in_length,
+		 void *out_data, u64 out_length);
+
 enum nfp_dump_diag {
 	NFP_DUMP_NSP_DIAG = 0,
 };
@@ -188,4 +206,11 @@ s64 nfp_net_dump_calculate_size(struct nfp_pf *pf, struct nfp_dumpspec *spec,
 int nfp_net_dump_populate_buffer(struct nfp_pf *pf, struct nfp_dumpspec *spec,
 				 struct ethtool_dump *dump_param, void *dest);
 
+int nfp_shared_buf_register(struct nfp_pf *pf);
+void nfp_shared_buf_unregister(struct nfp_pf *pf);
+int nfp_shared_buf_pool_get(struct nfp_pf *pf, unsigned int sb, u16 pool_index,
+			    struct devlink_sb_pool_info *pool_info);
+int nfp_shared_buf_pool_set(struct nfp_pf *pf, unsigned int sb,
+			    u16 pool_index, u32 size,
+			    enum devlink_sb_threshold_type threshold_type);
 #endif /* NFP_MAIN_H */
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h
index bd7d8ae31e17..57cb035dcc6d 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h
@@ -545,6 +545,7 @@ struct nfp_net_dp {
 /**
  * struct nfp_net - NFP network device structure
  * @dp:			Datapath structure
+ * @id:			vNIC id within the PF (0 for VFs)
  * @fw_ver:		Firmware version
  * @cap:                Capabilities advertised by the Firmware
  * @max_mtu:            Maximum support MTU advertised by the Firmware
@@ -597,6 +598,8 @@ struct nfp_net {
 
 	struct nfp_net_fw_version fw_ver;
 
+	u32 id;
+
 	u32 cap;
 	u32 max_mtu;
 
@@ -909,7 +912,7 @@ int nfp_net_ring_reconfig(struct nfp_net *nn, struct nfp_net_dp *new,
 void nfp_net_debugfs_create(void);
 void nfp_net_debugfs_destroy(void);
 struct dentry *nfp_net_debugfs_device_add(struct pci_dev *pdev);
-void nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir, int id);
+void nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir);
 void nfp_net_debugfs_dir_clean(struct dentry **dir);
 #else
 static inline void nfp_net_debugfs_create(void)
@@ -926,7 +929,7 @@ static inline struct dentry *nfp_net_debugfs_device_add(struct pci_dev *pdev)
 }
 
 static inline void
-nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir, int id)
+nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir)
 {
 }
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index 1eb6549f2a54..75110c8d6a90 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -1722,7 +1722,7 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
 
 			act = bpf_prog_run_xdp(xdp_prog, &xdp);
 
-			pkt_len -= xdp.data - orig_data;
+			pkt_len = xdp.data_end - xdp.data;
 			pkt_off += xdp.data - orig_data;
 
 			switch (act) {
@@ -3277,6 +3277,25 @@ nfp_net_features_check(struct sk_buff *skb, struct net_device *dev,
 	return features;
 }
 
+static int
+nfp_net_get_phys_port_name(struct net_device *netdev, char *name, size_t len)
+{
+	struct nfp_net *nn = netdev_priv(netdev);
+	int n;
+
+	if (nn->port)
+		return nfp_port_get_phys_port_name(netdev, name, len);
+
+	if (nn->dp.is_vf)
+		return -EOPNOTSUPP;
+
+	n = snprintf(name, len, "n%d", nn->id);
+	if (n >= len)
+		return -EINVAL;
+
+	return 0;
+}
+
 /**
  * nfp_net_set_vxlan_port() - set vxlan port in SW and reconfigure HW
  * @nn:   NFP Net device to reconfigure
@@ -3475,7 +3494,7 @@ const struct net_device_ops nfp_net_netdev_ops = {
 	.ndo_set_mac_address	= nfp_net_set_mac_address,
 	.ndo_set_features	= nfp_net_set_features,
 	.ndo_features_check	= nfp_net_features_check,
-	.ndo_get_phys_port_name	= nfp_port_get_phys_port_name,
+	.ndo_get_phys_port_name	= nfp_net_get_phys_port_name,
 	.ndo_udp_tunnel_add	= nfp_net_add_vxlan_port,
 	.ndo_udp_tunnel_del	= nfp_net_del_vxlan_port,
 	.ndo_bpf		= nfp_net_xdp,
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c b/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c
index 67cdd8330c59..099b63d67451 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c
@@ -201,7 +201,7 @@ static const struct file_operations nfp_xdp_q_fops = {
 	.llseek = seq_lseek
 };
 
-void nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir, int id)
+void nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir)
 {
 	struct dentry *queues, *tx, *rx, *xdp;
 	char name[20];
@@ -211,7 +211,7 @@ void nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir, int id)
 		return;
 
 	if (nfp_net_is_data_vnic(nn))
-		sprintf(name, "vnic%d", id);
+		sprintf(name, "vnic%d", nn->id);
 	else
 		strcpy(name, "ctrl-vnic");
 	nn->debugfs_dir = debugfs_create_dir(name, ddir);
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
index c9016419bfa0..26d1cc4e2906 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
@@ -437,7 +437,7 @@ static int nfp_net_set_ringparam(struct net_device *netdev,
 	return nfp_net_set_ring_size(nn, rxd_cnt, txd_cnt);
 }
 
-static __printf(2, 3) u8 *nfp_pr_et(u8 *data, const char *fmt, ...)
+__printf(2, 3) u8 *nfp_pr_et(u8 *data, const char *fmt, ...)
 {
 	va_list args;
 
@@ -637,6 +637,7 @@ static void nfp_net_get_strings(struct net_device *netdev,
 						     nn->dp.num_tx_rings,
 						     false);
 		data = nfp_mac_get_stats_strings(netdev, data);
+		data = nfp_app_port_get_stats_strings(nn->port, data);
 		break;
 	}
 }
@@ -651,6 +652,7 @@ nfp_net_get_stats(struct net_device *netdev, struct ethtool_stats *stats,
 	data = nfp_vnic_get_hw_stats(data, nn->dp.ctrl_bar,
 				     nn->dp.num_rx_rings, nn->dp.num_tx_rings);
 	data = nfp_mac_get_stats(netdev, data);
+	data = nfp_app_port_get_stats(nn->port, data);
 }
 
 static int nfp_net_get_sset_count(struct net_device *netdev, int sset)
@@ -662,7 +664,8 @@ static int nfp_net_get_sset_count(struct net_device *netdev, int sset)
 		return nfp_vnic_get_sw_stats_count(netdev) +
 		       nfp_vnic_get_hw_stats_count(nn->dp.num_rx_rings,
 						   nn->dp.num_tx_rings) +
-		       nfp_mac_get_stats_count(netdev);
+		       nfp_mac_get_stats_count(netdev) +
+		       nfp_app_port_get_stats_count(nn->port);
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -679,6 +682,7 @@ static void nfp_port_get_strings(struct net_device *netdev,
 			data = nfp_vnic_get_hw_stats_strings(data, 0, 0, true);
 		else
 			data = nfp_mac_get_stats_strings(netdev, data);
+		data = nfp_app_port_get_stats_strings(port, data);
 		break;
 	}
 }
@@ -693,6 +697,7 @@ nfp_port_get_stats(struct net_device *netdev, struct ethtool_stats *stats,
 		data = nfp_vnic_get_hw_stats(data, port->vnic, 0, 0);
 	else
 		data = nfp_mac_get_stats(netdev, data);
+	data = nfp_app_port_get_stats(port, data);
 }
 
 static int nfp_port_get_sset_count(struct net_device *netdev, int sset)
@@ -706,6 +711,7 @@ static int nfp_port_get_sset_count(struct net_device *netdev, int sset)
 			count = nfp_vnic_get_hw_stats_count(0, 0);
 		else
 			count = nfp_mac_get_stats_count(netdev);
+		count += nfp_app_port_get_stats_count(port);
 		return count;
 	default:
 		return -EOPNOTSUPP;
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
index 45cd2092e498..28516eecccc8 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
@@ -101,48 +101,15 @@ nfp_net_find_port(struct nfp_eth_table *eth_tbl, unsigned int index)
 	return NULL;
 }
 
-static int
-nfp_net_pf_rtsym_read_optional(struct nfp_pf *pf, const char *format,
-			       unsigned int default_val)
-{
-	char name[256];
-	int err = 0;
-	u64 val;
-
-	snprintf(name, sizeof(name), format, nfp_cppcore_pcie_unit(pf->cpp));
-
-	val = nfp_rtsym_read_le(pf->rtbl, name, &err);
-	if (err) {
-		if (err == -ENOENT)
-			return default_val;
-		nfp_err(pf->cpp, "Unable to read symbol %s\n", name);
-		return err;
-	}
-
-	return val;
-}
-
 static int nfp_net_pf_get_num_ports(struct nfp_pf *pf)
 {
-	return nfp_net_pf_rtsym_read_optional(pf, "nfd_cfg_pf%u_num_ports", 1);
+	return nfp_pf_rtsym_read_optional(pf, "nfd_cfg_pf%u_num_ports", 1);
 }
 
 static int nfp_net_pf_get_app_id(struct nfp_pf *pf)
 {
-	return nfp_net_pf_rtsym_read_optional(pf, "_pf%u_net_app_id",
-					      NFP_APP_CORE_NIC);
-}
-
-static u8 __iomem *
-nfp_net_pf_map_rtsym(struct nfp_pf *pf, const char *name, const char *sym_fmt,
-		     unsigned int min_size, struct nfp_cpp_area **area)
-{
-	char pf_symbol[256];
-
-	snprintf(pf_symbol, sizeof(pf_symbol), sym_fmt,
-		 nfp_cppcore_pcie_unit(pf->cpp));
-
-	return nfp_rtsym_map(pf->rtbl, pf_symbol, name, min_size, area);
+	return nfp_pf_rtsym_read_optional(pf, "_pf%u_net_app_id",
+					  NFP_APP_CORE_NIC);
 }
 
 static void nfp_net_pf_free_vnic(struct nfp_pf *pf, struct nfp_net *nn)
@@ -211,11 +178,13 @@ nfp_net_pf_init_vnic(struct nfp_pf *pf, struct nfp_net *nn, unsigned int id)
 {
 	int err;
 
+	nn->id = id;
+
 	err = nfp_net_init(nn);
 	if (err)
 		return err;
 
-	nfp_net_debugfs_vnic_add(nn, pf->ddir, id);
+	nfp_net_debugfs_vnic_add(nn, pf->ddir);
 
 	if (nn->port) {
 		err = nfp_devlink_port_register(pf->app, nn->port);
@@ -379,9 +348,8 @@ nfp_net_pf_app_init(struct nfp_pf *pf, u8 __iomem *qc_bar, unsigned int stride)
 	if (!nfp_app_needs_ctrl_vnic(pf->app))
 		return 0;
 
-	ctrl_bar = nfp_net_pf_map_rtsym(pf, "net.ctrl", "_pf%u_net_ctrl_bar",
-					NFP_PF_CSR_SLICE_SIZE,
-					&pf->ctrl_vnic_bar);
+	ctrl_bar = nfp_pf_map_rtsym(pf, "net.ctrl", "_pf%u_net_ctrl_bar",
+				    NFP_PF_CSR_SLICE_SIZE, &pf->ctrl_vnic_bar);
 	if (IS_ERR(ctrl_bar)) {
 		nfp_err(pf->cpp, "Failed to find ctrl vNIC memory symbol\n");
 		err = PTR_ERR(ctrl_bar);
@@ -507,8 +475,8 @@ static int nfp_net_pci_map_mem(struct nfp_pf *pf)
 	int err;
 
 	min_size = pf->max_data_vnics * NFP_PF_CSR_SLICE_SIZE;
-	mem = nfp_net_pf_map_rtsym(pf, "net.bar0", "_pf%d_net_bar0",
-				   min_size, &pf->data_vnic_bar);
+	mem = nfp_pf_map_rtsym(pf, "net.bar0", "_pf%d_net_bar0",
+			       min_size, &pf->data_vnic_bar);
 	if (IS_ERR(mem)) {
 		nfp_err(pf->cpp, "Failed to find data vNIC memory symbol\n");
 		return PTR_ERR(mem);
@@ -528,10 +496,9 @@ static int nfp_net_pci_map_mem(struct nfp_pf *pf)
 		}
 	}
 
-	pf->vf_cfg_mem = nfp_net_pf_map_rtsym(pf, "net.vfcfg",
-					      "_pf%d_net_vf_bar",
-					      NFP_NET_CFG_BAR_SZ *
-					      pf->limit_vfs, &pf->vf_cfg_bar);
+	pf->vf_cfg_mem = nfp_pf_map_rtsym(pf, "net.vfcfg", "_pf%d_net_vf_bar",
+					  NFP_NET_CFG_BAR_SZ * pf->limit_vfs,
+					  &pf->vf_cfg_bar);
 	if (IS_ERR(pf->vf_cfg_mem)) {
 		if (PTR_ERR(pf->vf_cfg_mem) != -ENOENT) {
 			err = PTR_ERR(pf->vf_cfg_mem);
@@ -541,9 +508,9 @@ static int nfp_net_pci_map_mem(struct nfp_pf *pf)
 	}
 
 	min_size = NFP_NET_VF_CFG_SZ * pf->limit_vfs + NFP_NET_VF_CFG_MB_SZ;
-	pf->vfcfg_tbl2 = nfp_net_pf_map_rtsym(pf, "net.vfcfg_tbl2",
-					      "_pf%d_net_vf_cfg2",
-					      min_size, &pf->vfcfg_tbl2_area);
+	pf->vfcfg_tbl2 = nfp_pf_map_rtsym(pf, "net.vfcfg_tbl2",
+					  "_pf%d_net_vf_cfg2",
+					  min_size, &pf->vfcfg_tbl2_area);
 	if (IS_ERR(pf->vfcfg_tbl2)) {
 		if (PTR_ERR(pf->vfcfg_tbl2) != -ENOENT) {
 			err = PTR_ERR(pf->vfcfg_tbl2);
@@ -763,6 +730,10 @@ int nfp_net_pci_probe(struct nfp_pf *pf)
 	if (err)
 		goto err_app_clean;
 
+	err = nfp_shared_buf_register(pf);
+	if (err)
+		goto err_devlink_unreg;
+
 	mutex_lock(&pf->lock);
 	pf->ddir = nfp_net_debugfs_device_add(pf->pdev);
 
@@ -796,6 +767,8 @@ int nfp_net_pci_probe(struct nfp_pf *pf)
 err_clean_ddir:
 	nfp_net_debugfs_dir_clean(&pf->ddir);
 	mutex_unlock(&pf->lock);
+	nfp_shared_buf_unregister(pf);
+err_devlink_unreg:
 	cancel_work_sync(&pf->port_refresh_work);
 	devlink_unregister(devlink);
 err_app_clean:
@@ -823,6 +796,7 @@ void nfp_net_pci_remove(struct nfp_pf *pf)
 
 	mutex_unlock(&pf->lock);
 
+	nfp_shared_buf_unregister(pf);
 	devlink_unregister(priv_to_devlink(pf));
 
 	nfp_net_pf_free_irqs(pf);
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
index 0cd077addb26..d7b712f6362f 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
@@ -277,6 +277,7 @@ const struct net_device_ops nfp_repr_netdev_ops = {
 	.ndo_get_vf_config	= nfp_app_get_vf_config,
 	.ndo_set_vf_link_state	= nfp_app_set_vf_link_state,
 	.ndo_set_features	= nfp_port_set_features,
+	.ndo_set_mac_address    = eth_mac_addr,
 };
 
 static void nfp_repr_clean(struct nfp_repr *repr)
@@ -348,18 +349,24 @@ int nfp_repr_init(struct nfp_app *app, struct net_device *netdev,
 	return err;
 }
 
-static void nfp_repr_free(struct nfp_repr *repr)
+static void __nfp_repr_free(struct nfp_repr *repr)
 {
 	free_percpu(repr->stats);
 	free_netdev(repr->netdev);
 }
 
-struct net_device *nfp_repr_alloc(struct nfp_app *app)
+void nfp_repr_free(struct net_device *netdev)
+{
+	__nfp_repr_free(netdev_priv(netdev));
+}
+
+struct net_device *
+nfp_repr_alloc_mqs(struct nfp_app *app, unsigned int txqs, unsigned int rxqs)
 {
 	struct net_device *netdev;
 	struct nfp_repr *repr;
 
-	netdev = alloc_etherdev(sizeof(*repr));
+	netdev = alloc_etherdev_mqs(sizeof(*repr), txqs, rxqs);
 	if (!netdev)
 		return NULL;
 
@@ -380,12 +387,12 @@ struct net_device *nfp_repr_alloc(struct nfp_app *app)
 	return NULL;
 }
 
-static void nfp_repr_clean_and_free(struct nfp_repr *repr)
+void nfp_repr_clean_and_free(struct nfp_repr *repr)
 {
 	nfp_info(repr->app->cpp, "Destroying Representor(%s)\n",
 		 repr->netdev->name);
 	nfp_repr_clean(repr);
-	nfp_repr_free(repr);
+	__nfp_repr_free(repr);
 }
 
 void nfp_reprs_clean_and_free(struct nfp_app *app, struct nfp_reprs *reprs)
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h
index a621e8ff528e..1bf2b18109ab 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h
@@ -76,6 +76,7 @@ struct nfp_repr_pcpu_stats {
  * @port:	Port of representor
  * @app:	APP handle
  * @stats:	Statistic of packets hitting CPU
+ * @app_priv:	Pointer for APP data
  */
 struct nfp_repr {
 	struct net_device *netdev;
@@ -83,6 +84,7 @@ struct nfp_repr {
 	struct nfp_port *port;
 	struct nfp_app *app;
 	struct nfp_repr_pcpu_stats __percpu *stats;
+	void *app_priv;
 };
 
 /**
@@ -123,11 +125,18 @@ void nfp_repr_inc_rx_stats(struct net_device *netdev, unsigned int len);
 int nfp_repr_init(struct nfp_app *app, struct net_device *netdev,
 		  u32 cmsg_port_id, struct nfp_port *port,
 		  struct net_device *pf_netdev);
-struct net_device *nfp_repr_alloc(struct nfp_app *app);
+void nfp_repr_free(struct net_device *netdev);
+struct net_device *
+nfp_repr_alloc_mqs(struct nfp_app *app, unsigned int txqs, unsigned int rxqs);
+void nfp_repr_clean_and_free(struct nfp_repr *repr);
 void nfp_reprs_clean_and_free(struct nfp_app *app, struct nfp_reprs *reprs);
 void nfp_reprs_clean_and_free_by_type(struct nfp_app *app,
 				      enum nfp_repr_type type);
 struct nfp_reprs *nfp_reprs_alloc(unsigned int num_reprs);
 int nfp_reprs_resync_phys_ports(struct nfp_app *app);
 
+static inline struct net_device *nfp_repr_alloc(struct nfp_app *app)
+{
+	return nfp_repr_alloc_mqs(app, 1, 1);
+}
 #endif /* NFP_NET_REPR_H */
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
index b802a1d55449..68928c86b698 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
@@ -283,7 +283,7 @@ static int nfp_netvf_pci_probe(struct pci_dev *pdev,
 
 	nfp_net_info(nn);
 	vf->ddir = nfp_net_debugfs_device_add(pdev);
-	nfp_net_debugfs_vnic_add(nn, vf->ddir, 0);
+	nfp_net_debugfs_vnic_add(nn, vf->ddir);
 
 	return 0;
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.c b/drivers/net/ethernet/netronome/nfp/nfp_port.c
index 7bd8be5c833b..9c1298114c70 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_port.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_port.c
@@ -181,7 +181,11 @@ nfp_port_get_phys_port_name(struct net_device *netdev, char *name, size_t len)
 				     eth_port->label_subport);
 		break;
 	case NFP_PORT_PF_PORT:
-		n = snprintf(name, len, "pf%d", port->pf_id);
+		if (!port->pf_split)
+			n = snprintf(name, len, "pf%d", port->pf_id);
+		else
+			n = snprintf(name, len, "pf%ds%d", port->pf_id,
+				     port->pf_split_id);
 		break;
 	case NFP_PORT_VF_PORT:
 		n = snprintf(name, len, "pf%dvf%d", port->pf_id, port->vf_id);
@@ -218,6 +222,8 @@ int nfp_port_configure(struct net_device *netdev, bool configed)
 	eth_port = __nfp_port_get_eth_port(port);
 	if (!eth_port)
 		return 0;
+	if (port->eth_forced)
+		return 0;
 
 	err = nfp_eth_set_configured(port->app->cpp, eth_port->index, configed);
 	return err < 0 && err != -EOPNOTSUPP ? err : 0;
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.h b/drivers/net/ethernet/netronome/nfp/nfp_port.h
index fa7e669a969c..51f10ae2d53e 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_port.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_port.h
@@ -77,10 +77,13 @@ enum nfp_port_flags {
  * @app:	backpointer to the app structure
  * @dl_port:	devlink port structure
  * @eth_id:	for %NFP_PORT_PHYS_PORT port ID in NFP enumeration scheme
+ * @eth_forced:	for %NFP_PORT_PHYS_PORT port is forced UP or DOWN, don't change
  * @eth_port:	for %NFP_PORT_PHYS_PORT translated ETH Table port entry
  * @eth_stats:	for %NFP_PORT_PHYS_PORT MAC stats if available
  * @pf_id:	for %NFP_PORT_PF_PORT, %NFP_PORT_VF_PORT ID of the PCI PF (0-3)
  * @vf_id:	for %NFP_PORT_VF_PORT ID of the PCI VF within @pf_id
+ * @pf_split:	for %NFP_PORT_PF_PORT %true if PCI PF has more than one vNIC
+ * @pf_split_id:for %NFP_PORT_PF_PORT ID of PCI PF vNIC (valid if @pf_split)
  * @vnic:	for %NFP_PORT_PF_PORT, %NFP_PORT_VF_PORT vNIC ctrl memory
  * @port_list:	entry on pf's list of ports
  */
@@ -99,6 +102,7 @@ struct nfp_port {
 		/* NFP_PORT_PHYS_PORT */
 		struct {
 			unsigned int eth_id;
+			bool eth_forced;
 			struct nfp_eth_table_port *eth_port;
 			u8 __iomem *eth_stats;
 		};
@@ -106,6 +110,8 @@ struct nfp_port {
 		struct {
 			unsigned int pf_id;
 			unsigned int vf_id;
+			bool pf_split;
+			unsigned int pf_split_id;
 			u8 __iomem *vnic;
 		};
 	};
@@ -116,6 +122,8 @@ struct nfp_port {
 extern const struct ethtool_ops nfp_port_ethtool_ops;
 extern const struct switchdev_ops nfp_port_switchdev_ops;
 
+__printf(2, 3) u8 *nfp_pr_et(u8 *data, const char *fmt, ...);
+
 int nfp_port_setup_tc(struct net_device *netdev, enum tc_setup_type type,
 		      void *type_data);
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_shared_buf.c b/drivers/net/ethernet/netronome/nfp/nfp_shared_buf.c
new file mode 100644
index 000000000000..0ecd83705368
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfp_shared_buf.c
@@ -0,0 +1,180 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+/*
+ * Copyright (C) 2018 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <net/devlink.h>
+
+#include "nfpcore/nfp_cpp.h"
+#include "nfpcore/nfp_nffw.h"
+#include "nfp_abi.h"
+#include "nfp_app.h"
+#include "nfp_main.h"
+
+static u32 nfp_shared_buf_pool_unit(struct nfp_pf *pf, unsigned int sb)
+{
+	__le32 sb_id = cpu_to_le32(sb);
+	unsigned int i;
+
+	for (i = 0; i < pf->num_shared_bufs; i++)
+		if (pf->shared_bufs[i].id == sb_id)
+			return le32_to_cpu(pf->shared_bufs[i].pool_size_unit);
+
+	WARN_ON_ONCE(1);
+	return 0;
+}
+
+int nfp_shared_buf_pool_get(struct nfp_pf *pf, unsigned int sb, u16 pool_index,
+			    struct devlink_sb_pool_info *pool_info)
+{
+	struct nfp_shared_buf_pool_info_get get_data;
+	struct nfp_shared_buf_pool_id id = {
+		.shared_buf	= cpu_to_le32(sb),
+		.pool		= cpu_to_le32(pool_index),
+	};
+	unsigned int unit_size;
+	int n;
+
+	unit_size = nfp_shared_buf_pool_unit(pf, sb);
+	if (!unit_size)
+		return -EINVAL;
+
+	n = nfp_mbox_cmd(pf, NFP_MBOX_POOL_GET, &id, sizeof(id),
+			 &get_data, sizeof(get_data));
+	if (n < 0)
+		return n;
+	if (n < sizeof(get_data))
+		return -EIO;
+
+	pool_info->pool_type = le32_to_cpu(get_data.pool_type);
+	pool_info->threshold_type = le32_to_cpu(get_data.threshold_type);
+	pool_info->size = le32_to_cpu(get_data.size) * unit_size;
+
+	return 0;
+}
+
+int nfp_shared_buf_pool_set(struct nfp_pf *pf, unsigned int sb,
+			    u16 pool_index, u32 size,
+			    enum devlink_sb_threshold_type threshold_type)
+{
+	struct nfp_shared_buf_pool_info_set set_data = {
+		.id = {
+			.shared_buf	= cpu_to_le32(sb),
+			.pool		= cpu_to_le32(pool_index),
+		},
+		.threshold_type	= cpu_to_le32(threshold_type),
+	};
+	unsigned int unit_size;
+
+	unit_size = nfp_shared_buf_pool_unit(pf, sb);
+	if (!unit_size || size % unit_size)
+		return -EINVAL;
+	set_data.size = cpu_to_le32(size / unit_size);
+
+	return nfp_mbox_cmd(pf, NFP_MBOX_POOL_SET, &set_data, sizeof(set_data),
+			    NULL, 0);
+}
+
+int nfp_shared_buf_register(struct nfp_pf *pf)
+{
+	struct devlink *devlink = priv_to_devlink(pf);
+	unsigned int i, num_entries, entry_sz;
+	struct nfp_cpp_area *sb_desc_area;
+	u8 __iomem *sb_desc;
+	int n, err;
+
+	if (!pf->mbox)
+		return 0;
+
+	n = nfp_pf_rtsym_read_optional(pf, NFP_SHARED_BUF_COUNT_SYM_NAME, 0);
+	if (n <= 0)
+		return n;
+	num_entries = n;
+
+	sb_desc = nfp_pf_map_rtsym(pf, "sb_tbl", NFP_SHARED_BUF_TABLE_SYM_NAME,
+				   num_entries * sizeof(pf->shared_bufs[0]),
+				   &sb_desc_area);
+	if (IS_ERR(sb_desc))
+		return PTR_ERR(sb_desc);
+
+	entry_sz = nfp_cpp_area_size(sb_desc_area) / num_entries;
+
+	pf->shared_bufs = kmalloc_array(num_entries, sizeof(pf->shared_bufs[0]),
+					GFP_KERNEL);
+	if (!pf->shared_bufs) {
+		err = -ENOMEM;
+		goto err_release_area;
+	}
+
+	for (i = 0; i < num_entries; i++) {
+		struct nfp_shared_buf *sb = &pf->shared_bufs[i];
+
+		/* Entries may be larger in future FW */
+		memcpy_fromio(sb, sb_desc + i * entry_sz, sizeof(*sb));
+
+		err = devlink_sb_register(devlink,
+					  le32_to_cpu(sb->id),
+					  le32_to_cpu(sb->size),
+					  le16_to_cpu(sb->ingress_pools_count),
+					  le16_to_cpu(sb->egress_pools_count),
+					  le16_to_cpu(sb->ingress_tc_count),
+					  le16_to_cpu(sb->egress_tc_count));
+		if (err)
+			goto err_unreg_prev;
+	}
+	pf->num_shared_bufs = num_entries;
+
+	nfp_cpp_area_release_free(sb_desc_area);
+
+	return 0;
+
+err_unreg_prev:
+	while (i--)
+		devlink_sb_unregister(devlink,
+				      le32_to_cpu(pf->shared_bufs[i].id));
+	kfree(pf->shared_bufs);
+err_release_area:
+	nfp_cpp_area_release_free(sb_desc_area);
+	return err;
+}
+
+void nfp_shared_buf_unregister(struct nfp_pf *pf)
+{
+	struct devlink *devlink = priv_to_devlink(pf);
+	unsigned int i;
+
+	for (i = 0; i < pf->num_shared_bufs; i++)
+		devlink_sb_unregister(devlink,
+				      le32_to_cpu(pf->shared_bufs[i].id));
+	kfree(pf->shared_bufs);
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h
index ced62d112aa2..f44d0a857314 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h
@@ -94,6 +94,8 @@ int nfp_nsp_read_sensors(struct nfp_nsp *state, unsigned int sensor_mask,
 /* MAC Statistics Accumulator */
 #define NFP_RESOURCE_MAC_STATISTICS	"mac.stat"
 
+int nfp_resource_table_init(struct nfp_cpp *cpp);
+
 struct nfp_resource *
 nfp_resource_acquire(struct nfp_cpp *cpp, const char *name);
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c
index cd678323bacb..749655c329b2 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c
@@ -933,7 +933,6 @@ static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr,
 	u32 *wrptr32 = kernel_vaddr;
 	const u32 __iomem *rdptr32;
 	int n, width;
-	bool is_64;
 
 	priv = nfp_cpp_area_priv(area);
 	rdptr64 = priv->iomem + offset;
@@ -943,10 +942,15 @@ static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr,
 		return -EFAULT;
 
 	width = priv->width.read;
-
 	if (width <= 0)
 		return -EINVAL;
 
+	/* MU reads via a PCIe2CPP BAR support 32bit (and other) lengths */
+	if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) &&
+	    priv->action == NFP_CPP_ACTION_RW &&
+	    (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4))
+		width = TARGET_WIDTH_32;
+
 	/* Unaligned? Translate to an explicit access */
 	if ((priv->offset + offset) & (width - 1))
 		return nfp_cpp_explicit_read(nfp_cpp_area_cpp(area),
@@ -956,36 +960,29 @@ static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr,
 					     priv->offset + offset,
 					     kernel_vaddr, length, width);
 
-	is_64 = width == TARGET_WIDTH_64;
-
-	/* MU reads via a PCIe2CPP BAR supports 32bit (and other) lengths */
-	if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
-	    priv->action == NFP_CPP_ACTION_RW)
-		is_64 = false;
+	if (WARN_ON(!priv->bar))
+		return -EFAULT;
 
-	if (is_64) {
-		if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
-			return -EINVAL;
-	} else {
+	switch (width) {
+	case TARGET_WIDTH_32:
 		if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
 			return -EINVAL;
-	}
 
-	if (WARN_ON(!priv->bar))
-		return -EFAULT;
+		for (n = 0; n < length; n += sizeof(u32))
+			*wrptr32++ = __raw_readl(rdptr32++);
+		return n;
+#ifdef __raw_readq
+	case TARGET_WIDTH_64:
+		if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
+			return -EINVAL;
 
-	if (is_64)
-#ifndef __raw_readq
-		return -EINVAL;
-#else
 		for (n = 0; n < length; n += sizeof(u64))
 			*wrptr64++ = __raw_readq(rdptr64++);
+		return n;
 #endif
-	else
-		for (n = 0; n < length; n += sizeof(u32))
-			*wrptr32++ = __raw_readl(rdptr32++);
-
-	return n;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int
@@ -999,7 +996,6 @@ nfp6000_area_write(struct nfp_cpp_area *area,
 	struct nfp6000_area_priv *priv;
 	u32 __iomem *wrptr32;
 	int n, width;
-	bool is_64;
 
 	priv = nfp_cpp_area_priv(area);
 	wrptr64 = priv->iomem + offset;
@@ -1009,10 +1005,15 @@ nfp6000_area_write(struct nfp_cpp_area *area,
 		return -EFAULT;
 
 	width = priv->width.write;
-
 	if (width <= 0)
 		return -EINVAL;
 
+	/* MU writes via a PCIe2CPP BAR support 32bit (and other) lengths */
+	if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
+	    priv->action == NFP_CPP_ACTION_RW &&
+	    (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4))
+		width = TARGET_WIDTH_32;
+
 	/* Unaligned? Translate to an explicit access */
 	if ((priv->offset + offset) & (width - 1))
 		return nfp_cpp_explicit_write(nfp_cpp_area_cpp(area),
@@ -1022,40 +1023,33 @@ nfp6000_area_write(struct nfp_cpp_area *area,
 					      priv->offset + offset,
 					      kernel_vaddr, length, width);
 
-	is_64 = width == TARGET_WIDTH_64;
-
-	/* MU writes via a PCIe2CPP BAR supports 32bit (and other) lengths */
-	if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
-	    priv->action == NFP_CPP_ACTION_RW)
-		is_64 = false;
+	if (WARN_ON(!priv->bar))
+		return -EFAULT;
 
-	if (is_64) {
-		if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
-			return -EINVAL;
-	} else {
+	switch (width) {
+	case TARGET_WIDTH_32:
 		if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
 			return -EINVAL;
-	}
 
-	if (WARN_ON(!priv->bar))
-		return -EFAULT;
+		for (n = 0; n < length; n += sizeof(u32)) {
+			__raw_writel(*rdptr32++, wrptr32++);
+			wmb();
+		}
+		return n;
+#ifdef __raw_writeq
+	case TARGET_WIDTH_64:
+		if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
+			return -EINVAL;
 
-	if (is_64)
-#ifndef __raw_writeq
-		return -EINVAL;
-#else
 		for (n = 0; n < length; n += sizeof(u64)) {
 			__raw_writeq(*rdptr64++, wrptr64++);
 			wmb();
 		}
+		return n;
 #endif
-	else
-		for (n = 0; n < length; n += sizeof(u32)) {
-			__raw_writel(*rdptr32++, wrptr32++);
-			wmb();
-		}
-
-	return n;
+	default:
+		return -EINVAL;
+	}
 }
 
 struct nfp6000_explicit_priv {
@@ -1330,6 +1324,7 @@ struct nfp_cpp *nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev)
 	/*  Finished with card initialization. */
 	dev_info(&pdev->dev,
 		 "Netronome Flow Processor NFP4000/NFP6000 PCIe Card Probe\n");
+	pcie_print_link_status(pdev);
 
 	nfp = kzalloc(sizeof(*nfp), GFP_KERNEL);
 	if (!nfp) {
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h
index c8f2c064cce3..b0da3d436850 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h
@@ -87,6 +87,11 @@ struct resource;
 
 #define NFP_CPP_TARGET_ID_MASK          0x1f
 
+#define NFP_CPP_ATOMIC_RD(target, island) \
+	NFP_CPP_ISLAND_ID((target), 3, 0, (island))
+#define NFP_CPP_ATOMIC_WR(target, island) \
+	NFP_CPP_ISLAND_ID((target), 4, 0, (island))
+
 /**
  * NFP_CPP_ID() - pack target, token, and action into a CPP ID.
  * @target:     NFP CPP target id
@@ -295,6 +300,8 @@ void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex);
 int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex);
 int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex);
 int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex);
+int nfp_cpp_mutex_reclaim(struct nfp_cpp *cpp, int target,
+			  unsigned long long address);
 
 /**
  * nfp_cppcore_pcie_unit() - Get PCI Unit of a CPP handle
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_mutex.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_mutex.c
index cb28ac03e4ca..c88bf673cb76 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_mutex.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_mutex.c
@@ -59,6 +59,11 @@ static u32 nfp_mutex_unlocked(u16 interface)
 	return (u32)interface << 16 | 0x0000;
 }
 
+static u32 nfp_mutex_owner(u32 val)
+{
+	return val >> 16;
+}
+
 static bool nfp_mutex_is_locked(u32 val)
 {
 	return (val & 0xffff) == 0x000f;
@@ -351,3 +356,43 @@ int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex)
 
 	return nfp_mutex_is_locked(tmp) ? -EBUSY : -EINVAL;
 }
+
+/**
+ * nfp_cpp_mutex_reclaim() - Unlock mutex if held by local endpoint
+ * @cpp:	NFP CPP handle
+ * @target:	NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
+ * @address:	Offset into the address space of the NFP CPP target ID
+ *
+ * Release lock if held by local system.  Extreme care is advised, call only
+ * when no local lock users can exist.
+ *
+ * Return:      0 if the lock was OK, 1 if locked by us, -errno on invalid mutex
+ */
+int nfp_cpp_mutex_reclaim(struct nfp_cpp *cpp, int target,
+			  unsigned long long address)
+{
+	const u32 mur = NFP_CPP_ID(target, 3, 0);	/* atomic_read */
+	const u32 muw = NFP_CPP_ID(target, 4, 0);	/* atomic_write */
+	u16 interface = nfp_cpp_interface(cpp);
+	int err;
+	u32 tmp;
+
+	err = nfp_cpp_mutex_validate(interface, &target, address);
+	if (err)
+		return err;
+
+	/* Check lock */
+	err = nfp_cpp_readl(cpp, mur, address, &tmp);
+	if (err < 0)
+		return err;
+
+	if (nfp_mutex_is_unlocked(tmp) || nfp_mutex_owner(tmp) != interface)
+		return 0;
+
+	/* Bust the lock */
+	err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_unlocked(interface));
+	if (err < 0)
+		return err;
+
+	return 1;
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.h
index c9724fb7ea4b..df599d5b6bb3 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.h
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.h
@@ -100,6 +100,8 @@ nfp_rtsym_lookup(struct nfp_rtsym_table *rtbl, const char *name);
 
 u64 nfp_rtsym_read_le(struct nfp_rtsym_table *rtbl, const char *name,
 		      int *error);
+int nfp_rtsym_write_le(struct nfp_rtsym_table *rtbl, const char *name,
+		       u64 value);
 u8 __iomem *
 nfp_rtsym_map(struct nfp_rtsym_table *rtbl, const char *name, const char *id,
 	      unsigned int min_size, struct nfp_cpp_area **area);
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
index 7e14725055c7..2dd89dba9311 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
@@ -338,3 +338,62 @@ u64 nfp_resource_size(struct nfp_resource *res)
 {
 	return res->size;
 }
+
+/**
+ * nfp_resource_table_init() - Run initial checks on the resource table
+ * @cpp:	NFP CPP handle
+ *
+ * Start-of-day init procedure for resource table.  Must be called before
+ * any local resource table users may exist.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+int nfp_resource_table_init(struct nfp_cpp *cpp)
+{
+	struct nfp_cpp_mutex *dev_mutex;
+	int i, err;
+
+	err = nfp_cpp_mutex_reclaim(cpp, NFP_RESOURCE_TBL_TARGET,
+				    NFP_RESOURCE_TBL_BASE);
+	if (err < 0) {
+		nfp_err(cpp, "Error: failed to reclaim resource table mutex\n");
+		return err;
+	}
+	if (err)
+		nfp_warn(cpp, "Warning: busted main resource table mutex\n");
+
+	dev_mutex = nfp_cpp_mutex_alloc(cpp, NFP_RESOURCE_TBL_TARGET,
+					NFP_RESOURCE_TBL_BASE,
+					NFP_RESOURCE_TBL_KEY);
+	if (!dev_mutex)
+		return -ENOMEM;
+
+	if (nfp_cpp_mutex_lock(dev_mutex)) {
+		nfp_err(cpp, "Error: failed to claim resource table mutex\n");
+		nfp_cpp_mutex_free(dev_mutex);
+		return -EINVAL;
+	}
+
+	/* Resource 0 is the dev_mutex, start from 1 */
+	for (i = 1; i < NFP_RESOURCE_TBL_ENTRIES; i++) {
+		u64 addr = NFP_RESOURCE_TBL_BASE +
+			sizeof(struct nfp_resource_entry) * i;
+
+		err = nfp_cpp_mutex_reclaim(cpp, NFP_RESOURCE_TBL_TARGET, addr);
+		if (err < 0) {
+			nfp_err(cpp,
+				"Error: failed to reclaim resource %d mutex\n",
+				i);
+			goto err_unlock;
+		}
+		if (err)
+			nfp_warn(cpp, "Warning: busted resource %d mutex\n", i);
+	}
+
+	err = 0;
+err_unlock:
+	nfp_cpp_mutex_unlock(dev_mutex);
+	nfp_cpp_mutex_free(dev_mutex);
+
+	return err;
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c
index 46107aefad1c..9e34216578da 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c
@@ -286,6 +286,49 @@ u64 nfp_rtsym_read_le(struct nfp_rtsym_table *rtbl, const char *name,
 	return val;
 }
 
+/**
+ * nfp_rtsym_write_le() - Write an unsigned scalar value to a symbol
+ * @rtbl:	NFP RTsym table
+ * @name:	Symbol name
+ * @value:	Value to write
+ *
+ * Lookup a symbol and write a value to it. Symbol can be 4 or 8 bytes in size.
+ * If 4 bytes then the lower 32-bits of 'value' are used. Value will be
+ * written as simple little-endian unsigned value.
+ *
+ * Return: 0 on success or error code.
+ */
+int nfp_rtsym_write_le(struct nfp_rtsym_table *rtbl, const char *name,
+		       u64 value)
+{
+	const struct nfp_rtsym *sym;
+	int err;
+	u32 id;
+
+	sym = nfp_rtsym_lookup(rtbl, name);
+	if (!sym)
+		return -ENOENT;
+
+	id = NFP_CPP_ISLAND_ID(sym->target, NFP_CPP_ACTION_RW, 0, sym->domain);
+
+	switch (sym->size) {
+	case 4:
+		err = nfp_cpp_writel(rtbl->cpp, id, sym->addr, value);
+		break;
+	case 8:
+		err = nfp_cpp_writeq(rtbl->cpp, id, sym->addr, value);
+		break;
+	default:
+		nfp_err(rtbl->cpp,
+			"rtsym '%s' unsupported or non-scalar size: %lld\n",
+			name, sym->size);
+		err = -EINVAL;
+		break;
+	}
+
+	return err;
+}
+
 u8 __iomem *
 nfp_rtsym_map(struct nfp_rtsym_table *rtbl, const char *name, const char *id,
 	      unsigned int min_size, struct nfp_cpp_area **area)
diff --git a/drivers/net/ethernet/qlogic/qed/Makefile b/drivers/net/ethernet/qlogic/qed/Makefile
index c70cf2ad81c0..a0acb94d65f0 100644
--- a/drivers/net/ethernet/qlogic/qed/Makefile
+++ b/drivers/net/ethernet/qlogic/qed/Makefile
@@ -3,7 +3,7 @@ obj-$(CONFIG_QED) := qed.o
 
 qed-y := qed_cxt.o qed_dev.o qed_hw.o qed_init_fw_funcs.o qed_init_ops.o \
 	 qed_int.o qed_main.o qed_mcp.o qed_sp_commands.o qed_spq.o qed_l2.o \
-	 qed_selftest.o qed_dcbx.o qed_debug.o qed_ptp.o
+	 qed_selftest.o qed_dcbx.o qed_debug.o qed_ptp.o qed_mng_tlv.o
 qed-$(CONFIG_QED_SRIOV) += qed_sriov.o qed_vf.o
 qed-$(CONFIG_QED_LL2) += qed_ll2.o
 qed-$(CONFIG_QED_RDMA) += qed_roce.o qed_rdma.o qed_iwarp.o
diff --git a/drivers/net/ethernet/qlogic/qed/qed.h b/drivers/net/ethernet/qlogic/qed/qed.h
index e07460a68d30..00db3401b898 100644
--- a/drivers/net/ethernet/qlogic/qed/qed.h
+++ b/drivers/net/ethernet/qlogic/qed/qed.h
@@ -92,6 +92,8 @@ struct qed_eth_cb_ops;
 struct qed_dev_info;
 union qed_mcp_protocol_stats;
 enum qed_mcp_protocol_type;
+enum qed_mfw_tlv_type;
+union qed_mfw_tlv_data;
 
 /* helpers */
 #define QED_MFW_GET_FIELD(name, field) \
@@ -439,6 +441,59 @@ struct qed_fw_data {
 	u32			init_ops_size;
 };
 
+enum qed_mf_mode_bit {
+	/* Supports PF-classification based on tag */
+	QED_MF_OVLAN_CLSS,
+
+	/* Supports PF-classification based on MAC */
+	QED_MF_LLH_MAC_CLSS,
+
+	/* Supports PF-classification based on protocol type */
+	QED_MF_LLH_PROTO_CLSS,
+
+	/* Requires a default PF to be set */
+	QED_MF_NEED_DEF_PF,
+
+	/* Allow LL2 to multicast/broadcast */
+	QED_MF_LL2_NON_UNICAST,
+
+	/* Allow Cross-PF [& child VFs] Tx-switching */
+	QED_MF_INTER_PF_SWITCH,
+
+	/* Unified Fabtic Port support enabled */
+	QED_MF_UFP_SPECIFIC,
+
+	/* Disable Accelerated Receive Flow Steering (aRFS) */
+	QED_MF_DISABLE_ARFS,
+
+	/* Use vlan for steering */
+	QED_MF_8021Q_TAGGING,
+
+	/* Use stag for steering */
+	QED_MF_8021AD_TAGGING,
+
+	/* Allow DSCP to TC mapping */
+	QED_MF_DSCP_TO_TC_MAP,
+};
+
+enum qed_ufp_mode {
+	QED_UFP_MODE_ETS,
+	QED_UFP_MODE_VNIC_BW,
+	QED_UFP_MODE_UNKNOWN
+};
+
+enum qed_ufp_pri_type {
+	QED_UFP_PRI_OS,
+	QED_UFP_PRI_VNIC,
+	QED_UFP_PRI_UNKNOWN
+};
+
+struct qed_ufp_info {
+	enum qed_ufp_pri_type pri_type;
+	enum qed_ufp_mode mode;
+	u8 tc;
+};
+
 enum BAR_ID {
 	BAR_ID_0,		/* used for GRC */
 	BAR_ID_1		/* Used for doorbells */
@@ -460,6 +515,10 @@ struct qed_simd_fp_handler {
 	void	(*func)(void *);
 };
 
+enum qed_slowpath_wq_flag {
+	QED_SLOWPATH_MFW_TLV_REQ,
+};
+
 struct qed_hwfn {
 	struct qed_dev			*cdev;
 	u8				my_id;          /* ID inside the PF */
@@ -547,6 +606,8 @@ struct qed_hwfn {
 
 	struct qed_dcbx_info		*p_dcbx_info;
 
+	struct qed_ufp_info		ufp_info;
+
 	struct qed_dmae_info		dmae_info;
 
 	/* QM init */
@@ -587,6 +648,9 @@ struct qed_hwfn {
 #endif
 
 	struct z_stream_s		*stream;
+	struct workqueue_struct *slowpath_wq;
+	struct delayed_work slowpath_task;
+	unsigned long slowpath_task_flags;
 };
 
 struct pci_params {
@@ -669,10 +733,8 @@ struct qed_dev {
 	u8				num_funcs_in_port;
 
 	u8				path_id;
-	enum qed_mf_mode		mf_mode;
-#define IS_MF_DEFAULT(_p_hwfn)  (((_p_hwfn)->cdev)->mf_mode == QED_MF_DEFAULT)
-#define IS_MF_SI(_p_hwfn)       (((_p_hwfn)->cdev)->mf_mode == QED_MF_NPAR)
-#define IS_MF_SD(_p_hwfn)       (((_p_hwfn)->cdev)->mf_mode == QED_MF_OVLAN)
+
+	unsigned long			mf_bits;
 
 	int				pcie_width;
 	int				pcie_speed;
@@ -853,5 +915,9 @@ void qed_get_protocol_stats(struct qed_dev *cdev,
 			    union qed_mcp_protocol_stats *stats);
 int qed_slowpath_irq_req(struct qed_hwfn *hwfn);
 void qed_slowpath_irq_sync(struct qed_hwfn *p_hwfn);
+int qed_mfw_tlv_req(struct qed_hwfn *hwfn);
 
+int qed_mfw_fill_tlv_data(struct qed_hwfn *hwfn,
+			  enum qed_mfw_tlv_type type,
+			  union qed_mfw_tlv_data *tlv_data);
 #endif /* _QED_H */
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
index 449777f21237..8f31406ec894 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
@@ -274,8 +274,8 @@ qed_dcbx_process_tlv(struct qed_hwfn *p_hwfn,
 		     u32 pri_tc_tbl, int count, u8 dcbx_version)
 {
 	enum dcbx_protocol_type type;
+	bool enable, ieee, eth_tlv;
 	u8 tc, priority_map;
-	bool enable, ieee;
 	u16 protocol_id;
 	int priority;
 	int i;
@@ -283,6 +283,7 @@ qed_dcbx_process_tlv(struct qed_hwfn *p_hwfn,
 	DP_VERBOSE(p_hwfn, QED_MSG_DCB, "Num APP entries = %d\n", count);
 
 	ieee = (dcbx_version == DCBX_CONFIG_VERSION_IEEE);
+	eth_tlv = false;
 	/* Parse APP TLV */
 	for (i = 0; i < count; i++) {
 		protocol_id = QED_MFW_GET_FIELD(p_tbl[i].entry,
@@ -304,13 +305,22 @@ qed_dcbx_process_tlv(struct qed_hwfn *p_hwfn,
 			 * indication, but we only got here if there was an
 			 * app tlv for the protocol, so dcbx must be enabled.
 			 */
-			enable = !(type == DCBX_PROTOCOL_ETH);
+			if (type == DCBX_PROTOCOL_ETH) {
+				enable = false;
+				eth_tlv = true;
+			} else {
+				enable = true;
+			}
 
 			qed_dcbx_update_app_info(p_data, p_hwfn, enable,
 						 priority, tc, type);
 		}
 	}
 
+	/* If Eth TLV is not detected, use UFP TC as default TC */
+	if (test_bit(QED_MF_UFP_SPECIFIC, &p_hwfn->cdev->mf_bits) && !eth_tlv)
+		p_data->arr[DCBX_PROTOCOL_ETH].tc = p_hwfn->ufp_info.tc;
+
 	/* Update ramrod protocol data and hw_info fields
 	 * with default info when corresponding APP TLV's are not detected.
 	 * The enabled field has a different logic for ethernet as only for
diff --git a/drivers/net/ethernet/qlogic/qed/qed_debug.c b/drivers/net/ethernet/qlogic/qed/qed_debug.c
index 4926c5532fba..39124b594a36 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_debug.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_debug.c
@@ -419,6 +419,7 @@ struct phy_defs {
 #define NUM_RSS_MEM_TYPES		5
 
 #define NUM_BIG_RAM_TYPES		3
+#define BIG_RAM_NAME_LEN		3
 
 #define NUM_PHY_TBUS_ADDRESSES		2048
 #define PHY_DUMP_SIZE_DWORDS		(NUM_PHY_TBUS_ADDRESSES / 2)
@@ -3650,8 +3651,8 @@ static u32 qed_grc_dump_big_ram(struct qed_hwfn *p_hwfn,
 		     BIT(big_ram->is_256b_bit_offset[dev_data->chip_id]) ? 256
 									 : 128;
 
-	strscpy(type_name, big_ram->instance_name, sizeof(type_name));
-	strscpy(mem_name, big_ram->instance_name, sizeof(mem_name));
+	strncpy(type_name, big_ram->instance_name, BIG_RAM_NAME_LEN);
+	strncpy(mem_name, big_ram->instance_name, BIG_RAM_NAME_LEN);
 
 	/* Dump memory header */
 	offset += qed_grc_dump_mem_hdr(p_hwfn,
@@ -7778,6 +7779,57 @@ int qed_dbg_igu_fifo_size(struct qed_dev *cdev)
 	return qed_dbg_feature_size(cdev, DBG_FEATURE_IGU_FIFO);
 }
 
+int qed_dbg_nvm_image_length(struct qed_hwfn *p_hwfn,
+			     enum qed_nvm_images image_id, u32 *length)
+{
+	struct qed_nvm_image_att image_att;
+	int rc;
+
+	*length = 0;
+	rc = qed_mcp_get_nvm_image_att(p_hwfn, image_id, &image_att);
+	if (rc)
+		return rc;
+
+	*length = image_att.length;
+
+	return rc;
+}
+
+int qed_dbg_nvm_image(struct qed_dev *cdev, void *buffer,
+		      u32 *num_dumped_bytes, enum qed_nvm_images image_id)
+{
+	struct qed_hwfn *p_hwfn =
+		&cdev->hwfns[cdev->dbg_params.engine_for_debug];
+	u32 len_rounded, i;
+	__be32 val;
+	int rc;
+
+	*num_dumped_bytes = 0;
+	rc = qed_dbg_nvm_image_length(p_hwfn, image_id, &len_rounded);
+	if (rc)
+		return rc;
+
+	DP_NOTICE(p_hwfn->cdev,
+		  "Collecting a debug feature [\"nvram image %d\"]\n",
+		  image_id);
+
+	len_rounded = roundup(len_rounded, sizeof(u32));
+	rc = qed_mcp_get_nvm_image(p_hwfn, image_id, buffer, len_rounded);
+	if (rc)
+		return rc;
+
+	/* QED_NVM_IMAGE_NVM_META image is not swapped like other images */
+	if (image_id != QED_NVM_IMAGE_NVM_META)
+		for (i = 0; i < len_rounded; i += 4) {
+			val = cpu_to_be32(*(u32 *)(buffer + i));
+			*(u32 *)(buffer + i) = val;
+		}
+
+	*num_dumped_bytes = len_rounded;
+
+	return rc;
+}
+
 int qed_dbg_protection_override(struct qed_dev *cdev, void *buffer,
 				u32 *num_dumped_bytes)
 {
@@ -7831,6 +7883,9 @@ enum debug_print_features {
 	IGU_FIFO = 6,
 	PHY = 7,
 	FW_ASSERTS = 8,
+	NVM_CFG1 = 9,
+	DEFAULT_CFG = 10,
+	NVM_META = 11,
 };
 
 static u32 qed_calc_regdump_header(enum debug_print_features feature,
@@ -7965,13 +8020,61 @@ int qed_dbg_all_data(struct qed_dev *cdev, void *buffer)
 		DP_ERR(cdev, "qed_dbg_mcp_trace failed. rc = %d\n", rc);
 	}
 
+	/* nvm cfg1 */
+	rc = qed_dbg_nvm_image(cdev,
+			       (u8 *)buffer + offset + REGDUMP_HEADER_SIZE,
+			       &feature_size, QED_NVM_IMAGE_NVM_CFG1);
+	if (!rc) {
+		*(u32 *)((u8 *)buffer + offset) =
+		    qed_calc_regdump_header(NVM_CFG1, cur_engine,
+					    feature_size, omit_engine);
+		offset += (feature_size + REGDUMP_HEADER_SIZE);
+	} else if (rc != -ENOENT) {
+		DP_ERR(cdev,
+		       "qed_dbg_nvm_image failed for image  %d (%s), rc = %d\n",
+		       QED_NVM_IMAGE_NVM_CFG1, "QED_NVM_IMAGE_NVM_CFG1", rc);
+	}
+
+	/* nvm default */
+	rc = qed_dbg_nvm_image(cdev,
+			       (u8 *)buffer + offset + REGDUMP_HEADER_SIZE,
+			       &feature_size, QED_NVM_IMAGE_DEFAULT_CFG);
+	if (!rc) {
+		*(u32 *)((u8 *)buffer + offset) =
+		    qed_calc_regdump_header(DEFAULT_CFG, cur_engine,
+					    feature_size, omit_engine);
+		offset += (feature_size + REGDUMP_HEADER_SIZE);
+	} else if (rc != -ENOENT) {
+		DP_ERR(cdev,
+		       "qed_dbg_nvm_image failed for image %d (%s), rc = %d\n",
+		       QED_NVM_IMAGE_DEFAULT_CFG, "QED_NVM_IMAGE_DEFAULT_CFG",
+		       rc);
+	}
+
+	/* nvm meta */
+	rc = qed_dbg_nvm_image(cdev,
+			       (u8 *)buffer + offset + REGDUMP_HEADER_SIZE,
+			       &feature_size, QED_NVM_IMAGE_NVM_META);
+	if (!rc) {
+		*(u32 *)((u8 *)buffer + offset) =
+		    qed_calc_regdump_header(NVM_META, cur_engine,
+					    feature_size, omit_engine);
+		offset += (feature_size + REGDUMP_HEADER_SIZE);
+	} else if (rc != -ENOENT) {
+		DP_ERR(cdev,
+		       "qed_dbg_nvm_image failed for image %d (%s), rc = %d\n",
+		       QED_NVM_IMAGE_NVM_META, "QED_NVM_IMAGE_NVM_META", rc);
+	}
+
 	return 0;
 }
 
 int qed_dbg_all_data_size(struct qed_dev *cdev)
 {
+	struct qed_hwfn *p_hwfn =
+		&cdev->hwfns[cdev->dbg_params.engine_for_debug];
+	u32 regs_len = 0, image_len = 0;
 	u8 cur_engine, org_engine;
-	u32 regs_len = 0;
 
 	org_engine = qed_get_debug_engine(cdev);
 	for (cur_engine = 0; cur_engine < cdev->num_hwfns; cur_engine++) {
@@ -7993,6 +8096,15 @@ int qed_dbg_all_data_size(struct qed_dev *cdev)
 
 	/* Engine common */
 	regs_len += REGDUMP_HEADER_SIZE + qed_dbg_mcp_trace_size(cdev);
+	qed_dbg_nvm_image_length(p_hwfn, QED_NVM_IMAGE_NVM_CFG1, &image_len);
+	if (image_len)
+		regs_len += REGDUMP_HEADER_SIZE + image_len;
+	qed_dbg_nvm_image_length(p_hwfn, QED_NVM_IMAGE_DEFAULT_CFG, &image_len);
+	if (image_len)
+		regs_len += REGDUMP_HEADER_SIZE + image_len;
+	qed_dbg_nvm_image_length(p_hwfn, QED_NVM_IMAGE_NVM_META, &image_len);
+	if (image_len)
+		regs_len += REGDUMP_HEADER_SIZE + image_len;
 
 	return regs_len;
 }
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev.c b/drivers/net/ethernet/qlogic/qed/qed_dev.c
index d2ad5e92c74f..560528962658 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dev.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dev.c
@@ -1149,18 +1149,10 @@ static int qed_calc_hw_mode(struct qed_hwfn *p_hwfn)
 		return -EINVAL;
 	}
 
-	switch (p_hwfn->cdev->mf_mode) {
-	case QED_MF_DEFAULT:
-	case QED_MF_NPAR:
-		hw_mode |= 1 << MODE_MF_SI;
-		break;
-	case QED_MF_OVLAN:
+	if (test_bit(QED_MF_OVLAN_CLSS, &p_hwfn->cdev->mf_bits))
 		hw_mode |= 1 << MODE_MF_SD;
-		break;
-	default:
-		DP_NOTICE(p_hwfn, "Unsupported MF mode, init as DEFAULT\n");
+	else
 		hw_mode |= 1 << MODE_MF_SI;
-	}
 
 	hw_mode |= 1 << MODE_ASIC;
 
@@ -1507,6 +1499,11 @@ static int qed_hw_init_pf(struct qed_hwfn *p_hwfn,
 		STORE_RT_REG(p_hwfn, NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET, 1);
 		STORE_RT_REG(p_hwfn, NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET,
 			     p_hwfn->hw_info.ovlan);
+
+		DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
+			   "Configuring LLH_FUNC_FILTER_HDR_SEL\n");
+		STORE_RT_REG(p_hwfn, NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET,
+			     1);
 	}
 
 	/* Enable classification by MAC if needed */
@@ -1557,7 +1554,6 @@ static int qed_hw_init_pf(struct qed_hwfn *p_hwfn,
 
 		/* send function start command */
 		rc = qed_sp_pf_start(p_hwfn, p_ptt, p_tunn,
-				     p_hwfn->cdev->mf_mode,
 				     allow_npar_tx_switch);
 		if (rc) {
 			DP_NOTICE(p_hwfn, "Function start ramrod failed\n");
@@ -1644,6 +1640,7 @@ int qed_hw_init(struct qed_dev *cdev, struct qed_hw_init_params *p_params)
 	bool b_default_mtu = true;
 	struct qed_hwfn *p_hwfn;
 	int rc = 0, mfw_rc, i;
+	u16 ether_type;
 
 	if ((p_params->int_mode == QED_INT_MODE_MSI) && (cdev->num_hwfns > 1)) {
 		DP_NOTICE(cdev, "MSI mode is not supported for CMT devices\n");
@@ -1677,6 +1674,24 @@ int qed_hw_init(struct qed_dev *cdev, struct qed_hw_init_params *p_params)
 		if (rc)
 			return rc;
 
+		if (IS_PF(cdev) && (test_bit(QED_MF_8021Q_TAGGING,
+					     &cdev->mf_bits) ||
+				    test_bit(QED_MF_8021AD_TAGGING,
+					     &cdev->mf_bits))) {
+			if (test_bit(QED_MF_8021Q_TAGGING, &cdev->mf_bits))
+				ether_type = ETH_P_8021Q;
+			else
+				ether_type = ETH_P_8021AD;
+			STORE_RT_REG(p_hwfn, PRS_REG_TAG_ETHERTYPE_0_RT_OFFSET,
+				     ether_type);
+			STORE_RT_REG(p_hwfn, NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET,
+				     ether_type);
+			STORE_RT_REG(p_hwfn, PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET,
+				     ether_type);
+			STORE_RT_REG(p_hwfn, DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET,
+				     ether_type);
+		}
+
 		qed_fill_load_req_params(&load_req_params,
 					 p_params->p_drv_load_params);
 		rc = qed_mcp_load_req(p_hwfn, p_hwfn->p_main_ptt,
@@ -2639,31 +2654,57 @@ static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 		   link->pause.autoneg,
 		   p_caps->default_eee, p_caps->eee_lpi_timer);
 
-	/* Read Multi-function information from shmem */
-	addr = MCP_REG_SCRATCH + nvm_cfg1_offset +
-	       offsetof(struct nvm_cfg1, glob) +
-	       offsetof(struct nvm_cfg1_glob, generic_cont0);
+	if (IS_LEAD_HWFN(p_hwfn)) {
+		struct qed_dev *cdev = p_hwfn->cdev;
 
-	generic_cont0 = qed_rd(p_hwfn, p_ptt, addr);
+		/* Read Multi-function information from shmem */
+		addr = MCP_REG_SCRATCH + nvm_cfg1_offset +
+		       offsetof(struct nvm_cfg1, glob) +
+		       offsetof(struct nvm_cfg1_glob, generic_cont0);
 
-	mf_mode = (generic_cont0 & NVM_CFG1_GLOB_MF_MODE_MASK) >>
-		  NVM_CFG1_GLOB_MF_MODE_OFFSET;
+		generic_cont0 = qed_rd(p_hwfn, p_ptt, addr);
 
-	switch (mf_mode) {
-	case NVM_CFG1_GLOB_MF_MODE_MF_ALLOWED:
-		p_hwfn->cdev->mf_mode = QED_MF_OVLAN;
-		break;
-	case NVM_CFG1_GLOB_MF_MODE_NPAR1_0:
-		p_hwfn->cdev->mf_mode = QED_MF_NPAR;
-		break;
-	case NVM_CFG1_GLOB_MF_MODE_DEFAULT:
-		p_hwfn->cdev->mf_mode = QED_MF_DEFAULT;
-		break;
+		mf_mode = (generic_cont0 & NVM_CFG1_GLOB_MF_MODE_MASK) >>
+			  NVM_CFG1_GLOB_MF_MODE_OFFSET;
+
+		switch (mf_mode) {
+		case NVM_CFG1_GLOB_MF_MODE_MF_ALLOWED:
+			cdev->mf_bits = BIT(QED_MF_OVLAN_CLSS);
+			break;
+		case NVM_CFG1_GLOB_MF_MODE_UFP:
+			cdev->mf_bits = BIT(QED_MF_OVLAN_CLSS) |
+					BIT(QED_MF_LLH_PROTO_CLSS) |
+					BIT(QED_MF_UFP_SPECIFIC) |
+					BIT(QED_MF_8021Q_TAGGING);
+			break;
+		case NVM_CFG1_GLOB_MF_MODE_BD:
+			cdev->mf_bits = BIT(QED_MF_OVLAN_CLSS) |
+					BIT(QED_MF_LLH_PROTO_CLSS) |
+					BIT(QED_MF_8021AD_TAGGING);
+			break;
+		case NVM_CFG1_GLOB_MF_MODE_NPAR1_0:
+			cdev->mf_bits = BIT(QED_MF_LLH_MAC_CLSS) |
+					BIT(QED_MF_LLH_PROTO_CLSS) |
+					BIT(QED_MF_LL2_NON_UNICAST) |
+					BIT(QED_MF_INTER_PF_SWITCH);
+			break;
+		case NVM_CFG1_GLOB_MF_MODE_DEFAULT:
+			cdev->mf_bits = BIT(QED_MF_LLH_MAC_CLSS) |
+					BIT(QED_MF_LLH_PROTO_CLSS) |
+					BIT(QED_MF_LL2_NON_UNICAST);
+			if (QED_IS_BB(p_hwfn->cdev))
+				cdev->mf_bits |= BIT(QED_MF_NEED_DEF_PF);
+			break;
+		}
+
+		DP_INFO(p_hwfn, "Multi function mode is 0x%lx\n",
+			cdev->mf_bits);
 	}
-	DP_INFO(p_hwfn, "Multi function mode is %08x\n",
-		p_hwfn->cdev->mf_mode);
 
-	/* Read Multi-function information from shmem */
+	DP_INFO(p_hwfn, "Multi function mode is 0x%lx\n",
+		p_hwfn->cdev->mf_bits);
+
+	/* Read device capabilities information from shmem */
 	addr = MCP_REG_SCRATCH + nvm_cfg1_offset +
 		offsetof(struct nvm_cfg1, glob) +
 		offsetof(struct nvm_cfg1_glob, device_capabilities);
@@ -2856,6 +2897,8 @@ qed_get_hw_info(struct qed_hwfn *p_hwfn,
 		qed_mcp_cmd_port_init(p_hwfn, p_ptt);
 
 		qed_get_eee_caps(p_hwfn, p_ptt);
+
+		qed_mcp_read_ufp_config(p_hwfn, p_ptt);
 	}
 
 	if (qed_mcp_is_init(p_hwfn)) {
@@ -3462,7 +3505,7 @@ int qed_llh_add_mac_filter(struct qed_hwfn *p_hwfn,
 	u32 high = 0, low = 0, en;
 	int i;
 
-	if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn)))
+	if (!test_bit(QED_MF_LLH_MAC_CLSS, &p_hwfn->cdev->mf_bits))
 		return 0;
 
 	qed_llh_mac_to_filter(&high, &low, p_filter);
@@ -3507,7 +3550,7 @@ void qed_llh_remove_mac_filter(struct qed_hwfn *p_hwfn,
 	u32 high = 0, low = 0;
 	int i;
 
-	if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn)))
+	if (!test_bit(QED_MF_LLH_MAC_CLSS, &p_hwfn->cdev->mf_bits))
 		return;
 
 	qed_llh_mac_to_filter(&high, &low, p_filter);
@@ -3549,7 +3592,7 @@ qed_llh_add_protocol_filter(struct qed_hwfn *p_hwfn,
 	u32 high = 0, low = 0, en;
 	int i;
 
-	if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn)))
+	if (!test_bit(QED_MF_LLH_PROTO_CLSS, &p_hwfn->cdev->mf_bits))
 		return 0;
 
 	switch (type) {
@@ -3647,7 +3690,7 @@ qed_llh_remove_protocol_filter(struct qed_hwfn *p_hwfn,
 	u32 high = 0, low = 0;
 	int i;
 
-	if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn)))
+	if (!test_bit(QED_MF_LLH_PROTO_CLSS, &p_hwfn->cdev->mf_bits))
 		return;
 
 	switch (type) {
diff --git a/drivers/net/ethernet/qlogic/qed/qed_fcoe.c b/drivers/net/ethernet/qlogic/qed/qed_fcoe.c
index 2dc9b312a795..cc1b373c0ace 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_fcoe.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_fcoe.c
@@ -313,6 +313,9 @@ qed_sp_fcoe_conn_offload(struct qed_hwfn *p_hwfn,
 	p_data->d_id.addr_mid = p_conn->d_id.addr_mid;
 	p_data->d_id.addr_lo = p_conn->d_id.addr_lo;
 	p_data->flags = p_conn->flags;
+	if (test_bit(QED_MF_UFP_SPECIFIC, &p_hwfn->cdev->mf_bits))
+		SET_FIELD(p_data->flags,
+			  FCOE_CONN_OFFLOAD_RAMROD_DATA_B_SINGLE_VLAN, 1);
 	p_data->def_q_idx = p_conn->def_q_idx;
 
 	return qed_spq_post(p_hwfn, p_ent, NULL);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_hsi.h b/drivers/net/ethernet/qlogic/qed/qed_hsi.h
index 7f5ec42dde48..8e1e6e1eb40e 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_hsi.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_hsi.h
@@ -11863,6 +11863,8 @@ struct public_global {
 	u32 running_bundle_id;
 	s32 external_temperature;
 	u32 mdump_reason;
+	u32 data_ptr;
+	u32 data_size;
 };
 
 struct fw_flr_mb {
@@ -11993,6 +11995,16 @@ struct public_port {
 #define EEE_REMOTE_TW_TX_OFFSET 0
 #define EEE_REMOTE_TW_RX_MASK   0xffff0000
 #define EEE_REMOTE_TW_RX_OFFSET 16
+
+	u32 oem_cfg_port;
+#define OEM_CFG_CHANNEL_TYPE_MASK                       0x00000003
+#define OEM_CFG_CHANNEL_TYPE_OFFSET                     0
+#define OEM_CFG_CHANNEL_TYPE_VLAN_PARTITION             0x1
+#define OEM_CFG_CHANNEL_TYPE_STAGGED                    0x2
+#define OEM_CFG_SCHED_TYPE_MASK                         0x0000000C
+#define OEM_CFG_SCHED_TYPE_OFFSET                       2
+#define OEM_CFG_SCHED_TYPE_ETS                          0x1
+#define OEM_CFG_SCHED_TYPE_VNIC_BW                      0x2
 };
 
 struct public_func {
@@ -12069,6 +12081,23 @@ struct public_func {
 #define DRV_ID_DRV_INIT_HW_MASK		0x80000000
 #define DRV_ID_DRV_INIT_HW_SHIFT	31
 #define DRV_ID_DRV_INIT_HW_FLAG		(1 << DRV_ID_DRV_INIT_HW_SHIFT)
+
+	u32 oem_cfg_func;
+#define OEM_CFG_FUNC_TC_MASK                    0x0000000F
+#define OEM_CFG_FUNC_TC_OFFSET                  0
+#define OEM_CFG_FUNC_TC_0                       0x0
+#define OEM_CFG_FUNC_TC_1                       0x1
+#define OEM_CFG_FUNC_TC_2                       0x2
+#define OEM_CFG_FUNC_TC_3                       0x3
+#define OEM_CFG_FUNC_TC_4                       0x4
+#define OEM_CFG_FUNC_TC_5                       0x5
+#define OEM_CFG_FUNC_TC_6                       0x6
+#define OEM_CFG_FUNC_TC_7                       0x7
+
+#define OEM_CFG_FUNC_HOST_PRI_CTRL_MASK         0x00000030
+#define OEM_CFG_FUNC_HOST_PRI_CTRL_OFFSET       4
+#define OEM_CFG_FUNC_HOST_PRI_CTRL_VNIC         0x1
+#define OEM_CFG_FUNC_HOST_PRI_CTRL_OS           0x2
 };
 
 struct mcp_mac {
@@ -12295,6 +12324,7 @@ struct public_drv_mb {
 #define DRV_MSG_CODE_BIST_TEST			0x001e0000
 #define DRV_MSG_CODE_SET_LED_MODE		0x00200000
 #define DRV_MSG_CODE_RESOURCE_CMD	0x00230000
+#define DRV_MSG_CODE_GET_TLV_DONE		0x002f0000
 
 #define RESOURCE_CMD_REQ_RESC_MASK		0x0000001F
 #define RESOURCE_CMD_REQ_RESC_SHIFT		0
@@ -12495,6 +12525,8 @@ enum MFW_DRV_MSG_TYPE {
 	MFW_DRV_MSG_BW_UPDATE10,
 	MFW_DRV_MSG_TRANSCEIVER_STATE_CHANGE,
 	MFW_DRV_MSG_BW_UPDATE11,
+	MFW_DRV_MSG_OEM_CFG_UPDATE,
+	MFW_DRV_MSG_GET_TLV_REQ,
 	MFW_DRV_MSG_MAX
 };
 
@@ -12530,6 +12562,233 @@ struct mcp_public_data {
 	struct public_func func[MCP_GLOB_FUNC_MAX];
 };
 
+/* OCBB definitions */
+enum tlvs {
+	/* Category 1: Device Properties */
+	DRV_TLV_CLP_STR,
+	DRV_TLV_CLP_STR_CTD,
+	/* Category 6: Device Configuration */
+	DRV_TLV_SCSI_TO,
+	DRV_TLV_R_T_TOV,
+	DRV_TLV_R_A_TOV,
+	DRV_TLV_E_D_TOV,
+	DRV_TLV_CR_TOV,
+	DRV_TLV_BOOT_TYPE,
+	/* Category 8: Port Configuration */
+	DRV_TLV_NPIV_ENABLED,
+	/* Category 10: Function Configuration */
+	DRV_TLV_FEATURE_FLAGS,
+	DRV_TLV_LOCAL_ADMIN_ADDR,
+	DRV_TLV_ADDITIONAL_MAC_ADDR_1,
+	DRV_TLV_ADDITIONAL_MAC_ADDR_2,
+	DRV_TLV_LSO_MAX_OFFLOAD_SIZE,
+	DRV_TLV_LSO_MIN_SEGMENT_COUNT,
+	DRV_TLV_PROMISCUOUS_MODE,
+	DRV_TLV_TX_DESCRIPTORS_QUEUE_SIZE,
+	DRV_TLV_RX_DESCRIPTORS_QUEUE_SIZE,
+	DRV_TLV_NUM_OF_NET_QUEUE_VMQ_CFG,
+	DRV_TLV_FLEX_NIC_OUTER_VLAN_ID,
+	DRV_TLV_OS_DRIVER_STATES,
+	DRV_TLV_PXE_BOOT_PROGRESS,
+	/* Category 12: FC/FCoE Configuration */
+	DRV_TLV_NPIV_STATE,
+	DRV_TLV_NUM_OF_NPIV_IDS,
+	DRV_TLV_SWITCH_NAME,
+	DRV_TLV_SWITCH_PORT_NUM,
+	DRV_TLV_SWITCH_PORT_ID,
+	DRV_TLV_VENDOR_NAME,
+	DRV_TLV_SWITCH_MODEL,
+	DRV_TLV_SWITCH_FW_VER,
+	DRV_TLV_QOS_PRIORITY_PER_802_1P,
+	DRV_TLV_PORT_ALIAS,
+	DRV_TLV_PORT_STATE,
+	DRV_TLV_FIP_TX_DESCRIPTORS_QUEUE_SIZE,
+	DRV_TLV_FCOE_RX_DESCRIPTORS_QUEUE_SIZE,
+	DRV_TLV_LINK_FAILURE_COUNT,
+	DRV_TLV_FCOE_BOOT_PROGRESS,
+	/* Category 13: iSCSI Configuration */
+	DRV_TLV_TARGET_LLMNR_ENABLED,
+	DRV_TLV_HEADER_DIGEST_FLAG_ENABLED,
+	DRV_TLV_DATA_DIGEST_FLAG_ENABLED,
+	DRV_TLV_AUTHENTICATION_METHOD,
+	DRV_TLV_ISCSI_BOOT_TARGET_PORTAL,
+	DRV_TLV_MAX_FRAME_SIZE,
+	DRV_TLV_PDU_TX_DESCRIPTORS_QUEUE_SIZE,
+	DRV_TLV_PDU_RX_DESCRIPTORS_QUEUE_SIZE,
+	DRV_TLV_ISCSI_BOOT_PROGRESS,
+	/* Category 20: Device Data */
+	DRV_TLV_PCIE_BUS_RX_UTILIZATION,
+	DRV_TLV_PCIE_BUS_TX_UTILIZATION,
+	DRV_TLV_DEVICE_CPU_CORES_UTILIZATION,
+	DRV_TLV_LAST_VALID_DCC_TLV_RECEIVED,
+	DRV_TLV_NCSI_RX_BYTES_RECEIVED,
+	DRV_TLV_NCSI_TX_BYTES_SENT,
+	/* Category 22: Base Port Data */
+	DRV_TLV_RX_DISCARDS,
+	DRV_TLV_RX_ERRORS,
+	DRV_TLV_TX_ERRORS,
+	DRV_TLV_TX_DISCARDS,
+	DRV_TLV_RX_FRAMES_RECEIVED,
+	DRV_TLV_TX_FRAMES_SENT,
+	/* Category 23: FC/FCoE Port Data */
+	DRV_TLV_RX_BROADCAST_PACKETS,
+	DRV_TLV_TX_BROADCAST_PACKETS,
+	/* Category 28: Base Function Data */
+	DRV_TLV_NUM_OFFLOADED_CONNECTIONS_TCP_IPV4,
+	DRV_TLV_NUM_OFFLOADED_CONNECTIONS_TCP_IPV6,
+	DRV_TLV_TX_DESCRIPTOR_QUEUE_AVG_DEPTH,
+	DRV_TLV_RX_DESCRIPTORS_QUEUE_AVG_DEPTH,
+	DRV_TLV_PF_RX_FRAMES_RECEIVED,
+	DRV_TLV_RX_BYTES_RECEIVED,
+	DRV_TLV_PF_TX_FRAMES_SENT,
+	DRV_TLV_TX_BYTES_SENT,
+	DRV_TLV_IOV_OFFLOAD,
+	DRV_TLV_PCI_ERRORS_CAP_ID,
+	DRV_TLV_UNCORRECTABLE_ERROR_STATUS,
+	DRV_TLV_UNCORRECTABLE_ERROR_MASK,
+	DRV_TLV_CORRECTABLE_ERROR_STATUS,
+	DRV_TLV_CORRECTABLE_ERROR_MASK,
+	DRV_TLV_PCI_ERRORS_AECC_REGISTER,
+	DRV_TLV_TX_QUEUES_EMPTY,
+	DRV_TLV_RX_QUEUES_EMPTY,
+	DRV_TLV_TX_QUEUES_FULL,
+	DRV_TLV_RX_QUEUES_FULL,
+	/* Category 29: FC/FCoE Function Data */
+	DRV_TLV_FCOE_TX_DESCRIPTOR_QUEUE_AVG_DEPTH,
+	DRV_TLV_FCOE_RX_DESCRIPTORS_QUEUE_AVG_DEPTH,
+	DRV_TLV_FCOE_RX_FRAMES_RECEIVED,
+	DRV_TLV_FCOE_RX_BYTES_RECEIVED,
+	DRV_TLV_FCOE_TX_FRAMES_SENT,
+	DRV_TLV_FCOE_TX_BYTES_SENT,
+	DRV_TLV_CRC_ERROR_COUNT,
+	DRV_TLV_CRC_ERROR_1_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_CRC_ERROR_1_TIMESTAMP,
+	DRV_TLV_CRC_ERROR_2_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_CRC_ERROR_2_TIMESTAMP,
+	DRV_TLV_CRC_ERROR_3_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_CRC_ERROR_3_TIMESTAMP,
+	DRV_TLV_CRC_ERROR_4_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_CRC_ERROR_4_TIMESTAMP,
+	DRV_TLV_CRC_ERROR_5_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_CRC_ERROR_5_TIMESTAMP,
+	DRV_TLV_LOSS_OF_SYNC_ERROR_COUNT,
+	DRV_TLV_LOSS_OF_SIGNAL_ERRORS,
+	DRV_TLV_PRIMITIVE_SEQUENCE_PROTOCOL_ERROR_COUNT,
+	DRV_TLV_DISPARITY_ERROR_COUNT,
+	DRV_TLV_CODE_VIOLATION_ERROR_COUNT,
+	DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_1,
+	DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_2,
+	DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_3,
+	DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_4,
+	DRV_TLV_LAST_FLOGI_TIMESTAMP,
+	DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_1,
+	DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_2,
+	DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_3,
+	DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_4,
+	DRV_TLV_LAST_FLOGI_ACC_TIMESTAMP,
+	DRV_TLV_LAST_FLOGI_RJT,
+	DRV_TLV_LAST_FLOGI_RJT_TIMESTAMP,
+	DRV_TLV_FDISCS_SENT_COUNT,
+	DRV_TLV_FDISC_ACCS_RECEIVED,
+	DRV_TLV_FDISC_RJTS_RECEIVED,
+	DRV_TLV_PLOGI_SENT_COUNT,
+	DRV_TLV_PLOGI_ACCS_RECEIVED,
+	DRV_TLV_PLOGI_RJTS_RECEIVED,
+	DRV_TLV_PLOGI_1_SENT_DESTINATION_FC_ID,
+	DRV_TLV_PLOGI_1_TIMESTAMP,
+	DRV_TLV_PLOGI_2_SENT_DESTINATION_FC_ID,
+	DRV_TLV_PLOGI_2_TIMESTAMP,
+	DRV_TLV_PLOGI_3_SENT_DESTINATION_FC_ID,
+	DRV_TLV_PLOGI_3_TIMESTAMP,
+	DRV_TLV_PLOGI_4_SENT_DESTINATION_FC_ID,
+	DRV_TLV_PLOGI_4_TIMESTAMP,
+	DRV_TLV_PLOGI_5_SENT_DESTINATION_FC_ID,
+	DRV_TLV_PLOGI_5_TIMESTAMP,
+	DRV_TLV_PLOGI_1_ACC_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_PLOGI_1_ACC_TIMESTAMP,
+	DRV_TLV_PLOGI_2_ACC_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_PLOGI_2_ACC_TIMESTAMP,
+	DRV_TLV_PLOGI_3_ACC_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_PLOGI_3_ACC_TIMESTAMP,
+	DRV_TLV_PLOGI_4_ACC_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_PLOGI_4_ACC_TIMESTAMP,
+	DRV_TLV_PLOGI_5_ACC_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_PLOGI_5_ACC_TIMESTAMP,
+	DRV_TLV_LOGOS_ISSUED,
+	DRV_TLV_LOGO_ACCS_RECEIVED,
+	DRV_TLV_LOGO_RJTS_RECEIVED,
+	DRV_TLV_LOGO_1_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_LOGO_1_TIMESTAMP,
+	DRV_TLV_LOGO_2_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_LOGO_2_TIMESTAMP,
+	DRV_TLV_LOGO_3_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_LOGO_3_TIMESTAMP,
+	DRV_TLV_LOGO_4_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_LOGO_4_TIMESTAMP,
+	DRV_TLV_LOGO_5_RECEIVED_SOURCE_FC_ID,
+	DRV_TLV_LOGO_5_TIMESTAMP,
+	DRV_TLV_LOGOS_RECEIVED,
+	DRV_TLV_ACCS_ISSUED,
+	DRV_TLV_PRLIS_ISSUED,
+	DRV_TLV_ACCS_RECEIVED,
+	DRV_TLV_ABTS_SENT_COUNT,
+	DRV_TLV_ABTS_ACCS_RECEIVED,
+	DRV_TLV_ABTS_RJTS_RECEIVED,
+	DRV_TLV_ABTS_1_SENT_DESTINATION_FC_ID,
+	DRV_TLV_ABTS_1_TIMESTAMP,
+	DRV_TLV_ABTS_2_SENT_DESTINATION_FC_ID,
+	DRV_TLV_ABTS_2_TIMESTAMP,
+	DRV_TLV_ABTS_3_SENT_DESTINATION_FC_ID,
+	DRV_TLV_ABTS_3_TIMESTAMP,
+	DRV_TLV_ABTS_4_SENT_DESTINATION_FC_ID,
+	DRV_TLV_ABTS_4_TIMESTAMP,
+	DRV_TLV_ABTS_5_SENT_DESTINATION_FC_ID,
+	DRV_TLV_ABTS_5_TIMESTAMP,
+	DRV_TLV_RSCNS_RECEIVED,
+	DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_1,
+	DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_2,
+	DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_3,
+	DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_4,
+	DRV_TLV_LUN_RESETS_ISSUED,
+	DRV_TLV_ABORT_TASK_SETS_ISSUED,
+	DRV_TLV_TPRLOS_SENT,
+	DRV_TLV_NOS_SENT_COUNT,
+	DRV_TLV_NOS_RECEIVED_COUNT,
+	DRV_TLV_OLS_COUNT,
+	DRV_TLV_LR_COUNT,
+	DRV_TLV_LRR_COUNT,
+	DRV_TLV_LIP_SENT_COUNT,
+	DRV_TLV_LIP_RECEIVED_COUNT,
+	DRV_TLV_EOFA_COUNT,
+	DRV_TLV_EOFNI_COUNT,
+	DRV_TLV_SCSI_STATUS_CHECK_CONDITION_COUNT,
+	DRV_TLV_SCSI_STATUS_CONDITION_MET_COUNT,
+	DRV_TLV_SCSI_STATUS_BUSY_COUNT,
+	DRV_TLV_SCSI_STATUS_INTERMEDIATE_COUNT,
+	DRV_TLV_SCSI_STATUS_INTERMEDIATE_CONDITION_MET_COUNT,
+	DRV_TLV_SCSI_STATUS_RESERVATION_CONFLICT_COUNT,
+	DRV_TLV_SCSI_STATUS_TASK_SET_FULL_COUNT,
+	DRV_TLV_SCSI_STATUS_ACA_ACTIVE_COUNT,
+	DRV_TLV_SCSI_STATUS_TASK_ABORTED_COUNT,
+	DRV_TLV_SCSI_CHECK_CONDITION_1_RECEIVED_SK_ASC_ASCQ,
+	DRV_TLV_SCSI_CHECK_1_TIMESTAMP,
+	DRV_TLV_SCSI_CHECK_CONDITION_2_RECEIVED_SK_ASC_ASCQ,
+	DRV_TLV_SCSI_CHECK_2_TIMESTAMP,
+	DRV_TLV_SCSI_CHECK_CONDITION_3_RECEIVED_SK_ASC_ASCQ,
+	DRV_TLV_SCSI_CHECK_3_TIMESTAMP,
+	DRV_TLV_SCSI_CHECK_CONDITION_4_RECEIVED_SK_ASC_ASCQ,
+	DRV_TLV_SCSI_CHECK_4_TIMESTAMP,
+	DRV_TLV_SCSI_CHECK_CONDITION_5_RECEIVED_SK_ASC_ASCQ,
+	DRV_TLV_SCSI_CHECK_5_TIMESTAMP,
+	/* Category 30: iSCSI Function Data */
+	DRV_TLV_PDU_TX_DESCRIPTOR_QUEUE_AVG_DEPTH,
+	DRV_TLV_PDU_RX_DESCRIPTORS_QUEUE_AVG_DEPTH,
+	DRV_TLV_ISCSI_PDU_RX_FRAMES_RECEIVED,
+	DRV_TLV_ISCSI_PDU_RX_BYTES_RECEIVED,
+	DRV_TLV_ISCSI_PDU_TX_FRAMES_SENT,
+	DRV_TLV_ISCSI_PDU_TX_BYTES_SENT
+};
+
 struct nvm_cfg_mac_address {
 	u32 mac_addr_hi;
 #define NVM_CFG_MAC_ADDRESS_HI_MASK	0x0000FFFF
diff --git a/drivers/net/ethernet/qlogic/qed/qed_l2.c b/drivers/net/ethernet/qlogic/qed/qed_l2.c
index 8667799d0069..1c0d0c217936 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_l2.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_l2.c
@@ -1677,6 +1677,8 @@ static void __qed_get_vport_tstats(struct qed_hwfn *p_hwfn,
 	    HILO_64_REGPAIR(tstats.mftag_filter_discard);
 	p_stats->common.mac_filter_discards +=
 	    HILO_64_REGPAIR(tstats.eth_mac_filter_discard);
+	p_stats->common.gft_filter_drop +=
+		HILO_64_REGPAIR(tstats.eth_gft_drop_pkt);
 }
 
 static void __qed_get_vport_ustats_addrlen(struct qed_hwfn *p_hwfn,
@@ -1973,6 +1975,8 @@ qed_arfs_mode_to_hsi(enum qed_filter_config_mode mode)
 		return GFT_PROFILE_TYPE_4_TUPLE;
 	if (mode == QED_FILTER_CONFIG_MODE_IP_DEST)
 		return GFT_PROFILE_TYPE_IP_DST_ADDR;
+	if (mode == QED_FILTER_CONFIG_MODE_IP_SRC)
+		return GFT_PROFILE_TYPE_IP_SRC_ADDR;
 	return GFT_PROFILE_TYPE_L4_DST_PORT;
 }
 
@@ -2013,16 +2017,6 @@ qed_configure_rfs_ntuple_filter(struct qed_hwfn *p_hwfn,
 	u8 abs_vport_id = 0;
 	int rc = -EINVAL;
 
-	rc = qed_fw_vport(p_hwfn, p_params->vport_id, &abs_vport_id);
-	if (rc)
-		return rc;
-
-	if (p_params->qid != QED_RFS_NTUPLE_QID_RSS) {
-		rc = qed_fw_l2_queue(p_hwfn, p_params->qid, &abs_rx_q_id);
-		if (rc)
-			return rc;
-	}
-
 	/* Get SPQ entry */
 	memset(&init_data, 0, sizeof(init_data));
 	init_data.cid = qed_spq_get_cid(p_hwfn);
@@ -2047,15 +2041,28 @@ qed_configure_rfs_ntuple_filter(struct qed_hwfn *p_hwfn,
 	DMA_REGPAIR_LE(p_ramrod->pkt_hdr_addr, p_params->addr);
 	p_ramrod->pkt_hdr_length = cpu_to_le16(p_params->length);
 
-	if (p_params->qid != QED_RFS_NTUPLE_QID_RSS) {
-		p_ramrod->rx_qid_valid = 1;
-		p_ramrod->rx_qid = cpu_to_le16(abs_rx_q_id);
+	if (p_params->b_is_drop) {
+		p_ramrod->vport_id = cpu_to_le16(ETH_GFT_TRASHCAN_VPORT);
+	} else {
+		rc = qed_fw_vport(p_hwfn, p_params->vport_id, &abs_vport_id);
+		if (rc)
+			return rc;
+
+		if (p_params->qid != QED_RFS_NTUPLE_QID_RSS) {
+			rc = qed_fw_l2_queue(p_hwfn, p_params->qid,
+					     &abs_rx_q_id);
+			if (rc)
+				return rc;
+
+			p_ramrod->rx_qid_valid = 1;
+			p_ramrod->rx_qid = cpu_to_le16(abs_rx_q_id);
+		}
+
+		p_ramrod->vport_id = cpu_to_le16((u16)abs_vport_id);
 	}
 
 	p_ramrod->flow_id_valid = 0;
 	p_ramrod->flow_id = 0;
-
-	p_ramrod->vport_id = cpu_to_le16((u16)abs_vport_id);
 	p_ramrod->filter_action = p_params->b_is_add ? GFT_ADD_FILTER
 	    : GFT_DELETE_FILTER;
 
@@ -2848,6 +2855,24 @@ static int qed_fp_cqe_completion(struct qed_dev *dev,
 				      cqe);
 }
 
+static int qed_req_bulletin_update_mac(struct qed_dev *cdev, u8 *mac)
+{
+	int i, ret;
+
+	if (IS_PF(cdev))
+		return 0;
+
+	for_each_hwfn(cdev, i) {
+		struct qed_hwfn *p_hwfn = &cdev->hwfns[i];
+
+		ret = qed_vf_pf_bulletin_update_mac(p_hwfn, mac);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 #ifdef CONFIG_QED_SRIOV
 extern const struct qed_iov_hv_ops qed_iov_ops_pass;
 #endif
@@ -2885,6 +2910,7 @@ static const struct qed_eth_ops qed_eth_ops_pass = {
 	.ntuple_filter_config = &qed_ntuple_arfs_filter_config,
 	.configure_arfs_searcher = &qed_configure_arfs_searcher,
 	.get_coalesce = &qed_get_coalesce,
+	.req_bulletin_update_mac = &qed_req_bulletin_update_mac,
 };
 
 const struct qed_eth_ops *qed_get_eth_ops(void)
diff --git a/drivers/net/ethernet/qlogic/qed/qed_ll2.c b/drivers/net/ethernet/qlogic/qed/qed_ll2.c
index 468c59d2e491..c97ebd681c47 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_ll2.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_ll2.c
@@ -960,12 +960,16 @@ static int qed_sp_ll2_rx_queue_start(struct qed_hwfn *p_hwfn,
 	p_ramrod->drop_ttl0_flg = p_ll2_conn->input.rx_drop_ttl0_flg;
 	p_ramrod->inner_vlan_stripping_en =
 		p_ll2_conn->input.rx_vlan_removal_en;
+
+	if (test_bit(QED_MF_UFP_SPECIFIC, &p_hwfn->cdev->mf_bits) &&
+	    p_ll2_conn->input.conn_type == QED_LL2_TYPE_FCOE)
+		p_ramrod->report_outer_vlan = 1;
 	p_ramrod->queue_id = p_ll2_conn->queue_id;
 	p_ramrod->main_func_queue = p_ll2_conn->main_func_queue ? 1 : 0;
 
-	if ((IS_MF_DEFAULT(p_hwfn) || IS_MF_SI(p_hwfn)) &&
-	    p_ramrod->main_func_queue && (conn_type != QED_LL2_TYPE_ROCE) &&
-	    (conn_type != QED_LL2_TYPE_IWARP)) {
+	if (test_bit(QED_MF_LL2_NON_UNICAST, &p_hwfn->cdev->mf_bits) &&
+	    p_ramrod->main_func_queue && conn_type != QED_LL2_TYPE_ROCE &&
+	    conn_type != QED_LL2_TYPE_IWARP) {
 		p_ramrod->mf_si_bcast_accept_all = 1;
 		p_ramrod->mf_si_mcast_accept_all = 1;
 	} else {
@@ -1534,11 +1538,12 @@ int qed_ll2_establish_connection(void *cxt, u8 connection_handle)
 	qed_ll2_establish_connection_ooo(p_hwfn, p_ll2_conn);
 
 	if (p_ll2_conn->input.conn_type == QED_LL2_TYPE_FCOE) {
+		if (!test_bit(QED_MF_UFP_SPECIFIC, &p_hwfn->cdev->mf_bits))
+			qed_llh_add_protocol_filter(p_hwfn, p_ptt,
+						    ETH_P_FCOE, 0,
+						    QED_LLH_FILTER_ETHERTYPE);
 		qed_llh_add_protocol_filter(p_hwfn, p_ptt,
-					    0x8906, 0,
-					    QED_LLH_FILTER_ETHERTYPE);
-		qed_llh_add_protocol_filter(p_hwfn, p_ptt,
-					    0x8914, 0,
+					    ETH_P_FIP, 0,
 					    QED_LLH_FILTER_ETHERTYPE);
 	}
 
@@ -1694,11 +1699,16 @@ qed_ll2_prepare_tx_packet_set_bd(struct qed_hwfn *p_hwfn,
 
 	start_bd = (struct core_tx_bd *)qed_chain_produce(p_tx_chain);
 	if (QED_IS_IWARP_PERSONALITY(p_hwfn) &&
-	    p_ll2->input.conn_type == QED_LL2_TYPE_OOO)
+	    p_ll2->input.conn_type == QED_LL2_TYPE_OOO) {
 		start_bd->nw_vlan_or_lb_echo =
 		    cpu_to_le16(IWARP_LL2_IN_ORDER_TX_QUEUE);
-	else
+	} else {
 		start_bd->nw_vlan_or_lb_echo = cpu_to_le16(pkt->vlan);
+		if (test_bit(QED_MF_UFP_SPECIFIC, &p_hwfn->cdev->mf_bits) &&
+		    p_ll2->input.conn_type == QED_LL2_TYPE_FCOE)
+			pkt->remove_stag = true;
+	}
+
 	SET_FIELD(start_bd->bitfield1, CORE_TX_BD_L4_HDR_OFFSET_W,
 		  cpu_to_le16(pkt->l4_hdr_offset_w));
 	SET_FIELD(start_bd->bitfield1, CORE_TX_BD_TX_DST, tx_dest);
@@ -1709,6 +1719,9 @@ qed_ll2_prepare_tx_packet_set_bd(struct qed_hwfn *p_hwfn,
 	SET_FIELD(bd_data, CORE_TX_BD_DATA_IP_CSUM, !!(pkt->enable_ip_cksum));
 	SET_FIELD(bd_data, CORE_TX_BD_DATA_L4_CSUM, !!(pkt->enable_l4_cksum));
 	SET_FIELD(bd_data, CORE_TX_BD_DATA_IP_LEN, !!(pkt->calc_ip_len));
+	SET_FIELD(bd_data, CORE_TX_BD_DATA_DISABLE_STAG_INSERTION,
+		  !!(pkt->remove_stag));
+
 	start_bd->bd_data.as_bitfield = cpu_to_le16(bd_data);
 	DMA_REGPAIR_LE(start_bd->addr, pkt->first_frag);
 	start_bd->nbytes = cpu_to_le16(pkt->first_frag_len);
@@ -1933,11 +1946,12 @@ int qed_ll2_terminate_connection(void *cxt, u8 connection_handle)
 		qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info);
 
 	if (p_ll2_conn->input.conn_type == QED_LL2_TYPE_FCOE) {
+		if (!test_bit(QED_MF_UFP_SPECIFIC, &p_hwfn->cdev->mf_bits))
+			qed_llh_remove_protocol_filter(p_hwfn, p_ptt,
+						       ETH_P_FCOE, 0,
+						      QED_LLH_FILTER_ETHERTYPE);
 		qed_llh_remove_protocol_filter(p_hwfn, p_ptt,
-					       0x8906, 0,
-					       QED_LLH_FILTER_ETHERTYPE);
-		qed_llh_remove_protocol_filter(p_hwfn, p_ptt,
-					       0x8914, 0,
+					       ETH_P_FIP, 0,
 					       QED_LLH_FILTER_ETHERTYPE);
 	}
 
@@ -2399,7 +2413,8 @@ static int qed_ll2_stop(struct qed_dev *cdev)
 	return -EINVAL;
 }
 
-static int qed_ll2_start_xmit(struct qed_dev *cdev, struct sk_buff *skb)
+static int qed_ll2_start_xmit(struct qed_dev *cdev, struct sk_buff *skb,
+			      unsigned long xmit_flags)
 {
 	struct qed_ll2_tx_pkt_info pkt;
 	const skb_frag_t *frag;
@@ -2444,6 +2459,9 @@ static int qed_ll2_start_xmit(struct qed_dev *cdev, struct sk_buff *skb)
 	pkt.first_frag = mapping;
 	pkt.first_frag_len = skb->len;
 	pkt.cookie = skb;
+	if (test_bit(QED_MF_UFP_SPECIFIC, &cdev->mf_bits) &&
+	    test_bit(QED_LL2_XMIT_FLAGS_FIP_DISCOVERY, &xmit_flags))
+		pkt.remove_stag = true;
 
 	rc = qed_ll2_prepare_tx_packet(&cdev->hwfns[0], cdev->ll2->handle,
 				       &pkt, 1);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_main.c b/drivers/net/ethernet/qlogic/qed/qed_main.c
index 7870ae2a6f7e..68c4399ffd50 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_main.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_main.c
@@ -264,7 +264,6 @@ int qed_fill_dev_info(struct qed_dev *cdev,
 	dev_info->pci_mem_end = cdev->pci_params.mem_end;
 	dev_info->pci_irq = cdev->pci_params.irq;
 	dev_info->rdma_supported = QED_IS_RDMA_PERSONALITY(p_hwfn);
-	dev_info->is_mf_default = IS_MF_DEFAULT(&cdev->hwfns[0]);
 	dev_info->dev_type = cdev->type;
 	ether_addr_copy(dev_info->hw_mac, hw_info->hw_mac_addr);
 
@@ -273,7 +272,8 @@ int qed_fill_dev_info(struct qed_dev *cdev,
 		dev_info->fw_minor = FW_MINOR_VERSION;
 		dev_info->fw_rev = FW_REVISION_VERSION;
 		dev_info->fw_eng = FW_ENGINEERING_VERSION;
-		dev_info->mf_mode = cdev->mf_mode;
+		dev_info->b_inter_pf_switch = test_bit(QED_MF_INTER_PF_SWITCH,
+						       &cdev->mf_bits);
 		dev_info->tx_switching = true;
 
 		if (hw_info->b_wol_support == QED_WOL_SUPPORT_PME)
@@ -946,6 +946,68 @@ static void qed_update_pf_params(struct qed_dev *cdev,
 	}
 }
 
+static void qed_slowpath_wq_stop(struct qed_dev *cdev)
+{
+	int i;
+
+	if (IS_VF(cdev))
+		return;
+
+	for_each_hwfn(cdev, i) {
+		if (!cdev->hwfns[i].slowpath_wq)
+			continue;
+
+		flush_workqueue(cdev->hwfns[i].slowpath_wq);
+		destroy_workqueue(cdev->hwfns[i].slowpath_wq);
+	}
+}
+
+static void qed_slowpath_task(struct work_struct *work)
+{
+	struct qed_hwfn *hwfn = container_of(work, struct qed_hwfn,
+					     slowpath_task.work);
+	struct qed_ptt *ptt = qed_ptt_acquire(hwfn);
+
+	if (!ptt) {
+		queue_delayed_work(hwfn->slowpath_wq, &hwfn->slowpath_task, 0);
+		return;
+	}
+
+	if (test_and_clear_bit(QED_SLOWPATH_MFW_TLV_REQ,
+			       &hwfn->slowpath_task_flags))
+		qed_mfw_process_tlv_req(hwfn, ptt);
+
+	qed_ptt_release(hwfn, ptt);
+}
+
+static int qed_slowpath_wq_start(struct qed_dev *cdev)
+{
+	struct qed_hwfn *hwfn;
+	char name[NAME_SIZE];
+	int i;
+
+	if (IS_VF(cdev))
+		return 0;
+
+	for_each_hwfn(cdev, i) {
+		hwfn = &cdev->hwfns[i];
+
+		snprintf(name, NAME_SIZE, "slowpath-%02x:%02x.%02x",
+			 cdev->pdev->bus->number,
+			 PCI_SLOT(cdev->pdev->devfn), hwfn->abs_pf_id);
+
+		hwfn->slowpath_wq = alloc_workqueue(name, 0, 0);
+		if (!hwfn->slowpath_wq) {
+			DP_NOTICE(hwfn, "Cannot create slowpath workqueue\n");
+			return -ENOMEM;
+		}
+
+		INIT_DELAYED_WORK(&hwfn->slowpath_task, qed_slowpath_task);
+	}
+
+	return 0;
+}
+
 static int qed_slowpath_start(struct qed_dev *cdev,
 			      struct qed_slowpath_params *params)
 {
@@ -961,6 +1023,9 @@ static int qed_slowpath_start(struct qed_dev *cdev,
 	if (qed_iov_wq_start(cdev))
 		goto err;
 
+	if (qed_slowpath_wq_start(cdev))
+		goto err;
+
 	if (IS_PF(cdev)) {
 		rc = request_firmware(&cdev->firmware, QED_FW_FILE_NAME,
 				      &cdev->pdev->dev);
@@ -1095,6 +1160,8 @@ static int qed_slowpath_start(struct qed_dev *cdev,
 
 	qed_iov_wq_stop(cdev, false);
 
+	qed_slowpath_wq_stop(cdev);
+
 	return rc;
 }
 
@@ -1103,6 +1170,8 @@ static int qed_slowpath_stop(struct qed_dev *cdev)
 	if (!cdev)
 		return -ENODEV;
 
+	qed_slowpath_wq_stop(cdev);
+
 	qed_ll2_dealloc_if(cdev);
 
 	if (IS_PF(cdev)) {
@@ -1894,15 +1963,8 @@ static int qed_nvm_get_image(struct qed_dev *cdev, enum qed_nvm_images type,
 			     u8 *buf, u16 len)
 {
 	struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
-	struct qed_ptt *ptt = qed_ptt_acquire(hwfn);
-	int rc;
 
-	if (!ptt)
-		return -EAGAIN;
-
-	rc = qed_mcp_get_nvm_image(hwfn, ptt, type, buf, len);
-	qed_ptt_release(hwfn, ptt);
-	return rc;
+	return qed_mcp_get_nvm_image(hwfn, type, buf, len);
 }
 
 static int qed_set_coalesce(struct qed_dev *cdev, u16 rx_coal, u16 tx_coal,
@@ -2095,3 +2157,89 @@ void qed_get_protocol_stats(struct qed_dev *cdev,
 		return;
 	}
 }
+
+int qed_mfw_tlv_req(struct qed_hwfn *hwfn)
+{
+	DP_VERBOSE(hwfn->cdev, NETIF_MSG_DRV,
+		   "Scheduling slowpath task [Flag: %d]\n",
+		   QED_SLOWPATH_MFW_TLV_REQ);
+	smp_mb__before_atomic();
+	set_bit(QED_SLOWPATH_MFW_TLV_REQ, &hwfn->slowpath_task_flags);
+	smp_mb__after_atomic();
+	queue_delayed_work(hwfn->slowpath_wq, &hwfn->slowpath_task, 0);
+
+	return 0;
+}
+
+static void
+qed_fill_generic_tlv_data(struct qed_dev *cdev, struct qed_mfw_tlv_generic *tlv)
+{
+	struct qed_common_cb_ops *op = cdev->protocol_ops.common;
+	struct qed_eth_stats_common *p_common;
+	struct qed_generic_tlvs gen_tlvs;
+	struct qed_eth_stats stats;
+	int i;
+
+	memset(&gen_tlvs, 0, sizeof(gen_tlvs));
+	op->get_generic_tlv_data(cdev->ops_cookie, &gen_tlvs);
+
+	if (gen_tlvs.feat_flags & QED_TLV_IP_CSUM)
+		tlv->flags.ipv4_csum_offload = true;
+	if (gen_tlvs.feat_flags & QED_TLV_LSO)
+		tlv->flags.lso_supported = true;
+	tlv->flags.b_set = true;
+
+	for (i = 0; i < QED_TLV_MAC_COUNT; i++) {
+		if (is_valid_ether_addr(gen_tlvs.mac[i])) {
+			ether_addr_copy(tlv->mac[i], gen_tlvs.mac[i]);
+			tlv->mac_set[i] = true;
+		}
+	}
+
+	qed_get_vport_stats(cdev, &stats);
+	p_common = &stats.common;
+	tlv->rx_frames = p_common->rx_ucast_pkts + p_common->rx_mcast_pkts +
+			 p_common->rx_bcast_pkts;
+	tlv->rx_frames_set = true;
+	tlv->rx_bytes = p_common->rx_ucast_bytes + p_common->rx_mcast_bytes +
+			p_common->rx_bcast_bytes;
+	tlv->rx_bytes_set = true;
+	tlv->tx_frames = p_common->tx_ucast_pkts + p_common->tx_mcast_pkts +
+			 p_common->tx_bcast_pkts;
+	tlv->tx_frames_set = true;
+	tlv->tx_bytes = p_common->tx_ucast_bytes + p_common->tx_mcast_bytes +
+			p_common->tx_bcast_bytes;
+	tlv->rx_bytes_set = true;
+}
+
+int qed_mfw_fill_tlv_data(struct qed_hwfn *hwfn, enum qed_mfw_tlv_type type,
+			  union qed_mfw_tlv_data *tlv_buf)
+{
+	struct qed_dev *cdev = hwfn->cdev;
+	struct qed_common_cb_ops *ops;
+
+	ops = cdev->protocol_ops.common;
+	if (!ops || !ops->get_protocol_tlv_data || !ops->get_generic_tlv_data) {
+		DP_NOTICE(hwfn, "Can't collect TLV management info\n");
+		return -EINVAL;
+	}
+
+	switch (type) {
+	case QED_MFW_TLV_GENERIC:
+		qed_fill_generic_tlv_data(hwfn->cdev, &tlv_buf->generic);
+		break;
+	case QED_MFW_TLV_ETH:
+		ops->get_protocol_tlv_data(cdev->ops_cookie, &tlv_buf->eth);
+		break;
+	case QED_MFW_TLV_FCOE:
+		ops->get_protocol_tlv_data(cdev->ops_cookie, &tlv_buf->fcoe);
+		break;
+	case QED_MFW_TLV_ISCSI:
+		ops->get_protocol_tlv_data(cdev->ops_cookie, &tlv_buf->iscsi);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.c b/drivers/net/ethernet/qlogic/qed/qed_mcp.c
index ec0d425766a7..2612e3e458d9 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_mcp.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.c
@@ -40,6 +40,7 @@
 #include <linux/string.h>
 #include <linux/etherdevice.h>
 #include "qed.h"
+#include "qed_cxt.h"
 #include "qed_dcbx.h"
 #include "qed_hsi.h"
 #include "qed_hw.h"
@@ -1486,6 +1487,80 @@ static void qed_mcp_update_stag(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 		    &resp, &param);
 }
 
+void qed_mcp_read_ufp_config(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	struct public_func shmem_info;
+	u32 port_cfg, val;
+
+	if (!test_bit(QED_MF_UFP_SPECIFIC, &p_hwfn->cdev->mf_bits))
+		return;
+
+	memset(&p_hwfn->ufp_info, 0, sizeof(p_hwfn->ufp_info));
+	port_cfg = qed_rd(p_hwfn, p_ptt, p_hwfn->mcp_info->port_addr +
+			  offsetof(struct public_port, oem_cfg_port));
+	val = (port_cfg & OEM_CFG_CHANNEL_TYPE_MASK) >>
+		OEM_CFG_CHANNEL_TYPE_OFFSET;
+	if (val != OEM_CFG_CHANNEL_TYPE_STAGGED)
+		DP_NOTICE(p_hwfn, "Incorrect UFP Channel type  %d\n", val);
+
+	val = (port_cfg & OEM_CFG_SCHED_TYPE_MASK) >> OEM_CFG_SCHED_TYPE_OFFSET;
+	if (val == OEM_CFG_SCHED_TYPE_ETS) {
+		p_hwfn->ufp_info.mode = QED_UFP_MODE_ETS;
+	} else if (val == OEM_CFG_SCHED_TYPE_VNIC_BW) {
+		p_hwfn->ufp_info.mode = QED_UFP_MODE_VNIC_BW;
+	} else {
+		p_hwfn->ufp_info.mode = QED_UFP_MODE_UNKNOWN;
+		DP_NOTICE(p_hwfn, "Unknown UFP scheduling mode %d\n", val);
+	}
+
+	qed_mcp_get_shmem_func(p_hwfn, p_ptt, &shmem_info, MCP_PF_ID(p_hwfn));
+	val = (port_cfg & OEM_CFG_FUNC_TC_MASK) >> OEM_CFG_FUNC_TC_OFFSET;
+	p_hwfn->ufp_info.tc = (u8)val;
+	val = (port_cfg & OEM_CFG_FUNC_HOST_PRI_CTRL_MASK) >>
+		OEM_CFG_FUNC_HOST_PRI_CTRL_OFFSET;
+	if (val == OEM_CFG_FUNC_HOST_PRI_CTRL_VNIC) {
+		p_hwfn->ufp_info.pri_type = QED_UFP_PRI_VNIC;
+	} else if (val == OEM_CFG_FUNC_HOST_PRI_CTRL_OS) {
+		p_hwfn->ufp_info.pri_type = QED_UFP_PRI_OS;
+	} else {
+		p_hwfn->ufp_info.pri_type = QED_UFP_PRI_UNKNOWN;
+		DP_NOTICE(p_hwfn, "Unknown Host priority control %d\n", val);
+	}
+
+	DP_NOTICE(p_hwfn,
+		  "UFP shmem config: mode = %d tc = %d pri_type = %d\n",
+		  p_hwfn->ufp_info.mode,
+		  p_hwfn->ufp_info.tc, p_hwfn->ufp_info.pri_type);
+}
+
+static int
+qed_mcp_handle_ufp_event(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	qed_mcp_read_ufp_config(p_hwfn, p_ptt);
+
+	if (p_hwfn->ufp_info.mode == QED_UFP_MODE_VNIC_BW) {
+		p_hwfn->qm_info.ooo_tc = p_hwfn->ufp_info.tc;
+		p_hwfn->hw_info.offload_tc = p_hwfn->ufp_info.tc;
+
+		qed_qm_reconf(p_hwfn, p_ptt);
+	} else if (p_hwfn->ufp_info.mode == QED_UFP_MODE_ETS) {
+		/* Merge UFP TC with the dcbx TC data */
+		qed_dcbx_mib_update_event(p_hwfn, p_ptt,
+					  QED_DCBX_OPERATIONAL_MIB);
+	} else {
+		DP_ERR(p_hwfn, "Invalid sched type, discard the UFP config\n");
+		return -EINVAL;
+	}
+
+	/* update storm FW with negotiation results */
+	qed_sp_pf_update_ufp(p_hwfn);
+
+	/* update stag pcp value */
+	qed_sp_pf_update_stag(p_hwfn);
+
+	return 0;
+}
+
 int qed_mcp_handle_events(struct qed_hwfn *p_hwfn,
 			  struct qed_ptt *p_ptt)
 {
@@ -1529,6 +1604,9 @@ int qed_mcp_handle_events(struct qed_hwfn *p_hwfn,
 			qed_dcbx_mib_update_event(p_hwfn, p_ptt,
 						  QED_DCBX_OPERATIONAL_MIB);
 			break;
+		case MFW_DRV_MSG_OEM_CFG_UPDATE:
+			qed_mcp_handle_ufp_event(p_hwfn, p_ptt);
+			break;
 		case MFW_DRV_MSG_TRANSCEIVER_STATE_CHANGE:
 			qed_mcp_handle_transceiver_change(p_hwfn, p_ptt);
 			break;
@@ -1544,6 +1622,8 @@ int qed_mcp_handle_events(struct qed_hwfn *p_hwfn,
 		case MFW_DRV_MSG_S_TAG_UPDATE:
 			qed_mcp_update_stag(p_hwfn, p_ptt);
 			break;
+		case MFW_DRV_MSG_GET_TLV_REQ:
+			qed_mfw_tlv_req(p_hwfn);
 			break;
 		default:
 			DP_INFO(p_hwfn, "Unimplemented MFW message %d\n", i);
@@ -2529,9 +2609,8 @@ int qed_mcp_nvm_info_populate(struct qed_hwfn *p_hwfn)
 	return rc;
 }
 
-static int
+int
 qed_mcp_get_nvm_image_att(struct qed_hwfn *p_hwfn,
-			  struct qed_ptt *p_ptt,
 			  enum qed_nvm_images image_id,
 			  struct qed_nvm_image_att *p_image_att)
 {
@@ -2546,6 +2625,15 @@ qed_mcp_get_nvm_image_att(struct qed_hwfn *p_hwfn,
 	case QED_NVM_IMAGE_FCOE_CFG:
 		type = NVM_TYPE_FCOE_CFG;
 		break;
+	case QED_NVM_IMAGE_NVM_CFG1:
+		type = NVM_TYPE_NVM_CFG1;
+		break;
+	case QED_NVM_IMAGE_DEFAULT_CFG:
+		type = NVM_TYPE_DEFAULT_CFG;
+		break;
+	case QED_NVM_IMAGE_NVM_META:
+		type = NVM_TYPE_META;
+		break;
 	default:
 		DP_NOTICE(p_hwfn, "Unknown request of image_id %08x\n",
 			  image_id);
@@ -2569,7 +2657,6 @@ qed_mcp_get_nvm_image_att(struct qed_hwfn *p_hwfn,
 }
 
 int qed_mcp_get_nvm_image(struct qed_hwfn *p_hwfn,
-			  struct qed_ptt *p_ptt,
 			  enum qed_nvm_images image_id,
 			  u8 *p_buffer, u32 buffer_len)
 {
@@ -2578,7 +2665,7 @@ int qed_mcp_get_nvm_image(struct qed_hwfn *p_hwfn,
 
 	memset(p_buffer, 0, buffer_len);
 
-	rc = qed_mcp_get_nvm_image_att(p_hwfn, p_ptt, image_id, &image_att);
+	rc = qed_mcp_get_nvm_image_att(p_hwfn, image_id, &image_att);
 	if (rc)
 		return rc;
 
@@ -2590,9 +2677,6 @@ int qed_mcp_get_nvm_image(struct qed_hwfn *p_hwfn,
 		return -EINVAL;
 	}
 
-	/* Each NVM image is suffixed by CRC; Upper-layer has no need for it */
-	image_att.length -= 4;
-
 	if (image_att.length > buffer_len) {
 		DP_VERBOSE(p_hwfn,
 			   QED_MSG_STORAGE,
diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.h b/drivers/net/ethernet/qlogic/qed/qed_mcp.h
index 8a5c988d0c3c..632a838f1fe3 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_mcp.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.h
@@ -213,6 +213,44 @@ enum qed_ov_wol {
 	QED_OV_WOL_ENABLED
 };
 
+enum qed_mfw_tlv_type {
+	QED_MFW_TLV_GENERIC = 0x1,	/* Core driver TLVs */
+	QED_MFW_TLV_ETH = 0x2,		/* L2 driver TLVs */
+	QED_MFW_TLV_FCOE = 0x4,		/* FCoE protocol TLVs */
+	QED_MFW_TLV_ISCSI = 0x8,	/* SCSI protocol TLVs */
+	QED_MFW_TLV_MAX = 0x16,
+};
+
+struct qed_mfw_tlv_generic {
+#define QED_MFW_TLV_FLAGS_SIZE	2
+	struct {
+		u8 ipv4_csum_offload;
+		u8 lso_supported;
+		bool b_set;
+	} flags;
+
+#define QED_MFW_TLV_MAC_COUNT 3
+	/* First entry for primary MAC, 2 secondary MACs possible */
+	u8 mac[QED_MFW_TLV_MAC_COUNT][6];
+	bool mac_set[QED_MFW_TLV_MAC_COUNT];
+
+	u64 rx_frames;
+	bool rx_frames_set;
+	u64 rx_bytes;
+	bool rx_bytes_set;
+	u64 tx_frames;
+	bool tx_frames_set;
+	u64 tx_bytes;
+	bool tx_bytes_set;
+};
+
+union qed_mfw_tlv_data {
+	struct qed_mfw_tlv_generic generic;
+	struct qed_mfw_tlv_eth eth;
+	struct qed_mfw_tlv_fcoe fcoe;
+	struct qed_mfw_tlv_iscsi iscsi;
+};
+
 /**
  * @brief - returns the link params of the hw function
  *
@@ -486,7 +524,20 @@ struct qed_nvm_image_att {
  * @brief Allows reading a whole nvram image
  *
  * @param p_hwfn
- * @param p_ptt
+ * @param image_id - image to get attributes for
+ * @param p_image_att - image attributes structure into which to fill data
+ *
+ * @return int - 0 - operation was successful.
+ */
+int
+qed_mcp_get_nvm_image_att(struct qed_hwfn *p_hwfn,
+			  enum qed_nvm_images image_id,
+			  struct qed_nvm_image_att *p_image_att);
+
+/**
+ * @brief Allows reading a whole nvram image
+ *
+ * @param p_hwfn
  * @param image_id - image requested for reading
  * @param p_buffer - allocated buffer into which to fill data
  * @param buffer_len - length of the allocated buffer.
@@ -494,7 +545,6 @@ struct qed_nvm_image_att {
  * @return 0 iff p_buffer now contains the nvram image.
  */
 int qed_mcp_get_nvm_image(struct qed_hwfn *p_hwfn,
-			  struct qed_ptt *p_ptt,
 			  enum qed_nvm_images image_id,
 			  u8 *p_buffer, u32 buffer_len);
 
@@ -549,6 +599,17 @@ int qed_mcp_bist_nvm_get_image_att(struct qed_hwfn *p_hwfn,
 				   struct bist_nvm_image_att *p_image_att,
 				   u32 image_index);
 
+/**
+ * @brief - Processes the TLV request from MFW i.e., get the required TLV info
+ *          from the qed client and send it to the MFW.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ *
+ * @param return 0 upon success.
+ */
+int qed_mfw_process_tlv_req(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
+
 /* Using hwfn number (and not pf_num) is required since in CMT mode,
  * same pf_num may be used by two different hwfn
  * TODO - this shouldn't really be in .h file, but until all fields
@@ -609,6 +670,14 @@ struct qed_mcp_mb_params {
 	u32			mcp_param;
 };
 
+struct qed_drv_tlv_hdr {
+	u8 tlv_type;
+	u8 tlv_length;	/* In dwords - not including this header */
+	u8 tlv_reserved;
+#define QED_DRV_TLV_FLAGS_CHANGED 0x01
+	u8 tlv_flags;
+};
+
 /**
  * @brief Initialize the interface with the MCP
  *
@@ -992,6 +1061,14 @@ int qed_mcp_get_capabilities(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
  */
 int qed_mcp_set_capabilities(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
 
+/**
+ * @brief Read ufp config from the shared memory.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ */
+void qed_mcp_read_ufp_config(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
+
 /**
  * @brief Populate the nvm info shadow in the given hardware function
  *
diff --git a/drivers/net/ethernet/qlogic/qed/qed_mng_tlv.c b/drivers/net/ethernet/qlogic/qed/qed_mng_tlv.c
new file mode 100644
index 000000000000..6c16158d8090
--- /dev/null
+++ b/drivers/net/ethernet/qlogic/qed/qed_mng_tlv.c
@@ -0,0 +1,1337 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/types.h>
+#include <asm/byteorder.h>
+#include <linux/bug.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/vmalloc.h>
+#include "qed.h"
+#include "qed_hw.h"
+#include "qed_mcp.h"
+#include "qed_reg_addr.h"
+
+#define TLV_TYPE(p)     (p[0])
+#define TLV_LENGTH(p)   (p[1])
+#define TLV_FLAGS(p)    (p[3])
+
+#define QED_TLV_DATA_MAX (14)
+struct qed_tlv_parsed_buf {
+	/* To be filled with the address to set in Value field */
+	void *p_val;
+
+	/* To be used internally in case the value has to be modified */
+	u8 data[QED_TLV_DATA_MAX];
+};
+
+static int qed_mfw_get_tlv_group(u8 tlv_type, u8 *tlv_group)
+{
+	switch (tlv_type) {
+	case DRV_TLV_FEATURE_FLAGS:
+	case DRV_TLV_LOCAL_ADMIN_ADDR:
+	case DRV_TLV_ADDITIONAL_MAC_ADDR_1:
+	case DRV_TLV_ADDITIONAL_MAC_ADDR_2:
+	case DRV_TLV_OS_DRIVER_STATES:
+	case DRV_TLV_PXE_BOOT_PROGRESS:
+	case DRV_TLV_RX_FRAMES_RECEIVED:
+	case DRV_TLV_RX_BYTES_RECEIVED:
+	case DRV_TLV_TX_FRAMES_SENT:
+	case DRV_TLV_TX_BYTES_SENT:
+	case DRV_TLV_NPIV_ENABLED:
+	case DRV_TLV_PCIE_BUS_RX_UTILIZATION:
+	case DRV_TLV_PCIE_BUS_TX_UTILIZATION:
+	case DRV_TLV_DEVICE_CPU_CORES_UTILIZATION:
+	case DRV_TLV_LAST_VALID_DCC_TLV_RECEIVED:
+	case DRV_TLV_NCSI_RX_BYTES_RECEIVED:
+	case DRV_TLV_NCSI_TX_BYTES_SENT:
+		*tlv_group |= QED_MFW_TLV_GENERIC;
+		break;
+	case DRV_TLV_LSO_MAX_OFFLOAD_SIZE:
+	case DRV_TLV_LSO_MIN_SEGMENT_COUNT:
+	case DRV_TLV_PROMISCUOUS_MODE:
+	case DRV_TLV_TX_DESCRIPTORS_QUEUE_SIZE:
+	case DRV_TLV_RX_DESCRIPTORS_QUEUE_SIZE:
+	case DRV_TLV_NUM_OF_NET_QUEUE_VMQ_CFG:
+	case DRV_TLV_NUM_OFFLOADED_CONNECTIONS_TCP_IPV4:
+	case DRV_TLV_NUM_OFFLOADED_CONNECTIONS_TCP_IPV6:
+	case DRV_TLV_TX_DESCRIPTOR_QUEUE_AVG_DEPTH:
+	case DRV_TLV_RX_DESCRIPTORS_QUEUE_AVG_DEPTH:
+	case DRV_TLV_IOV_OFFLOAD:
+	case DRV_TLV_TX_QUEUES_EMPTY:
+	case DRV_TLV_RX_QUEUES_EMPTY:
+	case DRV_TLV_TX_QUEUES_FULL:
+	case DRV_TLV_RX_QUEUES_FULL:
+		*tlv_group |= QED_MFW_TLV_ETH;
+		break;
+	case DRV_TLV_SCSI_TO:
+	case DRV_TLV_R_T_TOV:
+	case DRV_TLV_R_A_TOV:
+	case DRV_TLV_E_D_TOV:
+	case DRV_TLV_CR_TOV:
+	case DRV_TLV_BOOT_TYPE:
+	case DRV_TLV_NPIV_STATE:
+	case DRV_TLV_NUM_OF_NPIV_IDS:
+	case DRV_TLV_SWITCH_NAME:
+	case DRV_TLV_SWITCH_PORT_NUM:
+	case DRV_TLV_SWITCH_PORT_ID:
+	case DRV_TLV_VENDOR_NAME:
+	case DRV_TLV_SWITCH_MODEL:
+	case DRV_TLV_SWITCH_FW_VER:
+	case DRV_TLV_QOS_PRIORITY_PER_802_1P:
+	case DRV_TLV_PORT_ALIAS:
+	case DRV_TLV_PORT_STATE:
+	case DRV_TLV_FIP_TX_DESCRIPTORS_QUEUE_SIZE:
+	case DRV_TLV_FCOE_RX_DESCRIPTORS_QUEUE_SIZE:
+	case DRV_TLV_LINK_FAILURE_COUNT:
+	case DRV_TLV_FCOE_BOOT_PROGRESS:
+	case DRV_TLV_RX_BROADCAST_PACKETS:
+	case DRV_TLV_TX_BROADCAST_PACKETS:
+	case DRV_TLV_FCOE_TX_DESCRIPTOR_QUEUE_AVG_DEPTH:
+	case DRV_TLV_FCOE_RX_DESCRIPTORS_QUEUE_AVG_DEPTH:
+	case DRV_TLV_FCOE_RX_FRAMES_RECEIVED:
+	case DRV_TLV_FCOE_RX_BYTES_RECEIVED:
+	case DRV_TLV_FCOE_TX_FRAMES_SENT:
+	case DRV_TLV_FCOE_TX_BYTES_SENT:
+	case DRV_TLV_CRC_ERROR_COUNT:
+	case DRV_TLV_CRC_ERROR_1_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_CRC_ERROR_1_TIMESTAMP:
+	case DRV_TLV_CRC_ERROR_2_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_CRC_ERROR_2_TIMESTAMP:
+	case DRV_TLV_CRC_ERROR_3_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_CRC_ERROR_3_TIMESTAMP:
+	case DRV_TLV_CRC_ERROR_4_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_CRC_ERROR_4_TIMESTAMP:
+	case DRV_TLV_CRC_ERROR_5_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_CRC_ERROR_5_TIMESTAMP:
+	case DRV_TLV_LOSS_OF_SYNC_ERROR_COUNT:
+	case DRV_TLV_LOSS_OF_SIGNAL_ERRORS:
+	case DRV_TLV_PRIMITIVE_SEQUENCE_PROTOCOL_ERROR_COUNT:
+	case DRV_TLV_DISPARITY_ERROR_COUNT:
+	case DRV_TLV_CODE_VIOLATION_ERROR_COUNT:
+	case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_1:
+	case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_2:
+	case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_3:
+	case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_4:
+	case DRV_TLV_LAST_FLOGI_TIMESTAMP:
+	case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_1:
+	case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_2:
+	case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_3:
+	case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_4:
+	case DRV_TLV_LAST_FLOGI_ACC_TIMESTAMP:
+	case DRV_TLV_LAST_FLOGI_RJT:
+	case DRV_TLV_LAST_FLOGI_RJT_TIMESTAMP:
+	case DRV_TLV_FDISCS_SENT_COUNT:
+	case DRV_TLV_FDISC_ACCS_RECEIVED:
+	case DRV_TLV_FDISC_RJTS_RECEIVED:
+	case DRV_TLV_PLOGI_SENT_COUNT:
+	case DRV_TLV_PLOGI_ACCS_RECEIVED:
+	case DRV_TLV_PLOGI_RJTS_RECEIVED:
+	case DRV_TLV_PLOGI_1_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_PLOGI_1_TIMESTAMP:
+	case DRV_TLV_PLOGI_2_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_PLOGI_2_TIMESTAMP:
+	case DRV_TLV_PLOGI_3_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_PLOGI_3_TIMESTAMP:
+	case DRV_TLV_PLOGI_4_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_PLOGI_4_TIMESTAMP:
+	case DRV_TLV_PLOGI_5_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_PLOGI_5_TIMESTAMP:
+	case DRV_TLV_PLOGI_1_ACC_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_PLOGI_1_ACC_TIMESTAMP:
+	case DRV_TLV_PLOGI_2_ACC_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_PLOGI_2_ACC_TIMESTAMP:
+	case DRV_TLV_PLOGI_3_ACC_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_PLOGI_3_ACC_TIMESTAMP:
+	case DRV_TLV_PLOGI_4_ACC_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_PLOGI_4_ACC_TIMESTAMP:
+	case DRV_TLV_PLOGI_5_ACC_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_PLOGI_5_ACC_TIMESTAMP:
+	case DRV_TLV_LOGOS_ISSUED:
+	case DRV_TLV_LOGO_ACCS_RECEIVED:
+	case DRV_TLV_LOGO_RJTS_RECEIVED:
+	case DRV_TLV_LOGO_1_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_LOGO_1_TIMESTAMP:
+	case DRV_TLV_LOGO_2_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_LOGO_2_TIMESTAMP:
+	case DRV_TLV_LOGO_3_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_LOGO_3_TIMESTAMP:
+	case DRV_TLV_LOGO_4_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_LOGO_4_TIMESTAMP:
+	case DRV_TLV_LOGO_5_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_LOGO_5_TIMESTAMP:
+	case DRV_TLV_LOGOS_RECEIVED:
+	case DRV_TLV_ACCS_ISSUED:
+	case DRV_TLV_PRLIS_ISSUED:
+	case DRV_TLV_ACCS_RECEIVED:
+	case DRV_TLV_ABTS_SENT_COUNT:
+	case DRV_TLV_ABTS_ACCS_RECEIVED:
+	case DRV_TLV_ABTS_RJTS_RECEIVED:
+	case DRV_TLV_ABTS_1_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_ABTS_1_TIMESTAMP:
+	case DRV_TLV_ABTS_2_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_ABTS_2_TIMESTAMP:
+	case DRV_TLV_ABTS_3_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_ABTS_3_TIMESTAMP:
+	case DRV_TLV_ABTS_4_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_ABTS_4_TIMESTAMP:
+	case DRV_TLV_ABTS_5_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_ABTS_5_TIMESTAMP:
+	case DRV_TLV_RSCNS_RECEIVED:
+	case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_1:
+	case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_2:
+	case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_3:
+	case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_4:
+	case DRV_TLV_LUN_RESETS_ISSUED:
+	case DRV_TLV_ABORT_TASK_SETS_ISSUED:
+	case DRV_TLV_TPRLOS_SENT:
+	case DRV_TLV_NOS_SENT_COUNT:
+	case DRV_TLV_NOS_RECEIVED_COUNT:
+	case DRV_TLV_OLS_COUNT:
+	case DRV_TLV_LR_COUNT:
+	case DRV_TLV_LRR_COUNT:
+	case DRV_TLV_LIP_SENT_COUNT:
+	case DRV_TLV_LIP_RECEIVED_COUNT:
+	case DRV_TLV_EOFA_COUNT:
+	case DRV_TLV_EOFNI_COUNT:
+	case DRV_TLV_SCSI_STATUS_CHECK_CONDITION_COUNT:
+	case DRV_TLV_SCSI_STATUS_CONDITION_MET_COUNT:
+	case DRV_TLV_SCSI_STATUS_BUSY_COUNT:
+	case DRV_TLV_SCSI_STATUS_INTERMEDIATE_COUNT:
+	case DRV_TLV_SCSI_STATUS_INTERMEDIATE_CONDITION_MET_COUNT:
+	case DRV_TLV_SCSI_STATUS_RESERVATION_CONFLICT_COUNT:
+	case DRV_TLV_SCSI_STATUS_TASK_SET_FULL_COUNT:
+	case DRV_TLV_SCSI_STATUS_ACA_ACTIVE_COUNT:
+	case DRV_TLV_SCSI_STATUS_TASK_ABORTED_COUNT:
+	case DRV_TLV_SCSI_CHECK_CONDITION_1_RECEIVED_SK_ASC_ASCQ:
+	case DRV_TLV_SCSI_CHECK_1_TIMESTAMP:
+	case DRV_TLV_SCSI_CHECK_CONDITION_2_RECEIVED_SK_ASC_ASCQ:
+	case DRV_TLV_SCSI_CHECK_2_TIMESTAMP:
+	case DRV_TLV_SCSI_CHECK_CONDITION_3_RECEIVED_SK_ASC_ASCQ:
+	case DRV_TLV_SCSI_CHECK_3_TIMESTAMP:
+	case DRV_TLV_SCSI_CHECK_CONDITION_4_RECEIVED_SK_ASC_ASCQ:
+	case DRV_TLV_SCSI_CHECK_4_TIMESTAMP:
+	case DRV_TLV_SCSI_CHECK_CONDITION_5_RECEIVED_SK_ASC_ASCQ:
+	case DRV_TLV_SCSI_CHECK_5_TIMESTAMP:
+		*tlv_group = QED_MFW_TLV_FCOE;
+		break;
+	case DRV_TLV_TARGET_LLMNR_ENABLED:
+	case DRV_TLV_HEADER_DIGEST_FLAG_ENABLED:
+	case DRV_TLV_DATA_DIGEST_FLAG_ENABLED:
+	case DRV_TLV_AUTHENTICATION_METHOD:
+	case DRV_TLV_ISCSI_BOOT_TARGET_PORTAL:
+	case DRV_TLV_MAX_FRAME_SIZE:
+	case DRV_TLV_PDU_TX_DESCRIPTORS_QUEUE_SIZE:
+	case DRV_TLV_PDU_RX_DESCRIPTORS_QUEUE_SIZE:
+	case DRV_TLV_ISCSI_BOOT_PROGRESS:
+	case DRV_TLV_PDU_TX_DESCRIPTOR_QUEUE_AVG_DEPTH:
+	case DRV_TLV_PDU_RX_DESCRIPTORS_QUEUE_AVG_DEPTH:
+	case DRV_TLV_ISCSI_PDU_RX_FRAMES_RECEIVED:
+	case DRV_TLV_ISCSI_PDU_RX_BYTES_RECEIVED:
+	case DRV_TLV_ISCSI_PDU_TX_FRAMES_SENT:
+	case DRV_TLV_ISCSI_PDU_TX_BYTES_SENT:
+		*tlv_group |= QED_MFW_TLV_ISCSI;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* Returns size of the data buffer or, -1 in case TLV data is not available. */
+static int
+qed_mfw_get_gen_tlv_value(struct qed_drv_tlv_hdr *p_tlv,
+			  struct qed_mfw_tlv_generic *p_drv_buf,
+			  struct qed_tlv_parsed_buf *p_buf)
+{
+	switch (p_tlv->tlv_type) {
+	case DRV_TLV_FEATURE_FLAGS:
+		if (p_drv_buf->flags.b_set) {
+			memset(p_buf->data, 0, sizeof(u8) * QED_TLV_DATA_MAX);
+			p_buf->data[0] = p_drv_buf->flags.ipv4_csum_offload ?
+			    1 : 0;
+			p_buf->data[0] |= (p_drv_buf->flags.lso_supported ?
+					   1 : 0) << 1;
+			p_buf->p_val = p_buf->data;
+			return QED_MFW_TLV_FLAGS_SIZE;
+		}
+		break;
+
+	case DRV_TLV_LOCAL_ADMIN_ADDR:
+	case DRV_TLV_ADDITIONAL_MAC_ADDR_1:
+	case DRV_TLV_ADDITIONAL_MAC_ADDR_2:
+		{
+			int idx = p_tlv->tlv_type - DRV_TLV_LOCAL_ADMIN_ADDR;
+
+			if (p_drv_buf->mac_set[idx]) {
+				p_buf->p_val = p_drv_buf->mac[idx];
+				return ETH_ALEN;
+			}
+			break;
+		}
+
+	case DRV_TLV_RX_FRAMES_RECEIVED:
+		if (p_drv_buf->rx_frames_set) {
+			p_buf->p_val = &p_drv_buf->rx_frames;
+			return sizeof(p_drv_buf->rx_frames);
+		}
+		break;
+	case DRV_TLV_RX_BYTES_RECEIVED:
+		if (p_drv_buf->rx_bytes_set) {
+			p_buf->p_val = &p_drv_buf->rx_bytes;
+			return sizeof(p_drv_buf->rx_bytes);
+		}
+		break;
+	case DRV_TLV_TX_FRAMES_SENT:
+		if (p_drv_buf->tx_frames_set) {
+			p_buf->p_val = &p_drv_buf->tx_frames;
+			return sizeof(p_drv_buf->tx_frames);
+		}
+		break;
+	case DRV_TLV_TX_BYTES_SENT:
+		if (p_drv_buf->tx_bytes_set) {
+			p_buf->p_val = &p_drv_buf->tx_bytes;
+			return sizeof(p_drv_buf->tx_bytes);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -1;
+}
+
+static int
+qed_mfw_get_eth_tlv_value(struct qed_drv_tlv_hdr *p_tlv,
+			  struct qed_mfw_tlv_eth *p_drv_buf,
+			  struct qed_tlv_parsed_buf *p_buf)
+{
+	switch (p_tlv->tlv_type) {
+	case DRV_TLV_LSO_MAX_OFFLOAD_SIZE:
+		if (p_drv_buf->lso_maxoff_size_set) {
+			p_buf->p_val = &p_drv_buf->lso_maxoff_size;
+			return sizeof(p_drv_buf->lso_maxoff_size);
+		}
+		break;
+	case DRV_TLV_LSO_MIN_SEGMENT_COUNT:
+		if (p_drv_buf->lso_minseg_size_set) {
+			p_buf->p_val = &p_drv_buf->lso_minseg_size;
+			return sizeof(p_drv_buf->lso_minseg_size);
+		}
+		break;
+	case DRV_TLV_PROMISCUOUS_MODE:
+		if (p_drv_buf->prom_mode_set) {
+			p_buf->p_val = &p_drv_buf->prom_mode;
+			return sizeof(p_drv_buf->prom_mode);
+		}
+		break;
+	case DRV_TLV_TX_DESCRIPTORS_QUEUE_SIZE:
+		if (p_drv_buf->tx_descr_size_set) {
+			p_buf->p_val = &p_drv_buf->tx_descr_size;
+			return sizeof(p_drv_buf->tx_descr_size);
+		}
+		break;
+	case DRV_TLV_RX_DESCRIPTORS_QUEUE_SIZE:
+		if (p_drv_buf->rx_descr_size_set) {
+			p_buf->p_val = &p_drv_buf->rx_descr_size;
+			return sizeof(p_drv_buf->rx_descr_size);
+		}
+		break;
+	case DRV_TLV_NUM_OF_NET_QUEUE_VMQ_CFG:
+		if (p_drv_buf->netq_count_set) {
+			p_buf->p_val = &p_drv_buf->netq_count;
+			return sizeof(p_drv_buf->netq_count);
+		}
+		break;
+	case DRV_TLV_NUM_OFFLOADED_CONNECTIONS_TCP_IPV4:
+		if (p_drv_buf->tcp4_offloads_set) {
+			p_buf->p_val = &p_drv_buf->tcp4_offloads;
+			return sizeof(p_drv_buf->tcp4_offloads);
+		}
+		break;
+	case DRV_TLV_NUM_OFFLOADED_CONNECTIONS_TCP_IPV6:
+		if (p_drv_buf->tcp6_offloads_set) {
+			p_buf->p_val = &p_drv_buf->tcp6_offloads;
+			return sizeof(p_drv_buf->tcp6_offloads);
+		}
+		break;
+	case DRV_TLV_TX_DESCRIPTOR_QUEUE_AVG_DEPTH:
+		if (p_drv_buf->tx_descr_qdepth_set) {
+			p_buf->p_val = &p_drv_buf->tx_descr_qdepth;
+			return sizeof(p_drv_buf->tx_descr_qdepth);
+		}
+		break;
+	case DRV_TLV_RX_DESCRIPTORS_QUEUE_AVG_DEPTH:
+		if (p_drv_buf->rx_descr_qdepth_set) {
+			p_buf->p_val = &p_drv_buf->rx_descr_qdepth;
+			return sizeof(p_drv_buf->rx_descr_qdepth);
+		}
+		break;
+	case DRV_TLV_IOV_OFFLOAD:
+		if (p_drv_buf->iov_offload_set) {
+			p_buf->p_val = &p_drv_buf->iov_offload;
+			return sizeof(p_drv_buf->iov_offload);
+		}
+		break;
+	case DRV_TLV_TX_QUEUES_EMPTY:
+		if (p_drv_buf->txqs_empty_set) {
+			p_buf->p_val = &p_drv_buf->txqs_empty;
+			return sizeof(p_drv_buf->txqs_empty);
+		}
+		break;
+	case DRV_TLV_RX_QUEUES_EMPTY:
+		if (p_drv_buf->rxqs_empty_set) {
+			p_buf->p_val = &p_drv_buf->rxqs_empty;
+			return sizeof(p_drv_buf->rxqs_empty);
+		}
+		break;
+	case DRV_TLV_TX_QUEUES_FULL:
+		if (p_drv_buf->num_txqs_full_set) {
+			p_buf->p_val = &p_drv_buf->num_txqs_full;
+			return sizeof(p_drv_buf->num_txqs_full);
+		}
+		break;
+	case DRV_TLV_RX_QUEUES_FULL:
+		if (p_drv_buf->num_rxqs_full_set) {
+			p_buf->p_val = &p_drv_buf->num_rxqs_full;
+			return sizeof(p_drv_buf->num_rxqs_full);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -1;
+}
+
+static int
+qed_mfw_get_tlv_time_value(struct qed_mfw_tlv_time *p_time,
+			   struct qed_tlv_parsed_buf *p_buf)
+{
+	if (!p_time->b_set)
+		return -1;
+
+	/* Validate numbers */
+	if (p_time->month > 12)
+		p_time->month = 0;
+	if (p_time->day > 31)
+		p_time->day = 0;
+	if (p_time->hour > 23)
+		p_time->hour = 0;
+	if (p_time->min > 59)
+		p_time->hour = 0;
+	if (p_time->msec > 999)
+		p_time->msec = 0;
+	if (p_time->usec > 999)
+		p_time->usec = 0;
+
+	memset(p_buf->data, 0, sizeof(u8) * QED_TLV_DATA_MAX);
+	snprintf(p_buf->data, 14, "%d%d%d%d%d%d",
+		 p_time->month, p_time->day,
+		 p_time->hour, p_time->min, p_time->msec, p_time->usec);
+
+	p_buf->p_val = p_buf->data;
+
+	return QED_MFW_TLV_TIME_SIZE;
+}
+
+static int
+qed_mfw_get_fcoe_tlv_value(struct qed_drv_tlv_hdr *p_tlv,
+			   struct qed_mfw_tlv_fcoe *p_drv_buf,
+			   struct qed_tlv_parsed_buf *p_buf)
+{
+	struct qed_mfw_tlv_time *p_time;
+	u8 idx;
+
+	switch (p_tlv->tlv_type) {
+	case DRV_TLV_SCSI_TO:
+		if (p_drv_buf->scsi_timeout_set) {
+			p_buf->p_val = &p_drv_buf->scsi_timeout;
+			return sizeof(p_drv_buf->scsi_timeout);
+		}
+		break;
+	case DRV_TLV_R_T_TOV:
+		if (p_drv_buf->rt_tov_set) {
+			p_buf->p_val = &p_drv_buf->rt_tov;
+			return sizeof(p_drv_buf->rt_tov);
+		}
+		break;
+	case DRV_TLV_R_A_TOV:
+		if (p_drv_buf->ra_tov_set) {
+			p_buf->p_val = &p_drv_buf->ra_tov;
+			return sizeof(p_drv_buf->ra_tov);
+		}
+		break;
+	case DRV_TLV_E_D_TOV:
+		if (p_drv_buf->ed_tov_set) {
+			p_buf->p_val = &p_drv_buf->ed_tov;
+			return sizeof(p_drv_buf->ed_tov);
+		}
+		break;
+	case DRV_TLV_CR_TOV:
+		if (p_drv_buf->cr_tov_set) {
+			p_buf->p_val = &p_drv_buf->cr_tov;
+			return sizeof(p_drv_buf->cr_tov);
+		}
+		break;
+	case DRV_TLV_BOOT_TYPE:
+		if (p_drv_buf->boot_type_set) {
+			p_buf->p_val = &p_drv_buf->boot_type;
+			return sizeof(p_drv_buf->boot_type);
+		}
+		break;
+	case DRV_TLV_NPIV_STATE:
+		if (p_drv_buf->npiv_state_set) {
+			p_buf->p_val = &p_drv_buf->npiv_state;
+			return sizeof(p_drv_buf->npiv_state);
+		}
+		break;
+	case DRV_TLV_NUM_OF_NPIV_IDS:
+		if (p_drv_buf->num_npiv_ids_set) {
+			p_buf->p_val = &p_drv_buf->num_npiv_ids;
+			return sizeof(p_drv_buf->num_npiv_ids);
+		}
+		break;
+	case DRV_TLV_SWITCH_NAME:
+		if (p_drv_buf->switch_name_set) {
+			p_buf->p_val = &p_drv_buf->switch_name;
+			return sizeof(p_drv_buf->switch_name);
+		}
+		break;
+	case DRV_TLV_SWITCH_PORT_NUM:
+		if (p_drv_buf->switch_portnum_set) {
+			p_buf->p_val = &p_drv_buf->switch_portnum;
+			return sizeof(p_drv_buf->switch_portnum);
+		}
+		break;
+	case DRV_TLV_SWITCH_PORT_ID:
+		if (p_drv_buf->switch_portid_set) {
+			p_buf->p_val = &p_drv_buf->switch_portid;
+			return sizeof(p_drv_buf->switch_portid);
+		}
+		break;
+	case DRV_TLV_VENDOR_NAME:
+		if (p_drv_buf->vendor_name_set) {
+			p_buf->p_val = &p_drv_buf->vendor_name;
+			return sizeof(p_drv_buf->vendor_name);
+		}
+		break;
+	case DRV_TLV_SWITCH_MODEL:
+		if (p_drv_buf->switch_model_set) {
+			p_buf->p_val = &p_drv_buf->switch_model;
+			return sizeof(p_drv_buf->switch_model);
+		}
+		break;
+	case DRV_TLV_SWITCH_FW_VER:
+		if (p_drv_buf->switch_fw_version_set) {
+			p_buf->p_val = &p_drv_buf->switch_fw_version;
+			return sizeof(p_drv_buf->switch_fw_version);
+		}
+		break;
+	case DRV_TLV_QOS_PRIORITY_PER_802_1P:
+		if (p_drv_buf->qos_pri_set) {
+			p_buf->p_val = &p_drv_buf->qos_pri;
+			return sizeof(p_drv_buf->qos_pri);
+		}
+		break;
+	case DRV_TLV_PORT_ALIAS:
+		if (p_drv_buf->port_alias_set) {
+			p_buf->p_val = &p_drv_buf->port_alias;
+			return sizeof(p_drv_buf->port_alias);
+		}
+		break;
+	case DRV_TLV_PORT_STATE:
+		if (p_drv_buf->port_state_set) {
+			p_buf->p_val = &p_drv_buf->port_state;
+			return sizeof(p_drv_buf->port_state);
+		}
+		break;
+	case DRV_TLV_FIP_TX_DESCRIPTORS_QUEUE_SIZE:
+		if (p_drv_buf->fip_tx_descr_size_set) {
+			p_buf->p_val = &p_drv_buf->fip_tx_descr_size;
+			return sizeof(p_drv_buf->fip_tx_descr_size);
+		}
+		break;
+	case DRV_TLV_FCOE_RX_DESCRIPTORS_QUEUE_SIZE:
+		if (p_drv_buf->fip_rx_descr_size_set) {
+			p_buf->p_val = &p_drv_buf->fip_rx_descr_size;
+			return sizeof(p_drv_buf->fip_rx_descr_size);
+		}
+		break;
+	case DRV_TLV_LINK_FAILURE_COUNT:
+		if (p_drv_buf->link_failures_set) {
+			p_buf->p_val = &p_drv_buf->link_failures;
+			return sizeof(p_drv_buf->link_failures);
+		}
+		break;
+	case DRV_TLV_FCOE_BOOT_PROGRESS:
+		if (p_drv_buf->fcoe_boot_progress_set) {
+			p_buf->p_val = &p_drv_buf->fcoe_boot_progress;
+			return sizeof(p_drv_buf->fcoe_boot_progress);
+		}
+		break;
+	case DRV_TLV_RX_BROADCAST_PACKETS:
+		if (p_drv_buf->rx_bcast_set) {
+			p_buf->p_val = &p_drv_buf->rx_bcast;
+			return sizeof(p_drv_buf->rx_bcast);
+		}
+		break;
+	case DRV_TLV_TX_BROADCAST_PACKETS:
+		if (p_drv_buf->tx_bcast_set) {
+			p_buf->p_val = &p_drv_buf->tx_bcast;
+			return sizeof(p_drv_buf->tx_bcast);
+		}
+		break;
+	case DRV_TLV_FCOE_TX_DESCRIPTOR_QUEUE_AVG_DEPTH:
+		if (p_drv_buf->fcoe_txq_depth_set) {
+			p_buf->p_val = &p_drv_buf->fcoe_txq_depth;
+			return sizeof(p_drv_buf->fcoe_txq_depth);
+		}
+		break;
+	case DRV_TLV_FCOE_RX_DESCRIPTORS_QUEUE_AVG_DEPTH:
+		if (p_drv_buf->fcoe_rxq_depth_set) {
+			p_buf->p_val = &p_drv_buf->fcoe_rxq_depth;
+			return sizeof(p_drv_buf->fcoe_rxq_depth);
+		}
+		break;
+	case DRV_TLV_FCOE_RX_FRAMES_RECEIVED:
+		if (p_drv_buf->fcoe_rx_frames_set) {
+			p_buf->p_val = &p_drv_buf->fcoe_rx_frames;
+			return sizeof(p_drv_buf->fcoe_rx_frames);
+		}
+		break;
+	case DRV_TLV_FCOE_RX_BYTES_RECEIVED:
+		if (p_drv_buf->fcoe_rx_bytes_set) {
+			p_buf->p_val = &p_drv_buf->fcoe_rx_bytes;
+			return sizeof(p_drv_buf->fcoe_rx_bytes);
+		}
+		break;
+	case DRV_TLV_FCOE_TX_FRAMES_SENT:
+		if (p_drv_buf->fcoe_tx_frames_set) {
+			p_buf->p_val = &p_drv_buf->fcoe_tx_frames;
+			return sizeof(p_drv_buf->fcoe_tx_frames);
+		}
+		break;
+	case DRV_TLV_FCOE_TX_BYTES_SENT:
+		if (p_drv_buf->fcoe_tx_bytes_set) {
+			p_buf->p_val = &p_drv_buf->fcoe_tx_bytes;
+			return sizeof(p_drv_buf->fcoe_tx_bytes);
+		}
+		break;
+	case DRV_TLV_CRC_ERROR_COUNT:
+		if (p_drv_buf->crc_count_set) {
+			p_buf->p_val = &p_drv_buf->crc_count;
+			return sizeof(p_drv_buf->crc_count);
+		}
+		break;
+	case DRV_TLV_CRC_ERROR_1_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_CRC_ERROR_2_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_CRC_ERROR_3_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_CRC_ERROR_4_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_CRC_ERROR_5_RECEIVED_SOURCE_FC_ID:
+		idx = (p_tlv->tlv_type -
+		       DRV_TLV_CRC_ERROR_1_RECEIVED_SOURCE_FC_ID) / 2;
+
+		if (p_drv_buf->crc_err_src_fcid_set[idx]) {
+			p_buf->p_val = &p_drv_buf->crc_err_src_fcid[idx];
+			return sizeof(p_drv_buf->crc_err_src_fcid[idx]);
+		}
+		break;
+	case DRV_TLV_CRC_ERROR_1_TIMESTAMP:
+	case DRV_TLV_CRC_ERROR_2_TIMESTAMP:
+	case DRV_TLV_CRC_ERROR_3_TIMESTAMP:
+	case DRV_TLV_CRC_ERROR_4_TIMESTAMP:
+	case DRV_TLV_CRC_ERROR_5_TIMESTAMP:
+		idx = (p_tlv->tlv_type - DRV_TLV_CRC_ERROR_1_TIMESTAMP) / 2;
+
+		return qed_mfw_get_tlv_time_value(&p_drv_buf->crc_err[idx],
+						  p_buf);
+	case DRV_TLV_LOSS_OF_SYNC_ERROR_COUNT:
+		if (p_drv_buf->losync_err_set) {
+			p_buf->p_val = &p_drv_buf->losync_err;
+			return sizeof(p_drv_buf->losync_err);
+		}
+		break;
+	case DRV_TLV_LOSS_OF_SIGNAL_ERRORS:
+		if (p_drv_buf->losig_err_set) {
+			p_buf->p_val = &p_drv_buf->losig_err;
+			return sizeof(p_drv_buf->losig_err);
+		}
+		break;
+	case DRV_TLV_PRIMITIVE_SEQUENCE_PROTOCOL_ERROR_COUNT:
+		if (p_drv_buf->primtive_err_set) {
+			p_buf->p_val = &p_drv_buf->primtive_err;
+			return sizeof(p_drv_buf->primtive_err);
+		}
+		break;
+	case DRV_TLV_DISPARITY_ERROR_COUNT:
+		if (p_drv_buf->disparity_err_set) {
+			p_buf->p_val = &p_drv_buf->disparity_err;
+			return sizeof(p_drv_buf->disparity_err);
+		}
+		break;
+	case DRV_TLV_CODE_VIOLATION_ERROR_COUNT:
+		if (p_drv_buf->code_violation_err_set) {
+			p_buf->p_val = &p_drv_buf->code_violation_err;
+			return sizeof(p_drv_buf->code_violation_err);
+		}
+		break;
+	case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_1:
+	case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_2:
+	case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_3:
+	case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_4:
+		idx = p_tlv->tlv_type -
+			DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_1;
+		if (p_drv_buf->flogi_param_set[idx]) {
+			p_buf->p_val = &p_drv_buf->flogi_param[idx];
+			return sizeof(p_drv_buf->flogi_param[idx]);
+		}
+		break;
+	case DRV_TLV_LAST_FLOGI_TIMESTAMP:
+		return qed_mfw_get_tlv_time_value(&p_drv_buf->flogi_tstamp,
+						  p_buf);
+	case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_1:
+	case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_2:
+	case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_3:
+	case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_4:
+		idx = p_tlv->tlv_type -
+			DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_1;
+
+		if (p_drv_buf->flogi_acc_param_set[idx]) {
+			p_buf->p_val = &p_drv_buf->flogi_acc_param[idx];
+			return sizeof(p_drv_buf->flogi_acc_param[idx]);
+		}
+		break;
+	case DRV_TLV_LAST_FLOGI_ACC_TIMESTAMP:
+		return qed_mfw_get_tlv_time_value(&p_drv_buf->flogi_acc_tstamp,
+						  p_buf);
+	case DRV_TLV_LAST_FLOGI_RJT:
+		if (p_drv_buf->flogi_rjt_set) {
+			p_buf->p_val = &p_drv_buf->flogi_rjt;
+			return sizeof(p_drv_buf->flogi_rjt);
+		}
+		break;
+	case DRV_TLV_LAST_FLOGI_RJT_TIMESTAMP:
+		return qed_mfw_get_tlv_time_value(&p_drv_buf->flogi_rjt_tstamp,
+						  p_buf);
+	case DRV_TLV_FDISCS_SENT_COUNT:
+		if (p_drv_buf->fdiscs_set) {
+			p_buf->p_val = &p_drv_buf->fdiscs;
+			return sizeof(p_drv_buf->fdiscs);
+		}
+		break;
+	case DRV_TLV_FDISC_ACCS_RECEIVED:
+		if (p_drv_buf->fdisc_acc_set) {
+			p_buf->p_val = &p_drv_buf->fdisc_acc;
+			return sizeof(p_drv_buf->fdisc_acc);
+		}
+		break;
+	case DRV_TLV_FDISC_RJTS_RECEIVED:
+		if (p_drv_buf->fdisc_rjt_set) {
+			p_buf->p_val = &p_drv_buf->fdisc_rjt;
+			return sizeof(p_drv_buf->fdisc_rjt);
+		}
+		break;
+	case DRV_TLV_PLOGI_SENT_COUNT:
+		if (p_drv_buf->plogi_set) {
+			p_buf->p_val = &p_drv_buf->plogi;
+			return sizeof(p_drv_buf->plogi);
+		}
+		break;
+	case DRV_TLV_PLOGI_ACCS_RECEIVED:
+		if (p_drv_buf->plogi_acc_set) {
+			p_buf->p_val = &p_drv_buf->plogi_acc;
+			return sizeof(p_drv_buf->plogi_acc);
+		}
+		break;
+	case DRV_TLV_PLOGI_RJTS_RECEIVED:
+		if (p_drv_buf->plogi_rjt_set) {
+			p_buf->p_val = &p_drv_buf->plogi_rjt;
+			return sizeof(p_drv_buf->plogi_rjt);
+		}
+		break;
+	case DRV_TLV_PLOGI_1_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_PLOGI_2_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_PLOGI_3_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_PLOGI_4_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_PLOGI_5_SENT_DESTINATION_FC_ID:
+		idx = (p_tlv->tlv_type -
+		       DRV_TLV_PLOGI_1_SENT_DESTINATION_FC_ID) / 2;
+
+		if (p_drv_buf->plogi_dst_fcid_set[idx]) {
+			p_buf->p_val = &p_drv_buf->plogi_dst_fcid[idx];
+			return sizeof(p_drv_buf->plogi_dst_fcid[idx]);
+		}
+		break;
+	case DRV_TLV_PLOGI_1_TIMESTAMP:
+	case DRV_TLV_PLOGI_2_TIMESTAMP:
+	case DRV_TLV_PLOGI_3_TIMESTAMP:
+	case DRV_TLV_PLOGI_4_TIMESTAMP:
+	case DRV_TLV_PLOGI_5_TIMESTAMP:
+		idx = (p_tlv->tlv_type - DRV_TLV_PLOGI_1_TIMESTAMP) / 2;
+
+		return qed_mfw_get_tlv_time_value(&p_drv_buf->plogi_tstamp[idx],
+						  p_buf);
+	case DRV_TLV_PLOGI_1_ACC_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_PLOGI_2_ACC_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_PLOGI_3_ACC_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_PLOGI_4_ACC_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_PLOGI_5_ACC_RECEIVED_SOURCE_FC_ID:
+		idx = (p_tlv->tlv_type -
+		       DRV_TLV_PLOGI_1_ACC_RECEIVED_SOURCE_FC_ID) / 2;
+
+		if (p_drv_buf->plogi_acc_src_fcid_set[idx]) {
+			p_buf->p_val = &p_drv_buf->plogi_acc_src_fcid[idx];
+			return sizeof(p_drv_buf->plogi_acc_src_fcid[idx]);
+		}
+		break;
+	case DRV_TLV_PLOGI_1_ACC_TIMESTAMP:
+	case DRV_TLV_PLOGI_2_ACC_TIMESTAMP:
+	case DRV_TLV_PLOGI_3_ACC_TIMESTAMP:
+	case DRV_TLV_PLOGI_4_ACC_TIMESTAMP:
+	case DRV_TLV_PLOGI_5_ACC_TIMESTAMP:
+		idx = (p_tlv->tlv_type - DRV_TLV_PLOGI_1_ACC_TIMESTAMP) / 2;
+		p_time = &p_drv_buf->plogi_acc_tstamp[idx];
+
+		return qed_mfw_get_tlv_time_value(p_time, p_buf);
+	case DRV_TLV_LOGOS_ISSUED:
+		if (p_drv_buf->tx_plogos_set) {
+			p_buf->p_val = &p_drv_buf->tx_plogos;
+			return sizeof(p_drv_buf->tx_plogos);
+		}
+		break;
+	case DRV_TLV_LOGO_ACCS_RECEIVED:
+		if (p_drv_buf->plogo_acc_set) {
+			p_buf->p_val = &p_drv_buf->plogo_acc;
+			return sizeof(p_drv_buf->plogo_acc);
+		}
+		break;
+	case DRV_TLV_LOGO_RJTS_RECEIVED:
+		if (p_drv_buf->plogo_rjt_set) {
+			p_buf->p_val = &p_drv_buf->plogo_rjt;
+			return sizeof(p_drv_buf->plogo_rjt);
+		}
+		break;
+	case DRV_TLV_LOGO_1_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_LOGO_2_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_LOGO_3_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_LOGO_4_RECEIVED_SOURCE_FC_ID:
+	case DRV_TLV_LOGO_5_RECEIVED_SOURCE_FC_ID:
+		idx = (p_tlv->tlv_type - DRV_TLV_LOGO_1_RECEIVED_SOURCE_FC_ID) /
+			2;
+
+		if (p_drv_buf->plogo_src_fcid_set[idx]) {
+			p_buf->p_val = &p_drv_buf->plogo_src_fcid[idx];
+			return sizeof(p_drv_buf->plogo_src_fcid[idx]);
+		}
+		break;
+	case DRV_TLV_LOGO_1_TIMESTAMP:
+	case DRV_TLV_LOGO_2_TIMESTAMP:
+	case DRV_TLV_LOGO_3_TIMESTAMP:
+	case DRV_TLV_LOGO_4_TIMESTAMP:
+	case DRV_TLV_LOGO_5_TIMESTAMP:
+		idx = (p_tlv->tlv_type - DRV_TLV_LOGO_1_TIMESTAMP) / 2;
+
+		return qed_mfw_get_tlv_time_value(&p_drv_buf->plogo_tstamp[idx],
+						  p_buf);
+	case DRV_TLV_LOGOS_RECEIVED:
+		if (p_drv_buf->rx_logos_set) {
+			p_buf->p_val = &p_drv_buf->rx_logos;
+			return sizeof(p_drv_buf->rx_logos);
+		}
+		break;
+	case DRV_TLV_ACCS_ISSUED:
+		if (p_drv_buf->tx_accs_set) {
+			p_buf->p_val = &p_drv_buf->tx_accs;
+			return sizeof(p_drv_buf->tx_accs);
+		}
+		break;
+	case DRV_TLV_PRLIS_ISSUED:
+		if (p_drv_buf->tx_prlis_set) {
+			p_buf->p_val = &p_drv_buf->tx_prlis;
+			return sizeof(p_drv_buf->tx_prlis);
+		}
+		break;
+	case DRV_TLV_ACCS_RECEIVED:
+		if (p_drv_buf->rx_accs_set) {
+			p_buf->p_val = &p_drv_buf->rx_accs;
+			return sizeof(p_drv_buf->rx_accs);
+		}
+		break;
+	case DRV_TLV_ABTS_SENT_COUNT:
+		if (p_drv_buf->tx_abts_set) {
+			p_buf->p_val = &p_drv_buf->tx_abts;
+			return sizeof(p_drv_buf->tx_abts);
+		}
+		break;
+	case DRV_TLV_ABTS_ACCS_RECEIVED:
+		if (p_drv_buf->rx_abts_acc_set) {
+			p_buf->p_val = &p_drv_buf->rx_abts_acc;
+			return sizeof(p_drv_buf->rx_abts_acc);
+		}
+		break;
+	case DRV_TLV_ABTS_RJTS_RECEIVED:
+		if (p_drv_buf->rx_abts_rjt_set) {
+			p_buf->p_val = &p_drv_buf->rx_abts_rjt;
+			return sizeof(p_drv_buf->rx_abts_rjt);
+		}
+		break;
+	case DRV_TLV_ABTS_1_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_ABTS_2_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_ABTS_3_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_ABTS_4_SENT_DESTINATION_FC_ID:
+	case DRV_TLV_ABTS_5_SENT_DESTINATION_FC_ID:
+		idx = (p_tlv->tlv_type -
+		       DRV_TLV_ABTS_1_SENT_DESTINATION_FC_ID) / 2;
+
+		if (p_drv_buf->abts_dst_fcid_set[idx]) {
+			p_buf->p_val = &p_drv_buf->abts_dst_fcid[idx];
+			return sizeof(p_drv_buf->abts_dst_fcid[idx]);
+		}
+		break;
+	case DRV_TLV_ABTS_1_TIMESTAMP:
+	case DRV_TLV_ABTS_2_TIMESTAMP:
+	case DRV_TLV_ABTS_3_TIMESTAMP:
+	case DRV_TLV_ABTS_4_TIMESTAMP:
+	case DRV_TLV_ABTS_5_TIMESTAMP:
+		idx = (p_tlv->tlv_type - DRV_TLV_ABTS_1_TIMESTAMP) / 2;
+
+		return qed_mfw_get_tlv_time_value(&p_drv_buf->abts_tstamp[idx],
+						  p_buf);
+	case DRV_TLV_RSCNS_RECEIVED:
+		if (p_drv_buf->rx_rscn_set) {
+			p_buf->p_val = &p_drv_buf->rx_rscn;
+			return sizeof(p_drv_buf->rx_rscn);
+		}
+		break;
+	case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_1:
+	case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_2:
+	case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_3:
+	case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_4:
+		idx = p_tlv->tlv_type - DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_1;
+
+		if (p_drv_buf->rx_rscn_nport_set[idx]) {
+			p_buf->p_val = &p_drv_buf->rx_rscn_nport[idx];
+			return sizeof(p_drv_buf->rx_rscn_nport[idx]);
+		}
+		break;
+	case DRV_TLV_LUN_RESETS_ISSUED:
+		if (p_drv_buf->tx_lun_rst_set) {
+			p_buf->p_val = &p_drv_buf->tx_lun_rst;
+			return sizeof(p_drv_buf->tx_lun_rst);
+		}
+		break;
+	case DRV_TLV_ABORT_TASK_SETS_ISSUED:
+		if (p_drv_buf->abort_task_sets_set) {
+			p_buf->p_val = &p_drv_buf->abort_task_sets;
+			return sizeof(p_drv_buf->abort_task_sets);
+		}
+		break;
+	case DRV_TLV_TPRLOS_SENT:
+		if (p_drv_buf->tx_tprlos_set) {
+			p_buf->p_val = &p_drv_buf->tx_tprlos;
+			return sizeof(p_drv_buf->tx_tprlos);
+		}
+		break;
+	case DRV_TLV_NOS_SENT_COUNT:
+		if (p_drv_buf->tx_nos_set) {
+			p_buf->p_val = &p_drv_buf->tx_nos;
+			return sizeof(p_drv_buf->tx_nos);
+		}
+		break;
+	case DRV_TLV_NOS_RECEIVED_COUNT:
+		if (p_drv_buf->rx_nos_set) {
+			p_buf->p_val = &p_drv_buf->rx_nos;
+			return sizeof(p_drv_buf->rx_nos);
+		}
+		break;
+	case DRV_TLV_OLS_COUNT:
+		if (p_drv_buf->ols_set) {
+			p_buf->p_val = &p_drv_buf->ols;
+			return sizeof(p_drv_buf->ols);
+		}
+		break;
+	case DRV_TLV_LR_COUNT:
+		if (p_drv_buf->lr_set) {
+			p_buf->p_val = &p_drv_buf->lr;
+			return sizeof(p_drv_buf->lr);
+		}
+		break;
+	case DRV_TLV_LRR_COUNT:
+		if (p_drv_buf->lrr_set) {
+			p_buf->p_val = &p_drv_buf->lrr;
+			return sizeof(p_drv_buf->lrr);
+		}
+		break;
+	case DRV_TLV_LIP_SENT_COUNT:
+		if (p_drv_buf->tx_lip_set) {
+			p_buf->p_val = &p_drv_buf->tx_lip;
+			return sizeof(p_drv_buf->tx_lip);
+		}
+		break;
+	case DRV_TLV_LIP_RECEIVED_COUNT:
+		if (p_drv_buf->rx_lip_set) {
+			p_buf->p_val = &p_drv_buf->rx_lip;
+			return sizeof(p_drv_buf->rx_lip);
+		}
+		break;
+	case DRV_TLV_EOFA_COUNT:
+		if (p_drv_buf->eofa_set) {
+			p_buf->p_val = &p_drv_buf->eofa;
+			return sizeof(p_drv_buf->eofa);
+		}
+		break;
+	case DRV_TLV_EOFNI_COUNT:
+		if (p_drv_buf->eofni_set) {
+			p_buf->p_val = &p_drv_buf->eofni;
+			return sizeof(p_drv_buf->eofni);
+		}
+		break;
+	case DRV_TLV_SCSI_STATUS_CHECK_CONDITION_COUNT:
+		if (p_drv_buf->scsi_chks_set) {
+			p_buf->p_val = &p_drv_buf->scsi_chks;
+			return sizeof(p_drv_buf->scsi_chks);
+		}
+		break;
+	case DRV_TLV_SCSI_STATUS_CONDITION_MET_COUNT:
+		if (p_drv_buf->scsi_cond_met_set) {
+			p_buf->p_val = &p_drv_buf->scsi_cond_met;
+			return sizeof(p_drv_buf->scsi_cond_met);
+		}
+		break;
+	case DRV_TLV_SCSI_STATUS_BUSY_COUNT:
+		if (p_drv_buf->scsi_busy_set) {
+			p_buf->p_val = &p_drv_buf->scsi_busy;
+			return sizeof(p_drv_buf->scsi_busy);
+		}
+		break;
+	case DRV_TLV_SCSI_STATUS_INTERMEDIATE_COUNT:
+		if (p_drv_buf->scsi_inter_set) {
+			p_buf->p_val = &p_drv_buf->scsi_inter;
+			return sizeof(p_drv_buf->scsi_inter);
+		}
+		break;
+	case DRV_TLV_SCSI_STATUS_INTERMEDIATE_CONDITION_MET_COUNT:
+		if (p_drv_buf->scsi_inter_cond_met_set) {
+			p_buf->p_val = &p_drv_buf->scsi_inter_cond_met;
+			return sizeof(p_drv_buf->scsi_inter_cond_met);
+		}
+		break;
+	case DRV_TLV_SCSI_STATUS_RESERVATION_CONFLICT_COUNT:
+		if (p_drv_buf->scsi_rsv_conflicts_set) {
+			p_buf->p_val = &p_drv_buf->scsi_rsv_conflicts;
+			return sizeof(p_drv_buf->scsi_rsv_conflicts);
+		}
+		break;
+	case DRV_TLV_SCSI_STATUS_TASK_SET_FULL_COUNT:
+		if (p_drv_buf->scsi_tsk_full_set) {
+			p_buf->p_val = &p_drv_buf->scsi_tsk_full;
+			return sizeof(p_drv_buf->scsi_tsk_full);
+		}
+		break;
+	case DRV_TLV_SCSI_STATUS_ACA_ACTIVE_COUNT:
+		if (p_drv_buf->scsi_aca_active_set) {
+			p_buf->p_val = &p_drv_buf->scsi_aca_active;
+			return sizeof(p_drv_buf->scsi_aca_active);
+		}
+		break;
+	case DRV_TLV_SCSI_STATUS_TASK_ABORTED_COUNT:
+		if (p_drv_buf->scsi_tsk_abort_set) {
+			p_buf->p_val = &p_drv_buf->scsi_tsk_abort;
+			return sizeof(p_drv_buf->scsi_tsk_abort);
+		}
+		break;
+	case DRV_TLV_SCSI_CHECK_CONDITION_1_RECEIVED_SK_ASC_ASCQ:
+	case DRV_TLV_SCSI_CHECK_CONDITION_2_RECEIVED_SK_ASC_ASCQ:
+	case DRV_TLV_SCSI_CHECK_CONDITION_3_RECEIVED_SK_ASC_ASCQ:
+	case DRV_TLV_SCSI_CHECK_CONDITION_4_RECEIVED_SK_ASC_ASCQ:
+	case DRV_TLV_SCSI_CHECK_CONDITION_5_RECEIVED_SK_ASC_ASCQ:
+		idx = (p_tlv->tlv_type -
+		       DRV_TLV_SCSI_CHECK_CONDITION_1_RECEIVED_SK_ASC_ASCQ) / 2;
+
+		if (p_drv_buf->scsi_rx_chk_set[idx]) {
+			p_buf->p_val = &p_drv_buf->scsi_rx_chk[idx];
+			return sizeof(p_drv_buf->scsi_rx_chk[idx]);
+		}
+		break;
+	case DRV_TLV_SCSI_CHECK_1_TIMESTAMP:
+	case DRV_TLV_SCSI_CHECK_2_TIMESTAMP:
+	case DRV_TLV_SCSI_CHECK_3_TIMESTAMP:
+	case DRV_TLV_SCSI_CHECK_4_TIMESTAMP:
+	case DRV_TLV_SCSI_CHECK_5_TIMESTAMP:
+		idx = (p_tlv->tlv_type - DRV_TLV_SCSI_CHECK_1_TIMESTAMP) / 2;
+		p_time = &p_drv_buf->scsi_chk_tstamp[idx];
+
+		return qed_mfw_get_tlv_time_value(p_time, p_buf);
+	default:
+		break;
+	}
+
+	return -1;
+}
+
+static int
+qed_mfw_get_iscsi_tlv_value(struct qed_drv_tlv_hdr *p_tlv,
+			    struct qed_mfw_tlv_iscsi *p_drv_buf,
+			    struct qed_tlv_parsed_buf *p_buf)
+{
+	switch (p_tlv->tlv_type) {
+	case DRV_TLV_TARGET_LLMNR_ENABLED:
+		if (p_drv_buf->target_llmnr_set) {
+			p_buf->p_val = &p_drv_buf->target_llmnr;
+			return sizeof(p_drv_buf->target_llmnr);
+		}
+		break;
+	case DRV_TLV_HEADER_DIGEST_FLAG_ENABLED:
+		if (p_drv_buf->header_digest_set) {
+			p_buf->p_val = &p_drv_buf->header_digest;
+			return sizeof(p_drv_buf->header_digest);
+		}
+		break;
+	case DRV_TLV_DATA_DIGEST_FLAG_ENABLED:
+		if (p_drv_buf->data_digest_set) {
+			p_buf->p_val = &p_drv_buf->data_digest;
+			return sizeof(p_drv_buf->data_digest);
+		}
+		break;
+	case DRV_TLV_AUTHENTICATION_METHOD:
+		if (p_drv_buf->auth_method_set) {
+			p_buf->p_val = &p_drv_buf->auth_method;
+			return sizeof(p_drv_buf->auth_method);
+		}
+		break;
+	case DRV_TLV_ISCSI_BOOT_TARGET_PORTAL:
+		if (p_drv_buf->boot_taget_portal_set) {
+			p_buf->p_val = &p_drv_buf->boot_taget_portal;
+			return sizeof(p_drv_buf->boot_taget_portal);
+		}
+		break;
+	case DRV_TLV_MAX_FRAME_SIZE:
+		if (p_drv_buf->frame_size_set) {
+			p_buf->p_val = &p_drv_buf->frame_size;
+			return sizeof(p_drv_buf->frame_size);
+		}
+		break;
+	case DRV_TLV_PDU_TX_DESCRIPTORS_QUEUE_SIZE:
+		if (p_drv_buf->tx_desc_size_set) {
+			p_buf->p_val = &p_drv_buf->tx_desc_size;
+			return sizeof(p_drv_buf->tx_desc_size);
+		}
+		break;
+	case DRV_TLV_PDU_RX_DESCRIPTORS_QUEUE_SIZE:
+		if (p_drv_buf->rx_desc_size_set) {
+			p_buf->p_val = &p_drv_buf->rx_desc_size;
+			return sizeof(p_drv_buf->rx_desc_size);
+		}
+		break;
+	case DRV_TLV_ISCSI_BOOT_PROGRESS:
+		if (p_drv_buf->boot_progress_set) {
+			p_buf->p_val = &p_drv_buf->boot_progress;
+			return sizeof(p_drv_buf->boot_progress);
+		}
+		break;
+	case DRV_TLV_PDU_TX_DESCRIPTOR_QUEUE_AVG_DEPTH:
+		if (p_drv_buf->tx_desc_qdepth_set) {
+			p_buf->p_val = &p_drv_buf->tx_desc_qdepth;
+			return sizeof(p_drv_buf->tx_desc_qdepth);
+		}
+		break;
+	case DRV_TLV_PDU_RX_DESCRIPTORS_QUEUE_AVG_DEPTH:
+		if (p_drv_buf->rx_desc_qdepth_set) {
+			p_buf->p_val = &p_drv_buf->rx_desc_qdepth;
+			return sizeof(p_drv_buf->rx_desc_qdepth);
+		}
+		break;
+	case DRV_TLV_ISCSI_PDU_RX_FRAMES_RECEIVED:
+		if (p_drv_buf->rx_frames_set) {
+			p_buf->p_val = &p_drv_buf->rx_frames;
+			return sizeof(p_drv_buf->rx_frames);
+		}
+		break;
+	case DRV_TLV_ISCSI_PDU_RX_BYTES_RECEIVED:
+		if (p_drv_buf->rx_bytes_set) {
+			p_buf->p_val = &p_drv_buf->rx_bytes;
+			return sizeof(p_drv_buf->rx_bytes);
+		}
+		break;
+	case DRV_TLV_ISCSI_PDU_TX_FRAMES_SENT:
+		if (p_drv_buf->tx_frames_set) {
+			p_buf->p_val = &p_drv_buf->tx_frames;
+			return sizeof(p_drv_buf->tx_frames);
+		}
+		break;
+	case DRV_TLV_ISCSI_PDU_TX_BYTES_SENT:
+		if (p_drv_buf->tx_bytes_set) {
+			p_buf->p_val = &p_drv_buf->tx_bytes;
+			return sizeof(p_drv_buf->tx_bytes);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -1;
+}
+
+static int qed_mfw_update_tlvs(struct qed_hwfn *p_hwfn,
+			       u8 tlv_group, u8 *p_mfw_buf, u32 size)
+{
+	union qed_mfw_tlv_data *p_tlv_data;
+	struct qed_tlv_parsed_buf buffer;
+	struct qed_drv_tlv_hdr tlv;
+	int len = 0;
+	u32 offset;
+	u8 *p_tlv;
+
+	p_tlv_data = vzalloc(sizeof(*p_tlv_data));
+	if (!p_tlv_data)
+		return -ENOMEM;
+
+	if (qed_mfw_fill_tlv_data(p_hwfn, tlv_group, p_tlv_data)) {
+		vfree(p_tlv_data);
+		return -EINVAL;
+	}
+
+	memset(&tlv, 0, sizeof(tlv));
+	for (offset = 0; offset < size;
+	     offset += sizeof(tlv) + sizeof(u32) * tlv.tlv_length) {
+		p_tlv = &p_mfw_buf[offset];
+		tlv.tlv_type = TLV_TYPE(p_tlv);
+		tlv.tlv_length = TLV_LENGTH(p_tlv);
+		tlv.tlv_flags = TLV_FLAGS(p_tlv);
+
+		DP_VERBOSE(p_hwfn, QED_MSG_SP,
+			   "Type %d length = %d flags = 0x%x\n", tlv.tlv_type,
+			   tlv.tlv_length, tlv.tlv_flags);
+
+		if (tlv_group == QED_MFW_TLV_GENERIC)
+			len = qed_mfw_get_gen_tlv_value(&tlv,
+							&p_tlv_data->generic,
+							&buffer);
+		else if (tlv_group == QED_MFW_TLV_ETH)
+			len = qed_mfw_get_eth_tlv_value(&tlv,
+							&p_tlv_data->eth,
+							&buffer);
+		else if (tlv_group == QED_MFW_TLV_FCOE)
+			len = qed_mfw_get_fcoe_tlv_value(&tlv,
+							 &p_tlv_data->fcoe,
+							 &buffer);
+		else
+			len = qed_mfw_get_iscsi_tlv_value(&tlv,
+							  &p_tlv_data->iscsi,
+							  &buffer);
+
+		if (len > 0) {
+			WARN(len > 4 * tlv.tlv_length,
+			     "Incorrect MFW TLV length %d, it shouldn't be greater than %d\n",
+			     len, 4 * tlv.tlv_length);
+			len = min_t(int, len, 4 * tlv.tlv_length);
+			tlv.tlv_flags |= QED_DRV_TLV_FLAGS_CHANGED;
+			TLV_FLAGS(p_tlv) = tlv.tlv_flags;
+			memcpy(p_mfw_buf + offset + sizeof(tlv),
+			       buffer.p_val, len);
+		}
+	}
+
+	vfree(p_tlv_data);
+
+	return 0;
+}
+
+int qed_mfw_process_tlv_req(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	u32 addr, size, offset, resp, param, val, global_offsize, global_addr;
+	u8 tlv_group = 0, id, *p_mfw_buf = NULL, *p_temp;
+	struct qed_drv_tlv_hdr tlv;
+	int rc;
+
+	addr = SECTION_OFFSIZE_ADDR(p_hwfn->mcp_info->public_base,
+				    PUBLIC_GLOBAL);
+	global_offsize = qed_rd(p_hwfn, p_ptt, addr);
+	global_addr = SECTION_ADDR(global_offsize, 0);
+	addr = global_addr + offsetof(struct public_global, data_ptr);
+	addr = qed_rd(p_hwfn, p_ptt, addr);
+	size = qed_rd(p_hwfn, p_ptt, global_addr +
+		      offsetof(struct public_global, data_size));
+
+	if (!size) {
+		DP_NOTICE(p_hwfn, "Invalid TLV req size = %d\n", size);
+		goto drv_done;
+	}
+
+	p_mfw_buf = vzalloc(size);
+	if (!p_mfw_buf) {
+		DP_NOTICE(p_hwfn, "Failed allocate memory for p_mfw_buf\n");
+		goto drv_done;
+	}
+
+	/* Read the TLV request to local buffer. MFW represents the TLV in
+	 * little endian format and mcp returns it bigendian format. Hence
+	 * driver need to convert data to little endian first and then do the
+	 * memcpy (casting) to preserve the MFW TLV format in the driver buffer.
+	 *
+	 */
+	for (offset = 0; offset < size; offset += sizeof(u32)) {
+		val = qed_rd(p_hwfn, p_ptt, addr + offset);
+		val = be32_to_cpu(val);
+		memcpy(&p_mfw_buf[offset], &val, sizeof(u32));
+	}
+
+	/* Parse the headers to enumerate the requested TLV groups */
+	for (offset = 0; offset < size;
+	     offset += sizeof(tlv) + sizeof(u32) * tlv.tlv_length) {
+		p_temp = &p_mfw_buf[offset];
+		tlv.tlv_type = TLV_TYPE(p_temp);
+		tlv.tlv_length = TLV_LENGTH(p_temp);
+		if (qed_mfw_get_tlv_group(tlv.tlv_type, &tlv_group))
+			DP_VERBOSE(p_hwfn, NETIF_MSG_DRV,
+				   "Un recognized TLV %d\n", tlv.tlv_type);
+	}
+
+	/* Sanitize the TLV groups according to personality */
+	if ((tlv_group & QED_MFW_TLV_ETH) && !QED_IS_L2_PERSONALITY(p_hwfn)) {
+		DP_VERBOSE(p_hwfn, QED_MSG_SP,
+			   "Skipping L2 TLVs for non-L2 function\n");
+		tlv_group &= ~QED_MFW_TLV_ETH;
+	}
+
+	if ((tlv_group & QED_MFW_TLV_FCOE) &&
+	    p_hwfn->hw_info.personality != QED_PCI_FCOE) {
+		DP_VERBOSE(p_hwfn, QED_MSG_SP,
+			   "Skipping FCoE TLVs for non-FCoE function\n");
+		tlv_group &= ~QED_MFW_TLV_FCOE;
+	}
+
+	if ((tlv_group & QED_MFW_TLV_ISCSI) &&
+	    p_hwfn->hw_info.personality != QED_PCI_ISCSI) {
+		DP_VERBOSE(p_hwfn, QED_MSG_SP,
+			   "Skipping iSCSI TLVs for non-iSCSI function\n");
+		tlv_group &= ~QED_MFW_TLV_ISCSI;
+	}
+
+	/* Update the TLV values in the local buffer */
+	for (id = QED_MFW_TLV_GENERIC; id < QED_MFW_TLV_MAX; id <<= 1) {
+		if (tlv_group & id)
+			if (qed_mfw_update_tlvs(p_hwfn, id, p_mfw_buf, size))
+				goto drv_done;
+	}
+
+	/* Write the TLV data to shared memory. The stream of 4 bytes first need
+	 * to be mem-copied to u32 element to make it as LSB format. And then
+	 * converted to big endian as required by mcp-write.
+	 */
+	for (offset = 0; offset < size; offset += sizeof(u32)) {
+		memcpy(&val, &p_mfw_buf[offset], sizeof(u32));
+		val = cpu_to_be32(val);
+		qed_wr(p_hwfn, p_ptt, addr + offset, val);
+	}
+
+drv_done:
+	rc = qed_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_GET_TLV_DONE, 0, &resp,
+			 &param);
+
+	vfree(p_mfw_buf);
+
+	return rc;
+}
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp.h b/drivers/net/ethernet/qlogic/qed/qed_sp.h
index ab4ad8a1e2a5..e95431f6acd4 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sp.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_sp.h
@@ -416,7 +416,6 @@ int qed_sp_init_request(struct qed_hwfn *p_hwfn,
  * @param p_hwfn
  * @param p_ptt
  * @param p_tunn
- * @param mode
  * @param allow_npar_tx_switch
  *
  * @return int
@@ -425,7 +424,7 @@ int qed_sp_init_request(struct qed_hwfn *p_hwfn,
 int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
 		    struct qed_ptt *p_ptt,
 		    struct qed_tunnel_info *p_tunn,
-		    enum qed_mf_mode mode, bool allow_npar_tx_switch);
+		    bool allow_npar_tx_switch);
 
 /**
  * @brief qed_sp_pf_update - PF Function Update Ramrod
@@ -463,6 +462,15 @@ int qed_sp_pf_update_stag(struct qed_hwfn *p_hwfn);
  * @return int
  */
 
+/**
+ * @brief qed_sp_pf_update_ufp - PF ufp update Ramrod
+ *
+ * @param p_hwfn
+ *
+ * @return int
+ */
+int qed_sp_pf_update_ufp(struct qed_hwfn *p_hwfn);
+
 int qed_sp_pf_stop(struct qed_hwfn *p_hwfn);
 
 int qed_sp_pf_update_tunn_cfg(struct qed_hwfn *p_hwfn,
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
index 5e927b6cac22..8de644b4721e 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
@@ -306,7 +306,7 @@ qed_tunn_set_pf_start_params(struct qed_hwfn *p_hwfn,
 int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
 		    struct qed_ptt *p_ptt,
 		    struct qed_tunnel_info *p_tunn,
-		    enum qed_mf_mode mode, bool allow_npar_tx_switch)
+		    bool allow_npar_tx_switch)
 {
 	struct pf_start_ramrod_data *p_ramrod = NULL;
 	u16 sb = qed_int_get_sp_sb_id(p_hwfn);
@@ -314,7 +314,7 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
 	struct qed_spq_entry *p_ent = NULL;
 	struct qed_sp_init_data init_data;
 	int rc = -EINVAL;
-	u8 page_cnt;
+	u8 page_cnt, i;
 
 	/* update initial eq producer */
 	qed_eq_prod_update(p_hwfn,
@@ -339,21 +339,36 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
 	p_ramrod->dont_log_ramrods	= 0;
 	p_ramrod->log_type_mask		= cpu_to_le16(0xf);
 
-	switch (mode) {
-	case QED_MF_DEFAULT:
-	case QED_MF_NPAR:
-		p_ramrod->mf_mode = MF_NPAR;
-		break;
-	case QED_MF_OVLAN:
+	if (test_bit(QED_MF_OVLAN_CLSS, &p_hwfn->cdev->mf_bits))
 		p_ramrod->mf_mode = MF_OVLAN;
-		break;
-	default:
-		DP_NOTICE(p_hwfn, "Unsupported MF mode, init as DEFAULT\n");
+	else
 		p_ramrod->mf_mode = MF_NPAR;
-	}
 
 	p_ramrod->outer_tag_config.outer_tag.tci =
-		cpu_to_le16(p_hwfn->hw_info.ovlan);
+				cpu_to_le16(p_hwfn->hw_info.ovlan);
+	if (test_bit(QED_MF_8021Q_TAGGING, &p_hwfn->cdev->mf_bits)) {
+		p_ramrod->outer_tag_config.outer_tag.tpid = ETH_P_8021Q;
+	} else if (test_bit(QED_MF_8021AD_TAGGING, &p_hwfn->cdev->mf_bits)) {
+		p_ramrod->outer_tag_config.outer_tag.tpid = ETH_P_8021AD;
+		p_ramrod->outer_tag_config.enable_stag_pri_change = 1;
+	}
+
+	p_ramrod->outer_tag_config.pri_map_valid = 1;
+	for (i = 0; i < QED_MAX_PFC_PRIORITIES; i++)
+		p_ramrod->outer_tag_config.inner_to_outer_pri_map[i] = i;
+
+	/* enable_stag_pri_change should be set if port is in BD mode or,
+	 * UFP with Host Control mode.
+	 */
+	if (test_bit(QED_MF_UFP_SPECIFIC, &p_hwfn->cdev->mf_bits)) {
+		if (p_hwfn->ufp_info.pri_type == QED_UFP_PRI_OS)
+			p_ramrod->outer_tag_config.enable_stag_pri_change = 1;
+		else
+			p_ramrod->outer_tag_config.enable_stag_pri_change = 0;
+
+		p_ramrod->outer_tag_config.outer_tag.tci |=
+		    cpu_to_le16(((u16)p_hwfn->ufp_info.tc << 13));
+	}
 
 	/* Place EQ address in RAMROD */
 	DMA_REGPAIR_LE(p_ramrod->event_ring_pbl_addr,
@@ -365,7 +380,7 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
 
 	qed_tunn_set_pf_start_params(p_hwfn, p_tunn, &p_ramrod->tunnel_config);
 
-	if (IS_MF_SI(p_hwfn))
+	if (test_bit(QED_MF_INTER_PF_SWITCH, &p_hwfn->cdev->mf_bits))
 		p_ramrod->allow_npar_tx_switching = allow_npar_tx_switch;
 
 	switch (p_hwfn->hw_info.personality) {
@@ -434,6 +449,39 @@ int qed_sp_pf_update(struct qed_hwfn *p_hwfn)
 	return qed_spq_post(p_hwfn, p_ent, NULL);
 }
 
+int qed_sp_pf_update_ufp(struct qed_hwfn *p_hwfn)
+{
+	struct qed_spq_entry *p_ent = NULL;
+	struct qed_sp_init_data init_data;
+	int rc = -EOPNOTSUPP;
+
+	if (p_hwfn->ufp_info.pri_type == QED_UFP_PRI_UNKNOWN) {
+		DP_INFO(p_hwfn, "Invalid priority type %d\n",
+			p_hwfn->ufp_info.pri_type);
+		return -EINVAL;
+	}
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = qed_spq_get_cid(p_hwfn);
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_CB;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 COMMON_RAMROD_PF_UPDATE, PROTOCOLID_COMMON,
+				 &init_data);
+	if (rc)
+		return rc;
+
+	p_ent->ramrod.pf_update.update_enable_stag_pri_change = true;
+	if (p_hwfn->ufp_info.pri_type == QED_UFP_PRI_OS)
+		p_ent->ramrod.pf_update.enable_stag_pri_change = 1;
+	else
+		p_ent->ramrod.pf_update.enable_stag_pri_change = 0;
+
+	return qed_spq_post(p_hwfn, p_ent, NULL);
+}
+
 /* Set pf update ramrod command params */
 int qed_sp_pf_update_tunn_cfg(struct qed_hwfn *p_hwfn,
 			      struct qed_ptt *p_ptt,
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.c b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
index 5acb91b3564c..f01bf52bc381 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
@@ -48,7 +48,7 @@ static int qed_sriov_eqe_event(struct qed_hwfn *p_hwfn,
 			       u8 opcode,
 			       __le16 echo,
 			       union event_ring_data *data, u8 fw_return_code);
-
+static int qed_iov_bulletin_set_mac(struct qed_hwfn *p_hwfn, u8 *mac, int vfid);
 
 static u8 qed_vf_calculate_legacy(struct qed_vf_info *p_vf)
 {
@@ -1790,7 +1790,8 @@ static int qed_iov_configure_vport_forced(struct qed_hwfn *p_hwfn,
 	if (!p_vf->vport_instance)
 		return -EINVAL;
 
-	if (events & BIT(MAC_ADDR_FORCED)) {
+	if ((events & BIT(MAC_ADDR_FORCED)) ||
+	    p_vf->p_vf_info.is_trusted_configured) {
 		/* Since there's no way [currently] of removing the MAC,
 		 * we can always assume this means we need to force it.
 		 */
@@ -1809,8 +1810,12 @@ static int qed_iov_configure_vport_forced(struct qed_hwfn *p_hwfn,
 				  "PF failed to configure MAC for VF\n");
 			return rc;
 		}
-
-		p_vf->configured_features |= 1 << MAC_ADDR_FORCED;
+		if (p_vf->p_vf_info.is_trusted_configured)
+			p_vf->configured_features |=
+				BIT(VFPF_BULLETIN_MAC_ADDR);
+		else
+			p_vf->configured_features |=
+				BIT(MAC_ADDR_FORCED);
 	}
 
 	if (events & BIT(VLAN_ADDR_FORCED)) {
@@ -3170,6 +3175,10 @@ static int qed_iov_vf_update_mac_shadow(struct qed_hwfn *p_hwfn,
 	if (p_vf->bulletin.p_virt->valid_bitmap & BIT(MAC_ADDR_FORCED))
 		return 0;
 
+	/* Don't keep track of shadow copy since we don't intend to restore. */
+	if (p_vf->p_vf_info.is_trusted_configured)
+		return 0;
+
 	/* First remove entries and then add new ones */
 	if (p_params->opcode == QED_FILTER_REMOVE) {
 		for (i = 0; i < QED_ETH_VF_NUM_MAC_FILTERS; i++) {
@@ -3244,9 +3253,17 @@ static int qed_iov_chk_ucast(struct qed_hwfn *hwfn,
 
 	/* No real decision to make; Store the configured MAC */
 	if (params->type == QED_FILTER_MAC ||
-	    params->type == QED_FILTER_MAC_VLAN)
+	    params->type == QED_FILTER_MAC_VLAN) {
 		ether_addr_copy(vf->mac, params->mac);
 
+		if (vf->is_trusted_configured) {
+			qed_iov_bulletin_set_mac(hwfn, vf->mac, vfid);
+
+			/* Update and post bulleitin again */
+			qed_schedule_iov(hwfn, QED_IOV_WQ_BULLETIN_UPDATE_FLAG);
+		}
+	}
+
 	return 0;
 }
 
@@ -3803,6 +3820,40 @@ static void qed_iov_get_link(struct qed_hwfn *p_hwfn,
 		__qed_vf_get_link_caps(p_hwfn, p_caps, p_bulletin);
 }
 
+static int
+qed_iov_vf_pf_bulletin_update_mac(struct qed_hwfn *p_hwfn,
+				  struct qed_ptt *p_ptt,
+				  struct qed_vf_info *p_vf)
+{
+	struct qed_bulletin_content *p_bulletin = p_vf->bulletin.p_virt;
+	struct qed_iov_vf_mbx *mbx = &p_vf->vf_mbx;
+	struct vfpf_bulletin_update_mac_tlv *p_req;
+	u8 status = PFVF_STATUS_SUCCESS;
+	int rc = 0;
+
+	if (!p_vf->p_vf_info.is_trusted_configured) {
+		DP_VERBOSE(p_hwfn,
+			   QED_MSG_IOV,
+			   "Blocking bulletin update request from untrusted VF[%d]\n",
+			   p_vf->abs_vf_id);
+		status = PFVF_STATUS_NOT_SUPPORTED;
+		rc = -EINVAL;
+		goto send_status;
+	}
+
+	p_req = &mbx->req_virt->bulletin_update_mac;
+	ether_addr_copy(p_bulletin->mac, p_req->mac);
+	DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+		   "Updated bulletin of VF[%d] with requested MAC[%pM]\n",
+		   p_vf->abs_vf_id, p_req->mac);
+
+send_status:
+	qed_iov_prepare_resp(p_hwfn, p_ptt, p_vf,
+			     CHANNEL_TLV_BULLETIN_UPDATE_MAC,
+			     sizeof(struct pfvf_def_resp_tlv), status);
+	return rc;
+}
+
 static void qed_iov_process_mbx_req(struct qed_hwfn *p_hwfn,
 				    struct qed_ptt *p_ptt, int vfid)
 {
@@ -3882,6 +3933,9 @@ static void qed_iov_process_mbx_req(struct qed_hwfn *p_hwfn,
 		case CHANNEL_TLV_COALESCE_READ:
 			qed_iov_vf_pf_get_coalesce(p_hwfn, p_ptt, p_vf);
 			break;
+		case CHANNEL_TLV_BULLETIN_UPDATE_MAC:
+			qed_iov_vf_pf_bulletin_update_mac(p_hwfn, p_ptt, p_vf);
+			break;
 		}
 	} else if (qed_iov_tlv_supported(mbx->first_tlv.tl.type)) {
 		DP_VERBOSE(p_hwfn, QED_MSG_IOV,
@@ -4081,16 +4135,60 @@ static void qed_iov_bulletin_set_forced_mac(struct qed_hwfn *p_hwfn,
 		return;
 	}
 
-	feature = 1 << MAC_ADDR_FORCED;
+	if (vf_info->p_vf_info.is_trusted_configured) {
+		feature = BIT(VFPF_BULLETIN_MAC_ADDR);
+		/* Trust mode will disable Forced MAC */
+		vf_info->bulletin.p_virt->valid_bitmap &=
+			~BIT(MAC_ADDR_FORCED);
+	} else {
+		feature = BIT(MAC_ADDR_FORCED);
+		/* Forced MAC will disable MAC_ADDR */
+		vf_info->bulletin.p_virt->valid_bitmap &=
+			~BIT(VFPF_BULLETIN_MAC_ADDR);
+	}
+
 	memcpy(vf_info->bulletin.p_virt->mac, mac, ETH_ALEN);
 
 	vf_info->bulletin.p_virt->valid_bitmap |= feature;
-	/* Forced MAC will disable MAC_ADDR */
-	vf_info->bulletin.p_virt->valid_bitmap &= ~BIT(VFPF_BULLETIN_MAC_ADDR);
 
 	qed_iov_configure_vport_forced(p_hwfn, vf_info, feature);
 }
 
+static int qed_iov_bulletin_set_mac(struct qed_hwfn *p_hwfn, u8 *mac, int vfid)
+{
+	struct qed_vf_info *vf_info;
+	u64 feature;
+
+	vf_info = qed_iov_get_vf_info(p_hwfn, (u16)vfid, true);
+	if (!vf_info) {
+		DP_NOTICE(p_hwfn->cdev, "Can not set MAC, invalid vfid [%d]\n",
+			  vfid);
+		return -EINVAL;
+	}
+
+	if (vf_info->b_malicious) {
+		DP_NOTICE(p_hwfn->cdev, "Can't set MAC to malicious VF [%d]\n",
+			  vfid);
+		return -EINVAL;
+	}
+
+	if (vf_info->bulletin.p_virt->valid_bitmap & BIT(MAC_ADDR_FORCED)) {
+		DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+			   "Can not set MAC, Forced MAC is configured\n");
+		return -EINVAL;
+	}
+
+	feature = BIT(VFPF_BULLETIN_MAC_ADDR);
+	ether_addr_copy(vf_info->bulletin.p_virt->mac, mac);
+
+	vf_info->bulletin.p_virt->valid_bitmap |= feature;
+
+	if (vf_info->p_vf_info.is_trusted_configured)
+		qed_iov_configure_vport_forced(p_hwfn, vf_info, feature);
+
+	return 0;
+}
+
 static void qed_iov_bulletin_set_forced_vlan(struct qed_hwfn *p_hwfn,
 					     u16 pvid, int vfid)
 {
@@ -4204,6 +4302,21 @@ static int qed_iov_spoofchk_set(struct qed_hwfn *p_hwfn, int vfid, bool val)
 	return rc;
 }
 
+static u8 *qed_iov_bulletin_get_mac(struct qed_hwfn *p_hwfn, u16 rel_vf_id)
+{
+	struct qed_vf_info *p_vf;
+
+	p_vf = qed_iov_get_vf_info(p_hwfn, rel_vf_id, true);
+	if (!p_vf || !p_vf->bulletin.p_virt)
+		return NULL;
+
+	if (!(p_vf->bulletin.p_virt->valid_bitmap &
+	      BIT(VFPF_BULLETIN_MAC_ADDR)))
+		return NULL;
+
+	return p_vf->bulletin.p_virt->mac;
+}
+
 static u8 *qed_iov_bulletin_get_forced_mac(struct qed_hwfn *p_hwfn,
 					   u16 rel_vf_id)
 {
@@ -4493,8 +4606,12 @@ static int qed_sriov_pf_set_mac(struct qed_dev *cdev, u8 *mac, int vfid)
 		if (!vf_info)
 			continue;
 
-		/* Set the forced MAC, and schedule the IOV task */
-		ether_addr_copy(vf_info->forced_mac, mac);
+		/* Set the MAC, and schedule the IOV task */
+		if (vf_info->is_trusted_configured)
+			ether_addr_copy(vf_info->mac, mac);
+		else
+			ether_addr_copy(vf_info->forced_mac, mac);
+
 		qed_schedule_iov(hwfn, QED_IOV_WQ_SET_UNICAST_FILTER_FLAG);
 	}
 
@@ -4802,6 +4919,33 @@ static void qed_handle_vf_msg(struct qed_hwfn *hwfn)
 	qed_ptt_release(hwfn, ptt);
 }
 
+static bool qed_pf_validate_req_vf_mac(struct qed_hwfn *hwfn,
+				       u8 *mac,
+				       struct qed_public_vf_info *info)
+{
+	if (info->is_trusted_configured) {
+		if (is_valid_ether_addr(info->mac) &&
+		    (!mac || !ether_addr_equal(mac, info->mac)))
+			return true;
+	} else {
+		if (is_valid_ether_addr(info->forced_mac) &&
+		    (!mac || !ether_addr_equal(mac, info->forced_mac)))
+			return true;
+	}
+
+	return false;
+}
+
+static void qed_set_bulletin_mac(struct qed_hwfn *hwfn,
+				 struct qed_public_vf_info *info,
+				 int vfid)
+{
+	if (info->is_trusted_configured)
+		qed_iov_bulletin_set_mac(hwfn, info->mac, vfid);
+	else
+		qed_iov_bulletin_set_forced_mac(hwfn, info->forced_mac, vfid);
+}
+
 static void qed_handle_pf_set_vf_unicast(struct qed_hwfn *hwfn)
 {
 	int i;
@@ -4816,18 +4960,20 @@ static void qed_handle_pf_set_vf_unicast(struct qed_hwfn *hwfn)
 			continue;
 
 		/* Update data on bulletin board */
-		mac = qed_iov_bulletin_get_forced_mac(hwfn, i);
-		if (is_valid_ether_addr(info->forced_mac) &&
-		    (!mac || !ether_addr_equal(mac, info->forced_mac))) {
+		if (info->is_trusted_configured)
+			mac = qed_iov_bulletin_get_mac(hwfn, i);
+		else
+			mac = qed_iov_bulletin_get_forced_mac(hwfn, i);
+
+		if (qed_pf_validate_req_vf_mac(hwfn, mac, info)) {
 			DP_VERBOSE(hwfn,
 				   QED_MSG_IOV,
 				   "Handling PF setting of VF MAC to VF 0x%02x [Abs 0x%02x]\n",
 				   i,
 				   hwfn->cdev->p_iov_info->first_vf_in_pf + i);
 
-			/* Update bulletin board with forced MAC */
-			qed_iov_bulletin_set_forced_mac(hwfn,
-							info->forced_mac, i);
+			/* Update bulletin board with MAC */
+			qed_set_bulletin_mac(hwfn, info, i);
 			update = true;
 		}
 
@@ -4867,6 +5013,72 @@ static void qed_handle_bulletin_post(struct qed_hwfn *hwfn)
 	qed_ptt_release(hwfn, ptt);
 }
 
+static void qed_update_mac_for_vf_trust_change(struct qed_hwfn *hwfn, int vf_id)
+{
+	struct qed_public_vf_info *vf_info;
+	struct qed_vf_info *vf;
+	u8 *force_mac;
+	int i;
+
+	vf_info = qed_iov_get_public_vf_info(hwfn, vf_id, true);
+	vf = qed_iov_get_vf_info(hwfn, vf_id, true);
+
+	if (!vf_info || !vf)
+		return;
+
+	/* Force MAC converted to generic MAC in case of VF trust on */
+	if (vf_info->is_trusted_configured &&
+	    (vf->bulletin.p_virt->valid_bitmap & BIT(MAC_ADDR_FORCED))) {
+		force_mac = qed_iov_bulletin_get_forced_mac(hwfn, vf_id);
+
+		if (force_mac) {
+			/* Clear existing shadow copy of MAC to have a clean
+			 * slate.
+			 */
+			for (i = 0; i < QED_ETH_VF_NUM_MAC_FILTERS; i++) {
+				if (ether_addr_equal(vf->shadow_config.macs[i],
+						     vf_info->mac)) {
+					memset(vf->shadow_config.macs[i], 0,
+					       ETH_ALEN);
+					DP_VERBOSE(hwfn, QED_MSG_IOV,
+						   "Shadow MAC %pM removed for VF 0x%02x, VF trust mode is ON\n",
+						    vf_info->mac, vf_id);
+					break;
+				}
+			}
+
+			ether_addr_copy(vf_info->mac, force_mac);
+			memset(vf_info->forced_mac, 0, ETH_ALEN);
+			vf->bulletin.p_virt->valid_bitmap &=
+					~BIT(MAC_ADDR_FORCED);
+			qed_schedule_iov(hwfn, QED_IOV_WQ_BULLETIN_UPDATE_FLAG);
+		}
+	}
+
+	/* Update shadow copy with VF MAC when trust mode is turned off */
+	if (!vf_info->is_trusted_configured) {
+		u8 empty_mac[ETH_ALEN];
+
+		memset(empty_mac, 0, ETH_ALEN);
+		for (i = 0; i < QED_ETH_VF_NUM_MAC_FILTERS; i++) {
+			if (ether_addr_equal(vf->shadow_config.macs[i],
+					     empty_mac)) {
+				ether_addr_copy(vf->shadow_config.macs[i],
+						vf_info->mac);
+				DP_VERBOSE(hwfn, QED_MSG_IOV,
+					   "Shadow is updated with %pM for VF 0x%02x, VF trust mode is OFF\n",
+					    vf_info->mac, vf_id);
+				break;
+			}
+		}
+		/* Clear bulletin when trust mode is turned off,
+		 * to have a clean slate for next (normal) operations.
+		 */
+		qed_iov_bulletin_set_mac(hwfn, empty_mac, vf_id);
+		qed_schedule_iov(hwfn, QED_IOV_WQ_BULLETIN_UPDATE_FLAG);
+	}
+}
+
 static void qed_iov_handle_trust_change(struct qed_hwfn *hwfn)
 {
 	struct qed_sp_vport_update_params params;
@@ -4890,6 +5102,9 @@ static void qed_iov_handle_trust_change(struct qed_hwfn *hwfn)
 			continue;
 		vf_info->is_trusted_configured = vf_info->is_trusted_request;
 
+		/* Handle forced MAC mode */
+		qed_update_mac_for_vf_trust_change(hwfn, i);
+
 		/* Validate that the VF has a configured vport */
 		vf = qed_iov_get_vf_info(hwfn, i, true);
 		if (!vf->vport_instance)
diff --git a/drivers/net/ethernet/qlogic/qed/qed_vf.c b/drivers/net/ethernet/qlogic/qed/qed_vf.c
index 91b5e9f02a62..2d7fcd6a0777 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_vf.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_vf.c
@@ -1374,6 +1374,35 @@ int qed_vf_pf_get_coalesce(struct qed_hwfn *p_hwfn,
 	return rc;
 }
 
+int
+qed_vf_pf_bulletin_update_mac(struct qed_hwfn *p_hwfn,
+			      u8 *p_mac)
+{
+	struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
+	struct vfpf_bulletin_update_mac_tlv *p_req;
+	struct pfvf_def_resp_tlv *p_resp;
+	int rc;
+
+	if (!p_mac)
+		return -EINVAL;
+
+	/* clear mailbox and prep header tlv */
+	p_req = qed_vf_pf_prep(p_hwfn, CHANNEL_TLV_BULLETIN_UPDATE_MAC,
+			       sizeof(*p_req));
+	ether_addr_copy(p_req->mac, p_mac);
+	DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+		   "Requesting bulletin update for MAC[%pM]\n", p_mac);
+
+	/* add list termination tlv */
+	qed_add_tlv(p_hwfn, &p_iov->offset, CHANNEL_TLV_LIST_END,
+		    sizeof(struct channel_list_end_tlv));
+
+	p_resp = &p_iov->pf2vf_reply->default_resp;
+	rc = qed_send_msg2pf(p_hwfn, &p_resp->hdr.status, sizeof(*p_resp));
+	qed_vf_pf_req_end(p_hwfn, rc);
+	return rc;
+}
+
 int
 qed_vf_pf_set_coalesce(struct qed_hwfn *p_hwfn,
 		       u16 rx_coal, u16 tx_coal, struct qed_queue_cid *p_cid)
diff --git a/drivers/net/ethernet/qlogic/qed/qed_vf.h b/drivers/net/ethernet/qlogic/qed/qed_vf.h
index 97d44dfb38ca..4f05d5eb3cf5 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_vf.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_vf.h
@@ -518,6 +518,12 @@ struct pfvf_read_coal_resp_tlv {
 	u8 padding[6];
 };
 
+struct vfpf_bulletin_update_mac_tlv {
+	struct vfpf_first_tlv first_tlv;
+	u8 mac[ETH_ALEN];
+	u8 padding[2];
+};
+
 union vfpf_tlvs {
 	struct vfpf_first_tlv first_tlv;
 	struct vfpf_acquire_tlv acquire;
@@ -532,6 +538,7 @@ union vfpf_tlvs {
 	struct vfpf_update_tunn_param_tlv tunn_param_update;
 	struct vfpf_update_coalesce update_coalesce;
 	struct vfpf_read_coal_req_tlv read_coal_req;
+	struct vfpf_bulletin_update_mac_tlv bulletin_update_mac;
 	struct tlv_buffer_size tlv_buf_size;
 };
 
@@ -650,6 +657,7 @@ enum {
 	CHANNEL_TLV_COALESCE_UPDATE,
 	CHANNEL_TLV_QID,
 	CHANNEL_TLV_COALESCE_READ,
+	CHANNEL_TLV_BULLETIN_UPDATE_MAC,
 	CHANNEL_TLV_MAX,
 
 	/* Required for iterating over vport-update tlvs.
@@ -1042,6 +1050,13 @@ int qed_vf_pf_tunnel_param_update(struct qed_hwfn *p_hwfn,
 				  struct qed_tunnel_info *p_tunn);
 
 u32 qed_vf_hw_bar_size(struct qed_hwfn *p_hwfn, enum BAR_ID bar_id);
+/**
+ * @brief - Ask PF to update the MAC address in it's bulletin board
+ *
+ * @param p_mac - mac address to be updated in bulletin board
+ */
+int qed_vf_pf_bulletin_update_mac(struct qed_hwfn *p_hwfn, u8 *p_mac);
+
 #else
 static inline void qed_vf_get_link_params(struct qed_hwfn *p_hwfn,
 					  struct qed_mcp_link_params *params)
@@ -1228,6 +1243,12 @@ static inline int qed_vf_pf_tunnel_param_update(struct qed_hwfn *p_hwfn,
 	return -EINVAL;
 }
 
+static inline int qed_vf_pf_bulletin_update_mac(struct qed_hwfn *p_hwfn,
+						u8 *p_mac)
+{
+	return -EINVAL;
+}
+
 static inline u32
 qed_vf_hw_bar_size(struct qed_hwfn  *p_hwfn,
 		   enum BAR_ID bar_id)
diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h
index 9935978c5542..81c5c8dfa2ef 100644
--- a/drivers/net/ethernet/qlogic/qede/qede.h
+++ b/drivers/net/ethernet/qlogic/qede/qede.h
@@ -75,6 +75,7 @@ struct qede_stats_common {
 	u64 rx_bcast_pkts;
 	u64 mftag_filter_discards;
 	u64 mac_filter_discards;
+	u64 gft_filter_drop;
 	u64 tx_ucast_bytes;
 	u64 tx_mcast_bytes;
 	u64 tx_bcast_bytes;
@@ -290,15 +291,12 @@ struct qede_agg_info {
 	 * aggregation.
 	 */
 	struct sw_rx_data buffer;
-	dma_addr_t buffer_mapping;
-
 	struct sk_buff *skb;
 
 	/* We need some structs from the start cookie until termination */
 	u16 vlan_tag;
-	u16 start_cqe_bd_len;
-	u8 start_cqe_placement_offset;
 
+	bool tpa_start_fail;
 	u8 state;
 	u8 frag_id;
 
diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
index ecbf1ded7a39..6906e04b609e 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
@@ -161,6 +161,7 @@ static const struct {
 	QEDE_STAT(no_buff_discards),
 	QEDE_PF_STAT(mftag_filter_discards),
 	QEDE_PF_STAT(mac_filter_discards),
+	QEDE_PF_STAT(gft_filter_drop),
 	QEDE_STAT(tx_err_drop_pkts),
 	QEDE_STAT(ttl0_discard),
 	QEDE_STAT(packet_too_big_discard),
@@ -1508,7 +1509,8 @@ static int qede_selftest_receive_traffic(struct qede_dev *edev)
 		len =  le16_to_cpu(fp_cqe->len_on_first_bd);
 		data_ptr = (u8 *)(page_address(sw_rx_data->data) +
 				  fp_cqe->placement_offset +
-				  sw_rx_data->page_offset);
+				  sw_rx_data->page_offset +
+				  rxq->rx_headroom);
 		if (ether_addr_equal(data_ptr,  edev->ndev->dev_addr) &&
 		    ether_addr_equal(data_ptr + ETH_ALEN,
 				     edev->ndev->dev_addr)) {
diff --git a/drivers/net/ethernet/qlogic/qede/qede_filter.c b/drivers/net/ethernet/qlogic/qede/qede_filter.c
index 6687e04d1558..e9e088d9c815 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_filter.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_filter.c
@@ -38,6 +38,7 @@
 #include <linux/qed/qed_if.h>
 #include "qede.h"
 
+#define QEDE_FILTER_PRINT_MAX_LEN	(64)
 struct qede_arfs_tuple {
 	union {
 		__be32 src_ipv4;
@@ -51,6 +52,18 @@ struct qede_arfs_tuple {
 	__be16  dst_port;
 	__be16  eth_proto;
 	u8      ip_proto;
+
+	/* Describe filtering mode needed for this kind of filter */
+	enum qed_filter_config_mode mode;
+
+	/* Used to compare new/old filters. Return true if IPs match */
+	bool (*ip_comp)(struct qede_arfs_tuple *a, struct qede_arfs_tuple *b);
+
+	/* Given an address into ethhdr build a header from tuple info */
+	void (*build_hdr)(struct qede_arfs_tuple *t, void *header);
+
+	/* Stringify the tuple for a print into the provided buffer */
+	void (*stringify)(struct qede_arfs_tuple *t, void *buffer);
 };
 
 struct qede_arfs_fltr_node {
@@ -73,9 +86,11 @@ struct qede_arfs_fltr_node {
 	u16 sw_id;
 	u16 rxq_id;
 	u16 next_rxq_id;
+	u8 vfid;
 	bool filter_op;
 	bool used;
 	u8 fw_rc;
+	bool b_is_drop;
 	struct hlist_node node;
 };
 
@@ -90,7 +105,9 @@ struct qede_arfs {
 	spinlock_t		arfs_list_lock;
 	unsigned long		*arfs_fltr_bmap;
 	int			filter_count;
-	bool			enable;
+
+	/* Currently configured filtering mode */
+	enum qed_filter_config_mode mode;
 };
 
 static void qede_configure_arfs_fltr(struct qede_dev *edev,
@@ -109,12 +126,22 @@ static void qede_configure_arfs_fltr(struct qede_dev *edev,
 	params.length = n->buf_len;
 	params.qid = rxq_id;
 	params.b_is_add = add_fltr;
+	params.b_is_drop = n->b_is_drop;
+
+	if (n->vfid) {
+		params.b_is_vf = true;
+		params.vf_id = n->vfid - 1;
+	}
 
-	DP_VERBOSE(edev, NETIF_MSG_RX_STATUS,
-		   "%s arfs filter flow_id=%d, sw_id=%d, src_port=%d, dst_port=%d, rxq=%d\n",
-		   add_fltr ? "Adding" : "Deleting",
-		   n->flow_id, n->sw_id, ntohs(n->tuple.src_port),
-		   ntohs(n->tuple.dst_port), rxq_id);
+	if (n->tuple.stringify) {
+		char tuple_buffer[QEDE_FILTER_PRINT_MAX_LEN];
+
+		n->tuple.stringify(&n->tuple, tuple_buffer);
+		DP_VERBOSE(edev, NETIF_MSG_RX_STATUS,
+			   "%s sw_id[0x%x]: %s [vf %u queue %d]\n",
+			   add_fltr ? "Adding" : "Deleting",
+			   n->sw_id, tuple_buffer, n->vfid, rxq_id);
+	}
 
 	n->used = true;
 	n->filter_op = add_fltr;
@@ -145,14 +172,13 @@ qede_enqueue_fltr_and_config_searcher(struct qede_dev *edev,
 	INIT_HLIST_NODE(&fltr->node);
 	hlist_add_head(&fltr->node,
 		       QEDE_ARFS_BUCKET_HEAD(edev, bucket_idx));
-	edev->arfs->filter_count++;
-
-	if (edev->arfs->filter_count == 1 && !edev->arfs->enable) {
-		enum qed_filter_config_mode mode;
 
-		mode = QED_FILTER_CONFIG_MODE_5_TUPLE;
-		edev->ops->configure_arfs_searcher(edev->cdev, mode);
-		edev->arfs->enable = true;
+	edev->arfs->filter_count++;
+	if (edev->arfs->filter_count == 1 &&
+	    edev->arfs->mode == QED_FILTER_CONFIG_MODE_DISABLE) {
+		edev->ops->configure_arfs_searcher(edev->cdev,
+						   fltr->tuple.mode);
+		edev->arfs->mode = fltr->tuple.mode;
 	}
 
 	return 0;
@@ -167,14 +193,15 @@ qede_dequeue_fltr_and_config_searcher(struct qede_dev *edev,
 			 fltr->buf_len, DMA_TO_DEVICE);
 
 	qede_free_arfs_filter(edev, fltr);
-	edev->arfs->filter_count--;
 
-	if (!edev->arfs->filter_count && edev->arfs->enable) {
+	edev->arfs->filter_count--;
+	if (!edev->arfs->filter_count &&
+	    edev->arfs->mode != QED_FILTER_CONFIG_MODE_DISABLE) {
 		enum qed_filter_config_mode mode;
 
 		mode = QED_FILTER_CONFIG_MODE_DISABLE;
-		edev->arfs->enable = false;
 		edev->ops->configure_arfs_searcher(edev->cdev, mode);
+		edev->arfs->mode = QED_FILTER_CONFIG_MODE_DISABLE;
 	}
 }
 
@@ -264,25 +291,17 @@ void qede_process_arfs_filters(struct qede_dev *edev, bool free_fltr)
 		}
 	}
 
+#ifdef CONFIG_RFS_ACCEL
 	spin_lock_bh(&edev->arfs->arfs_list_lock);
 
-	if (!edev->arfs->filter_count) {
-		if (edev->arfs->enable) {
-			enum qed_filter_config_mode mode;
-
-			mode = QED_FILTER_CONFIG_MODE_DISABLE;
-			edev->arfs->enable = false;
-			edev->ops->configure_arfs_searcher(edev->cdev, mode);
-		}
-#ifdef CONFIG_RFS_ACCEL
-	} else {
+	if (edev->arfs->filter_count) {
 		set_bit(QEDE_SP_ARFS_CONFIG, &edev->sp_flags);
 		schedule_delayed_work(&edev->sp_task,
 				      QEDE_SP_TASK_POLL_DELAY);
-#endif
 	}
 
 	spin_unlock_bh(&edev->arfs->arfs_list_lock);
+#endif
 }
 
 /* This function waits until all aRFS filters get deleted and freed.
@@ -512,6 +531,7 @@ int qede_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
 	eth->h_proto = skb->protocol;
 	n->tuple.eth_proto = skb->protocol;
 	n->tuple.ip_proto = ip_proto;
+	n->tuple.mode = QED_FILTER_CONFIG_MODE_5_TUPLE;
 	memcpy(n->data + ETH_HLEN, skb->data, skb_headlen(skb));
 
 	rc = qede_enqueue_fltr_and_config_searcher(edev, n, tbl_idx);
@@ -550,8 +570,7 @@ void qede_force_mac(void *dev, u8 *mac, bool forced)
 
 	__qede_lock(edev);
 
-	/* MAC hints take effect only if we haven't set one already */
-	if (is_valid_ether_addr(edev->ndev->dev_addr) && !forced) {
+	if (!is_valid_ether_addr(mac)) {
 		__qede_unlock(edev);
 		return;
 	}
@@ -1161,6 +1180,10 @@ int qede_set_mac_addr(struct net_device *ndev, void *p)
 	if (edev->state != QEDE_STATE_OPEN) {
 		DP_VERBOSE(edev, NETIF_MSG_IFDOWN,
 			   "The device is currently down\n");
+		/* Ask PF to explicitly update a copy in bulletin board */
+		if (IS_VF(edev) && edev->ops->req_bulletin_update_mac)
+			edev->ops->req_bulletin_update_mac(edev->cdev,
+							   ndev->dev_addr);
 		goto out;
 	}
 
@@ -1336,38 +1359,6 @@ qede_get_arfs_fltr_by_loc(struct hlist_head *head, u32 location)
 	return NULL;
 }
 
-static bool
-qede_compare_user_flow_ips(struct qede_arfs_fltr_node *tpos,
-			   struct ethtool_rx_flow_spec *fsp,
-			   __be16 proto)
-{
-	if (proto == htons(ETH_P_IP)) {
-		struct ethtool_tcpip4_spec *ip;
-
-		ip = &fsp->h_u.tcp_ip4_spec;
-
-		if (tpos->tuple.src_ipv4 == ip->ip4src &&
-		    tpos->tuple.dst_ipv4 == ip->ip4dst)
-			return true;
-		else
-			return false;
-	} else {
-		struct ethtool_tcpip6_spec *ip6;
-		struct in6_addr *src;
-
-		ip6 = &fsp->h_u.tcp_ip6_spec;
-		src = &tpos->tuple.src_ipv6;
-
-		if (!memcmp(src, &ip6->ip6src, sizeof(struct in6_addr)) &&
-		    !memcmp(&tpos->tuple.dst_ipv6, &ip6->ip6dst,
-			    sizeof(struct in6_addr)))
-			return true;
-		else
-			return false;
-	}
-	return false;
-}
-
 int qede_get_cls_rule_all(struct qede_dev *edev, struct ethtool_rxnfc *info,
 			  u32 *rule_locs)
 {
@@ -1452,102 +1443,444 @@ int qede_get_cls_rule_entry(struct qede_dev *edev, struct ethtool_rxnfc *cmd)
 
 	fsp->ring_cookie = fltr->rxq_id;
 
+	if (fltr->vfid) {
+		fsp->ring_cookie |= ((u64)fltr->vfid) <<
+					ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF;
+	}
+
+	if (fltr->b_is_drop)
+		fsp->ring_cookie = RX_CLS_FLOW_DISC;
 unlock:
 	__qede_unlock(edev);
 	return rc;
 }
 
 static int
-qede_validate_and_check_flow_exist(struct qede_dev *edev,
-				   struct ethtool_rx_flow_spec *fsp,
-				   int *min_hlen)
+qede_poll_arfs_filter_config(struct qede_dev *edev,
+			     struct qede_arfs_fltr_node *fltr)
 {
-	__be16 src_port = 0x0, dst_port = 0x0;
-	struct qede_arfs_fltr_node *fltr;
-	struct hlist_node *temp;
-	struct hlist_head *head;
-	__be16 eth_proto;
-	u8 ip_proto;
+	int count = QEDE_ARFS_POLL_COUNT;
 
-	if (fsp->location >= QEDE_RFS_MAX_FLTR ||
-	    fsp->ring_cookie >= QEDE_RSS_COUNT(edev))
-		return -EINVAL;
+	while (fltr->used && count) {
+		msleep(20);
+		count--;
+	}
+
+	if (count == 0 || fltr->fw_rc) {
+		DP_NOTICE(edev, "Timeout in polling filter config\n");
+		qede_dequeue_fltr_and_config_searcher(edev, fltr);
+		return -EIO;
+	}
+
+	return fltr->fw_rc;
+}
+
+static int qede_flow_get_min_header_size(struct qede_arfs_tuple *t)
+{
+	int size = ETH_HLEN;
+
+	if (t->eth_proto == htons(ETH_P_IP))
+		size += sizeof(struct iphdr);
+	else
+		size += sizeof(struct ipv6hdr);
+
+	if (t->ip_proto == IPPROTO_TCP)
+		size += sizeof(struct tcphdr);
+	else
+		size += sizeof(struct udphdr);
+
+	return size;
+}
+
+static bool qede_flow_spec_ipv4_cmp(struct qede_arfs_tuple *a,
+				    struct qede_arfs_tuple *b)
+{
+	if (a->eth_proto != htons(ETH_P_IP) ||
+	    b->eth_proto != htons(ETH_P_IP))
+		return false;
+
+	return (a->src_ipv4 == b->src_ipv4) &&
+	       (a->dst_ipv4 == b->dst_ipv4);
+}
+
+static void qede_flow_build_ipv4_hdr(struct qede_arfs_tuple *t,
+				     void *header)
+{
+	__be16 *ports = (__be16 *)(header + ETH_HLEN + sizeof(struct iphdr));
+	struct iphdr *ip = (struct iphdr *)(header + ETH_HLEN);
+	struct ethhdr *eth = (struct ethhdr *)header;
+
+	eth->h_proto = t->eth_proto;
+	ip->saddr = t->src_ipv4;
+	ip->daddr = t->dst_ipv4;
+	ip->version = 0x4;
+	ip->ihl = 0x5;
+	ip->protocol = t->ip_proto;
+	ip->tot_len = cpu_to_be16(qede_flow_get_min_header_size(t) - ETH_HLEN);
+
+	/* ports is weakly typed to suit both TCP and UDP ports */
+	ports[0] = t->src_port;
+	ports[1] = t->dst_port;
+}
+
+static void qede_flow_stringify_ipv4_hdr(struct qede_arfs_tuple *t,
+					 void *buffer)
+{
+	const char *prefix = t->ip_proto == IPPROTO_TCP ? "TCP" : "UDP";
+
+	snprintf(buffer, QEDE_FILTER_PRINT_MAX_LEN,
+		 "%s %pI4 (%04x) -> %pI4 (%04x)",
+		 prefix, &t->src_ipv4, t->src_port,
+		 &t->dst_ipv4, t->dst_port);
+}
+
+static bool qede_flow_spec_ipv6_cmp(struct qede_arfs_tuple *a,
+				    struct qede_arfs_tuple *b)
+{
+	if (a->eth_proto != htons(ETH_P_IPV6) ||
+	    b->eth_proto != htons(ETH_P_IPV6))
+		return false;
+
+	if (memcmp(&a->src_ipv6, &b->src_ipv6, sizeof(struct in6_addr)))
+		return false;
 
-	if (fsp->flow_type == TCP_V4_FLOW) {
-		*min_hlen += sizeof(struct iphdr) +
-				sizeof(struct tcphdr);
-		eth_proto = htons(ETH_P_IP);
-		ip_proto = IPPROTO_TCP;
-	} else if (fsp->flow_type == UDP_V4_FLOW) {
-		*min_hlen += sizeof(struct iphdr) +
-				sizeof(struct udphdr);
-		eth_proto = htons(ETH_P_IP);
-		ip_proto = IPPROTO_UDP;
-	} else if (fsp->flow_type == TCP_V6_FLOW) {
-		*min_hlen += sizeof(struct ipv6hdr) +
-				sizeof(struct tcphdr);
-		eth_proto = htons(ETH_P_IPV6);
-		ip_proto = IPPROTO_TCP;
-	} else if (fsp->flow_type == UDP_V6_FLOW) {
-		*min_hlen += sizeof(struct ipv6hdr) +
-				sizeof(struct udphdr);
-		eth_proto = htons(ETH_P_IPV6);
-		ip_proto = IPPROTO_UDP;
+	if (memcmp(&a->dst_ipv6, &b->dst_ipv6, sizeof(struct in6_addr)))
+		return false;
+
+	return true;
+}
+
+static void qede_flow_build_ipv6_hdr(struct qede_arfs_tuple *t,
+				     void *header)
+{
+	__be16 *ports = (__be16 *)(header + ETH_HLEN + sizeof(struct ipv6hdr));
+	struct ipv6hdr *ip6 = (struct ipv6hdr *)(header + ETH_HLEN);
+	struct ethhdr *eth = (struct ethhdr *)header;
+
+	eth->h_proto = t->eth_proto;
+	memcpy(&ip6->saddr, &t->src_ipv6, sizeof(struct in6_addr));
+	memcpy(&ip6->daddr, &t->dst_ipv6, sizeof(struct in6_addr));
+	ip6->version = 0x6;
+
+	if (t->ip_proto == IPPROTO_TCP) {
+		ip6->nexthdr = NEXTHDR_TCP;
+		ip6->payload_len = cpu_to_be16(sizeof(struct tcphdr));
 	} else {
-		DP_NOTICE(edev, "Unsupported flow type = 0x%x\n",
-			  fsp->flow_type);
-		return -EPROTONOSUPPORT;
+		ip6->nexthdr = NEXTHDR_UDP;
+		ip6->payload_len = cpu_to_be16(sizeof(struct udphdr));
 	}
 
-	if (eth_proto == htons(ETH_P_IP)) {
-		src_port = fsp->h_u.tcp_ip4_spec.psrc;
-		dst_port = fsp->h_u.tcp_ip4_spec.pdst;
+	/* ports is weakly typed to suit both TCP and UDP ports */
+	ports[0] = t->src_port;
+	ports[1] = t->dst_port;
+}
+
+/* Validate fields which are set and not accepted by the driver */
+static int qede_flow_spec_validate_unused(struct qede_dev *edev,
+					  struct ethtool_rx_flow_spec *fs)
+{
+	if (fs->flow_type & FLOW_MAC_EXT) {
+		DP_INFO(edev, "Don't support MAC extensions\n");
+		return -EOPNOTSUPP;
+	}
+
+	if ((fs->flow_type & FLOW_EXT) &&
+	    (fs->h_ext.vlan_etype || fs->h_ext.vlan_tci)) {
+		DP_INFO(edev, "Don't support vlan-based classification\n");
+		return -EOPNOTSUPP;
+	}
+
+	if ((fs->flow_type & FLOW_EXT) &&
+	    (fs->h_ext.data[0] || fs->h_ext.data[1])) {
+		DP_INFO(edev, "Don't support user defined data\n");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int qede_flow_spec_to_tuple_ipv4_common(struct qede_dev *edev,
+					       struct qede_arfs_tuple *t,
+					       struct ethtool_rx_flow_spec *fs)
+{
+	if ((fs->h_u.tcp_ip4_spec.ip4src &
+	     fs->m_u.tcp_ip4_spec.ip4src) != fs->h_u.tcp_ip4_spec.ip4src) {
+		DP_INFO(edev, "Don't support IP-masks\n");
+		return -EOPNOTSUPP;
+	}
+
+	if ((fs->h_u.tcp_ip4_spec.ip4dst &
+	     fs->m_u.tcp_ip4_spec.ip4dst) != fs->h_u.tcp_ip4_spec.ip4dst) {
+		DP_INFO(edev, "Don't support IP-masks\n");
+		return -EOPNOTSUPP;
+	}
+
+	if ((fs->h_u.tcp_ip4_spec.psrc &
+	     fs->m_u.tcp_ip4_spec.psrc) != fs->h_u.tcp_ip4_spec.psrc) {
+		DP_INFO(edev, "Don't support port-masks\n");
+		return -EOPNOTSUPP;
+	}
+
+	if ((fs->h_u.tcp_ip4_spec.pdst &
+	     fs->m_u.tcp_ip4_spec.pdst) != fs->h_u.tcp_ip4_spec.pdst) {
+		DP_INFO(edev, "Don't support port-masks\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (fs->h_u.tcp_ip4_spec.tos) {
+		DP_INFO(edev, "Don't support tos\n");
+		return -EOPNOTSUPP;
+	}
+
+	t->eth_proto = htons(ETH_P_IP);
+	t->src_ipv4 = fs->h_u.tcp_ip4_spec.ip4src;
+	t->dst_ipv4 = fs->h_u.tcp_ip4_spec.ip4dst;
+	t->src_port = fs->h_u.tcp_ip4_spec.psrc;
+	t->dst_port = fs->h_u.tcp_ip4_spec.pdst;
+
+	/* We must either have a valid 4-tuple or only dst port
+	 * or only src ip as an input
+	 */
+	if (t->src_port && t->dst_port && t->src_ipv4 && t->dst_ipv4) {
+		t->mode = QED_FILTER_CONFIG_MODE_5_TUPLE;
+	} else if (!t->src_port && t->dst_port &&
+		   !t->src_ipv4 && !t->dst_ipv4) {
+		t->mode = QED_FILTER_CONFIG_MODE_L4_PORT;
+	}  else if (!t->src_port && !t->dst_port &&
+		    !t->dst_ipv4 && t->src_ipv4) {
+		t->mode = QED_FILTER_CONFIG_MODE_IP_SRC;
 	} else {
-		src_port = fsp->h_u.tcp_ip6_spec.psrc;
-		dst_port = fsp->h_u.tcp_ip6_spec.pdst;
+		DP_INFO(edev, "Invalid N-tuple\n");
+		return -EOPNOTSUPP;
 	}
 
-	head = QEDE_ARFS_BUCKET_HEAD(edev, 0);
-	hlist_for_each_entry_safe(fltr, temp, head, node) {
-		if ((fltr->tuple.ip_proto == ip_proto &&
-		     fltr->tuple.eth_proto == eth_proto &&
-		     qede_compare_user_flow_ips(fltr, fsp, eth_proto) &&
-		     fltr->tuple.src_port == src_port &&
-		     fltr->tuple.dst_port == dst_port) ||
-		    fltr->sw_id == fsp->location)
-			return -EEXIST;
+	t->ip_comp = qede_flow_spec_ipv4_cmp;
+	t->build_hdr = qede_flow_build_ipv4_hdr;
+	t->stringify = qede_flow_stringify_ipv4_hdr;
+
+	return 0;
+}
+
+static int qede_flow_spec_to_tuple_tcpv4(struct qede_dev *edev,
+					 struct qede_arfs_tuple *t,
+					 struct ethtool_rx_flow_spec *fs)
+{
+	t->ip_proto = IPPROTO_TCP;
+
+	if (qede_flow_spec_to_tuple_ipv4_common(edev, t, fs))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int qede_flow_spec_to_tuple_udpv4(struct qede_dev *edev,
+					 struct qede_arfs_tuple *t,
+					 struct ethtool_rx_flow_spec *fs)
+{
+	t->ip_proto = IPPROTO_UDP;
+
+	if (qede_flow_spec_to_tuple_ipv4_common(edev, t, fs))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int qede_flow_spec_to_tuple_ipv6_common(struct qede_dev *edev,
+					       struct qede_arfs_tuple *t,
+					       struct ethtool_rx_flow_spec *fs)
+{
+	struct in6_addr zero_addr;
+	void *p;
+
+	p = &zero_addr;
+	memset(p, 0, sizeof(zero_addr));
+
+	if ((fs->h_u.tcp_ip6_spec.psrc &
+	     fs->m_u.tcp_ip6_spec.psrc) != fs->h_u.tcp_ip6_spec.psrc) {
+		DP_INFO(edev, "Don't support port-masks\n");
+		return -EOPNOTSUPP;
+	}
+
+	if ((fs->h_u.tcp_ip6_spec.pdst &
+	     fs->m_u.tcp_ip6_spec.pdst) != fs->h_u.tcp_ip6_spec.pdst) {
+		DP_INFO(edev, "Don't support port-masks\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (fs->h_u.tcp_ip6_spec.tclass) {
+		DP_INFO(edev, "Don't support tclass\n");
+		return -EOPNOTSUPP;
 	}
 
+	t->eth_proto = htons(ETH_P_IPV6);
+	memcpy(&t->src_ipv6, &fs->h_u.tcp_ip6_spec.ip6src,
+	       sizeof(struct in6_addr));
+	memcpy(&t->dst_ipv6, &fs->h_u.tcp_ip6_spec.ip6dst,
+	       sizeof(struct in6_addr));
+	t->src_port = fs->h_u.tcp_ip6_spec.psrc;
+	t->dst_port = fs->h_u.tcp_ip6_spec.pdst;
+
+	/* We must make sure we have a valid 4-tuple or only dest port
+	 * or only src ip as an input
+	 */
+	if (t->src_port && t->dst_port &&
+	    memcmp(&t->src_ipv6, p, sizeof(struct in6_addr)) &&
+	    memcmp(&t->dst_ipv6, p, sizeof(struct in6_addr))) {
+		t->mode = QED_FILTER_CONFIG_MODE_5_TUPLE;
+	} else if (!t->src_port && t->dst_port &&
+		   !memcmp(&t->src_ipv6, p, sizeof(struct in6_addr)) &&
+		   !memcmp(&t->dst_ipv6, p, sizeof(struct in6_addr))) {
+		t->mode = QED_FILTER_CONFIG_MODE_L4_PORT;
+	} else if (!t->src_port && !t->dst_port &&
+		   !memcmp(&t->dst_ipv6, p, sizeof(struct in6_addr)) &&
+		   memcmp(&t->src_ipv6, p, sizeof(struct in6_addr))) {
+		t->mode = QED_FILTER_CONFIG_MODE_IP_SRC;
+	} else {
+		DP_INFO(edev, "Invalid N-tuple\n");
+		return -EOPNOTSUPP;
+	}
+
+	t->ip_comp = qede_flow_spec_ipv6_cmp;
+	t->build_hdr = qede_flow_build_ipv6_hdr;
+
 	return 0;
 }
 
-static int
-qede_poll_arfs_filter_config(struct qede_dev *edev,
-			     struct qede_arfs_fltr_node *fltr)
+static int qede_flow_spec_to_tuple_tcpv6(struct qede_dev *edev,
+					 struct qede_arfs_tuple *t,
+					 struct ethtool_rx_flow_spec *fs)
 {
-	int count = QEDE_ARFS_POLL_COUNT;
+	t->ip_proto = IPPROTO_TCP;
 
-	while (fltr->used && count) {
-		msleep(20);
-		count--;
+	if (qede_flow_spec_to_tuple_ipv6_common(edev, t, fs))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int qede_flow_spec_to_tuple_udpv6(struct qede_dev *edev,
+					 struct qede_arfs_tuple *t,
+					 struct ethtool_rx_flow_spec *fs)
+{
+	t->ip_proto = IPPROTO_UDP;
+
+	if (qede_flow_spec_to_tuple_ipv6_common(edev, t, fs))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int qede_flow_spec_to_tuple(struct qede_dev *edev,
+				   struct qede_arfs_tuple *t,
+				   struct ethtool_rx_flow_spec *fs)
+{
+	memset(t, 0, sizeof(*t));
+
+	if (qede_flow_spec_validate_unused(edev, fs))
+		return -EOPNOTSUPP;
+
+	switch ((fs->flow_type & ~FLOW_EXT)) {
+	case TCP_V4_FLOW:
+		return qede_flow_spec_to_tuple_tcpv4(edev, t, fs);
+	case UDP_V4_FLOW:
+		return qede_flow_spec_to_tuple_udpv4(edev, t, fs);
+	case TCP_V6_FLOW:
+		return qede_flow_spec_to_tuple_tcpv6(edev, t, fs);
+	case UDP_V6_FLOW:
+		return qede_flow_spec_to_tuple_udpv6(edev, t, fs);
+	default:
+		DP_VERBOSE(edev, NETIF_MSG_IFUP,
+			   "Can't support flow of type %08x\n", fs->flow_type);
+		return -EOPNOTSUPP;
 	}
 
-	if (count == 0 || fltr->fw_rc) {
-		qede_dequeue_fltr_and_config_searcher(edev, fltr);
-		return -EIO;
+	return 0;
+}
+
+static int qede_flow_spec_validate(struct qede_dev *edev,
+				   struct ethtool_rx_flow_spec *fs,
+				   struct qede_arfs_tuple *t)
+{
+	if (fs->location >= QEDE_RFS_MAX_FLTR) {
+		DP_INFO(edev, "Location out-of-bounds\n");
+		return -EINVAL;
 	}
 
-	return fltr->fw_rc;
+	/* Check location isn't already in use */
+	if (test_bit(fs->location, edev->arfs->arfs_fltr_bmap)) {
+		DP_INFO(edev, "Location already in use\n");
+		return -EINVAL;
+	}
+
+	/* Check if the filtering-mode could support the filter */
+	if (edev->arfs->filter_count &&
+	    edev->arfs->mode != t->mode) {
+		DP_INFO(edev,
+			"flow_spec would require filtering mode %08x, but %08x is configured\n",
+			t->mode, edev->arfs->filter_count);
+		return -EINVAL;
+	}
+
+	/* If drop requested then no need to validate other data */
+	if (fs->ring_cookie == RX_CLS_FLOW_DISC)
+		return 0;
+
+	if (ethtool_get_flow_spec_ring_vf(fs->ring_cookie))
+		return 0;
+
+	if (fs->ring_cookie >= QEDE_RSS_COUNT(edev)) {
+		DP_INFO(edev, "Queue out-of-bounds\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* Must be called while qede lock is held */
+static struct qede_arfs_fltr_node *
+qede_flow_find_fltr(struct qede_dev *edev, struct qede_arfs_tuple *t)
+{
+	struct qede_arfs_fltr_node *fltr;
+	struct hlist_node *temp;
+	struct hlist_head *head;
+
+	head = QEDE_ARFS_BUCKET_HEAD(edev, 0);
+
+	hlist_for_each_entry_safe(fltr, temp, head, node) {
+		if (fltr->tuple.ip_proto == t->ip_proto &&
+		    fltr->tuple.src_port == t->src_port &&
+		    fltr->tuple.dst_port == t->dst_port &&
+		    t->ip_comp(&fltr->tuple, t))
+			return fltr;
+	}
+
+	return NULL;
+}
+
+static void qede_flow_set_destination(struct qede_dev *edev,
+				      struct qede_arfs_fltr_node *n,
+				      struct ethtool_rx_flow_spec *fs)
+{
+	if (fs->ring_cookie == RX_CLS_FLOW_DISC) {
+		n->b_is_drop = true;
+		return;
+	}
+
+	n->vfid = ethtool_get_flow_spec_ring_vf(fs->ring_cookie);
+	n->rxq_id = ethtool_get_flow_spec_ring(fs->ring_cookie);
+	n->next_rxq_id = n->rxq_id;
+
+	if (n->vfid)
+		DP_VERBOSE(edev, QED_MSG_SP,
+			   "Configuring N-tuple for VF 0x%02x\n", n->vfid - 1);
 }
 
 int qede_add_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info)
 {
 	struct ethtool_rx_flow_spec *fsp = &info->fs;
 	struct qede_arfs_fltr_node *n;
-	int min_hlen = ETH_HLEN, rc;
-	struct ethhdr *eth;
-	struct iphdr *ip;
-	__be16 *ports;
+	struct qede_arfs_tuple t;
+	int min_hlen, rc;
 
 	__qede_lock(edev);
 
@@ -1556,16 +1889,28 @@ int qede_add_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info)
 		goto unlock;
 	}
 
-	rc = qede_validate_and_check_flow_exist(edev, fsp, &min_hlen);
+	/* Translate the flow specification into something fittign our DB */
+	rc = qede_flow_spec_to_tuple(edev, &t, fsp);
 	if (rc)
 		goto unlock;
 
+	/* Make sure location is valid and filter isn't already set */
+	rc = qede_flow_spec_validate(edev, fsp, &t);
+	if (rc)
+		goto unlock;
+
+	if (qede_flow_find_fltr(edev, &t)) {
+		rc = -EINVAL;
+		goto unlock;
+	}
+
 	n = kzalloc(sizeof(*n), GFP_KERNEL);
 	if (!n) {
 		rc = -ENOMEM;
 		goto unlock;
 	}
 
+	min_hlen = qede_flow_get_min_header_size(&t);
 	n->data = kzalloc(min_hlen, GFP_KERNEL);
 	if (!n->data) {
 		kfree(n);
@@ -1576,68 +1921,13 @@ int qede_add_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info)
 	n->sw_id = fsp->location;
 	set_bit(n->sw_id, edev->arfs->arfs_fltr_bmap);
 	n->buf_len = min_hlen;
-	n->rxq_id = fsp->ring_cookie;
-	n->next_rxq_id = n->rxq_id;
-	eth = (struct ethhdr *)n->data;
 
-	if (info->fs.flow_type == TCP_V4_FLOW ||
-	    info->fs.flow_type == UDP_V4_FLOW) {
-		ports = (__be16 *)(n->data + ETH_HLEN +
-					sizeof(struct iphdr));
-		eth->h_proto = htons(ETH_P_IP);
-		n->tuple.eth_proto = htons(ETH_P_IP);
-		n->tuple.src_ipv4 = info->fs.h_u.tcp_ip4_spec.ip4src;
-		n->tuple.dst_ipv4 = info->fs.h_u.tcp_ip4_spec.ip4dst;
-		n->tuple.src_port = info->fs.h_u.tcp_ip4_spec.psrc;
-		n->tuple.dst_port = info->fs.h_u.tcp_ip4_spec.pdst;
-		ports[0] = n->tuple.src_port;
-		ports[1] = n->tuple.dst_port;
-		ip = (struct iphdr *)(n->data + ETH_HLEN);
-		ip->saddr = info->fs.h_u.tcp_ip4_spec.ip4src;
-		ip->daddr = info->fs.h_u.tcp_ip4_spec.ip4dst;
-		ip->version = 0x4;
-		ip->ihl = 0x5;
-
-		if (info->fs.flow_type == TCP_V4_FLOW) {
-			n->tuple.ip_proto = IPPROTO_TCP;
-			ip->protocol = IPPROTO_TCP;
-		} else {
-			n->tuple.ip_proto = IPPROTO_UDP;
-			ip->protocol = IPPROTO_UDP;
-		}
-		ip->tot_len = cpu_to_be16(min_hlen - ETH_HLEN);
-	} else {
-		struct ipv6hdr *ip6;
-
-		ip6 = (struct ipv6hdr *)(n->data + ETH_HLEN);
-		ports = (__be16 *)(n->data + ETH_HLEN +
-					sizeof(struct ipv6hdr));
-		eth->h_proto = htons(ETH_P_IPV6);
-		n->tuple.eth_proto = htons(ETH_P_IPV6);
-		memcpy(&n->tuple.src_ipv6, &info->fs.h_u.tcp_ip6_spec.ip6src,
-		       sizeof(struct in6_addr));
-		memcpy(&n->tuple.dst_ipv6, &info->fs.h_u.tcp_ip6_spec.ip6dst,
-		       sizeof(struct in6_addr));
-		n->tuple.src_port = info->fs.h_u.tcp_ip6_spec.psrc;
-		n->tuple.dst_port = info->fs.h_u.tcp_ip6_spec.pdst;
-		ports[0] = n->tuple.src_port;
-		ports[1] = n->tuple.dst_port;
-		memcpy(&ip6->saddr, &n->tuple.src_ipv6,
-		       sizeof(struct in6_addr));
-		memcpy(&ip6->daddr, &n->tuple.dst_ipv6,
-		       sizeof(struct in6_addr));
-		ip6->version = 0x6;
+	memcpy(&n->tuple, &t, sizeof(n->tuple));
 
-		if (info->fs.flow_type == TCP_V6_FLOW) {
-			n->tuple.ip_proto = IPPROTO_TCP;
-			ip6->nexthdr = NEXTHDR_TCP;
-			ip6->payload_len = cpu_to_be16(sizeof(struct tcphdr));
-		} else {
-			n->tuple.ip_proto = IPPROTO_UDP;
-			ip6->nexthdr = NEXTHDR_UDP;
-			ip6->payload_len = cpu_to_be16(sizeof(struct udphdr));
-		}
-	}
+	qede_flow_set_destination(edev, n, fsp);
+
+	/* Build a minimal header according to the flow */
+	n->tuple.build_hdr(&n->tuple, n->data);
 
 	rc = qede_enqueue_fltr_and_config_searcher(edev, n, 0);
 	if (rc)
@@ -1647,6 +1937,7 @@ int qede_add_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info)
 	rc = qede_poll_arfs_filter_config(edev, n);
 unlock:
 	__qede_unlock(edev);
+
 	return rc;
 }
 
diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c
index 14941303189d..6c702399b801 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_fp.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c
@@ -660,7 +660,8 @@ static int qede_fill_frag_skb(struct qede_dev *edev,
 
 	/* Add one frag and update the appropriate fields in the skb */
 	skb_fill_page_desc(skb, tpa_info->frag_id++,
-			   current_bd->data, current_bd->page_offset,
+			   current_bd->data,
+			   current_bd->page_offset + rxq->rx_headroom,
 			   len_on_bd);
 
 	if (unlikely(qede_realloc_rx_buffer(rxq, current_bd))) {
@@ -671,8 +672,7 @@ static int qede_fill_frag_skb(struct qede_dev *edev,
 		goto out;
 	}
 
-	qed_chain_consume(&rxq->rx_bd_ring);
-	rxq->sw_rx_cons++;
+	qede_rx_bd_ring_consume(rxq);
 
 	skb->data_len += len_on_bd;
 	skb->truesize += rxq->rx_buf_seg_size;
@@ -721,64 +721,129 @@ static u8 qede_check_tunn_csum(u16 flag)
 	return QEDE_CSUM_UNNECESSARY | tcsum;
 }
 
+static inline struct sk_buff *
+qede_build_skb(struct qede_rx_queue *rxq,
+	       struct sw_rx_data *bd, u16 len, u16 pad)
+{
+	struct sk_buff *skb;
+	void *buf;
+
+	buf = page_address(bd->data) + bd->page_offset;
+	skb = build_skb(buf, rxq->rx_buf_seg_size);
+
+	skb_reserve(skb, pad);
+	skb_put(skb, len);
+
+	return skb;
+}
+
+static struct sk_buff *
+qede_tpa_rx_build_skb(struct qede_dev *edev,
+		      struct qede_rx_queue *rxq,
+		      struct sw_rx_data *bd, u16 len, u16 pad,
+		      bool alloc_skb)
+{
+	struct sk_buff *skb;
+
+	skb = qede_build_skb(rxq, bd, len, pad);
+	bd->page_offset += rxq->rx_buf_seg_size;
+
+	if (bd->page_offset == PAGE_SIZE) {
+		if (unlikely(qede_alloc_rx_buffer(rxq, true))) {
+			DP_NOTICE(edev,
+				  "Failed to allocate RX buffer for tpa start\n");
+			bd->page_offset -= rxq->rx_buf_seg_size;
+			page_ref_inc(bd->data);
+			dev_kfree_skb_any(skb);
+			return NULL;
+		}
+	} else {
+		page_ref_inc(bd->data);
+		qede_reuse_page(rxq, bd);
+	}
+
+	/* We've consumed the first BD and prepared an SKB */
+	qede_rx_bd_ring_consume(rxq);
+
+	return skb;
+}
+
+static struct sk_buff *
+qede_rx_build_skb(struct qede_dev *edev,
+		  struct qede_rx_queue *rxq,
+		  struct sw_rx_data *bd, u16 len, u16 pad)
+{
+	struct sk_buff *skb = NULL;
+
+	/* For smaller frames still need to allocate skb, memcpy
+	 * data and benefit in reusing the page segment instead of
+	 * un-mapping it.
+	 */
+	if ((len + pad <= edev->rx_copybreak)) {
+		unsigned int offset = bd->page_offset + pad;
+
+		skb = netdev_alloc_skb(edev->ndev, QEDE_RX_HDR_SIZE);
+		if (unlikely(!skb))
+			return NULL;
+
+		skb_reserve(skb, pad);
+		memcpy(skb_put(skb, len),
+		       page_address(bd->data) + offset, len);
+		qede_reuse_page(rxq, bd);
+		goto out;
+	}
+
+	skb = qede_build_skb(rxq, bd, len, pad);
+
+	if (unlikely(qede_realloc_rx_buffer(rxq, bd))) {
+		/* Incr page ref count to reuse on allocation failure so
+		 * that it doesn't get freed while freeing SKB [as its
+		 * already mapped there].
+		 */
+		page_ref_inc(bd->data);
+		dev_kfree_skb_any(skb);
+		return NULL;
+	}
+out:
+	/* We've consumed the first BD and prepared an SKB */
+	qede_rx_bd_ring_consume(rxq);
+
+	return skb;
+}
+
 static void qede_tpa_start(struct qede_dev *edev,
 			   struct qede_rx_queue *rxq,
 			   struct eth_fast_path_rx_tpa_start_cqe *cqe)
 {
 	struct qede_agg_info *tpa_info = &rxq->tpa_info[cqe->tpa_agg_index];
-	struct eth_rx_bd *rx_bd_cons = qed_chain_consume(&rxq->rx_bd_ring);
-	struct eth_rx_bd *rx_bd_prod = qed_chain_produce(&rxq->rx_bd_ring);
-	struct sw_rx_data *replace_buf = &tpa_info->buffer;
-	dma_addr_t mapping = tpa_info->buffer_mapping;
 	struct sw_rx_data *sw_rx_data_cons;
-	struct sw_rx_data *sw_rx_data_prod;
+	u16 pad;
 
 	sw_rx_data_cons = &rxq->sw_rx_ring[rxq->sw_rx_cons & NUM_RX_BDS_MAX];
-	sw_rx_data_prod = &rxq->sw_rx_ring[rxq->sw_rx_prod & NUM_RX_BDS_MAX];
+	pad = cqe->placement_offset + rxq->rx_headroom;
 
-	/* Use pre-allocated replacement buffer - we can't release the agg.
-	 * start until its over and we don't want to risk allocation failing
-	 * here, so re-allocate when aggregation will be over.
-	 */
-	sw_rx_data_prod->mapping = replace_buf->mapping;
-
-	sw_rx_data_prod->data = replace_buf->data;
-	rx_bd_prod->addr.hi = cpu_to_le32(upper_32_bits(mapping));
-	rx_bd_prod->addr.lo = cpu_to_le32(lower_32_bits(mapping));
-	sw_rx_data_prod->page_offset = replace_buf->page_offset;
-
-	rxq->sw_rx_prod++;
+	tpa_info->skb = qede_tpa_rx_build_skb(edev, rxq, sw_rx_data_cons,
+					      le16_to_cpu(cqe->len_on_first_bd),
+					      pad, false);
+	tpa_info->buffer.page_offset = sw_rx_data_cons->page_offset;
+	tpa_info->buffer.mapping = sw_rx_data_cons->mapping;
 
-	/* move partial skb from cons to pool (don't unmap yet)
-	 * save mapping, incase we drop the packet later on.
-	 */
-	tpa_info->buffer = *sw_rx_data_cons;
-	mapping = HILO_U64(le32_to_cpu(rx_bd_cons->addr.hi),
-			   le32_to_cpu(rx_bd_cons->addr.lo));
-
-	tpa_info->buffer_mapping = mapping;
-	rxq->sw_rx_cons++;
-
-	/* set tpa state to start only if we are able to allocate skb
-	 * for this aggregation, otherwise mark as error and aggregation will
-	 * be dropped
-	 */
-	tpa_info->skb = netdev_alloc_skb(edev->ndev,
-					 le16_to_cpu(cqe->len_on_first_bd));
 	if (unlikely(!tpa_info->skb)) {
 		DP_NOTICE(edev, "Failed to allocate SKB for gro\n");
+
+		/* Consume from ring but do not produce since
+		 * this might be used by FW still, it will be re-used
+		 * at TPA end.
+		 */
+		tpa_info->tpa_start_fail = true;
+		qede_rx_bd_ring_consume(rxq);
 		tpa_info->state = QEDE_AGG_STATE_ERROR;
 		goto cons_buf;
 	}
 
-	/* Start filling in the aggregation info */
-	skb_put(tpa_info->skb, le16_to_cpu(cqe->len_on_first_bd));
 	tpa_info->frag_id = 0;
 	tpa_info->state = QEDE_AGG_STATE_START;
 
-	/* Store some information from first CQE */
-	tpa_info->start_cqe_placement_offset = cqe->placement_offset;
-	tpa_info->start_cqe_bd_len = le16_to_cpu(cqe->len_on_first_bd);
 	if ((le16_to_cpu(cqe->pars_flags.flags) >>
 	     PARSING_AND_ERR_FLAGS_TAG8021QEXIST_SHIFT) &
 	    PARSING_AND_ERR_FLAGS_TAG8021QEXIST_MASK)
@@ -899,6 +964,10 @@ static int qede_tpa_end(struct qede_dev *edev,
 	tpa_info = &rxq->tpa_info[cqe->tpa_agg_index];
 	skb = tpa_info->skb;
 
+	if (tpa_info->buffer.page_offset == PAGE_SIZE)
+		dma_unmap_page(rxq->dev, tpa_info->buffer.mapping,
+			       PAGE_SIZE, rxq->data_direction);
+
 	for (i = 0; cqe->len_list[i]; i++)
 		qede_fill_frag_skb(edev, rxq, cqe->tpa_agg_index,
 				   le16_to_cpu(cqe->len_list[i]));
@@ -919,11 +988,6 @@ static int qede_tpa_end(struct qede_dev *edev,
 		       "Strange - total packet len [cqe] is %4x but SKB has len %04x\n",
 		       le16_to_cpu(cqe->total_packet_len), skb->len);
 
-	memcpy(skb->data,
-	       page_address(tpa_info->buffer.data) +
-	       tpa_info->start_cqe_placement_offset +
-	       tpa_info->buffer.page_offset, tpa_info->start_cqe_bd_len);
-
 	/* Finalize the SKB */
 	skb->protocol = eth_type_trans(skb, edev->ndev);
 	skb->ip_summed = CHECKSUM_UNNECESSARY;
@@ -940,6 +1004,12 @@ static int qede_tpa_end(struct qede_dev *edev,
 	return 1;
 err:
 	tpa_info->state = QEDE_AGG_STATE_NONE;
+
+	if (tpa_info->tpa_start_fail) {
+		qede_reuse_page(rxq, &tpa_info->buffer);
+		tpa_info->tpa_start_fail = false;
+	}
+
 	dev_kfree_skb_any(tpa_info->skb);
 	tpa_info->skb = NULL;
 	return 0;
@@ -1058,65 +1128,6 @@ static bool qede_rx_xdp(struct qede_dev *edev,
 	return false;
 }
 
-static struct sk_buff *qede_rx_allocate_skb(struct qede_dev *edev,
-					    struct qede_rx_queue *rxq,
-					    struct sw_rx_data *bd, u16 len,
-					    u16 pad)
-{
-	unsigned int offset = bd->page_offset + pad;
-	struct skb_frag_struct *frag;
-	struct page *page = bd->data;
-	unsigned int pull_len;
-	struct sk_buff *skb;
-	unsigned char *va;
-
-	/* Allocate a new SKB with a sufficient large header len */
-	skb = netdev_alloc_skb(edev->ndev, QEDE_RX_HDR_SIZE);
-	if (unlikely(!skb))
-		return NULL;
-
-	/* Copy data into SKB - if it's small, we can simply copy it and
-	 * re-use the already allcoated & mapped memory.
-	 */
-	if (len + pad <= edev->rx_copybreak) {
-		skb_put_data(skb, page_address(page) + offset, len);
-		qede_reuse_page(rxq, bd);
-		goto out;
-	}
-
-	frag = &skb_shinfo(skb)->frags[0];
-
-	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
-			page, offset, len, rxq->rx_buf_seg_size);
-
-	va = skb_frag_address(frag);
-	pull_len = eth_get_headlen(va, QEDE_RX_HDR_SIZE);
-
-	/* Align the pull_len to optimize memcpy */
-	memcpy(skb->data, va, ALIGN(pull_len, sizeof(long)));
-
-	/* Correct the skb & frag sizes offset after the pull */
-	skb_frag_size_sub(frag, pull_len);
-	frag->page_offset += pull_len;
-	skb->data_len -= pull_len;
-	skb->tail += pull_len;
-
-	if (unlikely(qede_realloc_rx_buffer(rxq, bd))) {
-		/* Incr page ref count to reuse on allocation failure so
-		 * that it doesn't get freed while freeing SKB [as its
-		 * already mapped there].
-		 */
-		page_ref_inc(page);
-		dev_kfree_skb_any(skb);
-		return NULL;
-	}
-
-out:
-	/* We've consumed the first BD and prepared an SKB */
-	qede_rx_bd_ring_consume(rxq);
-	return skb;
-}
-
 static int qede_rx_build_jumbo(struct qede_dev *edev,
 			       struct qede_rx_queue *rxq,
 			       struct sk_buff *skb,
@@ -1157,7 +1168,7 @@ static int qede_rx_build_jumbo(struct qede_dev *edev,
 			       PAGE_SIZE, DMA_FROM_DEVICE);
 
 		skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
-				   bd->data, 0, cur_size);
+				   bd->data, rxq->rx_headroom, cur_size);
 
 		skb->truesize += PAGE_SIZE;
 		skb->data_len += cur_size;
@@ -1256,7 +1267,7 @@ static int qede_rx_process_cqe(struct qede_dev *edev,
 	/* Basic validation passed; Need to prepare an SKB. This would also
 	 * guarantee to finally consume the first BD upon success.
 	 */
-	skb = qede_rx_allocate_skb(edev, rxq, bd, len, pad);
+	skb = qede_rx_build_skb(edev, rxq, bd, len, pad);
 	if (!skb) {
 		rxq->rx_alloc_errors++;
 		qede_recycle_rx_bd_ring(rxq, fp_cqe->bd_num);
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index f6655e251bbd..d118771e1a7b 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -133,6 +133,9 @@ static int qede_probe(struct pci_dev *pdev, const struct pci_device_id *id);
 static void qede_remove(struct pci_dev *pdev);
 static void qede_shutdown(struct pci_dev *pdev);
 static void qede_link_update(void *dev, struct qed_link_output *link);
+static void qede_get_eth_tlv_data(void *edev, void *data);
+static void qede_get_generic_tlv_data(void *edev,
+				      struct qed_generic_tlvs *data);
 
 /* The qede lock is used to protect driver state change and driver flows that
  * are not reentrant.
@@ -199,7 +202,7 @@ static int qede_sriov_configure(struct pci_dev *pdev, int num_vfs_param)
 
 	/* Enable/Disable Tx switching for PF */
 	if ((rc == num_vfs_param) && netif_running(edev->ndev) &&
-	    qed_info->mf_mode != QED_MF_NPAR && qed_info->tx_switching) {
+	    !qed_info->b_inter_pf_switch && qed_info->tx_switching) {
 		vport_params->vport_id = 0;
 		vport_params->update_tx_switching_flg = 1;
 		vport_params->tx_switching_flg = num_vfs_param ? 1 : 0;
@@ -228,6 +231,8 @@ static struct qed_eth_cb_ops qede_ll_ops = {
 		.arfs_filter_op = qede_arfs_filter_op,
 #endif
 		.link_update = qede_link_update,
+		.get_generic_tlv_data = qede_get_generic_tlv_data,
+		.get_protocol_tlv_data = qede_get_eth_tlv_data,
 	},
 	.force_mac = qede_force_mac,
 	.ports_update = qede_udp_ports_update,
@@ -342,6 +347,7 @@ void qede_fill_by_demand_stats(struct qede_dev *edev)
 	p_common->rx_bcast_pkts = stats.common.rx_bcast_pkts;
 	p_common->mftag_filter_discards = stats.common.mftag_filter_discards;
 	p_common->mac_filter_discards = stats.common.mac_filter_discards;
+	p_common->gft_filter_drop = stats.common.gft_filter_drop;
 
 	p_common->tx_ucast_bytes = stats.common.tx_ucast_bytes;
 	p_common->tx_mcast_bytes = stats.common.tx_mcast_bytes;
@@ -1196,30 +1202,8 @@ static void qede_free_rx_buffers(struct qede_dev *edev,
 	}
 }
 
-static void qede_free_sge_mem(struct qede_dev *edev, struct qede_rx_queue *rxq)
-{
-	int i;
-
-	if (edev->gro_disable)
-		return;
-
-	for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) {
-		struct qede_agg_info *tpa_info = &rxq->tpa_info[i];
-		struct sw_rx_data *replace_buf = &tpa_info->buffer;
-
-		if (replace_buf->data) {
-			dma_unmap_page(&edev->pdev->dev,
-				       replace_buf->mapping,
-				       PAGE_SIZE, DMA_FROM_DEVICE);
-			__free_page(replace_buf->data);
-		}
-	}
-}
-
 static void qede_free_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
 {
-	qede_free_sge_mem(edev, rxq);
-
 	/* Free rx buffers */
 	qede_free_rx_buffers(edev, rxq);
 
@@ -1231,45 +1215,15 @@ static void qede_free_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
 	edev->ops->common->chain_free(edev->cdev, &rxq->rx_comp_ring);
 }
 
-static int qede_alloc_sge_mem(struct qede_dev *edev, struct qede_rx_queue *rxq)
+static void qede_set_tpa_param(struct qede_rx_queue *rxq)
 {
-	dma_addr_t mapping;
 	int i;
 
-	if (edev->gro_disable)
-		return 0;
-
 	for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) {
 		struct qede_agg_info *tpa_info = &rxq->tpa_info[i];
-		struct sw_rx_data *replace_buf = &tpa_info->buffer;
-
-		replace_buf->data = alloc_pages(GFP_ATOMIC, 0);
-		if (unlikely(!replace_buf->data)) {
-			DP_NOTICE(edev,
-				  "Failed to allocate TPA skb pool [replacement buffer]\n");
-			goto err;
-		}
-
-		mapping = dma_map_page(&edev->pdev->dev, replace_buf->data, 0,
-				       PAGE_SIZE, DMA_FROM_DEVICE);
-		if (unlikely(dma_mapping_error(&edev->pdev->dev, mapping))) {
-			DP_NOTICE(edev,
-				  "Failed to map TPA replacement buffer\n");
-			goto err;
-		}
 
-		replace_buf->mapping = mapping;
-		tpa_info->buffer.page_offset = 0;
-		tpa_info->buffer_mapping = mapping;
 		tpa_info->state = QEDE_AGG_STATE_NONE;
 	}
-
-	return 0;
-err:
-	qede_free_sge_mem(edev, rxq);
-	edev->gro_disable = 1;
-	edev->ndev->features &= ~NETIF_F_GRO_HW;
-	return -ENOMEM;
 }
 
 /* This function allocates all memory needed per Rx queue */
@@ -1280,19 +1234,24 @@ static int qede_alloc_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
 	rxq->num_rx_buffers = edev->q_num_rx_buffers;
 
 	rxq->rx_buf_size = NET_IP_ALIGN + ETH_OVERHEAD + edev->ndev->mtu;
-	rxq->rx_headroom = edev->xdp_prog ? XDP_PACKET_HEADROOM : 0;
+
+	rxq->rx_headroom = edev->xdp_prog ? XDP_PACKET_HEADROOM : NET_SKB_PAD;
+	size = rxq->rx_headroom +
+	       SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 
 	/* Make sure that the headroom and  payload fit in a single page */
-	if (rxq->rx_buf_size + rxq->rx_headroom > PAGE_SIZE)
-		rxq->rx_buf_size = PAGE_SIZE - rxq->rx_headroom;
+	if (rxq->rx_buf_size + size > PAGE_SIZE)
+		rxq->rx_buf_size = PAGE_SIZE - size;
 
-	/* Segment size to spilt a page in multiple equal parts,
+	/* Segment size to spilt a page in multiple equal parts ,
 	 * unless XDP is used in which case we'd use the entire page.
 	 */
-	if (!edev->xdp_prog)
-		rxq->rx_buf_seg_size = roundup_pow_of_two(rxq->rx_buf_size);
-	else
+	if (!edev->xdp_prog) {
+		size = size + rxq->rx_buf_size;
+		rxq->rx_buf_seg_size = roundup_pow_of_two(size);
+	} else {
 		rxq->rx_buf_seg_size = PAGE_SIZE;
+	}
 
 	/* Allocate the parallel driver ring for Rx buffers */
 	size = sizeof(*rxq->sw_rx_ring) * RX_RING_SIZE;
@@ -1336,7 +1295,8 @@ static int qede_alloc_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
 		}
 	}
 
-	rc = qede_alloc_sge_mem(edev, rxq);
+	if (!edev->gro_disable)
+		qede_set_tpa_param(rxq);
 err:
 	return rc;
 }
@@ -1927,7 +1887,7 @@ static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
 	vport_update_params->update_vport_active_flg = 1;
 	vport_update_params->vport_active_flg = 1;
 
-	if ((qed_info->mf_mode == QED_MF_NPAR || pci_num_vf(edev->pdev)) &&
+	if ((qed_info->b_inter_pf_switch || pci_num_vf(edev->pdev)) &&
 	    qed_info->tx_switching) {
 		vport_update_params->update_tx_switching_flg = 1;
 		vport_update_params->tx_switching_flg = 1;
@@ -2177,3 +2137,99 @@ static void qede_link_update(void *dev, struct qed_link_output *link)
 		}
 	}
 }
+
+static bool qede_is_txq_full(struct qede_dev *edev, struct qede_tx_queue *txq)
+{
+	struct netdev_queue *netdev_txq;
+
+	netdev_txq = netdev_get_tx_queue(edev->ndev, txq->index);
+	if (netif_xmit_stopped(netdev_txq))
+		return true;
+
+	return false;
+}
+
+static void qede_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data)
+{
+	struct qede_dev *edev = dev;
+	struct netdev_hw_addr *ha;
+	int i;
+
+	if (edev->ndev->features & NETIF_F_IP_CSUM)
+		data->feat_flags |= QED_TLV_IP_CSUM;
+	if (edev->ndev->features & NETIF_F_TSO)
+		data->feat_flags |= QED_TLV_LSO;
+
+	ether_addr_copy(data->mac[0], edev->ndev->dev_addr);
+	memset(data->mac[1], 0, ETH_ALEN);
+	memset(data->mac[2], 0, ETH_ALEN);
+	/* Copy the first two UC macs */
+	netif_addr_lock_bh(edev->ndev);
+	i = 1;
+	netdev_for_each_uc_addr(ha, edev->ndev) {
+		ether_addr_copy(data->mac[i++], ha->addr);
+		if (i == QED_TLV_MAC_COUNT)
+			break;
+	}
+
+	netif_addr_unlock_bh(edev->ndev);
+}
+
+static void qede_get_eth_tlv_data(void *dev, void *data)
+{
+	struct qed_mfw_tlv_eth *etlv = data;
+	struct qede_dev *edev = dev;
+	struct qede_fastpath *fp;
+	int i;
+
+	etlv->lso_maxoff_size = 0XFFFF;
+	etlv->lso_maxoff_size_set = true;
+	etlv->lso_minseg_size = (u16)ETH_TX_LSO_WINDOW_MIN_LEN;
+	etlv->lso_minseg_size_set = true;
+	etlv->prom_mode = !!(edev->ndev->flags & IFF_PROMISC);
+	etlv->prom_mode_set = true;
+	etlv->tx_descr_size = QEDE_TSS_COUNT(edev);
+	etlv->tx_descr_size_set = true;
+	etlv->rx_descr_size = QEDE_RSS_COUNT(edev);
+	etlv->rx_descr_size_set = true;
+	etlv->iov_offload = QED_MFW_TLV_IOV_OFFLOAD_VEB;
+	etlv->iov_offload_set = true;
+
+	/* Fill information regarding queues; Should be done under the qede
+	 * lock to guarantee those don't change beneath our feet.
+	 */
+	etlv->txqs_empty = true;
+	etlv->rxqs_empty = true;
+	etlv->num_txqs_full = 0;
+	etlv->num_rxqs_full = 0;
+
+	__qede_lock(edev);
+	for_each_queue(i) {
+		fp = &edev->fp_array[i];
+		if (fp->type & QEDE_FASTPATH_TX) {
+			if (fp->txq->sw_tx_cons != fp->txq->sw_tx_prod)
+				etlv->txqs_empty = false;
+			if (qede_is_txq_full(edev, fp->txq))
+				etlv->num_txqs_full++;
+		}
+		if (fp->type & QEDE_FASTPATH_RX) {
+			if (qede_has_rx_work(fp->rxq))
+				etlv->rxqs_empty = false;
+
+			/* This one is a bit tricky; Firmware might stop
+			 * placing packets if ring is not yet full.
+			 * Give an approximation.
+			 */
+			if (le16_to_cpu(*fp->rxq->hw_cons_ptr) -
+			    qed_chain_get_cons_idx(&fp->rxq->rx_comp_ring) >
+			    RX_RING_SIZE - 100)
+				etlv->num_rxqs_full++;
+		}
+	}
+	__qede_unlock(edev);
+
+	etlv->txqs_empty_set = true;
+	etlv->rxqs_empty_set = true;
+	etlv->num_txqs_full_set = true;
+	etlv->num_rxqs_full_set = true;
+}
diff --git a/drivers/net/ethernet/qualcomm/emac/emac-mac.c b/drivers/net/ethernet/qualcomm/emac/emac-mac.c
index d5a32b7c7dc5..031f6e6ee9c1 100644
--- a/drivers/net/ethernet/qualcomm/emac/emac-mac.c
+++ b/drivers/net/ethernet/qualcomm/emac/emac-mac.c
@@ -683,10 +683,11 @@ static int emac_tx_q_desc_alloc(struct emac_adapter *adpt,
 				struct emac_tx_queue *tx_q)
 {
 	struct emac_ring_header *ring_header = &adpt->ring_header;
+	int node = dev_to_node(adpt->netdev->dev.parent);
 	size_t size;
 
 	size = sizeof(struct emac_buffer) * tx_q->tpd.count;
-	tx_q->tpd.tpbuff = kzalloc(size, GFP_KERNEL);
+	tx_q->tpd.tpbuff = kzalloc_node(size, GFP_KERNEL, node);
 	if (!tx_q->tpd.tpbuff)
 		return -ENOMEM;
 
@@ -723,11 +724,12 @@ static void emac_rx_q_bufs_free(struct emac_adapter *adpt)
 static int emac_rx_descs_alloc(struct emac_adapter *adpt)
 {
 	struct emac_ring_header *ring_header = &adpt->ring_header;
+	int node = dev_to_node(adpt->netdev->dev.parent);
 	struct emac_rx_queue *rx_q = &adpt->rx_q;
 	size_t size;
 
 	size = sizeof(struct emac_buffer) * rx_q->rfd.count;
-	rx_q->rfd.rfbuff = kzalloc(size, GFP_KERNEL);
+	rx_q->rfd.rfbuff = kzalloc_node(size, GFP_KERNEL, node);
 	if (!rx_q->rfd.rfbuff)
 		return -ENOMEM;
 
@@ -920,14 +922,13 @@ static void emac_mac_rx_descs_refill(struct emac_adapter *adpt,
 static void emac_adjust_link(struct net_device *netdev)
 {
 	struct emac_adapter *adpt = netdev_priv(netdev);
-	struct emac_sgmii *sgmii = &adpt->phy;
 	struct phy_device *phydev = netdev->phydev;
 
 	if (phydev->link) {
 		emac_mac_start(adpt);
-		sgmii->link_up(adpt);
+		emac_sgmii_link_change(adpt, true);
 	} else {
-		sgmii->link_down(adpt);
+		emac_sgmii_link_change(adpt, false);
 		emac_mac_stop(adpt);
 	}
 
diff --git a/drivers/net/ethernet/qualcomm/emac/emac-sgmii.c b/drivers/net/ethernet/qualcomm/emac/emac-sgmii.c
index e8ab512ee7e3..e78e5db39458 100644
--- a/drivers/net/ethernet/qualcomm/emac/emac-sgmii.c
+++ b/drivers/net/ethernet/qualcomm/emac/emac-sgmii.c
@@ -53,6 +53,46 @@
 
 #define SERDES_START_WAIT_TIMES			100
 
+int emac_sgmii_init(struct emac_adapter *adpt)
+{
+	if (!(adpt->phy.sgmii_ops && adpt->phy.sgmii_ops->init))
+		return 0;
+
+	return adpt->phy.sgmii_ops->init(adpt);
+}
+
+int emac_sgmii_open(struct emac_adapter *adpt)
+{
+	if (!(adpt->phy.sgmii_ops && adpt->phy.sgmii_ops->open))
+		return 0;
+
+	return adpt->phy.sgmii_ops->open(adpt);
+}
+
+void emac_sgmii_close(struct emac_adapter *adpt)
+{
+	if (!(adpt->phy.sgmii_ops && adpt->phy.sgmii_ops->close))
+		return;
+
+	adpt->phy.sgmii_ops->close(adpt);
+}
+
+int emac_sgmii_link_change(struct emac_adapter *adpt, bool link_state)
+{
+	if (!(adpt->phy.sgmii_ops && adpt->phy.sgmii_ops->link_change))
+		return 0;
+
+	return adpt->phy.sgmii_ops->link_change(adpt, link_state);
+}
+
+void emac_sgmii_reset(struct emac_adapter *adpt)
+{
+	if (!(adpt->phy.sgmii_ops && adpt->phy.sgmii_ops->reset))
+		return;
+
+	adpt->phy.sgmii_ops->reset(adpt);
+}
+
 /* Initialize the SGMII link between the internal and external PHYs. */
 static void emac_sgmii_link_init(struct emac_adapter *adpt)
 {
@@ -163,21 +203,21 @@ static void emac_sgmii_reset_prepare(struct emac_adapter *adpt)
 	msleep(50);
 }
 
-void emac_sgmii_reset(struct emac_adapter *adpt)
+static void emac_sgmii_common_reset(struct emac_adapter *adpt)
 {
 	int ret;
 
 	emac_sgmii_reset_prepare(adpt);
 	emac_sgmii_link_init(adpt);
 
-	ret = adpt->phy.initialize(adpt);
+	ret = emac_sgmii_init(adpt);
 	if (ret)
 		netdev_err(adpt->netdev,
 			   "could not reinitialize internal PHY (error=%i)\n",
 			   ret);
 }
 
-static int emac_sgmii_open(struct emac_adapter *adpt)
+static int emac_sgmii_common_open(struct emac_adapter *adpt)
 {
 	struct emac_sgmii *sgmii = &adpt->phy;
 	int ret;
@@ -201,43 +241,63 @@ static int emac_sgmii_open(struct emac_adapter *adpt)
 	return 0;
 }
 
-static int emac_sgmii_close(struct emac_adapter *adpt)
+static void emac_sgmii_common_close(struct emac_adapter *adpt)
 {
 	struct emac_sgmii *sgmii = &adpt->phy;
 
 	/* Make sure interrupts are disabled */
 	writel(0, sgmii->base + EMAC_SGMII_PHY_INTERRUPT_MASK);
 	free_irq(sgmii->irq, adpt);
-
-	return 0;
 }
 
 /* The error interrupts are only valid after the link is up */
-static int emac_sgmii_link_up(struct emac_adapter *adpt)
+static int emac_sgmii_common_link_change(struct emac_adapter *adpt, bool linkup)
 {
 	struct emac_sgmii *sgmii = &adpt->phy;
 	int ret;
 
-	/* Clear and enable interrupts */
-	ret = emac_sgmii_irq_clear(adpt, 0xff);
-	if (ret)
-		return ret;
+	if (linkup) {
+		/* Clear and enable interrupts */
+		ret = emac_sgmii_irq_clear(adpt, 0xff);
+		if (ret)
+			return ret;
 
-	writel(SGMII_ISR_MASK, sgmii->base + EMAC_SGMII_PHY_INTERRUPT_MASK);
+		writel(SGMII_ISR_MASK,
+		       sgmii->base + EMAC_SGMII_PHY_INTERRUPT_MASK);
+	} else {
+		/* Disable interrupts */
+		writel(0, sgmii->base + EMAC_SGMII_PHY_INTERRUPT_MASK);
+		synchronize_irq(sgmii->irq);
+	}
 
 	return 0;
 }
 
-static int emac_sgmii_link_down(struct emac_adapter *adpt)
-{
-	struct emac_sgmii *sgmii = &adpt->phy;
+static struct sgmii_ops fsm9900_ops = {
+	.init = emac_sgmii_init_fsm9900,
+	.open = emac_sgmii_common_open,
+	.close = emac_sgmii_common_close,
+	.link_change = emac_sgmii_common_link_change,
+	.reset = emac_sgmii_common_reset,
+};
 
-	/* Disable interrupts */
-	writel(0, sgmii->base + EMAC_SGMII_PHY_INTERRUPT_MASK);
-	synchronize_irq(sgmii->irq);
+static struct sgmii_ops qdf2432_ops = {
+	.init = emac_sgmii_init_qdf2432,
+	.open = emac_sgmii_common_open,
+	.close = emac_sgmii_common_close,
+	.link_change = emac_sgmii_common_link_change,
+	.reset = emac_sgmii_common_reset,
+};
 
-	return 0;
-}
+#ifdef CONFIG_ACPI
+static struct sgmii_ops qdf2400_ops = {
+	.init = emac_sgmii_init_qdf2400,
+	.open = emac_sgmii_common_open,
+	.close = emac_sgmii_common_close,
+	.link_change = emac_sgmii_common_link_change,
+	.reset = emac_sgmii_common_reset,
+};
+#endif
 
 static int emac_sgmii_acpi_match(struct device *dev, void *data)
 {
@@ -249,7 +309,7 @@ static int emac_sgmii_acpi_match(struct device *dev, void *data)
 		{}
 	};
 	const struct acpi_device_id *id = acpi_match_device(match_table, dev);
-	emac_sgmii_function *initialize = data;
+	struct sgmii_ops **ops = data;
 
 	if (id) {
 		acpi_handle handle = ACPI_HANDLE(dev);
@@ -270,10 +330,10 @@ static int emac_sgmii_acpi_match(struct device *dev, void *data)
 
 		switch (hrv) {
 		case 1:
-			*initialize = emac_sgmii_init_qdf2432;
+			*ops = &qdf2432_ops;
 			return 1;
 		case 2:
-			*initialize = emac_sgmii_init_qdf2400;
+			*ops = &qdf2400_ops;
 			return 1;
 		}
 	}
@@ -285,23 +345,15 @@ static int emac_sgmii_acpi_match(struct device *dev, void *data)
 static const struct of_device_id emac_sgmii_dt_match[] = {
 	{
 		.compatible = "qcom,fsm9900-emac-sgmii",
-		.data = emac_sgmii_init_fsm9900,
+		.data = &fsm9900_ops,
 	},
 	{
 		.compatible = "qcom,qdf2432-emac-sgmii",
-		.data = emac_sgmii_init_qdf2432,
+		.data = &qdf2432_ops,
 	},
 	{}
 };
 
-/* Dummy function for systems without an internal PHY. This avoids having
- * to check for NULL pointers before calling the functions.
- */
-static int emac_sgmii_dummy(struct emac_adapter *adpt)
-{
-	return 0;
-}
-
 int emac_sgmii_config(struct platform_device *pdev, struct emac_adapter *adpt)
 {
 	struct platform_device *sgmii_pdev = NULL;
@@ -312,22 +364,11 @@ int emac_sgmii_config(struct platform_device *pdev, struct emac_adapter *adpt)
 	if (has_acpi_companion(&pdev->dev)) {
 		struct device *dev;
 
-		dev = device_find_child(&pdev->dev, &phy->initialize,
+		dev = device_find_child(&pdev->dev, &phy->sgmii_ops,
 					emac_sgmii_acpi_match);
 
 		if (!dev) {
 			dev_warn(&pdev->dev, "cannot find internal phy node\n");
-			/* There is typically no internal PHY on emulation
-			 * systems, so if we can't find the node, assume
-			 * we are on an emulation system and stub-out
-			 * support for the internal PHY.  These systems only
-			 * use ACPI.
-			 */
-			phy->open = emac_sgmii_dummy;
-			phy->close = emac_sgmii_dummy;
-			phy->link_up = emac_sgmii_dummy;
-			phy->link_down = emac_sgmii_dummy;
-
 			return 0;
 		}
 
@@ -355,14 +396,9 @@ int emac_sgmii_config(struct platform_device *pdev, struct emac_adapter *adpt)
 			goto error_put_device;
 		}
 
-		phy->initialize = (emac_sgmii_function)match->data;
+		phy->sgmii_ops = (struct sgmii_ops *)match->data;
 	}
 
-	phy->open = emac_sgmii_open;
-	phy->close = emac_sgmii_close;
-	phy->link_up = emac_sgmii_link_up;
-	phy->link_down = emac_sgmii_link_down;
-
 	/* Base address is the first address */
 	res = platform_get_resource(sgmii_pdev, IORESOURCE_MEM, 0);
 	if (!res) {
@@ -386,7 +422,7 @@ int emac_sgmii_config(struct platform_device *pdev, struct emac_adapter *adpt)
 		}
 	}
 
-	ret = phy->initialize(adpt);
+	ret = emac_sgmii_init(adpt);
 	if (ret)
 		goto error;
 
diff --git a/drivers/net/ethernet/qualcomm/emac/emac-sgmii.h b/drivers/net/ethernet/qualcomm/emac/emac-sgmii.h
index e7c0c3b2baa4..31ba21eb61d2 100644
--- a/drivers/net/ethernet/qualcomm/emac/emac-sgmii.h
+++ b/drivers/net/ethernet/qualcomm/emac/emac-sgmii.h
@@ -16,36 +16,44 @@
 struct emac_adapter;
 struct platform_device;
 
-typedef int (*emac_sgmii_function)(struct emac_adapter *adpt);
+/** emac_sgmii - internal emac phy
+ * @init initialization function
+ * @open called when the driver is opened
+ * @close called when the driver is closed
+ * @link_change called when the link state changes
+ */
+struct sgmii_ops {
+	int (*init)(struct emac_adapter *adpt);
+	int (*open)(struct emac_adapter *adpt);
+	void (*close)(struct emac_adapter *adpt);
+	int (*link_change)(struct emac_adapter *adpt, bool link_state);
+	void (*reset)(struct emac_adapter *adpt);
+};
 
 /** emac_sgmii - internal emac phy
  * @base base address
  * @digital per-lane digital block
  * @irq the interrupt number
  * @decode_error_count reference count of consecutive decode errors
- * @initialize initialization function
- * @open called when the driver is opened
- * @close called when the driver is closed
- * @link_up called when the link comes up
- * @link_down called when the link comes down
+ * @sgmii_ops sgmii ops
  */
 struct emac_sgmii {
 	void __iomem		*base;
 	void __iomem		*digital;
 	unsigned int		irq;
 	atomic_t		decode_error_count;
-	emac_sgmii_function	initialize;
-	emac_sgmii_function	open;
-	emac_sgmii_function	close;
-	emac_sgmii_function	link_up;
-	emac_sgmii_function	link_down;
+	struct	sgmii_ops	*sgmii_ops;
 };
 
 int emac_sgmii_config(struct platform_device *pdev, struct emac_adapter *adpt);
-void emac_sgmii_reset(struct emac_adapter *adpt);
 
 int emac_sgmii_init_fsm9900(struct emac_adapter *adpt);
 int emac_sgmii_init_qdf2432(struct emac_adapter *adpt);
 int emac_sgmii_init_qdf2400(struct emac_adapter *adpt);
 
+int emac_sgmii_init(struct emac_adapter *adpt);
+int emac_sgmii_open(struct emac_adapter *adpt);
+void emac_sgmii_close(struct emac_adapter *adpt);
+int emac_sgmii_link_change(struct emac_adapter *adpt, bool link_state);
+void emac_sgmii_reset(struct emac_adapter *adpt);
 #endif
diff --git a/drivers/net/ethernet/qualcomm/emac/emac.c b/drivers/net/ethernet/qualcomm/emac/emac.c
index 13235baf4766..2a0cbc535a2e 100644
--- a/drivers/net/ethernet/qualcomm/emac/emac.c
+++ b/drivers/net/ethernet/qualcomm/emac/emac.c
@@ -253,7 +253,7 @@ static int emac_open(struct net_device *netdev)
 		return ret;
 	}
 
-	ret = adpt->phy.open(adpt);
+	ret = emac_sgmii_open(adpt);
 	if (ret) {
 		emac_mac_rx_tx_rings_free_all(adpt);
 		free_irq(irq->irq, irq);
@@ -264,7 +264,7 @@ static int emac_open(struct net_device *netdev)
 	if (ret) {
 		emac_mac_rx_tx_rings_free_all(adpt);
 		free_irq(irq->irq, irq);
-		adpt->phy.close(adpt);
+		emac_sgmii_close(adpt);
 		return ret;
 	}
 
@@ -278,7 +278,7 @@ static int emac_close(struct net_device *netdev)
 
 	mutex_lock(&adpt->reset_lock);
 
-	adpt->phy.close(adpt);
+	emac_sgmii_close(adpt);
 	emac_mac_down(adpt);
 	emac_mac_rx_tx_rings_free_all(adpt);
 
@@ -761,11 +761,10 @@ static void emac_shutdown(struct platform_device *pdev)
 {
 	struct net_device *netdev = dev_get_drvdata(&pdev->dev);
 	struct emac_adapter *adpt = netdev_priv(netdev);
-	struct emac_sgmii *sgmii = &adpt->phy;
 
 	if (netdev->flags & IFF_UP) {
 		/* Closing the SGMII turns off its interrupts */
-		sgmii->close(adpt);
+		emac_sgmii_close(adpt);
 
 		/* Resetting the MAC turns off all DMA and its interrupts */
 		emac_mac_reset(adpt);
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
index 0b5b5da80198..34ac45a774e7 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
@@ -54,11 +54,24 @@ struct rmnet_pcpu_stats {
 	struct u64_stats_sync syncp;
 };
 
+struct rmnet_priv_stats {
+	u64 csum_ok;
+	u64 csum_valid_unset;
+	u64 csum_validation_failed;
+	u64 csum_err_bad_buffer;
+	u64 csum_err_invalid_ip_version;
+	u64 csum_err_invalid_transport;
+	u64 csum_fragmented_pkt;
+	u64 csum_skipped;
+	u64 csum_sw;
+};
+
 struct rmnet_priv {
 	u8 mux_id;
 	struct net_device *real_dev;
 	struct rmnet_pcpu_stats __percpu *pcpu_stats;
 	struct gro_cells gro_cells;
+	struct rmnet_priv_stats stats;
 };
 
 struct rmnet_port *rmnet_get_port(struct net_device *real_dev);
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
index 6fcd586e9804..7fd86d40a337 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
@@ -148,7 +148,7 @@ static int rmnet_map_egress_handler(struct sk_buff *skb,
 
 	if (skb_headroom(skb) < required_headroom) {
 		if (pskb_expand_head(skb, required_headroom, 0, GFP_KERNEL))
-			goto fail;
+			return -ENOMEM;
 	}
 
 	if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV4)
@@ -156,17 +156,13 @@ static int rmnet_map_egress_handler(struct sk_buff *skb,
 
 	map_header = rmnet_map_add_map_header(skb, additional_header_len, 0);
 	if (!map_header)
-		goto fail;
+		return -ENOMEM;
 
 	map_header->mux_id = mux_id;
 
 	skb->protocol = htons(ETH_P_MAP);
 
 	return 0;
-
-fail:
-	kfree_skb(skb);
-	return -ENOMEM;
 }
 
 static void
@@ -228,15 +224,18 @@ void rmnet_egress_handler(struct sk_buff *skb)
 	mux_id = priv->mux_id;
 
 	port = rmnet_get_port(skb->dev);
-	if (!port) {
-		kfree_skb(skb);
-		return;
-	}
+	if (!port)
+		goto drop;
 
 	if (rmnet_map_egress_handler(skb, port, mux_id, orig_dev))
-		return;
+		goto drop;
 
 	rmnet_vnd_tx_fixup(skb, orig_dev);
 
 	dev_queue_xmit(skb);
+	return;
+
+drop:
+	this_cpu_inc(priv->pcpu_stats->stats.tx_drops);
+	kfree_skb(skb);
 }
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
index 78fdad0c6f76..56a93df962e6 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
@@ -69,17 +69,9 @@ static void rmnet_map_send_ack(struct sk_buff *skb,
 	struct rmnet_map_control_command *cmd;
 	int xmit_status;
 
-	if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) {
-		if (skb->len < sizeof(struct rmnet_map_header) +
-		    RMNET_MAP_GET_LENGTH(skb) +
-		    sizeof(struct rmnet_map_dl_csum_trailer)) {
-			kfree_skb(skb);
-			return;
-		}
-
-		skb_trim(skb, skb->len -
-			 sizeof(struct rmnet_map_dl_csum_trailer));
-	}
+	if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4)
+		skb_trim(skb,
+			 skb->len - sizeof(struct rmnet_map_dl_csum_trailer));
 
 	skb->protocol = htons(ETH_P_MAP);
 
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
index a6ea09416f8d..57a9c314a665 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
@@ -48,7 +48,8 @@ static __sum16 *rmnet_map_get_csum_field(unsigned char protocol,
 
 static int
 rmnet_map_ipv4_dl_csum_trailer(struct sk_buff *skb,
-			       struct rmnet_map_dl_csum_trailer *csum_trailer)
+			       struct rmnet_map_dl_csum_trailer *csum_trailer,
+			       struct rmnet_priv *priv)
 {
 	__sum16 *csum_field, csum_temp, pseudo_csum, hdr_csum, ip_payload_csum;
 	u16 csum_value, csum_value_final;
@@ -58,19 +59,25 @@ rmnet_map_ipv4_dl_csum_trailer(struct sk_buff *skb,
 
 	ip4h = (struct iphdr *)(skb->data);
 	if ((ntohs(ip4h->frag_off) & IP_MF) ||
-	    ((ntohs(ip4h->frag_off) & IP_OFFSET) > 0))
+	    ((ntohs(ip4h->frag_off) & IP_OFFSET) > 0)) {
+		priv->stats.csum_fragmented_pkt++;
 		return -EOPNOTSUPP;
+	}
 
 	txporthdr = skb->data + ip4h->ihl * 4;
 
 	csum_field = rmnet_map_get_csum_field(ip4h->protocol, txporthdr);
 
-	if (!csum_field)
+	if (!csum_field) {
+		priv->stats.csum_err_invalid_transport++;
 		return -EPROTONOSUPPORT;
+	}
 
 	/* RFC 768 - Skip IPv4 UDP packets where sender checksum field is 0 */
-	if (*csum_field == 0 && ip4h->protocol == IPPROTO_UDP)
+	if (*csum_field == 0 && ip4h->protocol == IPPROTO_UDP) {
+		priv->stats.csum_skipped++;
 		return 0;
+	}
 
 	csum_value = ~ntohs(csum_trailer->csum_value);
 	hdr_csum = ~ip_fast_csum(ip4h, (int)ip4h->ihl);
@@ -102,16 +109,20 @@ rmnet_map_ipv4_dl_csum_trailer(struct sk_buff *skb,
 		}
 	}
 
-	if (csum_value_final == ntohs((__force __be16)*csum_field))
+	if (csum_value_final == ntohs((__force __be16)*csum_field)) {
+		priv->stats.csum_ok++;
 		return 0;
-	else
+	} else {
+		priv->stats.csum_validation_failed++;
 		return -EINVAL;
+	}
 }
 
 #if IS_ENABLED(CONFIG_IPV6)
 static int
 rmnet_map_ipv6_dl_csum_trailer(struct sk_buff *skb,
-			       struct rmnet_map_dl_csum_trailer *csum_trailer)
+			       struct rmnet_map_dl_csum_trailer *csum_trailer,
+			       struct rmnet_priv *priv)
 {
 	__sum16 *csum_field, ip6_payload_csum, pseudo_csum, csum_temp;
 	u16 csum_value, csum_value_final;
@@ -125,8 +136,10 @@ rmnet_map_ipv6_dl_csum_trailer(struct sk_buff *skb,
 	txporthdr = skb->data + sizeof(struct ipv6hdr);
 	csum_field = rmnet_map_get_csum_field(ip6h->nexthdr, txporthdr);
 
-	if (!csum_field)
+	if (!csum_field) {
+		priv->stats.csum_err_invalid_transport++;
 		return -EPROTONOSUPPORT;
+	}
 
 	csum_value = ~ntohs(csum_trailer->csum_value);
 	ip6_hdr_csum = (__force __be16)
@@ -164,10 +177,13 @@ rmnet_map_ipv6_dl_csum_trailer(struct sk_buff *skb,
 		}
 	}
 
-	if (csum_value_final == ntohs((__force __be16)*csum_field))
+	if (csum_value_final == ntohs((__force __be16)*csum_field)) {
+		priv->stats.csum_ok++;
 		return 0;
-	else
+	} else {
+		priv->stats.csum_validation_failed++;
 		return -EINVAL;
+	}
 }
 #endif
 
@@ -339,24 +355,34 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb,
  */
 int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len)
 {
+	struct rmnet_priv *priv = netdev_priv(skb->dev);
 	struct rmnet_map_dl_csum_trailer *csum_trailer;
 
-	if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM)))
+	if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM))) {
+		priv->stats.csum_sw++;
 		return -EOPNOTSUPP;
+	}
 
 	csum_trailer = (struct rmnet_map_dl_csum_trailer *)(skb->data + len);
 
-	if (!csum_trailer->valid)
+	if (!csum_trailer->valid) {
+		priv->stats.csum_valid_unset++;
 		return -EINVAL;
+	}
 
-	if (skb->protocol == htons(ETH_P_IP))
-		return rmnet_map_ipv4_dl_csum_trailer(skb, csum_trailer);
-	else if (skb->protocol == htons(ETH_P_IPV6))
+	if (skb->protocol == htons(ETH_P_IP)) {
+		return rmnet_map_ipv4_dl_csum_trailer(skb, csum_trailer, priv);
+	} else if (skb->protocol == htons(ETH_P_IPV6)) {
 #if IS_ENABLED(CONFIG_IPV6)
-		return rmnet_map_ipv6_dl_csum_trailer(skb, csum_trailer);
+		return rmnet_map_ipv6_dl_csum_trailer(skb, csum_trailer, priv);
 #else
+		priv->stats.csum_err_invalid_ip_version++;
 		return -EPROTONOSUPPORT;
 #endif
+	} else {
+		priv->stats.csum_err_invalid_ip_version++;
+		return -EPROTONOSUPPORT;
+	}
 
 	return 0;
 }
@@ -367,6 +393,7 @@ int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len)
 void rmnet_map_checksum_uplink_packet(struct sk_buff *skb,
 				      struct net_device *orig_dev)
 {
+	struct rmnet_priv *priv = netdev_priv(orig_dev);
 	struct rmnet_map_ul_csum_header *ul_header;
 	void *iphdr;
 
@@ -389,8 +416,11 @@ void rmnet_map_checksum_uplink_packet(struct sk_buff *skb,
 			rmnet_map_ipv6_ul_csum_header(iphdr, ul_header, skb);
 			return;
 #else
+			priv->stats.csum_err_invalid_ip_version++;
 			goto sw_csum;
 #endif
+		} else {
+			priv->stats.csum_err_invalid_ip_version++;
 		}
 	}
 
@@ -399,4 +429,6 @@ void rmnet_map_checksum_uplink_packet(struct sk_buff *skb,
 	ul_header->csum_insert_offset = 0;
 	ul_header->csum_enabled = 0;
 	ul_header->udp_ip4_ind = 0;
+
+	priv->stats.csum_sw++;
 }
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
index 2ea16a088de8..cb02e1a015c1 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
@@ -152,6 +152,56 @@ static const struct net_device_ops rmnet_vnd_ops = {
 	.ndo_get_stats64 = rmnet_get_stats64,
 };
 
+static const char rmnet_gstrings_stats[][ETH_GSTRING_LEN] = {
+	"Checksum ok",
+	"Checksum valid bit not set",
+	"Checksum validation failed",
+	"Checksum error bad buffer",
+	"Checksum error bad ip version",
+	"Checksum error bad transport",
+	"Checksum skipped on ip fragment",
+	"Checksum skipped",
+	"Checksum computed in software",
+};
+
+static void rmnet_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
+{
+	switch (stringset) {
+	case ETH_SS_STATS:
+		memcpy(buf, &rmnet_gstrings_stats,
+		       sizeof(rmnet_gstrings_stats));
+		break;
+	}
+}
+
+static int rmnet_get_sset_count(struct net_device *dev, int sset)
+{
+	switch (sset) {
+	case ETH_SS_STATS:
+		return ARRAY_SIZE(rmnet_gstrings_stats);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void rmnet_get_ethtool_stats(struct net_device *dev,
+				    struct ethtool_stats *stats, u64 *data)
+{
+	struct rmnet_priv *priv = netdev_priv(dev);
+	struct rmnet_priv_stats *st = &priv->stats;
+
+	if (!data)
+		return;
+
+	memcpy(data, st, ARRAY_SIZE(rmnet_gstrings_stats) * sizeof(u64));
+}
+
+static const struct ethtool_ops rmnet_ethtool_ops = {
+	.get_ethtool_stats = rmnet_get_ethtool_stats,
+	.get_strings = rmnet_get_strings,
+	.get_sset_count = rmnet_get_sset_count,
+};
+
 /* Called by kernel whenever a new rmnet<n> device is created. Sets MTU,
  * flags, ARP type, needed headroom, etc...
  */
@@ -170,6 +220,7 @@ void rmnet_vnd_setup(struct net_device *rmnet_dev)
 	rmnet_dev->flags &= ~(IFF_BROADCAST | IFF_MULTICAST);
 
 	rmnet_dev->needs_free_netdev = true;
+	rmnet_dev->ethtool_ops = &rmnet_ethtool_ops;
 }
 
 /* Exposed API */
diff --git a/drivers/net/ethernet/realtek/8139too.c b/drivers/net/ethernet/realtek/8139too.c
index d118da5a10a2..ffd68a7bc9e1 100644
--- a/drivers/net/ethernet/realtek/8139too.c
+++ b/drivers/net/ethernet/realtek/8139too.c
@@ -1104,7 +1104,6 @@ static int rtl8139_init_one(struct pci_dev *pdev,
 	return 0;
 
 err_out:
-	netif_napi_del(&tp->napi);
 	__rtl8139_cleanup_dev (dev);
 	pci_disable_device (pdev);
 	return i;
@@ -1119,7 +1118,6 @@ static void rtl8139_remove_one(struct pci_dev *pdev)
 	assert (dev != NULL);
 
 	cancel_delayed_work_sync(&tp->thread);
-	netif_napi_del(&tp->napi);
 
 	unregister_netdev (dev);
 
diff --git a/drivers/net/ethernet/realtek/r8169.c b/drivers/net/ethernet/realtek/r8169.c
index c7aac1fc99e8..75dfac0248f4 100644
--- a/drivers/net/ethernet/realtek/r8169.c
+++ b/drivers/net/ethernet/realtek/r8169.c
@@ -84,12 +84,11 @@
    The RTL chips use a 64 element hash table based on the Ethernet CRC. */
 static const int multicast_filter_limit = 32;
 
-#define MAX_READ_REQUEST_SHIFT	12
 #define TX_DMA_BURST	7	/* Maximum PCI burst, '7' is unlimited */
 #define InterFrameGap	0x03	/* 3 means InterFrameGap = the shortest one */
 
 #define R8169_REGS_SIZE		256
-#define R8169_NAPI_WEIGHT	64
+#define R8169_RX_BUF_SIZE	(SZ_16K - 1)
 #define NUM_TX_DESC	64	/* Number of Tx descriptor registers */
 #define NUM_RX_DESC	256U	/* Number of Rx descriptor registers */
 #define R8169_TX_RING_BYTES	(NUM_TX_DESC * sizeof(struct TxDesc))
@@ -172,12 +171,11 @@ enum rtl_tx_desc_version {
 #define JUMBO_7K	(7*1024 - ETH_HLEN - 2)
 #define JUMBO_9K	(9*1024 - ETH_HLEN - 2)
 
-#define _R(NAME,TD,FW,SZ,B) {	\
+#define _R(NAME,TD,FW,SZ) {	\
 	.name = NAME,		\
 	.txd_version = TD,	\
 	.fw_name = FW,		\
 	.jumbo_max = SZ,	\
-	.jumbo_tx_csum = B	\
 }
 
 static const struct {
@@ -185,135 +183,111 @@ static const struct {
 	enum rtl_tx_desc_version txd_version;
 	const char *fw_name;
 	u16 jumbo_max;
-	bool jumbo_tx_csum;
 } rtl_chip_infos[] = {
 	/* PCI devices. */
 	[RTL_GIGA_MAC_VER_01] =
-		_R("RTL8169",		RTL_TD_0, NULL, JUMBO_7K, true),
+		_R("RTL8169",		RTL_TD_0, NULL, JUMBO_7K),
 	[RTL_GIGA_MAC_VER_02] =
-		_R("RTL8169s",		RTL_TD_0, NULL, JUMBO_7K, true),
+		_R("RTL8169s",		RTL_TD_0, NULL, JUMBO_7K),
 	[RTL_GIGA_MAC_VER_03] =
-		_R("RTL8110s",		RTL_TD_0, NULL, JUMBO_7K, true),
+		_R("RTL8110s",		RTL_TD_0, NULL, JUMBO_7K),
 	[RTL_GIGA_MAC_VER_04] =
-		_R("RTL8169sb/8110sb",	RTL_TD_0, NULL, JUMBO_7K, true),
+		_R("RTL8169sb/8110sb",	RTL_TD_0, NULL, JUMBO_7K),
 	[RTL_GIGA_MAC_VER_05] =
-		_R("RTL8169sc/8110sc",	RTL_TD_0, NULL, JUMBO_7K, true),
+		_R("RTL8169sc/8110sc",	RTL_TD_0, NULL, JUMBO_7K),
 	[RTL_GIGA_MAC_VER_06] =
-		_R("RTL8169sc/8110sc",	RTL_TD_0, NULL, JUMBO_7K, true),
+		_R("RTL8169sc/8110sc",	RTL_TD_0, NULL, JUMBO_7K),
 	/* PCI-E devices. */
 	[RTL_GIGA_MAC_VER_07] =
-		_R("RTL8102e",		RTL_TD_1, NULL, JUMBO_1K, true),
+		_R("RTL8102e",		RTL_TD_1, NULL, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_08] =
-		_R("RTL8102e",		RTL_TD_1, NULL, JUMBO_1K, true),
+		_R("RTL8102e",		RTL_TD_1, NULL, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_09] =
-		_R("RTL8102e",		RTL_TD_1, NULL, JUMBO_1K, true),
+		_R("RTL8102e",		RTL_TD_1, NULL, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_10] =
-		_R("RTL8101e",		RTL_TD_0, NULL, JUMBO_1K, true),
+		_R("RTL8101e",		RTL_TD_0, NULL, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_11] =
-		_R("RTL8168b/8111b",	RTL_TD_0, NULL, JUMBO_4K, false),
+		_R("RTL8168b/8111b",	RTL_TD_0, NULL, JUMBO_4K),
 	[RTL_GIGA_MAC_VER_12] =
-		_R("RTL8168b/8111b",	RTL_TD_0, NULL, JUMBO_4K, false),
+		_R("RTL8168b/8111b",	RTL_TD_0, NULL, JUMBO_4K),
 	[RTL_GIGA_MAC_VER_13] =
-		_R("RTL8101e",		RTL_TD_0, NULL, JUMBO_1K, true),
+		_R("RTL8101e",		RTL_TD_0, NULL, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_14] =
-		_R("RTL8100e",		RTL_TD_0, NULL, JUMBO_1K, true),
+		_R("RTL8100e",		RTL_TD_0, NULL, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_15] =
-		_R("RTL8100e",		RTL_TD_0, NULL, JUMBO_1K, true),
+		_R("RTL8100e",		RTL_TD_0, NULL, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_16] =
-		_R("RTL8101e",		RTL_TD_0, NULL, JUMBO_1K, true),
+		_R("RTL8101e",		RTL_TD_0, NULL, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_17] =
-		_R("RTL8168b/8111b",	RTL_TD_0, NULL, JUMBO_4K, false),
+		_R("RTL8168b/8111b",	RTL_TD_0, NULL, JUMBO_4K),
 	[RTL_GIGA_MAC_VER_18] =
-		_R("RTL8168cp/8111cp",	RTL_TD_1, NULL, JUMBO_6K, false),
+		_R("RTL8168cp/8111cp",	RTL_TD_1, NULL, JUMBO_6K),
 	[RTL_GIGA_MAC_VER_19] =
-		_R("RTL8168c/8111c",	RTL_TD_1, NULL, JUMBO_6K, false),
+		_R("RTL8168c/8111c",	RTL_TD_1, NULL, JUMBO_6K),
 	[RTL_GIGA_MAC_VER_20] =
-		_R("RTL8168c/8111c",	RTL_TD_1, NULL, JUMBO_6K, false),
+		_R("RTL8168c/8111c",	RTL_TD_1, NULL, JUMBO_6K),
 	[RTL_GIGA_MAC_VER_21] =
-		_R("RTL8168c/8111c",	RTL_TD_1, NULL, JUMBO_6K, false),
+		_R("RTL8168c/8111c",	RTL_TD_1, NULL, JUMBO_6K),
 	[RTL_GIGA_MAC_VER_22] =
-		_R("RTL8168c/8111c",	RTL_TD_1, NULL, JUMBO_6K, false),
+		_R("RTL8168c/8111c",	RTL_TD_1, NULL, JUMBO_6K),
 	[RTL_GIGA_MAC_VER_23] =
-		_R("RTL8168cp/8111cp",	RTL_TD_1, NULL, JUMBO_6K, false),
+		_R("RTL8168cp/8111cp",	RTL_TD_1, NULL, JUMBO_6K),
 	[RTL_GIGA_MAC_VER_24] =
-		_R("RTL8168cp/8111cp",	RTL_TD_1, NULL, JUMBO_6K, false),
+		_R("RTL8168cp/8111cp",	RTL_TD_1, NULL, JUMBO_6K),
 	[RTL_GIGA_MAC_VER_25] =
-		_R("RTL8168d/8111d",	RTL_TD_1, FIRMWARE_8168D_1,
-							JUMBO_9K, false),
+		_R("RTL8168d/8111d",	RTL_TD_1, FIRMWARE_8168D_1, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_26] =
-		_R("RTL8168d/8111d",	RTL_TD_1, FIRMWARE_8168D_2,
-							JUMBO_9K, false),
+		_R("RTL8168d/8111d",	RTL_TD_1, FIRMWARE_8168D_2, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_27] =
-		_R("RTL8168dp/8111dp",	RTL_TD_1, NULL, JUMBO_9K, false),
+		_R("RTL8168dp/8111dp",	RTL_TD_1, NULL, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_28] =
-		_R("RTL8168dp/8111dp",	RTL_TD_1, NULL, JUMBO_9K, false),
+		_R("RTL8168dp/8111dp",	RTL_TD_1, NULL, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_29] =
-		_R("RTL8105e",		RTL_TD_1, FIRMWARE_8105E_1,
-							JUMBO_1K, true),
+		_R("RTL8105e",		RTL_TD_1, FIRMWARE_8105E_1, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_30] =
-		_R("RTL8105e",		RTL_TD_1, FIRMWARE_8105E_1,
-							JUMBO_1K, true),
+		_R("RTL8105e",		RTL_TD_1, FIRMWARE_8105E_1, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_31] =
-		_R("RTL8168dp/8111dp",	RTL_TD_1, NULL, JUMBO_9K, false),
+		_R("RTL8168dp/8111dp",	RTL_TD_1, NULL, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_32] =
-		_R("RTL8168e/8111e",	RTL_TD_1, FIRMWARE_8168E_1,
-							JUMBO_9K, false),
+		_R("RTL8168e/8111e",	RTL_TD_1, FIRMWARE_8168E_1, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_33] =
-		_R("RTL8168e/8111e",	RTL_TD_1, FIRMWARE_8168E_2,
-							JUMBO_9K, false),
+		_R("RTL8168e/8111e",	RTL_TD_1, FIRMWARE_8168E_2, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_34] =
-		_R("RTL8168evl/8111evl",RTL_TD_1, FIRMWARE_8168E_3,
-							JUMBO_9K, false),
+		_R("RTL8168evl/8111evl",RTL_TD_1, FIRMWARE_8168E_3, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_35] =
-		_R("RTL8168f/8111f",	RTL_TD_1, FIRMWARE_8168F_1,
-							JUMBO_9K, false),
+		_R("RTL8168f/8111f",	RTL_TD_1, FIRMWARE_8168F_1, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_36] =
-		_R("RTL8168f/8111f",	RTL_TD_1, FIRMWARE_8168F_2,
-							JUMBO_9K, false),
+		_R("RTL8168f/8111f",	RTL_TD_1, FIRMWARE_8168F_2, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_37] =
-		_R("RTL8402",		RTL_TD_1, FIRMWARE_8402_1,
-							JUMBO_1K, true),
+		_R("RTL8402",		RTL_TD_1, FIRMWARE_8402_1,  JUMBO_1K),
 	[RTL_GIGA_MAC_VER_38] =
-		_R("RTL8411",		RTL_TD_1, FIRMWARE_8411_1,
-							JUMBO_9K, false),
+		_R("RTL8411",		RTL_TD_1, FIRMWARE_8411_1,  JUMBO_9K),
 	[RTL_GIGA_MAC_VER_39] =
-		_R("RTL8106e",		RTL_TD_1, FIRMWARE_8106E_1,
-							JUMBO_1K, true),
+		_R("RTL8106e",		RTL_TD_1, FIRMWARE_8106E_1, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_40] =
-		_R("RTL8168g/8111g",	RTL_TD_1, FIRMWARE_8168G_2,
-							JUMBO_9K, false),
+		_R("RTL8168g/8111g",	RTL_TD_1, FIRMWARE_8168G_2, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_41] =
-		_R("RTL8168g/8111g",	RTL_TD_1, NULL, JUMBO_9K, false),
+		_R("RTL8168g/8111g",	RTL_TD_1, NULL, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_42] =
-		_R("RTL8168g/8111g",	RTL_TD_1, FIRMWARE_8168G_3,
-							JUMBO_9K, false),
+		_R("RTL8168g/8111g",	RTL_TD_1, FIRMWARE_8168G_3, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_43] =
-		_R("RTL8106e",		RTL_TD_1, FIRMWARE_8106E_2,
-							JUMBO_1K, true),
+		_R("RTL8106e",		RTL_TD_1, FIRMWARE_8106E_2, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_44] =
-		_R("RTL8411",		RTL_TD_1, FIRMWARE_8411_2,
-							JUMBO_9K, false),
+		_R("RTL8411",		RTL_TD_1, FIRMWARE_8411_2,  JUMBO_9K),
 	[RTL_GIGA_MAC_VER_45] =
-		_R("RTL8168h/8111h",	RTL_TD_1, FIRMWARE_8168H_1,
-							JUMBO_9K, false),
+		_R("RTL8168h/8111h",	RTL_TD_1, FIRMWARE_8168H_1, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_46] =
-		_R("RTL8168h/8111h",	RTL_TD_1, FIRMWARE_8168H_2,
-							JUMBO_9K, false),
+		_R("RTL8168h/8111h",	RTL_TD_1, FIRMWARE_8168H_2, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_47] =
-		_R("RTL8107e",		RTL_TD_1, FIRMWARE_8107E_1,
-							JUMBO_1K, false),
+		_R("RTL8107e",		RTL_TD_1, FIRMWARE_8107E_1, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_48] =
-		_R("RTL8107e",		RTL_TD_1, FIRMWARE_8107E_2,
-							JUMBO_1K, false),
+		_R("RTL8107e",		RTL_TD_1, FIRMWARE_8107E_2, JUMBO_1K),
 	[RTL_GIGA_MAC_VER_49] =
-		_R("RTL8168ep/8111ep",	RTL_TD_1, NULL,
-							JUMBO_9K, false),
+		_R("RTL8168ep/8111ep",	RTL_TD_1, NULL, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_50] =
-		_R("RTL8168ep/8111ep",	RTL_TD_1, NULL,
-							JUMBO_9K, false),
+		_R("RTL8168ep/8111ep",	RTL_TD_1, NULL, JUMBO_9K),
 	[RTL_GIGA_MAC_VER_51] =
-		_R("RTL8168ep/8111ep",	RTL_TD_1, NULL,
-							JUMBO_9K, false),
+		_R("RTL8168ep/8111ep",	RTL_TD_1, NULL, JUMBO_9K),
 };
 #undef _R
 
@@ -345,7 +319,6 @@ static const struct pci_device_id rtl8169_pci_tbl[] = {
 
 MODULE_DEVICE_TABLE(pci, rtl8169_pci_tbl);
 
-static int rx_buf_sz = 16383;
 static int use_dac = -1;
 static struct {
 	u32 msg_enable;
@@ -437,13 +410,8 @@ enum rtl8168_8101_registers {
 	CSIAR			= 0x68,
 #define	CSIAR_FLAG			0x80000000
 #define	CSIAR_WRITE_CMD			0x80000000
-#define	CSIAR_BYTE_ENABLE		0x0f
-#define	CSIAR_BYTE_ENABLE_SHIFT		12
-#define	CSIAR_ADDR_MASK			0x0fff
-#define CSIAR_FUNC_CARD			0x00000000
-#define CSIAR_FUNC_SDIO			0x00010000
-#define CSIAR_FUNC_NIC			0x00020000
-#define CSIAR_FUNC_NIC2			0x00010000
+#define	CSIAR_BYTE_ENABLE		0x0000f000
+#define	CSIAR_ADDR_MASK			0x00000fff
 	PMCH			= 0x6f,
 	EPHYAR			= 0x80,
 #define	EPHYAR_FLAG			0x80000000
@@ -626,6 +594,7 @@ enum rtl_register_content {
 	RxChkSum	= (1 << 5),
 	PCIDAC		= (1 << 4),
 	PCIMulRW	= (1 << 3),
+#define INTT_MASK	GENMASK(1, 0)
 	INTT_0		= 0x0000,	// 8168
 	INTT_1		= 0x0001,	// 8168
 	INTT_2		= 0x0002,	// 8168
@@ -716,6 +685,7 @@ enum rtl_rx_desc_bit {
 };
 
 #define RsvdMask	0x3fffc000
+#define CPCMD_QUIRK_MASK	(Normal_mode | RxVlan | RxChkSum | INTT_MASK)
 
 struct TxDesc {
 	__le32 opts1;
@@ -778,7 +748,6 @@ struct rtl8169_private {
 	struct net_device *dev;
 	struct napi_struct napi;
 	u32 msg_enable;
-	u16 txd_version;
 	u16 mac_version;
 	u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */
 	u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */
@@ -802,26 +771,16 @@ struct rtl8169_private {
 		int (*read)(struct rtl8169_private *, int);
 	} mdio_ops;
 
-	struct pll_power_ops {
-		void (*down)(struct rtl8169_private *);
-		void (*up)(struct rtl8169_private *);
-	} pll_power_ops;
-
 	struct jumbo_ops {
 		void (*enable)(struct rtl8169_private *);
 		void (*disable)(struct rtl8169_private *);
 	} jumbo_ops;
 
-	struct csi_ops {
-		void (*write)(struct rtl8169_private *, int, int);
-		u32 (*read)(struct rtl8169_private *, int);
-	} csi_ops;
-
 	int (*set_speed)(struct net_device *, u8 aneg, u16 sp, u8 dpx, u32 adv);
 	int (*get_link_ksettings)(struct net_device *,
 				  struct ethtool_link_ksettings *);
 	void (*phy_reset_enable)(struct rtl8169_private *tp);
-	void (*hw_start)(struct net_device *);
+	void (*hw_start)(struct rtl8169_private *tp);
 	unsigned int (*phy_reset_pending)(struct rtl8169_private *tp);
 	unsigned int (*link_ok)(struct rtl8169_private *tp);
 	int (*do_ioctl)(struct rtl8169_private *tp, struct mii_ioctl_data *data, int cmd);
@@ -833,14 +792,11 @@ struct rtl8169_private {
 		struct work_struct work;
 	} wk;
 
-	unsigned features;
-
 	struct mii_if_info mii;
 	dma_addr_t counters_phys_addr;
 	struct rtl8169_counters *counters;
 	struct rtl8169_tc_offsets tc_offset;
 	u32 saved_wolopts;
-	u32 opts1_mask;
 
 	struct rtl_fw {
 		const struct firmware *fw;
@@ -1645,23 +1601,8 @@ static u32 __rtl8169_get_wol(struct rtl8169_private *tp)
 	if (options & LinkUp)
 		wolopts |= WAKE_PHY;
 	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_34:
-	case RTL_GIGA_MAC_VER_35:
-	case RTL_GIGA_MAC_VER_36:
-	case RTL_GIGA_MAC_VER_37:
-	case RTL_GIGA_MAC_VER_38:
-	case RTL_GIGA_MAC_VER_40:
-	case RTL_GIGA_MAC_VER_41:
-	case RTL_GIGA_MAC_VER_42:
-	case RTL_GIGA_MAC_VER_43:
-	case RTL_GIGA_MAC_VER_44:
-	case RTL_GIGA_MAC_VER_45:
-	case RTL_GIGA_MAC_VER_46:
-	case RTL_GIGA_MAC_VER_47:
-	case RTL_GIGA_MAC_VER_48:
-	case RTL_GIGA_MAC_VER_49:
-	case RTL_GIGA_MAC_VER_50:
-	case RTL_GIGA_MAC_VER_51:
+	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_38:
+	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_51:
 		if (rtl_eri_read(tp, 0xdc, ERIAR_EXGMAC) & MagicPacket_v2)
 			wolopts |= WAKE_MAGIC;
 		break;
@@ -1722,23 +1663,8 @@ static void __rtl8169_set_wol(struct rtl8169_private *tp, u32 wolopts)
 	RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
 
 	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_34:
-	case RTL_GIGA_MAC_VER_35:
-	case RTL_GIGA_MAC_VER_36:
-	case RTL_GIGA_MAC_VER_37:
-	case RTL_GIGA_MAC_VER_38:
-	case RTL_GIGA_MAC_VER_40:
-	case RTL_GIGA_MAC_VER_41:
-	case RTL_GIGA_MAC_VER_42:
-	case RTL_GIGA_MAC_VER_43:
-	case RTL_GIGA_MAC_VER_44:
-	case RTL_GIGA_MAC_VER_45:
-	case RTL_GIGA_MAC_VER_46:
-	case RTL_GIGA_MAC_VER_47:
-	case RTL_GIGA_MAC_VER_48:
-	case RTL_GIGA_MAC_VER_49:
-	case RTL_GIGA_MAC_VER_50:
-	case RTL_GIGA_MAC_VER_51:
+	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_38:
+	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_51:
 		tmp = ARRAY_SIZE(cfg) - 1;
 		if (wolopts & WAKE_MAGIC)
 			rtl_w0w1_eri(tp,
@@ -1960,18 +1886,20 @@ static netdev_features_t rtl8169_fix_features(struct net_device *dev,
 		features &= ~NETIF_F_ALL_TSO;
 
 	if (dev->mtu > JUMBO_1K &&
-	    !rtl_chip_infos[tp->mac_version].jumbo_tx_csum)
+	    tp->mac_version > RTL_GIGA_MAC_VER_06)
 		features &= ~NETIF_F_IP_CSUM;
 
 	return features;
 }
 
-static void __rtl8169_set_features(struct net_device *dev,
-				   netdev_features_t features)
+static int rtl8169_set_features(struct net_device *dev,
+				netdev_features_t features)
 {
 	struct rtl8169_private *tp = netdev_priv(dev);
 	u32 rx_config;
 
+	rtl_lock_work(tp);
+
 	rx_config = RTL_R32(tp, RxConfig);
 	if (features & NETIF_F_RXALL)
 		rx_config |= (AcceptErr | AcceptRunt);
@@ -1990,28 +1918,14 @@ static void __rtl8169_set_features(struct net_device *dev,
 	else
 		tp->cp_cmd &= ~RxVlan;
 
-	tp->cp_cmd |= RTL_R16(tp, CPlusCmd) & ~(RxVlan | RxChkSum);
-
 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
 	RTL_R16(tp, CPlusCmd);
-}
-
-static int rtl8169_set_features(struct net_device *dev,
-				netdev_features_t features)
-{
-	struct rtl8169_private *tp = netdev_priv(dev);
-
-	features &= NETIF_F_RXALL | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_RX;
 
-	rtl_lock_work(tp);
-	if (features ^ dev->features)
-		__rtl8169_set_features(dev, features);
 	rtl_unlock_work(tp);
 
 	return 0;
 }
 
-
 static inline u32 rtl8169_tx_vlan_tag(struct sk_buff *skb)
 {
 	return (skb_vlan_tag_present(skb)) ?
@@ -2155,9 +2069,8 @@ DECLARE_RTL_COND(rtl_counters_cond)
 	return RTL_R32(tp, CounterAddrLow) & (CounterReset | CounterDump);
 }
 
-static bool rtl8169_do_counters(struct net_device *dev, u32 counter_cmd)
+static bool rtl8169_do_counters(struct rtl8169_private *tp, u32 counter_cmd)
 {
-	struct rtl8169_private *tp = netdev_priv(dev);
 	dma_addr_t paddr = tp->counters_phys_addr;
 	u32 cmd;
 
@@ -2170,10 +2083,8 @@ static bool rtl8169_do_counters(struct net_device *dev, u32 counter_cmd)
 	return rtl_udelay_loop_wait_low(tp, &rtl_counters_cond, 10, 1000);
 }
 
-static bool rtl8169_reset_counters(struct net_device *dev)
+static bool rtl8169_reset_counters(struct rtl8169_private *tp)
 {
-	struct rtl8169_private *tp = netdev_priv(dev);
-
 	/*
 	 * Versions prior to RTL_GIGA_MAC_VER_19 don't support resetting the
 	 * tally counters.
@@ -2181,13 +2092,11 @@ static bool rtl8169_reset_counters(struct net_device *dev)
 	if (tp->mac_version < RTL_GIGA_MAC_VER_19)
 		return true;
 
-	return rtl8169_do_counters(dev, CounterReset);
+	return rtl8169_do_counters(tp, CounterReset);
 }
 
-static bool rtl8169_update_counters(struct net_device *dev)
+static bool rtl8169_update_counters(struct rtl8169_private *tp)
 {
-	struct rtl8169_private *tp = netdev_priv(dev);
-
 	/*
 	 * Some chips are unable to dump tally counters when the receiver
 	 * is disabled.
@@ -2195,12 +2104,11 @@ static bool rtl8169_update_counters(struct net_device *dev)
 	if ((RTL_R8(tp, ChipCmd) & CmdRxEnb) == 0)
 		return true;
 
-	return rtl8169_do_counters(dev, CounterDump);
+	return rtl8169_do_counters(tp, CounterDump);
 }
 
-static bool rtl8169_init_counter_offsets(struct net_device *dev)
+static bool rtl8169_init_counter_offsets(struct rtl8169_private *tp)
 {
-	struct rtl8169_private *tp = netdev_priv(dev);
 	struct rtl8169_counters *counters = tp->counters;
 	bool ret = false;
 
@@ -2223,10 +2131,10 @@ static bool rtl8169_init_counter_offsets(struct net_device *dev)
 		return true;
 
 	/* If both, reset and update fail, propagate to caller. */
-	if (rtl8169_reset_counters(dev))
+	if (rtl8169_reset_counters(tp))
 		ret = true;
 
-	if (rtl8169_update_counters(dev))
+	if (rtl8169_update_counters(tp))
 		ret = true;
 
 	tp->tc_offset.tx_errors = counters->tx_errors;
@@ -2249,7 +2157,7 @@ static void rtl8169_get_ethtool_stats(struct net_device *dev,
 	pm_runtime_get_noresume(d);
 
 	if (pm_runtime_active(d))
-		rtl8169_update_counters(dev);
+		rtl8169_update_counters(tp);
 
 	pm_runtime_put_noidle(d);
 
@@ -2391,7 +2299,7 @@ static int rtl_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
 	if (IS_ERR(ci))
 		return PTR_ERR(ci);
 
-	scale = &ci->scalev[RTL_R16(tp, CPlusCmd) & 3];
+	scale = &ci->scalev[tp->cp_cmd & INTT_MASK];
 
 	/* read IntrMitigate and adjust according to scale */
 	for (w = RTL_R16(tp, IntrMitigate); w; w >>= RTL_COALESCE_SHIFT, p++) {
@@ -2490,7 +2398,7 @@ static int rtl_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
 
 	RTL_W16(tp, IntrMitigate, swab16(w));
 
-	tp->cp_cmd = (tp->cp_cmd & ~3) | cp01;
+	tp->cp_cmd = (tp->cp_cmd & ~INTT_MASK) | cp01;
 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
 	RTL_R16(tp, CPlusCmd);
 
@@ -2520,7 +2428,7 @@ static const struct ethtool_ops rtl8169_ethtool_ops = {
 };
 
 static void rtl8169_get_mac_version(struct rtl8169_private *tp,
-				    struct net_device *dev, u8 default_version)
+				    u8 default_version)
 {
 	/*
 	 * The driver currently handles the 8168Bf and the 8168Be identically
@@ -2560,12 +2468,10 @@ static void rtl8169_get_mac_version(struct rtl8169_private *tp,
 
 		/* 8168E family. */
 		{ 0x7c800000, 0x2c800000,	RTL_GIGA_MAC_VER_34 },
-		{ 0x7cf00000, 0x2c200000,	RTL_GIGA_MAC_VER_33 },
 		{ 0x7cf00000, 0x2c100000,	RTL_GIGA_MAC_VER_32 },
 		{ 0x7c800000, 0x2c000000,	RTL_GIGA_MAC_VER_33 },
 
 		/* 8168D family. */
-		{ 0x7cf00000, 0x28300000,	RTL_GIGA_MAC_VER_26 },
 		{ 0x7cf00000, 0x28100000,	RTL_GIGA_MAC_VER_25 },
 		{ 0x7c800000, 0x28000000,	RTL_GIGA_MAC_VER_26 },
 
@@ -2575,32 +2481,24 @@ static void rtl8169_get_mac_version(struct rtl8169_private *tp,
 		{ 0x7cf00000, 0x28b00000,	RTL_GIGA_MAC_VER_31 },
 
 		/* 8168C family. */
-		{ 0x7cf00000, 0x3cb00000,	RTL_GIGA_MAC_VER_24 },
 		{ 0x7cf00000, 0x3c900000,	RTL_GIGA_MAC_VER_23 },
 		{ 0x7cf00000, 0x3c800000,	RTL_GIGA_MAC_VER_18 },
 		{ 0x7c800000, 0x3c800000,	RTL_GIGA_MAC_VER_24 },
 		{ 0x7cf00000, 0x3c000000,	RTL_GIGA_MAC_VER_19 },
 		{ 0x7cf00000, 0x3c200000,	RTL_GIGA_MAC_VER_20 },
 		{ 0x7cf00000, 0x3c300000,	RTL_GIGA_MAC_VER_21 },
-		{ 0x7cf00000, 0x3c400000,	RTL_GIGA_MAC_VER_22 },
 		{ 0x7c800000, 0x3c000000,	RTL_GIGA_MAC_VER_22 },
 
 		/* 8168B family. */
 		{ 0x7cf00000, 0x38000000,	RTL_GIGA_MAC_VER_12 },
-		{ 0x7cf00000, 0x38500000,	RTL_GIGA_MAC_VER_17 },
 		{ 0x7c800000, 0x38000000,	RTL_GIGA_MAC_VER_17 },
 		{ 0x7c800000, 0x30000000,	RTL_GIGA_MAC_VER_11 },
 
 		/* 8101 family. */
-		{ 0x7cf00000, 0x44900000,	RTL_GIGA_MAC_VER_39 },
 		{ 0x7c800000, 0x44800000,	RTL_GIGA_MAC_VER_39 },
 		{ 0x7c800000, 0x44000000,	RTL_GIGA_MAC_VER_37 },
-		{ 0x7cf00000, 0x40b00000,	RTL_GIGA_MAC_VER_30 },
-		{ 0x7cf00000, 0x40a00000,	RTL_GIGA_MAC_VER_30 },
 		{ 0x7cf00000, 0x40900000,	RTL_GIGA_MAC_VER_29 },
 		{ 0x7c800000, 0x40800000,	RTL_GIGA_MAC_VER_30 },
-		{ 0x7cf00000, 0x34a00000,	RTL_GIGA_MAC_VER_09 },
-		{ 0x7cf00000, 0x24a00000,	RTL_GIGA_MAC_VER_09 },
 		{ 0x7cf00000, 0x34900000,	RTL_GIGA_MAC_VER_08 },
 		{ 0x7cf00000, 0x24900000,	RTL_GIGA_MAC_VER_08 },
 		{ 0x7cf00000, 0x34800000,	RTL_GIGA_MAC_VER_07 },
@@ -2635,8 +2533,8 @@ static void rtl8169_get_mac_version(struct rtl8169_private *tp,
 	tp->mac_version = p->mac_version;
 
 	if (tp->mac_version == RTL_GIGA_MAC_NONE) {
-		netif_notice(tp, probe, dev,
-			     "unknown MAC, using family default\n");
+		dev_notice(tp_to_dev(tp),
+			   "unknown MAC, using family default\n");
 		tp->mac_version = default_version;
 	} else if (tp->mac_version == RTL_GIGA_MAC_VER_42) {
 		tp->mac_version = tp->mii.supports_gmii ?
@@ -4685,18 +4583,7 @@ static void rtl_init_mdio_ops(struct rtl8169_private *tp)
 		ops->write	= r8168dp_2_mdio_write;
 		ops->read	= r8168dp_2_mdio_read;
 		break;
-	case RTL_GIGA_MAC_VER_40:
-	case RTL_GIGA_MAC_VER_41:
-	case RTL_GIGA_MAC_VER_42:
-	case RTL_GIGA_MAC_VER_43:
-	case RTL_GIGA_MAC_VER_44:
-	case RTL_GIGA_MAC_VER_45:
-	case RTL_GIGA_MAC_VER_46:
-	case RTL_GIGA_MAC_VER_47:
-	case RTL_GIGA_MAC_VER_48:
-	case RTL_GIGA_MAC_VER_49:
-	case RTL_GIGA_MAC_VER_50:
-	case RTL_GIGA_MAC_VER_51:
+	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_51:
 		ops->write	= r8168g_mdio_write;
 		ops->read	= r8168g_mdio_read;
 		break;
@@ -4741,21 +4628,7 @@ static void rtl_wol_suspend_quirk(struct rtl8169_private *tp)
 	case RTL_GIGA_MAC_VER_32:
 	case RTL_GIGA_MAC_VER_33:
 	case RTL_GIGA_MAC_VER_34:
-	case RTL_GIGA_MAC_VER_37:
-	case RTL_GIGA_MAC_VER_38:
-	case RTL_GIGA_MAC_VER_39:
-	case RTL_GIGA_MAC_VER_40:
-	case RTL_GIGA_MAC_VER_41:
-	case RTL_GIGA_MAC_VER_42:
-	case RTL_GIGA_MAC_VER_43:
-	case RTL_GIGA_MAC_VER_44:
-	case RTL_GIGA_MAC_VER_45:
-	case RTL_GIGA_MAC_VER_46:
-	case RTL_GIGA_MAC_VER_47:
-	case RTL_GIGA_MAC_VER_48:
-	case RTL_GIGA_MAC_VER_49:
-	case RTL_GIGA_MAC_VER_50:
-	case RTL_GIGA_MAC_VER_51:
+	case RTL_GIGA_MAC_VER_37 ... RTL_GIGA_MAC_VER_51:
 		RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) |
 			AcceptBroadcast | AcceptMulticast | AcceptMyPhys);
 		break;
@@ -4775,79 +4648,13 @@ static bool rtl_wol_pll_power_down(struct rtl8169_private *tp)
 	return true;
 }
 
-static void r810x_phy_power_down(struct rtl8169_private *tp)
-{
-	rtl_writephy(tp, 0x1f, 0x0000);
-	rtl_writephy(tp, MII_BMCR, BMCR_PDOWN);
-}
-
-static void r810x_phy_power_up(struct rtl8169_private *tp)
-{
-	rtl_writephy(tp, 0x1f, 0x0000);
-	rtl_writephy(tp, MII_BMCR, BMCR_ANENABLE);
-}
-
-static void r810x_pll_power_down(struct rtl8169_private *tp)
-{
-	if (rtl_wol_pll_power_down(tp))
-		return;
-
-	r810x_phy_power_down(tp);
-
-	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_07:
-	case RTL_GIGA_MAC_VER_08:
-	case RTL_GIGA_MAC_VER_09:
-	case RTL_GIGA_MAC_VER_10:
-	case RTL_GIGA_MAC_VER_13:
-	case RTL_GIGA_MAC_VER_16:
-		break;
-	default:
-		RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) & ~0x80);
-		break;
-	}
-}
-
-static void r810x_pll_power_up(struct rtl8169_private *tp)
-{
-	r810x_phy_power_up(tp);
-
-	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_07:
-	case RTL_GIGA_MAC_VER_08:
-	case RTL_GIGA_MAC_VER_09:
-	case RTL_GIGA_MAC_VER_10:
-	case RTL_GIGA_MAC_VER_13:
-	case RTL_GIGA_MAC_VER_16:
-		break;
-	case RTL_GIGA_MAC_VER_47:
-	case RTL_GIGA_MAC_VER_48:
-		RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | 0xc0);
-		break;
-	default:
-		RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | 0x80);
-		break;
-	}
-}
-
 static void r8168_phy_power_up(struct rtl8169_private *tp)
 {
 	rtl_writephy(tp, 0x1f, 0x0000);
 	switch (tp->mac_version) {
 	case RTL_GIGA_MAC_VER_11:
 	case RTL_GIGA_MAC_VER_12:
-	case RTL_GIGA_MAC_VER_17:
-	case RTL_GIGA_MAC_VER_18:
-	case RTL_GIGA_MAC_VER_19:
-	case RTL_GIGA_MAC_VER_20:
-	case RTL_GIGA_MAC_VER_21:
-	case RTL_GIGA_MAC_VER_22:
-	case RTL_GIGA_MAC_VER_23:
-	case RTL_GIGA_MAC_VER_24:
-	case RTL_GIGA_MAC_VER_25:
-	case RTL_GIGA_MAC_VER_26:
-	case RTL_GIGA_MAC_VER_27:
-	case RTL_GIGA_MAC_VER_28:
+	case RTL_GIGA_MAC_VER_17 ... RTL_GIGA_MAC_VER_28:
 	case RTL_GIGA_MAC_VER_31:
 		rtl_writephy(tp, 0x0e, 0x0000);
 		break;
@@ -4855,6 +4662,9 @@ static void r8168_phy_power_up(struct rtl8169_private *tp)
 		break;
 	}
 	rtl_writephy(tp, MII_BMCR, BMCR_ANENABLE);
+
+	/* give MAC/PHY some time to resume */
+	msleep(20);
 }
 
 static void r8168_phy_power_down(struct rtl8169_private *tp)
@@ -4870,18 +4680,7 @@ static void r8168_phy_power_down(struct rtl8169_private *tp)
 
 	case RTL_GIGA_MAC_VER_11:
 	case RTL_GIGA_MAC_VER_12:
-	case RTL_GIGA_MAC_VER_17:
-	case RTL_GIGA_MAC_VER_18:
-	case RTL_GIGA_MAC_VER_19:
-	case RTL_GIGA_MAC_VER_20:
-	case RTL_GIGA_MAC_VER_21:
-	case RTL_GIGA_MAC_VER_22:
-	case RTL_GIGA_MAC_VER_23:
-	case RTL_GIGA_MAC_VER_24:
-	case RTL_GIGA_MAC_VER_25:
-	case RTL_GIGA_MAC_VER_26:
-	case RTL_GIGA_MAC_VER_27:
-	case RTL_GIGA_MAC_VER_28:
+	case RTL_GIGA_MAC_VER_17 ... RTL_GIGA_MAC_VER_28:
 	case RTL_GIGA_MAC_VER_31:
 		rtl_writephy(tp, 0x0e, 0x0200);
 	default:
@@ -4895,12 +4694,6 @@ static void r8168_pll_power_down(struct rtl8169_private *tp)
 	if (r8168_check_dash(tp))
 		return;
 
-	if ((tp->mac_version == RTL_GIGA_MAC_VER_23 ||
-	     tp->mac_version == RTL_GIGA_MAC_VER_24) &&
-	    (RTL_R16(tp, CPlusCmd) & ASF)) {
-		return;
-	}
-
 	if (tp->mac_version == RTL_GIGA_MAC_VER_32 ||
 	    tp->mac_version == RTL_GIGA_MAC_VER_33)
 		rtl_ephy_write(tp, 0x19, 0xff64);
@@ -4911,16 +4704,15 @@ static void r8168_pll_power_down(struct rtl8169_private *tp)
 	r8168_phy_power_down(tp);
 
 	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_25:
-	case RTL_GIGA_MAC_VER_26:
-	case RTL_GIGA_MAC_VER_27:
-	case RTL_GIGA_MAC_VER_28:
-	case RTL_GIGA_MAC_VER_31:
-	case RTL_GIGA_MAC_VER_32:
-	case RTL_GIGA_MAC_VER_33:
+	case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_33:
+	case RTL_GIGA_MAC_VER_37:
+	case RTL_GIGA_MAC_VER_39:
+	case RTL_GIGA_MAC_VER_43:
 	case RTL_GIGA_MAC_VER_44:
 	case RTL_GIGA_MAC_VER_45:
 	case RTL_GIGA_MAC_VER_46:
+	case RTL_GIGA_MAC_VER_47:
+	case RTL_GIGA_MAC_VER_48:
 	case RTL_GIGA_MAC_VER_50:
 	case RTL_GIGA_MAC_VER_51:
 		RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) & ~0x80);
@@ -4938,18 +4730,17 @@ static void r8168_pll_power_down(struct rtl8169_private *tp)
 static void r8168_pll_power_up(struct rtl8169_private *tp)
 {
 	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_25:
-	case RTL_GIGA_MAC_VER_26:
-	case RTL_GIGA_MAC_VER_27:
-	case RTL_GIGA_MAC_VER_28:
-	case RTL_GIGA_MAC_VER_31:
-	case RTL_GIGA_MAC_VER_32:
-	case RTL_GIGA_MAC_VER_33:
+	case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_33:
+	case RTL_GIGA_MAC_VER_37:
+	case RTL_GIGA_MAC_VER_39:
+	case RTL_GIGA_MAC_VER_43:
 		RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | 0x80);
 		break;
 	case RTL_GIGA_MAC_VER_44:
 	case RTL_GIGA_MAC_VER_45:
 	case RTL_GIGA_MAC_VER_46:
+	case RTL_GIGA_MAC_VER_47:
+	case RTL_GIGA_MAC_VER_48:
 	case RTL_GIGA_MAC_VER_50:
 	case RTL_GIGA_MAC_VER_51:
 		RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | 0xc0);
@@ -4966,130 +4757,41 @@ static void r8168_pll_power_up(struct rtl8169_private *tp)
 	r8168_phy_power_up(tp);
 }
 
-static void rtl_generic_op(struct rtl8169_private *tp,
-			   void (*op)(struct rtl8169_private *))
-{
-	if (op)
-		op(tp);
-}
-
 static void rtl_pll_power_down(struct rtl8169_private *tp)
 {
-	rtl_generic_op(tp, tp->pll_power_ops.down);
+	switch (tp->mac_version) {
+	case RTL_GIGA_MAC_VER_01 ... RTL_GIGA_MAC_VER_06:
+	case RTL_GIGA_MAC_VER_13 ... RTL_GIGA_MAC_VER_15:
+		break;
+	default:
+		r8168_pll_power_down(tp);
+	}
 }
 
 static void rtl_pll_power_up(struct rtl8169_private *tp)
 {
-	rtl_generic_op(tp, tp->pll_power_ops.up);
-
-	/* give MAC/PHY some time to resume */
-	msleep(20);
-}
-
-static void rtl_init_pll_power_ops(struct rtl8169_private *tp)
-{
-	struct pll_power_ops *ops = &tp->pll_power_ops;
-
 	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_07:
-	case RTL_GIGA_MAC_VER_08:
-	case RTL_GIGA_MAC_VER_09:
-	case RTL_GIGA_MAC_VER_10:
-	case RTL_GIGA_MAC_VER_16:
-	case RTL_GIGA_MAC_VER_29:
-	case RTL_GIGA_MAC_VER_30:
-	case RTL_GIGA_MAC_VER_37:
-	case RTL_GIGA_MAC_VER_39:
-	case RTL_GIGA_MAC_VER_43:
-	case RTL_GIGA_MAC_VER_47:
-	case RTL_GIGA_MAC_VER_48:
-		ops->down	= r810x_pll_power_down;
-		ops->up		= r810x_pll_power_up;
+	case RTL_GIGA_MAC_VER_01 ... RTL_GIGA_MAC_VER_06:
+	case RTL_GIGA_MAC_VER_13 ... RTL_GIGA_MAC_VER_15:
 		break;
-
-	case RTL_GIGA_MAC_VER_11:
-	case RTL_GIGA_MAC_VER_12:
-	case RTL_GIGA_MAC_VER_17:
-	case RTL_GIGA_MAC_VER_18:
-	case RTL_GIGA_MAC_VER_19:
-	case RTL_GIGA_MAC_VER_20:
-	case RTL_GIGA_MAC_VER_21:
-	case RTL_GIGA_MAC_VER_22:
-	case RTL_GIGA_MAC_VER_23:
-	case RTL_GIGA_MAC_VER_24:
-	case RTL_GIGA_MAC_VER_25:
-	case RTL_GIGA_MAC_VER_26:
-	case RTL_GIGA_MAC_VER_27:
-	case RTL_GIGA_MAC_VER_28:
-	case RTL_GIGA_MAC_VER_31:
-	case RTL_GIGA_MAC_VER_32:
-	case RTL_GIGA_MAC_VER_33:
-	case RTL_GIGA_MAC_VER_34:
-	case RTL_GIGA_MAC_VER_35:
-	case RTL_GIGA_MAC_VER_36:
-	case RTL_GIGA_MAC_VER_38:
-	case RTL_GIGA_MAC_VER_40:
-	case RTL_GIGA_MAC_VER_41:
-	case RTL_GIGA_MAC_VER_42:
-	case RTL_GIGA_MAC_VER_44:
-	case RTL_GIGA_MAC_VER_45:
-	case RTL_GIGA_MAC_VER_46:
-	case RTL_GIGA_MAC_VER_49:
-	case RTL_GIGA_MAC_VER_50:
-	case RTL_GIGA_MAC_VER_51:
-		ops->down	= r8168_pll_power_down;
-		ops->up		= r8168_pll_power_up;
-		break;
-
 	default:
-		ops->down	= NULL;
-		ops->up		= NULL;
-		break;
+		r8168_pll_power_up(tp);
 	}
 }
 
 static void rtl_init_rxcfg(struct rtl8169_private *tp)
 {
 	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_01:
-	case RTL_GIGA_MAC_VER_02:
-	case RTL_GIGA_MAC_VER_03:
-	case RTL_GIGA_MAC_VER_04:
-	case RTL_GIGA_MAC_VER_05:
-	case RTL_GIGA_MAC_VER_06:
-	case RTL_GIGA_MAC_VER_10:
-	case RTL_GIGA_MAC_VER_11:
-	case RTL_GIGA_MAC_VER_12:
-	case RTL_GIGA_MAC_VER_13:
-	case RTL_GIGA_MAC_VER_14:
-	case RTL_GIGA_MAC_VER_15:
-	case RTL_GIGA_MAC_VER_16:
-	case RTL_GIGA_MAC_VER_17:
+	case RTL_GIGA_MAC_VER_01 ... RTL_GIGA_MAC_VER_06:
+	case RTL_GIGA_MAC_VER_10 ... RTL_GIGA_MAC_VER_17:
 		RTL_W32(tp, RxConfig, RX_FIFO_THRESH | RX_DMA_BURST);
 		break;
-	case RTL_GIGA_MAC_VER_18:
-	case RTL_GIGA_MAC_VER_19:
-	case RTL_GIGA_MAC_VER_20:
-	case RTL_GIGA_MAC_VER_21:
-	case RTL_GIGA_MAC_VER_22:
-	case RTL_GIGA_MAC_VER_23:
-	case RTL_GIGA_MAC_VER_24:
+	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_24:
 	case RTL_GIGA_MAC_VER_34:
 	case RTL_GIGA_MAC_VER_35:
 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST);
 		break;
-	case RTL_GIGA_MAC_VER_40:
-	case RTL_GIGA_MAC_VER_41:
-	case RTL_GIGA_MAC_VER_42:
-	case RTL_GIGA_MAC_VER_43:
-	case RTL_GIGA_MAC_VER_44:
-	case RTL_GIGA_MAC_VER_45:
-	case RTL_GIGA_MAC_VER_46:
-	case RTL_GIGA_MAC_VER_47:
-	case RTL_GIGA_MAC_VER_48:
-	case RTL_GIGA_MAC_VER_49:
-	case RTL_GIGA_MAC_VER_50:
-	case RTL_GIGA_MAC_VER_51:
+	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_51:
 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST | RX_EARLY_OFF);
 		break;
 	default:
@@ -5105,16 +4807,20 @@ static void rtl8169_init_ring_indexes(struct rtl8169_private *tp)
 
 static void rtl_hw_jumbo_enable(struct rtl8169_private *tp)
 {
-	RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
-	rtl_generic_op(tp, tp->jumbo_ops.enable);
-	RTL_W8(tp, Cfg9346, Cfg9346_Lock);
+	if (tp->jumbo_ops.enable) {
+		RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
+		tp->jumbo_ops.enable(tp);
+		RTL_W8(tp, Cfg9346, Cfg9346_Lock);
+	}
 }
 
 static void rtl_hw_jumbo_disable(struct rtl8169_private *tp)
 {
-	RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
-	rtl_generic_op(tp, tp->jumbo_ops.disable);
-	RTL_W8(tp, Cfg9346, Cfg9346_Lock);
+	if (tp->jumbo_ops.disable) {
+		RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
+		tp->jumbo_ops.disable(tp);
+		RTL_W8(tp, Cfg9346, Cfg9346_Lock);
+	}
 }
 
 static void r8168c_hw_jumbo_enable(struct rtl8169_private *tp)
@@ -5128,7 +4834,7 @@ static void r8168c_hw_jumbo_disable(struct rtl8169_private *tp)
 {
 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~Jumbo_En1);
-	rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+	rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 }
 
 static void r8168dp_hw_jumbo_enable(struct rtl8169_private *tp)
@@ -5154,7 +4860,7 @@ static void r8168e_hw_jumbo_disable(struct rtl8169_private *tp)
 	RTL_W8(tp, MaxTxPacketSize, 0x0c);
 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~0x01);
-	rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+	rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 }
 
 static void r8168b_0_hw_jumbo_enable(struct rtl8169_private *tp)
@@ -5166,7 +4872,7 @@ static void r8168b_0_hw_jumbo_enable(struct rtl8169_private *tp)
 static void r8168b_0_hw_jumbo_disable(struct rtl8169_private *tp)
 {
 	rtl_tx_performance_tweak(tp,
-		(0x5 << MAX_READ_REQUEST_SHIFT) | PCI_EXP_DEVCTL_NOSNOOP_EN);
+		PCI_EXP_DEVCTL_READRQ_4096B | PCI_EXP_DEVCTL_NOSNOOP_EN);
 }
 
 static void r8168b_1_hw_jumbo_enable(struct rtl8169_private *tp)
@@ -5226,18 +4932,7 @@ static void rtl_init_jumbo_ops(struct rtl8169_private *tp)
 	 * No action needed for jumbo frames with 8169.
 	 * No jumbo for 810x at all.
 	 */
-	case RTL_GIGA_MAC_VER_40:
-	case RTL_GIGA_MAC_VER_41:
-	case RTL_GIGA_MAC_VER_42:
-	case RTL_GIGA_MAC_VER_43:
-	case RTL_GIGA_MAC_VER_44:
-	case RTL_GIGA_MAC_VER_45:
-	case RTL_GIGA_MAC_VER_46:
-	case RTL_GIGA_MAC_VER_47:
-	case RTL_GIGA_MAC_VER_48:
-	case RTL_GIGA_MAC_VER_49:
-	case RTL_GIGA_MAC_VER_50:
-	case RTL_GIGA_MAC_VER_51:
+	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_51:
 	default:
 		ops->disable	= NULL;
 		ops->enable	= NULL;
@@ -5323,32 +5018,21 @@ static void rtl8169_hw_reset(struct rtl8169_private *tp)
 
 	rtl_rx_close(tp);
 
-	if (tp->mac_version == RTL_GIGA_MAC_VER_27 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_28 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_31) {
+	switch (tp->mac_version) {
+	case RTL_GIGA_MAC_VER_27:
+	case RTL_GIGA_MAC_VER_28:
+	case RTL_GIGA_MAC_VER_31:
 		rtl_udelay_loop_wait_low(tp, &rtl_npq_cond, 20, 42*42);
-	} else if (tp->mac_version == RTL_GIGA_MAC_VER_34 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_35 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_36 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_37 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_38 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_40 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_41 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_42 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_43 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_44 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_45 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_46 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_47 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_48 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_49 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_50 ||
-		   tp->mac_version == RTL_GIGA_MAC_VER_51) {
+		break;
+	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_38:
+	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_51:
 		RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
 		rtl_udelay_loop_wait_high(tp, &rtl_txcfg_empty_cond, 100, 666);
-	} else {
+		break;
+	default:
 		RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
 		udelay(100);
+		break;
 	}
 
 	rtl_hw_reset(tp);
@@ -5361,13 +5045,10 @@ static void rtl_set_rx_tx_config_registers(struct rtl8169_private *tp)
 		(InterFrameGap << TxInterFrameGapShift));
 }
 
-static void rtl_hw_start(struct net_device *dev)
+static void rtl_set_rx_max_size(struct rtl8169_private *tp)
 {
-	struct rtl8169_private *tp = netdev_priv(dev);
-
-	tp->hw_start(dev);
-
-	rtl_irq_enable_all(tp);
+	/* Low hurts. Let's disable the filtering. */
+	RTL_W16(tp, RxMaxSize, R8169_RX_BUF_SIZE + 1);
 }
 
 static void rtl_set_rx_tx_desc_registers(struct rtl8169_private *tp)
@@ -5383,21 +5064,6 @@ static void rtl_set_rx_tx_desc_registers(struct rtl8169_private *tp)
 	RTL_W32(tp, RxDescAddrLow, ((u64) tp->RxPhyAddr) & DMA_BIT_MASK(32));
 }
 
-static u16 rtl_rw_cpluscmd(struct rtl8169_private *tp)
-{
-	u16 cmd;
-
-	cmd = RTL_R16(tp, CPlusCmd);
-	RTL_W16(tp, CPlusCmd, cmd);
-	return cmd;
-}
-
-static void rtl_set_rx_max_size(struct rtl8169_private *tp, unsigned int rx_buf_sz)
-{
-	/* Low hurts. Let's disable the filtering. */
-	RTL_W16(tp, RxMaxSize, rx_buf_sz + 1);
-}
-
 static void rtl8169_set_magic_reg(struct rtl8169_private *tp, unsigned mac_version)
 {
 	static const struct rtl_cfg2_info {
@@ -5475,36 +5141,34 @@ static void rtl_set_rx_mode(struct net_device *dev)
 	RTL_W32(tp, RxConfig, tmp);
 }
 
-static void rtl_hw_start_8169(struct net_device *dev)
+static void rtl_hw_start(struct  rtl8169_private *tp)
 {
-	struct rtl8169_private *tp = netdev_priv(dev);
-	struct pci_dev *pdev = tp->pci_dev;
-
-	if (tp->mac_version == RTL_GIGA_MAC_VER_05) {
-		RTL_W16(tp, CPlusCmd, RTL_R16(tp, CPlusCmd) | PCIMulRW);
-		pci_write_config_byte(pdev, PCI_CACHE_LINE_SIZE, 0x08);
-	}
-
 	RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
-	if (tp->mac_version == RTL_GIGA_MAC_VER_01 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_02 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_03 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_04)
-		RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb);
 
-	rtl_init_rxcfg(tp);
+	tp->hw_start(tp);
 
-	RTL_W8(tp, EarlyTxThres, NoEarlyTx);
+	rtl_set_rx_max_size(tp);
+	rtl_set_rx_tx_desc_registers(tp);
+	rtl_set_rx_tx_config_registers(tp);
+	RTL_W8(tp, Cfg9346, Cfg9346_Lock);
+
+	/* Initially a 10 us delay. Turned it into a PCI commit. - FR */
+	RTL_R8(tp, IntrMask);
+	RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb);
+	rtl_set_rx_mode(tp->dev);
+	/* no early-rx interrupts */
+	RTL_W16(tp, MultiIntr, RTL_R16(tp, MultiIntr) & 0xf000);
+	rtl_irq_enable_all(tp);
+}
 
-	rtl_set_rx_max_size(tp, rx_buf_sz);
+static void rtl_hw_start_8169(struct rtl8169_private *tp)
+{
+	if (tp->mac_version == RTL_GIGA_MAC_VER_05)
+		pci_write_config_byte(tp->pci_dev, PCI_CACHE_LINE_SIZE, 0x08);
 
-	if (tp->mac_version == RTL_GIGA_MAC_VER_01 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_02 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_03 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_04)
-		rtl_set_rx_tx_config_registers(tp);
+	RTL_W8(tp, EarlyTxThres, NoEarlyTx);
 
-	tp->cp_cmd |= rtl_rw_cpluscmd(tp) | PCIMulRW;
+	tp->cp_cmd |= PCIMulRW;
 
 	if (tp->mac_version == RTL_GIGA_MAC_VER_02 ||
 	    tp->mac_version == RTL_GIGA_MAC_VER_03) {
@@ -5523,56 +5187,7 @@ static void rtl_hw_start_8169(struct net_device *dev)
 	 */
 	RTL_W16(tp, IntrMitigate, 0x0000);
 
-	rtl_set_rx_tx_desc_registers(tp);
-
-	if (tp->mac_version != RTL_GIGA_MAC_VER_01 &&
-	    tp->mac_version != RTL_GIGA_MAC_VER_02 &&
-	    tp->mac_version != RTL_GIGA_MAC_VER_03 &&
-	    tp->mac_version != RTL_GIGA_MAC_VER_04) {
-		RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb);
-		rtl_set_rx_tx_config_registers(tp);
-	}
-
-	RTL_W8(tp, Cfg9346, Cfg9346_Lock);
-
-	/* Initially a 10 us delay. Turned it into a PCI commit. - FR */
-	RTL_R8(tp, IntrMask);
-
 	RTL_W32(tp, RxMissed, 0);
-
-	rtl_set_rx_mode(dev);
-
-	/* no early-rx interrupts */
-	RTL_W16(tp, MultiIntr, RTL_R16(tp, MultiIntr) & 0xf000);
-}
-
-static void rtl_csi_write(struct rtl8169_private *tp, int addr, int value)
-{
-	if (tp->csi_ops.write)
-		tp->csi_ops.write(tp, addr, value);
-}
-
-static u32 rtl_csi_read(struct rtl8169_private *tp, int addr)
-{
-	return tp->csi_ops.read ? tp->csi_ops.read(tp, addr) : ~0;
-}
-
-static void rtl_csi_access_enable(struct rtl8169_private *tp, u32 bits)
-{
-	u32 csi;
-
-	csi = rtl_csi_read(tp, 0x070c) & 0x00ffffff;
-	rtl_csi_write(tp, 0x070c, csi | bits);
-}
-
-static void rtl_csi_access_enable_1(struct rtl8169_private *tp)
-{
-	rtl_csi_access_enable(tp, 0x17000000);
-}
-
-static void rtl_csi_access_enable_2(struct rtl8169_private *tp)
-{
-	rtl_csi_access_enable(tp, 0x27000000);
 }
 
 DECLARE_RTL_COND(rtl_csiar_cond)
@@ -5580,101 +5195,55 @@ DECLARE_RTL_COND(rtl_csiar_cond)
 	return RTL_R32(tp, CSIAR) & CSIAR_FLAG;
 }
 
-static void r8169_csi_write(struct rtl8169_private *tp, int addr, int value)
-{
-	RTL_W32(tp, CSIDR, value);
-	RTL_W32(tp, CSIAR, CSIAR_WRITE_CMD | (addr & CSIAR_ADDR_MASK) |
-		CSIAR_BYTE_ENABLE << CSIAR_BYTE_ENABLE_SHIFT);
-
-	rtl_udelay_loop_wait_low(tp, &rtl_csiar_cond, 10, 100);
-}
-
-static u32 r8169_csi_read(struct rtl8169_private *tp, int addr)
+static void rtl_csi_write(struct rtl8169_private *tp, int addr, int value)
 {
-	RTL_W32(tp, CSIAR, (addr & CSIAR_ADDR_MASK) |
-		CSIAR_BYTE_ENABLE << CSIAR_BYTE_ENABLE_SHIFT);
-
-	return rtl_udelay_loop_wait_high(tp, &rtl_csiar_cond, 10, 100) ?
-		RTL_R32(tp, CSIDR) : ~0;
-}
+	u32 func = PCI_FUNC(tp->pci_dev->devfn);
 
-static void r8402_csi_write(struct rtl8169_private *tp, int addr, int value)
-{
 	RTL_W32(tp, CSIDR, value);
 	RTL_W32(tp, CSIAR, CSIAR_WRITE_CMD | (addr & CSIAR_ADDR_MASK) |
-		CSIAR_BYTE_ENABLE << CSIAR_BYTE_ENABLE_SHIFT |
-		CSIAR_FUNC_NIC);
+		CSIAR_BYTE_ENABLE | func << 16);
 
 	rtl_udelay_loop_wait_low(tp, &rtl_csiar_cond, 10, 100);
 }
 
-static u32 r8402_csi_read(struct rtl8169_private *tp, int addr)
+static u32 rtl_csi_read(struct rtl8169_private *tp, int addr)
 {
-	RTL_W32(tp, CSIAR, (addr & CSIAR_ADDR_MASK) | CSIAR_FUNC_NIC |
-		CSIAR_BYTE_ENABLE << CSIAR_BYTE_ENABLE_SHIFT);
+	u32 func = PCI_FUNC(tp->pci_dev->devfn);
+
+	RTL_W32(tp, CSIAR, (addr & CSIAR_ADDR_MASK) | func << 16 |
+		CSIAR_BYTE_ENABLE);
 
 	return rtl_udelay_loop_wait_high(tp, &rtl_csiar_cond, 10, 100) ?
 		RTL_R32(tp, CSIDR) : ~0;
 }
 
-static void r8411_csi_write(struct rtl8169_private *tp, int addr, int value)
+static void rtl_csi_access_enable(struct rtl8169_private *tp, u8 val)
 {
-	RTL_W32(tp, CSIDR, value);
-	RTL_W32(tp, CSIAR, CSIAR_WRITE_CMD | (addr & CSIAR_ADDR_MASK) |
-		CSIAR_BYTE_ENABLE << CSIAR_BYTE_ENABLE_SHIFT |
-		CSIAR_FUNC_NIC2);
+	struct pci_dev *pdev = tp->pci_dev;
+	u32 csi;
 
-	rtl_udelay_loop_wait_low(tp, &rtl_csiar_cond, 10, 100);
+	/* According to Realtek the value at config space address 0x070f
+	 * controls the L0s/L1 entrance latency. We try standard ECAM access
+	 * first and if it fails fall back to CSI.
+	 */
+	if (pdev->cfg_size > 0x070f &&
+	    pci_write_config_byte(pdev, 0x070f, val) == PCIBIOS_SUCCESSFUL)
+		return;
+
+	netdev_notice_once(tp->dev,
+		"No native access to PCI extended config space, falling back to CSI\n");
+	csi = rtl_csi_read(tp, 0x070c) & 0x00ffffff;
+	rtl_csi_write(tp, 0x070c, csi | val << 24);
 }
 
-static u32 r8411_csi_read(struct rtl8169_private *tp, int addr)
+static void rtl_csi_access_enable_1(struct rtl8169_private *tp)
 {
-	RTL_W32(tp, CSIAR, (addr & CSIAR_ADDR_MASK) | CSIAR_FUNC_NIC2 |
-		CSIAR_BYTE_ENABLE << CSIAR_BYTE_ENABLE_SHIFT);
-
-	return rtl_udelay_loop_wait_high(tp, &rtl_csiar_cond, 10, 100) ?
-		RTL_R32(tp, CSIDR) : ~0;
+	rtl_csi_access_enable(tp, 0x17);
 }
 
-static void rtl_init_csi_ops(struct rtl8169_private *tp)
+static void rtl_csi_access_enable_2(struct rtl8169_private *tp)
 {
-	struct csi_ops *ops = &tp->csi_ops;
-
-	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_01:
-	case RTL_GIGA_MAC_VER_02:
-	case RTL_GIGA_MAC_VER_03:
-	case RTL_GIGA_MAC_VER_04:
-	case RTL_GIGA_MAC_VER_05:
-	case RTL_GIGA_MAC_VER_06:
-	case RTL_GIGA_MAC_VER_10:
-	case RTL_GIGA_MAC_VER_11:
-	case RTL_GIGA_MAC_VER_12:
-	case RTL_GIGA_MAC_VER_13:
-	case RTL_GIGA_MAC_VER_14:
-	case RTL_GIGA_MAC_VER_15:
-	case RTL_GIGA_MAC_VER_16:
-	case RTL_GIGA_MAC_VER_17:
-		ops->write	= NULL;
-		ops->read	= NULL;
-		break;
-
-	case RTL_GIGA_MAC_VER_37:
-	case RTL_GIGA_MAC_VER_38:
-		ops->write	= r8402_csi_write;
-		ops->read	= r8402_csi_read;
-		break;
-
-	case RTL_GIGA_MAC_VER_44:
-		ops->write	= r8411_csi_write;
-		ops->read	= r8411_csi_read;
-		break;
-
-	default:
-		ops->write	= r8169_csi_write;
-		ops->read	= r8169_csi_read;
-		break;
-	}
+	rtl_csi_access_enable(tp, 0x27);
 }
 
 struct ephy_info {
@@ -5721,25 +5290,15 @@ static void rtl_pcie_state_l2l3_enable(struct rtl8169_private *tp, bool enable)
 	RTL_W8(tp, Config3, data);
 }
 
-#define R8168_CPCMD_QUIRK_MASK (\
-	EnableBist | \
-	Mac_dbgo_oe | \
-	Force_half_dup | \
-	Force_rxflow_en | \
-	Force_txflow_en | \
-	Cxpl_dbg_sel | \
-	ASF | \
-	PktCntrDisable | \
-	Mac_dbgo_sel)
-
 static void rtl_hw_start_8168bb(struct rtl8169_private *tp)
 {
 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
 
-	RTL_W16(tp, CPlusCmd, RTL_R16(tp, CPlusCmd) & ~R8168_CPCMD_QUIRK_MASK);
+	tp->cp_cmd &= CPCMD_QUIRK_MASK;
+	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
 
 	if (tp->dev->mtu <= ETH_DATA_LEN) {
-		rtl_tx_performance_tweak(tp, (0x5 << MAX_READ_REQUEST_SHIFT) |
+		rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B |
 					 PCI_EXP_DEVCTL_NOSNOOP_EN);
 	}
 }
@@ -5760,11 +5319,12 @@ static void __rtl_hw_start_8168cp(struct rtl8169_private *tp)
 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
 
 	if (tp->dev->mtu <= ETH_DATA_LEN)
-		rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+		rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	rtl_disable_clock_request(tp);
 
-	RTL_W16(tp, CPlusCmd, RTL_R16(tp, CPlusCmd) & ~R8168_CPCMD_QUIRK_MASK);
+	tp->cp_cmd &= CPCMD_QUIRK_MASK;
+	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
 }
 
 static void rtl_hw_start_8168cp_1(struct rtl8169_private *tp)
@@ -5791,9 +5351,10 @@ static void rtl_hw_start_8168cp_2(struct rtl8169_private *tp)
 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
 
 	if (tp->dev->mtu <= ETH_DATA_LEN)
-		rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+		rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
-	RTL_W16(tp, CPlusCmd, RTL_R16(tp, CPlusCmd) & ~R8168_CPCMD_QUIRK_MASK);
+	tp->cp_cmd &= CPCMD_QUIRK_MASK;
+	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
 }
 
 static void rtl_hw_start_8168cp_3(struct rtl8169_private *tp)
@@ -5808,9 +5369,10 @@ static void rtl_hw_start_8168cp_3(struct rtl8169_private *tp)
 	RTL_W8(tp, MaxTxPacketSize, TxPacketMax);
 
 	if (tp->dev->mtu <= ETH_DATA_LEN)
-		rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+		rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
-	RTL_W16(tp, CPlusCmd, RTL_R16(tp, CPlusCmd) & ~R8168_CPCMD_QUIRK_MASK);
+	tp->cp_cmd &= CPCMD_QUIRK_MASK;
+	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
 }
 
 static void rtl_hw_start_8168c_1(struct rtl8169_private *tp)
@@ -5865,9 +5427,10 @@ static void rtl_hw_start_8168d(struct rtl8169_private *tp)
 	RTL_W8(tp, MaxTxPacketSize, TxPacketMax);
 
 	if (tp->dev->mtu <= ETH_DATA_LEN)
-		rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+		rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
-	RTL_W16(tp, CPlusCmd, RTL_R16(tp, CPlusCmd) & ~R8168_CPCMD_QUIRK_MASK);
+	tp->cp_cmd &= CPCMD_QUIRK_MASK;
+	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
 }
 
 static void rtl_hw_start_8168dp(struct rtl8169_private *tp)
@@ -5875,7 +5438,7 @@ static void rtl_hw_start_8168dp(struct rtl8169_private *tp)
 	rtl_csi_access_enable_1(tp);
 
 	if (tp->dev->mtu <= ETH_DATA_LEN)
-		rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+		rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	RTL_W8(tp, MaxTxPacketSize, TxPacketMax);
 
@@ -5892,7 +5455,7 @@ static void rtl_hw_start_8168d_4(struct rtl8169_private *tp)
 
 	rtl_csi_access_enable_1(tp);
 
-	rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+	rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	RTL_W8(tp, MaxTxPacketSize, TxPacketMax);
 
@@ -5924,7 +5487,7 @@ static void rtl_hw_start_8168e_1(struct rtl8169_private *tp)
 	rtl_ephy_init(tp, e_info_8168e_1, ARRAY_SIZE(e_info_8168e_1));
 
 	if (tp->dev->mtu <= ETH_DATA_LEN)
-		rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+		rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	RTL_W8(tp, MaxTxPacketSize, TxPacketMax);
 
@@ -5949,7 +5512,7 @@ static void rtl_hw_start_8168e_2(struct rtl8169_private *tp)
 	rtl_ephy_init(tp, e_info_8168e_2, ARRAY_SIZE(e_info_8168e_2));
 
 	if (tp->dev->mtu <= ETH_DATA_LEN)
-		rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+		rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000, ERIAR_EXGMAC);
 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000, ERIAR_EXGMAC);
@@ -5979,7 +5542,7 @@ static void rtl_hw_start_8168f(struct rtl8169_private *tp)
 {
 	rtl_csi_access_enable_2(tp);
 
-	rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+	rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000, ERIAR_EXGMAC);
 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000, ERIAR_EXGMAC);
@@ -6050,7 +5613,7 @@ static void rtl_hw_start_8168g(struct rtl8169_private *tp)
 
 	rtl_csi_access_enable_1(tp);
 
-	rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+	rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	rtl_w0w1_eri(tp, 0xdc, ERIAR_MASK_0001, 0x00, 0x01, ERIAR_EXGMAC);
 	rtl_w0w1_eri(tp, 0xdc, ERIAR_MASK_0001, 0x01, 0x00, ERIAR_EXGMAC);
@@ -6150,7 +5713,7 @@ static void rtl_hw_start_8168h_1(struct rtl8169_private *tp)
 
 	rtl_csi_access_enable_1(tp);
 
-	rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+	rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	rtl_w0w1_eri(tp, 0xdc, ERIAR_MASK_0001, 0x00, 0x01, ERIAR_EXGMAC);
 	rtl_w0w1_eri(tp, 0xdc, ERIAR_MASK_0001, 0x01, 0x00, ERIAR_EXGMAC);
@@ -6232,7 +5795,7 @@ static void rtl_hw_start_8168ep(struct rtl8169_private *tp)
 
 	rtl_csi_access_enable_1(tp);
 
-	rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+	rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	rtl_w0w1_eri(tp, 0xdc, ERIAR_MASK_0001, 0x00, 0x01, ERIAR_EXGMAC);
 	rtl_w0w1_eri(tp, 0xdc, ERIAR_MASK_0001, 0x01, 0x00, ERIAR_EXGMAC);
@@ -6328,18 +5891,12 @@ static void rtl_hw_start_8168ep_3(struct rtl8169_private *tp)
 	r8168_mac_ocp_write(tp, 0xe860, data);
 }
 
-static void rtl_hw_start_8168(struct net_device *dev)
+static void rtl_hw_start_8168(struct rtl8169_private *tp)
 {
-	struct rtl8169_private *tp = netdev_priv(dev);
-
-	RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
-
 	RTL_W8(tp, MaxTxPacketSize, TxPacketMax);
 
-	rtl_set_rx_max_size(tp, rx_buf_sz);
-
-	tp->cp_cmd |= RTL_R16(tp, CPlusCmd) | PktCntrDisable | INTT_1;
-
+	tp->cp_cmd &= ~INTT_MASK;
+	tp->cp_cmd |= PktCntrDisable | INTT_1;
 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
 
 	RTL_W16(tp, IntrMitigate, 0x5151);
@@ -6350,12 +5907,6 @@ static void rtl_hw_start_8168(struct net_device *dev)
 		tp->event_slow &= ~RxOverflow;
 	}
 
-	rtl_set_rx_tx_desc_registers(tp);
-
-	rtl_set_rx_tx_config_registers(tp);
-
-	RTL_R8(tp, IntrMask);
-
 	switch (tp->mac_version) {
 	case RTL_GIGA_MAC_VER_11:
 		rtl_hw_start_8168bb(tp);
@@ -6456,30 +6007,11 @@ static void rtl_hw_start_8168(struct net_device *dev)
 
 	default:
 		printk(KERN_ERR PFX "%s: unknown chipset (mac_version = %d).\n",
-			dev->name, tp->mac_version);
+		       tp->dev->name, tp->mac_version);
 		break;
 	}
-
-	RTL_W8(tp, Cfg9346, Cfg9346_Lock);
-
-	RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb);
-
-	rtl_set_rx_mode(dev);
-
-	RTL_W16(tp, MultiIntr, RTL_R16(tp, MultiIntr) & 0xf000);
 }
 
-#define R810X_CPCMD_QUIRK_MASK (\
-	EnableBist | \
-	Mac_dbgo_oe | \
-	Force_half_dup | \
-	Force_rxflow_en | \
-	Force_txflow_en | \
-	Cxpl_dbg_sel | \
-	ASF | \
-	PktCntrDisable | \
-	Mac_dbgo_sel)
-
 static void rtl_hw_start_8102e_1(struct rtl8169_private *tp)
 {
 	static const struct ephy_info e_info_8102e_1[] = {
@@ -6498,7 +6030,7 @@ static void rtl_hw_start_8102e_1(struct rtl8169_private *tp)
 
 	RTL_W8(tp, DBG_REG, FIX_NAK_1);
 
-	rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+	rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	RTL_W8(tp, Config1,
 	       LEDS1 | LEDS0 | Speed_down | MEMMAP | IOMAP | VPD | PMEnable);
@@ -6515,7 +6047,7 @@ static void rtl_hw_start_8102e_2(struct rtl8169_private *tp)
 {
 	rtl_csi_access_enable_2(tp);
 
-	rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+	rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	RTL_W8(tp, Config1, MEMMAP | IOMAP | VPD | PMEnable);
 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
@@ -6578,7 +6110,7 @@ static void rtl_hw_start_8402(struct rtl8169_private *tp)
 
 	rtl_ephy_init(tp, e_info_8402, ARRAY_SIZE(e_info_8402));
 
-	rtl_tx_performance_tweak(tp, 0x5 << MAX_READ_REQUEST_SHIFT);
+	rtl_tx_performance_tweak(tp, PCI_EXP_DEVCTL_READRQ_4096B);
 
 	rtl_eri_write(tp, 0xc8, ERIAR_MASK_1111, 0x00000002, ERIAR_EXGMAC);
 	rtl_eri_write(tp, 0xe8, ERIAR_MASK_1111, 0x00000006, ERIAR_EXGMAC);
@@ -6603,32 +6135,21 @@ static void rtl_hw_start_8106(struct rtl8169_private *tp)
 	rtl_pcie_state_l2l3_enable(tp, false);
 }
 
-static void rtl_hw_start_8101(struct net_device *dev)
+static void rtl_hw_start_8101(struct rtl8169_private *tp)
 {
-	struct rtl8169_private *tp = netdev_priv(dev);
-	struct pci_dev *pdev = tp->pci_dev;
-
 	if (tp->mac_version >= RTL_GIGA_MAC_VER_30)
 		tp->event_slow &= ~RxFIFOOver;
 
 	if (tp->mac_version == RTL_GIGA_MAC_VER_13 ||
 	    tp->mac_version == RTL_GIGA_MAC_VER_16)
-		pcie_capability_set_word(pdev, PCI_EXP_DEVCTL,
+		pcie_capability_set_word(tp->pci_dev, PCI_EXP_DEVCTL,
 					 PCI_EXP_DEVCTL_NOSNOOP_EN);
 
-	RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
-
 	RTL_W8(tp, MaxTxPacketSize, TxPacketMax);
 
-	rtl_set_rx_max_size(tp, rx_buf_sz);
-
-	tp->cp_cmd &= ~R810X_CPCMD_QUIRK_MASK;
+	tp->cp_cmd &= CPCMD_QUIRK_MASK;
 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
 
-	rtl_set_rx_tx_desc_registers(tp);
-
-	rtl_set_rx_tx_config_registers(tp);
-
 	switch (tp->mac_version) {
 	case RTL_GIGA_MAC_VER_07:
 		rtl_hw_start_8102e_1(tp);
@@ -6665,17 +6186,7 @@ static void rtl_hw_start_8101(struct net_device *dev)
 		break;
 	}
 
-	RTL_W8(tp, Cfg9346, Cfg9346_Lock);
-
 	RTL_W16(tp, IntrMitigate, 0x0000);
-
-	RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb);
-
-	rtl_set_rx_mode(dev);
-
-	RTL_R8(tp, IntrMask);
-
-	RTL_W16(tp, MultiIntr, RTL_R16(tp, MultiIntr) & 0xf000);
 }
 
 static int rtl8169_change_mtu(struct net_device *dev, int new_mtu)
@@ -6702,29 +6213,22 @@ static inline void rtl8169_make_unusable_by_asic(struct RxDesc *desc)
 static void rtl8169_free_rx_databuff(struct rtl8169_private *tp,
 				     void **data_buff, struct RxDesc *desc)
 {
-	dma_unmap_single(tp_to_dev(tp), le64_to_cpu(desc->addr), rx_buf_sz,
-			 DMA_FROM_DEVICE);
+	dma_unmap_single(tp_to_dev(tp), le64_to_cpu(desc->addr),
+			 R8169_RX_BUF_SIZE, DMA_FROM_DEVICE);
 
 	kfree(*data_buff);
 	*data_buff = NULL;
 	rtl8169_make_unusable_by_asic(desc);
 }
 
-static inline void rtl8169_mark_to_asic(struct RxDesc *desc, u32 rx_buf_sz)
+static inline void rtl8169_mark_to_asic(struct RxDesc *desc)
 {
 	u32 eor = le32_to_cpu(desc->opts1) & RingEnd;
 
 	/* Force memory writes to complete before releasing descriptor */
 	dma_wmb();
 
-	desc->opts1 = cpu_to_le32(DescOwn | eor | rx_buf_sz);
-}
-
-static inline void rtl8169_map_to_asic(struct RxDesc *desc, dma_addr_t mapping,
-				       u32 rx_buf_sz)
-{
-	desc->addr = cpu_to_le64(mapping);
-	rtl8169_mark_to_asic(desc, rx_buf_sz);
+	desc->opts1 = cpu_to_le32(DescOwn | eor | R8169_RX_BUF_SIZE);
 }
 
 static inline void *rtl8169_align(void *data)
@@ -6738,21 +6242,20 @@ static struct sk_buff *rtl8169_alloc_rx_data(struct rtl8169_private *tp,
 	void *data;
 	dma_addr_t mapping;
 	struct device *d = tp_to_dev(tp);
-	struct net_device *dev = tp->dev;
-	int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
+	int node = dev_to_node(d);
 
-	data = kmalloc_node(rx_buf_sz, GFP_KERNEL, node);
+	data = kmalloc_node(R8169_RX_BUF_SIZE, GFP_KERNEL, node);
 	if (!data)
 		return NULL;
 
 	if (rtl8169_align(data) != data) {
 		kfree(data);
-		data = kmalloc_node(rx_buf_sz + 15, GFP_KERNEL, node);
+		data = kmalloc_node(R8169_RX_BUF_SIZE + 15, GFP_KERNEL, node);
 		if (!data)
 			return NULL;
 	}
 
-	mapping = dma_map_single(d, rtl8169_align(data), rx_buf_sz,
+	mapping = dma_map_single(d, rtl8169_align(data), R8169_RX_BUF_SIZE,
 				 DMA_FROM_DEVICE);
 	if (unlikely(dma_mapping_error(d, mapping))) {
 		if (net_ratelimit())
@@ -6760,7 +6263,8 @@ static struct sk_buff *rtl8169_alloc_rx_data(struct rtl8169_private *tp,
 		goto err_out;
 	}
 
-	rtl8169_map_to_asic(desc, mapping, rx_buf_sz);
+	desc->addr = cpu_to_le64(mapping);
+	rtl8169_mark_to_asic(desc);
 	return data;
 
 err_out:
@@ -6792,9 +6296,6 @@ static int rtl8169_rx_fill(struct rtl8169_private *tp)
 	for (i = 0; i < NUM_RX_DESC; i++) {
 		void *data;
 
-		if (tp->Rx_databuff[i])
-			continue;
-
 		data = rtl8169_alloc_rx_data(tp, tp->RxDescArray + i);
 		if (!data) {
 			rtl8169_make_unusable_by_asic(tp->RxDescArray + i);
@@ -6811,14 +6312,12 @@ static int rtl8169_rx_fill(struct rtl8169_private *tp)
 	return -ENOMEM;
 }
 
-static int rtl8169_init_ring(struct net_device *dev)
+static int rtl8169_init_ring(struct rtl8169_private *tp)
 {
-	struct rtl8169_private *tp = netdev_priv(dev);
-
 	rtl8169_init_ring_indexes(tp);
 
-	memset(tp->tx_skb, 0x0, NUM_TX_DESC * sizeof(struct ring_info));
-	memset(tp->Rx_databuff, 0x0, NUM_RX_DESC * sizeof(void *));
+	memset(tp->tx_skb, 0, sizeof(tp->tx_skb));
+	memset(tp->Rx_databuff, 0, sizeof(tp->Rx_databuff));
 
 	return rtl8169_rx_fill(tp);
 }
@@ -6877,13 +6376,13 @@ static void rtl_reset_work(struct rtl8169_private *tp)
 	rtl8169_hw_reset(tp);
 
 	for (i = 0; i < NUM_RX_DESC; i++)
-		rtl8169_mark_to_asic(tp->RxDescArray + i, rx_buf_sz);
+		rtl8169_mark_to_asic(tp->RxDescArray + i);
 
 	rtl8169_tx_clear(tp);
 	rtl8169_init_ring_indexes(tp);
 
 	napi_enable(&tp->napi);
-	rtl_hw_start(dev);
+	rtl_hw_start(tp);
 	netif_wake_queue(dev);
 	rtl8169_check_link_status(dev, tp);
 }
@@ -7015,18 +6514,6 @@ static int msdn_giant_send_check(struct sk_buff *skb)
 	return ret;
 }
 
-static inline __be16 get_protocol(struct sk_buff *skb)
-{
-	__be16 protocol;
-
-	if (skb->protocol == htons(ETH_P_8021Q))
-		protocol = vlan_eth_hdr(skb)->h_vlan_encapsulated_proto;
-	else
-		protocol = skb->protocol;
-
-	return protocol;
-}
-
 static bool rtl8169_tso_csum_v1(struct rtl8169_private *tp,
 				struct sk_buff *skb, u32 *opts)
 {
@@ -7063,7 +6550,7 @@ static bool rtl8169_tso_csum_v2(struct rtl8169_private *tp,
 			return false;
 		}
 
-		switch (get_protocol(skb)) {
+		switch (vlan_get_protocol(skb)) {
 		case htons(ETH_P_IP):
 			opts[0] |= TD1_GTSENV4;
 			break;
@@ -7095,7 +6582,7 @@ static bool rtl8169_tso_csum_v2(struct rtl8169_private *tp,
 			return false;
 		}
 
-		switch (get_protocol(skb)) {
+		switch (vlan_get_protocol(skb)) {
 		case htons(ETH_P_IP):
 			opts[1] |= TD1_IPv4_CS;
 			ip_protocol = ip_hdr(skb)->protocol;
@@ -7365,7 +6852,7 @@ static struct sk_buff *rtl8169_try_rx_copy(void *data,
 	prefetch(data);
 	skb = napi_alloc_skb(&tp->napi, pkt_size);
 	if (skb)
-		memcpy(skb->data, data, pkt_size);
+		skb_copy_to_linear_data(skb, data, pkt_size);
 	dma_sync_single_for_device(d, addr, pkt_size, DMA_FROM_DEVICE);
 
 	return skb;
@@ -7383,7 +6870,7 @@ static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, u32 budget
 		struct RxDesc *desc = tp->RxDescArray + entry;
 		u32 status;
 
-		status = le32_to_cpu(desc->opts1) & tp->opts1_mask;
+		status = le32_to_cpu(desc->opts1);
 		if (status & DescOwn)
 			break;
 
@@ -7401,14 +6888,16 @@ static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, u32 budget
 				dev->stats.rx_length_errors++;
 			if (status & RxCRC)
 				dev->stats.rx_crc_errors++;
-			if (status & RxFOVF) {
+			/* RxFOVF is a reserved bit on later chip versions */
+			if (tp->mac_version == RTL_GIGA_MAC_VER_01 &&
+			    status & RxFOVF) {
 				rtl_schedule_task(tp, RTL_FLAG_TASK_RESET_PENDING);
 				dev->stats.rx_fifo_errors++;
-			}
-			if ((status & (RxRUNT | RxCRC)) &&
-			    !(status & (RxRWT | RxFOVF)) &&
-			    (dev->features & NETIF_F_RXALL))
+			} else if (status & (RxRUNT | RxCRC) &&
+				   !(status & RxRWT) &&
+				   dev->features & NETIF_F_RXALL) {
 				goto process_pkt;
+			}
 		} else {
 			struct sk_buff *skb;
 			dma_addr_t addr;
@@ -7457,7 +6946,7 @@ static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, u32 budget
 		}
 release_descriptor:
 		desc->opts2 = 0;
-		rtl8169_mark_to_asic(desc, rx_buf_sz);
+		rtl8169_mark_to_asic(desc);
 	}
 
 	count = cur_rx - tp->cur_rx;
@@ -7468,8 +6957,7 @@ static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, u32 budget
 
 static irqreturn_t rtl8169_interrupt(int irq, void *dev_instance)
 {
-	struct net_device *dev = dev_instance;
-	struct rtl8169_private *tp = netdev_priv(dev);
+	struct rtl8169_private *tp = dev_instance;
 	int handled = 0;
 	u16 status;
 
@@ -7480,7 +6968,7 @@ static irqreturn_t rtl8169_interrupt(int irq, void *dev_instance)
 			handled = 1;
 
 			rtl_irq_disable(tp);
-			napi_schedule(&tp->napi);
+			napi_schedule_irqoff(&tp->napi);
 		}
 	}
 	return IRQ_RETVAL(handled);
@@ -7631,7 +7119,7 @@ static int rtl8169_close(struct net_device *dev)
 	pm_runtime_get_sync(&pdev->dev);
 
 	/* Update counters before going down */
-	rtl8169_update_counters(dev);
+	rtl8169_update_counters(tp);
 
 	rtl_lock_work(tp);
 	clear_bit(RTL_FLAG_TASK_ENABLED, tp->wk.flags);
@@ -7641,7 +7129,7 @@ static int rtl8169_close(struct net_device *dev)
 
 	cancel_work_sync(&tp->wk.work);
 
-	pci_free_irq(pdev, 0, dev);
+	pci_free_irq(pdev, 0, tp);
 
 	dma_free_coherent(&pdev->dev, R8169_RX_RING_BYTES, tp->RxDescArray,
 			  tp->RxPhyAddr);
@@ -7686,7 +7174,7 @@ static int rtl_open(struct net_device *dev)
 	if (!tp->RxDescArray)
 		goto err_free_tx_0;
 
-	retval = rtl8169_init_ring(dev);
+	retval = rtl8169_init_ring(tp);
 	if (retval < 0)
 		goto err_free_rx_1;
 
@@ -7696,7 +7184,7 @@ static int rtl_open(struct net_device *dev)
 
 	rtl_request_firmware(tp);
 
-	retval = pci_request_irq(pdev, 0, rtl8169_interrupt, NULL, dev,
+	retval = pci_request_irq(pdev, 0, rtl8169_interrupt, NULL, tp,
 				 dev->name);
 	if (retval < 0)
 		goto err_release_fw_2;
@@ -7709,13 +7197,11 @@ static int rtl_open(struct net_device *dev)
 
 	rtl8169_init_phy(dev, tp);
 
-	__rtl8169_set_features(dev, dev->features);
-
 	rtl_pll_power_up(tp);
 
-	rtl_hw_start(dev);
+	rtl_hw_start(tp);
 
-	if (!rtl8169_init_counter_offsets(dev))
+	if (!rtl8169_init_counter_offsets(tp))
 		netif_warn(tp, hw, dev, "counter reset/update failed\n");
 
 	netif_start_queue(dev);
@@ -7784,7 +7270,7 @@ rtl8169_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
 	 * from tally counters.
 	 */
 	if (pm_runtime_active(&pdev->dev))
-		rtl8169_update_counters(dev);
+		rtl8169_update_counters(tp);
 
 	/*
 	 * Subtract values fetched during initalization.
@@ -7880,7 +7366,7 @@ static int rtl8169_runtime_suspend(struct device *device)
 
 	/* Update counters before going runtime suspend */
 	rtl8169_rx_missed(dev);
-	rtl8169_update_counters(dev);
+	rtl8169_update_counters(tp);
 
 	return 0;
 }
@@ -8020,9 +7506,7 @@ static const struct net_device_ops rtl_netdev_ops = {
 };
 
 static const struct rtl_cfg_info {
-	void (*hw_start)(struct net_device *);
-	unsigned int region;
-	unsigned int align;
+	void (*hw_start)(struct rtl8169_private *tp);
 	u16 event_slow;
 	unsigned int has_gmii:1;
 	const struct rtl_coalesce_info *coalesce_info;
@@ -8030,8 +7514,6 @@ static const struct rtl_cfg_info {
 } rtl_cfg_infos [] = {
 	[RTL_CFG_0] = {
 		.hw_start	= rtl_hw_start_8169,
-		.region		= 1,
-		.align		= 0,
 		.event_slow	= SYSErr | LinkChg | RxOverflow | RxFIFOOver,
 		.has_gmii	= 1,
 		.coalesce_info	= rtl_coalesce_info_8169,
@@ -8039,8 +7521,6 @@ static const struct rtl_cfg_info {
 	},
 	[RTL_CFG_1] = {
 		.hw_start	= rtl_hw_start_8168,
-		.region		= 2,
-		.align		= 8,
 		.event_slow	= SYSErr | LinkChg | RxOverflow,
 		.has_gmii	= 1,
 		.coalesce_info	= rtl_coalesce_info_8168_8136,
@@ -8048,8 +7528,6 @@ static const struct rtl_cfg_info {
 	},
 	[RTL_CFG_2] = {
 		.hw_start	= rtl_hw_start_8101,
-		.region		= 2,
-		.align		= 8,
 		.event_slow	= SYSErr | LinkChg | RxOverflow | RxFIFOOver |
 				  PCSTimeout,
 		.coalesce_info	= rtl_coalesce_info_8168_8136,
@@ -8125,20 +7603,10 @@ static void rtl_hw_init_8168ep(struct rtl8169_private *tp)
 static void rtl_hw_initialize(struct rtl8169_private *tp)
 {
 	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_40:
-	case RTL_GIGA_MAC_VER_41:
-	case RTL_GIGA_MAC_VER_42:
-	case RTL_GIGA_MAC_VER_43:
-	case RTL_GIGA_MAC_VER_44:
-	case RTL_GIGA_MAC_VER_45:
-	case RTL_GIGA_MAC_VER_46:
-	case RTL_GIGA_MAC_VER_47:
-	case RTL_GIGA_MAC_VER_48:
+	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_48:
 		rtl_hw_init_8168g(tp);
 		break;
-	case RTL_GIGA_MAC_VER_49:
-	case RTL_GIGA_MAC_VER_50:
-	case RTL_GIGA_MAC_VER_51:
+	case RTL_GIGA_MAC_VER_49 ... RTL_GIGA_MAC_VER_51:
 		rtl_hw_init_8168ep(tp);
 		break;
 	default:
@@ -8149,11 +7617,10 @@ static void rtl_hw_initialize(struct rtl8169_private *tp)
 static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	const struct rtl_cfg_info *cfg = rtl_cfg_infos + ent->driver_data;
-	const unsigned int region = cfg->region;
 	struct rtl8169_private *tp;
 	struct mii_if_info *mii;
 	struct net_device *dev;
-	int chipset, i;
+	int chipset, region, i;
 	int rc;
 
 	if (netif_msg_drv(&debug)) {
@@ -8188,43 +7655,41 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	/* enable device (incl. PCI PM wakeup and hotplug setup) */
 	rc = pcim_enable_device(pdev);
 	if (rc < 0) {
-		netif_err(tp, probe, dev, "enable failure\n");
+		dev_err(&pdev->dev, "enable failure\n");
 		return rc;
 	}
 
 	if (pcim_set_mwi(pdev) < 0)
-		netif_info(tp, probe, dev, "Mem-Wr-Inval unavailable\n");
+		dev_info(&pdev->dev, "Mem-Wr-Inval unavailable\n");
 
-	/* make sure PCI base addr 1 is MMIO */
-	if (!(pci_resource_flags(pdev, region) & IORESOURCE_MEM)) {
-		netif_err(tp, probe, dev,
-			  "region #%d not an MMIO resource, aborting\n",
-			  region);
+	/* use first MMIO region */
+	region = ffs(pci_select_bars(pdev, IORESOURCE_MEM)) - 1;
+	if (region < 0) {
+		dev_err(&pdev->dev, "no MMIO resource found\n");
 		return -ENODEV;
 	}
 
 	/* check for weird/broken PCI region reporting */
 	if (pci_resource_len(pdev, region) < R8169_REGS_SIZE) {
-		netif_err(tp, probe, dev,
-			  "Invalid PCI region size(s), aborting\n");
+		dev_err(&pdev->dev, "Invalid PCI region size(s), aborting\n");
 		return -ENODEV;
 	}
 
 	rc = pcim_iomap_regions(pdev, BIT(region), MODULENAME);
 	if (rc < 0) {
-		netif_err(tp, probe, dev, "cannot remap MMIO, aborting\n");
+		dev_err(&pdev->dev, "cannot remap MMIO, aborting\n");
 		return rc;
 	}
 
 	tp->mmio_addr = pcim_iomap_table(pdev)[region];
 
 	if (!pci_is_pcie(pdev))
-		netif_info(tp, probe, dev, "not PCI Express\n");
+		dev_info(&pdev->dev, "not PCI Express\n");
 
 	/* Identify chip attached to board */
-	rtl8169_get_mac_version(tp, dev, cfg->default_ver);
+	rtl8169_get_mac_version(tp, cfg->default_ver);
 
-	tp->cp_cmd = 0;
+	tp->cp_cmd = RTL_R16(tp, CPlusCmd);
 
 	if ((sizeof(dma_addr_t) > 4) &&
 	    (use_dac == 1 || (use_dac == -1 && pci_is_pcie(pdev) &&
@@ -8239,7 +7704,7 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	} else {
 		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
 		if (rc < 0) {
-			netif_err(tp, probe, dev, "DMA configuration failed\n");
+			dev_err(&pdev->dev, "DMA configuration failed\n");
 			return rc;
 		}
 	}
@@ -8257,18 +7722,15 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	pci_set_master(pdev);
 
 	rtl_init_mdio_ops(tp);
-	rtl_init_pll_power_ops(tp);
 	rtl_init_jumbo_ops(tp);
-	rtl_init_csi_ops(tp);
 
 	rtl8169_print_mac_version(tp);
 
 	chipset = tp->mac_version;
-	tp->txd_version = rtl_chip_infos[chipset].txd_version;
 
 	rc = rtl_alloc_irq(tp);
 	if (rc < 0) {
-		netif_err(tp, probe, dev, "Can't allocate interrupt\n");
+		dev_err(&pdev->dev, "Can't allocate interrupt\n");
 		return rc;
 	}
 
@@ -8296,29 +7758,18 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	u64_stats_init(&tp->tx_stats.syncp);
 
 	/* Get MAC address */
-	if (tp->mac_version == RTL_GIGA_MAC_VER_35 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_36 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_37 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_38 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_40 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_41 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_42 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_43 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_44 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_45 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_46 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_47 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_48 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_49 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_50 ||
-	    tp->mac_version == RTL_GIGA_MAC_VER_51) {
-		u16 mac_addr[3];
-
+	switch (tp->mac_version) {
+		u8 mac_addr[ETH_ALEN] __aligned(4);
+	case RTL_GIGA_MAC_VER_35 ... RTL_GIGA_MAC_VER_38:
+	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_51:
 		*(u32 *)&mac_addr[0] = rtl_eri_read(tp, 0xe0, ERIAR_EXGMAC);
-		*(u16 *)&mac_addr[2] = rtl_eri_read(tp, 0xe4, ERIAR_EXGMAC);
+		*(u16 *)&mac_addr[4] = rtl_eri_read(tp, 0xe4, ERIAR_EXGMAC);
 
-		if (is_valid_ether_addr((u8 *)mac_addr))
-			rtl_rar_set(tp, (u8 *)mac_addr);
+		if (is_valid_ether_addr(mac_addr))
+			rtl_rar_set(tp, mac_addr);
+		break;
+	default:
+		break;
 	}
 	for (i = 0; i < ETH_ALEN; i++)
 		dev->dev_addr[i] = RTL_R8(tp, MAC0 + i);
@@ -8326,7 +7777,7 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	dev->ethtool_ops = &rtl8169_ethtool_ops;
 	dev->watchdog_timeo = RTL8169_TX_TIMEOUT;
 
-	netif_napi_add(dev, &tp->napi, rtl8169_poll, R8169_NAPI_WEIGHT);
+	netif_napi_add(dev, &tp->napi, rtl8169_poll, NAPI_POLL_WEIGHT);
 
 	/* don't enable SG, IP_CSUM and TSO by default - it might not work
 	 * properly for all devices */
@@ -8349,13 +7800,17 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		/* Disallow toggling */
 		dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
 
-	if (tp->txd_version == RTL_TD_0)
+	switch (rtl_chip_infos[chipset].txd_version) {
+	case RTL_TD_0:
 		tp->tso_csum = rtl8169_tso_csum_v1;
-	else if (tp->txd_version == RTL_TD_1) {
+		break;
+	case RTL_TD_1:
 		tp->tso_csum = rtl8169_tso_csum_v2;
 		dev->hw_features |= NETIF_F_IPV6_CSUM | NETIF_F_TSO6;
-	} else
+		break;
+	default:
 		WARN_ON_ONCE(1);
+	}
 
 	dev->hw_features |= NETIF_F_RXALL;
 	dev->hw_features |= NETIF_F_RXFCS;
@@ -8368,9 +7823,6 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	tp->event_slow = cfg->event_slow;
 	tp->coalesce_info = cfg->coalesce_info;
 
-	tp->opts1_mask = (tp->mac_version != RTL_GIGA_MAC_VER_01) ?
-		~(RxBOVF | RxFOVF) : ~0;
-
 	timer_setup(&tp->timer, rtl8169_phy_timer, 0);
 
 	tp->rtl_fw = RTL_FIRMWARE_UNKNOWN;
@@ -8387,15 +7839,15 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (rc < 0)
 		return rc;
 
-	netif_info(tp, probe, dev, "%s at 0x%p, %pM, XID %08x IRQ %d\n",
-		   rtl_chip_infos[chipset].name, tp->mmio_addr, dev->dev_addr,
-		   (u32)(RTL_R32(tp, TxConfig) & 0x9cf0f8ff),
+	netif_info(tp, probe, dev, "%s, %pM, XID %08x, IRQ %d\n",
+		   rtl_chip_infos[chipset].name, dev->dev_addr,
+		   (u32)(RTL_R32(tp, TxConfig) & 0xfcf0f8ff),
 		   pci_irq_vector(pdev, 0));
 	if (rtl_chip_infos[chipset].jumbo_max != JUMBO_1K) {
 		netif_info(tp, probe, dev, "jumbo features [frames: %d bytes, "
 			   "tx checksumming: %s]\n",
 			   rtl_chip_infos[chipset].jumbo_max,
-			   rtl_chip_infos[chipset].jumbo_tx_csum ? "ok" : "ko");
+			  tp->mac_version <= RTL_GIGA_MAC_VER_06 ? "ok" : "ko");
 	}
 
 	if (r8168_check_dash(tp))
diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c
index b6b90a6314e3..d9cadfb1bc4a 100644
--- a/drivers/net/ethernet/renesas/sh_eth.c
+++ b/drivers/net/ethernet/renesas/sh_eth.c
@@ -442,12 +442,22 @@ static void sh_eth_modify(struct net_device *ndev, int enum_index, u32 clear,
 static void sh_eth_tsu_write(struct sh_eth_private *mdp, u32 data,
 			     int enum_index)
 {
-	iowrite32(data, mdp->tsu_addr + mdp->reg_offset[enum_index]);
+	u16 offset = mdp->reg_offset[enum_index];
+
+	if (WARN_ON(offset == SH_ETH_OFFSET_INVALID))
+		return;
+
+	iowrite32(data, mdp->tsu_addr + offset);
 }
 
 static u32 sh_eth_tsu_read(struct sh_eth_private *mdp, int enum_index)
 {
-	return ioread32(mdp->tsu_addr + mdp->reg_offset[enum_index]);
+	u16 offset = mdp->reg_offset[enum_index];
+
+	if (WARN_ON(offset == SH_ETH_OFFSET_INVALID))
+		return ~0U;
+
+	return ioread32(mdp->tsu_addr + offset);
 }
 
 static void sh_eth_select_mii(struct net_device *ndev)
@@ -456,6 +466,9 @@ static void sh_eth_select_mii(struct net_device *ndev)
 	u32 value;
 
 	switch (mdp->phy_interface) {
+	case PHY_INTERFACE_MODE_RGMII ... PHY_INTERFACE_MODE_RGMII_TXID:
+		value = 0x3;
+		break;
 	case PHY_INTERFACE_MODE_GMII:
 		value = 0x2;
 		break;
@@ -693,7 +706,7 @@ static struct sh_eth_cpu_data rcar_gen1_data = {
 			  EESIPR_RTLFIP | EESIPR_RTSFIP |
 			  EESIPR_PREIP | EESIPR_CERFIP,
 
-	.tx_check	= EESR_FTC | EESR_CND | EESR_DLC | EESR_CD | EESR_RTO,
+	.tx_check	= EESR_FTC | EESR_CND | EESR_DLC | EESR_CD | EESR_TRO,
 	.eesr_err_check	= EESR_TWB | EESR_TABT | EESR_RABT | EESR_RFE |
 			  EESR_RDE | EESR_RFRMER | EESR_TFE | EESR_TDE,
 	.fdr_value	= 0x00000f0f,
@@ -725,7 +738,7 @@ static struct sh_eth_cpu_data rcar_gen2_data = {
 			  EESIPR_RTLFIP | EESIPR_RTSFIP |
 			  EESIPR_PREIP | EESIPR_CERFIP,
 
-	.tx_check	= EESR_FTC | EESR_CND | EESR_DLC | EESR_CD | EESR_RTO,
+	.tx_check	= EESR_FTC | EESR_CND | EESR_DLC | EESR_CD | EESR_TRO,
 	.eesr_err_check	= EESR_TWB | EESR_TABT | EESR_RABT | EESR_RFE |
 			  EESR_RDE | EESR_RFRMER | EESR_TFE | EESR_TDE,
 	.fdr_value	= 0x00000f0f,
@@ -740,6 +753,49 @@ static struct sh_eth_cpu_data rcar_gen2_data = {
 	.rmiimode	= 1,
 	.magic		= 1,
 };
+
+/* R8A77980 */
+static struct sh_eth_cpu_data r8a77980_data = {
+	.soft_reset	= sh_eth_soft_reset_gether,
+
+	.set_duplex	= sh_eth_set_duplex,
+	.set_rate	= sh_eth_set_rate_gether,
+
+	.register_type  = SH_ETH_REG_GIGABIT,
+
+	.edtrr_trns	= EDTRR_TRNS_GETHER,
+	.ecsr_value	= ECSR_PSRTO | ECSR_LCHNG | ECSR_ICD | ECSR_MPD,
+	.ecsipr_value	= ECSIPR_PSRTOIP | ECSIPR_LCHNGIP | ECSIPR_ICDIP |
+			  ECSIPR_MPDIP,
+	.eesipr_value	= EESIPR_RFCOFIP | EESIPR_ECIIP |
+			  EESIPR_FTCIP | EESIPR_TDEIP | EESIPR_TFUFIP |
+			  EESIPR_FRIP | EESIPR_RDEIP | EESIPR_RFOFIP |
+			  EESIPR_RMAFIP | EESIPR_RRFIP |
+			  EESIPR_RTLFIP | EESIPR_RTSFIP |
+			  EESIPR_PREIP | EESIPR_CERFIP,
+
+	.tx_check       = EESR_FTC | EESR_CD | EESR_TRO,
+	.eesr_err_check = EESR_TWB1 | EESR_TWB | EESR_TABT | EESR_RABT |
+			  EESR_RFE | EESR_RDE | EESR_RFRMER |
+			  EESR_TFE | EESR_TDE | EESR_ECI,
+	.fdr_value	= 0x0000070f,
+
+	.apr		= 1,
+	.mpr		= 1,
+	.tpauser	= 1,
+	.bculr		= 1,
+	.hw_swap	= 1,
+	.nbst		= 1,
+	.rpadir		= 1,
+	.rpadir_value   = 2 << 16,
+	.no_trimd	= 1,
+	.no_ade		= 1,
+	.xdfar_rw	= 1,
+	.hw_checksum	= 1,
+	.select_mii	= 1,
+	.magic		= 1,
+	.cexcr		= 1,
+};
 #endif /* CONFIG_OF */
 
 static void sh_eth_set_rate_sh7724(struct net_device *ndev)
@@ -775,7 +831,7 @@ static struct sh_eth_cpu_data sh7724_data = {
 			  EESIPR_RTLFIP | EESIPR_RTSFIP |
 			  EESIPR_PREIP | EESIPR_CERFIP,
 
-	.tx_check	= EESR_FTC | EESR_CND | EESR_DLC | EESR_CD | EESR_RTO,
+	.tx_check	= EESR_FTC | EESR_CND | EESR_DLC | EESR_CD | EESR_TRO,
 	.eesr_err_check	= EESR_TWB | EESR_TABT | EESR_RABT | EESR_RFE |
 			  EESR_RDE | EESR_RFRMER | EESR_TFE | EESR_TDE,
 
@@ -820,7 +876,7 @@ static struct sh_eth_cpu_data sh7757_data = {
 			  EESIPR_RRFIP | EESIPR_RTLFIP | EESIPR_RTSFIP |
 			  EESIPR_PREIP | EESIPR_CERFIP,
 
-	.tx_check	= EESR_FTC | EESR_CND | EESR_DLC | EESR_CD | EESR_RTO,
+	.tx_check	= EESR_FTC | EESR_CND | EESR_DLC | EESR_CD | EESR_TRO,
 	.eesr_err_check	= EESR_TWB | EESR_TABT | EESR_RABT | EESR_RFE |
 			  EESR_RDE | EESR_RFRMER | EESR_TFE | EESR_TDE,
 
@@ -1421,8 +1477,13 @@ static int sh_eth_dev_init(struct net_device *ndev)
 
 	sh_eth_write(ndev, mdp->cd->trscer_err_mask, TRSCER);
 
+	/* DMA transfer burst mode */
+	if (mdp->cd->nbst)
+		sh_eth_modify(ndev, EDMR, EDMR_NBST, EDMR_NBST);
+
+	/* Burst cycle count upper-limit */
 	if (mdp->cd->bculr)
-		sh_eth_write(ndev, 0x800, BCULR);	/* Burst sycle set */
+		sh_eth_write(ndev, 0x800, BCULR);
 
 	sh_eth_write(ndev, mdp->cd->fcftr_value, FCFTR);
 
@@ -2610,12 +2671,6 @@ static int sh_eth_change_mtu(struct net_device *ndev, int new_mtu)
 }
 
 /* For TSU_POSTn. Please refer to the manual about this (strange) bitfields */
-static void *sh_eth_tsu_get_post_reg_offset(struct sh_eth_private *mdp,
-					    int entry)
-{
-	return sh_eth_tsu_get_offset(mdp, TSU_POST1) + (entry / 8 * 4);
-}
-
 static u32 sh_eth_tsu_get_post_mask(int entry)
 {
 	return 0x0f << (28 - ((entry % 8) * 4));
@@ -2630,27 +2685,25 @@ static void sh_eth_tsu_enable_cam_entry_post(struct net_device *ndev,
 					     int entry)
 {
 	struct sh_eth_private *mdp = netdev_priv(ndev);
+	int reg = TSU_POST1 + entry / 8;
 	u32 tmp;
-	void *reg_offset;
 
-	reg_offset = sh_eth_tsu_get_post_reg_offset(mdp, entry);
-	tmp = ioread32(reg_offset);
-	iowrite32(tmp | sh_eth_tsu_get_post_bit(mdp, entry), reg_offset);
+	tmp = sh_eth_tsu_read(mdp, reg);
+	sh_eth_tsu_write(mdp, tmp | sh_eth_tsu_get_post_bit(mdp, entry), reg);
 }
 
 static bool sh_eth_tsu_disable_cam_entry_post(struct net_device *ndev,
 					      int entry)
 {
 	struct sh_eth_private *mdp = netdev_priv(ndev);
+	int reg = TSU_POST1 + entry / 8;
 	u32 post_mask, ref_mask, tmp;
-	void *reg_offset;
 
-	reg_offset = sh_eth_tsu_get_post_reg_offset(mdp, entry);
 	post_mask = sh_eth_tsu_get_post_mask(entry);
 	ref_mask = sh_eth_tsu_get_post_bit(mdp, entry) & ~post_mask;
 
-	tmp = ioread32(reg_offset);
-	iowrite32(tmp & ~post_mask, reg_offset);
+	tmp = sh_eth_tsu_read(mdp, reg);
+	sh_eth_tsu_write(mdp, tmp & ~post_mask, reg);
 
 	/* If other port enables, the function returns "true" */
 	return tmp & ref_mask;
@@ -3023,15 +3076,10 @@ static int sh_mdio_init(struct sh_eth_private *mdp,
 		 pdev->name, pdev->id);
 
 	/* register MDIO bus */
-	if (dev->of_node) {
-		ret = of_mdiobus_register(mdp->mii_bus, dev->of_node);
-	} else {
-		if (pd->phy_irq > 0)
-			mdp->mii_bus->irq[pd->phy] = pd->phy_irq;
-
-		ret = mdiobus_register(mdp->mii_bus);
-	}
+	if (pd->phy_irq > 0)
+		mdp->mii_bus->irq[pd->phy] = pd->phy_irq;
 
+	ret = of_mdiobus_register(mdp->mii_bus, dev->of_node);
 	if (ret)
 		goto out_free_bus;
 
@@ -3130,6 +3178,7 @@ static const struct of_device_id sh_eth_match_table[] = {
 	{ .compatible = "renesas,ether-r8a7791", .data = &rcar_gen2_data },
 	{ .compatible = "renesas,ether-r8a7793", .data = &rcar_gen2_data },
 	{ .compatible = "renesas,ether-r8a7794", .data = &rcar_gen2_data },
+	{ .compatible = "renesas,gether-r8a77980", .data = &r8a77980_data },
 	{ .compatible = "renesas,ether-r7s72100", .data = &r7s72100_data },
 	{ .compatible = "renesas,rcar-gen1-ether", .data = &rcar_gen1_data },
 	{ .compatible = "renesas,rcar-gen2-ether", .data = &rcar_gen2_data },
diff --git a/drivers/net/ethernet/renesas/sh_eth.h b/drivers/net/ethernet/renesas/sh_eth.h
index 1bf930d4a1e5..5dee19b61aee 100644
--- a/drivers/net/ethernet/renesas/sh_eth.h
+++ b/drivers/net/ethernet/renesas/sh_eth.h
@@ -184,6 +184,7 @@ enum GECMR_BIT {
 
 /* EDMR */
 enum DMAC_M_BIT {
+	EDMR_NBST = 0x80,
 	EDMR_EL = 0x40, /* Litte endian */
 	EDMR_DL1 = 0x20, EDMR_DL0 = 0x10,
 	EDMR_SRST_GETHER = 0x03,
@@ -242,7 +243,7 @@ enum EESR_BIT {
 	EESR_CND	= 0x00000800,
 	EESR_DLC	= 0x00000400,
 	EESR_CD		= 0x00000200,
-	EESR_RTO	= 0x00000100,
+	EESR_TRO	= 0x00000100,
 	EESR_RMAF	= 0x00000080,
 	EESR_CEEF	= 0x00000040,
 	EESR_CELF	= 0x00000020,
@@ -262,7 +263,7 @@ enum EESR_BIT {
 				 EESR_CERF)  /* Recv frame CRC error */
 
 #define DEFAULT_TX_CHECK	(EESR_FTC | EESR_CND | EESR_DLC | EESR_CD | \
-				 EESR_RTO)
+				 EESR_TRO)
 #define DEFAULT_EESR_ERR_CHECK	(EESR_TWB | EESR_TABT | EESR_RABT | EESR_RFE | \
 				 EESR_RDE | EESR_RFRMER | EESR_ADE | \
 				 EESR_TFE | EESR_TDE)
@@ -498,20 +499,21 @@ struct sh_eth_cpu_data {
 
 	/* hardware features */
 	unsigned long irq_flags; /* IRQ configuration flags */
-	unsigned no_psr:1;	/* EtherC DO NOT have PSR */
-	unsigned apr:1;		/* EtherC have APR */
-	unsigned mpr:1;		/* EtherC have MPR */
-	unsigned tpauser:1;	/* EtherC have TPAUSER */
-	unsigned bculr:1;	/* EtherC have BCULR */
-	unsigned tsu:1;		/* EtherC have TSU */
-	unsigned hw_swap:1;	/* E-DMAC have DE bit in EDMR */
-	unsigned rpadir:1;	/* E-DMAC have RPADIR */
-	unsigned no_trimd:1;	/* E-DMAC DO NOT have TRIMD */
-	unsigned no_ade:1;	/* E-DMAC DO NOT have ADE bit in EESR */
+	unsigned no_psr:1;	/* EtherC DOES NOT have PSR */
+	unsigned apr:1;		/* EtherC has APR */
+	unsigned mpr:1;		/* EtherC has MPR */
+	unsigned tpauser:1;	/* EtherC has TPAUSER */
+	unsigned bculr:1;	/* EtherC has BCULR */
+	unsigned tsu:1;		/* EtherC has TSU */
+	unsigned hw_swap:1;	/* E-DMAC has DE bit in EDMR */
+	unsigned nbst:1;	/* E-DMAC has NBST bit in EDMR */
+	unsigned rpadir:1;	/* E-DMAC has RPADIR */
+	unsigned no_trimd:1;	/* E-DMAC DOES NOT have TRIMD */
+	unsigned no_ade:1;	/* E-DMAC DOES NOT have ADE bit in EESR */
 	unsigned no_xdfar:1;	/* E-DMAC DOES NOT have RDFAR/TDFAR */
 	unsigned xdfar_rw:1;	/* E-DMAC has writeable RDFAR/TDFAR */
 	unsigned hw_checksum:1;	/* E-DMAC has CSMR */
-	unsigned select_mii:1;	/* EtherC have RMII_MII (MII select register) */
+	unsigned select_mii:1;	/* EtherC has RMII_MII (MII select register) */
 	unsigned rmiimode:1;	/* EtherC has RMIIMODE register */
 	unsigned rtrate:1;	/* EtherC has RTRATE register */
 	unsigned magic:1;	/* EtherC has ECMR.MPDE and ECSR.MPD */
diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c
index 056cb6093630..e73e4febeedb 100644
--- a/drivers/net/ethernet/rocker/rocker_main.c
+++ b/drivers/net/ethernet/rocker/rocker_main.c
@@ -2738,6 +2738,8 @@ static void rocker_switchdev_event_work(struct work_struct *work)
 	switch (switchdev_work->event) {
 	case SWITCHDEV_FDB_ADD_TO_DEVICE:
 		fdb_info = &switchdev_work->fdb_info;
+		if (!fdb_info->added_by_user)
+			break;
 		err = rocker_world_port_fdb_add(rocker_port, fdb_info);
 		if (err) {
 			netdev_dbg(rocker_port->dev, "fdb add failed err=%d\n", err);
@@ -2747,6 +2749,8 @@ static void rocker_switchdev_event_work(struct work_struct *work)
 		break;
 	case SWITCHDEV_FDB_DEL_TO_DEVICE:
 		fdb_info = &switchdev_work->fdb_info;
+		if (!fdb_info->added_by_user)
+			break;
 		err = rocker_world_port_fdb_del(rocker_port, fdb_info);
 		if (err)
 			netdev_dbg(rocker_port->dev, "fdb add failed err=%d\n", err);
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index a4ebd8715494..ad4a354ce570 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -1289,9 +1289,8 @@ static int efx_init_io(struct efx_nic *efx)
 
 	pci_set_master(pci_dev);
 
-	/* Set the PCI DMA mask.  Try all possibilities from our
-	 * genuine mask down to 32 bits, because some architectures
-	 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit
+	/* Set the PCI DMA mask.  Try all possibilities from our genuine mask
+	 * down to 32 bits, because some architectures will allow 40 bit
 	 * masks event though they reject 46 bit masks.
 	 */
 	while (dma_mask > 0x7fffffffUL) {
@@ -1551,6 +1550,38 @@ static int efx_probe_interrupts(struct efx_nic *efx)
 	return 0;
 }
 
+#if defined(CONFIG_SMP)
+static void efx_set_interrupt_affinity(struct efx_nic *efx)
+{
+	struct efx_channel *channel;
+	unsigned int cpu;
+
+	efx_for_each_channel(channel, efx) {
+		cpu = cpumask_local_spread(channel->channel,
+					   pcibus_to_node(efx->pci_dev->bus));
+		irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
+	}
+}
+
+static void efx_clear_interrupt_affinity(struct efx_nic *efx)
+{
+	struct efx_channel *channel;
+
+	efx_for_each_channel(channel, efx)
+		irq_set_affinity_hint(channel->irq, NULL);
+}
+#else
+static void
+efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
+{
+}
+
+static void
+efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
+{
+}
+#endif /* CONFIG_SMP */
+
 static int efx_soft_enable_interrupts(struct efx_nic *efx)
 {
 	struct efx_channel *channel, *end_channel;
@@ -3308,6 +3339,7 @@ static void efx_pci_remove_main(struct efx_nic *efx)
 	cancel_work_sync(&efx->reset_work);
 
 	efx_disable_interrupts(efx);
+	efx_clear_interrupt_affinity(efx);
 	efx_nic_fini_interrupt(efx);
 	efx_fini_port(efx);
 	efx->type->fini(efx);
@@ -3457,6 +3489,8 @@ static int efx_pci_probe_main(struct efx_nic *efx)
 	rc = efx_nic_init_interrupt(efx);
 	if (rc)
 		goto fail5;
+
+	efx_set_interrupt_affinity(efx);
 	rc = efx_enable_interrupts(efx);
 	if (rc)
 		goto fail6;
@@ -3464,6 +3498,7 @@ static int efx_pci_probe_main(struct efx_nic *efx)
 	return 0;
 
  fail6:
+	efx_clear_interrupt_affinity(efx);
 	efx_nic_fini_interrupt(efx);
  fail5:
 	efx_fini_port(efx);
diff --git a/drivers/net/ethernet/sfc/falcon/efx.c b/drivers/net/ethernet/sfc/falcon/efx.c
index 3d6c91e96589..dd5530a4f8c8 100644
--- a/drivers/net/ethernet/sfc/falcon/efx.c
+++ b/drivers/net/ethernet/sfc/falcon/efx.c
@@ -1242,9 +1242,8 @@ static int ef4_init_io(struct ef4_nic *efx)
 
 	pci_set_master(pci_dev);
 
-	/* Set the PCI DMA mask.  Try all possibilities from our
-	 * genuine mask down to 32 bits, because some architectures
-	 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit
+	/* Set the PCI DMA mask.  Try all possibilities from our genuine mask
+	 * down to 32 bits, because some architectures will allow 40 bit
 	 * masks event though they reject 46 bit masks.
 	 */
 	while (dma_mask > 0x7fffffffUL) {
diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c
index cece961f2e82..c3ad564ac4c0 100644
--- a/drivers/net/ethernet/sfc/tx.c
+++ b/drivers/net/ethernet/sfc/tx.c
@@ -435,17 +435,18 @@ static int efx_tx_map_data(struct efx_tx_queue *tx_queue, struct sk_buff *skb,
 	} while (1);
 }
 
-/* Remove buffers put into a tx_queue.  None of the buffers must have
- * an skb attached.
+/* Remove buffers put into a tx_queue for the current packet.
+ * None of the buffers must have an skb attached.
  */
-static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue)
+static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue,
+			       unsigned int insert_count)
 {
 	struct efx_tx_buffer *buffer;
 	unsigned int bytes_compl = 0;
 	unsigned int pkts_compl = 0;
 
 	/* Work backwards until we hit the original insert pointer value */
-	while (tx_queue->insert_count != tx_queue->write_count) {
+	while (tx_queue->insert_count != insert_count) {
 		--tx_queue->insert_count;
 		buffer = __efx_tx_queue_get_insert_buffer(tx_queue);
 		efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl);
@@ -504,6 +505,8 @@ static int efx_tx_tso_fallback(struct efx_tx_queue *tx_queue,
  */
 netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
 {
+	unsigned int old_insert_count = tx_queue->insert_count;
+	bool xmit_more = skb->xmit_more;
 	bool data_mapped = false;
 	unsigned int segments;
 	unsigned int skb_len;
@@ -553,8 +556,10 @@ netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
 	/* Update BQL */
 	netdev_tx_sent_queue(tx_queue->core_txq, skb_len);
 
+	efx_tx_maybe_stop_queue(tx_queue);
+
 	/* Pass off to hardware */
-	if (!skb->xmit_more || netif_xmit_stopped(tx_queue->core_txq)) {
+	if (!xmit_more || netif_xmit_stopped(tx_queue->core_txq)) {
 		struct efx_tx_queue *txq2 = efx_tx_queue_partner(tx_queue);
 
 		/* There could be packets left on the partner queue if those
@@ -577,14 +582,26 @@ netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
 		tx_queue->tx_packets++;
 	}
 
-	efx_tx_maybe_stop_queue(tx_queue);
-
 	return NETDEV_TX_OK;
 
 
 err:
-	efx_enqueue_unwind(tx_queue);
+	efx_enqueue_unwind(tx_queue, old_insert_count);
 	dev_kfree_skb_any(skb);
+
+	/* If we're not expecting another transmit and we had something to push
+	 * on this queue or a partner queue then we need to push here to get the
+	 * previous packets out.
+	 */
+	if (!xmit_more) {
+		struct efx_tx_queue *txq2 = efx_tx_queue_partner(tx_queue);
+
+		if (txq2->xmit_more_available)
+			efx_nic_push_buffers(txq2);
+
+		efx_nic_push_buffers(tx_queue);
+	}
+
 	return NETDEV_TX_OK;
 }
 
diff --git a/drivers/net/ethernet/socionext/Kconfig b/drivers/net/ethernet/socionext/Kconfig
index 6bcfe27fc560..b80048ca82a0 100644
--- a/drivers/net/ethernet/socionext/Kconfig
+++ b/drivers/net/ethernet/socionext/Kconfig
@@ -14,6 +14,8 @@ if NET_VENDOR_SOCIONEXT
 config SNI_AVE
 	tristate "Socionext AVE ethernet support"
 	depends on (ARCH_UNIPHIER || COMPILE_TEST) && OF
+	depends on HAS_IOMEM
+	select MFD_SYSCON
 	select PHYLIB
 	---help---
 	  Driver for gigabit ethernet MACs, called AVE, in the
diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c
index f4c0b02ddad8..ce8071fc90c4 100644
--- a/drivers/net/ethernet/socionext/netsec.c
+++ b/drivers/net/ethernet/socionext/netsec.c
@@ -1057,7 +1057,8 @@ static int netsec_netdev_load_microcode(struct netsec_priv *priv)
 	return 0;
 }
 
-static int netsec_reset_hardware(struct netsec_priv *priv)
+static int netsec_reset_hardware(struct netsec_priv *priv,
+				 bool load_ucode)
 {
 	u32 value;
 	int err;
@@ -1102,11 +1103,14 @@ static int netsec_reset_hardware(struct netsec_priv *priv)
 	netsec_write(priv, NETSEC_REG_NRM_RX_CONFIG,
 		     1 << NETSEC_REG_DESC_ENDIAN);
 
-	err = netsec_netdev_load_microcode(priv);
-	if (err) {
-		netif_err(priv, probe, priv->ndev,
-			  "%s: failed to load microcode (%d)\n", __func__, err);
-		return err;
+	if (load_ucode) {
+		err = netsec_netdev_load_microcode(priv);
+		if (err) {
+			netif_err(priv, probe, priv->ndev,
+				  "%s: failed to load microcode (%d)\n",
+				  __func__, err);
+			return err;
+		}
 	}
 
 	/* start DMA engines */
@@ -1313,8 +1317,8 @@ static int netsec_netdev_open(struct net_device *ndev)
 	napi_enable(&priv->napi);
 	netif_start_queue(ndev);
 
-	/* Enable RX intr. */
-	netsec_write(priv, NETSEC_REG_INTEN_SET, NETSEC_IRQ_RX);
+	/* Enable TX+RX intr. */
+	netsec_write(priv, NETSEC_REG_INTEN_SET, NETSEC_IRQ_RX | NETSEC_IRQ_TX);
 
 	return 0;
 err3:
@@ -1328,6 +1332,7 @@ static int netsec_netdev_open(struct net_device *ndev)
 
 static int netsec_netdev_stop(struct net_device *ndev)
 {
+	int ret;
 	struct netsec_priv *priv = netdev_priv(ndev);
 
 	netif_stop_queue(priv->ndev);
@@ -1343,12 +1348,14 @@ static int netsec_netdev_stop(struct net_device *ndev)
 	netsec_uninit_pkt_dring(priv, NETSEC_RING_TX);
 	netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
 
+	ret = netsec_reset_hardware(priv, false);
+
 	phy_stop(ndev->phydev);
 	phy_disconnect(ndev->phydev);
 
 	pm_runtime_put_sync(priv->dev);
 
-	return 0;
+	return ret;
 }
 
 static int netsec_netdev_init(struct net_device *ndev)
@@ -1364,7 +1371,7 @@ static int netsec_netdev_init(struct net_device *ndev)
 	if (ret)
 		goto err1;
 
-	ret = netsec_reset_hardware(priv);
+	ret = netsec_reset_hardware(priv, true);
 	if (ret)
 		goto err2;
 
@@ -1674,8 +1681,8 @@ static int netsec_probe(struct platform_device *pdev)
 	if (ret)
 		goto unreg_napi;
 
-	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)))
-		dev_warn(&pdev->dev, "Failed to enable 64-bit DMA\n");
+	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)))
+		dev_warn(&pdev->dev, "Failed to set DMA mask\n");
 
 	ret = register_netdev(ndev);
 	if (ret) {
diff --git a/drivers/net/ethernet/socionext/sni_ave.c b/drivers/net/ethernet/socionext/sni_ave.c
index 0b3b7a460641..f7ecceeb1e28 100644
--- a/drivers/net/ethernet/socionext/sni_ave.c
+++ b/drivers/net/ethernet/socionext/sni_ave.c
@@ -11,6 +11,7 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/iopoll.h>
+#include <linux/mfd/syscon.h>
 #include <linux/mii.h>
 #include <linux/module.h>
 #include <linux/netdevice.h>
@@ -18,6 +19,7 @@
 #include <linux/of_mdio.h>
 #include <linux/of_platform.h>
 #include <linux/phy.h>
+#include <linux/regmap.h>
 #include <linux/reset.h>
 #include <linux/types.h>
 #include <linux/u64_stats_sync.h>
@@ -197,8 +199,16 @@
 #define AVE_INTM_COUNT		20
 #define AVE_FORCE_TXINTCNT	1
 
+/* SG */
+#define SG_ETPINMODE		0x540
+#define SG_ETPINMODE_EXTPHY	BIT(1)	/* for LD11 */
+#define SG_ETPINMODE_RMII(ins)	BIT(ins)
+
 #define IS_DESC_64BIT(p)	((p)->data->is_desc_64bit)
 
+#define AVE_MAX_CLKS		4
+#define AVE_MAX_RSTS		2
+
 enum desc_id {
 	AVE_DESCID_RX,
 	AVE_DESCID_TX,
@@ -225,10 +235,6 @@ struct ave_desc_info {
 	struct ave_desc *desc;	/* skb info related descriptor */
 };
 
-struct ave_soc_data {
-	bool	is_desc_64bit;
-};
-
 struct ave_stats {
 	struct	u64_stats_sync	syncp;
 	u64	packets;
@@ -245,11 +251,16 @@ struct ave_private {
 	int			phy_id;
 	unsigned int		desc_size;
 	u32			msg_enable;
-	struct clk		*clk;
-	struct reset_control	*rst;
+	int			nclks;
+	struct clk		*clk[AVE_MAX_CLKS];
+	int			nrsts;
+	struct reset_control	*rst[AVE_MAX_RSTS];
 	phy_interface_t		phy_mode;
 	struct phy_device	*phydev;
 	struct mii_bus		*mdio;
+	struct regmap		*regmap;
+	unsigned int		pinmode_mask;
+	unsigned int		pinmode_val;
 
 	/* stats */
 	struct ave_stats	stats_rx;
@@ -272,6 +283,14 @@ struct ave_private {
 	const struct ave_soc_data *data;
 };
 
+struct ave_soc_data {
+	bool	is_desc_64bit;
+	const char	*clock_names[AVE_MAX_CLKS];
+	const char	*reset_names[AVE_MAX_RSTS];
+	int	(*get_pinmode)(struct ave_private *priv,
+			       phy_interface_t phy_mode, u32 arg);
+};
+
 static u32 ave_desc_read(struct net_device *ndev, enum desc_id id, int entry,
 			 int offset)
 {
@@ -1153,20 +1172,30 @@ static int ave_init(struct net_device *ndev)
 	struct device_node *np = dev->of_node;
 	struct device_node *mdio_np;
 	struct phy_device *phydev;
-	int ret;
+	int nc, nr, ret;
 
 	/* enable clk because of hw access until ndo_open */
-	ret = clk_prepare_enable(priv->clk);
-	if (ret) {
-		dev_err(dev, "can't enable clock\n");
-		return ret;
+	for (nc = 0; nc < priv->nclks; nc++) {
+		ret = clk_prepare_enable(priv->clk[nc]);
+		if (ret) {
+			dev_err(dev, "can't enable clock\n");
+			goto out_clk_disable;
+		}
 	}
-	ret = reset_control_deassert(priv->rst);
-	if (ret) {
-		dev_err(dev, "can't deassert reset\n");
-		goto out_clk_disable;
+
+	for (nr = 0; nr < priv->nrsts; nr++) {
+		ret = reset_control_deassert(priv->rst[nr]);
+		if (ret) {
+			dev_err(dev, "can't deassert reset\n");
+			goto out_reset_assert;
+		}
 	}
 
+	ret = regmap_update_bits(priv->regmap, SG_ETPINMODE,
+				 priv->pinmode_mask, priv->pinmode_val);
+	if (ret)
+		return ret;
+
 	ave_global_reset(ndev);
 
 	mdio_np = of_get_child_by_name(np, "mdio");
@@ -1207,9 +1236,11 @@ static int ave_init(struct net_device *ndev)
 out_mdio_unregister:
 	mdiobus_unregister(priv->mdio);
 out_reset_assert:
-	reset_control_assert(priv->rst);
+	while (--nr >= 0)
+		reset_control_assert(priv->rst[nr]);
 out_clk_disable:
-	clk_disable_unprepare(priv->clk);
+	while (--nc >= 0)
+		clk_disable_unprepare(priv->clk[nc]);
 
 	return ret;
 }
@@ -1217,13 +1248,16 @@ static int ave_init(struct net_device *ndev)
 static void ave_uninit(struct net_device *ndev)
 {
 	struct ave_private *priv = netdev_priv(ndev);
+	int i;
 
 	phy_disconnect(priv->phydev);
 	mdiobus_unregister(priv->mdio);
 
 	/* disable clk because of hw access after ndo_stop */
-	reset_control_assert(priv->rst);
-	clk_disable_unprepare(priv->clk);
+	for (i = 0; i < priv->nrsts; i++)
+		reset_control_assert(priv->rst[i]);
+	for (i = 0; i < priv->nclks; i++)
+		clk_disable_unprepare(priv->clk[i]);
 }
 
 static int ave_open(struct net_device *ndev)
@@ -1520,6 +1554,7 @@ static int ave_probe(struct platform_device *pdev)
 	const struct ave_soc_data *data;
 	struct device *dev = &pdev->dev;
 	char buf[ETHTOOL_FWVERS_LEN];
+	struct of_phandle_args args;
 	phy_interface_t phy_mode;
 	struct ave_private *priv;
 	struct net_device *ndev;
@@ -1527,8 +1562,9 @@ static int ave_probe(struct platform_device *pdev)
 	struct resource	*res;
 	const void *mac_addr;
 	void __iomem *base;
+	const char *name;
+	int i, irq, ret;
 	u64 dma_mask;
-	int irq, ret;
 	u32 ave_id;
 
 	data = of_device_get_match_data(dev);
@@ -1541,12 +1577,6 @@ static int ave_probe(struct platform_device *pdev)
 		dev_err(dev, "phy-mode not found\n");
 		return -EINVAL;
 	}
-	if ((!phy_interface_mode_is_rgmii(phy_mode)) &&
-	    phy_mode != PHY_INTERFACE_MODE_RMII &&
-	    phy_mode != PHY_INTERFACE_MODE_MII) {
-		dev_err(dev, "phy-mode is invalid\n");
-		return -EINVAL;
-	}
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
@@ -1614,15 +1644,47 @@ static int ave_probe(struct platform_device *pdev)
 	u64_stats_init(&priv->stats_tx.syncp);
 	u64_stats_init(&priv->stats_rx.syncp);
 
-	priv->clk = devm_clk_get(dev, NULL);
-	if (IS_ERR(priv->clk)) {
-		ret = PTR_ERR(priv->clk);
-		goto out_free_netdev;
+	for (i = 0; i < AVE_MAX_CLKS; i++) {
+		name = priv->data->clock_names[i];
+		if (!name)
+			break;
+		priv->clk[i] = devm_clk_get(dev, name);
+		if (IS_ERR(priv->clk[i])) {
+			ret = PTR_ERR(priv->clk[i]);
+			goto out_free_netdev;
+		}
+		priv->nclks++;
+	}
+
+	for (i = 0; i < AVE_MAX_RSTS; i++) {
+		name = priv->data->reset_names[i];
+		if (!name)
+			break;
+		priv->rst[i] = devm_reset_control_get_shared(dev, name);
+		if (IS_ERR(priv->rst[i])) {
+			ret = PTR_ERR(priv->rst[i]);
+			goto out_free_netdev;
+		}
+		priv->nrsts++;
 	}
 
-	priv->rst = devm_reset_control_get_optional_shared(dev, NULL);
-	if (IS_ERR(priv->rst)) {
-		ret = PTR_ERR(priv->rst);
+	ret = of_parse_phandle_with_fixed_args(np,
+					       "socionext,syscon-phy-mode",
+					       1, 0, &args);
+	if (ret) {
+		netdev_err(ndev, "can't get syscon-phy-mode property\n");
+		goto out_free_netdev;
+	}
+	priv->regmap = syscon_node_to_regmap(args.np);
+	of_node_put(args.np);
+	if (IS_ERR(priv->regmap)) {
+		netdev_err(ndev, "can't map syscon-phy-mode\n");
+		ret = PTR_ERR(priv->regmap);
+		goto out_free_netdev;
+	}
+	ret = priv->data->get_pinmode(priv, phy_mode, args.args[0]);
+	if (ret) {
+		netdev_err(ndev, "invalid phy-mode setting\n");
 		goto out_free_netdev;
 	}
 
@@ -1685,24 +1747,148 @@ static int ave_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static int ave_pro4_get_pinmode(struct ave_private *priv,
+				phy_interface_t phy_mode, u32 arg)
+{
+	if (arg > 0)
+		return -EINVAL;
+
+	priv->pinmode_mask = SG_ETPINMODE_RMII(0);
+
+	switch (phy_mode) {
+	case PHY_INTERFACE_MODE_RMII:
+		priv->pinmode_val = SG_ETPINMODE_RMII(0);
+		break;
+	case PHY_INTERFACE_MODE_MII:
+	case PHY_INTERFACE_MODE_RGMII:
+		priv->pinmode_val = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ave_ld11_get_pinmode(struct ave_private *priv,
+				phy_interface_t phy_mode, u32 arg)
+{
+	if (arg > 0)
+		return -EINVAL;
+
+	priv->pinmode_mask = SG_ETPINMODE_EXTPHY | SG_ETPINMODE_RMII(0);
+
+	switch (phy_mode) {
+	case PHY_INTERFACE_MODE_INTERNAL:
+		priv->pinmode_val = 0;
+		break;
+	case PHY_INTERFACE_MODE_RMII:
+		priv->pinmode_val = SG_ETPINMODE_EXTPHY | SG_ETPINMODE_RMII(0);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ave_ld20_get_pinmode(struct ave_private *priv,
+				phy_interface_t phy_mode, u32 arg)
+{
+	if (arg > 0)
+		return -EINVAL;
+
+	priv->pinmode_mask = SG_ETPINMODE_RMII(0);
+
+	switch (phy_mode) {
+	case PHY_INTERFACE_MODE_RMII:
+		priv->pinmode_val = SG_ETPINMODE_RMII(0);
+		break;
+	case PHY_INTERFACE_MODE_RGMII:
+		priv->pinmode_val = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ave_pxs3_get_pinmode(struct ave_private *priv,
+				phy_interface_t phy_mode, u32 arg)
+{
+	if (arg > 1)
+		return -EINVAL;
+
+	priv->pinmode_mask = SG_ETPINMODE_RMII(arg);
+
+	switch (phy_mode) {
+	case PHY_INTERFACE_MODE_RMII:
+		priv->pinmode_val = SG_ETPINMODE_RMII(arg);
+		break;
+	case PHY_INTERFACE_MODE_RGMII:
+		priv->pinmode_val = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static const struct ave_soc_data ave_pro4_data = {
 	.is_desc_64bit = false,
+	.clock_names = {
+		"gio", "ether", "ether-gb", "ether-phy",
+	},
+	.reset_names = {
+		"gio", "ether",
+	},
+	.get_pinmode = ave_pro4_get_pinmode,
 };
 
 static const struct ave_soc_data ave_pxs2_data = {
 	.is_desc_64bit = false,
+	.clock_names = {
+		"ether",
+	},
+	.reset_names = {
+		"ether",
+	},
+	.get_pinmode = ave_pro4_get_pinmode,
 };
 
 static const struct ave_soc_data ave_ld11_data = {
 	.is_desc_64bit = false,
+	.clock_names = {
+		"ether",
+	},
+	.reset_names = {
+		"ether",
+	},
+	.get_pinmode = ave_ld11_get_pinmode,
 };
 
 static const struct ave_soc_data ave_ld20_data = {
 	.is_desc_64bit = true,
+	.clock_names = {
+		"ether",
+	},
+	.reset_names = {
+		"ether",
+	},
+	.get_pinmode = ave_ld20_get_pinmode,
 };
 
 static const struct ave_soc_data ave_pxs3_data = {
 	.is_desc_64bit = false,
+	.clock_names = {
+		"ether",
+	},
+	.reset_names = {
+		"ether",
+	},
+	.get_pinmode = ave_pxs3_get_pinmode,
 };
 
 static const struct of_device_id of_ave_match[] = {
diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile
index 972e4ef6d414..68e9e2640c62 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Makefile
+++ b/drivers/net/ethernet/stmicro/stmmac/Makefile
@@ -4,7 +4,8 @@ stmmac-objs:= stmmac_main.o stmmac_ethtool.o stmmac_mdio.o ring_mode.o	\
 	      chain_mode.o dwmac_lib.o dwmac1000_core.o dwmac1000_dma.o	\
 	      dwmac100_core.o dwmac100_dma.o enh_desc.o norm_desc.o	\
 	      mmc_core.o stmmac_hwtstamp.o stmmac_ptp.o dwmac4_descs.o	\
-	      dwmac4_dma.o dwmac4_lib.o dwmac4_core.o dwmac5.o $(stmmac-y)
+	      dwmac4_dma.o dwmac4_lib.o dwmac4_core.o dwmac5.o hwif.o \
+	      stmmac_tc.o $(stmmac-y)
 
 # Ordering matters. Generic driver must be last.
 obj-$(CONFIG_STMMAC_PLATFORM)	+= stmmac-platform.o
diff --git a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
index e93c40b4631e..b9c9003060c5 100644
--- a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
+++ b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
@@ -24,7 +24,7 @@
 
 #include "stmmac.h"
 
-static int stmmac_jumbo_frm(void *p, struct sk_buff *skb, int csum)
+static int jumbo_frm(void *p, struct sk_buff *skb, int csum)
 {
 	struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)p;
 	unsigned int nopaged_len = skb_headlen(skb);
@@ -51,8 +51,8 @@ static int stmmac_jumbo_frm(void *p, struct sk_buff *skb, int csum)
 	tx_q->tx_skbuff_dma[entry].buf = des2;
 	tx_q->tx_skbuff_dma[entry].len = bmax;
 	/* do not close the descriptor and do not set own bit */
-	priv->hw->desc->prepare_tx_desc(desc, 1, bmax, csum, STMMAC_CHAIN_MODE,
-					0, false, skb->len);
+	stmmac_prepare_tx_desc(priv, desc, 1, bmax, csum, STMMAC_CHAIN_MODE,
+			0, false, skb->len);
 
 	while (len != 0) {
 		tx_q->tx_skbuff[entry] = NULL;
@@ -68,9 +68,8 @@ static int stmmac_jumbo_frm(void *p, struct sk_buff *skb, int csum)
 				return -1;
 			tx_q->tx_skbuff_dma[entry].buf = des2;
 			tx_q->tx_skbuff_dma[entry].len = bmax;
-			priv->hw->desc->prepare_tx_desc(desc, 0, bmax, csum,
-							STMMAC_CHAIN_MODE, 1,
-							false, skb->len);
+			stmmac_prepare_tx_desc(priv, desc, 0, bmax, csum,
+					STMMAC_CHAIN_MODE, 1, false, skb->len);
 			len -= bmax;
 			i++;
 		} else {
@@ -83,9 +82,8 @@ static int stmmac_jumbo_frm(void *p, struct sk_buff *skb, int csum)
 			tx_q->tx_skbuff_dma[entry].buf = des2;
 			tx_q->tx_skbuff_dma[entry].len = len;
 			/* last descriptor can be set now */
-			priv->hw->desc->prepare_tx_desc(desc, 0, len, csum,
-							STMMAC_CHAIN_MODE, 1,
-							true, skb->len);
+			stmmac_prepare_tx_desc(priv, desc, 0, len, csum,
+					STMMAC_CHAIN_MODE, 1, true, skb->len);
 			len = 0;
 		}
 	}
@@ -95,7 +93,7 @@ static int stmmac_jumbo_frm(void *p, struct sk_buff *skb, int csum)
 	return entry;
 }
 
-static unsigned int stmmac_is_jumbo_frm(int len, int enh_desc)
+static unsigned int is_jumbo_frm(int len, int enh_desc)
 {
 	unsigned int ret = 0;
 
@@ -107,7 +105,7 @@ static unsigned int stmmac_is_jumbo_frm(int len, int enh_desc)
 	return ret;
 }
 
-static void stmmac_init_dma_chain(void *des, dma_addr_t phy_addr,
+static void init_dma_chain(void *des, dma_addr_t phy_addr,
 				  unsigned int size, unsigned int extend_desc)
 {
 	/*
@@ -137,7 +135,7 @@ static void stmmac_init_dma_chain(void *des, dma_addr_t phy_addr,
 	}
 }
 
-static void stmmac_refill_desc3(void *priv_ptr, struct dma_desc *p)
+static void refill_desc3(void *priv_ptr, struct dma_desc *p)
 {
 	struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)priv_ptr;
 	struct stmmac_priv *priv = rx_q->priv_data;
@@ -153,7 +151,7 @@ static void stmmac_refill_desc3(void *priv_ptr, struct dma_desc *p)
 				      sizeof(struct dma_desc)));
 }
 
-static void stmmac_clean_desc3(void *priv_ptr, struct dma_desc *p)
+static void clean_desc3(void *priv_ptr, struct dma_desc *p)
 {
 	struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)priv_ptr;
 	struct stmmac_priv *priv = tx_q->priv_data;
@@ -171,9 +169,9 @@ static void stmmac_clean_desc3(void *priv_ptr, struct dma_desc *p)
 }
 
 const struct stmmac_mode_ops chain_mode_ops = {
-	.init = stmmac_init_dma_chain,
-	.is_jumbo_frm = stmmac_is_jumbo_frm,
-	.jumbo_frm = stmmac_jumbo_frm,
-	.refill_desc3 = stmmac_refill_desc3,
-	.clean_desc3 = stmmac_clean_desc3,
+	.init = init_dma_chain,
+	.is_jumbo_frm = is_jumbo_frm,
+	.jumbo_frm = jumbo_frm,
+	.refill_desc3 = refill_desc3,
+	.clean_desc3 = clean_desc3,
 };
diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h
index ad2388aee463..a679cb729d1d 100644
--- a/drivers/net/ethernet/stmicro/stmmac/common.h
+++ b/drivers/net/ethernet/stmicro/stmmac/common.h
@@ -32,12 +32,14 @@
 #endif
 
 #include "descs.h"
+#include "hwif.h"
 #include "mmc.h"
 
 /* Synopsys Core versions */
 #define	DWMAC_CORE_3_40	0x34
 #define	DWMAC_CORE_3_50	0x35
 #define	DWMAC_CORE_4_00	0x40
+#define DWMAC_CORE_4_10	0x41
 #define DWMAC_CORE_5_00 0x50
 #define DWMAC_CORE_5_10 0x51
 #define STMMAC_CHAN0	0	/* Always supported and default for all chips */
@@ -351,6 +353,10 @@ struct dma_features {
 	unsigned int rx_fifo_size;
 	/* Automotive Safety Package */
 	unsigned int asp;
+	/* RX Parser */
+	unsigned int frpsel;
+	unsigned int frpbs;
+	unsigned int frpes;
 };
 
 /* GMAC TX FIFO is 8K, Rx FIFO is 16K */
@@ -377,197 +383,11 @@ struct dma_features {
 
 #define JUMBO_LEN		9000
 
-/* Descriptors helpers */
-struct stmmac_desc_ops {
-	/* DMA RX descriptor ring initialization */
-	void (*init_rx_desc) (struct dma_desc *p, int disable_rx_ic, int mode,
-			      int end);
-	/* DMA TX descriptor ring initialization */
-	void (*init_tx_desc) (struct dma_desc *p, int mode, int end);
-
-	/* Invoked by the xmit function to prepare the tx descriptor */
-	void (*prepare_tx_desc) (struct dma_desc *p, int is_fs, int len,
-				 bool csum_flag, int mode, bool tx_own,
-				 bool ls, unsigned int tot_pkt_len);
-	void (*prepare_tso_tx_desc)(struct dma_desc *p, int is_fs, int len1,
-				    int len2, bool tx_own, bool ls,
-				    unsigned int tcphdrlen,
-				    unsigned int tcppayloadlen);
-	/* Set/get the owner of the descriptor */
-	void (*set_tx_owner) (struct dma_desc *p);
-	int (*get_tx_owner) (struct dma_desc *p);
-	/* Clean the tx descriptor as soon as the tx irq is received */
-	void (*release_tx_desc) (struct dma_desc *p, int mode);
-	/* Clear interrupt on tx frame completion. When this bit is
-	 * set an interrupt happens as soon as the frame is transmitted */
-	void (*set_tx_ic)(struct dma_desc *p);
-	/* Last tx segment reports the transmit status */
-	int (*get_tx_ls) (struct dma_desc *p);
-	/* Return the transmit status looking at the TDES1 */
-	int (*tx_status) (void *data, struct stmmac_extra_stats *x,
-			  struct dma_desc *p, void __iomem *ioaddr);
-	/* Get the buffer size from the descriptor */
-	int (*get_tx_len) (struct dma_desc *p);
-	/* Handle extra events on specific interrupts hw dependent */
-	void (*set_rx_owner) (struct dma_desc *p);
-	/* Get the receive frame size */
-	int (*get_rx_frame_len) (struct dma_desc *p, int rx_coe_type);
-	/* Return the reception status looking at the RDES1 */
-	int (*rx_status) (void *data, struct stmmac_extra_stats *x,
-			  struct dma_desc *p);
-	void (*rx_extended_status) (void *data, struct stmmac_extra_stats *x,
-				    struct dma_extended_desc *p);
-	/* Set tx timestamp enable bit */
-	void (*enable_tx_timestamp) (struct dma_desc *p);
-	/* get tx timestamp status */
-	int (*get_tx_timestamp_status) (struct dma_desc *p);
-	/* get timestamp value */
-	 u64(*get_timestamp) (void *desc, u32 ats);
-	/* get rx timestamp status */
-	int (*get_rx_timestamp_status)(void *desc, void *next_desc, u32 ats);
-	/* Display ring */
-	void (*display_ring)(void *head, unsigned int size, bool rx);
-	/* set MSS via context descriptor */
-	void (*set_mss)(struct dma_desc *p, unsigned int mss);
-};
-
 extern const struct stmmac_desc_ops enh_desc_ops;
 extern const struct stmmac_desc_ops ndesc_ops;
 
-/* Specific DMA helpers */
-struct stmmac_dma_ops {
-	/* DMA core initialization */
-	int (*reset)(void __iomem *ioaddr);
-	void (*init)(void __iomem *ioaddr, struct stmmac_dma_cfg *dma_cfg,
-		     u32 dma_tx, u32 dma_rx, int atds);
-	void (*init_chan)(void __iomem *ioaddr,
-			  struct stmmac_dma_cfg *dma_cfg, u32 chan);
-	void (*init_rx_chan)(void __iomem *ioaddr,
-			     struct stmmac_dma_cfg *dma_cfg,
-			     u32 dma_rx_phy, u32 chan);
-	void (*init_tx_chan)(void __iomem *ioaddr,
-			     struct stmmac_dma_cfg *dma_cfg,
-			     u32 dma_tx_phy, u32 chan);
-	/* Configure the AXI Bus Mode Register */
-	void (*axi)(void __iomem *ioaddr, struct stmmac_axi *axi);
-	/* Dump DMA registers */
-	void (*dump_regs)(void __iomem *ioaddr, u32 *reg_space);
-	/* Set tx/rx threshold in the csr6 register
-	 * An invalid value enables the store-and-forward mode */
-	void (*dma_mode)(void __iomem *ioaddr, int txmode, int rxmode,
-			 int rxfifosz);
-	void (*dma_rx_mode)(void __iomem *ioaddr, int mode, u32 channel,
-			    int fifosz, u8 qmode);
-	void (*dma_tx_mode)(void __iomem *ioaddr, int mode, u32 channel,
-			    int fifosz, u8 qmode);
-	/* To track extra statistic (if supported) */
-	void (*dma_diagnostic_fr) (void *data, struct stmmac_extra_stats *x,
-				   void __iomem *ioaddr);
-	void (*enable_dma_transmission) (void __iomem *ioaddr);
-	void (*enable_dma_irq)(void __iomem *ioaddr, u32 chan);
-	void (*disable_dma_irq)(void __iomem *ioaddr, u32 chan);
-	void (*start_tx)(void __iomem *ioaddr, u32 chan);
-	void (*stop_tx)(void __iomem *ioaddr, u32 chan);
-	void (*start_rx)(void __iomem *ioaddr, u32 chan);
-	void (*stop_rx)(void __iomem *ioaddr, u32 chan);
-	int (*dma_interrupt) (void __iomem *ioaddr,
-			      struct stmmac_extra_stats *x, u32 chan);
-	/* If supported then get the optional core features */
-	void (*get_hw_feature)(void __iomem *ioaddr,
-			       struct dma_features *dma_cap);
-	/* Program the HW RX Watchdog */
-	void (*rx_watchdog)(void __iomem *ioaddr, u32 riwt, u32 number_chan);
-	void (*set_tx_ring_len)(void __iomem *ioaddr, u32 len, u32 chan);
-	void (*set_rx_ring_len)(void __iomem *ioaddr, u32 len, u32 chan);
-	void (*set_rx_tail_ptr)(void __iomem *ioaddr, u32 tail_ptr, u32 chan);
-	void (*set_tx_tail_ptr)(void __iomem *ioaddr, u32 tail_ptr, u32 chan);
-	void (*enable_tso)(void __iomem *ioaddr, bool en, u32 chan);
-};
-
 struct mac_device_info;
 
-/* Helpers to program the MAC core */
-struct stmmac_ops {
-	/* MAC core initialization */
-	void (*core_init)(struct mac_device_info *hw, struct net_device *dev);
-	/* Enable the MAC RX/TX */
-	void (*set_mac)(void __iomem *ioaddr, bool enable);
-	/* Enable and verify that the IPC module is supported */
-	int (*rx_ipc)(struct mac_device_info *hw);
-	/* Enable RX Queues */
-	void (*rx_queue_enable)(struct mac_device_info *hw, u8 mode, u32 queue);
-	/* RX Queues Priority */
-	void (*rx_queue_prio)(struct mac_device_info *hw, u32 prio, u32 queue);
-	/* TX Queues Priority */
-	void (*tx_queue_prio)(struct mac_device_info *hw, u32 prio, u32 queue);
-	/* RX Queues Routing */
-	void (*rx_queue_routing)(struct mac_device_info *hw, u8 packet,
-				 u32 queue);
-	/* Program RX Algorithms */
-	void (*prog_mtl_rx_algorithms)(struct mac_device_info *hw, u32 rx_alg);
-	/* Program TX Algorithms */
-	void (*prog_mtl_tx_algorithms)(struct mac_device_info *hw, u32 tx_alg);
-	/* Set MTL TX queues weight */
-	void (*set_mtl_tx_queue_weight)(struct mac_device_info *hw,
-					u32 weight, u32 queue);
-	/* RX MTL queue to RX dma mapping */
-	void (*map_mtl_to_dma)(struct mac_device_info *hw, u32 queue, u32 chan);
-	/* Configure AV Algorithm */
-	void (*config_cbs)(struct mac_device_info *hw, u32 send_slope,
-			   u32 idle_slope, u32 high_credit, u32 low_credit,
-			   u32 queue);
-	/* Dump MAC registers */
-	void (*dump_regs)(struct mac_device_info *hw, u32 *reg_space);
-	/* Handle extra events on specific interrupts hw dependent */
-	int (*host_irq_status)(struct mac_device_info *hw,
-			       struct stmmac_extra_stats *x);
-	/* Handle MTL interrupts */
-	int (*host_mtl_irq_status)(struct mac_device_info *hw, u32 chan);
-	/* Multicast filter setting */
-	void (*set_filter)(struct mac_device_info *hw, struct net_device *dev);
-	/* Flow control setting */
-	void (*flow_ctrl)(struct mac_device_info *hw, unsigned int duplex,
-			  unsigned int fc, unsigned int pause_time, u32 tx_cnt);
-	/* Set power management mode (e.g. magic frame) */
-	void (*pmt)(struct mac_device_info *hw, unsigned long mode);
-	/* Set/Get Unicast MAC addresses */
-	void (*set_umac_addr)(struct mac_device_info *hw, unsigned char *addr,
-			      unsigned int reg_n);
-	void (*get_umac_addr)(struct mac_device_info *hw, unsigned char *addr,
-			      unsigned int reg_n);
-	void (*set_eee_mode)(struct mac_device_info *hw,
-			     bool en_tx_lpi_clockgating);
-	void (*reset_eee_mode)(struct mac_device_info *hw);
-	void (*set_eee_timer)(struct mac_device_info *hw, int ls, int tw);
-	void (*set_eee_pls)(struct mac_device_info *hw, int link);
-	void (*debug)(void __iomem *ioaddr, struct stmmac_extra_stats *x,
-		      u32 rx_queues, u32 tx_queues);
-	/* PCS calls */
-	void (*pcs_ctrl_ane)(void __iomem *ioaddr, bool ane, bool srgmi_ral,
-			     bool loopback);
-	void (*pcs_rane)(void __iomem *ioaddr, bool restart);
-	void (*pcs_get_adv_lp)(void __iomem *ioaddr, struct rgmii_adv *adv);
-	/* Safety Features */
-	int (*safety_feat_config)(void __iomem *ioaddr, unsigned int asp);
-	bool (*safety_feat_irq_status)(struct net_device *ndev,
-			void __iomem *ioaddr, unsigned int asp,
-			struct stmmac_safety_stats *stats);
-	const char *(*safety_feat_dump)(struct stmmac_safety_stats *stats,
-			int index, unsigned long *count);
-};
-
-/* PTP and HW Timer helpers */
-struct stmmac_hwtimestamp {
-	void (*config_hw_tstamping) (void __iomem *ioaddr, u32 data);
-	u32 (*config_sub_second_increment)(void __iomem *ioaddr, u32 ptp_clock,
-					   int gmac4);
-	int (*init_systime) (void __iomem *ioaddr, u32 sec, u32 nsec);
-	int (*config_addend) (void __iomem *ioaddr, u32 addend);
-	int (*adjust_systime) (void __iomem *ioaddr, u32 sec, u32 nsec,
-			       int add_sub, int gmac4);
-	 u64(*get_systime) (void __iomem *ioaddr);
-};
-
 extern const struct stmmac_hwtimestamp stmmac_ptp;
 extern const struct stmmac_mode_ops dwmac4_ring_mode_ops;
 
@@ -590,24 +410,13 @@ struct mii_regs {
 	unsigned int clk_csr_mask;
 };
 
-/* Helpers to manage the descriptors for chain and ring modes */
-struct stmmac_mode_ops {
-	void (*init) (void *des, dma_addr_t phy_addr, unsigned int size,
-		      unsigned int extend_desc);
-	unsigned int (*is_jumbo_frm) (int len, int ehn_desc);
-	int (*jumbo_frm)(void *priv, struct sk_buff *skb, int csum);
-	int (*set_16kib_bfsize)(int mtu);
-	void (*init_desc3)(struct dma_desc *p);
-	void (*refill_desc3) (void *priv, struct dma_desc *p);
-	void (*clean_desc3) (void *priv, struct dma_desc *p);
-};
-
 struct mac_device_info {
 	const struct stmmac_ops *mac;
 	const struct stmmac_desc_ops *desc;
 	const struct stmmac_dma_ops *dma;
 	const struct stmmac_mode_ops *mode;
 	const struct stmmac_hwtimestamp *ptp;
+	const struct stmmac_tc_ops *tc;
 	struct mii_regs mii;	/* MII register Addresses */
 	struct mac_link link;
 	void __iomem *pcsr;     /* vpointer to device CSRs */
@@ -625,12 +434,9 @@ struct stmmac_rx_routing {
 	u32 reg_shift;
 };
 
-struct mac_device_info *dwmac1000_setup(void __iomem *ioaddr, int mcbins,
-					int perfect_uc_entries,
-					int *synopsys_id);
-struct mac_device_info *dwmac100_setup(void __iomem *ioaddr, int *synopsys_id);
-struct mac_device_info *dwmac4_setup(void __iomem *ioaddr, int mcbins,
-				     int perfect_uc_entries, int *synopsys_id);
+int dwmac100_setup(struct stmmac_priv *priv);
+int dwmac1000_setup(struct stmmac_priv *priv);
+int dwmac4_setup(struct stmmac_priv *priv);
 
 void stmmac_set_mac_addr(void __iomem *ioaddr, u8 addr[6],
 			 unsigned int high, unsigned int low);
@@ -650,24 +456,4 @@ extern const struct stmmac_mode_ops ring_mode_ops;
 extern const struct stmmac_mode_ops chain_mode_ops;
 extern const struct stmmac_desc_ops dwmac4_desc_ops;
 
-/**
- * stmmac_get_synopsys_id - return the SYINID.
- * @priv: driver private structure
- * Description: this simple function is to decode and return the SYINID
- * starting from the HW core register.
- */
-static inline u32 stmmac_get_synopsys_id(u32 hwid)
-{
-	/* Check Synopsys Id (not available on old chips) */
-	if (likely(hwid)) {
-		u32 uid = ((hwid & 0x0000ff00) >> 8);
-		u32 synid = (hwid & 0x000000ff);
-
-		pr_info("stmmac - user ID: 0x%x, Synopsys ID: 0x%x\n",
-			uid, synid);
-
-		return synid;
-	}
-	return 0;
-}
 #endif /* __COMMON_H__ */
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
index 7cb794094a70..4ff231df7322 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
@@ -18,6 +18,7 @@
 #include <linux/io.h>
 #include <linux/ioport.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/of_net.h>
 #include <linux/mfd/syscon.h>
 #include <linux/platform_device.h>
@@ -29,6 +30,10 @@
 
 #define PRG_ETH0_RGMII_MODE		BIT(0)
 
+#define PRG_ETH0_EXT_PHY_MODE_MASK	GENMASK(2, 0)
+#define PRG_ETH0_EXT_RGMII_MODE		1
+#define PRG_ETH0_EXT_RMII_MODE		4
+
 /* mux to choose between fclk_div2 (bit unset) and mpll2 (bit set) */
 #define PRG_ETH0_CLK_M250_SEL_SHIFT	4
 #define PRG_ETH0_CLK_M250_SEL_MASK	GENMASK(4, 4)
@@ -47,12 +52,20 @@
 
 #define MUX_CLK_NUM_PARENTS		2
 
+struct meson8b_dwmac;
+
+struct meson8b_dwmac_data {
+	int (*set_phy_mode)(struct meson8b_dwmac *dwmac);
+};
+
 struct meson8b_dwmac {
-	struct device		*dev;
-	void __iomem		*regs;
-	phy_interface_t		phy_mode;
-	struct clk		*rgmii_tx_clk;
-	u32			tx_delay_ns;
+	struct device			*dev;
+	void __iomem			*regs;
+
+	const struct meson8b_dwmac_data	*data;
+	phy_interface_t			phy_mode;
+	struct clk			*rgmii_tx_clk;
+	u32				tx_delay_ns;
 };
 
 struct meson8b_dwmac_clk_configs {
@@ -171,6 +184,59 @@ static int meson8b_init_rgmii_tx_clk(struct meson8b_dwmac *dwmac)
 	return 0;
 }
 
+static int meson8b_set_phy_mode(struct meson8b_dwmac *dwmac)
+{
+	switch (dwmac->phy_mode) {
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		/* enable RGMII mode */
+		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0,
+					PRG_ETH0_RGMII_MODE,
+					PRG_ETH0_RGMII_MODE);
+		break;
+	case PHY_INTERFACE_MODE_RMII:
+		/* disable RGMII mode -> enables RMII mode */
+		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0,
+					PRG_ETH0_RGMII_MODE, 0);
+		break;
+	default:
+		dev_err(dwmac->dev, "fail to set phy-mode %s\n",
+			phy_modes(dwmac->phy_mode));
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int meson_axg_set_phy_mode(struct meson8b_dwmac *dwmac)
+{
+	switch (dwmac->phy_mode) {
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		/* enable RGMII mode */
+		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0,
+					PRG_ETH0_EXT_PHY_MODE_MASK,
+					PRG_ETH0_EXT_RGMII_MODE);
+		break;
+	case PHY_INTERFACE_MODE_RMII:
+		/* disable RGMII mode -> enables RMII mode */
+		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0,
+					PRG_ETH0_EXT_PHY_MODE_MASK,
+					PRG_ETH0_EXT_RMII_MODE);
+		break;
+	default:
+		dev_err(dwmac->dev, "fail to set phy-mode %s\n",
+			phy_modes(dwmac->phy_mode));
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int meson8b_init_prg_eth(struct meson8b_dwmac *dwmac)
 {
 	int ret;
@@ -188,10 +254,6 @@ static int meson8b_init_prg_eth(struct meson8b_dwmac *dwmac)
 
 	case PHY_INTERFACE_MODE_RGMII_ID:
 	case PHY_INTERFACE_MODE_RGMII_TXID:
-		/* enable RGMII mode */
-		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_RGMII_MODE,
-					PRG_ETH0_RGMII_MODE);
-
 		/* only relevant for RMII mode -> disable in RGMII mode */
 		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0,
 					PRG_ETH0_INVERTED_RMII_CLK, 0);
@@ -224,10 +286,6 @@ static int meson8b_init_prg_eth(struct meson8b_dwmac *dwmac)
 		break;
 
 	case PHY_INTERFACE_MODE_RMII:
-		/* disable RGMII mode -> enables RMII mode */
-		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_RGMII_MODE,
-					0);
-
 		/* invert internal clk_rmii_i to generate 25/2.5 tx_rx_clk */
 		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0,
 					PRG_ETH0_INVERTED_RMII_CLK,
@@ -274,6 +332,11 @@ static int meson8b_dwmac_probe(struct platform_device *pdev)
 		goto err_remove_config_dt;
 	}
 
+	dwmac->data = (const struct meson8b_dwmac_data *)
+		of_device_get_match_data(&pdev->dev);
+	if (!dwmac->data)
+		return -EINVAL;
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 	dwmac->regs = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(dwmac->regs)) {
@@ -298,6 +361,10 @@ static int meson8b_dwmac_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_remove_config_dt;
 
+	ret = dwmac->data->set_phy_mode(dwmac);
+	if (ret)
+		goto err_remove_config_dt;
+
 	ret = meson8b_init_prg_eth(dwmac);
 	if (ret)
 		goto err_remove_config_dt;
@@ -316,10 +383,31 @@ static int meson8b_dwmac_probe(struct platform_device *pdev)
 	return ret;
 }
 
+static const struct meson8b_dwmac_data meson8b_dwmac_data = {
+	.set_phy_mode = meson8b_set_phy_mode,
+};
+
+static const struct meson8b_dwmac_data meson_axg_dwmac_data = {
+	.set_phy_mode = meson_axg_set_phy_mode,
+};
+
 static const struct of_device_id meson8b_dwmac_match[] = {
-	{ .compatible = "amlogic,meson8b-dwmac" },
-	{ .compatible = "amlogic,meson8m2-dwmac" },
-	{ .compatible = "amlogic,meson-gxbb-dwmac" },
+	{
+		.compatible = "amlogic,meson8b-dwmac",
+		.data = &meson8b_dwmac_data,
+	},
+	{
+		.compatible = "amlogic,meson8m2-dwmac",
+		.data = &meson8b_dwmac_data,
+	},
+	{
+		.compatible = "amlogic,meson-gxbb-dwmac",
+		.data = &meson8b_dwmac_data,
+	},
+	{
+		.compatible = "amlogic,meson-axg-dwmac",
+		.data = &meson_axg_dwmac_data,
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(of, meson8b_dwmac_match);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
index 13133b30b575..f08625a02cea 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
@@ -1104,30 +1104,20 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable)
 	} else {
 		if (bsp_priv->clk_enabled) {
 			if (phy_iface == PHY_INTERFACE_MODE_RMII) {
-				if (!IS_ERR(bsp_priv->mac_clk_rx))
-					clk_disable_unprepare(
-						bsp_priv->mac_clk_rx);
+				clk_disable_unprepare(bsp_priv->mac_clk_rx);
 
-				if (!IS_ERR(bsp_priv->clk_mac_ref))
-					clk_disable_unprepare(
-						bsp_priv->clk_mac_ref);
+				clk_disable_unprepare(bsp_priv->clk_mac_ref);
 
-				if (!IS_ERR(bsp_priv->clk_mac_refout))
-					clk_disable_unprepare(
-						bsp_priv->clk_mac_refout);
+				clk_disable_unprepare(bsp_priv->clk_mac_refout);
 			}
 
-			if (!IS_ERR(bsp_priv->clk_phy))
-				clk_disable_unprepare(bsp_priv->clk_phy);
+			clk_disable_unprepare(bsp_priv->clk_phy);
 
-			if (!IS_ERR(bsp_priv->aclk_mac))
-				clk_disable_unprepare(bsp_priv->aclk_mac);
+			clk_disable_unprepare(bsp_priv->aclk_mac);
 
-			if (!IS_ERR(bsp_priv->pclk_mac))
-				clk_disable_unprepare(bsp_priv->pclk_mac);
+			clk_disable_unprepare(bsp_priv->pclk_mac);
 
-			if (!IS_ERR(bsp_priv->mac_clk_tx))
-				clk_disable_unprepare(bsp_priv->mac_clk_tx);
+			clk_disable_unprepare(bsp_priv->mac_clk_tx);
 			/**
 			 * if (!IS_ERR(bsp_priv->clk_mac))
 			 *	clk_disable_unprepare(bsp_priv->clk_mac);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c
index 9e6db16af663..7e2e79dedebf 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c
@@ -16,49 +16,180 @@
 #include <linux/of_net.h>
 #include <linux/phy.h>
 #include <linux/platform_device.h>
+#include <linux/pm_wakeirq.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
 #include <linux/stmmac.h>
 
 #include "stmmac_platform.h"
 
-#define MII_PHY_SEL_MASK	BIT(23)
+#define SYSCFG_MCU_ETH_MASK		BIT(23)
+#define SYSCFG_MP1_ETH_MASK		GENMASK(23, 16)
+
+#define SYSCFG_PMCR_ETH_CLK_SEL		BIT(16)
+#define SYSCFG_PMCR_ETH_REF_CLK_SEL	BIT(17)
+#define SYSCFG_PMCR_ETH_SEL_MII		BIT(20)
+#define SYSCFG_PMCR_ETH_SEL_RGMII	BIT(21)
+#define SYSCFG_PMCR_ETH_SEL_RMII	BIT(23)
+#define SYSCFG_PMCR_ETH_SEL_GMII	0
+#define SYSCFG_MCU_ETH_SEL_MII		0
+#define SYSCFG_MCU_ETH_SEL_RMII		1
 
 struct stm32_dwmac {
 	struct clk *clk_tx;
 	struct clk *clk_rx;
+	struct clk *clk_eth_ck;
+	struct clk *clk_ethstp;
+	struct clk *syscfg_clk;
+	bool int_phyclk;	/* Clock from RCC to drive PHY */
 	u32 mode_reg;		/* MAC glue-logic mode register */
 	struct regmap *regmap;
 	u32 speed;
+	const struct stm32_ops *ops;
+	struct device *dev;
+};
+
+struct stm32_ops {
+	int (*set_mode)(struct plat_stmmacenet_data *plat_dat);
+	int (*clk_prepare)(struct stm32_dwmac *dwmac, bool prepare);
+	int (*suspend)(struct stm32_dwmac *dwmac);
+	void (*resume)(struct stm32_dwmac *dwmac);
+	int (*parse_data)(struct stm32_dwmac *dwmac,
+			  struct device *dev);
+	u32 syscfg_eth_mask;
 };
 
 static int stm32_dwmac_init(struct plat_stmmacenet_data *plat_dat)
 {
 	struct stm32_dwmac *dwmac = plat_dat->bsp_priv;
-	u32 reg = dwmac->mode_reg;
-	u32 val;
 	int ret;
 
-	val = (plat_dat->interface == PHY_INTERFACE_MODE_MII) ? 0 : 1;
-	ret = regmap_update_bits(dwmac->regmap, reg, MII_PHY_SEL_MASK, val);
-	if (ret)
-		return ret;
+	if (dwmac->ops->set_mode) {
+		ret = dwmac->ops->set_mode(plat_dat);
+		if (ret)
+			return ret;
+	}
 
 	ret = clk_prepare_enable(dwmac->clk_tx);
 	if (ret)
 		return ret;
 
-	ret = clk_prepare_enable(dwmac->clk_rx);
-	if (ret)
-		clk_disable_unprepare(dwmac->clk_tx);
+	if (!dwmac->dev->power.is_suspended) {
+		ret = clk_prepare_enable(dwmac->clk_rx);
+		if (ret) {
+			clk_disable_unprepare(dwmac->clk_tx);
+			return ret;
+		}
+	}
+
+	if (dwmac->ops->clk_prepare) {
+		ret = dwmac->ops->clk_prepare(dwmac, true);
+		if (ret) {
+			clk_disable_unprepare(dwmac->clk_rx);
+			clk_disable_unprepare(dwmac->clk_tx);
+		}
+	}
 
 	return ret;
 }
 
+static int stm32mp1_clk_prepare(struct stm32_dwmac *dwmac, bool prepare)
+{
+	int ret = 0;
+
+	if (prepare) {
+		ret = clk_prepare_enable(dwmac->syscfg_clk);
+		if (ret)
+			return ret;
+
+		if (dwmac->int_phyclk) {
+			ret = clk_prepare_enable(dwmac->clk_eth_ck);
+			if (ret) {
+				clk_disable_unprepare(dwmac->syscfg_clk);
+				return ret;
+			}
+		}
+	} else {
+		clk_disable_unprepare(dwmac->syscfg_clk);
+		if (dwmac->int_phyclk)
+			clk_disable_unprepare(dwmac->clk_eth_ck);
+	}
+	return ret;
+}
+
+static int stm32mp1_set_mode(struct plat_stmmacenet_data *plat_dat)
+{
+	struct stm32_dwmac *dwmac = plat_dat->bsp_priv;
+	u32 reg = dwmac->mode_reg;
+	int val;
+
+	switch (plat_dat->interface) {
+	case PHY_INTERFACE_MODE_MII:
+		val = SYSCFG_PMCR_ETH_SEL_MII;
+		pr_debug("SYSCFG init : PHY_INTERFACE_MODE_MII\n");
+		break;
+	case PHY_INTERFACE_MODE_GMII:
+		val = SYSCFG_PMCR_ETH_SEL_GMII;
+		if (dwmac->int_phyclk)
+			val |= SYSCFG_PMCR_ETH_CLK_SEL;
+		pr_debug("SYSCFG init : PHY_INTERFACE_MODE_GMII\n");
+		break;
+	case PHY_INTERFACE_MODE_RMII:
+		val = SYSCFG_PMCR_ETH_SEL_RMII;
+		if (dwmac->int_phyclk)
+			val |= SYSCFG_PMCR_ETH_REF_CLK_SEL;
+		pr_debug("SYSCFG init : PHY_INTERFACE_MODE_RMII\n");
+		break;
+	case PHY_INTERFACE_MODE_RGMII:
+		val = SYSCFG_PMCR_ETH_SEL_RGMII;
+		if (dwmac->int_phyclk)
+			val |= SYSCFG_PMCR_ETH_CLK_SEL;
+		pr_debug("SYSCFG init : PHY_INTERFACE_MODE_RGMII\n");
+		break;
+	default:
+		pr_debug("SYSCFG init :  Do not manage %d interface\n",
+			 plat_dat->interface);
+		/* Do not manage others interfaces */
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(dwmac->regmap, reg,
+				 dwmac->ops->syscfg_eth_mask, val);
+}
+
+static int stm32mcu_set_mode(struct plat_stmmacenet_data *plat_dat)
+{
+	struct stm32_dwmac *dwmac = plat_dat->bsp_priv;
+	u32 reg = dwmac->mode_reg;
+	int val;
+
+	switch (plat_dat->interface) {
+	case PHY_INTERFACE_MODE_MII:
+		val = SYSCFG_MCU_ETH_SEL_MII;
+		pr_debug("SYSCFG init : PHY_INTERFACE_MODE_MII\n");
+		break;
+	case PHY_INTERFACE_MODE_RMII:
+		val = SYSCFG_MCU_ETH_SEL_RMII;
+		pr_debug("SYSCFG init : PHY_INTERFACE_MODE_RMII\n");
+		break;
+	default:
+		pr_debug("SYSCFG init :  Do not manage %d interface\n",
+			 plat_dat->interface);
+		/* Do not manage others interfaces */
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(dwmac->regmap, reg,
+				 dwmac->ops->syscfg_eth_mask, val);
+}
+
 static void stm32_dwmac_clk_disable(struct stm32_dwmac *dwmac)
 {
 	clk_disable_unprepare(dwmac->clk_tx);
 	clk_disable_unprepare(dwmac->clk_rx);
+
+	if (dwmac->ops->clk_prepare)
+		dwmac->ops->clk_prepare(dwmac, false);
 }
 
 static int stm32_dwmac_parse_data(struct stm32_dwmac *dwmac,
@@ -70,15 +201,22 @@ static int stm32_dwmac_parse_data(struct stm32_dwmac *dwmac,
 	/*  Get TX/RX clocks */
 	dwmac->clk_tx = devm_clk_get(dev, "mac-clk-tx");
 	if (IS_ERR(dwmac->clk_tx)) {
-		dev_err(dev, "No tx clock provided...\n");
+		dev_err(dev, "No ETH Tx clock provided...\n");
 		return PTR_ERR(dwmac->clk_tx);
 	}
+
 	dwmac->clk_rx = devm_clk_get(dev, "mac-clk-rx");
 	if (IS_ERR(dwmac->clk_rx)) {
-		dev_err(dev, "No rx clock provided...\n");
+		dev_err(dev, "No ETH Rx clock provided...\n");
 		return PTR_ERR(dwmac->clk_rx);
 	}
 
+	if (dwmac->ops->parse_data) {
+		err = dwmac->ops->parse_data(dwmac, dev);
+		if (err)
+			return err;
+	}
+
 	/* Get mode register */
 	dwmac->regmap = syscon_regmap_lookup_by_phandle(np, "st,syscon");
 	if (IS_ERR(dwmac->regmap))
@@ -91,11 +229,46 @@ static int stm32_dwmac_parse_data(struct stm32_dwmac *dwmac,
 	return err;
 }
 
+static int stm32mp1_parse_data(struct stm32_dwmac *dwmac,
+			       struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+
+	dwmac->int_phyclk = of_property_read_bool(np, "st,int-phyclk");
+
+	/* Check if internal clk from RCC selected */
+	if (dwmac->int_phyclk) {
+		/*  Get ETH_CLK clocks */
+		dwmac->clk_eth_ck = devm_clk_get(dev, "eth-ck");
+		if (IS_ERR(dwmac->clk_eth_ck)) {
+			dev_err(dev, "No ETH CK clock provided...\n");
+			return PTR_ERR(dwmac->clk_eth_ck);
+		}
+	}
+
+	/*  Clock used for low power mode */
+	dwmac->clk_ethstp = devm_clk_get(dev, "ethstp");
+	if (IS_ERR(dwmac->clk_ethstp)) {
+		dev_err(dev, "No ETH peripheral clock provided for CStop mode ...\n");
+		return PTR_ERR(dwmac->clk_ethstp);
+	}
+
+	/*  Clock for sysconfig */
+	dwmac->syscfg_clk = devm_clk_get(dev, "syscfg-clk");
+	if (IS_ERR(dwmac->syscfg_clk)) {
+		dev_err(dev, "No syscfg clock provided...\n");
+		return PTR_ERR(dwmac->syscfg_clk);
+	}
+
+	return 0;
+}
+
 static int stm32_dwmac_probe(struct platform_device *pdev)
 {
 	struct plat_stmmacenet_data *plat_dat;
 	struct stmmac_resources stmmac_res;
 	struct stm32_dwmac *dwmac;
+	const struct stm32_ops *data;
 	int ret;
 
 	ret = stmmac_get_platform_resources(pdev, &stmmac_res);
@@ -112,6 +285,16 @@ static int stm32_dwmac_probe(struct platform_device *pdev)
 		goto err_remove_config_dt;
 	}
 
+	data = of_device_get_match_data(&pdev->dev);
+	if (!data) {
+		dev_err(&pdev->dev, "no of match data provided\n");
+		ret = -EINVAL;
+		goto err_remove_config_dt;
+	}
+
+	dwmac->ops = data;
+	dwmac->dev = &pdev->dev;
+
 	ret = stm32_dwmac_parse_data(dwmac, &pdev->dev);
 	if (ret) {
 		dev_err(&pdev->dev, "Unable to parse OF data\n");
@@ -149,15 +332,48 @@ static int stm32_dwmac_remove(struct platform_device *pdev)
 	return ret;
 }
 
+static int stm32mp1_suspend(struct stm32_dwmac *dwmac)
+{
+	int ret = 0;
+
+	ret = clk_prepare_enable(dwmac->clk_ethstp);
+	if (ret)
+		return ret;
+
+	clk_disable_unprepare(dwmac->clk_tx);
+	clk_disable_unprepare(dwmac->syscfg_clk);
+	if (dwmac->int_phyclk)
+		clk_disable_unprepare(dwmac->clk_eth_ck);
+
+	return ret;
+}
+
+static void stm32mp1_resume(struct stm32_dwmac *dwmac)
+{
+	clk_disable_unprepare(dwmac->clk_ethstp);
+}
+
+static int stm32mcu_suspend(struct stm32_dwmac *dwmac)
+{
+	clk_disable_unprepare(dwmac->clk_tx);
+	clk_disable_unprepare(dwmac->clk_rx);
+
+	return 0;
+}
+
 #ifdef CONFIG_PM_SLEEP
 static int stm32_dwmac_suspend(struct device *dev)
 {
 	struct net_device *ndev = dev_get_drvdata(dev);
 	struct stmmac_priv *priv = netdev_priv(ndev);
+	struct stm32_dwmac *dwmac = priv->plat->bsp_priv;
+
 	int ret;
 
 	ret = stmmac_suspend(dev);
-	stm32_dwmac_clk_disable(priv->plat->bsp_priv);
+
+	if (dwmac->ops->suspend)
+		ret = dwmac->ops->suspend(dwmac);
 
 	return ret;
 }
@@ -166,8 +382,12 @@ static int stm32_dwmac_resume(struct device *dev)
 {
 	struct net_device *ndev = dev_get_drvdata(dev);
 	struct stmmac_priv *priv = netdev_priv(ndev);
+	struct stm32_dwmac *dwmac = priv->plat->bsp_priv;
 	int ret;
 
+	if (dwmac->ops->resume)
+		dwmac->ops->resume(dwmac);
+
 	ret = stm32_dwmac_init(priv->plat);
 	if (ret)
 		return ret;
@@ -181,8 +401,24 @@ static int stm32_dwmac_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(stm32_dwmac_pm_ops,
 	stm32_dwmac_suspend, stm32_dwmac_resume);
 
+static struct stm32_ops stm32mcu_dwmac_data = {
+	.set_mode = stm32mcu_set_mode,
+	.suspend = stm32mcu_suspend,
+	.syscfg_eth_mask = SYSCFG_MCU_ETH_MASK
+};
+
+static struct stm32_ops stm32mp1_dwmac_data = {
+	.set_mode = stm32mp1_set_mode,
+	.clk_prepare = stm32mp1_clk_prepare,
+	.suspend = stm32mp1_suspend,
+	.resume = stm32mp1_resume,
+	.parse_data = stm32mp1_parse_data,
+	.syscfg_eth_mask = SYSCFG_MP1_ETH_MASK
+};
+
 static const struct of_device_id stm32_dwmac_match[] = {
-	{ .compatible = "st,stm32-dwmac"},
+	{ .compatible = "st,stm32-dwmac", .data = &stm32mcu_dwmac_data},
+	{ .compatible = "st,stm32mp1-dwmac", .data = &stm32mp1_dwmac_data},
 	{ }
 };
 MODULE_DEVICE_TABLE(of, stm32_dwmac_match);
@@ -199,5 +435,6 @@ static struct platform_driver stm32_dwmac_driver = {
 module_platform_driver(stm32_dwmac_driver);
 
 MODULE_AUTHOR("Alexandre Torgue <alexandre.torgue@gmail.com>");
-MODULE_DESCRIPTION("STMicroelectronics MCU DWMAC Specific Glue layer");
+MODULE_AUTHOR("Christophe Roullier <christophe.roullier@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics STM32 DWMAC Specific Glue layer");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
index a3fa65b1ca8e..2e6e2a96b4f2 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
@@ -42,17 +42,27 @@
  *				This value is used for disabling properly EMAC
  *				and used as a good starting value in case of the
  *				boot process(uboot) leave some stuff.
+ * @syscon_field		reg_field for the syscon's gmac register
  * @soc_has_internal_phy:	Does the MAC embed an internal PHY
  * @support_mii:		Does the MAC handle MII
  * @support_rmii:		Does the MAC handle RMII
  * @support_rgmii:		Does the MAC handle RGMII
+ *
+ * @rx_delay_max:		Maximum raw value for RX delay chain
+ * @tx_delay_max:		Maximum raw value for TX delay chain
+ *				These two also indicate the bitmask for
+ *				the RX and TX delay chain registers. A
+ *				value of zero indicates this is not supported.
  */
 struct emac_variant {
 	u32 default_syscon_value;
+	const struct reg_field *syscon_field;
 	bool soc_has_internal_phy;
 	bool support_mii;
 	bool support_rmii;
 	bool support_rgmii;
+	u8 rx_delay_max;
+	u8 tx_delay_max;
 };
 
 /* struct sunxi_priv_data - hold all sunxi private data
@@ -71,38 +81,70 @@ struct sunxi_priv_data {
 	struct regulator *regulator;
 	struct reset_control *rst_ephy;
 	const struct emac_variant *variant;
-	struct regmap *regmap;
+	struct regmap_field *regmap_field;
 	bool internal_phy_powered;
 	void *mux_handle;
 };
 
+/* EMAC clock register @ 0x30 in the "system control" address range */
+static const struct reg_field sun8i_syscon_reg_field = {
+	.reg = 0x30,
+	.lsb = 0,
+	.msb = 31,
+};
+
+/* EMAC clock register @ 0x164 in the CCU address range */
+static const struct reg_field sun8i_ccu_reg_field = {
+	.reg = 0x164,
+	.lsb = 0,
+	.msb = 31,
+};
+
 static const struct emac_variant emac_variant_h3 = {
 	.default_syscon_value = 0x58000,
+	.syscon_field = &sun8i_syscon_reg_field,
 	.soc_has_internal_phy = true,
 	.support_mii = true,
 	.support_rmii = true,
-	.support_rgmii = true
+	.support_rgmii = true,
+	.rx_delay_max = 31,
+	.tx_delay_max = 7,
 };
 
 static const struct emac_variant emac_variant_v3s = {
 	.default_syscon_value = 0x38000,
+	.syscon_field = &sun8i_syscon_reg_field,
 	.soc_has_internal_phy = true,
 	.support_mii = true
 };
 
 static const struct emac_variant emac_variant_a83t = {
 	.default_syscon_value = 0,
+	.syscon_field = &sun8i_syscon_reg_field,
 	.soc_has_internal_phy = false,
 	.support_mii = true,
-	.support_rgmii = true
+	.support_rgmii = true,
+	.rx_delay_max = 31,
+	.tx_delay_max = 7,
+};
+
+static const struct emac_variant emac_variant_r40 = {
+	.default_syscon_value = 0,
+	.syscon_field = &sun8i_ccu_reg_field,
+	.support_mii = true,
+	.support_rgmii = true,
+	.rx_delay_max = 7,
 };
 
 static const struct emac_variant emac_variant_a64 = {
 	.default_syscon_value = 0,
+	.syscon_field = &sun8i_syscon_reg_field,
 	.soc_has_internal_phy = false,
 	.support_mii = true,
 	.support_rmii = true,
-	.support_rgmii = true
+	.support_rgmii = true,
+	.rx_delay_max = 31,
+	.tx_delay_max = 7,
 };
 
 #define EMAC_BASIC_CTL0 0x00
@@ -206,9 +248,7 @@ static const struct emac_variant emac_variant_a64 = {
 #define SYSCON_RMII_EN		BIT(13) /* 1: enable RMII (overrides EPIT) */
 
 /* Generic system control EMAC_CLK bits */
-#define SYSCON_ETXDC_MASK		GENMASK(2, 0)
 #define SYSCON_ETXDC_SHIFT		10
-#define SYSCON_ERXDC_MASK		GENMASK(4, 0)
 #define SYSCON_ERXDC_SHIFT		5
 /* EMAC PHY Interface Type */
 #define SYSCON_EPIT			BIT(2) /* 1: RGMII, 0: MII */
@@ -216,7 +256,6 @@ static const struct emac_variant emac_variant_a64 = {
 #define SYSCON_ETCS_MII		0x0
 #define SYSCON_ETCS_EXT_GMII	0x1
 #define SYSCON_ETCS_INT_GMII	0x2
-#define SYSCON_EMAC_REG		0x30
 
 /* sun8i_dwmac_dma_reset() - reset the EMAC
  * Called from stmmac via stmmac_dma_ops->reset
@@ -237,17 +276,28 @@ static int sun8i_dwmac_dma_reset(void __iomem *ioaddr)
  * Called from stmmac via stmmac_dma_ops->init
  */
 static void sun8i_dwmac_dma_init(void __iomem *ioaddr,
-				 struct stmmac_dma_cfg *dma_cfg,
-				 u32 dma_tx, u32 dma_rx, int atds)
+				 struct stmmac_dma_cfg *dma_cfg, int atds)
 {
-	/* Write TX and RX descriptors address */
-	writel(dma_rx, ioaddr + EMAC_RX_DESC_LIST);
-	writel(dma_tx, ioaddr + EMAC_TX_DESC_LIST);
-
 	writel(EMAC_RX_INT | EMAC_TX_INT, ioaddr + EMAC_INT_EN);
 	writel(0x1FFFFFF, ioaddr + EMAC_INT_STA);
 }
 
+static void sun8i_dwmac_dma_init_rx(void __iomem *ioaddr,
+				    struct stmmac_dma_cfg *dma_cfg,
+				    u32 dma_rx_phy, u32 chan)
+{
+	/* Write RX descriptors address */
+	writel(dma_rx_phy, ioaddr + EMAC_RX_DESC_LIST);
+}
+
+static void sun8i_dwmac_dma_init_tx(void __iomem *ioaddr,
+				    struct stmmac_dma_cfg *dma_cfg,
+				    u32 dma_tx_phy, u32 chan)
+{
+	/* Write TX descriptors address */
+	writel(dma_tx_phy, ioaddr + EMAC_TX_DESC_LIST);
+}
+
 /* sun8i_dwmac_dump_regs() - Dump EMAC address space
  * Called from stmmac_dma_ops->dump_regs
  * Used for ethtool
@@ -398,13 +448,36 @@ static int sun8i_dwmac_dma_interrupt(void __iomem *ioaddr,
 	return ret;
 }
 
-static void sun8i_dwmac_dma_operation_mode(void __iomem *ioaddr, int txmode,
-					   int rxmode, int rxfifosz)
+static void sun8i_dwmac_dma_operation_mode_rx(void __iomem *ioaddr, int mode,
+					      u32 channel, int fifosz, u8 qmode)
+{
+	u32 v;
+
+	v = readl(ioaddr + EMAC_RX_CTL1);
+	if (mode == SF_DMA_MODE) {
+		v |= EMAC_RX_MD;
+	} else {
+		v &= ~EMAC_RX_MD;
+		v &= ~EMAC_RX_TH_MASK;
+		if (mode < 32)
+			v |= EMAC_RX_TH_32;
+		else if (mode < 64)
+			v |= EMAC_RX_TH_64;
+		else if (mode < 96)
+			v |= EMAC_RX_TH_96;
+		else if (mode < 128)
+			v |= EMAC_RX_TH_128;
+	}
+	writel(v, ioaddr + EMAC_RX_CTL1);
+}
+
+static void sun8i_dwmac_dma_operation_mode_tx(void __iomem *ioaddr, int mode,
+					      u32 channel, int fifosz, u8 qmode)
 {
 	u32 v;
 
 	v = readl(ioaddr + EMAC_TX_CTL1);
-	if (txmode == SF_DMA_MODE) {
+	if (mode == SF_DMA_MODE) {
 		v |= EMAC_TX_MD;
 		/* Undocumented bit (called TX_NEXT_FRM in BSP), the original
 		 * comment is
@@ -415,40 +488,26 @@ static void sun8i_dwmac_dma_operation_mode(void __iomem *ioaddr, int txmode,
 	} else {
 		v &= ~EMAC_TX_MD;
 		v &= ~EMAC_TX_TH_MASK;
-		if (txmode < 64)
+		if (mode < 64)
 			v |= EMAC_TX_TH_64;
-		else if (txmode < 128)
+		else if (mode < 128)
 			v |= EMAC_TX_TH_128;
-		else if (txmode < 192)
+		else if (mode < 192)
 			v |= EMAC_TX_TH_192;
-		else if (txmode < 256)
+		else if (mode < 256)
 			v |= EMAC_TX_TH_256;
 	}
 	writel(v, ioaddr + EMAC_TX_CTL1);
-
-	v = readl(ioaddr + EMAC_RX_CTL1);
-	if (rxmode == SF_DMA_MODE) {
-		v |= EMAC_RX_MD;
-	} else {
-		v &= ~EMAC_RX_MD;
-		v &= ~EMAC_RX_TH_MASK;
-		if (rxmode < 32)
-			v |= EMAC_RX_TH_32;
-		else if (rxmode < 64)
-			v |= EMAC_RX_TH_64;
-		else if (rxmode < 96)
-			v |= EMAC_RX_TH_96;
-		else if (rxmode < 128)
-			v |= EMAC_RX_TH_128;
-	}
-	writel(v, ioaddr + EMAC_RX_CTL1);
 }
 
 static const struct stmmac_dma_ops sun8i_dwmac_dma_ops = {
 	.reset = sun8i_dwmac_dma_reset,
 	.init = sun8i_dwmac_dma_init,
+	.init_rx_chan = sun8i_dwmac_dma_init_rx,
+	.init_tx_chan = sun8i_dwmac_dma_init_tx,
 	.dump_regs = sun8i_dwmac_dump_regs,
-	.dma_mode = sun8i_dwmac_dma_operation_mode,
+	.dma_rx_mode = sun8i_dwmac_dma_operation_mode_rx,
+	.dma_tx_mode = sun8i_dwmac_dma_operation_mode_tx,
 	.enable_dma_transmission = sun8i_dwmac_enable_dma_transmission,
 	.enable_dma_irq = sun8i_dwmac_enable_dma_irq,
 	.disable_dma_irq = sun8i_dwmac_disable_dma_irq,
@@ -745,7 +804,7 @@ static int mdio_mux_syscon_switch_fn(int current_child, int desired_child,
 	bool need_power_ephy = false;
 
 	if (current_child ^ desired_child) {
-		regmap_read(gmac->regmap, SYSCON_EMAC_REG, &reg);
+		regmap_field_read(gmac->regmap_field, &reg);
 		switch (desired_child) {
 		case DWMAC_SUN8I_MDIO_MUX_INTERNAL_ID:
 			dev_info(priv->device, "Switch mux to internal PHY");
@@ -763,7 +822,7 @@ static int mdio_mux_syscon_switch_fn(int current_child, int desired_child,
 				desired_child);
 			return -EINVAL;
 		}
-		regmap_write(gmac->regmap, SYSCON_EMAC_REG, val);
+		regmap_field_write(gmac->regmap_field, val);
 		if (need_power_ephy) {
 			ret = sun8i_dwmac_power_internal_phy(priv);
 			if (ret)
@@ -801,7 +860,7 @@ static int sun8i_dwmac_set_syscon(struct stmmac_priv *priv)
 	int ret;
 	u32 reg, val;
 
-	regmap_read(gmac->regmap, SYSCON_EMAC_REG, &val);
+	regmap_field_read(gmac->regmap_field, &val);
 	reg = gmac->variant->default_syscon_value;
 	if (reg != val)
 		dev_warn(priv->device,
@@ -835,8 +894,9 @@ static int sun8i_dwmac_set_syscon(struct stmmac_priv *priv)
 		}
 		val /= 100;
 		dev_dbg(priv->device, "set tx-delay to %x\n", val);
-		if (val <= SYSCON_ETXDC_MASK) {
-			reg &= ~(SYSCON_ETXDC_MASK << SYSCON_ETXDC_SHIFT);
+		if (val <= gmac->variant->tx_delay_max) {
+			reg &= ~(gmac->variant->tx_delay_max <<
+				 SYSCON_ETXDC_SHIFT);
 			reg |= (val << SYSCON_ETXDC_SHIFT);
 		} else {
 			dev_err(priv->device, "Invalid TX clock delay: %d\n",
@@ -852,8 +912,9 @@ static int sun8i_dwmac_set_syscon(struct stmmac_priv *priv)
 		}
 		val /= 100;
 		dev_dbg(priv->device, "set rx-delay to %x\n", val);
-		if (val <= SYSCON_ERXDC_MASK) {
-			reg &= ~(SYSCON_ERXDC_MASK << SYSCON_ERXDC_SHIFT);
+		if (val <= gmac->variant->rx_delay_max) {
+			reg &= ~(gmac->variant->rx_delay_max <<
+				 SYSCON_ERXDC_SHIFT);
 			reg |= (val << SYSCON_ERXDC_SHIFT);
 		} else {
 			dev_err(priv->device, "Invalid RX clock delay: %d\n",
@@ -883,7 +944,7 @@ static int sun8i_dwmac_set_syscon(struct stmmac_priv *priv)
 		return -EINVAL;
 	}
 
-	regmap_write(gmac->regmap, SYSCON_EMAC_REG, reg);
+	regmap_field_write(gmac->regmap_field, reg);
 
 	return 0;
 }
@@ -892,7 +953,7 @@ static void sun8i_dwmac_unset_syscon(struct sunxi_priv_data *gmac)
 {
 	u32 reg = gmac->variant->default_syscon_value;
 
-	regmap_write(gmac->regmap, SYSCON_EMAC_REG, reg);
+	regmap_field_write(gmac->regmap_field, reg);
 }
 
 static void sun8i_dwmac_exit(struct platform_device *pdev, void *priv)
@@ -971,6 +1032,34 @@ static struct mac_device_info *sun8i_dwmac_setup(void *ppriv)
 	return mac;
 }
 
+static struct regmap *sun8i_dwmac_get_syscon_from_dev(struct device_node *node)
+{
+	struct device_node *syscon_node;
+	struct platform_device *syscon_pdev;
+	struct regmap *regmap = NULL;
+
+	syscon_node = of_parse_phandle(node, "syscon", 0);
+	if (!syscon_node)
+		return ERR_PTR(-ENODEV);
+
+	syscon_pdev = of_find_device_by_node(syscon_node);
+	if (!syscon_pdev) {
+		/* platform device might not be probed yet */
+		regmap = ERR_PTR(-EPROBE_DEFER);
+		goto out_put_node;
+	}
+
+	/* If no regmap is found then the other device driver is at fault */
+	regmap = dev_get_regmap(&syscon_pdev->dev, NULL);
+	if (!regmap)
+		regmap = ERR_PTR(-EINVAL);
+
+	platform_device_put(syscon_pdev);
+out_put_node:
+	of_node_put(syscon_node);
+	return regmap;
+}
+
 static int sun8i_dwmac_probe(struct platform_device *pdev)
 {
 	struct plat_stmmacenet_data *plat_dat;
@@ -980,6 +1069,7 @@ static int sun8i_dwmac_probe(struct platform_device *pdev)
 	int ret;
 	struct stmmac_priv *priv;
 	struct net_device *ndev;
+	struct regmap *regmap;
 
 	ret = stmmac_get_platform_resources(pdev, &stmmac_res);
 	if (ret)
@@ -1014,14 +1104,41 @@ static int sun8i_dwmac_probe(struct platform_device *pdev)
 		gmac->regulator = NULL;
 	}
 
-	gmac->regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
-						       "syscon");
-	if (IS_ERR(gmac->regmap)) {
-		ret = PTR_ERR(gmac->regmap);
+	/* The "GMAC clock control" register might be located in the
+	 * CCU address range (on the R40), or the system control address
+	 * range (on most other sun8i and later SoCs).
+	 *
+	 * The former controls most if not all clocks in the SoC. The
+	 * latter has an SoC identification register, and on some SoCs,
+	 * controls to map device specific SRAM to either the intended
+	 * peripheral, or the CPU address space.
+	 *
+	 * In either case, there should be a coordinated and restricted
+	 * method of accessing the register needed here. This is done by
+	 * having the device export a custom regmap, instead of a generic
+	 * syscon, which grants all access to all registers.
+	 *
+	 * To support old device trees, we fall back to using the syscon
+	 * interface if possible.
+	 */
+	regmap = sun8i_dwmac_get_syscon_from_dev(pdev->dev.of_node);
+	if (IS_ERR(regmap))
+		regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
+							 "syscon");
+	if (IS_ERR(regmap)) {
+		ret = PTR_ERR(regmap);
 		dev_err(&pdev->dev, "Unable to map syscon: %d\n", ret);
 		return ret;
 	}
 
+	gmac->regmap_field = devm_regmap_field_alloc(dev, regmap,
+						     *gmac->variant->syscon_field);
+	if (IS_ERR(gmac->regmap_field)) {
+		ret = PTR_ERR(gmac->regmap_field);
+		dev_err(dev, "Unable to map syscon register: %d\n", ret);
+		return ret;
+	}
+
 	plat_dat->interface = of_get_phy_mode(dev->of_node);
 
 	/* platform data specifying hardware features and callbacks.
@@ -1078,6 +1195,8 @@ static const struct of_device_id sun8i_dwmac_match[] = {
 		.data = &emac_variant_v3s },
 	{ .compatible = "allwinner,sun8i-a83t-emac",
 		.data = &emac_variant_a83t },
+	{ .compatible = "allwinner,sun8i-r40-gmac",
+		.data = &emac_variant_r40 },
 	{ .compatible = "allwinner,sun50i-a64-emac",
 		.data = &emac_variant_a64 },
 	{ }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h
index c02d36629c52..184ca13c8f79 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h
@@ -29,7 +29,6 @@
 #define GMAC_MII_DATA		0x00000014	/* MII Data */
 #define GMAC_FLOW_CTRL		0x00000018	/* Flow Control */
 #define GMAC_VLAN_TAG		0x0000001c	/* VLAN Tag */
-#define GMAC_VERSION		0x00000020	/* GMAC CORE Version */
 #define GMAC_DEBUG		0x00000024	/* GMAC debug register */
 #define GMAC_WAKEUP_FILTER	0x00000028	/* Wake-up Frame Filter */
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
index ef10baf14186..0877bde6e860 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
@@ -27,6 +27,7 @@
 #include <linux/ethtool.h>
 #include <net/dsa.h>
 #include <asm/io.h>
+#include "stmmac.h"
 #include "stmmac_pcs.h"
 #include "dwmac1000.h"
 
@@ -498,7 +499,7 @@ static void dwmac1000_debug(void __iomem *ioaddr, struct stmmac_extra_stats *x,
 		x->mac_gmii_rx_proto_engine++;
 }
 
-static const struct stmmac_ops dwmac1000_ops = {
+const struct stmmac_ops dwmac1000_ops = {
 	.core_init = dwmac1000_core_init,
 	.set_mac = stmmac_set_mac,
 	.rx_ipc = dwmac1000_rx_ipc_enable,
@@ -519,28 +520,21 @@ static const struct stmmac_ops dwmac1000_ops = {
 	.pcs_get_adv_lp = dwmac1000_get_adv_lp,
 };
 
-struct mac_device_info *dwmac1000_setup(void __iomem *ioaddr, int mcbins,
-					int perfect_uc_entries,
-					int *synopsys_id)
+int dwmac1000_setup(struct stmmac_priv *priv)
 {
-	struct mac_device_info *mac;
-	u32 hwid = readl(ioaddr + GMAC_VERSION);
+	struct mac_device_info *mac = priv->hw;
 
-	mac = kzalloc(sizeof(const struct mac_device_info), GFP_KERNEL);
-	if (!mac)
-		return NULL;
+	dev_info(priv->device, "\tDWMAC1000\n");
 
-	mac->pcsr = ioaddr;
-	mac->multicast_filter_bins = mcbins;
-	mac->unicast_filter_entries = perfect_uc_entries;
+	priv->dev->priv_flags |= IFF_UNICAST_FLT;
+	mac->pcsr = priv->ioaddr;
+	mac->multicast_filter_bins = priv->plat->multicast_filter_bins;
+	mac->unicast_filter_entries = priv->plat->unicast_filter_entries;
 	mac->mcast_bits_log2 = 0;
 
 	if (mac->multicast_filter_bins)
 		mac->mcast_bits_log2 = ilog2(mac->multicast_filter_bins);
 
-	mac->mac = &dwmac1000_ops;
-	mac->dma = &dwmac1000_dma_ops;
-
 	mac->link.duplex = GMAC_CONTROL_DM;
 	mac->link.speed10 = GMAC_CONTROL_PS;
 	mac->link.speed100 = GMAC_CONTROL_PS | GMAC_CONTROL_FES;
@@ -555,8 +549,5 @@ struct mac_device_info *dwmac1000_setup(void __iomem *ioaddr, int mcbins,
 	mac->mii.clk_csr_shift = 2;
 	mac->mii.clk_csr_mask = GENMASK(5, 2);
 
-	/* Get and dump the chip ID */
-	*synopsys_id = stmmac_get_synopsys_id(hwid);
-
-	return mac;
+	return 0;
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c
index 7ecf549c7f1c..aacc4aa80e3c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c
@@ -81,8 +81,7 @@ static void dwmac1000_dma_axi(void __iomem *ioaddr, struct stmmac_axi *axi)
 }
 
 static void dwmac1000_dma_init(void __iomem *ioaddr,
-			       struct stmmac_dma_cfg *dma_cfg,
-			       u32 dma_tx, u32 dma_rx, int atds)
+			       struct stmmac_dma_cfg *dma_cfg, int atds)
 {
 	u32 value = readl(ioaddr + DMA_BUS_MODE);
 	int txpbl = dma_cfg->txpbl ?: dma_cfg->pbl;
@@ -119,12 +118,22 @@ static void dwmac1000_dma_init(void __iomem *ioaddr,
 
 	/* Mask interrupts by writing to CSR7 */
 	writel(DMA_INTR_DEFAULT_MASK, ioaddr + DMA_INTR_ENA);
+}
 
-	/* RX/TX descriptor base address lists must be written into
-	 * DMA CSR3 and CSR4, respectively
-	 */
-	writel(dma_tx, ioaddr + DMA_TX_BASE_ADDR);
-	writel(dma_rx, ioaddr + DMA_RCV_BASE_ADDR);
+static void dwmac1000_dma_init_rx(void __iomem *ioaddr,
+				  struct stmmac_dma_cfg *dma_cfg,
+				  u32 dma_rx_phy, u32 chan)
+{
+	/* RX descriptor base address list must be written into DMA CSR3 */
+	writel(dma_rx_phy, ioaddr + DMA_RCV_BASE_ADDR);
+}
+
+static void dwmac1000_dma_init_tx(void __iomem *ioaddr,
+				  struct stmmac_dma_cfg *dma_cfg,
+				  u32 dma_tx_phy, u32 chan)
+{
+	/* TX descriptor base address list must be written into DMA CSR4 */
+	writel(dma_tx_phy, ioaddr + DMA_TX_BASE_ADDR);
 }
 
 static u32 dwmac1000_configure_fc(u32 csr6, int rxfifosz)
@@ -148,12 +157,40 @@ static u32 dwmac1000_configure_fc(u32 csr6, int rxfifosz)
 	return csr6;
 }
 
-static void dwmac1000_dma_operation_mode(void __iomem *ioaddr, int txmode,
-					 int rxmode, int rxfifosz)
+static void dwmac1000_dma_operation_mode_rx(void __iomem *ioaddr, int mode,
+					    u32 channel, int fifosz, u8 qmode)
+{
+	u32 csr6 = readl(ioaddr + DMA_CONTROL);
+
+	if (mode == SF_DMA_MODE) {
+		pr_debug("GMAC: enable RX store and forward mode\n");
+		csr6 |= DMA_CONTROL_RSF;
+	} else {
+		pr_debug("GMAC: disable RX SF mode (threshold %d)\n", mode);
+		csr6 &= ~DMA_CONTROL_RSF;
+		csr6 &= DMA_CONTROL_TC_RX_MASK;
+		if (mode <= 32)
+			csr6 |= DMA_CONTROL_RTC_32;
+		else if (mode <= 64)
+			csr6 |= DMA_CONTROL_RTC_64;
+		else if (mode <= 96)
+			csr6 |= DMA_CONTROL_RTC_96;
+		else
+			csr6 |= DMA_CONTROL_RTC_128;
+	}
+
+	/* Configure flow control based on rx fifo size */
+	csr6 = dwmac1000_configure_fc(csr6, fifosz);
+
+	writel(csr6, ioaddr + DMA_CONTROL);
+}
+
+static void dwmac1000_dma_operation_mode_tx(void __iomem *ioaddr, int mode,
+					    u32 channel, int fifosz, u8 qmode)
 {
 	u32 csr6 = readl(ioaddr + DMA_CONTROL);
 
-	if (txmode == SF_DMA_MODE) {
+	if (mode == SF_DMA_MODE) {
 		pr_debug("GMAC: enable TX store and forward mode\n");
 		/* Transmit COE type 2 cannot be done in cut-through mode. */
 		csr6 |= DMA_CONTROL_TSF;
@@ -162,42 +199,22 @@ static void dwmac1000_dma_operation_mode(void __iomem *ioaddr, int txmode,
 		 */
 		csr6 |= DMA_CONTROL_OSF;
 	} else {
-		pr_debug("GMAC: disabling TX SF (threshold %d)\n", txmode);
+		pr_debug("GMAC: disabling TX SF (threshold %d)\n", mode);
 		csr6 &= ~DMA_CONTROL_TSF;
 		csr6 &= DMA_CONTROL_TC_TX_MASK;
 		/* Set the transmit threshold */
-		if (txmode <= 32)
+		if (mode <= 32)
 			csr6 |= DMA_CONTROL_TTC_32;
-		else if (txmode <= 64)
+		else if (mode <= 64)
 			csr6 |= DMA_CONTROL_TTC_64;
-		else if (txmode <= 128)
+		else if (mode <= 128)
 			csr6 |= DMA_CONTROL_TTC_128;
-		else if (txmode <= 192)
+		else if (mode <= 192)
 			csr6 |= DMA_CONTROL_TTC_192;
 		else
 			csr6 |= DMA_CONTROL_TTC_256;
 	}
 
-	if (rxmode == SF_DMA_MODE) {
-		pr_debug("GMAC: enable RX store and forward mode\n");
-		csr6 |= DMA_CONTROL_RSF;
-	} else {
-		pr_debug("GMAC: disable RX SF mode (threshold %d)\n", rxmode);
-		csr6 &= ~DMA_CONTROL_RSF;
-		csr6 &= DMA_CONTROL_TC_RX_MASK;
-		if (rxmode <= 32)
-			csr6 |= DMA_CONTROL_RTC_32;
-		else if (rxmode <= 64)
-			csr6 |= DMA_CONTROL_RTC_64;
-		else if (rxmode <= 96)
-			csr6 |= DMA_CONTROL_RTC_96;
-		else
-			csr6 |= DMA_CONTROL_RTC_128;
-	}
-
-	/* Configure flow control based on rx fifo size */
-	csr6 = dwmac1000_configure_fc(csr6, rxfifosz);
-
 	writel(csr6, ioaddr + DMA_CONTROL);
 }
 
@@ -256,9 +273,12 @@ static void dwmac1000_rx_watchdog(void __iomem *ioaddr, u32 riwt,
 const struct stmmac_dma_ops dwmac1000_dma_ops = {
 	.reset = dwmac_dma_reset,
 	.init = dwmac1000_dma_init,
+	.init_rx_chan = dwmac1000_dma_init_rx,
+	.init_tx_chan = dwmac1000_dma_init_tx,
 	.axi = dwmac1000_dma_axi,
 	.dump_regs = dwmac1000_dump_dma_regs,
-	.dma_mode = dwmac1000_dma_operation_mode,
+	.dma_rx_mode = dwmac1000_dma_operation_mode_rx,
+	.dma_tx_mode = dwmac1000_dma_operation_mode_tx,
 	.enable_dma_transmission = dwmac_enable_dma_transmission,
 	.enable_dma_irq = dwmac_enable_dma_irq,
 	.disable_dma_irq = dwmac_disable_dma_irq,
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
index 91b23f9db31a..b735143987e1 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
@@ -27,6 +27,7 @@
 #include <linux/crc32.h>
 #include <net/dsa.h>
 #include <asm/io.h>
+#include "stmmac.h"
 #include "dwmac100.h"
 
 static void dwmac100_core_init(struct mac_device_info *hw,
@@ -159,7 +160,7 @@ static void dwmac100_pmt(struct mac_device_info *hw, unsigned long mode)
 	return;
 }
 
-static const struct stmmac_ops dwmac100_ops = {
+const struct stmmac_ops dwmac100_ops = {
 	.core_init = dwmac100_core_init,
 	.set_mac = stmmac_set_mac,
 	.rx_ipc = dwmac100_rx_ipc_enable,
@@ -172,20 +173,13 @@ static const struct stmmac_ops dwmac100_ops = {
 	.get_umac_addr = dwmac100_get_umac_addr,
 };
 
-struct mac_device_info *dwmac100_setup(void __iomem *ioaddr, int *synopsys_id)
+int dwmac100_setup(struct stmmac_priv *priv)
 {
-	struct mac_device_info *mac;
+	struct mac_device_info *mac = priv->hw;
 
-	mac = kzalloc(sizeof(const struct mac_device_info), GFP_KERNEL);
-	if (!mac)
-		return NULL;
-
-	pr_info("\tDWMAC100\n");
-
-	mac->pcsr = ioaddr;
-	mac->mac = &dwmac100_ops;
-	mac->dma = &dwmac100_dma_ops;
+	dev_info(priv->device, "\tDWMAC100\n");
 
+	mac->pcsr = priv->ioaddr;
 	mac->link.duplex = MAC_CONTROL_F;
 	mac->link.speed10 = 0;
 	mac->link.speed100 = 0;
@@ -200,8 +194,5 @@ struct mac_device_info *dwmac100_setup(void __iomem *ioaddr, int *synopsys_id)
 	mac->mii.clk_csr_shift = 2;
 	mac->mii.clk_csr_mask = GENMASK(5, 2);
 
-	/* Synopsys Id is not available on old chips */
-	*synopsys_id = 0;
-
-	return mac;
+	return 0;
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c
index 6502b9aa3bf5..21dee25ee570 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c
@@ -29,8 +29,7 @@
 #include "dwmac_dma.h"
 
 static void dwmac100_dma_init(void __iomem *ioaddr,
-			      struct stmmac_dma_cfg *dma_cfg,
-			      u32 dma_tx, u32 dma_rx, int atds)
+			      struct stmmac_dma_cfg *dma_cfg, int atds)
 {
 	/* Enable Application Access by writing to DMA CSR0 */
 	writel(DMA_BUS_MODE_DEFAULT | (dma_cfg->pbl << DMA_BUS_MODE_PBL_SHIFT),
@@ -38,12 +37,22 @@ static void dwmac100_dma_init(void __iomem *ioaddr,
 
 	/* Mask interrupts by writing to CSR7 */
 	writel(DMA_INTR_DEFAULT_MASK, ioaddr + DMA_INTR_ENA);
+}
 
-	/* RX/TX descriptor base addr lists must be written into
-	 * DMA CSR3 and CSR4, respectively
-	 */
-	writel(dma_tx, ioaddr + DMA_TX_BASE_ADDR);
-	writel(dma_rx, ioaddr + DMA_RCV_BASE_ADDR);
+static void dwmac100_dma_init_rx(void __iomem *ioaddr,
+				 struct stmmac_dma_cfg *dma_cfg,
+				 u32 dma_rx_phy, u32 chan)
+{
+	/* RX descriptor base addr lists must be written into DMA CSR3 */
+	writel(dma_rx_phy, ioaddr + DMA_RCV_BASE_ADDR);
+}
+
+static void dwmac100_dma_init_tx(void __iomem *ioaddr,
+				 struct stmmac_dma_cfg *dma_cfg,
+				 u32 dma_tx_phy, u32 chan)
+{
+	/* TX descriptor base addr lists must be written into DMA CSR4 */
+	writel(dma_tx_phy, ioaddr + DMA_TX_BASE_ADDR);
 }
 
 /* Store and Forward capability is not used at all.
@@ -51,14 +60,14 @@ static void dwmac100_dma_init(void __iomem *ioaddr,
  * The transmit threshold can be programmed by setting the TTC bits in the DMA
  * control register.
  */
-static void dwmac100_dma_operation_mode(void __iomem *ioaddr, int txmode,
-					int rxmode, int rxfifosz)
+static void dwmac100_dma_operation_mode_tx(void __iomem *ioaddr, int mode,
+					   u32 channel, int fifosz, u8 qmode)
 {
 	u32 csr6 = readl(ioaddr + DMA_CONTROL);
 
-	if (txmode <= 32)
+	if (mode <= 32)
 		csr6 |= DMA_CONTROL_TTC_32;
-	else if (txmode <= 64)
+	else if (mode <= 64)
 		csr6 |= DMA_CONTROL_TTC_64;
 	else
 		csr6 |= DMA_CONTROL_TTC_128;
@@ -112,8 +121,10 @@ static void dwmac100_dma_diagnostic_fr(void *data, struct stmmac_extra_stats *x,
 const struct stmmac_dma_ops dwmac100_dma_ops = {
 	.reset = dwmac_dma_reset,
 	.init = dwmac100_dma_init,
+	.init_rx_chan = dwmac100_dma_init_rx,
+	.init_tx_chan = dwmac100_dma_init_tx,
 	.dump_regs = dwmac100_dump_dma_regs,
-	.dma_mode = dwmac100_dma_operation_mode,
+	.dma_tx_mode = dwmac100_dma_operation_mode_tx,
 	.dma_diagnostic_fr = dwmac100_dma_diagnostic_fr,
 	.enable_dma_transmission = dwmac_enable_dma_transmission,
 	.enable_dma_irq = dwmac_enable_dma_irq,
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
index dedd40613090..6330a55953df 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
@@ -34,7 +34,6 @@
 #define GMAC_PCS_BASE			0x000000e0
 #define GMAC_PHYIF_CONTROL_STATUS	0x000000f8
 #define GMAC_PMT			0x000000c0
-#define GMAC_VERSION			0x00000110
 #define GMAC_DEBUG			0x00000114
 #define GMAC_HW_FEATURE0		0x0000011c
 #define GMAC_HW_FEATURE1		0x00000120
@@ -195,6 +194,9 @@ enum power_event {
 
 /* MAC HW features3 bitmap */
 #define GMAC_HW_FEAT_ASP		GENMASK(29, 28)
+#define GMAC_HW_FEAT_FRPES		GENMASK(14, 13)
+#define GMAC_HW_FEAT_FRPBS		GENMASK(12, 11)
+#define GMAC_HW_FEAT_FRPSEL		BIT(10)
 
 /* MAC HW ADDR regs */
 #define GMAC_HI_DCS			GENMASK(18, 16)
@@ -203,6 +205,7 @@ enum power_event {
 
 /*  MTL registers */
 #define MTL_OPERATION_MODE		0x00000c00
+#define MTL_FRPE			BIT(15)
 #define MTL_OPERATION_SCHALG_MASK	GENMASK(6, 5)
 #define MTL_OPERATION_SCHALG_WRR	(0x0 << 5)
 #define MTL_OPERATION_SCHALG_WFQ	(0x1 << 5)
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
index 517b1f6736a8..a7121a7d9391 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
@@ -18,6 +18,7 @@
 #include <linux/ethtool.h>
 #include <linux/io.h>
 #include <net/dsa.h>
+#include "stmmac.h"
 #include "stmmac_pcs.h"
 #include "dwmac4.h"
 #include "dwmac5.h"
@@ -700,7 +701,7 @@ static void dwmac4_debug(void __iomem *ioaddr, struct stmmac_extra_stats *x,
 		x->mac_gmii_rx_proto_engine++;
 }
 
-static const struct stmmac_ops dwmac4_ops = {
+const struct stmmac_ops dwmac4_ops = {
 	.core_init = dwmac4_core_init,
 	.set_mac = stmmac_set_mac,
 	.rx_ipc = dwmac4_rx_ipc_enable,
@@ -731,7 +732,7 @@ static const struct stmmac_ops dwmac4_ops = {
 	.set_filter = dwmac4_set_filter,
 };
 
-static const struct stmmac_ops dwmac410_ops = {
+const struct stmmac_ops dwmac410_ops = {
 	.core_init = dwmac4_core_init,
 	.set_mac = stmmac_dwmac4_set_mac,
 	.rx_ipc = dwmac4_rx_ipc_enable,
@@ -762,7 +763,7 @@ static const struct stmmac_ops dwmac410_ops = {
 	.set_filter = dwmac4_set_filter,
 };
 
-static const struct stmmac_ops dwmac510_ops = {
+const struct stmmac_ops dwmac510_ops = {
 	.core_init = dwmac4_core_init,
 	.set_mac = stmmac_dwmac4_set_mac,
 	.rx_ipc = dwmac4_rx_ipc_enable,
@@ -794,21 +795,19 @@ static const struct stmmac_ops dwmac510_ops = {
 	.safety_feat_config = dwmac5_safety_feat_config,
 	.safety_feat_irq_status = dwmac5_safety_feat_irq_status,
 	.safety_feat_dump = dwmac5_safety_feat_dump,
+	.rxp_config = dwmac5_rxp_config,
 };
 
-struct mac_device_info *dwmac4_setup(void __iomem *ioaddr, int mcbins,
-				     int perfect_uc_entries, int *synopsys_id)
+int dwmac4_setup(struct stmmac_priv *priv)
 {
-	struct mac_device_info *mac;
-	u32 hwid = readl(ioaddr + GMAC_VERSION);
+	struct mac_device_info *mac = priv->hw;
 
-	mac = kzalloc(sizeof(const struct mac_device_info), GFP_KERNEL);
-	if (!mac)
-		return NULL;
+	dev_info(priv->device, "\tDWMAC4/5\n");
 
-	mac->pcsr = ioaddr;
-	mac->multicast_filter_bins = mcbins;
-	mac->unicast_filter_entries = perfect_uc_entries;
+	priv->dev->priv_flags |= IFF_UNICAST_FLT;
+	mac->pcsr = priv->ioaddr;
+	mac->multicast_filter_bins = priv->plat->multicast_filter_bins;
+	mac->unicast_filter_entries = priv->plat->unicast_filter_entries;
 	mac->mcast_bits_log2 = 0;
 
 	if (mac->multicast_filter_bins)
@@ -828,20 +827,5 @@ struct mac_device_info *dwmac4_setup(void __iomem *ioaddr, int mcbins,
 	mac->mii.clk_csr_shift = 8;
 	mac->mii.clk_csr_mask = GENMASK(11, 8);
 
-	/* Get and dump the chip ID */
-	*synopsys_id = stmmac_get_synopsys_id(hwid);
-
-	if (*synopsys_id > DWMAC_CORE_4_00)
-		mac->dma = &dwmac410_dma_ops;
-	else
-		mac->dma = &dwmac4_dma_ops;
-
-	if (*synopsys_id >= DWMAC_CORE_5_10)
-		mac->mac = &dwmac510_ops;
-	else if (*synopsys_id >= DWMAC_CORE_4_00)
-		mac->mac = &dwmac410_ops;
-	else
-		mac->mac = &dwmac4_ops;
-
-	return mac;
+	return 0;
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
index 2a6521d33e43..20299f6f65fc 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
@@ -189,9 +189,12 @@ static void dwmac4_set_tx_owner(struct dma_desc *p)
 	p->des3 |= cpu_to_le32(TDES3_OWN);
 }
 
-static void dwmac4_set_rx_owner(struct dma_desc *p)
+static void dwmac4_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
 {
-	p->des3 |= cpu_to_le32(RDES3_OWN);
+	p->des3 = cpu_to_le32(RDES3_OWN | RDES3_BUFFER1_VALID_ADDR);
+
+	if (!disable_rx_ic)
+		p->des3 |= cpu_to_le32(RDES3_INT_ON_COMPLETION_EN);
 }
 
 static int dwmac4_get_tx_ls(struct dma_desc *p)
@@ -223,7 +226,7 @@ static int dwmac4_wrback_get_tx_timestamp_status(struct dma_desc *p)
 	return 0;
 }
 
-static inline u64 dwmac4_get_timestamp(void *desc, u32 ats)
+static inline void dwmac4_get_timestamp(void *desc, u32 ats, u64 *ts)
 {
 	struct dma_desc *p = (struct dma_desc *)desc;
 	u64 ns;
@@ -232,7 +235,7 @@ static inline u64 dwmac4_get_timestamp(void *desc, u32 ats)
 	/* convert high/sec time stamp value to nanosecond */
 	ns += le32_to_cpu(p->des1) * 1000000000ULL;
 
-	return ns;
+	*ts = ns;
 }
 
 static int dwmac4_rx_check_timestamp(void *desc)
@@ -292,10 +295,7 @@ static int dwmac4_wrback_get_rx_timestamp_status(void *desc, void *next_desc,
 static void dwmac4_rd_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
 				   int mode, int end)
 {
-	p->des3 = cpu_to_le32(RDES3_OWN | RDES3_BUFFER1_VALID_ADDR);
-
-	if (!disable_rx_ic)
-		p->des3 |= cpu_to_le32(RDES3_INT_ON_COMPLETION_EN);
+	dwmac4_set_rx_owner(p, disable_rx_ic);
 }
 
 static void dwmac4_rd_init_tx_desc(struct dma_desc *p, int mode, int end)
@@ -424,6 +424,25 @@ static void dwmac4_set_mss_ctxt(struct dma_desc *p, unsigned int mss)
 	p->des3 = cpu_to_le32(TDES3_CONTEXT_TYPE | TDES3_CTXT_TCMSSV);
 }
 
+static void dwmac4_get_addr(struct dma_desc *p, unsigned int *addr)
+{
+	*addr = le32_to_cpu(p->des0);
+}
+
+static void dwmac4_set_addr(struct dma_desc *p, dma_addr_t addr)
+{
+	p->des0 = cpu_to_le32(addr);
+	p->des1 = 0;
+}
+
+static void dwmac4_clear(struct dma_desc *p)
+{
+	p->des0 = 0;
+	p->des1 = 0;
+	p->des2 = 0;
+	p->des3 = 0;
+}
+
 const struct stmmac_desc_ops dwmac4_desc_ops = {
 	.tx_status = dwmac4_wrback_get_tx_status,
 	.rx_status = dwmac4_wrback_get_rx_status,
@@ -445,6 +464,9 @@ const struct stmmac_desc_ops dwmac4_desc_ops = {
 	.init_tx_desc = dwmac4_rd_init_tx_desc,
 	.display_ring = dwmac4_display_ring,
 	.set_mss = dwmac4_set_mss_ctxt,
+	.get_addr = dwmac4_get_addr,
+	.set_addr = dwmac4_set_addr,
+	.clear = dwmac4_clear,
 };
 
 const struct stmmac_mode_ops dwmac4_ring_mode_ops = { };
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
index d37d457306d1..bf8e5a16f11c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
@@ -94,6 +94,10 @@ static void dwmac4_dma_init_tx_chan(void __iomem *ioaddr,
 
 	value = readl(ioaddr + DMA_CHAN_TX_CONTROL(chan));
 	value = value | (txpbl << DMA_BUS_MODE_PBL_SHIFT);
+
+	/* Enable OSP to get best performance */
+	value |= DMA_CONTROL_OSP;
+
 	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(chan));
 
 	writel(dma_tx_phy, ioaddr + DMA_CHAN_TX_BASE_ADDR(chan));
@@ -116,8 +120,7 @@ static void dwmac4_dma_init_channel(void __iomem *ioaddr,
 }
 
 static void dwmac4_dma_init(void __iomem *ioaddr,
-			    struct stmmac_dma_cfg *dma_cfg,
-			    u32 dma_tx, u32 dma_rx, int atds)
+			    struct stmmac_dma_cfg *dma_cfg, int atds)
 {
 	u32 value = readl(ioaddr + DMA_SYS_BUS_MODE);
 
@@ -379,6 +382,9 @@ static void dwmac4_get_hw_feature(void __iomem *ioaddr,
 
 	/* 5.10 Features */
 	dma_cap->asp = (hw_cap & GMAC_HW_FEAT_ASP) >> 28;
+	dma_cap->frpes = (hw_cap & GMAC_HW_FEAT_FRPES) >> 13;
+	dma_cap->frpbs = (hw_cap & GMAC_HW_FEAT_FRPBS) >> 11;
+	dma_cap->frpsel = (hw_cap & GMAC_HW_FEAT_FRPSEL) >> 10;
 }
 
 /* Enable/disable TSO feature and set MSS */
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h
index 8474bf961dd0..c63c1fe3f26b 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h
@@ -184,7 +184,6 @@
 #define DMA_CHAN0_DBG_STAT_RPS_SHIFT	8
 
 int dwmac4_dma_reset(void __iomem *ioaddr);
-void dwmac4_enable_dma_transmission(void __iomem *ioaddr, u32 tail_ptr);
 void dwmac4_enable_dma_irq(void __iomem *ioaddr, u32 chan);
 void dwmac410_enable_dma_irq(void __iomem *ioaddr, u32 chan);
 void dwmac4_disable_dma_irq(void __iomem *ioaddr, u32 chan);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c
index 860de39999c7..b2becb80a697 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c
@@ -7,6 +7,7 @@
 #include "common.h"
 #include "dwmac4.h"
 #include "dwmac5.h"
+#include "stmmac.h"
 
 struct dwmac5_error_desc {
 	bool valid;
@@ -237,15 +238,16 @@ int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp)
 	return 0;
 }
 
-bool dwmac5_safety_feat_irq_status(struct net_device *ndev,
+int dwmac5_safety_feat_irq_status(struct net_device *ndev,
 		void __iomem *ioaddr, unsigned int asp,
 		struct stmmac_safety_stats *stats)
 {
-	bool ret = false, err, corr;
+	bool err, corr;
 	u32 mtl, dma;
+	int ret = 0;
 
 	if (!asp)
-		return false;
+		return -EINVAL;
 
 	mtl = readl(ioaddr + MTL_SAFETY_INT_STATUS);
 	dma = readl(ioaddr + DMA_SAFETY_INT_STATUS);
@@ -282,17 +284,213 @@ static const struct dwmac5_error {
 	{ dwmac5_dma_errors },
 };
 
-const char *dwmac5_safety_feat_dump(struct stmmac_safety_stats *stats,
-			int index, unsigned long *count)
+int dwmac5_safety_feat_dump(struct stmmac_safety_stats *stats,
+			int index, unsigned long *count, const char **desc)
 {
 	int module = index / 32, offset = index % 32;
 	unsigned long *ptr = (unsigned long *)stats;
 
 	if (module >= ARRAY_SIZE(dwmac5_all_errors))
-		return NULL;
+		return -EINVAL;
 	if (!dwmac5_all_errors[module].desc[offset].valid)
-		return NULL;
+		return -EINVAL;
 	if (count)
 		*count = *(ptr + index);
-	return dwmac5_all_errors[module].desc[offset].desc;
+	if (desc)
+		*desc = dwmac5_all_errors[module].desc[offset].desc;
+	return 0;
+}
+
+static int dwmac5_rxp_disable(void __iomem *ioaddr)
+{
+	u32 val;
+	int ret;
+
+	val = readl(ioaddr + MTL_OPERATION_MODE);
+	val &= ~MTL_FRPE;
+	writel(val, ioaddr + MTL_OPERATION_MODE);
+
+	ret = readl_poll_timeout(ioaddr + MTL_RXP_CONTROL_STATUS, val,
+			val & RXPI, 1, 10000);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static void dwmac5_rxp_enable(void __iomem *ioaddr)
+{
+	u32 val;
+
+	val = readl(ioaddr + MTL_OPERATION_MODE);
+	val |= MTL_FRPE;
+	writel(val, ioaddr + MTL_OPERATION_MODE);
+}
+
+static int dwmac5_rxp_update_single_entry(void __iomem *ioaddr,
+					  struct stmmac_tc_entry *entry,
+					  int pos)
+{
+	int ret, i;
+
+	for (i = 0; i < (sizeof(entry->val) / sizeof(u32)); i++) {
+		int real_pos = pos * (sizeof(entry->val) / sizeof(u32)) + i;
+		u32 val;
+
+		/* Wait for ready */
+		ret = readl_poll_timeout(ioaddr + MTL_RXP_IACC_CTRL_STATUS,
+				val, !(val & STARTBUSY), 1, 10000);
+		if (ret)
+			return ret;
+
+		/* Write data */
+		val = *((u32 *)&entry->val + i);
+		writel(val, ioaddr + MTL_RXP_IACC_DATA);
+
+		/* Write pos */
+		val = real_pos & ADDR;
+		writel(val, ioaddr + MTL_RXP_IACC_CTRL_STATUS);
+
+		/* Write OP */
+		val |= WRRDN;
+		writel(val, ioaddr + MTL_RXP_IACC_CTRL_STATUS);
+
+		/* Start Write */
+		val |= STARTBUSY;
+		writel(val, ioaddr + MTL_RXP_IACC_CTRL_STATUS);
+
+		/* Wait for done */
+		ret = readl_poll_timeout(ioaddr + MTL_RXP_IACC_CTRL_STATUS,
+				val, !(val & STARTBUSY), 1, 10000);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static struct stmmac_tc_entry *
+dwmac5_rxp_get_next_entry(struct stmmac_tc_entry *entries, unsigned int count,
+			  u32 curr_prio)
+{
+	struct stmmac_tc_entry *entry;
+	u32 min_prio = ~0x0;
+	int i, min_prio_idx;
+	bool found = false;
+
+	for (i = count - 1; i >= 0; i--) {
+		entry = &entries[i];
+
+		/* Do not update unused entries */
+		if (!entry->in_use)
+			continue;
+		/* Do not update already updated entries (i.e. fragments) */
+		if (entry->in_hw)
+			continue;
+		/* Let last entry be updated last */
+		if (entry->is_last)
+			continue;
+		/* Do not return fragments */
+		if (entry->is_frag)
+			continue;
+		/* Check if we already checked this prio */
+		if (entry->prio < curr_prio)
+			continue;
+		/* Check if this is the minimum prio */
+		if (entry->prio < min_prio) {
+			min_prio = entry->prio;
+			min_prio_idx = i;
+			found = true;
+		}
+	}
+
+	if (found)
+		return &entries[min_prio_idx];
+	return NULL;
+}
+
+int dwmac5_rxp_config(void __iomem *ioaddr, struct stmmac_tc_entry *entries,
+		      unsigned int count)
+{
+	struct stmmac_tc_entry *entry, *frag;
+	int i, ret, nve = 0;
+	u32 curr_prio = 0;
+	u32 old_val, val;
+
+	/* Force disable RX */
+	old_val = readl(ioaddr + GMAC_CONFIG);
+	val = old_val & ~GMAC_CONFIG_RE;
+	writel(val, ioaddr + GMAC_CONFIG);
+
+	/* Disable RX Parser */
+	ret = dwmac5_rxp_disable(ioaddr);
+	if (ret)
+		goto re_enable;
+
+	/* Set all entries as NOT in HW */
+	for (i = 0; i < count; i++) {
+		entry = &entries[i];
+		entry->in_hw = false;
+	}
+
+	/* Update entries by reverse order */
+	while (1) {
+		entry = dwmac5_rxp_get_next_entry(entries, count, curr_prio);
+		if (!entry)
+			break;
+
+		curr_prio = entry->prio;
+		frag = entry->frag_ptr;
+
+		/* Set special fragment requirements */
+		if (frag) {
+			entry->val.af = 0;
+			entry->val.rf = 0;
+			entry->val.nc = 1;
+			entry->val.ok_index = nve + 2;
+		}
+
+		ret = dwmac5_rxp_update_single_entry(ioaddr, entry, nve);
+		if (ret)
+			goto re_enable;
+
+		entry->table_pos = nve++;
+		entry->in_hw = true;
+
+		if (frag && !frag->in_hw) {
+			ret = dwmac5_rxp_update_single_entry(ioaddr, frag, nve);
+			if (ret)
+				goto re_enable;
+			frag->table_pos = nve++;
+			frag->in_hw = true;
+		}
+	}
+
+	if (!nve)
+		goto re_enable;
+
+	/* Update all pass entry */
+	for (i = 0; i < count; i++) {
+		entry = &entries[i];
+		if (!entry->is_last)
+			continue;
+
+		ret = dwmac5_rxp_update_single_entry(ioaddr, entry, nve);
+		if (ret)
+			goto re_enable;
+
+		entry->table_pos = nve++;
+	}
+
+	/* Assume n. of parsable entries == n. of valid entries */
+	val = (nve << 16) & NPE;
+	val |= nve & NVE;
+	writel(val, ioaddr + MTL_RXP_CONTROL_STATUS);
+
+	/* Enable RX Parser */
+	dwmac5_rxp_enable(ioaddr);
+
+re_enable:
+	/* Re-enable RX */
+	writel(old_val, ioaddr + GMAC_CONFIG);
+	return ret;
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac5.h b/drivers/net/ethernet/stmicro/stmmac/dwmac5.h
index a0d2c44711b9..cc810aff7100 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac5.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac5.h
@@ -11,6 +11,17 @@
 #define PRTYEN				BIT(1)
 #define TMOUTEN				BIT(0)
 
+#define MTL_RXP_CONTROL_STATUS		0x00000ca0
+#define RXPI				BIT(31)
+#define NPE				GENMASK(23, 16)
+#define NVE				GENMASK(7, 0)
+#define MTL_RXP_IACC_CTRL_STATUS	0x00000cb0
+#define STARTBUSY			BIT(31)
+#define RXPEIEC				GENMASK(22, 21)
+#define RXPEIEE				BIT(20)
+#define WRRDN				BIT(16)
+#define ADDR				GENMASK(15, 0)
+#define MTL_RXP_IACC_DATA		0x00000cb4
 #define MTL_ECC_CONTROL			0x00000cc0
 #define TSOEE				BIT(4)
 #define MRXPEE				BIT(3)
@@ -43,10 +54,12 @@
 #define DMA_ECC_INT_STATUS		0x00001088
 
 int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp);
-bool dwmac5_safety_feat_irq_status(struct net_device *ndev,
+int dwmac5_safety_feat_irq_status(struct net_device *ndev,
 		void __iomem *ioaddr, unsigned int asp,
 		struct stmmac_safety_stats *stats);
-const char *dwmac5_safety_feat_dump(struct stmmac_safety_stats *stats,
-			int index, unsigned long *count);
+int dwmac5_safety_feat_dump(struct stmmac_safety_stats *stats,
+			int index, unsigned long *count, const char **desc);
+int dwmac5_rxp_config(void __iomem *ioaddr, struct stmmac_tc_entry *entries,
+		      unsigned int count);
 
 #endif /* __DWMAC5_H__ */
diff --git a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
index 6768a25b6aa0..77914c89d749 100644
--- a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
@@ -292,7 +292,7 @@ static void enh_desc_set_tx_owner(struct dma_desc *p)
 	p->des0 |= cpu_to_le32(ETDES0_OWN);
 }
 
-static void enh_desc_set_rx_owner(struct dma_desc *p)
+static void enh_desc_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
 {
 	p->des0 |= cpu_to_le32(RDES0_OWN);
 }
@@ -382,7 +382,7 @@ static int enh_desc_get_tx_timestamp_status(struct dma_desc *p)
 	return (le32_to_cpu(p->des0) & ETDES0_TIME_STAMP_STATUS) >> 17;
 }
 
-static u64 enh_desc_get_timestamp(void *desc, u32 ats)
+static void enh_desc_get_timestamp(void *desc, u32 ats, u64 *ts)
 {
 	u64 ns;
 
@@ -397,7 +397,7 @@ static u64 enh_desc_get_timestamp(void *desc, u32 ats)
 		ns += le32_to_cpu(p->des3) * 1000000000ULL;
 	}
 
-	return ns;
+	*ts = ns;
 }
 
 static int enh_desc_get_rx_timestamp_status(void *desc, void *next_desc,
@@ -437,6 +437,21 @@ static void enh_desc_display_ring(void *head, unsigned int size, bool rx)
 	pr_info("\n");
 }
 
+static void enh_desc_get_addr(struct dma_desc *p, unsigned int *addr)
+{
+	*addr = le32_to_cpu(p->des2);
+}
+
+static void enh_desc_set_addr(struct dma_desc *p, dma_addr_t addr)
+{
+	p->des2 = cpu_to_le32(addr);
+}
+
+static void enh_desc_clear(struct dma_desc *p)
+{
+	p->des2 = 0;
+}
+
 const struct stmmac_desc_ops enh_desc_ops = {
 	.tx_status = enh_desc_get_tx_status,
 	.rx_status = enh_desc_get_rx_status,
@@ -457,4 +472,7 @@ const struct stmmac_desc_ops enh_desc_ops = {
 	.get_timestamp = enh_desc_get_timestamp,
 	.get_rx_timestamp_status = enh_desc_get_rx_timestamp_status,
 	.display_ring = enh_desc_display_ring,
+	.get_addr = enh_desc_get_addr,
+	.set_addr = enh_desc_set_addr,
+	.clear = enh_desc_clear,
 };
diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.c b/drivers/net/ethernet/stmicro/stmmac/hwif.c
new file mode 100644
index 000000000000..14770fc8865e
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/hwif.c
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
+ * stmmac HW Interface Handling
+ */
+
+#include "common.h"
+#include "stmmac.h"
+#include "stmmac_ptp.h"
+
+static u32 stmmac_get_id(struct stmmac_priv *priv, u32 id_reg)
+{
+	u32 reg = readl(priv->ioaddr + id_reg);
+
+	if (!reg) {
+		dev_info(priv->device, "Version ID not available\n");
+		return 0x0;
+	}
+
+	dev_info(priv->device, "User ID: 0x%x, Synopsys ID: 0x%x\n",
+			(unsigned int)(reg & GENMASK(15, 8)) >> 8,
+			(unsigned int)(reg & GENMASK(7, 0)));
+	return reg & GENMASK(7, 0);
+}
+
+static void stmmac_dwmac_mode_quirk(struct stmmac_priv *priv)
+{
+	struct mac_device_info *mac = priv->hw;
+
+	if (priv->chain_mode) {
+		dev_info(priv->device, "Chain mode enabled\n");
+		priv->mode = STMMAC_CHAIN_MODE;
+		mac->mode = &chain_mode_ops;
+	} else {
+		dev_info(priv->device, "Ring mode enabled\n");
+		priv->mode = STMMAC_RING_MODE;
+		mac->mode = &ring_mode_ops;
+	}
+}
+
+static int stmmac_dwmac1_quirks(struct stmmac_priv *priv)
+{
+	struct mac_device_info *mac = priv->hw;
+
+	if (priv->plat->enh_desc) {
+		dev_info(priv->device, "Enhanced/Alternate descriptors\n");
+
+		/* GMAC older than 3.50 has no extended descriptors */
+		if (priv->synopsys_id >= DWMAC_CORE_3_50) {
+			dev_info(priv->device, "Enabled extended descriptors\n");
+			priv->extend_desc = 1;
+		} else {
+			dev_warn(priv->device, "Extended descriptors not supported\n");
+		}
+
+		mac->desc = &enh_desc_ops;
+	} else {
+		dev_info(priv->device, "Normal descriptors\n");
+		mac->desc = &ndesc_ops;
+	}
+
+	stmmac_dwmac_mode_quirk(priv);
+	return 0;
+}
+
+static int stmmac_dwmac4_quirks(struct stmmac_priv *priv)
+{
+	stmmac_dwmac_mode_quirk(priv);
+	return 0;
+}
+
+static const struct stmmac_hwif_entry {
+	bool gmac;
+	bool gmac4;
+	u32 min_id;
+	const struct stmmac_regs_off regs;
+	const void *desc;
+	const void *dma;
+	const void *mac;
+	const void *hwtimestamp;
+	const void *mode;
+	const void *tc;
+	int (*setup)(struct stmmac_priv *priv);
+	int (*quirks)(struct stmmac_priv *priv);
+} stmmac_hw[] = {
+	/* NOTE: New HW versions shall go to the end of this table */
+	{
+		.gmac = false,
+		.gmac4 = false,
+		.min_id = 0,
+		.regs = {
+			.ptp_off = PTP_GMAC3_X_OFFSET,
+			.mmc_off = MMC_GMAC3_X_OFFSET,
+		},
+		.desc = NULL,
+		.dma = &dwmac100_dma_ops,
+		.mac = &dwmac100_ops,
+		.hwtimestamp = &stmmac_ptp,
+		.mode = NULL,
+		.tc = NULL,
+		.setup = dwmac100_setup,
+		.quirks = stmmac_dwmac1_quirks,
+	}, {
+		.gmac = true,
+		.gmac4 = false,
+		.min_id = 0,
+		.regs = {
+			.ptp_off = PTP_GMAC3_X_OFFSET,
+			.mmc_off = MMC_GMAC3_X_OFFSET,
+		},
+		.desc = NULL,
+		.dma = &dwmac1000_dma_ops,
+		.mac = &dwmac1000_ops,
+		.hwtimestamp = &stmmac_ptp,
+		.mode = NULL,
+		.tc = NULL,
+		.setup = dwmac1000_setup,
+		.quirks = stmmac_dwmac1_quirks,
+	}, {
+		.gmac = false,
+		.gmac4 = true,
+		.min_id = 0,
+		.regs = {
+			.ptp_off = PTP_GMAC4_OFFSET,
+			.mmc_off = MMC_GMAC4_OFFSET,
+		},
+		.desc = &dwmac4_desc_ops,
+		.dma = &dwmac4_dma_ops,
+		.mac = &dwmac4_ops,
+		.hwtimestamp = &stmmac_ptp,
+		.mode = NULL,
+		.tc = NULL,
+		.setup = dwmac4_setup,
+		.quirks = stmmac_dwmac4_quirks,
+	}, {
+		.gmac = false,
+		.gmac4 = true,
+		.min_id = DWMAC_CORE_4_00,
+		.regs = {
+			.ptp_off = PTP_GMAC4_OFFSET,
+			.mmc_off = MMC_GMAC4_OFFSET,
+		},
+		.desc = &dwmac4_desc_ops,
+		.dma = &dwmac4_dma_ops,
+		.mac = &dwmac410_ops,
+		.hwtimestamp = &stmmac_ptp,
+		.mode = &dwmac4_ring_mode_ops,
+		.tc = NULL,
+		.setup = dwmac4_setup,
+		.quirks = NULL,
+	}, {
+		.gmac = false,
+		.gmac4 = true,
+		.min_id = DWMAC_CORE_4_10,
+		.regs = {
+			.ptp_off = PTP_GMAC4_OFFSET,
+			.mmc_off = MMC_GMAC4_OFFSET,
+		},
+		.desc = &dwmac4_desc_ops,
+		.dma = &dwmac410_dma_ops,
+		.mac = &dwmac410_ops,
+		.hwtimestamp = &stmmac_ptp,
+		.mode = &dwmac4_ring_mode_ops,
+		.tc = NULL,
+		.setup = dwmac4_setup,
+		.quirks = NULL,
+	}, {
+		.gmac = false,
+		.gmac4 = true,
+		.min_id = DWMAC_CORE_5_10,
+		.regs = {
+			.ptp_off = PTP_GMAC4_OFFSET,
+			.mmc_off = MMC_GMAC4_OFFSET,
+		},
+		.desc = &dwmac4_desc_ops,
+		.dma = &dwmac410_dma_ops,
+		.mac = &dwmac510_ops,
+		.hwtimestamp = &stmmac_ptp,
+		.mode = &dwmac4_ring_mode_ops,
+		.tc = &dwmac510_tc_ops,
+		.setup = dwmac4_setup,
+		.quirks = NULL,
+	}
+};
+
+int stmmac_hwif_init(struct stmmac_priv *priv)
+{
+	bool needs_gmac4 = priv->plat->has_gmac4;
+	bool needs_gmac = priv->plat->has_gmac;
+	const struct stmmac_hwif_entry *entry;
+	struct mac_device_info *mac;
+	bool needs_setup = true;
+	int i, ret;
+	u32 id;
+
+	if (needs_gmac) {
+		id = stmmac_get_id(priv, GMAC_VERSION);
+	} else if (needs_gmac4) {
+		id = stmmac_get_id(priv, GMAC4_VERSION);
+	} else {
+		id = 0;
+	}
+
+	/* Save ID for later use */
+	priv->synopsys_id = id;
+
+	/* Lets assume some safe values first */
+	priv->ptpaddr = priv->ioaddr +
+		(needs_gmac4 ? PTP_GMAC4_OFFSET : PTP_GMAC3_X_OFFSET);
+	priv->mmcaddr = priv->ioaddr +
+		(needs_gmac4 ? MMC_GMAC4_OFFSET : MMC_GMAC3_X_OFFSET);
+
+	/* Check for HW specific setup first */
+	if (priv->plat->setup) {
+		mac = priv->plat->setup(priv);
+		needs_setup = false;
+	} else {
+		mac = devm_kzalloc(priv->device, sizeof(*mac), GFP_KERNEL);
+	}
+
+	if (!mac)
+		return -ENOMEM;
+
+	/* Fallback to generic HW */
+	for (i = ARRAY_SIZE(stmmac_hw) - 1; i >= 0; i--) {
+		entry = &stmmac_hw[i];
+
+		if (needs_gmac ^ entry->gmac)
+			continue;
+		if (needs_gmac4 ^ entry->gmac4)
+			continue;
+		/* Use synopsys_id var because some setups can override this */
+		if (priv->synopsys_id < entry->min_id)
+			continue;
+
+		/* Only use generic HW helpers if needed */
+		mac->desc = mac->desc ? : entry->desc;
+		mac->dma = mac->dma ? : entry->dma;
+		mac->mac = mac->mac ? : entry->mac;
+		mac->ptp = mac->ptp ? : entry->hwtimestamp;
+		mac->mode = mac->mode ? : entry->mode;
+		mac->tc = mac->tc ? : entry->tc;
+
+		priv->hw = mac;
+		priv->ptpaddr = priv->ioaddr + entry->regs.ptp_off;
+		priv->mmcaddr = priv->ioaddr + entry->regs.mmc_off;
+
+		/* Entry found */
+		if (needs_setup) {
+			ret = entry->setup(priv);
+			if (ret)
+				return ret;
+		}
+
+		/* Run quirks, if needed */
+		if (entry->quirks) {
+			ret = entry->quirks(priv);
+			if (ret)
+				return ret;
+		}
+
+		return 0;
+	}
+
+	dev_err(priv->device, "Failed to find HW IF (id=0x%x, gmac=%d/%d)\n",
+			id, needs_gmac, needs_gmac4);
+	return -EINVAL;
+}
diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h
new file mode 100644
index 000000000000..f499a7fad6f0
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h
@@ -0,0 +1,470 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+// Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
+// stmmac HW Interface Callbacks
+
+#ifndef __STMMAC_HWIF_H__
+#define __STMMAC_HWIF_H__
+
+#include <linux/netdevice.h>
+
+#define stmmac_do_void_callback(__priv, __module, __cname,  __arg0, __args...) \
+({ \
+	int __result = -EINVAL; \
+	if ((__priv)->hw->__module && (__priv)->hw->__module->__cname) { \
+		(__priv)->hw->__module->__cname((__arg0), ##__args); \
+		__result = 0; \
+	} \
+	__result; \
+})
+#define stmmac_do_callback(__priv, __module, __cname,  __arg0, __args...) \
+({ \
+	int __result = -EINVAL; \
+	if ((__priv)->hw->__module && (__priv)->hw->__module->__cname) \
+		__result = (__priv)->hw->__module->__cname((__arg0), ##__args); \
+	__result; \
+})
+
+struct stmmac_extra_stats;
+struct stmmac_safety_stats;
+struct dma_desc;
+struct dma_extended_desc;
+
+/* Descriptors helpers */
+struct stmmac_desc_ops {
+	/* DMA RX descriptor ring initialization */
+	void (*init_rx_desc)(struct dma_desc *p, int disable_rx_ic, int mode,
+			int end);
+	/* DMA TX descriptor ring initialization */
+	void (*init_tx_desc)(struct dma_desc *p, int mode, int end);
+	/* Invoked by the xmit function to prepare the tx descriptor */
+	void (*prepare_tx_desc)(struct dma_desc *p, int is_fs, int len,
+			bool csum_flag, int mode, bool tx_own, bool ls,
+			unsigned int tot_pkt_len);
+	void (*prepare_tso_tx_desc)(struct dma_desc *p, int is_fs, int len1,
+			int len2, bool tx_own, bool ls, unsigned int tcphdrlen,
+			unsigned int tcppayloadlen);
+	/* Set/get the owner of the descriptor */
+	void (*set_tx_owner)(struct dma_desc *p);
+	int (*get_tx_owner)(struct dma_desc *p);
+	/* Clean the tx descriptor as soon as the tx irq is received */
+	void (*release_tx_desc)(struct dma_desc *p, int mode);
+	/* Clear interrupt on tx frame completion. When this bit is
+	 * set an interrupt happens as soon as the frame is transmitted */
+	void (*set_tx_ic)(struct dma_desc *p);
+	/* Last tx segment reports the transmit status */
+	int (*get_tx_ls)(struct dma_desc *p);
+	/* Return the transmit status looking at the TDES1 */
+	int (*tx_status)(void *data, struct stmmac_extra_stats *x,
+			struct dma_desc *p, void __iomem *ioaddr);
+	/* Get the buffer size from the descriptor */
+	int (*get_tx_len)(struct dma_desc *p);
+	/* Handle extra events on specific interrupts hw dependent */
+	void (*set_rx_owner)(struct dma_desc *p, int disable_rx_ic);
+	/* Get the receive frame size */
+	int (*get_rx_frame_len)(struct dma_desc *p, int rx_coe_type);
+	/* Return the reception status looking at the RDES1 */
+	int (*rx_status)(void *data, struct stmmac_extra_stats *x,
+			struct dma_desc *p);
+	void (*rx_extended_status)(void *data, struct stmmac_extra_stats *x,
+			struct dma_extended_desc *p);
+	/* Set tx timestamp enable bit */
+	void (*enable_tx_timestamp) (struct dma_desc *p);
+	/* get tx timestamp status */
+	int (*get_tx_timestamp_status) (struct dma_desc *p);
+	/* get timestamp value */
+	void (*get_timestamp)(void *desc, u32 ats, u64 *ts);
+	/* get rx timestamp status */
+	int (*get_rx_timestamp_status)(void *desc, void *next_desc, u32 ats);
+	/* Display ring */
+	void (*display_ring)(void *head, unsigned int size, bool rx);
+	/* set MSS via context descriptor */
+	void (*set_mss)(struct dma_desc *p, unsigned int mss);
+	/* get descriptor skbuff address */
+	void (*get_addr)(struct dma_desc *p, unsigned int *addr);
+	/* set descriptor skbuff address */
+	void (*set_addr)(struct dma_desc *p, dma_addr_t addr);
+	/* clear descriptor */
+	void (*clear)(struct dma_desc *p);
+};
+
+#define stmmac_init_rx_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, init_rx_desc, __args)
+#define stmmac_init_tx_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, init_tx_desc, __args)
+#define stmmac_prepare_tx_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, prepare_tx_desc, __args)
+#define stmmac_prepare_tso_tx_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, prepare_tso_tx_desc, __args)
+#define stmmac_set_tx_owner(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_tx_owner, __args)
+#define stmmac_get_tx_owner(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_tx_owner, __args)
+#define stmmac_release_tx_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, release_tx_desc, __args)
+#define stmmac_set_tx_ic(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_tx_ic, __args)
+#define stmmac_get_tx_ls(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_tx_ls, __args)
+#define stmmac_tx_status(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, tx_status, __args)
+#define stmmac_get_tx_len(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_tx_len, __args)
+#define stmmac_set_rx_owner(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_rx_owner, __args)
+#define stmmac_get_rx_frame_len(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_rx_frame_len, __args)
+#define stmmac_rx_status(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, rx_status, __args)
+#define stmmac_rx_extended_status(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, rx_extended_status, __args)
+#define stmmac_enable_tx_timestamp(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, enable_tx_timestamp, __args)
+#define stmmac_get_tx_timestamp_status(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_tx_timestamp_status, __args)
+#define stmmac_get_timestamp(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, get_timestamp, __args)
+#define stmmac_get_rx_timestamp_status(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_rx_timestamp_status, __args)
+#define stmmac_display_ring(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, display_ring, __args)
+#define stmmac_set_mss(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_mss, __args)
+#define stmmac_get_desc_addr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, get_addr, __args)
+#define stmmac_set_desc_addr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_addr, __args)
+#define stmmac_clear_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, clear, __args)
+
+struct stmmac_dma_cfg;
+struct dma_features;
+
+/* Specific DMA helpers */
+struct stmmac_dma_ops {
+	/* DMA core initialization */
+	int (*reset)(void __iomem *ioaddr);
+	void (*init)(void __iomem *ioaddr, struct stmmac_dma_cfg *dma_cfg,
+		     int atds);
+	void (*init_chan)(void __iomem *ioaddr,
+			  struct stmmac_dma_cfg *dma_cfg, u32 chan);
+	void (*init_rx_chan)(void __iomem *ioaddr,
+			     struct stmmac_dma_cfg *dma_cfg,
+			     u32 dma_rx_phy, u32 chan);
+	void (*init_tx_chan)(void __iomem *ioaddr,
+			     struct stmmac_dma_cfg *dma_cfg,
+			     u32 dma_tx_phy, u32 chan);
+	/* Configure the AXI Bus Mode Register */
+	void (*axi)(void __iomem *ioaddr, struct stmmac_axi *axi);
+	/* Dump DMA registers */
+	void (*dump_regs)(void __iomem *ioaddr, u32 *reg_space);
+	void (*dma_rx_mode)(void __iomem *ioaddr, int mode, u32 channel,
+			    int fifosz, u8 qmode);
+	void (*dma_tx_mode)(void __iomem *ioaddr, int mode, u32 channel,
+			    int fifosz, u8 qmode);
+	/* To track extra statistic (if supported) */
+	void (*dma_diagnostic_fr) (void *data, struct stmmac_extra_stats *x,
+				   void __iomem *ioaddr);
+	void (*enable_dma_transmission) (void __iomem *ioaddr);
+	void (*enable_dma_irq)(void __iomem *ioaddr, u32 chan);
+	void (*disable_dma_irq)(void __iomem *ioaddr, u32 chan);
+	void (*start_tx)(void __iomem *ioaddr, u32 chan);
+	void (*stop_tx)(void __iomem *ioaddr, u32 chan);
+	void (*start_rx)(void __iomem *ioaddr, u32 chan);
+	void (*stop_rx)(void __iomem *ioaddr, u32 chan);
+	int (*dma_interrupt) (void __iomem *ioaddr,
+			      struct stmmac_extra_stats *x, u32 chan);
+	/* If supported then get the optional core features */
+	void (*get_hw_feature)(void __iomem *ioaddr,
+			       struct dma_features *dma_cap);
+	/* Program the HW RX Watchdog */
+	void (*rx_watchdog)(void __iomem *ioaddr, u32 riwt, u32 number_chan);
+	void (*set_tx_ring_len)(void __iomem *ioaddr, u32 len, u32 chan);
+	void (*set_rx_ring_len)(void __iomem *ioaddr, u32 len, u32 chan);
+	void (*set_rx_tail_ptr)(void __iomem *ioaddr, u32 tail_ptr, u32 chan);
+	void (*set_tx_tail_ptr)(void __iomem *ioaddr, u32 tail_ptr, u32 chan);
+	void (*enable_tso)(void __iomem *ioaddr, bool en, u32 chan);
+};
+
+#define stmmac_reset(__priv, __args...) \
+	stmmac_do_callback(__priv, dma, reset, __args)
+#define stmmac_dma_init(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, init, __args)
+#define stmmac_init_chan(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, init_chan, __args)
+#define stmmac_init_rx_chan(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, init_rx_chan, __args)
+#define stmmac_init_tx_chan(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, init_tx_chan, __args)
+#define stmmac_axi(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, axi, __args)
+#define stmmac_dump_dma_regs(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, dump_regs, __args)
+#define stmmac_dma_rx_mode(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, dma_rx_mode, __args)
+#define stmmac_dma_tx_mode(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, dma_tx_mode, __args)
+#define stmmac_dma_diagnostic_fr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, dma_diagnostic_fr, __args)
+#define stmmac_enable_dma_transmission(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, enable_dma_transmission, __args)
+#define stmmac_enable_dma_irq(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, enable_dma_irq, __args)
+#define stmmac_disable_dma_irq(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, disable_dma_irq, __args)
+#define stmmac_start_tx(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, start_tx, __args)
+#define stmmac_stop_tx(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, stop_tx, __args)
+#define stmmac_start_rx(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, start_rx, __args)
+#define stmmac_stop_rx(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, stop_rx, __args)
+#define stmmac_dma_interrupt_status(__priv, __args...) \
+	stmmac_do_callback(__priv, dma, dma_interrupt, __args)
+#define stmmac_get_hw_feature(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, get_hw_feature, __args)
+#define stmmac_rx_watchdog(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, rx_watchdog, __args)
+#define stmmac_set_tx_ring_len(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, set_tx_ring_len, __args)
+#define stmmac_set_rx_ring_len(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, set_rx_ring_len, __args)
+#define stmmac_set_rx_tail_ptr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, set_rx_tail_ptr, __args)
+#define stmmac_set_tx_tail_ptr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, set_tx_tail_ptr, __args)
+#define stmmac_enable_tso(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, enable_tso, __args)
+
+struct mac_device_info;
+struct net_device;
+struct rgmii_adv;
+struct stmmac_safety_stats;
+struct stmmac_tc_entry;
+
+/* Helpers to program the MAC core */
+struct stmmac_ops {
+	/* MAC core initialization */
+	void (*core_init)(struct mac_device_info *hw, struct net_device *dev);
+	/* Enable the MAC RX/TX */
+	void (*set_mac)(void __iomem *ioaddr, bool enable);
+	/* Enable and verify that the IPC module is supported */
+	int (*rx_ipc)(struct mac_device_info *hw);
+	/* Enable RX Queues */
+	void (*rx_queue_enable)(struct mac_device_info *hw, u8 mode, u32 queue);
+	/* RX Queues Priority */
+	void (*rx_queue_prio)(struct mac_device_info *hw, u32 prio, u32 queue);
+	/* TX Queues Priority */
+	void (*tx_queue_prio)(struct mac_device_info *hw, u32 prio, u32 queue);
+	/* RX Queues Routing */
+	void (*rx_queue_routing)(struct mac_device_info *hw, u8 packet,
+				 u32 queue);
+	/* Program RX Algorithms */
+	void (*prog_mtl_rx_algorithms)(struct mac_device_info *hw, u32 rx_alg);
+	/* Program TX Algorithms */
+	void (*prog_mtl_tx_algorithms)(struct mac_device_info *hw, u32 tx_alg);
+	/* Set MTL TX queues weight */
+	void (*set_mtl_tx_queue_weight)(struct mac_device_info *hw,
+					u32 weight, u32 queue);
+	/* RX MTL queue to RX dma mapping */
+	void (*map_mtl_to_dma)(struct mac_device_info *hw, u32 queue, u32 chan);
+	/* Configure AV Algorithm */
+	void (*config_cbs)(struct mac_device_info *hw, u32 send_slope,
+			   u32 idle_slope, u32 high_credit, u32 low_credit,
+			   u32 queue);
+	/* Dump MAC registers */
+	void (*dump_regs)(struct mac_device_info *hw, u32 *reg_space);
+	/* Handle extra events on specific interrupts hw dependent */
+	int (*host_irq_status)(struct mac_device_info *hw,
+			       struct stmmac_extra_stats *x);
+	/* Handle MTL interrupts */
+	int (*host_mtl_irq_status)(struct mac_device_info *hw, u32 chan);
+	/* Multicast filter setting */
+	void (*set_filter)(struct mac_device_info *hw, struct net_device *dev);
+	/* Flow control setting */
+	void (*flow_ctrl)(struct mac_device_info *hw, unsigned int duplex,
+			  unsigned int fc, unsigned int pause_time, u32 tx_cnt);
+	/* Set power management mode (e.g. magic frame) */
+	void (*pmt)(struct mac_device_info *hw, unsigned long mode);
+	/* Set/Get Unicast MAC addresses */
+	void (*set_umac_addr)(struct mac_device_info *hw, unsigned char *addr,
+			      unsigned int reg_n);
+	void (*get_umac_addr)(struct mac_device_info *hw, unsigned char *addr,
+			      unsigned int reg_n);
+	void (*set_eee_mode)(struct mac_device_info *hw,
+			     bool en_tx_lpi_clockgating);
+	void (*reset_eee_mode)(struct mac_device_info *hw);
+	void (*set_eee_timer)(struct mac_device_info *hw, int ls, int tw);
+	void (*set_eee_pls)(struct mac_device_info *hw, int link);
+	void (*debug)(void __iomem *ioaddr, struct stmmac_extra_stats *x,
+		      u32 rx_queues, u32 tx_queues);
+	/* PCS calls */
+	void (*pcs_ctrl_ane)(void __iomem *ioaddr, bool ane, bool srgmi_ral,
+			     bool loopback);
+	void (*pcs_rane)(void __iomem *ioaddr, bool restart);
+	void (*pcs_get_adv_lp)(void __iomem *ioaddr, struct rgmii_adv *adv);
+	/* Safety Features */
+	int (*safety_feat_config)(void __iomem *ioaddr, unsigned int asp);
+	int (*safety_feat_irq_status)(struct net_device *ndev,
+			void __iomem *ioaddr, unsigned int asp,
+			struct stmmac_safety_stats *stats);
+	int (*safety_feat_dump)(struct stmmac_safety_stats *stats,
+			int index, unsigned long *count, const char **desc);
+	/* Flexible RX Parser */
+	int (*rxp_config)(void __iomem *ioaddr, struct stmmac_tc_entry *entries,
+			  unsigned int count);
+};
+
+#define stmmac_core_init(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, core_init, __args)
+#define stmmac_mac_set(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_mac, __args)
+#define stmmac_rx_ipc(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, rx_ipc, __args)
+#define stmmac_rx_queue_enable(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, rx_queue_enable, __args)
+#define stmmac_rx_queue_prio(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, rx_queue_prio, __args)
+#define stmmac_tx_queue_prio(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, tx_queue_prio, __args)
+#define stmmac_rx_queue_routing(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, rx_queue_routing, __args)
+#define stmmac_prog_mtl_rx_algorithms(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, prog_mtl_rx_algorithms, __args)
+#define stmmac_prog_mtl_tx_algorithms(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, prog_mtl_tx_algorithms, __args)
+#define stmmac_set_mtl_tx_queue_weight(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_mtl_tx_queue_weight, __args)
+#define stmmac_map_mtl_to_dma(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, map_mtl_to_dma, __args)
+#define stmmac_config_cbs(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, config_cbs, __args)
+#define stmmac_dump_mac_regs(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, dump_regs, __args)
+#define stmmac_host_irq_status(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, host_irq_status, __args)
+#define stmmac_host_mtl_irq_status(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, host_mtl_irq_status, __args)
+#define stmmac_set_filter(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_filter, __args)
+#define stmmac_flow_ctrl(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, flow_ctrl, __args)
+#define stmmac_pmt(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, pmt, __args)
+#define stmmac_set_umac_addr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_umac_addr, __args)
+#define stmmac_get_umac_addr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, get_umac_addr, __args)
+#define stmmac_set_eee_mode(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_eee_mode, __args)
+#define stmmac_reset_eee_mode(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, reset_eee_mode, __args)
+#define stmmac_set_eee_timer(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_eee_timer, __args)
+#define stmmac_set_eee_pls(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_eee_pls, __args)
+#define stmmac_mac_debug(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, debug, __args)
+#define stmmac_pcs_ctrl_ane(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, pcs_ctrl_ane, __args)
+#define stmmac_pcs_rane(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, pcs_rane, __args)
+#define stmmac_pcs_get_adv_lp(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, pcs_get_adv_lp, __args)
+#define stmmac_safety_feat_config(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, safety_feat_config, __args)
+#define stmmac_safety_feat_irq_status(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, safety_feat_irq_status, __args)
+#define stmmac_safety_feat_dump(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, safety_feat_dump, __args)
+#define stmmac_rxp_config(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, rxp_config, __args)
+
+/* PTP and HW Timer helpers */
+struct stmmac_hwtimestamp {
+	void (*config_hw_tstamping) (void __iomem *ioaddr, u32 data);
+	void (*config_sub_second_increment)(void __iomem *ioaddr, u32 ptp_clock,
+					   int gmac4, u32 *ssinc);
+	int (*init_systime) (void __iomem *ioaddr, u32 sec, u32 nsec);
+	int (*config_addend) (void __iomem *ioaddr, u32 addend);
+	int (*adjust_systime) (void __iomem *ioaddr, u32 sec, u32 nsec,
+			       int add_sub, int gmac4);
+	void (*get_systime) (void __iomem *ioaddr, u64 *systime);
+};
+
+#define stmmac_config_hw_tstamping(__priv, __args...) \
+	stmmac_do_void_callback(__priv, ptp, config_hw_tstamping, __args)
+#define stmmac_config_sub_second_increment(__priv, __args...) \
+	stmmac_do_void_callback(__priv, ptp, config_sub_second_increment, __args)
+#define stmmac_init_systime(__priv, __args...) \
+	stmmac_do_callback(__priv, ptp, init_systime, __args)
+#define stmmac_config_addend(__priv, __args...) \
+	stmmac_do_callback(__priv, ptp, config_addend, __args)
+#define stmmac_adjust_systime(__priv, __args...) \
+	stmmac_do_callback(__priv, ptp, adjust_systime, __args)
+#define stmmac_get_systime(__priv, __args...) \
+	stmmac_do_void_callback(__priv, ptp, get_systime, __args)
+
+/* Helpers to manage the descriptors for chain and ring modes */
+struct stmmac_mode_ops {
+	void (*init) (void *des, dma_addr_t phy_addr, unsigned int size,
+		      unsigned int extend_desc);
+	unsigned int (*is_jumbo_frm) (int len, int ehn_desc);
+	int (*jumbo_frm)(void *priv, struct sk_buff *skb, int csum);
+	int (*set_16kib_bfsize)(int mtu);
+	void (*init_desc3)(struct dma_desc *p);
+	void (*refill_desc3) (void *priv, struct dma_desc *p);
+	void (*clean_desc3) (void *priv, struct dma_desc *p);
+};
+
+#define stmmac_mode_init(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mode, init, __args)
+#define stmmac_is_jumbo_frm(__priv, __args...) \
+	stmmac_do_callback(__priv, mode, is_jumbo_frm, __args)
+#define stmmac_jumbo_frm(__priv, __args...) \
+	stmmac_do_callback(__priv, mode, jumbo_frm, __args)
+#define stmmac_set_16kib_bfsize(__priv, __args...) \
+	stmmac_do_callback(__priv, mode, set_16kib_bfsize, __args)
+#define stmmac_init_desc3(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mode, init_desc3, __args)
+#define stmmac_refill_desc3(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mode, refill_desc3, __args)
+#define stmmac_clean_desc3(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mode, clean_desc3, __args)
+
+struct stmmac_priv;
+struct tc_cls_u32_offload;
+
+struct stmmac_tc_ops {
+	int (*init)(struct stmmac_priv *priv);
+	int (*setup_cls_u32)(struct stmmac_priv *priv,
+			     struct tc_cls_u32_offload *cls);
+};
+
+#define stmmac_tc_init(__priv, __args...) \
+	stmmac_do_callback(__priv, tc, init, __args)
+#define stmmac_tc_setup_cls_u32(__priv, __args...) \
+	stmmac_do_callback(__priv, tc, setup_cls_u32, __args)
+
+struct stmmac_regs_off {
+	u32 ptp_off;
+	u32 mmc_off;
+};
+
+extern const struct stmmac_ops dwmac100_ops;
+extern const struct stmmac_dma_ops dwmac100_dma_ops;
+extern const struct stmmac_ops dwmac1000_ops;
+extern const struct stmmac_dma_ops dwmac1000_dma_ops;
+extern const struct stmmac_ops dwmac4_ops;
+extern const struct stmmac_dma_ops dwmac4_dma_ops;
+extern const struct stmmac_ops dwmac410_ops;
+extern const struct stmmac_dma_ops dwmac410_dma_ops;
+extern const struct stmmac_ops dwmac510_ops;
+extern const struct stmmac_tc_ops dwmac510_tc_ops;
+
+#define GMAC_VERSION		0x00000020	/* GMAC CORE Version */
+#define GMAC4_VERSION		0x00000110	/* GMAC4+ CORE Version */
+
+int stmmac_hwif_init(struct stmmac_priv *priv);
+
+#endif /* __STMMAC_HWIF_H__ */
diff --git a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
index ebd9e5e00f16..de65bb29feba 100644
--- a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
@@ -168,7 +168,7 @@ static void ndesc_set_tx_owner(struct dma_desc *p)
 	p->des0 |= cpu_to_le32(TDES0_OWN);
 }
 
-static void ndesc_set_rx_owner(struct dma_desc *p)
+static void ndesc_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
 {
 	p->des0 |= cpu_to_le32(RDES0_OWN);
 }
@@ -253,7 +253,7 @@ static int ndesc_get_tx_timestamp_status(struct dma_desc *p)
 	return (le32_to_cpu(p->des0) & TDES0_TIME_STAMP_STATUS) >> 17;
 }
 
-static u64 ndesc_get_timestamp(void *desc, u32 ats)
+static void ndesc_get_timestamp(void *desc, u32 ats, u64 *ts)
 {
 	struct dma_desc *p = (struct dma_desc *)desc;
 	u64 ns;
@@ -262,7 +262,7 @@ static u64 ndesc_get_timestamp(void *desc, u32 ats)
 	/* convert high/sec time stamp value to nanosecond */
 	ns += le32_to_cpu(p->des3) * 1000000000ULL;
 
-	return ns;
+	*ts = ns;
 }
 
 static int ndesc_get_rx_timestamp_status(void *desc, void *next_desc, u32 ats)
@@ -297,6 +297,21 @@ static void ndesc_display_ring(void *head, unsigned int size, bool rx)
 	pr_info("\n");
 }
 
+static void ndesc_get_addr(struct dma_desc *p, unsigned int *addr)
+{
+	*addr = le32_to_cpu(p->des2);
+}
+
+static void ndesc_set_addr(struct dma_desc *p, dma_addr_t addr)
+{
+	p->des2 = cpu_to_le32(addr);
+}
+
+static void ndesc_clear(struct dma_desc *p)
+{
+	p->des2 = 0;
+}
+
 const struct stmmac_desc_ops ndesc_ops = {
 	.tx_status = ndesc_get_tx_status,
 	.rx_status = ndesc_get_rx_status,
@@ -316,4 +331,7 @@ const struct stmmac_desc_ops ndesc_ops = {
 	.get_timestamp = ndesc_get_timestamp,
 	.get_rx_timestamp_status = ndesc_get_rx_timestamp_status,
 	.display_ring = ndesc_display_ring,
+	.get_addr = ndesc_get_addr,
+	.set_addr = ndesc_set_addr,
+	.clear = ndesc_clear,
 };
diff --git a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
index 28e4b5d50ce6..a7ffc73fffe8 100644
--- a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
+++ b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
@@ -24,7 +24,7 @@
 
 #include "stmmac.h"
 
-static int stmmac_jumbo_frm(void *p, struct sk_buff *skb, int csum)
+static int jumbo_frm(void *p, struct sk_buff *skb, int csum)
 {
 	struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)p;
 	unsigned int nopaged_len = skb_headlen(skb);
@@ -58,9 +58,8 @@ static int stmmac_jumbo_frm(void *p, struct sk_buff *skb, int csum)
 		tx_q->tx_skbuff_dma[entry].is_jumbo = true;
 
 		desc->des3 = cpu_to_le32(des2 + BUF_SIZE_4KiB);
-		priv->hw->desc->prepare_tx_desc(desc, 1, bmax, csum,
-						STMMAC_RING_MODE, 0,
-						false, skb->len);
+		stmmac_prepare_tx_desc(priv, desc, 1, bmax, csum,
+				STMMAC_RING_MODE, 0, false, skb->len);
 		tx_q->tx_skbuff[entry] = NULL;
 		entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
 
@@ -79,9 +78,8 @@ static int stmmac_jumbo_frm(void *p, struct sk_buff *skb, int csum)
 		tx_q->tx_skbuff_dma[entry].is_jumbo = true;
 
 		desc->des3 = cpu_to_le32(des2 + BUF_SIZE_4KiB);
-		priv->hw->desc->prepare_tx_desc(desc, 0, len, csum,
-						STMMAC_RING_MODE, 1,
-						true, skb->len);
+		stmmac_prepare_tx_desc(priv, desc, 0, len, csum,
+				STMMAC_RING_MODE, 1, true, skb->len);
 	} else {
 		des2 = dma_map_single(priv->device, skb->data,
 				      nopaged_len, DMA_TO_DEVICE);
@@ -92,9 +90,8 @@ static int stmmac_jumbo_frm(void *p, struct sk_buff *skb, int csum)
 		tx_q->tx_skbuff_dma[entry].len = nopaged_len;
 		tx_q->tx_skbuff_dma[entry].is_jumbo = true;
 		desc->des3 = cpu_to_le32(des2 + BUF_SIZE_4KiB);
-		priv->hw->desc->prepare_tx_desc(desc, 1, nopaged_len, csum,
-						STMMAC_RING_MODE, 0,
-						true, skb->len);
+		stmmac_prepare_tx_desc(priv, desc, 1, nopaged_len, csum,
+				STMMAC_RING_MODE, 0, true, skb->len);
 	}
 
 	tx_q->cur_tx = entry;
@@ -102,7 +99,7 @@ static int stmmac_jumbo_frm(void *p, struct sk_buff *skb, int csum)
 	return entry;
 }
 
-static unsigned int stmmac_is_jumbo_frm(int len, int enh_desc)
+static unsigned int is_jumbo_frm(int len, int enh_desc)
 {
 	unsigned int ret = 0;
 
@@ -112,7 +109,7 @@ static unsigned int stmmac_is_jumbo_frm(int len, int enh_desc)
 	return ret;
 }
 
-static void stmmac_refill_desc3(void *priv_ptr, struct dma_desc *p)
+static void refill_desc3(void *priv_ptr, struct dma_desc *p)
 {
 	struct stmmac_priv *priv = (struct stmmac_priv *)priv_ptr;
 
@@ -122,12 +119,12 @@ static void stmmac_refill_desc3(void *priv_ptr, struct dma_desc *p)
 }
 
 /* In ring mode we need to fill the desc3 because it is used as buffer */
-static void stmmac_init_desc3(struct dma_desc *p)
+static void init_desc3(struct dma_desc *p)
 {
 	p->des3 = cpu_to_le32(le32_to_cpu(p->des2) + BUF_SIZE_8KiB);
 }
 
-static void stmmac_clean_desc3(void *priv_ptr, struct dma_desc *p)
+static void clean_desc3(void *priv_ptr, struct dma_desc *p)
 {
 	struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)priv_ptr;
 	struct stmmac_priv *priv = tx_q->priv_data;
@@ -140,7 +137,7 @@ static void stmmac_clean_desc3(void *priv_ptr, struct dma_desc *p)
 		p->des3 = 0;
 }
 
-static int stmmac_set_16kib_bfsize(int mtu)
+static int set_16kib_bfsize(int mtu)
 {
 	int ret = 0;
 	if (unlikely(mtu >= BUF_SIZE_8KiB))
@@ -149,10 +146,10 @@ static int stmmac_set_16kib_bfsize(int mtu)
 }
 
 const struct stmmac_mode_ops ring_mode_ops = {
-	.is_jumbo_frm = stmmac_is_jumbo_frm,
-	.jumbo_frm = stmmac_jumbo_frm,
-	.refill_desc3 = stmmac_refill_desc3,
-	.init_desc3 = stmmac_init_desc3,
-	.clean_desc3 = stmmac_clean_desc3,
-	.set_16kib_bfsize = stmmac_set_16kib_bfsize,
+	.is_jumbo_frm = is_jumbo_frm,
+	.jumbo_frm = jumbo_frm,
+	.refill_desc3 = refill_desc3,
+	.init_desc3 = init_desc3,
+	.clean_desc3 = clean_desc3,
+	.set_16kib_bfsize = set_16kib_bfsize,
 };
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
index da50451f8999..fbfe5dcefa87 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
@@ -76,11 +76,36 @@ struct stmmac_rx_queue {
 	struct napi_struct napi ____cacheline_aligned_in_smp;
 };
 
+struct stmmac_tc_entry {
+	bool in_use;
+	bool in_hw;
+	bool is_last;
+	bool is_frag;
+	void *frag_ptr;
+	unsigned int table_pos;
+	u32 handle;
+	u32 prio;
+	struct {
+		u32 match_data;
+		u32 match_en;
+		u8 af:1;
+		u8 rf:1;
+		u8 im:1;
+		u8 nc:1;
+		u8 res1:4;
+		u8 frame_offset;
+		u8 ok_index;
+		u8 dma_ch_no;
+		u32 res2;
+	} __packed val;
+};
+
 struct stmmac_priv {
 	/* Frequently used values are kept adjacent for cache effect */
 	u32 tx_count_frames;
 	u32 tx_coal_frames;
 	u32 tx_coal_timer;
+	bool tx_timer_armed;
 
 	int tx_coalesce;
 	int hwts_tx_en;
@@ -97,7 +122,7 @@ struct stmmac_priv {
 	struct net_device *dev;
 	struct device *device;
 	struct mac_device_info *hw;
-	spinlock_t lock;
+	struct mutex lock;
 
 	/* RX Queue */
 	struct stmmac_rx_queue rx_queue[MTL_MAX_RX_QUEUES];
@@ -130,6 +155,7 @@ struct stmmac_priv {
 	int eee_active;
 	int tx_lpi_timer;
 	unsigned int mode;
+	unsigned int chain_mode;
 	int extend_desc;
 	struct ptp_clock *ptp_clock;
 	struct ptp_clock_info ptp_clock_ops;
@@ -150,6 +176,11 @@ struct stmmac_priv {
 	unsigned long state;
 	struct workqueue_struct *wq;
 	struct work_struct service_task;
+
+	/* TC Handling */
+	unsigned int tc_entries_max;
+	unsigned int tc_off_max;
+	struct stmmac_tc_entry *tc_entries;
 };
 
 enum stmmac_state {
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
index 2c6ed47704fc..5710864fa809 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
@@ -291,11 +291,9 @@ static int stmmac_ethtool_get_link_ksettings(struct net_device *dev,
 		cmd->base.speed = priv->xstats.pcs_speed;
 
 		/* Get and convert ADV/LP_ADV from the HW AN registers */
-		if (!priv->hw->mac->pcs_get_adv_lp)
+		if (stmmac_pcs_get_adv_lp(priv, priv->ioaddr, &adv))
 			return -EOPNOTSUPP;	/* should never happen indeed */
 
-		priv->hw->mac->pcs_get_adv_lp(priv->ioaddr, &adv);
-
 		/* Encoding of PSE bits is defined in 802.3z, 37.2.1.4 */
 
 		ethtool_convert_link_mode_to_legacy_u32(
@@ -392,13 +390,9 @@ stmmac_ethtool_set_link_ksettings(struct net_device *dev,
 			ADVERTISED_10baseT_Half |
 			ADVERTISED_10baseT_Full);
 
-		spin_lock(&priv->lock);
-
-		if (priv->hw->mac->pcs_ctrl_ane)
-			priv->hw->mac->pcs_ctrl_ane(priv->ioaddr, 1,
-						    priv->hw->ps, 0);
-
-		spin_unlock(&priv->lock);
+		mutex_lock(&priv->lock);
+		stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);
+		mutex_unlock(&priv->lock);
 
 		return 0;
 	}
@@ -442,8 +436,8 @@ static void stmmac_ethtool_gregs(struct net_device *dev,
 
 	memset(reg_space, 0x0, REG_SPACE_SIZE);
 
-	priv->hw->mac->dump_regs(priv->hw, reg_space);
-	priv->hw->dma->dump_regs(priv->ioaddr, reg_space);
+	stmmac_dump_mac_regs(priv, priv->hw, reg_space);
+	stmmac_dump_dma_regs(priv, priv->ioaddr, reg_space);
 	/* Copy DMA registers to where ethtool expects them */
 	memcpy(&reg_space[ETHTOOL_DMA_OFFSET], &reg_space[DMA_BUS_MODE / 4],
 	       NUM_DWMAC1000_DMA_REGS * 4);
@@ -454,15 +448,13 @@ stmmac_get_pauseparam(struct net_device *netdev,
 		      struct ethtool_pauseparam *pause)
 {
 	struct stmmac_priv *priv = netdev_priv(netdev);
+	struct rgmii_adv adv_lp;
 
 	pause->rx_pause = 0;
 	pause->tx_pause = 0;
 
-	if (priv->hw->pcs && priv->hw->mac->pcs_get_adv_lp) {
-		struct rgmii_adv adv_lp;
-
+	if (priv->hw->pcs && !stmmac_pcs_get_adv_lp(priv, priv->ioaddr, &adv_lp)) {
 		pause->autoneg = 1;
-		priv->hw->mac->pcs_get_adv_lp(priv->ioaddr, &adv_lp);
 		if (!adv_lp.pause)
 			return;
 	} else {
@@ -488,12 +480,10 @@ stmmac_set_pauseparam(struct net_device *netdev,
 	u32 tx_cnt = priv->plat->tx_queues_to_use;
 	struct phy_device *phy = netdev->phydev;
 	int new_pause = FLOW_OFF;
+	struct rgmii_adv adv_lp;
 
-	if (priv->hw->pcs && priv->hw->mac->pcs_get_adv_lp) {
-		struct rgmii_adv adv_lp;
-
+	if (priv->hw->pcs && !stmmac_pcs_get_adv_lp(priv, priv->ioaddr, &adv_lp)) {
 		pause->autoneg = 1;
-		priv->hw->mac->pcs_get_adv_lp(priv->ioaddr, &adv_lp);
 		if (!adv_lp.pause)
 			return -EOPNOTSUPP;
 	} else {
@@ -515,37 +505,32 @@ stmmac_set_pauseparam(struct net_device *netdev,
 			return phy_start_aneg(phy);
 	}
 
-	priv->hw->mac->flow_ctrl(priv->hw, phy->duplex, priv->flow_ctrl,
-				 priv->pause, tx_cnt);
+	stmmac_flow_ctrl(priv, priv->hw, phy->duplex, priv->flow_ctrl,
+			priv->pause, tx_cnt);
 	return 0;
 }
 
 static void stmmac_get_ethtool_stats(struct net_device *dev,
 				 struct ethtool_stats *dummy, u64 *data)
 {
-	const char *(*dump)(struct stmmac_safety_stats *stats, int index,
-			unsigned long *count);
 	struct stmmac_priv *priv = netdev_priv(dev);
 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
 	unsigned long count;
-	int i, j = 0;
-
-	if (priv->dma_cap.asp && priv->hw->mac->safety_feat_dump) {
-		dump = priv->hw->mac->safety_feat_dump;
+	int i, j = 0, ret;
 
+	if (priv->dma_cap.asp) {
 		for (i = 0; i < STMMAC_SAFETY_FEAT_SIZE; i++) {
-			if (dump(&priv->sstats, i, &count))
+			if (!stmmac_safety_feat_dump(priv, &priv->sstats, i,
+						&count, NULL))
 				data[j++] = count;
 		}
 	}
 
 	/* Update the DMA HW counters for dwmac10/100 */
-	if (priv->hw->dma->dma_diagnostic_fr)
-		priv->hw->dma->dma_diagnostic_fr(&dev->stats,
-						 (void *) &priv->xstats,
-						 priv->ioaddr);
-	else {
+	ret = stmmac_dma_diagnostic_fr(priv, &dev->stats, (void *) &priv->xstats,
+			priv->ioaddr);
+	if (ret) {
 		/* If supported, for new GMAC chips expose the MMC counters */
 		if (priv->dma_cap.rmon) {
 			dwmac_mmc_read(priv->mmcaddr, &priv->mmc);
@@ -565,11 +550,10 @@ static void stmmac_get_ethtool_stats(struct net_device *dev,
 				priv->xstats.phy_eee_wakeup_error_n = val;
 		}
 
-		if ((priv->hw->mac->debug) &&
-		    (priv->synopsys_id >= DWMAC_CORE_3_50))
-			priv->hw->mac->debug(priv->ioaddr,
-					     (void *)&priv->xstats,
-					     rx_queues_count, tx_queues_count);
+		if (priv->synopsys_id >= DWMAC_CORE_3_50)
+			stmmac_mac_debug(priv, priv->ioaddr,
+					(void *)&priv->xstats,
+					rx_queues_count, tx_queues_count);
 	}
 	for (i = 0; i < STMMAC_STATS_LEN; i++) {
 		char *p = (char *)priv + stmmac_gstrings_stats[i].stat_offset;
@@ -581,8 +565,6 @@ static void stmmac_get_ethtool_stats(struct net_device *dev,
 static int stmmac_get_sset_count(struct net_device *netdev, int sset)
 {
 	struct stmmac_priv *priv = netdev_priv(netdev);
-	const char *(*dump)(struct stmmac_safety_stats *stats, int index,
-			unsigned long *count);
 	int i, len, safety_len = 0;
 
 	switch (sset) {
@@ -591,11 +573,11 @@ static int stmmac_get_sset_count(struct net_device *netdev, int sset)
 
 		if (priv->dma_cap.rmon)
 			len += STMMAC_MMC_STATS_LEN;
-		if (priv->dma_cap.asp && priv->hw->mac->safety_feat_dump) {
-			dump = priv->hw->mac->safety_feat_dump;
-
+		if (priv->dma_cap.asp) {
 			for (i = 0; i < STMMAC_SAFETY_FEAT_SIZE; i++) {
-				if (dump(&priv->sstats, i, NULL))
+				if (!stmmac_safety_feat_dump(priv,
+							&priv->sstats, i,
+							NULL, NULL))
 					safety_len++;
 			}
 
@@ -613,17 +595,15 @@ static void stmmac_get_strings(struct net_device *dev, u32 stringset, u8 *data)
 	int i;
 	u8 *p = data;
 	struct stmmac_priv *priv = netdev_priv(dev);
-	const char *(*dump)(struct stmmac_safety_stats *stats, int index,
-			unsigned long *count);
 
 	switch (stringset) {
 	case ETH_SS_STATS:
-		if (priv->dma_cap.asp && priv->hw->mac->safety_feat_dump) {
-			dump = priv->hw->mac->safety_feat_dump;
+		if (priv->dma_cap.asp) {
 			for (i = 0; i < STMMAC_SAFETY_FEAT_SIZE; i++) {
-				const char *desc = dump(&priv->sstats, i, NULL);
-
-				if (desc) {
+				const char *desc;
+				if (!stmmac_safety_feat_dump(priv,
+							&priv->sstats, i,
+							NULL, &desc)) {
 					memcpy(p, desc, ETH_GSTRING_LEN);
 					p += ETH_GSTRING_LEN;
 				}
@@ -652,12 +632,12 @@ static void stmmac_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
 
-	spin_lock_irq(&priv->lock);
+	mutex_lock(&priv->lock);
 	if (device_can_wakeup(priv->device)) {
 		wol->supported = WAKE_MAGIC | WAKE_UCAST;
 		wol->wolopts = priv->wolopts;
 	}
-	spin_unlock_irq(&priv->lock);
+	mutex_unlock(&priv->lock);
 }
 
 static int stmmac_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
@@ -686,9 +666,9 @@ static int stmmac_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
 		disable_irq_wake(priv->wol_irq);
 	}
 
-	spin_lock_irq(&priv->lock);
+	mutex_lock(&priv->lock);
 	priv->wolopts = wol->wolopts;
-	spin_unlock_irq(&priv->lock);
+	mutex_unlock(&priv->lock);
 
 	return 0;
 }
@@ -810,7 +790,7 @@ static int stmmac_set_coalesce(struct net_device *dev,
 	priv->tx_coal_frames = ec->tx_max_coalesced_frames;
 	priv->tx_coal_timer = ec->tx_coalesce_usecs;
 	priv->rx_riwt = rx_riwt;
-	priv->hw->dma->rx_watchdog(priv->ioaddr, priv->rx_riwt, rx_cnt);
+	stmmac_rx_watchdog(priv, priv->ioaddr, priv->rx_riwt, rx_cnt);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c
index 08c19ebd5306..8d9cc2157afd 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c
@@ -24,13 +24,13 @@
 #include "common.h"
 #include "stmmac_ptp.h"
 
-static void stmmac_config_hw_tstamping(void __iomem *ioaddr, u32 data)
+static void config_hw_tstamping(void __iomem *ioaddr, u32 data)
 {
 	writel(data, ioaddr + PTP_TCR);
 }
 
-static u32 stmmac_config_sub_second_increment(void __iomem *ioaddr,
-					      u32 ptp_clock, int gmac4)
+static void config_sub_second_increment(void __iomem *ioaddr,
+		u32 ptp_clock, int gmac4, u32 *ssinc)
 {
 	u32 value = readl(ioaddr + PTP_TCR);
 	unsigned long data;
@@ -57,10 +57,11 @@ static u32 stmmac_config_sub_second_increment(void __iomem *ioaddr,
 
 	writel(reg_value, ioaddr + PTP_SSIR);
 
-	return data;
+	if (ssinc)
+		*ssinc = data;
 }
 
-static int stmmac_init_systime(void __iomem *ioaddr, u32 sec, u32 nsec)
+static int init_systime(void __iomem *ioaddr, u32 sec, u32 nsec)
 {
 	int limit;
 	u32 value;
@@ -85,7 +86,7 @@ static int stmmac_init_systime(void __iomem *ioaddr, u32 sec, u32 nsec)
 	return 0;
 }
 
-static int stmmac_config_addend(void __iomem *ioaddr, u32 addend)
+static int config_addend(void __iomem *ioaddr, u32 addend)
 {
 	u32 value;
 	int limit;
@@ -109,8 +110,8 @@ static int stmmac_config_addend(void __iomem *ioaddr, u32 addend)
 	return 0;
 }
 
-static int stmmac_adjust_systime(void __iomem *ioaddr, u32 sec, u32 nsec,
-				 int add_sub, int gmac4)
+static int adjust_systime(void __iomem *ioaddr, u32 sec, u32 nsec,
+		int add_sub, int gmac4)
 {
 	u32 value;
 	int limit;
@@ -152,7 +153,7 @@ static int stmmac_adjust_systime(void __iomem *ioaddr, u32 sec, u32 nsec,
 	return 0;
 }
 
-static u64 stmmac_get_systime(void __iomem *ioaddr)
+static void get_systime(void __iomem *ioaddr, u64 *systime)
 {
 	u64 ns;
 
@@ -161,14 +162,15 @@ static u64 stmmac_get_systime(void __iomem *ioaddr)
 	/* Get the TSS and convert sec time value to nanosecond */
 	ns += readl(ioaddr + PTP_STSR) * 1000000000ULL;
 
-	return ns;
+	if (systime)
+		*systime = ns;
 }
 
 const struct stmmac_hwtimestamp stmmac_ptp = {
-	.config_hw_tstamping = stmmac_config_hw_tstamping,
-	.init_systime = stmmac_init_systime,
-	.config_sub_second_increment = stmmac_config_sub_second_increment,
-	.config_addend = stmmac_config_addend,
-	.adjust_systime = stmmac_adjust_systime,
-	.get_systime = stmmac_get_systime,
+	.config_hw_tstamping = config_hw_tstamping,
+	.init_systime = init_systime,
+	.config_sub_second_increment = config_sub_second_increment,
+	.config_addend = config_addend,
+	.adjust_systime = adjust_systime,
+	.get_systime = get_systime,
 };
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index b65e2d144698..77af85c981db 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -45,11 +45,13 @@
 #include <linux/seq_file.h>
 #endif /* CONFIG_DEBUG_FS */
 #include <linux/net_tstamp.h>
+#include <net/pkt_cls.h>
 #include "stmmac_ptp.h"
 #include "stmmac.h"
 #include <linux/reset.h>
 #include <linux/of_mdio.h>
 #include "dwmac1000.h"
+#include "hwif.h"
 
 #define STMMAC_ALIGN(x)	L1_CACHE_ALIGN(x)
 #define	TSO_MAX_BUFF_SIZE	(SZ_16K - 1)
@@ -335,8 +337,8 @@ static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
 
 	/* Check and enter in LPI mode */
 	if (!priv->tx_path_in_lpi_mode)
-		priv->hw->mac->set_eee_mode(priv->hw,
-					    priv->plat->en_tx_lpi_clockgating);
+		stmmac_set_eee_mode(priv, priv->hw,
+				priv->plat->en_tx_lpi_clockgating);
 }
 
 /**
@@ -347,7 +349,7 @@ static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
  */
 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
 {
-	priv->hw->mac->reset_eee_mode(priv->hw);
+	stmmac_reset_eee_mode(priv, priv->hw);
 	del_timer_sync(&priv->eee_ctrl_timer);
 	priv->tx_path_in_lpi_mode = false;
 }
@@ -379,7 +381,6 @@ bool stmmac_eee_init(struct stmmac_priv *priv)
 {
 	struct net_device *ndev = priv->dev;
 	int interface = priv->plat->interface;
-	unsigned long flags;
 	bool ret = false;
 
 	if ((interface != PHY_INTERFACE_MODE_MII) &&
@@ -406,19 +407,19 @@ bool stmmac_eee_init(struct stmmac_priv *priv)
 			 * changed).
 			 * In that case the driver disable own timers.
 			 */
-			spin_lock_irqsave(&priv->lock, flags);
+			mutex_lock(&priv->lock);
 			if (priv->eee_active) {
 				netdev_dbg(priv->dev, "disable EEE\n");
 				del_timer_sync(&priv->eee_ctrl_timer);
-				priv->hw->mac->set_eee_timer(priv->hw, 0,
-							     tx_lpi_timer);
+				stmmac_set_eee_timer(priv, priv->hw, 0,
+						tx_lpi_timer);
 			}
 			priv->eee_active = 0;
-			spin_unlock_irqrestore(&priv->lock, flags);
+			mutex_unlock(&priv->lock);
 			goto out;
 		}
 		/* Activate the EEE and start timers */
-		spin_lock_irqsave(&priv->lock, flags);
+		mutex_lock(&priv->lock);
 		if (!priv->eee_active) {
 			priv->eee_active = 1;
 			timer_setup(&priv->eee_ctrl_timer,
@@ -426,15 +427,14 @@ bool stmmac_eee_init(struct stmmac_priv *priv)
 			mod_timer(&priv->eee_ctrl_timer,
 				  STMMAC_LPI_T(eee_timer));
 
-			priv->hw->mac->set_eee_timer(priv->hw,
-						     STMMAC_DEFAULT_LIT_LS,
-						     tx_lpi_timer);
+			stmmac_set_eee_timer(priv, priv->hw,
+					STMMAC_DEFAULT_LIT_LS, tx_lpi_timer);
 		}
 		/* Set HW EEE according to the speed */
-		priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link);
+		stmmac_set_eee_pls(priv, priv->hw, ndev->phydev->link);
 
 		ret = true;
-		spin_unlock_irqrestore(&priv->lock, flags);
+		mutex_unlock(&priv->lock);
 
 		netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
 	}
@@ -464,9 +464,9 @@ static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
 		return;
 
 	/* check tx tstamp status */
-	if (priv->hw->desc->get_tx_timestamp_status(p)) {
+	if (stmmac_get_tx_timestamp_status(priv, p)) {
 		/* get the valid tstamp */
-		ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
+		stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
 
 		memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 		shhwtstamp.hwtstamp = ns_to_ktime(ns);
@@ -502,8 +502,8 @@ static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
 		desc = np;
 
 	/* Check if timestamp is available */
-	if (priv->hw->desc->get_rx_timestamp_status(p, np, priv->adv_ts)) {
-		ns = priv->hw->desc->get_timestamp(desc, priv->adv_ts);
+	if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
+		stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
 		netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
 		shhwtstamp = skb_hwtstamps(skb);
 		memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
@@ -707,18 +707,18 @@ static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
 	priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
 
 	if (!priv->hwts_tx_en && !priv->hwts_rx_en)
-		priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0);
+		stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0);
 	else {
 		value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
 			 tstamp_all | ptp_v2 | ptp_over_ethernet |
 			 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
 			 ts_master_en | snap_type_sel);
-		priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value);
+		stmmac_config_hw_tstamping(priv, priv->ptpaddr, value);
 
 		/* program Sub Second Increment reg */
-		sec_inc = priv->hw->ptp->config_sub_second_increment(
-			priv->ptpaddr, priv->plat->clk_ptp_rate,
-			priv->plat->has_gmac4);
+		stmmac_config_sub_second_increment(priv,
+				priv->ptpaddr, priv->plat->clk_ptp_rate,
+				priv->plat->has_gmac4, &sec_inc);
 		temp = div_u64(1000000000ULL, sec_inc);
 
 		/* calculate default added value:
@@ -728,15 +728,14 @@ static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
 		 */
 		temp = (u64)(temp << 32);
 		priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
-		priv->hw->ptp->config_addend(priv->ptpaddr,
-					     priv->default_addend);
+		stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
 
 		/* initialize system time */
 		ktime_get_real_ts64(&now);
 
 		/* lower 32 bits of tv_sec are safe until y2106 */
-		priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec,
-					    now.tv_nsec);
+		stmmac_init_systime(priv, priv->ptpaddr,
+				(u32)now.tv_sec, now.tv_nsec);
 	}
 
 	return copy_to_user(ifr->ifr_data, &config,
@@ -770,7 +769,6 @@ static int stmmac_init_ptp(struct stmmac_priv *priv)
 		netdev_info(priv->dev,
 			    "IEEE 1588-2008 Advanced Timestamp supported\n");
 
-	priv->hw->ptp = &stmmac_ptp;
 	priv->hwts_tx_en = 0;
 	priv->hwts_rx_en = 0;
 
@@ -795,8 +793,8 @@ static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
 {
 	u32 tx_cnt = priv->plat->tx_queues_to_use;
 
-	priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl,
-				 priv->pause, tx_cnt);
+	stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,
+			priv->pause, tx_cnt);
 }
 
 /**
@@ -812,13 +810,12 @@ static void stmmac_adjust_link(struct net_device *dev)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
 	struct phy_device *phydev = dev->phydev;
-	unsigned long flags;
 	bool new_state = false;
 
 	if (!phydev)
 		return;
 
-	spin_lock_irqsave(&priv->lock, flags);
+	mutex_lock(&priv->lock);
 
 	if (phydev->link) {
 		u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
@@ -877,7 +874,7 @@ static void stmmac_adjust_link(struct net_device *dev)
 	if (new_state && netif_msg_link(priv))
 		phy_print_status(phydev);
 
-	spin_unlock_irqrestore(&priv->lock, flags);
+	mutex_unlock(&priv->lock);
 
 	if (phydev->is_pseudo_fixed_link)
 		/* Stop PHY layer to call the hook to adjust the link in case
@@ -1008,7 +1005,7 @@ static void stmmac_display_rx_rings(struct stmmac_priv *priv)
 			head_rx = (void *)rx_q->dma_rx;
 
 		/* Display RX ring */
-		priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true);
+		stmmac_display_ring(priv, head_rx, DMA_RX_SIZE, true);
 	}
 }
 
@@ -1029,7 +1026,7 @@ static void stmmac_display_tx_rings(struct stmmac_priv *priv)
 		else
 			head_tx = (void *)tx_q->dma_tx;
 
-		priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false);
+		stmmac_display_ring(priv, head_tx, DMA_TX_SIZE, false);
 	}
 }
 
@@ -1073,13 +1070,13 @@ static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
 	/* Clear the RX descriptors */
 	for (i = 0; i < DMA_RX_SIZE; i++)
 		if (priv->extend_desc)
-			priv->hw->desc->init_rx_desc(&rx_q->dma_erx[i].basic,
-						     priv->use_riwt, priv->mode,
-						     (i == DMA_RX_SIZE - 1));
+			stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
+					priv->use_riwt, priv->mode,
+					(i == DMA_RX_SIZE - 1));
 		else
-			priv->hw->desc->init_rx_desc(&rx_q->dma_rx[i],
-						     priv->use_riwt, priv->mode,
-						     (i == DMA_RX_SIZE - 1));
+			stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
+					priv->use_riwt, priv->mode,
+					(i == DMA_RX_SIZE - 1));
 }
 
 /**
@@ -1097,13 +1094,11 @@ static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
 	/* Clear the TX descriptors */
 	for (i = 0; i < DMA_TX_SIZE; i++)
 		if (priv->extend_desc)
-			priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
-						     priv->mode,
-						     (i == DMA_TX_SIZE - 1));
+			stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic,
+					priv->mode, (i == DMA_TX_SIZE - 1));
 		else
-			priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
-						     priv->mode,
-						     (i == DMA_TX_SIZE - 1));
+			stmmac_init_tx_desc(priv, &tx_q->dma_tx[i],
+					priv->mode, (i == DMA_TX_SIZE - 1));
 }
 
 /**
@@ -1159,14 +1154,10 @@ static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
 		return -EINVAL;
 	}
 
-	if (priv->synopsys_id >= DWMAC_CORE_4_00)
-		p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
-	else
-		p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
+	stmmac_set_desc_addr(priv, p, rx_q->rx_skbuff_dma[i]);
 
-	if ((priv->hw->mode->init_desc3) &&
-	    (priv->dma_buf_sz == BUF_SIZE_16KiB))
-		priv->hw->mode->init_desc3(p);
+	if (priv->dma_buf_sz == BUF_SIZE_16KiB)
+		stmmac_init_desc3(priv, p);
 
 	return 0;
 }
@@ -1232,13 +1223,14 @@ static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
 	u32 rx_count = priv->plat->rx_queues_to_use;
-	unsigned int bfsize = 0;
 	int ret = -ENOMEM;
+	int bfsize = 0;
 	int queue;
 	int i;
 
-	if (priv->hw->mode->set_16kib_bfsize)
-		bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu);
+	bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu);
+	if (bfsize < 0)
+		bfsize = 0;
 
 	if (bfsize < BUF_SIZE_16KiB)
 		bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
@@ -1282,13 +1274,11 @@ static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
 		/* Setup the chained descriptor addresses */
 		if (priv->mode == STMMAC_CHAIN_MODE) {
 			if (priv->extend_desc)
-				priv->hw->mode->init(rx_q->dma_erx,
-						     rx_q->dma_rx_phy,
-						     DMA_RX_SIZE, 1);
+				stmmac_mode_init(priv, rx_q->dma_erx,
+						rx_q->dma_rx_phy, DMA_RX_SIZE, 1);
 			else
-				priv->hw->mode->init(rx_q->dma_rx,
-						     rx_q->dma_rx_phy,
-						     DMA_RX_SIZE, 0);
+				stmmac_mode_init(priv, rx_q->dma_rx,
+						rx_q->dma_rx_phy, DMA_RX_SIZE, 0);
 		}
 	}
 
@@ -1335,13 +1325,11 @@ static int init_dma_tx_desc_rings(struct net_device *dev)
 		/* Setup the chained descriptor addresses */
 		if (priv->mode == STMMAC_CHAIN_MODE) {
 			if (priv->extend_desc)
-				priv->hw->mode->init(tx_q->dma_etx,
-						     tx_q->dma_tx_phy,
-						     DMA_TX_SIZE, 1);
+				stmmac_mode_init(priv, tx_q->dma_etx,
+						tx_q->dma_tx_phy, DMA_TX_SIZE, 1);
 			else
-				priv->hw->mode->init(tx_q->dma_tx,
-						     tx_q->dma_tx_phy,
-						     DMA_TX_SIZE, 0);
+				stmmac_mode_init(priv, tx_q->dma_tx,
+						tx_q->dma_tx_phy, DMA_TX_SIZE, 0);
 		}
 
 		for (i = 0; i < DMA_TX_SIZE; i++) {
@@ -1351,14 +1339,7 @@ static int init_dma_tx_desc_rings(struct net_device *dev)
 			else
 				p = tx_q->dma_tx + i;
 
-			if (priv->synopsys_id >= DWMAC_CORE_4_00) {
-				p->des0 = 0;
-				p->des1 = 0;
-				p->des2 = 0;
-				p->des3 = 0;
-			} else {
-				p->des2 = 0;
-			}
+			stmmac_clear_desc(priv, p);
 
 			tx_q->tx_skbuff_dma[i].buf = 0;
 			tx_q->tx_skbuff_dma[i].map_as_page = false;
@@ -1664,7 +1645,7 @@ static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
 
 	for (queue = 0; queue < rx_queues_count; queue++) {
 		mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
-		priv->hw->mac->rx_queue_enable(priv->hw, mode, queue);
+		stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
 	}
 }
 
@@ -1678,7 +1659,7 @@ static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
 {
 	netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
-	priv->hw->dma->start_rx(priv->ioaddr, chan);
+	stmmac_start_rx(priv, priv->ioaddr, chan);
 }
 
 /**
@@ -1691,7 +1672,7 @@ static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
 {
 	netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
-	priv->hw->dma->start_tx(priv->ioaddr, chan);
+	stmmac_start_tx(priv, priv->ioaddr, chan);
 }
 
 /**
@@ -1704,7 +1685,7 @@ static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
 {
 	netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
-	priv->hw->dma->stop_rx(priv->ioaddr, chan);
+	stmmac_stop_rx(priv, priv->ioaddr, chan);
 }
 
 /**
@@ -1717,7 +1698,7 @@ static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
 {
 	netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
-	priv->hw->dma->stop_tx(priv->ioaddr, chan);
+	stmmac_stop_tx(priv, priv->ioaddr, chan);
 }
 
 /**
@@ -1804,23 +1785,18 @@ static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
 	}
 
 	/* configure all channels */
-	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
-		for (chan = 0; chan < rx_channels_count; chan++) {
-			qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
+	for (chan = 0; chan < rx_channels_count; chan++) {
+		qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
 
-			priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
-						   rxfifosz, qmode);
-		}
+		stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
+				rxfifosz, qmode);
+	}
 
-		for (chan = 0; chan < tx_channels_count; chan++) {
-			qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
+	for (chan = 0; chan < tx_channels_count; chan++) {
+		qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
 
-			priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan,
-						   txfifosz, qmode);
-		}
-	} else {
-		priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
-					rxfifosz);
+		stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
+				txfifosz, qmode);
 	}
 }
 
@@ -1851,9 +1827,8 @@ static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
 		else
 			p = tx_q->dma_tx + entry;
 
-		status = priv->hw->desc->tx_status(&priv->dev->stats,
-						      &priv->xstats, p,
-						      priv->ioaddr);
+		status = stmmac_tx_status(priv, &priv->dev->stats,
+				&priv->xstats, p, priv->ioaddr);
 		/* Check if the descriptor is owned by the DMA */
 		if (unlikely(status & tx_dma_own))
 			break;
@@ -1891,8 +1866,7 @@ static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
 			tx_q->tx_skbuff_dma[entry].map_as_page = false;
 		}
 
-		if (priv->hw->mode->clean_desc3)
-			priv->hw->mode->clean_desc3(tx_q, p);
+		stmmac_clean_desc3(priv, tx_q, p);
 
 		tx_q->tx_skbuff_dma[entry].last_segment = false;
 		tx_q->tx_skbuff_dma[entry].is_jumbo = false;
@@ -1904,7 +1878,7 @@ static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
 			tx_q->tx_skbuff[entry] = NULL;
 		}
 
-		priv->hw->desc->release_tx_desc(p, priv->mode);
+		stmmac_release_tx_desc(priv, p, priv->mode);
 
 		entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
 	}
@@ -1929,16 +1903,6 @@ static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
 	netif_tx_unlock(priv->dev);
 }
 
-static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan)
-{
-	priv->hw->dma->enable_dma_irq(priv->ioaddr, chan);
-}
-
-static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan)
-{
-	priv->hw->dma->disable_dma_irq(priv->ioaddr, chan);
-}
-
 /**
  * stmmac_tx_err - to manage the tx error
  * @priv: driver private structure
@@ -1957,13 +1921,11 @@ static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
 	dma_free_tx_skbufs(priv, chan);
 	for (i = 0; i < DMA_TX_SIZE; i++)
 		if (priv->extend_desc)
-			priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
-						     priv->mode,
-						     (i == DMA_TX_SIZE - 1));
+			stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic,
+					priv->mode, (i == DMA_TX_SIZE - 1));
 		else
-			priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
-						     priv->mode,
-						     (i == DMA_TX_SIZE - 1));
+			stmmac_init_tx_desc(priv, &tx_q->dma_tx[i],
+					priv->mode, (i == DMA_TX_SIZE - 1));
 	tx_q->dirty_tx = 0;
 	tx_q->cur_tx = 0;
 	tx_q->mss = 0;
@@ -2003,31 +1965,22 @@ static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
 	rxfifosz /= rx_channels_count;
 	txfifosz /= tx_channels_count;
 
-	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
-		priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
-					   rxfifosz, rxqmode);
-		priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan,
-					   txfifosz, txqmode);
-	} else {
-		priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
-					rxfifosz);
-	}
+	stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
+	stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
 }
 
 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
 {
-	bool ret = false;
-
-	/* Safety features are only available in cores >= 5.10 */
-	if (priv->synopsys_id < DWMAC_CORE_5_10)
-		return ret;
-	if (priv->hw->mac->safety_feat_irq_status)
-		ret = priv->hw->mac->safety_feat_irq_status(priv->dev,
-				priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
+	int ret;
 
-	if (ret)
+	ret = stmmac_safety_feat_irq_status(priv, priv->dev,
+			priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
+	if (ret && (ret != -EINVAL)) {
 		stmmac_global_err(priv);
-	return ret;
+		return true;
+	}
+
+	return false;
 }
 
 /**
@@ -2045,7 +1998,11 @@ static void stmmac_dma_interrupt(struct stmmac_priv *priv)
 				tx_channel_count : rx_channel_count;
 	u32 chan;
 	bool poll_scheduled = false;
-	int status[channels_to_check];
+	int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
+
+	/* Make sure we never check beyond our status buffer. */
+	if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
+		channels_to_check = ARRAY_SIZE(status);
 
 	/* Each DMA channel can be used for rx and tx simultaneously, yet
 	 * napi_struct is embedded in struct stmmac_rx_queue rather than in a
@@ -2054,16 +2011,15 @@ static void stmmac_dma_interrupt(struct stmmac_priv *priv)
 	 * all tx queues rather than just a single tx queue.
 	 */
 	for (chan = 0; chan < channels_to_check; chan++)
-		status[chan] = priv->hw->dma->dma_interrupt(priv->ioaddr,
-							    &priv->xstats,
-							    chan);
+		status[chan] = stmmac_dma_interrupt_status(priv, priv->ioaddr,
+				&priv->xstats, chan);
 
 	for (chan = 0; chan < rx_channel_count; chan++) {
 		if (likely(status[chan] & handle_rx)) {
 			struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
 
 			if (likely(napi_schedule_prep(&rx_q->napi))) {
-				stmmac_disable_dma_irq(priv, chan);
+				stmmac_disable_dma_irq(priv, priv->ioaddr, chan);
 				__napi_schedule(&rx_q->napi);
 				poll_scheduled = true;
 			}
@@ -2084,7 +2040,8 @@ static void stmmac_dma_interrupt(struct stmmac_priv *priv)
 					&priv->rx_queue[0];
 
 				if (likely(napi_schedule_prep(&rx_q->napi))) {
-					stmmac_disable_dma_irq(priv, chan);
+					stmmac_disable_dma_irq(priv,
+							priv->ioaddr, chan);
 					__napi_schedule(&rx_q->napi);
 				}
 				break;
@@ -2126,14 +2083,6 @@ static void stmmac_mmc_setup(struct stmmac_priv *priv)
 	unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
 			    MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
 
-	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
-		priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET;
-		priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET;
-	} else {
-		priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET;
-		priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET;
-	}
-
 	dwmac_mmc_intr_all_mask(priv->mmcaddr);
 
 	if (priv->dma_cap.rmon) {
@@ -2143,32 +2092,6 @@ static void stmmac_mmc_setup(struct stmmac_priv *priv)
 		netdev_info(priv->dev, "No MAC Management Counters available\n");
 }
 
-/**
- * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors
- * @priv: driver private structure
- * Description: select the Enhanced/Alternate or Normal descriptors.
- * In case of Enhanced/Alternate, it checks if the extended descriptors are
- * supported by the HW capability register.
- */
-static void stmmac_selec_desc_mode(struct stmmac_priv *priv)
-{
-	if (priv->plat->enh_desc) {
-		dev_info(priv->device, "Enhanced/Alternate descriptors\n");
-
-		/* GMAC older than 3.50 has no extended descriptors */
-		if (priv->synopsys_id >= DWMAC_CORE_3_50) {
-			dev_info(priv->device, "Enabled extended descriptors\n");
-			priv->extend_desc = 1;
-		} else
-			dev_warn(priv->device, "Extended descriptors not supported\n");
-
-		priv->hw->desc = &enh_desc_ops;
-	} else {
-		dev_info(priv->device, "Normal descriptors\n");
-		priv->hw->desc = &ndesc_ops;
-	}
-}
-
 /**
  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
  * @priv: driver private structure
@@ -2180,15 +2103,7 @@ static void stmmac_selec_desc_mode(struct stmmac_priv *priv)
  */
 static int stmmac_get_hw_features(struct stmmac_priv *priv)
 {
-	u32 ret = 0;
-
-	if (priv->hw->dma->get_hw_feature) {
-		priv->hw->dma->get_hw_feature(priv->ioaddr,
-					      &priv->dma_cap);
-		ret = 1;
-	}
-
-	return ret;
+	return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
 }
 
 /**
@@ -2201,8 +2116,7 @@ static int stmmac_get_hw_features(struct stmmac_priv *priv)
 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
 {
 	if (!is_valid_ether_addr(priv->dev->dev_addr)) {
-		priv->hw->mac->get_umac_addr(priv->hw,
-					     priv->dev->dev_addr, 0);
+		stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0);
 		if (!is_valid_ether_addr(priv->dev->dev_addr))
 			eth_hw_addr_random(priv->dev);
 		netdev_info(priv->dev, "device MAC address %pM\n",
@@ -2222,10 +2136,9 @@ static int stmmac_init_dma_engine(struct stmmac_priv *priv)
 {
 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
+	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
 	struct stmmac_rx_queue *rx_q;
 	struct stmmac_tx_queue *tx_q;
-	u32 dummy_dma_rx_phy = 0;
-	u32 dummy_dma_tx_phy = 0;
 	u32 chan = 0;
 	int atds = 0;
 	int ret = 0;
@@ -2238,59 +2151,47 @@ static int stmmac_init_dma_engine(struct stmmac_priv *priv)
 	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
 		atds = 1;
 
-	ret = priv->hw->dma->reset(priv->ioaddr);
+	ret = stmmac_reset(priv, priv->ioaddr);
 	if (ret) {
 		dev_err(priv->device, "Failed to reset the dma\n");
 		return ret;
 	}
 
-	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
-		/* DMA Configuration */
-		priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
-				    dummy_dma_tx_phy, dummy_dma_rx_phy, atds);
+	/* DMA RX Channel Configuration */
+	for (chan = 0; chan < rx_channels_count; chan++) {
+		rx_q = &priv->rx_queue[chan];
 
-		/* DMA RX Channel Configuration */
-		for (chan = 0; chan < rx_channels_count; chan++) {
-			rx_q = &priv->rx_queue[chan];
+		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
+				    rx_q->dma_rx_phy, chan);
 
-			priv->hw->dma->init_rx_chan(priv->ioaddr,
-						    priv->plat->dma_cfg,
-						    rx_q->dma_rx_phy, chan);
+		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
+			    (DMA_RX_SIZE * sizeof(struct dma_desc));
+		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
+				       rx_q->rx_tail_addr, chan);
+	}
 
-			rx_q->rx_tail_addr = rx_q->dma_rx_phy +
-				    (DMA_RX_SIZE * sizeof(struct dma_desc));
-			priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
-						       rx_q->rx_tail_addr,
-						       chan);
-		}
+	/* DMA TX Channel Configuration */
+	for (chan = 0; chan < tx_channels_count; chan++) {
+		tx_q = &priv->tx_queue[chan];
 
-		/* DMA TX Channel Configuration */
-		for (chan = 0; chan < tx_channels_count; chan++) {
-			tx_q = &priv->tx_queue[chan];
+		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
+				    tx_q->dma_tx_phy, chan);
 
-			priv->hw->dma->init_chan(priv->ioaddr,
-						 priv->plat->dma_cfg,
-						 chan);
+		tx_q->tx_tail_addr = tx_q->dma_tx_phy +
+			    (DMA_TX_SIZE * sizeof(struct dma_desc));
+		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
+				       tx_q->tx_tail_addr, chan);
+	}
 
-			priv->hw->dma->init_tx_chan(priv->ioaddr,
-						    priv->plat->dma_cfg,
-						    tx_q->dma_tx_phy, chan);
+	/* DMA CSR Channel configuration */
+	for (chan = 0; chan < dma_csr_ch; chan++)
+		stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
 
-			tx_q->tx_tail_addr = tx_q->dma_tx_phy +
-				    (DMA_TX_SIZE * sizeof(struct dma_desc));
-			priv->hw->dma->set_tx_tail_ptr(priv->ioaddr,
-						       tx_q->tx_tail_addr,
-						       chan);
-		}
-	} else {
-		rx_q = &priv->rx_queue[chan];
-		tx_q = &priv->tx_queue[chan];
-		priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
-				    tx_q->dma_tx_phy, rx_q->dma_rx_phy, atds);
-	}
+	/* DMA Configuration */
+	stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds);
 
-	if (priv->plat->axi && priv->hw->dma->axi)
-		priv->hw->dma->axi(priv->ioaddr, priv->plat->axi);
+	if (priv->plat->axi)
+		stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
 
 	return ret;
 }
@@ -2336,18 +2237,14 @@ static void stmmac_set_rings_length(struct stmmac_priv *priv)
 	u32 chan;
 
 	/* set TX ring length */
-	if (priv->hw->dma->set_tx_ring_len) {
-		for (chan = 0; chan < tx_channels_count; chan++)
-			priv->hw->dma->set_tx_ring_len(priv->ioaddr,
-						       (DMA_TX_SIZE - 1), chan);
-	}
+	for (chan = 0; chan < tx_channels_count; chan++)
+		stmmac_set_tx_ring_len(priv, priv->ioaddr,
+				(DMA_TX_SIZE - 1), chan);
 
 	/* set RX ring length */
-	if (priv->hw->dma->set_rx_ring_len) {
-		for (chan = 0; chan < rx_channels_count; chan++)
-			priv->hw->dma->set_rx_ring_len(priv->ioaddr,
-						       (DMA_RX_SIZE - 1), chan);
-	}
+	for (chan = 0; chan < rx_channels_count; chan++)
+		stmmac_set_rx_ring_len(priv, priv->ioaddr,
+				(DMA_RX_SIZE - 1), chan);
 }
 
 /**
@@ -2363,7 +2260,7 @@ static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
 
 	for (queue = 0; queue < tx_queues_count; queue++) {
 		weight = priv->plat->tx_queues_cfg[queue].weight;
-		priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue);
+		stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
 	}
 }
 
@@ -2384,7 +2281,7 @@ static void stmmac_configure_cbs(struct stmmac_priv *priv)
 		if (mode_to_use == MTL_QUEUE_DCB)
 			continue;
 
-		priv->hw->mac->config_cbs(priv->hw,
+		stmmac_config_cbs(priv, priv->hw,
 				priv->plat->tx_queues_cfg[queue].send_slope,
 				priv->plat->tx_queues_cfg[queue].idle_slope,
 				priv->plat->tx_queues_cfg[queue].high_credit,
@@ -2406,7 +2303,7 @@ static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
 
 	for (queue = 0; queue < rx_queues_count; queue++) {
 		chan = priv->plat->rx_queues_cfg[queue].chan;
-		priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan);
+		stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
 	}
 }
 
@@ -2426,7 +2323,7 @@ static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
 			continue;
 
 		prio = priv->plat->rx_queues_cfg[queue].prio;
-		priv->hw->mac->rx_queue_prio(priv->hw, prio, queue);
+		stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
 	}
 }
 
@@ -2446,7 +2343,7 @@ static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
 			continue;
 
 		prio = priv->plat->tx_queues_cfg[queue].prio;
-		priv->hw->mac->tx_queue_prio(priv->hw, prio, queue);
+		stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
 	}
 }
 
@@ -2467,7 +2364,7 @@ static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
 			continue;
 
 		packet = priv->plat->rx_queues_cfg[queue].pkt_route;
-		priv->hw->mac->rx_queue_routing(priv->hw, packet, queue);
+		stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
 	}
 }
 
@@ -2481,50 +2378,47 @@ static void stmmac_mtl_configuration(struct stmmac_priv *priv)
 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
 
-	if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight)
+	if (tx_queues_count > 1)
 		stmmac_set_tx_queue_weight(priv);
 
 	/* Configure MTL RX algorithms */
-	if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms)
-		priv->hw->mac->prog_mtl_rx_algorithms(priv->hw,
-						priv->plat->rx_sched_algorithm);
+	if (rx_queues_count > 1)
+		stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
+				priv->plat->rx_sched_algorithm);
 
 	/* Configure MTL TX algorithms */
-	if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms)
-		priv->hw->mac->prog_mtl_tx_algorithms(priv->hw,
-						priv->plat->tx_sched_algorithm);
+	if (tx_queues_count > 1)
+		stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
+				priv->plat->tx_sched_algorithm);
 
 	/* Configure CBS in AVB TX queues */
-	if (tx_queues_count > 1 && priv->hw->mac->config_cbs)
+	if (tx_queues_count > 1)
 		stmmac_configure_cbs(priv);
 
 	/* Map RX MTL to DMA channels */
-	if (priv->hw->mac->map_mtl_to_dma)
-		stmmac_rx_queue_dma_chan_map(priv);
+	stmmac_rx_queue_dma_chan_map(priv);
 
 	/* Enable MAC RX Queues */
-	if (priv->hw->mac->rx_queue_enable)
-		stmmac_mac_enable_rx_queues(priv);
+	stmmac_mac_enable_rx_queues(priv);
 
 	/* Set RX priorities */
-	if (rx_queues_count > 1 && priv->hw->mac->rx_queue_prio)
+	if (rx_queues_count > 1)
 		stmmac_mac_config_rx_queues_prio(priv);
 
 	/* Set TX priorities */
-	if (tx_queues_count > 1 && priv->hw->mac->tx_queue_prio)
+	if (tx_queues_count > 1)
 		stmmac_mac_config_tx_queues_prio(priv);
 
 	/* Set RX routing */
-	if (rx_queues_count > 1 && priv->hw->mac->rx_queue_routing)
+	if (rx_queues_count > 1)
 		stmmac_mac_config_rx_queues_routing(priv);
 }
 
 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
 {
-	if (priv->hw->mac->safety_feat_config && priv->dma_cap.asp) {
+	if (priv->dma_cap.asp) {
 		netdev_info(priv->dev, "Enabling Safety Features\n");
-		priv->hw->mac->safety_feat_config(priv->ioaddr,
-				priv->dma_cap.asp);
+		stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp);
 	} else {
 		netdev_info(priv->dev, "No Safety Features support found\n");
 	}
@@ -2559,7 +2453,7 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
 	}
 
 	/* Copy the MAC addr into the HW  */
-	priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0);
+	stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
 
 	/* PS and related bits will be programmed according to the speed */
 	if (priv->hw->pcs) {
@@ -2575,17 +2469,15 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
 	}
 
 	/* Initialize the MAC Core */
-	priv->hw->mac->core_init(priv->hw, dev);
+	stmmac_core_init(priv, priv->hw, dev);
 
 	/* Initialize MTL*/
-	if (priv->synopsys_id >= DWMAC_CORE_4_00)
-		stmmac_mtl_configuration(priv);
+	stmmac_mtl_configuration(priv);
 
 	/* Initialize Safety Features */
-	if (priv->synopsys_id >= DWMAC_CORE_5_10)
-		stmmac_safety_feat_configuration(priv);
+	stmmac_safety_feat_configuration(priv);
 
-	ret = priv->hw->mac->rx_ipc(priv->hw);
+	ret = stmmac_rx_ipc(priv, priv->hw);
 	if (!ret) {
 		netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
 		priv->plat->rx_coe = STMMAC_RX_COE_NONE;
@@ -2593,7 +2485,7 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
 	}
 
 	/* Enable the MAC Rx/Tx */
-	priv->hw->mac->set_mac(priv->ioaddr, true);
+	stmmac_mac_set(priv, priv->ioaddr, true);
 
 	/* Set the HW DMA mode and the COE */
 	stmmac_dma_operation_mode(priv);
@@ -2623,13 +2515,14 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
 
 	priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
 
-	if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) {
-		priv->rx_riwt = MAX_DMA_RIWT;
-		priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt);
+	if (priv->use_riwt) {
+		ret = stmmac_rx_watchdog(priv, priv->ioaddr, MAX_DMA_RIWT, rx_cnt);
+		if (!ret)
+			priv->rx_riwt = MAX_DMA_RIWT;
 	}
 
-	if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane)
-		priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0);
+	if (priv->hw->pcs)
+		stmmac_pcs_ctrl_ane(priv, priv->hw, 1, priv->hw->ps, 0);
 
 	/* set TX and RX rings length */
 	stmmac_set_rings_length(priv);
@@ -2637,7 +2530,7 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
 	/* Enable TSO */
 	if (priv->tso) {
 		for (chan = 0; chan < tx_cnt; chan++)
-			priv->hw->dma->enable_tso(priv->ioaddr, 1, chan);
+			stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
 	}
 
 	return 0;
@@ -2808,7 +2701,7 @@ static int stmmac_release(struct net_device *dev)
 	free_dma_desc_resources(priv);
 
 	/* Disable the MAC Rx/Tx */
-	priv->hw->mac->set_mac(priv->ioaddr, false);
+	stmmac_mac_set(priv, priv->ioaddr, false);
 
 	netif_carrier_off(dev);
 
@@ -2851,10 +2744,10 @@ static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des,
 		buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
 			    TSO_MAX_BUFF_SIZE : tmp_len;
 
-		priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size,
-			0, 1,
-			(last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
-			0, 0);
+		stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
+				0, 1,
+				(last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
+				0, 0);
 
 		tmp_len -= TSO_MAX_BUFF_SIZE;
 	}
@@ -2926,7 +2819,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
 	/* set new MSS value if needed */
 	if (mss != tx_q->mss) {
 		mss_desc = tx_q->dma_tx + tx_q->cur_tx;
-		priv->hw->desc->set_mss(mss_desc, mss);
+		stmmac_set_mss(priv, mss_desc, mss);
 		tx_q->mss = mss;
 		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
 		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
@@ -3012,7 +2905,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
 			  STMMAC_COAL_TIMER(priv->tx_coal_timer));
 	} else {
 		priv->tx_count_frames = 0;
-		priv->hw->desc->set_tx_ic(desc);
+		stmmac_set_tx_ic(priv, desc);
 		priv->xstats.tx_set_ic_bit++;
 	}
 
@@ -3022,11 +2915,11 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
 		     priv->hwts_tx_en)) {
 		/* declare that device is doing timestamping */
 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
-		priv->hw->desc->enable_tx_timestamp(first);
+		stmmac_enable_tx_timestamp(priv, first);
 	}
 
 	/* Complete the first descriptor before granting the DMA */
-	priv->hw->desc->prepare_tso_tx_desc(first, 1,
+	stmmac_prepare_tso_tx_desc(priv, first, 1,
 			proto_hdr_len,
 			pay_len,
 			1, tx_q->tx_skbuff_dma[first_entry].last_segment,
@@ -3040,7 +2933,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
 		 * sure that MSS's own bit is the last thing written.
 		 */
 		dma_wmb();
-		priv->hw->desc->set_tx_owner(mss_desc);
+		stmmac_set_tx_owner(priv, mss_desc);
 	}
 
 	/* The own bit must be the latest setting done when prepare the
@@ -3054,8 +2947,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
 			__func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
 			tx_q->cur_tx, first, nfrags);
 
-		priv->hw->desc->display_ring((void *)tx_q->dma_tx, DMA_TX_SIZE,
-					     0);
+		stmmac_display_ring(priv, (void *)tx_q->dma_tx, DMA_TX_SIZE, 0);
 
 		pr_info(">>> frame to be transmitted: ");
 		print_pkt(skb->data, skb_headlen(skb));
@@ -3063,8 +2955,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
 
-	priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
-				       queue);
+	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
 
 	return NETDEV_TX_OK;
 
@@ -3136,12 +3027,11 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 	enh_desc = priv->plat->enh_desc;
 	/* To program the descriptors according to the size of the frame */
 	if (enh_desc)
-		is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc);
+		is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
 
-	if (unlikely(is_jumbo) && likely(priv->synopsys_id <
-					 DWMAC_CORE_4_00)) {
-		entry = priv->hw->mode->jumbo_frm(tx_q, skb, csum_insertion);
-		if (unlikely(entry < 0))
+	if (unlikely(is_jumbo)) {
+		entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
+		if (unlikely(entry < 0) && (entry != -EINVAL))
 			goto dma_map_err;
 	}
 
@@ -3164,19 +3054,16 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 			goto dma_map_err; /* should reuse desc w/o issues */
 
 		tx_q->tx_skbuff_dma[entry].buf = des;
-		if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
-			desc->des0 = cpu_to_le32(des);
-		else
-			desc->des2 = cpu_to_le32(des);
+
+		stmmac_set_desc_addr(priv, desc, des);
 
 		tx_q->tx_skbuff_dma[entry].map_as_page = true;
 		tx_q->tx_skbuff_dma[entry].len = len;
 		tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
 
 		/* Prepare the descriptor and set the own bit too */
-		priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion,
-						priv->mode, 1, last_segment,
-						skb->len);
+		stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
+				priv->mode, 1, last_segment, skb->len);
 	}
 
 	/* Only the last descriptor gets to point to the skb. */
@@ -3203,7 +3090,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 		else
 			tx_head = (void *)tx_q->dma_tx;
 
-		priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false);
+		stmmac_display_ring(priv, tx_head, DMA_TX_SIZE, false);
 
 		netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
 		print_pkt(skb->data, skb->len);
@@ -3223,13 +3110,16 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 	 * element in case of no SG.
 	 */
 	priv->tx_count_frames += nfrags + 1;
-	if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
+	if (likely(priv->tx_coal_frames > priv->tx_count_frames) &&
+	    !priv->tx_timer_armed) {
 		mod_timer(&priv->txtimer,
 			  STMMAC_COAL_TIMER(priv->tx_coal_timer));
+		priv->tx_timer_armed = true;
 	} else {
 		priv->tx_count_frames = 0;
-		priv->hw->desc->set_tx_ic(desc);
+		stmmac_set_tx_ic(priv, desc);
 		priv->xstats.tx_set_ic_bit++;
+		priv->tx_timer_armed = false;
 	}
 
 	skb_tx_timestamp(skb);
@@ -3247,10 +3137,8 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 			goto dma_map_err;
 
 		tx_q->tx_skbuff_dma[first_entry].buf = des;
-		if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
-			first->des0 = cpu_to_le32(des);
-		else
-			first->des2 = cpu_to_le32(des);
+
+		stmmac_set_desc_addr(priv, first, des);
 
 		tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
 		tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
@@ -3259,13 +3147,13 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 			     priv->hwts_tx_en)) {
 			/* declare that device is doing timestamping */
 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
-			priv->hw->desc->enable_tx_timestamp(first);
+			stmmac_enable_tx_timestamp(priv, first);
 		}
 
 		/* Prepare the first descriptor setting the OWN bit too */
-		priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len,
-						csum_insertion, priv->mode, 1,
-						last_segment, skb->len);
+		stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
+				csum_insertion, priv->mode, 1, last_segment,
+				skb->len);
 
 		/* The own bit must be the latest setting done when prepare the
 		 * descriptor and then barrier is needed to make sure that
@@ -3276,11 +3164,8 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
 
-	if (priv->synopsys_id < DWMAC_CORE_4_00)
-		priv->hw->dma->enable_dma_transmission(priv->ioaddr);
-	else
-		priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
-					       queue);
+	stmmac_enable_dma_transmission(priv, priv->ioaddr);
+	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
 
 	return NETDEV_TX_OK;
 
@@ -3364,14 +3249,8 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
 				break;
 			}
 
-			if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) {
-				p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
-				p->des1 = 0;
-			} else {
-				p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
-			}
-			if (priv->hw->mode->refill_desc3)
-				priv->hw->mode->refill_desc3(rx_q, p);
+			stmmac_set_desc_addr(priv, p, rx_q->rx_skbuff_dma[entry]);
+			stmmac_refill_desc3(priv, rx_q, p);
 
 			if (rx_q->rx_zeroc_thresh > 0)
 				rx_q->rx_zeroc_thresh--;
@@ -3381,10 +3260,7 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
 		}
 		dma_wmb();
 
-		if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
-			priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0);
-		else
-			priv->hw->desc->set_rx_owner(p);
+		stmmac_set_rx_owner(priv, p, priv->use_riwt);
 
 		dma_wmb();
 
@@ -3418,7 +3294,7 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 		else
 			rx_head = (void *)rx_q->dma_rx;
 
-		priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true);
+		stmmac_display_ring(priv, rx_head, DMA_RX_SIZE, true);
 	}
 	while (count < limit) {
 		int status;
@@ -3431,8 +3307,8 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 			p = rx_q->dma_rx + entry;
 
 		/* read the status of the incoming frame */
-		status = priv->hw->desc->rx_status(&priv->dev->stats,
-						   &priv->xstats, p);
+		status = stmmac_rx_status(priv, &priv->dev->stats,
+				&priv->xstats, p);
 		/* check if managed by the DMA otherwise go ahead */
 		if (unlikely(status & dma_own))
 			break;
@@ -3449,11 +3325,9 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 
 		prefetch(np);
 
-		if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status))
-			priv->hw->desc->rx_extended_status(&priv->dev->stats,
-							   &priv->xstats,
-							   rx_q->dma_erx +
-							   entry);
+		if (priv->extend_desc)
+			stmmac_rx_extended_status(priv, &priv->dev->stats,
+					&priv->xstats, rx_q->dma_erx + entry);
 		if (unlikely(status == discard_frame)) {
 			priv->dev->stats.rx_errors++;
 			if (priv->hwts_rx_en && !priv->extend_desc) {
@@ -3474,12 +3348,8 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 			int frame_len;
 			unsigned int des;
 
-			if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
-				des = le32_to_cpu(p->des0);
-			else
-				des = le32_to_cpu(p->des2);
-
-			frame_len = priv->hw->desc->get_rx_frame_len(p, coe);
+			stmmac_get_desc_addr(priv, p, &des);
+			frame_len = stmmac_get_rx_frame_len(priv, p, coe);
 
 			/*  If frame length is greater than skb buffer size
 			 *  (preallocated during init) then the packet is
@@ -3621,7 +3491,7 @@ static int stmmac_poll(struct napi_struct *napi, int budget)
 	work_done = stmmac_rx(priv, budget, rx_q->queue_index);
 	if (work_done < budget) {
 		napi_complete_done(napi, work_done);
-		stmmac_enable_dma_irq(priv, chan);
+		stmmac_enable_dma_irq(priv, priv->ioaddr, chan);
 	}
 	return work_done;
 }
@@ -3654,7 +3524,7 @@ static void stmmac_set_rx_mode(struct net_device *dev)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
 
-	priv->hw->mac->set_filter(priv->hw, dev);
+	stmmac_set_filter(priv, priv->hw, dev);
 }
 
 /**
@@ -3727,7 +3597,7 @@ static int stmmac_set_features(struct net_device *netdev,
 	/* No check needed because rx_coe has been set before and it will be
 	 * fixed in case of issue.
 	 */
-	priv->hw->mac->rx_ipc(priv->hw);
+	stmmac_rx_ipc(priv, priv->hw);
 
 	return 0;
 }
@@ -3771,8 +3641,8 @@ static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
 
 	/* To handle GMAC own interrupts */
 	if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) {
-		int status = priv->hw->mac->host_irq_status(priv->hw,
-							    &priv->xstats);
+		int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
+		int mtl_status;
 
 		if (unlikely(status)) {
 			/* For LPI we need to save the tx status */
@@ -3782,21 +3652,18 @@ static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
 				priv->tx_path_in_lpi_mode = false;
 		}
 
-		if (priv->synopsys_id >= DWMAC_CORE_4_00) {
-			for (queue = 0; queue < queues_count; queue++) {
-				struct stmmac_rx_queue *rx_q =
-				&priv->rx_queue[queue];
-
-				status |=
-				priv->hw->mac->host_mtl_irq_status(priv->hw,
-								   queue);
+		for (queue = 0; queue < queues_count; queue++) {
+			struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 
-				if (status & CORE_IRQ_MTL_RX_OVERFLOW &&
-				    priv->hw->dma->set_rx_tail_ptr)
-					priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
-								rx_q->rx_tail_addr,
+			mtl_status = stmmac_host_mtl_irq_status(priv, priv->hw,
 								queue);
-			}
+			if (mtl_status != -EINVAL)
+				status |= mtl_status;
+
+			if (status & CORE_IRQ_MTL_RX_OVERFLOW)
+				stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
+						       rx_q->rx_tail_addr,
+						       queue);
 		}
 
 		/* PCS link status */
@@ -3860,6 +3727,58 @@ static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 	return ret;
 }
 
+static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
+				    void *cb_priv)
+{
+	struct stmmac_priv *priv = cb_priv;
+	int ret = -EOPNOTSUPP;
+
+	stmmac_disable_all_queues(priv);
+
+	switch (type) {
+	case TC_SETUP_CLSU32:
+		if (tc_cls_can_offload_and_chain0(priv->dev, type_data))
+			ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
+		break;
+	default:
+		break;
+	}
+
+	stmmac_enable_all_queues(priv);
+	return ret;
+}
+
+static int stmmac_setup_tc_block(struct stmmac_priv *priv,
+				 struct tc_block_offload *f)
+{
+	if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
+		return -EOPNOTSUPP;
+
+	switch (f->command) {
+	case TC_BLOCK_BIND:
+		return tcf_block_cb_register(f->block, stmmac_setup_tc_block_cb,
+				priv, priv);
+	case TC_BLOCK_UNBIND:
+		tcf_block_cb_unregister(f->block, stmmac_setup_tc_block_cb, priv);
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
+			   void *type_data)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+
+	switch (type) {
+	case TC_SETUP_BLOCK:
+		return stmmac_setup_tc_block(priv, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
 {
 	struct stmmac_priv *priv = netdev_priv(ndev);
@@ -3869,7 +3788,7 @@ static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
 	if (ret)
 		return ret;
 
-	priv->hw->mac->set_umac_addr(priv->hw, ndev->dev_addr, 0);
+	stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
 
 	return ret;
 }
@@ -4098,6 +4017,7 @@ static const struct net_device_ops stmmac_netdev_ops = {
 	.ndo_set_rx_mode = stmmac_set_rx_mode,
 	.ndo_tx_timeout = stmmac_tx_timeout,
 	.ndo_do_ioctl = stmmac_ioctl,
+	.ndo_setup_tc = stmmac_setup_tc,
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	.ndo_poll_controller = stmmac_poll_controller,
 #endif
@@ -4145,49 +4065,17 @@ static void stmmac_service_task(struct work_struct *work)
  */
 static int stmmac_hw_init(struct stmmac_priv *priv)
 {
-	struct mac_device_info *mac;
-
-	/* Identify the MAC HW device */
-	if (priv->plat->setup) {
-		mac = priv->plat->setup(priv);
-	} else if (priv->plat->has_gmac) {
-		priv->dev->priv_flags |= IFF_UNICAST_FLT;
-		mac = dwmac1000_setup(priv->ioaddr,
-				      priv->plat->multicast_filter_bins,
-				      priv->plat->unicast_filter_entries,
-				      &priv->synopsys_id);
-	} else if (priv->plat->has_gmac4) {
-		priv->dev->priv_flags |= IFF_UNICAST_FLT;
-		mac = dwmac4_setup(priv->ioaddr,
-				   priv->plat->multicast_filter_bins,
-				   priv->plat->unicast_filter_entries,
-				   &priv->synopsys_id);
-	} else {
-		mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id);
-	}
-	if (!mac)
-		return -ENOMEM;
-
-	priv->hw = mac;
+	int ret;
 
 	/* dwmac-sun8i only work in chain mode */
 	if (priv->plat->has_sun8i)
 		chain_mode = 1;
+	priv->chain_mode = chain_mode;
 
-	/* To use the chained or ring mode */
-	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
-		priv->hw->mode = &dwmac4_ring_mode_ops;
-	} else {
-		if (chain_mode) {
-			priv->hw->mode = &chain_mode_ops;
-			dev_info(priv->device, "Chain mode enabled\n");
-			priv->mode = STMMAC_CHAIN_MODE;
-		} else {
-			priv->hw->mode = &ring_mode_ops;
-			dev_info(priv->device, "Ring mode enabled\n");
-			priv->mode = STMMAC_RING_MODE;
-		}
-	}
+	/* Initialize HW Interface */
+	ret = stmmac_hwif_init(priv);
+	if (ret)
+		return ret;
 
 	/* Get the HW capability (new GMAC newer than 3.50a) */
 	priv->hw_cap_support = stmmac_get_hw_features(priv);
@@ -4221,12 +4109,6 @@ static int stmmac_hw_init(struct stmmac_priv *priv)
 		dev_info(priv->device, "No HW DMA feature register supported\n");
 	}
 
-	/* To use alternate (extended), normal or GMAC4 descriptor structures */
-	if (priv->synopsys_id >= DWMAC_CORE_4_00)
-		priv->hw->desc = &dwmac4_desc_ops;
-	else
-		stmmac_selec_desc_mode(priv);
-
 	if (priv->plat->rx_coe) {
 		priv->hw->rx_csum = priv->plat->rx_coe;
 		dev_info(priv->device, "RX Checksum Offload Engine supported\n");
@@ -4335,6 +4217,11 @@ int stmmac_dvr_probe(struct device *device,
 	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
 			    NETIF_F_RXCSUM;
 
+	ret = stmmac_tc_init(priv, priv);
+	if (!ret) {
+		ndev->hw_features |= NETIF_F_HW_TC;
+	}
+
 	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
 		ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
 		priv->tso = true;
@@ -4386,7 +4273,7 @@ int stmmac_dvr_probe(struct device *device,
 			       (8 * priv->plat->rx_queues_to_use));
 	}
 
-	spin_lock_init(&priv->lock);
+	mutex_init(&priv->lock);
 
 	/* If a specific clk_csr value is passed from the platform
 	 * this means that the CSR Clock Range selection cannot be
@@ -4458,7 +4345,7 @@ int stmmac_dvr_remove(struct device *dev)
 
 	stmmac_stop_all_dma(priv);
 
-	priv->hw->mac->set_mac(priv->ioaddr, false);
+	stmmac_mac_set(priv, priv->ioaddr, false);
 	netif_carrier_off(ndev);
 	unregister_netdev(ndev);
 	if (priv->plat->stmmac_rst)
@@ -4470,6 +4357,7 @@ int stmmac_dvr_remove(struct device *dev)
 	    priv->hw->pcs != STMMAC_PCS_RTBI)
 		stmmac_mdio_unregister(ndev);
 	destroy_workqueue(priv->wq);
+	mutex_destroy(&priv->lock);
 	free_netdev(ndev);
 
 	return 0;
@@ -4487,7 +4375,6 @@ int stmmac_suspend(struct device *dev)
 {
 	struct net_device *ndev = dev_get_drvdata(dev);
 	struct stmmac_priv *priv = netdev_priv(ndev);
-	unsigned long flags;
 
 	if (!ndev || !netif_running(ndev))
 		return 0;
@@ -4495,7 +4382,7 @@ int stmmac_suspend(struct device *dev)
 	if (ndev->phydev)
 		phy_stop(ndev->phydev);
 
-	spin_lock_irqsave(&priv->lock, flags);
+	mutex_lock(&priv->lock);
 
 	netif_device_detach(ndev);
 	stmmac_stop_all_queues(priv);
@@ -4507,16 +4394,16 @@ int stmmac_suspend(struct device *dev)
 
 	/* Enable Power down mode by programming the PMT regs */
 	if (device_may_wakeup(priv->device)) {
-		priv->hw->mac->pmt(priv->hw, priv->wolopts);
+		stmmac_pmt(priv, priv->hw, priv->wolopts);
 		priv->irq_wake = 1;
 	} else {
-		priv->hw->mac->set_mac(priv->ioaddr, false);
+		stmmac_mac_set(priv, priv->ioaddr, false);
 		pinctrl_pm_select_sleep_state(priv->device);
 		/* Disable clock in case of PWM is off */
 		clk_disable(priv->plat->pclk);
 		clk_disable(priv->plat->stmmac_clk);
 	}
-	spin_unlock_irqrestore(&priv->lock, flags);
+	mutex_unlock(&priv->lock);
 
 	priv->oldlink = false;
 	priv->speed = SPEED_UNKNOWN;
@@ -4561,7 +4448,6 @@ int stmmac_resume(struct device *dev)
 {
 	struct net_device *ndev = dev_get_drvdata(dev);
 	struct stmmac_priv *priv = netdev_priv(ndev);
-	unsigned long flags;
 
 	if (!netif_running(ndev))
 		return 0;
@@ -4573,9 +4459,9 @@ int stmmac_resume(struct device *dev)
 	 * from another devices (e.g. serial console).
 	 */
 	if (device_may_wakeup(priv->device)) {
-		spin_lock_irqsave(&priv->lock, flags);
-		priv->hw->mac->pmt(priv->hw, 0);
-		spin_unlock_irqrestore(&priv->lock, flags);
+		mutex_lock(&priv->lock);
+		stmmac_pmt(priv, priv->hw, 0);
+		mutex_unlock(&priv->lock);
 		priv->irq_wake = 0;
 	} else {
 		pinctrl_pm_select_default_state(priv->device);
@@ -4589,7 +4475,7 @@ int stmmac_resume(struct device *dev)
 
 	netif_device_attach(ndev);
 
-	spin_lock_irqsave(&priv->lock, flags);
+	mutex_lock(&priv->lock);
 
 	stmmac_reset_queues_param(priv);
 
@@ -4603,7 +4489,7 @@ int stmmac_resume(struct device *dev)
 
 	stmmac_start_all_queues(priv);
 
-	spin_unlock_irqrestore(&priv->lock, flags);
+	mutex_unlock(&priv->lock);
 
 	if (ndev->phydev)
 		phy_start(ndev->phydev);
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
index f5f37bfa1d58..5df1a608e566 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
@@ -233,10 +233,7 @@ int stmmac_mdio_register(struct net_device *ndev)
 	new_bus->phy_mask = mdio_bus_data->phy_mask;
 	new_bus->parent = priv->device;
 
-	if (mdio_node)
-		err = of_mdiobus_register(new_bus, mdio_node);
-	else
-		err = mdiobus_register(new_bus);
+	err = of_mdiobus_register(new_bus, mdio_node);
 	if (err != 0) {
 		dev_err(dev, "Cannot register the MDIO bus\n");
 		goto bus_register_fail;
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
index ebd3e5ffa73c..6d141f3931eb 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
@@ -472,7 +472,8 @@ stmmac_probe_config_dt(struct platform_device *pdev, const char **mac)
 	}
 
 	if (of_device_is_compatible(np, "snps,dwmac-4.00") ||
-	    of_device_is_compatible(np, "snps,dwmac-4.10a")) {
+	    of_device_is_compatible(np, "snps,dwmac-4.10a") ||
+	    of_device_is_compatible(np, "snps,dwmac-4.20a")) {
 		plat->has_gmac4 = 1;
 		plat->has_gmac = 0;
 		plat->pmt = 1;
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
index e471a903c654..7d3a5c7f5db6 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
@@ -49,9 +49,7 @@ static int stmmac_adjust_freq(struct ptp_clock_info *ptp, s32 ppb)
 	addend = neg_adj ? (addend - diff) : (addend + diff);
 
 	spin_lock_irqsave(&priv->ptp_lock, flags);
-
-	priv->hw->ptp->config_addend(priv->ptpaddr, addend);
-
+	stmmac_config_addend(priv, priv->ptpaddr, addend);
 	spin_unlock_irqrestore(&priv->ptp_lock, flags);
 
 	return 0;
@@ -84,10 +82,8 @@ static int stmmac_adjust_time(struct ptp_clock_info *ptp, s64 delta)
 	nsec = reminder;
 
 	spin_lock_irqsave(&priv->ptp_lock, flags);
-
-	priv->hw->ptp->adjust_systime(priv->ptpaddr, sec, nsec, neg_adj,
-				      priv->plat->has_gmac4);
-
+	stmmac_adjust_systime(priv, priv->ptpaddr, sec, nsec, neg_adj,
+			priv->plat->has_gmac4);
 	spin_unlock_irqrestore(&priv->ptp_lock, flags);
 
 	return 0;
@@ -110,9 +106,7 @@ static int stmmac_get_time(struct ptp_clock_info *ptp, struct timespec64 *ts)
 	u64 ns;
 
 	spin_lock_irqsave(&priv->ptp_lock, flags);
-
-	ns = priv->hw->ptp->get_systime(priv->ptpaddr);
-
+	stmmac_get_systime(priv, priv->ptpaddr, &ns);
 	spin_unlock_irqrestore(&priv->ptp_lock, flags);
 
 	*ts = ns_to_timespec64(ns);
@@ -137,9 +131,7 @@ static int stmmac_set_time(struct ptp_clock_info *ptp,
 	unsigned long flags;
 
 	spin_lock_irqsave(&priv->ptp_lock, flags);
-
-	priv->hw->ptp->init_systime(priv->ptpaddr, ts->tv_sec, ts->tv_nsec);
-
+	stmmac_init_systime(priv, priv->ptpaddr, ts->tv_sec, ts->tv_nsec);
 	spin_unlock_irqrestore(&priv->ptp_lock, flags);
 
 	return 0;
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c
new file mode 100644
index 000000000000..881c94b73e2f
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c
@@ -0,0 +1,295 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
+ * stmmac TC Handling (HW only)
+ */
+
+#include <net/pkt_cls.h>
+#include <net/tc_act/tc_gact.h>
+#include "common.h"
+#include "dwmac4.h"
+#include "dwmac5.h"
+#include "stmmac.h"
+
+static void tc_fill_all_pass_entry(struct stmmac_tc_entry *entry)
+{
+	memset(entry, 0, sizeof(*entry));
+	entry->in_use = true;
+	entry->is_last = true;
+	entry->is_frag = false;
+	entry->prio = ~0x0;
+	entry->handle = 0;
+	entry->val.match_data = 0x0;
+	entry->val.match_en = 0x0;
+	entry->val.af = 1;
+	entry->val.dma_ch_no = 0x0;
+}
+
+static struct stmmac_tc_entry *tc_find_entry(struct stmmac_priv *priv,
+					     struct tc_cls_u32_offload *cls,
+					     bool free)
+{
+	struct stmmac_tc_entry *entry, *first = NULL, *dup = NULL;
+	u32 loc = cls->knode.handle;
+	int i;
+
+	for (i = 0; i < priv->tc_entries_max; i++) {
+		entry = &priv->tc_entries[i];
+		if (!entry->in_use && !first && free)
+			first = entry;
+		if (entry->handle == loc && !free)
+			dup = entry;
+	}
+
+	if (dup)
+		return dup;
+	if (first) {
+		first->handle = loc;
+		first->in_use = true;
+
+		/* Reset HW values */
+		memset(&first->val, 0, sizeof(first->val));
+	}
+
+	return first;
+}
+
+static int tc_fill_actions(struct stmmac_tc_entry *entry,
+			   struct stmmac_tc_entry *frag,
+			   struct tc_cls_u32_offload *cls)
+{
+	struct stmmac_tc_entry *action_entry = entry;
+	const struct tc_action *act;
+	struct tcf_exts *exts;
+	LIST_HEAD(actions);
+
+	exts = cls->knode.exts;
+	if (!tcf_exts_has_actions(exts))
+		return -EINVAL;
+	if (frag)
+		action_entry = frag;
+
+	tcf_exts_to_list(exts, &actions);
+	list_for_each_entry(act, &actions, list) {
+		/* Accept */
+		if (is_tcf_gact_ok(act)) {
+			action_entry->val.af = 1;
+			break;
+		}
+		/* Drop */
+		if (is_tcf_gact_shot(act)) {
+			action_entry->val.rf = 1;
+			break;
+		}
+
+		/* Unsupported */
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int tc_fill_entry(struct stmmac_priv *priv,
+			 struct tc_cls_u32_offload *cls)
+{
+	struct stmmac_tc_entry *entry, *frag = NULL;
+	struct tc_u32_sel *sel = cls->knode.sel;
+	u32 off, data, mask, real_off, rem;
+	u32 prio = cls->common.prio;
+	int ret;
+
+	/* Only 1 match per entry */
+	if (sel->nkeys <= 0 || sel->nkeys > 1)
+		return -EINVAL;
+
+	off = sel->keys[0].off << sel->offshift;
+	data = sel->keys[0].val;
+	mask = sel->keys[0].mask;
+
+	switch (ntohs(cls->common.protocol)) {
+	case ETH_P_ALL:
+		break;
+	case ETH_P_IP:
+		off += ETH_HLEN;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (off > priv->tc_off_max)
+		return -EINVAL;
+
+	real_off = off / 4;
+	rem = off % 4;
+
+	entry = tc_find_entry(priv, cls, true);
+	if (!entry)
+		return -EINVAL;
+
+	if (rem) {
+		frag = tc_find_entry(priv, cls, true);
+		if (!frag) {
+			ret = -EINVAL;
+			goto err_unuse;
+		}
+
+		entry->frag_ptr = frag;
+		entry->val.match_en = (mask << (rem * 8)) &
+			GENMASK(31, rem * 8);
+		entry->val.match_data = (data << (rem * 8)) &
+			GENMASK(31, rem * 8);
+		entry->val.frame_offset = real_off;
+		entry->prio = prio;
+
+		frag->val.match_en = (mask >> (rem * 8)) &
+			GENMASK(rem * 8 - 1, 0);
+		frag->val.match_data = (data >> (rem * 8)) &
+			GENMASK(rem * 8 - 1, 0);
+		frag->val.frame_offset = real_off + 1;
+		frag->prio = prio;
+		frag->is_frag = true;
+	} else {
+		entry->frag_ptr = NULL;
+		entry->val.match_en = mask;
+		entry->val.match_data = data;
+		entry->val.frame_offset = real_off;
+		entry->prio = prio;
+	}
+
+	ret = tc_fill_actions(entry, frag, cls);
+	if (ret)
+		goto err_unuse;
+
+	return 0;
+
+err_unuse:
+	if (frag)
+		frag->in_use = false;
+	entry->in_use = false;
+	return ret;
+}
+
+static void tc_unfill_entry(struct stmmac_priv *priv,
+			    struct tc_cls_u32_offload *cls)
+{
+	struct stmmac_tc_entry *entry;
+
+	entry = tc_find_entry(priv, cls, false);
+	if (!entry)
+		return;
+
+	entry->in_use = false;
+	if (entry->frag_ptr) {
+		entry = entry->frag_ptr;
+		entry->is_frag = false;
+		entry->in_use = false;
+	}
+}
+
+static int tc_config_knode(struct stmmac_priv *priv,
+			   struct tc_cls_u32_offload *cls)
+{
+	int ret;
+
+	ret = tc_fill_entry(priv, cls);
+	if (ret)
+		return ret;
+
+	ret = stmmac_rxp_config(priv, priv->hw->pcsr, priv->tc_entries,
+			priv->tc_entries_max);
+	if (ret)
+		goto err_unfill;
+
+	return 0;
+
+err_unfill:
+	tc_unfill_entry(priv, cls);
+	return ret;
+}
+
+static int tc_delete_knode(struct stmmac_priv *priv,
+			   struct tc_cls_u32_offload *cls)
+{
+	int ret;
+
+	/* Set entry and fragments as not used */
+	tc_unfill_entry(priv, cls);
+
+	ret = stmmac_rxp_config(priv, priv->hw->pcsr, priv->tc_entries,
+			priv->tc_entries_max);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int tc_setup_cls_u32(struct stmmac_priv *priv,
+			    struct tc_cls_u32_offload *cls)
+{
+	switch (cls->command) {
+	case TC_CLSU32_REPLACE_KNODE:
+		tc_unfill_entry(priv, cls);
+		/* Fall through */
+	case TC_CLSU32_NEW_KNODE:
+		return tc_config_knode(priv, cls);
+	case TC_CLSU32_DELETE_KNODE:
+		return tc_delete_knode(priv, cls);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int tc_init(struct stmmac_priv *priv)
+{
+	struct dma_features *dma_cap = &priv->dma_cap;
+	unsigned int count;
+
+	if (!dma_cap->frpsel)
+		return -EINVAL;
+
+	switch (dma_cap->frpbs) {
+	case 0x0:
+		priv->tc_off_max = 64;
+		break;
+	case 0x1:
+		priv->tc_off_max = 128;
+		break;
+	case 0x2:
+		priv->tc_off_max = 256;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (dma_cap->frpes) {
+	case 0x0:
+		count = 64;
+		break;
+	case 0x1:
+		count = 128;
+		break;
+	case 0x2:
+		count = 256;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Reserve one last filter which lets all pass */
+	priv->tc_entries_max = count;
+	priv->tc_entries = devm_kzalloc(priv->device,
+			sizeof(*priv->tc_entries) * count, GFP_KERNEL);
+	if (!priv->tc_entries)
+		return -ENOMEM;
+
+	tc_fill_all_pass_entry(&priv->tc_entries[count - 1]);
+
+	dev_info(priv->device, "Enabling HW TC (entries=%d, max_off=%d)\n",
+			priv->tc_entries_max, priv->tc_off_max);
+	return 0;
+}
+
+const struct stmmac_tc_ops dwmac510_tc_ops = {
+	.init = tc_init,
+	.setup_cls_u32 = tc_setup_cls_u32,
+};
diff --git a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig
index 48a541eb0af2..9263d638bd6d 100644
--- a/drivers/net/ethernet/ti/Kconfig
+++ b/drivers/net/ethernet/ti/Kconfig
@@ -18,7 +18,7 @@ if NET_VENDOR_TI
 
 config TI_DAVINCI_EMAC
 	tristate "TI DaVinci EMAC Support"
-	depends on ARM && ( ARCH_DAVINCI || ARCH_OMAP3 )
+	depends on ARM && ( ARCH_DAVINCI || ARCH_OMAP3 ) || COMPILE_TEST
 	select TI_DAVINCI_MDIO
 	select TI_DAVINCI_CPDMA
 	select PHYLIB
@@ -30,7 +30,7 @@ config TI_DAVINCI_EMAC
 
 config TI_DAVINCI_MDIO
 	tristate "TI DaVinci MDIO Support"
-	depends on ARCH_DAVINCI || ARCH_OMAP2PLUS || ARCH_KEYSTONE
+	depends on ARCH_DAVINCI || ARCH_OMAP2PLUS || ARCH_KEYSTONE || COMPILE_TEST
 	select PHYLIB
 	---help---
 	  This driver supports TI's DaVinci MDIO module.
@@ -40,7 +40,7 @@ config TI_DAVINCI_MDIO
 
 config TI_DAVINCI_CPDMA
 	tristate "TI DaVinci CPDMA Support"
-	depends on ARCH_DAVINCI || ARCH_OMAP2PLUS
+	depends on ARCH_DAVINCI || ARCH_OMAP2PLUS || COMPILE_TEST
 	---help---
 	  This driver supports TI's DaVinci CPDMA dma engine.
 
@@ -60,7 +60,7 @@ config TI_CPSW_ALE
 
 config TI_CPSW
 	tristate "TI CPSW Switch Support"
-	depends on ARCH_DAVINCI || ARCH_OMAP2PLUS
+	depends on ARCH_DAVINCI || ARCH_OMAP2PLUS || COMPILE_TEST
 	select TI_DAVINCI_CPDMA
 	select TI_DAVINCI_MDIO
 	select TI_CPSW_PHY_SEL
@@ -75,7 +75,7 @@ config TI_CPSW
 
 config TI_CPTS
 	bool "TI Common Platform Time Sync (CPTS) Support"
-	depends on TI_CPSW || TI_KEYSTONE_NETCP
+	depends on TI_CPSW || TI_KEYSTONE_NETCP || COMPILE_TEST
 	depends on POSIX_TIMERS
 	---help---
 	  This driver supports the Common Platform Time Sync unit of
diff --git a/drivers/net/ethernet/ti/cpsw-phy-sel.c b/drivers/net/ethernet/ti/cpsw-phy-sel.c
index 18013645e76c..0c1adad7415d 100644
--- a/drivers/net/ethernet/ti/cpsw-phy-sel.c
+++ b/drivers/net/ethernet/ti/cpsw-phy-sel.c
@@ -177,12 +177,18 @@ void cpsw_phy_sel(struct device *dev, phy_interface_t phy_mode, int slave)
 	}
 
 	dev = bus_find_device(&platform_bus_type, NULL, node, match);
-	of_node_put(node);
+	if (!dev) {
+		dev_err(dev, "unable to find platform device for %pOF\n", node);
+		goto out;
+	}
+
 	priv = dev_get_drvdata(dev);
 
 	priv->cpsw_phy_sel(priv, phy_mode, slave);
 
 	put_device(dev);
+out:
+	of_node_put(node);
 }
 EXPORT_SYMBOL_GPL(cpsw_phy_sel);
 
diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c
index 28d893b93d30..643cd2d9dfb6 100644
--- a/drivers/net/ethernet/ti/cpsw.c
+++ b/drivers/net/ethernet/ti/cpsw.c
@@ -36,6 +36,7 @@
 #include <linux/of_device.h>
 #include <linux/if_vlan.h>
 #include <linux/kmemleak.h>
+#include <linux/sys_soc.h>
 
 #include <linux/pinctrl/consumer.h>
 
@@ -957,7 +958,7 @@ static irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
+static int cpsw_tx_mq_poll(struct napi_struct *napi_tx, int budget)
 {
 	u32			ch_map;
 	int			num_tx, cur_budget, ch;
@@ -984,7 +985,21 @@ static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
 	if (num_tx < budget) {
 		napi_complete(napi_tx);
 		writel(0xff, &cpsw->wr_regs->tx_en);
-		if (cpsw->quirk_irq && cpsw->tx_irq_disabled) {
+	}
+
+	return num_tx;
+}
+
+static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
+{
+	struct cpsw_common *cpsw = napi_to_cpsw(napi_tx);
+	int num_tx;
+
+	num_tx = cpdma_chan_process(cpsw->txv[0].ch, budget);
+	if (num_tx < budget) {
+		napi_complete(napi_tx);
+		writel(0xff, &cpsw->wr_regs->tx_en);
+		if (cpsw->tx_irq_disabled) {
 			cpsw->tx_irq_disabled = false;
 			enable_irq(cpsw->irqs_table[1]);
 		}
@@ -993,7 +1008,7 @@ static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
 	return num_tx;
 }
 
-static int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
+static int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget)
 {
 	u32			ch_map;
 	int			num_rx, cur_budget, ch;
@@ -1020,7 +1035,21 @@ static int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
 	if (num_rx < budget) {
 		napi_complete_done(napi_rx, num_rx);
 		writel(0xff, &cpsw->wr_regs->rx_en);
-		if (cpsw->quirk_irq && cpsw->rx_irq_disabled) {
+	}
+
+	return num_rx;
+}
+
+static int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
+{
+	struct cpsw_common *cpsw = napi_to_cpsw(napi_rx);
+	int num_rx;
+
+	num_rx = cpdma_chan_process(cpsw->rxv[0].ch, budget);
+	if (num_rx < budget) {
+		napi_complete_done(napi_rx, num_rx);
+		writel(0xff, &cpsw->wr_regs->rx_en);
+		if (cpsw->rx_irq_disabled) {
 			cpsw->rx_irq_disabled = false;
 			enable_irq(cpsw->irqs_table[0]);
 		}
@@ -1252,8 +1281,8 @@ static void cpsw_add_ch_strings(u8 **p, int ch_num, int rx_dir)
 	for (i = 0; i < ch_stats_len; i++) {
 		line = i % CPSW_STATS_CH_LEN;
 		snprintf(*p, ETH_GSTRING_LEN,
-			 "%s DMA chan %d: %s", rx_dir ? "Rx" : "Tx",
-			 i / CPSW_STATS_CH_LEN,
+			 "%s DMA chan %ld: %s", rx_dir ? "Rx" : "Tx",
+			 (long)(i / CPSW_STATS_CH_LEN),
 			 cpsw_gstrings_ch_stats[line].stat_string);
 		*p += ETH_GSTRING_LEN;
 	}
@@ -2364,9 +2393,9 @@ static void cpsw_get_channels(struct net_device *ndev,
 {
 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 
+	ch->max_rx = cpsw->quirk_irq ? 1 : CPSW_MAX_QUEUES;
+	ch->max_tx = cpsw->quirk_irq ? 1 : CPSW_MAX_QUEUES;
 	ch->max_combined = 0;
-	ch->max_rx = CPSW_MAX_QUEUES;
-	ch->max_tx = CPSW_MAX_QUEUES;
 	ch->max_other = 0;
 	ch->other_count = 0;
 	ch->rx_count = cpsw->rx_ch_num;
@@ -2377,6 +2406,11 @@ static void cpsw_get_channels(struct net_device *ndev,
 static int cpsw_check_ch_settings(struct cpsw_common *cpsw,
 				  struct ethtool_channels *ch)
 {
+	if (cpsw->quirk_irq) {
+		dev_err(cpsw->dev, "Maximum one tx/rx queue is allowed");
+		return -EOPNOTSUPP;
+	}
+
 	if (ch->combined_count)
 		return -EINVAL;
 
@@ -2917,44 +2951,20 @@ static int cpsw_probe_dual_emac(struct cpsw_priv *priv)
 	return ret;
 }
 
-#define CPSW_QUIRK_IRQ		BIT(0)
-
-static const struct platform_device_id cpsw_devtype[] = {
-	{
-		/* keep it for existing comaptibles */
-		.name = "cpsw",
-		.driver_data = CPSW_QUIRK_IRQ,
-	}, {
-		.name = "am335x-cpsw",
-		.driver_data = CPSW_QUIRK_IRQ,
-	}, {
-		.name = "am4372-cpsw",
-		.driver_data = 0,
-	}, {
-		.name = "dra7-cpsw",
-		.driver_data = 0,
-	}, {
-		/* sentinel */
-	}
-};
-MODULE_DEVICE_TABLE(platform, cpsw_devtype);
-
-enum ti_cpsw_type {
-	CPSW = 0,
-	AM335X_CPSW,
-	AM4372_CPSW,
-	DRA7_CPSW,
-};
-
 static const struct of_device_id cpsw_of_mtable[] = {
-	{ .compatible = "ti,cpsw", .data = &cpsw_devtype[CPSW], },
-	{ .compatible = "ti,am335x-cpsw", .data = &cpsw_devtype[AM335X_CPSW], },
-	{ .compatible = "ti,am4372-cpsw", .data = &cpsw_devtype[AM4372_CPSW], },
-	{ .compatible = "ti,dra7-cpsw", .data = &cpsw_devtype[DRA7_CPSW], },
+	{ .compatible = "ti,cpsw"},
+	{ .compatible = "ti,am335x-cpsw"},
+	{ .compatible = "ti,am4372-cpsw"},
+	{ .compatible = "ti,dra7-cpsw"},
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, cpsw_of_mtable);
 
+static const struct soc_device_attribute cpsw_soc_devices[] = {
+	{ .family = "AM33xx", .revision = "ES1.0"},
+	{ /* sentinel */ }
+};
+
 static int cpsw_probe(struct platform_device *pdev)
 {
 	struct clk			*clk;
@@ -2966,9 +2976,9 @@ static int cpsw_probe(struct platform_device *pdev)
 	void __iomem			*ss_regs;
 	void __iomem			*cpts_regs;
 	struct resource			*res, *ss_res;
-	const struct of_device_id	*of_id;
 	struct gpio_descs		*mode;
 	u32 slave_offset, sliver_offset, slave_size;
+	const struct soc_device_attribute *soc;
 	struct cpsw_common		*cpsw;
 	int ret = 0, i;
 	int irq;
@@ -3141,6 +3151,10 @@ static int cpsw_probe(struct platform_device *pdev)
 		goto clean_dt_ret;
 	}
 
+	soc = soc_device_match(cpsw_soc_devices);
+	if (soc)
+		cpsw->quirk_irq = 1;
+
 	cpsw->txv[0].ch = cpdma_chan_create(cpsw->dma, 0, cpsw_tx_handler, 0);
 	if (IS_ERR(cpsw->txv[0].ch)) {
 		dev_err(priv->dev, "error initializing tx dma channel\n");
@@ -3180,19 +3194,16 @@ static int cpsw_probe(struct platform_device *pdev)
 		goto clean_dma_ret;
 	}
 
-	of_id = of_match_device(cpsw_of_mtable, &pdev->dev);
-	if (of_id) {
-		pdev->id_entry = of_id->data;
-		if (pdev->id_entry->driver_data)
-			cpsw->quirk_irq = true;
-	}
-
 	ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_RX;
 
 	ndev->netdev_ops = &cpsw_netdev_ops;
 	ndev->ethtool_ops = &cpsw_ethtool_ops;
-	netif_napi_add(ndev, &cpsw->napi_rx, cpsw_rx_poll, CPSW_POLL_WEIGHT);
-	netif_tx_napi_add(ndev, &cpsw->napi_tx, cpsw_tx_poll, CPSW_POLL_WEIGHT);
+	netif_napi_add(ndev, &cpsw->napi_rx,
+		       cpsw->quirk_irq ? cpsw_rx_poll : cpsw_rx_mq_poll,
+		       CPSW_POLL_WEIGHT);
+	netif_tx_napi_add(ndev, &cpsw->napi_tx,
+			  cpsw->quirk_irq ? cpsw_tx_poll : cpsw_tx_mq_poll,
+			  CPSW_POLL_WEIGHT);
 	cpsw_split_res(ndev);
 
 	/* register the network device */
diff --git a/drivers/net/ethernet/ti/cpts.c b/drivers/net/ethernet/ti/cpts.c
index e7b76f6b4f67..6f63c8729afc 100644
--- a/drivers/net/ethernet/ti/cpts.c
+++ b/drivers/net/ethernet/ti/cpts.c
@@ -294,7 +294,8 @@ static long cpts_overflow_check(struct ptp_clock_info *ptp)
 		delay = CPTS_SKB_TX_WORK_TIMEOUT;
 	spin_unlock_irqrestore(&cpts->lock, flags);
 
-	pr_debug("cpts overflow check at %lld.%09lu\n", ts.tv_sec, ts.tv_nsec);
+	pr_debug("cpts overflow check at %lld.%09ld\n",
+		 (long long)ts.tv_sec, ts.tv_nsec);
 	return (long)delay;
 }
 
@@ -564,7 +565,7 @@ struct cpts *cpts_create(struct device *dev, void __iomem *regs,
 	cpts->refclk = devm_clk_get(dev, "cpts");
 	if (IS_ERR(cpts->refclk)) {
 		dev_err(dev, "Failed to get cpts refclk\n");
-		return ERR_PTR(PTR_ERR(cpts->refclk));
+		return ERR_CAST(cpts->refclk);
 	}
 
 	clk_prepare(cpts->refclk);
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.c b/drivers/net/ethernet/ti/davinci_cpdma.c
index 31ae04117f0a..cdbddf16dd29 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.c
+++ b/drivers/net/ethernet/ti/davinci_cpdma.c
@@ -191,7 +191,7 @@ static void cpdma_desc_pool_destroy(struct cpdma_ctlr *ctlr)
 		return;
 
 	WARN(gen_pool_size(pool->gen_pool) != gen_pool_avail(pool->gen_pool),
-	     "cpdma_desc_pool size %d != avail %d",
+	     "cpdma_desc_pool size %zd != avail %zd",
 	     gen_pool_size(pool->gen_pool),
 	     gen_pool_avail(pool->gen_pool));
 	if (pool->cpumap)
@@ -1080,7 +1080,7 @@ int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data,
 	writel_relaxed(buffer, &desc->hw_buffer);
 	writel_relaxed(len, &desc->hw_len);
 	writel_relaxed(mode | len, &desc->hw_mode);
-	writel_relaxed(token, &desc->sw_token);
+	writel_relaxed((uintptr_t)token, &desc->sw_token);
 	writel_relaxed(buffer, &desc->sw_buffer);
 	writel_relaxed(len, &desc->sw_len);
 	desc_read(desc, sw_len);
@@ -1121,15 +1121,15 @@ static void __cpdma_chan_free(struct cpdma_chan *chan,
 	struct cpdma_desc_pool		*pool = ctlr->pool;
 	dma_addr_t			buff_dma;
 	int				origlen;
-	void				*token;
+	uintptr_t			token;
 
-	token      = (void *)desc_read(desc, sw_token);
+	token      = desc_read(desc, sw_token);
 	buff_dma   = desc_read(desc, sw_buffer);
 	origlen    = desc_read(desc, sw_len);
 
 	dma_unmap_single(ctlr->dev, buff_dma, origlen, chan->dir);
 	cpdma_desc_free(pool, desc, 1);
-	(*chan->handler)(token, outlen, status);
+	(*chan->handler)((void *)token, outlen, status);
 }
 
 static int __cpdma_chan_process(struct cpdma_chan *chan)
diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c
index abceea802ea1..be0fec17d95d 100644
--- a/drivers/net/ethernet/ti/davinci_emac.c
+++ b/drivers/net/ethernet/ti/davinci_emac.c
@@ -1930,8 +1930,8 @@ static int davinci_emac_probe(struct platform_device *pdev)
 
 	if (netif_msg_probe(priv)) {
 		dev_notice(&pdev->dev, "DaVinci EMAC Probe found device "
-			   "(regs: %p, irq: %d)\n",
-			   (void *)priv->emac_base_phys, ndev->irq);
+			   "(regs: %pa, irq: %d)\n",
+			   &priv->emac_base_phys, ndev->irq);
 	}
 	pm_runtime_put(&pdev->dev);
 
diff --git a/drivers/net/ethernet/ti/davinci_mdio.c b/drivers/net/ethernet/ti/davinci_mdio.c
index 3c33f4504d8e..a98aedae1b41 100644
--- a/drivers/net/ethernet/ti/davinci_mdio.c
+++ b/drivers/net/ethernet/ti/davinci_mdio.c
@@ -34,6 +34,7 @@
 #include <linux/clk.h>
 #include <linux/err.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/pm_runtime.h>
 #include <linux/davinci_emac.h>
 #include <linux/of.h>
@@ -227,14 +228,14 @@ static inline int wait_for_user_access(struct davinci_mdio_data *data)
 static inline int wait_for_idle(struct davinci_mdio_data *data)
 {
 	struct davinci_mdio_regs __iomem *regs = data->regs;
-	unsigned long timeout = jiffies + msecs_to_jiffies(MDIO_TIMEOUT);
+	u32 val, ret;
 
-	while (time_after(timeout, jiffies)) {
-		if (__raw_readl(&regs->control) & CONTROL_IDLE)
-			return 0;
-	}
-	dev_err(data->dev, "timed out waiting for idle\n");
-	return -ETIMEDOUT;
+	ret = readl_poll_timeout(&regs->control, val, val & CONTROL_IDLE,
+				 0, MDIO_TIMEOUT * 1000);
+	if (ret)
+		dev_err(data->dev, "timed out waiting for idle\n");
+
+	return ret;
 }
 
 static int davinci_mdio_read(struct mii_bus *bus, int phy_id, int phy_reg)
@@ -320,7 +321,6 @@ static int davinci_mdio_write(struct mii_bus *bus, int phy_id,
 	return ret;
 }
 
-#if IS_ENABLED(CONFIG_OF)
 static int davinci_mdio_probe_dt(struct mdio_platform_data *data,
 			 struct platform_device *pdev)
 {
@@ -338,7 +338,6 @@ static int davinci_mdio_probe_dt(struct mdio_platform_data *data,
 
 	return 0;
 }
-#endif
 
 #if IS_ENABLED(CONFIG_OF)
 static const struct davinci_mdio_of_param of_cpsw_mdio_data = {
@@ -373,7 +372,7 @@ static int davinci_mdio_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	}
 
-	if (dev->of_node) {
+	if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
 		const struct of_device_id	*of_id;
 
 		ret = davinci_mdio_probe_dt(&data->pdata, pdev);
@@ -428,12 +427,10 @@ static int davinci_mdio_probe(struct platform_device *pdev)
 	 * defined to support backward compatibility with DTs which assume that
 	 * Davinci MDIO will always scan the bus for PHYs detection.
 	 */
-	if (dev->of_node && of_get_child_count(dev->of_node)) {
+	if (dev->of_node && of_get_child_count(dev->of_node))
 		data->skip_scan = true;
-		ret = of_mdiobus_register(data->bus, dev->of_node);
-	} else {
-		ret = mdiobus_register(data->bus);
-	}
+
+	ret = of_mdiobus_register(data->bus, dev->of_node);
 	if (ret)
 		goto bail_out;
 
diff --git a/drivers/net/ethernet/ti/netcp.h b/drivers/net/ethernet/ti/netcp.h
index 8900a6fad318..c4ffdf47bad5 100644
--- a/drivers/net/ethernet/ti/netcp.h
+++ b/drivers/net/ethernet/ti/netcp.h
@@ -33,6 +33,8 @@
 #define SGMII_LINK_MAC_MAC_FORCED	2
 #define SGMII_LINK_MAC_FIBER		3
 #define SGMII_LINK_MAC_PHY_NO_MDIO	4
+#define RGMII_LINK_MAC_PHY		5
+#define RGMII_LINK_MAC_PHY_NO_MDIO	7
 #define XGMII_LINK_MAC_PHY		10
 #define XGMII_LINK_MAC_MAC_FORCED	11
 
@@ -212,6 +214,7 @@ struct netcp_module {
 	int	(*add_vid)(void *intf_priv, int vid);
 	int	(*del_vid)(void *intf_priv, int vid);
 	int	(*ioctl)(void *intf_priv, struct ifreq *req, int cmd);
+	int	(*set_rx_mode)(void *intf_priv, bool promisc);
 
 	/* used internally */
 	struct list_head	module_list;
diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c
index f5a7eb22d0f5..e40aa3e31af2 100644
--- a/drivers/net/ethernet/ti/netcp_core.c
+++ b/drivers/net/ethernet/ti/netcp_core.c
@@ -1509,6 +1509,24 @@ static void netcp_addr_sweep_add(struct netcp_intf *netcp)
 	}
 }
 
+static int netcp_set_promiscuous(struct netcp_intf *netcp, bool promisc)
+{
+	struct netcp_intf_modpriv *priv;
+	struct netcp_module *module;
+	int error;
+
+	for_each_module(netcp, priv) {
+		module = priv->netcp_module;
+		if (!module->set_rx_mode)
+			continue;
+
+		error = module->set_rx_mode(priv->module_priv, promisc);
+		if (error)
+			return error;
+	}
+	return 0;
+}
+
 static void netcp_set_rx_mode(struct net_device *ndev)
 {
 	struct netcp_intf *netcp = netdev_priv(ndev);
@@ -1538,6 +1556,7 @@ static void netcp_set_rx_mode(struct net_device *ndev)
 	/* finally sweep and callout into modules */
 	netcp_addr_sweep_del(netcp);
 	netcp_addr_sweep_add(netcp);
+	netcp_set_promiscuous(netcp, promisc);
 	spin_unlock(&netcp->lock);
 }
 
@@ -2155,8 +2174,13 @@ static int netcp_probe(struct platform_device *pdev)
 	struct device_node *child, *interfaces;
 	struct netcp_device *netcp_device;
 	struct device *dev = &pdev->dev;
+	struct netcp_module *module;
 	int ret;
 
+	if (!knav_dma_device_ready() ||
+	    !knav_qmss_device_ready())
+		return -EPROBE_DEFER;
+
 	if (!node) {
 		dev_err(dev, "could not find device info\n");
 		return -ENODEV;
@@ -2203,6 +2227,14 @@ static int netcp_probe(struct platform_device *pdev)
 	/* Add the device instance to the list */
 	list_add_tail(&netcp_device->device_list, &netcp_devices);
 
+	/* Probe & attach any modules already registered */
+	mutex_lock(&netcp_modules_lock);
+	for_each_netcp_module(module) {
+		ret = netcp_module_probe(netcp_device, module);
+		if (ret < 0)
+			dev_err(dev, "module(%s) probe failed\n", module->name);
+	}
+	mutex_unlock(&netcp_modules_lock);
 	return 0;
 
 probe_quit_interface:
diff --git a/drivers/net/ethernet/ti/netcp_ethss.c b/drivers/net/ethernet/ti/netcp_ethss.c
index 56dbc0b9fedc..6a728d35e776 100644
--- a/drivers/net/ethernet/ti/netcp_ethss.c
+++ b/drivers/net/ethernet/ti/netcp_ethss.c
@@ -21,6 +21,7 @@
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/of_mdio.h>
+#include <linux/of_net.h>
 #include <linux/of_address.h>
 #include <linux/if_vlan.h>
 #include <linux/ptp_classify.h>
@@ -42,7 +43,7 @@
 
 /* 1G Ethernet SS defines */
 #define GBE_MODULE_NAME			"netcp-gbe"
-#define GBE_SS_VERSION_14		0x4ed21104
+#define GBE_SS_VERSION_14		0x4ed2
 
 #define GBE_SS_REG_INDEX		0
 #define GBE_SGMII34_REG_INDEX		1
@@ -72,6 +73,11 @@
 #define IS_SS_ID_NU(d) \
 	(GBE_IDENT((d)->ss_version) == GBE_SS_ID_NU)
 
+#define IS_SS_ID_VER_14(d) \
+	(GBE_IDENT((d)->ss_version) == GBE_SS_VERSION_14)
+#define IS_SS_ID_2U(d) \
+	(GBE_IDENT((d)->ss_version) == GBE_SS_ID_2U)
+
 #define GBENU_SS_REG_INDEX		0
 #define GBENU_SM_REG_INDEX		1
 #define GBENU_SGMII_MODULE_OFFSET	0x100
@@ -86,7 +92,7 @@
 
 /* 10G Ethernet SS defines */
 #define XGBE_MODULE_NAME		"netcp-xgbe"
-#define XGBE_SS_VERSION_10		0x4ee42100
+#define XGBE_SS_VERSION_10		0x4ee4
 
 #define XGBE_SS_REG_INDEX		0
 #define XGBE_SM_REG_INDEX		1
@@ -166,6 +172,11 @@
 #define	GBE_RXHOOK_ORDER			0
 #define GBE_DEFAULT_ALE_AGEOUT			30
 #define SLAVE_LINK_IS_XGMII(s) ((s)->link_interface >= XGMII_LINK_MAC_PHY)
+#define SLAVE_LINK_IS_RGMII(s) \
+	(((s)->link_interface >= RGMII_LINK_MAC_PHY) && \
+	 ((s)->link_interface <= RGMII_LINK_MAC_PHY_NO_MDIO))
+#define SLAVE_LINK_IS_SGMII(s) \
+	((s)->link_interface <= SGMII_LINK_MAC_PHY_NO_MDIO)
 #define NETCP_LINK_STATE_INVALID		-1
 
 #define GBE_SET_REG_OFS(p, rb, rn) p->rb##_ofs.rn = \
@@ -549,6 +560,7 @@ struct gbe_ss_regs {
 struct gbe_ss_regs_ofs {
 	u16	id_ver;
 	u16	control;
+	u16	rgmii_status; /* 2U */
 };
 
 struct gbe_switch_regs {
@@ -591,6 +603,7 @@ struct gbe_port_regs {
 struct gbe_port_regs_ofs {
 	u16	port_vlan;
 	u16	tx_pri_map;
+	u16     rx_pri_map;
 	u16	sa_lo;
 	u16	sa_hi;
 	u16	ts_ctl;
@@ -695,6 +708,7 @@ struct gbe_slave {
 	u32				link_interface;
 	u32				mac_control;
 	u8				phy_port_t;
+	struct device_node		*node;
 	struct device_node		*phy_node;
 	struct ts_ctl                   ts_ctl;
 	struct list_head		slave_list;
@@ -1915,7 +1929,7 @@ static void keystone_get_ethtool_stats(struct net_device *ndev,
 
 	gbe_dev = gbe_intf->gbe_dev;
 	spin_lock_bh(&gbe_dev->hw_stats_lock);
-	if (gbe_dev->ss_version == GBE_SS_VERSION_14)
+	if (IS_SS_ID_VER_14(gbe_dev))
 		gbe_update_stats_ver14(gbe_dev, data);
 	else
 		gbe_update_stats(gbe_dev, data);
@@ -2091,8 +2105,9 @@ static void netcp_ethss_link_state_action(struct gbe_priv *gbe_dev,
 				     ALE_PORT_STATE_FORWARD);
 
 		if (ndev && slave->open &&
-		    slave->link_interface != SGMII_LINK_MAC_PHY &&
-		    slave->link_interface != XGMII_LINK_MAC_PHY)
+		    ((slave->link_interface != SGMII_LINK_MAC_PHY) &&
+		    (slave->link_interface != RGMII_LINK_MAC_PHY) &&
+		    (slave->link_interface != XGMII_LINK_MAC_PHY)))
 			netif_carrier_on(ndev);
 	} else {
 		writel(mac_control, GBE_REG_ADDR(slave, emac_regs,
@@ -2101,8 +2116,9 @@ static void netcp_ethss_link_state_action(struct gbe_priv *gbe_dev,
 				     ALE_PORT_STATE,
 				     ALE_PORT_STATE_DISABLE);
 		if (ndev &&
-		    slave->link_interface != SGMII_LINK_MAC_PHY &&
-		    slave->link_interface != XGMII_LINK_MAC_PHY)
+		    ((slave->link_interface != SGMII_LINK_MAC_PHY) &&
+		    (slave->link_interface != RGMII_LINK_MAC_PHY) &&
+		    (slave->link_interface != XGMII_LINK_MAC_PHY)))
 			netif_carrier_off(ndev);
 	}
 
@@ -2115,23 +2131,35 @@ static bool gbe_phy_link_status(struct gbe_slave *slave)
 	 return !slave->phy || slave->phy->link;
 }
 
+#define RGMII_REG_STATUS_LINK	BIT(0)
+
+static void netcp_2u_rgmii_get_port_link(struct gbe_priv *gbe_dev, bool *status)
+{
+	u32 val = 0;
+
+	val = readl(GBE_REG_ADDR(gbe_dev, ss_regs, rgmii_status));
+	*status = !!(val & RGMII_REG_STATUS_LINK);
+}
+
 static void netcp_ethss_update_link_state(struct gbe_priv *gbe_dev,
 					  struct gbe_slave *slave,
 					  struct net_device *ndev)
 {
-	int sp = slave->slave_num;
-	int phy_link_state, sgmii_link_state = 1, link_state;
+	bool sw_link_state = true, phy_link_state;
+	int sp = slave->slave_num, link_state;
 
 	if (!slave->open)
 		return;
 
-	if (!SLAVE_LINK_IS_XGMII(slave)) {
-		sgmii_link_state =
-			netcp_sgmii_get_port_link(SGMII_BASE(gbe_dev, sp), sp);
-	}
+	if (SLAVE_LINK_IS_RGMII(slave))
+		netcp_2u_rgmii_get_port_link(gbe_dev,
+					     &sw_link_state);
+	if (SLAVE_LINK_IS_SGMII(slave))
+		sw_link_state =
+		netcp_sgmii_get_port_link(SGMII_BASE(gbe_dev, sp), sp);
 
 	phy_link_state = gbe_phy_link_status(slave);
-	link_state = phy_link_state & sgmii_link_state;
+	link_state = phy_link_state & sw_link_state;
 
 	if (atomic_xchg(&slave->link_state, link_state) != link_state)
 		netcp_ethss_link_state_action(gbe_dev, ndev, slave,
@@ -2205,7 +2233,7 @@ static void gbe_port_config(struct gbe_priv *gbe_dev, struct gbe_slave *slave,
 		max_rx_len = NETCP_MAX_FRAME_SIZE;
 
 	/* Enable correct MII mode at SS level */
-	if ((gbe_dev->ss_version == XGBE_SS_VERSION_10) &&
+	if (IS_SS_ID_XGBE(gbe_dev) &&
 	    (slave->link_interface >= XGMII_LINK_MAC_PHY)) {
 		xgmii_mode = readl(GBE_REG_ADDR(gbe_dev, ss_regs, control));
 		xgmii_mode |= (1 << slave->slave_num);
@@ -2236,7 +2264,8 @@ static void gbe_slave_stop(struct gbe_intf *intf)
 	struct gbe_priv *gbe_dev = intf->gbe_dev;
 	struct gbe_slave *slave = intf->slave;
 
-	gbe_sgmii_rtreset(gbe_dev, slave, true);
+	if (!IS_SS_ID_2U(gbe_dev))
+		gbe_sgmii_rtreset(gbe_dev, slave, true);
 	gbe_port_reset(slave);
 	/* Disable forwarding */
 	cpsw_ale_control_set(gbe_dev->ale, slave->port_num,
@@ -2271,11 +2300,20 @@ static int gbe_slave_open(struct gbe_intf *gbe_intf)
 
 	void (*hndlr)(struct net_device *) = gbe_adjust_link;
 
-	gbe_sgmii_config(priv, slave);
+	if (!IS_SS_ID_2U(priv))
+		gbe_sgmii_config(priv, slave);
 	gbe_port_reset(slave);
-	gbe_sgmii_rtreset(priv, slave, false);
+	if (!IS_SS_ID_2U(priv))
+		gbe_sgmii_rtreset(priv, slave, false);
 	gbe_port_config(priv, slave, priv->rx_packet_max);
 	gbe_set_slave_mac(slave, gbe_intf);
+	/* For NU & 2U switch, map the vlan priorities to zero
+	 * as we only configure to use priority 0
+	 */
+	if (IS_SS_ID_MU(priv))
+		writel(HOST_TX_PRI_MAP_DEFAULT,
+		       GBE_REG_ADDR(slave, port_regs, rx_pri_map));
+
 	/* enable forwarding */
 	cpsw_ale_control_set(priv->ale, slave->port_num,
 			     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
@@ -2286,6 +2324,21 @@ static int gbe_slave_open(struct gbe_intf *gbe_intf)
 		has_phy = true;
 		phy_mode = PHY_INTERFACE_MODE_SGMII;
 		slave->phy_port_t = PORT_MII;
+	} else if (slave->link_interface == RGMII_LINK_MAC_PHY) {
+		has_phy = true;
+		phy_mode = of_get_phy_mode(slave->node);
+		/* if phy-mode is not present, default to
+		 * PHY_INTERFACE_MODE_RGMII
+		 */
+		if (phy_mode < 0)
+			phy_mode = PHY_INTERFACE_MODE_RGMII;
+
+		if (!phy_interface_mode_is_rgmii(phy_mode)) {
+			dev_err(priv->dev,
+				"Unsupported phy mode %d\n", phy_mode);
+			return -EINVAL;
+		}
+		slave->phy_port_t = PORT_MII;
 	} else if (slave->link_interface == XGMII_LINK_MAC_PHY) {
 		has_phy = true;
 		phy_mode = PHY_INTERFACE_MODE_NA;
@@ -2293,7 +2346,7 @@ static int gbe_slave_open(struct gbe_intf *gbe_intf)
 	}
 
 	if (has_phy) {
-		if (priv->ss_version == XGBE_SS_VERSION_10)
+		if (IS_SS_ID_XGBE(priv))
 			hndlr = xgbe_adjust_link;
 
 		slave->phy = of_phy_connect(gbe_intf->ndev,
@@ -2722,6 +2775,61 @@ static inline int gbe_hwtstamp_set(struct gbe_intf *gbe_intf, struct ifreq *req)
 }
 #endif /* CONFIG_TI_CPTS */
 
+static int gbe_set_rx_mode(void *intf_priv, bool promisc)
+{
+	struct gbe_intf *gbe_intf = intf_priv;
+	struct gbe_priv *gbe_dev = gbe_intf->gbe_dev;
+	struct cpsw_ale *ale = gbe_dev->ale;
+	unsigned long timeout;
+	int i, ret = -ETIMEDOUT;
+
+	/* Disable(1)/Enable(0) Learn for all ports (host is port 0 and
+	 * slaves are port 1 and up
+	 */
+	for (i = 0; i <= gbe_dev->num_slaves; i++) {
+		cpsw_ale_control_set(ale, i,
+				     ALE_PORT_NOLEARN, !!promisc);
+		cpsw_ale_control_set(ale, i,
+				     ALE_PORT_NO_SA_UPDATE, !!promisc);
+	}
+
+	if (!promisc) {
+		/* Don't Flood All Unicast Packets to Host port */
+		cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 0);
+		dev_vdbg(gbe_dev->dev, "promiscuous mode disabled\n");
+		return 0;
+	}
+
+	timeout = jiffies + HZ;
+
+	/* Clear All Untouched entries */
+	cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
+	do {
+		cpu_relax();
+		if (cpsw_ale_control_get(ale, 0, ALE_AGEOUT)) {
+			ret = 0;
+			break;
+		}
+
+	} while (time_after(timeout, jiffies));
+
+	/* Make sure it is not a false timeout */
+	if (ret && !cpsw_ale_control_get(ale, 0, ALE_AGEOUT))
+		return ret;
+
+	cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
+
+	/* Clear all mcast from ALE */
+	cpsw_ale_flush_multicast(ale,
+				 GBE_PORT_MASK(gbe_dev->ale_ports),
+				 -1);
+
+	/* Flood All Unicast Packets to Host port */
+	cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 1);
+	dev_vdbg(gbe_dev->dev, "promiscuous mode enabled\n");
+	return ret;
+}
+
 static int gbe_ioctl(void *intf_priv, struct ifreq *req, int cmd)
 {
 	struct gbe_intf *gbe_intf = intf_priv;
@@ -2764,7 +2872,7 @@ static void netcp_ethss_timer(struct timer_list *t)
 	/* A timer runs as a BH, no need to block them */
 	spin_lock(&gbe_dev->hw_stats_lock);
 
-	if (gbe_dev->ss_version == GBE_SS_VERSION_14)
+	if (IS_SS_ID_VER_14(gbe_dev))
 		gbe_update_stats_ver14(gbe_dev, NULL);
 	else
 		gbe_update_stats(gbe_dev, NULL);
@@ -2807,7 +2915,7 @@ static int gbe_open(void *intf_priv, struct net_device *ndev)
 		GBE_RTL_VERSION(reg), GBE_IDENT(reg));
 
 	/* For 10G and on NetCP 1.5, use directed to port */
-	if ((gbe_dev->ss_version == XGBE_SS_VERSION_10) || IS_SS_ID_MU(gbe_dev))
+	if (IS_SS_ID_XGBE(gbe_dev) || IS_SS_ID_MU(gbe_dev))
 		gbe_intf->tx_pipe.flags = SWITCH_TO_PORT_IN_TAGINFO;
 
 	if (gbe_dev->enable_ale)
@@ -2911,8 +3019,10 @@ static int init_slave(struct gbe_priv *gbe_dev, struct gbe_slave *slave,
 		slave->link_interface = SGMII_LINK_MAC_PHY;
 	}
 
+	slave->node = node;
 	slave->open = false;
 	if ((slave->link_interface == SGMII_LINK_MAC_PHY) ||
+	    (slave->link_interface == RGMII_LINK_MAC_PHY) ||
 	    (slave->link_interface == XGMII_LINK_MAC_PHY))
 		slave->phy_node = of_parse_phandle(node, "phy-handle", 0);
 	slave->port_num = gbe_get_slave_port(gbe_dev, slave->slave_num);
@@ -2924,7 +3034,7 @@ static int init_slave(struct gbe_priv *gbe_dev, struct gbe_slave *slave,
 
 	/* Emac regs memmap are contiguous but port regs are not */
 	port_reg_num = slave->slave_num;
-	if (gbe_dev->ss_version == GBE_SS_VERSION_14) {
+	if (IS_SS_ID_VER_14(gbe_dev)) {
 		if (slave->slave_num > 1) {
 			port_reg_ofs = GBE13_SLAVE_PORT2_OFFSET;
 			port_reg_num -= 2;
@@ -2939,7 +3049,7 @@ static int init_slave(struct gbe_priv *gbe_dev, struct gbe_slave *slave,
 		emac_reg_ofs = GBENU_EMAC_OFFSET;
 		port_reg_blk_sz = 0x1000;
 		emac_reg_blk_sz = 0x1000;
-	} else if (gbe_dev->ss_version == XGBE_SS_VERSION_10) {
+	} else if (IS_SS_ID_XGBE(gbe_dev)) {
 		port_reg_ofs = XGBE10_SLAVE_PORT_OFFSET;
 		emac_reg_ofs = XGBE10_EMAC_OFFSET;
 		port_reg_blk_sz = 0x30;
@@ -2955,7 +3065,7 @@ static int init_slave(struct gbe_priv *gbe_dev, struct gbe_slave *slave,
 	slave->emac_regs = gbe_dev->switch_regs + emac_reg_ofs +
 				(emac_reg_blk_sz * slave->slave_num);
 
-	if (gbe_dev->ss_version == GBE_SS_VERSION_14) {
+	if (IS_SS_ID_VER_14(gbe_dev)) {
 		/* Initialize  slave port register offsets */
 		GBE_SET_REG_OFS(slave, port_regs, port_vlan);
 		GBE_SET_REG_OFS(slave, port_regs, tx_pri_map);
@@ -2976,6 +3086,7 @@ static int init_slave(struct gbe_priv *gbe_dev, struct gbe_slave *slave,
 		/* Initialize  slave port register offsets */
 		GBENU_SET_REG_OFS(slave, port_regs, port_vlan);
 		GBENU_SET_REG_OFS(slave, port_regs, tx_pri_map);
+		GBENU_SET_REG_OFS(slave, port_regs, rx_pri_map);
 		GBENU_SET_REG_OFS(slave, port_regs, sa_lo);
 		GBENU_SET_REG_OFS(slave, port_regs, sa_hi);
 		GBENU_SET_REG_OFS(slave, port_regs, ts_ctl);
@@ -2989,7 +3100,7 @@ static int init_slave(struct gbe_priv *gbe_dev, struct gbe_slave *slave,
 		GBENU_SET_REG_OFS(slave, emac_regs, mac_control);
 		GBENU_SET_REG_OFS(slave, emac_regs, soft_reset);
 
-	} else if (gbe_dev->ss_version == XGBE_SS_VERSION_10) {
+	} else if (IS_SS_ID_XGBE(gbe_dev)) {
 		/* Initialize  slave port register offsets */
 		XGBE_SET_REG_OFS(slave, port_regs, port_vlan);
 		XGBE_SET_REG_OFS(slave, port_regs, tx_pri_map);
@@ -3039,7 +3150,8 @@ static void init_secondary_ports(struct gbe_priv *gbe_dev,
 			continue;
 		}
 
-		gbe_sgmii_config(gbe_dev, slave);
+		if (!IS_SS_ID_2U(gbe_dev))
+			gbe_sgmii_config(gbe_dev, slave);
 		gbe_port_reset(slave);
 		gbe_port_config(gbe_dev, slave, gbe_dev->rx_packet_max);
 		list_add_tail(&slave->slave_list, &gbe_dev->secondary_slaves);
@@ -3073,6 +3185,9 @@ static void init_secondary_ports(struct gbe_priv *gbe_dev,
 	if (slave->link_interface == SGMII_LINK_MAC_PHY) {
 		phy_mode = PHY_INTERFACE_MODE_SGMII;
 		slave->phy_port_t = PORT_MII;
+	} else if (slave->link_interface == RGMII_LINK_MAC_PHY) {
+		phy_mode = PHY_INTERFACE_MODE_RGMII;
+		slave->phy_port_t = PORT_MII;
 	} else {
 		phy_mode = PHY_INTERFACE_MODE_NA;
 		slave->phy_port_t = PORT_FIBRE;
@@ -3080,6 +3195,7 @@ static void init_secondary_ports(struct gbe_priv *gbe_dev,
 
 	for_each_sec_slave(slave, gbe_dev) {
 		if ((slave->link_interface != SGMII_LINK_MAC_PHY) &&
+		    (slave->link_interface != RGMII_LINK_MAC_PHY) &&
 		    (slave->link_interface != XGMII_LINK_MAC_PHY))
 			continue;
 		slave->phy =
@@ -3355,7 +3471,7 @@ static int set_gbenu_ethss_priv(struct gbe_priv *gbe_dev,
 	gbe_dev->num_stats_mods = gbe_dev->max_num_ports;
 	gbe_dev->et_stats = gbenu_et_stats;
 
-	if (IS_SS_ID_NU(gbe_dev))
+	if (IS_SS_ID_MU(gbe_dev))
 		gbe_dev->num_et_stats = GBENU_ET_STATS_HOST_SIZE +
 			(gbe_dev->max_num_slaves * GBENU_ET_STATS_PORT_SIZE);
 	else
@@ -3396,7 +3512,9 @@ static int set_gbenu_ethss_priv(struct gbe_priv *gbe_dev,
 	}
 	gbe_dev->switch_regs = regs;
 
-	gbe_dev->sgmii_port_regs = gbe_dev->ss_regs + GBENU_SGMII_MODULE_OFFSET;
+	if (!IS_SS_ID_2U(gbe_dev))
+		gbe_dev->sgmii_port_regs =
+		       gbe_dev->ss_regs + GBENU_SGMII_MODULE_OFFSET;
 
 	/* Although sgmii modules are mem mapped to one contiguous
 	 * region on GBENU devices, setting sgmii_port34_regs allows
@@ -3419,6 +3537,8 @@ static int set_gbenu_ethss_priv(struct gbe_priv *gbe_dev,
 
 	/* Subsystem registers */
 	GBENU_SET_REG_OFS(gbe_dev, ss_regs, id_ver);
+	/* ok to set for MU, but used by 2U only */
+	GBENU_SET_REG_OFS(gbe_dev, ss_regs, rgmii_status);
 
 	/* Switch module registers */
 	GBENU_SET_REG_OFS(gbe_dev, switch_regs, id_ver);
@@ -3464,6 +3584,7 @@ static int gbe_probe(struct netcp_device *netcp_device, struct device *dev,
 		gbe_dev->max_num_slaves = 8;
 	} else if (of_device_is_compatible(node, "ti,netcp-gbe-2")) {
 		gbe_dev->max_num_slaves = 1;
+		gbe_module.set_rx_mode = gbe_set_rx_mode;
 	} else if (of_device_is_compatible(node, "ti,netcp-xgbe")) {
 		gbe_dev->max_num_slaves = 2;
 	} else {
@@ -3508,7 +3629,7 @@ static int gbe_probe(struct netcp_device *netcp_device, struct device *dev,
 
 		dev_dbg(dev, "ss_version: 0x%08x\n", gbe_dev->ss_version);
 
-		if (gbe_dev->ss_version == GBE_SS_VERSION_14)
+		if (IS_SS_ID_VER_14(gbe_dev))
 			ret = set_gbe_ethss14_priv(gbe_dev, node);
 		else if (IS_SS_ID_MU(gbe_dev))
 			ret = set_gbenu_ethss_priv(gbe_dev, node);
@@ -3606,7 +3727,7 @@ static int gbe_probe(struct netcp_device *netcp_device, struct device *dev,
 
 	spin_lock_bh(&gbe_dev->hw_stats_lock);
 	for (i = 0; i < gbe_dev->num_stats_mods; i++) {
-		if (gbe_dev->ss_version == GBE_SS_VERSION_14)
+		if (IS_SS_ID_VER_14(gbe_dev))
 			gbe_reset_mod_stats_ver14(gbe_dev, i);
 		else
 			gbe_reset_mod_stats(gbe_dev, i);
diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
index b919e89a9b93..750eaa53bf0c 100644
--- a/drivers/net/geneve.c
+++ b/drivers/net/geneve.c
@@ -36,6 +36,8 @@ MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN");
 
 #define GENEVE_VER 0
 #define GENEVE_BASE_HLEN (sizeof(struct udphdr) + sizeof(struct genevehdr))
+#define GENEVE_IPV4_HLEN (ETH_HLEN + sizeof(struct iphdr) + GENEVE_BASE_HLEN)
+#define GENEVE_IPV6_HLEN (ETH_HLEN + sizeof(struct ipv6hdr) + GENEVE_BASE_HLEN)
 
 /* per-network namespace private data for this module */
 struct geneve_net {
@@ -826,8 +828,8 @@ static int geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 		return PTR_ERR(rt);
 
 	if (skb_dst(skb)) {
-		int mtu = dst_mtu(&rt->dst) - sizeof(struct iphdr) -
-			  GENEVE_BASE_HLEN - info->options_len - 14;
+		int mtu = dst_mtu(&rt->dst) - GENEVE_IPV4_HLEN -
+			  info->options_len;
 
 		skb_dst_update_pmtu(skb, mtu);
 	}
@@ -872,8 +874,7 @@ static int geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 		return PTR_ERR(dst);
 
 	if (skb_dst(skb)) {
-		int mtu = dst_mtu(dst) - sizeof(struct ipv6hdr) -
-			  GENEVE_BASE_HLEN - info->options_len - 14;
+		int mtu = dst_mtu(dst) - GENEVE_IPV6_HLEN - info->options_len;
 
 		skb_dst_update_pmtu(skb, mtu);
 	}
@@ -941,11 +942,10 @@ static netdev_tx_t geneve_xmit(struct sk_buff *skb, struct net_device *dev)
 
 static int geneve_change_mtu(struct net_device *dev, int new_mtu)
 {
-	/* Only possible if called internally, ndo_change_mtu path's new_mtu
-	 * is guaranteed to be between dev->min_mtu and dev->max_mtu.
-	 */
 	if (new_mtu > dev->max_mtu)
 		new_mtu = dev->max_mtu;
+	else if (new_mtu < dev->min_mtu)
+		new_mtu = dev->min_mtu;
 
 	dev->mtu = new_mtu;
 	return 0;
@@ -1261,7 +1261,7 @@ static int geneve_nl2info(struct nlattr *tb[], struct nlattr *data[],
 	}
 
 	if (data[IFLA_GENEVE_REMOTE6]) {
- #if IS_ENABLED(CONFIG_IPV6)
+#if IS_ENABLED(CONFIG_IPV6)
 		if (changelink && (ip_tunnel_info_af(info) == AF_INET)) {
 			attrtype = IFLA_GENEVE_REMOTE6;
 			goto change_notsup;
@@ -1387,6 +1387,48 @@ static int geneve_nl2info(struct nlattr *tb[], struct nlattr *data[],
 	return -EOPNOTSUPP;
 }
 
+static void geneve_link_config(struct net_device *dev,
+			       struct ip_tunnel_info *info, struct nlattr *tb[])
+{
+	struct geneve_dev *geneve = netdev_priv(dev);
+	int ldev_mtu = 0;
+
+	if (tb[IFLA_MTU]) {
+		geneve_change_mtu(dev, nla_get_u32(tb[IFLA_MTU]));
+		return;
+	}
+
+	switch (ip_tunnel_info_af(info)) {
+	case AF_INET: {
+		struct flowi4 fl4 = { .daddr = info->key.u.ipv4.dst };
+		struct rtable *rt = ip_route_output_key(geneve->net, &fl4);
+
+		if (!IS_ERR(rt) && rt->dst.dev) {
+			ldev_mtu = rt->dst.dev->mtu - GENEVE_IPV4_HLEN;
+			ip_rt_put(rt);
+		}
+		break;
+	}
+#if IS_ENABLED(CONFIG_IPV6)
+	case AF_INET6: {
+		struct rt6_info *rt = rt6_lookup(geneve->net,
+						 &info->key.u.ipv6.dst, NULL, 0,
+						 NULL, 0);
+
+		if (rt && rt->dst.dev)
+			ldev_mtu = rt->dst.dev->mtu - GENEVE_IPV6_HLEN;
+		ip6_rt_put(rt);
+		break;
+	}
+#endif
+	}
+
+	if (ldev_mtu <= 0)
+		return;
+
+	geneve_change_mtu(dev, ldev_mtu - info->options_len);
+}
+
 static int geneve_newlink(struct net *net, struct net_device *dev,
 			  struct nlattr *tb[], struct nlattr *data[],
 			  struct netlink_ext_ack *extack)
@@ -1402,8 +1444,14 @@ static int geneve_newlink(struct net *net, struct net_device *dev,
 	if (err)
 		return err;
 
-	return geneve_configure(net, dev, extack, &info, metadata,
-				use_udp6_rx_checksums);
+	err = geneve_configure(net, dev, extack, &info, metadata,
+			       use_udp6_rx_checksums);
+	if (err)
+		return err;
+
+	geneve_link_config(dev, &info, tb);
+
+	return 0;
 }
 
 /* Quiesces the geneve device data path for both TX and RX.
@@ -1477,8 +1525,10 @@ static int geneve_changelink(struct net_device *dev, struct nlattr *tb[],
 	if (err)
 		return err;
 
-	if (!geneve_dst_addr_equal(&geneve->info, &info))
+	if (!geneve_dst_addr_equal(&geneve->info, &info)) {
 		dst_cache_reset(&info.dst_cache);
+		geneve_link_config(dev, &info, tb);
+	}
 
 	geneve_quiesce(geneve, &gs4, &gs6);
 	geneve->info = info;
diff --git a/drivers/net/hamradio/bpqether.c b/drivers/net/hamradio/bpqether.c
index dfabbae72efd..f347fd9c5b28 100644
--- a/drivers/net/hamradio/bpqether.c
+++ b/drivers/net/hamradio/bpqether.c
@@ -449,20 +449,6 @@ static const struct seq_operations bpq_seqops = {
 	.show = bpq_seq_show,
 };
 
-static int bpq_info_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &bpq_seqops);
-}
-
-static const struct file_operations bpq_info_fops = {
-	.owner = THIS_MODULE,
-	.open = bpq_info_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = seq_release,
-};
-
-
 /* ------------------------------------------------------------------------ */
 
 static const struct net_device_ops bpq_netdev_ops = {
@@ -590,7 +576,7 @@ static int bpq_device_event(struct notifier_block *this,
 static int __init bpq_init_driver(void)
 {
 #ifdef CONFIG_PROC_FS
-	if (!proc_create("bpqether", 0444, init_net.proc_net, &bpq_info_fops)) {
+	if (!proc_create_seq("bpqether", 0444, init_net.proc_net, &bpq_seqops)) {
 		printk(KERN_ERR
 			"bpq: cannot create /proc/net/bpqether entry.\n");
 		return -ENOENT;
diff --git a/drivers/net/hamradio/mkiss.c b/drivers/net/hamradio/mkiss.c
index c180b480f8ef..13e4c1eff353 100644
--- a/drivers/net/hamradio/mkiss.c
+++ b/drivers/net/hamradio/mkiss.c
@@ -217,7 +217,7 @@ static int kiss_esc_crc(unsigned char *s, unsigned char *d, unsigned short crc,
 			c = *s++;
 		else if (len > 1)
 			c = crc >> 8;
-		else if (len > 0)
+		else
 			c = crc & 0xff;
 
 		len--;
diff --git a/drivers/net/hamradio/scc.c b/drivers/net/hamradio/scc.c
index 3de272959090..6c03932d8a6b 100644
--- a/drivers/net/hamradio/scc.c
+++ b/drivers/net/hamradio/scc.c
@@ -2084,21 +2084,6 @@ static const struct seq_operations scc_net_seq_ops = {
 	.stop   = scc_net_seq_stop,
 	.show   = scc_net_seq_show,
 };
-
-
-static int scc_net_seq_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &scc_net_seq_ops);
-}
-
-static const struct file_operations scc_net_seq_fops = {
-	.owner	 = THIS_MODULE,
-	.open	 = scc_net_seq_open,
-	.read	 = seq_read,
-	.llseek	 = seq_lseek,
-	.release = seq_release_private,
-};
-
 #endif /* CONFIG_PROC_FS */
 
  
@@ -2122,7 +2107,7 @@ static int __init scc_init_driver (void)
 	}
 	rtnl_unlock();
 
-	proc_create("z8530drv", 0, init_net.proc_net, &scc_net_seq_fops);
+	proc_create_seq("z8530drv", 0, init_net.proc_net, &scc_net_seq_ops);
 
 	return 0;
 }
diff --git a/drivers/net/hamradio/yam.c b/drivers/net/hamradio/yam.c
index 83034eb7ed4f..16ec7af6ab7b 100644
--- a/drivers/net/hamradio/yam.c
+++ b/drivers/net/hamradio/yam.c
@@ -841,20 +841,6 @@ static const struct seq_operations yam_seqops = {
 	.stop = yam_seq_stop,
 	.show = yam_seq_show,
 };
-
-static int yam_info_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &yam_seqops);
-}
-
-static const struct file_operations yam_info_fops = {
-	.owner = THIS_MODULE,
-	.open = yam_info_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = seq_release,
-};
-
 #endif
 
 
@@ -1168,7 +1154,7 @@ static int __init yam_init_driver(void)
 	yam_timer.expires = jiffies + HZ / 100;
 	add_timer(&yam_timer);
 
-	proc_create("yam", 0444, init_net.proc_net, &yam_info_fops);
+	proc_create_seq("yam", 0444, init_net.proc_net, &yam_seqops);
 	return 0;
  error:
 	while (--i >= 0) {
diff --git a/drivers/net/hippi/rrunner.c b/drivers/net/hippi/rrunner.c
index 1ab97d99b9ba..f41116488079 100644
--- a/drivers/net/hippi/rrunner.c
+++ b/drivers/net/hippi/rrunner.c
@@ -867,7 +867,7 @@ static u32 rr_handle_event(struct net_device *dev, u32 prodidx, u32 eidx)
 			       dev->name);
 			goto drop;
 		case E_FRM_ERR:
-			printk(KERN_WARNING "%s: Framming Error\n",
+			printk(KERN_WARNING "%s: Framing Error\n",
 			       dev->name);
 			goto drop;
 		case E_FLG_SYN_ERR:
diff --git a/drivers/net/hyperv/Kconfig b/drivers/net/hyperv/Kconfig
index 936968d23559..23a2d145813a 100644
--- a/drivers/net/hyperv/Kconfig
+++ b/drivers/net/hyperv/Kconfig
@@ -1,5 +1,7 @@
 config HYPERV_NET
 	tristate "Microsoft Hyper-V virtual network driver"
 	depends on HYPERV
+	select UCS2_STRING
+	select FAILOVER
 	help
 	  Select this option to enable the Hyper-V virtual network driver.
diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h
index 960f06141472..23304aca25f9 100644
--- a/drivers/net/hyperv/hyperv_net.h
+++ b/drivers/net/hyperv/hyperv_net.h
@@ -110,7 +110,7 @@ struct ndis_recv_scale_param { /* NDIS_RECEIVE_SCALE_PARAMETERS */
 	u16 hashkey_size;
 
 	/* The offset of the secret key from the beginning of this structure */
-	u32 kashkey_offset;
+	u32 hashkey_offset;
 
 	u32 processor_masks_offset;
 	u32 num_processor_masks;
@@ -189,7 +189,6 @@ struct netvsc_device;
 struct net_device_context;
 
 extern u32 netvsc_ring_bytes;
-extern struct reciprocal_value netvsc_ring_reciprocal;
 
 struct netvsc_device *netvsc_device_add(struct hv_device *device,
 					const struct netvsc_device_info *info);
@@ -237,6 +236,8 @@ void netvsc_switch_datapath(struct net_device *nv_dev, bool vf);
 #define NVSP_PROTOCOL_VERSION_2		0x30002
 #define NVSP_PROTOCOL_VERSION_4		0x40000
 #define NVSP_PROTOCOL_VERSION_5		0x50000
+#define NVSP_PROTOCOL_VERSION_6		0x60000
+#define NVSP_PROTOCOL_VERSION_61	0x60001
 
 enum {
 	NVSP_MSG_TYPE_NONE = 0,
@@ -308,6 +309,12 @@ enum {
 	NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE,
 
 	NVSP_MSG5_MAX = NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE,
+
+	/* Version 6 messages */
+	NVSP_MSG6_TYPE_PD_API,
+	NVSP_MSG6_TYPE_PD_POST_BATCH,
+
+	NVSP_MSG6_MAX = NVSP_MSG6_TYPE_PD_POST_BATCH
 };
 
 enum {
@@ -619,12 +626,168 @@ union nvsp_5_message_uber {
 	struct nvsp_5_send_indirect_table send_table;
 } __packed;
 
+enum nvsp_6_pd_api_op {
+	PD_API_OP_CONFIG = 1,
+	PD_API_OP_SW_DATAPATH, /* Switch Datapath */
+	PD_API_OP_OPEN_PROVIDER,
+	PD_API_OP_CLOSE_PROVIDER,
+	PD_API_OP_CREATE_QUEUE,
+	PD_API_OP_FLUSH_QUEUE,
+	PD_API_OP_FREE_QUEUE,
+	PD_API_OP_ALLOC_COM_BUF, /* Allocate Common Buffer */
+	PD_API_OP_FREE_COM_BUF, /* Free Common Buffer */
+	PD_API_OP_MAX
+};
+
+struct grp_affinity {
+	u64 mask;
+	u16 grp;
+	u16 reserved[3];
+} __packed;
+
+struct nvsp_6_pd_api_req {
+	u32 op;
+
+	union {
+		/* MMIO information is sent from the VM to VSP */
+		struct __packed {
+			u64 mmio_pa; /* MMIO Physical Address */
+			u32 mmio_len;
+
+			/* Number of PD queues a VM can support */
+			u16 num_subchn;
+		} config;
+
+		/* Switch Datapath */
+		struct __packed {
+			/* Host Datapath Is PacketDirect */
+			u8 host_dpath_is_pd;
+
+			/* Guest PacketDirect Is Enabled */
+			u8 guest_pd_enabled;
+		} sw_dpath;
+
+		/* Open Provider*/
+		struct __packed {
+			u32 prov_id; /* Provider id */
+			u32 flag;
+		} open_prov;
+
+		/* Close Provider */
+		struct __packed {
+			u32 prov_id;
+		} cls_prov;
+
+		/* Create Queue*/
+		struct __packed {
+			u32 prov_id;
+			u16 q_id;
+			u16 q_size;
+			u8 is_recv_q;
+			u8 is_rss_q;
+			u32 recv_data_len;
+			struct grp_affinity affy;
+		} cr_q;
+
+		/* Delete Queue*/
+		struct __packed {
+			u32 prov_id;
+			u16 q_id;
+		} del_q;
+
+		/* Flush Queue */
+		struct __packed {
+			u32 prov_id;
+			u16 q_id;
+		} flush_q;
+
+		/* Allocate Common Buffer */
+		struct __packed {
+			u32 len;
+			u32 pf_node; /* Preferred Node */
+			u16 region_id;
+		} alloc_com_buf;
+
+		/* Free Common Buffer */
+		struct __packed {
+			u32 len;
+			u64 pa; /* Physical Address */
+			u32 pf_node; /* Preferred Node */
+			u16 region_id;
+			u8 cache_type;
+		} free_com_buf;
+	} __packed;
+} __packed;
+
+struct nvsp_6_pd_api_comp {
+	u32 op;
+	u32 status;
+
+	union {
+		struct __packed {
+			/* actual number of PD queues allocated to the VM */
+			u16 num_pd_q;
+
+			/* Num Receive Rss PD Queues */
+			u8 num_rss_q;
+
+			u8 is_supported; /* Is supported by VSP */
+			u8 is_enabled; /* Is enabled by VSP */
+		} config;
+
+		/* Open Provider */
+		struct __packed {
+			u32 prov_id;
+		} open_prov;
+
+		/* Create Queue */
+		struct __packed {
+			u32 prov_id;
+			u16 q_id;
+			u16 q_size;
+			u32 recv_data_len;
+			struct grp_affinity affy;
+		} cr_q;
+
+		/* Allocate Common Buffer */
+		struct __packed {
+			u64 pa; /* Physical Address */
+			u32 len;
+			u32 pf_node; /* Preferred Node */
+			u16 region_id;
+			u8 cache_type;
+		} alloc_com_buf;
+	} __packed;
+} __packed;
+
+struct nvsp_6_pd_buf {
+	u32 region_offset;
+	u16 region_id;
+	u16 is_partial:1;
+	u16 reserved:15;
+} __packed;
+
+struct nvsp_6_pd_batch_msg {
+	struct nvsp_message_header hdr;
+	u16 count;
+	u16 guest2host:1;
+	u16 is_recv:1;
+	u16 reserved:14;
+	struct nvsp_6_pd_buf pd_buf[0];
+} __packed;
+
+union nvsp_6_message_uber {
+	struct nvsp_6_pd_api_req pd_req;
+	struct nvsp_6_pd_api_comp pd_comp;
+} __packed;
+
 union nvsp_all_messages {
 	union nvsp_message_init_uber init_msg;
 	union nvsp_1_message_uber v1_msg;
 	union nvsp_2_message_uber v2_msg;
 	union nvsp_4_message_uber v4_msg;
 	union nvsp_5_message_uber v5_msg;
+	union nvsp_6_message_uber v6_msg;
 } __packed;
 
 /* ALL Messages */
@@ -768,6 +931,8 @@ struct net_device_context {
 	u32 vf_alloc;
 	/* Serial number of the VF to team with */
 	u32 vf_serial;
+
+	struct failover *failover;
 };
 
 /* Per channel data */
diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c
index 04f611e6f678..5d5bd513847f 100644
--- a/drivers/net/hyperv/netvsc.c
+++ b/drivers/net/hyperv/netvsc.c
@@ -31,7 +31,6 @@
 #include <linux/vmalloc.h>
 #include <linux/rtnetlink.h>
 #include <linux/prefetch.h>
-#include <linux/reciprocal_div.h>
 
 #include <asm/sync_bitops.h>
 
@@ -525,7 +524,8 @@ static int netvsc_connect_vsp(struct hv_device *device,
 	struct net_device *ndev = hv_get_drvdata(device);
 	static const u32 ver_list[] = {
 		NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2,
-		NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5
+		NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5,
+		NVSP_PROTOCOL_VERSION_6, NVSP_PROTOCOL_VERSION_61
 	};
 	struct nvsp_message *init_packet;
 	int ndis_version, i, ret;
@@ -634,33 +634,20 @@ void netvsc_device_remove(struct hv_device *device)
 #define RING_AVAIL_PERCENT_HIWATER 20
 #define RING_AVAIL_PERCENT_LOWATER 10
 
-/*
- * Get the percentage of available bytes to write in the ring.
- * The return value is in range from 0 to 100.
- */
-static u32 hv_ringbuf_avail_percent(const struct hv_ring_buffer_info *ring_info)
-{
-	u32 avail_write = hv_get_bytes_to_write(ring_info);
-
-	return reciprocal_divide(avail_write  * 100, netvsc_ring_reciprocal);
-}
-
 static inline void netvsc_free_send_slot(struct netvsc_device *net_device,
 					 u32 index)
 {
 	sync_change_bit(index, net_device->send_section_map);
 }
 
-static void netvsc_send_tx_complete(struct netvsc_device *net_device,
-				    struct vmbus_channel *incoming_channel,
-				    struct hv_device *device,
+static void netvsc_send_tx_complete(struct net_device *ndev,
+				    struct netvsc_device *net_device,
+				    struct vmbus_channel *channel,
 				    const struct vmpacket_descriptor *desc,
 				    int budget)
 {
 	struct sk_buff *skb = (struct sk_buff *)(unsigned long)desc->trans_id;
-	struct net_device *ndev = hv_get_drvdata(device);
 	struct net_device_context *ndev_ctx = netdev_priv(ndev);
-	struct vmbus_channel *channel = device->channel;
 	u16 q_idx = 0;
 	int queue_sends;
 
@@ -674,7 +661,6 @@ static void netvsc_send_tx_complete(struct netvsc_device *net_device,
 		if (send_index != NETVSC_INVALID_INDEX)
 			netvsc_free_send_slot(net_device, send_index);
 		q_idx = packet->q_idx;
-		channel = incoming_channel;
 
 		tx_stats = &net_device->chan_table[q_idx].tx_stats;
 
@@ -696,22 +682,21 @@ static void netvsc_send_tx_complete(struct netvsc_device *net_device,
 		struct netdev_queue *txq = netdev_get_tx_queue(ndev, q_idx);
 
 		if (netif_tx_queue_stopped(txq) &&
-		    (hv_ringbuf_avail_percent(&channel->outbound) > RING_AVAIL_PERCENT_HIWATER ||
-		     queue_sends < 1)) {
+		    (hv_get_avail_to_write_percent(&channel->outbound) >
+		     RING_AVAIL_PERCENT_HIWATER || queue_sends < 1)) {
 			netif_tx_wake_queue(txq);
 			ndev_ctx->eth_stats.wake_queue++;
 		}
 	}
 }
 
-static void netvsc_send_completion(struct netvsc_device *net_device,
+static void netvsc_send_completion(struct net_device *ndev,
+				   struct netvsc_device *net_device,
 				   struct vmbus_channel *incoming_channel,
-				   struct hv_device *device,
 				   const struct vmpacket_descriptor *desc,
 				   int budget)
 {
-	struct nvsp_message *nvsp_packet = hv_pkt_data(desc);
-	struct net_device *ndev = hv_get_drvdata(device);
+	const struct nvsp_message *nvsp_packet = hv_pkt_data(desc);
 
 	switch (nvsp_packet->hdr.msg_type) {
 	case NVSP_MSG_TYPE_INIT_COMPLETE:
@@ -725,8 +710,8 @@ static void netvsc_send_completion(struct netvsc_device *net_device,
 		break;
 
 	case NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE:
-		netvsc_send_tx_complete(net_device, incoming_channel,
-					device, desc, budget);
+		netvsc_send_tx_complete(ndev, net_device, incoming_channel,
+					desc, budget);
 		break;
 
 	default:
@@ -805,7 +790,7 @@ static inline int netvsc_send_pkt(
 	struct netdev_queue *txq = netdev_get_tx_queue(ndev, packet->q_idx);
 	u64 req_id;
 	int ret;
-	u32 ring_avail = hv_ringbuf_avail_percent(&out_channel->outbound);
+	u32 ring_avail = hv_get_avail_to_write_percent(&out_channel->outbound);
 
 	nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;
 	if (skb)
@@ -1091,12 +1076,11 @@ static void enq_receive_complete(struct net_device *ndev,
 
 static int netvsc_receive(struct net_device *ndev,
 			  struct netvsc_device *net_device,
-			  struct net_device_context *net_device_ctx,
-			  struct hv_device *device,
 			  struct vmbus_channel *channel,
 			  const struct vmpacket_descriptor *desc,
-			  struct nvsp_message *nvsp)
+			  const struct nvsp_message *nvsp)
 {
+	struct net_device_context *net_device_ctx = netdev_priv(ndev);
 	const struct vmtransfer_page_packet_header *vmxferpage_packet
 		= container_of(desc, const struct vmtransfer_page_packet_header, d);
 	u16 q_idx = channel->offermsg.offer.sub_channel_index;
@@ -1157,13 +1141,12 @@ static int netvsc_receive(struct net_device *ndev,
 	return count;
 }
 
-static void netvsc_send_table(struct hv_device *hdev,
-			      struct nvsp_message *nvmsg)
+static void netvsc_send_table(struct net_device *ndev,
+			      const struct nvsp_message *nvmsg)
 {
-	struct net_device *ndev = hv_get_drvdata(hdev);
 	struct net_device_context *net_device_ctx = netdev_priv(ndev);
-	int i;
 	u32 count, *tab;
+	int i;
 
 	count = nvmsg->msg.v5_msg.send_table.count;
 	if (count != VRSS_SEND_TAB_SIZE) {
@@ -1178,24 +1161,25 @@ static void netvsc_send_table(struct hv_device *hdev,
 		net_device_ctx->tx_table[i] = tab[i];
 }
 
-static void netvsc_send_vf(struct net_device_context *net_device_ctx,
-			   struct nvsp_message *nvmsg)
+static void netvsc_send_vf(struct net_device *ndev,
+			   const struct nvsp_message *nvmsg)
 {
+	struct net_device_context *net_device_ctx = netdev_priv(ndev);
+
 	net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated;
 	net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial;
 }
 
-static inline void netvsc_receive_inband(struct hv_device *hdev,
-				 struct net_device_context *net_device_ctx,
-				 struct nvsp_message *nvmsg)
+static  void netvsc_receive_inband(struct net_device *ndev,
+				   const struct nvsp_message *nvmsg)
 {
 	switch (nvmsg->hdr.msg_type) {
 	case NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE:
-		netvsc_send_table(hdev, nvmsg);
+		netvsc_send_table(ndev, nvmsg);
 		break;
 
 	case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION:
-		netvsc_send_vf(net_device_ctx, nvmsg);
+		netvsc_send_vf(ndev, nvmsg);
 		break;
 	}
 }
@@ -1207,24 +1191,23 @@ static int netvsc_process_raw_pkt(struct hv_device *device,
 				  const struct vmpacket_descriptor *desc,
 				  int budget)
 {
-	struct net_device_context *net_device_ctx = netdev_priv(ndev);
-	struct nvsp_message *nvmsg = hv_pkt_data(desc);
+	const struct nvsp_message *nvmsg = hv_pkt_data(desc);
 
 	trace_nvsp_recv(ndev, channel, nvmsg);
 
 	switch (desc->type) {
 	case VM_PKT_COMP:
-		netvsc_send_completion(net_device, channel, device,
+		netvsc_send_completion(ndev, net_device, channel,
 				       desc, budget);
 		break;
 
 	case VM_PKT_DATA_USING_XFER_PAGES:
-		return netvsc_receive(ndev, net_device, net_device_ctx,
-				      device, channel, desc, nvmsg);
+		return netvsc_receive(ndev, net_device, channel,
+				      desc, nvmsg);
 		break;
 
 	case VM_PKT_DATA_INBAND:
-		netvsc_receive_inband(device, net_device_ctx, nvmsg);
+		netvsc_receive_inband(ndev, nvmsg);
 		break;
 
 	default:
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index da07ccdf84bf..fb0a01a02f79 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -35,7 +35,6 @@
 #include <linux/slab.h>
 #include <linux/rtnetlink.h>
 #include <linux/netpoll.h>
-#include <linux/reciprocal_div.h>
 
 #include <net/arp.h>
 #include <net/route.h>
@@ -43,6 +42,7 @@
 #include <net/pkt_sched.h>
 #include <net/checksum.h>
 #include <net/ip6_checksum.h>
+#include <net/failover.h>
 
 #include "hyperv_net.h"
 
@@ -58,7 +58,6 @@ static unsigned int ring_size __ro_after_init = 128;
 module_param(ring_size, uint, 0444);
 MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)");
 unsigned int netvsc_ring_bytes __ro_after_init;
-struct reciprocal_value netvsc_ring_reciprocal __ro_after_init;
 
 static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE |
 				NETIF_MSG_LINK | NETIF_MSG_IFUP |
@@ -1618,8 +1617,24 @@ static int netvsc_set_ringparam(struct net_device *ndev,
 	return ret;
 }
 
+static u32 netvsc_get_msglevel(struct net_device *ndev)
+{
+	struct net_device_context *ndev_ctx = netdev_priv(ndev);
+
+	return ndev_ctx->msg_enable;
+}
+
+static void netvsc_set_msglevel(struct net_device *ndev, u32 val)
+{
+	struct net_device_context *ndev_ctx = netdev_priv(ndev);
+
+	ndev_ctx->msg_enable = val;
+}
+
 static const struct ethtool_ops ethtool_ops = {
 	.get_drvinfo	= netvsc_get_drvinfo,
+	.get_msglevel	= netvsc_get_msglevel,
+	.set_msglevel	= netvsc_set_msglevel,
 	.get_link	= ethtool_op_get_link,
 	.get_ethtool_stats = netvsc_get_ethtool_stats,
 	.get_sset_count = netvsc_get_sset_count,
@@ -1763,46 +1778,6 @@ static void netvsc_link_change(struct work_struct *w)
 	rtnl_unlock();
 }
 
-static struct net_device *get_netvsc_bymac(const u8 *mac)
-{
-	struct net_device *dev;
-
-	ASSERT_RTNL();
-
-	for_each_netdev(&init_net, dev) {
-		if (dev->netdev_ops != &device_ops)
-			continue;	/* not a netvsc device */
-
-		if (ether_addr_equal(mac, dev->perm_addr))
-			return dev;
-	}
-
-	return NULL;
-}
-
-static struct net_device *get_netvsc_byref(struct net_device *vf_netdev)
-{
-	struct net_device *dev;
-
-	ASSERT_RTNL();
-
-	for_each_netdev(&init_net, dev) {
-		struct net_device_context *net_device_ctx;
-
-		if (dev->netdev_ops != &device_ops)
-			continue;	/* not a netvsc device */
-
-		net_device_ctx = netdev_priv(dev);
-		if (!rtnl_dereference(net_device_ctx->nvdev))
-			continue;	/* device is removed */
-
-		if (rtnl_dereference(net_device_ctx->vf_netdev) == vf_netdev)
-			return dev;	/* a match */
-	}
-
-	return NULL;
-}
-
 /* Called when VF is injecting data into network stack.
  * Change the associated network device from VF to netvsc.
  * note: already called with rcu_read_lock
@@ -1825,46 +1800,6 @@ static rx_handler_result_t netvsc_vf_handle_frame(struct sk_buff **pskb)
 	return RX_HANDLER_ANOTHER;
 }
 
-static int netvsc_vf_join(struct net_device *vf_netdev,
-			  struct net_device *ndev)
-{
-	struct net_device_context *ndev_ctx = netdev_priv(ndev);
-	int ret;
-
-	ret = netdev_rx_handler_register(vf_netdev,
-					 netvsc_vf_handle_frame, ndev);
-	if (ret != 0) {
-		netdev_err(vf_netdev,
-			   "can not register netvsc VF receive handler (err = %d)\n",
-			   ret);
-		goto rx_handler_failed;
-	}
-
-	ret = netdev_master_upper_dev_link(vf_netdev, ndev,
-					   NULL, NULL, NULL);
-	if (ret != 0) {
-		netdev_err(vf_netdev,
-			   "can not set master device %s (err = %d)\n",
-			   ndev->name, ret);
-		goto upper_link_failed;
-	}
-
-	/* set slave flag before open to prevent IPv6 addrconf */
-	vf_netdev->flags |= IFF_SLAVE;
-
-	schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT);
-
-	call_netdevice_notifiers(NETDEV_JOIN, vf_netdev);
-
-	netdev_info(vf_netdev, "joined to %s\n", ndev->name);
-	return 0;
-
-upper_link_failed:
-	netdev_rx_handler_unregister(vf_netdev);
-rx_handler_failed:
-	return ret;
-}
-
 static void __netvsc_vf_setup(struct net_device *ndev,
 			      struct net_device *vf_netdev)
 {
@@ -1915,85 +1850,95 @@ static void netvsc_vf_setup(struct work_struct *w)
 	rtnl_unlock();
 }
 
-static int netvsc_register_vf(struct net_device *vf_netdev)
+static int netvsc_pre_register_vf(struct net_device *vf_netdev,
+				  struct net_device *ndev)
 {
-	struct net_device *ndev;
 	struct net_device_context *net_device_ctx;
 	struct netvsc_device *netvsc_dev;
 
-	if (vf_netdev->addr_len != ETH_ALEN)
-		return NOTIFY_DONE;
-
-	/*
-	 * We will use the MAC address to locate the synthetic interface to
-	 * associate with the VF interface. If we don't find a matching
-	 * synthetic interface, move on.
-	 */
-	ndev = get_netvsc_bymac(vf_netdev->perm_addr);
-	if (!ndev)
-		return NOTIFY_DONE;
-
 	net_device_ctx = netdev_priv(ndev);
 	netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
 	if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev))
-		return NOTIFY_DONE;
+		return -ENODEV;
 
-	if (netvsc_vf_join(vf_netdev, ndev) != 0)
-		return NOTIFY_DONE;
+	return 0;
+}
 
-	netdev_info(ndev, "VF registering: %s\n", vf_netdev->name);
+static int netvsc_register_vf(struct net_device *vf_netdev,
+			      struct net_device *ndev)
+{
+	struct net_device_context *ndev_ctx = netdev_priv(ndev);
+
+	/* set slave flag before open to prevent IPv6 addrconf */
+	vf_netdev->flags |= IFF_SLAVE;
+
+	schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT);
+
+	call_netdevice_notifiers(NETDEV_JOIN, vf_netdev);
+
+	netdev_info(vf_netdev, "joined to %s\n", ndev->name);
 
 	dev_hold(vf_netdev);
-	rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
-	return NOTIFY_OK;
+	rcu_assign_pointer(ndev_ctx->vf_netdev, vf_netdev);
+
+	return 0;
 }
 
 /* VF up/down change detected, schedule to change data path */
-static int netvsc_vf_changed(struct net_device *vf_netdev)
+static int netvsc_vf_changed(struct net_device *vf_netdev,
+			     struct net_device *ndev)
 {
 	struct net_device_context *net_device_ctx;
 	struct netvsc_device *netvsc_dev;
-	struct net_device *ndev;
 	bool vf_is_up = netif_running(vf_netdev);
 
-	ndev = get_netvsc_byref(vf_netdev);
-	if (!ndev)
-		return NOTIFY_DONE;
-
 	net_device_ctx = netdev_priv(ndev);
 	netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
 	if (!netvsc_dev)
-		return NOTIFY_DONE;
+		return -ENODEV;
 
 	netvsc_switch_datapath(ndev, vf_is_up);
 	netdev_info(ndev, "Data path switched %s VF: %s\n",
 		    vf_is_up ? "to" : "from", vf_netdev->name);
 
-	return NOTIFY_OK;
+	return 0;
 }
 
-static int netvsc_unregister_vf(struct net_device *vf_netdev)
+static int netvsc_pre_unregister_vf(struct net_device *vf_netdev,
+				    struct net_device *ndev)
 {
-	struct net_device *ndev;
 	struct net_device_context *net_device_ctx;
 
-	ndev = get_netvsc_byref(vf_netdev);
-	if (!ndev)
-		return NOTIFY_DONE;
-
 	net_device_ctx = netdev_priv(ndev);
 	cancel_delayed_work_sync(&net_device_ctx->vf_takeover);
 
+	return 0;
+}
+
+static int netvsc_unregister_vf(struct net_device *vf_netdev,
+				struct net_device *ndev)
+{
+	struct net_device_context *net_device_ctx;
+
+	net_device_ctx = netdev_priv(ndev);
+
 	netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
 
-	netdev_rx_handler_unregister(vf_netdev);
-	netdev_upper_dev_unlink(vf_netdev, ndev);
 	RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
 	dev_put(vf_netdev);
 
-	return NOTIFY_OK;
+	return 0;
 }
 
+static struct failover_ops netvsc_failover_ops = {
+	.slave_pre_register	= netvsc_pre_register_vf,
+	.slave_register		= netvsc_register_vf,
+	.slave_pre_unregister	= netvsc_pre_unregister_vf,
+	.slave_unregister	= netvsc_unregister_vf,
+	.slave_link_change	= netvsc_vf_changed,
+	.slave_handle_frame	= netvsc_vf_handle_frame,
+};
+
 static int netvsc_probe(struct hv_device *dev,
 			const struct hv_vmbus_device_id *dev_id)
 {
@@ -2083,8 +2028,14 @@ static int netvsc_probe(struct hv_device *dev,
 		goto register_failed;
 	}
 
+	net_device_ctx->failover = failover_register(net, &netvsc_failover_ops);
+	if (IS_ERR(net_device_ctx->failover))
+		goto err_failover;
+
 	return ret;
 
+err_failover:
+	unregister_netdev(net);
 register_failed:
 	rndis_filter_device_remove(dev, nvdev);
 rndis_failed:
@@ -2125,13 +2076,15 @@ static int netvsc_remove(struct hv_device *dev)
 	rtnl_lock();
 	vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
 	if (vf_netdev)
-		netvsc_unregister_vf(vf_netdev);
+		failover_slave_unregister(vf_netdev);
 
 	if (nvdev)
 		rndis_filter_device_remove(dev, nvdev);
 
 	unregister_netdevice(net);
 
+	failover_unregister(ndev_ctx->failover);
+
 	rtnl_unlock();
 	rcu_read_unlock();
 
@@ -2158,54 +2111,8 @@ static struct  hv_driver netvsc_drv = {
 	.remove = netvsc_remove,
 };
 
-/*
- * On Hyper-V, every VF interface is matched with a corresponding
- * synthetic interface. The synthetic interface is presented first
- * to the guest. When the corresponding VF instance is registered,
- * we will take care of switching the data path.
- */
-static int netvsc_netdev_event(struct notifier_block *this,
-			       unsigned long event, void *ptr)
-{
-	struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
-
-	/* Skip our own events */
-	if (event_dev->netdev_ops == &device_ops)
-		return NOTIFY_DONE;
-
-	/* Avoid non-Ethernet type devices */
-	if (event_dev->type != ARPHRD_ETHER)
-		return NOTIFY_DONE;
-
-	/* Avoid Vlan dev with same MAC registering as VF */
-	if (is_vlan_dev(event_dev))
-		return NOTIFY_DONE;
-
-	/* Avoid Bonding master dev with same MAC registering as VF */
-	if ((event_dev->priv_flags & IFF_BONDING) &&
-	    (event_dev->flags & IFF_MASTER))
-		return NOTIFY_DONE;
-
-	switch (event) {
-	case NETDEV_REGISTER:
-		return netvsc_register_vf(event_dev);
-	case NETDEV_UNREGISTER:
-		return netvsc_unregister_vf(event_dev);
-	case NETDEV_UP:
-	case NETDEV_DOWN:
-		return netvsc_vf_changed(event_dev);
-	default:
-		return NOTIFY_DONE;
-	}
-}
-
-static struct notifier_block netvsc_netdev_notifier = {
-	.notifier_call = netvsc_netdev_event,
-};
-
 static void __exit netvsc_drv_exit(void)
 {
-	unregister_netdevice_notifier(&netvsc_netdev_notifier);
 	vmbus_driver_unregister(&netvsc_drv);
 }
 
@@ -2219,13 +2126,11 @@ static int __init netvsc_drv_init(void)
 			ring_size);
 	}
 	netvsc_ring_bytes = ring_size * PAGE_SIZE;
-	netvsc_ring_reciprocal = reciprocal_value(netvsc_ring_bytes);
 
 	ret = vmbus_driver_register(&netvsc_drv);
 	if (ret)
 		return ret;
 
-	register_netdevice_notifier(&netvsc_netdev_notifier);
 	return 0;
 }
 
diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c
index e7ca5b5f39ed..5428bb261102 100644
--- a/drivers/net/hyperv/rndis_filter.c
+++ b/drivers/net/hyperv/rndis_filter.c
@@ -29,6 +29,7 @@
 #include <linux/nls.h>
 #include <linux/vmalloc.h>
 #include <linux/rtnetlink.h>
+#include <linux/ucs2_string.h>
 
 #include "hyperv_net.h"
 #include "netvsc_trace.h"
@@ -751,7 +752,7 @@ int rndis_filter_set_rss_param(struct rndis_device *rdev,
 	rssp->indirect_tabsize = 4*ITAB_NUM;
 	rssp->indirect_taboffset = sizeof(struct ndis_recv_scale_param);
 	rssp->hashkey_size = NETVSC_HASH_KEYLEN;
-	rssp->kashkey_offset = rssp->indirect_taboffset +
+	rssp->hashkey_offset = rssp->indirect_taboffset +
 			       rssp->indirect_tabsize;
 
 	/* Set indirection table entries */
@@ -760,7 +761,7 @@ int rndis_filter_set_rss_param(struct rndis_device *rdev,
 		itab[i] = rdev->rx_table[i];
 
 	/* Set hask key values */
-	keyp = (u8 *)((unsigned long)rssp + rssp->kashkey_offset);
+	keyp = (u8 *)((unsigned long)rssp + rssp->hashkey_offset);
 	memcpy(keyp, rss_key, NETVSC_HASH_KEYLEN);
 
 	ret = rndis_filter_send_request(rdev, request);
@@ -1223,6 +1224,32 @@ static int rndis_netdev_set_hwcaps(struct rndis_device *rndis_device,
 	return ret;
 }
 
+static void rndis_get_friendly_name(struct net_device *net,
+				    struct rndis_device *rndis_device,
+				    struct netvsc_device *net_device)
+{
+	ucs2_char_t wname[256];
+	unsigned long len;
+	u8 ifalias[256];
+	u32 size;
+
+	size = sizeof(wname);
+	if (rndis_filter_query_device(rndis_device, net_device,
+				      RNDIS_OID_GEN_FRIENDLY_NAME,
+				      wname, &size) != 0)
+		return;	/* ignore if host does not support */
+
+	if (size == 0)
+		return;	/* name not set */
+
+	/* Convert Windows Unicode string to UTF-8 */
+	len = ucs2_as_utf8(ifalias, wname, sizeof(ifalias));
+
+	/* ignore the default value from host */
+	if (strcmp(ifalias, "Network Adapter") != 0)
+		dev_set_alias(net, ifalias, len);
+}
+
 struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
 				      struct netvsc_device_info *device_info)
 {
@@ -1276,6 +1303,10 @@ struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
 
 	memcpy(device_info->mac_adr, rndis_device->hw_mac_adr, ETH_ALEN);
 
+	/* Get friendly name as ifalias*/
+	if (!net->ifalias)
+		rndis_get_friendly_name(net, rndis_device, net_device);
+
 	/* Query and set hardware capabilities */
 	ret = rndis_netdev_set_hwcaps(rndis_device, net_device);
 	if (ret != 0)
diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
index 5f2897ec0edc..d345c61d476c 100644
--- a/drivers/net/ifb.c
+++ b/drivers/net/ifb.c
@@ -102,7 +102,7 @@ static void ifb_ri_tasklet(unsigned long _txp)
 		if (!skb->tc_from_ingress) {
 			dev_queue_xmit(skb);
 		} else {
-			skb_pull(skb, skb->mac_len);
+			skb_pull_rcsum(skb, skb->mac_len);
 			netif_receive_skb(skb);
 		}
 	}
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index 725f4b4afc6d..adde8fc45588 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -514,6 +514,7 @@ static int macvlan_queue_xmit(struct sk_buff *skb, struct net_device *dev)
 	const struct macvlan_dev *vlan = netdev_priv(dev);
 	const struct macvlan_port *port = vlan->port;
 	const struct macvlan_dev *dest;
+	void *accel_priv = NULL;
 
 	if (vlan->mode == MACVLAN_MODE_BRIDGE) {
 		const struct ethhdr *eth = (void *)skb->data;
@@ -533,9 +534,14 @@ static int macvlan_queue_xmit(struct sk_buff *skb, struct net_device *dev)
 		}
 	}
 
+	/* For packets that are non-multicast and not bridged we will pass
+	 * the necessary information so that the lowerdev can distinguish
+	 * the source of the packets via the accel_priv value.
+	 */
+	accel_priv = vlan->accel_priv;
 xmit_world:
 	skb->dev = vlan->lowerdev;
-	return dev_queue_xmit(skb);
+	return dev_queue_xmit_accel(skb, accel_priv);
 }
 
 static inline netdev_tx_t macvlan_netpoll_send_skb(struct macvlan_dev *vlan, struct sk_buff *skb)
@@ -552,19 +558,14 @@ static inline netdev_tx_t macvlan_netpoll_send_skb(struct macvlan_dev *vlan, str
 static netdev_tx_t macvlan_start_xmit(struct sk_buff *skb,
 				      struct net_device *dev)
 {
+	struct macvlan_dev *vlan = netdev_priv(dev);
 	unsigned int len = skb->len;
 	int ret;
-	struct macvlan_dev *vlan = netdev_priv(dev);
 
 	if (unlikely(netpoll_tx_running(dev)))
 		return macvlan_netpoll_send_skb(vlan, skb);
 
-	if (vlan->fwd_priv) {
-		skb->dev = vlan->lowerdev;
-		ret = dev_queue_xmit_accel(skb, vlan->fwd_priv);
-	} else {
-		ret = macvlan_queue_xmit(skb, dev);
-	}
+	ret = macvlan_queue_xmit(skb, dev);
 
 	if (likely(ret == NET_XMIT_SUCCESS || ret == NET_XMIT_CN)) {
 		struct vlan_pcpu_stats *pcpu_stats;
@@ -613,26 +614,27 @@ static int macvlan_open(struct net_device *dev)
 		goto hash_add;
 	}
 
-	if (lowerdev->features & NETIF_F_HW_L2FW_DOFFLOAD) {
-		vlan->fwd_priv =
-		      lowerdev->netdev_ops->ndo_dfwd_add_station(lowerdev, dev);
-
-		/* If we get a NULL pointer back, or if we get an error
-		 * then we should just fall through to the non accelerated path
-		 */
-		if (IS_ERR_OR_NULL(vlan->fwd_priv)) {
-			vlan->fwd_priv = NULL;
-		} else
-			return 0;
-	}
-
 	err = -EBUSY;
 	if (macvlan_addr_busy(vlan->port, dev->dev_addr))
 		goto out;
 
-	err = dev_uc_add(lowerdev, dev->dev_addr);
-	if (err < 0)
-		goto out;
+	/* Attempt to populate accel_priv which is used to offload the L2
+	 * forwarding requests for unicast packets.
+	 */
+	if (lowerdev->features & NETIF_F_HW_L2FW_DOFFLOAD)
+		vlan->accel_priv =
+		      lowerdev->netdev_ops->ndo_dfwd_add_station(lowerdev, dev);
+
+	/* If earlier attempt to offload failed, or accel_priv is not
+	 * populated we must add the unicast address to the lower device.
+	 */
+	if (IS_ERR_OR_NULL(vlan->accel_priv)) {
+		vlan->accel_priv = NULL;
+		err = dev_uc_add(lowerdev, dev->dev_addr);
+		if (err < 0)
+			goto out;
+	}
+
 	if (dev->flags & IFF_ALLMULTI) {
 		err = dev_set_allmulti(lowerdev, 1);
 		if (err < 0)
@@ -653,13 +655,14 @@ static int macvlan_open(struct net_device *dev)
 	if (dev->flags & IFF_ALLMULTI)
 		dev_set_allmulti(lowerdev, -1);
 del_unicast:
-	dev_uc_del(lowerdev, dev->dev_addr);
-out:
-	if (vlan->fwd_priv) {
+	if (vlan->accel_priv) {
 		lowerdev->netdev_ops->ndo_dfwd_del_station(lowerdev,
-							   vlan->fwd_priv);
-		vlan->fwd_priv = NULL;
+							   vlan->accel_priv);
+		vlan->accel_priv = NULL;
+	} else {
+		dev_uc_del(lowerdev, dev->dev_addr);
 	}
+out:
 	return err;
 }
 
@@ -668,11 +671,10 @@ static int macvlan_stop(struct net_device *dev)
 	struct macvlan_dev *vlan = netdev_priv(dev);
 	struct net_device *lowerdev = vlan->lowerdev;
 
-	if (vlan->fwd_priv) {
+	if (vlan->accel_priv) {
 		lowerdev->netdev_ops->ndo_dfwd_del_station(lowerdev,
-							   vlan->fwd_priv);
-		vlan->fwd_priv = NULL;
-		return 0;
+							   vlan->accel_priv);
+		vlan->accel_priv = NULL;
 	}
 
 	dev_uc_unsync(lowerdev, dev);
diff --git a/drivers/net/net_failover.c b/drivers/net/net_failover.c
new file mode 100644
index 000000000000..8b508e2cf29b
--- /dev/null
+++ b/drivers/net/net_failover.c
@@ -0,0 +1,836 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2018, Intel Corporation. */
+
+/* This provides a net_failover interface for paravirtual drivers to
+ * provide an alternate datapath by exporting APIs to create and
+ * destroy a upper 'net_failover' netdev. The upper dev manages the
+ * original paravirtual interface as a 'standby' netdev and uses the
+ * generic failover infrastructure to register and manage a direct
+ * attached VF as a 'primary' netdev. This enables live migration of
+ * a VM with direct attached VF by failing over to the paravirtual
+ * datapath when the VF is unplugged.
+ *
+ * Some of the netdev management routines are based on bond/team driver as
+ * this driver provides active-backup functionality similar to those drivers.
+ */
+
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/netdevice.h>
+#include <linux/netpoll.h>
+#include <linux/rtnetlink.h>
+#include <linux/if_vlan.h>
+#include <linux/pci.h>
+#include <net/sch_generic.h>
+#include <uapi/linux/if_arp.h>
+#include <net/net_failover.h>
+
+static bool net_failover_xmit_ready(struct net_device *dev)
+{
+	return netif_running(dev) && netif_carrier_ok(dev);
+}
+
+static int net_failover_open(struct net_device *dev)
+{
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	struct net_device *primary_dev, *standby_dev;
+	int err;
+
+	primary_dev = rtnl_dereference(nfo_info->primary_dev);
+	if (primary_dev) {
+		err = dev_open(primary_dev);
+		if (err)
+			goto err_primary_open;
+	}
+
+	standby_dev = rtnl_dereference(nfo_info->standby_dev);
+	if (standby_dev) {
+		err = dev_open(standby_dev);
+		if (err)
+			goto err_standby_open;
+	}
+
+	if ((primary_dev && net_failover_xmit_ready(primary_dev)) ||
+	    (standby_dev && net_failover_xmit_ready(standby_dev))) {
+		netif_carrier_on(dev);
+		netif_tx_wake_all_queues(dev);
+	}
+
+	return 0;
+
+err_standby_open:
+	dev_close(primary_dev);
+err_primary_open:
+	netif_tx_disable(dev);
+	return err;
+}
+
+static int net_failover_close(struct net_device *dev)
+{
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	struct net_device *slave_dev;
+
+	netif_tx_disable(dev);
+
+	slave_dev = rtnl_dereference(nfo_info->primary_dev);
+	if (slave_dev)
+		dev_close(slave_dev);
+
+	slave_dev = rtnl_dereference(nfo_info->standby_dev);
+	if (slave_dev)
+		dev_close(slave_dev);
+
+	return 0;
+}
+
+static netdev_tx_t net_failover_drop_xmit(struct sk_buff *skb,
+					  struct net_device *dev)
+{
+	atomic_long_inc(&dev->tx_dropped);
+	dev_kfree_skb_any(skb);
+	return NETDEV_TX_OK;
+}
+
+static netdev_tx_t net_failover_start_xmit(struct sk_buff *skb,
+					   struct net_device *dev)
+{
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	struct net_device *xmit_dev;
+
+	/* Try xmit via primary netdev followed by standby netdev */
+	xmit_dev = rcu_dereference_bh(nfo_info->primary_dev);
+	if (!xmit_dev || !net_failover_xmit_ready(xmit_dev)) {
+		xmit_dev = rcu_dereference_bh(nfo_info->standby_dev);
+		if (!xmit_dev || !net_failover_xmit_ready(xmit_dev))
+			return net_failover_drop_xmit(skb, dev);
+	}
+
+	skb->dev = xmit_dev;
+	skb->queue_mapping = qdisc_skb_cb(skb)->slave_dev_queue_mapping;
+
+	return dev_queue_xmit(skb);
+}
+
+static u16 net_failover_select_queue(struct net_device *dev,
+				     struct sk_buff *skb, void *accel_priv,
+				     select_queue_fallback_t fallback)
+{
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	struct net_device *primary_dev;
+	u16 txq;
+
+	primary_dev = rcu_dereference(nfo_info->primary_dev);
+	if (primary_dev) {
+		const struct net_device_ops *ops = primary_dev->netdev_ops;
+
+		if (ops->ndo_select_queue)
+			txq = ops->ndo_select_queue(primary_dev, skb,
+						    accel_priv, fallback);
+		else
+			txq = fallback(primary_dev, skb);
+
+		qdisc_skb_cb(skb)->slave_dev_queue_mapping = skb->queue_mapping;
+
+		return txq;
+	}
+
+	txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : 0;
+
+	/* Save the original txq to restore before passing to the driver */
+	qdisc_skb_cb(skb)->slave_dev_queue_mapping = skb->queue_mapping;
+
+	if (unlikely(txq >= dev->real_num_tx_queues)) {
+		do {
+			txq -= dev->real_num_tx_queues;
+		} while (txq >= dev->real_num_tx_queues);
+	}
+
+	return txq;
+}
+
+/* fold stats, assuming all rtnl_link_stats64 fields are u64, but
+ * that some drivers can provide 32bit values only.
+ */
+static void net_failover_fold_stats(struct rtnl_link_stats64 *_res,
+				    const struct rtnl_link_stats64 *_new,
+				    const struct rtnl_link_stats64 *_old)
+{
+	const u64 *new = (const u64 *)_new;
+	const u64 *old = (const u64 *)_old;
+	u64 *res = (u64 *)_res;
+	int i;
+
+	for (i = 0; i < sizeof(*_res) / sizeof(u64); i++) {
+		u64 nv = new[i];
+		u64 ov = old[i];
+		s64 delta = nv - ov;
+
+		/* detects if this particular field is 32bit only */
+		if (((nv | ov) >> 32) == 0)
+			delta = (s64)(s32)((u32)nv - (u32)ov);
+
+		/* filter anomalies, some drivers reset their stats
+		 * at down/up events.
+		 */
+		if (delta > 0)
+			res[i] += delta;
+	}
+}
+
+static void net_failover_get_stats(struct net_device *dev,
+				   struct rtnl_link_stats64 *stats)
+{
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	const struct rtnl_link_stats64 *new;
+	struct rtnl_link_stats64 temp;
+	struct net_device *slave_dev;
+
+	spin_lock(&nfo_info->stats_lock);
+	memcpy(stats, &nfo_info->failover_stats, sizeof(*stats));
+
+	rcu_read_lock();
+
+	slave_dev = rcu_dereference(nfo_info->primary_dev);
+	if (slave_dev) {
+		new = dev_get_stats(slave_dev, &temp);
+		net_failover_fold_stats(stats, new, &nfo_info->primary_stats);
+		memcpy(&nfo_info->primary_stats, new, sizeof(*new));
+	}
+
+	slave_dev = rcu_dereference(nfo_info->standby_dev);
+	if (slave_dev) {
+		new = dev_get_stats(slave_dev, &temp);
+		net_failover_fold_stats(stats, new, &nfo_info->standby_stats);
+		memcpy(&nfo_info->standby_stats, new, sizeof(*new));
+	}
+
+	rcu_read_unlock();
+
+	memcpy(&nfo_info->failover_stats, stats, sizeof(*stats));
+	spin_unlock(&nfo_info->stats_lock);
+}
+
+static int net_failover_change_mtu(struct net_device *dev, int new_mtu)
+{
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	struct net_device *primary_dev, *standby_dev;
+	int ret = 0;
+
+	primary_dev = rcu_dereference(nfo_info->primary_dev);
+	if (primary_dev) {
+		ret = dev_set_mtu(primary_dev, new_mtu);
+		if (ret)
+			return ret;
+	}
+
+	standby_dev = rcu_dereference(nfo_info->standby_dev);
+	if (standby_dev) {
+		ret = dev_set_mtu(standby_dev, new_mtu);
+		if (ret) {
+			if (primary_dev)
+				dev_set_mtu(primary_dev, dev->mtu);
+			return ret;
+		}
+	}
+
+	dev->mtu = new_mtu;
+
+	return 0;
+}
+
+static void net_failover_set_rx_mode(struct net_device *dev)
+{
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	struct net_device *slave_dev;
+
+	rcu_read_lock();
+
+	slave_dev = rcu_dereference(nfo_info->primary_dev);
+	if (slave_dev) {
+		dev_uc_sync_multiple(slave_dev, dev);
+		dev_mc_sync_multiple(slave_dev, dev);
+	}
+
+	slave_dev = rcu_dereference(nfo_info->standby_dev);
+	if (slave_dev) {
+		dev_uc_sync_multiple(slave_dev, dev);
+		dev_mc_sync_multiple(slave_dev, dev);
+	}
+
+	rcu_read_unlock();
+}
+
+static int net_failover_vlan_rx_add_vid(struct net_device *dev, __be16 proto,
+					u16 vid)
+{
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	struct net_device *primary_dev, *standby_dev;
+	int ret = 0;
+
+	primary_dev = rcu_dereference(nfo_info->primary_dev);
+	if (primary_dev) {
+		ret = vlan_vid_add(primary_dev, proto, vid);
+		if (ret)
+			return ret;
+	}
+
+	standby_dev = rcu_dereference(nfo_info->standby_dev);
+	if (standby_dev) {
+		ret = vlan_vid_add(standby_dev, proto, vid);
+		if (ret)
+			if (primary_dev)
+				vlan_vid_del(primary_dev, proto, vid);
+	}
+
+	return ret;
+}
+
+static int net_failover_vlan_rx_kill_vid(struct net_device *dev, __be16 proto,
+					 u16 vid)
+{
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	struct net_device *slave_dev;
+
+	slave_dev = rcu_dereference(nfo_info->primary_dev);
+	if (slave_dev)
+		vlan_vid_del(slave_dev, proto, vid);
+
+	slave_dev = rcu_dereference(nfo_info->standby_dev);
+	if (slave_dev)
+		vlan_vid_del(slave_dev, proto, vid);
+
+	return 0;
+}
+
+static const struct net_device_ops failover_dev_ops = {
+	.ndo_open		= net_failover_open,
+	.ndo_stop		= net_failover_close,
+	.ndo_start_xmit		= net_failover_start_xmit,
+	.ndo_select_queue	= net_failover_select_queue,
+	.ndo_get_stats64	= net_failover_get_stats,
+	.ndo_change_mtu		= net_failover_change_mtu,
+	.ndo_set_rx_mode	= net_failover_set_rx_mode,
+	.ndo_vlan_rx_add_vid	= net_failover_vlan_rx_add_vid,
+	.ndo_vlan_rx_kill_vid	= net_failover_vlan_rx_kill_vid,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_features_check	= passthru_features_check,
+};
+
+#define FAILOVER_NAME "net_failover"
+#define FAILOVER_VERSION "0.1"
+
+static void nfo_ethtool_get_drvinfo(struct net_device *dev,
+				    struct ethtool_drvinfo *drvinfo)
+{
+	strlcpy(drvinfo->driver, FAILOVER_NAME, sizeof(drvinfo->driver));
+	strlcpy(drvinfo->version, FAILOVER_VERSION, sizeof(drvinfo->version));
+}
+
+static int nfo_ethtool_get_link_ksettings(struct net_device *dev,
+					  struct ethtool_link_ksettings *cmd)
+{
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	struct net_device *slave_dev;
+
+	slave_dev = rtnl_dereference(nfo_info->primary_dev);
+	if (!slave_dev || !net_failover_xmit_ready(slave_dev)) {
+		slave_dev = rtnl_dereference(nfo_info->standby_dev);
+		if (!slave_dev || !net_failover_xmit_ready(slave_dev)) {
+			cmd->base.duplex = DUPLEX_UNKNOWN;
+			cmd->base.port = PORT_OTHER;
+			cmd->base.speed = SPEED_UNKNOWN;
+
+			return 0;
+		}
+	}
+
+	return __ethtool_get_link_ksettings(slave_dev, cmd);
+}
+
+static const struct ethtool_ops failover_ethtool_ops = {
+	.get_drvinfo            = nfo_ethtool_get_drvinfo,
+	.get_link               = ethtool_op_get_link,
+	.get_link_ksettings     = nfo_ethtool_get_link_ksettings,
+};
+
+/* Called when slave dev is injecting data into network stack.
+ * Change the associated network device from lower dev to failover dev.
+ * note: already called with rcu_read_lock
+ */
+static rx_handler_result_t net_failover_handle_frame(struct sk_buff **pskb)
+{
+	struct sk_buff *skb = *pskb;
+	struct net_device *dev = rcu_dereference(skb->dev->rx_handler_data);
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	struct net_device *primary_dev, *standby_dev;
+
+	primary_dev = rcu_dereference(nfo_info->primary_dev);
+	standby_dev = rcu_dereference(nfo_info->standby_dev);
+
+	if (primary_dev && skb->dev == standby_dev)
+		return RX_HANDLER_EXACT;
+
+	skb->dev = dev;
+
+	return RX_HANDLER_ANOTHER;
+}
+
+static void net_failover_compute_features(struct net_device *dev)
+{
+	u32 vlan_features = FAILOVER_VLAN_FEATURES & NETIF_F_ALL_FOR_ALL;
+	netdev_features_t enc_features  = FAILOVER_ENC_FEATURES;
+	unsigned short max_hard_header_len = ETH_HLEN;
+	unsigned int dst_release_flag = IFF_XMIT_DST_RELEASE |
+					IFF_XMIT_DST_RELEASE_PERM;
+	struct net_failover_info *nfo_info = netdev_priv(dev);
+	struct net_device *primary_dev, *standby_dev;
+
+	primary_dev = rcu_dereference(nfo_info->primary_dev);
+	if (primary_dev) {
+		vlan_features =
+			netdev_increment_features(vlan_features,
+						  primary_dev->vlan_features,
+						  FAILOVER_VLAN_FEATURES);
+		enc_features =
+			netdev_increment_features(enc_features,
+						  primary_dev->hw_enc_features,
+						  FAILOVER_ENC_FEATURES);
+
+		dst_release_flag &= primary_dev->priv_flags;
+		if (primary_dev->hard_header_len > max_hard_header_len)
+			max_hard_header_len = primary_dev->hard_header_len;
+	}
+
+	standby_dev = rcu_dereference(nfo_info->standby_dev);
+	if (standby_dev) {
+		vlan_features =
+			netdev_increment_features(vlan_features,
+						  standby_dev->vlan_features,
+						  FAILOVER_VLAN_FEATURES);
+		enc_features =
+			netdev_increment_features(enc_features,
+						  standby_dev->hw_enc_features,
+						  FAILOVER_ENC_FEATURES);
+
+		dst_release_flag &= standby_dev->priv_flags;
+		if (standby_dev->hard_header_len > max_hard_header_len)
+			max_hard_header_len = standby_dev->hard_header_len;
+	}
+
+	dev->vlan_features = vlan_features;
+	dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL;
+	dev->hard_header_len = max_hard_header_len;
+
+	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
+	if (dst_release_flag == (IFF_XMIT_DST_RELEASE |
+				 IFF_XMIT_DST_RELEASE_PERM))
+		dev->priv_flags |= IFF_XMIT_DST_RELEASE;
+
+	netdev_change_features(dev);
+}
+
+static void net_failover_lower_state_changed(struct net_device *slave_dev,
+					     struct net_device *primary_dev,
+					     struct net_device *standby_dev)
+{
+	struct netdev_lag_lower_state_info info;
+
+	if (netif_carrier_ok(slave_dev))
+		info.link_up = true;
+	else
+		info.link_up = false;
+
+	if (slave_dev == primary_dev) {
+		if (netif_running(primary_dev))
+			info.tx_enabled = true;
+		else
+			info.tx_enabled = false;
+	} else {
+		if ((primary_dev && netif_running(primary_dev)) ||
+		    (!netif_running(standby_dev)))
+			info.tx_enabled = false;
+		else
+			info.tx_enabled = true;
+	}
+
+	netdev_lower_state_changed(slave_dev, &info);
+}
+
+static int net_failover_slave_pre_register(struct net_device *slave_dev,
+					   struct net_device *failover_dev)
+{
+	struct net_device *standby_dev, *primary_dev;
+	struct net_failover_info *nfo_info;
+	bool slave_is_standby;
+
+	nfo_info = netdev_priv(failover_dev);
+	standby_dev = rtnl_dereference(nfo_info->standby_dev);
+	primary_dev = rtnl_dereference(nfo_info->primary_dev);
+	slave_is_standby = slave_dev->dev.parent == failover_dev->dev.parent;
+	if (slave_is_standby ? standby_dev : primary_dev) {
+		netdev_err(failover_dev, "%s attempting to register as slave dev when %s already present\n",
+			   slave_dev->name,
+			   slave_is_standby ? "standby" : "primary");
+		return -EINVAL;
+	}
+
+	/* We want to allow only a direct attached VF device as a primary
+	 * netdev. As there is no easy way to check for a VF device, restrict
+	 * this to a pci device.
+	 */
+	if (!slave_is_standby && (!slave_dev->dev.parent ||
+				  !dev_is_pci(slave_dev->dev.parent)))
+		return -EINVAL;
+
+	if (failover_dev->features & NETIF_F_VLAN_CHALLENGED &&
+	    vlan_uses_dev(failover_dev)) {
+		netdev_err(failover_dev, "Device %s is VLAN challenged and failover device has VLAN set up\n",
+			   failover_dev->name);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int net_failover_slave_register(struct net_device *slave_dev,
+				       struct net_device *failover_dev)
+{
+	struct net_device *standby_dev, *primary_dev;
+	struct net_failover_info *nfo_info;
+	bool slave_is_standby;
+	u32 orig_mtu;
+	int err;
+
+	/* Align MTU of slave with failover dev */
+	orig_mtu = slave_dev->mtu;
+	err = dev_set_mtu(slave_dev, failover_dev->mtu);
+	if (err) {
+		netdev_err(failover_dev, "unable to change mtu of %s to %u register failed\n",
+			   slave_dev->name, failover_dev->mtu);
+		goto done;
+	}
+
+	dev_hold(slave_dev);
+
+	if (netif_running(failover_dev)) {
+		err = dev_open(slave_dev);
+		if (err && (err != -EBUSY)) {
+			netdev_err(failover_dev, "Opening slave %s failed err:%d\n",
+				   slave_dev->name, err);
+			goto err_dev_open;
+		}
+	}
+
+	netif_addr_lock_bh(failover_dev);
+	dev_uc_sync_multiple(slave_dev, failover_dev);
+	dev_uc_sync_multiple(slave_dev, failover_dev);
+	netif_addr_unlock_bh(failover_dev);
+
+	err = vlan_vids_add_by_dev(slave_dev, failover_dev);
+	if (err) {
+		netdev_err(failover_dev, "Failed to add vlan ids to device %s err:%d\n",
+			   slave_dev->name, err);
+		goto err_vlan_add;
+	}
+
+	nfo_info = netdev_priv(failover_dev);
+	standby_dev = rtnl_dereference(nfo_info->standby_dev);
+	primary_dev = rtnl_dereference(nfo_info->primary_dev);
+	slave_is_standby = slave_dev->dev.parent == failover_dev->dev.parent;
+
+	if (slave_is_standby) {
+		rcu_assign_pointer(nfo_info->standby_dev, slave_dev);
+		standby_dev = slave_dev;
+		dev_get_stats(standby_dev, &nfo_info->standby_stats);
+	} else {
+		rcu_assign_pointer(nfo_info->primary_dev, slave_dev);
+		primary_dev = slave_dev;
+		dev_get_stats(primary_dev, &nfo_info->primary_stats);
+		failover_dev->min_mtu = slave_dev->min_mtu;
+		failover_dev->max_mtu = slave_dev->max_mtu;
+	}
+
+	net_failover_lower_state_changed(slave_dev, primary_dev, standby_dev);
+	net_failover_compute_features(failover_dev);
+
+	call_netdevice_notifiers(NETDEV_JOIN, slave_dev);
+
+	netdev_info(failover_dev, "failover %s slave:%s registered\n",
+		    slave_is_standby ? "standby" : "primary", slave_dev->name);
+
+	return 0;
+
+err_vlan_add:
+	dev_uc_unsync(slave_dev, failover_dev);
+	dev_mc_unsync(slave_dev, failover_dev);
+	dev_close(slave_dev);
+err_dev_open:
+	dev_put(slave_dev);
+	dev_set_mtu(slave_dev, orig_mtu);
+done:
+	return err;
+}
+
+static int net_failover_slave_pre_unregister(struct net_device *slave_dev,
+					     struct net_device *failover_dev)
+{
+	struct net_device *standby_dev, *primary_dev;
+	struct net_failover_info *nfo_info;
+
+	nfo_info = netdev_priv(failover_dev);
+	primary_dev = rtnl_dereference(nfo_info->primary_dev);
+	standby_dev = rtnl_dereference(nfo_info->standby_dev);
+
+	if (slave_dev != primary_dev && slave_dev != standby_dev)
+		return -ENODEV;
+
+	return 0;
+}
+
+static int net_failover_slave_unregister(struct net_device *slave_dev,
+					 struct net_device *failover_dev)
+{
+	struct net_device *standby_dev, *primary_dev;
+	struct net_failover_info *nfo_info;
+	bool slave_is_standby;
+
+	nfo_info = netdev_priv(failover_dev);
+	primary_dev = rtnl_dereference(nfo_info->primary_dev);
+	standby_dev = rtnl_dereference(nfo_info->standby_dev);
+
+	vlan_vids_del_by_dev(slave_dev, failover_dev);
+	dev_uc_unsync(slave_dev, failover_dev);
+	dev_mc_unsync(slave_dev, failover_dev);
+	dev_close(slave_dev);
+
+	nfo_info = netdev_priv(failover_dev);
+	dev_get_stats(failover_dev, &nfo_info->failover_stats);
+
+	slave_is_standby = slave_dev->dev.parent == failover_dev->dev.parent;
+	if (slave_is_standby) {
+		RCU_INIT_POINTER(nfo_info->standby_dev, NULL);
+	} else {
+		RCU_INIT_POINTER(nfo_info->primary_dev, NULL);
+		if (standby_dev) {
+			failover_dev->min_mtu = standby_dev->min_mtu;
+			failover_dev->max_mtu = standby_dev->max_mtu;
+		}
+	}
+
+	dev_put(slave_dev);
+
+	net_failover_compute_features(failover_dev);
+
+	netdev_info(failover_dev, "failover %s slave:%s unregistered\n",
+		    slave_is_standby ? "standby" : "primary", slave_dev->name);
+
+	return 0;
+}
+
+static int net_failover_slave_link_change(struct net_device *slave_dev,
+					  struct net_device *failover_dev)
+{
+	struct net_device *primary_dev, *standby_dev;
+	struct net_failover_info *nfo_info;
+
+	nfo_info = netdev_priv(failover_dev);
+
+	primary_dev = rtnl_dereference(nfo_info->primary_dev);
+	standby_dev = rtnl_dereference(nfo_info->standby_dev);
+
+	if (slave_dev != primary_dev && slave_dev != standby_dev)
+		return -ENODEV;
+
+	if ((primary_dev && net_failover_xmit_ready(primary_dev)) ||
+	    (standby_dev && net_failover_xmit_ready(standby_dev))) {
+		netif_carrier_on(failover_dev);
+		netif_tx_wake_all_queues(failover_dev);
+	} else {
+		dev_get_stats(failover_dev, &nfo_info->failover_stats);
+		netif_carrier_off(failover_dev);
+		netif_tx_stop_all_queues(failover_dev);
+	}
+
+	net_failover_lower_state_changed(slave_dev, primary_dev, standby_dev);
+
+	return 0;
+}
+
+static int net_failover_slave_name_change(struct net_device *slave_dev,
+					  struct net_device *failover_dev)
+{
+	struct net_device *primary_dev, *standby_dev;
+	struct net_failover_info *nfo_info;
+
+	nfo_info = netdev_priv(failover_dev);
+
+	primary_dev = rtnl_dereference(nfo_info->primary_dev);
+	standby_dev = rtnl_dereference(nfo_info->standby_dev);
+
+	if (slave_dev != primary_dev && slave_dev != standby_dev)
+		return -ENODEV;
+
+	/* We need to bring up the slave after the rename by udev in case
+	 * open failed with EBUSY when it was registered.
+	 */
+	dev_open(slave_dev);
+
+	return 0;
+}
+
+static struct failover_ops net_failover_ops = {
+	.slave_pre_register	= net_failover_slave_pre_register,
+	.slave_register		= net_failover_slave_register,
+	.slave_pre_unregister	= net_failover_slave_pre_unregister,
+	.slave_unregister	= net_failover_slave_unregister,
+	.slave_link_change	= net_failover_slave_link_change,
+	.slave_name_change	= net_failover_slave_name_change,
+	.slave_handle_frame	= net_failover_handle_frame,
+};
+
+/**
+ * net_failover_create - Create and register a failover instance
+ *
+ * @dev: standby netdev
+ *
+ * Creates a failover netdev and registers a failover instance for a standby
+ * netdev. Used by paravirtual drivers that use 3-netdev model.
+ * The failover netdev acts as a master device and controls 2 slave devices -
+ * the original standby netdev and a VF netdev with the same MAC gets
+ * registered as primary netdev.
+ *
+ * Return: pointer to failover instance
+ */
+struct failover *net_failover_create(struct net_device *standby_dev)
+{
+	struct device *dev = standby_dev->dev.parent;
+	struct net_device *failover_dev;
+	struct failover *failover;
+	int err;
+
+	/* Alloc at least 2 queues, for now we are going with 16 assuming
+	 * that VF devices being enslaved won't have too many queues.
+	 */
+	failover_dev = alloc_etherdev_mq(sizeof(struct net_failover_info), 16);
+	if (!failover_dev) {
+		dev_err(dev, "Unable to allocate failover_netdev!\n");
+		return ERR_PTR(-ENOMEM);
+	}
+
+	dev_net_set(failover_dev, dev_net(standby_dev));
+	SET_NETDEV_DEV(failover_dev, dev);
+
+	failover_dev->netdev_ops = &failover_dev_ops;
+	failover_dev->ethtool_ops = &failover_ethtool_ops;
+
+	/* Initialize the device options */
+	failover_dev->priv_flags |= IFF_UNICAST_FLT | IFF_NO_QUEUE;
+	failover_dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE |
+				       IFF_TX_SKB_SHARING);
+
+	/* don't acquire failover netdev's netif_tx_lock when transmitting */
+	failover_dev->features |= NETIF_F_LLTX;
+
+	/* Don't allow failover devices to change network namespaces. */
+	failover_dev->features |= NETIF_F_NETNS_LOCAL;
+
+	failover_dev->hw_features = FAILOVER_VLAN_FEATURES |
+				    NETIF_F_HW_VLAN_CTAG_TX |
+				    NETIF_F_HW_VLAN_CTAG_RX |
+				    NETIF_F_HW_VLAN_CTAG_FILTER;
+
+	failover_dev->hw_features |= NETIF_F_GSO_ENCAP_ALL;
+	failover_dev->features |= failover_dev->hw_features;
+
+	memcpy(failover_dev->dev_addr, standby_dev->dev_addr,
+	       failover_dev->addr_len);
+
+	failover_dev->min_mtu = standby_dev->min_mtu;
+	failover_dev->max_mtu = standby_dev->max_mtu;
+
+	err = register_netdev(failover_dev);
+	if (err) {
+		dev_err(dev, "Unable to register failover_dev!\n");
+		goto err_register_netdev;
+	}
+
+	netif_carrier_off(failover_dev);
+
+	failover = failover_register(failover_dev, &net_failover_ops);
+	if (IS_ERR(failover))
+		goto err_failover_register;
+
+	return failover;
+
+err_failover_register:
+	unregister_netdev(failover_dev);
+err_register_netdev:
+	free_netdev(failover_dev);
+
+	return ERR_PTR(err);
+}
+EXPORT_SYMBOL_GPL(net_failover_create);
+
+/**
+ * net_failover_destroy - Destroy a failover instance
+ *
+ * @failover: pointer to failover instance
+ *
+ * Unregisters any slave netdevs associated with the failover instance by
+ * calling failover_slave_unregister().
+ * unregisters the failover instance itself and finally frees the failover
+ * netdev. Used by paravirtual drivers that use 3-netdev model.
+ *
+ */
+void net_failover_destroy(struct failover *failover)
+{
+	struct net_failover_info *nfo_info;
+	struct net_device *failover_dev;
+	struct net_device *slave_dev;
+
+	if (!failover)
+		return;
+
+	failover_dev = rcu_dereference(failover->failover_dev);
+	nfo_info = netdev_priv(failover_dev);
+
+	netif_device_detach(failover_dev);
+
+	rtnl_lock();
+
+	slave_dev = rtnl_dereference(nfo_info->primary_dev);
+	if (slave_dev)
+		failover_slave_unregister(slave_dev);
+
+	slave_dev = rtnl_dereference(nfo_info->standby_dev);
+	if (slave_dev)
+		failover_slave_unregister(slave_dev);
+
+	failover_unregister(failover);
+
+	unregister_netdevice(failover_dev);
+
+	rtnl_unlock();
+
+	free_netdev(failover_dev);
+}
+EXPORT_SYMBOL_GPL(net_failover_destroy);
+
+static __init int
+net_failover_init(void)
+{
+	return 0;
+}
+module_init(net_failover_init);
+
+static __exit
+void net_failover_exit(void)
+{
+}
+module_exit(net_failover_exit);
+
+MODULE_DESCRIPTION("Failover driver for Paravirtual drivers");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig
index bdfbabb86ee0..343989f9f9d9 100644
--- a/drivers/net/phy/Kconfig
+++ b/drivers/net/phy/Kconfig
@@ -118,11 +118,18 @@ config MDIO_I2C
 
 config MDIO_MOXART
 	tristate "MOXA ART MDIO interface support"
-	depends on ARCH_MOXART
+	depends on ARCH_MOXART || COMPILE_TEST
 	help
 	  This driver supports the MDIO interface found in the network
 	  interface units of the MOXA ART SoC
 
+config MDIO_MSCC_MIIM
+	tristate "Microsemi MIIM interface support"
+	depends on HAS_IOMEM
+	help
+	  This driver supports the MIIM (MDIO) interface found in the network
+	  switches of the Microsemi SoCs
+
 config MDIO_OCTEON
 	tristate "Octeon and some ThunderX SOCs MDIO buses"
 	depends on 64BIT
@@ -135,7 +142,7 @@ config MDIO_OCTEON
 
 config MDIO_SUN4I
 	tristate "Allwinner sun4i MDIO interface support"
-	depends on ARCH_SUNXI
+	depends on ARCH_SUNXI || COMPILE_TEST
 	help
 	  This driver supports the MDIO interface found in the network
 	  interface units of the Allwinner SoC that have an EMAC (A10,
@@ -218,6 +225,12 @@ config AQUANTIA_PHY
 	---help---
 	  Currently supports the Aquantia AQ1202, AQ2104, AQR105, AQR405
 
+config ASIX_PHY
+	tristate "Asix PHYs"
+	help
+	  Currently supports the Asix Electronics PHY found in the X-Surf 100
+	  AX88796B package.
+
 config AT803X_PHY
 	tristate "AT803X PHYs"
 	---help---
@@ -285,6 +298,11 @@ config DP83822_PHY
 	---help---
 	  Supports the DP83822 PHY.
 
+config DP83TC811_PHY
+	tristate "Texas Instruments DP83TC822 PHY"
+	---help---
+	  Supports the DP83TC822 PHY.
+
 config DP83848_PHY
 	tristate "Texas Instruments DP83848 PHY"
 	---help---
@@ -354,6 +372,11 @@ config MICROCHIP_PHY
 	help
 	  Supports the LAN88XX PHYs.
 
+config MICROCHIP_T1_PHY
+	tristate "Microchip T1 PHYs"
+	---help---
+	  Supports the LAN87XX PHYs.
+
 config MICROSEMI_PHY
 	tristate "Microsemi PHYs"
 	---help---
diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile
index 01acbcb2c798..5805c0b7d60e 100644
--- a/drivers/net/phy/Makefile
+++ b/drivers/net/phy/Makefile
@@ -34,6 +34,7 @@ obj-$(CONFIG_MDIO_GPIO)		+= mdio-gpio.o
 obj-$(CONFIG_MDIO_HISI_FEMAC)	+= mdio-hisi-femac.o
 obj-$(CONFIG_MDIO_I2C)		+= mdio-i2c.o
 obj-$(CONFIG_MDIO_MOXART)	+= mdio-moxart.o
+obj-$(CONFIG_MDIO_MSCC_MIIM)	+= mdio-mscc-miim.o
 obj-$(CONFIG_MDIO_OCTEON)	+= mdio-octeon.o
 obj-$(CONFIG_MDIO_SUN4I)	+= mdio-sun4i.o
 obj-$(CONFIG_MDIO_THUNDER)	+= mdio-thunder.o
@@ -45,6 +46,7 @@ obj-y				+= $(sfp-obj-y) $(sfp-obj-m)
 
 obj-$(CONFIG_AMD_PHY)		+= amd.o
 obj-$(CONFIG_AQUANTIA_PHY)	+= aquantia.o
+obj-$(CONFIG_ASIX_PHY)		+= asix.o
 obj-$(CONFIG_AT803X_PHY)	+= at803x.o
 obj-$(CONFIG_BCM63XX_PHY)	+= bcm63xx.o
 obj-$(CONFIG_BCM7XXX_PHY)	+= bcm7xxx.o
@@ -57,6 +59,7 @@ obj-$(CONFIG_CORTINA_PHY)	+= cortina.o
 obj-$(CONFIG_DAVICOM_PHY)	+= davicom.o
 obj-$(CONFIG_DP83640_PHY)	+= dp83640.o
 obj-$(CONFIG_DP83822_PHY)	+= dp83822.o
+obj-$(CONFIG_DP83TC811_PHY)	+= dp83tc811.o
 obj-$(CONFIG_DP83848_PHY)	+= dp83848.o
 obj-$(CONFIG_DP83867_PHY)	+= dp83867.o
 obj-$(CONFIG_FIXED_PHY)		+= fixed_phy.o
@@ -70,6 +73,7 @@ obj-$(CONFIG_MESON_GXL_PHY)	+= meson-gxl.o
 obj-$(CONFIG_MICREL_KS8995MA)	+= spi_ks8995.o
 obj-$(CONFIG_MICREL_PHY)	+= micrel.o
 obj-$(CONFIG_MICROCHIP_PHY)	+= microchip.o
+obj-$(CONFIG_MICROCHIP_T1_PHY)	+= microchip_t1.o
 obj-$(CONFIG_MICROSEMI_PHY)	+= mscc.o
 obj-$(CONFIG_NATIONAL_PHY)	+= national.o
 obj-$(CONFIG_QSEMI_PHY)		+= qsemi.o
diff --git a/drivers/net/phy/asix.c b/drivers/net/phy/asix.c
new file mode 100644
index 000000000000..8ebe7f5484ae
--- /dev/null
+++ b/drivers/net/phy/asix.c
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Driver for Asix PHYs
+ *
+ * Author: Michael Schmitz <schmitzmic@gmail.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ *
+ */
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mii.h>
+#include <linux/phy.h>
+
+#define PHY_ID_ASIX_AX88796B		0x003b1841
+
+MODULE_DESCRIPTION("Asix PHY driver");
+MODULE_AUTHOR("Michael Schmitz <schmitzmic@gmail.com>");
+MODULE_LICENSE("GPL");
+
+/**
+ * asix_soft_reset - software reset the PHY via BMCR_RESET bit
+ * @phydev: target phy_device struct
+ *
+ * Description: Perform a software PHY reset using the standard
+ * BMCR_RESET bit and poll for the reset bit to be cleared.
+ * Toggle BMCR_RESET bit off to accommodate broken AX8796B PHY implementation
+ * such as used on the Individual Computers' X-Surf 100 Zorro card.
+ *
+ * Returns: 0 on success, < 0 on failure
+ */
+static int asix_soft_reset(struct phy_device *phydev)
+{
+	int ret;
+
+	/* Asix PHY won't reset unless reset bit toggles */
+	ret = phy_write(phydev, MII_BMCR, 0);
+	if (ret < 0)
+		return ret;
+
+	return genphy_soft_reset(phydev);
+}
+
+static struct phy_driver asix_driver[] = { {
+	.phy_id		= PHY_ID_ASIX_AX88796B,
+	.name		= "Asix Electronics AX88796B",
+	.phy_id_mask	= 0xfffffff0,
+	.features	= PHY_BASIC_FEATURES,
+	.soft_reset	= asix_soft_reset,
+} };
+
+module_phy_driver(asix_driver);
+
+static struct mdio_device_id __maybe_unused asix_tbl[] = {
+	{ PHY_ID_ASIX_AX88796B, 0xfffffff0 },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(mdio, asix_tbl);
diff --git a/drivers/net/phy/bcm-phy-lib.c b/drivers/net/phy/bcm-phy-lib.c
index d5e0833d69b9..e10e7b54ec4b 100644
--- a/drivers/net/phy/bcm-phy-lib.c
+++ b/drivers/net/phy/bcm-phy-lib.c
@@ -346,10 +346,6 @@ void bcm_phy_get_strings(struct phy_device *phydev, u8 *data)
 }
 EXPORT_SYMBOL_GPL(bcm_phy_get_strings);
 
-#ifndef UINT64_MAX
-#define UINT64_MAX              (u64)(~((u64)0))
-#endif
-
 /* Caller is supposed to provide appropriate storage for the library code to
  * access the shadow copy
  */
@@ -362,7 +358,7 @@ static u64 bcm_phy_get_stat(struct phy_device *phydev, u64 *shadow,
 
 	val = phy_read(phydev, stat.reg);
 	if (val < 0) {
-		ret = UINT64_MAX;
+		ret = U64_MAX;
 	} else {
 		val >>= stat.shift;
 		val = val & ((1 << stat.bits) - 1);
diff --git a/drivers/net/phy/dp83tc811.c b/drivers/net/phy/dp83tc811.c
new file mode 100644
index 000000000000..081d99aa3985
--- /dev/null
+++ b/drivers/net/phy/dp83tc811.c
@@ -0,0 +1,347 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for the Texas Instruments DP83TC811 PHY
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ */
+
+#include <linux/ethtool.h>
+#include <linux/etherdevice.h>
+#include <linux/kernel.h>
+#include <linux/mii.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/phy.h>
+#include <linux/netdevice.h>
+
+#define DP83TC811_PHY_ID	0x2000a253
+#define DP83811_DEVADDR		0x1f
+
+#define MII_DP83811_SGMII_CTRL	0x09
+#define MII_DP83811_INT_STAT1	0x12
+#define MII_DP83811_INT_STAT2	0x13
+#define MII_DP83811_RESET_CTRL	0x1f
+
+#define DP83811_HW_RESET	BIT(15)
+#define DP83811_SW_RESET	BIT(14)
+
+/* INT_STAT1 bits */
+#define DP83811_RX_ERR_HF_INT_EN	BIT(0)
+#define DP83811_MS_TRAINING_INT_EN	BIT(1)
+#define DP83811_ANEG_COMPLETE_INT_EN	BIT(2)
+#define DP83811_ESD_EVENT_INT_EN	BIT(3)
+#define DP83811_WOL_INT_EN		BIT(4)
+#define DP83811_LINK_STAT_INT_EN	BIT(5)
+#define DP83811_ENERGY_DET_INT_EN	BIT(6)
+#define DP83811_LINK_QUAL_INT_EN	BIT(7)
+
+/* INT_STAT2 bits */
+#define DP83811_JABBER_DET_INT_EN	BIT(0)
+#define DP83811_POLARITY_INT_EN		BIT(1)
+#define DP83811_SLEEP_MODE_INT_EN	BIT(2)
+#define DP83811_OVERTEMP_INT_EN		BIT(3)
+#define DP83811_OVERVOLTAGE_INT_EN	BIT(6)
+#define DP83811_UNDERVOLTAGE_INT_EN	BIT(7)
+
+#define MII_DP83811_RXSOP1	0x04a5
+#define MII_DP83811_RXSOP2	0x04a6
+#define MII_DP83811_RXSOP3	0x04a7
+
+/* WoL Registers */
+#define MII_DP83811_WOL_CFG	0x04a0
+#define MII_DP83811_WOL_STAT	0x04a1
+#define MII_DP83811_WOL_DA1	0x04a2
+#define MII_DP83811_WOL_DA2	0x04a3
+#define MII_DP83811_WOL_DA3	0x04a4
+
+/* WoL bits */
+#define DP83811_WOL_MAGIC_EN	BIT(0)
+#define DP83811_WOL_SECURE_ON	BIT(5)
+#define DP83811_WOL_EN		BIT(7)
+#define DP83811_WOL_INDICATION_SEL BIT(8)
+#define DP83811_WOL_CLR_INDICATION BIT(11)
+
+/* SGMII CTRL bits */
+#define DP83811_TDR_AUTO		BIT(8)
+#define DP83811_SGMII_EN		BIT(12)
+#define DP83811_SGMII_AUTO_NEG_EN	BIT(13)
+#define DP83811_SGMII_TX_ERR_DIS	BIT(14)
+#define DP83811_SGMII_SOFT_RESET	BIT(15)
+
+static int dp83811_ack_interrupt(struct phy_device *phydev)
+{
+	int err;
+
+	err = phy_read(phydev, MII_DP83811_INT_STAT1);
+	if (err < 0)
+		return err;
+
+	err = phy_read(phydev, MII_DP83811_INT_STAT2);
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+
+static int dp83811_set_wol(struct phy_device *phydev,
+			   struct ethtool_wolinfo *wol)
+{
+	struct net_device *ndev = phydev->attached_dev;
+	const u8 *mac;
+	u16 value;
+
+	if (wol->wolopts & (WAKE_MAGIC | WAKE_MAGICSECURE)) {
+		mac = (const u8 *)ndev->dev_addr;
+
+		if (!is_valid_ether_addr(mac))
+			return -EINVAL;
+
+		/* MAC addresses start with byte 5, but stored in mac[0].
+		 * 811 PHYs store bytes 4|5, 2|3, 0|1
+		 */
+		phy_write_mmd(phydev, DP83811_DEVADDR, MII_DP83811_WOL_DA1,
+			      (mac[1] << 8) | mac[0]);
+		phy_write_mmd(phydev, DP83811_DEVADDR, MII_DP83811_WOL_DA2,
+			      (mac[3] << 8) | mac[2]);
+		phy_write_mmd(phydev, DP83811_DEVADDR, MII_DP83811_WOL_DA3,
+			      (mac[5] << 8) | mac[4]);
+
+		value = phy_read_mmd(phydev, DP83811_DEVADDR,
+				     MII_DP83811_WOL_CFG);
+		if (wol->wolopts & WAKE_MAGIC)
+			value |= DP83811_WOL_MAGIC_EN;
+		else
+			value &= ~DP83811_WOL_MAGIC_EN;
+
+		if (wol->wolopts & WAKE_MAGICSECURE) {
+			phy_write_mmd(phydev, DP83811_DEVADDR,
+				      MII_DP83811_RXSOP1,
+				      (wol->sopass[1] << 8) | wol->sopass[0]);
+			phy_write_mmd(phydev, DP83811_DEVADDR,
+				      MII_DP83811_RXSOP2,
+				      (wol->sopass[3] << 8) | wol->sopass[2]);
+			phy_write_mmd(phydev, DP83811_DEVADDR,
+				      MII_DP83811_RXSOP3,
+				      (wol->sopass[5] << 8) | wol->sopass[4]);
+			value |= DP83811_WOL_SECURE_ON;
+		} else {
+			value &= ~DP83811_WOL_SECURE_ON;
+		}
+
+		value |= (DP83811_WOL_EN | DP83811_WOL_INDICATION_SEL |
+			  DP83811_WOL_CLR_INDICATION);
+		phy_write_mmd(phydev, DP83811_DEVADDR, MII_DP83811_WOL_CFG,
+			      value);
+	} else {
+		value = phy_read_mmd(phydev, DP83811_DEVADDR,
+				     MII_DP83811_WOL_CFG);
+		value &= ~DP83811_WOL_EN;
+		phy_write_mmd(phydev, DP83811_DEVADDR, MII_DP83811_WOL_CFG,
+			      value);
+	}
+
+	return 0;
+}
+
+static void dp83811_get_wol(struct phy_device *phydev,
+			    struct ethtool_wolinfo *wol)
+{
+	u16 sopass_val;
+	int value;
+
+	wol->supported = (WAKE_MAGIC | WAKE_MAGICSECURE);
+	wol->wolopts = 0;
+
+	value = phy_read_mmd(phydev, DP83811_DEVADDR, MII_DP83811_WOL_CFG);
+
+	if (value & DP83811_WOL_MAGIC_EN)
+		wol->wolopts |= WAKE_MAGIC;
+
+	if (value & DP83811_WOL_SECURE_ON) {
+		sopass_val = phy_read_mmd(phydev, DP83811_DEVADDR,
+					  MII_DP83811_RXSOP1);
+		wol->sopass[0] = (sopass_val & 0xff);
+		wol->sopass[1] = (sopass_val >> 8);
+
+		sopass_val = phy_read_mmd(phydev, DP83811_DEVADDR,
+					  MII_DP83811_RXSOP2);
+		wol->sopass[2] = (sopass_val & 0xff);
+		wol->sopass[3] = (sopass_val >> 8);
+
+		sopass_val = phy_read_mmd(phydev, DP83811_DEVADDR,
+					  MII_DP83811_RXSOP3);
+		wol->sopass[4] = (sopass_val & 0xff);
+		wol->sopass[5] = (sopass_val >> 8);
+
+		wol->wolopts |= WAKE_MAGICSECURE;
+	}
+
+	/* WoL is not enabled so set wolopts to 0 */
+	if (!(value & DP83811_WOL_EN))
+		wol->wolopts = 0;
+}
+
+static int dp83811_config_intr(struct phy_device *phydev)
+{
+	int misr_status, err;
+
+	if (phydev->interrupts == PHY_INTERRUPT_ENABLED) {
+		misr_status = phy_read(phydev, MII_DP83811_INT_STAT1);
+		if (misr_status < 0)
+			return misr_status;
+
+		misr_status |= (DP83811_RX_ERR_HF_INT_EN |
+				DP83811_MS_TRAINING_INT_EN |
+				DP83811_ANEG_COMPLETE_INT_EN |
+				DP83811_ESD_EVENT_INT_EN |
+				DP83811_WOL_INT_EN |
+				DP83811_LINK_STAT_INT_EN |
+				DP83811_ENERGY_DET_INT_EN |
+				DP83811_LINK_QUAL_INT_EN);
+
+		err = phy_write(phydev, MII_DP83811_INT_STAT1, misr_status);
+		if (err < 0)
+			return err;
+
+		misr_status = phy_read(phydev, MII_DP83811_INT_STAT2);
+		if (misr_status < 0)
+			return misr_status;
+
+		misr_status |= (DP83811_JABBER_DET_INT_EN |
+				DP83811_POLARITY_INT_EN |
+				DP83811_SLEEP_MODE_INT_EN |
+				DP83811_OVERTEMP_INT_EN |
+				DP83811_OVERVOLTAGE_INT_EN |
+				DP83811_UNDERVOLTAGE_INT_EN);
+
+		err = phy_write(phydev, MII_DP83811_INT_STAT2, misr_status);
+
+	} else {
+		err = phy_write(phydev, MII_DP83811_INT_STAT1, 0);
+		if (err < 0)
+			return err;
+
+		err = phy_write(phydev, MII_DP83811_INT_STAT1, 0);
+	}
+
+	return err;
+}
+
+static int dp83811_config_aneg(struct phy_device *phydev)
+{
+	int value, err;
+
+	if (phydev->interface == PHY_INTERFACE_MODE_SGMII) {
+		value = phy_read(phydev, MII_DP83811_SGMII_CTRL);
+		if (phydev->autoneg == AUTONEG_ENABLE) {
+			err = phy_write(phydev, MII_DP83811_SGMII_CTRL,
+					(DP83811_SGMII_AUTO_NEG_EN | value));
+			if (err < 0)
+				return err;
+		} else {
+			err = phy_write(phydev, MII_DP83811_SGMII_CTRL,
+					(~DP83811_SGMII_AUTO_NEG_EN & value));
+			if (err < 0)
+				return err;
+		}
+	}
+
+	return genphy_config_aneg(phydev);
+}
+
+static int dp83811_config_init(struct phy_device *phydev)
+{
+	int value, err;
+
+	err = genphy_config_init(phydev);
+	if (err < 0)
+		return err;
+
+	if (phydev->interface == PHY_INTERFACE_MODE_SGMII) {
+		value = phy_read(phydev, MII_DP83811_SGMII_CTRL);
+		if (!(value & DP83811_SGMII_EN)) {
+			err = phy_write(phydev, MII_DP83811_SGMII_CTRL,
+					(DP83811_SGMII_EN | value));
+			if (err < 0)
+				return err;
+		} else {
+			err = phy_write(phydev, MII_DP83811_SGMII_CTRL,
+					(~DP83811_SGMII_EN & value));
+			if (err < 0)
+				return err;
+		}
+	}
+
+	value = DP83811_WOL_MAGIC_EN | DP83811_WOL_SECURE_ON | DP83811_WOL_EN;
+
+	return phy_write_mmd(phydev, DP83811_DEVADDR, MII_DP83811_WOL_CFG,
+	      value);
+}
+
+static int dp83811_phy_reset(struct phy_device *phydev)
+{
+	int err;
+
+	err = phy_write(phydev, MII_DP83811_RESET_CTRL, DP83811_HW_RESET);
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+
+static int dp83811_suspend(struct phy_device *phydev)
+{
+	int value;
+
+	value = phy_read_mmd(phydev, DP83811_DEVADDR, MII_DP83811_WOL_CFG);
+
+	if (!(value & DP83811_WOL_EN))
+		genphy_suspend(phydev);
+
+	return 0;
+}
+
+static int dp83811_resume(struct phy_device *phydev)
+{
+	int value;
+
+	genphy_resume(phydev);
+
+	value = phy_read_mmd(phydev, DP83811_DEVADDR, MII_DP83811_WOL_CFG);
+
+	phy_write_mmd(phydev, DP83811_DEVADDR, MII_DP83811_WOL_CFG, value |
+		      DP83811_WOL_CLR_INDICATION);
+
+	return 0;
+}
+
+static struct phy_driver dp83811_driver[] = {
+	{
+		.phy_id = DP83TC811_PHY_ID,
+		.phy_id_mask = 0xfffffff0,
+		.name = "TI DP83TC811",
+		.features = PHY_BASIC_FEATURES,
+		.flags = PHY_HAS_INTERRUPT,
+		.config_init = dp83811_config_init,
+		.config_aneg = dp83811_config_aneg,
+		.soft_reset = dp83811_phy_reset,
+		.get_wol = dp83811_get_wol,
+		.set_wol = dp83811_set_wol,
+		.ack_interrupt = dp83811_ack_interrupt,
+		.config_intr = dp83811_config_intr,
+		.suspend = dp83811_suspend,
+		.resume = dp83811_resume,
+	 },
+};
+module_phy_driver(dp83811_driver);
+
+static struct mdio_device_id __maybe_unused dp83811_tbl[] = {
+	{ DP83TC811_PHY_ID, 0xfffffff0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(mdio, dp83811_tbl);
+
+MODULE_DESCRIPTION("Texas Instruments DP83TC811 PHY driver");
+MODULE_AUTHOR("Dan Murphy <dmurphy@ti.com");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c
index 25e2a099b71c..b8f57e9b9379 100644
--- a/drivers/net/phy/marvell.c
+++ b/drivers/net/phy/marvell.c
@@ -1482,9 +1482,6 @@ static void marvell_get_strings(struct phy_device *phydev, u8 *data)
 	}
 }
 
-#ifndef UINT64_MAX
-#define UINT64_MAX		(u64)(~((u64)0))
-#endif
 static u64 marvell_get_stat(struct phy_device *phydev, int i)
 {
 	struct marvell_hw_stat stat = marvell_hw_stats[i];
@@ -1494,7 +1491,7 @@ static u64 marvell_get_stat(struct phy_device *phydev, int i)
 
 	val = phy_read_paged(phydev, stat.page, stat.reg);
 	if (val < 0) {
-		ret = UINT64_MAX;
+		ret = U64_MAX;
 	} else {
 		val = val & ((1 << stat.bits) - 1);
 		priv->stats[i] += val;
diff --git a/drivers/net/phy/mdio-bitbang.c b/drivers/net/phy/mdio-bitbang.c
index 403b085f0a89..15352f987bdf 100644
--- a/drivers/net/phy/mdio-bitbang.c
+++ b/drivers/net/phy/mdio-bitbang.c
@@ -205,14 +205,6 @@ static int mdiobb_write(struct mii_bus *bus, int phy, int reg, u16 val)
 	return 0;
 }
 
-static int mdiobb_reset(struct mii_bus *bus)
-{
-	struct mdiobb_ctrl *ctrl = bus->priv;
-	if (ctrl->reset)
-		ctrl->reset(bus);
-	return 0;
-}
-
 struct mii_bus *alloc_mdio_bitbang(struct mdiobb_ctrl *ctrl)
 {
 	struct mii_bus *bus;
@@ -225,7 +217,6 @@ struct mii_bus *alloc_mdio_bitbang(struct mdiobb_ctrl *ctrl)
 
 	bus->read = mdiobb_read;
 	bus->write = mdiobb_write;
-	bus->reset = mdiobb_reset;
 	bus->priv = ctrl;
 
 	return bus;
diff --git a/drivers/net/phy/mdio-boardinfo.c b/drivers/net/phy/mdio-boardinfo.c
index 1861f387820d..863496fa5d13 100644
--- a/drivers/net/phy/mdio-boardinfo.c
+++ b/drivers/net/phy/mdio-boardinfo.c
@@ -30,17 +30,20 @@ void mdiobus_setup_mdiodev_from_board_info(struct mii_bus *bus,
 					    struct mdio_board_info *bi))
 {
 	struct mdio_board_entry *be;
+	struct mdio_board_entry *tmp;
 	struct mdio_board_info *bi;
 	int ret;
 
 	mutex_lock(&mdio_board_lock);
-	list_for_each_entry(be, &mdio_board_list, list) {
+	list_for_each_entry_safe(be, tmp, &mdio_board_list, list) {
 		bi = &be->board_info;
 
 		if (strcmp(bus->id, bi->bus_id))
 			continue;
 
+		mutex_unlock(&mdio_board_lock);
 		ret = cb(bus, bi);
+		mutex_lock(&mdio_board_lock);
 		if (ret)
 			continue;
 
diff --git a/drivers/net/phy/mdio-gpio.c b/drivers/net/phy/mdio-gpio.c
index 4333c6e14742..4e4c8daf44c3 100644
--- a/drivers/net/phy/mdio-gpio.c
+++ b/drivers/net/phy/mdio-gpio.c
@@ -24,8 +24,10 @@
 #include <linux/slab.h>
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
+#include <linux/mdio-bitbang.h>
+#include <linux/mdio-gpio.h>
 #include <linux/gpio.h>
-#include <linux/platform_data/mdio-gpio.h>
+#include <linux/gpio/consumer.h>
 
 #include <linux/of_gpio.h>
 #include <linux/of_mdio.h>
@@ -35,37 +37,22 @@ struct mdio_gpio_info {
 	struct gpio_desc *mdc, *mdio, *mdo;
 };
 
-static void *mdio_gpio_of_get_data(struct platform_device *pdev)
+static int mdio_gpio_get_data(struct device *dev,
+			      struct mdio_gpio_info *bitbang)
 {
-	struct device_node *np = pdev->dev.of_node;
-	struct mdio_gpio_platform_data *pdata;
-	enum of_gpio_flags flags;
-	int ret;
-
-	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
-	if (!pdata)
-		return NULL;
-
-	ret = of_get_gpio_flags(np, 0, &flags);
-	if (ret < 0)
-		return NULL;
-
-	pdata->mdc = ret;
-	pdata->mdc_active_low = flags & OF_GPIO_ACTIVE_LOW;
-
-	ret = of_get_gpio_flags(np, 1, &flags);
-	if (ret < 0)
-		return NULL;
-	pdata->mdio = ret;
-	pdata->mdio_active_low = flags & OF_GPIO_ACTIVE_LOW;
-
-	ret = of_get_gpio_flags(np, 2, &flags);
-	if (ret > 0) {
-		pdata->mdo = ret;
-		pdata->mdo_active_low = flags & OF_GPIO_ACTIVE_LOW;
-	}
-
-	return pdata;
+	bitbang->mdc = devm_gpiod_get_index(dev, NULL, MDIO_GPIO_MDC,
+					    GPIOD_OUT_LOW);
+	if (IS_ERR(bitbang->mdc))
+		return PTR_ERR(bitbang->mdc);
+
+	bitbang->mdio = devm_gpiod_get_index(dev, NULL, MDIO_GPIO_MDIO,
+					     GPIOD_IN);
+	if (IS_ERR(bitbang->mdio))
+		return PTR_ERR(bitbang->mdio);
+
+	bitbang->mdo = devm_gpiod_get_index_optional(dev, NULL, MDIO_GPIO_MDO,
+						     GPIOD_OUT_LOW);
+	return PTR_ERR_OR_ZERO(bitbang->mdo);
 }
 
 static void mdio_dir(struct mdiobb_ctrl *ctrl, int dir)
@@ -125,78 +112,28 @@ static const struct mdiobb_ops mdio_gpio_ops = {
 };
 
 static struct mii_bus *mdio_gpio_bus_init(struct device *dev,
-					  struct mdio_gpio_platform_data *pdata,
+					  struct mdio_gpio_info *bitbang,
 					  int bus_id)
 {
 	struct mii_bus *new_bus;
-	struct mdio_gpio_info *bitbang;
-	int i;
-	int mdc, mdio, mdo;
-	unsigned long mdc_flags = GPIOF_OUT_INIT_LOW;
-	unsigned long mdio_flags = GPIOF_DIR_IN;
-	unsigned long mdo_flags = GPIOF_OUT_INIT_HIGH;
-
-	bitbang = devm_kzalloc(dev, sizeof(*bitbang), GFP_KERNEL);
-	if (!bitbang)
-		goto out;
 
 	bitbang->ctrl.ops = &mdio_gpio_ops;
-	bitbang->ctrl.reset = pdata->reset;
-	mdc = pdata->mdc;
-	bitbang->mdc = gpio_to_desc(mdc);
-	if (pdata->mdc_active_low)
-		mdc_flags = GPIOF_OUT_INIT_HIGH | GPIOF_ACTIVE_LOW;
-	mdio = pdata->mdio;
-	bitbang->mdio = gpio_to_desc(mdio);
-	if (pdata->mdio_active_low)
-		mdio_flags |= GPIOF_ACTIVE_LOW;
-	mdo = pdata->mdo;
-	if (mdo) {
-		bitbang->mdo = gpio_to_desc(mdo);
-		if (pdata->mdo_active_low)
-			mdo_flags = GPIOF_OUT_INIT_LOW | GPIOF_ACTIVE_LOW;
-	}
 
 	new_bus = alloc_mdio_bitbang(&bitbang->ctrl);
 	if (!new_bus)
-		goto out;
-
-	new_bus->name = "GPIO Bitbanged MDIO",
+		return NULL;
 
-	new_bus->phy_mask = pdata->phy_mask;
-	new_bus->phy_ignore_ta_mask = pdata->phy_ignore_ta_mask;
-	memcpy(new_bus->irq, pdata->irqs, sizeof(new_bus->irq));
+	new_bus->name = "GPIO Bitbanged MDIO";
 	new_bus->parent = dev;
 
-	if (new_bus->phy_mask == ~0)
-		goto out_free_bus;
-
-	for (i = 0; i < PHY_MAX_ADDR; i++)
-		if (!new_bus->irq[i])
-			new_bus->irq[i] = PHY_POLL;
-
 	if (bus_id != -1)
 		snprintf(new_bus->id, MII_BUS_ID_SIZE, "gpio-%x", bus_id);
 	else
 		strncpy(new_bus->id, "gpio", MII_BUS_ID_SIZE);
 
-	if (devm_gpio_request_one(dev, mdc, mdc_flags, "mdc"))
-		goto out_free_bus;
-
-	if (devm_gpio_request_one(dev, mdio, mdio_flags, "mdio"))
-		goto out_free_bus;
-
-	if (mdo && devm_gpio_request_one(dev, mdo, mdo_flags, "mdo"))
-		goto out_free_bus;
-
 	dev_set_drvdata(dev, new_bus);
 
 	return new_bus;
-
-out_free_bus:
-	free_mdio_bitbang(new_bus);
-out:
-	return NULL;
 }
 
 static void mdio_gpio_bus_deinit(struct device *dev)
@@ -216,34 +153,33 @@ static void mdio_gpio_bus_destroy(struct device *dev)
 
 static int mdio_gpio_probe(struct platform_device *pdev)
 {
-	struct mdio_gpio_platform_data *pdata;
+	struct mdio_gpio_info *bitbang;
 	struct mii_bus *new_bus;
 	int ret, bus_id;
 
+	bitbang = devm_kzalloc(&pdev->dev, sizeof(*bitbang), GFP_KERNEL);
+	if (!bitbang)
+		return -ENOMEM;
+
+	ret = mdio_gpio_get_data(&pdev->dev, bitbang);
+	if (ret)
+		return ret;
+
 	if (pdev->dev.of_node) {
-		pdata = mdio_gpio_of_get_data(pdev);
 		bus_id = of_alias_get_id(pdev->dev.of_node, "mdio-gpio");
 		if (bus_id < 0) {
 			dev_warn(&pdev->dev, "failed to get alias id\n");
 			bus_id = 0;
 		}
 	} else {
-		pdata = dev_get_platdata(&pdev->dev);
 		bus_id = pdev->id;
 	}
 
-	if (!pdata)
-		return -ENODEV;
-
-	new_bus = mdio_gpio_bus_init(&pdev->dev, pdata, bus_id);
+	new_bus = mdio_gpio_bus_init(&pdev->dev, bitbang, bus_id);
 	if (!new_bus)
 		return -ENODEV;
 
-	if (pdev->dev.of_node)
-		ret = of_mdiobus_register(new_bus, pdev->dev.of_node);
-	else
-		ret = mdiobus_register(new_bus);
-
+	ret = of_mdiobus_register(new_bus, pdev->dev.of_node);
 	if (ret)
 		mdio_gpio_bus_deinit(&pdev->dev);
 
diff --git a/drivers/net/phy/mdio-mscc-miim.c b/drivers/net/phy/mdio-mscc-miim.c
new file mode 100644
index 000000000000..badbc99bedd3
--- /dev/null
+++ b/drivers/net/phy/mdio-mscc-miim.c
@@ -0,0 +1,193 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Driver for the MDIO interface of Microsemi network switches.
+ *
+ * Author: Alexandre Belloni <alexandre.belloni@bootlin.com>
+ * Copyright (c) 2017 Microsemi Corporation
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/phy.h>
+#include <linux/platform_device.h>
+#include <linux/bitops.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/of_mdio.h>
+
+#define MSCC_MIIM_REG_STATUS		0x0
+#define		MSCC_MIIM_STATUS_STAT_BUSY	BIT(3)
+#define MSCC_MIIM_REG_CMD		0x8
+#define		MSCC_MIIM_CMD_OPR_WRITE		BIT(1)
+#define		MSCC_MIIM_CMD_OPR_READ		BIT(2)
+#define		MSCC_MIIM_CMD_WRDATA_SHIFT	4
+#define		MSCC_MIIM_CMD_REGAD_SHIFT	20
+#define		MSCC_MIIM_CMD_PHYAD_SHIFT	25
+#define		MSCC_MIIM_CMD_VLD		BIT(31)
+#define MSCC_MIIM_REG_DATA		0xC
+#define		MSCC_MIIM_DATA_ERROR		(BIT(16) | BIT(17))
+
+#define MSCC_PHY_REG_PHY_CFG	0x0
+#define		PHY_CFG_PHY_ENA		(BIT(0) | BIT(1) | BIT(2) | BIT(3))
+#define		PHY_CFG_PHY_COMMON_RESET BIT(4)
+#define		PHY_CFG_PHY_RESET	(BIT(5) | BIT(6) | BIT(7) | BIT(8))
+#define MSCC_PHY_REG_PHY_STATUS	0x4
+
+struct mscc_miim_dev {
+	void __iomem *regs;
+	void __iomem *phy_regs;
+};
+
+static int mscc_miim_wait_ready(struct mii_bus *bus)
+{
+	struct mscc_miim_dev *miim = bus->priv;
+	u32 val;
+
+	readl_poll_timeout(miim->regs + MSCC_MIIM_REG_STATUS, val,
+			   !(val & MSCC_MIIM_STATUS_STAT_BUSY), 100, 250000);
+	if (val & MSCC_MIIM_STATUS_STAT_BUSY)
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+static int mscc_miim_read(struct mii_bus *bus, int mii_id, int regnum)
+{
+	struct mscc_miim_dev *miim = bus->priv;
+	u32 val;
+	int ret;
+
+	ret = mscc_miim_wait_ready(bus);
+	if (ret)
+		goto out;
+
+	writel(MSCC_MIIM_CMD_VLD | (mii_id << MSCC_MIIM_CMD_PHYAD_SHIFT) |
+	       (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) | MSCC_MIIM_CMD_OPR_READ,
+	       miim->regs + MSCC_MIIM_REG_CMD);
+
+	ret = mscc_miim_wait_ready(bus);
+	if (ret)
+		goto out;
+
+	val = readl(miim->regs + MSCC_MIIM_REG_DATA);
+	if (val & MSCC_MIIM_DATA_ERROR) {
+		ret = -EIO;
+		goto out;
+	}
+
+	ret = val & 0xFFFF;
+out:
+	return ret;
+}
+
+static int mscc_miim_write(struct mii_bus *bus, int mii_id,
+			   int regnum, u16 value)
+{
+	struct mscc_miim_dev *miim = bus->priv;
+	int ret;
+
+	ret = mscc_miim_wait_ready(bus);
+	if (ret < 0)
+		goto out;
+
+	writel(MSCC_MIIM_CMD_VLD | (mii_id << MSCC_MIIM_CMD_PHYAD_SHIFT) |
+	       (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) |
+	       (value << MSCC_MIIM_CMD_WRDATA_SHIFT) |
+	       MSCC_MIIM_CMD_OPR_WRITE,
+	       miim->regs + MSCC_MIIM_REG_CMD);
+
+out:
+	return ret;
+}
+
+static int mscc_miim_reset(struct mii_bus *bus)
+{
+	struct mscc_miim_dev *miim = bus->priv;
+
+	if (miim->phy_regs) {
+		writel(0, miim->phy_regs + MSCC_PHY_REG_PHY_CFG);
+		writel(0x1ff, miim->phy_regs + MSCC_PHY_REG_PHY_CFG);
+		mdelay(500);
+	}
+
+	return 0;
+}
+
+static int mscc_miim_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	struct mii_bus *bus;
+	struct mscc_miim_dev *dev;
+	int ret;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENODEV;
+
+	bus = devm_mdiobus_alloc_size(&pdev->dev, sizeof(*dev));
+	if (!bus)
+		return -ENOMEM;
+
+	bus->name = "mscc_miim";
+	bus->read = mscc_miim_read;
+	bus->write = mscc_miim_write;
+	bus->reset = mscc_miim_reset;
+	snprintf(bus->id, MII_BUS_ID_SIZE, "%s-mii", dev_name(&pdev->dev));
+	bus->parent = &pdev->dev;
+
+	dev = bus->priv;
+	dev->regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(dev->regs)) {
+		dev_err(&pdev->dev, "Unable to map MIIM registers\n");
+		return PTR_ERR(dev->regs);
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	if (res) {
+		dev->phy_regs = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(dev->phy_regs)) {
+			dev_err(&pdev->dev, "Unable to map internal phy registers\n");
+			return PTR_ERR(dev->phy_regs);
+		}
+	}
+
+	ret = of_mdiobus_register(bus, pdev->dev.of_node);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Cannot register MDIO bus (%d)\n", ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, bus);
+
+	return 0;
+}
+
+static int mscc_miim_remove(struct platform_device *pdev)
+{
+	struct mii_bus *bus = platform_get_drvdata(pdev);
+
+	mdiobus_unregister(bus);
+
+	return 0;
+}
+
+static const struct of_device_id mscc_miim_match[] = {
+	{ .compatible = "mscc,ocelot-miim" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mscc_miim_match);
+
+static struct platform_driver mscc_miim_driver = {
+	.probe = mscc_miim_probe,
+	.remove = mscc_miim_remove,
+	.driver = {
+		.name = "mscc-miim",
+		.of_match_table = mscc_miim_match,
+	},
+};
+
+module_platform_driver(mscc_miim_driver);
+
+MODULE_DESCRIPTION("Microsemi MIIM driver");
+MODULE_AUTHOR("Alexandre Belloni <alexandre.belloni@bootlin.com>");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c
index ab195f0916d6..3db06b40580d 100644
--- a/drivers/net/phy/micrel.c
+++ b/drivers/net/phy/micrel.c
@@ -681,9 +681,6 @@ static void kszphy_get_strings(struct phy_device *phydev, u8 *data)
 	}
 }
 
-#ifndef UINT64_MAX
-#define UINT64_MAX              (u64)(~((u64)0))
-#endif
 static u64 kszphy_get_stat(struct phy_device *phydev, int i)
 {
 	struct kszphy_hw_stat stat = kszphy_hw_stats[i];
@@ -693,7 +690,7 @@ static u64 kszphy_get_stat(struct phy_device *phydev, int i)
 
 	val = phy_read(phydev, stat.reg);
 	if (val < 0) {
-		ret = UINT64_MAX;
+		ret = U64_MAX;
 	} else {
 		val = val & ((1 << stat.bits) - 1);
 		priv->stats[i] += val;
diff --git a/drivers/net/phy/microchip.c b/drivers/net/phy/microchip.c
index a97ac8c12c4c..2d67937866a3 100644
--- a/drivers/net/phy/microchip.c
+++ b/drivers/net/phy/microchip.c
@@ -21,6 +21,8 @@
 #include <linux/phy.h>
 #include <linux/microchipphy.h>
 #include <linux/delay.h>
+#include <linux/of.h>
+#include <dt-bindings/net/microchip-lan78xx.h>
 
 #define DRIVER_AUTHOR	"WOOJUNG HUH <woojung.huh@microchip.com>"
 #define DRIVER_DESC	"Microchip LAN88XX PHY driver"
@@ -225,6 +227,8 @@ static int lan88xx_probe(struct phy_device *phydev)
 {
 	struct device *dev = &phydev->mdio.dev;
 	struct lan88xx_priv *priv;
+	u32 led_modes[4];
+	int len;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -232,6 +236,27 @@ static int lan88xx_probe(struct phy_device *phydev)
 
 	priv->wolopts = 0;
 
+	len = of_property_read_variable_u32_array(dev->of_node,
+						  "microchip,led-modes",
+						  led_modes,
+						  0,
+						  ARRAY_SIZE(led_modes));
+	if (len >= 0) {
+		u32 reg = 0;
+		int i;
+
+		for (i = 0; i < len; i++) {
+			if (led_modes[i] > 15)
+				return -EINVAL;
+			reg |= led_modes[i] << (i * 4);
+		}
+		for (; i < ARRAY_SIZE(led_modes); i++)
+			reg |= LAN78XX_FORCE_LED_OFF << (i * 4);
+		(void)phy_write(phydev, LAN78XX_PHY_LED_MODE_SELECT, reg);
+	} else if (len == -EOVERFLOW) {
+		return -EINVAL;
+	}
+
 	/* these values can be used to identify internal PHY */
 	priv->chip_id = phy_read_mmd(phydev, 3, LAN88XX_MMD3_CHIP_ID);
 	priv->chip_rev = phy_read_mmd(phydev, 3, LAN88XX_MMD3_CHIP_REV);
diff --git a/drivers/net/phy/microchip_t1.c b/drivers/net/phy/microchip_t1.c
new file mode 100644
index 000000000000..b1917dd1978a
--- /dev/null
+++ b/drivers/net/phy/microchip_t1.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2018 Microchip Technology
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mii.h>
+#include <linux/phy.h>
+
+/* Interrupt Source Register */
+#define LAN87XX_INTERRUPT_SOURCE                (0x18)
+
+/* Interrupt Mask Register */
+#define LAN87XX_INTERRUPT_MASK                  (0x19)
+#define LAN87XX_MASK_LINK_UP                    (0x0004)
+#define LAN87XX_MASK_LINK_DOWN                  (0x0002)
+
+#define DRIVER_AUTHOR	"Nisar Sayed <nisar.sayed@microchip.com>"
+#define DRIVER_DESC	"Microchip LAN87XX T1 PHY driver"
+
+static int lan87xx_phy_config_intr(struct phy_device *phydev)
+{
+	int rc, val = 0;
+
+	if (phydev->interrupts == PHY_INTERRUPT_ENABLED) {
+		/* unmask all source and clear them before enable */
+		rc = phy_write(phydev, LAN87XX_INTERRUPT_MASK, 0x7FFF);
+		rc = phy_read(phydev, LAN87XX_INTERRUPT_SOURCE);
+		val = LAN87XX_MASK_LINK_UP | LAN87XX_MASK_LINK_DOWN;
+	}
+
+	rc = phy_write(phydev, LAN87XX_INTERRUPT_MASK, val);
+
+	return rc < 0 ? rc : 0;
+}
+
+static int lan87xx_phy_ack_interrupt(struct phy_device *phydev)
+{
+	int rc = phy_read(phydev, LAN87XX_INTERRUPT_SOURCE);
+
+	return rc < 0 ? rc : 0;
+}
+
+static struct phy_driver microchip_t1_phy_driver[] = {
+	{
+		.phy_id         = 0x0007c150,
+		.phy_id_mask    = 0xfffffff0,
+		.name           = "Microchip LAN87xx T1",
+
+		.features       = SUPPORTED_100baseT_Full,
+		.flags          = PHY_HAS_INTERRUPT,
+
+		.config_init    = genphy_config_init,
+		.config_aneg    = genphy_config_aneg,
+
+		.ack_interrupt  = lan87xx_phy_ack_interrupt,
+		.config_intr    = lan87xx_phy_config_intr,
+
+		.suspend        = genphy_suspend,
+		.resume         = genphy_resume,
+	}
+};
+
+module_phy_driver(microchip_t1_phy_driver);
+
+static struct mdio_device_id __maybe_unused microchip_t1_tbl[] = {
+	{ 0x0007c150, 0xfffffff0 },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(mdio, microchip_t1_tbl);
+
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c
index c582b2d7546c..af4dc4425be2 100644
--- a/drivers/net/phy/phylink.c
+++ b/drivers/net/phy/phylink.c
@@ -19,6 +19,7 @@
 #include <linux/phylink.h>
 #include <linux/rtnetlink.h>
 #include <linux/spinlock.h>
+#include <linux/timer.h>
 #include <linux/workqueue.h>
 
 #include "sfp.h"
@@ -54,6 +55,7 @@ struct phylink {
 	/* The link configuration settings */
 	struct phylink_link_state link_config;
 	struct gpio_desc *link_gpio;
+	struct timer_list link_poll;
 	void (*get_fixed_state)(struct net_device *dev,
 				struct phylink_link_state *s);
 
@@ -360,7 +362,7 @@ static void phylink_get_fixed_state(struct phylink *pl, struct phylink_link_stat
 	if (pl->get_fixed_state)
 		pl->get_fixed_state(pl->netdev, state);
 	else if (pl->link_gpio)
-		state->link = !!gpiod_get_value(pl->link_gpio);
+		state->link = !!gpiod_get_value_cansleep(pl->link_gpio);
 }
 
 /* Flow control is resolved according to our and the link partners
@@ -500,6 +502,15 @@ static void phylink_run_resolve(struct phylink *pl)
 		queue_work(system_power_efficient_wq, &pl->resolve);
 }
 
+static void phylink_fixed_poll(struct timer_list *t)
+{
+	struct phylink *pl = container_of(t, struct phylink, link_poll);
+
+	mod_timer(t, jiffies + HZ);
+
+	phylink_run_resolve(pl);
+}
+
 static const struct sfp_upstream_ops sfp_phylink_ops;
 
 static int phylink_register_sfp(struct phylink *pl,
@@ -572,6 +583,7 @@ struct phylink *phylink_create(struct net_device *ndev,
 	pl->link_config.an_enabled = true;
 	pl->ops = ops;
 	__set_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state);
+	timer_setup(&pl->link_poll, phylink_fixed_poll, 0);
 
 	bitmap_fill(pl->supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
 	linkmode_copy(pl->link_config.advertising, pl->supported);
@@ -612,6 +624,8 @@ void phylink_destroy(struct phylink *pl)
 {
 	if (pl->sfp_bus)
 		sfp_unregister_upstream(pl->sfp_bus);
+	if (!IS_ERR_OR_NULL(pl->link_gpio))
+		gpiod_put(pl->link_gpio);
 
 	cancel_work_sync(&pl->resolve);
 	kfree(pl);
@@ -903,6 +917,8 @@ void phylink_start(struct phylink *pl)
 	clear_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state);
 	phylink_run_resolve(pl);
 
+	if (pl->link_an_mode == MLO_AN_FIXED && !IS_ERR(pl->link_gpio))
+		mod_timer(&pl->link_poll, jiffies + HZ);
 	if (pl->sfp_bus)
 		sfp_upstream_start(pl->sfp_bus);
 	if (pl->phydev)
@@ -927,6 +943,8 @@ void phylink_stop(struct phylink *pl)
 		phy_stop(pl->phydev);
 	if (pl->sfp_bus)
 		sfp_upstream_stop(pl->sfp_bus);
+	if (pl->link_an_mode == MLO_AN_FIXED && !IS_ERR(pl->link_gpio))
+		del_timer_sync(&pl->link_poll);
 
 	set_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state);
 	queue_work(system_power_efficient_wq, &pl->resolve);
diff --git a/drivers/net/phy/realtek.c b/drivers/net/phy/realtek.c
index 9f48ecf9c627..082fb40c656d 100644
--- a/drivers/net/phy/realtek.c
+++ b/drivers/net/phy/realtek.c
@@ -145,6 +145,20 @@ static int rtl8211f_config_init(struct phy_device *phydev)
 	return phy_modify_paged(phydev, 0xd08, 0x11, RTL8211F_TX_DELAY, val);
 }
 
+static int rtl8211b_suspend(struct phy_device *phydev)
+{
+	phy_write(phydev, MII_MMD_DATA, BIT(9));
+
+	return genphy_suspend(phydev);
+}
+
+static int rtl8211b_resume(struct phy_device *phydev)
+{
+	phy_write(phydev, MII_MMD_DATA, 0);
+
+	return genphy_resume(phydev);
+}
+
 static struct phy_driver realtek_drvs[] = {
 	{
 		.phy_id         = 0x00008201,
@@ -174,6 +188,8 @@ static struct phy_driver realtek_drvs[] = {
 		.config_intr	= &rtl8211b_config_intr,
 		.read_mmd	= &genphy_read_mmd_unsupported,
 		.write_mmd	= &genphy_write_mmd_unsupported,
+		.suspend	= rtl8211b_suspend,
+		.resume		= rtl8211b_resume,
 	}, {
 		.phy_id		= 0x001cc914,
 		.name		= "RTL8211DN Gigabit Ethernet",
diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c
index fd6c23f69c2f..d437f4f5ed52 100644
--- a/drivers/net/phy/sfp-bus.c
+++ b/drivers/net/phy/sfp-bus.c
@@ -132,6 +132,13 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 			br_max = br_nom + br_nom * id->ext.br_min / 100;
 			br_min = br_nom - br_nom * id->ext.br_min / 100;
 		}
+
+		/* When using passive cables, in case neither BR,min nor BR,max
+		 * are specified, set br_min to 0 as the nominal value is then
+		 * used as the maximum.
+		 */
+		if (br_min == br_max && id->base.sfp_ct_passive)
+			br_min = 0;
 	}
 
 	/* Set ethtool support from the compliance fields. */
diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c
index 4ab6e9a50bbe..c4c92db86dfa 100644
--- a/drivers/net/phy/sfp.c
+++ b/drivers/net/phy/sfp.c
@@ -976,6 +976,7 @@ static int sfp_probe(struct platform_device *pdev)
 	if (pdev->dev.of_node) {
 		struct device_node *node = pdev->dev.of_node;
 		const struct of_device_id *id;
+		struct i2c_adapter *i2c;
 		struct device_node *np;
 
 		id = of_match_node(sfp_of_match, node);
@@ -985,19 +986,20 @@ static int sfp_probe(struct platform_device *pdev)
 		sff = sfp->type = id->data;
 
 		np = of_parse_phandle(node, "i2c-bus", 0);
-		if (np) {
-			struct i2c_adapter *i2c;
-
-			i2c = of_find_i2c_adapter_by_node(np);
-			of_node_put(np);
-			if (!i2c)
-				return -EPROBE_DEFER;
-
-			err = sfp_i2c_configure(sfp, i2c);
-			if (err < 0) {
-				i2c_put_adapter(i2c);
-				return err;
-			}
+		if (!np) {
+			dev_err(sfp->dev, "missing 'i2c-bus' property\n");
+			return -ENODEV;
+		}
+
+		i2c = of_find_i2c_adapter_by_node(np);
+		of_node_put(np);
+		if (!i2c)
+			return -EPROBE_DEFER;
+
+		err = sfp_i2c_configure(sfp, i2c);
+		if (err < 0) {
+			i2c_put_adapter(i2c);
+			return err;
 		}
 	}
 
@@ -1065,6 +1067,15 @@ static int sfp_probe(struct platform_device *pdev)
 	if (poll)
 		mod_delayed_work(system_wq, &sfp->poll, poll_jiffies);
 
+	/* We could have an issue in cases no Tx disable pin is available or
+	 * wired as modules using a laser as their light source will continue to
+	 * be active when the fiber is removed. This could be a safety issue and
+	 * we should at least warn the user about that.
+	 */
+	if (!sfp->gpio[GPIO_TX_DISABLE])
+		dev_warn(sfp->dev,
+			 "No tx_disable pin: SFP modules will always be emitting.\n");
+
 	return 0;
 }
 
diff --git a/drivers/net/phy/smsc.c b/drivers/net/phy/smsc.c
index be399d645224..c328208388da 100644
--- a/drivers/net/phy/smsc.c
+++ b/drivers/net/phy/smsc.c
@@ -168,9 +168,6 @@ static void smsc_get_strings(struct phy_device *phydev, u8 *data)
 	}
 }
 
-#ifndef UINT64_MAX
-#define UINT64_MAX              (u64)(~((u64)0))
-#endif
 static u64 smsc_get_stat(struct phy_device *phydev, int i)
 {
 	struct smsc_hw_stat stat = smsc_hw_stats[i];
@@ -179,7 +176,7 @@ static u64 smsc_get_stat(struct phy_device *phydev, int i)
 
 	val = phy_read(phydev, stat.reg);
 	if (val < 0)
-		ret = UINT64_MAX;
+		ret = U64_MAX;
 	else
 		ret = val;
 
diff --git a/drivers/net/ppp/pppoe.c b/drivers/net/ppp/pppoe.c
index 7df07337d69c..de51e8f70f44 100644
--- a/drivers/net/ppp/pppoe.c
+++ b/drivers/net/ppp/pppoe.c
@@ -1096,21 +1096,6 @@ static const struct seq_operations pppoe_seq_ops = {
 	.stop		= pppoe_seq_stop,
 	.show		= pppoe_seq_show,
 };
-
-static int pppoe_seq_open(struct inode *inode, struct file *file)
-{
-	return seq_open_net(inode, file, &pppoe_seq_ops,
-			sizeof(struct seq_net_private));
-}
-
-static const struct file_operations pppoe_seq_fops = {
-	.owner		= THIS_MODULE,
-	.open		= pppoe_seq_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release_net,
-};
-
 #endif /* CONFIG_PROC_FS */
 
 static const struct proto_ops pppoe_ops = {
@@ -1122,7 +1107,7 @@ static const struct proto_ops pppoe_ops = {
 	.socketpair	= sock_no_socketpair,
 	.accept		= sock_no_accept,
 	.getname	= pppoe_getname,
-	.poll		= datagram_poll,
+	.poll_mask	= datagram_poll_mask,
 	.listen		= sock_no_listen,
 	.shutdown	= sock_no_shutdown,
 	.setsockopt	= sock_no_setsockopt,
@@ -1146,7 +1131,8 @@ static __net_init int pppoe_init_net(struct net *net)
 
 	rwlock_init(&pn->hash_lock);
 
-	pde = proc_create("pppoe", 0444, net->proc_net, &pppoe_seq_fops);
+	pde = proc_create_net("pppoe", 0444, net->proc_net,
+			&pppoe_seq_ops, sizeof(struct seq_net_private));
 #ifdef CONFIG_PROC_FS
 	if (!pde)
 		return -ENOMEM;
diff --git a/drivers/net/ppp/pptp.c b/drivers/net/ppp/pptp.c
index c4267ecefd85..157b67c1bf8e 100644
--- a/drivers/net/ppp/pptp.c
+++ b/drivers/net/ppp/pptp.c
@@ -624,7 +624,6 @@ static const struct proto_ops pptp_ops = {
 	.socketpair = sock_no_socketpair,
 	.accept     = sock_no_accept,
 	.getname    = pptp_getname,
-	.poll       = sock_no_poll,
 	.listen     = sock_no_listen,
 	.shutdown   = sock_no_shutdown,
 	.setsockopt = sock_no_setsockopt,
diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c
index ddb6bf85a59c..267dcc929f6c 100644
--- a/drivers/net/team/team.c
+++ b/drivers/net/team/team.c
@@ -1026,7 +1026,8 @@ static void __team_compute_features(struct team *team)
 	}
 
 	team->dev->vlan_features = vlan_features;
-	team->dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL;
+	team->dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL |
+				     NETIF_F_GSO_UDP_L4;
 	team->dev->hard_header_len = max_hard_header_len;
 
 	team->dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
@@ -1128,6 +1129,7 @@ static int team_upper_dev_link(struct team *team, struct team_port *port,
 	int err;
 
 	lag_upper_info.tx_type = team->mode->lag_tx_type;
+	lag_upper_info.hash_type = NETDEV_LAG_HASH_UNKNOWN;
 	err = netdev_master_upper_dev_link(port->dev, team->dev, NULL,
 					   &lag_upper_info, extack);
 	if (err)
@@ -2117,7 +2119,7 @@ static void team_setup(struct net_device *dev)
 			   NETIF_F_HW_VLAN_CTAG_RX |
 			   NETIF_F_HW_VLAN_CTAG_FILTER;
 
-	dev->hw_features |= NETIF_F_GSO_ENCAP_ALL;
+	dev->hw_features |= NETIF_F_GSO_ENCAP_ALL | NETIF_F_GSO_UDP_L4;
 	dev->features |= dev->hw_features;
 }
 
@@ -2424,7 +2426,6 @@ static int team_nl_send_options_get(struct team *team, u32 portid, u32 seq,
 nla_put_failure:
 	err = -EMSGSIZE;
 errout:
-	genlmsg_cancel(skb, hdr);
 	nlmsg_free(skb);
 	return err;
 }
@@ -2718,7 +2719,6 @@ static int team_nl_send_port_list_get(struct team *team, u32 portid, u32 seq,
 nla_put_failure:
 	err = -EMSGSIZE;
 errout:
-	genlmsg_cancel(skb, hdr);
 	nlmsg_free(skb);
 	return err;
 }
@@ -2942,7 +2942,7 @@ static int team_device_event(struct notifier_block *unused,
 	case NETDEV_CHANGE:
 		if (netif_running(port->dev))
 			team_port_change_check(port,
-					       !!netif_carrier_ok(port->dev));
+					       !!netif_oper_up(port->dev));
 		break;
 	case NETDEV_UNREGISTER:
 		team_del_slave(port->team->dev, dev);
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index 45d807796a18..c94fffee5ea9 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -70,6 +70,7 @@
 #include <net/netns/generic.h>
 #include <net/rtnetlink.h>
 #include <net/sock.h>
+#include <net/xdp.h>
 #include <linux/seq_file.h>
 #include <linux/uio.h>
 #include <linux/skb_array.h>
@@ -248,11 +249,11 @@ struct veth {
 	__be16 h_vlan_TCI;
 };
 
-bool tun_is_xdp_buff(void *ptr)
+bool tun_is_xdp_frame(void *ptr)
 {
 	return (unsigned long)ptr & TUN_XDP_FLAG;
 }
-EXPORT_SYMBOL(tun_is_xdp_buff);
+EXPORT_SYMBOL(tun_is_xdp_frame);
 
 void *tun_xdp_to_ptr(void *ptr)
 {
@@ -525,11 +526,6 @@ static void tun_flow_update(struct tun_struct *tun, u32 rxhash,
 
 	rcu_read_lock();
 
-	/* We may get a very small possibility of OOO during switching, not
-	 * worth to optimize.*/
-	if (tun->numqueues == 1 || tfile->detached)
-		goto unlock;
-
 	e = tun_flow_find(head, rxhash);
 	if (likely(e)) {
 		/* TODO: keep queueing to old queue until it's empty? */
@@ -548,7 +544,6 @@ static void tun_flow_update(struct tun_struct *tun, u32 rxhash,
 		spin_unlock_bh(&tun->lock);
 	}
 
-unlock:
 	rcu_read_unlock();
 }
 
@@ -660,10 +655,10 @@ void tun_ptr_free(void *ptr)
 {
 	if (!ptr)
 		return;
-	if (tun_is_xdp_buff(ptr)) {
-		struct xdp_buff *xdp = tun_ptr_to_xdp(ptr);
+	if (tun_is_xdp_frame(ptr)) {
+		struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
 
-		put_page(virt_to_head_page(xdp->data));
+		xdp_return_frame(xdpf);
 	} else {
 		__skb_array_destroy_skb(ptr);
 	}
@@ -848,6 +843,12 @@ static int tun_attach(struct tun_struct *tun, struct file *file,
 				       tun->dev, tfile->queue_index);
 		if (err < 0)
 			goto out;
+		err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq,
+						 MEM_TYPE_PAGE_SHARED, NULL);
+		if (err < 0) {
+			xdp_rxq_info_unreg(&tfile->xdp_rxq);
+			goto out;
+		}
 		err = 0;
 	}
 
@@ -1284,42 +1285,54 @@ static const struct net_device_ops tun_netdev_ops = {
 	.ndo_get_stats64	= tun_net_get_stats64,
 };
 
-static int tun_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
+static int tun_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames)
 {
 	struct tun_struct *tun = netdev_priv(dev);
-	struct xdp_buff *buff = xdp->data_hard_start;
-	int headroom = xdp->data - xdp->data_hard_start;
 	struct tun_file *tfile;
 	u32 numqueues;
-	int ret = 0;
-
-	/* Assure headroom is available and buff is properly aligned */
-	if (unlikely(headroom < sizeof(*xdp) || tun_is_xdp_buff(xdp)))
-		return -ENOSPC;
-
-	*buff = *xdp;
+	int drops = 0;
+	int cnt = n;
+	int i;
 
 	rcu_read_lock();
 
 	numqueues = READ_ONCE(tun->numqueues);
 	if (!numqueues) {
-		ret = -ENOSPC;
-		goto out;
+		rcu_read_unlock();
+		return -ENXIO; /* Caller will free/return all frames */
 	}
 
 	tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
 					    numqueues]);
-	/* Encode the XDP flag into lowest bit for consumer to differ
-	 * XDP buffer from sk_buff.
-	 */
-	if (ptr_ring_produce(&tfile->tx_ring, tun_xdp_to_ptr(buff))) {
-		this_cpu_inc(tun->pcpu_stats->tx_dropped);
-		ret = -ENOSPC;
+
+	spin_lock(&tfile->tx_ring.producer_lock);
+	for (i = 0; i < n; i++) {
+		struct xdp_frame *xdp = frames[i];
+		/* Encode the XDP flag into lowest bit for consumer to differ
+		 * XDP buffer from sk_buff.
+		 */
+		void *frame = tun_xdp_to_ptr(xdp);
+
+		if (__ptr_ring_produce(&tfile->tx_ring, frame)) {
+			this_cpu_inc(tun->pcpu_stats->tx_dropped);
+			xdp_return_frame_rx_napi(xdp);
+			drops++;
+		}
 	}
+	spin_unlock(&tfile->tx_ring.producer_lock);
 
-out:
 	rcu_read_unlock();
-	return ret;
+	return cnt - drops;
+}
+
+static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)
+{
+	struct xdp_frame *frame = convert_to_xdp_frame(xdp);
+
+	if (unlikely(!frame))
+		return -EOVERFLOW;
+
+	return tun_xdp_xmit(dev, 1, &frame);
 }
 
 static void tun_xdp_flush(struct net_device *dev)
@@ -1650,7 +1663,7 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun,
 	else
 		*skb_xdp = 0;
 
-	preempt_disable();
+	local_bh_disable();
 	rcu_read_lock();
 	xdp_prog = rcu_dereference(tun->xdp_prog);
 	if (xdp_prog && !*skb_xdp) {
@@ -1675,19 +1688,20 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun,
 			if (err)
 				goto err_redirect;
 			rcu_read_unlock();
-			preempt_enable();
+			local_bh_enable();
 			return NULL;
 		case XDP_TX:
 			get_page(alloc_frag->page);
 			alloc_frag->offset += buflen;
-			if (tun_xdp_xmit(tun->dev, &xdp))
+			if (tun_xdp_tx(tun->dev, &xdp))
 				goto err_redirect;
 			tun_xdp_flush(tun->dev);
 			rcu_read_unlock();
-			preempt_enable();
+			local_bh_enable();
 			return NULL;
 		case XDP_PASS:
 			delta = orig_data - xdp.data;
+			len = xdp.data_end - xdp.data;
 			break;
 		default:
 			bpf_warn_invalid_xdp_action(act);
@@ -1703,17 +1717,17 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun,
 	skb = build_skb(buf, buflen);
 	if (!skb) {
 		rcu_read_unlock();
-		preempt_enable();
+		local_bh_enable();
 		return ERR_PTR(-ENOMEM);
 	}
 
 	skb_reserve(skb, pad - delta);
-	skb_put(skb, len + delta);
+	skb_put(skb, len);
 	get_page(alloc_frag->page);
 	alloc_frag->offset += buflen;
 
 	rcu_read_unlock();
-	preempt_enable();
+	local_bh_enable();
 
 	return skb;
 
@@ -1721,7 +1735,7 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun,
 	put_page(alloc_frag->page);
 err_xdp:
 	rcu_read_unlock();
-	preempt_enable();
+	local_bh_enable();
 	this_cpu_inc(tun->pcpu_stats->rx_dropped);
 	return NULL;
 }
@@ -1917,22 +1931,28 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
 		struct bpf_prog *xdp_prog;
 		int ret;
 
+		local_bh_disable();
 		rcu_read_lock();
 		xdp_prog = rcu_dereference(tun->xdp_prog);
 		if (xdp_prog) {
 			ret = do_xdp_generic(xdp_prog, skb);
 			if (ret != XDP_PASS) {
 				rcu_read_unlock();
+				local_bh_enable();
 				return total_len;
 			}
 		}
 		rcu_read_unlock();
+		local_bh_enable();
 	}
 
-	rcu_read_lock();
-	if (!rcu_dereference(tun->steering_prog))
+	/* Compute the costly rx hash only if needed for flow updates.
+	 * We may get a very small possibility of OOO during switching, not
+	 * worth to optimize.
+	 */
+	if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 &&
+	    !tfile->detached)
 		rxhash = __skb_get_hash_symmetric(skb);
-	rcu_read_unlock();
 
 	if (frags) {
 		/* Exercise flow dissector code path. */
@@ -2001,11 +2021,11 @@ static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from)
 
 static ssize_t tun_put_user_xdp(struct tun_struct *tun,
 				struct tun_file *tfile,
-				struct xdp_buff *xdp,
+				struct xdp_frame *xdp_frame,
 				struct iov_iter *iter)
 {
 	int vnet_hdr_sz = 0;
-	size_t size = xdp->data_end - xdp->data;
+	size_t size = xdp_frame->len;
 	struct tun_pcpu_stats *stats;
 	size_t ret;
 
@@ -2021,7 +2041,7 @@ static ssize_t tun_put_user_xdp(struct tun_struct *tun,
 		iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
 	}
 
-	ret = copy_to_iter(xdp->data, size, iter) + vnet_hdr_sz;
+	ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz;
 
 	stats = get_cpu_ptr(tun->pcpu_stats);
 	u64_stats_update_begin(&stats->syncp);
@@ -2189,11 +2209,11 @@ static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
 			return err;
 	}
 
-	if (tun_is_xdp_buff(ptr)) {
-		struct xdp_buff *xdp = tun_ptr_to_xdp(ptr);
+	if (tun_is_xdp_frame(ptr)) {
+		struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
 
-		ret = tun_put_user_xdp(tun, tfile, xdp, to);
-		put_page(virt_to_head_page(xdp->data));
+		ret = tun_put_user_xdp(tun, tfile, xdpf, to);
+		xdp_return_frame(xdpf);
 	} else {
 		struct sk_buff *skb = ptr;
 
@@ -2432,10 +2452,10 @@ static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
 static int tun_ptr_peek_len(void *ptr)
 {
 	if (likely(ptr)) {
-		if (tun_is_xdp_buff(ptr)) {
-			struct xdp_buff *xdp = tun_ptr_to_xdp(ptr);
+		if (tun_is_xdp_frame(ptr)) {
+			struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
 
-			return xdp->data_end - xdp->data;
+			return xdpf->len;
 		}
 		return __skb_array_len_with_tag(ptr);
 	} else {
@@ -2849,10 +2869,10 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
 			    unsigned long arg, int ifreq_len)
 {
 	struct tun_file *tfile = file->private_data;
+	struct net *net = sock_net(&tfile->sk);
 	struct tun_struct *tun;
 	void __user* argp = (void __user*)arg;
 	struct ifreq ifr;
-	struct net *net;
 	kuid_t owner;
 	kgid_t group;
 	int sndbuf;
@@ -2876,14 +2896,18 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
 		 */
 		return put_user(IFF_TUN | IFF_TAP | TUN_FEATURES,
 				(unsigned int __user*)argp);
-	} else if (cmd == TUNSETQUEUE)
+	} else if (cmd == TUNSETQUEUE) {
 		return tun_set_queue(file, &ifr);
+	} else if (cmd == SIOCGSKNS) {
+		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
+			return -EPERM;
+		return open_related_ns(&net->ns, get_net_ns);
+	}
 
 	ret = 0;
 	rtnl_lock();
 
 	tun = tun_get(tfile);
-	net = sock_net(&tfile->sk);
 	if (cmd == TUNSETIFF) {
 		ret = -EEXIST;
 		if (tun)
@@ -2913,14 +2937,6 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
 		tfile->ifindex = ifindex;
 		goto unlock;
 	}
-	if (cmd == SIOCGSKNS) {
-		ret = -EPERM;
-		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
-			goto unlock;
-
-		ret = open_related_ns(&net->ns, get_net_ns);
-		goto unlock;
-	}
 
 	ret = -EBADFD;
 	if (!tun)
diff --git a/drivers/net/usb/Kconfig b/drivers/net/usb/Kconfig
index f28bd74ac275..418b0904cecb 100644
--- a/drivers/net/usb/Kconfig
+++ b/drivers/net/usb/Kconfig
@@ -111,6 +111,7 @@ config USB_LAN78XX
 	select MII
 	select PHYLIB
 	select MICROCHIP_PHY
+	select FIXED_PHY
 	help
 	  This option adds support for Microchip LAN78XX based USB 2
 	  & USB 3 10/100/1000 Ethernet adapters.
diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c
index 0867f7275852..8dff87ec6d99 100644
--- a/drivers/net/usb/lan78xx.c
+++ b/drivers/net/usb/lan78xx.c
@@ -36,13 +36,14 @@
 #include <linux/irq.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/microchipphy.h>
-#include <linux/phy.h>
+#include <linux/phy_fixed.h>
+#include <linux/of_mdio.h>
+#include <linux/of_net.h>
 #include "lan78xx.h"
 
 #define DRIVER_AUTHOR	"WOOJUNG HUH <woojung.huh@microchip.com>"
 #define DRIVER_DESC	"LAN78XX USB 3.0 Gigabit Ethernet Devices"
 #define DRIVER_NAME	"lan78xx"
-#define DRIVER_VERSION	"1.0.6"
 
 #define TX_TIMEOUT_JIFFIES		(5 * HZ)
 #define THROTTLE_JIFFIES		(HZ / 8)
@@ -278,6 +279,30 @@ struct lan78xx_statstage64 {
 	u64 eee_tx_lpi_time;
 };
 
+static u32 lan78xx_regs[] = {
+	ID_REV,
+	INT_STS,
+	HW_CFG,
+	PMT_CTL,
+	E2P_CMD,
+	E2P_DATA,
+	USB_STATUS,
+	VLAN_TYPE,
+	MAC_CR,
+	MAC_RX,
+	MAC_TX,
+	FLOW,
+	ERR_STS,
+	MII_ACC,
+	MII_DATA,
+	EEE_TX_LPI_REQ_DLY,
+	EEE_TW_TX_SYS,
+	EEE_TX_LPI_REM_DLY,
+	WUCSR
+};
+
+#define PHY_REG_SIZE (32 * sizeof(u32))
+
 struct lan78xx_net;
 
 struct lan78xx_priv {
@@ -1477,7 +1502,6 @@ static void lan78xx_get_drvinfo(struct net_device *net,
 	struct lan78xx_net *dev = netdev_priv(net);
 
 	strncpy(info->driver, DRIVER_NAME, sizeof(info->driver));
-	strncpy(info->version, DRIVER_VERSION, sizeof(info->version));
 	usb_make_path(dev->udev, info->bus_info, sizeof(info->bus_info));
 }
 
@@ -1605,6 +1629,34 @@ static int lan78xx_set_pause(struct net_device *net,
 	return ret;
 }
 
+static int lan78xx_get_regs_len(struct net_device *netdev)
+{
+	if (!netdev->phydev)
+		return (sizeof(lan78xx_regs));
+	else
+		return (sizeof(lan78xx_regs) + PHY_REG_SIZE);
+}
+
+static void
+lan78xx_get_regs(struct net_device *netdev, struct ethtool_regs *regs,
+		 void *buf)
+{
+	u32 *data = buf;
+	int i, j;
+	struct lan78xx_net *dev = netdev_priv(netdev);
+
+	/* Read Device/MAC registers */
+	for (i = 0; i < (sizeof(lan78xx_regs) / sizeof(u32)); i++)
+		lan78xx_read_reg(dev, lan78xx_regs[i], &data[i]);
+
+	if (!netdev->phydev)
+		return;
+
+	/* Read PHY registers */
+	for (j = 0; j < 32; i++, j++)
+		data[i] = phy_read(netdev->phydev, j);
+}
+
 static const struct ethtool_ops lan78xx_ethtool_ops = {
 	.get_link	= lan78xx_get_link,
 	.nway_reset	= phy_ethtool_nway_reset,
@@ -1625,6 +1677,8 @@ static const struct ethtool_ops lan78xx_ethtool_ops = {
 	.set_pauseparam	= lan78xx_set_pause,
 	.get_link_ksettings = lan78xx_get_link_ksettings,
 	.set_link_ksettings = lan78xx_set_link_ksettings,
+	.get_regs_len	= lan78xx_get_regs_len,
+	.get_regs	= lan78xx_get_regs,
 };
 
 static int lan78xx_ioctl(struct net_device *netdev, struct ifreq *rq, int cmd)
@@ -1652,34 +1706,31 @@ static void lan78xx_init_mac_address(struct lan78xx_net *dev)
 	addr[5] = (addr_hi >> 8) & 0xFF;
 
 	if (!is_valid_ether_addr(addr)) {
-		/* reading mac address from EEPROM or OTP */
-		if ((lan78xx_read_eeprom(dev, EEPROM_MAC_OFFSET, ETH_ALEN,
-					 addr) == 0) ||
-		    (lan78xx_read_otp(dev, EEPROM_MAC_OFFSET, ETH_ALEN,
-				      addr) == 0)) {
-			if (is_valid_ether_addr(addr)) {
-				/* eeprom values are valid so use them */
-				netif_dbg(dev, ifup, dev->net,
-					  "MAC address read from EEPROM");
-			} else {
-				/* generate random MAC */
-				random_ether_addr(addr);
-				netif_dbg(dev, ifup, dev->net,
-					  "MAC address set to random addr");
-			}
-
-			addr_lo = addr[0] | (addr[1] << 8) |
-				  (addr[2] << 16) | (addr[3] << 24);
-			addr_hi = addr[4] | (addr[5] << 8);
-
-			ret = lan78xx_write_reg(dev, RX_ADDRL, addr_lo);
-			ret = lan78xx_write_reg(dev, RX_ADDRH, addr_hi);
+		if (!eth_platform_get_mac_address(&dev->udev->dev, addr)) {
+			/* valid address present in Device Tree */
+			netif_dbg(dev, ifup, dev->net,
+				  "MAC address read from Device Tree");
+		} else if (((lan78xx_read_eeprom(dev, EEPROM_MAC_OFFSET,
+						 ETH_ALEN, addr) == 0) ||
+			    (lan78xx_read_otp(dev, EEPROM_MAC_OFFSET,
+					      ETH_ALEN, addr) == 0)) &&
+			   is_valid_ether_addr(addr)) {
+			/* eeprom values are valid so use them */
+			netif_dbg(dev, ifup, dev->net,
+				  "MAC address read from EEPROM");
 		} else {
 			/* generate random MAC */
 			random_ether_addr(addr);
 			netif_dbg(dev, ifup, dev->net,
 				  "MAC address set to random addr");
 		}
+
+		addr_lo = addr[0] | (addr[1] << 8) |
+			  (addr[2] << 16) | (addr[3] << 24);
+		addr_hi = addr[4] | (addr[5] << 8);
+
+		ret = lan78xx_write_reg(dev, RX_ADDRL, addr_lo);
+		ret = lan78xx_write_reg(dev, RX_ADDRH, addr_hi);
 	}
 
 	ret = lan78xx_write_reg(dev, MAF_LO(0), addr_lo);
@@ -1762,6 +1813,7 @@ static int lan78xx_mdiobus_write(struct mii_bus *bus, int phy_id, int idx,
 
 static int lan78xx_mdio_init(struct lan78xx_net *dev)
 {
+	struct device_node *node;
 	int ret;
 
 	dev->mdiobus = mdiobus_alloc();
@@ -1790,7 +1842,10 @@ static int lan78xx_mdio_init(struct lan78xx_net *dev)
 		break;
 	}
 
-	ret = mdiobus_register(dev->mdiobus);
+	node = of_get_child_by_name(dev->udev->dev.of_node, "mdio");
+	ret = of_mdiobus_register(dev->mdiobus, node);
+	if (node)
+		of_node_put(node);
 	if (ret) {
 		netdev_err(dev->net, "can't register MDIO bus\n");
 		goto exit1;
@@ -2003,52 +2058,91 @@ static int ksz9031rnx_fixup(struct phy_device *phydev)
 	return 1;
 }
 
-static int lan78xx_phy_init(struct lan78xx_net *dev)
+static struct phy_device *lan7801_phy_init(struct lan78xx_net *dev)
 {
+	u32 buf;
 	int ret;
-	u32 mii_adv;
+	struct fixed_phy_status fphy_status = {
+		.link = 1,
+		.speed = SPEED_1000,
+		.duplex = DUPLEX_FULL,
+	};
 	struct phy_device *phydev;
 
 	phydev = phy_find_first(dev->mdiobus);
 	if (!phydev) {
-		netdev_err(dev->net, "no PHY found\n");
-		return -EIO;
-	}
-
-	if ((dev->chipid == ID_REV_CHIP_ID_7800_) ||
-	    (dev->chipid == ID_REV_CHIP_ID_7850_)) {
-		phydev->is_internal = true;
-		dev->interface = PHY_INTERFACE_MODE_GMII;
-
-	} else if (dev->chipid == ID_REV_CHIP_ID_7801_) {
+		netdev_dbg(dev->net, "PHY Not Found!! Registering Fixed PHY\n");
+		phydev = fixed_phy_register(PHY_POLL, &fphy_status, -1,
+					    NULL);
+		if (IS_ERR(phydev)) {
+			netdev_err(dev->net, "No PHY/fixed_PHY found\n");
+			return NULL;
+		}
+		netdev_dbg(dev->net, "Registered FIXED PHY\n");
+		dev->interface = PHY_INTERFACE_MODE_RGMII;
+		ret = lan78xx_write_reg(dev, MAC_RGMII_ID,
+					MAC_RGMII_ID_TXC_DELAY_EN_);
+		ret = lan78xx_write_reg(dev, RGMII_TX_BYP_DLL, 0x3D00);
+		ret = lan78xx_read_reg(dev, HW_CFG, &buf);
+		buf |= HW_CFG_CLK125_EN_;
+		buf |= HW_CFG_REFCLK25_EN_;
+		ret = lan78xx_write_reg(dev, HW_CFG, buf);
+	} else {
 		if (!phydev->drv) {
 			netdev_err(dev->net, "no PHY driver found\n");
-			return -EIO;
+			return NULL;
 		}
-
 		dev->interface = PHY_INTERFACE_MODE_RGMII;
-
 		/* external PHY fixup for KSZ9031RNX */
 		ret = phy_register_fixup_for_uid(PHY_KSZ9031RNX, 0xfffffff0,
 						 ksz9031rnx_fixup);
 		if (ret < 0) {
-			netdev_err(dev->net, "fail to register fixup\n");
-			return ret;
+			netdev_err(dev->net, "Failed to register fixup for PHY_KSZ9031RNX\n");
+			return NULL;
 		}
 		/* external PHY fixup for LAN8835 */
 		ret = phy_register_fixup_for_uid(PHY_LAN8835, 0xfffffff0,
 						 lan8835_fixup);
 		if (ret < 0) {
-			netdev_err(dev->net, "fail to register fixup\n");
-			return ret;
+			netdev_err(dev->net, "Failed to register fixup for PHY_LAN8835\n");
+			return NULL;
 		}
 		/* add more external PHY fixup here if needed */
 
 		phydev->is_internal = false;
-	} else {
-		netdev_err(dev->net, "unknown ID found\n");
-		ret = -EIO;
-		goto error;
+	}
+	return phydev;
+}
+
+static int lan78xx_phy_init(struct lan78xx_net *dev)
+{
+	int ret;
+	u32 mii_adv;
+	struct phy_device *phydev;
+
+	switch (dev->chipid) {
+	case ID_REV_CHIP_ID_7801_:
+		phydev = lan7801_phy_init(dev);
+		if (!phydev) {
+			netdev_err(dev->net, "lan7801: PHY Init Failed");
+			return -EIO;
+		}
+		break;
+
+	case ID_REV_CHIP_ID_7800_:
+	case ID_REV_CHIP_ID_7850_:
+		phydev = phy_find_first(dev->mdiobus);
+		if (!phydev) {
+			netdev_err(dev->net, "no PHY found\n");
+			return -EIO;
+		}
+		phydev->is_internal = true;
+		dev->interface = PHY_INTERFACE_MODE_GMII;
+		break;
+
+	default:
+		netdev_err(dev->net, "Unknown CHIP ID found\n");
+		return -EIO;
 	}
 
 	/* if phyirq is not set, use polling mode in phylib */
@@ -2067,6 +2161,16 @@ static int lan78xx_phy_init(struct lan78xx_net *dev)
 	if (ret) {
 		netdev_err(dev->net, "can't attach PHY to %s\n",
 			   dev->mdiobus->id);
+		if (dev->chipid == ID_REV_CHIP_ID_7801_) {
+			if (phy_is_pseudo_fixed_link(phydev)) {
+				fixed_phy_unregister(phydev);
+			} else {
+				phy_unregister_fixup_for_uid(PHY_KSZ9031RNX,
+							     0xfffffff0);
+				phy_unregister_fixup_for_uid(PHY_LAN8835,
+							     0xfffffff0);
+			}
+		}
 		return -EIO;
 	}
 
@@ -2079,17 +2183,33 @@ static int lan78xx_phy_init(struct lan78xx_net *dev)
 	mii_adv = (u32)mii_advertise_flowctrl(dev->fc_request_control);
 	phydev->advertising |= mii_adv_to_ethtool_adv_t(mii_adv);
 
+	if (phydev->mdio.dev.of_node) {
+		u32 reg;
+		int len;
+
+		len = of_property_count_elems_of_size(phydev->mdio.dev.of_node,
+						      "microchip,led-modes",
+						      sizeof(u32));
+		if (len >= 0) {
+			/* Ensure the appropriate LEDs are enabled */
+			lan78xx_read_reg(dev, HW_CFG, &reg);
+			reg &= ~(HW_CFG_LED0_EN_ |
+				 HW_CFG_LED1_EN_ |
+				 HW_CFG_LED2_EN_ |
+				 HW_CFG_LED3_EN_);
+			reg |= (len > 0) * HW_CFG_LED0_EN_ |
+				(len > 1) * HW_CFG_LED1_EN_ |
+				(len > 2) * HW_CFG_LED2_EN_ |
+				(len > 3) * HW_CFG_LED3_EN_;
+			lan78xx_write_reg(dev, HW_CFG, reg);
+		}
+	}
+
 	genphy_config_aneg(phydev);
 
 	dev->fc_autoneg = phydev->autoneg;
 
 	return 0;
-
-error:
-	phy_unregister_fixup_for_uid(PHY_KSZ9031RNX, 0xfffffff0);
-	phy_unregister_fixup_for_uid(PHY_LAN8835, 0xfffffff0);
-
-	return ret;
 }
 
 static int lan78xx_set_rx_max_frame_length(struct lan78xx_net *dev, int size)
@@ -3487,6 +3607,7 @@ static void lan78xx_disconnect(struct usb_interface *intf)
 	struct lan78xx_net		*dev;
 	struct usb_device		*udev;
 	struct net_device		*net;
+	struct phy_device		*phydev;
 
 	dev = usb_get_intfdata(intf);
 	usb_set_intfdata(intf, NULL);
@@ -3495,12 +3616,16 @@ static void lan78xx_disconnect(struct usb_interface *intf)
 
 	udev = interface_to_usbdev(intf);
 	net = dev->net;
+	phydev = net->phydev;
 
 	phy_unregister_fixup_for_uid(PHY_KSZ9031RNX, 0xfffffff0);
 	phy_unregister_fixup_for_uid(PHY_LAN8835, 0xfffffff0);
 
 	phy_disconnect(net->phydev);
 
+	if (phy_is_pseudo_fixed_link(phydev))
+		fixed_phy_unregister(phydev);
+
 	unregister_netdev(net);
 
 	cancel_delayed_work_sync(&dev->wq);
diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c
index 42565dd33aa6..8e8b51f171f4 100644
--- a/drivers/net/usb/qmi_wwan.c
+++ b/drivers/net/usb/qmi_wwan.c
@@ -1103,6 +1103,7 @@ static const struct usb_device_id products[] = {
 	{QMI_FIXED_INTF(0x05c6, 0x920d, 5)},
 	{QMI_QUIRK_SET_DTR(0x05c6, 0x9625, 4)},	/* YUGA CLM920-NC5 */
 	{QMI_FIXED_INTF(0x0846, 0x68a2, 8)},
+	{QMI_FIXED_INTF(0x0846, 0x68d3, 8)},	/* Netgear Aircard 779S */
 	{QMI_FIXED_INTF(0x12d1, 0x140c, 1)},	/* Huawei E173 */
 	{QMI_FIXED_INTF(0x12d1, 0x14ac, 1)},	/* Huawei E1820 */
 	{QMI_FIXED_INTF(0x1435, 0xd181, 3)},	/* Wistron NeWeb D18Q1 */
@@ -1248,7 +1249,7 @@ static const struct usb_device_id products[] = {
 	{QMI_FIXED_INTF(0x03f0, 0x4e1d, 8)},	/* HP lt4111 LTE/EV-DO/HSPA+ Gobi 4G Module */
 	{QMI_FIXED_INTF(0x03f0, 0x9d1d, 1)},	/* HP lt4120 Snapdragon X5 LTE */
 	{QMI_FIXED_INTF(0x22de, 0x9061, 3)},	/* WeTelecom WPD-600N */
-	{QMI_FIXED_INTF(0x1e0e, 0x9001, 5)},	/* SIMCom 7230E */
+	{QMI_QUIRK_SET_DTR(0x1e0e, 0x9001, 5)},	/* SIMCom 7100E, 7230E, 7600E ++ */
 	{QMI_QUIRK_SET_DTR(0x2c7c, 0x0125, 4)},	/* Quectel EC25, EC20 R2.0  Mini PCIe */
 	{QMI_QUIRK_SET_DTR(0x2c7c, 0x0121, 4)},	/* Quectel EC21 Mini PCIe */
 	{QMI_FIXED_INTF(0x2c7c, 0x0296, 4)},	/* Quectel BG96 */
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 032e1ac10a30..8f08a3e1bbaa 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -30,8 +30,11 @@
 #include <linux/cpu.h>
 #include <linux/average.h>
 #include <linux/filter.h>
+#include <linux/netdevice.h>
+#include <linux/pci.h>
 #include <net/route.h>
 #include <net/xdp.h>
+#include <net/net_failover.h>
 
 static int napi_weight = NAPI_POLL_WEIGHT;
 module_param(napi_weight, int, 0444);
@@ -210,6 +213,9 @@ struct virtnet_info {
 	u32 speed;
 
 	unsigned long guest_offloads;
+
+	/* failover when STANDBY feature enabled */
+	struct failover *failover;
 };
 
 struct padded_vnet_hdr {
@@ -419,46 +425,70 @@ static void virtnet_xdp_flush(struct net_device *dev)
 	virtqueue_kick(sq->vq);
 }
 
-static bool __virtnet_xdp_xmit(struct virtnet_info *vi,
-			       struct xdp_buff *xdp)
+static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
+				   struct send_queue *sq,
+				   struct xdp_frame *xdpf)
 {
 	struct virtio_net_hdr_mrg_rxbuf *hdr;
-	unsigned int len;
-	struct send_queue *sq;
-	unsigned int qp;
-	void *xdp_sent;
 	int err;
 
-	qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();
-	sq = &vi->sq[qp];
-
-	/* Free up any pending old buffers before queueing new ones. */
-	while ((xdp_sent = virtqueue_get_buf(sq->vq, &len)) != NULL) {
-		struct page *sent_page = virt_to_head_page(xdp_sent);
+	/* virtqueue want to use data area in-front of packet */
+	if (unlikely(xdpf->metasize > 0))
+		return -EOPNOTSUPP;
 
-		put_page(sent_page);
-	}
+	if (unlikely(xdpf->headroom < vi->hdr_len))
+		return -EOVERFLOW;
 
-	xdp->data -= vi->hdr_len;
+	/* Make room for virtqueue hdr (also change xdpf->headroom?) */
+	xdpf->data -= vi->hdr_len;
 	/* Zero header and leave csum up to XDP layers */
-	hdr = xdp->data;
+	hdr = xdpf->data;
 	memset(hdr, 0, vi->hdr_len);
+	xdpf->len   += vi->hdr_len;
 
-	sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);
+	sg_init_one(sq->sg, xdpf->data, xdpf->len);
 
-	err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp->data, GFP_ATOMIC);
+	err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdpf, GFP_ATOMIC);
 	if (unlikely(err))
-		return false; /* Caller handle free/refcnt */
+		return -ENOSPC; /* Caller handle free/refcnt */
 
-	return true;
+	return 0;
 }
 
-static int virtnet_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
+static int __virtnet_xdp_tx_xmit(struct virtnet_info *vi,
+				   struct xdp_frame *xdpf)
+{
+	struct xdp_frame *xdpf_sent;
+	struct send_queue *sq;
+	unsigned int len;
+	unsigned int qp;
+
+	qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();
+	sq = &vi->sq[qp];
+
+	/* Free up any pending old buffers before queueing new ones. */
+	while ((xdpf_sent = virtqueue_get_buf(sq->vq, &len)) != NULL)
+		xdp_return_frame(xdpf_sent);
+
+	return __virtnet_xdp_xmit_one(vi, sq, xdpf);
+}
+
+static int virtnet_xdp_xmit(struct net_device *dev,
+			    int n, struct xdp_frame **frames)
 {
 	struct virtnet_info *vi = netdev_priv(dev);
 	struct receive_queue *rq = vi->rq;
+	struct xdp_frame *xdpf_sent;
 	struct bpf_prog *xdp_prog;
-	bool sent;
+	struct send_queue *sq;
+	unsigned int len;
+	unsigned int qp;
+	int drops = 0;
+	int err;
+	int i;
+
+	qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();
+	sq = &vi->sq[qp];
 
 	/* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
 	 * indicate XDP resources have been successfully allocated.
@@ -467,10 +497,20 @@ static int virtnet_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
 	if (!xdp_prog)
 		return -ENXIO;
 
-	sent = __virtnet_xdp_xmit(vi, xdp);
-	if (!sent)
-		return -ENOSPC;
-	return 0;
+	/* Free up any pending old buffers before queueing new ones. */
+	while ((xdpf_sent = virtqueue_get_buf(sq->vq, &len)) != NULL)
+		xdp_return_frame(xdpf_sent);
+
+	for (i = 0; i < n; i++) {
+		struct xdp_frame *xdpf = frames[i];
+
+		err = __virtnet_xdp_xmit_one(vi, sq, xdpf);
+		if (err) {
+			xdp_return_frame_rx_napi(xdpf);
+			drops++;
+		}
+	}
+	return n - drops;
 }
 
 static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
@@ -559,7 +599,6 @@ static struct sk_buff *receive_small(struct net_device *dev,
 	struct page *page = virt_to_head_page(buf);
 	unsigned int delta = 0;
 	struct page *xdp_page;
-	bool sent;
 	int err;
 
 	len -= vi->hdr_len;
@@ -568,6 +607,7 @@ static struct sk_buff *receive_small(struct net_device *dev,
 	xdp_prog = rcu_dereference(rq->xdp_prog);
 	if (xdp_prog) {
 		struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
+		struct xdp_frame *xdpf;
 		struct xdp_buff xdp;
 		void *orig_data;
 		u32 act;
@@ -608,10 +648,14 @@ static struct sk_buff *receive_small(struct net_device *dev,
 		case XDP_PASS:
 			/* Recalculate length in case bpf program changed it */
 			delta = orig_data - xdp.data;
+			len = xdp.data_end - xdp.data;
 			break;
 		case XDP_TX:
-			sent = __virtnet_xdp_xmit(vi, &xdp);
-			if (unlikely(!sent)) {
+			xdpf = convert_to_xdp_frame(&xdp);
+			if (unlikely(!xdpf))
+				goto err_xdp;
+			err = __virtnet_xdp_tx_xmit(vi, xdpf);
+			if (unlikely(err)) {
 				trace_xdp_exception(vi->dev, xdp_prog, act);
 				goto err_xdp;
 			}
@@ -641,7 +685,7 @@ static struct sk_buff *receive_small(struct net_device *dev,
 		goto err;
 	}
 	skb_reserve(skb, headroom - delta);
-	skb_put(skb, len + delta);
+	skb_put(skb, len);
 	if (!delta) {
 		buf += header_offset;
 		memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
@@ -694,7 +738,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	struct bpf_prog *xdp_prog;
 	unsigned int truesize;
 	unsigned int headroom = mergeable_ctx_to_headroom(ctx);
-	bool sent;
 	int err;
 
 	head_skb = NULL;
@@ -702,6 +745,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	rcu_read_lock();
 	xdp_prog = rcu_dereference(rq->xdp_prog);
 	if (xdp_prog) {
+		struct xdp_frame *xdpf;
 		struct page *xdp_page;
 		struct xdp_buff xdp;
 		void *data;
@@ -755,6 +799,10 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 			offset = xdp.data -
 					page_address(xdp_page) - vi->hdr_len;
 
+			/* recalculate len if xdp.data or xdp.data_end were
+			 * adjusted
+			 */
+			len = xdp.data_end - xdp.data + vi->hdr_len;
 			/* We can only create skb based on xdp_page. */
 			if (unlikely(xdp_page != page)) {
 				rcu_read_unlock();
@@ -765,8 +813,11 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 			}
 			break;
 		case XDP_TX:
-			sent = __virtnet_xdp_xmit(vi, &xdp);
-			if (unlikely(!sent)) {
+			xdpf = convert_to_xdp_frame(&xdp);
+			if (unlikely(!xdpf))
+				goto err_xdp;
+			err = __virtnet_xdp_tx_xmit(vi, xdpf);
+			if (unlikely(err)) {
 				trace_xdp_exception(vi->dev, xdp_prog, act);
 				if (unlikely(xdp_page != page))
 					put_page(xdp_page);
@@ -1311,6 +1362,13 @@ static int virtnet_open(struct net_device *dev)
 		if (err < 0)
 			return err;
 
+		err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq,
+						 MEM_TYPE_PAGE_SHARED, NULL);
+		if (err < 0) {
+			xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
+			return err;
+		}
+
 		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
 		virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);
 	}
@@ -1502,6 +1560,9 @@ static int virtnet_set_mac_address(struct net_device *dev, void *p)
 	struct sockaddr *addr;
 	struct scatterlist sg;
 
+	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
+		return -EOPNOTSUPP;
+
 	addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
 	if (!addr)
 		return -ENOMEM;
@@ -2285,6 +2346,22 @@ static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 	}
 }
 
+static int virtnet_get_phys_port_name(struct net_device *dev, char *buf,
+				      size_t len)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+	int ret;
+
+	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
+		return -EOPNOTSUPP;
+
+	ret = snprintf(buf, len, "sby");
+	if (ret >= len)
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
 static const struct net_device_ops virtnet_netdev = {
 	.ndo_open            = virtnet_open,
 	.ndo_stop   	     = virtnet_close,
@@ -2302,6 +2379,7 @@ static const struct net_device_ops virtnet_netdev = {
 	.ndo_xdp_xmit		= virtnet_xdp_xmit,
 	.ndo_xdp_flush		= virtnet_xdp_flush,
 	.ndo_features_check	= passthru_features_check,
+	.ndo_get_phys_port_name	= virtnet_get_phys_port_name,
 };
 
 static void virtnet_config_changed_work(struct work_struct *work)
@@ -2855,10 +2933,16 @@ static int virtnet_probe(struct virtio_device *vdev)
 
 	virtnet_init_settings(dev);
 
+	if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
+		vi->failover = net_failover_create(vi->dev);
+		if (IS_ERR(vi->failover))
+			goto free_vqs;
+	}
+
 	err = register_netdev(dev);
 	if (err) {
 		pr_debug("virtio_net: registering device failed\n");
-		goto free_vqs;
+		goto free_failover;
 	}
 
 	virtio_device_ready(vdev);
@@ -2895,6 +2979,8 @@ static int virtnet_probe(struct virtio_device *vdev)
 	vi->vdev->config->reset(vdev);
 
 	unregister_netdev(dev);
+free_failover:
+	net_failover_destroy(vi->failover);
 free_vqs:
 	cancel_delayed_work_sync(&vi->refill);
 	free_receive_page_frags(vi);
@@ -2929,6 +3015,8 @@ static void virtnet_remove(struct virtio_device *vdev)
 
 	unregister_netdev(vi->dev);
 
+	net_failover_destroy(vi->failover);
+
 	remove_vq_common(vi);
 
 	free_netdev(vi->dev);
@@ -2978,7 +3066,7 @@ static struct virtio_device_id id_table[] = {
 	VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
 	VIRTIO_NET_F_CTRL_MAC_ADDR, \
 	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
-	VIRTIO_NET_F_SPEED_DUPLEX
+	VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY
 
 static unsigned int features[] = {
 	VIRTNET_FEATURES,
diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
index 27a9bb8c9611..e454dfc9ad8f 100644
--- a/drivers/net/vmxnet3/vmxnet3_drv.c
+++ b/drivers/net/vmxnet3/vmxnet3_drv.c
@@ -2949,7 +2949,7 @@ vmxnet3_close(struct net_device *netdev)
 	 * completion.
 	 */
 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
-		msleep(1);
+		usleep_range(1000, 2000);
 
 	vmxnet3_quiesce_dev(adapter);
 
@@ -2999,7 +2999,7 @@ vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
 	 * completion.
 	 */
 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
-		msleep(1);
+		usleep_range(1000, 2000);
 
 	if (netif_running(netdev)) {
 		vmxnet3_quiesce_dev(adapter);
@@ -3589,7 +3589,7 @@ static void vmxnet3_shutdown_device(struct pci_dev *pdev)
 	 * completion.
 	 */
 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
-		msleep(1);
+		usleep_range(1000, 2000);
 
 	if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED,
 			     &adapter->state)) {
diff --git a/drivers/net/vmxnet3/vmxnet3_ethtool.c b/drivers/net/vmxnet3/vmxnet3_ethtool.c
index 2ff27314e047..559db051a500 100644
--- a/drivers/net/vmxnet3/vmxnet3_ethtool.c
+++ b/drivers/net/vmxnet3/vmxnet3_ethtool.c
@@ -600,7 +600,7 @@ vmxnet3_set_ringparam(struct net_device *netdev,
 	 * completion.
 	 */
 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
-		msleep(1);
+		usleep_range(1000, 2000);
 
 	if (netif_running(netdev)) {
 		vmxnet3_quiesce_dev(adapter);
diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c
index 0a2b180d138a..f93547f257fb 100644
--- a/drivers/net/vrf.c
+++ b/drivers/net/vrf.c
@@ -48,6 +48,9 @@ static unsigned int vrf_net_id;
 struct net_vrf {
 	struct rtable __rcu	*rth;
 	struct rt6_info	__rcu	*rt6;
+#if IS_ENABLED(CONFIG_IPV6)
+	struct fib6_table	*fib6_table;
+#endif
 	u32                     tb_id;
 };
 
@@ -496,7 +499,6 @@ static int vrf_rt6_create(struct net_device *dev)
 	int flags = DST_HOST | DST_NOPOLICY | DST_NOXFRM;
 	struct net_vrf *vrf = netdev_priv(dev);
 	struct net *net = dev_net(dev);
-	struct fib6_table *rt6i_table;
 	struct rt6_info *rt6;
 	int rc = -ENOMEM;
 
@@ -504,8 +506,8 @@ static int vrf_rt6_create(struct net_device *dev)
 	if (!ipv6_mod_enabled())
 		return 0;
 
-	rt6i_table = fib6_new_table(net, vrf->tb_id);
-	if (!rt6i_table)
+	vrf->fib6_table = fib6_new_table(net, vrf->tb_id);
+	if (!vrf->fib6_table)
 		goto out;
 
 	/* create a dst for routing packets out a VRF device */
@@ -513,7 +515,6 @@ static int vrf_rt6_create(struct net_device *dev)
 	if (!rt6)
 		goto out;
 
-	rt6->rt6i_table = rt6i_table;
 	rt6->dst.output	= vrf_output6;
 
 	rcu_assign_pointer(vrf->rt6, rt6);
@@ -946,22 +947,8 @@ static struct rt6_info *vrf_ip6_route_lookup(struct net *net,
 					     int flags)
 {
 	struct net_vrf *vrf = netdev_priv(dev);
-	struct fib6_table *table = NULL;
-	struct rt6_info *rt6;
-
-	rcu_read_lock();
-
-	/* fib6_table does not have a refcnt and can not be freed */
-	rt6 = rcu_dereference(vrf->rt6);
-	if (likely(rt6))
-		table = rt6->rt6i_table;
-
-	rcu_read_unlock();
-
-	if (!table)
-		return NULL;
 
-	return ip6_pol_route(net, table, ifindex, fl6, skb, flags);
+	return ip6_pol_route(net, vrf->fib6_table, ifindex, fl6, skb, flags);
 }
 
 static void vrf_ip6_input_dst(struct sk_buff *skb, struct net_device *vrf_dev,
@@ -1267,7 +1254,7 @@ static void vrf_setup(struct net_device *dev)
 
 	/* enable offload features */
 	dev->features   |= NETIF_F_GSO_SOFTWARE;
-	dev->features   |= NETIF_F_RXCSUM | NETIF_F_HW_CSUM;
+	dev->features   |= NETIF_F_RXCSUM | NETIF_F_HW_CSUM | NETIF_F_SCTP_CRC;
 	dev->features   |= NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HIGHDMA;
 
 	dev->hw_features = dev->features;
diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
index fab7a4db249e..aee0e60471f1 100644
--- a/drivers/net/vxlan.c
+++ b/drivers/net/vxlan.c
@@ -2085,9 +2085,13 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		local_ip = vxlan->cfg.saddr;
 		dst_cache = &rdst->dst_cache;
 		md->gbp = skb->mark;
-		ttl = vxlan->cfg.ttl;
-		if (!ttl && vxlan_addr_multicast(dst))
-			ttl = 1;
+		if (flags & VXLAN_F_TTL_INHERIT) {
+			ttl = ip_tunnel_get_ttl(old_iph, skb);
+		} else {
+			ttl = vxlan->cfg.ttl;
+			if (!ttl && vxlan_addr_multicast(dst))
+				ttl = 1;
+		}
 
 		tos = vxlan->cfg.tos;
 		if (tos == 1)
@@ -2709,6 +2713,7 @@ static const struct nla_policy vxlan_policy[IFLA_VXLAN_MAX + 1] = {
 	[IFLA_VXLAN_GBP]	= { .type = NLA_FLAG, },
 	[IFLA_VXLAN_GPE]	= { .type = NLA_FLAG, },
 	[IFLA_VXLAN_REMCSUM_NOPARTIAL]	= { .type = NLA_FLAG },
+	[IFLA_VXLAN_TTL_INHERIT]	= { .type = NLA_FLAG },
 };
 
 static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[],
@@ -3254,6 +3259,12 @@ static int vxlan_nl2conf(struct nlattr *tb[], struct nlattr *data[],
 	if (data[IFLA_VXLAN_TTL])
 		conf->ttl = nla_get_u8(data[IFLA_VXLAN_TTL]);
 
+	if (data[IFLA_VXLAN_TTL_INHERIT]) {
+		if (changelink)
+			return -EOPNOTSUPP;
+		conf->flags |= VXLAN_F_TTL_INHERIT;
+	}
+
 	if (data[IFLA_VXLAN_LABEL])
 		conf->label = nla_get_be32(data[IFLA_VXLAN_LABEL]) &
 			     IPV6_FLOWLABEL_MASK;
diff --git a/drivers/net/wireless/ath/ath10k/Kconfig b/drivers/net/wireless/ath/ath10k/Kconfig
index deb5ae21a559..84f071ac0d84 100644
--- a/drivers/net/wireless/ath/ath10k/Kconfig
+++ b/drivers/net/wireless/ath/ath10k/Kconfig
@@ -4,12 +4,16 @@ config ATH10K
 	select ATH_COMMON
 	select CRC32
 	select WANT_DEV_COREDUMP
+	select ATH10K_CE
         ---help---
           This module adds support for wireless adapters based on
           Atheros IEEE 802.11ac family of chipsets.
 
           If you choose to build a module, it'll be called ath10k.
 
+config ATH10K_CE
+	bool
+
 config ATH10K_PCI
 	tristate "Atheros ath10k PCI support"
 	depends on ATH10K && PCI
@@ -36,6 +40,14 @@ config ATH10K_USB
 	  This module adds experimental support for USB bus. Currently
 	  work in progress and will not fully work.
 
+config ATH10K_SNOC
+        tristate "Qualcomm ath10k SNOC support (EXPERIMENTAL)"
+        depends on ATH10K && ARCH_QCOM
+        ---help---
+          This module adds support for integrated WCN3990 chip connected
+          to system NOC(SNOC). Currently work in progress and will not
+          fully work.
+
 config ATH10K_DEBUG
 	bool "Atheros ath10k debugging"
 	depends on ATH10K
diff --git a/drivers/net/wireless/ath/ath10k/Makefile b/drivers/net/wireless/ath/ath10k/Makefile
index 6739ac26fd29..44d60a61b242 100644
--- a/drivers/net/wireless/ath/ath10k/Makefile
+++ b/drivers/net/wireless/ath/ath10k/Makefile
@@ -22,10 +22,10 @@ ath10k_core-$(CONFIG_THERMAL) += thermal.o
 ath10k_core-$(CONFIG_MAC80211_DEBUGFS) += debugfs_sta.o
 ath10k_core-$(CONFIG_PM) += wow.o
 ath10k_core-$(CONFIG_DEV_COREDUMP) += coredump.o
+ath10k_core-$(CONFIG_ATH10K_CE) += ce.o
 
 obj-$(CONFIG_ATH10K_PCI) += ath10k_pci.o
-ath10k_pci-y += pci.o \
-		ce.o
+ath10k_pci-y += pci.o
 
 ath10k_pci-$(CONFIG_ATH10K_AHB) += ahb.o
 
@@ -35,5 +35,8 @@ ath10k_sdio-y += sdio.o
 obj-$(CONFIG_ATH10K_USB) += ath10k_usb.o
 ath10k_usb-y += usb.o
 
+obj-$(CONFIG_ATH10K_SNOC) += ath10k_snoc.o
+ath10k_snoc-y += snoc.o
+
 # for tracing framework to find trace.h
 CFLAGS_trace.o := -I$(src)
diff --git a/drivers/net/wireless/ath/ath10k/ahb.c b/drivers/net/wireless/ath/ath10k/ahb.c
index 35d10490f6c3..fa39ffffd34d 100644
--- a/drivers/net/wireless/ath/ath10k/ahb.c
+++ b/drivers/net/wireless/ath/ath10k/ahb.c
@@ -180,14 +180,11 @@ static void ath10k_ahb_clock_disable(struct ath10k *ar)
 {
 	struct ath10k_ahb *ar_ahb = ath10k_ahb_priv(ar);
 
-	if (!IS_ERR_OR_NULL(ar_ahb->cmd_clk))
-		clk_disable_unprepare(ar_ahb->cmd_clk);
+	clk_disable_unprepare(ar_ahb->cmd_clk);
 
-	if (!IS_ERR_OR_NULL(ar_ahb->ref_clk))
-		clk_disable_unprepare(ar_ahb->ref_clk);
+	clk_disable_unprepare(ar_ahb->ref_clk);
 
-	if (!IS_ERR_OR_NULL(ar_ahb->rtc_clk))
-		clk_disable_unprepare(ar_ahb->rtc_clk);
+	clk_disable_unprepare(ar_ahb->rtc_clk);
 }
 
 static int ath10k_ahb_rst_ctrl_init(struct ath10k *ar)
diff --git a/drivers/net/wireless/ath/ath10k/ce.c b/drivers/net/wireless/ath/ath10k/ce.c
index b9def7bace2f..3b96a43fbda4 100644
--- a/drivers/net/wireless/ath/ath10k/ce.c
+++ b/drivers/net/wireless/ath/ath10k/ce.c
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018 The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -58,6 +59,74 @@
  * the buffer is sent/received.
  */
 
+static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
+					struct ath10k_ce_pipe *ce_state)
+{
+	u32 ce_id = ce_state->id;
+	u32 addr = 0;
+
+	switch (ce_id) {
+	case 0:
+		addr = 0x00032000;
+		break;
+	case 3:
+		addr = 0x0003200C;
+		break;
+	case 4:
+		addr = 0x00032010;
+		break;
+	case 5:
+		addr = 0x00032014;
+		break;
+	case 7:
+		addr = 0x0003201C;
+		break;
+	default:
+		ath10k_warn(ar, "invalid CE id: %d", ce_id);
+		break;
+	}
+	return addr;
+}
+
+static inline u32 shadow_dst_wr_ind_addr(struct ath10k *ar,
+					 struct ath10k_ce_pipe *ce_state)
+{
+	u32 ce_id = ce_state->id;
+	u32 addr = 0;
+
+	switch (ce_id) {
+	case 1:
+		addr = 0x00032034;
+		break;
+	case 2:
+		addr = 0x00032038;
+		break;
+	case 5:
+		addr = 0x00032044;
+		break;
+	case 7:
+		addr = 0x0003204C;
+		break;
+	case 8:
+		addr = 0x00032050;
+		break;
+	case 9:
+		addr = 0x00032054;
+		break;
+	case 10:
+		addr = 0x00032058;
+		break;
+	case 11:
+		addr = 0x0003205C;
+		break;
+	default:
+		ath10k_warn(ar, "invalid CE id: %d", ce_id);
+		break;
+	}
+
+	return addr;
+}
+
 static inline unsigned int
 ath10k_set_ring_byte(unsigned int offset,
 		     struct ath10k_hw_ce_regs_addr_map *addr_map)
@@ -116,11 +185,46 @@ static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
 				ar->hw_ce_regs->sr_wr_index_addr);
 }
 
+static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
+							 u32 ce_id)
+{
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+
+	return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
+}
+
 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
 						    u32 ce_ctrl_addr)
 {
-	return ath10k_ce_read32(ar, ce_ctrl_addr +
-				ar->hw_ce_regs->current_srri_addr);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
+	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
+	u32 index;
+
+	if (ar->hw_params.rri_on_ddr &&
+	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))
+		index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
+	else
+		index = ath10k_ce_read32(ar, ce_ctrl_addr +
+					 ar->hw_ce_regs->current_srri_addr);
+
+	return index;
+}
+
+static inline void
+ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
+					  struct ath10k_ce_pipe *ce_state,
+					  unsigned int value)
+{
+	ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
+}
+
+static inline void
+ath10k_ce_shadow_dest_ring_write_index_set(struct ath10k *ar,
+					   struct ath10k_ce_pipe *ce_state,
+					   unsigned int value)
+{
+	ath10k_ce_write32(ar, shadow_dst_wr_ind_addr(ar, ce_state), value);
 }
 
 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
@@ -181,11 +285,31 @@ static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
 			  ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
 }
 
+static inline
+	u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
+{
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+
+	return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
+		CE_DDR_RRI_MASK;
+}
+
 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
 						     u32 ce_ctrl_addr)
 {
-	return ath10k_ce_read32(ar, ce_ctrl_addr +
-				ar->hw_ce_regs->current_drri_addr);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
+	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
+	u32 index;
+
+	if (ar->hw_params.rri_on_ddr &&
+	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))
+		index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
+	else
+		index = ath10k_ce_read32(ar, ce_ctrl_addr +
+					 ar->hw_ce_regs->current_drri_addr);
+
+	return index;
 }
 
 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
@@ -376,8 +500,14 @@ static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 
 	/* WORKAROUND */
-	if (!(flags & CE_SEND_FLAG_GATHER))
-		ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
+	if (!(flags & CE_SEND_FLAG_GATHER)) {
+		if (ar->hw_params.shadow_reg_support)
+			ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
+								  write_index);
+		else
+			ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
+							   write_index);
+	}
 
 	src_ring->write_index = write_index;
 exit:
@@ -395,7 +525,7 @@ static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 	struct ce_desc_64 *desc, sdesc;
 	unsigned int nentries_mask = src_ring->nentries_mask;
-	unsigned int sw_index = src_ring->sw_index;
+	unsigned int sw_index;
 	unsigned int write_index = src_ring->write_index;
 	u32 ctrl_addr = ce_state->ctrl_addr;
 	__le32 *addr;
@@ -409,6 +539,11 @@ static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
 		ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
 			    __func__, nbytes, ce_state->src_sz_max);
 
+	if (ar->hw_params.rri_on_ddr)
+		sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
+	else
+		sw_index = src_ring->sw_index;
+
 	if (unlikely(CE_RING_DELTA(nentries_mask,
 				   write_index, sw_index - 1) <= 0)) {
 		ret = -ENOSR;
@@ -464,6 +599,7 @@ int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 	return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
 				    buffer, nbytes, transfer_id, flags);
 }
+EXPORT_SYMBOL(ath10k_ce_send_nolock);
 
 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
 {
@@ -491,6 +627,7 @@ void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
 
 	src_ring->per_transfer_context[src_ring->write_index] = NULL;
 }
+EXPORT_SYMBOL(__ath10k_ce_send_revert);
 
 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 		   void *per_transfer_context,
@@ -510,6 +647,7 @@ int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 
 	return ret;
 }
+EXPORT_SYMBOL(ath10k_ce_send);
 
 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
 {
@@ -525,6 +663,7 @@ int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
 
 	return delta;
 }
+EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
 
 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
 {
@@ -539,6 +678,7 @@ int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
 
 	return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
 }
+EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
 
 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 				   dma_addr_t paddr)
@@ -615,13 +755,14 @@ void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
 	/* Prevent CE ring stuck issue that will occur when ring is full.
 	 * Make sure that write index is 1 less than read index.
 	 */
-	if ((cur_write_idx + nentries)  == dest_ring->sw_index)
+	if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
 		nentries -= 1;
 
 	write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
 	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 	dest_ring->write_index = write_index;
 }
+EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
 
 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 			  dma_addr_t paddr)
@@ -636,6 +777,7 @@ int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 
 	return ret;
 }
+EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
 
 /*
  * Guts of ath10k_ce_completed_recv_next.
@@ -748,6 +890,7 @@ int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 							    per_transfer_ctx,
 							    nbytesp);
 }
+EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
 
 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
 				  void **per_transfer_contextp,
@@ -766,6 +909,7 @@ int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
 
 	return ret;
 }
+EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
 
 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 				       void **per_transfer_contextp,
@@ -882,6 +1026,7 @@ int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 						  per_transfer_contextp,
 						  bufferp);
 }
+EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
 
 /*
  * Guts of ath10k_ce_completed_send_next.
@@ -915,7 +1060,10 @@ int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
 		src_ring->hw_index = read_index;
 	}
 
-	read_index = src_ring->hw_index;
+	if (ar->hw_params.rri_on_ddr)
+		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
+	else
+		read_index = src_ring->hw_index;
 
 	if (read_index == sw_index)
 		return -EIO;
@@ -936,6 +1084,7 @@ int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
 
 static void ath10k_ce_extract_desc_data(struct ath10k *ar,
 					struct ath10k_ce_ring *src_ring,
@@ -1025,6 +1174,7 @@ int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
 
 	return ret;
 }
+EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
 
 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
 				  void **per_transfer_contextp)
@@ -1040,6 +1190,7 @@ int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
 
 	return ret;
 }
+EXPORT_SYMBOL(ath10k_ce_completed_send_next);
 
 /*
  * Guts of interrupt handler for per-engine interrupts on a particular CE.
@@ -1078,6 +1229,7 @@ void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
 
 	spin_unlock_bh(&ce->ce_lock);
 }
+EXPORT_SYMBOL(ath10k_ce_per_engine_service);
 
 /*
  * Handler for per-engine interrupts on ALL active CEs.
@@ -1102,6 +1254,7 @@ void ath10k_ce_per_engine_service_any(struct ath10k *ar)
 		ath10k_ce_per_engine_service(ar, ce_id);
 	}
 }
+EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
 
 /*
  * Adjust interrupts for the copy complete handler.
@@ -1139,6 +1292,7 @@ int ath10k_ce_disable_interrupts(struct ath10k *ar)
 
 	return 0;
 }
+EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
 
 void ath10k_ce_enable_interrupts(struct ath10k *ar)
 {
@@ -1154,6 +1308,7 @@ void ath10k_ce_enable_interrupts(struct ath10k *ar)
 		ath10k_ce_per_engine_handler_adjust(ce_state);
 	}
 }
+EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
 
 static int ath10k_ce_init_src_ring(struct ath10k *ar,
 				   unsigned int ce_id,
@@ -1234,6 +1389,22 @@ static int ath10k_ce_init_dest_ring(struct ath10k *ar,
 	return 0;
 }
 
+static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
+				       struct ath10k_ce_ring *src_ring,
+				       u32 nentries)
+{
+	src_ring->shadow_base_unaligned = kcalloc(nentries,
+						  sizeof(struct ce_desc),
+						  GFP_KERNEL);
+	if (!src_ring->shadow_base_unaligned)
+		return -ENOMEM;
+
+	src_ring->shadow_base = (struct ce_desc *)
+			PTR_ALIGN(src_ring->shadow_base_unaligned,
+				  CE_DESC_RING_ALIGN);
+	return 0;
+}
+
 static struct ath10k_ce_ring *
 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
 			 const struct ce_attr *attr)
@@ -1241,6 +1412,7 @@ ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
 	struct ath10k_ce_ring *src_ring;
 	u32 nentries = attr->src_nentries;
 	dma_addr_t base_addr;
+	int ret;
 
 	nentries = roundup_pow_of_two(nentries);
 
@@ -1277,6 +1449,19 @@ ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
 			ALIGN(src_ring->base_addr_ce_space_unaligned,
 			      CE_DESC_RING_ALIGN);
 
+	if (ar->hw_params.shadow_reg_support) {
+		ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
+		if (ret) {
+			dma_free_coherent(ar->dev,
+					  (nentries * sizeof(struct ce_desc) +
+					   CE_DESC_RING_ALIGN),
+					  src_ring->base_addr_owner_space_unaligned,
+					  base_addr);
+			kfree(src_ring);
+			return ERR_PTR(ret);
+		}
+	}
+
 	return src_ring;
 }
 
@@ -1287,6 +1472,7 @@ ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
 	struct ath10k_ce_ring *src_ring;
 	u32 nentries = attr->src_nentries;
 	dma_addr_t base_addr;
+	int ret;
 
 	nentries = roundup_pow_of_two(nentries);
 
@@ -1322,6 +1508,19 @@ ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
 			ALIGN(src_ring->base_addr_ce_space_unaligned,
 			      CE_DESC_RING_ALIGN);
 
+	if (ar->hw_params.shadow_reg_support) {
+		ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
+		if (ret) {
+			dma_free_coherent(ar->dev,
+					  (nentries * sizeof(struct ce_desc) +
+					   CE_DESC_RING_ALIGN),
+					  src_ring->base_addr_owner_space_unaligned,
+					  base_addr);
+			kfree(src_ring);
+			return ERR_PTR(ret);
+		}
+	}
+
 	return src_ring;
 }
 
@@ -1454,6 +1653,7 @@ int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath10k_ce_init_pipe);
 
 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
 {
@@ -1479,6 +1679,7 @@ void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
 	ath10k_ce_deinit_src_ring(ar, ce_id);
 	ath10k_ce_deinit_dest_ring(ar, ce_id);
 }
+EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
 
 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
 {
@@ -1486,6 +1687,8 @@ static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 
 	if (ce_state->src_ring) {
+		if (ar->hw_params.shadow_reg_support)
+			kfree(ce_state->src_ring->shadow_base_unaligned);
 		dma_free_coherent(ar->dev,
 				  (ce_state->src_ring->nentries *
 				   sizeof(struct ce_desc) +
@@ -1515,6 +1718,8 @@ static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 
 	if (ce_state->src_ring) {
+		if (ar->hw_params.shadow_reg_support)
+			kfree(ce_state->src_ring->shadow_base_unaligned);
 		dma_free_coherent(ar->dev,
 				  (ce_state->src_ring->nentries *
 				   sizeof(struct ce_desc_64) +
@@ -1545,6 +1750,7 @@ void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
 
 	ce_state->ops->ce_free_pipe(ar, ce_id);
 }
+EXPORT_SYMBOL(ath10k_ce_free_pipe);
 
 void ath10k_ce_dump_registers(struct ath10k *ar,
 			      struct ath10k_fw_crash_data *crash_data)
@@ -1584,6 +1790,7 @@ void ath10k_ce_dump_registers(struct ath10k *ar,
 
 	spin_unlock_bh(&ce->ce_lock);
 }
+EXPORT_SYMBOL(ath10k_ce_dump_registers);
 
 static const struct ath10k_ce_ops ce_ops = {
 	.ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
@@ -1680,3 +1887,47 @@ int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
+
+void ath10k_ce_alloc_rri(struct ath10k *ar)
+{
+	int i;
+	u32 value;
+	u32 ctrl1_regs;
+	u32 ce_base_addr;
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+
+	ce->vaddr_rri = dma_alloc_coherent(ar->dev,
+					   (CE_COUNT * sizeof(u32)),
+					   &ce->paddr_rri, GFP_KERNEL);
+
+	if (!ce->vaddr_rri)
+		return;
+
+	ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
+			  lower_32_bits(ce->paddr_rri));
+	ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
+			  (upper_32_bits(ce->paddr_rri) &
+			  CE_DESC_FLAGS_GET_MASK));
+
+	for (i = 0; i < CE_COUNT; i++) {
+		ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
+		ce_base_addr = ath10k_ce_base_address(ar, i);
+		value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
+		value |= ar->hw_ce_regs->upd->mask;
+		ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
+	}
+
+	memset(ce->vaddr_rri, 0, CE_COUNT * sizeof(u32));
+}
+EXPORT_SYMBOL(ath10k_ce_alloc_rri);
+
+void ath10k_ce_free_rri(struct ath10k *ar)
+{
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+
+	dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
+			  ce->vaddr_rri,
+			  ce->paddr_rri);
+}
+EXPORT_SYMBOL(ath10k_ce_free_rri);
diff --git a/drivers/net/wireless/ath/ath10k/ce.h b/drivers/net/wireless/ath/ath10k/ce.h
index 2c3c8f5e90ea..dbeffaef6024 100644
--- a/drivers/net/wireless/ath/ath10k/ce.h
+++ b/drivers/net/wireless/ath/ath10k/ce.h
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018 The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -48,6 +49,9 @@ struct ath10k_ce_pipe;
 #define CE_DESC_FLAGS_META_DATA_MASK ar->hw_values->ce_desc_meta_data_mask
 #define CE_DESC_FLAGS_META_DATA_LSB  ar->hw_values->ce_desc_meta_data_lsb
 
+#define CE_DDR_RRI_MASK			GENMASK(15, 0)
+#define CE_DDR_DRRI_SHIFT		16
+
 struct ce_desc {
 	__le32 addr;
 	__le16 nbytes;
@@ -113,6 +117,9 @@ struct ath10k_ce_ring {
 	/* CE address space */
 	u32 base_addr_ce_space;
 
+	char *shadow_base_unaligned;
+	struct ce_desc *shadow_base;
+
 	/* keep last */
 	void *per_transfer_context[0];
 };
@@ -153,6 +160,8 @@ struct ath10k_ce {
 	spinlock_t ce_lock;
 	const struct ath10k_bus_ops *bus_ops;
 	struct ath10k_ce_pipe ce_states[CE_COUNT_MAX];
+	u32 *vaddr_rri;
+	dma_addr_t paddr_rri;
 };
 
 /*==================Send====================*/
@@ -261,6 +270,8 @@ int ath10k_ce_disable_interrupts(struct ath10k *ar);
 void ath10k_ce_enable_interrupts(struct ath10k *ar);
 void ath10k_ce_dump_registers(struct ath10k *ar,
 			      struct ath10k_fw_crash_data *crash_data);
+void ath10k_ce_alloc_rri(struct ath10k *ar);
+void ath10k_ce_free_rri(struct ath10k *ar);
 
 /* ce_attr.flags values */
 /* Use NonSnooping PCIe accesses? */
@@ -327,6 +338,9 @@ static inline u32 ath10k_ce_base_address(struct ath10k *ar, unsigned int ce_id)
 	return CE0_BASE_ADDRESS + (CE1_BASE_ADDRESS - CE0_BASE_ADDRESS) * ce_id;
 }
 
+#define COPY_ENGINE_ID(COPY_ENGINE_BASE_ADDRESS) (((COPY_ENGINE_BASE_ADDRESS) \
+		- CE0_BASE_ADDRESS) / (CE1_BASE_ADDRESS - CE0_BASE_ADDRESS))
+
 #define CE_SRC_RING_TO_DESC(baddr, idx) \
 	(&(((struct ce_desc *)baddr)[idx]))
 
@@ -355,14 +369,18 @@ static inline u32 ath10k_ce_base_address(struct ath10k *ar, unsigned int ce_id)
 	(((x) & CE_WRAPPER_INTERRUPT_SUMMARY_HOST_MSI_MASK) >> \
 		CE_WRAPPER_INTERRUPT_SUMMARY_HOST_MSI_LSB)
 #define CE_WRAPPER_INTERRUPT_SUMMARY_ADDRESS			0x0000
+#define CE_INTERRUPT_SUMMARY		(GENMASK(CE_COUNT_MAX - 1, 0))
 
 static inline u32 ath10k_ce_interrupt_summary(struct ath10k *ar)
 {
 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 
-	return CE_WRAPPER_INTERRUPT_SUMMARY_HOST_MSI_GET(
-		ce->bus_ops->read32((ar), CE_WRAPPER_BASE_ADDRESS +
-		CE_WRAPPER_INTERRUPT_SUMMARY_ADDRESS));
+	if (!ar->hw_params.per_ce_irq)
+		return CE_WRAPPER_INTERRUPT_SUMMARY_HOST_MSI_GET(
+			ce->bus_ops->read32((ar), CE_WRAPPER_BASE_ADDRESS +
+			CE_WRAPPER_INTERRUPT_SUMMARY_ADDRESS));
+	else
+		return CE_INTERRUPT_SUMMARY;
 }
 
 #endif /* _CE_H_ */
diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c
index 8a3020dbd4cf..ad4f6e3c0737 100644
--- a/drivers/net/wireless/ath/ath10k/core.c
+++ b/drivers/net/wireless/ath/ath10k/core.c
@@ -90,6 +90,8 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA988X_HW_2_0_VERSION,
@@ -119,6 +121,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA9887_HW_1_0_VERSION,
@@ -148,6 +153,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA6174_HW_2_1_VERSION,
@@ -176,6 +184,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA6174_HW_2_1_VERSION,
@@ -204,6 +215,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA6174_HW_3_0_VERSION,
@@ -232,6 +246,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA6174_HW_3_2_VERSION,
@@ -263,6 +280,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA99X0_HW_2_0_DEV_VERSION,
@@ -297,6 +317,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA9984_HW_1_0_DEV_VERSION,
@@ -336,6 +359,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA9888_HW_2_0_DEV_VERSION,
@@ -374,6 +400,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA9377_HW_1_0_DEV_VERSION,
@@ -402,6 +431,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA9377_HW_1_1_DEV_VERSION,
@@ -432,6 +464,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = QCA4019_HW_1_0_DEV_VERSION,
@@ -467,6 +502,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = 0x20,
 		.target_64bit = false,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+		.per_ce_irq = false,
+		.shadow_reg_support = false,
+		.rri_on_ddr = false,
 	},
 	{
 		.id = WCN3990_HW_1_0_DEV_VERSION,
@@ -487,6 +525,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.num_wds_entries = TARGET_HL_10_TLV_NUM_WDS_ENTRIES,
 		.target_64bit = true,
 		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL_DUAL_MAC,
+		.per_ce_irq = true,
+		.shadow_reg_support = true,
+		.rri_on_ddr = true,
 	},
 };
 
@@ -653,7 +694,7 @@ static const struct firmware *ath10k_fetch_fw_file(struct ath10k *ar,
 		dir = ".";
 
 	snprintf(filename, sizeof(filename), "%s/%s", dir, file);
-	ret = request_firmware(&fw, filename, ar->dev);
+	ret = firmware_request_nowarn(&fw, filename, ar->dev);
 	ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot fw request '%s': %d\n",
 		   filename, ret);
 
@@ -1253,14 +1294,61 @@ static int ath10k_core_parse_bd_ie_board(struct ath10k *ar,
 	return ret;
 }
 
+static int ath10k_core_search_bd(struct ath10k *ar,
+				 const char *boardname,
+				 const u8 *data,
+				 size_t len)
+{
+	size_t ie_len;
+	struct ath10k_fw_ie *hdr;
+	int ret = -ENOENT, ie_id;
+
+	while (len > sizeof(struct ath10k_fw_ie)) {
+		hdr = (struct ath10k_fw_ie *)data;
+		ie_id = le32_to_cpu(hdr->id);
+		ie_len = le32_to_cpu(hdr->len);
+
+		len -= sizeof(*hdr);
+		data = hdr->data;
+
+		if (len < ALIGN(ie_len, 4)) {
+			ath10k_err(ar, "invalid length for board ie_id %d ie_len %zu len %zu\n",
+				   ie_id, ie_len, len);
+			return -EINVAL;
+		}
+
+		switch (ie_id) {
+		case ATH10K_BD_IE_BOARD:
+			ret = ath10k_core_parse_bd_ie_board(ar, data, ie_len,
+							    boardname);
+			if (ret == -ENOENT)
+				/* no match found, continue */
+				break;
+
+			/* either found or error, so stop searching */
+			goto out;
+		}
+
+		/* jump over the padding */
+		ie_len = ALIGN(ie_len, 4);
+
+		len -= ie_len;
+		data += ie_len;
+	}
+
+out:
+	/* return result of parse_bd_ie_board() or -ENOENT */
+	return ret;
+}
+
 static int ath10k_core_fetch_board_data_api_n(struct ath10k *ar,
 					      const char *boardname,
+					      const char *fallback_boardname,
 					      const char *filename)
 {
-	size_t len, magic_len, ie_len;
-	struct ath10k_fw_ie *hdr;
+	size_t len, magic_len;
 	const u8 *data;
-	int ret, ie_id;
+	int ret;
 
 	ar->normal_mode_fw.board = ath10k_fetch_fw_file(ar,
 							ar->hw_params.fw.dir,
@@ -1298,69 +1386,23 @@ static int ath10k_core_fetch_board_data_api_n(struct ath10k *ar,
 	data += magic_len;
 	len -= magic_len;
 
-	while (len > sizeof(struct ath10k_fw_ie)) {
-		hdr = (struct ath10k_fw_ie *)data;
-		ie_id = le32_to_cpu(hdr->id);
-		ie_len = le32_to_cpu(hdr->len);
-
-		len -= sizeof(*hdr);
-		data = hdr->data;
-
-		if (len < ALIGN(ie_len, 4)) {
-			ath10k_err(ar, "invalid length for board ie_id %d ie_len %zu len %zu\n",
-				   ie_id, ie_len, len);
-			ret = -EINVAL;
-			goto err;
-		}
-
-		switch (ie_id) {
-		case ATH10K_BD_IE_BOARD:
-			ret = ath10k_core_parse_bd_ie_board(ar, data, ie_len,
-							    boardname);
-			if (ret == -ENOENT && ar->id.bdf_ext[0] != '\0') {
-				/* try default bdf if variant was not found */
-				char *s, *v = ",variant=";
-				char boardname2[100];
-
-				strlcpy(boardname2, boardname,
-					sizeof(boardname2));
-
-				s = strstr(boardname2, v);
-				if (s)
-					*s = '\0';  /* strip ",variant=%s" */
-
-				ret = ath10k_core_parse_bd_ie_board(ar, data,
-								    ie_len,
-								    boardname2);
-			}
-
-			if (ret == -ENOENT)
-				/* no match found, continue */
-				break;
-			else if (ret)
-				/* there was an error, bail out */
-				goto err;
+	/* attempt to find boardname in the IE list */
+	ret = ath10k_core_search_bd(ar, boardname, data, len);
 
-			/* board data found */
-			goto out;
-		}
+	/* if we didn't find it and have a fallback name, try that */
+	if (ret == -ENOENT && fallback_boardname)
+		ret = ath10k_core_search_bd(ar, fallback_boardname, data, len);
 
-		/* jump over the padding */
-		ie_len = ALIGN(ie_len, 4);
-
-		len -= ie_len;
-		data += ie_len;
-	}
-
-out:
-	if (!ar->normal_mode_fw.board_data || !ar->normal_mode_fw.board_len) {
+	if (ret == -ENOENT) {
 		ath10k_err(ar,
 			   "failed to fetch board data for %s from %s/%s\n",
 			   boardname, ar->hw_params.fw.dir, filename);
 		ret = -ENODATA;
-		goto err;
 	}
 
+	if (ret)
+		goto err;
+
 	return 0;
 
 err:
@@ -1369,12 +1411,12 @@ static int ath10k_core_fetch_board_data_api_n(struct ath10k *ar,
 }
 
 static int ath10k_core_create_board_name(struct ath10k *ar, char *name,
-					 size_t name_len)
+					 size_t name_len, bool with_variant)
 {
 	/* strlen(',variant=') + strlen(ar->id.bdf_ext) */
 	char variant[9 + ATH10K_SMBIOS_BDF_EXT_STR_LENGTH] = { 0 };
 
-	if (ar->id.bdf_ext[0] != '\0')
+	if (with_variant && ar->id.bdf_ext[0] != '\0')
 		scnprintf(variant, sizeof(variant), ",variant=%s",
 			  ar->id.bdf_ext);
 
@@ -1400,17 +1442,26 @@ static int ath10k_core_create_board_name(struct ath10k *ar, char *name,
 
 static int ath10k_core_fetch_board_file(struct ath10k *ar)
 {
-	char boardname[100];
+	char boardname[100], fallback_boardname[100];
 	int ret;
 
-	ret = ath10k_core_create_board_name(ar, boardname, sizeof(boardname));
+	ret = ath10k_core_create_board_name(ar, boardname,
+					    sizeof(boardname), true);
 	if (ret) {
 		ath10k_err(ar, "failed to create board name: %d", ret);
 		return ret;
 	}
 
+	ret = ath10k_core_create_board_name(ar, fallback_boardname,
+					    sizeof(boardname), false);
+	if (ret) {
+		ath10k_err(ar, "failed to create fallback board name: %d", ret);
+		return ret;
+	}
+
 	ar->bd_api = 2;
 	ret = ath10k_core_fetch_board_data_api_n(ar, boardname,
+						 fallback_boardname,
 						 ATH10K_BOARD_API2_FILE);
 	if (!ret)
 		goto success;
@@ -2472,6 +2523,14 @@ static int ath10k_core_probe_fw(struct ath10k *ar)
 		ar->hw->wiphy->hw_version = target_info.version;
 		break;
 	case ATH10K_BUS_SNOC:
+		memset(&target_info, 0, sizeof(target_info));
+		ret = ath10k_hif_get_target_info(ar, &target_info);
+		if (ret) {
+			ath10k_err(ar, "could not get target info (%d)\n", ret);
+			goto err_power_down;
+		}
+		ar->target_version = target_info.version;
+		ar->hw->wiphy->hw_version = target_info.version;
 		break;
 	default:
 		ath10k_err(ar, "incorrect hif bus type: %d\n", ar->hif.bus);
diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h
index c17d805d68cc..951dbdd1c9eb 100644
--- a/drivers/net/wireless/ath/ath10k/core.h
+++ b/drivers/net/wireless/ath/ath10k/core.h
@@ -44,6 +44,7 @@
 #define WO(_f)      ((_f##_OFFSET) >> 2)
 
 #define ATH10K_SCAN_ID 0
+#define ATH10K_SCAN_CHANNEL_SWITCH_WMI_EVT_OVERHEAD 10 /* msec */
 #define WMI_READY_TIMEOUT (5 * HZ)
 #define ATH10K_FLUSH_TIMEOUT_HZ (5 * HZ)
 #define ATH10K_CONNECTION_LOSS_HZ (3 * HZ)
@@ -52,6 +53,8 @@
 /* Antenna noise floor */
 #define ATH10K_DEFAULT_NOISE_FLOOR -95
 
+#define ATH10K_INVALID_RSSI 128
+
 #define ATH10K_MAX_NUM_MGMT_PENDING 128
 
 /* number of failed packets (20 packets with 16 sw reties each) */
@@ -173,6 +176,7 @@ struct ath10k_wmi {
 	struct completion service_ready;
 	struct completion unified_ready;
 	struct completion barrier;
+	struct completion radar_confirm;
 	wait_queue_head_t tx_credits_wq;
 	DECLARE_BITMAP(svc_map, WMI_SERVICE_MAX);
 	struct wmi_cmd_map *cmd;
@@ -375,6 +379,21 @@ struct ath10k_dfs_stats {
 	u32 radar_detected;
 };
 
+enum ath10k_radar_confirmation_state {
+	ATH10K_RADAR_CONFIRMATION_IDLE = 0,
+	ATH10K_RADAR_CONFIRMATION_INPROGRESS,
+	ATH10K_RADAR_CONFIRMATION_STOPPED,
+};
+
+struct ath10k_radar_found_info {
+	u32 pri_min;
+	u32 pri_max;
+	u32 width_min;
+	u32 width_max;
+	u32 sidx_min;
+	u32 sidx_max;
+};
+
 #define ATH10K_MAX_NUM_PEER_IDS (1 << 11) /* htt rx_desc limit */
 
 struct ath10k_peer {
@@ -1107,6 +1126,11 @@ struct ath10k {
 
 	u32 sta_tid_stats_mask;
 
+	/* protected by data_lock */
+	enum ath10k_radar_confirmation_state radar_conf_state;
+	struct ath10k_radar_found_info last_radar_info;
+	struct work_struct radar_confirmation_work;
+
 	/* must be last */
 	u8 drv_priv[0] __aligned(sizeof(void *));
 };
diff --git a/drivers/net/wireless/ath/ath10k/coredump.c b/drivers/net/wireless/ath/ath10k/coredump.c
index f90cec0ebb1c..4d28063052fe 100644
--- a/drivers/net/wireless/ath/ath10k/coredump.c
+++ b/drivers/net/wireless/ath/ath10k/coredump.c
@@ -701,6 +701,89 @@ static const struct ath10k_mem_region qca988x_hw20_mem_regions[] = {
 	},
 };
 
+static const struct ath10k_mem_region qca99x0_hw20_mem_regions[] = {
+	{
+		.type = ATH10K_MEM_REGION_TYPE_DRAM,
+		.start = 0x400000,
+		.len = 0x60000,
+		.name = "DRAM",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_REG,
+		.start = 0x98000,
+		.len = 0x50000,
+		.name = "IRAM",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_IOSRAM,
+		.start = 0xC0000,
+		.len = 0x40000,
+		.name = "SRAM",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_IOREG,
+		.start = 0x30000,
+		.len = 0x7000,
+		.name = "APB REG 1",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_IOREG,
+		.start = 0x3f000,
+		.len = 0x3000,
+		.name = "APB REG 2",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_IOREG,
+		.start = 0x43000,
+		.len = 0x3000,
+		.name = "WIFI REG",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_IOREG,
+		.start = 0x4A000,
+		.len = 0x5000,
+		.name = "CE REG",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_IOREG,
+		.start = 0x80000,
+		.len = 0x6000,
+		.name = "SOC REG",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+};
+
 static const struct ath10k_mem_region qca9984_hw10_mem_regions[] = {
 	{
 		.type = ATH10K_MEM_REGION_TYPE_DRAM,
@@ -848,6 +931,21 @@ static const struct ath10k_hw_mem_layout hw_mem_layouts[] = {
 			.size = ARRAY_SIZE(qca9984_hw10_mem_regions),
 		},
 	},
+	{
+		.hw_id = QCA9888_HW_2_0_DEV_VERSION,
+		.region_table = {
+			.regions = qca9984_hw10_mem_regions,
+			.size = ARRAY_SIZE(qca9984_hw10_mem_regions),
+		},
+	},
+	{
+		.hw_id = QCA99X0_HW_2_0_DEV_VERSION,
+		.region_table = {
+			.regions = qca99x0_hw20_mem_regions,
+			.size = ARRAY_SIZE(qca99x0_hw20_mem_regions),
+		},
+	},
+
 };
 
 static u32 ath10k_coredump_get_ramdump_size(struct ath10k *ar)
diff --git a/drivers/net/wireless/ath/ath10k/debug.c b/drivers/net/wireless/ath/ath10k/debug.c
index bac832ce1873..0d98c93a3aba 100644
--- a/drivers/net/wireless/ath/ath10k/debug.c
+++ b/drivers/net/wireless/ath/ath10k/debug.c
@@ -987,13 +987,13 @@ static ssize_t ath10k_write_htt_max_amsdu_ampdu(struct file *file,
 {
 	struct ath10k *ar = file->private_data;
 	int res;
-	char buf[64];
+	char buf[64] = {0};
 	unsigned int amsdu, ampdu;
 
-	simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, user_buf, count);
-
-	/* make sure that buf is null terminated */
-	buf[sizeof(buf) - 1] = 0;
+	res = simple_write_to_buffer(buf, sizeof(buf) - 1, ppos,
+				     user_buf, count);
+	if (res <= 0)
+		return res;
 
 	res = sscanf(buf, "%u %u", &amsdu, &ampdu);
 
@@ -1043,14 +1043,14 @@ static ssize_t ath10k_write_fw_dbglog(struct file *file,
 {
 	struct ath10k *ar = file->private_data;
 	int ret;
-	char buf[96];
+	char buf[96] = {0};
 	unsigned int log_level;
 	u64 mask;
 
-	simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, user_buf, count);
-
-	/* make sure that buf is null terminated */
-	buf[sizeof(buf) - 1] = 0;
+	ret = simple_write_to_buffer(buf, sizeof(buf) - 1, ppos,
+				     user_buf, count);
+	if (ret <= 0)
+		return ret;
 
 	ret = sscanf(buf, "%llx %u", &mask, &log_level);
 
@@ -1519,7 +1519,13 @@ static void ath10k_tpc_stats_print(struct ath10k_tpc_stats *tpc_stats,
 	*len += scnprintf(buf + *len, buf_len - *len,
 			  "********************************\n");
 	*len += scnprintf(buf + *len, buf_len - *len,
-			  "No.  Preamble Rate_code tpc_value1 tpc_value2 tpc_value3\n");
+			  "No.  Preamble Rate_code ");
+
+	for (i = 0; i < WMI_TPC_TX_N_CHAIN; i++)
+		*len += scnprintf(buf + *len, buf_len - *len,
+				  "tpc_value%d ", i);
+
+	*len += scnprintf(buf + *len, buf_len - *len, "\n");
 
 	for (i = 0; i < tpc_stats->rate_max; i++) {
 		*len += scnprintf(buf + *len, buf_len - *len,
diff --git a/drivers/net/wireless/ath/ath10k/debugfs_sta.c b/drivers/net/wireless/ath/ath10k/debugfs_sta.c
index 8f688f136c22..a63c97e2c50c 100644
--- a/drivers/net/wireless/ath/ath10k/debugfs_sta.c
+++ b/drivers/net/wireless/ath/ath10k/debugfs_sta.c
@@ -254,12 +254,12 @@ static ssize_t ath10k_dbg_sta_write_addba(struct file *file,
 	struct ath10k *ar = arsta->arvif->ar;
 	u32 tid, buf_size;
 	int ret;
-	char buf[64];
+	char buf[64] = {0};
 
-	simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, user_buf, count);
-
-	/* make sure that buf is null terminated */
-	buf[sizeof(buf) - 1] = '\0';
+	ret = simple_write_to_buffer(buf, sizeof(buf) - 1, ppos,
+				     user_buf, count);
+	if (ret <= 0)
+		return ret;
 
 	ret = sscanf(buf, "%u %u", &tid, &buf_size);
 	if (ret != 2)
@@ -305,12 +305,12 @@ static ssize_t ath10k_dbg_sta_write_addba_resp(struct file *file,
 	struct ath10k *ar = arsta->arvif->ar;
 	u32 tid, status;
 	int ret;
-	char buf[64];
-
-	simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, user_buf, count);
+	char buf[64] = {0};
 
-	/* make sure that buf is null terminated */
-	buf[sizeof(buf) - 1] = '\0';
+	ret = simple_write_to_buffer(buf, sizeof(buf) - 1, ppos,
+				     user_buf, count);
+	if (ret <= 0)
+		return ret;
 
 	ret = sscanf(buf, "%u %u", &tid, &status);
 	if (ret != 2)
@@ -355,12 +355,12 @@ static ssize_t ath10k_dbg_sta_write_delba(struct file *file,
 	struct ath10k *ar = arsta->arvif->ar;
 	u32 tid, initiator, reason;
 	int ret;
-	char buf[64];
-
-	simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, user_buf, count);
+	char buf[64] = {0};
 
-	/* make sure that buf is null terminated */
-	buf[sizeof(buf) - 1] = '\0';
+	ret = simple_write_to_buffer(buf, sizeof(buf) - 1, ppos,
+				     user_buf, count);
+	if (ret <= 0)
+		return ret;
 
 	ret = sscanf(buf, "%u %u %u", &tid, &initiator, &reason);
 	if (ret != 3)
diff --git a/drivers/net/wireless/ath/ath10k/hif.h b/drivers/net/wireless/ath/ath10k/hif.h
index 6da4e3369c5a..1a59ea0068c2 100644
--- a/drivers/net/wireless/ath/ath10k/hif.h
+++ b/drivers/net/wireless/ath/ath10k/hif.h
@@ -20,13 +20,14 @@
 
 #include <linux/kernel.h>
 #include "core.h"
+#include "bmi.h"
 #include "debug.h"
 
 struct ath10k_hif_sg_item {
 	u16 transfer_id;
 	void *transfer_context; /* NULL = tx completion callback not called */
 	void *vaddr; /* for debugging mostly */
-	u32 paddr;
+	dma_addr_t paddr;
 	u16 len;
 };
 
@@ -93,6 +94,9 @@ struct ath10k_hif_ops {
 	/* fetch calibration data from target eeprom */
 	int (*fetch_cal_eeprom)(struct ath10k *ar, void **data,
 				size_t *data_len);
+
+	int (*get_target_info)(struct ath10k *ar,
+			       struct bmi_target_info *target_info);
 };
 
 static inline int ath10k_hif_tx_sg(struct ath10k *ar, u8 pipe_id,
@@ -218,4 +222,13 @@ static inline int ath10k_hif_fetch_cal_eeprom(struct ath10k *ar,
 	return ar->hif.ops->fetch_cal_eeprom(ar, data, data_len);
 }
 
+static inline int ath10k_hif_get_target_info(struct ath10k *ar,
+					     struct bmi_target_info *tgt_info)
+{
+	if (!ar->hif.ops->get_target_info)
+		return -EOPNOTSUPP;
+
+	return ar->hif.ops->get_target_info(ar, tgt_info);
+}
+
 #endif /* _HIF_H_ */
diff --git a/drivers/net/wireless/ath/ath10k/htc.c b/drivers/net/wireless/ath/ath10k/htc.c
index 492dc5b4bbf2..8902720b4e49 100644
--- a/drivers/net/wireless/ath/ath10k/htc.c
+++ b/drivers/net/wireless/ath/ath10k/htc.c
@@ -542,8 +542,14 @@ static const char *htc_service_name(enum ath10k_htc_svc_id id)
 		return "NMI Data";
 	case ATH10K_HTC_SVC_ID_HTT_DATA_MSG:
 		return "HTT Data";
+	case ATH10K_HTC_SVC_ID_HTT_DATA2_MSG:
+		return "HTT Data";
+	case ATH10K_HTC_SVC_ID_HTT_DATA3_MSG:
+		return "HTT Data";
 	case ATH10K_HTC_SVC_ID_TEST_RAW_STREAMS:
 		return "RAW";
+	case ATH10K_HTC_SVC_ID_HTT_LOG_MSG:
+		return "PKTLOG";
 	}
 
 	return "Unknown";
diff --git a/drivers/net/wireless/ath/ath10k/htc.h b/drivers/net/wireless/ath/ath10k/htc.h
index a2f8814b3e53..34877597dd6a 100644
--- a/drivers/net/wireless/ath/ath10k/htc.h
+++ b/drivers/net/wireless/ath/ath10k/htc.h
@@ -248,6 +248,7 @@ enum ath10k_htc_svc_gid {
 	ATH10K_HTC_SVC_GRP_WMI = 1,
 	ATH10K_HTC_SVC_GRP_NMI = 2,
 	ATH10K_HTC_SVC_GRP_HTT = 3,
+	ATH10K_LOG_SERVICE_GROUP = 6,
 
 	ATH10K_HTC_SVC_GRP_TEST = 254,
 	ATH10K_HTC_SVC_GRP_LAST = 255,
@@ -273,6 +274,9 @@ enum ath10k_htc_svc_id {
 
 	ATH10K_HTC_SVC_ID_HTT_DATA_MSG	= SVC(ATH10K_HTC_SVC_GRP_HTT, 0),
 
+	ATH10K_HTC_SVC_ID_HTT_DATA2_MSG = SVC(ATH10K_HTC_SVC_GRP_HTT, 1),
+	ATH10K_HTC_SVC_ID_HTT_DATA3_MSG = SVC(ATH10K_HTC_SVC_GRP_HTT, 2),
+	ATH10K_HTC_SVC_ID_HTT_LOG_MSG = SVC(ATH10K_LOG_SERVICE_GROUP, 0),
 	/* raw stream service (i.e. flash, tcmd, calibration apps) */
 	ATH10K_HTC_SVC_ID_TEST_RAW_STREAMS = SVC(ATH10K_HTC_SVC_GRP_TEST, 0),
 };
diff --git a/drivers/net/wireless/ath/ath10k/htt.c b/drivers/net/wireless/ath/ath10k/htt.c
index 625198dea18b..21a67f82f037 100644
--- a/drivers/net/wireless/ath/ath10k/htt.c
+++ b/drivers/net/wireless/ath/ath10k/htt.c
@@ -257,11 +257,11 @@ int ath10k_htt_setup(struct ath10k_htt *htt)
 		return status;
 	}
 
-	status = htt->tx_ops->htt_send_frag_desc_bank_cfg(htt);
+	status = ath10k_htt_send_frag_desc_bank_cfg(htt);
 	if (status)
 		return status;
 
-	status = htt->tx_ops->htt_send_rx_ring_cfg(htt);
+	status = ath10k_htt_send_rx_ring_cfg(htt);
 	if (status) {
 		ath10k_warn(ar, "failed to setup rx ring: %d\n",
 			    status);
diff --git a/drivers/net/wireless/ath/ath10k/htt.h b/drivers/net/wireless/ath/ath10k/htt.h
index 8cc2a8b278e4..5d3ff80f3a1f 100644
--- a/drivers/net/wireless/ath/ath10k/htt.h
+++ b/drivers/net/wireless/ath/ath10k/htt.h
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -127,6 +128,19 @@ struct htt_msdu_ext_desc_64 {
 				 | HTT_MSDU_EXT_DESC_FLAG_TCP_IPV4_CSUM_ENABLE \
 				 | HTT_MSDU_EXT_DESC_FLAG_TCP_IPV6_CSUM_ENABLE)
 
+#define HTT_MSDU_EXT_DESC_FLAG_IPV4_CSUM_ENABLE_64		BIT(16)
+#define HTT_MSDU_EXT_DESC_FLAG_UDP_IPV4_CSUM_ENABLE_64		BIT(17)
+#define HTT_MSDU_EXT_DESC_FLAG_UDP_IPV6_CSUM_ENABLE_64		BIT(18)
+#define HTT_MSDU_EXT_DESC_FLAG_TCP_IPV4_CSUM_ENABLE_64		BIT(19)
+#define HTT_MSDU_EXT_DESC_FLAG_TCP_IPV6_CSUM_ENABLE_64		BIT(20)
+#define HTT_MSDU_EXT_DESC_FLAG_PARTIAL_CSUM_ENABLE_64		BIT(21)
+
+#define HTT_MSDU_CHECKSUM_ENABLE_64  (HTT_MSDU_EXT_DESC_FLAG_IPV4_CSUM_ENABLE_64 \
+				     | HTT_MSDU_EXT_DESC_FLAG_UDP_IPV4_CSUM_ENABLE_64 \
+				     | HTT_MSDU_EXT_DESC_FLAG_UDP_IPV6_CSUM_ENABLE_64 \
+				     | HTT_MSDU_EXT_DESC_FLAG_TCP_IPV4_CSUM_ENABLE_64 \
+				     | HTT_MSDU_EXT_DESC_FLAG_TCP_IPV6_CSUM_ENABLE_64)
+
 enum htt_data_tx_desc_flags0 {
 	HTT_DATA_TX_DESC_FLAGS0_MAC_HDR_PRESENT = 1 << 0,
 	HTT_DATA_TX_DESC_FLAGS0_NO_AGGR         = 1 << 1,
@@ -533,12 +547,18 @@ struct htt_ver_resp {
 	u8 rsvd0;
 } __packed;
 
+#define HTT_MGMT_TX_CMPL_FLAG_ACK_RSSI BIT(0)
+
+#define HTT_MGMT_TX_CMPL_INFO_ACK_RSSI_MASK	GENMASK(7, 0)
+
 struct htt_mgmt_tx_completion {
 	u8 rsvd0;
 	u8 rsvd1;
-	u8 rsvd2;
+	u8 flags;
 	__le32 desc_id;
 	__le32 status;
+	__le32 ppdu_id;
+	__le32 info;
 } __packed;
 
 #define HTT_RX_INDICATION_INFO0_EXT_TID_MASK  (0x1F)
@@ -1648,6 +1668,7 @@ struct htt_resp {
 struct htt_tx_done {
 	u16 msdu_id;
 	u16 status;
+	u8 ack_rssi;
 };
 
 enum htt_tx_compl_state {
@@ -1848,6 +1869,57 @@ struct ath10k_htt_tx_ops {
 	void (*htt_free_txbuff)(struct ath10k_htt *htt);
 };
 
+static inline int ath10k_htt_send_rx_ring_cfg(struct ath10k_htt *htt)
+{
+	if (!htt->tx_ops->htt_send_rx_ring_cfg)
+		return -EOPNOTSUPP;
+
+	return htt->tx_ops->htt_send_rx_ring_cfg(htt);
+}
+
+static inline int ath10k_htt_send_frag_desc_bank_cfg(struct ath10k_htt *htt)
+{
+	if (!htt->tx_ops->htt_send_frag_desc_bank_cfg)
+		return -EOPNOTSUPP;
+
+	return htt->tx_ops->htt_send_frag_desc_bank_cfg(htt);
+}
+
+static inline int ath10k_htt_alloc_frag_desc(struct ath10k_htt *htt)
+{
+	if (!htt->tx_ops->htt_alloc_frag_desc)
+		return -EOPNOTSUPP;
+
+	return htt->tx_ops->htt_alloc_frag_desc(htt);
+}
+
+static inline void ath10k_htt_free_frag_desc(struct ath10k_htt *htt)
+{
+	if (htt->tx_ops->htt_free_frag_desc)
+		htt->tx_ops->htt_free_frag_desc(htt);
+}
+
+static inline int ath10k_htt_tx(struct ath10k_htt *htt,
+				enum ath10k_hw_txrx_mode txmode,
+				struct sk_buff *msdu)
+{
+	return htt->tx_ops->htt_tx(htt, txmode, msdu);
+}
+
+static inline int ath10k_htt_alloc_txbuff(struct ath10k_htt *htt)
+{
+	if (!htt->tx_ops->htt_alloc_txbuff)
+		return -EOPNOTSUPP;
+
+	return htt->tx_ops->htt_alloc_txbuff(htt);
+}
+
+static inline void ath10k_htt_free_txbuff(struct ath10k_htt *htt)
+{
+	if (htt->tx_ops->htt_free_txbuff)
+		htt->tx_ops->htt_free_txbuff(htt);
+}
+
 struct ath10k_htt_rx_ops {
 	size_t (*htt_get_rx_ring_size)(struct ath10k_htt *htt);
 	void (*htt_config_paddrs_ring)(struct ath10k_htt *htt, void *vaddr);
@@ -1857,6 +1929,43 @@ struct ath10k_htt_rx_ops {
 	void (*htt_reset_paddrs_ring)(struct ath10k_htt *htt, int idx);
 };
 
+static inline size_t ath10k_htt_get_rx_ring_size(struct ath10k_htt *htt)
+{
+	if (!htt->rx_ops->htt_get_rx_ring_size)
+		return 0;
+
+	return htt->rx_ops->htt_get_rx_ring_size(htt);
+}
+
+static inline void ath10k_htt_config_paddrs_ring(struct ath10k_htt *htt,
+						 void *vaddr)
+{
+	if (htt->rx_ops->htt_config_paddrs_ring)
+		htt->rx_ops->htt_config_paddrs_ring(htt, vaddr);
+}
+
+static inline void ath10k_htt_set_paddrs_ring(struct ath10k_htt *htt,
+					      dma_addr_t paddr,
+					      int idx)
+{
+	if (htt->rx_ops->htt_set_paddrs_ring)
+		htt->rx_ops->htt_set_paddrs_ring(htt, paddr, idx);
+}
+
+static inline void *ath10k_htt_get_vaddr_ring(struct ath10k_htt *htt)
+{
+	if (!htt->rx_ops->htt_get_vaddr_ring)
+		return NULL;
+
+	return htt->rx_ops->htt_get_vaddr_ring(htt);
+}
+
+static inline void ath10k_htt_reset_paddrs_ring(struct ath10k_htt *htt, int idx)
+{
+	if (htt->rx_ops->htt_reset_paddrs_ring)
+		htt->rx_ops->htt_reset_paddrs_ring(htt, idx);
+}
+
 #define RX_HTT_HDR_STATUS_LEN 64
 
 /* This structure layout is programmed via rx ring setup
diff --git a/drivers/net/wireless/ath/ath10k/htt_rx.c b/drivers/net/wireless/ath/ath10k/htt_rx.c
index 5e02e26158f6..bd23f6940488 100644
--- a/drivers/net/wireless/ath/ath10k/htt_rx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_rx.c
@@ -25,6 +25,7 @@
 #include "mac.h"
 
 #include <linux/log2.h>
+#include <linux/bitfield.h>
 
 /* when under memory pressure rx ring refill may fail and needs a retry */
 #define HTT_RX_RING_REFILL_RETRY_MS 50
@@ -181,7 +182,7 @@ static int __ath10k_htt_rx_ring_fill_n(struct ath10k_htt *htt, int num)
 		rxcb = ATH10K_SKB_RXCB(skb);
 		rxcb->paddr = paddr;
 		htt->rx_ring.netbufs_ring[idx] = skb;
-		htt->rx_ops->htt_set_paddrs_ring(htt, paddr, idx);
+		ath10k_htt_set_paddrs_ring(htt, paddr, idx);
 		htt->rx_ring.fill_cnt++;
 
 		if (htt->rx_ring.in_ord_rx) {
@@ -286,8 +287,8 @@ void ath10k_htt_rx_free(struct ath10k_htt *htt)
 	ath10k_htt_rx_ring_free(htt);
 
 	dma_free_coherent(htt->ar->dev,
-			  htt->rx_ops->htt_get_rx_ring_size(htt),
-			  htt->rx_ops->htt_get_vaddr_ring(htt),
+			  ath10k_htt_get_rx_ring_size(htt),
+			  ath10k_htt_get_vaddr_ring(htt),
 			  htt->rx_ring.base_paddr);
 
 	dma_free_coherent(htt->ar->dev,
@@ -314,7 +315,7 @@ static inline struct sk_buff *ath10k_htt_rx_netbuf_pop(struct ath10k_htt *htt)
 	idx = htt->rx_ring.sw_rd_idx.msdu_payld;
 	msdu = htt->rx_ring.netbufs_ring[idx];
 	htt->rx_ring.netbufs_ring[idx] = NULL;
-	htt->rx_ops->htt_reset_paddrs_ring(htt, idx);
+	ath10k_htt_reset_paddrs_ring(htt, idx);
 
 	idx++;
 	idx &= htt->rx_ring.size_mask;
@@ -586,13 +587,13 @@ int ath10k_htt_rx_alloc(struct ath10k_htt *htt)
 	if (!htt->rx_ring.netbufs_ring)
 		goto err_netbuf;
 
-	size = htt->rx_ops->htt_get_rx_ring_size(htt);
+	size = ath10k_htt_get_rx_ring_size(htt);
 
 	vaddr_ring = dma_alloc_coherent(htt->ar->dev, size, &paddr, GFP_KERNEL);
 	if (!vaddr_ring)
 		goto err_dma_ring;
 
-	htt->rx_ops->htt_config_paddrs_ring(htt, vaddr_ring);
+	ath10k_htt_config_paddrs_ring(htt, vaddr_ring);
 	htt->rx_ring.base_paddr = paddr;
 
 	vaddr = dma_alloc_coherent(htt->ar->dev,
@@ -626,7 +627,7 @@ int ath10k_htt_rx_alloc(struct ath10k_htt *htt)
 
 err_dma_idx:
 	dma_free_coherent(htt->ar->dev,
-			  htt->rx_ops->htt_get_rx_ring_size(htt),
+			  ath10k_htt_get_rx_ring_size(htt),
 			  vaddr_ring,
 			  htt->rx_ring.base_paddr);
 err_dma_ring:
@@ -2719,12 +2720,21 @@ bool ath10k_htt_t2h_msg_handler(struct ath10k *ar, struct sk_buff *skb)
 	case HTT_T2H_MSG_TYPE_MGMT_TX_COMPLETION: {
 		struct htt_tx_done tx_done = {};
 		int status = __le32_to_cpu(resp->mgmt_tx_completion.status);
+		int info = __le32_to_cpu(resp->mgmt_tx_completion.info);
 
 		tx_done.msdu_id = __le32_to_cpu(resp->mgmt_tx_completion.desc_id);
 
 		switch (status) {
 		case HTT_MGMT_TX_STATUS_OK:
 			tx_done.status = HTT_TX_COMPL_STATE_ACK;
+			if (test_bit(WMI_SERVICE_HTT_MGMT_TX_COMP_VALID_FLAGS,
+				     ar->wmi.svc_map) &&
+			    (resp->mgmt_tx_completion.flags &
+			     HTT_MGMT_TX_CMPL_FLAG_ACK_RSSI)) {
+				tx_done.ack_rssi =
+				FIELD_GET(HTT_MGMT_TX_CMPL_INFO_ACK_RSSI_MASK,
+					  info);
+			}
 			break;
 		case HTT_MGMT_TX_STATUS_RETRY:
 			tx_done.status = HTT_TX_COMPL_STATE_NOACK;
diff --git a/drivers/net/wireless/ath/ath10k/htt_tx.c b/drivers/net/wireless/ath/ath10k/htt_tx.c
index d334b7be1fea..5d8b97a0ccaa 100644
--- a/drivers/net/wireless/ath/ath10k/htt_tx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_tx.c
@@ -443,13 +443,13 @@ static int ath10k_htt_tx_alloc_buf(struct ath10k_htt *htt)
 	struct ath10k *ar = htt->ar;
 	int ret;
 
-	ret = htt->tx_ops->htt_alloc_txbuff(htt);
+	ret = ath10k_htt_alloc_txbuff(htt);
 	if (ret) {
 		ath10k_err(ar, "failed to alloc cont tx buffer: %d\n", ret);
 		return ret;
 	}
 
-	ret = htt->tx_ops->htt_alloc_frag_desc(htt);
+	ret = ath10k_htt_alloc_frag_desc(htt);
 	if (ret) {
 		ath10k_err(ar, "failed to alloc cont frag desc: %d\n", ret);
 		goto free_txbuf;
@@ -473,10 +473,10 @@ static int ath10k_htt_tx_alloc_buf(struct ath10k_htt *htt)
 	ath10k_htt_tx_free_txq(htt);
 
 free_frag_desc:
-	htt->tx_ops->htt_free_frag_desc(htt);
+	ath10k_htt_free_frag_desc(htt);
 
 free_txbuf:
-	htt->tx_ops->htt_free_txbuff(htt);
+	ath10k_htt_free_txbuff(htt);
 
 	return ret;
 }
@@ -530,9 +530,9 @@ void ath10k_htt_tx_destroy(struct ath10k_htt *htt)
 	if (!htt->tx_mem_allocated)
 		return;
 
-	htt->tx_ops->htt_free_txbuff(htt);
+	ath10k_htt_free_txbuff(htt);
 	ath10k_htt_tx_free_txq(htt);
-	htt->tx_ops->htt_free_frag_desc(htt);
+	ath10k_htt_free_frag_desc(htt);
 	ath10k_htt_tx_free_txdone_fifo(htt);
 	htt->tx_mem_allocated = false;
 }
@@ -1475,8 +1475,11 @@ static int ath10k_htt_tx_64(struct ath10k_htt *htt,
 	    !test_bit(ATH10K_FLAG_RAW_MODE, &ar->dev_flags)) {
 		flags1 |= HTT_DATA_TX_DESC_FLAGS1_CKSUM_L3_OFFLOAD;
 		flags1 |= HTT_DATA_TX_DESC_FLAGS1_CKSUM_L4_OFFLOAD;
-		if (ar->hw_params.continuous_frag_desc)
-			ext_desc->flags |= HTT_MSDU_CHECKSUM_ENABLE;
+		if (ar->hw_params.continuous_frag_desc) {
+			memset(ext_desc->tso_flag, 0, sizeof(ext_desc->tso_flag));
+			ext_desc->tso_flag[3] |=
+				__cpu_to_le32(HTT_MSDU_CHECKSUM_ENABLE_64);
+		}
 	}
 
 	/* Prevent firmware from sending up tx inspection requests. There's
diff --git a/drivers/net/wireless/ath/ath10k/hw.c b/drivers/net/wireless/ath/ath10k/hw.c
index 497ac33e0fbf..677535b3d207 100644
--- a/drivers/net/wireless/ath/ath10k/hw.c
+++ b/drivers/net/wireless/ath/ath10k/hw.c
@@ -310,6 +310,12 @@ static struct ath10k_hw_ce_dst_src_wm_regs wcn3990_wm_dst_ring = {
 	.wm_high	= &wcn3990_dst_wm_high,
 };
 
+static struct ath10k_hw_ce_ctrl1_upd wcn3990_ctrl1_upd = {
+	.shift = 19,
+	.mask = 0x00080000,
+	.enable = 0x00000000,
+};
+
 const struct ath10k_hw_ce_regs wcn3990_ce_regs = {
 	.sr_base_addr		= 0x00000000,
 	.sr_size_addr		= 0x00000008,
@@ -320,8 +326,6 @@ const struct ath10k_hw_ce_regs wcn3990_ce_regs = {
 	.dst_wr_index_addr	= 0x00000040,
 	.current_srri_addr	= 0x00000044,
 	.current_drri_addr	= 0x00000048,
-	.ddr_addr_for_rri_low	= 0x00000004,
-	.ddr_addr_for_rri_high	= 0x00000008,
 	.ce_rri_low		= 0x0024C004,
 	.ce_rri_high		= 0x0024C008,
 	.host_ie_addr		= 0x0000002c,
@@ -331,6 +335,7 @@ const struct ath10k_hw_ce_regs wcn3990_ce_regs = {
 	.misc_regs		= &wcn3990_misc_reg,
 	.wm_srcr		= &wcn3990_wm_src_ring,
 	.wm_dstr		= &wcn3990_wm_dst_ring,
+	.upd			= &wcn3990_ctrl1_upd,
 };
 
 const struct ath10k_hw_values wcn3990_values = {
diff --git a/drivers/net/wireless/ath/ath10k/hw.h b/drivers/net/wireless/ath/ath10k/hw.h
index 413b1b4321f7..23467e9fefeb 100644
--- a/drivers/net/wireless/ath/ath10k/hw.h
+++ b/drivers/net/wireless/ath/ath10k/hw.h
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018 The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -84,11 +85,11 @@ enum qca9377_chip_id_rev {
 	QCA9377_HW_1_1_CHIP_ID_REV = 0x1,
 };
 
-#define QCA6174_HW_2_1_FW_DIR		"ath10k/QCA6174/hw2.1"
+#define QCA6174_HW_2_1_FW_DIR		ATH10K_FW_DIR "/QCA6174/hw2.1"
 #define QCA6174_HW_2_1_BOARD_DATA_FILE	"board.bin"
 #define QCA6174_HW_2_1_PATCH_LOAD_ADDR	0x1234
 
-#define QCA6174_HW_3_0_FW_DIR		"ath10k/QCA6174/hw3.0"
+#define QCA6174_HW_3_0_FW_DIR		ATH10K_FW_DIR "/QCA6174/hw3.0"
 #define QCA6174_HW_3_0_BOARD_DATA_FILE	"board.bin"
 #define QCA6174_HW_3_0_PATCH_LOAD_ADDR	0x1234
 
@@ -131,7 +132,7 @@ enum qca9377_chip_id_rev {
 
 /* WCN3990 1.0 definitions */
 #define WCN3990_HW_1_0_DEV_VERSION	ATH10K_HW_WCN3990
-#define WCN3990_HW_1_0_FW_DIR		ATH10K_FW_DIR "/WCN3990/hw3.0"
+#define WCN3990_HW_1_0_FW_DIR		ATH10K_FW_DIR "/WCN3990/hw1.0"
 
 #define ATH10K_FW_FILE_BASE		"firmware"
 #define ATH10K_FW_API_MAX		6
@@ -335,6 +336,12 @@ struct ath10k_hw_ce_dst_src_wm_regs {
 	struct ath10k_hw_ce_regs_addr_map *wm_low;
 	struct ath10k_hw_ce_regs_addr_map *wm_high; };
 
+struct ath10k_hw_ce_ctrl1_upd {
+	u32 shift;
+	u32 mask;
+	u32 enable;
+};
+
 struct ath10k_hw_ce_regs {
 	u32 sr_base_addr;
 	u32 sr_size_addr;
@@ -357,7 +364,9 @@ struct ath10k_hw_ce_regs {
 	struct ath10k_hw_ce_cmd_halt *cmd_halt;
 	struct ath10k_hw_ce_host_ie *host_ie;
 	struct ath10k_hw_ce_dst_src_wm_regs *wm_srcr;
-	struct ath10k_hw_ce_dst_src_wm_regs *wm_dstr; };
+	struct ath10k_hw_ce_dst_src_wm_regs *wm_dstr;
+	struct ath10k_hw_ce_ctrl1_upd *upd;
+};
 
 struct ath10k_hw_values {
 	u32 rtc_state_val_on;
@@ -568,6 +577,15 @@ struct ath10k_hw_params {
 
 	/* Target rx ring fill level */
 	u32 rx_ring_fill_level;
+
+	/* target supporting per ce IRQ */
+	bool per_ce_irq;
+
+	/* target supporting shadow register for ce write */
+	bool shadow_reg_support;
+
+	/* target supporting retention restore on ddr */
+	bool rri_on_ddr;
 };
 
 struct htt_rx_desc;
diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c
index bf05a3689558..e9c2fb318c03 100644
--- a/drivers/net/wireless/ath/ath10k/mac.c
+++ b/drivers/net/wireless/ath/ath10k/mac.c
@@ -3217,6 +3217,15 @@ static void ath10k_reg_notifier(struct wiphy *wiphy,
 					       ar->hw->wiphy->bands[NL80211_BAND_5GHZ]);
 }
 
+static void ath10k_stop_radar_confirmation(struct ath10k *ar)
+{
+	spin_lock_bh(&ar->data_lock);
+	ar->radar_conf_state = ATH10K_RADAR_CONFIRMATION_STOPPED;
+	spin_unlock_bh(&ar->data_lock);
+
+	cancel_work_sync(&ar->radar_confirmation_work);
+}
+
 /***************/
 /* TX handlers */
 /***************/
@@ -3598,7 +3607,7 @@ static int ath10k_mac_tx_submit(struct ath10k *ar,
 
 	switch (txpath) {
 	case ATH10K_MAC_TX_HTT:
-		ret = htt->tx_ops->htt_tx(htt, txmode, skb);
+		ret = ath10k_htt_tx(htt, txmode, skb);
 		break;
 	case ATH10K_MAC_TX_HTT_MGMT:
 		ret = ath10k_htt_mgmt_tx(htt, skb);
@@ -4290,7 +4299,7 @@ static void ath10k_mac_op_wake_tx_queue(struct ieee80211_hw *hw,
 
 	while (ath10k_mac_tx_can_push(hw, f_txq) && max--) {
 		ret = ath10k_mac_tx_push_txq(hw, f_txq);
-		if (ret)
+		if (ret < 0)
 			break;
 	}
 	if (ret != -ENOENT)
@@ -4333,6 +4342,7 @@ void ath10k_halt(struct ath10k *ar)
 
 	ath10k_scan_finish(ar);
 	ath10k_peer_cleanup_all(ar);
+	ath10k_stop_radar_confirmation(ar);
 	ath10k_core_stop(ar);
 	ath10k_hif_power_down(ar);
 
@@ -4679,6 +4689,13 @@ static int ath10k_start(struct ieee80211_hw *hw)
 		}
 	}
 
+	param = ar->wmi.pdev_param->idle_ps_config;
+	ret = ath10k_wmi_pdev_set_param(ar, param, 1);
+	if (ret && ret != -EOPNOTSUPP) {
+		ath10k_warn(ar, "failed to enable idle_ps_config: %d\n", ret);
+		goto err_core_stop;
+	}
+
 	__ath10k_set_antenna(ar, ar->cfg_tx_chainmask, ar->cfg_rx_chainmask);
 
 	/*
@@ -4751,6 +4768,8 @@ static int ath10k_start(struct ieee80211_hw *hw)
 	ath10k_spectral_start(ar);
 	ath10k_thermal_set_throttling(ar);
 
+	ar->radar_conf_state = ATH10K_RADAR_CONFIRMATION_IDLE;
+
 	mutex_unlock(&ar->conf_mutex);
 	return 0;
 
@@ -5668,6 +5687,7 @@ static int ath10k_hw_scan(struct ieee80211_hw *hw,
 	struct wmi_start_scan_arg arg;
 	int ret = 0;
 	int i;
+	u32 scan_timeout;
 
 	mutex_lock(&ar->conf_mutex);
 
@@ -5717,12 +5737,34 @@ static int ath10k_hw_scan(struct ieee80211_hw *hw,
 		arg.scan_ctrl_flags |= WMI_SCAN_FLAG_PASSIVE;
 	}
 
+	if (req->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) {
+		arg.scan_ctrl_flags |=  WMI_SCAN_ADD_SPOOFED_MAC_IN_PROBE_REQ;
+		ether_addr_copy(arg.mac_addr.addr, req->mac_addr);
+		ether_addr_copy(arg.mac_mask.addr, req->mac_addr_mask);
+	}
+
 	if (req->n_channels) {
 		arg.n_channels = req->n_channels;
 		for (i = 0; i < arg.n_channels; i++)
 			arg.channels[i] = req->channels[i]->center_freq;
 	}
 
+	/* if duration is set, default dwell times will be overwritten */
+	if (req->duration) {
+		arg.dwell_time_active = req->duration;
+		arg.dwell_time_passive = req->duration;
+		arg.burst_duration_ms = req->duration;
+
+		scan_timeout = min_t(u32, arg.max_rest_time *
+				(arg.n_channels - 1) + (req->duration +
+				ATH10K_SCAN_CHANNEL_SWITCH_WMI_EVT_OVERHEAD) *
+				arg.n_channels, arg.max_scan_time + 200);
+
+	} else {
+		/* Add a 200ms margin to account for event/command processing */
+		scan_timeout = arg.max_scan_time + 200;
+	}
+
 	ret = ath10k_start_scan(ar, &arg);
 	if (ret) {
 		ath10k_warn(ar, "failed to start hw scan: %d\n", ret);
@@ -5731,10 +5773,8 @@ static int ath10k_hw_scan(struct ieee80211_hw *hw,
 		spin_unlock_bh(&ar->data_lock);
 	}
 
-	/* Add a 200ms margin to account for event/command processing */
 	ieee80211_queue_delayed_work(ar->hw, &ar->scan.timeout,
-				     msecs_to_jiffies(arg.max_scan_time +
-						      200));
+				     msecs_to_jiffies(scan_timeout));
 
 exit:
 	mutex_unlock(&ar->conf_mutex);
@@ -8351,6 +8391,8 @@ int ath10k_mac_register(struct ath10k *ar)
 	}
 
 	wiphy_ext_feature_set(ar->hw->wiphy, NL80211_EXT_FEATURE_VHT_IBSS);
+	wiphy_ext_feature_set(ar->hw->wiphy,
+			      NL80211_EXT_FEATURE_SET_SCAN_DWELL);
 
 	/*
 	 * on LL hardware queues are managed entirely by the FW
@@ -8433,6 +8475,17 @@ int ath10k_mac_register(struct ath10k *ar)
 		goto err_dfs_detector_exit;
 	}
 
+	if (test_bit(WMI_SERVICE_SPOOF_MAC_SUPPORT, ar->wmi.svc_map)) {
+		ret = ath10k_wmi_scan_prob_req_oui(ar, ar->mac_addr);
+		if (ret) {
+			ath10k_err(ar, "failed to set prob req oui: %i\n", ret);
+			goto err_dfs_detector_exit;
+		}
+
+		ar->hw->wiphy->features |=
+			NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR;
+	}
+
 	ar->hw->wiphy->cipher_suites = cipher_suites;
 
 	/* QCA988x and QCA6174 family chips do not support CCMP-256, GCMP-128
diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c
index fd1566cd7d2b..af2cf55c4c1e 100644
--- a/drivers/net/wireless/ath/ath10k/pci.c
+++ b/drivers/net/wireless/ath/ath10k/pci.c
@@ -1383,8 +1383,8 @@ int ath10k_pci_hif_tx_sg(struct ath10k *ar, u8 pipe_id,
 
 	for (i = 0; i < n_items - 1; i++) {
 		ath10k_dbg(ar, ATH10K_DBG_PCI,
-			   "pci tx item %d paddr 0x%08x len %d n_items %d\n",
-			   i, items[i].paddr, items[i].len, n_items);
+			   "pci tx item %d paddr %pad len %d n_items %d\n",
+			   i, &items[i].paddr, items[i].len, n_items);
 		ath10k_dbg_dump(ar, ATH10K_DBG_PCI_DUMP, NULL, "pci tx data: ",
 				items[i].vaddr, items[i].len);
 
@@ -1401,8 +1401,8 @@ int ath10k_pci_hif_tx_sg(struct ath10k *ar, u8 pipe_id,
 	/* `i` is equal to `n_items -1` after for() */
 
 	ath10k_dbg(ar, ATH10K_DBG_PCI,
-		   "pci tx item %d paddr 0x%08x len %d n_items %d\n",
-		   i, items[i].paddr, items[i].len, n_items);
+		   "pci tx item %d paddr %pad len %d n_items %d\n",
+		   i, &items[i].paddr, items[i].len, n_items);
 	ath10k_dbg_dump(ar, ATH10K_DBG_PCI_DUMP, NULL, "pci tx data: ",
 			items[i].vaddr, items[i].len);
 
diff --git a/drivers/net/wireless/ath/ath10k/rx_desc.h b/drivers/net/wireless/ath/ath10k/rx_desc.h
index 545deb6d7af1..ea4075d456fa 100644
--- a/drivers/net/wireless/ath/ath10k/rx_desc.h
+++ b/drivers/net/wireless/ath/ath10k/rx_desc.h
@@ -18,39 +18,41 @@
 #ifndef _RX_DESC_H_
 #define _RX_DESC_H_
 
+#include <linux/bitops.h>
+
 enum rx_attention_flags {
-	RX_ATTENTION_FLAGS_FIRST_MPDU          = 1 << 0,
-	RX_ATTENTION_FLAGS_LAST_MPDU           = 1 << 1,
-	RX_ATTENTION_FLAGS_MCAST_BCAST         = 1 << 2,
-	RX_ATTENTION_FLAGS_PEER_IDX_INVALID    = 1 << 3,
-	RX_ATTENTION_FLAGS_PEER_IDX_TIMEOUT    = 1 << 4,
-	RX_ATTENTION_FLAGS_POWER_MGMT          = 1 << 5,
-	RX_ATTENTION_FLAGS_NON_QOS             = 1 << 6,
-	RX_ATTENTION_FLAGS_NULL_DATA           = 1 << 7,
-	RX_ATTENTION_FLAGS_MGMT_TYPE           = 1 << 8,
-	RX_ATTENTION_FLAGS_CTRL_TYPE           = 1 << 9,
-	RX_ATTENTION_FLAGS_MORE_DATA           = 1 << 10,
-	RX_ATTENTION_FLAGS_EOSP                = 1 << 11,
-	RX_ATTENTION_FLAGS_U_APSD_TRIGGER      = 1 << 12,
-	RX_ATTENTION_FLAGS_FRAGMENT            = 1 << 13,
-	RX_ATTENTION_FLAGS_ORDER               = 1 << 14,
-	RX_ATTENTION_FLAGS_CLASSIFICATION      = 1 << 15,
-	RX_ATTENTION_FLAGS_OVERFLOW_ERR        = 1 << 16,
-	RX_ATTENTION_FLAGS_MSDU_LENGTH_ERR     = 1 << 17,
-	RX_ATTENTION_FLAGS_TCP_UDP_CHKSUM_FAIL = 1 << 18,
-	RX_ATTENTION_FLAGS_IP_CHKSUM_FAIL      = 1 << 19,
-	RX_ATTENTION_FLAGS_SA_IDX_INVALID      = 1 << 20,
-	RX_ATTENTION_FLAGS_DA_IDX_INVALID      = 1 << 21,
-	RX_ATTENTION_FLAGS_SA_IDX_TIMEOUT      = 1 << 22,
-	RX_ATTENTION_FLAGS_DA_IDX_TIMEOUT      = 1 << 23,
-	RX_ATTENTION_FLAGS_ENCRYPT_REQUIRED    = 1 << 24,
-	RX_ATTENTION_FLAGS_DIRECTED            = 1 << 25,
-	RX_ATTENTION_FLAGS_BUFFER_FRAGMENT     = 1 << 26,
-	RX_ATTENTION_FLAGS_MPDU_LENGTH_ERR     = 1 << 27,
-	RX_ATTENTION_FLAGS_TKIP_MIC_ERR        = 1 << 28,
-	RX_ATTENTION_FLAGS_DECRYPT_ERR         = 1 << 29,
-	RX_ATTENTION_FLAGS_FCS_ERR             = 1 << 30,
-	RX_ATTENTION_FLAGS_MSDU_DONE           = 1 << 31,
+	RX_ATTENTION_FLAGS_FIRST_MPDU          = BIT(0),
+	RX_ATTENTION_FLAGS_LAST_MPDU           = BIT(1),
+	RX_ATTENTION_FLAGS_MCAST_BCAST         = BIT(2),
+	RX_ATTENTION_FLAGS_PEER_IDX_INVALID    = BIT(3),
+	RX_ATTENTION_FLAGS_PEER_IDX_TIMEOUT    = BIT(4),
+	RX_ATTENTION_FLAGS_POWER_MGMT          = BIT(5),
+	RX_ATTENTION_FLAGS_NON_QOS             = BIT(6),
+	RX_ATTENTION_FLAGS_NULL_DATA           = BIT(7),
+	RX_ATTENTION_FLAGS_MGMT_TYPE           = BIT(8),
+	RX_ATTENTION_FLAGS_CTRL_TYPE           = BIT(9),
+	RX_ATTENTION_FLAGS_MORE_DATA           = BIT(10),
+	RX_ATTENTION_FLAGS_EOSP                = BIT(11),
+	RX_ATTENTION_FLAGS_U_APSD_TRIGGER      = BIT(12),
+	RX_ATTENTION_FLAGS_FRAGMENT            = BIT(13),
+	RX_ATTENTION_FLAGS_ORDER               = BIT(14),
+	RX_ATTENTION_FLAGS_CLASSIFICATION      = BIT(15),
+	RX_ATTENTION_FLAGS_OVERFLOW_ERR        = BIT(16),
+	RX_ATTENTION_FLAGS_MSDU_LENGTH_ERR     = BIT(17),
+	RX_ATTENTION_FLAGS_TCP_UDP_CHKSUM_FAIL = BIT(18),
+	RX_ATTENTION_FLAGS_IP_CHKSUM_FAIL      = BIT(19),
+	RX_ATTENTION_FLAGS_SA_IDX_INVALID      = BIT(20),
+	RX_ATTENTION_FLAGS_DA_IDX_INVALID      = BIT(21),
+	RX_ATTENTION_FLAGS_SA_IDX_TIMEOUT      = BIT(22),
+	RX_ATTENTION_FLAGS_DA_IDX_TIMEOUT      = BIT(23),
+	RX_ATTENTION_FLAGS_ENCRYPT_REQUIRED    = BIT(24),
+	RX_ATTENTION_FLAGS_DIRECTED            = BIT(25),
+	RX_ATTENTION_FLAGS_BUFFER_FRAGMENT     = BIT(26),
+	RX_ATTENTION_FLAGS_MPDU_LENGTH_ERR     = BIT(27),
+	RX_ATTENTION_FLAGS_TKIP_MIC_ERR        = BIT(28),
+	RX_ATTENTION_FLAGS_DECRYPT_ERR         = BIT(29),
+	RX_ATTENTION_FLAGS_FCS_ERR             = BIT(30),
+	RX_ATTENTION_FLAGS_MSDU_DONE           = BIT(31),
 };
 
 struct rx_attention {
@@ -254,15 +256,15 @@ enum htt_rx_mpdu_encrypt_type {
 #define RX_MPDU_START_INFO0_SEQ_NUM_LSB       16
 #define RX_MPDU_START_INFO0_ENCRYPT_TYPE_MASK 0xf0000000
 #define RX_MPDU_START_INFO0_ENCRYPT_TYPE_LSB  28
-#define RX_MPDU_START_INFO0_FROM_DS           (1 << 11)
-#define RX_MPDU_START_INFO0_TO_DS             (1 << 12)
-#define RX_MPDU_START_INFO0_ENCRYPTED         (1 << 13)
-#define RX_MPDU_START_INFO0_RETRY             (1 << 14)
-#define RX_MPDU_START_INFO0_TXBF_H_INFO       (1 << 15)
+#define RX_MPDU_START_INFO0_FROM_DS           BIT(11)
+#define RX_MPDU_START_INFO0_TO_DS             BIT(12)
+#define RX_MPDU_START_INFO0_ENCRYPTED         BIT(13)
+#define RX_MPDU_START_INFO0_RETRY             BIT(14)
+#define RX_MPDU_START_INFO0_TXBF_H_INFO       BIT(15)
 
 #define RX_MPDU_START_INFO1_TID_MASK 0xf0000000
 #define RX_MPDU_START_INFO1_TID_LSB  28
-#define RX_MPDU_START_INFO1_DIRECTED (1 << 16)
+#define RX_MPDU_START_INFO1_DIRECTED BIT(16)
 
 struct rx_mpdu_start {
 	__le32 info0;
@@ -357,13 +359,13 @@ struct rx_mpdu_start {
 #define RX_MPDU_END_INFO0_RESERVED_0_LSB      0
 #define RX_MPDU_END_INFO0_POST_DELIM_CNT_MASK 0x0fff0000
 #define RX_MPDU_END_INFO0_POST_DELIM_CNT_LSB  16
-#define RX_MPDU_END_INFO0_OVERFLOW_ERR        (1 << 13)
-#define RX_MPDU_END_INFO0_LAST_MPDU           (1 << 14)
-#define RX_MPDU_END_INFO0_POST_DELIM_ERR      (1 << 15)
-#define RX_MPDU_END_INFO0_MPDU_LENGTH_ERR     (1 << 28)
-#define RX_MPDU_END_INFO0_TKIP_MIC_ERR        (1 << 29)
-#define RX_MPDU_END_INFO0_DECRYPT_ERR         (1 << 30)
-#define RX_MPDU_END_INFO0_FCS_ERR             (1 << 31)
+#define RX_MPDU_END_INFO0_OVERFLOW_ERR        BIT(13)
+#define RX_MPDU_END_INFO0_LAST_MPDU           BIT(14)
+#define RX_MPDU_END_INFO0_POST_DELIM_ERR      BIT(15)
+#define RX_MPDU_END_INFO0_MPDU_LENGTH_ERR     BIT(28)
+#define RX_MPDU_END_INFO0_TKIP_MIC_ERR        BIT(29)
+#define RX_MPDU_END_INFO0_DECRYPT_ERR         BIT(30)
+#define RX_MPDU_END_INFO0_FCS_ERR             BIT(31)
 
 struct rx_mpdu_end {
 	__le32 info0;
@@ -422,12 +424,12 @@ struct rx_mpdu_end {
 #define RX_MSDU_START_INFO1_DECAP_FORMAT_LSB    8
 #define RX_MSDU_START_INFO1_SA_IDX_MASK         0x07ff0000
 #define RX_MSDU_START_INFO1_SA_IDX_LSB          16
-#define RX_MSDU_START_INFO1_IPV4_PROTO          (1 << 10)
-#define RX_MSDU_START_INFO1_IPV6_PROTO          (1 << 11)
-#define RX_MSDU_START_INFO1_TCP_PROTO           (1 << 12)
-#define RX_MSDU_START_INFO1_UDP_PROTO           (1 << 13)
-#define RX_MSDU_START_INFO1_IP_FRAG             (1 << 14)
-#define RX_MSDU_START_INFO1_TCP_ONLY_ACK        (1 << 15)
+#define RX_MSDU_START_INFO1_IPV4_PROTO          BIT(10)
+#define RX_MSDU_START_INFO1_IPV6_PROTO          BIT(11)
+#define RX_MSDU_START_INFO1_TCP_PROTO           BIT(12)
+#define RX_MSDU_START_INFO1_UDP_PROTO           BIT(13)
+#define RX_MSDU_START_INFO1_IP_FRAG             BIT(14)
+#define RX_MSDU_START_INFO1_TCP_ONLY_ACK        BIT(15)
 
 #define RX_MSDU_START_INFO2_DA_IDX_MASK         0x000007ff
 #define RX_MSDU_START_INFO2_DA_IDX_LSB          0
@@ -568,10 +570,10 @@ struct rx_msdu_start {
 
 #define RX_MSDU_END_INFO0_REPORTED_MPDU_LENGTH_MASK 0x00003fff
 #define RX_MSDU_END_INFO0_REPORTED_MPDU_LENGTH_LSB  0
-#define RX_MSDU_END_INFO0_FIRST_MSDU                (1 << 14)
-#define RX_MSDU_END_INFO0_LAST_MSDU                 (1 << 15)
-#define RX_MSDU_END_INFO0_PRE_DELIM_ERR             (1 << 30)
-#define RX_MSDU_END_INFO0_RESERVED_3B               (1 << 31)
+#define RX_MSDU_END_INFO0_FIRST_MSDU                BIT(14)
+#define RX_MSDU_END_INFO0_LAST_MSDU                 BIT(15)
+#define RX_MSDU_END_INFO0_PRE_DELIM_ERR             BIT(30)
+#define RX_MSDU_END_INFO0_RESERVED_3B               BIT(31)
 
 struct rx_msdu_end_common {
 	__le16 ip_hdr_cksum;
@@ -691,7 +693,7 @@ struct rx_msdu_end {
 #define HTT_RX_PPDU_START_PREAMBLE_VHT           0x0C
 #define HTT_RX_PPDU_START_PREAMBLE_VHT_WITH_TXBF 0x0D
 
-#define RX_PPDU_START_INFO0_IS_GREENFIELD (1 << 0)
+#define RX_PPDU_START_INFO0_IS_GREENFIELD BIT(0)
 
 #define RX_PPDU_START_INFO1_L_SIG_RATE_MASK    0x0000000f
 #define RX_PPDU_START_INFO1_L_SIG_RATE_LSB     0
@@ -701,15 +703,15 @@ struct rx_msdu_end {
 #define RX_PPDU_START_INFO1_L_SIG_TAIL_LSB     18
 #define RX_PPDU_START_INFO1_PREAMBLE_TYPE_MASK 0xff000000
 #define RX_PPDU_START_INFO1_PREAMBLE_TYPE_LSB  24
-#define RX_PPDU_START_INFO1_L_SIG_RATE_SELECT  (1 << 4)
-#define RX_PPDU_START_INFO1_L_SIG_PARITY       (1 << 17)
+#define RX_PPDU_START_INFO1_L_SIG_RATE_SELECT  BIT(4)
+#define RX_PPDU_START_INFO1_L_SIG_PARITY       BIT(17)
 
 #define RX_PPDU_START_INFO2_HT_SIG_VHT_SIG_A_1_MASK 0x00ffffff
 #define RX_PPDU_START_INFO2_HT_SIG_VHT_SIG_A_1_LSB  0
 
 #define RX_PPDU_START_INFO3_HT_SIG_VHT_SIG_A_2_MASK 0x00ffffff
 #define RX_PPDU_START_INFO3_HT_SIG_VHT_SIG_A_2_LSB  0
-#define RX_PPDU_START_INFO3_TXBF_H_INFO             (1 << 24)
+#define RX_PPDU_START_INFO3_TXBF_H_INFO             BIT(24)
 
 #define RX_PPDU_START_INFO4_VHT_SIG_B_MASK 0x1fffffff
 #define RX_PPDU_START_INFO4_VHT_SIG_B_LSB  0
@@ -898,14 +900,14 @@ struct rx_ppdu_start {
  *		Reserved: HW should fill with 0, FW should ignore.
  */
 
-#define RX_PPDU_END_FLAGS_PHY_ERR             (1 << 0)
-#define RX_PPDU_END_FLAGS_RX_LOCATION         (1 << 1)
-#define RX_PPDU_END_FLAGS_TXBF_H_INFO         (1 << 2)
+#define RX_PPDU_END_FLAGS_PHY_ERR             BIT(0)
+#define RX_PPDU_END_FLAGS_RX_LOCATION         BIT(1)
+#define RX_PPDU_END_FLAGS_TXBF_H_INFO         BIT(2)
 
 #define RX_PPDU_END_INFO0_RX_ANTENNA_MASK     0x00ffffff
 #define RX_PPDU_END_INFO0_RX_ANTENNA_LSB      0
-#define RX_PPDU_END_INFO0_FLAGS_TX_HT_VHT_ACK (1 << 24)
-#define RX_PPDU_END_INFO0_BB_CAPTURED_CHANNEL (1 << 25)
+#define RX_PPDU_END_INFO0_FLAGS_TX_HT_VHT_ACK BIT(24)
+#define RX_PPDU_END_INFO0_BB_CAPTURED_CHANNEL BIT(25)
 
 #define RX_PPDU_END_INFO1_PEER_IDX_MASK       0x1ffc
 #define RX_PPDU_END_INFO1_PEER_IDX_LSB        2
@@ -1265,9 +1267,9 @@ struct rx_ppdu_end {
  *		to 0.
  */
 
-#define FW_RX_DESC_INFO0_DISCARD  (1 << 0)
-#define FW_RX_DESC_INFO0_FORWARD  (1 << 1)
-#define FW_RX_DESC_INFO0_INSPECT  (1 << 5)
+#define FW_RX_DESC_INFO0_DISCARD  BIT(0)
+#define FW_RX_DESC_INFO0_FORWARD  BIT(1)
+#define FW_RX_DESC_INFO0_INSPECT  BIT(5)
 #define FW_RX_DESC_INFO0_EXT_MASK 0xC0
 #define FW_RX_DESC_INFO0_EXT_LSB  6
 
diff --git a/drivers/net/wireless/ath/ath10k/sdio.c b/drivers/net/wireless/ath/ath10k/sdio.c
index 03a69e5b1116..d612ce8c9cff 100644
--- a/drivers/net/wireless/ath/ath10k/sdio.c
+++ b/drivers/net/wireless/ath/ath10k/sdio.c
@@ -1957,25 +1957,25 @@ static int ath10k_sdio_probe(struct sdio_func *func,
 	ar_sdio = ath10k_sdio_priv(ar);
 
 	ar_sdio->irq_data.irq_proc_reg =
-		kzalloc(sizeof(struct ath10k_sdio_irq_proc_regs),
-			GFP_KERNEL);
+		devm_kzalloc(ar->dev, sizeof(struct ath10k_sdio_irq_proc_regs),
+			     GFP_KERNEL);
 	if (!ar_sdio->irq_data.irq_proc_reg) {
 		ret = -ENOMEM;
 		goto err_core_destroy;
 	}
 
 	ar_sdio->irq_data.irq_en_reg =
-		kzalloc(sizeof(struct ath10k_sdio_irq_enable_regs),
-			GFP_KERNEL);
+		devm_kzalloc(ar->dev, sizeof(struct ath10k_sdio_irq_enable_regs),
+			     GFP_KERNEL);
 	if (!ar_sdio->irq_data.irq_en_reg) {
 		ret = -ENOMEM;
-		goto err_free_proc_reg;
+		goto err_core_destroy;
 	}
 
-	ar_sdio->bmi_buf = kzalloc(BMI_MAX_CMDBUF_SIZE, GFP_KERNEL);
+	ar_sdio->bmi_buf = devm_kzalloc(ar->dev, BMI_MAX_CMDBUF_SIZE, GFP_KERNEL);
 	if (!ar_sdio->bmi_buf) {
 		ret = -ENOMEM;
-		goto err_free_en_reg;
+		goto err_core_destroy;
 	}
 
 	ar_sdio->func = func;
@@ -1995,7 +1995,7 @@ static int ath10k_sdio_probe(struct sdio_func *func,
 	ar_sdio->workqueue = create_singlethread_workqueue("ath10k_sdio_wq");
 	if (!ar_sdio->workqueue) {
 		ret = -ENOMEM;
-		goto err_free_bmi_buf;
+		goto err_core_destroy;
 	}
 
 	for (i = 0; i < ATH10K_SDIO_BUS_REQUEST_MAX_NUM; i++)
@@ -2011,7 +2011,7 @@ static int ath10k_sdio_probe(struct sdio_func *func,
 		ret = -ENODEV;
 		ath10k_err(ar, "unsupported device id %u (0x%x)\n",
 			   dev_id_base, id->device);
-		goto err_free_bmi_buf;
+		goto err_free_wq;
 	}
 
 	ar->id.vendor = id->vendor;
@@ -2040,12 +2040,6 @@ static int ath10k_sdio_probe(struct sdio_func *func,
 
 err_free_wq:
 	destroy_workqueue(ar_sdio->workqueue);
-err_free_bmi_buf:
-	kfree(ar_sdio->bmi_buf);
-err_free_en_reg:
-	kfree(ar_sdio->irq_data.irq_en_reg);
-err_free_proc_reg:
-	kfree(ar_sdio->irq_data.irq_proc_reg);
 err_core_destroy:
 	ath10k_core_destroy(ar);
 
diff --git a/drivers/net/wireless/ath/ath10k/snoc.c b/drivers/net/wireless/ath/ath10k/snoc.c
new file mode 100644
index 000000000000..a3a7042fe13a
--- /dev/null
+++ b/drivers/net/wireless/ath/ath10k/snoc.c
@@ -0,0 +1,1413 @@
+/*
+ * Copyright (c) 2018 The Linux Foundation. All rights reserved.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include "debug.h"
+#include "hif.h"
+#include "htc.h"
+#include "ce.h"
+#include "snoc.h"
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/clk.h>
+#define  WCN3990_CE_ATTR_FLAGS 0
+#define ATH10K_SNOC_RX_POST_RETRY_MS 50
+#define CE_POLL_PIPE 4
+
+static char *const ce_name[] = {
+	"WLAN_CE_0",
+	"WLAN_CE_1",
+	"WLAN_CE_2",
+	"WLAN_CE_3",
+	"WLAN_CE_4",
+	"WLAN_CE_5",
+	"WLAN_CE_6",
+	"WLAN_CE_7",
+	"WLAN_CE_8",
+	"WLAN_CE_9",
+	"WLAN_CE_10",
+	"WLAN_CE_11",
+};
+
+static struct ath10k_wcn3990_vreg_info vreg_cfg[] = {
+	{NULL, "vdd-0.8-cx-mx", 800000, 800000, 0, 0, false},
+	{NULL, "vdd-1.8-xo", 1800000, 1800000, 0, 0, false},
+	{NULL, "vdd-1.3-rfa", 1304000, 1304000, 0, 0, false},
+	{NULL, "vdd-3.3-ch0", 3312000, 3312000, 0, 0, false},
+};
+
+static struct ath10k_wcn3990_clk_info clk_cfg[] = {
+	{NULL, "cxo_ref_clk_pin", 0, false},
+};
+
+static void ath10k_snoc_htc_tx_cb(struct ath10k_ce_pipe *ce_state);
+static void ath10k_snoc_htt_tx_cb(struct ath10k_ce_pipe *ce_state);
+static void ath10k_snoc_htc_rx_cb(struct ath10k_ce_pipe *ce_state);
+static void ath10k_snoc_htt_rx_cb(struct ath10k_ce_pipe *ce_state);
+static void ath10k_snoc_htt_htc_rx_cb(struct ath10k_ce_pipe *ce_state);
+
+static const struct ath10k_snoc_drv_priv drv_priv = {
+	.hw_rev = ATH10K_HW_WCN3990,
+	.dma_mask = DMA_BIT_MASK(37),
+};
+
+static struct ce_attr host_ce_config_wlan[] = {
+	/* CE0: host->target HTC control streams */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 16,
+		.src_sz_max = 2048,
+		.dest_nentries = 0,
+		.send_cb = ath10k_snoc_htc_tx_cb,
+	},
+
+	/* CE1: target->host HTT + HTC control */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 512,
+		.recv_cb = ath10k_snoc_htt_htc_rx_cb,
+	},
+
+	/* CE2: target->host WMI */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 64,
+		.recv_cb = ath10k_snoc_htc_rx_cb,
+	},
+
+	/* CE3: host->target WMI */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 32,
+		.src_sz_max = 2048,
+		.dest_nentries = 0,
+		.send_cb = ath10k_snoc_htc_tx_cb,
+	},
+
+	/* CE4: host->target HTT */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 256,
+		.src_sz_max = 256,
+		.dest_nentries = 0,
+		.send_cb = ath10k_snoc_htt_tx_cb,
+	},
+
+	/* CE5: target->host HTT (ipa_uc->target ) */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 512,
+		.dest_nentries = 512,
+		.recv_cb = ath10k_snoc_htt_rx_cb,
+	},
+
+	/* CE6: target autonomous hif_memcpy */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE7: ce_diag, the Diagnostic Window */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 2,
+		.src_sz_max = 2048,
+		.dest_nentries = 2,
+	},
+
+	/* CE8: Target to uMC */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 128,
+	},
+
+	/* CE9 target->host HTT */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 512,
+		.recv_cb = ath10k_snoc_htt_htc_rx_cb,
+	},
+
+	/* CE10: target->host HTT */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 512,
+		.recv_cb = ath10k_snoc_htt_htc_rx_cb,
+	},
+
+	/* CE11: target -> host PKTLOG */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 512,
+		.recv_cb = ath10k_snoc_htt_htc_rx_cb,
+	},
+};
+
+static struct service_to_pipe target_service_to_ce_map_wlan[] = {
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_WMI_DATA_VO),
+		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_WMI_DATA_VO),
+		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_WMI_DATA_BK),
+		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_WMI_DATA_BK),
+		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_WMI_DATA_BE),
+		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_WMI_DATA_BE),
+		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_WMI_DATA_VI),
+		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_WMI_DATA_VI),
+		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_WMI_CONTROL),
+		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_WMI_CONTROL),
+		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_RSVD_CTRL),
+		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
+		__cpu_to_le32(0),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_RSVD_CTRL),
+		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
+		__cpu_to_le32(2),
+	},
+	{ /* not used */
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_TEST_RAW_STREAMS),
+		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
+		__cpu_to_le32(0),
+	},
+	{ /* not used */
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_TEST_RAW_STREAMS),
+		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_HTT_DATA_MSG),
+		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
+		__cpu_to_le32(4),
+	},
+	{
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_HTT_DATA_MSG),
+		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
+		__cpu_to_le32(1),
+	},
+	{ /* not used */
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_TEST_RAW_STREAMS),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(5),
+	},
+	{ /* in = DL = target -> host */
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_HTT_DATA2_MSG),
+		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
+		__cpu_to_le32(9),
+	},
+	{ /* in = DL = target -> host */
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_HTT_DATA3_MSG),
+		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
+		__cpu_to_le32(10),
+	},
+	{ /* in = DL = target -> host pktlog */
+		__cpu_to_le32(ATH10K_HTC_SVC_ID_HTT_LOG_MSG),
+		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
+		__cpu_to_le32(11),
+	},
+	/* (Additions here) */
+
+	{ /* must be last */
+		__cpu_to_le32(0),
+		__cpu_to_le32(0),
+		__cpu_to_le32(0),
+	},
+};
+
+void ath10k_snoc_write32(struct ath10k *ar, u32 offset, u32 value)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+
+	iowrite32(value, ar_snoc->mem + offset);
+}
+
+u32 ath10k_snoc_read32(struct ath10k *ar, u32 offset)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	u32 val;
+
+	val = ioread32(ar_snoc->mem + offset);
+
+	return val;
+}
+
+static int __ath10k_snoc_rx_post_buf(struct ath10k_snoc_pipe *pipe)
+{
+	struct ath10k_ce_pipe *ce_pipe = pipe->ce_hdl;
+	struct ath10k *ar = pipe->hif_ce_state;
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct sk_buff *skb;
+	dma_addr_t paddr;
+	int ret;
+
+	skb = dev_alloc_skb(pipe->buf_sz);
+	if (!skb)
+		return -ENOMEM;
+
+	WARN_ONCE((unsigned long)skb->data & 3, "unaligned skb");
+
+	paddr = dma_map_single(ar->dev, skb->data,
+			       skb->len + skb_tailroom(skb),
+			       DMA_FROM_DEVICE);
+	if (unlikely(dma_mapping_error(ar->dev, paddr))) {
+		ath10k_warn(ar, "failed to dma map snoc rx buf\n");
+		dev_kfree_skb_any(skb);
+		return -EIO;
+	}
+
+	ATH10K_SKB_RXCB(skb)->paddr = paddr;
+
+	spin_lock_bh(&ce->ce_lock);
+	ret = ce_pipe->ops->ce_rx_post_buf(ce_pipe, skb, paddr);
+	spin_unlock_bh(&ce->ce_lock);
+	if (ret) {
+		dma_unmap_single(ar->dev, paddr, skb->len + skb_tailroom(skb),
+				 DMA_FROM_DEVICE);
+		dev_kfree_skb_any(skb);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void ath10k_snoc_rx_post_pipe(struct ath10k_snoc_pipe *pipe)
+{
+	struct ath10k *ar = pipe->hif_ce_state;
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	struct ath10k_ce_pipe *ce_pipe = pipe->ce_hdl;
+	int ret, num;
+
+	if (pipe->buf_sz == 0)
+		return;
+
+	if (!ce_pipe->dest_ring)
+		return;
+
+	spin_lock_bh(&ce->ce_lock);
+	num = __ath10k_ce_rx_num_free_bufs(ce_pipe);
+	spin_unlock_bh(&ce->ce_lock);
+	while (num--) {
+		ret = __ath10k_snoc_rx_post_buf(pipe);
+		if (ret) {
+			if (ret == -ENOSPC)
+				break;
+			ath10k_warn(ar, "failed to post rx buf: %d\n", ret);
+			mod_timer(&ar_snoc->rx_post_retry, jiffies +
+				  ATH10K_SNOC_RX_POST_RETRY_MS);
+			break;
+		}
+	}
+}
+
+static void ath10k_snoc_rx_post(struct ath10k *ar)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	int i;
+
+	for (i = 0; i < CE_COUNT; i++)
+		ath10k_snoc_rx_post_pipe(&ar_snoc->pipe_info[i]);
+}
+
+static void ath10k_snoc_process_rx_cb(struct ath10k_ce_pipe *ce_state,
+				      void (*callback)(struct ath10k *ar,
+						       struct sk_buff *skb))
+{
+	struct ath10k *ar = ce_state->ar;
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	struct ath10k_snoc_pipe *pipe_info =  &ar_snoc->pipe_info[ce_state->id];
+	struct sk_buff *skb;
+	struct sk_buff_head list;
+	void *transfer_context;
+	unsigned int nbytes, max_nbytes;
+
+	__skb_queue_head_init(&list);
+	while (ath10k_ce_completed_recv_next(ce_state, &transfer_context,
+					     &nbytes) == 0) {
+		skb = transfer_context;
+		max_nbytes = skb->len + skb_tailroom(skb);
+		dma_unmap_single(ar->dev, ATH10K_SKB_RXCB(skb)->paddr,
+				 max_nbytes, DMA_FROM_DEVICE);
+
+		if (unlikely(max_nbytes < nbytes)) {
+			ath10k_warn(ar, "rxed more than expected (nbytes %d, max %d)",
+				    nbytes, max_nbytes);
+			dev_kfree_skb_any(skb);
+			continue;
+		}
+
+		skb_put(skb, nbytes);
+		__skb_queue_tail(&list, skb);
+	}
+
+	while ((skb = __skb_dequeue(&list))) {
+		ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc rx ce pipe %d len %d\n",
+			   ce_state->id, skb->len);
+
+		callback(ar, skb);
+	}
+
+	ath10k_snoc_rx_post_pipe(pipe_info);
+}
+
+static void ath10k_snoc_htc_rx_cb(struct ath10k_ce_pipe *ce_state)
+{
+	ath10k_snoc_process_rx_cb(ce_state, ath10k_htc_rx_completion_handler);
+}
+
+static void ath10k_snoc_htt_htc_rx_cb(struct ath10k_ce_pipe *ce_state)
+{
+	/* CE4 polling needs to be done whenever CE pipe which transports
+	 * HTT Rx (target->host) is processed.
+	 */
+	ath10k_ce_per_engine_service(ce_state->ar, CE_POLL_PIPE);
+
+	ath10k_snoc_process_rx_cb(ce_state, ath10k_htc_rx_completion_handler);
+}
+
+static void ath10k_snoc_htt_rx_deliver(struct ath10k *ar, struct sk_buff *skb)
+{
+	skb_pull(skb, sizeof(struct ath10k_htc_hdr));
+	ath10k_htt_t2h_msg_handler(ar, skb);
+}
+
+static void ath10k_snoc_htt_rx_cb(struct ath10k_ce_pipe *ce_state)
+{
+	ath10k_ce_per_engine_service(ce_state->ar, CE_POLL_PIPE);
+	ath10k_snoc_process_rx_cb(ce_state, ath10k_snoc_htt_rx_deliver);
+}
+
+static void ath10k_snoc_rx_replenish_retry(struct timer_list *t)
+{
+	struct ath10k_pci *ar_snoc = from_timer(ar_snoc, t, rx_post_retry);
+	struct ath10k *ar = ar_snoc->ar;
+
+	ath10k_snoc_rx_post(ar);
+}
+
+static void ath10k_snoc_htc_tx_cb(struct ath10k_ce_pipe *ce_state)
+{
+	struct ath10k *ar = ce_state->ar;
+	struct sk_buff_head list;
+	struct sk_buff *skb;
+
+	__skb_queue_head_init(&list);
+	while (ath10k_ce_completed_send_next(ce_state, (void **)&skb) == 0) {
+		if (!skb)
+			continue;
+
+		__skb_queue_tail(&list, skb);
+	}
+
+	while ((skb = __skb_dequeue(&list)))
+		ath10k_htc_tx_completion_handler(ar, skb);
+}
+
+static void ath10k_snoc_htt_tx_cb(struct ath10k_ce_pipe *ce_state)
+{
+	struct ath10k *ar = ce_state->ar;
+	struct sk_buff *skb;
+
+	while (ath10k_ce_completed_send_next(ce_state, (void **)&skb) == 0) {
+		if (!skb)
+			continue;
+
+		dma_unmap_single(ar->dev, ATH10K_SKB_CB(skb)->paddr,
+				 skb->len, DMA_TO_DEVICE);
+		ath10k_htt_hif_tx_complete(ar, skb);
+	}
+}
+
+static int ath10k_snoc_hif_tx_sg(struct ath10k *ar, u8 pipe_id,
+				 struct ath10k_hif_sg_item *items, int n_items)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_snoc_pipe *snoc_pipe;
+	struct ath10k_ce_pipe *ce_pipe;
+	int err, i = 0;
+
+	snoc_pipe = &ar_snoc->pipe_info[pipe_id];
+	ce_pipe = snoc_pipe->ce_hdl;
+	spin_lock_bh(&ce->ce_lock);
+
+	for (i = 0; i < n_items - 1; i++) {
+		ath10k_dbg(ar, ATH10K_DBG_SNOC,
+			   "snoc tx item %d paddr %pad len %d n_items %d\n",
+			   i, &items[i].paddr, items[i].len, n_items);
+
+		err = ath10k_ce_send_nolock(ce_pipe,
+					    items[i].transfer_context,
+					    items[i].paddr,
+					    items[i].len,
+					    items[i].transfer_id,
+					    CE_SEND_FLAG_GATHER);
+		if (err)
+			goto err;
+	}
+
+	ath10k_dbg(ar, ATH10K_DBG_SNOC,
+		   "snoc tx item %d paddr %pad len %d n_items %d\n",
+		   i, &items[i].paddr, items[i].len, n_items);
+
+	err = ath10k_ce_send_nolock(ce_pipe,
+				    items[i].transfer_context,
+				    items[i].paddr,
+				    items[i].len,
+				    items[i].transfer_id,
+				    0);
+	if (err)
+		goto err;
+
+	spin_unlock_bh(&ce->ce_lock);
+
+	return 0;
+
+err:
+	for (; i > 0; i--)
+		__ath10k_ce_send_revert(ce_pipe);
+
+	spin_unlock_bh(&ce->ce_lock);
+	return err;
+}
+
+static int ath10k_snoc_hif_get_target_info(struct ath10k *ar,
+					   struct bmi_target_info *target_info)
+{
+	target_info->version = ATH10K_HW_WCN3990;
+	target_info->type = ATH10K_HW_WCN3990;
+
+	return 0;
+}
+
+static u16 ath10k_snoc_hif_get_free_queue_number(struct ath10k *ar, u8 pipe)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+
+	ath10k_dbg(ar, ATH10K_DBG_SNOC, "hif get free queue number\n");
+
+	return ath10k_ce_num_free_src_entries(ar_snoc->pipe_info[pipe].ce_hdl);
+}
+
+static void ath10k_snoc_hif_send_complete_check(struct ath10k *ar, u8 pipe,
+						int force)
+{
+	int resources;
+
+	ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc hif send complete check\n");
+
+	if (!force) {
+		resources = ath10k_snoc_hif_get_free_queue_number(ar, pipe);
+
+		if (resources > (host_ce_config_wlan[pipe].src_nentries >> 1))
+			return;
+	}
+	ath10k_ce_per_engine_service(ar, pipe);
+}
+
+static int ath10k_snoc_hif_map_service_to_pipe(struct ath10k *ar,
+					       u16 service_id,
+					       u8 *ul_pipe, u8 *dl_pipe)
+{
+	const struct service_to_pipe *entry;
+	bool ul_set = false, dl_set = false;
+	int i;
+
+	ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc hif map service\n");
+
+	for (i = 0; i < ARRAY_SIZE(target_service_to_ce_map_wlan); i++) {
+		entry = &target_service_to_ce_map_wlan[i];
+
+		if (__le32_to_cpu(entry->service_id) != service_id)
+			continue;
+
+		switch (__le32_to_cpu(entry->pipedir)) {
+		case PIPEDIR_NONE:
+			break;
+		case PIPEDIR_IN:
+			WARN_ON(dl_set);
+			*dl_pipe = __le32_to_cpu(entry->pipenum);
+			dl_set = true;
+			break;
+		case PIPEDIR_OUT:
+			WARN_ON(ul_set);
+			*ul_pipe = __le32_to_cpu(entry->pipenum);
+			ul_set = true;
+			break;
+		case PIPEDIR_INOUT:
+			WARN_ON(dl_set);
+			WARN_ON(ul_set);
+			*dl_pipe = __le32_to_cpu(entry->pipenum);
+			*ul_pipe = __le32_to_cpu(entry->pipenum);
+			dl_set = true;
+			ul_set = true;
+			break;
+		}
+	}
+
+	if (WARN_ON(!ul_set || !dl_set))
+		return -ENOENT;
+
+	return 0;
+}
+
+static void ath10k_snoc_hif_get_default_pipe(struct ath10k *ar,
+					     u8 *ul_pipe, u8 *dl_pipe)
+{
+	ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc hif get default pipe\n");
+
+	(void)ath10k_snoc_hif_map_service_to_pipe(ar,
+						 ATH10K_HTC_SVC_ID_RSVD_CTRL,
+						 ul_pipe, dl_pipe);
+}
+
+static inline void ath10k_snoc_irq_disable(struct ath10k *ar)
+{
+	ath10k_ce_disable_interrupts(ar);
+}
+
+static inline void ath10k_snoc_irq_enable(struct ath10k *ar)
+{
+	ath10k_ce_enable_interrupts(ar);
+}
+
+static void ath10k_snoc_rx_pipe_cleanup(struct ath10k_snoc_pipe *snoc_pipe)
+{
+	struct ath10k_ce_pipe *ce_pipe;
+	struct ath10k_ce_ring *ce_ring;
+	struct sk_buff *skb;
+	struct ath10k *ar;
+	int i;
+
+	ar = snoc_pipe->hif_ce_state;
+	ce_pipe = snoc_pipe->ce_hdl;
+	ce_ring = ce_pipe->dest_ring;
+
+	if (!ce_ring)
+		return;
+
+	if (!snoc_pipe->buf_sz)
+		return;
+
+	for (i = 0; i < ce_ring->nentries; i++) {
+		skb = ce_ring->per_transfer_context[i];
+		if (!skb)
+			continue;
+
+		ce_ring->per_transfer_context[i] = NULL;
+
+		dma_unmap_single(ar->dev, ATH10K_SKB_RXCB(skb)->paddr,
+				 skb->len + skb_tailroom(skb),
+				 DMA_FROM_DEVICE);
+		dev_kfree_skb_any(skb);
+	}
+}
+
+static void ath10k_snoc_tx_pipe_cleanup(struct ath10k_snoc_pipe *snoc_pipe)
+{
+	struct ath10k_ce_pipe *ce_pipe;
+	struct ath10k_ce_ring *ce_ring;
+	struct ath10k_snoc *ar_snoc;
+	struct sk_buff *skb;
+	struct ath10k *ar;
+	int i;
+
+	ar = snoc_pipe->hif_ce_state;
+	ar_snoc = ath10k_snoc_priv(ar);
+	ce_pipe = snoc_pipe->ce_hdl;
+	ce_ring = ce_pipe->src_ring;
+
+	if (!ce_ring)
+		return;
+
+	if (!snoc_pipe->buf_sz)
+		return;
+
+	for (i = 0; i < ce_ring->nentries; i++) {
+		skb = ce_ring->per_transfer_context[i];
+		if (!skb)
+			continue;
+
+		ce_ring->per_transfer_context[i] = NULL;
+
+		ath10k_htc_tx_completion_handler(ar, skb);
+	}
+}
+
+static void ath10k_snoc_buffer_cleanup(struct ath10k *ar)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	struct ath10k_snoc_pipe *pipe_info;
+	int pipe_num;
+
+	del_timer_sync(&ar_snoc->rx_post_retry);
+	for (pipe_num = 0; pipe_num < CE_COUNT; pipe_num++) {
+		pipe_info = &ar_snoc->pipe_info[pipe_num];
+		ath10k_snoc_rx_pipe_cleanup(pipe_info);
+		ath10k_snoc_tx_pipe_cleanup(pipe_info);
+	}
+}
+
+static void ath10k_snoc_hif_stop(struct ath10k *ar)
+{
+	ath10k_snoc_irq_disable(ar);
+	ath10k_snoc_buffer_cleanup(ar);
+	napi_synchronize(&ar->napi);
+	napi_disable(&ar->napi);
+	ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot hif stop\n");
+}
+
+static int ath10k_snoc_hif_start(struct ath10k *ar)
+{
+	ath10k_snoc_irq_enable(ar);
+	ath10k_snoc_rx_post(ar);
+
+	ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot hif start\n");
+
+	return 0;
+}
+
+static int ath10k_snoc_init_pipes(struct ath10k *ar)
+{
+	int i, ret;
+
+	for (i = 0; i < CE_COUNT; i++) {
+		ret = ath10k_ce_init_pipe(ar, i, &host_ce_config_wlan[i]);
+		if (ret) {
+			ath10k_err(ar, "failed to initialize copy engine pipe %d: %d\n",
+				   i, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int ath10k_snoc_wlan_enable(struct ath10k *ar)
+{
+	return 0;
+}
+
+static void ath10k_snoc_wlan_disable(struct ath10k *ar)
+{
+}
+
+static void ath10k_snoc_hif_power_down(struct ath10k *ar)
+{
+	ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot hif power down\n");
+
+	ath10k_snoc_wlan_disable(ar);
+	ath10k_ce_free_rri(ar);
+}
+
+static int ath10k_snoc_hif_power_up(struct ath10k *ar)
+{
+	int ret;
+
+	ath10k_dbg(ar, ATH10K_DBG_SNOC, "%s:WCN3990 driver state = %d\n",
+		   __func__, ar->state);
+
+	ret = ath10k_snoc_wlan_enable(ar);
+	if (ret) {
+		ath10k_err(ar, "failed to enable wcn3990: %d\n", ret);
+		return ret;
+	}
+
+	ath10k_ce_alloc_rri(ar);
+
+	ret = ath10k_snoc_init_pipes(ar);
+	if (ret) {
+		ath10k_err(ar, "failed to initialize CE: %d\n", ret);
+		goto err_wlan_enable;
+	}
+
+	napi_enable(&ar->napi);
+	return 0;
+
+err_wlan_enable:
+	ath10k_snoc_wlan_disable(ar);
+
+	return ret;
+}
+
+static const struct ath10k_hif_ops ath10k_snoc_hif_ops = {
+	.read32		= ath10k_snoc_read32,
+	.write32	= ath10k_snoc_write32,
+	.start		= ath10k_snoc_hif_start,
+	.stop		= ath10k_snoc_hif_stop,
+	.map_service_to_pipe	= ath10k_snoc_hif_map_service_to_pipe,
+	.get_default_pipe	= ath10k_snoc_hif_get_default_pipe,
+	.power_up		= ath10k_snoc_hif_power_up,
+	.power_down		= ath10k_snoc_hif_power_down,
+	.tx_sg			= ath10k_snoc_hif_tx_sg,
+	.send_complete_check	= ath10k_snoc_hif_send_complete_check,
+	.get_free_queue_number	= ath10k_snoc_hif_get_free_queue_number,
+	.get_target_info	= ath10k_snoc_hif_get_target_info,
+};
+
+static const struct ath10k_bus_ops ath10k_snoc_bus_ops = {
+	.read32		= ath10k_snoc_read32,
+	.write32	= ath10k_snoc_write32,
+};
+
+int ath10k_snoc_get_ce_id_from_irq(struct ath10k *ar, int irq)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	int i;
+
+	for (i = 0; i < CE_COUNT_MAX; i++) {
+		if (ar_snoc->ce_irqs[i].irq_line == irq)
+			return i;
+	}
+	ath10k_err(ar, "No matching CE id for irq %d\n", irq);
+
+	return -EINVAL;
+}
+
+static irqreturn_t ath10k_snoc_per_engine_handler(int irq, void *arg)
+{
+	struct ath10k *ar = arg;
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	int ce_id = ath10k_snoc_get_ce_id_from_irq(ar, irq);
+
+	if (ce_id < 0 || ce_id >= ARRAY_SIZE(ar_snoc->pipe_info)) {
+		ath10k_warn(ar, "unexpected/invalid irq %d ce_id %d\n", irq,
+			    ce_id);
+		return IRQ_HANDLED;
+	}
+
+	ath10k_snoc_irq_disable(ar);
+	napi_schedule(&ar->napi);
+
+	return IRQ_HANDLED;
+}
+
+static int ath10k_snoc_napi_poll(struct napi_struct *ctx, int budget)
+{
+	struct ath10k *ar = container_of(ctx, struct ath10k, napi);
+	int done = 0;
+
+	ath10k_ce_per_engine_service_any(ar);
+	done = ath10k_htt_txrx_compl_task(ar, budget);
+
+	if (done < budget) {
+		napi_complete(ctx);
+		ath10k_snoc_irq_enable(ar);
+	}
+
+	return done;
+}
+
+void ath10k_snoc_init_napi(struct ath10k *ar)
+{
+	netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_snoc_napi_poll,
+		       ATH10K_NAPI_BUDGET);
+}
+
+static int ath10k_snoc_request_irq(struct ath10k *ar)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	int irqflags = IRQF_TRIGGER_RISING;
+	int ret, id;
+
+	for (id = 0; id < CE_COUNT_MAX; id++) {
+		ret = request_irq(ar_snoc->ce_irqs[id].irq_line,
+				  ath10k_snoc_per_engine_handler,
+				  irqflags, ce_name[id], ar);
+		if (ret) {
+			ath10k_err(ar,
+				   "failed to register IRQ handler for CE %d: %d",
+				   id, ret);
+			goto err_irq;
+		}
+	}
+
+	return 0;
+
+err_irq:
+	for (id -= 1; id >= 0; id--)
+		free_irq(ar_snoc->ce_irqs[id].irq_line, ar);
+
+	return ret;
+}
+
+static void ath10k_snoc_free_irq(struct ath10k *ar)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	int id;
+
+	for (id = 0; id < CE_COUNT_MAX; id++)
+		free_irq(ar_snoc->ce_irqs[id].irq_line, ar);
+}
+
+static int ath10k_snoc_resource_init(struct ath10k *ar)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	struct platform_device *pdev;
+	struct resource *res;
+	int i, ret = 0;
+
+	pdev = ar_snoc->dev;
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "membase");
+	if (!res) {
+		ath10k_err(ar, "Memory base not found in DT\n");
+		return -EINVAL;
+	}
+
+	ar_snoc->mem_pa = res->start;
+	ar_snoc->mem = devm_ioremap(&pdev->dev, ar_snoc->mem_pa,
+				    resource_size(res));
+	if (!ar_snoc->mem) {
+		ath10k_err(ar, "Memory base ioremap failed with physical address %pa\n",
+			   &ar_snoc->mem_pa);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < CE_COUNT; i++) {
+		res = platform_get_resource(ar_snoc->dev, IORESOURCE_IRQ, i);
+		if (!res) {
+			ath10k_err(ar, "failed to get IRQ%d\n", i);
+			ret = -ENODEV;
+			goto out;
+		}
+		ar_snoc->ce_irqs[i].irq_line = res->start;
+	}
+
+out:
+	return ret;
+}
+
+static int ath10k_snoc_setup_resource(struct ath10k *ar)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_snoc_pipe *pipe;
+	int i, ret;
+
+	timer_setup(&ar_snoc->rx_post_retry, ath10k_snoc_rx_replenish_retry, 0);
+	spin_lock_init(&ce->ce_lock);
+	for (i = 0; i < CE_COUNT; i++) {
+		pipe = &ar_snoc->pipe_info[i];
+		pipe->ce_hdl = &ce->ce_states[i];
+		pipe->pipe_num = i;
+		pipe->hif_ce_state = ar;
+
+		ret = ath10k_ce_alloc_pipe(ar, i, &host_ce_config_wlan[i]);
+		if (ret) {
+			ath10k_err(ar, "failed to allocate copy engine pipe %d: %d\n",
+				   i, ret);
+			return ret;
+		}
+
+		pipe->buf_sz = host_ce_config_wlan[i].src_sz_max;
+	}
+	ath10k_snoc_init_napi(ar);
+
+	return 0;
+}
+
+static void ath10k_snoc_release_resource(struct ath10k *ar)
+{
+	int i;
+
+	netif_napi_del(&ar->napi);
+	for (i = 0; i < CE_COUNT; i++)
+		ath10k_ce_free_pipe(ar, i);
+}
+
+static int ath10k_get_vreg_info(struct ath10k *ar, struct device *dev,
+				struct ath10k_wcn3990_vreg_info *vreg_info)
+{
+	struct regulator *reg;
+	int ret = 0;
+
+	reg = devm_regulator_get_optional(dev, vreg_info->name);
+
+	if (IS_ERR(reg)) {
+		ret = PTR_ERR(reg);
+
+		if (ret  == -EPROBE_DEFER) {
+			ath10k_err(ar, "EPROBE_DEFER for regulator: %s\n",
+				   vreg_info->name);
+			return ret;
+		}
+		if (vreg_info->required) {
+			ath10k_err(ar, "Regulator %s doesn't exist: %d\n",
+				   vreg_info->name, ret);
+			return ret;
+		}
+		ath10k_dbg(ar, ATH10K_DBG_SNOC,
+			   "Optional regulator %s doesn't exist: %d\n",
+			   vreg_info->name, ret);
+		goto done;
+	}
+
+	vreg_info->reg = reg;
+
+done:
+	ath10k_dbg(ar, ATH10K_DBG_SNOC,
+		   "snog vreg %s min_v %u max_v %u load_ua %u settle_delay %lu\n",
+		   vreg_info->name, vreg_info->min_v, vreg_info->max_v,
+		   vreg_info->load_ua, vreg_info->settle_delay);
+
+	return 0;
+}
+
+static int ath10k_get_clk_info(struct ath10k *ar, struct device *dev,
+			       struct ath10k_wcn3990_clk_info *clk_info)
+{
+	struct clk *handle;
+	int ret = 0;
+
+	handle = devm_clk_get(dev, clk_info->name);
+	if (IS_ERR(handle)) {
+		ret = PTR_ERR(handle);
+		if (clk_info->required) {
+			ath10k_err(ar, "snoc clock %s isn't available: %d\n",
+				   clk_info->name, ret);
+			return ret;
+		}
+		ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc ignoring clock %s: %d\n",
+			   clk_info->name,
+			   ret);
+		return 0;
+	}
+
+	ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc clock %s freq %u\n",
+		   clk_info->name, clk_info->freq);
+
+	clk_info->handle = handle;
+
+	return ret;
+}
+
+static int ath10k_wcn3990_vreg_on(struct ath10k *ar)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	struct ath10k_wcn3990_vreg_info *vreg_info;
+	int ret = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(vreg_cfg); i++) {
+		vreg_info = &ar_snoc->vreg[i];
+
+		if (!vreg_info->reg)
+			continue;
+
+		ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc regulator %s being enabled\n",
+			   vreg_info->name);
+
+		ret = regulator_set_voltage(vreg_info->reg, vreg_info->min_v,
+					    vreg_info->max_v);
+		if (ret) {
+			ath10k_err(ar,
+				   "failed to set regulator %s voltage-min: %d voltage-max: %d\n",
+				   vreg_info->name, vreg_info->min_v, vreg_info->max_v);
+			goto err_reg_config;
+		}
+
+		if (vreg_info->load_ua) {
+			ret = regulator_set_load(vreg_info->reg,
+						 vreg_info->load_ua);
+			if (ret < 0) {
+				ath10k_err(ar,
+					   "failed to set regulator %s load: %d\n",
+					   vreg_info->name,
+					   vreg_info->load_ua);
+				goto err_reg_config;
+			}
+		}
+
+		ret = regulator_enable(vreg_info->reg);
+		if (ret) {
+			ath10k_err(ar, "failed to enable regulator %s\n",
+				   vreg_info->name);
+			goto err_reg_config;
+		}
+
+		if (vreg_info->settle_delay)
+			udelay(vreg_info->settle_delay);
+	}
+
+	return 0;
+
+err_reg_config:
+	for (; i >= 0; i--) {
+		vreg_info = &ar_snoc->vreg[i];
+
+		if (!vreg_info->reg)
+			continue;
+
+		regulator_disable(vreg_info->reg);
+		regulator_set_load(vreg_info->reg, 0);
+		regulator_set_voltage(vreg_info->reg, 0, vreg_info->max_v);
+	}
+
+	return ret;
+}
+
+static int ath10k_wcn3990_vreg_off(struct ath10k *ar)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	struct ath10k_wcn3990_vreg_info *vreg_info;
+	int ret = 0;
+	int i;
+
+	for (i = ARRAY_SIZE(vreg_cfg) - 1; i >= 0; i--) {
+		vreg_info = &ar_snoc->vreg[i];
+
+		if (!vreg_info->reg)
+			continue;
+
+		ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc regulator %s being disabled\n",
+			   vreg_info->name);
+
+		ret = regulator_disable(vreg_info->reg);
+		if (ret)
+			ath10k_err(ar, "failed to disable regulator %s\n",
+				   vreg_info->name);
+
+		ret = regulator_set_load(vreg_info->reg, 0);
+		if (ret < 0)
+			ath10k_err(ar, "failed to set load %s\n",
+				   vreg_info->name);
+
+		ret = regulator_set_voltage(vreg_info->reg, 0,
+					    vreg_info->max_v);
+		if (ret)
+			ath10k_err(ar, "failed to set voltage %s\n",
+				   vreg_info->name);
+	}
+
+	return ret;
+}
+
+static int ath10k_wcn3990_clk_init(struct ath10k *ar)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	struct ath10k_wcn3990_clk_info *clk_info;
+	int ret = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(clk_cfg); i++) {
+		clk_info = &ar_snoc->clk[i];
+
+		if (!clk_info->handle)
+			continue;
+
+		ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc clock %s being enabled\n",
+			   clk_info->name);
+
+		if (clk_info->freq) {
+			ret = clk_set_rate(clk_info->handle, clk_info->freq);
+
+			if (ret) {
+				ath10k_err(ar, "failed to set clock %s freq %u\n",
+					   clk_info->name, clk_info->freq);
+				goto err_clock_config;
+			}
+		}
+
+		ret = clk_prepare_enable(clk_info->handle);
+		if (ret) {
+			ath10k_err(ar, "failed to enable clock %s\n",
+				   clk_info->name);
+			goto err_clock_config;
+		}
+	}
+
+	return 0;
+
+err_clock_config:
+	for (; i >= 0; i--) {
+		clk_info = &ar_snoc->clk[i];
+
+		if (!clk_info->handle)
+			continue;
+
+		clk_disable_unprepare(clk_info->handle);
+	}
+
+	return ret;
+}
+
+static int ath10k_wcn3990_clk_deinit(struct ath10k *ar)
+{
+	struct ath10k_snoc *ar_snoc = ath10k_snoc_priv(ar);
+	struct ath10k_wcn3990_clk_info *clk_info;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(clk_cfg); i++) {
+		clk_info = &ar_snoc->clk[i];
+
+		if (!clk_info->handle)
+			continue;
+
+		ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc clock %s being disabled\n",
+			   clk_info->name);
+
+		clk_disable_unprepare(clk_info->handle);
+	}
+
+	return 0;
+}
+
+static int ath10k_hw_power_on(struct ath10k *ar)
+{
+	int ret;
+
+	ath10k_dbg(ar, ATH10K_DBG_SNOC, "soc power on\n");
+
+	ret = ath10k_wcn3990_vreg_on(ar);
+	if (ret)
+		return ret;
+
+	ret = ath10k_wcn3990_clk_init(ar);
+	if (ret)
+		goto vreg_off;
+
+	return ret;
+
+vreg_off:
+	ath10k_wcn3990_vreg_off(ar);
+	return ret;
+}
+
+static int ath10k_hw_power_off(struct ath10k *ar)
+{
+	int ret;
+
+	ath10k_dbg(ar, ATH10K_DBG_SNOC, "soc power off\n");
+
+	ath10k_wcn3990_clk_deinit(ar);
+
+	ret = ath10k_wcn3990_vreg_off(ar);
+
+	return ret;
+}
+
+static const struct of_device_id ath10k_snoc_dt_match[] = {
+	{ .compatible = "qcom,wcn3990-wifi",
+	 .data = &drv_priv,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ath10k_snoc_dt_match);
+
+static int ath10k_snoc_probe(struct platform_device *pdev)
+{
+	const struct ath10k_snoc_drv_priv *drv_data;
+	const struct of_device_id *of_id;
+	struct ath10k_snoc *ar_snoc;
+	struct device *dev;
+	struct ath10k *ar;
+	int ret;
+	u32 i;
+
+	of_id = of_match_device(ath10k_snoc_dt_match, &pdev->dev);
+	if (!of_id) {
+		dev_err(&pdev->dev, "failed to find matching device tree id\n");
+		return -EINVAL;
+	}
+
+	drv_data = of_id->data;
+	dev = &pdev->dev;
+
+	ret = dma_set_mask_and_coherent(dev, drv_data->dma_mask);
+	if (ret) {
+		dev_err(dev, "failed to set dma mask: %d", ret);
+		return ret;
+	}
+
+	ar = ath10k_core_create(sizeof(*ar_snoc), dev, ATH10K_BUS_SNOC,
+				drv_data->hw_rev, &ath10k_snoc_hif_ops);
+	if (!ar) {
+		dev_err(dev, "failed to allocate core\n");
+		return -ENOMEM;
+	}
+
+	ar_snoc = ath10k_snoc_priv(ar);
+	ar_snoc->dev = pdev;
+	platform_set_drvdata(pdev, ar);
+	ar_snoc->ar = ar;
+	ar_snoc->ce.bus_ops = &ath10k_snoc_bus_ops;
+	ar->ce_priv = &ar_snoc->ce;
+
+	ath10k_snoc_resource_init(ar);
+	if (ret) {
+		ath10k_warn(ar, "failed to initialize resource: %d\n", ret);
+		goto err_core_destroy;
+	}
+
+	ath10k_snoc_setup_resource(ar);
+	if (ret) {
+		ath10k_warn(ar, "failed to setup resource: %d\n", ret);
+		goto err_core_destroy;
+	}
+	ret = ath10k_snoc_request_irq(ar);
+	if (ret) {
+		ath10k_warn(ar, "failed to request irqs: %d\n", ret);
+		goto err_release_resource;
+	}
+
+	ar_snoc->vreg = vreg_cfg;
+	for (i = 0; i < ARRAY_SIZE(vreg_cfg); i++) {
+		ret = ath10k_get_vreg_info(ar, dev, &ar_snoc->vreg[i]);
+		if (ret)
+			goto err_free_irq;
+	}
+
+	ar_snoc->clk = clk_cfg;
+	for (i = 0; i < ARRAY_SIZE(clk_cfg); i++) {
+		ret = ath10k_get_clk_info(ar, dev, &ar_snoc->clk[i]);
+		if (ret)
+			goto err_free_irq;
+	}
+
+	ret = ath10k_hw_power_on(ar);
+	if (ret) {
+		ath10k_err(ar, "failed to power on device: %d\n", ret);
+		goto err_free_irq;
+	}
+
+	ret = ath10k_core_register(ar, drv_data->hw_rev);
+	if (ret) {
+		ath10k_err(ar, "failed to register driver core: %d\n", ret);
+		goto err_hw_power_off;
+	}
+
+	ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc probe\n");
+	ath10k_warn(ar, "Warning: SNOC support is still work-in-progress, it will not work properly!");
+
+	return 0;
+
+err_hw_power_off:
+	ath10k_hw_power_off(ar);
+
+err_free_irq:
+	ath10k_snoc_free_irq(ar);
+
+err_release_resource:
+	ath10k_snoc_release_resource(ar);
+
+err_core_destroy:
+	ath10k_core_destroy(ar);
+
+	return ret;
+}
+
+static int ath10k_snoc_remove(struct platform_device *pdev)
+{
+	struct ath10k *ar = platform_get_drvdata(pdev);
+
+	ath10k_dbg(ar, ATH10K_DBG_SNOC, "snoc remove\n");
+	ath10k_core_unregister(ar);
+	ath10k_hw_power_off(ar);
+	ath10k_snoc_free_irq(ar);
+	ath10k_snoc_release_resource(ar);
+	ath10k_core_destroy(ar);
+
+	return 0;
+}
+
+static struct platform_driver ath10k_snoc_driver = {
+		.probe  = ath10k_snoc_probe,
+		.remove = ath10k_snoc_remove,
+		.driver = {
+			.name   = "ath10k_snoc",
+			.of_match_table = ath10k_snoc_dt_match,
+		},
+};
+
+static int __init ath10k_snoc_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&ath10k_snoc_driver);
+	if (ret)
+		pr_err("failed to register ath10k snoc driver: %d\n",
+		       ret);
+
+	return ret;
+}
+module_init(ath10k_snoc_init);
+
+static void __exit ath10k_snoc_exit(void)
+{
+	platform_driver_unregister(&ath10k_snoc_driver);
+}
+module_exit(ath10k_snoc_exit);
+
+MODULE_AUTHOR("Qualcomm");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("Driver support for Atheros WCN3990 SNOC devices");
diff --git a/drivers/net/wireless/ath/ath10k/snoc.h b/drivers/net/wireless/ath/ath10k/snoc.h
new file mode 100644
index 000000000000..05dc98f46ccd
--- /dev/null
+++ b/drivers/net/wireless/ath/ath10k/snoc.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2018 The Linux Foundation. All rights reserved.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef _SNOC_H_
+#define _SNOC_H_
+
+#include "hw.h"
+#include "ce.h"
+#include "pci.h"
+
+struct ath10k_snoc_drv_priv {
+	enum ath10k_hw_rev hw_rev;
+	u64 dma_mask;
+};
+
+struct snoc_state {
+	u32 pipe_cfg_addr;
+	u32 svc_to_pipe_map;
+};
+
+struct ath10k_snoc_pipe {
+	struct ath10k_ce_pipe *ce_hdl;
+	u8 pipe_num;
+	struct ath10k *hif_ce_state;
+	size_t buf_sz;
+	/* protect ce info */
+	spinlock_t pipe_lock;
+	struct ath10k_snoc *ar_snoc;
+};
+
+struct ath10k_snoc_target_info {
+	u32 target_version;
+	u32 target_type;
+	u32 target_revision;
+	u32 soc_version;
+};
+
+struct ath10k_snoc_ce_irq {
+	u32 irq_line;
+};
+
+struct ath10k_wcn3990_vreg_info {
+	struct regulator *reg;
+	const char *name;
+	u32 min_v;
+	u32 max_v;
+	u32 load_ua;
+	unsigned long settle_delay;
+	bool required;
+};
+
+struct ath10k_wcn3990_clk_info {
+	struct clk *handle;
+	const char *name;
+	u32 freq;
+	bool required;
+};
+
+struct ath10k_snoc {
+	struct platform_device *dev;
+	struct ath10k *ar;
+	void __iomem *mem;
+	dma_addr_t mem_pa;
+	struct ath10k_snoc_target_info target_info;
+	size_t mem_len;
+	struct ath10k_snoc_pipe pipe_info[CE_COUNT_MAX];
+	struct ath10k_snoc_ce_irq ce_irqs[CE_COUNT_MAX];
+	struct ath10k_ce ce;
+	struct timer_list rx_post_retry;
+	struct ath10k_wcn3990_vreg_info *vreg;
+	struct ath10k_wcn3990_clk_info *clk;
+};
+
+static inline struct ath10k_snoc *ath10k_snoc_priv(struct ath10k *ar)
+{
+	return (struct ath10k_snoc *)ar->drv_priv;
+}
+
+void ath10k_snoc_write32(struct ath10k *ar, u32 offset, u32 value);
+u32 ath10k_snoc_read32(struct ath10k *ar, u32 offset);
+
+#endif /* _SNOC_H_ */
diff --git a/drivers/net/wireless/ath/ath10k/testmode.c b/drivers/net/wireless/ath/ath10k/testmode.c
index 568810b41657..c24ee616833c 100644
--- a/drivers/net/wireless/ath/ath10k/testmode.c
+++ b/drivers/net/wireless/ath/ath10k/testmode.c
@@ -157,7 +157,7 @@ static int ath10k_tm_fetch_utf_firmware_api_1(struct ath10k *ar,
 		 ar->hw_params.fw.dir, ATH10K_FW_UTF_FILE);
 
 	/* load utf firmware image */
-	ret = request_firmware_direct(&fw_file->firmware, filename, ar->dev);
+	ret = firmware_request_nowarn(&fw_file->firmware, filename, ar->dev);
 	ath10k_dbg(ar, ATH10K_DBG_TESTMODE, "testmode fw request '%s': %d\n",
 		   filename, ret);
 
diff --git a/drivers/net/wireless/ath/ath10k/txrx.c b/drivers/net/wireless/ath/ath10k/txrx.c
index 70e23bbf7171..cda164f6e9f6 100644
--- a/drivers/net/wireless/ath/ath10k/txrx.c
+++ b/drivers/net/wireless/ath/ath10k/txrx.c
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
  * Copyright (c) 2011-2016 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -119,6 +120,13 @@ int ath10k_txrx_tx_unref(struct ath10k_htt *htt,
 			info->flags &= ~IEEE80211_TX_STAT_ACK;
 	}
 
+	if (tx_done->status == HTT_TX_COMPL_STATE_ACK &&
+	    tx_done->ack_rssi != ATH10K_INVALID_RSSI) {
+		info->status.ack_signal = ATH10K_DEFAULT_NOISE_FLOOR +
+						tx_done->ack_rssi;
+		info->status.is_valid_ack_signal = true;
+	}
+
 	ieee80211_tx_status(htt->ar->hw, msdu);
 	/* we do not own the msdu anymore */
 
diff --git a/drivers/net/wireless/ath/ath10k/wmi-ops.h b/drivers/net/wireless/ath/ath10k/wmi-ops.h
index c35e45340b4f..5ecce04005d2 100644
--- a/drivers/net/wireless/ath/ath10k/wmi-ops.h
+++ b/drivers/net/wireless/ath/ath10k/wmi-ops.h
@@ -25,6 +25,7 @@ struct sk_buff;
 struct wmi_ops {
 	void (*rx)(struct ath10k *ar, struct sk_buff *skb);
 	void (*map_svc)(const __le32 *in, unsigned long *out, size_t len);
+	void (*map_svc_ext)(const __le32 *in, unsigned long *out, size_t len);
 
 	int (*pull_scan)(struct ath10k *ar, struct sk_buff *skb,
 			 struct wmi_scan_ev_arg *arg);
@@ -54,6 +55,11 @@ struct wmi_ops {
 			      struct wmi_wow_ev_arg *arg);
 	int (*pull_echo_ev)(struct ath10k *ar, struct sk_buff *skb,
 			    struct wmi_echo_ev_arg *arg);
+	int (*pull_dfs_status_ev)(struct ath10k *ar, struct sk_buff *skb,
+				  struct wmi_dfs_status_ev_arg *arg);
+	int (*pull_svc_avail)(struct ath10k *ar, struct sk_buff *skb,
+			      struct wmi_svc_avail_ev_arg *arg);
+
 	enum wmi_txbf_conf (*get_txbf_conf_scheme)(struct ath10k *ar);
 
 	struct sk_buff *(*gen_pdev_suspend)(struct ath10k *ar, u32 suspend_opt);
@@ -115,6 +121,8 @@ struct wmi_ops {
 					 u32 value);
 	struct sk_buff *(*gen_scan_chan_list)(struct ath10k *ar,
 					      const struct wmi_scan_chan_list_arg *arg);
+	struct sk_buff *(*gen_scan_prob_req_oui)(struct ath10k *ar,
+						 u32 prob_req_oui);
 	struct sk_buff *(*gen_beacon_dma)(struct ath10k *ar, u32 vdev_id,
 					  const void *bcn, size_t bcn_len,
 					  u32 bcn_paddr, bool dtim_zero,
@@ -182,6 +190,9 @@ struct wmi_ops {
 						const struct wmi_tdls_peer_update_cmd_arg *arg,
 						const struct wmi_tdls_peer_capab_arg *cap,
 						const struct wmi_channel_arg *chan);
+	struct sk_buff *(*gen_radar_found)
+			(struct ath10k *ar,
+			 const struct ath10k_radar_found_info *arg);
 	struct sk_buff *(*gen_adaptive_qcs)(struct ath10k *ar, bool enable);
 	struct sk_buff *(*gen_pdev_get_tpc_config)(struct ath10k *ar,
 						   u32 param);
@@ -229,6 +240,17 @@ ath10k_wmi_map_svc(struct ath10k *ar, const __le32 *in, unsigned long *out,
 	return 0;
 }
 
+static inline int
+ath10k_wmi_map_svc_ext(struct ath10k *ar, const __le32 *in, unsigned long *out,
+		       size_t len)
+{
+	if (!ar->wmi.ops->map_svc_ext)
+		return -EOPNOTSUPP;
+
+	ar->wmi.ops->map_svc_ext(in, out, len);
+	return 0;
+}
+
 static inline int
 ath10k_wmi_pull_scan(struct ath10k *ar, struct sk_buff *skb,
 		     struct wmi_scan_ev_arg *arg)
@@ -329,6 +351,15 @@ ath10k_wmi_pull_rdy(struct ath10k *ar, struct sk_buff *skb,
 	return ar->wmi.ops->pull_rdy(ar, skb, arg);
 }
 
+static inline int
+ath10k_wmi_pull_svc_avail(struct ath10k *ar, struct sk_buff *skb,
+			  struct wmi_svc_avail_ev_arg *arg)
+{
+	if (!ar->wmi.ops->pull_svc_avail)
+		return -EOPNOTSUPP;
+	return ar->wmi.ops->pull_svc_avail(ar, skb, arg);
+}
+
 static inline int
 ath10k_wmi_pull_fw_stats(struct ath10k *ar, struct sk_buff *skb,
 			 struct ath10k_fw_stats *stats)
@@ -369,6 +400,16 @@ ath10k_wmi_pull_echo_ev(struct ath10k *ar, struct sk_buff *skb,
 	return ar->wmi.ops->pull_echo_ev(ar, skb, arg);
 }
 
+static inline int
+ath10k_wmi_pull_dfs_status(struct ath10k *ar, struct sk_buff *skb,
+			   struct wmi_dfs_status_ev_arg *arg)
+{
+	if (!ar->wmi.ops->pull_dfs_status_ev)
+		return -EOPNOTSUPP;
+
+	return ar->wmi.ops->pull_dfs_status_ev(ar, skb, arg);
+}
+
 static inline enum wmi_txbf_conf
 ath10k_wmi_get_txbf_conf_scheme(struct ath10k *ar)
 {
@@ -890,6 +931,26 @@ ath10k_wmi_scan_chan_list(struct ath10k *ar,
 	return ath10k_wmi_cmd_send(ar, skb, ar->wmi.cmd->scan_chan_list_cmdid);
 }
 
+static inline int
+ath10k_wmi_scan_prob_req_oui(struct ath10k *ar, const u8 mac_addr[ETH_ALEN])
+{
+	struct sk_buff *skb;
+	u32 prob_req_oui;
+
+	prob_req_oui = (((u32)mac_addr[0]) << 16) |
+		       (((u32)mac_addr[1]) << 8) | mac_addr[2];
+
+	if (!ar->wmi.ops->gen_scan_prob_req_oui)
+		return -EOPNOTSUPP;
+
+	skb = ar->wmi.ops->gen_scan_prob_req_oui(ar, prob_req_oui);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	return ath10k_wmi_cmd_send(ar, skb,
+			ar->wmi.cmd->scan_prob_req_oui_cmdid);
+}
+
 static inline int
 ath10k_wmi_peer_assoc(struct ath10k *ar,
 		      const struct wmi_peer_assoc_complete_arg *arg)
@@ -1465,4 +1526,21 @@ ath10k_wmi_pdev_get_tpc_table_cmdid(struct ath10k *ar, u32 param)
 				   ar->wmi.cmd->pdev_get_tpc_table_cmdid);
 }
 
+static inline int
+ath10k_wmi_report_radar_found(struct ath10k *ar,
+			      const struct ath10k_radar_found_info *arg)
+{
+	struct sk_buff *skb;
+
+	if (!ar->wmi.ops->gen_radar_found)
+		return -EOPNOTSUPP;
+
+	skb = ar->wmi.ops->gen_radar_found(ar, arg);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	return ath10k_wmi_cmd_send(ar, skb,
+				   ar->wmi.cmd->radar_found_cmdid);
+}
+
 #endif
diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.c b/drivers/net/wireless/ath/ath10k/wmi-tlv.c
index 9d1b0a459069..2e34a1fc5ba6 100644
--- a/drivers/net/wireless/ath/ath10k/wmi-tlv.c
+++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.c
@@ -594,6 +594,9 @@ static void ath10k_wmi_tlv_op_rx(struct ath10k *ar, struct sk_buff *skb)
 	case WMI_TLV_READY_EVENTID:
 		ath10k_wmi_event_ready(ar, skb);
 		break;
+	case WMI_TLV_SERVICE_AVAILABLE_EVENTID:
+		ath10k_wmi_event_service_available(ar, skb);
+		break;
 	case WMI_TLV_OFFLOAD_BCN_TX_STATUS_EVENTID:
 		ath10k_wmi_tlv_event_bcn_tx_status(ar, skb);
 		break;
@@ -1117,6 +1120,39 @@ static int ath10k_wmi_tlv_op_pull_rdy_ev(struct ath10k *ar,
 	return 0;
 }
 
+static int ath10k_wmi_tlv_svc_avail_parse(struct ath10k *ar, u16 tag, u16 len,
+					  const void *ptr, void *data)
+{
+	struct wmi_svc_avail_ev_arg *arg = data;
+
+	switch (tag) {
+	case WMI_TLV_TAG_STRUCT_SERVICE_AVAILABLE_EVENT:
+		arg->service_map_ext_len = *(__le32 *)ptr;
+		arg->service_map_ext = ptr + sizeof(__le32);
+		return 0;
+	default:
+		break;
+	}
+	return -EPROTO;
+}
+
+static int ath10k_wmi_tlv_op_pull_svc_avail(struct ath10k *ar,
+					    struct sk_buff *skb,
+					    struct wmi_svc_avail_ev_arg *arg)
+{
+	int ret;
+
+	ret = ath10k_wmi_tlv_iter(ar, skb->data, skb->len,
+				  ath10k_wmi_tlv_svc_avail_parse, arg);
+
+	if (ret) {
+		ath10k_warn(ar, "failed to parse svc_avail tlv: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 static void ath10k_wmi_tlv_pull_vdev_stats(const struct wmi_tlv_vdev_stats *src,
 					   struct ath10k_fw_stats_vdev *dst)
 {
@@ -1600,6 +1636,8 @@ ath10k_wmi_tlv_op_gen_start_scan(struct ath10k *ar,
 	cmd->num_bssids = __cpu_to_le32(arg->n_bssids);
 	cmd->ie_len = __cpu_to_le32(arg->ie_len);
 	cmd->num_probes = __cpu_to_le32(3);
+	ether_addr_copy(cmd->mac_addr.addr, arg->mac_addr.addr);
+	ether_addr_copy(cmd->mac_mask.addr, arg->mac_mask.addr);
 
 	/* FIXME: There are some scan flag inconsistencies across firmwares,
 	 * e.g. WMI-TLV inverts the logic behind the following flag.
@@ -2446,6 +2484,27 @@ ath10k_wmi_tlv_op_gen_scan_chan_list(struct ath10k *ar,
 	return skb;
 }
 
+static struct sk_buff *
+ath10k_wmi_tlv_op_gen_scan_prob_req_oui(struct ath10k *ar, u32 prob_req_oui)
+{
+	struct wmi_scan_prob_req_oui_cmd *cmd;
+	struct wmi_tlv *tlv;
+	struct sk_buff *skb;
+
+	skb = ath10k_wmi_alloc_skb(ar, sizeof(*tlv) + sizeof(*cmd));
+	if (!skb)
+		return ERR_PTR(-ENOMEM);
+
+	tlv = (void *)skb->data;
+	tlv->tag = __cpu_to_le16(WMI_TLV_TAG_STRUCT_SCAN_PROB_REQ_OUI_CMD);
+	tlv->len = __cpu_to_le16(sizeof(*cmd));
+	cmd = (void *)tlv->value;
+	cmd->prob_req_oui = __cpu_to_le32(prob_req_oui);
+
+	ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi tlv scan prob req oui\n");
+	return skb;
+}
+
 static struct sk_buff *
 ath10k_wmi_tlv_op_gen_beacon_dma(struct ath10k *ar, u32 vdev_id,
 				 const void *bcn, size_t bcn_len,
@@ -3076,6 +3135,37 @@ ath10k_wmi_tlv_op_gen_tdls_peer_update(struct ath10k *ar,
 	return skb;
 }
 
+static struct sk_buff *
+ath10k_wmi_tlv_op_gen_pdev_set_quiet_mode(struct ath10k *ar, u32 period,
+					  u32 duration, u32 next_offset,
+					  u32 enabled)
+{
+	struct wmi_tlv_set_quiet_cmd *cmd;
+	struct wmi_tlv *tlv;
+	struct sk_buff *skb;
+
+	skb = ath10k_wmi_alloc_skb(ar, sizeof(*tlv) + sizeof(*cmd));
+	if (!skb)
+		return ERR_PTR(-ENOMEM);
+
+	tlv = (void *)skb->data;
+	tlv->tag = __cpu_to_le16(WMI_TLV_TAG_STRUCT_PDEV_SET_QUIET_CMD);
+	tlv->len = __cpu_to_le16(sizeof(*cmd));
+	cmd = (void *)tlv->value;
+
+	/* vdev_id is not in use, set to 0 */
+	cmd->vdev_id = __cpu_to_le32(0);
+	cmd->period = __cpu_to_le32(period);
+	cmd->duration = __cpu_to_le32(duration);
+	cmd->next_start = __cpu_to_le32(next_offset);
+	cmd->enabled = __cpu_to_le32(enabled);
+
+	ath10k_dbg(ar, ATH10K_DBG_WMI,
+		   "wmi tlv quiet param: period %u duration %u enabled %d\n",
+		   period, duration, enabled);
+	return skb;
+}
+
 static struct sk_buff *
 ath10k_wmi_tlv_op_gen_wow_enable(struct ath10k *ar)
 {
@@ -3416,6 +3506,7 @@ static struct wmi_cmd_map wmi_tlv_cmd_map = {
 	.stop_scan_cmdid = WMI_TLV_STOP_SCAN_CMDID,
 	.scan_chan_list_cmdid = WMI_TLV_SCAN_CHAN_LIST_CMDID,
 	.scan_sch_prio_tbl_cmdid = WMI_TLV_SCAN_SCH_PRIO_TBL_CMDID,
+	.scan_prob_req_oui_cmdid = WMI_TLV_SCAN_PROB_REQ_OUI_CMDID,
 	.pdev_set_regdomain_cmdid = WMI_TLV_PDEV_SET_REGDOMAIN_CMDID,
 	.pdev_set_channel_cmdid = WMI_TLV_PDEV_SET_CHANNEL_CMDID,
 	.pdev_set_param_cmdid = WMI_TLV_PDEV_SET_PARAM_CMDID,
@@ -3740,6 +3831,7 @@ static struct wmi_vdev_param_map wmi_tlv_vdev_param_map = {
 static const struct wmi_ops wmi_tlv_ops = {
 	.rx = ath10k_wmi_tlv_op_rx,
 	.map_svc = wmi_tlv_svc_map,
+	.map_svc_ext = wmi_tlv_svc_map_ext,
 
 	.pull_scan = ath10k_wmi_tlv_op_pull_scan_ev,
 	.pull_mgmt_rx = ath10k_wmi_tlv_op_pull_mgmt_rx_ev,
@@ -3751,6 +3843,7 @@ static const struct wmi_ops wmi_tlv_ops = {
 	.pull_phyerr = ath10k_wmi_op_pull_phyerr_ev,
 	.pull_svc_rdy = ath10k_wmi_tlv_op_pull_svc_rdy_ev,
 	.pull_rdy = ath10k_wmi_tlv_op_pull_rdy_ev,
+	.pull_svc_avail = ath10k_wmi_tlv_op_pull_svc_avail,
 	.pull_fw_stats = ath10k_wmi_tlv_op_pull_fw_stats,
 	.pull_roam_ev = ath10k_wmi_tlv_op_pull_roam_ev,
 	.pull_wow_event = ath10k_wmi_tlv_op_pull_wow_ev,
@@ -3782,6 +3875,7 @@ static const struct wmi_ops wmi_tlv_ops = {
 	.gen_set_sta_ps = ath10k_wmi_tlv_op_gen_set_sta_ps,
 	.gen_set_ap_ps = ath10k_wmi_tlv_op_gen_set_ap_ps,
 	.gen_scan_chan_list = ath10k_wmi_tlv_op_gen_scan_chan_list,
+	.gen_scan_prob_req_oui = ath10k_wmi_tlv_op_gen_scan_prob_req_oui,
 	.gen_beacon_dma = ath10k_wmi_tlv_op_gen_beacon_dma,
 	.gen_pdev_set_wmm = ath10k_wmi_tlv_op_gen_pdev_set_wmm,
 	.gen_request_stats = ath10k_wmi_tlv_op_gen_request_stats,
@@ -3791,7 +3885,7 @@ static const struct wmi_ops wmi_tlv_ops = {
 	.gen_dbglog_cfg = ath10k_wmi_tlv_op_gen_dbglog_cfg,
 	.gen_pktlog_enable = ath10k_wmi_tlv_op_gen_pktlog_enable,
 	.gen_pktlog_disable = ath10k_wmi_tlv_op_gen_pktlog_disable,
-	/* .gen_pdev_set_quiet_mode not implemented */
+	.gen_pdev_set_quiet_mode = ath10k_wmi_tlv_op_gen_pdev_set_quiet_mode,
 	.gen_pdev_get_temperature = ath10k_wmi_tlv_op_gen_pdev_get_temperature,
 	/* .gen_addba_clear_resp not implemented */
 	/* .gen_addba_send not implemented */
diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.h b/drivers/net/wireless/ath/ath10k/wmi-tlv.h
index fa3773ec7c68..3e1e340cd834 100644
--- a/drivers/net/wireless/ath/ath10k/wmi-tlv.h
+++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.h
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -295,6 +296,7 @@ enum wmi_tlv_cmd_id {
 enum wmi_tlv_event_id {
 	WMI_TLV_SERVICE_READY_EVENTID = 0x1,
 	WMI_TLV_READY_EVENTID,
+	WMI_TLV_SERVICE_AVAILABLE_EVENTID,
 	WMI_TLV_SCAN_EVENTID = WMI_TLV_EV(WMI_TLV_GRP_SCAN),
 	WMI_TLV_PDEV_TPC_CONFIG_EVENTID = WMI_TLV_EV(WMI_TLV_GRP_PDEV),
 	WMI_TLV_CHAN_INFO_EVENTID,
@@ -949,6 +951,275 @@ enum wmi_tlv_tag {
 	WMI_TLV_TAG_STRUCT_PACKET_FILTER_ENABLE,
 	WMI_TLV_TAG_STRUCT_SAP_SET_BLACKLIST_PARAM_CMD,
 	WMI_TLV_TAG_STRUCT_MGMT_TX_CMD,
+	WMI_TLV_TAG_STRUCT_MGMT_TX_COMPL_EVENT,
+	WMI_TLV_TAG_STRUCT_SOC_SET_ANTENNA_MODE_CMD,
+	WMI_TLV_TAG_STRUCT_WOW_UDP_SVC_OFLD_CMD,
+	WMI_TLV_TAG_STRUCT_LRO_INFO_CMD,
+	WMI_TLV_TAG_STRUCT_ROAM_EARLYSTOP_RSSI_THRES_PARAM,
+	WMI_TLV_TAG_STRUCT_SERVICE_READY_EXT_EVENT,
+	WMI_TLV_TAG_STRUCT_MAWC_SENSOR_REPORT_IND_CMD,
+	WMI_TLV_TAG_STRUCT_MAWC_ENABLE_SENSOR_EVENT,
+	WMI_TLV_TAG_STRUCT_ROAM_CONFIGURE_MAWC_CMD,
+	WMI_TLV_TAG_STRUCT_NLO_CONFIGURE_MAWC_CMD,
+	WMI_TLV_TAG_STRUCT_EXTSCAN_CONFIGURE_MAWC_CMD,
+	WMI_TLV_TAG_STRUCT_PEER_ASSOC_CONF_EVENT,
+	WMI_TLV_TAG_STRUCT_WOW_HOSTWAKEUP_GPIO_PIN_PATTERN_CONFIG_CMD,
+	WMI_TLV_TAG_STRUCT_AP_PS_EGAP_PARAM_CMD,
+	WMI_TLV_TAG_STRUCT_AP_PS_EGAP_INFO_EVENT,
+	WMI_TLV_TAG_STRUCT_PMF_OFFLOAD_SET_SA_QUERY_CMD,
+	WMI_TLV_TAG_STRUCT_TRANSFER_DATA_TO_FLASH_CMD,
+	WMI_TLV_TAG_STRUCT_TRANSFER_DATA_TO_FLASH_COMPLETE_EVENT,
+	WMI_TLV_TAG_STRUCT_SCPC_EVENT,
+	WMI_TLV_TAG_STRUCT_AP_PS_EGAP_INFO_CHAINMASK_LIST,
+	WMI_TLV_TAG_STRUCT_STA_SMPS_FORCE_MODE_COMPLETE_EVENT,
+	WMI_TLV_TAG_STRUCT_BPF_GET_CAPABILITY_CMD,
+	WMI_TLV_TAG_STRUCT_BPF_CAPABILITY_INFO_EVT,
+	WMI_TLV_TAG_STRUCT_BPF_GET_VDEV_STATS_CMD,
+	WMI_TLV_TAG_STRUCT_BPF_VDEV_STATS_INFO_EVT,
+	WMI_TLV_TAG_STRUCT_BPF_SET_VDEV_INSTRUCTIONS_CMD,
+	WMI_TLV_TAG_STRUCT_BPF_DEL_VDEV_INSTRUCTIONS_CMD,
+	WMI_TLV_TAG_STRUCT_VDEV_DELETE_RESP_EVENT,
+	WMI_TLV_TAG_STRUCT_PEER_DELETE_RESP_EVENT,
+	WMI_TLV_TAG_STRUCT_ROAM_DENSE_THRES_PARAM,
+	WMI_TLV_TAG_STRUCT_ENLO_CANDIDATE_SCORE_PARAM,
+	WMI_TLV_TAG_STRUCT_PEER_UPDATE_WDS_ENTRY_CMD,
+	WMI_TLV_TAG_STRUCT_VDEV_CONFIG_RATEMASK,
+	WMI_TLV_TAG_STRUCT_PDEV_FIPS_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SMART_ANT_ENABLE_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SMART_ANT_SET_RX_ANTENNA_CMD,
+	WMI_TLV_TAG_STRUCT_PEER_SMART_ANT_SET_TX_ANTENNA_CMD,
+	WMI_TLV_TAG_STRUCT_PEER_SMART_ANT_SET_TRAIN_ANTENNA_CMD,
+	WMI_TLV_TAG_STRUCT_PEER_SMART_ANT_SET_NODE_CONFIG_OPS_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_ANT_SWITCH_TBL_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_CTL_TABLE_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_MIMOGAIN_TABLE_CMD,
+	WMI_TLV_TAG_STRUCT_FWTEST_SET_PARAM_CMD,
+	WMI_TLV_TAG_STRUCT_PEER_ATF_REQUEST,
+	WMI_TLV_TAG_STRUCT_VDEV_ATF_REQUEST,
+	WMI_TLV_TAG_STRUCT_PDEV_GET_ANI_CCK_CONFIG_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_GET_ANI_OFDM_CONFIG_CMD,
+	WMI_TLV_TAG_STRUCT_INST_RSSI_STATS_RESP,
+	WMI_TLV_TAG_STRUCT_MED_UTIL_REPORT_EVENT,
+	WMI_TLV_TAG_STRUCT_PEER_STA_PS_STATECHANGE_EVENT,
+	WMI_TLV_TAG_STRUCT_WDS_ADDR_EVENT,
+	WMI_TLV_TAG_STRUCT_PEER_RATECODE_LIST_EVENT,
+	WMI_TLV_TAG_STRUCT_PDEV_NFCAL_POWER_ALL_CHANNELS_EVENT,
+	WMI_TLV_TAG_STRUCT_PDEV_TPC_EVENT,
+	WMI_TLV_TAG_STRUCT_ANI_OFDM_EVENT,
+	WMI_TLV_TAG_STRUCT_ANI_CCK_EVENT,
+	WMI_TLV_TAG_STRUCT_PDEV_CHANNEL_HOPPING_EVENT,
+	WMI_TLV_TAG_STRUCT_PDEV_FIPS_EVENT,
+	WMI_TLV_TAG_STRUCT_ATF_PEER_INFO,
+	WMI_TLV_TAG_STRUCT_PDEV_GET_TPC_CMD,
+	WMI_TLV_TAG_STRUCT_VDEV_FILTER_NRP_CONFIG_CMD,
+	WMI_TLV_TAG_STRUCT_QBOOST_CFG_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SMART_ANT_GPIO_HANDLE,
+	WMI_TLV_TAG_STRUCT_PEER_SMART_ANT_SET_TX_ANTENNA_SERIES,
+	WMI_TLV_TAG_STRUCT_PEER_SMART_ANT_SET_TRAIN_ANTENNA_PARAM,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_ANT_CTRL_CHAIN,
+	WMI_TLV_TAG_STRUCT_PEER_CCK_OFDM_RATE_INFO,
+	WMI_TLV_TAG_STRUCT_PEER_MCS_RATE_INFO,
+	WMI_TLV_TAG_STRUCT_PDEV_NFCAL_POWER_ALL_CHANNELS_NFDBR,
+	WMI_TLV_TAG_STRUCT_PDEV_NFCAL_POWER_ALL_CHANNELS_NFDBM,
+	WMI_TLV_TAG_STRUCT_PDEV_NFCAL_POWER_ALL_CHANNELS_FREQNUM,
+	WMI_TLV_TAG_STRUCT_MU_REPORT_TOTAL_MU,
+	WMI_TLV_TAG_STRUCT_VDEV_SET_DSCP_TID_MAP_CMD,
+	WMI_TLV_TAG_STRUCT_ROAM_SET_MBO,
+	WMI_TLV_TAG_STRUCT_MIB_STATS_ENABLE_CMD,
+	WMI_TLV_TAG_STRUCT_NAN_DISC_IFACE_CREATED_EVENT,
+	WMI_TLV_TAG_STRUCT_NAN_DISC_IFACE_DELETED_EVENT,
+	WMI_TLV_TAG_STRUCT_NAN_STARTED_CLUSTER_EVENT,
+	WMI_TLV_TAG_STRUCT_NAN_JOINED_CLUSTER_EVENT,
+	WMI_TLV_TAG_STRUCT_NDI_GET_CAP_REQ,
+	WMI_TLV_TAG_STRUCT_NDP_INITIATOR_REQ,
+	WMI_TLV_TAG_STRUCT_NDP_RESPONDER_REQ,
+	WMI_TLV_TAG_STRUCT_NDP_END_REQ,
+	WMI_TLV_TAG_STRUCT_NDI_CAP_RSP_EVENT,
+	WMI_TLV_TAG_STRUCT_NDP_INITIATOR_RSP_EVENT,
+	WMI_TLV_TAG_STRUCT_NDP_RESPONDER_RSP_EVENT,
+	WMI_TLV_TAG_STRUCT_NDP_END_RSP_EVENT,
+	WMI_TLV_TAG_STRUCT_NDP_INDICATION_EVENT,
+	WMI_TLV_TAG_STRUCT_NDP_CONFIRM_EVENT,
+	WMI_TLV_TAG_STRUCT_NDP_END_INDICATION_EVENT,
+	WMI_TLV_TAG_STRUCT_VDEV_SET_QUIET_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_PCL_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_HW_MODE_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_MAC_CONFIG_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_ANTENNA_MODE_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_HW_MODE_RESPONSE_EVENT,
+	WMI_TLV_TAG_STRUCT_PDEV_HW_MODE_TRANSITION_EVENT,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_HW_MODE_RESPONSE_VDEV_MAC_ENTRY,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_MAC_CONFIG_RESPONSE_EVENT,
+	WMI_TLV_TAG_STRUCT_COEX_CONFIG_CMD,
+	WMI_TLV_TAG_STRUCT_CONFIG_ENHANCED_MCAST_FILTER,
+	WMI_TLV_TAG_STRUCT_CHAN_AVOID_RPT_ALLOW_CMD,
+	WMI_TLV_TAG_STRUCT_SET_PERIODIC_CHANNEL_STATS_CONFIG,
+	WMI_TLV_TAG_STRUCT_VDEV_SET_CUSTOM_AGGR_SIZE_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_WAL_POWER_DEBUG_CMD,
+	WMI_TLV_TAG_STRUCT_MAC_PHY_CAPABILITIES,
+	WMI_TLV_TAG_STRUCT_HW_MODE_CAPABILITIES,
+	WMI_TLV_TAG_STRUCT_SOC_MAC_PHY_HW_MODE_CAPS,
+	WMI_TLV_TAG_STRUCT_HAL_REG_CAPABILITIES_EXT,
+	WMI_TLV_TAG_STRUCT_SOC_HAL_REG_CAPABILITIES,
+	WMI_TLV_TAG_STRUCT_VDEV_WISA_CMD,
+	WMI_TLV_TAG_STRUCT_TX_POWER_LEVEL_STATS_EVT,
+	WMI_TLV_TAG_STRUCT_SCAN_ADAPTIVE_DWELL_PARAMETERS_TLV,
+	WMI_TLV_TAG_STRUCT_SCAN_ADAPTIVE_DWELL_CONFIG,
+	WMI_TLV_TAG_STRUCT_WOW_SET_ACTION_WAKE_UP_CMD,
+	WMI_TLV_TAG_STRUCT_NDP_END_RSP_PER_NDI,
+	WMI_TLV_TAG_STRUCT_PEER_BWF_REQUEST,
+	WMI_TLV_TAG_STRUCT_BWF_PEER_INFO,
+	WMI_TLV_TAG_STRUCT_DBGLOG_TIME_STAMP_SYNC_CMD,
+	WMI_TLV_TAG_STRUCT_RMC_SET_LEADER_CMD,
+	WMI_TLV_TAG_STRUCT_RMC_MANUAL_LEADER_EVENT,
+	WMI_TLV_TAG_STRUCT_PER_CHAIN_RSSI_STATS,
+	WMI_TLV_TAG_STRUCT_RSSI_STATS,
+	WMI_TLV_TAG_STRUCT_P2P_LO_START_CMD,
+	WMI_TLV_TAG_STRUCT_P2P_LO_STOP_CMD,
+	WMI_TLV_TAG_STRUCT_P2P_LO_STOPPED_EVENT,
+	WMI_TLV_TAG_STRUCT_PEER_REORDER_QUEUE_SETUP_CMD,
+	WMI_TLV_TAG_STRUCT_PEER_REORDER_QUEUE_REMOVE_CMD,
+	WMI_TLV_TAG_STRUCT_SET_MULTIPLE_MCAST_FILTER_CMD,
+	WMI_TLV_TAG_STRUCT_MGMT_TX_COMPL_BUNDLE_EVENT,
+	WMI_TLV_TAG_STRUCT_READ_DATA_FROM_FLASH_CMD,
+	WMI_TLV_TAG_STRUCT_READ_DATA_FROM_FLASH_EVENT,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_REORDER_TIMEOUT_VAL_CMD,
+	WMI_TLV_TAG_STRUCT_PEER_SET_RX_BLOCKSIZE_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_WAKEUP_CONFIG_CMDID,
+	WMI_TLV_TAG_STRUCT_TLV_BUF_LEN_PARAM,
+	WMI_TLV_TAG_STRUCT_SERVICE_AVAILABLE_EVENT,
+	WMI_TLV_TAG_STRUCT_PEER_ANTDIV_INFO_REQ_CMD,
+	WMI_TLV_TAG_STRUCT_PEER_ANTDIV_INFO_EVENT,
+	WMI_TLV_TAG_STRUCT_PEER_ANTDIV_INFO,
+	WMI_TLV_TAG_STRUCT_PDEV_GET_ANTDIV_STATUS_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_ANTDIV_STATUS_EVENT,
+	WMI_TLV_TAG_STRUCT_MNT_FILTER_CMD,
+	WMI_TLV_TAG_STRUCT_GET_CHIP_POWER_STATS_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_CHIP_POWER_STATS_EVENT,
+	WMI_TLV_TAG_STRUCT_COEX_GET_ANTENNA_ISOLATION_CMD,
+	WMI_TLV_TAG_STRUCT_COEX_REPORT_ISOLATION_EVENT,
+	WMI_TLV_TAG_STRUCT_CHAN_CCA_STATS,
+	WMI_TLV_TAG_STRUCT_PEER_SIGNAL_STATS,
+	WMI_TLV_TAG_STRUCT_TX_STATS,
+	WMI_TLV_TAG_STRUCT_PEER_AC_TX_STATS,
+	WMI_TLV_TAG_STRUCT_RX_STATS,
+	WMI_TLV_TAG_STRUCT_PEER_AC_RX_STATS,
+	WMI_TLV_TAG_STRUCT_REPORT_STATS_EVENT,
+	WMI_TLV_TAG_STRUCT_CHAN_CCA_STATS_THRESH,
+	WMI_TLV_TAG_STRUCT_PEER_SIGNAL_STATS_THRESH,
+	WMI_TLV_TAG_STRUCT_TX_STATS_THRESH,
+	WMI_TLV_TAG_STRUCT_RX_STATS_THRESH,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_STATS_THRESHOLD_CMD,
+	WMI_TLV_TAG_STRUCT_REQUEST_WLAN_STATS_CMD,
+	WMI_TLV_TAG_STRUCT_RX_AGGR_FAILURE_EVENT,
+	WMI_TLV_TAG_STRUCT_RX_AGGR_FAILURE_INFO,
+	WMI_TLV_TAG_STRUCT_VDEV_ENCRYPT_DECRYPT_DATA_REQ_CMD,
+	WMI_TLV_TAG_STRUCT_VDEV_ENCRYPT_DECRYPT_DATA_RESP_EVENT,
+	WMI_TLV_TAG_STRUCT_PDEV_BAND_TO_MAC,
+	WMI_TLV_TAG_STRUCT_TBTT_OFFSET_INFO,
+	WMI_TLV_TAG_STRUCT_TBTT_OFFSET_EXT_EVENT,
+	WMI_TLV_TAG_STRUCT_SAR_LIMITS_CMD,
+	WMI_TLV_TAG_STRUCT_SAR_LIMIT_CMD_ROW,
+	WMI_TLV_TAG_STRUCT_PDEV_DFS_PHYERR_OFFLOAD_ENABLE_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_DFS_PHYERR_OFFLOAD_DISABLE_CMD,
+	WMI_TLV_TAG_STRUCT_VDEV_ADFS_CH_CFG_CMD,
+	WMI_TLV_TAG_STRUCT_VDEV_ADFS_OCAC_ABORT_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_DFS_RADAR_DETECTION_EVENT,
+	WMI_TLV_TAG_STRUCT_VDEV_ADFS_OCAC_COMPLETE_EVENT,
+	WMI_TLV_TAG_STRUCT_VDEV_DFS_CAC_COMPLETE_EVENT,
+	WMI_TLV_TAG_STRUCT_VENDOR_OUI,
+	WMI_TLV_TAG_STRUCT_REQUEST_RCPI_CMD,
+	WMI_TLV_TAG_STRUCT_UPDATE_RCPI_EVENT,
+	WMI_TLV_TAG_STRUCT_REQUEST_PEER_STATS_INFO_CMD,
+	WMI_TLV_TAG_STRUCT_PEER_STATS_INFO,
+	WMI_TLV_TAG_STRUCT_PEER_STATS_INFO_EVENT,
+	WMI_TLV_TAG_STRUCT_PKGID_EVENT,
+	WMI_TLV_TAG_STRUCT_CONNECTED_NLO_RSSI_PARAMS,
+	WMI_TLV_TAG_STRUCT_SET_CURRENT_COUNTRY_CMD,
+	WMI_TLV_TAG_STRUCT_REGULATORY_RULE_STRUCT,
+	WMI_TLV_TAG_STRUCT_REG_CHAN_LIST_CC_EVENT,
+	WMI_TLV_TAG_STRUCT_11D_SCAN_START_CMD,
+	WMI_TLV_TAG_STRUCT_11D_SCAN_STOP_CMD,
+	WMI_TLV_TAG_STRUCT_11D_NEW_COUNTRY_EVENT,
+	WMI_TLV_TAG_STRUCT_REQUEST_RADIO_CHAN_STATS_CMD,
+	WMI_TLV_TAG_STRUCT_RADIO_CHAN_STATS,
+	WMI_TLV_TAG_STRUCT_RADIO_CHAN_STATS_EVENT,
+	WMI_TLV_TAG_STRUCT_ROAM_PER_CONFIG,
+	WMI_TLV_TAG_STRUCT_VDEV_ADD_MAC_ADDR_TO_RX_FILTER_CMD,
+	WMI_TLV_TAG_STRUCT_VDEV_ADD_MAC_ADDR_TO_RX_FILTER_STATUS_EVENT,
+	WMI_TLV_TAG_STRUCT_BPF_SET_VDEV_ACTIVE_MODE_CMD,
+	WMI_TLV_TAG_STRUCT_HW_DATA_FILTER_CMD,
+	WMI_TLV_TAG_STRUCT_CONNECTED_NLO_BSS_BAND_RSSI_PREF,
+	WMI_TLV_TAG_STRUCT_PEER_OPER_MODE_CHANGE_EVENT,
+	WMI_TLV_TAG_STRUCT_CHIP_POWER_SAVE_FAILURE_DETECTED,
+	WMI_TLV_TAG_STRUCT_PDEV_MULTIPLE_VDEV_RESTART_REQUEST_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_CSA_SWITCH_COUNT_STATUS_EVENT,
+	WMI_TLV_TAG_STRUCT_PDEV_UPDATE_PKT_ROUTING_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_CHECK_CAL_VERSION_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_CHECK_CAL_VERSION_EVENT,
+	WMI_TLV_TAG_STRUCT_PDEV_SET_DIVERSITY_GAIN_CMD,
+	WMI_TLV_TAG_STRUCT_MAC_PHY_CHAINMASK_COMBO,
+	WMI_TLV_TAG_STRUCT_MAC_PHY_CHAINMASK_CAPABILITY,
+	WMI_TLV_TAG_STRUCT_VDEV_SET_ARP_STATS_CMD,
+	WMI_TLV_TAG_STRUCT_VDEV_GET_ARP_STATS_CMD,
+	WMI_TLV_TAG_STRUCT_VDEV_GET_ARP_STATS_EVENT,
+	WMI_TLV_TAG_STRUCT_IFACE_OFFLOAD_STATS,
+	WMI_TLV_TAG_STRUCT_REQUEST_STATS_CMD_SUB_STRUCT_PARAM,
+	WMI_TLV_TAG_STRUCT_RSSI_CTL_EXT,
+	WMI_TLV_TAG_STRUCT_SINGLE_PHYERR_EXT_RX_HDR,
+	WMI_TLV_TAG_STRUCT_COEX_BT_ACTIVITY_EVENT,
+	WMI_TLV_TAG_STRUCT_VDEV_GET_TX_POWER_CMD,
+	WMI_TLV_TAG_STRUCT_VDEV_TX_POWER_EVENT,
+	WMI_TLV_TAG_STRUCT_OFFCHAN_DATA_TX_COMPL_EVENT,
+	WMI_TLV_TAG_STRUCT_OFFCHAN_DATA_TX_SEND_CMD,
+	WMI_TLV_TAG_STRUCT_TX_SEND_PARAMS,
+	WMI_TLV_TAG_STRUCT_HE_RATE_SET,
+	WMI_TLV_TAG_STRUCT_CONGESTION_STATS,
+	WMI_TLV_TAG_STRUCT_SET_INIT_COUNTRY_CMD,
+	WMI_TLV_TAG_STRUCT_SCAN_DBS_DUTY_CYCLE,
+	WMI_TLV_TAG_STRUCT_SCAN_DBS_DUTY_CYCLE_PARAM_TLV,
+	WMI_TLV_TAG_STRUCT_PDEV_DIV_GET_RSSI_ANTID,
+	WMI_TLV_TAG_STRUCT_THERM_THROT_CONFIG_REQUEST,
+	WMI_TLV_TAG_STRUCT_THERM_THROT_LEVEL_CONFIG_INFO,
+	WMI_TLV_TAG_STRUCT_THERM_THROT_STATS_EVENT,
+	WMI_TLV_TAG_STRUCT_THERM_THROT_LEVEL_STATS_INFO,
+	WMI_TLV_TAG_STRUCT_PDEV_DIV_RSSI_ANTID_EVENT,
+	WMI_TLV_TAG_STRUCT_OEM_DMA_RING_CAPABILITIES,
+	WMI_TLV_TAG_STRUCT_OEM_DMA_RING_CFG_REQ,
+	WMI_TLV_TAG_STRUCT_OEM_DMA_RING_CFG_RSP,
+	WMI_TLV_TAG_STRUCT_OEM_INDIRECT_DATA,
+	WMI_TLV_TAG_STRUCT_OEM_DMA_BUF_RELEASE,
+	WMI_TLV_TAG_STRUCT_OEM_DMA_BUF_RELEASE_ENTRY,
+	WMI_TLV_TAG_STRUCT_PDEV_BSS_CHAN_INFO_REQUEST,
+	WMI_TLV_TAG_STRUCT_PDEV_BSS_CHAN_INFO_EVENT,
+	WMI_TLV_TAG_STRUCT_ROAM_LCA_DISALLOW_CONFIG_TLV_PARAM,
+	WMI_TLV_TAG_STRUCT_VDEV_LIMIT_OFFCHAN_CMD,
+	WMI_TLV_TAG_STRUCT_ROAM_RSSI_REJECTION_OCE_CONFIG_PARAM,
+	WMI_TLV_TAG_STRUCT_UNIT_TEST_EVENT,
+	WMI_TLV_TAG_STRUCT_ROAM_FILS_OFFLOAD_TLV_PARAM,
+	WMI_TLV_TAG_STRUCT_PDEV_UPDATE_PMK_CACHE_CMD,
+	WMI_TLV_TAG_STRUCT_PMK_CACHE,
+	WMI_TLV_TAG_STRUCT_PDEV_UPDATE_FILS_HLP_PKT_CMD,
+	WMI_TLV_TAG_STRUCT_ROAM_FILS_SYNCH_TLV_PARAM,
+	WMI_TLV_TAG_STRUCT_GTK_OFFLOAD_EXTENDED_TLV_PARAM,
+	WMI_TLV_TAG_STRUCT_ROAM_BG_SCAN_ROAMING_PARAM,
+	WMI_TLV_TAG_STRUCT_OIC_PING_OFFLOAD_PARAMS_CMD,
+	WMI_TLV_TAG_STRUCT_OIC_PING_OFFLOAD_SET_ENABLE_CMD,
+	WMI_TLV_TAG_STRUCT_OIC_PING_HANDOFF_EVENT,
+	WMI_TLV_TAG_STRUCT_DHCP_LEASE_RENEW_OFFLOAD_CMD,
+	WMI_TLV_TAG_STRUCT_DHCP_LEASE_RENEW_EVENT,
+	WMI_TLV_TAG_STRUCT_BTM_CONFIG,
+	WMI_TLV_TAG_STRUCT_DEBUG_MESG_FW_DATA_STALL_PARAM,
+	WMI_TLV_TAG_STRUCT_WLM_CONFIG_CMD,
+	WMI_TLV_TAG_STRUCT_PDEV_UPDATE_CTLTABLE_REQUEST,
+	WMI_TLV_TAG_STRUCT_PDEV_UPDATE_CTLTABLE_EVENT,
+	WMI_TLV_TAG_STRUCT_ROAM_CND_SCORING_PARAM,
+	WMI_TLV_TAG_STRUCT_PDEV_CONFIG_VENDOR_OUI_ACTION,
+	WMI_TLV_TAG_STRUCT_VENDOR_OUI_EXT,
+	WMI_TLV_TAG_STRUCT_ROAM_SYNCH_FRAME_EVENT,
+	WMI_TLV_TAG_STRUCT_FD_SEND_FROM_HOST_CMD,
+	WMI_TLV_TAG_STRUCT_ENABLE_FILS_CMD,
+	WMI_TLV_TAG_STRUCT_HOST_SWFDA_EVENT,
 
 	WMI_TLV_TAG_MAX
 };
@@ -1068,16 +1339,74 @@ enum wmi_tlv_service {
 	WMI_TLV_SERVICE_WLAN_STATS_REPORT,
 	WMI_TLV_SERVICE_TX_MSDU_ID_NEW_PARTITION_SUPPORT,
 	WMI_TLV_SERVICE_DFS_PHYERR_OFFLOAD,
+	WMI_TLV_SERVICE_RCPI_SUPPORT,
+	WMI_TLV_SERVICE_FW_MEM_DUMP_SUPPORT,
+	WMI_TLV_SERVICE_PEER_STATS_INFO,
+	WMI_TLV_SERVICE_REGULATORY_DB,
+	WMI_TLV_SERVICE_11D_OFFLOAD,
+	WMI_TLV_SERVICE_HW_DATA_FILTERING,
+	WMI_TLV_SERVICE_MULTIPLE_VDEV_RESTART,
+	WMI_TLV_SERVICE_PKT_ROUTING,
+	WMI_TLV_SERVICE_CHECK_CAL_VERSION,
+	WMI_TLV_SERVICE_OFFCHAN_TX_WMI,
+	WMI_TLV_SERVICE_8SS_TX_BFEE,
+	WMI_TLV_SERVICE_EXTENDED_NSS_SUPPORT,
+	WMI_TLV_SERVICE_ACK_TIMEOUT,
+	WMI_TLV_SERVICE_PDEV_BSS_CHANNEL_INFO_64,
+	WMI_TLV_MAX_SERVICE = 128,
+
+/* NOTE:
+ * The above service flags are delivered in the wmi_service_bitmap field
+ * of the WMI_TLV_SERVICE_READY_EVENT message.
+ * The below service flags are delivered in a WMI_TLV_SERVICE_AVAILABLE_EVENT
+ * message rather than in the WMI_TLV_SERVICE_READY_EVENT message's
+ * wmi_service_bitmap field.
+ * The WMI_TLV_SERVICE_AVAILABLE_EVENT message immediately precedes the
+ * WMI_TLV_SERVICE_READY_EVENT message.
+ */
+
+	WMI_TLV_SERVICE_CHAN_LOAD_INFO = 128,
+	WMI_TLV_SERVICE_TX_PPDU_INFO_STATS_SUPPORT,
+	WMI_TLV_SERVICE_VDEV_LIMIT_OFFCHAN_SUPPORT,
+	WMI_TLV_SERVICE_FILS_SUPPORT,
+	WMI_TLV_SERVICE_WLAN_OIC_PING_OFFLOAD,
+	WMI_TLV_SERVICE_WLAN_DHCP_RENEW,
+	WMI_TLV_SERVICE_MAWC_SUPPORT,
+	WMI_TLV_SERVICE_VDEV_LATENCY_CONFIG,
+	WMI_TLV_SERVICE_PDEV_UPDATE_CTLTABLE_SUPPORT,
+	WMI_TLV_SERVICE_PKTLOG_SUPPORT_OVER_HTT,
+	WMI_TLV_SERVICE_VDEV_MULTI_GROUP_KEY_SUPPORT,
+	WMI_TLV_SERVICE_SCAN_PHYMODE_SUPPORT,
+	WMI_TLV_SERVICE_THERM_THROT,
+	WMI_TLV_SERVICE_BCN_OFFLOAD_START_STOP_SUPPORT,
+	WMI_TLV_SERVICE_WOW_WAKEUP_BY_TIMER_PATTERN,
+	WMI_TLV_SERVICE_PEER_MAP_UNMAP_V2_SUPPORT = 143,
+	WMI_TLV_SERVICE_OFFCHAN_DATA_TID_SUPPORT = 144,
+	WMI_TLV_SERVICE_RX_PROMISC_ENABLE_SUPPORT = 145,
+	WMI_TLV_SERVICE_SUPPORT_DIRECT_DMA = 146,
+	WMI_TLV_SERVICE_AP_OBSS_DETECTION_OFFLOAD = 147,
+	WMI_TLV_SERVICE_11K_NEIGHBOUR_REPORT_SUPPORT = 148,
+	WMI_TLV_SERVICE_LISTEN_INTERVAL_OFFLOAD_SUPPORT = 149,
+	WMI_TLV_SERVICE_BSS_COLOR_OFFLOAD = 150,
+	WMI_TLV_SERVICE_RUNTIME_DPD_RECAL = 151,
+	WMI_TLV_SERVICE_STA_TWT = 152,
+	WMI_TLV_SERVICE_AP_TWT = 153,
+	WMI_TLV_SERVICE_GMAC_OFFLOAD_SUPPORT = 154,
+	WMI_TLV_SERVICE_SPOOF_MAC_SUPPORT = 155,
+
+	WMI_TLV_MAX_EXT_SERVICE = 256,
 };
 
-#define WMI_SERVICE_IS_ENABLED(wmi_svc_bmap, svc_id, len) \
-	((svc_id) < (len) && \
-	 __le32_to_cpu((wmi_svc_bmap)[(svc_id) / (sizeof(u32))]) & \
-	 BIT((svc_id) % (sizeof(u32))))
+#define WMI_TLV_EXT_SERVICE_IS_ENABLED(wmi_svc_bmap, svc_id, len) \
+	((svc_id) < (WMI_TLV_MAX_EXT_SERVICE) && \
+	 (svc_id) >= (len) && \
+	__le32_to_cpu((wmi_svc_bmap)[((svc_id) - (len)) / 32]) & \
+	BIT(((((svc_id) - (len)) % 32) & 0x1f)))
 
 #define SVCMAP(x, y, len) \
 	do { \
-		if (WMI_SERVICE_IS_ENABLED((in), (x), (len))) \
+		if ((WMI_SERVICE_IS_ENABLED((in), (x), (len))) || \
+			(WMI_TLV_EXT_SERVICE_IS_ENABLED((in), (x), (len)))) \
 			__set_bit(y, out); \
 	} while (0)
 
@@ -1228,6 +1557,14 @@ wmi_tlv_svc_map(const __le32 *in, unsigned long *out, size_t len)
 	       WMI_SERVICE_MGMT_TX_WMI, len);
 }
 
+static inline void
+wmi_tlv_svc_map_ext(const __le32 *in, unsigned long *out, size_t len)
+{
+	SVCMAP(WMI_TLV_SERVICE_SPOOF_MAC_SUPPORT,
+	       WMI_SERVICE_SPOOF_MAC_SUPPORT,
+	       WMI_TLV_MAX_SERVICE);
+}
+
 #undef SVCMAP
 
 struct wmi_tlv {
@@ -1370,6 +1707,15 @@ struct wmi_tlv_scan_chan_list_cmd {
 	__le32 num_scan_chans;
 } __packed;
 
+struct wmi_scan_prob_req_oui_cmd {
+/* OUI to be used in Probe Request frame when random MAC address is
+ * requested part of scan parameters. This is applied to both FW internal
+ * scans and host initiated scans. Host can request for random MAC address
+ * with WMI_SCAN_ADD_SPOOFED_MAC_IN_PROBE_REQ flag.
+ */
+	__le32 prob_req_oui;
+}  __packed;
+
 struct wmi_tlv_start_scan_cmd {
 	struct wmi_start_scan_common common;
 	__le32 burst_duration_ms;
@@ -1378,6 +1724,8 @@ struct wmi_tlv_start_scan_cmd {
 	__le32 num_ssids;
 	__le32 ie_len;
 	__le32 num_probes;
+	struct wmi_mac_addr mac_addr;
+	struct wmi_mac_addr mac_mask;
 } __packed;
 
 struct wmi_tlv_vdev_start_cmd {
@@ -1605,6 +1953,21 @@ struct wmi_tlv_wow_add_del_event_cmd {
 	__le32 event_bitmap;
 } __packed;
 
+/* Command to set/unset chip in quiet mode */
+struct wmi_tlv_set_quiet_cmd {
+	__le32 vdev_id;
+
+	/* in TUs */
+	__le32 period;
+
+	/* in TUs */
+	__le32 duration;
+
+	/* offset in TUs */
+	__le32 next_start;
+	__le32 enabled;
+} __packed;
+
 struct wmi_tlv_wow_enable_cmd {
 	__le32 enable;
 } __packed;
diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c
index c5e1ca5945db..f97ab795cf2e 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.c
+++ b/drivers/net/wireless/ath/ath10k/wmi.c
@@ -34,6 +34,7 @@
 
 #define ATH10K_WMI_BARRIER_ECHO_ID 0xBA991E9
 #define ATH10K_WMI_BARRIER_TIMEOUT_HZ (3 * HZ)
+#define ATH10K_WMI_DFS_CONF_TIMEOUT_HZ (HZ / 6)
 
 /* MAIN WMI cmd track */
 static struct wmi_cmd_map wmi_cmd_map = {
@@ -42,6 +43,7 @@ static struct wmi_cmd_map wmi_cmd_map = {
 	.stop_scan_cmdid = WMI_STOP_SCAN_CMDID,
 	.scan_chan_list_cmdid = WMI_SCAN_CHAN_LIST_CMDID,
 	.scan_sch_prio_tbl_cmdid = WMI_SCAN_SCH_PRIO_TBL_CMDID,
+	.scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED,
 	.pdev_set_regdomain_cmdid = WMI_PDEV_SET_REGDOMAIN_CMDID,
 	.pdev_set_channel_cmdid = WMI_PDEV_SET_CHANNEL_CMDID,
 	.pdev_set_param_cmdid = WMI_PDEV_SET_PARAM_CMDID,
@@ -198,6 +200,7 @@ static struct wmi_cmd_map wmi_cmd_map = {
 	.set_cca_params_cmdid = WMI_CMD_UNSUPPORTED,
 	.pdev_bss_chan_info_request_cmdid = WMI_CMD_UNSUPPORTED,
 	.pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED,
+	.radar_found_cmdid = WMI_CMD_UNSUPPORTED,
 };
 
 /* 10.X WMI cmd track */
@@ -207,6 +210,7 @@ static struct wmi_cmd_map wmi_10x_cmd_map = {
 	.stop_scan_cmdid = WMI_10X_STOP_SCAN_CMDID,
 	.scan_chan_list_cmdid = WMI_10X_SCAN_CHAN_LIST_CMDID,
 	.scan_sch_prio_tbl_cmdid = WMI_CMD_UNSUPPORTED,
+	.scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED,
 	.pdev_set_regdomain_cmdid = WMI_10X_PDEV_SET_REGDOMAIN_CMDID,
 	.pdev_set_channel_cmdid = WMI_10X_PDEV_SET_CHANNEL_CMDID,
 	.pdev_set_param_cmdid = WMI_10X_PDEV_SET_PARAM_CMDID,
@@ -365,6 +369,7 @@ static struct wmi_cmd_map wmi_10x_cmd_map = {
 	.set_cca_params_cmdid = WMI_CMD_UNSUPPORTED,
 	.pdev_bss_chan_info_request_cmdid = WMI_CMD_UNSUPPORTED,
 	.pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED,
+	.radar_found_cmdid = WMI_CMD_UNSUPPORTED,
 };
 
 /* 10.2.4 WMI cmd track */
@@ -374,6 +379,7 @@ static struct wmi_cmd_map wmi_10_2_4_cmd_map = {
 	.stop_scan_cmdid = WMI_10_2_STOP_SCAN_CMDID,
 	.scan_chan_list_cmdid = WMI_10_2_SCAN_CHAN_LIST_CMDID,
 	.scan_sch_prio_tbl_cmdid = WMI_CMD_UNSUPPORTED,
+	.scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED,
 	.pdev_set_regdomain_cmdid = WMI_10_2_PDEV_SET_REGDOMAIN_CMDID,
 	.pdev_set_channel_cmdid = WMI_10_2_PDEV_SET_CHANNEL_CMDID,
 	.pdev_set_param_cmdid = WMI_10_2_PDEV_SET_PARAM_CMDID,
@@ -532,6 +538,7 @@ static struct wmi_cmd_map wmi_10_2_4_cmd_map = {
 	.pdev_bss_chan_info_request_cmdid =
 		WMI_10_2_PDEV_BSS_CHAN_INFO_REQUEST_CMDID,
 	.pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED,
+	.radar_found_cmdid = WMI_CMD_UNSUPPORTED,
 };
 
 /* 10.4 WMI cmd track */
@@ -541,6 +548,7 @@ static struct wmi_cmd_map wmi_10_4_cmd_map = {
 	.stop_scan_cmdid = WMI_10_4_STOP_SCAN_CMDID,
 	.scan_chan_list_cmdid = WMI_10_4_SCAN_CHAN_LIST_CMDID,
 	.scan_sch_prio_tbl_cmdid = WMI_10_4_SCAN_SCH_PRIO_TBL_CMDID,
+	.scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED,
 	.pdev_set_regdomain_cmdid = WMI_10_4_PDEV_SET_REGDOMAIN_CMDID,
 	.pdev_set_channel_cmdid = WMI_10_4_PDEV_SET_CHANNEL_CMDID,
 	.pdev_set_param_cmdid = WMI_10_4_PDEV_SET_PARAM_CMDID,
@@ -741,6 +749,7 @@ static struct wmi_cmd_map wmi_10_4_cmd_map = {
 	.tdls_set_state_cmdid = WMI_10_4_TDLS_SET_STATE_CMDID,
 	.tdls_peer_update_cmdid = WMI_10_4_TDLS_PEER_UPDATE_CMDID,
 	.tdls_set_offchan_mode_cmdid = WMI_10_4_TDLS_SET_OFFCHAN_MODE_CMDID,
+	.radar_found_cmdid = WMI_10_4_RADAR_FOUND_CMDID,
 };
 
 /* MAIN WMI VDEV param map */
@@ -1338,6 +1347,7 @@ static struct wmi_cmd_map wmi_10_2_cmd_map = {
 	.stop_scan_cmdid = WMI_10_2_STOP_SCAN_CMDID,
 	.scan_chan_list_cmdid = WMI_10_2_SCAN_CHAN_LIST_CMDID,
 	.scan_sch_prio_tbl_cmdid = WMI_CMD_UNSUPPORTED,
+	.scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED,
 	.pdev_set_regdomain_cmdid = WMI_10_2_PDEV_SET_REGDOMAIN_CMDID,
 	.pdev_set_channel_cmdid = WMI_10_2_PDEV_SET_CHANNEL_CMDID,
 	.pdev_set_param_cmdid = WMI_10_2_PDEV_SET_PARAM_CMDID,
@@ -1485,6 +1495,7 @@ static struct wmi_cmd_map wmi_10_2_cmd_map = {
 	.pdev_get_ani_ofdm_config_cmdid = WMI_CMD_UNSUPPORTED,
 	.pdev_reserve_ast_entry_cmdid = WMI_CMD_UNSUPPORTED,
 	.pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED,
+	.radar_found_cmdid = WMI_CMD_UNSUPPORTED,
 };
 
 static struct wmi_pdev_param_map wmi_10_4_pdev_param_map = {
@@ -2313,7 +2324,6 @@ int ath10k_wmi_event_mgmt_rx(struct ath10k *ar, struct sk_buff *skb)
 	u32 phy_mode;
 	u32 snr;
 	u32 rate;
-	u32 buf_len;
 	u16 fc;
 	int ret;
 
@@ -2325,7 +2335,6 @@ int ath10k_wmi_event_mgmt_rx(struct ath10k *ar, struct sk_buff *skb)
 	}
 
 	channel = __le32_to_cpu(arg.channel);
-	buf_len = __le32_to_cpu(arg.buf_len);
 	rx_status = __le32_to_cpu(arg.status);
 	snr = __le32_to_cpu(arg.snr);
 	phy_mode = __le32_to_cpu(arg.phy_mode);
@@ -2735,14 +2744,13 @@ static int ath10k_wmi_main_op_pull_fw_stats(struct ath10k *ar,
 					    struct ath10k_fw_stats *stats)
 {
 	const struct wmi_stats_event *ev = (void *)skb->data;
-	u32 num_pdev_stats, num_vdev_stats, num_peer_stats;
+	u32 num_pdev_stats, num_peer_stats;
 	int i;
 
 	if (!skb_pull(skb, sizeof(*ev)))
 		return -EPROTO;
 
 	num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats);
-	num_vdev_stats = __le32_to_cpu(ev->num_vdev_stats);
 	num_peer_stats = __le32_to_cpu(ev->num_peer_stats);
 
 	for (i = 0; i < num_pdev_stats; i++) {
@@ -2790,14 +2798,13 @@ static int ath10k_wmi_10x_op_pull_fw_stats(struct ath10k *ar,
 					   struct ath10k_fw_stats *stats)
 {
 	const struct wmi_stats_event *ev = (void *)skb->data;
-	u32 num_pdev_stats, num_vdev_stats, num_peer_stats;
+	u32 num_pdev_stats, num_peer_stats;
 	int i;
 
 	if (!skb_pull(skb, sizeof(*ev)))
 		return -EPROTO;
 
 	num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats);
-	num_vdev_stats = __le32_to_cpu(ev->num_vdev_stats);
 	num_peer_stats = __le32_to_cpu(ev->num_peer_stats);
 
 	for (i = 0; i < num_pdev_stats; i++) {
@@ -2851,7 +2858,6 @@ static int ath10k_wmi_10_2_op_pull_fw_stats(struct ath10k *ar,
 	const struct wmi_10_2_stats_event *ev = (void *)skb->data;
 	u32 num_pdev_stats;
 	u32 num_pdev_ext_stats;
-	u32 num_vdev_stats;
 	u32 num_peer_stats;
 	int i;
 
@@ -2860,7 +2866,6 @@ static int ath10k_wmi_10_2_op_pull_fw_stats(struct ath10k *ar,
 
 	num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats);
 	num_pdev_ext_stats = __le32_to_cpu(ev->num_pdev_ext_stats);
-	num_vdev_stats = __le32_to_cpu(ev->num_vdev_stats);
 	num_peer_stats = __le32_to_cpu(ev->num_peer_stats);
 
 	for (i = 0; i < num_pdev_stats; i++) {
@@ -2930,7 +2935,6 @@ static int ath10k_wmi_10_2_4_op_pull_fw_stats(struct ath10k *ar,
 	const struct wmi_10_2_stats_event *ev = (void *)skb->data;
 	u32 num_pdev_stats;
 	u32 num_pdev_ext_stats;
-	u32 num_vdev_stats;
 	u32 num_peer_stats;
 	int i;
 
@@ -2939,7 +2943,6 @@ static int ath10k_wmi_10_2_4_op_pull_fw_stats(struct ath10k *ar,
 
 	num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats);
 	num_pdev_ext_stats = __le32_to_cpu(ev->num_pdev_ext_stats);
-	num_vdev_stats = __le32_to_cpu(ev->num_vdev_stats);
 	num_peer_stats = __le32_to_cpu(ev->num_peer_stats);
 
 	for (i = 0; i < num_pdev_stats; i++) {
@@ -3686,6 +3689,68 @@ void ath10k_wmi_event_tbttoffset_update(struct ath10k *ar, struct sk_buff *skb)
 	ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_TBTTOFFSET_UPDATE_EVENTID\n");
 }
 
+static void ath10k_radar_detected(struct ath10k *ar)
+{
+	ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs radar detected\n");
+	ATH10K_DFS_STAT_INC(ar, radar_detected);
+
+	/* Control radar events reporting in debugfs file
+	 * dfs_block_radar_events
+	 */
+	if (ar->dfs_block_radar_events)
+		ath10k_info(ar, "DFS Radar detected, but ignored as requested\n");
+	else
+		ieee80211_radar_detected(ar->hw);
+}
+
+static void ath10k_radar_confirmation_work(struct work_struct *work)
+{
+	struct ath10k *ar = container_of(work, struct ath10k,
+					 radar_confirmation_work);
+	struct ath10k_radar_found_info radar_info;
+	int ret, time_left;
+
+	reinit_completion(&ar->wmi.radar_confirm);
+
+	spin_lock_bh(&ar->data_lock);
+	memcpy(&radar_info, &ar->last_radar_info, sizeof(radar_info));
+	spin_unlock_bh(&ar->data_lock);
+
+	ret = ath10k_wmi_report_radar_found(ar, &radar_info);
+	if (ret) {
+		ath10k_warn(ar, "failed to send radar found %d\n", ret);
+		goto wait_complete;
+	}
+
+	time_left = wait_for_completion_timeout(&ar->wmi.radar_confirm,
+						ATH10K_WMI_DFS_CONF_TIMEOUT_HZ);
+	if (time_left) {
+		/* DFS Confirmation status event received and
+		 * necessary action completed.
+		 */
+		goto wait_complete;
+	} else {
+		/* DFS Confirmation event not received from FW.Considering this
+		 * as real radar.
+		 */
+		ath10k_dbg(ar, ATH10K_DBG_REGULATORY,
+			   "dfs confirmation not received from fw, considering as radar\n");
+		goto radar_detected;
+	}
+
+radar_detected:
+	ath10k_radar_detected(ar);
+
+	/* Reset state to allow sending confirmation on consecutive radar
+	 * detections, unless radar confirmation is disabled/stopped.
+	 */
+wait_complete:
+	spin_lock_bh(&ar->data_lock);
+	if (ar->radar_conf_state != ATH10K_RADAR_CONFIRMATION_STOPPED)
+		ar->radar_conf_state = ATH10K_RADAR_CONFIRMATION_IDLE;
+	spin_unlock_bh(&ar->data_lock);
+}
+
 static void ath10k_dfs_radar_report(struct ath10k *ar,
 				    struct wmi_phyerr_ev_arg *phyerr,
 				    const struct phyerr_radar_report *rr,
@@ -3694,8 +3759,10 @@ static void ath10k_dfs_radar_report(struct ath10k *ar,
 	u32 reg0, reg1, tsf32l;
 	struct ieee80211_channel *ch;
 	struct pulse_event pe;
+	struct radar_detector_specs rs;
 	u64 tsf64;
 	u8 rssi, width;
+	struct ath10k_radar_found_info *radar_info;
 
 	reg0 = __le32_to_cpu(rr->reg0);
 	reg1 = __le32_to_cpu(rr->reg1);
@@ -3760,25 +3827,46 @@ static void ath10k_dfs_radar_report(struct ath10k *ar,
 
 	ATH10K_DFS_STAT_INC(ar, pulses_detected);
 
-	if (!ar->dfs_detector->add_pulse(ar->dfs_detector, &pe)) {
+	if (!ar->dfs_detector->add_pulse(ar->dfs_detector, &pe, &rs)) {
 		ath10k_dbg(ar, ATH10K_DBG_REGULATORY,
 			   "dfs no pulse pattern detected, yet\n");
 		return;
 	}
 
-radar_detected:
-	ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs radar detected\n");
-	ATH10K_DFS_STAT_INC(ar, radar_detected);
+	if ((test_bit(WMI_SERVICE_HOST_DFS_CHECK_SUPPORT, ar->wmi.svc_map)) &&
+	    ar->dfs_detector->region == NL80211_DFS_FCC) {
+		/* Consecutive radar indications need not be
+		 * sent to the firmware until we get confirmation
+		 * for the previous detected radar.
+		 */
+		spin_lock_bh(&ar->data_lock);
+		if (ar->radar_conf_state != ATH10K_RADAR_CONFIRMATION_IDLE) {
+			spin_unlock_bh(&ar->data_lock);
+			return;
+		}
+		ar->radar_conf_state = ATH10K_RADAR_CONFIRMATION_INPROGRESS;
+		radar_info = &ar->last_radar_info;
 
-	/* Control radar events reporting in debugfs file
-	 * dfs_block_radar_events
-	 */
-	if (ar->dfs_block_radar_events) {
-		ath10k_info(ar, "DFS Radar detected, but ignored as requested\n");
+		radar_info->pri_min = rs.pri_min;
+		radar_info->pri_max = rs.pri_max;
+		radar_info->width_min = rs.width_min;
+		radar_info->width_max = rs.width_max;
+		/*TODO Find sidx_min and sidx_max */
+		radar_info->sidx_min = MS(reg0, RADAR_REPORT_REG0_PULSE_SIDX);
+		radar_info->sidx_max = MS(reg0, RADAR_REPORT_REG0_PULSE_SIDX);
+
+		ath10k_dbg(ar, ATH10K_DBG_REGULATORY,
+			   "sending wmi radar found cmd pri_min %d pri_max %d width_min %d width_max %d sidx_min %d sidx_max %d\n",
+			   radar_info->pri_min, radar_info->pri_max,
+			   radar_info->width_min, radar_info->width_max,
+			   radar_info->sidx_min, radar_info->sidx_max);
+		ieee80211_queue_work(ar->hw, &ar->radar_confirmation_work);
+		spin_unlock_bh(&ar->data_lock);
 		return;
 	}
 
-	ieee80211_radar_detected(ar->hw);
+radar_detected:
+	ath10k_radar_detected(ar);
 }
 
 static int ath10k_dfs_fft_report(struct ath10k *ar,
@@ -4128,6 +4216,47 @@ void ath10k_wmi_event_phyerr(struct ath10k *ar, struct sk_buff *skb)
 	}
 }
 
+static int
+ath10k_wmi_10_4_op_pull_dfs_status_ev(struct ath10k *ar, struct sk_buff *skb,
+				      struct wmi_dfs_status_ev_arg *arg)
+{
+	struct wmi_dfs_status_ev_arg *ev = (void *)skb->data;
+
+	if (skb->len < sizeof(*ev))
+		return -EPROTO;
+
+	arg->status = ev->status;
+
+	return 0;
+}
+
+static void
+ath10k_wmi_event_dfs_status_check(struct ath10k *ar, struct sk_buff *skb)
+{
+	struct wmi_dfs_status_ev_arg status_arg = {};
+	int ret;
+
+	ret = ath10k_wmi_pull_dfs_status(ar, skb, &status_arg);
+
+	if (ret) {
+		ath10k_warn(ar, "failed to parse dfs status event: %d\n", ret);
+		return;
+	}
+
+	ath10k_dbg(ar, ATH10K_DBG_REGULATORY,
+		   "dfs status event received from fw: %d\n",
+		   status_arg.status);
+
+	/* Even in case of radar detection failure we follow the same
+	 * behaviour as if radar is detected i.e to switch to a different
+	 * channel.
+	 */
+	if (status_arg.status == WMI_HW_RADAR_DETECTED ||
+	    status_arg.status == WMI_RADAR_DETECTION_FAIL)
+		ath10k_radar_detected(ar);
+	complete(&ar->wmi.radar_confirm);
+}
+
 void ath10k_wmi_event_roam(struct ath10k *ar, struct sk_buff *skb)
 {
 	struct wmi_roam_ev_arg arg = {};
@@ -4357,7 +4486,7 @@ static void ath10k_tpc_config_disp_tables(struct ath10k *ar,
 							    rate_code[i],
 							    type);
 			snprintf(buff, sizeof(buff), "%8d ", tpc[j]);
-			strncat(tpc_value, buff, strlen(buff));
+			strlcat(tpc_value, buff, sizeof(tpc_value));
 		}
 		tpc_stats->tpc_table[type].pream_idx[i] = pream_idx;
 		tpc_stats->tpc_table[type].rate_code[i] = rate_code[i];
@@ -4479,6 +4608,12 @@ void ath10k_wmi_event_pdev_tpc_config(struct ath10k *ar, struct sk_buff *skb)
 
 	num_tx_chain = __le32_to_cpu(ev->num_tx_chain);
 
+	if (num_tx_chain > WMI_TPC_TX_N_CHAIN) {
+		ath10k_warn(ar, "number of tx chain is %d greater than TPC configured tx chain %d\n",
+			    num_tx_chain, WMI_TPC_TX_N_CHAIN);
+		return;
+	}
+
 	ath10k_wmi_tpc_config_get_rate_code(rate_code, pream_table,
 					    num_tx_chain);
 
@@ -4694,7 +4829,7 @@ ath10k_wmi_tpc_stats_final_disp_tables(struct ath10k *ar,
 							       rate_code[i],
 							       type, pream_idx);
 			snprintf(buff, sizeof(buff), "%8d ", tpc[j]);
-			strncat(tpc_value, buff, strlen(buff));
+			strlcat(tpc_value, buff, sizeof(tpc_value));
 		}
 		tpc_stats->tpc_table_final[type].pream_idx[i] = pream_idx;
 		tpc_stats->tpc_table_final[type].rate_code[i] = rate_code[i];
@@ -5059,7 +5194,6 @@ static void ath10k_wmi_event_service_ready_work(struct work_struct *work)
 		return;
 	}
 
-	memset(&ar->wmi.svc_map, 0, sizeof(ar->wmi.svc_map));
 	ath10k_wmi_map_svc(ar, arg.service_map, ar->wmi.svc_map,
 			   arg.service_map_len);
 
@@ -5269,6 +5403,21 @@ int ath10k_wmi_event_ready(struct ath10k *ar, struct sk_buff *skb)
 	return 0;
 }
 
+void ath10k_wmi_event_service_available(struct ath10k *ar, struct sk_buff *skb)
+{
+	int ret;
+	struct wmi_svc_avail_ev_arg arg = {};
+
+	ret = ath10k_wmi_pull_svc_avail(ar, skb, &arg);
+	if (ret) {
+		ath10k_warn(ar, "failed to parse service available event: %d\n",
+			    ret);
+	}
+
+	ath10k_wmi_map_svc_ext(ar, arg.service_map_ext, ar->wmi.svc_map,
+			       __le32_to_cpu(arg.service_map_ext_len));
+}
+
 static int ath10k_wmi_event_temperature(struct ath10k *ar, struct sk_buff *skb)
 {
 	const struct wmi_pdev_temperature_event *ev;
@@ -5465,6 +5614,9 @@ static void ath10k_wmi_op_rx(struct ath10k *ar, struct sk_buff *skb)
 		ath10k_wmi_event_ready(ar, skb);
 		ath10k_wmi_queue_set_coverage_class_work(ar);
 		break;
+	case WMI_SERVICE_AVAILABLE_EVENTID:
+		ath10k_wmi_event_service_available(ar, skb);
+		break;
 	default:
 		ath10k_warn(ar, "Unknown eventid: %d\n", id);
 		break;
@@ -5856,6 +6008,9 @@ static void ath10k_wmi_10_4_op_rx(struct ath10k *ar, struct sk_buff *skb)
 	case WMI_10_4_PDEV_TPC_TABLE_EVENTID:
 		ath10k_wmi_event_tpc_final_table(ar, skb);
 		break;
+	case WMI_10_4_DFS_STATUS_CHECK_EVENTID:
+		ath10k_wmi_event_dfs_status_check(ar, skb);
+		break;
 	default:
 		ath10k_warn(ar, "Unknown eventid: %d\n", id);
 		break;
@@ -5880,6 +6035,8 @@ int ath10k_wmi_connect(struct ath10k *ar)
 	struct ath10k_htc_svc_conn_req conn_req;
 	struct ath10k_htc_svc_conn_resp conn_resp;
 
+	memset(&ar->wmi.svc_map, 0, sizeof(ar->wmi.svc_map));
+
 	memset(&conn_req, 0, sizeof(conn_req));
 	memset(&conn_resp, 0, sizeof(conn_resp));
 
@@ -7648,7 +7805,7 @@ ath10k_wmi_10_2_4_op_gen_pdev_get_tpc_config(struct ath10k *ar, u32 param)
 	cmd->param = __cpu_to_le32(param);
 
 	ath10k_dbg(ar, ATH10K_DBG_WMI,
-		   "wmi pdev get tcp config param:%d\n", param);
+		   "wmi pdev get tpc config param %d\n", param);
 	return skb;
 }
 
@@ -7768,7 +7925,7 @@ ath10k_wmi_fw_pdev_tx_stats_fill(const struct ath10k_fw_stats_pdev *pdev,
 	len += scnprintf(buf + len, buf_len - len, "%30s %10d\n",
 			 "HW rate", pdev->data_rc);
 	len += scnprintf(buf + len, buf_len - len, "%30s %10d\n",
-			 "Sched self tiggers", pdev->self_triggers);
+			 "Sched self triggers", pdev->self_triggers);
 	len += scnprintf(buf + len, buf_len - len, "%30s %10d\n",
 			 "Dropped due to SW retries",
 			 pdev->sw_retry_failure);
@@ -8439,6 +8596,32 @@ ath10k_wmi_10_4_gen_tdls_peer_update(struct ath10k *ar,
 	return skb;
 }
 
+static struct sk_buff *
+ath10k_wmi_10_4_gen_radar_found(struct ath10k *ar,
+				const struct ath10k_radar_found_info *arg)
+{
+	struct wmi_radar_found_info *cmd;
+	struct sk_buff *skb;
+
+	skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd));
+	if (!skb)
+		return ERR_PTR(-ENOMEM);
+
+	cmd = (struct wmi_radar_found_info *)skb->data;
+	cmd->pri_min   = __cpu_to_le32(arg->pri_min);
+	cmd->pri_max   = __cpu_to_le32(arg->pri_max);
+	cmd->width_min = __cpu_to_le32(arg->width_min);
+	cmd->width_max = __cpu_to_le32(arg->width_max);
+	cmd->sidx_min  = __cpu_to_le32(arg->sidx_min);
+	cmd->sidx_max  = __cpu_to_le32(arg->sidx_max);
+
+	ath10k_dbg(ar, ATH10K_DBG_WMI,
+		   "wmi radar found pri_min %d pri_max %d width_min %d width_max %d sidx_min %d sidx_max %d\n",
+		   arg->pri_min, arg->pri_max, arg->width_min,
+		   arg->width_max, arg->sidx_min, arg->sidx_max);
+	return skb;
+}
+
 static struct sk_buff *
 ath10k_wmi_op_gen_echo(struct ath10k *ar, u32 value)
 {
@@ -8776,6 +8959,7 @@ static const struct wmi_ops wmi_10_4_ops = {
 	.pull_svc_rdy = ath10k_wmi_main_op_pull_svc_rdy_ev,
 	.pull_rdy = ath10k_wmi_op_pull_rdy_ev,
 	.pull_roam_ev = ath10k_wmi_op_pull_roam_ev,
+	.pull_dfs_status_ev = ath10k_wmi_10_4_op_pull_dfs_status_ev,
 	.get_txbf_conf_scheme = ath10k_wmi_10_4_txbf_conf_scheme,
 
 	.gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend,
@@ -8822,6 +9006,7 @@ static const struct wmi_ops wmi_10_4_ops = {
 	.gen_tdls_peer_update = ath10k_wmi_10_4_gen_tdls_peer_update,
 	.gen_pdev_get_tpc_table_cmdid =
 			ath10k_wmi_10_4_op_gen_pdev_get_tpc_table_cmdid,
+	.gen_radar_found = ath10k_wmi_10_4_gen_radar_found,
 
 	/* shared with 10.2 */
 	.pull_echo_ev = ath10k_wmi_op_pull_echo_ev,
@@ -8884,8 +9069,11 @@ int ath10k_wmi_attach(struct ath10k *ar)
 	init_completion(&ar->wmi.service_ready);
 	init_completion(&ar->wmi.unified_ready);
 	init_completion(&ar->wmi.barrier);
+	init_completion(&ar->wmi.radar_confirm);
 
 	INIT_WORK(&ar->svc_rdy_work, ath10k_wmi_event_service_ready_work);
+	INIT_WORK(&ar->radar_confirmation_work,
+		  ath10k_radar_confirmation_work);
 
 	return 0;
 }
diff --git a/drivers/net/wireless/ath/ath10k/wmi.h b/drivers/net/wireless/ath/ath10k/wmi.h
index 6fbc84c29521..b48db54e9865 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.h
+++ b/drivers/net/wireless/ath/ath10k/wmi.h
@@ -201,6 +201,8 @@ enum wmi_service {
 	WMI_SERVICE_HTT_MGMT_TX_COMP_VALID_FLAGS,
 	WMI_SERVICE_HOST_DFS_CHECK_SUPPORT,
 	WMI_SERVICE_TPC_STATS_FINAL,
+	WMI_SERVICE_RESET_CHIP,
+	WMI_SERVICE_SPOOF_MAC_SUPPORT,
 
 	/* keep last */
 	WMI_SERVICE_MAX,
@@ -238,6 +240,8 @@ enum wmi_10x_service {
 	WMI_10X_SERVICE_MESH,
 	WMI_10X_SERVICE_EXT_RES_CFG_SUPPORT,
 	WMI_10X_SERVICE_PEER_STATS,
+	WMI_10X_SERVICE_RESET_CHIP,
+	WMI_10X_SERVICE_HTT_MGMT_TX_COMP_VALID_FLAGS,
 };
 
 enum wmi_main_service {
@@ -548,6 +552,10 @@ static inline void wmi_10x_svc_map(const __le32 *in, unsigned long *out,
 	       WMI_SERVICE_EXT_RES_CFG_SUPPORT, len);
 	SVCMAP(WMI_10X_SERVICE_PEER_STATS,
 	       WMI_SERVICE_PEER_STATS, len);
+	SVCMAP(WMI_10X_SERVICE_RESET_CHIP,
+	       WMI_SERVICE_RESET_CHIP, len);
+	SVCMAP(WMI_10X_SERVICE_HTT_MGMT_TX_COMP_VALID_FLAGS,
+	       WMI_SERVICE_HTT_MGMT_TX_COMP_VALID_FLAGS, len);
 }
 
 static inline void wmi_main_svc_map(const __le32 *in, unsigned long *out,
@@ -783,6 +791,7 @@ struct wmi_cmd_map {
 	u32 stop_scan_cmdid;
 	u32 scan_chan_list_cmdid;
 	u32 scan_sch_prio_tbl_cmdid;
+	u32 scan_prob_req_oui_cmdid;
 	u32 pdev_set_regdomain_cmdid;
 	u32 pdev_set_channel_cmdid;
 	u32 pdev_set_param_cmdid;
@@ -960,6 +969,7 @@ struct wmi_cmd_map {
 	u32 vdev_sifs_trigger_time_cmdid;
 	u32 pdev_wds_entry_list_cmdid;
 	u32 tdls_set_offchan_mode_cmdid;
+	u32 radar_found_cmdid;
 };
 
 /*
@@ -1183,6 +1193,7 @@ enum wmi_cmd_id {
 enum wmi_event_id {
 	WMI_SERVICE_READY_EVENTID = 0x1,
 	WMI_READY_EVENTID,
+	WMI_SERVICE_AVAILABLE_EVENTID,
 
 	/* Scan specific events */
 	WMI_SCAN_EVENTID = WMI_EVT_GRP_START_ID(WMI_GRP_SCAN),
@@ -1793,6 +1804,11 @@ enum wmi_10_4_cmd_id {
 	WMI_10_4_TDLS_SET_STATE_CMDID,
 	WMI_10_4_TDLS_PEER_UPDATE_CMDID,
 	WMI_10_4_TDLS_SET_OFFCHAN_MODE_CMDID,
+	WMI_10_4_PDEV_SEND_FD_CMDID,
+	WMI_10_4_ENABLE_FILS_CMDID,
+	WMI_10_4_PDEV_SET_BRIDGE_MACADDR_CMDID,
+	WMI_10_4_ATF_GROUP_WMM_AC_CONFIG_REQUEST_CMDID,
+	WMI_10_4_RADAR_FOUND_CMDID,
 	WMI_10_4_PDEV_UTF_CMDID = WMI_10_4_END_CMDID - 1,
 };
 
@@ -1868,6 +1884,9 @@ enum wmi_10_4_event_id {
 	WMI_10_4_PDEV_TPC_TABLE_EVENTID,
 	WMI_10_4_PDEV_WDS_ENTRY_LIST_EVENTID,
 	WMI_10_4_TDLS_PEER_EVENTID,
+	WMI_10_4_HOST_SWFDA_EVENTID,
+	WMI_10_4_ESP_ESTIMATE_EVENTID,
+	WMI_10_4_DFS_STATUS_CHECK_EVENTID,
 	WMI_10_4_PDEV_UTF_EVENTID = WMI_10_4_END_EVENTID - 1,
 };
 
@@ -3159,6 +3178,8 @@ struct wmi_start_scan_arg {
 	u16 channels[64];
 	struct wmi_ssid_arg ssids[WLAN_SCAN_PARAMS_MAX_SSID];
 	struct wmi_bssid_arg bssids[WLAN_SCAN_PARAMS_MAX_BSSID];
+	struct wmi_mac_addr mac_addr;
+	struct wmi_mac_addr mac_mask;
 };
 
 /* scan control flags */
@@ -3182,6 +3203,12 @@ struct wmi_start_scan_arg {
  */
 #define WMI_SCAN_CONTINUE_ON_ERROR 0x80
 
+/* Use random MAC address for TA for Probe Request frame and add
+ * OUI specified by WMI_SCAN_PROB_REQ_OUI_CMDID to the Probe Request frame.
+ * if OUI is not set by WMI_SCAN_PROB_REQ_OUI_CMDID then the flag is ignored.
+ */
+#define WMI_SCAN_ADD_SPOOFED_MAC_IN_PROBE_REQ   0x1000
+
 /* WMI_SCAN_CLASS_MASK must be the same value as IEEE80211_SCAN_CLASS_MASK */
 #define WMI_SCAN_CLASS_MASK 0xFF000000
 
@@ -3380,6 +3407,25 @@ struct wmi_10_4_phyerr_event {
 	u8 buf[0];
 } __packed;
 
+struct wmi_radar_found_info {
+	__le32 pri_min;
+	__le32 pri_max;
+	__le32 width_min;
+	__le32 width_max;
+	__le32 sidx_min;
+	__le32 sidx_max;
+} __packed;
+
+enum wmi_radar_confirmation_status {
+	/* Detected radar was due to SW pulses */
+	WMI_SW_RADAR_DETECTED    = 0,
+
+	WMI_RADAR_DETECTION_FAIL = 1,
+
+	/* Real radar detected */
+	WMI_HW_RADAR_DETECTED    = 2,
+};
+
 #define PHYERR_TLV_SIG				0xBB
 #define PHYERR_TLV_TAG_SEARCH_FFT_REPORT	0xFB
 #define PHYERR_TLV_TAG_RADAR_PULSE_SUMMARY	0xF8
@@ -4008,9 +4054,9 @@ struct wmi_pdev_get_tpc_config_cmd {
 } __packed;
 
 #define WMI_TPC_CONFIG_PARAM		1
-#define WMI_TPC_RATE_MAX		160
 #define WMI_TPC_FINAL_RATE_MAX		240
 #define WMI_TPC_TX_N_CHAIN		4
+#define WMI_TPC_RATE_MAX               (WMI_TPC_TX_N_CHAIN * 65)
 #define WMI_TPC_PREAM_TABLE_MAX		10
 #define WMI_TPC_FLAG			3
 #define WMI_TPC_BUF_SIZE		10
@@ -6613,6 +6659,10 @@ struct wmi_phyerr_hdr_arg {
 	const void *phyerrs;
 };
 
+struct wmi_dfs_status_ev_arg {
+	u32 status;
+};
+
 struct wmi_svc_rdy_ev_arg {
 	__le32 min_tx_power;
 	__le32 max_tx_power;
@@ -6632,6 +6682,11 @@ struct wmi_svc_rdy_ev_arg {
 	const struct wlan_host_mem_req *mem_reqs[WMI_MAX_MEM_REQS];
 };
 
+struct wmi_svc_avail_ev_arg {
+	__le32 service_map_ext_len;
+	const __le32 *service_map_ext;
+};
+
 struct wmi_rdy_ev_arg {
 	__le32 sw_version;
 	__le32 abi_version;
@@ -6812,6 +6867,10 @@ struct wmi_wow_ev_arg {
 #define WOW_MIN_PATTERN_SIZE	1
 #define WOW_MAX_PATTERN_SIZE	148
 #define WOW_MAX_PKT_OFFSET	128
+#define WOW_HDR_LEN	(sizeof(struct ieee80211_hdr_3addr) + \
+	sizeof(struct rfc1042_hdr))
+#define WOW_MAX_REDUCE	(WOW_HDR_LEN - sizeof(struct ethhdr) - \
+	offsetof(struct ieee80211_hdr_3addr, addr1))
 
 enum wmi_tdls_state {
 	WMI_TDLS_DISABLE,
@@ -7052,6 +7111,7 @@ void ath10k_wmi_event_vdev_standby_req(struct ath10k *ar, struct sk_buff *skb);
 void ath10k_wmi_event_vdev_resume_req(struct ath10k *ar, struct sk_buff *skb);
 void ath10k_wmi_event_service_ready(struct ath10k *ar, struct sk_buff *skb);
 int ath10k_wmi_event_ready(struct ath10k *ar, struct sk_buff *skb);
+void ath10k_wmi_event_service_available(struct ath10k *ar, struct sk_buff *skb);
 int ath10k_wmi_op_pull_phyerr_ev(struct ath10k *ar, const void *phyerr_buf,
 				 int left_len, struct wmi_phyerr_ev_arg *arg);
 void ath10k_wmi_main_op_fw_stats_fill(struct ath10k *ar,
diff --git a/drivers/net/wireless/ath/ath10k/wow.c b/drivers/net/wireless/ath/ath10k/wow.c
index c4cbccb29b31..a6b179f88d36 100644
--- a/drivers/net/wireless/ath/ath10k/wow.c
+++ b/drivers/net/wireless/ath/ath10k/wow.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2015-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -76,6 +77,109 @@ static int ath10k_wow_cleanup(struct ath10k *ar)
 	return 0;
 }
 
+/**
+ * Convert a 802.3 format to a 802.11 format.
+ *         +------------+-----------+--------+----------------+
+ * 802.3:  |dest mac(6B)|src mac(6B)|type(2B)|     body...    |
+ *         +------------+-----------+--------+----------------+
+ *                |__         |_______    |____________  |________
+ *                   |                |                |          |
+ *         +--+------------+----+-----------+---------------+-----------+
+ * 802.11: |4B|dest mac(6B)| 6B |src mac(6B)|  8B  |type(2B)|  body...  |
+ *         +--+------------+----+-----------+---------------+-----------+
+ */
+static void ath10k_wow_convert_8023_to_80211
+					(struct cfg80211_pkt_pattern *new,
+					const struct cfg80211_pkt_pattern *old)
+{
+	u8 hdr_8023_pattern[ETH_HLEN] = {};
+	u8 hdr_8023_bit_mask[ETH_HLEN] = {};
+	u8 hdr_80211_pattern[WOW_HDR_LEN] = {};
+	u8 hdr_80211_bit_mask[WOW_HDR_LEN] = {};
+
+	int total_len = old->pkt_offset + old->pattern_len;
+	int hdr_80211_end_offset;
+
+	struct ieee80211_hdr_3addr *new_hdr_pattern =
+		(struct ieee80211_hdr_3addr *)hdr_80211_pattern;
+	struct ieee80211_hdr_3addr *new_hdr_mask =
+		(struct ieee80211_hdr_3addr *)hdr_80211_bit_mask;
+	struct ethhdr *old_hdr_pattern = (struct ethhdr *)hdr_8023_pattern;
+	struct ethhdr *old_hdr_mask = (struct ethhdr *)hdr_8023_bit_mask;
+	int hdr_len = sizeof(*new_hdr_pattern);
+
+	struct rfc1042_hdr *new_rfc_pattern =
+		(struct rfc1042_hdr *)(hdr_80211_pattern + hdr_len);
+	struct rfc1042_hdr *new_rfc_mask =
+		(struct rfc1042_hdr *)(hdr_80211_bit_mask + hdr_len);
+	int rfc_len = sizeof(*new_rfc_pattern);
+
+	memcpy(hdr_8023_pattern + old->pkt_offset,
+	       old->pattern, ETH_HLEN - old->pkt_offset);
+	memcpy(hdr_8023_bit_mask + old->pkt_offset,
+	       old->mask, ETH_HLEN - old->pkt_offset);
+
+	/* Copy destination address */
+	memcpy(new_hdr_pattern->addr1, old_hdr_pattern->h_dest, ETH_ALEN);
+	memcpy(new_hdr_mask->addr1, old_hdr_mask->h_dest, ETH_ALEN);
+
+	/* Copy source address */
+	memcpy(new_hdr_pattern->addr3, old_hdr_pattern->h_source, ETH_ALEN);
+	memcpy(new_hdr_mask->addr3, old_hdr_mask->h_source, ETH_ALEN);
+
+	/* Copy logic link type */
+	memcpy(&new_rfc_pattern->snap_type,
+	       &old_hdr_pattern->h_proto,
+	       sizeof(old_hdr_pattern->h_proto));
+	memcpy(&new_rfc_mask->snap_type,
+	       &old_hdr_mask->h_proto,
+	       sizeof(old_hdr_mask->h_proto));
+
+	/* Caculate new pkt_offset */
+	if (old->pkt_offset < ETH_ALEN)
+		new->pkt_offset = old->pkt_offset +
+			offsetof(struct ieee80211_hdr_3addr, addr1);
+	else if (old->pkt_offset < offsetof(struct ethhdr, h_proto))
+		new->pkt_offset = old->pkt_offset +
+			offsetof(struct ieee80211_hdr_3addr, addr3) -
+			offsetof(struct ethhdr, h_source);
+	else
+		new->pkt_offset = old->pkt_offset + hdr_len + rfc_len - ETH_HLEN;
+
+	/* Caculate new hdr end offset */
+	if (total_len > ETH_HLEN)
+		hdr_80211_end_offset = hdr_len + rfc_len;
+	else if (total_len > offsetof(struct ethhdr, h_proto))
+		hdr_80211_end_offset = hdr_len + rfc_len + total_len - ETH_HLEN;
+	else if (total_len > ETH_ALEN)
+		hdr_80211_end_offset = total_len - ETH_ALEN +
+			offsetof(struct ieee80211_hdr_3addr, addr3);
+	else
+		hdr_80211_end_offset = total_len +
+			offsetof(struct ieee80211_hdr_3addr, addr1);
+
+	new->pattern_len = hdr_80211_end_offset - new->pkt_offset;
+
+	memcpy((u8 *)new->pattern,
+	       hdr_80211_pattern + new->pkt_offset,
+	       new->pattern_len);
+	memcpy((u8 *)new->mask,
+	       hdr_80211_bit_mask + new->pkt_offset,
+	       new->pattern_len);
+
+	if (total_len > ETH_HLEN) {
+		/* Copy frame body */
+		memcpy((u8 *)new->pattern + new->pattern_len,
+		       (void *)old->pattern + ETH_HLEN - old->pkt_offset,
+		       total_len - ETH_HLEN);
+		memcpy((u8 *)new->mask + new->pattern_len,
+		       (void *)old->mask + ETH_HLEN - old->pkt_offset,
+		       total_len - ETH_HLEN);
+
+		new->pattern_len += total_len - ETH_HLEN;
+	}
+}
+
 static int ath10k_vif_wow_set_wakeups(struct ath10k_vif *arvif,
 				      struct cfg80211_wowlan *wowlan)
 {
@@ -116,22 +220,40 @@ static int ath10k_vif_wow_set_wakeups(struct ath10k_vif *arvif,
 
 	for (i = 0; i < wowlan->n_patterns; i++) {
 		u8 bitmask[WOW_MAX_PATTERN_SIZE] = {};
+		u8 ath_pattern[WOW_MAX_PATTERN_SIZE] = {};
+		u8 ath_bitmask[WOW_MAX_PATTERN_SIZE] = {};
+		struct cfg80211_pkt_pattern new_pattern = {};
+		struct cfg80211_pkt_pattern old_pattern = patterns[i];
 		int j;
 
+		new_pattern.pattern = ath_pattern;
+		new_pattern.mask = ath_bitmask;
 		if (patterns[i].pattern_len > WOW_MAX_PATTERN_SIZE)
 			continue;
-
 		/* convert bytemask to bitmask */
 		for (j = 0; j < patterns[i].pattern_len; j++)
 			if (patterns[i].mask[j / 8] & BIT(j % 8))
 				bitmask[j] = 0xff;
+		old_pattern.mask = bitmask;
+		new_pattern = old_pattern;
+
+		if (ar->wmi.rx_decap_mode == ATH10K_HW_TXRX_NATIVE_WIFI) {
+			if (patterns[i].pkt_offset < ETH_HLEN)
+				ath10k_wow_convert_8023_to_80211(&new_pattern,
+								 &old_pattern);
+			else
+				new_pattern.pkt_offset += WOW_HDR_LEN - ETH_HLEN;
+		}
+
+		if (WARN_ON(new_pattern.pattern_len > WOW_MAX_PATTERN_SIZE))
+			return -EINVAL;
 
 		ret = ath10k_wmi_wow_add_pattern(ar, arvif->vdev_id,
 						 pattern_id,
-						 patterns[i].pattern,
-						 bitmask,
-						 patterns[i].pattern_len,
-						 patterns[i].pkt_offset);
+						 new_pattern.pattern,
+						 new_pattern.mask,
+						 new_pattern.pattern_len,
+						 new_pattern.pkt_offset);
 		if (ret) {
 			ath10k_warn(ar, "failed to add pattern %i to vdev %i: %d\n",
 				    pattern_id,
@@ -345,6 +467,12 @@ int ath10k_wow_init(struct ath10k *ar)
 		return -EINVAL;
 
 	ar->wow.wowlan_support = ath10k_wowlan_support;
+
+	if (ar->wmi.rx_decap_mode == ATH10K_HW_TXRX_NATIVE_WIFI) {
+		ar->wow.wowlan_support.pattern_max_len -= WOW_MAX_REDUCE;
+		ar->wow.wowlan_support.max_pkt_offset -= WOW_MAX_REDUCE;
+	}
+
 	ar->wow.wowlan_support.n_patterns = ar->wow.max_num_patterns;
 	ar->hw->wiphy->wowlan = &ar->wow.wowlan_support;
 
diff --git a/drivers/net/wireless/ath/ath6kl/core.h b/drivers/net/wireless/ath/ath6kl/core.h
index e23d450babd2..0d30e762c090 100644
--- a/drivers/net/wireless/ath/ath6kl/core.h
+++ b/drivers/net/wireless/ath/ath6kl/core.h
@@ -914,7 +914,7 @@ void ath6kl_tx_data_cleanup(struct ath6kl *ar);
 
 struct ath6kl_cookie *ath6kl_alloc_cookie(struct ath6kl *ar);
 void ath6kl_free_cookie(struct ath6kl *ar, struct ath6kl_cookie *cookie);
-int ath6kl_data_tx(struct sk_buff *skb, struct net_device *dev);
+netdev_tx_t ath6kl_data_tx(struct sk_buff *skb, struct net_device *dev);
 
 struct aggr_info *aggr_init(struct ath6kl_vif *vif);
 void aggr_conn_init(struct ath6kl_vif *vif, struct aggr_info *aggr_info,
diff --git a/drivers/net/wireless/ath/ath6kl/debug.c b/drivers/net/wireless/ath/ath6kl/debug.c
index 0f965e9f38a4..4e94b22eaada 100644
--- a/drivers/net/wireless/ath/ath6kl/debug.c
+++ b/drivers/net/wireless/ath/ath6kl/debug.c
@@ -645,7 +645,7 @@ static ssize_t read_file_tgt_stats(struct file *file, char __user *user_buf,
 	len += scnprintf(buf + len, buf_len - len, "%20s %10llu\n",
 			 "CRC Err", tgt_stats->rx_crc_err);
 	len += scnprintf(buf + len, buf_len - len, "%20s %10llu\n",
-			 "Key chache miss", tgt_stats->rx_key_cache_miss);
+			 "Key cache miss", tgt_stats->rx_key_cache_miss);
 	len += scnprintf(buf + len, buf_len - len, "%20s %10llu\n",
 			 "Decrypt Err", tgt_stats->rx_decrypt_err);
 	len += scnprintf(buf + len, buf_len - len, "%20s %10llu\n",
diff --git a/drivers/net/wireless/ath/ath6kl/main.c b/drivers/net/wireless/ath/ath6kl/main.c
index db95f85751e3..808fb30be9ad 100644
--- a/drivers/net/wireless/ath/ath6kl/main.c
+++ b/drivers/net/wireless/ath/ath6kl/main.c
@@ -426,7 +426,7 @@ void ath6kl_connect_ap_mode_sta(struct ath6kl_vif *vif, u16 aid, u8 *mac_addr,
 {
 	u8 *ies = NULL, *wpa_ie = NULL, *pos;
 	size_t ies_len = 0;
-	struct station_info sinfo;
+	struct station_info *sinfo;
 
 	ath6kl_dbg(ATH6KL_DBG_TRC, "new station %pM aid=%d\n", mac_addr, aid);
 
@@ -482,16 +482,20 @@ void ath6kl_connect_ap_mode_sta(struct ath6kl_vif *vif, u16 aid, u8 *mac_addr,
 			   keymgmt, ucipher, auth, apsd_info);
 
 	/* send event to application */
-	memset(&sinfo, 0, sizeof(sinfo));
+	sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL);
+	if (!sinfo)
+		return;
 
 	/* TODO: sinfo.generation */
 
-	sinfo.assoc_req_ies = ies;
-	sinfo.assoc_req_ies_len = ies_len;
+	sinfo->assoc_req_ies = ies;
+	sinfo->assoc_req_ies_len = ies_len;
 
-	cfg80211_new_sta(vif->ndev, mac_addr, &sinfo, GFP_KERNEL);
+	cfg80211_new_sta(vif->ndev, mac_addr, sinfo, GFP_KERNEL);
 
 	netif_wake_queue(vif->ndev);
+
+	kfree(sinfo);
 }
 
 void disconnect_timer_handler(struct timer_list *t)
diff --git a/drivers/net/wireless/ath/ath6kl/txrx.c b/drivers/net/wireless/ath/ath6kl/txrx.c
index 8da9506f8c2b..618d12ed4b40 100644
--- a/drivers/net/wireless/ath/ath6kl/txrx.c
+++ b/drivers/net/wireless/ath/ath6kl/txrx.c
@@ -353,7 +353,7 @@ int ath6kl_control_tx(void *devt, struct sk_buff *skb,
 	return status;
 }
 
-int ath6kl_data_tx(struct sk_buff *skb, struct net_device *dev)
+netdev_tx_t ath6kl_data_tx(struct sk_buff *skb, struct net_device *dev)
 {
 	struct ath6kl *ar = ath6kl_priv(dev);
 	struct ath6kl_cookie *cookie = NULL;
diff --git a/drivers/net/wireless/ath/ath9k/dfs.c b/drivers/net/wireless/ath/ath9k/dfs.c
index 6fee9a464cce..acb9602aa464 100644
--- a/drivers/net/wireless/ath/ath9k/dfs.c
+++ b/drivers/net/wireless/ath/ath9k/dfs.c
@@ -41,7 +41,7 @@ static const int BIN_DELTA_MAX		= 10;
 
 /* we need at least 3 deltas / 4 samples for a reliable chirp detection */
 #define NUM_DIFFS 3
-static const int FFT_NUM_SAMPLES	= (NUM_DIFFS + 1);
+#define FFT_NUM_SAMPLES		(NUM_DIFFS + 1)
 
 /* Threshold for difference of delta peaks */
 static const int MAX_DIFF		= 2;
@@ -114,7 +114,7 @@ static bool ath9k_check_chirping(struct ath_softc *sc, u8 *data,
 
 		ath_dbg(common, DFS, "HT40: datalen=%d, num_fft_packets=%d\n",
 			datalen, num_fft_packets);
-		if (num_fft_packets < (FFT_NUM_SAMPLES)) {
+		if (num_fft_packets < FFT_NUM_SAMPLES) {
 			ath_dbg(common, DFS, "not enough packets for chirp\n");
 			return false;
 		}
@@ -136,7 +136,7 @@ static bool ath9k_check_chirping(struct ath_softc *sc, u8 *data,
 			return false;
 		ath_dbg(common, DFS, "HT20: datalen=%d, num_fft_packets=%d\n",
 			datalen, num_fft_packets);
-		if (num_fft_packets < (FFT_NUM_SAMPLES)) {
+		if (num_fft_packets < FFT_NUM_SAMPLES) {
 			ath_dbg(common, DFS, "not enough packets for chirp\n");
 			return false;
 		}
@@ -277,7 +277,7 @@ ath9k_dfs_process_radar_pulse(struct ath_softc *sc, struct pulse_event *pe)
 	DFS_STAT_INC(sc, pulses_processed);
 	if (pd == NULL)
 		return;
-	if (!pd->add_pulse(pd, pe))
+	if (!pd->add_pulse(pd, pe, NULL))
 		return;
 	DFS_STAT_INC(sc, radar_detected);
 	ieee80211_radar_detected(sc->hw);
diff --git a/drivers/net/wireless/ath/ath9k/main.c b/drivers/net/wireless/ath/ath9k/main.c
index a3be8add56e1..b6663c80e7dd 100644
--- a/drivers/net/wireless/ath/ath9k/main.c
+++ b/drivers/net/wireless/ath/ath9k/main.c
@@ -2544,7 +2544,8 @@ static void ath9k_unassign_vif_chanctx(struct ieee80211_hw *hw,
 }
 
 static void ath9k_mgd_prepare_tx(struct ieee80211_hw *hw,
-				 struct ieee80211_vif *vif)
+				 struct ieee80211_vif *vif,
+				 u16 duration)
 {
 	struct ath_softc *sc = hw->priv;
 	struct ath_common *common = ath9k_hw_common(sc->sc_ah);
diff --git a/drivers/net/wireless/ath/dfs_pattern_detector.c b/drivers/net/wireless/ath/dfs_pattern_detector.c
index 448b83eea810..d52b31b45df7 100644
--- a/drivers/net/wireless/ath/dfs_pattern_detector.c
+++ b/drivers/net/wireless/ath/dfs_pattern_detector.c
@@ -268,7 +268,8 @@ static void dpd_exit(struct dfs_pattern_detector *dpd)
 }
 
 static bool
-dpd_add_pulse(struct dfs_pattern_detector *dpd, struct pulse_event *event)
+dpd_add_pulse(struct dfs_pattern_detector *dpd, struct pulse_event *event,
+	      struct radar_detector_specs *rs)
 {
 	u32 i;
 	struct channel_detector *cd;
@@ -294,6 +295,8 @@ dpd_add_pulse(struct dfs_pattern_detector *dpd, struct pulse_event *event)
 		struct pri_detector *pd = cd->detectors[i];
 		struct pri_sequence *ps = pd->add_pulse(pd, event);
 		if (ps != NULL) {
+			if (rs != NULL)
+				memcpy(rs, pd->rs, sizeof(*rs));
 			ath_dbg(dpd->common, DFS,
 				"DFS: radar found on freq=%d: id=%d, pri=%d, "
 				"count=%d, count_false=%d\n",
diff --git a/drivers/net/wireless/ath/dfs_pattern_detector.h b/drivers/net/wireless/ath/dfs_pattern_detector.h
index 92be3530e9b5..18db6f4f3568 100644
--- a/drivers/net/wireless/ath/dfs_pattern_detector.h
+++ b/drivers/net/wireless/ath/dfs_pattern_detector.h
@@ -97,7 +97,8 @@ struct dfs_pattern_detector {
 	bool (*set_dfs_domain)(struct dfs_pattern_detector *dpd,
 			   enum nl80211_dfs_regions region);
 	bool (*add_pulse)(struct dfs_pattern_detector *dpd,
-			  struct pulse_event *pe);
+			  struct pulse_event *pe,
+			  struct radar_detector_specs *rs);
 
 	struct ath_dfs_pool_stats (*get_stats)(struct dfs_pattern_detector *dpd);
 	enum nl80211_dfs_regions region;
diff --git a/drivers/net/wireless/ath/dfs_pri_detector.h b/drivers/net/wireless/ath/dfs_pri_detector.h
index 79f0fff4d1e6..86339f2b4d3a 100644
--- a/drivers/net/wireless/ath/dfs_pri_detector.h
+++ b/drivers/net/wireless/ath/dfs_pri_detector.h
@@ -62,8 +62,9 @@ struct pri_detector {
 	     (*add_pulse)(struct pri_detector *de, struct pulse_event *e);
 	void (*reset)    (struct pri_detector *de, u64 ts);
 
-/* private: internal use only */
 	const struct radar_detector_specs *rs;
+
+/* private: internal use only */
 	u64 last_ts;
 	struct list_head sequences;
 	struct list_head pulses;
diff --git a/drivers/net/wireless/ath/regd.h b/drivers/net/wireless/ath/regd.h
index 5d80be213fac..d73e45e26547 100644
--- a/drivers/net/wireless/ath/regd.h
+++ b/drivers/net/wireless/ath/regd.h
@@ -68,12 +68,14 @@ enum CountryCode {
 	CTRY_AUSTRALIA = 36,
 	CTRY_AUSTRIA = 40,
 	CTRY_AZERBAIJAN = 31,
+	CTRY_BAHAMAS = 44,
 	CTRY_BAHRAIN = 48,
 	CTRY_BANGLADESH = 50,
 	CTRY_BARBADOS = 52,
 	CTRY_BELARUS = 112,
 	CTRY_BELGIUM = 56,
 	CTRY_BELIZE = 84,
+	CTRY_BERMUDA = 60,
 	CTRY_BOLIVIA = 68,
 	CTRY_BOSNIA_HERZ = 70,
 	CTRY_BRAZIL = 76,
@@ -136,8 +138,10 @@ enum CountryCode {
 	CTRY_MACEDONIA = 807,
 	CTRY_MALAYSIA = 458,
 	CTRY_MALTA = 470,
+	CTRY_MAURITIUS = 480,
 	CTRY_MEXICO = 484,
 	CTRY_MONACO = 492,
+	CTRY_MONTENEGRO = 499,
 	CTRY_MOROCCO = 504,
 	CTRY_NEPAL = 524,
 	CTRY_NETHERLANDS = 528,
@@ -159,6 +163,7 @@ enum CountryCode {
 	CTRY_ROMANIA = 642,
 	CTRY_RUSSIA = 643,
 	CTRY_SAUDI_ARABIA = 682,
+	CTRY_SERBIA = 688,
 	CTRY_SERBIA_MONTENEGRO = 891,
 	CTRY_SINGAPORE = 702,
 	CTRY_SLOVAKIA = 703,
@@ -170,11 +175,13 @@ enum CountryCode {
 	CTRY_SWITZERLAND = 756,
 	CTRY_SYRIA = 760,
 	CTRY_TAIWAN = 158,
+	CTRY_TANZANIA = 834,
 	CTRY_THAILAND = 764,
 	CTRY_TRINIDAD_Y_TOBAGO = 780,
 	CTRY_TUNISIA = 788,
 	CTRY_TURKEY = 792,
 	CTRY_UAE = 784,
+	CTRY_UGANDA = 800,
 	CTRY_UKRAINE = 804,
 	CTRY_UNITED_KINGDOM = 826,
 	CTRY_UNITED_STATES = 840,
diff --git a/drivers/net/wireless/ath/regd_common.h b/drivers/net/wireless/ath/regd_common.h
index bdd2b4d61f2f..4021e37a225a 100644
--- a/drivers/net/wireless/ath/regd_common.h
+++ b/drivers/net/wireless/ath/regd_common.h
@@ -35,6 +35,7 @@ enum EnumRd {
 	FRANCE_RES = 0x31,
 	FCC3_FCCA = 0x3A,
 	FCC3_WORLD = 0x3B,
+	FCC3_ETSIC = 0x3F,
 
 	ETSI1_WORLD = 0x37,
 	ETSI3_ETSIA = 0x32,
@@ -44,6 +45,8 @@ enum EnumRd {
 	ETSI4_ETSIC = 0x38,
 	ETSI5_WORLD = 0x39,
 	ETSI6_WORLD = 0x34,
+	ETSI8_WORLD = 0x3D,
+	ETSI9_WORLD = 0x3E,
 	ETSI_RESERVED = 0x33,
 
 	MKK1_MKKA = 0x40,
@@ -59,6 +62,7 @@ enum EnumRd {
 	MKK1_MKKA1 = 0x4A,
 	MKK1_MKKA2 = 0x4B,
 	MKK1_MKKC = 0x4C,
+	APL2_FCCA = 0x4D,
 
 	APL3_FCCA = 0x50,
 	APL1_WORLD = 0x52,
@@ -67,6 +71,7 @@ enum EnumRd {
 	APL1_ETSIC = 0x55,
 	APL2_ETSIC = 0x56,
 	APL5_WORLD = 0x58,
+	APL13_WORLD = 0x5A,
 	APL6_WORLD = 0x5B,
 	APL7_FCCA = 0x5C,
 	APL8_WORLD = 0x5D,
@@ -168,6 +173,7 @@ static struct reg_dmn_pair_mapping regDomainPairs[] = {
 	{FCC2_ETSIC, CTL_FCC, CTL_ETSI},
 	{FCC3_FCCA, CTL_FCC, CTL_FCC},
 	{FCC3_WORLD, CTL_FCC, CTL_ETSI},
+	{FCC3_ETSIC, CTL_FCC, CTL_ETSI},
 	{FCC4_FCCA, CTL_FCC, CTL_FCC},
 	{FCC5_FCCA, CTL_FCC, CTL_FCC},
 	{FCC6_FCCA, CTL_FCC, CTL_FCC},
@@ -179,6 +185,8 @@ static struct reg_dmn_pair_mapping regDomainPairs[] = {
 	{ETSI4_WORLD, CTL_ETSI, CTL_ETSI},
 	{ETSI5_WORLD, CTL_ETSI, CTL_ETSI},
 	{ETSI6_WORLD, CTL_ETSI, CTL_ETSI},
+	{ETSI8_WORLD, CTL_ETSI, CTL_ETSI},
+	{ETSI9_WORLD, CTL_ETSI, CTL_ETSI},
 
 	/* XXX: For ETSI3_ETSIA, Was NO_CTL meant for the 2 GHz band ? */
 	{ETSI3_ETSIA, CTL_ETSI, CTL_ETSI},
@@ -188,9 +196,11 @@ static struct reg_dmn_pair_mapping regDomainPairs[] = {
 	{FCC1_FCCA, CTL_FCC, CTL_FCC},
 	{APL1_WORLD, CTL_FCC, CTL_ETSI},
 	{APL2_WORLD, CTL_FCC, CTL_ETSI},
+	{APL2_FCCA, CTL_FCC, CTL_FCC},
 	{APL3_WORLD, CTL_FCC, CTL_ETSI},
 	{APL4_WORLD, CTL_FCC, CTL_ETSI},
 	{APL5_WORLD, CTL_FCC, CTL_ETSI},
+	{APL13_WORLD, CTL_ETSI, CTL_ETSI},
 	{APL6_WORLD, CTL_ETSI, CTL_ETSI},
 	{APL8_WORLD, CTL_ETSI, CTL_ETSI},
 	{APL9_WORLD, CTL_ETSI, CTL_ETSI},
@@ -289,37 +299,39 @@ static struct reg_dmn_pair_mapping regDomainPairs[] = {
 static struct country_code_to_enum_rd allCountries[] = {
 	{CTRY_DEBUG, NO_ENUMRD, "DB"},
 	{CTRY_DEFAULT, FCC1_FCCA, "CO"},
-	{CTRY_ALBANIA, NULL1_WORLD, "AL"},
-	{CTRY_ALGERIA, NULL1_WORLD, "DZ"},
+	{CTRY_ALBANIA, ETSI1_WORLD, "AL"},
+	{CTRY_ALGERIA, APL13_WORLD, "DZ"},
 	{CTRY_ARGENTINA, FCC3_WORLD, "AR"},
 	{CTRY_ARMENIA, ETSI4_WORLD, "AM"},
 	{CTRY_ARUBA, ETSI1_WORLD, "AW"},
-	{CTRY_AUSTRALIA, FCC2_WORLD, "AU"},
+	{CTRY_AUSTRALIA, FCC3_WORLD, "AU"},
 	{CTRY_AUSTRALIA2, FCC6_WORLD, "AU"},
 	{CTRY_AUSTRIA, ETSI1_WORLD, "AT"},
 	{CTRY_AZERBAIJAN, ETSI4_WORLD, "AZ"},
+	{CTRY_BAHAMAS, FCC3_WORLD, "BS"},
 	{CTRY_BAHRAIN, APL6_WORLD, "BH"},
-	{CTRY_BANGLADESH, NULL1_WORLD, "BD"},
+	{CTRY_BANGLADESH, APL1_WORLD, "BD"},
 	{CTRY_BARBADOS, FCC2_WORLD, "BB"},
 	{CTRY_BELARUS, ETSI1_WORLD, "BY"},
 	{CTRY_BELGIUM, ETSI1_WORLD, "BE"},
 	{CTRY_BELGIUM2, ETSI4_WORLD, "BL"},
 	{CTRY_BELIZE, APL1_ETSIC, "BZ"},
+	{CTRY_BERMUDA, FCC3_FCCA, "BM"},
 	{CTRY_BOLIVIA, APL1_ETSIC, "BO"},
 	{CTRY_BOSNIA_HERZ, ETSI1_WORLD, "BA"},
 	{CTRY_BRAZIL, FCC3_WORLD, "BR"},
-	{CTRY_BRUNEI_DARUSSALAM, APL1_WORLD, "BN"},
-	{CTRY_BULGARIA, ETSI6_WORLD, "BG"},
+	{CTRY_BRUNEI_DARUSSALAM, APL6_WORLD, "BN"},
+	{CTRY_BULGARIA, ETSI1_WORLD, "BG"},
 	{CTRY_CAMBODIA, ETSI1_WORLD, "KH"},
 	{CTRY_CANADA, FCC3_FCCA, "CA"},
 	{CTRY_CANADA2, FCC6_FCCA, "CA"},
 	{CTRY_CHILE, APL6_WORLD, "CL"},
 	{CTRY_CHINA, APL1_WORLD, "CN"},
-	{CTRY_COLOMBIA, FCC1_FCCA, "CO"},
+	{CTRY_COLOMBIA, FCC3_WORLD, "CO"},
 	{CTRY_COSTA_RICA, FCC1_WORLD, "CR"},
 	{CTRY_CROATIA, ETSI1_WORLD, "HR"},
 	{CTRY_CYPRUS, ETSI1_WORLD, "CY"},
-	{CTRY_CZECH, ETSI3_WORLD, "CZ"},
+	{CTRY_CZECH, ETSI1_WORLD, "CZ"},
 	{CTRY_DENMARK, ETSI1_WORLD, "DK"},
 	{CTRY_DOMINICAN_REPUBLIC, FCC1_FCCA, "DO"},
 	{CTRY_ECUADOR, FCC1_WORLD, "EC"},
@@ -336,7 +348,7 @@ static struct country_code_to_enum_rd allCountries[] = {
 	{CTRY_GUAM, FCC1_FCCA, "GU"},
 	{CTRY_GUATEMALA, FCC1_FCCA, "GT"},
 	{CTRY_HAITI, ETSI1_WORLD, "HT"},
-	{CTRY_HONDURAS, NULL1_WORLD, "HN"},
+	{CTRY_HONDURAS, FCC3_WORLD, "HN"},
 	{CTRY_HONG_KONG, FCC3_WORLD, "HK"},
 	{CTRY_HUNGARY, ETSI1_WORLD, "HU"},
 	{CTRY_ICELAND, ETSI1_WORLD, "IS"},
@@ -344,7 +356,7 @@ static struct country_code_to_enum_rd allCountries[] = {
 	{CTRY_INDONESIA, NULL1_WORLD, "ID"},
 	{CTRY_IRAN, APL1_WORLD, "IR"},
 	{CTRY_IRELAND, ETSI1_WORLD, "IE"},
-	{CTRY_ISRAEL, NULL1_WORLD, "IL"},
+	{CTRY_ISRAEL, ETSI3_WORLD, "IL"},
 	{CTRY_ITALY, ETSI1_WORLD, "IT"},
 	{CTRY_JAMAICA, FCC3_WORLD, "JM"},
 
@@ -409,6 +421,7 @@ static struct country_code_to_enum_rd allCountries[] = {
 
 	{CTRY_JORDAN, ETSI2_WORLD, "JO"},
 	{CTRY_KAZAKHSTAN, NULL1_WORLD, "KZ"},
+	{CTRY_KENYA, APL1_WORLD, "KE"},
 	{CTRY_KOREA_NORTH, APL9_WORLD, "KP"},
 	{CTRY_KOREA_ROC, APL9_WORLD, "KR"},
 	{CTRY_KOREA_ROC2, APL2_WORLD, "K2"},
@@ -420,32 +433,37 @@ static struct country_code_to_enum_rd allCountries[] = {
 	{CTRY_LITHUANIA, ETSI1_WORLD, "LT"},
 	{CTRY_LUXEMBOURG, ETSI1_WORLD, "LU"},
 	{CTRY_MACAU, FCC2_WORLD, "MO"},
-	{CTRY_MACEDONIA, NULL1_WORLD, "MK"},
-	{CTRY_MALAYSIA, APL8_WORLD, "MY"},
+	{CTRY_MACEDONIA, ETSI1_WORLD, "MK"},
+	{CTRY_MALAYSIA, FCC1_WORLD, "MY"},
 	{CTRY_MALTA, ETSI1_WORLD, "MT"},
+	{CTRY_MAURITIUS, ETSI1_WORLD, "MU"},
 	{CTRY_MEXICO, FCC1_FCCA, "MX"},
 	{CTRY_MONACO, ETSI4_WORLD, "MC"},
+	{CTRY_MONTENEGRO, ETSI1_WORLD, "ME"},
 	{CTRY_MOROCCO, APL4_WORLD, "MA"},
 	{CTRY_NEPAL, APL1_WORLD, "NP"},
 	{CTRY_NETHERLANDS, ETSI1_WORLD, "NL"},
 	{CTRY_NETHERLANDS_ANTILLES, ETSI1_WORLD, "AN"},
-	{CTRY_NEW_ZEALAND, FCC2_ETSIC, "NZ"},
+	{CTRY_NEW_ZEALAND, FCC3_ETSIC, "NZ"},
+	{CTRY_NICARAGUA, FCC3_FCCA, "NI"},
 	{CTRY_NORWAY, ETSI1_WORLD, "NO"},
 	{CTRY_OMAN, FCC3_WORLD, "OM"},
 	{CTRY_PAKISTAN, NULL1_WORLD, "PK"},
 	{CTRY_PANAMA, FCC1_FCCA, "PA"},
 	{CTRY_PAPUA_NEW_GUINEA, FCC1_WORLD, "PG"},
-	{CTRY_PERU, APL1_WORLD, "PE"},
-	{CTRY_PHILIPPINES, APL1_WORLD, "PH"},
+	{CTRY_PARAGUAY, FCC3_WORLD, "PY"},
+	{CTRY_PERU, FCC3_WORLD, "PE"},
+	{CTRY_PHILIPPINES, FCC3_WORLD, "PH"},
 	{CTRY_POLAND, ETSI1_WORLD, "PL"},
 	{CTRY_PORTUGAL, ETSI1_WORLD, "PT"},
 	{CTRY_PUERTO_RICO, FCC1_FCCA, "PR"},
 	{CTRY_QATAR, APL1_WORLD, "QA"},
-	{CTRY_ROMANIA, NULL1_WORLD, "RO"},
-	{CTRY_RUSSIA, NULL1_WORLD, "RU"},
+	{CTRY_ROMANIA, ETSI1_WORLD, "RO"},
+	{CTRY_RUSSIA, ETSI8_WORLD, "RU"},
 	{CTRY_SAUDI_ARABIA, NULL1_WORLD, "SA"},
+	{CTRY_SERBIA, ETSI1_WORLD, "RS"},
 	{CTRY_SERBIA_MONTENEGRO, ETSI1_WORLD, "CS"},
-	{CTRY_SINGAPORE, APL6_WORLD, "SG"},
+	{CTRY_SINGAPORE, FCC3_WORLD, "SG"},
 	{CTRY_SLOVAKIA, ETSI1_WORLD, "SK"},
 	{CTRY_SLOVENIA, ETSI1_WORLD, "SI"},
 	{CTRY_SOUTH_AFRICA, FCC3_WORLD, "ZA"},
@@ -455,11 +473,13 @@ static struct country_code_to_enum_rd allCountries[] = {
 	{CTRY_SWITZERLAND, ETSI1_WORLD, "CH"},
 	{CTRY_SYRIA, NULL1_WORLD, "SY"},
 	{CTRY_TAIWAN, APL3_FCCA, "TW"},
+	{CTRY_TANZANIA, APL1_WORLD, "TZ"},
 	{CTRY_THAILAND, FCC3_WORLD, "TH"},
 	{CTRY_TRINIDAD_Y_TOBAGO, FCC3_WORLD, "TT"},
 	{CTRY_TUNISIA, ETSI3_WORLD, "TN"},
 	{CTRY_TURKEY, ETSI3_WORLD, "TR"},
-	{CTRY_UKRAINE, NULL1_WORLD, "UA"},
+	{CTRY_UGANDA, FCC3_WORLD, "UG"},
+	{CTRY_UKRAINE, ETSI9_WORLD, "UA"},
 	{CTRY_UAE, NULL1_WORLD, "AE"},
 	{CTRY_UNITED_KINGDOM, ETSI1_WORLD, "GB"},
 	{CTRY_UNITED_STATES, FCC3_FCCA, "US"},
@@ -472,7 +492,7 @@ static struct country_code_to_enum_rd allCountries[] = {
 	{CTRY_VENEZUELA, APL2_ETSIC, "VE"},
 	{CTRY_VIET_NAM, NULL1_WORLD, "VN"},
 	{CTRY_YEMEN, NULL1_WORLD, "YE"},
-	{CTRY_ZIMBABWE, NULL1_WORLD, "ZW"},
+	{CTRY_ZIMBABWE, ETSI1_WORLD, "ZW"},
 };
 
 #endif
diff --git a/drivers/net/wireless/ath/wcn36xx/Makefile b/drivers/net/wireless/ath/wcn36xx/Makefile
index 3b09435104eb..582049f65735 100644
--- a/drivers/net/wireless/ath/wcn36xx/Makefile
+++ b/drivers/net/wireless/ath/wcn36xx/Makefile
@@ -6,3 +6,5 @@ wcn36xx-y +=   main.o \
                smd.o \
                pmc.o \
                debug.o
+
+wcn36xx-$(CONFIG_NL80211_TESTMODE) += testmode.o
diff --git a/drivers/net/wireless/ath/wcn36xx/dxe.c b/drivers/net/wireless/ath/wcn36xx/dxe.c
index 2c3b899a88fa..06cfe8d311f3 100644
--- a/drivers/net/wireless/ath/wcn36xx/dxe.c
+++ b/drivers/net/wireless/ath/wcn36xx/dxe.c
@@ -78,7 +78,6 @@ static int wcn36xx_dxe_allocate_ctl_block(struct wcn36xx_dxe_ch *ch)
 		if (!cur_ctl)
 			goto out_fail;
 
-		spin_lock_init(&cur_ctl->skb_lock);
 		cur_ctl->ctl_blk_order = i;
 		if (i == 0) {
 			ch->head_blk_ctl = cur_ctl;
@@ -275,12 +274,14 @@ static int wcn36xx_dxe_enable_ch_int(struct wcn36xx *wcn, u16 wcn_ch)
 	return 0;
 }
 
-static int wcn36xx_dxe_fill_skb(struct device *dev, struct wcn36xx_dxe_ctl *ctl)
+static int wcn36xx_dxe_fill_skb(struct device *dev,
+				struct wcn36xx_dxe_ctl *ctl,
+				gfp_t gfp)
 {
 	struct wcn36xx_dxe_desc *dxe = ctl->desc;
 	struct sk_buff *skb;
 
-	skb = alloc_skb(WCN36XX_PKT_SIZE, GFP_ATOMIC);
+	skb = alloc_skb(WCN36XX_PKT_SIZE, gfp);
 	if (skb == NULL)
 		return -ENOMEM;
 
@@ -307,7 +308,7 @@ static int wcn36xx_dxe_ch_alloc_skb(struct wcn36xx *wcn,
 	cur_ctl = wcn_ch->head_blk_ctl;
 
 	for (i = 0; i < wcn_ch->desc_num; i++) {
-		wcn36xx_dxe_fill_skb(wcn->dev, cur_ctl);
+		wcn36xx_dxe_fill_skb(wcn->dev, cur_ctl, GFP_KERNEL);
 		cur_ctl = cur_ctl->next;
 	}
 
@@ -367,9 +368,11 @@ static void reap_tx_dxes(struct wcn36xx *wcn, struct wcn36xx_dxe_ch *ch)
 	spin_lock_irqsave(&ch->lock, flags);
 	ctl = ch->tail_blk_ctl;
 	do {
-		if (ctl->desc->ctrl & WCN36xx_DXE_CTRL_VLD)
+		if (READ_ONCE(ctl->desc->ctrl) & WCN36xx_DXE_CTRL_VLD)
 			break;
-		if (ctl->skb) {
+
+		if (ctl->skb &&
+		    READ_ONCE(ctl->desc->ctrl) & WCN36xx_DXE_CTRL_EOP) {
 			dma_unmap_single(wcn->dev, ctl->desc->src_addr_l,
 					 ctl->skb->len, DMA_TO_DEVICE);
 			info = IEEE80211_SKB_CB(ctl->skb);
@@ -377,18 +380,16 @@ static void reap_tx_dxes(struct wcn36xx *wcn, struct wcn36xx_dxe_ch *ch)
 				/* Keep frame until TX status comes */
 				ieee80211_free_txskb(wcn->hw, ctl->skb);
 			}
-			spin_lock(&ctl->skb_lock);
+
 			if (wcn->queues_stopped) {
 				wcn->queues_stopped = false;
 				ieee80211_wake_queues(wcn->hw);
 			}
-			spin_unlock(&ctl->skb_lock);
 
 			ctl->skb = NULL;
 		}
 		ctl = ctl->next;
-	} while (ctl != ch->head_blk_ctl &&
-	       !(ctl->desc->ctrl & WCN36xx_DXE_CTRL_VLD));
+	} while (ctl != ch->head_blk_ctl);
 
 	ch->tail_blk_ctl = ctl;
 	spin_unlock_irqrestore(&ch->lock, flags);
@@ -431,8 +432,12 @@ static irqreturn_t wcn36xx_irq_tx_complete(int irq, void *dev)
 						   WCN36XX_INT_MASK_CHAN_TX_H);
 		}
 
-		wcn36xx_dbg(WCN36XX_DBG_DXE, "dxe tx ready high\n");
-		reap_tx_dxes(wcn, &wcn->dxe_tx_h_ch);
+		wcn36xx_dbg(WCN36XX_DBG_DXE, "dxe tx ready high, reason %08x\n",
+			    int_reason);
+
+		if (int_reason & (WCN36XX_CH_STAT_INT_DONE_MASK |
+				  WCN36XX_CH_STAT_INT_ED_MASK))
+			reap_tx_dxes(wcn, &wcn->dxe_tx_h_ch);
 	}
 
 	if (int_src & WCN36XX_INT_MASK_CHAN_TX_L) {
@@ -466,8 +471,12 @@ static irqreturn_t wcn36xx_irq_tx_complete(int irq, void *dev)
 						   WCN36XX_INT_MASK_CHAN_TX_L);
 		}
 
-		wcn36xx_dbg(WCN36XX_DBG_DXE, "dxe tx ready low\n");
-		reap_tx_dxes(wcn, &wcn->dxe_tx_l_ch);
+		wcn36xx_dbg(WCN36XX_DBG_DXE, "dxe tx ready low, reason %08x\n",
+			    int_reason);
+
+		if (int_reason & (WCN36XX_CH_STAT_INT_DONE_MASK |
+				  WCN36XX_CH_STAT_INT_ED_MASK))
+			reap_tx_dxes(wcn, &wcn->dxe_tx_l_ch);
 	}
 
 	return IRQ_HANDLED;
@@ -477,9 +486,8 @@ static irqreturn_t wcn36xx_irq_rx_ready(int irq, void *dev)
 {
 	struct wcn36xx *wcn = (struct wcn36xx *)dev;
 
-	disable_irq_nosync(wcn->rx_irq);
 	wcn36xx_dxe_rx_frame(wcn);
-	enable_irq(wcn->rx_irq);
+
 	return IRQ_HANDLED;
 }
 
@@ -513,27 +521,53 @@ static int wcn36xx_dxe_request_irqs(struct wcn36xx *wcn)
 }
 
 static int wcn36xx_rx_handle_packets(struct wcn36xx *wcn,
-				     struct wcn36xx_dxe_ch *ch)
+				     struct wcn36xx_dxe_ch *ch,
+				     u32 ctrl,
+				     u32 en_mask,
+				     u32 int_mask,
+				     u32 status_reg)
 {
-	struct wcn36xx_dxe_ctl *ctl = ch->head_blk_ctl;
-	struct wcn36xx_dxe_desc *dxe = ctl->desc;
+	struct wcn36xx_dxe_desc *dxe;
+	struct wcn36xx_dxe_ctl *ctl;
 	dma_addr_t  dma_addr;
 	struct sk_buff *skb;
-	int ret = 0, int_mask;
-	u32 value;
+	u32 int_reason;
+	int ret;
 
-	if (ch->ch_type == WCN36XX_DXE_CH_RX_L) {
-		value = WCN36XX_DXE_CTRL_RX_L;
-		int_mask = WCN36XX_DXE_INT_CH1_MASK;
-	} else {
-		value = WCN36XX_DXE_CTRL_RX_H;
-		int_mask = WCN36XX_DXE_INT_CH3_MASK;
+	wcn36xx_dxe_read_register(wcn, status_reg, &int_reason);
+	wcn36xx_dxe_write_register(wcn, WCN36XX_DXE_0_INT_CLR, int_mask);
+
+	if (int_reason & WCN36XX_CH_STAT_INT_ERR_MASK) {
+		wcn36xx_dxe_write_register(wcn,
+					   WCN36XX_DXE_0_INT_ERR_CLR,
+					   int_mask);
+
+		wcn36xx_err("DXE IRQ reported error on RX channel\n");
 	}
 
-	while (!(dxe->ctrl & WCN36xx_DXE_CTRL_VLD)) {
+	if (int_reason & WCN36XX_CH_STAT_INT_DONE_MASK)
+		wcn36xx_dxe_write_register(wcn,
+					   WCN36XX_DXE_0_INT_DONE_CLR,
+					   int_mask);
+
+	if (int_reason & WCN36XX_CH_STAT_INT_ED_MASK)
+		wcn36xx_dxe_write_register(wcn,
+					   WCN36XX_DXE_0_INT_ED_CLR,
+					   int_mask);
+
+	if (!(int_reason & (WCN36XX_CH_STAT_INT_DONE_MASK |
+			    WCN36XX_CH_STAT_INT_ED_MASK)))
+		return 0;
+
+	spin_lock(&ch->lock);
+
+	ctl = ch->head_blk_ctl;
+	dxe = ctl->desc;
+
+	while (!(READ_ONCE(dxe->ctrl) & WCN36xx_DXE_CTRL_VLD)) {
 		skb = ctl->skb;
 		dma_addr = dxe->dst_addr_l;
-		ret = wcn36xx_dxe_fill_skb(wcn->dev, ctl);
+		ret = wcn36xx_dxe_fill_skb(wcn->dev, ctl, GFP_ATOMIC);
 		if (0 == ret) {
 			/* new skb allocation ok. Use the new one and queue
 			 * the old one to network system.
@@ -543,13 +577,16 @@ static int wcn36xx_rx_handle_packets(struct wcn36xx *wcn,
 			wcn36xx_rx_skb(wcn, skb);
 		} /* else keep old skb not submitted and use it for rx DMA */
 
-		dxe->ctrl = value;
+		dxe->ctrl = ctrl;
 		ctl = ctl->next;
 		dxe = ctl->desc;
 	}
-	wcn36xx_dxe_write_register(wcn, WCN36XX_DXE_ENCH_ADDR, int_mask);
+	wcn36xx_dxe_write_register(wcn, WCN36XX_DXE_ENCH_ADDR, en_mask);
 
 	ch->head_blk_ctl = ctl;
+
+	spin_unlock(&ch->lock);
+
 	return 0;
 }
 
@@ -560,19 +597,20 @@ void wcn36xx_dxe_rx_frame(struct wcn36xx *wcn)
 	wcn36xx_dxe_read_register(wcn, WCN36XX_DXE_INT_SRC_RAW_REG, &int_src);
 
 	/* RX_LOW_PRI */
-	if (int_src & WCN36XX_DXE_INT_CH1_MASK) {
-		wcn36xx_dxe_write_register(wcn, WCN36XX_DXE_0_INT_CLR,
-					   WCN36XX_DXE_INT_CH1_MASK);
-		wcn36xx_rx_handle_packets(wcn, &(wcn->dxe_rx_l_ch));
-	}
+	if (int_src & WCN36XX_DXE_INT_CH1_MASK)
+		wcn36xx_rx_handle_packets(wcn, &wcn->dxe_rx_l_ch,
+					  WCN36XX_DXE_CTRL_RX_L,
+					  WCN36XX_DXE_INT_CH1_MASK,
+					  WCN36XX_INT_MASK_CHAN_RX_L,
+					  WCN36XX_DXE_CH_STATUS_REG_ADDR_RX_L);
 
 	/* RX_HIGH_PRI */
-	if (int_src & WCN36XX_DXE_INT_CH3_MASK) {
-		/* Clean up all the INT within this channel */
-		wcn36xx_dxe_write_register(wcn, WCN36XX_DXE_0_INT_CLR,
-					   WCN36XX_DXE_INT_CH3_MASK);
-		wcn36xx_rx_handle_packets(wcn, &(wcn->dxe_rx_h_ch));
-	}
+	if (int_src & WCN36XX_DXE_INT_CH3_MASK)
+		wcn36xx_rx_handle_packets(wcn, &wcn->dxe_rx_h_ch,
+					  WCN36XX_DXE_CTRL_RX_H,
+					  WCN36XX_DXE_INT_CH3_MASK,
+					  WCN36XX_INT_MASK_CHAN_RX_H,
+					  WCN36XX_DXE_CH_STATUS_REG_ADDR_RX_H);
 
 	if (!int_src)
 		wcn36xx_warn("No DXE interrupt pending\n");
@@ -643,8 +681,8 @@ int wcn36xx_dxe_tx_frame(struct wcn36xx *wcn,
 			 struct sk_buff *skb,
 			 bool is_low)
 {
-	struct wcn36xx_dxe_ctl *ctl = NULL;
-	struct wcn36xx_dxe_desc *desc = NULL;
+	struct wcn36xx_dxe_desc *desc_bd, *desc_skb;
+	struct wcn36xx_dxe_ctl *ctl_bd, *ctl_skb;
 	struct wcn36xx_dxe_ch *ch = NULL;
 	unsigned long flags;
 	int ret;
@@ -652,73 +690,75 @@ int wcn36xx_dxe_tx_frame(struct wcn36xx *wcn,
 	ch = is_low ? &wcn->dxe_tx_l_ch : &wcn->dxe_tx_h_ch;
 
 	spin_lock_irqsave(&ch->lock, flags);
-	ctl = ch->head_blk_ctl;
-
-	spin_lock(&ctl->next->skb_lock);
+	ctl_bd = ch->head_blk_ctl;
+	ctl_skb = ctl_bd->next;
 
 	/*
 	 * If skb is not null that means that we reached the tail of the ring
 	 * hence ring is full. Stop queues to let mac80211 back off until ring
 	 * has an empty slot again.
 	 */
-	if (NULL != ctl->next->skb) {
+	if (NULL != ctl_skb->skb) {
 		ieee80211_stop_queues(wcn->hw);
 		wcn->queues_stopped = true;
-		spin_unlock(&ctl->next->skb_lock);
 		spin_unlock_irqrestore(&ch->lock, flags);
 		return -EBUSY;
 	}
-	spin_unlock(&ctl->next->skb_lock);
 
-	ctl->skb = NULL;
-	desc = ctl->desc;
+	if (unlikely(ctl_skb->bd_cpu_addr)) {
+		wcn36xx_err("bd_cpu_addr cannot be NULL for skb DXE\n");
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	desc_bd = ctl_bd->desc;
+	desc_skb = ctl_skb->desc;
+
+	ctl_bd->skb = NULL;
 
 	/* write buffer descriptor */
-	memcpy(ctl->bd_cpu_addr, bd, sizeof(*bd));
+	memcpy(ctl_bd->bd_cpu_addr, bd, sizeof(*bd));
 
 	/* Set source address of the BD we send */
-	desc->src_addr_l = ctl->bd_phy_addr;
-
-	desc->dst_addr_l = ch->dxe_wq;
-	desc->fr_len = sizeof(struct wcn36xx_tx_bd);
-	desc->ctrl = ch->ctrl_bd;
+	desc_bd->src_addr_l = ctl_bd->bd_phy_addr;
+	desc_bd->dst_addr_l = ch->dxe_wq;
+	desc_bd->fr_len = sizeof(struct wcn36xx_tx_bd);
 
 	wcn36xx_dbg(WCN36XX_DBG_DXE, "DXE TX\n");
 
 	wcn36xx_dbg_dump(WCN36XX_DBG_DXE_DUMP, "DESC1 >>> ",
-			 (char *)desc, sizeof(*desc));
+			 (char *)desc_bd, sizeof(*desc_bd));
 	wcn36xx_dbg_dump(WCN36XX_DBG_DXE_DUMP,
-			 "BD   >>> ", (char *)ctl->bd_cpu_addr,
+			 "BD   >>> ", (char *)ctl_bd->bd_cpu_addr,
 			 sizeof(struct wcn36xx_tx_bd));
 
-	/* Set source address of the SKB we send */
-	ctl = ctl->next;
-	ctl->skb = skb;
-	desc = ctl->desc;
-	if (ctl->bd_cpu_addr) {
-		wcn36xx_err("bd_cpu_addr cannot be NULL for skb DXE\n");
-		ret = -EINVAL;
+	desc_skb->src_addr_l = dma_map_single(wcn->dev,
+					      skb->data,
+					      skb->len,
+					      DMA_TO_DEVICE);
+	if (dma_mapping_error(wcn->dev, desc_skb->src_addr_l)) {
+		dev_err(wcn->dev, "unable to DMA map src_addr_l\n");
+		ret = -ENOMEM;
 		goto unlock;
 	}
 
-	desc->src_addr_l = dma_map_single(wcn->dev,
-					  ctl->skb->data,
-					  ctl->skb->len,
-					  DMA_TO_DEVICE);
-
-	desc->dst_addr_l = ch->dxe_wq;
-	desc->fr_len = ctl->skb->len;
-
-	/* set dxe descriptor to VALID */
-	desc->ctrl = ch->ctrl_skb;
+	ctl_skb->skb = skb;
+	desc_skb->dst_addr_l = ch->dxe_wq;
+	desc_skb->fr_len = ctl_skb->skb->len;
 
 	wcn36xx_dbg_dump(WCN36XX_DBG_DXE_DUMP, "DESC2 >>> ",
-			 (char *)desc, sizeof(*desc));
+			 (char *)desc_skb, sizeof(*desc_skb));
 	wcn36xx_dbg_dump(WCN36XX_DBG_DXE_DUMP, "SKB   >>> ",
-			 (char *)ctl->skb->data, ctl->skb->len);
+			 (char *)ctl_skb->skb->data, ctl_skb->skb->len);
 
 	/* Move the head of the ring to the next empty descriptor */
-	 ch->head_blk_ctl = ctl->next;
+	 ch->head_blk_ctl = ctl_skb->next;
+
+	/* Commit all previous writes and set descriptors to VALID */
+	wmb();
+	desc_skb->ctrl = ch->ctrl_skb;
+	wmb();
+	desc_bd->ctrl = ch->ctrl_bd;
 
 	/*
 	 * When connected and trying to send data frame chip can be in sleep
diff --git a/drivers/net/wireless/ath/wcn36xx/dxe.h b/drivers/net/wireless/ath/wcn36xx/dxe.h
index ce580960d109..31b81b7547a3 100644
--- a/drivers/net/wireless/ath/wcn36xx/dxe.h
+++ b/drivers/net/wireless/ath/wcn36xx/dxe.h
@@ -422,7 +422,6 @@ struct wcn36xx_dxe_ctl {
 	unsigned int		desc_phy_addr;
 	int			ctl_blk_order;
 	struct sk_buff		*skb;
-	spinlock_t              skb_lock;
 	void			*bd_cpu_addr;
 	dma_addr_t		bd_phy_addr;
 };
diff --git a/drivers/net/wireless/ath/wcn36xx/hal.h b/drivers/net/wireless/ath/wcn36xx/hal.h
index 182963522941..8abda2760e04 100644
--- a/drivers/net/wireless/ath/wcn36xx/hal.h
+++ b/drivers/net/wireless/ath/wcn36xx/hal.h
@@ -88,6 +88,12 @@
 /* version string max length (including NULL) */
 #define WCN36XX_HAL_VERSION_LENGTH  64
 
+/* How many frames until we start a-mpdu TX session */
+#define WCN36XX_AMPDU_START_THRESH	20
+
+#define WCN36XX_MAX_SCAN_SSIDS		9
+#define WCN36XX_MAX_SCAN_IE_LEN		500
+
 /* message types for messages exchanged between WDI and HAL */
 enum wcn36xx_hal_host_msg_type {
 	/* Init/De-Init */
@@ -1170,7 +1176,7 @@ struct wcn36xx_hal_start_scan_offload_req_msg {
 
 	/* IE field */
 	u16 ie_len;
-	u8 ie[0];
+	u8 ie[WCN36XX_MAX_SCAN_IE_LEN];
 } __packed;
 
 struct wcn36xx_hal_start_scan_offload_rsp_msg {
@@ -2230,6 +2236,22 @@ struct wcn36xx_hal_switch_channel_rsp_msg {
 
 } __packed;
 
+struct wcn36xx_hal_process_ptt_msg_req_msg {
+	struct wcn36xx_hal_msg_header header;
+
+	/* Actual FTM Command body */
+	u8 ptt_msg[0];
+} __packed;
+
+struct wcn36xx_hal_process_ptt_msg_rsp_msg {
+	struct wcn36xx_hal_msg_header header;
+
+	/* FTM Command response status */
+	u32 ptt_msg_resp_status;
+	/* Actual FTM Command body */
+	u8 ptt_msg[0];
+} __packed;
+
 struct update_edca_params_req_msg {
 	struct wcn36xx_hal_msg_header header;
 
diff --git a/drivers/net/wireless/ath/wcn36xx/main.c b/drivers/net/wireless/ath/wcn36xx/main.c
index 69d6be59d97f..aeb5e6e806be 100644
--- a/drivers/net/wireless/ath/wcn36xx/main.c
+++ b/drivers/net/wireless/ath/wcn36xx/main.c
@@ -26,6 +26,7 @@
 #include <linux/soc/qcom/smem_state.h>
 #include <linux/soc/qcom/wcnss_ctrl.h>
 #include "wcn36xx.h"
+#include "testmode.h"
 
 unsigned int wcn36xx_dbg_mask;
 module_param_named(debug_mask, wcn36xx_dbg_mask, uint, 0644);
@@ -353,6 +354,19 @@ static void wcn36xx_stop(struct ieee80211_hw *hw)
 
 	wcn36xx_dbg(WCN36XX_DBG_MAC, "mac stop\n");
 
+	cancel_work_sync(&wcn->scan_work);
+
+	mutex_lock(&wcn->scan_lock);
+	if (wcn->scan_req) {
+		struct cfg80211_scan_info scan_info = {
+			.aborted = true,
+		};
+
+		ieee80211_scan_completed(wcn->hw, &scan_info);
+	}
+	wcn->scan_req = NULL;
+	mutex_unlock(&wcn->scan_lock);
+
 	wcn36xx_debugfs_exit(wcn);
 	wcn36xx_smd_stop(wcn);
 	wcn36xx_dxe_deinit(wcn);
@@ -549,6 +563,7 @@ static int wcn36xx_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 		} else {
 			wcn36xx_smd_set_bsskey(wcn,
 				vif_priv->encrypt_type,
+				vif_priv->bss_index,
 				key_conf->keyidx,
 				key_conf->keylen,
 				key);
@@ -566,10 +581,13 @@ static int wcn36xx_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 		break;
 	case DISABLE_KEY:
 		if (!(IEEE80211_KEY_FLAG_PAIRWISE & key_conf->flags)) {
+			if (vif_priv->bss_index != WCN36XX_HAL_BSS_INVALID_IDX)
+				wcn36xx_smd_remove_bsskey(wcn,
+					vif_priv->encrypt_type,
+					vif_priv->bss_index,
+					key_conf->keyidx);
+
 			vif_priv->encrypt_type = WCN36XX_HAL_ED_NONE;
-			wcn36xx_smd_remove_bsskey(wcn,
-				vif_priv->encrypt_type,
-				key_conf->keyidx);
 		} else {
 			sta_priv->is_data_encrypted = false;
 			/* do not remove key if disassociated */
@@ -670,10 +688,18 @@ static void wcn36xx_cancel_hw_scan(struct ieee80211_hw *hw,
 	wcn->scan_aborted = true;
 	mutex_unlock(&wcn->scan_lock);
 
-	/* ieee80211_scan_completed will be called on FW scan indication */
-	wcn36xx_smd_stop_hw_scan(wcn);
-
-	cancel_work_sync(&wcn->scan_work);
+	if (get_feat_caps(wcn->fw_feat_caps, SCAN_OFFLOAD)) {
+		/* ieee80211_scan_completed will be called on FW scan
+		 * indication */
+		wcn36xx_smd_stop_hw_scan(wcn);
+	} else {
+		struct cfg80211_scan_info scan_info = {
+			.aborted = true,
+		};
+
+		cancel_work_sync(&wcn->scan_work);
+		ieee80211_scan_completed(wcn->hw, &scan_info);
+	}
 }
 
 static void wcn36xx_update_allowed_rates(struct ieee80211_sta *sta,
@@ -773,6 +799,8 @@ static void wcn36xx_bss_info_changed(struct ieee80211_hw *hw,
 		if (!is_zero_ether_addr(bss_conf->bssid)) {
 			vif_priv->is_joining = true;
 			vif_priv->bss_index = WCN36XX_HAL_BSS_INVALID_IDX;
+			wcn36xx_smd_set_link_st(wcn, bss_conf->bssid, vif->addr,
+						WCN36XX_HAL_LINK_PREASSOC_STATE);
 			wcn36xx_smd_join(wcn, bss_conf->bssid,
 					 vif->addr, WCN36XX_HW_CHANNEL(wcn));
 			wcn36xx_smd_config_bss(wcn, vif, NULL,
@@ -780,6 +808,8 @@ static void wcn36xx_bss_info_changed(struct ieee80211_hw *hw,
 		} else {
 			vif_priv->is_joining = false;
 			wcn36xx_smd_delete_bss(wcn, vif);
+			wcn36xx_smd_set_link_st(wcn, bss_conf->bssid, vif->addr,
+						WCN36XX_HAL_LINK_IDLE_STATE);
 			vif_priv->encrypt_type = WCN36XX_HAL_ED_NONE;
 		}
 	}
@@ -953,6 +983,7 @@ static int wcn36xx_add_interface(struct ieee80211_hw *hw,
 
 	mutex_lock(&wcn->conf_mutex);
 
+	vif_priv->bss_index = WCN36XX_HAL_BSS_INVALID_IDX;
 	list_add(&vif_priv->list, &wcn->vif_list);
 	wcn36xx_smd_add_sta_self(wcn, vif);
 
@@ -1116,6 +1147,8 @@ static const struct ieee80211_ops wcn36xx_ops = {
 	.sta_add		= wcn36xx_sta_add,
 	.sta_remove		= wcn36xx_sta_remove,
 	.ampdu_action		= wcn36xx_ampdu_action,
+
+	CFG80211_TESTMODE_CMD(wcn36xx_tm_cmd)
 };
 
 static int wcn36xx_init_ieee80211(struct wcn36xx *wcn)
@@ -1283,6 +1316,12 @@ static int wcn36xx_probe(struct platform_device *pdev)
 	mutex_init(&wcn->hal_mutex);
 	mutex_init(&wcn->scan_lock);
 
+	ret = dma_set_mask_and_coherent(wcn->dev, DMA_BIT_MASK(32));
+	if (ret < 0) {
+		wcn36xx_err("failed to set DMA mask: %d\n", ret);
+		goto out_wq;
+	}
+
 	INIT_WORK(&wcn->scan_work, wcn36xx_hw_scan_worker);
 
 	wcn->smd_channel = qcom_wcnss_open_channel(wcnss, "WLAN_CTRL", wcn36xx_smd_rsp_process, hw);
diff --git a/drivers/net/wireless/ath/wcn36xx/smd.c b/drivers/net/wireless/ath/wcn36xx/smd.c
index 8932af5e4d8d..b4dadf75d565 100644
--- a/drivers/net/wireless/ath/wcn36xx/smd.c
+++ b/drivers/net/wireless/ath/wcn36xx/smd.c
@@ -252,23 +252,29 @@ static int wcn36xx_smd_send_and_wait(struct wcn36xx *wcn, size_t len)
 {
 	int ret = 0;
 	unsigned long start;
+	struct wcn36xx_hal_msg_header *hdr =
+		(struct wcn36xx_hal_msg_header *)wcn->hal_buf;
+	u16 req_type = hdr->msg_type;
+
 	wcn36xx_dbg_dump(WCN36XX_DBG_SMD_DUMP, "HAL >>> ", wcn->hal_buf, len);
 
 	init_completion(&wcn->hal_rsp_compl);
 	start = jiffies;
 	ret = rpmsg_send(wcn->smd_channel, wcn->hal_buf, len);
 	if (ret) {
-		wcn36xx_err("HAL TX failed\n");
+		wcn36xx_err("HAL TX failed for req %d\n", req_type);
 		goto out;
 	}
 	if (wait_for_completion_timeout(&wcn->hal_rsp_compl,
 		msecs_to_jiffies(HAL_MSG_TIMEOUT)) <= 0) {
-		wcn36xx_err("Timeout! No SMD response in %dms\n",
-			    HAL_MSG_TIMEOUT);
+		wcn36xx_err("Timeout! No SMD response to req %d in %dms\n",
+			    req_type, HAL_MSG_TIMEOUT);
 		ret = -ETIME;
 		goto out;
 	}
-	wcn36xx_dbg(WCN36XX_DBG_SMD, "SMD command completed in %dms",
+	wcn36xx_dbg(WCN36XX_DBG_SMD,
+		    "SMD command (req %d, rsp %d) completed in %dms\n",
+		    req_type, hdr->msg_type,
 		    jiffies_to_msecs(jiffies - start));
 out:
 	return ret;
@@ -292,12 +298,26 @@ static void init_hal_msg(struct wcn36xx_hal_msg_header *hdr,
 		msg_body.header.len = sizeof(msg_body);			\
 	} while (0)							\
 
+#define INIT_HAL_PTT_MSG(p_msg_body, ppt_msg_len) \
+	do { \
+		memset(p_msg_body, 0, sizeof(*p_msg_body) + ppt_msg_len); \
+		p_msg_body->header.msg_type = WCN36XX_HAL_PROCESS_PTT_REQ; \
+		p_msg_body->header.msg_version = WCN36XX_HAL_MSG_VERSION0; \
+		p_msg_body->header.len = sizeof(*p_msg_body) + ppt_msg_len; \
+	} while (0)
+
 #define PREPARE_HAL_BUF(send_buf, msg_body) \
 	do {							\
 		memset(send_buf, 0, msg_body.header.len);	\
 		memcpy(send_buf, &msg_body, sizeof(msg_body));	\
 	} while (0)						\
 
+#define PREPARE_HAL_PTT_MSG_BUF(send_buf, p_msg_body) \
+	do {							\
+		memset(send_buf, 0, p_msg_body->header.len); \
+		memcpy(send_buf, p_msg_body, p_msg_body->header.len); \
+	} while (0)
+
 static int wcn36xx_smd_rsp_status_check(void *buf, size_t len)
 {
 	struct wcn36xx_fw_msg_status_rsp *rsp;
@@ -620,9 +640,13 @@ int wcn36xx_smd_finish_scan(struct wcn36xx *wcn,
 int wcn36xx_smd_start_hw_scan(struct wcn36xx *wcn, struct ieee80211_vif *vif,
 			      struct cfg80211_scan_request *req)
 {
+	struct wcn36xx_vif *vif_priv = wcn36xx_vif_to_priv(vif);
 	struct wcn36xx_hal_start_scan_offload_req_msg msg_body;
 	int ret, i;
 
+	if (req->ie_len > WCN36XX_MAX_SCAN_IE_LEN)
+		return -EINVAL;
+
 	mutex_lock(&wcn->hal_mutex);
 	INIT_HAL_MSG(msg_body, WCN36XX_HAL_START_SCAN_OFFLOAD_REQ);
 
@@ -631,6 +655,7 @@ int wcn36xx_smd_start_hw_scan(struct wcn36xx *wcn, struct ieee80211_vif *vif,
 	msg_body.max_ch_time = 100;
 	msg_body.scan_hidden = 1;
 	memcpy(msg_body.mac, vif->addr, ETH_ALEN);
+	msg_body.bss_type = vif_priv->bss_type;
 	msg_body.p2p_search = vif->p2p;
 
 	msg_body.num_ssid = min_t(u8, req->n_ssids, ARRAY_SIZE(msg_body.ssids));
@@ -646,6 +671,14 @@ int wcn36xx_smd_start_hw_scan(struct wcn36xx *wcn, struct ieee80211_vif *vif,
 	for (i = 0; i < msg_body.num_channel; i++)
 		msg_body.channels[i] = req->channels[i]->hw_value;
 
+	msg_body.header.len -= WCN36XX_MAX_SCAN_IE_LEN;
+
+	if (req->ie_len > 0) {
+		msg_body.ie_len = req->ie_len;
+		msg_body.header.len += req->ie_len;
+		memcpy(msg_body.ie, req->ie, req->ie_len);
+	}
+
 	PREPARE_HAL_BUF(wcn->hal_buf, msg_body);
 
 	wcn36xx_dbg(WCN36XX_DBG_HAL,
@@ -741,6 +774,71 @@ int wcn36xx_smd_switch_channel(struct wcn36xx *wcn,
 	return ret;
 }
 
+static int wcn36xx_smd_process_ptt_msg_rsp(void *buf, size_t len,
+					   void **p_ptt_rsp_msg)
+{
+	struct wcn36xx_hal_process_ptt_msg_rsp_msg *rsp;
+	int ret;
+
+	ret = wcn36xx_smd_rsp_status_check(buf, len);
+	if (ret)
+		return ret;
+
+	rsp = (struct wcn36xx_hal_process_ptt_msg_rsp_msg *)buf;
+
+	wcn36xx_dbg(WCN36XX_DBG_HAL, "process ptt msg responded with length %d\n",
+		    rsp->header.len);
+	wcn36xx_dbg_dump(WCN36XX_DBG_HAL_DUMP, "HAL_PTT_MSG_RSP:", rsp->ptt_msg,
+			 rsp->header.len - sizeof(rsp->ptt_msg_resp_status));
+
+	if (rsp->header.len > 0) {
+		*p_ptt_rsp_msg = kmalloc(rsp->header.len, GFP_ATOMIC);
+		if (!*p_ptt_rsp_msg)
+			return -ENOMEM;
+		memcpy(*p_ptt_rsp_msg, rsp->ptt_msg, rsp->header.len);
+	}
+	return ret;
+}
+
+int wcn36xx_smd_process_ptt_msg(struct wcn36xx *wcn,
+				struct ieee80211_vif *vif, void *ptt_msg, size_t len,
+		void **ptt_rsp_msg)
+{
+	struct wcn36xx_hal_process_ptt_msg_req_msg *p_msg_body;
+	int ret;
+
+	mutex_lock(&wcn->hal_mutex);
+	p_msg_body = kmalloc(
+		sizeof(struct wcn36xx_hal_process_ptt_msg_req_msg) + len,
+		GFP_ATOMIC);
+	if (!p_msg_body) {
+		ret = -ENOMEM;
+		goto out_nomem;
+	}
+	INIT_HAL_PTT_MSG(p_msg_body, len);
+
+	memcpy(&p_msg_body->ptt_msg, ptt_msg, len);
+
+	PREPARE_HAL_PTT_MSG_BUF(wcn->hal_buf, p_msg_body);
+
+	ret = wcn36xx_smd_send_and_wait(wcn, p_msg_body->header.len);
+	if (ret) {
+		wcn36xx_err("Sending hal_process_ptt_msg failed\n");
+		goto out;
+	}
+	ret = wcn36xx_smd_process_ptt_msg_rsp(wcn->hal_buf, wcn->hal_rsp_len,
+					      ptt_rsp_msg);
+	if (ret) {
+		wcn36xx_err("process_ptt_msg response failed err=%d\n", ret);
+		goto out;
+	}
+out:
+	kfree(p_msg_body);
+out_nomem:
+	mutex_unlock(&wcn->hal_mutex);
+	return ret;
+}
+
 static int wcn36xx_smd_update_scan_params_rsp(void *buf, size_t len)
 {
 	struct wcn36xx_hal_update_scan_params_resp *rsp;
@@ -1399,9 +1497,10 @@ int wcn36xx_smd_config_bss(struct wcn36xx *wcn, struct ieee80211_vif *vif,
 	bss->spectrum_mgt_enable = 0;
 	bss->tx_mgmt_power = 0;
 	bss->max_tx_power = WCN36XX_MAX_POWER(wcn);
-
 	bss->action = update;
 
+	vif_priv->bss_type = bss->bss_type;
+
 	wcn36xx_dbg(WCN36XX_DBG_HAL,
 		    "hal config bss bssid %pM self_mac_addr %pM bss_type %d oper_mode %d nw_type %d\n",
 		    bss->bssid, bss->self_mac_addr, bss->bss_type,
@@ -1446,6 +1545,10 @@ int wcn36xx_smd_delete_bss(struct wcn36xx *wcn, struct ieee80211_vif *vif)
 	int ret = 0;
 
 	mutex_lock(&wcn->hal_mutex);
+
+	if (vif_priv->bss_index == WCN36XX_HAL_BSS_INVALID_IDX)
+		goto out;
+
 	INIT_HAL_MSG(msg_body, WCN36XX_HAL_DELETE_BSS_REQ);
 
 	msg_body.bss_index = vif_priv->bss_index;
@@ -1464,6 +1567,8 @@ int wcn36xx_smd_delete_bss(struct wcn36xx *wcn, struct ieee80211_vif *vif)
 		wcn36xx_err("hal_delete_bss response failed err=%d\n", ret);
 		goto out;
 	}
+
+	vif_priv->bss_index = WCN36XX_HAL_BSS_INVALID_IDX;
 out:
 	mutex_unlock(&wcn->hal_mutex);
 	return ret;
@@ -1630,6 +1735,7 @@ int wcn36xx_smd_set_stakey(struct wcn36xx *wcn,
 
 int wcn36xx_smd_set_bsskey(struct wcn36xx *wcn,
 			   enum ani_ed_type enc_type,
+			   u8 bssidx,
 			   u8 keyidx,
 			   u8 keylen,
 			   u8 *key)
@@ -1639,7 +1745,7 @@ int wcn36xx_smd_set_bsskey(struct wcn36xx *wcn,
 
 	mutex_lock(&wcn->hal_mutex);
 	INIT_HAL_MSG(msg_body, WCN36XX_HAL_SET_BSSKEY_REQ);
-	msg_body.bss_idx = 0;
+	msg_body.bss_idx = bssidx;
 	msg_body.enc_type = enc_type;
 	msg_body.num_keys = 1;
 	msg_body.keys[0].id = keyidx;
@@ -1700,6 +1806,7 @@ int wcn36xx_smd_remove_stakey(struct wcn36xx *wcn,
 
 int wcn36xx_smd_remove_bsskey(struct wcn36xx *wcn,
 			      enum ani_ed_type enc_type,
+			      u8 bssidx,
 			      u8 keyidx)
 {
 	struct wcn36xx_hal_remove_bss_key_req_msg msg_body;
@@ -1707,7 +1814,7 @@ int wcn36xx_smd_remove_bsskey(struct wcn36xx *wcn,
 
 	mutex_lock(&wcn->hal_mutex);
 	INIT_HAL_MSG(msg_body, WCN36XX_HAL_RMV_BSSKEY_REQ);
-	msg_body.bss_idx = 0;
+	msg_body.bss_idx = bssidx;
 	msg_body.enc_type = enc_type;
 	msg_body.key_id = keyidx;
 
@@ -2132,11 +2239,13 @@ static int wcn36xx_smd_hw_scan_ind(struct wcn36xx *wcn, void *buf, size_t len)
 		return -EIO;
 	}
 
-	wcn36xx_dbg(WCN36XX_DBG_HAL, "scan indication (type %x)", rsp->type);
+	wcn36xx_dbg(WCN36XX_DBG_HAL, "scan indication (type %x)\n", rsp->type);
 
 	switch (rsp->type) {
 	case WCN36XX_HAL_SCAN_IND_FAILED:
+	case WCN36XX_HAL_SCAN_IND_DEQUEUED:
 		scan_info.aborted = true;
+		/* fall through */
 	case WCN36XX_HAL_SCAN_IND_COMPLETED:
 		mutex_lock(&wcn->scan_lock);
 		wcn->scan_req = NULL;
@@ -2147,7 +2256,6 @@ static int wcn36xx_smd_hw_scan_ind(struct wcn36xx *wcn, void *buf, size_t len)
 		break;
 	case WCN36XX_HAL_SCAN_IND_STARTED:
 	case WCN36XX_HAL_SCAN_IND_FOREIGN_CHANNEL:
-	case WCN36XX_HAL_SCAN_IND_DEQUEUED:
 	case WCN36XX_HAL_SCAN_IND_PREEMPTED:
 	case WCN36XX_HAL_SCAN_IND_RESTARTED:
 		break;
@@ -2367,6 +2475,7 @@ int wcn36xx_smd_rsp_process(struct rpmsg_device *rpdev,
 	case WCN36XX_HAL_JOIN_RSP:
 	case WCN36XX_HAL_UPDATE_SCAN_PARAM_RSP:
 	case WCN36XX_HAL_CH_SWITCH_RSP:
+	case WCN36XX_HAL_PROCESS_PTT_RSP:
 	case WCN36XX_HAL_FEATURE_CAPS_EXCHANGE_RSP:
 	case WCN36XX_HAL_8023_MULTICAST_LIST_RSP:
 	case WCN36XX_HAL_START_SCAN_OFFLOAD_RSP:
@@ -2407,6 +2516,7 @@ int wcn36xx_smd_rsp_process(struct rpmsg_device *rpdev,
 
 	return 0;
 }
+
 static void wcn36xx_ind_smd_work(struct work_struct *work)
 {
 	struct wcn36xx *wcn =
@@ -2471,24 +2581,24 @@ static void wcn36xx_ind_smd_work(struct work_struct *work)
 }
 int wcn36xx_smd_open(struct wcn36xx *wcn)
 {
-	int ret = 0;
 	wcn->hal_ind_wq = create_freezable_workqueue("wcn36xx_smd_ind");
-	if (!wcn->hal_ind_wq) {
-		wcn36xx_err("failed to allocate wq\n");
-		ret = -ENOMEM;
-		goto out;
-	}
+	if (!wcn->hal_ind_wq)
+		return -ENOMEM;
+
 	INIT_WORK(&wcn->hal_ind_work, wcn36xx_ind_smd_work);
 	INIT_LIST_HEAD(&wcn->hal_ind_queue);
 	spin_lock_init(&wcn->hal_ind_lock);
 
 	return 0;
-
-out:
-	return ret;
 }
 
 void wcn36xx_smd_close(struct wcn36xx *wcn)
 {
+	struct wcn36xx_hal_ind_msg *msg, *tmp;
+
+	cancel_work_sync(&wcn->hal_ind_work);
 	destroy_workqueue(wcn->hal_ind_wq);
+
+	list_for_each_entry_safe(msg, tmp, &wcn->hal_ind_queue, list)
+		kfree(msg);
 }
diff --git a/drivers/net/wireless/ath/wcn36xx/smd.h b/drivers/net/wireless/ath/wcn36xx/smd.h
index 8076edf40ac8..ff15df8ab56f 100644
--- a/drivers/net/wireless/ath/wcn36xx/smd.h
+++ b/drivers/net/wireless/ath/wcn36xx/smd.h
@@ -86,6 +86,10 @@ int wcn36xx_smd_send_beacon(struct wcn36xx *wcn, struct ieee80211_vif *vif,
 			    u16 p2p_off);
 int wcn36xx_smd_switch_channel(struct wcn36xx *wcn,
 			       struct ieee80211_vif *vif, int ch);
+int wcn36xx_smd_process_ptt_msg(struct wcn36xx *wcn,
+				struct ieee80211_vif *vif,
+				void *ptt_msg, size_t len,
+				void **ptt_rsp_msg);
 int wcn36xx_smd_update_proberesp_tmpl(struct wcn36xx *wcn,
 				      struct ieee80211_vif *vif,
 				      struct sk_buff *skb);
@@ -97,6 +101,7 @@ int wcn36xx_smd_set_stakey(struct wcn36xx *wcn,
 			   u8 sta_index);
 int wcn36xx_smd_set_bsskey(struct wcn36xx *wcn,
 			   enum ani_ed_type enc_type,
+			   u8 bssidx,
 			   u8 keyidx,
 			   u8 keylen,
 			   u8 *key);
@@ -106,6 +111,7 @@ int wcn36xx_smd_remove_stakey(struct wcn36xx *wcn,
 			      u8 sta_index);
 int wcn36xx_smd_remove_bsskey(struct wcn36xx *wcn,
 			      enum ani_ed_type enc_type,
+			      u8 bssidx,
 			      u8 keyidx);
 int wcn36xx_smd_enter_bmps(struct wcn36xx *wcn, struct ieee80211_vif *vif);
 int wcn36xx_smd_exit_bmps(struct wcn36xx *wcn, struct ieee80211_vif *vif);
diff --git a/drivers/net/wireless/ath/wcn36xx/testmode.c b/drivers/net/wireless/ath/wcn36xx/testmode.c
new file mode 100644
index 000000000000..1279064a3b71
--- /dev/null
+++ b/drivers/net/wireless/ath/wcn36xx/testmode.c
@@ -0,0 +1,149 @@
+﻿/*
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <net/netlink.h>
+#include <linux/firmware.h>
+#include <net/cfg80211.h>
+#include "wcn36xx.h"
+
+#include "testmode.h"
+#include "testmode_i.h"
+#include "hal.h"
+#include "smd.h"
+
+static const struct nla_policy wcn36xx_tm_policy[WCN36XX_TM_ATTR_MAX + 1] = {
+	[WCN36XX_TM_ATTR_CMD] = { .type = NLA_U16 },
+	[WCN36XX_TM_ATTR_DATA] = { .type = NLA_BINARY,
+	.len = WCN36XX_TM_DATA_MAX_LEN },
+};
+
+struct build_release_number {
+	u16 drv_major;
+	u16 drv_minor;
+	u16 drv_patch;
+	u16 drv_build;
+	u16 ptt_max;
+	u16 ptt_min;
+	u16 fw_ver;
+} __packed;
+
+static int wcn36xx_tm_cmd_ptt(struct wcn36xx *wcn, struct ieee80211_vif *vif,
+			      struct nlattr *tb[])
+{
+	int ret = 0, buf_len;
+	void *buf;
+	struct ftm_rsp_msg *msg, *rsp = NULL;
+	struct sk_buff *skb;
+
+	if (!tb[WCN36XX_TM_ATTR_DATA])
+		return -EINVAL;
+
+	buf = nla_data(tb[WCN36XX_TM_ATTR_DATA]);
+	buf_len = nla_len(tb[WCN36XX_TM_ATTR_DATA]);
+	msg = (struct ftm_rsp_msg *)buf;
+
+	wcn36xx_dbg(WCN36XX_DBG_TESTMODE,
+		    "testmode cmd wmi msg_id 0x%04X msg_len %d buf %pK buf_len %d\n",
+		   msg->msg_id, msg->msg_body_length,
+		   buf, buf_len);
+
+	wcn36xx_dbg_dump(WCN36XX_DBG_TESTMODE_DUMP, "REQ ", buf, buf_len);
+
+	if (msg->msg_id == MSG_GET_BUILD_RELEASE_NUMBER) {
+		struct build_release_number *body =
+				(struct build_release_number *)
+				msg->msg_response;
+
+		body->drv_major = wcn->fw_major;
+		body->drv_minor = wcn->fw_minor;
+		body->drv_patch = wcn->fw_version;
+		body->drv_build = wcn->fw_revision;
+		body->ptt_max = 10;
+		body->ptt_min = 0;
+
+		rsp = msg;
+		rsp->resp_status = 0;
+	} else {
+		wcn36xx_dbg(WCN36XX_DBG_TESTMODE,
+			    "PPT Request >> HAL size %d\n",
+				msg->msg_body_length);
+
+		msg->resp_status = wcn36xx_smd_process_ptt_msg(wcn, vif, msg,
+							       msg->msg_body_length, (void *)(&rsp));
+
+		wcn36xx_dbg(WCN36XX_DBG_TESTMODE,
+			    "Response status = %d\n",
+				msg->resp_status);
+		if (rsp)
+			wcn36xx_dbg(WCN36XX_DBG_TESTMODE,
+				    "PPT Response << HAL size %d\n",
+					rsp->msg_body_length);
+	}
+
+	if (!rsp) {
+		rsp = msg;
+		wcn36xx_warn("No response! Echoing request with response status %d\n",
+			     rsp->resp_status);
+	}
+	wcn36xx_dbg_dump(WCN36XX_DBG_TESTMODE_DUMP, "RSP ",
+			 rsp, rsp->msg_body_length);
+
+	skb = cfg80211_testmode_alloc_reply_skb(wcn->hw->wiphy,
+						nla_total_size(msg->msg_body_length));
+	if (!skb) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = nla_put(skb, WCN36XX_TM_ATTR_DATA, rsp->msg_body_length, rsp);
+	if (ret) {
+		kfree_skb(skb);
+		goto out;
+	}
+
+	ret = cfg80211_testmode_reply(skb);
+
+out:
+	if (rsp != msg)
+		kfree(rsp);
+
+	return ret;
+}
+
+int wcn36xx_tm_cmd(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		   void *data, int len)
+{
+	struct wcn36xx *wcn = hw->priv;
+	struct nlattr *tb[WCN36XX_TM_ATTR_MAX + 1];
+	int ret = 0;
+	unsigned short attr;
+
+	wcn36xx_dbg_dump(WCN36XX_DBG_TESTMODE_DUMP, "Data:", data, len);
+	ret = nla_parse(tb, WCN36XX_TM_ATTR_MAX, data, len,
+			wcn36xx_tm_policy, NULL);
+	if (ret)
+		return ret;
+
+	if (!tb[WCN36XX_TM_ATTR_CMD])
+		return -EINVAL;
+
+	attr = nla_get_u16(tb[WCN36XX_TM_ATTR_CMD]);
+
+	if (attr != WCN36XX_TM_CMD_PTT)
+		return -EOPNOTSUPP;
+
+	return wcn36xx_tm_cmd_ptt(wcn, vif, tb);
+}
diff --git a/drivers/net/wireless/ath/wcn36xx/testmode.h b/drivers/net/wireless/ath/wcn36xx/testmode.h
new file mode 100644
index 000000000000..4c6cfdb46580
--- /dev/null
+++ b/drivers/net/wireless/ath/wcn36xx/testmode.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "wcn36xx.h"
+
+struct ftm_rsp_msg {
+	u16 msg_id;
+	u16 msg_body_length;
+	u32 resp_status;
+	u8 msg_response[0];
+} __packed;
+
+/* The request buffer of FTM which contains a byte of command and the request */
+struct ftm_payload {
+	u16 ftm_cmd_type;
+	struct ftm_rsp_msg ftm_cmd_msg;
+} __packed;
+
+#define MSG_GET_BUILD_RELEASE_NUMBER 0x32A2
+
+#ifdef CONFIG_NL80211_TESTMODE
+int wcn36xx_tm_cmd(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		   void *data, int len);
+
+#else
+static inline int wcn36xx_tm_cmd(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				void *data, int len)
+{
+	return 0;
+}
+
+#endif
diff --git a/drivers/net/wireless/ath/wcn36xx/testmode_i.h b/drivers/net/wireless/ath/wcn36xx/testmode_i.h
new file mode 100644
index 000000000000..8a1477ffd5a0
--- /dev/null
+++ b/drivers/net/wireless/ath/wcn36xx/testmode_i.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#define WCN36XX_TM_DATA_MAX_LEN		5000
+
+enum wcn36xx_tm_attr {
+	__WCN36XX_TM_ATTR_INVALID	= 0,
+	WCN36XX_TM_ATTR_CMD		= 1,
+	WCN36XX_TM_ATTR_DATA		= 2,
+
+	/* keep last */
+	__WCN36XX_TM_ATTR_AFTER_LAST,
+	WCN36XX_TM_ATTR_MAX		= __WCN36XX_TM_ATTR_AFTER_LAST - 1,
+};
+
+#define WCN36XX_TM_CMD_PTT 3
diff --git a/drivers/net/wireless/ath/wcn36xx/txrx.c b/drivers/net/wireless/ath/wcn36xx/txrx.c
index b1768ed6b0be..a6902371e89c 100644
--- a/drivers/net/wireless/ath/wcn36xx/txrx.c
+++ b/drivers/net/wireless/ath/wcn36xx/txrx.c
@@ -273,6 +273,7 @@ int wcn36xx_start_tx(struct wcn36xx *wcn,
 	bool bcast = is_broadcast_ether_addr(hdr->addr1) ||
 		is_multicast_ether_addr(hdr->addr1);
 	struct wcn36xx_tx_bd bd;
+	int ret;
 
 	memset(&bd, 0, sizeof(bd));
 
@@ -317,5 +318,17 @@ int wcn36xx_start_tx(struct wcn36xx *wcn,
 	buff_to_be((u32 *)&bd, sizeof(bd)/sizeof(u32));
 	bd.tx_bd_sign = 0xbdbdbdbd;
 
-	return wcn36xx_dxe_tx_frame(wcn, vif_priv, &bd, skb, is_low);
+	ret = wcn36xx_dxe_tx_frame(wcn, vif_priv, &bd, skb, is_low);
+	if (ret && bd.tx_comp) {
+		/* If the skb has not been transmitted,
+		 * don't keep a reference to it.
+		 */
+		spin_lock_irqsave(&wcn->dxe_lock, flags);
+		wcn->tx_ack_skb = NULL;
+		spin_unlock_irqrestore(&wcn->dxe_lock, flags);
+
+		ieee80211_wake_queues(wcn->hw);
+	}
+
+	return ret;
 }
diff --git a/drivers/net/wireless/ath/wcn36xx/wcn36xx.h b/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
index 5854adf43f3a..11e74015c79a 100644
--- a/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
+++ b/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
@@ -32,12 +32,6 @@
 #define WLAN_NV_FILE               "wlan/prima/WCNSS_qcom_wlan_nv.bin"
 #define WCN36XX_AGGR_BUFFER_SIZE 64
 
-/* How many frames until we start a-mpdu TX session */
-#define WCN36XX_AMPDU_START_THRESH	20
-
-#define WCN36XX_MAX_SCAN_SSIDS		9
-#define WCN36XX_MAX_SCAN_IE_LEN		500
-
 extern unsigned int wcn36xx_dbg_mask;
 
 enum wcn36xx_debug_mask {
@@ -56,6 +50,8 @@ enum wcn36xx_debug_mask {
 	WCN36XX_DBG_BEACON_DUMP	= 0x00001000,
 	WCN36XX_DBG_PMC		= 0x00002000,
 	WCN36XX_DBG_PMC_DUMP	= 0x00004000,
+	WCN36XX_DBG_TESTMODE		= 0x00008000,
+	WCN36XX_DBG_TESTMODE_DUMP	= 0x00010000,
 	WCN36XX_DBG_ANY		= 0xffffffff,
 };
 
@@ -123,6 +119,7 @@ struct wcn36xx_vif {
 	bool is_joining;
 	bool sta_assoc;
 	struct wcn36xx_hal_mac_ssid ssid;
+	enum wcn36xx_hal_bss_type bss_type;
 
 	/* Power management */
 	enum wcn36xx_power_state pw_state;
diff --git a/drivers/net/wireless/ath/wil6210/Kconfig b/drivers/net/wireless/ath/wil6210/Kconfig
index b448926b0c0f..3548e8d5e18e 100644
--- a/drivers/net/wireless/ath/wil6210/Kconfig
+++ b/drivers/net/wireless/ath/wil6210/Kconfig
@@ -33,7 +33,7 @@ config WIL6210_TRACING
 	bool "wil6210 tracing support"
 	depends on WIL6210
 	depends on EVENT_TRACING
-	default y
+	default n
 	---help---
 	  Say Y here to enable tracepoints for the wil6210 driver
 	  using the kernel tracing infrastructure.  Select this
diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c
index cdbb393863f3..78946f28d0c7 100644
--- a/drivers/net/wireless/ath/wil6210/cfg80211.c
+++ b/drivers/net/wireless/ath/wil6210/cfg80211.c
@@ -276,6 +276,8 @@ int wil_cid_fill_sinfo(struct wil6210_vif *vif, int cid,
 	struct wil_net_stats *stats = &wil->sta[cid].stats;
 	int rc;
 
+	memset(&reply, 0, sizeof(reply));
+
 	rc = wmi_call(wil, WMI_NOTIFY_REQ_CMDID, vif->mid, &cmd, sizeof(cmd),
 		      WMI_NOTIFY_REQ_DONE_EVENTID, &reply, sizeof(reply), 20);
 	if (rc)
@@ -1081,17 +1083,11 @@ int wil_cfg80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 			 u64 *cookie)
 {
 	const u8 *buf = params->buf;
-	size_t len = params->len, total;
+	size_t len = params->len;
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
 	struct wil6210_vif *vif = wdev_to_vif(wil, wdev);
 	int rc;
-	bool tx_status = false;
-	struct ieee80211_mgmt *mgmt_frame = (void *)buf;
-	struct wmi_sw_tx_req_cmd *cmd;
-	struct {
-		struct wmi_cmd_hdr wmi;
-		struct wmi_sw_tx_complete_event evt;
-	} __packed evt;
+	bool tx_status;
 
 	/* Note, currently we do not support the "wait" parameter, user-space
 	 * must call remain_on_channel before mgmt_tx or listen on a channel
@@ -1100,34 +1096,9 @@ int wil_cfg80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 	 * different from currently "listened" channel and fail if it is.
 	 */
 
-	wil_dbg_misc(wil, "mgmt_tx mid %d\n", vif->mid);
-	wil_hex_dump_misc("mgmt tx frame ", DUMP_PREFIX_OFFSET, 16, 1, buf,
-			  len, true);
-
-	if (len < sizeof(struct ieee80211_hdr_3addr))
-		return -EINVAL;
-
-	total = sizeof(*cmd) + len;
-	if (total < len)
-		return -EINVAL;
-
-	cmd = kmalloc(total, GFP_KERNEL);
-	if (!cmd) {
-		rc = -ENOMEM;
-		goto out;
-	}
-
-	memcpy(cmd->dst_mac, mgmt_frame->da, WMI_MAC_LEN);
-	cmd->len = cpu_to_le16(len);
-	memcpy(cmd->payload, buf, len);
-
-	rc = wmi_call(wil, WMI_SW_TX_REQ_CMDID, vif->mid, cmd, total,
-		      WMI_SW_TX_COMPLETE_EVENTID, &evt, sizeof(evt), 2000);
-	if (rc == 0)
-		tx_status = !evt.evt.status;
+	rc = wmi_mgmt_tx(vif, buf, len);
+	tx_status = (rc == 0);
 
-	kfree(cmd);
- out:
 	cfg80211_mgmt_tx_status(wdev, cookie ? *cookie : 0, buf, len,
 				tx_status, GFP_KERNEL);
 	return rc;
@@ -2277,7 +2248,9 @@ static int wil_rf_sector_get_cfg(struct wiphy *wiphy,
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_get_rf_sector_params_done_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_RF_SECTOR_STATUS_NOT_SUPPORTED_ERROR},
+	};
 	struct sk_buff *msg;
 	struct nlattr *nl_cfgs, *nl_cfg;
 	u32 i;
@@ -2323,7 +2296,6 @@ static int wil_rf_sector_get_cfg(struct wiphy *wiphy,
 	cmd.sector_idx = cpu_to_le16(sector_index);
 	cmd.sector_type = sector_type;
 	cmd.rf_modules_vec = rf_modules_vec & 0xFF;
-	memset(&reply, 0, sizeof(reply));
 	rc = wmi_call(wil, WMI_GET_RF_SECTOR_PARAMS_CMDID, vif->mid,
 		      &cmd, sizeof(cmd), WMI_GET_RF_SECTOR_PARAMS_DONE_EVENTID,
 		      &reply, sizeof(reply),
@@ -2398,7 +2370,9 @@ static int wil_rf_sector_set_cfg(struct wiphy *wiphy,
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_set_rf_sector_params_done_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_RF_SECTOR_STATUS_NOT_SUPPORTED_ERROR},
+	};
 	struct nlattr *nl_cfg;
 	struct wmi_rf_sector_info *si;
 
@@ -2481,7 +2455,6 @@ static int wil_rf_sector_set_cfg(struct wiphy *wiphy,
 	}
 
 	cmd.rf_modules_vec = rf_modules_vec & 0xFF;
-	memset(&reply, 0, sizeof(reply));
 	rc = wmi_call(wil, WMI_SET_RF_SECTOR_PARAMS_CMDID, vif->mid,
 		      &cmd, sizeof(cmd), WMI_SET_RF_SECTOR_PARAMS_DONE_EVENTID,
 		      &reply, sizeof(reply),
@@ -2505,7 +2478,9 @@ static int wil_rf_sector_get_selected(struct wiphy *wiphy,
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_get_selected_rf_sector_index_done_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_RF_SECTOR_STATUS_NOT_SUPPORTED_ERROR},
+	};
 	struct sk_buff *msg;
 
 	if (!test_bit(WMI_FW_CAPABILITY_RF_SECTORS, wil->fw_capabilities))
@@ -2545,7 +2520,6 @@ static int wil_rf_sector_get_selected(struct wiphy *wiphy,
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.cid = (u8)cid;
 	cmd.sector_type = sector_type;
-	memset(&reply, 0, sizeof(reply));
 	rc = wmi_call(wil, WMI_GET_SELECTED_RF_SECTOR_INDEX_CMDID, vif->mid,
 		      &cmd, sizeof(cmd),
 		      WMI_GET_SELECTED_RF_SECTOR_INDEX_DONE_EVENTID,
@@ -2586,14 +2560,15 @@ static int wil_rf_sector_wmi_set_selected(struct wil6210_priv *wil,
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_set_selected_rf_sector_index_done_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_RF_SECTOR_STATUS_NOT_SUPPORTED_ERROR},
+	};
 	int rc;
 
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.sector_idx = cpu_to_le16(sector_index);
 	cmd.sector_type = sector_type;
 	cmd.cid = (u8)cid;
-	memset(&reply, 0, sizeof(reply));
 	rc = wmi_call(wil, WMI_SET_SELECTED_RF_SECTOR_INDEX_CMDID, mid,
 		      &cmd, sizeof(cmd),
 		      WMI_SET_SELECTED_RF_SECTOR_INDEX_DONE_EVENTID,
diff --git a/drivers/net/wireless/ath/wil6210/debugfs.c b/drivers/net/wireless/ath/wil6210/debugfs.c
index 8c90b3111f0b..ebfdff4d328c 100644
--- a/drivers/net/wireless/ath/wil6210/debugfs.c
+++ b/drivers/net/wireless/ath/wil6210/debugfs.c
@@ -1078,6 +1078,8 @@ static int wil_bf_debugfs_show(struct seq_file *s, void *data)
 		struct wmi_notify_req_done_event evt;
 	} __packed reply;
 
+	memset(&reply, 0, sizeof(reply));
+
 	for (i = 0; i < ARRAY_SIZE(wil->sta); i++) {
 		u32 status;
 
@@ -1200,8 +1202,12 @@ static const struct file_operations fops_freq = {
 static int wil_link_debugfs_show(struct seq_file *s, void *data)
 {
 	struct wil6210_priv *wil = s->private;
-	struct station_info sinfo;
-	int i, rc;
+	struct station_info *sinfo;
+	int i, rc = 0;
+
+	sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL);
+	if (!sinfo)
+		return -ENOMEM;
 
 	for (i = 0; i < ARRAY_SIZE(wil->sta); i++) {
 		struct wil_sta_info *p = &wil->sta[i];
@@ -1229,19 +1235,21 @@ static int wil_link_debugfs_show(struct seq_file *s, void *data)
 
 		vif = (mid < wil->max_vifs) ? wil->vifs[mid] : NULL;
 		if (vif) {
-			rc = wil_cid_fill_sinfo(vif, i, &sinfo);
+			rc = wil_cid_fill_sinfo(vif, i, sinfo);
 			if (rc)
-				return rc;
+				goto out;
 
-			seq_printf(s, "  Tx_mcs = %d\n", sinfo.txrate.mcs);
-			seq_printf(s, "  Rx_mcs = %d\n", sinfo.rxrate.mcs);
-			seq_printf(s, "  SQ     = %d\n", sinfo.signal);
+			seq_printf(s, "  Tx_mcs = %d\n", sinfo->txrate.mcs);
+			seq_printf(s, "  Rx_mcs = %d\n", sinfo->rxrate.mcs);
+			seq_printf(s, "  SQ     = %d\n", sinfo->signal);
 		} else {
 			seq_puts(s, "  INVALID MID\n");
 		}
 	}
 
-	return 0;
+out:
+	kfree(sinfo);
+	return rc;
 }
 
 static int wil_link_seq_open(struct inode *inode, struct file *file)
@@ -1377,8 +1385,7 @@ static void wil_print_rxtid(struct seq_file *s, struct wil_tid_ampdu_rx *r)
 	u16 index = ((r->head_seq_num - r->ssn) & 0xfff) % r->buf_size;
 	unsigned long long drop_dup = r->drop_dup, drop_old = r->drop_old;
 
-	seq_printf(s, "([%2d] %3d TU) 0x%03x [", r->buf_size, r->timeout,
-		   r->head_seq_num);
+	seq_printf(s, "([%2d]) 0x%03x [", r->buf_size, r->head_seq_num);
 	for (i = 0; i < r->buf_size; i++) {
 		if (i == index)
 			seq_printf(s, "%c", r->reorder_buf[i] ? 'O' : '|');
diff --git a/drivers/net/wireless/ath/wil6210/main.c b/drivers/net/wireless/ath/wil6210/main.c
index a4b413e8d55a..e7006c2428a0 100644
--- a/drivers/net/wireless/ath/wil6210/main.c
+++ b/drivers/net/wireless/ath/wil6210/main.c
@@ -342,6 +342,8 @@ void wil_disconnect_worker(struct work_struct *work)
 		/* already disconnected */
 		return;
 
+	memset(&reply, 0, sizeof(reply));
+
 	rc = wmi_call(wil, WMI_DISCONNECT_CMDID, vif->mid, NULL, 0,
 		      WMI_DISCONNECT_EVENTID, &reply, sizeof(reply),
 		      WIL6210_DISCONNECT_TO_MS);
@@ -391,7 +393,7 @@ static void wil_fw_error_worker(struct work_struct *work)
 	struct wil6210_priv *wil = container_of(work, struct wil6210_priv,
 						fw_error_worker);
 	struct net_device *ndev = wil->main_ndev;
-	struct wireless_dev *wdev = ndev->ieee80211_ptr;
+	struct wireless_dev *wdev;
 
 	wil_dbg_misc(wil, "fw error worker\n");
 
@@ -399,6 +401,7 @@ static void wil_fw_error_worker(struct work_struct *work)
 		wil_info(wil, "No recovery - interface is down\n");
 		return;
 	}
+	wdev = ndev->ieee80211_ptr;
 
 	/* increment @recovery_count if less then WIL6210_FW_RECOVERY_TO
 	 * passed since last recovery attempt
diff --git a/drivers/net/wireless/ath/wil6210/netdev.c b/drivers/net/wireless/ath/wil6210/netdev.c
index 05e9408e7ea3..eb6c14ed65a4 100644
--- a/drivers/net/wireless/ath/wil6210/netdev.c
+++ b/drivers/net/wireless/ath/wil6210/netdev.c
@@ -457,16 +457,16 @@ void wil_vif_remove(struct wil6210_priv *wil, u8 mid)
 		return;
 	}
 
+	mutex_lock(&wil->mutex);
+	wil6210_disconnect(vif, NULL, WLAN_REASON_DEAUTH_LEAVING, false);
+	mutex_unlock(&wil->mutex);
+
 	ndev = vif_to_ndev(vif);
 	/* during unregister_netdevice cfg80211_leave may perform operations
 	 * such as stop AP, disconnect, so we only clear the VIF afterwards
 	 */
 	unregister_netdevice(ndev);
 
-	mutex_lock(&wil->mutex);
-	wil6210_disconnect(vif, NULL, WLAN_REASON_DEAUTH_LEAVING, false);
-	mutex_unlock(&wil->mutex);
-
 	if (any_active && vif->mid != 0)
 		wmi_port_delete(wil, vif->mid);
 
diff --git a/drivers/net/wireless/ath/wil6210/rx_reorder.c b/drivers/net/wireless/ath/wil6210/rx_reorder.c
index 14dcb0698dee..76f8084c1fd8 100644
--- a/drivers/net/wireless/ath/wil6210/rx_reorder.c
+++ b/drivers/net/wireless/ath/wil6210/rx_reorder.c
@@ -206,7 +206,6 @@ __acquires(&sta->tid_rx_lock) __releases(&sta->tid_rx_lock)
 
 	/* put the frame in the reordering buffer */
 	r->reorder_buf[index] = skb;
-	r->reorder_time[index] = jiffies;
 	r->stored_mpdu_num++;
 	wil_reorder_release(ndev, r);
 
@@ -252,11 +251,8 @@ struct wil_tid_ampdu_rx *wil_tid_ampdu_rx_alloc(struct wil6210_priv *wil,
 
 	r->reorder_buf =
 		kcalloc(size, sizeof(struct sk_buff *), GFP_KERNEL);
-	r->reorder_time =
-		kcalloc(size, sizeof(unsigned long), GFP_KERNEL);
-	if (!r->reorder_buf || !r->reorder_time) {
+	if (!r->reorder_buf) {
 		kfree(r->reorder_buf);
-		kfree(r->reorder_time);
 		kfree(r);
 		return NULL;
 	}
@@ -286,7 +282,6 @@ void wil_tid_ampdu_rx_free(struct wil6210_priv *wil,
 		kfree_skb(r->reorder_buf[i]);
 
 	kfree(r->reorder_buf);
-	kfree(r->reorder_time);
 	kfree(r);
 }
 
diff --git a/drivers/net/wireless/ath/wil6210/txrx.c b/drivers/net/wireless/ath/wil6210/txrx.c
index b60b9fcaaebd..b9a9fa828961 100644
--- a/drivers/net/wireless/ath/wil6210/txrx.c
+++ b/drivers/net/wireless/ath/wil6210/txrx.c
@@ -652,8 +652,8 @@ static int wil_rx_refill(struct wil6210_priv *wil, int count)
 			v->swtail = next_tail) {
 		rc = wil_vring_alloc_skb(wil, v, v->swtail, headroom);
 		if (unlikely(rc)) {
-			wil_err(wil, "Error %d in wil_rx_refill[%d]\n",
-				rc, v->swtail);
+			wil_err_ratelimited(wil, "Error %d in rx refill[%d]\n",
+					    rc, v->swtail);
 			break;
 		}
 	}
@@ -963,7 +963,9 @@ int wil_vring_init_tx(struct wil6210_vif *vif, int id, int size,
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_vring_cfg_done_event cmd;
-	} __packed reply;
+	} __packed reply = {
+		.cmd = {.status = WMI_FW_STATUS_FAILURE},
+	};
 	struct vring *vring = &wil->vring_tx[id];
 	struct vring_tx_data *txdata = &wil->vring_tx_data[id];
 
@@ -1045,7 +1047,9 @@ int wil_vring_init_bcast(struct wil6210_vif *vif, int id, int size)
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_vring_cfg_done_event cmd;
-	} __packed reply;
+	} __packed reply = {
+		.cmd = {.status = WMI_FW_STATUS_FAILURE},
+	};
 	struct vring *vring = &wil->vring_tx[id];
 	struct vring_tx_data *txdata = &wil->vring_tx_data[id];
 
diff --git a/drivers/net/wireless/ath/wil6210/wil6210.h b/drivers/net/wireless/ath/wil6210/wil6210.h
index f9c5155025bc..b623510c6f6c 100644
--- a/drivers/net/wireless/ath/wil6210/wil6210.h
+++ b/drivers/net/wireless/ath/wil6210/wil6210.h
@@ -493,38 +493,28 @@ struct pci_dev;
  * struct tid_ampdu_rx - TID aggregation information (Rx).
  *
  * @reorder_buf: buffer to reorder incoming aggregated MPDUs
- * @reorder_time: jiffies when skb was added
- * @session_timer: check if peer keeps Tx-ing on the TID (by timeout value)
- * @reorder_timer: releases expired frames from the reorder buffer.
  * @last_rx: jiffies of last rx activity
  * @head_seq_num: head sequence number in reordering buffer.
  * @stored_mpdu_num: number of MPDUs in reordering buffer
  * @ssn: Starting Sequence Number expected to be aggregated.
  * @buf_size: buffer size for incoming A-MPDUs
- * @timeout: reset timer value (in TUs).
  * @ssn_last_drop: SSN of the last dropped frame
  * @total: total number of processed incoming frames
  * @drop_dup: duplicate frames dropped for this reorder buffer
  * @drop_old: old frames dropped for this reorder buffer
- * @dialog_token: dialog token for aggregation session
  * @first_time: true when this buffer used 1-st time
  */
 struct wil_tid_ampdu_rx {
 	struct sk_buff **reorder_buf;
-	unsigned long *reorder_time;
-	struct timer_list session_timer;
-	struct timer_list reorder_timer;
 	unsigned long last_rx;
 	u16 head_seq_num;
 	u16 stored_mpdu_num;
 	u16 ssn;
 	u16 buf_size;
-	u16 timeout;
 	u16 ssn_last_drop;
 	unsigned long long total; /* frames processed */
 	unsigned long long drop_dup;
 	unsigned long long drop_old;
-	u8 dialog_token;
 	bool first_time; /* is it 1-st time this buffer used? */
 };
 
@@ -986,7 +976,7 @@ int wmi_read_hdr(struct wil6210_priv *wil, __le32 ptr,
 int wmi_send(struct wil6210_priv *wil, u16 cmdid, u8 mid, void *buf, u16 len);
 void wmi_recv_cmd(struct wil6210_priv *wil);
 int wmi_call(struct wil6210_priv *wil, u16 cmdid, u8 mid, void *buf, u16 len,
-	     u16 reply_id, void *reply, u8 reply_size, int to_msec);
+	     u16 reply_id, void *reply, u16 reply_size, int to_msec);
 void wmi_event_worker(struct work_struct *work);
 void wmi_event_flush(struct wil6210_priv *wil);
 int wmi_set_ssid(struct wil6210_vif *vif, u8 ssid_len, const void *ssid);
@@ -1150,5 +1140,6 @@ void wil6210_clear_halp(struct wil6210_priv *wil);
 int wmi_start_sched_scan(struct wil6210_priv *wil,
 			 struct cfg80211_sched_scan_request *request);
 int wmi_stop_sched_scan(struct wil6210_priv *wil);
+int wmi_mgmt_tx(struct wil6210_vif *vif, const u8 *buf, size_t len);
 
 #endif /* __WIL6210_H__ */
diff --git a/drivers/net/wireless/ath/wil6210/wmi.c b/drivers/net/wireless/ath/wil6210/wmi.c
index a3dda9a97c1f..5d991243cdb5 100644
--- a/drivers/net/wireless/ath/wil6210/wmi.c
+++ b/drivers/net/wireless/ath/wil6210/wmi.c
@@ -824,7 +824,7 @@ static void wmi_evt_connect(struct wil6210_vif *vif, int id, void *d, int len)
 	struct wireless_dev *wdev = vif_to_wdev(vif);
 	struct wmi_connect_event *evt = d;
 	int ch; /* channel number */
-	struct station_info sinfo;
+	struct station_info *sinfo;
 	u8 *assoc_req_ie, *assoc_resp_ie;
 	size_t assoc_req_ielen, assoc_resp_ielen;
 	/* capinfo(u16) + listen_interval(u16) + IEs */
@@ -940,6 +940,7 @@ static void wmi_evt_connect(struct wil6210_vif *vif, int id, void *d, int len)
 		vif->bss = NULL;
 	} else if ((wdev->iftype == NL80211_IFTYPE_AP) ||
 		   (wdev->iftype == NL80211_IFTYPE_P2P_GO)) {
+
 		if (rc) {
 			if (disable_ap_sme)
 				/* notify new_sta has failed */
@@ -947,16 +948,22 @@ static void wmi_evt_connect(struct wil6210_vif *vif, int id, void *d, int len)
 			goto out;
 		}
 
-		memset(&sinfo, 0, sizeof(sinfo));
+		sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL);
+		if (!sinfo) {
+			rc = -ENOMEM;
+			goto out;
+		}
 
-		sinfo.generation = wil->sinfo_gen++;
+		sinfo->generation = wil->sinfo_gen++;
 
 		if (assoc_req_ie) {
-			sinfo.assoc_req_ies = assoc_req_ie;
-			sinfo.assoc_req_ies_len = assoc_req_ielen;
+			sinfo->assoc_req_ies = assoc_req_ie;
+			sinfo->assoc_req_ies_len = assoc_req_ielen;
 		}
 
-		cfg80211_new_sta(ndev, evt->bssid, &sinfo, GFP_KERNEL);
+		cfg80211_new_sta(ndev, evt->bssid, sinfo, GFP_KERNEL);
+
+		kfree(sinfo);
 	} else {
 		wil_err(wil, "unhandled iftype %d for CID %d\n", wdev->iftype,
 			evt->cid);
@@ -1416,7 +1423,7 @@ void wmi_recv_cmd(struct wil6210_priv *wil)
 }
 
 int wmi_call(struct wil6210_priv *wil, u16 cmdid, u8 mid, void *buf, u16 len,
-	     u16 reply_id, void *reply, u8 reply_size, int to_msec)
+	     u16 reply_id, void *reply, u16 reply_size, int to_msec)
 {
 	int rc;
 	unsigned long remain;
@@ -1509,7 +1516,9 @@ int wmi_led_cfg(struct wil6210_priv *wil, bool enable)
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_led_cfg_done_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = cpu_to_le32(WMI_FW_STATUS_FAILURE)},
+	};
 
 	if (led_id == WIL_LED_INVALID_ID)
 		goto out;
@@ -1554,13 +1563,17 @@ int wmi_pcp_start(struct wil6210_vif *vif,
 		.pcp_max_assoc_sta = max_assoc_sta,
 		.hidden_ssid = hidden_ssid,
 		.is_go = is_go,
-		.disable_ap_sme = disable_ap_sme,
+		.ap_sme_offload_mode = disable_ap_sme ?
+				       WMI_AP_SME_OFFLOAD_PARTIAL :
+				       WMI_AP_SME_OFFLOAD_FULL,
 		.abft_len = wil->abft_len,
 	};
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_pcp_started_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_FW_STATUS_FAILURE},
+	};
 
 	if (!vif->privacy)
 		cmd.disable_sec = 1;
@@ -1574,7 +1587,7 @@ int wmi_pcp_start(struct wil6210_vif *vif,
 	}
 
 	if (disable_ap_sme &&
-	    !test_bit(WMI_FW_CAPABILITY_DISABLE_AP_SME,
+	    !test_bit(WMI_FW_CAPABILITY_AP_SME_OFFLOAD_PARTIAL,
 		      wil->fw_capabilities)) {
 		wil_err(wil, "disable_ap_sme not supported by FW\n");
 		return -EOPNOTSUPP;
@@ -1637,6 +1650,8 @@ int wmi_get_ssid(struct wil6210_vif *vif, u8 *ssid_len, void *ssid)
 	} __packed reply;
 	int len; /* reply.cmd.ssid_len in CPU order */
 
+	memset(&reply, 0, sizeof(reply));
+
 	rc = wmi_call(wil, WMI_GET_SSID_CMDID, vif->mid, NULL, 0,
 		      WMI_GET_SSID_EVENTID, &reply, sizeof(reply), 20);
 	if (rc)
@@ -1672,6 +1687,8 @@ int wmi_get_channel(struct wil6210_priv *wil, int *channel)
 		struct wmi_set_pcp_channel_cmd cmd;
 	} __packed reply;
 
+	memset(&reply, 0, sizeof(reply));
+
 	rc = wmi_call(wil, WMI_GET_PCP_CHANNEL_CMDID, vif->mid, NULL, 0,
 		      WMI_GET_PCP_CHANNEL_EVENTID, &reply, sizeof(reply), 20);
 	if (rc)
@@ -1697,7 +1714,9 @@ int wmi_p2p_cfg(struct wil6210_vif *vif, int channel, int bi)
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_p2p_cfg_done_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_FW_STATUS_FAILURE},
+	};
 
 	wil_dbg_wmi(wil, "sending WMI_P2P_CFG_CMDID\n");
 
@@ -1718,7 +1737,9 @@ int wmi_start_listen(struct wil6210_vif *vif)
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_listen_started_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_FW_STATUS_FAILURE},
+	};
 
 	wil_dbg_wmi(wil, "sending WMI_START_LISTEN_CMDID\n");
 
@@ -1740,7 +1761,9 @@ int wmi_start_search(struct wil6210_vif *vif)
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_search_started_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_FW_STATUS_FAILURE},
+	};
 
 	wil_dbg_wmi(wil, "sending WMI_START_SEARCH_CMDID\n");
 
@@ -1866,7 +1889,9 @@ int wmi_rxon(struct wil6210_priv *wil, bool on)
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_listen_started_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_FW_STATUS_FAILURE},
+	};
 
 	wil_info(wil, "(%s)\n", on ? "on" : "off");
 
@@ -1908,6 +1933,8 @@ int wmi_rx_chain_add(struct wil6210_priv *wil, struct vring *vring)
 	} __packed evt;
 	int rc;
 
+	memset(&evt, 0, sizeof(evt));
+
 	if (wdev->iftype == NL80211_IFTYPE_MONITOR) {
 		struct ieee80211_channel *ch = wil->monitor_chandef.chan;
 
@@ -1937,14 +1964,14 @@ int wmi_rx_chain_add(struct wil6210_priv *wil, struct vring *vring)
 	if (rc)
 		return rc;
 
+	if (le32_to_cpu(evt.evt.status) != WMI_CFG_RX_CHAIN_SUCCESS)
+		rc = -EINVAL;
+
 	vring->hwtail = le32_to_cpu(evt.evt.rx_ring_tail_ptr);
 
 	wil_dbg_misc(wil, "Rx init: status %d tail 0x%08x\n",
 		     le32_to_cpu(evt.evt.status), vring->hwtail);
 
-	if (le32_to_cpu(evt.evt.status) != WMI_CFG_RX_CHAIN_SUCCESS)
-		rc = -EINVAL;
-
 	return rc;
 }
 
@@ -1962,6 +1989,8 @@ int wmi_get_temperature(struct wil6210_priv *wil, u32 *t_bb, u32 *t_rf)
 		struct wmi_temp_sense_done_event evt;
 	} __packed reply;
 
+	memset(&reply, 0, sizeof(reply));
+
 	rc = wmi_call(wil, WMI_TEMP_SENSE_CMDID, vif->mid, &cmd, sizeof(cmd),
 		      WMI_TEMP_SENSE_DONE_EVENTID, &reply, sizeof(reply), 100);
 	if (rc)
@@ -1994,6 +2023,7 @@ int wmi_disconnect_sta(struct wil6210_vif *vif, const u8 *mac,
 
 	wil_dbg_wmi(wil, "disconnect_sta: (%pM, reason %d)\n", mac, reason);
 
+	memset(&reply, 0, sizeof(reply));
 	vif->locally_generated_disc = true;
 	if (del_sta) {
 		ether_addr_copy(del_sta_cmd.dst_mac, mac);
@@ -2092,7 +2122,9 @@ int wmi_addba_rx_resp(struct wil6210_priv *wil,
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_rcp_addba_resp_sent_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = cpu_to_le16(WMI_FW_STATUS_FAILURE)},
+	};
 
 	wil_dbg_wmi(wil,
 		    "ADDBA response for MID %d CID %d TID %d size %d timeout %d status %d AMSDU%s\n",
@@ -2125,13 +2157,13 @@ int wmi_ps_dev_profile_cfg(struct wil6210_priv *wil,
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_ps_dev_profile_cfg_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = cpu_to_le32(WMI_PS_CFG_CMD_STATUS_ERROR)},
+	};
 	u32 status;
 
 	wil_dbg_wmi(wil, "Setting ps dev profile %d\n", ps_profile);
 
-	reply.evt.status = cpu_to_le32(WMI_PS_CFG_CMD_STATUS_ERROR);
-
 	rc = wmi_call(wil, WMI_PS_DEV_PROFILE_CFG_CMDID, vif->mid,
 		      &cmd, sizeof(cmd),
 		      WMI_PS_DEV_PROFILE_CFG_EVENTID, &reply, sizeof(reply),
@@ -2160,15 +2192,15 @@ int wmi_set_mgmt_retry(struct wil6210_priv *wil, u8 retry_short)
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_set_mgmt_retry_limit_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_FW_STATUS_FAILURE},
+	};
 
 	wil_dbg_wmi(wil, "Setting mgmt retry short %d\n", retry_short);
 
 	if (!test_bit(WMI_FW_CAPABILITY_MGMT_RETRY_LIMIT, wil->fw_capabilities))
 		return -ENOTSUPP;
 
-	reply.evt.status = WMI_FW_STATUS_FAILURE;
-
 	rc = wmi_call(wil, WMI_SET_MGMT_RETRY_LIMIT_CMDID, vif->mid,
 		      &cmd, sizeof(cmd),
 		      WMI_SET_MGMT_RETRY_LIMIT_EVENTID, &reply, sizeof(reply),
@@ -2199,7 +2231,7 @@ int wmi_get_mgmt_retry(struct wil6210_priv *wil, u8 *retry_short)
 	if (!test_bit(WMI_FW_CAPABILITY_MGMT_RETRY_LIMIT, wil->fw_capabilities))
 		return -ENOTSUPP;
 
-	reply.evt.mgmt_retry_limit = 0;
+	memset(&reply, 0, sizeof(reply));
 	rc = wmi_call(wil, WMI_GET_MGMT_RETRY_LIMIT_CMDID, vif->mid, NULL, 0,
 		      WMI_GET_MGMT_RETRY_LIMIT_EVENTID, &reply, sizeof(reply),
 		      100);
@@ -2282,14 +2314,15 @@ int wmi_suspend(struct wil6210_priv *wil)
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_traffic_suspend_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_TRAFFIC_SUSPEND_REJECTED_LINK_NOT_IDLE},
+	};
+
 	u32 suspend_to = WIL_WAIT_FOR_SUSPEND_RESUME_COMP;
 
 	wil->suspend_resp_rcvd = false;
 	wil->suspend_resp_comp = false;
 
-	reply.evt.status = WMI_TRAFFIC_SUSPEND_REJECTED_LINK_NOT_IDLE;
-
 	rc = wmi_call(wil, WMI_TRAFFIC_SUSPEND_CMDID, vif->mid,
 		      &cmd, sizeof(cmd),
 		      WMI_TRAFFIC_SUSPEND_EVENTID, &reply, sizeof(reply),
@@ -2365,10 +2398,11 @@ int wmi_resume(struct wil6210_priv *wil)
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_traffic_resume_event evt;
-	} __packed reply;
-
-	reply.evt.status = WMI_TRAFFIC_RESUME_FAILED;
-	reply.evt.resume_triggers = WMI_RESUME_TRIGGER_UNKNOWN;
+	} __packed reply = {
+		.evt = {.status = WMI_TRAFFIC_RESUME_FAILED,
+			.resume_triggers =
+				cpu_to_le32(WMI_RESUME_TRIGGER_UNKNOWN)},
+	};
 
 	rc = wmi_call(wil, WMI_TRAFFIC_RESUME_CMDID, vif->mid, NULL, 0,
 		      WMI_TRAFFIC_RESUME_EVENTID, &reply, sizeof(reply),
@@ -2394,7 +2428,9 @@ int wmi_port_allocate(struct wil6210_priv *wil, u8 mid,
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_port_allocated_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_FW_STATUS_FAILURE},
+	};
 
 	wil_dbg_misc(wil, "port allocate, mid %d iftype %d, mac %pM\n",
 		     mid, iftype, mac);
@@ -2419,8 +2455,6 @@ int wmi_port_allocate(struct wil6210_priv *wil, u8 mid,
 		return -EINVAL;
 	}
 
-	reply.evt.status = WMI_FW_STATUS_FAILURE;
-
 	rc = wmi_call(wil, WMI_PORT_ALLOCATE_CMDID, mid,
 		      &cmd, sizeof(cmd),
 		      WMI_PORT_ALLOCATED_EVENTID, &reply,
@@ -2447,12 +2481,12 @@ int wmi_port_delete(struct wil6210_priv *wil, u8 mid)
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_port_deleted_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.status = WMI_FW_STATUS_FAILURE},
+	};
 
 	wil_dbg_misc(wil, "port delete, mid %d\n", mid);
 
-	reply.evt.status = WMI_FW_STATUS_FAILURE;
-
 	rc = wmi_call(wil, WMI_PORT_DELETE_CMDID, mid,
 		      &cmd, sizeof(cmd),
 		      WMI_PORT_DELETED_EVENTID, &reply,
@@ -2709,7 +2743,9 @@ int wmi_start_sched_scan(struct wil6210_priv *wil,
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_start_sched_scan_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.result = WMI_PNO_REJECT},
+	};
 
 	if (!test_bit(WMI_FW_CAPABILITY_PNO, wil->fw_capabilities))
 		return -ENOTSUPP;
@@ -2725,8 +2761,6 @@ int wmi_start_sched_scan(struct wil6210_priv *wil,
 	wmi_sched_scan_set_plans(wil, &cmd,
 				 request->scan_plans, request->n_scan_plans);
 
-	reply.evt.result = WMI_PNO_REJECT;
-
 	rc = wmi_call(wil, WMI_START_SCHED_SCAN_CMDID, vif->mid,
 		      &cmd, sizeof(cmd),
 		      WMI_START_SCHED_SCAN_EVENTID, &reply, sizeof(reply),
@@ -2750,13 +2784,13 @@ int wmi_stop_sched_scan(struct wil6210_priv *wil)
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_stop_sched_scan_event evt;
-	} __packed reply;
+	} __packed reply = {
+		.evt = {.result = WMI_PNO_REJECT},
+	};
 
 	if (!test_bit(WMI_FW_CAPABILITY_PNO, wil->fw_capabilities))
 		return -ENOTSUPP;
 
-	reply.evt.result = WMI_PNO_REJECT;
-
 	rc = wmi_call(wil, WMI_STOP_SCHED_SCAN_CMDID, vif->mid, NULL, 0,
 		      WMI_STOP_SCHED_SCAN_EVENTID, &reply, sizeof(reply),
 		      WIL_WMI_CALL_GENERAL_TO_MS);
@@ -2771,3 +2805,50 @@ int wmi_stop_sched_scan(struct wil6210_priv *wil)
 
 	return 0;
 }
+
+int wmi_mgmt_tx(struct wil6210_vif *vif, const u8 *buf, size_t len)
+{
+	size_t total;
+	struct wil6210_priv *wil = vif_to_wil(vif);
+	struct ieee80211_mgmt *mgmt_frame = (void *)buf;
+	struct wmi_sw_tx_req_cmd *cmd;
+	struct {
+		struct wmi_cmd_hdr wmi;
+		struct wmi_sw_tx_complete_event evt;
+	} __packed evt = {
+		.evt = {.status = WMI_FW_STATUS_FAILURE},
+	};
+	int rc;
+
+	wil_dbg_misc(wil, "mgmt_tx mid %d\n", vif->mid);
+	wil_hex_dump_misc("mgmt tx frame ", DUMP_PREFIX_OFFSET, 16, 1, buf,
+			  len, true);
+
+	if (len < sizeof(struct ieee80211_hdr_3addr))
+		return -EINVAL;
+
+	total = sizeof(*cmd) + len;
+	if (total < len) {
+		wil_err(wil, "mgmt_tx invalid len %zu\n", len);
+		return -EINVAL;
+	}
+
+	cmd = kmalloc(total, GFP_KERNEL);
+	if (!cmd)
+		return -ENOMEM;
+
+	memcpy(cmd->dst_mac, mgmt_frame->da, WMI_MAC_LEN);
+	cmd->len = cpu_to_le16(len);
+	memcpy(cmd->payload, buf, len);
+
+	rc = wmi_call(wil, WMI_SW_TX_REQ_CMDID, vif->mid, cmd, total,
+		      WMI_SW_TX_COMPLETE_EVENTID, &evt, sizeof(evt), 2000);
+	if (!rc && evt.evt.status != WMI_FW_STATUS_SUCCESS) {
+		wil_err(wil, "mgmt_tx failed with status %d\n", evt.evt.status);
+		rc = -EINVAL;
+	}
+
+	kfree(cmd);
+
+	return rc;
+}
diff --git a/drivers/net/wireless/ath/wil6210/wmi.h b/drivers/net/wireless/ath/wil6210/wmi.h
index d3e75f0ff245..dc503d903786 100644
--- a/drivers/net/wireless/ath/wil6210/wmi.h
+++ b/drivers/net/wireless/ath/wil6210/wmi.h
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
  * Copyright (c) 2006-2012 Wilocity
  *
@@ -29,8 +30,6 @@
 #ifndef __WILOCITY_WMI_H__
 #define __WILOCITY_WMI_H__
 
-/* General */
-#define WMI_MAX_ASSOC_STA		(8)
 #define WMI_DEFAULT_ASSOC_STA		(1)
 #define WMI_MAC_LEN			(6)
 #define WMI_PROX_RANGE_NUM		(3)
@@ -41,6 +40,19 @@
 #define WMI_RF_ETYPE_LENGTH		(3)
 #define WMI_RF_RX2TX_LENGTH		(3)
 #define WMI_RF_ETYPE_VAL_PER_RANGE	(5)
+/* DTYPE configuration array size
+ * must always be kept equal to (WMI_RF_DTYPE_LENGTH+1)
+ */
+#define WMI_RF_DTYPE_CONF_LENGTH	(4)
+/* ETYPE configuration array size
+ * must always be kept equal to
+ * (WMI_RF_ETYPE_LENGTH+WMI_RF_ETYPE_VAL_PER_RANGE)
+ */
+#define WMI_RF_ETYPE_CONF_LENGTH	(8)
+/* RX2TX configuration array size
+ * must always be kept equal to (WMI_RF_RX2TX_LENGTH+1)
+ */
+#define WMI_RF_RX2TX_CONF_LENGTH	(4)
 
 /* Mailbox interface
  * used for commands and events
@@ -61,7 +73,7 @@ enum wmi_fw_capability {
 	WMI_FW_CAPABILITY_PS_CONFIG			= 1,
 	WMI_FW_CAPABILITY_RF_SECTORS			= 2,
 	WMI_FW_CAPABILITY_MGMT_RETRY_LIMIT		= 3,
-	WMI_FW_CAPABILITY_DISABLE_AP_SME		= 4,
+	WMI_FW_CAPABILITY_AP_SME_OFFLOAD_PARTIAL	= 4,
 	WMI_FW_CAPABILITY_WMI_ONLY			= 5,
 	WMI_FW_CAPABILITY_THERMAL_THROTTLING		= 7,
 	WMI_FW_CAPABILITY_D3_SUSPEND			= 8,
@@ -73,6 +85,7 @@ enum wmi_fw_capability {
 	WMI_FW_CAPABILITY_LO_POWER_CALIB_FROM_OTP	= 14,
 	WMI_FW_CAPABILITY_PNO				= 15,
 	WMI_FW_CAPABILITY_REF_CLOCK_CONTROL		= 18,
+	WMI_FW_CAPABILITY_AP_SME_OFFLOAD_NONE		= 19,
 	WMI_FW_CAPABILITY_MAX,
 };
 
@@ -164,12 +177,14 @@ enum wmi_command_id {
 	WMI_SET_ACTIVE_SILENT_RSSI_TABLE_CMDID		= 0x85C,
 	WMI_RF_PWR_ON_DELAY_CMDID			= 0x85D,
 	WMI_SET_HIGH_POWER_TABLE_PARAMS_CMDID		= 0x85E,
+	WMI_FIXED_SCHEDULING_UL_CONFIG_CMDID		= 0x85F,
 	/* Performance monitoring commands */
 	WMI_BF_CTRL_CMDID				= 0x862,
 	WMI_NOTIFY_REQ_CMDID				= 0x863,
 	WMI_GET_STATUS_CMDID				= 0x864,
 	WMI_GET_RF_STATUS_CMDID				= 0x866,
 	WMI_GET_BASEBAND_TYPE_CMDID			= 0x867,
+	WMI_VRING_SWITCH_TIMING_CONFIG_CMDID		= 0x868,
 	WMI_UNIT_TEST_CMDID				= 0x900,
 	WMI_FLASH_READ_CMDID				= 0x902,
 	WMI_FLASH_WRITE_CMDID				= 0x903,
@@ -202,6 +217,7 @@ enum wmi_command_id {
 	WMI_GET_THERMAL_THROTTLING_CFG_CMDID		= 0x941,
 	/* Read Power Save profile type */
 	WMI_PS_DEV_PROFILE_CFG_READ_CMDID		= 0x942,
+	WMI_TSF_SYNC_CMDID				= 0x973,
 	WMI_TOF_SESSION_START_CMDID			= 0x991,
 	WMI_TOF_GET_CAPABILITIES_CMDID			= 0x992,
 	WMI_TOF_SET_LCR_CMDID				= 0x993,
@@ -218,11 +234,16 @@ enum wmi_command_id {
 	WMI_PRIO_TX_SECTORS_ORDER_CMDID			= 0x9A5,
 	WMI_PRIO_TX_SECTORS_NUMBER_CMDID		= 0x9A6,
 	WMI_PRIO_TX_SECTORS_SET_DEFAULT_CFG_CMDID	= 0x9A7,
+	WMI_BF_CONTROL_CMDID				= 0x9AA,
 	WMI_SCHEDULING_SCHEME_CMDID			= 0xA01,
 	WMI_FIXED_SCHEDULING_CONFIG_CMDID		= 0xA02,
 	WMI_ENABLE_FIXED_SCHEDULING_CMDID		= 0xA03,
 	WMI_SET_MULTI_DIRECTED_OMNIS_CONFIG_CMDID	= 0xA04,
 	WMI_SET_LONG_RANGE_CONFIG_CMDID			= 0xA05,
+	WMI_GET_ASSOC_LIST_CMDID			= 0xA06,
+	WMI_GET_CCA_INDICATIONS_CMDID			= 0xA07,
+	WMI_SET_CCA_INDICATIONS_BI_AVG_NUM_CMDID	= 0xA08,
+	WMI_INTERNAL_FW_IOCTL_CMDID			= 0xA0B,
 	WMI_SET_MAC_ADDRESS_CMDID			= 0xF003,
 	WMI_ABORT_SCAN_CMDID				= 0xF007,
 	WMI_SET_PROMISCUOUS_MODE_CMDID			= 0xF041,
@@ -484,6 +505,18 @@ enum wmi_rf_mgmt_type {
 	WMI_RF_MGMT_GET_STATUS	= 0x02,
 };
 
+/* WMI_BF_CONTROL_CMDID */
+enum wmi_bf_triggers {
+	WMI_BF_TRIGGER_RS_MCS1_TH_FAILURE		= 0x01,
+	WMI_BF_TRIGGER_RS_MCS1_NO_BACK_FAILURE		= 0x02,
+	WMI_BF_TRIGGER_MAX_CTS_FAILURE_IN_TXOP		= 0x04,
+	WMI_BF_TRIGGER_MAX_BACK_FAILURE			= 0x08,
+	WMI_BF_TRIGGER_FW				= 0x10,
+	WMI_BF_TRIGGER_MAX_CTS_FAILURE_IN_KEEP_ALIVE	= 0x20,
+	WMI_BF_TRIGGER_AOA				= 0x40,
+	WMI_BF_TRIGGER_MAX_CTS_FAILURE_IN_UPM		= 0x80,
+};
+
 /* WMI_RF_MGMT_CMDID */
 struct wmi_rf_mgmt_cmd {
 	__le32 rf_mgmt_type;
@@ -519,7 +552,9 @@ struct wmi_bcon_ctrl_cmd {
 	u8 disable_sec;
 	u8 hidden_ssid;
 	u8 is_go;
-	u8 reserved[2];
+	/* A-BFT length override if non-0 */
+	u8 abft_len;
+	u8 reserved;
 } __packed;
 
 /* WMI_PORT_ALLOCATE_CMDID */
@@ -584,6 +619,16 @@ struct wmi_power_mgmt_cfg_cmd {
 	u8 reserved[3];
 } __packed;
 
+/* WMI_PCP_START_CMDID */
+enum wmi_ap_sme_offload_mode {
+	/* Full AP SME in FW */
+	WMI_AP_SME_OFFLOAD_FULL		= 0x00,
+	/* Probe AP SME in FW */
+	WMI_AP_SME_OFFLOAD_PARTIAL	= 0x01,
+	/* AP SME in host */
+	WMI_AP_SME_OFFLOAD_NONE		= 0x02,
+};
+
 /* WMI_PCP_START_CMDID */
 struct wmi_pcp_start_cmd {
 	__le16 bcon_interval;
@@ -593,7 +638,8 @@ struct wmi_pcp_start_cmd {
 	u8 reserved0[5];
 	/* A-BFT length override if non-0 */
 	u8 abft_len;
-	u8 disable_ap_sme;
+	/* enum wmi_ap_sme_offload_mode_e */
+	u8 ap_sme_offload_mode;
 	u8 network_type;
 	u8 channel;
 	u8 disable_sec_offload;
@@ -607,6 +653,17 @@ struct wmi_sw_tx_req_cmd {
 	u8 payload[0];
 } __packed;
 
+/* WMI_VRING_SWITCH_TIMING_CONFIG_CMDID */
+struct wmi_vring_switch_timing_config_cmd {
+	/* Set vring timing configuration:
+	 *
+	 * defined interval for vring switch
+	 */
+	__le32 interval_usec;
+	/* vring inactivity threshold */
+	__le32 idle_th_usec;
+} __packed;
+
 struct wmi_sw_ring_cfg {
 	__le64 ring_mem_base;
 	__le16 ring_size;
@@ -642,6 +699,7 @@ enum wmi_vring_cfg_schd_params_priority {
 	WMI_SCH_PRIO_HIGH	= 0x01,
 };
 
+#define CIDXTID_EXTENDED_CID_TID		(0xFF)
 #define CIDXTID_CID_POS				(0)
 #define CIDXTID_CID_LEN				(4)
 #define CIDXTID_CID_MSK				(0xF)
@@ -662,6 +720,9 @@ struct wmi_vring_cfg {
 	struct wmi_sw_ring_cfg tx_sw_ring;
 	/* 0-23 vrings */
 	u8 ringid;
+	/* Used for cid less than 8. For higher cid set
+	 * CIDXTID_EXTENDED_CID_TID here and use cid and tid members instead
+	 */
 	u8 cidxtid;
 	u8 encap_trans_type;
 	/* 802.3 DS cfg */
@@ -671,6 +732,11 @@ struct wmi_vring_cfg {
 	u8 to_resolution;
 	u8 agg_max_wsize;
 	struct wmi_vring_cfg_schd schd_params;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 cid;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 tid;
+	u8 reserved[2];
 } __packed;
 
 enum wmi_vring_cfg_cmd_action {
@@ -868,23 +934,42 @@ struct wmi_cfg_rx_chain_cmd {
 
 /* WMI_RCP_ADDBA_RESP_CMDID */
 struct wmi_rcp_addba_resp_cmd {
+	/* Used for cid less than 8. For higher cid set
+	 * CIDXTID_EXTENDED_CID_TID here and use cid and tid members instead
+	 */
 	u8 cidxtid;
 	u8 dialog_token;
 	__le16 status_code;
 	/* ieee80211_ba_parameterset field to send */
 	__le16 ba_param_set;
 	__le16 ba_timeout;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 cid;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 tid;
+	u8 reserved[2];
 } __packed;
 
 /* WMI_RCP_DELBA_CMDID */
 struct wmi_rcp_delba_cmd {
+	/* Used for cid less than 8. For higher cid set
+	 * CIDXTID_EXTENDED_CID_TID here and use cid and tid members instead
+	 */
 	u8 cidxtid;
 	u8 reserved;
 	__le16 reason;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 cid;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 tid;
+	u8 reserved2[2];
 } __packed;
 
 /* WMI_RCP_ADDBA_REQ_CMDID */
 struct wmi_rcp_addba_req_cmd {
+	/* Used for cid less than 8. For higher cid set
+	 * CIDXTID_EXTENDED_CID_TID here and use cid and tid members instead
+	 */
 	u8 cidxtid;
 	u8 dialog_token;
 	/* ieee80211_ba_parameterset field as it received */
@@ -892,6 +977,11 @@ struct wmi_rcp_addba_req_cmd {
 	__le16 ba_timeout;
 	/* ieee80211_ba_seqstrl field as it received */
 	__le16 ba_seq_ctrl;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 cid;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 tid;
+	u8 reserved[2];
 } __packed;
 
 /* WMI_SET_MAC_ADDRESS_CMDID */
@@ -902,15 +992,20 @@ struct wmi_set_mac_address_cmd {
 
 /* WMI_ECHO_CMDID
  * Check FW is alive
- * WMI_DEEP_ECHO_CMDID
- * Check FW and ucode are alive
  * Returned event: WMI_ECHO_RSP_EVENTID
- * same event for both commands
  */
 struct wmi_echo_cmd {
 	__le32 value;
 } __packed;
 
+/* WMI_DEEP_ECHO_CMDID
+ * Check FW and ucode are alive
+ * Returned event: WMI_ECHO_RSP_EVENTID
+ */
+struct wmi_deep_echo_cmd {
+	__le32 value;
+} __packed;
+
 /* WMI_RF_PWR_ON_DELAY_CMDID
  * set FW time parameters used through RF resetting
  *  RF reset consists of bringing its power down for a period of time, then
@@ -928,7 +1023,7 @@ struct wmi_rf_pwr_on_delay_cmd {
 	__le16 up_delay_usec;
 } __packed;
 
-/* \WMI_SET_HIGH_POWER_TABLE_PARAMS_CMDID
+/* WMI_SET_HIGH_POWER_TABLE_PARAMS_CMDID
  * This API controls the Tx and Rx gain over temperature.
  * It controls the Tx D-type, Rx D-type and Rx E-type amplifiers.
  * It also controls the Tx gain index, by controlling the Rx to Tx gain index
@@ -942,25 +1037,46 @@ struct wmi_set_high_power_table_params_cmd {
 	u8 tx_dtype_temp[WMI_RF_DTYPE_LENGTH];
 	u8 reserved0;
 	/* Tx D-type values to be used for each temperature range */
-	__le32 tx_dtype_conf[WMI_RF_DTYPE_LENGTH + 1];
+	__le32 tx_dtype_conf[WMI_RF_DTYPE_CONF_LENGTH];
+	/* Temperature range for Tx E-type parameters */
+	u8 tx_etype_temp[WMI_RF_ETYPE_LENGTH];
+	u8 reserved1;
+	/* Tx E-type values to be used for each temperature range.
+	 * The last 4 values of any range are the first 4 values of the next
+	 * range and so on
+	 */
+	__le32 tx_etype_conf[WMI_RF_ETYPE_CONF_LENGTH];
 	/* Temperature range for Rx D-type parameters */
 	u8 rx_dtype_temp[WMI_RF_DTYPE_LENGTH];
-	u8 reserved1;
+	u8 reserved2;
 	/* Rx D-type values to be used for each temperature range */
-	__le32 rx_dtype_conf[WMI_RF_DTYPE_LENGTH + 1];
+	__le32 rx_dtype_conf[WMI_RF_DTYPE_CONF_LENGTH];
 	/* Temperature range for Rx E-type parameters */
 	u8 rx_etype_temp[WMI_RF_ETYPE_LENGTH];
-	u8 reserved2;
+	u8 reserved3;
 	/* Rx E-type values to be used for each temperature range.
 	 * The last 4 values of any range are the first 4 values of the next
 	 * range and so on
 	 */
-	__le32 rx_etype_conf[WMI_RF_ETYPE_VAL_PER_RANGE + WMI_RF_ETYPE_LENGTH];
+	__le32 rx_etype_conf[WMI_RF_ETYPE_CONF_LENGTH];
 	/* Temperature range for rx_2_tx_offs parameters */
 	u8 rx_2_tx_temp[WMI_RF_RX2TX_LENGTH];
-	u8 reserved3;
+	u8 reserved4;
 	/* Rx to Tx gain index offset */
-	s8 rx_2_tx_offs[WMI_RF_RX2TX_LENGTH + 1];
+	s8 rx_2_tx_offs[WMI_RF_RX2TX_CONF_LENGTH];
+} __packed;
+
+/* WMI_FIXED_SCHEDULING_UL_CONFIG_CMDID
+ * This API sets rd parameter per mcs.
+ * Relevant only in Fixed Scheduling mode.
+ * Returned event: WMI_FIXED_SCHEDULING_UL_CONFIG_EVENTID
+ */
+struct wmi_fixed_scheduling_ul_config_cmd {
+	/* Use mcs -1 to set for every mcs */
+	s8 mcs;
+	/* Number of frames with rd bit set in a single virtual slot */
+	u8 rd_count_per_slot;
+	u8 reserved[2];
 } __packed;
 
 /* CMD: WMI_RF_XPM_READ_CMDID */
@@ -1267,6 +1383,93 @@ struct wmi_set_long_range_config_complete_event {
 	u8 reserved[3];
 } __packed;
 
+/* payload max size is 236 bytes: max event buffer size (256) - WMI headers
+ * (16) - prev struct field size (4)
+ */
+#define WMI_MAX_IOCTL_PAYLOAD_SIZE		(236)
+#define WMI_MAX_IOCTL_REPLY_PAYLOAD_SIZE	(236)
+#define WMI_MAX_INTERNAL_EVENT_PAYLOAD_SIZE	(236)
+
+enum wmi_internal_fw_ioctl_code {
+	WMI_INTERNAL_FW_CODE_NONE	= 0x0,
+	WMI_INTERNAL_FW_CODE_QCOM	= 0x1,
+};
+
+/* WMI_INTERNAL_FW_IOCTL_CMDID */
+struct wmi_internal_fw_ioctl_cmd {
+	/* enum wmi_internal_fw_ioctl_code */
+	__le16 code;
+	__le16 length;
+	/* payload max size is WMI_MAX_IOCTL_PAYLOAD_SIZE
+	 * Must be the last member of the struct
+	 */
+	__le32 payload[0];
+} __packed;
+
+/* WMI_INTERNAL_FW_IOCTL_EVENTID */
+struct wmi_internal_fw_ioctl_event {
+	/* wmi_fw_status */
+	u8 status;
+	u8 reserved;
+	__le16 length;
+	/* payload max size is WMI_MAX_IOCTL_REPLY_PAYLOAD_SIZE
+	 * Must be the last member of the struct
+	 */
+	__le32 payload[0];
+} __packed;
+
+/* WMI_INTERNAL_FW_EVENT_EVENTID */
+struct wmi_internal_fw_event_event {
+	__le16 id;
+	__le16 length;
+	/* payload max size is WMI_MAX_INTERNAL_EVENT_PAYLOAD_SIZE
+	 * Must be the last member of the struct
+	 */
+	__le32 payload[0];
+} __packed;
+
+/* WMI_BF_CONTROL_CMDID */
+struct wmi_bf_control_cmd {
+	/* wmi_bf_triggers */
+	__le32 triggers;
+	u8 cid;
+	/* DISABLED = 0, ENABLED = 1 , DRY_RUN = 2 */
+	u8 txss_mode;
+	/* DISABLED = 0, ENABLED = 1, DRY_RUN = 2 */
+	u8 brp_mode;
+	/* Max cts threshold (correspond to
+	 * WMI_BF_TRIGGER_MAX_CTS_FAILURE_IN_TXOP)
+	 */
+	u8 bf_trigger_max_cts_failure_thr;
+	/* Max cts threshold in dense (correspond to
+	 * WMI_BF_TRIGGER_MAX_CTS_FAILURE_IN_TXOP)
+	 */
+	u8 bf_trigger_max_cts_failure_dense_thr;
+	/* Max b-ack threshold (correspond to
+	 * WMI_BF_TRIGGER_MAX_BACK_FAILURE)
+	 */
+	u8 bf_trigger_max_back_failure_thr;
+	/* Max b-ack threshold in dense (correspond to
+	 * WMI_BF_TRIGGER_MAX_BACK_FAILURE)
+	 */
+	u8 bf_trigger_max_back_failure_dense_thr;
+	u8 reserved0;
+	/* Wrong sectors threshold */
+	__le32 wrong_sector_bis_thr;
+	/* BOOL to enable/disable long term trigger */
+	u8 long_term_enable;
+	/* 1 = Update long term thresholds from the long_term_mbps_th_tbl and
+	 * long_term_trig_timeout_per_mcs arrays, 0 = Ignore
+	 */
+	u8 long_term_update_thr;
+	/* Long term throughput threshold [Mbps] */
+	u8 long_term_mbps_th_tbl[WMI_NUM_MCS];
+	u8 reserved1;
+	/* Long term timeout threshold table [msec] */
+	__le16 long_term_trig_timeout_per_mcs[WMI_NUM_MCS];
+	u8 reserved2[2];
+} __packed;
+
 /* WMI Events
  * List of Events (target to host)
  */
@@ -1325,6 +1528,7 @@ enum wmi_event_id {
 	WMI_SET_SILENT_RSSI_TABLE_DONE_EVENTID		= 0x185C,
 	WMI_RF_PWR_ON_DELAY_RSP_EVENTID			= 0x185D,
 	WMI_SET_HIGH_POWER_TABLE_PARAMS_EVENTID		= 0x185E,
+	WMI_FIXED_SCHEDULING_UL_CONFIG_EVENTID		= 0x185F,
 	/* Performance monitoring events */
 	WMI_DATA_PORT_OPEN_EVENTID			= 0x1860,
 	WMI_WBE_LINK_DOWN_EVENTID			= 0x1861,
@@ -1334,6 +1538,7 @@ enum wmi_event_id {
 	WMI_VRING_EN_EVENTID				= 0x1865,
 	WMI_GET_RF_STATUS_EVENTID			= 0x1866,
 	WMI_GET_BASEBAND_TYPE_EVENTID			= 0x1867,
+	WMI_VRING_SWITCH_TIMING_CONFIG_EVENTID		= 0x1868,
 	WMI_UNIT_TEST_EVENTID				= 0x1900,
 	WMI_FLASH_READ_DONE_EVENTID			= 0x1902,
 	WMI_FLASH_WRITE_DONE_EVENTID			= 0x1903,
@@ -1363,6 +1568,7 @@ enum wmi_event_id {
 	WMI_GET_THERMAL_THROTTLING_CFG_EVENTID		= 0x1941,
 	/* return the Power Save profile */
 	WMI_PS_DEV_PROFILE_CFG_READ_EVENTID		= 0x1942,
+	WMI_TSF_SYNC_STATUS_EVENTID			= 0x1973,
 	WMI_TOF_SESSION_END_EVENTID			= 0x1991,
 	WMI_TOF_GET_CAPABILITIES_EVENTID		= 0x1992,
 	WMI_TOF_SET_LCR_EVENTID				= 0x1993,
@@ -1380,17 +1586,24 @@ enum wmi_event_id {
 	WMI_PRIO_TX_SECTORS_ORDER_EVENTID		= 0x19A5,
 	WMI_PRIO_TX_SECTORS_NUMBER_EVENTID		= 0x19A6,
 	WMI_PRIO_TX_SECTORS_SET_DEFAULT_CFG_EVENTID	= 0x19A7,
+	WMI_BF_CONTROL_EVENTID				= 0x19AA,
 	WMI_SCHEDULING_SCHEME_EVENTID			= 0x1A01,
 	WMI_FIXED_SCHEDULING_CONFIG_COMPLETE_EVENTID	= 0x1A02,
 	WMI_ENABLE_FIXED_SCHEDULING_COMPLETE_EVENTID	= 0x1A03,
 	WMI_SET_MULTI_DIRECTED_OMNIS_CONFIG_EVENTID	= 0x1A04,
 	WMI_SET_LONG_RANGE_CONFIG_COMPLETE_EVENTID	= 0x1A05,
+	WMI_GET_ASSOC_LIST_RES_EVENTID			= 0x1A06,
+	WMI_GET_CCA_INDICATIONS_EVENTID			= 0x1A07,
+	WMI_SET_CCA_INDICATIONS_BI_AVG_NUM_EVENTID	= 0x1A08,
+	WMI_INTERNAL_FW_EVENT_EVENTID			= 0x1A0A,
+	WMI_INTERNAL_FW_IOCTL_EVENTID			= 0x1A0B,
 	WMI_SET_CHANNEL_EVENTID				= 0x9000,
 	WMI_ASSOC_REQ_EVENTID				= 0x9001,
 	WMI_EAPOL_RX_EVENTID				= 0x9002,
 	WMI_MAC_ADDR_RESP_EVENTID			= 0x9003,
 	WMI_FW_VER_EVENTID				= 0x9004,
 	WMI_ACS_PASSIVE_SCAN_COMPLETE_EVENTID		= 0x9005,
+	WMI_INTERNAL_FW_SET_CHANNEL			= 0x9006,
 	WMI_COMMAND_NOT_SUPPORTED_EVENTID		= 0xFFFF,
 };
 
@@ -1462,12 +1675,16 @@ enum rf_type {
 	RF_UNKNOWN	= 0x00,
 	RF_MARLON	= 0x01,
 	RF_SPARROW	= 0x02,
+	RF_TALYNA1	= 0x03,
+	RF_TALYNA2	= 0x04,
 };
 
 /* WMI_GET_RF_STATUS_EVENTID */
 enum board_file_rf_type {
 	BF_RF_MARLON	= 0x00,
 	BF_RF_SPARROW	= 0x01,
+	BF_RF_TALYNA1	= 0x02,
+	BF_RF_TALYNA2	= 0x03,
 };
 
 /* WMI_GET_RF_STATUS_EVENTID */
@@ -1507,6 +1724,7 @@ enum baseband_type {
 	BASEBAND_SPARROW_M_C0	= 0x06,
 	BASEBAND_SPARROW_M_D0	= 0x07,
 	BASEBAND_TALYN_M_A0	= 0x08,
+	BASEBAND_TALYN_M_B0	= 0x09,
 };
 
 /* WMI_GET_BASEBAND_TYPE_EVENTID */
@@ -1551,7 +1769,11 @@ struct wmi_ready_event {
 	u8 numof_additional_mids;
 	/* rfc read calibration result. 5..15 */
 	u8 rfc_read_calib_result;
-	u8 reserved[3];
+	/* Max associated STAs supported by FW in AP mode (default 0 means 8
+	 * STA)
+	 */
+	u8 max_assoc_sta;
+	u8 reserved[2];
 } __packed;
 
 /* WMI_NOTIFY_REQ_DONE_EVENTID */
@@ -1666,13 +1888,13 @@ enum wmi_pno_result {
 };
 
 struct wmi_start_sched_scan_event {
-	/* pno_result */
+	/* wmi_pno_result */
 	u8 result;
 	u8 reserved[3];
 } __packed;
 
 struct wmi_stop_sched_scan_event {
-	/* pno_result */
+	/* wmi_pno_result */
 	u8 result;
 	u8 reserved[3];
 } __packed;
@@ -1739,9 +1961,17 @@ struct wmi_ba_status_event {
 
 /* WMI_DELBA_EVENTID */
 struct wmi_delba_event {
+	/* Used for cid less than 8. For higher cid set
+	 * CIDXTID_EXTENDED_CID_TID here and use cid and tid members instead
+	 */
 	u8 cidxtid;
 	u8 from_initiator;
 	__le16 reason;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 cid;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 tid;
+	u8 reserved[2];
 } __packed;
 
 /* WMI_VRING_CFG_DONE_EVENTID */
@@ -1754,13 +1984,24 @@ struct wmi_vring_cfg_done_event {
 
 /* WMI_RCP_ADDBA_RESP_SENT_EVENTID */
 struct wmi_rcp_addba_resp_sent_event {
+	/* Used for cid less than 8. For higher cid set
+	 * CIDXTID_EXTENDED_CID_TID here and use cid and tid members instead
+	 */
 	u8 cidxtid;
 	u8 reserved;
 	__le16 status;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 cid;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 tid;
+	u8 reserved2[2];
 } __packed;
 
 /* WMI_RCP_ADDBA_REQ_EVENTID */
 struct wmi_rcp_addba_req_event {
+	/* Used for cid less than 8. For higher cid set
+	 * CIDXTID_EXTENDED_CID_TID here and use cid and tid members instead
+	 */
 	u8 cidxtid;
 	u8 dialog_token;
 	/* ieee80211_ba_parameterset as it received */
@@ -1768,6 +2009,11 @@ struct wmi_rcp_addba_req_event {
 	__le16 ba_timeout;
 	/* ieee80211_ba_seqstrl field as it received */
 	__le16 ba_seq_ctrl;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 cid;
+	/* Used when cidxtid = CIDXTID_EXTENDED_CID_TID */
+	u8 tid;
+	u8 reserved[2];
 } __packed;
 
 /* WMI_CFG_RX_CHAIN_DONE_EVENTID */
@@ -1942,6 +2188,13 @@ struct wmi_set_high_power_table_params_event {
 	u8 reserved[3];
 } __packed;
 
+/* WMI_FIXED_SCHEDULING_UL_CONFIG_EVENTID */
+struct wmi_fixed_scheduling_ul_config_event {
+	/* wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
 /* WMI_TEMP_SENSE_DONE_EVENTID
  *
  * Measure MAC and radio temperatures
@@ -2290,6 +2543,8 @@ struct wmi_link_maintain_cfg {
 	__le32 bad_beacons_num_threshold;
 	/* SNR limit for bad_beacons_detector */
 	__le32 bad_beacons_snr_threshold_db;
+	/* timeout for disassoc response frame in uSec */
+	__le32 disconnect_timeout;
 } __packed;
 
 /* WMI_LINK_MAINTAIN_CFG_WRITE_CMDID */
@@ -2519,6 +2774,7 @@ enum wmi_tof_session_end_status {
 	WMI_TOF_SESSION_END_FAIL		= 0x01,
 	WMI_TOF_SESSION_END_PARAMS_ERROR	= 0x02,
 	WMI_TOF_SESSION_END_ABORTED		= 0x03,
+	WMI_TOF_SESSION_END_BUSY		= 0x04,
 };
 
 /* WMI_TOF_SESSION_END_EVENTID */
@@ -2925,7 +3181,40 @@ struct wmi_set_silent_rssi_table_done_event {
 	__le32 table;
 } __packed;
 
-/* \WMI_COMMAND_NOT_SUPPORTED_EVENTID */
+/* WMI_VRING_SWITCH_TIMING_CONFIG_EVENTID */
+struct wmi_vring_switch_timing_config_event {
+	/* enum wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_GET_ASSOC_LIST_RES_EVENTID */
+struct wmi_assoc_sta_info {
+	u8 mac[WMI_MAC_LEN];
+	u8 omni_index_address;
+	u8 reserved;
+} __packed;
+
+#define WMI_GET_ASSOC_LIST_SIZE	(8)
+
+/* WMI_GET_ASSOC_LIST_RES_EVENTID
+ * Returns up to MAX_ASSOC_STA_LIST_SIZE associated STAs
+ */
+struct wmi_get_assoc_list_res_event {
+	struct wmi_assoc_sta_info assoc_sta_list[WMI_GET_ASSOC_LIST_SIZE];
+	/* STA count */
+	u8 count;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_BF_CONTROL_EVENTID */
+struct wmi_bf_control_event {
+	/* wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_COMMAND_NOT_SUPPORTED_EVENTID */
 struct wmi_command_not_supported_event {
 	/* device id */
 	u8 mid;
@@ -2936,4 +3225,62 @@ struct wmi_command_not_supported_event {
 	__le16 reserved1;
 } __packed;
 
+/* WMI_TSF_SYNC_CMDID */
+struct wmi_tsf_sync_cmd {
+	/* The time interval to send announce frame in one BI */
+	u8 interval_ms;
+	/* The mcs to send announce frame */
+	u8 mcs;
+	u8 reserved[6];
+} __packed;
+
+/* WMI_TSF_SYNC_STATUS_EVENTID */
+enum wmi_tsf_sync_status {
+	WMI_TSF_SYNC_SUCCESS	= 0x00,
+	WMI_TSF_SYNC_FAILED	= 0x01,
+	WMI_TSF_SYNC_REJECTED	= 0x02,
+};
+
+/* WMI_TSF_SYNC_STATUS_EVENTID */
+struct wmi_tsf_sync_status_event {
+	/* enum wmi_tsf_sync_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_GET_CCA_INDICATIONS_EVENTID */
+struct wmi_get_cca_indications_event {
+	/* wmi_fw_status */
+	u8 status;
+	/* CCA-Energy Detect in percentage over last BI (0..100) */
+	u8 cca_ed_percent;
+	/* Averaged CCA-Energy Detect in percent over number of BIs (0..100) */
+	u8 cca_ed_avg_percent;
+	/* NAV percent over last BI (0..100) */
+	u8 nav_percent;
+	/* Averaged NAV percent over number of BIs (0..100) */
+	u8 nav_avg_percent;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_SET_CCA_INDICATIONS_BI_AVG_NUM_CMDID */
+struct wmi_set_cca_indications_bi_avg_num_cmd {
+	/* set the number of bis to average cca_ed (0..255) */
+	u8 bi_number;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_SET_CCA_INDICATIONS_BI_AVG_NUM_EVENTID */
+struct wmi_set_cca_indications_bi_avg_num_event {
+	/* wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_INTERNAL_FW_SET_CHANNEL */
+struct wmi_internal_fw_set_channel_event {
+	u8 channel_num;
+	u8 reserved[3];
+} __packed;
+
 #endif /* __WILOCITY_WMI_H__ */
diff --git a/drivers/net/wireless/atmel/atmel.c b/drivers/net/wireless/atmel/atmel.c
index d122386c382b..b01dc34d55af 100644
--- a/drivers/net/wireless/atmel/atmel.c
+++ b/drivers/net/wireless/atmel/atmel.c
@@ -1482,18 +1482,6 @@ static int atmel_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int atmel_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, atmel_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations atmel_proc_fops = {
-	.open		= atmel_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static const struct net_device_ops atmel_netdev_ops = {
 	.ndo_open 		= atmel_open,
 	.ndo_stop		= atmel_close,
@@ -1614,7 +1602,8 @@ struct net_device *init_atmel_card(unsigned short irq, unsigned long port,
 
 	netif_carrier_off(dev);
 
-	if (!proc_create_data("driver/atmel", 0, NULL, &atmel_proc_fops, priv))
+	if (!proc_create_single_data("driver/atmel", 0, NULL, atmel_proc_show,
+			priv))
 		printk(KERN_WARNING "atmel: unable to create /proc entry.\n");
 
 	printk(KERN_INFO "%s: Atmel at76c50x. Version %d.%d. MAC %pM\n",
diff --git a/drivers/net/wireless/atmel/atmel_pci.c b/drivers/net/wireless/atmel/atmel_pci.c
index bcf1f274a251..30df58a41a83 100644
--- a/drivers/net/wireless/atmel/atmel_pci.c
+++ b/drivers/net/wireless/atmel/atmel_pci.c
@@ -61,8 +61,10 @@ static int atmel_pci_probe(struct pci_dev *pdev,
 	dev = init_atmel_card(pdev->irq, pdev->resource[1].start,
 			      ATMEL_FW_TYPE_506,
 			      &pdev->dev, NULL, NULL);
-	if (!dev)
+	if (!dev) {
+		pci_disable_device(pdev);
 		return -ENODEV;
+	}
 
 	pci_set_drvdata(pdev, dev);
 	return 0;
diff --git a/drivers/net/wireless/broadcom/b43/dma.c b/drivers/net/wireless/broadcom/b43/dma.c
index 6837064908be..6b0e1ec346cb 100644
--- a/drivers/net/wireless/broadcom/b43/dma.c
+++ b/drivers/net/wireless/broadcom/b43/dma.c
@@ -1484,7 +1484,7 @@ void b43_dma_handle_txstatus(struct b43_wldev *dev,
 	int slot, firstused;
 	bool frame_succeed;
 	int skip;
-	static u8 err_out1, err_out2;
+	static u8 err_out1;
 
 	ring = parse_cookie(dev, status->cookie, &slot);
 	if (unlikely(!ring))
@@ -1518,13 +1518,13 @@ void b43_dma_handle_txstatus(struct b43_wldev *dev,
 			}
 		} else {
 			/* More than a single header/data pair were missed.
-			 * Report this error once.
+			 * Report this error, and reset the controller to
+			 * revive operation.
 			 */
-			if (!err_out2)
-				b43dbg(dev->wl,
-				       "Out of order TX status report on DMA ring %d. Expected %d, but got %d\n",
-				       ring->index, firstused, slot);
-			err_out2 = 1;
+			b43dbg(dev->wl,
+			       "Out of order TX status report on DMA ring %d. Expected %d, but got %d\n",
+			       ring->index, firstused, slot);
+			b43_controller_restart(dev, "Out of order TX");
 			return;
 		}
 	}
diff --git a/drivers/net/wireless/broadcom/b43legacy/dma.c b/drivers/net/wireless/broadcom/b43legacy/dma.c
index cfa617ddb2f1..2f0c64cef65f 100644
--- a/drivers/net/wireless/broadcom/b43legacy/dma.c
+++ b/drivers/net/wireless/broadcom/b43legacy/dma.c
@@ -1064,7 +1064,7 @@ static int dma_tx_fragment(struct b43legacy_dmaring *ring,
 	meta->dmaaddr = map_descbuffer(ring, skb->data, skb->len, 1);
 	/* create a bounce buffer in zone_dma on mapping failure. */
 	if (b43legacy_dma_mapping_error(ring, meta->dmaaddr, skb->len, 1)) {
-		bounce_skb = alloc_skb(skb->len, GFP_ATOMIC | GFP_DMA);
+		bounce_skb = alloc_skb(skb->len, GFP_KERNEL | GFP_DMA);
 		if (!bounce_skb) {
 			ring->current_slot = old_top_slot;
 			ring->used_slots = old_used_slots;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
index 0b68240ec7b4..d2f788d88668 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
@@ -963,6 +963,7 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = {
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43340),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43341),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43362),
+ 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43364),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4335_4339),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4339),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43430),
@@ -1164,6 +1165,7 @@ static struct sdio_driver brcmf_sdmmc_driver = {
 #ifdef CONFIG_PM_SLEEP
 		.pm = &brcmf_sdio_pm_ops,
 #endif	/* CONFIG_PM_SLEEP */
+		.coredump = brcmf_dev_coredump,
 	},
 };
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h
index 27e693e93f21..c4965184cdf3 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h
@@ -250,6 +250,8 @@ int brcmf_attach(struct device *dev, struct brcmf_mp_device *settings);
 void brcmf_detach(struct device *dev);
 /* Indication from bus module that dongle should be reset */
 void brcmf_dev_reset(struct device *dev);
+/* Request from bus module to initiate a coredump */
+void brcmf_dev_coredump(struct device *dev);
 
 /* Configure the "global" bus state used by upper layers */
 void brcmf_bus_change_state(struct brcmf_bus *bus, enum brcmf_bus_state state);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
index 89b86251910e..f5b405c98047 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
@@ -2728,9 +2728,8 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg,
 				   struct brcmf_bss_info_le *bi)
 {
 	struct wiphy *wiphy = cfg_to_wiphy(cfg);
-	struct ieee80211_channel *notify_channel;
 	struct cfg80211_bss *bss;
-	struct ieee80211_supported_band *band;
+	enum nl80211_band band;
 	struct brcmu_chan ch;
 	u16 channel;
 	u32 freq;
@@ -2738,7 +2737,7 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg,
 	u16 notify_interval;
 	u8 *notify_ie;
 	size_t notify_ielen;
-	s32 notify_signal;
+	struct cfg80211_inform_bss bss_data = {};
 
 	if (le32_to_cpu(bi->length) > WL_BSS_INFO_MAX) {
 		brcmf_err("Bss info is larger than buffer. Discarding\n");
@@ -2753,32 +2752,33 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg,
 	channel = bi->ctl_ch;
 
 	if (channel <= CH_MAX_2G_CHANNEL)
-		band = wiphy->bands[NL80211_BAND_2GHZ];
+		band = NL80211_BAND_2GHZ;
 	else
-		band = wiphy->bands[NL80211_BAND_5GHZ];
+		band = NL80211_BAND_5GHZ;
 
-	freq = ieee80211_channel_to_frequency(channel, band->band);
-	notify_channel = ieee80211_get_channel(wiphy, freq);
+	freq = ieee80211_channel_to_frequency(channel, band);
+	bss_data.chan = ieee80211_get_channel(wiphy, freq);
+	bss_data.scan_width = NL80211_BSS_CHAN_WIDTH_20;
+	bss_data.boottime_ns = ktime_to_ns(ktime_get_boottime());
 
 	notify_capability = le16_to_cpu(bi->capability);
 	notify_interval = le16_to_cpu(bi->beacon_period);
 	notify_ie = (u8 *)bi + le16_to_cpu(bi->ie_offset);
 	notify_ielen = le32_to_cpu(bi->ie_length);
-	notify_signal = (s16)le16_to_cpu(bi->RSSI) * 100;
+	bss_data.signal = (s16)le16_to_cpu(bi->RSSI) * 100;
 
 	brcmf_dbg(CONN, "bssid: %pM\n", bi->BSSID);
 	brcmf_dbg(CONN, "Channel: %d(%d)\n", channel, freq);
 	brcmf_dbg(CONN, "Capability: %X\n", notify_capability);
 	brcmf_dbg(CONN, "Beacon interval: %d\n", notify_interval);
-	brcmf_dbg(CONN, "Signal: %d\n", notify_signal);
+	brcmf_dbg(CONN, "Signal: %d\n", bss_data.signal);
 
-	bss = cfg80211_inform_bss(wiphy, notify_channel,
-				  CFG80211_BSS_FTYPE_UNKNOWN,
-				  (const u8 *)bi->BSSID,
-				  0, notify_capability,
-				  notify_interval, notify_ie,
-				  notify_ielen, notify_signal,
-				  GFP_KERNEL);
+	bss = cfg80211_inform_bss_data(wiphy, &bss_data,
+				       CFG80211_BSS_FTYPE_UNKNOWN,
+				       (const u8 *)bi->BSSID,
+				       0, notify_capability,
+				       notify_interval, notify_ie,
+				       notify_ielen, GFP_KERNEL);
 
 	if (!bss)
 		return -ENOMEM;
@@ -5498,7 +5498,7 @@ brcmf_notify_connect_status_ap(struct brcmf_cfg80211_info *cfg,
 	static int generation;
 	u32 event = e->event_code;
 	u32 reason = e->reason;
-	struct station_info sinfo;
+	struct station_info *sinfo;
 
 	brcmf_dbg(CONN, "event %s (%u), reason %d\n",
 		  brcmf_fweh_event_name(event), event, reason);
@@ -5511,16 +5511,22 @@ brcmf_notify_connect_status_ap(struct brcmf_cfg80211_info *cfg,
 
 	if (((event == BRCMF_E_ASSOC_IND) || (event == BRCMF_E_REASSOC_IND)) &&
 	    (reason == BRCMF_E_STATUS_SUCCESS)) {
-		memset(&sinfo, 0, sizeof(sinfo));
 		if (!data) {
 			brcmf_err("No IEs present in ASSOC/REASSOC_IND");
 			return -EINVAL;
 		}
-		sinfo.assoc_req_ies = data;
-		sinfo.assoc_req_ies_len = e->datalen;
+
+		sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL);
+		if (!sinfo)
+			return -ENOMEM;
+
+		sinfo->assoc_req_ies = data;
+		sinfo->assoc_req_ies_len = e->datalen;
 		generation++;
-		sinfo.generation = generation;
-		cfg80211_new_sta(ndev, e->addr, &sinfo, GFP_KERNEL);
+		sinfo->generation = generation;
+		cfg80211_new_sta(ndev, e->addr, sinfo, GFP_KERNEL);
+
+		kfree(sinfo);
 	} else if ((event == BRCMF_E_DISASSOC_IND) ||
 		   (event == BRCMF_E_DEAUTH_IND) ||
 		   (event == BRCMF_E_DEAUTH)) {
@@ -6512,6 +6518,7 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp)
 
 	wiphy->flags |= WIPHY_FLAG_NETNS_OK |
 			WIPHY_FLAG_PS_ON_BY_DEFAULT |
+			WIPHY_FLAG_HAVE_AP_SME |
 			WIPHY_FLAG_OFFCHAN_TX |
 			WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL;
 	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_TDLS))
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
index 3b829fed8631..927d62b3d41b 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
@@ -689,6 +689,7 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci)
 	case BRCM_CC_43525_CHIP_ID:
 	case BRCM_CC_4365_CHIP_ID:
 	case BRCM_CC_4366_CHIP_ID:
+	case BRCM_CC_43664_CHIP_ID:
 		return 0x200000;
 	case CY_CC_4373_CHIP_ID:
 		return 0x160000;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
index 105b8774fca9..cd3651069d0c 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
@@ -36,8 +36,6 @@ MODULE_AUTHOR("Broadcom Corporation");
 MODULE_DESCRIPTION("Broadcom 802.11 wireless LAN fullmac driver.");
 MODULE_LICENSE("Dual BSD/GPL");
 
-const u8 ALLFFMAC[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
-
 #define BRCMF_DEFAULT_SCAN_CHANNEL_TIME	40
 #define BRCMF_DEFAULT_SCAN_UNASSOC_TIME	40
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
index ef914619e8e1..a34642cb4d2f 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
@@ -19,8 +19,6 @@
 #include <linux/platform_data/brcmfmac.h>
 #include "fwil_types.h"
 
-extern const u8 ALLFFMAC[ETH_ALEN];
-
 #define BRCMF_FW_ALTPATH_LEN			256
 
 /* Definitions for the module global and device specific settings are defined
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
index 8d4511eaa9b9..72954fd6df3b 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
@@ -1180,6 +1180,14 @@ void brcmf_dev_reset(struct device *dev)
 		brcmf_fil_cmd_int_set(drvr->iflist[0], BRCMF_C_TERMINATED, 1);
 }
 
+void brcmf_dev_coredump(struct device *dev)
+{
+	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
+
+	if (brcmf_debug_create_memdump(bus_if, NULL, 0) < 0)
+		brcmf_dbg(TRACE, "failed to create coredump\n");
+}
+
 void brcmf_detach(struct device *dev)
 {
 	s32 i;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c
index 504832084eca..489b5dfdf5b9 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c
@@ -40,7 +40,8 @@ int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data,
 	if (!dump)
 		return -ENOMEM;
 
-	memcpy(dump, data, len);
+	if (data && len > 0)
+		memcpy(dump, data, len);
 	err = brcmf_bus_get_memdump(bus, dump + len, ramsize);
 	if (err) {
 		vfree(dump);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
index 876731c57bf5..800a423c7bc2 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
@@ -165,6 +165,41 @@ static void brcmf_feat_firmware_capabilities(struct brcmf_if *ifp)
 	}
 }
 
+/**
+ * brcmf_feat_fwcap_debugfs_read() - expose firmware capabilities to debugfs.
+ *
+ * @seq: sequence for debugfs entry.
+ * @data: raw data pointer.
+ */
+static int brcmf_feat_fwcap_debugfs_read(struct seq_file *seq, void *data)
+{
+	struct brcmf_bus *bus_if = dev_get_drvdata(seq->private);
+	struct brcmf_if *ifp = brcmf_get_ifp(bus_if->drvr, 0);
+	char caps[MAX_CAPS_BUFFER_SIZE + 1] = { };
+	char *tmp;
+	int err;
+
+	err = brcmf_fil_iovar_data_get(ifp, "cap", caps, sizeof(caps));
+	if (err) {
+		brcmf_err("could not get firmware cap (%d)\n", err);
+		return err;
+	}
+
+	/* Put every capability in a new line */
+	for (tmp = caps; *tmp; tmp++) {
+		if (*tmp == ' ')
+			*tmp = '\n';
+	}
+
+	/* Usually there is a space at the end of capabilities string */
+	seq_printf(seq, "%s", caps);
+	/* So make sure we don't print two line breaks */
+	if (tmp > caps && *(tmp - 1) != '\n')
+		seq_printf(seq, "\n");
+
+	return 0;
+}
+
 void brcmf_feat_attach(struct brcmf_pub *drvr)
 {
 	struct brcmf_if *ifp = brcmf_get_ifp(drvr, 0);
@@ -233,6 +268,7 @@ void brcmf_feat_attach(struct brcmf_pub *drvr)
 void brcmf_feat_debugfs_create(struct brcmf_pub *drvr)
 {
 	brcmf_debugfs_add_entry(drvr, "features", brcmf_feat_debugfs_read);
+	brcmf_debugfs_add_entry(drvr, "fwcap", brcmf_feat_fwcap_debugfs_read);
 }
 
 bool brcmf_feat_is_enabled(struct brcmf_if *ifp, enum brcmf_feat_id id)
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
index 94e177d7c9b5..9095b830ae4d 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
@@ -634,7 +634,7 @@ int brcmf_fw_get_firmwares(struct device *dev, struct brcmf_fw_request *req,
 
 struct brcmf_fw_request *
 brcmf_fw_alloc_request(u32 chip, u32 chiprev,
-		       struct brcmf_firmware_mapping mapping_table[],
+		       const struct brcmf_firmware_mapping mapping_table[],
 		       u32 table_size, struct brcmf_fw_name *fwnames,
 		       u32 n_fwnames)
 {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h
index 79a21095c349..2893e56910f0 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h
@@ -80,7 +80,7 @@ struct brcmf_fw_name {
 
 struct brcmf_fw_request *
 brcmf_fw_alloc_request(u32 chip, u32 chiprev,
-		       struct brcmf_firmware_mapping mapping_table[],
+		       const struct brcmf_firmware_mapping mapping_table[],
 		       u32 table_size, struct brcmf_fw_name *fwnames,
 		       u32 n_fwnames);
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c
index d0b738da2458..d0d8b32af7d0 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c
@@ -46,6 +46,8 @@ static const u8 brcmf_flowring_prio2fifo[] = {
 	3
 };
 
+static const u8 ALLFFMAC[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+
 
 static bool
 brcmf_flowring_is_tdls_mac(struct brcmf_flowring *flow, u8 mac[ETH_ALEN])
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h
index f93ba6be1ef8..692235d25277 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h
@@ -27,8 +27,10 @@
 #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE	40
 #define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE	32
 #define BRCMF_D2H_MSGRING_CONTROL_COMPLETE_ITEMSIZE	24
-#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE		16
-#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE		32
+#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE_PRE_V7	16
+#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE		24
+#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE_PRE_V7	32
+#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE		40
 #define BRCMF_H2D_TXFLOWRING_ITEMSIZE			48
 
 struct msgbuf_buf_addr {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
index bcef208a81a5..4b2149b48362 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
@@ -2073,6 +2073,13 @@ static struct wireless_dev *brcmf_p2p_create_p2pdev(struct brcmf_p2p_info *p2p,
 	}
 
 	pri_ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp;
+
+	/* firmware requires unique mac address for p2pdev interface */
+	if (addr && ether_addr_equal(addr, pri_ifp->mac_addr)) {
+		brcmf_err("discovery vif must be different from primary interface\n");
+		return ERR_PTR(-EINVAL);
+	}
+
 	brcmf_p2p_generate_bss_mac(p2p, addr);
 	brcmf_p2p_set_firmware(pri_ifp, p2p->dev_addr);
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
index 091c191ce259..45928b5b8d97 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
@@ -59,7 +59,7 @@ BRCMF_FW_DEF(4366B, "brcmfmac4366b-pcie");
 BRCMF_FW_DEF(4366C, "brcmfmac4366c-pcie");
 BRCMF_FW_DEF(4371, "brcmfmac4371-pcie");
 
-static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
+static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602),
 	BRCMF_FW_ENTRY(BRCM_CC_43465_CHIP_ID, 0xFFFFFFF0, 4366C),
 	BRCMF_FW_ENTRY(BRCM_CC_4350_CHIP_ID, 0x000000FF, 4350C),
@@ -75,6 +75,7 @@ static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_4365_CHIP_ID, 0xFFFFFFF0, 4365C),
 	BRCMF_FW_ENTRY(BRCM_CC_4366_CHIP_ID, 0x0000000F, 4366B),
 	BRCMF_FW_ENTRY(BRCM_CC_4366_CHIP_ID, 0xFFFFFFF0, 4366C),
+	BRCMF_FW_ENTRY(BRCM_CC_43664_CHIP_ID, 0xFFFFFFF0, 4366C),
 	BRCMF_FW_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371),
 };
 
@@ -104,7 +105,8 @@ static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 #define BRCMF_PCIE_PCIE2REG_MAILBOXMASK		0x4C
 #define BRCMF_PCIE_PCIE2REG_CONFIGADDR		0x120
 #define BRCMF_PCIE_PCIE2REG_CONFIGDATA		0x124
-#define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX		0x140
+#define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0	0x140
+#define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1	0x144
 
 #define BRCMF_PCIE2_INTA			0x01
 #define BRCMF_PCIE2_INTB			0x02
@@ -134,11 +136,13 @@ static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 						 BRCMF_PCIE_MB_INT_D2H3_DB0 | \
 						 BRCMF_PCIE_MB_INT_D2H3_DB1)
 
+#define BRCMF_PCIE_SHARED_VERSION_7		7
 #define BRCMF_PCIE_MIN_SHARED_VERSION		5
-#define BRCMF_PCIE_MAX_SHARED_VERSION		6
+#define BRCMF_PCIE_MAX_SHARED_VERSION		BRCMF_PCIE_SHARED_VERSION_7
 #define BRCMF_PCIE_SHARED_VERSION_MASK		0x00FF
 #define BRCMF_PCIE_SHARED_DMA_INDEX		0x10000
 #define BRCMF_PCIE_SHARED_DMA_2B_IDX		0x100000
+#define BRCMF_PCIE_SHARED_HOSTRDY_DB1		0x10000000
 
 #define BRCMF_PCIE_FLAGS_HTOD_SPLIT		0x4000
 #define BRCMF_PCIE_FLAGS_DTOH_SPLIT		0x8000
@@ -178,6 +182,7 @@ static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 #define BRCMF_D2H_DEV_D3_ACK			0x00000001
 #define BRCMF_D2H_DEV_DS_ENTER_REQ		0x00000002
 #define BRCMF_D2H_DEV_DS_EXIT_NOTE		0x00000004
+#define BRCMF_D2H_DEV_FWHALT			0x10000000
 
 #define BRCMF_H2D_HOST_D3_INFORM		0x00000001
 #define BRCMF_H2D_HOST_DS_ACK			0x00000002
@@ -315,6 +320,14 @@ static const u32 brcmf_ring_max_item[BRCMF_NROF_COMMON_MSGRINGS] = {
 	BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM
 };
 
+static const u32 brcmf_ring_itemsize_pre_v7[BRCMF_NROF_COMMON_MSGRINGS] = {
+	BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE,
+	BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE,
+	BRCMF_D2H_MSGRING_CONTROL_COMPLETE_ITEMSIZE,
+	BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE_PRE_V7,
+	BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE_PRE_V7
+};
+
 static const u32 brcmf_ring_itemsize[BRCMF_NROF_COMMON_MSGRINGS] = {
 	BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE,
 	BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE,
@@ -705,6 +718,10 @@ static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo)
 		devinfo->mbdata_completed = true;
 		wake_up(&devinfo->mbdata_resp_wait);
 	}
+	if (dtoh_mb_data & BRCMF_D2H_DEV_FWHALT) {
+		brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n");
+		brcmf_dev_coredump(&devinfo->pdev->dev);
+	}
 }
 
 
@@ -781,6 +798,12 @@ static void brcmf_pcie_intr_enable(struct brcmf_pciedev_info *devinfo)
 			       BRCMF_PCIE_MB_INT_FN0_1);
 }
 
+static void brcmf_pcie_hostready(struct brcmf_pciedev_info *devinfo)
+{
+	if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1)
+		brcmf_pcie_write_reg32(devinfo,
+				       BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, 1);
+}
 
 static irqreturn_t brcmf_pcie_quick_check_isr(int irq, void *arg)
 {
@@ -923,7 +946,7 @@ static int brcmf_pcie_ring_mb_ring_bell(void *ctx)
 
 	brcmf_dbg(PCIE, "RING !\n");
 	/* Any arbitrary value will do, lets use 1 */
-	brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX, 1);
+	brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, 1);
 
 	return 0;
 }
@@ -998,8 +1021,14 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id,
 	struct brcmf_pcie_ringbuf *ring;
 	u32 size;
 	u32 addr;
+	const u32 *ring_itemsize_array;
+
+	if (devinfo->shared.version < BRCMF_PCIE_SHARED_VERSION_7)
+		ring_itemsize_array = brcmf_ring_itemsize_pre_v7;
+	else
+		ring_itemsize_array = brcmf_ring_itemsize;
 
-	size = brcmf_ring_max_item[ring_id] * brcmf_ring_itemsize[ring_id];
+	size = brcmf_ring_max_item[ring_id] * ring_itemsize_array[ring_id];
 	dma_buf = brcmf_pcie_init_dmabuffer_for_device(devinfo, size,
 			tcm_ring_phys_addr + BRCMF_RING_MEM_BASE_ADDR_OFFSET,
 			&dma_handle);
@@ -1009,7 +1038,7 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id,
 	addr = tcm_ring_phys_addr + BRCMF_RING_MAX_ITEM_OFFSET;
 	brcmf_pcie_write_tcm16(devinfo, addr, brcmf_ring_max_item[ring_id]);
 	addr = tcm_ring_phys_addr + BRCMF_RING_LEN_ITEMS_OFFSET;
-	brcmf_pcie_write_tcm16(devinfo, addr, brcmf_ring_itemsize[ring_id]);
+	brcmf_pcie_write_tcm16(devinfo, addr, ring_itemsize_array[ring_id]);
 
 	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
 	if (!ring) {
@@ -1018,7 +1047,7 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id,
 		return NULL;
 	}
 	brcmf_commonring_config(&ring->commonring, brcmf_ring_max_item[ring_id],
-				brcmf_ring_itemsize[ring_id], dma_buf);
+				ring_itemsize_array[ring_id], dma_buf);
 	ring->dma_handle = dma_handle;
 	ring->devinfo = devinfo;
 	brcmf_commonring_register_cb(&ring->commonring,
@@ -1727,6 +1756,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret,
 	init_waitqueue_head(&devinfo->mbdata_resp_wait);
 
 	brcmf_pcie_intr_enable(devinfo);
+	brcmf_pcie_hostready(devinfo);
 	if (brcmf_attach(&devinfo->pdev->dev, devinfo->settings) == 0)
 		return;
 
@@ -1949,6 +1979,7 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev)
 		brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2);
 		brcmf_bus_change_state(bus, BRCMF_BUS_UP);
 		brcmf_pcie_intr_enable(devinfo);
+		brcmf_pcie_hostready(devinfo);
 		return 0;
 	}
 
@@ -2018,6 +2049,7 @@ static struct pci_driver brcmf_pciedrvr = {
 #ifdef CONFIG_PM
 	.driver.pm = &brcmf_pciedrvr_pm,
 #endif
+	.driver.coredump = brcmf_dev_coredump,
 };
 
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
index 1037df7297bb..c99a191e8d69 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
@@ -619,7 +619,7 @@ BRCMF_FW_DEF(4354, "brcmfmac4354-sdio");
 BRCMF_FW_DEF(4356, "brcmfmac4356-sdio");
 BRCMF_FW_DEF(4373, "brcmfmac4373-sdio");
 
-static struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = {
+static const struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143),
 	BRCMF_FW_ENTRY(BRCM_CC_43241_CHIP_ID, 0x0000001F, 43241B0),
 	BRCMF_FW_ENTRY(BRCM_CC_43241_CHIP_ID, 0x00000020, 43241B4),
@@ -1072,8 +1072,10 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus)
 	bus->sdcnt.f1regdata += 2;
 
 	/* dongle indicates the firmware has halted/crashed */
-	if (hmb_data & HMB_DATA_FWHALT)
+	if (hmb_data & HMB_DATA_FWHALT) {
 		brcmf_err("mailbox indicates firmware halted\n");
+		brcmf_dev_coredump(&sdiod->func1->dev);
+	}
 
 	/* Dongle recomposed rx frames, accept them again */
 	if (hmb_data & HMB_DATA_NAKHANDLED) {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
index a0873adcc01c..a4308c6e72d7 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
@@ -52,7 +52,7 @@ BRCMF_FW_DEF(43242A, "brcmfmac43242a");
 BRCMF_FW_DEF(43569, "brcmfmac43569");
 BRCMF_FW_DEF(4373, "brcmfmac4373");
 
-static struct brcmf_firmware_mapping brcmf_usb_fwnames[] = {
+static const struct brcmf_firmware_mapping brcmf_usb_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143),
 	BRCMF_FW_ENTRY(BRCM_CC_43235_CHIP_ID, 0x00000008, 43236B),
 	BRCMF_FW_ENTRY(BRCM_CC_43236_CHIP_ID, 0x00000008, 43236B),
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c
index 93d4cde0eb31..9d830d27b229 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c
@@ -3388,13 +3388,8 @@ void wlc_lcnphy_deaf_mode(struct brcms_phy *pi, bool mode)
 	u8 phybw40;
 	phybw40 = CHSPEC_IS40(pi->radio_chanspec);
 
-	if (LCNREV_LT(pi->pubpi.phy_rev, 2)) {
-		mod_phy_reg(pi, 0x4b0, (0x1 << 5), (mode) << 5);
-		mod_phy_reg(pi, 0x4b1, (0x1 << 9), 0 << 9);
-	} else {
-		mod_phy_reg(pi, 0x4b0, (0x1 << 5), (mode) << 5);
-		mod_phy_reg(pi, 0x4b1, (0x1 << 9), 0 << 9);
-	}
+	mod_phy_reg(pi, 0x4b0, (0x1 << 5), (mode) << 5);
+	mod_phy_reg(pi, 0x4b1, (0x1 << 9), 0 << 9);
 
 	if (phybw40 == 0) {
 		mod_phy_reg((pi), 0x410,
diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
index 57544a3a3ce4..686f7a85a045 100644
--- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
+++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
@@ -57,6 +57,7 @@
 #define BRCM_CC_43602_CHIP_ID		43602
 #define BRCM_CC_4365_CHIP_ID		0x4365
 #define BRCM_CC_4366_CHIP_ID		0x4366
+#define BRCM_CC_43664_CHIP_ID		43664
 #define BRCM_CC_4371_CHIP_ID		0x4371
 #define CY_CC_4373_CHIP_ID		0x4373
 
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
index 236b52423506..7c4f550a1475 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
@@ -3732,7 +3732,7 @@ IPW2100_ORD(STAT_TX_HOST_REQUESTS, "requested Host Tx's (MSDU)"),
 	    IPW2100_ORD(ASSOCIATED_AP_PTR,
 				"0 if not associated, else pointer to AP table entry"),
 	    IPW2100_ORD(AVAILABLE_AP_CNT,
-				"AP's decsribed in the AP table"),
+				"AP's described in the AP table"),
 	    IPW2100_ORD(AP_LIST_PTR, "Ptr to list of available APs"),
 	    IPW2100_ORD(STAT_AP_ASSNS, "associations"),
 	    IPW2100_ORD(STAT_ASSN_FAIL, "association failures"),
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.h b/drivers/net/wireless/intel/ipw2x00/ipw2100.h
index 193947865efd..ce3e35f6b60f 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2100.h
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.h
@@ -1009,7 +1009,7 @@ typedef enum _ORDINAL_TABLE_1 {	// NS - means Not Supported by FW
 	IPW_ORD_STAT_PERCENT_RETRIES,	// current calculation of % missed tx retries
 	IPW_ORD_ASSOCIATED_AP_PTR,	// If associated, this is ptr to the associated
 	// AP table entry. set to 0 if not associated
-	IPW_ORD_AVAILABLE_AP_CNT,	// # of AP's decsribed in the AP table
+	IPW_ORD_AVAILABLE_AP_CNT,	// # of AP's described in the AP table
 	IPW_ORD_AP_LIST_PTR,	// Ptr to list of available APs
 	IPW_ORD_STAT_AP_ASSNS,	// # of associations
 	IPW_ORD_STAT_ASSN_FAIL,	// # of association failures
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.c b/drivers/net/wireless/intel/ipw2x00/ipw2200.c
index 87a5e414c2f7..ba3fb1d2ddb4 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2200.c
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.c
@@ -12012,7 +12012,7 @@ MODULE_PARM_DESC(rtap_iface, "create the rtap interface (1 - create, default 0)"
 
 #ifdef CONFIG_IPW2200_QOS
 module_param(qos_enable, int, 0444);
-MODULE_PARM_DESC(qos_enable, "enable all QoS functionalitis");
+MODULE_PARM_DESC(qos_enable, "enable all QoS functionalities");
 
 module_param(qos_burst_enable, int, 0444);
 MODULE_PARM_DESC(qos_burst_enable, "enable QoS burst mode");
diff --git a/drivers/net/wireless/intel/iwlwifi/Makefile b/drivers/net/wireless/intel/iwlwifi/Makefile
index e6205eae51fd..4d08d78c6b71 100644
--- a/drivers/net/wireless/intel/iwlwifi/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/Makefile
@@ -13,7 +13,7 @@ iwlwifi-$(CONFIG_IWLMVM) += cfg/7000.o cfg/8000.o cfg/9000.o cfg/22000.o
 iwlwifi-objs		+= iwl-trans.o
 iwlwifi-objs		+= fw/notif-wait.o
 iwlwifi-$(CONFIG_IWLMVM) += fw/paging.o fw/smem.o fw/init.o fw/dbg.o
-iwlwifi-$(CONFIG_IWLMVM) += fw/common_rx.o fw/nvm.o
+iwlwifi-$(CONFIG_IWLMVM) += fw/common_rx.o
 iwlwifi-$(CONFIG_ACPI) += fw/acpi.o
 iwlwifi-$(CONFIG_IWLWIFI_DEBUGFS) += fw/debugfs.o
 
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/1000.c b/drivers/net/wireless/intel/iwlwifi/cfg/1000.c
index b2573b1d1506..591687984962 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/1000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/1000.c
@@ -1,6 +1,7 @@
 /******************************************************************************
  *
  * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of version 2 of the GNU General Public License as
@@ -27,7 +28,6 @@
 #include <linux/module.h>
 #include <linux/stringify.h>
 #include "iwl-config.h"
-#include "iwl-csr.h"
 #include "iwl-agn-hw.h"
 
 /* Highest firmware API version supported */
@@ -91,7 +91,8 @@ static const struct iwl_eeprom_params iwl1000_eeprom_params = {
 	.base_params = &iwl1000_base_params,			\
 	.eeprom_params = &iwl1000_eeprom_params,		\
 	.led_mode = IWL_LED_BLINK,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl1000_bgn_cfg = {
 	.name = "Intel(R) Centrino(R) Wireless-N 1000 BGN",
@@ -117,7 +118,8 @@ const struct iwl_cfg iwl1000_bg_cfg = {
 	.eeprom_params = &iwl1000_eeprom_params,		\
 	.led_mode = IWL_LED_RF_STATE,				\
 	.rx_with_siso_diversity = true,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl100_bgn_cfg = {
 	.name = "Intel(R) Centrino(R) Wireless-N 100 BGN",
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/2000.c b/drivers/net/wireless/intel/iwlwifi/cfg/2000.c
index 1b32ad413b9e..a63ca8820568 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/2000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/2000.c
@@ -1,6 +1,7 @@
 /******************************************************************************
  *
  * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of version 2 of the GNU General Public License as
@@ -115,7 +116,8 @@ static const struct iwl_eeprom_params iwl20x0_eeprom_params = {
 	.base_params = &iwl2000_base_params,			\
 	.eeprom_params = &iwl20x0_eeprom_params,		\
 	.led_mode = IWL_LED_RF_STATE,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 
 const struct iwl_cfg iwl2000_2bgn_cfg = {
@@ -142,7 +144,8 @@ const struct iwl_cfg iwl2000_2bgn_d_cfg = {
 	.base_params = &iwl2030_base_params,			\
 	.eeprom_params = &iwl20x0_eeprom_params,		\
 	.led_mode = IWL_LED_RF_STATE,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl2030_2bgn_cfg = {
 	.name = "Intel(R) Centrino(R) Wireless-N 2230 BGN",
@@ -163,7 +166,8 @@ const struct iwl_cfg iwl2030_2bgn_cfg = {
 	.eeprom_params = &iwl20x0_eeprom_params,		\
 	.led_mode = IWL_LED_RF_STATE,				\
 	.rx_with_siso_diversity = true,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl105_bgn_cfg = {
 	.name = "Intel(R) Centrino(R) Wireless-N 105 BGN",
@@ -190,7 +194,8 @@ const struct iwl_cfg iwl105_bgn_d_cfg = {
 	.eeprom_params = &iwl20x0_eeprom_params,		\
 	.led_mode = IWL_LED_RF_STATE,				\
 	.rx_with_siso_diversity = true,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl135_bgn_cfg = {
 	.name = "Intel(R) Centrino(R) Wireless-N 135 BGN",
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
index dffd9df782b0..d4ba66aecdc9 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
@@ -54,7 +54,6 @@
 #include <linux/module.h>
 #include <linux/stringify.h>
 #include "iwl-config.h"
-#include "iwl-agn-hw.h"
 
 /* Highest firmware API version supported */
 #define IWL_22000_UCODE_API_MAX	38
@@ -115,8 +114,6 @@ static const struct iwl_ht_params iwl_22000_ht_params = {
 	.ucode_api_max = IWL_22000_UCODE_API_MAX,			\
 	.ucode_api_min = IWL_22000_UCODE_API_MIN,			\
 	.device_family = IWL_DEVICE_FAMILY_22000,			\
-	.max_inst_size = IWL60_RTC_INST_SIZE,				\
-	.max_data_size = IWL60_RTC_DATA_SIZE,				\
 	.base_params = &iwl_22000_base_params,				\
 	.led_mode = IWL_LED_RF_STATE,					\
 	.nvm_hw_section_num = NVM_HW_SECTION_NUM_FAMILY_22000,		\
@@ -137,13 +134,13 @@ static const struct iwl_ht_params iwl_22000_ht_params = {
 	.gen2 = true,							\
 	.nvm_type = IWL_NVM_EXT,					\
 	.dbgc_supported = true,						\
-	.tx_cmd_queue_size = 32,					\
 	.min_umac_error_event_table = 0x400000
 
 const struct iwl_cfg iwl22000_2ac_cfg_hr = {
 	.name = "Intel(R) Dual Band Wireless AC 22000",
 	.fw_name_pre = IWL_22000_HR_FW_PRE,
 	IWL_DEVICE_22000,
+	.csr = &iwl_csr_v1,
 	.ht_params = &iwl_22000_ht_params,
 	.nvm_ver = IWL_22000_NVM_VERSION,
 	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
@@ -154,6 +151,7 @@ const struct iwl_cfg iwl22000_2ac_cfg_hr_cdb = {
 	.name = "Intel(R) Dual Band Wireless AC 22000",
 	.fw_name_pre = IWL_22000_HR_CDB_FW_PRE,
 	IWL_DEVICE_22000,
+	.csr = &iwl_csr_v1,
 	.ht_params = &iwl_22000_ht_params,
 	.nvm_ver = IWL_22000_NVM_VERSION,
 	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
@@ -165,6 +163,7 @@ const struct iwl_cfg iwl22000_2ac_cfg_jf = {
 	.name = "Intel(R) Dual Band Wireless AC 22000",
 	.fw_name_pre = IWL_22000_JF_FW_PRE,
 	IWL_DEVICE_22000,
+	.csr = &iwl_csr_v1,
 	.ht_params = &iwl_22000_ht_params,
 	.nvm_ver = IWL_22000_NVM_VERSION,
 	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
@@ -175,6 +174,7 @@ const struct iwl_cfg iwl22000_2ax_cfg_hr = {
 	.name = "Intel(R) Dual Band Wireless AX 22000",
 	.fw_name_pre = IWL_22000_HR_FW_PRE,
 	IWL_DEVICE_22000,
+	.csr = &iwl_csr_v1,
 	.ht_params = &iwl_22000_ht_params,
 	.nvm_ver = IWL_22000_NVM_VERSION,
 	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
@@ -185,6 +185,7 @@ const struct iwl_cfg iwl22000_2ax_cfg_qnj_hr_f0 = {
 	.name = "Intel(R) Dual Band Wireless AX 22000",
 	.fw_name_pre = IWL_22000_HR_F0_FW_PRE,
 	IWL_DEVICE_22000,
+	.csr = &iwl_csr_v1,
 	.ht_params = &iwl_22000_ht_params,
 	.nvm_ver = IWL_22000_NVM_VERSION,
 	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
@@ -195,6 +196,7 @@ const struct iwl_cfg iwl22000_2ax_cfg_qnj_jf_b0 = {
 	.name = "Intel(R) Dual Band Wireless AX 22000",
 	.fw_name_pre = IWL_22000_JF_B0_FW_PRE,
 	IWL_DEVICE_22000,
+	.csr = &iwl_csr_v1,
 	.ht_params = &iwl_22000_ht_params,
 	.nvm_ver = IWL_22000_NVM_VERSION,
 	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
@@ -205,6 +207,7 @@ const struct iwl_cfg iwl22000_2ax_cfg_qnj_hr_a0 = {
 	.name = "Intel(R) Dual Band Wireless AX 22000",
 	.fw_name_pre = IWL_22000_HR_A0_FW_PRE,
 	IWL_DEVICE_22000,
+	.csr = &iwl_csr_v1,
 	.ht_params = &iwl_22000_ht_params,
 	.nvm_ver = IWL_22000_NVM_VERSION,
 	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/5000.c b/drivers/net/wireless/intel/iwlwifi/cfg/5000.c
index 4aa8f0a05c8a..a224f1be1ec2 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/5000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/5000.c
@@ -1,6 +1,7 @@
 /******************************************************************************
  *
  * Copyright(c) 2007 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of version 2 of the GNU General Public License as
@@ -28,7 +29,6 @@
 #include <linux/stringify.h>
 #include "iwl-config.h"
 #include "iwl-agn-hw.h"
-#include "iwl-csr.h"
 
 /* Highest firmware API version supported */
 #define IWL5000_UCODE_API_MAX 5
@@ -89,7 +89,8 @@ static const struct iwl_eeprom_params iwl5000_eeprom_params = {
 	.base_params = &iwl5000_base_params,			\
 	.eeprom_params = &iwl5000_eeprom_params,		\
 	.led_mode = IWL_LED_BLINK,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl5300_agn_cfg = {
 	.name = "Intel(R) Ultimate N WiFi Link 5300 AGN",
@@ -153,7 +154,8 @@ const struct iwl_cfg iwl5350_agn_cfg = {
 	.eeprom_params = &iwl5000_eeprom_params,		\
 	.led_mode = IWL_LED_BLINK,				\
 	.internal_wimax_coex = true,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl5150_agn_cfg = {
 	.name = "Intel(R) WiMAX/WiFi Link 5150 AGN",
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/6000.c b/drivers/net/wireless/intel/iwlwifi/cfg/6000.c
index 39335b7b0c16..dbcec7ce7863 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/6000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/6000.c
@@ -1,6 +1,7 @@
 /******************************************************************************
  *
  * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of version 2 of the GNU General Public License as
@@ -135,7 +136,8 @@ static const struct iwl_eeprom_params iwl6000_eeprom_params = {
 	.base_params = &iwl6000_g2_base_params,			\
 	.eeprom_params = &iwl6000_eeprom_params,		\
 	.led_mode = IWL_LED_RF_STATE,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl6005_2agn_cfg = {
 	.name = "Intel(R) Centrino(R) Advanced-N 6205 AGN",
@@ -189,7 +191,8 @@ const struct iwl_cfg iwl6005_2agn_mow2_cfg = {
 	.base_params = &iwl6000_g2_base_params,			\
 	.eeprom_params = &iwl6000_eeprom_params,		\
 	.led_mode = IWL_LED_RF_STATE,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl6030_2agn_cfg = {
 	.name = "Intel(R) Centrino(R) Advanced-N 6230 AGN",
@@ -225,7 +228,8 @@ const struct iwl_cfg iwl6030_2bg_cfg = {
 	.base_params = &iwl6000_g2_base_params,			\
 	.eeprom_params = &iwl6000_eeprom_params,		\
 	.led_mode = IWL_LED_RF_STATE,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl6035_2agn_cfg = {
 	.name = "Intel(R) Centrino(R) Advanced-N 6235 AGN",
@@ -280,7 +284,8 @@ const struct iwl_cfg iwl130_bg_cfg = {
 	.base_params = &iwl6000_base_params,			\
 	.eeprom_params = &iwl6000_eeprom_params,		\
 	.led_mode = IWL_LED_BLINK,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl6000i_2agn_cfg = {
 	.name = "Intel(R) Centrino(R) Advanced-N 6200 AGN",
@@ -313,7 +318,8 @@ const struct iwl_cfg iwl6000i_2bg_cfg = {
 	.eeprom_params = &iwl6000_eeprom_params,		\
 	.led_mode = IWL_LED_BLINK,				\
 	.internal_wimax_coex = true,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl6050_2agn_cfg = {
 	.name = "Intel(R) Centrino(R) Advanced-N + WiMAX 6250 AGN",
@@ -339,7 +345,8 @@ const struct iwl_cfg iwl6050_2abg_cfg = {
 	.eeprom_params = &iwl6000_eeprom_params,		\
 	.led_mode = IWL_LED_BLINK,				\
 	.internal_wimax_coex = true,				\
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl6150_bgn_cfg = {
 	.name = "Intel(R) Centrino(R) Wireless-N + WiMAX 6150 BGN",
@@ -366,6 +373,7 @@ const struct iwl_cfg iwl6000_3agn_cfg = {
 	.eeprom_params = &iwl6000_eeprom_params,
 	.ht_params = &iwl6000_ht_params,
 	.led_mode = IWL_LED_BLINK,
+	.csr = &iwl_csr_v1,
 };
 
 MODULE_FIRMWARE(IWL6000_MODULE_FIRMWARE(IWL6000_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/7000.c b/drivers/net/wireless/intel/iwlwifi/cfg/7000.c
index ce741beec1fc..69bfa827e82a 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/7000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/7000.c
@@ -7,7 +7,8 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
- * Copyright(c) 2015 Intel Deutschland GmbH
+ * Copyright(c) 2015        Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -34,7 +35,8 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
- * Copyright(c) 2015 Intel Deutschland GmbH
+ * Copyright(c) 2015        Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -68,7 +70,6 @@
 #include <linux/module.h>
 #include <linux/stringify.h>
 #include "iwl-config.h"
-#include "iwl-agn-hw.h"
 
 /* Highest firmware API version supported */
 #define IWL7260_UCODE_API_MAX	17
@@ -160,14 +161,13 @@ static const struct iwl_ht_params iwl7000_ht_params = {
 
 #define IWL_DEVICE_7000_COMMON					\
 	.device_family = IWL_DEVICE_FAMILY_7000,		\
-	.max_inst_size = IWL60_RTC_INST_SIZE,			\
-	.max_data_size = IWL60_RTC_DATA_SIZE,			\
 	.base_params = &iwl7000_base_params,			\
 	.led_mode = IWL_LED_RF_STATE,				\
 	.nvm_hw_section_num = NVM_HW_SECTION_NUM_FAMILY_7000,	\
 	.non_shared_ant = ANT_A,				\
 	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,	\
-	.dccm_offset = IWL7000_DCCM_OFFSET
+	.dccm_offset = IWL7000_DCCM_OFFSET,			\
+	.csr = &iwl_csr_v1
 
 #define IWL_DEVICE_7000						\
 	IWL_DEVICE_7000_COMMON,					\
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/8000.c b/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
index 3f4d9bac9f73..7262e973e0d6 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
@@ -7,7 +7,8 @@
  *
  * Copyright(c) 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2014 - 2015 Intel Mobile Communications GmbH
- * Copyright(c) 2016 Intel Deutschland GmbH
+ * Copyright(c) 2016        Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -34,6 +35,7 @@
  *
  * Copyright(c) 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2014 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -67,7 +69,6 @@
 #include <linux/module.h>
 #include <linux/stringify.h>
 #include "iwl-config.h"
-#include "iwl-agn-hw.h"
 
 /* Highest firmware API version supported */
 #define IWL8000_UCODE_API_MAX	36
@@ -140,8 +141,6 @@ static const struct iwl_tt_params iwl8000_tt_params = {
 
 #define IWL_DEVICE_8000_COMMON						\
 	.device_family = IWL_DEVICE_FAMILY_8000,			\
-	.max_inst_size = IWL60_RTC_INST_SIZE,				\
-	.max_data_size = IWL60_RTC_DATA_SIZE,				\
 	.base_params = &iwl8000_base_params,				\
 	.led_mode = IWL_LED_RF_STATE,					\
 	.nvm_hw_section_num = NVM_HW_SECTION_NUM_FAMILY_8000,		\
@@ -158,7 +157,8 @@ static const struct iwl_tt_params iwl8000_tt_params = {
 	.apmg_not_supported = true,					\
 	.nvm_type = IWL_NVM_EXT,					\
 	.dbgc_supported = true,						\
-	.min_umac_error_event_table = 0x800000
+	.min_umac_error_event_table = 0x800000,				\
+	.csr = &iwl_csr_v1
 
 #define IWL_DEVICE_8000							\
 	IWL_DEVICE_8000_COMMON,						\
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/9000.c b/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
index e1c869a1f8cc..e20c30b29c03 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
@@ -54,7 +54,6 @@
 #include <linux/module.h>
 #include <linux/stringify.h>
 #include "iwl-config.h"
-#include "iwl-agn-hw.h"
 #include "fw/file.h"
 
 /* Highest firmware API version supported */
@@ -135,12 +134,10 @@ static const struct iwl_tt_params iwl9000_tt_params = {
 	.ucode_api_max = IWL9000_UCODE_API_MAX,				\
 	.ucode_api_min = IWL9000_UCODE_API_MIN,				\
 	.device_family = IWL_DEVICE_FAMILY_9000,			\
-	.max_inst_size = IWL60_RTC_INST_SIZE,				\
-	.max_data_size = IWL60_RTC_DATA_SIZE,				\
 	.base_params = &iwl9000_base_params,				\
 	.led_mode = IWL_LED_RF_STATE,					\
 	.nvm_hw_section_num = NVM_HW_SECTION_NUM_FAMILY_9000,		\
-	.non_shared_ant = ANT_A,					\
+	.non_shared_ant = ANT_B,					\
 	.dccm_offset = IWL9000_DCCM_OFFSET,				\
 	.dccm_len = IWL9000_DCCM_LEN,					\
 	.dccm2_offset = IWL9000_DCCM2_OFFSET,				\
@@ -156,7 +153,8 @@ static const struct iwl_tt_params iwl9000_tt_params = {
 	.rf_id = true,							\
 	.nvm_type = IWL_NVM_EXT,					\
 	.dbgc_supported = true,						\
-	.min_umac_error_event_table = 0x800000
+	.min_umac_error_event_table = 0x800000,				\
+	.csr = &iwl_csr_v1
 
 const struct iwl_cfg iwl9160_2ac_cfg = {
 	.name = "Intel(R) Dual Band Wireless AC 9160",
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/main.c b/drivers/net/wireless/intel/iwlwifi/dvm/main.c
index e68254e12764..030482b357a3 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/main.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/main.c
@@ -1200,16 +1200,16 @@ static int iwl_eeprom_init_hw_params(struct iwl_priv *priv)
 		return -EINVAL;
 	}
 
-	if (!data->sku_cap_11n_enable && !data->sku_cap_band_24GHz_enable &&
-	    !data->sku_cap_band_52GHz_enable) {
+	if (!data->sku_cap_11n_enable && !data->sku_cap_band_24ghz_enable &&
+	    !data->sku_cap_band_52ghz_enable) {
 		IWL_ERR(priv, "Invalid device sku\n");
 		return -EINVAL;
 	}
 
 	IWL_DEBUG_INFO(priv,
 		       "Device SKU: 24GHz %s %s, 52GHz %s %s, 11.n %s %s\n",
-		       data->sku_cap_band_24GHz_enable ? "" : "NOT", "enabled",
-		       data->sku_cap_band_52GHz_enable ? "" : "NOT", "enabled",
+		       data->sku_cap_band_24ghz_enable ? "" : "NOT", "enabled",
+		       data->sku_cap_band_52ghz_enable ? "" : "NOT", "enabled",
 		       data->sku_cap_11n_enable ? "" : "NOT", "enabled");
 
 	priv->hw_params.tx_chains_num =
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
index a57c7223df0f..5f6e855006dd 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
@@ -87,11 +87,6 @@ enum iwl_data_path_subcmd_ids {
 	 */
 	TLC_MNG_CONFIG_CMD = 0xF,
 
-	/**
-	 * @TLC_MNG_NOTIF_REQ_CMD: &struct iwl_tlc_notif_req_config_cmd
-	 */
-	TLC_MNG_NOTIF_REQ_CMD = 0x10,
-
 	/**
 	 * @TLC_MNG_UPDATE_NOTIF: &struct iwl_tlc_update_notif
 	 */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h b/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
index 37c57bcbfb4a..8d6dc9189985 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
@@ -189,23 +189,37 @@ struct iwl_nvm_get_info_general {
 	u8 reserved;
 } __packed; /* GRP_REGULATORY_NVM_GET_INFO_GENERAL_S_VER_1 */
 
+/**
+ * enum iwl_nvm_mac_sku_flags - flags in &iwl_nvm_get_info_sku
+ * @NVM_MAC_SKU_FLAGS_BAND_2_4_ENABLED: true if 2.4 band enabled
+ * @NVM_MAC_SKU_FLAGS_BAND_5_2_ENABLED: true if 5.2 band enabled
+ * @NVM_MAC_SKU_FLAGS_802_11N_ENABLED: true if 11n enabled
+ * @NVM_MAC_SKU_FLAGS_802_11AC_ENABLED: true if 11ac enabled
+ * @NVM_MAC_SKU_FLAGS_802_11AX_ENABLED: true if 11ax enabled
+ * @NVM_MAC_SKU_FLAGS_MIMO_DISABLED: true if MIMO disabled
+ * @NVM_MAC_SKU_FLAGS_WAPI_ENABLED: true if WAPI enabled
+ * @NVM_MAC_SKU_FLAGS_REG_CHECK_ENABLED: true if regulatory checker enabled
+ * @NVM_MAC_SKU_FLAGS_API_LOCK_ENABLED: true if API lock enabled
+ */
+enum iwl_nvm_mac_sku_flags {
+	NVM_MAC_SKU_FLAGS_BAND_2_4_ENABLED	= BIT(0),
+	NVM_MAC_SKU_FLAGS_BAND_5_2_ENABLED	= BIT(1),
+	NVM_MAC_SKU_FLAGS_802_11N_ENABLED	= BIT(2),
+	NVM_MAC_SKU_FLAGS_802_11AC_ENABLED	= BIT(3),
+	NVM_MAC_SKU_FLAGS_802_11AX_ENABLED	= BIT(4),
+	NVM_MAC_SKU_FLAGS_MIMO_DISABLED		= BIT(5),
+	NVM_MAC_SKU_FLAGS_WAPI_ENABLED		= BIT(8),
+	NVM_MAC_SKU_FLAGS_REG_CHECK_ENABLED	= BIT(14),
+	NVM_MAC_SKU_FLAGS_API_LOCK_ENABLED	= BIT(15),
+};
+
 /**
  * struct iwl_nvm_get_info_sku - mac information
- * @enable_24g: band 2.4G enabled
- * @enable_5g: band 5G enabled
- * @enable_11n: 11n enabled
- * @enable_11ac: 11ac enabled
- * @mimo_disable: MIMO enabled
- * @ext_crypto: Extended crypto enabled
+ * @mac_sku_flags: flags for SKU, see &enum iwl_nvm_mac_sku_flags
  */
 struct iwl_nvm_get_info_sku {
-	__le32 enable_24g;
-	__le32 enable_5g;
-	__le32 enable_11n;
-	__le32 enable_11ac;
-	__le32 mimo_disable;
-	__le32 ext_crypto;
-} __packed; /* GRP_REGULATORY_NVM_GET_INFO_MAC_SKU_SECTION_S_VER_1 */
+	__le32 mac_sku_flags;
+} __packed; /* REGULATORY_NVM_GET_INFO_MAC_SKU_SECTION_S_VER_2 */
 
 /**
  * struct iwl_nvm_get_info_phy - phy information
@@ -243,7 +257,7 @@ struct iwl_nvm_get_info_rsp {
 	struct iwl_nvm_get_info_sku mac_sku;
 	struct iwl_nvm_get_info_phy phy_sku;
 	struct iwl_nvm_get_info_regulatory regulatory;
-} __packed; /* GRP_REGULATORY_NVM_GET_INFO_CMD_RSP_S_VER_1 */
+} __packed; /* GRP_REGULATORY_NVM_GET_INFO_CMD_RSP_S_VER_2 */
 
 /**
  * struct iwl_nvm_access_complete_cmd - NVM_ACCESS commands are completed
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
index e49a6f7be613..21e13a315421 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
@@ -7,6 +7,7 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -28,6 +29,7 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -64,62 +66,38 @@
 
 /**
  * enum iwl_tlc_mng_cfg_flags_enum - options for TLC config flags
- * @IWL_TLC_MNG_CFG_FLAGS_CCK_MSK: CCK support
- * @IWL_TLC_MNG_CFG_FLAGS_DD_MSK: enable DD
  * @IWL_TLC_MNG_CFG_FLAGS_STBC_MSK: enable STBC
  * @IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK: enable LDPC
- * @IWL_TLC_MNG_CFG_FLAGS_BF_MSK: enable BFER
- * @IWL_TLC_MNG_CFG_FLAGS_DCM_MSK: enable DCM
  */
 enum iwl_tlc_mng_cfg_flags {
-	IWL_TLC_MNG_CFG_FLAGS_CCK_MSK	= BIT(0),
-	IWL_TLC_MNG_CFG_FLAGS_DD_MSK	= BIT(1),
-	IWL_TLC_MNG_CFG_FLAGS_STBC_MSK	= BIT(2),
-	IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK	= BIT(3),
-	IWL_TLC_MNG_CFG_FLAGS_BF_MSK	= BIT(4),
-	IWL_TLC_MNG_CFG_FLAGS_DCM_MSK	= BIT(5),
+	IWL_TLC_MNG_CFG_FLAGS_STBC_MSK		= BIT(0),
+	IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK		= BIT(1),
 };
 
 /**
  * enum iwl_tlc_mng_cfg_cw - channel width options
- * @IWL_TLC_MNG_MAX_CH_WIDTH_20MHZ: 20MHZ channel
- * @IWL_TLC_MNG_MAX_CH_WIDTH_40MHZ: 40MHZ channel
- * @IWL_TLC_MNG_MAX_CH_WIDTH_80MHZ: 80MHZ channel
- * @IWL_TLC_MNG_MAX_CH_WIDTH_160MHZ: 160MHZ channel
- * @IWL_TLC_MNG_MAX_CH_WIDTH_LAST: maximum value
+ * @IWL_TLC_MNG_CH_WIDTH_20MHZ: 20MHZ channel
+ * @IWL_TLC_MNG_CH_WIDTH_40MHZ: 40MHZ channel
+ * @IWL_TLC_MNG_CH_WIDTH_80MHZ: 80MHZ channel
+ * @IWL_TLC_MNG_CH_WIDTH_160MHZ: 160MHZ channel
+ * @IWL_TLC_MNG_CH_WIDTH_LAST: maximum value
  */
 enum iwl_tlc_mng_cfg_cw {
-	IWL_TLC_MNG_MAX_CH_WIDTH_20MHZ,
-	IWL_TLC_MNG_MAX_CH_WIDTH_40MHZ,
-	IWL_TLC_MNG_MAX_CH_WIDTH_80MHZ,
-	IWL_TLC_MNG_MAX_CH_WIDTH_160MHZ,
-	IWL_TLC_MNG_MAX_CH_WIDTH_LAST = IWL_TLC_MNG_MAX_CH_WIDTH_160MHZ,
+	IWL_TLC_MNG_CH_WIDTH_20MHZ,
+	IWL_TLC_MNG_CH_WIDTH_40MHZ,
+	IWL_TLC_MNG_CH_WIDTH_80MHZ,
+	IWL_TLC_MNG_CH_WIDTH_160MHZ,
+	IWL_TLC_MNG_CH_WIDTH_LAST = IWL_TLC_MNG_CH_WIDTH_160MHZ,
 };
 
 /**
  * enum iwl_tlc_mng_cfg_chains - possible chains
  * @IWL_TLC_MNG_CHAIN_A_MSK: chain A
  * @IWL_TLC_MNG_CHAIN_B_MSK: chain B
- * @IWL_TLC_MNG_CHAIN_C_MSK: chain C
  */
 enum iwl_tlc_mng_cfg_chains {
 	IWL_TLC_MNG_CHAIN_A_MSK = BIT(0),
 	IWL_TLC_MNG_CHAIN_B_MSK = BIT(1),
-	IWL_TLC_MNG_CHAIN_C_MSK = BIT(2),
-};
-
-/**
- * enum iwl_tlc_mng_cfg_gi - guard interval options
- * @IWL_TLC_MNG_SGI_20MHZ_MSK: enable short GI for 20MHZ
- * @IWL_TLC_MNG_SGI_40MHZ_MSK: enable short GI for 40MHZ
- * @IWL_TLC_MNG_SGI_80MHZ_MSK: enable short GI for 80MHZ
- * @IWL_TLC_MNG_SGI_160MHZ_MSK: enable short GI for 160MHZ
- */
-enum iwl_tlc_mng_cfg_gi {
-	IWL_TLC_MNG_SGI_20MHZ_MSK  = BIT(0),
-	IWL_TLC_MNG_SGI_40MHZ_MSK  = BIT(1),
-	IWL_TLC_MNG_SGI_80MHZ_MSK  = BIT(2),
-	IWL_TLC_MNG_SGI_160MHZ_MSK = BIT(3),
 };
 
 /**
@@ -145,25 +123,7 @@ enum iwl_tlc_mng_cfg_mode {
 };
 
 /**
- * enum iwl_tlc_mng_vht_he_types - VHT HE types
- * @IWL_TLC_MNG_VALID_VHT_HE_TYPES_SU: VHT HT single user
- * @IWL_TLC_MNG_VALID_VHT_HE_TYPES_SU_EXT: VHT HT single user extended
- * @IWL_TLC_MNG_VALID_VHT_HE_TYPES_MU: VHT HT multiple users
- * @IWL_TLC_MNG_VALID_VHT_HE_TYPES_TRIG_BASED: trigger based
- * @IWL_TLC_MNG_VALID_VHT_HE_TYPES_NUM: a count of possible types
- */
-enum iwl_tlc_mng_vht_he_types {
-	IWL_TLC_MNG_VALID_VHT_HE_TYPES_SU = 0,
-	IWL_TLC_MNG_VALID_VHT_HE_TYPES_SU_EXT,
-	IWL_TLC_MNG_VALID_VHT_HE_TYPES_MU,
-	IWL_TLC_MNG_VALID_VHT_HE_TYPES_TRIG_BASED,
-	IWL_TLC_MNG_VALID_VHT_HE_TYPES_NUM =
-		IWL_TLC_MNG_VALID_VHT_HE_TYPES_TRIG_BASED,
-
-};
-
-/**
- * enum iwl_tlc_mng_ht_rates - HT/VHT rates
+ * enum iwl_tlc_mng_ht_rates - HT/VHT/HE rates
  * @IWL_TLC_MNG_HT_RATE_MCS0: index of MCS0
  * @IWL_TLC_MNG_HT_RATE_MCS1: index of MCS1
  * @IWL_TLC_MNG_HT_RATE_MCS2: index of MCS2
@@ -174,6 +134,8 @@ enum iwl_tlc_mng_vht_he_types {
  * @IWL_TLC_MNG_HT_RATE_MCS7: index of MCS7
  * @IWL_TLC_MNG_HT_RATE_MCS8: index of MCS8
  * @IWL_TLC_MNG_HT_RATE_MCS9: index of MCS9
+ * @IWL_TLC_MNG_HT_RATE_MCS10: index of MCS10
+ * @IWL_TLC_MNG_HT_RATE_MCS11: index of MCS11
  * @IWL_TLC_MNG_HT_RATE_MAX: maximal rate for HT/VHT
  */
 enum iwl_tlc_mng_ht_rates {
@@ -187,81 +149,73 @@ enum iwl_tlc_mng_ht_rates {
 	IWL_TLC_MNG_HT_RATE_MCS7,
 	IWL_TLC_MNG_HT_RATE_MCS8,
 	IWL_TLC_MNG_HT_RATE_MCS9,
-	IWL_TLC_MNG_HT_RATE_MAX = IWL_TLC_MNG_HT_RATE_MCS9,
+	IWL_TLC_MNG_HT_RATE_MCS10,
+	IWL_TLC_MNG_HT_RATE_MCS11,
+	IWL_TLC_MNG_HT_RATE_MAX = IWL_TLC_MNG_HT_RATE_MCS11,
 };
 
 /* Maximum supported tx antennas number */
-#define MAX_RS_ANT_NUM 3
+#define MAX_NSS 2
 
 /**
  * struct tlc_config_cmd - TLC configuration
  * @sta_id: station id
  * @reserved1: reserved
- * @max_supp_ch_width: channel width
- * @flags: bitmask of &enum iwl_tlc_mng_cfg_flags
- * @chains: bitmask of &enum iwl_tlc_mng_cfg_chains
- * @max_supp_ss: valid values are 0-3, 0 - spatial streams are not supported
- * @valid_vht_he_types: bitmap of &enum iwl_tlc_mng_vht_he_types
- * @non_ht_supp_rates: bitmap of supported legacy rates
- * @ht_supp_rates: bitmap of supported HT/VHT rates, valid bits are 0-9
+ * @max_ch_width: max supported channel width from @enum iwl_tlc_mng_cfg_cw
  * @mode: &enum iwl_tlc_mng_cfg_mode
- * @reserved2: reserved
- * @he_supp_rates: bitmap of supported HE rates
+ * @chains: bitmask of &enum iwl_tlc_mng_cfg_chains
+ * @amsdu: TX amsdu is supported
+ * @flags: bitmask of &enum iwl_tlc_mng_cfg_flags
+ * @non_ht_rates: bitmap of supported legacy rates
+ * @ht_rates: bitmap of &enum iwl_tlc_mng_ht_rates, per <nss, channel-width>
+ *	      pair (0 - 80mhz width and below, 1 - 160mhz).
+ * @max_mpdu_len: max MPDU length, in bytes
  * @sgi_ch_width_supp: bitmap of SGI support per channel width
- * @he_gi_support: 11ax HE guard interval
- * @max_ampdu_cnt: max AMPDU size (frames count)
+ *		       use BIT(@enum iwl_tlc_mng_cfg_cw)
+ * @reserved2: reserved
  */
 struct iwl_tlc_config_cmd {
 	u8 sta_id;
 	u8 reserved1[3];
-	u8 max_supp_ch_width;
+	u8 max_ch_width;
+	u8 mode;
 	u8 chains;
-	u8 max_supp_ss;
-	u8 valid_vht_he_types;
+	u8 amsdu;
 	__le16 flags;
-	__le16 non_ht_supp_rates;
-	__le16 ht_supp_rates[MAX_RS_ANT_NUM];
-	u8 mode;
-	u8 reserved2;
-	__le16 he_supp_rates;
+	__le16 non_ht_rates;
+	__le16 ht_rates[MAX_NSS][2];
+	__le16 max_mpdu_len;
 	u8 sgi_ch_width_supp;
-	u8 he_gi_support;
-	__le32 max_ampdu_cnt;
-} __packed; /* TLC_MNG_CONFIG_CMD_API_S_VER_1 */
-
-#define IWL_TLC_NOTIF_INIT_RATE_POS 0
-#define IWL_TLC_NOTIF_INIT_RATE_MSK BIT(IWL_TLC_NOTIF_INIT_RATE_POS)
-#define IWL_TLC_NOTIF_REQ_INTERVAL (500)
+	u8 reserved2[1];
+} __packed; /* TLC_MNG_CONFIG_CMD_API_S_VER_2 */
 
 /**
- * struct iwl_tlc_notif_req_config_cmd - request notif on specific changes
- * @sta_id: relevant station
- * @reserved1: reserved
- * @flags: bitmap of requested notifications %IWL_TLC_NOTIF_INIT_\*
- * @interval: minimum time between notifications from TLC to the driver (msec)
- * @reserved2: reserved
+ * enum iwl_tlc_update_flags - updated fields
+ * @IWL_TLC_NOTIF_FLAG_RATE: last initial rate update
+ * @IWL_TLC_NOTIF_FLAG_AMSDU: umsdu parameters update
  */
-struct iwl_tlc_notif_req_config_cmd {
-	u8 sta_id;
-	u8 reserved1;
-	__le16 flags;
-	__le16 interval;
-	__le16 reserved2;
-} __packed; /* TLC_MNG_NOTIF_REQ_CMD_API_S_VER_1 */
+enum iwl_tlc_update_flags {
+	IWL_TLC_NOTIF_FLAG_RATE  = BIT(0),
+	IWL_TLC_NOTIF_FLAG_AMSDU = BIT(1),
+};
 
 /**
  * struct iwl_tlc_update_notif - TLC notification from FW
  * @sta_id: station id
  * @reserved: reserved
  * @flags: bitmap of notifications reported
- * @values: field per flag in struct iwl_tlc_notif_req_config_cmd
+ * @rate: current initial rate
+ * @amsdu_size: Max AMSDU size, in bytes
+ * @amsdu_enabled: bitmap for per-TID AMSDU enablement
  */
 struct iwl_tlc_update_notif {
 	u8 sta_id;
-	u8 reserved;
-	__le16 flags;
-	__le32 values[16];
-} __packed; /* TLC_MNG_UPDATE_NTFY_API_S_VER_1 */
+	u8 reserved[3];
+	__le32 flags;
+	__le32 rate;
+	__le32 amsdu_size;
+	__le32 amsdu_enabled;
+} __packed; /* TLC_MNG_UPDATE_NTFY_API_S_VER_2 */
 
 /**
  * enum iwl_tlc_debug_flags - debug options
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
index e7565f37ece9..7e570c4a9df0 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
@@ -295,6 +295,7 @@ enum iwl_rx_mpdu_status {
 	IWL_RX_MPDU_STATUS_MIC_OK		= BIT(6),
 	IWL_RX_MPDU_RES_STATUS_TTAK_OK		= BIT(7),
 	IWL_RX_MPDU_STATUS_SEC_MASK		= 0x7 << 8,
+	IWL_RX_MPDU_STATUS_SEC_UNKNOWN		= IWL_RX_MPDU_STATUS_SEC_MASK,
 	IWL_RX_MPDU_STATUS_SEC_NONE		= 0x0 << 8,
 	IWL_RX_MPDU_STATUS_SEC_WEP		= 0x1 << 8,
 	IWL_RX_MPDU_STATUS_SEC_CCM		= 0x2 << 8,
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/txq.h b/drivers/net/wireless/intel/iwlwifi/fw/api/txq.h
index dfa111bb411e..6ac240b6eace 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/txq.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/txq.h
@@ -131,6 +131,8 @@ enum iwl_tx_queue_cfg_actions {
 	TX_QUEUE_CFG_TFD_SHORT_FORMAT		= BIT(1),
 };
 
+#define IWL_DEFAULT_QUEUE_SIZE 256
+#define IWL_MGMT_QUEUE_SIZE 16
 /**
  * struct iwl_tx_queue_cfg_cmd - txq hw scheduler config command
  * @sta_id: station id
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.h b/drivers/net/wireless/intel/iwlwifi/fw/dbg.h
index 72259bff9922..507d9a49fa97 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.h
@@ -227,4 +227,40 @@ static inline void iwl_fw_cancel_dump(struct iwl_fw_runtime *fwrt)
 	cancel_delayed_work_sync(&fwrt->dump.wk);
 }
 
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+static inline void iwl_fw_cancel_timestamp(struct iwl_fw_runtime *fwrt)
+{
+	fwrt->timestamp.delay = 0;
+	cancel_delayed_work_sync(&fwrt->timestamp.wk);
+}
+
+void iwl_fw_trigger_timestamp(struct iwl_fw_runtime *fwrt, u32 delay);
+
+static inline void iwl_fw_suspend_timestamp(struct iwl_fw_runtime *fwrt)
+{
+	cancel_delayed_work_sync(&fwrt->timestamp.wk);
+}
+
+static inline void iwl_fw_resume_timestamp(struct iwl_fw_runtime *fwrt)
+{
+	if (!fwrt->timestamp.delay)
+		return;
+
+	schedule_delayed_work(&fwrt->timestamp.wk,
+			      round_jiffies_relative(fwrt->timestamp.delay));
+}
+
+#else
+
+static inline void iwl_fw_cancel_timestamp(struct iwl_fw_runtime *fwrt) {}
+
+static inline void iwl_fw_trigger_timestamp(struct iwl_fw_runtime *fwrt,
+					    u32 delay) {}
+
+static inline void iwl_fw_suspend_timestamp(struct iwl_fw_runtime *fwrt) {}
+
+static inline void iwl_fw_resume_timestamp(struct iwl_fw_runtime *fwrt) {}
+
+#endif /* CONFIG_IWLWIFI_DEBUGFS */
+
 #endif  /* __iwl_fw_dbg_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
index 8f005cd69559..8ba5a60ec9ed 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
@@ -64,6 +64,7 @@
  *****************************************************************************/
 #include "api/commands.h"
 #include "debugfs.h"
+#include "dbg.h"
 
 #define FWRT_DEBUGFS_READ_FILE_OPS(name)				\
 static ssize_t iwl_dbgfs_##name##_read(struct iwl_fw_runtime *fwrt,	\
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.h b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.h
index d93f6a4bb22d..cbbfa8e9e66d 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.h
@@ -69,28 +69,6 @@
 int iwl_fwrt_dbgfs_register(struct iwl_fw_runtime *fwrt,
 			    struct dentry *dbgfs_dir);
 
-static inline void iwl_fw_cancel_timestamp(struct iwl_fw_runtime *fwrt)
-{
-	fwrt->timestamp.delay = 0;
-	cancel_delayed_work_sync(&fwrt->timestamp.wk);
-}
-
-static inline void iwl_fw_suspend_timestamp(struct iwl_fw_runtime *fwrt)
-{
-	cancel_delayed_work_sync(&fwrt->timestamp.wk);
-}
-
-static inline void iwl_fw_resume_timestamp(struct iwl_fw_runtime *fwrt)
-{
-	if (!fwrt->timestamp.delay)
-		return;
-
-	schedule_delayed_work(&fwrt->timestamp.wk,
-			      round_jiffies_relative(fwrt->timestamp.delay));
-}
-
-void iwl_fw_trigger_timestamp(struct iwl_fw_runtime *fwrt, u32 delay);
-
 #else
 static inline int iwl_fwrt_dbgfs_register(struct iwl_fw_runtime *fwrt,
 					  struct dentry *dbgfs_dir)
@@ -98,13 +76,4 @@ static inline int iwl_fwrt_dbgfs_register(struct iwl_fw_runtime *fwrt,
 	return 0;
 }
 
-static inline void iwl_fw_cancel_timestamp(struct iwl_fw_runtime *fwrt) {}
-
-static inline void iwl_fw_suspend_timestamp(struct iwl_fw_runtime *fwrt) {}
-
-static inline void iwl_fw_resume_timestamp(struct iwl_fw_runtime *fwrt) {}
-
-static inline void iwl_fw_trigger_timestamp(struct iwl_fw_runtime *fwrt,
-					    u32 delay) {}
-
 #endif /* CONFIG_IWLWIFI_DEBUGFS */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h
index 9b2805e1e3b1..9d939cbaf6c6 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/file.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h
@@ -8,6 +8,7 @@
  * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -35,6 +36,7 @@
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -143,6 +145,7 @@ enum iwl_ucode_tlv_type {
 	IWL_UCODE_TLV_FW_DBG_TRIGGER	= 40,
 	IWL_UCODE_TLV_FW_GSCAN_CAPA	= 50,
 	IWL_UCODE_TLV_FW_MEM_SEG	= 51,
+	IWL_UCODE_TLV_IML		= 52,
 };
 
 struct iwl_ucode_tlv {
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/img.h b/drivers/net/wireless/intel/iwlwifi/fw/img.h
index b23ffe12ad84..f4912382b6af 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/img.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/img.h
@@ -8,6 +8,7 @@
  * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016        Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -35,6 +36,7 @@
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016        Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -241,6 +243,8 @@ enum iwl_fw_type {
  * @ucode_ver: ucode version from the ucode file
  * @fw_version: firmware version string
  * @img: ucode image like ucode_rt, ucode_init, ucode_wowlan.
+ * @iml_len: length of the image loader image
+ * @iml: image loader fw image
  * @ucode_capa: capabilities parsed from the ucode file.
  * @enhance_sensitivity_table: device can do enhanced sensitivity.
  * @init_evtlog_ptr: event log offset for init ucode.
@@ -267,6 +271,8 @@ struct iwl_fw {
 
 	/* ucode images */
 	struct fw_img img[IWL_UCODE_TYPE_MAX];
+	size_t iml_len;
+	u8 *iml;
 
 	struct iwl_ucode_capabilities ucode_capa;
 	bool enhance_sensitivity_table;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/nvm.c b/drivers/net/wireless/intel/iwlwifi/fw/nvm.c
deleted file mode 100644
index bd2e1fb43f5a..000000000000
--- a/drivers/net/wireless/intel/iwlwifi/fw/nvm.c
+++ /dev/null
@@ -1,162 +0,0 @@
-/******************************************************************************
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
- * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
- * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
- * The full GNU General Public License is included in this distribution
- * in the file called COPYING.
- *
- * Contact Information:
- *  Intel Linux Wireless <linuxwifi@intel.com>
- * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
- *
- * BSD LICENSE
- *
- * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
- * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
- * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *  * Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  * Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *  * Neither the name Intel Corporation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- *****************************************************************************/
-#include "iwl-drv.h"
-#include "runtime.h"
-#include "fw/api/nvm-reg.h"
-#include "fw/api/commands.h"
-#include "iwl-nvm-parse.h"
-
-struct iwl_nvm_data *iwl_fw_get_nvm(struct iwl_fw_runtime *fwrt)
-{
-	struct iwl_nvm_get_info cmd = {};
-	struct iwl_nvm_get_info_rsp *rsp;
-	struct iwl_trans *trans = fwrt->trans;
-	struct iwl_nvm_data *nvm;
-	struct iwl_host_cmd hcmd = {
-		.flags = CMD_WANT_SKB | CMD_SEND_IN_RFKILL,
-		.data = { &cmd, },
-		.len = { sizeof(cmd) },
-		.id = WIDE_ID(REGULATORY_AND_NVM_GROUP, NVM_GET_INFO)
-	};
-	int  ret;
-	bool lar_fw_supported = !iwlwifi_mod_params.lar_disable &&
-				fw_has_capa(&fwrt->fw->ucode_capa,
-					    IWL_UCODE_TLV_CAPA_LAR_SUPPORT);
-
-	ret = iwl_trans_send_cmd(trans, &hcmd);
-	if (ret)
-		return ERR_PTR(ret);
-
-	if (WARN(iwl_rx_packet_payload_len(hcmd.resp_pkt) != sizeof(*rsp),
-		 "Invalid payload len in NVM response from FW %d",
-		 iwl_rx_packet_payload_len(hcmd.resp_pkt))) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	rsp = (void *)hcmd.resp_pkt->data;
-	if (le32_to_cpu(rsp->general.flags) & NVM_GENERAL_FLAGS_EMPTY_OTP)
-		IWL_INFO(fwrt, "OTP is empty\n");
-
-	nvm = kzalloc(sizeof(*nvm) +
-		      sizeof(struct ieee80211_channel) * IWL_NUM_CHANNELS,
-		      GFP_KERNEL);
-	if (!nvm) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	iwl_set_hw_address_from_csr(trans, nvm);
-	/* TODO: if platform NVM has MAC address - override it here */
-
-	if (!is_valid_ether_addr(nvm->hw_addr)) {
-		IWL_ERR(fwrt, "no valid mac address was found\n");
-		ret = -EINVAL;
-		goto err_free;
-	}
-
-	IWL_INFO(trans, "base HW address: %pM\n", nvm->hw_addr);
-
-	/* Initialize general data */
-	nvm->nvm_version = le16_to_cpu(rsp->general.nvm_version);
-
-	/* Initialize MAC sku data */
-	nvm->sku_cap_11ac_enable =
-		le32_to_cpu(rsp->mac_sku.enable_11ac);
-	nvm->sku_cap_11n_enable =
-		le32_to_cpu(rsp->mac_sku.enable_11n);
-	nvm->sku_cap_band_24GHz_enable =
-		le32_to_cpu(rsp->mac_sku.enable_24g);
-	nvm->sku_cap_band_52GHz_enable =
-		le32_to_cpu(rsp->mac_sku.enable_5g);
-	nvm->sku_cap_mimo_disabled =
-		le32_to_cpu(rsp->mac_sku.mimo_disable);
-
-	/* Initialize PHY sku data */
-	nvm->valid_tx_ant = (u8)le32_to_cpu(rsp->phy_sku.tx_chains);
-	nvm->valid_rx_ant = (u8)le32_to_cpu(rsp->phy_sku.rx_chains);
-
-	/* Initialize regulatory data */
-	nvm->lar_enabled =
-		le32_to_cpu(rsp->regulatory.lar_enabled) && lar_fw_supported;
-
-	iwl_init_sbands(trans->dev, trans->cfg, nvm,
-			rsp->regulatory.channel_profile,
-			nvm->valid_tx_ant & fwrt->fw->valid_tx_ant,
-			nvm->valid_rx_ant & fwrt->fw->valid_rx_ant,
-			nvm->lar_enabled, false);
-
-	iwl_free_resp(&hcmd);
-	return nvm;
-
-err_free:
-	kfree(nvm);
-out:
-	iwl_free_resp(&hcmd);
-	return ERR_PTR(ret);
-}
-IWL_EXPORT_SYMBOL(iwl_fw_get_nvm);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/paging.c b/drivers/net/wireless/intel/iwlwifi/fw/paging.c
index 1fec8e3a6b35..9b8dd7fe7112 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/paging.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/paging.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -30,6 +31,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -163,7 +165,7 @@ static int iwl_alloc_fw_paging_mem(struct iwl_fw_runtime *fwrt,
 static int iwl_fill_paging_mem(struct iwl_fw_runtime *fwrt,
 			       const struct fw_img *image)
 {
-	int sec_idx, idx;
+	int sec_idx, idx, ret;
 	u32 offset = 0;
 
 	/*
@@ -190,17 +192,23 @@ static int iwl_fill_paging_mem(struct iwl_fw_runtime *fwrt,
 	 */
 	if (sec_idx >= image->num_sec - 1) {
 		IWL_ERR(fwrt, "Paging: Missing CSS and/or paging sections\n");
-		iwl_free_fw_paging(fwrt);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto err;
 	}
 
 	/* copy the CSS block to the dram */
 	IWL_DEBUG_FW(fwrt, "Paging: load paging CSS to FW, sec = %d\n",
 		     sec_idx);
 
+	if (image->sec[sec_idx].len > fwrt->fw_paging_db[0].fw_paging_size) {
+		IWL_ERR(fwrt, "CSS block is larger than paging size\n");
+		ret = -EINVAL;
+		goto err;
+	}
+
 	memcpy(page_address(fwrt->fw_paging_db[0].fw_paging_block),
 	       image->sec[sec_idx].data,
-	       fwrt->fw_paging_db[0].fw_paging_size);
+	       image->sec[sec_idx].len);
 	dma_sync_single_for_device(fwrt->trans->dev,
 				   fwrt->fw_paging_db[0].fw_paging_phys,
 				   fwrt->fw_paging_db[0].fw_paging_size,
@@ -213,17 +221,39 @@ static int iwl_fill_paging_mem(struct iwl_fw_runtime *fwrt,
 	sec_idx++;
 
 	/*
-	 * copy the paging blocks to the dram
-	 * loop index start from 1 since that CSS block already copied to dram
-	 * and CSS index is 0.
-	 * loop stop at num_of_paging_blk since that last block is not full.
+	 * Copy the paging blocks to the dram.  The loop index starts
+	 * from 1 since the CSS block (index 0) was already copied to
+	 * dram.  We use num_of_paging_blk + 1 to account for that.
 	 */
-	for (idx = 1; idx < fwrt->num_of_paging_blk; idx++) {
+	for (idx = 1; idx < fwrt->num_of_paging_blk + 1; idx++) {
 		struct iwl_fw_paging *block = &fwrt->fw_paging_db[idx];
+		int remaining = image->sec[sec_idx].len - offset;
+		int len = block->fw_paging_size;
+
+		/*
+		 * For the last block, we copy all that is remaining,
+		 * for all other blocks, we copy fw_paging_size at a
+		 * time. */
+		if (idx == fwrt->num_of_paging_blk) {
+			len = remaining;
+			if (remaining !=
+			    fwrt->num_of_pages_in_last_blk * FW_PAGING_SIZE) {
+				IWL_ERR(fwrt,
+					"Paging: last block contains more data than expected %d\n",
+					remaining);
+				ret = -EINVAL;
+				goto err;
+			}
+		} else if (block->fw_paging_size > remaining) {
+			IWL_ERR(fwrt,
+				"Paging: not enough data in other in block %d (%d)\n",
+				idx, remaining);
+			ret = -EINVAL;
+			goto err;
+		}
 
 		memcpy(page_address(block->fw_paging_block),
-		       image->sec[sec_idx].data + offset,
-		       block->fw_paging_size);
+		       image->sec[sec_idx].data + offset, len);
 		dma_sync_single_for_device(fwrt->trans->dev,
 					   block->fw_paging_phys,
 					   block->fw_paging_size,
@@ -231,30 +261,16 @@ static int iwl_fill_paging_mem(struct iwl_fw_runtime *fwrt,
 
 		IWL_DEBUG_FW(fwrt,
 			     "Paging: copied %d paging bytes to block %d\n",
-			     fwrt->fw_paging_db[idx].fw_paging_size,
-			     idx);
-
-		offset += fwrt->fw_paging_db[idx].fw_paging_size;
-	}
-
-	/* copy the last paging block */
-	if (fwrt->num_of_pages_in_last_blk > 0) {
-		struct iwl_fw_paging *block = &fwrt->fw_paging_db[idx];
+			     len, idx);
 
-		memcpy(page_address(block->fw_paging_block),
-		       image->sec[sec_idx].data + offset,
-		       FW_PAGING_SIZE * fwrt->num_of_pages_in_last_blk);
-		dma_sync_single_for_device(fwrt->trans->dev,
-					   block->fw_paging_phys,
-					   block->fw_paging_size,
-					   DMA_BIDIRECTIONAL);
-
-		IWL_DEBUG_FW(fwrt,
-			     "Paging: copied %d pages in the last block %d\n",
-			     fwrt->num_of_pages_in_last_blk, idx);
+		offset += block->fw_paging_size;
 	}
 
 	return 0;
+
+err:
+	iwl_free_fw_paging(fwrt);
+	return ret;
 }
 
 static int iwl_save_fw_paging(struct iwl_fw_runtime *fwrt,
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
index 3fb940ebd74a..d8db1dd100b0 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
@@ -170,6 +170,5 @@ void iwl_get_shared_mem_conf(struct iwl_fw_runtime *fwrt);
 
 void iwl_fwrt_handle_notification(struct iwl_fw_runtime *fwrt,
 				  struct iwl_rx_cmd_buffer *rxb);
-struct iwl_nvm_data *iwl_fw_get_nvm(struct iwl_fw_runtime *fwrt);
 
 #endif /* __iwl_fw_runtime_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
index f0f5636dd3ea..c503b26793f6 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
@@ -7,6 +7,7 @@
  *
  * Copyright(c) 2007 - 2014 Intel Corporation. All rights reserved.
  * Copyright (C) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -33,6 +34,7 @@
  *
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright (C) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -69,6 +71,7 @@
 #include <linux/netdevice.h>
 #include <linux/ieee80211.h>
 #include <linux/nl80211.h>
+#include "iwl-csr.h"
 
 enum iwl_device_family {
 	IWL_DEVICE_FAMILY_UNDEFINED,
@@ -151,6 +154,8 @@ enum iwl_nvm_type {
 #define	ANT_AC		(ANT_A | ANT_C)
 #define ANT_BC		(ANT_B | ANT_C)
 #define ANT_ABC		(ANT_A | ANT_B | ANT_C)
+#define MAX_ANT_NUM 3
+
 
 static inline u8 num_of_ant(u8 mask)
 {
@@ -282,6 +287,52 @@ struct iwl_pwr_tx_backoff {
 	u32 backoff;
 };
 
+/**
+ * struct iwl_csr_params
+ *
+ * @flag_sw_reset: reset the device
+ * @flag_mac_clock_ready:
+ *	Indicates MAC (ucode processor, etc.) is powered up and can run.
+ *	Internal resources are accessible.
+ *	NOTE:  This does not indicate that the processor is actually running.
+ *	NOTE:  This does not indicate that device has completed
+ *	       init or post-power-down restore of internal SRAM memory.
+ *	       Use CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP as indication that
+ *	       SRAM is restored and uCode is in normal operation mode.
+ *	       This note is relevant only for pre 5xxx devices.
+ *	NOTE:  After device reset, this bit remains "0" until host sets
+ *	       INIT_DONE
+ * @flag_init_done: Host sets this to put device into fully operational
+ *	D0 power mode. Host resets this after SW_RESET to put device into
+ *	low power mode.
+ * @flag_mac_access_req: Host sets this to request and maintain MAC wakeup,
+ *	to allow host access to device-internal resources. Host must wait for
+ *	mac_clock_ready (and !GOING_TO_SLEEP) before accessing non-CSR device
+ *	registers.
+ * @flag_val_mac_access_en: mac access is enabled
+ * @flag_master_dis: disable master
+ * @flag_stop_master: stop master
+ * @addr_sw_reset: address for resetting the device
+ * @mac_addr0_otp: first part of MAC address from OTP
+ * @mac_addr1_otp: second part of MAC address from OTP
+ * @mac_addr0_strap: first part of MAC address from strap
+ * @mac_addr1_strap: second part of MAC address from strap
+ */
+struct iwl_csr_params {
+	u8 flag_sw_reset;
+	u8 flag_mac_clock_ready;
+	u8 flag_init_done;
+	u8 flag_mac_access_req;
+	u8 flag_val_mac_access_en;
+	u8 flag_master_dis;
+	u8 flag_stop_master;
+	u8 addr_sw_reset;
+	u32 mac_addr0_otp;
+	u32 mac_addr1_otp;
+	u32 mac_addr0_strap;
+	u32 mac_addr1_strap;
+};
+
 /**
  * struct iwl_cfg
  * @name: Official name of the device
@@ -294,8 +345,8 @@ struct iwl_pwr_tx_backoff {
  *	next step. Supported only in integrated solutions.
  * @ucode_api_max: Highest version of uCode API supported by driver.
  * @ucode_api_min: Lowest version of uCode API supported by driver.
- * @max_inst_size: The maximal length of the fw inst section
- * @max_data_size: The maximal length of the fw data section
+ * @max_inst_size: The maximal length of the fw inst section (only DVM)
+ * @max_data_size: The maximal length of the fw data section (only DVM)
  * @valid_tx_ant: valid transmit antenna
  * @valid_rx_ant: valid receive antenna
  * @non_shared_ant: the antenna that is for WiFi only
@@ -314,6 +365,7 @@ struct iwl_pwr_tx_backoff {
  * @mac_addr_from_csr: read HW address from CSR registers
  * @features: hw features, any combination of feature_whitelist
  * @pwr_tx_backoffs: translation table between power limits and backoffs
+ * @csr: csr flags and addresses that are different across devices
  * @max_rx_agg_size: max RX aggregation size of the ADDBA request/response
  * @max_tx_agg_size: max TX aggregation size of the ADDBA request/response
  * @max_ht_ampdu_factor: the exponent of the max length of A-MPDU that the
@@ -333,8 +385,6 @@ struct iwl_pwr_tx_backoff {
  * @gen2: 22000 and on transport operation
  * @cdb: CDB support
  * @nvm_type: see &enum iwl_nvm_type
- * @tx_cmd_queue_size: size of the cmd queue. If zero, use the same value as
- *	the regular queues
  *
  * We enable the driver to be backward compatible wrt. hardware features.
  * API differences in uCode shouldn't be handled here but through TLVs
@@ -354,6 +404,7 @@ struct iwl_cfg {
 	const struct iwl_pwr_tx_backoff *pwr_tx_backoffs;
 	const char *default_nvm_file_C_step;
 	const struct iwl_tt_params *thermal_params;
+	const struct iwl_csr_params *csr;
 	enum iwl_device_family device_family;
 	enum iwl_led_mode led_mode;
 	enum iwl_nvm_type nvm_type;
@@ -369,7 +420,7 @@ struct iwl_cfg {
 	u32 soc_latency;
 	u16 nvm_ver;
 	u16 nvm_calib_ver;
-	u16 rx_with_siso_diversity:1,
+	u32 rx_with_siso_diversity:1,
 	    bt_shared_single_ant:1,
 	    internal_wimax_coex:1,
 	    host_interrupt_operation_mode:1,
@@ -386,7 +437,6 @@ struct iwl_cfg {
 	    gen2:1,
 	    cdb:1,
 	    dbgc_supported:1;
-	u16 tx_cmd_queue_size;
 	u8 valid_tx_ant;
 	u8 valid_rx_ant;
 	u8 non_shared_ant;
@@ -401,6 +451,36 @@ struct iwl_cfg {
 	u32 extra_phy_cfg_flags;
 };
 
+static const struct iwl_csr_params iwl_csr_v1 = {
+	.flag_mac_clock_ready = 0,
+	.flag_val_mac_access_en = 0,
+	.flag_init_done = 2,
+	.flag_mac_access_req = 3,
+	.flag_sw_reset = 7,
+	.flag_master_dis = 8,
+	.flag_stop_master = 9,
+	.addr_sw_reset = (CSR_BASE + 0x020),
+	.mac_addr0_otp = 0x380,
+	.mac_addr1_otp = 0x384,
+	.mac_addr0_strap = 0x388,
+	.mac_addr1_strap = 0x38C
+};
+
+static const struct iwl_csr_params iwl_csr_v2 = {
+	.flag_init_done = 6,
+	.flag_mac_clock_ready = 20,
+	.flag_val_mac_access_en = 20,
+	.flag_mac_access_req = 21,
+	.flag_master_dis = 28,
+	.flag_stop_master = 29,
+	.flag_sw_reset = 31,
+	.addr_sw_reset = (CSR_BASE + 0x024),
+	.mac_addr0_otp = 0x30,
+	.mac_addr1_otp = 0x34,
+	.mac_addr0_strap = 0x38,
+	.mac_addr1_strap = 0x3C
+};
+
 /*
  * This list declares the config structures for all devices.
  */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
index 4f0d070eda54..ba971d3946e2 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
@@ -8,6 +8,7 @@
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
  * Copyright(c) 2016        Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -34,6 +35,7 @@
  *
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -257,7 +259,6 @@
 /* RESET */
 #define CSR_RESET_REG_FLAG_NEVO_RESET                (0x00000001)
 #define CSR_RESET_REG_FLAG_FORCE_NMI                 (0x00000002)
-#define CSR_RESET_REG_FLAG_SW_RESET                  (0x00000080)
 #define CSR_RESET_REG_FLAG_MASTER_DISABLED           (0x00000100)
 #define CSR_RESET_REG_FLAG_STOP_MASTER               (0x00000200)
 #define CSR_RESET_LINK_PWR_MGMT_DISABLED             (0x80000000)
@@ -280,35 +281,10 @@
  *     4:  GOING_TO_SLEEP
  *         Indicates MAC is entering a power-saving sleep power-down.
  *         Not a good time to access device-internal resources.
- *     3:  MAC_ACCESS_REQ
- *         Host sets this to request and maintain MAC wakeup, to allow host
- *         access to device-internal resources.  Host must wait for
- *         MAC_CLOCK_READY (and !GOING_TO_SLEEP) before accessing non-CSR
- *         device registers.
- *     2:  INIT_DONE
- *         Host sets this to put device into fully operational D0 power mode.
- *         Host resets this after SW_RESET to put device into low power mode.
- *     0:  MAC_CLOCK_READY
- *         Indicates MAC (ucode processor, etc.) is powered up and can run.
- *         Internal resources are accessible.
- *         NOTE:  This does not indicate that the processor is actually running.
- *         NOTE:  This does not indicate that device has completed
- *                init or post-power-down restore of internal SRAM memory.
- *                Use CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP as indication that
- *                SRAM is restored and uCode is in normal operation mode.
- *                Later devices (5xxx/6xxx/1xxx) use non-volatile SRAM, and
- *                do not need to save/restore it.
- *         NOTE:  After device reset, this bit remains "0" until host sets
- *                INIT_DONE
  */
-#define CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY        (0x00000001)
-#define CSR_GP_CNTRL_REG_FLAG_INIT_DONE              (0x00000004)
-#define CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ         (0x00000008)
 #define CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP         (0x00000010)
 #define CSR_GP_CNTRL_REG_FLAG_XTAL_ON		     (0x00000400)
 
-#define CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN           (0x00000001)
-
 #define CSR_GP_CNTRL_REG_MSK_POWER_SAVE_TYPE         (0x07000000)
 #define CSR_GP_CNTRL_REG_FLAG_RFKILL_WAKE_L1A_EN     (0x04000000)
 #define CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW          (0x08000000)
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
index aa2d5c14e202..c59ce4f8a5ed 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
@@ -179,6 +179,7 @@ static void iwl_dealloc_ucode(struct iwl_drv *drv)
 	for (i = 0; i < ARRAY_SIZE(drv->fw.dbg_trigger_tlv); i++)
 		kfree(drv->fw.dbg_trigger_tlv[i]);
 	kfree(drv->fw.dbg_mem_tlv);
+	kfree(drv->fw.iml);
 
 	for (i = 0; i < IWL_UCODE_TYPE_MAX; i++)
 		iwl_free_fw_img(drv, drv->fw.img + i);
@@ -1126,6 +1127,13 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 			pieces->n_dbg_mem_tlv++;
 			break;
 			}
+		case IWL_UCODE_TLV_IML: {
+			drv->fw.iml_len = tlv_len;
+			drv->fw.iml = kmemdup(tlv_data, tlv_len, GFP_KERNEL);
+			if (!drv->fw.iml)
+				return -ENOMEM;
+			break;
+			}
 		default:
 			IWL_DEBUG_INFO(drv, "unknown TLV: %d\n", tlv_type);
 			break;
@@ -1842,3 +1850,9 @@ MODULE_PARM_DESC(d0i3_timeout, "Timeout to D0i3 entry when idle (ms)");
 
 module_param_named(disable_11ac, iwlwifi_mod_params.disable_11ac, bool, 0444);
 MODULE_PARM_DESC(disable_11ac, "Disable VHT capabilities (default: false)");
+
+module_param_named(remove_when_gone,
+		   iwlwifi_mod_params.remove_when_gone, bool,
+		   0444);
+MODULE_PARM_DESC(remove_when_gone,
+		 "Remove dev from PCIe bus if it is deemed inaccessible (default: false)");
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.c
index 3199d345b427..777f5df8a0c6 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.c
@@ -899,8 +899,8 @@ iwl_parse_eeprom_data(struct device *dev, const struct iwl_cfg *cfg,
 				 EEPROM_SKU_CAP);
 	data->sku_cap_11n_enable = sku & EEPROM_SKU_CAP_11N_ENABLE;
 	data->sku_cap_amt_enable = sku & EEPROM_SKU_CAP_AMT_ENABLE;
-	data->sku_cap_band_24GHz_enable = sku & EEPROM_SKU_CAP_BAND_24GHZ;
-	data->sku_cap_band_52GHz_enable = sku & EEPROM_SKU_CAP_BAND_52GHZ;
+	data->sku_cap_band_24ghz_enable = sku & EEPROM_SKU_CAP_BAND_24GHZ;
+	data->sku_cap_band_52ghz_enable = sku & EEPROM_SKU_CAP_BAND_52GHZ;
 	data->sku_cap_ipan_enable = sku & EEPROM_SKU_CAP_IPAN_ENABLE;
 	if (iwlwifi_mod_params.disable_11n & IWL_DISABLE_HT_ALL)
 		data->sku_cap_11n_enable = false;
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h b/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h
index b33888991b94..8be50ed12300 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h
@@ -81,10 +81,11 @@ struct iwl_nvm_data {
 	__le16 kelvin_voltage;
 	__le16 xtal_calib[2];
 
-	bool sku_cap_band_24GHz_enable;
-	bool sku_cap_band_52GHz_enable;
+	bool sku_cap_band_24ghz_enable;
+	bool sku_cap_band_52ghz_enable;
 	bool sku_cap_11n_enable;
 	bool sku_cap_11ac_enable;
+	bool sku_cap_11ax_enable;
 	bool sku_cap_amt_enable;
 	bool sku_cap_ipan_enable;
 	bool sku_cap_mimo_disabled;
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-read.c b/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-read.c
index f2cea1c7befc..ac965c34a2f8 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-read.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-eeprom-read.c
@@ -6,6 +6,7 @@
  * GPL LICENSE SUMMARY
  *
  * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -31,6 +32,7 @@
  * BSD LICENSE
  *
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -199,12 +201,12 @@ static int iwl_init_otp_access(struct iwl_trans *trans)
 	/* Enable 40MHz radio clock */
 	iwl_write32(trans, CSR_GP_CNTRL,
 		    iwl_read32(trans, CSR_GP_CNTRL) |
-		    CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
+		    BIT(trans->cfg->csr->flag_init_done));
 
 	/* wait for clock to be ready */
 	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
-			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
-			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
+			   BIT(trans->cfg->csr->flag_mac_clock_ready),
+			   BIT(trans->cfg->csr->flag_mac_clock_ready),
 			   25000);
 	if (ret < 0) {
 		IWL_ERR(trans, "Time out access OTP\n");
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h b/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h
index a41c46e63eb1..a7dd8a8cddf9 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-modparams.h
@@ -122,6 +122,7 @@ enum iwl_uapsd_disable {
  * @lar_disable: disable LAR (regulatory), default = 0
  * @fw_monitor: allow to use firmware monitor
  * @disable_11ac: disable VHT capabilities, default = false.
+ * @remove_when_gone: remove an inaccessible device from the PCIe bus.
  */
 struct iwl_mod_params {
 	int swcrypto;
@@ -143,6 +144,7 @@ struct iwl_mod_params {
 	bool lar_disable;
 	bool fw_monitor;
 	bool disable_11ac;
+	bool remove_when_gone;
 };
 
 #endif /* #__iwl_modparams_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
index ca0174680af9..b815ba38dbdb 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -35,6 +36,7 @@
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -68,6 +70,7 @@
 #include <linux/export.h>
 #include <linux/etherdevice.h>
 #include <linux/pci.h>
+#include <linux/firmware.h>
 
 #include "iwl-drv.h"
 #include "iwl-modparams.h"
@@ -77,6 +80,9 @@
 #include "iwl-csr.h"
 #include "fw/acpi.h"
 #include "fw/api/nvm-reg.h"
+#include "fw/api/commands.h"
+#include "fw/api/cmdhdr.h"
+#include "fw/img.h"
 
 /* NVM offsets (in words) definitions */
 enum nvm_offsets {
@@ -292,7 +298,7 @@ static u32 iwl_get_channel_flags(u8 ch_num, int ch_idx, bool is_5ghz,
 static int iwl_init_channel_map(struct device *dev, const struct iwl_cfg *cfg,
 				struct iwl_nvm_data *data,
 				const __le16 * const nvm_ch_flags,
-				bool lar_supported, bool no_wide_in_5ghz)
+				u32 sbands_flags)
 {
 	int ch_idx;
 	int n_channels = 0;
@@ -316,11 +322,12 @@ static int iwl_init_channel_map(struct device *dev, const struct iwl_cfg *cfg,
 
 		ch_flags = __le16_to_cpup(nvm_ch_flags + ch_idx);
 
-		if (is_5ghz && !data->sku_cap_band_52GHz_enable)
+		if (is_5ghz && !data->sku_cap_band_52ghz_enable)
 			continue;
 
 		/* workaround to disable wide channels in 5GHz */
-		if (no_wide_in_5ghz && is_5ghz) {
+		if ((sbands_flags & IWL_NVM_SBANDS_FLAGS_NO_WIDE_IN_5GHZ) &&
+		    is_5ghz) {
 			ch_flags &= ~(NVM_CHANNEL_40MHZ |
 				     NVM_CHANNEL_80MHZ |
 				     NVM_CHANNEL_160MHZ);
@@ -329,7 +336,8 @@ static int iwl_init_channel_map(struct device *dev, const struct iwl_cfg *cfg,
 		if (ch_flags & NVM_CHANNEL_160MHZ)
 			data->vht160_supported = true;
 
-		if (!lar_supported && !(ch_flags & NVM_CHANNEL_VALID)) {
+		if (!(sbands_flags & IWL_NVM_SBANDS_FLAGS_LAR) &&
+		    !(ch_flags & NVM_CHANNEL_VALID)) {
 			/*
 			 * Channels might become valid later if lar is
 			 * supported, hence we still want to add them to
@@ -359,7 +367,7 @@ static int iwl_init_channel_map(struct device *dev, const struct iwl_cfg *cfg,
 		channel->max_power = IWL_DEFAULT_MAX_TX_POWER;
 
 		/* don't put limitations in case we're using LAR */
-		if (!lar_supported)
+		if (!(sbands_flags & IWL_NVM_SBANDS_FLAGS_LAR))
 			channel->flags = iwl_get_channel_flags(nvm_chan[ch_idx],
 							       ch_idx, is_5ghz,
 							       ch_flags, cfg);
@@ -455,17 +463,17 @@ static void iwl_init_vht_hw_capab(const struct iwl_cfg *cfg,
 	vht_cap->vht_mcs.tx_mcs_map = vht_cap->vht_mcs.rx_mcs_map;
 }
 
-void iwl_init_sbands(struct device *dev, const struct iwl_cfg *cfg,
-		     struct iwl_nvm_data *data, const __le16 *nvm_ch_flags,
-		     u8 tx_chains, u8 rx_chains, bool lar_supported,
-		     bool no_wide_in_5ghz)
+static void iwl_init_sbands(struct device *dev, const struct iwl_cfg *cfg,
+			    struct iwl_nvm_data *data,
+			    const __le16 *nvm_ch_flags, u8 tx_chains,
+			    u8 rx_chains, u32 sbands_flags)
 {
 	int n_channels;
 	int n_used = 0;
 	struct ieee80211_supported_band *sband;
 
 	n_channels = iwl_init_channel_map(dev, cfg, data, nvm_ch_flags,
-					  lar_supported, no_wide_in_5ghz);
+					  sbands_flags);
 	sband = &data->bands[NL80211_BAND_2GHZ];
 	sband->band = NL80211_BAND_2GHZ;
 	sband->bitrates = &iwl_cfg80211_rates[RATES_24_OFFS];
@@ -491,7 +499,6 @@ void iwl_init_sbands(struct device *dev, const struct iwl_cfg *cfg,
 		IWL_ERR_DEV(dev, "NVM: used only %d of %d channels\n",
 			    n_used, n_channels);
 }
-IWL_EXPORT_SYMBOL(iwl_init_sbands);
 
 static int iwl_get_sku(const struct iwl_cfg *cfg, const __le16 *nvm_sw,
 		       const __le16 *phy_sku)
@@ -569,11 +576,15 @@ static void iwl_flip_hw_address(__le32 mac_addr0, __le32 mac_addr1, u8 *dest)
 	dest[5] = hw_addr[0];
 }
 
-void iwl_set_hw_address_from_csr(struct iwl_trans *trans,
-				 struct iwl_nvm_data *data)
+static void iwl_set_hw_address_from_csr(struct iwl_trans *trans,
+					struct iwl_nvm_data *data)
 {
-	__le32 mac_addr0 = cpu_to_le32(iwl_read32(trans, CSR_MAC_ADDR0_STRAP));
-	__le32 mac_addr1 = cpu_to_le32(iwl_read32(trans, CSR_MAC_ADDR1_STRAP));
+	__le32 mac_addr0 =
+		cpu_to_le32(iwl_read32(trans,
+				       trans->cfg->csr->mac_addr0_strap));
+	__le32 mac_addr1 =
+		cpu_to_le32(iwl_read32(trans,
+				       trans->cfg->csr->mac_addr1_strap));
 
 	iwl_flip_hw_address(mac_addr0, mac_addr1, data->hw_addr);
 	/*
@@ -583,12 +594,13 @@ void iwl_set_hw_address_from_csr(struct iwl_trans *trans,
 	if (is_valid_ether_addr(data->hw_addr))
 		return;
 
-	mac_addr0 = cpu_to_le32(iwl_read32(trans, CSR_MAC_ADDR0_OTP));
-	mac_addr1 = cpu_to_le32(iwl_read32(trans, CSR_MAC_ADDR1_OTP));
+	mac_addr0 = cpu_to_le32(iwl_read32(trans,
+					   trans->cfg->csr->mac_addr0_otp));
+	mac_addr1 = cpu_to_le32(iwl_read32(trans,
+					   trans->cfg->csr->mac_addr1_otp));
 
 	iwl_flip_hw_address(mac_addr0, mac_addr1, data->hw_addr);
 }
-IWL_EXPORT_SYMBOL(iwl_set_hw_address_from_csr);
 
 static void iwl_set_hw_address_family_8000(struct iwl_trans *trans,
 					   const struct iwl_cfg *cfg,
@@ -713,8 +725,8 @@ iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	struct device *dev = trans->dev;
 	struct iwl_nvm_data *data;
 	bool lar_enabled;
-	bool no_wide_in_5ghz = iwl_nvm_no_wide_in_5ghz(dev, cfg, nvm_hw);
 	u32 sku, radio_cfg;
+	u32 sbands_flags = 0;
 	u16 lar_config;
 	const __le16 *ch_section;
 
@@ -741,8 +753,8 @@ iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 		rx_chains &= data->valid_rx_ant;
 
 	sku = iwl_get_sku(cfg, nvm_sw, phy_sku);
-	data->sku_cap_band_24GHz_enable = sku & NVM_SKU_CAP_BAND_24GHZ;
-	data->sku_cap_band_52GHz_enable = sku & NVM_SKU_CAP_BAND_52GHZ;
+	data->sku_cap_band_24ghz_enable = sku & NVM_SKU_CAP_BAND_24GHZ;
+	data->sku_cap_band_52ghz_enable = sku & NVM_SKU_CAP_BAND_52GHZ;
 	data->sku_cap_11n_enable = sku & NVM_SKU_CAP_11N_ENABLE;
 	if (iwlwifi_mod_params.disable_11n & IWL_DISABLE_HT_ALL)
 		data->sku_cap_11n_enable = false;
@@ -787,8 +799,14 @@ iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 		return NULL;
 	}
 
+	if (lar_fw_supported && lar_enabled)
+		sbands_flags |= IWL_NVM_SBANDS_FLAGS_LAR;
+
+	if (iwl_nvm_no_wide_in_5ghz(dev, cfg, nvm_hw))
+		sbands_flags |= IWL_NVM_SBANDS_FLAGS_NO_WIDE_IN_5GHZ;
+
 	iwl_init_sbands(dev, cfg, data, ch_section, tx_chains, rx_chains,
-			lar_fw_supported && lar_enabled, no_wide_in_5ghz);
+			sbands_flags);
 	data->calib_version = 255;
 
 	return data;
@@ -1007,8 +1025,7 @@ iwl_parse_nvm_mcc_info(struct device *dev, const struct iwl_cfg *cfg,
 			continue;
 
 		copy_rd->reg_rules[i].wmm_rule = d_wmm +
-			(regd->reg_rules[i].wmm_rule - s_wmm) /
-			sizeof(struct ieee80211_wmm_rule);
+			(regd->reg_rules[i].wmm_rule - s_wmm);
 	}
 
 out:
@@ -1017,3 +1034,294 @@ iwl_parse_nvm_mcc_info(struct device *dev, const struct iwl_cfg *cfg,
 	return copy_rd;
 }
 IWL_EXPORT_SYMBOL(iwl_parse_nvm_mcc_info);
+
+#define IWL_MAX_NVM_SECTION_SIZE	0x1b58
+#define IWL_MAX_EXT_NVM_SECTION_SIZE	0x1ffc
+#define MAX_NVM_FILE_LEN	16384
+
+void iwl_nvm_fixups(u32 hw_id, unsigned int section, u8 *data,
+		    unsigned int len)
+{
+#define IWL_4165_DEVICE_ID	0x5501
+#define NVM_SKU_CAP_MIMO_DISABLE BIT(5)
+
+	if (section == NVM_SECTION_TYPE_PHY_SKU &&
+	    hw_id == IWL_4165_DEVICE_ID && data && len >= 5 &&
+	    (data[4] & NVM_SKU_CAP_MIMO_DISABLE))
+		/* OTP 0x52 bug work around: it's a 1x1 device */
+		data[3] = ANT_B | (ANT_B << 4);
+}
+IWL_EXPORT_SYMBOL(iwl_nvm_fixups);
+
+/*
+ * Reads external NVM from a file into mvm->nvm_sections
+ *
+ * HOW TO CREATE THE NVM FILE FORMAT:
+ * ------------------------------
+ * 1. create hex file, format:
+ *      3800 -> header
+ *      0000 -> header
+ *      5a40 -> data
+ *
+ *   rev - 6 bit (word1)
+ *   len - 10 bit (word1)
+ *   id - 4 bit (word2)
+ *   rsv - 12 bit (word2)
+ *
+ * 2. flip 8bits with 8 bits per line to get the right NVM file format
+ *
+ * 3. create binary file from the hex file
+ *
+ * 4. save as "iNVM_xxx.bin" under /lib/firmware
+ */
+int iwl_read_external_nvm(struct iwl_trans *trans,
+			  const char *nvm_file_name,
+			  struct iwl_nvm_section *nvm_sections)
+{
+	int ret, section_size;
+	u16 section_id;
+	const struct firmware *fw_entry;
+	const struct {
+		__le16 word1;
+		__le16 word2;
+		u8 data[];
+	} *file_sec;
+	const u8 *eof;
+	u8 *temp;
+	int max_section_size;
+	const __le32 *dword_buff;
+
+#define NVM_WORD1_LEN(x) (8 * (x & 0x03FF))
+#define NVM_WORD2_ID(x) (x >> 12)
+#define EXT_NVM_WORD2_LEN(x) (2 * (((x) & 0xFF) << 8 | (x) >> 8))
+#define EXT_NVM_WORD1_ID(x) ((x) >> 4)
+#define NVM_HEADER_0	(0x2A504C54)
+#define NVM_HEADER_1	(0x4E564D2A)
+#define NVM_HEADER_SIZE	(4 * sizeof(u32))
+
+	IWL_DEBUG_EEPROM(trans->dev, "Read from external NVM\n");
+
+	/* Maximal size depends on NVM version */
+	if (trans->cfg->nvm_type != IWL_NVM_EXT)
+		max_section_size = IWL_MAX_NVM_SECTION_SIZE;
+	else
+		max_section_size = IWL_MAX_EXT_NVM_SECTION_SIZE;
+
+	/*
+	 * Obtain NVM image via request_firmware. Since we already used
+	 * request_firmware_nowait() for the firmware binary load and only
+	 * get here after that we assume the NVM request can be satisfied
+	 * synchronously.
+	 */
+	ret = request_firmware(&fw_entry, nvm_file_name, trans->dev);
+	if (ret) {
+		IWL_ERR(trans, "ERROR: %s isn't available %d\n",
+			nvm_file_name, ret);
+		return ret;
+	}
+
+	IWL_INFO(trans, "Loaded NVM file %s (%zu bytes)\n",
+		 nvm_file_name, fw_entry->size);
+
+	if (fw_entry->size > MAX_NVM_FILE_LEN) {
+		IWL_ERR(trans, "NVM file too large\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	eof = fw_entry->data + fw_entry->size;
+	dword_buff = (__le32 *)fw_entry->data;
+
+	/* some NVM file will contain a header.
+	 * The header is identified by 2 dwords header as follow:
+	 * dword[0] = 0x2A504C54
+	 * dword[1] = 0x4E564D2A
+	 *
+	 * This header must be skipped when providing the NVM data to the FW.
+	 */
+	if (fw_entry->size > NVM_HEADER_SIZE &&
+	    dword_buff[0] == cpu_to_le32(NVM_HEADER_0) &&
+	    dword_buff[1] == cpu_to_le32(NVM_HEADER_1)) {
+		file_sec = (void *)(fw_entry->data + NVM_HEADER_SIZE);
+		IWL_INFO(trans, "NVM Version %08X\n", le32_to_cpu(dword_buff[2]));
+		IWL_INFO(trans, "NVM Manufacturing date %08X\n",
+			 le32_to_cpu(dword_buff[3]));
+
+		/* nvm file validation, dword_buff[2] holds the file version */
+		if (trans->cfg->device_family == IWL_DEVICE_FAMILY_8000 &&
+		    CSR_HW_REV_STEP(trans->hw_rev) == SILICON_C_STEP &&
+		    le32_to_cpu(dword_buff[2]) < 0xE4A) {
+			ret = -EFAULT;
+			goto out;
+		}
+	} else {
+		file_sec = (void *)fw_entry->data;
+	}
+
+	while (true) {
+		if (file_sec->data > eof) {
+			IWL_ERR(trans,
+				"ERROR - NVM file too short for section header\n");
+			ret = -EINVAL;
+			break;
+		}
+
+		/* check for EOF marker */
+		if (!file_sec->word1 && !file_sec->word2) {
+			ret = 0;
+			break;
+		}
+
+		if (trans->cfg->nvm_type != IWL_NVM_EXT) {
+			section_size =
+				2 * NVM_WORD1_LEN(le16_to_cpu(file_sec->word1));
+			section_id = NVM_WORD2_ID(le16_to_cpu(file_sec->word2));
+		} else {
+			section_size = 2 * EXT_NVM_WORD2_LEN(
+						le16_to_cpu(file_sec->word2));
+			section_id = EXT_NVM_WORD1_ID(
+						le16_to_cpu(file_sec->word1));
+		}
+
+		if (section_size > max_section_size) {
+			IWL_ERR(trans, "ERROR - section too large (%d)\n",
+				section_size);
+			ret = -EINVAL;
+			break;
+		}
+
+		if (!section_size) {
+			IWL_ERR(trans, "ERROR - section empty\n");
+			ret = -EINVAL;
+			break;
+		}
+
+		if (file_sec->data + section_size > eof) {
+			IWL_ERR(trans,
+				"ERROR - NVM file too short for section (%d bytes)\n",
+				section_size);
+			ret = -EINVAL;
+			break;
+		}
+
+		if (WARN(section_id >= NVM_MAX_NUM_SECTIONS,
+			 "Invalid NVM section ID %d\n", section_id)) {
+			ret = -EINVAL;
+			break;
+		}
+
+		temp = kmemdup(file_sec->data, section_size, GFP_KERNEL);
+		if (!temp) {
+			ret = -ENOMEM;
+			break;
+		}
+
+		iwl_nvm_fixups(trans->hw_id, section_id, temp, section_size);
+
+		kfree(nvm_sections[section_id].data);
+		nvm_sections[section_id].data = temp;
+		nvm_sections[section_id].length = section_size;
+
+		/* advance to the next section */
+		file_sec = (void *)(file_sec->data + section_size);
+	}
+out:
+	release_firmware(fw_entry);
+	return ret;
+}
+IWL_EXPORT_SYMBOL(iwl_read_external_nvm);
+
+struct iwl_nvm_data *iwl_get_nvm(struct iwl_trans *trans,
+				 const struct iwl_fw *fw)
+{
+	struct iwl_nvm_get_info cmd = {};
+	struct iwl_nvm_get_info_rsp *rsp;
+	struct iwl_nvm_data *nvm;
+	struct iwl_host_cmd hcmd = {
+		.flags = CMD_WANT_SKB | CMD_SEND_IN_RFKILL,
+		.data = { &cmd, },
+		.len = { sizeof(cmd) },
+		.id = WIDE_ID(REGULATORY_AND_NVM_GROUP, NVM_GET_INFO)
+	};
+	int  ret;
+	bool lar_fw_supported = !iwlwifi_mod_params.lar_disable &&
+				fw_has_capa(&fw->ucode_capa,
+					    IWL_UCODE_TLV_CAPA_LAR_SUPPORT);
+	u32 mac_flags;
+	u32 sbands_flags = 0;
+
+	ret = iwl_trans_send_cmd(trans, &hcmd);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (WARN(iwl_rx_packet_payload_len(hcmd.resp_pkt) != sizeof(*rsp),
+		 "Invalid payload len in NVM response from FW %d",
+		 iwl_rx_packet_payload_len(hcmd.resp_pkt))) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	rsp = (void *)hcmd.resp_pkt->data;
+	if (le32_to_cpu(rsp->general.flags) & NVM_GENERAL_FLAGS_EMPTY_OTP)
+		IWL_INFO(trans, "OTP is empty\n");
+
+	nvm = kzalloc(sizeof(*nvm) +
+		      sizeof(struct ieee80211_channel) * IWL_NUM_CHANNELS,
+		      GFP_KERNEL);
+	if (!nvm) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	iwl_set_hw_address_from_csr(trans, nvm);
+	/* TODO: if platform NVM has MAC address - override it here */
+
+	if (!is_valid_ether_addr(nvm->hw_addr)) {
+		IWL_ERR(trans, "no valid mac address was found\n");
+		ret = -EINVAL;
+		goto err_free;
+	}
+
+	IWL_INFO(trans, "base HW address: %pM\n", nvm->hw_addr);
+
+	/* Initialize general data */
+	nvm->nvm_version = le16_to_cpu(rsp->general.nvm_version);
+
+	/* Initialize MAC sku data */
+	mac_flags = le32_to_cpu(rsp->mac_sku.mac_sku_flags);
+	nvm->sku_cap_11ac_enable =
+		!!(mac_flags & NVM_MAC_SKU_FLAGS_802_11AC_ENABLED);
+	nvm->sku_cap_11n_enable =
+		!!(mac_flags & NVM_MAC_SKU_FLAGS_802_11N_ENABLED);
+	nvm->sku_cap_band_24ghz_enable =
+		!!(mac_flags & NVM_MAC_SKU_FLAGS_BAND_2_4_ENABLED);
+	nvm->sku_cap_band_52ghz_enable =
+		!!(mac_flags & NVM_MAC_SKU_FLAGS_BAND_5_2_ENABLED);
+	nvm->sku_cap_mimo_disabled =
+		!!(mac_flags & NVM_MAC_SKU_FLAGS_MIMO_DISABLED);
+
+	/* Initialize PHY sku data */
+	nvm->valid_tx_ant = (u8)le32_to_cpu(rsp->phy_sku.tx_chains);
+	nvm->valid_rx_ant = (u8)le32_to_cpu(rsp->phy_sku.rx_chains);
+
+	if (le32_to_cpu(rsp->regulatory.lar_enabled) && lar_fw_supported) {
+		nvm->lar_enabled = true;
+		sbands_flags |= IWL_NVM_SBANDS_FLAGS_LAR;
+	}
+
+	iwl_init_sbands(trans->dev, trans->cfg, nvm,
+			rsp->regulatory.channel_profile,
+			nvm->valid_tx_ant & fw->valid_tx_ant,
+			nvm->valid_rx_ant & fw->valid_rx_ant,
+			sbands_flags);
+
+	iwl_free_resp(&hcmd);
+	return nvm;
+
+err_free:
+	kfree(nvm);
+out:
+	iwl_free_resp(&hcmd);
+	return ERR_PTR(ret);
+}
+IWL_EXPORT_SYMBOL(iwl_get_nvm);
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
index 3071a23b7606..234d1009a9de 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
@@ -7,6 +7,7 @@
  *
  * Copyright(c) 2008 - 2015 Intel Corporation. All rights reserved.
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -33,6 +34,7 @@
  *
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -67,6 +69,17 @@
 #include <net/cfg80211.h>
 #include "iwl-eeprom-parse.h"
 
+/**
+ * enum iwl_nvm_sbands_flags - modification flags for the channel profiles
+ *
+ * @IWL_NVM_SBANDS_FLAGS_LAR: LAR is enabled
+ * @IWL_NVM_SBANDS_FLAGS_NO_WIDE_IN_5GHZ: disallow 40, 80 and 160MHz on 5GHz
+ */
+enum iwl_nvm_sbands_flags {
+	IWL_NVM_SBANDS_FLAGS_LAR		= BIT(0),
+	IWL_NVM_SBANDS_FLAGS_NO_WIDE_IN_5GHZ	= BIT(1),
+};
+
 /**
  * iwl_parse_nvm_data - parse NVM data and return values
  *
@@ -82,20 +95,6 @@ iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 		   const __le16 *mac_override, const __le16 *phy_sku,
 		   u8 tx_chains, u8 rx_chains, bool lar_fw_supported);
 
-/**
- * iwl_set_hw_address_from_csr - sets HW address for 9000 devices and on
- */
-void iwl_set_hw_address_from_csr(struct iwl_trans *trans,
-				 struct iwl_nvm_data *data);
-
-/**
- * iwl_init_sbands - parse and set all channel profiles
- */
-void iwl_init_sbands(struct device *dev, const struct iwl_cfg *cfg,
-		     struct iwl_nvm_data *data, const __le16 *nvm_ch_flags,
-		     u8 tx_chains, u8 rx_chains, bool lar_supported,
-		     bool no_wide_in_5ghz);
-
 /**
  * iwl_parse_mcc_info - parse MCC (mobile country code) info coming from FW
  *
@@ -111,4 +110,33 @@ iwl_parse_nvm_mcc_info(struct device *dev, const struct iwl_cfg *cfg,
 		       int num_of_ch, __le32 *channels, u16 fw_mcc,
 		       u16 geo_info);
 
+/**
+ * struct iwl_nvm_section - describes an NVM section in memory.
+ *
+ * This struct holds an NVM section read from the NIC using NVM_ACCESS_CMD,
+ * and saved for later use by the driver. Not all NVM sections are saved
+ * this way, only the needed ones.
+ */
+struct iwl_nvm_section {
+	u16 length;
+	const u8 *data;
+};
+
+/**
+ * iwl_read_external_nvm - Reads external NVM from a file into nvm_sections
+ */
+int iwl_read_external_nvm(struct iwl_trans *trans,
+			  const char *nvm_file_name,
+			  struct iwl_nvm_section *nvm_sections);
+void iwl_nvm_fixups(u32 hw_id, unsigned int section, u8 *data,
+		    unsigned int len);
+
+/**
+ * iwl_get_nvm - retrieve NVM data from firmware
+ *
+ * Allocates a new iwl_nvm_data structure, fills it with
+ * NVM data, and returns it to caller.
+ */
+struct iwl_nvm_data *iwl_get_nvm(struct iwl_trans *trans,
+				 const struct iwl_fw *fw);
 #endif /* __iwl_nvm_parse_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
index c25ed1a0bbb0..1b9c627ee34d 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
@@ -554,7 +554,7 @@ struct iwl_trans_ops {
 	/* 22000 functions */
 	int (*txq_alloc)(struct iwl_trans *trans,
 			 struct iwl_tx_queue_cfg_cmd *cmd,
-			 int cmd_id,
+			 int cmd_id, int size,
 			 unsigned int queue_wdg_timeout);
 	void (*txq_free)(struct iwl_trans *trans, int queue);
 
@@ -691,6 +691,8 @@ enum iwl_plat_pm_mode {
  * @wide_cmd_header: true when ucode supports wide command header format
  * @num_rx_queues: number of RX queues allocated by the transport;
  *	the transport must set this before calling iwl_drv_start()
+ * @iml_len: the length of the image loader
+ * @iml: a pointer to the image loader itself
  * @dev_cmd_pool: pool for Tx cmd allocation - for internal use only.
  *	The user should use iwl_trans_{alloc,free}_tx_cmd.
  * @rx_mpdu_cmd: MPDU RX command ID, must be assigned by opmode before
@@ -735,6 +737,9 @@ struct iwl_trans {
 
 	u8 num_rx_queues;
 
+	size_t iml_len;
+	u8 *iml;
+
 	/* The following fields are internal only */
 	struct kmem_cache *dev_cmd_pool;
 	char dev_cmd_pool_name[50];
@@ -952,8 +957,8 @@ iwl_trans_txq_free(struct iwl_trans *trans, int queue)
 static inline int
 iwl_trans_txq_alloc(struct iwl_trans *trans,
 		    struct iwl_tx_queue_cfg_cmd *cmd,
-		    int cmd_id,
-		    unsigned int queue_wdg_timeout)
+		    int cmd_id, int size,
+		    unsigned int wdg_timeout)
 {
 	might_sleep();
 
@@ -965,7 +970,7 @@ iwl_trans_txq_alloc(struct iwl_trans *trans,
 		return -EIO;
 	}
 
-	return trans->ops->txq_alloc(trans, cmd, cmd_id, queue_wdg_timeout);
+	return trans->ops->txq_alloc(trans, cmd, cmd_id, size, wdg_timeout);
 }
 
 static inline void iwl_trans_txq_set_shared_mode(struct iwl_trans *trans,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/coex.c b/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
index 890dbfff3a06..016e03a5034f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
@@ -279,6 +279,8 @@ struct iwl_bt_iterator_data {
 	struct ieee80211_chanctx_conf *primary;
 	struct ieee80211_chanctx_conf *secondary;
 	bool primary_ll;
+	u8 primary_load;
+	u8 secondary_load;
 };
 
 static inline
@@ -295,6 +297,30 @@ void iwl_mvm_bt_coex_enable_rssi_event(struct iwl_mvm *mvm,
 		enable ? -IWL_MVM_BT_COEX_DIS_RED_TXP_THRESH : 0;
 }
 
+#define MVM_COEX_TCM_PERIOD (HZ * 10)
+
+static void iwl_mvm_bt_coex_tcm_based_ci(struct iwl_mvm *mvm,
+					 struct iwl_bt_iterator_data *data)
+{
+	unsigned long now = jiffies;
+
+	if (!time_after(now, mvm->bt_coex_last_tcm_ts + MVM_COEX_TCM_PERIOD))
+		return;
+
+	mvm->bt_coex_last_tcm_ts = now;
+
+	/* We assume here that we don't have more than 2 vifs on 2.4GHz */
+
+	/* if the primary is low latency, it will stay primary */
+	if (data->primary_ll)
+		return;
+
+	if (data->primary_load >= data->secondary_load)
+		return;
+
+	swap(data->primary, data->secondary);
+}
+
 /* must be called under rcu_read_lock */
 static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 				      struct ieee80211_vif *vif)
@@ -385,6 +411,11 @@ static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 			/* there is low latency vif - we will be secondary */
 			data->secondary = chanctx_conf;
 		}
+
+		if (data->primary == chanctx_conf)
+			data->primary_load = mvm->tcm.result.load[mvmvif->id];
+		else if (data->secondary == chanctx_conf)
+			data->secondary_load = mvm->tcm.result.load[mvmvif->id];
 		return;
 	}
 
@@ -398,6 +429,10 @@ static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 		/* if secondary is not NULL, it might be a GO */
 		data->secondary = chanctx_conf;
 
+	if (data->primary == chanctx_conf)
+		data->primary_load = mvm->tcm.result.load[mvmvif->id];
+	else if (data->secondary == chanctx_conf)
+		data->secondary_load = mvm->tcm.result.load[mvmvif->id];
 	/*
 	 * don't reduce the Tx power if one of these is true:
 	 *  we are in LOOSE
@@ -449,6 +484,8 @@ static void iwl_mvm_bt_coex_notif_handle(struct iwl_mvm *mvm)
 					mvm->hw, IEEE80211_IFACE_ITER_NORMAL,
 					iwl_mvm_bt_notif_iterator, &data);
 
+	iwl_mvm_bt_coex_tcm_based_ci(mvm, &data);
+
 	if (data.primary) {
 		struct ieee80211_chanctx_conf *chan = data.primary;
 		if (WARN_ON(!chan->def.chan)) {
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/constants.h b/drivers/net/wireless/intel/iwlwifi/mvm/constants.h
index 96b52a275ee3..d61ff66ce07b 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/constants.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/constants.h
@@ -69,6 +69,8 @@
 
 #include <linux/ieee80211.h>
 
+#define IWL_MVM_UAPSD_NOAGG_BSSIDS_NUM		20
+
 #define IWL_MVM_DEFAULT_PS_TX_DATA_TIMEOUT	(100 * USEC_PER_MSEC)
 #define IWL_MVM_DEFAULT_PS_RX_DATA_TIMEOUT	(100 * USEC_PER_MSEC)
 #define IWL_MVM_WOWLAN_PS_TX_DATA_TIMEOUT	(10 * USEC_PER_MSEC)
@@ -112,6 +114,11 @@
 #define IWL_MVM_PARSE_NVM			0
 #define IWL_MVM_ADWELL_ENABLE			1
 #define IWL_MVM_ADWELL_MAX_BUDGET		0
+#define IWL_MVM_TCM_LOAD_MEDIUM_THRESH		10 /* percentage */
+#define IWL_MVM_TCM_LOAD_HIGH_THRESH		50 /* percentage */
+#define IWL_MVM_TCM_LOWLAT_ENABLE_THRESH	100 /* packets/10 seconds */
+#define IWL_MVM_UAPSD_NONAGG_PERIOD		5000 /* msecs */
+#define IWL_MVM_UAPSD_NOAGG_LIST_LEN		IWL_MVM_UAPSD_NOAGG_BSSIDS_NUM
 #define IWL_MVM_RS_NUM_TRY_BEFORE_ANT_TOGGLE    1
 #define IWL_MVM_RS_HT_VHT_RETRIES_PER_RATE      2
 #define IWL_MVM_RS_HT_VHT_RETRIES_PER_RATE_TW   1
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
index 2efe9b099556..3fcf489f3120 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -18,11 +19,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -35,6 +31,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -693,6 +690,14 @@ iwl_mvm_get_wowlan_config(struct iwl_mvm *mvm,
 				    IWL_WOWLAN_WAKEUP_LINK_CHANGE);
 	}
 
+	if (wowlan->any) {
+		wowlan_config_cmd->wakeup_filter |=
+			cpu_to_le32(IWL_WOWLAN_WAKEUP_BEACON_MISS |
+				    IWL_WOWLAN_WAKEUP_LINK_CHANGE |
+				    IWL_WOWLAN_WAKEUP_RX_FRAME |
+				    IWL_WOWLAN_WAKEUP_BCN_FILTERING);
+	}
+
 	return 0;
 }
 
@@ -1097,6 +1102,7 @@ int iwl_mvm_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan)
 
 	/* make sure the d0i3 exit work is not pending */
 	flush_work(&mvm->d0i3_exit_work);
+	iwl_mvm_pause_tcm(mvm, true);
 
 	iwl_fw_runtime_suspend(&mvm->fwrt);
 
@@ -2014,6 +2020,8 @@ int iwl_mvm_resume(struct ieee80211_hw *hw)
 
 	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
 
+	iwl_mvm_resume_tcm(mvm);
+
 	iwl_fw_runtime_resume(&mvm->fwrt);
 
 	return ret;
@@ -2042,6 +2050,8 @@ static int iwl_mvm_d3_test_open(struct inode *inode, struct file *file)
 
 	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_D3;
 
+	iwl_mvm_pause_tcm(mvm, true);
+
 	iwl_fw_runtime_suspend(&mvm->fwrt);
 
 	/* start pseudo D3 */
@@ -2104,6 +2114,8 @@ static int iwl_mvm_d3_test_release(struct inode *inode, struct file *file)
 	__iwl_mvm_resume(mvm, true);
 	rtnl_unlock();
 
+	iwl_mvm_resume_tcm(mvm);
+
 	iwl_fw_runtime_resume(&mvm->fwrt);
 
 	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
index f7fcf700196b..798605c4f122 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
@@ -269,6 +269,8 @@ static ssize_t iwl_dbgfs_mac_params_read(struct file *file,
 			 mvmvif->id, mvmvif->color);
 	pos += scnprintf(buf+pos, bufsz-pos, "bssid: %pM\n",
 			 vif->bss_conf.bssid);
+	pos += scnprintf(buf+pos, bufsz-pos, "Load: %d\n",
+			 mvm->tcm.result.load[mvmvif->id]);
 	pos += scnprintf(buf+pos, bufsz-pos, "QoS:\n");
 	for (i = 0; i < ARRAY_SIZE(mvmvif->queue_params); i++)
 		pos += scnprintf(buf+pos, bufsz-pos,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
index 0e6401cd7ccc..1c4178f20441 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
@@ -1728,6 +1728,27 @@ iwl_dbgfs_send_echo_cmd_write(struct iwl_mvm *mvm, char *buf,
 	return ret ?: count;
 }
 
+static ssize_t
+iwl_dbgfs_uapsd_noagg_bssids_read(struct file *file, char __user *user_buf,
+				  size_t count, loff_t *ppos)
+{
+	struct iwl_mvm *mvm = file->private_data;
+	u8 buf[IWL_MVM_UAPSD_NOAGG_BSSIDS_NUM * ETH_ALEN * 3 + 1];
+	unsigned int pos = 0;
+	size_t bufsz = sizeof(buf);
+	int i;
+
+	mutex_lock(&mvm->mutex);
+
+	for (i = 0; i < IWL_MVM_UAPSD_NOAGG_LIST_LEN; i++)
+		pos += scnprintf(buf + pos, bufsz - pos, "%pM\n",
+				 mvm->uapsd_noagg_bssids[i].addr);
+
+	mutex_unlock(&mvm->mutex);
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
+}
+
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(prph_reg, 64);
 
 /* Device wide debugfs entries */
@@ -1762,6 +1783,8 @@ MVM_DEBUGFS_WRITE_FILE_OPS(indirection_tbl,
 			   (IWL_RSS_INDIRECTION_TABLE_SIZE * 2));
 MVM_DEBUGFS_WRITE_FILE_OPS(inject_packet, 512);
 
+MVM_DEBUGFS_READ_FILE_OPS(uapsd_noagg_bssids);
+
 #ifdef CONFIG_IWLWIFI_BCAST_FILTERING
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(bcast_filters, 256);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(bcast_filters_macs, 256);
@@ -1972,6 +1995,8 @@ int iwl_mvm_dbgfs_register(struct iwl_mvm *mvm, struct dentry *dbgfs_dir)
 				 mvm->debugfs_dir, &mvm->drop_bcn_ap_mode))
 		goto err;
 
+	MVM_DEBUGFS_ADD_FILE(uapsd_noagg_bssids, mvm->debugfs_dir, S_IRUSR);
+
 #ifdef CONFIG_IWLWIFI_BCAST_FILTERING
 	if (mvm->fw->ucode_capa.flags & IWL_UCODE_TLV_FLAGS_BCAST_FILTERING) {
 		bcast_dir = debugfs_create_dir("bcast_filtering",
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
index 3c59109bea20..866c91c923be 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -35,6 +36,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -79,6 +81,8 @@
 #include "mvm.h"
 #include "fw/dbg.h"
 #include "iwl-phy-db.h"
+#include "iwl-modparams.h"
+#include "iwl-nvm-parse.h"
 
 #define MVM_UCODE_ALIVE_TIMEOUT	HZ
 #define MVM_UCODE_CALIB_TIMEOUT	(2*HZ)
@@ -381,7 +385,8 @@ static int iwl_run_unified_mvm_ucode(struct iwl_mvm *mvm, bool read_nvm)
 
 	/* Load NVM to NIC if needed */
 	if (mvm->nvm_file_name) {
-		iwl_mvm_read_external_nvm(mvm);
+		iwl_read_external_nvm(mvm->trans, mvm->nvm_file_name,
+				      mvm->nvm_sections);
 		iwl_mvm_load_nvm_to_nic(mvm);
 	}
 
@@ -410,7 +415,7 @@ static int iwl_run_unified_mvm_ucode(struct iwl_mvm *mvm, bool read_nvm)
 
 	/* Read the NVM only at driver load time, no need to do this twice */
 	if (!IWL_MVM_PARSE_NVM && read_nvm) {
-		mvm->nvm_data = iwl_fw_get_nvm(&mvm->fwrt);
+		mvm->nvm_data = iwl_get_nvm(mvm->trans, mvm->fw);
 		if (IS_ERR(mvm->nvm_data)) {
 			ret = PTR_ERR(mvm->nvm_data);
 			mvm->nvm_data = NULL;
@@ -1093,6 +1098,7 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 
 	if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_UMAC_SCAN)) {
 		mvm->scan_type = IWL_SCAN_TYPE_NOT_SET;
+		mvm->hb_scan_type = IWL_SCAN_TYPE_NOT_SET;
 		ret = iwl_mvm_config_scan(mvm);
 		if (ret)
 			goto error;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
index 90f8c89ea59c..94202e202fd3 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
@@ -953,6 +953,16 @@ static int iwl_mvm_mac_ampdu_action(struct ieee80211_hw *hw,
 
 	switch (action) {
 	case IEEE80211_AMPDU_RX_START:
+		if (iwl_mvm_vif_from_mac80211(vif)->ap_sta_id ==
+				iwl_mvm_sta_from_mac80211(sta)->sta_id) {
+			struct iwl_mvm_vif *mvmvif;
+			u16 macid = iwl_mvm_vif_from_mac80211(vif)->id;
+			struct iwl_mvm_tcm_mac *mdata = &mvm->tcm.data[macid];
+
+			mdata->opened_rx_ba_sessions = true;
+			mvmvif = iwl_mvm_vif_from_mac80211(vif);
+			cancel_delayed_work(&mvmvif->uapsd_nonagg_detected_wk);
+		}
 		if (!iwl_enable_rx_ampdu(mvm->cfg)) {
 			ret = -EINVAL;
 			break;
@@ -1436,6 +1446,8 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 		mvm->p2p_device_vif = vif;
 	}
 
+	iwl_mvm_tcm_add_vif(mvm, vif);
+
 	if (vif->type == NL80211_IFTYPE_MONITOR)
 		mvm->monitor_on = true;
 
@@ -1487,6 +1499,10 @@ static void iwl_mvm_mac_remove_interface(struct ieee80211_hw *hw,
 
 	iwl_mvm_prepare_mac_removal(mvm, vif);
 
+	if (!(vif->type == NL80211_IFTYPE_AP ||
+	      vif->type == NL80211_IFTYPE_ADHOC))
+		iwl_mvm_tcm_rm_vif(mvm, vif);
+
 	mutex_lock(&mvm->mutex);
 
 	if (mvm->bf_allowed_vif == mvmvif) {
@@ -2536,6 +2552,16 @@ static void iwl_mvm_sta_pre_rcu_remove(struct ieee80211_hw *hw,
 static void iwl_mvm_check_uapsd(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 				const u8 *bssid)
 {
+	int i;
+
+	if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
+		struct iwl_mvm_tcm_mac *mdata;
+
+		mdata = &mvm->tcm.data[iwl_mvm_vif_from_mac80211(vif)->id];
+		ewma_rate_init(&mdata->uapsd_nonagg_detect.rate);
+		mdata->opened_rx_ba_sessions = false;
+	}
+
 	if (!(mvm->fw->ucode_capa.flags & IWL_UCODE_TLV_FLAGS_UAPSD_SUPPORT))
 		return;
 
@@ -2550,6 +2576,13 @@ static void iwl_mvm_check_uapsd(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		return;
 	}
 
+	for (i = 0; i < IWL_MVM_UAPSD_NOAGG_LIST_LEN; i++) {
+		if (ether_addr_equal(mvm->uapsd_noagg_bssids[i].addr, bssid)) {
+			vif->driver_flags &= ~IEEE80211_VIF_SUPPORTS_UAPSD;
+			return;
+		}
+	}
+
 	vif->driver_flags |= IEEE80211_VIF_SUPPORTS_UAPSD;
 }
 
@@ -2652,7 +2685,7 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 
 	mutex_lock(&mvm->mutex);
 	/* track whether or not the station is associated */
-	mvm_sta->associated = new_state >= IEEE80211_STA_ASSOC;
+	mvm_sta->sta_state = new_state;
 
 	if (old_state == IEEE80211_STA_NOTEXIST &&
 	    new_state == IEEE80211_STA_NONE) {
@@ -2704,8 +2737,7 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 			iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL);
 		}
 
-		iwl_mvm_rs_rate_init(mvm, sta, mvmvif->phy_ctxt->channel->band,
-				     true);
+		iwl_mvm_rs_rate_init(mvm, sta, mvmvif->phy_ctxt->channel->band);
 		ret = iwl_mvm_update_sta(mvm, vif, sta);
 	} else if (old_state == IEEE80211_STA_ASSOC &&
 		   new_state == IEEE80211_STA_AUTHORIZED) {
@@ -2721,8 +2753,7 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 		/* enable beacon filtering */
 		WARN_ON(iwl_mvm_enable_beacon_filter(mvm, vif, 0));
 
-		iwl_mvm_rs_rate_init(mvm, sta, mvmvif->phy_ctxt->channel->band,
-				     false);
+		iwl_mvm_rs_rate_init(mvm, sta, mvmvif->phy_ctxt->channel->band);
 
 		ret = 0;
 	} else if (old_state == IEEE80211_STA_AUTHORIZED &&
@@ -2811,7 +2842,8 @@ static int iwl_mvm_mac_conf_tx(struct ieee80211_hw *hw,
 }
 
 static void iwl_mvm_mac_mgd_prepare_tx(struct ieee80211_hw *hw,
-				      struct ieee80211_vif *vif)
+				       struct ieee80211_vif *vif,
+				       u16 req_duration)
 {
 	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
 	u32 duration = IWL_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS;
@@ -2824,6 +2856,9 @@ static void iwl_mvm_mac_mgd_prepare_tx(struct ieee80211_hw *hw,
 	if (iwl_mvm_ref_sync(mvm, IWL_MVM_REF_PREPARE_TX))
 		return;
 
+	if (req_duration > duration)
+		duration = req_duration;
+
 	mutex_lock(&mvm->mutex);
 	/* Try really hard to protect the session and hear a beacon */
 	iwl_mvm_protect_session(mvm, vif, duration, min_duration, 500, false);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
index d2cf751db68d..6a4ba160c59e 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -35,6 +36,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -90,7 +92,9 @@
 #include "fw/runtime.h"
 #include "fw/dbg.h"
 #include "fw/acpi.h"
-#include "fw/debugfs.h"
+#include "iwl-nvm-parse.h"
+
+#include <linux/average.h>
 
 #define IWL_MVM_MAX_ADDRESSES		5
 /* RSSI offset for WkP */
@@ -444,6 +448,8 @@ struct iwl_mvm_vif {
 	/* FW identified misbehaving AP */
 	u8 uapsd_misbehaving_bssid[ETH_ALEN];
 
+	struct delayed_work uapsd_nonagg_detected_wk;
+
 	/* Indicates that CSA countdown may be started */
 	bool csa_countdown;
 	bool csa_failed;
@@ -502,18 +508,6 @@ enum iwl_mvm_sched_scan_pass_all_states {
 	SCHED_SCAN_PASS_ALL_FOUND,
 };
 
-/**
- * struct iwl_nvm_section - describes an NVM section in memory.
- *
- * This struct holds an NVM section read from the NIC using NVM_ACCESS_CMD,
- * and saved for later use by the driver. Not all NVM sections are saved
- * this way, only the needed ones.
- */
-struct iwl_nvm_section {
-	u16 length;
-	const u8 *data;
-};
-
 /**
  * struct iwl_mvm_tt_mgnt - Thermal Throttling Management structure
  * @ct_kill_exit: worker to exit thermal kill
@@ -595,6 +589,53 @@ enum iwl_mvm_tdls_cs_state {
 	IWL_MVM_TDLS_SW_ACTIVE,
 };
 
+enum iwl_mvm_traffic_load {
+	IWL_MVM_TRAFFIC_LOW,
+	IWL_MVM_TRAFFIC_MEDIUM,
+	IWL_MVM_TRAFFIC_HIGH,
+};
+
+DECLARE_EWMA(rate, 16, 16)
+
+struct iwl_mvm_tcm_mac {
+	struct {
+		u32 pkts[IEEE80211_NUM_ACS];
+		u32 airtime;
+	} tx;
+	struct {
+		u32 pkts[IEEE80211_NUM_ACS];
+		u32 airtime;
+		u32 last_ampdu_ref;
+	} rx;
+	struct {
+		/* track AP's transfer in client mode */
+		u64 rx_bytes;
+		struct ewma_rate rate;
+		bool detected;
+	} uapsd_nonagg_detect;
+	bool opened_rx_ba_sessions;
+};
+
+struct iwl_mvm_tcm {
+	struct delayed_work work;
+	spinlock_t lock; /* used when time elapsed */
+	unsigned long ts; /* timestamp when period ends */
+	unsigned long ll_ts;
+	unsigned long uapsd_nonagg_ts;
+	bool paused;
+	struct iwl_mvm_tcm_mac data[NUM_MAC_INDEX_DRIVER];
+	struct {
+		u32 elapsed; /* milliseconds for this TCM period */
+		u32 airtime[NUM_MAC_INDEX_DRIVER];
+		enum iwl_mvm_traffic_load load[NUM_MAC_INDEX_DRIVER];
+		enum iwl_mvm_traffic_load band_load[NUM_NL80211_BANDS];
+		enum iwl_mvm_traffic_load global_load;
+		bool low_latency[NUM_MAC_INDEX_DRIVER];
+		bool change[NUM_MAC_INDEX_DRIVER];
+		bool global_change;
+	} result;
+};
+
 /**
  * struct iwl_mvm_reorder_buffer - per ra/tid/queue reorder buffer
  * @head_sn: reorder window head sn
@@ -829,7 +870,10 @@ struct iwl_mvm {
 	unsigned int scan_status;
 	void *scan_cmd;
 	struct iwl_mcast_filter_cmd *mcast_filter_cmd;
+	/* For CDB this is low band scan type, for non-CDB - type. */
 	enum iwl_mvm_scan_type scan_type;
+	enum iwl_mvm_scan_type hb_scan_type;
+
 	enum iwl_mvm_sched_scan_pass_all_states sched_scan_pass_all;
 	struct delayed_work scan_timeout_dwork;
 
@@ -978,6 +1022,13 @@ struct iwl_mvm {
 	 */
 	bool temperature_test;  /* Debug test temperature is enabled */
 
+	unsigned long bt_coex_last_tcm_ts;
+	struct iwl_mvm_tcm tcm;
+
+	u8 uapsd_noagg_bssid_write_idx;
+	struct mac_address uapsd_noagg_bssids[IWL_MVM_UAPSD_NOAGG_BSSIDS_NUM]
+		__aligned(2);
+
 	struct iwl_time_quota_cmd last_quota_cmd;
 
 #ifdef CONFIG_NL80211_TESTMODE
@@ -1293,6 +1344,16 @@ static inline bool iwl_mvm_is_cdb_supported(struct iwl_mvm *mvm)
 			   IWL_UCODE_TLV_CAPA_CDB_SUPPORT);
 }
 
+static inline bool iwl_mvm_cdb_scan_api(struct iwl_mvm *mvm)
+{
+	/*
+	 * TODO: should this be the same as iwl_mvm_is_cdb_supported()?
+	 * but then there's a little bit of code in scan that won't make
+	 * any sense...
+	 */
+	return mvm->trans->cfg->device_family >= IWL_DEVICE_FAMILY_22000;
+}
+
 static inline bool iwl_mvm_has_new_rx_stats_api(struct iwl_mvm *mvm)
 {
 	return fw_has_api(&mvm->fw->ucode_capa,
@@ -1438,7 +1499,6 @@ void iwl_mvm_accu_radio_stats(struct iwl_mvm *mvm);
 /* NVM */
 int iwl_nvm_init(struct iwl_mvm *mvm);
 int iwl_mvm_load_nvm_to_nic(struct iwl_mvm *mvm);
-int iwl_mvm_read_external_nvm(struct iwl_mvm *mvm);
 
 static inline u8 iwl_mvm_get_valid_tx_ant(struct iwl_mvm *mvm)
 {
@@ -1771,6 +1831,8 @@ int iwl_mvm_update_low_latency(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			      enum iwl_mvm_low_latency_cause cause);
 /* get SystemLowLatencyMode - only needed for beacon threshold? */
 bool iwl_mvm_low_latency(struct iwl_mvm *mvm);
+bool iwl_mvm_low_latency_band(struct iwl_mvm *mvm, enum nl80211_band band);
+
 /* get VMACLowLatencyMode */
 static inline bool iwl_mvm_vif_low_latency(struct iwl_mvm_vif *mvmvif)
 {
@@ -1906,6 +1968,17 @@ bool iwl_mvm_is_vif_assoc(struct iwl_mvm *mvm);
 
 void iwl_mvm_inactivity_check(struct iwl_mvm *mvm);
 
+#define MVM_TCM_PERIOD_MSEC 500
+#define MVM_TCM_PERIOD (HZ * MVM_TCM_PERIOD_MSEC / 1000)
+#define MVM_LL_PERIOD (10 * HZ)
+void iwl_mvm_tcm_work(struct work_struct *work);
+void iwl_mvm_recalc_tcm(struct iwl_mvm *mvm);
+void iwl_mvm_pause_tcm(struct iwl_mvm *mvm, bool with_cancel);
+void iwl_mvm_resume_tcm(struct iwl_mvm *mvm);
+void iwl_mvm_tcm_add_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
+void iwl_mvm_tcm_rm_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
+u8 iwl_mvm_tcm_load_percentage(u32 airtime, u32 elapsed);
+
 void iwl_mvm_nic_restart(struct iwl_mvm *mvm, bool fw_error);
 unsigned int iwl_mvm_get_wd_timeout(struct iwl_mvm *mvm,
 				    struct ieee80211_vif *vif,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c b/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
index 5bfe5306524c..cf48517944ec 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -35,6 +36,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -76,9 +78,7 @@
 #include "fw/acpi.h"
 
 /* Default NVM size to read */
-#define IWL_NVM_DEFAULT_CHUNK_SIZE (2*1024)
-#define IWL_MAX_NVM_SECTION_SIZE	0x1b58
-#define IWL_MAX_EXT_NVM_SECTION_SIZE	0x1ffc
+#define IWL_NVM_DEFAULT_CHUNK_SIZE (2 * 1024)
 
 #define NVM_WRITE_OPCODE 1
 #define NVM_READ_OPCODE 0
@@ -229,19 +229,6 @@ static int iwl_nvm_write_section(struct iwl_mvm *mvm, u16 section,
 	return 0;
 }
 
-static void iwl_mvm_nvm_fixups(struct iwl_mvm *mvm, unsigned int section,
-			       u8 *data, unsigned int len)
-{
-#define IWL_4165_DEVICE_ID	0x5501
-#define NVM_SKU_CAP_MIMO_DISABLE BIT(5)
-
-	if (section == NVM_SECTION_TYPE_PHY_SKU &&
-	    mvm->trans->hw_id == IWL_4165_DEVICE_ID && data && len >= 5 &&
-	    (data[4] & NVM_SKU_CAP_MIMO_DISABLE))
-		/* OTP 0x52 bug work around: it's a 1x1 device */
-		data[3] = ANT_B | (ANT_B << 4);
-}
-
 /*
  * Reads an NVM section completely.
  * NICs prior to 7000 family doesn't have a real NVM, but just read
@@ -282,7 +269,7 @@ static int iwl_nvm_read_section(struct iwl_mvm *mvm, u16 section,
 		offset += ret;
 	}
 
-	iwl_mvm_nvm_fixups(mvm, section, data, offset);
+	iwl_nvm_fixups(mvm->trans->hw_id, section, data, offset);
 
 	IWL_DEBUG_EEPROM(mvm->trans->dev,
 			 "NVM section %d read completed\n", section);
@@ -355,184 +342,6 @@ iwl_parse_nvm_sections(struct iwl_mvm *mvm)
 				  lar_enabled);
 }
 
-#define MAX_NVM_FILE_LEN	16384
-
-/*
- * Reads external NVM from a file into mvm->nvm_sections
- *
- * HOW TO CREATE THE NVM FILE FORMAT:
- * ------------------------------
- * 1. create hex file, format:
- *      3800 -> header
- *      0000 -> header
- *      5a40 -> data
- *
- *   rev - 6 bit (word1)
- *   len - 10 bit (word1)
- *   id - 4 bit (word2)
- *   rsv - 12 bit (word2)
- *
- * 2. flip 8bits with 8 bits per line to get the right NVM file format
- *
- * 3. create binary file from the hex file
- *
- * 4. save as "iNVM_xxx.bin" under /lib/firmware
- */
-int iwl_mvm_read_external_nvm(struct iwl_mvm *mvm)
-{
-	int ret, section_size;
-	u16 section_id;
-	const struct firmware *fw_entry;
-	const struct {
-		__le16 word1;
-		__le16 word2;
-		u8 data[];
-	} *file_sec;
-	const u8 *eof;
-	u8 *temp;
-	int max_section_size;
-	const __le32 *dword_buff;
-
-#define NVM_WORD1_LEN(x) (8 * (x & 0x03FF))
-#define NVM_WORD2_ID(x) (x >> 12)
-#define EXT_NVM_WORD2_LEN(x) (2 * (((x) & 0xFF) << 8 | (x) >> 8))
-#define EXT_NVM_WORD1_ID(x) ((x) >> 4)
-#define NVM_HEADER_0	(0x2A504C54)
-#define NVM_HEADER_1	(0x4E564D2A)
-#define NVM_HEADER_SIZE	(4 * sizeof(u32))
-
-	IWL_DEBUG_EEPROM(mvm->trans->dev, "Read from external NVM\n");
-
-	/* Maximal size depends on NVM version */
-	if (mvm->trans->cfg->nvm_type != IWL_NVM_EXT)
-		max_section_size = IWL_MAX_NVM_SECTION_SIZE;
-	else
-		max_section_size = IWL_MAX_EXT_NVM_SECTION_SIZE;
-
-	/*
-	 * Obtain NVM image via request_firmware. Since we already used
-	 * request_firmware_nowait() for the firmware binary load and only
-	 * get here after that we assume the NVM request can be satisfied
-	 * synchronously.
-	 */
-	ret = request_firmware(&fw_entry, mvm->nvm_file_name,
-			       mvm->trans->dev);
-	if (ret) {
-		IWL_ERR(mvm, "ERROR: %s isn't available %d\n",
-			mvm->nvm_file_name, ret);
-		return ret;
-	}
-
-	IWL_INFO(mvm, "Loaded NVM file %s (%zu bytes)\n",
-		 mvm->nvm_file_name, fw_entry->size);
-
-	if (fw_entry->size > MAX_NVM_FILE_LEN) {
-		IWL_ERR(mvm, "NVM file too large\n");
-		ret = -EINVAL;
-		goto out;
-	}
-
-	eof = fw_entry->data + fw_entry->size;
-	dword_buff = (__le32 *)fw_entry->data;
-
-	/* some NVM file will contain a header.
-	 * The header is identified by 2 dwords header as follow:
-	 * dword[0] = 0x2A504C54
-	 * dword[1] = 0x4E564D2A
-	 *
-	 * This header must be skipped when providing the NVM data to the FW.
-	 */
-	if (fw_entry->size > NVM_HEADER_SIZE &&
-	    dword_buff[0] == cpu_to_le32(NVM_HEADER_0) &&
-	    dword_buff[1] == cpu_to_le32(NVM_HEADER_1)) {
-		file_sec = (void *)(fw_entry->data + NVM_HEADER_SIZE);
-		IWL_INFO(mvm, "NVM Version %08X\n", le32_to_cpu(dword_buff[2]));
-		IWL_INFO(mvm, "NVM Manufacturing date %08X\n",
-			 le32_to_cpu(dword_buff[3]));
-
-		/* nvm file validation, dword_buff[2] holds the file version */
-		if (mvm->trans->cfg->device_family == IWL_DEVICE_FAMILY_8000 &&
-		    CSR_HW_REV_STEP(mvm->trans->hw_rev) == SILICON_C_STEP &&
-		    le32_to_cpu(dword_buff[2]) < 0xE4A) {
-			ret = -EFAULT;
-			goto out;
-		}
-	} else {
-		file_sec = (void *)fw_entry->data;
-	}
-
-	while (true) {
-		if (file_sec->data > eof) {
-			IWL_ERR(mvm,
-				"ERROR - NVM file too short for section header\n");
-			ret = -EINVAL;
-			break;
-		}
-
-		/* check for EOF marker */
-		if (!file_sec->word1 && !file_sec->word2) {
-			ret = 0;
-			break;
-		}
-
-		if (mvm->trans->cfg->nvm_type != IWL_NVM_EXT) {
-			section_size =
-				2 * NVM_WORD1_LEN(le16_to_cpu(file_sec->word1));
-			section_id = NVM_WORD2_ID(le16_to_cpu(file_sec->word2));
-		} else {
-			section_size = 2 * EXT_NVM_WORD2_LEN(
-						le16_to_cpu(file_sec->word2));
-			section_id = EXT_NVM_WORD1_ID(
-						le16_to_cpu(file_sec->word1));
-		}
-
-		if (section_size > max_section_size) {
-			IWL_ERR(mvm, "ERROR - section too large (%d)\n",
-				section_size);
-			ret = -EINVAL;
-			break;
-		}
-
-		if (!section_size) {
-			IWL_ERR(mvm, "ERROR - section empty\n");
-			ret = -EINVAL;
-			break;
-		}
-
-		if (file_sec->data + section_size > eof) {
-			IWL_ERR(mvm,
-				"ERROR - NVM file too short for section (%d bytes)\n",
-				section_size);
-			ret = -EINVAL;
-			break;
-		}
-
-		if (WARN(section_id >= NVM_MAX_NUM_SECTIONS,
-			 "Invalid NVM section ID %d\n", section_id)) {
-			ret = -EINVAL;
-			break;
-		}
-
-		temp = kmemdup(file_sec->data, section_size, GFP_KERNEL);
-		if (!temp) {
-			ret = -ENOMEM;
-			break;
-		}
-
-		iwl_mvm_nvm_fixups(mvm, section_id, temp, section_size);
-
-		kfree(mvm->nvm_sections[section_id].data);
-		mvm->nvm_sections[section_id].data = temp;
-		mvm->nvm_sections[section_id].length = section_size;
-
-		/* advance to the next section */
-		file_sec = (void *)(file_sec->data + section_size);
-	}
-out:
-	release_firmware(fw_entry);
-	return ret;
-}
-
 /* Loads the NVM data stored in mvm->nvm_sections into the NIC */
 int iwl_mvm_load_nvm_to_nic(struct iwl_mvm *mvm)
 {
@@ -585,7 +394,7 @@ int iwl_nvm_init(struct iwl_mvm *mvm)
 			break;
 		}
 
-		iwl_mvm_nvm_fixups(mvm, section, temp, ret);
+		iwl_nvm_fixups(mvm->trans->hw_id, section, temp, ret);
 
 		mvm->nvm_sections[section].data = temp;
 		mvm->nvm_sections[section].length = ret;
@@ -624,14 +433,17 @@ int iwl_nvm_init(struct iwl_mvm *mvm)
 	/* Only if PNVM selected in the mod param - load external NVM  */
 	if (mvm->nvm_file_name) {
 		/* read External NVM file from the mod param */
-		ret = iwl_mvm_read_external_nvm(mvm);
+		ret = iwl_read_external_nvm(mvm->trans, mvm->nvm_file_name,
+					    mvm->nvm_sections);
 		if (ret) {
 			mvm->nvm_file_name = nvm_file_C;
 
 			if ((ret == -EFAULT || ret == -ENOENT) &&
 			    mvm->nvm_file_name) {
 				/* in case nvm file was failed try again */
-				ret = iwl_mvm_read_external_nvm(mvm);
+				ret = iwl_read_external_nvm(mvm->trans,
+							    mvm->nvm_file_name,
+							    mvm->nvm_sections);
 				if (ret)
 					return ret;
 			} else {
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
index 224bfa1bcf53..ff1e518096c5 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
@@ -250,6 +250,9 @@ static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 	RX_HANDLER(TX_CMD, iwl_mvm_rx_tx_cmd, RX_HANDLER_SYNC),
 	RX_HANDLER(BA_NOTIF, iwl_mvm_rx_ba_notif, RX_HANDLER_SYNC),
 
+	RX_HANDLER_GRP(DATA_PATH_GROUP, TLC_MNG_UPDATE_NOTIF,
+		       iwl_mvm_tlc_update_notif, RX_HANDLER_SYNC),
+
 	RX_HANDLER(BT_PROFILE_NOTIFICATION, iwl_mvm_rx_bt_coex_notif,
 		   RX_HANDLER_ASYNC_LOCKED),
 	RX_HANDLER(BEACON_NOTIFICATION, iwl_mvm_rx_beacon_notif,
@@ -667,6 +670,12 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 
 	SET_IEEE80211_DEV(mvm->hw, mvm->trans->dev);
 
+	spin_lock_init(&mvm->tcm.lock);
+	INIT_DELAYED_WORK(&mvm->tcm.work, iwl_mvm_tcm_work);
+	mvm->tcm.ts = jiffies;
+	mvm->tcm.ll_ts = jiffies;
+	mvm->tcm.uapsd_nonagg_ts = jiffies;
+
 	INIT_DELAYED_WORK(&mvm->cs_tx_unblock_dwork, iwl_mvm_tx_unblock_dwork);
 
 	/*
@@ -730,6 +739,9 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	       sizeof(trans->dbg_conf_tlv));
 	trans->dbg_trigger_tlv = mvm->fw->dbg_trigger_tlv;
 
+	trans->iml = mvm->fw->iml;
+	trans->iml_len = mvm->fw->iml_len;
+
 	/* set up notification wait support */
 	iwl_notification_wait_init(&mvm->notif_wait);
 
@@ -859,6 +871,8 @@ static void iwl_op_mode_mvm_stop(struct iwl_op_mode *op_mode)
 	for (i = 0; i < NVM_MAX_NUM_SECTIONS; i++)
 		kfree(mvm->nvm_sections[i].data);
 
+	cancel_delayed_work_sync(&mvm->tcm.work);
+
 	iwl_mvm_tof_clean(mvm);
 
 	mutex_destroy(&mvm->mutex);
@@ -1026,8 +1040,6 @@ static void iwl_mvm_rx_mq(struct iwl_op_mode *op_mode,
 		iwl_mvm_rx_queue_notif(mvm, rxb, 0);
 	else if (cmd == WIDE_ID(LEGACY_GROUP, FRAME_RELEASE))
 		iwl_mvm_rx_frame_release(mvm, napi, rxb, 0);
-	else if (cmd == WIDE_ID(DATA_PATH_GROUP, TLC_MNG_UPDATE_NOTIF))
-		iwl_mvm_tlc_update_notif(mvm, pkt);
 	else
 		iwl_mvm_rx_common(mvm, rxb, pkt);
 }
@@ -1432,6 +1444,7 @@ int iwl_mvm_enter_d0i3(struct iwl_op_mode *op_mode)
 		mvm->d0i3_offloading = false;
 	}
 
+	iwl_mvm_pause_tcm(mvm, true);
 	/* make sure we have no running tx while configuring the seqno */
 	synchronize_net();
 
@@ -1615,6 +1628,7 @@ static void iwl_mvm_d0i3_exit_work(struct work_struct *wk)
 	/* the FW might have updated the regdomain */
 	iwl_mvm_update_changed_regdom(mvm);
 
+	iwl_mvm_resume_tcm(mvm);
 	iwl_mvm_unref(mvm, IWL_MVM_REF_EXIT_WORK);
 	mutex_unlock(&mvm->mutex);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
index fb5745660509..b8b2b819e8e7 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
@@ -6,6 +6,7 @@
  * GPL LICENSE SUMMARY
  *
  * Copyright(c) 2017        Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -26,6 +27,7 @@
  * BSD LICENSE
  *
  * Copyright(c) 2017        Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -65,14 +67,14 @@ static u8 rs_fw_bw_from_sta_bw(struct ieee80211_sta *sta)
 {
 	switch (sta->bandwidth) {
 	case IEEE80211_STA_RX_BW_160:
-		return IWL_TLC_MNG_MAX_CH_WIDTH_160MHZ;
+		return IWL_TLC_MNG_CH_WIDTH_160MHZ;
 	case IEEE80211_STA_RX_BW_80:
-		return IWL_TLC_MNG_MAX_CH_WIDTH_80MHZ;
+		return IWL_TLC_MNG_CH_WIDTH_80MHZ;
 	case IEEE80211_STA_RX_BW_40:
-		return IWL_TLC_MNG_MAX_CH_WIDTH_40MHZ;
+		return IWL_TLC_MNG_CH_WIDTH_40MHZ;
 	case IEEE80211_STA_RX_BW_20:
 	default:
-		return IWL_TLC_MNG_MAX_CH_WIDTH_20MHZ;
+		return IWL_TLC_MNG_CH_WIDTH_20MHZ;
 	}
 }
 
@@ -85,7 +87,9 @@ static u8 rs_fw_set_active_chains(u8 chains)
 	if (chains & ANT_B)
 		fw_chains |= IWL_TLC_MNG_CHAIN_B_MSK;
 	if (chains & ANT_C)
-		fw_chains |= IWL_TLC_MNG_CHAIN_C_MSK;
+		WARN(false,
+		     "tlc offload doesn't support antenna C. chains: 0x%x\n",
+		     chains);
 
 	return fw_chains;
 }
@@ -97,13 +101,13 @@ static u8 rs_fw_sgi_cw_support(struct ieee80211_sta *sta)
 	u8 supp = 0;
 
 	if (ht_cap->cap & IEEE80211_HT_CAP_SGI_20)
-		supp |= IWL_TLC_MNG_SGI_20MHZ_MSK;
+		supp |= BIT(IWL_TLC_MNG_CH_WIDTH_20MHZ);
 	if (ht_cap->cap & IEEE80211_HT_CAP_SGI_40)
-		supp |= IWL_TLC_MNG_SGI_40MHZ_MSK;
+		supp |= BIT(IWL_TLC_MNG_CH_WIDTH_40MHZ);
 	if (vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_80)
-		supp |= IWL_TLC_MNG_SGI_80MHZ_MSK;
+		supp |= BIT(IWL_TLC_MNG_CH_WIDTH_80MHZ);
 	if (vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_160)
-		supp |= IWL_TLC_MNG_SGI_160MHZ_MSK;
+		supp |= BIT(IWL_TLC_MNG_CH_WIDTH_160MHZ);
 
 	return supp;
 }
@@ -114,9 +118,7 @@ static u16 rs_fw_set_config_flags(struct iwl_mvm *mvm,
 	struct ieee80211_sta_ht_cap *ht_cap = &sta->ht_cap;
 	struct ieee80211_sta_vht_cap *vht_cap = &sta->vht_cap;
 	bool vht_ena = vht_cap && vht_cap->vht_supported;
-	u16 flags = IWL_TLC_MNG_CFG_FLAGS_CCK_MSK |
-		    IWL_TLC_MNG_CFG_FLAGS_DCM_MSK |
-		    IWL_TLC_MNG_CFG_FLAGS_DD_MSK;
+	u16 flags = 0;
 
 	if (mvm->cfg->ht_params->stbc &&
 	    (num_of_ant(iwl_mvm_get_valid_tx_ant(mvm)) > 1) &&
@@ -129,16 +131,11 @@ static u16 rs_fw_set_config_flags(struct iwl_mvm *mvm,
 	     (vht_ena && (vht_cap->cap & IEEE80211_VHT_CAP_RXLDPC))))
 		flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
 
-	if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_BEAMFORMER) &&
-	    (num_of_ant(iwl_mvm_get_valid_tx_ant(mvm)) > 1) &&
-	    (vht_cap->cap & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE))
-		flags |= IWL_TLC_MNG_CFG_FLAGS_BF_MSK;
-
 	return flags;
 }
 
 static
-int rs_fw_vht_highest_rx_mcs_index(struct ieee80211_sta_vht_cap *vht_cap,
+int rs_fw_vht_highest_rx_mcs_index(const struct ieee80211_sta_vht_cap *vht_cap,
 				   int nss)
 {
 	u16 rx_mcs = le16_to_cpu(vht_cap->vht_mcs.rx_mcs_map) &
@@ -160,15 +157,16 @@ int rs_fw_vht_highest_rx_mcs_index(struct ieee80211_sta_vht_cap *vht_cap,
 	return 0;
 }
 
-static void rs_fw_vht_set_enabled_rates(struct ieee80211_sta *sta,
-					struct ieee80211_sta_vht_cap *vht_cap,
-					struct iwl_tlc_config_cmd *cmd)
+static void
+rs_fw_vht_set_enabled_rates(const struct ieee80211_sta *sta,
+			    const struct ieee80211_sta_vht_cap *vht_cap,
+			    struct iwl_tlc_config_cmd *cmd)
 {
 	u16 supp;
 	int i, highest_mcs;
 
 	for (i = 0; i < sta->rx_nss; i++) {
-		if (i == MAX_RS_ANT_NUM)
+		if (i == MAX_NSS)
 			break;
 
 		highest_mcs = rs_fw_vht_highest_rx_mcs_index(vht_cap, i + 1);
@@ -179,7 +177,9 @@ static void rs_fw_vht_set_enabled_rates(struct ieee80211_sta *sta,
 		if (sta->bandwidth == IEEE80211_STA_RX_BW_20)
 			supp &= ~BIT(IWL_TLC_MNG_HT_RATE_MCS9);
 
-		cmd->ht_supp_rates[i] = cpu_to_le16(supp);
+		cmd->ht_rates[i][0] = cpu_to_le16(supp);
+		if (sta->bandwidth == IEEE80211_STA_RX_BW_160)
+			cmd->ht_rates[i][1] = cmd->ht_rates[i][0];
 	}
 }
 
@@ -190,8 +190,8 @@ static void rs_fw_set_supp_rates(struct ieee80211_sta *sta,
 	int i;
 	unsigned long tmp;
 	unsigned long supp; /* must be unsigned long for for_each_set_bit */
-	struct ieee80211_sta_ht_cap *ht_cap = &sta->ht_cap;
-	struct ieee80211_sta_vht_cap *vht_cap = &sta->vht_cap;
+	const struct ieee80211_sta_ht_cap *ht_cap = &sta->ht_cap;
+	const struct ieee80211_sta_vht_cap *vht_cap = &sta->vht_cap;
 
 	/* non HT rates */
 	supp = 0;
@@ -199,45 +199,40 @@ static void rs_fw_set_supp_rates(struct ieee80211_sta *sta,
 	for_each_set_bit(i, &tmp, BITS_PER_LONG)
 		supp |= BIT(sband->bitrates[i].hw_value);
 
-	cmd->non_ht_supp_rates = cpu_to_le16(supp);
+	cmd->non_ht_rates = cpu_to_le16(supp);
 	cmd->mode = IWL_TLC_MNG_MODE_NON_HT;
 
-	/* HT/VHT rates */
 	if (vht_cap && vht_cap->vht_supported) {
 		cmd->mode = IWL_TLC_MNG_MODE_VHT;
 		rs_fw_vht_set_enabled_rates(sta, vht_cap, cmd);
 	} else if (ht_cap && ht_cap->ht_supported) {
 		cmd->mode = IWL_TLC_MNG_MODE_HT;
-		cmd->ht_supp_rates[0] = cpu_to_le16(ht_cap->mcs.rx_mask[0]);
-		cmd->ht_supp_rates[1] = cpu_to_le16(ht_cap->mcs.rx_mask[1]);
+		cmd->ht_rates[0][0] = cpu_to_le16(ht_cap->mcs.rx_mask[0]);
+		cmd->ht_rates[1][0] = cpu_to_le16(ht_cap->mcs.rx_mask[1]);
 	}
 }
 
-static void rs_fw_tlc_mng_notif_req_config(struct iwl_mvm *mvm, u8 sta_id)
-{
-	u32 cmd_id = iwl_cmd_id(TLC_MNG_NOTIF_REQ_CMD, DATA_PATH_GROUP, 0);
-	struct iwl_tlc_notif_req_config_cmd cfg_cmd = {
-		.sta_id = sta_id,
-		.flags = cpu_to_le16(IWL_TLC_NOTIF_INIT_RATE_MSK),
-		.interval = cpu_to_le16(IWL_TLC_NOTIF_REQ_INTERVAL),
-	};
-	int ret;
-
-	ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, sizeof(cfg_cmd), &cfg_cmd);
-	if (ret)
-		IWL_ERR(mvm, "Failed to send TLC notif request (%d)\n", ret);
-}
-
-void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm, struct iwl_rx_packet *pkt)
+void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
+			      struct iwl_rx_cmd_buffer *rxb)
 {
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_tlc_update_notif *notif;
+	struct ieee80211_sta *sta;
 	struct iwl_mvm_sta *mvmsta;
 	struct iwl_lq_sta_rs_fw *lq_sta;
+	u32 flags;
 
 	rcu_read_lock();
 
 	notif = (void *)pkt->data;
-	mvmsta = iwl_mvm_sta_from_staid_rcu(mvm, notif->sta_id);
+	sta = rcu_dereference(mvm->fw_id_to_mac_id[notif->sta_id]);
+	if (IS_ERR_OR_NULL(sta)) {
+		IWL_ERR(mvm, "Invalid sta id (%d) in FW TLC notification\n",
+			notif->sta_id);
+		goto out;
+	}
+
+	mvmsta = iwl_mvm_sta_from_mac80211(sta);
 
 	if (!mvmsta) {
 		IWL_ERR(mvm, "Invalid sta id (%d) in FW TLC notification\n",
@@ -245,14 +240,30 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm, struct iwl_rx_packet *pkt)
 		goto out;
 	}
 
+	flags = le32_to_cpu(notif->flags);
+
 	lq_sta = &mvmsta->lq_sta.rs_fw;
 
-	if (le16_to_cpu(notif->flags) & IWL_TLC_NOTIF_INIT_RATE_MSK) {
-		lq_sta->last_rate_n_flags =
-			le32_to_cpu(notif->values[IWL_TLC_NOTIF_INIT_RATE_POS]);
+	if (flags & IWL_TLC_NOTIF_FLAG_RATE) {
+		lq_sta->last_rate_n_flags = le32_to_cpu(notif->rate);
 		IWL_DEBUG_RATE(mvm, "new rate_n_flags: 0x%X\n",
 			       lq_sta->last_rate_n_flags);
 	}
+
+	if (flags & IWL_TLC_NOTIF_FLAG_AMSDU) {
+		u16 size = le32_to_cpu(notif->amsdu_size);
+
+		if (WARN_ON(sta->max_amsdu_len < size))
+			goto out;
+
+		mvmsta->amsdu_enabled = le32_to_cpu(notif->amsdu_enabled);
+		mvmsta->max_amsdu_len = size;
+
+		IWL_DEBUG_RATE(mvm,
+			       "AMSDU update. AMSDU size: %d, AMSDU selected size: %d, AMSDU TID bitmap 0x%X\n",
+			       le32_to_cpu(notif->amsdu_size), size,
+			       mvmsta->amsdu_enabled);
+	}
 out:
 	rcu_read_unlock();
 }
@@ -267,12 +278,12 @@ void rs_fw_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	struct ieee80211_supported_band *sband;
 	struct iwl_tlc_config_cmd cfg_cmd = {
 		.sta_id = mvmsta->sta_id,
-		.max_supp_ch_width = rs_fw_bw_from_sta_bw(sta),
+		.max_ch_width = rs_fw_bw_from_sta_bw(sta),
 		.flags = cpu_to_le16(rs_fw_set_config_flags(mvm, sta)),
 		.chains = rs_fw_set_active_chains(iwl_mvm_get_valid_tx_ant(mvm)),
-		.max_supp_ss = sta->rx_nss,
-		.max_ampdu_cnt = cpu_to_le32(mvmsta->max_agg_bufsize),
+		.max_mpdu_len = cpu_to_le16(sta->max_amsdu_len),
 		.sgi_ch_width_supp = rs_fw_sgi_cw_support(sta),
+		.amsdu = iwl_mvm_is_csum_supported(mvm),
 	};
 	int ret;
 
@@ -287,8 +298,6 @@ void rs_fw_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, sizeof(cfg_cmd), &cfg_cmd);
 	if (ret)
 		IWL_ERR(mvm, "Failed to send rate scale config (%d)\n", ret);
-
-	rs_fw_tlc_mng_notif_req_config(mvm, cfg_cmd.sta_id);
 }
 
 int rs_fw_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
index 5d776ec1840f..642da10b0b7f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
@@ -3,6 +3,7 @@
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of version 2 of the GNU General Public License as
@@ -13,10 +14,6 @@
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110, USA
- *
  * The full GNU General Public License is included in this distribution in the
  * file called LICENSE.
  *
@@ -651,9 +648,10 @@ static void rs_tl_turn_on_agg(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
 	}
 
 	tid_data = &mvmsta->tid_data[tid];
-	if ((tid_data->state == IWL_AGG_OFF) &&
+	if (mvmsta->sta_state >= IEEE80211_STA_AUTHORIZED &&
+	    tid_data->state == IWL_AGG_OFF &&
 	    (lq_sta->tx_agg_tid_en & BIT(tid)) &&
-	    (tid_data->tx_count_last >= IWL_MVM_RS_AGG_START_THRESHOLD)) {
+	    tid_data->tx_count_last >= IWL_MVM_RS_AGG_START_THRESHOLD) {
 		IWL_DEBUG_RATE(mvm, "try to aggregate tid %d\n", tid);
 		if (rs_tl_turn_on_agg_for_tid(mvm, lq_sta, tid, sta) == 0)
 			tid_data->state = IWL_AGG_QUEUED;
@@ -1257,7 +1255,7 @@ void iwl_mvm_rs_tx_status(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		       (unsigned long)(lq_sta->last_tx +
 				       (IWL_MVM_RS_IDLE_TIMEOUT * HZ)))) {
 		IWL_DEBUG_RATE(mvm, "Tx idle for too long. reinit rs\n");
-		iwl_mvm_rs_rate_init(mvm, sta, info->band, false);
+		iwl_mvm_rs_rate_init(mvm, sta, info->band);
 		return;
 	}
 	lq_sta->last_tx = jiffies;
@@ -1715,12 +1713,18 @@ static void rs_set_amsdu_len(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 {
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 
+	/*
+	 * In case TLC offload is not active amsdu_enabled is either 0xFFFF
+	 * or 0, since there is no per-TID alg.
+	 */
 	if ((!is_vht(&tbl->rate) && !is_ht(&tbl->rate)) ||
 	    tbl->rate.index < IWL_RATE_MCS_5_INDEX ||
 	    scale_action == RS_ACTION_DOWNSCALE)
-		mvmsta->tlc_amsdu = false;
+		mvmsta->amsdu_enabled = 0;
 	else
-		mvmsta->tlc_amsdu = true;
+		mvmsta->amsdu_enabled = 0xFFFF;
+
+	mvmsta->max_amsdu_len = sta->max_amsdu_len;
 }
 
 /*
@@ -2684,9 +2688,9 @@ static void rs_get_initial_rate(struct iwl_mvm *mvm,
 				struct ieee80211_sta *sta,
 				struct iwl_lq_sta *lq_sta,
 				enum nl80211_band band,
-				struct rs_rate *rate,
-				bool init)
+				struct rs_rate *rate)
 {
+	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	int i, nentries;
 	unsigned long active_rate;
 	s8 best_rssi = S8_MIN;
@@ -2748,7 +2752,8 @@ static void rs_get_initial_rate(struct iwl_mvm *mvm,
 		 * bandwidth rate, and after authorization, when the phy context
 		 * is already up-to-date, re-init rs with the correct bw.
 		 */
-		u32 bw = init ? RATE_MCS_CHAN_WIDTH_20 : rs_bw_from_sta_bw(sta);
+		u32 bw = mvmsta->sta_state < IEEE80211_STA_AUTHORIZED ?
+				RATE_MCS_CHAN_WIDTH_20 : rs_bw_from_sta_bw(sta);
 
 		switch (bw) {
 		case RATE_MCS_CHAN_WIDTH_40:
@@ -2833,9 +2838,9 @@ void rs_update_last_rssi(struct iwl_mvm *mvm,
 static void rs_initialize_lq(struct iwl_mvm *mvm,
 			     struct ieee80211_sta *sta,
 			     struct iwl_lq_sta *lq_sta,
-			     enum nl80211_band band,
-			     bool init)
+			     enum nl80211_band band)
 {
+	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_scale_tbl_info *tbl;
 	struct rs_rate *rate;
 	u8 active_tbl = 0;
@@ -2851,7 +2856,7 @@ static void rs_initialize_lq(struct iwl_mvm *mvm,
 	tbl = &(lq_sta->lq_info[active_tbl]);
 	rate = &tbl->rate;
 
-	rs_get_initial_rate(mvm, sta, lq_sta, band, rate, init);
+	rs_get_initial_rate(mvm, sta, lq_sta, band, rate);
 	rs_init_optimal_rate(mvm, sta, lq_sta);
 
 	WARN_ONCE(rate->ant != ANT_A && rate->ant != ANT_B,
@@ -2864,7 +2869,8 @@ static void rs_initialize_lq(struct iwl_mvm *mvm,
 	rs_set_expected_tpt_table(lq_sta, tbl);
 	rs_fill_lq_cmd(mvm, sta, lq_sta, rate);
 	/* TODO restore station should remember the lq cmd */
-	iwl_mvm_send_lq_cmd(mvm, &lq_sta->lq, init);
+	iwl_mvm_send_lq_cmd(mvm, &lq_sta->lq,
+			    mvmsta->sta_state < IEEE80211_STA_AUTHORIZED);
 }
 
 static void rs_drv_get_rate(void *mvm_r, struct ieee80211_sta *sta,
@@ -3117,7 +3123,7 @@ void iwl_mvm_update_frame_stats(struct iwl_mvm *mvm, u32 rate, bool agg)
  * Called after adding a new station to initialize rate scaling
  */
 static void rs_drv_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
-			     enum nl80211_band band, bool init)
+			     enum nl80211_band band)
 {
 	int i, j;
 	struct ieee80211_hw *hw = mvm->hw;
@@ -3134,7 +3140,8 @@ static void rs_drv_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	sband = hw->wiphy->bands[band];
 
 	lq_sta->lq.sta_id = mvmsta->sta_id;
-	mvmsta->tlc_amsdu = false;
+	mvmsta->amsdu_enabled = 0;
+	mvmsta->max_amsdu_len = sta->max_amsdu_len;
 
 	for (j = 0; j < LQ_SIZE; j++)
 		rs_rate_scale_clear_tbl_windows(mvm, &lq_sta->lq_info[j]);
@@ -3196,7 +3203,7 @@ static void rs_drv_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 	iwl_mvm_reset_frame_stats(mvm);
 #endif
-	rs_initialize_lq(mvm, sta, lq_sta, band, init);
+	rs_initialize_lq(mvm, sta, lq_sta, band);
 }
 
 static void rs_drv_rate_update(void *mvm_r,
@@ -3216,7 +3223,7 @@ static void rs_drv_rate_update(void *mvm_r,
 	for (tid = 0; tid < IWL_MAX_TID_COUNT; tid++)
 		ieee80211_stop_tx_ba_session(sta, tid);
 
-	iwl_mvm_rs_rate_init(mvm, sta, sband->band, false);
+	iwl_mvm_rs_rate_init(mvm, sta, sband->band);
 }
 
 #ifdef CONFIG_MAC80211_DEBUGFS
@@ -3744,7 +3751,7 @@ static ssize_t rs_sta_dbgfs_scale_table_read(struct file *file,
 				(rate->sgi) ? "SGI" : "NGI",
 				(rate->ldpc) ? "LDPC" : "BCC",
 				(lq_sta->is_agg) ? "AGG on" : "",
-				(mvmsta->tlc_amsdu) ? "AMSDU on" : "");
+				(mvmsta->amsdu_enabled) ? "AMSDU on" : "");
 	}
 	desc += scnprintf(buff + desc, bufsz - desc, "last tx rate=0x%X\n",
 			lq_sta->last_rate_n_flags);
@@ -4062,12 +4069,12 @@ static const struct rate_control_ops rs_mvm_ops_drv = {
 };
 
 void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
-			  enum nl80211_band band, bool init)
+			  enum nl80211_band band)
 {
 	if (iwl_mvm_has_tlc_offload(mvm))
 		rs_fw_rate_init(mvm, sta, band);
 	else
-		rs_drv_rate_init(mvm, sta, band, init);
+		rs_drv_rate_init(mvm, sta, band);
 }
 
 int iwl_mvm_rate_control_register(void)
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs.h b/drivers/net/wireless/intel/iwlwifi/mvm/rs.h
index fb18cb8c233d..cffb8c852934 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs.h
@@ -3,6 +3,7 @@
  * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of version 2 of the GNU General Public License as
@@ -13,10 +14,6 @@
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110, USA
- *
  * The full GNU General Public License is included in this distribution in the
  * file called LICENSE.
  *
@@ -410,7 +407,7 @@ struct iwl_lq_sta {
 
 /* Initialize station's rate scaling information after adding station */
 void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
-			  enum nl80211_band band, bool init);
+			  enum nl80211_band band);
 
 /* Notify RS about Tx status */
 void iwl_mvm_rs_tx_status(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
@@ -454,5 +451,6 @@ void rs_fw_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		     enum nl80211_band band);
 int rs_fw_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
 			bool enable);
-void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm, struct iwl_rx_packet *pkt);
+void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
+			      struct iwl_rx_cmd_buffer *rxb);
 #endif /* __rs__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
index d26833c5ce1f..bfb163419c67 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
@@ -254,6 +254,74 @@ static u32 iwl_mvm_set_mac80211_rx_flag(struct iwl_mvm *mvm,
 	return 0;
 }
 
+static void iwl_mvm_rx_handle_tcm(struct iwl_mvm *mvm,
+				  struct ieee80211_sta *sta,
+				  struct ieee80211_hdr *hdr, u32 len,
+				  struct iwl_rx_phy_info *phy_info,
+				  u32 rate_n_flags)
+{
+	struct iwl_mvm_sta *mvmsta;
+	struct iwl_mvm_tcm_mac *mdata;
+	int mac;
+	int ac = IEEE80211_AC_BE; /* treat non-QoS as BE */
+	struct iwl_mvm_vif *mvmvif;
+	/* expected throughput in 100Kbps, single stream, 20 MHz */
+	static const u8 thresh_tpt[] = {
+		9, 18, 30, 42, 60, 78, 90, 96, 120, 135,
+	};
+	u16 thr;
+
+	if (ieee80211_is_data_qos(hdr->frame_control))
+		ac = tid_to_mac80211_ac[ieee80211_get_tid(hdr)];
+
+	mvmsta = iwl_mvm_sta_from_mac80211(sta);
+	mac = mvmsta->mac_id_n_color & FW_CTXT_ID_MSK;
+
+	if (time_after(jiffies, mvm->tcm.ts + MVM_TCM_PERIOD))
+		schedule_delayed_work(&mvm->tcm.work, 0);
+	mdata = &mvm->tcm.data[mac];
+	mdata->rx.pkts[ac]++;
+
+	/* count the airtime only once for each ampdu */
+	if (mdata->rx.last_ampdu_ref != mvm->ampdu_ref) {
+		mdata->rx.last_ampdu_ref = mvm->ampdu_ref;
+		mdata->rx.airtime += le16_to_cpu(phy_info->frame_time);
+	}
+
+	if (!(rate_n_flags & (RATE_MCS_HT_MSK | RATE_MCS_VHT_MSK)))
+		return;
+
+	mvmvif = iwl_mvm_vif_from_mac80211(mvmsta->vif);
+
+	if (mdata->opened_rx_ba_sessions ||
+	    mdata->uapsd_nonagg_detect.detected ||
+	    (!mvmvif->queue_params[IEEE80211_AC_VO].uapsd &&
+	     !mvmvif->queue_params[IEEE80211_AC_VI].uapsd &&
+	     !mvmvif->queue_params[IEEE80211_AC_BE].uapsd &&
+	     !mvmvif->queue_params[IEEE80211_AC_BK].uapsd) ||
+	    mvmsta->sta_id != mvmvif->ap_sta_id)
+		return;
+
+	if (rate_n_flags & RATE_MCS_HT_MSK) {
+		thr = thresh_tpt[rate_n_flags & RATE_HT_MCS_RATE_CODE_MSK];
+		thr *= 1 + ((rate_n_flags & RATE_HT_MCS_NSS_MSK) >>
+					RATE_HT_MCS_NSS_POS);
+	} else {
+		if (WARN_ON((rate_n_flags & RATE_VHT_MCS_RATE_CODE_MSK) >=
+				ARRAY_SIZE(thresh_tpt)))
+			return;
+		thr = thresh_tpt[rate_n_flags & RATE_VHT_MCS_RATE_CODE_MSK];
+		thr *= 1 + ((rate_n_flags & RATE_VHT_MCS_NSS_MSK) >>
+					RATE_VHT_MCS_NSS_POS);
+	}
+
+	thr <<= ((rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) >>
+				RATE_MCS_CHAN_WIDTH_POS);
+
+	mdata->uapsd_nonagg_detect.rx_bytes += len;
+	ewma_rate_add(&mdata->uapsd_nonagg_detect.rate, thr);
+}
+
 static void iwl_mvm_rx_csum(struct ieee80211_sta *sta,
 			    struct sk_buff *skb,
 			    u32 status)
@@ -408,6 +476,12 @@ void iwl_mvm_rx_rx_mpdu(struct iwl_mvm *mvm, struct napi_struct *napi,
 							NULL);
 		}
 
+		if (!mvm->tcm.paused && len >= sizeof(*hdr) &&
+		    !is_multicast_ether_addr(hdr->addr1) &&
+		    ieee80211_is_data(hdr->frame_control))
+			iwl_mvm_rx_handle_tcm(mvm, sta, hdr, len, phy_info,
+					      rate_n_flags);
+
 		if (ieee80211_is_data(hdr->frame_control))
 			iwl_mvm_rx_csum(sta, skb, rx_pkt_status);
 	}
@@ -654,7 +728,8 @@ void iwl_mvm_handle_rx_statistics(struct iwl_mvm *mvm,
 	int expected_size;
 	int i;
 	u8 *energy;
-	__le32 *bytes, *air_time;
+	__le32 *bytes;
+	__le32 *air_time;
 	__le32 flags;
 
 	if (!iwl_mvm_has_new_rx_stats_api(mvm)) {
@@ -752,6 +827,32 @@ void iwl_mvm_handle_rx_statistics(struct iwl_mvm *mvm,
 		sta->avg_energy = energy[i];
 	}
 	rcu_read_unlock();
+
+	/*
+	 * Don't update in case the statistics are not cleared, since
+	 * we will end up counting twice the same airtime, once in TCM
+	 * request and once in statistics notification.
+	 */
+	if (!(le32_to_cpu(flags) & IWL_STATISTICS_REPLY_FLG_CLEAR))
+		return;
+
+	spin_lock(&mvm->tcm.lock);
+	for (i = 0; i < NUM_MAC_INDEX_DRIVER; i++) {
+		struct iwl_mvm_tcm_mac *mdata = &mvm->tcm.data[i];
+		u32 airtime = le32_to_cpu(air_time[i]);
+		u32 rx_bytes = le32_to_cpu(bytes[i]);
+
+		mdata->uapsd_nonagg_detect.rx_bytes += rx_bytes;
+		if (airtime) {
+			/* re-init every time to store rate from FW */
+			ewma_rate_init(&mdata->uapsd_nonagg_detect.rate);
+			ewma_rate_add(&mdata->uapsd_nonagg_detect.rate,
+				      rx_bytes * 8 / airtime);
+		}
+
+		mdata->rx.airtime += airtime;
+	}
+	spin_unlock(&mvm->tcm.lock);
 }
 
 void iwl_mvm_rx_statistics(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
index 4a4ccfd11e5b..129c4c09648d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
@@ -112,7 +112,7 @@ static inline int iwl_mvm_check_pn(struct iwl_mvm *mvm, struct sk_buff *skb,
 		return -1;
 
 	if (ieee80211_is_data_qos(hdr->frame_control))
-		tid = *ieee80211_get_qos_ctl(hdr) & IEEE80211_QOS_CTL_TID_MASK;
+		tid = ieee80211_get_tid(hdr);
 	else
 		tid = 0;
 
@@ -151,17 +151,9 @@ static void iwl_mvm_create_skb(struct sk_buff *skb, struct ieee80211_hdr *hdr,
 	unsigned int hdrlen = ieee80211_hdrlen(hdr->frame_control);
 
 	if (desc->mac_flags2 & IWL_RX_MPDU_MFLG2_PAD) {
+		len -= 2;
 		pad_len = 2;
-
-		/*
-		 * If the device inserted padding it means that (it thought)
-		 * the 802.11 header wasn't a multiple of 4 bytes long. In
-		 * this case, reserve two bytes at the start of the SKB to
-		 * align the payload properly in case we end up copying it.
-		 */
-		skb_reserve(skb, pad_len);
 	}
-	len -= pad_len;
 
 	/* If frame is small enough to fit in skb->head, pull it completely.
 	 * If not, only pull ieee80211_hdr (including crypto if present, and
@@ -235,12 +227,24 @@ static void iwl_mvm_get_signal_strength(struct iwl_mvm *mvm,
 }
 
 static int iwl_mvm_rx_crypto(struct iwl_mvm *mvm, struct ieee80211_hdr *hdr,
-			     struct ieee80211_rx_status *stats,
-			     struct iwl_rx_mpdu_desc *desc, u32 pkt_flags,
-			     int queue, u8 *crypt_len)
+			     struct ieee80211_rx_status *stats, u16 phy_info,
+			     struct iwl_rx_mpdu_desc *desc,
+			     u32 pkt_flags, int queue, u8 *crypt_len)
 {
 	u16 status = le16_to_cpu(desc->status);
 
+	/*
+	 * Drop UNKNOWN frames in aggregation, unless in monitor mode
+	 * (where we don't have the keys).
+	 * We limit this to aggregation because in TKIP this is a valid
+	 * scenario, since we may not have the (correct) TTAK (phase 1
+	 * key) in the firmware.
+	 */
+	if (phy_info & IWL_RX_MPDU_PHY_AMPDU &&
+	    (status & IWL_RX_MPDU_STATUS_SEC_MASK) ==
+	    IWL_RX_MPDU_STATUS_SEC_UNKNOWN && !mvm->monitor_on)
+		return -1;
+
 	if (!ieee80211_has_protected(hdr->frame_control) ||
 	    (status & IWL_RX_MPDU_STATUS_SEC_MASK) ==
 	    IWL_RX_MPDU_STATUS_SEC_NONE)
@@ -347,8 +351,7 @@ static bool iwl_mvm_is_dup(struct ieee80211_sta *sta, int queue,
 
 	if (ieee80211_is_data_qos(hdr->frame_control))
 		/* frame has qos control */
-		tid = *ieee80211_get_qos_ctl(hdr) &
-			IEEE80211_QOS_CTL_TID_MASK;
+		tid = ieee80211_get_tid(hdr);
 	else
 		tid = IWL_MAX_TID_COUNT;
 
@@ -587,14 +590,10 @@ void iwl_mvm_rx_queue_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb,
 	notif = (void *)pkt->data;
 	internal_notif = (void *)notif->payload;
 
-	if (internal_notif->sync) {
-		if (mvm->queue_sync_cookie != internal_notif->cookie) {
-			WARN_ONCE(1,
-				  "Received expired RX queue sync message\n");
-			return;
-		}
-		if (!atomic_dec_return(&mvm->queue_sync_counter))
-			wake_up(&mvm->rx_sync_waitq);
+	if (internal_notif->sync &&
+	    mvm->queue_sync_cookie != internal_notif->cookie) {
+		WARN_ONCE(1, "Received expired RX queue sync message\n");
+		return;
 	}
 
 	switch (internal_notif->type) {
@@ -606,6 +605,10 @@ void iwl_mvm_rx_queue_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb,
 	default:
 		WARN_ONCE(1, "Invalid identifier %d", internal_notif->type);
 	}
+
+	if (internal_notif->sync &&
+	    !atomic_dec_return(&mvm->queue_sync_counter))
+		wake_up(&mvm->rx_sync_waitq);
 }
 
 /*
@@ -628,7 +631,7 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 	bool amsdu = desc->mac_flags2 & IWL_RX_MPDU_MFLG2_AMSDU;
 	bool last_subframe =
 		desc->amsdu_info & IWL_RX_MPDU_AMSDU_LAST_SUBFRAME;
-	u8 tid = *ieee80211_get_qos_ctl(hdr) & IEEE80211_QOS_CTL_TID_MASK;
+	u8 tid = ieee80211_get_tid(hdr);
 	u8 sub_frame_idx = desc->amsdu_info &
 			   IWL_RX_MPDU_AMSDU_SUBFRAME_IDX_MASK;
 	struct iwl_mvm_reorder_buf_entry *entries;
@@ -867,9 +870,19 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		return;
 	}
 
+	if (desc->mac_flags2 & IWL_RX_MPDU_MFLG2_PAD) {
+		/*
+		 * If the device inserted padding it means that (it thought)
+		 * the 802.11 header wasn't a multiple of 4 bytes long. In
+		 * this case, reserve two bytes at the start of the SKB to
+		 * align the payload properly in case we end up copying it.
+		 */
+		skb_reserve(skb, 2);
+	}
+
 	rx_status = IEEE80211_SKB_RXCB(skb);
 
-	if (iwl_mvm_rx_crypto(mvm, hdr, rx_status, desc,
+	if (iwl_mvm_rx_crypto(mvm, hdr, rx_status, phy_info, desc,
 			      le32_to_cpu(pkt->len_n_flags), queue,
 			      &crypt_len)) {
 		kfree_skb(skb);
@@ -941,6 +954,12 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 			       IWL_RX_MPDU_REORDER_BAID_MASK) >>
 			       IWL_RX_MPDU_REORDER_BAID_SHIFT);
 
+		if (!mvm->tcm.paused && len >= sizeof(*hdr) &&
+		    !is_multicast_ether_addr(hdr->addr1) &&
+		    ieee80211_is_data(hdr->frame_control) &&
+		    time_after(jiffies, mvm->tcm.ts + MVM_TCM_PERIOD))
+			schedule_delayed_work(&mvm->tcm.work, 0);
+
 		/*
 		 * We have tx blocked stations (with CS bit). If we heard
 		 * frames from a blocked station on a new channel we can
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
index b31f0ffbbbf0..4b3753d78d03 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -19,9 +20,7 @@
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
+ * along with this program
  *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
@@ -76,12 +75,6 @@
 #define IWL_DENSE_EBS_SCAN_RATIO 5
 #define IWL_SPARSE_EBS_SCAN_RATIO 1
 
-enum iwl_mvm_traffic_load {
-	IWL_MVM_TRAFFIC_LOW,
-	IWL_MVM_TRAFFIC_MEDIUM,
-	IWL_MVM_TRAFFIC_HIGH,
-};
-
 #define IWL_SCAN_DWELL_ACTIVE		10
 #define IWL_SCAN_DWELL_PASSIVE		110
 #define IWL_SCAN_DWELL_FRAGMENTED	44
@@ -123,7 +116,9 @@ static struct iwl_mvm_scan_timing_params scan_timing[] = {
 };
 
 struct iwl_mvm_scan_params {
+	/* For CDB this is low band scan type, for non-CDB - type. */
 	enum iwl_mvm_scan_type type;
+	enum iwl_mvm_scan_type hb_type;
 	u32 n_channels;
 	u16 delay;
 	int n_ssids;
@@ -152,7 +147,7 @@ static inline void *iwl_mvm_get_scan_req_umac_data(struct iwl_mvm *mvm)
 	if (iwl_mvm_is_adaptive_dwell_supported(mvm))
 		return (void *)&cmd->v7.data;
 
-	if (iwl_mvm_has_new_tx_api(mvm))
+	if (iwl_mvm_cdb_scan_api(mvm))
 		return (void *)&cmd->v6.data;
 
 	return (void *)&cmd->v1.data;
@@ -169,7 +164,7 @@ iwl_mvm_get_scan_req_umac_channel(struct iwl_mvm *mvm)
 	if (iwl_mvm_is_adaptive_dwell_supported(mvm))
 		return &cmd->v7.channel;
 
-	if (iwl_mvm_has_new_tx_api(mvm))
+	if (iwl_mvm_cdb_scan_api(mvm))
 		return &cmd->v6.channel;
 
 	return &cmd->v1.channel;
@@ -234,15 +229,21 @@ static void iwl_mvm_scan_condition_iterator(void *data, u8 *mac,
 
 static enum iwl_mvm_traffic_load iwl_mvm_get_traffic_load(struct iwl_mvm *mvm)
 {
-	return IWL_MVM_TRAFFIC_LOW;
+	return mvm->tcm.result.global_load;
+}
+
+static enum iwl_mvm_traffic_load
+iwl_mvm_get_traffic_load_band(struct iwl_mvm *mvm, enum nl80211_band band)
+{
+	return mvm->tcm.result.band_load[band];
 }
 
 static enum
-iwl_mvm_scan_type iwl_mvm_get_scan_type(struct iwl_mvm *mvm, bool p2p_device)
+iwl_mvm_scan_type _iwl_mvm_get_scan_type(struct iwl_mvm *mvm, bool p2p_device,
+					 enum iwl_mvm_traffic_load load,
+					 bool low_latency)
 {
 	int global_cnt = 0;
-	enum iwl_mvm_traffic_load load;
-	bool low_latency;
 
 	ieee80211_iterate_active_interfaces_atomic(mvm->hw,
 					    IEEE80211_IFACE_ITER_NORMAL,
@@ -251,9 +252,6 @@ iwl_mvm_scan_type iwl_mvm_get_scan_type(struct iwl_mvm *mvm, bool p2p_device)
 	if (!global_cnt)
 		return IWL_SCAN_TYPE_UNASSOC;
 
-	load = iwl_mvm_get_traffic_load(mvm);
-	low_latency = iwl_mvm_low_latency(mvm);
-
 	if ((load == IWL_MVM_TRAFFIC_HIGH || low_latency) && !p2p_device &&
 	    fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_FRAGMENTED_SCAN))
 		return IWL_SCAN_TYPE_FRAGMENTED;
@@ -264,25 +262,57 @@ iwl_mvm_scan_type iwl_mvm_get_scan_type(struct iwl_mvm *mvm, bool p2p_device)
 	return IWL_SCAN_TYPE_WILD;
 }
 
+static enum
+iwl_mvm_scan_type iwl_mvm_get_scan_type(struct iwl_mvm *mvm, bool p2p_device)
+{
+	enum iwl_mvm_traffic_load load;
+	bool low_latency;
+
+	load = iwl_mvm_get_traffic_load(mvm);
+	low_latency = iwl_mvm_low_latency(mvm);
+
+	return _iwl_mvm_get_scan_type(mvm, p2p_device, load, low_latency);
+}
+
+static enum
+iwl_mvm_scan_type iwl_mvm_get_scan_type_band(struct iwl_mvm *mvm,
+					     bool p2p_device,
+					     enum nl80211_band band)
+{
+	enum iwl_mvm_traffic_load load;
+	bool low_latency;
+
+	load = iwl_mvm_get_traffic_load_band(mvm, band);
+	low_latency = iwl_mvm_low_latency_band(mvm, band);
+
+	return _iwl_mvm_get_scan_type(mvm, p2p_device, load, low_latency);
+}
+
 static int
 iwl_mvm_get_measurement_dwell(struct iwl_mvm *mvm,
 			      struct cfg80211_scan_request *req,
 			      struct iwl_mvm_scan_params *params)
 {
+	u32 duration = scan_timing[params->type].max_out_time;
+
 	if (!req->duration)
 		return 0;
 
-	if (req->duration_mandatory &&
-	    req->duration > scan_timing[params->type].max_out_time) {
+	if (iwl_mvm_is_cdb_supported(mvm)) {
+		u32 hb_time = scan_timing[params->hb_type].max_out_time;
+
+		duration = min_t(u32, duration, hb_time);
+	}
+
+	if (req->duration_mandatory && req->duration > duration) {
 		IWL_DEBUG_SCAN(mvm,
 			       "Measurement scan - too long dwell %hu (max out time %u)\n",
 			       req->duration,
-			       scan_timing[params->type].max_out_time);
+			       duration);
 		return -EOPNOTSUPP;
 	}
 
-	return min_t(u32, (u32)req->duration,
-		     scan_timing[params->type].max_out_time);
+	return min_t(u32, (u32)req->duration, duration);
 }
 
 static inline bool iwl_mvm_rrm_scan_needed(struct iwl_mvm *mvm)
@@ -437,6 +467,7 @@ void iwl_mvm_rx_lmac_scan_complete_notif(struct iwl_mvm *mvm,
 		ieee80211_scan_completed(mvm->hw, &info);
 		iwl_mvm_unref(mvm, IWL_MVM_REF_SCAN);
 		cancel_delayed_work(&mvm->scan_timeout_dwork);
+		iwl_mvm_resume_tcm(mvm);
 	} else {
 		IWL_ERR(mvm,
 			"got scan complete notification but no scan is running\n");
@@ -1030,22 +1061,38 @@ static void iwl_mvm_fill_scan_config_v1(struct iwl_mvm *mvm, void *config,
 static void iwl_mvm_fill_scan_config(struct iwl_mvm *mvm, void *config,
 				     u32 flags, u8 channel_flags)
 {
-	enum iwl_mvm_scan_type type = iwl_mvm_get_scan_type(mvm, false);
 	struct iwl_scan_config *cfg = config;
 
 	cfg->flags = cpu_to_le32(flags);
 	cfg->tx_chains = cpu_to_le32(iwl_mvm_get_valid_tx_ant(mvm));
 	cfg->rx_chains = cpu_to_le32(iwl_mvm_scan_rx_ant(mvm));
 	cfg->legacy_rates = iwl_mvm_scan_config_rates(mvm);
-	cfg->out_of_channel_time[0] =
-		cpu_to_le32(scan_timing[type].max_out_time);
-	cfg->suspend_time[0] = cpu_to_le32(scan_timing[type].suspend_time);
 
 	if (iwl_mvm_is_cdb_supported(mvm)) {
-		cfg->suspend_time[1] =
-			cpu_to_le32(scan_timing[type].suspend_time);
-		cfg->out_of_channel_time[1] =
+		enum iwl_mvm_scan_type lb_type, hb_type;
+
+		lb_type = iwl_mvm_get_scan_type_band(mvm, false,
+						     NL80211_BAND_2GHZ);
+		hb_type = iwl_mvm_get_scan_type_band(mvm, false,
+						     NL80211_BAND_5GHZ);
+
+		cfg->out_of_channel_time[SCAN_LB_LMAC_IDX] =
+			cpu_to_le32(scan_timing[lb_type].max_out_time);
+		cfg->suspend_time[SCAN_LB_LMAC_IDX] =
+			cpu_to_le32(scan_timing[lb_type].suspend_time);
+
+		cfg->out_of_channel_time[SCAN_HB_LMAC_IDX] =
+			cpu_to_le32(scan_timing[hb_type].max_out_time);
+		cfg->suspend_time[SCAN_HB_LMAC_IDX] =
+			cpu_to_le32(scan_timing[hb_type].suspend_time);
+	} else {
+		enum iwl_mvm_scan_type type =
+			iwl_mvm_get_scan_type(mvm, false);
+
+		cfg->out_of_channel_time[SCAN_LB_LMAC_IDX] =
 			cpu_to_le32(scan_timing[type].max_out_time);
+		cfg->suspend_time[SCAN_LB_LMAC_IDX] =
+			cpu_to_le32(scan_timing[type].suspend_time);
 	}
 
 	iwl_mvm_fill_scan_dwell(mvm, &cfg->dwell);
@@ -1065,7 +1112,8 @@ int iwl_mvm_config_scan(struct iwl_mvm *mvm)
 	struct iwl_host_cmd cmd = {
 		.id = iwl_cmd_id(SCAN_CFG_CMD, IWL_ALWAYS_LONG_GROUP, 0),
 	};
-	enum iwl_mvm_scan_type type = iwl_mvm_get_scan_type(mvm, false);
+	enum iwl_mvm_scan_type type;
+	enum iwl_mvm_scan_type hb_type = IWL_SCAN_TYPE_NOT_SET;
 	int num_channels =
 		mvm->nvm_data->bands[NL80211_BAND_2GHZ].n_channels +
 		mvm->nvm_data->bands[NL80211_BAND_5GHZ].n_channels;
@@ -1075,10 +1123,20 @@ int iwl_mvm_config_scan(struct iwl_mvm *mvm)
 	if (WARN_ON(num_channels > mvm->fw->ucode_capa.n_scan_channels))
 		return -ENOBUFS;
 
-	if (type == mvm->scan_type)
-		return 0;
+	if (iwl_mvm_is_cdb_supported(mvm)) {
+		type = iwl_mvm_get_scan_type_band(mvm, false,
+						  NL80211_BAND_2GHZ);
+		hb_type = iwl_mvm_get_scan_type_band(mvm, false,
+						     NL80211_BAND_5GHZ);
+		if (type == mvm->scan_type && hb_type == mvm->hb_scan_type)
+			return 0;
+	} else {
+		type = iwl_mvm_get_scan_type(mvm, false);
+		if (type == mvm->scan_type)
+			return 0;
+	}
 
-	if (iwl_mvm_has_new_tx_api(mvm))
+	if (iwl_mvm_cdb_scan_api(mvm))
 		cmd_size = sizeof(struct iwl_scan_config);
 	else
 		cmd_size = sizeof(struct iwl_scan_config_v1);
@@ -1107,10 +1165,15 @@ int iwl_mvm_config_scan(struct iwl_mvm *mvm)
 			IWL_CHANNEL_FLAG_EBS_ADD |
 			IWL_CHANNEL_FLAG_PRE_SCAN_PASSIVE2ACTIVE;
 
-	if (iwl_mvm_has_new_tx_api(mvm)) {
-		flags |= (type == IWL_SCAN_TYPE_FRAGMENTED) ?
-			 SCAN_CONFIG_FLAG_SET_LMAC2_FRAGMENTED :
-			 SCAN_CONFIG_FLAG_CLEAR_LMAC2_FRAGMENTED;
+	/*
+	 * Check for fragmented scan on LMAC2 - high band.
+	 * LMAC1 - low band is checked above.
+	 */
+	if (iwl_mvm_cdb_scan_api(mvm)) {
+		if (iwl_mvm_is_cdb_supported(mvm))
+			flags |= (hb_type == IWL_SCAN_TYPE_FRAGMENTED) ?
+				 SCAN_CONFIG_FLAG_SET_LMAC2_FRAGMENTED :
+				 SCAN_CONFIG_FLAG_CLEAR_LMAC2_FRAGMENTED;
 		iwl_mvm_fill_scan_config(mvm, cfg, flags, channel_flags);
 	} else {
 		iwl_mvm_fill_scan_config_v1(mvm, cfg, flags, channel_flags);
@@ -1123,8 +1186,10 @@ int iwl_mvm_config_scan(struct iwl_mvm *mvm)
 	IWL_DEBUG_SCAN(mvm, "Sending UMAC scan config\n");
 
 	ret = iwl_mvm_send_cmd(mvm, &cmd);
-	if (!ret)
+	if (!ret) {
 		mvm->scan_type = type;
+		mvm->hb_scan_type = hb_type;
+	}
 
 	kfree(cfg);
 	return ret;
@@ -1178,7 +1243,7 @@ static void iwl_mvm_scan_umac_dwell(struct iwl_mvm *mvm,
 			cpu_to_le32(timing->suspend_time);
 
 		if (iwl_mvm_is_cdb_supported(mvm)) {
-			hb_timing = &scan_timing[params->type];
+			hb_timing = &scan_timing[params->hb_type];
 
 			cmd->v7.max_out_time[SCAN_HB_LMAC_IDX] =
 				cpu_to_le32(hb_timing->max_out_time);
@@ -1208,7 +1273,7 @@ static void iwl_mvm_scan_umac_dwell(struct iwl_mvm *mvm,
 		cmd->v1.fragmented_dwell = IWL_SCAN_DWELL_FRAGMENTED;
 
 		if (iwl_mvm_is_cdb_supported(mvm)) {
-			hb_timing = &scan_timing[params->type];
+			hb_timing = &scan_timing[params->hb_type];
 
 			cmd->v6.max_out_time[SCAN_HB_LMAC_IDX] =
 					cpu_to_le32(hb_timing->max_out_time);
@@ -1216,7 +1281,7 @@ static void iwl_mvm_scan_umac_dwell(struct iwl_mvm *mvm,
 					cpu_to_le32(hb_timing->suspend_time);
 		}
 
-		if (iwl_mvm_has_new_tx_api(mvm)) {
+		if (iwl_mvm_cdb_scan_api(mvm)) {
 			cmd->v6.scan_priority =
 				cpu_to_le32(IWL_SCAN_PRIORITY_EXT_6);
 			cmd->v6.max_out_time[SCAN_LB_LMAC_IDX] =
@@ -1232,6 +1297,11 @@ static void iwl_mvm_scan_umac_dwell(struct iwl_mvm *mvm,
 				cpu_to_le32(timing->suspend_time);
 		}
 	}
+
+	if (iwl_mvm_is_regular_scan(params))
+		cmd->ooc_priority = cpu_to_le32(IWL_SCAN_PRIORITY_EXT_6);
+	else
+		cmd->ooc_priority = cpu_to_le32(IWL_SCAN_PRIORITY_EXT_2);
 }
 
 static void
@@ -1262,11 +1332,12 @@ static u16 iwl_mvm_scan_umac_flags(struct iwl_mvm *mvm,
 	if (params->n_ssids == 1 && params->ssids[0].ssid_len != 0)
 		flags |= IWL_UMAC_SCAN_GEN_FLAGS_PRE_CONNECT;
 
-	if (params->type == IWL_SCAN_TYPE_FRAGMENTED) {
+	if (params->type == IWL_SCAN_TYPE_FRAGMENTED)
 		flags |= IWL_UMAC_SCAN_GEN_FLAGS_FRAGMENTED;
-		if (iwl_mvm_is_cdb_supported(mvm))
-			flags |= IWL_UMAC_SCAN_GEN_FLAGS_LMAC2_FRAGMENTED;
-	}
+
+	if (iwl_mvm_is_cdb_supported(mvm) &&
+	    params->hb_type == IWL_SCAN_TYPE_FRAGMENTED)
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_LMAC2_FRAGMENTED;
 
 	if (iwl_mvm_rrm_scan_needed(mvm) &&
 	    fw_has_capa(&mvm->fw->ucode_capa,
@@ -1497,6 +1568,21 @@ void iwl_mvm_scan_timeout_wk(struct work_struct *work)
 	iwl_force_nmi(mvm->trans);
 }
 
+static void iwl_mvm_fill_scan_type(struct iwl_mvm *mvm,
+				   struct iwl_mvm_scan_params *params,
+				   bool p2p)
+{
+	if (iwl_mvm_is_cdb_supported(mvm)) {
+		params->type =
+			iwl_mvm_get_scan_type_band(mvm, p2p,
+						   NL80211_BAND_2GHZ);
+		params->hb_type =
+			iwl_mvm_get_scan_type_band(mvm, p2p,
+						   NL80211_BAND_5GHZ);
+	} else {
+		params->type = iwl_mvm_get_scan_type(mvm, p2p);
+	}
+}
 int iwl_mvm_reg_scan_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			   struct cfg80211_scan_request *req,
 			   struct ieee80211_scan_ies *ies)
@@ -1544,9 +1630,8 @@ int iwl_mvm_reg_scan_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	params.scan_plans = &scan_plan;
 	params.n_scan_plans = 1;
 
-	params.type =
-		iwl_mvm_get_scan_type(mvm,
-				      vif->type == NL80211_IFTYPE_P2P_DEVICE);
+	iwl_mvm_fill_scan_type(mvm, &params,
+			       vif->type == NL80211_IFTYPE_P2P_DEVICE);
 
 	ret = iwl_mvm_get_measurement_dwell(mvm, req, &params);
 	if (ret < 0)
@@ -1568,6 +1653,8 @@ int iwl_mvm_reg_scan_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	if (ret)
 		return ret;
 
+	iwl_mvm_pause_tcm(mvm, false);
+
 	ret = iwl_mvm_send_cmd(mvm, &hcmd);
 	if (ret) {
 		/* If the scan failed, it usually means that the FW was unable
@@ -1575,6 +1662,7 @@ int iwl_mvm_reg_scan_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		 * should try to send the command again with different params.
 		 */
 		IWL_ERR(mvm, "Scan failed! ret %d\n", ret);
+		iwl_mvm_resume_tcm(mvm);
 		return ret;
 	}
 
@@ -1638,9 +1726,8 @@ int iwl_mvm_sched_scan_start(struct iwl_mvm *mvm,
 	params.n_scan_plans = req->n_scan_plans;
 	params.scan_plans = req->scan_plans;
 
-	params.type =
-		iwl_mvm_get_scan_type(mvm,
-				      vif->type == NL80211_IFTYPE_P2P_DEVICE);
+	iwl_mvm_fill_scan_type(mvm, &params,
+			       vif->type == NL80211_IFTYPE_P2P_DEVICE);
 
 	/* In theory, LMAC scans can handle a 32-bit delay, but since
 	 * waiting for over 18 hours to start the scan is a bit silly
@@ -1711,6 +1798,7 @@ void iwl_mvm_rx_umac_scan_complete_notif(struct iwl_mvm *mvm,
 		mvm->scan_vif = NULL;
 		iwl_mvm_unref(mvm, IWL_MVM_REF_SCAN);
 		cancel_delayed_work(&mvm->scan_timeout_dwork);
+		iwl_mvm_resume_tcm(mvm);
 	} else if (mvm->scan_uid_status[uid] == IWL_MVM_SCAN_SCHED) {
 		ieee80211_sched_scan_stopped(mvm->hw);
 		mvm->sched_scan_pass_all = SCHED_SCAN_PASS_ALL_DISABLED;
@@ -1827,7 +1915,7 @@ int iwl_mvm_scan_size(struct iwl_mvm *mvm)
 		base_size = IWL_SCAN_REQ_UMAC_SIZE_V8;
 	else if (iwl_mvm_is_adaptive_dwell_supported(mvm))
 		base_size = IWL_SCAN_REQ_UMAC_SIZE_V7;
-	else if (iwl_mvm_has_new_tx_api(mvm))
+	else if (iwl_mvm_cdb_scan_api(mvm))
 		base_size = IWL_SCAN_REQ_UMAC_SIZE_V6;
 
 	if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_UMAC_SCAN))
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
index 80067eb9ea05..9263b9aa8b72 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2015 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -35,6 +36,7 @@
  * Copyright(c) 2012 - 2015 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -214,7 +216,7 @@ int iwl_mvm_sta_send_to_fw(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		cpu_to_le32(agg_size << STA_FLG_MAX_AGG_SIZE_SHIFT);
 	add_sta_cmd.station_flags |=
 		cpu_to_le32(mpdu_dens << STA_FLG_AGG_MPDU_DENS_SHIFT);
-	if (mvm_sta->associated)
+	if (mvm_sta->sta_state >= IEEE80211_STA_ASSOC)
 		add_sta_cmd.assoc_id = cpu_to_le16(sta->aid);
 
 	if (sta->wme) {
@@ -2466,6 +2468,15 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 	lockdep_assert_held(&mvm->mutex);
 
+	if (mvmsta->tid_data[tid].txq_id == IWL_MVM_INVALID_QUEUE &&
+	    iwl_mvm_has_new_tx_api(mvm)) {
+		u8 ac = tid_to_mac80211_ac[tid];
+
+		ret = iwl_mvm_sta_alloc_queue_tvqm(mvm, sta, ac, tid);
+		if (ret)
+			return ret;
+	}
+
 	spin_lock_bh(&mvmsta->lock);
 
 	/* possible race condition - we entered D0i3 while starting agg */
@@ -2887,7 +2898,7 @@ static int iwl_mvm_send_sta_key(struct iwl_mvm *mvm,
 				u32 sta_id,
 				struct ieee80211_key_conf *key, bool mcast,
 				u32 tkip_iv32, u16 *tkip_p1k, u32 cmd_flags,
-				u8 key_offset)
+				u8 key_offset, bool mfp)
 {
 	union {
 		struct iwl_mvm_add_sta_key_cmd_v1 cmd_v1;
@@ -2960,6 +2971,8 @@ static int iwl_mvm_send_sta_key(struct iwl_mvm *mvm,
 
 	if (mcast)
 		key_flags |= cpu_to_le16(STA_KEY_MULTICAST);
+	if (mfp)
+		key_flags |= cpu_to_le16(STA_KEY_MFP);
 
 	u.cmd.common.key_offset = key_offset;
 	u.cmd.common.key_flags = key_flags;
@@ -3101,11 +3114,13 @@ static int __iwl_mvm_set_sta_key(struct iwl_mvm *mvm,
 	struct ieee80211_key_seq seq;
 	u16 p1k[5];
 	u32 sta_id;
+	bool mfp = false;
 
 	if (sta) {
 		struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 
 		sta_id = mvm_sta->sta_id;
+		mfp = sta->mfp;
 	} else if (vif->type == NL80211_IFTYPE_AP &&
 		   !(keyconf->flags & IEEE80211_KEY_FLAG_PAIRWISE)) {
 		struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
@@ -3127,7 +3142,8 @@ static int __iwl_mvm_set_sta_key(struct iwl_mvm *mvm,
 		ieee80211_get_key_rx_seq(keyconf, 0, &seq);
 		ieee80211_get_tkip_rx_p1k(keyconf, addr, seq.tkip.iv32, p1k);
 		ret = iwl_mvm_send_sta_key(mvm, sta_id, keyconf, mcast,
-					   seq.tkip.iv32, p1k, 0, key_offset);
+					   seq.tkip.iv32, p1k, 0, key_offset,
+					   mfp);
 		break;
 	case WLAN_CIPHER_SUITE_CCMP:
 	case WLAN_CIPHER_SUITE_WEP40:
@@ -3135,11 +3151,11 @@ static int __iwl_mvm_set_sta_key(struct iwl_mvm *mvm,
 	case WLAN_CIPHER_SUITE_GCMP:
 	case WLAN_CIPHER_SUITE_GCMP_256:
 		ret = iwl_mvm_send_sta_key(mvm, sta_id, keyconf, mcast,
-					   0, NULL, 0, key_offset);
+					   0, NULL, 0, key_offset, mfp);
 		break;
 	default:
 		ret = iwl_mvm_send_sta_key(mvm, sta_id, keyconf, mcast,
-					   0, NULL, 0, key_offset);
+					   0, NULL, 0, key_offset, mfp);
 	}
 
 	return ret;
@@ -3366,6 +3382,7 @@ void iwl_mvm_update_tkip_key(struct iwl_mvm *mvm,
 {
 	struct iwl_mvm_sta *mvm_sta;
 	bool mcast = !(keyconf->flags & IEEE80211_KEY_FLAG_PAIRWISE);
+	bool mfp = sta ? sta->mfp : false;
 
 	rcu_read_lock();
 
@@ -3373,7 +3390,8 @@ void iwl_mvm_update_tkip_key(struct iwl_mvm *mvm,
 	if (WARN_ON_ONCE(!mvm_sta))
 		goto unlock;
 	iwl_mvm_send_sta_key(mvm, mvm_sta->sta_id, keyconf, mcast,
-			     iv32, phase1key, CMD_ASYNC, keyconf->hw_key_idx);
+			     iv32, phase1key, CMD_ASYNC, keyconf->hw_key_idx,
+			     mfp);
 
  unlock:
 	rcu_read_unlock();
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
index 5ffd6adbc383..1c43ea8dd8cc 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
  * Copyright(c) 2015 - 2016 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -18,11 +19,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -35,6 +31,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
  * Copyright(c) 2015 - 2016 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -376,6 +373,7 @@ struct iwl_mvm_rxq_dup_data {
  *	tid.
  * @max_agg_bufsize: the maximal size of the AGG buffer for this station
  * @sta_type: station type
+ * @sta_state: station state according to enum %ieee80211_sta_state
  * @bt_reduced_txpower: is reduced tx power enabled for this station
  * @next_status_eosp: the next reclaimed packet is a PS-Poll response and
  *	we need to signal the EOSP
@@ -391,7 +389,9 @@ struct iwl_mvm_rxq_dup_data {
  * @tx_protection: reference counter for controlling the Tx protection.
  * @tt_tx_protection: is thermal throttling enable Tx protection?
  * @disable_tx: is tx to this STA disabled?
- * @tlc_amsdu: true if A-MSDU is allowed
+ * @amsdu_enabled: bitmap of TX AMSDU allowed TIDs.
+ *	In case TLC offload is not active it is either 0xFFFF or 0.
+ * @max_amsdu_len: max AMSDU length
  * @agg_tids: bitmap of tids whose status is operational aggregated (IWL_AGG_ON)
  * @sleep_tx_count: the number of frames that we told the firmware to let out
  *	even when that station is asleep. This is useful in case the queue
@@ -414,6 +414,7 @@ struct iwl_mvm_sta {
 	u16 tid_disable_agg;
 	u8 max_agg_bufsize;
 	enum iwl_sta_type sta_type;
+	enum ieee80211_sta_state sta_state;
 	bool bt_reduced_txpower;
 	bool next_status_eosp;
 	spinlock_t lock;
@@ -436,9 +437,9 @@ struct iwl_mvm_sta {
 	bool tt_tx_protection;
 
 	bool disable_tx;
-	bool tlc_amsdu;
+	u16 amsdu_enabled;
+	u16 max_amsdu_len;
 	bool sleeping;
-	bool associated;
 	u8 agg_tids;
 	u8 sleep_tx_count;
 	u8 avg_energy;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c b/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c
index 3d97436bbdf5..67f360c0d17e 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c
@@ -7,6 +7,7 @@
  *
  * Copyright(c) 2014 Intel Mobile Communications GmbH
  * Copyright(c) 2017 Intel Deutschland GmbH
+ * Copyright(C) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -18,9 +19,7 @@
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
+ * along with this program.
  *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
@@ -33,6 +32,7 @@
  *
  * Copyright(c) 2014 Intel Mobile Communications GmbH
  * Copyright(c) 2017 Intel Deutschland GmbH
+ * Copyright(C) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -188,8 +188,14 @@ void iwl_mvm_recalc_tdls_state(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	if (tdls_sta_cnt == 1 && sta_added)
 		iwl_mvm_power_update_mac(mvm);
 
-	/* configure the FW with TDLS peer info */
-	iwl_mvm_tdls_config(mvm, vif);
+	/* Configure the FW with TDLS peer info only if TDLS channel switch
+	 * capability is set.
+	 * TDLS config data is used currently only in TDLS channel switch code.
+	 * Supposed to serve also TDLS buffer station which is not implemneted
+	 * yet in FW*/
+	if (fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_TDLS_CHANNEL_SWITCH))
+		iwl_mvm_tdls_config(mvm, vif);
 
 	/* when the last peer leaves, send a power update last */
 	if (tdls_sta_cnt == 0 && !sta_added)
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
index 795065974d78..cf2591f2ac23 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
@@ -767,16 +767,16 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	u16 snap_ip_tcp, pad;
 	unsigned int dbg_max_amsdu_len;
 	netdev_features_t netdev_flags = NETIF_F_CSUM_MASK | NETIF_F_SG;
-	u8 *qc, tid, txf;
+	u8 tid, txf;
 
 	snap_ip_tcp = 8 + skb_transport_header(skb) - skb_network_header(skb) +
 		tcp_hdrlen(skb);
 
 	dbg_max_amsdu_len = READ_ONCE(mvm->max_amsdu_len);
 
-	if (!sta->max_amsdu_len ||
+	if (!mvmsta->max_amsdu_len ||
 	    !ieee80211_is_data_qos(hdr->frame_control) ||
-	    (!mvmsta->tlc_amsdu && !dbg_max_amsdu_len))
+	    (!mvmsta->amsdu_enabled && !dbg_max_amsdu_len))
 		return iwl_mvm_tx_tso_segment(skb, 1, netdev_flags, mpdus_skb);
 
 	/*
@@ -790,8 +790,7 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 		return iwl_mvm_tx_tso_segment(skb, 1, netdev_flags, mpdus_skb);
 	}
 
-	qc = ieee80211_get_qos_ctl(hdr);
-	tid = *qc & IEEE80211_QOS_CTL_TID_MASK;
+	tid = ieee80211_get_tid(hdr);
 	if (WARN_ON_ONCE(tid >= IWL_MAX_TID_COUNT))
 		return -EINVAL;
 
@@ -803,7 +802,11 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	    !mvmsta->tid_data[tid].amsdu_in_ampdu_allowed)
 		return iwl_mvm_tx_tso_segment(skb, 1, netdev_flags, mpdus_skb);
 
-	max_amsdu_len = sta->max_amsdu_len;
+	if (iwl_mvm_vif_low_latency(iwl_mvm_vif_from_mac80211(mvmsta->vif)) ||
+	    !(mvmsta->amsdu_enabled & BIT(tid)))
+		return iwl_mvm_tx_tso_segment(skb, 1, netdev_flags, mpdus_skb);
+
+	max_amsdu_len = mvmsta->max_amsdu_len;
 
 	/* the Tx FIFO to which this A-MSDU will be routed */
 	txf = iwl_mvm_mac_ac_to_tx_fifo(mvm, tid_to_mac80211_ac[tid]);
@@ -840,8 +843,10 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	 * N * subf_len + (N - 1) * pad.
 	 */
 	num_subframes = (max_amsdu_len + pad) / (subf_len + pad);
-	if (num_subframes > 1)
-		*qc |= IEEE80211_QOS_CTL_A_MSDU_PRESENT;
+
+	if (sta->max_amsdu_subframes &&
+	    num_subframes > sta->max_amsdu_subframes)
+		num_subframes = sta->max_amsdu_subframes;
 
 	tcp_payload_len = skb_tail_pointer(skb) - skb_transport_header(skb) -
 		tcp_hdrlen(skb) + skb->data_len;
@@ -852,10 +857,12 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	 *	1 more for each fragment
 	 *	1 more for the potential data in the header
 	 */
-	num_subframes =
-		min_t(unsigned int, num_subframes,
-		      (mvm->trans->max_skb_frags - 1 -
-		       skb_shinfo(skb)->nr_frags) / 2);
+	if ((num_subframes * 2 + skb_shinfo(skb)->nr_frags + 1) >
+	    mvm->trans->max_skb_frags)
+		num_subframes = 1;
+
+	if (num_subframes > 1)
+		*ieee80211_get_qos_ctl(hdr) |= IEEE80211_QOS_CTL_A_MSDU_PRESENT;
 
 	/* This skb fits in one single A-MSDU */
 	if (num_subframes * mss >= tcp_payload_len) {
@@ -930,6 +937,32 @@ static bool iwl_mvm_txq_should_update(struct iwl_mvm *mvm, int txq_id)
 	return false;
 }
 
+static void iwl_mvm_tx_airtime(struct iwl_mvm *mvm,
+			       struct iwl_mvm_sta *mvmsta,
+			       int airtime)
+{
+	int mac = mvmsta->mac_id_n_color & FW_CTXT_ID_MSK;
+	struct iwl_mvm_tcm_mac *mdata = &mvm->tcm.data[mac];
+
+	if (mvm->tcm.paused)
+		return;
+
+	if (time_after(jiffies, mvm->tcm.ts + MVM_TCM_PERIOD))
+		schedule_delayed_work(&mvm->tcm.work, 0);
+
+	mdata->tx.airtime += airtime;
+}
+
+static void iwl_mvm_tx_pkt_queued(struct iwl_mvm *mvm,
+				  struct iwl_mvm_sta *mvmsta, int tid)
+{
+	u32 ac = tid_to_mac80211_ac[tid];
+	int mac = mvmsta->mac_id_n_color & FW_CTXT_ID_MSK;
+	struct iwl_mvm_tcm_mac *mdata = &mvm->tcm.data[mac];
+
+	mdata->tx.pkts[ac]++;
+}
+
 /*
  * Sets the fields in the Tx cmd that are crypto related
  */
@@ -976,9 +1009,7 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 	 * assignment of MGMT TID
 	 */
 	if (ieee80211_is_data_qos(fc) && !ieee80211_is_qos_nullfunc(fc)) {
-		u8 *qc = NULL;
-		qc = ieee80211_get_qos_ctl(hdr);
-		tid = qc[0] & IEEE80211_QOS_CTL_TID_MASK;
+		tid = ieee80211_get_tid(hdr);
 		if (WARN_ON_ONCE(tid >= IWL_MAX_TID_COUNT))
 			goto drop_unlock_sta;
 
@@ -1067,6 +1098,8 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 
 	spin_unlock(&mvmsta->lock);
 
+	iwl_mvm_tx_pkt_queued(mvm, mvmsta, tid == IWL_MAX_TID_COUNT ? 0 : tid);
+
 	return 0;
 
 drop_unlock_sta:
@@ -1469,6 +1502,9 @@ static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm,
 	if (!IS_ERR(sta)) {
 		mvmsta = iwl_mvm_sta_from_mac80211(sta);
 
+		iwl_mvm_tx_airtime(mvm, mvmsta,
+				   le16_to_cpu(tx_resp->wireless_media_time));
+
 		if (tid != IWL_TID_NON_QOS && tid != IWL_MGMT_TID) {
 			struct iwl_mvm_tid_data *tid_data =
 				&mvmsta->tid_data[tid];
@@ -1610,6 +1646,8 @@ static void iwl_mvm_rx_tx_cmd_agg(struct iwl_mvm *mvm,
 			le16_to_cpu(tx_resp->wireless_media_time);
 		mvmsta->tid_data[tid].lq_color =
 			TX_RES_RATE_TABLE_COL_GET(tx_resp->tlc_info);
+		iwl_mvm_tx_airtime(mvm, mvmsta,
+				   le16_to_cpu(tx_resp->wireless_media_time));
 	}
 
 	rcu_read_unlock();
@@ -1800,6 +1838,8 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 					   le32_to_cpu(ba_res->tx_rate));
 		}
 
+		iwl_mvm_tx_airtime(mvm, mvmsta,
+				   le32_to_cpu(ba_res->wireless_time));
 out_unlock:
 		rcu_read_unlock();
 out:
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
index d99d9ea78e4c..b002a7afb5f5 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -35,6 +36,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -278,8 +280,8 @@ u8 iwl_mvm_next_antenna(struct iwl_mvm *mvm, u8 valid, u8 last_idx)
 	u8 ind = last_idx;
 	int i;
 
-	for (i = 0; i < MAX_RS_ANT_NUM; i++) {
-		ind = (ind + 1) % MAX_RS_ANT_NUM;
+	for (i = 0; i < MAX_ANT_NUM; i++) {
+		ind = (ind + 1) % MAX_ANT_NUM;
 		if (valid & BIT(ind))
 			return ind;
 	}
@@ -520,15 +522,15 @@ static void iwl_mvm_dump_lmac_error_log(struct iwl_mvm *mvm, u32 base)
 
 		/* set INIT_DONE flag */
 		iwl_set_bit(trans, CSR_GP_CNTRL,
-			    CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
+			    BIT(trans->cfg->csr->flag_init_done));
 
 		/* and wait for clock stabilization */
 		if (trans->cfg->device_family == IWL_DEVICE_FAMILY_8000)
 			udelay(2);
 
 		err = iwl_poll_bit(trans, CSR_GP_CNTRL,
-				   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
-				   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
+				   BIT(trans->cfg->csr->flag_mac_clock_ready),
+				   BIT(trans->cfg->csr->flag_mac_clock_ready),
 				   25000);
 		if (err < 0) {
 			IWL_DEBUG_INFO(trans,
@@ -728,12 +730,14 @@ int iwl_mvm_tvqm_enable_txq(struct iwl_mvm *mvm, int mac80211_queue,
 		.sta_id = sta_id,
 		.tid = tid,
 	};
-	int queue;
+	int queue, size = IWL_DEFAULT_QUEUE_SIZE;
 
-	if (cmd.tid == IWL_MAX_TID_COUNT)
+	if (cmd.tid == IWL_MAX_TID_COUNT) {
 		cmd.tid = IWL_MGMT_TID;
+		size = IWL_MGMT_QUEUE_SIZE;
+	}
 	queue = iwl_trans_txq_alloc(mvm->trans, (void *)&cmd,
-				    SCD_QUEUE_CFG, timeout);
+				    SCD_QUEUE_CFG, size, timeout);
 
 	if (queue < 0) {
 		IWL_DEBUG_TX_QUEUES(mvm,
@@ -1074,23 +1078,48 @@ int iwl_mvm_update_low_latency(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	return iwl_mvm_power_update_mac(mvm);
 }
 
+struct iwl_mvm_low_latency_iter {
+	bool result;
+	bool result_per_band[NUM_NL80211_BANDS];
+};
+
 static void iwl_mvm_ll_iter(void *_data, u8 *mac, struct ieee80211_vif *vif)
 {
-	bool *result = _data;
+	struct iwl_mvm_low_latency_iter *result = _data;
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	enum nl80211_band band;
 
-	if (iwl_mvm_vif_low_latency(iwl_mvm_vif_from_mac80211(vif)))
-		*result = true;
+	if (iwl_mvm_vif_low_latency(mvmvif)) {
+		result->result = true;
+
+		if (!mvmvif->phy_ctxt)
+			return;
+
+		band = mvmvif->phy_ctxt->channel->band;
+		result->result_per_band[band] = true;
+	}
 }
 
 bool iwl_mvm_low_latency(struct iwl_mvm *mvm)
 {
-	bool result = false;
+	struct iwl_mvm_low_latency_iter data = {};
 
 	ieee80211_iterate_active_interfaces_atomic(
 			mvm->hw, IEEE80211_IFACE_ITER_NORMAL,
-			iwl_mvm_ll_iter, &result);
+			iwl_mvm_ll_iter, &data);
 
-	return result;
+	return data.result;
+}
+
+bool iwl_mvm_low_latency_band(struct iwl_mvm *mvm, enum nl80211_band band)
+{
+	struct iwl_mvm_low_latency_iter data = {};
+
+	ieee80211_iterate_active_interfaces_atomic(
+			mvm->hw, IEEE80211_IFACE_ITER_NORMAL,
+			iwl_mvm_ll_iter, &data);
+
+	return data.result_per_band[band];
 }
 
 struct iwl_bss_iter_data {
@@ -1429,6 +1458,387 @@ void iwl_mvm_event_frame_timeout_callback(struct iwl_mvm *mvm,
 				sta->addr, tid);
 }
 
+u8 iwl_mvm_tcm_load_percentage(u32 airtime, u32 elapsed)
+{
+	if (!elapsed)
+		return 0;
+
+	return (100 * airtime / elapsed) / USEC_PER_MSEC;
+}
+
+static enum iwl_mvm_traffic_load
+iwl_mvm_tcm_load(struct iwl_mvm *mvm, u32 airtime, unsigned long elapsed)
+{
+	u8 load = iwl_mvm_tcm_load_percentage(airtime, elapsed);
+
+	if (load > IWL_MVM_TCM_LOAD_HIGH_THRESH)
+		return IWL_MVM_TRAFFIC_HIGH;
+	if (load > IWL_MVM_TCM_LOAD_MEDIUM_THRESH)
+		return IWL_MVM_TRAFFIC_MEDIUM;
+
+	return IWL_MVM_TRAFFIC_LOW;
+}
+
+struct iwl_mvm_tcm_iter_data {
+	struct iwl_mvm *mvm;
+	bool any_sent;
+};
+
+static void iwl_mvm_tcm_iter(void *_data, u8 *mac, struct ieee80211_vif *vif)
+{
+	struct iwl_mvm_tcm_iter_data *data = _data;
+	struct iwl_mvm *mvm = data->mvm;
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	bool low_latency, prev = mvmvif->low_latency & LOW_LATENCY_TRAFFIC;
+
+	if (mvmvif->id >= NUM_MAC_INDEX_DRIVER)
+		return;
+
+	low_latency = mvm->tcm.result.low_latency[mvmvif->id];
+
+	if (!mvm->tcm.result.change[mvmvif->id] &&
+	    prev == low_latency) {
+		iwl_mvm_update_quotas(mvm, false, NULL);
+		return;
+	}
+
+	if (prev != low_latency) {
+		/* this sends traffic load and updates quota as well */
+		iwl_mvm_update_low_latency(mvm, vif, low_latency,
+					   LOW_LATENCY_TRAFFIC);
+	} else {
+		iwl_mvm_update_quotas(mvm, false, NULL);
+	}
+
+	data->any_sent = true;
+}
+
+static void iwl_mvm_tcm_results(struct iwl_mvm *mvm)
+{
+	struct iwl_mvm_tcm_iter_data data = {
+		.mvm = mvm,
+		.any_sent = false,
+	};
+
+	mutex_lock(&mvm->mutex);
+
+	ieee80211_iterate_active_interfaces(
+		mvm->hw, IEEE80211_IFACE_ITER_NORMAL,
+		iwl_mvm_tcm_iter, &data);
+
+	if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_UMAC_SCAN))
+		iwl_mvm_config_scan(mvm);
+
+	mutex_unlock(&mvm->mutex);
+}
+
+static void iwl_mvm_tcm_uapsd_nonagg_detected_wk(struct work_struct *wk)
+{
+	struct iwl_mvm *mvm;
+	struct iwl_mvm_vif *mvmvif;
+	struct ieee80211_vif *vif;
+
+	mvmvif = container_of(wk, struct iwl_mvm_vif,
+			      uapsd_nonagg_detected_wk.work);
+	vif = container_of((void *)mvmvif, struct ieee80211_vif, drv_priv);
+	mvm = mvmvif->mvm;
+
+	if (mvm->tcm.data[mvmvif->id].opened_rx_ba_sessions)
+		return;
+
+	/* remember that this AP is broken */
+	memcpy(mvm->uapsd_noagg_bssids[mvm->uapsd_noagg_bssid_write_idx].addr,
+	       vif->bss_conf.bssid, ETH_ALEN);
+	mvm->uapsd_noagg_bssid_write_idx++;
+	if (mvm->uapsd_noagg_bssid_write_idx >= IWL_MVM_UAPSD_NOAGG_LIST_LEN)
+		mvm->uapsd_noagg_bssid_write_idx = 0;
+
+	iwl_mvm_connection_loss(mvm, vif,
+				"AP isn't using AMPDU with uAPSD enabled");
+}
+
+static void iwl_mvm_uapsd_agg_disconnect_iter(void *data, u8 *mac,
+					      struct ieee80211_vif *vif)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm *mvm = mvmvif->mvm;
+	int *mac_id = data;
+
+	if (vif->type != NL80211_IFTYPE_STATION)
+		return;
+
+	if (mvmvif->id != *mac_id)
+		return;
+
+	if (!vif->bss_conf.assoc)
+		return;
+
+	if (!mvmvif->queue_params[IEEE80211_AC_VO].uapsd &&
+	    !mvmvif->queue_params[IEEE80211_AC_VI].uapsd &&
+	    !mvmvif->queue_params[IEEE80211_AC_BE].uapsd &&
+	    !mvmvif->queue_params[IEEE80211_AC_BK].uapsd)
+		return;
+
+	if (mvm->tcm.data[*mac_id].uapsd_nonagg_detect.detected)
+		return;
+
+	mvm->tcm.data[*mac_id].uapsd_nonagg_detect.detected = true;
+	IWL_INFO(mvm,
+		 "detected AP should do aggregation but isn't, likely due to U-APSD\n");
+	schedule_delayed_work(&mvmvif->uapsd_nonagg_detected_wk, 15 * HZ);
+}
+
+static void iwl_mvm_check_uapsd_agg_expected_tpt(struct iwl_mvm *mvm,
+						 unsigned int elapsed,
+						 int mac)
+{
+	u64 bytes = mvm->tcm.data[mac].uapsd_nonagg_detect.rx_bytes;
+	u64 tpt;
+	unsigned long rate;
+
+	rate = ewma_rate_read(&mvm->tcm.data[mac].uapsd_nonagg_detect.rate);
+
+	if (!rate || mvm->tcm.data[mac].opened_rx_ba_sessions ||
+	    mvm->tcm.data[mac].uapsd_nonagg_detect.detected)
+		return;
+
+	if (iwl_mvm_has_new_rx_api(mvm)) {
+		tpt = 8 * bytes; /* kbps */
+		do_div(tpt, elapsed);
+		rate *= 1000; /* kbps */
+		if (tpt < 22 * rate / 100)
+			return;
+	} else {
+		/*
+		 * the rate here is actually the threshold, in 100Kbps units,
+		 * so do the needed conversion from bytes to 100Kbps:
+		 * 100kb = bits / (100 * 1000),
+		 * 100kbps = 100kb / (msecs / 1000) ==
+		 *           (bits / (100 * 1000)) / (msecs / 1000) ==
+		 *           bits / (100 * msecs)
+		 */
+		tpt = (8 * bytes);
+		do_div(tpt, elapsed * 100);
+		if (tpt < rate)
+			return;
+	}
+
+	ieee80211_iterate_active_interfaces_atomic(
+		mvm->hw, IEEE80211_IFACE_ITER_NORMAL,
+		iwl_mvm_uapsd_agg_disconnect_iter, &mac);
+}
+
+static void iwl_mvm_tcm_iterator(void *_data, u8 *mac,
+				 struct ieee80211_vif *vif)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	u32 *band = _data;
+
+	if (!mvmvif->phy_ctxt)
+		return;
+
+	band[mvmvif->id] = mvmvif->phy_ctxt->channel->band;
+}
+
+static unsigned long iwl_mvm_calc_tcm_stats(struct iwl_mvm *mvm,
+					    unsigned long ts,
+					    bool handle_uapsd)
+{
+	unsigned int elapsed = jiffies_to_msecs(ts - mvm->tcm.ts);
+	unsigned int uapsd_elapsed =
+		jiffies_to_msecs(ts - mvm->tcm.uapsd_nonagg_ts);
+	u32 total_airtime = 0;
+	u32 band_airtime[NUM_NL80211_BANDS] = {0};
+	u32 band[NUM_MAC_INDEX_DRIVER] = {0};
+	int ac, mac, i;
+	bool low_latency = false;
+	enum iwl_mvm_traffic_load load, band_load;
+	bool handle_ll = time_after(ts, mvm->tcm.ll_ts + MVM_LL_PERIOD);
+
+	if (handle_ll)
+		mvm->tcm.ll_ts = ts;
+	if (handle_uapsd)
+		mvm->tcm.uapsd_nonagg_ts = ts;
+
+	mvm->tcm.result.elapsed = elapsed;
+
+	ieee80211_iterate_active_interfaces_atomic(mvm->hw,
+						   IEEE80211_IFACE_ITER_NORMAL,
+						   iwl_mvm_tcm_iterator,
+						   &band);
+
+	for (mac = 0; mac < NUM_MAC_INDEX_DRIVER; mac++) {
+		struct iwl_mvm_tcm_mac *mdata = &mvm->tcm.data[mac];
+		u32 vo_vi_pkts = 0;
+		u32 airtime = mdata->rx.airtime + mdata->tx.airtime;
+
+		total_airtime += airtime;
+		band_airtime[band[mac]] += airtime;
+
+		load = iwl_mvm_tcm_load(mvm, airtime, elapsed);
+		mvm->tcm.result.change[mac] = load != mvm->tcm.result.load[mac];
+		mvm->tcm.result.load[mac] = load;
+		mvm->tcm.result.airtime[mac] = airtime;
+
+		for (ac = IEEE80211_AC_VO; ac <= IEEE80211_AC_VI; ac++)
+			vo_vi_pkts += mdata->rx.pkts[ac] +
+				      mdata->tx.pkts[ac];
+
+		/* enable immediately with enough packets but defer disabling */
+		if (vo_vi_pkts > IWL_MVM_TCM_LOWLAT_ENABLE_THRESH)
+			mvm->tcm.result.low_latency[mac] = true;
+		else if (handle_ll)
+			mvm->tcm.result.low_latency[mac] = false;
+
+		if (handle_ll) {
+			/* clear old data */
+			memset(&mdata->rx.pkts, 0, sizeof(mdata->rx.pkts));
+			memset(&mdata->tx.pkts, 0, sizeof(mdata->tx.pkts));
+		}
+		low_latency |= mvm->tcm.result.low_latency[mac];
+
+		if (!mvm->tcm.result.low_latency[mac] && handle_uapsd)
+			iwl_mvm_check_uapsd_agg_expected_tpt(mvm, uapsd_elapsed,
+							     mac);
+		/* clear old data */
+		if (handle_uapsd)
+			mdata->uapsd_nonagg_detect.rx_bytes = 0;
+		memset(&mdata->rx.airtime, 0, sizeof(mdata->rx.airtime));
+		memset(&mdata->tx.airtime, 0, sizeof(mdata->tx.airtime));
+	}
+
+	load = iwl_mvm_tcm_load(mvm, total_airtime, elapsed);
+	mvm->tcm.result.global_change = load != mvm->tcm.result.global_load;
+	mvm->tcm.result.global_load = load;
+
+	for (i = 0; i < NUM_NL80211_BANDS; i++) {
+		band_load = iwl_mvm_tcm_load(mvm, band_airtime[i], elapsed);
+		mvm->tcm.result.band_load[i] = band_load;
+	}
+
+	/*
+	 * If the current load isn't low we need to force re-evaluation
+	 * in the TCM period, so that we can return to low load if there
+	 * was no traffic at all (and thus iwl_mvm_recalc_tcm didn't get
+	 * triggered by traffic).
+	 */
+	if (load != IWL_MVM_TRAFFIC_LOW)
+		return MVM_TCM_PERIOD;
+	/*
+	 * If low-latency is active we need to force re-evaluation after
+	 * (the longer) MVM_LL_PERIOD, so that we can disable low-latency
+	 * when there's no traffic at all.
+	 */
+	if (low_latency)
+		return MVM_LL_PERIOD;
+	/*
+	 * Otherwise, we don't need to run the work struct because we're
+	 * in the default "idle" state - traffic indication is low (which
+	 * also covers the "no traffic" case) and low-latency is disabled
+	 * so there's no state that may need to be disabled when there's
+	 * no traffic at all.
+	 *
+	 * Note that this has no impact on the regular scheduling of the
+	 * updates triggered by traffic - those happen whenever one of the
+	 * two timeouts expire (if there's traffic at all.)
+	 */
+	return 0;
+}
+
+void iwl_mvm_recalc_tcm(struct iwl_mvm *mvm)
+{
+	unsigned long ts = jiffies;
+	bool handle_uapsd =
+		time_after(ts, mvm->tcm.uapsd_nonagg_ts +
+			       msecs_to_jiffies(IWL_MVM_UAPSD_NONAGG_PERIOD));
+
+	spin_lock(&mvm->tcm.lock);
+	if (mvm->tcm.paused || !time_after(ts, mvm->tcm.ts + MVM_TCM_PERIOD)) {
+		spin_unlock(&mvm->tcm.lock);
+		return;
+	}
+	spin_unlock(&mvm->tcm.lock);
+
+	if (handle_uapsd && iwl_mvm_has_new_rx_api(mvm)) {
+		mutex_lock(&mvm->mutex);
+		if (iwl_mvm_request_statistics(mvm, true))
+			handle_uapsd = false;
+		mutex_unlock(&mvm->mutex);
+	}
+
+	spin_lock(&mvm->tcm.lock);
+	/* re-check if somebody else won the recheck race */
+	if (!mvm->tcm.paused && time_after(ts, mvm->tcm.ts + MVM_TCM_PERIOD)) {
+		/* calculate statistics */
+		unsigned long work_delay = iwl_mvm_calc_tcm_stats(mvm, ts,
+								  handle_uapsd);
+
+		/* the memset needs to be visible before the timestamp */
+		smp_mb();
+		mvm->tcm.ts = ts;
+		if (work_delay)
+			schedule_delayed_work(&mvm->tcm.work, work_delay);
+	}
+	spin_unlock(&mvm->tcm.lock);
+
+	iwl_mvm_tcm_results(mvm);
+}
+
+void iwl_mvm_tcm_work(struct work_struct *work)
+{
+	struct delayed_work *delayed_work = to_delayed_work(work);
+	struct iwl_mvm *mvm = container_of(delayed_work, struct iwl_mvm,
+					   tcm.work);
+
+	iwl_mvm_recalc_tcm(mvm);
+}
+
+void iwl_mvm_pause_tcm(struct iwl_mvm *mvm, bool with_cancel)
+{
+	spin_lock_bh(&mvm->tcm.lock);
+	mvm->tcm.paused = true;
+	spin_unlock_bh(&mvm->tcm.lock);
+	if (with_cancel)
+		cancel_delayed_work_sync(&mvm->tcm.work);
+}
+
+void iwl_mvm_resume_tcm(struct iwl_mvm *mvm)
+{
+	int mac;
+
+	spin_lock_bh(&mvm->tcm.lock);
+	mvm->tcm.ts = jiffies;
+	mvm->tcm.ll_ts = jiffies;
+	for (mac = 0; mac < NUM_MAC_INDEX_DRIVER; mac++) {
+		struct iwl_mvm_tcm_mac *mdata = &mvm->tcm.data[mac];
+
+		memset(&mdata->rx.pkts, 0, sizeof(mdata->rx.pkts));
+		memset(&mdata->tx.pkts, 0, sizeof(mdata->tx.pkts));
+		memset(&mdata->rx.airtime, 0, sizeof(mdata->rx.airtime));
+		memset(&mdata->tx.airtime, 0, sizeof(mdata->tx.airtime));
+	}
+	/* The TCM data needs to be reset before "paused" flag changes */
+	smp_mb();
+	mvm->tcm.paused = false;
+	spin_unlock_bh(&mvm->tcm.lock);
+}
+
+void iwl_mvm_tcm_add_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+
+	INIT_DELAYED_WORK(&mvmvif->uapsd_nonagg_detected_wk,
+			  iwl_mvm_tcm_uapsd_nonagg_detected_wk);
+}
+
+void iwl_mvm_tcm_rm_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+
+	cancel_delayed_work_sync(&mvmvif->uapsd_nonagg_detected_wk);
+}
+
+
 void iwl_mvm_get_sync_time(struct iwl_mvm *mvm, u32 *gp2, u64 *boottime)
 {
 	bool ps_disabled;
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c b/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
index 5ef216f3a60b..3fc4343581ee 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
@@ -244,7 +244,7 @@ int iwl_pcie_ctxt_info_init(struct iwl_trans *trans,
 	ctxt_info->hcmd_cfg.cmd_queue_addr =
 		cpu_to_le64(trans_pcie->txq[trans_pcie->cmd_queue]->dma_addr);
 	ctxt_info->hcmd_cfg.cmd_queue_size =
-		TFD_QUEUE_CB_SIZE(trans_pcie->tx_cmd_queue_size);
+		TFD_QUEUE_CB_SIZE(TFD_CMD_SLOTS);
 
 	/* allocate ucode sections in dram and set addresses */
 	ret = iwl_pcie_ctxt_info_init_fw_sec(trans, fw, ctxt_info);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
index 959de2f8bb28..38234bda9017 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
@@ -836,6 +836,9 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct iwl_trans *iwl_trans;
 	int ret;
 
+	if (WARN_ONCE(!cfg->csr, "CSR addresses aren't configured\n"))
+		return -EINVAL;
+
 	iwl_trans = iwl_trans_pcie_alloc(pdev, ent, cfg);
 	if (IS_ERR(iwl_trans))
 		return PTR_ERR(iwl_trans);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
index ca3b64ff4dad..45ea32796cda 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
@@ -383,7 +383,8 @@ struct iwl_self_init_dram {
  * @hw_init_mask: initial unmasked hw causes
  * @fh_mask: current unmasked fh causes
  * @hw_mask: current unmasked hw causes
- * @tx_cmd_queue_size: the size of the tx command queue
+ * @in_rescan: true if we have triggered a device rescan
+ * @scheduled_for_removal: true if we have scheduled a device removal
  */
 struct iwl_trans_pcie {
 	struct iwl_rxq *rxq;
@@ -466,6 +467,8 @@ struct iwl_trans_pcie {
 	u32 hw_mask;
 	cpumask_t affinity_mask[IWL_MAX_RX_HW_QUEUES];
 	u16 tx_cmd_queue_size;
+	bool in_rescan;
+	bool scheduled_for_removal;
 };
 
 static inline struct iwl_trans_pcie *
@@ -537,7 +540,6 @@ void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
 void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 			    struct sk_buff_head *skbs);
 void iwl_trans_pcie_tx_reset(struct iwl_trans *trans);
-void iwl_pcie_set_tx_cmd_queue_size(struct iwl_trans *trans);
 
 static inline u16 iwl_pcie_tfd_tb_get_len(struct iwl_trans *trans, void *_tfd,
 					  u8 idx)
@@ -822,7 +824,7 @@ int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
 void iwl_trans_pcie_gen2_fw_alive(struct iwl_trans *trans, u32 scd_addr);
 int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans *trans,
 				 struct iwl_tx_queue_cfg_cmd *cmd,
-				 int cmd_id,
+				 int cmd_id, int size,
 				 unsigned int timeout);
 void iwl_trans_pcie_dyn_txq_free(struct iwl_trans *trans, int queue);
 int iwl_trans_pcie_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb,
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
index f25ce3a1ea50..d15f5ba2dc77 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
@@ -3,6 +3,7 @@
  * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * Portions of this file are derived from the ipw3945 project, as well
  * as portions of the ieee80211 subsystem header files.
@@ -201,7 +202,7 @@ static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans,
 			IWL_DEBUG_INFO(trans, "Rx queue requesting wakeup, GP1 = 0x%x\n",
 				       reg);
 			iwl_set_bit(trans, CSR_GP_CNTRL,
-				    CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+				    BIT(trans->cfg->csr->flag_mac_access_req));
 			rxq->need_update = true;
 			return;
 		}
@@ -901,6 +902,8 @@ static int _iwl_pcie_rx_init(struct iwl_trans *trans)
 	}
 	def_rxq = trans_pcie->rxq;
 
+	cancel_work_sync(&rba->rx_alloc);
+
 	spin_lock(&rba->lock);
 	atomic_set(&rba->req_pending, 0);
 	atomic_set(&rba->req_ready, 0);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
index cb4012541f45..b8e8dac2895d 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
@@ -6,6 +6,7 @@
  * GPL LICENSE SUMMARY
  *
  * Copyright(c) 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -19,6 +20,7 @@
  * BSD LICENSE
  *
  * Copyright(c) 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -92,7 +94,8 @@ static int iwl_pcie_gen2_apm_init(struct iwl_trans *trans)
 	 * Set "initialization complete" bit to move adapter from
 	 * D0U* --> D0A* (powered-up active) state.
 	 */
-	iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
+	iwl_set_bit(trans, CSR_GP_CNTRL,
+		    BIT(trans->cfg->csr->flag_init_done));
 
 	/*
 	 * Wait for clock stabilization; once stabilized, access to
@@ -100,8 +103,9 @@ static int iwl_pcie_gen2_apm_init(struct iwl_trans *trans)
 	 * and accesses to uCode SRAM.
 	 */
 	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
-			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
-			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, 25000);
+			   BIT(trans->cfg->csr->flag_mac_clock_ready),
+			   BIT(trans->cfg->csr->flag_mac_clock_ready),
+			   25000);
 	if (ret < 0) {
 		IWL_DEBUG_INFO(trans, "Failed to init the card\n");
 		return ret;
@@ -143,7 +147,8 @@ static void iwl_pcie_gen2_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
 	 * Clear "initialization complete" bit to move adapter from
 	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
 	 */
-	iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
+	iwl_clear_bit(trans, CSR_GP_CNTRL,
+		      BIT(trans->cfg->csr->flag_init_done));
 }
 
 void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans, bool low_power)
@@ -187,7 +192,7 @@ void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans, bool low_power)
 
 	/* Make sure (redundant) we've released our request to stay awake */
 	iwl_clear_bit(trans, CSR_GP_CNTRL,
-		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+		      BIT(trans->cfg->csr->flag_mac_access_req));
 
 	/* Stop the device, and put it in low power state */
 	iwl_pcie_gen2_apm_stop(trans, false);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
index f8a0234d332c..7229991ae70d 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2007 - 2015 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -35,6 +36,7 @@
  * Copyright(c) 2005 - 2015 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -73,6 +75,7 @@
 #include <linux/gfp.h>
 #include <linux/vmalloc.h>
 #include <linux/pm_runtime.h>
+#include <linux/module.h>
 
 #include "iwl-drv.h"
 #include "iwl-trans.h"
@@ -179,7 +182,8 @@ static void iwl_trans_pcie_dump_regs(struct iwl_trans *trans)
 static void iwl_trans_pcie_sw_reset(struct iwl_trans *trans)
 {
 	/* Reset entire device - do controller reset (results in SHRD_HW_RST) */
-	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);
+	iwl_set_bit(trans, trans->cfg->csr->addr_sw_reset,
+		    BIT(trans->cfg->csr->flag_sw_reset));
 	usleep_range(5000, 6000);
 }
 
@@ -372,7 +376,8 @@ static int iwl_pcie_apm_init(struct iwl_trans *trans)
 	 * Set "initialization complete" bit to move adapter from
 	 * D0U* --> D0A* (powered-up active) state.
 	 */
-	iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
+	iwl_set_bit(trans, CSR_GP_CNTRL,
+		    BIT(trans->cfg->csr->flag_init_done));
 
 	/*
 	 * Wait for clock stabilization; once stabilized, access to
@@ -380,8 +385,9 @@ static int iwl_pcie_apm_init(struct iwl_trans *trans)
 	 * and accesses to uCode SRAM.
 	 */
 	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
-			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
-			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, 25000);
+			   BIT(trans->cfg->csr->flag_mac_clock_ready),
+			   BIT(trans->cfg->csr->flag_mac_clock_ready),
+			   25000);
 	if (ret < 0) {
 		IWL_ERR(trans, "Failed to init the card\n");
 		return ret;
@@ -459,15 +465,16 @@ static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
 	 * Set "initialization complete" bit to move adapter from
 	 * D0U* --> D0A* (powered-up active) state.
 	 */
-	iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
+	iwl_set_bit(trans, CSR_GP_CNTRL,
+		    BIT(trans->cfg->csr->flag_init_done));
 
 	/*
 	 * Wait for clock stabilization; once stabilized, access to
 	 * device-internal resources is possible.
 	 */
 	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
-			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
-			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
+			   BIT(trans->cfg->csr->flag_mac_clock_ready),
+			   BIT(trans->cfg->csr->flag_mac_clock_ready),
 			   25000);
 	if (WARN_ON(ret < 0)) {
 		IWL_ERR(trans, "Access time out - failed to enable LP XTAL\n");
@@ -519,7 +526,7 @@ static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
 	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
 	 */
 	iwl_clear_bit(trans, CSR_GP_CNTRL,
-		      CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
+		      BIT(trans->cfg->csr->flag_init_done));
 
 	/* Activates XTAL resources monitor */
 	__iwl_trans_pcie_set_bit(trans, CSR_MONITOR_CFG_REG,
@@ -541,11 +548,12 @@ void iwl_pcie_apm_stop_master(struct iwl_trans *trans)
 	int ret;
 
 	/* stop device's busmaster DMA activity */
-	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_STOP_MASTER);
+	iwl_set_bit(trans, trans->cfg->csr->addr_sw_reset,
+		    BIT(trans->cfg->csr->flag_stop_master));
 
-	ret = iwl_poll_bit(trans, CSR_RESET,
-			   CSR_RESET_REG_FLAG_MASTER_DISABLED,
-			   CSR_RESET_REG_FLAG_MASTER_DISABLED, 100);
+	ret = iwl_poll_bit(trans, trans->cfg->csr->addr_sw_reset,
+			   BIT(trans->cfg->csr->flag_master_dis),
+			   BIT(trans->cfg->csr->flag_master_dis), 100);
 	if (ret < 0)
 		IWL_WARN(trans, "Master Disable Timed Out, 100 usec\n");
 
@@ -594,7 +602,7 @@ static void iwl_pcie_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
 	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
 	 */
 	iwl_clear_bit(trans, CSR_GP_CNTRL,
-		      CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
+		      BIT(trans->cfg->csr->flag_init_done));
 }
 
 static int iwl_pcie_nic_init(struct iwl_trans *trans)
@@ -1267,7 +1275,7 @@ static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool low_power)
 
 	/* Make sure (redundant) we've released our request to stay awake */
 	iwl_clear_bit(trans, CSR_GP_CNTRL,
-		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+		      BIT(trans->cfg->csr->flag_mac_access_req));
 
 	/* Stop the device, and put it in low power state */
 	iwl_pcie_apm_stop(trans, false);
@@ -1497,9 +1505,9 @@ static void iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test,
 	iwl_pcie_synchronize_irqs(trans);
 
 	iwl_clear_bit(trans, CSR_GP_CNTRL,
-		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+		      BIT(trans->cfg->csr->flag_mac_access_req));
 	iwl_clear_bit(trans, CSR_GP_CNTRL,
-		      CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
+		      BIT(trans->cfg->csr->flag_init_done));
 
 	iwl_pcie_enable_rx_wake(trans, false);
 
@@ -1543,15 +1551,17 @@ static int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
 		iwl_pcie_reset_ict(trans);
 	iwl_enable_interrupts(trans);
 
-	iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
-	iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
+	iwl_set_bit(trans, CSR_GP_CNTRL,
+		    BIT(trans->cfg->csr->flag_mac_access_req));
+	iwl_set_bit(trans, CSR_GP_CNTRL,
+		    BIT(trans->cfg->csr->flag_init_done));
 
 	if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000)
 		udelay(2);
 
 	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
-			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
-			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
+			   BIT(trans->cfg->csr->flag_mac_clock_ready),
+			   BIT(trans->cfg->csr->flag_mac_clock_ready),
 			   25000);
 	if (ret < 0) {
 		IWL_ERR(trans, "Failed to resume the device (mac ready)\n");
@@ -1562,7 +1572,7 @@ static int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
 
 	if (!reset) {
 		iwl_clear_bit(trans, CSR_GP_CNTRL,
-			      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+			      BIT(trans->cfg->csr->flag_mac_access_req));
 	} else {
 		iwl_trans_pcie_tx_reset(trans);
 
@@ -1590,14 +1600,13 @@ static void iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
 					struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	int max_irqs, num_irqs, i, ret, nr_online_cpus;
+	int max_irqs, num_irqs, i, ret;
 	u16 pci_cmd;
 
 	if (!trans->cfg->mq_rx_supported)
 		goto enable_msi;
 
-	nr_online_cpus = num_online_cpus();
-	max_irqs = min_t(u32, nr_online_cpus + 2, IWL_MAX_RX_HW_QUEUES);
+	max_irqs = min_t(u32, num_online_cpus() + 2, IWL_MAX_RX_HW_QUEUES);
 	for (i = 0; i < max_irqs; i++)
 		trans_pcie->msix_entries[i].entry = i;
 
@@ -1623,16 +1632,17 @@ static void iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
 	 * Two interrupts less: non rx causes shared with FBQ and RSS.
 	 * More than two interrupts: we will use fewer RSS queues.
 	 */
-	if (num_irqs <= nr_online_cpus) {
+	if (num_irqs <= max_irqs - 2) {
 		trans_pcie->trans->num_rx_queues = num_irqs + 1;
 		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX |
 			IWL_SHARED_IRQ_FIRST_RSS;
-	} else if (num_irqs == nr_online_cpus + 1) {
+	} else if (num_irqs == max_irqs - 1) {
 		trans_pcie->trans->num_rx_queues = num_irqs;
 		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX;
 	} else {
 		trans_pcie->trans->num_rx_queues = num_irqs - 1;
 	}
+	WARN_ON(trans_pcie->trans->num_rx_queues > IWL_MAX_RX_HW_QUEUES);
 
 	trans_pcie->alloc_vecs = num_irqs;
 	trans_pcie->msix_enabled = true;
@@ -1926,6 +1936,29 @@ static void iwl_trans_pcie_set_pmi(struct iwl_trans *trans, bool state)
 		clear_bit(STATUS_TPOWER_PMI, &trans->status);
 }
 
+struct iwl_trans_pcie_removal {
+	struct pci_dev *pdev;
+	struct work_struct work;
+};
+
+static void iwl_trans_pcie_removal_wk(struct work_struct *wk)
+{
+	struct iwl_trans_pcie_removal *removal =
+		container_of(wk, struct iwl_trans_pcie_removal, work);
+	struct pci_dev *pdev = removal->pdev;
+	char *prop[] = {"EVENT=INACCESSIBLE", NULL};
+
+	dev_err(&pdev->dev, "Device gone - attempting removal\n");
+	kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, prop);
+	pci_lock_rescan_remove();
+	pci_dev_put(pdev);
+	pci_stop_and_remove_bus_device(pdev);
+	pci_unlock_rescan_remove();
+
+	kfree(removal);
+	module_put(THIS_MODULE);
+}
+
 static bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans,
 					   unsigned long *flags)
 {
@@ -1939,7 +1972,7 @@ static bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans,
 
 	/* this bit wakes up the NIC */
 	__iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
-				 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+				 BIT(trans->cfg->csr->flag_mac_access_req));
 	if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000)
 		udelay(2);
 
@@ -1964,15 +1997,59 @@ static bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans,
 	 * and do not save/restore SRAM when power cycling.
 	 */
 	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
-			   CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN,
-			   (CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY |
+			   BIT(trans->cfg->csr->flag_val_mac_access_en),
+			   (BIT(trans->cfg->csr->flag_mac_clock_ready) |
 			    CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP), 15000);
 	if (unlikely(ret < 0)) {
-		iwl_trans_pcie_dump_regs(trans);
-		iwl_write32(trans, CSR_RESET, CSR_RESET_REG_FLAG_FORCE_NMI);
+		u32 cntrl = iwl_read32(trans, CSR_GP_CNTRL);
+
 		WARN_ONCE(1,
 			  "Timeout waiting for hardware access (CSR_GP_CNTRL 0x%08x)\n",
-			  iwl_read32(trans, CSR_GP_CNTRL));
+			  cntrl);
+
+		iwl_trans_pcie_dump_regs(trans);
+
+		if (iwlwifi_mod_params.remove_when_gone && cntrl == ~0U) {
+			struct iwl_trans_pcie_removal *removal;
+
+			if (trans_pcie->scheduled_for_removal)
+				goto err;
+
+			IWL_ERR(trans, "Device gone - scheduling removal!\n");
+
+			/*
+			 * get a module reference to avoid doing this
+			 * while unloading anyway and to avoid
+			 * scheduling a work with code that's being
+			 * removed.
+			 */
+			if (!try_module_get(THIS_MODULE)) {
+				IWL_ERR(trans,
+					"Module is being unloaded - abort\n");
+				goto err;
+			}
+
+			removal = kzalloc(sizeof(*removal), GFP_ATOMIC);
+			if (!removal) {
+				module_put(THIS_MODULE);
+				goto err;
+			}
+			/*
+			 * we don't need to clear this flag, because
+			 * the trans will be freed and reallocated.
+			*/
+			trans_pcie->scheduled_for_removal = true;
+
+			removal->pdev = to_pci_dev(trans->dev);
+			INIT_WORK(&removal->work, iwl_trans_pcie_removal_wk);
+			pci_dev_get(removal->pdev);
+			schedule_work(&removal->work);
+		} else {
+			iwl_write32(trans, CSR_RESET,
+				    CSR_RESET_REG_FLAG_FORCE_NMI);
+		}
+
+err:
 		spin_unlock_irqrestore(&trans_pcie->reg_lock, *flags);
 		return false;
 	}
@@ -2003,7 +2080,7 @@ static void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans,
 		goto out;
 
 	__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
-				   CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+				   BIT(trans->cfg->csr->flag_mac_access_req));
 	/*
 	 * Above we read the CSR_GP_CNTRL register, which will flush
 	 * any previous writes, but we need the write that clears the
@@ -3232,12 +3309,12 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 		 * id located at the AUX bus MISC address space.
 		 */
 		iwl_set_bit(trans, CSR_GP_CNTRL,
-			    CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
+			    BIT(trans->cfg->csr->flag_init_done));
 		udelay(2);
 
 		ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
-				   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
-				   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
+				   BIT(trans->cfg->csr->flag_mac_clock_ready),
+				   BIT(trans->cfg->csr->flag_mac_clock_ready),
 				   25000);
 		if (ret < 0) {
 			IWL_DEBUG_INFO(trans, "Failed to wake up the nic\n");
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c
index fabae0f60683..48890a1c825f 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c
@@ -488,6 +488,23 @@ int iwl_trans_pcie_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb,
 
 	spin_lock(&txq->lock);
 
+	if (iwl_queue_space(txq) < txq->high_mark) {
+		iwl_stop_queue(trans, txq);
+
+		/* don't put the packet on the ring, if there is no room */
+		if (unlikely(iwl_queue_space(txq) < 3)) {
+			struct iwl_device_cmd **dev_cmd_ptr;
+
+			dev_cmd_ptr = (void *)((u8 *)skb->cb +
+					       trans_pcie->dev_cmd_offs);
+
+			*dev_cmd_ptr = dev_cmd;
+			__skb_queue_tail(&txq->overflow_q, skb);
+			spin_unlock(&txq->lock);
+			return 0;
+		}
+	}
+
 	idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr);
 
 	/* Set up driver data for this TFD */
@@ -523,9 +540,6 @@ int iwl_trans_pcie_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb,
 	/* Tell device the write index *just past* this latest filled TFD */
 	txq->write_ptr = iwl_queue_inc_wrap(txq->write_ptr);
 	iwl_pcie_gen2_txq_inc_wr_ptr(trans, txq);
-	if (iwl_queue_space(txq) < txq->high_mark)
-		iwl_stop_queue(trans, txq);
-
 	/*
 	 * At this point the frame is "transmitted" successfully
 	 * and we will get a TX status notification eventually.
@@ -555,15 +569,13 @@ static int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
 	unsigned long flags;
 	void *dup_buf = NULL;
 	dma_addr_t phys_addr;
-	int i, cmd_pos, idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr);
+	int i, cmd_pos, idx;
 	u16 copy_size, cmd_size, tb0_size;
 	bool had_nocopy = false;
 	u8 group_id = iwl_cmd_groupid(cmd->id);
 	const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD];
 	u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD];
-	struct iwl_tfh_tfd *tfd = iwl_pcie_get_tfd(trans, txq, txq->write_ptr);
-
-	memset(tfd, 0, sizeof(*tfd));
+	struct iwl_tfh_tfd *tfd;
 
 	copy_size = sizeof(struct iwl_cmd_header_wide);
 	cmd_size = sizeof(struct iwl_cmd_header_wide);
@@ -634,6 +646,10 @@ static int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
 
 	spin_lock_bh(&txq->lock);
 
+	idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr);
+	tfd = iwl_pcie_get_tfd(trans, txq, txq->write_ptr);
+	memset(tfd, 0, sizeof(*tfd));
+
 	if (iwl_queue_space(txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) {
 		spin_unlock_bh(&txq->lock);
 
@@ -957,6 +973,13 @@ void iwl_pcie_gen2_txq_unmap(struct iwl_trans *trans, int txq_id)
 			spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
 		}
 	}
+
+	while (!skb_queue_empty(&txq->overflow_q)) {
+		struct sk_buff *skb = __skb_dequeue(&txq->overflow_q);
+
+		iwl_op_mode_free_skb(trans->op_mode, skb);
+	}
+
 	spin_unlock_bh(&txq->lock);
 
 	/* just in case - this queue may have been stopped */
@@ -972,7 +995,7 @@ static void iwl_pcie_gen2_txq_free_memory(struct iwl_trans *trans,
 	/* De-alloc circular buffer of TFDs */
 	if (txq->tfds) {
 		dma_free_coherent(dev,
-				  trans_pcie->tfd_size * TFD_QUEUE_SIZE_MAX,
+				  trans_pcie->tfd_size * txq->n_window,
 				  txq->tfds, txq->dma_addr);
 		dma_free_coherent(dev,
 				  sizeof(*txq->first_tb_bufs) * txq->n_window,
@@ -1020,7 +1043,7 @@ static void iwl_pcie_gen2_txq_free(struct iwl_trans *trans, int txq_id)
 
 int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans *trans,
 				 struct iwl_tx_queue_cfg_cmd *cmd,
-				 int cmd_id,
+				 int cmd_id, int size,
 				 unsigned int timeout)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
@@ -1046,12 +1069,12 @@ int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans *trans,
 		return -ENOMEM;
 	}
 
-	ret = iwl_pcie_txq_alloc(trans, txq, TFD_TX_CMD_SLOTS, false);
+	ret = iwl_pcie_txq_alloc(trans, txq, size, false);
 	if (ret) {
 		IWL_ERR(trans, "Tx queue alloc failed\n");
 		goto error;
 	}
-	ret = iwl_pcie_txq_init(trans, txq, TFD_TX_CMD_SLOTS, false);
+	ret = iwl_pcie_txq_init(trans, txq, size, false);
 	if (ret) {
 		IWL_ERR(trans, "Tx queue init failed\n");
 		goto error;
@@ -1061,7 +1084,7 @@ int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans *trans,
 
 	cmd->tfdq_addr = cpu_to_le64(txq->dma_addr);
 	cmd->byte_cnt_addr = cpu_to_le64(txq->bc_tbl.dma);
-	cmd->cb_size = cpu_to_le32(TFD_QUEUE_CB_SIZE(TFD_TX_CMD_SLOTS));
+	cmd->cb_size = cpu_to_le32(TFD_QUEUE_CB_SIZE(size));
 
 	ret = iwl_trans_send_cmd(trans, &hcmd);
 	if (ret)
@@ -1152,8 +1175,6 @@ int iwl_pcie_gen2_tx_init(struct iwl_trans *trans)
 	struct iwl_txq *cmd_queue;
 	int txq_id = trans_pcie->cmd_queue, ret;
 
-	iwl_pcie_set_tx_cmd_queue_size(trans);
-
 	/* alloc and init the command queue */
 	if (!trans_pcie->txq[txq_id]) {
 		cmd_queue = kzalloc(sizeof(*cmd_queue), GFP_KERNEL);
@@ -1162,8 +1183,7 @@ int iwl_pcie_gen2_tx_init(struct iwl_trans *trans)
 			return -ENOMEM;
 		}
 		trans_pcie->txq[txq_id] = cmd_queue;
-		ret = iwl_pcie_txq_alloc(trans, cmd_queue,
-					 trans_pcie->tx_cmd_queue_size, true);
+		ret = iwl_pcie_txq_alloc(trans, cmd_queue, TFD_CMD_SLOTS, true);
 		if (ret) {
 			IWL_ERR(trans, "Tx %d queue init failed\n", txq_id);
 			goto error;
@@ -1172,8 +1192,7 @@ int iwl_pcie_gen2_tx_init(struct iwl_trans *trans)
 		cmd_queue = trans_pcie->txq[txq_id];
 	}
 
-	ret = iwl_pcie_txq_init(trans, cmd_queue,
-				trans_pcie->tx_cmd_queue_size, true);
+	ret = iwl_pcie_txq_init(trans, cmd_queue, TFD_CMD_SLOTS, true);
 	if (ret) {
 		IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id);
 		goto error;
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
index 1a566287993d..473fe7ccb07c 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
@@ -3,6 +3,7 @@
  * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * Portions of this file are derived from the ipw3945 project, as well
  * as portions of the ieee80211 subsystem header files.
@@ -273,7 +274,7 @@ static void iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans,
 			IWL_DEBUG_INFO(trans, "Tx queue %d requesting wakeup, GP1 = 0x%x\n",
 				       txq_id, reg);
 			iwl_set_bit(trans, CSR_GP_CNTRL,
-				    CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+				    BIT(trans->cfg->csr->flag_mac_access_req));
 			txq->need_update = true;
 			return;
 		}
@@ -495,6 +496,9 @@ int iwl_pcie_txq_alloc(struct iwl_trans *trans, struct iwl_txq *txq,
 	if (WARN_ON(txq->entries || txq->tfds))
 		return -EINVAL;
 
+	if (trans->cfg->use_tfh)
+		tfd_sz = trans_pcie->tfd_size * slots_num;
+
 	timer_setup(&txq->stuck_timer, iwl_pcie_txq_stuck_timer, 0);
 	txq->trans_pcie = trans_pcie;
 
@@ -608,7 +612,7 @@ static void iwl_pcie_clear_cmd_in_flight(struct iwl_trans *trans)
 
 	trans_pcie->cmd_hold_nic_awake = false;
 	__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
-				   CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+				   BIT(trans->cfg->csr->flag_mac_access_req));
 }
 
 /*
@@ -950,8 +954,7 @@ static int iwl_pcie_tx_alloc(struct iwl_trans *trans)
 	     txq_id++) {
 		bool cmd_queue = (txq_id == trans_pcie->cmd_queue);
 
-		slots_num = cmd_queue ? trans_pcie->tx_cmd_queue_size :
-			TFD_TX_CMD_SLOTS;
+		slots_num = cmd_queue ? TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS;
 		trans_pcie->txq[txq_id] = &trans_pcie->txq_memory[txq_id];
 		ret = iwl_pcie_txq_alloc(trans, trans_pcie->txq[txq_id],
 					 slots_num, cmd_queue);
@@ -970,21 +973,6 @@ static int iwl_pcie_tx_alloc(struct iwl_trans *trans)
 	return ret;
 }
 
-void iwl_pcie_set_tx_cmd_queue_size(struct iwl_trans *trans)
-{
-	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	int queue_size = TFD_CMD_SLOTS;
-
-	if (trans->cfg->tx_cmd_queue_size)
-		queue_size = trans->cfg->tx_cmd_queue_size;
-
-	if (WARN_ON(!(is_power_of_2(queue_size) &&
-		      TFD_QUEUE_CB_SIZE(queue_size) > 0)))
-		trans_pcie->tx_cmd_queue_size = TFD_CMD_SLOTS;
-	else
-		trans_pcie->tx_cmd_queue_size = queue_size;
-}
-
 int iwl_pcie_tx_init(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
@@ -992,8 +980,6 @@ int iwl_pcie_tx_init(struct iwl_trans *trans)
 	int txq_id, slots_num;
 	bool alloc = false;
 
-	iwl_pcie_set_tx_cmd_queue_size(trans);
-
 	if (!trans_pcie->txq_memory) {
 		ret = iwl_pcie_tx_alloc(trans);
 		if (ret)
@@ -1017,8 +1003,7 @@ int iwl_pcie_tx_init(struct iwl_trans *trans)
 	     txq_id++) {
 		bool cmd_queue = (txq_id == trans_pcie->cmd_queue);
 
-		slots_num = cmd_queue ? trans_pcie->tx_cmd_queue_size :
-			TFD_TX_CMD_SLOTS;
+		slots_num = cmd_queue ? TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS;
 		ret = iwl_pcie_txq_init(trans, trans_pcie->txq[txq_id],
 					slots_num, cmd_queue);
 		if (ret) {
@@ -1166,7 +1151,7 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 			 * In that case, iwl_queue_space will be small again
 			 * and we won't wake mac80211's queue.
 			 */
-			iwl_trans_pcie_tx(trans, skb, dev_cmd_ptr, txq_id);
+			iwl_trans_tx(trans, skb, dev_cmd_ptr, txq_id);
 		}
 		spin_lock_bh(&txq->lock);
 
@@ -1187,6 +1172,7 @@ static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans,
 				      const struct iwl_host_cmd *cmd)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+	const struct iwl_cfg *cfg = trans->cfg;
 	int ret;
 
 	lockdep_assert_held(&trans_pcie->reg_lock);
@@ -1204,19 +1190,19 @@ static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans,
 	 * returned. This needs to be done only on NICs that have
 	 * apmg_wake_up_wa set.
 	 */
-	if (trans->cfg->base_params->apmg_wake_up_wa &&
+	if (cfg->base_params->apmg_wake_up_wa &&
 	    !trans_pcie->cmd_hold_nic_awake) {
 		__iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
-					 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+					 BIT(cfg->csr->flag_mac_access_req));
 
 		ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
-				   CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN,
-				   (CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY |
+				   BIT(cfg->csr->flag_val_mac_access_en),
+				   (BIT(cfg->csr->flag_mac_clock_ready) |
 				    CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP),
 				   15000);
 		if (ret < 0) {
 			__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
-					CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
+					BIT(cfg->csr->flag_mac_access_req));
 			IWL_ERR(trans, "Failed to wake NIC for hcmd\n");
 			return -EIO;
 		}
@@ -2411,7 +2397,13 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 		goto out_err;
 	iwl_pcie_txq_build_tfd(trans, txq, tb1_phys, tb1_len, false);
 
-	if (amsdu) {
+	/*
+	 * If gso_size wasn't set, don't give the frame "amsdu treatment"
+	 * (adding subframes, etc.).
+	 * This can happen in some testing flows when the amsdu was already
+	 * pre-built, and we just need to send the resulting skb.
+	 */
+	if (amsdu && skb_shinfo(skb)->gso_size) {
 		if (unlikely(iwl_fill_data_tbs_amsdu(trans, skb, txq, hdr_len,
 						     out_meta, dev_cmd,
 						     tb1_len)))
diff --git a/drivers/net/wireless/intersil/hostap/hostap_ap.c b/drivers/net/wireless/intersil/hostap/hostap_ap.c
index b4dfe1893d18..d1884b8913e7 100644
--- a/drivers/net/wireless/intersil/hostap/hostap_ap.c
+++ b/drivers/net/wireless/intersil/hostap/hostap_ap.c
@@ -69,7 +69,7 @@ static void prism2_send_mgmt(struct net_device *dev,
 #ifndef PRISM2_NO_PROCFS_DEBUG
 static int ap_debug_proc_show(struct seq_file *m, void *v)
 {
-	struct ap_data *ap = m->private;
+	struct ap_data *ap = PDE_DATA(file_inode(m->file));
 
 	seq_printf(m, "BridgedUnicastFrames=%u\n", ap->bridged_unicast);
 	seq_printf(m, "BridgedMulticastFrames=%u\n", ap->bridged_multicast);
@@ -81,18 +81,6 @@ static int ap_debug_proc_show(struct seq_file *m, void *v)
 	seq_printf(m, "tx_drop_nonassoc=%u\n", ap->tx_drop_nonassoc);
 	return 0;
 }
-
-static int ap_debug_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ap_debug_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations ap_debug_proc_fops = {
-	.open		= ap_debug_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
 #endif /* PRISM2_NO_PROCFS_DEBUG */
 
 
@@ -333,7 +321,7 @@ void hostap_deauth_all_stas(struct net_device *dev, struct ap_data *ap,
 
 static int ap_control_proc_show(struct seq_file *m, void *v)
 {
-	struct ap_data *ap = m->private;
+	struct ap_data *ap = PDE_DATA(file_inode(m->file));
 	char *policy_txt;
 	struct mac_entry *entry;
 
@@ -365,20 +353,20 @@ static int ap_control_proc_show(struct seq_file *m, void *v)
 
 static void *ap_control_proc_start(struct seq_file *m, loff_t *_pos)
 {
-	struct ap_data *ap = m->private;
+	struct ap_data *ap = PDE_DATA(file_inode(m->file));
 	spin_lock_bh(&ap->mac_restrictions.lock);
 	return seq_list_start_head(&ap->mac_restrictions.mac_list, *_pos);
 }
 
 static void *ap_control_proc_next(struct seq_file *m, void *v, loff_t *_pos)
 {
-	struct ap_data *ap = m->private;
+	struct ap_data *ap = PDE_DATA(file_inode(m->file));
 	return seq_list_next(v, &ap->mac_restrictions.mac_list, _pos);
 }
 
 static void ap_control_proc_stop(struct seq_file *m, void *v)
 {
-	struct ap_data *ap = m->private;
+	struct ap_data *ap = PDE_DATA(file_inode(m->file));
 	spin_unlock_bh(&ap->mac_restrictions.lock);
 }
 
@@ -389,24 +377,6 @@ static const struct seq_operations ap_control_proc_seqops = {
 	.show	= ap_control_proc_show,
 };
 
-static int ap_control_proc_open(struct inode *inode, struct file *file)
-{
-	int ret = seq_open(file, &ap_control_proc_seqops);
-	if (ret == 0) {
-		struct seq_file *m = file->private_data;
-		m->private = PDE_DATA(inode);
-	}
-	return ret;
-}
-
-static const struct file_operations ap_control_proc_fops = {
-	.open		= ap_control_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
-
 int ap_control_add_mac(struct mac_restrictions *mac_restrictions, u8 *mac)
 {
 	struct mac_entry *entry;
@@ -585,20 +555,20 @@ static int prism2_ap_proc_show(struct seq_file *m, void *v)
 
 static void *prism2_ap_proc_start(struct seq_file *m, loff_t *_pos)
 {
-	struct ap_data *ap = m->private;
+	struct ap_data *ap = PDE_DATA(file_inode(m->file));
 	spin_lock_bh(&ap->sta_table_lock);
 	return seq_list_start_head(&ap->sta_list, *_pos);
 }
 
 static void *prism2_ap_proc_next(struct seq_file *m, void *v, loff_t *_pos)
 {
-	struct ap_data *ap = m->private;
+	struct ap_data *ap = PDE_DATA(file_inode(m->file));
 	return seq_list_next(v, &ap->sta_list, _pos);
 }
 
 static void prism2_ap_proc_stop(struct seq_file *m, void *v)
 {
-	struct ap_data *ap = m->private;
+	struct ap_data *ap = PDE_DATA(file_inode(m->file));
 	spin_unlock_bh(&ap->sta_table_lock);
 }
 
@@ -608,23 +578,6 @@ static const struct seq_operations prism2_ap_proc_seqops = {
 	.stop	= prism2_ap_proc_stop,
 	.show	= prism2_ap_proc_show,
 };
-
-static int prism2_ap_proc_open(struct inode *inode, struct file *file)
-{
-	int ret = seq_open(file, &prism2_ap_proc_seqops);
-	if (ret == 0) {
-		struct seq_file *m = file->private_data;
-		m->private = PDE_DATA(inode);
-	}
-	return ret;
-}
-
-static const struct file_operations prism2_ap_proc_fops = {
-	.open		= prism2_ap_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
 #endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
 
 
@@ -896,12 +849,13 @@ void hostap_init_ap_proc(local_info_t *local)
 		return;
 
 #ifndef PRISM2_NO_PROCFS_DEBUG
-	proc_create_data("ap_debug", 0, ap->proc, &ap_debug_proc_fops, ap);
+	proc_create_single_data("ap_debug", 0, ap->proc, ap_debug_proc_show, ap);
 #endif /* PRISM2_NO_PROCFS_DEBUG */
 
 #ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	proc_create_data("ap_control", 0, ap->proc, &ap_control_proc_fops, ap);
-	proc_create_data("ap", 0, ap->proc, &prism2_ap_proc_fops, ap);
+	proc_create_seq_data("ap_control", 0, ap->proc, &ap_control_proc_seqops,
+			ap);
+	proc_create_seq_data("ap", 0, ap->proc, &prism2_ap_proc_seqops, ap);
 #endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
 
 }
@@ -1106,18 +1060,6 @@ static int prism2_sta_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int prism2_sta_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, prism2_sta_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations prism2_sta_proc_fops = {
-	.open		= prism2_sta_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static void handle_add_proc_queue(struct work_struct *work)
 {
 	struct ap_data *ap = container_of(work, struct ap_data,
@@ -1138,9 +1080,9 @@ static void handle_add_proc_queue(struct work_struct *work)
 
 		if (sta) {
 			sprintf(name, "%pM", sta->addr);
-			sta->proc = proc_create_data(
+			sta->proc = proc_create_single_data(
 				name, 0, ap->proc,
-				&prism2_sta_proc_fops, sta);
+				prism2_sta_proc_show, sta);
 
 			atomic_dec(&sta->users);
 		}
diff --git a/drivers/net/wireless/intersil/hostap/hostap_hw.c b/drivers/net/wireless/intersil/hostap/hostap_hw.c
index 5c4a17a18968..2720aa39f530 100644
--- a/drivers/net/wireless/intersil/hostap/hostap_hw.c
+++ b/drivers/net/wireless/intersil/hostap/hostap_hw.c
@@ -2951,19 +2951,6 @@ static int prism2_registers_proc_show(struct seq_file *m, void *v)
 
 	return 0;
 }
-
-static int prism2_registers_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, prism2_registers_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations prism2_registers_proc_fops = {
-	.open		= prism2_registers_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 #endif /* PRISM2_NO_PROCFS_DEBUG */
 
 
@@ -3279,8 +3266,8 @@ static int hostap_hw_ready(struct net_device *dev)
 		}
 		hostap_init_proc(local);
 #ifndef PRISM2_NO_PROCFS_DEBUG
-		proc_create_data("registers", 0, local->proc,
-				 &prism2_registers_proc_fops, local);
+		proc_create_single_data("registers", 0, local->proc,
+				 prism2_registers_proc_show, local);
 #endif /* PRISM2_NO_PROCFS_DEBUG */
 		hostap_init_ap_proc(local);
 		return 0;
diff --git a/drivers/net/wireless/intersil/hostap/hostap_proc.c b/drivers/net/wireless/intersil/hostap/hostap_proc.c
index d234231bf532..5b33ccab9188 100644
--- a/drivers/net/wireless/intersil/hostap/hostap_proc.c
+++ b/drivers/net/wireless/intersil/hostap/hostap_proc.c
@@ -43,18 +43,6 @@ static int prism2_debug_proc_show(struct seq_file *m, void *v)
 
 	return 0;
 }
-
-static int prism2_debug_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, prism2_debug_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations prism2_debug_proc_fops = {
-	.open		= prism2_debug_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
 #endif /* PRISM2_NO_PROCFS_DEBUG */
 
 
@@ -95,19 +83,6 @@ static int prism2_stats_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int prism2_stats_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, prism2_stats_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations prism2_stats_proc_fops = {
-	.open		= prism2_stats_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-
 static int prism2_wds_proc_show(struct seq_file *m, void *v)
 {
 	struct list_head *ptr = v;
@@ -122,20 +97,20 @@ static int prism2_wds_proc_show(struct seq_file *m, void *v)
 
 static void *prism2_wds_proc_start(struct seq_file *m, loff_t *_pos)
 {
-	local_info_t *local = m->private;
+	local_info_t *local = PDE_DATA(file_inode(m->file));
 	read_lock_bh(&local->iface_lock);
 	return seq_list_start(&local->hostap_interfaces, *_pos);
 }
 
 static void *prism2_wds_proc_next(struct seq_file *m, void *v, loff_t *_pos)
 {
-	local_info_t *local = m->private;
+	local_info_t *local = PDE_DATA(file_inode(m->file));
 	return seq_list_next(v, &local->hostap_interfaces, _pos);
 }
 
 static void prism2_wds_proc_stop(struct seq_file *m, void *v)
 {
-	local_info_t *local = m->private;
+	local_info_t *local = PDE_DATA(file_inode(m->file));
 	read_unlock_bh(&local->iface_lock);
 }
 
@@ -146,27 +121,9 @@ static const struct seq_operations prism2_wds_proc_seqops = {
 	.show	= prism2_wds_proc_show,
 };
 
-static int prism2_wds_proc_open(struct inode *inode, struct file *file)
-{
-	int ret = seq_open(file, &prism2_wds_proc_seqops);
-	if (ret == 0) {
-		struct seq_file *m = file->private_data;
-		m->private = PDE_DATA(inode);
-	}
-	return ret;
-}
-
-static const struct file_operations prism2_wds_proc_fops = {
-	.open		= prism2_wds_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
-
 static int prism2_bss_list_proc_show(struct seq_file *m, void *v)
 {
-	local_info_t *local = m->private;
+	local_info_t *local = PDE_DATA(file_inode(m->file));
 	struct list_head *ptr = v;
 	struct hostap_bss_info *bss;
 
@@ -193,20 +150,20 @@ static int prism2_bss_list_proc_show(struct seq_file *m, void *v)
 
 static void *prism2_bss_list_proc_start(struct seq_file *m, loff_t *_pos)
 {
-	local_info_t *local = m->private;
+	local_info_t *local = PDE_DATA(file_inode(m->file));
 	spin_lock_bh(&local->lock);
 	return seq_list_start_head(&local->bss_list, *_pos);
 }
 
 static void *prism2_bss_list_proc_next(struct seq_file *m, void *v, loff_t *_pos)
 {
-	local_info_t *local = m->private;
+	local_info_t *local = PDE_DATA(file_inode(m->file));
 	return seq_list_next(v, &local->bss_list, _pos);
 }
 
 static void prism2_bss_list_proc_stop(struct seq_file *m, void *v)
 {
-	local_info_t *local = m->private;
+	local_info_t *local = PDE_DATA(file_inode(m->file));
 	spin_unlock_bh(&local->lock);
 }
 
@@ -217,24 +174,6 @@ static const struct seq_operations prism2_bss_list_proc_seqops = {
 	.show	= prism2_bss_list_proc_show,
 };
 
-static int prism2_bss_list_proc_open(struct inode *inode, struct file *file)
-{
-	int ret = seq_open(file, &prism2_bss_list_proc_seqops);
-	if (ret == 0) {
-		struct seq_file *m = file->private_data;
-		m->private = PDE_DATA(inode);
-	}
-	return ret;
-}
-
-static const struct file_operations prism2_bss_list_proc_fops = {
-	.open		= prism2_bss_list_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
-
 static int prism2_crypt_proc_show(struct seq_file *m, void *v)
 {
 	local_info_t *local = m->private;
@@ -252,19 +191,6 @@ static int prism2_crypt_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int prism2_crypt_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, prism2_crypt_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations prism2_crypt_proc_fops = {
-	.open		= prism2_crypt_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-
 static ssize_t prism2_pda_proc_read(struct file *file, char __user *buf,
 				    size_t count, loff_t *_pos)
 {
@@ -342,7 +268,7 @@ static int prism2_io_debug_proc_read(char *page, char **start, off_t off,
 #ifndef PRISM2_NO_STATION_MODES
 static int prism2_scan_results_proc_show(struct seq_file *m, void *v)
 {
-	local_info_t *local = m->private;
+	local_info_t *local = PDE_DATA(file_inode(m->file));
 	unsigned long entry;
 	int i, len;
 	struct hfa384x_hostscan_result *scanres;
@@ -392,7 +318,7 @@ static int prism2_scan_results_proc_show(struct seq_file *m, void *v)
 
 static void *prism2_scan_results_proc_start(struct seq_file *m, loff_t *_pos)
 {
-	local_info_t *local = m->private;
+	local_info_t *local = PDE_DATA(file_inode(m->file));
 	spin_lock_bh(&local->lock);
 
 	/* We have a header (pos 0) + N results to show (pos 1...N) */
@@ -403,7 +329,7 @@ static void *prism2_scan_results_proc_start(struct seq_file *m, loff_t *_pos)
 
 static void *prism2_scan_results_proc_next(struct seq_file *m, void *v, loff_t *_pos)
 {
-	local_info_t *local = m->private;
+	local_info_t *local = PDE_DATA(file_inode(m->file));
 
 	++*_pos;
 	if (*_pos > local->last_scan_results_count)
@@ -413,7 +339,7 @@ static void *prism2_scan_results_proc_next(struct seq_file *m, void *v, loff_t *
 
 static void prism2_scan_results_proc_stop(struct seq_file *m, void *v)
 {
-	local_info_t *local = m->private;
+	local_info_t *local = PDE_DATA(file_inode(m->file));
 	spin_unlock_bh(&local->lock);
 }
 
@@ -423,25 +349,6 @@ static const struct seq_operations prism2_scan_results_proc_seqops = {
 	.stop	= prism2_scan_results_proc_stop,
 	.show	= prism2_scan_results_proc_show,
 };
-
-static int prism2_scan_results_proc_open(struct inode *inode, struct file *file)
-{
-	int ret = seq_open(file, &prism2_scan_results_proc_seqops);
-	if (ret == 0) {
-		struct seq_file *m = file->private_data;
-		m->private = PDE_DATA(inode);
-	}
-	return ret;
-}
-
-static const struct file_operations prism2_scan_results_proc_fops = {
-	.open		= prism2_scan_results_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
-
 #endif /* PRISM2_NO_STATION_MODES */
 
 
@@ -463,29 +370,29 @@ void hostap_init_proc(local_info_t *local)
 	}
 
 #ifndef PRISM2_NO_PROCFS_DEBUG
-	proc_create_data("debug", 0, local->proc,
-			 &prism2_debug_proc_fops, local);
+	proc_create_single_data("debug", 0, local->proc,
+			prism2_debug_proc_show, local);
 #endif /* PRISM2_NO_PROCFS_DEBUG */
-	proc_create_data("stats", 0, local->proc,
-			 &prism2_stats_proc_fops, local);
-	proc_create_data("wds", 0, local->proc,
-			 &prism2_wds_proc_fops, local);
+	proc_create_single_data("stats", 0, local->proc, prism2_stats_proc_show,
+			local);
+	proc_create_seq_data("wds", 0, local->proc,
+			&prism2_wds_proc_seqops, local);
 	proc_create_data("pda", 0, local->proc,
 			 &prism2_pda_proc_fops, local);
 	proc_create_data("aux_dump", 0, local->proc,
 			 local->func->read_aux_fops ?: &prism2_aux_dump_proc_fops,
 			 local);
-	proc_create_data("bss_list", 0, local->proc,
-			 &prism2_bss_list_proc_fops, local);
-	proc_create_data("crypt", 0, local->proc,
-			 &prism2_crypt_proc_fops, local);
+	proc_create_seq_data("bss_list", 0, local->proc,
+			&prism2_bss_list_proc_seqops, local);
+	proc_create_single_data("crypt", 0, local->proc, prism2_crypt_proc_show,
+		local);
 #ifdef PRISM2_IO_DEBUG
-	proc_create_data("io_debug", 0, local->proc,
-			 &prism2_io_debug_proc_fops, local);
+	proc_create_single_data("io_debug", 0, local->proc,
+			prism2_debug_proc_show, local);
 #endif /* PRISM2_IO_DEBUG */
 #ifndef PRISM2_NO_STATION_MODES
-	proc_create_data("scan_results", 0, local->proc,
-			 &prism2_scan_results_proc_fops, local);
+	proc_create_seq_data("scan_results", 0, local->proc,
+			&prism2_scan_results_proc_seqops, local);
 #endif /* PRISM2_NO_STATION_MODES */
 }
 
diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c
index 920c23e542a5..9825bfd42abc 100644
--- a/drivers/net/wireless/mac80211_hwsim.c
+++ b/drivers/net/wireless/mac80211_hwsim.c
@@ -2514,7 +2514,6 @@ static void hwsim_mcast_new_radio(int id, struct genl_info *info,
 	return;
 
 out_err:
-	genlmsg_cancel(mcast_skb, data);
 	nlmsg_free(mcast_skb);
 }
 
@@ -2650,6 +2649,7 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 	ieee80211_hw_set(hw, AMPDU_AGGREGATION);
 	ieee80211_hw_set(hw, MFP_CAPABLE);
 	ieee80211_hw_set(hw, SIGNAL_DBM);
+	ieee80211_hw_set(hw, SUPPORTS_PS);
 	ieee80211_hw_set(hw, TDLS_WIDER_BW);
 	if (rctbl)
 		ieee80211_hw_set(hw, SUPPORTS_RC_TABLE);
diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
index 7f7e9de2db1c..a67e2d66ac9d 100644
--- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c
+++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
@@ -929,7 +929,7 @@ mwifiex_init_new_priv_params(struct mwifiex_private *priv,
 	adapter->rx_locked = false;
 	spin_unlock_irqrestore(&adapter->rx_proc_lock, flags);
 
-	mwifiex_set_mac_address(priv, dev);
+	mwifiex_set_mac_address(priv, dev, false, NULL);
 
 	return 0;
 }
@@ -1158,6 +1158,7 @@ mwifiex_cfg80211_change_virtual_intf(struct wiphy *wiphy,
 		case NL80211_IFTYPE_UNSPECIFIED:
 			mwifiex_dbg(priv->adapter, INFO,
 				    "%s: kept type as IBSS\n", dev->name);
+			/* fall through */
 		case NL80211_IFTYPE_ADHOC:	/* This shouldn't happen */
 			return 0;
 		default:
@@ -1188,6 +1189,7 @@ mwifiex_cfg80211_change_virtual_intf(struct wiphy *wiphy,
 		case NL80211_IFTYPE_UNSPECIFIED:
 			mwifiex_dbg(priv->adapter, INFO,
 				    "%s: kept type as STA\n", dev->name);
+			/* fall through */
 		case NL80211_IFTYPE_STATION:	/* This shouldn't happen */
 			return 0;
 		default:
@@ -1210,6 +1212,7 @@ mwifiex_cfg80211_change_virtual_intf(struct wiphy *wiphy,
 		case NL80211_IFTYPE_UNSPECIFIED:
 			mwifiex_dbg(priv->adapter, INFO,
 				    "%s: kept type as AP\n", dev->name);
+			/* fall through */
 		case NL80211_IFTYPE_AP:		/* This shouldn't happen */
 			return 0;
 		default:
@@ -1249,6 +1252,7 @@ mwifiex_cfg80211_change_virtual_intf(struct wiphy *wiphy,
 		case NL80211_IFTYPE_UNSPECIFIED:
 			mwifiex_dbg(priv->adapter, INFO,
 				    "%s: kept type as P2P\n", dev->name);
+			/* fall through */
 		case NL80211_IFTYPE_P2P_CLIENT:
 		case NL80211_IFTYPE_P2P_GO:
 			return 0;
@@ -1979,7 +1983,8 @@ static int mwifiex_cfg80211_start_ap(struct wiphy *wiphy,
 		bss_cfg->bcast_ssid_ctl = 0;
 		break;
 	case NL80211_HIDDEN_SSID_ZERO_CONTENTS:
-		/* firmware doesn't support this type of hidden SSID */
+		bss_cfg->bcast_ssid_ctl = 2;
+		break;
 	default:
 		kfree(bss_cfg);
 		return -EINVAL;
@@ -2257,7 +2262,7 @@ mwifiex_cfg80211_assoc(struct mwifiex_private *priv, size_t ssid_len,
 
 		if (!bss) {
 			if (is_scanning_required) {
-				mwifiex_dbg(priv->adapter, WARN,
+				mwifiex_dbg(priv->adapter, MSG,
 					    "assoc: requested bss not found in scan results\n");
 				break;
 			}
@@ -2978,7 +2983,7 @@ struct wireless_dev *mwifiex_add_virtual_intf(struct wiphy *wiphy,
 	priv->netdev = dev;
 
 	if (!adapter->mfg_mode) {
-		mwifiex_set_mac_address(priv, dev);
+		mwifiex_set_mac_address(priv, dev, false, NULL);
 
 		ret = mwifiex_send_cmd(priv, HostCmd_CMD_SET_BSS_MODE,
 				       HostCmd_ACT_GEN_SET, 0, NULL, true);
@@ -3554,6 +3559,9 @@ static int mwifiex_set_rekey_data(struct wiphy *wiphy, struct net_device *dev,
 {
 	struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev);
 
+	if (!ISSUPP_FIRMWARE_SUPPLICANT(priv->adapter->fw_cap_info))
+		return -EOPNOTSUPP;
+
 	return mwifiex_send_cmd(priv, HostCmd_CMD_GTK_REKEY_OFFLOAD_CFG,
 				HostCmd_ACT_GEN_SET, 0, data, true);
 }
@@ -4189,6 +4197,16 @@ static const struct wiphy_wowlan_support mwifiex_wowlan_support = {
 	.max_pkt_offset = MWIFIEX_MAX_OFFSET_LEN,
 	.max_nd_match_sets = MWIFIEX_MAX_ND_MATCH_SETS,
 };
+
+static const struct wiphy_wowlan_support mwifiex_wowlan_support_no_gtk = {
+	.flags = WIPHY_WOWLAN_MAGIC_PKT | WIPHY_WOWLAN_DISCONNECT |
+		 WIPHY_WOWLAN_NET_DETECT,
+	.n_patterns = MWIFIEX_MEF_MAX_FILTERS,
+	.pattern_min_len = 1,
+	.pattern_max_len = MWIFIEX_MAX_PATTERN_LEN,
+	.max_pkt_offset = MWIFIEX_MAX_OFFSET_LEN,
+	.max_nd_match_sets = MWIFIEX_MAX_ND_MATCH_SETS,
+};
 #endif
 
 static bool mwifiex_is_valid_alpha2(const char *alpha2)
@@ -4307,7 +4325,10 @@ int mwifiex_register_cfg80211(struct mwifiex_adapter *adapter)
 				WIPHY_FLAG_TDLS_EXTERNAL_SETUP;
 
 #ifdef CONFIG_PM
-	wiphy->wowlan = &mwifiex_wowlan_support;
+	if (ISSUPP_FIRMWARE_SUPPLICANT(priv->adapter->fw_cap_info))
+		wiphy->wowlan = &mwifiex_wowlan_support;
+	else
+		wiphy->wowlan = &mwifiex_wowlan_support_no_gtk;
 #endif
 
 	wiphy->coalesce = &mwifiex_coalesce_support;
diff --git a/drivers/net/wireless/marvell/mwifiex/cmdevt.c b/drivers/net/wireless/marvell/mwifiex/cmdevt.c
index 7014f440e6f8..9cfcdf6bec52 100644
--- a/drivers/net/wireless/marvell/mwifiex/cmdevt.c
+++ b/drivers/net/wireless/marvell/mwifiex/cmdevt.c
@@ -25,7 +25,6 @@
 #include "main.h"
 #include "wmm.h"
 #include "11n.h"
-#include "11ac.h"
 
 static void mwifiex_cancel_pending_ioctl(struct mwifiex_adapter *adapter);
 
diff --git a/drivers/net/wireless/marvell/mwifiex/debugfs.c b/drivers/net/wireless/marvell/mwifiex/debugfs.c
index db2872daae97..07453932f703 100644
--- a/drivers/net/wireless/marvell/mwifiex/debugfs.c
+++ b/drivers/net/wireless/marvell/mwifiex/debugfs.c
@@ -153,34 +153,6 @@ mwifiex_info_read(struct file *file, char __user *ubuf,
 	return ret;
 }
 
-/*
- * Proc device dump read handler.
- *
- * This function is called when the 'device_dump' file is opened for
- * reading.
- * This function dumps driver information and firmware memory segments
- * (ex. DTCM, ITCM, SQRAM etc.) for
- * debugging.
- */
-static ssize_t
-mwifiex_device_dump_read(struct file *file, char __user *ubuf,
-			 size_t count, loff_t *ppos)
-{
-	struct mwifiex_private *priv = file->private_data;
-
-	/* For command timeouts, USB firmware will automatically emit
-	 * firmware dump events, so we don't implement device_dump().
-	 * For user-initiated dumps, we trigger it ourselves.
-	 */
-	if (priv->adapter->iface_type == MWIFIEX_USB)
-		mwifiex_send_cmd(priv, HostCmd_CMD_FW_DUMP_EVENT,
-				 HostCmd_ACT_GEN_SET, 0, NULL, true);
-	else
-		priv->adapter->if_ops.device_dump(priv->adapter);
-
-	return 0;
-}
-
 /*
  * Proc getlog file read handler.
  *
@@ -980,7 +952,6 @@ static const struct file_operations mwifiex_dfs_##name##_fops = {       \
 MWIFIEX_DFS_FILE_READ_OPS(info);
 MWIFIEX_DFS_FILE_READ_OPS(debug);
 MWIFIEX_DFS_FILE_READ_OPS(getlog);
-MWIFIEX_DFS_FILE_READ_OPS(device_dump);
 MWIFIEX_DFS_FILE_OPS(regrdwr);
 MWIFIEX_DFS_FILE_OPS(rdeeprom);
 MWIFIEX_DFS_FILE_OPS(memrw);
@@ -1011,7 +982,7 @@ mwifiex_dev_debugfs_init(struct mwifiex_private *priv)
 	MWIFIEX_DFS_ADD_FILE(getlog);
 	MWIFIEX_DFS_ADD_FILE(regrdwr);
 	MWIFIEX_DFS_ADD_FILE(rdeeprom);
-	MWIFIEX_DFS_ADD_FILE(device_dump);
+
 	MWIFIEX_DFS_ADD_FILE(memrw);
 	MWIFIEX_DFS_ADD_FILE(hscfg);
 	MWIFIEX_DFS_ADD_FILE(histogram);
diff --git a/drivers/net/wireless/marvell/mwifiex/fw.h b/drivers/net/wireless/marvell/mwifiex/fw.h
index c5dc518f768b..b73f99dc5a72 100644
--- a/drivers/net/wireless/marvell/mwifiex/fw.h
+++ b/drivers/net/wireless/marvell/mwifiex/fw.h
@@ -249,6 +249,7 @@ enum MWIFIEX_802_11_PRIVACY_FILTER {
 #define ISSUPP_SDIO_SPA_ENABLED(FwCapInfo) (FwCapInfo & BIT(16))
 #define ISSUPP_ADHOC_ENABLED(FwCapInfo) (FwCapInfo & BIT(25))
 #define ISSUPP_RANDOM_MAC(FwCapInfo) (FwCapInfo & BIT(27))
+#define ISSUPP_FIRMWARE_SUPPLICANT(FwCapInfo) (FwCapInfo & BIT(21))
 
 #define MWIFIEX_DEF_HT_CAP	(IEEE80211_HT_CAP_DSSSCCK40 | \
 				 (1 << IEEE80211_HT_CAP_RX_STBC_SHIFT) | \
diff --git a/drivers/net/wireless/marvell/mwifiex/ie.c b/drivers/net/wireless/marvell/mwifiex/ie.c
index 922e3d69fd84..b10baacb51c9 100644
--- a/drivers/net/wireless/marvell/mwifiex/ie.c
+++ b/drivers/net/wireless/marvell/mwifiex/ie.c
@@ -349,6 +349,7 @@ static int mwifiex_uap_parse_tail_ies(struct mwifiex_private *priv,
 		case WLAN_EID_SUPP_RATES:
 		case WLAN_EID_COUNTRY:
 		case WLAN_EID_PWR_CONSTRAINT:
+		case WLAN_EID_ERP_INFO:
 		case WLAN_EID_EXT_SUPP_RATES:
 		case WLAN_EID_HT_CAPABILITY:
 		case WLAN_EID_HT_OPERATION:
diff --git a/drivers/net/wireless/marvell/mwifiex/main.c b/drivers/net/wireless/marvell/mwifiex/main.c
index b6484582845a..510f6b8e717d 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.c
+++ b/drivers/net/wireless/marvell/mwifiex/main.c
@@ -858,7 +858,7 @@ mwifiex_clone_skb_for_tx_status(struct mwifiex_private *priv,
 /*
  * CFG802.11 network device handler for data transmission.
  */
-static int
+static netdev_tx_t
 mwifiex_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev);
@@ -940,28 +940,32 @@ mwifiex_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
 }
 
 int mwifiex_set_mac_address(struct mwifiex_private *priv,
-			    struct net_device *dev)
+			    struct net_device *dev, bool external,
+			    u8 *new_mac)
 {
 	int ret;
 	u64 mac_addr, old_mac_addr;
 
-	if (priv->bss_type == MWIFIEX_BSS_TYPE_ANY)
-		return -ENOTSUPP;
+	old_mac_addr = ether_addr_to_u64(priv->curr_addr);
 
-	mac_addr = ether_addr_to_u64(priv->curr_addr);
-	old_mac_addr = mac_addr;
+	if (external) {
+		mac_addr = ether_addr_to_u64(new_mac);
+	} else {
+		/* Internal mac address change */
+		if (priv->bss_type == MWIFIEX_BSS_TYPE_ANY)
+			return -ENOTSUPP;
 
-	if (priv->bss_type == MWIFIEX_BSS_TYPE_P2P)
-		mac_addr |= BIT_ULL(MWIFIEX_MAC_LOCAL_ADMIN_BIT);
+		mac_addr = old_mac_addr;
 
-	if (mwifiex_get_intf_num(priv->adapter, priv->bss_type) > 1) {
-		/* Set mac address based on bss_type/bss_num */
-		mac_addr ^= BIT_ULL(priv->bss_type + 8);
-		mac_addr += priv->bss_num;
-	}
+		if (priv->bss_type == MWIFIEX_BSS_TYPE_P2P)
+			mac_addr |= BIT_ULL(MWIFIEX_MAC_LOCAL_ADMIN_BIT);
 
-	if (mac_addr == old_mac_addr)
-		goto done;
+		if (mwifiex_get_intf_num(priv->adapter, priv->bss_type) > 1) {
+			/* Set mac address based on bss_type/bss_num */
+			mac_addr ^= BIT_ULL(priv->bss_type + 8);
+			mac_addr += priv->bss_num;
+		}
+	}
 
 	u64_to_ether_addr(mac_addr, priv->curr_addr);
 
@@ -976,7 +980,6 @@ int mwifiex_set_mac_address(struct mwifiex_private *priv,
 		return ret;
 	}
 
-done:
 	ether_addr_copy(dev->dev_addr, priv->curr_addr);
 	return 0;
 }
@@ -989,8 +992,7 @@ mwifiex_ndo_set_mac_address(struct net_device *dev, void *addr)
 	struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev);
 	struct sockaddr *hw_addr = addr;
 
-	memcpy(priv->curr_addr, hw_addr->sa_data, ETH_ALEN);
-	return mwifiex_set_mac_address(priv, dev);
+	return mwifiex_set_mac_address(priv, dev, true, hw_addr->sa_data);
 }
 
 /*
@@ -1331,7 +1333,10 @@ void mwifiex_init_priv_params(struct mwifiex_private *priv,
 	priv->assocresp_idx = MWIFIEX_AUTO_IDX_MASK;
 	priv->gen_idx = MWIFIEX_AUTO_IDX_MASK;
 	priv->num_tx_timeout = 0;
-	ether_addr_copy(priv->curr_addr, priv->adapter->perm_addr);
+	if (is_valid_ether_addr(dev->dev_addr))
+		ether_addr_copy(priv->curr_addr, dev->dev_addr);
+	else
+		ether_addr_copy(priv->curr_addr, priv->adapter->perm_addr);
 
 	if (GET_BSS_ROLE(priv) == MWIFIEX_BSS_ROLE_STA ||
 	    GET_BSS_ROLE(priv) == MWIFIEX_BSS_ROLE_UAP) {
diff --git a/drivers/net/wireless/marvell/mwifiex/main.h b/drivers/net/wireless/marvell/mwifiex/main.h
index 9bde181700dc..69ac0a22c28c 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.h
+++ b/drivers/net/wireless/marvell/mwifiex/main.h
@@ -84,8 +84,8 @@ enum {
 #define MWIFIEX_TIMER_10S			10000
 #define MWIFIEX_TIMER_1S			1000
 
-#define MAX_TX_PENDING      100
-#define LOW_TX_PENDING      80
+#define MAX_TX_PENDING      400
+#define LOW_TX_PENDING      380
 
 #define HIGH_RX_PENDING     50
 #define LOW_RX_PENDING      20
@@ -1691,6 +1691,7 @@ void mwifiex_drv_info_dump(struct mwifiex_adapter *adapter);
 void mwifiex_prepare_fw_dump_info(struct mwifiex_adapter *adapter);
 void mwifiex_upload_device_dump(struct mwifiex_adapter *adapter);
 void *mwifiex_alloc_dma_align_buf(int rx_len, gfp_t flags);
+void mwifiex_fw_dump_event(struct mwifiex_private *priv);
 void mwifiex_queue_main_work(struct mwifiex_adapter *adapter);
 int mwifiex_get_wakeup_reason(struct mwifiex_private *priv, u16 action,
 			      int cmd_type,
@@ -1709,7 +1710,8 @@ void mwifiex_process_multi_chan_event(struct mwifiex_private *priv,
 				      struct sk_buff *event_skb);
 void mwifiex_multi_chan_resync(struct mwifiex_adapter *adapter);
 int mwifiex_set_mac_address(struct mwifiex_private *priv,
-			    struct net_device *dev);
+			    struct net_device *dev,
+			    bool external, u8 *new_mac);
 void mwifiex_devdump_tmo_func(unsigned long function_context);
 
 #ifdef CONFIG_DEBUG_FS
diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c
index 97a6199692ab..0c42b7296ddd 100644
--- a/drivers/net/wireless/marvell/mwifiex/pcie.c
+++ b/drivers/net/wireless/marvell/mwifiex/pcie.c
@@ -320,6 +320,19 @@ static void mwifiex_pcie_shutdown(struct pci_dev *pdev)
 	return;
 }
 
+static void mwifiex_pcie_coredump(struct device *dev)
+{
+	struct pci_dev *pdev;
+	struct pcie_service_card *card;
+
+	pdev = container_of(dev, struct pci_dev, dev);
+	card = pci_get_drvdata(pdev);
+
+	if (!test_and_set_bit(MWIFIEX_IFACE_WORK_DEVICE_DUMP,
+			      &card->work_flags))
+		schedule_work(&card->work);
+}
+
 static const struct pci_device_id mwifiex_ids[] = {
 	{
 		PCIE_VENDOR_ID_MARVELL, PCIE_DEVICE_ID_MARVELL_88W8766P,
@@ -415,11 +428,12 @@ static struct pci_driver __refdata mwifiex_pcie = {
 	.id_table = mwifiex_ids,
 	.probe    = mwifiex_pcie_probe,
 	.remove   = mwifiex_pcie_remove,
-#ifdef CONFIG_PM_SLEEP
 	.driver   = {
+		.coredump = mwifiex_pcie_coredump,
+#ifdef CONFIG_PM_SLEEP
 		.pm = &mwifiex_pcie_pm_ops,
-	},
 #endif
+	},
 	.shutdown = mwifiex_pcie_shutdown,
 	.err_handler = &mwifiex_pcie_err_handler,
 };
@@ -1881,7 +1895,8 @@ static int mwifiex_pcie_process_event_ready(struct mwifiex_adapter *adapter)
 		mwifiex_dbg(adapter, EVENT,
 			    "info: Event length: %d\n", evt_len);
 
-		if ((evt_len > 0) && (evt_len  < MAX_EVENT_SIZE))
+		if (evt_len > MWIFIEX_EVENT_HEADER_LEN &&
+		    evt_len < MAX_EVENT_SIZE)
 			memcpy(adapter->event_body, skb_cmd->data +
 			       MWIFIEX_EVENT_HEADER_LEN, evt_len -
 			       MWIFIEX_EVENT_HEADER_LEN);
diff --git a/drivers/net/wireless/marvell/mwifiex/scan.c b/drivers/net/wireless/marvell/mwifiex/scan.c
index d7ce7f75ae38..895b806cdb03 100644
--- a/drivers/net/wireless/marvell/mwifiex/scan.c
+++ b/drivers/net/wireless/marvell/mwifiex/scan.c
@@ -482,7 +482,8 @@ mwifiex_scan_create_channel_list(struct mwifiex_private *priv,
 				scan_chan_list[chan_idx].max_scan_time =
 					cpu_to_le16((u16) user_scan_in->
 					chan_list[0].scan_time);
-			else if (ch->flags & IEEE80211_CHAN_NO_IR)
+			else if ((ch->flags & IEEE80211_CHAN_NO_IR) ||
+				 (ch->flags & IEEE80211_CHAN_RADAR))
 				scan_chan_list[chan_idx].max_scan_time =
 					cpu_to_le16(adapter->passive_scan_time);
 			else
@@ -502,10 +503,12 @@ mwifiex_scan_create_channel_list(struct mwifiex_private *priv,
 			scan_chan_list[chan_idx].chan_scan_mode_bitmap
 					|= MWIFIEX_DISABLE_CHAN_FILT;
 
-			if (filtered_scan) {
+			if (filtered_scan &&
+			    !((ch->flags & IEEE80211_CHAN_NO_IR) ||
+			      (ch->flags & IEEE80211_CHAN_RADAR)))
 				scan_chan_list[chan_idx].max_scan_time =
 				cpu_to_le16(adapter->specific_scan_time);
-			}
+
 			chan_idx++;
 		}
 
@@ -1308,6 +1311,7 @@ int mwifiex_update_bss_desc_with_ie(struct mwifiex_adapter *adapter,
 
 		case WLAN_EID_CHANNEL_SWITCH:
 			bss_entry->chan_sw_ie_present = true;
+			/* fall through */
 		case WLAN_EID_PWR_CAPABILITY:
 		case WLAN_EID_TPC_REPORT:
 		case WLAN_EID_QUIET:
diff --git a/drivers/net/wireless/marvell/mwifiex/sdio.c b/drivers/net/wireless/marvell/mwifiex/sdio.c
index a82880132af4..47d2dcc3f28f 100644
--- a/drivers/net/wireless/marvell/mwifiex/sdio.c
+++ b/drivers/net/wireless/marvell/mwifiex/sdio.c
@@ -466,6 +466,17 @@ static int mwifiex_sdio_suspend(struct device *dev)
 	return ret;
 }
 
+static void mwifiex_sdio_coredump(struct device *dev)
+{
+	struct sdio_func *func = dev_to_sdio_func(dev);
+	struct sdio_mmc_card *card;
+
+	card = sdio_get_drvdata(func);
+	if (!test_and_set_bit(MWIFIEX_IFACE_WORK_DEVICE_DUMP,
+			      &card->work_flags))
+		schedule_work(&card->work);
+}
+
 /* Device ID for SD8786 */
 #define SDIO_DEVICE_ID_MARVELL_8786   (0x9116)
 /* Device ID for SD8787 */
@@ -515,6 +526,7 @@ static struct sdio_driver mwifiex_sdio = {
 	.remove = mwifiex_sdio_remove,
 	.drv = {
 		.owner = THIS_MODULE,
+		.coredump = mwifiex_sdio_coredump,
 		.pm = &mwifiex_sdio_pm_ops,
 	}
 };
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_event.c b/drivers/net/wireless/marvell/mwifiex/sta_event.c
index 93dfb76cd8a6..03a6492662ca 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_event.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_event.c
@@ -241,6 +241,9 @@ void mwifiex_reset_connect_state(struct mwifiex_private *priv, u16 reason_code,
 	if (netif_carrier_ok(priv->netdev))
 		netif_carrier_off(priv->netdev);
 
+	if (!ISSUPP_FIRMWARE_SUPPLICANT(priv->adapter->fw_cap_info))
+		return;
+
 	mwifiex_send_cmd(priv, HostCmd_CMD_GTK_REKEY_OFFLOAD_CFG,
 			 HostCmd_ACT_GEN_REMOVE, 0, NULL, false);
 }
diff --git a/drivers/net/wireless/marvell/mwifiex/uap_event.c b/drivers/net/wireless/marvell/mwifiex/uap_event.c
index e8c8728db15a..e86217a6b9ca 100644
--- a/drivers/net/wireless/marvell/mwifiex/uap_event.c
+++ b/drivers/net/wireless/marvell/mwifiex/uap_event.c
@@ -108,7 +108,7 @@ int mwifiex_process_uap_event(struct mwifiex_private *priv)
 	struct mwifiex_adapter *adapter = priv->adapter;
 	int len, i;
 	u32 eventcause = adapter->event_cause;
-	struct station_info sinfo;
+	struct station_info *sinfo;
 	struct mwifiex_assoc_event *event;
 	struct mwifiex_sta_node *node;
 	u8 *deauth_mac;
@@ -117,7 +117,10 @@ int mwifiex_process_uap_event(struct mwifiex_private *priv)
 
 	switch (eventcause) {
 	case EVENT_UAP_STA_ASSOC:
-		memset(&sinfo, 0, sizeof(sinfo));
+		sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL);
+		if (!sinfo)
+			return -ENOMEM;
+
 		event = (struct mwifiex_assoc_event *)
 			(adapter->event_body + MWIFIEX_UAP_EVENT_EXTRA_HEADER);
 		if (le16_to_cpu(event->type) == TLV_TYPE_UAP_MGMT_FRAME) {
@@ -132,28 +135,31 @@ int mwifiex_process_uap_event(struct mwifiex_private *priv)
 				len = ETH_ALEN;
 
 			if (len != -1) {
-				sinfo.assoc_req_ies = &event->data[len];
-				len = (u8 *)sinfo.assoc_req_ies -
+				sinfo->assoc_req_ies = &event->data[len];
+				len = (u8 *)sinfo->assoc_req_ies -
 				      (u8 *)&event->frame_control;
-				sinfo.assoc_req_ies_len =
+				sinfo->assoc_req_ies_len =
 					le16_to_cpu(event->len) - (u16)len;
 			}
 		}
-		cfg80211_new_sta(priv->netdev, event->sta_addr, &sinfo,
+		cfg80211_new_sta(priv->netdev, event->sta_addr, sinfo,
 				 GFP_KERNEL);
 
 		node = mwifiex_add_sta_entry(priv, event->sta_addr);
 		if (!node) {
 			mwifiex_dbg(adapter, ERROR,
 				    "could not create station entry!\n");
+			kfree(sinfo);
 			return -1;
 		}
 
-		if (!priv->ap_11n_enabled)
+		if (!priv->ap_11n_enabled) {
+			kfree(sinfo);
 			break;
+		}
 
-		mwifiex_set_sta_ht_cap(priv, sinfo.assoc_req_ies,
-				       sinfo.assoc_req_ies_len, node);
+		mwifiex_set_sta_ht_cap(priv, sinfo->assoc_req_ies,
+				       sinfo->assoc_req_ies_len, node);
 
 		for (i = 0; i < MAX_NUM_TID; i++) {
 			if (node->is_11n_enabled)
@@ -163,6 +169,7 @@ int mwifiex_process_uap_event(struct mwifiex_private *priv)
 				node->ampdu_sta[i] = BA_STREAM_NOT_ALLOWED;
 		}
 		memset(node->rx_seq, 0xff, sizeof(node->rx_seq));
+		kfree(sinfo);
 		break;
 	case EVENT_UAP_STA_DEAUTH:
 		deauth_mac = adapter->event_body +
diff --git a/drivers/net/wireless/marvell/mwifiex/usb.c b/drivers/net/wireless/marvell/mwifiex/usb.c
index 4bc244801636..6e3cf9817730 100644
--- a/drivers/net/wireless/marvell/mwifiex/usb.c
+++ b/drivers/net/wireless/marvell/mwifiex/usb.c
@@ -644,6 +644,9 @@ static void mwifiex_usb_disconnect(struct usb_interface *intf)
 					 MWIFIEX_FUNC_SHUTDOWN);
 	}
 
+	if (adapter->workqueue)
+		flush_workqueue(adapter->workqueue);
+
 	mwifiex_usb_free(card);
 
 	mwifiex_dbg(adapter, FATAL,
@@ -653,6 +656,15 @@ static void mwifiex_usb_disconnect(struct usb_interface *intf)
 	usb_put_dev(interface_to_usbdev(intf));
 }
 
+static void mwifiex_usb_coredump(struct device *dev)
+{
+	struct usb_interface *intf = to_usb_interface(dev);
+	struct usb_card_rec *card = usb_get_intfdata(intf);
+
+	mwifiex_fw_dump_event(mwifiex_get_priv(card->adapter,
+					       MWIFIEX_BSS_ROLE_ANY));
+}
+
 static struct usb_driver mwifiex_usb_driver = {
 	.name = "mwifiex_usb",
 	.probe = mwifiex_usb_probe,
@@ -661,6 +673,9 @@ static struct usb_driver mwifiex_usb_driver = {
 	.suspend = mwifiex_usb_suspend,
 	.resume = mwifiex_usb_resume,
 	.soft_unbind = 1,
+	.drvwrap.driver = {
+		.coredump = mwifiex_usb_coredump,
+	},
 };
 
 static int mwifiex_write_data_sync(struct mwifiex_adapter *adapter, u8 *pbuf,
diff --git a/drivers/net/wireless/marvell/mwifiex/util.c b/drivers/net/wireless/marvell/mwifiex/util.c
index 0cd68ffc2c74..6dd212898117 100644
--- a/drivers/net/wireless/marvell/mwifiex/util.c
+++ b/drivers/net/wireless/marvell/mwifiex/util.c
@@ -708,12 +708,14 @@ void mwifiex_hist_data_set(struct mwifiex_private *priv, u8 rx_rate, s8 snr,
 			   s8 nflr)
 {
 	struct mwifiex_histogram_data *phist_data = priv->hist_data;
+	s8 nf   = -nflr;
+	s8 rssi = snr - nflr;
 
 	atomic_inc(&phist_data->num_samples);
 	atomic_inc(&phist_data->rx_rate[rx_rate]);
-	atomic_inc(&phist_data->snr[snr]);
-	atomic_inc(&phist_data->noise_flr[128 + nflr]);
-	atomic_inc(&phist_data->sig_str[nflr - snr]);
+	atomic_inc(&phist_data->snr[snr + 128]);
+	atomic_inc(&phist_data->noise_flr[nf + 128]);
+	atomic_inc(&phist_data->sig_str[rssi + 128]);
 }
 
 /* function to reset histogram data during init/reset */
@@ -755,3 +757,10 @@ void *mwifiex_alloc_dma_align_buf(int rx_len, gfp_t flags)
 	return skb;
 }
 EXPORT_SYMBOL_GPL(mwifiex_alloc_dma_align_buf);
+
+void mwifiex_fw_dump_event(struct mwifiex_private *priv)
+{
+	mwifiex_send_cmd(priv, HostCmd_CMD_FW_DUMP_EVENT, HostCmd_ACT_GEN_SET,
+			 0, NULL, true);
+}
+EXPORT_SYMBOL_GPL(mwifiex_fw_dump_event);
diff --git a/drivers/net/wireless/mediatek/mt76/agg-rx.c b/drivers/net/wireless/mediatek/mt76/agg-rx.c
index fcb208d1f276..b67acc6189bf 100644
--- a/drivers/net/wireless/mediatek/mt76/agg-rx.c
+++ b/drivers/net/wireless/mediatek/mt76/agg-rx.c
@@ -103,6 +103,7 @@ mt76_rx_aggr_reorder_work(struct work_struct *work)
 	__skb_queue_head_init(&frames);
 
 	local_bh_disable();
+	rcu_read_lock();
 
 	spin_lock(&tid->lock);
 	mt76_rx_aggr_check_release(tid, &frames);
@@ -114,6 +115,7 @@ mt76_rx_aggr_reorder_work(struct work_struct *work)
 					     REORDER_TIMEOUT);
 	mt76_rx_complete(dev, &frames, -1);
 
+	rcu_read_unlock();
 	local_bh_enable();
 }
 
@@ -147,12 +149,13 @@ mt76_rx_aggr_check_ctl(struct sk_buff *skb, struct sk_buff_head *frames)
 void mt76_rx_aggr_reorder(struct sk_buff *skb, struct sk_buff_head *frames)
 {
 	struct mt76_rx_status *status = (struct mt76_rx_status *) skb->cb;
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
 	struct mt76_wcid *wcid = status->wcid;
 	struct ieee80211_sta *sta;
 	struct mt76_rx_tid *tid;
 	bool sn_less;
 	u16 seqno, head, size;
-	u8 idx;
+	u8 ackp, idx;
 
 	__skb_queue_tail(frames, skb);
 
@@ -165,10 +168,17 @@ void mt76_rx_aggr_reorder(struct sk_buff *skb, struct sk_buff_head *frames)
 		return;
 	}
 
+	/* not part of a BA session */
+	ackp = *ieee80211_get_qos_ctl(hdr) & IEEE80211_QOS_CTL_ACK_POLICY_MASK;
+	if (ackp != IEEE80211_QOS_CTL_ACK_POLICY_BLOCKACK &&
+	    ackp != IEEE80211_QOS_CTL_ACK_POLICY_NORMAL)
+		return;
+
 	tid = rcu_dereference(wcid->aggr[status->tid]);
 	if (!tid)
 		return;
 
+	status->flag |= RX_FLAG_DUP_VALIDATED;
 	spin_lock_bh(&tid->lock);
 
 	if (tid->stopped)
@@ -258,6 +268,8 @@ static void mt76_rx_aggr_shutdown(struct mt76_dev *dev, struct mt76_rx_tid *tid)
 	u8 size = tid->size;
 	int i;
 
+	cancel_delayed_work(&tid->reorder_work);
+
 	spin_lock_bh(&tid->lock);
 
 	tid->stopped = true;
@@ -272,8 +284,6 @@ static void mt76_rx_aggr_shutdown(struct mt76_dev *dev, struct mt76_rx_tid *tid)
 	}
 
 	spin_unlock_bh(&tid->lock);
-
-	cancel_delayed_work_sync(&tid->reorder_work);
 }
 
 void mt76_rx_aggr_stop(struct mt76_dev *dev, struct mt76_wcid *wcid, u8 tidno)
diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c
index 3518703524e7..3dbedcedc2c4 100644
--- a/drivers/net/wireless/mediatek/mt76/dma.c
+++ b/drivers/net/wireless/mediatek/mt76/dma.c
@@ -178,6 +178,10 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)
 		mt76_dma_sync_idx(dev, q);
 
 	wake = wake && qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;
+
+	if (!q->queued)
+		wake_up(&dev->tx_wait);
+
 	spin_unlock_bh(&q->lock);
 
 	if (wake)
diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c
index 4f30cdcd2b53..fcd079a96782 100644
--- a/drivers/net/wireless/mediatek/mt76/mac80211.c
+++ b/drivers/net/wireless/mediatek/mt76/mac80211.c
@@ -213,7 +213,8 @@ mt76_init_sband(struct mt76_dev *dev, struct mt76_sband *msband,
 	vht_cap->vht_supported = true;
 	vht_cap->cap |= IEEE80211_VHT_CAP_RXLDPC |
 			IEEE80211_VHT_CAP_RXSTBC_1 |
-			IEEE80211_VHT_CAP_SHORT_GI_80;
+			IEEE80211_VHT_CAP_SHORT_GI_80 |
+			(3 << IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT);
 
 	return 0;
 }
@@ -267,6 +268,27 @@ mt76_check_sband(struct mt76_dev *dev, int band)
 	dev->hw->wiphy->bands[band] = NULL;
 }
 
+struct mt76_dev *
+mt76_alloc_device(unsigned int size, const struct ieee80211_ops *ops)
+{
+	struct ieee80211_hw *hw;
+	struct mt76_dev *dev;
+
+	hw = ieee80211_alloc_hw(size, ops);
+	if (!hw)
+		return NULL;
+
+	dev = hw->priv;
+	dev->hw = hw;
+	spin_lock_init(&dev->rx_lock);
+	spin_lock_init(&dev->lock);
+	spin_lock_init(&dev->cc_lock);
+	init_waitqueue_head(&dev->tx_wait);
+
+	return dev;
+}
+EXPORT_SYMBOL_GPL(mt76_alloc_device);
+
 int mt76_register_device(struct mt76_dev *dev, bool vht,
 			 struct ieee80211_rate *rates, int n_rates)
 {
@@ -276,8 +298,6 @@ int mt76_register_device(struct mt76_dev *dev, bool vht,
 
 	dev_set_drvdata(dev->dev, dev);
 
-	spin_lock_init(&dev->lock);
-	spin_lock_init(&dev->cc_lock);
 	INIT_LIST_HEAD(&dev->txwi_cache);
 
 	SET_IEEE80211_DEV(hw, dev->dev);
@@ -358,12 +378,32 @@ void mt76_rx(struct mt76_dev *dev, enum mt76_rxq_id q, struct sk_buff *skb)
 }
 EXPORT_SYMBOL_GPL(mt76_rx);
 
+static bool mt76_has_tx_pending(struct mt76_dev *dev)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++) {
+		if (dev->q_tx[i].queued)
+			return true;
+	}
+
+	return false;
+}
+
 void mt76_set_channel(struct mt76_dev *dev)
 {
 	struct ieee80211_hw *hw = dev->hw;
 	struct cfg80211_chan_def *chandef = &hw->conf.chandef;
 	struct mt76_channel_state *state;
 	bool offchannel = hw->conf.flags & IEEE80211_CONF_OFFCHANNEL;
+	int timeout = HZ / 5;
+
+	if (offchannel)
+		set_bit(MT76_OFFCHANNEL, &dev->state);
+	else
+		clear_bit(MT76_OFFCHANNEL, &dev->state);
+
+	wait_event_timeout(dev->tx_wait, !mt76_has_tx_pending(dev), timeout);
 
 	if (dev->drv->update_survey)
 		dev->drv->update_survey(dev);
@@ -541,15 +581,13 @@ mt76_check_ps(struct mt76_dev *dev, struct sk_buff *skb)
 	if (!!test_bit(MT_WCID_FLAG_PS, &wcid->flags) == ps)
 		return;
 
-	if (ps) {
+	if (ps)
 		set_bit(MT_WCID_FLAG_PS, &wcid->flags);
-		mt76_stop_tx_queues(dev, sta, true);
-	} else {
+	else
 		clear_bit(MT_WCID_FLAG_PS, &wcid->flags);
-	}
 
-	ieee80211_sta_ps_transition(sta, ps);
 	dev->drv->sta_ps(dev, sta, ps);
+	ieee80211_sta_ps_transition(sta, ps);
 }
 
 void mt76_rx_complete(struct mt76_dev *dev, struct sk_buff_head *frames,
@@ -562,6 +600,7 @@ void mt76_rx_complete(struct mt76_dev *dev, struct sk_buff_head *frames,
 	if (queue >= 0)
 	    napi = &dev->napi[queue];
 
+	spin_lock(&dev->rx_lock);
 	while ((skb = __skb_dequeue(frames)) != NULL) {
 		if (mt76_check_ccmp_pn(skb)) {
 			dev_kfree_skb(skb);
@@ -571,6 +610,7 @@ void mt76_rx_complete(struct mt76_dev *dev, struct sk_buff_head *frames,
 		sta = mt76_rx_convert(skb);
 		ieee80211_rx_napi(dev->hw, sta, skb, napi);
 	}
+	spin_unlock(&dev->rx_lock);
 }
 
 void mt76_rx_poll_complete(struct mt76_dev *dev, enum mt76_rxq_id q)
diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h
index 065ff78059c3..d2166fbf50ff 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76.h
@@ -189,6 +189,7 @@ enum {
 	MT76_STATE_RUNNING,
 	MT76_SCANNING,
 	MT76_RESET,
+	MT76_OFFCHANNEL,
 };
 
 struct mt76_hw_cap {
@@ -241,6 +242,7 @@ struct mt76_dev {
 	struct device *dev;
 
 	struct net_device napi_dev;
+	spinlock_t rx_lock;
 	struct napi_struct napi[__MT_RXQ_MAX];
 	struct sk_buff_head rx_skb[__MT_RXQ_MAX];
 
@@ -249,6 +251,8 @@ struct mt76_dev {
 	struct mt76_queue q_rx[__MT_RXQ_MAX];
 	const struct mt76_queue_ops *queue_ops;
 
+	wait_queue_head_t tx_wait;
+
 	u8 macaddr[ETH_ALEN];
 	u32 rev;
 	unsigned long state;
@@ -375,6 +379,8 @@ mt76_channel_state(struct mt76_dev *dev, struct ieee80211_channel *c)
 	return &msband->chan[idx];
 }
 
+struct mt76_dev *mt76_alloc_device(unsigned int size,
+				   const struct ieee80211_ops *ops);
 int mt76_register_device(struct mt76_dev *dev, bool vht,
 			 struct ieee80211_rate *rates, int n_rates);
 void mt76_unregister_device(struct mt76_dev *dev);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2.h b/drivers/net/wireless/mediatek/mt76/mt76x2.h
index 783b8122ec3c..dc12bbdbb2ee 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x2.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2.h
@@ -39,6 +39,9 @@
 
 #define MT_CALIBRATE_INTERVAL	HZ
 
+#define MT_MAX_VIFS		8
+#define MT_VIF_WCID(_n)		(254 - ((_n) & 7))
+
 #include "mt76.h"
 #include "mt76x2_regs.h"
 #include "mt76x2_mac.h"
@@ -117,7 +120,6 @@ struct mt76x2_dev {
 	u8 beacon_mask;
 	u8 beacon_data_mask;
 
-	u32 rev;
 	u32 rxfilter;
 
 	u16 chainmask;
@@ -151,7 +153,7 @@ struct mt76x2_sta {
 
 static inline bool is_mt7612(struct mt76x2_dev *dev)
 {
-	return (dev->rev >> 16) == 0x7612;
+	return mt76_chip(&dev->mt76) == 0x7612;
 }
 
 void mt76x2_set_irq_mask(struct mt76x2_dev *dev, u32 clear, u32 set);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_dma.h b/drivers/net/wireless/mediatek/mt76/mt76x2_dma.h
index 47f79d83fcb4..e9d426bbf91a 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x2_dma.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_dma.h
@@ -19,7 +19,7 @@
 
 #include "dma.h"
 
-#define MT_TXD_INFO_LEN			GENMASK(13, 0)
+#define MT_TXD_INFO_LEN			GENMASK(15, 0)
 #define MT_TXD_INFO_NEXT_VLD		BIT(16)
 #define MT_TXD_INFO_TX_BURST		BIT(17)
 #define MT_TXD_INFO_80211		BIT(19)
@@ -27,9 +27,8 @@
 #define MT_TXD_INFO_CSO			BIT(21)
 #define MT_TXD_INFO_WIV			BIT(24)
 #define MT_TXD_INFO_QSEL		GENMASK(26, 25)
-#define MT_TXD_INFO_TCO			BIT(29)
-#define MT_TXD_INFO_UCO			BIT(30)
-#define MT_TXD_INFO_ICO			BIT(31)
+#define MT_TXD_INFO_DPORT		GENMASK(29, 27)
+#define MT_TXD_INFO_TYPE		GENMASK(31, 30)
 
 #define MT_RX_FCE_INFO_LEN		GENMASK(13, 0)
 #define MT_RX_FCE_INFO_SELF_GEN		BIT(15)
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.c b/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.c
index 5bb50027c1e8..95d5f7d888f0 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.c
@@ -609,17 +609,13 @@ int mt76x2_get_temp_comp(struct mt76x2_dev *dev, struct mt76x2_temp_comp *t)
 
 	memset(t, 0, sizeof(*t));
 
-	val = mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_1);
-	if (!(val & MT_EE_NIC_CONF_1_TEMP_TX_ALC))
+	if (!mt76x2_temp_tx_alc_enabled(dev))
 		return -EINVAL;
 
 	if (!mt76x2_ext_pa_enabled(dev, band))
 		return -EINVAL;
 
 	val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_EXT_PA_5G) >> 8;
-	if (!(val & BIT(7)))
-		return -EINVAL;
-
 	t->temp_25_ref = val & 0x7f;
 	if (band == NL80211_BAND_5GHZ) {
 		slope = mt76x2_eeprom_get(dev, MT_EE_RF_TEMP_COMP_SLOPE_5G);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.h b/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.h
index d79122728dca..aa0b0c040375 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.h
@@ -159,6 +159,12 @@ void mt76x2_read_rx_gain(struct mt76x2_dev *dev);
 static inline bool
 mt76x2_temp_tx_alc_enabled(struct mt76x2_dev *dev)
 {
+	u16 val;
+
+	val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_EXT_PA_5G);
+	if (!(val & BIT(15)))
+		return false;
+
 	return mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_1) &
 	       MT_EE_NIC_CONF_1_TEMP_TX_ALC;
 }
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_init.c b/drivers/net/wireless/mediatek/mt76/mt76x2_init.c
index 934c331d995e..79ab93613e06 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x2_init.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_init.c
@@ -228,7 +228,7 @@ mt76x2_init_beacon_offsets(struct mt76x2_dev *dev)
 		mt76_wr(dev, MT_BCN_OFFSET(i), regs[i]);
 }
 
-int mt76x2_mac_reset(struct mt76x2_dev *dev, bool hard)
+static int mt76x2_mac_reset(struct mt76x2_dev *dev, bool hard)
 {
 	static const u8 null_addr[ETH_ALEN] = {};
 	const u8 *macaddr = dev->mt76.macaddr;
@@ -296,6 +296,9 @@ int mt76x2_mac_reset(struct mt76x2_dev *dev, bool hard)
 	for (i = 0; i < 256; i++)
 		mt76x2_mac_wcid_setup(dev, i, 0, NULL);
 
+	for (i = 0; i < MT_MAX_VIFS; i++)
+		mt76x2_mac_wcid_setup(dev, MT_VIF_WCID(i), i, NULL);
+
 	for (i = 0; i < 16; i++)
 		for (k = 0; k < 4; k++)
 			mt76x2_mac_shared_key_setup(dev, i, k, NULL);
@@ -370,12 +373,12 @@ void mt76x2_mac_stop(struct mt76x2_dev *dev, bool force)
 
 	/* Wait for MAC to become idle */
 	for (i = 0; i < 300; i++) {
-		if (mt76_rr(dev, MT_MAC_STATUS) &
-		    (MT_MAC_STATUS_RX | MT_MAC_STATUS_TX))
-			continue;
-
-		if (mt76_rr(dev, MT_BBP(IBI, 12)))
+		if ((mt76_rr(dev, MT_MAC_STATUS) &
+		     (MT_MAC_STATUS_RX | MT_MAC_STATUS_TX)) ||
+		    mt76_rr(dev, MT_BBP(IBI, 12))) {
+			udelay(1);
 			continue;
+		}
 
 		stopped = true;
 		break;
@@ -482,7 +485,10 @@ void mt76x2_set_tx_ackto(struct mt76x2_dev *dev)
 {
 	u8 ackto, sifs, slottime = dev->slottime;
 
+	/* As defined by IEEE 802.11-2007 17.3.8.6 */
 	slottime += 3 * dev->coverage_class;
+	mt76_rmw_field(dev, MT_BKOFF_SLOT_CFG,
+		       MT_BKOFF_SLOT_CFG_SLOTTIME, slottime);
 
 	sifs = mt76_get_field(dev, MT_XIFS_TIME_CFG,
 			      MT_XIFS_TIME_CFG_OFDM_SIFS);
@@ -632,17 +638,16 @@ struct mt76x2_dev *mt76x2_alloc_device(struct device *pdev)
 		.rx_poll_complete = mt76x2_rx_poll_complete,
 		.sta_ps = mt76x2_sta_ps,
 	};
-	struct ieee80211_hw *hw;
 	struct mt76x2_dev *dev;
+	struct mt76_dev *mdev;
 
-	hw = ieee80211_alloc_hw(sizeof(*dev), &mt76x2_ops);
-	if (!hw)
+	mdev = mt76_alloc_device(sizeof(*dev), &mt76x2_ops);
+	if (!mdev)
 		return NULL;
 
-	dev = hw->priv;
-	dev->mt76.dev = pdev;
-	dev->mt76.hw = hw;
-	dev->mt76.drv = &drv_ops;
+	dev = container_of(mdev, struct mt76x2_dev, mt76);
+	mdev->dev = pdev;
+	mdev->drv = &drv_ops;
 	mutex_init(&dev->mutex);
 	spin_lock_init(&dev->irq_lock);
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_mac.c b/drivers/net/wireless/mediatek/mt76/mt76x2_mac.c
index d18315652583..b49aea4da2d6 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x2_mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_mac.c
@@ -301,6 +301,9 @@ int mt76x2_mac_process_rx(struct mt76x2_dev *dev, struct sk_buff *skb,
 	u8 wcid;
 	int len;
 
+	if (!test_bit(MT76_STATE_RUNNING, &dev->mt76.state))
+		return -EINVAL;
+
 	if (rxinfo & MT_RXINFO_L2PAD)
 		pad_len += 2;
 
@@ -410,7 +413,6 @@ mt76x2_mac_process_tx_rate(struct ieee80211_tx_rate *txrate, u16 rate,
 		break;
 	default:
 		return -EINVAL;
-		break;
 	}
 
 	if (rate & MT_RXWI_RATE_SGI)
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_mac.h b/drivers/net/wireless/mediatek/mt76/mt76x2_mac.h
index 8a8a25e32d5f..c048cd06df6b 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x2_mac.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_mac.h
@@ -157,7 +157,6 @@ mt76x2_skb_tx_info(struct sk_buff *skb)
 	return (void *) info->status.status_driver_data;
 }
 
-int mt76x2_mac_reset(struct mt76x2_dev *dev, bool hard);
 int mt76x2_mac_start(struct mt76x2_dev *dev);
 void mt76x2_mac_stop(struct mt76x2_dev *dev, bool force);
 void mt76x2_mac_resume(struct mt76x2_dev *dev);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_main.c b/drivers/net/wireless/mediatek/mt76/mt76x2_main.c
index 73c127f92613..ce90ff999b49 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x2_main.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_main.c
@@ -104,7 +104,7 @@ mt76x2_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 		idx += 8;
 
 	mvif->idx = idx;
-	mvif->group_wcid.idx = 254 - idx;
+	mvif->group_wcid.idx = MT_VIF_WCID(idx);
 	mvif->group_wcid.hw_key_idx = -1;
 	mt76x2_txq_init(dev, vif->txq);
 
@@ -124,11 +124,14 @@ mt76x2_set_channel(struct mt76x2_dev *dev, struct cfg80211_chan_def *chandef)
 {
 	int ret;
 
+	cancel_delayed_work_sync(&dev->cal_work);
+
+	set_bit(MT76_RESET, &dev->mt76.state);
+
 	mt76_set_channel(&dev->mt76);
 
 	tasklet_disable(&dev->pre_tbtt_tasklet);
 	tasklet_disable(&dev->dfs_pd.dfs_tasklet);
-	cancel_delayed_work_sync(&dev->cal_work);
 
 	mt76x2_mac_stop(dev, true);
 	ret = mt76x2_phy_set_channel(dev, chandef);
@@ -143,6 +146,10 @@ mt76x2_set_channel(struct mt76x2_dev *dev, struct cfg80211_chan_def *chandef)
 	tasklet_enable(&dev->dfs_pd.dfs_tasklet);
 	tasklet_enable(&dev->pre_tbtt_tasklet);
 
+	clear_bit(MT76_RESET, &dev->mt76.state);
+
+	mt76_txq_schedule_all(&dev->mt76);
+
 	return ret;
 }
 
@@ -247,8 +254,7 @@ mt76x2_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		int slottime = info->use_short_slot ? 9 : 20;
 
 		dev->slottime = slottime;
-		mt76_rmw_field(dev, MT_BKOFF_SLOT_CFG,
-			       MT_BKOFF_SLOT_CFG_SLOTTIME, slottime);
+		mt76x2_set_tx_ackto(dev);
 	}
 
 	mutex_unlock(&dev->mutex);
@@ -321,6 +327,7 @@ mt76x2_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps)
 	struct mt76x2_dev *dev = container_of(mdev, struct mt76x2_dev, mt76);
 	int idx = msta->wcid.idx;
 
+	mt76_stop_tx_queues(&dev->mt76, sta, true);
 	mt76x2_mac_wcid_set_drop(dev, idx, ps);
 }
 
@@ -452,7 +459,6 @@ mt76x2_sw_scan_complete(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 
 	clear_bit(MT76_SCANNING, &dev->mt76.state);
 	tasklet_enable(&dev->pre_tbtt_tasklet);
-	mt76_txq_schedule_all(&dev->mt76);
 }
 
 static void
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_phy.c b/drivers/net/wireless/mediatek/mt76/mt76x2_phy.c
index fcc37eb7ce0b..c1c38ca3330a 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x2_phy.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_phy.c
@@ -73,16 +73,6 @@ int mt76x2_phy_get_rssi(struct mt76x2_dev *dev, s8 rssi, int chain)
 	return rssi;
 }
 
-static u8
-mt76x2_txpower_check(int value)
-{
-	if (value < 0)
-		return 0;
-	if (value > 0x2f)
-		return 0x2f;
-	return value;
-}
-
 static void
 mt76x2_add_rate_power_offset(struct mt76_rate_power *r, int offset)
 {
@@ -102,6 +92,26 @@ mt76x2_limit_rate_power(struct mt76_rate_power *r, int limit)
 			r->all[i] = limit;
 }
 
+static int
+mt76x2_get_min_rate_power(struct mt76_rate_power *r)
+{
+	int i;
+	s8 ret = 0;
+
+	for (i = 0; i < sizeof(r->all); i++) {
+		if (!r->all[i])
+			continue;
+
+		if (ret)
+			ret = min(ret, r->all[i]);
+		else
+			ret = r->all[i];
+	}
+
+	return ret;
+}
+
+
 void mt76x2_phy_set_txpower(struct mt76x2_dev *dev)
 {
 	enum nl80211_chan_width width = dev->mt76.chandef.width;
@@ -109,6 +119,7 @@ void mt76x2_phy_set_txpower(struct mt76x2_dev *dev)
 	struct mt76x2_tx_power_info txp;
 	int txp_0, txp_1, delta = 0;
 	struct mt76_rate_power t = {};
+	int base_power, gain;
 
 	mt76x2_get_power_info(dev, &txp, chan);
 
@@ -117,26 +128,32 @@ void mt76x2_phy_set_txpower(struct mt76x2_dev *dev)
 	else if (width == NL80211_CHAN_WIDTH_80)
 		delta = txp.delta_bw80;
 
-	if (txp.target_power > dev->txpower_conf)
-		delta -= txp.target_power - dev->txpower_conf;
-
 	mt76x2_get_rate_power(dev, &t, chan);
-	mt76x2_add_rate_power_offset(&t, txp.chain[0].target_power +
-				   txp.chain[0].delta);
+	mt76x2_add_rate_power_offset(&t, txp.chain[0].target_power);
 	mt76x2_limit_rate_power(&t, dev->txpower_conf);
 	dev->txpower_cur = mt76x2_get_max_rate_power(&t);
-	mt76x2_add_rate_power_offset(&t, -(txp.chain[0].target_power +
-					 txp.chain[0].delta + delta));
-	dev->target_power = txp.chain[0].target_power;
-	dev->target_power_delta[0] = txp.chain[0].delta + delta;
-	dev->target_power_delta[1] = txp.chain[1].delta + delta;
-	dev->rate_power = t;
 
-	txp_0 = mt76x2_txpower_check(txp.chain[0].target_power +
-				   txp.chain[0].delta + delta);
+	base_power = mt76x2_get_min_rate_power(&t);
+	delta += base_power - txp.chain[0].target_power;
+	txp_0 = txp.chain[0].target_power + txp.chain[0].delta + delta;
+	txp_1 = txp.chain[1].target_power + txp.chain[1].delta + delta;
+
+	gain = min(txp_0, txp_1);
+	if (gain < 0) {
+		base_power -= gain;
+		txp_0 -= gain;
+		txp_1 -= gain;
+	} else if (gain > 0x2f) {
+		base_power -= gain - 0x2f;
+		txp_0 = 0x2f;
+		txp_1 = 0x2f;
+	}
 
-	txp_1 = mt76x2_txpower_check(txp.chain[1].target_power +
-				   txp.chain[1].delta + delta);
+	mt76x2_add_rate_power_offset(&t, -base_power);
+	dev->target_power = txp.chain[0].target_power;
+	dev->target_power_delta[0] = txp_0 - txp.chain[0].target_power;
+	dev->target_power_delta[1] = txp_1 - txp.chain[0].target_power;
+	dev->rate_power = t;
 
 	mt76_rmw_field(dev, MT_TX_ALC_CFG_0, MT_TX_ALC_CFG_0_CH_INIT_0, txp_0);
 	mt76_rmw_field(dev, MT_TX_ALC_CFG_0, MT_TX_ALC_CFG_0_CH_INIT_1, txp_1);
@@ -180,7 +197,7 @@ mt76x2_phy_tssi_init_cal(struct mt76x2_dev *dev)
 	if (mt76x2_channel_silent(dev))
 		return false;
 
-	if (chan->band == NL80211_BAND_2GHZ)
+	if (chan->band == NL80211_BAND_5GHZ)
 		flag |= BIT(0);
 
 	if (mt76x2_ext_pa_enabled(dev, chan->band))
@@ -257,7 +274,6 @@ mt76x2_phy_set_txpower_regs(struct mt76x2_dev *dev, enum nl80211_band band)
 			mt76_wr(dev, MT_TX_ALC_CFG_2, 0x1b0f0400);
 			mt76_wr(dev, MT_TX_ALC_CFG_3, 0x1b0f0476);
 		}
-		mt76_wr(dev, MT_TX_ALC_CFG_4, 0);
 
 		if (mt76x2_ext_pa_enabled(dev, band))
 			pa_mode_adj = 0x04000000;
@@ -492,8 +508,10 @@ mt76x2_phy_update_channel_gain(struct mt76x2_dev *dev)
 	u8 gain_delta;
 	int low_gain;
 
-	dev->cal.avg_rssi[0] = (dev->cal.avg_rssi[0] * 15) / 16 + (rssi0 << 8);
-	dev->cal.avg_rssi[1] = (dev->cal.avg_rssi[1] * 15) / 16 + (rssi1 << 8);
+	dev->cal.avg_rssi[0] = (dev->cal.avg_rssi[0] * 15) / 16 +
+			       (rssi0 << 8) / 16;
+	dev->cal.avg_rssi[1] = (dev->cal.avg_rssi[1] * 15) / 16 +
+			       (rssi1 << 8) / 16;
 	dev->cal.avg_rssi_all = (dev->cal.avg_rssi[0] +
 				 dev->cal.avg_rssi[1]) / 512;
 
@@ -661,6 +679,14 @@ int mt76x2_phy_set_channel(struct mt76x2_dev *dev,
 	memcpy(dev->cal.agc_gain_cur, dev->cal.agc_gain_init,
 	       sizeof(dev->cal.agc_gain_cur));
 
+	/* init default values for temp compensation */
+	if (mt76x2_tssi_enabled(dev)) {
+		mt76_rmw_field(dev, MT_TX_ALC_CFG_1, MT_TX_ALC_CFG_1_TEMP_COMP,
+			       0x38);
+		mt76_rmw_field(dev, MT_TX_ALC_CFG_2, MT_TX_ALC_CFG_2_TEMP_COMP,
+			       0x38);
+	}
+
 	ieee80211_queue_delayed_work(mt76_hw(dev), &dev->cal_work,
 				     MT_CALIBRATE_INTERVAL);
 
diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c
index 4eef69bd8a9e..e96956710fb2 100644
--- a/drivers/net/wireless/mediatek/mt76/tx.c
+++ b/drivers/net/wireless/mediatek/mt76/tx.c
@@ -332,7 +332,7 @@ mt76_txq_send_burst(struct mt76_dev *dev, struct mt76_queue *hwq,
 		if (probe)
 			break;
 
-		if (test_bit(MT76_SCANNING, &dev->state) ||
+		if (test_bit(MT76_OFFCHANNEL, &dev->state) ||
 		    test_bit(MT76_RESET, &dev->state))
 			return -EBUSY;
 
@@ -385,6 +385,10 @@ mt76_txq_schedule_list(struct mt76_dev *dev, struct mt76_queue *hwq)
 		bool empty = false;
 		int cur;
 
+		if (test_bit(MT76_OFFCHANNEL, &dev->state) ||
+		    test_bit(MT76_RESET, &dev->state))
+			return -EBUSY;
+
 		mtxq = list_first_entry(&hwq->swq, struct mt76_txq, list);
 		if (mtxq->send_bar && mtxq->aggr) {
 			struct ieee80211_txq *txq = mtxq_to_txq(mtxq);
@@ -422,12 +426,14 @@ void mt76_txq_schedule(struct mt76_dev *dev, struct mt76_queue *hwq)
 {
 	int len;
 
+	rcu_read_lock();
 	do {
 		if (hwq->swq_queued >= 4 || list_empty(&hwq->swq))
 			break;
 
 		len = mt76_txq_schedule_list(dev, hwq);
 	} while (len > 0);
+	rcu_read_unlock();
 }
 EXPORT_SYMBOL_GPL(mt76_txq_schedule);
 
diff --git a/drivers/net/wireless/mediatek/mt7601u/mac.c b/drivers/net/wireless/mediatek/mt7601u/mac.c
index d55d7040a56d..148c36d3d2e5 100644
--- a/drivers/net/wireless/mediatek/mt7601u/mac.c
+++ b/drivers/net/wireless/mediatek/mt7601u/mac.c
@@ -453,7 +453,7 @@ mt7601u_rx_monitor_beacon(struct mt7601u_dev *dev, struct mt7601u_rxwi *rxwi,
 {
 	dev->bcn_freq_off = rxwi->freq_off;
 	dev->bcn_phy_mode = FIELD_GET(MT_RXWI_RATE_PHY, rate);
-	dev->avg_rssi = (dev->avg_rssi * 15) / 16 + (rssi << 8);
+	ewma_rssi_add(&dev->avg_rssi, -rssi);
 }
 
 static int
@@ -503,7 +503,7 @@ u32 mt76_mac_process_rx(struct mt7601u_dev *dev, struct sk_buff *skb,
 	if (mt7601u_rx_is_our_beacon(dev, data))
 		mt7601u_rx_monitor_beacon(dev, rxwi, rate, rssi);
 	else if (rxwi->rxinfo & cpu_to_le32(MT_RXINFO_U2M))
-		dev->avg_rssi = (dev->avg_rssi * 15) / 16 + (rssi << 8);
+		ewma_rssi_add(&dev->avg_rssi, -rssi);
 	spin_unlock_bh(&dev->con_mon_lock);
 
 	return len;
diff --git a/drivers/net/wireless/mediatek/mt7601u/main.c b/drivers/net/wireless/mediatek/mt7601u/main.c
index 3c9ea40d9584..7b21016012c3 100644
--- a/drivers/net/wireless/mediatek/mt7601u/main.c
+++ b/drivers/net/wireless/mediatek/mt7601u/main.c
@@ -288,6 +288,12 @@ mt7601u_sw_scan_complete(struct ieee80211_hw *hw,
 
 	mt7601u_agc_restore(dev);
 	clear_bit(MT7601U_STATE_SCANNING, &dev->state);
+
+	ieee80211_queue_delayed_work(dev->hw, &dev->cal_work,
+				     MT_CALIBRATE_INTERVAL);
+	if (dev->freq_cal.enabled)
+		ieee80211_queue_delayed_work(dev->hw, &dev->freq_cal.work,
+					     MT_FREQ_CAL_INIT_DELAY);
 }
 
 static int
diff --git a/drivers/net/wireless/mediatek/mt7601u/mt7601u.h b/drivers/net/wireless/mediatek/mt7601u/mt7601u.h
index 9233744451a9..db317d8c1652 100644
--- a/drivers/net/wireless/mediatek/mt7601u/mt7601u.h
+++ b/drivers/net/wireless/mediatek/mt7601u/mt7601u.h
@@ -23,6 +23,7 @@
 #include <linux/completion.h>
 #include <net/mac80211.h>
 #include <linux/debugfs.h>
+#include <linux/average.h>
 
 #include "regs.h"
 
@@ -138,6 +139,8 @@ enum {
 	MT7601U_STATE_MORE_STATS,
 };
 
+DECLARE_EWMA(rssi, 10, 4);
+
 /**
  * struct mt7601u_dev - adapter structure
  * @lock:		protects @wcid->tx_rate.
@@ -220,7 +223,7 @@ struct mt7601u_dev {
 	s8 bcn_freq_off;
 	u8 bcn_phy_mode;
 
-	int avg_rssi; /* starts at 0 and converges */
+	struct ewma_rssi avg_rssi;
 
 	u8 agc_save;
 
diff --git a/drivers/net/wireless/mediatek/mt7601u/phy.c b/drivers/net/wireless/mediatek/mt7601u/phy.c
index ca09a5d4305e..9d2f9a776ef1 100644
--- a/drivers/net/wireless/mediatek/mt7601u/phy.c
+++ b/drivers/net/wireless/mediatek/mt7601u/phy.c
@@ -795,6 +795,7 @@ mt7601u_phy_rf_pa_mode_val(struct mt7601u_dev *dev, int phy_mode, int tx_rate)
 	switch (phy_mode) {
 	case MT_PHY_TYPE_OFDM:
 		tx_rate += 4;
+		/* fall through */
 	case MT_PHY_TYPE_CCK:
 		reg = dev->rf_pa_mode[0];
 		break;
@@ -974,6 +975,7 @@ void mt7601u_agc_restore(struct mt7601u_dev *dev)
 static void mt7601u_agc_tune(struct mt7601u_dev *dev)
 {
 	u8 val = mt7601u_agc_default(dev);
+	long avg_rssi;
 
 	if (test_bit(MT7601U_STATE_SCANNING, &dev->state))
 		return;
@@ -983,9 +985,12 @@ static void mt7601u_agc_tune(struct mt7601u_dev *dev)
 	 *	 Rssi updates are only on beacons and U2M so should work...
 	 */
 	spin_lock_bh(&dev->con_mon_lock);
-	if (dev->avg_rssi <= -70)
+	avg_rssi = ewma_rssi_read(&dev->avg_rssi);
+	WARN_ON_ONCE(avg_rssi == 0);
+	avg_rssi = -avg_rssi;
+	if (avg_rssi <= -70)
 		val -= 0x20;
-	else if (dev->avg_rssi <= -60)
+	else if (avg_rssi <= -60)
 		val -= 0x10;
 	spin_unlock_bh(&dev->con_mon_lock);
 
@@ -1101,7 +1106,7 @@ void mt7601u_phy_con_cal_onoff(struct mt7601u_dev *dev,
 	/* Start/stop collecting beacon data */
 	spin_lock_bh(&dev->con_mon_lock);
 	ether_addr_copy(dev->ap_bssid, info->bssid);
-	dev->avg_rssi = 0;
+	ewma_rssi_init(&dev->avg_rssi);
 	dev->bcn_freq_off = MT_FREQ_OFFSET_INVALID;
 	spin_unlock_bh(&dev->con_mon_lock);
 
diff --git a/drivers/net/wireless/quantenna/qtnfmac/bus.h b/drivers/net/wireless/quantenna/qtnfmac/bus.h
index 0a1604683bab..323e47cea1e2 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/bus.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/bus.h
@@ -27,7 +27,8 @@ enum qtnf_fw_state {
 	QTNF_FW_STATE_FW_DNLD_DONE,
 	QTNF_FW_STATE_BOOT_DONE,
 	QTNF_FW_STATE_ACTIVE,
-	QTNF_FW_STATE_DEAD,
+	QTNF_FW_STATE_DETACHED,
+	QTNF_FW_STATE_EP_DEAD,
 };
 
 struct qtnf_bus;
diff --git a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
index 0398bece5782..220e2b710208 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
@@ -177,8 +177,6 @@ int qtnf_del_virtual_intf(struct wiphy *wiphy, struct wireless_dev *wdev)
 	vif->netdev->ieee80211_ptr = NULL;
 	vif->netdev = NULL;
 	vif->wdev.iftype = NL80211_IFTYPE_UNSPECIFIED;
-	eth_zero_addr(vif->mac_addr);
-	eth_zero_addr(vif->bssid);
 
 	return 0;
 }
@@ -216,10 +214,12 @@ static struct wireless_dev *qtnf_add_virtual_intf(struct wiphy *wiphy,
 		}
 
 		eth_zero_addr(vif->mac_addr);
+		eth_zero_addr(vif->bssid);
 		vif->bss_priority = QTNF_DEF_BSS_PRIORITY;
+		vif->sta_state = QTNF_STA_DISCONNECTED;
+		memset(&vif->wdev, 0, sizeof(vif->wdev));
 		vif->wdev.wiphy = wiphy;
 		vif->wdev.iftype = type;
-		vif->sta_state = QTNF_STA_DISCONNECTED;
 		break;
 	default:
 		pr_err("MAC%u: unsupported IF type %d\n", mac->macid, type);
@@ -255,8 +255,6 @@ static struct wireless_dev *qtnf_add_virtual_intf(struct wiphy *wiphy,
 	qtnf_cmd_send_del_intf(vif);
 err_cmd:
 	vif->wdev.iftype = NL80211_IFTYPE_UNSPECIFIED;
-	eth_zero_addr(vif->mac_addr);
-	eth_zero_addr(vif->bssid);
 
 	return ERR_PTR(-EFAULT);
 }
@@ -651,28 +649,37 @@ qtnf_disconnect(struct wiphy *wiphy, struct net_device *dev,
 {
 	struct qtnf_wmac *mac = wiphy_priv(wiphy);
 	struct qtnf_vif *vif;
-	int ret;
+	int ret = 0;
 
 	vif = qtnf_mac_get_base_vif(mac);
 	if (!vif) {
 		pr_err("MAC%u: primary VIF is not configured\n", mac->macid);
-		return -EFAULT;
+		ret = -EFAULT;
+		goto out;
 	}
 
-	if (vif->wdev.iftype != NL80211_IFTYPE_STATION)
-		return -EOPNOTSUPP;
+	if (vif->wdev.iftype != NL80211_IFTYPE_STATION) {
+		ret = -EOPNOTSUPP;
+		goto out;
+	}
+
+	qtnf_scan_done(mac, true);
 
 	if (vif->sta_state == QTNF_STA_DISCONNECTED)
-		return 0;
+		goto out;
 
 	ret = qtnf_cmd_send_disconnect(vif, reason_code);
 	if (ret) {
 		pr_err("VIF%u.%u: failed to disconnect\n", mac->macid,
 		       vif->vifid);
-		return ret;
+		goto out;
 	}
 
-	return 0;
+out:
+	if (vif->sta_state == QTNF_STA_CONNECTING)
+		vif->sta_state = QTNF_STA_DISCONNECTED;
+
+	return ret;
 }
 
 static int
@@ -813,6 +820,9 @@ static int qtnf_start_radar_detection(struct wiphy *wiphy,
 	struct qtnf_vif *vif = qtnf_netdev_get_priv(ndev);
 	int ret;
 
+	if (wiphy_ext_feature_isset(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD))
+		return -ENOTSUPP;
+
 	ret = qtnf_cmd_start_cac(vif, chandef, cac_time_ms);
 	if (ret)
 		pr_err("%s: failed to start CAC ret=%d\n", ndev->name, ret);
@@ -909,6 +919,9 @@ struct wiphy *qtnf_wiphy_allocate(struct qtnf_bus *bus)
 {
 	struct wiphy *wiphy;
 
+	if (bus->hw_info.hw_capab & QLINK_HW_CAPAB_DFS_OFFLOAD)
+		qtn_cfg80211_ops.start_radar_detection = NULL;
+
 	wiphy = wiphy_new(&qtn_cfg80211_ops, sizeof(struct qtnf_wmac));
 	if (!wiphy)
 		return NULL;
@@ -949,6 +962,7 @@ qtnf_wiphy_setup_if_comb(struct wiphy *wiphy, struct qtnf_mac_info *mac_info)
 int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 {
 	struct wiphy *wiphy = priv_to_wiphy(mac);
+	struct qtnf_mac_info *macinfo = &mac->macinfo;
 	int ret;
 
 	if (!wiphy) {
@@ -956,20 +970,20 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 		return -EFAULT;
 	}
 
-	wiphy->frag_threshold = mac->macinfo.frag_thr;
-	wiphy->rts_threshold = mac->macinfo.rts_thr;
-	wiphy->retry_short = mac->macinfo.sretry_limit;
-	wiphy->retry_long = mac->macinfo.lretry_limit;
-	wiphy->coverage_class = mac->macinfo.coverage_class;
+	wiphy->frag_threshold = macinfo->frag_thr;
+	wiphy->rts_threshold = macinfo->rts_thr;
+	wiphy->retry_short = macinfo->sretry_limit;
+	wiphy->retry_long = macinfo->lretry_limit;
+	wiphy->coverage_class = macinfo->coverage_class;
 
 	wiphy->max_scan_ssids = QTNF_MAX_SSID_LIST_LENGTH;
 	wiphy->max_scan_ie_len = QTNF_MAX_VSIE_LEN;
 	wiphy->mgmt_stypes = qtnf_mgmt_stypes;
 	wiphy->max_remain_on_channel_duration = 5000;
-	wiphy->max_acl_mac_addrs = mac->macinfo.max_acl_mac_addrs;
+	wiphy->max_acl_mac_addrs = macinfo->max_acl_mac_addrs;
 	wiphy->max_num_csa_counters = 2;
 
-	ret = qtnf_wiphy_setup_if_comb(wiphy, &mac->macinfo);
+	ret = qtnf_wiphy_setup_if_comb(wiphy, macinfo);
 	if (ret)
 		goto out;
 
@@ -982,15 +996,18 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 			WIPHY_FLAG_AP_UAPSD |
 			WIPHY_FLAG_HAS_CHANNEL_SWITCH;
 
+	if (hw_info->hw_capab & QLINK_HW_CAPAB_DFS_OFFLOAD)
+		wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD);
+
 	wiphy->probe_resp_offload = NL80211_PROBE_RESP_OFFLOAD_SUPPORT_WPS |
 				    NL80211_PROBE_RESP_OFFLOAD_SUPPORT_WPS2;
 
-	wiphy->available_antennas_tx = mac->macinfo.num_tx_chain;
-	wiphy->available_antennas_rx = mac->macinfo.num_rx_chain;
+	wiphy->available_antennas_tx = macinfo->num_tx_chain;
+	wiphy->available_antennas_rx = macinfo->num_rx_chain;
 
-	wiphy->max_ap_assoc_sta = mac->macinfo.max_ap_assoc_sta;
-	wiphy->ht_capa_mod_mask = &mac->macinfo.ht_cap_mod_mask;
-	wiphy->vht_capa_mod_mask = &mac->macinfo.vht_cap_mod_mask;
+	wiphy->max_ap_assoc_sta = macinfo->max_ap_assoc_sta;
+	wiphy->ht_capa_mod_mask = &macinfo->ht_cap_mod_mask;
+	wiphy->vht_capa_mod_mask = &macinfo->vht_cap_mod_mask;
 
 	ether_addr_copy(wiphy->perm_addr, mac->macaddr);
 
diff --git a/drivers/net/wireless/quantenna/qtnfmac/commands.c b/drivers/net/wireless/quantenna/qtnfmac/commands.c
index deca0060eb27..5eb143667539 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/commands.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/commands.c
@@ -55,6 +55,28 @@ static int qtnf_cmd_check_reply_header(const struct qlink_resp *resp,
 	return 0;
 }
 
+static int qtnf_cmd_resp_result_decode(enum qlink_cmd_result qcode)
+{
+	switch (qcode) {
+	case QLINK_CMD_RESULT_OK:
+		return 0;
+	case QLINK_CMD_RESULT_INVALID:
+		return -EINVAL;
+	case QLINK_CMD_RESULT_ENOTSUPP:
+		return -ENOTSUPP;
+	case QLINK_CMD_RESULT_ENOTFOUND:
+		return -ENOENT;
+	case QLINK_CMD_RESULT_EALREADY:
+		return -EALREADY;
+	case QLINK_CMD_RESULT_EADDRINUSE:
+		return -EADDRINUSE;
+	case QLINK_CMD_RESULT_EADDRNOTAVAIL:
+		return -EADDRNOTAVAIL;
+	default:
+		return -EFAULT;
+	}
+}
+
 static int qtnf_cmd_send_with_reply(struct qtnf_bus *bus,
 				    struct sk_buff *cmd_skb,
 				    struct sk_buff **response_skb,
@@ -80,6 +102,7 @@ static int qtnf_cmd_send_with_reply(struct qtnf_bus *bus,
 		pr_warn("VIF%u.%u: drop cmd 0x%.4X in fw state %d\n",
 			mac_id, vif_id, le16_to_cpu(cmd->cmd_id),
 			bus->fw_state);
+		dev_kfree_skb(cmd_skb);
 		return -ENODEV;
 	}
 
@@ -810,10 +833,10 @@ static int qtnf_cmd_send_add_change_intf(struct qtnf_vif *vif,
 	if (unlikely(ret))
 		goto out;
 
-	if (unlikely(res_code != QLINK_CMD_RESULT_OK)) {
+	ret = qtnf_cmd_resp_result_decode(res_code);
+	if (ret) {
 		pr_err("VIF%u.%u: CMD %d failed: %u\n", vif->mac->macid,
 		       vif->vifid, cmd_type, res_code);
-		ret = -EFAULT;
 		goto out;
 	}
 
@@ -2316,13 +2339,11 @@ int qtnf_cmd_send_connect(struct qtnf_vif *vif,
 	else
 		eth_zero_addr(cmd->prev_bssid);
 
-	if ((sme->bg_scan_period > 0) &&
-	    (sme->bg_scan_period <= QTNF_MAX_BG_SCAN_PERIOD))
+	if ((sme->bg_scan_period >= 0) &&
+	    (sme->bg_scan_period <= SHRT_MAX))
 		cmd->bg_scan_period = cpu_to_le16(sme->bg_scan_period);
-	else if (sme->bg_scan_period == -1)
-		cmd->bg_scan_period = cpu_to_le16(QTNF_DEFAULT_BG_SCAN_PERIOD);
 	else
-		cmd->bg_scan_period = 0; /* disabled */
+		cmd->bg_scan_period = cpu_to_le16(-1); /* use default value */
 
 	if (sme->flags & ASSOC_REQ_DISABLE_HT)
 		connect_flags |= QLINK_STA_CONNECT_DISABLE_HT;
diff --git a/drivers/net/wireless/quantenna/qtnfmac/commands.h b/drivers/net/wireless/quantenna/qtnfmac/commands.h
index 69a7d56f7e58..cf9274add26d 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/commands.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/commands.h
@@ -58,11 +58,6 @@ int qtnf_cmd_send_change_sta(struct qtnf_vif *vif, const u8 *mac,
 			     struct station_parameters *params);
 int qtnf_cmd_send_del_sta(struct qtnf_vif *vif,
 			  struct station_del_parameters *params);
-
-int qtnf_cmd_resp_parse(struct qtnf_bus *bus, struct sk_buff *resp_skb);
-int qtnf_cmd_resp_check(const struct qtnf_vif *vif,
-			const struct sk_buff *resp_skb, u16 cmd_id,
-			u16 *result, const u8 **payload, size_t *payload_size);
 int qtnf_cmd_send_scan(struct qtnf_wmac *mac);
 int qtnf_cmd_send_connect(struct qtnf_vif *vif,
 			  struct cfg80211_connect_params *sme);
diff --git a/drivers/net/wireless/quantenna/qtnfmac/core.c b/drivers/net/wireless/quantenna/qtnfmac/core.c
index cf26c15a84f8..a6a450984f9a 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/core.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/core.c
@@ -76,7 +76,7 @@ static int qtnf_netdev_close(struct net_device *ndev)
 
 /* Netdev handler for data transmission.
  */
-static int
+static netdev_tx_t
 qtnf_netdev_hard_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 {
 	struct qtnf_vif *vif;
@@ -394,7 +394,6 @@ int qtnf_core_net_attach(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 	dev = alloc_netdev_mqs(sizeof(struct qtnf_vif *), name,
 			       name_assign_type, ether_setup, 1, 1);
 	if (!dev) {
-		memset(&vif->wdev, 0, sizeof(vif->wdev));
 		vif->wdev.iftype = NL80211_IFTYPE_UNSPECIFIED;
 		return -ENOMEM;
 	}
@@ -629,7 +628,7 @@ void qtnf_core_detach(struct qtnf_bus *bus)
 	if (bus->fw_state == QTNF_FW_STATE_ACTIVE)
 		qtnf_cmd_send_deinit_fw(bus);
 
-	bus->fw_state = QTNF_FW_STATE_DEAD;
+	bus->fw_state = QTNF_FW_STATE_DETACHED;
 
 	if (bus->workqueue) {
 		flush_workqueue(bus->workqueue);
diff --git a/drivers/net/wireless/quantenna/qtnfmac/core.h b/drivers/net/wireless/quantenna/qtnfmac/core.h
index 3b884c80b6ab..214435448335 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/core.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/core.h
@@ -44,8 +44,6 @@
 #define QTNF_MAX_VSIE_LEN		255
 #define QTNF_MAX_INTF			8
 #define QTNF_MAX_EVENT_QUEUE_LEN	255
-#define QTNF_DEFAULT_BG_SCAN_PERIOD	300
-#define QTNF_MAX_BG_SCAN_PERIOD		0xffff
 #define QTNF_SCAN_TIMEOUT_SEC		15
 
 #define QTNF_DEF_BSS_PRIORITY		0
diff --git a/drivers/net/wireless/quantenna/qtnfmac/event.c b/drivers/net/wireless/quantenna/qtnfmac/event.c
index bcd415f96412..68da81bec4e9 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/event.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/event.c
@@ -34,12 +34,13 @@ qtnf_event_handle_sta_assoc(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 {
 	const u8 *sta_addr;
 	u16 frame_control;
-	struct station_info sinfo = { 0 };
+	struct station_info *sinfo;
 	size_t payload_len;
 	u16 tlv_type;
 	u16 tlv_value_len;
 	size_t tlv_full_len;
 	const struct qlink_tlv_hdr *tlv;
+	int ret = 0;
 
 	if (unlikely(len < sizeof(*sta_assoc))) {
 		pr_err("VIF%u.%u: payload is too short (%u < %zu)\n",
@@ -53,6 +54,10 @@ qtnf_event_handle_sta_assoc(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 		return -EPROTO;
 	}
 
+	sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL);
+	if (!sinfo)
+		return -ENOMEM;
+
 	sta_addr = sta_assoc->sta_addr;
 	frame_control = le16_to_cpu(sta_assoc->frame_control);
 
@@ -61,9 +66,9 @@ qtnf_event_handle_sta_assoc(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 
 	qtnf_sta_list_add(vif, sta_addr);
 
-	sinfo.assoc_req_ies = NULL;
-	sinfo.assoc_req_ies_len = 0;
-	sinfo.generation = vif->generation;
+	sinfo->assoc_req_ies = NULL;
+	sinfo->assoc_req_ies_len = 0;
+	sinfo->generation = vif->generation;
 
 	payload_len = len - sizeof(*sta_assoc);
 	tlv = (const struct qlink_tlv_hdr *)sta_assoc->ies;
@@ -73,23 +78,27 @@ qtnf_event_handle_sta_assoc(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 		tlv_value_len = le16_to_cpu(tlv->len);
 		tlv_full_len = tlv_value_len + sizeof(struct qlink_tlv_hdr);
 
-		if (tlv_full_len > payload_len)
-			return -EINVAL;
+		if (tlv_full_len > payload_len) {
+			ret = -EINVAL;
+			goto out;
+		}
 
 		if (tlv_type == QTN_TLV_ID_IE_SET) {
 			const struct qlink_tlv_ie_set *ie_set;
 			unsigned int ie_len;
 
-			if (payload_len < sizeof(*ie_set))
-				return -EINVAL;
+			if (payload_len < sizeof(*ie_set)) {
+				ret = -EINVAL;
+				goto out;
+			}
 
 			ie_set = (const struct qlink_tlv_ie_set *)tlv;
 			ie_len = tlv_value_len -
 				(sizeof(*ie_set) - sizeof(ie_set->hdr));
 
 			if (ie_set->type == QLINK_IE_SET_ASSOC_REQ && ie_len) {
-				sinfo.assoc_req_ies = ie_set->ie_data;
-				sinfo.assoc_req_ies_len = ie_len;
+				sinfo->assoc_req_ies = ie_set->ie_data;
+				sinfo->assoc_req_ies_len = ie_len;
 			}
 		}
 
@@ -97,13 +106,17 @@ qtnf_event_handle_sta_assoc(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 		tlv = (struct qlink_tlv_hdr *)(tlv->val + tlv_value_len);
 	}
 
-	if (payload_len)
-		return -EINVAL;
+	if (payload_len) {
+		ret = -EINVAL;
+		goto out;
+	}
 
-	cfg80211_new_sta(vif->netdev, sta_assoc->sta_addr, &sinfo,
+	cfg80211_new_sta(vif->netdev, sta_assoc->sta_addr, sinfo,
 			 GFP_KERNEL);
 
-	return 0;
+out:
+	kfree(sinfo);
+	return ret;
 }
 
 static int
@@ -198,11 +211,9 @@ qtnf_event_handle_bss_leave(struct qtnf_vif *vif,
 		return -EPROTO;
 	}
 
-	if (vif->sta_state != QTNF_STA_CONNECTED) {
-		pr_err("VIF%u.%u: BSS_LEAVE event when STA is not connected\n",
-		       vif->mac->macid, vif->vifid);
-		return -EPROTO;
-	}
+	if (vif->sta_state != QTNF_STA_CONNECTED)
+		pr_warn("VIF%u.%u: BSS_LEAVE event when STA is not connected\n",
+			vif->mac->macid, vif->vifid);
 
 	pr_debug("VIF%u.%u: disconnected\n", vif->mac->macid, vif->vifid);
 
@@ -443,6 +454,17 @@ static int qtnf_event_handle_radar(struct qtnf_vif *vif,
 		cfg80211_cac_event(vif->netdev, &chandef,
 				   NL80211_RADAR_CAC_ABORTED, GFP_KERNEL);
 		break;
+	case QLINK_RADAR_CAC_STARTED:
+		if (vif->wdev.cac_started)
+			break;
+
+		if (!wiphy_ext_feature_isset(wiphy,
+					     NL80211_EXT_FEATURE_DFS_OFFLOAD))
+			break;
+
+		cfg80211_cac_event(vif->netdev, &chandef,
+				   NL80211_RADAR_CAC_STARTED, GFP_KERNEL);
+		break;
 	default:
 		pr_warn("%s: unhandled radar event %u\n",
 			vif->netdev->name, ev->event);
diff --git a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c
index f117904d9120..3120d49df565 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c
@@ -751,8 +751,16 @@ static int qtnf_pcie_data_tx(struct qtnf_bus *bus, struct sk_buff *skb)
 static int qtnf_pcie_control_tx(struct qtnf_bus *bus, struct sk_buff *skb)
 {
 	struct qtnf_pcie_bus_priv *priv = (void *)get_bus_priv(bus);
+	int ret;
+
+	ret = qtnf_shm_ipc_send(&priv->shm_ipc_ep_in, skb->data, skb->len);
+
+	if (ret == -ETIMEDOUT) {
+		pr_err("EP firmware is dead\n");
+		bus->fw_state = QTNF_FW_STATE_EP_DEAD;
+	}
 
-	return qtnf_shm_ipc_send(&priv->shm_ipc_ep_in, skb->data, skb->len);
+	return ret;
 }
 
 static irqreturn_t qtnf_interrupt(int irq, void *data)
@@ -1185,6 +1193,10 @@ static void qtnf_fw_work_handler(struct work_struct *work)
 	if (qtnf_poll_state(&priv->bda->bda_ep_state, QTN_EP_FW_LOADRDY,
 			    QTN_FW_DL_TIMEOUT_MS)) {
 		pr_err("card is not ready\n");
+
+		if (!flashboot)
+			release_firmware(fw);
+
 		goto fw_load_fail;
 	}
 
@@ -1234,7 +1246,7 @@ static void qtnf_fw_work_handler(struct work_struct *work)
 	goto fw_load_exit;
 
 fw_load_fail:
-	bus->fw_state = QTNF_FW_STATE_DEAD;
+	bus->fw_state = QTNF_FW_STATE_DETACHED;
 
 fw_load_exit:
 	complete(&bus->firmware_init_complete);
@@ -1404,7 +1416,8 @@ static void qtnf_pcie_remove(struct pci_dev *pdev)
 
 	wait_for_completion(&bus->firmware_init_complete);
 
-	if (bus->fw_state == QTNF_FW_STATE_ACTIVE)
+	if (bus->fw_state == QTNF_FW_STATE_ACTIVE ||
+	    bus->fw_state == QTNF_FW_STATE_EP_DEAD)
 		qtnf_core_detach(bus);
 
 	priv = get_bus_priv(bus);
diff --git a/drivers/net/wireless/quantenna/qtnfmac/qlink.h b/drivers/net/wireless/quantenna/qtnfmac/qlink.h
index 9bf3ae4d1b3b..f85deda703fb 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/qlink.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/qlink.h
@@ -68,10 +68,12 @@ struct qlink_msg_header {
  * @QLINK_HW_CAPAB_STA_INACT_TIMEOUT: device implements a logic to kick-out
  *	associated STAs due to inactivity. Inactivity timeout period is taken
  *	from QLINK_CMD_START_AP parameters.
+ * @QLINK_HW_CAPAB_DFS_OFFLOAD: device implements DFS offload functionality
  */
 enum qlink_hw_capab {
-	QLINK_HW_CAPAB_REG_UPDATE = BIT(0),
-	QLINK_HW_CAPAB_STA_INACT_TIMEOUT = BIT(1),
+	QLINK_HW_CAPAB_REG_UPDATE		= BIT(0),
+	QLINK_HW_CAPAB_STA_INACT_TIMEOUT	= BIT(1),
+	QLINK_HW_CAPAB_DFS_OFFLOAD		= BIT(2),
 };
 
 enum qlink_iface_type {
@@ -672,6 +674,8 @@ enum qlink_cmd_result {
 	QLINK_CMD_RESULT_ENOTSUPP,
 	QLINK_CMD_RESULT_ENOTFOUND,
 	QLINK_CMD_RESULT_EALREADY,
+	QLINK_CMD_RESULT_EADDRINUSE,
+	QLINK_CMD_RESULT_EADDRNOTAVAIL,
 };
 
 /**
@@ -1031,6 +1035,7 @@ enum qlink_radar_event {
 	QLINK_RADAR_CAC_ABORTED,
 	QLINK_RADAR_NOP_FINISHED,
 	QLINK_RADAR_PRE_CAC_EXPIRED,
+	QLINK_RADAR_CAC_STARTED,
 };
 
 /**
diff --git a/drivers/net/wireless/quantenna/qtnfmac/trans.c b/drivers/net/wireless/quantenna/qtnfmac/trans.c
index ccddfebc508a..345f34ec9750 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/trans.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/trans.c
@@ -35,8 +35,10 @@ int qtnf_trans_send_cmd_with_resp(struct qtnf_bus *bus, struct sk_buff *cmd_skb,
 	bool resp_not_handled = true;
 	struct sk_buff *resp_skb = NULL;
 
-	if (unlikely(!response_skb))
+	if (unlikely(!response_skb)) {
+		dev_kfree_skb(cmd_skb);
 		return -EFAULT;
+	}
 
 	spin_lock(&ctl_node->resp_lock);
 	ctl_node->seq_num++;
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800.h b/drivers/net/wireless/ralink/rt2x00/rt2800.h
index 6a8c93fb6a43..b05ed2f3025a 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800.h
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800.h
@@ -94,6 +94,7 @@
 #define REV_RT3390E			0x0211
 #define REV_RT3593E			0x0211
 #define REV_RT5390F			0x0502
+#define REV_RT5370G			0x0503
 #define REV_RT5390R			0x1502
 #define REV_RT5592C			0x0221
 
@@ -1193,10 +1194,10 @@
 #define TX_PWR_CFG_3_MCS13		FIELD32(0x000000f0)
 #define TX_PWR_CFG_3_MCS14		FIELD32(0x00000f00)
 #define TX_PWR_CFG_3_MCS15		FIELD32(0x0000f000)
-#define TX_PWR_CFG_3_UKNOWN1		FIELD32(0x000f0000)
-#define TX_PWR_CFG_3_UKNOWN2		FIELD32(0x00f00000)
-#define TX_PWR_CFG_3_UKNOWN3		FIELD32(0x0f000000)
-#define TX_PWR_CFG_3_UKNOWN4		FIELD32(0xf0000000)
+#define TX_PWR_CFG_3_UNKNOWN1		FIELD32(0x000f0000)
+#define TX_PWR_CFG_3_UNKNOWN2		FIELD32(0x00f00000)
+#define TX_PWR_CFG_3_UNKNOWN3		FIELD32(0x0f000000)
+#define TX_PWR_CFG_3_UNKNOWN4		FIELD32(0xf0000000)
 /* bits for 3T devices */
 #define TX_PWR_CFG_3_MCS12_CH0		FIELD32(0x0000000f)
 #define TX_PWR_CFG_3_MCS12_CH1		FIELD32(0x000000f0)
@@ -1216,10 +1217,10 @@
  * TX_PWR_CFG_4:
  */
 #define TX_PWR_CFG_4			0x1324
-#define TX_PWR_CFG_4_UKNOWN5		FIELD32(0x0000000f)
-#define TX_PWR_CFG_4_UKNOWN6		FIELD32(0x000000f0)
-#define TX_PWR_CFG_4_UKNOWN7		FIELD32(0x00000f00)
-#define TX_PWR_CFG_4_UKNOWN8		FIELD32(0x0000f000)
+#define TX_PWR_CFG_4_UNKNOWN5		FIELD32(0x0000000f)
+#define TX_PWR_CFG_4_UNKNOWN6		FIELD32(0x000000f0)
+#define TX_PWR_CFG_4_UNKNOWN7		FIELD32(0x00000f00)
+#define TX_PWR_CFG_4_UNKNOWN8		FIELD32(0x0000f000)
 /* bits for 3T devices */
 #define TX_PWR_CFG_4_STBC4_CH0		FIELD32(0x0000000f)
 #define TX_PWR_CFG_4_STBC4_CH1		FIELD32(0x000000f0)
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
index 429d07b651dd..a567bc273ffc 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
@@ -1557,12 +1557,13 @@ static void rt2800_set_max_psdu_len(struct rt2x00_dev *rt2x00dev)
 	rt2800_register_write(rt2x00dev, MAX_LEN_CFG, reg);
 }
 
-int rt2800_sta_add(struct rt2x00_dev *rt2x00dev, struct ieee80211_vif *vif,
+int rt2800_sta_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		   struct ieee80211_sta *sta)
 {
-	int wcid;
-	struct rt2x00_sta *sta_priv = sta_to_rt2x00_sta(sta);
+	struct rt2x00_dev *rt2x00dev = hw->priv;
 	struct rt2800_drv_data *drv_data = rt2x00dev->drv_data;
+	struct rt2x00_sta *sta_priv = sta_to_rt2x00_sta(sta);
+	int wcid;
 
 	/*
 	 * Limit global maximum TX AMPDU length to smallest value of all
@@ -1608,8 +1609,10 @@ int rt2800_sta_add(struct rt2x00_dev *rt2x00dev, struct ieee80211_vif *vif,
 }
 EXPORT_SYMBOL_GPL(rt2800_sta_add);
 
-int rt2800_sta_remove(struct rt2x00_dev *rt2x00dev, struct ieee80211_sta *sta)
+int rt2800_sta_remove(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		      struct ieee80211_sta *sta)
 {
+	struct rt2x00_dev *rt2x00dev = hw->priv;
 	struct rt2800_drv_data *drv_data = rt2x00dev->drv_data;
 	struct rt2x00_sta *sta_priv = sta_to_rt2x00_sta(sta);
 	int wcid = sta_priv->wcid;
@@ -6220,8 +6223,9 @@ static void rt2800_init_bbp_53xx(struct rt2x00_dev *rt2x00dev)
 		rt2800_register_write(rt2x00dev, GPIO_CTRL, reg);
 	}
 
-	/* This chip has hardware antenna diversity*/
-	if (rt2x00_rt_rev_gte(rt2x00dev, RT5390, REV_RT5390R)) {
+	/* These chips have hardware RX antenna diversity */
+	if (rt2x00_rt_rev_gte(rt2x00dev, RT5390, REV_RT5390R) ||
+	    rt2x00_rt_rev_gte(rt2x00dev, RT5390, REV_RT5370G)) {
 		rt2800_bbp_write(rt2x00dev, 150, 0); /* Disable Antenna Software OFDM */
 		rt2800_bbp_write(rt2x00dev, 151, 0); /* Disable Antenna Software CCK */
 		rt2800_bbp_write(rt2x00dev, 154, 0); /* Clear previously selected antenna */
@@ -8748,7 +8752,9 @@ static int rt2800_init_eeprom(struct rt2x00_dev *rt2x00dev)
 		rt2x00dev->default_ant.rx = ANTENNA_A;
 	}
 
-	if (rt2x00_rt_rev_gte(rt2x00dev, RT5390, REV_RT5390R)) {
+	/* These chips have hardware RX antenna diversity */
+	if (rt2x00_rt_rev_gte(rt2x00dev, RT5390, REV_RT5390R) ||
+	    rt2x00_rt_rev_gte(rt2x00dev, RT5390, REV_RT5370G)) {
 		rt2x00dev->default_ant.tx = ANTENNA_HW_DIVERSITY; /* Unused */
 		rt2x00dev->default_ant.rx = ANTENNA_HW_DIVERSITY; /* Unused */
 	}
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.h b/drivers/net/wireless/ralink/rt2x00/rt2800lib.h
index 275e3969abdd..51d9c2a932cc 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.h
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.h
@@ -208,9 +208,10 @@ int rt2800_config_shared_key(struct rt2x00_dev *rt2x00dev,
 int rt2800_config_pairwise_key(struct rt2x00_dev *rt2x00dev,
 			       struct rt2x00lib_crypto *crypto,
 			       struct ieee80211_key_conf *key);
-int rt2800_sta_add(struct rt2x00_dev *rt2x00dev, struct ieee80211_vif *vif,
+int rt2800_sta_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		   struct ieee80211_sta *sta);
-int rt2800_sta_remove(struct rt2x00_dev *rt2x00dev, struct ieee80211_sta *sta);
+int rt2800_sta_remove(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		      struct ieee80211_sta *sta);
 void rt2800_config_filter(struct rt2x00_dev *rt2x00dev,
 			  const unsigned int filter_flags);
 void rt2800_config_intf(struct rt2x00_dev *rt2x00dev, struct rt2x00_intf *intf,
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800mmio.c b/drivers/net/wireless/ralink/rt2x00/rt2800mmio.c
index 1123e2bed803..e1a7ed7e4892 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800mmio.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800mmio.c
@@ -600,6 +600,7 @@ void rt2800mmio_kick_queue(struct data_queue *queue)
 	case QID_AC_VI:
 	case QID_AC_BE:
 	case QID_AC_BK:
+		WARN_ON_ONCE(rt2x00queue_empty(queue));
 		entry = rt2x00queue_get_entry(queue, Q_INDEX);
 		rt2x00mmio_register_write(rt2x00dev, TX_CTX_IDX(queue->qid),
 					  entry->entry_idx);
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800pci.c b/drivers/net/wireless/ralink/rt2x00/rt2800pci.c
index 1172eefd1c1a..71b1affc3885 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800pci.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800pci.c
@@ -311,8 +311,8 @@ static const struct ieee80211_ops rt2800pci_mac80211_ops = {
 	.get_stats		= rt2x00mac_get_stats,
 	.get_key_seq		= rt2800_get_key_seq,
 	.set_rts_threshold	= rt2800_set_rts_threshold,
-	.sta_add		= rt2x00mac_sta_add,
-	.sta_remove		= rt2x00mac_sta_remove,
+	.sta_add		= rt2800_sta_add,
+	.sta_remove		= rt2800_sta_remove,
 	.bss_info_changed	= rt2x00mac_bss_info_changed,
 	.conf_tx		= rt2800_conf_tx,
 	.get_tsf		= rt2800_get_tsf,
@@ -377,8 +377,6 @@ static const struct rt2x00lib_ops rt2800pci_rt2x00_ops = {
 	.config_erp		= rt2800_config_erp,
 	.config_ant		= rt2800_config_ant,
 	.config			= rt2800_config,
-	.sta_add		= rt2800_sta_add,
-	.sta_remove		= rt2800_sta_remove,
 };
 
 static const struct rt2x00_ops rt2800pci_ops = {
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800soc.c b/drivers/net/wireless/ralink/rt2x00/rt2800soc.c
index 6848ebc83534..a502816214ab 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800soc.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800soc.c
@@ -150,8 +150,8 @@ static const struct ieee80211_ops rt2800soc_mac80211_ops = {
 	.get_stats		= rt2x00mac_get_stats,
 	.get_key_seq		= rt2800_get_key_seq,
 	.set_rts_threshold	= rt2800_set_rts_threshold,
-	.sta_add		= rt2x00mac_sta_add,
-	.sta_remove		= rt2x00mac_sta_remove,
+	.sta_add		= rt2800_sta_add,
+	.sta_remove		= rt2800_sta_remove,
 	.bss_info_changed	= rt2x00mac_bss_info_changed,
 	.conf_tx		= rt2800_conf_tx,
 	.get_tsf		= rt2800_get_tsf,
@@ -216,8 +216,6 @@ static const struct rt2x00lib_ops rt2800soc_rt2x00_ops = {
 	.config_erp		= rt2800_config_erp,
 	.config_ant		= rt2800_config_ant,
 	.config			= rt2800_config,
-	.sta_add		= rt2800_sta_add,
-	.sta_remove		= rt2800_sta_remove,
 };
 
 static const struct rt2x00_ops rt2800soc_ops = {
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800usb.c b/drivers/net/wireless/ralink/rt2x00/rt2800usb.c
index d901a41d36e4..98a7313fea4a 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800usb.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800usb.c
@@ -797,8 +797,8 @@ static const struct ieee80211_ops rt2800usb_mac80211_ops = {
 	.get_stats		= rt2x00mac_get_stats,
 	.get_key_seq		= rt2800_get_key_seq,
 	.set_rts_threshold	= rt2800_set_rts_threshold,
-	.sta_add		= rt2x00mac_sta_add,
-	.sta_remove		= rt2x00mac_sta_remove,
+	.sta_add		= rt2800_sta_add,
+	.sta_remove		= rt2800_sta_remove,
 	.bss_info_changed	= rt2x00mac_bss_info_changed,
 	.conf_tx		= rt2800_conf_tx,
 	.get_tsf		= rt2800_get_tsf,
@@ -858,8 +858,6 @@ static const struct rt2x00lib_ops rt2800usb_rt2x00_ops = {
 	.config_erp		= rt2800_config_erp,
 	.config_ant		= rt2800_config_ant,
 	.config			= rt2800_config,
-	.sta_add		= rt2800_sta_add,
-	.sta_remove		= rt2800_sta_remove,
 };
 
 static void rt2800usb_queue_init(struct data_queue *queue)
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00.h b/drivers/net/wireless/ralink/rt2x00/rt2x00.h
index 1f38c338ca7a..a279a4363bc1 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00.h
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00.h
@@ -1457,10 +1457,6 @@ int rt2x00mac_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 #else
 #define rt2x00mac_set_key	NULL
 #endif /* CONFIG_RT2X00_LIB_CRYPTO */
-int rt2x00mac_sta_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-		      struct ieee80211_sta *sta);
-int rt2x00mac_sta_remove(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-			 struct ieee80211_sta *sta);
 void rt2x00mac_sw_scan_start(struct ieee80211_hw *hw,
 			     struct ieee80211_vif *vif,
 			     const u8 *mac_addr);
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c b/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c
index a971bc7a6b63..c380c1f56ba6 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c
@@ -739,8 +739,7 @@ void rt2x00mac_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		return;
 
 	tx_queue_for_each(rt2x00dev, queue)
-		if (!rt2x00queue_empty(queue))
-			rt2x00queue_flush_queue(queue, drop);
+		rt2x00queue_flush_queue(queue, drop);
 }
 EXPORT_SYMBOL_GPL(rt2x00mac_flush);
 
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c b/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c
index a6884e73d2ab..710e9641552e 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c
@@ -372,16 +372,15 @@ static void rt2x00queue_create_tx_descriptor_ht(struct rt2x00_dev *rt2x00dev,
 
 	/*
 	 * Determine IFS values
-	 * - Use TXOP_BACKOFF for probe and management frames except beacons
+	 * - Use TXOP_BACKOFF for management frames except beacons
 	 * - Use TXOP_SIFS for fragment bursts
 	 * - Use TXOP_HTTXOP for everything else
 	 *
 	 * Note: rt2800 devices won't use CTS protection (if used)
 	 * for frames not transmitted with TXOP_HTTXOP
 	 */
-	if ((ieee80211_is_mgmt(hdr->frame_control) &&
-	     !ieee80211_is_beacon(hdr->frame_control)) ||
-	    (tx_info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE))
+	if (ieee80211_is_mgmt(hdr->frame_control) &&
+	    !ieee80211_is_beacon(hdr->frame_control))
 		txdesc->u.ht.txop = TXOP_BACKOFF;
 	else if (!(tx_info->flags & IEEE80211_TX_CTL_FIRST_FRAGMENT))
 		txdesc->u.ht.txop = TXOP_SIFS;
@@ -1000,6 +999,8 @@ void rt2x00queue_flush_queue(struct data_queue *queue, bool drop)
 		(queue->qid == QID_AC_BE) ||
 		(queue->qid == QID_AC_BK);
 
+	if (rt2x00queue_empty(queue))
+		return;
 
 	/*
 	 * If we are not supposed to drop any pending
diff --git a/drivers/net/wireless/ray_cs.c b/drivers/net/wireless/ray_cs.c
index 7f9b16b97ea3..a7e0a17aa7e8 100644
--- a/drivers/net/wireless/ray_cs.c
+++ b/drivers/net/wireless/ray_cs.c
@@ -2663,19 +2663,6 @@ static int ray_cs_proc_show(struct seq_file *m, void *v)
 	}
 	return 0;
 }
-
-static int ray_cs_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ray_cs_proc_show, NULL);
-}
-
-static const struct file_operations ray_cs_proc_fops = {
-	.owner = THIS_MODULE,
-	.open = ray_cs_proc_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
 #endif
 /*===========================================================================*/
 static int build_auth_frame(ray_dev_t *local, UCHAR *dest, int auth_type)
@@ -2814,7 +2801,7 @@ static int __init init_ray_cs(void)
 #ifdef CONFIG_PROC_FS
 	proc_mkdir("driver/ray_cs", NULL);
 
-	proc_create("driver/ray_cs/ray_cs", 0, NULL, &ray_cs_proc_fops);
+	proc_create_single("driver/ray_cs/ray_cs", 0, NULL, ray_cs_proc_show);
 	proc_create("driver/ray_cs/essid", 0200, NULL, &ray_cs_essid_proc_fops);
 	proc_create_data("driver/ray_cs/net_type", 0200, NULL, &int_proc_fops,
 			 &net_type);
diff --git a/drivers/net/wireless/realtek/rtlwifi/base.c b/drivers/net/wireless/realtek/rtlwifi/base.c
index 762a29cdf7ad..39c817eddd78 100644
--- a/drivers/net/wireless/realtek/rtlwifi/base.c
+++ b/drivers/net/wireless/realtek/rtlwifi/base.c
@@ -396,7 +396,6 @@ static void _rtl_init_mac80211(struct ieee80211_hw *hw)
 	ieee80211_hw_set(hw, SIGNAL_DBM);
 	ieee80211_hw_set(hw, RX_INCLUDES_FCS);
 	ieee80211_hw_set(hw, AMPDU_AGGREGATION);
-	ieee80211_hw_set(hw, CONNECTION_MONITOR);
 	ieee80211_hw_set(hw, MFP_CAPABLE);
 	ieee80211_hw_set(hw, REPORTS_TX_ACK_STATUS);
 	ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU);
@@ -572,8 +571,9 @@ int rtl_init_core(struct ieee80211_hw *hw)
 	spin_lock_init(&rtlpriv->locks.iqk_lock);
 	/* <5> init list */
 	INIT_LIST_HEAD(&rtlpriv->entry_list);
-	INIT_LIST_HEAD(&rtlpriv->c2hcmd_list);
 	INIT_LIST_HEAD(&rtlpriv->scan_list.list);
+	skb_queue_head_init(&rtlpriv->tx_report.queue);
+	skb_queue_head_init(&rtlpriv->c2hcmd_queue);
 
 	rtlmac->link_state = MAC80211_NOLINK;
 
@@ -585,11 +585,14 @@ int rtl_init_core(struct ieee80211_hw *hw)
 EXPORT_SYMBOL_GPL(rtl_init_core);
 
 static void rtl_free_entries_from_scan_list(struct ieee80211_hw *hw);
+static void rtl_free_entries_from_ack_queue(struct ieee80211_hw *hw,
+					    bool timeout);
 
 void rtl_deinit_core(struct ieee80211_hw *hw)
 {
 	rtl_c2hcmd_launcher(hw, 0);
 	rtl_free_entries_from_scan_list(hw);
+	rtl_free_entries_from_ack_queue(hw, false);
 }
 EXPORT_SYMBOL_GPL(rtl_deinit_core);
 
@@ -1575,22 +1578,52 @@ u8 rtl_is_special_data(struct ieee80211_hw *hw, struct sk_buff *skb, u8 is_tx,
 }
 EXPORT_SYMBOL_GPL(rtl_is_special_data);
 
+void rtl_tx_ackqueue(struct ieee80211_hw *hw, struct sk_buff *skb)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_tx_report *tx_report = &rtlpriv->tx_report;
+
+	__skb_queue_tail(&tx_report->queue, skb);
+}
+EXPORT_SYMBOL_GPL(rtl_tx_ackqueue);
+
+static void rtl_tx_status(struct ieee80211_hw *hw, struct sk_buff *skb,
+			  bool ack)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct ieee80211_tx_info *info;
+
+	info = IEEE80211_SKB_CB(skb);
+	ieee80211_tx_info_clear_status(info);
+	if (ack) {
+		RT_TRACE(rtlpriv, COMP_TX_REPORT, DBG_LOUD,
+			 "tx report: ack\n");
+		info->flags |= IEEE80211_TX_STAT_ACK;
+	} else {
+		RT_TRACE(rtlpriv, COMP_TX_REPORT, DBG_LOUD,
+			 "tx report: not ack\n");
+		info->flags &= ~IEEE80211_TX_STAT_ACK;
+	}
+	ieee80211_tx_status_irqsafe(hw, skb);
+}
+
 bool rtl_is_tx_report_skb(struct ieee80211_hw *hw, struct sk_buff *skb)
 {
 	u16 ether_type;
 	const u8 *ether_type_ptr;
+	__le16 fc = rtl_get_fc(skb);
 
 	ether_type_ptr = rtl_skb_ether_type_ptr(hw, skb, true);
 	ether_type = be16_to_cpup((__be16 *)ether_type_ptr);
 
-	/* EAPOL */
-	if (ether_type == ETH_P_PAE)
+	if (ether_type == ETH_P_PAE || ieee80211_is_nullfunc(fc))
 		return true;
 
 	return false;
 }
 
-static u16 rtl_get_tx_report_sn(struct ieee80211_hw *hw)
+static u16 rtl_get_tx_report_sn(struct ieee80211_hw *hw,
+				struct rtlwifi_tx_info *tx_info)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_tx_report *tx_report = &rtlpriv->tx_report;
@@ -1604,29 +1637,33 @@ static u16 rtl_get_tx_report_sn(struct ieee80211_hw *hw)
 
 	tx_report->last_sent_sn = sn;
 	tx_report->last_sent_time = jiffies;
-
+	tx_info->sn = sn;
+	tx_info->send_time = tx_report->last_sent_time;
 	RT_TRACE(rtlpriv, COMP_TX_REPORT, DBG_DMESG,
 		 "Send TX-Report sn=0x%X\n", sn);
 
 	return sn;
 }
 
-void rtl_get_tx_report(struct rtl_tcb_desc *ptcb_desc, u8 *pdesc,
-		       struct ieee80211_hw *hw)
+void rtl_set_tx_report(struct rtl_tcb_desc *ptcb_desc, u8 *pdesc,
+		       struct ieee80211_hw *hw, struct rtlwifi_tx_info *tx_info)
 {
 	if (ptcb_desc->use_spe_rpt) {
-		u16 sn = rtl_get_tx_report_sn(hw);
+		u16 sn = rtl_get_tx_report_sn(hw, tx_info);
 
 		SET_TX_DESC_SPE_RPT(pdesc, 1);
 		SET_TX_DESC_SW_DEFINE(pdesc, sn);
 	}
 }
-EXPORT_SYMBOL_GPL(rtl_get_tx_report);
+EXPORT_SYMBOL_GPL(rtl_set_tx_report);
 
 void rtl_tx_report_handler(struct ieee80211_hw *hw, u8 *tmp_buf, u8 c2h_cmd_len)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_tx_report *tx_report = &rtlpriv->tx_report;
+	struct rtlwifi_tx_info *tx_info;
+	struct sk_buff_head *queue = &tx_report->queue;
+	struct sk_buff *skb;
 	u16 sn;
 	u8 st, retry;
 
@@ -1642,6 +1679,14 @@ void rtl_tx_report_handler(struct ieee80211_hw *hw, u8 *tmp_buf, u8 c2h_cmd_len)
 
 	tx_report->last_recv_sn = sn;
 
+	skb_queue_walk(queue, skb) {
+		tx_info = rtl_tx_skb_cb_info(skb);
+		if (tx_info->sn == sn) {
+			skb_unlink(skb, queue);
+			rtl_tx_status(hw, skb, st == 0);
+			break;
+		}
+	}
 	RT_TRACE(rtlpriv, COMP_TX_REPORT, DBG_DMESG,
 		 "Recv TX-Report st=0x%02X sn=0x%X retry=0x%X\n",
 		 st, sn, retry);
@@ -1909,6 +1954,25 @@ static void rtl_free_entries_from_scan_list(struct ieee80211_hw *hw)
 	}
 }
 
+static void rtl_free_entries_from_ack_queue(struct ieee80211_hw *hw,
+					    bool chk_timeout)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_tx_report *tx_report = &rtlpriv->tx_report;
+	struct sk_buff_head *queue = &tx_report->queue;
+	struct sk_buff *skb, *tmp;
+	struct rtlwifi_tx_info *tx_info;
+
+	skb_queue_walk_safe(queue, skb, tmp) {
+		tx_info = rtl_tx_skb_cb_info(skb);
+		if (chk_timeout &&
+		    time_after(tx_info->send_time + HZ, jiffies))
+			continue;
+		skb_unlink(skb, queue);
+		rtl_tx_status(hw, skb, false);
+	}
+}
+
 void rtl_scan_list_expire(struct ieee80211_hw *hw)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
@@ -2173,6 +2237,9 @@ void rtl_watchdog_wq_callback(void *data)
 
 	/* <6> scan list */
 	rtl_scan_list_expire(hw);
+
+	/* <7> check ack queue */
+	rtl_free_entries_from_ack_queue(hw, true);
 }
 
 void rtl_watch_dog_timer_callback(struct timer_list *t)
@@ -2195,81 +2262,122 @@ void rtl_fwevt_wq_callback(void *data)
 	rtlpriv->cfg->ops->c2h_command_handle(hw);
 }
 
-void rtl_c2hcmd_enqueue(struct ieee80211_hw *hw, u8 tag, u8 len, u8 *val)
-{
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	unsigned long flags;
-	struct rtl_c2hcmd *c2hcmd;
+static void rtl_c2h_content_parsing(struct ieee80211_hw *hw,
+				    struct sk_buff *skb);
 
-	c2hcmd = kmalloc(sizeof(*c2hcmd),
-			 in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+static bool rtl_c2h_fast_cmd(struct ieee80211_hw *hw, struct sk_buff *skb)
+{
+	u8 cmd_id = GET_C2H_CMD_ID(skb->data);
 
-	if (!c2hcmd)
-		goto label_err;
+	switch (cmd_id) {
+	case C2H_BT_MP:
+		return true;
+	default:
+		break;
+	}
 
-	c2hcmd->val = kmalloc(len,
-			      in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+	return false;
+}
 
-	if (!c2hcmd->val)
-		goto label_err2;
+void rtl_c2hcmd_enqueue(struct ieee80211_hw *hw, struct sk_buff *skb)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	unsigned long flags;
 
-	/* fill data */
-	c2hcmd->tag = tag;
-	c2hcmd->len = len;
-	memcpy(c2hcmd->val, val, len);
+	if (rtl_c2h_fast_cmd(hw, skb)) {
+		rtl_c2h_content_parsing(hw, skb);
+		return;
+	}
 
 	/* enqueue */
 	spin_lock_irqsave(&rtlpriv->locks.c2hcmd_lock, flags);
 
-	list_add_tail(&c2hcmd->list, &rtlpriv->c2hcmd_list);
+	__skb_queue_tail(&rtlpriv->c2hcmd_queue, skb);
 
 	spin_unlock_irqrestore(&rtlpriv->locks.c2hcmd_lock, flags);
 
 	/* wake up wq */
 	queue_delayed_work(rtlpriv->works.rtl_wq, &rtlpriv->works.c2hcmd_wq, 0);
-
-	return;
-
-label_err2:
-	kfree(c2hcmd);
-
-label_err:
-	RT_TRACE(rtlpriv, COMP_CMD, DBG_WARNING,
-		 "C2H cmd enqueue fail.\n");
 }
 EXPORT_SYMBOL(rtl_c2hcmd_enqueue);
 
+static void rtl_c2h_content_parsing(struct ieee80211_hw *hw,
+				    struct sk_buff *skb)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal_ops *hal_ops = rtlpriv->cfg->ops;
+	const struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
+	u8 cmd_id, cmd_seq, cmd_len;
+	u8 *cmd_buf = NULL;
+
+	cmd_id = GET_C2H_CMD_ID(skb->data);
+	cmd_seq = GET_C2H_SEQ(skb->data);
+	cmd_len = skb->len - C2H_DATA_OFFSET;
+	cmd_buf = GET_C2H_DATA_PTR(skb->data);
+
+	switch (cmd_id) {
+	case C2H_DBG:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD, "[C2H], C2H_DBG!!\n");
+		break;
+	case C2H_TXBF:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+			 "[C2H], C2H_TXBF!!\n");
+		break;
+	case C2H_TX_REPORT:
+		rtl_tx_report_handler(hw, cmd_buf, cmd_len);
+		break;
+	case C2H_RA_RPT:
+		if (hal_ops->c2h_ra_report_handler)
+			hal_ops->c2h_ra_report_handler(hw, cmd_buf, cmd_len);
+		break;
+	case C2H_BT_INFO:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+			 "[C2H], C2H_BT_INFO!!\n");
+		if (rtlpriv->cfg->ops->get_btc_status())
+			btc_ops->btc_btinfo_notify(rtlpriv, cmd_buf, cmd_len);
+		break;
+	case C2H_BT_MP:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+			 "[C2H], C2H_BT_MP!!\n");
+		if (rtlpriv->cfg->ops->get_btc_status())
+			btc_ops->btc_btmpinfo_notify(rtlpriv, cmd_buf, cmd_len);
+		break;
+	default:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+			 "[C2H], Unknown packet!! cmd_id(%#X)!\n", cmd_id);
+		break;
+	}
+}
+
 void rtl_c2hcmd_launcher(struct ieee80211_hw *hw, int exec)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct sk_buff *skb;
 	unsigned long flags;
-	struct rtl_c2hcmd *c2hcmd;
 	int i;
 
 	for (i = 0; i < 200; i++) {
 		/* dequeue a task */
 		spin_lock_irqsave(&rtlpriv->locks.c2hcmd_lock, flags);
 
-		c2hcmd = list_first_entry_or_null(&rtlpriv->c2hcmd_list,
-						  struct rtl_c2hcmd, list);
-
-		if (c2hcmd)
-			list_del(&c2hcmd->list);
+		skb = __skb_dequeue(&rtlpriv->c2hcmd_queue);
 
 		spin_unlock_irqrestore(&rtlpriv->locks.c2hcmd_lock, flags);
 
 		/* do it */
-		if (!c2hcmd)
+		if (!skb)
 			break;
 
-		if (rtlpriv->cfg->ops->c2h_content_parsing && exec)
-			rtlpriv->cfg->ops->c2h_content_parsing(hw,
-					c2hcmd->tag, c2hcmd->len, c2hcmd->val);
+		RT_TRACE(rtlpriv, COMP_FW, DBG_DMESG, "C2H rx_desc_shift=%d\n",
+			 *((u8 *)skb->cb));
+		RT_PRINT_DATA(rtlpriv, COMP_FW, DBG_DMESG,
+			      "C2H data: ", skb->data, skb->len);
 
-		/* free */
-		kfree(c2hcmd->val);
+		if (exec)
+			rtl_c2h_content_parsing(hw, skb);
 
-		kfree(c2hcmd);
+		/* free */
+		dev_kfree_skb_any(skb);
 	}
 }
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/base.h b/drivers/net/wireless/realtek/rtlwifi/base.h
index acc924635818..912f205779c3 100644
--- a/drivers/net/wireless/realtek/rtlwifi/base.h
+++ b/drivers/net/wireless/realtek/rtlwifi/base.h
@@ -130,9 +130,10 @@ bool rtl_tx_mgmt_proc(struct ieee80211_hw *hw, struct sk_buff *skb);
 u8 rtl_is_special_data(struct ieee80211_hw *hw, struct sk_buff *skb, u8 is_tx,
 		       bool is_enc);
 
+void rtl_tx_ackqueue(struct ieee80211_hw *hw, struct sk_buff *skb);
 bool rtl_is_tx_report_skb(struct ieee80211_hw *hw, struct sk_buff *skb);
-void rtl_get_tx_report(struct rtl_tcb_desc *ptcb_desc, u8 *pdesc,
-		       struct ieee80211_hw *hw);
+void rtl_set_tx_report(struct rtl_tcb_desc *ptcb_desc, u8 *pdesc,
+		       struct ieee80211_hw *hw, struct rtlwifi_tx_info *info);
 void rtl_tx_report_handler(struct ieee80211_hw *hw, u8 *tmp_buf,
 			   u8 c2h_cmd_len);
 bool rtl_check_tx_report_acked(struct ieee80211_hw *hw);
@@ -160,7 +161,7 @@ void rtl_watchdog_wq_callback(void *data);
 void rtl_fwevt_wq_callback(void *data);
 void rtl_c2hcmd_wq_callback(void *data);
 void rtl_c2hcmd_launcher(struct ieee80211_hw *hw, int exec);
-void rtl_c2hcmd_enqueue(struct ieee80211_hw *hw, u8 tag, u8 len, u8 *val);
+void rtl_c2hcmd_enqueue(struct ieee80211_hw *hw, struct sk_buff *skb);
 
 u8 rtl_mrate_idx_to_arfr_id(struct ieee80211_hw *hw, u8 rate_index,
 			    enum wireless_mode wirelessmode);
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.c
index 8fce371749d3..f22fec093f1d 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.c
@@ -1783,7 +1783,7 @@ static void btc8192e2ant_tdma_duration_adjust(struct btc_coexist *btcoexist,
 		bool scan = false, link = false, roam = false;
 
 		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
-			 "[BTCoex], PsTdma type dismatch!!!, ");
+			 "[BTCoex], PsTdma type mismatch!!!, ");
 		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
 			 "curPsTdma=%d, recordPsTdma=%d\n",
 			 coex_dm->cur_ps_tdma, coex_dm->tdma_adj_type);
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c
index 73ec31972944..df3facc8e5a4 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c
@@ -2766,7 +2766,7 @@ static void btc8723b2ant_tdma_duration_adjust(struct btc_coexist *btcoexist,
 	if (coex_dm->cur_ps_tdma != coex_dm->ps_tdma_du_adj_type) {
 		bool scan = false, link = false, roam = false;
 		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
-			 "[BTCoex], PsTdma type dismatch!!!, curPsTdma=%d, recordPsTdma=%d\n",
+			 "[BTCoex], PsTdma type mismatch!!!, curPsTdma=%d, recordPsTdma=%d\n",
 			 coex_dm->cur_ps_tdma, coex_dm->ps_tdma_du_adj_type);
 
 		btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
@@ -2876,25 +2876,10 @@ static void btc8723b2ant_action_hid(struct btc_coexist *btcoexist)
 		btc8723b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 13);
 
 	/* sw mechanism */
-	if (BTC_WIFI_BW_HT40 == wifi_bw) {
-		if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) ||
-		    (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) {
-			btc8723b2ant_sw_mechanism(btcoexist, true, true,
-						  false, false);
-		} else {
-			btc8723b2ant_sw_mechanism(btcoexist, true, true,
-						  false, false);
-		}
-	} else {
-		if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) ||
-		    (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) {
-			btc8723b2ant_sw_mechanism(btcoexist, false, true,
-						  false, false);
-		} else {
-			btc8723b2ant_sw_mechanism(btcoexist, false, true,
-						  false, false);
-		}
-	}
+	if (wifi_bw == BTC_WIFI_BW_HT40)
+		btc8723b2ant_sw_mechanism(btcoexist, true, true, false, false);
+	else
+		btc8723b2ant_sw_mechanism(btcoexist, false, true, false, false);
 }
 
 /* A2DP only / PAN(EDR) only/ A2DP+PAN(HS) */
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.c
index 202597cf8915..b5d65872db84 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.c
@@ -1584,11 +1584,7 @@ static void btc8821a1ant_act_bt_sco_hid_only_busy(struct btc_coexist *btcoexist,
 	/* tdma and coex table */
 	btc8821a1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 5);
 
-	if (BT_8821A_1ANT_WIFI_STATUS_NON_CONNECTED_ASSO_AUTH_SCAN ==
-	    wifi_status)
-		btc8821a1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 1);
-	else
-		btc8821a1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 1);
+	btc8821a1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 1);
 }
 
 static void btc8821a1ant_act_wifi_con_bt_acl_busy(struct btc_coexist *btcoexist,
@@ -1991,16 +1987,9 @@ static void btc8821a1ant_run_coexist_mechanism(struct btc_coexist *btcoexist)
 			wifi_rssi_state =
 				btc8821a1ant_wifi_rssi_state(btcoexist, 1, 2,
 							     30, 0);
-			if ((wifi_rssi_state == BTC_RSSI_STATE_HIGH) ||
-			    (wifi_rssi_state == BTC_RSSI_STATE_STAY_HIGH)) {
-				btc8821a1ant_limited_tx(btcoexist,
-							NORMAL_EXEC, 1, 1,
-							0, 1);
-			} else {
-				btc8821a1ant_limited_tx(btcoexist,
-							NORMAL_EXEC, 1, 1,
-							0, 1);
-			}
+			btc8821a1ant_limited_tx(btcoexist,
+						NORMAL_EXEC, 1, 1,
+						0, 1);
 		} else {
 			btc8821a1ant_limited_tx(btcoexist, NORMAL_EXEC,
 						0, 0, 0, 0);
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.c
index 2202d5e18977..01a9d303603b 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.c
@@ -2614,7 +2614,7 @@ static void btc8821a2ant_tdma_duration_adjust(struct btc_coexist *btcoexist,
 		bool scan = false, link = false, roam = false;
 
 		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
-			 "[BTCoex], PsTdma type dismatch!!!, cur_ps_tdma = %d, recordPsTdma = %d\n",
+			 "[BTCoex], PsTdma type mismatch!!!, cur_ps_tdma = %d, recordPsTdma = %d\n",
 			 coex_dm->cur_ps_tdma, coex_dm->ps_tdma_du_adj_type);
 
 		btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c
index 57bb8f049e59..ae13bcfb3bf0 100644
--- a/drivers/net/wireless/realtek/rtlwifi/pci.c
+++ b/drivers/net/wireless/realtek/rtlwifi/pci.c
@@ -619,12 +619,15 @@ static void _rtl_pci_tx_isr(struct ieee80211_hw *hw, int prio)
 			rtlpriv->link_info.tidtx_inperiod[tid]++;
 
 		info = IEEE80211_SKB_CB(skb);
-		ieee80211_tx_info_clear_status(info);
 
-		info->flags |= IEEE80211_TX_STAT_ACK;
-		/*info->status.rates[0].count = 1; */
-
-		ieee80211_tx_status_irqsafe(hw, skb);
+		if (likely(!ieee80211_is_nullfunc(fc))) {
+			ieee80211_tx_info_clear_status(info);
+			info->flags |= IEEE80211_TX_STAT_ACK;
+			/*info->status.rates[0].count = 1; */
+			ieee80211_tx_status_irqsafe(hw, skb);
+		} else {
+			rtl_tx_ackqueue(hw, skb);
+		}
 
 		if ((ring->entries - skb_queue_len(&ring->queue)) <= 4) {
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
@@ -827,9 +830,8 @@ static void _rtl_pci_rx_interrupt(struct ieee80211_hw *hw)
 			goto new_trx_end;
 		}
 		/* handle command packet here */
-		if (rtlpriv->cfg->ops->rx_command_packet &&
-		    rtlpriv->cfg->ops->rx_command_packet(hw, &stats, skb)) {
-			dev_kfree_skb_any(skb);
+		if (stats.packet_report_type == C2H_PACKET) {
+			rtl_c2hcmd_enqueue(hw, skb);
 			goto new_trx_end;
 		}
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/def.h b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/def.h
index 0532b9852444..45c866d3ca88 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/def.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/def.h
@@ -137,13 +137,6 @@ enum version_8188e {
 	VERSION_UNKNOWN = 0xFF,
 };
 
-enum rx_packet_type {
-	NORMAL_RX,
-	TX_REPORT1,
-	TX_REPORT2,
-	HIS_REPORT,
-};
-
 enum rtl819x_loopback_e {
 	RTL819X_NO_LOOPBACK = 0,
 	RTL819X_MAC_LOOPBACK = 1,
@@ -183,31 +176,6 @@ enum interface_select_pci {
 	INTF_SEL3_RSV = 3,
 };
 
-enum hal_fw_c2h_cmd_id {
-	HAL_FW_C2H_CMD_READ_MACREG = 0,
-	HAL_FW_C2H_CMD_READ_BBREG = 1,
-	HAL_FW_C2H_CMD_READ_RFREG = 2,
-	HAL_FW_C2H_CMD_READ_EEPROM = 3,
-	HAL_FW_C2H_CMD_READ_EFUSE = 4,
-	HAL_FW_C2H_CMD_READ_CAM = 5,
-	HAL_FW_C2H_CMD_GET_BASICRATE = 6,
-	HAL_FW_C2H_CMD_GET_DATARATE = 7,
-	HAL_FW_C2H_CMD_SURVEY = 8,
-	HAL_FW_C2H_CMD_SURVEYDONE = 9,
-	HAL_FW_C2H_CMD_JOINBSS = 10,
-	HAL_FW_C2H_CMD_ADDSTA = 11,
-	HAL_FW_C2H_CMD_DELSTA = 12,
-	HAL_FW_C2H_CMD_ATIMDONE = 13,
-	HAL_FW_C2H_CMD_TX_REPORT = 14,
-	HAL_FW_C2H_CMD_CCX_REPORT = 15,
-	HAL_FW_C2H_CMD_DTM_REPORT = 16,
-	HAL_FW_C2H_CMD_TX_RATE_STATISTICS = 17,
-	HAL_FW_C2H_CMD_C2HLBK = 18,
-	HAL_FW_C2H_CMD_C2HDBG = 19,
-	HAL_FW_C2H_CMD_C2HFEEDBACK = 20,
-	HAL_FW_C2H_CMD_MAX
-};
-
 enum rtl_desc_qsel {
 	QSLT_BK = 0x2,
 	QSLT_BE = 0x0,
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c
index 82681b96ef93..8c15ffd3568b 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c
@@ -263,8 +263,6 @@ static struct rtl_hal_ops rtl8188ee_hal_ops = {
 	.get_rfreg = rtl88e_phy_query_rf_reg,
 	.set_rfreg = rtl88e_phy_set_rf_reg,
 	.get_btc_status = rtl88e_get_btc_status,
-	.rx_command_packet = rtl88ee_rx_command_packet,
-
 };
 
 static struct rtl_mod_params rtl88ee_mod_params = {
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/trx.c
index 9670732b2bc6..4c1f8b08fc10 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/trx.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/trx.c
@@ -850,10 +850,3 @@ void rtl88ee_tx_polling(struct ieee80211_hw *hw, u8 hw_queue)
 			       BIT(0) << (hw_queue));
 	}
 }
-
-u32 rtl88ee_rx_command_packet(struct ieee80211_hw *hw,
-			      const struct rtl_stats *status,
-			      struct sk_buff *skb)
-{
-	return 0;
-}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/trx.h b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/trx.h
index f902d6769aa8..127ba977206f 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/trx.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/trx.h
@@ -790,8 +790,4 @@ void rtl88ee_tx_polling(struct ieee80211_hw *hw, u8 hw_queue);
 void rtl88ee_tx_fill_cmddesc(struct ieee80211_hw *hw, u8 *pdesc,
 			     bool firstseg, bool lastseg,
 			     struct sk_buff *skb);
-u32 rtl88ee_rx_command_packet(struct ieee80211_hw *hw,
-			      const struct rtl_stats *status,
-			      struct sk_buff *skb);
-
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/def.h b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/def.h
index b90aaf128072..d2005d7e9ad2 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/def.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/def.h
@@ -142,31 +142,6 @@ enum interface_select_pci {
 	INTF_SEL3_RSV = 3,
 };
 
-enum hal_fw_c2h_cmd_id {
-	HAL_FW_C2H_CMD_Read_MACREG = 0,
-	HAL_FW_C2H_CMD_Read_BBREG = 1,
-	HAL_FW_C2H_CMD_Read_RFREG = 2,
-	HAL_FW_C2H_CMD_Read_EEPROM = 3,
-	HAL_FW_C2H_CMD_Read_EFUSE = 4,
-	HAL_FW_C2H_CMD_Read_CAM = 5,
-	HAL_FW_C2H_CMD_Get_BasicRate = 6,
-	HAL_FW_C2H_CMD_Get_DataRate = 7,
-	HAL_FW_C2H_CMD_Survey = 8,
-	HAL_FW_C2H_CMD_SurveyDone = 9,
-	HAL_FW_C2H_CMD_JoinBss = 10,
-	HAL_FW_C2H_CMD_AddSTA = 11,
-	HAL_FW_C2H_CMD_DelSTA = 12,
-	HAL_FW_C2H_CMD_AtimDone = 13,
-	HAL_FW_C2H_CMD_TX_Report = 14,
-	HAL_FW_C2H_CMD_CCX_Report = 15,
-	HAL_FW_C2H_CMD_DTM_Report = 16,
-	HAL_FW_C2H_CMD_TX_Rate_Statistics = 17,
-	HAL_FW_C2H_CMD_C2HLBK = 18,
-	HAL_FW_C2H_CMD_C2HDBG = 19,
-	HAL_FW_C2H_CMD_C2HFEEDBACK = 20,
-	HAL_FW_C2H_CMD_MAX
-};
-
 enum rtl_desc_qsel {
 	QSLT_BK = 0x2,
 	QSLT_BE = 0x0,
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/def.h b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/def.h
index 60f5728b4e2d..9f7e7bb8610b 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/def.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/def.h
@@ -47,14 +47,6 @@ enum version_8192e {
 	VERSION_UNKNOWN = 0xFF,
 };
 
-enum rx_packet_type {
-	NORMAL_RX,
-	TX_REPORT1,
-	TX_REPORT2,
-	HIS_REPORT,
-	C2H_PACKET,
-};
-
 enum rtl_desc_qsel {
 	QSLT_BK = 0x2,
 	QSLT_BE = 0x0,
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c
index f9563ae301ad..84a0d0eb72e1 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c
@@ -876,85 +876,11 @@ void rtl92ee_set_p2p_ps_offload_cmd(struct ieee80211_hw *hw, u8 p2p_ps_state)
 			     (u8 *)p2p_ps_offload);
 }
 
-static void _rtl92ee_c2h_ra_report_handler(struct ieee80211_hw *hw,
-					   u8 *cmd_buf, u8 cmd_len)
+void rtl92ee_c2h_ra_report_handler(struct ieee80211_hw *hw,
+				   u8 *cmd_buf, u8 cmd_len)
 {
 	u8 rate = cmd_buf[0] & 0x3F;
 	bool collision_state = cmd_buf[3] & BIT(0);
 
 	rtl92ee_dm_dynamic_arfb_select(hw, rate, collision_state);
 }
-
-void rtl92ee_c2h_content_parsing(struct ieee80211_hw *hw, u8 c2h_cmd_id,
-				 u8 c2h_cmd_len, u8 *tmp_buf)
-{
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
-
-	switch (c2h_cmd_id) {
-	case C2H_8192E_DBG:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-			 "[C2H], C2H_8723BE_DBG!!\n");
-		break;
-	case C2H_8192E_TXBF:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-			 "[C2H], C2H_8192E_TXBF!!\n");
-		break;
-	case C2H_8192E_TX_REPORT:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE ,
-			 "[C2H], C2H_8723BE_TX_REPORT!\n");
-		rtl_tx_report_handler(hw, tmp_buf, c2h_cmd_len);
-		break;
-	case C2H_8192E_BT_INFO:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-			 "[C2H], C2H_8723BE_BT_INFO!!\n");
-		if (rtlpriv->cfg->ops->get_btc_status())
-			btc_ops->btc_btinfo_notify(rtlpriv, tmp_buf,
-						   c2h_cmd_len);
-		break;
-	case C2H_8192E_BT_MP:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-			 "[C2H], C2H_8723BE_BT_MP!!\n");
-		if (rtlpriv->cfg->ops->get_btc_status())
-			btc_ops->btc_btmpinfo_notify(rtlpriv, tmp_buf,
-						     c2h_cmd_len);
-		break;
-	case C2H_8192E_RA_RPT:
-		_rtl92ee_c2h_ra_report_handler(hw, tmp_buf, c2h_cmd_len);
-		break;
-	default:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-			 "[C2H], Unknown packet!! CmdId(%#X)!\n", c2h_cmd_id);
-		break;
-	}
-}
-
-void rtl92ee_c2h_packet_handler(struct ieee80211_hw *hw, u8 *buffer, u8 len)
-{
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	u8 c2h_cmd_id = 0, c2h_cmd_seq = 0, c2h_cmd_len = 0;
-	u8 *tmp_buf = NULL;
-
-	c2h_cmd_id = buffer[0];
-	c2h_cmd_seq = buffer[1];
-	c2h_cmd_len = len - 2;
-	tmp_buf = buffer + 2;
-
-	RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-		 "[C2H packet], c2hCmdId=0x%x, c2hCmdSeq=0x%x, c2hCmdLen=%d\n",
-		 c2h_cmd_id, c2h_cmd_seq, c2h_cmd_len);
-
-	RT_PRINT_DATA(rtlpriv, COMP_FW, DBG_TRACE,
-		      "[C2H packet], Content Hex:\n", tmp_buf, c2h_cmd_len);
-
-	switch (c2h_cmd_id) {
-	case C2H_8192E_BT_INFO:
-	case C2H_8192E_BT_MP:
-		rtl_c2hcmd_enqueue(hw, c2h_cmd_id, c2h_cmd_len, tmp_buf);
-		break;
-	default:
-		rtl92ee_c2h_content_parsing(hw, c2h_cmd_id, c2h_cmd_len,
-					    tmp_buf);
-		break;
-	}
-}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.h b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.h
index b770f722daa6..6a2fbf20d579 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.h
@@ -128,17 +128,6 @@ enum rtl8192e_h2c_cmd {
 	MAX_92E_H2CCMD
 };
 
-enum rtl8192e_c2h_evt {
-	C2H_8192E_DBG = 0,
-	C2H_8192E_LB = 1,
-	C2H_8192E_TXBF = 2,
-	C2H_8192E_TX_REPORT = 3,
-	C2H_8192E_BT_INFO = 9,
-	C2H_8192E_BT_MP = 11,
-	C2H_8192E_RA_RPT = 12,
-	MAX_8192E_C2HEVENT
-};
-
 #define pagenum_128(_len)	\
 	(u32)(((_len) >> 7) + ((_len) & 0x7F ? 1 : 0))
 
@@ -190,7 +179,6 @@ void rtl92ee_set_fw_pwrmode_cmd(struct ieee80211_hw *hw, u8 mode);
 void rtl92ee_set_fw_media_status_rpt_cmd(struct ieee80211_hw *hw, u8 mstatus);
 void rtl92ee_set_fw_rsvdpagepkt(struct ieee80211_hw *hw, bool b_dl_finished);
 void rtl92ee_set_p2p_ps_offload_cmd(struct ieee80211_hw *hw, u8 p2p_ps_state);
-void rtl92ee_c2h_packet_handler(struct ieee80211_hw *hw, u8 *buffer, u8 len);
-void rtl92ee_c2h_content_parsing(struct ieee80211_hw *hw, u8 c2h_cmd_id,
-				 u8 c2h_cmd_len, u8 *tmp_buf);
+void rtl92ee_c2h_ra_report_handler(struct ieee80211_hw *hw,
+				   u8 *cmd_buf, u8 cmd_len);
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
index fd7928fdbd1a..3f2295fdb02d 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
@@ -2574,11 +2574,11 @@ void rtl92ee_read_bt_coexist_info_from_hwpg(struct ieee80211_hw *hw,
 			rtlpriv->btcoexist.btc_info.btcoexist = 0;
 
 		rtlpriv->btcoexist.btc_info.bt_type = BT_RTL8192E;
-		rtlpriv->btcoexist.btc_info.ant_num = ANT_TOTAL_X2;
+		rtlpriv->btcoexist.btc_info.ant_num = ANT_X2;
 	} else {
 		rtlpriv->btcoexist.btc_info.btcoexist = 1;
 		rtlpriv->btcoexist.btc_info.bt_type = BT_RTL8192E;
-		rtlpriv->btcoexist.btc_info.ant_num = ANT_TOTAL_X1;
+		rtlpriv->btcoexist.btc_info.ant_num = ANT_X1;
 	}
 }
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c
index ef92a789871d..9ea62599ecbb 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c
@@ -250,8 +250,7 @@ static struct rtl_hal_ops rtl8192ee_hal_ops = {
 	.set_rfreg = rtl92ee_phy_set_rf_reg,
 	.fill_h2c_cmd = rtl92ee_fill_h2c_cmd,
 	.get_btc_status = rtl92ee_get_btc_status,
-	.rx_command_packet = rtl92ee_rx_command_packet,
-	.c2h_content_parsing = rtl92ee_c2h_content_parsing,
+	.c2h_ra_report_handler = rtl92ee_c2h_ra_report_handler,
 };
 
 static struct rtl_mod_params rtl92ee_mod_params = {
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c
index 4f7444331b07..14d6e3fc5767 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c
@@ -662,6 +662,7 @@ void rtl92ee_tx_fill_desc(struct ieee80211_hw *hw,
 	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct rtlwifi_tx_info *tx_info = rtl_tx_skb_cb_info(skb);
 	u8 *pdesc = (u8 *)pdesc_tx;
 	u16 seq_number;
 	__le16 fc = hdr->frame_control;
@@ -723,8 +724,6 @@ void rtl92ee_tx_fill_desc(struct ieee80211_hw *hw,
 			SET_TX_DESC_OFFSET(pdesc, USB_HWDESC_HEADER_LEN);
 		}
 
-		/* tx report */
-		rtl_get_tx_report(ptcb_desc, pdesc, hw);
 
 		SET_TX_DESC_TX_RATE(pdesc, ptcb_desc->hw_rate);
 
@@ -827,6 +826,8 @@ void rtl92ee_tx_fill_desc(struct ieee80211_hw *hw,
 				SET_TX_DESC_HTC(pdesc, 1);
 			}
 		}
+		/* tx report */
+		rtl_set_tx_report(ptcb_desc, pdesc, hw, tx_info);
 	}
 
 	SET_TX_DESC_FIRST_SEG(pdesc, (firstseg ? 1 : 0));
@@ -1071,27 +1072,3 @@ bool rtl92ee_is_tx_desc_closed(struct ieee80211_hw *hw, u8 hw_queue, u16 index)
 void rtl92ee_tx_polling(struct ieee80211_hw *hw, u8 hw_queue)
 {
 }
-
-u32 rtl92ee_rx_command_packet(struct ieee80211_hw *hw,
-			      const struct rtl_stats *status,
-			      struct sk_buff *skb)
-{
-	u32 result = 0;
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-
-	switch (status->packet_report_type) {
-	case NORMAL_RX:
-		result = 0;
-		break;
-	case C2H_PACKET:
-		rtl92ee_c2h_packet_handler(hw, skb->data, (u8)skb->len);
-		result = 1;
-		break;
-	default:
-		RT_TRACE(rtlpriv, COMP_RECV, DBG_TRACE,
-			 "Unknown packet type %d\n", status->packet_report_type);
-		break;
-	}
-
-	return result;
-}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.h b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.h
index 48c16fff20c6..45df3e79f490 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.h
@@ -762,7 +762,4 @@ void rtl92ee_tx_polling(struct ieee80211_hw *hw, u8 hw_queue);
 void rtl92ee_tx_fill_cmddesc(struct ieee80211_hw *hw, u8 *pdesc,
 			     bool firstseg, bool lastseg,
 			     struct sk_buff *skb);
-u32 rtl92ee_rx_command_packet(struct ieee80211_hw *hw,
-			      const struct rtl_stats *status,
-			      struct sk_buff *skb);
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/def.h b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/def.h
index bcdf2273688e..847544817549 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/def.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/def.h
@@ -152,31 +152,6 @@ enum interface_select_pci {
 	INTF_SEL3_RSV = 3,
 };
 
-enum hal_fw_c2h_cmd_id {
-	HAL_FW_C2H_CMD_Read_MACREG = 0,
-	HAL_FW_C2H_CMD_Read_BBREG = 1,
-	HAL_FW_C2H_CMD_Read_RFREG = 2,
-	HAL_FW_C2H_CMD_Read_EEPROM = 3,
-	HAL_FW_C2H_CMD_Read_EFUSE = 4,
-	HAL_FW_C2H_CMD_Read_CAM = 5,
-	HAL_FW_C2H_CMD_Get_BasicRate = 6,
-	HAL_FW_C2H_CMD_Get_DataRate = 7,
-	HAL_FW_C2H_CMD_Survey = 8,
-	HAL_FW_C2H_CMD_SurveyDone = 9,
-	HAL_FW_C2H_CMD_JoinBss = 10,
-	HAL_FW_C2H_CMD_AddSTA = 11,
-	HAL_FW_C2H_CMD_DelSTA = 12,
-	HAL_FW_C2H_CMD_AtimDone = 13,
-	HAL_FW_C2H_CMD_TX_Report = 14,
-	HAL_FW_C2H_CMD_CCX_Report = 15,
-	HAL_FW_C2H_CMD_DTM_Report = 16,
-	HAL_FW_C2H_CMD_TX_Rate_Statistics = 17,
-	HAL_FW_C2H_CMD_C2HLBK = 18,
-	HAL_FW_C2H_CMD_C2HDBG = 19,
-	HAL_FW_C2H_CMD_C2HFEEDBACK = 20,
-	HAL_FW_C2H_CMD_MAX
-};
-
 enum rtl_desc_qsel {
 	QSLT_BK = 0x2,
 	QSLT_BE = 0x0,
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hal_btc.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hal_btc.c
index ec9bcf32f0ab..788de88ab1ee 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hal_btc.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hal_btc.c
@@ -1749,13 +1749,13 @@ void rtl_8723e_c2h_command_handle(struct ieee80211_hw *hw)
 
 
 	switch (c2h_event.cmd_id) {
-	case C2H_BT_RSSI:
+	case C2H_V0_BT_RSSI:
 			break;
 
-	case C2H_BT_OP_MODE:
+	case C2H_V0_BT_OP_MODE:
 			break;
 
-	case BT_INFO:
+	case C2H_V0_BT_INFO:
 		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
 			"BT info Byte[0] (ID) is 0x%x\n",
 			c2h_event.cmd_id);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hal_btc.h b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hal_btc.h
index 3723d7476717..756868897d8b 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hal_btc.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hal_btc.h
@@ -130,17 +130,17 @@ enum bt_state {
 	BT_INFO_STATE_MAX = 7
 };
 
-enum rtl8723e_c2h_evt {
-	C2H_DBG = 0,
-	C2H_TSF = 1,
-	C2H_AP_RPT_RSP = 2,
+enum rtl8723e_c2h_evt_v0 {
+	C2H_V0_DBG = 0,
+	C2H_V0_TSF = 1,
+	C2H_V0_AP_RPT_RSP = 2,
 	/* The FW notify the report of the specific tx packet. */
-	C2H_CCX_TX_RPT = 3,
-	C2H_BT_RSSI = 4,
-	C2H_BT_OP_MODE = 5,
-	C2H_HW_INFO_EXCH = 10,
-	C2H_C2H_H2C_TEST = 11,
-	BT_INFO = 12,
+	C2H_V0_CCX_TX_RPT = 3,
+	C2H_V0_BT_RSSI = 4,
+	C2H_V0_BT_OP_MODE = 5,
+	C2H_V0_HW_INFO_EXCH = 10,
+	C2H_V0_C2H_H2C_TEST = 11,
+	C2H_V0_BT_INFO = 12,
 	MAX_C2HEVENT
 };
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c
index a545ea317323..07b82700d1de 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c
@@ -260,7 +260,6 @@ static struct rtl_hal_ops rtl8723e_hal_ops = {
 	.bt_coex_off_before_lps =
 		rtl8723e_dm_bt_turn_off_bt_coexist_before_enter_lps,
 	.get_btc_status = rtl8723e_get_btc_status,
-	.rx_command_packet = rtl8723e_rx_command_packet,
 	.is_fw_header = is_fw_header,
 };
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/trx.c
index 23485602a9a1..d461d0c9631f 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/trx.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/trx.c
@@ -709,10 +709,3 @@ void rtl8723e_tx_polling(struct ieee80211_hw *hw, u8 hw_queue)
 			       BIT(0) << (hw_queue));
 	}
 }
-
-u32 rtl8723e_rx_command_packet(struct ieee80211_hw *hw,
-			       const struct rtl_stats *status,
-			       struct sk_buff *skb)
-{
-	return 0;
-}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/trx.h b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/trx.h
index 985ce0b77ea5..d592b08d4ac8 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/trx.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/trx.h
@@ -716,7 +716,4 @@ void rtl8723e_tx_polling(struct ieee80211_hw *hw, u8 hw_queue);
 void rtl8723e_tx_fill_cmddesc(struct ieee80211_hw *hw, u8 *pdesc,
 			      bool firstseg, bool lastseg,
 			      struct sk_buff *skb);
-u32 rtl8723e_rx_command_packet(struct ieee80211_hw *hw,
-			       const struct rtl_stats *status,
-			       struct sk_buff *skb);
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/def.h b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/def.h
index 025ea5c0f3f6..5e5403d69220 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/def.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/def.h
@@ -38,14 +38,6 @@
 /* Currently only for RTL8723B */
 #define EXT_VENDOR_ID				(BIT(18) | BIT(19))
 
-enum rx_packet_type {
-	NORMAL_RX,
-	TX_REPORT1,
-	TX_REPORT2,
-	HIS_REPORT,
-	C2H_PACKET,
-};
-
 enum rtl_desc_qsel {
 	QSLT_BK = 0x2,
 	QSLT_BE = 0x0,
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c
index 4b963fd27d64..f2441fbb92f1 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c
@@ -703,72 +703,3 @@ void rtl8723be_set_p2p_ps_offload_cmd(struct ieee80211_hw *hw,
 	rtl8723be_fill_h2c_cmd(hw, H2C_8723B_P2P_PS_OFFLOAD, 1,
 			       (u8 *)p2p_ps_offload);
 }
-
-void rtl8723be_c2h_content_parsing(struct ieee80211_hw *hw,
-				   u8 c2h_cmd_id,
-				   u8 c2h_cmd_len, u8 *tmp_buf)
-{
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
-
-	switch (c2h_cmd_id) {
-	case C2H_8723B_DBG:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-			 "[C2H], C2H_8723BE_DBG!!\n");
-		break;
-	case C2H_8723B_TX_REPORT:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-			 "[C2H], C2H_8723BE_TX_REPORT!\n");
-		rtl_tx_report_handler(hw, tmp_buf, c2h_cmd_len);
-		break;
-	case C2H_8723B_BT_INFO:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-			 "[C2H], C2H_8723BE_BT_INFO!!\n");
-		if (rtlpriv->cfg->ops->get_btc_status())
-			btc_ops->btc_btinfo_notify(rtlpriv, tmp_buf,
-						   c2h_cmd_len);
-		break;
-	case C2H_8723B_BT_MP:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-			 "[C2H], C2H_8723BE_BT_MP!!\n");
-		if (rtlpriv->cfg->ops->get_btc_status())
-			btc_ops->btc_btmpinfo_notify(rtlpriv, tmp_buf,
-						     c2h_cmd_len);
-		break;
-	default:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-			 "[C2H], Unknown packet!! CmdId(%#X)!\n", c2h_cmd_id);
-		break;
-	}
-}
-
-void rtl8723be_c2h_packet_handler(struct ieee80211_hw *hw, u8 *buffer, u8 len)
-{
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	u8 c2h_cmd_id = 0, c2h_cmd_seq = 0, c2h_cmd_len = 0;
-	u8 *tmp_buf = NULL;
-
-	c2h_cmd_id = buffer[0];
-	c2h_cmd_seq = buffer[1];
-	c2h_cmd_len = len - 2;
-	tmp_buf = buffer + 2;
-
-	RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-		 "[C2H packet], c2hCmdId=0x%x, c2hCmdSeq=0x%x, c2hCmdLen=%d\n",
-		 c2h_cmd_id, c2h_cmd_seq, c2h_cmd_len);
-
-	RT_PRINT_DATA(rtlpriv, COMP_FW, DBG_TRACE,
-		      "[C2H packet], Content Hex:\n", tmp_buf, c2h_cmd_len);
-
-	switch (c2h_cmd_id) {
-	case C2H_8723B_BT_INFO:
-	case C2H_8723B_BT_MP:
-		rtl_c2hcmd_enqueue(hw, c2h_cmd_id, c2h_cmd_len, tmp_buf);
-		break;
-
-	default:
-		rtl8723be_c2h_content_parsing(hw, c2h_cmd_id, c2h_cmd_len,
-					      tmp_buf);
-		break;
-	}
-}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.h b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.h
index 2de4edb62bca..948d28646364 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.h
@@ -100,16 +100,6 @@ enum rtl8723b_h2c_cmd {
 	MAX_8723B_H2CCMD
 };
 
-enum rtl8723b_c2h_evt {
-	C2H_8723B_DBG = 0,
-	C2H_8723B_LB = 1,
-	C2H_8723B_TXBF = 2,
-	C2H_8723B_TX_REPORT = 3,
-	C2H_8723B_BT_INFO = 9,
-	C2H_8723B_BT_MP = 11,
-	MAX_8723B_C2HEVENT
-};
-
 #define pagenum_128(_len) (u32)(((_len)>>7) + ((_len)&0x7F ? 1 : 0))
 
 
@@ -153,7 +143,4 @@ void rtl8723be_set_fw_pwrmode_cmd(struct ieee80211_hw *hw, u8 mode);
 void rtl8723be_set_fw_media_status_rpt_cmd(struct ieee80211_hw *hw, u8 mstatus);
 void rtl8723be_set_fw_rsvdpagepkt(struct ieee80211_hw *hw, bool b_dl_finished);
 void rtl8723be_set_p2p_ps_offload_cmd(struct ieee80211_hw *hw, u8 p2p_ps_state);
-void rtl8723be_c2h_packet_handler(struct ieee80211_hw *hw, u8 *buffer, u8 len);
-void rtl8723be_c2h_content_parsing(struct ieee80211_hw *hw, u8 c2h_cmd_id,
-				   u8 c2h_cmd_len, u8 *tmp_buf);
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c
index 6a42988aad65..c9f7b042d9c6 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c
@@ -261,9 +261,7 @@ static struct rtl_hal_ops rtl8723be_hal_ops = {
 	.set_rfreg = rtl8723be_phy_set_rf_reg,
 	.fill_h2c_cmd = rtl8723be_fill_h2c_cmd,
 	.get_btc_status = rtl8723be_get_btc_status,
-	.rx_command_packet = rtl8723be_rx_command_packet,
 	.is_fw_header = is_fw_header,
-	.c2h_content_parsing = rtl8723be_c2h_content_parsing,
 };
 
 static struct rtl_mod_params rtl8723be_mod_params = {
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/trx.c
index fd9b38aa08a1..9f8dfb5af774 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/trx.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/trx.c
@@ -431,6 +431,7 @@ void rtl8723be_tx_fill_desc(struct ieee80211_hw *hw,
 	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct rtlwifi_tx_info *tx_info = rtl_tx_skb_cb_info(skb);
 	u8 *pdesc = (u8 *)pdesc_tx;
 	u16 seq_number;
 	__le16 fc = hdr->frame_control;
@@ -488,8 +489,6 @@ void rtl8723be_tx_fill_desc(struct ieee80211_hw *hw,
 			SET_TX_DESC_OFFSET(pdesc, USB_HWDESC_HEADER_LEN);
 		}
 
-		/* tx report */
-		rtl_get_tx_report(ptcb_desc, pdesc, hw);
 
 		/* ptcb_desc->use_driver_rate = true; */
 		SET_TX_DESC_TX_RATE(pdesc, ptcb_desc->hw_rate);
@@ -578,6 +577,8 @@ void rtl8723be_tx_fill_desc(struct ieee80211_hw *hw,
 				SET_TX_DESC_HTC(pdesc, 1);
 			}
 		}
+		/* tx report */
+		rtl_set_tx_report(ptcb_desc, pdesc, hw, tx_info);
 	}
 
 	SET_TX_DESC_FIRST_SEG(pdesc, (firstseg ? 1 : 0));
@@ -760,28 +761,3 @@ void rtl8723be_tx_polling(struct ieee80211_hw *hw, u8 hw_queue)
 			       BIT(0) << (hw_queue));
 	}
 }
-
-u32 rtl8723be_rx_command_packet(struct ieee80211_hw *hw,
-				const struct rtl_stats *status,
-				struct sk_buff *skb)
-{
-	u32 result = 0;
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-
-	switch (status->packet_report_type) {
-	case NORMAL_RX:
-			result = 0;
-			break;
-	case C2H_PACKET:
-			rtl8723be_c2h_packet_handler(hw, skb->data,
-						     (u8)skb->len);
-			result = 1;
-			break;
-	default:
-			RT_TRACE(rtlpriv, COMP_RECV, DBG_TRACE,
-				 "No this packet type!!\n");
-			break;
-	}
-
-	return result;
-}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/trx.h b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/trx.h
index 988bf0586674..609f7ad7f787 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/trx.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/trx.h
@@ -632,7 +632,4 @@ void rtl8723be_tx_polling(struct ieee80211_hw *hw, u8 hw_queue);
 void rtl8723be_tx_fill_cmddesc(struct ieee80211_hw *hw, u8 *pdesc,
 			       bool firstseg, bool lastseg,
 			       struct sk_buff *skb);
-u32 rtl8723be_rx_command_packet(struct ieee80211_hw *hw,
-				const struct rtl_stats *status,
-				struct sk_buff *skb);
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/def.h b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/def.h
index dfbdf539de1a..3fe3aaa5fe3c 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/def.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/def.h
@@ -296,31 +296,6 @@ enum interface_select_pci {
 	INTF_SEL3_RSV = 3,
 };
 
-enum hal_fw_c2h_cmd_id {
-	HAL_FW_C2H_CMD_READ_MACREG = 0,
-	HAL_FW_C2H_CMD_READ_BBREG = 1,
-	HAL_FW_C2H_CMD_READ_RFREG = 2,
-	HAL_FW_C2H_CMD_READ_EEPROM = 3,
-	HAL_FW_C2H_CMD_READ_EFUSE = 4,
-	HAL_FW_C2H_CMD_READ_CAM = 5,
-	HAL_FW_C2H_CMD_GET_BASICRATE = 6,
-	HAL_FW_C2H_CMD_GET_DATARATE = 7,
-	HAL_FW_C2H_CMD_SURVEY = 8,
-	HAL_FW_C2H_CMD_SURVEYDONE = 9,
-	HAL_FW_C2H_CMD_JOINBSS = 10,
-	HAL_FW_C2H_CMD_ADDSTA = 11,
-	HAL_FW_C2H_CMD_DELSTA = 12,
-	HAL_FW_C2H_CMD_ATIMDONE = 13,
-	HAL_FW_C2H_CMD_TX_REPORT = 14,
-	HAL_FW_C2H_CMD_CCX_REPORT = 15,
-	HAL_FW_C2H_CMD_DTM_REPORT = 16,
-	HAL_FW_C2H_CMD_TX_RATE_STATISTICS = 17,
-	HAL_FW_C2H_CMD_C2HLBK = 18,
-	HAL_FW_C2H_CMD_C2HDBG = 19,
-	HAL_FW_C2H_CMD_C2HFEEDBACK = 20,
-	HAL_FW_C2H_CMD_MAX
-};
-
 enum rtl_desc_qsel {
 	QSLT_BK = 0x2,
 	QSLT_BE = 0x0,
@@ -332,14 +307,6 @@ enum rtl_desc_qsel {
 	QSLT_CMD = 0x13,
 };
 
-enum rx_packet_type {
-	NORMAL_RX,
-	TX_REPORT1,
-	TX_REPORT2,
-	HIS_REPORT,
-	C2H_PACKET,
-};
-
 struct phy_sts_cck_8821ae_t {
 	u8 adc_pwdb_X[4];
 	u8 sq_rpt;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c
index f2b2c549e5b2..d868a034659f 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c
@@ -1907,7 +1907,7 @@ void rtl8821ae_set_p2p_ps_offload_cmd(struct ieee80211_hw *hw, u8 p2p_ps_state)
 			H2C_8821AE_P2P_PS_OFFLOAD, 1, (u8 *)p2p_ps_offload);
 }
 
-static void rtl8821ae_c2h_ra_report_handler(struct ieee80211_hw *hw,
+void rtl8821ae_c2h_ra_report_handler(struct ieee80211_hw *hw,
 				     u8 *cmd_buf, u8 cmd_len)
 {
 	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
@@ -1917,70 +1917,3 @@ static void rtl8821ae_c2h_ra_report_handler(struct ieee80211_hw *hw,
 
 	rtl8821ae_dm_update_init_rate(hw, rate);
 }
-
-void rtl8821ae_c2h_content_parsing(struct ieee80211_hw *hw,
-				   u8 c2h_cmd_id, u8 c2h_cmd_len,
-				   u8 *tmp_buf)
-{
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
-
-	switch (c2h_cmd_id) {
-	case C2H_8812_DBG:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD, "[C2H], C2H_8812_DBG!!\n");
-		break;
-	case C2H_8812_TX_REPORT:
-		rtl_tx_report_handler(hw, tmp_buf, c2h_cmd_len);
-		break;
-	case C2H_8812_RA_RPT:
-		rtl8821ae_c2h_ra_report_handler(hw, tmp_buf, c2h_cmd_len);
-		break;
-	case C2H_8812_BT_INFO:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD,
-			 "[C2H], C2H_8812_BT_INFO!!\n");
-		if (rtlpriv->cfg->ops->get_btc_status())
-			btc_ops->btc_btinfo_notify(rtlpriv, tmp_buf,
-						   c2h_cmd_len);
-		break;
-	case C2H_8812_BT_MP:
-		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
-			 "[C2H], C2H_8812_BT_MP!!\n");
-		if (rtlpriv->cfg->ops->get_btc_status())
-			btc_ops->btc_btmpinfo_notify(rtlpriv, tmp_buf,
-						     c2h_cmd_len);
-		break;
-	default:
-		break;
-	}
-}
-
-void rtl8821ae_c2h_packet_handler(struct ieee80211_hw *hw, u8 *buffer,
-				  u8 length)
-{
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	u8 c2h_cmd_id = 0, c2h_cmd_seq = 0, c2h_cmd_len = 0;
-	u8 *tmp_buf = NULL;
-
-	c2h_cmd_id = buffer[0];
-	c2h_cmd_seq = buffer[1];
-	c2h_cmd_len = length - 2;
-	tmp_buf = buffer + 2;
-
-	RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD,
-		 "[C2H packet], c2hCmdId=0x%x, c2hCmdSeq=0x%x, c2hCmdLen=%d\n",
-		 c2h_cmd_id, c2h_cmd_seq, c2h_cmd_len);
-
-	RT_PRINT_DATA(rtlpriv, COMP_FW, DBG_LOUD,
-		      "[C2H packet], Content Hex:\n", tmp_buf, c2h_cmd_len);
-
-	switch (c2h_cmd_id) {
-	case C2H_8812_BT_INFO:
-		rtl_c2hcmd_enqueue(hw, c2h_cmd_id, c2h_cmd_len, tmp_buf);
-		break;
-
-	default:
-		rtl8821ae_c2h_content_parsing(hw, c2h_cmd_id, c2h_cmd_len,
-					      tmp_buf);
-		break;
-	}
-}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.h b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.h
index 32d46d7128f5..99c902ff0b84 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.h
@@ -137,20 +137,6 @@
 #define	FW_PWR_STATE_ACTIVE	((FW_PS_RF_ON) | (FW_PS_REGISTER_ACTIVE))
 #define	FW_PWR_STATE_RF_OFF	0
 
-enum rtl8812_c2h_evt {
-	C2H_8812_DBG = 0,
-	C2H_8812_LB = 1,
-	C2H_8812_TXBF = 2,
-	C2H_8812_TX_REPORT = 3,
-	C2H_8812_BT_INFO = 9,
-	C2H_8812_BT_MP = 11,
-	C2H_8812_RA_RPT = 12,
-
-	C2H_8812_FW_SWCHNL = 0x10,
-	C2H_8812_IQK_FINISH = 0x11,
-	MAX_8812_C2HEVENT
-};
-
 enum rtl8821a_h2c_cmd {
 	H2C_8821AE_RSVDPAGE = 0,
 	H2C_8821AE_MSRRPT = 1,
@@ -331,9 +317,6 @@ void rtl8821ae_set_fw_keep_alive_cmd(struct ieee80211_hw *hw, bool func_en);
 void rtl8821ae_set_fw_disconnect_decision_ctrl_cmd(struct ieee80211_hw *hw,
 						   bool enabled);
 void rtl8821ae_set_fw_global_info_cmd(struct ieee80211_hw *hw);
-void rtl8821ae_c2h_packet_handler(struct ieee80211_hw *hw,
-				  u8 *buffer, u8 length);
-void rtl8821ae_c2h_content_parsing(struct ieee80211_hw *hw,
-				   u8 c2h_cmd_id, u8 c2h_cmd_len,
-				   u8 *tmp_buf);
+void rtl8821ae_c2h_ra_report_handler(struct ieee80211_hw *hw,
+				     u8 *cmd_buf, u8 cmd_len);
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c
index 9bb3d9dfce79..77f6401021c9 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c
@@ -303,8 +303,7 @@ static struct rtl_hal_ops rtl8821ae_hal_ops = {
 	.set_rfreg = rtl8821ae_phy_set_rf_reg,
 	.fill_h2c_cmd = rtl8821ae_fill_h2c_cmd,
 	.get_btc_status = rtl8821ae_get_btc_status,
-	.rx_command_packet = rtl8821ae_rx_command_packet,
-	.c2h_content_parsing = rtl8821ae_c2h_content_parsing,
+	.c2h_ra_report_handler = rtl8821ae_c2h_ra_report_handler,
 	.add_wowlan_pattern = rtl8821ae_add_wowlan_pattern,
 };
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.c
index 1e1bacf562f3..d7960dd5bf1a 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.c
@@ -691,6 +691,7 @@ void rtl8821ae_tx_fill_desc(struct ieee80211_hw *hw,
 	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct rtlwifi_tx_info *tx_info = rtl_tx_skb_cb_info(skb);
 	u8 *pdesc = (u8 *)pdesc_tx;
 	u16 seq_number;
 	__le16 fc = hdr->frame_control;
@@ -740,8 +741,6 @@ void rtl8821ae_tx_fill_desc(struct ieee80211_hw *hw,
 			SET_TX_DESC_OFFSET(pdesc, USB_HWDESC_HEADER_LEN);
 		}
 
-		/* tx report */
-		rtl_get_tx_report(ptcb_desc, pdesc, hw);
 
 		/* ptcb_desc->use_driver_rate = true; */
 		SET_TX_DESC_TX_RATE(pdesc, ptcb_desc->hw_rate);
@@ -818,6 +817,8 @@ void rtl8821ae_tx_fill_desc(struct ieee80211_hw *hw,
 				SET_TX_DESC_HTC(pdesc, 1);
 			}
 		}
+		/* tx report */
+		rtl_set_tx_report(ptcb_desc, pdesc, hw, tx_info);
 	}
 
 	SET_TX_DESC_FIRST_SEG(pdesc, (firstseg ? 1 : 0));
@@ -1004,29 +1005,3 @@ void rtl8821ae_tx_polling(struct ieee80211_hw *hw, u8 hw_queue)
 			       BIT(0) << (hw_queue));
 	}
 }
-
-u32 rtl8821ae_rx_command_packet(struct ieee80211_hw *hw,
-				const struct rtl_stats *status,
-				struct sk_buff *skb)
-{
-	u32 result = 0;
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-
-	switch (status->packet_report_type) {
-	case NORMAL_RX:
-		result = 0;
-		break;
-	case C2H_PACKET:
-		rtl8821ae_c2h_packet_handler(hw, skb->data, (u8)skb->len);
-		result = 1;
-		RT_TRACE(rtlpriv, COMP_RECV, DBG_LOUD,
-			 "skb->len=%d\n\n", skb->len);
-		break;
-	default:
-		RT_TRACE(rtlpriv, COMP_RECV, DBG_LOUD,
-			 "No this packet type!!\n");
-		break;
-	}
-
-	return result;
-}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.h b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.h
index 221dd2b29d3b..4ff0968dba81 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.h
@@ -628,7 +628,4 @@ void rtl8821ae_tx_polling(struct ieee80211_hw *hw, u8 hw_queue);
 void rtl8821ae_tx_fill_cmddesc(struct ieee80211_hw *hw, u8 *pdesc,
 			       bool firstseg, bool lastseg,
 			       struct sk_buff *skb);
-u32 rtl8821ae_rx_command_packet(struct ieee80211_hw *hw,
-				const struct rtl_stats *status,
-				struct sk_buff *skb);
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/wifi.h b/drivers/net/wireless/realtek/rtlwifi/wifi.h
index ce1754054a07..0f3b98c5227f 100644
--- a/drivers/net/wireless/realtek/rtlwifi/wifi.h
+++ b/drivers/net/wireless/realtek/rtlwifi/wifi.h
@@ -158,6 +158,30 @@ enum {
 	H2C_BT_PORT_ID = 0x71,
 };
 
+enum rtl_c2h_evt_v1 {
+	C2H_DBG = 0,
+	C2H_LB = 1,
+	C2H_TXBF = 2,
+	C2H_TX_REPORT = 3,
+	C2H_BT_INFO = 9,
+	C2H_BT_MP = 11,
+	C2H_RA_RPT = 12,
+
+	C2H_FW_SWCHNL = 0x10,
+	C2H_IQK_FINISH = 0x11,
+
+	C2H_EXT_V2 = 0xFF,
+};
+
+enum rtl_c2h_evt_v2 {
+	C2H_V2_CCX_RPT = 0x0F,
+};
+
+#define GET_C2H_CMD_ID(c2h)	({u8 *__c2h = c2h; __c2h[0]; })
+#define GET_C2H_SEQ(c2h)	({u8 *__c2h = c2h; __c2h[1]; })
+#define C2H_DATA_OFFSET		2
+#define GET_C2H_DATA_PTR(c2h)	({u8 *__c2h = c2h; &__c2h[C2H_DATA_OFFSET]; })
+
 #define GET_TX_REPORT_SN_V1(c2h)	(c2h[6])
 #define GET_TX_REPORT_ST_V1(c2h)	(c2h[0] & 0xC0)
 #define GET_TX_REPORT_RETRY_V1(c2h)	(c2h[2] & 0x3F)
@@ -1010,6 +1034,29 @@ enum dm_info_query {
 	DM_INFO_SIZE,
 };
 
+enum rx_packet_type {
+	NORMAL_RX,
+	TX_REPORT1,
+	TX_REPORT2,
+	HIS_REPORT,
+	C2H_PACKET,
+};
+
+struct rtlwifi_tx_info {
+	int sn;
+	unsigned long send_time;
+};
+
+static inline struct rtlwifi_tx_info *rtl_tx_skb_cb_info(struct sk_buff *skb)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+
+	BUILD_BUG_ON(sizeof(struct rtlwifi_tx_info) >
+		     sizeof(info->status.status_driver_data));
+
+	return (struct rtlwifi_tx_info *)(info->status.status_driver_data);
+}
+
 struct octet_string {
 	u8 *octet;
 	u16 length;
@@ -1967,6 +2014,7 @@ struct rtl_tx_report {
 	u16 last_sent_sn;
 	unsigned long last_sent_time;
 	u16 last_recv_sn;
+	struct sk_buff_head queue;
 };
 
 struct rtl_ps_ctl {
@@ -2297,14 +2345,12 @@ struct rtl_hal_ops {
 	void (*set_default_port_id_cmd)(struct ieee80211_hw *hw);
 	bool (*get_btc_status) (void);
 	bool (*is_fw_header)(struct rtlwifi_firmware_header *hdr);
-	u32 (*rx_command_packet)(struct ieee80211_hw *hw,
-				 const struct rtl_stats *status, struct sk_buff *skb);
 	void (*add_wowlan_pattern)(struct ieee80211_hw *hw,
 				   struct rtl_wow_pattern *rtl_pattern,
 				   u8 index);
 	u16 (*get_available_desc)(struct ieee80211_hw *hw, u8 q_idx);
-	void (*c2h_content_parsing)(struct ieee80211_hw *hw, u8 tag, u8 len,
-				    u8 *val);
+	void (*c2h_ra_report_handler)(struct ieee80211_hw *hw,
+				      u8 *cmd_buf, u8 cmd_len);
 };
 
 struct rtl_intf_ops {
@@ -2750,7 +2796,7 @@ struct rtl_priv {
 	struct list_head entry_list;
 
 	/* c2hcmd list for kthread level access */
-	struct list_head c2hcmd_list;
+	struct sk_buff_head c2hcmd_queue;
 
 	struct rtl_debug dbg;
 	int max_fw_size;
@@ -2842,11 +2888,6 @@ enum bt_co_type {
 	BT_RTL8812A = 11,
 };
 
-enum bt_total_ant_num {
-	ANT_TOTAL_X2 = 0,
-	ANT_TOTAL_X1 = 1
-};
-
 enum bt_cur_state {
 	BT_OFF = 0,
 	BT_ON = 1,
diff --git a/drivers/net/wireless/rsi/rsi_91x_coex.c b/drivers/net/wireless/rsi/rsi_91x_coex.c
index d055099dadf1..c8ba148f8c6c 100644
--- a/drivers/net/wireless/rsi/rsi_91x_coex.c
+++ b/drivers/net/wireless/rsi/rsi_91x_coex.c
@@ -73,6 +73,7 @@ int rsi_coex_recv_pkt(struct rsi_common *common, u8 *msg)
 	switch (msg_type) {
 	case COMMON_CARD_READY_IND:
 		rsi_dbg(INFO_ZONE, "common card ready received\n");
+		common->hibernate_resume = false;
 		rsi_handle_card_ready(common, msg);
 		break;
 	case SLEEP_NOTIFY_IND:
diff --git a/drivers/net/wireless/rsi/rsi_91x_core.c b/drivers/net/wireless/rsi/rsi_91x_core.c
index 5dafd2e1306c..3644d7d99463 100644
--- a/drivers/net/wireless/rsi/rsi_91x_core.c
+++ b/drivers/net/wireless/rsi/rsi_91x_core.c
@@ -411,11 +411,30 @@ void rsi_core_xmit(struct rsi_common *common, struct sk_buff *skb)
 	if ((ieee80211_is_mgmt(wh->frame_control)) ||
 	    (ieee80211_is_ctl(wh->frame_control)) ||
 	    (ieee80211_is_qos_nullfunc(wh->frame_control))) {
+		if (ieee80211_is_assoc_req(wh->frame_control) ||
+		    ieee80211_is_reassoc_req(wh->frame_control)) {
+			struct ieee80211_bss_conf *bss = &vif->bss_conf;
+
+			common->eapol4_confirm = false;
+			rsi_hal_send_sta_notify_frame(common,
+						      RSI_IFTYPE_STATION,
+						      STA_CONNECTED, bss->bssid,
+						      bss->qos, bss->aid, 0,
+						      vif);
+		}
+
 		q_num = MGMT_SOFT_Q;
 		skb->priority = q_num;
+
+		if (rsi_prepare_mgmt_desc(common, skb)) {
+			rsi_dbg(ERR_ZONE, "Failed to prepare desc\n");
+			goto xmit_fail;
+		}
 	} else {
 		if (ieee80211_is_data_qos(wh->frame_control)) {
-			tid = (skb->data[24] & IEEE80211_QOS_TID);
+			u8 *qos = ieee80211_get_qos_ctl(wh);
+
+			tid = *qos & IEEE80211_QOS_CTL_TID_MASK;
 			skb->priority = TID_TO_WME_AC(tid);
 		} else {
 			tid = IEEE80211_NONQOS_TID;
@@ -433,6 +452,8 @@ void rsi_core_xmit(struct rsi_common *common, struct sk_buff *skb)
 			if (!rsta)
 				goto xmit_fail;
 			tx_params->sta_id = rsta->sta_id;
+		} else {
+			tx_params->sta_id = 0;
 		}
 
 		if (rsta) {
@@ -443,6 +464,14 @@ void rsi_core_xmit(struct rsi_common *common, struct sk_buff *skb)
 							      tid, 0);
 			}
 		}
+		if (skb->protocol == cpu_to_be16(ETH_P_PAE)) {
+			q_num = MGMT_SOFT_Q;
+			skb->priority = q_num;
+		}
+		if (rsi_prepare_data_desc(common, skb)) {
+			rsi_dbg(ERR_ZONE, "Failed to prepare data desc\n");
+			goto xmit_fail;
+		}
 	}
 
 	if ((q_num < MGMT_SOFT_Q) &&
@@ -456,7 +485,7 @@ void rsi_core_xmit(struct rsi_common *common, struct sk_buff *skb)
 	}
 
 	rsi_core_queue_pkt(common, skb);
-	rsi_dbg(DATA_TX_ZONE, "%s: ===> Scheduling TX thead <===\n", __func__);
+	rsi_dbg(DATA_TX_ZONE, "%s: ===> Scheduling TX thread <===\n", __func__);
 	rsi_set_event(&common->tx_thread.event);
 
 	return;
diff --git a/drivers/net/wireless/rsi/rsi_91x_hal.c b/drivers/net/wireless/rsi/rsi_91x_hal.c
index de608ae365a4..0761e61591bd 100644
--- a/drivers/net/wireless/rsi/rsi_91x_hal.c
+++ b/drivers/net/wireless/rsi/rsi_91x_hal.c
@@ -45,7 +45,7 @@ int rsi_send_pkt_to_bus(struct rsi_common *common, struct sk_buff *skb)
 	return status;
 }
 
-static int rsi_prepare_mgmt_desc(struct rsi_common *common, struct sk_buff *skb)
+int rsi_prepare_mgmt_desc(struct rsi_common *common, struct sk_buff *skb)
 {
 	struct rsi_hw *adapter = common->priv;
 	struct ieee80211_hdr *wh = NULL;
@@ -55,7 +55,7 @@ static int rsi_prepare_mgmt_desc(struct rsi_common *common, struct sk_buff *skb)
 	struct rsi_mgmt_desc *mgmt_desc;
 	struct skb_info *tx_params;
 	struct ieee80211_bss_conf *bss = NULL;
-	struct xtended_desc *xtend_desc = NULL;
+	struct rsi_xtended_desc *xtend_desc = NULL;
 	u8 header_size;
 	u32 dword_align_bytes = 0;
 
@@ -69,7 +69,7 @@ static int rsi_prepare_mgmt_desc(struct rsi_common *common, struct sk_buff *skb)
 	vif = tx_params->vif;
 
 	/* Update header size */
-	header_size = FRAME_DESC_SZ + sizeof(struct xtended_desc);
+	header_size = FRAME_DESC_SZ + sizeof(struct rsi_xtended_desc);
 	if (header_size > skb_headroom(skb)) {
 		rsi_dbg(ERR_ZONE,
 			"%s: Failed to add extended descriptor\n",
@@ -92,7 +92,7 @@ static int rsi_prepare_mgmt_desc(struct rsi_common *common, struct sk_buff *skb)
 	wh = (struct ieee80211_hdr *)&skb->data[header_size];
 
 	mgmt_desc = (struct rsi_mgmt_desc *)skb->data;
-	xtend_desc = (struct xtended_desc *)&skb->data[FRAME_DESC_SZ];
+	xtend_desc = (struct rsi_xtended_desc *)&skb->data[FRAME_DESC_SZ];
 
 	rsi_set_len_qno(&mgmt_desc->len_qno, (skb->len - FRAME_DESC_SZ),
 			RSI_WIFI_MGMT_Q);
@@ -113,17 +113,6 @@ static int rsi_prepare_mgmt_desc(struct rsi_common *common, struct sk_buff *skb)
 	if (conf_is_ht40(conf))
 		mgmt_desc->bbp_info = cpu_to_le16(FULL40M_ENABLE);
 
-	if (ieee80211_is_probe_req(wh->frame_control)) {
-		if (!bss->assoc) {
-			rsi_dbg(INFO_ZONE,
-				"%s: blocking mgmt queue\n", __func__);
-			mgmt_desc->misc_flags = RSI_DESC_REQUIRE_CFM_TO_HOST;
-			xtend_desc->confirm_frame_type = PROBEREQ_CONFIRM;
-			common->mgmt_q_block = true;
-			rsi_dbg(INFO_ZONE, "Mgmt queue blocked\n");
-		}
-	}
-
 	if (ieee80211_is_probe_resp(wh->frame_control)) {
 		mgmt_desc->misc_flags |= (RSI_ADD_DELTA_TSF_VAP_ID |
 					  RSI_FETCH_RETRY_CNT_FRM_HST);
@@ -149,7 +138,7 @@ static int rsi_prepare_mgmt_desc(struct rsi_common *common, struct sk_buff *skb)
 }
 
 /* This function prepares descriptor for given data packet */
-static int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb)
+int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb)
 {
 	struct rsi_hw *adapter = common->priv;
 	struct ieee80211_vif *vif;
@@ -158,7 +147,7 @@ static int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb)
 	struct skb_info *tx_params;
 	struct ieee80211_bss_conf *bss;
 	struct rsi_data_desc *data_desc;
-	struct xtended_desc *xtend_desc;
+	struct rsi_xtended_desc *xtend_desc;
 	u8 ieee80211_size = MIN_802_11_HDR_LEN;
 	u8 header_size;
 	u8 vap_id = 0;
@@ -170,7 +159,7 @@ static int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb)
 	bss = &vif->bss_conf;
 	tx_params = (struct skb_info *)info->driver_data;
 
-	header_size = FRAME_DESC_SZ + sizeof(struct xtended_desc);
+	header_size = FRAME_DESC_SZ + sizeof(struct rsi_xtended_desc);
 	if (header_size > skb_headroom(skb)) {
 		rsi_dbg(ERR_ZONE, "%s: Unable to send pkt\n", __func__);
 		return -ENOSPC;
@@ -188,7 +177,7 @@ static int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb)
 	data_desc = (struct rsi_data_desc *)skb->data;
 	memset(data_desc, 0, header_size);
 
-	xtend_desc = (struct xtended_desc *)&skb->data[FRAME_DESC_SZ];
+	xtend_desc = (struct rsi_xtended_desc *)&skb->data[FRAME_DESC_SZ];
 	wh = (struct ieee80211_hdr *)&skb->data[header_size];
 	seq_num = IEEE80211_SEQ_TO_SN(le16_to_cpu(wh->seq_ctrl));
 
@@ -243,6 +232,18 @@ static int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb)
 		data_desc->misc_flags |= RSI_FETCH_RETRY_CNT_FRM_HST;
 #define EAPOL_RETRY_CNT 15
 		xtend_desc->retry_cnt = EAPOL_RETRY_CNT;
+
+		if (common->eapol4_confirm)
+			skb->priority = VO_Q;
+		else
+			rsi_set_len_qno(&data_desc->len_qno,
+					(skb->len - FRAME_DESC_SZ),
+					RSI_WIFI_MGMT_Q);
+		if ((skb->len - header_size) == EAPOL4_PACKET_LEN) {
+			data_desc->misc_flags |=
+				RSI_DESC_REQUIRE_CFM_TO_HOST;
+			xtend_desc->confirm_frame_type = EAPOL4_CONFIRM;
+		}
 	}
 
 	data_desc->mac_flags = cpu_to_le16(seq_num & 0xfff);
@@ -282,8 +283,11 @@ int rsi_send_data_pkt(struct rsi_common *common, struct sk_buff *skb)
 	struct rsi_hw *adapter = common->priv;
 	struct ieee80211_vif *vif;
 	struct ieee80211_tx_info *info;
+	struct skb_info *tx_params;
 	struct ieee80211_bss_conf *bss;
+	struct ieee80211_hdr *wh;
 	int status = -EINVAL;
+	u8 header_size;
 
 	if (!skb)
 		return 0;
@@ -295,16 +299,15 @@ int rsi_send_data_pkt(struct rsi_common *common, struct sk_buff *skb)
 		goto err;
 	vif = info->control.vif;
 	bss = &vif->bss_conf;
+	tx_params = (struct skb_info *)info->driver_data;
+	header_size = tx_params->internal_hdr_size;
+	wh = (struct ieee80211_hdr *)&skb->data[header_size];
 
 	if (((vif->type == NL80211_IFTYPE_STATION) ||
 	     (vif->type == NL80211_IFTYPE_P2P_CLIENT)) &&
 	    (!bss->assoc))
 		goto err;
 
-	status = rsi_prepare_data_desc(common, skb);
-	if (status)
-		goto err;
-
 	status = rsi_send_pkt_to_bus(common, skb);
 	if (status)
 		rsi_dbg(ERR_ZONE, "%s: Failed to write pkt\n", __func__);
@@ -327,12 +330,18 @@ int rsi_send_mgmt_pkt(struct rsi_common *common,
 		      struct sk_buff *skb)
 {
 	struct rsi_hw *adapter = common->priv;
+	struct ieee80211_bss_conf *bss;
+	struct ieee80211_hdr *wh;
 	struct ieee80211_tx_info *info;
 	struct skb_info *tx_params;
+	struct rsi_mgmt_desc *mgmt_desc;
+	struct rsi_xtended_desc *xtend_desc;
 	int status = -E2BIG;
+	u8 header_size;
 
 	info = IEEE80211_SKB_CB(skb);
 	tx_params = (struct skb_info *)info->driver_data;
+	header_size = tx_params->internal_hdr_size;
 
 	if (tx_params->flags & INTERNAL_MGMT_PKT) {
 		status = adapter->host_intf_ops->write_pkt(common->priv,
@@ -346,15 +355,25 @@ int rsi_send_mgmt_pkt(struct rsi_common *common,
 		return status;
 	}
 
-	if (FRAME_DESC_SZ > skb_headroom(skb))
-		goto err;
+	bss = &info->control.vif->bss_conf;
+	wh = (struct ieee80211_hdr *)&skb->data[header_size];
+	mgmt_desc = (struct rsi_mgmt_desc *)skb->data;
+	xtend_desc = (struct rsi_xtended_desc *)&skb->data[FRAME_DESC_SZ];
+
+	/* Indicate to firmware to give cfm for probe */
+	if (ieee80211_is_probe_req(wh->frame_control) && !bss->assoc) {
+		rsi_dbg(INFO_ZONE,
+			"%s: blocking mgmt queue\n", __func__);
+		mgmt_desc->misc_flags = RSI_DESC_REQUIRE_CFM_TO_HOST;
+		xtend_desc->confirm_frame_type = PROBEREQ_CONFIRM;
+		common->mgmt_q_block = true;
+		rsi_dbg(INFO_ZONE, "Mgmt queue blocked\n");
+	}
 
-	rsi_prepare_mgmt_desc(common, skb);
 	status = rsi_send_pkt_to_bus(common, skb);
 	if (status)
 		rsi_dbg(ERR_ZONE, "%s: Failed to write the packet\n", __func__);
 
-err:
 	rsi_indicate_tx_status(common->priv, skb, status);
 	return status;
 }
@@ -616,28 +635,32 @@ static int bl_write_header(struct rsi_hw *adapter, u8 *flash_content,
 			   u32 content_size)
 {
 	struct rsi_host_intf_ops *hif_ops = adapter->host_intf_ops;
-	struct bl_header bl_hdr;
+	struct bl_header *bl_hdr;
 	u32 write_addr, write_len;
 	int status;
 
-	bl_hdr.flags = 0;
-	bl_hdr.image_no = cpu_to_le32(adapter->priv->coex_mode);
-	bl_hdr.check_sum = cpu_to_le32(
-				*(u32 *)&flash_content[CHECK_SUM_OFFSET]);
-	bl_hdr.flash_start_address = cpu_to_le32(
-					*(u32 *)&flash_content[ADDR_OFFSET]);
-	bl_hdr.flash_len = cpu_to_le32(*(u32 *)&flash_content[LEN_OFFSET]);
+	bl_hdr = kzalloc(sizeof(*bl_hdr), GFP_KERNEL);
+	if (!bl_hdr)
+		return -ENOMEM;
+
+	bl_hdr->flags = 0;
+	bl_hdr->image_no = cpu_to_le32(adapter->priv->coex_mode);
+	bl_hdr->check_sum =
+		cpu_to_le32(*(u32 *)&flash_content[CHECK_SUM_OFFSET]);
+	bl_hdr->flash_start_address =
+		cpu_to_le32(*(u32 *)&flash_content[ADDR_OFFSET]);
+	bl_hdr->flash_len = cpu_to_le32(*(u32 *)&flash_content[LEN_OFFSET]);
 	write_len = sizeof(struct bl_header);
 
 	if (adapter->rsi_host_intf == RSI_HOST_INTF_USB) {
 		write_addr = PING_BUFFER_ADDRESS;
 		status = hif_ops->write_reg_multiple(adapter, write_addr,
-						 (u8 *)&bl_hdr, write_len);
+						 (u8 *)bl_hdr, write_len);
 		if (status < 0) {
 			rsi_dbg(ERR_ZONE,
 				"%s: Failed to load Version/CRC structure\n",
 				__func__);
-			return status;
+			goto fail;
 		}
 	} else {
 		write_addr = PING_BUFFER_ADDRESS >> 16;
@@ -646,20 +669,23 @@ static int bl_write_header(struct rsi_hw *adapter, u8 *flash_content,
 			rsi_dbg(ERR_ZONE,
 				"%s: Unable to set ms word to common reg\n",
 				__func__);
-			return status;
+			goto fail;
 		}
 		write_addr = RSI_SD_REQUEST_MASTER |
 			     (PING_BUFFER_ADDRESS & 0xFFFF);
 		status = hif_ops->write_reg_multiple(adapter, write_addr,
-						 (u8 *)&bl_hdr, write_len);
+						 (u8 *)bl_hdr, write_len);
 		if (status < 0) {
 			rsi_dbg(ERR_ZONE,
 				"%s: Failed to load Version/CRC structure\n",
 				__func__);
-			return status;
+			goto fail;
 		}
 	}
-	return 0;
+	status = 0;
+fail:
+	kfree(bl_hdr);
+	return status;
 }
 
 static u32 read_flash_capacity(struct rsi_hw *adapter)
diff --git a/drivers/net/wireless/rsi/rsi_91x_mac80211.c b/drivers/net/wireless/rsi/rsi_91x_mac80211.c
index 32f5cb46fd4f..2ca7464b7fa3 100644
--- a/drivers/net/wireless/rsi/rsi_91x_mac80211.c
+++ b/drivers/net/wireless/rsi/rsi_91x_mac80211.c
@@ -245,6 +245,7 @@ void rsi_mac80211_detach(struct rsi_hw *adapter)
 		ieee80211_stop_queues(hw);
 		ieee80211_unregister_hw(hw);
 		ieee80211_free_hw(hw);
+		adapter->hw = NULL;
 	}
 
 	for (band = 0; band < NUM_NL80211_BANDS; band++) {
@@ -614,7 +615,7 @@ static int rsi_mac80211_config(struct ieee80211_hw *hw,
 
 	/* Power save parameters */
 	if (changed & IEEE80211_CONF_CHANGE_PS) {
-		struct ieee80211_vif *vif;
+		struct ieee80211_vif *vif, *sta_vif = NULL;
 		unsigned long flags;
 		int i, set_ps = 1;
 
@@ -628,13 +629,17 @@ static int rsi_mac80211_config(struct ieee80211_hw *hw,
 				set_ps = 0;
 				break;
 			}
+			if ((vif->type == NL80211_IFTYPE_STATION ||
+			     vif->type == NL80211_IFTYPE_P2P_CLIENT) &&
+			    (!sta_vif || vif->bss_conf.assoc))
+				sta_vif = vif;
 		}
-		if (set_ps) {
+		if (set_ps && sta_vif) {
 			spin_lock_irqsave(&adapter->ps_lock, flags);
 			if (conf->flags & IEEE80211_CONF_PS)
-				rsi_enable_ps(adapter, vif);
+				rsi_enable_ps(adapter, sta_vif);
 			else
-				rsi_disable_ps(adapter, vif);
+				rsi_disable_ps(adapter, sta_vif);
 			spin_unlock_irqrestore(&adapter->ps_lock, flags);
 		}
 	}
@@ -737,7 +742,8 @@ static void rsi_mac80211_bss_info_changed(struct ieee80211_hw *hw,
 				      bss_conf->bssid,
 				      bss_conf->qos,
 				      bss_conf->aid,
-				      NULL, 0, vif);
+				      NULL, 0,
+				      bss_conf->assoc_capability, vif);
 		adapter->ps_info.dtim_interval_duration = bss->dtim_period;
 		adapter->ps_info.listen_interval = conf->listen_interval;
 
@@ -906,14 +912,25 @@ static int rsi_hal_key_config(struct ieee80211_hw *hw,
 		}
 	}
 
-	return rsi_hal_load_key(adapter->priv,
-				key->key,
-				key->keylen,
-				key_type,
-				key->keyidx,
-				key->cipher,
-				sta_id,
-				vif);
+	status = rsi_hal_load_key(adapter->priv,
+				  key->key,
+				  key->keylen,
+				  key_type,
+				  key->keyidx,
+				  key->cipher,
+				  sta_id,
+				  vif);
+	if (status)
+		return status;
+
+	if (vif->type == NL80211_IFTYPE_STATION && key->key &&
+	    (key->cipher == WLAN_CIPHER_SUITE_WEP104 ||
+	     key->cipher == WLAN_CIPHER_SUITE_WEP40)) {
+		if (!rsi_send_block_unblock_frame(adapter->priv, false))
+			adapter->priv->hw_data_qs_blocked = false;
+	}
+
+	return 0;
 }
 
 /**
@@ -1086,7 +1103,7 @@ static int rsi_mac80211_ampdu_action(struct ieee80211_hw *hw,
 		break;
 
 	default:
-		rsi_dbg(ERR_ZONE, "%s: Uknown AMPDU action\n", __func__);
+		rsi_dbg(ERR_ZONE, "%s: Unknown AMPDU action\n", __func__);
 		break;
 	}
 
@@ -1391,7 +1408,7 @@ static int rsi_mac80211_sta_add(struct ieee80211_hw *hw,
 			rsi_dbg(INFO_ZONE, "Indicate bss status to device\n");
 			rsi_inform_bss_status(common, RSI_OPMODE_AP, 1,
 					      sta->addr, sta->wme, sta->aid,
-					      sta, sta_idx, vif);
+					      sta, sta_idx, 0, vif);
 
 			if (common->key) {
 				struct ieee80211_key_conf *key = common->key;
@@ -1469,7 +1486,7 @@ static int rsi_mac80211_sta_remove(struct ieee80211_hw *hw,
 				rsi_inform_bss_status(common, RSI_OPMODE_AP, 0,
 						      sta->addr, sta->wme,
 						      sta->aid, sta, sta_idx,
-						      vif);
+						      0, vif);
 				rsta->sta = NULL;
 				rsta->sta_id = -1;
 				for (cnt = 0; cnt < IEEE80211_NUM_TIDS; cnt++)
@@ -1788,15 +1805,21 @@ int rsi_config_wowlan(struct rsi_hw *adapter, struct cfg80211_wowlan *wowlan)
 	struct rsi_common *common = adapter->priv;
 	u16 triggers = 0;
 	u16 rx_filter_word = 0;
-	struct ieee80211_bss_conf *bss = &adapter->vifs[0]->bss_conf;
+	struct ieee80211_bss_conf *bss = NULL;
 
 	rsi_dbg(INFO_ZONE, "Config WoWLAN to device\n");
 
+	if (!adapter->vifs[0])
+		return -EINVAL;
+
+	bss = &adapter->vifs[0]->bss_conf;
+
 	if (WARN_ON(!wowlan)) {
 		rsi_dbg(ERR_ZONE, "WoW triggers not enabled\n");
 		return -EINVAL;
 	}
 
+	common->wow_flags |= RSI_WOW_ENABLED;
 	triggers = rsi_wow_map_triggers(common, wowlan);
 	if (!triggers) {
 		rsi_dbg(ERR_ZONE, "%s:No valid WoW triggers\n", __func__);
@@ -1819,7 +1842,6 @@ int rsi_config_wowlan(struct rsi_hw *adapter, struct cfg80211_wowlan *wowlan)
 
 	rx_filter_word = (ALLOW_DATA_ASSOC_PEER | DISALLOW_BEACONS);
 	rsi_send_rx_filter_frame(common, rx_filter_word);
-	common->wow_flags |= RSI_WOW_ENABLED;
 
 	return 0;
 }
@@ -1939,9 +1961,8 @@ int rsi_mac80211_attach(struct rsi_common *common)
 	hw->uapsd_queues = RSI_IEEE80211_UAPSD_QUEUES;
 	hw->uapsd_max_sp_len = IEEE80211_WMM_IE_STA_QOSINFO_SP_ALL;
 
-	hw->max_tx_aggregation_subframes = 6;
-	rsi_register_rates_channels(adapter, NL80211_BAND_2GHZ);
-	rsi_register_rates_channels(adapter, NL80211_BAND_5GHZ);
+	hw->max_tx_aggregation_subframes = RSI_MAX_TX_AGGR_FRMS;
+	hw->max_rx_aggregation_subframes = RSI_MAX_RX_AGGR_FRMS;
 	hw->rate_control_algorithm = "AARF";
 
 	SET_IEEE80211_PERM_ADDR(hw, common->mac_addr);
@@ -1962,10 +1983,15 @@ int rsi_mac80211_attach(struct rsi_common *common)
 
 	wiphy->available_antennas_rx = 1;
 	wiphy->available_antennas_tx = 1;
+
+	rsi_register_rates_channels(adapter, NL80211_BAND_2GHZ);
 	wiphy->bands[NL80211_BAND_2GHZ] =
 		&adapter->sbands[NL80211_BAND_2GHZ];
-	wiphy->bands[NL80211_BAND_5GHZ] =
-		&adapter->sbands[NL80211_BAND_5GHZ];
+	if (common->num_supp_bands > 1) {
+		rsi_register_rates_channels(adapter, NL80211_BAND_5GHZ);
+		wiphy->bands[NL80211_BAND_5GHZ] =
+			&adapter->sbands[NL80211_BAND_5GHZ];
+	}
 
 	/* AP Parameters */
 	wiphy->max_ap_assoc_sta = rsi_max_ap_stas[common->oper_mode - 1];
@@ -1991,6 +2017,9 @@ int rsi_mac80211_attach(struct rsi_common *common)
 	wiphy->iface_combinations = rsi_iface_combinations;
 	wiphy->n_iface_combinations = ARRAY_SIZE(rsi_iface_combinations);
 
+	if (common->coex_mode > 1)
+		wiphy->flags |= WIPHY_FLAG_PS_ON_BY_DEFAULT;
+
 	status = ieee80211_register_hw(hw);
 	if (status)
 		return status;
diff --git a/drivers/net/wireless/rsi/rsi_91x_mgmt.c b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
index c21fca750fd4..d0e5937cad6d 100644
--- a/drivers/net/wireless/rsi/rsi_91x_mgmt.c
+++ b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
@@ -325,8 +325,8 @@ static int rsi_load_radio_caps(struct rsi_common *common)
 	radio_caps->channel_num = common->channel;
 	radio_caps->rf_model = RSI_RF_TYPE;
 
+	radio_caps->radio_cfg_info = RSI_LMAC_CLOCK_80MHZ;
 	if (common->channel_width == BW_40MHZ) {
-		radio_caps->radio_cfg_info = RSI_LMAC_CLOCK_80MHZ;
 		radio_caps->radio_cfg_info |= RSI_ENABLE_40MHZ;
 
 		if (common->fsm_state == FSM_MAC_INIT_DONE) {
@@ -454,14 +454,10 @@ static int rsi_mgmt_pkt_to_core(struct rsi_common *common,
  *
  * Return: status: 0 on success, corresponding negative error code on failure.
  */
-static int rsi_hal_send_sta_notify_frame(struct rsi_common *common,
-					 enum opmode opmode,
-					 u8 notify_event,
-					 const unsigned char *bssid,
-					 u8 qos_enable,
-					 u16 aid,
-					 u16 sta_id,
-					 struct ieee80211_vif *vif)
+int rsi_hal_send_sta_notify_frame(struct rsi_common *common, enum opmode opmode,
+				  u8 notify_event, const unsigned char *bssid,
+				  u8 qos_enable, u16 aid, u16 sta_id,
+				  struct ieee80211_vif *vif)
 {
 	struct sk_buff *skb = NULL;
 	struct rsi_peer_notify *peer_notify;
@@ -1194,6 +1190,7 @@ static int rsi_send_auto_rate_request(struct rsi_common *common,
 		return -ENOMEM;
 	}
 
+	memset(skb->data, 0, frame_len);
 	selected_rates = kzalloc(2 * RSI_TBL_SZ, GFP_KERNEL);
 	if (!selected_rates) {
 		rsi_dbg(ERR_ZONE, "%s: Failed in allocation of mem\n",
@@ -1328,6 +1325,7 @@ void rsi_inform_bss_status(struct rsi_common *common,
 			   u16 aid,
 			   struct ieee80211_sta *sta,
 			   u16 sta_id,
+			   u16 assoc_cap,
 			   struct ieee80211_vif *vif)
 {
 	if (status) {
@@ -1342,10 +1340,10 @@ void rsi_inform_bss_status(struct rsi_common *common,
 					      vif);
 		if (common->min_rate == 0xffff)
 			rsi_send_auto_rate_request(common, sta, sta_id, vif);
-		if (opmode == RSI_OPMODE_STA) {
-			if (!rsi_send_block_unblock_frame(common, false))
-				common->hw_data_qs_blocked = false;
-		}
+		if (opmode == RSI_OPMODE_STA &&
+		    !(assoc_cap & WLAN_CAPABILITY_PRIVACY) &&
+		    !rsi_send_block_unblock_frame(common, false))
+			common->hw_data_qs_blocked = false;
 	} else {
 		if (opmode == RSI_OPMODE_STA)
 			common->hw_data_qs_blocked = true;
@@ -1850,10 +1848,19 @@ int rsi_mgmt_pkt_recv(struct rsi_common *common, u8 *msg)
 			__func__);
 		return rsi_handle_card_ready(common, msg);
 	case TX_STATUS_IND:
-		if (msg[15] == PROBEREQ_CONFIRM) {
+		switch (msg[RSI_TX_STATUS_TYPE]) {
+		case PROBEREQ_CONFIRM:
 			common->mgmt_q_block = false;
 			rsi_dbg(FSM_ZONE, "%s: Probe confirm received\n",
 				__func__);
+			break;
+		case EAPOL4_CONFIRM:
+			if (msg[RSI_TX_STATUS]) {
+				common->eapol4_confirm = true;
+				if (!rsi_send_block_unblock_frame(common,
+								  false))
+					common->hw_data_qs_blocked = false;
+			}
 		}
 		break;
 	case BEACON_EVENT_IND:
diff --git a/drivers/net/wireless/rsi/rsi_91x_sdio.c b/drivers/net/wireless/rsi/rsi_91x_sdio.c
index d76e69c0beaa..416981d99229 100644
--- a/drivers/net/wireless/rsi/rsi_91x_sdio.c
+++ b/drivers/net/wireless/rsi/rsi_91x_sdio.c
@@ -170,7 +170,6 @@ static void rsi_reset_card(struct sdio_func *pfunction)
 	int err;
 	struct mmc_card *card = pfunction->card;
 	struct mmc_host *host = card->host;
-	s32 bit = (fls(host->ocr_avail) - 1);
 	u8 cmd52_resp;
 	u32 clock, resp, i;
 	u16 rca;
@@ -190,7 +189,6 @@ static void rsi_reset_card(struct sdio_func *pfunction)
 	msleep(20);
 
 	/* Initialize the SDIO card */
-	host->ios.vdd = bit;
 	host->ios.chip_select = MMC_CS_DONTCARE;
 	host->ios.bus_mode = MMC_BUSMODE_OPENDRAIN;
 	host->ios.power_mode = MMC_POWER_UP;
@@ -660,8 +658,6 @@ static int rsi_sdio_master_reg_read(struct rsi_hw *adapter, u32 addr,
 	if (!data)
 		return -ENOMEM;
 
-	data = PTR_ALIGN(data, 8);
-
 	ms_addr = (addr >> 16);
 	status = rsi_sdio_master_access_msword(adapter, ms_addr);
 	if (status < 0) {
@@ -724,8 +720,6 @@ static int rsi_sdio_master_reg_write(struct rsi_hw *adapter,
 	if (!data_aligned)
 		return -ENOMEM;
 
-	data_aligned = PTR_ALIGN(data_aligned, 8);
-
 	if (size == 2) {
 		*data_aligned = ((data << 16) | (data & 0xFFFF));
 	} else if (size == 1) {
@@ -1042,17 +1036,21 @@ static void ulp_read_write(struct rsi_hw *adapter, u16 addr, u32 data,
 /*This function resets and re-initializes the chip.*/
 static void rsi_reset_chip(struct rsi_hw *adapter)
 {
-	__le32 data;
+	u8 *data;
 	u8 sdio_interrupt_status = 0;
 	u8 request = 1;
 	int ret;
 
+	data = kzalloc(sizeof(u32), GFP_KERNEL);
+	if (!data)
+		return;
+
 	rsi_dbg(INFO_ZONE, "Writing disable to wakeup register\n");
 	ret =  rsi_sdio_write_register(adapter, 0, SDIO_WAKEUP_REG, &request);
 	if (ret < 0) {
 		rsi_dbg(ERR_ZONE,
 			"%s: Failed to write SDIO wakeup register\n", __func__);
-		return;
+		goto err;
 	}
 	msleep(20);
 	ret =  rsi_sdio_read_register(adapter, RSI_FN1_INT_REGISTER,
@@ -1060,7 +1058,7 @@ static void rsi_reset_chip(struct rsi_hw *adapter)
 	if (ret < 0) {
 		rsi_dbg(ERR_ZONE, "%s: Failed to Read Intr Status Register\n",
 			__func__);
-		return;
+		goto err;
 	}
 	rsi_dbg(INFO_ZONE, "%s: Intr Status Register value = %d\n",
 		__func__, sdio_interrupt_status);
@@ -1070,17 +1068,17 @@ static void rsi_reset_chip(struct rsi_hw *adapter)
 		rsi_dbg(ERR_ZONE,
 			"%s: Unable to set ms word to common reg\n",
 			__func__);
-		return;
+		goto err;
 	}
 
-	data = TA_HOLD_THREAD_VALUE;
+	put_unaligned_le32(TA_HOLD_THREAD_VALUE, data);
 	if (rsi_sdio_write_register_multiple(adapter, TA_HOLD_THREAD_REG |
 					     RSI_SD_REQUEST_MASTER,
-					     (u8 *)&data, 4)) {
+					     data, 4)) {
 		rsi_dbg(ERR_ZONE,
 			"%s: Unable to hold Thread-Arch processor threads\n",
 			__func__);
-		return;
+		goto err;
 	}
 
 	/* This msleep will ensure Thread-Arch processor to go to hold
@@ -1101,6 +1099,9 @@ static void rsi_reset_chip(struct rsi_hw *adapter)
 	 * read write operations to complete for chip reset.
 	 */
 	msleep(500);
+err:
+	kfree(data);
+	return;
 }
 
 /**
diff --git a/drivers/net/wireless/rsi/rsi_91x_usb.c b/drivers/net/wireless/rsi/rsi_91x_usb.c
index 7b8bae313aa9..6ce6b754df12 100644
--- a/drivers/net/wireless/rsi/rsi_91x_usb.c
+++ b/drivers/net/wireless/rsi/rsi_91x_usb.c
@@ -687,6 +687,14 @@ static int rsi_reset_card(struct rsi_hw *adapter)
 	 */
 	msleep(100);
 
+	ret = rsi_usb_master_reg_write(adapter, SWBL_REGOUT,
+				       RSI_FW_WDT_DISABLE_REQ,
+				       RSI_COMMON_REG_SIZE);
+	if (ret < 0) {
+		rsi_dbg(ERR_ZONE, "Disabling firmware watchdog timer failed\n");
+		goto fail;
+	}
+
 	ret = usb_ulp_read_write(adapter, RSI_WATCH_DOG_TIMER_1,
 				 RSI_ULP_WRITE_2, 32);
 	if (ret < 0)
diff --git a/drivers/net/wireless/rsi/rsi_boot_params.h b/drivers/net/wireless/rsi/rsi_boot_params.h
index 238ee96434ec..ad903b22440e 100644
--- a/drivers/net/wireless/rsi/rsi_boot_params.h
+++ b/drivers/net/wireless/rsi/rsi_boot_params.h
@@ -46,7 +46,8 @@
 	(((TA_PLL_M_VAL_20 + 1) * 40) / \
 	 ((TA_PLL_N_VAL_20 + 1) * (TA_PLL_P_VAL_20 + 1)))
 #define VALID_20 \
-	(WIFI_PLL960_CONFIGS | WIFI_AFEPLL_CONFIGS | WIFI_SWITCH_CLK_CONFIGS)
+	(WIFI_TAPLL_CONFIGS | WIFI_PLL960_CONFIGS | WIFI_AFEPLL_CONFIGS | \
+	 WIFI_SWITCH_CLK_CONFIGS | BOOTUP_MODE_INFO | CRYSTAL_GOOD_TIME)
 #define UMAC_CLK_40BW   \
 	(((TA_PLL_M_VAL_40 + 1) * 40) / \
 	 ((TA_PLL_N_VAL_40 + 1) * (TA_PLL_P_VAL_40 + 1)))
diff --git a/drivers/net/wireless/rsi/rsi_hal.h b/drivers/net/wireless/rsi/rsi_hal.h
index 786dccd0b732..327638cdd30b 100644
--- a/drivers/net/wireless/rsi/rsi_hal.h
+++ b/drivers/net/wireless/rsi/rsi_hal.h
@@ -115,6 +115,7 @@
 #define FW_FLASH_OFFSET			0x820
 #define LMAC_VER_OFFSET			(FW_FLASH_OFFSET + 0x200)
 #define MAX_DWORD_ALIGN_BYTES		64
+#define RSI_COMMON_REG_SIZE		2
 
 struct bl_header {
 	__le32 flags;
@@ -167,6 +168,8 @@ struct rsi_bt_desc {
 } __packed;
 
 int rsi_hal_device_init(struct rsi_hw *adapter);
+int rsi_prepare_mgmt_desc(struct rsi_common *common, struct sk_buff *skb);
+int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb);
 int rsi_prepare_beacon(struct rsi_common *common, struct sk_buff *skb);
 int rsi_send_pkt_to_bus(struct rsi_common *common, struct sk_buff *skb);
 int rsi_send_bt_pkt(struct rsi_common *common, struct sk_buff *skb);
diff --git a/drivers/net/wireless/rsi/rsi_main.h b/drivers/net/wireless/rsi/rsi_main.h
index ef4fa323694b..a084f224bb03 100644
--- a/drivers/net/wireless/rsi/rsi_main.h
+++ b/drivers/net/wireless/rsi/rsi_main.h
@@ -190,12 +190,6 @@ struct cqm_info {
 	u32 rssi_hyst;
 };
 
-struct xtended_desc {
-	u8 confirm_frame_type;
-	u8 retry_cnt;
-	u16 reserved;
-};
-
 enum rsi_dfs_regions {
 	RSI_REGION_FCC = 0,
 	RSI_REGION_ETSI,
@@ -293,6 +287,7 @@ struct rsi_common {
 	struct timer_list roc_timer;
 	struct ieee80211_vif *roc_vif;
 
+	bool eapol4_confirm;
 	void *bt_adapter;
 };
 
diff --git a/drivers/net/wireless/rsi/rsi_mgmt.h b/drivers/net/wireless/rsi/rsi_mgmt.h
index cf6567ae5bbe..14620935c925 100644
--- a/drivers/net/wireless/rsi/rsi_mgmt.h
+++ b/drivers/net/wireless/rsi/rsi_mgmt.h
@@ -33,6 +33,7 @@
 #define WMM_SHORT_SLOT_TIME             9
 #define SIFS_DURATION                   16
 
+#define EAPOL4_PACKET_LEN		0x85
 #define KEY_TYPE_CLEAR                  0
 #define RSI_PAIRWISE_KEY                1
 #define RSI_GROUP_KEY                   2
@@ -62,9 +63,12 @@
 #define RX_DOT11_MGMT                   0x02
 #define TX_STATUS_IND                   0x04
 #define BEACON_EVENT_IND		0x08
+#define EAPOL4_CONFIRM                  1
 #define PROBEREQ_CONFIRM                2
 #define CARD_READY_IND                  0x00
 #define SLEEP_NOTIFY_IND                0x06
+#define RSI_TX_STATUS_TYPE		15
+#define RSI_TX_STATUS			12
 
 #define RSI_DELETE_PEER                 0x0
 #define RSI_ADD_PEER                    0x1
@@ -221,6 +225,9 @@
 #define RSI_WOW_DISCONNECT		BIT(5)
 #endif
 
+#define RSI_MAX_TX_AGGR_FRMS		8
+#define RSI_MAX_RX_AGGR_FRMS		8
+
 enum opmode {
 	RSI_OPMODE_UNSUPPORTED = -1,
 	RSI_OPMODE_AP = 0,
@@ -301,6 +308,12 @@ struct rsi_mac_frame {
 #define ENCAP_MGMT_PKT			BIT(7)
 #define DESC_IMMEDIATE_WAKEUP		BIT(15)
 
+struct rsi_xtended_desc {
+	u8 confirm_frame_type;
+	u8 retry_cnt;
+	u16 reserved;
+};
+
 struct rsi_cmd_desc_dword0 {
 	__le16 len_qno;
 	u8 frame_type;
@@ -654,10 +667,14 @@ int rsi_set_channel(struct rsi_common *common,
 		    struct ieee80211_channel *channel);
 int rsi_send_vap_dynamic_update(struct rsi_common *common);
 int rsi_send_block_unblock_frame(struct rsi_common *common, bool event);
+int rsi_hal_send_sta_notify_frame(struct rsi_common *common, enum opmode opmode,
+				  u8 notify_event, const unsigned char *bssid,
+				  u8 qos_enable, u16 aid, u16 sta_id,
+				  struct ieee80211_vif *vif);
 void rsi_inform_bss_status(struct rsi_common *common, enum opmode opmode,
 			   u8 status, const u8 *addr, u8 qos_enable, u16 aid,
 			   struct ieee80211_sta *sta, u16 sta_id,
-			   struct ieee80211_vif *vif);
+			   u16 assoc_cap, struct ieee80211_vif *vif);
 void rsi_indicate_pkt_to_os(struct rsi_common *common, struct sk_buff *skb);
 int rsi_mac80211_attach(struct rsi_common *common);
 void rsi_indicate_tx_status(struct rsi_hw *common, struct sk_buff *skb,
diff --git a/drivers/net/wireless/rsi/rsi_sdio.h b/drivers/net/wireless/rsi/rsi_sdio.h
index ead8e7c4df3a..353dbdf31e75 100644
--- a/drivers/net/wireless/rsi/rsi_sdio.h
+++ b/drivers/net/wireless/rsi/rsi_sdio.h
@@ -87,7 +87,7 @@ enum sdio_interrupt_type {
 #define TA_SOFT_RST_CLR              0
 #define TA_SOFT_RST_SET              BIT(0)
 #define TA_PC_ZERO                   0
-#define TA_HOLD_THREAD_VALUE         cpu_to_le32(0xF)
+#define TA_HOLD_THREAD_VALUE         0xF
 #define TA_RELEASE_THREAD_VALUE      cpu_to_le32(0xF)
 #define TA_BASE_ADDR                 0x2200
 #define MISC_CFG_BASE_ADDR           0x4105
diff --git a/drivers/net/wireless/rsi/rsi_usb.h b/drivers/net/wireless/rsi/rsi_usb.h
index a88d59295a98..b6fe79f0a513 100644
--- a/drivers/net/wireless/rsi/rsi_usb.h
+++ b/drivers/net/wireless/rsi/rsi_usb.h
@@ -26,6 +26,7 @@
 #define RSI_USB_READY_MAGIC_NUM      0xab
 #define FW_STATUS_REG                0x41050012
 #define RSI_TA_HOLD_REG              0x22000844
+#define RSI_FW_WDT_DISABLE_REQ	     0x69
 
 #define USB_VENDOR_REGISTER_READ     0x15
 #define USB_VENDOR_REGISTER_WRITE    0x16
diff --git a/drivers/net/wireless/st/cw1200/txrx.c b/drivers/net/wireless/st/cw1200/txrx.c
index e9050b41157a..f7b1b0062db3 100644
--- a/drivers/net/wireless/st/cw1200/txrx.c
+++ b/drivers/net/wireless/st/cw1200/txrx.c
@@ -1069,7 +1069,7 @@ void cw1200_rx_cb(struct cw1200_common *priv,
 	}
 
 	if (skb->len < sizeof(struct ieee80211_pspoll)) {
-		wiphy_warn(priv->hw->wiphy, "Mailformed SDU rx'ed. Size is lesser than IEEE header.\n");
+		wiphy_warn(priv->hw->wiphy, "Malformed SDU rx'ed. Size is lesser than IEEE header.\n");
 		goto drop;
 	}
 
diff --git a/drivers/net/wireless/ti/wlcore/sdio.c b/drivers/net/wireless/ti/wlcore/sdio.c
index 1f727babbea0..750bea3574ee 100644
--- a/drivers/net/wireless/ti/wlcore/sdio.c
+++ b/drivers/net/wireless/ti/wlcore/sdio.c
@@ -155,47 +155,40 @@ static int wl12xx_sdio_power_on(struct wl12xx_sdio_glue *glue)
 	struct mmc_card *card = func->card;
 
 	ret = pm_runtime_get_sync(&card->dev);
-	if (ret) {
-		/*
-		 * Runtime PM might be temporarily disabled, or the device
-		 * might have a positive reference counter. Make sure it is
-		 * really powered on.
-		 */
-		ret = mmc_power_restore_host(card->host);
-		if (ret < 0) {
-			pm_runtime_put_sync(&card->dev);
-			goto out;
-		}
+	if (ret < 0) {
+		pm_runtime_put_noidle(&card->dev);
+		dev_err(glue->dev, "%s: failed to get_sync(%d)\n",
+			__func__, ret);
+
+		return ret;
 	}
 
 	sdio_claim_host(func);
 	sdio_enable_func(func);
 	sdio_release_host(func);
 
-out:
-	return ret;
+	return 0;
 }
 
 static int wl12xx_sdio_power_off(struct wl12xx_sdio_glue *glue)
 {
-	int ret;
 	struct sdio_func *func = dev_to_sdio_func(glue->dev);
 	struct mmc_card *card = func->card;
+	int error;
 
 	sdio_claim_host(func);
 	sdio_disable_func(func);
 	sdio_release_host(func);
 
-	/* Power off the card manually in case it wasn't powered off above */
-	ret = mmc_power_save_host(card->host);
-	if (ret < 0)
-		goto out;
-
 	/* Let runtime PM know the card is powered off */
-	pm_runtime_put_sync(&card->dev);
+	error = pm_runtime_put(&card->dev);
+	if (error < 0 && error != -EBUSY) {
+		dev_err(&card->dev, "%s failed: %i\n", __func__, error);
 
-out:
-	return ret;
+		return error;
+	}
+
+	return 0;
 }
 
 static int wl12xx_sdio_set_power(struct device *child, bool enable)
@@ -406,6 +399,11 @@ static int wl1271_suspend(struct device *dev)
 	mmc_pm_flag_t sdio_flags;
 	int ret = 0;
 
+	if (!wl) {
+		dev_err(dev, "no wilink module was probed\n");
+		goto out;
+	}
+
 	dev_dbg(dev, "wl1271 suspend. wow_enabled: %d\n",
 		wl->wow_enabled);
 
diff --git a/drivers/ntb/hw/idt/ntb_hw_idt.c b/drivers/ntb/hw/idt/ntb_hw_idt.c
index 8d98872d0983..dbe72f116017 100644
--- a/drivers/ntb/hw/idt/ntb_hw_idt.c
+++ b/drivers/ntb/hw/idt/ntb_hw_idt.c
@@ -1401,7 +1401,7 @@ static int idt_ntb_peer_mw_clear_trans(struct ntb_dev *ntb, int pidx,
  *                          5. Doorbell operations
  *
  *    Doorbell functionality of IDT PCIe-switches is pretty unusual. First of
- * all there is global doorbell register which state can by changed by any
+ * all there is global doorbell register which state can be changed by any
  * NT-function of the IDT device in accordance with global permissions. These
  * permissions configs are not supported by NTB API, so it must be done by
  * either BIOS or EEPROM settings. In the same way the state of the global
diff --git a/drivers/ntb/hw/intel/Makefile b/drivers/ntb/hw/intel/Makefile
index 1b434568d2ad..4ff22af967c6 100644
--- a/drivers/ntb/hw/intel/Makefile
+++ b/drivers/ntb/hw/intel/Makefile
@@ -1 +1,2 @@
 obj-$(CONFIG_NTB_INTEL) += ntb_hw_intel.o
+ntb_hw_intel-y := ntb_hw_gen1.o ntb_hw_gen3.o
diff --git a/drivers/ntb/hw/intel/ntb_hw_gen1.c b/drivers/ntb/hw/intel/ntb_hw_gen1.c
new file mode 100644
index 000000000000..ffdee98e8ece
--- /dev/null
+++ b/drivers/ntb/hw/intel/ntb_hw_gen1.c
@@ -0,0 +1,2070 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ *   redistributing this file, you may do so under either license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright(c) 2012 Intel Corporation. All rights reserved.
+ *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of version 2 of the GNU General Public License as
+ *   published by the Free Software Foundation.
+ *
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2012 Intel Corporation. All rights reserved.
+ *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copy
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Intel PCIe NTB Linux driver
+ */
+
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+#include <linux/ntb.h>
+
+#include "ntb_hw_intel.h"
+#include "ntb_hw_gen1.h"
+#include "ntb_hw_gen3.h"
+
+#define NTB_NAME	"ntb_hw_intel"
+#define NTB_DESC	"Intel(R) PCI-E Non-Transparent Bridge Driver"
+#define NTB_VER		"2.0"
+
+MODULE_DESCRIPTION(NTB_DESC);
+MODULE_VERSION(NTB_VER);
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("Intel Corporation");
+
+#define bar0_off(base, bar) ((base) + ((bar) << 2))
+#define bar2_off(base, bar) bar0_off(base, (bar) - 2)
+
+static const struct intel_ntb_reg xeon_reg;
+static const struct intel_ntb_alt_reg xeon_pri_reg;
+static const struct intel_ntb_alt_reg xeon_sec_reg;
+static const struct intel_ntb_alt_reg xeon_b2b_reg;
+static const struct intel_ntb_xlat_reg xeon_pri_xlat;
+static const struct intel_ntb_xlat_reg xeon_sec_xlat;
+static const struct ntb_dev_ops intel_ntb_ops;
+
+static const struct file_operations intel_ntb_debugfs_info;
+static struct dentry *debugfs_dir;
+
+static int b2b_mw_idx = -1;
+module_param(b2b_mw_idx, int, 0644);
+MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb.  A "
+		 "value of zero or positive starts from first mw idx, and a "
+		 "negative value starts from last mw idx.  Both sides MUST "
+		 "set the same value here!");
+
+static unsigned int b2b_mw_share;
+module_param(b2b_mw_share, uint, 0644);
+MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
+		 "ntb so that the peer ntb only occupies the first half of "
+		 "the mw, so the second half can still be used as a mw.  Both "
+		 "sides MUST set the same value here!");
+
+module_param_named(xeon_b2b_usd_bar2_addr64,
+		   xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
+MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
+		 "XEON B2B USD BAR 2 64-bit address");
+
+module_param_named(xeon_b2b_usd_bar4_addr64,
+		   xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
+MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64,
+		 "XEON B2B USD BAR 4 64-bit address");
+
+module_param_named(xeon_b2b_usd_bar4_addr32,
+		   xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
+MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32,
+		 "XEON B2B USD split-BAR 4 32-bit address");
+
+module_param_named(xeon_b2b_usd_bar5_addr32,
+		   xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
+MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32,
+		 "XEON B2B USD split-BAR 5 32-bit address");
+
+module_param_named(xeon_b2b_dsd_bar2_addr64,
+		   xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
+MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
+		 "XEON B2B DSD BAR 2 64-bit address");
+
+module_param_named(xeon_b2b_dsd_bar4_addr64,
+		   xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
+MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64,
+		 "XEON B2B DSD BAR 4 64-bit address");
+
+module_param_named(xeon_b2b_dsd_bar4_addr32,
+		   xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
+MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32,
+		 "XEON B2B DSD split-BAR 4 32-bit address");
+
+module_param_named(xeon_b2b_dsd_bar5_addr32,
+		   xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
+MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32,
+		 "XEON B2B DSD split-BAR 5 32-bit address");
+
+
+static int xeon_init_isr(struct intel_ntb_dev *ndev);
+
+static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
+{
+	ndev->unsafe_flags = 0;
+	ndev->unsafe_flags_ignore = 0;
+
+	/* Only B2B has a workaround to avoid SDOORBELL */
+	if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
+		if (!ntb_topo_is_b2b(ndev->ntb.topo))
+			ndev->unsafe_flags |= NTB_UNSAFE_DB;
+
+	/* No low level workaround to avoid SB01BASE */
+	if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
+		ndev->unsafe_flags |= NTB_UNSAFE_DB;
+		ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
+	}
+}
+
+static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
+				 unsigned long flag)
+{
+	return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
+}
+
+static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
+				     unsigned long flag)
+{
+	flag &= ndev->unsafe_flags;
+	ndev->unsafe_flags_ignore |= flag;
+
+	return !!flag;
+}
+
+int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
+{
+	if (idx < 0 || idx >= ndev->mw_count)
+		return -EINVAL;
+	return ndev->reg->mw_bar[idx];
+}
+
+static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
+			       phys_addr_t *db_addr, resource_size_t *db_size,
+			       phys_addr_t reg_addr, unsigned long reg)
+{
+	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
+		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
+
+	if (db_addr) {
+		*db_addr = reg_addr + reg;
+		dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr);
+	}
+
+	if (db_size) {
+		*db_size = ndev->reg->db_size;
+		dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size);
+	}
+
+	return 0;
+}
+
+u64 ndev_db_read(struct intel_ntb_dev *ndev,
+			       void __iomem *mmio)
+{
+	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
+		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
+
+	return ndev->reg->db_ioread(mmio);
+}
+
+int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
+				void __iomem *mmio)
+{
+	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
+		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
+
+	if (db_bits & ~ndev->db_valid_mask)
+		return -EINVAL;
+
+	ndev->reg->db_iowrite(db_bits, mmio);
+
+	return 0;
+}
+
+static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
+				   void __iomem *mmio)
+{
+	unsigned long irqflags;
+
+	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
+		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
+
+	if (db_bits & ~ndev->db_valid_mask)
+		return -EINVAL;
+
+	spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
+	{
+		ndev->db_mask |= db_bits;
+		ndev->reg->db_iowrite(ndev->db_mask, mmio);
+	}
+	spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
+
+	return 0;
+}
+
+static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
+				     void __iomem *mmio)
+{
+	unsigned long irqflags;
+
+	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
+		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
+
+	if (db_bits & ~ndev->db_valid_mask)
+		return -EINVAL;
+
+	spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
+	{
+		ndev->db_mask &= ~db_bits;
+		ndev->reg->db_iowrite(ndev->db_mask, mmio);
+	}
+	spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
+
+	return 0;
+}
+
+static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
+{
+	u64 shift, mask;
+
+	shift = ndev->db_vec_shift;
+	mask = BIT_ULL(shift) - 1;
+
+	return mask << (shift * db_vector);
+}
+
+static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
+				 phys_addr_t *spad_addr, phys_addr_t reg_addr,
+				 unsigned long reg)
+{
+	if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
+		pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return -EINVAL;
+
+	if (spad_addr) {
+		*spad_addr = reg_addr + reg + (idx << 2);
+		dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n",
+			*spad_addr);
+	}
+
+	return 0;
+}
+
+static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
+				 void __iomem *mmio)
+{
+	if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
+		pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return 0;
+
+	return ioread32(mmio + (idx << 2));
+}
+
+static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
+				  void __iomem *mmio)
+{
+	if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
+		pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
+
+	if (idx < 0 || idx >= ndev->spad_count)
+		return -EINVAL;
+
+	iowrite32(val, mmio + (idx << 2));
+
+	return 0;
+}
+
+static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
+{
+	u64 vec_mask;
+
+	vec_mask = ndev_vec_mask(ndev, vec);
+
+	if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31))
+		vec_mask |= ndev->db_link_mask;
+
+	dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask);
+
+	ndev->last_ts = jiffies;
+
+	if (vec_mask & ndev->db_link_mask) {
+		if (ndev->reg->poll_link(ndev))
+			ntb_link_event(&ndev->ntb);
+	}
+
+	if (vec_mask & ndev->db_valid_mask)
+		ntb_db_event(&ndev->ntb, vec);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t ndev_vec_isr(int irq, void *dev)
+{
+	struct intel_ntb_vec *nvec = dev;
+
+	dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d  nvec->num: %d\n",
+		irq, nvec->num);
+
+	return ndev_interrupt(nvec->ndev, nvec->num);
+}
+
+static irqreturn_t ndev_irq_isr(int irq, void *dev)
+{
+	struct intel_ntb_dev *ndev = dev;
+
+	return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
+}
+
+int ndev_init_isr(struct intel_ntb_dev *ndev,
+			 int msix_min, int msix_max,
+			 int msix_shift, int total_shift)
+{
+	struct pci_dev *pdev;
+	int rc, i, msix_count, node;
+
+	pdev = ndev->ntb.pdev;
+
+	node = dev_to_node(&pdev->dev);
+
+	/* Mask all doorbell interrupts */
+	ndev->db_mask = ndev->db_valid_mask;
+	ndev->reg->db_iowrite(ndev->db_mask,
+			      ndev->self_mmio +
+			      ndev->self_reg->db_mask);
+
+	/* Try to set up msix irq */
+
+	ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
+				 GFP_KERNEL, node);
+	if (!ndev->vec)
+		goto err_msix_vec_alloc;
+
+	ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
+				  GFP_KERNEL, node);
+	if (!ndev->msix)
+		goto err_msix_alloc;
+
+	for (i = 0; i < msix_max; ++i)
+		ndev->msix[i].entry = i;
+
+	msix_count = pci_enable_msix_range(pdev, ndev->msix,
+					   msix_min, msix_max);
+	if (msix_count < 0)
+		goto err_msix_enable;
+
+	for (i = 0; i < msix_count; ++i) {
+		ndev->vec[i].ndev = ndev;
+		ndev->vec[i].num = i;
+		rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
+				 "ndev_vec_isr", &ndev->vec[i]);
+		if (rc)
+			goto err_msix_request;
+	}
+
+	dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count);
+	ndev->db_vec_count = msix_count;
+	ndev->db_vec_shift = msix_shift;
+	return 0;
+
+err_msix_request:
+	while (i-- > 0)
+		free_irq(ndev->msix[i].vector, &ndev->vec[i]);
+	pci_disable_msix(pdev);
+err_msix_enable:
+	kfree(ndev->msix);
+err_msix_alloc:
+	kfree(ndev->vec);
+err_msix_vec_alloc:
+	ndev->msix = NULL;
+	ndev->vec = NULL;
+
+	/* Try to set up msi irq */
+
+	rc = pci_enable_msi(pdev);
+	if (rc)
+		goto err_msi_enable;
+
+	rc = request_irq(pdev->irq, ndev_irq_isr, 0,
+			 "ndev_irq_isr", ndev);
+	if (rc)
+		goto err_msi_request;
+
+	dev_dbg(&pdev->dev, "Using msi interrupts\n");
+	ndev->db_vec_count = 1;
+	ndev->db_vec_shift = total_shift;
+	return 0;
+
+err_msi_request:
+	pci_disable_msi(pdev);
+err_msi_enable:
+
+	/* Try to set up intx irq */
+
+	pci_intx(pdev, 1);
+
+	rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
+			 "ndev_irq_isr", ndev);
+	if (rc)
+		goto err_intx_request;
+
+	dev_dbg(&pdev->dev, "Using intx interrupts\n");
+	ndev->db_vec_count = 1;
+	ndev->db_vec_shift = total_shift;
+	return 0;
+
+err_intx_request:
+	return rc;
+}
+
+static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
+{
+	struct pci_dev *pdev;
+	int i;
+
+	pdev = ndev->ntb.pdev;
+
+	/* Mask all doorbell interrupts */
+	ndev->db_mask = ndev->db_valid_mask;
+	ndev->reg->db_iowrite(ndev->db_mask,
+			      ndev->self_mmio +
+			      ndev->self_reg->db_mask);
+
+	if (ndev->msix) {
+		i = ndev->db_vec_count;
+		while (i--)
+			free_irq(ndev->msix[i].vector, &ndev->vec[i]);
+		pci_disable_msix(pdev);
+		kfree(ndev->msix);
+		kfree(ndev->vec);
+	} else {
+		free_irq(pdev->irq, ndev);
+		if (pci_dev_msi_enabled(pdev))
+			pci_disable_msi(pdev);
+	}
+}
+
+static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf,
+				     size_t count, loff_t *offp)
+{
+	struct intel_ntb_dev *ndev;
+	struct pci_dev *pdev;
+	void __iomem *mmio;
+	char *buf;
+	size_t buf_size;
+	ssize_t ret, off;
+	union { u64 v64; u32 v32; u16 v16; u8 v8; } u;
+
+	ndev = filp->private_data;
+	pdev = ndev->ntb.pdev;
+	mmio = ndev->self_mmio;
+
+	buf_size = min(count, 0x800ul);
+
+	buf = kmalloc(buf_size, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	off = 0;
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "NTB Device Information:\n");
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Connection Topology -\t%s\n",
+			 ntb_topo_string(ndev->ntb.topo));
+
+	if (ndev->b2b_idx != UINT_MAX) {
+		off += scnprintf(buf + off, buf_size - off,
+				 "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
+		off += scnprintf(buf + off, buf_size - off,
+				 "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
+	}
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "BAR4 Split -\t\t%s\n",
+			 ndev->bar4_split ? "yes" : "no");
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
+	off += scnprintf(buf + off, buf_size - off,
+			 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
+
+	if (!ndev->reg->link_is_up(ndev)) {
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Status -\t\tDown\n");
+	} else {
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Status -\t\tUp\n");
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Speed -\t\tPCI-E Gen %u\n",
+				 NTB_LNK_STA_SPEED(ndev->lnk_sta));
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Width -\t\tx%u\n",
+				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
+	}
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Memory Window Count -\t%u\n", ndev->mw_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Scratchpad Count -\t%u\n", ndev->spad_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Count -\t%u\n", ndev->db_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
+
+	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Mask -\t\t%#llx\n", u.v64);
+
+	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Bell -\t\t%#llx\n", u.v64);
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "\nNTB Window Size:\n");
+
+	pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8);
+	off += scnprintf(buf + off, buf_size - off,
+			 "PBAR23SZ %hhu\n", u.v8);
+	if (!ndev->bar4_split) {
+		pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8);
+		off += scnprintf(buf + off, buf_size - off,
+				 "PBAR45SZ %hhu\n", u.v8);
+	} else {
+		pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8);
+		off += scnprintf(buf + off, buf_size - off,
+				 "PBAR4SZ %hhu\n", u.v8);
+		pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8);
+		off += scnprintf(buf + off, buf_size - off,
+				 "PBAR5SZ %hhu\n", u.v8);
+	}
+
+	pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8);
+	off += scnprintf(buf + off, buf_size - off,
+			 "SBAR23SZ %hhu\n", u.v8);
+	if (!ndev->bar4_split) {
+		pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8);
+		off += scnprintf(buf + off, buf_size - off,
+				 "SBAR45SZ %hhu\n", u.v8);
+	} else {
+		pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8);
+		off += scnprintf(buf + off, buf_size - off,
+				 "SBAR4SZ %hhu\n", u.v8);
+		pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8);
+		off += scnprintf(buf + off, buf_size - off,
+				 "SBAR5SZ %hhu\n", u.v8);
+	}
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "\nNTB Incoming XLAT:\n");
+
+	u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
+	off += scnprintf(buf + off, buf_size - off,
+			 "XLAT23 -\t\t%#018llx\n", u.v64);
+
+	if (ndev->bar4_split) {
+		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
+		off += scnprintf(buf + off, buf_size - off,
+				 "XLAT4 -\t\t\t%#06x\n", u.v32);
+
+		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
+		off += scnprintf(buf + off, buf_size - off,
+				 "XLAT5 -\t\t\t%#06x\n", u.v32);
+	} else {
+		u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
+		off += scnprintf(buf + off, buf_size - off,
+				 "XLAT45 -\t\t%#018llx\n", u.v64);
+	}
+
+	u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
+	off += scnprintf(buf + off, buf_size - off,
+			 "LMT23 -\t\t\t%#018llx\n", u.v64);
+
+	if (ndev->bar4_split) {
+		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
+		off += scnprintf(buf + off, buf_size - off,
+				 "LMT4 -\t\t\t%#06x\n", u.v32);
+		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
+		off += scnprintf(buf + off, buf_size - off,
+				 "LMT5 -\t\t\t%#06x\n", u.v32);
+	} else {
+		u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
+		off += scnprintf(buf + off, buf_size - off,
+				 "LMT45 -\t\t\t%#018llx\n", u.v64);
+	}
+
+	if (pdev_is_gen1(pdev)) {
+		if (ntb_topo_is_b2b(ndev->ntb.topo)) {
+			off += scnprintf(buf + off, buf_size - off,
+					 "\nNTB Outgoing B2B XLAT:\n");
+
+			u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
+			off += scnprintf(buf + off, buf_size - off,
+					 "B2B XLAT23 -\t\t%#018llx\n", u.v64);
+
+			if (ndev->bar4_split) {
+				u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
+				off += scnprintf(buf + off, buf_size - off,
+						 "B2B XLAT4 -\t\t%#06x\n",
+						 u.v32);
+				u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
+				off += scnprintf(buf + off, buf_size - off,
+						 "B2B XLAT5 -\t\t%#06x\n",
+						 u.v32);
+			} else {
+				u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
+				off += scnprintf(buf + off, buf_size - off,
+						 "B2B XLAT45 -\t\t%#018llx\n",
+						 u.v64);
+			}
+
+			u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
+			off += scnprintf(buf + off, buf_size - off,
+					 "B2B LMT23 -\t\t%#018llx\n", u.v64);
+
+			if (ndev->bar4_split) {
+				u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
+				off += scnprintf(buf + off, buf_size - off,
+						 "B2B LMT4 -\t\t%#06x\n",
+						 u.v32);
+				u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
+				off += scnprintf(buf + off, buf_size - off,
+						 "B2B LMT5 -\t\t%#06x\n",
+						 u.v32);
+			} else {
+				u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
+				off += scnprintf(buf + off, buf_size - off,
+						 "B2B LMT45 -\t\t%#018llx\n",
+						 u.v64);
+			}
+
+			off += scnprintf(buf + off, buf_size - off,
+					 "\nNTB Secondary BAR:\n");
+
+			u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
+			off += scnprintf(buf + off, buf_size - off,
+					 "SBAR01 -\t\t%#018llx\n", u.v64);
+
+			u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
+			off += scnprintf(buf + off, buf_size - off,
+					 "SBAR23 -\t\t%#018llx\n", u.v64);
+
+			if (ndev->bar4_split) {
+				u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
+				off += scnprintf(buf + off, buf_size - off,
+						 "SBAR4 -\t\t\t%#06x\n", u.v32);
+				u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
+				off += scnprintf(buf + off, buf_size - off,
+						 "SBAR5 -\t\t\t%#06x\n", u.v32);
+			} else {
+				u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
+				off += scnprintf(buf + off, buf_size - off,
+						 "SBAR45 -\t\t%#018llx\n",
+						 u.v64);
+			}
+		}
+
+		off += scnprintf(buf + off, buf_size - off,
+				 "\nXEON NTB Statistics:\n");
+
+		u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
+		off += scnprintf(buf + off, buf_size - off,
+				 "Upstream Memory Miss -\t%u\n", u.v16);
+
+		off += scnprintf(buf + off, buf_size - off,
+				 "\nXEON NTB Hardware Errors:\n");
+
+		if (!pci_read_config_word(pdev,
+					  XEON_DEVSTS_OFFSET, &u.v16))
+			off += scnprintf(buf + off, buf_size - off,
+					 "DEVSTS -\t\t%#06x\n", u.v16);
+
+		if (!pci_read_config_word(pdev,
+					  XEON_LINK_STATUS_OFFSET, &u.v16))
+			off += scnprintf(buf + off, buf_size - off,
+					 "LNKSTS -\t\t%#06x\n", u.v16);
+
+		if (!pci_read_config_dword(pdev,
+					   XEON_UNCERRSTS_OFFSET, &u.v32))
+			off += scnprintf(buf + off, buf_size - off,
+					 "UNCERRSTS -\t\t%#06x\n", u.v32);
+
+		if (!pci_read_config_dword(pdev,
+					   XEON_CORERRSTS_OFFSET, &u.v32))
+			off += scnprintf(buf + off, buf_size - off,
+					 "CORERRSTS -\t\t%#06x\n", u.v32);
+	}
+
+	ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
+	kfree(buf);
+	return ret;
+}
+
+static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
+				 size_t count, loff_t *offp)
+{
+	struct intel_ntb_dev *ndev = filp->private_data;
+
+	if (pdev_is_gen1(ndev->ntb.pdev))
+		return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
+	else if (pdev_is_gen3(ndev->ntb.pdev))
+		return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
+
+	return -ENXIO;
+}
+
+static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
+{
+	if (!debugfs_dir) {
+		ndev->debugfs_dir = NULL;
+		ndev->debugfs_info = NULL;
+	} else {
+		ndev->debugfs_dir =
+			debugfs_create_dir(pci_name(ndev->ntb.pdev),
+					   debugfs_dir);
+		if (!ndev->debugfs_dir)
+			ndev->debugfs_info = NULL;
+		else
+			ndev->debugfs_info =
+				debugfs_create_file("info", S_IRUSR,
+						    ndev->debugfs_dir, ndev,
+						    &intel_ntb_debugfs_info);
+	}
+}
+
+static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
+{
+	debugfs_remove_recursive(ndev->debugfs_dir);
+}
+
+int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx)
+{
+	if (pidx != NTB_DEF_PEER_IDX)
+		return -EINVAL;
+
+	return ntb_ndev(ntb)->mw_count;
+}
+
+int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
+			   resource_size_t *addr_align,
+			   resource_size_t *size_align,
+			   resource_size_t *size_max)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+	resource_size_t bar_size, mw_size;
+	int bar;
+
+	if (pidx != NTB_DEF_PEER_IDX)
+		return -EINVAL;
+
+	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
+		idx += 1;
+
+	bar = ndev_mw_to_bar(ndev, idx);
+	if (bar < 0)
+		return bar;
+
+	bar_size = pci_resource_len(ndev->ntb.pdev, bar);
+
+	if (idx == ndev->b2b_idx)
+		mw_size = bar_size - ndev->b2b_off;
+	else
+		mw_size = bar_size;
+
+	if (addr_align)
+		*addr_align = pci_resource_len(ndev->ntb.pdev, bar);
+
+	if (size_align)
+		*size_align = 1;
+
+	if (size_max)
+		*size_max = mw_size;
+
+	return 0;
+}
+
+static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
+				  dma_addr_t addr, resource_size_t size)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+	unsigned long base_reg, xlat_reg, limit_reg;
+	resource_size_t bar_size, mw_size;
+	void __iomem *mmio;
+	u64 base, limit, reg_val;
+	int bar;
+
+	if (pidx != NTB_DEF_PEER_IDX)
+		return -EINVAL;
+
+	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
+		idx += 1;
+
+	bar = ndev_mw_to_bar(ndev, idx);
+	if (bar < 0)
+		return bar;
+
+	bar_size = pci_resource_len(ndev->ntb.pdev, bar);
+
+	if (idx == ndev->b2b_idx)
+		mw_size = bar_size - ndev->b2b_off;
+	else
+		mw_size = bar_size;
+
+	/* hardware requires that addr is aligned to bar size */
+	if (addr & (bar_size - 1))
+		return -EINVAL;
+
+	/* make sure the range fits in the usable mw size */
+	if (size > mw_size)
+		return -EINVAL;
+
+	mmio = ndev->self_mmio;
+	base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
+	xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
+	limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
+
+	if (bar < 4 || !ndev->bar4_split) {
+		base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
+
+		/* Set the limit if supported, if size is not mw_size */
+		if (limit_reg && size != mw_size)
+			limit = base + size;
+		else
+			limit = 0;
+
+		/* set and verify setting the translation address */
+		iowrite64(addr, mmio + xlat_reg);
+		reg_val = ioread64(mmio + xlat_reg);
+		if (reg_val != addr) {
+			iowrite64(0, mmio + xlat_reg);
+			return -EIO;
+		}
+
+		/* set and verify setting the limit */
+		iowrite64(limit, mmio + limit_reg);
+		reg_val = ioread64(mmio + limit_reg);
+		if (reg_val != limit) {
+			iowrite64(base, mmio + limit_reg);
+			iowrite64(0, mmio + xlat_reg);
+			return -EIO;
+		}
+	} else {
+		/* split bar addr range must all be 32 bit */
+		if (addr & (~0ull << 32))
+			return -EINVAL;
+		if ((addr + size) & (~0ull << 32))
+			return -EINVAL;
+
+		base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
+
+		/* Set the limit if supported, if size is not mw_size */
+		if (limit_reg && size != mw_size)
+			limit = base + size;
+		else
+			limit = 0;
+
+		/* set and verify setting the translation address */
+		iowrite32(addr, mmio + xlat_reg);
+		reg_val = ioread32(mmio + xlat_reg);
+		if (reg_val != addr) {
+			iowrite32(0, mmio + xlat_reg);
+			return -EIO;
+		}
+
+		/* set and verify setting the limit */
+		iowrite32(limit, mmio + limit_reg);
+		reg_val = ioread32(mmio + limit_reg);
+		if (reg_val != limit) {
+			iowrite32(base, mmio + limit_reg);
+			iowrite32(0, mmio + xlat_reg);
+			return -EIO;
+		}
+	}
+
+	return 0;
+}
+
+u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed,
+			 enum ntb_width *width)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	if (ndev->reg->link_is_up(ndev)) {
+		if (speed)
+			*speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
+		if (width)
+			*width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
+		return 1;
+	} else {
+		/* TODO MAYBE: is it possible to observe the link speed and
+		 * width while link is training? */
+		if (speed)
+			*speed = NTB_SPEED_NONE;
+		if (width)
+			*width = NTB_WIDTH_NONE;
+		return 0;
+	}
+}
+
+static int intel_ntb_link_enable(struct ntb_dev *ntb,
+				 enum ntb_speed max_speed,
+				 enum ntb_width max_width)
+{
+	struct intel_ntb_dev *ndev;
+	u32 ntb_ctl;
+
+	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
+
+	if (ndev->ntb.topo == NTB_TOPO_SEC)
+		return -EINVAL;
+
+	dev_dbg(&ntb->pdev->dev,
+		"Enabling link with max_speed %d max_width %d\n",
+		max_speed, max_width);
+	if (max_speed != NTB_SPEED_AUTO)
+		dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
+	if (max_width != NTB_WIDTH_AUTO)
+		dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
+
+	ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
+	ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
+	ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
+	ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
+	if (ndev->bar4_split)
+		ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
+	iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
+
+	return 0;
+}
+
+int intel_ntb_link_disable(struct ntb_dev *ntb)
+{
+	struct intel_ntb_dev *ndev;
+	u32 ntb_cntl;
+
+	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
+
+	if (ndev->ntb.topo == NTB_TOPO_SEC)
+		return -EINVAL;
+
+	dev_dbg(&ntb->pdev->dev, "Disabling link\n");
+
+	/* Bring NTB link down */
+	ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
+	ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
+	ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
+	if (ndev->bar4_split)
+		ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
+	ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
+	iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
+
+	return 0;
+}
+
+int intel_ntb_peer_mw_count(struct ntb_dev *ntb)
+{
+	/* Numbers of inbound and outbound memory windows match */
+	return ntb_ndev(ntb)->mw_count;
+}
+
+int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
+			       phys_addr_t *base, resource_size_t *size)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+	int bar;
+
+	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
+		idx += 1;
+
+	bar = ndev_mw_to_bar(ndev, idx);
+	if (bar < 0)
+		return bar;
+
+	if (base)
+		*base = pci_resource_start(ndev->ntb.pdev, bar) +
+			(idx == ndev->b2b_idx ? ndev->b2b_off : 0);
+
+	if (size)
+		*size = pci_resource_len(ndev->ntb.pdev, bar) -
+			(idx == ndev->b2b_idx ? ndev->b2b_off : 0);
+
+	return 0;
+}
+
+static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
+{
+	return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
+}
+
+u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
+{
+	return ntb_ndev(ntb)->db_valid_mask;
+}
+
+int intel_ntb_db_vector_count(struct ntb_dev *ntb)
+{
+	struct intel_ntb_dev *ndev;
+
+	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
+
+	return ndev->db_vec_count;
+}
+
+u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	if (db_vector < 0 || db_vector > ndev->db_vec_count)
+		return 0;
+
+	return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
+}
+
+static u64 intel_ntb_db_read(struct ntb_dev *ntb)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_db_read(ndev,
+			    ndev->self_mmio +
+			    ndev->self_reg->db_bell);
+}
+
+static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_db_write(ndev, db_bits,
+			     ndev->self_mmio +
+			     ndev->self_reg->db_bell);
+}
+
+int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_db_set_mask(ndev, db_bits,
+				ndev->self_mmio +
+				ndev->self_reg->db_mask);
+}
+
+int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_db_clear_mask(ndev, db_bits,
+				  ndev->self_mmio +
+				  ndev->self_reg->db_mask);
+}
+
+int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
+			   resource_size_t *db_size)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
+			    ndev->peer_reg->db_bell);
+}
+
+static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_db_write(ndev, db_bits,
+			     ndev->peer_mmio +
+			     ndev->peer_reg->db_bell);
+}
+
+int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
+{
+	return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
+}
+
+int intel_ntb_spad_count(struct ntb_dev *ntb)
+{
+	struct intel_ntb_dev *ndev;
+
+	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
+
+	return ndev->spad_count;
+}
+
+u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_spad_read(ndev, idx,
+			      ndev->self_mmio +
+			      ndev->self_reg->spad);
+}
+
+int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_spad_write(ndev, idx, val,
+			       ndev->self_mmio +
+			       ndev->self_reg->spad);
+}
+
+int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
+			     phys_addr_t *spad_addr)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
+			      ndev->peer_reg->spad);
+}
+
+u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_spad_read(ndev, sidx,
+			      ndev->peer_mmio +
+			      ndev->peer_reg->spad);
+}
+
+int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx,
+			      u32 val)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_spad_write(ndev, sidx, val,
+			       ndev->peer_mmio +
+			       ndev->peer_reg->spad);
+}
+
+static u64 xeon_db_ioread(void __iomem *mmio)
+{
+	return (u64)ioread16(mmio);
+}
+
+static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
+{
+	iowrite16((u16)bits, mmio);
+}
+
+static int xeon_poll_link(struct intel_ntb_dev *ndev)
+{
+	u16 reg_val;
+	int rc;
+
+	ndev->reg->db_iowrite(ndev->db_link_mask,
+			      ndev->self_mmio +
+			      ndev->self_reg->db_bell);
+
+	rc = pci_read_config_word(ndev->ntb.pdev,
+				  XEON_LINK_STATUS_OFFSET, &reg_val);
+	if (rc)
+		return 0;
+
+	if (reg_val == ndev->lnk_sta)
+		return 0;
+
+	ndev->lnk_sta = reg_val;
+
+	return 1;
+}
+
+int xeon_link_is_up(struct intel_ntb_dev *ndev)
+{
+	if (ndev->ntb.topo == NTB_TOPO_SEC)
+		return 1;
+
+	return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
+}
+
+enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
+{
+	switch (ppd & XEON_PPD_TOPO_MASK) {
+	case XEON_PPD_TOPO_B2B_USD:
+		return NTB_TOPO_B2B_USD;
+
+	case XEON_PPD_TOPO_B2B_DSD:
+		return NTB_TOPO_B2B_DSD;
+
+	case XEON_PPD_TOPO_PRI_USD:
+	case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
+		return NTB_TOPO_PRI;
+
+	case XEON_PPD_TOPO_SEC_USD:
+	case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
+		return NTB_TOPO_SEC;
+	}
+
+	return NTB_TOPO_NONE;
+}
+
+static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
+{
+	if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
+		dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd);
+		return 1;
+	}
+	return 0;
+}
+
+static int xeon_init_isr(struct intel_ntb_dev *ndev)
+{
+	return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
+			     XEON_DB_MSIX_VECTOR_COUNT,
+			     XEON_DB_MSIX_VECTOR_SHIFT,
+			     XEON_DB_TOTAL_SHIFT);
+}
+
+static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
+{
+	ndev_deinit_isr(ndev);
+}
+
+static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
+			     const struct intel_b2b_addr *addr,
+			     const struct intel_b2b_addr *peer_addr)
+{
+	struct pci_dev *pdev;
+	void __iomem *mmio;
+	resource_size_t bar_size;
+	phys_addr_t bar_addr;
+	int b2b_bar;
+	u8 bar_sz;
+
+	pdev = ndev->ntb.pdev;
+	mmio = ndev->self_mmio;
+
+	if (ndev->b2b_idx == UINT_MAX) {
+		dev_dbg(&pdev->dev, "not using b2b mw\n");
+		b2b_bar = 0;
+		ndev->b2b_off = 0;
+	} else {
+		b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
+		if (b2b_bar < 0)
+			return -EIO;
+
+		dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
+
+		bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
+
+		dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
+
+		if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
+			dev_dbg(&pdev->dev, "b2b using first half of bar\n");
+			ndev->b2b_off = bar_size >> 1;
+		} else if (XEON_B2B_MIN_SIZE <= bar_size) {
+			dev_dbg(&pdev->dev, "b2b using whole bar\n");
+			ndev->b2b_off = 0;
+			--ndev->mw_count;
+		} else {
+			dev_dbg(&pdev->dev, "b2b bar size is too small\n");
+			return -EIO;
+		}
+	}
+
+	/* Reset the secondary bar sizes to match the primary bar sizes,
+	 * except disable or halve the size of the b2b secondary bar.
+	 *
+	 * Note: code for each specific bar size register, because the register
+	 * offsets are not in a consistent order (bar5sz comes after ppd, odd).
+	 */
+	pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
+	dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz);
+	if (b2b_bar == 2) {
+		if (ndev->b2b_off)
+			bar_sz -= 1;
+		else
+			bar_sz = 0;
+	}
+	pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
+	pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
+	dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz);
+
+	if (!ndev->bar4_split) {
+		pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
+		dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz);
+		if (b2b_bar == 4) {
+			if (ndev->b2b_off)
+				bar_sz -= 1;
+			else
+				bar_sz = 0;
+		}
+		pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
+		pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
+		dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz);
+	} else {
+		pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
+		dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz);
+		if (b2b_bar == 4) {
+			if (ndev->b2b_off)
+				bar_sz -= 1;
+			else
+				bar_sz = 0;
+		}
+		pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
+		pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
+		dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz);
+
+		pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
+		dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz);
+		if (b2b_bar == 5) {
+			if (ndev->b2b_off)
+				bar_sz -= 1;
+			else
+				bar_sz = 0;
+		}
+		pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
+		pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
+		dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz);
+	}
+
+	/* SBAR01 hit by first part of the b2b bar */
+	if (b2b_bar == 0)
+		bar_addr = addr->bar0_addr;
+	else if (b2b_bar == 2)
+		bar_addr = addr->bar2_addr64;
+	else if (b2b_bar == 4 && !ndev->bar4_split)
+		bar_addr = addr->bar4_addr64;
+	else if (b2b_bar == 4)
+		bar_addr = addr->bar4_addr32;
+	else if (b2b_bar == 5)
+		bar_addr = addr->bar5_addr32;
+	else
+		return -EIO;
+
+	dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr);
+	iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
+
+	/* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
+	 * The b2b bar is either disabled above, or configured half-size, and
+	 * it starts at the PBAR xlat + offset.
+	 */
+
+	bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
+	iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
+	bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
+	dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr);
+
+	if (!ndev->bar4_split) {
+		bar_addr = addr->bar4_addr64 +
+			(b2b_bar == 4 ? ndev->b2b_off : 0);
+		iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
+		bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
+		dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr);
+	} else {
+		bar_addr = addr->bar4_addr32 +
+			(b2b_bar == 4 ? ndev->b2b_off : 0);
+		iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
+		bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
+		dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr);
+
+		bar_addr = addr->bar5_addr32 +
+			(b2b_bar == 5 ? ndev->b2b_off : 0);
+		iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
+		bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
+		dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr);
+	}
+
+	/* setup incoming bar limits == base addrs (zero length windows) */
+
+	bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
+	iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
+	bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
+	dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr);
+
+	if (!ndev->bar4_split) {
+		bar_addr = addr->bar4_addr64 +
+			(b2b_bar == 4 ? ndev->b2b_off : 0);
+		iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
+		bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
+		dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr);
+	} else {
+		bar_addr = addr->bar4_addr32 +
+			(b2b_bar == 4 ? ndev->b2b_off : 0);
+		iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
+		bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
+		dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr);
+
+		bar_addr = addr->bar5_addr32 +
+			(b2b_bar == 5 ? ndev->b2b_off : 0);
+		iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
+		bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
+		dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr);
+	}
+
+	/* zero incoming translation addrs */
+	iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
+
+	if (!ndev->bar4_split) {
+		iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
+	} else {
+		iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
+		iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
+	}
+
+	/* zero outgoing translation limits (whole bar size windows) */
+	iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
+	if (!ndev->bar4_split) {
+		iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
+	} else {
+		iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
+		iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
+	}
+
+	/* set outgoing translation offsets */
+	bar_addr = peer_addr->bar2_addr64;
+	iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
+	bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
+	dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr);
+
+	if (!ndev->bar4_split) {
+		bar_addr = peer_addr->bar4_addr64;
+		iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
+		bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
+		dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr);
+	} else {
+		bar_addr = peer_addr->bar4_addr32;
+		iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
+		bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
+		dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr);
+
+		bar_addr = peer_addr->bar5_addr32;
+		iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
+		bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
+		dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr);
+	}
+
+	/* set the translation offset for b2b registers */
+	if (b2b_bar == 0)
+		bar_addr = peer_addr->bar0_addr;
+	else if (b2b_bar == 2)
+		bar_addr = peer_addr->bar2_addr64;
+	else if (b2b_bar == 4 && !ndev->bar4_split)
+		bar_addr = peer_addr->bar4_addr64;
+	else if (b2b_bar == 4)
+		bar_addr = peer_addr->bar4_addr32;
+	else if (b2b_bar == 5)
+		bar_addr = peer_addr->bar5_addr32;
+	else
+		return -EIO;
+
+	/* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
+	dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr);
+	iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
+	iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
+
+	if (b2b_bar) {
+		/* map peer ntb mmio config space registers */
+		ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
+					    XEON_B2B_MIN_SIZE);
+		if (!ndev->peer_mmio)
+			return -EIO;
+
+		ndev->peer_addr = pci_resource_start(pdev, b2b_bar);
+	}
+
+	return 0;
+}
+
+static int xeon_init_ntb(struct intel_ntb_dev *ndev)
+{
+	struct device *dev = &ndev->ntb.pdev->dev;
+	int rc;
+	u32 ntb_ctl;
+
+	if (ndev->bar4_split)
+		ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
+	else
+		ndev->mw_count = XEON_MW_COUNT;
+
+	ndev->spad_count = XEON_SPAD_COUNT;
+	ndev->db_count = XEON_DB_COUNT;
+	ndev->db_link_mask = XEON_DB_LINK_BIT;
+
+	switch (ndev->ntb.topo) {
+	case NTB_TOPO_PRI:
+		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
+			dev_err(dev, "NTB Primary config disabled\n");
+			return -EINVAL;
+		}
+
+		/* enable link to allow secondary side device to appear */
+		ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
+		ntb_ctl &= ~NTB_CTL_DISABLE;
+		iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
+
+		/* use half the spads for the peer */
+		ndev->spad_count >>= 1;
+		ndev->self_reg = &xeon_pri_reg;
+		ndev->peer_reg = &xeon_sec_reg;
+		ndev->xlat_reg = &xeon_sec_xlat;
+		break;
+
+	case NTB_TOPO_SEC:
+		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
+			dev_err(dev, "NTB Secondary config disabled\n");
+			return -EINVAL;
+		}
+		/* use half the spads for the peer */
+		ndev->spad_count >>= 1;
+		ndev->self_reg = &xeon_sec_reg;
+		ndev->peer_reg = &xeon_pri_reg;
+		ndev->xlat_reg = &xeon_pri_xlat;
+		break;
+
+	case NTB_TOPO_B2B_USD:
+	case NTB_TOPO_B2B_DSD:
+		ndev->self_reg = &xeon_pri_reg;
+		ndev->peer_reg = &xeon_b2b_reg;
+		ndev->xlat_reg = &xeon_sec_xlat;
+
+		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
+			ndev->peer_reg = &xeon_pri_reg;
+
+			if (b2b_mw_idx < 0)
+				ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
+			else
+				ndev->b2b_idx = b2b_mw_idx;
+
+			if (ndev->b2b_idx >= ndev->mw_count) {
+				dev_dbg(dev,
+					"b2b_mw_idx %d invalid for mw_count %u\n",
+					b2b_mw_idx, ndev->mw_count);
+				return -EINVAL;
+			}
+
+			dev_dbg(dev, "setting up b2b mw idx %d means %d\n",
+				b2b_mw_idx, ndev->b2b_idx);
+
+		} else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
+			dev_warn(dev, "Reduce doorbell count by 1\n");
+			ndev->db_count -= 1;
+		}
+
+		if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
+			rc = xeon_setup_b2b_mw(ndev,
+					       &xeon_b2b_dsd_addr,
+					       &xeon_b2b_usd_addr);
+		} else {
+			rc = xeon_setup_b2b_mw(ndev,
+					       &xeon_b2b_usd_addr,
+					       &xeon_b2b_dsd_addr);
+		}
+		if (rc)
+			return rc;
+
+		/* Enable Bus Master and Memory Space on the secondary side */
+		iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
+			  ndev->self_mmio + XEON_SPCICMD_OFFSET);
+
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
+
+	ndev->reg->db_iowrite(ndev->db_valid_mask,
+			      ndev->self_mmio +
+			      ndev->self_reg->db_mask);
+
+	return 0;
+}
+
+static int xeon_init_dev(struct intel_ntb_dev *ndev)
+{
+	struct pci_dev *pdev;
+	u8 ppd;
+	int rc, mem;
+
+	pdev = ndev->ntb.pdev;
+
+	switch (pdev->device) {
+	/* There is a Xeon hardware errata related to writes to SDOORBELL or
+	 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
+	 * which may hang the system.  To workaround this use the second memory
+	 * window to access the interrupt and scratch pad registers on the
+	 * remote system.
+	 */
+	case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
+		ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
+		break;
+	}
+
+	switch (pdev->device) {
+	/* There is a hardware errata related to accessing any register in
+	 * SB01BASE in the presence of bidirectional traffic crossing the NTB.
+	 */
+	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
+		ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
+		break;
+	}
+
+	switch (pdev->device) {
+	/* HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
+	 * mirrored to the remote system.  Shrink the number of bits by one,
+	 * since bit 14 is the last bit.
+	 */
+	case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
+		ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
+		break;
+	}
+
+	ndev->reg = &xeon_reg;
+
+	rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
+	if (rc)
+		return -EIO;
+
+	ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
+	dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
+		ntb_topo_string(ndev->ntb.topo));
+	if (ndev->ntb.topo == NTB_TOPO_NONE)
+		return -EINVAL;
+
+	if (ndev->ntb.topo != NTB_TOPO_SEC) {
+		ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
+		dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n",
+			ppd, ndev->bar4_split);
+	} else {
+		/* This is a way for transparent BAR to figure out if we are
+		 * doing split BAR or not. There is no way for the hw on the
+		 * transparent side to know and set the PPD.
+		 */
+		mem = pci_select_bars(pdev, IORESOURCE_MEM);
+		ndev->bar4_split = hweight32(mem) ==
+			HSX_SPLIT_BAR_MW_COUNT + 1;
+		dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n",
+			mem, ndev->bar4_split);
+	}
+
+	rc = xeon_init_ntb(ndev);
+	if (rc)
+		return rc;
+
+	return xeon_init_isr(ndev);
+}
+
+static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
+{
+	xeon_deinit_isr(ndev);
+}
+
+static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
+{
+	int rc;
+
+	pci_set_drvdata(pdev, ndev);
+
+	rc = pci_enable_device(pdev);
+	if (rc)
+		goto err_pci_enable;
+
+	rc = pci_request_regions(pdev, NTB_NAME);
+	if (rc)
+		goto err_pci_regions;
+
+	pci_set_master(pdev);
+
+	rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
+	if (rc) {
+		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+		if (rc)
+			goto err_dma_mask;
+		dev_warn(&pdev->dev, "Cannot DMA highmem\n");
+	}
+
+	rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+	if (rc) {
+		rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+		if (rc)
+			goto err_dma_mask;
+		dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
+	}
+	rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
+					  dma_get_mask(&pdev->dev));
+	if (rc)
+		goto err_dma_mask;
+
+	ndev->self_mmio = pci_iomap(pdev, 0, 0);
+	if (!ndev->self_mmio) {
+		rc = -EIO;
+		goto err_mmio;
+	}
+	ndev->peer_mmio = ndev->self_mmio;
+	ndev->peer_addr = pci_resource_start(pdev, 0);
+
+	return 0;
+
+err_mmio:
+err_dma_mask:
+	pci_clear_master(pdev);
+	pci_release_regions(pdev);
+err_pci_regions:
+	pci_disable_device(pdev);
+err_pci_enable:
+	pci_set_drvdata(pdev, NULL);
+	return rc;
+}
+
+static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
+{
+	struct pci_dev *pdev = ndev->ntb.pdev;
+
+	if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
+		pci_iounmap(pdev, ndev->peer_mmio);
+	pci_iounmap(pdev, ndev->self_mmio);
+
+	pci_clear_master(pdev);
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+}
+
+static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
+				    struct pci_dev *pdev)
+{
+	ndev->ntb.pdev = pdev;
+	ndev->ntb.topo = NTB_TOPO_NONE;
+	ndev->ntb.ops = &intel_ntb_ops;
+
+	ndev->b2b_off = 0;
+	ndev->b2b_idx = UINT_MAX;
+
+	ndev->bar4_split = 0;
+
+	ndev->mw_count = 0;
+	ndev->spad_count = 0;
+	ndev->db_count = 0;
+	ndev->db_vec_count = 0;
+	ndev->db_vec_shift = 0;
+
+	ndev->ntb_ctl = 0;
+	ndev->lnk_sta = 0;
+
+	ndev->db_valid_mask = 0;
+	ndev->db_link_mask = 0;
+	ndev->db_mask = 0;
+
+	spin_lock_init(&ndev->db_mask_lock);
+}
+
+static int intel_ntb_pci_probe(struct pci_dev *pdev,
+			       const struct pci_device_id *id)
+{
+	struct intel_ntb_dev *ndev;
+	int rc, node;
+
+	node = dev_to_node(&pdev->dev);
+
+	if (pdev_is_gen1(pdev)) {
+		ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
+		if (!ndev) {
+			rc = -ENOMEM;
+			goto err_ndev;
+		}
+
+		ndev_init_struct(ndev, pdev);
+
+		rc = intel_ntb_init_pci(ndev, pdev);
+		if (rc)
+			goto err_init_pci;
+
+		rc = xeon_init_dev(ndev);
+		if (rc)
+			goto err_init_dev;
+
+	} else if (pdev_is_gen3(pdev)) {
+		ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
+		if (!ndev) {
+			rc = -ENOMEM;
+			goto err_ndev;
+		}
+
+		ndev_init_struct(ndev, pdev);
+		ndev->ntb.ops = &intel_ntb3_ops;
+
+		rc = intel_ntb_init_pci(ndev, pdev);
+		if (rc)
+			goto err_init_pci;
+
+		rc = gen3_init_dev(ndev);
+		if (rc)
+			goto err_init_dev;
+
+	} else {
+		rc = -EINVAL;
+		goto err_ndev;
+	}
+
+	ndev_reset_unsafe_flags(ndev);
+
+	ndev->reg->poll_link(ndev);
+
+	ndev_init_debugfs(ndev);
+
+	rc = ntb_register_device(&ndev->ntb);
+	if (rc)
+		goto err_register;
+
+	dev_info(&pdev->dev, "NTB device registered.\n");
+
+	return 0;
+
+err_register:
+	ndev_deinit_debugfs(ndev);
+	if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev))
+		xeon_deinit_dev(ndev);
+err_init_dev:
+	intel_ntb_deinit_pci(ndev);
+err_init_pci:
+	kfree(ndev);
+err_ndev:
+	return rc;
+}
+
+static void intel_ntb_pci_remove(struct pci_dev *pdev)
+{
+	struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
+
+	ntb_unregister_device(&ndev->ntb);
+	ndev_deinit_debugfs(ndev);
+	if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev))
+		xeon_deinit_dev(ndev);
+	intel_ntb_deinit_pci(ndev);
+	kfree(ndev);
+}
+
+static const struct intel_ntb_reg xeon_reg = {
+	.poll_link		= xeon_poll_link,
+	.link_is_up		= xeon_link_is_up,
+	.db_ioread		= xeon_db_ioread,
+	.db_iowrite		= xeon_db_iowrite,
+	.db_size		= sizeof(u32),
+	.ntb_ctl		= XEON_NTBCNTL_OFFSET,
+	.mw_bar			= {2, 4, 5},
+};
+
+static const struct intel_ntb_alt_reg xeon_pri_reg = {
+	.db_bell		= XEON_PDOORBELL_OFFSET,
+	.db_mask		= XEON_PDBMSK_OFFSET,
+	.spad			= XEON_SPAD_OFFSET,
+};
+
+static const struct intel_ntb_alt_reg xeon_sec_reg = {
+	.db_bell		= XEON_SDOORBELL_OFFSET,
+	.db_mask		= XEON_SDBMSK_OFFSET,
+	/* second half of the scratchpads */
+	.spad			= XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
+};
+
+static const struct intel_ntb_alt_reg xeon_b2b_reg = {
+	.db_bell		= XEON_B2B_DOORBELL_OFFSET,
+	.spad			= XEON_B2B_SPAD_OFFSET,
+};
+
+static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
+	/* Note: no primary .bar0_base visible to the secondary side.
+	 *
+	 * The secondary side cannot get the base address stored in primary
+	 * bars.  The base address is necessary to set the limit register to
+	 * any value other than zero, or unlimited.
+	 *
+	 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
+	 * window by setting the limit equal to base, nor can it limit the size
+	 * of the memory window by setting the limit to base + size.
+	 */
+	.bar2_limit		= XEON_PBAR23LMT_OFFSET,
+	.bar2_xlat		= XEON_PBAR23XLAT_OFFSET,
+};
+
+static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
+	.bar0_base		= XEON_SBAR0BASE_OFFSET,
+	.bar2_limit		= XEON_SBAR23LMT_OFFSET,
+	.bar2_xlat		= XEON_SBAR23XLAT_OFFSET,
+};
+
+struct intel_b2b_addr xeon_b2b_usd_addr = {
+	.bar2_addr64		= XEON_B2B_BAR2_ADDR64,
+	.bar4_addr64		= XEON_B2B_BAR4_ADDR64,
+	.bar4_addr32		= XEON_B2B_BAR4_ADDR32,
+	.bar5_addr32		= XEON_B2B_BAR5_ADDR32,
+};
+
+struct intel_b2b_addr xeon_b2b_dsd_addr = {
+	.bar2_addr64		= XEON_B2B_BAR2_ADDR64,
+	.bar4_addr64		= XEON_B2B_BAR4_ADDR64,
+	.bar4_addr32		= XEON_B2B_BAR4_ADDR32,
+	.bar5_addr32		= XEON_B2B_BAR5_ADDR32,
+};
+
+/* operations for primary side of local ntb */
+static const struct ntb_dev_ops intel_ntb_ops = {
+	.mw_count		= intel_ntb_mw_count,
+	.mw_get_align		= intel_ntb_mw_get_align,
+	.mw_set_trans		= intel_ntb_mw_set_trans,
+	.peer_mw_count		= intel_ntb_peer_mw_count,
+	.peer_mw_get_addr	= intel_ntb_peer_mw_get_addr,
+	.link_is_up		= intel_ntb_link_is_up,
+	.link_enable		= intel_ntb_link_enable,
+	.link_disable		= intel_ntb_link_disable,
+	.db_is_unsafe		= intel_ntb_db_is_unsafe,
+	.db_valid_mask		= intel_ntb_db_valid_mask,
+	.db_vector_count	= intel_ntb_db_vector_count,
+	.db_vector_mask		= intel_ntb_db_vector_mask,
+	.db_read		= intel_ntb_db_read,
+	.db_clear		= intel_ntb_db_clear,
+	.db_set_mask		= intel_ntb_db_set_mask,
+	.db_clear_mask		= intel_ntb_db_clear_mask,
+	.peer_db_addr		= intel_ntb_peer_db_addr,
+	.peer_db_set		= intel_ntb_peer_db_set,
+	.spad_is_unsafe		= intel_ntb_spad_is_unsafe,
+	.spad_count		= intel_ntb_spad_count,
+	.spad_read		= intel_ntb_spad_read,
+	.spad_write		= intel_ntb_spad_write,
+	.peer_spad_addr		= intel_ntb_peer_spad_addr,
+	.peer_spad_read		= intel_ntb_peer_spad_read,
+	.peer_spad_write	= intel_ntb_peer_spad_write,
+};
+
+static const struct file_operations intel_ntb_debugfs_info = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.read = ndev_debugfs_read,
+};
+
+static const struct pci_device_id intel_ntb_pci_tbl[] = {
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
+	{0}
+};
+MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
+
+static struct pci_driver intel_ntb_pci_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = intel_ntb_pci_tbl,
+	.probe = intel_ntb_pci_probe,
+	.remove = intel_ntb_pci_remove,
+};
+
+static int __init intel_ntb_pci_driver_init(void)
+{
+	pr_info("%s %s\n", NTB_DESC, NTB_VER);
+
+	if (debugfs_initialized())
+		debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
+
+	return pci_register_driver(&intel_ntb_pci_driver);
+}
+module_init(intel_ntb_pci_driver_init);
+
+static void __exit intel_ntb_pci_driver_exit(void)
+{
+	pci_unregister_driver(&intel_ntb_pci_driver);
+
+	debugfs_remove_recursive(debugfs_dir);
+}
+module_exit(intel_ntb_pci_driver_exit);
diff --git a/drivers/ntb/hw/intel/ntb_hw_gen1.h b/drivers/ntb/hw/intel/ntb_hw_gen1.h
new file mode 100644
index 000000000000..ad8ec1444436
--- /dev/null
+++ b/drivers/ntb/hw/intel/ntb_hw_gen1.h
@@ -0,0 +1,182 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ *   redistributing this file, you may do so under either license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright(c) 2012-2017 Intel Corporation. All rights reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of version 2 of the GNU General Public License as
+ *   published by the Free Software Foundation.
+ *
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2012-2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copy
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _NTB_INTEL_GEN1_H_
+#define _NTB_INTEL_GEN1_H_
+
+#include "ntb_hw_intel.h"
+
+/* Intel Gen1 Xeon hardware */
+#define XEON_PBAR23LMT_OFFSET		0x0000
+#define XEON_PBAR45LMT_OFFSET		0x0008
+#define XEON_PBAR4LMT_OFFSET		0x0008
+#define XEON_PBAR5LMT_OFFSET		0x000c
+#define XEON_PBAR23XLAT_OFFSET		0x0010
+#define XEON_PBAR45XLAT_OFFSET		0x0018
+#define XEON_PBAR4XLAT_OFFSET		0x0018
+#define XEON_PBAR5XLAT_OFFSET		0x001c
+#define XEON_SBAR23LMT_OFFSET		0x0020
+#define XEON_SBAR45LMT_OFFSET		0x0028
+#define XEON_SBAR4LMT_OFFSET		0x0028
+#define XEON_SBAR5LMT_OFFSET		0x002c
+#define XEON_SBAR23XLAT_OFFSET		0x0030
+#define XEON_SBAR45XLAT_OFFSET		0x0038
+#define XEON_SBAR4XLAT_OFFSET		0x0038
+#define XEON_SBAR5XLAT_OFFSET		0x003c
+#define XEON_SBAR0BASE_OFFSET		0x0040
+#define XEON_SBAR23BASE_OFFSET		0x0048
+#define XEON_SBAR45BASE_OFFSET		0x0050
+#define XEON_SBAR4BASE_OFFSET		0x0050
+#define XEON_SBAR5BASE_OFFSET		0x0054
+#define XEON_SBDF_OFFSET		0x005c
+#define XEON_NTBCNTL_OFFSET		0x0058
+#define XEON_PDOORBELL_OFFSET		0x0060
+#define XEON_PDBMSK_OFFSET		0x0062
+#define XEON_SDOORBELL_OFFSET		0x0064
+#define XEON_SDBMSK_OFFSET		0x0066
+#define XEON_USMEMMISS_OFFSET		0x0070
+#define XEON_SPAD_OFFSET		0x0080
+#define XEON_PBAR23SZ_OFFSET		0x00d0
+#define XEON_PBAR45SZ_OFFSET		0x00d1
+#define XEON_PBAR4SZ_OFFSET		0x00d1
+#define XEON_SBAR23SZ_OFFSET		0x00d2
+#define XEON_SBAR45SZ_OFFSET		0x00d3
+#define XEON_SBAR4SZ_OFFSET		0x00d3
+#define XEON_PPD_OFFSET			0x00d4
+#define XEON_PBAR5SZ_OFFSET		0x00d5
+#define XEON_SBAR5SZ_OFFSET		0x00d6
+#define XEON_WCCNTRL_OFFSET		0x00e0
+#define XEON_UNCERRSTS_OFFSET		0x014c
+#define XEON_CORERRSTS_OFFSET		0x0158
+#define XEON_LINK_STATUS_OFFSET		0x01a2
+#define XEON_SPCICMD_OFFSET		0x0504
+#define XEON_DEVCTRL_OFFSET		0x0598
+#define XEON_DEVSTS_OFFSET		0x059a
+#define XEON_SLINK_STATUS_OFFSET	0x05a2
+#define XEON_B2B_SPAD_OFFSET		0x0100
+#define XEON_B2B_DOORBELL_OFFSET	0x0140
+#define XEON_B2B_XLAT_OFFSETL		0x0144
+#define XEON_B2B_XLAT_OFFSETU		0x0148
+#define XEON_PPD_CONN_MASK		0x03
+#define XEON_PPD_CONN_TRANSPARENT	0x00
+#define XEON_PPD_CONN_B2B		0x01
+#define XEON_PPD_CONN_RP		0x02
+#define XEON_PPD_DEV_MASK		0x10
+#define XEON_PPD_DEV_USD		0x00
+#define XEON_PPD_DEV_DSD		0x10
+#define XEON_PPD_SPLIT_BAR_MASK		0x40
+
+#define XEON_PPD_TOPO_MASK	(XEON_PPD_CONN_MASK | XEON_PPD_DEV_MASK)
+#define XEON_PPD_TOPO_PRI_USD	(XEON_PPD_CONN_RP | XEON_PPD_DEV_USD)
+#define XEON_PPD_TOPO_PRI_DSD	(XEON_PPD_CONN_RP | XEON_PPD_DEV_DSD)
+#define XEON_PPD_TOPO_SEC_USD	(XEON_PPD_CONN_TRANSPARENT | XEON_PPD_DEV_USD)
+#define XEON_PPD_TOPO_SEC_DSD	(XEON_PPD_CONN_TRANSPARENT | XEON_PPD_DEV_DSD)
+#define XEON_PPD_TOPO_B2B_USD	(XEON_PPD_CONN_B2B | XEON_PPD_DEV_USD)
+#define XEON_PPD_TOPO_B2B_DSD	(XEON_PPD_CONN_B2B | XEON_PPD_DEV_DSD)
+
+#define XEON_MW_COUNT			2
+#define HSX_SPLIT_BAR_MW_COUNT		3
+#define XEON_DB_COUNT			15
+#define XEON_DB_LINK			15
+#define XEON_DB_LINK_BIT			BIT_ULL(XEON_DB_LINK)
+#define XEON_DB_MSIX_VECTOR_COUNT	4
+#define XEON_DB_MSIX_VECTOR_SHIFT	5
+#define XEON_DB_TOTAL_SHIFT		16
+#define XEON_SPAD_COUNT			16
+
+/* Use the following addresses for translation between b2b ntb devices in case
+ * the hardware default values are not reliable. */
+#define XEON_B2B_BAR0_ADDR	0x1000000000000000ull
+#define XEON_B2B_BAR2_ADDR64	0x2000000000000000ull
+#define XEON_B2B_BAR4_ADDR64	0x4000000000000000ull
+#define XEON_B2B_BAR4_ADDR32	0x20000000u
+#define XEON_B2B_BAR5_ADDR32	0x40000000u
+
+/* The peer ntb secondary config space is 32KB fixed size */
+#define XEON_B2B_MIN_SIZE		0x8000
+
+/* flags to indicate hardware errata */
+#define NTB_HWERR_SDOORBELL_LOCKUP	BIT_ULL(0)
+#define NTB_HWERR_SB01BASE_LOCKUP	BIT_ULL(1)
+#define NTB_HWERR_B2BDOORBELL_BIT14	BIT_ULL(2)
+#define NTB_HWERR_MSIX_VECTOR32_BAD	BIT_ULL(3)
+
+extern struct intel_b2b_addr xeon_b2b_usd_addr;
+extern struct intel_b2b_addr xeon_b2b_dsd_addr;
+
+int ndev_init_isr(struct intel_ntb_dev *ndev, int msix_min, int msix_max,
+		int msix_shift, int total_shift);
+enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd);
+u64 ndev_db_read(struct intel_ntb_dev *ndev, void __iomem *mmio);
+int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
+				void __iomem *mmio);
+int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx);
+int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx);
+int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
+		resource_size_t *addr_align, resource_size_t *size_align,
+		resource_size_t *size_max);
+int intel_ntb_peer_mw_count(struct ntb_dev *ntb);
+int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
+		phys_addr_t *base, resource_size_t *size);
+u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed,
+		enum ntb_width *width);
+int intel_ntb_link_disable(struct ntb_dev *ntb);
+u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb);
+int intel_ntb_db_vector_count(struct ntb_dev *ntb);
+u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector);
+int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits);
+int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits);
+int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
+		resource_size_t *db_size);
+int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb);
+int intel_ntb_spad_count(struct ntb_dev *ntb);
+u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx);
+int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val);
+u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx);
+int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx,
+		u32 val);
+int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
+				    phys_addr_t *spad_addr);
+int xeon_link_is_up(struct intel_ntb_dev *ndev);
+
+#endif
diff --git a/drivers/ntb/hw/intel/ntb_hw_gen3.c b/drivers/ntb/hw/intel/ntb_hw_gen3.c
new file mode 100644
index 000000000000..b3fa24778f94
--- /dev/null
+++ b/drivers/ntb/hw/intel/ntb_hw_gen3.c
@@ -0,0 +1,597 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ *   redistributing this file, you may do so under either license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of version 2 of the GNU General Public License as
+ *   published by the Free Software Foundation.
+ *
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copy
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Intel PCIe GEN3 NTB Linux driver
+ *
+ */
+
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+#include <linux/ntb.h>
+
+#include "ntb_hw_intel.h"
+#include "ntb_hw_gen1.h"
+#include "ntb_hw_gen3.h"
+
+static int gen3_poll_link(struct intel_ntb_dev *ndev);
+
+static const struct intel_ntb_reg gen3_reg = {
+	.poll_link		= gen3_poll_link,
+	.link_is_up		= xeon_link_is_up,
+	.db_ioread		= gen3_db_ioread,
+	.db_iowrite		= gen3_db_iowrite,
+	.db_size		= sizeof(u32),
+	.ntb_ctl		= GEN3_NTBCNTL_OFFSET,
+	.mw_bar			= {2, 4},
+};
+
+static const struct intel_ntb_alt_reg gen3_pri_reg = {
+	.db_bell		= GEN3_EM_DOORBELL_OFFSET,
+	.db_clear		= GEN3_IM_INT_STATUS_OFFSET,
+	.db_mask		= GEN3_IM_INT_DISABLE_OFFSET,
+	.spad			= GEN3_IM_SPAD_OFFSET,
+};
+
+static const struct intel_ntb_alt_reg gen3_b2b_reg = {
+	.db_bell		= GEN3_IM_DOORBELL_OFFSET,
+	.db_clear		= GEN3_EM_INT_STATUS_OFFSET,
+	.db_mask		= GEN3_EM_INT_DISABLE_OFFSET,
+	.spad			= GEN3_B2B_SPAD_OFFSET,
+};
+
+static const struct intel_ntb_xlat_reg gen3_sec_xlat = {
+/*	.bar0_base		= GEN3_EMBAR0_OFFSET, */
+	.bar2_limit		= GEN3_IMBAR1XLMT_OFFSET,
+	.bar2_xlat		= GEN3_IMBAR1XBASE_OFFSET,
+};
+
+static int gen3_poll_link(struct intel_ntb_dev *ndev)
+{
+	u16 reg_val;
+	int rc;
+
+	ndev->reg->db_iowrite(ndev->db_link_mask,
+			      ndev->self_mmio +
+			      ndev->self_reg->db_clear);
+
+	rc = pci_read_config_word(ndev->ntb.pdev,
+				  GEN3_LINK_STATUS_OFFSET, &reg_val);
+	if (rc)
+		return 0;
+
+	if (reg_val == ndev->lnk_sta)
+		return 0;
+
+	ndev->lnk_sta = reg_val;
+
+	return 1;
+}
+
+static int gen3_init_isr(struct intel_ntb_dev *ndev)
+{
+	int i;
+
+	/*
+	 * The MSIX vectors and the interrupt status bits are not lined up
+	 * on Skylake. By default the link status bit is bit 32, however it
+	 * is by default MSIX vector0. We need to fixup to line them up.
+	 * The vectors at reset is 1-32,0. We need to reprogram to 0-32.
+	 */
+
+	for (i = 0; i < GEN3_DB_MSIX_VECTOR_COUNT; i++)
+		iowrite8(i, ndev->self_mmio + GEN3_INTVEC_OFFSET + i);
+
+	/* move link status down one as workaround */
+	if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) {
+		iowrite8(GEN3_DB_MSIX_VECTOR_COUNT - 2,
+			 ndev->self_mmio + GEN3_INTVEC_OFFSET +
+			 (GEN3_DB_MSIX_VECTOR_COUNT - 1));
+	}
+
+	return ndev_init_isr(ndev, GEN3_DB_MSIX_VECTOR_COUNT,
+			     GEN3_DB_MSIX_VECTOR_COUNT,
+			     GEN3_DB_MSIX_VECTOR_SHIFT,
+			     GEN3_DB_TOTAL_SHIFT);
+}
+
+static int gen3_setup_b2b_mw(struct intel_ntb_dev *ndev,
+			    const struct intel_b2b_addr *addr,
+			    const struct intel_b2b_addr *peer_addr)
+{
+	struct pci_dev *pdev;
+	void __iomem *mmio;
+	phys_addr_t bar_addr;
+
+	pdev = ndev->ntb.pdev;
+	mmio = ndev->self_mmio;
+
+	/* setup incoming bar limits == base addrs (zero length windows) */
+	bar_addr = addr->bar2_addr64;
+	iowrite64(bar_addr, mmio + GEN3_IMBAR1XLMT_OFFSET);
+	bar_addr = ioread64(mmio + GEN3_IMBAR1XLMT_OFFSET);
+	dev_dbg(&pdev->dev, "IMBAR1XLMT %#018llx\n", bar_addr);
+
+	bar_addr = addr->bar4_addr64;
+	iowrite64(bar_addr, mmio + GEN3_IMBAR2XLMT_OFFSET);
+	bar_addr = ioread64(mmio + GEN3_IMBAR2XLMT_OFFSET);
+	dev_dbg(&pdev->dev, "IMBAR2XLMT %#018llx\n", bar_addr);
+
+	/* zero incoming translation addrs */
+	iowrite64(0, mmio + GEN3_IMBAR1XBASE_OFFSET);
+	iowrite64(0, mmio + GEN3_IMBAR2XBASE_OFFSET);
+
+	ndev->peer_mmio = ndev->self_mmio;
+
+	return 0;
+}
+
+static int gen3_init_ntb(struct intel_ntb_dev *ndev)
+{
+	int rc;
+
+
+	ndev->mw_count = XEON_MW_COUNT;
+	ndev->spad_count = GEN3_SPAD_COUNT;
+	ndev->db_count = GEN3_DB_COUNT;
+	ndev->db_link_mask = GEN3_DB_LINK_BIT;
+
+	/* DB fixup for using 31 right now */
+	if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD)
+		ndev->db_link_mask |= BIT_ULL(31);
+
+	switch (ndev->ntb.topo) {
+	case NTB_TOPO_B2B_USD:
+	case NTB_TOPO_B2B_DSD:
+		ndev->self_reg = &gen3_pri_reg;
+		ndev->peer_reg = &gen3_b2b_reg;
+		ndev->xlat_reg = &gen3_sec_xlat;
+
+		if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
+			rc = gen3_setup_b2b_mw(ndev,
+					      &xeon_b2b_dsd_addr,
+					      &xeon_b2b_usd_addr);
+		} else {
+			rc = gen3_setup_b2b_mw(ndev,
+					      &xeon_b2b_usd_addr,
+					      &xeon_b2b_dsd_addr);
+		}
+
+		if (rc)
+			return rc;
+
+		/* Enable Bus Master and Memory Space on the secondary side */
+		iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
+			  ndev->self_mmio + GEN3_SPCICMD_OFFSET);
+
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
+
+	ndev->reg->db_iowrite(ndev->db_valid_mask,
+			      ndev->self_mmio +
+			      ndev->self_reg->db_mask);
+
+	return 0;
+}
+
+int gen3_init_dev(struct intel_ntb_dev *ndev)
+{
+	struct pci_dev *pdev;
+	u8 ppd;
+	int rc;
+
+	pdev = ndev->ntb.pdev;
+
+	ndev->reg = &gen3_reg;
+
+	rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
+	if (rc)
+		return -EIO;
+
+	ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
+	dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
+		ntb_topo_string(ndev->ntb.topo));
+	if (ndev->ntb.topo == NTB_TOPO_NONE)
+		return -EINVAL;
+
+	ndev->hwerr_flags |= NTB_HWERR_MSIX_VECTOR32_BAD;
+
+	rc = gen3_init_ntb(ndev);
+	if (rc)
+		return rc;
+
+	return gen3_init_isr(ndev);
+}
+
+ssize_t ndev_ntb3_debugfs_read(struct file *filp, char __user *ubuf,
+				      size_t count, loff_t *offp)
+{
+	struct intel_ntb_dev *ndev;
+	void __iomem *mmio;
+	char *buf;
+	size_t buf_size;
+	ssize_t ret, off;
+	union { u64 v64; u32 v32; u16 v16; } u;
+
+	ndev = filp->private_data;
+	mmio = ndev->self_mmio;
+
+	buf_size = min(count, 0x800ul);
+
+	buf = kmalloc(buf_size, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	off = 0;
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "NTB Device Information:\n");
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Connection Topology -\t%s\n",
+			 ntb_topo_string(ndev->ntb.topo));
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
+	off += scnprintf(buf + off, buf_size - off,
+			 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
+
+	if (!ndev->reg->link_is_up(ndev))
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Status -\t\tDown\n");
+	else {
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Status -\t\tUp\n");
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Speed -\t\tPCI-E Gen %u\n",
+				 NTB_LNK_STA_SPEED(ndev->lnk_sta));
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Width -\t\tx%u\n",
+				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
+	}
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Memory Window Count -\t%u\n", ndev->mw_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Scratchpad Count -\t%u\n", ndev->spad_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Count -\t%u\n", ndev->db_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
+
+	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Mask -\t\t%#llx\n", u.v64);
+
+	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Bell -\t\t%#llx\n", u.v64);
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "\nNTB Incoming XLAT:\n");
+
+	u.v64 = ioread64(mmio + GEN3_IMBAR1XBASE_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "IMBAR1XBASE -\t\t%#018llx\n", u.v64);
+
+	u.v64 = ioread64(mmio + GEN3_IMBAR2XBASE_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "IMBAR2XBASE -\t\t%#018llx\n", u.v64);
+
+	u.v64 = ioread64(mmio + GEN3_IMBAR1XLMT_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "IMBAR1XLMT -\t\t\t%#018llx\n", u.v64);
+
+	u.v64 = ioread64(mmio + GEN3_IMBAR2XLMT_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "IMBAR2XLMT -\t\t\t%#018llx\n", u.v64);
+
+	if (ntb_topo_is_b2b(ndev->ntb.topo)) {
+		off += scnprintf(buf + off, buf_size - off,
+				 "\nNTB Outgoing B2B XLAT:\n");
+
+		u.v64 = ioread64(mmio + GEN3_EMBAR1XBASE_OFFSET);
+		off += scnprintf(buf + off, buf_size - off,
+				 "EMBAR1XBASE -\t\t%#018llx\n", u.v64);
+
+		u.v64 = ioread64(mmio + GEN3_EMBAR2XBASE_OFFSET);
+		off += scnprintf(buf + off, buf_size - off,
+				 "EMBAR2XBASE -\t\t%#018llx\n", u.v64);
+
+		u.v64 = ioread64(mmio + GEN3_EMBAR1XLMT_OFFSET);
+		off += scnprintf(buf + off, buf_size - off,
+				 "EMBAR1XLMT -\t\t%#018llx\n", u.v64);
+
+		u.v64 = ioread64(mmio + GEN3_EMBAR2XLMT_OFFSET);
+		off += scnprintf(buf + off, buf_size - off,
+				 "EMBAR2XLMT -\t\t%#018llx\n", u.v64);
+
+		off += scnprintf(buf + off, buf_size - off,
+				 "\nNTB Secondary BAR:\n");
+
+		u.v64 = ioread64(mmio + GEN3_EMBAR0_OFFSET);
+		off += scnprintf(buf + off, buf_size - off,
+				 "EMBAR0 -\t\t%#018llx\n", u.v64);
+
+		u.v64 = ioread64(mmio + GEN3_EMBAR1_OFFSET);
+		off += scnprintf(buf + off, buf_size - off,
+				 "EMBAR1 -\t\t%#018llx\n", u.v64);
+
+		u.v64 = ioread64(mmio + GEN3_EMBAR2_OFFSET);
+		off += scnprintf(buf + off, buf_size - off,
+				 "EMBAR2 -\t\t%#018llx\n", u.v64);
+	}
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "\nNTB Statistics:\n");
+
+	u.v16 = ioread16(mmio + GEN3_USMEMMISS_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Upstream Memory Miss -\t%u\n", u.v16);
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "\nNTB Hardware Errors:\n");
+
+	if (!pci_read_config_word(ndev->ntb.pdev,
+				  GEN3_DEVSTS_OFFSET, &u.v16))
+		off += scnprintf(buf + off, buf_size - off,
+				 "DEVSTS -\t\t%#06x\n", u.v16);
+
+	if (!pci_read_config_word(ndev->ntb.pdev,
+				  GEN3_LINK_STATUS_OFFSET, &u.v16))
+		off += scnprintf(buf + off, buf_size - off,
+				 "LNKSTS -\t\t%#06x\n", u.v16);
+
+	if (!pci_read_config_dword(ndev->ntb.pdev,
+				   GEN3_UNCERRSTS_OFFSET, &u.v32))
+		off += scnprintf(buf + off, buf_size - off,
+				 "UNCERRSTS -\t\t%#06x\n", u.v32);
+
+	if (!pci_read_config_dword(ndev->ntb.pdev,
+				   GEN3_CORERRSTS_OFFSET, &u.v32))
+		off += scnprintf(buf + off, buf_size - off,
+				 "CORERRSTS -\t\t%#06x\n", u.v32);
+
+	ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
+	kfree(buf);
+	return ret;
+}
+
+static int intel_ntb3_link_enable(struct ntb_dev *ntb,
+				  enum ntb_speed max_speed,
+				  enum ntb_width max_width)
+{
+	struct intel_ntb_dev *ndev;
+	u32 ntb_ctl;
+
+	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
+
+	dev_dbg(&ntb->pdev->dev,
+		"Enabling link with max_speed %d max_width %d\n",
+		max_speed, max_width);
+
+	if (max_speed != NTB_SPEED_AUTO)
+		dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
+	if (max_width != NTB_WIDTH_AUTO)
+		dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
+
+	ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
+	ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
+	ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
+	ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
+	iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
+
+	return 0;
+}
+static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
+				   dma_addr_t addr, resource_size_t size)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+	unsigned long xlat_reg, limit_reg;
+	resource_size_t bar_size, mw_size;
+	void __iomem *mmio;
+	u64 base, limit, reg_val;
+	int bar;
+
+	if (pidx != NTB_DEF_PEER_IDX)
+		return -EINVAL;
+
+	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
+		idx += 1;
+
+	bar = ndev_mw_to_bar(ndev, idx);
+	if (bar < 0)
+		return bar;
+
+	bar_size = pci_resource_len(ndev->ntb.pdev, bar);
+
+	if (idx == ndev->b2b_idx)
+		mw_size = bar_size - ndev->b2b_off;
+	else
+		mw_size = bar_size;
+
+	/* hardware requires that addr is aligned to bar size */
+	if (addr & (bar_size - 1))
+		return -EINVAL;
+
+	/* make sure the range fits in the usable mw size */
+	if (size > mw_size)
+		return -EINVAL;
+
+	mmio = ndev->self_mmio;
+	xlat_reg = ndev->xlat_reg->bar2_xlat + (idx * 0x10);
+	limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10);
+	base = pci_resource_start(ndev->ntb.pdev, bar);
+
+	/* Set the limit if supported, if size is not mw_size */
+	if (limit_reg && size != mw_size)
+		limit = base + size;
+	else
+		limit = base + mw_size;
+
+	/* set and verify setting the translation address */
+	iowrite64(addr, mmio + xlat_reg);
+	reg_val = ioread64(mmio + xlat_reg);
+	if (reg_val != addr) {
+		iowrite64(0, mmio + xlat_reg);
+		return -EIO;
+	}
+
+	dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXBASE: %#Lx\n", bar, reg_val);
+
+	/* set and verify setting the limit */
+	iowrite64(limit, mmio + limit_reg);
+	reg_val = ioread64(mmio + limit_reg);
+	if (reg_val != limit) {
+		iowrite64(base, mmio + limit_reg);
+		iowrite64(0, mmio + xlat_reg);
+		return -EIO;
+	}
+
+	dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXLMT: %#Lx\n", bar, reg_val);
+
+	/* setup the EP */
+	limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10) + 0x4000;
+	base = ioread64(mmio + GEN3_EMBAR1_OFFSET + (8 * idx));
+	base &= ~0xf;
+
+	if (limit_reg && size != mw_size)
+		limit = base + size;
+	else
+		limit = base + mw_size;
+
+	/* set and verify setting the limit */
+	iowrite64(limit, mmio + limit_reg);
+	reg_val = ioread64(mmio + limit_reg);
+	if (reg_val != limit) {
+		iowrite64(base, mmio + limit_reg);
+		iowrite64(0, mmio + xlat_reg);
+		return -EIO;
+	}
+
+	dev_dbg(&ntb->pdev->dev, "BAR %d EMBARXLMT: %#Lx\n", bar, reg_val);
+
+	return 0;
+}
+
+static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+	int bit;
+
+	if (db_bits & ~ndev->db_valid_mask)
+		return -EINVAL;
+
+	while (db_bits) {
+		bit = __ffs(db_bits);
+		iowrite32(1, ndev->peer_mmio +
+				ndev->peer_reg->db_bell + (bit * 4));
+		db_bits &= db_bits - 1;
+	}
+
+	return 0;
+}
+
+static u64 intel_ntb3_db_read(struct ntb_dev *ntb)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_db_read(ndev,
+			    ndev->self_mmio +
+			    ndev->self_reg->db_clear);
+}
+
+static int intel_ntb3_db_clear(struct ntb_dev *ntb, u64 db_bits)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+
+	return ndev_db_write(ndev, db_bits,
+			     ndev->self_mmio +
+			     ndev->self_reg->db_clear);
+}
+
+const struct ntb_dev_ops intel_ntb3_ops = {
+	.mw_count		= intel_ntb_mw_count,
+	.mw_get_align		= intel_ntb_mw_get_align,
+	.mw_set_trans		= intel_ntb3_mw_set_trans,
+	.peer_mw_count		= intel_ntb_peer_mw_count,
+	.peer_mw_get_addr	= intel_ntb_peer_mw_get_addr,
+	.link_is_up		= intel_ntb_link_is_up,
+	.link_enable		= intel_ntb3_link_enable,
+	.link_disable		= intel_ntb_link_disable,
+	.db_valid_mask		= intel_ntb_db_valid_mask,
+	.db_vector_count	= intel_ntb_db_vector_count,
+	.db_vector_mask		= intel_ntb_db_vector_mask,
+	.db_read		= intel_ntb3_db_read,
+	.db_clear		= intel_ntb3_db_clear,
+	.db_set_mask		= intel_ntb_db_set_mask,
+	.db_clear_mask		= intel_ntb_db_clear_mask,
+	.peer_db_addr		= intel_ntb_peer_db_addr,
+	.peer_db_set		= intel_ntb3_peer_db_set,
+	.spad_is_unsafe		= intel_ntb_spad_is_unsafe,
+	.spad_count		= intel_ntb_spad_count,
+	.spad_read		= intel_ntb_spad_read,
+	.spad_write		= intel_ntb_spad_write,
+	.peer_spad_addr		= intel_ntb_peer_spad_addr,
+	.peer_spad_read		= intel_ntb_peer_spad_read,
+	.peer_spad_write	= intel_ntb_peer_spad_write,
+};
+
diff --git a/drivers/ntb/hw/intel/ntb_hw_gen3.h b/drivers/ntb/hw/intel/ntb_hw_gen3.h
new file mode 100644
index 000000000000..75fb86ca27bb
--- /dev/null
+++ b/drivers/ntb/hw/intel/ntb_hw_gen3.h
@@ -0,0 +1,110 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ *   redistributing this file, you may do so under either license.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ *   Copyright(c) 2012-2017 Intel Corporation. All rights reserved.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of version 2 of the GNU General Public License as
+ *   published by the Free Software Foundation.
+ *
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2012-2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copy
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _NTB_INTEL_GEN3_H_
+#define _NTB_INTEL_GEN3_H_
+
+#include "ntb_hw_intel.h"
+
+/* Intel Skylake Xeon hardware */
+#define GEN3_IMBAR1SZ_OFFSET		0x00d0
+#define GEN3_IMBAR2SZ_OFFSET		0x00d1
+#define GEN3_EMBAR1SZ_OFFSET		0x00d2
+#define GEN3_EMBAR2SZ_OFFSET		0x00d3
+#define GEN3_DEVCTRL_OFFSET		0x0098
+#define GEN3_DEVSTS_OFFSET		0x009a
+#define GEN3_UNCERRSTS_OFFSET		0x014c
+#define GEN3_CORERRSTS_OFFSET		0x0158
+#define GEN3_LINK_STATUS_OFFSET		0x01a2
+
+#define GEN3_NTBCNTL_OFFSET		0x0000
+#define GEN3_IMBAR1XBASE_OFFSET		0x0010		/* SBAR2XLAT */
+#define GEN3_IMBAR1XLMT_OFFSET		0x0018		/* SBAR2LMT */
+#define GEN3_IMBAR2XBASE_OFFSET		0x0020		/* SBAR4XLAT */
+#define GEN3_IMBAR2XLMT_OFFSET		0x0028		/* SBAR4LMT */
+#define GEN3_IM_INT_STATUS_OFFSET	0x0040
+#define GEN3_IM_INT_DISABLE_OFFSET	0x0048
+#define GEN3_IM_SPAD_OFFSET		0x0080		/* SPAD */
+#define GEN3_USMEMMISS_OFFSET		0x0070
+#define GEN3_INTVEC_OFFSET		0x00d0
+#define GEN3_IM_DOORBELL_OFFSET		0x0100		/* SDOORBELL0 */
+#define GEN3_B2B_SPAD_OFFSET		0x0180		/* B2B SPAD */
+#define GEN3_EMBAR0XBASE_OFFSET		0x4008		/* B2B_XLAT */
+#define GEN3_EMBAR1XBASE_OFFSET		0x4010		/* PBAR2XLAT */
+#define GEN3_EMBAR1XLMT_OFFSET		0x4018		/* PBAR2LMT */
+#define GEN3_EMBAR2XBASE_OFFSET		0x4020		/* PBAR4XLAT */
+#define GEN3_EMBAR2XLMT_OFFSET		0x4028		/* PBAR4LMT */
+#define GEN3_EM_INT_STATUS_OFFSET	0x4040
+#define GEN3_EM_INT_DISABLE_OFFSET	0x4048
+#define GEN3_EM_SPAD_OFFSET		0x4080		/* remote SPAD */
+#define GEN3_EM_DOORBELL_OFFSET		0x4100		/* PDOORBELL0 */
+#define GEN3_SPCICMD_OFFSET		0x4504		/* SPCICMD */
+#define GEN3_EMBAR0_OFFSET		0x4510		/* SBAR0BASE */
+#define GEN3_EMBAR1_OFFSET		0x4518		/* SBAR23BASE */
+#define GEN3_EMBAR2_OFFSET		0x4520		/* SBAR45BASE */
+
+#define GEN3_DB_COUNT			32
+#define GEN3_DB_LINK			32
+#define GEN3_DB_LINK_BIT		BIT_ULL(GEN3_DB_LINK)
+#define GEN3_DB_MSIX_VECTOR_COUNT	33
+#define GEN3_DB_MSIX_VECTOR_SHIFT	1
+#define GEN3_DB_TOTAL_SHIFT		33
+#define GEN3_SPAD_COUNT			16
+
+static inline u64 gen3_db_ioread(void __iomem *mmio)
+{
+	return ioread64(mmio);
+}
+
+static inline void gen3_db_iowrite(u64 bits, void __iomem *mmio)
+{
+	iowrite64(bits, mmio);
+}
+
+ssize_t ndev_ntb3_debugfs_read(struct file *filp, char __user *ubuf,
+				      size_t count, loff_t *offp);
+int gen3_init_dev(struct intel_ntb_dev *ndev);
+
+extern const struct ntb_dev_ops intel_ntb3_ops;
+
+#endif
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c b/drivers/ntb/hw/intel/ntb_hw_intel.c
deleted file mode 100644
index 156b45cd4a19..000000000000
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ /dev/null
@@ -1,2691 +0,0 @@
-/*
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- *   redistributing this file, you may do so under either license.
- *
- *   GPL LICENSE SUMMARY
- *
- *   Copyright(c) 2012 Intel Corporation. All rights reserved.
- *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
- *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of version 2 of the GNU General Public License as
- *   published by the Free Software Foundation.
- *
- *   BSD LICENSE
- *
- *   Copyright(c) 2012 Intel Corporation. All rights reserved.
- *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
- *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copy
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Intel PCIe NTB Linux driver
- *
- * Contact Information:
- * Jon Mason <jon.mason@intel.com>
- */
-
-#include <linux/debugfs.h>
-#include <linux/delay.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <linux/random.h>
-#include <linux/slab.h>
-#include <linux/ntb.h>
-
-#include "ntb_hw_intel.h"
-
-#define NTB_NAME	"ntb_hw_intel"
-#define NTB_DESC	"Intel(R) PCI-E Non-Transparent Bridge Driver"
-#define NTB_VER		"2.0"
-
-MODULE_DESCRIPTION(NTB_DESC);
-MODULE_VERSION(NTB_VER);
-MODULE_LICENSE("Dual BSD/GPL");
-MODULE_AUTHOR("Intel Corporation");
-
-#define bar0_off(base, bar) ((base) + ((bar) << 2))
-#define bar2_off(base, bar) bar0_off(base, (bar) - 2)
-
-static const struct intel_ntb_reg xeon_reg;
-static const struct intel_ntb_alt_reg xeon_pri_reg;
-static const struct intel_ntb_alt_reg xeon_sec_reg;
-static const struct intel_ntb_alt_reg xeon_b2b_reg;
-static const struct intel_ntb_xlat_reg xeon_pri_xlat;
-static const struct intel_ntb_xlat_reg xeon_sec_xlat;
-static struct intel_b2b_addr xeon_b2b_usd_addr;
-static struct intel_b2b_addr xeon_b2b_dsd_addr;
-static const struct intel_ntb_reg skx_reg;
-static const struct intel_ntb_alt_reg skx_pri_reg;
-static const struct intel_ntb_alt_reg skx_b2b_reg;
-static const struct intel_ntb_xlat_reg skx_sec_xlat;
-static const struct ntb_dev_ops intel_ntb_ops;
-static const struct ntb_dev_ops intel_ntb3_ops;
-
-static const struct file_operations intel_ntb_debugfs_info;
-static struct dentry *debugfs_dir;
-
-static int b2b_mw_idx = -1;
-module_param(b2b_mw_idx, int, 0644);
-MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb.  A "
-		 "value of zero or positive starts from first mw idx, and a "
-		 "negative value starts from last mw idx.  Both sides MUST "
-		 "set the same value here!");
-
-static unsigned int b2b_mw_share;
-module_param(b2b_mw_share, uint, 0644);
-MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
-		 "ntb so that the peer ntb only occupies the first half of "
-		 "the mw, so the second half can still be used as a mw.  Both "
-		 "sides MUST set the same value here!");
-
-module_param_named(xeon_b2b_usd_bar2_addr64,
-		   xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
-MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
-		 "XEON B2B USD BAR 2 64-bit address");
-
-module_param_named(xeon_b2b_usd_bar4_addr64,
-		   xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
-MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64,
-		 "XEON B2B USD BAR 4 64-bit address");
-
-module_param_named(xeon_b2b_usd_bar4_addr32,
-		   xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
-MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32,
-		 "XEON B2B USD split-BAR 4 32-bit address");
-
-module_param_named(xeon_b2b_usd_bar5_addr32,
-		   xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
-MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32,
-		 "XEON B2B USD split-BAR 5 32-bit address");
-
-module_param_named(xeon_b2b_dsd_bar2_addr64,
-		   xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
-MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
-		 "XEON B2B DSD BAR 2 64-bit address");
-
-module_param_named(xeon_b2b_dsd_bar4_addr64,
-		   xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
-MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64,
-		 "XEON B2B DSD BAR 4 64-bit address");
-
-module_param_named(xeon_b2b_dsd_bar4_addr32,
-		   xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
-MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32,
-		 "XEON B2B DSD split-BAR 4 32-bit address");
-
-module_param_named(xeon_b2b_dsd_bar5_addr32,
-		   xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
-MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32,
-		 "XEON B2B DSD split-BAR 5 32-bit address");
-
-static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd);
-static int xeon_init_isr(struct intel_ntb_dev *ndev);
-
-#ifndef ioread64
-#ifdef readq
-#define ioread64 readq
-#else
-#define ioread64 _ioread64
-static inline u64 _ioread64(void __iomem *mmio)
-{
-	u64 low, high;
-
-	low = ioread32(mmio);
-	high = ioread32(mmio + sizeof(u32));
-	return low | (high << 32);
-}
-#endif
-#endif
-
-#ifndef iowrite64
-#ifdef writeq
-#define iowrite64 writeq
-#else
-#define iowrite64 _iowrite64
-static inline void _iowrite64(u64 val, void __iomem *mmio)
-{
-	iowrite32(val, mmio);
-	iowrite32(val >> 32, mmio + sizeof(u32));
-}
-#endif
-#endif
-
-static inline int pdev_is_xeon(struct pci_dev *pdev)
-{
-	switch (pdev->device) {
-	case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
-		return 1;
-	}
-	return 0;
-}
-
-static inline int pdev_is_skx_xeon(struct pci_dev *pdev)
-{
-	if (pdev->device == PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)
-		return 1;
-
-	return 0;
-}
-
-static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
-{
-	ndev->unsafe_flags = 0;
-	ndev->unsafe_flags_ignore = 0;
-
-	/* Only B2B has a workaround to avoid SDOORBELL */
-	if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
-		if (!ntb_topo_is_b2b(ndev->ntb.topo))
-			ndev->unsafe_flags |= NTB_UNSAFE_DB;
-
-	/* No low level workaround to avoid SB01BASE */
-	if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
-		ndev->unsafe_flags |= NTB_UNSAFE_DB;
-		ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
-	}
-}
-
-static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
-				 unsigned long flag)
-{
-	return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
-}
-
-static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
-				     unsigned long flag)
-{
-	flag &= ndev->unsafe_flags;
-	ndev->unsafe_flags_ignore |= flag;
-
-	return !!flag;
-}
-
-static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
-{
-	if (idx < 0 || idx >= ndev->mw_count)
-		return -EINVAL;
-	return ndev->reg->mw_bar[idx];
-}
-
-static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
-			       phys_addr_t *db_addr, resource_size_t *db_size,
-			       phys_addr_t reg_addr, unsigned long reg)
-{
-	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
-		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
-
-	if (db_addr) {
-		*db_addr = reg_addr + reg;
-		dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr);
-	}
-
-	if (db_size) {
-		*db_size = ndev->reg->db_size;
-		dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size);
-	}
-
-	return 0;
-}
-
-static inline u64 ndev_db_read(struct intel_ntb_dev *ndev,
-			       void __iomem *mmio)
-{
-	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
-		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
-
-	return ndev->reg->db_ioread(mmio);
-}
-
-static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
-				void __iomem *mmio)
-{
-	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
-		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
-
-	if (db_bits & ~ndev->db_valid_mask)
-		return -EINVAL;
-
-	ndev->reg->db_iowrite(db_bits, mmio);
-
-	return 0;
-}
-
-static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
-				   void __iomem *mmio)
-{
-	unsigned long irqflags;
-
-	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
-		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
-
-	if (db_bits & ~ndev->db_valid_mask)
-		return -EINVAL;
-
-	spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
-	{
-		ndev->db_mask |= db_bits;
-		ndev->reg->db_iowrite(ndev->db_mask, mmio);
-	}
-	spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
-
-	return 0;
-}
-
-static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
-				     void __iomem *mmio)
-{
-	unsigned long irqflags;
-
-	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
-		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
-
-	if (db_bits & ~ndev->db_valid_mask)
-		return -EINVAL;
-
-	spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
-	{
-		ndev->db_mask &= ~db_bits;
-		ndev->reg->db_iowrite(ndev->db_mask, mmio);
-	}
-	spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
-
-	return 0;
-}
-
-static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
-{
-	u64 shift, mask;
-
-	shift = ndev->db_vec_shift;
-	mask = BIT_ULL(shift) - 1;
-
-	return mask << (shift * db_vector);
-}
-
-static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
-				 phys_addr_t *spad_addr, phys_addr_t reg_addr,
-				 unsigned long reg)
-{
-	if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
-		pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
-
-	if (idx < 0 || idx >= ndev->spad_count)
-		return -EINVAL;
-
-	if (spad_addr) {
-		*spad_addr = reg_addr + reg + (idx << 2);
-		dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n",
-			*spad_addr);
-	}
-
-	return 0;
-}
-
-static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
-				 void __iomem *mmio)
-{
-	if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
-		pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
-
-	if (idx < 0 || idx >= ndev->spad_count)
-		return 0;
-
-	return ioread32(mmio + (idx << 2));
-}
-
-static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
-				  void __iomem *mmio)
-{
-	if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
-		pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
-
-	if (idx < 0 || idx >= ndev->spad_count)
-		return -EINVAL;
-
-	iowrite32(val, mmio + (idx << 2));
-
-	return 0;
-}
-
-static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
-{
-	u64 vec_mask;
-
-	vec_mask = ndev_vec_mask(ndev, vec);
-
-	if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31))
-		vec_mask |= ndev->db_link_mask;
-
-	dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask);
-
-	ndev->last_ts = jiffies;
-
-	if (vec_mask & ndev->db_link_mask) {
-		if (ndev->reg->poll_link(ndev))
-			ntb_link_event(&ndev->ntb);
-	}
-
-	if (vec_mask & ndev->db_valid_mask)
-		ntb_db_event(&ndev->ntb, vec);
-
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t ndev_vec_isr(int irq, void *dev)
-{
-	struct intel_ntb_vec *nvec = dev;
-
-	dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d  nvec->num: %d\n",
-		irq, nvec->num);
-
-	return ndev_interrupt(nvec->ndev, nvec->num);
-}
-
-static irqreturn_t ndev_irq_isr(int irq, void *dev)
-{
-	struct intel_ntb_dev *ndev = dev;
-
-	return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
-}
-
-static int ndev_init_isr(struct intel_ntb_dev *ndev,
-			 int msix_min, int msix_max,
-			 int msix_shift, int total_shift)
-{
-	struct pci_dev *pdev;
-	int rc, i, msix_count, node;
-
-	pdev = ndev->ntb.pdev;
-
-	node = dev_to_node(&pdev->dev);
-
-	/* Mask all doorbell interrupts */
-	ndev->db_mask = ndev->db_valid_mask;
-	ndev->reg->db_iowrite(ndev->db_mask,
-			      ndev->self_mmio +
-			      ndev->self_reg->db_mask);
-
-	/* Try to set up msix irq */
-
-	ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
-				 GFP_KERNEL, node);
-	if (!ndev->vec)
-		goto err_msix_vec_alloc;
-
-	ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
-				  GFP_KERNEL, node);
-	if (!ndev->msix)
-		goto err_msix_alloc;
-
-	for (i = 0; i < msix_max; ++i)
-		ndev->msix[i].entry = i;
-
-	msix_count = pci_enable_msix_range(pdev, ndev->msix,
-					   msix_min, msix_max);
-	if (msix_count < 0)
-		goto err_msix_enable;
-
-	for (i = 0; i < msix_count; ++i) {
-		ndev->vec[i].ndev = ndev;
-		ndev->vec[i].num = i;
-		rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
-				 "ndev_vec_isr", &ndev->vec[i]);
-		if (rc)
-			goto err_msix_request;
-	}
-
-	dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count);
-	ndev->db_vec_count = msix_count;
-	ndev->db_vec_shift = msix_shift;
-	return 0;
-
-err_msix_request:
-	while (i-- > 0)
-		free_irq(ndev->msix[i].vector, &ndev->vec[i]);
-	pci_disable_msix(pdev);
-err_msix_enable:
-	kfree(ndev->msix);
-err_msix_alloc:
-	kfree(ndev->vec);
-err_msix_vec_alloc:
-	ndev->msix = NULL;
-	ndev->vec = NULL;
-
-	/* Try to set up msi irq */
-
-	rc = pci_enable_msi(pdev);
-	if (rc)
-		goto err_msi_enable;
-
-	rc = request_irq(pdev->irq, ndev_irq_isr, 0,
-			 "ndev_irq_isr", ndev);
-	if (rc)
-		goto err_msi_request;
-
-	dev_dbg(&pdev->dev, "Using msi interrupts\n");
-	ndev->db_vec_count = 1;
-	ndev->db_vec_shift = total_shift;
-	return 0;
-
-err_msi_request:
-	pci_disable_msi(pdev);
-err_msi_enable:
-
-	/* Try to set up intx irq */
-
-	pci_intx(pdev, 1);
-
-	rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
-			 "ndev_irq_isr", ndev);
-	if (rc)
-		goto err_intx_request;
-
-	dev_dbg(&pdev->dev, "Using intx interrupts\n");
-	ndev->db_vec_count = 1;
-	ndev->db_vec_shift = total_shift;
-	return 0;
-
-err_intx_request:
-	return rc;
-}
-
-static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
-{
-	struct pci_dev *pdev;
-	int i;
-
-	pdev = ndev->ntb.pdev;
-
-	/* Mask all doorbell interrupts */
-	ndev->db_mask = ndev->db_valid_mask;
-	ndev->reg->db_iowrite(ndev->db_mask,
-			      ndev->self_mmio +
-			      ndev->self_reg->db_mask);
-
-	if (ndev->msix) {
-		i = ndev->db_vec_count;
-		while (i--)
-			free_irq(ndev->msix[i].vector, &ndev->vec[i]);
-		pci_disable_msix(pdev);
-		kfree(ndev->msix);
-		kfree(ndev->vec);
-	} else {
-		free_irq(pdev->irq, ndev);
-		if (pci_dev_msi_enabled(pdev))
-			pci_disable_msi(pdev);
-	}
-}
-
-static ssize_t ndev_ntb3_debugfs_read(struct file *filp, char __user *ubuf,
-				      size_t count, loff_t *offp)
-{
-	struct intel_ntb_dev *ndev;
-	void __iomem *mmio;
-	char *buf;
-	size_t buf_size;
-	ssize_t ret, off;
-	union { u64 v64; u32 v32; u16 v16; } u;
-
-	ndev = filp->private_data;
-	mmio = ndev->self_mmio;
-
-	buf_size = min(count, 0x800ul);
-
-	buf = kmalloc(buf_size, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
-
-	off = 0;
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "NTB Device Information:\n");
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "Connection Topology -\t%s\n",
-			 ntb_topo_string(ndev->ntb.topo));
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
-	off += scnprintf(buf + off, buf_size - off,
-			 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
-
-	if (!ndev->reg->link_is_up(ndev))
-		off += scnprintf(buf + off, buf_size - off,
-				 "Link Status -\t\tDown\n");
-	else {
-		off += scnprintf(buf + off, buf_size - off,
-				 "Link Status -\t\tUp\n");
-		off += scnprintf(buf + off, buf_size - off,
-				 "Link Speed -\t\tPCI-E Gen %u\n",
-				 NTB_LNK_STA_SPEED(ndev->lnk_sta));
-		off += scnprintf(buf + off, buf_size - off,
-				 "Link Width -\t\tx%u\n",
-				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
-	}
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "Memory Window Count -\t%u\n", ndev->mw_count);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Scratchpad Count -\t%u\n", ndev->spad_count);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Count -\t%u\n", ndev->db_count);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
-
-	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Mask -\t\t%#llx\n", u.v64);
-
-	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Bell -\t\t%#llx\n", u.v64);
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "\nNTB Incoming XLAT:\n");
-
-	u.v64 = ioread64(mmio + SKX_IMBAR1XBASE_OFFSET);
-	off += scnprintf(buf + off, buf_size - off,
-			 "IMBAR1XBASE -\t\t%#018llx\n", u.v64);
-
-	u.v64 = ioread64(mmio + SKX_IMBAR2XBASE_OFFSET);
-	off += scnprintf(buf + off, buf_size - off,
-			 "IMBAR2XBASE -\t\t%#018llx\n", u.v64);
-
-	u.v64 = ioread64(mmio + SKX_IMBAR1XLMT_OFFSET);
-	off += scnprintf(buf + off, buf_size - off,
-			 "IMBAR1XLMT -\t\t\t%#018llx\n", u.v64);
-
-	u.v64 = ioread64(mmio + SKX_IMBAR2XLMT_OFFSET);
-	off += scnprintf(buf + off, buf_size - off,
-			 "IMBAR2XLMT -\t\t\t%#018llx\n", u.v64);
-
-	if (ntb_topo_is_b2b(ndev->ntb.topo)) {
-		off += scnprintf(buf + off, buf_size - off,
-				 "\nNTB Outgoing B2B XLAT:\n");
-
-		u.v64 = ioread64(mmio + SKX_EMBAR1XBASE_OFFSET);
-		off += scnprintf(buf + off, buf_size - off,
-				 "EMBAR1XBASE -\t\t%#018llx\n", u.v64);
-
-		u.v64 = ioread64(mmio + SKX_EMBAR2XBASE_OFFSET);
-		off += scnprintf(buf + off, buf_size - off,
-				 "EMBAR2XBASE -\t\t%#018llx\n", u.v64);
-
-		u.v64 = ioread64(mmio + SKX_EMBAR1XLMT_OFFSET);
-		off += scnprintf(buf + off, buf_size - off,
-				 "EMBAR1XLMT -\t\t%#018llx\n", u.v64);
-
-		u.v64 = ioread64(mmio + SKX_EMBAR2XLMT_OFFSET);
-		off += scnprintf(buf + off, buf_size - off,
-				 "EMBAR2XLMT -\t\t%#018llx\n", u.v64);
-
-		off += scnprintf(buf + off, buf_size - off,
-				 "\nNTB Secondary BAR:\n");
-
-		u.v64 = ioread64(mmio + SKX_EMBAR0_OFFSET);
-		off += scnprintf(buf + off, buf_size - off,
-				 "EMBAR0 -\t\t%#018llx\n", u.v64);
-
-		u.v64 = ioread64(mmio + SKX_EMBAR1_OFFSET);
-		off += scnprintf(buf + off, buf_size - off,
-				 "EMBAR1 -\t\t%#018llx\n", u.v64);
-
-		u.v64 = ioread64(mmio + SKX_EMBAR2_OFFSET);
-		off += scnprintf(buf + off, buf_size - off,
-				 "EMBAR2 -\t\t%#018llx\n", u.v64);
-	}
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "\nNTB Statistics:\n");
-
-	u.v16 = ioread16(mmio + SKX_USMEMMISS_OFFSET);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Upstream Memory Miss -\t%u\n", u.v16);
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "\nNTB Hardware Errors:\n");
-
-	if (!pci_read_config_word(ndev->ntb.pdev,
-				  SKX_DEVSTS_OFFSET, &u.v16))
-		off += scnprintf(buf + off, buf_size - off,
-				 "DEVSTS -\t\t%#06x\n", u.v16);
-
-	if (!pci_read_config_word(ndev->ntb.pdev,
-				  SKX_LINK_STATUS_OFFSET, &u.v16))
-		off += scnprintf(buf + off, buf_size - off,
-				 "LNKSTS -\t\t%#06x\n", u.v16);
-
-	if (!pci_read_config_dword(ndev->ntb.pdev,
-				   SKX_UNCERRSTS_OFFSET, &u.v32))
-		off += scnprintf(buf + off, buf_size - off,
-				 "UNCERRSTS -\t\t%#06x\n", u.v32);
-
-	if (!pci_read_config_dword(ndev->ntb.pdev,
-				   SKX_CORERRSTS_OFFSET, &u.v32))
-		off += scnprintf(buf + off, buf_size - off,
-				 "CORERRSTS -\t\t%#06x\n", u.v32);
-
-	ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
-	kfree(buf);
-	return ret;
-}
-
-static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf,
-				     size_t count, loff_t *offp)
-{
-	struct intel_ntb_dev *ndev;
-	struct pci_dev *pdev;
-	void __iomem *mmio;
-	char *buf;
-	size_t buf_size;
-	ssize_t ret, off;
-	union { u64 v64; u32 v32; u16 v16; u8 v8; } u;
-
-	ndev = filp->private_data;
-	pdev = ndev->ntb.pdev;
-	mmio = ndev->self_mmio;
-
-	buf_size = min(count, 0x800ul);
-
-	buf = kmalloc(buf_size, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
-
-	off = 0;
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "NTB Device Information:\n");
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "Connection Topology -\t%s\n",
-			 ntb_topo_string(ndev->ntb.topo));
-
-	if (ndev->b2b_idx != UINT_MAX) {
-		off += scnprintf(buf + off, buf_size - off,
-				 "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
-		off += scnprintf(buf + off, buf_size - off,
-				 "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
-	}
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "BAR4 Split -\t\t%s\n",
-			 ndev->bar4_split ? "yes" : "no");
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
-	off += scnprintf(buf + off, buf_size - off,
-			 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
-
-	if (!ndev->reg->link_is_up(ndev)) {
-		off += scnprintf(buf + off, buf_size - off,
-				 "Link Status -\t\tDown\n");
-	} else {
-		off += scnprintf(buf + off, buf_size - off,
-				 "Link Status -\t\tUp\n");
-		off += scnprintf(buf + off, buf_size - off,
-				 "Link Speed -\t\tPCI-E Gen %u\n",
-				 NTB_LNK_STA_SPEED(ndev->lnk_sta));
-		off += scnprintf(buf + off, buf_size - off,
-				 "Link Width -\t\tx%u\n",
-				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
-	}
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "Memory Window Count -\t%u\n", ndev->mw_count);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Scratchpad Count -\t%u\n", ndev->spad_count);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Count -\t%u\n", ndev->db_count);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
-
-	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Mask -\t\t%#llx\n", u.v64);
-
-	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
-	off += scnprintf(buf + off, buf_size - off,
-			 "Doorbell Bell -\t\t%#llx\n", u.v64);
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "\nNTB Window Size:\n");
-
-	pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8);
-	off += scnprintf(buf + off, buf_size - off,
-			 "PBAR23SZ %hhu\n", u.v8);
-	if (!ndev->bar4_split) {
-		pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8);
-		off += scnprintf(buf + off, buf_size - off,
-				 "PBAR45SZ %hhu\n", u.v8);
-	} else {
-		pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8);
-		off += scnprintf(buf + off, buf_size - off,
-				 "PBAR4SZ %hhu\n", u.v8);
-		pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8);
-		off += scnprintf(buf + off, buf_size - off,
-				 "PBAR5SZ %hhu\n", u.v8);
-	}
-
-	pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8);
-	off += scnprintf(buf + off, buf_size - off,
-			 "SBAR23SZ %hhu\n", u.v8);
-	if (!ndev->bar4_split) {
-		pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8);
-		off += scnprintf(buf + off, buf_size - off,
-				 "SBAR45SZ %hhu\n", u.v8);
-	} else {
-		pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8);
-		off += scnprintf(buf + off, buf_size - off,
-				 "SBAR4SZ %hhu\n", u.v8);
-		pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8);
-		off += scnprintf(buf + off, buf_size - off,
-				 "SBAR5SZ %hhu\n", u.v8);
-	}
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "\nNTB Incoming XLAT:\n");
-
-	u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
-	off += scnprintf(buf + off, buf_size - off,
-			 "XLAT23 -\t\t%#018llx\n", u.v64);
-
-	if (ndev->bar4_split) {
-		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
-		off += scnprintf(buf + off, buf_size - off,
-				 "XLAT4 -\t\t\t%#06x\n", u.v32);
-
-		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
-		off += scnprintf(buf + off, buf_size - off,
-				 "XLAT5 -\t\t\t%#06x\n", u.v32);
-	} else {
-		u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
-		off += scnprintf(buf + off, buf_size - off,
-				 "XLAT45 -\t\t%#018llx\n", u.v64);
-	}
-
-	u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
-	off += scnprintf(buf + off, buf_size - off,
-			 "LMT23 -\t\t\t%#018llx\n", u.v64);
-
-	if (ndev->bar4_split) {
-		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
-		off += scnprintf(buf + off, buf_size - off,
-				 "LMT4 -\t\t\t%#06x\n", u.v32);
-		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
-		off += scnprintf(buf + off, buf_size - off,
-				 "LMT5 -\t\t\t%#06x\n", u.v32);
-	} else {
-		u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
-		off += scnprintf(buf + off, buf_size - off,
-				 "LMT45 -\t\t\t%#018llx\n", u.v64);
-	}
-
-	if (pdev_is_xeon(pdev)) {
-		if (ntb_topo_is_b2b(ndev->ntb.topo)) {
-			off += scnprintf(buf + off, buf_size - off,
-					 "\nNTB Outgoing B2B XLAT:\n");
-
-			u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
-			off += scnprintf(buf + off, buf_size - off,
-					 "B2B XLAT23 -\t\t%#018llx\n", u.v64);
-
-			if (ndev->bar4_split) {
-				u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
-				off += scnprintf(buf + off, buf_size - off,
-						 "B2B XLAT4 -\t\t%#06x\n",
-						 u.v32);
-				u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
-				off += scnprintf(buf + off, buf_size - off,
-						 "B2B XLAT5 -\t\t%#06x\n",
-						 u.v32);
-			} else {
-				u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
-				off += scnprintf(buf + off, buf_size - off,
-						 "B2B XLAT45 -\t\t%#018llx\n",
-						 u.v64);
-			}
-
-			u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
-			off += scnprintf(buf + off, buf_size - off,
-					 "B2B LMT23 -\t\t%#018llx\n", u.v64);
-
-			if (ndev->bar4_split) {
-				u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
-				off += scnprintf(buf + off, buf_size - off,
-						 "B2B LMT4 -\t\t%#06x\n",
-						 u.v32);
-				u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
-				off += scnprintf(buf + off, buf_size - off,
-						 "B2B LMT5 -\t\t%#06x\n",
-						 u.v32);
-			} else {
-				u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
-				off += scnprintf(buf + off, buf_size - off,
-						 "B2B LMT45 -\t\t%#018llx\n",
-						 u.v64);
-			}
-
-			off += scnprintf(buf + off, buf_size - off,
-					 "\nNTB Secondary BAR:\n");
-
-			u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
-			off += scnprintf(buf + off, buf_size - off,
-					 "SBAR01 -\t\t%#018llx\n", u.v64);
-
-			u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
-			off += scnprintf(buf + off, buf_size - off,
-					 "SBAR23 -\t\t%#018llx\n", u.v64);
-
-			if (ndev->bar4_split) {
-				u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
-				off += scnprintf(buf + off, buf_size - off,
-						 "SBAR4 -\t\t\t%#06x\n", u.v32);
-				u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
-				off += scnprintf(buf + off, buf_size - off,
-						 "SBAR5 -\t\t\t%#06x\n", u.v32);
-			} else {
-				u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
-				off += scnprintf(buf + off, buf_size - off,
-						 "SBAR45 -\t\t%#018llx\n",
-						 u.v64);
-			}
-		}
-
-		off += scnprintf(buf + off, buf_size - off,
-				 "\nXEON NTB Statistics:\n");
-
-		u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
-		off += scnprintf(buf + off, buf_size - off,
-				 "Upstream Memory Miss -\t%u\n", u.v16);
-
-		off += scnprintf(buf + off, buf_size - off,
-				 "\nXEON NTB Hardware Errors:\n");
-
-		if (!pci_read_config_word(pdev,
-					  XEON_DEVSTS_OFFSET, &u.v16))
-			off += scnprintf(buf + off, buf_size - off,
-					 "DEVSTS -\t\t%#06x\n", u.v16);
-
-		if (!pci_read_config_word(pdev,
-					  XEON_LINK_STATUS_OFFSET, &u.v16))
-			off += scnprintf(buf + off, buf_size - off,
-					 "LNKSTS -\t\t%#06x\n", u.v16);
-
-		if (!pci_read_config_dword(pdev,
-					   XEON_UNCERRSTS_OFFSET, &u.v32))
-			off += scnprintf(buf + off, buf_size - off,
-					 "UNCERRSTS -\t\t%#06x\n", u.v32);
-
-		if (!pci_read_config_dword(pdev,
-					   XEON_CORERRSTS_OFFSET, &u.v32))
-			off += scnprintf(buf + off, buf_size - off,
-					 "CORERRSTS -\t\t%#06x\n", u.v32);
-	}
-
-	ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
-	kfree(buf);
-	return ret;
-}
-
-static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
-				 size_t count, loff_t *offp)
-{
-	struct intel_ntb_dev *ndev = filp->private_data;
-
-	if (pdev_is_xeon(ndev->ntb.pdev))
-		return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
-	else if (pdev_is_skx_xeon(ndev->ntb.pdev))
-		return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
-
-	return -ENXIO;
-}
-
-static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
-{
-	if (!debugfs_dir) {
-		ndev->debugfs_dir = NULL;
-		ndev->debugfs_info = NULL;
-	} else {
-		ndev->debugfs_dir =
-			debugfs_create_dir(pci_name(ndev->ntb.pdev),
-					   debugfs_dir);
-		if (!ndev->debugfs_dir)
-			ndev->debugfs_info = NULL;
-		else
-			ndev->debugfs_info =
-				debugfs_create_file("info", S_IRUSR,
-						    ndev->debugfs_dir, ndev,
-						    &intel_ntb_debugfs_info);
-	}
-}
-
-static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
-{
-	debugfs_remove_recursive(ndev->debugfs_dir);
-}
-
-static int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx)
-{
-	if (pidx != NTB_DEF_PEER_IDX)
-		return -EINVAL;
-
-	return ntb_ndev(ntb)->mw_count;
-}
-
-static int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
-				  resource_size_t *addr_align,
-				  resource_size_t *size_align,
-				  resource_size_t *size_max)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-	resource_size_t bar_size, mw_size;
-	int bar;
-
-	if (pidx != NTB_DEF_PEER_IDX)
-		return -EINVAL;
-
-	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
-		idx += 1;
-
-	bar = ndev_mw_to_bar(ndev, idx);
-	if (bar < 0)
-		return bar;
-
-	bar_size = pci_resource_len(ndev->ntb.pdev, bar);
-
-	if (idx == ndev->b2b_idx)
-		mw_size = bar_size - ndev->b2b_off;
-	else
-		mw_size = bar_size;
-
-	if (addr_align)
-		*addr_align = pci_resource_len(ndev->ntb.pdev, bar);
-
-	if (size_align)
-		*size_align = 1;
-
-	if (size_max)
-		*size_max = mw_size;
-
-	return 0;
-}
-
-static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
-				  dma_addr_t addr, resource_size_t size)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-	unsigned long base_reg, xlat_reg, limit_reg;
-	resource_size_t bar_size, mw_size;
-	void __iomem *mmio;
-	u64 base, limit, reg_val;
-	int bar;
-
-	if (pidx != NTB_DEF_PEER_IDX)
-		return -EINVAL;
-
-	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
-		idx += 1;
-
-	bar = ndev_mw_to_bar(ndev, idx);
-	if (bar < 0)
-		return bar;
-
-	bar_size = pci_resource_len(ndev->ntb.pdev, bar);
-
-	if (idx == ndev->b2b_idx)
-		mw_size = bar_size - ndev->b2b_off;
-	else
-		mw_size = bar_size;
-
-	/* hardware requires that addr is aligned to bar size */
-	if (addr & (bar_size - 1))
-		return -EINVAL;
-
-	/* make sure the range fits in the usable mw size */
-	if (size > mw_size)
-		return -EINVAL;
-
-	mmio = ndev->self_mmio;
-	base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
-	xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
-	limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
-
-	if (bar < 4 || !ndev->bar4_split) {
-		base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
-
-		/* Set the limit if supported, if size is not mw_size */
-		if (limit_reg && size != mw_size)
-			limit = base + size;
-		else
-			limit = 0;
-
-		/* set and verify setting the translation address */
-		iowrite64(addr, mmio + xlat_reg);
-		reg_val = ioread64(mmio + xlat_reg);
-		if (reg_val != addr) {
-			iowrite64(0, mmio + xlat_reg);
-			return -EIO;
-		}
-
-		/* set and verify setting the limit */
-		iowrite64(limit, mmio + limit_reg);
-		reg_val = ioread64(mmio + limit_reg);
-		if (reg_val != limit) {
-			iowrite64(base, mmio + limit_reg);
-			iowrite64(0, mmio + xlat_reg);
-			return -EIO;
-		}
-	} else {
-		/* split bar addr range must all be 32 bit */
-		if (addr & (~0ull << 32))
-			return -EINVAL;
-		if ((addr + size) & (~0ull << 32))
-			return -EINVAL;
-
-		base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
-
-		/* Set the limit if supported, if size is not mw_size */
-		if (limit_reg && size != mw_size)
-			limit = base + size;
-		else
-			limit = 0;
-
-		/* set and verify setting the translation address */
-		iowrite32(addr, mmio + xlat_reg);
-		reg_val = ioread32(mmio + xlat_reg);
-		if (reg_val != addr) {
-			iowrite32(0, mmio + xlat_reg);
-			return -EIO;
-		}
-
-		/* set and verify setting the limit */
-		iowrite32(limit, mmio + limit_reg);
-		reg_val = ioread32(mmio + limit_reg);
-		if (reg_val != limit) {
-			iowrite32(base, mmio + limit_reg);
-			iowrite32(0, mmio + xlat_reg);
-			return -EIO;
-		}
-	}
-
-	return 0;
-}
-
-static u64 intel_ntb_link_is_up(struct ntb_dev *ntb,
-				enum ntb_speed *speed,
-				enum ntb_width *width)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	if (ndev->reg->link_is_up(ndev)) {
-		if (speed)
-			*speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
-		if (width)
-			*width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
-		return 1;
-	} else {
-		/* TODO MAYBE: is it possible to observe the link speed and
-		 * width while link is training? */
-		if (speed)
-			*speed = NTB_SPEED_NONE;
-		if (width)
-			*width = NTB_WIDTH_NONE;
-		return 0;
-	}
-}
-
-static int intel_ntb_link_enable(struct ntb_dev *ntb,
-				 enum ntb_speed max_speed,
-				 enum ntb_width max_width)
-{
-	struct intel_ntb_dev *ndev;
-	u32 ntb_ctl;
-
-	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
-
-	if (ndev->ntb.topo == NTB_TOPO_SEC)
-		return -EINVAL;
-
-	dev_dbg(&ntb->pdev->dev,
-		"Enabling link with max_speed %d max_width %d\n",
-		max_speed, max_width);
-	if (max_speed != NTB_SPEED_AUTO)
-		dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
-	if (max_width != NTB_WIDTH_AUTO)
-		dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
-
-	ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
-	ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
-	ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
-	ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
-	if (ndev->bar4_split)
-		ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
-	iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
-
-	return 0;
-}
-
-static int intel_ntb_link_disable(struct ntb_dev *ntb)
-{
-	struct intel_ntb_dev *ndev;
-	u32 ntb_cntl;
-
-	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
-
-	if (ndev->ntb.topo == NTB_TOPO_SEC)
-		return -EINVAL;
-
-	dev_dbg(&ntb->pdev->dev, "Disabling link\n");
-
-	/* Bring NTB link down */
-	ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
-	ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
-	ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
-	if (ndev->bar4_split)
-		ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
-	ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
-	iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
-
-	return 0;
-}
-
-static int intel_ntb_peer_mw_count(struct ntb_dev *ntb)
-{
-	/* Numbers of inbound and outbound memory windows match */
-	return ntb_ndev(ntb)->mw_count;
-}
-
-static int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
-				     phys_addr_t *base, resource_size_t *size)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-	int bar;
-
-	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
-		idx += 1;
-
-	bar = ndev_mw_to_bar(ndev, idx);
-	if (bar < 0)
-		return bar;
-
-	if (base)
-		*base = pci_resource_start(ndev->ntb.pdev, bar) +
-			(idx == ndev->b2b_idx ? ndev->b2b_off : 0);
-
-	if (size)
-		*size = pci_resource_len(ndev->ntb.pdev, bar) -
-			(idx == ndev->b2b_idx ? ndev->b2b_off : 0);
-
-	return 0;
-}
-
-static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
-{
-	return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
-}
-
-static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
-{
-	return ntb_ndev(ntb)->db_valid_mask;
-}
-
-static int intel_ntb_db_vector_count(struct ntb_dev *ntb)
-{
-	struct intel_ntb_dev *ndev;
-
-	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
-
-	return ndev->db_vec_count;
-}
-
-static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	if (db_vector < 0 || db_vector > ndev->db_vec_count)
-		return 0;
-
-	return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
-}
-
-static u64 intel_ntb_db_read(struct ntb_dev *ntb)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_db_read(ndev,
-			    ndev->self_mmio +
-			    ndev->self_reg->db_bell);
-}
-
-static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_db_write(ndev, db_bits,
-			     ndev->self_mmio +
-			     ndev->self_reg->db_bell);
-}
-
-static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_db_set_mask(ndev, db_bits,
-				ndev->self_mmio +
-				ndev->self_reg->db_mask);
-}
-
-static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_db_clear_mask(ndev, db_bits,
-				  ndev->self_mmio +
-				  ndev->self_reg->db_mask);
-}
-
-static int intel_ntb_peer_db_addr(struct ntb_dev *ntb,
-				  phys_addr_t *db_addr,
-				  resource_size_t *db_size)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
-			    ndev->peer_reg->db_bell);
-}
-
-static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_db_write(ndev, db_bits,
-			     ndev->peer_mmio +
-			     ndev->peer_reg->db_bell);
-}
-
-static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
-{
-	return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
-}
-
-static int intel_ntb_spad_count(struct ntb_dev *ntb)
-{
-	struct intel_ntb_dev *ndev;
-
-	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
-
-	return ndev->spad_count;
-}
-
-static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_spad_read(ndev, idx,
-			      ndev->self_mmio +
-			      ndev->self_reg->spad);
-}
-
-static int intel_ntb_spad_write(struct ntb_dev *ntb,
-				int idx, u32 val)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_spad_write(ndev, idx, val,
-			       ndev->self_mmio +
-			       ndev->self_reg->spad);
-}
-
-static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
-				    phys_addr_t *spad_addr)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
-			      ndev->peer_reg->spad);
-}
-
-static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_spad_read(ndev, sidx,
-			      ndev->peer_mmio +
-			      ndev->peer_reg->spad);
-}
-
-static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
-				     int sidx, u32 val)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_spad_write(ndev, sidx, val,
-			       ndev->peer_mmio +
-			       ndev->peer_reg->spad);
-}
-
-/* Skylake Xeon NTB */
-
-static int skx_poll_link(struct intel_ntb_dev *ndev)
-{
-	u16 reg_val;
-	int rc;
-
-	ndev->reg->db_iowrite(ndev->db_link_mask,
-			      ndev->self_mmio +
-			      ndev->self_reg->db_clear);
-
-	rc = pci_read_config_word(ndev->ntb.pdev,
-				  SKX_LINK_STATUS_OFFSET, &reg_val);
-	if (rc)
-		return 0;
-
-	if (reg_val == ndev->lnk_sta)
-		return 0;
-
-	ndev->lnk_sta = reg_val;
-
-	return 1;
-}
-
-static u64 skx_db_ioread(void __iomem *mmio)
-{
-	return ioread64(mmio);
-}
-
-static void skx_db_iowrite(u64 bits, void __iomem *mmio)
-{
-	iowrite64(bits, mmio);
-}
-
-static int skx_init_isr(struct intel_ntb_dev *ndev)
-{
-	int i;
-
-	/*
-	 * The MSIX vectors and the interrupt status bits are not lined up
-	 * on Skylake. By default the link status bit is bit 32, however it
-	 * is by default MSIX vector0. We need to fixup to line them up.
-	 * The vectors at reset is 1-32,0. We need to reprogram to 0-32.
-	 */
-
-	for (i = 0; i < SKX_DB_MSIX_VECTOR_COUNT; i++)
-		iowrite8(i, ndev->self_mmio + SKX_INTVEC_OFFSET + i);
-
-	/* move link status down one as workaround */
-	if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) {
-		iowrite8(SKX_DB_MSIX_VECTOR_COUNT - 2,
-			 ndev->self_mmio + SKX_INTVEC_OFFSET +
-			 (SKX_DB_MSIX_VECTOR_COUNT - 1));
-	}
-
-	return ndev_init_isr(ndev, SKX_DB_MSIX_VECTOR_COUNT,
-			     SKX_DB_MSIX_VECTOR_COUNT,
-			     SKX_DB_MSIX_VECTOR_SHIFT,
-			     SKX_DB_TOTAL_SHIFT);
-}
-
-static int skx_setup_b2b_mw(struct intel_ntb_dev *ndev,
-			    const struct intel_b2b_addr *addr,
-			    const struct intel_b2b_addr *peer_addr)
-{
-	struct pci_dev *pdev;
-	void __iomem *mmio;
-	phys_addr_t bar_addr;
-
-	pdev = ndev->ntb.pdev;
-	mmio = ndev->self_mmio;
-
-	/* setup incoming bar limits == base addrs (zero length windows) */
-	bar_addr = addr->bar2_addr64;
-	iowrite64(bar_addr, mmio + SKX_IMBAR1XLMT_OFFSET);
-	bar_addr = ioread64(mmio + SKX_IMBAR1XLMT_OFFSET);
-	dev_dbg(&pdev->dev, "IMBAR1XLMT %#018llx\n", bar_addr);
-
-	bar_addr = addr->bar4_addr64;
-	iowrite64(bar_addr, mmio + SKX_IMBAR2XLMT_OFFSET);
-	bar_addr = ioread64(mmio + SKX_IMBAR2XLMT_OFFSET);
-	dev_dbg(&pdev->dev, "IMBAR2XLMT %#018llx\n", bar_addr);
-
-	/* zero incoming translation addrs */
-	iowrite64(0, mmio + SKX_IMBAR1XBASE_OFFSET);
-	iowrite64(0, mmio + SKX_IMBAR2XBASE_OFFSET);
-
-	ndev->peer_mmio = ndev->self_mmio;
-
-	return 0;
-}
-
-static int skx_init_ntb(struct intel_ntb_dev *ndev)
-{
-	int rc;
-
-
-	ndev->mw_count = XEON_MW_COUNT;
-	ndev->spad_count = SKX_SPAD_COUNT;
-	ndev->db_count = SKX_DB_COUNT;
-	ndev->db_link_mask = SKX_DB_LINK_BIT;
-
-	/* DB fixup for using 31 right now */
-	if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD)
-		ndev->db_link_mask |= BIT_ULL(31);
-
-	switch (ndev->ntb.topo) {
-	case NTB_TOPO_B2B_USD:
-	case NTB_TOPO_B2B_DSD:
-		ndev->self_reg = &skx_pri_reg;
-		ndev->peer_reg = &skx_b2b_reg;
-		ndev->xlat_reg = &skx_sec_xlat;
-
-		if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
-			rc = skx_setup_b2b_mw(ndev,
-					      &xeon_b2b_dsd_addr,
-					      &xeon_b2b_usd_addr);
-		} else {
-			rc = skx_setup_b2b_mw(ndev,
-					      &xeon_b2b_usd_addr,
-					      &xeon_b2b_dsd_addr);
-		}
-
-		if (rc)
-			return rc;
-
-		/* Enable Bus Master and Memory Space on the secondary side */
-		iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
-			  ndev->self_mmio + SKX_SPCICMD_OFFSET);
-
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
-
-	ndev->reg->db_iowrite(ndev->db_valid_mask,
-			      ndev->self_mmio +
-			      ndev->self_reg->db_mask);
-
-	return 0;
-}
-
-static int skx_init_dev(struct intel_ntb_dev *ndev)
-{
-	struct pci_dev *pdev;
-	u8 ppd;
-	int rc;
-
-	pdev = ndev->ntb.pdev;
-
-	ndev->reg = &skx_reg;
-
-	rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
-	if (rc)
-		return -EIO;
-
-	ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
-	dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
-		ntb_topo_string(ndev->ntb.topo));
-	if (ndev->ntb.topo == NTB_TOPO_NONE)
-		return -EINVAL;
-
-	if (pdev_is_skx_xeon(pdev))
-		ndev->hwerr_flags |= NTB_HWERR_MSIX_VECTOR32_BAD;
-
-	rc = skx_init_ntb(ndev);
-	if (rc)
-		return rc;
-
-	return skx_init_isr(ndev);
-}
-
-static int intel_ntb3_link_enable(struct ntb_dev *ntb,
-				  enum ntb_speed max_speed,
-				  enum ntb_width max_width)
-{
-	struct intel_ntb_dev *ndev;
-	u32 ntb_ctl;
-
-	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
-
-	dev_dbg(&ntb->pdev->dev,
-		"Enabling link with max_speed %d max_width %d\n",
-		max_speed, max_width);
-
-	if (max_speed != NTB_SPEED_AUTO)
-		dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
-	if (max_width != NTB_WIDTH_AUTO)
-		dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
-
-	ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
-	ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
-	ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
-	ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
-	iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
-
-	return 0;
-}
-static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
-				   dma_addr_t addr, resource_size_t size)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-	unsigned long xlat_reg, limit_reg;
-	resource_size_t bar_size, mw_size;
-	void __iomem *mmio;
-	u64 base, limit, reg_val;
-	int bar;
-
-	if (pidx != NTB_DEF_PEER_IDX)
-		return -EINVAL;
-
-	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
-		idx += 1;
-
-	bar = ndev_mw_to_bar(ndev, idx);
-	if (bar < 0)
-		return bar;
-
-	bar_size = pci_resource_len(ndev->ntb.pdev, bar);
-
-	if (idx == ndev->b2b_idx)
-		mw_size = bar_size - ndev->b2b_off;
-	else
-		mw_size = bar_size;
-
-	/* hardware requires that addr is aligned to bar size */
-	if (addr & (bar_size - 1))
-		return -EINVAL;
-
-	/* make sure the range fits in the usable mw size */
-	if (size > mw_size)
-		return -EINVAL;
-
-	mmio = ndev->self_mmio;
-	xlat_reg = ndev->xlat_reg->bar2_xlat + (idx * 0x10);
-	limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10);
-	base = pci_resource_start(ndev->ntb.pdev, bar);
-
-	/* Set the limit if supported, if size is not mw_size */
-	if (limit_reg && size != mw_size)
-		limit = base + size;
-	else
-		limit = base + mw_size;
-
-	/* set and verify setting the translation address */
-	iowrite64(addr, mmio + xlat_reg);
-	reg_val = ioread64(mmio + xlat_reg);
-	if (reg_val != addr) {
-		iowrite64(0, mmio + xlat_reg);
-		return -EIO;
-	}
-
-	dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXBASE: %#Lx\n", bar, reg_val);
-
-	/* set and verify setting the limit */
-	iowrite64(limit, mmio + limit_reg);
-	reg_val = ioread64(mmio + limit_reg);
-	if (reg_val != limit) {
-		iowrite64(base, mmio + limit_reg);
-		iowrite64(0, mmio + xlat_reg);
-		return -EIO;
-	}
-
-	dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXLMT: %#Lx\n", bar, reg_val);
-
-	/* setup the EP */
-	limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10) + 0x4000;
-	base = ioread64(mmio + SKX_EMBAR1_OFFSET + (8 * idx));
-	base &= ~0xf;
-
-	if (limit_reg && size != mw_size)
-		limit = base + size;
-	else
-		limit = base + mw_size;
-
-	/* set and verify setting the limit */
-	iowrite64(limit, mmio + limit_reg);
-	reg_val = ioread64(mmio + limit_reg);
-	if (reg_val != limit) {
-		iowrite64(base, mmio + limit_reg);
-		iowrite64(0, mmio + xlat_reg);
-		return -EIO;
-	}
-
-	dev_dbg(&ntb->pdev->dev, "BAR %d EMBARXLMT: %#Lx\n", bar, reg_val);
-
-	return 0;
-}
-
-static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-	int bit;
-
-	if (db_bits & ~ndev->db_valid_mask)
-		return -EINVAL;
-
-	while (db_bits) {
-		bit = __ffs(db_bits);
-		iowrite32(1, ndev->peer_mmio +
-				ndev->peer_reg->db_bell + (bit * 4));
-		db_bits &= db_bits - 1;
-	}
-
-	return 0;
-}
-
-static u64 intel_ntb3_db_read(struct ntb_dev *ntb)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_db_read(ndev,
-			    ndev->self_mmio +
-			    ndev->self_reg->db_clear);
-}
-
-static int intel_ntb3_db_clear(struct ntb_dev *ntb, u64 db_bits)
-{
-	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-
-	return ndev_db_write(ndev, db_bits,
-			     ndev->self_mmio +
-			     ndev->self_reg->db_clear);
-}
-
-/* XEON */
-
-static u64 xeon_db_ioread(void __iomem *mmio)
-{
-	return (u64)ioread16(mmio);
-}
-
-static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
-{
-	iowrite16((u16)bits, mmio);
-}
-
-static int xeon_poll_link(struct intel_ntb_dev *ndev)
-{
-	u16 reg_val;
-	int rc;
-
-	ndev->reg->db_iowrite(ndev->db_link_mask,
-			      ndev->self_mmio +
-			      ndev->self_reg->db_bell);
-
-	rc = pci_read_config_word(ndev->ntb.pdev,
-				  XEON_LINK_STATUS_OFFSET, &reg_val);
-	if (rc)
-		return 0;
-
-	if (reg_val == ndev->lnk_sta)
-		return 0;
-
-	ndev->lnk_sta = reg_val;
-
-	return 1;
-}
-
-static int xeon_link_is_up(struct intel_ntb_dev *ndev)
-{
-	if (ndev->ntb.topo == NTB_TOPO_SEC)
-		return 1;
-
-	return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
-}
-
-static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
-{
-	switch (ppd & XEON_PPD_TOPO_MASK) {
-	case XEON_PPD_TOPO_B2B_USD:
-		return NTB_TOPO_B2B_USD;
-
-	case XEON_PPD_TOPO_B2B_DSD:
-		return NTB_TOPO_B2B_DSD;
-
-	case XEON_PPD_TOPO_PRI_USD:
-	case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
-		return NTB_TOPO_PRI;
-
-	case XEON_PPD_TOPO_SEC_USD:
-	case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
-		return NTB_TOPO_SEC;
-	}
-
-	return NTB_TOPO_NONE;
-}
-
-static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
-{
-	if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
-		dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd);
-		return 1;
-	}
-	return 0;
-}
-
-static int xeon_init_isr(struct intel_ntb_dev *ndev)
-{
-	return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
-			     XEON_DB_MSIX_VECTOR_COUNT,
-			     XEON_DB_MSIX_VECTOR_SHIFT,
-			     XEON_DB_TOTAL_SHIFT);
-}
-
-static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
-{
-	ndev_deinit_isr(ndev);
-}
-
-static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
-			     const struct intel_b2b_addr *addr,
-			     const struct intel_b2b_addr *peer_addr)
-{
-	struct pci_dev *pdev;
-	void __iomem *mmio;
-	resource_size_t bar_size;
-	phys_addr_t bar_addr;
-	int b2b_bar;
-	u8 bar_sz;
-
-	pdev = ndev->ntb.pdev;
-	mmio = ndev->self_mmio;
-
-	if (ndev->b2b_idx == UINT_MAX) {
-		dev_dbg(&pdev->dev, "not using b2b mw\n");
-		b2b_bar = 0;
-		ndev->b2b_off = 0;
-	} else {
-		b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
-		if (b2b_bar < 0)
-			return -EIO;
-
-		dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
-
-		bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
-
-		dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
-
-		if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
-			dev_dbg(&pdev->dev, "b2b using first half of bar\n");
-			ndev->b2b_off = bar_size >> 1;
-		} else if (XEON_B2B_MIN_SIZE <= bar_size) {
-			dev_dbg(&pdev->dev, "b2b using whole bar\n");
-			ndev->b2b_off = 0;
-			--ndev->mw_count;
-		} else {
-			dev_dbg(&pdev->dev, "b2b bar size is too small\n");
-			return -EIO;
-		}
-	}
-
-	/* Reset the secondary bar sizes to match the primary bar sizes,
-	 * except disable or halve the size of the b2b secondary bar.
-	 *
-	 * Note: code for each specific bar size register, because the register
-	 * offsets are not in a consistent order (bar5sz comes after ppd, odd).
-	 */
-	pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
-	dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz);
-	if (b2b_bar == 2) {
-		if (ndev->b2b_off)
-			bar_sz -= 1;
-		else
-			bar_sz = 0;
-	}
-	pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
-	pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
-	dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz);
-
-	if (!ndev->bar4_split) {
-		pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
-		dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz);
-		if (b2b_bar == 4) {
-			if (ndev->b2b_off)
-				bar_sz -= 1;
-			else
-				bar_sz = 0;
-		}
-		pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
-		pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
-		dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz);
-	} else {
-		pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
-		dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz);
-		if (b2b_bar == 4) {
-			if (ndev->b2b_off)
-				bar_sz -= 1;
-			else
-				bar_sz = 0;
-		}
-		pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
-		pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
-		dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz);
-
-		pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
-		dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz);
-		if (b2b_bar == 5) {
-			if (ndev->b2b_off)
-				bar_sz -= 1;
-			else
-				bar_sz = 0;
-		}
-		pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
-		pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
-		dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz);
-	}
-
-	/* SBAR01 hit by first part of the b2b bar */
-	if (b2b_bar == 0)
-		bar_addr = addr->bar0_addr;
-	else if (b2b_bar == 2)
-		bar_addr = addr->bar2_addr64;
-	else if (b2b_bar == 4 && !ndev->bar4_split)
-		bar_addr = addr->bar4_addr64;
-	else if (b2b_bar == 4)
-		bar_addr = addr->bar4_addr32;
-	else if (b2b_bar == 5)
-		bar_addr = addr->bar5_addr32;
-	else
-		return -EIO;
-
-	dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr);
-	iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
-
-	/* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
-	 * The b2b bar is either disabled above, or configured half-size, and
-	 * it starts at the PBAR xlat + offset.
-	 */
-
-	bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
-	iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
-	bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
-	dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr);
-
-	if (!ndev->bar4_split) {
-		bar_addr = addr->bar4_addr64 +
-			(b2b_bar == 4 ? ndev->b2b_off : 0);
-		iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
-		bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
-		dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr);
-	} else {
-		bar_addr = addr->bar4_addr32 +
-			(b2b_bar == 4 ? ndev->b2b_off : 0);
-		iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
-		bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
-		dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr);
-
-		bar_addr = addr->bar5_addr32 +
-			(b2b_bar == 5 ? ndev->b2b_off : 0);
-		iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
-		bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
-		dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr);
-	}
-
-	/* setup incoming bar limits == base addrs (zero length windows) */
-
-	bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
-	iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
-	bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
-	dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr);
-
-	if (!ndev->bar4_split) {
-		bar_addr = addr->bar4_addr64 +
-			(b2b_bar == 4 ? ndev->b2b_off : 0);
-		iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
-		bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
-		dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr);
-	} else {
-		bar_addr = addr->bar4_addr32 +
-			(b2b_bar == 4 ? ndev->b2b_off : 0);
-		iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
-		bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
-		dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr);
-
-		bar_addr = addr->bar5_addr32 +
-			(b2b_bar == 5 ? ndev->b2b_off : 0);
-		iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
-		bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
-		dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr);
-	}
-
-	/* zero incoming translation addrs */
-	iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
-
-	if (!ndev->bar4_split) {
-		iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
-	} else {
-		iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
-		iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
-	}
-
-	/* zero outgoing translation limits (whole bar size windows) */
-	iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
-	if (!ndev->bar4_split) {
-		iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
-	} else {
-		iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
-		iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
-	}
-
-	/* set outgoing translation offsets */
-	bar_addr = peer_addr->bar2_addr64;
-	iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
-	bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
-	dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr);
-
-	if (!ndev->bar4_split) {
-		bar_addr = peer_addr->bar4_addr64;
-		iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
-		bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
-		dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr);
-	} else {
-		bar_addr = peer_addr->bar4_addr32;
-		iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
-		bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
-		dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr);
-
-		bar_addr = peer_addr->bar5_addr32;
-		iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
-		bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
-		dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr);
-	}
-
-	/* set the translation offset for b2b registers */
-	if (b2b_bar == 0)
-		bar_addr = peer_addr->bar0_addr;
-	else if (b2b_bar == 2)
-		bar_addr = peer_addr->bar2_addr64;
-	else if (b2b_bar == 4 && !ndev->bar4_split)
-		bar_addr = peer_addr->bar4_addr64;
-	else if (b2b_bar == 4)
-		bar_addr = peer_addr->bar4_addr32;
-	else if (b2b_bar == 5)
-		bar_addr = peer_addr->bar5_addr32;
-	else
-		return -EIO;
-
-	/* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
-	dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr);
-	iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
-	iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
-
-	if (b2b_bar) {
-		/* map peer ntb mmio config space registers */
-		ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
-					    XEON_B2B_MIN_SIZE);
-		if (!ndev->peer_mmio)
-			return -EIO;
-
-		ndev->peer_addr = pci_resource_start(pdev, b2b_bar);
-	}
-
-	return 0;
-}
-
-static int xeon_init_ntb(struct intel_ntb_dev *ndev)
-{
-	struct device *dev = &ndev->ntb.pdev->dev;
-	int rc;
-	u32 ntb_ctl;
-
-	if (ndev->bar4_split)
-		ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
-	else
-		ndev->mw_count = XEON_MW_COUNT;
-
-	ndev->spad_count = XEON_SPAD_COUNT;
-	ndev->db_count = XEON_DB_COUNT;
-	ndev->db_link_mask = XEON_DB_LINK_BIT;
-
-	switch (ndev->ntb.topo) {
-	case NTB_TOPO_PRI:
-		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
-			dev_err(dev, "NTB Primary config disabled\n");
-			return -EINVAL;
-		}
-
-		/* enable link to allow secondary side device to appear */
-		ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
-		ntb_ctl &= ~NTB_CTL_DISABLE;
-		iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
-
-		/* use half the spads for the peer */
-		ndev->spad_count >>= 1;
-		ndev->self_reg = &xeon_pri_reg;
-		ndev->peer_reg = &xeon_sec_reg;
-		ndev->xlat_reg = &xeon_sec_xlat;
-		break;
-
-	case NTB_TOPO_SEC:
-		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
-			dev_err(dev, "NTB Secondary config disabled\n");
-			return -EINVAL;
-		}
-		/* use half the spads for the peer */
-		ndev->spad_count >>= 1;
-		ndev->self_reg = &xeon_sec_reg;
-		ndev->peer_reg = &xeon_pri_reg;
-		ndev->xlat_reg = &xeon_pri_xlat;
-		break;
-
-	case NTB_TOPO_B2B_USD:
-	case NTB_TOPO_B2B_DSD:
-		ndev->self_reg = &xeon_pri_reg;
-		ndev->peer_reg = &xeon_b2b_reg;
-		ndev->xlat_reg = &xeon_sec_xlat;
-
-		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
-			ndev->peer_reg = &xeon_pri_reg;
-
-			if (b2b_mw_idx < 0)
-				ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
-			else
-				ndev->b2b_idx = b2b_mw_idx;
-
-			if (ndev->b2b_idx >= ndev->mw_count) {
-				dev_dbg(dev,
-					"b2b_mw_idx %d invalid for mw_count %u\n",
-					b2b_mw_idx, ndev->mw_count);
-				return -EINVAL;
-			}
-
-			dev_dbg(dev, "setting up b2b mw idx %d means %d\n",
-				b2b_mw_idx, ndev->b2b_idx);
-
-		} else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
-			dev_warn(dev, "Reduce doorbell count by 1\n");
-			ndev->db_count -= 1;
-		}
-
-		if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
-			rc = xeon_setup_b2b_mw(ndev,
-					       &xeon_b2b_dsd_addr,
-					       &xeon_b2b_usd_addr);
-		} else {
-			rc = xeon_setup_b2b_mw(ndev,
-					       &xeon_b2b_usd_addr,
-					       &xeon_b2b_dsd_addr);
-		}
-		if (rc)
-			return rc;
-
-		/* Enable Bus Master and Memory Space on the secondary side */
-		iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
-			  ndev->self_mmio + XEON_SPCICMD_OFFSET);
-
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
-
-	ndev->reg->db_iowrite(ndev->db_valid_mask,
-			      ndev->self_mmio +
-			      ndev->self_reg->db_mask);
-
-	return 0;
-}
-
-static int xeon_init_dev(struct intel_ntb_dev *ndev)
-{
-	struct pci_dev *pdev;
-	u8 ppd;
-	int rc, mem;
-
-	pdev = ndev->ntb.pdev;
-
-	switch (pdev->device) {
-	/* There is a Xeon hardware errata related to writes to SDOORBELL or
-	 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
-	 * which may hang the system.  To workaround this use the second memory
-	 * window to access the interrupt and scratch pad registers on the
-	 * remote system.
-	 */
-	case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
-		ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
-		break;
-	}
-
-	switch (pdev->device) {
-	/* There is a hardware errata related to accessing any register in
-	 * SB01BASE in the presence of bidirectional traffic crossing the NTB.
-	 */
-	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
-		ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
-		break;
-	}
-
-	switch (pdev->device) {
-	/* HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
-	 * mirrored to the remote system.  Shrink the number of bits by one,
-	 * since bit 14 is the last bit.
-	 */
-	case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
-	case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
-	case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
-		ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
-		break;
-	}
-
-	ndev->reg = &xeon_reg;
-
-	rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
-	if (rc)
-		return -EIO;
-
-	ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
-	dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
-		ntb_topo_string(ndev->ntb.topo));
-	if (ndev->ntb.topo == NTB_TOPO_NONE)
-		return -EINVAL;
-
-	if (ndev->ntb.topo != NTB_TOPO_SEC) {
-		ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
-		dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n",
-			ppd, ndev->bar4_split);
-	} else {
-		/* This is a way for transparent BAR to figure out if we are
-		 * doing split BAR or not. There is no way for the hw on the
-		 * transparent side to know and set the PPD.
-		 */
-		mem = pci_select_bars(pdev, IORESOURCE_MEM);
-		ndev->bar4_split = hweight32(mem) ==
-			HSX_SPLIT_BAR_MW_COUNT + 1;
-		dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n",
-			mem, ndev->bar4_split);
-	}
-
-	rc = xeon_init_ntb(ndev);
-	if (rc)
-		return rc;
-
-	return xeon_init_isr(ndev);
-}
-
-static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
-{
-	xeon_deinit_isr(ndev);
-}
-
-static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
-{
-	int rc;
-
-	pci_set_drvdata(pdev, ndev);
-
-	rc = pci_enable_device(pdev);
-	if (rc)
-		goto err_pci_enable;
-
-	rc = pci_request_regions(pdev, NTB_NAME);
-	if (rc)
-		goto err_pci_regions;
-
-	pci_set_master(pdev);
-
-	rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
-	if (rc) {
-		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
-		if (rc)
-			goto err_dma_mask;
-		dev_warn(&pdev->dev, "Cannot DMA highmem\n");
-	}
-
-	rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
-	if (rc) {
-		rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
-		if (rc)
-			goto err_dma_mask;
-		dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
-	}
-	rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
-					  dma_get_mask(&pdev->dev));
-	if (rc)
-		goto err_dma_mask;
-
-	ndev->self_mmio = pci_iomap(pdev, 0, 0);
-	if (!ndev->self_mmio) {
-		rc = -EIO;
-		goto err_mmio;
-	}
-	ndev->peer_mmio = ndev->self_mmio;
-	ndev->peer_addr = pci_resource_start(pdev, 0);
-
-	return 0;
-
-err_mmio:
-err_dma_mask:
-	pci_clear_master(pdev);
-	pci_release_regions(pdev);
-err_pci_regions:
-	pci_disable_device(pdev);
-err_pci_enable:
-	pci_set_drvdata(pdev, NULL);
-	return rc;
-}
-
-static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
-{
-	struct pci_dev *pdev = ndev->ntb.pdev;
-
-	if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
-		pci_iounmap(pdev, ndev->peer_mmio);
-	pci_iounmap(pdev, ndev->self_mmio);
-
-	pci_clear_master(pdev);
-	pci_release_regions(pdev);
-	pci_disable_device(pdev);
-	pci_set_drvdata(pdev, NULL);
-}
-
-static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
-				    struct pci_dev *pdev)
-{
-	ndev->ntb.pdev = pdev;
-	ndev->ntb.topo = NTB_TOPO_NONE;
-	ndev->ntb.ops = &intel_ntb_ops;
-
-	ndev->b2b_off = 0;
-	ndev->b2b_idx = UINT_MAX;
-
-	ndev->bar4_split = 0;
-
-	ndev->mw_count = 0;
-	ndev->spad_count = 0;
-	ndev->db_count = 0;
-	ndev->db_vec_count = 0;
-	ndev->db_vec_shift = 0;
-
-	ndev->ntb_ctl = 0;
-	ndev->lnk_sta = 0;
-
-	ndev->db_valid_mask = 0;
-	ndev->db_link_mask = 0;
-	ndev->db_mask = 0;
-
-	spin_lock_init(&ndev->db_mask_lock);
-}
-
-static int intel_ntb_pci_probe(struct pci_dev *pdev,
-			       const struct pci_device_id *id)
-{
-	struct intel_ntb_dev *ndev;
-	int rc, node;
-
-	node = dev_to_node(&pdev->dev);
-
-	if (pdev_is_xeon(pdev)) {
-		ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
-		if (!ndev) {
-			rc = -ENOMEM;
-			goto err_ndev;
-		}
-
-		ndev_init_struct(ndev, pdev);
-
-		rc = intel_ntb_init_pci(ndev, pdev);
-		if (rc)
-			goto err_init_pci;
-
-		rc = xeon_init_dev(ndev);
-		if (rc)
-			goto err_init_dev;
-
-	} else if (pdev_is_skx_xeon(pdev)) {
-		ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
-		if (!ndev) {
-			rc = -ENOMEM;
-			goto err_ndev;
-		}
-
-		ndev_init_struct(ndev, pdev);
-		ndev->ntb.ops = &intel_ntb3_ops;
-
-		rc = intel_ntb_init_pci(ndev, pdev);
-		if (rc)
-			goto err_init_pci;
-
-		rc = skx_init_dev(ndev);
-		if (rc)
-			goto err_init_dev;
-
-	} else {
-		rc = -EINVAL;
-		goto err_ndev;
-	}
-
-	ndev_reset_unsafe_flags(ndev);
-
-	ndev->reg->poll_link(ndev);
-
-	ndev_init_debugfs(ndev);
-
-	rc = ntb_register_device(&ndev->ntb);
-	if (rc)
-		goto err_register;
-
-	dev_info(&pdev->dev, "NTB device registered.\n");
-
-	return 0;
-
-err_register:
-	ndev_deinit_debugfs(ndev);
-	if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
-		xeon_deinit_dev(ndev);
-err_init_dev:
-	intel_ntb_deinit_pci(ndev);
-err_init_pci:
-	kfree(ndev);
-err_ndev:
-	return rc;
-}
-
-static void intel_ntb_pci_remove(struct pci_dev *pdev)
-{
-	struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
-
-	ntb_unregister_device(&ndev->ntb);
-	ndev_deinit_debugfs(ndev);
-	if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
-		xeon_deinit_dev(ndev);
-	intel_ntb_deinit_pci(ndev);
-	kfree(ndev);
-}
-
-static const struct intel_ntb_reg xeon_reg = {
-	.poll_link		= xeon_poll_link,
-	.link_is_up		= xeon_link_is_up,
-	.db_ioread		= xeon_db_ioread,
-	.db_iowrite		= xeon_db_iowrite,
-	.db_size		= sizeof(u32),
-	.ntb_ctl		= XEON_NTBCNTL_OFFSET,
-	.mw_bar			= {2, 4, 5},
-};
-
-static const struct intel_ntb_alt_reg xeon_pri_reg = {
-	.db_bell		= XEON_PDOORBELL_OFFSET,
-	.db_mask		= XEON_PDBMSK_OFFSET,
-	.spad			= XEON_SPAD_OFFSET,
-};
-
-static const struct intel_ntb_alt_reg xeon_sec_reg = {
-	.db_bell		= XEON_SDOORBELL_OFFSET,
-	.db_mask		= XEON_SDBMSK_OFFSET,
-	/* second half of the scratchpads */
-	.spad			= XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
-};
-
-static const struct intel_ntb_alt_reg xeon_b2b_reg = {
-	.db_bell		= XEON_B2B_DOORBELL_OFFSET,
-	.spad			= XEON_B2B_SPAD_OFFSET,
-};
-
-static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
-	/* Note: no primary .bar0_base visible to the secondary side.
-	 *
-	 * The secondary side cannot get the base address stored in primary
-	 * bars.  The base address is necessary to set the limit register to
-	 * any value other than zero, or unlimited.
-	 *
-	 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
-	 * window by setting the limit equal to base, nor can it limit the size
-	 * of the memory window by setting the limit to base + size.
-	 */
-	.bar2_limit		= XEON_PBAR23LMT_OFFSET,
-	.bar2_xlat		= XEON_PBAR23XLAT_OFFSET,
-};
-
-static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
-	.bar0_base		= XEON_SBAR0BASE_OFFSET,
-	.bar2_limit		= XEON_SBAR23LMT_OFFSET,
-	.bar2_xlat		= XEON_SBAR23XLAT_OFFSET,
-};
-
-static struct intel_b2b_addr xeon_b2b_usd_addr = {
-	.bar2_addr64		= XEON_B2B_BAR2_ADDR64,
-	.bar4_addr64		= XEON_B2B_BAR4_ADDR64,
-	.bar4_addr32		= XEON_B2B_BAR4_ADDR32,
-	.bar5_addr32		= XEON_B2B_BAR5_ADDR32,
-};
-
-static struct intel_b2b_addr xeon_b2b_dsd_addr = {
-	.bar2_addr64		= XEON_B2B_BAR2_ADDR64,
-	.bar4_addr64		= XEON_B2B_BAR4_ADDR64,
-	.bar4_addr32		= XEON_B2B_BAR4_ADDR32,
-	.bar5_addr32		= XEON_B2B_BAR5_ADDR32,
-};
-
-static const struct intel_ntb_reg skx_reg = {
-	.poll_link		= skx_poll_link,
-	.link_is_up		= xeon_link_is_up,
-	.db_ioread		= skx_db_ioread,
-	.db_iowrite		= skx_db_iowrite,
-	.db_size		= sizeof(u32),
-	.ntb_ctl		= SKX_NTBCNTL_OFFSET,
-	.mw_bar			= {2, 4},
-};
-
-static const struct intel_ntb_alt_reg skx_pri_reg = {
-	.db_bell		= SKX_EM_DOORBELL_OFFSET,
-	.db_clear		= SKX_IM_INT_STATUS_OFFSET,
-	.db_mask		= SKX_IM_INT_DISABLE_OFFSET,
-	.spad			= SKX_IM_SPAD_OFFSET,
-};
-
-static const struct intel_ntb_alt_reg skx_b2b_reg = {
-	.db_bell		= SKX_IM_DOORBELL_OFFSET,
-	.db_clear		= SKX_EM_INT_STATUS_OFFSET,
-	.db_mask		= SKX_EM_INT_DISABLE_OFFSET,
-	.spad			= SKX_B2B_SPAD_OFFSET,
-};
-
-static const struct intel_ntb_xlat_reg skx_sec_xlat = {
-/*	.bar0_base		= SKX_EMBAR0_OFFSET, */
-	.bar2_limit		= SKX_IMBAR1XLMT_OFFSET,
-	.bar2_xlat		= SKX_IMBAR1XBASE_OFFSET,
-};
-
-/* operations for primary side of local ntb */
-static const struct ntb_dev_ops intel_ntb_ops = {
-	.mw_count		= intel_ntb_mw_count,
-	.mw_get_align		= intel_ntb_mw_get_align,
-	.mw_set_trans		= intel_ntb_mw_set_trans,
-	.peer_mw_count		= intel_ntb_peer_mw_count,
-	.peer_mw_get_addr	= intel_ntb_peer_mw_get_addr,
-	.link_is_up		= intel_ntb_link_is_up,
-	.link_enable		= intel_ntb_link_enable,
-	.link_disable		= intel_ntb_link_disable,
-	.db_is_unsafe		= intel_ntb_db_is_unsafe,
-	.db_valid_mask		= intel_ntb_db_valid_mask,
-	.db_vector_count	= intel_ntb_db_vector_count,
-	.db_vector_mask		= intel_ntb_db_vector_mask,
-	.db_read		= intel_ntb_db_read,
-	.db_clear		= intel_ntb_db_clear,
-	.db_set_mask		= intel_ntb_db_set_mask,
-	.db_clear_mask		= intel_ntb_db_clear_mask,
-	.peer_db_addr		= intel_ntb_peer_db_addr,
-	.peer_db_set		= intel_ntb_peer_db_set,
-	.spad_is_unsafe		= intel_ntb_spad_is_unsafe,
-	.spad_count		= intel_ntb_spad_count,
-	.spad_read		= intel_ntb_spad_read,
-	.spad_write		= intel_ntb_spad_write,
-	.peer_spad_addr		= intel_ntb_peer_spad_addr,
-	.peer_spad_read		= intel_ntb_peer_spad_read,
-	.peer_spad_write	= intel_ntb_peer_spad_write,
-};
-
-static const struct ntb_dev_ops intel_ntb3_ops = {
-	.mw_count		= intel_ntb_mw_count,
-	.mw_get_align		= intel_ntb_mw_get_align,
-	.mw_set_trans		= intel_ntb3_mw_set_trans,
-	.peer_mw_count		= intel_ntb_peer_mw_count,
-	.peer_mw_get_addr	= intel_ntb_peer_mw_get_addr,
-	.link_is_up		= intel_ntb_link_is_up,
-	.link_enable		= intel_ntb3_link_enable,
-	.link_disable		= intel_ntb_link_disable,
-	.db_valid_mask		= intel_ntb_db_valid_mask,
-	.db_vector_count	= intel_ntb_db_vector_count,
-	.db_vector_mask		= intel_ntb_db_vector_mask,
-	.db_read		= intel_ntb3_db_read,
-	.db_clear		= intel_ntb3_db_clear,
-	.db_set_mask		= intel_ntb_db_set_mask,
-	.db_clear_mask		= intel_ntb_db_clear_mask,
-	.peer_db_addr		= intel_ntb_peer_db_addr,
-	.peer_db_set		= intel_ntb3_peer_db_set,
-	.spad_is_unsafe		= intel_ntb_spad_is_unsafe,
-	.spad_count		= intel_ntb_spad_count,
-	.spad_read		= intel_ntb_spad_read,
-	.spad_write		= intel_ntb_spad_write,
-	.peer_spad_addr		= intel_ntb_peer_spad_addr,
-	.peer_spad_read		= intel_ntb_peer_spad_read,
-	.peer_spad_write	= intel_ntb_peer_spad_write,
-};
-
-static const struct file_operations intel_ntb_debugfs_info = {
-	.owner = THIS_MODULE,
-	.open = simple_open,
-	.read = ndev_debugfs_read,
-};
-
-static const struct pci_device_id intel_ntb_pci_tbl[] = {
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
-	{0}
-};
-MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
-
-static struct pci_driver intel_ntb_pci_driver = {
-	.name = KBUILD_MODNAME,
-	.id_table = intel_ntb_pci_tbl,
-	.probe = intel_ntb_pci_probe,
-	.remove = intel_ntb_pci_remove,
-};
-
-static int __init intel_ntb_pci_driver_init(void)
-{
-	pr_info("%s %s\n", NTB_DESC, NTB_VER);
-
-	if (debugfs_initialized())
-		debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
-
-	return pci_register_driver(&intel_ntb_pci_driver);
-}
-module_init(intel_ntb_pci_driver_init);
-
-static void __exit intel_ntb_pci_driver_exit(void)
-{
-	pci_unregister_driver(&intel_ntb_pci_driver);
-
-	debugfs_remove_recursive(debugfs_dir);
-}
-module_exit(intel_ntb_pci_driver_exit);
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.h b/drivers/ntb/hw/intel/ntb_hw_intel.h
index 4415aa7ea775..c49ff8970ce3 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.h
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.h
@@ -54,6 +54,7 @@
 #include <linux/ntb.h>
 #include <linux/pci.h>
 
+/* PCI device IDs */
 #define PCI_DEVICE_ID_INTEL_NTB_B2B_JSF	0x3725
 #define PCI_DEVICE_ID_INTEL_NTB_PS_JSF	0x3726
 #define PCI_DEVICE_ID_INTEL_NTB_SS_JSF	0x3727
@@ -71,132 +72,7 @@
 #define PCI_DEVICE_ID_INTEL_NTB_SS_BDX	0x6F0F
 #define PCI_DEVICE_ID_INTEL_NTB_B2B_SKX	0x201C
 
-/* Intel Xeon hardware */
-
-#define XEON_PBAR23LMT_OFFSET		0x0000
-#define XEON_PBAR45LMT_OFFSET		0x0008
-#define XEON_PBAR4LMT_OFFSET		0x0008
-#define XEON_PBAR5LMT_OFFSET		0x000c
-#define XEON_PBAR23XLAT_OFFSET		0x0010
-#define XEON_PBAR45XLAT_OFFSET		0x0018
-#define XEON_PBAR4XLAT_OFFSET		0x0018
-#define XEON_PBAR5XLAT_OFFSET		0x001c
-#define XEON_SBAR23LMT_OFFSET		0x0020
-#define XEON_SBAR45LMT_OFFSET		0x0028
-#define XEON_SBAR4LMT_OFFSET		0x0028
-#define XEON_SBAR5LMT_OFFSET		0x002c
-#define XEON_SBAR23XLAT_OFFSET		0x0030
-#define XEON_SBAR45XLAT_OFFSET		0x0038
-#define XEON_SBAR4XLAT_OFFSET		0x0038
-#define XEON_SBAR5XLAT_OFFSET		0x003c
-#define XEON_SBAR0BASE_OFFSET		0x0040
-#define XEON_SBAR23BASE_OFFSET		0x0048
-#define XEON_SBAR45BASE_OFFSET		0x0050
-#define XEON_SBAR4BASE_OFFSET		0x0050
-#define XEON_SBAR5BASE_OFFSET		0x0054
-#define XEON_SBDF_OFFSET		0x005c
-#define XEON_NTBCNTL_OFFSET		0x0058
-#define XEON_PDOORBELL_OFFSET		0x0060
-#define XEON_PDBMSK_OFFSET		0x0062
-#define XEON_SDOORBELL_OFFSET		0x0064
-#define XEON_SDBMSK_OFFSET		0x0066
-#define XEON_USMEMMISS_OFFSET		0x0070
-#define XEON_SPAD_OFFSET		0x0080
-#define XEON_PBAR23SZ_OFFSET		0x00d0
-#define XEON_PBAR45SZ_OFFSET		0x00d1
-#define XEON_PBAR4SZ_OFFSET		0x00d1
-#define XEON_SBAR23SZ_OFFSET		0x00d2
-#define XEON_SBAR45SZ_OFFSET		0x00d3
-#define XEON_SBAR4SZ_OFFSET		0x00d3
-#define XEON_PPD_OFFSET			0x00d4
-#define XEON_PBAR5SZ_OFFSET		0x00d5
-#define XEON_SBAR5SZ_OFFSET		0x00d6
-#define XEON_WCCNTRL_OFFSET		0x00e0
-#define XEON_UNCERRSTS_OFFSET		0x014c
-#define XEON_CORERRSTS_OFFSET		0x0158
-#define XEON_LINK_STATUS_OFFSET		0x01a2
-#define XEON_SPCICMD_OFFSET		0x0504
-#define XEON_DEVCTRL_OFFSET		0x0598
-#define XEON_DEVSTS_OFFSET		0x059a
-#define XEON_SLINK_STATUS_OFFSET	0x05a2
-#define XEON_B2B_SPAD_OFFSET		0x0100
-#define XEON_B2B_DOORBELL_OFFSET	0x0140
-#define XEON_B2B_XLAT_OFFSETL		0x0144
-#define XEON_B2B_XLAT_OFFSETU		0x0148
-#define XEON_PPD_CONN_MASK		0x03
-#define XEON_PPD_CONN_TRANSPARENT	0x00
-#define XEON_PPD_CONN_B2B		0x01
-#define XEON_PPD_CONN_RP		0x02
-#define XEON_PPD_DEV_MASK		0x10
-#define XEON_PPD_DEV_USD		0x00
-#define XEON_PPD_DEV_DSD		0x10
-#define XEON_PPD_SPLIT_BAR_MASK		0x40
-
-#define XEON_PPD_TOPO_MASK	(XEON_PPD_CONN_MASK | XEON_PPD_DEV_MASK)
-#define XEON_PPD_TOPO_PRI_USD	(XEON_PPD_CONN_RP | XEON_PPD_DEV_USD)
-#define XEON_PPD_TOPO_PRI_DSD	(XEON_PPD_CONN_RP | XEON_PPD_DEV_DSD)
-#define XEON_PPD_TOPO_SEC_USD	(XEON_PPD_CONN_TRANSPARENT | XEON_PPD_DEV_USD)
-#define XEON_PPD_TOPO_SEC_DSD	(XEON_PPD_CONN_TRANSPARENT | XEON_PPD_DEV_DSD)
-#define XEON_PPD_TOPO_B2B_USD	(XEON_PPD_CONN_B2B | XEON_PPD_DEV_USD)
-#define XEON_PPD_TOPO_B2B_DSD	(XEON_PPD_CONN_B2B | XEON_PPD_DEV_DSD)
-
-#define XEON_MW_COUNT			2
-#define HSX_SPLIT_BAR_MW_COUNT		3
-#define XEON_DB_COUNT			15
-#define XEON_DB_LINK			15
-#define XEON_DB_LINK_BIT			BIT_ULL(XEON_DB_LINK)
-#define XEON_DB_MSIX_VECTOR_COUNT	4
-#define XEON_DB_MSIX_VECTOR_SHIFT	5
-#define XEON_DB_TOTAL_SHIFT		16
-#define XEON_SPAD_COUNT			16
-
-/* Intel Skylake Xeon hardware */
-#define SKX_IMBAR1SZ_OFFSET		0x00d0
-#define SKX_IMBAR2SZ_OFFSET		0x00d1
-#define SKX_EMBAR1SZ_OFFSET		0x00d2
-#define SKX_EMBAR2SZ_OFFSET		0x00d3
-#define SKX_DEVCTRL_OFFSET		0x0098
-#define SKX_DEVSTS_OFFSET		0x009a
-#define SKX_UNCERRSTS_OFFSET		0x014c
-#define SKX_CORERRSTS_OFFSET		0x0158
-#define SKX_LINK_STATUS_OFFSET		0x01a2
-
-#define SKX_NTBCNTL_OFFSET		0x0000
-#define SKX_IMBAR1XBASE_OFFSET		0x0010		/* SBAR2XLAT */
-#define SKX_IMBAR1XLMT_OFFSET		0x0018		/* SBAR2LMT */
-#define SKX_IMBAR2XBASE_OFFSET		0x0020		/* SBAR4XLAT */
-#define SKX_IMBAR2XLMT_OFFSET		0x0028		/* SBAR4LMT */
-#define SKX_IM_INT_STATUS_OFFSET	0x0040
-#define SKX_IM_INT_DISABLE_OFFSET	0x0048
-#define SKX_IM_SPAD_OFFSET		0x0080		/* SPAD */
-#define SKX_USMEMMISS_OFFSET		0x0070
-#define SKX_INTVEC_OFFSET		0x00d0
-#define SKX_IM_DOORBELL_OFFSET		0x0100		/* SDOORBELL0 */
-#define SKX_B2B_SPAD_OFFSET		0x0180		/* B2B SPAD */
-#define SKX_EMBAR0XBASE_OFFSET		0x4008		/* B2B_XLAT */
-#define SKX_EMBAR1XBASE_OFFSET		0x4010		/* PBAR2XLAT */
-#define SKX_EMBAR1XLMT_OFFSET		0x4018		/* PBAR2LMT */
-#define SKX_EMBAR2XBASE_OFFSET		0x4020		/* PBAR4XLAT */
-#define SKX_EMBAR2XLMT_OFFSET		0x4028		/* PBAR4LMT */
-#define SKX_EM_INT_STATUS_OFFSET	0x4040
-#define SKX_EM_INT_DISABLE_OFFSET	0x4048
-#define SKX_EM_SPAD_OFFSET		0x4080		/* remote SPAD */
-#define SKX_EM_DOORBELL_OFFSET		0x4100		/* PDOORBELL0 */
-#define SKX_SPCICMD_OFFSET		0x4504		/* SPCICMD */
-#define SKX_EMBAR0_OFFSET		0x4510		/* SBAR0BASE */
-#define SKX_EMBAR1_OFFSET		0x4518		/* SBAR23BASE */
-#define SKX_EMBAR2_OFFSET		0x4520		/* SBAR45BASE */
-
-#define SKX_DB_COUNT			32
-#define SKX_DB_LINK			32
-#define SKX_DB_LINK_BIT			BIT_ULL(SKX_DB_LINK)
-#define SKX_DB_MSIX_VECTOR_COUNT	33
-#define SKX_DB_MSIX_VECTOR_SHIFT	1
-#define SKX_DB_TOTAL_SHIFT		33
-#define SKX_SPAD_COUNT			16
-
 /* Ntb control and link status */
-
 #define NTB_CTL_CFG_LOCK		BIT(0)
 #define NTB_CTL_DISABLE			BIT(1)
 #define NTB_CTL_S2P_BAR2_SNOOP		BIT(2)
@@ -213,23 +89,6 @@
 #define NTB_LNK_STA_SPEED(x)		((x) & NTB_LNK_STA_SPEED_MASK)
 #define NTB_LNK_STA_WIDTH(x)		(((x) & NTB_LNK_STA_WIDTH_MASK) >> 4)
 
-/* Use the following addresses for translation between b2b ntb devices in case
- * the hardware default values are not reliable. */
-#define XEON_B2B_BAR0_ADDR	0x1000000000000000ull
-#define XEON_B2B_BAR2_ADDR64	0x2000000000000000ull
-#define XEON_B2B_BAR4_ADDR64	0x4000000000000000ull
-#define XEON_B2B_BAR4_ADDR32	0x20000000u
-#define XEON_B2B_BAR5_ADDR32	0x40000000u
-
-/* The peer ntb secondary config space is 32KB fixed size */
-#define XEON_B2B_MIN_SIZE		0x8000
-
-/* flags to indicate hardware errata */
-#define NTB_HWERR_SDOORBELL_LOCKUP	BIT_ULL(0)
-#define NTB_HWERR_SB01BASE_LOCKUP	BIT_ULL(1)
-#define NTB_HWERR_B2BDOORBELL_BIT14	BIT_ULL(2)
-#define NTB_HWERR_MSIX_VECTOR32_BAD	BIT_ULL(3)
-
 /* flags to indicate unsafe api */
 #define NTB_UNSAFE_DB			BIT_ULL(0)
 #define NTB_UNSAFE_SPAD			BIT_ULL(1)
@@ -328,4 +187,64 @@ struct intel_ntb_dev {
 #define hb_ndev(__work) container_of(__work, struct intel_ntb_dev, \
 				     hb_timer.work)
 
+static inline int pdev_is_gen1(struct pci_dev *pdev)
+{
+	switch (pdev->device) {
+	case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
+	case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
+		return 1;
+	}
+	return 0;
+}
+
+static inline int pdev_is_gen3(struct pci_dev *pdev)
+{
+	if (pdev->device == PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)
+		return 1;
+
+	return 0;
+}
+
+#ifndef ioread64
+#ifdef readq
+#define ioread64 readq
+#else
+#define ioread64 _ioread64
+static inline u64 _ioread64(void __iomem *mmio)
+{
+	u64 low, high;
+
+	low = ioread32(mmio);
+	high = ioread32(mmio + sizeof(u32));
+	return low | (high << 32);
+}
+#endif
+#endif
+
+#ifndef iowrite64
+#ifdef writeq
+#define iowrite64 writeq
+#else
+#define iowrite64 _iowrite64
+static inline void _iowrite64(u64 val, void __iomem *mmio)
+{
+	iowrite32(val, mmio);
+	iowrite32(val >> 32, mmio + sizeof(u32));
+}
+#endif
+#endif
+
 #endif
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 9878c48826e3..8145be34328b 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -637,7 +637,7 @@ static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
 	 */
 	node = dev_to_node(&ndev->dev);
 	for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) {
-		entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node);
+		entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
 		if (!entry)
 			return -ENOMEM;
 
@@ -1828,7 +1828,7 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
 		qp->rx_dma_chan ? "DMA" : "CPU");
 
 	for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
-		entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node);
+		entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
 		if (!entry)
 			goto err1;
 
@@ -1839,7 +1839,7 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
 	qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES;
 
 	for (i = 0; i < qp->tx_max_entry; i++) {
-		entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node);
+		entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
 		if (!entry)
 			goto err2;
 
diff --git a/drivers/nubus/bus.c b/drivers/nubus/bus.c
index d306c348c857..a59b6c4bb5b8 100644
--- a/drivers/nubus/bus.c
+++ b/drivers/nubus/bus.c
@@ -63,20 +63,15 @@ static struct device nubus_parent = {
 	.init_name	= "nubus",
 };
 
-int __init nubus_bus_register(void)
+static int __init nubus_bus_register(void)
 {
-	int err;
-
-	err = device_register(&nubus_parent);
-	if (err)
-		return err;
-
-	err = bus_register(&nubus_bus_type);
-	if (!err)
-		return 0;
+	return bus_register(&nubus_bus_type);
+}
+postcore_initcall(nubus_bus_register);
 
-	device_unregister(&nubus_parent);
-	return err;
+int __init nubus_parent_device_register(void)
+{
+	return device_register(&nubus_parent);
 }
 
 static void nubus_device_release(struct device *dev)
diff --git a/drivers/nubus/nubus.c b/drivers/nubus/nubus.c
index 4621ff98138c..bb0d63a44f41 100644
--- a/drivers/nubus/nubus.c
+++ b/drivers/nubus/nubus.c
@@ -875,7 +875,7 @@ static int __init nubus_init(void)
 		return 0;
 
 	nubus_proc_init();
-	err = nubus_bus_register();
+	err = nubus_parent_device_register();
 	if (err)
 		return err;
 	nubus_scan_bus();
diff --git a/drivers/nubus/proc.c b/drivers/nubus/proc.c
index c2e5a7e6bd3e..88e1f9a0faaf 100644
--- a/drivers/nubus/proc.c
+++ b/drivers/nubus/proc.c
@@ -45,18 +45,6 @@ nubus_devices_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int nubus_devices_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, nubus_devices_proc_show, NULL);
-}
-
-static const struct file_operations nubus_devices_proc_fops = {
-	.open		= nubus_devices_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static struct proc_dir_entry *proc_bus_nubus_dir;
 
 /*
@@ -149,18 +137,6 @@ static int nubus_proc_rsrc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int nubus_proc_rsrc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, nubus_proc_rsrc_show, inode);
-}
-
-static const struct file_operations nubus_proc_rsrc_fops = {
-	.open		= nubus_proc_rsrc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 void nubus_proc_add_rsrc_mem(struct proc_dir_entry *procdir,
 			     const struct nubus_dirent *ent,
 			     unsigned int size)
@@ -176,8 +152,8 @@ void nubus_proc_add_rsrc_mem(struct proc_dir_entry *procdir,
 		pde_data = nubus_proc_alloc_pde_data(nubus_dirptr(ent), size);
 	else
 		pde_data = NULL;
-	proc_create_data(name, S_IFREG | 0444, procdir,
-			 &nubus_proc_rsrc_fops, pde_data);
+	proc_create_single_data(name, S_IFREG | 0444, procdir,
+			nubus_proc_rsrc_show, pde_data);
 }
 
 void nubus_proc_add_rsrc(struct proc_dir_entry *procdir,
@@ -190,32 +166,21 @@ void nubus_proc_add_rsrc(struct proc_dir_entry *procdir,
 		return;
 
 	snprintf(name, sizeof(name), "%x", ent->type);
-	proc_create_data(name, S_IFREG | 0444, procdir,
-			 &nubus_proc_rsrc_fops,
-			 nubus_proc_alloc_pde_data(data, 0));
+	proc_create_single_data(name, S_IFREG | 0444, procdir,
+			nubus_proc_rsrc_show,
+			nubus_proc_alloc_pde_data(data, 0));
 }
 
 /*
  * /proc/nubus stuff
  */
 
-static int nubus_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, nubus_proc_show, NULL);
-}
-
-static const struct file_operations nubus_proc_fops = {
-	.open		= nubus_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 void __init nubus_proc_init(void)
 {
-	proc_create("nubus", 0, NULL, &nubus_proc_fops);
+	proc_create_single("nubus", 0, NULL, nubus_proc_show);
 	proc_bus_nubus_dir = proc_mkdir("bus/nubus", NULL);
 	if (!proc_bus_nubus_dir)
 		return;
-	proc_create("devices", 0, proc_bus_nubus_dir, &nubus_devices_proc_fops);
+	proc_create_single("devices", 0, proc_bus_nubus_dir,
+			nubus_devices_proc_show);
 }
diff --git a/drivers/nvdimm/claim.c b/drivers/nvdimm/claim.c
index 30852270484f..2e96b34bc936 100644
--- a/drivers/nvdimm/claim.c
+++ b/drivers/nvdimm/claim.c
@@ -276,7 +276,8 @@ static int nsio_rw_bytes(struct nd_namespace_common *ndns,
 	if (rw == READ) {
 		if (unlikely(is_bad_pmem(&nsio->bb, sector, sz_align)))
 			return -EIO;
-		return memcpy_mcsafe(buf, nsio->addr + offset, size);
+		if (memcpy_mcsafe(buf, nsio->addr + offset, size) != 0)
+			return -EIO;
 	}
 
 	if (unlikely(is_bad_pmem(&nsio->bb, sector, sz_align))) {
diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c
index 30b08791597d..3f7ad5bc443e 100644
--- a/drivers/nvdimm/pfn_devs.c
+++ b/drivers/nvdimm/pfn_devs.c
@@ -561,8 +561,6 @@ static int __nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap)
 	res->start += start_pad;
 	res->end -= end_trunc;
 
-	pgmap->type = MEMORY_DEVICE_HOST;
-
 	if (nd_pfn->mode == PFN_MODE_RAM) {
 		if (offset < SZ_8K)
 			return -EINVAL;
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index 9d714926ecf5..47219fe7e3b7 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -101,15 +101,15 @@ static blk_status_t read_pmem(struct page *page, unsigned int off,
 		void *pmem_addr, unsigned int len)
 {
 	unsigned int chunk;
-	int rc;
+	unsigned long rem;
 	void *mem;
 
 	while (len) {
 		mem = kmap_atomic(page);
 		chunk = min_t(unsigned int, len, PAGE_SIZE);
-		rc = memcpy_mcsafe(mem + off, pmem_addr, chunk);
+		rem = memcpy_mcsafe(mem + off, pmem_addr, chunk);
 		kunmap_atomic(mem);
-		if (rc)
+		if (rem)
 			return BLK_STS_IOERR;
 		len -= chunk;
 		off = 0;
@@ -264,9 +264,16 @@ static size_t pmem_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
 	return copy_from_iter_flushcache(addr, bytes, i);
 }
 
+static size_t pmem_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,
+		void *addr, size_t bytes, struct iov_iter *i)
+{
+	return copy_to_iter_mcsafe(addr, bytes, i);
+}
+
 static const struct dax_operations pmem_dax_ops = {
 	.direct_access = pmem_dax_direct_access,
 	.copy_from_iter = pmem_copy_from_iter,
+	.copy_to_iter = pmem_copy_to_iter,
 };
 
 static const struct attribute_group *pmem_attribute_groups[] = {
@@ -294,6 +301,27 @@ static void pmem_release_disk(void *__pmem)
 	put_disk(pmem->disk);
 }
 
+static void pmem_release_pgmap_ops(void *__pgmap)
+{
+	dev_pagemap_put_ops();
+}
+
+static void fsdax_pagefree(struct page *page, void *data)
+{
+	wake_up_var(&page->_refcount);
+}
+
+static int setup_pagemap_fsdax(struct device *dev, struct dev_pagemap *pgmap)
+{
+	dev_pagemap_get_ops();
+	if (devm_add_action_or_reset(dev, pmem_release_pgmap_ops, pgmap))
+		return -ENOMEM;
+	pgmap->type = MEMORY_DEVICE_FS_DAX;
+	pgmap->page_free = fsdax_pagefree;
+
+	return 0;
+}
+
 static int pmem_attach_disk(struct device *dev,
 		struct nd_namespace_common *ndns)
 {
@@ -353,6 +381,8 @@ static int pmem_attach_disk(struct device *dev,
 	pmem->pfn_flags = PFN_DEV;
 	pmem->pgmap.ref = &q->q_usage_counter;
 	if (is_nd_pfn(dev)) {
+		if (setup_pagemap_fsdax(dev, &pmem->pgmap))
+			return -ENOMEM;
 		addr = devm_memremap_pages(dev, &pmem->pgmap);
 		pfn_sb = nd_pfn->pfn_sb;
 		pmem->data_offset = le64_to_cpu(pfn_sb->dataoff);
@@ -364,6 +394,8 @@ static int pmem_attach_disk(struct device *dev,
 	} else if (pmem_should_map_pages(dev)) {
 		memcpy(&pmem->pgmap.res, &nsio->res, sizeof(pmem->pgmap.res));
 		pmem->pgmap.altmap_valid = false;
+		if (setup_pagemap_fsdax(dev, &pmem->pgmap))
+			return -ENOMEM;
 		addr = devm_memremap_pages(dev, &pmem->pgmap);
 		pmem->pfn_flags |= PFN_MAP;
 		memcpy(&bb_res, &pmem->pgmap.res, sizeof(bb_res));
diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
index dbb7464c018c..88a8b5916624 100644
--- a/drivers/nvme/host/Kconfig
+++ b/drivers/nvme/host/Kconfig
@@ -27,7 +27,7 @@ config NVME_FABRICS
 
 config NVME_RDMA
 	tristate "NVM Express over Fabrics RDMA host driver"
-	depends on INFINIBAND_ADDR_TRANS && BLOCK
+	depends on INFINIBAND && INFINIBAND_ADDR_TRANS && BLOCK
 	select NVME_CORE
 	select NVME_FABRICS
 	select SG_POOL
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 99b857e5a7a9..cf8fd0113c88 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -244,9 +244,6 @@ EXPORT_SYMBOL_GPL(nvme_complete_rq);
 
 void nvme_cancel_request(struct request *req, void *data, bool reserved)
 {
-	if (!blk_mq_request_started(req))
-		return;
-
 	dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
 				"Cancelling I/O %d", req->tag);
 
@@ -351,7 +348,7 @@ static void nvme_free_ns_head(struct kref *ref)
 	nvme_mpath_remove_disk(head);
 	ida_simple_remove(&head->subsys->ns_ida, head->instance);
 	list_del_init(&head->entry);
-	cleanup_srcu_struct(&head->srcu);
+	cleanup_srcu_struct_quiesced(&head->srcu);
 	nvme_put_subsystem(head->subsys);
 	kfree(head);
 }
@@ -1351,13 +1348,19 @@ static void nvme_set_chunk_size(struct nvme_ns *ns)
 	blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(chunk_size));
 }
 
-static void nvme_config_discard(struct nvme_ctrl *ctrl,
-		unsigned stream_alignment, struct request_queue *queue)
+static void nvme_config_discard(struct nvme_ns *ns)
 {
+	struct nvme_ctrl *ctrl = ns->ctrl;
+	struct request_queue *queue = ns->queue;
 	u32 size = queue_logical_block_size(queue);
 
-	if (stream_alignment)
-		size *= stream_alignment;
+	if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
+		blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
+		return;
+	}
+
+	if (ctrl->nr_streams && ns->sws && ns->sgs)
+		size *= ns->sws * ns->sgs;
 
 	BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
 			NVME_DSM_MAX_RANGES);
@@ -1365,9 +1368,12 @@ static void nvme_config_discard(struct nvme_ctrl *ctrl,
 	queue->limits.discard_alignment = 0;
 	queue->limits.discard_granularity = size;
 
+	/* If discard is already enabled, don't reset queue limits */
+	if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
+		return;
+
 	blk_queue_max_discard_sectors(queue, UINT_MAX);
 	blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
-	blk_queue_flag_set(QUEUE_FLAG_DISCARD, queue);
 
 	if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
 		blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
@@ -1411,10 +1417,6 @@ static void nvme_update_disk_info(struct gendisk *disk,
 {
 	sector_t capacity = le64_to_cpup(&id->nsze) << (ns->lba_shift - 9);
 	unsigned short bs = 1 << ns->lba_shift;
-	unsigned stream_alignment = 0;
-
-	if (ns->ctrl->nr_streams && ns->sws && ns->sgs)
-		stream_alignment = ns->sws * ns->sgs;
 
 	blk_mq_freeze_queue(disk->queue);
 	blk_integrity_unregister(disk);
@@ -1428,10 +1430,9 @@ static void nvme_update_disk_info(struct gendisk *disk,
 		nvme_init_integrity(disk, ns->ms, ns->pi_type);
 	if (ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk))
 		capacity = 0;
-	set_capacity(disk, capacity);
 
-	if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)
-		nvme_config_discard(ns->ctrl, stream_alignment, disk->queue);
+	set_capacity(disk, capacity);
+	nvme_config_discard(ns);
 	blk_mq_unfreeze_queue(disk->queue);
 }
 
@@ -1577,7 +1578,7 @@ static int nvme_pr_reserve(struct block_device *bdev, u64 key,
 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
 		enum pr_type type, bool abort)
 {
-	u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
+	u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
 	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
 }
 
@@ -1589,7 +1590,7 @@ static int nvme_pr_clear(struct block_device *bdev, u64 key)
 
 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
 {
-	u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
+	u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
 	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
 }
 
@@ -2183,7 +2184,8 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 		 * Verify that the subsystem actually supports multiple
 		 * controllers, else bail out.
 		 */
-		if (nvme_active_ctrls(found) && !(id->cmic & (1 << 1))) {
+		if (!ctrl->opts->discovery_nqn &&
+		    nvme_active_ctrls(found) && !(id->cmic & (1 << 1))) {
 			dev_err(ctrl->device,
 				"ignoring ctrl due to duplicate subnqn (%s).\n",
 				found->subnqn);
@@ -2314,7 +2316,7 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
 	if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
 		ret = nvme_get_effects_log(ctrl);
 		if (ret < 0)
-			return ret;
+			goto out_free;
 	}
 
 	if (!ctrl->identified) {
@@ -3323,7 +3325,7 @@ static void nvme_fw_act_work(struct work_struct *work)
 }
 
 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
-		union nvme_result *res)
+		volatile union nvme_result *res)
 {
 	u32 result = le32_to_cpu(res->u32);
 
diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c
index 7ae732a77fe8..aa318136460e 100644
--- a/drivers/nvme/host/fabrics.c
+++ b/drivers/nvme/host/fabrics.c
@@ -57,7 +57,7 @@ static struct nvmf_host *nvmf_host_add(const char *hostnqn)
 		goto out_unlock;
 
 	kref_init(&host->ref);
-	memcpy(host->nqn, hostnqn, NVMF_NQN_SIZE);
+	strlcpy(host->nqn, hostnqn, NVMF_NQN_SIZE);
 
 	list_add_tail(&host->list, &nvmf_hosts);
 out_unlock:
@@ -689,10 +689,6 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 			opts->discovery_nqn =
 				!(strcmp(opts->subsysnqn,
 					 NVME_DISC_SUBSYS_NAME));
-			if (opts->discovery_nqn) {
-				opts->kato = 0;
-				opts->nr_io_queues = 0;
-			}
 			break;
 		case NVMF_OPT_TRADDR:
 			p = match_strdup(args);
@@ -851,6 +847,11 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 		}
 	}
 
+	if (opts->discovery_nqn) {
+		opts->kato = 0;
+		opts->nr_io_queues = 0;
+		opts->duplicate_connect = true;
+	}
 	if (ctrl_loss_tmo < 0)
 		opts->max_reconnects = -1;
 	else
@@ -983,16 +984,6 @@ nvmf_create_ctrl(struct device *dev, const char *buf, size_t count)
 		goto out_module_put;
 	}
 
-	if (strcmp(ctrl->subsys->subnqn, opts->subsysnqn)) {
-		dev_warn(ctrl->device,
-			"controller returned incorrect NQN: \"%s\".\n",
-			ctrl->subsys->subnqn);
-		module_put(ops->module);
-		up_read(&nvmf_transports_rwsem);
-		nvme_delete_ctrl_sync(ctrl);
-		return ERR_PTR(-EINVAL);
-	}
-
 	module_put(ops->module);
 	up_read(&nvmf_transports_rwsem);
 	return ctrl;
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index 6cb26bcf6ec0..0bad65803271 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -1686,16 +1686,6 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
 		goto check_error;
 	}
 
-	/*
-	 * Force failures of commands if we're killing the controller
-	 * or have an error on a command used to create an new association
-	 */
-	if (status &&
-	    (blk_queue_dying(rq->q) ||
-	     ctrl->ctrl.state == NVME_CTRL_NEW ||
-	     ctrl->ctrl.state == NVME_CTRL_CONNECTING))
-		status |= cpu_to_le16(NVME_SC_DNR << 1);
-
 	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
 	nvme_end_request(rq, status, result);
 
@@ -2403,9 +2393,6 @@ nvme_fc_terminate_exchange(struct request *req, void *data, bool reserved)
 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req);
 
-	if (!blk_mq_request_started(req))
-		return;
-
 	__nvme_fc_abort_op(ctrl, op);
 }
 
@@ -3284,6 +3271,8 @@ nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
 	}
 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
 
+	pr_warn("%s: %s - %s combination not found\n",
+		__func__, opts->traddr, opts->host_traddr);
 	return ERR_PTR(-ENOENT);
 }
 
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 17d2f7cf3fed..1bbd7ea52330 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -22,6 +22,7 @@
 #include <linux/lightnvm.h>
 #include <linux/sed-opal.h>
 #include <linux/fault-inject.h>
+#include <linux/rcupdate.h>
 
 extern unsigned int nvme_io_timeout;
 #define NVME_IO_TIMEOUT	(nvme_io_timeout * HZ)
@@ -405,7 +406,7 @@ int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
 		bool send);
 
 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
-		union nvme_result *res);
+		volatile union nvme_result *res);
 
 void nvme_stop_queues(struct nvme_ctrl *ctrl);
 void nvme_start_queues(struct nvme_ctrl *ctrl);
@@ -454,7 +455,7 @@ static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
 {
 	struct nvme_ns_head *head = ns->head;
 
-	if (head && ns == srcu_dereference(head->current_path, &head->srcu))
+	if (head && ns == rcu_access_pointer(head->current_path))
 		rcu_assign_pointer(head->current_path, NULL);
 }
 struct nvme_ns *nvme_find_path(struct nvme_ns_head *head);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 17a0190bd88f..ad3eb7d55712 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -13,6 +13,7 @@
  */
 
 #include <linux/aer.h>
+#include <linux/async.h>
 #include <linux/blkdev.h>
 #include <linux/blk-mq.h>
 #include <linux/blk-mq-pci.h>
@@ -68,7 +69,6 @@ MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2");
 struct nvme_dev;
 struct nvme_queue;
 
-static void nvme_process_cq(struct nvme_queue *nvmeq);
 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
 
 /*
@@ -147,9 +147,10 @@ static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
 struct nvme_queue {
 	struct device *q_dmadev;
 	struct nvme_dev *dev;
-	spinlock_t q_lock;
+	spinlock_t sq_lock;
 	struct nvme_command *sq_cmds;
 	struct nvme_command __iomem *sq_cmds_io;
+	spinlock_t cq_lock ____cacheline_aligned_in_smp;
 	volatile struct nvme_completion *cqes;
 	struct blk_mq_tags **tags;
 	dma_addr_t sq_dma_addr;
@@ -159,9 +160,9 @@ struct nvme_queue {
 	s16 cq_vector;
 	u16 sq_tail;
 	u16 cq_head;
+	u16 last_cq_head;
 	u16 qid;
 	u8 cq_phase;
-	u8 cqe_seen;
 	u32 *dbbuf_sq_db;
 	u32 *dbbuf_cq_db;
 	u32 *dbbuf_sq_ei;
@@ -872,6 +873,13 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 	struct nvme_command cmnd;
 	blk_status_t ret;
 
+	/*
+	 * We should not need to do this, but we're still using this to
+	 * ensure we can drain requests on a dying queue.
+	 */
+	if (unlikely(nvmeq->cq_vector < 0))
+		return BLK_STS_IOERR;
+
 	ret = nvme_setup_cmd(ns, req, &cmnd);
 	if (ret)
 		return ret;
@@ -888,15 +896,9 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 
 	blk_mq_start_request(req);
 
-	spin_lock_irq(&nvmeq->q_lock);
-	if (unlikely(nvmeq->cq_vector < 0)) {
-		ret = BLK_STS_IOERR;
-		spin_unlock_irq(&nvmeq->q_lock);
-		goto out_cleanup_iod;
-	}
+	spin_lock(&nvmeq->sq_lock);
 	__nvme_submit_cmd(nvmeq, &cmnd);
-	nvme_process_cq(nvmeq);
-	spin_unlock_irq(&nvmeq->q_lock);
+	spin_unlock(&nvmeq->sq_lock);
 	return BLK_STS_OK;
 out_cleanup_iod:
 	nvme_free_iod(dev, req);
@@ -914,10 +916,10 @@ static void nvme_pci_complete_rq(struct request *req)
 }
 
 /* We read the CQE phase first to check if the rest of the entry is valid */
-static inline bool nvme_cqe_valid(struct nvme_queue *nvmeq, u16 head,
-		u16 phase)
+static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
 {
-	return (le16_to_cpu(nvmeq->cqes[head].status) & 1) == phase;
+	return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
+			nvmeq->cq_phase;
 }
 
 static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
@@ -931,9 +933,9 @@ static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
 	}
 }
 
-static inline void nvme_handle_cqe(struct nvme_queue *nvmeq,
-		struct nvme_completion *cqe)
+static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx)
 {
+	volatile struct nvme_completion *cqe = &nvmeq->cqes[idx];
 	struct request *req;
 
 	if (unlikely(cqe->command_id >= nvmeq->q_depth)) {
@@ -956,83 +958,87 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq,
 		return;
 	}
 
-	nvmeq->cqe_seen = 1;
 	req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id);
 	nvme_end_request(req, cqe->status, cqe->result);
 }
 
-static inline bool nvme_read_cqe(struct nvme_queue *nvmeq,
-		struct nvme_completion *cqe)
+static void nvme_complete_cqes(struct nvme_queue *nvmeq, u16 start, u16 end)
 {
-	if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase)) {
-		*cqe = nvmeq->cqes[nvmeq->cq_head];
+	while (start != end) {
+		nvme_handle_cqe(nvmeq, start);
+		if (++start == nvmeq->q_depth)
+			start = 0;
+	}
+}
 
-		if (++nvmeq->cq_head == nvmeq->q_depth) {
-			nvmeq->cq_head = 0;
-			nvmeq->cq_phase = !nvmeq->cq_phase;
-		}
-		return true;
+static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
+{
+	if (++nvmeq->cq_head == nvmeq->q_depth) {
+		nvmeq->cq_head = 0;
+		nvmeq->cq_phase = !nvmeq->cq_phase;
 	}
-	return false;
 }
 
-static void nvme_process_cq(struct nvme_queue *nvmeq)
+static inline bool nvme_process_cq(struct nvme_queue *nvmeq, u16 *start,
+		u16 *end, int tag)
 {
-	struct nvme_completion cqe;
-	int consumed = 0;
+	bool found = false;
 
-	while (nvme_read_cqe(nvmeq, &cqe)) {
-		nvme_handle_cqe(nvmeq, &cqe);
-		consumed++;
+	*start = nvmeq->cq_head;
+	while (!found && nvme_cqe_pending(nvmeq)) {
+		if (nvmeq->cqes[nvmeq->cq_head].command_id == tag)
+			found = true;
+		nvme_update_cq_head(nvmeq);
 	}
+	*end = nvmeq->cq_head;
 
-	if (consumed)
+	if (*start != *end)
 		nvme_ring_cq_doorbell(nvmeq);
+	return found;
 }
 
 static irqreturn_t nvme_irq(int irq, void *data)
 {
-	irqreturn_t result;
 	struct nvme_queue *nvmeq = data;
-	spin_lock(&nvmeq->q_lock);
-	nvme_process_cq(nvmeq);
-	result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
-	nvmeq->cqe_seen = 0;
-	spin_unlock(&nvmeq->q_lock);
-	return result;
+	irqreturn_t ret = IRQ_NONE;
+	u16 start, end;
+
+	spin_lock(&nvmeq->cq_lock);
+	if (nvmeq->cq_head != nvmeq->last_cq_head)
+		ret = IRQ_HANDLED;
+	nvme_process_cq(nvmeq, &start, &end, -1);
+	nvmeq->last_cq_head = nvmeq->cq_head;
+	spin_unlock(&nvmeq->cq_lock);
+
+	if (start != end) {
+		nvme_complete_cqes(nvmeq, start, end);
+		return IRQ_HANDLED;
+	}
+
+	return ret;
 }
 
 static irqreturn_t nvme_irq_check(int irq, void *data)
 {
 	struct nvme_queue *nvmeq = data;
-	if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase))
+	if (nvme_cqe_pending(nvmeq))
 		return IRQ_WAKE_THREAD;
 	return IRQ_NONE;
 }
 
 static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag)
 {
-	struct nvme_completion cqe;
-	int found = 0, consumed = 0;
+	u16 start, end;
+	bool found;
 
-	if (!nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase))
+	if (!nvme_cqe_pending(nvmeq))
 		return 0;
 
-	spin_lock_irq(&nvmeq->q_lock);
-	while (nvme_read_cqe(nvmeq, &cqe)) {
-		nvme_handle_cqe(nvmeq, &cqe);
-		consumed++;
-
-		if (tag == cqe.command_id) {
-			found = 1;
-			break;
-		}
-       }
-
-	if (consumed)
-		nvme_ring_cq_doorbell(nvmeq);
-	spin_unlock_irq(&nvmeq->q_lock);
+	spin_lock_irq(&nvmeq->cq_lock);
+	found = nvme_process_cq(nvmeq, &start, &end, tag);
+	spin_unlock_irq(&nvmeq->cq_lock);
 
+	nvme_complete_cqes(nvmeq, start, end);
 	return found;
 }
 
@@ -1053,9 +1059,9 @@ static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl)
 	c.common.opcode = nvme_admin_async_event;
 	c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
 
-	spin_lock_irq(&nvmeq->q_lock);
+	spin_lock(&nvmeq->sq_lock);
 	__nvme_submit_cmd(nvmeq, &c);
-	spin_unlock_irq(&nvmeq->q_lock);
+	spin_unlock(&nvmeq->sq_lock);
 }
 
 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
@@ -1070,7 +1076,7 @@ static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
 }
 
 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
-						struct nvme_queue *nvmeq)
+		struct nvme_queue *nvmeq, s16 vector)
 {
 	struct nvme_command c;
 	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
@@ -1085,7 +1091,7 @@ static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
 	c.create_cq.cqid = cpu_to_le16(qid);
 	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 	c.create_cq.cq_flags = cpu_to_le16(flags);
-	c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
+	c.create_cq.irq_vector = cpu_to_le16(vector);
 
 	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
 }
@@ -1208,7 +1214,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
 		nvme_warn_reset(dev, csts);
 		nvme_dev_disable(dev, false);
 		nvme_reset_ctrl(&dev->ctrl);
-		return BLK_EH_HANDLED;
+		return BLK_EH_DONE;
 	}
 
 	/*
@@ -1218,14 +1224,14 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
 		dev_warn(dev->ctrl.device,
 			 "I/O %d QID %d timeout, completion polled\n",
 			 req->tag, nvmeq->qid);
-		return BLK_EH_HANDLED;
+		return BLK_EH_DONE;
 	}
 
 	/*
 	 * Shutdown immediately if controller times out while starting. The
 	 * reset work will see the pci device disabled when it gets the forced
 	 * cancellation error. All outstanding requests are completed on
-	 * shutdown, so we return BLK_EH_HANDLED.
+	 * shutdown, so we return BLK_EH_DONE.
 	 */
 	switch (dev->ctrl.state) {
 	case NVME_CTRL_CONNECTING:
@@ -1235,7 +1241,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
 			 req->tag, nvmeq->qid);
 		nvme_dev_disable(dev, false);
 		nvme_req(req)->flags |= NVME_REQ_CANCELLED;
-		return BLK_EH_HANDLED;
+		return BLK_EH_DONE;
 	default:
 		break;
 	}
@@ -1252,12 +1258,8 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
 		nvme_dev_disable(dev, false);
 		nvme_reset_ctrl(&dev->ctrl);
 
-		/*
-		 * Mark the request as handled, since the inline shutdown
-		 * forces all outstanding requests to complete.
-		 */
 		nvme_req(req)->flags |= NVME_REQ_CANCELLED;
-		return BLK_EH_HANDLED;
+		return BLK_EH_DONE;
 	}
 
 	if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
@@ -1321,15 +1323,21 @@ static int nvme_suspend_queue(struct nvme_queue *nvmeq)
 {
 	int vector;
 
-	spin_lock_irq(&nvmeq->q_lock);
+	spin_lock_irq(&nvmeq->cq_lock);
 	if (nvmeq->cq_vector == -1) {
-		spin_unlock_irq(&nvmeq->q_lock);
+		spin_unlock_irq(&nvmeq->cq_lock);
 		return 1;
 	}
 	vector = nvmeq->cq_vector;
 	nvmeq->dev->online_queues--;
 	nvmeq->cq_vector = -1;
-	spin_unlock_irq(&nvmeq->q_lock);
+	spin_unlock_irq(&nvmeq->cq_lock);
+
+	/*
+	 * Ensure that nvme_queue_rq() sees it ->cq_vector == -1 without
+	 * having to grab the lock.
+	 */
+	mb();
 
 	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
 		blk_mq_quiesce_queue(nvmeq->dev->ctrl.admin_q);
@@ -1342,15 +1350,18 @@ static int nvme_suspend_queue(struct nvme_queue *nvmeq)
 static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
 {
 	struct nvme_queue *nvmeq = &dev->queues[0];
+	u16 start, end;
 
 	if (shutdown)
 		nvme_shutdown_ctrl(&dev->ctrl);
 	else
 		nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);
 
-	spin_lock_irq(&nvmeq->q_lock);
-	nvme_process_cq(nvmeq);
-	spin_unlock_irq(&nvmeq->q_lock);
+	spin_lock_irq(&nvmeq->cq_lock);
+	nvme_process_cq(nvmeq, &start, &end, -1);
+	spin_unlock_irq(&nvmeq->cq_lock);
+
+	nvme_complete_cqes(nvmeq, start, end);
 }
 
 static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
@@ -1408,7 +1419,8 @@ static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
 
 	nvmeq->q_dmadev = dev->dev;
 	nvmeq->dev = dev;
-	spin_lock_init(&nvmeq->q_lock);
+	spin_lock_init(&nvmeq->sq_lock);
+	spin_lock_init(&nvmeq->cq_lock);
 	nvmeq->cq_head = 0;
 	nvmeq->cq_phase = 1;
 	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
@@ -1444,7 +1456,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
 {
 	struct nvme_dev *dev = nvmeq->dev;
 
-	spin_lock_irq(&nvmeq->q_lock);
+	spin_lock_irq(&nvmeq->cq_lock);
 	nvmeq->sq_tail = 0;
 	nvmeq->cq_head = 0;
 	nvmeq->cq_phase = 1;
@@ -1452,13 +1464,14 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
 	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
 	nvme_dbbuf_init(dev, nvmeq, qid);
 	dev->online_queues++;
-	spin_unlock_irq(&nvmeq->q_lock);
+	spin_unlock_irq(&nvmeq->cq_lock);
 }
 
 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
 {
 	struct nvme_dev *dev = nvmeq->dev;
 	int result;
+	s16 vector;
 
 	if (dev->cmb && use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) {
 		unsigned offset = (qid - 1) * roundup(SQ_SIZE(nvmeq->q_depth),
@@ -1471,15 +1484,21 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
 	 * A queue's vector matches the queue identifier unless the controller
 	 * has only one vector available.
 	 */
-	nvmeq->cq_vector = dev->num_vecs == 1 ? 0 : qid;
-	result = adapter_alloc_cq(dev, qid, nvmeq);
+	vector = dev->num_vecs == 1 ? 0 : qid;
+	result = adapter_alloc_cq(dev, qid, nvmeq, vector);
 	if (result < 0)
-		goto release_vector;
+		goto out;
 
 	result = adapter_alloc_sq(dev, qid, nvmeq);
 	if (result < 0)
 		goto release_cq;
 
+	/*
+	 * Set cq_vector after alloc cq/sq, otherwise nvme_suspend_queue will
+	 * invoke free_irq for it and cause a 'Trying to free already-free IRQ
+	 * xxx' warning if the create CQ/SQ command times out.
+	 */
+	nvmeq->cq_vector = vector;
 	nvme_init_queue(nvmeq, qid);
 	result = queue_request_irq(nvmeq);
 	if (result < 0)
@@ -1487,13 +1506,13 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
 
 	return result;
 
- release_sq:
+release_sq:
+	nvmeq->cq_vector = -1;
 	dev->online_queues--;
 	adapter_delete_sq(dev, qid);
- release_cq:
+release_cq:
 	adapter_delete_cq(dev, qid);
- release_vector:
-	nvmeq->cq_vector = -1;
+out:
 	return result;
 }
 
@@ -1997,19 +2016,22 @@ static void nvme_del_queue_end(struct request *req, blk_status_t error)
 static void nvme_del_cq_end(struct request *req, blk_status_t error)
 {
 	struct nvme_queue *nvmeq = req->end_io_data;
+	u16 start, end;
 
 	if (!error) {
 		unsigned long flags;
 
 		/*
-		 * We might be called with the AQ q_lock held
-		 * and the I/O queue q_lock should always
+		 * We might be called with the AQ cq_lock held
+		 * and the I/O queue cq_lock should always
 		 * nest inside the AQ one.
 		 */
-		spin_lock_irqsave_nested(&nvmeq->q_lock, flags,
+		spin_lock_irqsave_nested(&nvmeq->cq_lock, flags,
 					SINGLE_DEPTH_NESTING);
-		nvme_process_cq(nvmeq);
-		spin_unlock_irqrestore(&nvmeq->q_lock, flags);
+		nvme_process_cq(nvmeq, &start, &end, -1);
+		spin_unlock_irqrestore(&nvmeq->cq_lock, flags);
+
+		nvme_complete_cqes(nvmeq, start, end);
 	}
 
 	nvme_del_queue_end(req, error);
@@ -2497,6 +2519,15 @@ static unsigned long check_vendor_combination_bug(struct pci_dev *pdev)
 	return 0;
 }
 
+static void nvme_async_probe(void *data, async_cookie_t cookie)
+{
+	struct nvme_dev *dev = data;
+
+	nvme_reset_ctrl_sync(&dev->ctrl);
+	flush_work(&dev->ctrl.scan_work);
+	nvme_put_ctrl(&dev->ctrl);
+}
+
 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	int node, result = -ENOMEM;
@@ -2541,7 +2572,8 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));
 
-	nvme_reset_ctrl(&dev->ctrl);
+	nvme_get_ctrl(&dev->ctrl);
+	async_schedule(nvme_async_probe, dev);
 
 	return 0;
 
@@ -2607,24 +2639,6 @@ static void nvme_remove(struct pci_dev *pdev)
 	nvme_put_ctrl(&dev->ctrl);
 }
 
-static int nvme_pci_sriov_configure(struct pci_dev *pdev, int numvfs)
-{
-	int ret = 0;
-
-	if (numvfs == 0) {
-		if (pci_vfs_assigned(pdev)) {
-			dev_warn(&pdev->dev,
-				"Cannot disable SR-IOV VFs while assigned\n");
-			return -EPERM;
-		}
-		pci_disable_sriov(pdev);
-		return 0;
-	}
-
-	ret = pci_enable_sriov(pdev, numvfs);
-	return ret ? ret : numvfs;
-}
-
 #ifdef CONFIG_PM_SLEEP
 static int nvme_suspend(struct device *dev)
 {
@@ -2685,6 +2699,9 @@ static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
 
 static void nvme_error_resume(struct pci_dev *pdev)
 {
+	struct nvme_dev *dev = pci_get_drvdata(pdev);
+
+	flush_work(&dev->ctrl.reset_work);
 	pci_cleanup_aer_uncorrect_error_status(pdev);
 }
 
@@ -2714,6 +2731,8 @@ static const struct pci_device_id nvme_id_table[] = {
 				NVME_QUIRK_MEDIUM_PRIO_SQ },
 	{ PCI_VDEVICE(INTEL, 0x5845),	/* Qemu emulated controller */
 		.driver_data = NVME_QUIRK_IDENTIFY_CNS, },
+	{ PCI_DEVICE(0x1bb1, 0x0100),   /* Seagate Nytro Flash Storage */
+		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
 	{ PCI_DEVICE(0x1c58, 0x0003),	/* HGST adapter */
 		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
 	{ PCI_DEVICE(0x1c58, 0x0023),	/* WDC SN200 adapter */
@@ -2728,6 +2747,8 @@ static const struct pci_device_id nvme_id_table[] = {
 		.driver_data = NVME_QUIRK_LIGHTNVM, },
 	{ PCI_DEVICE(0x1d1d, 0x2807),	/* CNEX WL */
 		.driver_data = NVME_QUIRK_LIGHTNVM, },
+	{ PCI_DEVICE(0x1d1d, 0x2601),	/* CNEX Granby */
+		.driver_data = NVME_QUIRK_LIGHTNVM, },
 	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
@@ -2744,7 +2765,7 @@ static struct pci_driver nvme_driver = {
 	.driver		= {
 		.pm	= &nvme_dev_pm_ops,
 	},
-	.sriov_configure = nvme_pci_sriov_configure,
+	.sriov_configure = pci_sriov_configure_simple,
 	.err_handler	= &nvme_err_handler,
 };
 
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 1eb4438a8763..7b3f08410430 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -778,7 +778,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
 	if (error) {
 		dev_err(ctrl->ctrl.device,
 			"prop_get NVME_REG_CAP failed\n");
-		goto out_cleanup_queue;
+		goto out_stop_queue;
 	}
 
 	ctrl->ctrl.sqsize =
@@ -786,23 +786,25 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
 
 	error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
 	if (error)
-		goto out_cleanup_queue;
+		goto out_stop_queue;
 
 	ctrl->ctrl.max_hw_sectors =
 		(ctrl->max_fr_pages - 1) << (ilog2(SZ_4K) - 9);
 
 	error = nvme_init_identify(&ctrl->ctrl);
 	if (error)
-		goto out_cleanup_queue;
+		goto out_stop_queue;
 
 	error = nvme_rdma_alloc_qe(ctrl->queues[0].device->dev,
 			&ctrl->async_event_sqe, sizeof(struct nvme_command),
 			DMA_TO_DEVICE);
 	if (error)
-		goto out_cleanup_queue;
+		goto out_stop_queue;
 
 	return 0;
 
+out_stop_queue:
+	nvme_rdma_stop_queue(&ctrl->queues[0]);
 out_cleanup_queue:
 	if (new)
 		blk_cleanup_queue(ctrl->ctrl.admin_q);
@@ -1598,7 +1600,7 @@ nvme_rdma_timeout(struct request *rq, bool reserved)
 	/* fail with DNR on cmd timeout */
 	nvme_req(rq)->status = NVME_SC_ABORT_REQ | NVME_SC_DNR;
 
-	return BLK_EH_HANDLED;
+	return BLK_EH_DONE;
 }
 
 static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
diff --git a/drivers/nvme/host/trace.h b/drivers/nvme/host/trace.h
index ea91fccd1bc0..01390f0e1671 100644
--- a/drivers/nvme/host/trace.h
+++ b/drivers/nvme/host/trace.h
@@ -148,8 +148,8 @@ TRACE_EVENT(nvme_complete_rq,
 		    __entry->flags = nvme_req(req)->flags;
 		    __entry->status = nvme_req(req)->status;
 	    ),
-	    TP_printk("cmdid=%u, qid=%d, res=%llu, retries=%u, flags=0x%x, status=%u",
-		      __entry->cid, __entry->qid, __entry->result,
+	    TP_printk("qid=%d, cmdid=%u, res=%llu, retries=%u, flags=0x%x, status=%u",
+		      __entry->qid, __entry->cid, __entry->result,
 		      __entry->retries, __entry->flags, __entry->status)
 
 );
diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig
index 7595664ee753..3c7b61ddb0d1 100644
--- a/drivers/nvme/target/Kconfig
+++ b/drivers/nvme/target/Kconfig
@@ -27,7 +27,7 @@ config NVME_TARGET_LOOP
 
 config NVME_TARGET_RDMA
 	tristate "NVMe over Fabrics RDMA target support"
-	depends on INFINIBAND_ADDR_TRANS
+	depends on INFINIBAND && INFINIBAND_ADDR_TRANS
 	depends on NVME_TARGET
 	select SGL_ALLOC
 	help
diff --git a/drivers/nvme/target/Makefile b/drivers/nvme/target/Makefile
index 488250189c99..8118c93391c6 100644
--- a/drivers/nvme/target/Makefile
+++ b/drivers/nvme/target/Makefile
@@ -6,8 +6,8 @@ obj-$(CONFIG_NVME_TARGET_RDMA)		+= nvmet-rdma.o
 obj-$(CONFIG_NVME_TARGET_FC)		+= nvmet-fc.o
 obj-$(CONFIG_NVME_TARGET_FCLOOP)	+= nvme-fcloop.o
 
-nvmet-y		+= core.o configfs.o admin-cmd.o io-cmd.o fabrics-cmd.o \
-			discovery.o
+nvmet-y		+= core.o configfs.o admin-cmd.o fabrics-cmd.o \
+			discovery.o io-cmd-file.o io-cmd-bdev.o
 nvme-loop-y	+= loop.o
 nvmet-rdma-y	+= rdma.o
 nvmet-fc-y	+= fc.o
diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index 5e0e9fcc0d4d..b2ba95b2eef7 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -45,6 +45,10 @@ static u16 nvmet_get_smart_log_nsid(struct nvmet_req *req,
 		return NVME_SC_INVALID_NS;
 	}
 
+	/* we don't have the right data for file backed ns */
+	if (!ns->bdev)
+		goto out;
+
 	host_reads = part_stat_read(ns->bdev->bd_part, ios[READ]);
 	data_units_read = part_stat_read(ns->bdev->bd_part, sectors[READ]);
 	host_writes = part_stat_read(ns->bdev->bd_part, ios[WRITE]);
@@ -54,6 +58,7 @@ static u16 nvmet_get_smart_log_nsid(struct nvmet_req *req,
 	put_unaligned_le64(data_units_read, &slog->data_units_read[0]);
 	put_unaligned_le64(host_writes, &slog->host_writes[0]);
 	put_unaligned_le64(data_units_written, &slog->data_units_written[0]);
+out:
 	nvmet_put_namespace(ns);
 
 	return NVME_SC_SUCCESS;
@@ -71,6 +76,9 @@ static u16 nvmet_get_smart_log_all(struct nvmet_req *req,
 
 	rcu_read_lock();
 	list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link) {
+		/* we don't have the right data for file backed ns */
+		if (!ns->bdev)
+			continue;
 		host_reads += part_stat_read(ns->bdev->bd_part, ios[READ]);
 		data_units_read +=
 			part_stat_read(ns->bdev->bd_part, sectors[READ]);
@@ -548,8 +556,6 @@ u16 nvmet_parse_admin_cmd(struct nvmet_req *req)
 	struct nvme_command *cmd = req->cmd;
 	u16 ret;
 
-	req->ns = NULL;
-
 	ret = nvmet_check_ctrl_status(req, cmd);
 	if (unlikely(ret))
 		return ret;
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index e95424f172fd..800aaf96ddcd 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -271,6 +271,12 @@ void nvmet_put_namespace(struct nvmet_ns *ns)
 	percpu_ref_put(&ns->ref);
 }
 
+static void nvmet_ns_dev_disable(struct nvmet_ns *ns)
+{
+	nvmet_bdev_ns_disable(ns);
+	nvmet_file_ns_disable(ns);
+}
+
 int nvmet_ns_enable(struct nvmet_ns *ns)
 {
 	struct nvmet_subsys *subsys = ns->subsys;
@@ -281,23 +287,16 @@ int nvmet_ns_enable(struct nvmet_ns *ns)
 	if (ns->enabled)
 		goto out_unlock;
 
-	ns->bdev = blkdev_get_by_path(ns->device_path, FMODE_READ | FMODE_WRITE,
-			NULL);
-	if (IS_ERR(ns->bdev)) {
-		pr_err("failed to open block device %s: (%ld)\n",
-		       ns->device_path, PTR_ERR(ns->bdev));
-		ret = PTR_ERR(ns->bdev);
-		ns->bdev = NULL;
+	ret = nvmet_bdev_ns_enable(ns);
+	if (ret)
+		ret = nvmet_file_ns_enable(ns);
+	if (ret)
 		goto out_unlock;
-	}
-
-	ns->size = i_size_read(ns->bdev->bd_inode);
-	ns->blksize_shift = blksize_bits(bdev_logical_block_size(ns->bdev));
 
 	ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace,
 				0, GFP_KERNEL);
 	if (ret)
-		goto out_blkdev_put;
+		goto out_dev_put;
 
 	if (ns->nsid > subsys->max_nsid)
 		subsys->max_nsid = ns->nsid;
@@ -328,9 +327,8 @@ int nvmet_ns_enable(struct nvmet_ns *ns)
 out_unlock:
 	mutex_unlock(&subsys->lock);
 	return ret;
-out_blkdev_put:
-	blkdev_put(ns->bdev, FMODE_WRITE|FMODE_READ);
-	ns->bdev = NULL;
+out_dev_put:
+	nvmet_ns_dev_disable(ns);
 	goto out_unlock;
 }
 
@@ -366,8 +364,7 @@ void nvmet_ns_disable(struct nvmet_ns *ns)
 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
 		nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE, 0, 0);
 
-	if (ns->bdev)
-		blkdev_put(ns->bdev, FMODE_WRITE|FMODE_READ);
+	nvmet_ns_dev_disable(ns);
 out_unlock:
 	mutex_unlock(&subsys->lock);
 }
@@ -499,6 +496,25 @@ int nvmet_sq_init(struct nvmet_sq *sq)
 }
 EXPORT_SYMBOL_GPL(nvmet_sq_init);
 
+static u16 nvmet_parse_io_cmd(struct nvmet_req *req)
+{
+	struct nvme_command *cmd = req->cmd;
+	u16 ret;
+
+	ret = nvmet_check_ctrl_status(req, cmd);
+	if (unlikely(ret))
+		return ret;
+
+	req->ns = nvmet_find_namespace(req->sq->ctrl, cmd->rw.nsid);
+	if (unlikely(!req->ns))
+		return NVME_SC_INVALID_NS | NVME_SC_DNR;
+
+	if (req->ns->file)
+		return nvmet_file_parse_io_cmd(req);
+	else
+		return nvmet_bdev_parse_io_cmd(req);
+}
+
 bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,
 		struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops)
 {
@@ -710,15 +726,14 @@ u16 nvmet_ctrl_find_get(const char *subsysnqn, const char *hostnqn, u16 cntlid,
 u16 nvmet_check_ctrl_status(struct nvmet_req *req, struct nvme_command *cmd)
 {
 	if (unlikely(!(req->sq->ctrl->cc & NVME_CC_ENABLE))) {
-		pr_err("got io cmd %d while CC.EN == 0 on qid = %d\n",
+		pr_err("got cmd %d while CC.EN == 0 on qid = %d\n",
 		       cmd->common.opcode, req->sq->qid);
 		return NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR;
 	}
 
 	if (unlikely(!(req->sq->ctrl->csts & NVME_CSTS_RDY))) {
-		pr_err("got io cmd %d while CSTS.RDY == 0 on qid = %d\n",
+		pr_err("got cmd %d while CSTS.RDY == 0 on qid = %d\n",
 		       cmd->common.opcode, req->sq->qid);
-		req->ns = NULL;
 		return NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR;
 	}
 	return 0;
diff --git a/drivers/nvme/target/discovery.c b/drivers/nvme/target/discovery.c
index 231e04e0a496..08656b849bd6 100644
--- a/drivers/nvme/target/discovery.c
+++ b/drivers/nvme/target/discovery.c
@@ -187,8 +187,6 @@ u16 nvmet_parse_discovery_cmd(struct nvmet_req *req)
 {
 	struct nvme_command *cmd = req->cmd;
 
-	req->ns = NULL;
-
 	if (unlikely(!(req->sq->ctrl->csts & NVME_CSTS_RDY))) {
 		pr_err("got cmd %d while not ready\n",
 		       cmd->common.opcode);
diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c
index 19e9e42ae943..d84ae004cb85 100644
--- a/drivers/nvme/target/fabrics-cmd.c
+++ b/drivers/nvme/target/fabrics-cmd.c
@@ -77,8 +77,6 @@ u16 nvmet_parse_fabrics_cmd(struct nvmet_req *req)
 {
 	struct nvme_command *cmd = req->cmd;
 
-	req->ns = NULL;
-
 	switch (cmd->fabrics.fctype) {
 	case nvme_fabrics_type_property_set:
 		req->data_len = 0;
@@ -242,8 +240,6 @@ u16 nvmet_parse_connect_cmd(struct nvmet_req *req)
 {
 	struct nvme_command *cmd = req->cmd;
 
-	req->ns = NULL;
-
 	if (cmd->common.opcode != nvme_fabrics_command) {
 		pr_err("invalid command 0x%x on unconnected queue.\n",
 			cmd->fabrics.opcode);
diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c
index 33ee8d3145f8..408279cb6f2c 100644
--- a/drivers/nvme/target/fc.c
+++ b/drivers/nvme/target/fc.c
@@ -31,7 +31,7 @@
 /* *************************** Data Structures/Defines ****************** */
 
 
-#define NVMET_LS_CTX_COUNT		4
+#define NVMET_LS_CTX_COUNT		256
 
 /* for this implementation, assume small single frame rqst/rsp */
 #define NVME_FC_MAX_LS_BUFFER_SIZE		2048
diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c
new file mode 100644
index 000000000000..e0b0f7df70c2
--- /dev/null
+++ b/drivers/nvme/target/io-cmd-bdev.c
@@ -0,0 +1,241 @@
+/*
+ * NVMe I/O command implementation.
+ * Copyright (c) 2015-2016 HGST, a Western Digital Company.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/blkdev.h>
+#include <linux/module.h>
+#include "nvmet.h"
+
+int nvmet_bdev_ns_enable(struct nvmet_ns *ns)
+{
+	int ret;
+
+	ns->bdev = blkdev_get_by_path(ns->device_path,
+			FMODE_READ | FMODE_WRITE, NULL);
+	if (IS_ERR(ns->bdev)) {
+		ret = PTR_ERR(ns->bdev);
+		if (ret != -ENOTBLK) {
+			pr_err("failed to open block device %s: (%ld)\n",
+					ns->device_path, PTR_ERR(ns->bdev));
+		}
+		ns->bdev = NULL;
+		return ret;
+	}
+	ns->size = i_size_read(ns->bdev->bd_inode);
+	ns->blksize_shift = blksize_bits(bdev_logical_block_size(ns->bdev));
+	return 0;
+}
+
+void nvmet_bdev_ns_disable(struct nvmet_ns *ns)
+{
+	if (ns->bdev) {
+		blkdev_put(ns->bdev, FMODE_WRITE | FMODE_READ);
+		ns->bdev = NULL;
+	}
+}
+
+static void nvmet_bio_done(struct bio *bio)
+{
+	struct nvmet_req *req = bio->bi_private;
+
+	nvmet_req_complete(req,
+		bio->bi_status ? NVME_SC_INTERNAL | NVME_SC_DNR : 0);
+
+	if (bio != &req->b.inline_bio)
+		bio_put(bio);
+}
+
+static void nvmet_bdev_execute_rw(struct nvmet_req *req)
+{
+	int sg_cnt = req->sg_cnt;
+	struct bio *bio = &req->b.inline_bio;
+	struct scatterlist *sg;
+	sector_t sector;
+	blk_qc_t cookie;
+	int op, op_flags = 0, i;
+
+	if (!req->sg_cnt) {
+		nvmet_req_complete(req, 0);
+		return;
+	}
+
+	if (req->cmd->rw.opcode == nvme_cmd_write) {
+		op = REQ_OP_WRITE;
+		op_flags = REQ_SYNC | REQ_IDLE;
+		if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
+			op_flags |= REQ_FUA;
+	} else {
+		op = REQ_OP_READ;
+	}
+
+	sector = le64_to_cpu(req->cmd->rw.slba);
+	sector <<= (req->ns->blksize_shift - 9);
+
+	bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec));
+	bio_set_dev(bio, req->ns->bdev);
+	bio->bi_iter.bi_sector = sector;
+	bio->bi_private = req;
+	bio->bi_end_io = nvmet_bio_done;
+	bio_set_op_attrs(bio, op, op_flags);
+
+	for_each_sg(req->sg, sg, req->sg_cnt, i) {
+		while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
+				!= sg->length) {
+			struct bio *prev = bio;
+
+			bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES));
+			bio_set_dev(bio, req->ns->bdev);
+			bio->bi_iter.bi_sector = sector;
+			bio_set_op_attrs(bio, op, op_flags);
+
+			bio_chain(bio, prev);
+			submit_bio(prev);
+		}
+
+		sector += sg->length >> 9;
+		sg_cnt--;
+	}
+
+	cookie = submit_bio(bio);
+
+	blk_poll(bdev_get_queue(req->ns->bdev), cookie);
+}
+
+static void nvmet_bdev_execute_flush(struct nvmet_req *req)
+{
+	struct bio *bio = &req->b.inline_bio;
+
+	bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec));
+	bio_set_dev(bio, req->ns->bdev);
+	bio->bi_private = req;
+	bio->bi_end_io = nvmet_bio_done;
+	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
+
+	submit_bio(bio);
+}
+
+static u16 nvmet_bdev_discard_range(struct nvmet_ns *ns,
+		struct nvme_dsm_range *range, struct bio **bio)
+{
+	int ret;
+
+	ret = __blkdev_issue_discard(ns->bdev,
+			le64_to_cpu(range->slba) << (ns->blksize_shift - 9),
+			le32_to_cpu(range->nlb) << (ns->blksize_shift - 9),
+			GFP_KERNEL, 0, bio);
+	if (ret && ret != -EOPNOTSUPP)
+		return NVME_SC_INTERNAL | NVME_SC_DNR;
+	return 0;
+}
+
+static void nvmet_bdev_execute_discard(struct nvmet_req *req)
+{
+	struct nvme_dsm_range range;
+	struct bio *bio = NULL;
+	int i;
+	u16 status;
+
+	for (i = 0; i <= le32_to_cpu(req->cmd->dsm.nr); i++) {
+		status = nvmet_copy_from_sgl(req, i * sizeof(range), &range,
+				sizeof(range));
+		if (status)
+			break;
+
+		status = nvmet_bdev_discard_range(req->ns, &range, &bio);
+		if (status)
+			break;
+	}
+
+	if (bio) {
+		bio->bi_private = req;
+		bio->bi_end_io = nvmet_bio_done;
+		if (status) {
+			bio->bi_status = BLK_STS_IOERR;
+			bio_endio(bio);
+		} else {
+			submit_bio(bio);
+		}
+	} else {
+		nvmet_req_complete(req, status);
+	}
+}
+
+static void nvmet_bdev_execute_dsm(struct nvmet_req *req)
+{
+	switch (le32_to_cpu(req->cmd->dsm.attributes)) {
+	case NVME_DSMGMT_AD:
+		nvmet_bdev_execute_discard(req);
+		return;
+	case NVME_DSMGMT_IDR:
+	case NVME_DSMGMT_IDW:
+	default:
+		/* Not supported yet */
+		nvmet_req_complete(req, 0);
+		return;
+	}
+}
+
+static void nvmet_bdev_execute_write_zeroes(struct nvmet_req *req)
+{
+	struct nvme_write_zeroes_cmd *write_zeroes = &req->cmd->write_zeroes;
+	struct bio *bio = NULL;
+	u16 status = NVME_SC_SUCCESS;
+	sector_t sector;
+	sector_t nr_sector;
+
+	sector = le64_to_cpu(write_zeroes->slba) <<
+		(req->ns->blksize_shift - 9);
+	nr_sector = (((sector_t)le16_to_cpu(write_zeroes->length) + 1) <<
+		(req->ns->blksize_shift - 9));
+
+	if (__blkdev_issue_zeroout(req->ns->bdev, sector, nr_sector,
+				GFP_KERNEL, &bio, 0))
+		status = NVME_SC_INTERNAL | NVME_SC_DNR;
+
+	if (bio) {
+		bio->bi_private = req;
+		bio->bi_end_io = nvmet_bio_done;
+		submit_bio(bio);
+	} else {
+		nvmet_req_complete(req, status);
+	}
+}
+
+u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req)
+{
+	struct nvme_command *cmd = req->cmd;
+
+	switch (cmd->common.opcode) {
+	case nvme_cmd_read:
+	case nvme_cmd_write:
+		req->execute = nvmet_bdev_execute_rw;
+		req->data_len = nvmet_rw_len(req);
+		return 0;
+	case nvme_cmd_flush:
+		req->execute = nvmet_bdev_execute_flush;
+		req->data_len = 0;
+		return 0;
+	case nvme_cmd_dsm:
+		req->execute = nvmet_bdev_execute_dsm;
+		req->data_len = (le32_to_cpu(cmd->dsm.nr) + 1) *
+			sizeof(struct nvme_dsm_range);
+		return 0;
+	case nvme_cmd_write_zeroes:
+		req->execute = nvmet_bdev_execute_write_zeroes;
+		return 0;
+	default:
+		pr_err("unhandled cmd %d on qid %d\n", cmd->common.opcode,
+		       req->sq->qid);
+		return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
+	}
+}
diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c
new file mode 100644
index 000000000000..ca1ccf867856
--- /dev/null
+++ b/drivers/nvme/target/io-cmd-file.c
@@ -0,0 +1,300 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * NVMe Over Fabrics Target File I/O commands implementation.
+ * Copyright (c) 2017-2018 Western Digital Corporation or its
+ * affiliates.
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/uio.h>
+#include <linux/falloc.h>
+#include <linux/file.h>
+#include "nvmet.h"
+
+#define NVMET_MAX_MPOOL_BVEC		16
+#define NVMET_MIN_MPOOL_OBJ		16
+
+void nvmet_file_ns_disable(struct nvmet_ns *ns)
+{
+	if (ns->file) {
+		mempool_destroy(ns->bvec_pool);
+		ns->bvec_pool = NULL;
+		kmem_cache_destroy(ns->bvec_cache);
+		ns->bvec_cache = NULL;
+		fput(ns->file);
+		ns->file = NULL;
+	}
+}
+
+int nvmet_file_ns_enable(struct nvmet_ns *ns)
+{
+	int ret;
+	struct kstat stat;
+
+	ns->file = filp_open(ns->device_path,
+			O_RDWR | O_LARGEFILE | O_DIRECT, 0);
+	if (IS_ERR(ns->file)) {
+		pr_err("failed to open file %s: (%ld)\n",
+				ns->device_path, PTR_ERR(ns->bdev));
+		return PTR_ERR(ns->file);
+	}
+
+	ret = vfs_getattr(&ns->file->f_path,
+			&stat, STATX_SIZE, AT_STATX_FORCE_SYNC);
+	if (ret)
+		goto err;
+
+	ns->size = stat.size;
+	ns->blksize_shift = file_inode(ns->file)->i_blkbits;
+
+	ns->bvec_cache = kmem_cache_create("nvmet-bvec",
+			NVMET_MAX_MPOOL_BVEC * sizeof(struct bio_vec),
+			0, SLAB_HWCACHE_ALIGN, NULL);
+	if (!ns->bvec_cache)
+		goto err;
+
+	ns->bvec_pool = mempool_create(NVMET_MIN_MPOOL_OBJ, mempool_alloc_slab,
+			mempool_free_slab, ns->bvec_cache);
+
+	if (!ns->bvec_pool)
+		goto err;
+
+	return ret;
+err:
+	ns->size = 0;
+	ns->blksize_shift = 0;
+	nvmet_file_ns_disable(ns);
+	return ret;
+}
+
+static void nvmet_file_init_bvec(struct bio_vec *bv, struct sg_page_iter *iter)
+{
+	bv->bv_page = sg_page_iter_page(iter);
+	bv->bv_offset = iter->sg->offset;
+	bv->bv_len = PAGE_SIZE - iter->sg->offset;
+}
+
+static ssize_t nvmet_file_submit_bvec(struct nvmet_req *req, loff_t pos,
+		unsigned long nr_segs, size_t count)
+{
+	struct kiocb *iocb = &req->f.iocb;
+	ssize_t (*call_iter)(struct kiocb *iocb, struct iov_iter *iter);
+	struct iov_iter iter;
+	int ki_flags = 0, rw;
+	ssize_t ret;
+
+	if (req->cmd->rw.opcode == nvme_cmd_write) {
+		if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
+			ki_flags = IOCB_DSYNC;
+		call_iter = req->ns->file->f_op->write_iter;
+		rw = WRITE;
+	} else {
+		call_iter = req->ns->file->f_op->read_iter;
+		rw = READ;
+	}
+
+	iov_iter_bvec(&iter, ITER_BVEC | rw, req->f.bvec, nr_segs, count);
+
+	iocb->ki_pos = pos;
+	iocb->ki_filp = req->ns->file;
+	iocb->ki_flags = IOCB_DIRECT | ki_flags;
+
+	ret = call_iter(iocb, &iter);
+
+	if (ret != -EIOCBQUEUED && iocb->ki_complete)
+		iocb->ki_complete(iocb, ret, 0);
+
+	return ret;
+}
+
+static void nvmet_file_io_done(struct kiocb *iocb, long ret, long ret2)
+{
+	struct nvmet_req *req = container_of(iocb, struct nvmet_req, f.iocb);
+
+	if (req->f.bvec != req->inline_bvec) {
+		if (likely(req->f.mpool_alloc == false))
+			kfree(req->f.bvec);
+		else
+			mempool_free(req->f.bvec, req->ns->bvec_pool);
+	}
+
+	nvmet_req_complete(req, ret != req->data_len ?
+			NVME_SC_INTERNAL | NVME_SC_DNR : 0);
+}
+
+static void nvmet_file_execute_rw(struct nvmet_req *req)
+{
+	ssize_t nr_bvec = DIV_ROUND_UP(req->data_len, PAGE_SIZE);
+	struct sg_page_iter sg_pg_iter;
+	unsigned long bv_cnt = 0;
+	bool is_sync = false;
+	size_t len = 0, total_len = 0;
+	ssize_t ret = 0;
+	loff_t pos;
+
+	if (!req->sg_cnt || !nr_bvec) {
+		nvmet_req_complete(req, 0);
+		return;
+	}
+
+	if (nr_bvec > NVMET_MAX_INLINE_BIOVEC)
+		req->f.bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec),
+				GFP_KERNEL);
+	else
+		req->f.bvec = req->inline_bvec;
+
+	req->f.mpool_alloc = false;
+	if (unlikely(!req->f.bvec)) {
+		/* fallback under memory pressure */
+		req->f.bvec = mempool_alloc(req->ns->bvec_pool, GFP_KERNEL);
+		req->f.mpool_alloc = true;
+		if (nr_bvec > NVMET_MAX_MPOOL_BVEC)
+			is_sync = true;
+	}
+
+	pos = le64_to_cpu(req->cmd->rw.slba) << req->ns->blksize_shift;
+
+	memset(&req->f.iocb, 0, sizeof(struct kiocb));
+	for_each_sg_page(req->sg, &sg_pg_iter, req->sg_cnt, 0) {
+		nvmet_file_init_bvec(&req->f.bvec[bv_cnt], &sg_pg_iter);
+		len += req->f.bvec[bv_cnt].bv_len;
+		total_len += req->f.bvec[bv_cnt].bv_len;
+		bv_cnt++;
+
+		WARN_ON_ONCE((nr_bvec - 1) < 0);
+
+		if (unlikely(is_sync) &&
+		    (nr_bvec - 1 == 0 || bv_cnt == NVMET_MAX_MPOOL_BVEC)) {
+			ret = nvmet_file_submit_bvec(req, pos, bv_cnt, len);
+			if (ret < 0)
+				goto out;
+			pos += len;
+			bv_cnt = 0;
+			len = 0;
+		}
+		nr_bvec--;
+	}
+
+	if (WARN_ON_ONCE(total_len != req->data_len))
+		ret = -EIO;
+out:
+	if (unlikely(is_sync || ret)) {
+		nvmet_file_io_done(&req->f.iocb, ret < 0 ? ret : total_len, 0);
+		return;
+	}
+	req->f.iocb.ki_complete = nvmet_file_io_done;
+	nvmet_file_submit_bvec(req, pos, bv_cnt, total_len);
+}
+
+static void nvmet_file_flush_work(struct work_struct *w)
+{
+	struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);
+	int ret;
+
+	ret = vfs_fsync(req->ns->file, 1);
+
+	nvmet_req_complete(req, ret < 0 ? NVME_SC_INTERNAL | NVME_SC_DNR : 0);
+}
+
+static void nvmet_file_execute_flush(struct nvmet_req *req)
+{
+	INIT_WORK(&req->f.work, nvmet_file_flush_work);
+	schedule_work(&req->f.work);
+}
+
+static void nvmet_file_execute_discard(struct nvmet_req *req)
+{
+	int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
+	struct nvme_dsm_range range;
+	loff_t offset;
+	loff_t len;
+	int i, ret;
+
+	for (i = 0; i <= le32_to_cpu(req->cmd->dsm.nr); i++) {
+		if (nvmet_copy_from_sgl(req, i * sizeof(range), &range,
+					sizeof(range)))
+			break;
+		offset = le64_to_cpu(range.slba) << req->ns->blksize_shift;
+		len = le32_to_cpu(range.nlb) << req->ns->blksize_shift;
+		ret = vfs_fallocate(req->ns->file, mode, offset, len);
+		if (ret)
+			break;
+	}
+
+	nvmet_req_complete(req, ret < 0 ? NVME_SC_INTERNAL | NVME_SC_DNR : 0);
+}
+
+static void nvmet_file_dsm_work(struct work_struct *w)
+{
+	struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);
+
+	switch (le32_to_cpu(req->cmd->dsm.attributes)) {
+	case NVME_DSMGMT_AD:
+		nvmet_file_execute_discard(req);
+		return;
+	case NVME_DSMGMT_IDR:
+	case NVME_DSMGMT_IDW:
+	default:
+		/* Not supported yet */
+		nvmet_req_complete(req, 0);
+		return;
+	}
+}
+
+static void nvmet_file_execute_dsm(struct nvmet_req *req)
+{
+	INIT_WORK(&req->f.work, nvmet_file_dsm_work);
+	schedule_work(&req->f.work);
+}
+
+static void nvmet_file_write_zeroes_work(struct work_struct *w)
+{
+	struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);
+	struct nvme_write_zeroes_cmd *write_zeroes = &req->cmd->write_zeroes;
+	int mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE;
+	loff_t offset;
+	loff_t len;
+	int ret;
+
+	offset = le64_to_cpu(write_zeroes->slba) << req->ns->blksize_shift;
+	len = (((sector_t)le16_to_cpu(write_zeroes->length) + 1) <<
+			req->ns->blksize_shift);
+
+	ret = vfs_fallocate(req->ns->file, mode, offset, len);
+	nvmet_req_complete(req, ret < 0 ? NVME_SC_INTERNAL | NVME_SC_DNR : 0);
+}
+
+static void nvmet_file_execute_write_zeroes(struct nvmet_req *req)
+{
+	INIT_WORK(&req->f.work, nvmet_file_write_zeroes_work);
+	schedule_work(&req->f.work);
+}
+
+u16 nvmet_file_parse_io_cmd(struct nvmet_req *req)
+{
+	struct nvme_command *cmd = req->cmd;
+
+	switch (cmd->common.opcode) {
+	case nvme_cmd_read:
+	case nvme_cmd_write:
+		req->execute = nvmet_file_execute_rw;
+		req->data_len = nvmet_rw_len(req);
+		return 0;
+	case nvme_cmd_flush:
+		req->execute = nvmet_file_execute_flush;
+		req->data_len = 0;
+		return 0;
+	case nvme_cmd_dsm:
+		req->execute = nvmet_file_execute_dsm;
+		req->data_len = (le32_to_cpu(cmd->dsm.nr) + 1) *
+			sizeof(struct nvme_dsm_range);
+		return 0;
+	case nvme_cmd_write_zeroes:
+		req->execute = nvmet_file_execute_write_zeroes;
+		req->data_len = 0;
+		return 0;
+	default:
+		pr_err("unhandled cmd for file ns %d on qid %d\n",
+				cmd->common.opcode, req->sq->qid);
+		return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
+	}
+}
diff --git a/drivers/nvme/target/io-cmd.c b/drivers/nvme/target/io-cmd.c
deleted file mode 100644
index cd2344179673..000000000000
--- a/drivers/nvme/target/io-cmd.c
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * NVMe I/O command implementation.
- * Copyright (c) 2015-2016 HGST, a Western Digital Company.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-#include <linux/blkdev.h>
-#include <linux/module.h>
-#include "nvmet.h"
-
-static void nvmet_bio_done(struct bio *bio)
-{
-	struct nvmet_req *req = bio->bi_private;
-
-	nvmet_req_complete(req,
-		bio->bi_status ? NVME_SC_INTERNAL | NVME_SC_DNR : 0);
-
-	if (bio != &req->inline_bio)
-		bio_put(bio);
-}
-
-static inline u32 nvmet_rw_len(struct nvmet_req *req)
-{
-	return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) <<
-			req->ns->blksize_shift;
-}
-
-static void nvmet_execute_rw(struct nvmet_req *req)
-{
-	int sg_cnt = req->sg_cnt;
-	struct bio *bio = &req->inline_bio;
-	struct scatterlist *sg;
-	sector_t sector;
-	blk_qc_t cookie;
-	int op, op_flags = 0, i;
-
-	if (!req->sg_cnt) {
-		nvmet_req_complete(req, 0);
-		return;
-	}
-
-	if (req->cmd->rw.opcode == nvme_cmd_write) {
-		op = REQ_OP_WRITE;
-		op_flags = REQ_SYNC | REQ_IDLE;
-		if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
-			op_flags |= REQ_FUA;
-	} else {
-		op = REQ_OP_READ;
-	}
-
-	sector = le64_to_cpu(req->cmd->rw.slba);
-	sector <<= (req->ns->blksize_shift - 9);
-
-	bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec));
-	bio_set_dev(bio, req->ns->bdev);
-	bio->bi_iter.bi_sector = sector;
-	bio->bi_private = req;
-	bio->bi_end_io = nvmet_bio_done;
-	bio_set_op_attrs(bio, op, op_flags);
-
-	for_each_sg(req->sg, sg, req->sg_cnt, i) {
-		while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
-				!= sg->length) {
-			struct bio *prev = bio;
-
-			bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES));
-			bio_set_dev(bio, req->ns->bdev);
-			bio->bi_iter.bi_sector = sector;
-			bio_set_op_attrs(bio, op, op_flags);
-
-			bio_chain(bio, prev);
-			submit_bio(prev);
-		}
-
-		sector += sg->length >> 9;
-		sg_cnt--;
-	}
-
-	cookie = submit_bio(bio);
-
-	blk_poll(bdev_get_queue(req->ns->bdev), cookie);
-}
-
-static void nvmet_execute_flush(struct nvmet_req *req)
-{
-	struct bio *bio = &req->inline_bio;
-
-	bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec));
-	bio_set_dev(bio, req->ns->bdev);
-	bio->bi_private = req;
-	bio->bi_end_io = nvmet_bio_done;
-	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
-
-	submit_bio(bio);
-}
-
-static u16 nvmet_discard_range(struct nvmet_ns *ns,
-		struct nvme_dsm_range *range, struct bio **bio)
-{
-	int ret;
-
-	ret = __blkdev_issue_discard(ns->bdev,
-			le64_to_cpu(range->slba) << (ns->blksize_shift - 9),
-			le32_to_cpu(range->nlb) << (ns->blksize_shift - 9),
-			GFP_KERNEL, 0, bio);
-	if (ret && ret != -EOPNOTSUPP)
-		return NVME_SC_INTERNAL | NVME_SC_DNR;
-	return 0;
-}
-
-static void nvmet_execute_discard(struct nvmet_req *req)
-{
-	struct nvme_dsm_range range;
-	struct bio *bio = NULL;
-	int i;
-	u16 status;
-
-	for (i = 0; i <= le32_to_cpu(req->cmd->dsm.nr); i++) {
-		status = nvmet_copy_from_sgl(req, i * sizeof(range), &range,
-				sizeof(range));
-		if (status)
-			break;
-
-		status = nvmet_discard_range(req->ns, &range, &bio);
-		if (status)
-			break;
-	}
-
-	if (bio) {
-		bio->bi_private = req;
-		bio->bi_end_io = nvmet_bio_done;
-		if (status) {
-			bio->bi_status = BLK_STS_IOERR;
-			bio_endio(bio);
-		} else {
-			submit_bio(bio);
-		}
-	} else {
-		nvmet_req_complete(req, status);
-	}
-}
-
-static void nvmet_execute_dsm(struct nvmet_req *req)
-{
-	switch (le32_to_cpu(req->cmd->dsm.attributes)) {
-	case NVME_DSMGMT_AD:
-		nvmet_execute_discard(req);
-		return;
-	case NVME_DSMGMT_IDR:
-	case NVME_DSMGMT_IDW:
-	default:
-		/* Not supported yet */
-		nvmet_req_complete(req, 0);
-		return;
-	}
-}
-
-static void nvmet_execute_write_zeroes(struct nvmet_req *req)
-{
-	struct nvme_write_zeroes_cmd *write_zeroes = &req->cmd->write_zeroes;
-	struct bio *bio = NULL;
-	u16 status = NVME_SC_SUCCESS;
-	sector_t sector;
-	sector_t nr_sector;
-
-	sector = le64_to_cpu(write_zeroes->slba) <<
-		(req->ns->blksize_shift - 9);
-	nr_sector = (((sector_t)le16_to_cpu(write_zeroes->length) + 1) <<
-		(req->ns->blksize_shift - 9));
-
-	if (__blkdev_issue_zeroout(req->ns->bdev, sector, nr_sector,
-				GFP_KERNEL, &bio, 0))
-		status = NVME_SC_INTERNAL | NVME_SC_DNR;
-
-	if (bio) {
-		bio->bi_private = req;
-		bio->bi_end_io = nvmet_bio_done;
-		submit_bio(bio);
-	} else {
-		nvmet_req_complete(req, status);
-	}
-}
-
-u16 nvmet_parse_io_cmd(struct nvmet_req *req)
-{
-	struct nvme_command *cmd = req->cmd;
-	u16 ret;
-
-	ret = nvmet_check_ctrl_status(req, cmd);
-	if (unlikely(ret)) {
-		req->ns = NULL;
-		return ret;
-	}
-
-	req->ns = nvmet_find_namespace(req->sq->ctrl, cmd->rw.nsid);
-	if (unlikely(!req->ns))
-		return NVME_SC_INVALID_NS | NVME_SC_DNR;
-
-	switch (cmd->common.opcode) {
-	case nvme_cmd_read:
-	case nvme_cmd_write:
-		req->execute = nvmet_execute_rw;
-		req->data_len = nvmet_rw_len(req);
-		return 0;
-	case nvme_cmd_flush:
-		req->execute = nvmet_execute_flush;
-		req->data_len = 0;
-		return 0;
-	case nvme_cmd_dsm:
-		req->execute = nvmet_execute_dsm;
-		req->data_len = (le32_to_cpu(cmd->dsm.nr) + 1) *
-			sizeof(struct nvme_dsm_range);
-		return 0;
-	case nvme_cmd_write_zeroes:
-		req->execute = nvmet_execute_write_zeroes;
-		return 0;
-	default:
-		pr_err("unhandled cmd %d on qid %d\n", cmd->common.opcode,
-		       req->sq->qid);
-		return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
-	}
-}
diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c
index 27a8561c0cb9..02cc316fad39 100644
--- a/drivers/nvme/target/loop.c
+++ b/drivers/nvme/target/loop.c
@@ -146,7 +146,7 @@ nvme_loop_timeout(struct request *rq, bool reserved)
 	/* fail with DNR on admin cmd timeout */
 	nvme_req(rq)->status = NVME_SC_ABORT_REQ | NVME_SC_DNR;
 
-	return BLK_EH_HANDLED;
+	return BLK_EH_DONE;
 }
 
 static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
@@ -174,7 +174,7 @@ static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
 			&queue->nvme_sq, &nvme_loop_ops))
 		return BLK_STS_OK;
 
-	if (blk_rq_payload_bytes(req)) {
+	if (blk_rq_nr_phys_segments(req)) {
 		iod->sg_table.sgl = iod->first_sgl;
 		if (sg_alloc_table_chained(&iod->sg_table,
 				blk_rq_nr_phys_segments(req),
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 15fd84ab21f8..2d09afcfe505 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -43,6 +43,7 @@ struct nvmet_ns {
 	struct list_head	dev_link;
 	struct percpu_ref	ref;
 	struct block_device	*bdev;
+	struct file		*file;
 	u32			nsid;
 	u32			blksize_shift;
 	loff_t			size;
@@ -57,6 +58,8 @@ struct nvmet_ns {
 	struct config_group	group;
 
 	struct completion	disable_done;
+	mempool_t		*bvec_pool;
+	struct kmem_cache	*bvec_cache;
 };
 
 static inline struct nvmet_ns *to_nvmet_ns(struct config_item *item)
@@ -222,8 +225,18 @@ struct nvmet_req {
 	struct nvmet_cq		*cq;
 	struct nvmet_ns		*ns;
 	struct scatterlist	*sg;
-	struct bio		inline_bio;
 	struct bio_vec		inline_bvec[NVMET_MAX_INLINE_BIOVEC];
+	union {
+		struct {
+			struct bio      inline_bio;
+		} b;
+		struct {
+			bool			mpool_alloc;
+			struct kiocb            iocb;
+			struct bio_vec          *bvec;
+			struct work_struct      work;
+		} f;
+	};
 	int			sg_cnt;
 	/* data length as parsed from the command: */
 	size_t			data_len;
@@ -263,7 +276,8 @@ struct nvmet_async_event {
 };
 
 u16 nvmet_parse_connect_cmd(struct nvmet_req *req);
-u16 nvmet_parse_io_cmd(struct nvmet_req *req);
+u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req);
+u16 nvmet_file_parse_io_cmd(struct nvmet_req *req);
 u16 nvmet_parse_admin_cmd(struct nvmet_req *req);
 u16 nvmet_parse_discovery_cmd(struct nvmet_req *req);
 u16 nvmet_parse_fabrics_cmd(struct nvmet_req *req);
@@ -338,4 +352,14 @@ extern struct rw_semaphore nvmet_config_sem;
 bool nvmet_host_allowed(struct nvmet_req *req, struct nvmet_subsys *subsys,
 		const char *hostnqn);
 
+int nvmet_bdev_ns_enable(struct nvmet_ns *ns);
+int nvmet_file_ns_enable(struct nvmet_ns *ns);
+void nvmet_bdev_ns_disable(struct nvmet_ns *ns);
+void nvmet_file_ns_disable(struct nvmet_ns *ns);
+
+static inline u32 nvmet_rw_len(struct nvmet_req *req)
+{
+	return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) <<
+			req->ns->blksize_shift;
+}
 #endif /* _NVMET_H */
diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig
index 1090924efdb1..54a3c298247b 100644
--- a/drivers/nvmem/Kconfig
+++ b/drivers/nvmem/Kconfig
@@ -175,4 +175,10 @@ config NVMEM_SNVS_LPGPR
 	  This driver can also be built as a module. If so, the module
 	  will be called nvmem-snvs-lpgpr.
 
+config RAVE_SP_EEPROM
+	tristate "Rave SP EEPROM Support"
+	depends on RAVE_SP_CORE
+	help
+	  Say y here to enable Rave SP EEPROM support.
+
 endif
diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile
index e54dcfa6565a..27e96a8efd1c 100644
--- a/drivers/nvmem/Makefile
+++ b/drivers/nvmem/Makefile
@@ -37,3 +37,6 @@ obj-$(CONFIG_MESON_MX_EFUSE)	+= nvmem_meson_mx_efuse.o
 nvmem_meson_mx_efuse-y		:= meson-mx-efuse.o
 obj-$(CONFIG_NVMEM_SNVS_LPGPR)	+= nvmem_snvs_lpgpr.o
 nvmem_snvs_lpgpr-y		:= snvs_lpgpr.o
+obj-$(CONFIG_RAVE_SP_EEPROM)	+= nvmem-rave-sp-eeprom.o
+nvmem-rave-sp-eeprom-y		:= rave-sp-eeprom.o
+
diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c
index b05aa8e81303..b5b0cdc21d01 100644
--- a/drivers/nvmem/core.c
+++ b/drivers/nvmem/core.c
@@ -353,18 +353,27 @@ static int nvmem_cell_info_to_nvmem_cell(struct nvmem_device *nvmem,
 	return 0;
 }
 
-static int nvmem_add_cells(struct nvmem_device *nvmem,
-			   const struct nvmem_config *cfg)
+/**
+ * nvmem_add_cells() - Add cell information to an nvmem device
+ *
+ * @nvmem: nvmem device to add cells to.
+ * @info: nvmem cell info to add to the device
+ * @ncells: number of cells in info
+ *
+ * Return: 0 or negative error code on failure.
+ */
+int nvmem_add_cells(struct nvmem_device *nvmem,
+		    const struct nvmem_cell_info *info,
+		    int ncells)
 {
 	struct nvmem_cell **cells;
-	const struct nvmem_cell_info *info = cfg->cells;
 	int i, rval;
 
-	cells = kcalloc(cfg->ncells, sizeof(*cells), GFP_KERNEL);
+	cells = kcalloc(ncells, sizeof(*cells), GFP_KERNEL);
 	if (!cells)
 		return -ENOMEM;
 
-	for (i = 0; i < cfg->ncells; i++) {
+	for (i = 0; i < ncells; i++) {
 		cells[i] = kzalloc(sizeof(**cells), GFP_KERNEL);
 		if (!cells[i]) {
 			rval = -ENOMEM;
@@ -380,7 +389,7 @@ static int nvmem_add_cells(struct nvmem_device *nvmem,
 		nvmem_cell_add(cells[i]);
 	}
 
-	nvmem->ncells = cfg->ncells;
+	nvmem->ncells = ncells;
 	/* remove tmp array */
 	kfree(cells);
 
@@ -393,6 +402,7 @@ static int nvmem_add_cells(struct nvmem_device *nvmem,
 
 	return rval;
 }
+EXPORT_SYMBOL_GPL(nvmem_add_cells);
 
 /*
  * nvmem_setup_compat() - Create an additional binary entry in
@@ -509,7 +519,7 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 	}
 
 	if (config->cells)
-		nvmem_add_cells(nvmem, config);
+		nvmem_add_cells(nvmem, config->cells, config->ncells);
 
 	return nvmem;
 
@@ -559,6 +569,7 @@ static void devm_nvmem_release(struct device *dev, void *res)
  * nvmem_config.
  * Also creates an binary entry in /sys/bus/nvmem/devices/dev-name/nvmem
  *
+ * @dev: Device that uses the nvmem device.
  * @config: nvmem device configuration with which nvmem device is created.
  *
  * Return: Will be an ERR_PTR() on error or a valid pointer to nvmem_device
@@ -597,6 +608,7 @@ static int devm_nvmem_match(struct device *dev, void *res, void *data)
  * devm_nvmem_unregister() - Unregister previously registered managed nvmem
  * device.
  *
+ * @dev: Device that uses the nvmem device.
  * @nvmem: Pointer to previously registered nvmem device.
  *
  * Return: Will be an negative on error or a zero on success.
@@ -1107,6 +1119,8 @@ static void *nvmem_cell_prepare_write_buffer(struct nvmem_cell *cell,
 
 		/* setup the first byte with lsb bits from nvmem */
 		rc = nvmem_reg_read(nvmem, cell->offset, &v, 1);
+		if (rc)
+			goto err;
 		*b++ |= GENMASK(bit_offset - 1, 0) & v;
 
 		/* setup rest of the byte if any */
@@ -1125,11 +1139,16 @@ static void *nvmem_cell_prepare_write_buffer(struct nvmem_cell *cell,
 		/* setup the last byte with msb bits from nvmem */
 		rc = nvmem_reg_read(nvmem,
 				    cell->offset + cell->bytes - 1, &v, 1);
+		if (rc)
+			goto err;
 		*p |= GENMASK(7, (nbits + bit_offset) % BITS_PER_BYTE) & v;
 
 	}
 
 	return buf;
+err:
+	kfree(buf);
+	return ERR_PTR(rc);
 }
 
 /**
diff --git a/drivers/nvmem/meson-efuse.c b/drivers/nvmem/meson-efuse.c
index 71823d1403c5..d769840d1e18 100644
--- a/drivers/nvmem/meson-efuse.c
+++ b/drivers/nvmem/meson-efuse.c
@@ -24,23 +24,16 @@
 static int meson_efuse_read(void *context, unsigned int offset,
 			    void *val, size_t bytes)
 {
-	u8 *buf = val;
-	int ret;
-
-	ret = meson_sm_call_read(buf, bytes, SM_EFUSE_READ, offset,
-				 bytes, 0, 0, 0);
-	if (ret < 0)
-		return ret;
-
-	return 0;
+	return meson_sm_call_read((u8 *)val, bytes, SM_EFUSE_READ, offset,
+				  bytes, 0, 0, 0);
 }
 
-static struct nvmem_config econfig = {
-	.name = "meson-efuse",
-	.stride = 1,
-	.word_size = 1,
-	.read_only = true,
-};
+static int meson_efuse_write(void *context, unsigned int offset,
+			     void *val, size_t bytes)
+{
+	return meson_sm_call_write((u8 *)val, bytes, SM_EFUSE_WRITE, offset,
+				   bytes, 0, 0, 0);
+}
 
 static const struct of_device_id meson_efuse_match[] = {
 	{ .compatible = "amlogic,meson-gxbb-efuse", },
@@ -50,17 +43,27 @@ MODULE_DEVICE_TABLE(of, meson_efuse_match);
 
 static int meson_efuse_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
 	struct nvmem_device *nvmem;
+	struct nvmem_config *econfig;
 	unsigned int size;
 
 	if (meson_sm_call(SM_EFUSE_USER_MAX, &size, 0, 0, 0, 0, 0) < 0)
 		return -EINVAL;
 
-	econfig.dev = &pdev->dev;
-	econfig.reg_read = meson_efuse_read;
-	econfig.size = size;
+	econfig = devm_kzalloc(dev, sizeof(*econfig), GFP_KERNEL);
+	if (!econfig)
+		return -ENOMEM;
+
+	econfig->dev = dev;
+	econfig->name = dev_name(dev);
+	econfig->stride = 1;
+	econfig->word_size = 1;
+	econfig->reg_read = meson_efuse_read;
+	econfig->reg_write = meson_efuse_write;
+	econfig->size = size;
 
-	nvmem = devm_nvmem_register(&pdev->dev, &econfig);
+	nvmem = devm_nvmem_register(&pdev->dev, econfig);
 
 	return PTR_ERR_OR_ZERO(nvmem);
 }
diff --git a/drivers/nvmem/rave-sp-eeprom.c b/drivers/nvmem/rave-sp-eeprom.c
new file mode 100644
index 000000000000..50aeea6ec6cc
--- /dev/null
+++ b/drivers/nvmem/rave-sp-eeprom.c
@@ -0,0 +1,357 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/*
+ * EEPROM driver for RAVE SP
+ *
+ * Copyright (C) 2018 Zodiac Inflight Innovations
+ *
+ */
+#include <linux/kernel.h>
+#include <linux/mfd/rave-sp.h>
+#include <linux/module.h>
+#include <linux/nvmem-provider.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/sizes.h>
+
+/**
+ * enum rave_sp_eeprom_access_type - Supported types of EEPROM access
+ *
+ * @RAVE_SP_EEPROM_WRITE:	EEPROM write
+ * @RAVE_SP_EEPROM_READ:	EEPROM read
+ */
+enum rave_sp_eeprom_access_type {
+	RAVE_SP_EEPROM_WRITE = 0,
+	RAVE_SP_EEPROM_READ  = 1,
+};
+
+/**
+ * enum rave_sp_eeprom_header_size - EEPROM command header sizes
+ *
+ * @RAVE_SP_EEPROM_HEADER_SMALL: EEPROM header size for "small" devices (< 8K)
+ * @RAVE_SP_EEPROM_HEADER_BIG:	 EEPROM header size for "big" devices (> 8K)
+ */
+enum rave_sp_eeprom_header_size {
+	RAVE_SP_EEPROM_HEADER_SMALL = 4U,
+	RAVE_SP_EEPROM_HEADER_BIG   = 5U,
+};
+
+#define	RAVE_SP_EEPROM_PAGE_SIZE	32U
+
+/**
+ * struct rave_sp_eeprom_page - RAVE SP EEPROM page
+ *
+ * @type:	Access type (see enum rave_sp_eeprom_access_type)
+ * @success:	Success flag (Success = 1, Failure = 0)
+ * @data:	Read data
+
+ * Note this structure corresponds to RSP_*_EEPROM payload from RAVE
+ * SP ICD
+ */
+struct rave_sp_eeprom_page {
+	u8  type;
+	u8  success;
+	u8  data[RAVE_SP_EEPROM_PAGE_SIZE];
+} __packed;
+
+/**
+ * struct rave_sp_eeprom - RAVE SP EEPROM device
+ *
+ * @sp:			Pointer to parent RAVE SP device
+ * @mutex:		Lock protecting access to EEPROM
+ * @address:		EEPROM device address
+ * @header_size:	Size of EEPROM command header for this device
+ * @dev:		Pointer to corresponding struct device used for logging
+ */
+struct rave_sp_eeprom {
+	struct rave_sp *sp;
+	struct mutex mutex;
+	u8 address;
+	unsigned int header_size;
+	struct device *dev;
+};
+
+/**
+ * rave_sp_eeprom_io - Low-level part of EEPROM page access
+ *
+ * @eeprom:	EEPROM device to write to
+ * @type:	EEPROM access type (read or write)
+ * @idx:	number of the EEPROM page
+ * @page:	Data to write or buffer to store result (via page->data)
+ *
+ * This function does all of the low-level work required to perform a
+ * EEPROM access. This includes formatting correct command payload,
+ * sending it and checking received results.
+ *
+ * Returns zero in case of success or negative error code in
+ * case of failure.
+ */
+static int rave_sp_eeprom_io(struct rave_sp_eeprom *eeprom,
+			     enum rave_sp_eeprom_access_type type,
+			     u16 idx,
+			     struct rave_sp_eeprom_page *page)
+{
+	const bool is_write = type == RAVE_SP_EEPROM_WRITE;
+	const unsigned int data_size = is_write ? sizeof(page->data) : 0;
+	const unsigned int cmd_size = eeprom->header_size + data_size;
+	const unsigned int rsp_size =
+		is_write ? sizeof(*page) - sizeof(page->data) : sizeof(*page);
+	unsigned int offset = 0;
+	u8 cmd[cmd_size];
+	int ret;
+
+	cmd[offset++] = eeprom->address;
+	cmd[offset++] = 0;
+	cmd[offset++] = type;
+	cmd[offset++] = idx;
+
+	/*
+	 * If there's still room in this command's header it means we
+	 * are talkin to EEPROM that uses 16-bit page numbers and we
+	 * have to specify index's MSB in payload as well.
+	 */
+	if (offset < eeprom->header_size)
+		cmd[offset++] = idx >> 8;
+	/*
+	 * Copy our data to write to command buffer first. In case of
+	 * a read data_size should be zero and memcpy would become a
+	 * no-op
+	 */
+	memcpy(&cmd[offset], page->data, data_size);
+
+	ret = rave_sp_exec(eeprom->sp, cmd, cmd_size, page, rsp_size);
+	if (ret)
+		return ret;
+
+	if (page->type != type)
+		return -EPROTO;
+
+	if (!page->success)
+		return -EIO;
+
+	return 0;
+}
+
+/**
+ * rave_sp_eeprom_page_access - Access single EEPROM page
+ *
+ * @eeprom:	EEPROM device to access
+ * @type:	Access type to perform (read or write)
+ * @offset:	Offset within EEPROM to access
+ * @data:	Data buffer
+ * @data_len:	Size of the data buffer
+ *
+ * This function performs a generic access to a single page or a
+ * portion thereof. Requested access MUST NOT cross the EEPROM page
+ * boundary.
+ *
+ * Returns zero in case of success or negative error code in
+ * case of failure.
+ */
+static int
+rave_sp_eeprom_page_access(struct rave_sp_eeprom *eeprom,
+			   enum rave_sp_eeprom_access_type type,
+			   unsigned int offset, u8 *data,
+			   size_t data_len)
+{
+	const unsigned int page_offset = offset % RAVE_SP_EEPROM_PAGE_SIZE;
+	const unsigned int page_nr     = offset / RAVE_SP_EEPROM_PAGE_SIZE;
+	struct rave_sp_eeprom_page page;
+	int ret;
+
+	/*
+	 * This function will not work if data access we've been asked
+	 * to do is crossing EEPROM page boundary. Normally this
+	 * should never happen and getting here would indicate a bug
+	 * in the code.
+	 */
+	if (WARN_ON(data_len > sizeof(page.data) - page_offset))
+		return -EINVAL;
+
+	if (type == RAVE_SP_EEPROM_WRITE) {
+		/*
+		 * If doing a partial write we need to do a read first
+		 * to fill the rest of the page with correct data.
+		 */
+		if (data_len < RAVE_SP_EEPROM_PAGE_SIZE) {
+			ret = rave_sp_eeprom_io(eeprom, RAVE_SP_EEPROM_READ,
+						page_nr, &page);
+			if (ret)
+				return ret;
+		}
+
+		memcpy(&page.data[page_offset], data, data_len);
+	}
+
+	ret = rave_sp_eeprom_io(eeprom, type, page_nr, &page);
+	if (ret)
+		return ret;
+
+	/*
+	 * Since we receive the result of the read via 'page.data'
+	 * buffer we need to copy that to 'data'
+	 */
+	if (type == RAVE_SP_EEPROM_READ)
+		memcpy(data, &page.data[page_offset], data_len);
+
+	return 0;
+}
+
+/**
+ * rave_sp_eeprom_access - Access EEPROM data
+ *
+ * @eeprom:	EEPROM device to access
+ * @type:	Access type to perform (read or write)
+ * @offset:	Offset within EEPROM to access
+ * @data:	Data buffer
+ * @data_len:	Size of the data buffer
+ *
+ * This function performs a generic access (either read or write) at
+ * arbitrary offset (not necessary page aligned) of arbitrary length
+ * (is not constrained by EEPROM page size).
+ *
+ * Returns zero in case of success or negative error code in case of
+ * failure.
+ */
+static int rave_sp_eeprom_access(struct rave_sp_eeprom *eeprom,
+				 enum rave_sp_eeprom_access_type type,
+				 unsigned int offset, u8 *data,
+				 unsigned int data_len)
+{
+	unsigned int residue;
+	unsigned int chunk;
+	unsigned int head;
+	int ret;
+
+	mutex_lock(&eeprom->mutex);
+
+	head    = offset % RAVE_SP_EEPROM_PAGE_SIZE;
+	residue = data_len;
+
+	do {
+		/*
+		 * First iteration, if we are doing an access that is
+		 * not 32-byte aligned, we need to access only data up
+		 * to a page boundary to avoid corssing it in
+		 * rave_sp_eeprom_page_access()
+		 */
+		if (unlikely(head)) {
+			chunk = RAVE_SP_EEPROM_PAGE_SIZE - head;
+			/*
+			 * This can only happen once per
+			 * rave_sp_eeprom_access() call, so we set
+			 * head to zero to process all the other
+			 * iterations normally.
+			 */
+			head  = 0;
+		} else {
+			chunk = RAVE_SP_EEPROM_PAGE_SIZE;
+		}
+
+		/*
+		 * We should never read more that 'residue' bytes
+		 */
+		chunk = min(chunk, residue);
+		ret = rave_sp_eeprom_page_access(eeprom, type, offset,
+						 data, chunk);
+		if (ret)
+			goto out;
+
+		residue -= chunk;
+		offset  += chunk;
+		data    += chunk;
+	} while (residue);
+out:
+	mutex_unlock(&eeprom->mutex);
+	return ret;
+}
+
+static int rave_sp_eeprom_reg_read(void *eeprom, unsigned int offset,
+				   void *val, size_t bytes)
+{
+	return rave_sp_eeprom_access(eeprom, RAVE_SP_EEPROM_READ,
+				     offset, val, bytes);
+}
+
+static int rave_sp_eeprom_reg_write(void *eeprom, unsigned int offset,
+				    void *val, size_t bytes)
+{
+	return rave_sp_eeprom_access(eeprom, RAVE_SP_EEPROM_WRITE,
+				     offset, val, bytes);
+}
+
+static int rave_sp_eeprom_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct rave_sp *sp = dev_get_drvdata(dev->parent);
+	struct device_node *np = dev->of_node;
+	struct nvmem_config config = { 0 };
+	struct rave_sp_eeprom *eeprom;
+	struct nvmem_device *nvmem;
+	u32 reg[2], size;
+
+	if (of_property_read_u32_array(np, "reg", reg, ARRAY_SIZE(reg))) {
+		dev_err(dev, "Failed to parse \"reg\" property\n");
+		return -EINVAL;
+	}
+
+	size = reg[1];
+	/*
+	 * Per ICD, we have no more than 2 bytes to specify EEPROM
+	 * page.
+	 */
+	if (size > U16_MAX * RAVE_SP_EEPROM_PAGE_SIZE) {
+		dev_err(dev, "Specified size is too big\n");
+		return -EINVAL;
+	}
+
+	eeprom = devm_kzalloc(dev, sizeof(*eeprom), GFP_KERNEL);
+	if (!eeprom)
+		return -ENOMEM;
+
+	eeprom->address = reg[0];
+	eeprom->sp      = sp;
+	eeprom->dev     = dev;
+
+	if (size > SZ_8K)
+		eeprom->header_size = RAVE_SP_EEPROM_HEADER_BIG;
+	else
+		eeprom->header_size = RAVE_SP_EEPROM_HEADER_SMALL;
+
+	mutex_init(&eeprom->mutex);
+
+	config.id		= -1;
+	of_property_read_string(np, "zii,eeprom-name", &config.name);
+	config.priv		= eeprom;
+	config.dev		= dev;
+	config.size		= size;
+	config.reg_read		= rave_sp_eeprom_reg_read;
+	config.reg_write	= rave_sp_eeprom_reg_write;
+	config.word_size	= 1;
+	config.stride		= 1;
+
+	nvmem = devm_nvmem_register(dev, &config);
+
+	return PTR_ERR_OR_ZERO(nvmem);
+}
+
+static const struct of_device_id rave_sp_eeprom_of_match[] = {
+	{ .compatible = "zii,rave-sp-eeprom" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, rave_sp_eeprom_of_match);
+
+static struct platform_driver rave_sp_eeprom_driver = {
+	.probe = rave_sp_eeprom_probe,
+	.driver	= {
+		.name = KBUILD_MODNAME,
+		.of_match_table = rave_sp_eeprom_of_match,
+	},
+};
+module_platform_driver(rave_sp_eeprom_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Andrey Vostrikov <andrey.vostrikov@cogentembedded.com>");
+MODULE_AUTHOR("Nikita Yushchenko <nikita.yoush@cogentembedded.com>");
+MODULE_AUTHOR("Andrey Smirnov <andrew.smirnov@gmail.com>");
+MODULE_DESCRIPTION("RAVE SP EEPROM driver");
diff --git a/drivers/of/device.c b/drivers/of/device.c
index 064c818105bd..33d85511d790 100644
--- a/drivers/of/device.c
+++ b/drivers/of/device.c
@@ -76,6 +76,8 @@ int of_device_add(struct platform_device *ofdev)
  * of_dma_configure - Setup DMA configuration
  * @dev:	Device to apply DMA configuration
  * @np:		Pointer to OF node having DMA configuration
+ * @force_dma:  Whether device is to be set up by of_dma_configure() even if
+ *		DMA capability is not explicitly described by firmware.
  *
  * Try to get devices's DMA configuration from DT and update it
  * accordingly.
@@ -84,7 +86,7 @@ int of_device_add(struct platform_device *ofdev)
  * can use a platform bus notifier and handle BUS_NOTIFY_ADD_DEVICE events
  * to fix up DMA configuration.
  */
-int of_dma_configure(struct device *dev, struct device_node *np)
+int of_dma_configure(struct device *dev, struct device_node *np, bool force_dma)
 {
 	u64 dma_addr, paddr, size = 0;
 	int ret;
@@ -100,7 +102,7 @@ int of_dma_configure(struct device *dev, struct device_node *np)
 		 * DMA configuration regardless of whether "dma-ranges" is
 		 * correctly specified or not.
 		 */
-		if (!dev->bus->force_dma)
+		if (!force_dma)
 			return ret == -ENODEV ? 0 : ret;
 
 		dma_addr = offset = 0;
diff --git a/drivers/of/of_mdio.c b/drivers/of/of_mdio.c
index 8c0c92712fc9..d963baf8e53a 100644
--- a/drivers/of/of_mdio.c
+++ b/drivers/of/of_mdio.c
@@ -204,6 +204,9 @@ int of_mdiobus_register(struct mii_bus *mdio, struct device_node *np)
 	bool scanphys = false;
 	int addr, rc;
 
+	if (!np)
+		return mdiobus_register(mdio);
+
 	/* Do not continue if the node is disabled */
 	if (!of_device_is_available(np))
 		return -ENODEV;
diff --git a/drivers/of/of_numa.c b/drivers/of/of_numa.c
index f9d5480a4ae5..27d9b4bba535 100644
--- a/drivers/of/of_numa.c
+++ b/drivers/of/of_numa.c
@@ -177,7 +177,6 @@ int of_node_to_nid(struct device_node *device)
 
 	return NUMA_NO_NODE;
 }
-EXPORT_SYMBOL(of_node_to_nid);
 
 int __init of_numa_init(void)
 {
diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c
index 9a4f4246231d..895c83e0c7b6 100644
--- a/drivers/of/of_reserved_mem.c
+++ b/drivers/of/of_reserved_mem.c
@@ -353,7 +353,7 @@ int of_reserved_mem_device_init_by_idx(struct device *dev,
 		/* ensure that dma_ops is set for virtual devices
 		 * using reserved memory
 		 */
-		of_dma_configure(dev, np);
+		of_dma_configure(dev, np, true);
 
 		dev_info(dev, "assigned reserved memory node %s\n", rmem->name);
 	} else {
diff --git a/drivers/of/platform.c b/drivers/of/platform.c
index c00d81dfac0b..37108952ddd3 100644
--- a/drivers/of/platform.c
+++ b/drivers/of/platform.c
@@ -32,6 +32,11 @@ const struct of_device_id of_default_bus_match_table[] = {
 	{} /* Empty terminated list */
 };
 
+static const struct of_device_id of_skipped_node_table[] = {
+	{ .compatible = "operating-points-v2", },
+	{} /* Empty terminated list */
+};
+
 static int of_dev_node_match(struct device *dev, void *data)
 {
 	return dev->of_node == data;
@@ -356,6 +361,12 @@ static int of_platform_bus_create(struct device_node *bus,
 		return 0;
 	}
 
+	/* Skip nodes for which we don't want to create devices */
+	if (unlikely(of_match_node(of_skipped_node_table, bus))) {
+		pr_debug("%s() - skipping %pOF node\n", __func__, bus);
+		return 0;
+	}
+
 	if (of_node_check_flag(bus, OF_POPULATED_BUS)) {
 		pr_debug("%s() - skipping %pOF, already populated\n",
 			__func__, bus);
@@ -494,6 +505,7 @@ EXPORT_SYMBOL_GPL(of_platform_default_populate);
 #ifndef CONFIG_PPC
 static const struct of_device_id reserved_mem_matches[] = {
 	{ .compatible = "qcom,rmtfs-mem" },
+	{ .compatible = "qcom,cmd-db" },
 	{ .compatible = "ramoops" },
 	{}
 };
diff --git a/drivers/of/resolver.c b/drivers/of/resolver.c
index 65d0b7adfcd4..7edfac6f1914 100644
--- a/drivers/of/resolver.c
+++ b/drivers/of/resolver.c
@@ -122,6 +122,11 @@ static int update_usages_of_a_phandle_reference(struct device_node *overlay,
 			goto err_fail;
 		}
 
+		if (offset < 0 || offset + sizeof(__be32) > prop->length) {
+			err = -EINVAL;
+			goto err_fail;
+		}
+
 		*(__be32 *)(prop->value + offset) = cpu_to_be32(phandle);
 	}
 
diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c
index 6bb37c18292a..ecee50d10d14 100644
--- a/drivers/of/unittest.c
+++ b/drivers/of/unittest.c
@@ -165,20 +165,20 @@ static void __init of_unittest_dynamic(void)
 	/* Add a new property - should pass*/
 	prop->name = "new-property";
 	prop->value = "new-property-data";
-	prop->length = strlen(prop->value);
+	prop->length = strlen(prop->value) + 1;
 	unittest(of_add_property(np, prop) == 0, "Adding a new property failed\n");
 
 	/* Try to add an existing property - should fail */
 	prop++;
 	prop->name = "new-property";
 	prop->value = "new-property-data-should-fail";
-	prop->length = strlen(prop->value);
+	prop->length = strlen(prop->value) + 1;
 	unittest(of_add_property(np, prop) != 0,
 		 "Adding an existing property should have failed\n");
 
 	/* Try to modify an existing property - should pass */
 	prop->value = "modify-property-data-should-pass";
-	prop->length = strlen(prop->value);
+	prop->length = strlen(prop->value) + 1;
 	unittest(of_update_property(np, prop) == 0,
 		 "Updating an existing property should have passed\n");
 
@@ -186,7 +186,7 @@ static void __init of_unittest_dynamic(void)
 	prop++;
 	prop->name = "modify-property";
 	prop->value = "modify-missing-property-data-should-pass";
-	prop->length = strlen(prop->value);
+	prop->length = strlen(prop->value) + 1;
 	unittest(of_update_property(np, prop) == 0,
 		 "Updating a missing property should have passed\n");
 
diff --git a/drivers/opp/core.c b/drivers/opp/core.c
index 92fa94a6dcc1..ab2f3fead6b1 100644
--- a/drivers/opp/core.c
+++ b/drivers/opp/core.c
@@ -33,8 +33,6 @@ LIST_HEAD(opp_tables);
 /* Lock to allow exclusive modification to the device and opp lists */
 DEFINE_MUTEX(opp_table_lock);
 
-static void dev_pm_opp_get(struct dev_pm_opp *opp);
-
 static struct opp_device *_find_opp_dev(const struct device *dev,
 					struct opp_table *opp_table)
 {
@@ -281,6 +279,23 @@ unsigned long dev_pm_opp_get_suspend_opp_freq(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_get_suspend_opp_freq);
 
+int _get_opp_count(struct opp_table *opp_table)
+{
+	struct dev_pm_opp *opp;
+	int count = 0;
+
+	mutex_lock(&opp_table->lock);
+
+	list_for_each_entry(opp, &opp_table->opp_list, node) {
+		if (opp->available)
+			count++;
+	}
+
+	mutex_unlock(&opp_table->lock);
+
+	return count;
+}
+
 /**
  * dev_pm_opp_get_opp_count() - Get number of opps available in the opp table
  * @dev:	device for which we do this operation
@@ -291,25 +306,17 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_get_suspend_opp_freq);
 int dev_pm_opp_get_opp_count(struct device *dev)
 {
 	struct opp_table *opp_table;
-	struct dev_pm_opp *temp_opp;
-	int count = 0;
+	int count;
 
 	opp_table = _find_opp_table(dev);
 	if (IS_ERR(opp_table)) {
 		count = PTR_ERR(opp_table);
 		dev_dbg(dev, "%s: OPP table not found (%d)\n",
 			__func__, count);
-		return count;
-	}
-
-	mutex_lock(&opp_table->lock);
-
-	list_for_each_entry(temp_opp, &opp_table->opp_list, node) {
-		if (temp_opp->available)
-			count++;
+		return 0;
 	}
 
-	mutex_unlock(&opp_table->lock);
+	count = _get_opp_count(opp_table);
 	dev_pm_opp_put_opp_table(opp_table);
 
 	return count;
@@ -892,7 +899,7 @@ static void _opp_kref_release(struct kref *kref)
 	dev_pm_opp_put_opp_table(opp_table);
 }
 
-static void dev_pm_opp_get(struct dev_pm_opp *opp)
+void dev_pm_opp_get(struct dev_pm_opp *opp)
 {
 	kref_get(&opp->kref);
 }
@@ -985,22 +992,11 @@ static bool _opp_supported_by_regulators(struct dev_pm_opp *opp,
 	return true;
 }
 
-/*
- * Returns:
- * 0: On success. And appropriate error message for duplicate OPPs.
- * -EBUSY: For OPP with same freq/volt and is available. The callers of
- *  _opp_add() must return 0 if they receive -EBUSY from it. This is to make
- *  sure we don't print error messages unnecessarily if different parts of
- *  kernel try to initialize the OPP table.
- * -EEXIST: For OPP with same freq but different volt or is unavailable. This
- *  should be considered an error by the callers of _opp_add().
- */
-int _opp_add(struct device *dev, struct dev_pm_opp *new_opp,
-	     struct opp_table *opp_table)
+static int _opp_is_duplicate(struct device *dev, struct dev_pm_opp *new_opp,
+			     struct opp_table *opp_table,
+			     struct list_head **head)
 {
 	struct dev_pm_opp *opp;
-	struct list_head *head;
-	int ret;
 
 	/*
 	 * Insert new OPP in order of increasing frequency and discard if
@@ -1010,17 +1006,14 @@ int _opp_add(struct device *dev, struct dev_pm_opp *new_opp,
 	 * loop, don't replace it with head otherwise it will become an infinite
 	 * loop.
 	 */
-	mutex_lock(&opp_table->lock);
-	head = &opp_table->opp_list;
-
 	list_for_each_entry(opp, &opp_table->opp_list, node) {
 		if (new_opp->rate > opp->rate) {
-			head = &opp->node;
+			*head = &opp->node;
 			continue;
 		}
 
 		if (new_opp->rate < opp->rate)
-			break;
+			return 0;
 
 		/* Duplicate OPPs */
 		dev_warn(dev, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n",
@@ -1029,15 +1022,39 @@ int _opp_add(struct device *dev, struct dev_pm_opp *new_opp,
 			 new_opp->supplies[0].u_volt, new_opp->available);
 
 		/* Should we compare voltages for all regulators here ? */
-		ret = opp->available &&
-		      new_opp->supplies[0].u_volt == opp->supplies[0].u_volt ? -EBUSY : -EEXIST;
-
-		mutex_unlock(&opp_table->lock);
-		return ret;
+		return opp->available &&
+		       new_opp->supplies[0].u_volt == opp->supplies[0].u_volt ? -EBUSY : -EEXIST;
 	}
 
-	if (opp_table->get_pstate)
-		new_opp->pstate = opp_table->get_pstate(dev, new_opp->rate);
+	return 0;
+}
+
+/*
+ * Returns:
+ * 0: On success. And appropriate error message for duplicate OPPs.
+ * -EBUSY: For OPP with same freq/volt and is available. The callers of
+ *  _opp_add() must return 0 if they receive -EBUSY from it. This is to make
+ *  sure we don't print error messages unnecessarily if different parts of
+ *  kernel try to initialize the OPP table.
+ * -EEXIST: For OPP with same freq but different volt or is unavailable. This
+ *  should be considered an error by the callers of _opp_add().
+ */
+int _opp_add(struct device *dev, struct dev_pm_opp *new_opp,
+	     struct opp_table *opp_table, bool rate_not_available)
+{
+	struct list_head *head;
+	int ret;
+
+	mutex_lock(&opp_table->lock);
+	head = &opp_table->opp_list;
+
+	if (likely(!rate_not_available)) {
+		ret = _opp_is_duplicate(dev, new_opp, opp_table, &head);
+		if (ret) {
+			mutex_unlock(&opp_table->lock);
+			return ret;
+		}
+	}
 
 	list_add(&new_opp->node, head);
 	mutex_unlock(&opp_table->lock);
@@ -1104,7 +1121,7 @@ int _opp_add_v1(struct opp_table *opp_table, struct device *dev,
 	new_opp->available = true;
 	new_opp->dynamic = dynamic;
 
-	ret = _opp_add(dev, new_opp, opp_table);
+	ret = _opp_add(dev, new_opp, opp_table, false);
 	if (ret) {
 		/* Don't return error for duplicate OPPs */
 		if (ret == -EBUSY)
@@ -1140,7 +1157,6 @@ struct opp_table *dev_pm_opp_set_supported_hw(struct device *dev,
 			const u32 *versions, unsigned int count)
 {
 	struct opp_table *opp_table;
-	int ret;
 
 	opp_table = dev_pm_opp_get_opp_table(dev);
 	if (!opp_table)
@@ -1149,29 +1165,20 @@ struct opp_table *dev_pm_opp_set_supported_hw(struct device *dev,
 	/* Make sure there are no concurrent readers while updating opp_table */
 	WARN_ON(!list_empty(&opp_table->opp_list));
 
-	/* Do we already have a version hierarchy associated with opp_table? */
-	if (opp_table->supported_hw) {
-		dev_err(dev, "%s: Already have supported hardware list\n",
-			__func__);
-		ret = -EBUSY;
-		goto err;
-	}
+	/* Another CPU that shares the OPP table has set the property ? */
+	if (opp_table->supported_hw)
+		return opp_table;
 
 	opp_table->supported_hw = kmemdup(versions, count * sizeof(*versions),
 					GFP_KERNEL);
 	if (!opp_table->supported_hw) {
-		ret = -ENOMEM;
-		goto err;
+		dev_pm_opp_put_opp_table(opp_table);
+		return ERR_PTR(-ENOMEM);
 	}
 
 	opp_table->supported_hw_count = count;
 
 	return opp_table;
-
-err:
-	dev_pm_opp_put_opp_table(opp_table);
-
-	return ERR_PTR(ret);
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_set_supported_hw);
 
@@ -1188,12 +1195,6 @@ void dev_pm_opp_put_supported_hw(struct opp_table *opp_table)
 	/* Make sure there are no concurrent readers while updating opp_table */
 	WARN_ON(!list_empty(&opp_table->opp_list));
 
-	if (!opp_table->supported_hw) {
-		pr_err("%s: Doesn't have supported hardware list\n",
-		       __func__);
-		return;
-	}
-
 	kfree(opp_table->supported_hw);
 	opp_table->supported_hw = NULL;
 	opp_table->supported_hw_count = 0;
@@ -1215,7 +1216,6 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_put_supported_hw);
 struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name)
 {
 	struct opp_table *opp_table;
-	int ret;
 
 	opp_table = dev_pm_opp_get_opp_table(dev);
 	if (!opp_table)
@@ -1224,26 +1224,17 @@ struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name)
 	/* Make sure there are no concurrent readers while updating opp_table */
 	WARN_ON(!list_empty(&opp_table->opp_list));
 
-	/* Do we already have a prop-name associated with opp_table? */
-	if (opp_table->prop_name) {
-		dev_err(dev, "%s: Already have prop-name %s\n", __func__,
-			opp_table->prop_name);
-		ret = -EBUSY;
-		goto err;
-	}
+	/* Another CPU that shares the OPP table has set the property ? */
+	if (opp_table->prop_name)
+		return opp_table;
 
 	opp_table->prop_name = kstrdup(name, GFP_KERNEL);
 	if (!opp_table->prop_name) {
-		ret = -ENOMEM;
-		goto err;
+		dev_pm_opp_put_opp_table(opp_table);
+		return ERR_PTR(-ENOMEM);
 	}
 
 	return opp_table;
-
-err:
-	dev_pm_opp_put_opp_table(opp_table);
-
-	return ERR_PTR(ret);
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_set_prop_name);
 
@@ -1260,11 +1251,6 @@ void dev_pm_opp_put_prop_name(struct opp_table *opp_table)
 	/* Make sure there are no concurrent readers while updating opp_table */
 	WARN_ON(!list_empty(&opp_table->opp_list));
 
-	if (!opp_table->prop_name) {
-		pr_err("%s: Doesn't have a prop-name\n", __func__);
-		return;
-	}
-
 	kfree(opp_table->prop_name);
 	opp_table->prop_name = NULL;
 
@@ -1334,11 +1320,9 @@ struct opp_table *dev_pm_opp_set_regulators(struct device *dev,
 		goto err;
 	}
 
-	/* Already have regulators set */
-	if (opp_table->regulators) {
-		ret = -EBUSY;
-		goto err;
-	}
+	/* Another CPU that shares the OPP table has set the regulators ? */
+	if (opp_table->regulators)
+		return opp_table;
 
 	opp_table->regulators = kmalloc_array(count,
 					      sizeof(*opp_table->regulators),
@@ -1392,10 +1376,8 @@ void dev_pm_opp_put_regulators(struct opp_table *opp_table)
 {
 	int i;
 
-	if (!opp_table->regulators) {
-		pr_err("%s: Doesn't have regulators set\n", __func__);
-		return;
-	}
+	if (!opp_table->regulators)
+		goto put_opp_table;
 
 	/* Make sure there are no concurrent readers while updating opp_table */
 	WARN_ON(!list_empty(&opp_table->opp_list));
@@ -1409,6 +1391,7 @@ void dev_pm_opp_put_regulators(struct opp_table *opp_table)
 	opp_table->regulators = NULL;
 	opp_table->regulator_count = 0;
 
+put_opp_table:
 	dev_pm_opp_put_opp_table(opp_table);
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_put_regulators);
@@ -1494,7 +1477,6 @@ struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev,
 			int (*set_opp)(struct dev_pm_set_opp_data *data))
 {
 	struct opp_table *opp_table;
-	int ret;
 
 	if (!set_opp)
 		return ERR_PTR(-EINVAL);
@@ -1505,24 +1487,15 @@ struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev,
 
 	/* This should be called before OPPs are initialized */
 	if (WARN_ON(!list_empty(&opp_table->opp_list))) {
-		ret = -EBUSY;
-		goto err;
-	}
-
-	/* Already have custom set_opp helper */
-	if (WARN_ON(opp_table->set_opp)) {
-		ret = -EBUSY;
-		goto err;
+		dev_pm_opp_put_opp_table(opp_table);
+		return ERR_PTR(-EBUSY);
 	}
 
-	opp_table->set_opp = set_opp;
+	/* Another CPU that shares the OPP table has set the helper ? */
+	if (!opp_table->set_opp)
+		opp_table->set_opp = set_opp;
 
 	return opp_table;
-
-err:
-	dev_pm_opp_put_opp_table(opp_table);
-
-	return ERR_PTR(ret);
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_register_set_opp_helper);
 
@@ -1535,96 +1508,14 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_register_set_opp_helper);
  */
 void dev_pm_opp_unregister_set_opp_helper(struct opp_table *opp_table)
 {
-	if (!opp_table->set_opp) {
-		pr_err("%s: Doesn't have custom set_opp helper set\n",
-		       __func__);
-		return;
-	}
-
 	/* Make sure there are no concurrent readers while updating opp_table */
 	WARN_ON(!list_empty(&opp_table->opp_list));
 
 	opp_table->set_opp = NULL;
-
 	dev_pm_opp_put_opp_table(opp_table);
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_unregister_set_opp_helper);
 
-/**
- * dev_pm_opp_register_get_pstate_helper() - Register get_pstate() helper.
- * @dev: Device for which the helper is getting registered.
- * @get_pstate: Helper.
- *
- * TODO: Remove this callback after the same information is available via Device
- * Tree.
- *
- * This allows a platform to initialize the performance states of individual
- * OPPs for its devices, until we get similar information directly from DT.
- *
- * This must be called before the OPPs are initialized for the device.
- */
-struct opp_table *dev_pm_opp_register_get_pstate_helper(struct device *dev,
-		int (*get_pstate)(struct device *dev, unsigned long rate))
-{
-	struct opp_table *opp_table;
-	int ret;
-
-	if (!get_pstate)
-		return ERR_PTR(-EINVAL);
-
-	opp_table = dev_pm_opp_get_opp_table(dev);
-	if (!opp_table)
-		return ERR_PTR(-ENOMEM);
-
-	/* This should be called before OPPs are initialized */
-	if (WARN_ON(!list_empty(&opp_table->opp_list))) {
-		ret = -EBUSY;
-		goto err;
-	}
-
-	/* Already have genpd_performance_state set */
-	if (WARN_ON(opp_table->genpd_performance_state)) {
-		ret = -EBUSY;
-		goto err;
-	}
-
-	opp_table->genpd_performance_state = true;
-	opp_table->get_pstate = get_pstate;
-
-	return opp_table;
-
-err:
-	dev_pm_opp_put_opp_table(opp_table);
-
-	return ERR_PTR(ret);
-}
-EXPORT_SYMBOL_GPL(dev_pm_opp_register_get_pstate_helper);
-
-/**
- * dev_pm_opp_unregister_get_pstate_helper() - Releases resources blocked for
- *					   get_pstate() helper
- * @opp_table: OPP table returned from dev_pm_opp_register_get_pstate_helper().
- *
- * Release resources blocked for platform specific get_pstate() helper.
- */
-void dev_pm_opp_unregister_get_pstate_helper(struct opp_table *opp_table)
-{
-	if (!opp_table->genpd_performance_state) {
-		pr_err("%s: Doesn't have performance states set\n",
-		       __func__);
-		return;
-	}
-
-	/* Make sure there are no concurrent readers while updating opp_table */
-	WARN_ON(!list_empty(&opp_table->opp_list));
-
-	opp_table->genpd_performance_state = false;
-	opp_table->get_pstate = NULL;
-
-	dev_pm_opp_put_opp_table(opp_table);
-}
-EXPORT_SYMBOL_GPL(dev_pm_opp_unregister_get_pstate_helper);
-
 /**
  * dev_pm_opp_add()  - Add an OPP table from a table definitions
  * @dev:	device for which we do this operation
diff --git a/drivers/opp/debugfs.c b/drivers/opp/debugfs.c
index b03c03576a62..e6828e5f81b0 100644
--- a/drivers/opp/debugfs.c
+++ b/drivers/opp/debugfs.c
@@ -77,10 +77,21 @@ int opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table)
 {
 	struct dentry *pdentry = opp_table->dentry;
 	struct dentry *d;
+	unsigned long id;
 	char name[25];	/* 20 chars for 64 bit value + 5 (opp:\0) */
 
-	/* Rate is unique to each OPP, use it to give opp-name */
-	snprintf(name, sizeof(name), "opp:%lu", opp->rate);
+	/*
+	 * Get directory name for OPP.
+	 *
+	 * - Normally rate is unique to each OPP, use it to get unique opp-name.
+	 * - For some devices rate isn't available, use index instead.
+	 */
+	if (likely(opp->rate))
+		id = opp->rate;
+	else
+		id = _get_opp_count(opp_table);
+
+	snprintf(name, sizeof(name), "opp:%lu", id);
 
 	/* Create per-opp directory */
 	d = debugfs_create_dir(name, pdentry);
diff --git a/drivers/opp/of.c b/drivers/opp/of.c
index cb716aa2f44b..7af0ddec936b 100644
--- a/drivers/opp/of.c
+++ b/drivers/opp/of.c
@@ -17,6 +17,7 @@
 #include <linux/errno.h>
 #include <linux/device.h>
 #include <linux/of_device.h>
+#include <linux/pm_domain.h>
 #include <linux/slab.h>
 #include <linux/export.h>
 
@@ -250,20 +251,17 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_of_remove_table);
 
 /* Returns opp descriptor node for a device node, caller must
  * do of_node_put() */
-static struct device_node *_opp_of_get_opp_desc_node(struct device_node *np)
+static struct device_node *_opp_of_get_opp_desc_node(struct device_node *np,
+						     int index)
 {
-	/*
-	 * There should be only ONE phandle present in "operating-points-v2"
-	 * property.
-	 */
-
-	return of_parse_phandle(np, "operating-points-v2", 0);
+	/* "operating-points-v2" can be an array for power domain providers */
+	return of_parse_phandle(np, "operating-points-v2", index);
 }
 
 /* Returns opp descriptor node for a device, caller must do of_node_put() */
 struct device_node *dev_pm_opp_of_get_opp_desc_node(struct device *dev)
 {
-	return _opp_of_get_opp_desc_node(dev->of_node);
+	return _opp_of_get_opp_desc_node(dev->of_node, 0);
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_of_get_opp_desc_node);
 
@@ -289,9 +287,10 @@ static int _opp_add_static_v2(struct opp_table *opp_table, struct device *dev,
 			      struct device_node *np)
 {
 	struct dev_pm_opp *new_opp;
-	u64 rate;
+	u64 rate = 0;
 	u32 val;
 	int ret;
+	bool rate_not_available = false;
 
 	new_opp = _opp_allocate(opp_table);
 	if (!new_opp)
@@ -299,8 +298,21 @@ static int _opp_add_static_v2(struct opp_table *opp_table, struct device *dev,
 
 	ret = of_property_read_u64(np, "opp-hz", &rate);
 	if (ret < 0) {
-		dev_err(dev, "%s: opp-hz not found\n", __func__);
-		goto free_opp;
+		/* "opp-hz" is optional for devices like power domains. */
+		if (!of_find_property(dev->of_node, "#power-domain-cells",
+				      NULL)) {
+			dev_err(dev, "%s: opp-hz not found\n", __func__);
+			goto free_opp;
+		}
+
+		rate_not_available = true;
+	} else {
+		/*
+		 * Rate is defined as an unsigned long in clk API, and so
+		 * casting explicitly to its type. Must be fixed once rate is 64
+		 * bit guaranteed in clk API.
+		 */
+		new_opp->rate = (unsigned long)rate;
 	}
 
 	/* Check if the OPP supports hardware's hierarchy of versions or not */
@@ -309,12 +321,6 @@ static int _opp_add_static_v2(struct opp_table *opp_table, struct device *dev,
 		goto free_opp;
 	}
 
-	/*
-	 * Rate is defined as an unsigned long in clk API, and so casting
-	 * explicitly to its type. Must be fixed once rate is 64 bit
-	 * guaranteed in clk API.
-	 */
-	new_opp->rate = (unsigned long)rate;
 	new_opp->turbo = of_property_read_bool(np, "turbo-mode");
 
 	new_opp->np = np;
@@ -324,11 +330,13 @@ static int _opp_add_static_v2(struct opp_table *opp_table, struct device *dev,
 	if (!of_property_read_u32(np, "clock-latency-ns", &val))
 		new_opp->clock_latency_ns = val;
 
+	new_opp->pstate = of_genpd_opp_to_performance_state(dev, np);
+
 	ret = opp_parse_supplies(new_opp, dev, opp_table);
 	if (ret)
 		goto free_opp;
 
-	ret = _opp_add(dev, new_opp, opp_table);
+	ret = _opp_add(dev, new_opp, opp_table, rate_not_available);
 	if (ret) {
 		/* Don't return error for duplicate OPPs */
 		if (ret == -EBUSY)
@@ -374,7 +382,8 @@ static int _of_add_opp_table_v2(struct device *dev, struct device_node *opp_np)
 {
 	struct device_node *np;
 	struct opp_table *opp_table;
-	int ret = 0, count = 0;
+	int ret = 0, count = 0, pstate_count = 0;
+	struct dev_pm_opp *opp;
 
 	opp_table = _managed_opp(opp_np);
 	if (opp_table) {
@@ -408,6 +417,20 @@ static int _of_add_opp_table_v2(struct device *dev, struct device_node *opp_np)
 		goto put_opp_table;
 	}
 
+	list_for_each_entry(opp, &opp_table->opp_list, node)
+		pstate_count += !!opp->pstate;
+
+	/* Either all or none of the nodes shall have performance state set */
+	if (pstate_count && pstate_count != count) {
+		dev_err(dev, "Not all nodes have performance state set (%d: %d)\n",
+			count, pstate_count);
+		ret = -ENOENT;
+		goto put_opp_table;
+	}
+
+	if (pstate_count)
+		opp_table->genpd_performance_state = true;
+
 	opp_table->np = opp_np;
 	if (of_property_read_bool(opp_np, "opp-shared"))
 		opp_table->shared_opp = OPP_TABLE_ACCESS_SHARED;
@@ -509,6 +532,54 @@ int dev_pm_opp_of_add_table(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_of_add_table);
 
+/**
+ * dev_pm_opp_of_add_table_indexed() - Initialize indexed opp table from device tree
+ * @dev:	device pointer used to lookup OPP table.
+ * @index:	Index number.
+ *
+ * Register the initial OPP table with the OPP library for given device only
+ * using the "operating-points-v2" property.
+ *
+ * Return:
+ * 0		On success OR
+ *		Duplicate OPPs (both freq and volt are same) and opp->available
+ * -EEXIST	Freq are same and volt are different OR
+ *		Duplicate OPPs (both freq and volt are same) and !opp->available
+ * -ENOMEM	Memory allocation failure
+ * -ENODEV	when 'operating-points' property is not found or is invalid data
+ *		in device node.
+ * -ENODATA	when empty 'operating-points' property is found
+ * -EINVAL	when invalid entries are found in opp-v2 table
+ */
+int dev_pm_opp_of_add_table_indexed(struct device *dev, int index)
+{
+	struct device_node *opp_np;
+	int ret, count;
+
+again:
+	opp_np = _opp_of_get_opp_desc_node(dev->of_node, index);
+	if (!opp_np) {
+		/*
+		 * If only one phandle is present, then the same OPP table
+		 * applies for all index requests.
+		 */
+		count = of_count_phandle_with_args(dev->of_node,
+						   "operating-points-v2", NULL);
+		if (count == 1 && index) {
+			index = 0;
+			goto again;
+		}
+
+		return -ENODEV;
+	}
+
+	ret = _of_add_opp_table_v2(dev, opp_np);
+	of_node_put(opp_np);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dev_pm_opp_of_add_table_indexed);
+
 /* CPU device specific helpers */
 
 /**
@@ -613,7 +684,7 @@ int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev,
 		}
 
 		/* Get OPP descriptor node */
-		tmp_np = _opp_of_get_opp_desc_node(cpu_np);
+		tmp_np = _opp_of_get_opp_desc_node(cpu_np, 0);
 		of_node_put(cpu_np);
 		if (!tmp_np) {
 			pr_err("%pOF: Couldn't find opp node\n", cpu_np);
@@ -633,3 +704,76 @@ int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev,
 	return ret;
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_of_get_sharing_cpus);
+
+/**
+ * of_dev_pm_opp_find_required_opp() - Search for required OPP.
+ * @dev: The device whose OPP node is referenced by the 'np' DT node.
+ * @np: Node that contains the "required-opps" property.
+ *
+ * Returns the OPP of the device 'dev', whose phandle is present in the "np"
+ * node. Although the "required-opps" property supports having multiple
+ * phandles, this helper routine only parses the very first phandle in the list.
+ *
+ * Return: Matching opp, else returns ERR_PTR in case of error and should be
+ * handled using IS_ERR.
+ *
+ * The callers are required to call dev_pm_opp_put() for the returned OPP after
+ * use.
+ */
+struct dev_pm_opp *of_dev_pm_opp_find_required_opp(struct device *dev,
+						   struct device_node *np)
+{
+	struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ENODEV);
+	struct device_node *required_np;
+	struct opp_table *opp_table;
+
+	opp_table = _find_opp_table(dev);
+	if (IS_ERR(opp_table))
+		return ERR_CAST(opp_table);
+
+	required_np = of_parse_phandle(np, "required-opps", 0);
+	if (unlikely(!required_np)) {
+		dev_err(dev, "Unable to parse required-opps\n");
+		goto put_opp_table;
+	}
+
+	mutex_lock(&opp_table->lock);
+
+	list_for_each_entry(temp_opp, &opp_table->opp_list, node) {
+		if (temp_opp->available && temp_opp->np == required_np) {
+			opp = temp_opp;
+
+			/* Increment the reference count of OPP */
+			dev_pm_opp_get(opp);
+			break;
+		}
+	}
+
+	mutex_unlock(&opp_table->lock);
+
+	of_node_put(required_np);
+put_opp_table:
+	dev_pm_opp_put_opp_table(opp_table);
+
+	return opp;
+}
+EXPORT_SYMBOL_GPL(of_dev_pm_opp_find_required_opp);
+
+/**
+ * dev_pm_opp_get_of_node() - Gets the DT node corresponding to an opp
+ * @opp:	opp for which DT node has to be returned for
+ *
+ * Return: DT node corresponding to the opp, else 0 on success.
+ *
+ * The caller needs to put the node with of_node_put() after using it.
+ */
+struct device_node *dev_pm_opp_get_of_node(struct dev_pm_opp *opp)
+{
+	if (IS_ERR_OR_NULL(opp)) {
+		pr_err("%s: Invalid parameters\n", __func__);
+		return NULL;
+	}
+
+	return of_node_get(opp->np);
+}
+EXPORT_SYMBOL_GPL(dev_pm_opp_get_of_node);
diff --git a/drivers/opp/opp.h b/drivers/opp/opp.h
index 4d00061648a3..7c540fd063b2 100644
--- a/drivers/opp/opp.h
+++ b/drivers/opp/opp.h
@@ -140,7 +140,6 @@ enum opp_table_access {
  * @genpd_performance_state: Device's power domain support performance state.
  * @set_opp: Platform specific set_opp callback
  * @set_opp_data: Data to be passed to set_opp callback
- * @get_pstate: Platform specific get_pstate callback
  * @dentry:	debugfs dentry pointer of the real device directory (not links).
  * @dentry_name: Name of the real dentry.
  *
@@ -178,7 +177,6 @@ struct opp_table {
 
 	int (*set_opp)(struct dev_pm_set_opp_data *data);
 	struct dev_pm_set_opp_data *set_opp_data;
-	int (*get_pstate)(struct device *dev, unsigned long rate);
 
 #ifdef CONFIG_DEBUG_FS
 	struct dentry *dentry;
@@ -187,14 +185,16 @@ struct opp_table {
 };
 
 /* Routines internal to opp core */
+void dev_pm_opp_get(struct dev_pm_opp *opp);
 void _get_opp_table_kref(struct opp_table *opp_table);
+int _get_opp_count(struct opp_table *opp_table);
 struct opp_table *_find_opp_table(struct device *dev);
 struct opp_device *_add_opp_dev(const struct device *dev, struct opp_table *opp_table);
 void _dev_pm_opp_remove_table(struct opp_table *opp_table, struct device *dev, bool remove_all);
 void _dev_pm_opp_find_and_remove_table(struct device *dev, bool remove_all);
 struct dev_pm_opp *_opp_allocate(struct opp_table *opp_table);
 void _opp_free(struct dev_pm_opp *opp);
-int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, struct opp_table *opp_table);
+int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, struct opp_table *opp_table, bool rate_not_available);
 int _opp_add_v1(struct opp_table *opp_table, struct device *dev, unsigned long freq, long u_volt, bool dynamic);
 void _dev_pm_opp_cpumask_remove_table(const struct cpumask *cpumask, bool of);
 struct opp_table *_add_opp_table(struct device *dev);
diff --git a/drivers/parisc/Kconfig b/drivers/parisc/Kconfig
index 3a102a84d637..5a48b5606110 100644
--- a/drivers/parisc/Kconfig
+++ b/drivers/parisc/Kconfig
@@ -103,11 +103,6 @@ config IOMMU_SBA
 	depends on PCI_LBA
 	default PCI_LBA
 
-config IOMMU_HELPER
-	bool
-	depends on IOMMU_SBA || IOMMU_CCIO
-	default y
-
 source "drivers/pcmcia/Kconfig"
 
 endmenu
diff --git a/drivers/parisc/ccio-dma.c b/drivers/parisc/ccio-dma.c
index 297599fcbc32..614823617b8b 100644
--- a/drivers/parisc/ccio-dma.c
+++ b/drivers/parisc/ccio-dma.c
@@ -1108,19 +1108,6 @@ static int ccio_proc_info(struct seq_file *m, void *p)
 	return 0;
 }
 
-static int ccio_proc_info_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, &ccio_proc_info, NULL);
-}
-
-static const struct file_operations ccio_proc_info_fops = {
-	.owner = THIS_MODULE,
-	.open = ccio_proc_info_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-
 static int ccio_proc_bitmap_info(struct seq_file *m, void *p)
 {
 	struct ioc *ioc = ioc_list;
@@ -1135,19 +1122,6 @@ static int ccio_proc_bitmap_info(struct seq_file *m, void *p)
 
 	return 0;
 }
-
-static int ccio_proc_bitmap_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, &ccio_proc_bitmap_info, NULL);
-}
-
-static const struct file_operations ccio_proc_bitmap_fops = {
-	.owner = THIS_MODULE,
-	.open = ccio_proc_bitmap_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
 #endif /* CONFIG_PROC_FS */
 
 /**
@@ -1589,15 +1563,13 @@ static int __init ccio_probe(struct parisc_device *dev)
 
 #ifdef CONFIG_PROC_FS
 	if (ioc_count == 0) {
-		proc_create(MODULE_NAME, 0, proc_runway_root,
-			    &ccio_proc_info_fops);
-		proc_create(MODULE_NAME"-bitmap", 0, proc_runway_root,
-			    &ccio_proc_bitmap_fops);
+		proc_create_single(MODULE_NAME, 0, proc_runway_root,
+				ccio_proc_info);
+		proc_create_single(MODULE_NAME"-bitmap", 0, proc_runway_root,
+				ccio_proc_bitmap_info);
 	}
 #endif
 	ioc_count++;
-
-	parisc_has_iommu();
 	return 0;
 }
 
diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c
index 0a9c762a70fa..11de0eccf968 100644
--- a/drivers/parisc/sba_iommu.c
+++ b/drivers/parisc/sba_iommu.c
@@ -1863,20 +1863,6 @@ static int sba_proc_info(struct seq_file *m, void *p)
 	return 0;
 }
 
-static int
-sba_proc_open(struct inode *i, struct file *f)
-{
-	return single_open(f, &sba_proc_info, NULL);
-}
-
-static const struct file_operations sba_proc_fops = {
-	.owner = THIS_MODULE,
-	.open = sba_proc_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-
 static int
 sba_proc_bitmap_info(struct seq_file *m, void *p)
 {
@@ -1889,20 +1875,6 @@ sba_proc_bitmap_info(struct seq_file *m, void *p)
 
 	return 0;
 }
-
-static int
-sba_proc_bitmap_open(struct inode *i, struct file *f)
-{
-	return single_open(f, &sba_proc_bitmap_info, NULL);
-}
-
-static const struct file_operations sba_proc_bitmap_fops = {
-	.owner = THIS_MODULE,
-	.open = sba_proc_bitmap_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
 #endif /* CONFIG_PROC_FS */
 
 static const struct parisc_device_id sba_tbl[] __initconst = {
@@ -2014,11 +1986,9 @@ static int __init sba_driver_callback(struct parisc_device *dev)
 		break;
 	}
 
-	proc_create("sba_iommu", 0, root, &sba_proc_fops);
-	proc_create("sba_iommu-bitmap", 0, root, &sba_proc_bitmap_fops);
+	proc_create_single("sba_iommu", 0, root, sba_proc_info);
+	proc_create_single("sba_iommu-bitmap", 0, root, sba_proc_bitmap_info);
 #endif
-
-	parisc_has_iommu();
 	return 0;
 }
 
diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
index 34b56a8f8480..b2f07635e94d 100644
--- a/drivers/pci/Kconfig
+++ b/drivers/pci/Kconfig
@@ -5,10 +5,6 @@
 
 source "drivers/pci/pcie/Kconfig"
 
-config PCI_BUS_ADDR_T_64BIT
-	def_bool y if (ARCH_DMA_ADDR_T_64BIT || 64BIT)
-	depends on PCI
-
 config PCI_MSI
 	bool "Message Signaled Interrupts (MSI and MSI-X)"
 	depends on PCI
@@ -71,6 +67,18 @@ config PCI_STUB
 
 	  When in doubt, say N.
 
+config PCI_PF_STUB
+	tristate "PCI PF Stub driver"
+	depends on PCI
+	depends on PCI_IOV
+	help
+	  Say Y or M here if you want to enable support for devices that
+	  require SR-IOV support, while at the same time the PF itself is
+	  not providing any actual services on the host itself such as
+	  storage or networking.
+
+	  When in doubt, say N.
+
 config XEN_PCIDEV_FRONTEND
         tristate "Xen PCI Frontend"
         depends on PCI && X86 && XEN
diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile
index 952addc7bacf..84c9eef6b1c3 100644
--- a/drivers/pci/Makefile
+++ b/drivers/pci/Makefile
@@ -24,6 +24,7 @@ obj-$(CONFIG_PCI_LABEL)		+= pci-label.o
 obj-$(CONFIG_X86_INTEL_MID)	+= pci-mid.o
 obj-$(CONFIG_PCI_SYSCALL)	+= syscall.o
 obj-$(CONFIG_PCI_STUB)		+= pci-stub.o
+obj-$(CONFIG_PCI_PF_STUB)	+= pci-pf-stub.o
 obj-$(CONFIG_PCI_ECAM)		+= ecam.o
 obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o
 
diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c
index 89305b569d3d..4923a2a8e14b 100644
--- a/drivers/pci/ats.c
+++ b/drivers/pci/ats.c
@@ -20,6 +20,9 @@ void pci_ats_init(struct pci_dev *dev)
 {
 	int pos;
 
+	if (pci_ats_disabled())
+		return;
+
 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
 	if (!pos)
 		return;
diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c
index bc2ded4c451f..35b7fc87eac5 100644
--- a/drivers/pci/bus.c
+++ b/drivers/pci/bus.c
@@ -120,7 +120,7 @@ int devm_request_pci_bus_resources(struct device *dev,
 EXPORT_SYMBOL_GPL(devm_request_pci_bus_resources);
 
 static struct pci_bus_region pci_32_bit = {0, 0xffffffffULL};
-#ifdef CONFIG_PCI_BUS_ADDR_T_64BIT
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 static struct pci_bus_region pci_64_bit = {0,
 				(pci_bus_addr_t) 0xffffffffffffffffULL};
 static struct pci_bus_region pci_high = {(pci_bus_addr_t) 0x100000000ULL,
@@ -230,7 +230,7 @@ int pci_bus_alloc_resource(struct pci_bus *bus, struct resource *res,
 					  resource_size_t),
 		void *alignf_data)
 {
-#ifdef CONFIG_PCI_BUS_ADDR_T_64BIT
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 	int rc;
 
 	if (res->flags & IORESOURCE_MEM_64) {
diff --git a/drivers/pci/dwc/Kconfig b/drivers/pci/dwc/Kconfig
index 2f3f5c50aa48..16f52c626b4b 100644
--- a/drivers/pci/dwc/Kconfig
+++ b/drivers/pci/dwc/Kconfig
@@ -1,13 +1,13 @@
 # SPDX-License-Identifier: GPL-2.0
 
 menu "DesignWare PCI Core Support"
+	depends on PCI
 
 config PCIE_DW
 	bool
 
 config PCIE_DW_HOST
         bool
-	depends on PCI
 	depends on PCI_MSI_IRQ_DOMAIN
         select PCIE_DW
 
@@ -22,7 +22,7 @@ config PCI_DRA7XX
 config PCI_DRA7XX_HOST
 	bool "TI DRA7xx PCIe controller Host Mode"
 	depends on SOC_DRA7XX || COMPILE_TEST
-	depends on PCI && PCI_MSI_IRQ_DOMAIN
+	depends on PCI_MSI_IRQ_DOMAIN
 	depends on OF && HAS_IOMEM && TI_PIPE3
 	select PCIE_DW_HOST
 	select PCI_DRA7XX
@@ -51,50 +51,62 @@ config PCI_DRA7XX_EP
 	  This uses the DesignWare core.
 
 config PCIE_DW_PLAT
-	bool "Platform bus based DesignWare PCIe Controller"
-	depends on PCI
-	depends on PCI_MSI_IRQ_DOMAIN
-	select PCIE_DW_HOST
-	---help---
-	 This selects the DesignWare PCIe controller support. Select this if
-	 you have a PCIe controller on Platform bus.
+	bool
 
-	 If you have a controller with this interface, say Y or M here.
+config PCIE_DW_PLAT_HOST
+	bool "Platform bus based DesignWare PCIe Controller - Host mode"
+	depends on PCI && PCI_MSI_IRQ_DOMAIN
+	select PCIE_DW_HOST
+	select PCIE_DW_PLAT
+	default y
+	help
+	  Enables support for the PCIe controller in the Designware IP to
+	  work in host mode. There are two instances of PCIe controller in
+	  Designware IP.
+	  This controller can work either as EP or RC. In order to enable
+	  host-specific features PCIE_DW_PLAT_HOST must be selected and in
+	  order to enable device-specific features PCI_DW_PLAT_EP must be
+	  selected.
 
-	 If unsure, say N.
+config PCIE_DW_PLAT_EP
+	bool "Platform bus based DesignWare PCIe Controller - Endpoint mode"
+	depends on PCI && PCI_MSI_IRQ_DOMAIN
+	depends on PCI_ENDPOINT
+	select PCIE_DW_EP
+	select PCIE_DW_PLAT
+	help
+	  Enables support for the PCIe controller in the Designware IP to
+	  work in endpoint mode. There are two instances of PCIe controller
+	  in Designware IP.
+	  This controller can work either as EP or RC. In order to enable
+	  host-specific features PCIE_DW_PLAT_HOST must be selected and in
+	  order to enable device-specific features PCI_DW_PLAT_EP must be
+	  selected.
 
 config PCI_EXYNOS
 	bool "Samsung Exynos PCIe controller"
-	depends on PCI
-	depends on SOC_EXYNOS5440
+	depends on SOC_EXYNOS5440 || COMPILE_TEST
 	depends on PCI_MSI_IRQ_DOMAIN
-	select PCIEPORTBUS
 	select PCIE_DW_HOST
 
 config PCI_IMX6
 	bool "Freescale i.MX6 PCIe controller"
-	depends on PCI
-	depends on SOC_IMX6Q
+	depends on SOC_IMX6Q || (ARM && COMPILE_TEST)
 	depends on PCI_MSI_IRQ_DOMAIN
-	select PCIEPORTBUS
 	select PCIE_DW_HOST
 
 config PCIE_SPEAR13XX
 	bool "STMicroelectronics SPEAr PCIe controller"
-	depends on PCI
-	depends on ARCH_SPEAR13XX
+	depends on ARCH_SPEAR13XX || COMPILE_TEST
 	depends on PCI_MSI_IRQ_DOMAIN
-	select PCIEPORTBUS
 	select PCIE_DW_HOST
 	help
 	  Say Y here if you want PCIe support on SPEAr13XX SoCs.
 
 config PCI_KEYSTONE
 	bool "TI Keystone PCIe controller"
-	depends on PCI
-	depends on ARCH_KEYSTONE
+	depends on ARCH_KEYSTONE || (ARM && COMPILE_TEST)
 	depends on PCI_MSI_IRQ_DOMAIN
-	select PCIEPORTBUS
 	select PCIE_DW_HOST
 	help
 	  Say Y here if you want to enable PCI controller support on Keystone
@@ -104,8 +116,7 @@ config PCI_KEYSTONE
 
 config PCI_LAYERSCAPE
 	bool "Freescale Layerscape PCIe controller"
-	depends on PCI
-	depends on OF && (ARM || ARCH_LAYERSCAPE)
+	depends on OF && (ARM || ARCH_LAYERSCAPE || COMPILE_TEST)
 	depends on PCI_MSI_IRQ_DOMAIN
 	select MFD_SYSCON
 	select PCIE_DW_HOST
@@ -113,11 +124,9 @@ config PCI_LAYERSCAPE
 	  Say Y here if you want PCIe controller support on Layerscape SoCs.
 
 config PCI_HISI
-	depends on OF && ARM64
+	depends on OF && (ARM64 || COMPILE_TEST)
 	bool "HiSilicon Hip05 and Hip06 SoCs PCIe controllers"
-	depends on PCI
 	depends on PCI_MSI_IRQ_DOMAIN
-	select PCIEPORTBUS
 	select PCIE_DW_HOST
 	select PCI_HOST_COMMON
 	help
@@ -126,10 +135,8 @@ config PCI_HISI
 
 config PCIE_QCOM
 	bool "Qualcomm PCIe controller"
-	depends on PCI
-	depends on ARCH_QCOM && OF
+	depends on OF && (ARCH_QCOM || COMPILE_TEST)
 	depends on PCI_MSI_IRQ_DOMAIN
-	select PCIEPORTBUS
 	select PCIE_DW_HOST
 	help
 	  Say Y here to enable PCIe controller support on Qualcomm SoCs. The
@@ -138,10 +145,8 @@ config PCIE_QCOM
 
 config PCIE_ARMADA_8K
 	bool "Marvell Armada-8K PCIe controller"
-	depends on PCI
-	depends on ARCH_MVEBU
+	depends on ARCH_MVEBU || COMPILE_TEST
 	depends on PCI_MSI_IRQ_DOMAIN
-	select PCIEPORTBUS
 	select PCIE_DW_HOST
 	help
 	  Say Y here if you want to enable PCIe controller support on
@@ -154,9 +159,8 @@ config PCIE_ARTPEC6
 
 config PCIE_ARTPEC6_HOST
 	bool "Axis ARTPEC-6 PCIe controller Host Mode"
-	depends on MACH_ARTPEC6
-	depends on PCI && PCI_MSI_IRQ_DOMAIN
-	select PCIEPORTBUS
+	depends on MACH_ARTPEC6 || COMPILE_TEST
+	depends on PCI_MSI_IRQ_DOMAIN
 	select PCIE_DW_HOST
 	select PCIE_ARTPEC6
 	help
@@ -165,7 +169,7 @@ config PCIE_ARTPEC6_HOST
 
 config PCIE_ARTPEC6_EP
 	bool "Axis ARTPEC-6 PCIe controller Endpoint Mode"
-	depends on MACH_ARTPEC6
+	depends on MACH_ARTPEC6 || COMPILE_TEST
 	depends on PCI_ENDPOINT
 	select PCIE_DW_EP
 	select PCIE_ARTPEC6
@@ -174,11 +178,9 @@ config PCIE_ARTPEC6_EP
 	  endpoint mode. This uses the DesignWare core.
 
 config PCIE_KIRIN
-	depends on OF && ARM64
+	depends on OF && (ARM64 || COMPILE_TEST)
 	bool "HiSilicon Kirin series SoCs PCIe controllers"
 	depends on PCI_MSI_IRQ_DOMAIN
-	depends on PCI
-	select PCIEPORTBUS
 	select PCIE_DW_HOST
 	help
 	  Say Y here if you want PCIe controller support
@@ -186,10 +188,8 @@ config PCIE_KIRIN
 
 config PCIE_HISI_STB
 	bool "HiSilicon STB SoCs PCIe controllers"
-	depends on ARCH_HISI
-	depends on PCI
+	depends on ARCH_HISI || COMPILE_TEST
 	depends on PCI_MSI_IRQ_DOMAIN
-	select PCIEPORTBUS
 	select PCIE_DW_HOST
 	help
           Say Y here if you want PCIe controller support on HiSilicon STB SoCs
diff --git a/drivers/pci/dwc/pci-dra7xx.c b/drivers/pci/dwc/pci-dra7xx.c
index ed8558d638e5..f688204e50c5 100644
--- a/drivers/pci/dwc/pci-dra7xx.c
+++ b/drivers/pci/dwc/pci-dra7xx.c
@@ -27,6 +27,7 @@
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
 
+#include "../pci.h"
 #include "pcie-designware.h"
 
 /* PCIe controller wrapper DRA7XX configuration registers */
@@ -406,14 +407,14 @@ static int __init dra7xx_add_pcie_ep(struct dra7xx_pcie *dra7xx,
 	ep->ops = &pcie_ep_ops;
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ep_dbics");
-	pci->dbi_base = devm_ioremap(dev, res->start, resource_size(res));
-	if (!pci->dbi_base)
-		return -ENOMEM;
+	pci->dbi_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(pci->dbi_base))
+		return PTR_ERR(pci->dbi_base);
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ep_dbics2");
-	pci->dbi_base2 = devm_ioremap(dev, res->start, resource_size(res));
-	if (!pci->dbi_base2)
-		return -ENOMEM;
+	pci->dbi_base2 = devm_ioremap_resource(dev, res);
+	if (IS_ERR(pci->dbi_base2))
+		return PTR_ERR(pci->dbi_base2);
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space");
 	if (!res)
@@ -459,9 +460,9 @@ static int __init dra7xx_add_pcie_port(struct dra7xx_pcie *dra7xx,
 		return ret;
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rc_dbics");
-	pci->dbi_base = devm_ioremap(dev, res->start, resource_size(res));
-	if (!pci->dbi_base)
-		return -ENOMEM;
+	pci->dbi_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(pci->dbi_base))
+		return PTR_ERR(pci->dbi_base);
 
 	pp->ops = &dra7xx_pcie_host_ops;
 
diff --git a/drivers/pci/dwc/pci-imx6.c b/drivers/pci/dwc/pci-imx6.c
index 4818ef875f8a..80f604602783 100644
--- a/drivers/pci/dwc/pci-imx6.c
+++ b/drivers/pci/dwc/pci-imx6.c
@@ -338,7 +338,7 @@ static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie)
 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
 				   IMX6SX_GPR12_PCIE_TEST_POWERDOWN, 0);
 		break;
-	case IMX6QP: 		/* FALLTHROUGH */
+	case IMX6QP:		/* FALLTHROUGH */
 	case IMX6Q:
 		/* power up core phy and enable ref clock */
 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
diff --git a/drivers/pci/dwc/pci-keystone.c b/drivers/pci/dwc/pci-keystone.c
index d55ae0716adf..3722a5f31e5e 100644
--- a/drivers/pci/dwc/pci-keystone.c
+++ b/drivers/pci/dwc/pci-keystone.c
@@ -89,7 +89,7 @@ static int ks_pcie_establish_link(struct keystone_pcie *ks_pcie)
 	dw_pcie_setup_rc(pp);
 
 	if (dw_pcie_link_up(pci)) {
-		dev_err(dev, "Link already up\n");
+		dev_info(dev, "Link already up\n");
 		return 0;
 	}
 
diff --git a/drivers/pci/dwc/pcie-armada8k.c b/drivers/pci/dwc/pcie-armada8k.c
index b587352f8b9f..072fd7ecc29f 100644
--- a/drivers/pci/dwc/pcie-armada8k.c
+++ b/drivers/pci/dwc/pcie-armada8k.c
@@ -28,6 +28,7 @@
 struct armada8k_pcie {
 	struct dw_pcie *pci;
 	struct clk *clk;
+	struct clk *clk_reg;
 };
 
 #define PCIE_VENDOR_REGS_OFFSET		0x8000
@@ -229,26 +230,38 @@ static int armada8k_pcie_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	pcie->clk_reg = devm_clk_get(dev, "reg");
+	if (pcie->clk_reg == ERR_PTR(-EPROBE_DEFER)) {
+		ret = -EPROBE_DEFER;
+		goto fail;
+	}
+	if (!IS_ERR(pcie->clk_reg)) {
+		ret = clk_prepare_enable(pcie->clk_reg);
+		if (ret)
+			goto fail_clkreg;
+	}
+
 	/* Get the dw-pcie unit configuration/control registers base. */
 	base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ctrl");
 	pci->dbi_base = devm_pci_remap_cfg_resource(dev, base);
 	if (IS_ERR(pci->dbi_base)) {
 		dev_err(dev, "couldn't remap regs base %p\n", base);
 		ret = PTR_ERR(pci->dbi_base);
-		goto fail;
+		goto fail_clkreg;
 	}
 
 	platform_set_drvdata(pdev, pcie);
 
 	ret = armada8k_add_pcie_port(pcie, pdev);
 	if (ret)
-		goto fail;
+		goto fail_clkreg;
 
 	return 0;
 
+fail_clkreg:
+	clk_disable_unprepare(pcie->clk_reg);
 fail:
-	if (!IS_ERR(pcie->clk))
-		clk_disable_unprepare(pcie->clk);
+	clk_disable_unprepare(pcie->clk);
 
 	return ret;
 }
diff --git a/drivers/pci/dwc/pcie-artpec6.c b/drivers/pci/dwc/pcie-artpec6.c
index e66cede2b5b7..321b56cfd5d0 100644
--- a/drivers/pci/dwc/pcie-artpec6.c
+++ b/drivers/pci/dwc/pcie-artpec6.c
@@ -463,9 +463,9 @@ static int artpec6_add_pcie_ep(struct artpec6_pcie *artpec6_pcie,
 	ep->ops = &pcie_ep_ops;
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi2");
-	pci->dbi_base2 = devm_ioremap(dev, res->start, resource_size(res));
-	if (!pci->dbi_base2)
-		return -ENOMEM;
+	pci->dbi_base2 = devm_ioremap_resource(dev, res);
+	if (IS_ERR(pci->dbi_base2))
+		return PTR_ERR(pci->dbi_base2);
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space");
 	if (!res)
diff --git a/drivers/pci/dwc/pcie-designware-ep.c b/drivers/pci/dwc/pcie-designware-ep.c
index f07678bf7cfc..1eec4415a77f 100644
--- a/drivers/pci/dwc/pcie-designware-ep.c
+++ b/drivers/pci/dwc/pcie-designware-ep.c
@@ -75,7 +75,7 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar,
 
 	free_win = find_first_zero_bit(ep->ib_window_map, ep->num_ib_windows);
 	if (free_win >= ep->num_ib_windows) {
-		dev_err(pci->dev, "no free inbound window\n");
+		dev_err(pci->dev, "No free inbound window\n");
 		return -EINVAL;
 	}
 
@@ -100,7 +100,7 @@ static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr,
 
 	free_win = find_first_zero_bit(ep->ob_window_map, ep->num_ob_windows);
 	if (free_win >= ep->num_ob_windows) {
-		dev_err(pci->dev, "no free outbound window\n");
+		dev_err(pci->dev, "No free outbound window\n");
 		return -EINVAL;
 	}
 
@@ -204,7 +204,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no,
 
 	ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size);
 	if (ret) {
-		dev_err(pci->dev, "failed to enable address\n");
+		dev_err(pci->dev, "Failed to enable address\n");
 		return ret;
 	}
 
@@ -348,21 +348,21 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 
 	ret = of_property_read_u32(np, "num-ib-windows", &ep->num_ib_windows);
 	if (ret < 0) {
-		dev_err(dev, "unable to read *num-ib-windows* property\n");
+		dev_err(dev, "Unable to read *num-ib-windows* property\n");
 		return ret;
 	}
 	if (ep->num_ib_windows > MAX_IATU_IN) {
-		dev_err(dev, "invalid *num-ib-windows*\n");
+		dev_err(dev, "Invalid *num-ib-windows*\n");
 		return -EINVAL;
 	}
 
 	ret = of_property_read_u32(np, "num-ob-windows", &ep->num_ob_windows);
 	if (ret < 0) {
-		dev_err(dev, "unable to read *num-ob-windows* property\n");
+		dev_err(dev, "Unable to read *num-ob-windows* property\n");
 		return ret;
 	}
 	if (ep->num_ob_windows > MAX_IATU_OUT) {
-		dev_err(dev, "invalid *num-ob-windows*\n");
+		dev_err(dev, "Invalid *num-ob-windows*\n");
 		return -EINVAL;
 	}
 
@@ -389,7 +389,7 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 
 	epc = devm_pci_epc_create(dev, &epc_ops);
 	if (IS_ERR(epc)) {
-		dev_err(dev, "failed to create epc device\n");
+		dev_err(dev, "Failed to create epc device\n");
 		return PTR_ERR(epc);
 	}
 
@@ -411,6 +411,9 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 		return -ENOMEM;
 	}
 
+	epc->features = EPC_FEATURE_NO_LINKUP_NOTIFIER;
+	EPC_FEATURE_SET_BAR(epc->features, BAR_0);
+
 	ep->epc = epc;
 	epc_set_drvdata(epc, ep);
 	dw_pcie_setup(pci);
diff --git a/drivers/pci/dwc/pcie-designware-host.c b/drivers/pci/dwc/pcie-designware-host.c
index 6c409079d514..cba1432e395d 100644
--- a/drivers/pci/dwc/pcie-designware-host.c
+++ b/drivers/pci/dwc/pcie-designware-host.c
@@ -15,6 +15,7 @@
 #include <linux/pci_regs.h>
 #include <linux/platform_device.h>
 
+#include "../pci.h"
 #include "pcie-designware.h"
 
 static struct pci_ops dw_pcie_ops;
@@ -83,18 +84,23 @@ irqreturn_t dw_handle_msi_irq(struct pcie_port *pp)
 	num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
 
 	for (i = 0; i < num_ctrls; i++) {
-		dw_pcie_rd_own_conf(pp, PCIE_MSI_INTR0_STATUS + i * 12, 4,
-				    &val);
+		dw_pcie_rd_own_conf(pp, PCIE_MSI_INTR0_STATUS +
+					(i * MSI_REG_CTRL_BLOCK_SIZE),
+				    4, &val);
 		if (!val)
 			continue;
 
 		ret = IRQ_HANDLED;
 		pos = 0;
-		while ((pos = find_next_bit((unsigned long *) &val, 32,
-					    pos)) != 32) {
-			irq = irq_find_mapping(pp->irq_domain, i * 32 + pos);
+		while ((pos = find_next_bit((unsigned long *) &val,
+					    MAX_MSI_IRQS_PER_CTRL,
+					    pos)) != MAX_MSI_IRQS_PER_CTRL) {
+			irq = irq_find_mapping(pp->irq_domain,
+					       (i * MAX_MSI_IRQS_PER_CTRL) +
+					       pos);
 			generic_handle_irq(irq);
-			dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_STATUS + i * 12,
+			dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_STATUS +
+						(i * MSI_REG_CTRL_BLOCK_SIZE),
 					    4, 1 << pos);
 			pos++;
 		}
@@ -157,9 +163,9 @@ static void dw_pci_bottom_mask(struct irq_data *data)
 	if (pp->ops->msi_clear_irq) {
 		pp->ops->msi_clear_irq(pp, data->hwirq);
 	} else {
-		ctrl = data->hwirq / 32;
-		res = ctrl * 12;
-		bit = data->hwirq % 32;
+		ctrl = data->hwirq / MAX_MSI_IRQS_PER_CTRL;
+		res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
+		bit = data->hwirq % MAX_MSI_IRQS_PER_CTRL;
 
 		pp->irq_status[ctrl] &= ~(1 << bit);
 		dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_ENABLE + res, 4,
@@ -180,9 +186,9 @@ static void dw_pci_bottom_unmask(struct irq_data *data)
 	if (pp->ops->msi_set_irq) {
 		pp->ops->msi_set_irq(pp, data->hwirq);
 	} else {
-		ctrl = data->hwirq / 32;
-		res = ctrl * 12;
-		bit = data->hwirq % 32;
+		ctrl = data->hwirq / MAX_MSI_IRQS_PER_CTRL;
+		res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
+		bit = data->hwirq % MAX_MSI_IRQS_PER_CTRL;
 
 		pp->irq_status[ctrl] |= 1 << bit;
 		dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_ENABLE + res, 4,
@@ -248,8 +254,10 @@ static void dw_pcie_irq_domain_free(struct irq_domain *domain,
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&pp->lock, flags);
+
 	bitmap_release_region(pp->msi_irq_in_use, data->hwirq,
 			      order_base_2(nr_irqs));
+
 	raw_spin_unlock_irqrestore(&pp->lock, flags);
 }
 
@@ -266,7 +274,7 @@ int dw_pcie_allocate_domains(struct pcie_port *pp)
 	pp->irq_domain = irq_domain_create_linear(fwnode, pp->num_vectors,
 					       &dw_pcie_msi_domain_ops, pp);
 	if (!pp->irq_domain) {
-		dev_err(pci->dev, "failed to create IRQ domain\n");
+		dev_err(pci->dev, "Failed to create IRQ domain\n");
 		return -ENOMEM;
 	}
 
@@ -274,7 +282,7 @@ int dw_pcie_allocate_domains(struct pcie_port *pp)
 						   &dw_pcie_msi_domain_info,
 						   pp->irq_domain);
 	if (!pp->msi_domain) {
-		dev_err(pci->dev, "failed to create MSI domain\n");
+		dev_err(pci->dev, "Failed to create MSI domain\n");
 		irq_domain_remove(pp->irq_domain);
 		return -ENOMEM;
 	}
@@ -301,13 +309,13 @@ void dw_pcie_msi_init(struct pcie_port *pp)
 	page = alloc_page(GFP_KERNEL);
 	pp->msi_data = dma_map_page(dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
 	if (dma_mapping_error(dev, pp->msi_data)) {
-		dev_err(dev, "failed to map MSI data\n");
+		dev_err(dev, "Failed to map MSI data\n");
 		__free_page(page);
 		return;
 	}
 	msi_target = (u64)pp->msi_data;
 
-	/* program the msi_data */
+	/* Program the msi_data */
 	dw_pcie_wr_own_conf(pp, PCIE_MSI_ADDR_LO, 4,
 			    lower_32_bits(msi_target));
 	dw_pcie_wr_own_conf(pp, PCIE_MSI_ADDR_HI, 4,
@@ -330,19 +338,19 @@ int dw_pcie_host_init(struct pcie_port *pp)
 
 	cfg_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
 	if (cfg_res) {
-		pp->cfg0_size = resource_size(cfg_res) / 2;
-		pp->cfg1_size = resource_size(cfg_res) / 2;
+		pp->cfg0_size = resource_size(cfg_res) >> 1;
+		pp->cfg1_size = resource_size(cfg_res) >> 1;
 		pp->cfg0_base = cfg_res->start;
 		pp->cfg1_base = cfg_res->start + pp->cfg0_size;
 	} else if (!pp->va_cfg0_base) {
-		dev_err(dev, "missing *config* reg space\n");
+		dev_err(dev, "Missing *config* reg space\n");
 	}
 
 	bridge = pci_alloc_host_bridge(0);
 	if (!bridge)
 		return -ENOMEM;
 
-	ret = of_pci_get_host_bridge_resources(np, 0, 0xff,
+	ret = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff,
 					&bridge->windows, &pp->io_base);
 	if (ret)
 		return ret;
@@ -357,7 +365,7 @@ int dw_pcie_host_init(struct pcie_port *pp)
 		case IORESOURCE_IO:
 			ret = pci_remap_iospace(win->res, pp->io_base);
 			if (ret) {
-				dev_warn(dev, "error %d: failed to map resource %pR\n",
+				dev_warn(dev, "Error %d: failed to map resource %pR\n",
 					 ret, win->res);
 				resource_list_destroy_entry(win);
 			} else {
@@ -375,8 +383,8 @@ int dw_pcie_host_init(struct pcie_port *pp)
 			break;
 		case 0:
 			pp->cfg = win->res;
-			pp->cfg0_size = resource_size(pp->cfg) / 2;
-			pp->cfg1_size = resource_size(pp->cfg) / 2;
+			pp->cfg0_size = resource_size(pp->cfg) >> 1;
+			pp->cfg1_size = resource_size(pp->cfg) >> 1;
 			pp->cfg0_base = pp->cfg->start;
 			pp->cfg1_base = pp->cfg->start + pp->cfg0_size;
 			break;
@@ -391,7 +399,7 @@ int dw_pcie_host_init(struct pcie_port *pp)
 						pp->cfg->start,
 						resource_size(pp->cfg));
 		if (!pci->dbi_base) {
-			dev_err(dev, "error with ioremap\n");
+			dev_err(dev, "Error with ioremap\n");
 			ret = -ENOMEM;
 			goto error;
 		}
@@ -403,7 +411,7 @@ int dw_pcie_host_init(struct pcie_port *pp)
 		pp->va_cfg0_base = devm_pci_remap_cfgspace(dev,
 					pp->cfg0_base, pp->cfg0_size);
 		if (!pp->va_cfg0_base) {
-			dev_err(dev, "error with ioremap in function\n");
+			dev_err(dev, "Error with ioremap in function\n");
 			ret = -ENOMEM;
 			goto error;
 		}
@@ -414,7 +422,7 @@ int dw_pcie_host_init(struct pcie_port *pp)
 						pp->cfg1_base,
 						pp->cfg1_size);
 		if (!pp->va_cfg1_base) {
-			dev_err(dev, "error with ioremap\n");
+			dev_err(dev, "Error with ioremap\n");
 			ret = -ENOMEM;
 			goto error;
 		}
@@ -586,7 +594,7 @@ static int dw_pcie_valid_device(struct pcie_port *pp, struct pci_bus *bus,
 			return 0;
 	}
 
-	/* access only one slot on each root port */
+	/* Access only one slot on each root port */
 	if (bus->number == pp->root_bus_nr && dev > 0)
 		return 0;
 
@@ -650,13 +658,15 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 
 	/* Initialize IRQ Status array */
 	for (ctrl = 0; ctrl < num_ctrls; ctrl++)
-		dw_pcie_rd_own_conf(pp, PCIE_MSI_INTR0_ENABLE + (ctrl * 12), 4,
-				    &pp->irq_status[ctrl]);
-	/* setup RC BARs */
+		dw_pcie_rd_own_conf(pp, PCIE_MSI_INTR0_ENABLE +
+					(ctrl * MSI_REG_CTRL_BLOCK_SIZE),
+				    4, &pp->irq_status[ctrl]);
+
+	/* Setup RC BARs */
 	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0x00000004);
 	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0x00000000);
 
-	/* setup interrupt pins */
+	/* Setup interrupt pins */
 	dw_pcie_dbi_ro_wr_en(pci);
 	val = dw_pcie_readl_dbi(pci, PCI_INTERRUPT_LINE);
 	val &= 0xffff00ff;
@@ -664,13 +674,13 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 	dw_pcie_writel_dbi(pci, PCI_INTERRUPT_LINE, val);
 	dw_pcie_dbi_ro_wr_dis(pci);
 
-	/* setup bus numbers */
+	/* Setup bus numbers */
 	val = dw_pcie_readl_dbi(pci, PCI_PRIMARY_BUS);
 	val &= 0xff000000;
 	val |= 0x00ff0100;
 	dw_pcie_writel_dbi(pci, PCI_PRIMARY_BUS, val);
 
-	/* setup command register */
+	/* Setup command register */
 	val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
 	val &= 0xffff0000;
 	val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY |
@@ -683,7 +693,7 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 	 * we should not program the ATU here.
 	 */
 	if (!pp->ops->rd_other_conf) {
-		/* get iATU unroll support */
+		/* Get iATU unroll support */
 		pci->iatu_unroll_enabled = dw_pcie_iatu_unroll_enabled(pci);
 		dev_dbg(pci->dev, "iATU unroll: %s\n",
 			pci->iatu_unroll_enabled ? "enabled" : "disabled");
@@ -701,7 +711,7 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 
 	/* Enable write permission for the DBI read-only register */
 	dw_pcie_dbi_ro_wr_en(pci);
-	/* program correct class for RC */
+	/* Program correct class for RC */
 	dw_pcie_wr_own_conf(pp, PCI_CLASS_DEVICE, 2, PCI_CLASS_BRIDGE_PCI);
 	/* Better disable write permission right after the update */
 	dw_pcie_dbi_ro_wr_dis(pci);
diff --git a/drivers/pci/dwc/pcie-designware-plat.c b/drivers/pci/dwc/pcie-designware-plat.c
index 5416aa8a07a5..5937fed4c938 100644
--- a/drivers/pci/dwc/pcie-designware-plat.c
+++ b/drivers/pci/dwc/pcie-designware-plat.c
@@ -12,19 +12,29 @@
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
+#include <linux/of_device.h>
 #include <linux/of_gpio.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
 #include <linux/resource.h>
 #include <linux/signal.h>
 #include <linux/types.h>
+#include <linux/regmap.h>
 
 #include "pcie-designware.h"
 
 struct dw_plat_pcie {
-	struct dw_pcie		*pci;
+	struct dw_pcie			*pci;
+	struct regmap			*regmap;
+	enum dw_pcie_device_mode	mode;
 };
 
+struct dw_plat_pcie_of_data {
+	enum dw_pcie_device_mode	mode;
+};
+
+static const struct of_device_id dw_plat_pcie_of_match[];
+
 static int dw_plat_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
@@ -38,13 +48,63 @@ static int dw_plat_pcie_host_init(struct pcie_port *pp)
 	return 0;
 }
 
+static void dw_plat_set_num_vectors(struct pcie_port *pp)
+{
+	pp->num_vectors = MAX_MSI_IRQS;
+}
+
 static const struct dw_pcie_host_ops dw_plat_pcie_host_ops = {
 	.host_init = dw_plat_pcie_host_init,
+	.set_num_vectors = dw_plat_set_num_vectors,
+};
+
+static int dw_plat_pcie_establish_link(struct dw_pcie *pci)
+{
+	return 0;
+}
+
+static const struct dw_pcie_ops dw_pcie_ops = {
+	.start_link = dw_plat_pcie_establish_link,
 };
 
-static int dw_plat_add_pcie_port(struct pcie_port *pp,
+static void dw_plat_pcie_ep_init(struct dw_pcie_ep *ep)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	enum pci_barno bar;
+
+	for (bar = BAR_0; bar <= BAR_5; bar++)
+		dw_pcie_ep_reset_bar(pci, bar);
+}
+
+static int dw_plat_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
+				     enum pci_epc_irq_type type,
+				     u8 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+
+	switch (type) {
+	case PCI_EPC_IRQ_LEGACY:
+		dev_err(pci->dev, "EP cannot trigger legacy IRQs\n");
+		return -EINVAL;
+	case PCI_EPC_IRQ_MSI:
+		return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num);
+	default:
+		dev_err(pci->dev, "UNKNOWN IRQ type\n");
+	}
+
+	return 0;
+}
+
+static struct dw_pcie_ep_ops pcie_ep_ops = {
+	.ep_init = dw_plat_pcie_ep_init,
+	.raise_irq = dw_plat_pcie_ep_raise_irq,
+};
+
+static int dw_plat_add_pcie_port(struct dw_plat_pcie *dw_plat_pcie,
 				 struct platform_device *pdev)
 {
+	struct dw_pcie *pci = dw_plat_pcie->pci;
+	struct pcie_port *pp = &pci->pp;
 	struct device *dev = &pdev->dev;
 	int ret;
 
@@ -63,15 +123,44 @@ static int dw_plat_add_pcie_port(struct pcie_port *pp,
 
 	ret = dw_pcie_host_init(pp);
 	if (ret) {
-		dev_err(dev, "failed to initialize host\n");
+		dev_err(dev, "Failed to initialize host\n");
 		return ret;
 	}
 
 	return 0;
 }
 
-static const struct dw_pcie_ops dw_pcie_ops = {
-};
+static int dw_plat_add_pcie_ep(struct dw_plat_pcie *dw_plat_pcie,
+			       struct platform_device *pdev)
+{
+	int ret;
+	struct dw_pcie_ep *ep;
+	struct resource *res;
+	struct device *dev = &pdev->dev;
+	struct dw_pcie *pci = dw_plat_pcie->pci;
+
+	ep = &pci->ep;
+	ep->ops = &pcie_ep_ops;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi2");
+	pci->dbi_base2 = devm_ioremap_resource(dev, res);
+	if (IS_ERR(pci->dbi_base2))
+		return PTR_ERR(pci->dbi_base2);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space");
+	if (!res)
+		return -EINVAL;
+
+	ep->phys_base = res->start;
+	ep->addr_size = resource_size(res);
+
+	ret = dw_pcie_ep_init(ep);
+	if (ret) {
+		dev_err(dev, "Failed to initialize endpoint\n");
+		return ret;
+	}
+	return 0;
+}
 
 static int dw_plat_pcie_probe(struct platform_device *pdev)
 {
@@ -80,6 +169,16 @@ static int dw_plat_pcie_probe(struct platform_device *pdev)
 	struct dw_pcie *pci;
 	struct resource *res;  /* Resource from DT */
 	int ret;
+	const struct of_device_id *match;
+	const struct dw_plat_pcie_of_data *data;
+	enum dw_pcie_device_mode mode;
+
+	match = of_match_device(dw_plat_pcie_of_match, dev);
+	if (!match)
+		return -EINVAL;
+
+	data = (struct dw_plat_pcie_of_data *)match->data;
+	mode = (enum dw_pcie_device_mode)data->mode;
 
 	dw_plat_pcie = devm_kzalloc(dev, sizeof(*dw_plat_pcie), GFP_KERNEL);
 	if (!dw_plat_pcie)
@@ -93,23 +192,59 @@ static int dw_plat_pcie_probe(struct platform_device *pdev)
 	pci->ops = &dw_pcie_ops;
 
 	dw_plat_pcie->pci = pci;
+	dw_plat_pcie->mode = mode;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
+	if (!res)
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	pci->dbi_base = devm_ioremap_resource(dev, res);
 	if (IS_ERR(pci->dbi_base))
 		return PTR_ERR(pci->dbi_base);
 
 	platform_set_drvdata(pdev, dw_plat_pcie);
 
-	ret = dw_plat_add_pcie_port(&pci->pp, pdev);
-	if (ret < 0)
-		return ret;
+	switch (dw_plat_pcie->mode) {
+	case DW_PCIE_RC_TYPE:
+		if (!IS_ENABLED(CONFIG_PCIE_DW_PLAT_HOST))
+			return -ENODEV;
+
+		ret = dw_plat_add_pcie_port(dw_plat_pcie, pdev);
+		if (ret < 0)
+			return ret;
+		break;
+	case DW_PCIE_EP_TYPE:
+		if (!IS_ENABLED(CONFIG_PCIE_DW_PLAT_EP))
+			return -ENODEV;
+
+		ret = dw_plat_add_pcie_ep(dw_plat_pcie, pdev);
+		if (ret < 0)
+			return ret;
+		break;
+	default:
+		dev_err(dev, "INVALID device type %d\n", dw_plat_pcie->mode);
+	}
 
 	return 0;
 }
 
+static const struct dw_plat_pcie_of_data dw_plat_pcie_rc_of_data = {
+	.mode = DW_PCIE_RC_TYPE,
+};
+
+static const struct dw_plat_pcie_of_data dw_plat_pcie_ep_of_data = {
+	.mode = DW_PCIE_EP_TYPE,
+};
+
 static const struct of_device_id dw_plat_pcie_of_match[] = {
-	{ .compatible = "snps,dw-pcie", },
+	{
+		.compatible = "snps,dw-pcie",
+		.data = &dw_plat_pcie_rc_of_data,
+	},
+	{
+		.compatible = "snps,dw-pcie-ep",
+		.data = &dw_plat_pcie_ep_of_data,
+	},
 	{},
 };
 
diff --git a/drivers/pci/dwc/pcie-designware.c b/drivers/pci/dwc/pcie-designware.c
index 1b7282e5b494..778c4f76a884 100644
--- a/drivers/pci/dwc/pcie-designware.c
+++ b/drivers/pci/dwc/pcie-designware.c
@@ -69,7 +69,7 @@ u32 __dw_pcie_read_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg,
 
 	ret = dw_pcie_read(base + reg, size, &val);
 	if (ret)
-		dev_err(pci->dev, "read DBI address failed\n");
+		dev_err(pci->dev, "Read DBI address failed\n");
 
 	return val;
 }
@@ -86,7 +86,7 @@ void __dw_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg,
 
 	ret = dw_pcie_write(base + reg, size, val);
 	if (ret)
-		dev_err(pci->dev, "write DBI address failed\n");
+		dev_err(pci->dev, "Write DBI address failed\n");
 }
 
 static u32 dw_pcie_readl_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg)
@@ -137,7 +137,7 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
 
 		usleep_range(LINK_WAIT_IATU_MIN, LINK_WAIT_IATU_MAX);
 	}
-	dev_err(pci->dev, "outbound iATU is not being enabled\n");
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n");
 }
 
 void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type,
@@ -180,7 +180,7 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type,
 
 		usleep_range(LINK_WAIT_IATU_MIN, LINK_WAIT_IATU_MAX);
 	}
-	dev_err(pci->dev, "outbound iATU is not being enabled\n");
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n");
 }
 
 static u32 dw_pcie_readl_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg)
@@ -238,7 +238,7 @@ static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
 
 		usleep_range(LINK_WAIT_IATU_MIN, LINK_WAIT_IATU_MAX);
 	}
-	dev_err(pci->dev, "inbound iATU is not being enabled\n");
+	dev_err(pci->dev, "Inbound iATU is not being enabled\n");
 
 	return -EBUSY;
 }
@@ -284,7 +284,7 @@ int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
 
 		usleep_range(LINK_WAIT_IATU_MIN, LINK_WAIT_IATU_MAX);
 	}
-	dev_err(pci->dev, "inbound iATU is not being enabled\n");
+	dev_err(pci->dev, "Inbound iATU is not being enabled\n");
 
 	return -EBUSY;
 }
@@ -313,16 +313,16 @@ int dw_pcie_wait_for_link(struct dw_pcie *pci)
 {
 	int retries;
 
-	/* check if the link is up or not */
+	/* Check if the link is up or not */
 	for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
 		if (dw_pcie_link_up(pci)) {
-			dev_info(pci->dev, "link up\n");
+			dev_info(pci->dev, "Link up\n");
 			return 0;
 		}
 		usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
 	}
 
-	dev_err(pci->dev, "phy link never came up\n");
+	dev_err(pci->dev, "Phy link never came up\n");
 
 	return -ETIMEDOUT;
 }
@@ -351,7 +351,7 @@ void dw_pcie_setup(struct dw_pcie *pci)
 	if (ret)
 		lanes = 0;
 
-	/* set the number of lanes */
+	/* Set the number of lanes */
 	val = dw_pcie_readl_dbi(pci, PCIE_PORT_LINK_CONTROL);
 	val &= ~PORT_LINK_MODE_MASK;
 	switch (lanes) {
@@ -373,7 +373,7 @@ void dw_pcie_setup(struct dw_pcie *pci)
 	}
 	dw_pcie_writel_dbi(pci, PCIE_PORT_LINK_CONTROL, val);
 
-	/* set link width speed control register */
+	/* Set link width speed control register */
 	val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
 	val &= ~PORT_LOGIC_LINK_WIDTH_MASK;
 	switch (lanes) {
diff --git a/drivers/pci/dwc/pcie-designware.h b/drivers/pci/dwc/pcie-designware.h
index fe811dbc12cf..bee4e2535a61 100644
--- a/drivers/pci/dwc/pcie-designware.h
+++ b/drivers/pci/dwc/pcie-designware.h
@@ -110,6 +110,7 @@
 #define MAX_MSI_IRQS			256
 #define MAX_MSI_IRQS_PER_CTRL		32
 #define MAX_MSI_CTRLS			(MAX_MSI_IRQS / MAX_MSI_IRQS_PER_CTRL)
+#define MSI_REG_CTRL_BLOCK_SIZE		12
 #define MSI_DEF_NUM_VECTORS		32
 
 /* Maximum number of inbound/outbound iATUs */
diff --git a/drivers/pci/dwc/pcie-qcom.c b/drivers/pci/dwc/pcie-qcom.c
index 5897af7d3355..429113051a83 100644
--- a/drivers/pci/dwc/pcie-qcom.c
+++ b/drivers/pci/dwc/pcie-qcom.c
@@ -19,6 +19,7 @@
 #include <linux/of_device.h>
 #include <linux/of_gpio.h>
 #include <linux/pci.h>
+#include <linux/pm_runtime.h>
 #include <linux/platform_device.h>
 #include <linux/phy/phy.h>
 #include <linux/regulator/consumer.h>
@@ -869,7 +870,7 @@ static int qcom_pcie_init_2_4_0(struct qcom_pcie *pcie)
 
 	/* enable PCIe clocks and resets */
 	val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL);
-	val &= !BIT(0);
+	val &= ~BIT(0);
 	writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL);
 
 	/* change DBI base address */
@@ -1088,6 +1089,7 @@ static int qcom_pcie_host_init(struct pcie_port *pp)
 	struct qcom_pcie *pcie = to_qcom_pcie(pci);
 	int ret;
 
+	pm_runtime_get_sync(pci->dev);
 	qcom_ep_reset_assert(pcie);
 
 	ret = pcie->ops->init(pcie);
@@ -1124,6 +1126,7 @@ static int qcom_pcie_host_init(struct pcie_port *pp)
 	phy_power_off(pcie->phy);
 err_deinit:
 	pcie->ops->deinit(pcie);
+	pm_runtime_put(pci->dev);
 
 	return ret;
 }
@@ -1212,6 +1215,7 @@ static int qcom_pcie_probe(struct platform_device *pdev)
 	if (!pci)
 		return -ENOMEM;
 
+	pm_runtime_enable(dev);
 	pci->dev = dev;
 	pci->ops = &dw_pcie_ops;
 	pp = &pci->pp;
@@ -1257,14 +1261,17 @@ static int qcom_pcie_probe(struct platform_device *pdev)
 	}
 
 	ret = phy_init(pcie->phy);
-	if (ret)
+	if (ret) {
+		pm_runtime_disable(&pdev->dev);
 		return ret;
+	}
 
 	platform_set_drvdata(pdev, pcie);
 
 	ret = dw_pcie_host_init(pp);
 	if (ret) {
 		dev_err(dev, "cannot initialize host\n");
+		pm_runtime_disable(&pdev->dev);
 		return ret;
 	}
 
diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
index 7cef85124325..63ed706445b9 100644
--- a/drivers/pci/endpoint/functions/pci-epf-test.c
+++ b/drivers/pci/endpoint/functions/pci-epf-test.c
@@ -87,7 +87,7 @@ static int pci_epf_test_copy(struct pci_epf_test *epf_test)
 
 	src_addr = pci_epc_mem_alloc_addr(epc, &src_phys_addr, reg->size);
 	if (!src_addr) {
-		dev_err(dev, "failed to allocate source address\n");
+		dev_err(dev, "Failed to allocate source address\n");
 		reg->status = STATUS_SRC_ADDR_INVALID;
 		ret = -ENOMEM;
 		goto err;
@@ -96,14 +96,14 @@ static int pci_epf_test_copy(struct pci_epf_test *epf_test)
 	ret = pci_epc_map_addr(epc, epf->func_no, src_phys_addr, reg->src_addr,
 			       reg->size);
 	if (ret) {
-		dev_err(dev, "failed to map source address\n");
+		dev_err(dev, "Failed to map source address\n");
 		reg->status = STATUS_SRC_ADDR_INVALID;
 		goto err_src_addr;
 	}
 
 	dst_addr = pci_epc_mem_alloc_addr(epc, &dst_phys_addr, reg->size);
 	if (!dst_addr) {
-		dev_err(dev, "failed to allocate destination address\n");
+		dev_err(dev, "Failed to allocate destination address\n");
 		reg->status = STATUS_DST_ADDR_INVALID;
 		ret = -ENOMEM;
 		goto err_src_map_addr;
@@ -112,7 +112,7 @@ static int pci_epf_test_copy(struct pci_epf_test *epf_test)
 	ret = pci_epc_map_addr(epc, epf->func_no, dst_phys_addr, reg->dst_addr,
 			       reg->size);
 	if (ret) {
-		dev_err(dev, "failed to map destination address\n");
+		dev_err(dev, "Failed to map destination address\n");
 		reg->status = STATUS_DST_ADDR_INVALID;
 		goto err_dst_addr;
 	}
@@ -149,7 +149,7 @@ static int pci_epf_test_read(struct pci_epf_test *epf_test)
 
 	src_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size);
 	if (!src_addr) {
-		dev_err(dev, "failed to allocate address\n");
+		dev_err(dev, "Failed to allocate address\n");
 		reg->status = STATUS_SRC_ADDR_INVALID;
 		ret = -ENOMEM;
 		goto err;
@@ -158,7 +158,7 @@ static int pci_epf_test_read(struct pci_epf_test *epf_test)
 	ret = pci_epc_map_addr(epc, epf->func_no, phys_addr, reg->src_addr,
 			       reg->size);
 	if (ret) {
-		dev_err(dev, "failed to map address\n");
+		dev_err(dev, "Failed to map address\n");
 		reg->status = STATUS_SRC_ADDR_INVALID;
 		goto err_addr;
 	}
@@ -201,7 +201,7 @@ static int pci_epf_test_write(struct pci_epf_test *epf_test)
 
 	dst_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size);
 	if (!dst_addr) {
-		dev_err(dev, "failed to allocate address\n");
+		dev_err(dev, "Failed to allocate address\n");
 		reg->status = STATUS_DST_ADDR_INVALID;
 		ret = -ENOMEM;
 		goto err;
@@ -210,7 +210,7 @@ static int pci_epf_test_write(struct pci_epf_test *epf_test)
 	ret = pci_epc_map_addr(epc, epf->func_no, phys_addr, reg->dst_addr,
 			       reg->size);
 	if (ret) {
-		dev_err(dev, "failed to map address\n");
+		dev_err(dev, "Failed to map address\n");
 		reg->status = STATUS_DST_ADDR_INVALID;
 		goto err_addr;
 	}
@@ -230,7 +230,7 @@ static int pci_epf_test_write(struct pci_epf_test *epf_test)
 	 * wait 1ms inorder for the write to complete. Without this delay L3
 	 * error in observed in the host system.
 	 */
-	mdelay(1);
+	usleep_range(1000, 2000);
 
 	kfree(buf);
 
@@ -379,7 +379,7 @@ static int pci_epf_test_set_bar(struct pci_epf *epf)
 		ret = pci_epc_set_bar(epc, epf->func_no, epf_bar);
 		if (ret) {
 			pci_epf_free_space(epf, epf_test->reg[bar], bar);
-			dev_err(dev, "failed to set BAR%d\n", bar);
+			dev_err(dev, "Failed to set BAR%d\n", bar);
 			if (bar == test_reg_bar)
 				return ret;
 		}
@@ -406,7 +406,7 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf)
 	base = pci_epf_alloc_space(epf, sizeof(struct pci_epf_test_reg),
 				   test_reg_bar);
 	if (!base) {
-		dev_err(dev, "failed to allocated register space\n");
+		dev_err(dev, "Failed to allocated register space\n");
 		return -ENOMEM;
 	}
 	epf_test->reg[test_reg_bar] = base;
@@ -416,7 +416,7 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf)
 			continue;
 		base = pci_epf_alloc_space(epf, bar_size[bar], bar);
 		if (!base)
-			dev_err(dev, "failed to allocate space for BAR%d\n",
+			dev_err(dev, "Failed to allocate space for BAR%d\n",
 				bar);
 		epf_test->reg[bar] = base;
 	}
@@ -435,9 +435,16 @@ static int pci_epf_test_bind(struct pci_epf *epf)
 	if (WARN_ON_ONCE(!epc))
 		return -EINVAL;
 
+	if (epc->features & EPC_FEATURE_NO_LINKUP_NOTIFIER)
+		epf_test->linkup_notifier = false;
+	else
+		epf_test->linkup_notifier = true;
+
+	epf_test->test_reg_bar = EPC_FEATURE_GET_BAR(epc->features);
+
 	ret = pci_epc_write_header(epc, epf->func_no, header);
 	if (ret) {
-		dev_err(dev, "configuration header write failed\n");
+		dev_err(dev, "Configuration header write failed\n");
 		return ret;
 	}
 
@@ -519,7 +526,7 @@ static int __init pci_epf_test_init(void)
 					     WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
 	ret = pci_epf_register_driver(&test_driver);
 	if (ret) {
-		pr_err("failed to register pci epf test driver --> %d\n", ret);
+		pr_err("Failed to register pci epf test driver --> %d\n", ret);
 		return ret;
 	}
 
diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c
index 465b5f058b6d..523a8cab3bfb 100644
--- a/drivers/pci/endpoint/pci-epf-core.c
+++ b/drivers/pci/endpoint/pci-epf-core.c
@@ -15,6 +15,8 @@
 #include <linux/pci-epf.h>
 #include <linux/pci-ep-cfs.h>
 
+static DEFINE_MUTEX(pci_epf_mutex);
+
 static struct bus_type pci_epf_bus_type;
 static const struct device_type pci_epf_type;
 
@@ -143,7 +145,13 @@ EXPORT_SYMBOL_GPL(pci_epf_alloc_space);
  */
 void pci_epf_unregister_driver(struct pci_epf_driver *driver)
 {
-	pci_ep_cfs_remove_epf_group(driver->group);
+	struct config_group *group;
+
+	mutex_lock(&pci_epf_mutex);
+	list_for_each_entry(group, &driver->epf_group, group_entry)
+		pci_ep_cfs_remove_epf_group(group);
+	list_del(&driver->epf_group);
+	mutex_unlock(&pci_epf_mutex);
 	driver_unregister(&driver->driver);
 }
 EXPORT_SYMBOL_GPL(pci_epf_unregister_driver);
@@ -159,6 +167,8 @@ int __pci_epf_register_driver(struct pci_epf_driver *driver,
 			      struct module *owner)
 {
 	int ret;
+	struct config_group *group;
+	const struct pci_epf_device_id *id;
 
 	if (!driver->ops)
 		return -EINVAL;
@@ -173,7 +183,16 @@ int __pci_epf_register_driver(struct pci_epf_driver *driver,
 	if (ret)
 		return ret;
 
-	driver->group = pci_ep_cfs_add_epf_group(driver->driver.name);
+	INIT_LIST_HEAD(&driver->epf_group);
+
+	id = driver->id_table;
+	while (id->name[0]) {
+		group = pci_ep_cfs_add_epf_group(id->name);
+		mutex_lock(&pci_epf_mutex);
+		list_add_tail(&group->group_entry, &driver->epf_group);
+		mutex_unlock(&pci_epf_mutex);
+		id++;
+	}
 
 	return 0;
 }
diff --git a/drivers/pci/host/Kconfig b/drivers/pci/host/Kconfig
index 0d0177ce436c..a96e23bda664 100644
--- a/drivers/pci/host/Kconfig
+++ b/drivers/pci/host/Kconfig
@@ -5,13 +5,14 @@ menu "PCI host controller drivers"
 
 config PCI_MVEBU
 	bool "Marvell EBU PCIe controller"
-	depends on ARCH_MVEBU || ARCH_DOVE
+	depends on ARCH_MVEBU || ARCH_DOVE || COMPILE_TEST
+	depends on MVEBU_MBUS
 	depends on ARM
 	depends on OF
 
 config PCI_AARDVARK
 	bool "Aardvark PCIe controller"
-	depends on ARCH_MVEBU && ARM64
+	depends on (ARCH_MVEBU && ARM64) || COMPILE_TEST
 	depends on OF
 	depends on PCI_MSI_IRQ_DOMAIN
 	help
@@ -21,7 +22,7 @@ config PCI_AARDVARK
 
 config PCIE_XILINX_NWL
 	bool "NWL PCIe Core"
-	depends on ARCH_ZYNQMP
+	depends on ARCH_ZYNQMP || COMPILE_TEST
 	depends on PCI_MSI_IRQ_DOMAIN
 	help
 	 Say 'Y' here if you want kernel support for Xilinx
@@ -32,12 +33,11 @@ config PCIE_XILINX_NWL
 config PCI_FTPCI100
 	bool "Faraday Technology FTPCI100 PCI controller"
 	depends on OF
-	depends on ARM
 	default ARCH_GEMINI
 
 config PCI_TEGRA
 	bool "NVIDIA Tegra PCIe controller"
-	depends on ARCH_TEGRA
+	depends on ARCH_TEGRA || COMPILE_TEST
 	depends on PCI_MSI_IRQ_DOMAIN
 	help
 	  Say Y here if you want support for the PCIe host controller found
@@ -45,8 +45,8 @@ config PCI_TEGRA
 
 config PCI_RCAR_GEN2
 	bool "Renesas R-Car Gen2 Internal PCI controller"
-	depends on ARM
 	depends on ARCH_RENESAS || COMPILE_TEST
+	depends on ARM
 	help
 	  Say Y here if you want internal PCI support on R-Car Gen2 SoC.
 	  There are 3 internal PCI controllers available with a single
@@ -54,7 +54,7 @@ config PCI_RCAR_GEN2
 
 config PCIE_RCAR
 	bool "Renesas R-Car PCIe controller"
-	depends on ARCH_RENESAS || (ARM && COMPILE_TEST)
+	depends on ARCH_RENESAS || COMPILE_TEST
 	depends on PCI_MSI_IRQ_DOMAIN
 	help
 	  Say Y here if you want PCIe controller support on R-Car SoCs.
@@ -65,25 +65,25 @@ config PCI_HOST_COMMON
 
 config PCI_HOST_GENERIC
 	bool "Generic PCI host controller"
-	depends on (ARM || ARM64) && OF
+	depends on OF
 	select PCI_HOST_COMMON
 	select IRQ_DOMAIN
+	select PCI_DOMAINS
 	help
 	  Say Y here if you want to support a simple generic PCI host
 	  controller, such as the one emulated by kvmtool.
 
 config PCIE_XILINX
 	bool "Xilinx AXI PCIe host bridge support"
-	depends on ARCH_ZYNQ || MICROBLAZE || (MIPS && PCI_DRIVERS_GENERIC)
+	depends on ARCH_ZYNQ || MICROBLAZE || (MIPS && PCI_DRIVERS_GENERIC) || COMPILE_TEST
 	help
 	  Say 'Y' here if you want kernel to support the Xilinx AXI PCIe
 	  Host Bridge driver.
 
 config PCI_XGENE
 	bool "X-Gene PCIe controller"
-	depends on ARM64
+	depends on ARM64 || COMPILE_TEST
 	depends on OF || (ACPI && PCI_QUIRKS)
-	select PCIEPORTBUS
 	help
 	  Say Y here if you want internal PCI support on APM X-Gene SoC.
 	  There are 5 internal PCIe ports available. Each port is GEN3 capable
@@ -101,7 +101,7 @@ config PCI_XGENE_MSI
 config PCI_V3_SEMI
 	bool "V3 Semiconductor PCI controller"
 	depends on OF
-	depends on ARM
+	depends on ARM || COMPILE_TEST
 	default ARCH_INTEGRATOR_AP
 
 config PCI_VERSATILE
@@ -147,8 +147,7 @@ config PCIE_IPROC_MSI
 
 config PCIE_ALTERA
 	bool "Altera PCIe controller"
-	depends on ARM || NIOS2
-	depends on OF_PCI
+	depends on ARM || NIOS2 || COMPILE_TEST
 	select PCI_DOMAINS
 	help
 	  Say Y here if you want to enable PCIe controller support on Altera
@@ -164,7 +163,7 @@ config PCIE_ALTERA_MSI
 
 config PCI_HOST_THUNDER_PEM
 	bool "Cavium Thunder PCIe controller to off-chip devices"
-	depends on ARM64
+	depends on ARM64 || COMPILE_TEST
 	depends on OF || (ACPI && PCI_QUIRKS)
 	select PCI_HOST_COMMON
 	help
@@ -172,29 +171,45 @@ config PCI_HOST_THUNDER_PEM
 
 config PCI_HOST_THUNDER_ECAM
 	bool "Cavium Thunder ECAM controller to on-chip devices on pass-1.x silicon"
-	depends on ARM64
+	depends on ARM64 || COMPILE_TEST
 	depends on OF || (ACPI && PCI_QUIRKS)
 	select PCI_HOST_COMMON
 	help
 	  Say Y here if you want ECAM support for CN88XX-Pass-1.x Cavium Thunder SoCs.
 
 config PCIE_ROCKCHIP
-	tristate "Rockchip PCIe controller"
+	bool
+	depends on PCI
+
+config PCIE_ROCKCHIP_HOST
+	tristate "Rockchip PCIe host controller"
 	depends on ARCH_ROCKCHIP || COMPILE_TEST
 	depends on OF
 	depends on PCI_MSI_IRQ_DOMAIN
 	select MFD_SYSCON
+	select PCIE_ROCKCHIP
 	help
 	  Say Y here if you want internal PCI support on Rockchip SoC.
 	  There is 1 internal PCIe port available to support GEN2 with
 	  4 slots.
 
+config PCIE_ROCKCHIP_EP
+	bool "Rockchip PCIe endpoint controller"
+	depends on ARCH_ROCKCHIP || COMPILE_TEST
+	depends on OF
+	depends on PCI_ENDPOINT
+	select MFD_SYSCON
+	select PCIE_ROCKCHIP
+	help
+	  Say Y here if you want to support Rockchip PCIe controller in
+	  endpoint mode on Rockchip SoC. There is 1 internal PCIe port
+	  available to support GEN2 with 4 slots.
+
 config PCIE_MEDIATEK
 	bool "MediaTek PCIe controller"
-	depends on (ARM || ARM64) && (ARCH_MEDIATEK || COMPILE_TEST)
+	depends on ARCH_MEDIATEK || COMPILE_TEST
 	depends on OF
-	depends on PCI
-	select PCIEPORTBUS
+	depends on PCI_MSI_IRQ_DOMAIN
 	help
 	  Say Y here if you want to enable PCIe controller support on
 	  MediaTek SoCs.
diff --git a/drivers/pci/host/Makefile b/drivers/pci/host/Makefile
index 3b1059190867..11d21b026d37 100644
--- a/drivers/pci/host/Makefile
+++ b/drivers/pci/host/Makefile
@@ -20,6 +20,8 @@ obj-$(CONFIG_PCIE_IPROC_BCMA) += pcie-iproc-bcma.o
 obj-$(CONFIG_PCIE_ALTERA) += pcie-altera.o
 obj-$(CONFIG_PCIE_ALTERA_MSI) += pcie-altera-msi.o
 obj-$(CONFIG_PCIE_ROCKCHIP) += pcie-rockchip.o
+obj-$(CONFIG_PCIE_ROCKCHIP_EP) += pcie-rockchip-ep.o
+obj-$(CONFIG_PCIE_ROCKCHIP_HOST) += pcie-rockchip-host.o
 obj-$(CONFIG_PCIE_MEDIATEK) += pcie-mediatek.o
 obj-$(CONFIG_PCIE_TANGO_SMP8759) += pcie-tango.o
 obj-$(CONFIG_VMD) += vmd.o
diff --git a/drivers/pci/host/pci-aardvark.c b/drivers/pci/host/pci-aardvark.c
index 9abf549631b4..d3172d5d3d35 100644
--- a/drivers/pci/host/pci-aardvark.c
+++ b/drivers/pci/host/pci-aardvark.c
@@ -19,6 +19,8 @@
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
 
+#include "../pci.h"
+
 /* PCIe core registers */
 #define PCIE_CORE_CMD_STATUS_REG				0x4
 #define     PCIE_CORE_CMD_IO_ACCESS_EN				BIT(0)
@@ -822,14 +824,13 @@ static int advk_pcie_parse_request_of_pci_ranges(struct advk_pcie *pcie)
 {
 	int err, res_valid = 0;
 	struct device *dev = &pcie->pdev->dev;
-	struct device_node *np = dev->of_node;
 	struct resource_entry *win, *tmp;
 	resource_size_t iobase;
 
 	INIT_LIST_HEAD(&pcie->resources);
 
-	err = of_pci_get_host_bridge_resources(np, 0, 0xff, &pcie->resources,
-					       &iobase);
+	err = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff,
+						    &pcie->resources, &iobase);
 	if (err)
 		return err;
 
diff --git a/drivers/pci/host/pci-ftpci100.c b/drivers/pci/host/pci-ftpci100.c
index 5008fd87956a..a1ebe9ed441f 100644
--- a/drivers/pci/host/pci-ftpci100.c
+++ b/drivers/pci/host/pci-ftpci100.c
@@ -28,6 +28,8 @@
 #include <linux/irq.h>
 #include <linux/clk.h>
 
+#include "../pci.h"
+
 /*
  * Special configuration registers directly in the first few words
  * in I/O space.
@@ -476,8 +478,8 @@ static int faraday_pci_probe(struct platform_device *pdev)
 	if (IS_ERR(p->base))
 		return PTR_ERR(p->base);
 
-	ret = of_pci_get_host_bridge_resources(dev->of_node, 0, 0xff,
-					       &res, &io_base);
+	ret = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff,
+						    &res, &io_base);
 	if (ret)
 		return ret;
 
diff --git a/drivers/pci/host/pci-host-common.c b/drivers/pci/host/pci-host-common.c
index 5d028f53fdcd..d8f10451f273 100644
--- a/drivers/pci/host/pci-host-common.c
+++ b/drivers/pci/host/pci-host-common.c
@@ -101,5 +101,18 @@ int pci_host_common_probe(struct platform_device *pdev,
 		return ret;
 	}
 
+	platform_set_drvdata(pdev, bridge->bus);
+	return 0;
+}
+
+int pci_host_common_remove(struct platform_device *pdev)
+{
+	struct pci_bus *bus = platform_get_drvdata(pdev);
+
+	pci_lock_rescan_remove();
+	pci_stop_root_bus(bus);
+	pci_remove_root_bus(bus);
+	pci_unlock_rescan_remove();
+
 	return 0;
 }
diff --git a/drivers/pci/host/pci-host-generic.c b/drivers/pci/host/pci-host-generic.c
index 45319ee3b484..dea3ec7592a2 100644
--- a/drivers/pci/host/pci-host-generic.c
+++ b/drivers/pci/host/pci-host-generic.c
@@ -95,5 +95,6 @@ static struct platform_driver gen_pci_driver = {
 		.suppress_bind_attrs = true,
 	},
 	.probe = gen_pci_probe,
+	.remove = pci_host_common_remove,
 };
 builtin_platform_driver(gen_pci_driver);
diff --git a/drivers/pci/host/pci-hyperv.c b/drivers/pci/host/pci-hyperv.c
index 50cdefe3f6d3..6cc5036ac83c 100644
--- a/drivers/pci/host/pci-hyperv.c
+++ b/drivers/pci/host/pci-hyperv.c
@@ -433,7 +433,7 @@ enum hv_pcibus_state {
 struct hv_pcibus_device {
 	struct pci_sysdata sysdata;
 	enum hv_pcibus_state state;
-	atomic_t remove_lock;
+	refcount_t remove_lock;
 	struct hv_device *hdev;
 	resource_size_t low_mmio_space;
 	resource_size_t high_mmio_space;
@@ -488,17 +488,6 @@ enum hv_pcichild_state {
 	hv_pcichild_maximum
 };
 
-enum hv_pcidev_ref_reason {
-	hv_pcidev_ref_invalid = 0,
-	hv_pcidev_ref_initial,
-	hv_pcidev_ref_by_slot,
-	hv_pcidev_ref_packet,
-	hv_pcidev_ref_pnp,
-	hv_pcidev_ref_childlist,
-	hv_pcidev_irqdata,
-	hv_pcidev_ref_max
-};
-
 struct hv_pci_dev {
 	/* List protected by pci_rescan_remove_lock */
 	struct list_head list_entry;
@@ -548,14 +537,41 @@ static void hv_pci_generic_compl(void *context, struct pci_response *resp,
 
 static struct hv_pci_dev *get_pcichild_wslot(struct hv_pcibus_device *hbus,
 						u32 wslot);
-static void get_pcichild(struct hv_pci_dev *hv_pcidev,
-			 enum hv_pcidev_ref_reason reason);
-static void put_pcichild(struct hv_pci_dev *hv_pcidev,
-			 enum hv_pcidev_ref_reason reason);
+
+static void get_pcichild(struct hv_pci_dev *hpdev)
+{
+	refcount_inc(&hpdev->refs);
+}
+
+static void put_pcichild(struct hv_pci_dev *hpdev)
+{
+	if (refcount_dec_and_test(&hpdev->refs))
+		kfree(hpdev);
+}
 
 static void get_hvpcibus(struct hv_pcibus_device *hv_pcibus);
 static void put_hvpcibus(struct hv_pcibus_device *hv_pcibus);
 
+/*
+ * There is no good way to get notified from vmbus_onoffer_rescind(),
+ * so let's use polling here, since this is not a hot path.
+ */
+static int wait_for_response(struct hv_device *hdev,
+			     struct completion *comp)
+{
+	while (true) {
+		if (hdev->channel->rescind) {
+			dev_warn_once(&hdev->device, "The device is gone.\n");
+			return -ENODEV;
+		}
+
+		if (wait_for_completion_timeout(comp, HZ / 10))
+			break;
+	}
+
+	return 0;
+}
+
 /**
  * devfn_to_wslot() - Convert from Linux PCI slot to Windows
  * @devfn:	The Linux representation of PCI slot
@@ -762,7 +778,7 @@ static int hv_pcifront_read_config(struct pci_bus *bus, unsigned int devfn,
 
 	_hv_pcifront_read_config(hpdev, where, size, val);
 
-	put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+	put_pcichild(hpdev);
 	return PCIBIOS_SUCCESSFUL;
 }
 
@@ -790,7 +806,7 @@ static int hv_pcifront_write_config(struct pci_bus *bus, unsigned int devfn,
 
 	_hv_pcifront_write_config(hpdev, where, size, val);
 
-	put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+	put_pcichild(hpdev);
 	return PCIBIOS_SUCCESSFUL;
 }
 
@@ -856,7 +872,7 @@ static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info,
 	}
 
 	hv_int_desc_free(hpdev, int_desc);
-	put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+	put_pcichild(hpdev);
 }
 
 static int hv_set_affinity(struct irq_data *data, const struct cpumask *dest,
@@ -1186,13 +1202,13 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 	msg->address_lo = comp.int_desc.address & 0xffffffff;
 	msg->data = comp.int_desc.data;
 
-	put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+	put_pcichild(hpdev);
 	return;
 
 free_int_desc:
 	kfree(int_desc);
 drop_reference:
-	put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+	put_pcichild(hpdev);
 return_null_message:
 	msg->address_hi = 0;
 	msg->address_lo = 0;
@@ -1283,7 +1299,6 @@ static u64 get_bar_size(u64 bar_val)
  */
 static void survey_child_resources(struct hv_pcibus_device *hbus)
 {
-	struct list_head *iter;
 	struct hv_pci_dev *hpdev;
 	resource_size_t bar_size = 0;
 	unsigned long flags;
@@ -1309,8 +1324,7 @@ static void survey_child_resources(struct hv_pcibus_device *hbus)
 	 * for a child device are a power of 2 in size and aligned in memory,
 	 * so it's sufficient to just add them up without tracking alignment.
 	 */
-	list_for_each(iter, &hbus->children) {
-		hpdev = container_of(iter, struct hv_pci_dev, list_entry);
+	list_for_each_entry(hpdev, &hbus->children, list_entry) {
 		for (i = 0; i < 6; i++) {
 			if (hpdev->probed_bar[i] & PCI_BASE_ADDRESS_SPACE_IO)
 				dev_err(&hbus->hdev->device,
@@ -1363,7 +1377,6 @@ static void prepopulate_bars(struct hv_pcibus_device *hbus)
 	resource_size_t low_base = 0;
 	resource_size_t bar_size;
 	struct hv_pci_dev *hpdev;
-	struct list_head *iter;
 	unsigned long flags;
 	u64 bar_val;
 	u32 command;
@@ -1385,9 +1398,7 @@ static void prepopulate_bars(struct hv_pcibus_device *hbus)
 
 	/* Pick addresses for the BARs. */
 	do {
-		list_for_each(iter, &hbus->children) {
-			hpdev = container_of(iter, struct hv_pci_dev,
-					     list_entry);
+		list_for_each_entry(hpdev, &hbus->children, list_entry) {
 			for (i = 0; i < 6; i++) {
 				bar_val = hpdev->probed_bar[i];
 				if (bar_val == 0)
@@ -1508,19 +1519,6 @@ static void q_resource_requirements(void *context, struct pci_response *resp,
 	complete(&completion->host_event);
 }
 
-static void get_pcichild(struct hv_pci_dev *hpdev,
-			    enum hv_pcidev_ref_reason reason)
-{
-	refcount_inc(&hpdev->refs);
-}
-
-static void put_pcichild(struct hv_pci_dev *hpdev,
-			    enum hv_pcidev_ref_reason reason)
-{
-	if (refcount_dec_and_test(&hpdev->refs))
-		kfree(hpdev);
-}
-
 /**
  * new_pcichild_device() - Create a new child device
  * @hbus:	The internal struct tracking this root PCI bus.
@@ -1568,24 +1566,14 @@ static struct hv_pci_dev *new_pcichild_device(struct hv_pcibus_device *hbus,
 	if (ret)
 		goto error;
 
-	wait_for_completion(&comp_pkt.host_event);
+	if (wait_for_response(hbus->hdev, &comp_pkt.host_event))
+		goto error;
 
 	hpdev->desc = *desc;
 	refcount_set(&hpdev->refs, 1);
-	get_pcichild(hpdev, hv_pcidev_ref_childlist);
+	get_pcichild(hpdev);
 	spin_lock_irqsave(&hbus->device_list_lock, flags);
 
-	/*
-	 * When a device is being added to the bus, we set the PCI domain
-	 * number to be the device serial number, which is non-zero and
-	 * unique on the same VM.  The serial numbers start with 1, and
-	 * increase by 1 for each device.  So device names including this
-	 * can have shorter names than based on the bus instance UUID.
-	 * Only the first device serial number is used for domain, so the
-	 * domain number will not change after the first device is added.
-	 */
-	if (list_empty(&hbus->children))
-		hbus->sysdata.domain = desc->ser;
 	list_add_tail(&hpdev->list_entry, &hbus->children);
 	spin_unlock_irqrestore(&hbus->device_list_lock, flags);
 	return hpdev;
@@ -1618,7 +1606,7 @@ static struct hv_pci_dev *get_pcichild_wslot(struct hv_pcibus_device *hbus,
 	list_for_each_entry(iter, &hbus->children, list_entry) {
 		if (iter->desc.win_slot.slot == wslot) {
 			hpdev = iter;
-			get_pcichild(hpdev, hv_pcidev_ref_by_slot);
+			get_pcichild(hpdev);
 			break;
 		}
 	}
@@ -1654,7 +1642,6 @@ static void pci_devices_present_work(struct work_struct *work)
 {
 	u32 child_no;
 	bool found;
-	struct list_head *iter;
 	struct pci_function_description *new_desc;
 	struct hv_pci_dev *hpdev;
 	struct hv_pcibus_device *hbus;
@@ -1691,10 +1678,8 @@ static void pci_devices_present_work(struct work_struct *work)
 
 	/* First, mark all existing children as reported missing. */
 	spin_lock_irqsave(&hbus->device_list_lock, flags);
-	list_for_each(iter, &hbus->children) {
-			hpdev = container_of(iter, struct hv_pci_dev,
-					     list_entry);
-			hpdev->reported_missing = true;
+	list_for_each_entry(hpdev, &hbus->children, list_entry) {
+		hpdev->reported_missing = true;
 	}
 	spin_unlock_irqrestore(&hbus->device_list_lock, flags);
 
@@ -1704,11 +1689,8 @@ static void pci_devices_present_work(struct work_struct *work)
 		new_desc = &dr->func[child_no];
 
 		spin_lock_irqsave(&hbus->device_list_lock, flags);
-		list_for_each(iter, &hbus->children) {
-			hpdev = container_of(iter, struct hv_pci_dev,
-					     list_entry);
-			if ((hpdev->desc.win_slot.slot ==
-			     new_desc->win_slot.slot) &&
+		list_for_each_entry(hpdev, &hbus->children, list_entry) {
+			if ((hpdev->desc.win_slot.slot == new_desc->win_slot.slot) &&
 			    (hpdev->desc.v_id == new_desc->v_id) &&
 			    (hpdev->desc.d_id == new_desc->d_id) &&
 			    (hpdev->desc.ser == new_desc->ser)) {
@@ -1730,12 +1712,10 @@ static void pci_devices_present_work(struct work_struct *work)
 	spin_lock_irqsave(&hbus->device_list_lock, flags);
 	do {
 		found = false;
-		list_for_each(iter, &hbus->children) {
-			hpdev = container_of(iter, struct hv_pci_dev,
-					     list_entry);
+		list_for_each_entry(hpdev, &hbus->children, list_entry) {
 			if (hpdev->reported_missing) {
 				found = true;
-				put_pcichild(hpdev, hv_pcidev_ref_childlist);
+				put_pcichild(hpdev);
 				list_move_tail(&hpdev->list_entry, &removed);
 				break;
 			}
@@ -1748,7 +1728,7 @@ static void pci_devices_present_work(struct work_struct *work)
 		hpdev = list_first_entry(&removed, struct hv_pci_dev,
 					 list_entry);
 		list_del(&hpdev->list_entry);
-		put_pcichild(hpdev, hv_pcidev_ref_initial);
+		put_pcichild(hpdev);
 	}
 
 	switch (hbus->state) {
@@ -1883,8 +1863,8 @@ static void hv_eject_device_work(struct work_struct *work)
 			 sizeof(*ejct_pkt), (unsigned long)&ctxt.pkt,
 			 VM_PKT_DATA_INBAND, 0);
 
-	put_pcichild(hpdev, hv_pcidev_ref_childlist);
-	put_pcichild(hpdev, hv_pcidev_ref_pnp);
+	put_pcichild(hpdev);
+	put_pcichild(hpdev);
 	put_hvpcibus(hpdev->hbus);
 }
 
@@ -1899,7 +1879,7 @@ static void hv_eject_device_work(struct work_struct *work)
 static void hv_pci_eject_device(struct hv_pci_dev *hpdev)
 {
 	hpdev->state = hv_pcichild_ejecting;
-	get_pcichild(hpdev, hv_pcidev_ref_pnp);
+	get_pcichild(hpdev);
 	INIT_WORK(&hpdev->wrk, hv_eject_device_work);
 	get_hvpcibus(hpdev->hbus);
 	queue_work(hpdev->hbus->wq, &hpdev->wrk);
@@ -1999,8 +1979,7 @@ static void hv_pci_onchannelcallback(void *context)
 						      dev_message->wslot.slot);
 				if (hpdev) {
 					hv_pci_eject_device(hpdev);
-					put_pcichild(hpdev,
-							hv_pcidev_ref_by_slot);
+					put_pcichild(hpdev);
 				}
 				break;
 
@@ -2069,15 +2048,16 @@ static int hv_pci_protocol_negotiation(struct hv_device *hdev)
 				sizeof(struct pci_version_request),
 				(unsigned long)pkt, VM_PKT_DATA_INBAND,
 				VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+		if (!ret)
+			ret = wait_for_response(hdev, &comp_pkt.host_event);
+
 		if (ret) {
 			dev_err(&hdev->device,
-				"PCI Pass-through VSP failed sending version reqquest: %#x",
+				"PCI Pass-through VSP failed to request version: %d",
 				ret);
 			goto exit;
 		}
 
-		wait_for_completion(&comp_pkt.host_event);
-
 		if (comp_pkt.completion_status >= 0) {
 			pci_protocol_version = pci_protocol_versions[i];
 			dev_info(&hdev->device,
@@ -2286,11 +2266,12 @@ static int hv_pci_enter_d0(struct hv_device *hdev)
 	ret = vmbus_sendpacket(hdev->channel, d0_entry, sizeof(*d0_entry),
 			       (unsigned long)pkt, VM_PKT_DATA_INBAND,
 			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+	if (!ret)
+		ret = wait_for_response(hdev, &comp_pkt.host_event);
+
 	if (ret)
 		goto exit;
 
-	wait_for_completion(&comp_pkt.host_event);
-
 	if (comp_pkt.completion_status < 0) {
 		dev_err(&hdev->device,
 			"PCI Pass-through VSP failed D0 Entry with status %x\n",
@@ -2330,11 +2311,10 @@ static int hv_pci_query_relations(struct hv_device *hdev)
 
 	ret = vmbus_sendpacket(hdev->channel, &message, sizeof(message),
 			       0, VM_PKT_DATA_INBAND, 0);
-	if (ret)
-		return ret;
+	if (!ret)
+		ret = wait_for_response(hdev, &comp);
 
-	wait_for_completion(&comp);
-	return 0;
+	return ret;
 }
 
 /**
@@ -2398,17 +2378,17 @@ static int hv_send_resources_allocated(struct hv_device *hdev)
 				PCI_RESOURCES_ASSIGNED2;
 			res_assigned2->wslot.slot = hpdev->desc.win_slot.slot;
 		}
-		put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+		put_pcichild(hpdev);
 
 		ret = vmbus_sendpacket(hdev->channel, &pkt->message,
 				size_res, (unsigned long)pkt,
 				VM_PKT_DATA_INBAND,
 				VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+		if (!ret)
+			ret = wait_for_response(hdev, &comp_pkt.host_event);
 		if (ret)
 			break;
 
-		wait_for_completion(&comp_pkt.host_event);
-
 		if (comp_pkt.completion_status < 0) {
 			ret = -EPROTO;
 			dev_err(&hdev->device,
@@ -2446,7 +2426,7 @@ static int hv_send_resources_released(struct hv_device *hdev)
 		pkt.message_type.type = PCI_RESOURCES_RELEASED;
 		pkt.wslot.slot = hpdev->desc.win_slot.slot;
 
-		put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+		put_pcichild(hpdev);
 
 		ret = vmbus_sendpacket(hdev->channel, &pkt, sizeof(pkt), 0,
 				       VM_PKT_DATA_INBAND, 0);
@@ -2459,12 +2439,12 @@ static int hv_send_resources_released(struct hv_device *hdev)
 
 static void get_hvpcibus(struct hv_pcibus_device *hbus)
 {
-	atomic_inc(&hbus->remove_lock);
+	refcount_inc(&hbus->remove_lock);
 }
 
 static void put_hvpcibus(struct hv_pcibus_device *hbus)
 {
-	if (atomic_dec_and_test(&hbus->remove_lock))
+	if (refcount_dec_and_test(&hbus->remove_lock))
 		complete(&hbus->remove_event);
 }
 
@@ -2508,7 +2488,7 @@ static int hv_pci_probe(struct hv_device *hdev,
 			       hdev->dev_instance.b[8] << 8;
 
 	hbus->hdev = hdev;
-	atomic_inc(&hbus->remove_lock);
+	refcount_set(&hbus->remove_lock, 1);
 	INIT_LIST_HEAD(&hbus->children);
 	INIT_LIST_HEAD(&hbus->dr_list);
 	INIT_LIST_HEAD(&hbus->resources_for_children);
diff --git a/drivers/pci/host/pci-mvebu.c b/drivers/pci/host/pci-mvebu.c
index 5d4dccfc9d81..23e270839e6a 100644
--- a/drivers/pci/host/pci-mvebu.c
+++ b/drivers/pci/host/pci-mvebu.c
@@ -21,6 +21,8 @@
 #include <linux/of_pci.h>
 #include <linux/of_platform.h>
 
+#include "../pci.h"
+
 /*
  * PCIe unit register offsets.
  */
diff --git a/drivers/pci/host/pci-rcar-gen2.c b/drivers/pci/host/pci-rcar-gen2.c
index dd4f1a6b57c5..326171cb1a97 100644
--- a/drivers/pci/host/pci-rcar-gen2.c
+++ b/drivers/pci/host/pci-rcar-gen2.c
@@ -21,6 +21,8 @@
 #include <linux/sizes.h>
 #include <linux/slab.h>
 
+#include "../pci.h"
+
 /* AHB-PCI Bridge PCI communication registers */
 #define RCAR_AHBPCI_PCICOM_OFFSET	0x800
 
diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index 389e74be846c..f4f53d092e00 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -40,6 +40,8 @@
 #include <soc/tegra/cpuidle.h>
 #include <soc/tegra/pmc.h>
 
+#include "../pci.h"
+
 #define INT_PCI_MSI_NR (8 * 32)
 
 /* register definitions */
diff --git a/drivers/pci/host/pci-v3-semi.c b/drivers/pci/host/pci-v3-semi.c
index 0a4dea796663..68b8bfbdb867 100644
--- a/drivers/pci/host/pci-v3-semi.c
+++ b/drivers/pci/host/pci-v3-semi.c
@@ -33,6 +33,8 @@
 #include <linux/regmap.h>
 #include <linux/clk.h>
 
+#include "../pci.h"
+
 #define V3_PCI_VENDOR			0x00000000
 #define V3_PCI_DEVICE			0x00000002
 #define V3_PCI_CMD			0x00000004
@@ -791,7 +793,8 @@ static int v3_pci_probe(struct platform_device *pdev)
 	if (IS_ERR(v3->config_base))
 		return PTR_ERR(v3->config_base);
 
-	ret = of_pci_get_host_bridge_resources(np, 0, 0xff, &res, &io_base);
+	ret = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff, &res,
+						    &io_base);
 	if (ret)
 		return ret;
 
diff --git a/drivers/pci/host/pci-versatile.c b/drivers/pci/host/pci-versatile.c
index 5b3876f5312b..994f32061b32 100644
--- a/drivers/pci/host/pci-versatile.c
+++ b/drivers/pci/host/pci-versatile.c
@@ -15,6 +15,8 @@
 #include <linux/pci.h>
 #include <linux/platform_device.h>
 
+#include "../pci.h"
+
 static void __iomem *versatile_pci_base;
 static void __iomem *versatile_cfg_base[2];
 
@@ -64,11 +66,10 @@ static int versatile_pci_parse_request_of_pci_ranges(struct device *dev,
 						     struct list_head *res)
 {
 	int err, mem = 1, res_valid = 0;
-	struct device_node *np = dev->of_node;
 	resource_size_t iobase;
 	struct resource_entry *win, *tmp;
 
-	err = of_pci_get_host_bridge_resources(np, 0, 0xff, res, &iobase);
+	err = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff, res, &iobase);
 	if (err)
 		return err;
 
diff --git a/drivers/pci/host/pci-xgene.c b/drivers/pci/host/pci-xgene.c
index 0a0d7ee6d3c9..d854d67e873c 100644
--- a/drivers/pci/host/pci-xgene.c
+++ b/drivers/pci/host/pci-xgene.c
@@ -22,6 +22,8 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 
+#include "../pci.h"
+
 #define PCIECORE_CTLANDSTATUS		0x50
 #define PIM1_1L				0x80
 #define IBAR2				0x98
@@ -632,7 +634,8 @@ static int xgene_pcie_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	ret = of_pci_get_host_bridge_resources(dn, 0, 0xff, &res, &iobase);
+	ret = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff, &res,
+						    &iobase);
 	if (ret)
 		return ret;
 
diff --git a/drivers/pci/host/pcie-altera.c b/drivers/pci/host/pcie-altera.c
index a6af62e0256d..7d05e51205b3 100644
--- a/drivers/pci/host/pcie-altera.c
+++ b/drivers/pci/host/pcie-altera.c
@@ -17,6 +17,8 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 
+#include "../pci.h"
+
 #define RP_TX_REG0			0x2000
 #define RP_TX_REG1			0x2004
 #define RP_TX_CNTRL			0x2008
@@ -488,11 +490,10 @@ static int altera_pcie_parse_request_of_pci_ranges(struct altera_pcie *pcie)
 {
 	int err, res_valid = 0;
 	struct device *dev = &pcie->pdev->dev;
-	struct device_node *np = dev->of_node;
 	struct resource_entry *win;
 
-	err = of_pci_get_host_bridge_resources(np, 0, 0xff, &pcie->resources,
-					       NULL);
+	err = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff,
+						    &pcie->resources, NULL);
 	if (err)
 		return err;
 
diff --git a/drivers/pci/host/pcie-iproc-platform.c b/drivers/pci/host/pcie-iproc-platform.c
index e764a2a2693c..f30f5f3fb5c1 100644
--- a/drivers/pci/host/pcie-iproc-platform.c
+++ b/drivers/pci/host/pcie-iproc-platform.c
@@ -16,6 +16,7 @@
 #include <linux/of_platform.h>
 #include <linux/phy/phy.h>
 
+#include "../pci.h"
 #include "pcie-iproc.h"
 
 static const struct of_device_id iproc_pcie_of_match_table[] = {
@@ -99,8 +100,8 @@ static int iproc_pcie_pltfm_probe(struct platform_device *pdev)
 		pcie->phy = NULL;
 	}
 
-	ret = of_pci_get_host_bridge_resources(np, 0, 0xff, &resources,
-					       &iobase);
+	ret = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff, &resources,
+						    &iobase);
 	if (ret) {
 		dev_err(dev, "unable to get PCI host bridge resources\n");
 		return ret;
diff --git a/drivers/pci/host/pcie-mediatek.c b/drivers/pci/host/pcie-mediatek.c
index a8b20c5012a9..0baabe30858f 100644
--- a/drivers/pci/host/pcie-mediatek.c
+++ b/drivers/pci/host/pcie-mediatek.c
@@ -11,8 +11,10 @@
 #include <linux/delay.h>
 #include <linux/iopoll.h>
 #include <linux/irq.h>
+#include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
+#include <linux/msi.h>
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
 #include <linux/of_platform.h>
@@ -22,6 +24,8 @@
 #include <linux/pm_runtime.h>
 #include <linux/reset.h>
 
+#include "../pci.h"
+
 /* PCIe shared registers */
 #define PCIE_SYS_CFG		0x00
 #define PCIE_INT_ENABLE		0x0c
@@ -66,6 +70,10 @@
 
 /* PCIe V2 per-port registers */
 #define PCIE_MSI_VECTOR		0x0c0
+
+#define PCIE_CONF_VEND_ID	0x100
+#define PCIE_CONF_CLASS_ID	0x106
+
 #define PCIE_INT_MASK		0x420
 #define INTX_MASK		GENMASK(19, 16)
 #define INTX_SHIFT		16
@@ -125,13 +133,13 @@ struct mtk_pcie_port;
 
 /**
  * struct mtk_pcie_soc - differentiate between host generations
- * @has_msi: whether this host supports MSI interrupts or not
+ * @need_fix_class_id: whether this host's class ID needed to be fixed or not
  * @ops: pointer to configuration access functions
  * @startup: pointer to controller setting functions
  * @setup_irq: pointer to initialize IRQ functions
  */
 struct mtk_pcie_soc {
-	bool has_msi;
+	bool need_fix_class_id;
 	struct pci_ops *ops;
 	int (*startup)(struct mtk_pcie_port *port);
 	int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node);
@@ -155,7 +163,9 @@ struct mtk_pcie_soc {
  * @lane: lane count
  * @slot: port slot
  * @irq_domain: legacy INTx IRQ domain
+ * @inner_domain: inner IRQ domain
  * @msi_domain: MSI IRQ domain
+ * @lock: protect the msi_irq_in_use bitmap
  * @msi_irq_in_use: bit map for assigned MSI IRQ
  */
 struct mtk_pcie_port {
@@ -173,7 +183,9 @@ struct mtk_pcie_port {
 	u32 lane;
 	u32 slot;
 	struct irq_domain *irq_domain;
+	struct irq_domain *inner_domain;
 	struct irq_domain *msi_domain;
+	struct mutex lock;
 	DECLARE_BITMAP(msi_irq_in_use, MTK_MSI_IRQS_NUM);
 };
 
@@ -375,6 +387,7 @@ static int mtk_pcie_startup_port_v2(struct mtk_pcie_port *port)
 {
 	struct mtk_pcie *pcie = port->pcie;
 	struct resource *mem = &pcie->mem;
+	const struct mtk_pcie_soc *soc = port->pcie->soc;
 	u32 val;
 	size_t size;
 	int err;
@@ -403,6 +416,15 @@ static int mtk_pcie_startup_port_v2(struct mtk_pcie_port *port)
 	       PCIE_MAC_SRSTB | PCIE_CRSTB;
 	writel(val, port->base + PCIE_RST_CTRL);
 
+	/* Set up vendor ID and class code */
+	if (soc->need_fix_class_id) {
+		val = PCI_VENDOR_ID_MEDIATEK;
+		writew(val, port->base + PCIE_CONF_VEND_ID);
+
+		val = PCI_CLASS_BRIDGE_HOST;
+		writew(val, port->base + PCIE_CONF_CLASS_ID);
+	}
+
 	/* 100ms timeout value should be enough for Gen1/2 training */
 	err = readl_poll_timeout(port->base + PCIE_LINK_STATUS_V2, val,
 				 !!(val & PCIE_PORT_LINKUP_V2), 20,
@@ -430,103 +452,130 @@ static int mtk_pcie_startup_port_v2(struct mtk_pcie_port *port)
 	return 0;
 }
 
-static int mtk_pcie_msi_alloc(struct mtk_pcie_port *port)
+static void mtk_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 {
-	int msi;
+	struct mtk_pcie_port *port = irq_data_get_irq_chip_data(data);
+	phys_addr_t addr;
 
-	msi = find_first_zero_bit(port->msi_irq_in_use, MTK_MSI_IRQS_NUM);
-	if (msi < MTK_MSI_IRQS_NUM)
-		set_bit(msi, port->msi_irq_in_use);
-	else
-		return -ENOSPC;
+	/* MT2712/MT7622 only support 32-bit MSI addresses */
+	addr = virt_to_phys(port->base + PCIE_MSI_VECTOR);
+	msg->address_hi = 0;
+	msg->address_lo = lower_32_bits(addr);
+
+	msg->data = data->hwirq;
 
-	return msi;
+	dev_dbg(port->pcie->dev, "msi#%d address_hi %#x address_lo %#x\n",
+		(int)data->hwirq, msg->address_hi, msg->address_lo);
 }
 
-static void mtk_pcie_msi_free(struct mtk_pcie_port *port, unsigned long hwirq)
+static int mtk_msi_set_affinity(struct irq_data *irq_data,
+				const struct cpumask *mask, bool force)
 {
-	clear_bit(hwirq, port->msi_irq_in_use);
+	 return -EINVAL;
 }
 
-static int mtk_pcie_msi_setup_irq(struct msi_controller *chip,
-				  struct pci_dev *pdev, struct msi_desc *desc)
+static void mtk_msi_ack_irq(struct irq_data *data)
 {
-	struct mtk_pcie_port *port;
-	struct msi_msg msg;
-	unsigned int irq;
-	int hwirq;
-	phys_addr_t msg_addr;
+	struct mtk_pcie_port *port = irq_data_get_irq_chip_data(data);
+	u32 hwirq = data->hwirq;
 
-	port = mtk_pcie_find_port(pdev->bus, pdev->devfn);
-	if (!port)
-		return -EINVAL;
+	writel(1 << hwirq, port->base + PCIE_IMSI_STATUS);
+}
 
-	hwirq = mtk_pcie_msi_alloc(port);
-	if (hwirq < 0)
-		return hwirq;
+static struct irq_chip mtk_msi_bottom_irq_chip = {
+	.name			= "MTK MSI",
+	.irq_compose_msi_msg	= mtk_compose_msi_msg,
+	.irq_set_affinity	= mtk_msi_set_affinity,
+	.irq_ack		= mtk_msi_ack_irq,
+};
 
-	irq = irq_create_mapping(port->msi_domain, hwirq);
-	if (!irq) {
-		mtk_pcie_msi_free(port, hwirq);
-		return -EINVAL;
-	}
+static int mtk_pcie_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
+				     unsigned int nr_irqs, void *args)
+{
+	struct mtk_pcie_port *port = domain->host_data;
+	unsigned long bit;
+
+	WARN_ON(nr_irqs != 1);
+	mutex_lock(&port->lock);
 
-	chip->dev = &pdev->dev;
+	bit = find_first_zero_bit(port->msi_irq_in_use, MTK_MSI_IRQS_NUM);
+	if (bit >= MTK_MSI_IRQS_NUM) {
+		mutex_unlock(&port->lock);
+		return -ENOSPC;
+	}
 
-	irq_set_msi_desc(irq, desc);
+	__set_bit(bit, port->msi_irq_in_use);
 
-	/* MT2712/MT7622 only support 32-bit MSI addresses */
-	msg_addr = virt_to_phys(port->base + PCIE_MSI_VECTOR);
-	msg.address_hi = 0;
-	msg.address_lo = lower_32_bits(msg_addr);
-	msg.data = hwirq;
+	mutex_unlock(&port->lock);
 
-	pci_write_msi_msg(irq, &msg);
+	irq_domain_set_info(domain, virq, bit, &mtk_msi_bottom_irq_chip,
+			    domain->host_data, handle_edge_irq,
+			    NULL, NULL);
 
 	return 0;
 }
 
-static void mtk_msi_teardown_irq(struct msi_controller *chip, unsigned int irq)
+static void mtk_pcie_irq_domain_free(struct irq_domain *domain,
+				     unsigned int virq, unsigned int nr_irqs)
 {
-	struct pci_dev *pdev = to_pci_dev(chip->dev);
-	struct irq_data *d = irq_get_irq_data(irq);
-	irq_hw_number_t hwirq = irqd_to_hwirq(d);
-	struct mtk_pcie_port *port;
+	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
+	struct mtk_pcie_port *port = irq_data_get_irq_chip_data(d);
 
-	port = mtk_pcie_find_port(pdev->bus, pdev->devfn);
-	if (!port)
-		return;
+	mutex_lock(&port->lock);
 
-	irq_dispose_mapping(irq);
-	mtk_pcie_msi_free(port, hwirq);
+	if (!test_bit(d->hwirq, port->msi_irq_in_use))
+		dev_err(port->pcie->dev, "trying to free unused MSI#%lu\n",
+			d->hwirq);
+	else
+		__clear_bit(d->hwirq, port->msi_irq_in_use);
+
+	mutex_unlock(&port->lock);
+
+	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
 }
 
-static struct msi_controller mtk_pcie_msi_chip = {
-	.setup_irq = mtk_pcie_msi_setup_irq,
-	.teardown_irq = mtk_msi_teardown_irq,
+static const struct irq_domain_ops msi_domain_ops = {
+	.alloc	= mtk_pcie_irq_domain_alloc,
+	.free	= mtk_pcie_irq_domain_free,
 };
 
 static struct irq_chip mtk_msi_irq_chip = {
-	.name = "MTK PCIe MSI",
-	.irq_enable = pci_msi_unmask_irq,
-	.irq_disable = pci_msi_mask_irq,
-	.irq_mask = pci_msi_mask_irq,
-	.irq_unmask = pci_msi_unmask_irq,
+	.name		= "MTK PCIe MSI",
+	.irq_ack	= irq_chip_ack_parent,
+	.irq_mask	= pci_msi_mask_irq,
+	.irq_unmask	= pci_msi_unmask_irq,
 };
 
-static int mtk_pcie_msi_map(struct irq_domain *domain, unsigned int irq,
-			    irq_hw_number_t hwirq)
+static struct msi_domain_info mtk_msi_domain_info = {
+	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+		   MSI_FLAG_PCI_MSIX),
+	.chip	= &mtk_msi_irq_chip,
+};
+
+static int mtk_pcie_allocate_msi_domains(struct mtk_pcie_port *port)
 {
-	irq_set_chip_and_handler(irq, &mtk_msi_irq_chip, handle_simple_irq);
-	irq_set_chip_data(irq, domain->host_data);
+	struct fwnode_handle *fwnode = of_node_to_fwnode(port->pcie->dev->of_node);
+
+	mutex_init(&port->lock);
+
+	port->inner_domain = irq_domain_create_linear(fwnode, MTK_MSI_IRQS_NUM,
+						      &msi_domain_ops, port);
+	if (!port->inner_domain) {
+		dev_err(port->pcie->dev, "failed to create IRQ domain\n");
+		return -ENOMEM;
+	}
+
+	port->msi_domain = pci_msi_create_irq_domain(fwnode, &mtk_msi_domain_info,
+						     port->inner_domain);
+	if (!port->msi_domain) {
+		dev_err(port->pcie->dev, "failed to create MSI domain\n");
+		irq_domain_remove(port->inner_domain);
+		return -ENOMEM;
+	}
 
 	return 0;
 }
 
-static const struct irq_domain_ops msi_domain_ops = {
-	.map = mtk_pcie_msi_map,
-};
-
 static void mtk_pcie_enable_msi(struct mtk_pcie_port *port)
 {
 	u32 val;
@@ -559,6 +608,7 @@ static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port,
 {
 	struct device *dev = port->pcie->dev;
 	struct device_node *pcie_intc_node;
+	int ret;
 
 	/* Setup INTx */
 	pcie_intc_node = of_get_next_child(node, NULL);
@@ -575,27 +625,28 @@ static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port,
 	}
 
 	if (IS_ENABLED(CONFIG_PCI_MSI)) {
-		port->msi_domain = irq_domain_add_linear(node, MTK_MSI_IRQS_NUM,
-							 &msi_domain_ops,
-							 &mtk_pcie_msi_chip);
-		if (!port->msi_domain) {
-			dev_err(dev, "failed to create MSI IRQ domain\n");
-			return -ENODEV;
-		}
+		ret = mtk_pcie_allocate_msi_domains(port);
+		if (ret)
+			return ret;
+
 		mtk_pcie_enable_msi(port);
 	}
 
 	return 0;
 }
 
-static irqreturn_t mtk_pcie_intr_handler(int irq, void *data)
+static void mtk_pcie_intr_handler(struct irq_desc *desc)
 {
-	struct mtk_pcie_port *port = (struct mtk_pcie_port *)data;
+	struct mtk_pcie_port *port = irq_desc_get_handler_data(desc);
+	struct irq_chip *irqchip = irq_desc_get_chip(desc);
 	unsigned long status;
 	u32 virq;
 	u32 bit = INTX_SHIFT;
 
-	while ((status = readl(port->base + PCIE_INT_STATUS)) & INTX_MASK) {
+	chained_irq_enter(irqchip, desc);
+
+	status = readl(port->base + PCIE_INT_STATUS);
+	if (status & INTX_MASK) {
 		for_each_set_bit_from(bit, &status, PCI_NUM_INTX + INTX_SHIFT) {
 			/* Clear the INTx */
 			writel(1 << bit, port->base + PCIE_INT_STATUS);
@@ -606,14 +657,12 @@ static irqreturn_t mtk_pcie_intr_handler(int irq, void *data)
 	}
 
 	if (IS_ENABLED(CONFIG_PCI_MSI)) {
-		while ((status = readl(port->base + PCIE_INT_STATUS)) & MSI_STATUS) {
+		if (status & MSI_STATUS){
 			unsigned long imsi_status;
 
 			while ((imsi_status = readl(port->base + PCIE_IMSI_STATUS))) {
 				for_each_set_bit(bit, &imsi_status, MTK_MSI_IRQS_NUM) {
-					/* Clear the MSI */
-					writel(1 << bit, port->base + PCIE_IMSI_STATUS);
-					virq = irq_find_mapping(port->msi_domain, bit);
+					virq = irq_find_mapping(port->inner_domain, bit);
 					generic_handle_irq(virq);
 				}
 			}
@@ -622,7 +671,9 @@ static irqreturn_t mtk_pcie_intr_handler(int irq, void *data)
 		}
 	}
 
-	return IRQ_HANDLED;
+	chained_irq_exit(irqchip, desc);
+
+	return;
 }
 
 static int mtk_pcie_setup_irq(struct mtk_pcie_port *port,
@@ -633,20 +684,15 @@ static int mtk_pcie_setup_irq(struct mtk_pcie_port *port,
 	struct platform_device *pdev = to_platform_device(dev);
 	int err, irq;
 
-	irq = platform_get_irq(pdev, port->slot);
-	err = devm_request_irq(dev, irq, mtk_pcie_intr_handler,
-			       IRQF_SHARED, "mtk-pcie", port);
-	if (err) {
-		dev_err(dev, "unable to request IRQ %d\n", irq);
-		return err;
-	}
-
 	err = mtk_pcie_init_irq_domain(port, node);
 	if (err) {
 		dev_err(dev, "failed to init PCIe IRQ domain\n");
 		return err;
 	}
 
+	irq = platform_get_irq(pdev, port->slot);
+	irq_set_chained_handler_and_data(irq, mtk_pcie_intr_handler, port);
+
 	return 0;
 }
 
@@ -1080,8 +1126,6 @@ static int mtk_pcie_register_host(struct pci_host_bridge *host)
 	host->map_irq = of_irq_parse_and_map_pci;
 	host->swizzle_irq = pci_common_swizzle;
 	host->sysdata = pcie;
-	if (IS_ENABLED(CONFIG_PCI_MSI) && pcie->soc->has_msi)
-		host->msi = &mtk_pcie_msi_chip;
 
 	err = pci_scan_root_bus_bridge(host);
 	if (err < 0)
@@ -1142,8 +1186,14 @@ static const struct mtk_pcie_soc mtk_pcie_soc_v1 = {
 	.startup = mtk_pcie_startup_port,
 };
 
-static const struct mtk_pcie_soc mtk_pcie_soc_v2 = {
-	.has_msi = true,
+static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
+	.ops = &mtk_pcie_ops_v2,
+	.startup = mtk_pcie_startup_port_v2,
+	.setup_irq = mtk_pcie_setup_irq,
+};
+
+static const struct mtk_pcie_soc mtk_pcie_soc_mt7622 = {
+	.need_fix_class_id = true,
 	.ops = &mtk_pcie_ops_v2,
 	.startup = mtk_pcie_startup_port_v2,
 	.setup_irq = mtk_pcie_setup_irq,
@@ -1152,8 +1202,8 @@ static const struct mtk_pcie_soc mtk_pcie_soc_v2 = {
 static const struct of_device_id mtk_pcie_ids[] = {
 	{ .compatible = "mediatek,mt2701-pcie", .data = &mtk_pcie_soc_v1 },
 	{ .compatible = "mediatek,mt7623-pcie", .data = &mtk_pcie_soc_v1 },
-	{ .compatible = "mediatek,mt2712-pcie", .data = &mtk_pcie_soc_v2 },
-	{ .compatible = "mediatek,mt7622-pcie", .data = &mtk_pcie_soc_v2 },
+	{ .compatible = "mediatek,mt2712-pcie", .data = &mtk_pcie_soc_mt2712 },
+	{ .compatible = "mediatek,mt7622-pcie", .data = &mtk_pcie_soc_mt7622 },
 	{},
 };
 
diff --git a/drivers/pci/host/pcie-mobiveil.c b/drivers/pci/host/pcie-mobiveil.c
new file mode 100644
index 000000000000..f3fbe9d5e67c
--- /dev/null
+++ b/drivers/pci/host/pcie-mobiveil.c
@@ -0,0 +1,865 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PCIe host controller driver for Mobiveil PCIe Host controller
+ *
+ * Copyright (c) 2018 Mobiveil Inc.
+ * Author: Subrahmanya Lingappa <l.subrahmanya@mobiveil.co.in>
+ */
+
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/msi.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/of_pci.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+/* register offsets and bit positions */
+
+/*
+ * translation tables are grouped into windows, each window registers are
+ * grouped into blocks of 4 or 16 registers each
+ */
+#define PAB_REG_BLOCK_SIZE	16
+#define PAB_EXT_REG_BLOCK_SIZE	4
+
+#define PAB_REG_ADDR(offset, win) (offset + (win * PAB_REG_BLOCK_SIZE))
+#define PAB_EXT_REG_ADDR(offset, win) (offset + (win * PAB_EXT_REG_BLOCK_SIZE))
+
+#define LTSSM_STATUS		0x0404
+#define  LTSSM_STATUS_L0_MASK	0x3f
+#define  LTSSM_STATUS_L0	0x2d
+
+#define PAB_CTRL		0x0808
+#define  AMBA_PIO_ENABLE_SHIFT	0
+#define  PEX_PIO_ENABLE_SHIFT	1
+#define  PAGE_SEL_SHIFT	13
+#define  PAGE_SEL_MASK		0x3f
+#define  PAGE_LO_MASK		0x3ff
+#define  PAGE_SEL_EN		0xc00
+#define  PAGE_SEL_OFFSET_SHIFT	10
+
+#define PAB_AXI_PIO_CTRL	0x0840
+#define  APIO_EN_MASK		0xf
+
+#define PAB_PEX_PIO_CTRL	0x08c0
+#define  PIO_ENABLE_SHIFT	0
+
+#define PAB_INTP_AMBA_MISC_ENB		0x0b0c
+#define PAB_INTP_AMBA_MISC_STAT	0x0b1c
+#define  PAB_INTP_INTX_MASK		0x01e0
+#define  PAB_INTP_MSI_MASK		0x8
+
+#define PAB_AXI_AMAP_CTRL(win)	PAB_REG_ADDR(0x0ba0, win)
+#define  WIN_ENABLE_SHIFT	0
+#define  WIN_TYPE_SHIFT	1
+
+#define PAB_EXT_AXI_AMAP_SIZE(win)	PAB_EXT_REG_ADDR(0xbaf0, win)
+
+#define PAB_AXI_AMAP_AXI_WIN(win)	PAB_REG_ADDR(0x0ba4, win)
+#define  AXI_WINDOW_ALIGN_MASK		3
+
+#define PAB_AXI_AMAP_PEX_WIN_L(win)	PAB_REG_ADDR(0x0ba8, win)
+#define  PAB_BUS_SHIFT		24
+#define  PAB_DEVICE_SHIFT	19
+#define  PAB_FUNCTION_SHIFT	16
+
+#define PAB_AXI_AMAP_PEX_WIN_H(win)	PAB_REG_ADDR(0x0bac, win)
+#define PAB_INTP_AXI_PIO_CLASS		0x474
+
+#define PAB_PEX_AMAP_CTRL(win)	PAB_REG_ADDR(0x4ba0, win)
+#define  AMAP_CTRL_EN_SHIFT	0
+#define  AMAP_CTRL_TYPE_SHIFT	1
+
+#define PAB_EXT_PEX_AMAP_SIZEN(win)	PAB_EXT_REG_ADDR(0xbef0, win)
+#define PAB_PEX_AMAP_AXI_WIN(win)	PAB_REG_ADDR(0x4ba4, win)
+#define PAB_PEX_AMAP_PEX_WIN_L(win)	PAB_REG_ADDR(0x4ba8, win)
+#define PAB_PEX_AMAP_PEX_WIN_H(win)	PAB_REG_ADDR(0x4bac, win)
+
+/* starting offset of INTX bits in status register */
+#define PAB_INTX_START	5
+
+/* supported number of MSI interrupts */
+#define PCI_NUM_MSI	16
+
+/* MSI registers */
+#define MSI_BASE_LO_OFFSET	0x04
+#define MSI_BASE_HI_OFFSET	0x08
+#define MSI_SIZE_OFFSET	0x0c
+#define MSI_ENABLE_OFFSET	0x14
+#define MSI_STATUS_OFFSET	0x18
+#define MSI_DATA_OFFSET	0x20
+#define MSI_ADDR_L_OFFSET	0x24
+#define MSI_ADDR_H_OFFSET	0x28
+
+/* outbound and inbound window definitions */
+#define WIN_NUM_0		0
+#define WIN_NUM_1		1
+#define CFG_WINDOW_TYPE	0
+#define IO_WINDOW_TYPE		1
+#define MEM_WINDOW_TYPE	2
+#define IB_WIN_SIZE		(256 * 1024 * 1024 * 1024)
+#define MAX_PIO_WINDOWS	8
+
+/* Parameters for the waiting for link up routine */
+#define LINK_WAIT_MAX_RETRIES	10
+#define LINK_WAIT_MIN	90000
+#define LINK_WAIT_MAX	100000
+
+struct mobiveil_msi {			/* MSI information */
+	struct mutex lock;		/* protect bitmap variable */
+	struct irq_domain *msi_domain;
+	struct irq_domain *dev_domain;
+	phys_addr_t msi_pages_phys;
+	int num_of_vectors;
+	DECLARE_BITMAP(msi_irq_in_use, PCI_NUM_MSI);
+};
+
+struct mobiveil_pcie {
+	struct platform_device *pdev;
+	struct list_head resources;
+	void __iomem *config_axi_slave_base;	/* endpoint config base */
+	void __iomem *csr_axi_slave_base;	/* root port config base */
+	void __iomem *apb_csr_base;	/* MSI register base */
+	void __iomem *pcie_reg_base;	/* Physical PCIe Controller Base */
+	struct irq_domain *intx_domain;
+	raw_spinlock_t intx_mask_lock;
+	int irq;
+	int apio_wins;
+	int ppio_wins;
+	int ob_wins_configured;		/* configured outbound windows */
+	int ib_wins_configured;		/* configured inbound windows */
+	struct resource *ob_io_res;
+	char root_bus_nr;
+	struct mobiveil_msi msi;
+};
+
+static inline void csr_writel(struct mobiveil_pcie *pcie, const u32 value,
+		const u32 reg)
+{
+	writel_relaxed(value, pcie->csr_axi_slave_base + reg);
+}
+
+static inline u32 csr_readl(struct mobiveil_pcie *pcie, const u32 reg)
+{
+	return readl_relaxed(pcie->csr_axi_slave_base + reg);
+}
+
+static bool mobiveil_pcie_link_up(struct mobiveil_pcie *pcie)
+{
+	return (csr_readl(pcie, LTSSM_STATUS) &
+		LTSSM_STATUS_L0_MASK) == LTSSM_STATUS_L0;
+}
+
+static bool mobiveil_pcie_valid_device(struct pci_bus *bus, unsigned int devfn)
+{
+	struct mobiveil_pcie *pcie = bus->sysdata;
+
+	/* Only one device down on each root port */
+	if ((bus->number == pcie->root_bus_nr) && (devfn > 0))
+		return false;
+
+	/*
+	 * Do not read more than one device on the bus directly
+	 * attached to RC
+	 */
+	if ((bus->primary == pcie->root_bus_nr) && (devfn > 0))
+		return false;
+
+	return true;
+}
+
+/*
+ * mobiveil_pcie_map_bus - routine to get the configuration base of either
+ * root port or endpoint
+ */
+static void __iomem *mobiveil_pcie_map_bus(struct pci_bus *bus,
+					unsigned int devfn, int where)
+{
+	struct mobiveil_pcie *pcie = bus->sysdata;
+
+	if (!mobiveil_pcie_valid_device(bus, devfn))
+		return NULL;
+
+	if (bus->number == pcie->root_bus_nr) {
+		/* RC config access */
+		return pcie->csr_axi_slave_base + where;
+	}
+
+	/*
+	 * EP config access (in Config/APIO space)
+	 * Program PEX Address base (31..16 bits) with appropriate value
+	 * (BDF) in PAB_AXI_AMAP_PEX_WIN_L0 Register.
+	 * Relies on pci_lock serialization
+	 */
+	csr_writel(pcie, bus->number << PAB_BUS_SHIFT |
+			PCI_SLOT(devfn) << PAB_DEVICE_SHIFT |
+			PCI_FUNC(devfn) << PAB_FUNCTION_SHIFT,
+			PAB_AXI_AMAP_PEX_WIN_L(WIN_NUM_0));
+	return pcie->config_axi_slave_base + where;
+}
+
+static struct pci_ops mobiveil_pcie_ops = {
+	.map_bus = mobiveil_pcie_map_bus,
+	.read = pci_generic_config_read,
+	.write = pci_generic_config_write,
+};
+
+static void mobiveil_pcie_isr(struct irq_desc *desc)
+{
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	struct mobiveil_pcie *pcie = irq_desc_get_handler_data(desc);
+	struct device *dev = &pcie->pdev->dev;
+	struct mobiveil_msi *msi = &pcie->msi;
+	u32 msi_data, msi_addr_lo, msi_addr_hi;
+	u32 intr_status, msi_status;
+	unsigned long shifted_status;
+	u32 bit, virq, val, mask;
+
+	/*
+	 * The core provides a single interrupt for both INTx/MSI messages.
+	 * So we'll read both INTx and MSI status
+	 */
+
+	chained_irq_enter(chip, desc);
+
+	/* read INTx status */
+	val = csr_readl(pcie, PAB_INTP_AMBA_MISC_STAT);
+	mask = csr_readl(pcie, PAB_INTP_AMBA_MISC_ENB);
+	intr_status = val & mask;
+
+	/* Handle INTx */
+	if (intr_status & PAB_INTP_INTX_MASK) {
+		shifted_status = csr_readl(pcie, PAB_INTP_AMBA_MISC_STAT) >>
+			PAB_INTX_START;
+		do {
+			for_each_set_bit(bit, &shifted_status, PCI_NUM_INTX) {
+				virq = irq_find_mapping(pcie->intx_domain,
+						bit + 1);
+				if (virq)
+					generic_handle_irq(virq);
+				else
+					dev_err_ratelimited(dev,
+						"unexpected IRQ, INT%d\n", bit);
+
+				/* clear interrupt */
+				csr_writel(pcie,
+					shifted_status << PAB_INTX_START,
+					PAB_INTP_AMBA_MISC_STAT);
+			}
+		} while ((shifted_status >> PAB_INTX_START) != 0);
+	}
+
+	/* read extra MSI status register */
+	msi_status = readl_relaxed(pcie->apb_csr_base + MSI_STATUS_OFFSET);
+
+	/* handle MSI interrupts */
+	while (msi_status & 1) {
+		msi_data = readl_relaxed(pcie->apb_csr_base
+				+ MSI_DATA_OFFSET);
+
+		/*
+		 * MSI_STATUS_OFFSET register gets updated to zero
+		 * once we pop not only the MSI data but also address
+		 * from MSI hardware FIFO. So keeping these following
+		 * two dummy reads.
+		 */
+		msi_addr_lo = readl_relaxed(pcie->apb_csr_base +
+				MSI_ADDR_L_OFFSET);
+		msi_addr_hi = readl_relaxed(pcie->apb_csr_base +
+				MSI_ADDR_H_OFFSET);
+		dev_dbg(dev, "MSI registers, data: %08x, addr: %08x:%08x\n",
+				msi_data, msi_addr_hi, msi_addr_lo);
+
+		virq = irq_find_mapping(msi->dev_domain, msi_data);
+		if (virq)
+			generic_handle_irq(virq);
+
+		msi_status = readl_relaxed(pcie->apb_csr_base +
+				MSI_STATUS_OFFSET);
+	}
+
+	/* Clear the interrupt status */
+	csr_writel(pcie, intr_status, PAB_INTP_AMBA_MISC_STAT);
+	chained_irq_exit(chip, desc);
+}
+
+static int mobiveil_pcie_parse_dt(struct mobiveil_pcie *pcie)
+{
+	struct device *dev = &pcie->pdev->dev;
+	struct platform_device *pdev = pcie->pdev;
+	struct device_node *node = dev->of_node;
+	struct resource *res;
+	const char *type;
+
+	type = of_get_property(node, "device_type", NULL);
+	if (!type || strcmp(type, "pci")) {
+		dev_err(dev, "invalid \"device_type\" %s\n", type);
+		return -EINVAL;
+	}
+
+	/* map config resource */
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+			"config_axi_slave");
+	pcie->config_axi_slave_base = devm_pci_remap_cfg_resource(dev, res);
+	if (IS_ERR(pcie->config_axi_slave_base))
+		return PTR_ERR(pcie->config_axi_slave_base);
+	pcie->ob_io_res = res;
+
+	/* map csr resource */
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+			"csr_axi_slave");
+	pcie->csr_axi_slave_base = devm_pci_remap_cfg_resource(dev, res);
+	if (IS_ERR(pcie->csr_axi_slave_base))
+		return PTR_ERR(pcie->csr_axi_slave_base);
+	pcie->pcie_reg_base = res->start;
+
+	/* map MSI config resource */
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "apb_csr");
+	pcie->apb_csr_base = devm_pci_remap_cfg_resource(dev, res);
+	if (IS_ERR(pcie->apb_csr_base))
+		return PTR_ERR(pcie->apb_csr_base);
+
+	/* read the number of windows requested */
+	if (of_property_read_u32(node, "apio-wins", &pcie->apio_wins))
+		pcie->apio_wins = MAX_PIO_WINDOWS;
+
+	if (of_property_read_u32(node, "ppio-wins", &pcie->ppio_wins))
+		pcie->ppio_wins = MAX_PIO_WINDOWS;
+
+	pcie->irq = platform_get_irq(pdev, 0);
+	if (pcie->irq <= 0) {
+		dev_err(dev, "failed to map IRQ: %d\n", pcie->irq);
+		return -ENODEV;
+	}
+
+	irq_set_chained_handler_and_data(pcie->irq, mobiveil_pcie_isr, pcie);
+
+	return 0;
+}
+
+/*
+ * select_paged_register - routine to access paged register of root complex
+ *
+ * registers of RC are paged, for this scheme to work
+ * extracted higher 6 bits of the offset will be written to pg_sel
+ * field of PAB_CTRL register and rest of the lower 10 bits enabled with
+ * PAGE_SEL_EN are used as offset of the register.
+ */
+static void select_paged_register(struct mobiveil_pcie *pcie, u32 offset)
+{
+	int pab_ctrl_dw, pg_sel;
+
+	/* clear pg_sel field */
+	pab_ctrl_dw = csr_readl(pcie, PAB_CTRL);
+	pab_ctrl_dw = (pab_ctrl_dw & ~(PAGE_SEL_MASK << PAGE_SEL_SHIFT));
+
+	/* set pg_sel field */
+	pg_sel = (offset >> PAGE_SEL_OFFSET_SHIFT) & PAGE_SEL_MASK;
+	pab_ctrl_dw |= ((pg_sel << PAGE_SEL_SHIFT));
+	csr_writel(pcie, pab_ctrl_dw, PAB_CTRL);
+}
+
+static void write_paged_register(struct mobiveil_pcie *pcie,
+		u32 val, u32 offset)
+{
+	u32 off = (offset & PAGE_LO_MASK) | PAGE_SEL_EN;
+
+	select_paged_register(pcie, offset);
+	csr_writel(pcie, val, off);
+}
+
+static u32 read_paged_register(struct mobiveil_pcie *pcie, u32 offset)
+{
+	u32 off = (offset & PAGE_LO_MASK) | PAGE_SEL_EN;
+
+	select_paged_register(pcie, offset);
+	return csr_readl(pcie, off);
+}
+
+static void program_ib_windows(struct mobiveil_pcie *pcie, int win_num,
+		int pci_addr, u32 type, u64 size)
+{
+	int pio_ctrl_val;
+	int amap_ctrl_dw;
+	u64 size64 = ~(size - 1);
+
+	if ((pcie->ib_wins_configured + 1) > pcie->ppio_wins) {
+		dev_err(&pcie->pdev->dev,
+			"ERROR: max inbound windows reached !\n");
+		return;
+	}
+
+	pio_ctrl_val = csr_readl(pcie, PAB_PEX_PIO_CTRL);
+	csr_writel(pcie,
+		pio_ctrl_val | (1 << PIO_ENABLE_SHIFT), PAB_PEX_PIO_CTRL);
+	amap_ctrl_dw = read_paged_register(pcie, PAB_PEX_AMAP_CTRL(win_num));
+	amap_ctrl_dw = (amap_ctrl_dw | (type << AMAP_CTRL_TYPE_SHIFT));
+	amap_ctrl_dw = (amap_ctrl_dw | (1 << AMAP_CTRL_EN_SHIFT));
+
+	write_paged_register(pcie, amap_ctrl_dw | lower_32_bits(size64),
+				PAB_PEX_AMAP_CTRL(win_num));
+
+	write_paged_register(pcie, upper_32_bits(size64),
+				PAB_EXT_PEX_AMAP_SIZEN(win_num));
+
+	write_paged_register(pcie, pci_addr, PAB_PEX_AMAP_AXI_WIN(win_num));
+	write_paged_register(pcie, pci_addr, PAB_PEX_AMAP_PEX_WIN_L(win_num));
+	write_paged_register(pcie, 0, PAB_PEX_AMAP_PEX_WIN_H(win_num));
+}
+
+/*
+ * routine to program the outbound windows
+ */
+static void program_ob_windows(struct mobiveil_pcie *pcie, int win_num,
+		u64 cpu_addr, u64 pci_addr, u32 config_io_bit, u64 size)
+{
+
+	u32 value, type;
+	u64 size64 = ~(size - 1);
+
+	if ((pcie->ob_wins_configured + 1) > pcie->apio_wins) {
+		dev_err(&pcie->pdev->dev,
+			"ERROR: max outbound windows reached !\n");
+		return;
+	}
+
+	/*
+	 * program Enable Bit to 1, Type Bit to (00) base 2, AXI Window Size Bit
+	 * to 4 KB in PAB_AXI_AMAP_CTRL register
+	 */
+	type = config_io_bit;
+	value = csr_readl(pcie, PAB_AXI_AMAP_CTRL(win_num));
+	csr_writel(pcie, 1 << WIN_ENABLE_SHIFT | type << WIN_TYPE_SHIFT |
+			lower_32_bits(size64), PAB_AXI_AMAP_CTRL(win_num));
+
+	write_paged_register(pcie, upper_32_bits(size64),
+				PAB_EXT_AXI_AMAP_SIZE(win_num));
+
+	/*
+	 * program AXI window base with appropriate value in
+	 * PAB_AXI_AMAP_AXI_WIN0 register
+	 */
+	value = csr_readl(pcie, PAB_AXI_AMAP_AXI_WIN(win_num));
+	csr_writel(pcie, cpu_addr & (~AXI_WINDOW_ALIGN_MASK),
+			PAB_AXI_AMAP_AXI_WIN(win_num));
+
+	value = csr_readl(pcie, PAB_AXI_AMAP_PEX_WIN_H(win_num));
+
+	csr_writel(pcie, lower_32_bits(pci_addr),
+			PAB_AXI_AMAP_PEX_WIN_L(win_num));
+	csr_writel(pcie, upper_32_bits(pci_addr),
+			PAB_AXI_AMAP_PEX_WIN_H(win_num));
+
+	pcie->ob_wins_configured++;
+}
+
+static int mobiveil_bringup_link(struct mobiveil_pcie *pcie)
+{
+	int retries;
+
+	/* check if the link is up or not */
+	for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
+		if (mobiveil_pcie_link_up(pcie))
+			return 0;
+
+		usleep_range(LINK_WAIT_MIN, LINK_WAIT_MAX);
+	}
+	dev_err(&pcie->pdev->dev, "link never came up\n");
+	return -ETIMEDOUT;
+}
+
+static void mobiveil_pcie_enable_msi(struct mobiveil_pcie *pcie)
+{
+	phys_addr_t msg_addr = pcie->pcie_reg_base;
+	struct mobiveil_msi *msi = &pcie->msi;
+
+	pcie->msi.num_of_vectors = PCI_NUM_MSI;
+	msi->msi_pages_phys = (phys_addr_t)msg_addr;
+
+	writel_relaxed(lower_32_bits(msg_addr),
+		pcie->apb_csr_base + MSI_BASE_LO_OFFSET);
+	writel_relaxed(upper_32_bits(msg_addr),
+		pcie->apb_csr_base + MSI_BASE_HI_OFFSET);
+	writel_relaxed(4096, pcie->apb_csr_base + MSI_SIZE_OFFSET);
+	writel_relaxed(1, pcie->apb_csr_base + MSI_ENABLE_OFFSET);
+}
+
+static int mobiveil_host_init(struct mobiveil_pcie *pcie)
+{
+	u32 value, pab_ctrl, type = 0;
+	int err;
+	struct resource_entry *win, *tmp;
+
+	err = mobiveil_bringup_link(pcie);
+	if (err) {
+		dev_info(&pcie->pdev->dev, "link bring-up failed\n");
+		return err;
+	}
+
+	/*
+	 * program Bus Master Enable Bit in Command Register in PAB Config
+	 * Space
+	 */
+	value = csr_readl(pcie, PCI_COMMAND);
+	csr_writel(pcie, value | PCI_COMMAND_IO | PCI_COMMAND_MEMORY |
+		PCI_COMMAND_MASTER, PCI_COMMAND);
+
+	/*
+	 * program PIO Enable Bit to 1 (and PEX PIO Enable to 1) in PAB_CTRL
+	 * register
+	 */
+	pab_ctrl = csr_readl(pcie, PAB_CTRL);
+	csr_writel(pcie, pab_ctrl | (1 << AMBA_PIO_ENABLE_SHIFT) |
+		(1 << PEX_PIO_ENABLE_SHIFT), PAB_CTRL);
+
+	csr_writel(pcie, (PAB_INTP_INTX_MASK | PAB_INTP_MSI_MASK),
+		PAB_INTP_AMBA_MISC_ENB);
+
+	/*
+	 * program PIO Enable Bit to 1 and Config Window Enable Bit to 1 in
+	 * PAB_AXI_PIO_CTRL Register
+	 */
+	value = csr_readl(pcie, PAB_AXI_PIO_CTRL);
+	csr_writel(pcie, value | APIO_EN_MASK, PAB_AXI_PIO_CTRL);
+
+	/*
+	 * we'll program one outbound window for config reads and
+	 * another default inbound window for all the upstream traffic
+	 * rest of the outbound windows will be configured according to
+	 * the "ranges" field defined in device tree
+	 */
+
+	/* config outbound translation window */
+	program_ob_windows(pcie, pcie->ob_wins_configured,
+			pcie->ob_io_res->start, 0, CFG_WINDOW_TYPE,
+			resource_size(pcie->ob_io_res));
+
+	/* memory inbound translation window */
+	program_ib_windows(pcie, WIN_NUM_1, 0, MEM_WINDOW_TYPE, IB_WIN_SIZE);
+
+	/* Get the I/O and memory ranges from DT */
+	resource_list_for_each_entry_safe(win, tmp, &pcie->resources) {
+		type = 0;
+		if (resource_type(win->res) == IORESOURCE_MEM)
+			type = MEM_WINDOW_TYPE;
+		if (resource_type(win->res) == IORESOURCE_IO)
+			type = IO_WINDOW_TYPE;
+		if (type) {
+			/* configure outbound translation window */
+			program_ob_windows(pcie, pcie->ob_wins_configured,
+				win->res->start, 0, type,
+				resource_size(win->res));
+		}
+	}
+
+	/* setup MSI hardware registers */
+	mobiveil_pcie_enable_msi(pcie);
+
+	return err;
+}
+
+static void mobiveil_mask_intx_irq(struct irq_data *data)
+{
+	struct irq_desc *desc = irq_to_desc(data->irq);
+	struct mobiveil_pcie *pcie;
+	unsigned long flags;
+	u32 mask, shifted_val;
+
+	pcie = irq_desc_get_chip_data(desc);
+	mask = 1 << ((data->hwirq + PAB_INTX_START) - 1);
+	raw_spin_lock_irqsave(&pcie->intx_mask_lock, flags);
+	shifted_val = csr_readl(pcie, PAB_INTP_AMBA_MISC_ENB);
+	csr_writel(pcie, (shifted_val & (~mask)), PAB_INTP_AMBA_MISC_ENB);
+	raw_spin_unlock_irqrestore(&pcie->intx_mask_lock, flags);
+}
+
+static void mobiveil_unmask_intx_irq(struct irq_data *data)
+{
+	struct irq_desc *desc = irq_to_desc(data->irq);
+	struct mobiveil_pcie *pcie;
+	unsigned long flags;
+	u32 shifted_val, mask;
+
+	pcie = irq_desc_get_chip_data(desc);
+	mask = 1 << ((data->hwirq + PAB_INTX_START) - 1);
+	raw_spin_lock_irqsave(&pcie->intx_mask_lock, flags);
+	shifted_val = csr_readl(pcie, PAB_INTP_AMBA_MISC_ENB);
+	csr_writel(pcie, (shifted_val | mask), PAB_INTP_AMBA_MISC_ENB);
+	raw_spin_unlock_irqrestore(&pcie->intx_mask_lock, flags);
+}
+
+static struct irq_chip intx_irq_chip = {
+	.name = "mobiveil_pcie:intx",
+	.irq_enable = mobiveil_unmask_intx_irq,
+	.irq_disable = mobiveil_mask_intx_irq,
+	.irq_mask = mobiveil_mask_intx_irq,
+	.irq_unmask = mobiveil_unmask_intx_irq,
+};
+
+/* routine to setup the INTx related data */
+static int mobiveil_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
+		irq_hw_number_t hwirq)
+{
+	irq_set_chip_and_handler(irq, &intx_irq_chip, handle_level_irq);
+	irq_set_chip_data(irq, domain->host_data);
+	return 0;
+}
+
+/* INTx domain operations structure */
+static const struct irq_domain_ops intx_domain_ops = {
+	.map = mobiveil_pcie_intx_map,
+};
+
+static struct irq_chip mobiveil_msi_irq_chip = {
+	.name = "Mobiveil PCIe MSI",
+	.irq_mask = pci_msi_mask_irq,
+	.irq_unmask = pci_msi_unmask_irq,
+};
+
+static struct msi_domain_info mobiveil_msi_domain_info = {
+	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+		MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX),
+	.chip	= &mobiveil_msi_irq_chip,
+};
+
+static void mobiveil_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+	struct mobiveil_pcie *pcie = irq_data_get_irq_chip_data(data);
+	phys_addr_t addr = pcie->pcie_reg_base + (data->hwirq * sizeof(int));
+
+	msg->address_lo = lower_32_bits(addr);
+	msg->address_hi = upper_32_bits(addr);
+	msg->data = data->hwirq;
+
+	dev_dbg(&pcie->pdev->dev, "msi#%d address_hi %#x address_lo %#x\n",
+		(int)data->hwirq, msg->address_hi, msg->address_lo);
+}
+
+static int mobiveil_msi_set_affinity(struct irq_data *irq_data,
+		const struct cpumask *mask, bool force)
+{
+	return -EINVAL;
+}
+
+static struct irq_chip mobiveil_msi_bottom_irq_chip = {
+	.name			= "Mobiveil MSI",
+	.irq_compose_msi_msg	= mobiveil_compose_msi_msg,
+	.irq_set_affinity	= mobiveil_msi_set_affinity,
+};
+
+static int mobiveil_irq_msi_domain_alloc(struct irq_domain *domain,
+		unsigned int virq, unsigned int nr_irqs, void *args)
+{
+	struct mobiveil_pcie *pcie = domain->host_data;
+	struct mobiveil_msi *msi = &pcie->msi;
+	unsigned long bit;
+
+	WARN_ON(nr_irqs != 1);
+	mutex_lock(&msi->lock);
+
+	bit = find_first_zero_bit(msi->msi_irq_in_use, msi->num_of_vectors);
+	if (bit >= msi->num_of_vectors) {
+		mutex_unlock(&msi->lock);
+		return -ENOSPC;
+	}
+
+	set_bit(bit, msi->msi_irq_in_use);
+
+	mutex_unlock(&msi->lock);
+
+	irq_domain_set_info(domain, virq, bit, &mobiveil_msi_bottom_irq_chip,
+				domain->host_data, handle_level_irq,
+				NULL, NULL);
+	return 0;
+}
+
+static void mobiveil_irq_msi_domain_free(struct irq_domain *domain,
+		unsigned int virq, unsigned int nr_irqs)
+{
+	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
+	struct mobiveil_pcie *pcie = irq_data_get_irq_chip_data(d);
+	struct mobiveil_msi *msi = &pcie->msi;
+
+	mutex_lock(&msi->lock);
+
+	if (!test_bit(d->hwirq, msi->msi_irq_in_use)) {
+		dev_err(&pcie->pdev->dev, "trying to free unused MSI#%lu\n",
+			d->hwirq);
+	} else {
+		__clear_bit(d->hwirq, msi->msi_irq_in_use);
+	}
+
+	mutex_unlock(&msi->lock);
+}
+static const struct irq_domain_ops msi_domain_ops = {
+	.alloc	= mobiveil_irq_msi_domain_alloc,
+	.free	= mobiveil_irq_msi_domain_free,
+};
+
+static int mobiveil_allocate_msi_domains(struct mobiveil_pcie *pcie)
+{
+	struct device *dev = &pcie->pdev->dev;
+	struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
+	struct mobiveil_msi *msi = &pcie->msi;
+
+	mutex_init(&pcie->msi.lock);
+	msi->dev_domain = irq_domain_add_linear(NULL, msi->num_of_vectors,
+						&msi_domain_ops, pcie);
+	if (!msi->dev_domain) {
+		dev_err(dev, "failed to create IRQ domain\n");
+		return -ENOMEM;
+	}
+
+	msi->msi_domain = pci_msi_create_irq_domain(fwnode,
+				&mobiveil_msi_domain_info, msi->dev_domain);
+	if (!msi->msi_domain) {
+		dev_err(dev, "failed to create MSI domain\n");
+		irq_domain_remove(msi->dev_domain);
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+static int mobiveil_pcie_init_irq_domain(struct mobiveil_pcie *pcie)
+{
+	struct device *dev = &pcie->pdev->dev;
+	struct device_node *node = dev->of_node;
+	int ret;
+
+	/* setup INTx */
+	pcie->intx_domain = irq_domain_add_linear(node,
+				PCI_NUM_INTX, &intx_domain_ops, pcie);
+
+	if (!pcie->intx_domain) {
+		dev_err(dev, "Failed to get a INTx IRQ domain\n");
+		return -ENODEV;
+	}
+
+	raw_spin_lock_init(&pcie->intx_mask_lock);
+
+	/* setup MSI */
+	ret = mobiveil_allocate_msi_domains(pcie);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int mobiveil_pcie_probe(struct platform_device *pdev)
+{
+	struct mobiveil_pcie *pcie;
+	struct pci_bus *bus;
+	struct pci_bus *child;
+	struct pci_host_bridge *bridge;
+	struct device *dev = &pdev->dev;
+	resource_size_t iobase;
+	int ret;
+
+	/* allocate the PCIe port */
+	bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
+	if (!bridge)
+		return -ENODEV;
+
+	pcie = pci_host_bridge_priv(bridge);
+	if (!pcie)
+		return -ENOMEM;
+
+	pcie->pdev = pdev;
+
+	ret = mobiveil_pcie_parse_dt(pcie);
+	if (ret) {
+		dev_err(dev, "Parsing DT failed, ret: %x\n", ret);
+		return ret;
+	}
+
+	INIT_LIST_HEAD(&pcie->resources);
+
+	/* parse the host bridge base addresses from the device tree file */
+	ret = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff,
+						    &pcie->resources, &iobase);
+	if (ret) {
+		dev_err(dev, "Getting bridge resources failed\n");
+		return -ENOMEM;
+	}
+
+	/*
+	 * configure all inbound and outbound windows and prepare the RC for
+	 * config access
+	 */
+	ret = mobiveil_host_init(pcie);
+	if (ret) {
+		dev_err(dev, "Failed to initialize host\n");
+		goto error;
+	}
+
+	/* fixup for PCIe class register */
+	csr_writel(pcie, 0x060402ab, PAB_INTP_AXI_PIO_CLASS);
+
+	/* initialize the IRQ domains */
+	ret = mobiveil_pcie_init_irq_domain(pcie);
+	if (ret) {
+		dev_err(dev, "Failed creating IRQ Domain\n");
+		goto error;
+	}
+
+	ret = devm_request_pci_bus_resources(dev, &pcie->resources);
+	if (ret)
+		goto error;
+
+	/* Initialize bridge */
+	list_splice_init(&pcie->resources, &bridge->windows);
+	bridge->dev.parent = dev;
+	bridge->sysdata = pcie;
+	bridge->busnr = pcie->root_bus_nr;
+	bridge->ops = &mobiveil_pcie_ops;
+	bridge->map_irq = of_irq_parse_and_map_pci;
+	bridge->swizzle_irq = pci_common_swizzle;
+
+	/* setup the kernel resources for the newly added PCIe root bus */
+	ret = pci_scan_root_bus_bridge(bridge);
+	if (ret)
+		goto error;
+
+	bus = bridge->bus;
+
+	pci_assign_unassigned_bus_resources(bus);
+	list_for_each_entry(child, &bus->children, node)
+		pcie_bus_configure_settings(child);
+	pci_bus_add_devices(bus);
+
+	return 0;
+error:
+	pci_free_resource_list(&pcie->resources);
+	return ret;
+}
+
+static const struct of_device_id mobiveil_pcie_of_match[] = {
+	{.compatible = "mbvl,gpex40-pcie",},
+};
+
+MODULE_DEVICE_TABLE(of, mobiveil_pcie_of_match);
+
+static struct platform_driver mobiveil_pcie_driver = {
+	.probe = mobiveil_pcie_probe,
+	.driver = {
+			.name = "mobiveil-pcie",
+			.of_match_table = mobiveil_pcie_of_match,
+			.suppress_bind_attrs = true,
+		},
+};
+
+builtin_platform_driver(mobiveil_pcie_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Mobiveil PCIe host controller driver");
+MODULE_AUTHOR("Subrahmanya Lingappa <l.subrahmanya@mobiveil.co.in>");
diff --git a/drivers/pci/host/pcie-rcar.c b/drivers/pci/host/pcie-rcar.c
index 6ab28f29ac6a..874d75c9ee4a 100644
--- a/drivers/pci/host/pcie-rcar.c
+++ b/drivers/pci/host/pcie-rcar.c
@@ -11,6 +11,7 @@
  * Author: Phil Edworthy <phil.edworthy@renesas.com>
  */
 
+#include <linux/bitops.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
@@ -24,18 +25,23 @@
 #include <linux/of_pci.h>
 #include <linux/of_platform.h>
 #include <linux/pci.h>
+#include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
 
+#include "../pci.h"
+
 #define PCIECAR			0x000010
 #define PCIECCTLR		0x000018
-#define  CONFIG_SEND_ENABLE	(1 << 31)
+#define  CONFIG_SEND_ENABLE	BIT(31)
 #define  TYPE0			(0 << 8)
-#define  TYPE1			(1 << 8)
+#define  TYPE1			BIT(8)
 #define PCIECDR			0x000020
 #define PCIEMSR			0x000028
 #define PCIEINTXR		0x000400
+#define PCIEPHYSR		0x0007f0
+#define  PHYRDY			BIT(0)
 #define PCIEMSITXR		0x000840
 
 /* Transfer control */
@@ -44,7 +50,7 @@
 #define PCIETSTR		0x02004
 #define  DATA_LINK_ACTIVE	1
 #define PCIEERRFR		0x02020
-#define  UNSUPPORTED_REQUEST	(1 << 4)
+#define  UNSUPPORTED_REQUEST	BIT(4)
 #define PCIEMSIFR		0x02044
 #define PCIEMSIALR		0x02048
 #define  MSIFE			1
@@ -57,17 +63,17 @@
 /* local address reg & mask */
 #define PCIELAR(x)		(0x02200 + ((x) * 0x20))
 #define PCIELAMR(x)		(0x02208 + ((x) * 0x20))
-#define  LAM_PREFETCH		(1 << 3)
-#define  LAM_64BIT		(1 << 2)
-#define  LAR_ENABLE		(1 << 1)
+#define  LAM_PREFETCH		BIT(3)
+#define  LAM_64BIT		BIT(2)
+#define  LAR_ENABLE		BIT(1)
 
 /* PCIe address reg & mask */
 #define PCIEPALR(x)		(0x03400 + ((x) * 0x20))
 #define PCIEPAUR(x)		(0x03404 + ((x) * 0x20))
 #define PCIEPAMR(x)		(0x03408 + ((x) * 0x20))
 #define PCIEPTCTLR(x)		(0x0340c + ((x) * 0x20))
-#define  PAR_ENABLE		(1 << 31)
-#define  IO_SPACE		(1 << 8)
+#define  PAR_ENABLE		BIT(31)
+#define  IO_SPACE		BIT(8)
 
 /* Configuration */
 #define PCICONF(x)		(0x010000 + ((x) * 0x4))
@@ -79,47 +85,46 @@
 #define IDSETR1			0x011004
 #define TLCTLR			0x011048
 #define MACSR			0x011054
-#define  SPCHGFIN		(1 << 4)
-#define  SPCHGFAIL		(1 << 6)
-#define  SPCHGSUC		(1 << 7)
+#define  SPCHGFIN		BIT(4)
+#define  SPCHGFAIL		BIT(6)
+#define  SPCHGSUC		BIT(7)
 #define  LINK_SPEED		(0xf << 16)
 #define  LINK_SPEED_2_5GTS	(1 << 16)
 #define  LINK_SPEED_5_0GTS	(2 << 16)
 #define MACCTLR			0x011058
-#define  SPEED_CHANGE		(1 << 24)
-#define  SCRAMBLE_DISABLE	(1 << 27)
+#define  SPEED_CHANGE		BIT(24)
+#define  SCRAMBLE_DISABLE	BIT(27)
 #define MACS2R			0x011078
 #define MACCGSPSETR		0x011084
-#define  SPCNGRSN		(1 << 31)
+#define  SPCNGRSN		BIT(31)
 
 /* R-Car H1 PHY */
 #define H1_PCIEPHYADRR		0x04000c
-#define  WRITE_CMD		(1 << 16)
-#define  PHY_ACK		(1 << 24)
+#define  WRITE_CMD		BIT(16)
+#define  PHY_ACK		BIT(24)
 #define  RATE_POS		12
 #define  LANE_POS		8
 #define  ADR_POS		0
 #define H1_PCIEPHYDOUTR		0x040014
-#define H1_PCIEPHYSR		0x040018
 
 /* R-Car Gen2 PHY */
 #define GEN2_PCIEPHYADDR	0x780
 #define GEN2_PCIEPHYDATA	0x784
 #define GEN2_PCIEPHYCTRL	0x78c
 
-#define INT_PCI_MSI_NR	32
+#define INT_PCI_MSI_NR		32
 
-#define RCONF(x)	(PCICONF(0)+(x))
-#define RPMCAP(x)	(PMCAP(0)+(x))
-#define REXPCAP(x)	(EXPCAP(0)+(x))
-#define RVCCAP(x)	(VCCAP(0)+(x))
+#define RCONF(x)		(PCICONF(0) + (x))
+#define RPMCAP(x)		(PMCAP(0) + (x))
+#define REXPCAP(x)		(EXPCAP(0) + (x))
+#define RVCCAP(x)		(VCCAP(0) + (x))
 
-#define  PCIE_CONF_BUS(b)	(((b) & 0xff) << 24)
-#define  PCIE_CONF_DEV(d)	(((d) & 0x1f) << 19)
-#define  PCIE_CONF_FUNC(f)	(((f) & 0x7) << 16)
+#define PCIE_CONF_BUS(b)	(((b) & 0xff) << 24)
+#define PCIE_CONF_DEV(d)	(((d) & 0x1f) << 19)
+#define PCIE_CONF_FUNC(f)	(((f) & 0x7) << 16)
 
-#define RCAR_PCI_MAX_RESOURCES 4
-#define MAX_NR_INBOUND_MAPS 6
+#define RCAR_PCI_MAX_RESOURCES	4
+#define MAX_NR_INBOUND_MAPS	6
 
 struct rcar_msi {
 	DECLARE_BITMAP(used, INT_PCI_MSI_NR);
@@ -139,10 +144,10 @@ static inline struct rcar_msi *to_rcar_msi(struct msi_controller *chip)
 /* Structure representing the PCIe interface */
 struct rcar_pcie {
 	struct device		*dev;
+	struct phy		*phy;
 	void __iomem		*base;
 	struct list_head	resources;
 	int			root_bus_nr;
-	struct clk		*clk;
 	struct clk		*bus_clk;
 	struct			rcar_msi msi;
 };
@@ -527,12 +532,12 @@ static void phy_write_reg(struct rcar_pcie *pcie,
 	phy_wait_for_ack(pcie);
 }
 
-static int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie)
+static int rcar_pcie_wait_for_phyrdy(struct rcar_pcie *pcie)
 {
 	unsigned int timeout = 10;
 
 	while (timeout--) {
-		if ((rcar_pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE))
+		if (rcar_pci_read_reg(pcie, PCIEPHYSR) & PHYRDY)
 			return 0;
 
 		msleep(5);
@@ -541,6 +546,21 @@ static int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie)
 	return -ETIMEDOUT;
 }
 
+static int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie)
+{
+	unsigned int timeout = 10000;
+
+	while (timeout--) {
+		if ((rcar_pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE))
+			return 0;
+
+		udelay(5);
+		cpu_relax();
+	}
+
+	return -ETIMEDOUT;
+}
+
 static int rcar_pcie_hw_init(struct rcar_pcie *pcie)
 {
 	int err;
@@ -551,6 +571,10 @@ static int rcar_pcie_hw_init(struct rcar_pcie *pcie)
 	/* Set mode */
 	rcar_pci_write_reg(pcie, 1, PCIEMSR);
 
+	err = rcar_pcie_wait_for_phyrdy(pcie);
+	if (err)
+		return err;
+
 	/*
 	 * Initial header for port config space is type 1, set the device
 	 * class to match. Hardware takes care of propagating the IDSETR
@@ -605,10 +629,8 @@ static int rcar_pcie_hw_init(struct rcar_pcie *pcie)
 	return 0;
 }
 
-static int rcar_pcie_hw_init_h1(struct rcar_pcie *pcie)
+static int rcar_pcie_phy_init_h1(struct rcar_pcie *pcie)
 {
-	unsigned int timeout = 10;
-
 	/* Initialize the phy */
 	phy_write_reg(pcie, 0, 0x42, 0x1, 0x0EC34191);
 	phy_write_reg(pcie, 1, 0x42, 0x1, 0x0EC34180);
@@ -627,17 +649,10 @@ static int rcar_pcie_hw_init_h1(struct rcar_pcie *pcie)
 	phy_write_reg(pcie, 0, 0x64, 0x1, 0x3F0F1F0F);
 	phy_write_reg(pcie, 0, 0x66, 0x1, 0x00008000);
 
-	while (timeout--) {
-		if (rcar_pci_read_reg(pcie, H1_PCIEPHYSR))
-			return rcar_pcie_hw_init(pcie);
-
-		msleep(5);
-	}
-
-	return -ETIMEDOUT;
+	return 0;
 }
 
-static int rcar_pcie_hw_init_gen2(struct rcar_pcie *pcie)
+static int rcar_pcie_phy_init_gen2(struct rcar_pcie *pcie)
 {
 	/*
 	 * These settings come from the R-Car Series, 2nd Generation User's
@@ -654,7 +669,18 @@ static int rcar_pcie_hw_init_gen2(struct rcar_pcie *pcie)
 	rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL);
 	rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL);
 
-	return rcar_pcie_hw_init(pcie);
+	return 0;
+}
+
+static int rcar_pcie_phy_init_gen3(struct rcar_pcie *pcie)
+{
+	int err;
+
+	err = phy_init(pcie->phy);
+	if (err)
+		return err;
+
+	return phy_power_on(pcie->phy);
 }
 
 static int rcar_msi_alloc(struct rcar_msi *chip)
@@ -842,6 +868,20 @@ static const struct irq_domain_ops msi_domain_ops = {
 	.map = rcar_msi_map,
 };
 
+static void rcar_pcie_unmap_msi(struct rcar_pcie *pcie)
+{
+	struct rcar_msi *msi = &pcie->msi;
+	int i, irq;
+
+	for (i = 0; i < INT_PCI_MSI_NR; i++) {
+		irq = irq_find_mapping(msi->domain, i);
+		if (irq > 0)
+			irq_dispose_mapping(irq);
+	}
+
+	irq_domain_remove(msi->domain);
+}
+
 static int rcar_pcie_enable_msi(struct rcar_pcie *pcie)
 {
 	struct device *dev = pcie->dev;
@@ -896,16 +936,35 @@ static int rcar_pcie_enable_msi(struct rcar_pcie *pcie)
 	return 0;
 
 err:
-	irq_domain_remove(msi->domain);
+	rcar_pcie_unmap_msi(pcie);
 	return err;
 }
 
+static void rcar_pcie_teardown_msi(struct rcar_pcie *pcie)
+{
+	struct rcar_msi *msi = &pcie->msi;
+
+	/* Disable all MSI interrupts */
+	rcar_pci_write_reg(pcie, 0, PCIEMSIIER);
+
+	/* Disable address decoding of the MSI interrupt, MSIFE */
+	rcar_pci_write_reg(pcie, 0, PCIEMSIALR);
+
+	free_pages(msi->pages, 0);
+
+	rcar_pcie_unmap_msi(pcie);
+}
+
 static int rcar_pcie_get_resources(struct rcar_pcie *pcie)
 {
 	struct device *dev = pcie->dev;
 	struct resource res;
 	int err, i;
 
+	pcie->phy = devm_phy_optional_get(dev, "pcie");
+	if (IS_ERR(pcie->phy))
+		return PTR_ERR(pcie->phy);
+
 	err = of_address_to_resource(dev->of_node, 0, &res);
 	if (err)
 		return err;
@@ -914,30 +973,17 @@ static int rcar_pcie_get_resources(struct rcar_pcie *pcie)
 	if (IS_ERR(pcie->base))
 		return PTR_ERR(pcie->base);
 
-	pcie->clk = devm_clk_get(dev, "pcie");
-	if (IS_ERR(pcie->clk)) {
-		dev_err(dev, "cannot get platform clock\n");
-		return PTR_ERR(pcie->clk);
-	}
-	err = clk_prepare_enable(pcie->clk);
-	if (err)
-		return err;
-
 	pcie->bus_clk = devm_clk_get(dev, "pcie_bus");
 	if (IS_ERR(pcie->bus_clk)) {
 		dev_err(dev, "cannot get pcie bus clock\n");
-		err = PTR_ERR(pcie->bus_clk);
-		goto fail_clk;
+		return PTR_ERR(pcie->bus_clk);
 	}
-	err = clk_prepare_enable(pcie->bus_clk);
-	if (err)
-		goto fail_clk;
 
 	i = irq_of_parse_and_map(dev->of_node, 0);
 	if (!i) {
 		dev_err(dev, "cannot get platform resources for msi interrupt\n");
 		err = -ENOENT;
-		goto err_map_reg;
+		goto err_irq1;
 	}
 	pcie->msi.irq1 = i;
 
@@ -945,17 +991,15 @@ static int rcar_pcie_get_resources(struct rcar_pcie *pcie)
 	if (!i) {
 		dev_err(dev, "cannot get platform resources for msi interrupt\n");
 		err = -ENOENT;
-		goto err_map_reg;
+		goto err_irq2;
 	}
 	pcie->msi.irq2 = i;
 
 	return 0;
 
-err_map_reg:
-	clk_disable_unprepare(pcie->bus_clk);
-fail_clk:
-	clk_disable_unprepare(pcie->clk);
-
+err_irq2:
+	irq_dispose_mapping(pcie->msi.irq1);
+err_irq1:
 	return err;
 }
 
@@ -1051,63 +1095,28 @@ static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie *pcie,
 }
 
 static const struct of_device_id rcar_pcie_of_match[] = {
-	{ .compatible = "renesas,pcie-r8a7779", .data = rcar_pcie_hw_init_h1 },
+	{ .compatible = "renesas,pcie-r8a7779",
+	  .data = rcar_pcie_phy_init_h1 },
 	{ .compatible = "renesas,pcie-r8a7790",
-	  .data = rcar_pcie_hw_init_gen2 },
+	  .data = rcar_pcie_phy_init_gen2 },
 	{ .compatible = "renesas,pcie-r8a7791",
-	  .data = rcar_pcie_hw_init_gen2 },
+	  .data = rcar_pcie_phy_init_gen2 },
 	{ .compatible = "renesas,pcie-rcar-gen2",
-	  .data = rcar_pcie_hw_init_gen2 },
-	{ .compatible = "renesas,pcie-r8a7795", .data = rcar_pcie_hw_init },
-	{ .compatible = "renesas,pcie-rcar-gen3", .data = rcar_pcie_hw_init },
+	  .data = rcar_pcie_phy_init_gen2 },
+	{ .compatible = "renesas,pcie-r8a7795",
+	  .data = rcar_pcie_phy_init_gen3 },
+	{ .compatible = "renesas,pcie-rcar-gen3",
+	  .data = rcar_pcie_phy_init_gen3 },
 	{},
 };
 
-static int rcar_pcie_parse_request_of_pci_ranges(struct rcar_pcie *pci)
-{
-	int err;
-	struct device *dev = pci->dev;
-	struct device_node *np = dev->of_node;
-	resource_size_t iobase;
-	struct resource_entry *win, *tmp;
-
-	err = of_pci_get_host_bridge_resources(np, 0, 0xff, &pci->resources,
-					       &iobase);
-	if (err)
-		return err;
-
-	err = devm_request_pci_bus_resources(dev, &pci->resources);
-	if (err)
-		goto out_release_res;
-
-	resource_list_for_each_entry_safe(win, tmp, &pci->resources) {
-		struct resource *res = win->res;
-
-		if (resource_type(res) == IORESOURCE_IO) {
-			err = pci_remap_iospace(res, iobase);
-			if (err) {
-				dev_warn(dev, "error %d: failed to map resource %pR\n",
-					 err, res);
-
-				resource_list_destroy_entry(win);
-			}
-		}
-	}
-
-	return 0;
-
-out_release_res:
-	pci_free_resource_list(&pci->resources);
-	return err;
-}
-
 static int rcar_pcie_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct rcar_pcie *pcie;
 	unsigned int data;
 	int err;
-	int (*hw_init_fn)(struct rcar_pcie *);
+	int (*phy_init_fn)(struct rcar_pcie *);
 	struct pci_host_bridge *bridge;
 
 	bridge = pci_alloc_host_bridge(sizeof(*pcie));
@@ -1118,36 +1127,45 @@ static int rcar_pcie_probe(struct platform_device *pdev)
 
 	pcie->dev = dev;
 
-	INIT_LIST_HEAD(&pcie->resources);
-
-	err = rcar_pcie_parse_request_of_pci_ranges(pcie);
+	err = pci_parse_request_of_pci_ranges(dev, &pcie->resources, NULL);
 	if (err)
 		goto err_free_bridge;
 
+	pm_runtime_enable(pcie->dev);
+	err = pm_runtime_get_sync(pcie->dev);
+	if (err < 0) {
+		dev_err(pcie->dev, "pm_runtime_get_sync failed\n");
+		goto err_pm_disable;
+	}
+
 	err = rcar_pcie_get_resources(pcie);
 	if (err < 0) {
 		dev_err(dev, "failed to request resources: %d\n", err);
-		goto err_free_resource_list;
+		goto err_pm_put;
+	}
+
+	err = clk_prepare_enable(pcie->bus_clk);
+	if (err) {
+		dev_err(dev, "failed to enable bus clock: %d\n", err);
+		goto err_unmap_msi_irqs;
 	}
 
 	err = rcar_pcie_parse_map_dma_ranges(pcie, dev->of_node);
 	if (err)
-		goto err_free_resource_list;
+		goto err_clk_disable;
 
-	pm_runtime_enable(dev);
-	err = pm_runtime_get_sync(dev);
-	if (err < 0) {
-		dev_err(dev, "pm_runtime_get_sync failed\n");
-		goto err_pm_disable;
+	phy_init_fn = of_device_get_match_data(dev);
+	err = phy_init_fn(pcie);
+	if (err) {
+		dev_err(dev, "failed to init PCIe PHY\n");
+		goto err_clk_disable;
 	}
 
 	/* Failure to get a link might just be that no cards are inserted */
-	hw_init_fn = of_device_get_match_data(dev);
-	err = hw_init_fn(pcie);
-	if (err) {
+	if (rcar_pcie_hw_init(pcie)) {
 		dev_info(dev, "PCIe link down\n");
 		err = -ENODEV;
-		goto err_pm_put;
+		goto err_clk_disable;
 	}
 
 	data = rcar_pci_read_reg(pcie, MACSR);
@@ -1159,24 +1177,34 @@ static int rcar_pcie_probe(struct platform_device *pdev)
 			dev_err(dev,
 				"failed to enable MSI support: %d\n",
 				err);
-			goto err_pm_put;
+			goto err_clk_disable;
 		}
 	}
 
 	err = rcar_pcie_enable(pcie);
 	if (err)
-		goto err_pm_put;
+		goto err_msi_teardown;
 
 	return 0;
 
+err_msi_teardown:
+	if (IS_ENABLED(CONFIG_PCI_MSI))
+		rcar_pcie_teardown_msi(pcie);
+
+err_clk_disable:
+	clk_disable_unprepare(pcie->bus_clk);
+
+err_unmap_msi_irqs:
+	irq_dispose_mapping(pcie->msi.irq2);
+	irq_dispose_mapping(pcie->msi.irq1);
+
 err_pm_put:
 	pm_runtime_put(dev);
 
 err_pm_disable:
 	pm_runtime_disable(dev);
-
-err_free_resource_list:
 	pci_free_resource_list(&pcie->resources);
+
 err_free_bridge:
 	pci_free_host_bridge(bridge);
 
diff --git a/drivers/pci/host/pcie-rockchip-ep.c b/drivers/pci/host/pcie-rockchip-ep.c
new file mode 100644
index 000000000000..fc267a49a932
--- /dev/null
+++ b/drivers/pci/host/pcie-rockchip-ep.c
@@ -0,0 +1,642 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Rockchip AXI PCIe endpoint controller driver
+ *
+ * Copyright (c) 2018 Rockchip, Inc.
+ *
+ * Author: Shawn Lin <shawn.lin@rock-chips.com>
+ *         Simon Xue <xxm@rock-chips.com>
+ */
+
+#include <linux/configfs.h>
+#include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/pci-epc.h>
+#include <linux/platform_device.h>
+#include <linux/pci-epf.h>
+#include <linux/sizes.h>
+
+#include "pcie-rockchip.h"
+
+/**
+ * struct rockchip_pcie_ep - private data for PCIe endpoint controller driver
+ * @rockchip: Rockchip PCIe controller
+ * @max_regions: maximum number of regions supported by hardware
+ * @ob_region_map: bitmask of mapped outbound regions
+ * @ob_addr: base addresses in the AXI bus where the outbound regions start
+ * @irq_phys_addr: base address on the AXI bus where the MSI/legacy IRQ
+ *		   dedicated outbound regions is mapped.
+ * @irq_cpu_addr: base address in the CPU space where a write access triggers
+ *		  the sending of a memory write (MSI) / normal message (legacy
+ *		  IRQ) TLP through the PCIe bus.
+ * @irq_pci_addr: used to save the current mapping of the MSI/legacy IRQ
+ *		  dedicated outbound region.
+ * @irq_pci_fn: the latest PCI function that has updated the mapping of
+ *		the MSI/legacy IRQ dedicated outbound region.
+ * @irq_pending: bitmask of asserted legacy IRQs.
+ */
+struct rockchip_pcie_ep {
+	struct rockchip_pcie	rockchip;
+	struct pci_epc		*epc;
+	u32			max_regions;
+	unsigned long		ob_region_map;
+	phys_addr_t		*ob_addr;
+	phys_addr_t		irq_phys_addr;
+	void __iomem		*irq_cpu_addr;
+	u64			irq_pci_addr;
+	u8			irq_pci_fn;
+	u8			irq_pending;
+};
+
+static void rockchip_pcie_clear_ep_ob_atu(struct rockchip_pcie *rockchip,
+					  u32 region)
+{
+	rockchip_pcie_write(rockchip, 0,
+			    ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(region));
+	rockchip_pcie_write(rockchip, 0,
+			    ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(region));
+	rockchip_pcie_write(rockchip, 0,
+			    ROCKCHIP_PCIE_AT_OB_REGION_DESC0(region));
+	rockchip_pcie_write(rockchip, 0,
+			    ROCKCHIP_PCIE_AT_OB_REGION_DESC1(region));
+	rockchip_pcie_write(rockchip, 0,
+			    ROCKCHIP_PCIE_AT_OB_REGION_CPU_ADDR0(region));
+	rockchip_pcie_write(rockchip, 0,
+			    ROCKCHIP_PCIE_AT_OB_REGION_CPU_ADDR1(region));
+}
+
+static void rockchip_pcie_prog_ep_ob_atu(struct rockchip_pcie *rockchip, u8 fn,
+					 u32 r, u32 type, u64 cpu_addr,
+					 u64 pci_addr, size_t size)
+{
+	u64 sz = 1ULL << fls64(size - 1);
+	int num_pass_bits = ilog2(sz);
+	u32 addr0, addr1, desc0, desc1;
+	bool is_nor_msg = (type == AXI_WRAPPER_NOR_MSG);
+
+	/* The minimal region size is 1MB */
+	if (num_pass_bits < 8)
+		num_pass_bits = 8;
+
+	cpu_addr -= rockchip->mem_res->start;
+	addr0 = ((is_nor_msg ? 0x10 : (num_pass_bits - 1)) &
+		PCIE_CORE_OB_REGION_ADDR0_NUM_BITS) |
+		(lower_32_bits(cpu_addr) & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR);
+	addr1 = upper_32_bits(is_nor_msg ? cpu_addr : pci_addr);
+	desc0 = ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN(fn) | type;
+	desc1 = 0;
+
+	if (is_nor_msg) {
+		rockchip_pcie_write(rockchip, 0,
+				    ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(r));
+		rockchip_pcie_write(rockchip, 0,
+				    ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(r));
+		rockchip_pcie_write(rockchip, desc0,
+				    ROCKCHIP_PCIE_AT_OB_REGION_DESC0(r));
+		rockchip_pcie_write(rockchip, desc1,
+				    ROCKCHIP_PCIE_AT_OB_REGION_DESC1(r));
+	} else {
+		/* PCI bus address region */
+		rockchip_pcie_write(rockchip, addr0,
+				    ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(r));
+		rockchip_pcie_write(rockchip, addr1,
+				    ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(r));
+		rockchip_pcie_write(rockchip, desc0,
+				    ROCKCHIP_PCIE_AT_OB_REGION_DESC0(r));
+		rockchip_pcie_write(rockchip, desc1,
+				    ROCKCHIP_PCIE_AT_OB_REGION_DESC1(r));
+
+		addr0 =
+		    ((num_pass_bits - 1) & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS) |
+		    (lower_32_bits(cpu_addr) &
+		     PCIE_CORE_OB_REGION_ADDR0_LO_ADDR);
+		addr1 = upper_32_bits(cpu_addr);
+	}
+
+	/* CPU bus address region */
+	rockchip_pcie_write(rockchip, addr0,
+			    ROCKCHIP_PCIE_AT_OB_REGION_CPU_ADDR0(r));
+	rockchip_pcie_write(rockchip, addr1,
+			    ROCKCHIP_PCIE_AT_OB_REGION_CPU_ADDR1(r));
+}
+
+static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn,
+					 struct pci_epf_header *hdr)
+{
+	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
+	struct rockchip_pcie *rockchip = &ep->rockchip;
+
+	/* All functions share the same vendor ID with function 0 */
+	if (fn == 0) {
+		u32 vid_regs = (hdr->vendorid & GENMASK(15, 0)) |
+			       (hdr->subsys_vendor_id & GENMASK(31, 16)) << 16;
+
+		rockchip_pcie_write(rockchip, vid_regs,
+				    PCIE_CORE_CONFIG_VENDOR);
+	}
+
+	rockchip_pcie_write(rockchip, hdr->deviceid << 16,
+			    ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + PCI_VENDOR_ID);
+
+	rockchip_pcie_write(rockchip,
+			    hdr->revid |
+			    hdr->progif_code << 8 |
+			    hdr->subclass_code << 16 |
+			    hdr->baseclass_code << 24,
+			    ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + PCI_REVISION_ID);
+	rockchip_pcie_write(rockchip, hdr->cache_line_size,
+			    ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+			    PCI_CACHE_LINE_SIZE);
+	rockchip_pcie_write(rockchip, hdr->subsys_id << 16,
+			    ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+			    PCI_SUBSYSTEM_VENDOR_ID);
+	rockchip_pcie_write(rockchip, hdr->interrupt_pin << 8,
+			    ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+			    PCI_INTERRUPT_LINE);
+
+	return 0;
+}
+
+static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn,
+				    struct pci_epf_bar *epf_bar)
+{
+	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
+	struct rockchip_pcie *rockchip = &ep->rockchip;
+	dma_addr_t bar_phys = epf_bar->phys_addr;
+	enum pci_barno bar = epf_bar->barno;
+	int flags = epf_bar->flags;
+	u32 addr0, addr1, reg, cfg, b, aperture, ctrl;
+	u64 sz;
+
+	/* BAR size is 2^(aperture + 7) */
+	sz = max_t(size_t, epf_bar->size, MIN_EP_APERTURE);
+
+	/*
+	 * roundup_pow_of_two() returns an unsigned long, which is not suited
+	 * for 64bit values.
+	 */
+	sz = 1ULL << fls64(sz - 1);
+	aperture = ilog2(sz) - 7; /* 128B -> 0, 256B -> 1, 512B -> 2, ... */
+
+	if ((flags & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) {
+		ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_IO_32BITS;
+	} else {
+		bool is_prefetch = !!(flags & PCI_BASE_ADDRESS_MEM_PREFETCH);
+		bool is_64bits = sz > SZ_2G;
+
+		if (is_64bits && (bar & 1))
+			return -EINVAL;
+
+		if (is_64bits && is_prefetch)
+			ctrl =
+			    ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_64BITS;
+		else if (is_prefetch)
+			ctrl =
+			    ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_32BITS;
+		else if (is_64bits)
+			ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_64BITS;
+		else
+			ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_32BITS;
+	}
+
+	if (bar < BAR_4) {
+		reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn);
+		b = bar;
+	} else {
+		reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn);
+		b = bar - BAR_4;
+	}
+
+	addr0 = lower_32_bits(bar_phys);
+	addr1 = upper_32_bits(bar_phys);
+
+	cfg = rockchip_pcie_read(rockchip, reg);
+	cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
+		 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
+	cfg |= (ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) |
+		ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl));
+
+	rockchip_pcie_write(rockchip, cfg, reg);
+	rockchip_pcie_write(rockchip, addr0,
+			    ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar));
+	rockchip_pcie_write(rockchip, addr1,
+			    ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar));
+
+	return 0;
+}
+
+static void rockchip_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn,
+				       struct pci_epf_bar *epf_bar)
+{
+	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
+	struct rockchip_pcie *rockchip = &ep->rockchip;
+	u32 reg, cfg, b, ctrl;
+	enum pci_barno bar = epf_bar->barno;
+
+	if (bar < BAR_4) {
+		reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn);
+		b = bar;
+	} else {
+		reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn);
+		b = bar - BAR_4;
+	}
+
+	ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_DISABLED;
+	cfg = rockchip_pcie_read(rockchip, reg);
+	cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
+		 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
+	cfg |= ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl);
+
+	rockchip_pcie_write(rockchip, cfg, reg);
+	rockchip_pcie_write(rockchip, 0x0,
+			    ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar));
+	rockchip_pcie_write(rockchip, 0x0,
+			    ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar));
+}
+
+static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn,
+				     phys_addr_t addr, u64 pci_addr,
+				     size_t size)
+{
+	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
+	struct rockchip_pcie *pcie = &ep->rockchip;
+	u32 r;
+
+	r = find_first_zero_bit(&ep->ob_region_map,
+				sizeof(ep->ob_region_map) * BITS_PER_LONG);
+	/*
+	 * Region 0 is reserved for configuration space and shouldn't
+	 * be used elsewhere per TRM, so leave it out.
+	 */
+	if (r >= ep->max_regions - 1) {
+		dev_err(&epc->dev, "no free outbound region\n");
+		return -EINVAL;
+	}
+
+	rockchip_pcie_prog_ep_ob_atu(pcie, fn, r, AXI_WRAPPER_MEM_WRITE, addr,
+				     pci_addr, size);
+
+	set_bit(r, &ep->ob_region_map);
+	ep->ob_addr[r] = addr;
+
+	return 0;
+}
+
+static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn,
+					phys_addr_t addr)
+{
+	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
+	struct rockchip_pcie *rockchip = &ep->rockchip;
+	u32 r;
+
+	for (r = 0; r < ep->max_regions - 1; r++)
+		if (ep->ob_addr[r] == addr)
+			break;
+
+	/*
+	 * Region 0 is reserved for configuration space and shouldn't
+	 * be used elsewhere per TRM, so leave it out.
+	 */
+	if (r == ep->max_regions - 1)
+		return;
+
+	rockchip_pcie_clear_ep_ob_atu(rockchip, r);
+
+	ep->ob_addr[r] = 0;
+	clear_bit(r, &ep->ob_region_map);
+}
+
+static int rockchip_pcie_ep_set_msi(struct pci_epc *epc, u8 fn,
+				    u8 multi_msg_cap)
+{
+	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
+	struct rockchip_pcie *rockchip = &ep->rockchip;
+	u16 flags;
+
+	flags = rockchip_pcie_read(rockchip,
+				   ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+				   ROCKCHIP_PCIE_EP_MSI_CTRL_REG);
+	flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_MASK;
+	flags |=
+	   ((multi_msg_cap << 1) <<  ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_OFFSET) |
+	   PCI_MSI_FLAGS_64BIT;
+	flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MASK_MSI_CAP;
+	rockchip_pcie_write(rockchip, flags,
+			    ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+			    ROCKCHIP_PCIE_EP_MSI_CTRL_REG);
+	return 0;
+}
+
+static int rockchip_pcie_ep_get_msi(struct pci_epc *epc, u8 fn)
+{
+	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
+	struct rockchip_pcie *rockchip = &ep->rockchip;
+	u16 flags;
+
+	flags = rockchip_pcie_read(rockchip,
+				   ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+				   ROCKCHIP_PCIE_EP_MSI_CTRL_REG);
+	if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME))
+		return -EINVAL;
+
+	return ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >>
+			ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET);
+}
+
+static void rockchip_pcie_ep_assert_intx(struct rockchip_pcie_ep *ep, u8 fn,
+					 u8 intx, bool is_asserted)
+{
+	struct rockchip_pcie *rockchip = &ep->rockchip;
+	u32 r = ep->max_regions - 1;
+	u32 offset;
+	u16 status;
+	u8 msg_code;
+
+	if (unlikely(ep->irq_pci_addr != ROCKCHIP_PCIE_EP_PCI_LEGACY_IRQ_ADDR ||
+		     ep->irq_pci_fn != fn)) {
+		rockchip_pcie_prog_ep_ob_atu(rockchip, fn, r,
+					     AXI_WRAPPER_NOR_MSG,
+					     ep->irq_phys_addr, 0, 0);
+		ep->irq_pci_addr = ROCKCHIP_PCIE_EP_PCI_LEGACY_IRQ_ADDR;
+		ep->irq_pci_fn = fn;
+	}
+
+	intx &= 3;
+	if (is_asserted) {
+		ep->irq_pending |= BIT(intx);
+		msg_code = ROCKCHIP_PCIE_MSG_CODE_ASSERT_INTA + intx;
+	} else {
+		ep->irq_pending &= ~BIT(intx);
+		msg_code = ROCKCHIP_PCIE_MSG_CODE_DEASSERT_INTA + intx;
+	}
+
+	status = rockchip_pcie_read(rockchip,
+				    ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+				    ROCKCHIP_PCIE_EP_CMD_STATUS);
+	status &= ROCKCHIP_PCIE_EP_CMD_STATUS_IS;
+
+	if ((status != 0) ^ (ep->irq_pending != 0)) {
+		status ^= ROCKCHIP_PCIE_EP_CMD_STATUS_IS;
+		rockchip_pcie_write(rockchip, status,
+				    ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+				    ROCKCHIP_PCIE_EP_CMD_STATUS);
+	}
+
+	offset =
+	   ROCKCHIP_PCIE_MSG_ROUTING(ROCKCHIP_PCIE_MSG_ROUTING_LOCAL_INTX) |
+	   ROCKCHIP_PCIE_MSG_CODE(msg_code) | ROCKCHIP_PCIE_MSG_NO_DATA;
+	writel(0, ep->irq_cpu_addr + offset);
+}
+
+static int rockchip_pcie_ep_send_legacy_irq(struct rockchip_pcie_ep *ep, u8 fn,
+					    u8 intx)
+{
+	u16 cmd;
+
+	cmd = rockchip_pcie_read(&ep->rockchip,
+				 ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+				 ROCKCHIP_PCIE_EP_CMD_STATUS);
+
+	if (cmd & PCI_COMMAND_INTX_DISABLE)
+		return -EINVAL;
+
+	/*
+	 * Should add some delay between toggling INTx per TRM vaguely saying
+	 * it depends on some cycles of the AHB bus clock to function it. So
+	 * add sufficient 1ms here.
+	 */
+	rockchip_pcie_ep_assert_intx(ep, fn, intx, true);
+	mdelay(1);
+	rockchip_pcie_ep_assert_intx(ep, fn, intx, false);
+	return 0;
+}
+
+static int rockchip_pcie_ep_send_msi_irq(struct rockchip_pcie_ep *ep, u8 fn,
+					 u8 interrupt_num)
+{
+	struct rockchip_pcie *rockchip = &ep->rockchip;
+	u16 flags, mme, data, data_mask;
+	u8 msi_count;
+	u64 pci_addr, pci_addr_mask = 0xff;
+
+	/* Check MSI enable bit */
+	flags = rockchip_pcie_read(&ep->rockchip,
+				   ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+				   ROCKCHIP_PCIE_EP_MSI_CTRL_REG);
+	if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME))
+		return -EINVAL;
+
+	/* Get MSI numbers from MME */
+	mme = ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >>
+			ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET);
+	msi_count = 1 << mme;
+	if (!interrupt_num || interrupt_num > msi_count)
+		return -EINVAL;
+
+	/* Set MSI private data */
+	data_mask = msi_count - 1;
+	data = rockchip_pcie_read(rockchip,
+				  ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+				  ROCKCHIP_PCIE_EP_MSI_CTRL_REG +
+				  PCI_MSI_DATA_64);
+	data = (data & ~data_mask) | ((interrupt_num - 1) & data_mask);
+
+	/* Get MSI PCI address */
+	pci_addr = rockchip_pcie_read(rockchip,
+				      ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+				      ROCKCHIP_PCIE_EP_MSI_CTRL_REG +
+				      PCI_MSI_ADDRESS_HI);
+	pci_addr <<= 32;
+	pci_addr |= rockchip_pcie_read(rockchip,
+				       ROCKCHIP_PCIE_EP_FUNC_BASE(fn) +
+				       ROCKCHIP_PCIE_EP_MSI_CTRL_REG +
+				       PCI_MSI_ADDRESS_LO);
+	pci_addr &= GENMASK_ULL(63, 2);
+
+	/* Set the outbound region if needed. */
+	if (unlikely(ep->irq_pci_addr != (pci_addr & ~pci_addr_mask) ||
+		     ep->irq_pci_fn != fn)) {
+		rockchip_pcie_prog_ep_ob_atu(rockchip, fn, ep->max_regions - 1,
+					     AXI_WRAPPER_MEM_WRITE,
+					     ep->irq_phys_addr,
+					     pci_addr & ~pci_addr_mask,
+					     pci_addr_mask + 1);
+		ep->irq_pci_addr = (pci_addr & ~pci_addr_mask);
+		ep->irq_pci_fn = fn;
+	}
+
+	writew(data, ep->irq_cpu_addr + (pci_addr & pci_addr_mask));
+	return 0;
+}
+
+static int rockchip_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn,
+				      enum pci_epc_irq_type type,
+				      u8 interrupt_num)
+{
+	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
+
+	switch (type) {
+	case PCI_EPC_IRQ_LEGACY:
+		return rockchip_pcie_ep_send_legacy_irq(ep, fn, 0);
+	case PCI_EPC_IRQ_MSI:
+		return rockchip_pcie_ep_send_msi_irq(ep, fn, interrupt_num);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int rockchip_pcie_ep_start(struct pci_epc *epc)
+{
+	struct rockchip_pcie_ep *ep = epc_get_drvdata(epc);
+	struct rockchip_pcie *rockchip = &ep->rockchip;
+	struct pci_epf *epf;
+	u32 cfg;
+
+	cfg = BIT(0);
+	list_for_each_entry(epf, &epc->pci_epf, list)
+		cfg |= BIT(epf->func_no);
+
+	rockchip_pcie_write(rockchip, cfg, PCIE_CORE_PHY_FUNC_CFG);
+
+	list_for_each_entry(epf, &epc->pci_epf, list)
+		pci_epf_linkup(epf);
+
+	return 0;
+}
+
+static const struct pci_epc_ops rockchip_pcie_epc_ops = {
+	.write_header	= rockchip_pcie_ep_write_header,
+	.set_bar	= rockchip_pcie_ep_set_bar,
+	.clear_bar	= rockchip_pcie_ep_clear_bar,
+	.map_addr	= rockchip_pcie_ep_map_addr,
+	.unmap_addr	= rockchip_pcie_ep_unmap_addr,
+	.set_msi	= rockchip_pcie_ep_set_msi,
+	.get_msi	= rockchip_pcie_ep_get_msi,
+	.raise_irq	= rockchip_pcie_ep_raise_irq,
+	.start		= rockchip_pcie_ep_start,
+};
+
+static int rockchip_pcie_parse_ep_dt(struct rockchip_pcie *rockchip,
+				     struct rockchip_pcie_ep *ep)
+{
+	struct device *dev = rockchip->dev;
+	int err;
+
+	err = rockchip_pcie_parse_dt(rockchip);
+	if (err)
+		return err;
+
+	err = rockchip_pcie_get_phys(rockchip);
+	if (err)
+		return err;
+
+	err = of_property_read_u32(dev->of_node,
+				   "rockchip,max-outbound-regions",
+				   &ep->max_regions);
+	if (err < 0 || ep->max_regions > MAX_REGION_LIMIT)
+		ep->max_regions = MAX_REGION_LIMIT;
+
+	err = of_property_read_u8(dev->of_node, "max-functions",
+				  &ep->epc->max_functions);
+	if (err < 0)
+		ep->epc->max_functions = 1;
+
+	return 0;
+}
+
+static const struct of_device_id rockchip_pcie_ep_of_match[] = {
+	{ .compatible = "rockchip,rk3399-pcie-ep"},
+	{},
+};
+
+static int rockchip_pcie_ep_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct rockchip_pcie_ep *ep;
+	struct rockchip_pcie *rockchip;
+	struct pci_epc *epc;
+	size_t max_regions;
+	int err;
+
+	ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL);
+	if (!ep)
+		return -ENOMEM;
+
+	rockchip = &ep->rockchip;
+	rockchip->is_rc = false;
+	rockchip->dev = dev;
+
+	epc = devm_pci_epc_create(dev, &rockchip_pcie_epc_ops);
+	if (IS_ERR(epc)) {
+		dev_err(dev, "failed to create epc device\n");
+		return PTR_ERR(epc);
+	}
+
+	ep->epc = epc;
+	epc_set_drvdata(epc, ep);
+
+	err = rockchip_pcie_parse_ep_dt(rockchip, ep);
+	if (err)
+		return err;
+
+	err = rockchip_pcie_enable_clocks(rockchip);
+	if (err)
+		return err;
+
+	err = rockchip_pcie_init_port(rockchip);
+	if (err)
+		goto err_disable_clocks;
+
+	/* Establish the link automatically */
+	rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
+			    PCIE_CLIENT_CONFIG);
+
+	max_regions = ep->max_regions;
+	ep->ob_addr = devm_kzalloc(dev, max_regions * sizeof(*ep->ob_addr),
+				   GFP_KERNEL);
+
+	if (!ep->ob_addr) {
+		err = -ENOMEM;
+		goto err_uninit_port;
+	}
+
+	/* Only enable function 0 by default */
+	rockchip_pcie_write(rockchip, BIT(0), PCIE_CORE_PHY_FUNC_CFG);
+
+	err = pci_epc_mem_init(epc, rockchip->mem_res->start,
+			       resource_size(rockchip->mem_res));
+	if (err < 0) {
+		dev_err(dev, "failed to initialize the memory space\n");
+		goto err_uninit_port;
+	}
+
+	ep->irq_cpu_addr = pci_epc_mem_alloc_addr(epc, &ep->irq_phys_addr,
+						  SZ_128K);
+	if (!ep->irq_cpu_addr) {
+		dev_err(dev, "failed to reserve memory space for MSI\n");
+		err = -ENOMEM;
+		goto err_epc_mem_exit;
+	}
+
+	ep->irq_pci_addr = ROCKCHIP_PCIE_EP_DUMMY_IRQ_ADDR;
+
+	return 0;
+err_epc_mem_exit:
+	pci_epc_mem_exit(epc);
+err_uninit_port:
+	rockchip_pcie_deinit_phys(rockchip);
+err_disable_clocks:
+	rockchip_pcie_disable_clocks(rockchip);
+	return err;
+}
+
+static struct platform_driver rockchip_pcie_ep_driver = {
+	.driver = {
+		.name = "rockchip-pcie-ep",
+		.of_match_table = rockchip_pcie_ep_of_match,
+	},
+	.probe = rockchip_pcie_ep_probe,
+};
+
+builtin_platform_driver(rockchip_pcie_ep_driver);
diff --git a/drivers/pci/host/pcie-rockchip-host.c b/drivers/pci/host/pcie-rockchip-host.c
new file mode 100644
index 000000000000..1372d270764f
--- /dev/null
+++ b/drivers/pci/host/pcie-rockchip-host.c
@@ -0,0 +1,1142 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Rockchip AXI PCIe host controller driver
+ *
+ * Copyright (c) 2016 Rockchip, Inc.
+ *
+ * Author: Shawn Lin <shawn.lin@rock-chips.com>
+ *         Wenrui Li <wenrui.li@rock-chips.com>
+ *
+ * Bits taken from Synopsys DesignWare Host controller driver and
+ * ARM PCI Host generic driver.
+ */
+
+#include <linux/bitrev.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/irq.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/of_irq.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/regmap.h>
+
+#include "../pci.h"
+#include "pcie-rockchip.h"
+
+static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
+{
+	u32 status;
+
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+	status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+}
+
+static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
+{
+	u32 status;
+
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+	status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+}
+
+static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)
+{
+	u32 val;
+
+	/* Update Tx credit maximum update interval */
+	val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);
+	val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;
+	val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000);	/* ns */
+	rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);
+}
+
+static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
+				      struct pci_bus *bus, int dev)
+{
+	/* access only one slot on each root port */
+	if (bus->number == rockchip->root_bus_nr && dev > 0)
+		return 0;
+
+	/*
+	 * do not read more than one device on the bus directly attached
+	 * to RC's downstream side.
+	 */
+	if (bus->primary == rockchip->root_bus_nr && dev > 0)
+		return 0;
+
+	return 1;
+}
+
+static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip)
+{
+	u32 val;
+	u8 map;
+
+	if (rockchip->legacy_phy)
+		return GENMASK(MAX_LANE_NUM - 1, 0);
+
+	val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP);
+	map = val & PCIE_CORE_LANE_MAP_MASK;
+
+	/* The link may be using a reverse-indexed mapping. */
+	if (val & PCIE_CORE_LANE_MAP_REVERSE)
+		map = bitrev8(map) >> 4;
+
+	return map;
+}
+
+static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
+				     int where, int size, u32 *val)
+{
+	void __iomem *addr;
+
+	addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where;
+
+	if (!IS_ALIGNED((uintptr_t)addr, size)) {
+		*val = 0;
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+	}
+
+	if (size == 4) {
+		*val = readl(addr);
+	} else if (size == 2) {
+		*val = readw(addr);
+	} else if (size == 1) {
+		*val = readb(addr);
+	} else {
+		*val = 0;
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+	}
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,
+				     int where, int size, u32 val)
+{
+	u32 mask, tmp, offset;
+	void __iomem *addr;
+
+	offset = where & ~0x3;
+	addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset;
+
+	if (size == 4) {
+		writel(val, addr);
+		return PCIBIOS_SUCCESSFUL;
+	}
+
+	mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
+
+	/*
+	 * N.B. This read/modify/write isn't safe in general because it can
+	 * corrupt RW1C bits in adjacent registers.  But the hardware
+	 * doesn't support smaller writes.
+	 */
+	tmp = readl(addr) & mask;
+	tmp |= val << ((where & 0x3) * 8);
+	writel(tmp, addr);
+
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,
+				       struct pci_bus *bus, u32 devfn,
+				       int where, int size, u32 *val)
+{
+	u32 busdev;
+
+	busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
+				PCI_FUNC(devfn), where);
+
+	if (!IS_ALIGNED(busdev, size)) {
+		*val = 0;
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+	}
+
+	if (bus->parent->number == rockchip->root_bus_nr)
+		rockchip_pcie_cfg_configuration_accesses(rockchip,
+						AXI_WRAPPER_TYPE0_CFG);
+	else
+		rockchip_pcie_cfg_configuration_accesses(rockchip,
+						AXI_WRAPPER_TYPE1_CFG);
+
+	if (size == 4) {
+		*val = readl(rockchip->reg_base + busdev);
+	} else if (size == 2) {
+		*val = readw(rockchip->reg_base + busdev);
+	} else if (size == 1) {
+		*val = readb(rockchip->reg_base + busdev);
+	} else {
+		*val = 0;
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+	}
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,
+				       struct pci_bus *bus, u32 devfn,
+				       int where, int size, u32 val)
+{
+	u32 busdev;
+
+	busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
+				PCI_FUNC(devfn), where);
+	if (!IS_ALIGNED(busdev, size))
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+
+	if (bus->parent->number == rockchip->root_bus_nr)
+		rockchip_pcie_cfg_configuration_accesses(rockchip,
+						AXI_WRAPPER_TYPE0_CFG);
+	else
+		rockchip_pcie_cfg_configuration_accesses(rockchip,
+						AXI_WRAPPER_TYPE1_CFG);
+
+	if (size == 4)
+		writel(val, rockchip->reg_base + busdev);
+	else if (size == 2)
+		writew(val, rockchip->reg_base + busdev);
+	else if (size == 1)
+		writeb(val, rockchip->reg_base + busdev);
+	else
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
+				 int size, u32 *val)
+{
+	struct rockchip_pcie *rockchip = bus->sysdata;
+
+	if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) {
+		*val = 0xffffffff;
+		return PCIBIOS_DEVICE_NOT_FOUND;
+	}
+
+	if (bus->number == rockchip->root_bus_nr)
+		return rockchip_pcie_rd_own_conf(rockchip, where, size, val);
+
+	return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size,
+					   val);
+}
+
+static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
+				 int where, int size, u32 val)
+{
+	struct rockchip_pcie *rockchip = bus->sysdata;
+
+	if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
+		return PCIBIOS_DEVICE_NOT_FOUND;
+
+	if (bus->number == rockchip->root_bus_nr)
+		return rockchip_pcie_wr_own_conf(rockchip, where, size, val);
+
+	return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size,
+					   val);
+}
+
+static struct pci_ops rockchip_pcie_ops = {
+	.read = rockchip_pcie_rd_conf,
+	.write = rockchip_pcie_wr_conf,
+};
+
+static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
+{
+	int curr;
+	u32 status, scale, power;
+
+	if (IS_ERR(rockchip->vpcie3v3))
+		return;
+
+	/*
+	 * Set RC's captured slot power limit and scale if
+	 * vpcie3v3 available. The default values are both zero
+	 * which means the software should set these two according
+	 * to the actual power supply.
+	 */
+	curr = regulator_get_current_limit(rockchip->vpcie3v3);
+	if (curr <= 0)
+		return;
+
+	scale = 3; /* 0.001x */
+	curr = curr / 1000; /* convert to mA */
+	power = (curr * 3300) / 1000; /* milliwatt */
+	while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) {
+		if (!scale) {
+			dev_warn(rockchip->dev, "invalid power supply\n");
+			return;
+		}
+		scale--;
+		power = power / 10;
+	}
+
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR);
+	status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) |
+		  (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT);
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR);
+}
+
+/**
+ * rockchip_pcie_host_init_port - Initialize hardware
+ * @rockchip: PCIe port information
+ */
+static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	int err, i = MAX_LANE_NUM;
+	u32 status;
+
+	gpiod_set_value_cansleep(rockchip->ep_gpio, 0);
+
+	err = rockchip_pcie_init_port(rockchip);
+	if (err)
+		return err;
+
+	/* Fix the transmitted FTS count desired to exit from L0s. */
+	status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
+	status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |
+		 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
+	rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
+
+	rockchip_pcie_set_power_limit(rockchip);
+
+	/* Set RC's clock architecture as common clock */
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+	status |= PCI_EXP_LNKSTA_SLC << 16;
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+
+	/* Set RC's RCB to 128 */
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+	status |= PCI_EXP_LNKCTL_RCB;
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+
+	/* Enable Gen1 training */
+	rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
+			    PCIE_CLIENT_CONFIG);
+
+	gpiod_set_value_cansleep(rockchip->ep_gpio, 1);
+
+	/* 500ms timeout value should be enough for Gen1/2 training */
+	err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,
+				 status, PCIE_LINK_UP(status), 20,
+				 500 * USEC_PER_MSEC);
+	if (err) {
+		dev_err(dev, "PCIe link training gen1 timeout!\n");
+		goto err_power_off_phy;
+	}
+
+	if (rockchip->link_gen == 2) {
+		/*
+		 * Enable retrain for gen2. This should be configured only after
+		 * gen1 finished.
+		 */
+		status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+		status |= PCI_EXP_LNKCTL_RL;
+		rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+
+		err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL,
+					 status, PCIE_LINK_IS_GEN2(status), 20,
+					 500 * USEC_PER_MSEC);
+		if (err)
+			dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
+	}
+
+	/* Check the final link width from negotiated lane counter from MGMT */
+	status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
+	status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
+			  PCIE_CORE_PL_CONF_LANE_SHIFT);
+	dev_dbg(dev, "current link width is x%d\n", status);
+
+	/* Power off unused lane(s) */
+	rockchip->lanes_map = rockchip_pcie_lane_map(rockchip);
+	for (i = 0; i < MAX_LANE_NUM; i++) {
+		if (!(rockchip->lanes_map & BIT(i))) {
+			dev_dbg(dev, "idling lane %d\n", i);
+			phy_power_off(rockchip->phys[i]);
+		}
+	}
+
+	rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,
+			    PCIE_CORE_CONFIG_VENDOR);
+	rockchip_pcie_write(rockchip,
+			    PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT,
+			    PCIE_RC_CONFIG_RID_CCR);
+
+	/* Clear THP cap's next cap pointer to remove L1 substate cap */
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);
+	status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);
+
+	/* Clear L0s from RC's link cap */
+	if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) {
+		status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP);
+		status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S;
+		rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP);
+	}
+
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR);
+	status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK;
+	status |= PCIE_RC_CONFIG_DCSR_MPS_256;
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR);
+
+	return 0;
+err_power_off_phy:
+	while (i--)
+		phy_power_off(rockchip->phys[i]);
+	i = MAX_LANE_NUM;
+	while (i--)
+		phy_exit(rockchip->phys[i]);
+	return err;
+}
+
+static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
+{
+	struct rockchip_pcie *rockchip = arg;
+	struct device *dev = rockchip->dev;
+	u32 reg;
+	u32 sub_reg;
+
+	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
+	if (reg & PCIE_CLIENT_INT_LOCAL) {
+		dev_dbg(dev, "local interrupt received\n");
+		sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);
+		if (sub_reg & PCIE_CORE_INT_PRFPE)
+			dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");
+
+		if (sub_reg & PCIE_CORE_INT_CRFPE)
+			dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");
+
+		if (sub_reg & PCIE_CORE_INT_RRPE)
+			dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");
+
+		if (sub_reg & PCIE_CORE_INT_PRFO)
+			dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");
+
+		if (sub_reg & PCIE_CORE_INT_CRFO)
+			dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");
+
+		if (sub_reg & PCIE_CORE_INT_RT)
+			dev_dbg(dev, "replay timer timed out\n");
+
+		if (sub_reg & PCIE_CORE_INT_RTR)
+			dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");
+
+		if (sub_reg & PCIE_CORE_INT_PE)
+			dev_dbg(dev, "phy error detected on receive side\n");
+
+		if (sub_reg & PCIE_CORE_INT_MTR)
+			dev_dbg(dev, "malformed TLP received from the link\n");
+
+		if (sub_reg & PCIE_CORE_INT_UCR)
+			dev_dbg(dev, "malformed TLP received from the link\n");
+
+		if (sub_reg & PCIE_CORE_INT_FCE)
+			dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");
+
+		if (sub_reg & PCIE_CORE_INT_CT)
+			dev_dbg(dev, "a request timed out waiting for completion\n");
+
+		if (sub_reg & PCIE_CORE_INT_UTC)
+			dev_dbg(dev, "unmapped TC error\n");
+
+		if (sub_reg & PCIE_CORE_INT_MMVC)
+			dev_dbg(dev, "MSI mask register changes\n");
+
+		rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);
+	} else if (reg & PCIE_CLIENT_INT_PHY) {
+		dev_dbg(dev, "phy link changes\n");
+		rockchip_pcie_update_txcredit_mui(rockchip);
+		rockchip_pcie_clr_bw_int(rockchip);
+	}
+
+	rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,
+			    PCIE_CLIENT_INT_STATUS);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)
+{
+	struct rockchip_pcie *rockchip = arg;
+	struct device *dev = rockchip->dev;
+	u32 reg;
+
+	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
+	if (reg & PCIE_CLIENT_INT_LEGACY_DONE)
+		dev_dbg(dev, "legacy done interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_MSG)
+		dev_dbg(dev, "message done interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_HOT_RST)
+		dev_dbg(dev, "hot reset interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_DPA)
+		dev_dbg(dev, "dpa interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_FATAL_ERR)
+		dev_dbg(dev, "fatal error interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_NFATAL_ERR)
+		dev_dbg(dev, "no fatal error interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_CORR_ERR)
+		dev_dbg(dev, "correctable error interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_PHY)
+		dev_dbg(dev, "phy interrupt received\n");
+
+	rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |
+			      PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |
+			      PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |
+			      PCIE_CLIENT_INT_NFATAL_ERR |
+			      PCIE_CLIENT_INT_CORR_ERR |
+			      PCIE_CLIENT_INT_PHY),
+		   PCIE_CLIENT_INT_STATUS);
+
+	return IRQ_HANDLED;
+}
+
+static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc)
+{
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
+	struct device *dev = rockchip->dev;
+	u32 reg;
+	u32 hwirq;
+	u32 virq;
+
+	chained_irq_enter(chip, desc);
+
+	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
+	reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;
+
+	while (reg) {
+		hwirq = ffs(reg) - 1;
+		reg &= ~BIT(hwirq);
+
+		virq = irq_find_mapping(rockchip->irq_domain, hwirq);
+		if (virq)
+			generic_handle_irq(virq);
+		else
+			dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);
+	}
+
+	chained_irq_exit(chip, desc);
+}
+
+static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip)
+{
+	int irq, err;
+	struct device *dev = rockchip->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+
+	irq = platform_get_irq_byname(pdev, "sys");
+	if (irq < 0) {
+		dev_err(dev, "missing sys IRQ resource\n");
+		return irq;
+	}
+
+	err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
+			       IRQF_SHARED, "pcie-sys", rockchip);
+	if (err) {
+		dev_err(dev, "failed to request PCIe subsystem IRQ\n");
+		return err;
+	}
+
+	irq = platform_get_irq_byname(pdev, "legacy");
+	if (irq < 0) {
+		dev_err(dev, "missing legacy IRQ resource\n");
+		return irq;
+	}
+
+	irq_set_chained_handler_and_data(irq,
+					 rockchip_pcie_legacy_int_handler,
+					 rockchip);
+
+	irq = platform_get_irq_byname(pdev, "client");
+	if (irq < 0) {
+		dev_err(dev, "missing client IRQ resource\n");
+		return irq;
+	}
+
+	err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
+			       IRQF_SHARED, "pcie-client", rockchip);
+	if (err) {
+		dev_err(dev, "failed to request PCIe client IRQ\n");
+		return err;
+	}
+
+	return 0;
+}
+
+/**
+ * rockchip_pcie_parse_host_dt - Parse Device Tree
+ * @rockchip: PCIe port information
+ *
+ * Return: '0' on success and error value on failure
+ */
+static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	int err;
+
+	err = rockchip_pcie_parse_dt(rockchip);
+	if (err)
+		return err;
+
+	err = rockchip_pcie_setup_irq(rockchip);
+	if (err)
+		return err;
+
+	rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v");
+	if (IS_ERR(rockchip->vpcie12v)) {
+		if (PTR_ERR(rockchip->vpcie12v) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_info(dev, "no vpcie12v regulator found\n");
+	}
+
+	rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
+	if (IS_ERR(rockchip->vpcie3v3)) {
+		if (PTR_ERR(rockchip->vpcie3v3) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_info(dev, "no vpcie3v3 regulator found\n");
+	}
+
+	rockchip->vpcie1v8 = devm_regulator_get_optional(dev, "vpcie1v8");
+	if (IS_ERR(rockchip->vpcie1v8)) {
+		if (PTR_ERR(rockchip->vpcie1v8) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_info(dev, "no vpcie1v8 regulator found\n");
+	}
+
+	rockchip->vpcie0v9 = devm_regulator_get_optional(dev, "vpcie0v9");
+	if (IS_ERR(rockchip->vpcie0v9)) {
+		if (PTR_ERR(rockchip->vpcie0v9) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_info(dev, "no vpcie0v9 regulator found\n");
+	}
+
+	return 0;
+}
+
+static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	int err;
+
+	if (!IS_ERR(rockchip->vpcie12v)) {
+		err = regulator_enable(rockchip->vpcie12v);
+		if (err) {
+			dev_err(dev, "fail to enable vpcie12v regulator\n");
+			goto err_out;
+		}
+	}
+
+	if (!IS_ERR(rockchip->vpcie3v3)) {
+		err = regulator_enable(rockchip->vpcie3v3);
+		if (err) {
+			dev_err(dev, "fail to enable vpcie3v3 regulator\n");
+			goto err_disable_12v;
+		}
+	}
+
+	if (!IS_ERR(rockchip->vpcie1v8)) {
+		err = regulator_enable(rockchip->vpcie1v8);
+		if (err) {
+			dev_err(dev, "fail to enable vpcie1v8 regulator\n");
+			goto err_disable_3v3;
+		}
+	}
+
+	if (!IS_ERR(rockchip->vpcie0v9)) {
+		err = regulator_enable(rockchip->vpcie0v9);
+		if (err) {
+			dev_err(dev, "fail to enable vpcie0v9 regulator\n");
+			goto err_disable_1v8;
+		}
+	}
+
+	return 0;
+
+err_disable_1v8:
+	if (!IS_ERR(rockchip->vpcie1v8))
+		regulator_disable(rockchip->vpcie1v8);
+err_disable_3v3:
+	if (!IS_ERR(rockchip->vpcie3v3))
+		regulator_disable(rockchip->vpcie3v3);
+err_disable_12v:
+	if (!IS_ERR(rockchip->vpcie12v))
+		regulator_disable(rockchip->vpcie12v);
+err_out:
+	return err;
+}
+
+static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)
+{
+	rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &
+			    (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);
+	rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),
+			    PCIE_CORE_INT_MASK);
+
+	rockchip_pcie_enable_bw_int(rockchip);
+}
+
+static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
+				  irq_hw_number_t hwirq)
+{
+	irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
+	irq_set_chip_data(irq, domain->host_data);
+
+	return 0;
+}
+
+static const struct irq_domain_ops intx_domain_ops = {
+	.map = rockchip_pcie_intx_map,
+};
+
+static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	struct device_node *intc = of_get_next_child(dev->of_node, NULL);
+
+	if (!intc) {
+		dev_err(dev, "missing child interrupt-controller node\n");
+		return -EINVAL;
+	}
+
+	rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX,
+						    &intx_domain_ops, rockchip);
+	if (!rockchip->irq_domain) {
+		dev_err(dev, "failed to get a INTx IRQ domain\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,
+				     int region_no, int type, u8 num_pass_bits,
+				     u32 lower_addr, u32 upper_addr)
+{
+	u32 ob_addr_0;
+	u32 ob_addr_1;
+	u32 ob_desc_0;
+	u32 aw_offset;
+
+	if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)
+		return -EINVAL;
+	if (num_pass_bits + 1 < 8)
+		return -EINVAL;
+	if (num_pass_bits > 63)
+		return -EINVAL;
+	if (region_no == 0) {
+		if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))
+			return -EINVAL;
+	}
+	if (region_no != 0) {
+		if (AXI_REGION_SIZE < (2ULL << num_pass_bits))
+			return -EINVAL;
+	}
+
+	aw_offset = (region_no << OB_REG_SIZE_SHIFT);
+
+	ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;
+	ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;
+	ob_addr_1 = upper_addr;
+	ob_desc_0 = (1 << 23 | type);
+
+	rockchip_pcie_write(rockchip, ob_addr_0,
+			    PCIE_CORE_OB_REGION_ADDR0 + aw_offset);
+	rockchip_pcie_write(rockchip, ob_addr_1,
+			    PCIE_CORE_OB_REGION_ADDR1 + aw_offset);
+	rockchip_pcie_write(rockchip, ob_desc_0,
+			    PCIE_CORE_OB_REGION_DESC0 + aw_offset);
+	rockchip_pcie_write(rockchip, 0,
+			    PCIE_CORE_OB_REGION_DESC1 + aw_offset);
+
+	return 0;
+}
+
+static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
+				     int region_no, u8 num_pass_bits,
+				     u32 lower_addr, u32 upper_addr)
+{
+	u32 ib_addr_0;
+	u32 ib_addr_1;
+	u32 aw_offset;
+
+	if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)
+		return -EINVAL;
+	if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)
+		return -EINVAL;
+	if (num_pass_bits > 63)
+		return -EINVAL;
+
+	aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);
+
+	ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;
+	ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;
+	ib_addr_1 = upper_addr;
+
+	rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);
+	rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);
+
+	return 0;
+}
+
+static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	int offset;
+	int err;
+	int reg_no;
+
+	rockchip_pcie_cfg_configuration_accesses(rockchip,
+						 AXI_WRAPPER_TYPE0_CFG);
+
+	for (reg_no = 0; reg_no < (rockchip->mem_size >> 20); reg_no++) {
+		err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
+						AXI_WRAPPER_MEM_WRITE,
+						20 - 1,
+						rockchip->mem_bus_addr +
+						(reg_no << 20),
+						0);
+		if (err) {
+			dev_err(dev, "program RC mem outbound ATU failed\n");
+			return err;
+		}
+	}
+
+	err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
+	if (err) {
+		dev_err(dev, "program RC mem inbound ATU failed\n");
+		return err;
+	}
+
+	offset = rockchip->mem_size >> 20;
+	for (reg_no = 0; reg_no < (rockchip->io_size >> 20); reg_no++) {
+		err = rockchip_pcie_prog_ob_atu(rockchip,
+						reg_no + 1 + offset,
+						AXI_WRAPPER_IO_WRITE,
+						20 - 1,
+						rockchip->io_bus_addr +
+						(reg_no << 20),
+						0);
+		if (err) {
+			dev_err(dev, "program RC io outbound ATU failed\n");
+			return err;
+		}
+	}
+
+	/* assign message regions */
+	rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset,
+				  AXI_WRAPPER_NOR_MSG,
+				  20 - 1, 0, 0);
+
+	rockchip->msg_bus_addr = rockchip->mem_bus_addr +
+					((reg_no + offset) << 20);
+	return err;
+}
+
+static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)
+{
+	u32 value;
+	int err;
+
+	/* send PME_TURN_OFF message */
+	writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF);
+
+	/* read LTSSM and wait for falling into L2 link state */
+	err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0,
+				 value, PCIE_LINK_IS_L2(value), 20,
+				 jiffies_to_usecs(5 * HZ));
+	if (err) {
+		dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n");
+		return err;
+	}
+
+	return 0;
+}
+
+static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev)
+{
+	struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
+	int ret;
+
+	/* disable core and cli int since we don't need to ack PME_ACK */
+	rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) |
+			    PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK);
+	rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK);
+
+	ret = rockchip_pcie_wait_l2(rockchip);
+	if (ret) {
+		rockchip_pcie_enable_interrupts(rockchip);
+		return ret;
+	}
+
+	rockchip_pcie_deinit_phys(rockchip);
+
+	rockchip_pcie_disable_clocks(rockchip);
+
+	if (!IS_ERR(rockchip->vpcie0v9))
+		regulator_disable(rockchip->vpcie0v9);
+
+	return ret;
+}
+
+static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev)
+{
+	struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
+	int err;
+
+	if (!IS_ERR(rockchip->vpcie0v9)) {
+		err = regulator_enable(rockchip->vpcie0v9);
+		if (err) {
+			dev_err(dev, "fail to enable vpcie0v9 regulator\n");
+			return err;
+		}
+	}
+
+	err = rockchip_pcie_enable_clocks(rockchip);
+	if (err)
+		goto err_disable_0v9;
+
+	err = rockchip_pcie_host_init_port(rockchip);
+	if (err)
+		goto err_pcie_resume;
+
+	err = rockchip_pcie_cfg_atu(rockchip);
+	if (err)
+		goto err_err_deinit_port;
+
+	/* Need this to enter L1 again */
+	rockchip_pcie_update_txcredit_mui(rockchip);
+	rockchip_pcie_enable_interrupts(rockchip);
+
+	return 0;
+
+err_err_deinit_port:
+	rockchip_pcie_deinit_phys(rockchip);
+err_pcie_resume:
+	rockchip_pcie_disable_clocks(rockchip);
+err_disable_0v9:
+	if (!IS_ERR(rockchip->vpcie0v9))
+		regulator_disable(rockchip->vpcie0v9);
+	return err;
+}
+
+static int rockchip_pcie_probe(struct platform_device *pdev)
+{
+	struct rockchip_pcie *rockchip;
+	struct device *dev = &pdev->dev;
+	struct pci_bus *bus, *child;
+	struct pci_host_bridge *bridge;
+	struct resource_entry *win;
+	resource_size_t io_base;
+	struct resource	*mem;
+	struct resource	*io;
+	int err;
+
+	LIST_HEAD(res);
+
+	if (!dev->of_node)
+		return -ENODEV;
+
+	bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip));
+	if (!bridge)
+		return -ENOMEM;
+
+	rockchip = pci_host_bridge_priv(bridge);
+
+	platform_set_drvdata(pdev, rockchip);
+	rockchip->dev = dev;
+	rockchip->is_rc = true;
+
+	err = rockchip_pcie_parse_host_dt(rockchip);
+	if (err)
+		return err;
+
+	err = rockchip_pcie_enable_clocks(rockchip);
+	if (err)
+		return err;
+
+	err = rockchip_pcie_set_vpcie(rockchip);
+	if (err) {
+		dev_err(dev, "failed to set vpcie regulator\n");
+		goto err_set_vpcie;
+	}
+
+	err = rockchip_pcie_host_init_port(rockchip);
+	if (err)
+		goto err_vpcie;
+
+	rockchip_pcie_enable_interrupts(rockchip);
+
+	err = rockchip_pcie_init_irq_domain(rockchip);
+	if (err < 0)
+		goto err_deinit_port;
+
+	err = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff,
+						    &res, &io_base);
+	if (err)
+		goto err_remove_irq_domain;
+
+	err = devm_request_pci_bus_resources(dev, &res);
+	if (err)
+		goto err_free_res;
+
+	/* Get the I/O and memory ranges from DT */
+	resource_list_for_each_entry(win, &res) {
+		switch (resource_type(win->res)) {
+		case IORESOURCE_IO:
+			io = win->res;
+			io->name = "I/O";
+			rockchip->io_size = resource_size(io);
+			rockchip->io_bus_addr = io->start - win->offset;
+			err = pci_remap_iospace(io, io_base);
+			if (err) {
+				dev_warn(dev, "error %d: failed to map resource %pR\n",
+					 err, io);
+				continue;
+			}
+			rockchip->io = io;
+			break;
+		case IORESOURCE_MEM:
+			mem = win->res;
+			mem->name = "MEM";
+			rockchip->mem_size = resource_size(mem);
+			rockchip->mem_bus_addr = mem->start - win->offset;
+			break;
+		case IORESOURCE_BUS:
+			rockchip->root_bus_nr = win->res->start;
+			break;
+		default:
+			continue;
+		}
+	}
+
+	err = rockchip_pcie_cfg_atu(rockchip);
+	if (err)
+		goto err_unmap_iospace;
+
+	rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M);
+	if (!rockchip->msg_region) {
+		err = -ENOMEM;
+		goto err_unmap_iospace;
+	}
+
+	list_splice_init(&res, &bridge->windows);
+	bridge->dev.parent = dev;
+	bridge->sysdata = rockchip;
+	bridge->busnr = 0;
+	bridge->ops = &rockchip_pcie_ops;
+	bridge->map_irq = of_irq_parse_and_map_pci;
+	bridge->swizzle_irq = pci_common_swizzle;
+
+	err = pci_scan_root_bus_bridge(bridge);
+	if (err < 0)
+		goto err_unmap_iospace;
+
+	bus = bridge->bus;
+
+	rockchip->root_bus = bus;
+
+	pci_bus_size_bridges(bus);
+	pci_bus_assign_resources(bus);
+	list_for_each_entry(child, &bus->children, node)
+		pcie_bus_configure_settings(child);
+
+	pci_bus_add_devices(bus);
+	return 0;
+
+err_unmap_iospace:
+	pci_unmap_iospace(rockchip->io);
+err_free_res:
+	pci_free_resource_list(&res);
+err_remove_irq_domain:
+	irq_domain_remove(rockchip->irq_domain);
+err_deinit_port:
+	rockchip_pcie_deinit_phys(rockchip);
+err_vpcie:
+	if (!IS_ERR(rockchip->vpcie12v))
+		regulator_disable(rockchip->vpcie12v);
+	if (!IS_ERR(rockchip->vpcie3v3))
+		regulator_disable(rockchip->vpcie3v3);
+	if (!IS_ERR(rockchip->vpcie1v8))
+		regulator_disable(rockchip->vpcie1v8);
+	if (!IS_ERR(rockchip->vpcie0v9))
+		regulator_disable(rockchip->vpcie0v9);
+err_set_vpcie:
+	rockchip_pcie_disable_clocks(rockchip);
+	return err;
+}
+
+static int rockchip_pcie_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
+
+	pci_stop_root_bus(rockchip->root_bus);
+	pci_remove_root_bus(rockchip->root_bus);
+	pci_unmap_iospace(rockchip->io);
+	irq_domain_remove(rockchip->irq_domain);
+
+	rockchip_pcie_deinit_phys(rockchip);
+
+	rockchip_pcie_disable_clocks(rockchip);
+
+	if (!IS_ERR(rockchip->vpcie12v))
+		regulator_disable(rockchip->vpcie12v);
+	if (!IS_ERR(rockchip->vpcie3v3))
+		regulator_disable(rockchip->vpcie3v3);
+	if (!IS_ERR(rockchip->vpcie1v8))
+		regulator_disable(rockchip->vpcie1v8);
+	if (!IS_ERR(rockchip->vpcie0v9))
+		regulator_disable(rockchip->vpcie0v9);
+
+	return 0;
+}
+
+static const struct dev_pm_ops rockchip_pcie_pm_ops = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq,
+				      rockchip_pcie_resume_noirq)
+};
+
+static const struct of_device_id rockchip_pcie_of_match[] = {
+	{ .compatible = "rockchip,rk3399-pcie", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match);
+
+static struct platform_driver rockchip_pcie_driver = {
+	.driver = {
+		.name = "rockchip-pcie",
+		.of_match_table = rockchip_pcie_of_match,
+		.pm = &rockchip_pcie_pm_ops,
+	},
+	.probe = rockchip_pcie_probe,
+	.remove = rockchip_pcie_remove,
+};
+module_platform_driver(rockchip_pcie_driver);
+
+MODULE_AUTHOR("Rockchip Inc");
+MODULE_DESCRIPTION("Rockchip AXI PCIe driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pci/host/pcie-rockchip.c b/drivers/pci/host/pcie-rockchip.c
index f1e8f97ea1fb..c53d1322a3d6 100644
--- a/drivers/pci/host/pcie-rockchip.c
+++ b/drivers/pci/host/pcie-rockchip.c
@@ -11,535 +11,154 @@
  * ARM PCI Host generic driver.
  */
 
-#include <linux/bitrev.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/gpio/consumer.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/iopoll.h>
-#include <linux/irq.h>
-#include <linux/irqchip/chained_irq.h>
-#include <linux/irqdomain.h>
-#include <linux/kernel.h>
-#include <linux/mfd/syscon.h>
-#include <linux/module.h>
-#include <linux/of_address.h>
-#include <linux/of_device.h>
 #include <linux/of_pci.h>
-#include <linux/of_platform.h>
-#include <linux/of_irq.h>
-#include <linux/pci.h>
-#include <linux/pci_ids.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/reset.h>
-#include <linux/regmap.h>
 
-/*
- * The upper 16 bits of PCIE_CLIENT_CONFIG are a write mask for the lower 16
- * bits.  This allows atomic updates of the register without locking.
- */
-#define HIWORD_UPDATE(mask, val)	(((mask) << 16) | (val))
-#define HIWORD_UPDATE_BIT(val)		HIWORD_UPDATE(val, val)
-
-#define ENCODE_LANES(x)			((((x) >> 1) & 3) << 4)
-#define MAX_LANE_NUM			4
-
-#define PCIE_CLIENT_BASE		0x0
-#define PCIE_CLIENT_CONFIG		(PCIE_CLIENT_BASE + 0x00)
-#define   PCIE_CLIENT_CONF_ENABLE	  HIWORD_UPDATE_BIT(0x0001)
-#define   PCIE_CLIENT_LINK_TRAIN_ENABLE	  HIWORD_UPDATE_BIT(0x0002)
-#define   PCIE_CLIENT_ARI_ENABLE	  HIWORD_UPDATE_BIT(0x0008)
-#define   PCIE_CLIENT_CONF_LANE_NUM(x)	  HIWORD_UPDATE(0x0030, ENCODE_LANES(x))
-#define   PCIE_CLIENT_MODE_RC		  HIWORD_UPDATE_BIT(0x0040)
-#define   PCIE_CLIENT_GEN_SEL_1		  HIWORD_UPDATE(0x0080, 0)
-#define   PCIE_CLIENT_GEN_SEL_2		  HIWORD_UPDATE_BIT(0x0080)
-#define PCIE_CLIENT_DEBUG_OUT_0		(PCIE_CLIENT_BASE + 0x3c)
-#define   PCIE_CLIENT_DEBUG_LTSSM_MASK		GENMASK(5, 0)
-#define   PCIE_CLIENT_DEBUG_LTSSM_L1		0x18
-#define   PCIE_CLIENT_DEBUG_LTSSM_L2		0x19
-#define PCIE_CLIENT_BASIC_STATUS1	(PCIE_CLIENT_BASE + 0x48)
-#define   PCIE_CLIENT_LINK_STATUS_UP		0x00300000
-#define   PCIE_CLIENT_LINK_STATUS_MASK		0x00300000
-#define PCIE_CLIENT_INT_MASK		(PCIE_CLIENT_BASE + 0x4c)
-#define PCIE_CLIENT_INT_STATUS		(PCIE_CLIENT_BASE + 0x50)
-#define   PCIE_CLIENT_INTR_MASK			GENMASK(8, 5)
-#define   PCIE_CLIENT_INTR_SHIFT		5
-#define   PCIE_CLIENT_INT_LEGACY_DONE		BIT(15)
-#define   PCIE_CLIENT_INT_MSG			BIT(14)
-#define   PCIE_CLIENT_INT_HOT_RST		BIT(13)
-#define   PCIE_CLIENT_INT_DPA			BIT(12)
-#define   PCIE_CLIENT_INT_FATAL_ERR		BIT(11)
-#define   PCIE_CLIENT_INT_NFATAL_ERR		BIT(10)
-#define   PCIE_CLIENT_INT_CORR_ERR		BIT(9)
-#define   PCIE_CLIENT_INT_INTD			BIT(8)
-#define   PCIE_CLIENT_INT_INTC			BIT(7)
-#define   PCIE_CLIENT_INT_INTB			BIT(6)
-#define   PCIE_CLIENT_INT_INTA			BIT(5)
-#define   PCIE_CLIENT_INT_LOCAL			BIT(4)
-#define   PCIE_CLIENT_INT_UDMA			BIT(3)
-#define   PCIE_CLIENT_INT_PHY			BIT(2)
-#define   PCIE_CLIENT_INT_HOT_PLUG		BIT(1)
-#define   PCIE_CLIENT_INT_PWR_STCG		BIT(0)
-
-#define PCIE_CLIENT_INT_LEGACY \
-	(PCIE_CLIENT_INT_INTA | PCIE_CLIENT_INT_INTB | \
-	PCIE_CLIENT_INT_INTC | PCIE_CLIENT_INT_INTD)
-
-#define PCIE_CLIENT_INT_CLI \
-	(PCIE_CLIENT_INT_CORR_ERR | PCIE_CLIENT_INT_NFATAL_ERR | \
-	PCIE_CLIENT_INT_FATAL_ERR | PCIE_CLIENT_INT_DPA | \
-	PCIE_CLIENT_INT_HOT_RST | PCIE_CLIENT_INT_MSG | \
-	PCIE_CLIENT_INT_LEGACY_DONE | PCIE_CLIENT_INT_LEGACY | \
-	PCIE_CLIENT_INT_PHY)
-
-#define PCIE_CORE_CTRL_MGMT_BASE	0x900000
-#define PCIE_CORE_CTRL			(PCIE_CORE_CTRL_MGMT_BASE + 0x000)
-#define   PCIE_CORE_PL_CONF_SPEED_5G		0x00000008
-#define   PCIE_CORE_PL_CONF_SPEED_MASK		0x00000018
-#define   PCIE_CORE_PL_CONF_LANE_MASK		0x00000006
-#define   PCIE_CORE_PL_CONF_LANE_SHIFT		1
-#define PCIE_CORE_CTRL_PLC1		(PCIE_CORE_CTRL_MGMT_BASE + 0x004)
-#define   PCIE_CORE_CTRL_PLC1_FTS_MASK		GENMASK(23, 8)
-#define   PCIE_CORE_CTRL_PLC1_FTS_SHIFT		8
-#define   PCIE_CORE_CTRL_PLC1_FTS_CNT		0xffff
-#define PCIE_CORE_TXCREDIT_CFG1		(PCIE_CORE_CTRL_MGMT_BASE + 0x020)
-#define   PCIE_CORE_TXCREDIT_CFG1_MUI_MASK	0xFFFF0000
-#define   PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT	16
-#define   PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(x) \
-		(((x) >> 3) << PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT)
-#define PCIE_CORE_LANE_MAP             (PCIE_CORE_CTRL_MGMT_BASE + 0x200)
-#define   PCIE_CORE_LANE_MAP_MASK              0x0000000f
-#define   PCIE_CORE_LANE_MAP_REVERSE           BIT(16)
-#define PCIE_CORE_INT_STATUS		(PCIE_CORE_CTRL_MGMT_BASE + 0x20c)
-#define   PCIE_CORE_INT_PRFPE			BIT(0)
-#define   PCIE_CORE_INT_CRFPE			BIT(1)
-#define   PCIE_CORE_INT_RRPE			BIT(2)
-#define   PCIE_CORE_INT_PRFO			BIT(3)
-#define   PCIE_CORE_INT_CRFO			BIT(4)
-#define   PCIE_CORE_INT_RT			BIT(5)
-#define   PCIE_CORE_INT_RTR			BIT(6)
-#define   PCIE_CORE_INT_PE			BIT(7)
-#define   PCIE_CORE_INT_MTR			BIT(8)
-#define   PCIE_CORE_INT_UCR			BIT(9)
-#define   PCIE_CORE_INT_FCE			BIT(10)
-#define   PCIE_CORE_INT_CT			BIT(11)
-#define   PCIE_CORE_INT_UTC			BIT(18)
-#define   PCIE_CORE_INT_MMVC			BIT(19)
-#define PCIE_CORE_CONFIG_VENDOR		(PCIE_CORE_CTRL_MGMT_BASE + 0x44)
-#define PCIE_CORE_INT_MASK		(PCIE_CORE_CTRL_MGMT_BASE + 0x210)
-#define PCIE_RC_BAR_CONF		(PCIE_CORE_CTRL_MGMT_BASE + 0x300)
-
-#define PCIE_CORE_INT \
-		(PCIE_CORE_INT_PRFPE | PCIE_CORE_INT_CRFPE | \
-		 PCIE_CORE_INT_RRPE | PCIE_CORE_INT_CRFO | \
-		 PCIE_CORE_INT_RT | PCIE_CORE_INT_RTR | \
-		 PCIE_CORE_INT_PE | PCIE_CORE_INT_MTR | \
-		 PCIE_CORE_INT_UCR | PCIE_CORE_INT_FCE | \
-		 PCIE_CORE_INT_CT | PCIE_CORE_INT_UTC | \
-		 PCIE_CORE_INT_MMVC)
-
-#define PCIE_RC_CONFIG_NORMAL_BASE	0x800000
-#define PCIE_RC_CONFIG_BASE		0xa00000
-#define PCIE_RC_CONFIG_RID_CCR		(PCIE_RC_CONFIG_BASE + 0x08)
-#define   PCIE_RC_CONFIG_SCC_SHIFT		16
-#define PCIE_RC_CONFIG_DCR		(PCIE_RC_CONFIG_BASE + 0xc4)
-#define   PCIE_RC_CONFIG_DCR_CSPL_SHIFT		18
-#define   PCIE_RC_CONFIG_DCR_CSPL_LIMIT		0xff
-#define   PCIE_RC_CONFIG_DCR_CPLS_SHIFT		26
-#define PCIE_RC_CONFIG_DCSR		(PCIE_RC_CONFIG_BASE + 0xc8)
-#define   PCIE_RC_CONFIG_DCSR_MPS_MASK		GENMASK(7, 5)
-#define   PCIE_RC_CONFIG_DCSR_MPS_256		(0x1 << 5)
-#define PCIE_RC_CONFIG_LINK_CAP		(PCIE_RC_CONFIG_BASE + 0xcc)
-#define   PCIE_RC_CONFIG_LINK_CAP_L0S		BIT(10)
-#define PCIE_RC_CONFIG_LCS		(PCIE_RC_CONFIG_BASE + 0xd0)
-#define PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 (PCIE_RC_CONFIG_BASE + 0x90c)
-#define PCIE_RC_CONFIG_THP_CAP		(PCIE_RC_CONFIG_BASE + 0x274)
-#define   PCIE_RC_CONFIG_THP_CAP_NEXT_MASK	GENMASK(31, 20)
-
-#define PCIE_CORE_AXI_CONF_BASE		0xc00000
-#define PCIE_CORE_OB_REGION_ADDR0	(PCIE_CORE_AXI_CONF_BASE + 0x0)
-#define   PCIE_CORE_OB_REGION_ADDR0_NUM_BITS	0x3f
-#define   PCIE_CORE_OB_REGION_ADDR0_LO_ADDR	0xffffff00
-#define PCIE_CORE_OB_REGION_ADDR1	(PCIE_CORE_AXI_CONF_BASE + 0x4)
-#define PCIE_CORE_OB_REGION_DESC0	(PCIE_CORE_AXI_CONF_BASE + 0x8)
-#define PCIE_CORE_OB_REGION_DESC1	(PCIE_CORE_AXI_CONF_BASE + 0xc)
-
-#define PCIE_CORE_AXI_INBOUND_BASE	0xc00800
-#define PCIE_RP_IB_ADDR0		(PCIE_CORE_AXI_INBOUND_BASE + 0x0)
-#define   PCIE_CORE_IB_REGION_ADDR0_NUM_BITS	0x3f
-#define   PCIE_CORE_IB_REGION_ADDR0_LO_ADDR	0xffffff00
-#define PCIE_RP_IB_ADDR1		(PCIE_CORE_AXI_INBOUND_BASE + 0x4)
-
-/* Size of one AXI Region (not Region 0) */
-#define AXI_REGION_SIZE				BIT(20)
-/* Size of Region 0, equal to sum of sizes of other regions */
-#define AXI_REGION_0_SIZE			(32 * (0x1 << 20))
-#define OB_REG_SIZE_SHIFT			5
-#define IB_ROOT_PORT_REG_SIZE_SHIFT		3
-#define AXI_WRAPPER_IO_WRITE			0x6
-#define AXI_WRAPPER_MEM_WRITE			0x2
-#define AXI_WRAPPER_TYPE0_CFG			0xa
-#define AXI_WRAPPER_TYPE1_CFG			0xb
-#define AXI_WRAPPER_NOR_MSG			0xc
-
-#define MAX_AXI_IB_ROOTPORT_REGION_NUM		3
-#define MIN_AXI_ADDR_BITS_PASSED		8
-#define PCIE_RC_SEND_PME_OFF			0x11960
-#define ROCKCHIP_VENDOR_ID			0x1d87
-#define PCIE_ECAM_BUS(x)			(((x) & 0xff) << 20)
-#define PCIE_ECAM_DEV(x)			(((x) & 0x1f) << 15)
-#define PCIE_ECAM_FUNC(x)			(((x) & 0x7) << 12)
-#define PCIE_ECAM_REG(x)			(((x) & 0xfff) << 0)
-#define PCIE_ECAM_ADDR(bus, dev, func, reg) \
-	  (PCIE_ECAM_BUS(bus) | PCIE_ECAM_DEV(dev) | \
-	   PCIE_ECAM_FUNC(func) | PCIE_ECAM_REG(reg))
-#define PCIE_LINK_IS_L2(x) \
-	(((x) & PCIE_CLIENT_DEBUG_LTSSM_MASK) == PCIE_CLIENT_DEBUG_LTSSM_L2)
-#define PCIE_LINK_UP(x) \
-	(((x) & PCIE_CLIENT_LINK_STATUS_MASK) == PCIE_CLIENT_LINK_STATUS_UP)
-#define PCIE_LINK_IS_GEN2(x) \
-	(((x) & PCIE_CORE_PL_CONF_SPEED_MASK) == PCIE_CORE_PL_CONF_SPEED_5G)
-
-#define RC_REGION_0_ADDR_TRANS_H		0x00000000
-#define RC_REGION_0_ADDR_TRANS_L		0x00000000
-#define RC_REGION_0_PASS_BITS			(25 - 1)
-#define RC_REGION_0_TYPE_MASK			GENMASK(3, 0)
-#define MAX_AXI_WRAPPER_REGION_NUM		33
-
-struct rockchip_pcie {
-	void	__iomem *reg_base;		/* DT axi-base */
-	void	__iomem *apb_base;		/* DT apb-base */
-	bool    legacy_phy;
-	struct  phy *phys[MAX_LANE_NUM];
-	struct	reset_control *core_rst;
-	struct	reset_control *mgmt_rst;
-	struct	reset_control *mgmt_sticky_rst;
-	struct	reset_control *pipe_rst;
-	struct	reset_control *pm_rst;
-	struct	reset_control *aclk_rst;
-	struct	reset_control *pclk_rst;
-	struct	clk *aclk_pcie;
-	struct	clk *aclk_perf_pcie;
-	struct	clk *hclk_pcie;
-	struct	clk *clk_pcie_pm;
-	struct	regulator *vpcie12v; /* 12V power supply */
-	struct	regulator *vpcie3v3; /* 3.3V power supply */
-	struct	regulator *vpcie1v8; /* 1.8V power supply */
-	struct	regulator *vpcie0v9; /* 0.9V power supply */
-	struct	gpio_desc *ep_gpio;
-	u32	lanes;
-	u8      lanes_map;
-	u8	root_bus_nr;
-	int	link_gen;
-	struct	device *dev;
-	struct	irq_domain *irq_domain;
-	int     offset;
-	struct pci_bus *root_bus;
-	struct resource *io;
-	phys_addr_t io_bus_addr;
-	u32     io_size;
-	void    __iomem *msg_region;
-	u32     mem_size;
-	phys_addr_t msg_bus_addr;
-	phys_addr_t mem_bus_addr;
-};
-
-static u32 rockchip_pcie_read(struct rockchip_pcie *rockchip, u32 reg)
-{
-	return readl(rockchip->apb_base + reg);
-}
-
-static void rockchip_pcie_write(struct rockchip_pcie *rockchip, u32 val,
-				u32 reg)
-{
-	writel(val, rockchip->apb_base + reg);
-}
-
-static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
-{
-	u32 status;
-
-	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
-	status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);
-	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
-}
-
-static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
-{
-	u32 status;
-
-	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
-	status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;
-	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
-}
-
-static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)
-{
-	u32 val;
-
-	/* Update Tx credit maximum update interval */
-	val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);
-	val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;
-	val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000);	/* ns */
-	rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);
-}
-
-static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
-				      struct pci_bus *bus, int dev)
-{
-	/* access only one slot on each root port */
-	if (bus->number == rockchip->root_bus_nr && dev > 0)
-		return 0;
-
-	/*
-	 * do not read more than one device on the bus directly attached
-	 * to RC's downstream side.
-	 */
-	if (bus->primary == rockchip->root_bus_nr && dev > 0)
-		return 0;
-
-	return 1;
-}
-
-static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip)
-{
-	u32 val;
-	u8 map;
-
-	if (rockchip->legacy_phy)
-		return GENMASK(MAX_LANE_NUM - 1, 0);
-
-	val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP);
-	map = val & PCIE_CORE_LANE_MAP_MASK;
+#include "../pci.h"
+#include "pcie-rockchip.h"
 
-	/* The link may be using a reverse-indexed mapping. */
-	if (val & PCIE_CORE_LANE_MAP_REVERSE)
-		map = bitrev8(map) >> 4;
-
-	return map;
-}
-
-static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
-				     int where, int size, u32 *val)
+int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
 {
-	void __iomem *addr;
-
-	addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where;
-
-	if (!IS_ALIGNED((uintptr_t)addr, size)) {
-		*val = 0;
-		return PCIBIOS_BAD_REGISTER_NUMBER;
-	}
+	struct device *dev = rockchip->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct device_node *node = dev->of_node;
+	struct resource *regs;
+	int err;
 
-	if (size == 4) {
-		*val = readl(addr);
-	} else if (size == 2) {
-		*val = readw(addr);
-	} else if (size == 1) {
-		*val = readb(addr);
+	if (rockchip->is_rc) {
+		regs = platform_get_resource_byname(pdev,
+						    IORESOURCE_MEM,
+						    "axi-base");
+		rockchip->reg_base = devm_pci_remap_cfg_resource(dev, regs);
+		if (IS_ERR(rockchip->reg_base))
+			return PTR_ERR(rockchip->reg_base);
 	} else {
-		*val = 0;
-		return PCIBIOS_BAD_REGISTER_NUMBER;
+		rockchip->mem_res =
+			platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						     "mem-base");
+		if (!rockchip->mem_res)
+			return -EINVAL;
 	}
-	return PCIBIOS_SUCCESSFUL;
-}
 
-static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,
-				     int where, int size, u32 val)
-{
-	u32 mask, tmp, offset;
-	void __iomem *addr;
+	regs = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+					    "apb-base");
+	rockchip->apb_base = devm_ioremap_resource(dev, regs);
+	if (IS_ERR(rockchip->apb_base))
+		return PTR_ERR(rockchip->apb_base);
 
-	offset = where & ~0x3;
-	addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset;
+	err = rockchip_pcie_get_phys(rockchip);
+	if (err)
+		return err;
 
-	if (size == 4) {
-		writel(val, addr);
-		return PCIBIOS_SUCCESSFUL;
+	rockchip->lanes = 1;
+	err = of_property_read_u32(node, "num-lanes", &rockchip->lanes);
+	if (!err && (rockchip->lanes == 0 ||
+		     rockchip->lanes == 3 ||
+		     rockchip->lanes > 4)) {
+		dev_warn(dev, "invalid num-lanes, default to use one lane\n");
+		rockchip->lanes = 1;
 	}
 
-	mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
-
-	/*
-	 * N.B. This read/modify/write isn't safe in general because it can
-	 * corrupt RW1C bits in adjacent registers.  But the hardware
-	 * doesn't support smaller writes.
-	 */
-	tmp = readl(addr) & mask;
-	tmp |= val << ((where & 0x3) * 8);
-	writel(tmp, addr);
-
-	return PCIBIOS_SUCCESSFUL;
-}
-
-static void rockchip_pcie_cfg_configuration_accesses(
-		struct rockchip_pcie *rockchip, u32 type)
-{
-	u32 ob_desc_0;
-
-	/* Configuration Accesses for region 0 */
-	rockchip_pcie_write(rockchip, 0x0, PCIE_RC_BAR_CONF);
-
-	rockchip_pcie_write(rockchip,
-			    (RC_REGION_0_ADDR_TRANS_L + RC_REGION_0_PASS_BITS),
-			    PCIE_CORE_OB_REGION_ADDR0);
-	rockchip_pcie_write(rockchip, RC_REGION_0_ADDR_TRANS_H,
-			    PCIE_CORE_OB_REGION_ADDR1);
-	ob_desc_0 = rockchip_pcie_read(rockchip, PCIE_CORE_OB_REGION_DESC0);
-	ob_desc_0 &= ~(RC_REGION_0_TYPE_MASK);
-	ob_desc_0 |= (type | (0x1 << 23));
-	rockchip_pcie_write(rockchip, ob_desc_0, PCIE_CORE_OB_REGION_DESC0);
-	rockchip_pcie_write(rockchip, 0x0, PCIE_CORE_OB_REGION_DESC1);
-}
-
-static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,
-				       struct pci_bus *bus, u32 devfn,
-				       int where, int size, u32 *val)
-{
-	u32 busdev;
-
-	busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
-				PCI_FUNC(devfn), where);
+	rockchip->link_gen = of_pci_get_max_link_speed(node);
+	if (rockchip->link_gen < 0 || rockchip->link_gen > 2)
+		rockchip->link_gen = 2;
 
-	if (!IS_ALIGNED(busdev, size)) {
-		*val = 0;
-		return PCIBIOS_BAD_REGISTER_NUMBER;
+	rockchip->core_rst = devm_reset_control_get_exclusive(dev, "core");
+	if (IS_ERR(rockchip->core_rst)) {
+		if (PTR_ERR(rockchip->core_rst) != -EPROBE_DEFER)
+			dev_err(dev, "missing core reset property in node\n");
+		return PTR_ERR(rockchip->core_rst);
 	}
 
-	if (bus->parent->number == rockchip->root_bus_nr)
-		rockchip_pcie_cfg_configuration_accesses(rockchip,
-						AXI_WRAPPER_TYPE0_CFG);
-	else
-		rockchip_pcie_cfg_configuration_accesses(rockchip,
-						AXI_WRAPPER_TYPE1_CFG);
-
-	if (size == 4) {
-		*val = readl(rockchip->reg_base + busdev);
-	} else if (size == 2) {
-		*val = readw(rockchip->reg_base + busdev);
-	} else if (size == 1) {
-		*val = readb(rockchip->reg_base + busdev);
-	} else {
-		*val = 0;
-		return PCIBIOS_BAD_REGISTER_NUMBER;
+	rockchip->mgmt_rst = devm_reset_control_get_exclusive(dev, "mgmt");
+	if (IS_ERR(rockchip->mgmt_rst)) {
+		if (PTR_ERR(rockchip->mgmt_rst) != -EPROBE_DEFER)
+			dev_err(dev, "missing mgmt reset property in node\n");
+		return PTR_ERR(rockchip->mgmt_rst);
 	}
-	return PCIBIOS_SUCCESSFUL;
-}
-
-static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,
-				       struct pci_bus *bus, u32 devfn,
-				       int where, int size, u32 val)
-{
-	u32 busdev;
-
-	busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
-				PCI_FUNC(devfn), where);
-	if (!IS_ALIGNED(busdev, size))
-		return PCIBIOS_BAD_REGISTER_NUMBER;
-
-	if (bus->parent->number == rockchip->root_bus_nr)
-		rockchip_pcie_cfg_configuration_accesses(rockchip,
-						AXI_WRAPPER_TYPE0_CFG);
-	else
-		rockchip_pcie_cfg_configuration_accesses(rockchip,
-						AXI_WRAPPER_TYPE1_CFG);
-
-	if (size == 4)
-		writel(val, rockchip->reg_base + busdev);
-	else if (size == 2)
-		writew(val, rockchip->reg_base + busdev);
-	else if (size == 1)
-		writeb(val, rockchip->reg_base + busdev);
-	else
-		return PCIBIOS_BAD_REGISTER_NUMBER;
-
-	return PCIBIOS_SUCCESSFUL;
-}
-
-static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
-				 int size, u32 *val)
-{
-	struct rockchip_pcie *rockchip = bus->sysdata;
 
-	if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) {
-		*val = 0xffffffff;
-		return PCIBIOS_DEVICE_NOT_FOUND;
+	rockchip->mgmt_sticky_rst = devm_reset_control_get_exclusive(dev,
+								     "mgmt-sticky");
+	if (IS_ERR(rockchip->mgmt_sticky_rst)) {
+		if (PTR_ERR(rockchip->mgmt_sticky_rst) != -EPROBE_DEFER)
+			dev_err(dev, "missing mgmt-sticky reset property in node\n");
+		return PTR_ERR(rockchip->mgmt_sticky_rst);
 	}
 
-	if (bus->number == rockchip->root_bus_nr)
-		return rockchip_pcie_rd_own_conf(rockchip, where, size, val);
-
-	return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, val);
-}
+	rockchip->pipe_rst = devm_reset_control_get_exclusive(dev, "pipe");
+	if (IS_ERR(rockchip->pipe_rst)) {
+		if (PTR_ERR(rockchip->pipe_rst) != -EPROBE_DEFER)
+			dev_err(dev, "missing pipe reset property in node\n");
+		return PTR_ERR(rockchip->pipe_rst);
+	}
 
-static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
-				 int where, int size, u32 val)
-{
-	struct rockchip_pcie *rockchip = bus->sysdata;
+	rockchip->pm_rst = devm_reset_control_get_exclusive(dev, "pm");
+	if (IS_ERR(rockchip->pm_rst)) {
+		if (PTR_ERR(rockchip->pm_rst) != -EPROBE_DEFER)
+			dev_err(dev, "missing pm reset property in node\n");
+		return PTR_ERR(rockchip->pm_rst);
+	}
 
-	if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
-		return PCIBIOS_DEVICE_NOT_FOUND;
+	rockchip->pclk_rst = devm_reset_control_get_exclusive(dev, "pclk");
+	if (IS_ERR(rockchip->pclk_rst)) {
+		if (PTR_ERR(rockchip->pclk_rst) != -EPROBE_DEFER)
+			dev_err(dev, "missing pclk reset property in node\n");
+		return PTR_ERR(rockchip->pclk_rst);
+	}
 
-	if (bus->number == rockchip->root_bus_nr)
-		return rockchip_pcie_wr_own_conf(rockchip, where, size, val);
+	rockchip->aclk_rst = devm_reset_control_get_exclusive(dev, "aclk");
+	if (IS_ERR(rockchip->aclk_rst)) {
+		if (PTR_ERR(rockchip->aclk_rst) != -EPROBE_DEFER)
+			dev_err(dev, "missing aclk reset property in node\n");
+		return PTR_ERR(rockchip->aclk_rst);
+	}
 
-	return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, val);
-}
+	if (rockchip->is_rc) {
+		rockchip->ep_gpio = devm_gpiod_get(dev, "ep", GPIOD_OUT_HIGH);
+		if (IS_ERR(rockchip->ep_gpio)) {
+			dev_err(dev, "missing ep-gpios property in node\n");
+			return PTR_ERR(rockchip->ep_gpio);
+		}
+	}
 
-static struct pci_ops rockchip_pcie_ops = {
-	.read = rockchip_pcie_rd_conf,
-	.write = rockchip_pcie_wr_conf,
-};
+	rockchip->aclk_pcie = devm_clk_get(dev, "aclk");
+	if (IS_ERR(rockchip->aclk_pcie)) {
+		dev_err(dev, "aclk clock not found\n");
+		return PTR_ERR(rockchip->aclk_pcie);
+	}
 
-static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
-{
-	int curr;
-	u32 status, scale, power;
+	rockchip->aclk_perf_pcie = devm_clk_get(dev, "aclk-perf");
+	if (IS_ERR(rockchip->aclk_perf_pcie)) {
+		dev_err(dev, "aclk_perf clock not found\n");
+		return PTR_ERR(rockchip->aclk_perf_pcie);
+	}
 
-	if (IS_ERR(rockchip->vpcie3v3))
-		return;
+	rockchip->hclk_pcie = devm_clk_get(dev, "hclk");
+	if (IS_ERR(rockchip->hclk_pcie)) {
+		dev_err(dev, "hclk clock not found\n");
+		return PTR_ERR(rockchip->hclk_pcie);
+	}
 
-	/*
-	 * Set RC's captured slot power limit and scale if
-	 * vpcie3v3 available. The default values are both zero
-	 * which means the software should set these two according
-	 * to the actual power supply.
-	 */
-	curr = regulator_get_current_limit(rockchip->vpcie3v3);
-	if (curr <= 0)
-		return;
-
-	scale = 3; /* 0.001x */
-	curr = curr / 1000; /* convert to mA */
-	power = (curr * 3300) / 1000; /* milliwatt */
-	while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) {
-		if (!scale) {
-			dev_warn(rockchip->dev, "invalid power supply\n");
-			return;
-		}
-		scale--;
-		power = power / 10;
+	rockchip->clk_pcie_pm = devm_clk_get(dev, "pm");
+	if (IS_ERR(rockchip->clk_pcie_pm)) {
+		dev_err(dev, "pm clock not found\n");
+		return PTR_ERR(rockchip->clk_pcie_pm);
 	}
 
-	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR);
-	status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) |
-		  (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT);
-	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR);
+	return 0;
 }
+EXPORT_SYMBOL_GPL(rockchip_pcie_parse_dt);
 
-/**
- * rockchip_pcie_init_port - Initialize hardware
- * @rockchip: PCIe port information
- */
-static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
+int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 {
 	struct device *dev = rockchip->dev;
 	int err, i;
-	u32 status;
-
-	gpiod_set_value_cansleep(rockchip->ep_gpio, 0);
+	u32 regs;
 
 	err = reset_control_assert(rockchip->aclk_rst);
 	if (err) {
@@ -618,13 +237,15 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 		rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_1,
 				    PCIE_CLIENT_CONFIG);
 
-	rockchip_pcie_write(rockchip,
-			    PCIE_CLIENT_CONF_ENABLE |
-			    PCIE_CLIENT_LINK_TRAIN_ENABLE |
-			    PCIE_CLIENT_ARI_ENABLE |
-			    PCIE_CLIENT_CONF_LANE_NUM(rockchip->lanes) |
-			    PCIE_CLIENT_MODE_RC,
-			    PCIE_CLIENT_CONFIG);
+	regs = PCIE_CLIENT_LINK_TRAIN_ENABLE | PCIE_CLIENT_ARI_ENABLE |
+	       PCIE_CLIENT_CONF_LANE_NUM(rockchip->lanes);
+
+	if (rockchip->is_rc)
+		regs |= PCIE_CLIENT_CONF_ENABLE | PCIE_CLIENT_MODE_RC;
+	else
+		regs |= PCIE_CLIENT_CONF_DISABLE | PCIE_CLIENT_MODE_EP;
+
+	rockchip_pcie_write(rockchip, regs, PCIE_CLIENT_CONFIG);
 
 	for (i = 0; i < MAX_LANE_NUM; i++) {
 		err = phy_power_on(rockchip->phys[i]);
@@ -662,105 +283,61 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 		goto err_power_off_phy;
 	}
 
-	/* Fix the transmitted FTS count desired to exit from L0s. */
-	status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
-	status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |
-		 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
-	rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
-
-	rockchip_pcie_set_power_limit(rockchip);
-
-	/* Set RC's clock architecture as common clock */
-	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
-	status |= PCI_EXP_LNKSTA_SLC << 16;
-	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
-
-	/* Set RC's RCB to 128 */
-	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
-	status |= PCI_EXP_LNKCTL_RCB;
-	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
-
-	/* Enable Gen1 training */
-	rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
-			    PCIE_CLIENT_CONFIG);
+	return 0;
+err_power_off_phy:
+	while (i--)
+		phy_power_off(rockchip->phys[i]);
+	i = MAX_LANE_NUM;
+err_exit_phy:
+	while (i--)
+		phy_exit(rockchip->phys[i]);
+	return err;
+}
+EXPORT_SYMBOL_GPL(rockchip_pcie_init_port);
 
-	gpiod_set_value_cansleep(rockchip->ep_gpio, 1);
+int rockchip_pcie_get_phys(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	struct phy *phy;
+	char *name;
+	u32 i;
 
-	/* 500ms timeout value should be enough for Gen1/2 training */
-	err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,
-				 status, PCIE_LINK_UP(status), 20,
-				 500 * USEC_PER_MSEC);
-	if (err) {
-		dev_err(dev, "PCIe link training gen1 timeout!\n");
-		goto err_power_off_phy;
+	phy = devm_phy_get(dev, "pcie-phy");
+	if (!IS_ERR(phy)) {
+		rockchip->legacy_phy = true;
+		rockchip->phys[0] = phy;
+		dev_warn(dev, "legacy phy model is deprecated!\n");
+		return 0;
 	}
 
-	if (rockchip->link_gen == 2) {
-		/*
-		 * Enable retrain for gen2. This should be configured only after
-		 * gen1 finished.
-		 */
-		status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
-		status |= PCI_EXP_LNKCTL_RL;
-		rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
-
-		err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL,
-					 status, PCIE_LINK_IS_GEN2(status), 20,
-					 500 * USEC_PER_MSEC);
-		if (err)
-			dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
-	}
+	if (PTR_ERR(phy) == -EPROBE_DEFER)
+		return PTR_ERR(phy);
 
-	/* Check the final link width from negotiated lane counter from MGMT */
-	status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
-	status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
-			  PCIE_CORE_PL_CONF_LANE_SHIFT);
-	dev_dbg(dev, "current link width is x%d\n", status);
+	dev_dbg(dev, "missing legacy phy; search for per-lane PHY\n");
 
-	/* Power off unused lane(s) */
-	rockchip->lanes_map = rockchip_pcie_lane_map(rockchip);
 	for (i = 0; i < MAX_LANE_NUM; i++) {
-		if (!(rockchip->lanes_map & BIT(i))) {
-			dev_dbg(dev, "idling lane %d\n", i);
-			phy_power_off(rockchip->phys[i]);
+		name = kasprintf(GFP_KERNEL, "pcie-phy-%u", i);
+		if (!name)
+			return -ENOMEM;
+
+		phy = devm_of_phy_get(dev, dev->of_node, name);
+		kfree(name);
+
+		if (IS_ERR(phy)) {
+			if (PTR_ERR(phy) != -EPROBE_DEFER)
+				dev_err(dev, "missing phy for lane %d: %ld\n",
+					i, PTR_ERR(phy));
+			return PTR_ERR(phy);
 		}
-	}
 
-	rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,
-			    PCIE_CORE_CONFIG_VENDOR);
-	rockchip_pcie_write(rockchip,
-			    PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT,
-			    PCIE_RC_CONFIG_RID_CCR);
-
-	/* Clear THP cap's next cap pointer to remove L1 substate cap */
-	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);
-	status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
-	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);
-
-	/* Clear L0s from RC's link cap */
-	if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) {
-		status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP);
-		status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S;
-		rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP);
+		rockchip->phys[i] = phy;
 	}
 
-	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR);
-	status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK;
-	status |= PCIE_RC_CONFIG_DCSR_MPS_256;
-	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR);
-
 	return 0;
-err_power_off_phy:
-	while (i--)
-		phy_power_off(rockchip->phys[i]);
-	i = MAX_LANE_NUM;
-err_exit_phy:
-	while (i--)
-		phy_exit(rockchip->phys[i]);
-	return err;
 }
+EXPORT_SYMBOL_GPL(rockchip_pcie_get_phys);
 
-static void rockchip_pcie_deinit_phys(struct rockchip_pcie *rockchip)
+void rockchip_pcie_deinit_phys(struct rockchip_pcie *rockchip)
 {
 	int i;
 
@@ -771,656 +348,36 @@ static void rockchip_pcie_deinit_phys(struct rockchip_pcie *rockchip)
 		phy_exit(rockchip->phys[i]);
 	}
 }
+EXPORT_SYMBOL_GPL(rockchip_pcie_deinit_phys);
 
-static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
+int rockchip_pcie_enable_clocks(struct rockchip_pcie *rockchip)
 {
-	struct rockchip_pcie *rockchip = arg;
 	struct device *dev = rockchip->dev;
-	u32 reg;
-	u32 sub_reg;
+	int err;
 
-	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
-	if (reg & PCIE_CLIENT_INT_LOCAL) {
-		dev_dbg(dev, "local interrupt received\n");
-		sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);
-		if (sub_reg & PCIE_CORE_INT_PRFPE)
-			dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");
+	err = clk_prepare_enable(rockchip->aclk_pcie);
+	if (err) {
+		dev_err(dev, "unable to enable aclk_pcie clock\n");
+		return err;
+	}
 
-		if (sub_reg & PCIE_CORE_INT_CRFPE)
-			dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");
+	err = clk_prepare_enable(rockchip->aclk_perf_pcie);
+	if (err) {
+		dev_err(dev, "unable to enable aclk_perf_pcie clock\n");
+		goto err_aclk_perf_pcie;
+	}
 
-		if (sub_reg & PCIE_CORE_INT_RRPE)
-			dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");
+	err = clk_prepare_enable(rockchip->hclk_pcie);
+	if (err) {
+		dev_err(dev, "unable to enable hclk_pcie clock\n");
+		goto err_hclk_pcie;
+	}
 
-		if (sub_reg & PCIE_CORE_INT_PRFO)
-			dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");
-
-		if (sub_reg & PCIE_CORE_INT_CRFO)
-			dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");
-
-		if (sub_reg & PCIE_CORE_INT_RT)
-			dev_dbg(dev, "replay timer timed out\n");
-
-		if (sub_reg & PCIE_CORE_INT_RTR)
-			dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");
-
-		if (sub_reg & PCIE_CORE_INT_PE)
-			dev_dbg(dev, "phy error detected on receive side\n");
-
-		if (sub_reg & PCIE_CORE_INT_MTR)
-			dev_dbg(dev, "malformed TLP received from the link\n");
-
-		if (sub_reg & PCIE_CORE_INT_UCR)
-			dev_dbg(dev, "malformed TLP received from the link\n");
-
-		if (sub_reg & PCIE_CORE_INT_FCE)
-			dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");
-
-		if (sub_reg & PCIE_CORE_INT_CT)
-			dev_dbg(dev, "a request timed out waiting for completion\n");
-
-		if (sub_reg & PCIE_CORE_INT_UTC)
-			dev_dbg(dev, "unmapped TC error\n");
-
-		if (sub_reg & PCIE_CORE_INT_MMVC)
-			dev_dbg(dev, "MSI mask register changes\n");
-
-		rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);
-	} else if (reg & PCIE_CLIENT_INT_PHY) {
-		dev_dbg(dev, "phy link changes\n");
-		rockchip_pcie_update_txcredit_mui(rockchip);
-		rockchip_pcie_clr_bw_int(rockchip);
-	}
-
-	rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,
-			    PCIE_CLIENT_INT_STATUS);
-
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)
-{
-	struct rockchip_pcie *rockchip = arg;
-	struct device *dev = rockchip->dev;
-	u32 reg;
-
-	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
-	if (reg & PCIE_CLIENT_INT_LEGACY_DONE)
-		dev_dbg(dev, "legacy done interrupt received\n");
-
-	if (reg & PCIE_CLIENT_INT_MSG)
-		dev_dbg(dev, "message done interrupt received\n");
-
-	if (reg & PCIE_CLIENT_INT_HOT_RST)
-		dev_dbg(dev, "hot reset interrupt received\n");
-
-	if (reg & PCIE_CLIENT_INT_DPA)
-		dev_dbg(dev, "dpa interrupt received\n");
-
-	if (reg & PCIE_CLIENT_INT_FATAL_ERR)
-		dev_dbg(dev, "fatal error interrupt received\n");
-
-	if (reg & PCIE_CLIENT_INT_NFATAL_ERR)
-		dev_dbg(dev, "no fatal error interrupt received\n");
-
-	if (reg & PCIE_CLIENT_INT_CORR_ERR)
-		dev_dbg(dev, "correctable error interrupt received\n");
-
-	if (reg & PCIE_CLIENT_INT_PHY)
-		dev_dbg(dev, "phy interrupt received\n");
-
-	rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |
-			      PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |
-			      PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |
-			      PCIE_CLIENT_INT_NFATAL_ERR |
-			      PCIE_CLIENT_INT_CORR_ERR |
-			      PCIE_CLIENT_INT_PHY),
-		   PCIE_CLIENT_INT_STATUS);
-
-	return IRQ_HANDLED;
-}
-
-static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc)
-{
-	struct irq_chip *chip = irq_desc_get_chip(desc);
-	struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
-	struct device *dev = rockchip->dev;
-	u32 reg;
-	u32 hwirq;
-	u32 virq;
-
-	chained_irq_enter(chip, desc);
-
-	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
-	reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;
-
-	while (reg) {
-		hwirq = ffs(reg) - 1;
-		reg &= ~BIT(hwirq);
-
-		virq = irq_find_mapping(rockchip->irq_domain, hwirq);
-		if (virq)
-			generic_handle_irq(virq);
-		else
-			dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);
-	}
-
-	chained_irq_exit(chip, desc);
-}
-
-static int rockchip_pcie_get_phys(struct rockchip_pcie *rockchip)
-{
-	struct device *dev = rockchip->dev;
-	struct phy *phy;
-	char *name;
-	u32 i;
-
-	phy = devm_phy_get(dev, "pcie-phy");
-	if (!IS_ERR(phy)) {
-		rockchip->legacy_phy = true;
-		rockchip->phys[0] = phy;
-		dev_warn(dev, "legacy phy model is deprecated!\n");
-		return 0;
-	}
-
-	if (PTR_ERR(phy) == -EPROBE_DEFER)
-		return PTR_ERR(phy);
-
-	dev_dbg(dev, "missing legacy phy; search for per-lane PHY\n");
-
-	for (i = 0; i < MAX_LANE_NUM; i++) {
-		name = kasprintf(GFP_KERNEL, "pcie-phy-%u", i);
-		if (!name)
-			return -ENOMEM;
-
-		phy = devm_of_phy_get(dev, dev->of_node, name);
-		kfree(name);
-
-		if (IS_ERR(phy)) {
-			if (PTR_ERR(phy) != -EPROBE_DEFER)
-				dev_err(dev, "missing phy for lane %d: %ld\n",
-					i, PTR_ERR(phy));
-			return PTR_ERR(phy);
-		}
-
-		rockchip->phys[i] = phy;
-	}
-
-	return 0;
-}
-
-static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip)
-{
-	int irq, err;
-	struct device *dev = rockchip->dev;
-	struct platform_device *pdev = to_platform_device(dev);
-
-	irq = platform_get_irq_byname(pdev, "sys");
-	if (irq < 0) {
-		dev_err(dev, "missing sys IRQ resource\n");
-		return irq;
-	}
-
-	err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
-			       IRQF_SHARED, "pcie-sys", rockchip);
-	if (err) {
-		dev_err(dev, "failed to request PCIe subsystem IRQ\n");
-		return err;
-	}
-
-	irq = platform_get_irq_byname(pdev, "legacy");
-	if (irq < 0) {
-		dev_err(dev, "missing legacy IRQ resource\n");
-		return irq;
-	}
-
-	irq_set_chained_handler_and_data(irq,
-					 rockchip_pcie_legacy_int_handler,
-					 rockchip);
-
-	irq = platform_get_irq_byname(pdev, "client");
-	if (irq < 0) {
-		dev_err(dev, "missing client IRQ resource\n");
-		return irq;
-	}
-
-	err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
-			       IRQF_SHARED, "pcie-client", rockchip);
-	if (err) {
-		dev_err(dev, "failed to request PCIe client IRQ\n");
-		return err;
-	}
-
-	return 0;
-}
-
-/**
- * rockchip_pcie_parse_dt - Parse Device Tree
- * @rockchip: PCIe port information
- *
- * Return: '0' on success and error value on failure
- */
-static int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
-{
-	struct device *dev = rockchip->dev;
-	struct platform_device *pdev = to_platform_device(dev);
-	struct device_node *node = dev->of_node;
-	struct resource *regs;
-	int err;
-
-	regs = platform_get_resource_byname(pdev,
-					    IORESOURCE_MEM,
-					    "axi-base");
-	rockchip->reg_base = devm_pci_remap_cfg_resource(dev, regs);
-	if (IS_ERR(rockchip->reg_base))
-		return PTR_ERR(rockchip->reg_base);
-
-	regs = platform_get_resource_byname(pdev,
-					    IORESOURCE_MEM,
-					    "apb-base");
-	rockchip->apb_base = devm_ioremap_resource(dev, regs);
-	if (IS_ERR(rockchip->apb_base))
-		return PTR_ERR(rockchip->apb_base);
-
-	err = rockchip_pcie_get_phys(rockchip);
-	if (err)
-		return err;
-
-	rockchip->lanes = 1;
-	err = of_property_read_u32(node, "num-lanes", &rockchip->lanes);
-	if (!err && (rockchip->lanes == 0 ||
-		     rockchip->lanes == 3 ||
-		     rockchip->lanes > 4)) {
-		dev_warn(dev, "invalid num-lanes, default to use one lane\n");
-		rockchip->lanes = 1;
-	}
-
-	rockchip->link_gen = of_pci_get_max_link_speed(node);
-	if (rockchip->link_gen < 0 || rockchip->link_gen > 2)
-		rockchip->link_gen = 2;
-
-	rockchip->core_rst = devm_reset_control_get_exclusive(dev, "core");
-	if (IS_ERR(rockchip->core_rst)) {
-		if (PTR_ERR(rockchip->core_rst) != -EPROBE_DEFER)
-			dev_err(dev, "missing core reset property in node\n");
-		return PTR_ERR(rockchip->core_rst);
-	}
-
-	rockchip->mgmt_rst = devm_reset_control_get_exclusive(dev, "mgmt");
-	if (IS_ERR(rockchip->mgmt_rst)) {
-		if (PTR_ERR(rockchip->mgmt_rst) != -EPROBE_DEFER)
-			dev_err(dev, "missing mgmt reset property in node\n");
-		return PTR_ERR(rockchip->mgmt_rst);
-	}
-
-	rockchip->mgmt_sticky_rst = devm_reset_control_get_exclusive(dev,
-								     "mgmt-sticky");
-	if (IS_ERR(rockchip->mgmt_sticky_rst)) {
-		if (PTR_ERR(rockchip->mgmt_sticky_rst) != -EPROBE_DEFER)
-			dev_err(dev, "missing mgmt-sticky reset property in node\n");
-		return PTR_ERR(rockchip->mgmt_sticky_rst);
-	}
-
-	rockchip->pipe_rst = devm_reset_control_get_exclusive(dev, "pipe");
-	if (IS_ERR(rockchip->pipe_rst)) {
-		if (PTR_ERR(rockchip->pipe_rst) != -EPROBE_DEFER)
-			dev_err(dev, "missing pipe reset property in node\n");
-		return PTR_ERR(rockchip->pipe_rst);
-	}
-
-	rockchip->pm_rst = devm_reset_control_get_exclusive(dev, "pm");
-	if (IS_ERR(rockchip->pm_rst)) {
-		if (PTR_ERR(rockchip->pm_rst) != -EPROBE_DEFER)
-			dev_err(dev, "missing pm reset property in node\n");
-		return PTR_ERR(rockchip->pm_rst);
-	}
-
-	rockchip->pclk_rst = devm_reset_control_get_exclusive(dev, "pclk");
-	if (IS_ERR(rockchip->pclk_rst)) {
-		if (PTR_ERR(rockchip->pclk_rst) != -EPROBE_DEFER)
-			dev_err(dev, "missing pclk reset property in node\n");
-		return PTR_ERR(rockchip->pclk_rst);
-	}
-
-	rockchip->aclk_rst = devm_reset_control_get_exclusive(dev, "aclk");
-	if (IS_ERR(rockchip->aclk_rst)) {
-		if (PTR_ERR(rockchip->aclk_rst) != -EPROBE_DEFER)
-			dev_err(dev, "missing aclk reset property in node\n");
-		return PTR_ERR(rockchip->aclk_rst);
-	}
-
-	rockchip->ep_gpio = devm_gpiod_get(dev, "ep", GPIOD_OUT_HIGH);
-	if (IS_ERR(rockchip->ep_gpio)) {
-		dev_err(dev, "missing ep-gpios property in node\n");
-		return PTR_ERR(rockchip->ep_gpio);
-	}
-
-	rockchip->aclk_pcie = devm_clk_get(dev, "aclk");
-	if (IS_ERR(rockchip->aclk_pcie)) {
-		dev_err(dev, "aclk clock not found\n");
-		return PTR_ERR(rockchip->aclk_pcie);
-	}
-
-	rockchip->aclk_perf_pcie = devm_clk_get(dev, "aclk-perf");
-	if (IS_ERR(rockchip->aclk_perf_pcie)) {
-		dev_err(dev, "aclk_perf clock not found\n");
-		return PTR_ERR(rockchip->aclk_perf_pcie);
-	}
-
-	rockchip->hclk_pcie = devm_clk_get(dev, "hclk");
-	if (IS_ERR(rockchip->hclk_pcie)) {
-		dev_err(dev, "hclk clock not found\n");
-		return PTR_ERR(rockchip->hclk_pcie);
-	}
-
-	rockchip->clk_pcie_pm = devm_clk_get(dev, "pm");
-	if (IS_ERR(rockchip->clk_pcie_pm)) {
-		dev_err(dev, "pm clock not found\n");
-		return PTR_ERR(rockchip->clk_pcie_pm);
-	}
-
-	err = rockchip_pcie_setup_irq(rockchip);
-	if (err)
-		return err;
-
-	rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v");
-	if (IS_ERR(rockchip->vpcie12v)) {
-		if (PTR_ERR(rockchip->vpcie12v) == -EPROBE_DEFER)
-			return -EPROBE_DEFER;
-		dev_info(dev, "no vpcie12v regulator found\n");
-	}
-
-	rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
-	if (IS_ERR(rockchip->vpcie3v3)) {
-		if (PTR_ERR(rockchip->vpcie3v3) == -EPROBE_DEFER)
-			return -EPROBE_DEFER;
-		dev_info(dev, "no vpcie3v3 regulator found\n");
-	}
-
-	rockchip->vpcie1v8 = devm_regulator_get_optional(dev, "vpcie1v8");
-	if (IS_ERR(rockchip->vpcie1v8)) {
-		if (PTR_ERR(rockchip->vpcie1v8) == -EPROBE_DEFER)
-			return -EPROBE_DEFER;
-		dev_info(dev, "no vpcie1v8 regulator found\n");
-	}
-
-	rockchip->vpcie0v9 = devm_regulator_get_optional(dev, "vpcie0v9");
-	if (IS_ERR(rockchip->vpcie0v9)) {
-		if (PTR_ERR(rockchip->vpcie0v9) == -EPROBE_DEFER)
-			return -EPROBE_DEFER;
-		dev_info(dev, "no vpcie0v9 regulator found\n");
-	}
-
-	return 0;
-}
-
-static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
-{
-	struct device *dev = rockchip->dev;
-	int err;
-
-	if (!IS_ERR(rockchip->vpcie12v)) {
-		err = regulator_enable(rockchip->vpcie12v);
-		if (err) {
-			dev_err(dev, "fail to enable vpcie12v regulator\n");
-			goto err_out;
-		}
-	}
-
-	if (!IS_ERR(rockchip->vpcie3v3)) {
-		err = regulator_enable(rockchip->vpcie3v3);
-		if (err) {
-			dev_err(dev, "fail to enable vpcie3v3 regulator\n");
-			goto err_disable_12v;
-		}
-	}
-
-	if (!IS_ERR(rockchip->vpcie1v8)) {
-		err = regulator_enable(rockchip->vpcie1v8);
-		if (err) {
-			dev_err(dev, "fail to enable vpcie1v8 regulator\n");
-			goto err_disable_3v3;
-		}
-	}
-
-	if (!IS_ERR(rockchip->vpcie0v9)) {
-		err = regulator_enable(rockchip->vpcie0v9);
-		if (err) {
-			dev_err(dev, "fail to enable vpcie0v9 regulator\n");
-			goto err_disable_1v8;
-		}
-	}
-
-	return 0;
-
-err_disable_1v8:
-	if (!IS_ERR(rockchip->vpcie1v8))
-		regulator_disable(rockchip->vpcie1v8);
-err_disable_3v3:
-	if (!IS_ERR(rockchip->vpcie3v3))
-		regulator_disable(rockchip->vpcie3v3);
-err_disable_12v:
-	if (!IS_ERR(rockchip->vpcie12v))
-		regulator_disable(rockchip->vpcie12v);
-err_out:
-	return err;
-}
-
-static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)
-{
-	rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &
-			    (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);
-	rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),
-			    PCIE_CORE_INT_MASK);
-
-	rockchip_pcie_enable_bw_int(rockchip);
-}
-
-static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
-				  irq_hw_number_t hwirq)
-{
-	irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
-	irq_set_chip_data(irq, domain->host_data);
-
-	return 0;
-}
-
-static const struct irq_domain_ops intx_domain_ops = {
-	.map = rockchip_pcie_intx_map,
-};
-
-static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
-{
-	struct device *dev = rockchip->dev;
-	struct device_node *intc = of_get_next_child(dev->of_node, NULL);
-
-	if (!intc) {
-		dev_err(dev, "missing child interrupt-controller node\n");
-		return -EINVAL;
-	}
-
-	rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX,
-						    &intx_domain_ops, rockchip);
-	if (!rockchip->irq_domain) {
-		dev_err(dev, "failed to get a INTx IRQ domain\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,
-				     int region_no, int type, u8 num_pass_bits,
-				     u32 lower_addr, u32 upper_addr)
-{
-	u32 ob_addr_0;
-	u32 ob_addr_1;
-	u32 ob_desc_0;
-	u32 aw_offset;
-
-	if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)
-		return -EINVAL;
-	if (num_pass_bits + 1 < 8)
-		return -EINVAL;
-	if (num_pass_bits > 63)
-		return -EINVAL;
-	if (region_no == 0) {
-		if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))
-			return -EINVAL;
-	}
-	if (region_no != 0) {
-		if (AXI_REGION_SIZE < (2ULL << num_pass_bits))
-			return -EINVAL;
-	}
-
-	aw_offset = (region_no << OB_REG_SIZE_SHIFT);
-
-	ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;
-	ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;
-	ob_addr_1 = upper_addr;
-	ob_desc_0 = (1 << 23 | type);
-
-	rockchip_pcie_write(rockchip, ob_addr_0,
-			    PCIE_CORE_OB_REGION_ADDR0 + aw_offset);
-	rockchip_pcie_write(rockchip, ob_addr_1,
-			    PCIE_CORE_OB_REGION_ADDR1 + aw_offset);
-	rockchip_pcie_write(rockchip, ob_desc_0,
-			    PCIE_CORE_OB_REGION_DESC0 + aw_offset);
-	rockchip_pcie_write(rockchip, 0,
-			    PCIE_CORE_OB_REGION_DESC1 + aw_offset);
-
-	return 0;
-}
-
-static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
-				     int region_no, u8 num_pass_bits,
-				     u32 lower_addr, u32 upper_addr)
-{
-	u32 ib_addr_0;
-	u32 ib_addr_1;
-	u32 aw_offset;
-
-	if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)
-		return -EINVAL;
-	if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)
-		return -EINVAL;
-	if (num_pass_bits > 63)
-		return -EINVAL;
-
-	aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);
-
-	ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;
-	ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;
-	ib_addr_1 = upper_addr;
-
-	rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);
-	rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);
-
-	return 0;
-}
-
-static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip)
-{
-	struct device *dev = rockchip->dev;
-	int offset;
-	int err;
-	int reg_no;
-
-	rockchip_pcie_cfg_configuration_accesses(rockchip,
-						 AXI_WRAPPER_TYPE0_CFG);
-
-	for (reg_no = 0; reg_no < (rockchip->mem_size >> 20); reg_no++) {
-		err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
-						AXI_WRAPPER_MEM_WRITE,
-						20 - 1,
-						rockchip->mem_bus_addr +
-						(reg_no << 20),
-						0);
-		if (err) {
-			dev_err(dev, "program RC mem outbound ATU failed\n");
-			return err;
-		}
-	}
-
-	err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
-	if (err) {
-		dev_err(dev, "program RC mem inbound ATU failed\n");
-		return err;
-	}
-
-	offset = rockchip->mem_size >> 20;
-	for (reg_no = 0; reg_no < (rockchip->io_size >> 20); reg_no++) {
-		err = rockchip_pcie_prog_ob_atu(rockchip,
-						reg_no + 1 + offset,
-						AXI_WRAPPER_IO_WRITE,
-						20 - 1,
-						rockchip->io_bus_addr +
-						(reg_no << 20),
-						0);
-		if (err) {
-			dev_err(dev, "program RC io outbound ATU failed\n");
-			return err;
-		}
-	}
-
-	/* assign message regions */
-	rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset,
-				  AXI_WRAPPER_NOR_MSG,
-				  20 - 1, 0, 0);
-
-	rockchip->msg_bus_addr = rockchip->mem_bus_addr +
-					((reg_no + offset) << 20);
-	return err;
-}
-
-static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)
-{
-	u32 value;
-	int err;
-
-	/* send PME_TURN_OFF message */
-	writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF);
-
-	/* read LTSSM and wait for falling into L2 link state */
-	err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0,
-				 value, PCIE_LINK_IS_L2(value), 20,
-				 jiffies_to_usecs(5 * HZ));
-	if (err) {
-		dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n");
-		return err;
-	}
-
-	return 0;
-}
-
-static int rockchip_pcie_enable_clocks(struct rockchip_pcie *rockchip)
-{
-	struct device *dev = rockchip->dev;
-	int err;
-
-	err = clk_prepare_enable(rockchip->aclk_pcie);
-	if (err) {
-		dev_err(dev, "unable to enable aclk_pcie clock\n");
-		return err;
-	}
-
-	err = clk_prepare_enable(rockchip->aclk_perf_pcie);
-	if (err) {
-		dev_err(dev, "unable to enable aclk_perf_pcie clock\n");
-		goto err_aclk_perf_pcie;
-	}
-
-	err = clk_prepare_enable(rockchip->hclk_pcie);
-	if (err) {
-		dev_err(dev, "unable to enable hclk_pcie clock\n");
-		goto err_hclk_pcie;
-	}
-
-	err = clk_prepare_enable(rockchip->clk_pcie_pm);
-	if (err) {
-		dev_err(dev, "unable to enable clk_pcie_pm clock\n");
-		goto err_clk_pcie_pm;
-	}
+	err = clk_prepare_enable(rockchip->clk_pcie_pm);
+	if (err) {
+		dev_err(dev, "unable to enable clk_pcie_pm clock\n");
+		goto err_clk_pcie_pm;
+	}
 
 	return 0;
 
@@ -1432,8 +389,9 @@ static int rockchip_pcie_enable_clocks(struct rockchip_pcie *rockchip)
 	clk_disable_unprepare(rockchip->aclk_pcie);
 	return err;
 }
+EXPORT_SYMBOL_GPL(rockchip_pcie_enable_clocks);
 
-static void rockchip_pcie_disable_clocks(void *data)
+void rockchip_pcie_disable_clocks(void *data)
 {
 	struct rockchip_pcie *rockchip = data;
 
@@ -1442,267 +400,25 @@ static void rockchip_pcie_disable_clocks(void *data)
 	clk_disable_unprepare(rockchip->aclk_perf_pcie);
 	clk_disable_unprepare(rockchip->aclk_pcie);
 }
+EXPORT_SYMBOL_GPL(rockchip_pcie_disable_clocks);
 
-static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev)
-{
-	struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
-	int ret;
-
-	/* disable core and cli int since we don't need to ack PME_ACK */
-	rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) |
-			    PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK);
-	rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK);
-
-	ret = rockchip_pcie_wait_l2(rockchip);
-	if (ret) {
-		rockchip_pcie_enable_interrupts(rockchip);
-		return ret;
-	}
-
-	rockchip_pcie_deinit_phys(rockchip);
-
-	rockchip_pcie_disable_clocks(rockchip);
-
-	if (!IS_ERR(rockchip->vpcie0v9))
-		regulator_disable(rockchip->vpcie0v9);
-
-	return ret;
-}
-
-static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev)
-{
-	struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
-	int err;
-
-	if (!IS_ERR(rockchip->vpcie0v9)) {
-		err = regulator_enable(rockchip->vpcie0v9);
-		if (err) {
-			dev_err(dev, "fail to enable vpcie0v9 regulator\n");
-			return err;
-		}
-	}
-
-	err = rockchip_pcie_enable_clocks(rockchip);
-	if (err)
-		goto err_disable_0v9;
-
-	err = rockchip_pcie_init_port(rockchip);
-	if (err)
-		goto err_pcie_resume;
-
-	err = rockchip_pcie_cfg_atu(rockchip);
-	if (err)
-		goto err_err_deinit_port;
-
-	/* Need this to enter L1 again */
-	rockchip_pcie_update_txcredit_mui(rockchip);
-	rockchip_pcie_enable_interrupts(rockchip);
-
-	return 0;
-
-err_err_deinit_port:
-	rockchip_pcie_deinit_phys(rockchip);
-err_pcie_resume:
-	rockchip_pcie_disable_clocks(rockchip);
-err_disable_0v9:
-	if (!IS_ERR(rockchip->vpcie0v9))
-		regulator_disable(rockchip->vpcie0v9);
-	return err;
-}
-
-static int rockchip_pcie_probe(struct platform_device *pdev)
-{
-	struct rockchip_pcie *rockchip;
-	struct device *dev = &pdev->dev;
-	struct pci_bus *bus, *child;
-	struct pci_host_bridge *bridge;
-	struct resource_entry *win;
-	resource_size_t io_base;
-	struct resource	*mem;
-	struct resource	*io;
-	int err;
-
-	LIST_HEAD(res);
-
-	if (!dev->of_node)
-		return -ENODEV;
-
-	bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip));
-	if (!bridge)
-		return -ENOMEM;
-
-	rockchip = pci_host_bridge_priv(bridge);
-
-	platform_set_drvdata(pdev, rockchip);
-	rockchip->dev = dev;
-
-	err = rockchip_pcie_parse_dt(rockchip);
-	if (err)
-		return err;
-
-	err = rockchip_pcie_enable_clocks(rockchip);
-	if (err)
-		return err;
-
-	err = rockchip_pcie_set_vpcie(rockchip);
-	if (err) {
-		dev_err(dev, "failed to set vpcie regulator\n");
-		goto err_set_vpcie;
-	}
-
-	err = rockchip_pcie_init_port(rockchip);
-	if (err)
-		goto err_vpcie;
-
-	rockchip_pcie_enable_interrupts(rockchip);
-
-	err = rockchip_pcie_init_irq_domain(rockchip);
-	if (err < 0)
-		goto err_deinit_port;
-
-	err = of_pci_get_host_bridge_resources(dev->of_node, 0, 0xff,
-					       &res, &io_base);
-	if (err)
-		goto err_remove_irq_domain;
-
-	err = devm_request_pci_bus_resources(dev, &res);
-	if (err)
-		goto err_free_res;
-
-	/* Get the I/O and memory ranges from DT */
-	resource_list_for_each_entry(win, &res) {
-		switch (resource_type(win->res)) {
-		case IORESOURCE_IO:
-			io = win->res;
-			io->name = "I/O";
-			rockchip->io_size = resource_size(io);
-			rockchip->io_bus_addr = io->start - win->offset;
-			err = pci_remap_iospace(io, io_base);
-			if (err) {
-				dev_warn(dev, "error %d: failed to map resource %pR\n",
-					 err, io);
-				continue;
-			}
-			rockchip->io = io;
-			break;
-		case IORESOURCE_MEM:
-			mem = win->res;
-			mem->name = "MEM";
-			rockchip->mem_size = resource_size(mem);
-			rockchip->mem_bus_addr = mem->start - win->offset;
-			break;
-		case IORESOURCE_BUS:
-			rockchip->root_bus_nr = win->res->start;
-			break;
-		default:
-			continue;
-		}
-	}
-
-	err = rockchip_pcie_cfg_atu(rockchip);
-	if (err)
-		goto err_unmap_iospace;
-
-	rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M);
-	if (!rockchip->msg_region) {
-		err = -ENOMEM;
-		goto err_unmap_iospace;
-	}
-
-	list_splice_init(&res, &bridge->windows);
-	bridge->dev.parent = dev;
-	bridge->sysdata = rockchip;
-	bridge->busnr = 0;
-	bridge->ops = &rockchip_pcie_ops;
-	bridge->map_irq = of_irq_parse_and_map_pci;
-	bridge->swizzle_irq = pci_common_swizzle;
-
-	err = pci_scan_root_bus_bridge(bridge);
-	if (err < 0)
-		goto err_unmap_iospace;
-
-	bus = bridge->bus;
-
-	rockchip->root_bus = bus;
-
-	pci_bus_size_bridges(bus);
-	pci_bus_assign_resources(bus);
-	list_for_each_entry(child, &bus->children, node)
-		pcie_bus_configure_settings(child);
-
-	pci_bus_add_devices(bus);
-	return 0;
-
-err_unmap_iospace:
-	pci_unmap_iospace(rockchip->io);
-err_free_res:
-	pci_free_resource_list(&res);
-err_remove_irq_domain:
-	irq_domain_remove(rockchip->irq_domain);
-err_deinit_port:
-	rockchip_pcie_deinit_phys(rockchip);
-err_vpcie:
-	if (!IS_ERR(rockchip->vpcie12v))
-		regulator_disable(rockchip->vpcie12v);
-	if (!IS_ERR(rockchip->vpcie3v3))
-		regulator_disable(rockchip->vpcie3v3);
-	if (!IS_ERR(rockchip->vpcie1v8))
-		regulator_disable(rockchip->vpcie1v8);
-	if (!IS_ERR(rockchip->vpcie0v9))
-		regulator_disable(rockchip->vpcie0v9);
-err_set_vpcie:
-	rockchip_pcie_disable_clocks(rockchip);
-	return err;
-}
-
-static int rockchip_pcie_remove(struct platform_device *pdev)
+void rockchip_pcie_cfg_configuration_accesses(
+		struct rockchip_pcie *rockchip, u32 type)
 {
-	struct device *dev = &pdev->dev;
-	struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
-
-	pci_stop_root_bus(rockchip->root_bus);
-	pci_remove_root_bus(rockchip->root_bus);
-	pci_unmap_iospace(rockchip->io);
-	irq_domain_remove(rockchip->irq_domain);
-
-	rockchip_pcie_deinit_phys(rockchip);
-
-	rockchip_pcie_disable_clocks(rockchip);
+	u32 ob_desc_0;
 
-	if (!IS_ERR(rockchip->vpcie12v))
-		regulator_disable(rockchip->vpcie12v);
-	if (!IS_ERR(rockchip->vpcie3v3))
-		regulator_disable(rockchip->vpcie3v3);
-	if (!IS_ERR(rockchip->vpcie1v8))
-		regulator_disable(rockchip->vpcie1v8);
-	if (!IS_ERR(rockchip->vpcie0v9))
-		regulator_disable(rockchip->vpcie0v9);
+	/* Configuration Accesses for region 0 */
+	rockchip_pcie_write(rockchip, 0x0, PCIE_RC_BAR_CONF);
 
-	return 0;
+	rockchip_pcie_write(rockchip,
+			    (RC_REGION_0_ADDR_TRANS_L + RC_REGION_0_PASS_BITS),
+			    PCIE_CORE_OB_REGION_ADDR0);
+	rockchip_pcie_write(rockchip, RC_REGION_0_ADDR_TRANS_H,
+			    PCIE_CORE_OB_REGION_ADDR1);
+	ob_desc_0 = rockchip_pcie_read(rockchip, PCIE_CORE_OB_REGION_DESC0);
+	ob_desc_0 &= ~(RC_REGION_0_TYPE_MASK);
+	ob_desc_0 |= (type | (0x1 << 23));
+	rockchip_pcie_write(rockchip, ob_desc_0, PCIE_CORE_OB_REGION_DESC0);
+	rockchip_pcie_write(rockchip, 0x0, PCIE_CORE_OB_REGION_DESC1);
 }
-
-static const struct dev_pm_ops rockchip_pcie_pm_ops = {
-	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq,
-				      rockchip_pcie_resume_noirq)
-};
-
-static const struct of_device_id rockchip_pcie_of_match[] = {
-	{ .compatible = "rockchip,rk3399-pcie", },
-	{}
-};
-MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match);
-
-static struct platform_driver rockchip_pcie_driver = {
-	.driver = {
-		.name = "rockchip-pcie",
-		.of_match_table = rockchip_pcie_of_match,
-		.pm = &rockchip_pcie_pm_ops,
-	},
-	.probe = rockchip_pcie_probe,
-	.remove = rockchip_pcie_remove,
-};
-module_platform_driver(rockchip_pcie_driver);
-
-MODULE_AUTHOR("Rockchip Inc");
-MODULE_DESCRIPTION("Rockchip AXI PCIe driver");
-MODULE_LICENSE("GPL v2");
+EXPORT_SYMBOL_GPL(rockchip_pcie_cfg_configuration_accesses);
diff --git a/drivers/pci/host/pcie-rockchip.h b/drivers/pci/host/pcie-rockchip.h
new file mode 100644
index 000000000000..8e87a059ce73
--- /dev/null
+++ b/drivers/pci/host/pcie-rockchip.h
@@ -0,0 +1,338 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Rockchip AXI PCIe controller driver
+ *
+ * Copyright (c) 2018 Rockchip, Inc.
+ *
+ * Author: Shawn Lin <shawn.lin@rock-chips.com>
+ *
+ */
+
+#ifndef _PCIE_ROCKCHIP_H
+#define _PCIE_ROCKCHIP_H
+
+#include <linux/kernel.h>
+#include <linux/pci.h>
+
+/*
+ * The upper 16 bits of PCIE_CLIENT_CONFIG are a write mask for the lower 16
+ * bits.  This allows atomic updates of the register without locking.
+ */
+#define HIWORD_UPDATE(mask, val)	(((mask) << 16) | (val))
+#define HIWORD_UPDATE_BIT(val)		HIWORD_UPDATE(val, val)
+
+#define ENCODE_LANES(x)			((((x) >> 1) & 3) << 4)
+#define MAX_LANE_NUM			4
+#define MAX_REGION_LIMIT		32
+#define MIN_EP_APERTURE			28
+
+#define PCIE_CLIENT_BASE		0x0
+#define PCIE_CLIENT_CONFIG		(PCIE_CLIENT_BASE + 0x00)
+#define   PCIE_CLIENT_CONF_ENABLE	  HIWORD_UPDATE_BIT(0x0001)
+#define   PCIE_CLIENT_CONF_DISABLE       HIWORD_UPDATE(0x0001, 0)
+#define   PCIE_CLIENT_LINK_TRAIN_ENABLE	  HIWORD_UPDATE_BIT(0x0002)
+#define   PCIE_CLIENT_ARI_ENABLE	  HIWORD_UPDATE_BIT(0x0008)
+#define   PCIE_CLIENT_CONF_LANE_NUM(x)	  HIWORD_UPDATE(0x0030, ENCODE_LANES(x))
+#define   PCIE_CLIENT_MODE_RC		  HIWORD_UPDATE_BIT(0x0040)
+#define   PCIE_CLIENT_MODE_EP            HIWORD_UPDATE(0x0040, 0)
+#define   PCIE_CLIENT_GEN_SEL_1		  HIWORD_UPDATE(0x0080, 0)
+#define   PCIE_CLIENT_GEN_SEL_2		  HIWORD_UPDATE_BIT(0x0080)
+#define PCIE_CLIENT_DEBUG_OUT_0		(PCIE_CLIENT_BASE + 0x3c)
+#define   PCIE_CLIENT_DEBUG_LTSSM_MASK		GENMASK(5, 0)
+#define   PCIE_CLIENT_DEBUG_LTSSM_L1		0x18
+#define   PCIE_CLIENT_DEBUG_LTSSM_L2		0x19
+#define PCIE_CLIENT_BASIC_STATUS1	(PCIE_CLIENT_BASE + 0x48)
+#define   PCIE_CLIENT_LINK_STATUS_UP		0x00300000
+#define   PCIE_CLIENT_LINK_STATUS_MASK		0x00300000
+#define PCIE_CLIENT_INT_MASK		(PCIE_CLIENT_BASE + 0x4c)
+#define PCIE_CLIENT_INT_STATUS		(PCIE_CLIENT_BASE + 0x50)
+#define   PCIE_CLIENT_INTR_MASK			GENMASK(8, 5)
+#define   PCIE_CLIENT_INTR_SHIFT		5
+#define   PCIE_CLIENT_INT_LEGACY_DONE		BIT(15)
+#define   PCIE_CLIENT_INT_MSG			BIT(14)
+#define   PCIE_CLIENT_INT_HOT_RST		BIT(13)
+#define   PCIE_CLIENT_INT_DPA			BIT(12)
+#define   PCIE_CLIENT_INT_FATAL_ERR		BIT(11)
+#define   PCIE_CLIENT_INT_NFATAL_ERR		BIT(10)
+#define   PCIE_CLIENT_INT_CORR_ERR		BIT(9)
+#define   PCIE_CLIENT_INT_INTD			BIT(8)
+#define   PCIE_CLIENT_INT_INTC			BIT(7)
+#define   PCIE_CLIENT_INT_INTB			BIT(6)
+#define   PCIE_CLIENT_INT_INTA			BIT(5)
+#define   PCIE_CLIENT_INT_LOCAL			BIT(4)
+#define   PCIE_CLIENT_INT_UDMA			BIT(3)
+#define   PCIE_CLIENT_INT_PHY			BIT(2)
+#define   PCIE_CLIENT_INT_HOT_PLUG		BIT(1)
+#define   PCIE_CLIENT_INT_PWR_STCG		BIT(0)
+
+#define PCIE_CLIENT_INT_LEGACY \
+	(PCIE_CLIENT_INT_INTA | PCIE_CLIENT_INT_INTB | \
+	PCIE_CLIENT_INT_INTC | PCIE_CLIENT_INT_INTD)
+
+#define PCIE_CLIENT_INT_CLI \
+	(PCIE_CLIENT_INT_CORR_ERR | PCIE_CLIENT_INT_NFATAL_ERR | \
+	PCIE_CLIENT_INT_FATAL_ERR | PCIE_CLIENT_INT_DPA | \
+	PCIE_CLIENT_INT_HOT_RST | PCIE_CLIENT_INT_MSG | \
+	PCIE_CLIENT_INT_LEGACY_DONE | PCIE_CLIENT_INT_LEGACY | \
+	PCIE_CLIENT_INT_PHY)
+
+#define PCIE_CORE_CTRL_MGMT_BASE	0x900000
+#define PCIE_CORE_CTRL			(PCIE_CORE_CTRL_MGMT_BASE + 0x000)
+#define   PCIE_CORE_PL_CONF_SPEED_5G		0x00000008
+#define   PCIE_CORE_PL_CONF_SPEED_MASK		0x00000018
+#define   PCIE_CORE_PL_CONF_LANE_MASK		0x00000006
+#define   PCIE_CORE_PL_CONF_LANE_SHIFT		1
+#define PCIE_CORE_CTRL_PLC1		(PCIE_CORE_CTRL_MGMT_BASE + 0x004)
+#define   PCIE_CORE_CTRL_PLC1_FTS_MASK		GENMASK(23, 8)
+#define   PCIE_CORE_CTRL_PLC1_FTS_SHIFT		8
+#define   PCIE_CORE_CTRL_PLC1_FTS_CNT		0xffff
+#define PCIE_CORE_TXCREDIT_CFG1		(PCIE_CORE_CTRL_MGMT_BASE + 0x020)
+#define   PCIE_CORE_TXCREDIT_CFG1_MUI_MASK	0xFFFF0000
+#define   PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT	16
+#define   PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(x) \
+		(((x) >> 3) << PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT)
+#define PCIE_CORE_LANE_MAP             (PCIE_CORE_CTRL_MGMT_BASE + 0x200)
+#define   PCIE_CORE_LANE_MAP_MASK              0x0000000f
+#define   PCIE_CORE_LANE_MAP_REVERSE           BIT(16)
+#define PCIE_CORE_INT_STATUS		(PCIE_CORE_CTRL_MGMT_BASE + 0x20c)
+#define   PCIE_CORE_INT_PRFPE			BIT(0)
+#define   PCIE_CORE_INT_CRFPE			BIT(1)
+#define   PCIE_CORE_INT_RRPE			BIT(2)
+#define   PCIE_CORE_INT_PRFO			BIT(3)
+#define   PCIE_CORE_INT_CRFO			BIT(4)
+#define   PCIE_CORE_INT_RT			BIT(5)
+#define   PCIE_CORE_INT_RTR			BIT(6)
+#define   PCIE_CORE_INT_PE			BIT(7)
+#define   PCIE_CORE_INT_MTR			BIT(8)
+#define   PCIE_CORE_INT_UCR			BIT(9)
+#define   PCIE_CORE_INT_FCE			BIT(10)
+#define   PCIE_CORE_INT_CT			BIT(11)
+#define   PCIE_CORE_INT_UTC			BIT(18)
+#define   PCIE_CORE_INT_MMVC			BIT(19)
+#define PCIE_CORE_CONFIG_VENDOR		(PCIE_CORE_CTRL_MGMT_BASE + 0x44)
+#define PCIE_CORE_INT_MASK		(PCIE_CORE_CTRL_MGMT_BASE + 0x210)
+#define PCIE_CORE_PHY_FUNC_CFG		(PCIE_CORE_CTRL_MGMT_BASE + 0x2c0)
+#define PCIE_RC_BAR_CONF		(PCIE_CORE_CTRL_MGMT_BASE + 0x300)
+#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_DISABLED		0x0
+#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_IO_32BITS		0x1
+#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_32BITS		0x4
+#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_32BITS	0x5
+#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_64BITS		0x6
+#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_64BITS	0x7
+
+#define PCIE_CORE_INT \
+		(PCIE_CORE_INT_PRFPE | PCIE_CORE_INT_CRFPE | \
+		 PCIE_CORE_INT_RRPE | PCIE_CORE_INT_CRFO | \
+		 PCIE_CORE_INT_RT | PCIE_CORE_INT_RTR | \
+		 PCIE_CORE_INT_PE | PCIE_CORE_INT_MTR | \
+		 PCIE_CORE_INT_UCR | PCIE_CORE_INT_FCE | \
+		 PCIE_CORE_INT_CT | PCIE_CORE_INT_UTC | \
+		 PCIE_CORE_INT_MMVC)
+
+#define PCIE_RC_RP_ATS_BASE		0x400000
+#define PCIE_RC_CONFIG_NORMAL_BASE	0x800000
+#define PCIE_RC_CONFIG_BASE		0xa00000
+#define PCIE_RC_CONFIG_RID_CCR		(PCIE_RC_CONFIG_BASE + 0x08)
+#define   PCIE_RC_CONFIG_SCC_SHIFT		16
+#define PCIE_RC_CONFIG_DCR		(PCIE_RC_CONFIG_BASE + 0xc4)
+#define   PCIE_RC_CONFIG_DCR_CSPL_SHIFT		18
+#define   PCIE_RC_CONFIG_DCR_CSPL_LIMIT		0xff
+#define   PCIE_RC_CONFIG_DCR_CPLS_SHIFT		26
+#define PCIE_RC_CONFIG_DCSR		(PCIE_RC_CONFIG_BASE + 0xc8)
+#define   PCIE_RC_CONFIG_DCSR_MPS_MASK		GENMASK(7, 5)
+#define   PCIE_RC_CONFIG_DCSR_MPS_256		(0x1 << 5)
+#define PCIE_RC_CONFIG_LINK_CAP		(PCIE_RC_CONFIG_BASE + 0xcc)
+#define   PCIE_RC_CONFIG_LINK_CAP_L0S		BIT(10)
+#define PCIE_RC_CONFIG_LCS		(PCIE_RC_CONFIG_BASE + 0xd0)
+#define PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 (PCIE_RC_CONFIG_BASE + 0x90c)
+#define PCIE_RC_CONFIG_THP_CAP		(PCIE_RC_CONFIG_BASE + 0x274)
+#define   PCIE_RC_CONFIG_THP_CAP_NEXT_MASK	GENMASK(31, 20)
+
+#define PCIE_CORE_AXI_CONF_BASE		0xc00000
+#define PCIE_CORE_OB_REGION_ADDR0	(PCIE_CORE_AXI_CONF_BASE + 0x0)
+#define   PCIE_CORE_OB_REGION_ADDR0_NUM_BITS	0x3f
+#define   PCIE_CORE_OB_REGION_ADDR0_LO_ADDR	0xffffff00
+#define PCIE_CORE_OB_REGION_ADDR1	(PCIE_CORE_AXI_CONF_BASE + 0x4)
+#define PCIE_CORE_OB_REGION_DESC0	(PCIE_CORE_AXI_CONF_BASE + 0x8)
+#define PCIE_CORE_OB_REGION_DESC1	(PCIE_CORE_AXI_CONF_BASE + 0xc)
+
+#define PCIE_CORE_AXI_INBOUND_BASE	0xc00800
+#define PCIE_RP_IB_ADDR0		(PCIE_CORE_AXI_INBOUND_BASE + 0x0)
+#define   PCIE_CORE_IB_REGION_ADDR0_NUM_BITS	0x3f
+#define   PCIE_CORE_IB_REGION_ADDR0_LO_ADDR	0xffffff00
+#define PCIE_RP_IB_ADDR1		(PCIE_CORE_AXI_INBOUND_BASE + 0x4)
+
+/* Size of one AXI Region (not Region 0) */
+#define AXI_REGION_SIZE				BIT(20)
+/* Size of Region 0, equal to sum of sizes of other regions */
+#define AXI_REGION_0_SIZE			(32 * (0x1 << 20))
+#define OB_REG_SIZE_SHIFT			5
+#define IB_ROOT_PORT_REG_SIZE_SHIFT		3
+#define AXI_WRAPPER_IO_WRITE			0x6
+#define AXI_WRAPPER_MEM_WRITE			0x2
+#define AXI_WRAPPER_TYPE0_CFG			0xa
+#define AXI_WRAPPER_TYPE1_CFG			0xb
+#define AXI_WRAPPER_NOR_MSG			0xc
+
+#define MAX_AXI_IB_ROOTPORT_REGION_NUM		3
+#define MIN_AXI_ADDR_BITS_PASSED		8
+#define PCIE_RC_SEND_PME_OFF			0x11960
+#define ROCKCHIP_VENDOR_ID			0x1d87
+#define PCIE_ECAM_BUS(x)			(((x) & 0xff) << 20)
+#define PCIE_ECAM_DEV(x)			(((x) & 0x1f) << 15)
+#define PCIE_ECAM_FUNC(x)			(((x) & 0x7) << 12)
+#define PCIE_ECAM_REG(x)			(((x) & 0xfff) << 0)
+#define PCIE_ECAM_ADDR(bus, dev, func, reg) \
+	  (PCIE_ECAM_BUS(bus) | PCIE_ECAM_DEV(dev) | \
+	   PCIE_ECAM_FUNC(func) | PCIE_ECAM_REG(reg))
+#define PCIE_LINK_IS_L2(x) \
+	(((x) & PCIE_CLIENT_DEBUG_LTSSM_MASK) == PCIE_CLIENT_DEBUG_LTSSM_L2)
+#define PCIE_LINK_UP(x) \
+	(((x) & PCIE_CLIENT_LINK_STATUS_MASK) == PCIE_CLIENT_LINK_STATUS_UP)
+#define PCIE_LINK_IS_GEN2(x) \
+	(((x) & PCIE_CORE_PL_CONF_SPEED_MASK) == PCIE_CORE_PL_CONF_SPEED_5G)
+
+#define RC_REGION_0_ADDR_TRANS_H		0x00000000
+#define RC_REGION_0_ADDR_TRANS_L		0x00000000
+#define RC_REGION_0_PASS_BITS			(25 - 1)
+#define RC_REGION_0_TYPE_MASK			GENMASK(3, 0)
+#define MAX_AXI_WRAPPER_REGION_NUM		33
+
+#define ROCKCHIP_PCIE_MSG_ROUTING_TO_RC		0x0
+#define ROCKCHIP_PCIE_MSG_ROUTING_VIA_ADDR		0x1
+#define ROCKCHIP_PCIE_MSG_ROUTING_VIA_ID		0x2
+#define ROCKCHIP_PCIE_MSG_ROUTING_BROADCAST		0x3
+#define ROCKCHIP_PCIE_MSG_ROUTING_LOCAL_INTX		0x4
+#define ROCKCHIP_PCIE_MSG_ROUTING_PME_ACK		0x5
+#define ROCKCHIP_PCIE_MSG_CODE_ASSERT_INTA		0x20
+#define ROCKCHIP_PCIE_MSG_CODE_ASSERT_INTB		0x21
+#define ROCKCHIP_PCIE_MSG_CODE_ASSERT_INTC		0x22
+#define ROCKCHIP_PCIE_MSG_CODE_ASSERT_INTD		0x23
+#define ROCKCHIP_PCIE_MSG_CODE_DEASSERT_INTA		0x24
+#define ROCKCHIP_PCIE_MSG_CODE_DEASSERT_INTB		0x25
+#define ROCKCHIP_PCIE_MSG_CODE_DEASSERT_INTC		0x26
+#define ROCKCHIP_PCIE_MSG_CODE_DEASSERT_INTD		0x27
+#define ROCKCHIP_PCIE_MSG_ROUTING_MASK			GENMASK(7, 5)
+#define ROCKCHIP_PCIE_MSG_ROUTING(route) \
+	(((route) << 5) & ROCKCHIP_PCIE_MSG_ROUTING_MASK)
+#define ROCKCHIP_PCIE_MSG_CODE_MASK			GENMASK(15, 8)
+#define ROCKCHIP_PCIE_MSG_CODE(code) \
+	(((code) << 8) & ROCKCHIP_PCIE_MSG_CODE_MASK)
+#define ROCKCHIP_PCIE_MSG_NO_DATA			BIT(16)
+
+#define ROCKCHIP_PCIE_EP_CMD_STATUS			0x4
+#define   ROCKCHIP_PCIE_EP_CMD_STATUS_IS		BIT(19)
+#define ROCKCHIP_PCIE_EP_MSI_CTRL_REG			0x90
+#define   ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_OFFSET		17
+#define   ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_MASK		GENMASK(19, 17)
+#define   ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET		20
+#define   ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK		GENMASK(22, 20)
+#define   ROCKCHIP_PCIE_EP_MSI_CTRL_ME				BIT(16)
+#define   ROCKCHIP_PCIE_EP_MSI_CTRL_MASK_MSI_CAP	BIT(24)
+#define ROCKCHIP_PCIE_EP_DUMMY_IRQ_ADDR				0x1
+#define ROCKCHIP_PCIE_EP_PCI_LEGACY_IRQ_ADDR		0x3
+#define ROCKCHIP_PCIE_EP_FUNC_BASE(fn)	(((fn) << 12) & GENMASK(19, 12))
+#define ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar) \
+	(PCIE_RC_RP_ATS_BASE + 0x0840 + (fn) * 0x0040 + (bar) * 0x0008)
+#define ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar) \
+	(PCIE_RC_RP_ATS_BASE + 0x0844 + (fn) * 0x0040 + (bar) * 0x0008)
+#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(r) \
+	(PCIE_RC_RP_ATS_BASE + 0x0000 + ((r) & 0x1f) * 0x0020)
+#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN_MASK	GENMASK(19, 12)
+#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) \
+	(((devfn) << 12) & \
+		 ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN_MASK)
+#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_BUS_MASK	GENMASK(27, 20)
+#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(bus) \
+		(((bus) << 20) & ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_BUS_MASK)
+#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(r) \
+		(PCIE_RC_RP_ATS_BASE + 0x0004 + ((r) & 0x1f) * 0x0020)
+#define ROCKCHIP_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID	BIT(23)
+#define ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN_MASK	GENMASK(31, 24)
+#define ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN(devfn) \
+		(((devfn) << 24) & ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN_MASK)
+#define ROCKCHIP_PCIE_AT_OB_REGION_DESC0(r) \
+		(PCIE_RC_RP_ATS_BASE + 0x0008 + ((r) & 0x1f) * 0x0020)
+#define ROCKCHIP_PCIE_AT_OB_REGION_DESC1(r)	\
+		(PCIE_RC_RP_ATS_BASE + 0x000c + ((r) & 0x1f) * 0x0020)
+#define ROCKCHIP_PCIE_AT_OB_REGION_CPU_ADDR0(r) \
+		(PCIE_RC_RP_ATS_BASE + 0x0018 + ((r) & 0x1f) * 0x0020)
+#define ROCKCHIP_PCIE_AT_OB_REGION_CPU_ADDR1(r) \
+		(PCIE_RC_RP_ATS_BASE + 0x001c + ((r) & 0x1f) * 0x0020)
+
+#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn) \
+		(PCIE_CORE_CTRL_MGMT_BASE + 0x0240 + (fn) * 0x0008)
+#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn) \
+		(PCIE_CORE_CTRL_MGMT_BASE + 0x0244 + (fn) * 0x0008)
+#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) \
+		(GENMASK(4, 0) << ((b) * 8))
+#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE(b, a) \
+		(((a) << ((b) * 8)) & \
+		 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b))
+#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b) \
+		(GENMASK(7, 5) << ((b) * 8))
+#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, c) \
+		(((c) << ((b) * 8 + 5)) & \
+		 ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b))
+
+struct rockchip_pcie {
+	void	__iomem *reg_base;		/* DT axi-base */
+	void	__iomem *apb_base;		/* DT apb-base */
+	bool    legacy_phy;
+	struct  phy *phys[MAX_LANE_NUM];
+	struct	reset_control *core_rst;
+	struct	reset_control *mgmt_rst;
+	struct	reset_control *mgmt_sticky_rst;
+	struct	reset_control *pipe_rst;
+	struct	reset_control *pm_rst;
+	struct	reset_control *aclk_rst;
+	struct	reset_control *pclk_rst;
+	struct	clk *aclk_pcie;
+	struct	clk *aclk_perf_pcie;
+	struct	clk *hclk_pcie;
+	struct	clk *clk_pcie_pm;
+	struct	regulator *vpcie12v; /* 12V power supply */
+	struct	regulator *vpcie3v3; /* 3.3V power supply */
+	struct	regulator *vpcie1v8; /* 1.8V power supply */
+	struct	regulator *vpcie0v9; /* 0.9V power supply */
+	struct	gpio_desc *ep_gpio;
+	u32	lanes;
+	u8      lanes_map;
+	u8	root_bus_nr;
+	int	link_gen;
+	struct	device *dev;
+	struct	irq_domain *irq_domain;
+	int     offset;
+	struct pci_bus *root_bus;
+	struct resource *io;
+	phys_addr_t io_bus_addr;
+	u32     io_size;
+	void    __iomem *msg_region;
+	u32     mem_size;
+	phys_addr_t msg_bus_addr;
+	phys_addr_t mem_bus_addr;
+	bool is_rc;
+	struct resource *mem_res;
+};
+
+static u32 rockchip_pcie_read(struct rockchip_pcie *rockchip, u32 reg)
+{
+	return readl(rockchip->apb_base + reg);
+}
+
+static void rockchip_pcie_write(struct rockchip_pcie *rockchip, u32 val,
+				u32 reg)
+{
+	writel(val, rockchip->apb_base + reg);
+}
+
+int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip);
+int rockchip_pcie_init_port(struct rockchip_pcie *rockchip);
+int rockchip_pcie_get_phys(struct rockchip_pcie *rockchip);
+void rockchip_pcie_deinit_phys(struct rockchip_pcie *rockchip);
+int rockchip_pcie_enable_clocks(struct rockchip_pcie *rockchip);
+void rockchip_pcie_disable_clocks(void *data);
+void rockchip_pcie_cfg_configuration_accesses(
+		struct rockchip_pcie *rockchip, u32 type);
+
+#endif /* _PCIE_ROCKCHIP_H */
diff --git a/drivers/pci/host/pcie-xilinx-nwl.c b/drivers/pci/host/pcie-xilinx-nwl.c
index 4839ae578711..6a4bbb5b3de0 100644
--- a/drivers/pci/host/pcie-xilinx-nwl.c
+++ b/drivers/pci/host/pcie-xilinx-nwl.c
@@ -21,6 +21,8 @@
 #include <linux/platform_device.h>
 #include <linux/irqchip/chained_irq.h>
 
+#include "../pci.h"
+
 /* Bridge core config registers */
 #define BRCFG_PCIE_RX0			0x00000000
 #define BRCFG_INTERRUPT			0x00000010
@@ -825,7 +827,6 @@ static const struct of_device_id nwl_pcie_of_match[] = {
 static int nwl_pcie_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct device_node *node = dev->of_node;
 	struct nwl_pcie *pcie;
 	struct pci_bus *bus;
 	struct pci_bus *child;
@@ -855,7 +856,8 @@ static int nwl_pcie_probe(struct platform_device *pdev)
 		return err;
 	}
 
-	err = of_pci_get_host_bridge_resources(node, 0, 0xff, &res, &iobase);
+	err = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff, &res,
+						    &iobase);
 	if (err) {
 		dev_err(dev, "Getting bridge resources failed\n");
 		return err;
diff --git a/drivers/pci/host/pcie-xilinx.c b/drivers/pci/host/pcie-xilinx.c
index 0ad188effc09..b110a3a814e3 100644
--- a/drivers/pci/host/pcie-xilinx.c
+++ b/drivers/pci/host/pcie-xilinx.c
@@ -23,6 +23,8 @@
 #include <linux/pci.h>
 #include <linux/platform_device.h>
 
+#include "../pci.h"
+
 /* Register definitions */
 #define XILINX_PCIE_REG_BIR		0x00000130
 #define XILINX_PCIE_REG_IDR		0x00000138
@@ -643,8 +645,8 @@ static int xilinx_pcie_probe(struct platform_device *pdev)
 		return err;
 	}
 
-	err = of_pci_get_host_bridge_resources(dev->of_node, 0, 0xff, &res,
-					       &iobase);
+	err = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff, &res,
+						    &iobase);
 	if (err) {
 		dev_err(dev, "Getting bridge resources failed\n");
 		return err;
diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c
index 930a8fa08bd6..942b64fc7f1f 100644
--- a/drivers/pci/host/vmd.c
+++ b/drivers/pci/host/vmd.c
@@ -24,6 +24,28 @@
 #define VMD_MEMBAR1	2
 #define VMD_MEMBAR2	4
 
+#define PCI_REG_VMCAP		0x40
+#define BUS_RESTRICT_CAP(vmcap)	(vmcap & 0x1)
+#define PCI_REG_VMCONFIG	0x44
+#define BUS_RESTRICT_CFG(vmcfg)	((vmcfg >> 8) & 0x3)
+#define PCI_REG_VMLOCK		0x70
+#define MB2_SHADOW_EN(vmlock)	(vmlock & 0x2)
+
+enum vmd_features {
+	/*
+	 * Device may contain registers which hint the physical location of the
+	 * membars, in order to allow proper address translation during
+	 * resource assignment to enable guest virtualization
+	 */
+	VMD_FEAT_HAS_MEMBAR_SHADOW	= (1 << 0),
+
+	/*
+	 * Device may provide root port configuration information which limits
+	 * bus numbering
+	 */
+	VMD_FEAT_HAS_BUS_RESTRICTIONS	= (1 << 1),
+};
+
 /*
  * Lock for manipulating VMD IRQ lists.
  */
@@ -546,7 +568,7 @@ static int vmd_find_free_domain(void)
 	return domain + 1;
 }
 
-static int vmd_enable_domain(struct vmd_dev *vmd)
+static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)
 {
 	struct pci_sysdata *sd = &vmd->sysdata;
 	struct fwnode_handle *fn;
@@ -554,12 +576,57 @@ static int vmd_enable_domain(struct vmd_dev *vmd)
 	u32 upper_bits;
 	unsigned long flags;
 	LIST_HEAD(resources);
+	resource_size_t offset[2] = {0};
+	resource_size_t membar2_offset = 0x2000, busn_start = 0;
+
+	/*
+	 * Shadow registers may exist in certain VMD device ids which allow
+	 * guests to correctly assign host physical addresses to the root ports
+	 * and child devices. These registers will either return the host value
+	 * or 0, depending on an enable bit in the VMD device.
+	 */
+	if (features & VMD_FEAT_HAS_MEMBAR_SHADOW) {
+		u32 vmlock;
+		int ret;
+
+		membar2_offset = 0x2018;
+		ret = pci_read_config_dword(vmd->dev, PCI_REG_VMLOCK, &vmlock);
+		if (ret || vmlock == ~0)
+			return -ENODEV;
+
+		if (MB2_SHADOW_EN(vmlock)) {
+			void __iomem *membar2;
+
+			membar2 = pci_iomap(vmd->dev, VMD_MEMBAR2, 0);
+			if (!membar2)
+				return -ENOMEM;
+			offset[0] = vmd->dev->resource[VMD_MEMBAR1].start -
+						readq(membar2 + 0x2008);
+			offset[1] = vmd->dev->resource[VMD_MEMBAR2].start -
+						readq(membar2 + 0x2010);
+			pci_iounmap(vmd->dev, membar2);
+		}
+	}
+
+	/*
+	 * Certain VMD devices may have a root port configuration option which
+	 * limits the bus range to between 0-127 or 128-255
+	 */
+	if (features & VMD_FEAT_HAS_BUS_RESTRICTIONS) {
+		u32 vmcap, vmconfig;
+
+		pci_read_config_dword(vmd->dev, PCI_REG_VMCAP, &vmcap);
+		pci_read_config_dword(vmd->dev, PCI_REG_VMCONFIG, &vmconfig);
+		if (BUS_RESTRICT_CAP(vmcap) &&
+		    (BUS_RESTRICT_CFG(vmconfig) == 0x1))
+			busn_start = 128;
+	}
 
 	res = &vmd->dev->resource[VMD_CFGBAR];
 	vmd->resources[0] = (struct resource) {
 		.name  = "VMD CFGBAR",
-		.start = 0,
-		.end   = (resource_size(res) >> 20) - 1,
+		.start = busn_start,
+		.end   = busn_start + (resource_size(res) >> 20) - 1,
 		.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED,
 	};
 
@@ -600,7 +667,7 @@ static int vmd_enable_domain(struct vmd_dev *vmd)
 		flags &= ~IORESOURCE_MEM_64;
 	vmd->resources[2] = (struct resource) {
 		.name  = "VMD MEMBAR2",
-		.start = res->start + 0x2000,
+		.start = res->start + membar2_offset,
 		.end   = res->end,
 		.flags = flags,
 		.parent = res,
@@ -624,10 +691,11 @@ static int vmd_enable_domain(struct vmd_dev *vmd)
 		return -ENODEV;
 
 	pci_add_resource(&resources, &vmd->resources[0]);
-	pci_add_resource(&resources, &vmd->resources[1]);
-	pci_add_resource(&resources, &vmd->resources[2]);
-	vmd->bus = pci_create_root_bus(&vmd->dev->dev, 0, &vmd_ops, sd,
-				       &resources);
+	pci_add_resource_offset(&resources, &vmd->resources[1], offset[0]);
+	pci_add_resource_offset(&resources, &vmd->resources[2], offset[1]);
+
+	vmd->bus = pci_create_root_bus(&vmd->dev->dev, busn_start, &vmd_ops,
+				       sd, &resources);
 	if (!vmd->bus) {
 		pci_free_resource_list(&resources);
 		irq_domain_remove(vmd->irq_domain);
@@ -713,7 +781,7 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
 
 	spin_lock_init(&vmd->cfg_lock);
 	pci_set_drvdata(dev, vmd);
-	err = vmd_enable_domain(vmd);
+	err = vmd_enable_domain(vmd, (unsigned long) id->driver_data);
 	if (err)
 		return err;
 
@@ -778,7 +846,10 @@ static int vmd_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(vmd_dev_pm_ops, vmd_suspend, vmd_resume);
 
 static const struct pci_device_id vmd_ids[] = {
-	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x201d),},
+	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_201D),},
+	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_28C0),
+		.driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW |
+				VMD_FEAT_HAS_BUS_RESTRICTIONS,},
 	{0,}
 };
 MODULE_DEVICE_TABLE(pci, vmd_ids);
diff --git a/drivers/pci/hotplug/Kconfig b/drivers/pci/hotplug/Kconfig
index a8f21d051e0c..e9f78eb390d2 100644
--- a/drivers/pci/hotplug/Kconfig
+++ b/drivers/pci/hotplug/Kconfig
@@ -104,14 +104,11 @@ config HOTPLUG_PCI_CPCI_GENERIC
 	  When in doubt, say N.
 
 config HOTPLUG_PCI_SHPC
-	tristate "SHPC PCI Hotplug driver"
+	bool "SHPC PCI Hotplug driver"
 	help
 	  Say Y here if you have a motherboard with a SHPC PCI Hotplug
 	  controller.
 
-	  To compile this driver as a module, choose M here: the
-	  module will be called shpchp.
-
 	  When in doubt, say N.
 
 config HOTPLUG_PCI_POWERNV
diff --git a/drivers/pci/hotplug/acpi_pcihp.c b/drivers/pci/hotplug/acpi_pcihp.c
index c9816166978e..597d22aeefc1 100644
--- a/drivers/pci/hotplug/acpi_pcihp.c
+++ b/drivers/pci/hotplug/acpi_pcihp.c
@@ -63,22 +63,17 @@ static acpi_status acpi_run_oshp(acpi_handle handle)
 /**
  * acpi_get_hp_hw_control_from_firmware
  * @dev: the pci_dev of the bridge that has a hotplug controller
- * @flags: requested control bits for _OSC
  *
  * Attempt to take hotplug control from firmware.
  */
-int acpi_get_hp_hw_control_from_firmware(struct pci_dev *pdev, u32 flags)
+int acpi_get_hp_hw_control_from_firmware(struct pci_dev *pdev)
 {
+	const struct pci_host_bridge *host;
+	const struct acpi_pci_root *root;
 	acpi_status status;
 	acpi_handle chandle, handle;
 	struct acpi_buffer string = { ACPI_ALLOCATE_BUFFER, NULL };
 
-	flags &= OSC_PCI_SHPC_NATIVE_HP_CONTROL;
-	if (!flags) {
-		err("Invalid flags %u specified!\n", flags);
-		return -EINVAL;
-	}
-
 	/*
 	 * Per PCI firmware specification, we should run the ACPI _OSC
 	 * method to get control of hotplug hardware before using it. If
@@ -88,25 +83,20 @@ int acpi_get_hp_hw_control_from_firmware(struct pci_dev *pdev, u32 flags)
 	 * OSHP within the scope of the hotplug controller and its parents,
 	 * up to the host bridge under which this controller exists.
 	 */
-	handle = acpi_find_root_bridge_handle(pdev);
-	if (handle) {
-		acpi_get_name(handle, ACPI_FULL_PATHNAME, &string);
-		dbg("Trying to get hotplug control for %s\n",
-				(char *)string.pointer);
-		status = acpi_pci_osc_control_set(handle, &flags, flags);
-		if (ACPI_SUCCESS(status))
-			goto got_one;
-		if (status == AE_SUPPORT)
-			goto no_control;
-		kfree(string.pointer);
-		string = (struct acpi_buffer){ ACPI_ALLOCATE_BUFFER, NULL };
-	}
+	host = pci_find_host_bridge(pdev->bus);
+	if (host->native_shpc_hotplug)
+		return 0;
+
+	/* If _OSC exists, we should not evaluate OSHP */
+	root = acpi_pci_find_root(ACPI_HANDLE(&host->dev));
+	if (root->osc_support_set)
+		goto no_control;
 
 	handle = ACPI_HANDLE(&pdev->dev);
 	if (!handle) {
 		/*
 		 * This hotplug controller was not listed in the ACPI name
-		 * space at all. Try to get acpi handle of parent pci bus.
+		 * space at all. Try to get ACPI handle of parent PCI bus.
 		 */
 		struct pci_bus *pbus;
 		for (pbus = pdev->bus; pbus; pbus = pbus->parent) {
@@ -118,8 +108,8 @@ int acpi_get_hp_hw_control_from_firmware(struct pci_dev *pdev, u32 flags)
 
 	while (handle) {
 		acpi_get_name(handle, ACPI_FULL_PATHNAME, &string);
-		dbg("Trying to get hotplug control for %s\n",
-		    (char *)string.pointer);
+		pci_info(pdev, "Requesting control of SHPC hotplug via OSHP (%s)\n",
+			 (char *)string.pointer);
 		status = acpi_run_oshp(handle);
 		if (ACPI_SUCCESS(status))
 			goto got_one;
@@ -131,13 +121,12 @@ int acpi_get_hp_hw_control_from_firmware(struct pci_dev *pdev, u32 flags)
 			break;
 	}
 no_control:
-	dbg("Cannot get control of hotplug hardware for pci %s\n",
-	    pci_name(pdev));
+	pci_info(pdev, "Cannot get control of SHPC hotplug\n");
 	kfree(string.pointer);
 	return -ENODEV;
 got_one:
-	dbg("Gained control for hotplug HW for pci %s (%s)\n",
-	    pci_name(pdev), (char *)string.pointer);
+	pci_info(pdev, "Gained control of SHPC hotplug (%s)\n",
+		 (char *)string.pointer);
 	kfree(string.pointer);
 	return 0;
 }
diff --git a/drivers/pci/hotplug/acpiphp_glue.c b/drivers/pci/hotplug/acpiphp_glue.c
index b45b375c0e6c..318b6a6f6341 100644
--- a/drivers/pci/hotplug/acpiphp_glue.c
+++ b/drivers/pci/hotplug/acpiphp_glue.c
@@ -481,7 +481,7 @@ static void enable_slot(struct acpiphp_slot *slot)
 		if (!dev) {
 			/* Do not set SLOT_ENABLED flag if some funcs
 			   are not added. */
-			slot->flags &= (~SLOT_ENABLED);
+			slot->flags &= ~SLOT_ENABLED;
 			continue;
 		}
 	}
@@ -510,7 +510,7 @@ static void disable_slot(struct acpiphp_slot *slot)
 	list_for_each_entry(func, &slot->funcs, sibling)
 		acpi_bus_trim(func_to_acpi_device(func));
 
-	slot->flags &= (~SLOT_ENABLED);
+	slot->flags &= ~SLOT_ENABLED;
 }
 
 static bool slot_no_hotplug(struct acpiphp_slot *slot)
diff --git a/drivers/pci/hotplug/ibmphp_core.c b/drivers/pci/hotplug/ibmphp_core.c
index b81ca3fa0e84..1869b0411ce0 100644
--- a/drivers/pci/hotplug/ibmphp_core.c
+++ b/drivers/pci/hotplug/ibmphp_core.c
@@ -379,7 +379,7 @@ static int get_adapter_present(struct hotplug_slot *hotplug_slot, u8 *value)
 
 static int get_max_bus_speed(struct slot *slot)
 {
-	int rc;
+	int rc = 0;
 	u8 mode = 0;
 	enum pci_bus_speed speed;
 	struct pci_bus *bus = slot->hotplug_slot->pci_slot->bus;
diff --git a/drivers/pci/hotplug/pciehp.h b/drivers/pci/hotplug/pciehp.h
index 88e917c9120f..5f892065585e 100644
--- a/drivers/pci/hotplug/pciehp.h
+++ b/drivers/pci/hotplug/pciehp.h
@@ -121,7 +121,7 @@ struct controller *pcie_init(struct pcie_device *dev);
 int pcie_init_notification(struct controller *ctrl);
 int pciehp_enable_slot(struct slot *p_slot);
 int pciehp_disable_slot(struct slot *p_slot);
-void pcie_enable_notification(struct controller *ctrl);
+void pcie_reenable_notification(struct controller *ctrl);
 int pciehp_power_on_slot(struct slot *slot);
 void pciehp_power_off_slot(struct slot *slot);
 void pciehp_get_power_status(struct slot *slot, u8 *status);
diff --git a/drivers/pci/hotplug/pciehp_core.c b/drivers/pci/hotplug/pciehp_core.c
index 332b723ff9e6..44a6a63802d5 100644
--- a/drivers/pci/hotplug/pciehp_core.c
+++ b/drivers/pci/hotplug/pciehp_core.c
@@ -283,7 +283,7 @@ static int pciehp_resume(struct pcie_device *dev)
 	ctrl = get_service_data(dev);
 
 	/* reinitialize the chipset's event detection logic */
-	pcie_enable_notification(ctrl);
+	pcie_reenable_notification(ctrl);
 
 	slot = ctrl->slot;
 
diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c
index 18a42f8f5dc5..718b6073afad 100644
--- a/drivers/pci/hotplug/pciehp_hpc.c
+++ b/drivers/pci/hotplug/pciehp_hpc.c
@@ -10,7 +10,6 @@
  * All rights reserved.
  *
  * Send feedback to <greg@kroah.com>,<kristen.c.accardi@intel.com>
- *
  */
 
 #include <linux/kernel.h>
@@ -147,25 +146,22 @@ static void pcie_wait_cmd(struct controller *ctrl)
 	else
 		rc = pcie_poll_cmd(ctrl, jiffies_to_msecs(timeout));
 
-	/*
-	 * Controllers with errata like Intel CF118 don't generate
-	 * completion notifications unless the power/indicator/interlock
-	 * control bits are changed.  On such controllers, we'll emit this
-	 * timeout message when we wait for completion of commands that
-	 * don't change those bits, e.g., commands that merely enable
-	 * interrupts.
-	 */
 	if (!rc)
 		ctrl_info(ctrl, "Timeout on hotplug command %#06x (issued %u msec ago)\n",
 			  ctrl->slot_ctrl,
 			  jiffies_to_msecs(jiffies - ctrl->cmd_started));
 }
 
+#define CC_ERRATUM_MASK		(PCI_EXP_SLTCTL_PCC |	\
+				 PCI_EXP_SLTCTL_PIC |	\
+				 PCI_EXP_SLTCTL_AIC |	\
+				 PCI_EXP_SLTCTL_EIC)
+
 static void pcie_do_write_cmd(struct controller *ctrl, u16 cmd,
 			      u16 mask, bool wait)
 {
 	struct pci_dev *pdev = ctrl_dev(ctrl);
-	u16 slot_ctrl;
+	u16 slot_ctrl_orig, slot_ctrl;
 
 	mutex_lock(&ctrl->ctrl_lock);
 
@@ -180,6 +176,7 @@ static void pcie_do_write_cmd(struct controller *ctrl, u16 cmd,
 		goto out;
 	}
 
+	slot_ctrl_orig = slot_ctrl;
 	slot_ctrl &= ~mask;
 	slot_ctrl |= (cmd & mask);
 	ctrl->cmd_busy = 1;
@@ -188,6 +185,17 @@ static void pcie_do_write_cmd(struct controller *ctrl, u16 cmd,
 	ctrl->cmd_started = jiffies;
 	ctrl->slot_ctrl = slot_ctrl;
 
+	/*
+	 * Controllers with the Intel CF118 and similar errata advertise
+	 * Command Completed support, but they only set Command Completed
+	 * if we change the "Control" bits for power, power indicator,
+	 * attention indicator, or interlock.  If we only change the
+	 * "Enable" bits, they never set the Command Completed bit.
+	 */
+	if (pdev->broken_cmd_compl &&
+	    (slot_ctrl_orig & CC_ERRATUM_MASK) == (slot_ctrl & CC_ERRATUM_MASK))
+		ctrl->cmd_busy = 0;
+
 	/*
 	 * Optionally wait for the hardware to be ready for a new command,
 	 * indicating completion of the above issued command.
@@ -231,25 +239,11 @@ bool pciehp_check_link_active(struct controller *ctrl)
 	return ret;
 }
 
-static void __pcie_wait_link_active(struct controller *ctrl, bool active)
-{
-	int timeout = 1000;
-
-	if (pciehp_check_link_active(ctrl) == active)
-		return;
-	while (timeout > 0) {
-		msleep(10);
-		timeout -= 10;
-		if (pciehp_check_link_active(ctrl) == active)
-			return;
-	}
-	ctrl_dbg(ctrl, "Data Link Layer Link Active not %s in 1000 msec\n",
-			active ? "set" : "cleared");
-}
-
 static void pcie_wait_link_active(struct controller *ctrl)
 {
-	__pcie_wait_link_active(ctrl, true);
+	struct pci_dev *pdev = ctrl_dev(ctrl);
+
+	pcie_wait_for_link(pdev, true);
 }
 
 static bool pci_bus_check_dev(struct pci_bus *bus, int devfn)
@@ -659,7 +653,7 @@ static irqreturn_t pcie_isr(int irq, void *dev_id)
 	return handled;
 }
 
-void pcie_enable_notification(struct controller *ctrl)
+static void pcie_enable_notification(struct controller *ctrl)
 {
 	u16 cmd, mask;
 
@@ -697,6 +691,17 @@ void pcie_enable_notification(struct controller *ctrl)
 		 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, cmd);
 }
 
+void pcie_reenable_notification(struct controller *ctrl)
+{
+	/*
+	 * Clear both Presence and Data Link Layer Changed to make sure
+	 * those events still fire after we have re-enabled them.
+	 */
+	pcie_capability_write_word(ctrl->pcie->port, PCI_EXP_SLTSTA,
+				   PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC);
+	pcie_enable_notification(ctrl);
+}
+
 static void pcie_disable_notification(struct controller *ctrl)
 {
 	u16 mask;
@@ -861,7 +866,7 @@ struct controller *pcie_init(struct pcie_device *dev)
 		PCI_EXP_SLTSTA_MRLSC | PCI_EXP_SLTSTA_CC |
 		PCI_EXP_SLTSTA_DLLSC);
 
-	ctrl_info(ctrl, "Slot #%d AttnBtn%c PwrCtrl%c MRL%c AttnInd%c PwrInd%c HotPlug%c Surprise%c Interlock%c NoCompl%c LLActRep%c\n",
+	ctrl_info(ctrl, "Slot #%d AttnBtn%c PwrCtrl%c MRL%c AttnInd%c PwrInd%c HotPlug%c Surprise%c Interlock%c NoCompl%c LLActRep%c%s\n",
 		(slot_cap & PCI_EXP_SLTCAP_PSN) >> 19,
 		FLAG(slot_cap, PCI_EXP_SLTCAP_ABP),
 		FLAG(slot_cap, PCI_EXP_SLTCAP_PCP),
@@ -872,7 +877,8 @@ struct controller *pcie_init(struct pcie_device *dev)
 		FLAG(slot_cap, PCI_EXP_SLTCAP_HPS),
 		FLAG(slot_cap, PCI_EXP_SLTCAP_EIP),
 		FLAG(slot_cap, PCI_EXP_SLTCAP_NCCS),
-		FLAG(link_cap, PCI_EXP_LNKCAP_DLLLARC));
+		FLAG(link_cap, PCI_EXP_LNKCAP_DLLLARC),
+		pdev->broken_cmd_compl ? " (with Cmd Compl erratum)" : "");
 
 	if (pcie_init_slot(ctrl))
 		goto abort_ctrl;
@@ -891,3 +897,21 @@ void pciehp_release_ctrl(struct controller *ctrl)
 	pcie_cleanup_slot(ctrl);
 	kfree(ctrl);
 }
+
+static void quirk_cmd_compl(struct pci_dev *pdev)
+{
+	u32 slot_cap;
+
+	if (pci_is_pcie(pdev)) {
+		pcie_capability_read_dword(pdev, PCI_EXP_SLTCAP, &slot_cap);
+		if (slot_cap & PCI_EXP_SLTCAP_HPC &&
+		    !(slot_cap & PCI_EXP_SLTCAP_NCCS))
+			pdev->broken_cmd_compl = 1;
+	}
+}
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, PCI_ANY_ID,
+			      PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_QCOM, 0x0400,
+			      PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_QCOM, 0x0401,
+			      PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
diff --git a/drivers/pci/hotplug/pnv_php.c b/drivers/pci/hotplug/pnv_php.c
index d44100687dfe..6c2e8d7307c6 100644
--- a/drivers/pci/hotplug/pnv_php.c
+++ b/drivers/pci/hotplug/pnv_php.c
@@ -220,12 +220,16 @@ static int pnv_php_populate_changeset(struct of_changeset *ocs,
 
 	for_each_child_of_node(dn, child) {
 		ret = of_changeset_attach_node(ocs, child);
-		if (ret)
+		if (ret) {
+			of_node_put(child);
 			break;
+		}
 
 		ret = pnv_php_populate_changeset(ocs, child);
-		if (ret)
+		if (ret) {
+			of_node_put(child);
 			break;
+		}
 	}
 
 	return ret;
diff --git a/drivers/pci/hotplug/shpchp.h b/drivers/pci/hotplug/shpchp.h
index c55730b61c9a..9675ab757323 100644
--- a/drivers/pci/hotplug/shpchp.h
+++ b/drivers/pci/hotplug/shpchp.h
@@ -173,17 +173,6 @@ static inline const char *slot_name(struct slot *slot)
 	return hotplug_slot_name(slot->hotplug_slot);
 }
 
-#ifdef CONFIG_ACPI
-#include <linux/pci-acpi.h>
-static inline int get_hp_hw_control_from_firmware(struct pci_dev *dev)
-{
-	u32 flags = OSC_PCI_SHPC_NATIVE_HP_CONTROL;
-	return acpi_get_hp_hw_control_from_firmware(dev, flags);
-}
-#else
-#define get_hp_hw_control_from_firmware(dev) (0)
-#endif
-
 struct ctrl_reg {
 	volatile u32 base_offset;
 	volatile u32 slot_avail1;
diff --git a/drivers/pci/hotplug/shpchp_core.c b/drivers/pci/hotplug/shpchp_core.c
index 1f0f96908b5a..47decc9b3bb3 100644
--- a/drivers/pci/hotplug/shpchp_core.c
+++ b/drivers/pci/hotplug/shpchp_core.c
@@ -277,7 +277,7 @@ static int is_shpc_capable(struct pci_dev *dev)
 		return 1;
 	if (!pci_find_capability(dev, PCI_CAP_ID_SHPC))
 		return 0;
-	if (get_hp_hw_control_from_firmware(dev))
+	if (acpi_get_hp_hw_control_from_firmware(dev))
 		return 0;
 	return 1;
 }
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 8adf4a64f291..d0d73dbbd5ca 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -469,6 +469,7 @@ static int sriov_init(struct pci_dev *dev, int pos)
 	iov->nres = nres;
 	iov->ctrl = ctrl;
 	iov->total_VFs = total;
+	iov->driver_max_VFs = total;
 	pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_device);
 	iov->pgsz = pgsz;
 	iov->self = dev;
@@ -827,9 +828,42 @@ int pci_sriov_get_totalvfs(struct pci_dev *dev)
 	if (!dev->is_physfn)
 		return 0;
 
-	if (dev->sriov->driver_max_VFs)
-		return dev->sriov->driver_max_VFs;
-
-	return dev->sriov->total_VFs;
+	return dev->sriov->driver_max_VFs;
 }
 EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs);
+
+/**
+ * pci_sriov_configure_simple - helper to configure SR-IOV
+ * @dev: the PCI device
+ * @nr_virtfn: number of virtual functions to enable, 0 to disable
+ *
+ * Enable or disable SR-IOV for devices that don't require any PF setup
+ * before enabling SR-IOV.  Return value is negative on error, or number of
+ * VFs allocated on success.
+ */
+int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn)
+{
+	int rc;
+
+	might_sleep();
+
+	if (!dev->is_physfn)
+		return -ENODEV;
+
+	if (pci_vfs_assigned(dev)) {
+		pci_warn(dev, "Cannot modify SR-IOV while VFs are assigned\n");
+		return -EPERM;
+	}
+
+	if (nr_virtfn == 0) {
+		sriov_disable(dev);
+		return 0;
+	}
+
+	rc = sriov_enable(dev, nr_virtfn);
+	if (rc < 0)
+		return rc;
+
+	return nr_virtfn;
+}
+EXPORT_SYMBOL_GPL(pci_sriov_configure_simple);
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 30250631efe7..f45b74fcc059 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -1434,6 +1434,9 @@ struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
 {
 	struct irq_domain *domain;
 
+	if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE))
+		info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;
+
 	if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
 		pci_msi_domain_update_dom_ops(info);
 	if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
diff --git a/drivers/pci/of.c b/drivers/pci/of.c
index a28355c273ae..d088c9147f10 100644
--- a/drivers/pci/of.c
+++ b/drivers/pci/of.c
@@ -244,8 +244,9 @@ EXPORT_SYMBOL_GPL(of_pci_check_probe_only);
 
 #if defined(CONFIG_OF_ADDRESS)
 /**
- * of_pci_get_host_bridge_resources - Parse PCI host bridge resources from DT
- * @dev: device node of the host bridge having the range property
+ * devm_of_pci_get_host_bridge_resources() - Resource-managed parsing of PCI
+ *                                           host bridge resources from DT
+ * @dev: host bridge device
  * @busno: bus number associated with the bridge root bus
  * @bus_max: maximum number of buses for this bridge
  * @resources: list where the range of resources will be added after DT parsing
@@ -253,8 +254,6 @@ EXPORT_SYMBOL_GPL(of_pci_check_probe_only);
  * address for the start of the I/O range. Can be NULL if the caller doesn't
  * expect I/O ranges to be present in the device tree.
  *
- * It is the caller's job to free the @resources list.
- *
  * This function will parse the "ranges" property of a PCI host bridge device
  * node and setup the resource mapping based on its content. It is expected
  * that the property conforms with the Power ePAPR document.
@@ -262,11 +261,11 @@ EXPORT_SYMBOL_GPL(of_pci_check_probe_only);
  * It returns zero if the range parsing has been successful or a standard error
  * value if it failed.
  */
-int of_pci_get_host_bridge_resources(struct device_node *dev,
+int devm_of_pci_get_host_bridge_resources(struct device *dev,
 			unsigned char busno, unsigned char bus_max,
 			struct list_head *resources, resource_size_t *io_base)
 {
-	struct resource_entry *window;
+	struct device_node *dev_node = dev->of_node;
 	struct resource *res;
 	struct resource *bus_range;
 	struct of_pci_range range;
@@ -277,19 +276,19 @@ int of_pci_get_host_bridge_resources(struct device_node *dev,
 	if (io_base)
 		*io_base = (resource_size_t)OF_BAD_ADDR;
 
-	bus_range = kzalloc(sizeof(*bus_range), GFP_KERNEL);
+	bus_range = devm_kzalloc(dev, sizeof(*bus_range), GFP_KERNEL);
 	if (!bus_range)
 		return -ENOMEM;
 
-	pr_info("host bridge %pOF ranges:\n", dev);
+	dev_info(dev, "host bridge %pOF ranges:\n", dev_node);
 
-	err = of_pci_parse_bus_range(dev, bus_range);
+	err = of_pci_parse_bus_range(dev_node, bus_range);
 	if (err) {
 		bus_range->start = busno;
 		bus_range->end = bus_max;
 		bus_range->flags = IORESOURCE_BUS;
-		pr_info("  No bus range found for %pOF, using %pR\n",
-			dev, bus_range);
+		dev_info(dev, "  No bus range found for %pOF, using %pR\n",
+			 dev_node, bus_range);
 	} else {
 		if (bus_range->end > bus_range->start + bus_max)
 			bus_range->end = bus_range->start + bus_max;
@@ -297,11 +296,11 @@ int of_pci_get_host_bridge_resources(struct device_node *dev,
 	pci_add_resource(resources, bus_range);
 
 	/* Check for ranges property */
-	err = of_pci_range_parser_init(&parser, dev);
+	err = of_pci_range_parser_init(&parser, dev_node);
 	if (err)
-		goto parse_failed;
+		goto failed;
 
-	pr_debug("Parsing ranges property...\n");
+	dev_dbg(dev, "Parsing ranges property...\n");
 	for_each_of_pci_range(&parser, &range) {
 		/* Read next ranges element */
 		if ((range.flags & IORESOURCE_TYPE_BITS) == IORESOURCE_IO)
@@ -310,9 +309,9 @@ int of_pci_get_host_bridge_resources(struct device_node *dev,
 			snprintf(range_type, 4, "MEM");
 		else
 			snprintf(range_type, 4, "err");
-		pr_info("  %s %#010llx..%#010llx -> %#010llx\n", range_type,
-			range.cpu_addr, range.cpu_addr + range.size - 1,
-			range.pci_addr);
+		dev_info(dev, "  %s %#010llx..%#010llx -> %#010llx\n",
+			 range_type, range.cpu_addr,
+			 range.cpu_addr + range.size - 1, range.pci_addr);
 
 		/*
 		 * If we failed translation or got a zero-sized region
@@ -321,28 +320,28 @@ int of_pci_get_host_bridge_resources(struct device_node *dev,
 		if (range.cpu_addr == OF_BAD_ADDR || range.size == 0)
 			continue;
 
-		res = kzalloc(sizeof(struct resource), GFP_KERNEL);
+		res = devm_kzalloc(dev, sizeof(struct resource), GFP_KERNEL);
 		if (!res) {
 			err = -ENOMEM;
-			goto parse_failed;
+			goto failed;
 		}
 
-		err = of_pci_range_to_resource(&range, dev, res);
+		err = of_pci_range_to_resource(&range, dev_node, res);
 		if (err) {
-			kfree(res);
+			devm_kfree(dev, res);
 			continue;
 		}
 
 		if (resource_type(res) == IORESOURCE_IO) {
 			if (!io_base) {
-				pr_err("I/O range found for %pOF. Please provide an io_base pointer to save CPU base address\n",
-					dev);
+				dev_err(dev, "I/O range found for %pOF. Please provide an io_base pointer to save CPU base address\n",
+					dev_node);
 				err = -EINVAL;
-				goto conversion_failed;
+				goto failed;
 			}
 			if (*io_base != (resource_size_t)OF_BAD_ADDR)
-				pr_warn("More than one I/O resource converted for %pOF. CPU base address for old range lost!\n",
-					dev);
+				dev_warn(dev, "More than one I/O resource converted for %pOF. CPU base address for old range lost!\n",
+					 dev_node);
 			*io_base = range.cpu_addr;
 		}
 
@@ -351,15 +350,11 @@ int of_pci_get_host_bridge_resources(struct device_node *dev,
 
 	return 0;
 
-conversion_failed:
-	kfree(res);
-parse_failed:
-	resource_list_for_each_entry(window, resources)
-		kfree(window->res);
+failed:
 	pci_free_resource_list(resources);
 	return err;
 }
-EXPORT_SYMBOL_GPL(of_pci_get_host_bridge_resources);
+EXPORT_SYMBOL_GPL(devm_of_pci_get_host_bridge_resources);
 #endif /* CONFIG_OF_ADDRESS */
 
 /**
@@ -599,12 +594,12 @@ int pci_parse_request_of_pci_ranges(struct device *dev,
 				    struct resource **bus_range)
 {
 	int err, res_valid = 0;
-	struct device_node *np = dev->of_node;
 	resource_size_t iobase;
 	struct resource_entry *win, *tmp;
 
 	INIT_LIST_HEAD(resources);
-	err = of_pci_get_host_bridge_resources(np, 0, 0xff, resources, &iobase);
+	err = devm_of_pci_get_host_bridge_resources(dev, 0, 0xff, resources,
+						    &iobase);
 	if (err)
 		return err;
 
diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c
index 1abdbf267c19..52b8434d4d6e 100644
--- a/drivers/pci/pci-acpi.c
+++ b/drivers/pci/pci-acpi.c
@@ -370,26 +370,28 @@ EXPORT_SYMBOL_GPL(pci_get_hp_params);
 
 /**
  * pciehp_is_native - Check whether a hotplug port is handled by the OS
- * @pdev: Hotplug port to check
+ * @bridge: Hotplug port to check
  *
- * Walk up from @pdev to the host bridge, obtain its cached _OSC Control Field
- * and return the value of the "PCI Express Native Hot Plug control" bit.
- * On failure to obtain the _OSC Control Field return %false.
+ * Returns true if the given @bridge is handled by the native PCIe hotplug
+ * driver.
  */
-bool pciehp_is_native(struct pci_dev *pdev)
+bool pciehp_is_native(struct pci_dev *bridge)
 {
-	struct acpi_pci_root *root;
-	acpi_handle handle;
+	const struct pci_host_bridge *host;
+	u32 slot_cap;
 
-	handle = acpi_find_root_bridge_handle(pdev);
-	if (!handle)
+	if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE))
 		return false;
 
-	root = acpi_pci_find_root(handle);
-	if (!root)
+	pcie_capability_read_dword(bridge, PCI_EXP_SLTCAP, &slot_cap);
+	if (!(slot_cap & PCI_EXP_SLTCAP_HPC))
 		return false;
 
-	return root->osc_control_set & OSC_PCI_EXPRESS_NATIVE_HP_CONTROL;
+	if (pcie_ports_native)
+		return true;
+
+	host = pci_find_host_bridge(bridge->bus);
+	return host->native_pcie_hotplug;
 }
 
 /**
diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c
index b9a131137e64..c125d53033c6 100644
--- a/drivers/pci/pci-driver.c
+++ b/drivers/pci/pci-driver.c
@@ -16,6 +16,8 @@
 #include <linux/pm_runtime.h>
 #include <linux/suspend.h>
 #include <linux/kexec.h>
+#include <linux/of_device.h>
+#include <linux/acpi.h>
 #include "pci.h"
 #include "pcie/portdrv.h"
 
@@ -753,10 +755,11 @@ static int pci_pm_suspend(struct device *dev)
 	 * better to resume the device from runtime suspend here.
 	 */
 	if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
-	    !pci_dev_keep_suspended(pci_dev))
+	    !pci_dev_keep_suspended(pci_dev)) {
 		pm_runtime_resume(dev);
+		pci_dev->state_saved = false;
+	}
 
-	pci_dev->state_saved = false;
 	if (pm->suspend) {
 		pci_power_t prev = pci_dev->current_state;
 		int error;
@@ -1536,7 +1539,7 @@ static int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
 	return 0;
 }
 
-#if defined(CONFIG_PCIEAER) || defined(CONFIG_EEH)
+#if defined(CONFIG_PCIEPORTBUS) || defined(CONFIG_EEH)
 /**
  * pci_uevent_ers - emit a uevent during recovery path of PCI device
  * @pdev: PCI device undergoing error recovery
@@ -1577,6 +1580,35 @@ static int pci_bus_num_vf(struct device *dev)
 	return pci_num_vf(to_pci_dev(dev));
 }
 
+/**
+ * pci_dma_configure - Setup DMA configuration
+ * @dev: ptr to dev structure
+ *
+ * Function to update PCI devices's DMA configuration using the same
+ * info from the OF node or ACPI node of host bridge's parent (if any).
+ */
+static int pci_dma_configure(struct device *dev)
+{
+	struct device *bridge;
+	int ret = 0;
+
+	bridge = pci_get_host_bridge_device(to_pci_dev(dev));
+
+	if (IS_ENABLED(CONFIG_OF) && bridge->parent &&
+	    bridge->parent->of_node) {
+		ret = of_dma_configure(dev, bridge->parent->of_node, true);
+	} else if (has_acpi_companion(bridge)) {
+		struct acpi_device *adev = to_acpi_device_node(bridge->fwnode);
+		enum dev_dma_attr attr = acpi_get_dma_attr(adev);
+
+		if (attr != DEV_DMA_NOT_SUPPORTED)
+			ret = acpi_dma_configure(dev, attr);
+	}
+
+	pci_put_host_bridge_device(bridge);
+	return ret;
+}
+
 struct bus_type pci_bus_type = {
 	.name		= "pci",
 	.match		= pci_bus_match,
@@ -1589,7 +1621,7 @@ struct bus_type pci_bus_type = {
 	.drv_groups	= pci_drv_groups,
 	.pm		= PCI_PM_OPS_PTR,
 	.num_vf		= pci_bus_num_vf,
-	.force_dma	= true,
+	.dma_configure	= pci_dma_configure,
 };
 EXPORT_SYMBOL(pci_bus_type);
 
diff --git a/drivers/pci/pci-pf-stub.c b/drivers/pci/pci-pf-stub.c
new file mode 100644
index 000000000000..9795649fc6f9
--- /dev/null
+++ b/drivers/pci/pci-pf-stub.c
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0
+/* pci-pf-stub - simple stub driver for PCI SR-IOV PF device
+ *
+ * This driver is meant to act as a "whitelist" for devices that provde
+ * SR-IOV functionality while at the same time not actually needing a
+ * driver of their own.
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+
+/**
+ * pci_pf_stub_whitelist - White list of devices to bind pci-pf-stub onto
+ *
+ * This table provides the list of IDs this driver is supposed to bind
+ * onto.  You could think of this as a list of "quirked" devices where we
+ * are adding support for SR-IOV here since there are no other drivers
+ * that they would be running under.
+ */
+static const struct pci_device_id pci_pf_stub_whitelist[] = {
+	{ PCI_VDEVICE(AMAZON, 0x0053) },
+	/* required last entry */
+	{ 0 }
+};
+MODULE_DEVICE_TABLE(pci, pci_pf_stub_whitelist);
+
+static int pci_pf_stub_probe(struct pci_dev *dev,
+			     const struct pci_device_id *id)
+{
+	pci_info(dev, "claimed by pci-pf-stub\n");
+	return 0;
+}
+
+static struct pci_driver pf_stub_driver = {
+	.name			= "pci-pf-stub",
+	.id_table		= pci_pf_stub_whitelist,
+	.probe			= pci_pf_stub_probe,
+	.sriov_configure	= pci_sriov_configure_simple,
+};
+
+static int __init pci_pf_stub_init(void)
+{
+	return pci_register_driver(&pf_stub_driver);
+}
+
+static void __exit pci_pf_stub_exit(void)
+{
+	pci_unregister_driver(&pf_stub_driver);
+}
+
+module_init(pci_pf_stub_init);
+module_exit(pci_pf_stub_exit);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index 366d93af051d..788a200fb2dc 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -288,13 +288,16 @@ static ssize_t enable_store(struct device *dev, struct device_attribute *attr,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EPERM;
 
-	if (!val) {
-		if (pci_is_enabled(pdev))
-			pci_disable_device(pdev);
-		else
-			result = -EIO;
-	} else
+	device_lock(dev);
+	if (dev->driver)
+		result = -EBUSY;
+	else if (val)
 		result = pci_enable_device(pdev);
+	else if (pci_is_enabled(pdev))
+		pci_disable_device(pdev);
+	else
+		result = -EIO;
+	device_unlock(dev);
 
 	return result < 0 ? result : count;
 }
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index dbfe7c4f3776..97acba712e4e 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -112,6 +112,14 @@ unsigned int pcibios_max_latency = 255;
 /* If set, the PCIe ARI capability will not be used. */
 static bool pcie_ari_disabled;
 
+/* If set, the PCIe ATS capability will not be used. */
+static bool pcie_ats_disabled;
+
+bool pci_ats_disabled(void)
+{
+	return pcie_ats_disabled;
+}
+
 /* Disable bridge_d3 for all PCIe ports */
 static bool pci_bridge_d3_disable;
 /* Force bridge_d3 for all PCIe ports */
@@ -2025,8 +2033,7 @@ static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup)
 
 	if (platform_pci_power_manageable(dev)) {
 		/*
-		 * Call the platform to choose the target state of the device
-		 * and enable wake-up from this state if supported.
+		 * Call the platform to find the target state for the device.
 		 */
 		pci_power_t state = platform_pci_choose_state(dev);
 
@@ -2059,8 +2066,7 @@ static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup)
 	if (wakeup) {
 		/*
 		 * Find the deepest state from which the device can generate
-		 * wake-up events, make it the target state and enable device
-		 * to generate PME#.
+		 * PME#.
 		 */
 		if (dev->pme_support) {
 			while (target_state
@@ -4155,6 +4161,35 @@ static int pci_pm_reset(struct pci_dev *dev, int probe)
 
 	return pci_dev_wait(dev, "PM D3->D0", PCIE_RESET_READY_POLL_MS);
 }
+/**
+ * pcie_wait_for_link - Wait until link is active or inactive
+ * @pdev: Bridge device
+ * @active: waiting for active or inactive?
+ *
+ * Use this to wait till link becomes active or inactive.
+ */
+bool pcie_wait_for_link(struct pci_dev *pdev, bool active)
+{
+	int timeout = 1000;
+	bool ret;
+	u16 lnk_status;
+
+	for (;;) {
+		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
+		ret = !!(lnk_status & PCI_EXP_LNKSTA_DLLLA);
+		if (ret == active)
+			return true;
+		if (timeout <= 0)
+			break;
+		msleep(10);
+		timeout -= 10;
+	}
+
+	pci_info(pdev, "Data Link Layer Link Active not %s in 1000 msec\n",
+		 active ? "set" : "cleared");
+
+	return false;
+}
 
 void pci_reset_secondary_bus(struct pci_dev *dev)
 {
@@ -5086,49 +5121,6 @@ int pcie_set_mps(struct pci_dev *dev, int mps)
 }
 EXPORT_SYMBOL(pcie_set_mps);
 
-/**
- * pcie_get_minimum_link - determine minimum link settings of a PCI device
- * @dev: PCI device to query
- * @speed: storage for minimum speed
- * @width: storage for minimum width
- *
- * This function will walk up the PCI device chain and determine the minimum
- * link width and speed of the device.
- */
-int pcie_get_minimum_link(struct pci_dev *dev, enum pci_bus_speed *speed,
-			  enum pcie_link_width *width)
-{
-	int ret;
-
-	*speed = PCI_SPEED_UNKNOWN;
-	*width = PCIE_LNK_WIDTH_UNKNOWN;
-
-	while (dev) {
-		u16 lnksta;
-		enum pci_bus_speed next_speed;
-		enum pcie_link_width next_width;
-
-		ret = pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta);
-		if (ret)
-			return ret;
-
-		next_speed = pcie_link_speed[lnksta & PCI_EXP_LNKSTA_CLS];
-		next_width = (lnksta & PCI_EXP_LNKSTA_NLW) >>
-			PCI_EXP_LNKSTA_NLW_SHIFT;
-
-		if (next_speed < *speed)
-			*speed = next_speed;
-
-		if (next_width < *width)
-			*width = next_width;
-
-		dev = dev->bus->self;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(pcie_get_minimum_link);
-
 /**
  * pcie_bandwidth_available - determine minimum link settings of a PCIe
  *			      device and its bandwidth limitation
@@ -5719,15 +5711,14 @@ static void pci_no_domains(void)
 #endif
 }
 
-#ifdef CONFIG_PCI_DOMAINS
+#ifdef CONFIG_PCI_DOMAINS_GENERIC
 static atomic_t __domain_nr = ATOMIC_INIT(-1);
 
-int pci_get_new_domain_nr(void)
+static int pci_get_new_domain_nr(void)
 {
 	return atomic_inc_return(&__domain_nr);
 }
 
-#ifdef CONFIG_PCI_DOMAINS_GENERIC
 static int of_pci_bus_find_domain_nr(struct device *parent)
 {
 	static int use_dt_domains = -1;
@@ -5782,7 +5773,6 @@ int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent)
 			       acpi_pci_bus_find_domain_nr(bus);
 }
 #endif
-#endif
 
 /**
  * pci_ext_cfg_avail - can we access extended PCI config space?
@@ -5810,6 +5800,9 @@ static int __init pci_setup(char *str)
 		if (*str && (str = pcibios_setup(str)) && *str) {
 			if (!strcmp(str, "nomsi")) {
 				pci_no_msi();
+			} else if (!strncmp(str, "noats", 5)) {
+				pr_info("PCIe: ATS is disabled\n");
+				pcie_ats_disabled = true;
 			} else if (!strcmp(str, "noaer")) {
 				pci_no_aer();
 			} else if (!strncmp(str, "realloc=", 8)) {
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 023f7cf25bff..c358e7a07f3f 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -353,6 +353,11 @@ static inline resource_size_t pci_resource_alignment(struct pci_dev *dev,
 
 void pci_enable_acs(struct pci_dev *dev);
 
+/* PCI error reporting and recovery */
+void pcie_do_fatal_recovery(struct pci_dev *dev, u32 service);
+void pcie_do_nonfatal_recovery(struct pci_dev *dev);
+
+bool pcie_wait_for_link(struct pci_dev *pdev, bool active);
 #ifdef CONFIG_PCIEASPM
 void pcie_aspm_init_link_state(struct pci_dev *pdev);
 void pcie_aspm_exit_link_state(struct pci_dev *pdev);
@@ -407,4 +412,44 @@ static inline u64 pci_rebar_size_to_bytes(int size)
 	return 1ULL << (size + 20);
 }
 
+struct device_node;
+
+#ifdef CONFIG_OF
+int of_pci_parse_bus_range(struct device_node *node, struct resource *res);
+int of_get_pci_domain_nr(struct device_node *node);
+int of_pci_get_max_link_speed(struct device_node *node);
+
+#else
+static inline int
+of_pci_parse_bus_range(struct device_node *node, struct resource *res)
+{
+	return -EINVAL;
+}
+
+static inline int
+of_get_pci_domain_nr(struct device_node *node)
+{
+	return -1;
+}
+
+static inline int
+of_pci_get_max_link_speed(struct device_node *node)
+{
+	return -EINVAL;
+}
+#endif /* CONFIG_OF */
+
+#if defined(CONFIG_OF_ADDRESS)
+int devm_of_pci_get_host_bridge_resources(struct device *dev,
+			unsigned char busno, unsigned char bus_max,
+			struct list_head *resources, resource_size_t *io_base);
+#else
+static inline int devm_of_pci_get_host_bridge_resources(struct device *dev,
+			unsigned char busno, unsigned char bus_max,
+			struct list_head *resources, resource_size_t *io_base)
+{
+	return -EINVAL;
+}
+#endif
+
 #endif /* DRIVERS_PCI_H */
diff --git a/drivers/pci/pcie/Makefile b/drivers/pci/pcie/Makefile
index 800e1d404a45..03f4e0b3a140 100644
--- a/drivers/pci/pcie/Makefile
+++ b/drivers/pci/pcie/Makefile
@@ -2,7 +2,7 @@
 #
 # Makefile for PCI Express features and port driver
 
-pcieportdrv-y			:= portdrv_core.o portdrv_pci.o
+pcieportdrv-y			:= portdrv_core.o portdrv_pci.o err.o
 
 obj-$(CONFIG_PCIEPORTBUS)	+= pcieportdrv.o
 
diff --git a/drivers/pci/pcie/aer/aerdrv.c b/drivers/pci/pcie/aer/aerdrv.c
index 779b3879b1b5..377e576e4f41 100644
--- a/drivers/pci/pcie/aer/aerdrv.c
+++ b/drivers/pci/pcie/aer/aerdrv.c
@@ -353,10 +353,7 @@ static void aer_error_resume(struct pci_dev *dev)
 	pos = dev->aer_cap;
 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
-	if (dev->error_state == pci_channel_io_normal)
-		status &= ~mask; /* Clear corresponding nonfatal bits */
-	else
-		status &= mask; /* Clear corresponding fatal bits */
+	status &= ~mask; /* Clear corresponding nonfatal bits */
 	pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
 }
 
diff --git a/drivers/pci/pcie/aer/aerdrv.h b/drivers/pci/pcie/aer/aerdrv.h
index 08b4584f62fe..b4c950683cc7 100644
--- a/drivers/pci/pcie/aer/aerdrv.h
+++ b/drivers/pci/pcie/aer/aerdrv.h
@@ -76,36 +76,6 @@ struct aer_rpc {
 					 */
 };
 
-struct aer_broadcast_data {
-	enum pci_channel_state state;
-	enum pci_ers_result result;
-};
-
-static inline pci_ers_result_t merge_result(enum pci_ers_result orig,
-		enum pci_ers_result new)
-{
-	if (new == PCI_ERS_RESULT_NO_AER_DRIVER)
-		return PCI_ERS_RESULT_NO_AER_DRIVER;
-
-	if (new == PCI_ERS_RESULT_NONE)
-		return orig;
-
-	switch (orig) {
-	case PCI_ERS_RESULT_CAN_RECOVER:
-	case PCI_ERS_RESULT_RECOVERED:
-		orig = new;
-		break;
-	case PCI_ERS_RESULT_DISCONNECT:
-		if (new == PCI_ERS_RESULT_NEED_RESET)
-			orig = PCI_ERS_RESULT_NEED_RESET;
-		break;
-	default:
-		break;
-	}
-
-	return orig;
-}
-
 extern struct bus_type pcie_port_bus_type;
 void aer_isr(struct work_struct *work);
 void aer_print_error(struct pci_dev *dev, struct aer_err_info *info);
diff --git a/drivers/pci/pcie/aer/aerdrv_core.c b/drivers/pci/pcie/aer/aerdrv_core.c
index 0ea5acc40323..36e622d35c48 100644
--- a/drivers/pci/pcie/aer/aerdrv_core.c
+++ b/drivers/pci/pcie/aer/aerdrv_core.c
@@ -20,6 +20,7 @@
 #include <linux/slab.h>
 #include <linux/kfifo.h>
 #include "aerdrv.h"
+#include "../../pci.h"
 
 #define	PCI_EXP_AER_FLAGS	(PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \
 				 PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE)
@@ -227,318 +228,6 @@ static bool find_source_device(struct pci_dev *parent,
 	return true;
 }
 
-static int report_error_detected(struct pci_dev *dev, void *data)
-{
-	pci_ers_result_t vote;
-	const struct pci_error_handlers *err_handler;
-	struct aer_broadcast_data *result_data;
-	result_data = (struct aer_broadcast_data *) data;
-
-	device_lock(&dev->dev);
-	dev->error_state = result_data->state;
-
-	if (!dev->driver ||
-		!dev->driver->err_handler ||
-		!dev->driver->err_handler->error_detected) {
-		if (result_data->state == pci_channel_io_frozen &&
-			dev->hdr_type != PCI_HEADER_TYPE_BRIDGE) {
-			/*
-			 * In case of fatal recovery, if one of down-
-			 * stream device has no driver. We might be
-			 * unable to recover because a later insmod
-			 * of a driver for this device is unaware of
-			 * its hw state.
-			 */
-			pci_printk(KERN_DEBUG, dev, "device has %s\n",
-				   dev->driver ?
-				   "no AER-aware driver" : "no driver");
-		}
-
-		/*
-		 * If there's any device in the subtree that does not
-		 * have an error_detected callback, returning
-		 * PCI_ERS_RESULT_NO_AER_DRIVER prevents calling of
-		 * the subsequent mmio_enabled/slot_reset/resume
-		 * callbacks of "any" device in the subtree. All the
-		 * devices in the subtree are left in the error state
-		 * without recovery.
-		 */
-
-		if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
-			vote = PCI_ERS_RESULT_NO_AER_DRIVER;
-		else
-			vote = PCI_ERS_RESULT_NONE;
-	} else {
-		err_handler = dev->driver->err_handler;
-		vote = err_handler->error_detected(dev, result_data->state);
-		pci_uevent_ers(dev, PCI_ERS_RESULT_NONE);
-	}
-
-	result_data->result = merge_result(result_data->result, vote);
-	device_unlock(&dev->dev);
-	return 0;
-}
-
-static int report_mmio_enabled(struct pci_dev *dev, void *data)
-{
-	pci_ers_result_t vote;
-	const struct pci_error_handlers *err_handler;
-	struct aer_broadcast_data *result_data;
-	result_data = (struct aer_broadcast_data *) data;
-
-	device_lock(&dev->dev);
-	if (!dev->driver ||
-		!dev->driver->err_handler ||
-		!dev->driver->err_handler->mmio_enabled)
-		goto out;
-
-	err_handler = dev->driver->err_handler;
-	vote = err_handler->mmio_enabled(dev);
-	result_data->result = merge_result(result_data->result, vote);
-out:
-	device_unlock(&dev->dev);
-	return 0;
-}
-
-static int report_slot_reset(struct pci_dev *dev, void *data)
-{
-	pci_ers_result_t vote;
-	const struct pci_error_handlers *err_handler;
-	struct aer_broadcast_data *result_data;
-	result_data = (struct aer_broadcast_data *) data;
-
-	device_lock(&dev->dev);
-	if (!dev->driver ||
-		!dev->driver->err_handler ||
-		!dev->driver->err_handler->slot_reset)
-		goto out;
-
-	err_handler = dev->driver->err_handler;
-	vote = err_handler->slot_reset(dev);
-	result_data->result = merge_result(result_data->result, vote);
-out:
-	device_unlock(&dev->dev);
-	return 0;
-}
-
-static int report_resume(struct pci_dev *dev, void *data)
-{
-	const struct pci_error_handlers *err_handler;
-
-	device_lock(&dev->dev);
-	dev->error_state = pci_channel_io_normal;
-
-	if (!dev->driver ||
-		!dev->driver->err_handler ||
-		!dev->driver->err_handler->resume)
-		goto out;
-
-	err_handler = dev->driver->err_handler;
-	err_handler->resume(dev);
-	pci_uevent_ers(dev, PCI_ERS_RESULT_RECOVERED);
-out:
-	device_unlock(&dev->dev);
-	return 0;
-}
-
-/**
- * broadcast_error_message - handle message broadcast to downstream drivers
- * @dev: pointer to from where in a hierarchy message is broadcasted down
- * @state: error state
- * @error_mesg: message to print
- * @cb: callback to be broadcasted
- *
- * Invoked during error recovery process. Once being invoked, the content
- * of error severity will be broadcasted to all downstream drivers in a
- * hierarchy in question.
- */
-static pci_ers_result_t broadcast_error_message(struct pci_dev *dev,
-	enum pci_channel_state state,
-	char *error_mesg,
-	int (*cb)(struct pci_dev *, void *))
-{
-	struct aer_broadcast_data result_data;
-
-	pci_printk(KERN_DEBUG, dev, "broadcast %s message\n", error_mesg);
-	result_data.state = state;
-	if (cb == report_error_detected)
-		result_data.result = PCI_ERS_RESULT_CAN_RECOVER;
-	else
-		result_data.result = PCI_ERS_RESULT_RECOVERED;
-
-	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
-		/*
-		 * If the error is reported by a bridge, we think this error
-		 * is related to the downstream link of the bridge, so we
-		 * do error recovery on all subordinates of the bridge instead
-		 * of the bridge and clear the error status of the bridge.
-		 */
-		if (cb == report_error_detected)
-			dev->error_state = state;
-		pci_walk_bus(dev->subordinate, cb, &result_data);
-		if (cb == report_resume) {
-			pci_cleanup_aer_uncorrect_error_status(dev);
-			dev->error_state = pci_channel_io_normal;
-		}
-	} else {
-		/*
-		 * If the error is reported by an end point, we think this
-		 * error is related to the upstream link of the end point.
-		 */
-		if (state == pci_channel_io_normal)
-			/*
-			 * the error is non fatal so the bus is ok, just invoke
-			 * the callback for the function that logged the error.
-			 */
-			cb(dev, &result_data);
-		else
-			pci_walk_bus(dev->bus, cb, &result_data);
-	}
-
-	return result_data.result;
-}
-
-/**
- * default_reset_link - default reset function
- * @dev: pointer to pci_dev data structure
- *
- * Invoked when performing link reset on a Downstream Port or a
- * Root Port with no aer driver.
- */
-static pci_ers_result_t default_reset_link(struct pci_dev *dev)
-{
-	pci_reset_bridge_secondary_bus(dev);
-	pci_printk(KERN_DEBUG, dev, "downstream link has been reset\n");
-	return PCI_ERS_RESULT_RECOVERED;
-}
-
-static int find_aer_service_iter(struct device *device, void *data)
-{
-	struct pcie_port_service_driver *service_driver, **drv;
-
-	drv = (struct pcie_port_service_driver **) data;
-
-	if (device->bus == &pcie_port_bus_type && device->driver) {
-		service_driver = to_service_driver(device->driver);
-		if (service_driver->service == PCIE_PORT_SERVICE_AER) {
-			*drv = service_driver;
-			return 1;
-		}
-	}
-
-	return 0;
-}
-
-static struct pcie_port_service_driver *find_aer_service(struct pci_dev *dev)
-{
-	struct pcie_port_service_driver *drv = NULL;
-
-	device_for_each_child(&dev->dev, &drv, find_aer_service_iter);
-
-	return drv;
-}
-
-static pci_ers_result_t reset_link(struct pci_dev *dev)
-{
-	struct pci_dev *udev;
-	pci_ers_result_t status;
-	struct pcie_port_service_driver *driver;
-
-	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
-		/* Reset this port for all subordinates */
-		udev = dev;
-	} else {
-		/* Reset the upstream component (likely downstream port) */
-		udev = dev->bus->self;
-	}
-
-	/* Use the aer driver of the component firstly */
-	driver = find_aer_service(udev);
-
-	if (driver && driver->reset_link) {
-		status = driver->reset_link(udev);
-	} else if (udev->has_secondary_link) {
-		status = default_reset_link(udev);
-	} else {
-		pci_printk(KERN_DEBUG, dev, "no link-reset support at upstream device %s\n",
-			pci_name(udev));
-		return PCI_ERS_RESULT_DISCONNECT;
-	}
-
-	if (status != PCI_ERS_RESULT_RECOVERED) {
-		pci_printk(KERN_DEBUG, dev, "link reset at upstream device %s failed\n",
-			pci_name(udev));
-		return PCI_ERS_RESULT_DISCONNECT;
-	}
-
-	return status;
-}
-
-/**
- * do_recovery - handle nonfatal/fatal error recovery process
- * @dev: pointer to a pci_dev data structure of agent detecting an error
- * @severity: error severity type
- *
- * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
- * error detected message to all downstream drivers within a hierarchy in
- * question and return the returned code.
- */
-static void do_recovery(struct pci_dev *dev, int severity)
-{
-	pci_ers_result_t status, result = PCI_ERS_RESULT_RECOVERED;
-	enum pci_channel_state state;
-
-	if (severity == AER_FATAL)
-		state = pci_channel_io_frozen;
-	else
-		state = pci_channel_io_normal;
-
-	status = broadcast_error_message(dev,
-			state,
-			"error_detected",
-			report_error_detected);
-
-	if (severity == AER_FATAL) {
-		result = reset_link(dev);
-		if (result != PCI_ERS_RESULT_RECOVERED)
-			goto failed;
-	}
-
-	if (status == PCI_ERS_RESULT_CAN_RECOVER)
-		status = broadcast_error_message(dev,
-				state,
-				"mmio_enabled",
-				report_mmio_enabled);
-
-	if (status == PCI_ERS_RESULT_NEED_RESET) {
-		/*
-		 * TODO: Should call platform-specific
-		 * functions to reset slot before calling
-		 * drivers' slot_reset callbacks?
-		 */
-		status = broadcast_error_message(dev,
-				state,
-				"slot_reset",
-				report_slot_reset);
-	}
-
-	if (status != PCI_ERS_RESULT_RECOVERED)
-		goto failed;
-
-	broadcast_error_message(dev,
-				state,
-				"resume",
-				report_resume);
-
-	pci_info(dev, "AER: Device recovery successful\n");
-	return;
-
-failed:
-	pci_uevent_ers(dev, PCI_ERS_RESULT_DISCONNECT);
-	/* TODO: Should kernel panic here? */
-	pci_info(dev, "AER: Device recovery failed\n");
-}
-
 /**
  * handle_error_source - handle logging error into an event log
  * @aerdev: pointer to pcie_device data structure of the root port
@@ -562,8 +251,10 @@ static void handle_error_source(struct pcie_device *aerdev,
 		if (pos)
 			pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS,
 					info->status);
-	} else
-		do_recovery(dev, info->severity);
+	} else if (info->severity == AER_NONFATAL)
+		pcie_do_nonfatal_recovery(dev);
+	else if (info->severity == AER_FATAL)
+		pcie_do_fatal_recovery(dev, PCIE_PORT_SERVICE_AER);
 }
 
 #ifdef CONFIG_ACPI_APEI_PCIEAER
@@ -627,8 +318,10 @@ static void aer_recover_work_func(struct work_struct *work)
 			continue;
 		}
 		cper_print_aer(pdev, entry.severity, entry.regs);
-		if (entry.severity != AER_CORRECTABLE)
-			do_recovery(pdev, entry.severity);
+		if (entry.severity == AER_NONFATAL)
+			pcie_do_nonfatal_recovery(pdev);
+		else if (entry.severity == AER_FATAL)
+			pcie_do_fatal_recovery(pdev, PCIE_PORT_SERVICE_AER);
 		pci_dev_put(pdev);
 	}
 }
diff --git a/drivers/pci/pcie/aer/aerdrv_errprint.c b/drivers/pci/pcie/aer/aerdrv_errprint.c
index cfc89dd57831..693ce439691e 100644
--- a/drivers/pci/pcie/aer/aerdrv_errprint.c
+++ b/drivers/pci/pcie/aer/aerdrv_errprint.c
@@ -163,21 +163,20 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info)
 	int id = ((dev->bus->number << 8) | dev->devfn);
 
 	if (!info->status) {
-		pci_err(dev, "PCIe Bus Error: severity=%s, type=Unaccessible, id=%04x(Unregistered Agent ID)\n",
-			aer_error_severity_string[info->severity], id);
+		pci_err(dev, "PCIe Bus Error: severity=%s, type=Unaccessible, (Unregistered Agent ID)\n",
+			aer_error_severity_string[info->severity]);
 		goto out;
 	}
 
 	layer = AER_GET_LAYER_ERROR(info->severity, info->status);
 	agent = AER_GET_AGENT(info->severity, info->status);
 
-	pci_err(dev, "PCIe Bus Error: severity=%s, type=%s, id=%04x(%s)\n",
+	pci_err(dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n",
 		aer_error_severity_string[info->severity],
-		aer_error_layer[layer], id, aer_agent_string[agent]);
+		aer_error_layer[layer], aer_agent_string[agent]);
 
-	pci_err(dev, "  device [%04x:%04x] error status/mask=%08x/%08x\n",
-		dev->vendor, dev->device,
-		info->status, info->mask);
+	pci_err(dev, "  error status/mask=%08x/%08x\n", info->status,
+		info->mask);
 
 	__aer_print_error(dev, info);
 
@@ -186,17 +185,21 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info)
 
 out:
 	if (info->id && info->error_dev_num > 1 && info->id == id)
-		pci_err(dev, "  Error of this Agent(%04x) is reported first\n", id);
+		pci_err(dev, "  Error of this Agent is reported first\n");
 
 	trace_aer_event(dev_name(&dev->dev), (info->status & ~info->mask),
-			info->severity);
+			info->severity, info->tlp_header_valid, &info->tlp);
 }
 
 void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info)
 {
-	pci_info(dev, "AER: %s%s error received: id=%04x\n",
+	u8 bus = info->id >> 8;
+	u8 devfn = info->id & 0xff;
+
+	pci_info(dev, "AER: %s%s error received: %04x:%02x:%02x.%d\n",
 		info->multi_error_valid ? "Multiple " : "",
-		aer_error_severity_string[info->severity], info->id);
+		aer_error_severity_string[info->severity],
+		pci_domain_nr(dev->bus), bus, devfn >> 3, devfn & 0x7);
 }
 
 #ifdef CONFIG_ACPI_APEI_PCIEAER
@@ -216,28 +219,30 @@ EXPORT_SYMBOL_GPL(cper_severity_to_aer);
 void cper_print_aer(struct pci_dev *dev, int aer_severity,
 		    struct aer_capability_regs *aer)
 {
-	int layer, agent, status_strs_size, tlp_header_valid = 0;
+	int layer, agent, tlp_header_valid = 0;
 	u32 status, mask;
-	const char **status_strs;
+	struct aer_err_info info;
 
 	if (aer_severity == AER_CORRECTABLE) {
 		status = aer->cor_status;
 		mask = aer->cor_mask;
-		status_strs = aer_correctable_error_string;
-		status_strs_size = ARRAY_SIZE(aer_correctable_error_string);
 	} else {
 		status = aer->uncor_status;
 		mask = aer->uncor_mask;
-		status_strs = aer_uncorrectable_error_string;
-		status_strs_size = ARRAY_SIZE(aer_uncorrectable_error_string);
 		tlp_header_valid = status & AER_LOG_TLP_MASKS;
 	}
 
 	layer = AER_GET_LAYER_ERROR(aer_severity, status);
 	agent = AER_GET_AGENT(aer_severity, status);
 
+	memset(&info, 0, sizeof(info));
+	info.severity = aer_severity;
+	info.status = status;
+	info.mask = mask;
+	info.first_error = PCI_ERR_CAP_FEP(aer->cap_control);
+
 	pci_err(dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", status, mask);
-	cper_print_bits("", status, status_strs, status_strs_size);
+	__aer_print_error(dev, &info);
 	pci_err(dev, "aer_layer=%s, aer_agent=%s\n",
 		aer_error_layer[layer], aer_agent_string[agent]);
 
@@ -249,6 +254,6 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity,
 		__print_tlp_header(dev, &aer->header_log);
 
 	trace_aer_event(dev_name(&dev->dev), (status & ~mask),
-			aer_severity);
+			aer_severity, tlp_header_valid, &aer->header_log);
 }
 #endif
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index f76eb7704f64..c687c817b47d 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -400,6 +400,15 @@ static void pcie_get_aspm_reg(struct pci_dev *pdev,
 		info->l1ss_cap = 0;
 		return;
 	}
+
+	/*
+	 * If we don't have LTR for the entire path from the Root Complex
+	 * to this device, we can't use ASPM L1.2 because it relies on the
+	 * LTR_L1.2_THRESHOLD.  See PCIe r4.0, secs 5.5.4, 6.18.
+	 */
+	if (!pdev->ltr_path)
+		info->l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
+
 	pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1,
 			      &info->l1ss_ctl1);
 	pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2,
diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c
index 8c57d607e603..d6436681c535 100644
--- a/drivers/pci/pcie/dpc.c
+++ b/drivers/pci/pcie/dpc.c
@@ -68,44 +68,35 @@ static int dpc_wait_rp_inactive(struct dpc_dev *dpc)
 
 static void dpc_wait_link_inactive(struct dpc_dev *dpc)
 {
-	unsigned long timeout = jiffies + HZ;
 	struct pci_dev *pdev = dpc->dev->port;
-	struct device *dev = &dpc->dev->device;
-	u16 lnk_status;
 
-	pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
-	while (lnk_status & PCI_EXP_LNKSTA_DLLLA &&
-					!time_after(jiffies, timeout)) {
-		msleep(10);
-		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
-	}
-	if (lnk_status & PCI_EXP_LNKSTA_DLLLA)
-		dev_warn(dev, "Link state not disabled for DPC event\n");
+	pcie_wait_for_link(pdev, false);
 }
 
-static void dpc_work(struct work_struct *work)
+static pci_ers_result_t dpc_reset_link(struct pci_dev *pdev)
 {
-	struct dpc_dev *dpc = container_of(work, struct dpc_dev, work);
-	struct pci_dev *dev, *temp, *pdev = dpc->dev->port;
-	struct pci_bus *parent = pdev->subordinate;
-	u16 cap = dpc->cap_pos, ctl;
-
-	pci_lock_rescan_remove();
-	list_for_each_entry_safe_reverse(dev, temp, &parent->devices,
-					 bus_list) {
-		pci_dev_get(dev);
-		pci_dev_set_disconnected(dev, NULL);
-		if (pci_has_subordinate(dev))
-			pci_walk_bus(dev->subordinate,
-				     pci_dev_set_disconnected, NULL);
-		pci_stop_and_remove_bus_device(dev);
-		pci_dev_put(dev);
-	}
-	pci_unlock_rescan_remove();
-
+	struct dpc_dev *dpc;
+	struct pcie_device *pciedev;
+	struct device *devdpc;
+	u16 cap, ctl;
+
+	/*
+	 * DPC disables the Link automatically in hardware, so it has
+	 * already been reset by the time we get here.
+	 */
+	devdpc = pcie_port_find_device(pdev, PCIE_PORT_SERVICE_DPC);
+	pciedev = to_pcie_device(devdpc);
+	dpc = get_service_data(pciedev);
+	cap = dpc->cap_pos;
+
+	/*
+	 * Wait until the Link is inactive, then clear DPC Trigger Status
+	 * to allow the Port to leave DPC.
+	 */
 	dpc_wait_link_inactive(dpc);
+
 	if (dpc->rp_extensions && dpc_wait_rp_inactive(dpc))
-		return;
+		return PCI_ERS_RESULT_DISCONNECT;
 	if (dpc->rp_extensions && dpc->rp_pio_status) {
 		pci_write_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_STATUS,
 				       dpc->rp_pio_status);
@@ -113,11 +104,22 @@ static void dpc_work(struct work_struct *work)
 	}
 
 	pci_write_config_word(pdev, cap + PCI_EXP_DPC_STATUS,
-		PCI_EXP_DPC_STATUS_TRIGGER | PCI_EXP_DPC_STATUS_INTERRUPT);
+			      PCI_EXP_DPC_STATUS_TRIGGER);
 
 	pci_read_config_word(pdev, cap + PCI_EXP_DPC_CTL, &ctl);
 	pci_write_config_word(pdev, cap + PCI_EXP_DPC_CTL,
 			      ctl | PCI_EXP_DPC_CTL_INT_EN);
+
+	return PCI_ERS_RESULT_RECOVERED;
+}
+
+static void dpc_work(struct work_struct *work)
+{
+	struct dpc_dev *dpc = container_of(work, struct dpc_dev, work);
+	struct pci_dev *pdev = dpc->dev->port;
+
+	/* We configure DPC so it only triggers on ERR_FATAL */
+	pcie_do_fatal_recovery(pdev, PCIE_PORT_SERVICE_DPC);
 }
 
 static void dpc_process_rp_pio_error(struct dpc_dev *dpc)
@@ -223,6 +225,9 @@ static irqreturn_t dpc_irq(int irq, void *context)
 	if (dpc->rp_extensions && reason == 3 && ext_reason == 0)
 		dpc_process_rp_pio_error(dpc);
 
+	pci_write_config_word(pdev, cap + PCI_EXP_DPC_STATUS,
+			      PCI_EXP_DPC_STATUS_INTERRUPT);
+
 	schedule_work(&dpc->work);
 
 	return IRQ_HANDLED;
@@ -270,7 +275,7 @@ static int dpc_probe(struct pcie_device *dev)
 		}
 	}
 
-	ctl = (ctl & 0xfff4) | PCI_EXP_DPC_CTL_EN_NONFATAL | PCI_EXP_DPC_CTL_INT_EN;
+	ctl = (ctl & 0xfff4) | PCI_EXP_DPC_CTL_EN_FATAL | PCI_EXP_DPC_CTL_INT_EN;
 	pci_write_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CTL, ctl);
 
 	dev_info(device, "DPC error containment capabilities: Int Msg #%d, RPExt%c PoisonedTLP%c SwTrigger%c RP PIO Log %d, DL_ActiveErr%c\n",
@@ -288,7 +293,7 @@ static void dpc_remove(struct pcie_device *dev)
 	u16 ctl;
 
 	pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CTL, &ctl);
-	ctl &= ~(PCI_EXP_DPC_CTL_EN_NONFATAL | PCI_EXP_DPC_CTL_INT_EN);
+	ctl &= ~(PCI_EXP_DPC_CTL_EN_FATAL | PCI_EXP_DPC_CTL_INT_EN);
 	pci_write_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CTL, ctl);
 }
 
@@ -298,6 +303,7 @@ static struct pcie_port_service_driver dpcdriver = {
 	.service	= PCIE_PORT_SERVICE_DPC,
 	.probe		= dpc_probe,
 	.remove		= dpc_remove,
+	.reset_link	= dpc_reset_link,
 };
 
 static int __init dpc_service_init(void)
diff --git a/drivers/pci/pcie/err.c b/drivers/pci/pcie/err.c
new file mode 100644
index 000000000000..f7ce0cb0b0b7
--- /dev/null
+++ b/drivers/pci/pcie/err.c
@@ -0,0 +1,388 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file implements the error recovery as a core part of PCIe error
+ * reporting. When a PCIe error is delivered, an error message will be
+ * collected and printed to console, then, an error recovery procedure
+ * will be executed by following the PCI error recovery rules.
+ *
+ * Copyright (C) 2006 Intel Corp.
+ *	Tom Long Nguyen (tom.l.nguyen@intel.com)
+ *	Zhang Yanmin (yanmin.zhang@intel.com)
+ */
+
+#include <linux/pci.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/aer.h>
+#include "portdrv.h"
+#include "../pci.h"
+
+struct aer_broadcast_data {
+	enum pci_channel_state state;
+	enum pci_ers_result result;
+};
+
+static pci_ers_result_t merge_result(enum pci_ers_result orig,
+				  enum pci_ers_result new)
+{
+	if (new == PCI_ERS_RESULT_NO_AER_DRIVER)
+		return PCI_ERS_RESULT_NO_AER_DRIVER;
+
+	if (new == PCI_ERS_RESULT_NONE)
+		return orig;
+
+	switch (orig) {
+	case PCI_ERS_RESULT_CAN_RECOVER:
+	case PCI_ERS_RESULT_RECOVERED:
+		orig = new;
+		break;
+	case PCI_ERS_RESULT_DISCONNECT:
+		if (new == PCI_ERS_RESULT_NEED_RESET)
+			orig = PCI_ERS_RESULT_NEED_RESET;
+		break;
+	default:
+		break;
+	}
+
+	return orig;
+}
+
+static int report_error_detected(struct pci_dev *dev, void *data)
+{
+	pci_ers_result_t vote;
+	const struct pci_error_handlers *err_handler;
+	struct aer_broadcast_data *result_data;
+
+	result_data = (struct aer_broadcast_data *) data;
+
+	device_lock(&dev->dev);
+	dev->error_state = result_data->state;
+
+	if (!dev->driver ||
+		!dev->driver->err_handler ||
+		!dev->driver->err_handler->error_detected) {
+		if (result_data->state == pci_channel_io_frozen &&
+			dev->hdr_type != PCI_HEADER_TYPE_BRIDGE) {
+			/*
+			 * In case of fatal recovery, if one of down-
+			 * stream device has no driver. We might be
+			 * unable to recover because a later insmod
+			 * of a driver for this device is unaware of
+			 * its hw state.
+			 */
+			pci_printk(KERN_DEBUG, dev, "device has %s\n",
+				   dev->driver ?
+				   "no AER-aware driver" : "no driver");
+		}
+
+		/*
+		 * If there's any device in the subtree that does not
+		 * have an error_detected callback, returning
+		 * PCI_ERS_RESULT_NO_AER_DRIVER prevents calling of
+		 * the subsequent mmio_enabled/slot_reset/resume
+		 * callbacks of "any" device in the subtree. All the
+		 * devices in the subtree are left in the error state
+		 * without recovery.
+		 */
+
+		if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
+			vote = PCI_ERS_RESULT_NO_AER_DRIVER;
+		else
+			vote = PCI_ERS_RESULT_NONE;
+	} else {
+		err_handler = dev->driver->err_handler;
+		vote = err_handler->error_detected(dev, result_data->state);
+		pci_uevent_ers(dev, PCI_ERS_RESULT_NONE);
+	}
+
+	result_data->result = merge_result(result_data->result, vote);
+	device_unlock(&dev->dev);
+	return 0;
+}
+
+static int report_mmio_enabled(struct pci_dev *dev, void *data)
+{
+	pci_ers_result_t vote;
+	const struct pci_error_handlers *err_handler;
+	struct aer_broadcast_data *result_data;
+
+	result_data = (struct aer_broadcast_data *) data;
+
+	device_lock(&dev->dev);
+	if (!dev->driver ||
+		!dev->driver->err_handler ||
+		!dev->driver->err_handler->mmio_enabled)
+		goto out;
+
+	err_handler = dev->driver->err_handler;
+	vote = err_handler->mmio_enabled(dev);
+	result_data->result = merge_result(result_data->result, vote);
+out:
+	device_unlock(&dev->dev);
+	return 0;
+}
+
+static int report_slot_reset(struct pci_dev *dev, void *data)
+{
+	pci_ers_result_t vote;
+	const struct pci_error_handlers *err_handler;
+	struct aer_broadcast_data *result_data;
+
+	result_data = (struct aer_broadcast_data *) data;
+
+	device_lock(&dev->dev);
+	if (!dev->driver ||
+		!dev->driver->err_handler ||
+		!dev->driver->err_handler->slot_reset)
+		goto out;
+
+	err_handler = dev->driver->err_handler;
+	vote = err_handler->slot_reset(dev);
+	result_data->result = merge_result(result_data->result, vote);
+out:
+	device_unlock(&dev->dev);
+	return 0;
+}
+
+static int report_resume(struct pci_dev *dev, void *data)
+{
+	const struct pci_error_handlers *err_handler;
+
+	device_lock(&dev->dev);
+	dev->error_state = pci_channel_io_normal;
+
+	if (!dev->driver ||
+		!dev->driver->err_handler ||
+		!dev->driver->err_handler->resume)
+		goto out;
+
+	err_handler = dev->driver->err_handler;
+	err_handler->resume(dev);
+	pci_uevent_ers(dev, PCI_ERS_RESULT_RECOVERED);
+out:
+	device_unlock(&dev->dev);
+	return 0;
+}
+
+/**
+ * default_reset_link - default reset function
+ * @dev: pointer to pci_dev data structure
+ *
+ * Invoked when performing link reset on a Downstream Port or a
+ * Root Port with no aer driver.
+ */
+static pci_ers_result_t default_reset_link(struct pci_dev *dev)
+{
+	pci_reset_bridge_secondary_bus(dev);
+	pci_printk(KERN_DEBUG, dev, "downstream link has been reset\n");
+	return PCI_ERS_RESULT_RECOVERED;
+}
+
+static pci_ers_result_t reset_link(struct pci_dev *dev, u32 service)
+{
+	struct pci_dev *udev;
+	pci_ers_result_t status;
+	struct pcie_port_service_driver *driver = NULL;
+
+	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
+		/* Reset this port for all subordinates */
+		udev = dev;
+	} else {
+		/* Reset the upstream component (likely downstream port) */
+		udev = dev->bus->self;
+	}
+
+	/* Use the aer driver of the component firstly */
+	driver = pcie_port_find_service(udev, service);
+
+	if (driver && driver->reset_link) {
+		status = driver->reset_link(udev);
+	} else if (udev->has_secondary_link) {
+		status = default_reset_link(udev);
+	} else {
+		pci_printk(KERN_DEBUG, dev, "no link-reset support at upstream device %s\n",
+			pci_name(udev));
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
+	if (status != PCI_ERS_RESULT_RECOVERED) {
+		pci_printk(KERN_DEBUG, dev, "link reset at upstream device %s failed\n",
+			pci_name(udev));
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
+	return status;
+}
+
+/**
+ * broadcast_error_message - handle message broadcast to downstream drivers
+ * @dev: pointer to from where in a hierarchy message is broadcasted down
+ * @state: error state
+ * @error_mesg: message to print
+ * @cb: callback to be broadcasted
+ *
+ * Invoked during error recovery process. Once being invoked, the content
+ * of error severity will be broadcasted to all downstream drivers in a
+ * hierarchy in question.
+ */
+static pci_ers_result_t broadcast_error_message(struct pci_dev *dev,
+	enum pci_channel_state state,
+	char *error_mesg,
+	int (*cb)(struct pci_dev *, void *))
+{
+	struct aer_broadcast_data result_data;
+
+	pci_printk(KERN_DEBUG, dev, "broadcast %s message\n", error_mesg);
+	result_data.state = state;
+	if (cb == report_error_detected)
+		result_data.result = PCI_ERS_RESULT_CAN_RECOVER;
+	else
+		result_data.result = PCI_ERS_RESULT_RECOVERED;
+
+	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
+		/*
+		 * If the error is reported by a bridge, we think this error
+		 * is related to the downstream link of the bridge, so we
+		 * do error recovery on all subordinates of the bridge instead
+		 * of the bridge and clear the error status of the bridge.
+		 */
+		if (cb == report_error_detected)
+			dev->error_state = state;
+		pci_walk_bus(dev->subordinate, cb, &result_data);
+		if (cb == report_resume) {
+			pci_cleanup_aer_uncorrect_error_status(dev);
+			dev->error_state = pci_channel_io_normal;
+		}
+	} else {
+		/*
+		 * If the error is reported by an end point, we think this
+		 * error is related to the upstream link of the end point.
+		 */
+		if (state == pci_channel_io_normal)
+			/*
+			 * the error is non fatal so the bus is ok, just invoke
+			 * the callback for the function that logged the error.
+			 */
+			cb(dev, &result_data);
+		else
+			pci_walk_bus(dev->bus, cb, &result_data);
+	}
+
+	return result_data.result;
+}
+
+/**
+ * pcie_do_fatal_recovery - handle fatal error recovery process
+ * @dev: pointer to a pci_dev data structure of agent detecting an error
+ *
+ * Invoked when an error is fatal. Once being invoked, removes the devices
+ * beneath this AER agent, followed by reset link e.g. secondary bus reset
+ * followed by re-enumeration of devices.
+ */
+void pcie_do_fatal_recovery(struct pci_dev *dev, u32 service)
+{
+	struct pci_dev *udev;
+	struct pci_bus *parent;
+	struct pci_dev *pdev, *temp;
+	pci_ers_result_t result;
+
+	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
+		udev = dev;
+	else
+		udev = dev->bus->self;
+
+	parent = udev->subordinate;
+	pci_lock_rescan_remove();
+	list_for_each_entry_safe_reverse(pdev, temp, &parent->devices,
+					 bus_list) {
+		pci_dev_get(pdev);
+		pci_dev_set_disconnected(pdev, NULL);
+		if (pci_has_subordinate(pdev))
+			pci_walk_bus(pdev->subordinate,
+				     pci_dev_set_disconnected, NULL);
+		pci_stop_and_remove_bus_device(pdev);
+		pci_dev_put(pdev);
+	}
+
+	result = reset_link(udev, service);
+
+	if ((service == PCIE_PORT_SERVICE_AER) &&
+	    (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)) {
+		/*
+		 * If the error is reported by a bridge, we think this error
+		 * is related to the downstream link of the bridge, so we
+		 * do error recovery on all subordinates of the bridge instead
+		 * of the bridge and clear the error status of the bridge.
+		 */
+		pci_cleanup_aer_uncorrect_error_status(dev);
+	}
+
+	if (result == PCI_ERS_RESULT_RECOVERED) {
+		if (pcie_wait_for_link(udev, true))
+			pci_rescan_bus(udev->bus);
+		pci_info(dev, "Device recovery from fatal error successful\n");
+	} else {
+		pci_uevent_ers(dev, PCI_ERS_RESULT_DISCONNECT);
+		pci_info(dev, "Device recovery from fatal error failed\n");
+	}
+
+	pci_unlock_rescan_remove();
+}
+
+/**
+ * pcie_do_nonfatal_recovery - handle nonfatal error recovery process
+ * @dev: pointer to a pci_dev data structure of agent detecting an error
+ *
+ * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
+ * error detected message to all downstream drivers within a hierarchy in
+ * question and return the returned code.
+ */
+void pcie_do_nonfatal_recovery(struct pci_dev *dev)
+{
+	pci_ers_result_t status;
+	enum pci_channel_state state;
+
+	state = pci_channel_io_normal;
+
+	status = broadcast_error_message(dev,
+			state,
+			"error_detected",
+			report_error_detected);
+
+	if (status == PCI_ERS_RESULT_CAN_RECOVER)
+		status = broadcast_error_message(dev,
+				state,
+				"mmio_enabled",
+				report_mmio_enabled);
+
+	if (status == PCI_ERS_RESULT_NEED_RESET) {
+		/*
+		 * TODO: Should call platform-specific
+		 * functions to reset slot before calling
+		 * drivers' slot_reset callbacks?
+		 */
+		status = broadcast_error_message(dev,
+				state,
+				"slot_reset",
+				report_slot_reset);
+	}
+
+	if (status != PCI_ERS_RESULT_RECOVERED)
+		goto failed;
+
+	broadcast_error_message(dev,
+				state,
+				"resume",
+				report_resume);
+
+	pci_info(dev, "AER: Device recovery successful\n");
+	return;
+
+failed:
+	pci_uevent_ers(dev, PCI_ERS_RESULT_DISCONNECT);
+
+	/* TODO: Should kernel panic here? */
+	pci_info(dev, "AER: Device recovery failed\n");
+}
diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h
index d0c6783dbfe3..2bb5db7b53e6 100644
--- a/drivers/pci/pcie/portdrv.h
+++ b/drivers/pci/pcie/portdrv.h
@@ -11,8 +11,6 @@
 
 #include <linux/compiler.h>
 
-extern bool pcie_ports_native;
-
 /* Service Type */
 #define PCIE_PORT_SERVICE_PME_SHIFT	0	/* Power Management Event */
 #define PCIE_PORT_SERVICE_PME		(1 << PCIE_PORT_SERVICE_PME_SHIFT)
@@ -112,4 +110,7 @@ static inline bool pcie_pme_no_msi(void) { return false; }
 static inline void pcie_pme_interrupt_enable(struct pci_dev *dev, bool en) {}
 #endif /* !CONFIG_PCIE_PME */
 
+struct pcie_port_service_driver *pcie_port_find_service(struct pci_dev *dev,
+							u32 service);
+struct device *pcie_port_find_device(struct pci_dev *dev, u32 service);
 #endif /* _PORTDRV_H_ */
diff --git a/drivers/pci/pcie/portdrv_acpi.c b/drivers/pci/pcie/portdrv_acpi.c
deleted file mode 100644
index 8ab5d434b9c6..000000000000
--- a/drivers/pci/pcie/portdrv_acpi.c
+++ /dev/null
@@ -1,57 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * PCIe Port Native Services Support, ACPI-Related Part
- *
- * Copyright (C) 2010 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc.
- */
-
-#include <linux/pci.h>
-#include <linux/kernel.h>
-#include <linux/errno.h>
-#include <linux/acpi.h>
-#include <linux/pci-acpi.h>
-
-#include "aer/aerdrv.h"
-#include "../pci.h"
-#include "portdrv.h"
-
-/**
- * pcie_port_acpi_setup - Request the BIOS to release control of PCIe services.
- * @port: PCIe Port service for a root port or event collector.
- * @srv_mask: Bit mask of services that can be enabled for @port.
- *
- * Invoked when @port is identified as a PCIe port device.  To avoid conflicts
- * with the BIOS PCIe port native services support requires the BIOS to yield
- * control of these services to the kernel.  The mask of services that the BIOS
- * allows to be enabled for @port is written to @srv_mask.
- *
- * NOTE: It turns out that we cannot do that for individual port services
- * separately, because that would make some systems work incorrectly.
- */
-void pcie_port_acpi_setup(struct pci_dev *port, int *srv_mask)
-{
-	struct acpi_pci_root *root;
-	acpi_handle handle;
-	u32 flags;
-
-	if (acpi_pci_disabled)
-		return;
-
-	handle = acpi_find_root_bridge_handle(port);
-	if (!handle)
-		return;
-
-	root = acpi_pci_find_root(handle);
-	if (!root)
-		return;
-
-	flags = root->osc_control_set;
-
-	*srv_mask = 0;
-	if (flags & OSC_PCI_EXPRESS_NATIVE_HP_CONTROL)
-		*srv_mask |= PCIE_PORT_SERVICE_HP;
-	if (flags & OSC_PCI_EXPRESS_PME_CONTROL)
-		*srv_mask |= PCIE_PORT_SERVICE_PME;
-	if (flags & OSC_PCI_EXPRESS_AER_CONTROL)
-		*srv_mask |= PCIE_PORT_SERVICE_AER | PCIE_PORT_SERVICE_DPC;
-}
diff --git a/drivers/pci/pcie/portdrv_core.c b/drivers/pci/pcie/portdrv_core.c
index c9c0663db282..fd1f65ff0d7d 100644
--- a/drivers/pci/pcie/portdrv_core.c
+++ b/drivers/pci/pcie/portdrv_core.c
@@ -18,6 +18,11 @@
 
 #include "../pci.h"
 #include "portdrv.h"
+struct portdrv_service_data {
+	struct pcie_port_service_driver *drv;
+	struct device *dev;
+	u32 service;
+};
 
 /**
  * release_pcie_device - free PCI Express port service device structure
@@ -199,7 +204,7 @@ static int get_port_device_capability(struct pci_dev *dev)
 	int services = 0;
 
 	if (dev->is_hotplug_bridge &&
-	    (pcie_ports_native || host->native_hotplug)) {
+	    (pcie_ports_native || host->native_pcie_hotplug)) {
 		services |= PCIE_PORT_SERVICE_HP;
 
 		/*
@@ -398,6 +403,68 @@ static int remove_iter(struct device *dev, void *data)
 	return 0;
 }
 
+static int find_service_iter(struct device *device, void *data)
+{
+	struct pcie_port_service_driver *service_driver;
+	struct portdrv_service_data *pdrvs;
+	u32 service;
+
+	pdrvs = (struct portdrv_service_data *) data;
+	service = pdrvs->service;
+
+	if (device->bus == &pcie_port_bus_type && device->driver) {
+		service_driver = to_service_driver(device->driver);
+		if (service_driver->service == service) {
+			pdrvs->drv = service_driver;
+			pdrvs->dev = device;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+/**
+ * pcie_port_find_service - find the service driver
+ * @dev: PCI Express port the service is associated with
+ * @service: Service to find
+ *
+ * Find PCI Express port service driver associated with given service
+ */
+struct pcie_port_service_driver *pcie_port_find_service(struct pci_dev *dev,
+							u32 service)
+{
+	struct pcie_port_service_driver *drv;
+	struct portdrv_service_data pdrvs;
+
+	pdrvs.drv = NULL;
+	pdrvs.service = service;
+	device_for_each_child(&dev->dev, &pdrvs, find_service_iter);
+
+	drv = pdrvs.drv;
+	return drv;
+}
+
+/**
+ * pcie_port_find_device - find the struct device
+ * @dev: PCI Express port the service is associated with
+ * @service: For the service to find
+ *
+ * Find the struct device associated with given service on a pci_dev
+ */
+struct device *pcie_port_find_device(struct pci_dev *dev,
+				      u32 service)
+{
+	struct device *device;
+	struct portdrv_service_data pdrvs;
+
+	pdrvs.dev = NULL;
+	pdrvs.service = service;
+	device_for_each_child(&dev->dev, &pdrvs, find_service_iter);
+
+	device = pdrvs.dev;
+	return device;
+}
+
 /**
  * pcie_port_device_remove - unregister PCI Express port service devices
  * @dev: PCI Express port the service devices to unregister are associated with
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index ac91b6fd0bcd..56771f356044 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -526,12 +526,14 @@ static void devm_pci_release_host_bridge_dev(struct device *dev)
 
 	if (bridge->release_fn)
 		bridge->release_fn(bridge);
+
+	pci_free_resource_list(&bridge->windows);
 }
 
 static void pci_release_host_bridge_dev(struct device *dev)
 {
 	devm_pci_release_host_bridge_dev(dev);
-	pci_free_host_bridge(to_pci_host_bridge(dev));
+	kfree(to_pci_host_bridge(dev));
 }
 
 struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
@@ -552,8 +554,10 @@ struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
 	 * OS from interfering.
 	 */
 	bridge->native_aer = 1;
-	bridge->native_hotplug = 1;
+	bridge->native_pcie_hotplug = 1;
+	bridge->native_shpc_hotplug = 1;
 	bridge->native_pme = 1;
+	bridge->native_ltr = 1;
 
 	return bridge;
 }
@@ -882,6 +886,45 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge)
 	return err;
 }
 
+static bool pci_bridge_child_ext_cfg_accessible(struct pci_dev *bridge)
+{
+	int pos;
+	u32 status;
+
+	/*
+	 * If extended config space isn't accessible on a bridge's primary
+	 * bus, we certainly can't access it on the secondary bus.
+	 */
+	if (bridge->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
+		return false;
+
+	/*
+	 * PCIe Root Ports and switch ports are PCIe on both sides, so if
+	 * extended config space is accessible on the primary, it's also
+	 * accessible on the secondary.
+	 */
+	if (pci_is_pcie(bridge) &&
+	    (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT ||
+	     pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM ||
+	     pci_pcie_type(bridge) == PCI_EXP_TYPE_DOWNSTREAM))
+		return true;
+
+	/*
+	 * For the other bridge types:
+	 *   - PCI-to-PCI bridges
+	 *   - PCIe-to-PCI/PCI-X forward bridges
+	 *   - PCI/PCI-X-to-PCIe reverse bridges
+	 * extended config space on the secondary side is only accessible
+	 * if the bridge supports PCI-X Mode 2.
+	 */
+	pos = pci_find_capability(bridge, PCI_CAP_ID_PCIX);
+	if (!pos)
+		return false;
+
+	pci_read_config_dword(bridge, pos + PCI_X_STATUS, &status);
+	return status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ);
+}
+
 static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
 					   struct pci_dev *bridge, int busnr)
 {
@@ -923,6 +966,16 @@ static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
 	pci_set_bus_of_node(child);
 	pci_set_bus_speed(child);
 
+	/*
+	 * Check whether extended config space is accessible on the child
+	 * bus.  Note that we currently assume it is always accessible on
+	 * the root bus.
+	 */
+	if (!pci_bridge_child_ext_cfg_accessible(bridge)) {
+		child->bus_flags |= PCI_BUS_FLAGS_NO_EXTCFG;
+		pci_info(child, "extended config space not accessible\n");
+	}
+
 	/* Set up default resource pointers and names */
 	for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
 		child->resource[i] = &bridge->resource[PCI_BRIDGE_RESOURCES+i];
@@ -1188,20 +1241,15 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
 		(is_cardbus ? "PCI CardBus %04x:%02x" : "PCI Bus %04x:%02x"),
 		pci_domain_nr(bus), child->number);
 
-	/* Has only triggered on CardBus, fixup is in yenta_socket */
+	/* Check that all devices are accessible */
 	while (bus->parent) {
 		if ((child->busn_res.end > bus->busn_res.end) ||
 		    (child->number > bus->busn_res.end) ||
 		    (child->number < bus->number) ||
 		    (child->busn_res.end < bus->number)) {
-			dev_info(&child->dev, "%pR %s hidden behind%s bridge %s %pR\n",
-				&child->busn_res,
-				(bus->number > child->busn_res.end &&
-				 bus->busn_res.end < child->number) ?
-					"wholly" : "partially",
-				bus->self->transparent ? " transparent" : "",
-				dev_name(&bus->dev),
-				&bus->busn_res);
+			dev_info(&dev->dev, "devices behind bridge are unusable because %pR cannot be assigned for them\n",
+				 &child->busn_res);
+			break;
 		}
 		bus = bus->parent;
 	}
@@ -1393,6 +1441,9 @@ int pci_cfg_space_size(struct pci_dev *dev)
 	u32 status;
 	u16 class;
 
+	if (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
+		return PCI_CFG_SPACE_SIZE;
+
 	class = dev->class >> 8;
 	if (class == PCI_CLASS_BRIDGE_HOST)
 		return pci_cfg_space_size_ext(dev);
@@ -1954,9 +2005,13 @@ static void pci_configure_relaxed_ordering(struct pci_dev *dev)
 static void pci_configure_ltr(struct pci_dev *dev)
 {
 #ifdef CONFIG_PCIEASPM
+	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
 	u32 cap;
 	struct pci_dev *bridge;
 
+	if (!host->native_ltr)
+		return;
+
 	if (!pci_is_pcie(dev))
 		return;
 
diff --git a/drivers/pci/proc.c b/drivers/pci/proc.c
index 1ee8927a0635..7ac035af39f0 100644
--- a/drivers/pci/proc.c
+++ b/drivers/pci/proc.c
@@ -435,25 +435,12 @@ int pci_proc_detach_bus(struct pci_bus *bus)
 	return 0;
 }
 
-static int proc_bus_pci_dev_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &proc_bus_pci_devices_op);
-}
-
-static const struct file_operations proc_bus_pci_dev_operations = {
-	.owner		= THIS_MODULE,
-	.open		= proc_bus_pci_dev_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
 static int __init pci_proc_init(void)
 {
 	struct pci_dev *dev = NULL;
 	proc_bus_pci_dir = proc_mkdir("bus/pci", NULL);
-	proc_create("devices", 0, proc_bus_pci_dir,
-		    &proc_bus_pci_dev_operations);
+	proc_create_seq("devices", 0, proc_bus_pci_dir,
+		    &proc_bus_pci_devices_op);
 	proc_initialized = 1;
 	for_each_pci_dev(dev)
 		pci_proc_attach_device(dev);
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index 2990ad1e7c99..f439de848658 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -30,6 +30,162 @@
 #include <asm/dma.h>	/* isa_dma_bridge_buggy */
 #include "pci.h"
 
+static ktime_t fixup_debug_start(struct pci_dev *dev,
+				 void (*fn)(struct pci_dev *dev))
+{
+	if (initcall_debug)
+		pci_info(dev, "calling  %pF @ %i\n", fn, task_pid_nr(current));
+
+	return ktime_get();
+}
+
+static void fixup_debug_report(struct pci_dev *dev, ktime_t calltime,
+			       void (*fn)(struct pci_dev *dev))
+{
+	ktime_t delta, rettime;
+	unsigned long long duration;
+
+	rettime = ktime_get();
+	delta = ktime_sub(rettime, calltime);
+	duration = (unsigned long long) ktime_to_ns(delta) >> 10;
+	if (initcall_debug || duration > 10000)
+		pci_info(dev, "%pF took %lld usecs\n", fn, duration);
+}
+
+static void pci_do_fixups(struct pci_dev *dev, struct pci_fixup *f,
+			  struct pci_fixup *end)
+{
+	ktime_t calltime;
+
+	for (; f < end; f++)
+		if ((f->class == (u32) (dev->class >> f->class_shift) ||
+		     f->class == (u32) PCI_ANY_ID) &&
+		    (f->vendor == dev->vendor ||
+		     f->vendor == (u16) PCI_ANY_ID) &&
+		    (f->device == dev->device ||
+		     f->device == (u16) PCI_ANY_ID)) {
+			calltime = fixup_debug_start(dev, f->hook);
+			f->hook(dev);
+			fixup_debug_report(dev, calltime, f->hook);
+		}
+}
+
+extern struct pci_fixup __start_pci_fixups_early[];
+extern struct pci_fixup __end_pci_fixups_early[];
+extern struct pci_fixup __start_pci_fixups_header[];
+extern struct pci_fixup __end_pci_fixups_header[];
+extern struct pci_fixup __start_pci_fixups_final[];
+extern struct pci_fixup __end_pci_fixups_final[];
+extern struct pci_fixup __start_pci_fixups_enable[];
+extern struct pci_fixup __end_pci_fixups_enable[];
+extern struct pci_fixup __start_pci_fixups_resume[];
+extern struct pci_fixup __end_pci_fixups_resume[];
+extern struct pci_fixup __start_pci_fixups_resume_early[];
+extern struct pci_fixup __end_pci_fixups_resume_early[];
+extern struct pci_fixup __start_pci_fixups_suspend[];
+extern struct pci_fixup __end_pci_fixups_suspend[];
+extern struct pci_fixup __start_pci_fixups_suspend_late[];
+extern struct pci_fixup __end_pci_fixups_suspend_late[];
+
+static bool pci_apply_fixup_final_quirks;
+
+void pci_fixup_device(enum pci_fixup_pass pass, struct pci_dev *dev)
+{
+	struct pci_fixup *start, *end;
+
+	switch (pass) {
+	case pci_fixup_early:
+		start = __start_pci_fixups_early;
+		end = __end_pci_fixups_early;
+		break;
+
+	case pci_fixup_header:
+		start = __start_pci_fixups_header;
+		end = __end_pci_fixups_header;
+		break;
+
+	case pci_fixup_final:
+		if (!pci_apply_fixup_final_quirks)
+			return;
+		start = __start_pci_fixups_final;
+		end = __end_pci_fixups_final;
+		break;
+
+	case pci_fixup_enable:
+		start = __start_pci_fixups_enable;
+		end = __end_pci_fixups_enable;
+		break;
+
+	case pci_fixup_resume:
+		start = __start_pci_fixups_resume;
+		end = __end_pci_fixups_resume;
+		break;
+
+	case pci_fixup_resume_early:
+		start = __start_pci_fixups_resume_early;
+		end = __end_pci_fixups_resume_early;
+		break;
+
+	case pci_fixup_suspend:
+		start = __start_pci_fixups_suspend;
+		end = __end_pci_fixups_suspend;
+		break;
+
+	case pci_fixup_suspend_late:
+		start = __start_pci_fixups_suspend_late;
+		end = __end_pci_fixups_suspend_late;
+		break;
+
+	default:
+		/* stupid compiler warning, you would think with an enum... */
+		return;
+	}
+	pci_do_fixups(dev, start, end);
+}
+EXPORT_SYMBOL(pci_fixup_device);
+
+static int __init pci_apply_final_quirks(void)
+{
+	struct pci_dev *dev = NULL;
+	u8 cls = 0;
+	u8 tmp;
+
+	if (pci_cache_line_size)
+		printk(KERN_DEBUG "PCI: CLS %u bytes\n",
+		       pci_cache_line_size << 2);
+
+	pci_apply_fixup_final_quirks = true;
+	for_each_pci_dev(dev) {
+		pci_fixup_device(pci_fixup_final, dev);
+		/*
+		 * If arch hasn't set it explicitly yet, use the CLS
+		 * value shared by all PCI devices.  If there's a
+		 * mismatch, fall back to the default value.
+		 */
+		if (!pci_cache_line_size) {
+			pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &tmp);
+			if (!cls)
+				cls = tmp;
+			if (!tmp || cls == tmp)
+				continue;
+
+			printk(KERN_DEBUG "PCI: CLS mismatch (%u != %u), using %u bytes\n",
+			       cls << 2, tmp << 2,
+			       pci_dfl_cache_line_size << 2);
+			pci_cache_line_size = pci_dfl_cache_line_size;
+		}
+	}
+
+	if (!pci_cache_line_size) {
+		printk(KERN_DEBUG "PCI: CLS %u bytes, default %u\n",
+		       cls << 2, pci_dfl_cache_line_size << 2);
+		pci_cache_line_size = cls ? cls : pci_dfl_cache_line_size;
+	}
+
+	return 0;
+}
+fs_initcall_sync(pci_apply_final_quirks);
+
 /*
  * Decoding should be disabled for a PCI device during BAR sizing to avoid
  * conflict. But doing so may cause problems on host bridge and perhaps other
@@ -43,9 +199,10 @@ static void quirk_mmio_always_on(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_ANY_ID, PCI_ANY_ID,
 				PCI_CLASS_BRIDGE_HOST, 8, quirk_mmio_always_on);
 
-/* The Mellanox Tavor device gives false positive parity errors
- * Mark this device with a broken_parity_status, to allow
- * PCI scanning code to "skip" this now blacklisted device.
+/*
+ * The Mellanox Tavor device gives false positive parity errors.  Mark this
+ * device with a broken_parity_status to allow PCI scanning code to "skip"
+ * this now blacklisted device.
  */
 static void quirk_mellanox_tavor(struct pci_dev *dev)
 {
@@ -54,15 +211,19 @@ static void quirk_mellanox_tavor(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR, quirk_mellanox_tavor);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR_BRIDGE, quirk_mellanox_tavor);
 
-/* Deal with broken BIOSes that neglect to enable passive release,
-   which can cause problems in combination with the 82441FX/PPro MTRRs */
+/*
+ * Deal with broken BIOSes that neglect to enable passive release,
+ * which can cause problems in combination with the 82441FX/PPro MTRRs
+ */
 static void quirk_passive_release(struct pci_dev *dev)
 {
 	struct pci_dev *d = NULL;
 	unsigned char dlc;
 
-	/* We have to make sure a particular bit is set in the PIIX3
-	   ISA bridge, so we have to go out and find it. */
+	/*
+	 * We have to make sure a particular bit is set in the PIIX3
+	 * ISA bridge, so we have to go out and find it.
+	 */
 	while ((d = pci_get_device(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371SB_0, d))) {
 		pci_read_config_byte(d, 0x82, &dlc);
 		if (!(dlc & 1<<1)) {
@@ -75,13 +236,14 @@ static void quirk_passive_release(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82441,	quirk_passive_release);
 DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82441,	quirk_passive_release);
 
-/*  The VIA VP2/VP3/MVP3 seem to have some 'features'. There may be a workaround
-    but VIA don't answer queries. If you happen to have good contacts at VIA
-    ask them for me please -- Alan
-
-    This appears to be BIOS not version dependent. So presumably there is a
-    chipset level fix */
-
+/*
+ * The VIA VP2/VP3/MVP3 seem to have some 'features'. There may be a
+ * workaround but VIA don't answer queries. If you happen to have good
+ * contacts at VIA ask them for me please -- Alan
+ *
+ * This appears to be BIOS not version dependent. So presumably there is a
+ * chipset level fix.
+ */
 static void quirk_isa_dma_hangs(struct pci_dev *dev)
 {
 	if (!isa_dma_bridge_buggy) {
@@ -89,10 +251,10 @@ static void quirk_isa_dma_hangs(struct pci_dev *dev)
 		pci_info(dev, "Activating ISA DMA hang workarounds\n");
 	}
 }
-	/*
-	 * Its not totally clear which chipsets are the problematic ones
-	 * We know 82C586 and 82C596 variants are affected.
-	 */
+/*
+ * It's not totally clear which chipsets are the problematic ones.  We know
+ * 82C586 and 82C596 variants are affected.
+ */
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_82C586_0,	quirk_isa_dma_hangs);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_82C596,	quirk_isa_dma_hangs);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL,    PCI_DEVICE_ID_INTEL_82371SB_0,  quirk_isa_dma_hangs);
@@ -121,9 +283,7 @@ static void quirk_tigerpoint_bm_sts(struct pci_dev *dev)
 }
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGP_LPC, quirk_tigerpoint_bm_sts);
 
-/*
- *	Chipsets where PCI->PCI transfers vanish or hang
- */
+/* Chipsets where PCI->PCI transfers vanish or hang */
 static void quirk_nopcipci(struct pci_dev *dev)
 {
 	if ((pci_pci_problems & PCIPCI_FAIL) == 0) {
@@ -146,9 +306,7 @@ static void quirk_nopciamd(struct pci_dev *dev)
 }
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD,	PCI_DEVICE_ID_AMD_8151_0,	quirk_nopciamd);
 
-/*
- *	Triton requires workarounds to be used by the drivers
- */
+/* Triton requires workarounds to be used by the drivers */
 static void quirk_triton(struct pci_dev *dev)
 {
 	if ((pci_pci_problems&PCIPCI_TRITON) == 0) {
@@ -162,53 +320,62 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82439,	quirk_tr
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82439TX,	quirk_triton);
 
 /*
- *	VIA Apollo KT133 needs PCI latency patch
- *	Made according to a windows driver based patch by George E. Breese
- *	see PCI Latency Adjust on http://www.viahardware.com/download/viatweak.shtm
- *	Also see http://www.au-ja.org/review-kt133a-1-en.phtml for
- *	the info on which Mr Breese based his work.
+ * VIA Apollo KT133 needs PCI latency patch
+ * Made according to a Windows driver-based patch by George E. Breese;
+ * see PCI Latency Adjust on http://www.viahardware.com/download/viatweak.shtm
+ * Also see http://www.au-ja.org/review-kt133a-1-en.phtml for the info on
+ * which Mr Breese based his work.
  *
- *	Updated based on further information from the site and also on
- *	information provided by VIA
+ * Updated based on further information from the site and also on
+ * information provided by VIA
  */
 static void quirk_vialatency(struct pci_dev *dev)
 {
 	struct pci_dev *p;
 	u8 busarb;
-	/* Ok we have a potential problem chipset here. Now see if we have
-	   a buggy southbridge */
 
+	/*
+	 * Ok, we have a potential problem chipset here. Now see if we have
+	 * a buggy southbridge.
+	 */
 	p = pci_get_device(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_82C686, NULL);
 	if (p != NULL) {
-		/* 0x40 - 0x4f == 686B, 0x10 - 0x2f == 686A; thanks Dan Hollis */
-		/* Check for buggy part revisions */
+
+		/*
+		 * 0x40 - 0x4f == 686B, 0x10 - 0x2f == 686A;
+		 * thanks Dan Hollis.
+		 * Check for buggy part revisions
+		 */
 		if (p->revision < 0x40 || p->revision > 0x42)
 			goto exit;
 	} else {
 		p = pci_get_device(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8231, NULL);
 		if (p == NULL)	/* No problem parts */
 			goto exit;
+
 		/* Check for buggy part revisions */
 		if (p->revision < 0x10 || p->revision > 0x12)
 			goto exit;
 	}
 
 	/*
-	 *	Ok we have the problem. Now set the PCI master grant to
-	 *	occur every master grant. The apparent bug is that under high
-	 *	PCI load (quite common in Linux of course) you can get data
-	 *	loss when the CPU is held off the bus for 3 bus master requests
-	 *	This happens to include the IDE controllers....
+	 * Ok we have the problem. Now set the PCI master grant to occur
+	 * every master grant. The apparent bug is that under high PCI load
+	 * (quite common in Linux of course) you can get data loss when the
+	 * CPU is held off the bus for 3 bus master requests.  This happens
+	 * to include the IDE controllers....
 	 *
-	 *	VIA only apply this fix when an SB Live! is present but under
-	 *	both Linux and Windows this isn't enough, and we have seen
-	 *	corruption without SB Live! but with things like 3 UDMA IDE
-	 *	controllers. So we ignore that bit of the VIA recommendation..
+	 * VIA only apply this fix when an SB Live! is present but under
+	 * both Linux and Windows this isn't enough, and we have seen
+	 * corruption without SB Live! but with things like 3 UDMA IDE
+	 * controllers. So we ignore that bit of the VIA recommendation..
 	 */
-
 	pci_read_config_byte(dev, 0x76, &busarb);
-	/* Set bit 4 and bi 5 of byte 76 to 0x01
-	   "Master priority rotation on every PCI master grant */
+
+	/*
+	 * Set bit 4 and bit 5 of byte 76 to 0x01
+	 * "Master priority rotation on every PCI master grant"
+	 */
 	busarb &= ~(1<<5);
 	busarb |= (1<<4);
 	pci_write_config_byte(dev, 0x76, busarb);
@@ -224,9 +391,7 @@ DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_8363_0,	quirk_vial
 DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_8371_1,	quirk_vialatency);
 DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_8361,		quirk_vialatency);
 
-/*
- *	VIA Apollo VP3 needs ETBF on BT848/878
- */
+/* VIA Apollo VP3 needs ETBF on BT848/878 */
 static void quirk_viaetbf(struct pci_dev *dev)
 {
 	if ((pci_pci_problems&PCIPCI_VIAETBF) == 0) {
@@ -246,10 +411,9 @@ static void quirk_vsfx(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_82C576,	quirk_vsfx);
 
 /*
- *	Ali Magik requires workarounds to be used by the drivers
- *	that DMA to AGP space. Latency must be set to 0xA and triton
- *	workaround applied too
- *	[Info kindly provided by ALi]
+ * ALi Magik requires workarounds to be used by the drivers that DMA to AGP
+ * space. Latency must be set to 0xA and Triton workaround applied too.
+ * [Info kindly provided by ALi]
  */
 static void quirk_alimagik(struct pci_dev *dev)
 {
@@ -261,10 +425,7 @@ static void quirk_alimagik(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AL,	PCI_DEVICE_ID_AL_M1647,		quirk_alimagik);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AL,	PCI_DEVICE_ID_AL_M1651,		quirk_alimagik);
 
-/*
- *	Natoma has some interesting boundary conditions with Zoran stuff
- *	at least
- */
+/* Natoma has some interesting boundary conditions with Zoran stuff at least */
 static void quirk_natoma(struct pci_dev *dev)
 {
 	if ((pci_pci_problems&PCIPCI_NATOMA) == 0) {
@@ -280,8 +441,8 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82443BX_1,	quir
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82443BX_2,	quirk_natoma);
 
 /*
- *  This chip can cause PCI parity errors if config register 0xA0 is read
- *  while DMAs are occurring.
+ * This chip can cause PCI parity errors if config register 0xA0 is read
+ * while DMAs are occurring.
  */
 static void quirk_citrine(struct pci_dev *dev)
 {
@@ -321,8 +482,8 @@ static void quirk_extend_bar_to_page(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_IBM, 0x034a, quirk_extend_bar_to_page);
 
 /*
- *  S3 868 and 968 chips report region size equal to 32M, but they decode 64M.
- *  If it's needed, re-allocate the region.
+ * S3 868 and 968 chips report region size equal to 32M, but they decode 64M.
+ * If it's needed, re-allocate the region.
  */
 static void quirk_s3_64M(struct pci_dev *dev)
 {
@@ -413,8 +574,8 @@ static void quirk_io_region(struct pci_dev *dev, int port,
 }
 
 /*
- *	ATI Northbridge setups MCE the processor if you even
- *	read somewhere between 0x3b0->0x3bb or read 0x3d3
+ * ATI Northbridge setups MCE the processor if you even read somewhere
+ * between 0x3b0->0x3bb or read 0x3d3
  */
 static void quirk_ati_exploding_mce(struct pci_dev *dev)
 {
@@ -429,6 +590,7 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI,	PCI_DEVICE_ID_ATI_RS100,   quirk_ati_
  * In the AMD NL platform, this device ([1022:7912]) has a class code of
  * PCI_CLASS_SERIAL_USB_XHCI (0x0c0330), which means the xhci driver will
  * claim it.
+ *
  * But the dwc3 driver is a more specific driver for this device, and we'd
  * prefer to use it instead of xhci. To prevent xhci from claiming the
  * device, change the class code to 0x0c03fe, which the PCI r3.0 spec
@@ -448,11 +610,10 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_NL_USB,
 		quirk_amd_nl_class);
 
 /*
- * Let's make the southbridge information explicit instead
- * of having to worry about people probing the ACPI areas,
- * for example.. (Yes, it happens, and if you read the wrong
- * ACPI register it will put the machine to sleep with no
- * way of waking it up again. Bummer).
+ * Let's make the southbridge information explicit instead of having to
+ * worry about people probing the ACPI areas, for example.. (Yes, it
+ * happens, and if you read the wrong ACPI register it will put the machine
+ * to sleep with no way of waking it up again. Bummer).
  *
  * ALI M7101: Two IO regions pointed to by words at
  *	0xE0 (64 bytes of ACPI registers)
@@ -508,6 +669,7 @@ static void piix4_mem_quirk(struct pci_dev *dev, const char *name, unsigned int
 			break;
 		size = bit;
 	}
+
 	/*
 	 * For now we only print it out. Eventually we'll want to
 	 * reserve it, but let's get enough confirmation reports first.
@@ -579,8 +741,7 @@ static void quirk_ich4_lpc_acpi(struct pci_dev *dev)
 	 * priority and can't tell whether the legacy device or the one created
 	 * here is really at that address.  This happens on boards with broken
 	 * BIOSes.
-	*/
-
+	 */
 	pci_read_config_byte(dev, ICH_ACPI_CNTL, &enable);
 	if (enable & ICH4_ACPI_EN)
 		quirk_io_region(dev, ICH_PMBASE, 128, PCI_BRIDGE_RESOURCES,
@@ -617,7 +778,8 @@ static void ich6_lpc_acpi_gpio(struct pci_dev *dev)
 				"ICH6 GPIO");
 }
 
-static void ich6_lpc_generic_decode(struct pci_dev *dev, unsigned reg, const char *name, int dynsize)
+static void ich6_lpc_generic_decode(struct pci_dev *dev, unsigned reg,
+				    const char *name, int dynsize)
 {
 	u32 val;
 	u32 size, base;
@@ -641,7 +803,10 @@ static void ich6_lpc_generic_decode(struct pci_dev *dev, unsigned reg, const cha
 	}
 	base &= ~(size-1);
 
-	/* Just print it out for now. We should reserve it after more debugging */
+	/*
+	 * Just print it out for now. We should reserve it after more
+	 * debugging.
+	 */
 	pci_info(dev, "%s PIO at %04x-%04x\n", name, base, base+size-1);
 }
 
@@ -657,7 +822,8 @@ static void quirk_ich6_lpc(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_ICH6_0, quirk_ich6_lpc);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_ICH6_1, quirk_ich6_lpc);
 
-static void ich7_lpc_generic_decode(struct pci_dev *dev, unsigned reg, const char *name)
+static void ich7_lpc_generic_decode(struct pci_dev *dev, unsigned reg,
+				    const char *name)
 {
 	u32 val;
 	u32 mask, base;
@@ -668,15 +834,15 @@ static void ich7_lpc_generic_decode(struct pci_dev *dev, unsigned reg, const cha
 	if (!(val & 1))
 		return;
 
-	/*
-	 * IO base in bits 15:2, mask in bits 23:18, both
-	 * are dword-based
-	 */
+	/* IO base in bits 15:2, mask in bits 23:18, both are dword-based */
 	base = val & 0xfffc;
 	mask = (val >> 16) & 0xfc;
 	mask |= 3;
 
-	/* Just print it out for now. We should reserve it after more debugging */
+	/*
+	 * Just print it out for now. We should reserve it after more
+	 * debugging.
+	 */
 	pci_info(dev, "%s PIO at %04x (mask %04x)\n", name, base, mask);
 }
 
@@ -748,8 +914,8 @@ static void quirk_vt8235_acpi(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_8235,	quirk_vt8235_acpi);
 
 /*
- * TI XIO2000a PCIe-PCI Bridge erroneously reports it supports fast back-to-back:
- *	Disable fast back-to-back on the secondary bus segment
+ * TI XIO2000a PCIe-PCI Bridge erroneously reports it supports fast
+ * back-to-back: Disable fast back-to-back on the secondary bus segment
  */
 static void quirk_xio2000a(struct pci_dev *dev)
 {
@@ -774,8 +940,8 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_XIO2000A,
  * VIA 686A/B: If an IO-APIC is active, we need to route all on-chip
  * devices to the external APIC.
  *
- * TODO: When we have device-specific interrupt routers,
- * this code will go away from quirks.
+ * TODO: When we have device-specific interrupt routers, this code will go
+ * away from quirks.
  */
 static void quirk_via_ioapic(struct pci_dev *dev)
 {
@@ -816,13 +982,13 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_8237,		quirk_via_vt
 DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_8237,		quirk_via_vt8237_bypass_apic_deassert);
 
 /*
- * The AMD io apic can hang the box when an apic irq is masked.
+ * The AMD IO-APIC can hang the box when an APIC IRQ is masked.
  * We check all revs >= B0 (yet not in the pre production!) as the bug
  * is currently marked NoFix
  *
  * We have multiple reports of hangs with this chipset that went away with
  * noapic specified. For the moment we assume it's the erratum. We may be wrong
- * of course. However the advice is demonstrably good even if so..
+ * of course. However the advice is demonstrably good even if so.
  */
 static void quirk_amd_ioapic(struct pci_dev *dev)
 {
@@ -838,7 +1004,7 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD,	PCI_DEVICE_ID_AMD_VIPER_7410,	quirk_a
 
 static void quirk_cavium_sriov_rnm_link(struct pci_dev *dev)
 {
-	/* Fix for improper SRIOV configuration on Cavium cn88xx  RNM device */
+	/* Fix for improper SR-IOV configuration on Cavium cn88xx RNM device */
 	if (dev->subsystem_device == 0xa118)
 		dev->sriov->link = dev->devfn;
 }
@@ -860,19 +1026,17 @@ static void quirk_amd_8131_mmrbc(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_8131_BRIDGE, quirk_amd_8131_mmrbc);
 
 /*
- * FIXME: it is questionable that quirk_via_acpi
- * is needed.  It shows up as an ISA bridge, and does not
- * support the PCI_INTERRUPT_LINE register at all.  Therefore
- * it seems like setting the pci_dev's 'irq' to the
- * value of the ACPI SCI interrupt is only done for convenience.
+ * FIXME: it is questionable that quirk_via_acpi() is needed.  It shows up
+ * as an ISA bridge, and does not support the PCI_INTERRUPT_LINE register
+ * at all.  Therefore it seems like setting the pci_dev's IRQ to the value
+ * of the ACPI SCI interrupt is only done for convenience.
  *	-jgarzik
  */
 static void quirk_via_acpi(struct pci_dev *d)
 {
-	/*
-	 * VIA ACPI device: SCI IRQ line in PCI config byte 0x42
-	 */
 	u8 irq;
+
+	/* VIA ACPI device: SCI IRQ line in PCI config byte 0x42 */
 	pci_read_config_byte(d, 0x42, &irq);
 	irq &= 0xf;
 	if (irq && (irq != 2))
@@ -881,11 +1045,7 @@ static void quirk_via_acpi(struct pci_dev *d)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_82C586_3,	quirk_via_acpi);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_82C686_4,	quirk_via_acpi);
 
-
-/*
- *	VIA bridges which have VLink
- */
-
+/* VIA bridges which have VLink */
 static int via_vlink_dev_lo = -1, via_vlink_dev_hi = 18;
 
 static void quirk_via_bridge(struct pci_dev *dev)
@@ -893,9 +1053,11 @@ static void quirk_via_bridge(struct pci_dev *dev)
 	/* See what bridge we have and find the device ranges */
 	switch (dev->device) {
 	case PCI_DEVICE_ID_VIA_82C686:
-		/* The VT82C686 is special, it attaches to PCI and can have
-		   any device number. All its subdevices are functions of
-		   that single device. */
+		/*
+		 * The VT82C686 is special; it attaches to PCI and can have
+		 * any device number. All its subdevices are functions of
+		 * that single device.
+		 */
 		via_vlink_dev_lo = PCI_SLOT(dev->devfn);
 		via_vlink_dev_hi = PCI_SLOT(dev->devfn);
 		break;
@@ -923,19 +1085,17 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_8235,		quirk_via_b
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_8237,		quirk_via_bridge);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_8237A,	quirk_via_bridge);
 
-/**
- *	quirk_via_vlink		-	VIA VLink IRQ number update
- *	@dev: PCI device
+/*
+ * quirk_via_vlink		-	VIA VLink IRQ number update
+ * @dev: PCI device
  *
- *	If the device we are dealing with is on a PIC IRQ we need to
- *	ensure that the IRQ line register which usually is not relevant
- *	for PCI cards, is actually written so that interrupts get sent
- *	to the right place.
- *	We only do this on systems where a VIA south bridge was detected,
- *	and only for VIA devices on the motherboard (see quirk_via_bridge
- *	above).
+ * If the device we are dealing with is on a PIC IRQ we need to ensure that
+ * the IRQ line register which usually is not relevant for PCI cards, is
+ * actually written so that interrupts get sent to the right place.
+ *
+ * We only do this on systems where a VIA south bridge was detected, and
+ * only for VIA devices on the motherboard (see quirk_via_bridge above).
  */
-
 static void quirk_via_vlink(struct pci_dev *dev)
 {
 	u8 irq, new_irq;
@@ -955,9 +1115,10 @@ static void quirk_via_vlink(struct pci_dev *dev)
 	    PCI_SLOT(dev->devfn) < via_vlink_dev_lo)
 		return;
 
-	/* This is an internal VLink device on a PIC interrupt. The BIOS
-	   ought to have set this but may not have, so we redo it */
-
+	/*
+	 * This is an internal VLink device on a PIC interrupt. The BIOS
+	 * ought to have set this but may not have, so we redo it.
+	 */
 	pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &irq);
 	if (new_irq != irq) {
 		pci_info(dev, "VIA VLink IRQ fixup, from %d to %d\n",
@@ -969,10 +1130,9 @@ static void quirk_via_vlink(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_VIA, PCI_ANY_ID, quirk_via_vlink);
 
 /*
- * VIA VT82C598 has its device ID settable and many BIOSes
- * set it to the ID of VT82C597 for backward compatibility.
- * We need to switch it off to be able to recognize the real
- * type of the chip.
+ * VIA VT82C598 has its device ID settable and many BIOSes set it to the ID
+ * of VT82C597 for backward compatibility.  We need to switch it off to be
+ * able to recognize the real type of the chip.
  */
 static void quirk_vt82c598_id(struct pci_dev *dev)
 {
@@ -982,10 +1142,10 @@ static void quirk_vt82c598_id(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_82C597_0,	quirk_vt82c598_id);
 
 /*
- * CardBus controllers have a legacy base address that enables them
- * to respond as i82365 pcmcia controllers.  We don't want them to
- * do this even if the Linux CardBus driver is not loaded, because
- * the Linux i82365 driver does not (and should not) handle CardBus.
+ * CardBus controllers have a legacy base address that enables them to
+ * respond as i82365 pcmcia controllers.  We don't want them to do this
+ * even if the Linux CardBus driver is not loaded, because the Linux i82365
+ * driver does not (and should not) handle CardBus.
  */
 static void quirk_cardbus_legacy(struct pci_dev *dev)
 {
@@ -997,11 +1157,11 @@ DECLARE_PCI_FIXUP_CLASS_RESUME_EARLY(PCI_ANY_ID, PCI_ANY_ID,
 			PCI_CLASS_BRIDGE_CARDBUS, 8, quirk_cardbus_legacy);
 
 /*
- * Following the PCI ordering rules is optional on the AMD762. I'm not
- * sure what the designers were smoking but let's not inhale...
+ * Following the PCI ordering rules is optional on the AMD762. I'm not sure
+ * what the designers were smoking but let's not inhale...
  *
- * To be fair to AMD, it follows the spec by default, its BIOS people
- * who turn it off!
+ * To be fair to AMD, it follows the spec by default, it's BIOS people who
+ * turn it off!
  */
 static void quirk_amd_ordering(struct pci_dev *dev)
 {
@@ -1020,11 +1180,11 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD,	PCI_DEVICE_ID_AMD_FE_GATE_700C, quirk
 DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_AMD,	PCI_DEVICE_ID_AMD_FE_GATE_700C, quirk_amd_ordering);
 
 /*
- *	DreamWorks provided workaround for Dunord I-3000 problem
+ * DreamWorks-provided workaround for Dunord I-3000 problem
  *
- *	This card decodes and responds to addresses not apparently
- *	assigned to it. We force a larger allocation to ensure that
- *	nothing gets put too close to it.
+ * This card decodes and responds to addresses not apparently assigned to
+ * it.  We force a larger allocation to ensure that nothing gets put too
+ * close to it.
  */
 static void quirk_dunord(struct pci_dev *dev)
 {
@@ -1037,10 +1197,9 @@ static void quirk_dunord(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_DUNORD,	PCI_DEVICE_ID_DUNORD_I3000,	quirk_dunord);
 
 /*
- * i82380FB mobile docking controller: its PCI-to-PCI bridge
- * is subtractive decoding (transparent), and does indicate this
- * in the ProgIf. Unfortunately, the ProgIf value is wrong - 0x80
- * instead of 0x01.
+ * i82380FB mobile docking controller: its PCI-to-PCI bridge is subtractive
+ * decoding (transparent), and does indicate this in the ProgIf.
+ * Unfortunately, the ProgIf value is wrong - 0x80 instead of 0x01.
  */
 static void quirk_transparent_bridge(struct pci_dev *dev)
 {
@@ -1050,10 +1209,10 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82380FB,	quirk
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TOSHIBA,	0x605,	quirk_transparent_bridge);
 
 /*
- * Common misconfiguration of the MediaGX/Geode PCI master that will
- * reduce PCI bandwidth from 70MB/s to 25MB/s.  See the GXM/GXLV/GX1
- * datasheets found at http://www.national.com/analog for info on what
- * these bits do.  <christer@weinigel.se>
+ * Common misconfiguration of the MediaGX/Geode PCI master that will reduce
+ * PCI bandwidth from 70MB/s to 25MB/s.  See the GXM/GXLV/GX1 datasheets
+ * found at http://www.national.com/analog for info on what these bits do.
+ * <christer@weinigel.se>
  */
 static void quirk_mediagx_master(struct pci_dev *dev)
 {
@@ -1071,9 +1230,9 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CYRIX,	PCI_DEVICE_ID_CYRIX_PCI_MASTER, qui
 DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_CYRIX,	PCI_DEVICE_ID_CYRIX_PCI_MASTER, quirk_mediagx_master);
 
 /*
- *	Ensure C0 rev restreaming is off. This is normally done by
- *	the BIOS but in the odd case it is not the results are corruption
- *	hence the presence of a Linux check
+ * Ensure C0 rev restreaming is off. This is normally done by the BIOS but
+ * in the odd case it is not the results are corruption hence the presence
+ * of a Linux check.
  */
 static void quirk_disable_pxb(struct pci_dev *pdev)
 {
@@ -1117,9 +1276,7 @@ DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_HUDSON2_SATA
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, 0x7900, quirk_amd_ide_mode);
 DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_AMD, 0x7900, quirk_amd_ide_mode);
 
-/*
- *	Serverworks CSB5 IDE does not fully support native mode
- */
+/* Serverworks CSB5 IDE does not fully support native mode */
 static void quirk_svwks_csb5ide(struct pci_dev *pdev)
 {
 	u8 prog;
@@ -1133,9 +1290,7 @@ static void quirk_svwks_csb5ide(struct pci_dev *pdev)
 }
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, PCI_DEVICE_ID_SERVERWORKS_CSB5IDE, quirk_svwks_csb5ide);
 
-/*
- *	Intel 82801CAM ICH3-M datasheet says IDE modes must be the same
- */
+/* Intel 82801CAM ICH3-M datasheet says IDE modes must be the same */
 static void quirk_ide_samemode(struct pci_dev *pdev)
 {
 	u8 prog;
@@ -1151,10 +1306,7 @@ static void quirk_ide_samemode(struct pci_dev *pdev)
 }
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801CA_10, quirk_ide_samemode);
 
-/*
- * Some ATA devices break if put into D3
- */
-
+/* Some ATA devices break if put into D3 */
 static void quirk_no_ata_d3(struct pci_dev *pdev)
 {
 	pdev->dev_flags |= PCI_DEV_FLAGS_NO_D3;
@@ -1172,7 +1324,8 @@ DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_AL, PCI_ANY_ID,
 DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_VIA, PCI_ANY_ID,
 				PCI_CLASS_STORAGE_IDE, 8, quirk_no_ata_d3);
 
-/* This was originally an Alpha specific thing, but it really fits here.
+/*
+ * This was originally an Alpha-specific thing, but it really fits here.
  * The i82375 PCI/EISA bridge appears as non-classified. Fix that.
  */
 static void quirk_eisa_bridge(struct pci_dev *dev)
@@ -1181,7 +1334,6 @@ static void quirk_eisa_bridge(struct pci_dev *dev)
 }
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82375,	quirk_eisa_bridge);
 
-
 /*
  * On ASUS P4B boards, the SMBus PCI Device within the ICH2/4 southbridge
  * is not activated. The myth is that Asus said that they do not want the
@@ -1398,15 +1550,19 @@ static void asus_hides_smbus_lpc_ich6_resume_early(struct pci_dev *dev)
 
 	if (likely(!asus_hides_smbus || !asus_rcba_base))
 		return;
+
 	/* read the Function Disable register, dword mode only */
 	val = readl(asus_rcba_base + 0x3418);
-	writel(val & 0xFFFFFFF7, asus_rcba_base + 0x3418); /* enable the SMBus device */
+
+	/* enable the SMBus device */
+	writel(val & 0xFFFFFFF7, asus_rcba_base + 0x3418);
 }
 
 static void asus_hides_smbus_lpc_ich6_resume(struct pci_dev *dev)
 {
 	if (likely(!asus_hides_smbus || !asus_rcba_base))
 		return;
+
 	iounmap(asus_rcba_base);
 	asus_rcba_base = NULL;
 	pci_info(dev, "Enabled ICH6/i801 SMBus device\n");
@@ -1423,9 +1579,7 @@ DECLARE_PCI_FIXUP_SUSPEND(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_ICH6_1,	asus_
 DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_ICH6_1,	asus_hides_smbus_lpc_ich6_resume);
 DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_ICH6_1,	asus_hides_smbus_lpc_ich6_resume_early);
 
-/*
- * SiS 96x south bridge: BIOS typically hides SMBus device...
- */
+/* SiS 96x south bridge: BIOS typically hides SMBus device...  */
 static void quirk_sis_96x_smbus(struct pci_dev *dev)
 {
 	u8 val = 0;
@@ -1448,7 +1602,7 @@ DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_SI,	PCI_DEVICE_ID_SI_LPC,		quirk_si
  * ... This is further complicated by the fact that some SiS96x south
  * bridges pretend to be 85C503/5513 instead.  In that case see if we
  * spotted a compatible north bridge to make sure.
- * (pci_find_device doesn't work yet)
+ * (pci_find_device() doesn't work yet)
  *
  * We can also enable the sis96x bit in the discovery register..
  */
@@ -1468,9 +1622,9 @@ static void quirk_sis_503(struct pci_dev *dev)
 	}
 
 	/*
-	 * Ok, it now shows up as a 96x.. run the 96x quirk by
-	 * hand in case it has already been processed.
-	 * (depends on link order, which is apparently not guaranteed)
+	 * Ok, it now shows up as a 96x.  Run the 96x quirk by hand in case
+	 * it has already been processed.  (Depends on link order, which is
+	 * apparently not guaranteed)
 	 */
 	dev->device = devid;
 	quirk_sis_96x_smbus(dev);
@@ -1478,7 +1632,6 @@ static void quirk_sis_503(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_SI,	PCI_DEVICE_ID_SI_503,		quirk_sis_503);
 DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_SI,	PCI_DEVICE_ID_SI_503,		quirk_sis_503);
 
-
 /*
  * On ASUS A8V and A8V Deluxe boards, the onboard AC97 audio controller
  * and MC97 modem controller are disabled when a second PCI soundcard is
@@ -1515,9 +1668,8 @@ DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_VIA,	PCI_DEVICE_ID_VIA_8237, asus_h
 #if defined(CONFIG_ATA) || defined(CONFIG_ATA_MODULE)
 
 /*
- *	If we are using libata we can drive this chip properly but must
- *	do this early on to make the additional device appear during
- *	the PCI scanning.
+ * If we are using libata we can drive this chip properly but must do this
+ * early on to make the additional device appear during the PCI scanning.
  */
 static void quirk_jmicron_ata(struct pci_dev *pdev)
 {
@@ -1613,14 +1765,18 @@ static void quirk_alder_ioapic(struct pci_dev *pdev)
 	if ((pdev->class >> 8) != 0xff00)
 		return;
 
-	/* the first BAR is the location of the IO APIC...we must
+	/*
+	 * The first BAR is the location of the IO-APIC... we must
 	 * not touch this (and it's already covered by the fixmap), so
-	 * forcibly insert it into the resource tree */
+	 * forcibly insert it into the resource tree.
+	 */
 	if (pci_resource_start(pdev, 0) && pci_resource_len(pdev, 0))
 		insert_resource(&iomem_resource, &pdev->resource[0]);
 
-	/* The next five BARs all seem to be rubbish, so just clean
-	 * them out */
+	/*
+	 * The next five BARs all seem to be rubbish, so just clean
+	 * them out.
+	 */
 	for (i = 1; i < 6; i++)
 		memset(&pdev->resource[i], 0, sizeof(pdev->resource[i]));
 }
@@ -1638,8 +1794,8 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_E7525_MCH,	quir
 DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_HUAWEI, 0x1610, PCI_CLASS_BRIDGE_PCI, 8, quirk_pcie_mch);
 
 /*
- * It's possible for the MSI to get corrupted if shpc and acpi
- * are used together on certain PXH-based systems.
+ * It's possible for the MSI to get corrupted if SHPC and ACPI are used
+ * together on certain PXH-based systems.
  */
 static void quirk_pcie_pxh(struct pci_dev *dev)
 {
@@ -1653,15 +1809,14 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_PXH_1,	quirk_pc
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_PXHV,	quirk_pcie_pxh);
 
 /*
- * Some Intel PCI Express chipsets have trouble with downstream
- * device power management.
+ * Some Intel PCI Express chipsets have trouble with downstream device
+ * power management.
  */
 static void quirk_intel_pcie_pm(struct pci_dev *dev)
 {
 	pci_pm_d3_delay = 120;
 	dev->no_d1d2 = 1;
 }
-
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL,	0x25e2, quirk_intel_pcie_pm);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL,	0x25e3, quirk_intel_pcie_pm);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL,	0x25e4, quirk_intel_pcie_pm);
@@ -1723,7 +1878,7 @@ static const struct dmi_system_id boot_interrupt_dmi_table[] = {
 
 /*
  * Boot interrupts on some chipsets cannot be turned off. For these chipsets,
- * remap the original interrupt in the linux kernel to the boot interrupt, so
+ * remap the original interrupt in the Linux kernel to the boot interrupt, so
  * that a PCI device's interrupt handler is installed on the boot interrupt
  * line instead.
  */
@@ -1760,7 +1915,7 @@ DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_80332_1,	quirk
  */
 
 /*
- * IO-APIC1 on 6300ESB generates boot interrupts, see intel order no
+ * IO-APIC1 on 6300ESB generates boot interrupts, see Intel order no
  * 300641-004US, section 5.7.3.
  */
 #define INTEL_6300_IOAPIC_ABAR		0x40
@@ -1783,9 +1938,7 @@ static void quirk_disable_intel_boot_interrupt(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL,   PCI_DEVICE_ID_INTEL_ESB_10,	quirk_disable_intel_boot_interrupt);
 DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_INTEL,   PCI_DEVICE_ID_INTEL_ESB_10,	quirk_disable_intel_boot_interrupt);
 
-/*
- * disable boot interrupts on HT-1000
- */
+/* Disable boot interrupts on HT-1000 */
 #define BC_HT1000_FEATURE_REG		0x64
 #define BC_HT1000_PIC_REGS_ENABLE	(1<<0)
 #define BC_HT1000_MAP_IDX		0xC00
@@ -1816,9 +1969,8 @@ static void quirk_disable_broadcom_boot_interrupt(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_SERVERWORKS,   PCI_DEVICE_ID_SERVERWORKS_HT1000SB,	quirk_disable_broadcom_boot_interrupt);
 DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_SERVERWORKS,   PCI_DEVICE_ID_SERVERWORKS_HT1000SB,	quirk_disable_broadcom_boot_interrupt);
 
-/*
- * disable boot interrupts on AMD and ATI chipsets
- */
+/* Disable boot interrupts on AMD and ATI chipsets */
+
 /*
  * NOIOAMODE needs to be disabled to disable "boot interrupts". For AMD 8131
  * rev. A0 and B0, NOIOAMODE needs to be disabled anyway to fix IO-APIC mode
@@ -1894,7 +2046,7 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TOSHIBA_2,
 			 quirk_tc86c001_ide);
 
 /*
- * PLX PCI 9050 PCI Target bridge controller has an errata that prevents the
+ * PLX PCI 9050 PCI Target bridge controller has an erratum that prevents the
  * local configuration registers accessible via BAR0 (memory) or BAR1 (i/o)
  * being read correctly if bit 7 of the base address is set.
  * The BAR0 or BAR1 region may be disabled (size 0) or enabled (size 128).
@@ -2087,15 +2239,17 @@ static void quirk_p64h2_1k_io(struct pci_dev *dev)
 		dev->io_window_1k = 1;
 	}
 }
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL,	0x1460,		quirk_p64h2_1k_io);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1460, quirk_p64h2_1k_io);
 
-/* Under some circumstances, AER is not linked with extended capabilities.
+/*
+ * Under some circumstances, AER is not linked with extended capabilities.
  * Force it to be linked by setting the corresponding control bit in the
  * config space.
  */
 static void quirk_nvidia_ck804_pcie_aer_ext_cap(struct pci_dev *dev)
 {
 	uint8_t b;
+
 	if (pci_read_config_byte(dev, 0xf41, &b) == 0) {
 		if (!(b & 0x20)) {
 			pci_write_config_byte(dev, 0xf41, b | 0x20);
@@ -2125,8 +2279,10 @@ static void quirk_via_cx700_pci_parking_caching(struct pci_dev *dev)
 		PCI_DEVICE_ID_VIA_8235_USB_2, NULL);
 	uint8_t b;
 
-	/* p should contain the first (internal) VT6212L -- see if we have
-	   an external one by searching again */
+	/*
+	 * p should contain the first (internal) VT6212L -- see if we have
+	 * an external one by searching again.
+	 */
 	p = pci_get_device(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8235_USB_2, p);
 	if (!p)
 		return;
@@ -2171,7 +2327,6 @@ static void quirk_brcm_5719_limit_mrrs(struct pci_dev *dev)
 			pcie_set_readrq(dev, 2048);
 	}
 }
-
 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_BROADCOM,
 			 PCI_DEVICE_ID_TIGON3_5719,
 			 quirk_brcm_5719_limit_mrrs);
@@ -2179,14 +2334,16 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_BROADCOM,
 #ifdef CONFIG_PCIE_IPROC_PLATFORM
 static void quirk_paxc_bridge(struct pci_dev *pdev)
 {
-	/* The PCI config space is shared with the PAXC root port and the first
+	/*
+	 * The PCI config space is shared with the PAXC root port and the first
 	 * Ethernet device.  So, we need to workaround this by telling the PCI
 	 * code that the bridge is not an Ethernet device.
 	 */
 	if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
 		pdev->class = PCI_CLASS_BRIDGE_PCI << 8;
 
-	/* MPSS is not being set properly (as it is currently 0).  This is
+	/*
+	 * MPSS is not being set properly (as it is currently 0).  This is
 	 * because that area of the PCI config space is hard coded to zero, and
 	 * is not modifiable by firmware.  Set this to 2 (e.g., 512 byte MPS)
 	 * so that the MPS can be set to the real max value.
@@ -2197,10 +2354,10 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16cd, quirk_paxc_bridge);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, quirk_paxc_bridge);
 #endif
 
-/* Originally in EDAC sources for i82875P:
- * Intel tells BIOS developers to hide device 6 which
- * configures the overflow device access containing
- * the DRBs - this is where we expose device 6.
+/*
+ * Originally in EDAC sources for i82875P: Intel tells BIOS developers to
+ * hide device 6 which configures the overflow device access containing the
+ * DRBs - this is where we expose device 6.
  * http://www.x86-secret.com/articles/tweak/pat/patsecrets-2.htm
  */
 static void quirk_unhide_mch_dev6(struct pci_dev *dev)
@@ -2212,18 +2369,18 @@ static void quirk_unhide_mch_dev6(struct pci_dev *dev)
 		pci_write_config_byte(dev, 0xF4, reg | 0x02);
 	}
 }
-
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82865_HB,
 			quirk_unhide_mch_dev6);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82875_HB,
 			quirk_unhide_mch_dev6);
 
 #ifdef CONFIG_PCI_MSI
-/* Some chipsets do not support MSI. We cannot easily rely on setting
- * PCI_BUS_FLAGS_NO_MSI in its bus flags because there are actually
- * some other buses controlled by the chipset even if Linux is not
- * aware of it.  Instead of setting the flag on all buses in the
- * machine, simply disable MSI globally.
+/*
+ * Some chipsets do not support MSI. We cannot easily rely on setting
+ * PCI_BUS_FLAGS_NO_MSI in its bus flags because there are actually some
+ * other buses controlled by the chipset even if Linux is not aware of it.
+ * Instead of setting the flag on all buses in the machine, simply disable
+ * MSI globally.
  */
 static void quirk_disable_all_msi(struct pci_dev *dev)
 {
@@ -2271,8 +2428,10 @@ static void quirk_amd_780_apc_msi(struct pci_dev *host_bridge)
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x9600, quirk_amd_780_apc_msi);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x9601, quirk_amd_780_apc_msi);
 
-/* Go through the list of Hypertransport capabilities and
- * return 1 if a HT MSI capability is found and enabled */
+/*
+ * Go through the list of HyperTransport capabilities and return 1 if a HT
+ * MSI capability is found and enabled.
+ */
 static int msi_ht_cap_enabled(struct pci_dev *dev)
 {
 	int pos, ttl = PCI_FIND_CAP_TTL;
@@ -2295,7 +2454,7 @@ static int msi_ht_cap_enabled(struct pci_dev *dev)
 	return 0;
 }
 
-/* Check the hypertransport MSI mapping to know whether MSI is enabled or not */
+/* Check the HyperTransport MSI mapping to know whether MSI is enabled or not */
 static void quirk_msi_ht_cap(struct pci_dev *dev)
 {
 	if (dev->subordinate && !msi_ht_cap_enabled(dev)) {
@@ -2306,8 +2465,9 @@ static void quirk_msi_ht_cap(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_SERVERWORKS, PCI_DEVICE_ID_SERVERWORKS_HT2000_PCIE,
 			quirk_msi_ht_cap);
 
-/* The nVidia CK804 chipset may have 2 HT MSI mappings.
- * MSI are supported if the MSI capability set in any of these mappings.
+/*
+ * The nVidia CK804 chipset may have 2 HT MSI mappings.  MSI is supported
+ * if the MSI capability is set in any of these mappings.
  */
 static void quirk_nvidia_ck804_msi_ht_cap(struct pci_dev *dev)
 {
@@ -2316,8 +2476,9 @@ static void quirk_nvidia_ck804_msi_ht_cap(struct pci_dev *dev)
 	if (!dev->subordinate)
 		return;
 
-	/* check HT MSI cap on this chipset and the root one.
-	 * a single one having MSI is enough to be sure that MSI are supported.
+	/*
+	 * Check HT MSI cap on this chipset and the root one.  A single one
+	 * having MSI is enough to be sure that MSI is supported.
 	 */
 	pdev = pci_get_slot(dev->bus, 0);
 	if (!pdev)
@@ -2354,13 +2515,13 @@ static void ht_enable_msi_mapping(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_SERVERWORKS,
 			 PCI_DEVICE_ID_SERVERWORKS_HT1000_PXB,
 			 ht_enable_msi_mapping);
-
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_8132_BRIDGE,
 			 ht_enable_msi_mapping);
 
-/* The P5N32-SLI motherboards from Asus have a problem with msi
- * for the MCP55 NIC. It is not yet determined whether the msi problem
- * also affects other devices. As for now, turn off msi for this device.
+/*
+ * The P5N32-SLI motherboards from Asus have a problem with MSI
+ * for the MCP55 NIC. It is not yet determined whether the MSI problem
+ * also affects other devices. As for now, turn off MSI for this device.
  */
 static void nvenet_msi_disable(struct pci_dev *dev)
 {
@@ -2397,16 +2558,14 @@ static void nvbridge_check_legacy_irq_routing(struct pci_dev *dev)
 	pci_read_config_dword(dev, 0x74, &cfg);
 
 	if (cfg & ((1 << 2) | (1 << 15))) {
-		printk(KERN_INFO "Rewriting irq routing register on MCP55\n");
+		printk(KERN_INFO "Rewriting IRQ routing register on MCP55\n");
 		cfg &= ~((1 << 2) | (1 << 15));
 		pci_write_config_dword(dev, 0x74, cfg);
 	}
 }
-
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA,
 			PCI_DEVICE_ID_NVIDIA_MCP55_BRIDGE_V0,
 			nvbridge_check_legacy_irq_routing);
-
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA,
 			PCI_DEVICE_ID_NVIDIA_MCP55_BRIDGE_V4,
 			nvbridge_check_legacy_irq_routing);
@@ -2416,7 +2575,7 @@ static int ht_check_msi_mapping(struct pci_dev *dev)
 	int pos, ttl = PCI_FIND_CAP_TTL;
 	int found = 0;
 
-	/* check if there is HT MSI cap or enabled on this device */
+	/* Check if there is HT MSI cap or enabled on this device */
 	pos = pci_find_ht_capability(dev, HT_CAPTYPE_MSI_MAPPING);
 	while (pos && ttl--) {
 		u8 flags;
@@ -2452,7 +2611,7 @@ static int host_bridge_with_leaf(struct pci_dev *host_bridge)
 		if (!dev)
 			continue;
 
-		/* found next host bridge ?*/
+		/* found next host bridge? */
 		pos = pci_find_ht_capability(dev, HT_CAPTYPE_SLAVE);
 		if (pos != 0) {
 			pci_dev_put(dev);
@@ -2611,27 +2770,27 @@ static void nv_msi_ht_cap_quirk_all(struct pci_dev *dev)
 {
 	return __nv_msi_ht_cap_quirk(dev, 1);
 }
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AL, PCI_ANY_ID, nv_msi_ht_cap_quirk_all);
+DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_AL, PCI_ANY_ID, nv_msi_ht_cap_quirk_all);
 
 static void nv_msi_ht_cap_quirk_leaf(struct pci_dev *dev)
 {
 	return __nv_msi_ht_cap_quirk(dev, 0);
 }
-
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, nv_msi_ht_cap_quirk_leaf);
 DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, nv_msi_ht_cap_quirk_leaf);
 
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AL, PCI_ANY_ID, nv_msi_ht_cap_quirk_all);
-DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_AL, PCI_ANY_ID, nv_msi_ht_cap_quirk_all);
-
 static void quirk_msi_intx_disable_bug(struct pci_dev *dev)
 {
 	dev->dev_flags |= PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG;
 }
+
 static void quirk_msi_intx_disable_ati_bug(struct pci_dev *dev)
 {
 	struct pci_dev *p;
 
-	/* SB700 MSI issue will be fixed at HW level from revision A21,
+	/*
+	 * SB700 MSI issue will be fixed at HW level from revision A21;
 	 * we need check PCI REVISION ID of SMBus controller to get SB700
 	 * revision.
 	 */
@@ -2644,6 +2803,7 @@ static void quirk_msi_intx_disable_ati_bug(struct pci_dev *dev)
 		dev->dev_flags |= PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG;
 	pci_dev_put(p);
 }
+
 static void quirk_msi_intx_disable_qca_bug(struct pci_dev *dev)
 {
 	/* AR816X/AR817X/E210X MSI is fixed at HW level from revision 0x18 */
@@ -2713,55 +2873,56 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATTANSIC, 0xe091,
 			quirk_msi_intx_disable_qca_bug);
 #endif /* CONFIG_PCI_MSI */
 
-/* Allow manual resource allocation for PCI hotplug bridges
- * via pci=hpmemsize=nnM and pci=hpiosize=nnM parameters. For
- * some PCI-PCI hotplug bridges, like PLX 6254 (former HINT HB6),
- * kernel fails to allocate resources when hotplug device is
- * inserted and PCI bus is rescanned.
+/*
+ * Allow manual resource allocation for PCI hotplug bridges via
+ * pci=hpmemsize=nnM and pci=hpiosize=nnM parameters. For some PCI-PCI
+ * hotplug bridges, like PLX 6254 (former HINT HB6), kernel fails to
+ * allocate resources when hotplug device is inserted and PCI bus is
+ * rescanned.
  */
 static void quirk_hotplug_bridge(struct pci_dev *dev)
 {
 	dev->is_hotplug_bridge = 1;
 }
-
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_HINT, 0x0020, quirk_hotplug_bridge);
 
 /*
- * This is a quirk for the Ricoh MMC controller found as a part of
- * some mulifunction chips.
-
+ * This is a quirk for the Ricoh MMC controller found as a part of some
+ * multifunction chips.
+ *
  * This is very similar and based on the ricoh_mmc driver written by
  * Philip Langdale. Thank you for these magic sequences.
  *
- * These chips implement the four main memory card controllers (SD, MMC, MS, xD)
- * and one or both of cardbus or firewire.
+ * These chips implement the four main memory card controllers (SD, MMC,
+ * MS, xD) and one or both of CardBus or FireWire.
  *
- * It happens that they implement SD and MMC
- * support as separate controllers (and PCI functions). The linux SDHCI
- * driver supports MMC cards but the chip detects MMC cards in hardware
- * and directs them to the MMC controller - so the SDHCI driver never sees
- * them.
+ * It happens that they implement SD and MMC support as separate
+ * controllers (and PCI functions). The Linux SDHCI driver supports MMC
+ * cards but the chip detects MMC cards in hardware and directs them to the
+ * MMC controller - so the SDHCI driver never sees them.
  *
- * To get around this, we must disable the useless MMC controller.
- * At that point, the SDHCI controller will start seeing them
- * It seems to be the case that the relevant PCI registers to deactivate the
- * MMC controller live on PCI function 0, which might be the cardbus controller
- * or the firewire controller, depending on the particular chip in question
+ * To get around this, we must disable the useless MMC controller.  At that
+ * point, the SDHCI controller will start seeing them.  It seems to be the
+ * case that the relevant PCI registers to deactivate the MMC controller
+ * live on PCI function 0, which might be the CardBus controller or the
+ * FireWire controller, depending on the particular chip in question
  *
  * This has to be done early, because as soon as we disable the MMC controller
- * other pci functions shift up one level, e.g. function #2 becomes function
- * #1, and this will confuse the pci core.
+ * other PCI functions shift up one level, e.g. function #2 becomes function
+ * #1, and this will confuse the PCI core.
  */
-
 #ifdef CONFIG_MMC_RICOH_MMC
 static void ricoh_mmc_fixup_rl5c476(struct pci_dev *dev)
 {
-	/* disable via cardbus interface */
 	u8 write_enable;
 	u8 write_target;
 	u8 disable;
 
-	/* disable must be done via function #0 */
+	/*
+	 * Disable via CardBus interface
+	 *
+	 * This must be done via function #0
+	 */
 	if (PCI_FUNC(dev->devfn))
 		return;
 
@@ -2777,7 +2938,7 @@ static void ricoh_mmc_fixup_rl5c476(struct pci_dev *dev)
 	pci_write_config_byte(dev, 0x8E, write_enable);
 	pci_write_config_byte(dev, 0x8D, write_target);
 
-	pci_notice(dev, "proprietary Ricoh MMC controller disabled (via cardbus function)\n");
+	pci_notice(dev, "proprietary Ricoh MMC controller disabled (via CardBus function)\n");
 	pci_notice(dev, "MMC cards are now supported by standard SDHCI controller\n");
 }
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_RICOH, PCI_DEVICE_ID_RICOH_RL5C476, ricoh_mmc_fixup_rl5c476);
@@ -2785,17 +2946,20 @@ DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_RICOH, PCI_DEVICE_ID_RICOH_RL5C476,
 
 static void ricoh_mmc_fixup_r5c832(struct pci_dev *dev)
 {
-	/* disable via firewire interface */
 	u8 write_enable;
 	u8 disable;
 
-	/* disable must be done via function #0 */
+	/*
+	 * Disable via FireWire interface
+	 *
+	 * This must be done via function #0
+	 */
 	if (PCI_FUNC(dev->devfn))
 		return;
 	/*
 	 * RICOH 0xe822 and 0xe823 SD/MMC card readers fail to recognize
-	 * certain types of SD/MMC cards. Lowering the SD base
-	 * clock frequency from 200Mhz to 50Mhz fixes this issue.
+	 * certain types of SD/MMC cards. Lowering the SD base clock
+	 * frequency from 200Mhz to 50Mhz fixes this issue.
 	 *
 	 * 0x150 - SD2.0 mode enable for changing base clock
 	 *	   frequency to 50Mhz
@@ -2826,7 +2990,7 @@ static void ricoh_mmc_fixup_r5c832(struct pci_dev *dev)
 	pci_write_config_byte(dev, 0xCB, disable | 0x02);
 	pci_write_config_byte(dev, 0xCA, write_enable);
 
-	pci_notice(dev, "proprietary Ricoh MMC controller disabled (via firewire function)\n");
+	pci_notice(dev, "proprietary Ricoh MMC controller disabled (via FireWire function)\n");
 	pci_notice(dev, "MMC cards are now supported by standard SDHCI controller\n");
 
 }
@@ -2842,13 +3006,13 @@ DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_RICOH, PCI_DEVICE_ID_RICOH_R5CE823,
 #define VTUNCERRMSK_REG	0x1ac
 #define VTD_MSK_SPEC_ERRORS	(1 << 31)
 /*
- * This is a quirk for masking vt-d spec defined errors to platform error
- * handling logic. With out this, platforms using Intel 7500, 5500 chipsets
+ * This is a quirk for masking VT-d spec-defined errors to platform error
+ * handling logic. Without this, platforms using Intel 7500, 5500 chipsets
  * (and the derivative chipsets like X58 etc) seem to generate NMI/SMI (based
- * on the RAS config settings of the platform) when a vt-d fault happens.
+ * on the RAS config settings of the platform) when a VT-d fault happens.
  * The resulting SMI caused the system to hang.
  *
- * VT-d spec related errors are already handled by the VT-d OS code, so no
+ * VT-d spec-related errors are already handled by the VT-d OS code, so no
  * need to report the same error through other channels.
  */
 static void vtd_mask_spec_errors(struct pci_dev *dev)
@@ -2874,7 +3038,8 @@ static void fixup_ti816x_class(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_TI, 0xb800,
 			      PCI_CLASS_NOT_DEFINED, 8, fixup_ti816x_class);
 
-/* Some PCIe devices do not work reliably with the claimed maximum
+/*
+ * Some PCIe devices do not work reliably with the claimed maximum
  * payload size supported.
  */
 static void fixup_mpss_256(struct pci_dev *dev)
@@ -2888,9 +3053,10 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_SOLARFLARE,
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_SOLARFLARE,
 			 PCI_DEVICE_ID_SOLARFLARE_SFC4000B, fixup_mpss_256);
 
-/* Intel 5000 and 5100 Memory controllers have an errata with read completion
+/*
+ * Intel 5000 and 5100 Memory controllers have an erratum with read completion
  * coalescing (which is enabled by default on some BIOSes) and MPS of 256B.
- * Since there is no way of knowing what the PCIE MPS on each fabric will be
+ * Since there is no way of knowing what the PCIe MPS on each fabric will be
  * until all of the devices are discovered and buses walked, read completion
  * coalescing must be disabled.  Unfortunately, it cannot be re-enabled because
  * it is possible to hotplug a device with MPS of 256B.
@@ -2904,9 +3070,10 @@ static void quirk_intel_mc_errata(struct pci_dev *dev)
 	    pcie_bus_config == PCIE_BUS_DEFAULT)
 		return;
 
-	/* Intel errata specifies bits to change but does not say what they are.
-	 * Keeping them magical until such time as the registers and values can
-	 * be explained.
+	/*
+	 * Intel erratum specifies bits to change but does not say what
+	 * they are.  Keeping them magical until such time as the registers
+	 * and values can be explained.
 	 */
 	err = pci_read_config_word(dev, 0x48, &rcc);
 	if (err) {
@@ -2925,7 +3092,7 @@ static void quirk_intel_mc_errata(struct pci_dev *dev)
 		return;
 	}
 
-	pr_info_once("Read completion coalescing disabled due to hardware errata relating to 256B MPS\n");
+	pr_info_once("Read completion coalescing disabled due to hardware erratum relating to 256B MPS\n");
 }
 /* Intel 5000 series memory controllers and ports 2-7 */
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x25c0, quirk_intel_mc_errata);
@@ -2955,11 +3122,10 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x65f8, quirk_intel_mc_errata);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x65f9, quirk_intel_mc_errata);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x65fa, quirk_intel_mc_errata);
 
-
 /*
- * Ivytown NTB BAR sizes are misreported by the hardware due to an erratum.  To
- * work around this, query the size it should be configured to by the device and
- * modify the resource end to correspond to this new size.
+ * Ivytown NTB BAR sizes are misreported by the hardware due to an erratum.
+ * To work around this, query the size it should be configured to by the
+ * device and modify the resource end to correspond to this new size.
  */
 static void quirk_intel_ntb(struct pci_dev *dev)
 {
@@ -2981,39 +3147,17 @@ static void quirk_intel_ntb(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0e08, quirk_intel_ntb);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0e0d, quirk_intel_ntb);
 
-static ktime_t fixup_debug_start(struct pci_dev *dev,
-				 void (*fn)(struct pci_dev *dev))
-{
-	if (initcall_debug)
-		pci_info(dev, "calling  %pF @ %i\n", fn, task_pid_nr(current));
-
-	return ktime_get();
-}
-
-static void fixup_debug_report(struct pci_dev *dev, ktime_t calltime,
-			       void (*fn)(struct pci_dev *dev))
-{
-	ktime_t delta, rettime;
-	unsigned long long duration;
-
-	rettime = ktime_get();
-	delta = ktime_sub(rettime, calltime);
-	duration = (unsigned long long) ktime_to_ns(delta) >> 10;
-	if (initcall_debug || duration > 10000)
-		pci_info(dev, "%pF took %lld usecs\n", fn, duration);
-}
-
 /*
- * Some BIOS implementations leave the Intel GPU interrupts enabled,
- * even though no one is handling them (f.e. i915 driver is never loaded).
- * Additionally the interrupt destination is not set up properly
+ * Some BIOS implementations leave the Intel GPU interrupts enabled, even
+ * though no one is handling them (e.g., if the i915 driver is never
+ * loaded).  Additionally the interrupt destination is not set up properly
  * and the interrupt ends up -somewhere-.
  *
- * These spurious interrupts are "sticky" and the kernel disables
- * the (shared) interrupt line after 100.000+ generated interrupts.
+ * These spurious interrupts are "sticky" and the kernel disables the
+ * (shared) interrupt line after 100,000+ generated interrupts.
  *
- * Fix it by disabling the still enabled interrupts.
- * This resolves crashes often seen on monitor unplug.
+ * Fix it by disabling the still enabled interrupts.  This resolves crashes
+ * often seen on monitor unplug.
  */
 #define I915_DEIER_REG 0x4400c
 static void disable_igfx_irq(struct pci_dev *dev)
@@ -3101,38 +3245,22 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_REALTEK, 0x8169,
  * Intel i40e (XL710/X710) 10/20/40GbE NICs all have broken INTx masking,
  * DisINTx can be set but the interrupt status bit is non-functional.
  */
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1572,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1574,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1580,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1581,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1583,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1584,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1585,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1586,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1587,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1588,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1589,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x158a,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x158b,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d0,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d1,
-			quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d2,
-			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1572, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1574, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1580, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1581, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1583, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1584, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1585, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1586, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1587, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1588, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1589, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x158a, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x158b, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d0, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d1, quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d2, quirk_broken_intx_masking);
 
 static u16 mellanox_broken_intx_devs[] = {
 	PCI_DEVICE_ID_MELLANOX_HERMON_SDR,
@@ -3177,7 +3305,8 @@ static void mellanox_check_broken_intx_masking(struct pci_dev *pdev)
 		}
 	}
 
-	/* Getting here means Connect-IB cards and up. Connect-IB has no INTx
+	/*
+	 * Getting here means Connect-IB cards and up. Connect-IB has no INTx
 	 * support so shouldn't be checked further
 	 */
 	if (pdev->device == PCI_DEVICE_ID_MELLANOX_CONNECTIB)
@@ -3297,8 +3426,8 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PORT_RIDGE,
  * shutdown before suspend. Otherwise the native host interface (NHI) will not
  * be present after resume if a device was plugged in before suspend.
  *
- * The thunderbolt controller consists of a pcie switch with downstream
- * bridges leading to the NHI and to the tunnel pci bridges.
+ * The Thunderbolt controller consists of a PCIe switch with downstream
+ * bridges leading to the NHI and to the tunnel PCI bridges.
  *
  * This quirk cuts power to the whole chip. Therefore we have to apply it
  * during suspend_noirq of the upstream bridge.
@@ -3316,17 +3445,19 @@ static void quirk_apple_poweroff_thunderbolt(struct pci_dev *dev)
 	bridge = ACPI_HANDLE(&dev->dev);
 	if (!bridge)
 		return;
+
 	/*
 	 * SXIO and SXLV are present only on machines requiring this quirk.
-	 * TB bridges in external devices might have the same device id as those
-	 * on the host, but they will not have the associated ACPI methods. This
-	 * implicitly checks that we are at the right bridge.
+	 * Thunderbolt bridges in external devices might have the same
+	 * device ID as those on the host, but they will not have the
+	 * associated ACPI methods. This implicitly checks that we are at
+	 * the right bridge.
 	 */
 	if (ACPI_FAILURE(acpi_get_handle(bridge, "DSB0.NHI0.SXIO", &SXIO))
 	    || ACPI_FAILURE(acpi_get_handle(bridge, "DSB0.NHI0.SXFP", &SXFP))
 	    || ACPI_FAILURE(acpi_get_handle(bridge, "DSB0.NHI0.SXLV", &SXLV)))
 		return;
-	pci_info(dev, "quirk: cutting power to thunderbolt controller...\n");
+	pci_info(dev, "quirk: cutting power to Thunderbolt controller...\n");
 
 	/* magic sequence */
 	acpi_execute_simple_method(SXIO, NULL, 1);
@@ -3341,9 +3472,9 @@ DECLARE_PCI_FIXUP_SUSPEND_LATE(PCI_VENDOR_ID_INTEL,
 			       quirk_apple_poweroff_thunderbolt);
 
 /*
- * Apple: Wait for the thunderbolt controller to reestablish pci tunnels.
+ * Apple: Wait for the Thunderbolt controller to reestablish PCI tunnels
  *
- * During suspend the thunderbolt controller is reset and all pci
+ * During suspend the Thunderbolt controller is reset and all PCI
  * tunnels are lost. The NHI driver will try to reestablish all tunnels
  * during resume. We have to manually wait for the NHI since there is
  * no parent child relationship between the NHI and the tunneled
@@ -3358,9 +3489,10 @@ static void quirk_apple_wait_for_thunderbolt(struct pci_dev *dev)
 		return;
 	if (pci_pcie_type(dev) != PCI_EXP_TYPE_DOWNSTREAM)
 		return;
+
 	/*
-	 * Find the NHI and confirm that we are a bridge on the tb host
-	 * controller and not on a tb endpoint.
+	 * Find the NHI and confirm that we are a bridge on the Thunderbolt
+	 * host controller and not on a Thunderbolt endpoint.
 	 */
 	sibling = pci_get_slot(dev->bus, 0x0);
 	if (sibling == dev)
@@ -3377,7 +3509,7 @@ static void quirk_apple_wait_for_thunderbolt(struct pci_dev *dev)
 			nhi->device != PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI)
 		    || nhi->class != PCI_CLASS_SYSTEM_OTHER << 8)
 		goto out;
-	pci_info(dev, "quirk: waiting for thunderbolt to reestablish PCI tunnels...\n");
+	pci_info(dev, "quirk: waiting for Thunderbolt to reestablish PCI tunnels...\n");
 	device_pm_wait_for_dev(&dev->dev, &nhi->dev);
 out:
 	pci_dev_put(nhi);
@@ -3397,142 +3529,6 @@ DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_INTEL,
 			       quirk_apple_wait_for_thunderbolt);
 #endif
 
-static void pci_do_fixups(struct pci_dev *dev, struct pci_fixup *f,
-			  struct pci_fixup *end)
-{
-	ktime_t calltime;
-
-	for (; f < end; f++)
-		if ((f->class == (u32) (dev->class >> f->class_shift) ||
-		     f->class == (u32) PCI_ANY_ID) &&
-		    (f->vendor == dev->vendor ||
-		     f->vendor == (u16) PCI_ANY_ID) &&
-		    (f->device == dev->device ||
-		     f->device == (u16) PCI_ANY_ID)) {
-			calltime = fixup_debug_start(dev, f->hook);
-			f->hook(dev);
-			fixup_debug_report(dev, calltime, f->hook);
-		}
-}
-
-extern struct pci_fixup __start_pci_fixups_early[];
-extern struct pci_fixup __end_pci_fixups_early[];
-extern struct pci_fixup __start_pci_fixups_header[];
-extern struct pci_fixup __end_pci_fixups_header[];
-extern struct pci_fixup __start_pci_fixups_final[];
-extern struct pci_fixup __end_pci_fixups_final[];
-extern struct pci_fixup __start_pci_fixups_enable[];
-extern struct pci_fixup __end_pci_fixups_enable[];
-extern struct pci_fixup __start_pci_fixups_resume[];
-extern struct pci_fixup __end_pci_fixups_resume[];
-extern struct pci_fixup __start_pci_fixups_resume_early[];
-extern struct pci_fixup __end_pci_fixups_resume_early[];
-extern struct pci_fixup __start_pci_fixups_suspend[];
-extern struct pci_fixup __end_pci_fixups_suspend[];
-extern struct pci_fixup __start_pci_fixups_suspend_late[];
-extern struct pci_fixup __end_pci_fixups_suspend_late[];
-
-static bool pci_apply_fixup_final_quirks;
-
-void pci_fixup_device(enum pci_fixup_pass pass, struct pci_dev *dev)
-{
-	struct pci_fixup *start, *end;
-
-	switch (pass) {
-	case pci_fixup_early:
-		start = __start_pci_fixups_early;
-		end = __end_pci_fixups_early;
-		break;
-
-	case pci_fixup_header:
-		start = __start_pci_fixups_header;
-		end = __end_pci_fixups_header;
-		break;
-
-	case pci_fixup_final:
-		if (!pci_apply_fixup_final_quirks)
-			return;
-		start = __start_pci_fixups_final;
-		end = __end_pci_fixups_final;
-		break;
-
-	case pci_fixup_enable:
-		start = __start_pci_fixups_enable;
-		end = __end_pci_fixups_enable;
-		break;
-
-	case pci_fixup_resume:
-		start = __start_pci_fixups_resume;
-		end = __end_pci_fixups_resume;
-		break;
-
-	case pci_fixup_resume_early:
-		start = __start_pci_fixups_resume_early;
-		end = __end_pci_fixups_resume_early;
-		break;
-
-	case pci_fixup_suspend:
-		start = __start_pci_fixups_suspend;
-		end = __end_pci_fixups_suspend;
-		break;
-
-	case pci_fixup_suspend_late:
-		start = __start_pci_fixups_suspend_late;
-		end = __end_pci_fixups_suspend_late;
-		break;
-
-	default:
-		/* stupid compiler warning, you would think with an enum... */
-		return;
-	}
-	pci_do_fixups(dev, start, end);
-}
-EXPORT_SYMBOL(pci_fixup_device);
-
-
-static int __init pci_apply_final_quirks(void)
-{
-	struct pci_dev *dev = NULL;
-	u8 cls = 0;
-	u8 tmp;
-
-	if (pci_cache_line_size)
-		printk(KERN_DEBUG "PCI: CLS %u bytes\n",
-		       pci_cache_line_size << 2);
-
-	pci_apply_fixup_final_quirks = true;
-	for_each_pci_dev(dev) {
-		pci_fixup_device(pci_fixup_final, dev);
-		/*
-		 * If arch hasn't set it explicitly yet, use the CLS
-		 * value shared by all PCI devices.  If there's a
-		 * mismatch, fall back to the default value.
-		 */
-		if (!pci_cache_line_size) {
-			pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &tmp);
-			if (!cls)
-				cls = tmp;
-			if (!tmp || cls == tmp)
-				continue;
-
-			printk(KERN_DEBUG "PCI: CLS mismatch (%u != %u), using %u bytes\n",
-			       cls << 2, tmp << 2,
-			       pci_dfl_cache_line_size << 2);
-			pci_cache_line_size = pci_dfl_cache_line_size;
-		}
-	}
-
-	if (!pci_cache_line_size) {
-		printk(KERN_DEBUG "PCI: CLS %u bytes, default %u\n",
-		       cls << 2, pci_dfl_cache_line_size << 2);
-		pci_cache_line_size = cls ? cls : pci_dfl_cache_line_size;
-	}
-
-	return 0;
-}
-
-fs_initcall_sync(pci_apply_final_quirks);
-
 /*
  * Following are device-specific reset methods which can be used to
  * reset a single function if other methods (e.g. FLR, PM D0->D3) are
@@ -3602,9 +3598,7 @@ static int reset_ivb_igd(struct pci_dev *dev, int probe)
 	return 0;
 }
 
-/*
- * Device-specific reset method for Chelsio T4-based adapters.
- */
+/* Device-specific reset method for Chelsio T4-based adapters */
 static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
 {
 	u16 old_command;
@@ -3887,7 +3881,7 @@ DECLARE_PCI_FIXUP_CLASS_EARLY(0x1797, 0x6869, PCI_CLASS_NOT_DEFINED, 8,
 /*
  * Some devices have problems with Transaction Layer Packets with the Relaxed
  * Ordering Attribute set.  Such devices should mark themselves and other
- * Device Drivers should check before sending TLPs with RO set.
+ * device drivers should check before sending TLPs with RO set.
  */
 static void quirk_relaxedordering_disable(struct pci_dev *dev)
 {
@@ -3897,7 +3891,7 @@ static void quirk_relaxedordering_disable(struct pci_dev *dev)
 
 /*
  * Intel Xeon processors based on Broadwell/Haswell microarchitecture Root
- * Complex has a Flow Control Credit issue which can cause performance
+ * Complex have a Flow Control Credit issue which can cause performance
  * problems with Upstream Transaction Layer Packets with Relaxed Ordering set.
  */
 DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f01, PCI_CLASS_NOT_DEFINED, 8,
@@ -3958,7 +3952,7 @@ DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f0e, PCI_CLASS_NOT_DEFINED
 			      quirk_relaxedordering_disable);
 
 /*
- * The AMD ARM A1100 (AKA "SEATTLE") SoC has a bug in its PCIe Root Complex
+ * The AMD ARM A1100 (aka "SEATTLE") SoC has a bug in its PCIe Root Complex
  * where Upstream Transaction Layer Packets with the Relaxed Ordering
  * Attribute clear are allowed to bypass earlier TLPs with Relaxed Ordering
  * set.  This is a violation of the PCIe 3.0 Transaction Ordering Rules
@@ -4022,7 +4016,7 @@ static void quirk_chelsio_T5_disable_root_port_attributes(struct pci_dev *pdev)
 	 * This mask/compare operation selects for Physical Function 4 on a
 	 * T5.  We only need to fix up the Root Port once for any of the
 	 * PFs.  PF[0..3] have PCI Device IDs of 0x50xx, but PF4 is uniquely
-	 * 0x54xx so we use that one,
+	 * 0x54xx so we use that one.
 	 */
 	if ((pdev->device & 0xff00) == 0x5400)
 		quirk_disable_root_port_attributes(pdev);
@@ -4113,7 +4107,7 @@ static int pci_quirk_cavium_acs(struct pci_dev *dev, u16 acs_flags)
 static int pci_quirk_xgene_acs(struct pci_dev *dev, u16 acs_flags)
 {
 	/*
-	 * X-Gene root matching this quirk do not allow peer-to-peer
+	 * X-Gene Root Ports matching this quirk do not allow peer-to-peer
 	 * transactions with others, allowing masking out these bits as if they
 	 * were unimplemented in the ACS capability.
 	 */
@@ -4230,11 +4224,29 @@ static int pci_quirk_qcom_rp_acs(struct pci_dev *dev, u16 acs_flags)
  * 0xa290-0xa29f PCI Express Root port #{0-16}
  * 0xa2e7-0xa2ee PCI Express Root port #{17-24}
  *
+ * Mobile chipsets are also affected, 7th & 8th Generation
+ * Specification update confirms ACS errata 22, status no fix: (7th Generation
+ * Intel Processor Family I/O for U/Y Platforms and 8th Generation Intel
+ * Processor Family I/O for U Quad Core Platforms Specification Update,
+ * August 2017, Revision 002, Document#: 334660-002)[6]
+ * Device IDs from I/O datasheet: (7th Generation Intel Processor Family I/O
+ * for U/Y Platforms and 8th Generation Intel ® Processor Family I/O for U
+ * Quad Core Platforms, Vol 1 of 2, August 2017, Document#: 334658-003)[7]
+ *
+ * 0x9d10-0x9d1b PCI Express Root port #{1-12}
+ *
+ * The 300 series chipset suffers from the same bug so include those root
+ * ports here as well.
+ *
+ * 0xa32c-0xa343 PCI Express Root port #{0-24}
+ *
  * [1] http://www.intel.com/content/www/us/en/chipsets/100-series-chipset-datasheet-vol-2.html
  * [2] http://www.intel.com/content/www/us/en/chipsets/100-series-chipset-datasheet-vol-1.html
  * [3] http://www.intel.com/content/www/us/en/chipsets/100-series-chipset-spec-update.html
  * [4] http://www.intel.com/content/www/us/en/chipsets/200-series-chipset-pch-spec-update.html
  * [5] http://www.intel.com/content/www/us/en/chipsets/200-series-chipset-pch-datasheet-vol-1.html
+ * [6] https://www.intel.com/content/www/us/en/processors/core/7th-gen-core-family-mobile-u-y-processor-lines-i-o-spec-update.html
+ * [7] https://www.intel.com/content/www/us/en/processors/core/7th-gen-core-family-mobile-u-y-processor-lines-i-o-datasheet-vol-1.html
  */
 static bool pci_quirk_intel_spt_pch_acs_match(struct pci_dev *dev)
 {
@@ -4244,6 +4256,8 @@ static bool pci_quirk_intel_spt_pch_acs_match(struct pci_dev *dev)
 	switch (dev->device) {
 	case 0xa110 ... 0xa11f: case 0xa167 ... 0xa16a: /* Sunrise Point */
 	case 0xa290 ... 0xa29f: case 0xa2e7 ... 0xa2ee: /* Union Point */
+	case 0x9d10 ... 0x9d1b: /* 7th & 8th Gen Mobile */
+	case 0xa32c ... 0xa343:				/* 300 series */
 		return true;
 	}
 
@@ -4361,8 +4375,8 @@ static const struct pci_dev_acs_enabled {
 	{ PCI_VENDOR_ID_INTEL, 0x15b7, pci_quirk_mf_endpoint_acs },
 	{ PCI_VENDOR_ID_INTEL, 0x15b8, pci_quirk_mf_endpoint_acs },
 	/* QCOM QDF2xxx root ports */
-	{ 0x17cb, 0x400, pci_quirk_qcom_rp_acs },
-	{ 0x17cb, 0x401, pci_quirk_qcom_rp_acs },
+	{ PCI_VENDOR_ID_QCOM, 0x0400, pci_quirk_qcom_rp_acs },
+	{ PCI_VENDOR_ID_QCOM, 0x0401, pci_quirk_qcom_rp_acs },
 	/* Intel PCH root ports */
 	{ PCI_VENDOR_ID_INTEL, PCI_ANY_ID, pci_quirk_intel_pch_acs },
 	{ PCI_VENDOR_ID_INTEL, PCI_ANY_ID, pci_quirk_intel_spt_pch_acs },
@@ -4436,7 +4450,7 @@ static int pci_quirk_enable_intel_lpc_acs(struct pci_dev *dev)
 	/*
 	 * Read the RCBA register from the LPC (D31:F0).  PCH root ports
 	 * are D28:F* and therefore get probed before LPC, thus we can't
-	 * use pci_get_slot/pci_read_config_dword here.
+	 * use pci_get_slot()/pci_read_config_dword() here.
 	 */
 	pci_bus_read_config_dword(dev->bus, PCI_DEVFN(31, 0),
 				  INTEL_LPC_RCBA_REG, &rcba);
@@ -4569,7 +4583,7 @@ int pci_dev_specific_enable_acs(struct pci_dev *dev)
 }
 
 /*
- * The PCI capabilities list for Intel DH895xCC VFs (device id 0x0443) with
+ * The PCI capabilities list for Intel DH895xCC VFs (device ID 0x0443) with
  * QuickAssist Technology (QAT) is prematurely terminated in hardware.  The
  * Next Capability pointer in the MSI Capability Structure should point to
  * the PCIe Capability Structure but is incorrectly hardwired as 0 terminating
@@ -4630,9 +4644,7 @@ static void quirk_intel_qat_vf_cap(struct pci_dev *pdev)
 		if (pci_find_saved_cap(pdev, PCI_CAP_ID_EXP))
 			return;
 
-		/*
-		 * Save PCIE cap
-		 */
+		/* Save PCIe cap */
 		state = kzalloc(sizeof(*state) + size, GFP_KERNEL);
 		if (!state)
 			return;
@@ -4653,7 +4665,7 @@ static void quirk_intel_qat_vf_cap(struct pci_dev *pdev)
 }
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap);
 
-/* FLR may cause some 82579 devices to hang. */
+/* FLR may cause some 82579 devices to hang */
 static void quirk_intel_no_flr(struct pci_dev *dev)
 {
 	dev->dev_flags |= PCI_DEV_FLAGS_NO_FLR_RESET;
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index 28bb5a029558..08ebaf7cca8b 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -6,30 +6,32 @@ menu "Performance monitor support"
 	depends on PERF_EVENTS
 
 config ARM_CCI_PMU
-	bool
+	tristate "ARM CCI PMU driver"
+	depends on (ARM && CPU_V7) || ARM64
 	select ARM_CCI
+	help
+	  Support for PMU events monitoring on the ARM CCI (Cache Coherent
+	  Interconnect) family of products.
+
+	  If compiled as a module, it will be called arm-cci.
 
 config ARM_CCI400_PMU
-	bool "ARM CCI400 PMU support"
-	depends on (ARM && CPU_V7) || ARM64
+	bool "support CCI-400"
+	default y
+	depends on ARM_CCI_PMU
 	select ARM_CCI400_COMMON
-	select ARM_CCI_PMU
 	help
-	  Support for PMU events monitoring on the ARM CCI-400 (cache coherent
-	  interconnect). CCI-400 supports counting events related to the
-	  connected slave/master interfaces.
+	  CCI-400 provides 4 independent event counters counting events related
+	  to the connected slave/master interfaces, plus a cycle counter.
 
 config ARM_CCI5xx_PMU
-	bool "ARM CCI-500/CCI-550 PMU support"
-	depends on (ARM && CPU_V7) || ARM64
-	select ARM_CCI_PMU
+	bool "support CCI-500/CCI-550"
+	default y
+	depends on ARM_CCI_PMU
 	help
-	  Support for PMU events monitoring on the ARM CCI-500/CCI-550 cache
-	  coherent interconnects. Both of them provide 8 independent event counters,
-	  which can count events pertaining to the slave/master interfaces as well
-	  as the internal events to the CCI.
-
-	  If unsure, say Y
+	  CCI-500/CCI-550 both provide 8 independent event counters, which can
+	  count events pertaining to the slave/master interfaces as well as the
+	  internal events to the CCI.
 
 config ARM_CCN
 	tristate "ARM CCN driver support"
@@ -94,7 +96,7 @@ config XGENE_PMU
 
 config ARM_SPE_PMU
 	tristate "Enable support for the ARMv8.2 Statistical Profiling Extension"
-	depends on PERF_EVENTS && ARM64
+	depends on ARM64
 	help
 	  Enable perf support for the ARMv8.2 Statistical Profiling
 	  Extension, which provides periodic sampling of operations in
diff --git a/drivers/perf/arm-cci.c b/drivers/perf/arm-cci.c
index 383b2d3dcbc6..0d09d8e669cd 100644
--- a/drivers/perf/arm-cci.c
+++ b/drivers/perf/arm-cci.c
@@ -120,9 +120,9 @@ enum cci_models {
 
 static void pmu_write_counters(struct cci_pmu *cci_pmu,
 				 unsigned long *mask);
-static ssize_t cci_pmu_format_show(struct device *dev,
+static ssize_t __maybe_unused cci_pmu_format_show(struct device *dev,
 			struct device_attribute *attr, char *buf);
-static ssize_t cci_pmu_event_show(struct device *dev,
+static ssize_t __maybe_unused cci_pmu_event_show(struct device *dev,
 			struct device_attribute *attr, char *buf);
 
 #define CCI_EXT_ATTR_ENTRY(_name, _func, _config) 				\
@@ -1184,16 +1184,11 @@ static int cci_pmu_add(struct perf_event *event, int flags)
 	struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
 	struct hw_perf_event *hwc = &event->hw;
 	int idx;
-	int err = 0;
-
-	perf_pmu_disable(event->pmu);
 
 	/* If we don't have a space for the counter then finish early. */
 	idx = pmu_get_event_idx(hw_events, event);
-	if (idx < 0) {
-		err = idx;
-		goto out;
-	}
+	if (idx < 0)
+		return idx;
 
 	event->hw.idx = idx;
 	hw_events->events[idx] = event;
@@ -1205,9 +1200,7 @@ static int cci_pmu_add(struct perf_event *event, int flags)
 	/* Propagate our changes to the userspace mapping. */
 	perf_event_update_userpage(event);
 
-out:
-	perf_pmu_enable(event->pmu);
-	return err;
+	return 0;
 }
 
 static void cci_pmu_del(struct perf_event *event, int flags)
@@ -1304,15 +1297,6 @@ static int __hw_perf_event_init(struct perf_event *event)
 	 */
 	hwc->config_base	    |= (unsigned long)mapping;
 
-	/*
-	 * Limit the sample_period to half of the counter width. That way, the
-	 * new counter value is far less likely to overtake the previous one
-	 * unless you have some serious IRQ latency issues.
-	 */
-	hwc->sample_period  = CCI_PMU_CNTR_MASK >> 1;
-	hwc->last_period    = hwc->sample_period;
-	local64_set(&hwc->period_left, hwc->sample_period);
-
 	if (event->group_leader != event) {
 		if (validate_group(event) != 0)
 			return -EINVAL;
@@ -1423,6 +1407,7 @@ static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev)
 	pmu_format_attr_group.attrs = model->format_attrs;
 
 	cci_pmu->pmu = (struct pmu) {
+		.module		= THIS_MODULE,
 		.name		= cci_pmu->model->name,
 		.task_ctx_nr	= perf_invalid_context,
 		.pmu_enable	= cci_pmu_enable,
@@ -1466,7 +1451,7 @@ static int cci_pmu_offline_cpu(unsigned int cpu)
 	return 0;
 }
 
-static struct cci_pmu_model cci_pmu_models[] = {
+static __maybe_unused struct cci_pmu_model cci_pmu_models[] = {
 #ifdef CONFIG_ARM_CCI400_PMU
 	[CCI400_R0] = {
 		.name = "CCI_400",
@@ -1588,6 +1573,7 @@ static const struct of_device_id arm_cci_pmu_matches[] = {
 #endif
 	{},
 };
+MODULE_DEVICE_TABLE(of, arm_cci_pmu_matches);
 
 static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs)
 {
@@ -1709,14 +1695,27 @@ static int cci_pmu_probe(struct platform_device *pdev)
 	return 0;
 }
 
+static int cci_pmu_remove(struct platform_device *pdev)
+{
+	if (!g_cci_pmu)
+		return 0;
+
+	cpuhp_remove_state(CPUHP_AP_PERF_ARM_CCI_ONLINE);
+	perf_pmu_unregister(&g_cci_pmu->pmu);
+	g_cci_pmu = NULL;
+
+	return 0;
+}
+
 static struct platform_driver cci_pmu_driver = {
 	.driver = {
 		   .name = DRIVER_NAME,
 		   .of_match_table = arm_cci_pmu_matches,
 		  },
 	.probe = cci_pmu_probe,
+	.remove = cci_pmu_remove,
 };
 
-builtin_platform_driver(cci_pmu_driver);
-MODULE_LICENSE("GPL");
+module_platform_driver(cci_pmu_driver);
+MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("ARM CCI PMU support");
diff --git a/drivers/perf/arm-ccn.c b/drivers/perf/arm-ccn.c
index 65b7e4042ece..b416ee18e6bb 100644
--- a/drivers/perf/arm-ccn.c
+++ b/drivers/perf/arm-ccn.c
@@ -736,7 +736,7 @@ static int arm_ccn_pmu_event_init(struct perf_event *event)
 	ccn = pmu_to_arm_ccn(event->pmu);
 
 	if (hw->sample_period) {
-		dev_warn(ccn->dev, "Sampling not supported!\n");
+		dev_dbg(ccn->dev, "Sampling not supported!\n");
 		return -EOPNOTSUPP;
 	}
 
@@ -744,12 +744,12 @@ static int arm_ccn_pmu_event_init(struct perf_event *event)
 			event->attr.exclude_kernel || event->attr.exclude_hv ||
 			event->attr.exclude_idle || event->attr.exclude_host ||
 			event->attr.exclude_guest) {
-		dev_warn(ccn->dev, "Can't exclude execution levels!\n");
+		dev_dbg(ccn->dev, "Can't exclude execution levels!\n");
 		return -EINVAL;
 	}
 
 	if (event->cpu < 0) {
-		dev_warn(ccn->dev, "Can't provide per-task data!\n");
+		dev_dbg(ccn->dev, "Can't provide per-task data!\n");
 		return -EOPNOTSUPP;
 	}
 	/*
@@ -771,13 +771,13 @@ static int arm_ccn_pmu_event_init(struct perf_event *event)
 	switch (type) {
 	case CCN_TYPE_MN:
 		if (node_xp != ccn->mn_id) {
-			dev_warn(ccn->dev, "Invalid MN ID %d!\n", node_xp);
+			dev_dbg(ccn->dev, "Invalid MN ID %d!\n", node_xp);
 			return -EINVAL;
 		}
 		break;
 	case CCN_TYPE_XP:
 		if (node_xp >= ccn->num_xps) {
-			dev_warn(ccn->dev, "Invalid XP ID %d!\n", node_xp);
+			dev_dbg(ccn->dev, "Invalid XP ID %d!\n", node_xp);
 			return -EINVAL;
 		}
 		break;
@@ -785,11 +785,11 @@ static int arm_ccn_pmu_event_init(struct perf_event *event)
 		break;
 	default:
 		if (node_xp >= ccn->num_nodes) {
-			dev_warn(ccn->dev, "Invalid node ID %d!\n", node_xp);
+			dev_dbg(ccn->dev, "Invalid node ID %d!\n", node_xp);
 			return -EINVAL;
 		}
 		if (!arm_ccn_pmu_type_eq(type, ccn->node[node_xp].type)) {
-			dev_warn(ccn->dev, "Invalid type 0x%x for node %d!\n",
+			dev_dbg(ccn->dev, "Invalid type 0x%x for node %d!\n",
 					type, node_xp);
 			return -EINVAL;
 		}
@@ -808,19 +808,19 @@ static int arm_ccn_pmu_event_init(struct perf_event *event)
 		if (event_id != e->event)
 			continue;
 		if (e->num_ports && port >= e->num_ports) {
-			dev_warn(ccn->dev, "Invalid port %d for node/XP %d!\n",
+			dev_dbg(ccn->dev, "Invalid port %d for node/XP %d!\n",
 					port, node_xp);
 			return -EINVAL;
 		}
 		if (e->num_vcs && vc >= e->num_vcs) {
-			dev_warn(ccn->dev, "Invalid vc %d for node/XP %d!\n",
+			dev_dbg(ccn->dev, "Invalid vc %d for node/XP %d!\n",
 					vc, node_xp);
 			return -EINVAL;
 		}
 		valid = 1;
 	}
 	if (!valid) {
-		dev_warn(ccn->dev, "Invalid event 0x%x for node/XP %d!\n",
+		dev_dbg(ccn->dev, "Invalid event 0x%x for node/XP %d!\n",
 				event_id, node_xp);
 		return -EINVAL;
 	}
@@ -1594,4 +1594,4 @@ module_init(arm_ccn_init);
 module_exit(arm_ccn_exit);
 
 MODULE_AUTHOR("Pawel Moll <pawel.moll@arm.com>");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
index 1a0d340b65cf..a6347d487635 100644
--- a/drivers/perf/arm_pmu.c
+++ b/drivers/perf/arm_pmu.c
@@ -339,7 +339,7 @@ static irqreturn_t armpmu_dispatch_irq(int irq, void *dev)
 		return IRQ_NONE;
 
 	start_clock = sched_clock();
-	ret = armpmu->handle_irq(irq, armpmu);
+	ret = armpmu->handle_irq(armpmu);
 	finish_clock = sched_clock();
 
 	perf_sample_event_took(finish_clock - start_clock);
diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c
index 28bb642af18b..54ec278d2fc4 100644
--- a/drivers/perf/arm_spe_pmu.c
+++ b/drivers/perf/arm_spe_pmu.c
@@ -131,8 +131,7 @@ static ssize_t arm_spe_pmu_cap_show(struct device *dev,
 				    struct device_attribute *attr,
 				    char *buf)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev);
+	struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);
 	struct dev_ext_attribute *ea =
 		container_of(attr, struct dev_ext_attribute, attr);
 	int cap = (long)ea->var;
@@ -247,8 +246,7 @@ static ssize_t arm_spe_pmu_get_attr_cpumask(struct device *dev,
 					    struct device_attribute *attr,
 					    char *buf)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev);
+	struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);
 
 	return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus);
 }
diff --git a/drivers/phy/marvell/phy-mvebu-cp110-comphy.c b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c
index a0d522154cdf..4ef429250d7b 100644
--- a/drivers/phy/marvell/phy-mvebu-cp110-comphy.c
+++ b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c
@@ -135,19 +135,25 @@ struct mvebu_comhy_conf {
 static const struct mvebu_comhy_conf mvebu_comphy_cp110_modes[] = {
 	/* lane 0 */
 	MVEBU_COMPHY_CONF(0, 1, PHY_MODE_SGMII, 0x1),
+	MVEBU_COMPHY_CONF(0, 1, PHY_MODE_2500SGMII, 0x1),
 	/* lane 1 */
 	MVEBU_COMPHY_CONF(1, 2, PHY_MODE_SGMII, 0x1),
+	MVEBU_COMPHY_CONF(1, 2, PHY_MODE_2500SGMII, 0x1),
 	/* lane 2 */
 	MVEBU_COMPHY_CONF(2, 0, PHY_MODE_SGMII, 0x1),
+	MVEBU_COMPHY_CONF(2, 0, PHY_MODE_2500SGMII, 0x1),
 	MVEBU_COMPHY_CONF(2, 0, PHY_MODE_10GKR, 0x1),
 	/* lane 3 */
 	MVEBU_COMPHY_CONF(3, 1, PHY_MODE_SGMII, 0x2),
+	MVEBU_COMPHY_CONF(3, 1, PHY_MODE_2500SGMII, 0x2),
 	/* lane 4 */
 	MVEBU_COMPHY_CONF(4, 0, PHY_MODE_SGMII, 0x2),
+	MVEBU_COMPHY_CONF(4, 0, PHY_MODE_2500SGMII, 0x2),
 	MVEBU_COMPHY_CONF(4, 0, PHY_MODE_10GKR, 0x2),
 	MVEBU_COMPHY_CONF(4, 1, PHY_MODE_SGMII, 0x1),
 	/* lane 5 */
 	MVEBU_COMPHY_CONF(5, 2, PHY_MODE_SGMII, 0x1),
+	MVEBU_COMPHY_CONF(5, 2, PHY_MODE_2500SGMII, 0x1),
 };
 
 struct mvebu_comphy_priv {
@@ -206,6 +212,10 @@ static void mvebu_comphy_ethernet_init_reset(struct mvebu_comphy_lane *lane,
 	if (mode == PHY_MODE_10GKR)
 		val |= MVEBU_COMPHY_SERDES_CFG0_GEN_RX(0xe) |
 		       MVEBU_COMPHY_SERDES_CFG0_GEN_TX(0xe);
+	else if (mode == PHY_MODE_2500SGMII)
+		val |= MVEBU_COMPHY_SERDES_CFG0_GEN_RX(0x8) |
+		       MVEBU_COMPHY_SERDES_CFG0_GEN_TX(0x8) |
+		       MVEBU_COMPHY_SERDES_CFG0_HALF_BUS;
 	else if (mode == PHY_MODE_SGMII)
 		val |= MVEBU_COMPHY_SERDES_CFG0_GEN_RX(0x6) |
 		       MVEBU_COMPHY_SERDES_CFG0_GEN_TX(0x6) |
@@ -296,13 +306,13 @@ static int mvebu_comphy_init_plls(struct mvebu_comphy_lane *lane,
 	return 0;
 }
 
-static int mvebu_comphy_set_mode_sgmii(struct phy *phy)
+static int mvebu_comphy_set_mode_sgmii(struct phy *phy, enum phy_mode mode)
 {
 	struct mvebu_comphy_lane *lane = phy_get_drvdata(phy);
 	struct mvebu_comphy_priv *priv = lane->priv;
 	u32 val;
 
-	mvebu_comphy_ethernet_init_reset(lane, PHY_MODE_SGMII);
+	mvebu_comphy_ethernet_init_reset(lane, mode);
 
 	val = readl(priv->base + MVEBU_COMPHY_RX_CTRL1(lane->id));
 	val &= ~MVEBU_COMPHY_RX_CTRL1_CLK8T_EN;
@@ -487,7 +497,8 @@ static int mvebu_comphy_power_on(struct phy *phy)
 
 	switch (lane->mode) {
 	case PHY_MODE_SGMII:
-		ret = mvebu_comphy_set_mode_sgmii(phy);
+	case PHY_MODE_2500SGMII:
+		ret = mvebu_comphy_set_mode_sgmii(phy, lane->mode);
 		break;
 	case PHY_MODE_10GKR:
 		ret = mvebu_comphy_set_mode_10gkr(phy);
diff --git a/drivers/phy/mediatek/Kconfig b/drivers/phy/mediatek/Kconfig
index 88ab4e25e34f..8857d00b3c65 100644
--- a/drivers/phy/mediatek/Kconfig
+++ b/drivers/phy/mediatek/Kconfig
@@ -12,3 +12,12 @@ config PHY_MTK_TPHY
 	  different banks layout, the T-PHY with shared banks between
 	  multi-ports is first version, otherwise is second veriosn,
 	  so you can easily distinguish them by banks layout.
+
+config PHY_MTK_XSPHY
+    tristate "MediaTek XS-PHY Driver"
+    depends on ARCH_MEDIATEK && OF
+    select GENERIC_PHY
+    help
+	  Enable this to support the SuperSpeedPlus XS-PHY transceiver for
+	  USB3.1 GEN2 controllers on MediaTek chips. The driver supports
+	  multiple USB2.0, USB3.1 GEN2 ports.
diff --git a/drivers/phy/mediatek/Makefile b/drivers/phy/mediatek/Makefile
index 763a92eefa00..e5074b607d3d 100644
--- a/drivers/phy/mediatek/Makefile
+++ b/drivers/phy/mediatek/Makefile
@@ -3,3 +3,4 @@
 #
 
 obj-$(CONFIG_PHY_MTK_TPHY)		+= phy-mtk-tphy.o
+obj-$(CONFIG_PHY_MTK_XSPHY)		+= phy-mtk-xsphy.o
diff --git a/drivers/phy/mediatek/phy-mtk-xsphy.c b/drivers/phy/mediatek/phy-mtk-xsphy.c
new file mode 100644
index 000000000000..020cd0227397
--- /dev/null
+++ b/drivers/phy/mediatek/phy-mtk-xsphy.c
@@ -0,0 +1,600 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * MediaTek USB3.1 gen2 xsphy Driver
+ *
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ *
+ */
+
+#include <dt-bindings/phy/phy.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+
+/* u2 phy banks */
+#define SSUSB_SIFSLV_MISC		0x000
+#define SSUSB_SIFSLV_U2FREQ		0x100
+#define SSUSB_SIFSLV_U2PHY_COM	0x300
+
+/* u3 phy shared banks */
+#define SSPXTP_SIFSLV_DIG_GLB		0x000
+#define SSPXTP_SIFSLV_PHYA_GLB		0x100
+
+/* u3 phy banks */
+#define SSPXTP_SIFSLV_DIG_LN_TOP	0x000
+#define SSPXTP_SIFSLV_DIG_LN_TX0	0x100
+#define SSPXTP_SIFSLV_DIG_LN_RX0	0x200
+#define SSPXTP_SIFSLV_DIG_LN_DAIF	0x300
+#define SSPXTP_SIFSLV_PHYA_LN		0x400
+
+#define XSP_U2FREQ_FMCR0	((SSUSB_SIFSLV_U2FREQ) + 0x00)
+#define P2F_RG_FREQDET_EN	BIT(24)
+#define P2F_RG_CYCLECNT		GENMASK(23, 0)
+#define P2F_RG_CYCLECNT_VAL(x)	((P2F_RG_CYCLECNT) & (x))
+
+#define XSP_U2FREQ_MMONR0  ((SSUSB_SIFSLV_U2FREQ) + 0x0c)
+
+#define XSP_U2FREQ_FMMONR1	((SSUSB_SIFSLV_U2FREQ) + 0x10)
+#define P2F_RG_FRCK_EN		BIT(8)
+#define P2F_USB_FM_VALID	BIT(0)
+
+#define XSP_USBPHYACR0	((SSUSB_SIFSLV_U2PHY_COM) + 0x00)
+#define P2A0_RG_INTR_EN	BIT(5)
+
+#define XSP_USBPHYACR1		((SSUSB_SIFSLV_U2PHY_COM) + 0x04)
+#define P2A1_RG_INTR_CAL		GENMASK(23, 19)
+#define P2A1_RG_INTR_CAL_VAL(x)	((0x1f & (x)) << 19)
+#define P2A1_RG_VRT_SEL			GENMASK(14, 12)
+#define P2A1_RG_VRT_SEL_VAL(x)	((0x7 & (x)) << 12)
+#define P2A1_RG_TERM_SEL		GENMASK(10, 8)
+#define P2A1_RG_TERM_SEL_VAL(x)	((0x7 & (x)) << 8)
+
+#define XSP_USBPHYACR5		((SSUSB_SIFSLV_U2PHY_COM) + 0x014)
+#define P2A5_RG_HSTX_SRCAL_EN	BIT(15)
+#define P2A5_RG_HSTX_SRCTRL		GENMASK(14, 12)
+#define P2A5_RG_HSTX_SRCTRL_VAL(x)	((0x7 & (x)) << 12)
+
+#define XSP_USBPHYACR6		((SSUSB_SIFSLV_U2PHY_COM) + 0x018)
+#define P2A6_RG_BC11_SW_EN	BIT(23)
+#define P2A6_RG_OTG_VBUSCMP_EN	BIT(20)
+
+#define XSP_U2PHYDTM1		((SSUSB_SIFSLV_U2PHY_COM) + 0x06C)
+#define P2D_FORCE_IDDIG		BIT(9)
+#define P2D_RG_VBUSVALID	BIT(5)
+#define P2D_RG_SESSEND		BIT(4)
+#define P2D_RG_AVALID		BIT(2)
+#define P2D_RG_IDDIG		BIT(1)
+
+#define SSPXTP_PHYA_GLB_00		((SSPXTP_SIFSLV_PHYA_GLB) + 0x00)
+#define RG_XTP_GLB_BIAS_INTR_CTRL		GENMASK(21, 16)
+#define RG_XTP_GLB_BIAS_INTR_CTRL_VAL(x)	((0x3f & (x)) << 16)
+
+#define SSPXTP_PHYA_LN_04	((SSPXTP_SIFSLV_PHYA_LN) + 0x04)
+#define RG_XTP_LN0_TX_IMPSEL		GENMASK(4, 0)
+#define RG_XTP_LN0_TX_IMPSEL_VAL(x)	(0x1f & (x))
+
+#define SSPXTP_PHYA_LN_14	((SSPXTP_SIFSLV_PHYA_LN) + 0x014)
+#define RG_XTP_LN0_RX_IMPSEL		GENMASK(4, 0)
+#define RG_XTP_LN0_RX_IMPSEL_VAL(x)	(0x1f & (x))
+
+#define XSP_REF_CLK		26	/* MHZ */
+#define XSP_SLEW_RATE_COEF	17
+#define XSP_SR_COEF_DIVISOR	1000
+#define XSP_FM_DET_CYCLE_CNT	1024
+
+struct xsphy_instance {
+	struct phy *phy;
+	void __iomem *port_base;
+	struct clk *ref_clk;	/* reference clock of anolog phy */
+	u32 index;
+	u32 type;
+	/* only for HQA test */
+	int efuse_intr;
+	int efuse_tx_imp;
+	int efuse_rx_imp;
+	/* u2 eye diagram */
+	int eye_src;
+	int eye_vrt;
+	int eye_term;
+};
+
+struct mtk_xsphy {
+	struct device *dev;
+	void __iomem *glb_base;	/* only shared u3 sif */
+	struct xsphy_instance **phys;
+	int nphys;
+	int src_ref_clk; /* MHZ, reference clock for slew rate calibrate */
+	int src_coef;    /* coefficient for slew rate calibrate */
+};
+
+static void u2_phy_slew_rate_calibrate(struct mtk_xsphy *xsphy,
+					struct xsphy_instance *inst)
+{
+	void __iomem *pbase = inst->port_base;
+	int calib_val;
+	int fm_out;
+	u32 tmp;
+
+	/* use force value */
+	if (inst->eye_src)
+		return;
+
+	/* enable USB ring oscillator */
+	tmp = readl(pbase + XSP_USBPHYACR5);
+	tmp |= P2A5_RG_HSTX_SRCAL_EN;
+	writel(tmp, pbase + XSP_USBPHYACR5);
+	udelay(1);	/* wait clock stable */
+
+	/* enable free run clock */
+	tmp = readl(pbase + XSP_U2FREQ_FMMONR1);
+	tmp |= P2F_RG_FRCK_EN;
+	writel(tmp, pbase + XSP_U2FREQ_FMMONR1);
+
+	/* set cycle count as 1024 */
+	tmp = readl(pbase + XSP_U2FREQ_FMCR0);
+	tmp &= ~(P2F_RG_CYCLECNT);
+	tmp |= P2F_RG_CYCLECNT_VAL(XSP_FM_DET_CYCLE_CNT);
+	writel(tmp, pbase + XSP_U2FREQ_FMCR0);
+
+	/* enable frequency meter */
+	tmp = readl(pbase + XSP_U2FREQ_FMCR0);
+	tmp |= P2F_RG_FREQDET_EN;
+	writel(tmp, pbase + XSP_U2FREQ_FMCR0);
+
+	/* ignore return value */
+	readl_poll_timeout(pbase + XSP_U2FREQ_FMMONR1, tmp,
+			   (tmp & P2F_USB_FM_VALID), 10, 200);
+
+	fm_out = readl(pbase + XSP_U2FREQ_MMONR0);
+
+	/* disable frequency meter */
+	tmp = readl(pbase + XSP_U2FREQ_FMCR0);
+	tmp &= ~P2F_RG_FREQDET_EN;
+	writel(tmp, pbase + XSP_U2FREQ_FMCR0);
+
+	/* disable free run clock */
+	tmp = readl(pbase + XSP_U2FREQ_FMMONR1);
+	tmp &= ~P2F_RG_FRCK_EN;
+	writel(tmp, pbase + XSP_U2FREQ_FMMONR1);
+
+	if (fm_out) {
+		/* (1024 / FM_OUT) x reference clock frequency x coefficient */
+		tmp = xsphy->src_ref_clk * xsphy->src_coef;
+		tmp = (tmp * XSP_FM_DET_CYCLE_CNT) / fm_out;
+		calib_val = DIV_ROUND_CLOSEST(tmp, XSP_SR_COEF_DIVISOR);
+	} else {
+		/* if FM detection fail, set default value */
+		calib_val = 3;
+	}
+	dev_dbg(xsphy->dev, "phy.%d, fm_out:%d, calib:%d (clk:%d, coef:%d)\n",
+		inst->index, fm_out, calib_val,
+		xsphy->src_ref_clk, xsphy->src_coef);
+
+	/* set HS slew rate */
+	tmp = readl(pbase + XSP_USBPHYACR5);
+	tmp &= ~P2A5_RG_HSTX_SRCTRL;
+	tmp |= P2A5_RG_HSTX_SRCTRL_VAL(calib_val);
+	writel(tmp, pbase + XSP_USBPHYACR5);
+
+	/* disable USB ring oscillator */
+	tmp = readl(pbase + XSP_USBPHYACR5);
+	tmp &= ~P2A5_RG_HSTX_SRCAL_EN;
+	writel(tmp, pbase + XSP_USBPHYACR5);
+}
+
+static void u2_phy_instance_init(struct mtk_xsphy *xsphy,
+				 struct xsphy_instance *inst)
+{
+	void __iomem *pbase = inst->port_base;
+	u32 tmp;
+
+	/* DP/DM BC1.1 path Disable */
+	tmp = readl(pbase + XSP_USBPHYACR6);
+	tmp &= ~P2A6_RG_BC11_SW_EN;
+	writel(tmp, pbase + XSP_USBPHYACR6);
+
+	tmp = readl(pbase + XSP_USBPHYACR0);
+	tmp |= P2A0_RG_INTR_EN;
+	writel(tmp, pbase + XSP_USBPHYACR0);
+}
+
+static void u2_phy_instance_power_on(struct mtk_xsphy *xsphy,
+				     struct xsphy_instance *inst)
+{
+	void __iomem *pbase = inst->port_base;
+	u32 index = inst->index;
+	u32 tmp;
+
+	tmp = readl(pbase + XSP_USBPHYACR6);
+	tmp |= P2A6_RG_OTG_VBUSCMP_EN;
+	writel(tmp, pbase + XSP_USBPHYACR6);
+
+	tmp = readl(pbase + XSP_U2PHYDTM1);
+	tmp |= P2D_RG_VBUSVALID | P2D_RG_AVALID;
+	tmp &= ~P2D_RG_SESSEND;
+	writel(tmp, pbase + XSP_U2PHYDTM1);
+
+	dev_dbg(xsphy->dev, "%s(%d)\n", __func__, index);
+}
+
+static void u2_phy_instance_power_off(struct mtk_xsphy *xsphy,
+				      struct xsphy_instance *inst)
+{
+	void __iomem *pbase = inst->port_base;
+	u32 index = inst->index;
+	u32 tmp;
+
+	tmp = readl(pbase + XSP_USBPHYACR6);
+	tmp &= ~P2A6_RG_OTG_VBUSCMP_EN;
+	writel(tmp, pbase + XSP_USBPHYACR6);
+
+	tmp = readl(pbase + XSP_U2PHYDTM1);
+	tmp &= ~(P2D_RG_VBUSVALID | P2D_RG_AVALID);
+	tmp |= P2D_RG_SESSEND;
+	writel(tmp, pbase + XSP_U2PHYDTM1);
+
+	dev_dbg(xsphy->dev, "%s(%d)\n", __func__, index);
+}
+
+static void u2_phy_instance_set_mode(struct mtk_xsphy *xsphy,
+				     struct xsphy_instance *inst,
+				     enum phy_mode mode)
+{
+	u32 tmp;
+
+	tmp = readl(inst->port_base + XSP_U2PHYDTM1);
+	switch (mode) {
+	case PHY_MODE_USB_DEVICE:
+		tmp |= P2D_FORCE_IDDIG | P2D_RG_IDDIG;
+		break;
+	case PHY_MODE_USB_HOST:
+		tmp |= P2D_FORCE_IDDIG;
+		tmp &= ~P2D_RG_IDDIG;
+		break;
+	case PHY_MODE_USB_OTG:
+		tmp &= ~(P2D_FORCE_IDDIG | P2D_RG_IDDIG);
+		break;
+	default:
+		return;
+	}
+	writel(tmp, inst->port_base + XSP_U2PHYDTM1);
+}
+
+static void phy_parse_property(struct mtk_xsphy *xsphy,
+				struct xsphy_instance *inst)
+{
+	struct device *dev = &inst->phy->dev;
+
+	switch (inst->type) {
+	case PHY_TYPE_USB2:
+		device_property_read_u32(dev, "mediatek,efuse-intr",
+					 &inst->efuse_intr);
+		device_property_read_u32(dev, "mediatek,eye-src",
+					 &inst->eye_src);
+		device_property_read_u32(dev, "mediatek,eye-vrt",
+					 &inst->eye_vrt);
+		device_property_read_u32(dev, "mediatek,eye-term",
+					 &inst->eye_term);
+		dev_dbg(dev, "intr:%d, src:%d, vrt:%d, term:%d\n",
+			inst->efuse_intr, inst->eye_src,
+			inst->eye_vrt, inst->eye_term);
+		break;
+	case PHY_TYPE_USB3:
+		device_property_read_u32(dev, "mediatek,efuse-intr",
+					 &inst->efuse_intr);
+		device_property_read_u32(dev, "mediatek,efuse-tx-imp",
+					 &inst->efuse_tx_imp);
+		device_property_read_u32(dev, "mediatek,efuse-rx-imp",
+					 &inst->efuse_rx_imp);
+		dev_dbg(dev, "intr:%d, tx-imp:%d, rx-imp:%d\n",
+			inst->efuse_intr, inst->efuse_tx_imp,
+			inst->efuse_rx_imp);
+		break;
+	default:
+		dev_err(xsphy->dev, "incompatible phy type\n");
+		return;
+	}
+}
+
+static void u2_phy_props_set(struct mtk_xsphy *xsphy,
+			     struct xsphy_instance *inst)
+{
+	void __iomem *pbase = inst->port_base;
+	u32 tmp;
+
+	if (inst->efuse_intr) {
+		tmp = readl(pbase + XSP_USBPHYACR1);
+		tmp &= ~P2A1_RG_INTR_CAL;
+		tmp |= P2A1_RG_INTR_CAL_VAL(inst->efuse_intr);
+		writel(tmp, pbase + XSP_USBPHYACR1);
+	}
+
+	if (inst->eye_src) {
+		tmp = readl(pbase + XSP_USBPHYACR5);
+		tmp &= ~P2A5_RG_HSTX_SRCTRL;
+		tmp |= P2A5_RG_HSTX_SRCTRL_VAL(inst->eye_src);
+		writel(tmp, pbase + XSP_USBPHYACR5);
+	}
+
+	if (inst->eye_vrt) {
+		tmp = readl(pbase + XSP_USBPHYACR1);
+		tmp &= ~P2A1_RG_VRT_SEL;
+		tmp |= P2A1_RG_VRT_SEL_VAL(inst->eye_vrt);
+		writel(tmp, pbase + XSP_USBPHYACR1);
+	}
+
+	if (inst->eye_term) {
+		tmp = readl(pbase + XSP_USBPHYACR1);
+		tmp &= ~P2A1_RG_TERM_SEL;
+		tmp |= P2A1_RG_TERM_SEL_VAL(inst->eye_term);
+		writel(tmp, pbase + XSP_USBPHYACR1);
+	}
+}
+
+static void u3_phy_props_set(struct mtk_xsphy *xsphy,
+			     struct xsphy_instance *inst)
+{
+	void __iomem *pbase = inst->port_base;
+	u32 tmp;
+
+	if (inst->efuse_intr) {
+		tmp = readl(xsphy->glb_base + SSPXTP_PHYA_GLB_00);
+		tmp &= ~RG_XTP_GLB_BIAS_INTR_CTRL;
+		tmp |= RG_XTP_GLB_BIAS_INTR_CTRL_VAL(inst->efuse_intr);
+		writel(tmp, xsphy->glb_base + SSPXTP_PHYA_GLB_00);
+	}
+
+	if (inst->efuse_tx_imp) {
+		tmp = readl(pbase + SSPXTP_PHYA_LN_04);
+		tmp &= ~RG_XTP_LN0_TX_IMPSEL;
+		tmp |= RG_XTP_LN0_TX_IMPSEL_VAL(inst->efuse_tx_imp);
+		writel(tmp, pbase + SSPXTP_PHYA_LN_04);
+	}
+
+	if (inst->efuse_rx_imp) {
+		tmp = readl(pbase + SSPXTP_PHYA_LN_14);
+		tmp &= ~RG_XTP_LN0_RX_IMPSEL;
+		tmp |= RG_XTP_LN0_RX_IMPSEL_VAL(inst->efuse_rx_imp);
+		writel(tmp, pbase + SSPXTP_PHYA_LN_14);
+	}
+}
+
+static int mtk_phy_init(struct phy *phy)
+{
+	struct xsphy_instance *inst = phy_get_drvdata(phy);
+	struct mtk_xsphy *xsphy = dev_get_drvdata(phy->dev.parent);
+	int ret;
+
+	ret = clk_prepare_enable(inst->ref_clk);
+	if (ret) {
+		dev_err(xsphy->dev, "failed to enable ref_clk\n");
+		return ret;
+	}
+
+	switch (inst->type) {
+	case PHY_TYPE_USB2:
+		u2_phy_instance_init(xsphy, inst);
+		u2_phy_props_set(xsphy, inst);
+		break;
+	case PHY_TYPE_USB3:
+		u3_phy_props_set(xsphy, inst);
+		break;
+	default:
+		dev_err(xsphy->dev, "incompatible phy type\n");
+		clk_disable_unprepare(inst->ref_clk);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int mtk_phy_power_on(struct phy *phy)
+{
+	struct xsphy_instance *inst = phy_get_drvdata(phy);
+	struct mtk_xsphy *xsphy = dev_get_drvdata(phy->dev.parent);
+
+	if (inst->type == PHY_TYPE_USB2) {
+		u2_phy_instance_power_on(xsphy, inst);
+		u2_phy_slew_rate_calibrate(xsphy, inst);
+	}
+
+	return 0;
+}
+
+static int mtk_phy_power_off(struct phy *phy)
+{
+	struct xsphy_instance *inst = phy_get_drvdata(phy);
+	struct mtk_xsphy *xsphy = dev_get_drvdata(phy->dev.parent);
+
+	if (inst->type == PHY_TYPE_USB2)
+		u2_phy_instance_power_off(xsphy, inst);
+
+	return 0;
+}
+
+static int mtk_phy_exit(struct phy *phy)
+{
+	struct xsphy_instance *inst = phy_get_drvdata(phy);
+
+	clk_disable_unprepare(inst->ref_clk);
+	return 0;
+}
+
+static int mtk_phy_set_mode(struct phy *phy, enum phy_mode mode)
+{
+	struct xsphy_instance *inst = phy_get_drvdata(phy);
+	struct mtk_xsphy *xsphy = dev_get_drvdata(phy->dev.parent);
+
+	if (inst->type == PHY_TYPE_USB2)
+		u2_phy_instance_set_mode(xsphy, inst, mode);
+
+	return 0;
+}
+
+static struct phy *mtk_phy_xlate(struct device *dev,
+				 struct of_phandle_args *args)
+{
+	struct mtk_xsphy *xsphy = dev_get_drvdata(dev);
+	struct xsphy_instance *inst = NULL;
+	struct device_node *phy_np = args->np;
+	int index;
+
+	if (args->args_count != 1) {
+		dev_err(dev, "invalid number of cells in 'phy' property\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	for (index = 0; index < xsphy->nphys; index++)
+		if (phy_np == xsphy->phys[index]->phy->dev.of_node) {
+			inst = xsphy->phys[index];
+			break;
+		}
+
+	if (!inst) {
+		dev_err(dev, "failed to find appropriate phy\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	inst->type = args->args[0];
+	if (!(inst->type == PHY_TYPE_USB2 ||
+	      inst->type == PHY_TYPE_USB3)) {
+		dev_err(dev, "unsupported phy type: %d\n", inst->type);
+		return ERR_PTR(-EINVAL);
+	}
+
+	phy_parse_property(xsphy, inst);
+
+	return inst->phy;
+}
+
+static const struct phy_ops mtk_xsphy_ops = {
+	.init		= mtk_phy_init,
+	.exit		= mtk_phy_exit,
+	.power_on	= mtk_phy_power_on,
+	.power_off	= mtk_phy_power_off,
+	.set_mode	= mtk_phy_set_mode,
+	.owner		= THIS_MODULE,
+};
+
+static const struct of_device_id mtk_xsphy_id_table[] = {
+	{ .compatible = "mediatek,xsphy", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, mtk_xsphy_id_table);
+
+static int mtk_xsphy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct device_node *child_np;
+	struct phy_provider *provider;
+	struct resource *glb_res;
+	struct mtk_xsphy *xsphy;
+	struct resource res;
+	int port, retval;
+
+	xsphy = devm_kzalloc(dev, sizeof(*xsphy), GFP_KERNEL);
+	if (!xsphy)
+		return -ENOMEM;
+
+	xsphy->nphys = of_get_child_count(np);
+	xsphy->phys = devm_kcalloc(dev, xsphy->nphys,
+				       sizeof(*xsphy->phys), GFP_KERNEL);
+	if (!xsphy->phys)
+		return -ENOMEM;
+
+	xsphy->dev = dev;
+	platform_set_drvdata(pdev, xsphy);
+
+	glb_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	/* optional, may not exist if no u3 phys */
+	if (glb_res) {
+		/* get banks shared by multiple u3 phys */
+		xsphy->glb_base = devm_ioremap_resource(dev, glb_res);
+		if (IS_ERR(xsphy->glb_base)) {
+			dev_err(dev, "failed to remap glb regs\n");
+			return PTR_ERR(xsphy->glb_base);
+		}
+	}
+
+	xsphy->src_ref_clk = XSP_REF_CLK;
+	xsphy->src_coef = XSP_SLEW_RATE_COEF;
+	/* update parameters of slew rate calibrate if exist */
+	device_property_read_u32(dev, "mediatek,src-ref-clk-mhz",
+				 &xsphy->src_ref_clk);
+	device_property_read_u32(dev, "mediatek,src-coef", &xsphy->src_coef);
+
+	port = 0;
+	for_each_child_of_node(np, child_np) {
+		struct xsphy_instance *inst;
+		struct phy *phy;
+
+		inst = devm_kzalloc(dev, sizeof(*inst), GFP_KERNEL);
+		if (!inst) {
+			retval = -ENOMEM;
+			goto put_child;
+		}
+
+		xsphy->phys[port] = inst;
+
+		phy = devm_phy_create(dev, child_np, &mtk_xsphy_ops);
+		if (IS_ERR(phy)) {
+			dev_err(dev, "failed to create phy\n");
+			retval = PTR_ERR(phy);
+			goto put_child;
+		}
+
+		retval = of_address_to_resource(child_np, 0, &res);
+		if (retval) {
+			dev_err(dev, "failed to get address resource(id-%d)\n",
+				port);
+			goto put_child;
+		}
+
+		inst->port_base = devm_ioremap_resource(&phy->dev, &res);
+		if (IS_ERR(inst->port_base)) {
+			dev_err(dev, "failed to remap phy regs\n");
+			retval = PTR_ERR(inst->port_base);
+			goto put_child;
+		}
+
+		inst->phy = phy;
+		inst->index = port;
+		phy_set_drvdata(phy, inst);
+		port++;
+
+		inst->ref_clk = devm_clk_get(&phy->dev, "ref");
+		if (IS_ERR(inst->ref_clk)) {
+			dev_err(dev, "failed to get ref_clk(id-%d)\n", port);
+			retval = PTR_ERR(inst->ref_clk);
+			goto put_child;
+		}
+	}
+
+	provider = devm_of_phy_provider_register(dev, mtk_phy_xlate);
+	return PTR_ERR_OR_ZERO(provider);
+
+put_child:
+	of_node_put(child_np);
+	return retval;
+}
+
+static struct platform_driver mtk_xsphy_driver = {
+	.probe		= mtk_xsphy_probe,
+	.driver		= {
+		.name	= "mtk-xsphy",
+		.of_match_table = mtk_xsphy_id_table,
+	},
+};
+
+module_platform_driver(mtk_xsphy_driver);
+
+MODULE_AUTHOR("Chunfeng Yun <chunfeng.yun@mediatek.com>");
+MODULE_DESCRIPTION("MediaTek USB XS-PHY driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/motorola/phy-mapphone-mdm6600.c b/drivers/phy/motorola/phy-mapphone-mdm6600.c
index 5439dd90d0dd..23705e1a0023 100644
--- a/drivers/phy/motorola/phy-mapphone-mdm6600.c
+++ b/drivers/phy/motorola/phy-mapphone-mdm6600.c
@@ -19,6 +19,8 @@
 
 #define PHY_MDM6600_PHY_DELAY_MS	4000	/* PHY enable 2.2s to 3.5s */
 #define PHY_MDM6600_ENABLED_DELAY_MS	8000	/* 8s more total for MDM6600 */
+#define MDM6600_MODEM_IDLE_DELAY_MS	1000	/* modem after USB suspend */
+#define MDM6600_MODEM_WAKE_DELAY_MS	200	/* modem response after idle */
 
 enum phy_mdm6600_ctrl_lines {
 	PHY_MDM6600_ENABLE,			/* USB PHY enable */
@@ -93,9 +95,11 @@ struct phy_mdm6600 {
 	struct gpio_descs *cmd_gpios;
 	struct delayed_work bootup_work;
 	struct delayed_work status_work;
+	struct delayed_work modem_wake_work;
 	struct completion ack;
 	bool enabled;				/* mdm6600 phy enabled */
 	bool running;				/* mdm6600 boot done */
+	bool awake;				/* mdm6600 respnds on n_gsm */
 	int status;
 };
 
@@ -446,6 +450,62 @@ static void phy_mdm6600_deferred_power_on(struct work_struct *work)
 		dev_err(ddata->dev, "Device not functional\n");
 }
 
+/*
+ * USB suspend puts mdm6600 into low power mode. For any n_gsm using apps,
+ * we need to keep the modem awake by kicking it's mode0 GPIO. This will
+ * keep the modem awake for about 1.2 seconds. When no n_gsm apps are using
+ * the modem, runtime PM auto mode can be enabled so modem can enter low
+ * power mode.
+ */
+static void phy_mdm6600_wake_modem(struct phy_mdm6600 *ddata)
+{
+	struct gpio_desc *mode_gpio0;
+
+	mode_gpio0 = ddata->mode_gpios->desc[PHY_MDM6600_MODE0];
+	gpiod_set_value_cansleep(mode_gpio0, 1);
+	usleep_range(5, 15);
+	gpiod_set_value_cansleep(mode_gpio0, 0);
+	if (ddata->awake)
+		usleep_range(5, 15);
+	else
+		msleep(MDM6600_MODEM_WAKE_DELAY_MS);
+}
+
+static void phy_mdm6600_modem_wake(struct work_struct *work)
+{
+	struct phy_mdm6600 *ddata;
+
+	ddata = container_of(work, struct phy_mdm6600, modem_wake_work.work);
+	phy_mdm6600_wake_modem(ddata);
+	schedule_delayed_work(&ddata->modem_wake_work,
+			      msecs_to_jiffies(MDM6600_MODEM_IDLE_DELAY_MS));
+}
+
+static int __maybe_unused phy_mdm6600_runtime_suspend(struct device *dev)
+{
+	struct phy_mdm6600 *ddata = dev_get_drvdata(dev);
+
+	cancel_delayed_work_sync(&ddata->modem_wake_work);
+	ddata->awake = false;
+
+	return 0;
+}
+
+static int __maybe_unused phy_mdm6600_runtime_resume(struct device *dev)
+{
+	struct phy_mdm6600 *ddata = dev_get_drvdata(dev);
+
+	phy_mdm6600_modem_wake(&ddata->modem_wake_work.work);
+	ddata->awake = true;
+
+	return 0;
+}
+
+static const struct dev_pm_ops phy_mdm6600_pm_ops = {
+	SET_RUNTIME_PM_OPS(phy_mdm6600_runtime_suspend,
+			   phy_mdm6600_runtime_resume, NULL)
+};
+
 static const struct of_device_id phy_mdm6600_id_table[] = {
 	{ .compatible = "motorola,mapphone-mdm6600", },
 	{},
@@ -464,6 +524,7 @@ static int phy_mdm6600_probe(struct platform_device *pdev)
 	INIT_DELAYED_WORK(&ddata->bootup_work,
 			  phy_mdm6600_deferred_power_on);
 	INIT_DELAYED_WORK(&ddata->status_work, phy_mdm6600_status);
+	INIT_DELAYED_WORK(&ddata->modem_wake_work, phy_mdm6600_modem_wake);
 	init_completion(&ddata->ack);
 
 	ddata->dev = &pdev->dev;
@@ -500,6 +561,24 @@ static int phy_mdm6600_probe(struct platform_device *pdev)
 	 */
 	msleep(PHY_MDM6600_PHY_DELAY_MS + 500);
 
+	/*
+	 * Enable PM runtime only after PHY has been powered up properly.
+	 * It is currently only needed after USB suspends mdm6600 and n_gsm
+	 * needs to access the device. We don't want to do this earlier as
+	 * gpio mode0 pin doubles as mdm6600 wake-up gpio.
+	 */
+	pm_runtime_use_autosuspend(ddata->dev);
+	pm_runtime_set_autosuspend_delay(ddata->dev,
+					 MDM6600_MODEM_IDLE_DELAY_MS);
+	pm_runtime_enable(ddata->dev);
+	error = pm_runtime_get_sync(ddata->dev);
+	if (error < 0) {
+		dev_warn(ddata->dev, "failed to wake modem: %i\n", error);
+		pm_runtime_put_noidle(ddata->dev);
+	}
+	pm_runtime_mark_last_busy(ddata->dev);
+	pm_runtime_put_autosuspend(ddata->dev);
+
 	return 0;
 
 cleanup:
@@ -512,6 +591,10 @@ static int phy_mdm6600_remove(struct platform_device *pdev)
 	struct phy_mdm6600 *ddata = platform_get_drvdata(pdev);
 	struct gpio_desc *reset_gpio = ddata->ctrl_gpios[PHY_MDM6600_RESET];
 
+	pm_runtime_dont_use_autosuspend(ddata->dev);
+	pm_runtime_put_sync(ddata->dev);
+	pm_runtime_disable(ddata->dev);
+
 	if (!ddata->running)
 		wait_for_completion_timeout(&ddata->ack,
 			msecs_to_jiffies(PHY_MDM6600_ENABLED_DELAY_MS));
@@ -519,6 +602,7 @@ static int phy_mdm6600_remove(struct platform_device *pdev)
 	gpiod_set_value_cansleep(reset_gpio, 1);
 	phy_mdm6600_device_power_off(ddata);
 
+	cancel_delayed_work_sync(&ddata->modem_wake_work);
 	cancel_delayed_work_sync(&ddata->bootup_work);
 	cancel_delayed_work_sync(&ddata->status_work);
 
@@ -530,6 +614,7 @@ static struct platform_driver phy_mdm6600_driver = {
 	.remove = phy_mdm6600_remove,
 	.driver = {
 		.name = "phy-mapphone-mdm6600",
+		.pm = &phy_mdm6600_pm_ops,
 		.of_match_table = of_match_ptr(phy_mdm6600_id_table),
 	},
 };
diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c
index 09ac8afb97ac..35fd38c5a4a1 100644
--- a/drivers/phy/phy-core.c
+++ b/drivers/phy/phy-core.c
@@ -153,6 +153,9 @@ int phy_pm_runtime_get(struct phy *phy)
 {
 	int ret;
 
+	if (!phy)
+		return 0;
+
 	if (!pm_runtime_enabled(&phy->dev))
 		return -ENOTSUPP;
 
@@ -168,6 +171,9 @@ int phy_pm_runtime_get_sync(struct phy *phy)
 {
 	int ret;
 
+	if (!phy)
+		return 0;
+
 	if (!pm_runtime_enabled(&phy->dev))
 		return -ENOTSUPP;
 
@@ -181,6 +187,9 @@ EXPORT_SYMBOL_GPL(phy_pm_runtime_get_sync);
 
 int phy_pm_runtime_put(struct phy *phy)
 {
+	if (!phy)
+		return 0;
+
 	if (!pm_runtime_enabled(&phy->dev))
 		return -ENOTSUPP;
 
@@ -190,6 +199,9 @@ EXPORT_SYMBOL_GPL(phy_pm_runtime_put);
 
 int phy_pm_runtime_put_sync(struct phy *phy)
 {
+	if (!phy)
+		return 0;
+
 	if (!pm_runtime_enabled(&phy->dev))
 		return -ENOTSUPP;
 
@@ -199,6 +211,9 @@ EXPORT_SYMBOL_GPL(phy_pm_runtime_put_sync);
 
 void phy_pm_runtime_allow(struct phy *phy)
 {
+	if (!phy)
+		return;
+
 	if (!pm_runtime_enabled(&phy->dev))
 		return;
 
@@ -208,6 +223,9 @@ EXPORT_SYMBOL_GPL(phy_pm_runtime_allow);
 
 void phy_pm_runtime_forbid(struct phy *phy)
 {
+	if (!phy)
+		return;
+
 	if (!pm_runtime_enabled(&phy->dev))
 		return;
 
diff --git a/drivers/phy/qualcomm/Kconfig b/drivers/phy/qualcomm/Kconfig
index 7bfa64baf837..632a0e73ee10 100644
--- a/drivers/phy/qualcomm/Kconfig
+++ b/drivers/phy/qualcomm/Kconfig
@@ -1,6 +1,15 @@
 #
-# Phy drivers for Qualcomm platforms
+# Phy drivers for Qualcomm and Atheros platforms
 #
+config PHY_ATH79_USB
+	tristate "Atheros AR71XX/9XXX USB PHY driver"
+	depends on OF && (ATH79 || COMPILE_TEST)
+	default y if USB_EHCI_HCD_PLATFORM || USB_OHCI_HCD_PLATFORM
+	select RESET_CONTROLLER
+	select GENERIC_PHY
+	help
+	  Enable this to support the USB PHY on Atheros AR71XX/9XXX SoCs.
+
 config PHY_QCOM_APQ8064_SATA
 	tristate "Qualcomm APQ8064 SATA SerDes/PHY driver"
 	depends on ARCH_QCOM
diff --git a/drivers/phy/qualcomm/Makefile b/drivers/phy/qualcomm/Makefile
index 9abb7899762a..deb831f453ae 100644
--- a/drivers/phy/qualcomm/Makefile
+++ b/drivers/phy/qualcomm/Makefile
@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_PHY_ATH79_USB)		+= phy-ath79-usb.o
 obj-$(CONFIG_PHY_QCOM_APQ8064_SATA)	+= phy-qcom-apq8064-sata.o
 obj-$(CONFIG_PHY_QCOM_IPQ806X_SATA)	+= phy-qcom-ipq806x-sata.o
 obj-$(CONFIG_PHY_QCOM_QMP)		+= phy-qcom-qmp.o
diff --git a/drivers/phy/qualcomm/phy-ath79-usb.c b/drivers/phy/qualcomm/phy-ath79-usb.c
new file mode 100644
index 000000000000..6fd6e07ab345
--- /dev/null
+++ b/drivers/phy/qualcomm/phy-ath79-usb.c
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Atheros AR71XX/9XXX USB PHY driver
+ *
+ * Copyright (C) 2015-2018 Alban Bedel <albeu@free.fr>
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/phy/phy.h>
+#include <linux/reset.h>
+
+struct ath79_usb_phy {
+	struct reset_control *reset;
+	/* The suspend override logic is inverted, hence the no prefix
+	 * to make the code a bit easier to understand.
+	 */
+	struct reset_control *no_suspend_override;
+};
+
+static int ath79_usb_phy_power_on(struct phy *phy)
+{
+	struct ath79_usb_phy *priv = phy_get_drvdata(phy);
+	int err = 0;
+
+	if (priv->no_suspend_override) {
+		err = reset_control_assert(priv->no_suspend_override);
+		if (err)
+			return err;
+	}
+
+	err = reset_control_deassert(priv->reset);
+	if (err && priv->no_suspend_override)
+		reset_control_assert(priv->no_suspend_override);
+
+	return err;
+}
+
+static int ath79_usb_phy_power_off(struct phy *phy)
+{
+	struct ath79_usb_phy *priv = phy_get_drvdata(phy);
+	int err = 0;
+
+	err = reset_control_assert(priv->reset);
+	if (err)
+		return err;
+
+	if (priv->no_suspend_override) {
+		err = reset_control_deassert(priv->no_suspend_override);
+		if (err)
+			reset_control_deassert(priv->reset);
+	}
+
+	return err;
+}
+
+static const struct phy_ops ath79_usb_phy_ops = {
+	.power_on	= ath79_usb_phy_power_on,
+	.power_off	= ath79_usb_phy_power_off,
+	.owner		= THIS_MODULE,
+};
+
+static int ath79_usb_phy_probe(struct platform_device *pdev)
+{
+	struct ath79_usb_phy *priv;
+	struct phy *phy;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->reset = devm_reset_control_get(&pdev->dev, "usb-phy");
+	if (IS_ERR(priv->reset))
+		return PTR_ERR(priv->reset);
+
+	priv->no_suspend_override = devm_reset_control_get_optional(
+		&pdev->dev, "usb-suspend-override");
+	if (IS_ERR(priv->no_suspend_override))
+		return PTR_ERR(priv->no_suspend_override);
+
+	phy = devm_phy_create(&pdev->dev, NULL, &ath79_usb_phy_ops);
+	if (IS_ERR(phy))
+		return PTR_ERR(phy);
+
+	phy_set_drvdata(phy, priv);
+
+	return PTR_ERR_OR_ZERO(devm_of_phy_provider_register(
+				&pdev->dev, of_phy_simple_xlate));
+}
+
+static const struct of_device_id ath79_usb_phy_of_match[] = {
+	{ .compatible = "qca,ar7100-usb-phy" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, ath79_usb_phy_of_match);
+
+static struct platform_driver ath79_usb_phy_driver = {
+	.probe	= ath79_usb_phy_probe,
+	.driver = {
+		.of_match_table	= ath79_usb_phy_of_match,
+		.name		= "ath79-usb-phy",
+	}
+};
+module_platform_driver(ath79_usb_phy_driver);
+
+MODULE_DESCRIPTION("ATH79 USB PHY driver");
+MODULE_AUTHOR("Alban Bedel <albeu@free.fr>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c
index 6470c5d61d1c..4c470104a0d6 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp.c
@@ -490,6 +490,118 @@ static const struct qmp_phy_init_tbl qmp_v3_usb3_pcs_tbl[] = {
 	QMP_PHY_INIT_CFG(QPHY_V3_PCS_RXEQTRAINING_RUN_TIME, 0x13),
 };
 
+static const struct qmp_phy_init_tbl qmp_v3_usb3_uniphy_serdes_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_PLL_IVCO, 0x07),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SYSCLK_EN_SEL, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_BIAS_EN_CLKBUFLR_EN, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CLK_SELECT, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SYS_CLK_CTRL, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_RESETSM_CNTRL2, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CMN_CONFIG, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SVS_MODE_CLK_SEL, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_HSCLK_SEL, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_DIV_FRAC_START1_MODE0, 0xab),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_DIV_FRAC_START2_MODE0, 0xea),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_DIV_FRAC_START3_MODE0, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CP_CTRL_MODE0, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_PLL_CCTRL_MODE0, 0x36),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_INTEGLOOP_GAIN0_MODE0, 0x3f),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_VCO_TUNE2_MODE0, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_VCO_TUNE1_MODE0, 0xc9),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CORECLK_DIV_MODE0, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_LOCK_CMP3_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_LOCK_CMP2_MODE0, 0x34),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_LOCK_CMP1_MODE0, 0x15),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_LOCK_CMP_EN, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CORE_CLK_EN, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_LOCK_CMP_CFG, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_VCO_TUNE_MAP, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SYSCLK_BUF_ENABLE, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_EN_CENTER, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_PER1, 0x31),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_PER2, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_ADJ_PER1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_ADJ_PER2, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_STEP_SIZE1, 0x85),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_STEP_SIZE2, 0x07),
+};
+
+static const struct qmp_phy_init_tbl qmp_v3_usb3_uniphy_tx_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_HIGHZ_DRVR_EN, 0x10),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_RCV_DETECT_LVL_2, 0x12),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_LANE_MODE_1, 0xc6),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_RES_CODE_LANE_OFFSET_RX, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_RES_CODE_LANE_OFFSET_TX, 0x06),
+};
+
+static const struct qmp_phy_init_tbl qmp_v3_usb3_uniphy_rx_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_VGA_CAL_CNTRL2, 0x0c),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_MODE_00, 0x50),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_UCDR_FASTLOCK_FO_GAIN, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0e),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4e),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_EQU_ADAPTOR_CNTRL4, 0x18),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x77),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_SIGDET_CNTRL, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_SIGDET_DEGLITCH_CNTRL, 0x1c),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x75),
+};
+
+static const struct qmp_phy_init_tbl qmp_v3_usb3_uniphy_pcs_tbl[] = {
+	/* FLL settings */
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_FLL_CNTRL2, 0x83),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_FLL_CNT_VAL_L, 0x09),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_FLL_CNT_VAL_H_TOL, 0xa2),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_FLL_MAN_CODE, 0x40),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_FLL_CNTRL1, 0x02),
+
+	/* Lock Det settings */
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_LOCK_DETECT_CONFIG1, 0xd1),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_LOCK_DETECT_CONFIG2, 0x1f),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_LOCK_DETECT_CONFIG3, 0x47),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_POWER_STATE_CONFIG2, 0x1b),
+
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_RX_SIGDET_LVL, 0xba),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXMGN_V0, 0x9f),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXMGN_V1, 0x9f),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXMGN_V2, 0xb5),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXMGN_V3, 0x4c),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXMGN_V4, 0x64),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXMGN_LS, 0x6a),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M6DB_V0, 0x15),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M3P5DB_V0, 0x0d),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M6DB_V1, 0x15),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M3P5DB_V1, 0x0d),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M6DB_V2, 0x15),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M3P5DB_V2, 0x0d),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M6DB_V3, 0x15),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M3P5DB_V3, 0x1d),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M6DB_V4, 0x15),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M3P5DB_V4, 0x0d),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M6DB_LS, 0x15),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TXDEEMPH_M3P5DB_LS, 0x0d),
+
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_RATE_SLEW_CNTRL, 0x02),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_PWRUP_RESET_DLY_TIME_AUXCLK, 0x04),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_TSYNC_RSYNC_TIME, 0x44),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_PWRUP_RESET_DLY_TIME_AUXCLK, 0x04),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_RCVR_DTCT_DLY_P1U2_L, 0xe7),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_RCVR_DTCT_DLY_P1U2_H, 0x03),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_RCVR_DTCT_DLY_U3_L, 0x40),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_RCVR_DTCT_DLY_U3_H, 0x00),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_RXEQTRAINING_WAIT_TIME, 0x75),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_LFPS_TX_ECSTART_EQTLOCK, 0x86),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_RXEQTRAINING_RUN_TIME, 0x13),
+
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_REFGEN_REQ_CONFIG1, 0x21),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_REFGEN_REQ_CONFIG2, 0x60),
+};
+
+
 /* struct qmp_phy_cfg - per-PHY initialization config */
 struct qmp_phy_cfg {
 	/* phy-type - PCIE/UFS/USB */
@@ -766,6 +878,7 @@ static const struct qmp_phy_cfg qmp_v3_usb3phy_cfg = {
 	.pwrdn_ctrl		= SW_PWRDN,
 	.mask_pcs_ready		= PHYSTATUS,
 
+	.has_pwrdn_delay	= true,
 	.pwrdn_delay_min	= POWER_DOWN_DELAY_US_MIN,
 	.pwrdn_delay_max	= POWER_DOWN_DELAY_US_MAX,
 
@@ -774,6 +887,35 @@ static const struct qmp_phy_cfg qmp_v3_usb3phy_cfg = {
 	.rx_b_lane_offset	= 0x400,
 };
 
+static const struct qmp_phy_cfg qmp_v3_usb3_uniphy_cfg = {
+	.type			= PHY_TYPE_USB3,
+	.nlanes			= 1,
+
+	.serdes_tbl		= qmp_v3_usb3_uniphy_serdes_tbl,
+	.serdes_tbl_num		= ARRAY_SIZE(qmp_v3_usb3_uniphy_serdes_tbl),
+	.tx_tbl			= qmp_v3_usb3_uniphy_tx_tbl,
+	.tx_tbl_num		= ARRAY_SIZE(qmp_v3_usb3_uniphy_tx_tbl),
+	.rx_tbl			= qmp_v3_usb3_uniphy_rx_tbl,
+	.rx_tbl_num		= ARRAY_SIZE(qmp_v3_usb3_uniphy_rx_tbl),
+	.pcs_tbl		= qmp_v3_usb3_uniphy_pcs_tbl,
+	.pcs_tbl_num		= ARRAY_SIZE(qmp_v3_usb3_uniphy_pcs_tbl),
+	.clk_list		= qmp_v3_phy_clk_l,
+	.num_clks		= ARRAY_SIZE(qmp_v3_phy_clk_l),
+	.reset_list		= msm8996_usb3phy_reset_l,
+	.num_resets		= ARRAY_SIZE(msm8996_usb3phy_reset_l),
+	.vreg_list		= msm8996_phy_vreg_l,
+	.num_vregs		= ARRAY_SIZE(msm8996_phy_vreg_l),
+	.regs			= qmp_v3_usb3phy_regs_layout,
+
+	.start_ctrl		= SERDES_START | PCS_START,
+	.pwrdn_ctrl		= SW_PWRDN,
+	.mask_pcs_ready		= PHYSTATUS,
+
+	.has_pwrdn_delay	= true,
+	.pwrdn_delay_min	= POWER_DOWN_DELAY_US_MIN,
+	.pwrdn_delay_max	= POWER_DOWN_DELAY_US_MAX,
+};
+
 static void qcom_qmp_phy_configure(void __iomem *base,
 				   const unsigned int *regs,
 				   const struct qmp_phy_init_tbl tbl[],
@@ -793,19 +935,6 @@ static void qcom_qmp_phy_configure(void __iomem *base,
 	}
 }
 
-static int qcom_qmp_phy_poweron(struct phy *phy)
-{
-	struct qmp_phy *qphy = phy_get_drvdata(phy);
-	struct qcom_qmp *qmp = qphy->qmp;
-	int ret;
-
-	ret = clk_prepare_enable(qphy->pipe_clk);
-	if (ret)
-		dev_err(qmp->dev, "pipe_clk enable failed, err=%d\n", ret);
-
-	return ret;
-}
-
 static int qcom_qmp_phy_com_init(struct qcom_qmp *qmp)
 {
 	const struct qmp_phy_cfg *cfg = qmp->cfg;
@@ -974,6 +1103,12 @@ static int qcom_qmp_phy_init(struct phy *phy)
 		}
 	}
 
+	ret = clk_prepare_enable(qphy->pipe_clk);
+	if (ret) {
+		dev_err(qmp->dev, "pipe_clk enable failed err=%d\n", ret);
+		goto err_clk_enable;
+	}
+
 	/* Tx, Rx, and PCS configurations */
 	qcom_qmp_phy_configure(tx, cfg->regs, cfg->tx_tbl, cfg->tx_tbl_num);
 	/* Configuration for other LANE for USB-DP combo PHY */
@@ -1019,6 +1154,8 @@ static int qcom_qmp_phy_init(struct phy *phy)
 	return ret;
 
 err_pcs_ready:
+	clk_disable_unprepare(qphy->pipe_clk);
+err_clk_enable:
 	if (cfg->has_lane_rst)
 		reset_control_assert(qphy->lane_rst);
 err_lane_rst:
@@ -1283,7 +1420,6 @@ static int phy_pipe_clk_register(struct qcom_qmp *qmp, struct device_node *np)
 static const struct phy_ops qcom_qmp_phy_gen_ops = {
 	.init		= qcom_qmp_phy_init,
 	.exit		= qcom_qmp_phy_exit,
-	.power_on	= qcom_qmp_phy_poweron,
 	.set_mode	= qcom_qmp_phy_set_mode,
 	.owner		= THIS_MODULE,
 };
@@ -1381,8 +1517,11 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = {
 		.compatible = "qcom,ipq8074-qmp-pcie-phy",
 		.data = &ipq8074_pciephy_cfg,
 	}, {
-		.compatible = "qcom,qmp-v3-usb3-phy",
+		.compatible = "qcom,sdm845-qmp-usb3-phy",
 		.data = &qmp_v3_usb3phy_cfg,
+	}, {
+		.compatible = "qcom,sdm845-qmp-usb3-uni-phy",
+		.data = &qmp_v3_usb3_uniphy_cfg,
 	},
 	{ },
 };
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.h b/drivers/phy/qualcomm/phy-qcom-qmp.h
index d1c6905d0439..5d78d43ba9fc 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp.h
+++ b/drivers/phy/qualcomm/phy-qcom-qmp.h
@@ -214,6 +214,8 @@
 #define QSERDES_V3_RX_UCDR_FASTLOCK_FO_GAIN		0x030
 #define QSERDES_V3_RX_UCDR_SO_SATURATION_AND_ENABLE	0x034
 #define QSERDES_V3_RX_RX_TERM_BW			0x07c
+#define QSERDES_V3_RX_VGA_CAL_CNTRL1			0x0bc
+#define QSERDES_V3_RX_VGA_CAL_CNTRL2			0x0c0
 #define QSERDES_V3_RX_RX_EQ_GAIN2_LSB			0x0c8
 #define QSERDES_V3_RX_RX_EQ_GAIN2_MSB			0x0cc
 #define QSERDES_V3_RX_RX_EQU_ADAPTOR_CNTRL2		0x0d4
@@ -227,6 +229,7 @@
 #define QSERDES_V3_RX_SIGDET_DEGLITCH_CNTRL		0x10c
 #define QSERDES_V3_RX_RX_BAND				0x110
 #define QSERDES_V3_RX_RX_INTERFACE_MODE			0x11c
+#define QSERDES_V3_RX_RX_MODE_00			0x164
 
 /* Only for QMP V3 PHY - PCS registers */
 #define QPHY_V3_PCS_POWER_DOWN_CONTROL			0x004
@@ -273,6 +276,8 @@
 #define QPHY_V3_PCS_FLL_CNT_VAL_H_TOL			0x0d0
 #define QPHY_V3_PCS_FLL_MAN_CODE			0x0d4
 #define QPHY_V3_PCS_RX_SIGDET_LVL			0x1d8
+#define QPHY_V3_PCS_REFGEN_REQ_CONFIG1			0x20c
+#define QPHY_V3_PCS_REFGEN_REQ_CONFIG2			0x210
 
 /* Only for QMP V3 PHY - PCS_MISC registers */
 #define QPHY_V3_PCS_MISC_CLAMP_ENABLE			0x0c
diff --git a/drivers/phy/qualcomm/phy-qcom-qusb2.c b/drivers/phy/qualcomm/phy-qcom-qusb2.c
index 94afeac1a19e..e70e425f26f5 100644
--- a/drivers/phy/qualcomm/phy-qcom-qusb2.c
+++ b/drivers/phy/qualcomm/phy-qcom-qusb2.c
@@ -20,6 +20,8 @@
 #include <linux/reset.h>
 #include <linux/slab.h>
 
+#include <dt-bindings/phy/phy-qcom-qusb2.h>
+
 #define QUSB2PHY_PLL_TEST		0x04
 #define CLK_REF_SEL			BIT(7)
 
@@ -60,6 +62,17 @@
 #define CORE_RESET				BIT(5)
 #define CORE_RESET_MUX				BIT(6)
 
+/* QUSB2PHY_IMP_CTRL1 register bits */
+#define IMP_RES_OFFSET_MASK			GENMASK(5, 0)
+#define IMP_RES_OFFSET_SHIFT			0x0
+
+/* QUSB2PHY_PORT_TUNE1 register bits */
+#define HSTX_TRIM_MASK				GENMASK(7, 4)
+#define HSTX_TRIM_SHIFT				0x4
+#define PREEMPH_WIDTH_HALF_BIT			BIT(2)
+#define PREEMPHASIS_EN_MASK			GENMASK(1, 0)
+#define PREEMPHASIS_EN_SHIFT			0x0
+
 #define QUSB2PHY_PLL_ANALOG_CONTROLS_TWO	0x04
 #define QUSB2PHY_PLL_CLOCK_INVERTERS		0x18c
 #define QUSB2PHY_PLL_CMODE			0x2c
@@ -139,7 +152,7 @@ static const struct qusb2_phy_init_tbl msm8996_init_tbl[] = {
 	QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_PWR_CTRL, 0x00),
 };
 
-static const unsigned int qusb2_v2_regs_layout[] = {
+static const unsigned int sdm845_regs_layout[] = {
 	[QUSB2PHY_PLL_CORE_INPUT_OVERRIDE] = 0xa8,
 	[QUSB2PHY_PLL_STATUS]		= 0x1a0,
 	[QUSB2PHY_PORT_TUNE1]		= 0x240,
@@ -153,7 +166,7 @@ static const unsigned int qusb2_v2_regs_layout[] = {
 	[QUSB2PHY_INTR_CTRL]		= 0x230,
 };
 
-static const struct qusb2_phy_init_tbl qusb2_v2_init_tbl[] = {
+static const struct qusb2_phy_init_tbl sdm845_init_tbl[] = {
 	QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_ANALOG_CONTROLS_TWO, 0x03),
 	QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_CLOCK_INVERTERS, 0x7c),
 	QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_CMODE, 0x80),
@@ -208,10 +221,10 @@ static const struct qusb2_phy_cfg msm8996_phy_cfg = {
 	.autoresume_en	 = BIT(3),
 };
 
-static const struct qusb2_phy_cfg qusb2_v2_phy_cfg = {
-	.tbl		= qusb2_v2_init_tbl,
-	.tbl_num	= ARRAY_SIZE(qusb2_v2_init_tbl),
-	.regs		= qusb2_v2_regs_layout,
+static const struct qusb2_phy_cfg sdm845_phy_cfg = {
+	.tbl		= sdm845_init_tbl,
+	.tbl_num	= ARRAY_SIZE(sdm845_init_tbl),
+	.regs		= sdm845_regs_layout,
 
 	.disable_ctrl	= (PWR_CTRL1_VREF_SUPPLY_TRIM | PWR_CTRL1_CLAMP_N_EN |
 			   POWER_DOWN),
@@ -241,6 +254,15 @@ static const char * const qusb2_phy_vreg_names[] = {
  * @tcsr: TCSR syscon register map
  * @cell: nvmem cell containing phy tuning value
  *
+ * @override_imp_res_offset: PHY should use different rescode offset
+ * @imp_res_offset_value: rescode offset to be updated in IMP_CTRL1 register
+ * @override_hstx_trim: PHY should use different HSTX o/p current value
+ * @hstx_trim_value: HSTX_TRIM value to be updated in TUNE1 register
+ * @override_preemphasis: PHY should use different pre-amphasis amplitude
+ * @preemphasis_level: Amplitude Pre-Emphasis to be updated in TUNE1 register
+ * @override_preemphasis_width: PHY should use different pre-emphasis duration
+ * @preemphasis_width: half/full-width Pre-Emphasis updated via TUNE1
+ *
  * @cfg: phy config data
  * @has_se_clk_scheme: indicate if PHY has single-ended ref clock scheme
  * @phy_initialized: indicate if PHY has been initialized
@@ -259,12 +281,35 @@ struct qusb2_phy {
 	struct regmap *tcsr;
 	struct nvmem_cell *cell;
 
+	bool override_imp_res_offset;
+	u8 imp_res_offset_value;
+	bool override_hstx_trim;
+	u8 hstx_trim_value;
+	bool override_preemphasis;
+	u8 preemphasis_level;
+	bool override_preemphasis_width;
+	u8 preemphasis_width;
+
 	const struct qusb2_phy_cfg *cfg;
 	bool has_se_clk_scheme;
 	bool phy_initialized;
 	enum phy_mode mode;
 };
 
+static inline void qusb2_write_mask(void __iomem *base, u32 offset,
+				    u32 val, u32 mask)
+{
+	u32 reg;
+
+	reg = readl(base + offset);
+	reg &= ~mask;
+	reg |= val & mask;
+	writel(reg, base + offset);
+
+	/* Ensure above write is completed */
+	readl(base + offset);
+}
+
 static inline void qusb2_setbits(void __iomem *base, u32 offset, u32 val)
 {
 	u32 reg;
@@ -304,6 +349,42 @@ void qcom_qusb2_phy_configure(void __iomem *base,
 	}
 }
 
+/*
+ * Update board specific PHY tuning override values if specified from
+ * device tree.
+ */
+static void qusb2_phy_override_phy_params(struct qusb2_phy *qphy)
+{
+	const struct qusb2_phy_cfg *cfg = qphy->cfg;
+
+	if (qphy->override_imp_res_offset)
+		qusb2_write_mask(qphy->base, QUSB2PHY_IMP_CTRL1,
+			     qphy->imp_res_offset_value << IMP_RES_OFFSET_SHIFT,
+			     IMP_RES_OFFSET_MASK);
+
+	if (qphy->override_hstx_trim)
+		qusb2_write_mask(qphy->base, cfg->regs[QUSB2PHY_PORT_TUNE1],
+				 qphy->hstx_trim_value << HSTX_TRIM_SHIFT,
+				 HSTX_TRIM_MASK);
+
+	if (qphy->override_preemphasis)
+		qusb2_write_mask(qphy->base, cfg->regs[QUSB2PHY_PORT_TUNE1],
+				qphy->preemphasis_level << PREEMPHASIS_EN_SHIFT,
+				PREEMPHASIS_EN_MASK);
+
+	if (qphy->override_preemphasis_width) {
+		if (qphy->preemphasis_width ==
+		    QUSB2_V2_PREEMPHASIS_WIDTH_HALF_BIT)
+			qusb2_setbits(qphy->base,
+				      cfg->regs[QUSB2PHY_PORT_TUNE1],
+				      PREEMPH_WIDTH_HALF_BIT);
+		else
+			qusb2_clrbits(qphy->base,
+				      cfg->regs[QUSB2PHY_PORT_TUNE1],
+				      PREEMPH_WIDTH_HALF_BIT);
+	}
+}
+
 /*
  * Fetches HS Tx tuning value from nvmem and sets the
  * QUSB2PHY_PORT_TUNE1/2 register.
@@ -315,6 +396,10 @@ static void qusb2_phy_set_tune2_param(struct qusb2_phy *qphy)
 	const struct qusb2_phy_cfg *cfg = qphy->cfg;
 	u8 *val;
 
+	/* efuse register is optional */
+	if (!qphy->cell)
+		return;
+
 	/*
 	 * Read efuse register having TUNE2/1 parameter's high nibble.
 	 * If efuse register shows value as 0x0, or if we fail to find
@@ -521,6 +606,9 @@ static int qusb2_phy_init(struct phy *phy)
 	qcom_qusb2_phy_configure(qphy->base, cfg->regs, cfg->tbl,
 				 cfg->tbl_num);
 
+	/* Override board specific PHY tuning values */
+	qusb2_phy_override_phy_params(qphy);
+
 	/* Set efuse value for tuning the PHY */
 	qusb2_phy_set_tune2_param(qphy);
 
@@ -643,8 +731,8 @@ static const struct of_device_id qusb2_phy_of_match_table[] = {
 		.compatible	= "qcom,msm8996-qusb2-phy",
 		.data		= &msm8996_phy_cfg,
 	}, {
-		.compatible	= "qcom,qusb2-v2-phy",
-		.data		= &qusb2_v2_phy_cfg,
+		.compatible	= "qcom,sdm845-qusb2-phy",
+		.data		= &sdm845_phy_cfg,
 	},
 	{ },
 };
@@ -664,6 +752,7 @@ static int qusb2_phy_probe(struct platform_device *pdev)
 	struct resource *res;
 	int ret, i;
 	int num;
+	u32 value;
 
 	qphy = devm_kzalloc(dev, sizeof(*qphy), GFP_KERNEL);
 	if (!qphy)
@@ -732,6 +821,31 @@ static int qusb2_phy_probe(struct platform_device *pdev)
 		qphy->cell = NULL;
 		dev_dbg(dev, "failed to lookup tune2 hstx trim value\n");
 	}
+
+	if (!of_property_read_u32(dev->of_node, "qcom,imp-res-offset-value",
+				  &value)) {
+		qphy->imp_res_offset_value = (u8)value;
+		qphy->override_imp_res_offset = true;
+	}
+
+	if (!of_property_read_u32(dev->of_node, "qcom,hstx-trim-value",
+				  &value)) {
+		qphy->hstx_trim_value = (u8)value;
+		qphy->override_hstx_trim = true;
+	}
+
+	if (!of_property_read_u32(dev->of_node, "qcom,preemphasis-level",
+				     &value)) {
+		qphy->preemphasis_level = (u8)value;
+		qphy->override_preemphasis = true;
+	}
+
+	if (!of_property_read_u32(dev->of_node, "qcom,preemphasis-width",
+				     &value)) {
+		qphy->preemphasis_width = (u8)value;
+		qphy->override_preemphasis_width = true;
+	}
+
 	pm_runtime_set_active(dev);
 	pm_runtime_enable(dev);
 	/*
diff --git a/drivers/phy/samsung/phy-exynos-mipi-video.c b/drivers/phy/samsung/phy-exynos-mipi-video.c
index c198886f80a3..00d89599c67d 100644
--- a/drivers/phy/samsung/phy-exynos-mipi-video.c
+++ b/drivers/phy/samsung/phy-exynos-mipi-video.c
@@ -231,33 +231,27 @@ struct exynos_mipi_video_phy {
 static int __set_phy_state(const struct exynos_mipi_phy_desc *data,
 			   struct exynos_mipi_video_phy *state, unsigned int on)
 {
-	u32 val;
+	struct regmap *enable_map = state->regmaps[data->enable_map];
+	struct regmap *resetn_map = state->regmaps[data->resetn_map];
 
 	spin_lock(&state->slock);
 
 	/* disable in PMU sysreg */
 	if (!on && data->coupled_phy_id >= 0 &&
-	    state->phys[data->coupled_phy_id].phy->power_count == 0) {
-		regmap_read(state->regmaps[data->enable_map], data->enable_reg,
-			    &val);
-		val &= ~data->enable_val;
-		regmap_write(state->regmaps[data->enable_map], data->enable_reg,
-			     val);
-	}
-
+	    state->phys[data->coupled_phy_id].phy->power_count == 0)
+		regmap_update_bits(enable_map, data->enable_reg,
+				   data->enable_val, 0);
 	/* PHY reset */
-	regmap_read(state->regmaps[data->resetn_map], data->resetn_reg, &val);
-	val = on ? (val | data->resetn_val) : (val & ~data->resetn_val);
-	regmap_write(state->regmaps[data->resetn_map], data->resetn_reg, val);
-
+	if (on)
+		regmap_update_bits(resetn_map, data->resetn_reg,
+				   data->resetn_val, data->resetn_val);
+	else
+		regmap_update_bits(resetn_map, data->resetn_reg,
+				   data->resetn_val, 0);
 	/* enable in PMU sysreg */
-	if (on) {
-		regmap_read(state->regmaps[data->enable_map], data->enable_reg,
-			    &val);
-		val |= data->enable_val;
-		regmap_write(state->regmaps[data->enable_map], data->enable_reg,
-			     val);
-	}
+	if (on)
+		regmap_update_bits(enable_map, data->enable_reg,
+				   data->enable_val, data->enable_val);
 
 	spin_unlock(&state->slock);
 
diff --git a/drivers/phy/st/phy-stm32-usbphyc.c b/drivers/phy/st/phy-stm32-usbphyc.c
index bc4e78a19c91..1255cd1d9a60 100644
--- a/drivers/phy/st/phy-stm32-usbphyc.c
+++ b/drivers/phy/st/phy-stm32-usbphyc.c
@@ -71,7 +71,6 @@ struct stm32_usbphyc {
 	struct stm32_usbphyc_phy **phys;
 	int nphys;
 	int switch_setup;
-	bool pll_enabled;
 };
 
 static inline void stm32_usbphyc_set_bits(void __iomem *reg, u32 bits)
@@ -84,7 +83,8 @@ static inline void stm32_usbphyc_clr_bits(void __iomem *reg, u32 bits)
 	writel_relaxed(readl_relaxed(reg) & ~bits, reg);
 }
 
-static void stm32_usbphyc_get_pll_params(u32 clk_rate, struct pll_params *pll_params)
+static void stm32_usbphyc_get_pll_params(u32 clk_rate,
+					 struct pll_params *pll_params)
 {
 	unsigned long long fvco, ndiv, frac;
 
@@ -271,7 +271,6 @@ static struct phy *stm32_usbphyc_of_xlate(struct device *dev,
 	struct stm32_usbphyc *usbphyc = dev_get_drvdata(dev);
 	struct stm32_usbphyc_phy *usbphyc_phy = NULL;
 	struct device_node *phynode = args->np;
-
 	int port = 0;
 
 	for (port = 0; port < usbphyc->nphys; port++) {
@@ -367,8 +366,8 @@ static int stm32_usbphyc_probe(struct platform_device *pdev)
 		if (IS_ERR(phy)) {
 			ret = PTR_ERR(phy);
 			if (ret != -EPROBE_DEFER)
-				dev_err(dev,
-					"failed to create phy%d: %d\n", i, ret);
+				dev_err(dev, "failed to create phy%d: %d\n",
+					port, ret);
 			goto put_child;
 		}
 
diff --git a/drivers/phy/tegra/xusb.c b/drivers/phy/tegra/xusb.c
index 11aa5902a9ac..de1b4ebe4de2 100644
--- a/drivers/phy/tegra/xusb.c
+++ b/drivers/phy/tegra/xusb.c
@@ -102,19 +102,6 @@ tegra_xusb_pad_find_phy_node(struct tegra_xusb_pad *pad, unsigned int index)
 	return np;
 }
 
-static int
-tegra_xusb_lane_lookup_function(struct tegra_xusb_lane *lane,
-				    const char *function)
-{
-	unsigned int i;
-
-	for (i = 0; i < lane->soc->num_funcs; i++)
-		if (strcmp(function, lane->soc->funcs[i]) == 0)
-			return i;
-
-	return -EINVAL;
-}
-
 int tegra_xusb_lane_parse_dt(struct tegra_xusb_lane *lane,
 			     struct device_node *np)
 {
@@ -126,7 +113,7 @@ int tegra_xusb_lane_parse_dt(struct tegra_xusb_lane *lane,
 	if (err < 0)
 		return err;
 
-	err = tegra_xusb_lane_lookup_function(lane, function);
+	err = match_string(lane->soc->funcs, lane->soc->num_funcs, function);
 	if (err < 0) {
 		dev_err(dev, "invalid function \"%s\" for lane \"%s\"\n",
 			function, np->name);
diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig
index 01fe8e0455a0..dd50371225bc 100644
--- a/drivers/pinctrl/Kconfig
+++ b/drivers/pinctrl/Kconfig
@@ -337,6 +337,7 @@ config PINCTRL_OCELOT
 	select GENERIC_PINMUX_FUNCTIONS
 	select REGMAP_MMIO
 
+source "drivers/pinctrl/actions/Kconfig"
 source "drivers/pinctrl/aspeed/Kconfig"
 source "drivers/pinctrl/bcm/Kconfig"
 source "drivers/pinctrl/berlin/Kconfig"
diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile
index 657332b121fb..de40863e7297 100644
--- a/drivers/pinctrl/Makefile
+++ b/drivers/pinctrl/Makefile
@@ -43,6 +43,7 @@ obj-$(CONFIG_PINCTRL_INGENIC)	+= pinctrl-ingenic.o
 obj-$(CONFIG_PINCTRL_RK805)	+= pinctrl-rk805.o
 obj-$(CONFIG_PINCTRL_OCELOT)	+= pinctrl-ocelot.o
 
+obj-y				+= actions/
 obj-$(CONFIG_ARCH_ASPEED)	+= aspeed/
 obj-y				+= bcm/
 obj-$(CONFIG_PINCTRL_BERLIN)	+= berlin/
diff --git a/drivers/pinctrl/actions/Kconfig b/drivers/pinctrl/actions/Kconfig
new file mode 100644
index 000000000000..490927b4ea76
--- /dev/null
+++ b/drivers/pinctrl/actions/Kconfig
@@ -0,0 +1,15 @@
+config PINCTRL_OWL
+	bool "Actions Semi OWL pinctrl driver"
+	depends on (ARCH_ACTIONS || COMPILE_TEST) && OF
+	select PINMUX
+	select PINCONF
+	select GENERIC_PINCONF
+	select GPIOLIB
+	help
+	  Say Y here to enable Actions Semi OWL pinctrl driver
+
+config PINCTRL_S900
+	bool "Actions Semi S900 pinctrl driver"
+	depends on PINCTRL_OWL
+	help
+	  Say Y here to enable Actions Semi S900 pinctrl driver
diff --git a/drivers/pinctrl/actions/Makefile b/drivers/pinctrl/actions/Makefile
new file mode 100644
index 000000000000..bd232d28400f
--- /dev/null
+++ b/drivers/pinctrl/actions/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_PINCTRL_OWL)	+= pinctrl-owl.o
+obj-$(CONFIG_PINCTRL_S900) 	+= pinctrl-s900.o
diff --git a/drivers/pinctrl/actions/pinctrl-owl.c b/drivers/pinctrl/actions/pinctrl-owl.c
new file mode 100644
index 000000000000..76243caa08c6
--- /dev/null
+++ b/drivers/pinctrl/actions/pinctrl-owl.c
@@ -0,0 +1,785 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * OWL SoC's Pinctrl driver
+ *
+ * Copyright (c) 2014 Actions Semi Inc.
+ * Author: David Liu <liuwei@actions-semi.com>
+ *
+ * Copyright (c) 2018 Linaro Ltd.
+ * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+ */
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/gpio/driver.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/machine.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include "../core.h"
+#include "../pinctrl-utils.h"
+#include "pinctrl-owl.h"
+
+/**
+ * struct owl_pinctrl - pinctrl state of the device
+ * @dev: device handle
+ * @pctrldev: pinctrl handle
+ * @chip: gpio chip
+ * @lock: spinlock to protect registers
+ * @soc: reference to soc_data
+ * @base: pinctrl register base address
+ */
+struct owl_pinctrl {
+	struct device *dev;
+	struct pinctrl_dev *pctrldev;
+	struct gpio_chip chip;
+	raw_spinlock_t lock;
+	struct clk *clk;
+	const struct owl_pinctrl_soc_data *soc;
+	void __iomem *base;
+};
+
+static void owl_update_bits(void __iomem *base, u32 mask, u32 val)
+{
+	u32 reg_val;
+
+	reg_val = readl_relaxed(base);
+
+	reg_val = (reg_val & ~mask) | (val & mask);
+
+	writel_relaxed(reg_val, base);
+}
+
+static u32 owl_read_field(struct owl_pinctrl *pctrl, u32 reg,
+				u32 bit, u32 width)
+{
+	u32 tmp, mask;
+
+	tmp = readl_relaxed(pctrl->base + reg);
+	mask = (1 << width) - 1;
+
+	return (tmp >> bit) & mask;
+}
+
+static void owl_write_field(struct owl_pinctrl *pctrl, u32 reg, u32 arg,
+				u32 bit, u32 width)
+{
+	u32 mask;
+
+	mask = (1 << width) - 1;
+	mask = mask << bit;
+
+	owl_update_bits(pctrl->base + reg, mask, (arg << bit));
+}
+
+static int owl_get_groups_count(struct pinctrl_dev *pctrldev)
+{
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+
+	return pctrl->soc->ngroups;
+}
+
+static const char *owl_get_group_name(struct pinctrl_dev *pctrldev,
+				unsigned int group)
+{
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+
+	return pctrl->soc->groups[group].name;
+}
+
+static int owl_get_group_pins(struct pinctrl_dev *pctrldev,
+				unsigned int group,
+				const unsigned int **pins,
+				unsigned int *num_pins)
+{
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+
+	*pins = pctrl->soc->groups[group].pads;
+	*num_pins = pctrl->soc->groups[group].npads;
+
+	return 0;
+}
+
+static void owl_pin_dbg_show(struct pinctrl_dev *pctrldev,
+				struct seq_file *s,
+				unsigned int offset)
+{
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+
+	seq_printf(s, "%s", dev_name(pctrl->dev));
+}
+
+static struct pinctrl_ops owl_pinctrl_ops = {
+	.get_groups_count = owl_get_groups_count,
+	.get_group_name = owl_get_group_name,
+	.get_group_pins = owl_get_group_pins,
+	.pin_dbg_show = owl_pin_dbg_show,
+	.dt_node_to_map = pinconf_generic_dt_node_to_map_all,
+	.dt_free_map = pinctrl_utils_free_map,
+};
+
+static int owl_get_funcs_count(struct pinctrl_dev *pctrldev)
+{
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+
+	return pctrl->soc->nfunctions;
+}
+
+static const char *owl_get_func_name(struct pinctrl_dev *pctrldev,
+				unsigned int function)
+{
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+
+	return pctrl->soc->functions[function].name;
+}
+
+static int owl_get_func_groups(struct pinctrl_dev *pctrldev,
+				unsigned int function,
+				const char * const **groups,
+				unsigned int * const num_groups)
+{
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+
+	*groups = pctrl->soc->functions[function].groups;
+	*num_groups = pctrl->soc->functions[function].ngroups;
+
+	return 0;
+}
+
+static inline int get_group_mfp_mask_val(const struct owl_pingroup *g,
+				int function,
+				u32 *mask,
+				u32 *val)
+{
+	int id;
+	u32 option_num;
+	u32 option_mask;
+
+	for (id = 0; id < g->nfuncs; id++) {
+		if (g->funcs[id] == function)
+			break;
+	}
+	if (WARN_ON(id == g->nfuncs))
+		return -EINVAL;
+
+	option_num = (1 << g->mfpctl_width);
+	if (id > option_num)
+		id -= option_num;
+
+	option_mask = option_num - 1;
+	*mask = (option_mask  << g->mfpctl_shift);
+	*val = (id << g->mfpctl_shift);
+
+	return 0;
+}
+
+static int owl_set_mux(struct pinctrl_dev *pctrldev,
+				unsigned int function,
+				unsigned int group)
+{
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+	const struct owl_pingroup *g;
+	unsigned long flags;
+	u32 val, mask;
+
+	g = &pctrl->soc->groups[group];
+
+	if (get_group_mfp_mask_val(g, function, &mask, &val))
+		return -EINVAL;
+
+	raw_spin_lock_irqsave(&pctrl->lock, flags);
+
+	owl_update_bits(pctrl->base + g->mfpctl_reg, mask, val);
+
+	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+
+	return 0;
+}
+
+static struct pinmux_ops owl_pinmux_ops = {
+	.get_functions_count = owl_get_funcs_count,
+	.get_function_name = owl_get_func_name,
+	.get_function_groups = owl_get_func_groups,
+	.set_mux = owl_set_mux,
+};
+
+static int owl_pad_pinconf_reg(const struct owl_padinfo *info,
+				unsigned int param,
+				u32 *reg,
+				u32 *bit,
+				u32 *width)
+{
+	switch (param) {
+	case PIN_CONFIG_BIAS_BUS_HOLD:
+	case PIN_CONFIG_BIAS_HIGH_IMPEDANCE:
+	case PIN_CONFIG_BIAS_PULL_DOWN:
+	case PIN_CONFIG_BIAS_PULL_UP:
+		if (!info->pullctl)
+			return -EINVAL;
+		*reg = info->pullctl->reg;
+		*bit = info->pullctl->shift;
+		*width = info->pullctl->width;
+		break;
+	case PIN_CONFIG_INPUT_SCHMITT_ENABLE:
+		if (!info->st)
+			return -EINVAL;
+		*reg = info->st->reg;
+		*bit = info->st->shift;
+		*width = info->st->width;
+		break;
+	default:
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static int owl_pad_pinconf_arg2val(const struct owl_padinfo *info,
+				unsigned int param,
+				u32 *arg)
+{
+	switch (param) {
+	case PIN_CONFIG_BIAS_BUS_HOLD:
+		*arg = OWL_PINCONF_PULL_HOLD;
+		break;
+	case PIN_CONFIG_BIAS_HIGH_IMPEDANCE:
+		*arg = OWL_PINCONF_PULL_HIZ;
+		break;
+	case PIN_CONFIG_BIAS_PULL_DOWN:
+		*arg = OWL_PINCONF_PULL_DOWN;
+		break;
+	case PIN_CONFIG_BIAS_PULL_UP:
+		*arg = OWL_PINCONF_PULL_UP;
+		break;
+	case PIN_CONFIG_INPUT_SCHMITT_ENABLE:
+		*arg = (*arg >= 1 ? 1 : 0);
+		break;
+	default:
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static int owl_pad_pinconf_val2arg(const struct owl_padinfo *padinfo,
+				unsigned int param,
+				u32 *arg)
+{
+	switch (param) {
+	case PIN_CONFIG_BIAS_BUS_HOLD:
+		*arg = *arg == OWL_PINCONF_PULL_HOLD;
+		break;
+	case PIN_CONFIG_BIAS_HIGH_IMPEDANCE:
+		*arg = *arg == OWL_PINCONF_PULL_HIZ;
+		break;
+	case PIN_CONFIG_BIAS_PULL_DOWN:
+		*arg = *arg == OWL_PINCONF_PULL_DOWN;
+		break;
+	case PIN_CONFIG_BIAS_PULL_UP:
+		*arg = *arg == OWL_PINCONF_PULL_UP;
+		break;
+	case PIN_CONFIG_INPUT_SCHMITT_ENABLE:
+		*arg = *arg == 1;
+		break;
+	default:
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static int owl_pin_config_get(struct pinctrl_dev *pctrldev,
+				unsigned int pin,
+				unsigned long *config)
+{
+	int ret = 0;
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+	const struct owl_padinfo *info;
+	unsigned int param = pinconf_to_config_param(*config);
+	u32 reg, bit, width, arg;
+
+	info = &pctrl->soc->padinfo[pin];
+
+	ret = owl_pad_pinconf_reg(info, param, &reg, &bit, &width);
+	if (ret)
+		return ret;
+
+	arg = owl_read_field(pctrl, reg, bit, width);
+
+	ret = owl_pad_pinconf_val2arg(info, param, &arg);
+	if (ret)
+		return ret;
+
+	*config = pinconf_to_config_packed(param, arg);
+
+	return ret;
+}
+
+static int owl_pin_config_set(struct pinctrl_dev *pctrldev,
+				unsigned int pin,
+				unsigned long *configs,
+				unsigned int num_configs)
+{
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+	const struct owl_padinfo *info;
+	unsigned long flags;
+	unsigned int param;
+	u32 reg, bit, width, arg;
+	int ret, i;
+
+	info = &pctrl->soc->padinfo[pin];
+
+	for (i = 0; i < num_configs; i++) {
+		param = pinconf_to_config_param(configs[i]);
+		arg = pinconf_to_config_argument(configs[i]);
+
+		ret = owl_pad_pinconf_reg(info, param, &reg, &bit, &width);
+		if (ret)
+			return ret;
+
+		ret = owl_pad_pinconf_arg2val(info, param, &arg);
+		if (ret)
+			return ret;
+
+		raw_spin_lock_irqsave(&pctrl->lock, flags);
+
+		owl_write_field(pctrl, reg, arg, bit, width);
+
+		raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+	}
+
+	return ret;
+}
+
+static int owl_group_pinconf_reg(const struct owl_pingroup *g,
+				unsigned int param,
+				u32 *reg,
+				u32 *bit,
+				u32 *width)
+{
+	switch (param) {
+	case PIN_CONFIG_DRIVE_STRENGTH:
+		if (g->drv_reg < 0)
+			return -EINVAL;
+		*reg = g->drv_reg;
+		*bit = g->drv_shift;
+		*width = g->drv_width;
+		break;
+	case PIN_CONFIG_SLEW_RATE:
+		if (g->sr_reg < 0)
+			return -EINVAL;
+		*reg = g->sr_reg;
+		*bit = g->sr_shift;
+		*width = g->sr_width;
+		break;
+	default:
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static int owl_group_pinconf_arg2val(const struct owl_pingroup *g,
+				unsigned int param,
+				u32 *arg)
+{
+	switch (param) {
+	case PIN_CONFIG_DRIVE_STRENGTH:
+		switch (*arg) {
+		case 2:
+			*arg = OWL_PINCONF_DRV_2MA;
+			break;
+		case 4:
+			*arg = OWL_PINCONF_DRV_4MA;
+			break;
+		case 8:
+			*arg = OWL_PINCONF_DRV_8MA;
+			break;
+		case 12:
+			*arg = OWL_PINCONF_DRV_12MA;
+			break;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case PIN_CONFIG_SLEW_RATE:
+		if (*arg)
+			*arg = OWL_PINCONF_SLEW_FAST;
+		else
+			*arg = OWL_PINCONF_SLEW_SLOW;
+		break;
+	default:
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static int owl_group_pinconf_val2arg(const struct owl_pingroup *g,
+				unsigned int param,
+				u32 *arg)
+{
+	switch (param) {
+	case PIN_CONFIG_DRIVE_STRENGTH:
+		switch (*arg) {
+		case OWL_PINCONF_DRV_2MA:
+			*arg = 2;
+			break;
+		case OWL_PINCONF_DRV_4MA:
+			*arg = 4;
+			break;
+		case OWL_PINCONF_DRV_8MA:
+			*arg = 8;
+			break;
+		case OWL_PINCONF_DRV_12MA:
+			*arg = 12;
+			break;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case PIN_CONFIG_SLEW_RATE:
+		if (*arg)
+			*arg = 1;
+		else
+			*arg = 0;
+		break;
+	default:
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static int owl_group_config_get(struct pinctrl_dev *pctrldev,
+				unsigned int group,
+				unsigned long *config)
+{
+	const struct owl_pingroup *g;
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+	unsigned int param = pinconf_to_config_param(*config);
+	u32 reg, bit, width, arg;
+	int ret;
+
+	g = &pctrl->soc->groups[group];
+
+	ret = owl_group_pinconf_reg(g, param, &reg, &bit, &width);
+	if (ret)
+		return ret;
+
+	arg = owl_read_field(pctrl, reg, bit, width);
+
+	ret = owl_group_pinconf_val2arg(g, param, &arg);
+	if (ret)
+		return ret;
+
+	*config = pinconf_to_config_packed(param, arg);
+
+	return ret;
+
+}
+
+static int owl_group_config_set(struct pinctrl_dev *pctrldev,
+				unsigned int group,
+				unsigned long *configs,
+				unsigned int num_configs)
+{
+	const struct owl_pingroup *g;
+	struct owl_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctrldev);
+	unsigned long flags;
+	unsigned int param;
+	u32 reg, bit, width, arg;
+	int ret, i;
+
+	g = &pctrl->soc->groups[group];
+
+	for (i = 0; i < num_configs; i++) {
+		param = pinconf_to_config_param(configs[i]);
+		arg = pinconf_to_config_argument(configs[i]);
+
+		ret = owl_group_pinconf_reg(g, param, &reg, &bit, &width);
+		if (ret)
+			return ret;
+
+		ret = owl_group_pinconf_arg2val(g, param, &arg);
+		if (ret)
+			return ret;
+
+		/* Update register */
+		raw_spin_lock_irqsave(&pctrl->lock, flags);
+
+		owl_write_field(pctrl, reg, arg, bit, width);
+
+		raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+	}
+
+	return 0;
+}
+
+static const struct pinconf_ops owl_pinconf_ops = {
+	.is_generic = true,
+	.pin_config_get = owl_pin_config_get,
+	.pin_config_set = owl_pin_config_set,
+	.pin_config_group_get = owl_group_config_get,
+	.pin_config_group_set = owl_group_config_set,
+};
+
+static struct pinctrl_desc owl_pinctrl_desc = {
+	.pctlops = &owl_pinctrl_ops,
+	.pmxops = &owl_pinmux_ops,
+	.confops = &owl_pinconf_ops,
+	.owner = THIS_MODULE,
+};
+
+static const struct owl_gpio_port *
+owl_gpio_get_port(struct owl_pinctrl *pctrl, unsigned int *pin)
+{
+	unsigned int start = 0, i;
+
+	for (i = 0; i < pctrl->soc->nports; i++) {
+		const struct owl_gpio_port *port = &pctrl->soc->ports[i];
+
+		if (*pin >= start && *pin < start + port->pins) {
+			*pin -= start;
+			return port;
+		}
+
+		start += port->pins;
+	}
+
+	return NULL;
+}
+
+static void owl_gpio_update_reg(void __iomem *base, unsigned int pin, int flag)
+{
+	u32 val;
+
+	val = readl_relaxed(base);
+
+	if (flag)
+		val |= BIT(pin);
+	else
+		val &= ~BIT(pin);
+
+	writel_relaxed(val, base);
+}
+
+static int owl_gpio_request(struct gpio_chip *chip, unsigned int offset)
+{
+	struct owl_pinctrl *pctrl = gpiochip_get_data(chip);
+	const struct owl_gpio_port *port;
+	void __iomem *gpio_base;
+	unsigned long flags;
+
+	port = owl_gpio_get_port(pctrl, &offset);
+	if (WARN_ON(port == NULL))
+		return -ENODEV;
+
+	gpio_base = pctrl->base + port->offset;
+
+	/*
+	 * GPIOs have higher priority over other modules, so either setting
+	 * them as OUT or IN is sufficient
+	 */
+	raw_spin_lock_irqsave(&pctrl->lock, flags);
+	owl_gpio_update_reg(gpio_base + port->outen, offset, true);
+	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+
+	return 0;
+}
+
+static void owl_gpio_free(struct gpio_chip *chip, unsigned int offset)
+{
+	struct owl_pinctrl *pctrl = gpiochip_get_data(chip);
+	const struct owl_gpio_port *port;
+	void __iomem *gpio_base;
+	unsigned long flags;
+
+	port = owl_gpio_get_port(pctrl, &offset);
+	if (WARN_ON(port == NULL))
+		return;
+
+	gpio_base = pctrl->base + port->offset;
+
+	raw_spin_lock_irqsave(&pctrl->lock, flags);
+	/* disable gpio output */
+	owl_gpio_update_reg(gpio_base + port->outen, offset, false);
+
+	/* disable gpio input */
+	owl_gpio_update_reg(gpio_base + port->inen, offset, false);
+	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+}
+
+static int owl_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+	struct owl_pinctrl *pctrl = gpiochip_get_data(chip);
+	const struct owl_gpio_port *port;
+	void __iomem *gpio_base;
+	unsigned long flags;
+	u32 val;
+
+	port = owl_gpio_get_port(pctrl, &offset);
+	if (WARN_ON(port == NULL))
+		return -ENODEV;
+
+	gpio_base = pctrl->base + port->offset;
+
+	raw_spin_lock_irqsave(&pctrl->lock, flags);
+	val = readl_relaxed(gpio_base + port->dat);
+	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+
+	return !!(val & BIT(offset));
+}
+
+static void owl_gpio_set(struct gpio_chip *chip, unsigned int offset, int value)
+{
+	struct owl_pinctrl *pctrl = gpiochip_get_data(chip);
+	const struct owl_gpio_port *port;
+	void __iomem *gpio_base;
+	unsigned long flags;
+
+	port = owl_gpio_get_port(pctrl, &offset);
+	if (WARN_ON(port == NULL))
+		return;
+
+	gpio_base = pctrl->base + port->offset;
+
+	raw_spin_lock_irqsave(&pctrl->lock, flags);
+	owl_gpio_update_reg(gpio_base + port->dat, offset, value);
+	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+}
+
+static int owl_gpio_direction_input(struct gpio_chip *chip, unsigned int offset)
+{
+	struct owl_pinctrl *pctrl = gpiochip_get_data(chip);
+	const struct owl_gpio_port *port;
+	void __iomem *gpio_base;
+	unsigned long flags;
+
+	port = owl_gpio_get_port(pctrl, &offset);
+	if (WARN_ON(port == NULL))
+		return -ENODEV;
+
+	gpio_base = pctrl->base + port->offset;
+
+	raw_spin_lock_irqsave(&pctrl->lock, flags);
+	owl_gpio_update_reg(gpio_base + port->outen, offset, false);
+	owl_gpio_update_reg(gpio_base + port->inen, offset, true);
+	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+
+	return 0;
+}
+
+static int owl_gpio_direction_output(struct gpio_chip *chip,
+				unsigned int offset, int value)
+{
+	struct owl_pinctrl *pctrl = gpiochip_get_data(chip);
+	const struct owl_gpio_port *port;
+	void __iomem *gpio_base;
+	unsigned long flags;
+
+	port = owl_gpio_get_port(pctrl, &offset);
+	if (WARN_ON(port == NULL))
+		return -ENODEV;
+
+	gpio_base = pctrl->base + port->offset;
+
+	raw_spin_lock_irqsave(&pctrl->lock, flags);
+	owl_gpio_update_reg(gpio_base + port->inen, offset, false);
+	owl_gpio_update_reg(gpio_base + port->outen, offset, true);
+	owl_gpio_update_reg(gpio_base + port->dat, offset, value);
+	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+
+	return 0;
+}
+
+static int owl_gpio_init(struct owl_pinctrl *pctrl)
+{
+	struct gpio_chip *chip;
+	int ret;
+
+	chip = &pctrl->chip;
+	chip->base = -1;
+	chip->ngpio = pctrl->soc->ngpios;
+	chip->label = dev_name(pctrl->dev);
+	chip->parent = pctrl->dev;
+	chip->owner = THIS_MODULE;
+	chip->of_node = pctrl->dev->of_node;
+
+	ret = gpiochip_add_data(&pctrl->chip, pctrl);
+	if (ret) {
+		dev_err(pctrl->dev, "failed to register gpiochip\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+int owl_pinctrl_probe(struct platform_device *pdev,
+				struct owl_pinctrl_soc_data *soc_data)
+{
+	struct resource *res;
+	struct owl_pinctrl *pctrl;
+	int ret;
+
+	pctrl = devm_kzalloc(&pdev->dev, sizeof(*pctrl), GFP_KERNEL);
+	if (!pctrl)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	pctrl->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(pctrl->base))
+		return PTR_ERR(pctrl->base);
+
+	/* enable GPIO/MFP clock */
+	pctrl->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(pctrl->clk)) {
+		dev_err(&pdev->dev, "no clock defined\n");
+		return PTR_ERR(pctrl->clk);
+	}
+
+	ret = clk_prepare_enable(pctrl->clk);
+	if (ret) {
+		dev_err(&pdev->dev, "clk enable failed\n");
+		return ret;
+	}
+
+	raw_spin_lock_init(&pctrl->lock);
+
+	owl_pinctrl_desc.name = dev_name(&pdev->dev);
+	owl_pinctrl_desc.pins = soc_data->pins;
+	owl_pinctrl_desc.npins = soc_data->npins;
+
+	pctrl->chip.direction_input  = owl_gpio_direction_input;
+	pctrl->chip.direction_output = owl_gpio_direction_output;
+	pctrl->chip.get = owl_gpio_get;
+	pctrl->chip.set = owl_gpio_set;
+	pctrl->chip.request = owl_gpio_request;
+	pctrl->chip.free = owl_gpio_free;
+
+	pctrl->soc = soc_data;
+	pctrl->dev = &pdev->dev;
+
+	pctrl->pctrldev = devm_pinctrl_register(&pdev->dev,
+					&owl_pinctrl_desc, pctrl);
+	if (IS_ERR(pctrl->pctrldev)) {
+		dev_err(&pdev->dev, "could not register Actions OWL pinmux driver\n");
+		return PTR_ERR(pctrl->pctrldev);
+	}
+
+	ret = owl_gpio_init(pctrl);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, pctrl);
+
+	return 0;
+}
diff --git a/drivers/pinctrl/actions/pinctrl-owl.h b/drivers/pinctrl/actions/pinctrl-owl.h
new file mode 100644
index 000000000000..74342378937c
--- /dev/null
+++ b/drivers/pinctrl/actions/pinctrl-owl.h
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * OWL SoC's Pinctrl definitions
+ *
+ * Copyright (c) 2014 Actions Semi Inc.
+ * Author: David Liu <liuwei@actions-semi.com>
+ *
+ * Copyright (c) 2018 Linaro Ltd.
+ * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+ */
+
+#ifndef __PINCTRL_OWL_H__
+#define __PINCTRL_OWL_H__
+
+#define OWL_PINCONF_SLEW_SLOW 0
+#define OWL_PINCONF_SLEW_FAST 1
+
+enum owl_pinconf_pull {
+	OWL_PINCONF_PULL_HIZ,
+	OWL_PINCONF_PULL_DOWN,
+	OWL_PINCONF_PULL_UP,
+	OWL_PINCONF_PULL_HOLD,
+};
+
+enum owl_pinconf_drv {
+	OWL_PINCONF_DRV_2MA,
+	OWL_PINCONF_DRV_4MA,
+	OWL_PINCONF_DRV_8MA,
+	OWL_PINCONF_DRV_12MA,
+};
+
+/**
+ * struct owl_pullctl - Actions pad pull control register
+ * @reg: offset to the pull control register
+ * @shift: shift value of the register
+ * @width: width of the register
+ */
+struct owl_pullctl {
+	int reg;
+	unsigned int shift;
+	unsigned int width;
+};
+
+/**
+ * struct owl_st - Actions pad schmitt trigger enable register
+ * @reg: offset to the schmitt trigger enable register
+ * @shift: shift value of the register
+ * @width: width of the register
+ */
+struct owl_st {
+	int reg;
+	unsigned int shift;
+	unsigned int width;
+};
+
+/**
+ * struct owl_pingroup - Actions pingroup definition
+ * @name: name of the  pin group
+ * @pads: list of pins assigned to this pingroup
+ * @npads: size of @pads array
+ * @funcs: list of pinmux functions for this pingroup
+ * @nfuncs: size of @funcs array
+ * @mfpctl_reg: multiplexing control register offset
+ * @mfpctl_shift: multiplexing control register bit mask
+ * @mfpctl_width: multiplexing control register width
+ * @drv_reg: drive control register offset
+ * @drv_shift: drive control register bit mask
+ * @drv_width: driver control register width
+ * @sr_reg: slew rate control register offset
+ * @sr_shift: slew rate control register bit mask
+ * @sr_width: slew rate control register width
+ */
+struct owl_pingroup {
+	const char *name;
+	unsigned int *pads;
+	unsigned int npads;
+	unsigned int *funcs;
+	unsigned int nfuncs;
+
+	int mfpctl_reg;
+	unsigned int mfpctl_shift;
+	unsigned int mfpctl_width;
+
+	int drv_reg;
+	unsigned int drv_shift;
+	unsigned int drv_width;
+
+	int sr_reg;
+	unsigned int sr_shift;
+	unsigned int sr_width;
+};
+
+/**
+ * struct owl_padinfo - Actions pinctrl pad info
+ * @pad: pad name of the SoC
+ * @pullctl: pull control register info
+ * @st: schmitt trigger register info
+ */
+struct owl_padinfo {
+	int pad;
+	struct owl_pullctl *pullctl;
+	struct owl_st *st;
+};
+
+/**
+ * struct owl_pinmux_func - Actions pinctrl mux functions
+ * @name: name of the pinmux function.
+ * @groups: array of pin groups that may select this function.
+ * @ngroups: number of entries in @groups.
+ */
+struct owl_pinmux_func {
+	const char *name;
+	const char * const *groups;
+	unsigned int ngroups;
+};
+
+/**
+ * struct owl_gpio_port - Actions GPIO port info
+ * @offset: offset of the GPIO port.
+ * @pins: number of pins belongs to the GPIO port.
+ * @outen: offset of the output enable register.
+ * @inen: offset of the input enable register.
+ * @dat: offset of the data register.
+ */
+struct owl_gpio_port {
+	unsigned int offset;
+	unsigned int pins;
+	unsigned int outen;
+	unsigned int inen;
+	unsigned int dat;
+};
+
+/**
+ * struct owl_pinctrl_soc_data - Actions pin controller driver configuration
+ * @pins: array describing all pins of the pin controller.
+ * @npins: number of entries in @pins.
+ * @functions: array describing all mux functions of this SoC.
+ * @nfunction: number of entries in @functions.
+ * @groups: array describing all pin groups of this SoC.
+ * @ngroups: number of entries in @groups.
+ * @padinfo: array describing the pad info of this SoC.
+ * @ngpios: number of pingroups the driver should expose as GPIOs.
+ * @port: array describing all GPIO ports of this SoC.
+ * @nports: number of GPIO ports in this SoC.
+ */
+struct owl_pinctrl_soc_data {
+	const struct pinctrl_pin_desc *pins;
+	unsigned int npins;
+	const struct owl_pinmux_func *functions;
+	unsigned int nfunctions;
+	const struct owl_pingroup *groups;
+	unsigned int ngroups;
+	const struct owl_padinfo *padinfo;
+	unsigned int ngpios;
+	const struct owl_gpio_port *ports;
+	unsigned int nports;
+};
+
+int owl_pinctrl_probe(struct platform_device *pdev,
+		struct owl_pinctrl_soc_data *soc_data);
+
+#endif /* __PINCTRL_OWL_H__ */
diff --git a/drivers/pinctrl/actions/pinctrl-s900.c b/drivers/pinctrl/actions/pinctrl-s900.c
new file mode 100644
index 000000000000..5503c7945764
--- /dev/null
+++ b/drivers/pinctrl/actions/pinctrl-s900.c
@@ -0,0 +1,1888 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * OWL S900 Pinctrl driver
+ *
+ * Copyright (c) 2014 Actions Semi Inc.
+ * Author: David Liu <liuwei@actions-semi.com>
+ *
+ * Copyright (c) 2018 Linaro Ltd.
+ * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/pinctrl.h>
+#include "pinctrl-owl.h"
+
+/* Pinctrl registers offset */
+#define MFCTL0			(0x0040)
+#define MFCTL1			(0x0044)
+#define MFCTL2			(0x0048)
+#define MFCTL3			(0x004C)
+#define PAD_PULLCTL0		(0x0060)
+#define PAD_PULLCTL1		(0x0064)
+#define PAD_PULLCTL2		(0x0068)
+#define PAD_ST0			(0x006C)
+#define PAD_ST1			(0x0070)
+#define PAD_CTL			(0x0074)
+#define PAD_DRV0		(0x0080)
+#define PAD_DRV1		(0x0084)
+#define PAD_DRV2		(0x0088)
+#define PAD_SR0			(0x0270)
+#define PAD_SR1			(0x0274)
+#define PAD_SR2			(0x0278)
+
+#define OWL_GPIO_PORT_A		0
+#define OWL_GPIO_PORT_B		1
+#define OWL_GPIO_PORT_C		2
+#define OWL_GPIO_PORT_D		3
+#define OWL_GPIO_PORT_E		4
+#define OWL_GPIO_PORT_F		5
+
+#define _GPIOA(offset)		(offset)
+#define _GPIOB(offset)		(32 + (offset))
+#define _GPIOC(offset)		(64 + (offset))
+#define _GPIOD(offset)		(76 + (offset))
+#define _GPIOE(offset)		(106 + (offset))
+#define _GPIOF(offset)		(138 + (offset))
+
+#define NUM_GPIOS		(_GPIOF(7) + 1)
+#define _PIN(offset)		(NUM_GPIOS + (offset))
+
+#define ETH_TXD0		_GPIOA(0)
+#define ETH_TXD1		_GPIOA(1)
+#define ETH_TXEN		_GPIOA(2)
+#define ETH_RXER		_GPIOA(3)
+#define ETH_CRS_DV		_GPIOA(4)
+#define ETH_RXD1		_GPIOA(5)
+#define ETH_RXD0		_GPIOA(6)
+#define ETH_REF_CLK		_GPIOA(7)
+#define ETH_MDC			_GPIOA(8)
+#define ETH_MDIO		_GPIOA(9)
+#define SIRQ0			_GPIOA(10)
+#define SIRQ1			_GPIOA(11)
+#define SIRQ2			_GPIOA(12)
+#define I2S_D0			_GPIOA(13)
+#define I2S_BCLK0		_GPIOA(14)
+#define I2S_LRCLK0		_GPIOA(15)
+#define I2S_MCLK0		_GPIOA(16)
+#define I2S_D1			_GPIOA(17)
+#define I2S_BCLK1		_GPIOA(18)
+#define I2S_LRCLK1		_GPIOA(19)
+#define I2S_MCLK1		_GPIOA(20)
+#define ERAM_A5			_GPIOA(21)
+#define ERAM_A6			_GPIOA(22)
+#define ERAM_A7			_GPIOA(23)
+#define ERAM_A8			_GPIOA(24)
+#define ERAM_A9			_GPIOA(25)
+#define ERAM_A10		_GPIOA(26)
+#define ERAM_A11		_GPIOA(27)
+#define SD0_D0			_GPIOA(28)
+#define SD0_D1			_GPIOA(29)
+#define SD0_D2			_GPIOA(30)
+#define SD0_D3			_GPIOA(31)
+
+#define SD1_D0			_GPIOB(0)
+#define SD1_D1			_GPIOB(1)
+#define SD1_D2			_GPIOB(2)
+#define SD1_D3			_GPIOB(3)
+#define SD0_CMD			_GPIOB(4)
+#define SD0_CLK			_GPIOB(5)
+#define SD1_CMD			_GPIOB(6)
+#define SD1_CLK			_GPIOB(7)
+#define SPI0_SCLK		_GPIOB(8)
+#define SPI0_SS			_GPIOB(9)
+#define SPI0_MISO		_GPIOB(10)
+#define SPI0_MOSI		_GPIOB(11)
+#define UART0_RX		_GPIOB(12)
+#define UART0_TX		_GPIOB(13)
+#define UART2_RX		_GPIOB(14)
+#define UART2_TX		_GPIOB(15)
+#define UART2_RTSB		_GPIOB(16)
+#define UART2_CTSB		_GPIOB(17)
+#define UART4_RX		_GPIOB(18)
+#define UART4_TX		_GPIOB(19)
+#define I2C0_SCLK		_GPIOB(20)
+#define I2C0_SDATA		_GPIOB(21)
+#define I2C1_SCLK		_GPIOB(22)
+#define I2C1_SDATA		_GPIOB(23)
+#define I2C2_SCLK		_GPIOB(24)
+#define I2C2_SDATA		_GPIOB(25)
+#define CSI0_DN0		_GPIOB(26)
+#define CSI0_DP0		_GPIOB(27)
+#define CSI0_DN1		_GPIOB(28)
+#define CSI0_DP1		_GPIOB(29)
+#define CSI0_CN			_GPIOB(30)
+#define CSI0_CP			_GPIOB(31)
+
+#define CSI0_DN2		_GPIOC(0)
+#define CSI0_DP2		_GPIOC(1)
+#define CSI0_DN3		_GPIOC(2)
+#define CSI0_DP3		_GPIOC(3)
+#define SENSOR0_PCLK		_GPIOC(4)
+#define CSI1_DN0		_GPIOC(5)
+#define CSI1_DP0		_GPIOC(6)
+#define CSI1_DN1		_GPIOC(7)
+#define CSI1_DP1		_GPIOC(8)
+#define CSI1_CN			_GPIOC(9)
+#define CSI1_CP			_GPIOC(10)
+#define SENSOR0_CKOUT		_GPIOC(11)
+
+#define LVDS_OEP		_GPIOD(0)
+#define LVDS_OEN		_GPIOD(1)
+#define LVDS_ODP		_GPIOD(2)
+#define LVDS_ODN		_GPIOD(3)
+#define LVDS_OCP		_GPIOD(4)
+#define LVDS_OCN		_GPIOD(5)
+#define LVDS_OBP		_GPIOD(6)
+#define LVDS_OBN		_GPIOD(7)
+#define LVDS_OAP		_GPIOD(8)
+#define LVDS_OAN		_GPIOD(9)
+#define LVDS_EEP		_GPIOD(10)
+#define LVDS_EEN		_GPIOD(11)
+#define LVDS_EDP		_GPIOD(12)
+#define LVDS_EDN		_GPIOD(13)
+#define LVDS_ECP		_GPIOD(14)
+#define LVDS_ECN		_GPIOD(15)
+#define LVDS_EBP		_GPIOD(16)
+#define LVDS_EBN		_GPIOD(17)
+#define LVDS_EAP		_GPIOD(18)
+#define LVDS_EAN		_GPIOD(19)
+#define DSI_DP3			_GPIOD(20)
+#define DSI_DN3			_GPIOD(21)
+#define DSI_DP1			_GPIOD(22)
+#define DSI_DN1			_GPIOD(23)
+#define DSI_CP			_GPIOD(24)
+#define DSI_CN			_GPIOD(25)
+#define DSI_DP0			_GPIOD(26)
+#define DSI_DN0			_GPIOD(27)
+#define DSI_DP2			_GPIOD(28)
+#define DSI_DN2			_GPIOD(29)
+
+#define NAND0_D0		_GPIOE(0)
+#define NAND0_D1		_GPIOE(1)
+#define NAND0_D2		_GPIOE(2)
+#define NAND0_D3		_GPIOE(3)
+#define NAND0_D4		_GPIOE(4)
+#define NAND0_D5		_GPIOE(5)
+#define NAND0_D6		_GPIOE(6)
+#define NAND0_D7		_GPIOE(7)
+#define NAND0_DQS		_GPIOE(8)
+#define NAND0_DQSN		_GPIOE(9)
+#define NAND0_ALE		_GPIOE(10)
+#define NAND0_CLE		_GPIOE(11)
+#define NAND0_CEB0		_GPIOE(12)
+#define NAND0_CEB1		_GPIOE(13)
+#define NAND0_CEB2		_GPIOE(14)
+#define NAND0_CEB3		_GPIOE(15)
+#define NAND1_D0		_GPIOE(16)
+#define NAND1_D1		_GPIOE(17)
+#define NAND1_D2		_GPIOE(18)
+#define NAND1_D3		_GPIOE(19)
+#define NAND1_D4		_GPIOE(20)
+#define NAND1_D5		_GPIOE(21)
+#define NAND1_D6		_GPIOE(22)
+#define NAND1_D7		_GPIOE(23)
+#define NAND1_DQS		_GPIOE(24)
+#define NAND1_DQSN		_GPIOE(25)
+#define NAND1_ALE		_GPIOE(26)
+#define NAND1_CLE		_GPIOE(27)
+#define NAND1_CEB0		_GPIOE(28)
+#define NAND1_CEB1		_GPIOE(29)
+#define NAND1_CEB2		_GPIOE(30)
+#define NAND1_CEB3		_GPIOE(31)
+
+#define PCM1_IN			_GPIOF(0)
+#define PCM1_CLK		_GPIOF(1)
+#define PCM1_SYNC		_GPIOF(2)
+#define PCM1_OUT		_GPIOF(3)
+#define UART3_RX		_GPIOF(4)
+#define UART3_TX		_GPIOF(5)
+#define UART3_RTSB		_GPIOF(6)
+#define UART3_CTSB		_GPIOF(7)
+
+/* System */
+#define SGPIO0			_PIN(0)
+#define SGPIO1			_PIN(1)
+#define SGPIO2			_PIN(2)
+#define SGPIO3			_PIN(3)
+
+#define NUM_PADS		(_PIN(3) + 1)
+
+/* Pad names as specified in datasheet */
+static const struct pinctrl_pin_desc s900_pads[] = {
+	PINCTRL_PIN(ETH_TXD0, "eth_txd0"),
+	PINCTRL_PIN(ETH_TXD1, "eth_txd1"),
+	PINCTRL_PIN(ETH_TXEN, "eth_txen"),
+	PINCTRL_PIN(ETH_RXER, "eth_rxer"),
+	PINCTRL_PIN(ETH_CRS_DV, "eth_crs_dv"),
+	PINCTRL_PIN(ETH_RXD1, "eth_rxd1"),
+	PINCTRL_PIN(ETH_RXD0, "eth_rxd0"),
+	PINCTRL_PIN(ETH_REF_CLK, "eth_ref_clk"),
+	PINCTRL_PIN(ETH_MDC, "eth_mdc"),
+	PINCTRL_PIN(ETH_MDIO, "eth_mdio"),
+	PINCTRL_PIN(SIRQ0, "sirq0"),
+	PINCTRL_PIN(SIRQ1, "sirq1"),
+	PINCTRL_PIN(SIRQ2, "sirq2"),
+	PINCTRL_PIN(I2S_D0, "i2s_d0"),
+	PINCTRL_PIN(I2S_BCLK0, "i2s_bclk0"),
+	PINCTRL_PIN(I2S_LRCLK0, "i2s_lrclk0"),
+	PINCTRL_PIN(I2S_MCLK0, "i2s_mclk0"),
+	PINCTRL_PIN(I2S_D1, "i2s_d1"),
+	PINCTRL_PIN(I2S_BCLK1, "i2s_bclk1"),
+	PINCTRL_PIN(I2S_LRCLK1, "i2s_lrclk1"),
+	PINCTRL_PIN(I2S_MCLK1, "i2s_mclk1"),
+	PINCTRL_PIN(PCM1_IN, "pcm1_in"),
+	PINCTRL_PIN(PCM1_CLK, "pcm1_clk"),
+	PINCTRL_PIN(PCM1_SYNC, "pcm1_sync"),
+	PINCTRL_PIN(PCM1_OUT, "pcm1_out"),
+	PINCTRL_PIN(ERAM_A5, "eram_a5"),
+	PINCTRL_PIN(ERAM_A6, "eram_a6"),
+	PINCTRL_PIN(ERAM_A7, "eram_a7"),
+	PINCTRL_PIN(ERAM_A8, "eram_a8"),
+	PINCTRL_PIN(ERAM_A9, "eram_a9"),
+	PINCTRL_PIN(ERAM_A10, "eram_a10"),
+	PINCTRL_PIN(ERAM_A11, "eram_a11"),
+	PINCTRL_PIN(LVDS_OEP, "lvds_oep"),
+	PINCTRL_PIN(LVDS_OEN, "lvds_oen"),
+	PINCTRL_PIN(LVDS_ODP, "lvds_odp"),
+	PINCTRL_PIN(LVDS_ODN, "lvds_odn"),
+	PINCTRL_PIN(LVDS_OCP, "lvds_ocp"),
+	PINCTRL_PIN(LVDS_OCN, "lvds_ocn"),
+	PINCTRL_PIN(LVDS_OBP, "lvds_obp"),
+	PINCTRL_PIN(LVDS_OBN, "lvds_obn"),
+	PINCTRL_PIN(LVDS_OAP, "lvds_oap"),
+	PINCTRL_PIN(LVDS_OAN, "lvds_oan"),
+	PINCTRL_PIN(LVDS_EEP, "lvds_eep"),
+	PINCTRL_PIN(LVDS_EEN, "lvds_een"),
+	PINCTRL_PIN(LVDS_EDP, "lvds_edp"),
+	PINCTRL_PIN(LVDS_EDN, "lvds_edn"),
+	PINCTRL_PIN(LVDS_ECP, "lvds_ecp"),
+	PINCTRL_PIN(LVDS_ECN, "lvds_ecn"),
+	PINCTRL_PIN(LVDS_EBP, "lvds_ebp"),
+	PINCTRL_PIN(LVDS_EBN, "lvds_ebn"),
+	PINCTRL_PIN(LVDS_EAP, "lvds_eap"),
+	PINCTRL_PIN(LVDS_EAN, "lvds_ean"),
+	PINCTRL_PIN(SD0_D0, "sd0_d0"),
+	PINCTRL_PIN(SD0_D1, "sd0_d1"),
+	PINCTRL_PIN(SD0_D2, "sd0_d2"),
+	PINCTRL_PIN(SD0_D3, "sd0_d3"),
+	PINCTRL_PIN(SD1_D0, "sd1_d0"),
+	PINCTRL_PIN(SD1_D1, "sd1_d1"),
+	PINCTRL_PIN(SD1_D2, "sd1_d2"),
+	PINCTRL_PIN(SD1_D3, "sd1_d3"),
+	PINCTRL_PIN(SD0_CMD, "sd0_cmd"),
+	PINCTRL_PIN(SD0_CLK, "sd0_clk"),
+	PINCTRL_PIN(SD1_CMD, "sd1_cmd"),
+	PINCTRL_PIN(SD1_CLK, "sd1_clk"),
+	PINCTRL_PIN(SPI0_SCLK, "spi0_sclk"),
+	PINCTRL_PIN(SPI0_SS, "spi0_ss"),
+	PINCTRL_PIN(SPI0_MISO, "spi0_miso"),
+	PINCTRL_PIN(SPI0_MOSI, "spi0_mosi"),
+	PINCTRL_PIN(UART0_RX, "uart0_rx"),
+	PINCTRL_PIN(UART0_TX, "uart0_tx"),
+	PINCTRL_PIN(UART2_RX, "uart2_rx"),
+	PINCTRL_PIN(UART2_TX, "uart2_tx"),
+	PINCTRL_PIN(UART2_RTSB, "uart2_rtsb"),
+	PINCTRL_PIN(UART2_CTSB, "uart2_ctsb"),
+	PINCTRL_PIN(UART3_RX, "uart3_rx"),
+	PINCTRL_PIN(UART3_TX, "uart3_tx"),
+	PINCTRL_PIN(UART3_RTSB, "uart3_rtsb"),
+	PINCTRL_PIN(UART3_CTSB, "uart3_ctsb"),
+	PINCTRL_PIN(UART4_RX, "uart4_rx"),
+	PINCTRL_PIN(UART4_TX, "uart4_tx"),
+	PINCTRL_PIN(I2C0_SCLK, "i2c0_sclk"),
+	PINCTRL_PIN(I2C0_SDATA, "i2c0_sdata"),
+	PINCTRL_PIN(I2C1_SCLK, "i2c1_sclk"),
+	PINCTRL_PIN(I2C1_SDATA, "i2c1_sdata"),
+	PINCTRL_PIN(I2C2_SCLK, "i2c2_sclk"),
+	PINCTRL_PIN(I2C2_SDATA, "i2c2_sdata"),
+	PINCTRL_PIN(CSI0_DN0, "csi0_dn0"),
+	PINCTRL_PIN(CSI0_DP0, "csi0_dp0"),
+	PINCTRL_PIN(CSI0_DN1, "csi0_dn1"),
+	PINCTRL_PIN(CSI0_DP1, "csi0_dp1"),
+	PINCTRL_PIN(CSI0_CN, "csi0_cn"),
+	PINCTRL_PIN(CSI0_CP, "csi0_cp"),
+	PINCTRL_PIN(CSI0_DN2, "csi0_dn2"),
+	PINCTRL_PIN(CSI0_DP2, "csi0_dp2"),
+	PINCTRL_PIN(CSI0_DN3, "csi0_dn3"),
+	PINCTRL_PIN(CSI0_DP3, "csi0_dp3"),
+	PINCTRL_PIN(DSI_DP3, "dsi_dp3"),
+	PINCTRL_PIN(DSI_DN3, "dsi_dn3"),
+	PINCTRL_PIN(DSI_DP1, "dsi_dp1"),
+	PINCTRL_PIN(DSI_DN1, "dsi_dn1"),
+	PINCTRL_PIN(DSI_CP, "dsi_cp"),
+	PINCTRL_PIN(DSI_CN, "dsi_cn"),
+	PINCTRL_PIN(DSI_DP0, "dsi_dp0"),
+	PINCTRL_PIN(DSI_DN0, "dsi_dn0"),
+	PINCTRL_PIN(DSI_DP2, "dsi_dp2"),
+	PINCTRL_PIN(DSI_DN2, "dsi_dn2"),
+	PINCTRL_PIN(SENSOR0_PCLK, "sensor0_pclk"),
+	PINCTRL_PIN(CSI1_DN0, "csi1_dn0"),
+	PINCTRL_PIN(CSI1_DP0, "csi1_dp0"),
+	PINCTRL_PIN(CSI1_DN1, "csi1_dn1"),
+	PINCTRL_PIN(CSI1_DP1, "csi1_dp1"),
+	PINCTRL_PIN(CSI1_CN, "csi1_cn"),
+	PINCTRL_PIN(CSI1_CP, "csi1_cp"),
+	PINCTRL_PIN(SENSOR0_CKOUT, "sensor0_ckout"),
+	PINCTRL_PIN(NAND0_D0, "nand0_d0"),
+	PINCTRL_PIN(NAND0_D1, "nand0_d1"),
+	PINCTRL_PIN(NAND0_D2, "nand0_d2"),
+	PINCTRL_PIN(NAND0_D3, "nand0_d3"),
+	PINCTRL_PIN(NAND0_D4, "nand0_d4"),
+	PINCTRL_PIN(NAND0_D5, "nand0_d5"),
+	PINCTRL_PIN(NAND0_D6, "nand0_d6"),
+	PINCTRL_PIN(NAND0_D7, "nand0_d7"),
+	PINCTRL_PIN(NAND0_DQS, "nand0_dqs"),
+	PINCTRL_PIN(NAND0_DQSN, "nand0_dqsn"),
+	PINCTRL_PIN(NAND0_ALE, "nand0_ale"),
+	PINCTRL_PIN(NAND0_CLE, "nand0_cle"),
+	PINCTRL_PIN(NAND0_CEB0, "nand0_ceb0"),
+	PINCTRL_PIN(NAND0_CEB1, "nand0_ceb1"),
+	PINCTRL_PIN(NAND0_CEB2, "nand0_ceb2"),
+	PINCTRL_PIN(NAND0_CEB3, "nand0_ceb3"),
+	PINCTRL_PIN(NAND1_D0, "nand1_d0"),
+	PINCTRL_PIN(NAND1_D1, "nand1_d1"),
+	PINCTRL_PIN(NAND1_D2, "nand1_d2"),
+	PINCTRL_PIN(NAND1_D3, "nand1_d3"),
+	PINCTRL_PIN(NAND1_D4, "nand1_d4"),
+	PINCTRL_PIN(NAND1_D5, "nand1_d5"),
+	PINCTRL_PIN(NAND1_D6, "nand1_d6"),
+	PINCTRL_PIN(NAND1_D7, "nand1_d7"),
+	PINCTRL_PIN(NAND1_DQS, "nand1_dqs"),
+	PINCTRL_PIN(NAND1_DQSN, "nand1_dqsn"),
+	PINCTRL_PIN(NAND1_ALE, "nand1_ale"),
+	PINCTRL_PIN(NAND1_CLE, "nand1_cle"),
+	PINCTRL_PIN(NAND1_CEB0, "nand1_ceb0"),
+	PINCTRL_PIN(NAND1_CEB1, "nand1_ceb1"),
+	PINCTRL_PIN(NAND1_CEB2, "nand1_ceb2"),
+	PINCTRL_PIN(NAND1_CEB3, "nand1_ceb3"),
+	PINCTRL_PIN(SGPIO0, "sgpio0"),
+	PINCTRL_PIN(SGPIO1, "sgpio1"),
+	PINCTRL_PIN(SGPIO2, "sgpio2"),
+	PINCTRL_PIN(SGPIO3, "sgpio3")
+};
+
+enum s900_pinmux_functions {
+	S900_MUX_ERAM,
+	S900_MUX_ETH_RMII,
+	S900_MUX_ETH_SMII,
+	S900_MUX_SPI0,
+	S900_MUX_SPI1,
+	S900_MUX_SPI2,
+	S900_MUX_SPI3,
+	S900_MUX_SENS0,
+	S900_MUX_UART0,
+	S900_MUX_UART1,
+	S900_MUX_UART2,
+	S900_MUX_UART3,
+	S900_MUX_UART4,
+	S900_MUX_UART5,
+	S900_MUX_UART6,
+	S900_MUX_I2S0,
+	S900_MUX_I2S1,
+	S900_MUX_PCM0,
+	S900_MUX_PCM1,
+	S900_MUX_JTAG,
+	S900_MUX_PWM0,
+	S900_MUX_PWM1,
+	S900_MUX_PWM2,
+	S900_MUX_PWM3,
+	S900_MUX_PWM4,
+	S900_MUX_PWM5,
+	S900_MUX_SD0,
+	S900_MUX_SD1,
+	S900_MUX_SD2,
+	S900_MUX_SD3,
+	S900_MUX_I2C0,
+	S900_MUX_I2C1,
+	S900_MUX_I2C2,
+	S900_MUX_I2C3,
+	S900_MUX_I2C4,
+	S900_MUX_I2C5,
+	S900_MUX_LVDS,
+	S900_MUX_USB20,
+	S900_MUX_USB30,
+	S900_MUX_GPU,
+	S900_MUX_MIPI_CSI0,
+	S900_MUX_MIPI_CSI1,
+	S900_MUX_MIPI_DSI,
+	S900_MUX_NAND0,
+	S900_MUX_NAND1,
+	S900_MUX_SPDIF,
+	S900_MUX_SIRQ0,
+	S900_MUX_SIRQ1,
+	S900_MUX_SIRQ2,
+	S900_MUX_AUX_START,
+	S900_MUX_MAX,
+	S900_MUX_RESERVED
+};
+
+/* mfp0_22 */
+static unsigned int lvds_oxx_uart4_mfp_pads[]	= { LVDS_OAP, LVDS_OAN };
+static unsigned int lvds_oxx_uart4_mfp_funcs[]	= { S900_MUX_ERAM,
+						    S900_MUX_UART4 };
+/* mfp0_21_20 */
+static unsigned int rmii_mdc_mfp_pads[]		= { ETH_MDC };
+static unsigned int rmii_mdc_mfp_funcs[]	= { S900_MUX_ETH_RMII,
+						    S900_MUX_PWM2,
+						    S900_MUX_UART2,
+						    S900_MUX_RESERVED };
+static unsigned int rmii_mdio_mfp_pads[]	= { ETH_MDIO };
+static unsigned int rmii_mdio_mfp_funcs[]	= { S900_MUX_ETH_RMII,
+						    S900_MUX_PWM3,
+						    S900_MUX_UART2,
+						    S900_MUX_RESERVED };
+/* mfp0_19 */
+static unsigned int sirq0_mfp_pads[]		= { SIRQ0 };
+static unsigned int sirq0_mfp_funcs[]		= { S900_MUX_SIRQ0,
+						    S900_MUX_PWM0 };
+static unsigned int sirq1_mfp_pads[]		= { SIRQ1 };
+static unsigned int sirq1_mfp_funcs[]		= { S900_MUX_SIRQ1,
+						    S900_MUX_PWM1 };
+/* mfp0_18_16 */
+static unsigned int rmii_txd0_mfp_pads[]	= { ETH_TXD0 };
+static unsigned int rmii_txd0_mfp_funcs[]	= { S900_MUX_ETH_RMII,
+						    S900_MUX_ETH_SMII,
+						    S900_MUX_SPI2,
+						    S900_MUX_UART6,
+						    S900_MUX_SENS0,
+						    S900_MUX_PWM0 };
+static unsigned int rmii_txd1_mfp_pads[]	= { ETH_TXD1 };
+static unsigned int rmii_txd1_mfp_funcs[]	= { S900_MUX_ETH_RMII,
+						    S900_MUX_ETH_SMII,
+						    S900_MUX_SPI2,
+						    S900_MUX_UART6,
+						    S900_MUX_SENS0,
+						    S900_MUX_PWM1 };
+/* mfp0_15_13 */
+static unsigned int rmii_txen_mfp_pads[]	= { ETH_TXEN };
+static unsigned int rmii_txen_mfp_funcs[]	= { S900_MUX_ETH_RMII,
+						    S900_MUX_UART2,
+						    S900_MUX_SPI3,
+						    S900_MUX_RESERVED,
+						    S900_MUX_RESERVED,
+						    S900_MUX_PWM2,
+						    S900_MUX_SENS0 };
+
+static unsigned int rmii_rxer_mfp_pads[]	= { ETH_RXER };
+static unsigned int rmii_rxer_mfp_funcs[]	= { S900_MUX_ETH_RMII,
+						    S900_MUX_UART2,
+						    S900_MUX_SPI3,
+						    S900_MUX_RESERVED,
+						    S900_MUX_RESERVED,
+						    S900_MUX_PWM3,
+						    S900_MUX_SENS0 };
+/* mfp0_12_11 */
+static unsigned int rmii_crs_dv_mfp_pads[]	= { ETH_CRS_DV };
+static unsigned int rmii_crs_dv_mfp_funcs[]	= { S900_MUX_ETH_RMII,
+						    S900_MUX_ETH_SMII,
+						    S900_MUX_SPI2,
+						    S900_MUX_UART4 };
+/* mfp0_10_8 */
+static unsigned int rmii_rxd1_mfp_pads[]	= { ETH_RXD1 };
+static unsigned int rmii_rxd1_mfp_funcs[]	= { S900_MUX_ETH_RMII,
+						    S900_MUX_UART2,
+						    S900_MUX_SPI3,
+						    S900_MUX_RESERVED,
+						    S900_MUX_UART5,
+						    S900_MUX_PWM0,
+						    S900_MUX_SENS0 };
+static unsigned int rmii_rxd0_mfp_pads[]	= { ETH_RXD0 };
+static unsigned int rmii_rxd0_mfp_funcs[]	= { S900_MUX_ETH_RMII,
+						    S900_MUX_UART2,
+						    S900_MUX_SPI3,
+						    S900_MUX_RESERVED,
+						    S900_MUX_UART5,
+						    S900_MUX_PWM1,
+						    S900_MUX_SENS0 };
+/* mfp0_7_6 */
+static unsigned int rmii_ref_clk_mfp_pads[]	= { ETH_REF_CLK };
+static unsigned int rmii_ref_clk_mfp_funcs[]	= { S900_MUX_ETH_RMII,
+						    S900_MUX_UART4,
+						    S900_MUX_SPI2,
+						    S900_MUX_RESERVED };
+/* mfp0_5 */
+static unsigned int i2s_d0_mfp_pads[]		= { I2S_D0 };
+static unsigned int i2s_d0_mfp_funcs[]		= { S900_MUX_I2S0,
+						    S900_MUX_PCM0 };
+static unsigned int i2s_d1_mfp_pads[]		= { I2S_D1 };
+static unsigned int i2s_d1_mfp_funcs[]		= { S900_MUX_I2S1,
+						    S900_MUX_PCM0 };
+
+/* mfp0_4_3 */
+static unsigned int i2s_lr_m_clk0_mfp_pads[]	= { I2S_LRCLK0,
+						    I2S_MCLK0 };
+static unsigned int i2s_lr_m_clk0_mfp_funcs[]	= { S900_MUX_I2S0,
+						    S900_MUX_PCM0,
+						    S900_MUX_PCM1,
+						    S900_MUX_RESERVED };
+/* mfp0_2 */
+static unsigned int i2s_bclk0_mfp_pads[]	= { I2S_BCLK0 };
+static unsigned int i2s_bclk0_mfp_funcs[]	= { S900_MUX_I2S0,
+						    S900_MUX_PCM0 };
+static unsigned int i2s_bclk1_mclk1_mfp_pads[]	= { I2S_BCLK1,
+						    I2S_LRCLK1,
+						    I2S_MCLK1 };
+static unsigned int i2s_bclk1_mclk1_mfp_funcs[] = { S900_MUX_I2S1,
+						    S900_MUX_PCM0 };
+/* mfp0_1_0 */
+static unsigned int pcm1_in_out_mfp_pads[]	= { PCM1_IN,
+						    PCM1_OUT };
+static unsigned int pcm1_in_out_mfp_funcs[]	= { S900_MUX_PCM1,
+						    S900_MUX_SPI1,
+						    S900_MUX_I2C3,
+						    S900_MUX_UART4 };
+static unsigned int pcm1_clk_mfp_pads[]		= { PCM1_CLK };
+static unsigned int pcm1_clk_mfp_funcs[]	= { S900_MUX_PCM1,
+						    S900_MUX_SPI1,
+						    S900_MUX_PWM4,
+						    S900_MUX_UART4 };
+static unsigned int pcm1_sync_mfp_pads[]	= { PCM1_SYNC };
+static unsigned int pcm1_sync_mfp_funcs[]	= { S900_MUX_PCM1,
+						    S900_MUX_SPI1,
+						    S900_MUX_PWM5,
+						    S900_MUX_UART4 };
+/* mfp1_31_29 */
+static unsigned int eram_a5_mfp_pads[]		= { ERAM_A5 };
+static unsigned int eram_a5_mfp_funcs[]		= { S900_MUX_UART4,
+						    S900_MUX_JTAG,
+						    S900_MUX_ERAM,
+						    S900_MUX_PWM0,
+						    S900_MUX_RESERVED,
+						    S900_MUX_SENS0 };
+static unsigned int eram_a6_mfp_pads[]		= { ERAM_A6 };
+static unsigned int eram_a6_mfp_funcs[]		= { S900_MUX_UART4,
+						    S900_MUX_JTAG,
+						    S900_MUX_ERAM,
+						    S900_MUX_PWM1,
+						    S900_MUX_RESERVED,
+						    S900_MUX_SENS0,
+};
+static unsigned int eram_a7_mfp_pads[]		= { ERAM_A7 };
+static unsigned int eram_a7_mfp_funcs[]		= { S900_MUX_RESERVED,
+						    S900_MUX_JTAG,
+						    S900_MUX_ERAM,
+						    S900_MUX_RESERVED,
+						    S900_MUX_RESERVED,
+						    S900_MUX_SENS0 };
+/* mfp1_28_26 */
+static unsigned int eram_a8_mfp_pads[]		= { ERAM_A8 };
+static unsigned int eram_a8_mfp_funcs[]		= { S900_MUX_RESERVED,
+						    S900_MUX_JTAG,
+						    S900_MUX_ERAM,
+						    S900_MUX_PWM1,
+						    S900_MUX_RESERVED,
+						    S900_MUX_SENS0 };
+static unsigned int eram_a9_mfp_pads[]		= { ERAM_A9 };
+static unsigned int eram_a9_mfp_funcs[]		= { S900_MUX_USB20,
+						    S900_MUX_UART5,
+						    S900_MUX_ERAM,
+						    S900_MUX_PWM2,
+						    S900_MUX_RESERVED,
+						    S900_MUX_SENS0 };
+static unsigned int eram_a10_mfp_pads[]		= { ERAM_A10 };
+static unsigned int eram_a10_mfp_funcs[]	= { S900_MUX_USB30,
+						    S900_MUX_JTAG,
+						    S900_MUX_ERAM,
+						    S900_MUX_PWM3,
+						    S900_MUX_RESERVED,
+						    S900_MUX_SENS0,
+						    S900_MUX_RESERVED,
+						    S900_MUX_RESERVED };
+/* mfp1_25_23 */
+static unsigned int eram_a11_mfp_pads[]		= { ERAM_A11 };
+static unsigned int eram_a11_mfp_funcs[]	= { S900_MUX_RESERVED,
+						    S900_MUX_RESERVED,
+						    S900_MUX_ERAM,
+						    S900_MUX_PWM2,
+						    S900_MUX_UART5,
+						    S900_MUX_RESERVED,
+						    S900_MUX_SENS0,
+						    S900_MUX_RESERVED };
+/* mfp1_22 */
+static unsigned int lvds_oep_odn_mfp_pads[]	= { LVDS_OEP,
+						    LVDS_OEN,
+						    LVDS_ODP,
+						    LVDS_ODN };
+static unsigned int lvds_oep_odn_mfp_funcs[]	= { S900_MUX_LVDS,
+						    S900_MUX_UART2 };
+static unsigned int lvds_ocp_obn_mfp_pads[]	= { LVDS_OCP,
+						    LVDS_OCN,
+						    LVDS_OBP,
+						    LVDS_OBN };
+static unsigned int lvds_ocp_obn_mfp_funcs[]	= { S900_MUX_LVDS,
+						    S900_MUX_PCM1 };
+static unsigned int lvds_oap_oan_mfp_pads[]	= { LVDS_OAP,
+						    LVDS_OAN };
+static unsigned int lvds_oap_oan_mfp_funcs[]	= { S900_MUX_LVDS,
+						    S900_MUX_ERAM };
+/* mfp1_21 */
+static unsigned int lvds_e_mfp_pads[]		= { LVDS_EEP,
+						    LVDS_EEN,
+						    LVDS_EDP,
+						    LVDS_EDN,
+						    LVDS_ECP,
+						    LVDS_ECN,
+						    LVDS_EBP,
+						    LVDS_EBN,
+						    LVDS_EAP,
+						    LVDS_EAN };
+static unsigned int lvds_e_mfp_funcs[]		= { S900_MUX_LVDS,
+						    S900_MUX_ERAM };
+/* mfp1_5_4 */
+static unsigned int spi0_sclk_mosi_mfp_pads[]	= { SPI0_SCLK,
+						    SPI0_MOSI };
+static unsigned int spi0_sclk_mosi_mfp_funcs[]	= { S900_MUX_SPI0,
+						    S900_MUX_ERAM,
+						    S900_MUX_I2C3,
+						    S900_MUX_PCM0 };
+/* mfp1_3_1 */
+static unsigned int spi0_ss_mfp_pads[]		= { SPI0_SS };
+static unsigned int spi0_ss_mfp_funcs[]		= { S900_MUX_SPI0,
+						    S900_MUX_ERAM,
+						    S900_MUX_I2S1,
+						    S900_MUX_PCM1,
+						    S900_MUX_PCM0,
+						    S900_MUX_PWM4 };
+static unsigned int spi0_miso_mfp_pads[]	= { SPI0_MISO };
+static unsigned int spi0_miso_mfp_funcs[]	= { S900_MUX_SPI0,
+						    S900_MUX_ERAM,
+						    S900_MUX_I2S1,
+						    S900_MUX_PCM1,
+						    S900_MUX_PCM0,
+						    S900_MUX_PWM5 };
+/* mfp2_23 */
+static unsigned int uart2_rtsb_mfp_pads[]	= { UART2_RTSB };
+static unsigned int uart2_rtsb_mfp_funcs[]	= { S900_MUX_UART2,
+						    S900_MUX_UART0 };
+/* mfp2_22 */
+static unsigned int uart2_ctsb_mfp_pads[]	= { UART2_CTSB };
+static unsigned int uart2_ctsb_mfp_funcs[]	= { S900_MUX_UART2,
+						    S900_MUX_UART0 };
+/* mfp2_21 */
+static unsigned int uart3_rtsb_mfp_pads[]	= { UART3_RTSB };
+static unsigned int uart3_rtsb_mfp_funcs[]	= { S900_MUX_UART3,
+						    S900_MUX_UART5 };
+/* mfp2_20 */
+static unsigned int uart3_ctsb_mfp_pads[]	= { UART3_CTSB };
+static unsigned int uart3_ctsb_mfp_funcs[]	= { S900_MUX_UART3,
+						    S900_MUX_UART5 };
+/* mfp2_19_17 */
+static unsigned int sd0_d0_mfp_pads[]		= { SD0_D0 };
+static unsigned int sd0_d0_mfp_funcs[]		= { S900_MUX_SD0,
+						    S900_MUX_ERAM,
+						    S900_MUX_RESERVED,
+						    S900_MUX_JTAG,
+						    S900_MUX_UART2,
+						    S900_MUX_UART5,
+						    S900_MUX_GPU };
+/* mfp2_16_14 */
+static unsigned int sd0_d1_mfp_pads[]		= { SD0_D1 };
+static unsigned int sd0_d1_mfp_funcs[]		= { S900_MUX_SD0,
+						    S900_MUX_ERAM,
+						    S900_MUX_GPU,
+						    S900_MUX_RESERVED,
+						    S900_MUX_UART2,
+						    S900_MUX_UART5 };
+/* mfp_13_11 */
+static unsigned int sd0_d2_d3_mfp_pads[]	= { SD0_D2,
+						    SD0_D3 };
+static unsigned int sd0_d2_d3_mfp_funcs[]	= { S900_MUX_SD0,
+						    S900_MUX_ERAM,
+						    S900_MUX_RESERVED,
+						    S900_MUX_JTAG,
+						    S900_MUX_UART2,
+						    S900_MUX_UART1,
+						    S900_MUX_GPU };
+/* mfp2_10_9 */
+static unsigned int sd1_d0_d3_mfp_pads[]	= { SD1_D0, SD1_D1,
+						    SD1_D2, SD1_D3 };
+static unsigned int sd1_d0_d3_mfp_funcs[]	= { S900_MUX_SD1,
+						    S900_MUX_ERAM };
+/* mfp2_8_7 */
+static unsigned int sd0_cmd_mfp_pads[]		= { SD0_CMD };
+static unsigned int sd0_cmd_mfp_funcs[]		= { S900_MUX_SD0,
+						    S900_MUX_ERAM,
+						    S900_MUX_GPU,
+						    S900_MUX_JTAG };
+/* mfp2_6_5 */
+static unsigned int sd0_clk_mfp_pads[]		= { SD0_CLK };
+static unsigned int sd0_clk_mfp_funcs[]		= { S900_MUX_SD0,
+						    S900_MUX_ERAM,
+						    S900_MUX_JTAG,
+						    S900_MUX_GPU };
+/* mfp2_4_3 */
+static unsigned int sd1_cmd_clk_mfp_pads[]	= { SD1_CMD, SD1_CLK };
+static unsigned int sd1_cmd_clk_mfp_funcs[]	= { S900_MUX_SD1,
+						    S900_MUX_ERAM };
+/* mfp2_2_0 */
+static unsigned int uart0_rx_mfp_pads[]		= { UART0_RX };
+static unsigned int uart0_rx_mfp_funcs[]	= { S900_MUX_UART0,
+						    S900_MUX_UART2,
+						    S900_MUX_SPI1,
+						    S900_MUX_I2C5,
+						    S900_MUX_PCM1,
+						    S900_MUX_I2S1 };
+/* mfp3_27 */
+static unsigned int nand0_d0_ceb3_mfp_pads[]	= { NAND0_D0, NAND0_D1,
+						    NAND0_D2, NAND0_D3,
+						    NAND0_D4, NAND0_D5,
+						    NAND0_D6, NAND0_D7,
+						    NAND0_DQSN, NAND0_CEB3 };
+static unsigned int nand0_d0_ceb3_mfp_funcs[]	= { S900_MUX_NAND0,
+						    S900_MUX_SD2 };
+/* mfp3_21_19 */
+static unsigned int uart0_tx_mfp_pads[]		= { UART0_TX };
+static unsigned int uart0_tx_mfp_funcs[]	= { S900_MUX_UART0,
+						    S900_MUX_UART2,
+						    S900_MUX_SPI1,
+						    S900_MUX_I2C5,
+						    S900_MUX_SPDIF,
+						    S900_MUX_PCM1,
+						    S900_MUX_I2S1 };
+/* mfp3_18_16 */
+static unsigned int i2c0_mfp_pads[]		= { I2C0_SCLK, I2C0_SDATA };
+static unsigned int i2c0_mfp_funcs[]		= { S900_MUX_I2C0,
+						    S900_MUX_UART2,
+						    S900_MUX_I2C1,
+						    S900_MUX_UART1,
+						    S900_MUX_SPI1 };
+/* mfp3_15 */
+static unsigned int csi0_cn_cp_mfp_pads[]	= { CSI0_CN, CSI0_CP };
+static unsigned int csi0_cn_cp_mfp_funcs[]	= { S900_MUX_SENS0,
+						    S900_MUX_SENS0 };
+/* mfp3_14 */
+static unsigned int csi0_dn0_dp3_mfp_pads[]	= { CSI0_DN0, CSI0_DP0,
+						    CSI0_DN1, CSI0_DP1,
+						    CSI0_CN, CSI0_CP,
+						    CSI0_DP2, CSI0_DN2,
+						    CSI0_DN3, CSI0_DP3 };
+static unsigned int csi0_dn0_dp3_mfp_funcs[]	= { S900_MUX_MIPI_CSI0,
+						    S900_MUX_SENS0 };
+/* mfp3_13 */
+static unsigned int csi1_dn0_cp_mfp_pads[]	= { CSI1_DN0, CSI1_DP0,
+						    CSI1_DN1, CSI1_DP1,
+						    CSI1_CN, CSI1_CP };
+static unsigned int csi1_dn0_cp_mfp_funcs[]	= { S900_MUX_MIPI_CSI1,
+						    S900_MUX_SENS0 };
+/* mfp3_12_dsi */
+static unsigned int dsi_dp3_dn1_mfp_pads[]	= { DSI_DP3, DSI_DN2,
+						    DSI_DP1, DSI_DN1 };
+static unsigned int dsi_dp3_dn1_mfp_funcs[]	= { S900_MUX_MIPI_DSI,
+						    S900_MUX_UART2 };
+static unsigned int dsi_cp_dn0_mfp_pads[]	= { DSI_CP, DSI_CN,
+						    DSI_DP0, DSI_DN0 };
+static unsigned int dsi_cp_dn0_mfp_funcs[]	= { S900_MUX_MIPI_DSI,
+						    S900_MUX_PCM1 };
+static unsigned int dsi_dp2_dn2_mfp_pads[]	= { DSI_DP2, DSI_DN2 };
+static unsigned int dsi_dp2_dn2_mfp_funcs[]	= { S900_MUX_MIPI_DSI,
+						    S900_MUX_UART4 };
+/* mfp3_11 */
+static unsigned int nand1_d0_ceb1_mfp_pads[]	= { NAND1_D0, NAND1_D1,
+						    NAND1_D2, NAND1_D3,
+						    NAND1_D4, NAND1_D5,
+						    NAND1_D6, NAND1_D7,
+						    NAND1_DQSN, NAND1_CEB1 };
+static unsigned int nand1_d0_ceb1_mfp_funcs[]	= { S900_MUX_NAND1,
+						    S900_MUX_SD3 };
+/* mfp3_10 */
+static unsigned int nand1_ceb3_mfp_pads[]	= { NAND1_CEB3 };
+static unsigned int nand1_ceb3_mfp_funcs[]	= { S900_MUX_NAND1,
+						    S900_MUX_PWM0 };
+static unsigned int nand1_ceb0_mfp_pads[]	= { NAND1_CEB0 };
+static unsigned int nand1_ceb0_mfp_funcs[]	= { S900_MUX_NAND1,
+						    S900_MUX_PWM1 };
+/* mfp3_9 */
+static unsigned int csi1_dn0_dp0_mfp_pads[]	= { CSI1_DN0, CSI1_DP0 };
+static unsigned int csi1_dn0_dp0_mfp_funcs[]	= { S900_MUX_SENS0,
+						    S900_MUX_SENS0 };
+/* mfp3_8 */
+static unsigned int uart4_rx_tx_mfp_pads[]	= { UART4_RX, UART4_TX };
+static unsigned int uart4_rx_tx_mfp_funcs[]	= { S900_MUX_UART4,
+						    S900_MUX_I2C4 };
+/* PADDRV group data */
+/* drv0 */
+static unsigned int sgpio3_drv_pads[]		= { SGPIO3 };
+static unsigned int sgpio2_drv_pads[]		= { SGPIO2 };
+static unsigned int sgpio1_drv_pads[]		= { SGPIO1 };
+static unsigned int sgpio0_drv_pads[]		= { SGPIO0 };
+static unsigned int rmii_tx_d0_d1_drv_pads[]	= { ETH_TXD0, ETH_TXD1 };
+static unsigned int rmii_txen_rxer_drv_pads[]	= { ETH_TXEN, ETH_RXER };
+static unsigned int rmii_crs_dv_drv_pads[]	= { ETH_CRS_DV };
+static unsigned int rmii_rx_d1_d0_drv_pads[]	= { ETH_RXD1, ETH_RXD0 };
+static unsigned int rmii_ref_clk_drv_pads[]	= { ETH_REF_CLK };
+static unsigned int rmii_mdc_mdio_drv_pads[]	= { ETH_MDC, ETH_MDIO };
+static unsigned int sirq_0_1_drv_pads[]		= { SIRQ0, SIRQ1 };
+static unsigned int sirq2_drv_pads[]		= { SIRQ2 };
+static unsigned int i2s_d0_d1_drv_pads[]	= { I2S_D0, I2S_D1 };
+static unsigned int i2s_lr_m_clk0_drv_pads[]	= { I2S_LRCLK0, I2S_MCLK0 };
+static unsigned int i2s_blk1_mclk1_drv_pads[]	= { I2S_BCLK0, I2S_BCLK1,
+						    I2S_LRCLK1, I2S_MCLK1 };
+static unsigned int pcm1_in_out_drv_pads[]	= { PCM1_IN, PCM1_CLK,
+						    PCM1_SYNC, PCM1_OUT };
+/* drv1 */
+static unsigned int lvds_oap_oan_drv_pads[]	= { LVDS_OAP, LVDS_OAN };
+static unsigned int lvds_oep_odn_drv_pads[]	= { LVDS_OEP, LVDS_OEN,
+						    LVDS_ODP, LVDS_ODN };
+static unsigned int lvds_ocp_obn_drv_pads[]	= { LVDS_OCP, LVDS_OCN,
+						    LVDS_OBP, LVDS_OBN };
+static unsigned int lvds_e_drv_pads[]		= { LVDS_EEP, LVDS_EEN,
+						    LVDS_EDP, LVDS_EDN,
+						    LVDS_ECP, LVDS_ECN,
+						    LVDS_EBP, LVDS_EBN };
+static unsigned int sd0_d3_d0_drv_pads[]	= { SD0_D3, SD0_D2,
+						    SD0_D1, SD0_D0 };
+static unsigned int sd1_d3_d0_drv_pads[]	= { SD1_D3, SD1_D2,
+						    SD1_D1, SD1_D0 };
+static unsigned int sd0_sd1_cmd_clk_drv_pads[]	= { SD0_CLK, SD0_CMD,
+						    SD1_CLK, SD1_CMD };
+static unsigned int spi0_sclk_mosi_drv_pads[]	= { SPI0_SCLK, SPI0_MOSI };
+static unsigned int spi0_ss_miso_drv_pads[]	= { SPI0_SS, SPI0_MISO };
+static unsigned int uart0_rx_tx_drv_pads[]	= { UART0_RX, UART0_TX };
+static unsigned int uart4_rx_tx_drv_pads[]	= { UART4_RX, UART4_TX };
+static unsigned int uart2_drv_pads[]		= { UART2_RX, UART2_TX,
+						    UART2_RTSB, UART2_CTSB };
+static unsigned int uart3_drv_pads[]		= { UART3_RX, UART3_TX,
+						    UART3_RTSB, UART3_CTSB };
+/* drv2 */
+static unsigned int i2c0_drv_pads[]		= { I2C0_SCLK, I2C0_SDATA };
+static unsigned int i2c1_drv_pads[]		= { I2C1_SCLK, I2C1_SDATA };
+static unsigned int i2c2_drv_pads[]		= { I2C2_SCLK, I2C2_SDATA };
+static unsigned int sensor0_drv_pads[]		= { SENSOR0_PCLK,
+						    SENSOR0_CKOUT };
+/* SR group data */
+/* sr0 */
+static unsigned int sgpio3_sr_pads[]		= { SGPIO3 };
+static unsigned int sgpio2_sr_pads[]		= { SGPIO2 };
+static unsigned int sgpio1_sr_pads[]		= { SGPIO1 };
+static unsigned int sgpio0_sr_pads[]		= { SGPIO0 };
+static unsigned int rmii_tx_d0_d1_sr_pads[]	= { ETH_TXD0, ETH_TXD1 };
+static unsigned int rmii_txen_rxer_sr_pads[]	= { ETH_TXEN, ETH_RXER };
+static unsigned int rmii_crs_dv_sr_pads[]	= { ETH_CRS_DV };
+static unsigned int rmii_rx_d1_d0_sr_pads[]	= { ETH_RXD1, ETH_RXD0 };
+static unsigned int rmii_ref_clk_sr_pads[]	= { ETH_REF_CLK };
+static unsigned int rmii_mdc_mdio_sr_pads[]	= { ETH_MDC, ETH_MDIO };
+static unsigned int sirq_0_1_sr_pads[]		= { SIRQ0, SIRQ1 };
+static unsigned int sirq2_sr_pads[]		= { SIRQ2 };
+static unsigned int i2s_do_d1_sr_pads[]		= { I2S_D0, I2S_D1 };
+static unsigned int i2s_lr_m_clk0_sr_pads[]	= { I2S_LRCLK0, I2S_MCLK0 };
+static unsigned int i2s_bclk0_mclk1_sr_pads[]	= { I2S_BCLK0, I2S_BCLK1,
+						    I2S_LRCLK1, I2S_MCLK1 };
+static unsigned int pcm1_in_out_sr_pads[]	= { PCM1_IN, PCM1_CLK,
+						    PCM1_SYNC, PCM1_OUT };
+/* sr1 */
+static unsigned int sd1_d3_d0_sr_pads[]		= { SD1_D3, SD1_D2,
+						    SD1_D1, SD1_D0 };
+static unsigned int sd0_sd1_clk_cmd_sr_pads[]	= { SD0_CLK, SD0_CMD,
+						    SD1_CLK, SD1_CMD };
+static unsigned int spi0_sclk_mosi_sr_pads[]	= { SPI0_SCLK, SPI0_MOSI };
+static unsigned int spi0_ss_miso_sr_pads[]	= { SPI0_SS, SPI0_MISO };
+static unsigned int uart0_rx_tx_sr_pads[]	= { UART0_RX, UART0_TX };
+static unsigned int uart4_rx_tx_sr_pads[]	= { UART4_RX, UART4_TX };
+static unsigned int uart2_sr_pads[]		= { UART2_RX, UART2_TX,
+						    UART2_RTSB, UART2_CTSB };
+static unsigned int uart3_sr_pads[]		= { UART3_RX, UART3_TX,
+						    UART3_RTSB, UART3_CTSB };
+/* sr2 */
+static unsigned int i2c0_sr_pads[]		= { I2C0_SCLK, I2C0_SDATA };
+static unsigned int i2c1_sr_pads[]		= { I2C1_SCLK, I2C1_SDATA };
+static unsigned int i2c2_sr_pads[]		= { I2C2_SCLK, I2C2_SDATA };
+static unsigned int sensor0_sr_pads[]		= { SENSOR0_PCLK,
+						    SENSOR0_CKOUT };
+
+#define MUX_PG(group_name, reg, shift, width)				\
+	{								\
+		.name = #group_name,					\
+		.pads = group_name##_pads,				\
+		.npads = ARRAY_SIZE(group_name##_pads),			\
+		.funcs = group_name##_funcs,				\
+		.nfuncs = ARRAY_SIZE(group_name##_funcs),		\
+		.mfpctl_reg  = MFCTL##reg,				\
+		.mfpctl_shift = shift,					\
+		.mfpctl_width = width,					\
+		.drv_reg = -1,						\
+		.drv_shift = -1,					\
+		.drv_width = -1,					\
+		.sr_reg = -1,						\
+		.sr_shift = -1,						\
+		.sr_width = -1,						\
+	}
+
+#define DRV_PG(group_name, reg, shift, width)				\
+	{								\
+		.name = #group_name,					\
+		.pads = group_name##_pads,				\
+		.npads = ARRAY_SIZE(group_name##_pads),			\
+		.mfpctl_reg  = -1,					\
+		.mfpctl_shift = -1,					\
+		.mfpctl_width = -1,					\
+		.drv_reg = PAD_DRV##reg,				\
+		.drv_shift = shift,					\
+		.drv_width = width,					\
+		.sr_reg = -1,						\
+		.sr_shift = -1,						\
+		.sr_width = -1,						\
+	}
+
+#define SR_PG(group_name, reg, shift, width)				\
+	{								\
+		.name = #group_name,					\
+		.pads = group_name##_pads,				\
+		.npads = ARRAY_SIZE(group_name##_pads),			\
+		.mfpctl_reg  = -1,					\
+		.mfpctl_shift = -1,					\
+		.mfpctl_width = -1,					\
+		.drv_reg = -1,						\
+		.drv_shift = -1,					\
+		.drv_width = -1,					\
+		.sr_reg = PAD_SR##reg,					\
+		.sr_shift = shift,					\
+		.sr_width = width,					\
+	}
+
+/* Pinctrl groups */
+static const struct owl_pingroup s900_groups[] = {
+	MUX_PG(lvds_oxx_uart4_mfp, 0, 22, 1),
+	MUX_PG(rmii_mdc_mfp, 0, 20, 2),
+	MUX_PG(rmii_mdio_mfp, 0, 20, 2),
+	MUX_PG(sirq0_mfp, 0, 19, 1),
+	MUX_PG(sirq1_mfp, 0, 19, 1),
+	MUX_PG(rmii_txd0_mfp, 0, 16, 3),
+	MUX_PG(rmii_txd1_mfp, 0, 16, 3),
+	MUX_PG(rmii_txen_mfp, 0, 13, 3),
+	MUX_PG(rmii_rxer_mfp, 0, 13, 3),
+	MUX_PG(rmii_crs_dv_mfp, 0, 11, 2),
+	MUX_PG(rmii_rxd1_mfp, 0, 8, 3),
+	MUX_PG(rmii_rxd0_mfp, 0, 8, 3),
+	MUX_PG(rmii_ref_clk_mfp, 0, 6, 2),
+	MUX_PG(i2s_d0_mfp, 0, 5, 1),
+	MUX_PG(i2s_d1_mfp, 0, 5, 1),
+	MUX_PG(i2s_lr_m_clk0_mfp, 0, 3, 2),
+	MUX_PG(i2s_bclk0_mfp, 0, 2, 1),
+	MUX_PG(i2s_bclk1_mclk1_mfp, 0, 2, 1),
+	MUX_PG(pcm1_in_out_mfp, 0, 0, 2),
+	MUX_PG(pcm1_clk_mfp, 0, 0, 2),
+	MUX_PG(pcm1_sync_mfp, 0, 0, 2),
+	MUX_PG(eram_a5_mfp, 1, 29, 3),
+	MUX_PG(eram_a6_mfp, 1, 29, 3),
+	MUX_PG(eram_a7_mfp, 1, 29, 3),
+	MUX_PG(eram_a8_mfp, 1, 26, 3),
+	MUX_PG(eram_a9_mfp, 1, 26, 3),
+	MUX_PG(eram_a10_mfp, 1, 26, 3),
+	MUX_PG(eram_a11_mfp, 1, 23, 3),
+	MUX_PG(lvds_oep_odn_mfp, 1, 22, 1),
+	MUX_PG(lvds_ocp_obn_mfp, 1, 22, 1),
+	MUX_PG(lvds_oap_oan_mfp, 1, 22, 1),
+	MUX_PG(lvds_e_mfp, 1, 21, 1),
+	MUX_PG(spi0_sclk_mosi_mfp, 1, 4, 2),
+	MUX_PG(spi0_ss_mfp, 1, 1, 3),
+	MUX_PG(spi0_miso_mfp, 1, 1, 3),
+	MUX_PG(uart2_rtsb_mfp, 2, 23, 1),
+	MUX_PG(uart2_ctsb_mfp, 2, 22, 1),
+	MUX_PG(uart3_rtsb_mfp, 2, 21, 1),
+	MUX_PG(uart3_ctsb_mfp, 2, 20, 1),
+	MUX_PG(sd0_d0_mfp, 2, 17, 3),
+	MUX_PG(sd0_d1_mfp, 2, 14, 3),
+	MUX_PG(sd0_d2_d3_mfp, 2, 11, 3),
+	MUX_PG(sd1_d0_d3_mfp, 2, 9, 2),
+	MUX_PG(sd0_cmd_mfp, 2, 7, 2),
+	MUX_PG(sd0_clk_mfp, 2, 5, 2),
+	MUX_PG(sd1_cmd_clk_mfp, 2, 3, 2),
+	MUX_PG(uart0_rx_mfp, 2, 0, 3),
+	MUX_PG(nand0_d0_ceb3_mfp, 3, 27, 1),
+	MUX_PG(uart0_tx_mfp, 3, 19, 3),
+	MUX_PG(i2c0_mfp, 3, 16, 3),
+	MUX_PG(csi0_cn_cp_mfp, 3, 15, 1),
+	MUX_PG(csi0_dn0_dp3_mfp, 3, 14, 1),
+	MUX_PG(csi1_dn0_cp_mfp, 3, 13, 1),
+	MUX_PG(dsi_dp3_dn1_mfp, 3, 12, 1),
+	MUX_PG(dsi_cp_dn0_mfp, 3, 12, 1),
+	MUX_PG(dsi_dp2_dn2_mfp, 3, 12, 1),
+	MUX_PG(nand1_d0_ceb1_mfp, 3, 11, 1),
+	MUX_PG(nand1_ceb3_mfp, 3, 10, 1),
+	MUX_PG(nand1_ceb0_mfp, 3, 10, 1),
+	MUX_PG(csi1_dn0_dp0_mfp, 3, 9, 1),
+	MUX_PG(uart4_rx_tx_mfp, 3, 8, 1),
+
+	DRV_PG(sgpio3_drv, 0, 30, 2),
+	DRV_PG(sgpio2_drv, 0, 28, 2),
+	DRV_PG(sgpio1_drv, 0, 26, 2),
+	DRV_PG(sgpio0_drv, 0, 24, 2),
+	DRV_PG(rmii_tx_d0_d1_drv, 0, 22, 2),
+	DRV_PG(rmii_txen_rxer_drv, 0, 20, 2),
+	DRV_PG(rmii_crs_dv_drv, 0, 18, 2),
+	DRV_PG(rmii_rx_d1_d0_drv, 0, 16, 2),
+	DRV_PG(rmii_ref_clk_drv, 0, 14, 2),
+	DRV_PG(rmii_mdc_mdio_drv, 0, 12, 2),
+	DRV_PG(sirq_0_1_drv, 0, 10, 2),
+	DRV_PG(sirq2_drv, 0, 8, 2),
+	DRV_PG(i2s_d0_d1_drv, 0, 6, 2),
+	DRV_PG(i2s_lr_m_clk0_drv, 0, 4, 2),
+	DRV_PG(i2s_blk1_mclk1_drv, 0, 2, 2),
+	DRV_PG(pcm1_in_out_drv, 0, 0, 2),
+	DRV_PG(lvds_oap_oan_drv, 1, 28, 2),
+	DRV_PG(lvds_oep_odn_drv, 1, 26, 2),
+	DRV_PG(lvds_ocp_obn_drv, 1, 24, 2),
+	DRV_PG(lvds_e_drv, 1, 22, 2),
+	DRV_PG(sd0_d3_d0_drv, 1, 20, 2),
+	DRV_PG(sd1_d3_d0_drv, 1, 18, 2),
+	DRV_PG(sd0_sd1_cmd_clk_drv, 1, 16, 2),
+	DRV_PG(spi0_sclk_mosi_drv, 1, 14, 2),
+	DRV_PG(spi0_ss_miso_drv, 1, 12, 2),
+	DRV_PG(uart0_rx_tx_drv, 1, 10, 2),
+	DRV_PG(uart4_rx_tx_drv, 1, 8, 2),
+	DRV_PG(uart2_drv, 1, 6, 2),
+	DRV_PG(uart3_drv, 1, 4, 2),
+	DRV_PG(i2c0_drv, 2, 30, 2),
+	DRV_PG(i2c1_drv, 2, 28, 2),
+	DRV_PG(i2c2_drv, 2, 26, 2),
+	DRV_PG(sensor0_drv, 2, 20, 2),
+
+	SR_PG(sgpio3_sr, 0, 15, 1),
+	SR_PG(sgpio2_sr, 0, 14, 1),
+	SR_PG(sgpio1_sr, 0, 13, 1),
+	SR_PG(sgpio0_sr, 0, 12, 1),
+	SR_PG(rmii_tx_d0_d1_sr, 0, 11, 1),
+	SR_PG(rmii_txen_rxer_sr, 0, 10, 1),
+	SR_PG(rmii_crs_dv_sr, 0, 9, 1),
+	SR_PG(rmii_rx_d1_d0_sr, 0, 8, 1),
+	SR_PG(rmii_ref_clk_sr, 0, 7, 1),
+	SR_PG(rmii_mdc_mdio_sr, 0, 6, 1),
+	SR_PG(sirq_0_1_sr, 0, 5, 1),
+	SR_PG(sirq2_sr, 0, 4, 1),
+	SR_PG(i2s_do_d1_sr, 0, 3, 1),
+	SR_PG(i2s_lr_m_clk0_sr, 0, 2, 1),
+	SR_PG(i2s_bclk0_mclk1_sr, 0, 1, 1),
+	SR_PG(pcm1_in_out_sr, 0, 0, 1),
+	SR_PG(sd1_d3_d0_sr, 1, 25, 1),
+	SR_PG(sd0_sd1_clk_cmd_sr, 1, 24, 1),
+	SR_PG(spi0_sclk_mosi_sr, 1, 23, 1),
+	SR_PG(spi0_ss_miso_sr, 1, 22, 1),
+	SR_PG(uart0_rx_tx_sr, 1, 21, 1),
+	SR_PG(uart4_rx_tx_sr, 1, 20, 1),
+	SR_PG(uart2_sr, 1, 19, 1),
+	SR_PG(uart3_sr, 1, 18, 1),
+	SR_PG(i2c0_sr, 2, 31, 1),
+	SR_PG(i2c1_sr, 2, 30, 1),
+	SR_PG(i2c2_sr, 2, 29, 1),
+	SR_PG(sensor0_sr, 2, 25, 1)
+};
+
+static const char * const eram_groups[] = {
+	"lvds_oxx_uart4_mfp",
+	"eram_a5_mfp",
+	"eram_a6_mfp",
+	"eram_a7_mfp",
+	"eram_a8_mfp",
+	"eram_a9_mfp",
+	"eram_a10_mfp",
+	"eram_a11_mfp",
+	"lvds_oap_oan_mfp",
+	"lvds_e_mfp",
+	"spi0_sclk_mosi_mfp",
+	"spi0_ss_mfp",
+	"spi0_miso_mfp",
+	"sd0_d0_mfp",
+	"sd0_d1_mfp",
+	"sd0_d2_d3_mfp",
+	"sd1_d0_d3_mfp",
+	"sd0_cmd_mfp",
+	"sd0_clk_mfp",
+	"sd1_cmd_clk_mfp",
+};
+
+static const char * const eth_rmii_groups[] = {
+	"rmii_mdc_mfp",
+	"rmii_mdio_mfp",
+	"rmii_txd0_mfp",
+	"rmii_txd1_mfp",
+	"rmii_txen_mfp",
+	"rmii_rxer_mfp",
+	"rmii_crs_dv_mfp",
+	"rmii_rxd1_mfp",
+	"rmii_rxd0_mfp",
+	"rmii_ref_clk_mfp",
+	"eth_smi_dummy",
+};
+
+static const char * const eth_smii_groups[] = {
+	"rmii_txd0_mfp",
+	"rmii_txd1_mfp",
+	"rmii_crs_dv_mfp",
+	"eth_smi_dummy",
+};
+
+static const char * const spi0_groups[] = {
+	"spi0_sclk_mosi_mfp",
+	"spi0_ss_mfp",
+	"spi0_miso_mfp",
+	"spi0_sclk_mosi_mfp",
+	"spi0_ss_mfp",
+	"spi0_miso_mfp",
+};
+
+static const char * const spi1_groups[] = {
+	"pcm1_in_out_mfp",
+	"pcm1_clk_mfp",
+	"pcm1_sync_mfp",
+	"uart0_rx_mfp",
+	"uart0_tx_mfp",
+	"i2c0_mfp",
+};
+
+static const char * const spi2_groups[] = {
+	"rmii_txd0_mfp",
+	"rmii_txd1_mfp",
+	"rmii_crs_dv_mfp",
+	"rmii_ref_clk_mfp",
+};
+
+static const char * const spi3_groups[] = {
+	"rmii_txen_mfp",
+	"rmii_rxer_mfp",
+};
+
+static const char * const sens0_groups[] = {
+	"rmii_txd0_mfp",
+	"rmii_txd1_mfp",
+	"rmii_txen_mfp",
+	"rmii_rxer_mfp",
+	"rmii_rxd1_mfp",
+	"rmii_rxd0_mfp",
+	"eram_a5_mfp",
+	"eram_a6_mfp",
+	"eram_a7_mfp",
+	"eram_a8_mfp",
+	"eram_a9_mfp",
+	"csi0_cn_cp_mfp",
+	"csi0_dn0_dp3_mfp",
+	"csi1_dn0_cp_mfp",
+	"csi1_dn0_dp0_mfp",
+};
+
+static const char * const uart0_groups[] = {
+	"uart2_rtsb_mfp",
+	"uart2_ctsb_mfp",
+	"uart0_rx_mfp",
+	"uart0_tx_mfp",
+};
+
+static const char * const uart1_groups[] = {
+	"sd0_d2_d3_mfp",
+	"i2c0_mfp",
+};
+
+static const char * const uart2_groups[] = {
+	"rmii_mdc_mfp",
+	"rmii_mdio_mfp",
+	"rmii_txen_mfp",
+	"rmii_rxer_mfp",
+	"rmii_rxd1_mfp",
+	"rmii_rxd0_mfp",
+	"lvds_oep_odn_mfp",
+	"uart2_rtsb_mfp",
+	"uart2_ctsb_mfp",
+	"sd0_d0_mfp",
+	"sd0_d1_mfp",
+	"sd0_d2_d3_mfp",
+	"uart0_rx_mfp",
+	"uart0_tx_mfp_pads",
+	"i2c0_mfp_pads",
+	"dsi_dp3_dn1_mfp",
+	"uart2_dummy"
+};
+
+static const char * const uart3_groups[] = {
+	"uart3_rtsb_mfp",
+	"uart3_ctsb_mfp",
+	"uart3_dummy"
+};
+
+static const char * const uart4_groups[] = {
+	"lvds_oxx_uart4_mfp",
+	"rmii_crs_dv_mfp",
+	"rmii_ref_clk_mfp",
+	"pcm1_in_out_mfp",
+	"pcm1_clk_mfp",
+	"pcm1_sync_mfp",
+	"eram_a5_mfp",
+	"eram_a6_mfp",
+	"dsi_dp2_dn2_mfp",
+	"uart4_rx_tx_mfp_pads",
+	"uart4_dummy"
+};
+
+static const char * const uart5_groups[] = {
+	"rmii_rxd1_mfp",
+	"rmii_rxd0_mfp",
+	"eram_a9_mfp",
+	"eram_a11_mfp",
+	"uart3_rtsb_mfp",
+	"uart3_ctsb_mfp",
+	"sd0_d0_mfp",
+	"sd0_d1_mfp",
+};
+
+static const char * const uart6_groups[] = {
+	"rmii_txd0_mfp",
+	"rmii_txd1_mfp",
+};
+
+static const char * const i2s0_groups[] = {
+	"i2s_d0_mfp",
+	"i2s_lr_m_clk0_mfp",
+	"i2s_bclk0_mfp",
+	"i2s0_dummy",
+};
+
+static const char * const i2s1_groups[] = {
+	"i2s_d1_mfp",
+	"i2s_bclk1_mclk1_mfp",
+	"spi0_ss_mfp",
+	"spi0_miso_mfp",
+	"uart0_rx_mfp",
+	"uart0_tx_mfp",
+	"i2s1_dummy",
+};
+
+static const char * const pcm0_groups[] = {
+	"i2s_d0_mfp",
+	"i2s_d1_mfp",
+	"i2s_lr_m_clk0_mfp",
+	"i2s_bclk0_mfp",
+	"i2s_bclk1_mclk1_mfp",
+	"spi0_sclk_mosi_mfp",
+	"spi0_ss_mfp",
+	"spi0_miso_mfp",
+};
+
+static const char * const pcm1_groups[] = {
+	"i2s_lr_m_clk0_mfp",
+	"pcm1_in_out_mfp",
+	"pcm1_clk_mfp",
+	"pcm1_sync_mfp",
+	"lvds_oep_odn_mfp",
+	"spi0_ss_mfp",
+	"spi0_miso_mfp",
+	"uart0_rx_mfp",
+	"uart0_tx_mfp",
+	"dsi_cp_dn0_mfp",
+	"pcm1_dummy",
+};
+
+static const char * const jtag_groups[] = {
+	"eram_a5_mfp",
+	"eram_a6_mfp",
+	"eram_a7_mfp",
+	"eram_a8_mfp",
+	"eram_a10_mfp",
+	"eram_a10_mfp",
+	"sd0_d2_d3_mfp",
+	"sd0_cmd_mfp",
+	"sd0_clk_mfp",
+};
+
+static const char * const pwm0_groups[] = {
+	"sirq0_mfp",
+	"rmii_txd0_mfp",
+	"rmii_rxd1_mfp",
+	"eram_a5_mfp",
+	"nand1_ceb3_mfp",
+};
+
+static const char * const pwm1_groups[] = {
+	"sirq1_mfp",
+	"rmii_txd1_mfp",
+	"rmii_rxd0_mfp",
+	"eram_a6_mfp",
+	"eram_a8_mfp",
+	"nand1_ceb0_mfp",
+};
+
+static const char * const pwm2_groups[] = {
+	"rmii_mdc_mfp",
+	"rmii_txen_mfp",
+	"eram_a9_mfp",
+	"eram_a11_mfp",
+};
+
+static const char * const pwm3_groups[] = {
+	"rmii_mdio_mfp",
+	"rmii_rxer_mfp",
+	"eram_a10_mfp",
+};
+
+static const char * const pwm4_groups[] = {
+	"pcm1_clk_mfp",
+	"spi0_ss_mfp",
+};
+
+static const char * const pwm5_groups[] = {
+	"pcm1_sync_mfp",
+	"spi0_miso_mfp",
+};
+
+static const char * const sd0_groups[] = {
+	"sd0_d0_mfp",
+	"sd0_d1_mfp",
+	"sd0_d2_d3_mfp",
+	"sd0_cmd_mfp",
+	"sd0_clk_mfp",
+};
+
+static const char * const sd1_groups[] = {
+	"sd1_d0_d3_mfp",
+	"sd1_cmd_clk_mfp",
+	"sd1_dummy",
+};
+
+static const char * const sd2_groups[] = {
+	"nand0_d0_ceb3_mfp",
+};
+
+static const char * const sd3_groups[] = {
+	"nand1_d0_ceb1_mfp",
+};
+
+static const char * const i2c0_groups[] = {
+	"i2c0_mfp",
+};
+
+static const char * const i2c1_groups[] = {
+	"i2c0_mfp",
+	"i2c1_dummy"
+};
+
+static const char * const i2c2_groups[] = {
+	"i2c2_dummy"
+};
+
+static const char * const i2c3_groups[] = {
+	"pcm1_in_out_mfp",
+	"spi0_sclk_mosi_mfp",
+};
+
+static const char * const i2c4_groups[] = {
+	"uart4_rx_tx_mfp",
+};
+
+static const char * const i2c5_groups[] = {
+	"uart0_rx_mfp",
+	"uart0_tx_mfp",
+};
+
+
+static const char * const lvds_groups[] = {
+	"lvds_oep_odn_mfp",
+	"lvds_ocp_obn_mfp",
+	"lvds_oap_oan_mfp",
+	"lvds_e_mfp",
+};
+
+static const char * const usb20_groups[] = {
+	"eram_a9_mfp",
+};
+
+static const char * const usb30_groups[] = {
+	"eram_a10_mfp",
+};
+
+static const char * const gpu_groups[] = {
+	"sd0_d0_mfp",
+	"sd0_d1_mfp",
+	"sd0_d2_d3_mfp",
+	"sd0_cmd_mfp",
+	"sd0_clk_mfp",
+};
+
+static const char * const mipi_csi0_groups[] = {
+	"csi0_dn0_dp3_mfp",
+};
+
+static const char * const mipi_csi1_groups[] = {
+	"csi1_dn0_cp_mfp",
+};
+
+static const char * const mipi_dsi_groups[] = {
+	"dsi_dp3_dn1_mfp",
+	"dsi_cp_dn0_mfp",
+	"dsi_dp2_dn2_mfp",
+	"mipi_dsi_dummy",
+};
+
+static const char * const nand0_groups[] = {
+	"nand0_d0_ceb3_mfp",
+	"nand0_dummy",
+};
+
+static const char * const nand1_groups[] = {
+	"nand1_d0_ceb1_mfp",
+	"nand1_ceb3_mfp",
+	"nand1_ceb0_mfp",
+	"nand1_dummy",
+};
+
+static const char * const spdif_groups[] = {
+	"uart0_tx_mfp",
+};
+
+static const char * const sirq0_groups[] = {
+	"sirq0_mfp",
+	"sirq0_dummy",
+};
+
+static const char * const sirq1_groups[] = {
+	"sirq1_mfp",
+	"sirq1_dummy",
+};
+
+static const char * const sirq2_groups[] = {
+	"sirq2_dummy",
+};
+
+#define FUNCTION(fname)					\
+	{						\
+		.name = #fname,				\
+		.groups = fname##_groups,		\
+		.ngroups = ARRAY_SIZE(fname##_groups),	\
+	}
+
+static const struct owl_pinmux_func s900_functions[] = {
+	[S900_MUX_ERAM] = FUNCTION(eram),
+	[S900_MUX_ETH_RMII] = FUNCTION(eth_rmii),
+	[S900_MUX_ETH_SMII] = FUNCTION(eth_smii),
+	[S900_MUX_SPI0] = FUNCTION(spi0),
+	[S900_MUX_SPI1] = FUNCTION(spi1),
+	[S900_MUX_SPI2] = FUNCTION(spi2),
+	[S900_MUX_SPI3] = FUNCTION(spi3),
+	[S900_MUX_SENS0] = FUNCTION(sens0),
+	[S900_MUX_UART0] = FUNCTION(uart0),
+	[S900_MUX_UART1] = FUNCTION(uart1),
+	[S900_MUX_UART2] = FUNCTION(uart2),
+	[S900_MUX_UART3] = FUNCTION(uart3),
+	[S900_MUX_UART4] = FUNCTION(uart4),
+	[S900_MUX_UART5] = FUNCTION(uart5),
+	[S900_MUX_UART6] = FUNCTION(uart6),
+	[S900_MUX_I2S0] = FUNCTION(i2s0),
+	[S900_MUX_I2S1] = FUNCTION(i2s1),
+	[S900_MUX_PCM0] = FUNCTION(pcm0),
+	[S900_MUX_PCM1] = FUNCTION(pcm1),
+	[S900_MUX_JTAG] = FUNCTION(jtag),
+	[S900_MUX_PWM0] = FUNCTION(pwm0),
+	[S900_MUX_PWM1] = FUNCTION(pwm1),
+	[S900_MUX_PWM2] = FUNCTION(pwm2),
+	[S900_MUX_PWM3] = FUNCTION(pwm3),
+	[S900_MUX_PWM4] = FUNCTION(pwm4),
+	[S900_MUX_PWM5] = FUNCTION(pwm5),
+	[S900_MUX_SD0] = FUNCTION(sd0),
+	[S900_MUX_SD1] = FUNCTION(sd1),
+	[S900_MUX_SD2] = FUNCTION(sd2),
+	[S900_MUX_SD3] = FUNCTION(sd3),
+	[S900_MUX_I2C0] = FUNCTION(i2c0),
+	[S900_MUX_I2C1] = FUNCTION(i2c1),
+	[S900_MUX_I2C2] = FUNCTION(i2c2),
+	[S900_MUX_I2C3] = FUNCTION(i2c3),
+	[S900_MUX_I2C4] = FUNCTION(i2c4),
+	[S900_MUX_I2C5] = FUNCTION(i2c5),
+	[S900_MUX_LVDS] = FUNCTION(lvds),
+	[S900_MUX_USB30] = FUNCTION(usb30),
+	[S900_MUX_USB20] = FUNCTION(usb20),
+	[S900_MUX_GPU] = FUNCTION(gpu),
+	[S900_MUX_MIPI_CSI0] = FUNCTION(mipi_csi0),
+	[S900_MUX_MIPI_CSI1] = FUNCTION(mipi_csi1),
+	[S900_MUX_MIPI_DSI] = FUNCTION(mipi_dsi),
+	[S900_MUX_NAND0] = FUNCTION(nand0),
+	[S900_MUX_NAND1] = FUNCTION(nand1),
+	[S900_MUX_SPDIF] = FUNCTION(spdif),
+	[S900_MUX_SIRQ0] = FUNCTION(sirq0),
+	[S900_MUX_SIRQ1] = FUNCTION(sirq1),
+	[S900_MUX_SIRQ2] = FUNCTION(sirq2)
+};
+/* PAD PULL UP/DOWN CONFIGURES */
+#define PULLCTL_CONF(pull_reg, pull_sft, pull_wdt)			\
+	{								\
+		.reg = PAD_PULLCTL##pull_reg,				\
+		.shift = pull_sft,					\
+		.width = pull_wdt,					\
+	}
+
+#define PAD_PULLCTL_CONF(pad_name, pull_reg, pull_sft, pull_wdt)	\
+	struct owl_pullctl pad_name##_pullctl_conf			\
+		= PULLCTL_CONF(pull_reg, pull_sft, pull_wdt)
+
+#define ST_CONF(st_reg, st_sft, st_wdt)					\
+	{								\
+		.reg = PAD_ST##st_reg,					\
+		.shift = st_sft,					\
+		.width = st_wdt,					\
+	}
+
+#define PAD_ST_CONF(pad_name, st_reg, st_sft, st_wdt)			\
+	struct owl_st pad_name##_st_conf				\
+		= ST_CONF(st_reg, st_sft, st_wdt)
+
+/* PAD_PULLCTL0 */
+static PAD_PULLCTL_CONF(ETH_RXER, 0, 18, 2);
+static PAD_PULLCTL_CONF(SIRQ0, 0, 16, 2);
+static PAD_PULLCTL_CONF(SIRQ1, 0, 14, 2);
+static PAD_PULLCTL_CONF(SIRQ2, 0, 12, 2);
+static PAD_PULLCTL_CONF(I2C0_SDATA, 0, 10, 2);
+static PAD_PULLCTL_CONF(I2C0_SCLK, 0, 8, 2);
+static PAD_PULLCTL_CONF(ERAM_A5, 0, 6, 2);
+static PAD_PULLCTL_CONF(ERAM_A6, 0, 4, 2);
+static PAD_PULLCTL_CONF(ERAM_A7, 0, 2, 2);
+static PAD_PULLCTL_CONF(ERAM_A10, 0, 0, 2);
+
+/* PAD_PULLCTL1 */
+static PAD_PULLCTL_CONF(PCM1_IN, 1, 30, 2);
+static PAD_PULLCTL_CONF(PCM1_OUT, 1, 28, 2);
+static PAD_PULLCTL_CONF(SD0_D0, 1, 26, 2);
+static PAD_PULLCTL_CONF(SD0_D1, 1, 24, 2);
+static PAD_PULLCTL_CONF(SD0_D2, 1, 22, 2);
+static PAD_PULLCTL_CONF(SD0_D3, 1, 20, 2);
+static PAD_PULLCTL_CONF(SD0_CMD, 1, 18, 2);
+static PAD_PULLCTL_CONF(SD0_CLK, 1, 16, 2);
+static PAD_PULLCTL_CONF(SD1_CMD, 1, 14, 2);
+static PAD_PULLCTL_CONF(SD1_D0, 1, 12, 2);
+static PAD_PULLCTL_CONF(SD1_D1, 1, 10, 2);
+static PAD_PULLCTL_CONF(SD1_D2, 1, 8, 2);
+static PAD_PULLCTL_CONF(SD1_D3, 1, 6, 2);
+static PAD_PULLCTL_CONF(UART0_RX, 1, 4, 2);
+static PAD_PULLCTL_CONF(UART0_TX, 1, 2, 2);
+
+/* PAD_PULLCTL2 */
+static PAD_PULLCTL_CONF(I2C2_SDATA, 2, 26, 2);
+static PAD_PULLCTL_CONF(I2C2_SCLK, 2, 24, 2);
+static PAD_PULLCTL_CONF(SPI0_SCLK, 2, 22, 2);
+static PAD_PULLCTL_CONF(SPI0_MOSI, 2, 20, 2);
+static PAD_PULLCTL_CONF(I2C1_SDATA, 2, 18, 2);
+static PAD_PULLCTL_CONF(I2C1_SCLK, 2, 16, 2);
+static PAD_PULLCTL_CONF(NAND0_D0, 2, 15, 1);
+static PAD_PULLCTL_CONF(NAND0_D1, 2, 15, 1);
+static PAD_PULLCTL_CONF(NAND0_D2, 2, 15, 1);
+static PAD_PULLCTL_CONF(NAND0_D3, 2, 15, 1);
+static PAD_PULLCTL_CONF(NAND0_D4, 2, 15, 1);
+static PAD_PULLCTL_CONF(NAND0_D5, 2, 15, 1);
+static PAD_PULLCTL_CONF(NAND0_D6, 2, 15, 1);
+static PAD_PULLCTL_CONF(NAND0_D7, 2, 15, 1);
+static PAD_PULLCTL_CONF(NAND0_DQSN, 2, 14, 1);
+static PAD_PULLCTL_CONF(NAND0_DQS, 2, 13, 1);
+static PAD_PULLCTL_CONF(NAND1_D0, 2, 12, 1);
+static PAD_PULLCTL_CONF(NAND1_D1, 2, 12, 1);
+static PAD_PULLCTL_CONF(NAND1_D2, 2, 12, 1);
+static PAD_PULLCTL_CONF(NAND1_D3, 2, 12, 1);
+static PAD_PULLCTL_CONF(NAND1_D4, 2, 12, 1);
+static PAD_PULLCTL_CONF(NAND1_D5, 2, 12, 1);
+static PAD_PULLCTL_CONF(NAND1_D6, 2, 12, 1);
+static PAD_PULLCTL_CONF(NAND1_D7, 2, 12, 1);
+static PAD_PULLCTL_CONF(NAND1_DQSN, 2, 11, 1);
+static PAD_PULLCTL_CONF(NAND1_DQS, 2, 10, 1);
+static PAD_PULLCTL_CONF(SGPIO2, 2, 8, 2);
+static PAD_PULLCTL_CONF(SGPIO3, 2, 6, 2);
+static PAD_PULLCTL_CONF(UART4_RX, 2, 4, 2);
+static PAD_PULLCTL_CONF(UART4_TX, 2, 2, 2);
+
+/* PAD_ST0 */
+static PAD_ST_CONF(I2C0_SDATA, 0, 30, 1);
+static PAD_ST_CONF(UART0_RX, 0, 29, 1);
+static PAD_ST_CONF(ETH_MDC, 0, 28, 1);
+static PAD_ST_CONF(I2S_MCLK1, 0, 23, 1);
+static PAD_ST_CONF(ETH_REF_CLK, 0, 22, 1);
+static PAD_ST_CONF(ETH_TXEN, 0, 21, 1);
+static PAD_ST_CONF(ETH_TXD0, 0, 20, 1);
+static PAD_ST_CONF(I2S_LRCLK1, 0, 19, 1);
+static PAD_ST_CONF(SGPIO2, 0, 18, 1);
+static PAD_ST_CONF(SGPIO3, 0, 17, 1);
+static PAD_ST_CONF(UART4_TX, 0, 16, 1);
+static PAD_ST_CONF(I2S_D1, 0, 15, 1);
+static PAD_ST_CONF(UART0_TX, 0, 14, 1);
+static PAD_ST_CONF(SPI0_SCLK, 0, 13, 1);
+static PAD_ST_CONF(SD0_CLK, 0, 12, 1);
+static PAD_ST_CONF(ERAM_A5, 0, 11, 1);
+static PAD_ST_CONF(I2C0_SCLK, 0, 7, 1);
+static PAD_ST_CONF(ERAM_A9, 0, 6, 1);
+static PAD_ST_CONF(LVDS_OEP, 0, 5, 1);
+static PAD_ST_CONF(LVDS_ODN, 0, 4, 1);
+static PAD_ST_CONF(LVDS_OAP, 0, 3, 1);
+static PAD_ST_CONF(I2S_BCLK1, 0, 2, 1);
+
+/* PAD_ST1 */
+static PAD_ST_CONF(I2S_LRCLK0, 1, 29, 1);
+static PAD_ST_CONF(UART4_RX, 1, 28, 1);
+static PAD_ST_CONF(UART3_CTSB, 1, 27, 1);
+static PAD_ST_CONF(UART3_RTSB, 1, 26, 1);
+static PAD_ST_CONF(UART3_RX, 1, 25, 1);
+static PAD_ST_CONF(UART2_RTSB, 1, 24, 1);
+static PAD_ST_CONF(UART2_CTSB, 1, 23, 1);
+static PAD_ST_CONF(UART2_RX, 1, 22, 1);
+static PAD_ST_CONF(ETH_RXD0, 1, 21, 1);
+static PAD_ST_CONF(ETH_RXD1, 1, 20, 1);
+static PAD_ST_CONF(ETH_CRS_DV, 1, 19, 1);
+static PAD_ST_CONF(ETH_RXER, 1, 18, 1);
+static PAD_ST_CONF(ETH_TXD1, 1, 17, 1);
+static PAD_ST_CONF(LVDS_OCP, 1, 16, 1);
+static PAD_ST_CONF(LVDS_OBP, 1, 15, 1);
+static PAD_ST_CONF(LVDS_OBN, 1, 14, 1);
+static PAD_ST_CONF(PCM1_OUT, 1, 12, 1);
+static PAD_ST_CONF(PCM1_CLK, 1, 11, 1);
+static PAD_ST_CONF(PCM1_IN, 1, 10, 1);
+static PAD_ST_CONF(PCM1_SYNC, 1, 9, 1);
+static PAD_ST_CONF(I2C1_SCLK, 1, 8, 1);
+static PAD_ST_CONF(I2C1_SDATA, 1, 7, 1);
+static PAD_ST_CONF(I2C2_SCLK, 1, 6, 1);
+static PAD_ST_CONF(I2C2_SDATA, 1, 5, 1);
+static PAD_ST_CONF(SPI0_MOSI, 1, 4, 1);
+static PAD_ST_CONF(SPI0_MISO, 1, 3, 1);
+static PAD_ST_CONF(SPI0_SS, 1, 2, 1);
+static PAD_ST_CONF(I2S_BCLK0, 1, 1, 1);
+static PAD_ST_CONF(I2S_MCLK0, 1, 0, 1);
+
+#define PAD_INFO(name)							\
+	{								\
+		.pad = name,						\
+		.pullctl = NULL,					\
+		.st = NULL,						\
+	}
+
+#define PAD_INFO_ST(name)						\
+	{								\
+		.pad = name,						\
+		.pullctl = NULL,					\
+		.st = &name##_st_conf,					\
+	}
+
+#define PAD_INFO_PULLCTL(name)						\
+	{								\
+		.pad = name,						\
+		.pullctl = &name##_pullctl_conf,			\
+		.st = NULL,						\
+	}
+
+#define PAD_INFO_PULLCTL_ST(name)					\
+	{								\
+		.pad = name,						\
+		.pullctl = &name##_pullctl_conf,			\
+		.st = &name##_st_conf,					\
+	}
+
+/* Pad info table */
+static struct owl_padinfo s900_padinfo[NUM_PADS] = {
+	[ETH_TXD0] = PAD_INFO_ST(ETH_TXD0),
+	[ETH_TXD1] = PAD_INFO_ST(ETH_TXD1),
+	[ETH_TXEN] = PAD_INFO_ST(ETH_TXEN),
+	[ETH_RXER] = PAD_INFO_PULLCTL_ST(ETH_RXER),
+	[ETH_CRS_DV] = PAD_INFO_ST(ETH_CRS_DV),
+	[ETH_RXD1] = PAD_INFO_ST(ETH_RXD1),
+	[ETH_RXD0] = PAD_INFO_ST(ETH_RXD0),
+	[ETH_REF_CLK] = PAD_INFO_ST(ETH_REF_CLK),
+	[ETH_MDC] = PAD_INFO_ST(ETH_MDC),
+	[ETH_MDIO] = PAD_INFO(ETH_MDIO),
+	[SIRQ0] = PAD_INFO_PULLCTL(SIRQ0),
+	[SIRQ1] = PAD_INFO_PULLCTL(SIRQ1),
+	[SIRQ2] = PAD_INFO_PULLCTL(SIRQ2),
+	[I2S_D0] = PAD_INFO(I2S_D0),
+	[I2S_BCLK0] = PAD_INFO_ST(I2S_BCLK0),
+	[I2S_LRCLK0] = PAD_INFO_ST(I2S_LRCLK0),
+	[I2S_MCLK0] = PAD_INFO_ST(I2S_MCLK0),
+	[I2S_D1] = PAD_INFO_ST(I2S_D1),
+	[I2S_BCLK1] = PAD_INFO_ST(I2S_BCLK1),
+	[I2S_LRCLK1] = PAD_INFO_ST(I2S_LRCLK1),
+	[I2S_MCLK1] = PAD_INFO_ST(I2S_MCLK1),
+	[PCM1_IN] = PAD_INFO_PULLCTL_ST(PCM1_IN),
+	[PCM1_CLK] = PAD_INFO_ST(PCM1_CLK),
+	[PCM1_SYNC] = PAD_INFO_ST(PCM1_SYNC),
+	[PCM1_OUT] = PAD_INFO_PULLCTL_ST(PCM1_OUT),
+	[ERAM_A5] = PAD_INFO_PULLCTL_ST(ERAM_A5),
+	[ERAM_A6] = PAD_INFO_PULLCTL(ERAM_A6),
+	[ERAM_A7] = PAD_INFO_PULLCTL(ERAM_A7),
+	[ERAM_A8] = PAD_INFO(ERAM_A8),
+	[ERAM_A9] = PAD_INFO_ST(ERAM_A9),
+	[ERAM_A10] = PAD_INFO_PULLCTL(ERAM_A10),
+	[ERAM_A11] = PAD_INFO(ERAM_A11),
+	[LVDS_OEP] = PAD_INFO_ST(LVDS_OEP),
+	[LVDS_OEN] = PAD_INFO(LVDS_OEN),
+	[LVDS_ODP] = PAD_INFO(LVDS_ODP),
+	[LVDS_ODN] = PAD_INFO_ST(LVDS_ODN),
+	[LVDS_OCP] = PAD_INFO_ST(LVDS_OCP),
+	[LVDS_OCN] = PAD_INFO(LVDS_OCN),
+	[LVDS_OBP] = PAD_INFO_ST(LVDS_OBP),
+	[LVDS_OBN] = PAD_INFO_ST(LVDS_OBN),
+	[LVDS_OAP] = PAD_INFO_ST(LVDS_OAP),
+	[LVDS_OAN] = PAD_INFO(LVDS_OAN),
+	[LVDS_EEP] = PAD_INFO(LVDS_EEP),
+	[LVDS_EEN] = PAD_INFO(LVDS_EEN),
+	[LVDS_EDP] = PAD_INFO(LVDS_EDP),
+	[LVDS_EDN] = PAD_INFO(LVDS_EDN),
+	[LVDS_ECP] = PAD_INFO(LVDS_ECP),
+	[LVDS_ECN] = PAD_INFO(LVDS_ECN),
+	[LVDS_EBP] = PAD_INFO(LVDS_EBP),
+	[LVDS_EBN] = PAD_INFO(LVDS_EBN),
+	[LVDS_EAP] = PAD_INFO(LVDS_EAP),
+	[LVDS_EAN] = PAD_INFO(LVDS_EAN),
+	[SD0_D0] = PAD_INFO_PULLCTL(SD0_D0),
+	[SD0_D1] = PAD_INFO_PULLCTL(SD0_D1),
+	[SD0_D2] = PAD_INFO_PULLCTL(SD0_D2),
+	[SD0_D3] = PAD_INFO_PULLCTL(SD0_D3),
+	[SD1_D0] = PAD_INFO_PULLCTL(SD1_D0),
+	[SD1_D1] = PAD_INFO_PULLCTL(SD1_D1),
+	[SD1_D2] = PAD_INFO_PULLCTL(SD1_D2),
+	[SD1_D3] = PAD_INFO_PULLCTL(SD1_D3),
+	[SD0_CMD] = PAD_INFO_PULLCTL(SD0_CMD),
+	[SD0_CLK] = PAD_INFO_PULLCTL_ST(SD0_CLK),
+	[SD1_CMD] = PAD_INFO_PULLCTL(SD1_CMD),
+	[SD1_CLK] = PAD_INFO(SD1_CLK),
+	[SPI0_SCLK] = PAD_INFO_PULLCTL_ST(SPI0_SCLK),
+	[SPI0_SS] = PAD_INFO_ST(SPI0_SS),
+	[SPI0_MISO] = PAD_INFO_ST(SPI0_MISO),
+	[SPI0_MOSI] = PAD_INFO_PULLCTL_ST(SPI0_MOSI),
+	[UART0_RX] = PAD_INFO_PULLCTL_ST(UART0_RX),
+	[UART0_TX] = PAD_INFO_PULLCTL_ST(UART0_TX),
+	[UART2_RX] = PAD_INFO_ST(UART2_RX),
+	[UART2_TX] = PAD_INFO(UART2_TX),
+	[UART2_RTSB] = PAD_INFO_ST(UART2_RTSB),
+	[UART2_CTSB] = PAD_INFO_ST(UART2_CTSB),
+	[UART3_RX] = PAD_INFO_ST(UART3_RX),
+	[UART3_TX] = PAD_INFO(UART3_TX),
+	[UART3_RTSB] = PAD_INFO_ST(UART3_RTSB),
+	[UART3_CTSB] = PAD_INFO_ST(UART3_CTSB),
+	[UART4_RX] = PAD_INFO_PULLCTL_ST(UART4_RX),
+	[UART4_TX] = PAD_INFO_PULLCTL_ST(UART4_TX),
+	[I2C0_SCLK] = PAD_INFO_PULLCTL_ST(I2C0_SCLK),
+	[I2C0_SDATA] = PAD_INFO_PULLCTL_ST(I2C0_SDATA),
+	[I2C1_SCLK] = PAD_INFO_PULLCTL_ST(I2C1_SCLK),
+	[I2C1_SDATA] = PAD_INFO_PULLCTL_ST(I2C1_SDATA),
+	[I2C2_SCLK] = PAD_INFO_PULLCTL_ST(I2C2_SCLK),
+	[I2C2_SDATA] = PAD_INFO_PULLCTL_ST(I2C2_SDATA),
+	[CSI0_DN0] = PAD_INFO(CSI0_DN0),
+	[CSI0_DP0] = PAD_INFO(CSI0_DP0),
+	[CSI0_DN1] = PAD_INFO(CSI0_DN1),
+	[CSI0_DP1] = PAD_INFO(CSI0_DP1),
+	[CSI0_CN] = PAD_INFO(CSI0_CN),
+	[CSI0_CP] = PAD_INFO(CSI0_CP),
+	[CSI0_DN2] = PAD_INFO(CSI0_DN2),
+	[CSI0_DP2] = PAD_INFO(CSI0_DP2),
+	[CSI0_DN3] = PAD_INFO(CSI0_DN3),
+	[CSI0_DP3] = PAD_INFO(CSI0_DP3),
+	[DSI_DP3] = PAD_INFO(DSI_DP3),
+	[DSI_DN3] = PAD_INFO(DSI_DN3),
+	[DSI_DP1] = PAD_INFO(DSI_DP1),
+	[DSI_DN1] = PAD_INFO(DSI_DN1),
+	[DSI_CP] = PAD_INFO(DSI_CP),
+	[DSI_CN] = PAD_INFO(DSI_CN),
+	[DSI_DP0] = PAD_INFO(DSI_DP0),
+	[DSI_DN0] = PAD_INFO(DSI_DN0),
+	[DSI_DP2] = PAD_INFO(DSI_DP2),
+	[DSI_DN2] = PAD_INFO(DSI_DN2),
+	[SENSOR0_PCLK] = PAD_INFO(SENSOR0_PCLK),
+	[CSI1_DN0] = PAD_INFO(CSI1_DN0),
+	[CSI1_DP0] = PAD_INFO(CSI1_DP0),
+	[CSI1_DN1] = PAD_INFO(CSI1_DN1),
+	[CSI1_DP1] = PAD_INFO(CSI1_DP1),
+	[CSI1_CN] = PAD_INFO(CSI1_CN),
+	[CSI1_CP] = PAD_INFO(CSI1_CP),
+	[SENSOR0_CKOUT] = PAD_INFO(SENSOR0_CKOUT),
+	[NAND0_D0] = PAD_INFO_PULLCTL(NAND0_D0),
+	[NAND0_D1] = PAD_INFO_PULLCTL(NAND0_D1),
+	[NAND0_D2] = PAD_INFO_PULLCTL(NAND0_D2),
+	[NAND0_D3] = PAD_INFO_PULLCTL(NAND0_D3),
+	[NAND0_D4] = PAD_INFO_PULLCTL(NAND0_D4),
+	[NAND0_D5] = PAD_INFO_PULLCTL(NAND0_D5),
+	[NAND0_D6] = PAD_INFO_PULLCTL(NAND0_D6),
+	[NAND0_D7] = PAD_INFO_PULLCTL(NAND0_D7),
+	[NAND0_DQS] = PAD_INFO_PULLCTL(NAND0_DQS),
+	[NAND0_DQSN] = PAD_INFO_PULLCTL(NAND0_DQSN),
+	[NAND0_ALE] = PAD_INFO(NAND0_ALE),
+	[NAND0_CLE] = PAD_INFO(NAND0_CLE),
+	[NAND0_CEB0] = PAD_INFO(NAND0_CEB0),
+	[NAND0_CEB1] = PAD_INFO(NAND0_CEB1),
+	[NAND0_CEB2] = PAD_INFO(NAND0_CEB2),
+	[NAND0_CEB3] = PAD_INFO(NAND0_CEB3),
+	[NAND1_D0] = PAD_INFO_PULLCTL(NAND1_D0),
+	[NAND1_D1] = PAD_INFO_PULLCTL(NAND1_D1),
+	[NAND1_D2] = PAD_INFO_PULLCTL(NAND1_D2),
+	[NAND1_D3] = PAD_INFO_PULLCTL(NAND1_D3),
+	[NAND1_D4] = PAD_INFO_PULLCTL(NAND1_D4),
+	[NAND1_D5] = PAD_INFO_PULLCTL(NAND1_D5),
+	[NAND1_D6] = PAD_INFO_PULLCTL(NAND1_D6),
+	[NAND1_D7] = PAD_INFO_PULLCTL(NAND1_D7),
+	[NAND1_DQS] = PAD_INFO_PULLCTL(NAND1_DQS),
+	[NAND1_DQSN] = PAD_INFO_PULLCTL(NAND1_DQSN),
+	[NAND1_ALE] = PAD_INFO(NAND1_ALE),
+	[NAND1_CLE] = PAD_INFO(NAND1_CLE),
+	[NAND1_CEB0] = PAD_INFO(NAND1_CEB0),
+	[NAND1_CEB1] = PAD_INFO(NAND1_CEB1),
+	[NAND1_CEB2] = PAD_INFO(NAND1_CEB2),
+	[NAND1_CEB3] = PAD_INFO(NAND1_CEB3),
+	[SGPIO0] = PAD_INFO(SGPIO0),
+	[SGPIO1] = PAD_INFO(SGPIO1),
+	[SGPIO2] = PAD_INFO_PULLCTL_ST(SGPIO2),
+	[SGPIO3] = PAD_INFO_PULLCTL_ST(SGPIO3)
+};
+
+#define OWL_GPIO_PORT(port, base, count, _outen, _inen, _dat)	\
+	[OWL_GPIO_PORT_##port] = {				\
+		.offset = base,					\
+		.pins = count,					\
+		.outen = _outen,				\
+		.inen = _inen,					\
+		.dat = _dat,					\
+	}
+
+static const struct owl_gpio_port s900_gpio_ports[] = {
+	OWL_GPIO_PORT(A, 0x0000, 32, 0x0, 0x4, 0x8),
+	OWL_GPIO_PORT(B, 0x000C, 32, 0x0, 0x4, 0x8),
+	OWL_GPIO_PORT(C, 0x0018, 12, 0x0, 0x4, 0x8),
+	OWL_GPIO_PORT(D, 0x0024, 30, 0x0, 0x4, 0x8),
+	OWL_GPIO_PORT(E, 0x0030, 32, 0x0, 0x4, 0x8),
+	OWL_GPIO_PORT(F, 0x00F0, 8, 0x0, 0x4, 0x8)
+};
+
+static struct owl_pinctrl_soc_data s900_pinctrl_data = {
+	.padinfo = s900_padinfo,
+	.pins = (const struct pinctrl_pin_desc *)s900_pads,
+	.npins = ARRAY_SIZE(s900_pads),
+	.functions = s900_functions,
+	.nfunctions = ARRAY_SIZE(s900_functions),
+	.groups = s900_groups,
+	.ngroups = ARRAY_SIZE(s900_groups),
+	.ngpios = NUM_GPIOS,
+	.ports = s900_gpio_ports,
+	.nports = ARRAY_SIZE(s900_gpio_ports)
+};
+
+static int s900_pinctrl_probe(struct platform_device *pdev)
+{
+	return owl_pinctrl_probe(pdev, &s900_pinctrl_data);
+}
+
+static const struct of_device_id s900_pinctrl_of_match[] = {
+	{ .compatible = "actions,s900-pinctrl", },
+	{ }
+};
+
+static struct platform_driver s900_pinctrl_driver = {
+	.driver = {
+		.name = "pinctrl-s900",
+		.of_match_table = of_match_ptr(s900_pinctrl_of_match),
+	},
+	.probe = s900_pinctrl_probe,
+};
+
+static int __init s900_pinctrl_init(void)
+{
+	return platform_driver_register(&s900_pinctrl_driver);
+}
+arch_initcall(s900_pinctrl_init);
+
+static void __exit s900_pinctrl_exit(void)
+{
+	platform_driver_unregister(&s900_pinctrl_driver);
+}
+module_exit(s900_pinctrl_exit);
+
+MODULE_AUTHOR("Actions Semi Inc.");
+MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
+MODULE_DESCRIPTION("Actions Semi S900 SoC Pinctrl Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/bcm/Kconfig b/drivers/pinctrl/bcm/Kconfig
index e8c4e4f934a6..0f38d51f47c6 100644
--- a/drivers/pinctrl/bcm/Kconfig
+++ b/drivers/pinctrl/bcm/Kconfig
@@ -20,6 +20,7 @@ config PINCTRL_BCM2835
 	bool
 	select PINMUX
 	select PINCONF
+	select GENERIC_PINCONF
 	select GPIOLIB_IRQCHIP
 
 config PINCTRL_IPROC_GPIO
diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
index 785c366fd6d6..136ccaf53df8 100644
--- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c
+++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
@@ -36,11 +36,13 @@
 #include <linux/pinctrl/pinconf.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/pinconf-generic.h>
 #include <linux/platform_device.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
+#include <dt-bindings/pinctrl/bcm2835.h>
 
 #define MODULE_NAME "pinctrl-bcm2835"
 #define BCM2835_NUM_GPIOS 54
@@ -72,13 +74,9 @@
 
 enum bcm2835_pinconf_param {
 	/* argument: bcm2835_pinconf_pull */
-	BCM2835_PINCONF_PARAM_PULL,
+	BCM2835_PINCONF_PARAM_PULL = (PIN_CONFIG_END + 1),
 };
 
-#define BCM2835_PINCONF_PACK(_param_, _arg_) ((_param_) << 16 | (_arg_))
-#define BCM2835_PINCONF_UNPACK_PARAM(_conf_) ((_conf_) >> 16)
-#define BCM2835_PINCONF_UNPACK_ARG(_conf_) ((_conf_) & 0xffff)
-
 struct bcm2835_pinctrl {
 	struct device *dev;
 	void __iomem *base;
@@ -213,14 +211,6 @@ static const char * const bcm2835_gpio_groups[] = {
 };
 
 enum bcm2835_fsel {
-	BCM2835_FSEL_GPIO_IN = 0,
-	BCM2835_FSEL_GPIO_OUT = 1,
-	BCM2835_FSEL_ALT0 = 4,
-	BCM2835_FSEL_ALT1 = 5,
-	BCM2835_FSEL_ALT2 = 6,
-	BCM2835_FSEL_ALT3 = 7,
-	BCM2835_FSEL_ALT4 = 3,
-	BCM2835_FSEL_ALT5 = 2,
 	BCM2835_FSEL_COUNT = 8,
 	BCM2835_FSEL_MASK = 0x7,
 };
@@ -714,7 +704,7 @@ static int bcm2835_pctl_dt_node_to_map_pull(struct bcm2835_pinctrl *pc,
 	configs = kzalloc(sizeof(*configs), GFP_KERNEL);
 	if (!configs)
 		return -ENOMEM;
-	configs[0] = BCM2835_PINCONF_PACK(BCM2835_PINCONF_PARAM_PULL, pull);
+	configs[0] = pinconf_to_config_packed(BCM2835_PINCONF_PARAM_PULL, pull);
 
 	map->type = PIN_MAP_TYPE_CONFIGS_PIN;
 	map->data.configs.group_or_pin = bcm2835_gpio_pins[pin].name;
@@ -727,7 +717,7 @@ static int bcm2835_pctl_dt_node_to_map_pull(struct bcm2835_pinctrl *pc,
 
 static int bcm2835_pctl_dt_node_to_map(struct pinctrl_dev *pctldev,
 		struct device_node *np,
-		struct pinctrl_map **map, unsigned *num_maps)
+		struct pinctrl_map **map, unsigned int *num_maps)
 {
 	struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev);
 	struct property *pins, *funcs, *pulls;
@@ -736,6 +726,12 @@ static int bcm2835_pctl_dt_node_to_map(struct pinctrl_dev *pctldev,
 	int i, err;
 	u32 pin, func, pull;
 
+	/* Check for generic binding in this node */
+	err = pinconf_generic_dt_node_to_map_all(pctldev, np, map, num_maps);
+	if (err || *num_maps)
+		return err;
+
+	/* Generic binding did not find anything continue with legacy parse */
 	pins = of_find_property(np, "brcm,pins", NULL);
 	if (!pins) {
 		dev_err(pc->dev, "%pOF: missing brcm,pins property\n", np);
@@ -917,37 +913,67 @@ static int bcm2835_pinconf_get(struct pinctrl_dev *pctldev,
 	return -ENOTSUPP;
 }
 
+static void bcm2835_pull_config_set(struct bcm2835_pinctrl *pc,
+		unsigned int pin, unsigned int arg)
+{
+	u32 off, bit;
+
+	off = GPIO_REG_OFFSET(pin);
+	bit = GPIO_REG_SHIFT(pin);
+
+	bcm2835_gpio_wr(pc, GPPUD, arg & 3);
+	/*
+	 * BCM2835 datasheet say to wait 150 cycles, but not of what.
+	 * But the VideoCore firmware delay for this operation
+	 * based nearly on the same amount of VPU cycles and this clock
+	 * runs at 250 MHz.
+	 */
+	udelay(1);
+	bcm2835_gpio_wr(pc, GPPUDCLK0 + (off * 4), BIT(bit));
+	udelay(1);
+	bcm2835_gpio_wr(pc, GPPUDCLK0 + (off * 4), 0);
+}
+
 static int bcm2835_pinconf_set(struct pinctrl_dev *pctldev,
-			unsigned pin, unsigned long *configs,
-			unsigned num_configs)
+			unsigned int pin, unsigned long *configs,
+			unsigned int num_configs)
 {
 	struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev);
-	enum bcm2835_pinconf_param param;
-	u16 arg;
-	u32 off, bit;
+	u32 param, arg;
 	int i;
 
 	for (i = 0; i < num_configs; i++) {
-		param = BCM2835_PINCONF_UNPACK_PARAM(configs[i]);
-		arg = BCM2835_PINCONF_UNPACK_ARG(configs[i]);
+		param = pinconf_to_config_param(configs[i]);
+		arg = pinconf_to_config_argument(configs[i]);
 
-		if (param != BCM2835_PINCONF_PARAM_PULL)
-			return -EINVAL;
+		switch (param) {
+		/* Set legacy brcm,pull */
+		case BCM2835_PINCONF_PARAM_PULL:
+			bcm2835_pull_config_set(pc, pin, arg);
+			break;
 
-		off = GPIO_REG_OFFSET(pin);
-		bit = GPIO_REG_SHIFT(pin);
+		/* Set pull generic bindings */
+		case PIN_CONFIG_BIAS_DISABLE:
+			bcm2835_pull_config_set(pc, pin, BCM2835_PUD_OFF);
+			break;
 
-		bcm2835_gpio_wr(pc, GPPUD, arg & 3);
-		/*
-		 * BCM2835 datasheet say to wait 150 cycles, but not of what.
-		 * But the VideoCore firmware delay for this operation
-		 * based nearly on the same amount of VPU cycles and this clock
-		 * runs at 250 MHz.
-		 */
-		udelay(1);
-		bcm2835_gpio_wr(pc, GPPUDCLK0 + (off * 4), BIT(bit));
-		udelay(1);
-		bcm2835_gpio_wr(pc, GPPUDCLK0 + (off * 4), 0);
+		case PIN_CONFIG_BIAS_PULL_DOWN:
+			bcm2835_pull_config_set(pc, pin, BCM2835_PUD_DOWN);
+			break;
+
+		case PIN_CONFIG_BIAS_PULL_UP:
+			bcm2835_pull_config_set(pc, pin, BCM2835_PUD_UP);
+			break;
+
+		/* Set output-high or output-low */
+		case PIN_CONFIG_OUTPUT:
+			bcm2835_gpio_set_bit(pc, arg ? GPSET0 : GPCLR0, pin);
+			break;
+
+		default:
+			return -EINVAL;
+
+		} /* switch param type */
 	} /* for each config */
 
 	return 0;
diff --git a/drivers/pinctrl/berlin/berlin-bg2.c b/drivers/pinctrl/berlin/berlin-bg2.c
index bf2e17d0d6e4..acbd413340e8 100644
--- a/drivers/pinctrl/berlin/berlin-bg2.c
+++ b/drivers/pinctrl/berlin/berlin-bg2.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Marvell Berlin BG2 pinctrl driver.
  *
  * Copyright (C) 2014 Marvell Technology Group Ltd.
  *
  * Antoine Ténart <antoine.tenart@free-electrons.com>
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2. This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
  */
 
 #include <linux/init.h>
diff --git a/drivers/pinctrl/berlin/berlin-bg2cd.c b/drivers/pinctrl/berlin/berlin-bg2cd.c
index 9bee7bd1650f..c0f5d86d5d01 100644
--- a/drivers/pinctrl/berlin/berlin-bg2cd.c
+++ b/drivers/pinctrl/berlin/berlin-bg2cd.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Marvell Berlin BG2CD pinctrl driver.
  *
  * Copyright (C) 2014 Marvell Technology Group Ltd.
  *
  * Antoine Ténart <antoine.tenart@free-electrons.com>
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2. This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
  */
 
 #include <linux/init.h>
diff --git a/drivers/pinctrl/berlin/berlin-bg2q.c b/drivers/pinctrl/berlin/berlin-bg2q.c
index eee6763f114c..20a3216ede07 100644
--- a/drivers/pinctrl/berlin/berlin-bg2q.c
+++ b/drivers/pinctrl/berlin/berlin-bg2q.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Marvell Berlin BG2Q pinctrl driver
  *
  * Copyright (C) 2014 Marvell Technology Group Ltd.
  *
  * Antoine Ténart <antoine.tenart@free-electrons.com>
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2. This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
  */
 
 #include <linux/init.h>
diff --git a/drivers/pinctrl/berlin/berlin-bg4ct.c b/drivers/pinctrl/berlin/berlin-bg4ct.c
index e6740656ee7c..6a7fe929a68b 100644
--- a/drivers/pinctrl/berlin/berlin-bg4ct.c
+++ b/drivers/pinctrl/berlin/berlin-bg4ct.c
@@ -1,21 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Marvell berlin4ct pinctrl driver
  *
  * Copyright (C) 2015 Marvell Technology Group Ltd.
  *
  * Author: Jisheng Zhang <jszhang@marvell.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <linux/init.h>
diff --git a/drivers/pinctrl/berlin/berlin.c b/drivers/pinctrl/berlin/berlin.c
index cc3bd2efafe3..a620a8e8fa78 100644
--- a/drivers/pinctrl/berlin/berlin.c
+++ b/drivers/pinctrl/berlin/berlin.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Marvell Berlin SoC pinctrl core driver
  *
  * Copyright (C) 2014 Marvell Technology Group Ltd.
  *
  * Antoine Ténart <antoine.tenart@free-electrons.com>
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2. This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
  */
 
 #include <linux/io.h>
diff --git a/drivers/pinctrl/berlin/berlin.h b/drivers/pinctrl/berlin/berlin.h
index e9b30f95b03e..d7787754d1ed 100644
--- a/drivers/pinctrl/berlin/berlin.h
+++ b/drivers/pinctrl/berlin/berlin.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Marvell Berlin SoC pinctrl driver.
  *
  * Copyright (C) 2014 Marvell Technology Group Ltd.
  *
  * Antoine Ténart <antoine.tenart@free-electrons.com>
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2. This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
  */
 
 #ifndef __PINCTRL_BERLIN_H
diff --git a/drivers/pinctrl/freescale/pinctrl-imx.c b/drivers/pinctrl/freescale/pinctrl-imx.c
index 24aaddd760a0..e582a21cfe54 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx.c
@@ -1,16 +1,11 @@
-/*
- * Core driver for the imx pin controller
- *
- * Copyright (C) 2012 Freescale Semiconductor, Inc.
- * Copyright (C) 2012 Linaro Ltd.
- *
- * Author: Dong Aisheng <dong.aisheng@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Core driver for the imx pin controller
+//
+// Copyright (C) 2012 Freescale Semiconductor, Inc.
+// Copyright (C) 2012 Linaro Ltd.
+//
+// Author: Dong Aisheng <dong.aisheng@linaro.org>
 
 #include <linux/err.h>
 #include <linux/init.h>
@@ -371,7 +366,7 @@ static void imx_pinconf_dbg_show(struct pinctrl_dev *pctldev,
 	unsigned long config;
 
 	if (!pin_reg || pin_reg->conf_reg == -1) {
-		seq_printf(s, "N/A");
+		seq_puts(s, "N/A");
 		return;
 	}
 
@@ -390,7 +385,7 @@ static void imx_pinconf_group_dbg_show(struct pinctrl_dev *pctldev,
 	if (group > pctldev->num_groups)
 		return;
 
-	seq_printf(s, "\n");
+	seq_puts(s, "\n");
 	grp = pinctrl_generic_get_group(pctldev, group);
 	if (!grp)
 		return;
@@ -414,11 +409,18 @@ static const struct pinconf_ops imx_pinconf_ops = {
 };
 
 /*
- * Each pin represented in fsl,pins consists of 5 u32 PIN_FUNC_ID and
- * 1 u32 CONFIG, so 24 types in total for each pin.
+ * Each pin represented in fsl,pins consists of a number of u32 PIN_FUNC_ID
+ * and 1 u32 CONFIG, the total size is PIN_FUNC_ID + CONFIG for each pin.
+ * For generic_pinconf case, there's no extra u32 CONFIG.
+ *
+ * PIN_FUNC_ID format:
+ * Default:
+ *     <mux_reg conf_reg input_reg mux_mode input_val>
+ * SHARE_MUX_CONF_REG:
+ *     <mux_conf_reg input_reg mux_mode input_val>
  */
 #define FSL_PIN_SIZE 24
-#define SHARE_FSL_PIN_SIZE 20
+#define FSL_PIN_SHARE_SIZE 20
 
 static int imx_pinctrl_parse_groups(struct device_node *np,
 				    struct group_desc *grp,
@@ -434,7 +436,7 @@ static int imx_pinctrl_parse_groups(struct device_node *np,
 	dev_dbg(ipctl->dev, "group(%d): %s\n", index, np->name);
 
 	if (info->flags & SHARE_MUX_CONF_REG)
-		pin_size = SHARE_FSL_PIN_SIZE;
+		pin_size = FSL_PIN_SHARE_SIZE;
 	else
 		pin_size = FSL_PIN_SIZE;
 
@@ -617,7 +619,7 @@ static int imx_pinctrl_probe_dt(struct platform_device *pdev,
 		nfuncs = 1;
 	} else {
 		nfuncs = of_get_child_count(np);
-		if (nfuncs <= 0) {
+		if (nfuncs == 0) {
 			dev_err(&pdev->dev, "no functions defined\n");
 			return -EINVAL;
 		}
diff --git a/drivers/pinctrl/freescale/pinctrl-imx.h b/drivers/pinctrl/freescale/pinctrl-imx.h
index 038e8c0e5b96..4b8225ccb03a 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx.h
+++ b/drivers/pinctrl/freescale/pinctrl-imx.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * IMX pinmux core definitions
  *
@@ -5,11 +6,6 @@
  * Copyright (C) 2012 Linaro Ltd.
  *
  * Author: Dong Aisheng <dong.aisheng@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #ifndef __DRIVERS_PINCTRL_IMX_H
diff --git a/drivers/pinctrl/freescale/pinctrl-imx1-core.c b/drivers/pinctrl/freescale/pinctrl-imx1-core.c
index a4e9f430d452..5af89de0ff02 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx1-core.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx1-core.c
@@ -1,19 +1,14 @@
-/*
- * Core driver for the imx pin controller in imx1/21/27
- *
- * Copyright (C) 2013 Pengutronix
- * Author: Markus Pargmann <mpa@pengutronix.de>
- *
- * Based on pinctrl-imx.c:
- *	Author: Dong Aisheng <dong.aisheng@linaro.org>
- *	Copyright (C) 2012 Freescale Semiconductor, Inc.
- *	Copyright (C) 2012 Linaro Ltd.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Core driver for the imx pin controller in imx1/21/27
+//
+// Copyright (C) 2013 Pengutronix
+// Author: Markus Pargmann <mpa@pengutronix.de>
+//
+// Based on pinctrl-imx.c:
+//	Author: Dong Aisheng <dong.aisheng@linaro.org>
+//	Copyright (C) 2012 Freescale Semiconductor, Inc.
+//	Copyright (C) 2012 Linaro Ltd.
 
 #include <linux/bitops.h>
 #include <linux/err.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx1.c b/drivers/pinctrl/freescale/pinctrl-imx1.c
index fc8efc748734..faf770f13bc7 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx1.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx1.c
@@ -1,13 +1,8 @@
-/*
- * i.MX1 pinctrl driver based on imx pinmux core
- *
- * Copyright (C) 2014 Alexander Shiyan <shc_work@mail.ru>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// i.MX1 pinctrl driver based on imx pinmux core
+//
+// Copyright (C) 2014 Alexander Shiyan <shc_work@mail.ru>
 
 #include <linux/init.h>
 #include <linux/of.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx1.h b/drivers/pinctrl/freescale/pinctrl-imx1.h
index 174074308d6c..f1b9dabf7601 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx1.h
+++ b/drivers/pinctrl/freescale/pinctrl-imx1.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * IMX pinmux core definitions
  *
@@ -5,11 +6,6 @@
  * Copyright (C) 2012 Linaro Ltd.
  *
  * Author: Dong Aisheng <dong.aisheng@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #ifndef __DRIVERS_PINCTRL_IMX1_H
diff --git a/drivers/pinctrl/freescale/pinctrl-imx21.c b/drivers/pinctrl/freescale/pinctrl-imx21.c
index 73e26bc12f09..8a102275a053 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx21.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx21.c
@@ -1,13 +1,8 @@
-/*
- * i.MX21 pinctrl driver based on imx pinmux core
- *
- * Copyright (C) 2014 Alexander Shiyan <shc_work@mail.ru>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// i.MX21 pinctrl driver based on imx pinmux core
+//
+// Copyright (C) 2014 Alexander Shiyan <shc_work@mail.ru>
 
 #include <linux/init.h>
 #include <linux/of.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx23.c b/drivers/pinctrl/freescale/pinctrl-imx23.c
index c9405685971b..144020764a4b 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx23.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx23.c
@@ -1,16 +1,9 @@
-/*
- * Freescale i.MX23 pinctrl driver
- *
- * Author: Shawn Guo <shawn.guo@linaro.org>
- * Copyright 2012 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Freescale i.MX23 pinctrl driver
+//
+// Author: Shawn Guo <shawn.guo@linaro.org>
+// Copyright 2012 Freescale Semiconductor, Inc.
 
 #include <linux/init.h>
 #include <linux/of_device.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx25.c b/drivers/pinctrl/freescale/pinctrl-imx25.c
index db6d9d1382f9..a899a398b6bb 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx25.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx25.c
@@ -1,19 +1,15 @@
-/*
- * imx25 pinctrl driver.
- *
- * Copyright 2013 Eukréa Electromatique <denis@eukrea.com>
- *
- * This driver was mostly copied from the imx51 pinctrl driver which has:
- *
- * Copyright (C) 2012 Freescale Semiconductor, Inc.
- * Copyright (C) 2012 Linaro, Inc.
- *
- * Author: Denis Carikli <denis@eukrea.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as published
- * by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// imx25 pinctrl driver.
+//
+// Copyright 2013 Eukréa Electromatique <denis@eukrea.com>
+//
+// This driver was mostly copied from the imx51 pinctrl driver which has:
+//
+// Copyright (C) 2012 Freescale Semiconductor, Inc.
+// Copyright (C) 2012 Linaro, Inc.
+//
+// Author: Denis Carikli <denis@eukrea.com>
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx27.c b/drivers/pinctrl/freescale/pinctrl-imx27.c
index e5992036fc6c..b4dfc1676cbc 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx27.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx27.c
@@ -1,15 +1,10 @@
-/*
- * imx27 pinctrl driver based on imx pinmux core
- *
- * Copyright (C) 2013 Pengutronix
- *
- * Author: Markus Pargmann <mpa@pengutronix.de>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// imx27 pinctrl driver based on imx pinmux core
+//
+// Copyright (C) 2013 Pengutronix
+//
+// Author: Markus Pargmann <mpa@pengutronix.de>
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx28.c b/drivers/pinctrl/freescale/pinctrl-imx28.c
index 87deb9ec938a..13730dd193f1 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx28.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx28.c
@@ -1,16 +1,9 @@
-/*
- * Freescale i.MX28 pinctrl driver
- *
- * Author: Shawn Guo <shawn.guo@linaro.org>
- * Copyright 2012 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Freescale i.MX28 pinctrl driver
+//
+// Author: Shawn Guo <shawn.guo@linaro.org>
+// Copyright 2012 Freescale Semiconductor, Inc.
 
 #include <linux/init.h>
 #include <linux/of_device.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx35.c b/drivers/pinctrl/freescale/pinctrl-imx35.c
index 6927946ae4b5..871bb419e2f0 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx35.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx35.c
@@ -1,17 +1,13 @@
-/*
- * imx35 pinctrl driver.
- *
- * This driver was mostly copied from the imx51 pinctrl driver which has:
- *
- * Copyright (C) 2012 Freescale Semiconductor, Inc.
- * Copyright (C) 2012 Linaro, Inc.
- *
- * Author: Dong Aisheng <dong.aisheng@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as published
- * by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// imx35 pinctrl driver.
+//
+// This driver was mostly copied from the imx51 pinctrl driver which has:
+//
+// Copyright (C) 2012 Freescale Semiconductor, Inc.
+// Copyright (C) 2012 Linaro, Inc.
+//
+// Author: Dong Aisheng <dong.aisheng@linaro.org>
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx50.c b/drivers/pinctrl/freescale/pinctrl-imx50.c
index eb349b97290f..cf182c040e0b 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx50.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx50.c
@@ -1,15 +1,10 @@
-/*
- * imx50 pinctrl driver based on imx pinmux core
- *
- * Copyright (C) 2013 Greg Ungerer <gerg@uclinux.org>
- * Copyright (C) 2012 Freescale Semiconductor, Inc.
- * Copyright (C) 2012 Linaro, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// imx50 pinctrl driver based on imx pinmux core
+//
+// Copyright (C) 2013 Greg Ungerer <gerg@uclinux.org>
+// Copyright (C) 2012 Freescale Semiconductor, Inc.
+// Copyright (C) 2012 Linaro, Inc.
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx51.c b/drivers/pinctrl/freescale/pinctrl-imx51.c
index 49acd991b5fb..e5c261e2bf1e 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx51.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx51.c
@@ -1,16 +1,11 @@
-/*
- * imx51 pinctrl driver based on imx pinmux core
- *
- * Copyright (C) 2012 Freescale Semiconductor, Inc.
- * Copyright (C) 2012 Linaro, Inc.
- *
- * Author: Dong Aisheng <dong.aisheng@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// imx51 pinctrl driver based on imx pinmux core
+//
+// Copyright (C) 2012 Freescale Semiconductor, Inc.
+// Copyright (C) 2012 Linaro, Inc.
+//
+// Author: Dong Aisheng <dong.aisheng@linaro.org>
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx53.c b/drivers/pinctrl/freescale/pinctrl-imx53.c
index 6dd0c60eaea4..64c97aaf20c7 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx53.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx53.c
@@ -1,16 +1,11 @@
-/*
- * imx53 pinctrl driver based on imx pinmux core
- *
- * Copyright (C) 2012 Freescale Semiconductor, Inc.
- * Copyright (C) 2012 Linaro, Inc.
- *
- * Author: Dong Aisheng <dong.aisheng@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// imx53 pinctrl driver based on imx pinmux core
+//
+// Copyright (C) 2012 Freescale Semiconductor, Inc.
+// Copyright (C) 2012 Linaro, Inc.
+//
+// Author: Dong Aisheng <dong.aisheng@linaro.org>
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx6dl.c b/drivers/pinctrl/freescale/pinctrl-imx6dl.c
index 91b85fc01de8..0858b4d79ed2 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx6dl.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx6dl.c
@@ -1,13 +1,9 @@
-/*
- * Freescale imx6dl pinctrl driver
- *
- * Author: Shawn Guo <shawn.guo@linaro.org>
- * Copyright (C) 2013 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Freescale imx6dl pinctrl driver
+//
+// Author: Shawn Guo <shawn.guo@linaro.org>
+// Copyright (C) 2013 Freescale Semiconductor, Inc.
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx6q.c b/drivers/pinctrl/freescale/pinctrl-imx6q.c
index 5f653d69d0f5..078ed6a331fd 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx6q.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx6q.c
@@ -1,16 +1,11 @@
-/*
- * imx6q pinctrl driver based on imx pinmux core
- *
- * Copyright (C) 2012 Freescale Semiconductor, Inc.
- * Copyright (C) 2012 Linaro, Inc.
- *
- * Author: Dong Aisheng <dong.aisheng@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// imx6q pinctrl driver based on imx pinmux core
+//
+// Copyright (C) 2012 Freescale Semiconductor, Inc.
+// Copyright (C) 2012 Linaro, Inc.
+//
+// Author: Dong Aisheng <dong.aisheng@linaro.org>
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx6sl.c b/drivers/pinctrl/freescale/pinctrl-imx6sl.c
index 1167dc273c04..9d2e6f987aa7 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx6sl.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx6sl.c
@@ -1,13 +1,9 @@
-/*
- * Freescale imx6sl pinctrl driver
- *
- * Author: Shawn Guo <shawn.guo@linaro.org>
- * Copyright (C) 2013 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Freescale imx6sl pinctrl driver
+//
+// Author: Shawn Guo <shawn.guo@linaro.org>
+// Copyright (C) 2013 Freescale Semiconductor, Inc.
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx6sll.c b/drivers/pinctrl/freescale/pinctrl-imx6sll.c
index 0fbea9cf536d..0618f4d887fd 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx6sll.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx6sll.c
@@ -1,9 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2016 Freescale Semiconductor, Inc.
- * Copyright 2017-2018 NXP.
- *
- */
+//
+// Copyright (C) 2016 Freescale Semiconductor, Inc.
+// Copyright 2017-2018 NXP.
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx6sx.c b/drivers/pinctrl/freescale/pinctrl-imx6sx.c
index 15ea56c75f68..c7e2b1f94f01 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx6sx.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx6sx.c
@@ -1,13 +1,9 @@
-/*
- * Freescale imx6sx pinctrl driver
- *
- * Author: Anson Huang <Anson.Huang@freescale.com>
- * Copyright (C) 2014 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Freescale imx6sx pinctrl driver
+//
+// Author: Anson Huang <Anson.Huang@freescale.com>
+// Copyright (C) 2014 Freescale Semiconductor, Inc.
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx6ul.c b/drivers/pinctrl/freescale/pinctrl-imx6ul.c
index 4580717ade19..7e37627c63f5 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx6ul.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx6ul.c
@@ -1,13 +1,9 @@
-/*
- * Freescale imx6ul pinctrl driver
- *
- * Author: Anson Huang <Anson.Huang@freescale.com>
- * Copyright (C) 2015 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Freescale imx6ul pinctrl driver
+//
+// Author: Anson Huang <Anson.Huang@freescale.com>
+// Copyright (C) 2015 Freescale Semiconductor, Inc.
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx7d.c b/drivers/pinctrl/freescale/pinctrl-imx7d.c
index 0b0a2f33b06a..369d3e59fdd6 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx7d.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx7d.c
@@ -1,13 +1,9 @@
-/*
- * Freescale imx7d pinctrl driver
- *
- * Author: Anson Huang <Anson.Huang@freescale.com>
- * Copyright (C) 2014-2015 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Freescale imx7d pinctrl driver
+//
+// Author: Anson Huang <Anson.Huang@freescale.com>
+// Copyright (C) 2014-2015 Freescale Semiconductor, Inc.
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-imx7ulp.c b/drivers/pinctrl/freescale/pinctrl-imx7ulp.c
index f363e45fd246..f521bdb53f62 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx7ulp.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx7ulp.c
@@ -1,14 +1,9 @@
-/*
- * Copyright (C) 2016 Freescale Semiconductor, Inc.
- * Copyright (C) 2017 NXP
- *
- * Author: Dong Aisheng <aisheng.dong@nxp.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2016 Freescale Semiconductor, Inc.
+// Copyright (C) 2017 NXP
+//
+// Author: Dong Aisheng <aisheng.dong@nxp.com>
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-mxs.c b/drivers/pinctrl/freescale/pinctrl-mxs.c
index 6852010a6d70..594f3e5ce9a9 100644
--- a/drivers/pinctrl/freescale/pinctrl-mxs.c
+++ b/drivers/pinctrl/freescale/pinctrl-mxs.c
@@ -1,13 +1,6 @@
-/*
- * Copyright 2012 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/freescale/pinctrl-mxs.h b/drivers/pinctrl/freescale/pinctrl-mxs.h
index 34dbf75208dc..ab9f834b03e6 100644
--- a/drivers/pinctrl/freescale/pinctrl-mxs.h
+++ b/drivers/pinctrl/freescale/pinctrl-mxs.h
@@ -1,12 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Copyright 2012 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
  */
 
 #ifndef __PINCTRL_MXS_H
diff --git a/drivers/pinctrl/freescale/pinctrl-vf610.c b/drivers/pinctrl/freescale/pinctrl-vf610.c
index c078f859ae15..37602b053ed2 100644
--- a/drivers/pinctrl/freescale/pinctrl-vf610.c
+++ b/drivers/pinctrl/freescale/pinctrl-vf610.c
@@ -1,13 +1,8 @@
-/*
- * VF610 pinctrl driver based on imx pinmux and pinconf core
- *
- * Copyright 2013 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// VF610 pinctrl driver based on imx pinmux and pinconf core
+//
+// Copyright 2013 Freescale Semiconductor, Inc.
 
 #include <linux/err.h>
 #include <linux/init.h>
diff --git a/drivers/pinctrl/intel/pinctrl-cherryview.c b/drivers/pinctrl/intel/pinctrl-cherryview.c
index fee9225ca559..0f1019ae3993 100644
--- a/drivers/pinctrl/intel/pinctrl-cherryview.c
+++ b/drivers/pinctrl/intel/pinctrl-cherryview.c
@@ -1527,6 +1527,7 @@ static const struct dmi_system_id chv_no_valid_mask[] = {
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "GOOGLE"),
 			DMI_MATCH(DMI_PRODUCT_FAMILY, "Intel_Strago"),
+			DMI_MATCH(DMI_BOARD_VERSION, "1.0"),
 		},
 	},
 	{
@@ -1534,6 +1535,7 @@ static const struct dmi_system_id chv_no_valid_mask[] = {
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "HP"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "Setzer"),
+			DMI_MATCH(DMI_BOARD_VERSION, "1.0"),
 		},
 	},
 	{
@@ -1541,6 +1543,7 @@ static const struct dmi_system_id chv_no_valid_mask[] = {
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "GOOGLE"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "Cyan"),
+			DMI_MATCH(DMI_BOARD_VERSION, "1.0"),
 		},
 	},
 	{
@@ -1548,6 +1551,7 @@ static const struct dmi_system_id chv_no_valid_mask[] = {
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "GOOGLE"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "Celes"),
+			DMI_MATCH(DMI_BOARD_VERSION, "1.0"),
 		},
 	},
 	{}
diff --git a/drivers/pinctrl/mediatek/Kconfig b/drivers/pinctrl/mediatek/Kconfig
index 862c5dbc6977..9905dc672f6b 100644
--- a/drivers/pinctrl/mediatek/Kconfig
+++ b/drivers/pinctrl/mediatek/Kconfig
@@ -1,12 +1,18 @@
 menu "MediaTek pinctrl drivers"
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 
+config EINT_MTK
+	bool "MediaTek External Interrupt Support"
+	depends on PINCTRL_MTK || PINCTRL_MT7622 || COMPILE_TEST
+	select IRQ_DOMAIN
+
 config PINCTRL_MTK
 	bool
 	depends on OF
 	select PINMUX
 	select GENERIC_PINCONF
 	select GPIOLIB
+	select EINT_MTK
 	select OF_GPIO
 
 # For ARMv7 SoCs
diff --git a/drivers/pinctrl/mediatek/Makefile b/drivers/pinctrl/mediatek/Makefile
index 7959e773533f..3de7156df345 100644
--- a/drivers/pinctrl/mediatek/Makefile
+++ b/drivers/pinctrl/mediatek/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 # Core
+obj-$(CONFIG_EINT_MTK)		+= mtk-eint.o
 obj-$(CONFIG_PINCTRL_MTK)	+= pinctrl-mtk-common.o
 
 # SoC Drivers
diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c
new file mode 100644
index 000000000000..30f3316747e2
--- /dev/null
+++ b/drivers/pinctrl/mediatek/mtk-eint.c
@@ -0,0 +1,492 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2014-2018 MediaTek Inc.
+
+/*
+ * Library for MediaTek External Interrupt Support
+ *
+ * Author: Maoguang Meng <maoguang.meng@mediatek.com>
+ *	   Sean Wang <sean.wang@mediatek.com>
+ *
+ */
+
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/gpio.h>
+#include <linux/io.h>
+#include <linux/irqdomain.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+
+#include "mtk-eint.h"
+
+#define MTK_EINT_EDGE_SENSITIVE           0
+#define MTK_EINT_LEVEL_SENSITIVE          1
+#define MTK_EINT_DBNC_SET_DBNC_BITS	  4
+#define MTK_EINT_DBNC_RST_BIT		  (0x1 << 1)
+#define MTK_EINT_DBNC_SET_EN		  (0x1 << 0)
+
+static const struct mtk_eint_regs mtk_generic_eint_regs = {
+	.stat      = 0x000,
+	.ack       = 0x040,
+	.mask      = 0x080,
+	.mask_set  = 0x0c0,
+	.mask_clr  = 0x100,
+	.sens      = 0x140,
+	.sens_set  = 0x180,
+	.sens_clr  = 0x1c0,
+	.soft      = 0x200,
+	.soft_set  = 0x240,
+	.soft_clr  = 0x280,
+	.pol       = 0x300,
+	.pol_set   = 0x340,
+	.pol_clr   = 0x380,
+	.dom_en    = 0x400,
+	.dbnc_ctrl = 0x500,
+	.dbnc_set  = 0x600,
+	.dbnc_clr  = 0x700,
+};
+
+static void __iomem *mtk_eint_get_offset(struct mtk_eint *eint,
+					 unsigned int eint_num,
+					 unsigned int offset)
+{
+	unsigned int eint_base = 0;
+	void __iomem *reg;
+
+	if (eint_num >= eint->hw->ap_num)
+		eint_base = eint->hw->ap_num;
+
+	reg = eint->base + offset + ((eint_num - eint_base) / 32) * 4;
+
+	return reg;
+}
+
+static unsigned int mtk_eint_can_en_debounce(struct mtk_eint *eint,
+					     unsigned int eint_num)
+{
+	unsigned int sens;
+	unsigned int bit = BIT(eint_num % 32);
+	void __iomem *reg = mtk_eint_get_offset(eint, eint_num,
+						eint->regs->sens);
+
+	if (readl(reg) & bit)
+		sens = MTK_EINT_LEVEL_SENSITIVE;
+	else
+		sens = MTK_EINT_EDGE_SENSITIVE;
+
+	if (eint_num < eint->hw->db_cnt && sens != MTK_EINT_EDGE_SENSITIVE)
+		return 1;
+	else
+		return 0;
+}
+
+static int mtk_eint_flip_edge(struct mtk_eint *eint, int hwirq)
+{
+	int start_level, curr_level;
+	unsigned int reg_offset;
+	u32 mask = BIT(hwirq & 0x1f);
+	u32 port = (hwirq >> 5) & eint->hw->port_mask;
+	void __iomem *reg = eint->base + (port << 2);
+
+	curr_level = eint->gpio_xlate->get_gpio_state(eint->pctl, hwirq);
+
+	do {
+		start_level = curr_level;
+		if (start_level)
+			reg_offset = eint->regs->pol_clr;
+		else
+			reg_offset = eint->regs->pol_set;
+		writel(mask, reg + reg_offset);
+
+		curr_level = eint->gpio_xlate->get_gpio_state(eint->pctl,
+							      hwirq);
+	} while (start_level != curr_level);
+
+	return start_level;
+}
+
+static void mtk_eint_mask(struct irq_data *d)
+{
+	struct mtk_eint *eint = irq_data_get_irq_chip_data(d);
+	u32 mask = BIT(d->hwirq & 0x1f);
+	void __iomem *reg = mtk_eint_get_offset(eint, d->hwirq,
+						eint->regs->mask_set);
+
+	writel(mask, reg);
+}
+
+static void mtk_eint_unmask(struct irq_data *d)
+{
+	struct mtk_eint *eint = irq_data_get_irq_chip_data(d);
+	u32 mask = BIT(d->hwirq & 0x1f);
+	void __iomem *reg = mtk_eint_get_offset(eint, d->hwirq,
+						eint->regs->mask_clr);
+
+	writel(mask, reg);
+
+	if (eint->dual_edge[d->hwirq])
+		mtk_eint_flip_edge(eint, d->hwirq);
+}
+
+static unsigned int mtk_eint_get_mask(struct mtk_eint *eint,
+				      unsigned int eint_num)
+{
+	unsigned int bit = BIT(eint_num % 32);
+	void __iomem *reg = mtk_eint_get_offset(eint, eint_num,
+						eint->regs->mask);
+
+	return !!(readl(reg) & bit);
+}
+
+static void mtk_eint_ack(struct irq_data *d)
+{
+	struct mtk_eint *eint = irq_data_get_irq_chip_data(d);
+	u32 mask = BIT(d->hwirq & 0x1f);
+	void __iomem *reg = mtk_eint_get_offset(eint, d->hwirq,
+						eint->regs->ack);
+
+	writel(mask, reg);
+}
+
+static int mtk_eint_set_type(struct irq_data *d, unsigned int type)
+{
+	struct mtk_eint *eint = irq_data_get_irq_chip_data(d);
+	u32 mask = BIT(d->hwirq & 0x1f);
+	void __iomem *reg;
+
+	if (((type & IRQ_TYPE_EDGE_BOTH) && (type & IRQ_TYPE_LEVEL_MASK)) ||
+	    ((type & IRQ_TYPE_LEVEL_MASK) == IRQ_TYPE_LEVEL_MASK)) {
+		dev_err(eint->dev,
+			"Can't configure IRQ%d (EINT%lu) for type 0x%X\n",
+			d->irq, d->hwirq, type);
+		return -EINVAL;
+	}
+
+	if ((type & IRQ_TYPE_EDGE_BOTH) == IRQ_TYPE_EDGE_BOTH)
+		eint->dual_edge[d->hwirq] = 1;
+	else
+		eint->dual_edge[d->hwirq] = 0;
+
+	if (type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING)) {
+		reg = mtk_eint_get_offset(eint, d->hwirq, eint->regs->pol_clr);
+		writel(mask, reg);
+	} else {
+		reg = mtk_eint_get_offset(eint, d->hwirq, eint->regs->pol_set);
+		writel(mask, reg);
+	}
+
+	if (type & (IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING)) {
+		reg = mtk_eint_get_offset(eint, d->hwirq, eint->regs->sens_clr);
+		writel(mask, reg);
+	} else {
+		reg = mtk_eint_get_offset(eint, d->hwirq, eint->regs->sens_set);
+		writel(mask, reg);
+	}
+
+	if (eint->dual_edge[d->hwirq])
+		mtk_eint_flip_edge(eint, d->hwirq);
+
+	return 0;
+}
+
+static int mtk_eint_irq_set_wake(struct irq_data *d, unsigned int on)
+{
+	struct mtk_eint *eint = irq_data_get_irq_chip_data(d);
+	int shift = d->hwirq & 0x1f;
+	int reg = d->hwirq >> 5;
+
+	if (on)
+		eint->wake_mask[reg] |= BIT(shift);
+	else
+		eint->wake_mask[reg] &= ~BIT(shift);
+
+	return 0;
+}
+
+static void mtk_eint_chip_write_mask(const struct mtk_eint *eint,
+				     void __iomem *base, u32 *buf)
+{
+	int port;
+	void __iomem *reg;
+
+	for (port = 0; port < eint->hw->ports; port++) {
+		reg = base + (port << 2);
+		writel_relaxed(~buf[port], reg + eint->regs->mask_set);
+		writel_relaxed(buf[port], reg + eint->regs->mask_clr);
+	}
+}
+
+static void mtk_eint_chip_read_mask(const struct mtk_eint *eint,
+				    void __iomem *base, u32 *buf)
+{
+	int port;
+	void __iomem *reg;
+
+	for (port = 0; port < eint->hw->ports; port++) {
+		reg = base + eint->regs->mask + (port << 2);
+		buf[port] = ~readl_relaxed(reg);
+		/* Mask is 0 when irq is enabled, and 1 when disabled. */
+	}
+}
+
+static int mtk_eint_irq_request_resources(struct irq_data *d)
+{
+	struct mtk_eint *eint = irq_data_get_irq_chip_data(d);
+	struct gpio_chip *gpio_c;
+	unsigned int gpio_n;
+	int err;
+
+	err = eint->gpio_xlate->get_gpio_n(eint->pctl, d->hwirq,
+					   &gpio_n, &gpio_c);
+	if (err < 0) {
+		dev_err(eint->dev, "Can not find pin\n");
+		return err;
+	}
+
+	err = gpiochip_lock_as_irq(gpio_c, gpio_n);
+	if (err < 0) {
+		dev_err(eint->dev, "unable to lock HW IRQ %lu for IRQ\n",
+			irqd_to_hwirq(d));
+		return err;
+	}
+
+	err = eint->gpio_xlate->set_gpio_as_eint(eint->pctl, d->hwirq);
+	if (err < 0) {
+		dev_err(eint->dev, "Can not eint mode\n");
+		return err;
+	}
+
+	return 0;
+}
+
+static void mtk_eint_irq_release_resources(struct irq_data *d)
+{
+	struct mtk_eint *eint = irq_data_get_irq_chip_data(d);
+	struct gpio_chip *gpio_c;
+	unsigned int gpio_n;
+
+	eint->gpio_xlate->get_gpio_n(eint->pctl, d->hwirq, &gpio_n,
+				     &gpio_c);
+
+	gpiochip_unlock_as_irq(gpio_c, gpio_n);
+}
+
+static struct irq_chip mtk_eint_irq_chip = {
+	.name = "mt-eint",
+	.irq_disable = mtk_eint_mask,
+	.irq_mask = mtk_eint_mask,
+	.irq_unmask = mtk_eint_unmask,
+	.irq_ack = mtk_eint_ack,
+	.irq_set_type = mtk_eint_set_type,
+	.irq_set_wake = mtk_eint_irq_set_wake,
+	.irq_request_resources = mtk_eint_irq_request_resources,
+	.irq_release_resources = mtk_eint_irq_release_resources,
+};
+
+static unsigned int mtk_eint_hw_init(struct mtk_eint *eint)
+{
+	void __iomem *reg = eint->base + eint->regs->dom_en;
+	unsigned int i;
+
+	for (i = 0; i < eint->hw->ap_num; i += 32) {
+		writel(0xffffffff, reg);
+		reg += 4;
+	}
+
+	return 0;
+}
+
+static inline void
+mtk_eint_debounce_process(struct mtk_eint *eint, int index)
+{
+	unsigned int rst, ctrl_offset;
+	unsigned int bit, dbnc;
+
+	ctrl_offset = (index / 4) * 4 + eint->regs->dbnc_ctrl;
+	dbnc = readl(eint->base + ctrl_offset);
+	bit = MTK_EINT_DBNC_SET_EN << ((index % 4) * 8);
+	if ((bit & dbnc) > 0) {
+		ctrl_offset = (index / 4) * 4 + eint->regs->dbnc_set;
+		rst = MTK_EINT_DBNC_RST_BIT << ((index % 4) * 8);
+		writel(rst, eint->base + ctrl_offset);
+	}
+}
+
+static void mtk_eint_irq_handler(struct irq_desc *desc)
+{
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	struct mtk_eint *eint = irq_desc_get_handler_data(desc);
+	unsigned int status, eint_num;
+	int offset, index, virq;
+	void __iomem *reg =  mtk_eint_get_offset(eint, 0, eint->regs->stat);
+	int dual_edge, start_level, curr_level;
+
+	chained_irq_enter(chip, desc);
+	for (eint_num = 0; eint_num < eint->hw->ap_num; eint_num += 32,
+	     reg += 4) {
+		status = readl(reg);
+		while (status) {
+			offset = __ffs(status);
+			index = eint_num + offset;
+			virq = irq_find_mapping(eint->domain, index);
+			status &= ~BIT(offset);
+
+			dual_edge = eint->dual_edge[index];
+			if (dual_edge) {
+				/*
+				 * Clear soft-irq in case we raised it last
+				 * time.
+				 */
+				writel(BIT(offset), reg - eint->regs->stat +
+				       eint->regs->soft_clr);
+
+				start_level =
+				eint->gpio_xlate->get_gpio_state(eint->pctl,
+								 index);
+			}
+
+			generic_handle_irq(virq);
+
+			if (dual_edge) {
+				curr_level = mtk_eint_flip_edge(eint, index);
+
+				/*
+				 * If level changed, we might lost one edge
+				 * interrupt, raised it through soft-irq.
+				 */
+				if (start_level != curr_level)
+					writel(BIT(offset), reg -
+					       eint->regs->stat +
+					       eint->regs->soft_set);
+			}
+
+			if (index < eint->hw->db_cnt)
+				mtk_eint_debounce_process(eint, index);
+		}
+	}
+	chained_irq_exit(chip, desc);
+}
+
+int mtk_eint_do_suspend(struct mtk_eint *eint)
+{
+	mtk_eint_chip_read_mask(eint, eint->base, eint->cur_mask);
+	mtk_eint_chip_write_mask(eint, eint->base, eint->wake_mask);
+
+	return 0;
+}
+
+int mtk_eint_do_resume(struct mtk_eint *eint)
+{
+	mtk_eint_chip_write_mask(eint, eint->base, eint->cur_mask);
+
+	return 0;
+}
+
+int mtk_eint_set_debounce(struct mtk_eint *eint, unsigned long eint_num,
+			  unsigned int debounce)
+{
+	int virq, eint_offset;
+	unsigned int set_offset, bit, clr_bit, clr_offset, rst, i, unmask,
+		     dbnc;
+	static const unsigned int debounce_time[] = {500, 1000, 16000, 32000,
+						     64000, 128000, 256000};
+	struct irq_data *d;
+
+	virq = irq_find_mapping(eint->domain, eint_num);
+	eint_offset = (eint_num % 4) * 8;
+	d = irq_get_irq_data(virq);
+
+	set_offset = (eint_num / 4) * 4 + eint->regs->dbnc_set;
+	clr_offset = (eint_num / 4) * 4 + eint->regs->dbnc_clr;
+
+	if (!mtk_eint_can_en_debounce(eint, eint_num))
+		return -EINVAL;
+
+	dbnc = ARRAY_SIZE(debounce_time);
+	for (i = 0; i < ARRAY_SIZE(debounce_time); i++) {
+		if (debounce <= debounce_time[i]) {
+			dbnc = i;
+			break;
+		}
+	}
+
+	if (!mtk_eint_get_mask(eint, eint_num)) {
+		mtk_eint_mask(d);
+		unmask = 1;
+	} else {
+		unmask = 0;
+	}
+
+	clr_bit = 0xff << eint_offset;
+	writel(clr_bit, eint->base + clr_offset);
+
+	bit = ((dbnc << MTK_EINT_DBNC_SET_DBNC_BITS) | MTK_EINT_DBNC_SET_EN) <<
+		eint_offset;
+	rst = MTK_EINT_DBNC_RST_BIT << eint_offset;
+	writel(rst | bit, eint->base + set_offset);
+
+	/*
+	 * Delay a while (more than 2T) to wait for hw debounce counter reset
+	 * work correctly.
+	 */
+	udelay(1);
+	if (unmask == 1)
+		mtk_eint_unmask(d);
+
+	return 0;
+}
+
+int mtk_eint_find_irq(struct mtk_eint *eint, unsigned long eint_n)
+{
+	int irq;
+
+	irq = irq_find_mapping(eint->domain, eint_n);
+	if (!irq)
+		return -EINVAL;
+
+	return irq;
+}
+
+int mtk_eint_do_init(struct mtk_eint *eint)
+{
+	int i;
+
+	/* If clients don't assign a specific regs, let's use generic one */
+	if (!eint->regs)
+		eint->regs = &mtk_generic_eint_regs;
+
+	eint->wake_mask = devm_kcalloc(eint->dev, eint->hw->ports,
+				       sizeof(*eint->wake_mask), GFP_KERNEL);
+	if (!eint->wake_mask)
+		return -ENOMEM;
+
+	eint->cur_mask = devm_kcalloc(eint->dev, eint->hw->ports,
+				      sizeof(*eint->cur_mask), GFP_KERNEL);
+	if (!eint->cur_mask)
+		return -ENOMEM;
+
+	eint->dual_edge = devm_kcalloc(eint->dev, eint->hw->ap_num,
+				       sizeof(int), GFP_KERNEL);
+	if (!eint->dual_edge)
+		return -ENOMEM;
+
+	eint->domain = irq_domain_add_linear(eint->dev->of_node,
+					     eint->hw->ap_num,
+					     &irq_domain_simple_ops, NULL);
+	if (!eint->domain)
+		return -ENOMEM;
+
+	mtk_eint_hw_init(eint);
+	for (i = 0; i < eint->hw->ap_num; i++) {
+		int virq = irq_create_mapping(eint->domain, i);
+
+		irq_set_chip_and_handler(virq, &mtk_eint_irq_chip,
+					 handle_level_irq);
+		irq_set_chip_data(virq, eint);
+	}
+
+	irq_set_chained_handler_and_data(eint->irq, mtk_eint_irq_handler,
+					 eint);
+
+	return 0;
+}
diff --git a/drivers/pinctrl/mediatek/mtk-eint.h b/drivers/pinctrl/mediatek/mtk-eint.h
new file mode 100644
index 000000000000..c286a9b940f2
--- /dev/null
+++ b/drivers/pinctrl/mediatek/mtk-eint.h
@@ -0,0 +1,106 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2014-2018 MediaTek Inc.
+ *
+ * Author: Maoguang Meng <maoguang.meng@mediatek.com>
+ *	   Sean Wang <sean.wang@mediatek.com>
+ *
+ */
+#ifndef __MTK_EINT_H
+#define __MTK_EINT_H
+
+#include <linux/irqdomain.h>
+
+struct mtk_eint_regs {
+	unsigned int	stat;
+	unsigned int	ack;
+	unsigned int	mask;
+	unsigned int	mask_set;
+	unsigned int	mask_clr;
+	unsigned int	sens;
+	unsigned int	sens_set;
+	unsigned int	sens_clr;
+	unsigned int	soft;
+	unsigned int	soft_set;
+	unsigned int	soft_clr;
+	unsigned int	pol;
+	unsigned int	pol_set;
+	unsigned int	pol_clr;
+	unsigned int	dom_en;
+	unsigned int	dbnc_ctrl;
+	unsigned int	dbnc_set;
+	unsigned int	dbnc_clr;
+};
+
+struct mtk_eint_hw {
+	u8		port_mask;
+	u8		ports;
+	unsigned int	ap_num;
+	unsigned int	db_cnt;
+};
+
+struct mtk_eint;
+
+struct mtk_eint_xt {
+	int (*get_gpio_n)(void *data, unsigned long eint_n,
+			  unsigned int *gpio_n,
+			  struct gpio_chip **gpio_chip);
+	int (*get_gpio_state)(void *data, unsigned long eint_n);
+	int (*set_gpio_as_eint)(void *data, unsigned long eint_n);
+};
+
+struct mtk_eint {
+	struct device *dev;
+	void __iomem *base;
+	struct irq_domain *domain;
+	int irq;
+
+	int *dual_edge;
+	u32 *wake_mask;
+	u32 *cur_mask;
+
+	/* Used to fit into various EINT device */
+	const struct mtk_eint_hw *hw;
+	const struct mtk_eint_regs *regs;
+
+	/* Used to fit into various pinctrl device */
+	void *pctl;
+	const struct mtk_eint_xt *gpio_xlate;
+};
+
+#if IS_ENABLED(CONFIG_EINT_MTK)
+int mtk_eint_do_init(struct mtk_eint *eint);
+int mtk_eint_do_suspend(struct mtk_eint *eint);
+int mtk_eint_do_resume(struct mtk_eint *eint);
+int mtk_eint_set_debounce(struct mtk_eint *eint, unsigned long eint_n,
+			  unsigned int debounce);
+int mtk_eint_find_irq(struct mtk_eint *eint, unsigned long eint_n);
+
+#else
+static inline int mtk_eint_do_init(struct mtk_eint *eint)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int mtk_eint_do_suspend(struct mtk_eint *eint)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int mtk_eint_do_resume(struct mtk_eint *eint)
+{
+	return -EOPNOTSUPP;
+}
+
+int mtk_eint_set_debounce(struct mtk_eint *eint, unsigned long eint_n,
+			  unsigned int debounce)
+{
+	return -EOPNOTSUPP;
+}
+
+int mtk_eint_find_irq(struct mtk_eint *eint, unsigned long eint_n)
+{
+	return -EOPNOTSUPP;
+}
+#endif
+#endif /* __MTK_EINT_H */
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt2701.c b/drivers/pinctrl/mediatek/pinctrl-mt2701.c
index f86f3b379607..e91c314f3b75 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt2701.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt2701.c
@@ -531,31 +531,12 @@ static const struct mtk_pinctrl_devdata mt2701_pinctrl_data = {
 	.port_shf = 4,
 	.port_mask = 0x1f,
 	.port_align = 4,
-	.eint_offsets = {
-		.name = "mt2701_eint",
-		.stat      = 0x000,
-		.ack       = 0x040,
-		.mask      = 0x080,
-		.mask_set  = 0x0c0,
-		.mask_clr  = 0x100,
-		.sens      = 0x140,
-		.sens_set  = 0x180,
-		.sens_clr  = 0x1c0,
-		.soft      = 0x200,
-		.soft_set  = 0x240,
-		.soft_clr  = 0x280,
-		.pol       = 0x300,
-		.pol_set   = 0x340,
-		.pol_clr   = 0x380,
-		.dom_en    = 0x400,
-		.dbnc_ctrl = 0x500,
-		.dbnc_set  = 0x600,
-		.dbnc_clr  = 0x700,
+	.eint_hw = {
 		.port_mask = 6,
 		.ports     = 6,
+		.ap_num    = 169,
+		.db_cnt    = 16,
 	},
-	.ap_num = 169,
-	.db_cnt = 16,
 };
 
 static int mt2701_pinctrl_probe(struct platform_device *pdev)
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt2712.c b/drivers/pinctrl/mediatek/pinctrl-mt2712.c
index 81e11f9e70f1..8398d55c01cb 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt2712.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt2712.c
@@ -576,31 +576,12 @@ static const struct mtk_pinctrl_devdata mt2712_pinctrl_data = {
 	.port_shf = 4,
 	.port_mask = 0xf,
 	.port_align = 4,
-	.eint_offsets = {
-		.name = "mt2712_eint",
-		.stat      = 0x000,
-		.ack       = 0x040,
-		.mask      = 0x080,
-		.mask_set  = 0x0c0,
-		.mask_clr  = 0x100,
-		.sens      = 0x140,
-		.sens_set  = 0x180,
-		.sens_clr  = 0x1c0,
-		.soft      = 0x200,
-		.soft_set  = 0x240,
-		.soft_clr  = 0x280,
-		.pol       = 0x300,
-		.pol_set   = 0x340,
-		.pol_clr   = 0x380,
-		.dom_en    = 0x400,
-		.dbnc_ctrl = 0x500,
-		.dbnc_set  = 0x600,
-		.dbnc_clr  = 0x700,
+	.eint_hw = {
 		.port_mask = 0xf,
 		.ports     = 8,
+		.ap_num    = 229,
+		.db_cnt    = 40,
 	},
-	.ap_num = 229,
-	.db_cnt = 40,
 };
 
 static int mt2712_pinctrl_probe(struct platform_device *pdev)
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt7622.c b/drivers/pinctrl/mediatek/pinctrl-mt7622.c
index 06e8406c4440..ad6da1184c9f 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt7622.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt7622.c
@@ -19,6 +19,7 @@
 #include <linux/init.h>
 #include <linux/mfd/syscon.h>
 #include <linux/of.h>
+#include <linux/of_irq.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/pinctrl/pinctrl.h>
@@ -30,6 +31,7 @@
 #include "../core.h"
 #include "../pinconf.h"
 #include "../pinmux.h"
+#include "mtk-eint.h"
 
 #define PINCTRL_PINCTRL_DEV		KBUILD_MODNAME
 #define MTK_RANGE(_a)		{ .range = (_a), .nranges = ARRAY_SIZE(_a), }
@@ -123,6 +125,8 @@ struct mtk_pin_soc {
 	unsigned int			ngrps;
 	const struct function_desc	*funcs;
 	unsigned int			nfuncs;
+	const struct mtk_eint_regs	*eint_regs;
+	const struct mtk_eint_hw	*eint_hw;
 };
 
 struct mtk_pinctrl {
@@ -131,6 +135,7 @@ struct mtk_pinctrl {
 	struct device			*dev;
 	struct gpio_chip		chip;
 	const struct mtk_pin_soc	*soc;
+	struct mtk_eint			*eint;
 };
 
 static const struct mtk_pin_field_calc mt7622_pin_mode_range[] = {
@@ -913,6 +918,13 @@ static const struct pin_config_item mtk_conf_items[] = {
 };
 #endif
 
+static const struct mtk_eint_hw mt7622_eint_hw = {
+	.port_mask = 7,
+	.ports     = 7,
+	.ap_num    = ARRAY_SIZE(mt7622_pins),
+	.db_cnt    = 20,
+};
+
 static const struct mtk_pin_soc mt7622_data = {
 	.reg_cal = mt7622_reg_cals,
 	.pins = mt7622_pins,
@@ -921,6 +933,7 @@ static const struct mtk_pin_soc mt7622_data = {
 	.ngrps = ARRAY_SIZE(mt7622_groups),
 	.funcs = mt7622_functions,
 	.nfuncs = ARRAY_SIZE(mt7622_functions),
+	.eint_hw = &mt7622_eint_hw,
 };
 
 static void mtk_w32(struct mtk_pinctrl *pctl, u32 reg, u32 val)
@@ -1441,6 +1454,32 @@ static int mtk_gpio_direction_output(struct gpio_chip *chip, unsigned int gpio,
 	return pinctrl_gpio_direction_output(chip->base + gpio);
 }
 
+static int mtk_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
+{
+	struct mtk_pinctrl *hw = gpiochip_get_data(chip);
+	unsigned long eint_n;
+
+	eint_n = offset;
+
+	return mtk_eint_find_irq(hw->eint, eint_n);
+}
+
+static int mtk_gpio_set_config(struct gpio_chip *chip, unsigned int offset,
+			       unsigned long config)
+{
+	struct mtk_pinctrl *hw = gpiochip_get_data(chip);
+	unsigned long eint_n;
+	u32 debounce;
+
+	if (pinconf_to_config_param(config) != PIN_CONFIG_INPUT_DEBOUNCE)
+		return -ENOTSUPP;
+
+	debounce = pinconf_to_config_argument(config);
+	eint_n = offset;
+
+	return mtk_eint_set_debounce(hw->eint, eint_n, debounce);
+}
+
 static int mtk_build_gpiochip(struct mtk_pinctrl *hw, struct device_node *np)
 {
 	struct gpio_chip *chip = &hw->chip;
@@ -1454,6 +1493,8 @@ static int mtk_build_gpiochip(struct mtk_pinctrl *hw, struct device_node *np)
 	chip->direction_output	= mtk_gpio_direction_output;
 	chip->get		= mtk_gpio_get;
 	chip->set		= mtk_gpio_set;
+	chip->to_irq		= mtk_gpio_to_irq,
+	chip->set_config	= mtk_gpio_set_config,
 	chip->base		= -1;
 	chip->ngpio		= hw->soc->npins;
 	chip->of_node		= np;
@@ -1514,6 +1555,103 @@ static int mtk_build_functions(struct mtk_pinctrl *hw)
 	return 0;
 }
 
+static int mtk_xt_get_gpio_n(void *data, unsigned long eint_n,
+			     unsigned int *gpio_n,
+			     struct gpio_chip **gpio_chip)
+{
+	struct mtk_pinctrl *hw = (struct mtk_pinctrl *)data;
+
+	*gpio_chip = &hw->chip;
+	*gpio_n = eint_n;
+
+	return 0;
+}
+
+static int mtk_xt_get_gpio_state(void *data, unsigned long eint_n)
+{
+	struct mtk_pinctrl *hw = (struct mtk_pinctrl *)data;
+	struct gpio_chip *gpio_chip;
+	unsigned int gpio_n;
+	int err;
+
+	err = mtk_xt_get_gpio_n(hw, eint_n, &gpio_n, &gpio_chip);
+	if (err)
+		return err;
+
+	return mtk_gpio_get(gpio_chip, gpio_n);
+}
+
+static int mtk_xt_set_gpio_as_eint(void *data, unsigned long eint_n)
+{
+	struct mtk_pinctrl *hw = (struct mtk_pinctrl *)data;
+	struct gpio_chip *gpio_chip;
+	unsigned int gpio_n;
+	int err;
+
+	err = mtk_xt_get_gpio_n(hw, eint_n, &gpio_n, &gpio_chip);
+	if (err)
+		return err;
+
+	err = mtk_hw_set_value(hw, gpio_n, PINCTRL_PIN_REG_MODE,
+			       MTK_GPIO_MODE);
+	if (err)
+		return err;
+
+	err = mtk_hw_set_value(hw, gpio_n, PINCTRL_PIN_REG_DIR, MTK_INPUT);
+	if (err)
+		return err;
+
+	err = mtk_hw_set_value(hw, gpio_n, PINCTRL_PIN_REG_SMT, MTK_ENABLE);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static const struct mtk_eint_xt mtk_eint_xt = {
+	.get_gpio_n = mtk_xt_get_gpio_n,
+	.get_gpio_state = mtk_xt_get_gpio_state,
+	.set_gpio_as_eint = mtk_xt_set_gpio_as_eint,
+};
+
+static int
+mtk_build_eint(struct mtk_pinctrl *hw, struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct resource *res;
+
+	if (!IS_ENABLED(CONFIG_EINT_MTK))
+		return 0;
+
+	if (!of_property_read_bool(np, "interrupt-controller"))
+		return -ENODEV;
+
+	hw->eint = devm_kzalloc(hw->dev, sizeof(*hw->eint), GFP_KERNEL);
+	if (!hw->eint)
+		return -ENOMEM;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "eint");
+	if (!res) {
+		dev_err(&pdev->dev, "Unable to get eint resource\n");
+		return -ENODEV;
+	}
+
+	hw->eint->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(hw->eint->base))
+		return PTR_ERR(hw->eint->base);
+
+	hw->eint->irq = irq_of_parse_and_map(np, 0);
+	if (!hw->eint->irq)
+		return -EINVAL;
+
+	hw->eint->dev = &pdev->dev;
+	hw->eint->hw = hw->soc->eint_hw;
+	hw->eint->pctl = hw;
+	hw->eint->gpio_xlate = &mtk_eint_xt;
+
+	return mtk_eint_do_init(hw->eint);
+}
+
 static const struct of_device_id mtk_pinctrl_of_match[] = {
 	{ .compatible = "mediatek,mt7622-pinctrl", .data = &mt7622_data},
 	{ }
@@ -1577,6 +1715,11 @@ static int mtk_pinctrl_probe(struct platform_device *pdev)
 		return err;
 	}
 
+	err = mtk_build_eint(hw, pdev);
+	if (err)
+		dev_warn(&pdev->dev,
+			 "Failed to add EINT, but pinctrl still can work\n");
+
 	platform_set_drvdata(pdev, hw);
 
 	return 0;
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8127.c b/drivers/pinctrl/mediatek/pinctrl-mt8127.c
index d76491574841..2e4cc9257e00 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt8127.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt8127.c
@@ -300,31 +300,12 @@ static const struct mtk_pinctrl_devdata mt8127_pinctrl_data = {
 	.port_shf = 4,
 	.port_mask = 0xf,
 	.port_align = 4,
-	.eint_offsets = {
-		.name = "mt8127_eint",
-		.stat      = 0x000,
-		.ack       = 0x040,
-		.mask      = 0x080,
-		.mask_set  = 0x0c0,
-		.mask_clr  = 0x100,
-		.sens      = 0x140,
-		.sens_set  = 0x180,
-		.sens_clr  = 0x1c0,
-		.soft      = 0x200,
-		.soft_set  = 0x240,
-		.soft_clr  = 0x280,
-		.pol       = 0x300,
-		.pol_set   = 0x340,
-		.pol_clr   = 0x380,
-		.dom_en    = 0x400,
-		.dbnc_ctrl = 0x500,
-		.dbnc_set  = 0x600,
-		.dbnc_clr  = 0x700,
+	.eint_hw = {
 		.port_mask = 7,
 		.ports     = 6,
+		.ap_num    = 143,
+		.db_cnt    = 16,
 	},
-	.ap_num = 143,
-	.db_cnt = 16,
 };
 
 static int mt8127_pinctrl_probe(struct platform_device *pdev)
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8135.c b/drivers/pinctrl/mediatek/pinctrl-mt8135.c
index d8c645f16f21..7f5edfaffdc5 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt8135.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt8135.c
@@ -313,31 +313,12 @@ static const struct mtk_pinctrl_devdata mt8135_pinctrl_data = {
 	.port_shf = 4,
 	.port_mask = 0xf,
 	.port_align = 4,
-	.eint_offsets = {
-		.name = "mt8135_eint",
-		.stat      = 0x000,
-		.ack       = 0x040,
-		.mask      = 0x080,
-		.mask_set  = 0x0c0,
-		.mask_clr  = 0x100,
-		.sens      = 0x140,
-		.sens_set  = 0x180,
-		.sens_clr  = 0x1c0,
-		.soft      = 0x200,
-		.soft_set  = 0x240,
-		.soft_clr  = 0x280,
-		.pol       = 0x300,
-		.pol_set   = 0x340,
-		.pol_clr   = 0x380,
-		.dom_en    = 0x400,
-		.dbnc_ctrl = 0x500,
-		.dbnc_set  = 0x600,
-		.dbnc_clr  = 0x700,
+	.eint_hw = {
 		.port_mask = 7,
 		.ports     = 6,
+		.ap_num    = 192,
+		.db_cnt    = 16,
 	},
-	.ap_num = 192,
-	.db_cnt = 16,
 };
 
 static int mt8135_pinctrl_probe(struct platform_device *pdev)
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8173.c b/drivers/pinctrl/mediatek/pinctrl-mt8173.c
index 8bfd427b9135..c449c9a043da 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt8173.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt8173.c
@@ -340,31 +340,12 @@ static const struct mtk_pinctrl_devdata mt8173_pinctrl_data = {
 	.port_shf = 4,
 	.port_mask = 0xf,
 	.port_align = 4,
-	.eint_offsets = {
-		.name = "mt8173_eint",
-		.stat      = 0x000,
-		.ack       = 0x040,
-		.mask      = 0x080,
-		.mask_set  = 0x0c0,
-		.mask_clr  = 0x100,
-		.sens      = 0x140,
-		.sens_set  = 0x180,
-		.sens_clr  = 0x1c0,
-		.soft      = 0x200,
-		.soft_set  = 0x240,
-		.soft_clr  = 0x280,
-		.pol       = 0x300,
-		.pol_set   = 0x340,
-		.pol_clr   = 0x380,
-		.dom_en    = 0x400,
-		.dbnc_ctrl = 0x500,
-		.dbnc_set  = 0x600,
-		.dbnc_clr  = 0x700,
+	.eint_hw = {
 		.port_mask = 7,
 		.ports     = 6,
+		.ap_num    = 224,
+		.db_cnt    = 16,
 	},
-	.ap_num = 224,
-	.db_cnt = 16,
 };
 
 static int mt8173_pinctrl_probe(struct platform_device *pdev)
diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
index c3975a04d1cd..b3799695d8db 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
@@ -38,6 +38,7 @@
 #include "../core.h"
 #include "../pinconf.h"
 #include "../pinctrl-utils.h"
+#include "mtk-eint.h"
 #include "pinctrl-mtk-common.h"
 
 #define MAX_GPIO_MODE_PER_REG 5
@@ -831,243 +832,38 @@ static int mtk_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static int mtk_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	const struct mtk_desc_pin *pin;
 	struct mtk_pinctrl *pctl = gpiochip_get_data(chip);
-	int irq;
-
-	pin = pctl->devdata->pins + offset;
-	if (pin->eint.eintnum == NO_EINT_SUPPORT)
-		return -EINVAL;
-
-	irq = irq_find_mapping(pctl->domain, pin->eint.eintnum);
-	if (!irq)
-		return -EINVAL;
-
-	return irq;
-}
-
-static int mtk_pinctrl_irq_request_resources(struct irq_data *d)
-{
-	struct mtk_pinctrl *pctl = irq_data_get_irq_chip_data(d);
-	const struct mtk_desc_pin *pin;
-	int ret;
-
-	pin = mtk_find_pin_by_eint_num(pctl, d->hwirq);
-
-	if (!pin) {
-		dev_err(pctl->dev, "Can not find pin\n");
-		return -EINVAL;
-	}
-
-	ret = gpiochip_lock_as_irq(pctl->chip, pin->pin.number);
-	if (ret) {
-		dev_err(pctl->dev, "unable to lock HW IRQ %lu for IRQ\n",
-			irqd_to_hwirq(d));
-		return ret;
-	}
-
-	/* set mux to INT mode */
-	mtk_pmx_set_mode(pctl->pctl_dev, pin->pin.number, pin->eint.eintmux);
-	/* set gpio direction to input */
-	mtk_pmx_gpio_set_direction(pctl->pctl_dev, NULL, pin->pin.number, true);
-	/* set input-enable */
-	mtk_pconf_set_ies_smt(pctl, pin->pin.number, 1, PIN_CONFIG_INPUT_ENABLE);
-
-	return 0;
-}
-
-static void mtk_pinctrl_irq_release_resources(struct irq_data *d)
-{
-	struct mtk_pinctrl *pctl = irq_data_get_irq_chip_data(d);
-	const struct mtk_desc_pin *pin;
-
-	pin = mtk_find_pin_by_eint_num(pctl, d->hwirq);
-
-	if (!pin) {
-		dev_err(pctl->dev, "Can not find pin\n");
-		return;
-	}
-
-	gpiochip_unlock_as_irq(pctl->chip, pin->pin.number);
-}
-
-static void __iomem *mtk_eint_get_offset(struct mtk_pinctrl *pctl,
-	unsigned int eint_num, unsigned int offset)
-{
-	unsigned int eint_base = 0;
-	void __iomem *reg;
-
-	if (eint_num >= pctl->devdata->ap_num)
-		eint_base = pctl->devdata->ap_num;
-
-	reg = pctl->eint_reg_base + offset + ((eint_num - eint_base) / 32) * 4;
-
-	return reg;
-}
-
-/*
- * mtk_can_en_debounce: Check the EINT number is able to enable debounce or not
- * @eint_num: the EINT number to setmtk_pinctrl
- */
-static unsigned int mtk_eint_can_en_debounce(struct mtk_pinctrl *pctl,
-	unsigned int eint_num)
-{
-	unsigned int sens;
-	unsigned int bit = BIT(eint_num % 32);
-	const struct mtk_eint_offsets *eint_offsets =
-		&pctl->devdata->eint_offsets;
-
-	void __iomem *reg = mtk_eint_get_offset(pctl, eint_num,
-			eint_offsets->sens);
-
-	if (readl(reg) & bit)
-		sens = MT_LEVEL_SENSITIVE;
-	else
-		sens = MT_EDGE_SENSITIVE;
-
-	if ((eint_num < pctl->devdata->db_cnt) && (sens != MT_EDGE_SENSITIVE))
-		return 1;
-	else
-		return 0;
-}
-
-/*
- * mtk_eint_get_mask: To get the eint mask
- * @eint_num: the EINT number to get
- */
-static unsigned int mtk_eint_get_mask(struct mtk_pinctrl *pctl,
-	unsigned int eint_num)
-{
-	unsigned int bit = BIT(eint_num % 32);
-	const struct mtk_eint_offsets *eint_offsets =
-		&pctl->devdata->eint_offsets;
-
-	void __iomem *reg = mtk_eint_get_offset(pctl, eint_num,
-			eint_offsets->mask);
-
-	return !!(readl(reg) & bit);
-}
-
-static int mtk_eint_flip_edge(struct mtk_pinctrl *pctl, int hwirq)
-{
-	int start_level, curr_level;
-	unsigned int reg_offset;
-	const struct mtk_eint_offsets *eint_offsets = &(pctl->devdata->eint_offsets);
-	u32 mask = BIT(hwirq & 0x1f);
-	u32 port = (hwirq >> 5) & eint_offsets->port_mask;
-	void __iomem *reg = pctl->eint_reg_base + (port << 2);
 	const struct mtk_desc_pin *pin;
-
-	pin = mtk_find_pin_by_eint_num(pctl, hwirq);
-	curr_level = mtk_gpio_get(pctl->chip, pin->pin.number);
-	do {
-		start_level = curr_level;
-		if (start_level)
-			reg_offset = eint_offsets->pol_clr;
-		else
-			reg_offset = eint_offsets->pol_set;
-		writel(mask, reg + reg_offset);
-
-		curr_level = mtk_gpio_get(pctl->chip, pin->pin.number);
-	} while (start_level != curr_level);
-
-	return start_level;
-}
-
-static void mtk_eint_mask(struct irq_data *d)
-{
-	struct mtk_pinctrl *pctl = irq_data_get_irq_chip_data(d);
-	const struct mtk_eint_offsets *eint_offsets =
-			&pctl->devdata->eint_offsets;
-	u32 mask = BIT(d->hwirq & 0x1f);
-	void __iomem *reg = mtk_eint_get_offset(pctl, d->hwirq,
-			eint_offsets->mask_set);
-
-	writel(mask, reg);
-}
-
-static void mtk_eint_unmask(struct irq_data *d)
-{
-	struct mtk_pinctrl *pctl = irq_data_get_irq_chip_data(d);
-	const struct mtk_eint_offsets *eint_offsets =
-		&pctl->devdata->eint_offsets;
-	u32 mask = BIT(d->hwirq & 0x1f);
-	void __iomem *reg = mtk_eint_get_offset(pctl, d->hwirq,
-			eint_offsets->mask_clr);
-
-	writel(mask, reg);
-
-	if (pctl->eint_dual_edges[d->hwirq])
-		mtk_eint_flip_edge(pctl, d->hwirq);
-}
-
-static int mtk_gpio_set_debounce(struct gpio_chip *chip, unsigned offset,
-	unsigned debounce)
-{
-	struct mtk_pinctrl *pctl = dev_get_drvdata(chip->parent);
-	int eint_num, virq, eint_offset;
-	unsigned int set_offset, bit, clr_bit, clr_offset, rst, i, unmask, dbnc;
-	static const unsigned int debounce_time[] = {500, 1000, 16000, 32000, 64000,
-						128000, 256000};
-	const struct mtk_desc_pin *pin;
-	struct irq_data *d;
+	unsigned long eint_n;
 
 	pin = pctl->devdata->pins + offset;
 	if (pin->eint.eintnum == NO_EINT_SUPPORT)
 		return -EINVAL;
 
-	eint_num = pin->eint.eintnum;
-	virq = irq_find_mapping(pctl->domain, eint_num);
-	eint_offset = (eint_num % 4) * 8;
-	d = irq_get_irq_data(virq);
+	eint_n = pin->eint.eintnum;
 
-	set_offset = (eint_num / 4) * 4 + pctl->devdata->eint_offsets.dbnc_set;
-	clr_offset = (eint_num / 4) * 4 + pctl->devdata->eint_offsets.dbnc_clr;
-	if (!mtk_eint_can_en_debounce(pctl, eint_num))
-		return -ENOSYS;
-
-	dbnc = ARRAY_SIZE(debounce_time);
-	for (i = 0; i < ARRAY_SIZE(debounce_time); i++) {
-		if (debounce <= debounce_time[i]) {
-			dbnc = i;
-			break;
-		}
-	}
-
-	if (!mtk_eint_get_mask(pctl, eint_num)) {
-		mtk_eint_mask(d);
-		unmask = 1;
-	} else {
-		unmask = 0;
-	}
-
-	clr_bit = 0xff << eint_offset;
-	writel(clr_bit, pctl->eint_reg_base + clr_offset);
-
-	bit = ((dbnc << EINT_DBNC_SET_DBNC_BITS) | EINT_DBNC_SET_EN) <<
-		eint_offset;
-	rst = EINT_DBNC_RST_BIT << eint_offset;
-	writel(rst | bit, pctl->eint_reg_base + set_offset);
-
-	/* Delay a while (more than 2T) to wait for hw debounce counter reset
-	work correctly */
-	udelay(1);
-	if (unmask == 1)
-		mtk_eint_unmask(d);
-
-	return 0;
+	return mtk_eint_find_irq(pctl->eint, eint_n);
 }
 
 static int mtk_gpio_set_config(struct gpio_chip *chip, unsigned offset,
 			       unsigned long config)
 {
+	struct mtk_pinctrl *pctl = gpiochip_get_data(chip);
+	const struct mtk_desc_pin *pin;
+	unsigned long eint_n;
 	u32 debounce;
 
 	if (pinconf_to_config_param(config) != PIN_CONFIG_INPUT_DEBOUNCE)
 		return -ENOTSUPP;
 
+	pin = pctl->devdata->pins + offset;
+	if (pin->eint.eintnum == NO_EINT_SUPPORT)
+		return -EINVAL;
+
 	debounce = pinconf_to_config_argument(config);
-	return mtk_gpio_set_debounce(chip, offset, debounce);
+	eint_n = pin->eint.eintnum;
+
+	return mtk_eint_set_debounce(pctl->eint, eint_n, debounce);
 }
 
 static const struct gpio_chip mtk_gpio_chip = {
@@ -1084,117 +880,18 @@ static const struct gpio_chip mtk_gpio_chip = {
 	.of_gpio_n_cells	= 2,
 };
 
-static int mtk_eint_set_type(struct irq_data *d,
-				      unsigned int type)
-{
-	struct mtk_pinctrl *pctl = irq_data_get_irq_chip_data(d);
-	const struct mtk_eint_offsets *eint_offsets =
-		&pctl->devdata->eint_offsets;
-	u32 mask = BIT(d->hwirq & 0x1f);
-	void __iomem *reg;
-
-	if (((type & IRQ_TYPE_EDGE_BOTH) && (type & IRQ_TYPE_LEVEL_MASK)) ||
-		((type & IRQ_TYPE_LEVEL_MASK) == IRQ_TYPE_LEVEL_MASK)) {
-		dev_err(pctl->dev, "Can't configure IRQ%d (EINT%lu) for type 0x%X\n",
-			d->irq, d->hwirq, type);
-		return -EINVAL;
-	}
-
-	if ((type & IRQ_TYPE_EDGE_BOTH) == IRQ_TYPE_EDGE_BOTH)
-		pctl->eint_dual_edges[d->hwirq] = 1;
-	else
-		pctl->eint_dual_edges[d->hwirq] = 0;
-
-	if (type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING)) {
-		reg = mtk_eint_get_offset(pctl, d->hwirq,
-			eint_offsets->pol_clr);
-		writel(mask, reg);
-	} else {
-		reg = mtk_eint_get_offset(pctl, d->hwirq,
-			eint_offsets->pol_set);
-		writel(mask, reg);
-	}
-
-	if (type & (IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING)) {
-		reg = mtk_eint_get_offset(pctl, d->hwirq,
-			eint_offsets->sens_clr);
-		writel(mask, reg);
-	} else {
-		reg = mtk_eint_get_offset(pctl, d->hwirq,
-			eint_offsets->sens_set);
-		writel(mask, reg);
-	}
-
-	if (pctl->eint_dual_edges[d->hwirq])
-		mtk_eint_flip_edge(pctl, d->hwirq);
-
-	return 0;
-}
-
-static int mtk_eint_irq_set_wake(struct irq_data *d, unsigned int on)
-{
-	struct mtk_pinctrl *pctl = irq_data_get_irq_chip_data(d);
-	int shift = d->hwirq & 0x1f;
-	int reg = d->hwirq >> 5;
-
-	if (on)
-		pctl->wake_mask[reg] |= BIT(shift);
-	else
-		pctl->wake_mask[reg] &= ~BIT(shift);
-
-	return 0;
-}
-
-static void mtk_eint_chip_write_mask(const struct mtk_eint_offsets *chip,
-		void __iomem *eint_reg_base, u32 *buf)
-{
-	int port;
-	void __iomem *reg;
-
-	for (port = 0; port < chip->ports; port++) {
-		reg = eint_reg_base + (port << 2);
-		writel_relaxed(~buf[port], reg + chip->mask_set);
-		writel_relaxed(buf[port], reg + chip->mask_clr);
-	}
-}
-
-static void mtk_eint_chip_read_mask(const struct mtk_eint_offsets *chip,
-		void __iomem *eint_reg_base, u32 *buf)
-{
-	int port;
-	void __iomem *reg;
-
-	for (port = 0; port < chip->ports; port++) {
-		reg = eint_reg_base + chip->mask + (port << 2);
-		buf[port] = ~readl_relaxed(reg);
-		/* Mask is 0 when irq is enabled, and 1 when disabled. */
-	}
-}
-
 static int mtk_eint_suspend(struct device *device)
 {
-	void __iomem *reg;
 	struct mtk_pinctrl *pctl = dev_get_drvdata(device);
-	const struct mtk_eint_offsets *eint_offsets =
-			&pctl->devdata->eint_offsets;
-
-	reg = pctl->eint_reg_base;
-	mtk_eint_chip_read_mask(eint_offsets, reg, pctl->cur_mask);
-	mtk_eint_chip_write_mask(eint_offsets, reg, pctl->wake_mask);
 
-	return 0;
+	return mtk_eint_do_suspend(pctl->eint);
 }
 
 static int mtk_eint_resume(struct device *device)
 {
 	struct mtk_pinctrl *pctl = dev_get_drvdata(device);
-	const struct mtk_eint_offsets *eint_offsets =
-			&pctl->devdata->eint_offsets;
-
-	mtk_eint_chip_write_mask(eint_offsets,
-			pctl->eint_reg_base, pctl->cur_mask);
 
-	return 0;
+	return mtk_eint_do_resume(pctl->eint);
 }
 
 const struct dev_pm_ops mtk_eint_pm_ops = {
@@ -1202,117 +899,6 @@ const struct dev_pm_ops mtk_eint_pm_ops = {
 	.resume_noirq = mtk_eint_resume,
 };
 
-static void mtk_eint_ack(struct irq_data *d)
-{
-	struct mtk_pinctrl *pctl = irq_data_get_irq_chip_data(d);
-	const struct mtk_eint_offsets *eint_offsets =
-		&pctl->devdata->eint_offsets;
-	u32 mask = BIT(d->hwirq & 0x1f);
-	void __iomem *reg = mtk_eint_get_offset(pctl, d->hwirq,
-			eint_offsets->ack);
-
-	writel(mask, reg);
-}
-
-static struct irq_chip mtk_pinctrl_irq_chip = {
-	.name = "mt-eint",
-	.irq_disable = mtk_eint_mask,
-	.irq_mask = mtk_eint_mask,
-	.irq_unmask = mtk_eint_unmask,
-	.irq_ack = mtk_eint_ack,
-	.irq_set_type = mtk_eint_set_type,
-	.irq_set_wake = mtk_eint_irq_set_wake,
-	.irq_request_resources = mtk_pinctrl_irq_request_resources,
-	.irq_release_resources = mtk_pinctrl_irq_release_resources,
-};
-
-static unsigned int mtk_eint_init(struct mtk_pinctrl *pctl)
-{
-	const struct mtk_eint_offsets *eint_offsets =
-		&pctl->devdata->eint_offsets;
-	void __iomem *reg = pctl->eint_reg_base + eint_offsets->dom_en;
-	unsigned int i;
-
-	for (i = 0; i < pctl->devdata->ap_num; i += 32) {
-		writel(0xffffffff, reg);
-		reg += 4;
-	}
-	return 0;
-}
-
-static inline void
-mtk_eint_debounce_process(struct mtk_pinctrl *pctl, int index)
-{
-	unsigned int rst, ctrl_offset;
-	unsigned int bit, dbnc;
-	const struct mtk_eint_offsets *eint_offsets =
-		&pctl->devdata->eint_offsets;
-
-	ctrl_offset = (index / 4) * 4 + eint_offsets->dbnc_ctrl;
-	dbnc = readl(pctl->eint_reg_base + ctrl_offset);
-	bit = EINT_DBNC_SET_EN << ((index % 4) * 8);
-	if ((bit & dbnc) > 0) {
-		ctrl_offset = (index / 4) * 4 + eint_offsets->dbnc_set;
-		rst = EINT_DBNC_RST_BIT << ((index % 4) * 8);
-		writel(rst, pctl->eint_reg_base + ctrl_offset);
-	}
-}
-
-static void mtk_eint_irq_handler(struct irq_desc *desc)
-{
-	struct irq_chip *chip = irq_desc_get_chip(desc);
-	struct mtk_pinctrl *pctl = irq_desc_get_handler_data(desc);
-	unsigned int status, eint_num;
-	int offset, index, virq;
-	const struct mtk_eint_offsets *eint_offsets =
-		&pctl->devdata->eint_offsets;
-	void __iomem *reg =  mtk_eint_get_offset(pctl, 0, eint_offsets->stat);
-	int dual_edges, start_level, curr_level;
-	const struct mtk_desc_pin *pin;
-
-	chained_irq_enter(chip, desc);
-	for (eint_num = 0;
-	     eint_num < pctl->devdata->ap_num;
-	     eint_num += 32, reg += 4) {
-		status = readl(reg);
-		while (status) {
-			offset = __ffs(status);
-			index = eint_num + offset;
-			virq = irq_find_mapping(pctl->domain, index);
-			status &= ~BIT(offset);
-
-			dual_edges = pctl->eint_dual_edges[index];
-			if (dual_edges) {
-				/* Clear soft-irq in case we raised it
-				   last time */
-				writel(BIT(offset), reg - eint_offsets->stat +
-					eint_offsets->soft_clr);
-
-				pin = mtk_find_pin_by_eint_num(pctl, index);
-				start_level = mtk_gpio_get(pctl->chip,
-							   pin->pin.number);
-			}
-
-			generic_handle_irq(virq);
-
-			if (dual_edges) {
-				curr_level = mtk_eint_flip_edge(pctl, index);
-
-				/* If level changed, we might lost one edge
-				   interrupt, raised it through soft-irq */
-				if (start_level != curr_level)
-					writel(BIT(offset), reg -
-						eint_offsets->stat +
-						eint_offsets->soft_set);
-			}
-
-			if (index < pctl->devdata->db_cnt)
-				mtk_eint_debounce_process(pctl , index);
-		}
-	}
-	chained_irq_exit(chip, desc);
-}
-
 static int mtk_pctrl_build_state(struct platform_device *pdev)
 {
 	struct mtk_pinctrl *pctl = platform_get_drvdata(pdev);
@@ -1345,6 +931,101 @@ static int mtk_pctrl_build_state(struct platform_device *pdev)
 	return 0;
 }
 
+static int
+mtk_xt_get_gpio_n(void *data, unsigned long eint_n, unsigned int *gpio_n,
+		  struct gpio_chip **gpio_chip)
+{
+	struct mtk_pinctrl *pctl = (struct mtk_pinctrl *)data;
+	const struct mtk_desc_pin *pin;
+
+	pin = mtk_find_pin_by_eint_num(pctl, eint_n);
+	if (!pin)
+		return -EINVAL;
+
+	*gpio_chip = pctl->chip;
+	*gpio_n = pin->pin.number;
+
+	return 0;
+}
+
+static int mtk_xt_get_gpio_state(void *data, unsigned long eint_n)
+{
+	struct mtk_pinctrl *pctl = (struct mtk_pinctrl *)data;
+	const struct mtk_desc_pin *pin;
+
+	pin = mtk_find_pin_by_eint_num(pctl, eint_n);
+	if (!pin)
+		return -EINVAL;
+
+	return mtk_gpio_get(pctl->chip, pin->pin.number);
+}
+
+static int mtk_xt_set_gpio_as_eint(void *data, unsigned long eint_n)
+{
+	struct mtk_pinctrl *pctl = (struct mtk_pinctrl *)data;
+	const struct mtk_desc_pin *pin;
+
+	pin = mtk_find_pin_by_eint_num(pctl, eint_n);
+	if (!pin)
+		return -EINVAL;
+
+	/* set mux to INT mode */
+	mtk_pmx_set_mode(pctl->pctl_dev, pin->pin.number, pin->eint.eintmux);
+	/* set gpio direction to input */
+	mtk_pmx_gpio_set_direction(pctl->pctl_dev, NULL, pin->pin.number,
+				   true);
+	/* set input-enable */
+	mtk_pconf_set_ies_smt(pctl, pin->pin.number, 1,
+			      PIN_CONFIG_INPUT_ENABLE);
+
+	return 0;
+}
+
+static const struct mtk_eint_xt mtk_eint_xt = {
+	.get_gpio_n = mtk_xt_get_gpio_n,
+	.get_gpio_state = mtk_xt_get_gpio_state,
+	.set_gpio_as_eint = mtk_xt_set_gpio_as_eint,
+};
+
+static int mtk_eint_init(struct mtk_pinctrl *pctl, struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct resource *res;
+
+	if (!of_property_read_bool(np, "interrupt-controller"))
+		return -ENODEV;
+
+	pctl->eint = devm_kzalloc(pctl->dev, sizeof(*pctl->eint), GFP_KERNEL);
+	if (!pctl->eint)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(&pdev->dev, "Unable to get eint resource\n");
+		return -ENODEV;
+	}
+
+	pctl->eint->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(pctl->eint->base))
+		return PTR_ERR(pctl->eint->base);
+
+	pctl->eint->irq = irq_of_parse_and_map(np, 0);
+	if (!pctl->eint->irq)
+		return -EINVAL;
+
+	pctl->eint->dev = &pdev->dev;
+	/*
+	 * If pctl->eint->regs == NULL, it would fall back into using a generic
+	 * register map in mtk_eint_do_init calls.
+	 */
+	pctl->eint->regs = pctl->devdata->eint_regs;
+	pctl->eint->hw = &pctl->devdata->eint_hw;
+	pctl->eint->pctl = pctl;
+	pctl->eint->gpio_xlate = &mtk_eint_xt;
+
+	return mtk_eint_do_init(pctl->eint);
+}
+
 int mtk_pctrl_init(struct platform_device *pdev,
 		const struct mtk_pinctrl_devdata *data,
 		struct regmap *regmap)
@@ -1353,8 +1034,7 @@ int mtk_pctrl_init(struct platform_device *pdev,
 	struct mtk_pinctrl *pctl;
 	struct device_node *np = pdev->dev.of_node, *node;
 	struct property *prop;
-	struct resource *res;
-	int i, ret, irq, ports_buf;
+	int ret, i;
 
 	pctl = devm_kzalloc(&pdev->dev, sizeof(*pctl), GFP_KERNEL);
 	if (!pctl)
@@ -1441,70 +1121,10 @@ int mtk_pctrl_init(struct platform_device *pdev,
 		goto chip_error;
 	}
 
-	if (!of_property_read_bool(np, "interrupt-controller"))
-		return 0;
-
-	/* Get EINT register base from dts. */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		dev_err(&pdev->dev, "Unable to get Pinctrl resource\n");
-		ret = -EINVAL;
-		goto chip_error;
-	}
-
-	pctl->eint_reg_base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(pctl->eint_reg_base)) {
-		ret = -EINVAL;
-		goto chip_error;
-	}
-
-	ports_buf = pctl->devdata->eint_offsets.ports;
-	pctl->wake_mask = devm_kcalloc(&pdev->dev, ports_buf,
-					sizeof(*pctl->wake_mask), GFP_KERNEL);
-	if (!pctl->wake_mask) {
-		ret = -ENOMEM;
-		goto chip_error;
-	}
-
-	pctl->cur_mask = devm_kcalloc(&pdev->dev, ports_buf,
-					sizeof(*pctl->cur_mask), GFP_KERNEL);
-	if (!pctl->cur_mask) {
-		ret = -ENOMEM;
-		goto chip_error;
-	}
-
-	pctl->eint_dual_edges = devm_kcalloc(&pdev->dev, pctl->devdata->ap_num,
-					     sizeof(int), GFP_KERNEL);
-	if (!pctl->eint_dual_edges) {
-		ret = -ENOMEM;
-		goto chip_error;
-	}
-
-	irq = irq_of_parse_and_map(np, 0);
-	if (!irq) {
-		dev_err(&pdev->dev, "couldn't parse and map irq\n");
-		ret = -EINVAL;
-		goto chip_error;
-	}
-
-	pctl->domain = irq_domain_add_linear(np,
-		pctl->devdata->ap_num, &irq_domain_simple_ops, NULL);
-	if (!pctl->domain) {
-		dev_err(&pdev->dev, "Couldn't register IRQ domain\n");
-		ret = -ENOMEM;
+	ret = mtk_eint_init(pctl, pdev);
+	if (ret)
 		goto chip_error;
-	}
-
-	mtk_eint_init(pctl);
-	for (i = 0; i < pctl->devdata->ap_num; i++) {
-		int virq = irq_create_mapping(pctl->domain, i);
-
-		irq_set_chip_and_handler(virq, &mtk_pinctrl_irq_chip,
-			handle_level_irq);
-		irq_set_chip_data(virq, pctl);
-	}
 
-	irq_set_chained_handler_and_data(irq, mtk_eint_irq_handler, pctl);
 	return 0;
 
 chip_error:
diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.h b/drivers/pinctrl/mediatek/pinctrl-mtk-common.h
index 8543bc478a1e..bf13eb0a68d6 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.h
+++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.h
@@ -19,6 +19,8 @@
 #include <linux/regmap.h>
 #include <linux/pinctrl/pinconf-generic.h>
 
+#include "mtk-eint.h"
+
 #define NO_EINT_SUPPORT    255
 #define MT_EDGE_SENSITIVE           0
 #define MT_LEVEL_SENSITIVE          1
@@ -258,9 +260,8 @@ struct mtk_pinctrl_devdata {
 	unsigned char  port_shf;
 	unsigned char  port_mask;
 	unsigned char  port_align;
-	struct mtk_eint_offsets eint_offsets;
-	unsigned int	ap_num;
-	unsigned int	db_cnt;
+	struct mtk_eint_hw eint_hw;
+	struct mtk_eint_regs *eint_regs;
 };
 
 struct mtk_pinctrl {
@@ -274,11 +275,7 @@ struct mtk_pinctrl {
 	const char          **grp_names;
 	struct pinctrl_dev      *pctl_dev;
 	const struct mtk_pinctrl_devdata  *devdata;
-	void __iomem		*eint_reg_base;
-	struct irq_domain	*domain;
-	int			*eint_dual_edges;
-	u32 *wake_mask;
-	u32 *cur_mask;
+	struct mtk_eint *eint;
 };
 
 int mtk_pctrl_init(struct platform_device *pdev,
diff --git a/drivers/pinctrl/meson/pinctrl-meson-axg.c b/drivers/pinctrl/meson/pinctrl-meson-axg.c
index 99a6ceac8e53..46a0918bd284 100644
--- a/drivers/pinctrl/meson/pinctrl-meson-axg.c
+++ b/drivers/pinctrl/meson/pinctrl-meson-axg.c
@@ -312,6 +312,47 @@ static const unsigned int pdm_din1_pins[] = {GPIOA_16};
 static const unsigned int pdm_din2_pins[] = {GPIOA_17};
 static const unsigned int pdm_din3_pins[] = {GPIOA_18};
 
+/* mclk */
+static const unsigned int mclk_c_pins[] = {GPIOA_0};
+static const unsigned int mclk_b_pins[] = {GPIOA_1};
+
+/* tdm */
+static const unsigned int tdma_sclk_pins[] = {GPIOX_12};
+static const unsigned int tdma_sclk_slv_pins[] = {GPIOX_12};
+static const unsigned int tdma_fs_pins[] = {GPIOX_13};
+static const unsigned int tdma_fs_slv_pins[] = {GPIOX_13};
+static const unsigned int tdma_din0_pins[] = {GPIOX_14};
+static const unsigned int tdma_dout0_x14_pins[] = {GPIOX_14};
+static const unsigned int tdma_dout0_x15_pins[] = {GPIOX_15};
+static const unsigned int tdma_dout1_pins[] = {GPIOX_15};
+static const unsigned int tdma_din1_pins[] = {GPIOX_15};
+
+static const unsigned int tdmc_sclk_pins[] = {GPIOA_2};
+static const unsigned int tdmc_sclk_slv_pins[] = {GPIOA_2};
+static const unsigned int tdmc_fs_pins[] = {GPIOA_3};
+static const unsigned int tdmc_fs_slv_pins[] = {GPIOA_3};
+static const unsigned int tdmc_din0_pins[] = {GPIOA_4};
+static const unsigned int tdmc_dout0_pins[] = {GPIOA_4};
+static const unsigned int tdmc_din1_pins[] = {GPIOA_5};
+static const unsigned int tdmc_dout1_pins[] = {GPIOA_5};
+static const unsigned int tdmc_din2_pins[] = {GPIOA_6};
+static const unsigned int tdmc_dout2_pins[] = {GPIOA_6};
+static const unsigned int tdmc_din3_pins[] = {GPIOA_7};
+static const unsigned int tdmc_dout3_pins[] = {GPIOA_7};
+
+static const unsigned int tdmb_sclk_pins[] = {GPIOA_8};
+static const unsigned int tdmb_sclk_slv_pins[] = {GPIOA_8};
+static const unsigned int tdmb_fs_pins[] = {GPIOA_9};
+static const unsigned int tdmb_fs_slv_pins[] = {GPIOA_9};
+static const unsigned int tdmb_din0_pins[] = {GPIOA_10};
+static const unsigned int tdmb_dout0_pins[] = {GPIOA_10};
+static const unsigned int tdmb_din1_pins[] = {GPIOA_11};
+static const unsigned int tdmb_dout1_pins[] = {GPIOA_11};
+static const unsigned int tdmb_din2_pins[] = {GPIOA_12};
+static const unsigned int tdmb_dout2_pins[] = {GPIOA_12};
+static const unsigned int tdmb_din3_pins[] = {GPIOA_13};
+static const unsigned int tdmb_dout3_pins[] = {GPIOA_13};
+
 static struct meson_pmx_group meson_axg_periphs_groups[] = {
 	GPIO_GROUP(GPIOZ_0),
 	GPIO_GROUP(GPIOZ_1),
@@ -495,6 +536,15 @@ static struct meson_pmx_group meson_axg_periphs_groups[] = {
 	GROUP(eth_rx_dv_x, 4),
 	GROUP(eth_mdio_x, 4),
 	GROUP(eth_mdc_x, 4),
+	GROUP(tdma_sclk, 1),
+	GROUP(tdma_sclk_slv, 2),
+	GROUP(tdma_fs, 1),
+	GROUP(tdma_fs_slv, 2),
+	GROUP(tdma_din0, 1),
+	GROUP(tdma_dout0_x14, 2),
+	GROUP(tdma_dout0_x15, 1),
+	GROUP(tdma_dout1, 2),
+	GROUP(tdma_din1, 3),
 
 	/* bank GPIOY */
 	GROUP(eth_txd0_y, 1),
@@ -544,6 +594,32 @@ static struct meson_pmx_group meson_axg_periphs_groups[] = {
 	GROUP(pdm_din1, 1),
 	GROUP(pdm_din2, 1),
 	GROUP(pdm_din3, 1),
+	GROUP(mclk_c, 1),
+	GROUP(mclk_b, 1),
+	GROUP(tdmc_sclk, 1),
+	GROUP(tdmc_sclk_slv, 2),
+	GROUP(tdmc_fs, 1),
+	GROUP(tdmc_fs_slv, 2),
+	GROUP(tdmc_din0, 2),
+	GROUP(tdmc_dout0, 1),
+	GROUP(tdmc_din1, 2),
+	GROUP(tdmc_dout1, 1),
+	GROUP(tdmc_din2, 2),
+	GROUP(tdmc_dout2, 1),
+	GROUP(tdmc_din3, 2),
+	GROUP(tdmc_dout3, 1),
+	GROUP(tdmb_sclk, 1),
+	GROUP(tdmb_sclk_slv, 2),
+	GROUP(tdmb_fs, 1),
+	GROUP(tdmb_fs_slv, 2),
+	GROUP(tdmb_din0, 2),
+	GROUP(tdmb_dout0, 1),
+	GROUP(tdmb_din1, 2),
+	GROUP(tdmb_dout1, 1),
+	GROUP(tdmb_din2, 2),
+	GROUP(tdmb_dout2, 1),
+	GROUP(tdmb_din3, 2),
+	GROUP(tdmb_dout3, 1),
 };
 
 /* uart_ao_a */
@@ -845,6 +921,32 @@ static const char * const jtag_ao_groups[] = {
 	"jtag_ao_tdi", "jtag_ao_tdo", "jtag_ao_clk", "jtag_ao_tms",
 };
 
+static const char * const mclk_c_groups[] = {
+	"mclk_c",
+};
+
+static const char * const mclk_b_groups[] = {
+	"mclk_b",
+};
+
+static const char * const tdma_groups[] = {
+	"tdma_sclk", "tdma_sclk_slv", "tdma_fs", "tdma_fs_slv",
+	"tdma_din0", "tdma_dout0_x14", "tdma_dout0_x15", "tdma_dout1",
+	"tdma_din1",
+};
+
+static const char * const tdmc_groups[] = {
+	"tdmc_sclk", "tdmc_sclk_slv", "tdmc_fs", "tdmc_fs_slv",
+	"tdmc_din0", "tdmc_dout0", "tdmc_din1",	"tdmc_dout1",
+	"tdmc_din2", "tdmc_dout2", "tdmc_din3",	"tdmc_dout3",
+};
+
+static const char * const tdmb_groups[] = {
+	"tdmb_sclk", "tdmb_sclk_slv", "tdmb_fs", "tdmb_fs_slv",
+	"tdmb_din0", "tdmb_dout0", "tdmb_din1",	"tdmb_dout1",
+	"tdmb_din2", "tdmb_dout2", "tdmb_din3",	"tdmb_dout3",
+};
+
 static struct meson_pmx_func meson_axg_periphs_functions[] = {
 	FUNCTION(gpio_periphs),
 	FUNCTION(emmc),
@@ -870,6 +972,11 @@ static struct meson_pmx_func meson_axg_periphs_functions[] = {
 	FUNCTION(spdif_in),
 	FUNCTION(jtag_ee),
 	FUNCTION(pdm),
+	FUNCTION(mclk_b),
+	FUNCTION(mclk_c),
+	FUNCTION(tdma),
+	FUNCTION(tdmb),
+	FUNCTION(tdmc),
 };
 
 static struct meson_pmx_func meson_axg_aobus_functions[] = {
diff --git a/drivers/pinctrl/meson/pinctrl-meson-gxbb.c b/drivers/pinctrl/meson/pinctrl-meson-gxbb.c
index 9079020259c5..2c97a2e07a5f 100644
--- a/drivers/pinctrl/meson/pinctrl-meson-gxbb.c
+++ b/drivers/pinctrl/meson/pinctrl-meson-gxbb.c
@@ -627,8 +627,8 @@ static const char * const sdio_groups[] = {
 };
 
 static const char * const nand_groups[] = {
-	"nand_ce0", "nand_ce1", "nand_rb0", "nand_ale", "nand_cle",
-	"nand_wen_clk", "nand_ren_wr", "nand_dqs",
+	"emmc_nand_d07", "nand_ce0", "nand_ce1", "nand_rb0", "nand_ale",
+	"nand_cle", "nand_wen_clk", "nand_ren_wr", "nand_dqs",
 };
 
 static const char * const uart_a_groups[] = {
diff --git a/drivers/pinctrl/meson/pinctrl-meson-gxl.c b/drivers/pinctrl/meson/pinctrl-meson-gxl.c
index b3786cde963d..7dae1d7bf6b0 100644
--- a/drivers/pinctrl/meson/pinctrl-meson-gxl.c
+++ b/drivers/pinctrl/meson/pinctrl-meson-gxl.c
@@ -617,8 +617,8 @@ static const char * const sdio_groups[] = {
 };
 
 static const char * const nand_groups[] = {
-	"nand_ce0", "nand_ce1", "nand_rb0", "nand_ale", "nand_cle",
-	"nand_wen_clk", "nand_ren_wr", "nand_dqs",
+	"emmc_nand_d07", "nand_ce0", "nand_ce1", "nand_rb0", "nand_ale",
+	"nand_cle", "nand_wen_clk", "nand_ren_wr", "nand_dqs",
 };
 
 static const char * const uart_a_groups[] = {
diff --git a/drivers/pinctrl/meson/pinctrl-meson8.c b/drivers/pinctrl/meson/pinctrl-meson8.c
index 49c7ce03547b..c6d79315218f 100644
--- a/drivers/pinctrl/meson/pinctrl-meson8.c
+++ b/drivers/pinctrl/meson/pinctrl-meson8.c
@@ -1,5 +1,5 @@
 /*
- * Pin controller and GPIO driver for Amlogic Meson8.
+ * Pin controller and GPIO driver for Amlogic Meson8 and Meson8m2.
  *
  * Copyright (C) 2014 Beniamino Galvani <b.galvani@gmail.com>
  *
@@ -299,6 +299,10 @@ static const unsigned int spi_mosi_1_pins[]	= { GPIOZ_12 };
 static const unsigned int spi_miso_1_pins[]	= { GPIOZ_13 };
 static const unsigned int spi_ss2_1_pins[]	= { GPIOZ_14 };
 
+static const unsigned int eth_txd3_pins[]	= { GPIOZ_0 };
+static const unsigned int eth_txd2_pins[]	= { GPIOZ_1 };
+static const unsigned int eth_rxd3_pins[]	= { GPIOZ_2 };
+static const unsigned int eth_rxd2_pins[]	= { GPIOZ_3 };
 static const unsigned int eth_tx_clk_50m_pins[]	= { GPIOZ_4 };
 static const unsigned int eth_tx_en_pins[]	= { GPIOZ_5 };
 static const unsigned int eth_txd1_pins[]	= { GPIOZ_6 };
@@ -650,6 +654,12 @@ static struct meson_pmx_group meson8_cbus_groups[] = {
 	GROUP(eth_mdio,		6,	6),
 	GROUP(eth_mdc,		6,	5),
 
+	/* NOTE: the following four groups are only available on Meson8m2: */
+	GROUP(eth_rxd2,		6,	3),
+	GROUP(eth_rxd3,		6,	2),
+	GROUP(eth_txd2,		6,	1),
+	GROUP(eth_txd3,		6,	0),
+
 	GROUP(i2c_sda_a0,	5,	31),
 	GROUP(i2c_sck_a0,	5,	30),
 
@@ -877,7 +887,8 @@ static const char * const spi_groups[] = {
 static const char * const ethernet_groups[] = {
 	"eth_tx_clk_50m", "eth_tx_en", "eth_txd1",
 	"eth_txd0", "eth_rx_clk_in", "eth_rx_dv",
-	"eth_rxd1", "eth_rxd0", "eth_mdio", "eth_mdc"
+	"eth_rxd1", "eth_rxd0", "eth_mdio", "eth_mdc", "eth_rxd2",
+	"eth_rxd3", "eth_txd2", "eth_txd3"
 };
 
 static const char * const i2c_a_groups[] = {
@@ -1080,6 +1091,14 @@ static const struct of_device_id meson8_pinctrl_dt_match[] = {
 		.compatible = "amlogic,meson8-aobus-pinctrl",
 		.data = &meson8_aobus_pinctrl_data,
 	},
+	{
+		.compatible = "amlogic,meson8m2-cbus-pinctrl",
+		.data = &meson8_cbus_pinctrl_data,
+	},
+	{
+		.compatible = "amlogic,meson8m2-aobus-pinctrl",
+		.data = &meson8_aobus_pinctrl_data,
+	},
 	{ },
 };
 
diff --git a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c
index 5b63248c8209..674ffdf8103c 100644
--- a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c
+++ b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c
@@ -214,18 +214,6 @@ static inline void armada_37xx_update_reg(unsigned int *reg,
 	}
 }
 
-static int armada_37xx_get_func_reg(struct armada_37xx_pin_group *grp,
-				    const char *func)
-{
-	int f;
-
-	for (f = 0; (f < NB_FUNCS) && grp->funcs[f]; f++)
-		if (!strcmp(grp->funcs[f], func))
-			return f;
-
-	return -ENOTSUPP;
-}
-
 static struct armada_37xx_pin_group *armada_37xx_find_next_grp_by_pin(
 	struct armada_37xx_pinctrl *info, int pin, int *grp)
 {
@@ -344,10 +332,9 @@ static int armada_37xx_pmx_set_by_name(struct pinctrl_dev *pctldev,
 	dev_dbg(info->dev, "enable function %s group %s\n",
 		name, grp->name);
 
-	func = armada_37xx_get_func_reg(grp, name);
-
+	func = match_string(grp->funcs, NB_FUNCS, name);
 	if (func < 0)
-		return func;
+		return -ENOTSUPP;
 
 	val = grp->val[func];
 
@@ -679,12 +666,13 @@ static void armada_37xx_irq_handler(struct irq_desc *desc)
 					writel(1 << hwirq,
 					       info->base +
 					       IRQ_STATUS + 4 * i);
-					continue;
+					goto update_status;
 				}
 			}
 
 			generic_handle_irq(virq);
 
+update_status:
 			/* Update status in case a new IRQ appears */
 			spin_lock_irqsave(&info->irq_lock, flags);
 			status = readl_relaxed(info->base +
@@ -932,12 +920,12 @@ static int armada_37xx_fill_func(struct armada_37xx_pinctrl *info)
 			struct armada_37xx_pin_group *gp = &info->groups[g];
 			int f;
 
-			for (f = 0; (f < NB_FUNCS) && gp->funcs[f]; f++) {
-				if (strcmp(gp->funcs[f], name) == 0) {
-					*groups = gp->name;
-					groups++;
-				}
-			}
+			f = match_string(gp->funcs, NB_FUNCS, name);
+			if (f < 0)
+				continue;
+
+			*groups = gp->name;
+			groups++;
 		}
 	}
 	return 0;
diff --git a/drivers/pinctrl/mvebu/pinctrl-armada-xp.c b/drivers/pinctrl/mvebu/pinctrl-armada-xp.c
index b854f1ee5de5..5e828468e43d 100644
--- a/drivers/pinctrl/mvebu/pinctrl-armada-xp.c
+++ b/drivers/pinctrl/mvebu/pinctrl-armada-xp.c
@@ -431,40 +431,40 @@ static struct mvebu_mpp_mode mv98dx3236_mpp_modes[] = {
 	MPP_MODE(19,
 		 MPP_VAR_FUNCTION(0x0, "gpio", NULL,         V_98DX3236_PLUS),
 		 MPP_VAR_FUNCTION(0x3, "uart1", "rxd",       V_98DX3236_PLUS),
-		 MPP_VAR_FUNCTION(0x4, "dev", "rb",          V_98DX3236_PLUS)),
+		 MPP_VAR_FUNCTION(0x4, "nand", "rb",         V_98DX3236_PLUS)),
 	MPP_MODE(20,
 		 MPP_VAR_FUNCTION(0x0, "gpo", NULL,          V_98DX3236_PLUS),
 		 MPP_VAR_FUNCTION(0x4, "dev", "we0",         V_98DX3236_PLUS)),
 	MPP_MODE(21,
 		 MPP_VAR_FUNCTION(0x0, "gpo", NULL,          V_98DX3236_PLUS),
-		 MPP_VAR_FUNCTION(0x1, "dev", "ad0",         V_98DX3236_PLUS)),
+		 MPP_VAR_FUNCTION(0x4, "dev", "ad0",         V_98DX3236_PLUS)),
 	MPP_MODE(22,
 		 MPP_VAR_FUNCTION(0x0, "gpo", NULL,          V_98DX3236_PLUS),
-		 MPP_VAR_FUNCTION(0x1, "dev", "ad1",         V_98DX3236_PLUS)),
+		 MPP_VAR_FUNCTION(0x4, "dev", "ad1",         V_98DX3236_PLUS)),
 	MPP_MODE(23,
 		 MPP_VAR_FUNCTION(0x0, "gpo", NULL,          V_98DX3236_PLUS),
-		 MPP_VAR_FUNCTION(0x1, "dev", "ad2",         V_98DX3236_PLUS)),
+		 MPP_VAR_FUNCTION(0x4, "dev", "ad2",         V_98DX3236_PLUS)),
 	MPP_MODE(24,
 		 MPP_VAR_FUNCTION(0x0, "gpo", NULL,          V_98DX3236_PLUS),
-		 MPP_VAR_FUNCTION(0x1, "dev", "ad3",         V_98DX3236_PLUS)),
+		 MPP_VAR_FUNCTION(0x4, "dev", "ad3",         V_98DX3236_PLUS)),
 	MPP_MODE(25,
 		 MPP_VAR_FUNCTION(0x0, "gpo", NULL,          V_98DX3236_PLUS),
-		 MPP_VAR_FUNCTION(0x1, "dev", "ad4",         V_98DX3236_PLUS)),
+		 MPP_VAR_FUNCTION(0x4, "dev", "ad4",         V_98DX3236_PLUS)),
 	MPP_MODE(26,
 		 MPP_VAR_FUNCTION(0x0, "gpo", NULL,          V_98DX3236_PLUS),
-		 MPP_VAR_FUNCTION(0x1, "dev", "ad5",         V_98DX3236_PLUS)),
+		 MPP_VAR_FUNCTION(0x4, "dev", "ad5",         V_98DX3236_PLUS)),
 	MPP_MODE(27,
 		 MPP_VAR_FUNCTION(0x0, "gpo", NULL,          V_98DX3236_PLUS),
-		 MPP_VAR_FUNCTION(0x1, "dev", "ad6",         V_98DX3236_PLUS)),
+		 MPP_VAR_FUNCTION(0x4, "dev", "ad6",         V_98DX3236_PLUS)),
 	MPP_MODE(28,
 		 MPP_VAR_FUNCTION(0x0, "gpo", NULL,          V_98DX3236_PLUS),
-		 MPP_VAR_FUNCTION(0x1, "dev", "ad7",         V_98DX3236_PLUS)),
+		 MPP_VAR_FUNCTION(0x4, "dev", "ad7",         V_98DX3236_PLUS)),
 	MPP_MODE(29,
 		 MPP_VAR_FUNCTION(0x0, "gpo", NULL,          V_98DX3236_PLUS),
-		 MPP_VAR_FUNCTION(0x1, "dev", "a0",          V_98DX3236_PLUS)),
+		 MPP_VAR_FUNCTION(0x4, "dev", "a0",          V_98DX3236_PLUS)),
 	MPP_MODE(30,
 		 MPP_VAR_FUNCTION(0x0, "gpo", NULL,          V_98DX3236_PLUS),
-		 MPP_VAR_FUNCTION(0x1, "dev", "a1",          V_98DX3236_PLUS)),
+		 MPP_VAR_FUNCTION(0x4, "dev", "a1",          V_98DX3236_PLUS)),
 	MPP_MODE(31,
 		 MPP_VAR_FUNCTION(0x0, "gpio", NULL,         V_98DX3236_PLUS),
 		 MPP_VAR_FUNCTION(0x1, "slv_smi", "mdc",     V_98DX3236_PLUS),
diff --git a/drivers/pinctrl/pinctrl-at91-pio4.c b/drivers/pinctrl/pinctrl-at91-pio4.c
index 4b57a13758a4..bafb3d40545e 100644
--- a/drivers/pinctrl/pinctrl-at91-pio4.c
+++ b/drivers/pinctrl/pinctrl-at91-pio4.c
@@ -576,8 +576,10 @@ static int atmel_pctl_dt_node_to_map(struct pinctrl_dev *pctldev,
 		for_each_child_of_node(np_config, np) {
 			ret = atmel_pctl_dt_subnode_to_map(pctldev, np, map,
 						    &reserved_maps, num_maps);
-			if (ret < 0)
+			if (ret < 0) {
+				of_node_put(np);
 				break;
+			}
 		}
 	}
 
diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c
index 3924779f5578..1882713e68f9 100644
--- a/drivers/pinctrl/pinctrl-rockchip.c
+++ b/drivers/pinctrl/pinctrl-rockchip.c
@@ -59,6 +59,7 @@
 #define GPIO_LS_SYNC		0x60
 
 enum rockchip_pinctrl_type {
+	PX30,
 	RV1108,
 	RK2928,
 	RK3066B,
@@ -701,6 +702,66 @@ static void rockchip_get_recalced_mux(struct rockchip_pin_bank *bank, int pin,
 	*bit = data->bit;
 }
 
+static struct rockchip_mux_route_data px30_mux_route_data[] = {
+	{
+		/* cif-d2m0 */
+		.bank_num = 2,
+		.pin = 0,
+		.func = 1,
+		.route_offset = 0x184,
+		.route_val = BIT(16 + 7),
+	}, {
+		/* cif-d2m1 */
+		.bank_num = 3,
+		.pin = 3,
+		.func = 3,
+		.route_offset = 0x184,
+		.route_val = BIT(16 + 7) | BIT(7),
+	}, {
+		/* pdm-m0 */
+		.bank_num = 3,
+		.pin = 22,
+		.func = 2,
+		.route_offset = 0x184,
+		.route_val = BIT(16 + 8),
+	}, {
+		/* pdm-m1 */
+		.bank_num = 2,
+		.pin = 22,
+		.func = 1,
+		.route_offset = 0x184,
+		.route_val = BIT(16 + 8) | BIT(8),
+	}, {
+		/* uart2-rxm0 */
+		.bank_num = 1,
+		.pin = 27,
+		.func = 2,
+		.route_offset = 0x184,
+		.route_val = BIT(16 + 10),
+	}, {
+		/* uart2-rxm1 */
+		.bank_num = 2,
+		.pin = 14,
+		.func = 2,
+		.route_offset = 0x184,
+		.route_val = BIT(16 + 10) | BIT(10),
+	}, {
+		/* uart3-rxm0 */
+		.bank_num = 0,
+		.pin = 17,
+		.func = 2,
+		.route_offset = 0x184,
+		.route_val = BIT(16 + 9),
+	}, {
+		/* uart3-rxm1 */
+		.bank_num = 1,
+		.pin = 15,
+		.func = 2,
+		.route_offset = 0x184,
+		.route_val = BIT(16 + 9) | BIT(9),
+	},
+};
+
 static struct rockchip_mux_route_data rk3128_mux_route_data[] = {
 	{
 		/* spi-0 */
@@ -1202,6 +1263,97 @@ static int rockchip_set_mux(struct rockchip_pin_bank *bank, int pin, int mux)
 	return ret;
 }
 
+#define PX30_PULL_PMU_OFFSET		0x10
+#define PX30_PULL_GRF_OFFSET		0x60
+#define PX30_PULL_BITS_PER_PIN		2
+#define PX30_PULL_PINS_PER_REG		8
+#define PX30_PULL_BANK_STRIDE		16
+
+static void px30_calc_pull_reg_and_bit(struct rockchip_pin_bank *bank,
+				       int pin_num, struct regmap **regmap,
+				       int *reg, u8 *bit)
+{
+	struct rockchip_pinctrl *info = bank->drvdata;
+
+	/* The first 32 pins of the first bank are located in PMU */
+	if (bank->bank_num == 0) {
+		*regmap = info->regmap_pmu;
+		*reg = PX30_PULL_PMU_OFFSET;
+	} else {
+		*regmap = info->regmap_base;
+		*reg = PX30_PULL_GRF_OFFSET;
+
+		/* correct the offset, as we're starting with the 2nd bank */
+		*reg -= 0x10;
+		*reg += bank->bank_num * PX30_PULL_BANK_STRIDE;
+	}
+
+	*reg += ((pin_num / PX30_PULL_PINS_PER_REG) * 4);
+	*bit = (pin_num % PX30_PULL_PINS_PER_REG);
+	*bit *= PX30_PULL_BITS_PER_PIN;
+}
+
+#define PX30_DRV_PMU_OFFSET		0x20
+#define PX30_DRV_GRF_OFFSET		0xf0
+#define PX30_DRV_BITS_PER_PIN		2
+#define PX30_DRV_PINS_PER_REG		8
+#define PX30_DRV_BANK_STRIDE		16
+
+static void px30_calc_drv_reg_and_bit(struct rockchip_pin_bank *bank,
+				      int pin_num, struct regmap **regmap,
+				      int *reg, u8 *bit)
+{
+	struct rockchip_pinctrl *info = bank->drvdata;
+
+	/* The first 32 pins of the first bank are located in PMU */
+	if (bank->bank_num == 0) {
+		*regmap = info->regmap_pmu;
+		*reg = PX30_DRV_PMU_OFFSET;
+	} else {
+		*regmap = info->regmap_base;
+		*reg = PX30_DRV_GRF_OFFSET;
+
+		/* correct the offset, as we're starting with the 2nd bank */
+		*reg -= 0x10;
+		*reg += bank->bank_num * PX30_DRV_BANK_STRIDE;
+	}
+
+	*reg += ((pin_num / PX30_DRV_PINS_PER_REG) * 4);
+	*bit = (pin_num % PX30_DRV_PINS_PER_REG);
+	*bit *= PX30_DRV_BITS_PER_PIN;
+}
+
+#define PX30_SCHMITT_PMU_OFFSET			0x38
+#define PX30_SCHMITT_GRF_OFFSET			0xc0
+#define PX30_SCHMITT_PINS_PER_PMU_REG		16
+#define PX30_SCHMITT_BANK_STRIDE		16
+#define PX30_SCHMITT_PINS_PER_GRF_REG		8
+
+static int px30_calc_schmitt_reg_and_bit(struct rockchip_pin_bank *bank,
+					 int pin_num,
+					 struct regmap **regmap,
+					 int *reg, u8 *bit)
+{
+	struct rockchip_pinctrl *info = bank->drvdata;
+	int pins_per_reg;
+
+	if (bank->bank_num == 0) {
+		*regmap = info->regmap_pmu;
+		*reg = PX30_SCHMITT_PMU_OFFSET;
+		pins_per_reg = PX30_SCHMITT_PINS_PER_PMU_REG;
+	} else {
+		*regmap = info->regmap_base;
+		*reg = PX30_SCHMITT_GRF_OFFSET;
+		pins_per_reg = PX30_SCHMITT_PINS_PER_GRF_REG;
+		*reg += (bank->bank_num  - 1) * PX30_SCHMITT_BANK_STRIDE;
+	}
+
+	*reg += ((pin_num / pins_per_reg) * 4);
+	*bit = pin_num % pins_per_reg;
+
+	return 0;
+}
+
 #define RV1108_PULL_PMU_OFFSET		0x10
 #define RV1108_PULL_OFFSET		0x110
 #define RV1108_PULL_PINS_PER_REG	8
@@ -1798,6 +1950,7 @@ static int rockchip_get_pull(struct rockchip_pin_bank *bank, int pin_num)
 		return !(data & BIT(bit))
 				? PIN_CONFIG_BIAS_PULL_PIN_DEFAULT
 				: PIN_CONFIG_BIAS_DISABLE;
+	case PX30:
 	case RV1108:
 	case RK3188:
 	case RK3288:
@@ -1841,6 +1994,7 @@ static int rockchip_set_pull(struct rockchip_pin_bank *bank,
 			data |= BIT(bit);
 		ret = regmap_write(regmap, reg, data);
 		break;
+	case PX30:
 	case RV1108:
 	case RK3188:
 	case RK3288:
@@ -2103,6 +2257,7 @@ static bool rockchip_pinconf_pull_valid(struct rockchip_pin_ctrl *ctrl,
 					pull == PIN_CONFIG_BIAS_DISABLE);
 	case RK3066B:
 		return pull ? false : true;
+	case PX30:
 	case RV1108:
 	case RK3188:
 	case RK3288:
@@ -2555,6 +2710,57 @@ static int rockchip_gpio_direction_output(struct gpio_chip *gc,
 	return pinctrl_gpio_direction_output(gc->base + offset);
 }
 
+static void rockchip_gpio_set_debounce(struct gpio_chip *gc,
+				       unsigned int offset, bool enable)
+{
+	struct rockchip_pin_bank *bank = gpiochip_get_data(gc);
+	void __iomem *reg = bank->reg_base + GPIO_DEBOUNCE;
+	unsigned long flags;
+	u32 data;
+
+	clk_enable(bank->clk);
+	raw_spin_lock_irqsave(&bank->slock, flags);
+
+	data = readl(reg);
+	if (enable)
+		data |= BIT(offset);
+	else
+		data &= ~BIT(offset);
+	writel(data, reg);
+
+	raw_spin_unlock_irqrestore(&bank->slock, flags);
+	clk_disable(bank->clk);
+}
+
+/*
+ * gpiolib set_config callback function. The setting of the pin
+ * mux function as 'gpio output' will be handled by the pinctrl subsystem
+ * interface.
+ */
+static int rockchip_gpio_set_config(struct gpio_chip *gc, unsigned int offset,
+				  unsigned long config)
+{
+	enum pin_config_param param = pinconf_to_config_param(config);
+
+	switch (param) {
+	case PIN_CONFIG_INPUT_DEBOUNCE:
+		rockchip_gpio_set_debounce(gc, offset, true);
+		/*
+		 * Rockchip's gpio could only support up to one period
+		 * of the debounce clock(pclk), which is far away from
+		 * satisftying the requirement, as pclk is usually near
+		 * 100MHz shared by all peripherals. So the fact is it
+		 * has crippled debounce capability could only be useful
+		 * to prevent any spurious glitches from waking up the system
+		 * if the gpio is conguired as wakeup interrupt source. Let's
+		 * still return -ENOTSUPP as before, to make sure the caller
+		 * of gpiod_set_debounce won't change its behaviour.
+		 */
+	default:
+		return -ENOTSUPP;
+	}
+}
+
 /*
  * gpiolib gpio_to_irq callback function. Creates a mapping between a GPIO pin
  * and a virtual IRQ, if not already present.
@@ -2580,6 +2786,7 @@ static const struct gpio_chip rockchip_gpiolib_chip = {
 	.get_direction	= rockchip_gpio_get_direction,
 	.direction_input = rockchip_gpio_direction_input,
 	.direction_output = rockchip_gpio_direction_output,
+	.set_config = rockchip_gpio_set_config,
 	.to_irq = rockchip_gpio_to_irq,
 	.owner = THIS_MODULE,
 };
@@ -3237,6 +3444,43 @@ static int rockchip_pinctrl_probe(struct platform_device *pdev)
 	return 0;
 }
 
+static struct rockchip_pin_bank px30_pin_banks[] = {
+	PIN_BANK_IOMUX_FLAGS(0, 32, "gpio0", IOMUX_SOURCE_PMU,
+					     IOMUX_SOURCE_PMU,
+					     IOMUX_SOURCE_PMU,
+					     IOMUX_SOURCE_PMU
+			    ),
+	PIN_BANK_IOMUX_FLAGS(1, 32, "gpio1", IOMUX_WIDTH_4BIT,
+					     IOMUX_WIDTH_4BIT,
+					     IOMUX_WIDTH_4BIT,
+					     IOMUX_WIDTH_4BIT
+			    ),
+	PIN_BANK_IOMUX_FLAGS(2, 32, "gpio2", IOMUX_WIDTH_4BIT,
+					     IOMUX_WIDTH_4BIT,
+					     IOMUX_WIDTH_4BIT,
+					     IOMUX_WIDTH_4BIT
+			    ),
+	PIN_BANK_IOMUX_FLAGS(3, 32, "gpio3", IOMUX_WIDTH_4BIT,
+					     IOMUX_WIDTH_4BIT,
+					     IOMUX_WIDTH_4BIT,
+					     IOMUX_WIDTH_4BIT
+			    ),
+};
+
+static struct rockchip_pin_ctrl px30_pin_ctrl = {
+		.pin_banks		= px30_pin_banks,
+		.nr_banks		= ARRAY_SIZE(px30_pin_banks),
+		.label			= "PX30-GPIO",
+		.type			= PX30,
+		.grf_mux_offset		= 0x0,
+		.pmu_mux_offset		= 0x0,
+		.iomux_routes		= px30_mux_route_data,
+		.niomux_routes		= ARRAY_SIZE(px30_mux_route_data),
+		.pull_calc_reg		= px30_calc_pull_reg_and_bit,
+		.drv_calc_reg		= px30_calc_drv_reg_and_bit,
+		.schmitt_calc_reg	= px30_calc_schmitt_reg_and_bit,
+};
+
 static struct rockchip_pin_bank rv1108_pin_banks[] = {
 	PIN_BANK_IOMUX_FLAGS(0, 32, "gpio0", IOMUX_SOURCE_PMU,
 					     IOMUX_SOURCE_PMU,
@@ -3545,6 +3789,8 @@ static struct rockchip_pin_ctrl rk3399_pin_ctrl = {
 };
 
 static const struct of_device_id rockchip_pinctrl_dt_match[] = {
+	{ .compatible = "rockchip,px30-pinctrl",
+		.data = &px30_pin_ctrl },
 	{ .compatible = "rockchip,rv1108-pinctrl",
 		.data = &rv1108_pin_ctrl },
 	{ .compatible = "rockchip,rk2928-pinctrl",
diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c
index a7c5eb39b1eb..9c3c00515aa0 100644
--- a/drivers/pinctrl/pinctrl-single.c
+++ b/drivers/pinctrl/pinctrl-single.c
@@ -144,6 +144,7 @@ struct pcs_soc_data {
  * struct pcs_device - pinctrl device instance
  * @res:	resources
  * @base:	virtual address of the controller
+ * @saved_vals: saved values for the controller
  * @size:	size of the ioremapped area
  * @dev:	device entry
  * @np:		device tree node
@@ -172,11 +173,13 @@ struct pcs_soc_data {
 struct pcs_device {
 	struct resource *res;
 	void __iomem *base;
+	void *saved_vals;
 	unsigned size;
 	struct device *dev;
 	struct device_node *np;
 	struct pinctrl_dev *pctl;
 	unsigned flags;
+#define PCS_CONTEXT_LOSS_OFF	(1 << 3)
 #define PCS_QUIRK_SHARED_IRQ	(1 << 2)
 #define PCS_FEAT_IRQ		(1 << 1)
 #define PCS_FEAT_PINCONF	(1 << 0)
@@ -1576,6 +1579,67 @@ static int pcs_irq_init_chained_handler(struct pcs_device *pcs,
 }
 
 #ifdef CONFIG_PM
+static int pcs_save_context(struct pcs_device *pcs)
+{
+	int i, mux_bytes;
+	u64 *regsl;
+	u32 *regsw;
+	u16 *regshw;
+
+	mux_bytes = pcs->width / BITS_PER_BYTE;
+
+	if (!pcs->saved_vals)
+		pcs->saved_vals = devm_kzalloc(pcs->dev, pcs->size, GFP_ATOMIC);
+
+	switch (pcs->width) {
+	case 64:
+		regsl = (u64 *)pcs->saved_vals;
+		for (i = 0; i < pcs->size / mux_bytes; i++)
+			regsl[i] = pcs->read(pcs->base + i * mux_bytes);
+		break;
+	case 32:
+		regsw = (u32 *)pcs->saved_vals;
+		for (i = 0; i < pcs->size / mux_bytes; i++)
+			regsw[i] = pcs->read(pcs->base + i * mux_bytes);
+		break;
+	case 16:
+		regshw = (u16 *)pcs->saved_vals;
+		for (i = 0; i < pcs->size / mux_bytes; i++)
+			regshw[i] = pcs->read(pcs->base + i * mux_bytes);
+		break;
+	}
+
+	return 0;
+}
+
+static void pcs_restore_context(struct pcs_device *pcs)
+{
+	int i, mux_bytes;
+	u64 *regsl;
+	u32 *regsw;
+	u16 *regshw;
+
+	mux_bytes = pcs->width / BITS_PER_BYTE;
+
+	switch (pcs->width) {
+	case 64:
+		regsl = (u64 *)pcs->saved_vals;
+		for (i = 0; i < pcs->size / mux_bytes; i++)
+			pcs->write(regsl[i], pcs->base + i * mux_bytes);
+		break;
+	case 32:
+		regsw = (u32 *)pcs->saved_vals;
+		for (i = 0; i < pcs->size / mux_bytes; i++)
+			pcs->write(regsw[i], pcs->base + i * mux_bytes);
+		break;
+	case 16:
+		regshw = (u16 *)pcs->saved_vals;
+		for (i = 0; i < pcs->size / mux_bytes; i++)
+			pcs->write(regshw[i], pcs->base + i * mux_bytes);
+		break;
+	}
+}
+
 static int pinctrl_single_suspend(struct platform_device *pdev,
 					pm_message_t state)
 {
@@ -1585,6 +1649,9 @@ static int pinctrl_single_suspend(struct platform_device *pdev,
 	if (!pcs)
 		return -EINVAL;
 
+	if (pcs->flags & PCS_CONTEXT_LOSS_OFF)
+		pcs_save_context(pcs);
+
 	return pinctrl_force_sleep(pcs->pctl);
 }
 
@@ -1596,6 +1663,9 @@ static int pinctrl_single_resume(struct platform_device *pdev)
 	if (!pcs)
 		return -EINVAL;
 
+	if (pcs->flags & PCS_CONTEXT_LOSS_OFF)
+		pcs_restore_context(pcs);
+
 	return pinctrl_force_default(pcs->pctl);
 }
 #endif
@@ -1824,7 +1894,7 @@ static const struct pcs_soc_data pinctrl_single_dra7 = {
 };
 
 static const struct pcs_soc_data pinctrl_single_am437x = {
-	.flags = PCS_QUIRK_SHARED_IRQ,
+	.flags = PCS_QUIRK_SHARED_IRQ | PCS_CONTEXT_LOSS_OFF,
 	.irq_enable_mask = (1 << 29),   /* OMAP_WAKEUP_EN */
 	.irq_status_mask = (1 << 30),   /* OMAP_WAKEUP_EVENT */
 };
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c
index ad80a17c9990..0e22f52b2a19 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm.c
@@ -58,7 +58,10 @@ struct msm_pinctrl {
 	struct device *dev;
 	struct pinctrl_dev *pctrl;
 	struct gpio_chip chip;
+	struct pinctrl_desc desc;
 	struct notifier_block restart_nb;
+
+	struct irq_chip irq_chip;
 	int irq;
 
 	raw_spinlock_t lock;
@@ -390,13 +393,6 @@ static const struct pinconf_ops msm_pinconf_ops = {
 	.pin_config_group_set	= msm_config_group_set,
 };
 
-static struct pinctrl_desc msm_pinctrl_desc = {
-	.pctlops = &msm_pinctrl_ops,
-	.pmxops = &msm_pinmux_ops,
-	.confops = &msm_pinconf_ops,
-	.owner = THIS_MODULE,
-};
-
 static int msm_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 {
 	const struct msm_pingroup *g;
@@ -506,29 +502,46 @@ static void msm_gpio_dbg_show_one(struct seq_file *s,
 	int is_out;
 	int drive;
 	int pull;
-	u32 ctl_reg;
+	int val;
+	u32 ctl_reg, io_reg;
 
-	static const char * const pulls[] = {
+	static const char * const pulls_keeper[] = {
 		"no pull",
 		"pull down",
 		"keeper",
 		"pull up"
 	};
 
+	static const char * const pulls_no_keeper[] = {
+		"no pull",
+		"pull down",
+		"pull up",
+	};
+
 	if (!gpiochip_line_is_valid(chip, offset))
 		return;
 
 	g = &pctrl->soc->groups[offset];
 	ctl_reg = readl(pctrl->regs + g->ctl_reg);
+	io_reg = readl(pctrl->regs + g->io_reg);
 
 	is_out = !!(ctl_reg & BIT(g->oe_bit));
 	func = (ctl_reg >> g->mux_bit) & 7;
 	drive = (ctl_reg >> g->drv_bit) & 7;
 	pull = (ctl_reg >> g->pull_bit) & 3;
 
-	seq_printf(s, " %-8s: %-3s %d", g->name, is_out ? "out" : "in", func);
+	if (is_out)
+		val = !!(io_reg & BIT(g->out_bit));
+	else
+		val = !!(io_reg & BIT(g->in_bit));
+
+	seq_printf(s, " %-8s: %-3s", g->name, is_out ? "out" : "in");
+	seq_printf(s, " %-4s func%d", val ? "high" : "low", func);
 	seq_printf(s, " %dmA", msm_regval_to_drive(drive));
-	seq_printf(s, " %s", pulls[pull]);
+	if (pctrl->soc->pull_no_keeper)
+		seq_printf(s, " %s", pulls_no_keeper[pull]);
+	else
+		seq_printf(s, " %s", pulls_keeper[pull]);
 	seq_puts(s, "\n");
 }
 
@@ -776,15 +789,6 @@ static int msm_gpio_irq_set_wake(struct irq_data *d, unsigned int on)
 	return 0;
 }
 
-static struct irq_chip msm_gpio_irq_chip = {
-	.name           = "msmgpio",
-	.irq_mask       = msm_gpio_irq_mask,
-	.irq_unmask     = msm_gpio_irq_unmask,
-	.irq_ack        = msm_gpio_irq_ack,
-	.irq_set_type   = msm_gpio_irq_set_type,
-	.irq_set_wake   = msm_gpio_irq_set_wake,
-};
-
 static void msm_gpio_irq_handler(struct irq_desc *desc)
 {
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
@@ -877,6 +881,13 @@ static int msm_gpio_init(struct msm_pinctrl *pctrl)
 	chip->of_node = pctrl->dev->of_node;
 	chip->need_valid_mask = msm_gpio_needs_valid_mask(pctrl);
 
+	pctrl->irq_chip.name = "msmgpio";
+	pctrl->irq_chip.irq_mask = msm_gpio_irq_mask;
+	pctrl->irq_chip.irq_unmask = msm_gpio_irq_unmask;
+	pctrl->irq_chip.irq_ack = msm_gpio_irq_ack;
+	pctrl->irq_chip.irq_set_type = msm_gpio_irq_set_type;
+	pctrl->irq_chip.irq_set_wake = msm_gpio_irq_set_wake;
+
 	ret = gpiochip_add_data(&pctrl->chip, pctrl);
 	if (ret) {
 		dev_err(pctrl->dev, "Failed register gpiochip\n");
@@ -890,15 +901,28 @@ static int msm_gpio_init(struct msm_pinctrl *pctrl)
 		return ret;
 	}
 
-	ret = gpiochip_add_pin_range(&pctrl->chip, dev_name(pctrl->dev), 0, 0, chip->ngpio);
-	if (ret) {
-		dev_err(pctrl->dev, "Failed to add pin range\n");
-		gpiochip_remove(&pctrl->chip);
-		return ret;
+	/*
+	 * For DeviceTree-supported systems, the gpio core checks the
+	 * pinctrl's device node for the "gpio-ranges" property.
+	 * If it is present, it takes care of adding the pin ranges
+	 * for the driver. In this case the driver can skip ahead.
+	 *
+	 * In order to remain compatible with older, existing DeviceTree
+	 * files which don't set the "gpio-ranges" property or systems that
+	 * utilize ACPI the driver has to call gpiochip_add_pin_range().
+	 */
+	if (!of_property_read_bool(pctrl->dev->of_node, "gpio-ranges")) {
+		ret = gpiochip_add_pin_range(&pctrl->chip,
+			dev_name(pctrl->dev), 0, 0, chip->ngpio);
+		if (ret) {
+			dev_err(pctrl->dev, "Failed to add pin range\n");
+			gpiochip_remove(&pctrl->chip);
+			return ret;
+		}
 	}
 
 	ret = gpiochip_irqchip_add(chip,
-				   &msm_gpio_irq_chip,
+				   &pctrl->irq_chip,
 				   0,
 				   handle_edge_irq,
 				   IRQ_TYPE_NONE);
@@ -908,7 +932,7 @@ static int msm_gpio_init(struct msm_pinctrl *pctrl)
 		return -ENOSYS;
 	}
 
-	gpiochip_set_chained_irqchip(chip, &msm_gpio_irq_chip, pctrl->irq,
+	gpiochip_set_chained_irqchip(chip, &pctrl->irq_chip, pctrl->irq,
 				     msm_gpio_irq_handler);
 
 	return 0;
@@ -979,11 +1003,15 @@ int msm_pinctrl_probe(struct platform_device *pdev,
 		return pctrl->irq;
 	}
 
-	msm_pinctrl_desc.name = dev_name(&pdev->dev);
-	msm_pinctrl_desc.pins = pctrl->soc->pins;
-	msm_pinctrl_desc.npins = pctrl->soc->npins;
-	pctrl->pctrl = devm_pinctrl_register(&pdev->dev, &msm_pinctrl_desc,
-					     pctrl);
+	pctrl->desc.owner = THIS_MODULE;
+	pctrl->desc.pctlops = &msm_pinctrl_ops;
+	pctrl->desc.pmxops = &msm_pinmux_ops;
+	pctrl->desc.confops = &msm_pinconf_ops;
+	pctrl->desc.name = dev_name(&pdev->dev);
+	pctrl->desc.pins = pctrl->soc->pins;
+	pctrl->desc.npins = pctrl->soc->npins;
+
+	pctrl->pctrl = devm_pinctrl_register(&pdev->dev, &pctrl->desc, pctrl);
 	if (IS_ERR(pctrl->pctrl)) {
 		dev_err(&pdev->dev, "Couldn't register pinctrl driver\n");
 		return PTR_ERR(pctrl->pctrl);
diff --git a/drivers/pinctrl/qcom/pinctrl-qdf2xxx.c b/drivers/pinctrl/qcom/pinctrl-qdf2xxx.c
index bb3ce5c3e18b..1dfbe42dd895 100644
--- a/drivers/pinctrl/qcom/pinctrl-qdf2xxx.c
+++ b/drivers/pinctrl/qcom/pinctrl-qdf2xxx.c
@@ -30,9 +30,7 @@
 
 #include "pinctrl-msm.h"
 
-static struct msm_pinctrl_soc_data qdf2xxx_pinctrl;
-
-/* A reasonable limit to the number of GPIOS */
+/* A maximum of 256 allows us to use a u8 array to hold the GPIO numbers */
 #define MAX_GPIOS	256
 
 /* maximum size of each gpio name (enough room for "gpioXXX" + null) */
@@ -40,77 +38,111 @@ static struct msm_pinctrl_soc_data qdf2xxx_pinctrl;
 
 static int qdf2xxx_pinctrl_probe(struct platform_device *pdev)
 {
+	struct msm_pinctrl_soc_data *pinctrl;
 	struct pinctrl_pin_desc *pins;
 	struct msm_pingroup *groups;
 	char (*names)[NAME_SIZE];
 	unsigned int i;
 	u32 num_gpios;
+	unsigned int avail_gpios; /* The number of GPIOs we support */
+	u8 gpios[MAX_GPIOS];      /* An array of supported GPIOs */
 	int ret;
 
 	/* Query the number of GPIOs from ACPI */
 	ret = device_property_read_u32(&pdev->dev, "num-gpios", &num_gpios);
 	if (ret < 0) {
-		dev_warn(&pdev->dev, "missing num-gpios property\n");
+		dev_err(&pdev->dev, "missing 'num-gpios' property\n");
 		return ret;
 	}
-
 	if (!num_gpios || num_gpios > MAX_GPIOS) {
-		dev_warn(&pdev->dev, "invalid num-gpios property\n");
+		dev_err(&pdev->dev, "invalid 'num-gpios' property\n");
+		return -ENODEV;
+	}
+
+	/* The number of GPIOs in the approved list */
+	ret = device_property_read_u8_array(&pdev->dev, "gpios", NULL, 0);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "missing 'gpios' property\n");
+		return ret;
+	}
+	/*
+	 * The number of available GPIOs should be non-zero, and no
+	 * more than the total number of GPIOS.
+	 */
+	if (!ret || ret > num_gpios) {
+		dev_err(&pdev->dev, "invalid 'gpios' property\n");
 		return -ENODEV;
 	}
+	avail_gpios = ret;
 
+	ret = device_property_read_u8_array(&pdev->dev, "gpios", gpios,
+					    avail_gpios);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "could not read list of GPIOs\n");
+		return ret;
+	}
+
+	pinctrl = devm_kzalloc(&pdev->dev, sizeof(*pinctrl), GFP_KERNEL);
 	pins = devm_kcalloc(&pdev->dev, num_gpios,
 		sizeof(struct pinctrl_pin_desc), GFP_KERNEL);
 	groups = devm_kcalloc(&pdev->dev, num_gpios,
 		sizeof(struct msm_pingroup), GFP_KERNEL);
-	names = devm_kcalloc(&pdev->dev, num_gpios, NAME_SIZE, GFP_KERNEL);
+	names = devm_kcalloc(&pdev->dev, avail_gpios, NAME_SIZE, GFP_KERNEL);
 
-	if (!pins || !groups || !names)
+	if (!pinctrl || !pins || !groups || !names)
 		return -ENOMEM;
 
+	/*
+	 * Initialize the array.  GPIOs not listed in the 'gpios' array
+	 * still need a number, but nothing else.
+	 */
 	for (i = 0; i < num_gpios; i++) {
-		snprintf(names[i], NAME_SIZE, "gpio%u", i);
-
 		pins[i].number = i;
-		pins[i].name = names[i];
-
-		groups[i].npins = 1;
-		groups[i].name = names[i];
 		groups[i].pins = &pins[i].number;
+	}
 
-		groups[i].ctl_reg = 0x10000 * i;
-		groups[i].io_reg = 0x04 + 0x10000 * i;
-		groups[i].intr_cfg_reg = 0x08 + 0x10000 * i;
-		groups[i].intr_status_reg = 0x0c + 0x10000 * i;
-		groups[i].intr_target_reg = 0x08 + 0x10000 * i;
-
-		groups[i].mux_bit = 2;
-		groups[i].pull_bit = 0;
-		groups[i].drv_bit = 6;
-		groups[i].oe_bit = 9;
-		groups[i].in_bit = 0;
-		groups[i].out_bit = 1;
-		groups[i].intr_enable_bit = 0;
-		groups[i].intr_status_bit = 0;
-		groups[i].intr_target_bit = 5;
-		groups[i].intr_target_kpss_val = 1;
-		groups[i].intr_raw_status_bit = 4;
-		groups[i].intr_polarity_bit = 1;
-		groups[i].intr_detection_bit = 2;
-		groups[i].intr_detection_width = 2;
+	/* Populate the entries that are meant to be exposed as GPIOs. */
+	for (i = 0; i < avail_gpios; i++) {
+		unsigned int gpio = gpios[i];
+
+		groups[gpio].npins = 1;
+		snprintf(names[i], NAME_SIZE, "gpio%u", gpio);
+		pins[gpio].name = names[i];
+		groups[gpio].name = names[i];
+
+		groups[gpio].ctl_reg = 0x10000 * gpio;
+		groups[gpio].io_reg = 0x04 + 0x10000 * gpio;
+		groups[gpio].intr_cfg_reg = 0x08 + 0x10000 * gpio;
+		groups[gpio].intr_status_reg = 0x0c + 0x10000 * gpio;
+		groups[gpio].intr_target_reg = 0x08 + 0x10000 * gpio;
+
+		groups[gpio].mux_bit = 2;
+		groups[gpio].pull_bit = 0;
+		groups[gpio].drv_bit = 6;
+		groups[gpio].oe_bit = 9;
+		groups[gpio].in_bit = 0;
+		groups[gpio].out_bit = 1;
+		groups[gpio].intr_enable_bit = 0;
+		groups[gpio].intr_status_bit = 0;
+		groups[gpio].intr_target_bit = 5;
+		groups[gpio].intr_target_kpss_val = 1;
+		groups[gpio].intr_raw_status_bit = 4;
+		groups[gpio].intr_polarity_bit = 1;
+		groups[gpio].intr_detection_bit = 2;
+		groups[gpio].intr_detection_width = 2;
 	}
 
-	qdf2xxx_pinctrl.pins = pins;
-	qdf2xxx_pinctrl.groups = groups;
-	qdf2xxx_pinctrl.npins = num_gpios;
-	qdf2xxx_pinctrl.ngroups = num_gpios;
-	qdf2xxx_pinctrl.ngpios = num_gpios;
+	pinctrl->pins = pins;
+	pinctrl->groups = groups;
+	pinctrl->npins = num_gpios;
+	pinctrl->ngroups = num_gpios;
+	pinctrl->ngpios = num_gpios;
 
-	return msm_pinctrl_probe(pdev, &qdf2xxx_pinctrl);
+	return msm_pinctrl_probe(pdev, pinctrl);
 }
 
 static const struct acpi_device_id qdf2xxx_acpi_ids[] = {
-	{"QCOM8001"},
+	{"QCOM8002"},
 	{},
 };
 MODULE_DEVICE_TABLE(acpi, qdf2xxx_acpi_ids);
diff --git a/drivers/pinctrl/samsung/Kconfig b/drivers/pinctrl/samsung/Kconfig
index 11b5eeb14c4a..425fadd6c346 100644
--- a/drivers/pinctrl/samsung/Kconfig
+++ b/drivers/pinctrl/samsung/Kconfig
@@ -8,26 +8,20 @@ config PINCTRL_SAMSUNG
 	select PINCONF
 
 config PINCTRL_EXYNOS
-	bool "Pinctrl driver data for Samsung EXYNOS SoCs other than 5440"
+	bool "Pinctrl driver data for Samsung EXYNOS SoCs"
 	depends on OF && GPIOLIB && (ARCH_EXYNOS || ARCH_S5PV210)
 	select PINCTRL_SAMSUNG
 	select PINCTRL_EXYNOS_ARM if ARM && (ARCH_EXYNOS || ARCH_S5PV210)
 	select PINCTRL_EXYNOS_ARM64 if ARM64 && ARCH_EXYNOS
 
 config PINCTRL_EXYNOS_ARM
-	bool "ARMv7-specific pinctrl driver data for Exynos (except Exynos5440)" if COMPILE_TEST
+	bool "ARMv7-specific pinctrl driver data for Exynos" if COMPILE_TEST
 	depends on PINCTRL_EXYNOS
 
 config PINCTRL_EXYNOS_ARM64
 	bool "ARMv8-specific pinctrl driver data for Exynos" if COMPILE_TEST
 	depends on PINCTRL_EXYNOS
 
-config PINCTRL_EXYNOS5440
-	bool "Samsung EXYNOS5440 SoC pinctrl driver"
-	depends on SOC_EXYNOS5440
-	select PINMUX
-	select PINCONF
-
 config PINCTRL_S3C24XX
 	bool "Samsung S3C24XX SoC pinctrl driver"
 	depends on ARCH_S3C24XX && OF
diff --git a/drivers/pinctrl/samsung/Makefile b/drivers/pinctrl/samsung/Makefile
index df426561d067..ed951df6a112 100644
--- a/drivers/pinctrl/samsung/Makefile
+++ b/drivers/pinctrl/samsung/Makefile
@@ -5,6 +5,5 @@ obj-$(CONFIG_PINCTRL_SAMSUNG)	+= pinctrl-samsung.o
 obj-$(CONFIG_PINCTRL_EXYNOS)	+= pinctrl-exynos.o
 obj-$(CONFIG_PINCTRL_EXYNOS_ARM)	+= pinctrl-exynos-arm.o
 obj-$(CONFIG_PINCTRL_EXYNOS_ARM64)	+= pinctrl-exynos-arm64.o
-obj-$(CONFIG_PINCTRL_EXYNOS5440)	+= pinctrl-exynos5440.o
 obj-$(CONFIG_PINCTRL_S3C24XX)	+= pinctrl-s3c24xx.o
 obj-$(CONFIG_PINCTRL_S3C64XX)	+= pinctrl-s3c64xx.o
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos-arm.c b/drivers/pinctrl/samsung/pinctrl-exynos-arm.c
index 90c274490181..d82820fc349a 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos-arm.c
+++ b/drivers/pinctrl/samsung/pinctrl-exynos-arm.c
@@ -88,6 +88,7 @@ static const struct samsung_retention_data s5pv210_retention_data __initconst =
 
 /* pin banks of s5pv210 pin-controller */
 static const struct samsung_pin_bank_data s5pv210_pin_bank[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpa0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(4, 0x020, "gpa1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x040, "gpb", 0x08),
@@ -105,12 +106,12 @@ static const struct samsung_pin_bank_data s5pv210_pin_bank[] __initconst = {
 	EXYNOS_PIN_BANK_EINTG(7, 0x1c0, "gpg1", 0x38),
 	EXYNOS_PIN_BANK_EINTG(7, 0x1e0, "gpg2", 0x3c),
 	EXYNOS_PIN_BANK_EINTG(7, 0x200, "gpg3", 0x40),
-	EXYNOS_PIN_BANK_EINTN(7, 0x220, "gpi"),
 	EXYNOS_PIN_BANK_EINTG(8, 0x240, "gpj0", 0x44),
 	EXYNOS_PIN_BANK_EINTG(6, 0x260, "gpj1", 0x48),
 	EXYNOS_PIN_BANK_EINTG(8, 0x280, "gpj2", 0x4c),
 	EXYNOS_PIN_BANK_EINTG(8, 0x2a0, "gpj3", 0x50),
 	EXYNOS_PIN_BANK_EINTG(5, 0x2c0, "gpj4", 0x54),
+	EXYNOS_PIN_BANK_EINTN(7, 0x220, "gpi"),
 	EXYNOS_PIN_BANK_EINTN(8, 0x2e0, "mp01"),
 	EXYNOS_PIN_BANK_EINTN(4, 0x300, "mp02"),
 	EXYNOS_PIN_BANK_EINTN(8, 0x320, "mp03"),
@@ -147,6 +148,7 @@ static atomic_t exynos_shared_retention_refcnt;
 
 /* pin banks of exynos3250 pin-controller 0 */
 static const struct samsung_pin_bank_data exynos3250_pin_banks0[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpa0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(6, 0x020, "gpa1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x040, "gpb",  0x08),
@@ -158,6 +160,7 @@ static const struct samsung_pin_bank_data exynos3250_pin_banks0[] __initconst =
 
 /* pin banks of exynos3250 pin-controller 1 */
 static const struct samsung_pin_bank_data exynos3250_pin_banks1[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTN(8, 0x120, "gpe0"),
 	EXYNOS_PIN_BANK_EINTN(8, 0x140, "gpe1"),
 	EXYNOS_PIN_BANK_EINTN(3, 0x180, "gpe2"),
@@ -232,6 +235,7 @@ const struct samsung_pinctrl_of_match_data exynos3250_of_data __initconst = {
 
 /* pin banks of exynos4210 pin-controller 0 */
 static const struct samsung_pin_bank_data exynos4210_pin_banks0[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpa0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(6, 0x020, "gpa1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x040, "gpb", 0x08),
@@ -252,6 +256,7 @@ static const struct samsung_pin_bank_data exynos4210_pin_banks0[] __initconst =
 
 /* pin banks of exynos4210 pin-controller 1 */
 static const struct samsung_pin_bank_data exynos4210_pin_banks1[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpj0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(5, 0x020, "gpj1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(7, 0x040, "gpk0", 0x08),
@@ -276,6 +281,7 @@ static const struct samsung_pin_bank_data exynos4210_pin_banks1[] __initconst =
 
 /* pin banks of exynos4210 pin-controller 2 */
 static const struct samsung_pin_bank_data exynos4210_pin_banks2[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTN(7, 0x000, "gpz"),
 };
 
@@ -346,6 +352,7 @@ const struct samsung_pinctrl_of_match_data exynos4210_of_data __initconst = {
 
 /* pin banks of exynos4x12 pin-controller 0 */
 static const struct samsung_pin_bank_data exynos4x12_pin_banks0[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpa0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(6, 0x020, "gpa1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x040, "gpb", 0x08),
@@ -363,6 +370,7 @@ static const struct samsung_pin_bank_data exynos4x12_pin_banks0[] __initconst =
 
 /* pin banks of exynos4x12 pin-controller 1 */
 static const struct samsung_pin_bank_data exynos4x12_pin_banks1[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(7, 0x040, "gpk0", 0x08),
 	EXYNOS_PIN_BANK_EINTG(7, 0x060, "gpk1", 0x0c),
 	EXYNOS_PIN_BANK_EINTG(7, 0x080, "gpk2", 0x10),
@@ -390,11 +398,13 @@ static const struct samsung_pin_bank_data exynos4x12_pin_banks1[] __initconst =
 
 /* pin banks of exynos4x12 pin-controller 2 */
 static const struct samsung_pin_bank_data exynos4x12_pin_banks2[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(7, 0x000, "gpz", 0x00),
 };
 
 /* pin banks of exynos4x12 pin-controller 3 */
 static const struct samsung_pin_bank_data exynos4x12_pin_banks3[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpv0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(8, 0x020, "gpv1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x040, "gpv2", 0x08),
@@ -449,6 +459,7 @@ const struct samsung_pinctrl_of_match_data exynos4x12_of_data __initconst = {
 
 /* pin banks of exynos5250 pin-controller 0 */
 static const struct samsung_pin_bank_data exynos5250_pin_banks0[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpa0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(6, 0x020, "gpa1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x040, "gpa2", 0x08),
@@ -478,6 +489,7 @@ static const struct samsung_pin_bank_data exynos5250_pin_banks0[] __initconst =
 
 /* pin banks of exynos5250 pin-controller 1 */
 static const struct samsung_pin_bank_data exynos5250_pin_banks1[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpe0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(2, 0x020, "gpe1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(4, 0x040, "gpf0", 0x08),
@@ -491,6 +503,7 @@ static const struct samsung_pin_bank_data exynos5250_pin_banks1[] __initconst =
 
 /* pin banks of exynos5250 pin-controller 2 */
 static const struct samsung_pin_bank_data exynos5250_pin_banks2[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpv0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(8, 0x020, "gpv1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x060, "gpv2", 0x08),
@@ -500,6 +513,7 @@ static const struct samsung_pin_bank_data exynos5250_pin_banks2[] __initconst =
 
 /* pin banks of exynos5250 pin-controller 3 */
 static const struct samsung_pin_bank_data exynos5250_pin_banks3[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(7, 0x000, "gpz", 0x00),
 };
 
@@ -550,6 +564,7 @@ const struct samsung_pinctrl_of_match_data exynos5250_of_data __initconst = {
 
 /* pin banks of exynos5260 pin-controller 0 */
 static const struct samsung_pin_bank_data exynos5260_pin_banks0[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(4, 0x000, "gpa0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(7, 0x020, "gpa1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x040, "gpa2", 0x08),
@@ -575,6 +590,7 @@ static const struct samsung_pin_bank_data exynos5260_pin_banks0[] __initconst =
 
 /* pin banks of exynos5260 pin-controller 1 */
 static const struct samsung_pin_bank_data exynos5260_pin_banks1[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(7, 0x000, "gpc0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(6, 0x020, "gpc1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(7, 0x040, "gpc2", 0x08),
@@ -584,6 +600,7 @@ static const struct samsung_pin_bank_data exynos5260_pin_banks1[] __initconst =
 
 /* pin banks of exynos5260 pin-controller 2 */
 static const struct samsung_pin_bank_data exynos5260_pin_banks2[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(7, 0x000, "gpz0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(4, 0x020, "gpz1", 0x04),
 };
@@ -619,6 +636,7 @@ const struct samsung_pinctrl_of_match_data exynos5260_of_data __initconst = {
 
 /* pin banks of exynos5410 pin-controller 0 */
 static const struct samsung_pin_bank_data exynos5410_pin_banks0[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpa0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(6, 0x020, "gpa1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x040, "gpa2", 0x08),
@@ -630,7 +648,6 @@ static const struct samsung_pin_bank_data exynos5410_pin_banks0[] __initconst =
 	EXYNOS_PIN_BANK_EINTG(4, 0x100, "gpc3", 0x20),
 	EXYNOS_PIN_BANK_EINTG(7, 0x120, "gpc1", 0x24),
 	EXYNOS_PIN_BANK_EINTG(7, 0x140, "gpc2", 0x28),
-	EXYNOS_PIN_BANK_EINTN(2, 0x160, "gpm5"),
 	EXYNOS_PIN_BANK_EINTG(8, 0x180, "gpd1", 0x2c),
 	EXYNOS_PIN_BANK_EINTG(8, 0x1A0, "gpe0", 0x30),
 	EXYNOS_PIN_BANK_EINTG(2, 0x1C0, "gpe1", 0x34),
@@ -641,6 +658,7 @@ static const struct samsung_pin_bank_data exynos5410_pin_banks0[] __initconst =
 	EXYNOS_PIN_BANK_EINTG(2, 0x260, "gpg2", 0x48),
 	EXYNOS_PIN_BANK_EINTG(4, 0x280, "gph0", 0x4c),
 	EXYNOS_PIN_BANK_EINTG(8, 0x2A0, "gph1", 0x50),
+	EXYNOS_PIN_BANK_EINTN(2, 0x160, "gpm5"),
 	EXYNOS_PIN_BANK_EINTN(8, 0x2C0, "gpm7"),
 	EXYNOS_PIN_BANK_EINTN(6, 0x2E0, "gpy0"),
 	EXYNOS_PIN_BANK_EINTN(4, 0x300, "gpy1"),
@@ -658,6 +676,7 @@ static const struct samsung_pin_bank_data exynos5410_pin_banks0[] __initconst =
 
 /* pin banks of exynos5410 pin-controller 1 */
 static const struct samsung_pin_bank_data exynos5410_pin_banks1[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(5, 0x000, "gpj0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(8, 0x020, "gpj1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x040, "gpj2", 0x08),
@@ -671,6 +690,7 @@ static const struct samsung_pin_bank_data exynos5410_pin_banks1[] __initconst =
 
 /* pin banks of exynos5410 pin-controller 2 */
 static const struct samsung_pin_bank_data exynos5410_pin_banks2[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpv0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(8, 0x020, "gpv1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x060, "gpv2", 0x08),
@@ -680,6 +700,7 @@ static const struct samsung_pin_bank_data exynos5410_pin_banks2[] __initconst =
 
 /* pin banks of exynos5410 pin-controller 3 */
 static const struct samsung_pin_bank_data exynos5410_pin_banks3[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(7, 0x000, "gpz", 0x00),
 };
 
@@ -727,6 +748,7 @@ const struct samsung_pinctrl_of_match_data exynos5410_of_data __initconst = {
 
 /* pin banks of exynos5420 pin-controller 0 */
 static const struct samsung_pin_bank_data exynos5420_pin_banks0[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpy7", 0x00),
 	EXYNOS_PIN_BANK_EINTW(8, 0xC00, "gpx0", 0x00),
 	EXYNOS_PIN_BANK_EINTW(8, 0xC20, "gpx1", 0x04),
@@ -736,6 +758,7 @@ static const struct samsung_pin_bank_data exynos5420_pin_banks0[] __initconst =
 
 /* pin banks of exynos5420 pin-controller 1 */
 static const struct samsung_pin_bank_data exynos5420_pin_banks1[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpc0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(8, 0x020, "gpc1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(7, 0x040, "gpc2", 0x08),
@@ -753,6 +776,7 @@ static const struct samsung_pin_bank_data exynos5420_pin_banks1[] __initconst =
 
 /* pin banks of exynos5420 pin-controller 2 */
 static const struct samsung_pin_bank_data exynos5420_pin_banks2[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpe0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(2, 0x020, "gpe1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(6, 0x040, "gpf0", 0x08),
@@ -765,6 +789,7 @@ static const struct samsung_pin_bank_data exynos5420_pin_banks2[] __initconst =
 
 /* pin banks of exynos5420 pin-controller 3 */
 static const struct samsung_pin_bank_data exynos5420_pin_banks3[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x000, "gpa0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(6, 0x020, "gpa1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(8, 0x040, "gpa2", 0x08),
@@ -778,6 +803,7 @@ static const struct samsung_pin_bank_data exynos5420_pin_banks3[] __initconst =
 
 /* pin banks of exynos5420 pin-controller 4 */
 static const struct samsung_pin_bank_data exynos5420_pin_banks4[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(7, 0x000, "gpz", 0x00),
 };
 
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c b/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c
index 71c9d1d9f345..b6e56422a700 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c
+++ b/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c
@@ -45,6 +45,7 @@ static atomic_t exynos_shared_retention_refcnt;
 
 /* pin banks of exynos5433 pin-controller - ALIVE */
 static const struct samsung_pin_bank_data exynos5433_pin_banks0[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS5433_PIN_BANK_EINTW(8, 0x000, "gpa0", 0x00),
 	EXYNOS5433_PIN_BANK_EINTW(8, 0x020, "gpa1", 0x04),
 	EXYNOS5433_PIN_BANK_EINTW(8, 0x040, "gpa2", 0x08),
@@ -58,27 +59,32 @@ static const struct samsung_pin_bank_data exynos5433_pin_banks0[] __initconst =
 
 /* pin banks of exynos5433 pin-controller - AUD */
 static const struct samsung_pin_bank_data exynos5433_pin_banks1[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS5433_PIN_BANK_EINTG(7, 0x000, "gpz0", 0x00),
 	EXYNOS5433_PIN_BANK_EINTG(4, 0x020, "gpz1", 0x04),
 };
 
 /* pin banks of exynos5433 pin-controller - CPIF */
 static const struct samsung_pin_bank_data exynos5433_pin_banks2[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS5433_PIN_BANK_EINTG(2, 0x000, "gpv6", 0x00),
 };
 
 /* pin banks of exynos5433 pin-controller - eSE */
 static const struct samsung_pin_bank_data exynos5433_pin_banks3[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS5433_PIN_BANK_EINTG(3, 0x000, "gpj2", 0x00),
 };
 
 /* pin banks of exynos5433 pin-controller - FINGER */
 static const struct samsung_pin_bank_data exynos5433_pin_banks4[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS5433_PIN_BANK_EINTG(4, 0x000, "gpd5", 0x00),
 };
 
 /* pin banks of exynos5433 pin-controller - FSYS */
 static const struct samsung_pin_bank_data exynos5433_pin_banks5[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS5433_PIN_BANK_EINTG(6, 0x000, "gph1", 0x00),
 	EXYNOS5433_PIN_BANK_EINTG(7, 0x020, "gpr4", 0x04),
 	EXYNOS5433_PIN_BANK_EINTG(5, 0x040, "gpr0", 0x08),
@@ -89,16 +95,19 @@ static const struct samsung_pin_bank_data exynos5433_pin_banks5[] __initconst =
 
 /* pin banks of exynos5433 pin-controller - IMEM */
 static const struct samsung_pin_bank_data exynos5433_pin_banks6[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS5433_PIN_BANK_EINTG(8, 0x000, "gpf0", 0x00),
 };
 
 /* pin banks of exynos5433 pin-controller - NFC */
 static const struct samsung_pin_bank_data exynos5433_pin_banks7[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS5433_PIN_BANK_EINTG(3, 0x000, "gpj0", 0x00),
 };
 
 /* pin banks of exynos5433 pin-controller - PERIC */
 static const struct samsung_pin_bank_data exynos5433_pin_banks8[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS5433_PIN_BANK_EINTG(6, 0x000, "gpv7", 0x00),
 	EXYNOS5433_PIN_BANK_EINTG(5, 0x020, "gpb0", 0x04),
 	EXYNOS5433_PIN_BANK_EINTG(8, 0x040, "gpc0", 0x08),
@@ -120,6 +129,7 @@ static const struct samsung_pin_bank_data exynos5433_pin_banks8[] __initconst =
 
 /* pin banks of exynos5433 pin-controller - TOUCH */
 static const struct samsung_pin_bank_data exynos5433_pin_banks9[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS5433_PIN_BANK_EINTG(3, 0x000, "gpj1", 0x00),
 };
 
@@ -267,6 +277,7 @@ const struct samsung_pinctrl_of_match_data exynos5433_of_data __initconst = {
 
 /* pin banks of exynos7 pin-controller - ALIVE */
 static const struct samsung_pin_bank_data exynos7_pin_banks0[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTW(8, 0x000, "gpa0", 0x00),
 	EXYNOS_PIN_BANK_EINTW(8, 0x020, "gpa1", 0x04),
 	EXYNOS_PIN_BANK_EINTW(8, 0x040, "gpa2", 0x08),
@@ -275,6 +286,7 @@ static const struct samsung_pin_bank_data exynos7_pin_banks0[] __initconst = {
 
 /* pin banks of exynos7 pin-controller - BUS0 */
 static const struct samsung_pin_bank_data exynos7_pin_banks1[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(5, 0x000, "gpb0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(8, 0x020, "gpc0", 0x04),
 	EXYNOS_PIN_BANK_EINTG(2, 0x040, "gpc1", 0x08),
@@ -294,31 +306,37 @@ static const struct samsung_pin_bank_data exynos7_pin_banks1[] __initconst = {
 
 /* pin banks of exynos7 pin-controller - NFC */
 static const struct samsung_pin_bank_data exynos7_pin_banks2[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(3, 0x000, "gpj0", 0x00),
 };
 
 /* pin banks of exynos7 pin-controller - TOUCH */
 static const struct samsung_pin_bank_data exynos7_pin_banks3[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(3, 0x000, "gpj1", 0x00),
 };
 
 /* pin banks of exynos7 pin-controller - FF */
 static const struct samsung_pin_bank_data exynos7_pin_banks4[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(4, 0x000, "gpg4", 0x00),
 };
 
 /* pin banks of exynos7 pin-controller - ESE */
 static const struct samsung_pin_bank_data exynos7_pin_banks5[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(5, 0x000, "gpv7", 0x00),
 };
 
 /* pin banks of exynos7 pin-controller - FSYS0 */
 static const struct samsung_pin_bank_data exynos7_pin_banks6[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(7, 0x000, "gpr4", 0x00),
 };
 
 /* pin banks of exynos7 pin-controller - FSYS1 */
 static const struct samsung_pin_bank_data exynos7_pin_banks7[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(4, 0x000, "gpr0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(8, 0x020, "gpr1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(5, 0x040, "gpr2", 0x08),
@@ -327,6 +345,7 @@ static const struct samsung_pin_bank_data exynos7_pin_banks7[] __initconst = {
 
 /* pin banks of exynos7 pin-controller - BUS1 */
 static const struct samsung_pin_bank_data exynos7_pin_banks8[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(8, 0x020, "gpf0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(8, 0x040, "gpf1", 0x04),
 	EXYNOS_PIN_BANK_EINTG(4, 0x060, "gpf2", 0x08),
@@ -340,6 +359,7 @@ static const struct samsung_pin_bank_data exynos7_pin_banks8[] __initconst = {
 };
 
 static const struct samsung_pin_bank_data exynos7_pin_banks9[] __initconst = {
+	/* Must start with EINTG banks, ordered by EINT group number. */
 	EXYNOS_PIN_BANK_EINTG(7, 0x000, "gpz0", 0x00),
 	EXYNOS_PIN_BANK_EINTG(4, 0x020, "gpz1", 0x04),
 };
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.h b/drivers/pinctrl/samsung/pinctrl-exynos.h
index abd43aa7eb0d..da1ec13697e7 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos.h
+++ b/drivers/pinctrl/samsung/pinctrl-exynos.h
@@ -99,7 +99,7 @@
 
 #define EXYNOS5433_PIN_BANK_EINTW_EXT(pins, reg, id, offs, pctl_idx) \
 	{							\
-		.type           = &exynos5433_bank_type_alive,	\
+		.type           = &exynos5433_bank_type_off,	\
 		.pctl_offset    = reg,				\
 		.nr_pins        = pins,				\
 		.eint_type      = EINT_TYPE_WKUP,		\
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos5440.c b/drivers/pinctrl/samsung/pinctrl-exynos5440.c
deleted file mode 100644
index 3d8d5e812839..000000000000
--- a/drivers/pinctrl/samsung/pinctrl-exynos5440.c
+++ /dev/null
@@ -1,1005 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-//
-// pin-controller/pin-mux/pin-config/gpio-driver for Samsung's EXYNOS5440 SoC.
-//
-// Author: Thomas Abraham <thomas.ab@samsung.com>
-//
-// Copyright (c) 2012 Samsung Electronics Co., Ltd.
-//		http://www.samsung.com
-
-#include <linux/init.h>
-#include <linux/platform_device.h>
-#include <linux/io.h>
-#include <linux/slab.h>
-#include <linux/err.h>
-#include <linux/gpio/driver.h>
-#include <linux/device.h>
-#include <linux/pinctrl/pinctrl.h>
-#include <linux/pinctrl/pinmux.h>
-#include <linux/pinctrl/pinconf.h>
-#include <linux/interrupt.h>
-#include <linux/irqdomain.h>
-#include <linux/of_irq.h>
-#include "../core.h"
-
-/* EXYNOS5440 GPIO and Pinctrl register offsets */
-#define GPIO_MUX		0x00
-#define GPIO_IE			0x04
-#define GPIO_INT		0x08
-#define GPIO_TYPE		0x0C
-#define GPIO_VAL		0x10
-#define GPIO_OE			0x14
-#define GPIO_IN			0x18
-#define GPIO_PE			0x1C
-#define GPIO_PS			0x20
-#define GPIO_SR			0x24
-#define GPIO_DS0		0x28
-#define GPIO_DS1		0x2C
-
-#define EXYNOS5440_MAX_PINS		23
-#define EXYNOS5440_MAX_GPIO_INT	8
-#define PIN_NAME_LENGTH		10
-
-#define GROUP_SUFFIX		"-grp"
-#define FUNCTION_SUFFIX		"-mux"
-
-/*
- * pin configuration type and its value are packed together into a 16-bits.
- * The upper 8-bits represent the configuration type and the lower 8-bits
- * hold the value of the configuration type.
- */
-#define PINCFG_TYPE_MASK		0xFF
-#define PINCFG_VALUE_SHIFT		8
-#define PINCFG_VALUE_MASK		(0xFF << PINCFG_VALUE_SHIFT)
-#define PINCFG_PACK(type, value)	(((value) << PINCFG_VALUE_SHIFT) | type)
-#define PINCFG_UNPACK_TYPE(cfg)		((cfg) & PINCFG_TYPE_MASK)
-#define PINCFG_UNPACK_VALUE(cfg)	(((cfg) & PINCFG_VALUE_MASK) >> \
-						PINCFG_VALUE_SHIFT)
-
-/**
- * enum pincfg_type - possible pin configuration types supported.
- * @PINCFG_TYPE_PUD: Pull up/down configuration.
- * @PINCFG_TYPE_DRV: Drive strength configuration.
- * @PINCFG_TYPE_SKEW_RATE: Skew rate configuration.
- * @PINCFG_TYPE_INPUT_TYPE: Pin input type configuration.
- */
-enum pincfg_type {
-	PINCFG_TYPE_PUD,
-	PINCFG_TYPE_DRV,
-	PINCFG_TYPE_SKEW_RATE,
-	PINCFG_TYPE_INPUT_TYPE
-};
-
-/**
- * struct exynos5440_pin_group: represent group of pins for pincfg setting.
- * @name: name of the pin group, used to lookup the group.
- * @pins: the pins included in this group.
- * @num_pins: number of pins included in this group.
- */
-struct exynos5440_pin_group {
-	const char		*name;
-	const unsigned int	*pins;
-	u8			num_pins;
-};
-
-/**
- * struct exynos5440_pmx_func: represent a pin function.
- * @name: name of the pin function, used to lookup the function.
- * @groups: one or more names of pin groups that provide this function.
- * @num_groups: number of groups included in @groups.
- * @function: the function number to be programmed when selected.
- */
-struct exynos5440_pmx_func {
-	const char		*name;
-	const char		**groups;
-	u8			num_groups;
-	unsigned long		function;
-};
-
-/**
- * struct exynos5440_pinctrl_priv_data: driver's private runtime data.
- * @reg_base: ioremapped based address of the register space.
- * @gc: gpio chip registered with gpiolib.
- * @pin_groups: list of pin groups parsed from device tree.
- * @nr_groups: number of pin groups available.
- * @pmx_functions: list of pin functions parsed from device tree.
- * @nr_functions: number of pin functions available.
- * @range: gpio range to register with pinctrl
- */
-struct exynos5440_pinctrl_priv_data {
-	void __iomem			*reg_base;
-	struct gpio_chip		*gc;
-	struct irq_domain		*irq_domain;
-
-	const struct exynos5440_pin_group	*pin_groups;
-	unsigned int			nr_groups;
-	const struct exynos5440_pmx_func	*pmx_functions;
-	unsigned int			nr_functions;
-	struct pinctrl_gpio_range	range;
-};
-
-/**
- * struct exynos5440_gpio_intr_data: private data for gpio interrupts.
- * @priv: driver's private runtime data.
- * @gpio_int: gpio interrupt number.
- */
-struct exynos5440_gpio_intr_data {
-	struct exynos5440_pinctrl_priv_data	*priv;
-	unsigned int				gpio_int;
-};
-
-/* list of all possible config options supported */
-static struct pin_config {
-	char		*prop_cfg;
-	unsigned int	cfg_type;
-} pcfgs[] = {
-	{ "samsung,exynos5440-pin-pud", PINCFG_TYPE_PUD },
-	{ "samsung,exynos5440-pin-drv", PINCFG_TYPE_DRV },
-	{ "samsung,exynos5440-pin-skew-rate", PINCFG_TYPE_SKEW_RATE },
-	{ "samsung,exynos5440-pin-input-type", PINCFG_TYPE_INPUT_TYPE },
-};
-
-/* check if the selector is a valid pin group selector */
-static int exynos5440_get_group_count(struct pinctrl_dev *pctldev)
-{
-	struct exynos5440_pinctrl_priv_data *priv;
-
-	priv = pinctrl_dev_get_drvdata(pctldev);
-	return priv->nr_groups;
-}
-
-/* return the name of the group selected by the group selector */
-static const char *exynos5440_get_group_name(struct pinctrl_dev *pctldev,
-						unsigned selector)
-{
-	struct exynos5440_pinctrl_priv_data *priv;
-
-	priv = pinctrl_dev_get_drvdata(pctldev);
-	return priv->pin_groups[selector].name;
-}
-
-/* return the pin numbers associated with the specified group */
-static int exynos5440_get_group_pins(struct pinctrl_dev *pctldev,
-		unsigned selector, const unsigned **pins, unsigned *num_pins)
-{
-	struct exynos5440_pinctrl_priv_data *priv;
-
-	priv = pinctrl_dev_get_drvdata(pctldev);
-	*pins = priv->pin_groups[selector].pins;
-	*num_pins = priv->pin_groups[selector].num_pins;
-	return 0;
-}
-
-/* create pinctrl_map entries by parsing device tree nodes */
-static int exynos5440_dt_node_to_map(struct pinctrl_dev *pctldev,
-			struct device_node *np, struct pinctrl_map **maps,
-			unsigned *nmaps)
-{
-	struct device *dev = pctldev->dev;
-	struct pinctrl_map *map;
-	unsigned long *cfg = NULL;
-	char *gname, *fname;
-	int cfg_cnt = 0, map_cnt = 0, idx = 0;
-
-	/* count the number of config options specfied in the node */
-	for (idx = 0; idx < ARRAY_SIZE(pcfgs); idx++)
-		if (of_find_property(np, pcfgs[idx].prop_cfg, NULL))
-			cfg_cnt++;
-
-	/*
-	 * Find out the number of map entries to create. All the config options
-	 * can be accomadated into a single config map entry.
-	 */
-	if (cfg_cnt)
-		map_cnt = 1;
-	if (of_find_property(np, "samsung,exynos5440-pin-function", NULL))
-		map_cnt++;
-	if (!map_cnt) {
-		dev_err(dev, "node %s does not have either config or function "
-				"configurations\n", np->name);
-		return -EINVAL;
-	}
-
-	/* Allocate memory for pin-map entries */
-	map = kzalloc(sizeof(*map) * map_cnt, GFP_KERNEL);
-	if (!map)
-		return -ENOMEM;
-	*nmaps = 0;
-
-	/*
-	 * Allocate memory for pin group name. The pin group name is derived
-	 * from the node name from which these map entries are be created.
-	 */
-	gname = kasprintf(GFP_KERNEL, "%s%s", np->name, GROUP_SUFFIX);
-	if (!gname)
-		goto free_map;
-
-	/*
-	 * don't have config options? then skip over to creating function
-	 * map entries.
-	 */
-	if (!cfg_cnt)
-		goto skip_cfgs;
-
-	/* Allocate memory for config entries */
-	cfg = kzalloc(sizeof(*cfg) * cfg_cnt, GFP_KERNEL);
-	if (!cfg)
-		goto free_gname;
-
-	/* Prepare a list of config settings */
-	for (idx = 0, cfg_cnt = 0; idx < ARRAY_SIZE(pcfgs); idx++) {
-		u32 value;
-		if (!of_property_read_u32(np, pcfgs[idx].prop_cfg, &value))
-			cfg[cfg_cnt++] =
-				PINCFG_PACK(pcfgs[idx].cfg_type, value);
-	}
-
-	/* create the config map entry */
-	map[*nmaps].data.configs.group_or_pin = gname;
-	map[*nmaps].data.configs.configs = cfg;
-	map[*nmaps].data.configs.num_configs = cfg_cnt;
-	map[*nmaps].type = PIN_MAP_TYPE_CONFIGS_GROUP;
-	*nmaps += 1;
-
-skip_cfgs:
-	/* create the function map entry */
-	if (of_find_property(np, "samsung,exynos5440-pin-function", NULL)) {
-		fname = kasprintf(GFP_KERNEL,
-				  "%s%s", np->name, FUNCTION_SUFFIX);
-		if (!fname)
-			goto free_cfg;
-
-		map[*nmaps].data.mux.group = gname;
-		map[*nmaps].data.mux.function = fname;
-		map[*nmaps].type = PIN_MAP_TYPE_MUX_GROUP;
-		*nmaps += 1;
-	}
-
-	*maps = map;
-	return 0;
-
-free_cfg:
-	kfree(cfg);
-free_gname:
-	kfree(gname);
-free_map:
-	kfree(map);
-	return -ENOMEM;
-}
-
-/* free the memory allocated to hold the pin-map table */
-static void exynos5440_dt_free_map(struct pinctrl_dev *pctldev,
-			     struct pinctrl_map *map, unsigned num_maps)
-{
-	int idx;
-
-	for (idx = 0; idx < num_maps; idx++) {
-		if (map[idx].type == PIN_MAP_TYPE_MUX_GROUP) {
-			kfree(map[idx].data.mux.function);
-			if (!idx)
-				kfree(map[idx].data.mux.group);
-		} else if (map->type == PIN_MAP_TYPE_CONFIGS_GROUP) {
-			kfree(map[idx].data.configs.configs);
-			if (!idx)
-				kfree(map[idx].data.configs.group_or_pin);
-		}
-	}
-
-	kfree(map);
-}
-
-/* list of pinctrl callbacks for the pinctrl core */
-static const struct pinctrl_ops exynos5440_pctrl_ops = {
-	.get_groups_count	= exynos5440_get_group_count,
-	.get_group_name		= exynos5440_get_group_name,
-	.get_group_pins		= exynos5440_get_group_pins,
-	.dt_node_to_map		= exynos5440_dt_node_to_map,
-	.dt_free_map		= exynos5440_dt_free_map,
-};
-
-/* check if the selector is a valid pin function selector */
-static int exynos5440_get_functions_count(struct pinctrl_dev *pctldev)
-{
-	struct exynos5440_pinctrl_priv_data *priv;
-
-	priv = pinctrl_dev_get_drvdata(pctldev);
-	return priv->nr_functions;
-}
-
-/* return the name of the pin function specified */
-static const char *exynos5440_pinmux_get_fname(struct pinctrl_dev *pctldev,
-						unsigned selector)
-{
-	struct exynos5440_pinctrl_priv_data *priv;
-
-	priv = pinctrl_dev_get_drvdata(pctldev);
-	return priv->pmx_functions[selector].name;
-}
-
-/* return the groups associated for the specified function selector */
-static int exynos5440_pinmux_get_groups(struct pinctrl_dev *pctldev,
-		unsigned selector, const char * const **groups,
-		unsigned * const num_groups)
-{
-	struct exynos5440_pinctrl_priv_data *priv;
-
-	priv = pinctrl_dev_get_drvdata(pctldev);
-	*groups = priv->pmx_functions[selector].groups;
-	*num_groups = priv->pmx_functions[selector].num_groups;
-	return 0;
-}
-
-/* enable or disable a pinmux function */
-static void exynos5440_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector,
-					unsigned group, bool enable)
-{
-	struct exynos5440_pinctrl_priv_data *priv;
-	void __iomem *base;
-	u32 function;
-	u32 data;
-
-	priv = pinctrl_dev_get_drvdata(pctldev);
-	base = priv->reg_base;
-	function = priv->pmx_functions[selector].function;
-
-	data = readl(base + GPIO_MUX);
-	if (enable)
-		data |= (1 << function);
-	else
-		data &= ~(1 << function);
-	writel(data, base + GPIO_MUX);
-}
-
-/* enable a specified pinmux by writing to registers */
-static int exynos5440_pinmux_set_mux(struct pinctrl_dev *pctldev,
-				     unsigned selector,
-				     unsigned group)
-{
-	exynos5440_pinmux_setup(pctldev, selector, group, true);
-	return 0;
-}
-
-/*
- * The calls to gpio_direction_output() and gpio_direction_input()
- * leads to this function call (via the pinctrl_gpio_direction_{input|output}()
- * function called from the gpiolib interface).
- */
-static int exynos5440_pinmux_gpio_set_direction(struct pinctrl_dev *pctldev,
-		struct pinctrl_gpio_range *range, unsigned offset, bool input)
-{
-	return 0;
-}
-
-/* list of pinmux callbacks for the pinmux vertical in pinctrl core */
-static const struct pinmux_ops exynos5440_pinmux_ops = {
-	.get_functions_count	= exynos5440_get_functions_count,
-	.get_function_name	= exynos5440_pinmux_get_fname,
-	.get_function_groups	= exynos5440_pinmux_get_groups,
-	.set_mux		= exynos5440_pinmux_set_mux,
-	.gpio_set_direction	= exynos5440_pinmux_gpio_set_direction,
-};
-
-/* set the pin config settings for a specified pin */
-static int exynos5440_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
-				unsigned long *configs,
-				unsigned num_configs)
-{
-	struct exynos5440_pinctrl_priv_data *priv;
-	void __iomem *base;
-	enum pincfg_type cfg_type;
-	u32 cfg_value;
-	u32 data;
-	int i;
-
-	priv = pinctrl_dev_get_drvdata(pctldev);
-	base = priv->reg_base;
-
-	for (i = 0; i < num_configs; i++) {
-		cfg_type = PINCFG_UNPACK_TYPE(configs[i]);
-		cfg_value = PINCFG_UNPACK_VALUE(configs[i]);
-
-		switch (cfg_type) {
-		case PINCFG_TYPE_PUD:
-			/* first set pull enable/disable bit */
-			data = readl(base + GPIO_PE);
-			data &= ~(1 << pin);
-			if (cfg_value)
-				data |= (1 << pin);
-			writel(data, base + GPIO_PE);
-
-			/* then set pull up/down bit */
-			data = readl(base + GPIO_PS);
-			data &= ~(1 << pin);
-			if (cfg_value == 2)
-				data |= (1 << pin);
-			writel(data, base + GPIO_PS);
-			break;
-
-		case PINCFG_TYPE_DRV:
-			/* set the first bit of the drive strength */
-			data = readl(base + GPIO_DS0);
-			data &= ~(1 << pin);
-			data |= ((cfg_value & 1) << pin);
-			writel(data, base + GPIO_DS0);
-			cfg_value >>= 1;
-
-			/* set the second bit of the driver strength */
-			data = readl(base + GPIO_DS1);
-			data &= ~(1 << pin);
-			data |= ((cfg_value & 1) << pin);
-			writel(data, base + GPIO_DS1);
-			break;
-		case PINCFG_TYPE_SKEW_RATE:
-			data = readl(base + GPIO_SR);
-			data &= ~(1 << pin);
-			data |= ((cfg_value & 1) << pin);
-			writel(data, base + GPIO_SR);
-			break;
-		case PINCFG_TYPE_INPUT_TYPE:
-			data = readl(base + GPIO_TYPE);
-			data &= ~(1 << pin);
-			data |= ((cfg_value & 1) << pin);
-			writel(data, base + GPIO_TYPE);
-			break;
-		default:
-			WARN_ON(1);
-			return -EINVAL;
-		}
-	} /* for each config */
-
-	return 0;
-}
-
-/* get the pin config settings for a specified pin */
-static int exynos5440_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin,
-					unsigned long *config)
-{
-	struct exynos5440_pinctrl_priv_data *priv;
-	void __iomem *base;
-	enum pincfg_type cfg_type = PINCFG_UNPACK_TYPE(*config);
-	u32 data;
-
-	priv = pinctrl_dev_get_drvdata(pctldev);
-	base = priv->reg_base;
-
-	switch (cfg_type) {
-	case PINCFG_TYPE_PUD:
-		data = readl(base + GPIO_PE);
-		data = (data >> pin) & 1;
-		if (!data)
-			*config = 0;
-		else
-			*config = ((readl(base + GPIO_PS) >> pin) & 1) + 1;
-		break;
-	case PINCFG_TYPE_DRV:
-		data = readl(base + GPIO_DS0);
-		data = (data >> pin) & 1;
-		*config = data;
-		data = readl(base + GPIO_DS1);
-		data = (data >> pin) & 1;
-		*config |= (data << 1);
-		break;
-	case PINCFG_TYPE_SKEW_RATE:
-		data = readl(base + GPIO_SR);
-		*config = (data >> pin) & 1;
-		break;
-	case PINCFG_TYPE_INPUT_TYPE:
-		data = readl(base + GPIO_TYPE);
-		*config = (data >> pin) & 1;
-		break;
-	default:
-		WARN_ON(1);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/* set the pin config settings for a specified pin group */
-static int exynos5440_pinconf_group_set(struct pinctrl_dev *pctldev,
-			unsigned group, unsigned long *configs,
-			unsigned num_configs)
-{
-	struct exynos5440_pinctrl_priv_data *priv;
-	const unsigned int *pins;
-	unsigned int cnt;
-
-	priv = pinctrl_dev_get_drvdata(pctldev);
-	pins = priv->pin_groups[group].pins;
-
-	for (cnt = 0; cnt < priv->pin_groups[group].num_pins; cnt++)
-		exynos5440_pinconf_set(pctldev, pins[cnt], configs,
-				       num_configs);
-
-	return 0;
-}
-
-/* get the pin config settings for a specified pin group */
-static int exynos5440_pinconf_group_get(struct pinctrl_dev *pctldev,
-				unsigned int group, unsigned long *config)
-{
-	struct exynos5440_pinctrl_priv_data *priv;
-	const unsigned int *pins;
-
-	priv = pinctrl_dev_get_drvdata(pctldev);
-	pins = priv->pin_groups[group].pins;
-	exynos5440_pinconf_get(pctldev, pins[0], config);
-	return 0;
-}
-
-/* list of pinconfig callbacks for pinconfig vertical in the pinctrl code */
-static const struct pinconf_ops exynos5440_pinconf_ops = {
-	.pin_config_get		= exynos5440_pinconf_get,
-	.pin_config_set		= exynos5440_pinconf_set,
-	.pin_config_group_get	= exynos5440_pinconf_group_get,
-	.pin_config_group_set	= exynos5440_pinconf_group_set,
-};
-
-/* gpiolib gpio_set callback function */
-static void exynos5440_gpio_set(struct gpio_chip *gc, unsigned offset, int value)
-{
-	struct exynos5440_pinctrl_priv_data *priv = gpiochip_get_data(gc);
-	void __iomem *base = priv->reg_base;
-	u32 data;
-
-	data = readl(base + GPIO_VAL);
-	data &= ~(1 << offset);
-	if (value)
-		data |= 1 << offset;
-	writel(data, base + GPIO_VAL);
-}
-
-/* gpiolib gpio_get callback function */
-static int exynos5440_gpio_get(struct gpio_chip *gc, unsigned offset)
-{
-	struct exynos5440_pinctrl_priv_data *priv = gpiochip_get_data(gc);
-	void __iomem *base = priv->reg_base;
-	u32 data;
-
-	data = readl(base + GPIO_IN);
-	data >>= offset;
-	data &= 1;
-	return data;
-}
-
-/* gpiolib gpio_direction_input callback function */
-static int exynos5440_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
-{
-	struct exynos5440_pinctrl_priv_data *priv = gpiochip_get_data(gc);
-	void __iomem *base = priv->reg_base;
-	u32 data;
-
-	/* first disable the data output enable on this pin */
-	data = readl(base + GPIO_OE);
-	data &= ~(1 << offset);
-	writel(data, base + GPIO_OE);
-
-	/* now enable input on this pin */
-	data =  readl(base + GPIO_IE);
-	data |= 1 << offset;
-	writel(data, base + GPIO_IE);
-	return 0;
-}
-
-/* gpiolib gpio_direction_output callback function */
-static int exynos5440_gpio_direction_output(struct gpio_chip *gc, unsigned offset,
-							int value)
-{
-	struct exynos5440_pinctrl_priv_data *priv = gpiochip_get_data(gc);
-	void __iomem *base = priv->reg_base;
-	u32 data;
-
-	exynos5440_gpio_set(gc, offset, value);
-
-	/* first disable the data input enable on this pin */
-	data = readl(base + GPIO_IE);
-	data &= ~(1 << offset);
-	writel(data, base + GPIO_IE);
-
-	/* now enable output on this pin */
-	data =  readl(base + GPIO_OE);
-	data |= 1 << offset;
-	writel(data, base + GPIO_OE);
-	return 0;
-}
-
-/* gpiolib gpio_to_irq callback function */
-static int exynos5440_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
-{
-	struct exynos5440_pinctrl_priv_data *priv = gpiochip_get_data(gc);
-	unsigned int virq;
-
-	if (offset < 16 || offset > 23)
-		return -ENXIO;
-
-	if (!priv->irq_domain)
-		return -ENXIO;
-
-	virq = irq_create_mapping(priv->irq_domain, offset - 16);
-	return virq ? : -ENXIO;
-}
-
-/* parse the pin numbers listed in the 'samsung,exynos5440-pins' property */
-static int exynos5440_pinctrl_parse_dt_pins(struct platform_device *pdev,
-			struct device_node *cfg_np, unsigned int **pin_list,
-			unsigned int *npins)
-{
-	struct device *dev = &pdev->dev;
-	struct property *prop;
-
-	prop = of_find_property(cfg_np, "samsung,exynos5440-pins", NULL);
-	if (!prop)
-		return -ENOENT;
-
-	*npins = prop->length / sizeof(unsigned long);
-	if (!*npins) {
-		dev_err(dev, "invalid pin list in %s node", cfg_np->name);
-		return -EINVAL;
-	}
-
-	*pin_list = devm_kzalloc(dev, *npins * sizeof(**pin_list), GFP_KERNEL);
-	if (!*pin_list)
-		return -ENOMEM;
-
-	return of_property_read_u32_array(cfg_np, "samsung,exynos5440-pins",
-			*pin_list, *npins);
-}
-
-/*
- * Parse the information about all the available pin groups and pin functions
- * from device node of the pin-controller.
- */
-static int exynos5440_pinctrl_parse_dt(struct platform_device *pdev,
-				struct exynos5440_pinctrl_priv_data *priv)
-{
-	struct device *dev = &pdev->dev;
-	struct device_node *dev_np = dev->of_node;
-	struct device_node *cfg_np;
-	struct exynos5440_pin_group *groups, *grp;
-	struct exynos5440_pmx_func *functions, *func;
-	unsigned *pin_list;
-	unsigned int npins, grp_cnt, func_idx = 0;
-	char *gname, *fname;
-	int ret;
-
-	grp_cnt = of_get_child_count(dev_np);
-	if (!grp_cnt)
-		return -EINVAL;
-
-	groups = devm_kzalloc(dev, grp_cnt * sizeof(*groups), GFP_KERNEL);
-	if (!groups)
-		return -EINVAL;
-
-	grp = groups;
-
-	functions = devm_kzalloc(dev, grp_cnt * sizeof(*functions), GFP_KERNEL);
-	if (!functions)
-		return -EINVAL;
-
-	func = functions;
-
-	/*
-	 * Iterate over all the child nodes of the pin controller node
-	 * and create pin groups and pin function lists.
-	 */
-	for_each_child_of_node(dev_np, cfg_np) {
-		u32 function;
-
-		ret = exynos5440_pinctrl_parse_dt_pins(pdev, cfg_np,
-					&pin_list, &npins);
-		if (ret) {
-			gname = NULL;
-			goto skip_to_pin_function;
-		}
-
-		/* derive pin group name from the node name */
-		gname = devm_kasprintf(dev, GFP_KERNEL,
-				       "%s%s", cfg_np->name, GROUP_SUFFIX);
-		if (!gname)
-			return -ENOMEM;
-
-		grp->name = gname;
-		grp->pins = pin_list;
-		grp->num_pins = npins;
-		grp++;
-
-skip_to_pin_function:
-		ret = of_property_read_u32(cfg_np, "samsung,exynos5440-pin-function",
-						&function);
-		if (ret)
-			continue;
-
-		/* derive function name from the node name */
-		fname = devm_kasprintf(dev, GFP_KERNEL,
-				       "%s%s", cfg_np->name, FUNCTION_SUFFIX);
-		if (!fname)
-			return -ENOMEM;
-
-		func->name = fname;
-		func->groups = devm_kzalloc(dev, sizeof(char *), GFP_KERNEL);
-		if (!func->groups)
-			return -ENOMEM;
-		func->groups[0] = gname;
-		func->num_groups = gname ? 1 : 0;
-		func->function = function;
-		func++;
-		func_idx++;
-	}
-
-	priv->pin_groups = groups;
-	priv->nr_groups = grp_cnt;
-	priv->pmx_functions = functions;
-	priv->nr_functions = func_idx;
-	return 0;
-}
-
-/* register the pinctrl interface with the pinctrl subsystem */
-static int exynos5440_pinctrl_register(struct platform_device *pdev,
-				struct exynos5440_pinctrl_priv_data *priv)
-{
-	struct device *dev = &pdev->dev;
-	struct pinctrl_desc *ctrldesc;
-	struct pinctrl_dev *pctl_dev;
-	struct pinctrl_pin_desc *pindesc, *pdesc;
-	char *pin_names;
-	int pin, ret;
-
-	ctrldesc = devm_kzalloc(dev, sizeof(*ctrldesc), GFP_KERNEL);
-	if (!ctrldesc)
-		return -ENOMEM;
-
-	ctrldesc->name = "exynos5440-pinctrl";
-	ctrldesc->owner = THIS_MODULE;
-	ctrldesc->pctlops = &exynos5440_pctrl_ops;
-	ctrldesc->pmxops = &exynos5440_pinmux_ops;
-	ctrldesc->confops = &exynos5440_pinconf_ops;
-
-	pindesc = devm_kzalloc(&pdev->dev, sizeof(*pindesc) *
-				EXYNOS5440_MAX_PINS, GFP_KERNEL);
-	if (!pindesc)
-		return -ENOMEM;
-	ctrldesc->pins = pindesc;
-	ctrldesc->npins = EXYNOS5440_MAX_PINS;
-
-	/* dynamically populate the pin number and pin name for pindesc */
-	for (pin = 0, pdesc = pindesc; pin < ctrldesc->npins; pin++, pdesc++)
-		pdesc->number = pin;
-
-	/*
-	 * allocate space for storing the dynamically generated names for all
-	 * the pins which belong to this pin-controller.
-	 */
-	pin_names = devm_kzalloc(&pdev->dev, sizeof(char) * PIN_NAME_LENGTH *
-					ctrldesc->npins, GFP_KERNEL);
-	if (!pin_names)
-		return -ENOMEM;
-
-	/* for each pin, set the name of the pin */
-	for (pin = 0; pin < ctrldesc->npins; pin++) {
-		snprintf(pin_names, 6, "gpio%02d", pin);
-		pdesc = pindesc + pin;
-		pdesc->name = pin_names;
-		pin_names += PIN_NAME_LENGTH;
-	}
-
-	ret = exynos5440_pinctrl_parse_dt(pdev, priv);
-	if (ret)
-		return ret;
-
-	pctl_dev = devm_pinctrl_register(&pdev->dev, ctrldesc, priv);
-	if (IS_ERR(pctl_dev)) {
-		dev_err(&pdev->dev, "could not register pinctrl driver\n");
-		return PTR_ERR(pctl_dev);
-	}
-
-	priv->range.name = "exynos5440-pctrl-gpio-range";
-	priv->range.id = 0;
-	priv->range.base = 0;
-	priv->range.npins = EXYNOS5440_MAX_PINS;
-	priv->range.gc = priv->gc;
-	pinctrl_add_gpio_range(pctl_dev, &priv->range);
-	return 0;
-}
-
-/* register the gpiolib interface with the gpiolib subsystem */
-static int exynos5440_gpiolib_register(struct platform_device *pdev,
-				struct exynos5440_pinctrl_priv_data *priv)
-{
-	struct gpio_chip *gc;
-	int ret;
-
-	gc = devm_kzalloc(&pdev->dev, sizeof(*gc), GFP_KERNEL);
-	if (!gc)
-		return -ENOMEM;
-
-	priv->gc = gc;
-	gc->base = 0;
-	gc->ngpio = EXYNOS5440_MAX_PINS;
-	gc->parent = &pdev->dev;
-	gc->set = exynos5440_gpio_set;
-	gc->get = exynos5440_gpio_get;
-	gc->direction_input = exynos5440_gpio_direction_input;
-	gc->direction_output = exynos5440_gpio_direction_output;
-	gc->to_irq = exynos5440_gpio_to_irq;
-	gc->label = "gpiolib-exynos5440";
-	gc->owner = THIS_MODULE;
-	ret = gpiochip_add_data(gc, priv);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to register gpio_chip %s, error "
-					"code: %d\n", gc->label, ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-/* unregister the gpiolib interface with the gpiolib subsystem */
-static int exynos5440_gpiolib_unregister(struct platform_device *pdev,
-				struct exynos5440_pinctrl_priv_data *priv)
-{
-	gpiochip_remove(priv->gc);
-	return 0;
-}
-
-static void exynos5440_gpio_irq_unmask(struct irq_data *irqd)
-{
-	struct exynos5440_pinctrl_priv_data *d;
-	unsigned long gpio_int;
-
-	d = irq_data_get_irq_chip_data(irqd);
-	gpio_int = readl(d->reg_base + GPIO_INT);
-	gpio_int |= 1 << irqd->hwirq;
-	writel(gpio_int, d->reg_base + GPIO_INT);
-}
-
-static void exynos5440_gpio_irq_mask(struct irq_data *irqd)
-{
-	struct exynos5440_pinctrl_priv_data *d;
-	unsigned long gpio_int;
-
-	d = irq_data_get_irq_chip_data(irqd);
-	gpio_int = readl(d->reg_base + GPIO_INT);
-	gpio_int &= ~(1 << irqd->hwirq);
-	writel(gpio_int, d->reg_base + GPIO_INT);
-}
-
-/* irq_chip for gpio interrupts */
-static struct irq_chip exynos5440_gpio_irq_chip = {
-	.name		= "exynos5440_gpio_irq_chip",
-	.irq_unmask	= exynos5440_gpio_irq_unmask,
-	.irq_mask	= exynos5440_gpio_irq_mask,
-};
-
-/* interrupt handler for GPIO interrupts 0..7 */
-static irqreturn_t exynos5440_gpio_irq(int irq, void *data)
-{
-	struct exynos5440_gpio_intr_data *intd = data;
-	struct exynos5440_pinctrl_priv_data *d = intd->priv;
-	int virq;
-
-	virq = irq_linear_revmap(d->irq_domain, intd->gpio_int);
-	if (!virq)
-		return IRQ_NONE;
-	generic_handle_irq(virq);
-	return IRQ_HANDLED;
-}
-
-static int exynos5440_gpio_irq_map(struct irq_domain *h, unsigned int virq,
-					irq_hw_number_t hw)
-{
-	struct exynos5440_pinctrl_priv_data *d = h->host_data;
-
-	irq_set_chip_data(virq, d);
-	irq_set_chip_and_handler(virq, &exynos5440_gpio_irq_chip,
-					handle_level_irq);
-	return 0;
-}
-
-/* irq domain callbacks for gpio interrupt controller */
-static const struct irq_domain_ops exynos5440_gpio_irqd_ops = {
-	.map	= exynos5440_gpio_irq_map,
-	.xlate	= irq_domain_xlate_twocell,
-};
-
-/* setup handling of gpio interrupts */
-static int exynos5440_gpio_irq_init(struct platform_device *pdev,
-				struct exynos5440_pinctrl_priv_data *priv)
-{
-	struct device *dev = &pdev->dev;
-	struct exynos5440_gpio_intr_data *intd;
-	int i, irq, ret;
-
-	intd = devm_kzalloc(dev, sizeof(*intd) * EXYNOS5440_MAX_GPIO_INT,
-					GFP_KERNEL);
-	if (!intd)
-		return -ENOMEM;
-
-	for (i = 0; i < EXYNOS5440_MAX_GPIO_INT; i++) {
-		irq = irq_of_parse_and_map(dev->of_node, i);
-		if (irq <= 0) {
-			dev_err(dev, "irq parsing failed\n");
-			return -EINVAL;
-		}
-
-		intd->gpio_int = i;
-		intd->priv = priv;
-		ret = devm_request_irq(dev, irq, exynos5440_gpio_irq,
-					0, dev_name(dev), intd++);
-		if (ret) {
-			dev_err(dev, "irq request failed\n");
-			return -ENXIO;
-		}
-	}
-
-	priv->irq_domain = irq_domain_add_linear(dev->of_node,
-				EXYNOS5440_MAX_GPIO_INT,
-				&exynos5440_gpio_irqd_ops, priv);
-	if (!priv->irq_domain) {
-		dev_err(dev, "failed to create irq domain\n");
-		return -ENXIO;
-	}
-
-	return 0;
-}
-
-static int exynos5440_pinctrl_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct exynos5440_pinctrl_priv_data *priv;
-	struct resource *res;
-	int ret;
-
-	if (!dev->of_node) {
-		dev_err(dev, "device tree node not found\n");
-		return -ENODEV;
-	}
-
-	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	priv->reg_base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(priv->reg_base))
-		return PTR_ERR(priv->reg_base);
-
-	ret = exynos5440_gpiolib_register(pdev, priv);
-	if (ret)
-		return ret;
-
-	ret = exynos5440_pinctrl_register(pdev, priv);
-	if (ret) {
-		exynos5440_gpiolib_unregister(pdev, priv);
-		return ret;
-	}
-
-	ret = exynos5440_gpio_irq_init(pdev, priv);
-	if (ret) {
-		dev_err(dev, "failed to setup gpio interrupts\n");
-		return ret;
-	}
-
-	platform_set_drvdata(pdev, priv);
-	dev_info(dev, "EXYNOS5440 pinctrl driver registered\n");
-	return 0;
-}
-
-static const struct of_device_id exynos5440_pinctrl_dt_match[] = {
-	{ .compatible = "samsung,exynos5440-pinctrl" },
-	{},
-};
-
-static struct platform_driver exynos5440_pinctrl_driver = {
-	.probe		= exynos5440_pinctrl_probe,
-	.driver = {
-		.name	= "exynos5440-pinctrl",
-		.of_match_table = exynos5440_pinctrl_dt_match,
-		.suppress_bind_attrs = true,
-	},
-};
-
-static int __init exynos5440_pinctrl_drv_register(void)
-{
-	return platform_driver_register(&exynos5440_pinctrl_driver);
-}
-postcore_initcall(exynos5440_pinctrl_drv_register);
diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c
index 336e88d7bdb9..618945a0fd38 100644
--- a/drivers/pinctrl/samsung/pinctrl-samsung.c
+++ b/drivers/pinctrl/samsung/pinctrl-samsung.c
@@ -279,6 +279,32 @@ static int samsung_dt_node_to_map(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_FS
+/* Forward declaration which can be used by samsung_pin_dbg_show */
+static int samsung_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin,
+					unsigned long *config);
+static const char * const reg_names[] = {"CON", "DAT", "PUD", "DRV", "CON_PDN",
+					 "PUD_PDN"};
+
+static void samsung_pin_dbg_show(struct pinctrl_dev *pctldev,
+				struct seq_file *s, unsigned int pin)
+{
+	enum pincfg_type cfg_type;
+	unsigned long config;
+	int ret;
+
+	for (cfg_type = 0; cfg_type < PINCFG_TYPE_NUM; cfg_type++) {
+		config = PINCFG_PACK(cfg_type, 0);
+		ret = samsung_pinconf_get(pctldev, pin, &config);
+		if (ret < 0)
+			continue;
+
+		seq_printf(s, " %s(0x%lx)", reg_names[cfg_type],
+			   PINCFG_UNPACK_VALUE(config));
+	}
+}
+#endif
+
 /* list of pinctrl callbacks for the pinctrl core */
 static const struct pinctrl_ops samsung_pctrl_ops = {
 	.get_groups_count	= samsung_get_group_count,
@@ -286,6 +312,9 @@ static const struct pinctrl_ops samsung_pctrl_ops = {
 	.get_group_pins		= samsung_get_group_pins,
 	.dt_node_to_map		= samsung_dt_node_to_map,
 	.dt_free_map		= samsung_dt_free_map,
+#ifdef CONFIG_DEBUG_FS
+	.pin_dbg_show		= samsung_pin_dbg_show,
+#endif
 };
 
 /* check if the selector is a valid pin function selector */
diff --git a/drivers/pinctrl/sh-pfc/Kconfig b/drivers/pinctrl/sh-pfc/Kconfig
index c11b789ec583..43d950c16528 100644
--- a/drivers/pinctrl/sh-pfc/Kconfig
+++ b/drivers/pinctrl/sh-pfc/Kconfig
@@ -44,6 +44,11 @@ config PINCTRL_PFC_R8A7745
         depends on ARCH_R8A7745
         select PINCTRL_SH_PFC
 
+config PINCTRL_PFC_R8A77470
+        def_bool y
+        depends on ARCH_R8A77470
+        select PINCTRL_SH_PFC
+
 config PINCTRL_PFC_R8A7778
 	def_bool y
 	depends on ARCH_R8A7778
@@ -104,6 +109,11 @@ config PINCTRL_PFC_R8A77980
 	depends on ARCH_R8A77980
 	select PINCTRL_SH_PFC
 
+config PINCTRL_PFC_R8A77990
+        def_bool y
+        depends on ARCH_R8A77990
+        select PINCTRL_SH_PFC
+
 config PINCTRL_PFC_R8A77995
         def_bool y
         depends on ARCH_R8A77995
diff --git a/drivers/pinctrl/sh-pfc/Makefile b/drivers/pinctrl/sh-pfc/Makefile
index 463775f28cf1..d0b29c51c159 100644
--- a/drivers/pinctrl/sh-pfc/Makefile
+++ b/drivers/pinctrl/sh-pfc/Makefile
@@ -6,6 +6,7 @@ obj-$(CONFIG_PINCTRL_PFC_R8A73A4)	+= pfc-r8a73a4.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7740)	+= pfc-r8a7740.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7743)	+= pfc-r8a7791.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7745)	+= pfc-r8a7794.o
+obj-$(CONFIG_PINCTRL_PFC_R8A77470)	+= pfc-r8a77470.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7778)	+= pfc-r8a7778.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7779)	+= pfc-r8a7779.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7790)	+= pfc-r8a7790.o
@@ -19,6 +20,7 @@ obj-$(CONFIG_PINCTRL_PFC_R8A7796)	+= pfc-r8a7796.o
 obj-$(CONFIG_PINCTRL_PFC_R8A77965)	+= pfc-r8a77965.o
 obj-$(CONFIG_PINCTRL_PFC_R8A77970)	+= pfc-r8a77970.o
 obj-$(CONFIG_PINCTRL_PFC_R8A77980)	+= pfc-r8a77980.o
+obj-$(CONFIG_PINCTRL_PFC_R8A77990)	+= pfc-r8a77990.o
 obj-$(CONFIG_PINCTRL_PFC_R8A77995)	+= pfc-r8a77995.o
 obj-$(CONFIG_PINCTRL_PFC_SH7203)	+= pfc-sh7203.o
 obj-$(CONFIG_PINCTRL_PFC_SH7264)	+= pfc-sh7264.o
diff --git a/drivers/pinctrl/sh-pfc/core.c b/drivers/pinctrl/sh-pfc/core.c
index 74861b7b5b0d..eb06981538b4 100644
--- a/drivers/pinctrl/sh-pfc/core.c
+++ b/drivers/pinctrl/sh-pfc/core.c
@@ -503,6 +503,12 @@ static const struct of_device_id sh_pfc_of_table[] = {
 		.data = &r8a7745_pinmux_info,
 	},
 #endif
+#ifdef CONFIG_PINCTRL_PFC_R8A77470
+	{
+		.compatible = "renesas,pfc-r8a77470",
+		.data = &r8a77470_pinmux_info,
+	},
+#endif
 #ifdef CONFIG_PINCTRL_PFC_R8A7778
 	{
 		.compatible = "renesas,pfc-r8a7778",
@@ -575,6 +581,12 @@ static const struct of_device_id sh_pfc_of_table[] = {
 		.data = &r8a77980_pinmux_info,
 	},
 #endif
+#ifdef CONFIG_PINCTRL_PFC_R8A77990
+	{
+		.compatible = "renesas,pfc-r8a77990",
+		.data = &r8a77990_pinmux_info,
+	},
+#endif
 #ifdef CONFIG_PINCTRL_PFC_R8A77995
 	{
 		.compatible = "renesas,pfc-r8a77995",
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77470.c b/drivers/pinctrl/sh-pfc/pfc-r8a77470.c
new file mode 100644
index 000000000000..9d3ed438ec7b
--- /dev/null
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77470.c
@@ -0,0 +1,2343 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * R8A77470 processor support - PFC hardware block.
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ */
+
+#include <linux/kernel.h>
+
+#include "sh_pfc.h"
+
+#define CPU_ALL_PORT(fn, sfx)						\
+	PORT_GP_23(0, fn, sfx),						\
+	PORT_GP_23(1, fn, sfx),						\
+	PORT_GP_32(2, fn, sfx),						\
+	PORT_GP_17(3, fn, sfx),						\
+	PORT_GP_1(3, 27, fn, sfx),					\
+	PORT_GP_1(3, 28, fn, sfx),					\
+	PORT_GP_1(3, 29, fn, sfx),					\
+	PORT_GP_26(4, fn, sfx),						\
+	PORT_GP_32(5, fn, sfx)
+
+enum {
+	PINMUX_RESERVED = 0,
+
+	PINMUX_DATA_BEGIN,
+	GP_ALL(DATA),
+	PINMUX_DATA_END,
+
+	PINMUX_FUNCTION_BEGIN,
+	GP_ALL(FN),
+
+	/* GPSR0 */
+	FN_USB0_PWEN, FN_USB0_OVC, FN_USB1_PWEN, FN_USB1_OVC, FN_CLKOUT,
+	FN_IP0_3_0, FN_IP0_7_4, FN_IP0_11_8, FN_IP0_15_12, FN_IP0_19_16,
+	FN_IP0_23_20, FN_IP0_27_24, FN_IP0_31_28, FN_MMC0_CLK_SDHI1_CLK,
+	FN_MMC0_CMD_SDHI1_CMD, FN_MMC0_D0_SDHI1_D0, FN_MMC0_D1_SDHI1_D1,
+	FN_MMC0_D2_SDHI1_D2, FN_MMC0_D3_SDHI1_D3, FN_IP1_3_0,
+	FN_IP1_7_4, FN_MMC0_D6, FN_MMC0_D7,
+
+	/* GPSR1 */
+	FN_IP1_11_8, FN_IP1_15_12, FN_IP1_19_16, FN_IP1_23_20, FN_IP1_27_24,
+	FN_IP1_31_28, FN_IP2_3_0, FN_IP2_7_4, FN_IP2_11_8, FN_IP2_15_12,
+	FN_IP2_19_16, FN_IP2_23_20, FN_IP2_27_24, FN_IP2_31_28, FN_IP3_3_0,
+	FN_IP3_7_4, FN_IP3_11_8, FN_IP3_15_12, FN_IP3_19_16, FN_IP3_23_20,
+	FN_IP3_27_24, FN_IP3_31_28, FN_IP4_3_0,
+
+	/* GPSR2 */
+	FN_IP4_7_4, FN_IP4_11_8, FN_IP4_15_12, FN_IP4_19_16, FN_IP4_23_20,
+	FN_IP4_27_24, FN_IP4_31_28, FN_IP5_3_0, FN_IP5_7_4, FN_IP5_11_8,
+	FN_IP5_15_12, FN_IP5_19_16, FN_IP5_23_20, FN_IP5_27_24, FN_IP5_31_28,
+	FN_IP6_3_0, FN_IP6_7_4, FN_IP6_11_8, FN_IP6_15_12, FN_IP6_19_16,
+	FN_IP6_23_20, FN_IP6_27_24, FN_IP6_31_28, FN_IP7_3_0, FN_IP7_7_4,
+	FN_IP7_11_8, FN_IP7_15_12, FN_IP7_19_16, FN_IP7_23_20, FN_IP7_27_24,
+	FN_IP7_31_28, FN_IP8_3_0,
+
+	/* GPSR3 */
+	FN_IP8_7_4, FN_IP8_11_8, FN_IP8_15_12, FN_IP8_19_16, FN_IP8_23_20,
+	FN_IP8_27_24, FN_IP8_31_28, FN_IP9_3_0, FN_IP9_7_4, FN_IP9_11_8,
+	FN_IP9_15_12, FN_IP9_19_16, FN_IP9_23_20, FN_IP9_27_24, FN_IP9_31_28,
+	FN_IP10_3_0, FN_IP10_7_4, FN_IP10_11_8, FN_IP10_15_12, FN_IP10_19_16,
+
+	/* GPSR4 */
+	FN_IP10_23_20, FN_IP10_27_24, FN_IP10_31_28, FN_IP11_3_0, FN_IP11_7_4,
+	FN_IP11_11_8, FN_IP11_15_12, FN_IP11_19_16, FN_IP11_23_20,
+	FN_IP11_27_24, FN_IP11_31_28, FN_IP12_3_0, FN_IP12_7_4, FN_IP12_11_8,
+	FN_IP12_15_12, FN_IP12_19_16, FN_IP12_23_20, FN_IP12_27_24,
+	FN_IP12_31_28, FN_IP13_3_0, FN_IP13_7_4, FN_IP13_11_8, FN_IP13_15_12,
+	FN_IP13_19_16, FN_IP13_23_20, FN_IP13_27_24,
+
+	/* GPSR5 */
+	FN_IP13_31_28, FN_IP14_3_0, FN_IP14_7_4, FN_IP14_11_8, FN_IP14_15_12,
+	FN_IP14_19_16, FN_IP14_23_20, FN_IP14_27_24, FN_IP14_31_28,
+	FN_IP15_3_0, FN_IP15_7_4, FN_IP15_11_8, FN_IP15_15_12, FN_IP15_19_16,
+	FN_IP15_23_20, FN_IP15_27_24, FN_IP15_31_28, FN_IP16_3_0, FN_IP16_7_4,
+	FN_IP16_11_8, FN_IP16_15_12, FN_IP16_19_16, FN_IP16_23_20,
+	FN_IP16_27_24, FN_IP16_31_28, FN_IP17_3_0, FN_IP17_7_4, FN_IP17_11_8,
+	FN_IP17_15_12, FN_IP17_19_16, FN_IP17_23_20, FN_IP17_27_24,
+
+	/* IPSR0 */
+	FN_SD0_CLK, FN_SSI_SCK1_C, FN_RX3_C,
+	FN_SD0_CMD, FN_SSI_WS1_C, FN_TX3_C,
+	FN_SD0_DAT0, FN_SSI_SDATA1_C, FN_RX4_E,
+	FN_SD0_DAT1, FN_SSI_SCK0129_B, FN_TX4_E,
+	FN_SD0_DAT2, FN_SSI_WS0129_B, FN_RX5_E,
+	FN_SD0_DAT3, FN_SSI_SDATA0_B, FN_TX5_E,
+	FN_SD0_CD, FN_CAN0_RX_A,
+	FN_SD0_WP, FN_IRQ7, FN_CAN0_TX_A,
+
+	/* IPSR1 */
+	FN_MMC0_D4, FN_SD1_CD,
+	FN_MMC0_D5, FN_SD1_WP,
+	FN_D0, FN_SCL3_B, FN_RX5_B, FN_IRQ4, FN_MSIOF2_RXD_C, FN_SSI_SDATA5_B,
+	FN_D1, FN_SDA3_B, FN_TX5_B, FN_MSIOF2_TXD_C, FN_SSI_WS5_B,
+	FN_D2, FN_RX4_B, FN_SCL0_D, FN_PWM1_C, FN_MSIOF2_SCK_C, FN_SSI_SCK5_B,
+	FN_D3, FN_TX4_B, FN_SDA0_D, FN_PWM0_A, FN_MSIOF2_SYNC_C,
+	FN_D4, FN_IRQ3, FN_TCLK1_A, FN_PWM6_C,
+	FN_D5, FN_HRX2, FN_SCL1_B, FN_PWM2_C, FN_TCLK2_B,
+
+	/* IPSR2 */
+	FN_D6, FN_HTX2, FN_SDA1_B, FN_PWM4_C,
+	FN_D7, FN_HSCK2, FN_SCIF1_SCK_C, FN_IRQ6, FN_PWM5_C,
+	FN_D8, FN_HCTS2_N, FN_RX1_C, FN_SCL1_D, FN_PWM3_C,
+	FN_D9, FN_HRTS2_N, FN_TX1_C, FN_SDA1_D,
+	FN_D10, FN_MSIOF2_RXD_A, FN_HRX0_B,
+	FN_D11, FN_MSIOF2_TXD_A, FN_HTX0_B,
+	FN_D12, FN_MSIOF2_SCK_A, FN_HSCK0, FN_CAN_CLK_C,
+	FN_D13,	FN_MSIOF2_SYNC_A, FN_RX4_C,
+
+	/* IPSR3 */
+	FN_D14, FN_MSIOF2_SS1, FN_TX4_C, FN_CAN1_RX_B, FN_AVB_AVTP_CAPTURE_A,
+	FN_D15, FN_MSIOF2_SS2, FN_PWM4_A, FN_CAN1_TX_B, FN_IRQ2, FN_AVB_AVTP_MATCH_A,
+	FN_QSPI0_SPCLK, FN_WE0_N,
+	FN_QSPI0_MOSI_QSPI0_IO0, FN_BS_N,
+	FN_QSPI0_MISO_QSPI0_IO1, FN_RD_WR_N,
+	FN_QSPI0_IO2, FN_CS0_N,
+	FN_QSPI0_IO3, FN_RD_N,
+	FN_QSPI0_SSL, FN_WE1_N,
+
+	/* IPSR4 */
+	FN_EX_WAIT0, FN_CAN_CLK_B, FN_SCIF_CLK_A,
+	FN_DU0_DR0, FN_RX5_C, FN_SCL2_D, FN_A0,
+	FN_DU0_DR1, FN_TX5_C, FN_SDA2_D, FN_A1,
+	FN_DU0_DR2, FN_RX0_D, FN_SCL0_E, FN_A2,
+	FN_DU0_DR3, FN_TX0_D, FN_SDA0_E, FN_PWM0_B, FN_A3,
+	FN_DU0_DR4, FN_RX1_D, FN_A4,
+	FN_DU0_DR5, FN_TX1_D, FN_PWM1_B, FN_A5,
+	FN_DU0_DR6, FN_RX2_C, FN_A6,
+
+	/* IPSR5 */
+	FN_DU0_DR7, FN_TX2_C, FN_PWM2_B, FN_A7,
+	FN_DU0_DG0, FN_RX3_B, FN_SCL3_D, FN_A8,
+	FN_DU0_DG1, FN_TX3_B, FN_SDA3_D, FN_PWM3_B, FN_A9,
+	FN_DU0_DG2, FN_RX4_D, FN_A10,
+	FN_DU0_DG3, FN_TX4_D, FN_PWM4_B, FN_A11,
+	FN_DU0_DG4, FN_HRX0_A, FN_A12,
+	FN_DU0_DG5, FN_HTX0_A, FN_PWM5_B, FN_A13,
+	FN_DU0_DG6, FN_HRX1_C, FN_A14,
+
+	/* IPSR6 */
+	FN_DU0_DG7, FN_HTX1_C, FN_PWM6_B, FN_A15,
+	FN_DU0_DB0, FN_SCL4_D, FN_CAN0_RX_C, FN_A16,
+	FN_DU0_DB1, FN_SDA4_D, FN_CAN0_TX_C, FN_A17,
+	FN_DU0_DB2, FN_HCTS0_N, FN_A18,
+	FN_DU0_DB3, FN_HRTS0_N, FN_A19,
+	FN_DU0_DB4, FN_HCTS1_N_C, FN_A20,
+	FN_DU0_DB5, FN_HRTS1_N_C, FN_A21,
+	FN_DU0_DB6, FN_A22,
+
+	/* IPSR7 */
+	FN_DU0_DB7, FN_A23,
+	FN_DU0_DOTCLKIN, FN_A24,
+	FN_DU0_DOTCLKOUT0, FN_A25,
+	FN_DU0_DOTCLKOUT1, FN_MSIOF2_RXD_B, FN_CS1_N_A26,
+	FN_DU0_EXHSYNC_DU0_HSYNC, FN_MSIOF2_TXD_B, FN_DREQ0_N,
+	FN_DU0_EXVSYNC_DU0_VSYNC, FN_MSIOF2_SYNC_B, FN_DACK0,
+	FN_DU0_EXODDF_DU0_ODDF_DISP_CDE, FN_MSIOF2_SCK_B, FN_DRACK0,
+	FN_DU0_DISP, FN_CAN1_RX_C,
+
+	/* IPSR8 */
+	FN_DU0_CDE, FN_CAN1_TX_C,
+	FN_VI1_CLK, FN_AVB_RX_CLK, FN_ETH_REF_CLK,
+	FN_VI1_DATA0, FN_AVB_RX_DV, FN_ETH_CRS_DV,
+	FN_VI1_DATA1, FN_AVB_RXD0, FN_ETH_RXD0,
+	FN_VI1_DATA2, FN_AVB_RXD1, FN_ETH_RXD1,
+	FN_VI1_DATA3, FN_AVB_RXD2, FN_ETH_MDIO,
+	FN_VI1_DATA4, FN_AVB_RXD3, FN_ETH_RX_ER,
+	FN_VI1_DATA5, FN_AVB_RXD4, FN_ETH_LINK,
+
+	/* IPSR9 */
+	FN_VI1_DATA6, FN_AVB_RXD5, FN_ETH_TXD1,
+	FN_VI1_DATA7, FN_AVB_RXD6, FN_ETH_TX_EN,
+	FN_VI1_CLKENB, FN_SCL3_A, FN_AVB_RXD7, FN_ETH_MAGIC,
+	FN_VI1_FIELD, FN_SDA3_A, FN_AVB_RX_ER, FN_ETH_TXD0,
+	FN_VI1_HSYNC_N,	FN_RX0_B, FN_SCL0_C, FN_AVB_GTXREFCLK, FN_ETH_MDC,
+	FN_VI1_VSYNC_N,	FN_TX0_B, FN_SDA0_C, FN_AUDIO_CLKOUT_B, FN_AVB_TX_CLK,
+	FN_VI1_DATA8, FN_SCL2_B, FN_AVB_TX_EN,
+	FN_VI1_DATA9, FN_SDA2_B, FN_AVB_TXD0,
+
+	/* IPSR10 */
+	FN_VI1_DATA10, FN_CAN0_RX_B, FN_AVB_TXD1,
+	FN_VI1_DATA11, FN_CAN0_TX_B, FN_AVB_TXD2,
+	FN_AVB_TXD3, FN_AUDIO_CLKA_B, FN_SSI_SCK1_D, FN_RX5_F, FN_MSIOF0_RXD_B,
+	FN_AVB_TXD4, FN_AUDIO_CLKB_B, FN_SSI_WS1_D, FN_TX5_F, FN_MSIOF0_TXD_B,
+	FN_AVB_TXD5, FN_SCIF_CLK_B, FN_AUDIO_CLKC_B, FN_SSI_SDATA1_D, FN_MSIOF0_SCK_B,
+	FN_SCL0_A, FN_RX0_C, FN_PWM5_A,	FN_TCLK1_B, FN_AVB_TXD6, FN_CAN1_RX_D, FN_MSIOF0_SYNC_B,
+	FN_SDA0_A, FN_TX0_C, FN_IRQ5, FN_CAN_CLK_A, FN_AVB_GTX_CLK, FN_CAN1_TX_D, FN_DVC_MUTE,
+	FN_SCL1_A, FN_RX4_A, FN_PWM5_D, FN_DU1_DR0, FN_SSI_SCK6_B, FN_VI0_G0,
+
+	/* IPSR11 */
+	FN_SDA1_A, FN_TX4_A, FN_DU1_DR1, FN_SSI_WS6_B, FN_VI0_G1,
+	FN_MSIOF0_RXD_A, FN_RX5_A, FN_SCL2_C, FN_DU1_DR2, FN_QSPI1_MOSI_QSPI1_IO0, FN_SSI_SDATA6_B, FN_VI0_G2,
+	FN_MSIOF0_TXD_A, FN_TX5_A, FN_SDA2_C, FN_DU1_DR3, FN_QSPI1_MISO_QSPI1_IO1, FN_SSI_WS78_B, FN_VI0_G3,
+	FN_MSIOF0_SCK_A, FN_IRQ0, FN_DU1_DR4, FN_QSPI1_SPCLK, FN_SSI_SCK78_B, FN_VI0_G4,
+	FN_MSIOF0_SYNC_A, FN_PWM1_A, FN_DU1_DR5, FN_QSPI1_IO2, FN_SSI_SDATA7_B,
+	FN_MSIOF0_SS1_A, FN_DU1_DR6, FN_QSPI1_IO3, FN_SSI_SDATA8_B,
+	FN_MSIOF0_SS2_A, FN_DU1_DR7, FN_QSPI1_SSL,
+	FN_HRX1_A, FN_SCL4_A, FN_PWM6_A, FN_DU1_DG0, FN_RX0_A,
+
+	/* IPSR12 */
+	FN_HTX1_A, FN_SDA4_A, FN_DU1_DG1, FN_TX0_A,
+	FN_HCTS1_N_A, FN_PWM2_A, FN_DU1_DG2, FN_REMOCON_B,
+	FN_HRTS1_N_A, FN_DU1_DG3, FN_SSI_WS1_B, FN_IRQ1,
+	FN_SD2_CLK, FN_HSCK1, FN_DU1_DG4, FN_SSI_SCK1_B,
+	FN_SD2_CMD, FN_SCIF1_SCK_A, FN_TCLK2_A, FN_DU1_DG5, FN_SSI_SCK2_B, FN_PWM3_A,
+	FN_SD2_DAT0, FN_RX1_A, FN_SCL1_E, FN_DU1_DG6, FN_SSI_SDATA1_B,
+	FN_SD2_DAT1, FN_TX1_A, FN_SDA1_E, FN_DU1_DG7, FN_SSI_WS2_B,
+	FN_SD2_DAT2, FN_RX2_A, FN_DU1_DB0, FN_SSI_SDATA2_B,
+
+	/* IPSR13 */
+	FN_SD2_DAT3, FN_TX2_A, FN_DU1_DB1, FN_SSI_WS9_B,
+	FN_SD2_CD, FN_SCIF2_SCK_A, FN_DU1_DB2, FN_SSI_SCK9_B,
+	FN_SD2_WP, FN_SCIF3_SCK, FN_DU1_DB3, FN_SSI_SDATA9_B,
+	FN_RX3_A, FN_SCL1_C, FN_MSIOF1_RXD_B, FN_DU1_DB4, FN_AUDIO_CLKA_C, FN_SSI_SDATA4_B,
+	FN_TX3_A, FN_SDA1_C, FN_MSIOF1_TXD_B, FN_DU1_DB5, FN_AUDIO_CLKB_C, FN_SSI_WS4_B,
+	FN_SCL2_A, FN_MSIOF1_SCK_B, FN_DU1_DB6, FN_AUDIO_CLKC_C, FN_SSI_SCK4_B,
+	FN_SDA2_A, FN_MSIOF1_SYNC_B, FN_DU1_DB7, FN_AUDIO_CLKOUT_C,
+	FN_SSI_SCK5_A, FN_DU1_DOTCLKOUT1,
+
+	/* IPSR14 */
+	FN_SSI_WS5_A, FN_SCL3_C, FN_DU1_DOTCLKIN,
+	FN_SSI_SDATA5_A, FN_SDA3_C, FN_DU1_DOTCLKOUT0,
+	FN_SSI_SCK6_A, FN_DU1_EXODDF_DU1_ODDF_DISP_CDE,
+	FN_SSI_WS6_A, FN_SCL4_C, FN_DU1_EXHSYNC_DU1_HSYNC,
+	FN_SSI_SDATA6_A, FN_SDA4_C, FN_DU1_EXVSYNC_DU1_VSYNC,
+	FN_SSI_SCK78_A, FN_SDA4_E, FN_DU1_DISP,
+	FN_SSI_WS78_A, FN_SCL4_E, FN_DU1_CDE,
+	FN_SSI_SDATA7_A, FN_IRQ8, FN_AUDIO_CLKA_D, FN_CAN_CLK_D, FN_VI0_G5,
+
+	/* IPSR15 */
+	FN_SSI_SCK0129_A, FN_MSIOF1_RXD_A, FN_RX5_D, FN_VI0_G6,
+	FN_SSI_WS0129_A, FN_MSIOF1_TXD_A, FN_TX5_D, FN_VI0_G7,
+	FN_SSI_SDATA0_A, FN_MSIOF1_SYNC_A, FN_PWM0_C, FN_VI0_R0,
+	FN_SSI_SCK34, FN_MSIOF1_SCK_A, FN_AVB_MDC, FN_DACK1, FN_VI0_R1,
+	FN_SSI_WS34, FN_MSIOF1_SS1_A, FN_AVB_MDIO, FN_CAN1_RX_A, FN_DREQ1_N, FN_VI0_R2,
+	FN_SSI_SDATA3, FN_MSIOF1_SS2_A, FN_AVB_LINK, FN_CAN1_TX_A, FN_DREQ2_N, FN_VI0_R3,
+	FN_SSI_SCK4_A, FN_AVB_MAGIC, FN_VI0_R4,
+	FN_SSI_WS4_A, FN_AVB_PHY_INT, FN_VI0_R5,
+
+	/* IPSR16 */
+	FN_SSI_SDATA4_A, FN_AVB_CRS, FN_VI0_R6,
+	FN_SSI_SCK1_A, FN_SCIF1_SCK_B, FN_PWM1_D, FN_IRQ9, FN_REMOCON_A, FN_DACK2, FN_VI0_CLK, FN_AVB_COL,
+	FN_SSI_SDATA8_A, FN_RX1_B, FN_CAN0_RX_D, FN_AVB_AVTP_CAPTURE_B,	FN_VI0_R7,
+	FN_SSI_WS1_A, FN_TX1_B, FN_CAN0_TX_D, FN_AVB_AVTP_MATCH_B, FN_VI0_DATA0_VI0_B0,
+	FN_SSI_SDATA1_A, FN_HRX1_B, FN_VI0_DATA1_VI0_B1,
+	FN_SSI_SCK2_A, FN_HTX1_B, FN_AVB_TXD7, FN_VI0_DATA2_VI0_B2,
+	FN_SSI_WS2_A, FN_HCTS1_N_B, FN_AVB_TX_ER, FN_VI0_DATA3_VI0_B3,
+	FN_SSI_SDATA2_A, FN_HRTS1_N_B, FN_VI0_DATA4_VI0_B4,
+
+	/* IPSR17 */
+	FN_SSI_SCK9_A, FN_RX2_B, FN_SCL3_E, FN_EX_WAIT1, FN_VI0_DATA5_VI0_B5,
+	FN_SSI_WS9_A, FN_TX2_B, FN_SDA3_E, FN_VI0_DATA6_VI0_B6,
+	FN_SSI_SDATA9_A, FN_SCIF2_SCK_B, FN_PWM2_D, FN_VI0_DATA7_VI0_B7,
+	FN_AUDIO_CLKA_A, FN_SCL0_B, FN_VI0_CLKENB,
+	FN_AUDIO_CLKB_A, FN_SDA0_B, FN_VI0_FIELD,
+	FN_AUDIO_CLKC_A, FN_SCL4_B, FN_VI0_HSYNC_N,
+	FN_AUDIO_CLKOUT_A, FN_SDA4_B, FN_VI0_VSYNC_N,
+
+	/* MOD_SEL0 */
+	FN_SEL_ADGA_0, FN_SEL_ADGA_1, FN_SEL_ADGA_2, FN_SEL_ADGA_3,
+	FN_SEL_CANCLK_0, FN_SEL_CANCLK_1, FN_SEL_CANCLK_2, FN_SEL_CANCLK_3,
+	FN_SEL_CAN1_0, FN_SEL_CAN1_1, FN_SEL_CAN1_2, FN_SEL_CAN1_3,
+	FN_SEL_CAN0_0, FN_SEL_CAN0_1, FN_SEL_CAN0_2, FN_SEL_CAN0_3,
+	FN_SEL_I2C04_0, FN_SEL_I2C04_1,	FN_SEL_I2C04_2, FN_SEL_I2C04_3, FN_SEL_I2C04_4,
+	FN_SEL_I2C03_0,	FN_SEL_I2C03_1, FN_SEL_I2C03_2, FN_SEL_I2C03_3, FN_SEL_I2C03_4,
+	FN_SEL_I2C02_0, FN_SEL_I2C02_1, FN_SEL_I2C02_2, FN_SEL_I2C02_3,
+	FN_SEL_I2C01_0, FN_SEL_I2C01_1, FN_SEL_I2C01_2, FN_SEL_I2C01_3, FN_SEL_I2C01_4,
+	FN_SEL_I2C00_0, FN_SEL_I2C00_1, FN_SEL_I2C00_2, FN_SEL_I2C00_3, FN_SEL_I2C00_4,
+	FN_SEL_AVB_0, FN_SEL_AVB_1,
+
+	/* MOD_SEL1 */
+	FN_SEL_SCIFCLK_0, FN_SEL_SCIFCLK_1,
+	FN_SEL_SCIF5_0, FN_SEL_SCIF5_1,	FN_SEL_SCIF5_2, FN_SEL_SCIF5_3, FN_SEL_SCIF5_4, FN_SEL_SCIF5_5,
+	FN_SEL_SCIF4_0, FN_SEL_SCIF4_1, FN_SEL_SCIF4_2, FN_SEL_SCIF4_3,	FN_SEL_SCIF4_4,
+	FN_SEL_SCIF3_0, FN_SEL_SCIF3_1, FN_SEL_SCIF3_2,
+	FN_SEL_SCIF2_0, FN_SEL_SCIF2_1, FN_SEL_SCIF2_2,
+	FN_SEL_SCIF2_CLK_0, FN_SEL_SCIF2_CLK_1,
+	FN_SEL_SCIF1_0, FN_SEL_SCIF1_1, FN_SEL_SCIF1_2,	FN_SEL_SCIF1_3,
+	FN_SEL_SCIF0_0, FN_SEL_SCIF0_1, FN_SEL_SCIF0_2,	FN_SEL_SCIF0_3,
+	FN_SEL_MSIOF2_0, FN_SEL_MSIOF2_1, FN_SEL_MSIOF2_2,
+	FN_SEL_MSIOF1_0, FN_SEL_MSIOF1_1,
+	FN_SEL_MSIOF0_0, FN_SEL_MSIOF0_1,
+	FN_SEL_RCN_0, FN_SEL_RCN_1,
+	FN_SEL_TMU2_0, FN_SEL_TMU2_1,
+	FN_SEL_TMU1_0, FN_SEL_TMU1_1,
+	FN_SEL_HSCIF1_0, FN_SEL_HSCIF1_1, FN_SEL_HSCIF1_2,
+	FN_SEL_HSCIF0_0, FN_SEL_HSCIF0_1,
+
+	/* MOD_SEL2 */
+	FN_SEL_ADGB_0, FN_SEL_ADGB_1, FN_SEL_ADGB_2,
+	FN_SEL_ADGC_0, FN_SEL_ADGC_1, FN_SEL_ADGC_2,
+	FN_SEL_SSI9_0, FN_SEL_SSI9_1,
+	FN_SEL_SSI8_0, FN_SEL_SSI8_1,
+	FN_SEL_SSI7_0, FN_SEL_SSI7_1,
+	FN_SEL_SSI6_0, FN_SEL_SSI6_1,
+	FN_SEL_SSI5_0, FN_SEL_SSI5_1,
+	FN_SEL_SSI4_0, FN_SEL_SSI4_1,
+	FN_SEL_SSI2_0, FN_SEL_SSI2_1,
+	FN_SEL_SSI1_0, FN_SEL_SSI1_1, FN_SEL_SSI1_2, FN_SEL_SSI1_3,
+	FN_SEL_SSI0_0, FN_SEL_SSI0_1,
+	PINMUX_FUNCTION_END,
+
+	PINMUX_MARK_BEGIN,
+
+	USB0_PWEN_MARK, USB0_OVC_MARK, USB1_PWEN_MARK, USB1_OVC_MARK,
+	CLKOUT_MARK, MMC0_CLK_SDHI1_CLK_MARK, MMC0_CMD_SDHI1_CMD_MARK,
+	MMC0_D0_SDHI1_D0_MARK, MMC0_D1_SDHI1_D1_MARK,
+	MMC0_D2_SDHI1_D2_MARK, MMC0_D3_SDHI1_D3_MARK, MMC0_D6_MARK,
+	MMC0_D7_MARK,
+
+	/* IPSR0 */
+	SD0_CLK_MARK, SSI_SCK1_C_MARK, RX3_C_MARK,
+	SD0_CMD_MARK, SSI_WS1_C_MARK, TX3_C_MARK,
+	SD0_DAT0_MARK, SSI_SDATA1_C_MARK, RX4_E_MARK,
+	SD0_DAT1_MARK, SSI_SCK0129_B_MARK, TX4_E_MARK,
+	SD0_DAT2_MARK, SSI_WS0129_B_MARK, RX5_E_MARK,
+	SD0_DAT3_MARK, SSI_SDATA0_B_MARK, TX5_E_MARK,
+	SD0_CD_MARK, CAN0_RX_A_MARK,
+	SD0_WP_MARK, IRQ7_MARK,	CAN0_TX_A_MARK,
+
+	/* IPSR1 */
+	MMC0_D4_MARK, SD1_CD_MARK,
+	MMC0_D5_MARK, SD1_WP_MARK,
+	D0_MARK, SCL3_B_MARK, RX5_B_MARK, IRQ4_MARK, MSIOF2_RXD_C_MARK,	SSI_SDATA5_B_MARK,
+	D1_MARK, SDA3_B_MARK, TX5_B_MARK, MSIOF2_TXD_C_MARK, SSI_WS5_B_MARK,
+	D2_MARK, RX4_B_MARK, SCL0_D_MARK, PWM1_C_MARK, MSIOF2_SCK_C_MARK, SSI_SCK5_B_MARK,
+	D3_MARK, TX4_B_MARK, SDA0_D_MARK, PWM0_A_MARK, MSIOF2_SYNC_C_MARK,
+	D4_MARK, IRQ3_MARK, TCLK1_A_MARK, PWM6_C_MARK,
+	D5_MARK, HRX2_MARK, SCL1_B_MARK, PWM2_C_MARK, TCLK2_B_MARK,
+
+	/* IPSR2 */
+	D6_MARK, HTX2_MARK, SDA1_B_MARK, PWM4_C_MARK,
+	D7_MARK, HSCK2_MARK, SCIF1_SCK_C_MARK, IRQ6_MARK, PWM5_C_MARK,
+	D8_MARK, HCTS2_N_MARK, RX1_C_MARK, SCL1_D_MARK,	PWM3_C_MARK,
+	D9_MARK, HRTS2_N_MARK, TX1_C_MARK, SDA1_D_MARK,
+	D10_MARK, MSIOF2_RXD_A_MARK, HRX0_B_MARK,
+	D11_MARK, MSIOF2_TXD_A_MARK, HTX0_B_MARK,
+	D12_MARK, MSIOF2_SCK_A_MARK, HSCK0_MARK, CAN_CLK_C_MARK,
+	D13_MARK, MSIOF2_SYNC_A_MARK, RX4_C_MARK,
+
+	/* IPSR3 */
+	D14_MARK, MSIOF2_SS1_MARK, TX4_C_MARK, CAN1_RX_B_MARK, AVB_AVTP_CAPTURE_A_MARK,
+	D15_MARK, MSIOF2_SS2_MARK, PWM4_A_MARK, CAN1_TX_B_MARK, IRQ2_MARK, AVB_AVTP_MATCH_A_MARK,
+	QSPI0_SPCLK_MARK, WE0_N_MARK,
+	QSPI0_MOSI_QSPI0_IO0_MARK, BS_N_MARK,
+	QSPI0_MISO_QSPI0_IO1_MARK, RD_WR_N_MARK,
+	QSPI0_IO2_MARK, CS0_N_MARK,
+	QSPI0_IO3_MARK, RD_N_MARK,
+	QSPI0_SSL_MARK, WE1_N_MARK,
+
+	/* IPSR4 */
+	EX_WAIT0_MARK, CAN_CLK_B_MARK, SCIF_CLK_A_MARK,
+	DU0_DR0_MARK, RX5_C_MARK, SCL2_D_MARK, A0_MARK,
+	DU0_DR1_MARK, TX5_C_MARK, SDA2_D_MARK, A1_MARK,
+	DU0_DR2_MARK, RX0_D_MARK, SCL0_E_MARK, A2_MARK,
+	DU0_DR3_MARK, TX0_D_MARK, SDA0_E_MARK, PWM0_B_MARK, A3_MARK,
+	DU0_DR4_MARK, RX1_D_MARK, A4_MARK,
+	DU0_DR5_MARK, TX1_D_MARK, PWM1_B_MARK, A5_MARK,
+	DU0_DR6_MARK, RX2_C_MARK, A6_MARK,
+
+	/* IPSR5 */
+	DU0_DR7_MARK, TX2_C_MARK, PWM2_B_MARK, A7_MARK,
+	DU0_DG0_MARK, RX3_B_MARK, SCL3_D_MARK, A8_MARK,
+	DU0_DG1_MARK, TX3_B_MARK, SDA3_D_MARK, PWM3_B_MARK, A9_MARK,
+	DU0_DG2_MARK, RX4_D_MARK, A10_MARK,
+	DU0_DG3_MARK, TX4_D_MARK, PWM4_B_MARK, A11_MARK,
+	DU0_DG4_MARK, HRX0_A_MARK, A12_MARK,
+	DU0_DG5_MARK, HTX0_A_MARK, PWM5_B_MARK, A13_MARK,
+	DU0_DG6_MARK, HRX1_C_MARK, A14_MARK,
+
+	/* IPSR6 */
+	DU0_DG7_MARK, HTX1_C_MARK, PWM6_B_MARK, A15_MARK,
+	DU0_DB0_MARK, SCL4_D_MARK, CAN0_RX_C_MARK, A16_MARK,
+	DU0_DB1_MARK, SDA4_D_MARK, CAN0_TX_C_MARK, A17_MARK,
+	DU0_DB2_MARK, HCTS0_N_MARK, A18_MARK,
+	DU0_DB3_MARK, HRTS0_N_MARK, A19_MARK,
+	DU0_DB4_MARK, HCTS1_N_C_MARK, A20_MARK,
+	DU0_DB5_MARK, HRTS1_N_C_MARK, A21_MARK,
+	DU0_DB6_MARK, A22_MARK,
+
+	/* IPSR7 */
+	DU0_DB7_MARK, A23_MARK,
+	DU0_DOTCLKIN_MARK, A24_MARK,
+	DU0_DOTCLKOUT0_MARK, A25_MARK,
+	DU0_DOTCLKOUT1_MARK, MSIOF2_RXD_B_MARK, CS1_N_A26_MARK,
+	DU0_EXHSYNC_DU0_HSYNC_MARK, MSIOF2_TXD_B_MARK, DREQ0_N_MARK,
+	DU0_EXVSYNC_DU0_VSYNC_MARK, MSIOF2_SYNC_B_MARK, DACK0_MARK,
+	DU0_EXODDF_DU0_ODDF_DISP_CDE_MARK, MSIOF2_SCK_B_MARK, DRACK0_MARK,
+	DU0_DISP_MARK, CAN1_RX_C_MARK,
+
+	/* IPSR8 */
+	DU0_CDE_MARK, CAN1_TX_C_MARK,
+	VI1_CLK_MARK, AVB_RX_CLK_MARK, ETH_REF_CLK_MARK,
+	VI1_DATA0_MARK, AVB_RX_DV_MARK, ETH_CRS_DV_MARK,
+	VI1_DATA1_MARK, AVB_RXD0_MARK, ETH_RXD0_MARK,
+	VI1_DATA2_MARK, AVB_RXD1_MARK, ETH_RXD1_MARK,
+	VI1_DATA3_MARK, AVB_RXD2_MARK, ETH_MDIO_MARK,
+	VI1_DATA4_MARK, AVB_RXD3_MARK, ETH_RX_ER_MARK,
+	VI1_DATA5_MARK, AVB_RXD4_MARK, ETH_LINK_MARK,
+
+	/* IPSR9 */
+	VI1_DATA6_MARK, AVB_RXD5_MARK, ETH_TXD1_MARK,
+	VI1_DATA7_MARK,	AVB_RXD6_MARK, ETH_TX_EN_MARK,
+	VI1_CLKENB_MARK, SCL3_A_MARK, AVB_RXD7_MARK, ETH_MAGIC_MARK,
+	VI1_FIELD_MARK, SDA3_A_MARK, AVB_RX_ER_MARK, ETH_TXD0_MARK,
+	VI1_HSYNC_N_MARK, RX0_B_MARK, SCL0_C_MARK, AVB_GTXREFCLK_MARK, ETH_MDC_MARK,
+	VI1_VSYNC_N_MARK, TX0_B_MARK, SDA0_C_MARK, AUDIO_CLKOUT_B_MARK, AVB_TX_CLK_MARK,
+	VI1_DATA8_MARK, SCL2_B_MARK, AVB_TX_EN_MARK,
+	VI1_DATA9_MARK, SDA2_B_MARK, AVB_TXD0_MARK,
+
+	/* IPSR10 */
+	VI1_DATA10_MARK, CAN0_RX_B_MARK, AVB_TXD1_MARK,
+	VI1_DATA11_MARK, CAN0_TX_B_MARK, AVB_TXD2_MARK,
+	AVB_TXD3_MARK, AUDIO_CLKA_B_MARK, SSI_SCK1_D_MARK, RX5_F_MARK, MSIOF0_RXD_B_MARK,
+	AVB_TXD4_MARK, AUDIO_CLKB_B_MARK, SSI_WS1_D_MARK, TX5_F_MARK, MSIOF0_TXD_B_MARK,
+	AVB_TXD5_MARK, SCIF_CLK_B_MARK, AUDIO_CLKC_B_MARK, SSI_SDATA1_D_MARK, MSIOF0_SCK_B_MARK,
+	SCL0_A_MARK, RX0_C_MARK, PWM5_A_MARK, TCLK1_B_MARK, AVB_TXD6_MARK, CAN1_RX_D_MARK, MSIOF0_SYNC_B_MARK,
+	SDA0_A_MARK, TX0_C_MARK, IRQ5_MARK, CAN_CLK_A_MARK, AVB_GTX_CLK_MARK, CAN1_TX_D_MARK, DVC_MUTE_MARK,
+	SCL1_A_MARK, RX4_A_MARK, PWM5_D_MARK, DU1_DR0_MARK, SSI_SCK6_B_MARK, VI0_G0_MARK,
+
+	/* IPSR11 */
+	SDA1_A_MARK, TX4_A_MARK, DU1_DR1_MARK, SSI_WS6_B_MARK, VI0_G1_MARK,
+	MSIOF0_RXD_A_MARK, RX5_A_MARK, SCL2_C_MARK, DU1_DR2_MARK, QSPI1_MOSI_QSPI1_IO0_MARK, SSI_SDATA6_B_MARK, VI0_G2_MARK,
+	MSIOF0_TXD_A_MARK, TX5_A_MARK, SDA2_C_MARK, DU1_DR3_MARK, QSPI1_MISO_QSPI1_IO1_MARK, SSI_WS78_B_MARK, VI0_G3_MARK,
+	MSIOF0_SCK_A_MARK, IRQ0_MARK, DU1_DR4_MARK, QSPI1_SPCLK_MARK, SSI_SCK78_B_MARK, VI0_G4_MARK,
+	MSIOF0_SYNC_A_MARK, PWM1_A_MARK, DU1_DR5_MARK, QSPI1_IO2_MARK, SSI_SDATA7_B_MARK,
+	MSIOF0_SS1_A_MARK, DU1_DR6_MARK, QSPI1_IO3_MARK, SSI_SDATA8_B_MARK,
+	MSIOF0_SS2_A_MARK, DU1_DR7_MARK, QSPI1_SSL_MARK,
+	HRX1_A_MARK, SCL4_A_MARK, PWM6_A_MARK, DU1_DG0_MARK, RX0_A_MARK,
+
+	/* IPSR12 */
+	HTX1_A_MARK, SDA4_A_MARK, DU1_DG1_MARK, TX0_A_MARK,
+	HCTS1_N_A_MARK, PWM2_A_MARK, DU1_DG2_MARK, REMOCON_B_MARK,
+	HRTS1_N_A_MARK, DU1_DG3_MARK, SSI_WS1_B_MARK, IRQ1_MARK,
+	SD2_CLK_MARK, HSCK1_MARK, DU1_DG4_MARK, SSI_SCK1_B_MARK,
+	SD2_CMD_MARK, SCIF1_SCK_A_MARK, TCLK2_A_MARK, DU1_DG5_MARK, SSI_SCK2_B_MARK, PWM3_A_MARK,
+	SD2_DAT0_MARK, RX1_A_MARK, SCL1_E_MARK, DU1_DG6_MARK, SSI_SDATA1_B_MARK,
+	SD2_DAT1_MARK, TX1_A_MARK, SDA1_E_MARK, DU1_DG7_MARK, SSI_WS2_B_MARK,
+	SD2_DAT2_MARK, RX2_A_MARK, DU1_DB0_MARK, SSI_SDATA2_B_MARK,
+
+	/* IPSR13 */
+	SD2_DAT3_MARK, TX2_A_MARK, DU1_DB1_MARK, SSI_WS9_B_MARK,
+	SD2_CD_MARK, SCIF2_SCK_A_MARK, DU1_DB2_MARK, SSI_SCK9_B_MARK,
+	SD2_WP_MARK, SCIF3_SCK_MARK, DU1_DB3_MARK, SSI_SDATA9_B_MARK,
+	RX3_A_MARK, SCL1_C_MARK, MSIOF1_RXD_B_MARK, DU1_DB4_MARK, AUDIO_CLKA_C_MARK, SSI_SDATA4_B_MARK,
+	TX3_A_MARK, SDA1_C_MARK, MSIOF1_TXD_B_MARK, DU1_DB5_MARK, AUDIO_CLKB_C_MARK, SSI_WS4_B_MARK,
+	SCL2_A_MARK, MSIOF1_SCK_B_MARK, DU1_DB6_MARK, AUDIO_CLKC_C_MARK, SSI_SCK4_B_MARK,
+	SDA2_A_MARK, MSIOF1_SYNC_B_MARK, DU1_DB7_MARK, AUDIO_CLKOUT_C_MARK,
+	SSI_SCK5_A_MARK, DU1_DOTCLKOUT1_MARK,
+
+	/* IPSR14 */
+	SSI_WS5_A_MARK, SCL3_C_MARK, DU1_DOTCLKIN_MARK,
+	SSI_SDATA5_A_MARK, SDA3_C_MARK, DU1_DOTCLKOUT0_MARK,
+	SSI_SCK6_A_MARK, DU1_EXODDF_DU1_ODDF_DISP_CDE_MARK,
+	SSI_WS6_A_MARK, SCL4_C_MARK, DU1_EXHSYNC_DU1_HSYNC_MARK,
+	SSI_SDATA6_A_MARK, SDA4_C_MARK, DU1_EXVSYNC_DU1_VSYNC_MARK,
+	SSI_SCK78_A_MARK, SDA4_E_MARK, DU1_DISP_MARK,
+	SSI_WS78_A_MARK, SCL4_E_MARK, DU1_CDE_MARK,
+	SSI_SDATA7_A_MARK, IRQ8_MARK, AUDIO_CLKA_D_MARK, CAN_CLK_D_MARK, VI0_G5_MARK,
+
+	/* IPSR15 */
+	SSI_SCK0129_A_MARK, MSIOF1_RXD_A_MARK, RX5_D_MARK, VI0_G6_MARK,
+	SSI_WS0129_A_MARK, MSIOF1_TXD_A_MARK, TX5_D_MARK, VI0_G7_MARK,
+	SSI_SDATA0_A_MARK, MSIOF1_SYNC_A_MARK, PWM0_C_MARK, VI0_R0_MARK,
+	SSI_SCK34_MARK, MSIOF1_SCK_A_MARK, AVB_MDC_MARK, DACK1_MARK, VI0_R1_MARK,
+	SSI_WS34_MARK, MSIOF1_SS1_A_MARK, AVB_MDIO_MARK, CAN1_RX_A_MARK, DREQ1_N_MARK, VI0_R2_MARK,
+	SSI_SDATA3_MARK, MSIOF1_SS2_A_MARK, AVB_LINK_MARK, CAN1_TX_A_MARK, DREQ2_N_MARK, VI0_R3_MARK,
+	SSI_SCK4_A_MARK, AVB_MAGIC_MARK, VI0_R4_MARK,
+	SSI_WS4_A_MARK, AVB_PHY_INT_MARK, VI0_R5_MARK,
+
+	/* IPSR16 */
+	SSI_SDATA4_A_MARK, AVB_CRS_MARK, VI0_R6_MARK,
+	SSI_SCK1_A_MARK, SCIF1_SCK_B_MARK, PWM1_D_MARK, IRQ9_MARK, REMOCON_A_MARK, DACK2_MARK, VI0_CLK_MARK, AVB_COL_MARK,
+	SSI_SDATA8_A_MARK, RX1_B_MARK, CAN0_RX_D_MARK, AVB_AVTP_CAPTURE_B_MARK, VI0_R7_MARK,
+	SSI_WS1_A_MARK,	TX1_B_MARK, CAN0_TX_D_MARK, AVB_AVTP_MATCH_B_MARK, VI0_DATA0_VI0_B0_MARK,
+	SSI_SDATA1_A_MARK, HRX1_B_MARK, VI0_DATA1_VI0_B1_MARK,
+	SSI_SCK2_A_MARK, HTX1_B_MARK, AVB_TXD7_MARK, VI0_DATA2_VI0_B2_MARK,
+	SSI_WS2_A_MARK, HCTS1_N_B_MARK, AVB_TX_ER_MARK, VI0_DATA3_VI0_B3_MARK,
+	SSI_SDATA2_A_MARK, HRTS1_N_B_MARK, VI0_DATA4_VI0_B4_MARK,
+
+	/* IPSR17 */
+	SSI_SCK9_A_MARK, RX2_B_MARK, SCL3_E_MARK, EX_WAIT1_MARK, VI0_DATA5_VI0_B5_MARK,
+	SSI_WS9_A_MARK, TX2_B_MARK, SDA3_E_MARK, VI0_DATA6_VI0_B6_MARK,
+	SSI_SDATA9_A_MARK, SCIF2_SCK_B_MARK, PWM2_D_MARK, VI0_DATA7_VI0_B7_MARK,
+	AUDIO_CLKA_A_MARK, SCL0_B_MARK, VI0_CLKENB_MARK,
+	AUDIO_CLKB_A_MARK, SDA0_B_MARK,	VI0_FIELD_MARK,
+	AUDIO_CLKC_A_MARK, SCL4_B_MARK, VI0_HSYNC_N_MARK,
+	AUDIO_CLKOUT_A_MARK, SDA4_B_MARK, VI0_VSYNC_N_MARK,
+
+	PINMUX_MARK_END,
+};
+
+static const u16 pinmux_data[] = {
+	PINMUX_DATA_GP_ALL(), /* PINMUX_DATA(GP_M_N_DATA, GP_M_N_FN...), */
+
+	PINMUX_SINGLE(USB0_PWEN),
+	PINMUX_SINGLE(USB0_OVC),
+	PINMUX_SINGLE(USB1_PWEN),
+	PINMUX_SINGLE(USB1_OVC),
+	PINMUX_SINGLE(CLKOUT),
+	PINMUX_SINGLE(MMC0_CLK_SDHI1_CLK),
+	PINMUX_SINGLE(MMC0_CMD_SDHI1_CMD),
+	PINMUX_SINGLE(MMC0_D0_SDHI1_D0),
+	PINMUX_SINGLE(MMC0_D1_SDHI1_D1),
+	PINMUX_SINGLE(MMC0_D2_SDHI1_D2),
+	PINMUX_SINGLE(MMC0_D3_SDHI1_D3),
+	PINMUX_SINGLE(MMC0_D6),
+	PINMUX_SINGLE(MMC0_D7),
+
+	/* IPSR0 */
+	PINMUX_IPSR_GPSR(IP0_3_0, SD0_CLK),
+	PINMUX_IPSR_MSEL(IP0_3_0, SSI_SCK1_C, SEL_SSI1_2),
+	PINMUX_IPSR_MSEL(IP0_3_0, RX3_C, SEL_SCIF3_2),
+	PINMUX_IPSR_GPSR(IP0_7_4, SD0_CMD),
+	PINMUX_IPSR_MSEL(IP0_7_4, SSI_WS1_C, SEL_SSI1_2),
+	PINMUX_IPSR_MSEL(IP0_7_4, TX3_C, SEL_SCIF3_2),
+	PINMUX_IPSR_GPSR(IP0_11_8, SD0_DAT0),
+	PINMUX_IPSR_MSEL(IP0_11_8, SSI_SDATA1_C, SEL_SSI1_2),
+	PINMUX_IPSR_MSEL(IP0_11_8, RX4_E, SEL_SCIF4_4),
+	PINMUX_IPSR_GPSR(IP0_15_12, SD0_DAT1),
+	PINMUX_IPSR_MSEL(IP0_15_12, SSI_SCK0129_B, SEL_SSI0_1),
+	PINMUX_IPSR_MSEL(IP0_15_12, TX4_E, SEL_SCIF4_4),
+	PINMUX_IPSR_GPSR(IP0_19_16, SD0_DAT2),
+	PINMUX_IPSR_MSEL(IP0_19_16, SSI_WS0129_B, SEL_SSI0_1),
+	PINMUX_IPSR_MSEL(IP0_19_16, RX5_E, SEL_SCIF5_4),
+	PINMUX_IPSR_GPSR(IP0_23_20, SD0_DAT3),
+	PINMUX_IPSR_MSEL(IP0_23_20, SSI_SDATA0_B, SEL_SSI0_1),
+	PINMUX_IPSR_MSEL(IP0_23_20, TX5_E, SEL_SCIF5_4),
+	PINMUX_IPSR_GPSR(IP0_27_24, SD0_CD),
+	PINMUX_IPSR_MSEL(IP0_27_24, CAN0_RX_A, SEL_CAN0_0),
+	PINMUX_IPSR_GPSR(IP0_31_28, SD0_WP),
+	PINMUX_IPSR_GPSR(IP0_31_28, IRQ7),
+	PINMUX_IPSR_MSEL(IP0_31_28, CAN0_TX_A, SEL_CAN0_0),
+
+	/* IPSR1 */
+	PINMUX_IPSR_GPSR(IP1_3_0, MMC0_D4),
+	PINMUX_IPSR_GPSR(IP1_3_0, SD1_CD),
+	PINMUX_IPSR_GPSR(IP1_7_4, MMC0_D5),
+	PINMUX_IPSR_GPSR(IP1_7_4, SD1_WP),
+	PINMUX_IPSR_GPSR(IP1_11_8, D0),
+	PINMUX_IPSR_MSEL(IP1_11_8, SCL3_B, SEL_I2C03_1),
+	PINMUX_IPSR_MSEL(IP1_11_8, RX5_B, SEL_SCIF5_1),
+	PINMUX_IPSR_GPSR(IP1_11_8, IRQ4),
+	PINMUX_IPSR_MSEL(IP1_11_8, MSIOF2_RXD_C, SEL_MSIOF2_2),
+	PINMUX_IPSR_MSEL(IP1_11_8, SSI_SDATA5_B, SEL_SSI5_1),
+	PINMUX_IPSR_GPSR(IP1_15_12, D1),
+	PINMUX_IPSR_MSEL(IP1_15_12, SDA3_B, SEL_I2C03_1),
+	PINMUX_IPSR_MSEL(IP1_15_12, TX5_B, SEL_SCIF5_1),
+	PINMUX_IPSR_MSEL(IP1_15_12, MSIOF2_TXD_C, SEL_MSIOF2_2),
+	PINMUX_IPSR_MSEL(IP1_15_12, SSI_WS5_B, SEL_SSI5_1),
+	PINMUX_IPSR_GPSR(IP1_19_16, D2),
+	PINMUX_IPSR_MSEL(IP1_19_16, RX4_B, SEL_SCIF4_1),
+	PINMUX_IPSR_MSEL(IP1_19_16, SCL0_D, SEL_I2C00_3),
+	PINMUX_IPSR_GPSR(IP1_19_16, PWM1_C),
+	PINMUX_IPSR_MSEL(IP1_19_16, MSIOF2_SCK_C, SEL_MSIOF2_2),
+	PINMUX_IPSR_MSEL(IP1_19_16, SSI_SCK5_B, SEL_SSI5_1),
+	PINMUX_IPSR_GPSR(IP1_23_20, D3),
+	PINMUX_IPSR_MSEL(IP1_23_20, TX4_B, SEL_SCIF4_1),
+	PINMUX_IPSR_MSEL(IP1_23_20, SDA0_D, SEL_I2C00_3),
+	PINMUX_IPSR_GPSR(IP1_23_20, PWM0_A),
+	PINMUX_IPSR_MSEL(IP1_23_20, MSIOF2_SYNC_C, SEL_MSIOF2_2),
+	PINMUX_IPSR_GPSR(IP1_27_24, D4),
+	PINMUX_IPSR_GPSR(IP1_27_24, IRQ3),
+	PINMUX_IPSR_MSEL(IP1_27_24, TCLK1_A, SEL_TMU1_0),
+	PINMUX_IPSR_GPSR(IP1_27_24, PWM6_C),
+	PINMUX_IPSR_GPSR(IP1_31_28, D5),
+	PINMUX_IPSR_GPSR(IP1_31_28, HRX2),
+	PINMUX_IPSR_MSEL(IP1_31_28, SCL1_B, SEL_I2C01_1),
+	PINMUX_IPSR_GPSR(IP1_31_28, PWM2_C),
+	PINMUX_IPSR_MSEL(IP1_31_28, TCLK2_B, SEL_TMU2_1),
+
+	/* IPSR2 */
+	PINMUX_IPSR_GPSR(IP2_3_0, D6),
+	PINMUX_IPSR_GPSR(IP2_3_0, HTX2),
+	PINMUX_IPSR_MSEL(IP2_3_0, SDA1_B, SEL_I2C01_1),
+	PINMUX_IPSR_GPSR(IP2_3_0, PWM4_C),
+	PINMUX_IPSR_GPSR(IP2_7_4, D7),
+	PINMUX_IPSR_GPSR(IP2_7_4, HSCK2),
+	PINMUX_IPSR_MSEL(IP2_7_4, SCIF1_SCK_C, SEL_SCIF1_2),
+	PINMUX_IPSR_GPSR(IP2_7_4, IRQ6),
+	PINMUX_IPSR_GPSR(IP2_7_4, PWM5_C),
+	PINMUX_IPSR_GPSR(IP2_11_8, D8),
+	PINMUX_IPSR_GPSR(IP2_11_8, HCTS2_N),
+	PINMUX_IPSR_MSEL(IP2_11_8, RX1_C, SEL_SCIF1_2),
+	PINMUX_IPSR_MSEL(IP2_11_8, SCL1_D, SEL_I2C01_3),
+	PINMUX_IPSR_GPSR(IP2_11_8, PWM3_C),
+	PINMUX_IPSR_GPSR(IP2_15_12, D9),
+	PINMUX_IPSR_GPSR(IP2_15_12, HRTS2_N),
+	PINMUX_IPSR_MSEL(IP2_15_12, TX1_C, SEL_SCIF1_2),
+	PINMUX_IPSR_MSEL(IP2_15_12, SDA1_D, SEL_I2C01_3),
+	PINMUX_IPSR_GPSR(IP2_19_16, D10),
+	PINMUX_IPSR_MSEL(IP2_19_16, MSIOF2_RXD_A, SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP2_19_16, HRX0_B, SEL_HSCIF0_1),
+	PINMUX_IPSR_GPSR(IP2_23_20, D11),
+	PINMUX_IPSR_MSEL(IP2_23_20, MSIOF2_TXD_A, SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP2_23_20, HTX0_B, SEL_HSCIF0_1),
+	PINMUX_IPSR_GPSR(IP2_27_24, D12),
+	PINMUX_IPSR_MSEL(IP2_27_24, MSIOF2_SCK_A, SEL_MSIOF2_0),
+	PINMUX_IPSR_GPSR(IP2_27_24, HSCK0),
+	PINMUX_IPSR_MSEL(IP2_27_24, CAN_CLK_C, SEL_CANCLK_2),
+	PINMUX_IPSR_GPSR(IP2_31_28, D13),
+	PINMUX_IPSR_MSEL(IP2_31_28, MSIOF2_SYNC_A, SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP2_31_28, RX4_C, SEL_SCIF4_2),
+
+	/* IPSR3 */
+	PINMUX_IPSR_GPSR(IP3_3_0, D14),
+	PINMUX_IPSR_GPSR(IP3_3_0, MSIOF2_SS1),
+	PINMUX_IPSR_MSEL(IP3_3_0, TX4_C, SEL_SCIF4_2),
+	PINMUX_IPSR_MSEL(IP3_3_0, CAN1_RX_B, SEL_CAN1_1),
+	PINMUX_IPSR_MSEL(IP3_3_0, AVB_AVTP_CAPTURE_A, SEL_AVB_0),
+	PINMUX_IPSR_GPSR(IP3_7_4, D15),
+	PINMUX_IPSR_GPSR(IP3_7_4, MSIOF2_SS2),
+	PINMUX_IPSR_GPSR(IP3_7_4, PWM4_A),
+	PINMUX_IPSR_MSEL(IP3_7_4, CAN1_TX_B, SEL_CAN1_1),
+	PINMUX_IPSR_GPSR(IP3_7_4, IRQ2),
+	PINMUX_IPSR_MSEL(IP3_7_4, AVB_AVTP_MATCH_A, SEL_AVB_0),
+	PINMUX_IPSR_GPSR(IP3_11_8, QSPI0_SPCLK),
+	PINMUX_IPSR_GPSR(IP3_11_8, WE0_N),
+	PINMUX_IPSR_GPSR(IP3_15_12, QSPI0_MOSI_QSPI0_IO0),
+	PINMUX_IPSR_GPSR(IP3_15_12, BS_N),
+	PINMUX_IPSR_GPSR(IP3_19_16, QSPI0_MISO_QSPI0_IO1),
+	PINMUX_IPSR_GPSR(IP3_19_16, RD_WR_N),
+	PINMUX_IPSR_GPSR(IP3_23_20, QSPI0_IO2),
+	PINMUX_IPSR_GPSR(IP3_23_20, CS0_N),
+	PINMUX_IPSR_GPSR(IP3_27_24, QSPI0_IO3),
+	PINMUX_IPSR_GPSR(IP3_27_24, RD_N),
+	PINMUX_IPSR_GPSR(IP3_31_28, QSPI0_SSL),
+	PINMUX_IPSR_GPSR(IP3_31_28, WE1_N),
+
+	/* IPSR4 */
+	PINMUX_IPSR_GPSR(IP4_3_0, EX_WAIT0),
+	PINMUX_IPSR_MSEL(IP4_3_0, CAN_CLK_B, SEL_CANCLK_1),
+	PINMUX_IPSR_MSEL(IP4_3_0, SCIF_CLK_A, SEL_SCIFCLK_0),
+	PINMUX_IPSR_GPSR(IP4_7_4, DU0_DR0),
+	PINMUX_IPSR_MSEL(IP4_7_4, RX5_C, SEL_SCIF5_2),
+	PINMUX_IPSR_MSEL(IP4_7_4, SCL2_D, SEL_I2C02_3),
+	PINMUX_IPSR_GPSR(IP4_7_4, A0),
+	PINMUX_IPSR_GPSR(IP4_11_8, DU0_DR1),
+	PINMUX_IPSR_MSEL(IP4_11_8, TX5_C, SEL_SCIF5_2),
+	PINMUX_IPSR_MSEL(IP4_11_8, SDA2_D, SEL_I2C02_3),
+	PINMUX_IPSR_GPSR(IP4_11_8, A1),
+	PINMUX_IPSR_GPSR(IP4_15_12, DU0_DR2),
+	PINMUX_IPSR_MSEL(IP4_15_12, RX0_D, SEL_SCIF0_3),
+	PINMUX_IPSR_MSEL(IP4_15_12, SCL0_E, SEL_I2C00_4),
+	PINMUX_IPSR_GPSR(IP4_15_12, A2),
+	PINMUX_IPSR_GPSR(IP4_19_16, DU0_DR3),
+	PINMUX_IPSR_MSEL(IP4_19_16, TX0_D, SEL_SCIF0_3),
+	PINMUX_IPSR_MSEL(IP4_19_16, SDA0_E, SEL_I2C00_4),
+	PINMUX_IPSR_GPSR(IP4_19_16, PWM0_B),
+	PINMUX_IPSR_GPSR(IP4_19_16, A3),
+	PINMUX_IPSR_GPSR(IP4_23_20, DU0_DR4),
+	PINMUX_IPSR_MSEL(IP4_23_20, RX1_D, SEL_SCIF1_3),
+	PINMUX_IPSR_GPSR(IP4_23_20, A4),
+	PINMUX_IPSR_GPSR(IP4_27_24, DU0_DR5),
+	PINMUX_IPSR_MSEL(IP4_27_24, TX1_D, SEL_SCIF1_3),
+	PINMUX_IPSR_GPSR(IP4_27_24, PWM1_B),
+	PINMUX_IPSR_GPSR(IP4_27_24, A5),
+	PINMUX_IPSR_GPSR(IP4_31_28, DU0_DR6),
+	PINMUX_IPSR_MSEL(IP4_31_28, RX2_C, SEL_SCIF2_2),
+	PINMUX_IPSR_GPSR(IP4_31_28, A6),
+
+	/* IPSR5 */
+	PINMUX_IPSR_GPSR(IP5_3_0, DU0_DR7),
+	PINMUX_IPSR_MSEL(IP5_3_0, TX2_C, SEL_SCIF2_2),
+	PINMUX_IPSR_GPSR(IP5_3_0, PWM2_B),
+	PINMUX_IPSR_GPSR(IP5_3_0, A7),
+	PINMUX_IPSR_GPSR(IP5_7_4, DU0_DG0),
+	PINMUX_IPSR_MSEL(IP5_7_4, RX3_B, SEL_SCIF3_1),
+	PINMUX_IPSR_MSEL(IP5_7_4, SCL3_D, SEL_I2C03_3),
+	PINMUX_IPSR_GPSR(IP5_7_4, A8),
+	PINMUX_IPSR_GPSR(IP5_11_8, DU0_DG1),
+	PINMUX_IPSR_MSEL(IP5_11_8, TX3_B, SEL_SCIF3_1),
+	PINMUX_IPSR_MSEL(IP5_11_8, SDA3_D, SEL_I2C03_3),
+	PINMUX_IPSR_GPSR(IP5_11_8, PWM3_B),
+	PINMUX_IPSR_GPSR(IP5_11_8, A9),
+	PINMUX_IPSR_GPSR(IP5_15_12, DU0_DG2),
+	PINMUX_IPSR_MSEL(IP5_15_12, RX4_D, SEL_SCIF4_3),
+	PINMUX_IPSR_GPSR(IP5_15_12, A10),
+	PINMUX_IPSR_GPSR(IP5_19_16, DU0_DG3),
+	PINMUX_IPSR_MSEL(IP5_19_16, TX4_D, SEL_SCIF4_3),
+	PINMUX_IPSR_GPSR(IP5_19_16, PWM4_B),
+	PINMUX_IPSR_GPSR(IP5_19_16, A11),
+	PINMUX_IPSR_GPSR(IP5_23_20, DU0_DG4),
+	PINMUX_IPSR_MSEL(IP5_23_20, HRX0_A, SEL_HSCIF0_0),
+	PINMUX_IPSR_GPSR(IP5_23_20, A12),
+	PINMUX_IPSR_GPSR(IP5_27_24, DU0_DG5),
+	PINMUX_IPSR_MSEL(IP5_27_24, HTX0_A, SEL_HSCIF0_0),
+	PINMUX_IPSR_GPSR(IP5_27_24, PWM5_B),
+	PINMUX_IPSR_GPSR(IP5_27_24, A13),
+	PINMUX_IPSR_GPSR(IP5_31_28, DU0_DG6),
+	PINMUX_IPSR_MSEL(IP5_31_28, HRX1_C, SEL_HSCIF1_2),
+	PINMUX_IPSR_GPSR(IP5_31_28, A14),
+
+	/* IPSR6 */
+	PINMUX_IPSR_GPSR(IP6_3_0, DU0_DG7),
+	PINMUX_IPSR_MSEL(IP6_3_0, HTX1_C, SEL_HSCIF1_2),
+	PINMUX_IPSR_GPSR(IP6_3_0, PWM6_B),
+	PINMUX_IPSR_GPSR(IP6_3_0, A15),
+	PINMUX_IPSR_GPSR(IP6_7_4, DU0_DB0),
+	PINMUX_IPSR_MSEL(IP6_7_4, SCL4_D, SEL_I2C04_3),
+	PINMUX_IPSR_MSEL(IP6_7_4, CAN0_RX_C, SEL_CAN0_2),
+	PINMUX_IPSR_GPSR(IP6_7_4, A16),
+	PINMUX_IPSR_GPSR(IP6_11_8, DU0_DB1),
+	PINMUX_IPSR_MSEL(IP6_11_8, SDA4_D, SEL_I2C04_3),
+	PINMUX_IPSR_MSEL(IP6_11_8, CAN0_TX_C, SEL_CAN0_2),
+	PINMUX_IPSR_GPSR(IP6_11_8, A17),
+	PINMUX_IPSR_GPSR(IP6_15_12, DU0_DB2),
+	PINMUX_IPSR_GPSR(IP6_15_12, HCTS0_N),
+	PINMUX_IPSR_GPSR(IP6_15_12, A18),
+	PINMUX_IPSR_GPSR(IP6_19_16, DU0_DB3),
+	PINMUX_IPSR_GPSR(IP6_19_16, HRTS0_N),
+	PINMUX_IPSR_GPSR(IP6_19_16, A19),
+	PINMUX_IPSR_GPSR(IP6_23_20, DU0_DB4),
+	PINMUX_IPSR_MSEL(IP6_23_20, HCTS1_N_C, SEL_HSCIF1_2),
+	PINMUX_IPSR_GPSR(IP6_23_20, A20),
+	PINMUX_IPSR_GPSR(IP6_27_24, DU0_DB5),
+	PINMUX_IPSR_MSEL(IP6_27_24, HRTS1_N_C, SEL_HSCIF1_2),
+	PINMUX_IPSR_GPSR(IP6_27_24, A21),
+	PINMUX_IPSR_GPSR(IP6_31_28, DU0_DB6),
+	PINMUX_IPSR_GPSR(IP6_31_28, A22),
+
+	/* IPSR7 */
+	PINMUX_IPSR_GPSR(IP7_3_0, DU0_DB7),
+	PINMUX_IPSR_GPSR(IP7_3_0, A23),
+	PINMUX_IPSR_GPSR(IP7_7_4, DU0_DOTCLKIN),
+	PINMUX_IPSR_GPSR(IP7_7_4, A24),
+	PINMUX_IPSR_GPSR(IP7_11_8, DU0_DOTCLKOUT0),
+	PINMUX_IPSR_GPSR(IP7_11_8, A25),
+	PINMUX_IPSR_GPSR(IP7_15_12, DU0_DOTCLKOUT1),
+	PINMUX_IPSR_MSEL(IP7_15_12, MSIOF2_RXD_B, SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP7_15_12, CS1_N_A26),
+	PINMUX_IPSR_GPSR(IP7_19_16, DU0_EXHSYNC_DU0_HSYNC),
+	PINMUX_IPSR_MSEL(IP7_19_16, MSIOF2_TXD_B, SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP7_19_16, DREQ0_N),
+	PINMUX_IPSR_GPSR(IP7_23_20, DU0_EXVSYNC_DU0_VSYNC),
+	PINMUX_IPSR_MSEL(IP7_23_20, MSIOF2_SYNC_B, SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP7_23_20, DACK0),
+	PINMUX_IPSR_GPSR(IP7_27_24, DU0_EXODDF_DU0_ODDF_DISP_CDE),
+	PINMUX_IPSR_MSEL(IP7_27_24, MSIOF2_SCK_B, SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP7_27_24, DRACK0),
+	PINMUX_IPSR_GPSR(IP7_31_28, DU0_DISP),
+	PINMUX_IPSR_MSEL(IP7_31_28, CAN1_RX_C, SEL_CAN1_2),
+
+	/* IPSR8 */
+	PINMUX_IPSR_GPSR(IP8_3_0, DU0_CDE),
+	PINMUX_IPSR_MSEL(IP8_3_0, CAN1_TX_C, SEL_CAN1_2),
+	PINMUX_IPSR_GPSR(IP8_7_4, VI1_CLK),
+	PINMUX_IPSR_GPSR(IP8_7_4, AVB_RX_CLK),
+	PINMUX_IPSR_GPSR(IP8_7_4, ETH_REF_CLK),
+	PINMUX_IPSR_GPSR(IP8_11_8, VI1_DATA0),
+	PINMUX_IPSR_GPSR(IP8_11_8, AVB_RX_DV),
+	PINMUX_IPSR_GPSR(IP8_11_8, ETH_CRS_DV),
+	PINMUX_IPSR_GPSR(IP8_15_12, VI1_DATA1),
+	PINMUX_IPSR_GPSR(IP8_15_12, AVB_RXD0),
+	PINMUX_IPSR_GPSR(IP8_15_12, ETH_RXD0),
+	PINMUX_IPSR_GPSR(IP8_19_16, VI1_DATA2),
+	PINMUX_IPSR_GPSR(IP8_19_16, AVB_RXD1),
+	PINMUX_IPSR_GPSR(IP8_19_16, ETH_RXD1),
+	PINMUX_IPSR_GPSR(IP8_23_20, VI1_DATA3),
+	PINMUX_IPSR_GPSR(IP8_23_20, AVB_RXD2),
+	PINMUX_IPSR_GPSR(IP8_23_20, ETH_MDIO),
+	PINMUX_IPSR_GPSR(IP8_27_24, VI1_DATA4),
+	PINMUX_IPSR_GPSR(IP8_27_24, AVB_RXD3),
+	PINMUX_IPSR_GPSR(IP8_27_24, ETH_RX_ER),
+	PINMUX_IPSR_GPSR(IP8_31_28, VI1_DATA5),
+	PINMUX_IPSR_GPSR(IP8_31_28, AVB_RXD4),
+	PINMUX_IPSR_GPSR(IP8_31_28, ETH_LINK),
+
+	/* IPSR9 */
+	PINMUX_IPSR_GPSR(IP9_3_0, VI1_DATA6),
+	PINMUX_IPSR_GPSR(IP9_3_0, AVB_RXD5),
+	PINMUX_IPSR_GPSR(IP9_3_0, ETH_TXD1),
+	PINMUX_IPSR_GPSR(IP9_7_4, VI1_DATA7),
+	PINMUX_IPSR_GPSR(IP9_7_4, AVB_RXD6),
+	PINMUX_IPSR_GPSR(IP9_7_4, ETH_TX_EN),
+	PINMUX_IPSR_GPSR(IP9_11_8, VI1_CLKENB),
+	PINMUX_IPSR_MSEL(IP9_11_8, SCL3_A, SEL_I2C03_0),
+	PINMUX_IPSR_GPSR(IP9_11_8, AVB_RXD7),
+	PINMUX_IPSR_GPSR(IP9_11_8, ETH_MAGIC),
+	PINMUX_IPSR_GPSR(IP9_15_12, VI1_FIELD),
+	PINMUX_IPSR_MSEL(IP9_15_12, SDA3_A, SEL_I2C03_0),
+	PINMUX_IPSR_GPSR(IP9_15_12, AVB_RX_ER),
+	PINMUX_IPSR_GPSR(IP9_15_12, ETH_TXD0),
+	PINMUX_IPSR_GPSR(IP9_19_16, VI1_HSYNC_N),
+	PINMUX_IPSR_MSEL(IP9_19_16, RX0_B, SEL_SCIF0_1),
+	PINMUX_IPSR_MSEL(IP9_19_16, SCL0_C, SEL_I2C00_2),
+	PINMUX_IPSR_GPSR(IP9_19_16, AVB_GTXREFCLK),
+	PINMUX_IPSR_GPSR(IP9_19_16, ETH_MDC),
+	PINMUX_IPSR_GPSR(IP9_23_20, VI1_VSYNC_N),
+	PINMUX_IPSR_MSEL(IP9_23_20, TX0_B, SEL_SCIF0_1),
+	PINMUX_IPSR_MSEL(IP9_23_20, SDA0_C, SEL_I2C00_2),
+	PINMUX_IPSR_GPSR(IP9_23_20, AUDIO_CLKOUT_B),
+	PINMUX_IPSR_GPSR(IP9_23_20, AVB_TX_CLK),
+	PINMUX_IPSR_GPSR(IP9_27_24, VI1_DATA8),
+	PINMUX_IPSR_MSEL(IP9_27_24, SCL2_B, SEL_I2C02_1),
+	PINMUX_IPSR_GPSR(IP9_27_24, AVB_TX_EN),
+	PINMUX_IPSR_GPSR(IP9_31_28, VI1_DATA9),
+	PINMUX_IPSR_MSEL(IP9_31_28, SDA2_B, SEL_I2C02_1),
+	PINMUX_IPSR_GPSR(IP9_31_28, AVB_TXD0),
+
+	/* IPSR10 */
+	PINMUX_IPSR_GPSR(IP10_3_0, VI1_DATA10),
+	PINMUX_IPSR_MSEL(IP10_3_0, CAN0_RX_B, SEL_CAN0_1),
+	PINMUX_IPSR_GPSR(IP10_3_0, AVB_TXD1),
+	PINMUX_IPSR_GPSR(IP10_7_4, VI1_DATA11),
+	PINMUX_IPSR_MSEL(IP10_7_4, CAN0_TX_B, SEL_CAN0_1),
+	PINMUX_IPSR_GPSR(IP10_7_4, AVB_TXD2),
+	PINMUX_IPSR_GPSR(IP10_11_8, AVB_TXD3),
+	PINMUX_IPSR_MSEL(IP10_11_8, AUDIO_CLKA_B, SEL_ADGA_1),
+	PINMUX_IPSR_MSEL(IP10_11_8, SSI_SCK1_D, SEL_SSI1_3),
+	PINMUX_IPSR_MSEL(IP10_11_8, RX5_F, SEL_SCIF5_5),
+	PINMUX_IPSR_MSEL(IP10_11_8, MSIOF0_RXD_B, SEL_MSIOF0_1),
+	PINMUX_IPSR_GPSR(IP10_15_12, AVB_TXD4),
+	PINMUX_IPSR_MSEL(IP10_15_12, AUDIO_CLKB_B, SEL_ADGB_1),
+	PINMUX_IPSR_MSEL(IP10_15_12, SSI_WS1_D, SEL_SSI1_3),
+	PINMUX_IPSR_MSEL(IP10_15_12, TX5_F, SEL_SCIF5_5),
+	PINMUX_IPSR_MSEL(IP10_15_12, MSIOF0_TXD_B, SEL_MSIOF0_1),
+	PINMUX_IPSR_GPSR(IP10_19_16, AVB_TXD5),
+	PINMUX_IPSR_MSEL(IP10_19_16, SCIF_CLK_B, SEL_SCIFCLK_1),
+	PINMUX_IPSR_MSEL(IP10_19_16, AUDIO_CLKC_B, SEL_ADGC_1),
+	PINMUX_IPSR_MSEL(IP10_19_16, SSI_SDATA1_D, SEL_SSI1_3),
+	PINMUX_IPSR_MSEL(IP10_19_16, MSIOF0_SCK_B, SEL_MSIOF0_1),
+	PINMUX_IPSR_MSEL(IP10_23_20, SCL0_A, SEL_I2C00_0),
+	PINMUX_IPSR_MSEL(IP10_23_20, RX0_C, SEL_SCIF0_2),
+	PINMUX_IPSR_GPSR(IP10_23_20, PWM5_A),
+	PINMUX_IPSR_MSEL(IP10_23_20, TCLK1_B, SEL_TMU1_1),
+	PINMUX_IPSR_GPSR(IP10_23_20, AVB_TXD6),
+	PINMUX_IPSR_MSEL(IP10_23_20, CAN1_RX_D, SEL_CAN1_3),
+	PINMUX_IPSR_MSEL(IP10_23_20, MSIOF0_SYNC_B, SEL_MSIOF0_1),
+	PINMUX_IPSR_MSEL(IP10_27_24, SDA0_A, SEL_I2C00_0),
+	PINMUX_IPSR_MSEL(IP10_27_24, TX0_C, SEL_SCIF0_2),
+	PINMUX_IPSR_GPSR(IP10_27_24, IRQ5),
+	PINMUX_IPSR_MSEL(IP10_27_24, CAN_CLK_A, SEL_CANCLK_0),
+	PINMUX_IPSR_GPSR(IP10_27_24, AVB_GTX_CLK),
+	PINMUX_IPSR_MSEL(IP10_27_24, CAN1_TX_D, SEL_CAN1_3),
+	PINMUX_IPSR_GPSR(IP10_27_24, DVC_MUTE),
+	PINMUX_IPSR_MSEL(IP10_31_28, SCL1_A, SEL_I2C01_0),
+	PINMUX_IPSR_MSEL(IP10_31_28, RX4_A, SEL_SCIF4_0),
+	PINMUX_IPSR_GPSR(IP10_31_28, PWM5_D),
+	PINMUX_IPSR_GPSR(IP10_31_28, DU1_DR0),
+	PINMUX_IPSR_MSEL(IP10_31_28, SSI_SCK6_B, SEL_SSI6_1),
+	PINMUX_IPSR_GPSR(IP10_31_28, VI0_G0),
+
+	/* IPSR11 */
+	PINMUX_IPSR_MSEL(IP11_3_0, SDA1_A, SEL_I2C01_0),
+	PINMUX_IPSR_MSEL(IP11_3_0, TX4_A, SEL_SCIF4_0),
+	PINMUX_IPSR_GPSR(IP11_3_0, DU1_DR1),
+	PINMUX_IPSR_MSEL(IP11_3_0, SSI_WS6_B, SEL_SSI6_1),
+	PINMUX_IPSR_GPSR(IP11_3_0, VI0_G1),
+	PINMUX_IPSR_MSEL(IP11_7_4, MSIOF0_RXD_A, SEL_MSIOF0_0),
+	PINMUX_IPSR_MSEL(IP11_7_4, RX5_A, SEL_SCIF5_0),
+	PINMUX_IPSR_MSEL(IP11_7_4, SCL2_C, SEL_I2C02_2),
+	PINMUX_IPSR_GPSR(IP11_7_4, DU1_DR2),
+	PINMUX_IPSR_GPSR(IP11_7_4, QSPI1_MOSI_QSPI1_IO0),
+	PINMUX_IPSR_MSEL(IP11_7_4, SSI_SDATA6_B, SEL_SSI6_1),
+	PINMUX_IPSR_GPSR(IP11_7_4, VI0_G2),
+	PINMUX_IPSR_MSEL(IP11_11_8, MSIOF0_TXD_A, SEL_MSIOF0_0),
+	PINMUX_IPSR_MSEL(IP11_11_8, TX5_A, SEL_SCIF5_0),
+	PINMUX_IPSR_MSEL(IP11_11_8, SDA2_C, SEL_I2C02_2),
+	PINMUX_IPSR_GPSR(IP11_11_8, DU1_DR3),
+	PINMUX_IPSR_GPSR(IP11_11_8, QSPI1_MISO_QSPI1_IO1),
+	PINMUX_IPSR_MSEL(IP11_11_8, SSI_WS78_B, SEL_SSI7_1),
+	PINMUX_IPSR_GPSR(IP11_11_8, VI0_G3),
+	PINMUX_IPSR_MSEL(IP11_15_12, MSIOF0_SCK_A, SEL_MSIOF0_0),
+	PINMUX_IPSR_GPSR(IP11_15_12, IRQ0),
+	PINMUX_IPSR_GPSR(IP11_15_12, DU1_DR4),
+	PINMUX_IPSR_GPSR(IP11_15_12, QSPI1_SPCLK),
+	PINMUX_IPSR_MSEL(IP11_15_12, SSI_SCK78_B, SEL_SSI7_1),
+	PINMUX_IPSR_GPSR(IP11_15_12, VI0_G4),
+	PINMUX_IPSR_MSEL(IP11_19_16, MSIOF0_SYNC_A, SEL_MSIOF0_0),
+	PINMUX_IPSR_GPSR(IP11_19_16, PWM1_A),
+	PINMUX_IPSR_GPSR(IP11_19_16, DU1_DR5),
+	PINMUX_IPSR_GPSR(IP11_19_16, QSPI1_IO2),
+	PINMUX_IPSR_MSEL(IP11_19_16, SSI_SDATA7_B, SEL_SSI7_1),
+	PINMUX_IPSR_MSEL(IP11_23_20, MSIOF0_SS1_A, SEL_MSIOF0_0),
+	PINMUX_IPSR_GPSR(IP11_23_20, DU1_DR6),
+	PINMUX_IPSR_GPSR(IP11_23_20, QSPI1_IO3),
+	PINMUX_IPSR_MSEL(IP11_23_20, SSI_SDATA8_B, SEL_SSI8_1),
+	PINMUX_IPSR_MSEL(IP11_27_24, MSIOF0_SS2_A, SEL_MSIOF0_0),
+	PINMUX_IPSR_GPSR(IP11_27_24, DU1_DR7),
+	PINMUX_IPSR_GPSR(IP11_27_24, QSPI1_SSL),
+	PINMUX_IPSR_MSEL(IP11_31_28, HRX1_A, SEL_HSCIF1_0),
+	PINMUX_IPSR_MSEL(IP11_31_28, SCL4_A, SEL_I2C04_0),
+	PINMUX_IPSR_GPSR(IP11_31_28, PWM6_A),
+	PINMUX_IPSR_GPSR(IP11_31_28, DU1_DG0),
+	PINMUX_IPSR_MSEL(IP11_31_28, RX0_A, SEL_SCIF0_0),
+
+	/* IPSR12 */
+	PINMUX_IPSR_MSEL(IP12_3_0, HTX1_A, SEL_HSCIF1_0),
+	PINMUX_IPSR_MSEL(IP12_3_0, SDA4_A, SEL_I2C04_0),
+	PINMUX_IPSR_GPSR(IP12_3_0, DU1_DG1),
+	PINMUX_IPSR_MSEL(IP12_3_0, TX0_A, SEL_SCIF0_0),
+	PINMUX_IPSR_MSEL(IP12_7_4, HCTS1_N_A, SEL_HSCIF1_0),
+	PINMUX_IPSR_GPSR(IP12_7_4, PWM2_A),
+	PINMUX_IPSR_GPSR(IP12_7_4, DU1_DG2),
+	PINMUX_IPSR_MSEL(IP12_7_4, REMOCON_B, SEL_RCN_1),
+	PINMUX_IPSR_MSEL(IP12_11_8, HRTS1_N_A, SEL_HSCIF1_0),
+	PINMUX_IPSR_GPSR(IP12_11_8, DU1_DG3),
+	PINMUX_IPSR_MSEL(IP12_11_8, SSI_WS1_B, SEL_SSI1_1),
+	PINMUX_IPSR_GPSR(IP12_11_8, IRQ1),
+	PINMUX_IPSR_GPSR(IP12_15_12, SD2_CLK),
+	PINMUX_IPSR_GPSR(IP12_15_12, HSCK1),
+	PINMUX_IPSR_GPSR(IP12_15_12, DU1_DG4),
+	PINMUX_IPSR_MSEL(IP12_15_12, SSI_SCK1_B, SEL_SSI1_1),
+	PINMUX_IPSR_GPSR(IP12_19_16, SD2_CMD),
+	PINMUX_IPSR_MSEL(IP12_19_16, SCIF1_SCK_A, SEL_SCIF1_0),
+	PINMUX_IPSR_MSEL(IP12_19_16, TCLK2_A, SEL_TMU2_0),
+	PINMUX_IPSR_GPSR(IP12_19_16, DU1_DG5),
+	PINMUX_IPSR_MSEL(IP12_19_16, SSI_SCK2_B, SEL_SSI2_1),
+	PINMUX_IPSR_GPSR(IP12_19_16, PWM3_A),
+	PINMUX_IPSR_GPSR(IP12_23_20, SD2_DAT0),
+	PINMUX_IPSR_MSEL(IP12_23_20, RX1_A, SEL_SCIF1_0),
+	PINMUX_IPSR_MSEL(IP12_23_20, SCL1_E, SEL_I2C01_4),
+	PINMUX_IPSR_GPSR(IP12_23_20, DU1_DG6),
+	PINMUX_IPSR_MSEL(IP12_23_20, SSI_SDATA1_B, SEL_SSI1_1),
+	PINMUX_IPSR_GPSR(IP12_27_24, SD2_DAT1),
+	PINMUX_IPSR_MSEL(IP12_27_24, TX1_A, SEL_SCIF1_0),
+	PINMUX_IPSR_MSEL(IP12_27_24, SDA1_E, SEL_I2C01_4),
+	PINMUX_IPSR_GPSR(IP12_27_24, DU1_DG7),
+	PINMUX_IPSR_MSEL(IP12_27_24, SSI_WS2_B, SEL_SSI2_1),
+	PINMUX_IPSR_GPSR(IP12_31_28, SD2_DAT2),
+	PINMUX_IPSR_MSEL(IP12_31_28, RX2_A, SEL_SCIF2_0),
+	PINMUX_IPSR_GPSR(IP12_31_28, DU1_DB0),
+	PINMUX_IPSR_MSEL(IP12_31_28, SSI_SDATA2_B, SEL_SSI2_1),
+
+	/* IPSR13 */
+	PINMUX_IPSR_GPSR(IP13_3_0, SD2_DAT3),
+	PINMUX_IPSR_MSEL(IP13_3_0, TX2_A, SEL_SCIF2_0),
+	PINMUX_IPSR_GPSR(IP13_3_0, DU1_DB1),
+	PINMUX_IPSR_MSEL(IP13_3_0, SSI_WS9_B, SEL_SSI9_1),
+	PINMUX_IPSR_GPSR(IP13_7_4, SD2_CD),
+	PINMUX_IPSR_MSEL(IP13_7_4, SCIF2_SCK_A, SEL_SCIF2_CLK_0),
+	PINMUX_IPSR_GPSR(IP13_7_4, DU1_DB2),
+	PINMUX_IPSR_MSEL(IP13_7_4, SSI_SCK9_B, SEL_SSI9_1),
+	PINMUX_IPSR_GPSR(IP13_11_8, SD2_WP),
+	PINMUX_IPSR_GPSR(IP13_11_8, SCIF3_SCK),
+	PINMUX_IPSR_GPSR(IP13_11_8, DU1_DB3),
+	PINMUX_IPSR_MSEL(IP13_11_8, SSI_SDATA9_B, SEL_SSI9_1),
+	PINMUX_IPSR_MSEL(IP13_15_12, RX3_A, SEL_SCIF3_0),
+	PINMUX_IPSR_MSEL(IP13_15_12, SCL1_C, SEL_I2C01_2),
+	PINMUX_IPSR_MSEL(IP13_15_12, MSIOF1_RXD_B, SEL_MSIOF1_1),
+	PINMUX_IPSR_GPSR(IP13_15_12, DU1_DB4),
+	PINMUX_IPSR_MSEL(IP13_15_12, AUDIO_CLKA_C, SEL_ADGA_2),
+	PINMUX_IPSR_MSEL(IP13_15_12, SSI_SDATA4_B, SEL_SSI4_1),
+	PINMUX_IPSR_MSEL(IP13_19_16, TX3_A, SEL_SCIF3_0),
+	PINMUX_IPSR_MSEL(IP13_19_16, SDA1_C, SEL_I2C01_2),
+	PINMUX_IPSR_MSEL(IP13_19_16, MSIOF1_TXD_B, SEL_MSIOF1_1),
+	PINMUX_IPSR_GPSR(IP13_19_16, DU1_DB5),
+	PINMUX_IPSR_MSEL(IP13_19_16, AUDIO_CLKB_C, SEL_ADGB_2),
+	PINMUX_IPSR_MSEL(IP13_19_16, SSI_WS4_B, SEL_SSI4_1),
+	PINMUX_IPSR_MSEL(IP13_23_20, SCL2_A, SEL_I2C02_0),
+	PINMUX_IPSR_MSEL(IP13_23_20, MSIOF1_SCK_B, SEL_MSIOF1_1),
+	PINMUX_IPSR_GPSR(IP13_23_20, DU1_DB6),
+	PINMUX_IPSR_MSEL(IP13_23_20, AUDIO_CLKC_C, SEL_ADGC_2),
+	PINMUX_IPSR_MSEL(IP13_23_20, SSI_SCK4_B, SEL_SSI4_1),
+	PINMUX_IPSR_MSEL(IP13_27_24, SDA2_A, SEL_I2C02_0),
+	PINMUX_IPSR_MSEL(IP13_27_24, MSIOF1_SYNC_B, SEL_MSIOF1_1),
+	PINMUX_IPSR_GPSR(IP13_27_24, DU1_DB7),
+	PINMUX_IPSR_GPSR(IP13_27_24, AUDIO_CLKOUT_C),
+	PINMUX_IPSR_MSEL(IP13_31_28, SSI_SCK5_A, SEL_SSI5_0),
+	PINMUX_IPSR_GPSR(IP13_31_28, DU1_DOTCLKOUT1),
+
+	/* IPSR14 */
+	PINMUX_IPSR_MSEL(IP14_3_0, SSI_WS5_A, SEL_SSI5_0),
+	PINMUX_IPSR_MSEL(IP14_3_0, SCL3_C, SEL_I2C03_2),
+	PINMUX_IPSR_GPSR(IP14_3_0, DU1_DOTCLKIN),
+	PINMUX_IPSR_MSEL(IP14_7_4, SSI_SDATA5_A, SEL_SSI5_0),
+	PINMUX_IPSR_MSEL(IP14_7_4, SDA3_C, SEL_I2C03_2),
+	PINMUX_IPSR_GPSR(IP14_7_4, DU1_DOTCLKOUT0),
+	PINMUX_IPSR_MSEL(IP14_11_8, SSI_SCK6_A, SEL_SSI6_0),
+	PINMUX_IPSR_GPSR(IP14_11_8, DU1_EXODDF_DU1_ODDF_DISP_CDE),
+	PINMUX_IPSR_MSEL(IP14_15_12, SSI_WS6_A, SEL_SSI6_0),
+	PINMUX_IPSR_MSEL(IP14_15_12, SCL4_C, SEL_I2C04_2),
+	PINMUX_IPSR_GPSR(IP14_15_12, DU1_EXHSYNC_DU1_HSYNC),
+	PINMUX_IPSR_MSEL(IP14_19_16, SSI_SDATA6_A, SEL_SSI6_0),
+	PINMUX_IPSR_MSEL(IP14_19_16, SDA4_C, SEL_I2C04_2),
+	PINMUX_IPSR_GPSR(IP14_19_16, DU1_EXVSYNC_DU1_VSYNC),
+	PINMUX_IPSR_MSEL(IP14_23_20, SSI_SCK78_A, SEL_SSI7_0),
+	PINMUX_IPSR_MSEL(IP14_23_20, SDA4_E, SEL_I2C04_4),
+	PINMUX_IPSR_GPSR(IP14_23_20, DU1_DISP),
+	PINMUX_IPSR_MSEL(IP14_27_24, SSI_WS78_A, SEL_SSI7_0),
+	PINMUX_IPSR_MSEL(IP14_27_24, SCL4_E, SEL_I2C04_4),
+	PINMUX_IPSR_GPSR(IP14_27_24, DU1_CDE),
+	PINMUX_IPSR_MSEL(IP14_31_28, SSI_SDATA7_A, SEL_SSI7_0),
+	PINMUX_IPSR_GPSR(IP14_31_28, IRQ8),
+	PINMUX_IPSR_MSEL(IP14_31_28, AUDIO_CLKA_D, SEL_ADGA_3),
+	PINMUX_IPSR_MSEL(IP14_31_28, CAN_CLK_D, SEL_CANCLK_3),
+	PINMUX_IPSR_GPSR(IP14_31_28, VI0_G5),
+
+	/* IPSR15 */
+	PINMUX_IPSR_MSEL(IP15_3_0, SSI_SCK0129_A, SEL_SSI0_0),
+	PINMUX_IPSR_MSEL(IP15_3_0, MSIOF1_RXD_A, SEL_MSIOF1_0),
+	PINMUX_IPSR_MSEL(IP15_3_0, RX5_D, SEL_SCIF5_3),
+	PINMUX_IPSR_GPSR(IP15_3_0, VI0_G6),
+	PINMUX_IPSR_MSEL(IP15_7_4, SSI_WS0129_A, SEL_SSI0_0),
+	PINMUX_IPSR_MSEL(IP15_7_4, MSIOF1_TXD_A, SEL_MSIOF1_0),
+	PINMUX_IPSR_MSEL(IP15_7_4, TX5_D, SEL_SCIF5_3),
+	PINMUX_IPSR_GPSR(IP15_7_4, VI0_G7),
+	PINMUX_IPSR_MSEL(IP15_11_8, SSI_SDATA0_A, SEL_SSI0_0),
+	PINMUX_IPSR_MSEL(IP15_11_8, MSIOF1_SYNC_A, SEL_MSIOF1_0),
+	PINMUX_IPSR_GPSR(IP15_11_8, PWM0_C),
+	PINMUX_IPSR_GPSR(IP15_11_8, VI0_R0),
+	PINMUX_IPSR_GPSR(IP15_15_12, SSI_SCK34),
+	PINMUX_IPSR_MSEL(IP15_15_12, MSIOF1_SCK_A, SEL_MSIOF1_0),
+	PINMUX_IPSR_GPSR(IP15_15_12, AVB_MDC),
+	PINMUX_IPSR_GPSR(IP15_15_12, DACK1),
+	PINMUX_IPSR_GPSR(IP15_15_12, VI0_R1),
+	PINMUX_IPSR_GPSR(IP15_19_16, SSI_WS34),
+	PINMUX_IPSR_MSEL(IP15_19_16, MSIOF1_SS1_A, SEL_MSIOF1_0),
+	PINMUX_IPSR_GPSR(IP15_19_16, AVB_MDIO),
+	PINMUX_IPSR_MSEL(IP15_19_16, CAN1_RX_A, SEL_CAN1_0),
+	PINMUX_IPSR_GPSR(IP15_19_16, DREQ1_N),
+	PINMUX_IPSR_GPSR(IP15_19_16, VI0_R2),
+	PINMUX_IPSR_GPSR(IP15_23_20, SSI_SDATA3),
+	PINMUX_IPSR_MSEL(IP15_23_20, MSIOF1_SS2_A, SEL_MSIOF1_0),
+	PINMUX_IPSR_GPSR(IP15_23_20, AVB_LINK),
+	PINMUX_IPSR_MSEL(IP15_23_20, CAN1_TX_A, SEL_CAN1_0),
+	PINMUX_IPSR_GPSR(IP15_23_20, DREQ2_N),
+	PINMUX_IPSR_GPSR(IP15_23_20, VI0_R3),
+	PINMUX_IPSR_MSEL(IP15_27_24, SSI_SCK4_A, SEL_SSI4_0),
+	PINMUX_IPSR_GPSR(IP15_27_24, AVB_MAGIC),
+	PINMUX_IPSR_GPSR(IP15_27_24, VI0_R4),
+	PINMUX_IPSR_MSEL(IP15_31_28, SSI_WS4_A, SEL_SSI4_0),
+	PINMUX_IPSR_GPSR(IP15_31_28, AVB_PHY_INT),
+	PINMUX_IPSR_GPSR(IP15_31_28, VI0_R5),
+
+	/* IPSR16 */
+	PINMUX_IPSR_MSEL(IP16_3_0, SSI_SDATA4_A, SEL_SSI4_0),
+	PINMUX_IPSR_GPSR(IP16_3_0, AVB_CRS),
+	PINMUX_IPSR_GPSR(IP16_3_0, VI0_R6),
+	PINMUX_IPSR_MSEL(IP16_7_4, SSI_SCK1_A, SEL_SSI1_0),
+	PINMUX_IPSR_MSEL(IP16_7_4, SCIF1_SCK_B, SEL_SCIF1_1),
+	PINMUX_IPSR_GPSR(IP16_7_4, PWM1_D),
+	PINMUX_IPSR_GPSR(IP16_7_4, IRQ9),
+	PINMUX_IPSR_MSEL(IP16_7_4, REMOCON_A, SEL_RCN_0),
+	PINMUX_IPSR_GPSR(IP16_7_4, DACK2),
+	PINMUX_IPSR_GPSR(IP16_7_4, VI0_CLK),
+	PINMUX_IPSR_GPSR(IP16_7_4, AVB_COL),
+	PINMUX_IPSR_MSEL(IP16_11_8, SSI_SDATA8_A, SEL_SSI8_0),
+	PINMUX_IPSR_MSEL(IP16_11_8, RX1_B, SEL_SCIF1_1),
+	PINMUX_IPSR_MSEL(IP16_11_8, CAN0_RX_D, SEL_CAN0_3),
+	PINMUX_IPSR_MSEL(IP16_11_8, AVB_AVTP_CAPTURE_B, SEL_AVB_1),
+	PINMUX_IPSR_GPSR(IP16_11_8, VI0_R7),
+	PINMUX_IPSR_MSEL(IP16_15_12, SSI_WS1_A, SEL_SSI1_0),
+	PINMUX_IPSR_MSEL(IP16_15_12, TX1_B, SEL_SCIF1_1),
+	PINMUX_IPSR_MSEL(IP16_15_12, CAN0_TX_D, SEL_CAN0_3),
+	PINMUX_IPSR_MSEL(IP16_15_12, AVB_AVTP_MATCH_B, SEL_AVB_1),
+	PINMUX_IPSR_GPSR(IP16_15_12, VI0_DATA0_VI0_B0),
+	PINMUX_IPSR_MSEL(IP16_19_16, SSI_SDATA1_A, SEL_SSI1_0),
+	PINMUX_IPSR_MSEL(IP16_19_16, HRX1_B, SEL_HSCIF1_1),
+	PINMUX_IPSR_GPSR(IP16_19_16, VI0_DATA1_VI0_B1),
+	PINMUX_IPSR_MSEL(IP16_23_20, SSI_SCK2_A, SEL_SSI2_0),
+	PINMUX_IPSR_MSEL(IP16_23_20, HTX1_B, SEL_HSCIF1_1),
+	PINMUX_IPSR_GPSR(IP16_23_20, AVB_TXD7),
+	PINMUX_IPSR_GPSR(IP16_23_20, VI0_DATA2_VI0_B2),
+	PINMUX_IPSR_MSEL(IP16_27_24, SSI_WS2_A, SEL_SSI2_0),
+	PINMUX_IPSR_MSEL(IP16_27_24, HCTS1_N_B, SEL_HSCIF1_1),
+	PINMUX_IPSR_GPSR(IP16_27_24, AVB_TX_ER),
+	PINMUX_IPSR_GPSR(IP16_27_24, VI0_DATA3_VI0_B3),
+	PINMUX_IPSR_MSEL(IP16_31_28, SSI_SDATA2_A, SEL_SSI2_0),
+	PINMUX_IPSR_MSEL(IP16_31_28, HRTS1_N_B, SEL_HSCIF1_1),
+	PINMUX_IPSR_GPSR(IP16_31_28, VI0_DATA4_VI0_B4),
+
+	/* IPSR17 */
+	PINMUX_IPSR_MSEL(IP17_3_0, SSI_SCK9_A, SEL_SSI9_0),
+	PINMUX_IPSR_MSEL(IP17_3_0, RX2_B, SEL_SCIF2_1),
+	PINMUX_IPSR_MSEL(IP17_3_0, SCL3_E, SEL_I2C03_4),
+	PINMUX_IPSR_GPSR(IP17_3_0, EX_WAIT1),
+	PINMUX_IPSR_GPSR(IP17_3_0, VI0_DATA5_VI0_B5),
+	PINMUX_IPSR_MSEL(IP17_7_4, SSI_WS9_A, SEL_SSI9_0),
+	PINMUX_IPSR_MSEL(IP17_7_4, TX2_B, SEL_SCIF2_1),
+	PINMUX_IPSR_MSEL(IP17_7_4, SDA3_E, SEL_I2C03_4),
+	PINMUX_IPSR_GPSR(IP17_7_4, VI0_DATA6_VI0_B6),
+	PINMUX_IPSR_MSEL(IP17_11_8, SSI_SDATA9_A, SEL_SSI9_0),
+	PINMUX_IPSR_GPSR(IP17_11_8, SCIF2_SCK_B),
+	PINMUX_IPSR_GPSR(IP17_11_8, PWM2_D),
+	PINMUX_IPSR_GPSR(IP17_11_8, VI0_DATA7_VI0_B7),
+	PINMUX_IPSR_MSEL(IP17_15_12, AUDIO_CLKA_A, SEL_ADGA_0),
+	PINMUX_IPSR_MSEL(IP17_15_12, SCL0_B, SEL_I2C00_1),
+	PINMUX_IPSR_GPSR(IP17_15_12, VI0_CLKENB),
+	PINMUX_IPSR_MSEL(IP17_19_16, AUDIO_CLKB_A, SEL_ADGB_0),
+	PINMUX_IPSR_MSEL(IP17_19_16, SDA0_B, SEL_I2C00_1),
+	PINMUX_IPSR_GPSR(IP17_19_16, VI0_FIELD),
+	PINMUX_IPSR_MSEL(IP17_23_20, AUDIO_CLKC_A, SEL_ADGC_0),
+	PINMUX_IPSR_MSEL(IP17_23_20, SCL4_B, SEL_I2C04_1),
+	PINMUX_IPSR_GPSR(IP17_23_20, VI0_HSYNC_N),
+	PINMUX_IPSR_GPSR(IP17_27_24, AUDIO_CLKOUT_A),
+	PINMUX_IPSR_MSEL(IP17_27_24, SDA4_B, SEL_I2C04_1),
+	PINMUX_IPSR_GPSR(IP17_27_24, VI0_VSYNC_N),
+};
+
+static const struct sh_pfc_pin pinmux_pins[] = {
+	PINMUX_GPIO_GP_ALL(),
+};
+
+/* - MMC -------------------------------------------------------------------- */
+static const unsigned int mmc_data1_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(0, 15),
+};
+static const unsigned int mmc_data1_mux[] = {
+	MMC0_D0_SDHI1_D0_MARK,
+};
+static const unsigned int mmc_data4_pins[] = {
+	/* D[0:3] */
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 16),
+	RCAR_GP_PIN(0, 17), RCAR_GP_PIN(0, 18),
+};
+static const unsigned int mmc_data4_mux[] = {
+	MMC0_D0_SDHI1_D0_MARK, MMC0_D1_SDHI1_D1_MARK,
+	MMC0_D2_SDHI1_D2_MARK, MMC0_D3_SDHI1_D3_MARK,
+};
+static const unsigned int mmc_data8_pins[] = {
+	/* D[0:3] */
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 16),
+	RCAR_GP_PIN(0, 17), RCAR_GP_PIN(0, 18),
+	RCAR_GP_PIN(0, 19), RCAR_GP_PIN(0, 20),
+	RCAR_GP_PIN(0, 21), RCAR_GP_PIN(0, 22),
+};
+static const unsigned int mmc_data8_mux[] = {
+	MMC0_D0_SDHI1_D0_MARK, MMC0_D1_SDHI1_D1_MARK,
+	MMC0_D2_SDHI1_D2_MARK, MMC0_D3_SDHI1_D3_MARK,
+	MMC0_D4_MARK, MMC0_D5_MARK,
+	MMC0_D6_MARK, MMC0_D7_MARK,
+};
+static const unsigned int mmc_ctrl_pins[] = {
+	/* CLK, CMD */
+	RCAR_GP_PIN(0, 13), RCAR_GP_PIN(0, 14),
+};
+static const unsigned int mmc_ctrl_mux[] = {
+	MMC0_CLK_SDHI1_CLK_MARK, MMC0_CMD_SDHI1_CMD_MARK,
+};
+/* - SCIF0 ------------------------------------------------------------------ */
+static const unsigned int scif0_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(4, 10), RCAR_GP_PIN(4, 11),
+};
+static const unsigned int scif0_data_a_mux[] = {
+	RX0_A_MARK, TX0_A_MARK,
+};
+static const unsigned int scif0_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(3, 11), RCAR_GP_PIN(3, 12),
+};
+static const unsigned int scif0_data_b_mux[] = {
+	RX0_B_MARK, TX0_B_MARK,
+};
+static const unsigned int scif0_data_c_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(4, 0), RCAR_GP_PIN(4, 1),
+};
+static const unsigned int scif0_data_c_mux[] = {
+	RX0_C_MARK, TX0_C_MARK,
+};
+static const unsigned int scif0_data_d_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 2), RCAR_GP_PIN(2, 3),
+};
+static const unsigned int scif0_data_d_mux[] = {
+	RX0_D_MARK, TX0_D_MARK,
+};
+/* - SCIF1 ------------------------------------------------------------------ */
+static const unsigned int scif1_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(4, 16), RCAR_GP_PIN(4, 17),
+};
+static const unsigned int scif1_data_a_mux[] = {
+	RX1_A_MARK, TX1_A_MARK,
+};
+static const unsigned int scif1_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(4, 15),
+};
+static const unsigned int scif1_clk_a_mux[] = {
+	SCIF1_SCK_A_MARK,
+};
+static const unsigned int scif1_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 19), RCAR_GP_PIN(5, 20),
+};
+static const unsigned int scif1_data_b_mux[] = {
+	RX1_B_MARK, TX1_B_MARK,
+};
+static const unsigned int scif1_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 18),
+};
+static const unsigned int scif1_clk_b_mux[] = {
+	SCIF1_SCK_B_MARK,
+};
+static const unsigned int scif1_data_c_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 8), RCAR_GP_PIN(1, 9),
+};
+static const unsigned int scif1_data_c_mux[] = {
+	RX1_C_MARK, TX1_C_MARK,
+};
+static const unsigned int scif1_clk_c_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 7),
+};
+static const unsigned int scif1_clk_c_mux[] = {
+	SCIF1_SCK_C_MARK,
+};
+static const unsigned int scif1_data_d_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 4), RCAR_GP_PIN(2, 5),
+};
+static const unsigned int scif1_data_d_mux[] = {
+	RX1_D_MARK, TX1_D_MARK,
+};
+/* - SCIF2 ------------------------------------------------------------------ */
+static const unsigned int scif2_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(4, 18), RCAR_GP_PIN(4, 19),
+};
+static const unsigned int scif2_data_a_mux[] = {
+	RX2_A_MARK, TX2_A_MARK,
+};
+static const unsigned int scif2_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(4, 20),
+};
+static const unsigned int scif2_clk_a_mux[] = {
+	SCIF2_SCK_A_MARK,
+};
+static const unsigned int scif2_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 25), RCAR_GP_PIN(5, 26),
+};
+static const unsigned int scif2_data_b_mux[] = {
+	RX2_B_MARK, TX2_B_MARK,
+};
+static const unsigned int scif2_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 27),
+};
+static const unsigned int scif2_clk_b_mux[] = {
+	SCIF2_SCK_B_MARK,
+};
+static const unsigned int scif2_data_c_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 6), RCAR_GP_PIN(2, 7),
+};
+static const unsigned int scif2_data_c_mux[] = {
+	RX2_C_MARK, TX2_C_MARK,
+};
+/* - SCIF3 ------------------------------------------------------------------ */
+static const unsigned int scif3_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(4, 22), RCAR_GP_PIN(4, 23),
+};
+static const unsigned int scif3_data_a_mux[] = {
+	RX3_A_MARK, TX3_A_MARK,
+};
+static const unsigned int scif3_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(4, 21),
+};
+static const unsigned int scif3_clk_mux[] = {
+	SCIF3_SCK_MARK,
+};
+static const unsigned int scif3_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 8), RCAR_GP_PIN(2, 9),
+};
+static const unsigned int scif3_data_b_mux[] = {
+	RX3_B_MARK, TX3_B_MARK,
+};
+static const unsigned int scif3_data_c_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(0, 5), RCAR_GP_PIN(0, 6),
+};
+static const unsigned int scif3_data_c_mux[] = {
+	RX3_C_MARK, TX3_C_MARK,
+};
+/* - SCIF4 ------------------------------------------------------------------ */
+static const unsigned int scif4_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(4, 2), RCAR_GP_PIN(4, 3),
+};
+static const unsigned int scif4_data_a_mux[] = {
+	RX4_A_MARK, TX4_A_MARK,
+};
+static const unsigned int scif4_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 2), RCAR_GP_PIN(1, 3),
+};
+static const unsigned int scif4_data_b_mux[] = {
+	RX4_B_MARK, TX4_B_MARK,
+};
+static const unsigned int scif4_data_c_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 13), RCAR_GP_PIN(1, 14),
+};
+static const unsigned int scif4_data_c_mux[] = {
+	RX4_C_MARK, TX4_C_MARK,
+};
+static const unsigned int scif4_data_d_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 10), RCAR_GP_PIN(2, 11),
+};
+static const unsigned int scif4_data_d_mux[] = {
+	RX4_D_MARK, TX4_D_MARK,
+};
+static const unsigned int scif4_data_e_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(0, 7), RCAR_GP_PIN(0, 8),
+};
+static const unsigned int scif4_data_e_mux[] = {
+	RX4_E_MARK, TX4_E_MARK,
+};
+/* - SCIF5 ------------------------------------------------------------------ */
+static const unsigned int scif5_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(4, 4), RCAR_GP_PIN(4, 5),
+};
+static const unsigned int scif5_data_a_mux[] = {
+	RX5_A_MARK, TX5_A_MARK,
+};
+static const unsigned int scif5_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 0), RCAR_GP_PIN(1, 1),
+};
+static const unsigned int scif5_data_b_mux[] = {
+	RX5_B_MARK, TX5_B_MARK,
+};
+static const unsigned int scif5_data_c_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 0), RCAR_GP_PIN(2, 1),
+};
+static const unsigned int scif5_data_c_mux[] = {
+	RX5_C_MARK, TX5_C_MARK,
+};
+static const unsigned int scif5_data_d_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 9), RCAR_GP_PIN(5, 10),
+};
+static const unsigned int scif5_data_d_mux[] = {
+	RX5_D_MARK, TX5_D_MARK,
+};
+static const unsigned int scif5_data_e_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(0, 9), RCAR_GP_PIN(0, 10),
+};
+static const unsigned int scif5_data_e_mux[] = {
+	RX5_E_MARK, TX5_E_MARK,
+};
+static const unsigned int scif5_data_f_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(3, 27), RCAR_GP_PIN(3, 28),
+};
+static const unsigned int scif5_data_f_mux[] = {
+	RX5_F_MARK, TX5_F_MARK,
+};
+/* - SCIF Clock ------------------------------------------------------------- */
+static const unsigned int scif_clk_a_pins[] = {
+	/* SCIF_CLK */
+	RCAR_GP_PIN(1, 22),
+};
+static const unsigned int scif_clk_a_mux[] = {
+	SCIF_CLK_A_MARK,
+};
+static const unsigned int scif_clk_b_pins[] = {
+	/* SCIF_CLK */
+	RCAR_GP_PIN(3, 29),
+};
+static const unsigned int scif_clk_b_mux[] = {
+	SCIF_CLK_B_MARK,
+};
+
+static const struct sh_pfc_pin_group pinmux_groups[] = {
+	SH_PFC_PIN_GROUP(mmc_data1),
+	SH_PFC_PIN_GROUP(mmc_data4),
+	SH_PFC_PIN_GROUP(mmc_data8),
+	SH_PFC_PIN_GROUP(mmc_ctrl),
+	SH_PFC_PIN_GROUP(scif0_data_a),
+	SH_PFC_PIN_GROUP(scif0_data_b),
+	SH_PFC_PIN_GROUP(scif0_data_c),
+	SH_PFC_PIN_GROUP(scif0_data_d),
+	SH_PFC_PIN_GROUP(scif1_data_a),
+	SH_PFC_PIN_GROUP(scif1_clk_a),
+	SH_PFC_PIN_GROUP(scif1_data_b),
+	SH_PFC_PIN_GROUP(scif1_clk_b),
+	SH_PFC_PIN_GROUP(scif1_data_c),
+	SH_PFC_PIN_GROUP(scif1_clk_c),
+	SH_PFC_PIN_GROUP(scif1_data_d),
+	SH_PFC_PIN_GROUP(scif2_data_a),
+	SH_PFC_PIN_GROUP(scif2_clk_a),
+	SH_PFC_PIN_GROUP(scif2_data_b),
+	SH_PFC_PIN_GROUP(scif2_clk_b),
+	SH_PFC_PIN_GROUP(scif2_data_c),
+	SH_PFC_PIN_GROUP(scif3_data_a),
+	SH_PFC_PIN_GROUP(scif3_clk),
+	SH_PFC_PIN_GROUP(scif3_data_b),
+	SH_PFC_PIN_GROUP(scif3_data_c),
+	SH_PFC_PIN_GROUP(scif4_data_a),
+	SH_PFC_PIN_GROUP(scif4_data_b),
+	SH_PFC_PIN_GROUP(scif4_data_c),
+	SH_PFC_PIN_GROUP(scif4_data_d),
+	SH_PFC_PIN_GROUP(scif4_data_e),
+	SH_PFC_PIN_GROUP(scif5_data_a),
+	SH_PFC_PIN_GROUP(scif5_data_b),
+	SH_PFC_PIN_GROUP(scif5_data_c),
+	SH_PFC_PIN_GROUP(scif5_data_d),
+	SH_PFC_PIN_GROUP(scif5_data_e),
+	SH_PFC_PIN_GROUP(scif5_data_f),
+	SH_PFC_PIN_GROUP(scif_clk_a),
+	SH_PFC_PIN_GROUP(scif_clk_b),
+};
+
+static const char * const mmc_groups[] = {
+	"mmc_data1",
+	"mmc_data4",
+	"mmc_data8",
+	"mmc_ctrl",
+};
+
+static const char * const scif0_groups[] = {
+	"scif0_data_a",
+	"scif0_data_b",
+	"scif0_data_c",
+	"scif0_data_d",
+};
+
+static const char * const scif1_groups[] = {
+	"scif1_data_a",
+	"scif1_clk_a",
+	"scif1_data_b",
+	"scif1_clk_b",
+	"scif1_data_c",
+	"scif1_clk_c",
+	"scif1_data_d",
+};
+
+static const char * const scif2_groups[] = {
+	"scif2_data_a",
+	"scif2_clk_a",
+	"scif2_data_b",
+	"scif2_clk_b",
+	"scif2_data_c",
+};
+
+static const char * const scif3_groups[] = {
+	"scif3_data_a",
+	"scif3_clk",
+	"scif3_data_b",
+	"scif3_data_c",
+};
+
+static const char * const scif4_groups[] = {
+	"scif4_data_a",
+	"scif4_data_b",
+	"scif4_data_c",
+	"scif4_data_d",
+	"scif4_data_e",
+};
+
+static const char * const scif5_groups[] = {
+	"scif5_data_a",
+	"scif5_data_b",
+	"scif5_data_c",
+	"scif5_data_d",
+	"scif5_data_e",
+	"scif5_data_f",
+};
+
+static const char * const scif_clk_groups[] = {
+	"scif_clk_a",
+	"scif_clk_b",
+};
+
+static const struct sh_pfc_function pinmux_functions[] = {
+	SH_PFC_FUNCTION(mmc),
+	SH_PFC_FUNCTION(scif0),
+	SH_PFC_FUNCTION(scif1),
+	SH_PFC_FUNCTION(scif2),
+	SH_PFC_FUNCTION(scif3),
+	SH_PFC_FUNCTION(scif4),
+	SH_PFC_FUNCTION(scif5),
+	SH_PFC_FUNCTION(scif_clk),
+};
+
+static const struct pinmux_cfg_reg pinmux_config_regs[] = {
+	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_0_22_FN, FN_MMC0_D7,
+		GP_0_21_FN, FN_MMC0_D6,
+		GP_0_20_FN, FN_IP1_7_4,
+		GP_0_19_FN, FN_IP1_3_0,
+		GP_0_18_FN, FN_MMC0_D3_SDHI1_D3,
+		GP_0_17_FN, FN_MMC0_D2_SDHI1_D2,
+		GP_0_16_FN, FN_MMC0_D1_SDHI1_D1,
+		GP_0_15_FN, FN_MMC0_D0_SDHI1_D0,
+		GP_0_14_FN, FN_MMC0_CMD_SDHI1_CMD,
+		GP_0_13_FN, FN_MMC0_CLK_SDHI1_CLK,
+		GP_0_12_FN, FN_IP0_31_28,
+		GP_0_11_FN, FN_IP0_27_24,
+		GP_0_10_FN, FN_IP0_23_20,
+		GP_0_9_FN, FN_IP0_19_16,
+		GP_0_8_FN, FN_IP0_15_12,
+		GP_0_7_FN, FN_IP0_11_8,
+		GP_0_6_FN, FN_IP0_7_4,
+		GP_0_5_FN, FN_IP0_3_0,
+		GP_0_4_FN, FN_CLKOUT,
+		GP_0_3_FN, FN_USB1_OVC,
+		GP_0_2_FN, FN_USB1_PWEN,
+		GP_0_1_FN, FN_USB0_OVC,
+		GP_0_0_FN, FN_USB0_PWEN, }
+	},
+	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_1_22_FN, FN_IP4_3_0,
+		GP_1_21_FN, FN_IP3_31_28,
+		GP_1_20_FN, FN_IP3_27_24,
+		GP_1_19_FN, FN_IP3_23_20,
+		GP_1_18_FN, FN_IP3_19_16,
+		GP_1_17_FN, FN_IP3_15_12,
+		GP_1_16_FN, FN_IP3_11_8,
+		GP_1_15_FN, FN_IP3_7_4,
+		GP_1_14_FN, FN_IP3_3_0,
+		GP_1_13_FN, FN_IP2_31_28,
+		GP_1_12_FN, FN_IP2_27_24,
+		GP_1_11_FN, FN_IP2_23_20,
+		GP_1_10_FN, FN_IP2_19_16,
+		GP_1_9_FN, FN_IP2_15_12,
+		GP_1_8_FN, FN_IP2_11_8,
+		GP_1_7_FN, FN_IP2_7_4,
+		GP_1_6_FN, FN_IP2_3_0,
+		GP_1_5_FN, FN_IP1_31_28,
+		GP_1_4_FN, FN_IP1_27_24,
+		GP_1_3_FN, FN_IP1_23_20,
+		GP_1_2_FN, FN_IP1_19_16,
+		GP_1_1_FN, FN_IP1_15_12,
+		GP_1_0_FN, FN_IP1_11_8, }
+	},
+	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1) {
+		GP_2_31_FN, FN_IP8_3_0,
+		GP_2_30_FN, FN_IP7_31_28,
+		GP_2_29_FN, FN_IP7_27_24,
+		GP_2_28_FN, FN_IP7_23_20,
+		GP_2_27_FN, FN_IP7_19_16,
+		GP_2_26_FN, FN_IP7_15_12,
+		GP_2_25_FN, FN_IP7_11_8,
+		GP_2_24_FN, FN_IP7_7_4,
+		GP_2_23_FN, FN_IP7_3_0,
+		GP_2_22_FN, FN_IP6_31_28,
+		GP_2_21_FN, FN_IP6_27_24,
+		GP_2_20_FN, FN_IP6_23_20,
+		GP_2_19_FN, FN_IP6_19_16,
+		GP_2_18_FN, FN_IP6_15_12,
+		GP_2_17_FN, FN_IP6_11_8,
+		GP_2_16_FN, FN_IP6_7_4,
+		GP_2_15_FN, FN_IP6_3_0,
+		GP_2_14_FN, FN_IP5_31_28,
+		GP_2_13_FN, FN_IP5_27_24,
+		GP_2_12_FN, FN_IP5_23_20,
+		GP_2_11_FN, FN_IP5_19_16,
+		GP_2_10_FN, FN_IP5_15_12,
+		GP_2_9_FN, FN_IP5_11_8,
+		GP_2_8_FN, FN_IP5_7_4,
+		GP_2_7_FN, FN_IP5_3_0,
+		GP_2_6_FN, FN_IP4_31_28,
+		GP_2_5_FN, FN_IP4_27_24,
+		GP_2_4_FN, FN_IP4_23_20,
+		GP_2_3_FN, FN_IP4_19_16,
+		GP_2_2_FN, FN_IP4_15_12,
+		GP_2_1_FN, FN_IP4_11_8,
+		GP_2_0_FN, FN_IP4_7_4, }
+	},
+	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1) {
+		0, 0,
+		0, 0,
+		GP_3_29_FN, FN_IP10_19_16,
+		GP_3_28_FN, FN_IP10_15_12,
+		GP_3_27_FN, FN_IP10_11_8,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_3_16_FN, FN_IP10_7_4,
+		GP_3_15_FN, FN_IP10_3_0,
+		GP_3_14_FN, FN_IP9_31_28,
+		GP_3_13_FN, FN_IP9_27_24,
+		GP_3_12_FN, FN_IP9_23_20,
+		GP_3_11_FN, FN_IP9_19_16,
+		GP_3_10_FN, FN_IP9_15_12,
+		GP_3_9_FN, FN_IP9_11_8,
+		GP_3_8_FN, FN_IP9_7_4,
+		GP_3_7_FN, FN_IP9_3_0,
+		GP_3_6_FN, FN_IP8_31_28,
+		GP_3_5_FN, FN_IP8_27_24,
+		GP_3_4_FN, FN_IP8_23_20,
+		GP_3_3_FN, FN_IP8_19_16,
+		GP_3_2_FN, FN_IP8_15_12,
+		GP_3_1_FN, FN_IP8_11_8,
+		GP_3_0_FN, FN_IP8_7_4, }
+	},
+	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_4_25_FN, FN_IP13_27_24,
+		GP_4_24_FN, FN_IP13_23_20,
+		GP_4_23_FN, FN_IP13_19_16,
+		GP_4_22_FN, FN_IP13_15_12,
+		GP_4_21_FN, FN_IP13_11_8,
+		GP_4_20_FN, FN_IP13_7_4,
+		GP_4_19_FN, FN_IP13_3_0,
+		GP_4_18_FN, FN_IP12_31_28,
+		GP_4_17_FN, FN_IP12_27_24,
+		GP_4_16_FN, FN_IP12_23_20,
+		GP_4_15_FN, FN_IP12_19_16,
+		GP_4_14_FN, FN_IP12_15_12,
+		GP_4_13_FN, FN_IP12_11_8,
+		GP_4_12_FN, FN_IP12_7_4,
+		GP_4_11_FN, FN_IP12_3_0,
+		GP_4_10_FN, FN_IP11_31_28,
+		GP_4_9_FN, FN_IP11_27_24,
+		GP_4_8_FN, FN_IP11_23_20,
+		GP_4_7_FN, FN_IP11_19_16,
+		GP_4_6_FN, FN_IP11_15_12,
+		GP_4_5_FN, FN_IP11_11_8,
+		GP_4_4_FN, FN_IP11_7_4,
+		GP_4_3_FN, FN_IP11_3_0,
+		GP_4_2_FN, FN_IP10_31_28,
+		GP_4_1_FN, FN_IP10_27_24,
+		GP_4_0_FN, FN_IP10_23_20, }
+	},
+	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1) {
+		GP_5_31_FN, FN_IP17_27_24,
+		GP_5_30_FN, FN_IP17_23_20,
+		GP_5_29_FN, FN_IP17_19_16,
+		GP_5_28_FN, FN_IP17_15_12,
+		GP_5_27_FN, FN_IP17_11_8,
+		GP_5_26_FN, FN_IP17_7_4,
+		GP_5_25_FN, FN_IP17_3_0,
+		GP_5_24_FN, FN_IP16_31_28,
+		GP_5_23_FN, FN_IP16_27_24,
+		GP_5_22_FN, FN_IP16_23_20,
+		GP_5_21_FN, FN_IP16_19_16,
+		GP_5_20_FN, FN_IP16_15_12,
+		GP_5_19_FN, FN_IP16_11_8,
+		GP_5_18_FN, FN_IP16_7_4,
+		GP_5_17_FN, FN_IP16_3_0,
+		GP_5_16_FN, FN_IP15_31_28,
+		GP_5_15_FN, FN_IP15_27_24,
+		GP_5_14_FN, FN_IP15_23_20,
+		GP_5_13_FN, FN_IP15_19_16,
+		GP_5_12_FN, FN_IP15_15_12,
+		GP_5_11_FN, FN_IP15_11_8,
+		GP_5_10_FN, FN_IP15_7_4,
+		GP_5_9_FN, FN_IP15_3_0,
+		GP_5_8_FN, FN_IP14_31_28,
+		GP_5_7_FN, FN_IP14_27_24,
+		GP_5_6_FN, FN_IP14_23_20,
+		GP_5_5_FN, FN_IP14_19_16,
+		GP_5_4_FN, FN_IP14_15_12,
+		GP_5_3_FN, FN_IP14_11_8,
+		GP_5_2_FN, FN_IP14_7_4,
+		GP_5_1_FN, FN_IP14_3_0,
+		GP_5_0_FN, FN_IP13_31_28, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR0", 0xE6060040, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP0_31_28 [4] */
+		FN_SD0_WP, FN_IRQ7, FN_CAN0_TX_A, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP0_27_24 [4] */
+		FN_SD0_CD, 0, FN_CAN0_RX_A, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP0_23_20 [4] */
+		FN_SD0_DAT3, 0, 0, FN_SSI_SDATA0_B, FN_TX5_E, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP0_19_16 [4] */
+		FN_SD0_DAT2, 0, 0, FN_SSI_WS0129_B, FN_RX5_E, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP0_15_12 [4] */
+		FN_SD0_DAT1, 0, 0, FN_SSI_SCK0129_B, FN_TX4_E, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP0_11_8 [4] */
+		FN_SD0_DAT0, 0, 0, FN_SSI_SDATA1_C, FN_RX4_E, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP0_7_4 [4] */
+		FN_SD0_CMD, 0, 0, FN_SSI_WS1_C, FN_TX3_C, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP0_3_0 [4] */
+		FN_SD0_CLK, 0, 0, FN_SSI_SCK1_C, FN_RX3_C, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR1", 0xE6060044, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP1_31_28 [4] */
+		FN_D5, FN_HRX2, FN_SCL1_B, FN_PWM2_C, FN_TCLK2_B, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP1_27_24 [4] */
+		FN_D4, 0, FN_IRQ3, FN_TCLK1_A, FN_PWM6_C, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP1_23_20 [4] */
+		FN_D3, 0, FN_TX4_B, FN_SDA0_D, FN_PWM0_A,
+		FN_MSIOF2_SYNC_C, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP1_19_16 [4] */
+		FN_D2, 0, FN_RX4_B, FN_SCL0_D, FN_PWM1_C,
+		FN_MSIOF2_SCK_C, FN_SSI_SCK5_B, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP1_15_12 [4] */
+		FN_D1, 0, FN_SDA3_B, FN_TX5_B, 0, FN_MSIOF2_TXD_C,
+		FN_SSI_WS5_B, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP1_11_8 [4] */
+		FN_D0, 0, FN_SCL3_B, FN_RX5_B, FN_IRQ4,
+		FN_MSIOF2_RXD_C, FN_SSI_SDATA5_B, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP1_7_4 [4] */
+		FN_MMC0_D5, FN_SD1_WP, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP1_3_0 [4] */
+		FN_MMC0_D4, FN_SD1_CD, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR2", 0xE6060048, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP2_31_28 [4] */
+		FN_D13, FN_MSIOF2_SYNC_A, 0, FN_RX4_C, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0,
+		/* IP2_27_24 [4] */
+		FN_D12, FN_MSIOF2_SCK_A, FN_HSCK0, 0, FN_CAN_CLK_C,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP2_23_20 [4] */
+		FN_D11, FN_MSIOF2_TXD_A, FN_HTX0_B, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0,
+		/* IP2_19_16 [4] */
+		FN_D10, FN_MSIOF2_RXD_A, FN_HRX0_B, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0,
+		/* IP2_15_12 [4] */
+		FN_D9, FN_HRTS2_N, FN_TX1_C, FN_SDA1_D, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP2_11_8 [4] */
+		FN_D8, FN_HCTS2_N, FN_RX1_C, FN_SCL1_D, FN_PWM3_C, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP2_7_4 [4] */
+		FN_D7, FN_HSCK2, FN_SCIF1_SCK_C, FN_IRQ6, FN_PWM5_C,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP2_3_0 [4] */
+		FN_D6, FN_HTX2, FN_SDA1_B, FN_PWM4_C, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR3", 0xE606004C, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP3_31_28 [4] */
+		FN_QSPI0_SSL, FN_WE1_N, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0,
+		/* IP3_27_24 [4] */
+		FN_QSPI0_IO3, FN_RD_N, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0,
+		/* IP3_23_20 [4] */
+		FN_QSPI0_IO2, FN_CS0_N, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0,
+		/* IP3_19_16 [4] */
+		FN_QSPI0_MISO_QSPI0_IO1, FN_RD_WR_N, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0,
+		/* IP3_15_12 [4] */
+		FN_QSPI0_MOSI_QSPI0_IO0, FN_BS_N, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0,
+		/* IP3_11_8 [4] */
+		FN_QSPI0_SPCLK, FN_WE0_N, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0,
+		/* IP3_7_4 [4] */
+		FN_D15, FN_MSIOF2_SS2, FN_PWM4_A, 0, FN_CAN1_TX_B, FN_IRQ2,
+		FN_AVB_AVTP_MATCH_A, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP3_3_0 [4] */
+		FN_D14, FN_MSIOF2_SS1, 0, FN_TX4_C, FN_CAN1_RX_B,
+		0, FN_AVB_AVTP_CAPTURE_A,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR4", 0xE6060050, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP4_31_28 [4] */
+		FN_DU0_DR6, 0, FN_RX2_C, 0, 0, 0, FN_A6, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP4_27_24 [4] */
+		FN_DU0_DR5, 0, FN_TX1_D, 0, FN_PWM1_B, 0, FN_A5, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP4_23_20 [4] */
+		FN_DU0_DR4, 0, FN_RX1_D, 0, 0, 0, FN_A4, 0, 0, 0, 0,
+		0, 0, 0, 0, 0,
+		/* IP4_19_16 [4] */
+		FN_DU0_DR3, 0, FN_TX0_D, FN_SDA0_E, FN_PWM0_B, 0,
+		FN_A3, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP4_15_12 [4] */
+		FN_DU0_DR2, 0, FN_RX0_D, FN_SCL0_E, 0, 0, FN_A2, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP4_11_8 [4] */
+		FN_DU0_DR1, 0, FN_TX5_C, FN_SDA2_D, 0, 0, FN_A1, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP4_7_4 [4] */
+		FN_DU0_DR0, 0, FN_RX5_C, FN_SCL2_D, 0, 0, FN_A0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP4_3_0 [4] */
+		FN_EX_WAIT0, FN_CAN_CLK_B, FN_SCIF_CLK_A, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR5", 0xE6060054, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP5_31_28 [4] */
+		FN_DU0_DG6, 0, FN_HRX1_C, 0, 0, 0, FN_A14,  0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP5_27_24 [4] */
+		FN_DU0_DG5, 0, FN_HTX0_A, 0, FN_PWM5_B, 0, FN_A13,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP5_23_20 [4] */
+		FN_DU0_DG4, 0, FN_HRX0_A, 0, 0, 0, FN_A12, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP5_19_16 [4] */
+		FN_DU0_DG3, 0, FN_TX4_D, 0, FN_PWM4_B, 0, FN_A11, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP5_15_12 [4] */
+		FN_DU0_DG2, 0, FN_RX4_D, 0, 0, 0, FN_A10, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP5_11_8 [4] */
+		FN_DU0_DG1, 0, FN_TX3_B, FN_SDA3_D, FN_PWM3_B, 0,
+		FN_A9, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP5_7_4 [4] */
+		FN_DU0_DG0, 0, FN_RX3_B, FN_SCL3_D, 0, 0, FN_A8, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP5_3_0 [4] */
+		FN_DU0_DR7, 0, FN_TX2_C, 0, FN_PWM2_B, 0, FN_A7, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR6", 0xE6060058, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP6_31_28 [4] */
+		FN_DU0_DB6, 0, 0, 0, 0, 0, FN_A22, 0, 0,
+		0, 0, 0, 0, 0, 0, 0,
+		/* IP6_27_24 [4] */
+		FN_DU0_DB5, 0, FN_HRTS1_N_C, 0, 0, 0,
+		FN_A21, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP6_23_20 [4] */
+		FN_DU0_DB4, 0, FN_HCTS1_N_C, 0, 0, 0,
+		FN_A20, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP6_19_16 [4] */
+		FN_DU0_DB3, 0, FN_HRTS0_N, 0, 0, 0, FN_A19, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP6_15_12 [4] */
+		FN_DU0_DB2, 0, FN_HCTS0_N, 0, 0, 0, FN_A18, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP6_11_8 [4] */
+		FN_DU0_DB1, 0, 0, FN_SDA4_D, FN_CAN0_TX_C, 0, FN_A17,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP6_7_4 [4] */
+		FN_DU0_DB0, 0, 0, FN_SCL4_D, FN_CAN0_RX_C, 0, FN_A16,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP6_3_0 [4] */
+		FN_DU0_DG7, 0, FN_HTX1_C, 0,  FN_PWM6_B, 0, FN_A15,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR7", 0xE606005C, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP7_31_28 [4] */
+		FN_DU0_DISP, 0, 0, 0, FN_CAN1_RX_C, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0,
+		/* IP7_27_24 [4] */
+		FN_DU0_EXODDF_DU0_ODDF_DISP_CDE, 0, FN_MSIOF2_SCK_B,
+		0, 0, 0, FN_DRACK0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP7_23_20 [4] */
+		FN_DU0_EXVSYNC_DU0_VSYNC, 0, FN_MSIOF2_SYNC_B, 0,
+		0, 0, FN_DACK0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP7_19_16 [4] */
+		FN_DU0_EXHSYNC_DU0_HSYNC, 0, FN_MSIOF2_TXD_B, 0,
+		0, 0, FN_DREQ0_N, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP7_15_12 [4] */
+		FN_DU0_DOTCLKOUT1, 0, FN_MSIOF2_RXD_B, 0, 0, 0,
+		FN_CS1_N_A26, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP7_11_8 [4] */
+		FN_DU0_DOTCLKOUT0, 0, 0, 0, 0, 0, FN_A25, 0, 0, 0, 0,
+		0, 0, 0, 0, 0,
+		/* IP7_7_4 [4] */
+		FN_DU0_DOTCLKIN, 0, 0, 0, 0, 0, FN_A24, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP7_3_0 [4] */
+		FN_DU0_DB7, 0, 0, 0, 0, 0, FN_A23, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR8", 0xE6060060, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP8_31_28 [4] */
+		FN_VI1_DATA5, 0, 0, 0, FN_AVB_RXD4, FN_ETH_LINK, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP8_27_24 [4] */
+		FN_VI1_DATA4, 0, 0, 0, FN_AVB_RXD3, FN_ETH_RX_ER, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP8_23_20 [4] */
+		FN_VI1_DATA3, 0, 0, 0, FN_AVB_RXD2, FN_ETH_MDIO, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP8_19_16 [4] */
+		FN_VI1_DATA2, 0, 0, 0, FN_AVB_RXD1, FN_ETH_RXD1, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP8_15_12 [4] */
+		FN_VI1_DATA1, 0, 0, 0, FN_AVB_RXD0, FN_ETH_RXD0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP8_11_8 [4] */
+		FN_VI1_DATA0, 0, 0, 0, FN_AVB_RX_DV, FN_ETH_CRS_DV, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0,
+		/* IP8_7_4 [4] */
+		FN_VI1_CLK, 0, 0, 0, FN_AVB_RX_CLK, FN_ETH_REF_CLK, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0,
+		/* IP8_3_0 [4] */
+		FN_DU0_CDE, 0, 0, 0, FN_CAN1_TX_C, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR9", 0xE6060064, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP9_31_28 [4] */
+		FN_VI1_DATA9, 0, 0, FN_SDA2_B, FN_AVB_TXD0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0,
+		/* IP9_27_24 [4] */
+		FN_VI1_DATA8, 0, 0, FN_SCL2_B, FN_AVB_TX_EN, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0,
+		/* IP9_23_20 [4] */
+		FN_VI1_VSYNC_N, FN_TX0_B, FN_SDA0_C, FN_AUDIO_CLKOUT_B,
+		FN_AVB_TX_CLK, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP9_19_16 [4] */
+		FN_VI1_HSYNC_N, FN_RX0_B, FN_SCL0_C, 0, FN_AVB_GTXREFCLK,
+		FN_ETH_MDC, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP9_15_12 [4] */
+		FN_VI1_FIELD, FN_SDA3_A, 0, 0, FN_AVB_RX_ER, FN_ETH_TXD0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP9_11_8 [4] */
+		FN_VI1_CLKENB, FN_SCL3_A, 0, 0, FN_AVB_RXD7, FN_ETH_MAGIC, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP9_7_4 [4] */
+		FN_VI1_DATA7, 0, 0, 0, FN_AVB_RXD6, FN_ETH_TX_EN, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP9_3_0 [4] */
+		FN_VI1_DATA6, 0, 0, 0, FN_AVB_RXD5, FN_ETH_TXD1, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR10", 0xE6060068, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP10_31_28 [4] */
+		FN_SCL1_A, FN_RX4_A, FN_PWM5_D, FN_DU1_DR0, 0, 0,
+		FN_SSI_SCK6_B, FN_VI0_G0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP10_27_24 [4] */
+		FN_SDA0_A, FN_TX0_C, FN_IRQ5, FN_CAN_CLK_A, FN_AVB_GTX_CLK,
+		FN_CAN1_TX_D, FN_DVC_MUTE, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP10_23_20 [4] */
+		FN_SCL0_A, FN_RX0_C, FN_PWM5_A, FN_TCLK1_B, FN_AVB_TXD6,
+		FN_CAN1_RX_D, FN_MSIOF0_SYNC_B, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP10_19_16 [4] */
+		FN_AVB_TXD5, FN_SCIF_CLK_B, FN_AUDIO_CLKC_B, 0,
+		FN_SSI_SDATA1_D, 0, FN_MSIOF0_SCK_B, 0, 0, 0, 0, 0, 0, 0,
+		0, 0,
+		/* IP10_15_12 [4] */
+		FN_AVB_TXD4, 0, FN_AUDIO_CLKB_B, 0, FN_SSI_WS1_D, FN_TX5_F,
+		FN_MSIOF0_TXD_B, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP10_11_8 [4] */
+		FN_AVB_TXD3, 0, FN_AUDIO_CLKA_B, 0, FN_SSI_SCK1_D, FN_RX5_F,
+		FN_MSIOF0_RXD_B, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP10_7_4 [4] */
+		FN_VI1_DATA11, 0, 0, FN_CAN0_TX_B, FN_AVB_TXD2, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0,
+		/* IP10_3_0 [4] */
+		FN_VI1_DATA10, 0, 0, FN_CAN0_RX_B, FN_AVB_TXD1, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR11", 0xE606006C, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP11_31_28 [4] */
+		FN_HRX1_A, FN_SCL4_A, FN_PWM6_A, FN_DU1_DG0, FN_RX0_A, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP11_27_24 [4] */
+		FN_MSIOF0_SS2_A, 0, 0, FN_DU1_DR7, 0,
+		FN_QSPI1_SSL, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP11_23_20 [4] */
+		FN_MSIOF0_SS1_A, 0, 0, FN_DU1_DR6, 0,
+		FN_QSPI1_IO3, FN_SSI_SDATA8_B, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP11_19_16 [4] */
+		FN_MSIOF0_SYNC_A, FN_PWM1_A, 0, FN_DU1_DR5,
+		0, FN_QSPI1_IO2, FN_SSI_SDATA7_B, 0, 0, 0, 0, 0,
+		0, 0, 0, 0,
+		/* IP11_15_12 [4] */
+		FN_MSIOF0_SCK_A, FN_IRQ0, 0, FN_DU1_DR4,
+		0, FN_QSPI1_SPCLK, FN_SSI_SCK78_B, FN_VI0_G4,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP11_11_8 [4] */
+		FN_MSIOF0_TXD_A, FN_TX5_A, FN_SDA2_C, FN_DU1_DR3, 0,
+		FN_QSPI1_MISO_QSPI1_IO1, FN_SSI_WS78_B, FN_VI0_G3,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP11_7_4 [4] */
+		FN_MSIOF0_RXD_A, FN_RX5_A, FN_SCL2_C, FN_DU1_DR2, 0,
+		FN_QSPI1_MOSI_QSPI1_IO0, FN_SSI_SDATA6_B, FN_VI0_G2,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP11_3_0 [4] */
+		FN_SDA1_A, FN_TX4_A, 0, FN_DU1_DR1, 0, 0, FN_SSI_WS6_B,
+		FN_VI0_G1, 0, 0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR12", 0xE6060070, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP12_31_28 [4] */
+		FN_SD2_DAT2, FN_RX2_A, 0, FN_DU1_DB0, FN_SSI_SDATA2_B, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP12_27_24 [4] */
+		FN_SD2_DAT1, FN_TX1_A, FN_SDA1_E, FN_DU1_DG7, FN_SSI_WS2_B,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP12_23_20 [4] */
+		FN_SD2_DAT0, FN_RX1_A, FN_SCL1_E, FN_DU1_DG6,
+		FN_SSI_SDATA1_B, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP12_19_16 [4] */
+		FN_SD2_CMD, FN_SCIF1_SCK_A, FN_TCLK2_A, FN_DU1_DG5,
+		FN_SSI_SCK2_B, FN_PWM3_A, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP12_15_12 [4] */
+		FN_SD2_CLK, FN_HSCK1, 0, FN_DU1_DG4, FN_SSI_SCK1_B, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP12_11_8 [4] */
+		FN_HRTS1_N_A, 0, 0, FN_DU1_DG3, FN_SSI_WS1_B, FN_IRQ1, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP12_7_4 [4] */
+		FN_HCTS1_N_A, FN_PWM2_A, 0, FN_DU1_DG2, FN_REMOCON_B,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP12_3_0 [4] */
+		FN_HTX1_A, FN_SDA4_A, 0, FN_DU1_DG1, FN_TX0_A, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR13", 0xE6060074, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP13_31_28 [4] */
+		FN_SSI_SCK5_A, 0, 0, FN_DU1_DOTCLKOUT1, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0,
+		/* IP13_27_24 [4] */
+		FN_SDA2_A, 0, FN_MSIOF1_SYNC_B, FN_DU1_DB7, FN_AUDIO_CLKOUT_C,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP13_23_20 [4] */
+		FN_SCL2_A, 0, FN_MSIOF1_SCK_B, FN_DU1_DB6, FN_AUDIO_CLKC_C,
+		FN_SSI_SCK4_B, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP13_19_16 [4] */
+		FN_TX3_A, FN_SDA1_C, FN_MSIOF1_TXD_B, FN_DU1_DB5,
+		FN_AUDIO_CLKB_C, FN_SSI_WS4_B, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP13_15_12 [4] */
+		FN_RX3_A, FN_SCL1_C, FN_MSIOF1_RXD_B, FN_DU1_DB4,
+		FN_AUDIO_CLKA_C, FN_SSI_SDATA4_B, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0,
+		/* IP13_11_8 [4] */
+		FN_SD2_WP, FN_SCIF3_SCK, 0, FN_DU1_DB3, FN_SSI_SDATA9_B, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP13_7_4 [4] */
+		FN_SD2_CD, FN_SCIF2_SCK_A, 0, FN_DU1_DB2, FN_SSI_SCK9_B, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP13_3_0 [4] */
+		FN_SD2_DAT3, FN_TX2_A, 0, FN_DU1_DB1, FN_SSI_WS9_B, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR14", 0xE6060078, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP14_31_28 [4] */
+		FN_SSI_SDATA7_A, 0, 0, FN_IRQ8, FN_AUDIO_CLKA_D, FN_CAN_CLK_D,
+		FN_VI0_G5, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP14_27_24 [4] */
+		FN_SSI_WS78_A, 0, FN_SCL4_E, FN_DU1_CDE, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0,
+		/* IP14_23_20 [4] */
+		FN_SSI_SCK78_A, 0, FN_SDA4_E, FN_DU1_DISP, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP14_19_16 [4] */
+		FN_SSI_SDATA6_A, 0, FN_SDA4_C, FN_DU1_EXVSYNC_DU1_VSYNC, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP14_15_12 [4] */
+		FN_SSI_WS6_A, 0, FN_SCL4_C, FN_DU1_EXHSYNC_DU1_HSYNC, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP14_11_8 [4] */
+		FN_SSI_SCK6_A, 0, 0, FN_DU1_EXODDF_DU1_ODDF_DISP_CDE, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP14_7_4 [4] */
+		FN_SSI_SDATA5_A, 0, FN_SDA3_C, FN_DU1_DOTCLKOUT0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP14_3_0 [4] */
+		FN_SSI_WS5_A, 0, FN_SCL3_C, FN_DU1_DOTCLKIN, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR15", 0xE606007C, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP15_31_28 [4] */
+		FN_SSI_WS4_A, 0, FN_AVB_PHY_INT, 0, 0, 0, FN_VI0_R5, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP15_27_24 [4] */
+		FN_SSI_SCK4_A, 0, FN_AVB_MAGIC, 0, 0, 0, FN_VI0_R4, 0, 0, 0,
+		0, 0, 0, 0, 0, 0,
+		/* IP15_23_20 [4] */
+		FN_SSI_SDATA3, FN_MSIOF1_SS2_A, FN_AVB_LINK, 0, FN_CAN1_TX_A,
+		FN_DREQ2_N, FN_VI0_R3, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP15_19_16 [4] */
+		FN_SSI_WS34, FN_MSIOF1_SS1_A, FN_AVB_MDIO, 0, FN_CAN1_RX_A,
+		FN_DREQ1_N, FN_VI0_R2, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP15_15_12 [4] */
+		FN_SSI_SCK34, FN_MSIOF1_SCK_A, FN_AVB_MDC, 0, 0, FN_DACK1,
+		FN_VI0_R1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP15_11_8 [4] */
+		FN_SSI_SDATA0_A, FN_MSIOF1_SYNC_A, FN_PWM0_C, 0, 0, 0,
+		FN_VI0_R0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP15_7_4 [4] */
+		FN_SSI_WS0129_A, FN_MSIOF1_TXD_A, FN_TX5_D, 0, 0, 0,
+		FN_VI0_G7, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP15_3_0 [4] */
+		FN_SSI_SCK0129_A, FN_MSIOF1_RXD_A, FN_RX5_D, 0, 0, 0,
+		FN_VI0_G6, 0, 0, 0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR16", 0xE6060080, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP16_31_28 [4] */
+		FN_SSI_SDATA2_A, FN_HRTS1_N_B, 0, 0, 0, 0,
+		FN_VI0_DATA4_VI0_B4, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP16_27_24 [4] */
+		FN_SSI_WS2_A, FN_HCTS1_N_B, 0, 0, 0, FN_AVB_TX_ER,
+		FN_VI0_DATA3_VI0_B3, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP16_23_20 [4] */
+		FN_SSI_SCK2_A, FN_HTX1_B, 0, 0, 0, FN_AVB_TXD7,
+		FN_VI0_DATA2_VI0_B2, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP16_19_16 [4] */
+		FN_SSI_SDATA1_A, FN_HRX1_B, 0, 0, 0, 0, FN_VI0_DATA1_VI0_B1,
+		0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP16_15_12 [4] */
+		FN_SSI_WS1_A, FN_TX1_B, 0, 0, FN_CAN0_TX_D,
+		FN_AVB_AVTP_MATCH_B, FN_VI0_DATA0_VI0_B0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0,
+		/* IP16_11_8 [4] */
+		FN_SSI_SDATA8_A, FN_RX1_B, 0, 0, FN_CAN0_RX_D,
+		FN_AVB_AVTP_CAPTURE_B, FN_VI0_R7, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP16_7_4 [4] */
+		FN_SSI_SCK1_A, FN_SCIF1_SCK_B, FN_PWM1_D, FN_IRQ9, FN_REMOCON_A,
+		FN_DACK2, FN_VI0_CLK, FN_AVB_COL, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP16_3_0 [4] */
+		FN_SSI_SDATA4_A, 0, FN_AVB_CRS, 0, 0, 0, FN_VI0_R6, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR17", 0xE6060084, 32,
+			     4, 4, 4, 4, 4, 4, 4, 4) {
+		/* IP17_31_28 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP17_27_24 [4] */
+		FN_AUDIO_CLKOUT_A, FN_SDA4_B, 0, 0, 0, 0,
+		FN_VI0_VSYNC_N, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP17_23_20 [4] */
+		FN_AUDIO_CLKC_A, FN_SCL4_B, 0, 0, 0, 0,
+		FN_VI0_HSYNC_N, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP17_19_16 [4] */
+		FN_AUDIO_CLKB_A, FN_SDA0_B, 0, 0, 0, 0,
+		FN_VI0_FIELD, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP17_15_12 [4] */
+		FN_AUDIO_CLKA_A, FN_SCL0_B, 0, 0, 0, 0,
+		FN_VI0_CLKENB, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP17_11_8 [4] */
+		FN_SSI_SDATA9_A, FN_SCIF2_SCK_B, FN_PWM2_D, 0, 0, 0,
+		FN_VI0_DATA7_VI0_B7, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP17_7_4 [4] */
+		FN_SSI_WS9_A, FN_TX2_B, FN_SDA3_E, 0, 0, 0,
+		FN_VI0_DATA6_VI0_B6, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP17_3_0 [4] */
+		FN_SSI_SCK9_A, FN_RX2_B, FN_SCL3_E, 0, 0, FN_EX_WAIT1,
+		FN_VI0_DATA5_VI0_B5, 0, 0, 0, 0, 0, 0, 0, 0, 0, }
+	},
+	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xE60600C0, 32,
+			     1, 1, 1, 1, 1, 2, 1, 1, 2, 2, 2, 1, 3, 3,
+				 1, 2, 3, 3, 1) {
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* SEL_ADGA [2] */
+		FN_SEL_ADGA_0, FN_SEL_ADGA_1, FN_SEL_ADGA_2, FN_SEL_ADGA_3,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* SEL_CANCLK [2] */
+		FN_SEL_CANCLK_0, FN_SEL_CANCLK_1, FN_SEL_CANCLK_2,
+		FN_SEL_CANCLK_3,
+		/* SEL_CAN1 [2] */
+		FN_SEL_CAN1_0, FN_SEL_CAN1_1, FN_SEL_CAN1_2, FN_SEL_CAN1_3,
+		/* SEL_CAN0 [2] */
+		FN_SEL_CAN0_0, FN_SEL_CAN0_1, FN_SEL_CAN0_2, FN_SEL_CAN0_3,
+		/* RESERVED [1] */
+		0, 0,
+		/* SEL_I2C04 [3] */
+		FN_SEL_I2C04_0, FN_SEL_I2C04_1, FN_SEL_I2C04_2, FN_SEL_I2C04_3,
+		FN_SEL_I2C04_4, 0, 0, 0,
+		/* SEL_I2C03 [3] */
+		FN_SEL_I2C03_0, FN_SEL_I2C03_1, FN_SEL_I2C03_2, FN_SEL_I2C03_3,
+		FN_SEL_I2C03_4, 0, 0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* SEL_I2C02 [2] */
+		FN_SEL_I2C02_0, FN_SEL_I2C02_1, FN_SEL_I2C02_2, FN_SEL_I2C02_3,
+		/* SEL_I2C01 [3] */
+		FN_SEL_I2C01_0, FN_SEL_I2C01_1, FN_SEL_I2C01_2, FN_SEL_I2C01_3,
+		FN_SEL_I2C01_4, 0, 0, 0,
+		/* SEL_I2C00 [3] */
+		FN_SEL_I2C00_0, FN_SEL_I2C00_1, FN_SEL_I2C00_2, FN_SEL_I2C00_3,
+		FN_SEL_I2C00_4, 0, 0, 0,
+		/* SEL_AVB [1] */
+		FN_SEL_AVB_0, FN_SEL_AVB_1, }
+	},
+	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xE60600C4, 32,
+			     1, 3, 3, 2, 2, 1, 2, 2,
+			     2, 1, 1, 1, 1, 1, 2, 1, 1, 2, 2, 1) {
+		/* SEL_SCIFCLK [1] */
+		FN_SEL_SCIFCLK_0, FN_SEL_SCIFCLK_1,
+		/* SEL_SCIF5 [3] */
+		FN_SEL_SCIF5_0, FN_SEL_SCIF5_1, FN_SEL_SCIF5_2, FN_SEL_SCIF5_3,
+		FN_SEL_SCIF5_4, FN_SEL_SCIF5_5, 0, 0,
+		/* SEL_SCIF4 [3] */
+		FN_SEL_SCIF4_0, FN_SEL_SCIF4_1, FN_SEL_SCIF4_2, FN_SEL_SCIF4_3,
+		FN_SEL_SCIF4_4, 0, 0, 0,
+		/* SEL_SCIF3 [2] */
+		FN_SEL_SCIF3_0, FN_SEL_SCIF3_1, FN_SEL_SCIF3_2, 0,
+		/* SEL_SCIF2 [2] */
+		FN_SEL_SCIF2_0, FN_SEL_SCIF2_1, FN_SEL_SCIF2_2, 0,
+		/* SEL_SCIF2_CLK [1] */
+		FN_SEL_SCIF2_CLK_0, FN_SEL_SCIF2_CLK_1,
+		/* SEL_SCIF1 [2] */
+		FN_SEL_SCIF1_0, FN_SEL_SCIF1_1, FN_SEL_SCIF1_2, FN_SEL_SCIF1_3,
+		/* SEL_SCIF0 [2] */
+		FN_SEL_SCIF0_0, FN_SEL_SCIF0_1, FN_SEL_SCIF0_2, FN_SEL_SCIF0_3,
+		/* SEL_MSIOF2 [2] */
+		FN_SEL_MSIOF2_0, FN_SEL_MSIOF2_1, FN_SEL_MSIOF2_2, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* SEL_MSIOF1 [1] */
+		FN_SEL_MSIOF1_0, FN_SEL_MSIOF1_1,
+		/* RESERVED [1] */
+		0, 0,
+		/* SEL_MSIOF0 [1] */
+		FN_SEL_MSIOF0_0, FN_SEL_MSIOF0_1,
+		/* SEL_RCN [1] */
+		FN_SEL_RCN_0, FN_SEL_RCN_1,
+		/* RESERVED [2] */
+		0, 0, 0, 0,
+		/* SEL_TMU2 [1] */
+		FN_SEL_TMU2_0, FN_SEL_TMU2_1,
+		/* SEL_TMU1 [1] */
+		FN_SEL_TMU1_0, FN_SEL_TMU1_1,
+		/* RESERVED [2] */
+		0, 0, 0, 0,
+		/* SEL_HSCIF1 [2] */
+		FN_SEL_HSCIF1_0, FN_SEL_HSCIF1_1, FN_SEL_HSCIF1_2, 0,
+		/* SEL_HSCIF0 [1] */
+		FN_SEL_HSCIF0_0, FN_SEL_HSCIF0_1,}
+	},
+	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xE60600C8, 32,
+			     1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2) {
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* RESERVED [1] */
+		0, 0,
+		/* SEL_ADGB [2] */
+		FN_SEL_ADGB_0, FN_SEL_ADGB_1, FN_SEL_ADGB_2, 0,
+		/* SEL_ADGC [2] */
+		FN_SEL_ADGC_0, FN_SEL_ADGC_1, FN_SEL_ADGC_2, 0,
+		/* SEL_SSI9 [2] */
+		FN_SEL_SSI9_0, FN_SEL_SSI9_1, 0, 0,
+		/* SEL_SSI8 [2] */
+		FN_SEL_SSI8_0, FN_SEL_SSI8_1, 0, 0,
+		/* SEL_SSI7 [2] */
+		FN_SEL_SSI7_0, FN_SEL_SSI7_1, 0, 0,
+		/* SEL_SSI6 [2] */
+		FN_SEL_SSI6_0, FN_SEL_SSI6_1, 0, 0,
+		/* SEL_SSI5 [2] */
+		FN_SEL_SSI5_0, FN_SEL_SSI5_1, 0, 0,
+		/* SEL_SSI4 [2] */
+		FN_SEL_SSI4_0, FN_SEL_SSI4_1, 0, 0,
+		/* SEL_SSI2 [2] */
+		FN_SEL_SSI2_0, FN_SEL_SSI2_1, 0, 0,
+		/* SEL_SSI1 [2] */
+		FN_SEL_SSI1_0, FN_SEL_SSI1_1, FN_SEL_SSI1_2, FN_SEL_SSI1_3,
+		/* SEL_SSI0 [2] */
+		FN_SEL_SSI0_0, FN_SEL_SSI0_1, 0, 0, }
+	},
+	{ },
+};
+
+#ifdef CONFIG_PINCTRL_PFC_R8A77470
+const struct sh_pfc_soc_info r8a77470_pinmux_info = {
+	.name = "r8a77470_pfc",
+	.unlock_reg = 0xe6060000, /* PMMR */
+
+	.function = { PINMUX_FUNCTION_BEGIN, PINMUX_FUNCTION_END },
+
+	.pins = pinmux_pins,
+	.nr_pins = ARRAY_SIZE(pinmux_pins),
+	.groups = pinmux_groups,
+	.nr_groups = ARRAY_SIZE(pinmux_groups),
+	.functions = pinmux_functions,
+	.nr_functions = ARRAY_SIZE(pinmux_functions),
+
+	.cfg_regs = pinmux_config_regs,
+
+	.pinmux_data = pinmux_data,
+	.pinmux_data_size = ARRAY_SIZE(pinmux_data),
+};
+#endif
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7795-es1.c b/drivers/pinctrl/sh-pfc/pfc-r8a7795-es1.c
index 82a1c411c952..a6c5d50557e6 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7795-es1.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7795-es1.c
@@ -1432,10 +1432,10 @@ static const u16 pinmux_data[] = {
 
 /*
  * Static pins can not be muxed between different functions but
- * still needs a mark entry in the pinmux list. Add each static
+ * still need mark entries in the pinmux list. Add each static
  * pin to the list without an associated function. The sh-pfc
- * core will do the right thing and skip trying to mux then pin
- * while still applying configuration to it
+ * core will do the right thing and skip trying to mux the pin
+ * while still applying configuration to it.
  */
 #define FM(x)	PINMUX_DATA(x##_MARK, 0),
 	PINMUX_STATIC
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
index 7100a2dd65f8..4f55b1562ad4 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
@@ -1493,10 +1493,10 @@ static const u16 pinmux_data[] = {
 
 /*
  * Static pins can not be muxed between different functions but
- * still needs a mark entry in the pinmux list. Add each static
+ * still need mark entries in the pinmux list. Add each static
  * pin to the list without an associated function. The sh-pfc
- * core will do the right thing and skip trying to mux then pin
- * while still applying configuration to it
+ * core will do the right thing and skip trying to mux the pin
+ * while still applying configuration to it.
  */
 #define FM(x)	PINMUX_DATA(x##_MARK, 0),
 	PINMUX_STATIC
@@ -3122,7 +3122,7 @@ static const unsigned int msiof3_ss1_e_mux[] = {
 	MSIOF3_SS1_E_MARK,
 };
 static const unsigned int msiof3_ss2_e_pins[] = {
-	/* SS1 */
+	/* SS2 */
 	RCAR_GP_PIN(2, 0),
 };
 static const unsigned int msiof3_ss2_e_mux[] = {
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7796.c b/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
index 4bc5b1f820c1..3ea133cfb241 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
@@ -1499,10 +1499,10 @@ static const u16 pinmux_data[] = {
 
 /*
  * Static pins can not be muxed between different functions but
- * still needs a mark entry in the pinmux list. Add each static
+ * still need mark entries in the pinmux list. Add each static
  * pin to the list without an associated function. The sh-pfc
- * core will do the right thing and skip trying to mux then pin
- * while still applying configuration to it
+ * core will do the right thing and skip trying to mux the pin
+ * while still applying configuration to it.
  */
 #define FM(x)   PINMUX_DATA(x##_MARK, 0),
 	PINMUX_STATIC
@@ -3122,7 +3122,7 @@ static const unsigned int msiof3_ss1_e_mux[] = {
 	MSIOF3_SS1_E_MARK,
 };
 static const unsigned int msiof3_ss2_e_pins[] = {
-	/* SS1 */
+	/* SS2 */
 	RCAR_GP_PIN(2, 0),
 };
 static const unsigned int msiof3_ss2_e_mux[] = {
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77965.c b/drivers/pinctrl/sh-pfc/pfc-r8a77965.c
index cea9d0599c12..d2bbee656381 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a77965.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77965.c
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.
+// SPDX-License-Identifier: GPL-2.0
 /*
  * R8A77965 processor support - PFC hardware block.
  *
@@ -1501,10 +1501,10 @@ static const u16 pinmux_data[] = {
 
 /*
  * Static pins can not be muxed between different functions but
- * still needs a mark entry in the pinmux list. Add each static
+ * still need mark entries in the pinmux list. Add each static
  * pin to the list without an associated function. The sh-pfc
- * core will do the right thing and skip trying to mux then pin
- * while still applying configuration to it
+ * core will do the right thing and skip trying to mux the pin
+ * while still applying configuration to it.
  */
 #define FM(x)   PINMUX_DATA(x##_MARK, 0),
 	PINMUX_STATIC
@@ -1662,6 +1662,153 @@ static const unsigned int avb_avtp_capture_b_mux[] = {
 	AVB_AVTP_CAPTURE_B_MARK,
 };
 
+/* - DU --------------------------------------------------------------------- */
+static const unsigned int du_rgb666_pins[] = {
+	/* R[7:2], G[7:2], B[7:2] */
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 14), RCAR_GP_PIN(0, 13),
+	RCAR_GP_PIN(0, 12), RCAR_GP_PIN(0, 11), RCAR_GP_PIN(0, 10),
+	RCAR_GP_PIN(1, 15), RCAR_GP_PIN(1, 14), RCAR_GP_PIN(1, 13),
+	RCAR_GP_PIN(1, 12), RCAR_GP_PIN(1, 19), RCAR_GP_PIN(1, 18),
+	RCAR_GP_PIN(1, 7),  RCAR_GP_PIN(1, 6),  RCAR_GP_PIN(1, 5),
+	RCAR_GP_PIN(1, 4),  RCAR_GP_PIN(1, 3),  RCAR_GP_PIN(1, 2),
+};
+
+static const unsigned int du_rgb666_mux[] = {
+	DU_DR7_MARK, DU_DR6_MARK, DU_DR5_MARK, DU_DR4_MARK,
+	DU_DR3_MARK, DU_DR2_MARK,
+	DU_DG7_MARK, DU_DG6_MARK, DU_DG5_MARK, DU_DG4_MARK,
+	DU_DG3_MARK, DU_DG2_MARK,
+	DU_DB7_MARK, DU_DB6_MARK, DU_DB5_MARK, DU_DB4_MARK,
+	DU_DB3_MARK, DU_DB2_MARK,
+};
+
+static const unsigned int du_rgb888_pins[] = {
+	/* R[7:0], G[7:0], B[7:0] */
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 14), RCAR_GP_PIN(0, 13),
+	RCAR_GP_PIN(0, 12), RCAR_GP_PIN(0, 11), RCAR_GP_PIN(0, 10),
+	RCAR_GP_PIN(0, 9),  RCAR_GP_PIN(0, 8),
+	RCAR_GP_PIN(1, 15), RCAR_GP_PIN(1, 14), RCAR_GP_PIN(1, 13),
+	RCAR_GP_PIN(1, 12), RCAR_GP_PIN(1, 19), RCAR_GP_PIN(1, 18),
+	RCAR_GP_PIN(1, 17), RCAR_GP_PIN(1, 16),
+	RCAR_GP_PIN(1, 7),  RCAR_GP_PIN(1, 6),  RCAR_GP_PIN(1, 5),
+	RCAR_GP_PIN(1, 4),  RCAR_GP_PIN(1, 3),  RCAR_GP_PIN(1, 2),
+	RCAR_GP_PIN(1, 1),  RCAR_GP_PIN(1, 0),
+};
+
+static const unsigned int du_rgb888_mux[] = {
+	DU_DR7_MARK, DU_DR6_MARK, DU_DR5_MARK, DU_DR4_MARK,
+	DU_DR3_MARK, DU_DR2_MARK, DU_DR1_MARK, DU_DR0_MARK,
+	DU_DG7_MARK, DU_DG6_MARK, DU_DG5_MARK, DU_DG4_MARK,
+	DU_DG3_MARK, DU_DG2_MARK, DU_DG1_MARK, DU_DG0_MARK,
+	DU_DB7_MARK, DU_DB6_MARK, DU_DB5_MARK, DU_DB4_MARK,
+	DU_DB3_MARK, DU_DB2_MARK, DU_DB1_MARK, DU_DB0_MARK,
+};
+
+static const unsigned int du_clk_out_0_pins[] = {
+	/* CLKOUT */
+	RCAR_GP_PIN(1, 27),
+};
+
+static const unsigned int du_clk_out_0_mux[] = {
+	DU_DOTCLKOUT0_MARK
+};
+
+static const unsigned int du_clk_out_1_pins[] = {
+	/* CLKOUT */
+	RCAR_GP_PIN(2, 3),
+};
+
+static const unsigned int du_clk_out_1_mux[] = {
+	DU_DOTCLKOUT1_MARK
+};
+
+static const unsigned int du_sync_pins[] = {
+	/* EXVSYNC/VSYNC, EXHSYNC/HSYNC */
+	RCAR_GP_PIN(2, 5), RCAR_GP_PIN(2, 4),
+};
+
+static const unsigned int du_sync_mux[] = {
+	DU_EXVSYNC_DU_VSYNC_MARK, DU_EXHSYNC_DU_HSYNC_MARK
+};
+
+static const unsigned int du_oddf_pins[] = {
+	/* EXDISP/EXODDF/EXCDE */
+	RCAR_GP_PIN(2, 2),
+};
+
+static const unsigned int du_oddf_mux[] = {
+	DU_EXODDF_DU_ODDF_DISP_CDE_MARK,
+};
+
+static const unsigned int du_cde_pins[] = {
+	/* CDE */
+	RCAR_GP_PIN(2, 0),
+};
+
+static const unsigned int du_cde_mux[] = {
+	DU_CDE_MARK,
+};
+
+static const unsigned int du_disp_pins[] = {
+	/* DISP */
+	RCAR_GP_PIN(2, 1),
+};
+
+static const unsigned int du_disp_mux[] = {
+	DU_DISP_MARK,
+};
+
+/* - I2C -------------------------------------------------------------------- */
+static const unsigned int i2c1_a_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(5, 11), RCAR_GP_PIN(5, 10),
+};
+static const unsigned int i2c1_a_mux[] = {
+	SDA1_A_MARK, SCL1_A_MARK,
+};
+static const unsigned int i2c1_b_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(5, 24), RCAR_GP_PIN(5, 23),
+};
+static const unsigned int i2c1_b_mux[] = {
+	SDA1_B_MARK, SCL1_B_MARK,
+};
+static const unsigned int i2c2_a_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(5, 0), RCAR_GP_PIN(5, 4),
+};
+static const unsigned int i2c2_a_mux[] = {
+	SDA2_A_MARK, SCL2_A_MARK,
+};
+static const unsigned int i2c2_b_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(3, 13), RCAR_GP_PIN(3, 12),
+};
+static const unsigned int i2c2_b_mux[] = {
+	SDA2_B_MARK, SCL2_B_MARK,
+};
+static const unsigned int i2c6_a_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(1, 8), RCAR_GP_PIN(1, 11),
+};
+static const unsigned int i2c6_a_mux[] = {
+	SDA6_A_MARK, SCL6_A_MARK,
+};
+static const unsigned int i2c6_b_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(1, 26), RCAR_GP_PIN(1, 25),
+};
+static const unsigned int i2c6_b_mux[] = {
+	SDA6_B_MARK, SCL6_B_MARK,
+};
+static const unsigned int i2c6_c_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 14),
+};
+static const unsigned int i2c6_c_mux[] = {
+	SDA6_C_MARK, SCL6_C_MARK,
+};
+
 /* - INTC-EX ---------------------------------------------------------------- */
 static const unsigned int intc_ex_irq0_pins[] = {
 	/* IRQ0 */
@@ -1706,6 +1853,803 @@ static const unsigned int intc_ex_irq5_mux[] = {
 	IRQ5_MARK,
 };
 
+/* - MSIOF0 ----------------------------------------------------------------- */
+static const unsigned int msiof0_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 17),
+};
+static const unsigned int msiof0_clk_mux[] = {
+	MSIOF0_SCK_MARK,
+};
+static const unsigned int msiof0_sync_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(5, 18),
+};
+static const unsigned int msiof0_sync_mux[] = {
+	MSIOF0_SYNC_MARK,
+};
+static const unsigned int msiof0_ss1_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(5, 19),
+};
+static const unsigned int msiof0_ss1_mux[] = {
+	MSIOF0_SS1_MARK,
+};
+static const unsigned int msiof0_ss2_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(5, 21),
+};
+static const unsigned int msiof0_ss2_mux[] = {
+	MSIOF0_SS2_MARK,
+};
+static const unsigned int msiof0_txd_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(5, 20),
+};
+static const unsigned int msiof0_txd_mux[] = {
+	MSIOF0_TXD_MARK,
+};
+static const unsigned int msiof0_rxd_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(5, 22),
+};
+static const unsigned int msiof0_rxd_mux[] = {
+	MSIOF0_RXD_MARK,
+};
+/* - MSIOF1 ----------------------------------------------------------------- */
+static const unsigned int msiof1_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(6, 8),
+};
+static const unsigned int msiof1_clk_a_mux[] = {
+	MSIOF1_SCK_A_MARK,
+};
+static const unsigned int msiof1_sync_a_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(6, 9),
+};
+static const unsigned int msiof1_sync_a_mux[] = {
+	MSIOF1_SYNC_A_MARK,
+};
+static const unsigned int msiof1_ss1_a_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(6, 5),
+};
+static const unsigned int msiof1_ss1_a_mux[] = {
+	MSIOF1_SS1_A_MARK,
+};
+static const unsigned int msiof1_ss2_a_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(6, 6),
+};
+static const unsigned int msiof1_ss2_a_mux[] = {
+	MSIOF1_SS2_A_MARK,
+};
+static const unsigned int msiof1_txd_a_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(6, 7),
+};
+static const unsigned int msiof1_txd_a_mux[] = {
+	MSIOF1_TXD_A_MARK,
+};
+static const unsigned int msiof1_rxd_a_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(6, 10),
+};
+static const unsigned int msiof1_rxd_a_mux[] = {
+	MSIOF1_RXD_A_MARK,
+};
+static const unsigned int msiof1_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 9),
+};
+static const unsigned int msiof1_clk_b_mux[] = {
+	MSIOF1_SCK_B_MARK,
+};
+static const unsigned int msiof1_sync_b_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(5, 3),
+};
+static const unsigned int msiof1_sync_b_mux[] = {
+	MSIOF1_SYNC_B_MARK,
+};
+static const unsigned int msiof1_ss1_b_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(5, 4),
+};
+static const unsigned int msiof1_ss1_b_mux[] = {
+	MSIOF1_SS1_B_MARK,
+};
+static const unsigned int msiof1_ss2_b_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(5, 0),
+};
+static const unsigned int msiof1_ss2_b_mux[] = {
+	MSIOF1_SS2_B_MARK,
+};
+static const unsigned int msiof1_txd_b_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(5, 8),
+};
+static const unsigned int msiof1_txd_b_mux[] = {
+	MSIOF1_TXD_B_MARK,
+};
+static const unsigned int msiof1_rxd_b_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(5, 7),
+};
+static const unsigned int msiof1_rxd_b_mux[] = {
+	MSIOF1_RXD_B_MARK,
+};
+static const unsigned int msiof1_clk_c_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(6, 17),
+};
+static const unsigned int msiof1_clk_c_mux[] = {
+	MSIOF1_SCK_C_MARK,
+};
+static const unsigned int msiof1_sync_c_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(6, 18),
+};
+static const unsigned int msiof1_sync_c_mux[] = {
+	MSIOF1_SYNC_C_MARK,
+};
+static const unsigned int msiof1_ss1_c_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(6, 21),
+};
+static const unsigned int msiof1_ss1_c_mux[] = {
+	MSIOF1_SS1_C_MARK,
+};
+static const unsigned int msiof1_ss2_c_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(6, 27),
+};
+static const unsigned int msiof1_ss2_c_mux[] = {
+	MSIOF1_SS2_C_MARK,
+};
+static const unsigned int msiof1_txd_c_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(6, 20),
+};
+static const unsigned int msiof1_txd_c_mux[] = {
+	MSIOF1_TXD_C_MARK,
+};
+static const unsigned int msiof1_rxd_c_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(6, 19),
+};
+static const unsigned int msiof1_rxd_c_mux[] = {
+	MSIOF1_RXD_C_MARK,
+};
+static const unsigned int msiof1_clk_d_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 12),
+};
+static const unsigned int msiof1_clk_d_mux[] = {
+	MSIOF1_SCK_D_MARK,
+};
+static const unsigned int msiof1_sync_d_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(5, 15),
+};
+static const unsigned int msiof1_sync_d_mux[] = {
+	MSIOF1_SYNC_D_MARK,
+};
+static const unsigned int msiof1_ss1_d_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(5, 16),
+};
+static const unsigned int msiof1_ss1_d_mux[] = {
+	MSIOF1_SS1_D_MARK,
+};
+static const unsigned int msiof1_ss2_d_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(5, 21),
+};
+static const unsigned int msiof1_ss2_d_mux[] = {
+	MSIOF1_SS2_D_MARK,
+};
+static const unsigned int msiof1_txd_d_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(5, 14),
+};
+static const unsigned int msiof1_txd_d_mux[] = {
+	MSIOF1_TXD_D_MARK,
+};
+static const unsigned int msiof1_rxd_d_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(5, 13),
+};
+static const unsigned int msiof1_rxd_d_mux[] = {
+	MSIOF1_RXD_D_MARK,
+};
+static const unsigned int msiof1_clk_e_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(3, 0),
+};
+static const unsigned int msiof1_clk_e_mux[] = {
+	MSIOF1_SCK_E_MARK,
+};
+static const unsigned int msiof1_sync_e_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(3, 1),
+};
+static const unsigned int msiof1_sync_e_mux[] = {
+	MSIOF1_SYNC_E_MARK,
+};
+static const unsigned int msiof1_ss1_e_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(3, 4),
+};
+static const unsigned int msiof1_ss1_e_mux[] = {
+	MSIOF1_SS1_E_MARK,
+};
+static const unsigned int msiof1_ss2_e_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(3, 5),
+};
+static const unsigned int msiof1_ss2_e_mux[] = {
+	MSIOF1_SS2_E_MARK,
+};
+static const unsigned int msiof1_txd_e_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(3, 3),
+};
+static const unsigned int msiof1_txd_e_mux[] = {
+	MSIOF1_TXD_E_MARK,
+};
+static const unsigned int msiof1_rxd_e_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(3, 2),
+};
+static const unsigned int msiof1_rxd_e_mux[] = {
+	MSIOF1_RXD_E_MARK,
+};
+static const unsigned int msiof1_clk_f_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 23),
+};
+static const unsigned int msiof1_clk_f_mux[] = {
+	MSIOF1_SCK_F_MARK,
+};
+static const unsigned int msiof1_sync_f_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(5, 24),
+};
+static const unsigned int msiof1_sync_f_mux[] = {
+	MSIOF1_SYNC_F_MARK,
+};
+static const unsigned int msiof1_ss1_f_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(6, 1),
+};
+static const unsigned int msiof1_ss1_f_mux[] = {
+	MSIOF1_SS1_F_MARK,
+};
+static const unsigned int msiof1_ss2_f_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(6, 2),
+};
+static const unsigned int msiof1_ss2_f_mux[] = {
+	MSIOF1_SS2_F_MARK,
+};
+static const unsigned int msiof1_txd_f_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(6, 0),
+};
+static const unsigned int msiof1_txd_f_mux[] = {
+	MSIOF1_TXD_F_MARK,
+};
+static const unsigned int msiof1_rxd_f_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(5, 25),
+};
+static const unsigned int msiof1_rxd_f_mux[] = {
+	MSIOF1_RXD_F_MARK,
+};
+static const unsigned int msiof1_clk_g_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(3, 6),
+};
+static const unsigned int msiof1_clk_g_mux[] = {
+	MSIOF1_SCK_G_MARK,
+};
+static const unsigned int msiof1_sync_g_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(3, 7),
+};
+static const unsigned int msiof1_sync_g_mux[] = {
+	MSIOF1_SYNC_G_MARK,
+};
+static const unsigned int msiof1_ss1_g_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(3, 10),
+};
+static const unsigned int msiof1_ss1_g_mux[] = {
+	MSIOF1_SS1_G_MARK,
+};
+static const unsigned int msiof1_ss2_g_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(3, 11),
+};
+static const unsigned int msiof1_ss2_g_mux[] = {
+	MSIOF1_SS2_G_MARK,
+};
+static const unsigned int msiof1_txd_g_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(3, 9),
+};
+static const unsigned int msiof1_txd_g_mux[] = {
+	MSIOF1_TXD_G_MARK,
+};
+static const unsigned int msiof1_rxd_g_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(3, 8),
+};
+static const unsigned int msiof1_rxd_g_mux[] = {
+	MSIOF1_RXD_G_MARK,
+};
+/* - MSIOF2 ----------------------------------------------------------------- */
+static const unsigned int msiof2_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 9),
+};
+static const unsigned int msiof2_clk_a_mux[] = {
+	MSIOF2_SCK_A_MARK,
+};
+static const unsigned int msiof2_sync_a_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(1, 8),
+};
+static const unsigned int msiof2_sync_a_mux[] = {
+	MSIOF2_SYNC_A_MARK,
+};
+static const unsigned int msiof2_ss1_a_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(1, 6),
+};
+static const unsigned int msiof2_ss1_a_mux[] = {
+	MSIOF2_SS1_A_MARK,
+};
+static const unsigned int msiof2_ss2_a_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(1, 7),
+};
+static const unsigned int msiof2_ss2_a_mux[] = {
+	MSIOF2_SS2_A_MARK,
+};
+static const unsigned int msiof2_txd_a_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(1, 11),
+};
+static const unsigned int msiof2_txd_a_mux[] = {
+	MSIOF2_TXD_A_MARK,
+};
+static const unsigned int msiof2_rxd_a_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(1, 10),
+};
+static const unsigned int msiof2_rxd_a_mux[] = {
+	MSIOF2_RXD_A_MARK,
+};
+static const unsigned int msiof2_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(0, 4),
+};
+static const unsigned int msiof2_clk_b_mux[] = {
+	MSIOF2_SCK_B_MARK,
+};
+static const unsigned int msiof2_sync_b_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(0, 5),
+};
+static const unsigned int msiof2_sync_b_mux[] = {
+	MSIOF2_SYNC_B_MARK,
+};
+static const unsigned int msiof2_ss1_b_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(0, 0),
+};
+static const unsigned int msiof2_ss1_b_mux[] = {
+	MSIOF2_SS1_B_MARK,
+};
+static const unsigned int msiof2_ss2_b_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(0, 1),
+};
+static const unsigned int msiof2_ss2_b_mux[] = {
+	MSIOF2_SS2_B_MARK,
+};
+static const unsigned int msiof2_txd_b_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(0, 7),
+};
+static const unsigned int msiof2_txd_b_mux[] = {
+	MSIOF2_TXD_B_MARK,
+};
+static const unsigned int msiof2_rxd_b_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(0, 6),
+};
+static const unsigned int msiof2_rxd_b_mux[] = {
+	MSIOF2_RXD_B_MARK,
+};
+static const unsigned int msiof2_clk_c_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(2, 12),
+};
+static const unsigned int msiof2_clk_c_mux[] = {
+	MSIOF2_SCK_C_MARK,
+};
+static const unsigned int msiof2_sync_c_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(2, 11),
+};
+static const unsigned int msiof2_sync_c_mux[] = {
+	MSIOF2_SYNC_C_MARK,
+};
+static const unsigned int msiof2_ss1_c_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(2, 10),
+};
+static const unsigned int msiof2_ss1_c_mux[] = {
+	MSIOF2_SS1_C_MARK,
+};
+static const unsigned int msiof2_ss2_c_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(2, 9),
+};
+static const unsigned int msiof2_ss2_c_mux[] = {
+	MSIOF2_SS2_C_MARK,
+};
+static const unsigned int msiof2_txd_c_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(2, 14),
+};
+static const unsigned int msiof2_txd_c_mux[] = {
+	MSIOF2_TXD_C_MARK,
+};
+static const unsigned int msiof2_rxd_c_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(2, 13),
+};
+static const unsigned int msiof2_rxd_c_mux[] = {
+	MSIOF2_RXD_C_MARK,
+};
+static const unsigned int msiof2_clk_d_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(0, 8),
+};
+static const unsigned int msiof2_clk_d_mux[] = {
+	MSIOF2_SCK_D_MARK,
+};
+static const unsigned int msiof2_sync_d_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(0, 9),
+};
+static const unsigned int msiof2_sync_d_mux[] = {
+	MSIOF2_SYNC_D_MARK,
+};
+static const unsigned int msiof2_ss1_d_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(0, 12),
+};
+static const unsigned int msiof2_ss1_d_mux[] = {
+	MSIOF2_SS1_D_MARK,
+};
+static const unsigned int msiof2_ss2_d_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(0, 13),
+};
+static const unsigned int msiof2_ss2_d_mux[] = {
+	MSIOF2_SS2_D_MARK,
+};
+static const unsigned int msiof2_txd_d_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(0, 11),
+};
+static const unsigned int msiof2_txd_d_mux[] = {
+	MSIOF2_TXD_D_MARK,
+};
+static const unsigned int msiof2_rxd_d_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(0, 10),
+};
+static const unsigned int msiof2_rxd_d_mux[] = {
+	MSIOF2_RXD_D_MARK,
+};
+/* - MSIOF3 ----------------------------------------------------------------- */
+static const unsigned int msiof3_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(0, 0),
+};
+static const unsigned int msiof3_clk_a_mux[] = {
+	MSIOF3_SCK_A_MARK,
+};
+static const unsigned int msiof3_sync_a_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(0, 1),
+};
+static const unsigned int msiof3_sync_a_mux[] = {
+	MSIOF3_SYNC_A_MARK,
+};
+static const unsigned int msiof3_ss1_a_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(0, 14),
+};
+static const unsigned int msiof3_ss1_a_mux[] = {
+	MSIOF3_SS1_A_MARK,
+};
+static const unsigned int msiof3_ss2_a_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(0, 15),
+};
+static const unsigned int msiof3_ss2_a_mux[] = {
+	MSIOF3_SS2_A_MARK,
+};
+static const unsigned int msiof3_txd_a_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(0, 3),
+};
+static const unsigned int msiof3_txd_a_mux[] = {
+	MSIOF3_TXD_A_MARK,
+};
+static const unsigned int msiof3_rxd_a_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(0, 2),
+};
+static const unsigned int msiof3_rxd_a_mux[] = {
+	MSIOF3_RXD_A_MARK,
+};
+static const unsigned int msiof3_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 2),
+};
+static const unsigned int msiof3_clk_b_mux[] = {
+	MSIOF3_SCK_B_MARK,
+};
+static const unsigned int msiof3_sync_b_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(1, 0),
+};
+static const unsigned int msiof3_sync_b_mux[] = {
+	MSIOF3_SYNC_B_MARK,
+};
+static const unsigned int msiof3_ss1_b_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(1, 4),
+};
+static const unsigned int msiof3_ss1_b_mux[] = {
+	MSIOF3_SS1_B_MARK,
+};
+static const unsigned int msiof3_ss2_b_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(1, 5),
+};
+static const unsigned int msiof3_ss2_b_mux[] = {
+	MSIOF3_SS2_B_MARK,
+};
+static const unsigned int msiof3_txd_b_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(1, 1),
+};
+static const unsigned int msiof3_txd_b_mux[] = {
+	MSIOF3_TXD_B_MARK,
+};
+static const unsigned int msiof3_rxd_b_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(1, 3),
+};
+static const unsigned int msiof3_rxd_b_mux[] = {
+	MSIOF3_RXD_B_MARK,
+};
+static const unsigned int msiof3_clk_c_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 12),
+};
+static const unsigned int msiof3_clk_c_mux[] = {
+	MSIOF3_SCK_C_MARK,
+};
+static const unsigned int msiof3_sync_c_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(1, 13),
+};
+static const unsigned int msiof3_sync_c_mux[] = {
+	MSIOF3_SYNC_C_MARK,
+};
+static const unsigned int msiof3_txd_c_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(1, 15),
+};
+static const unsigned int msiof3_txd_c_mux[] = {
+	MSIOF3_TXD_C_MARK,
+};
+static const unsigned int msiof3_rxd_c_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(1, 14),
+};
+static const unsigned int msiof3_rxd_c_mux[] = {
+	MSIOF3_RXD_C_MARK,
+};
+static const unsigned int msiof3_clk_d_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 22),
+};
+static const unsigned int msiof3_clk_d_mux[] = {
+	MSIOF3_SCK_D_MARK,
+};
+static const unsigned int msiof3_sync_d_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(1, 23),
+};
+static const unsigned int msiof3_sync_d_mux[] = {
+	MSIOF3_SYNC_D_MARK,
+};
+static const unsigned int msiof3_ss1_d_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(1, 26),
+};
+static const unsigned int msiof3_ss1_d_mux[] = {
+	MSIOF3_SS1_D_MARK,
+};
+static const unsigned int msiof3_txd_d_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(1, 25),
+};
+static const unsigned int msiof3_txd_d_mux[] = {
+	MSIOF3_TXD_D_MARK,
+};
+static const unsigned int msiof3_rxd_d_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(1, 24),
+};
+static const unsigned int msiof3_rxd_d_mux[] = {
+	MSIOF3_RXD_D_MARK,
+};
+static const unsigned int msiof3_clk_e_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(2, 3),
+};
+static const unsigned int msiof3_clk_e_mux[] = {
+	MSIOF3_SCK_E_MARK,
+};
+static const unsigned int msiof3_sync_e_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(2, 2),
+};
+static const unsigned int msiof3_sync_e_mux[] = {
+	MSIOF3_SYNC_E_MARK,
+};
+static const unsigned int msiof3_ss1_e_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(2, 1),
+};
+static const unsigned int msiof3_ss1_e_mux[] = {
+	MSIOF3_SS1_E_MARK,
+};
+static const unsigned int msiof3_ss2_e_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(2, 0),
+};
+static const unsigned int msiof3_ss2_e_mux[] = {
+	MSIOF3_SS2_E_MARK,
+};
+static const unsigned int msiof3_txd_e_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(2, 5),
+};
+static const unsigned int msiof3_txd_e_mux[] = {
+	MSIOF3_TXD_E_MARK,
+};
+static const unsigned int msiof3_rxd_e_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(2, 4),
+};
+static const unsigned int msiof3_rxd_e_mux[] = {
+	MSIOF3_RXD_E_MARK,
+};
+
+/* - PWM0 --------------------------------------------------------------------*/
+static const unsigned int pwm0_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(2, 6),
+};
+static const unsigned int pwm0_mux[] = {
+	PWM0_MARK,
+};
+/* - PWM1 --------------------------------------------------------------------*/
+static const unsigned int pwm1_a_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(2, 7),
+};
+static const unsigned int pwm1_a_mux[] = {
+	PWM1_A_MARK,
+};
+static const unsigned int pwm1_b_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(1, 8),
+};
+static const unsigned int pwm1_b_mux[] = {
+	PWM1_B_MARK,
+};
+/* - PWM2 --------------------------------------------------------------------*/
+static const unsigned int pwm2_a_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(2, 8),
+};
+static const unsigned int pwm2_a_mux[] = {
+	PWM2_A_MARK,
+};
+static const unsigned int pwm2_b_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(1, 11),
+};
+static const unsigned int pwm2_b_mux[] = {
+	PWM2_B_MARK,
+};
+/* - PWM3 --------------------------------------------------------------------*/
+static const unsigned int pwm3_a_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(1, 0),
+};
+static const unsigned int pwm3_a_mux[] = {
+	PWM3_A_MARK,
+};
+static const unsigned int pwm3_b_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(2, 2),
+};
+static const unsigned int pwm3_b_mux[] = {
+	PWM3_B_MARK,
+};
+/* - PWM4 --------------------------------------------------------------------*/
+static const unsigned int pwm4_a_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(1, 1),
+};
+static const unsigned int pwm4_a_mux[] = {
+	PWM4_A_MARK,
+};
+static const unsigned int pwm4_b_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(2, 3),
+};
+static const unsigned int pwm4_b_mux[] = {
+	PWM4_B_MARK,
+};
+/* - PWM5 --------------------------------------------------------------------*/
+static const unsigned int pwm5_a_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(1, 2),
+};
+static const unsigned int pwm5_a_mux[] = {
+	PWM5_A_MARK,
+};
+static const unsigned int pwm5_b_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(2, 4),
+};
+static const unsigned int pwm5_b_mux[] = {
+	PWM5_B_MARK,
+};
+/* - PWM6 --------------------------------------------------------------------*/
+static const unsigned int pwm6_a_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(1, 3),
+};
+static const unsigned int pwm6_a_mux[] = {
+	PWM6_A_MARK,
+};
+static const unsigned int pwm6_b_pins[] = {
+	/* PWM */
+	RCAR_GP_PIN(2, 5),
+};
+static const unsigned int pwm6_b_mux[] = {
+	PWM6_B_MARK,
+};
+
 /* - SCIF0 ------------------------------------------------------------------ */
 static const unsigned int scif0_data_pins[] = {
 	/* RX, TX */
@@ -1917,6 +2861,264 @@ static const unsigned int scif_clk_b_mux[] = {
 	SCIF_CLK_B_MARK,
 };
 
+/* - SDHI0 ------------------------------------------------------------------ */
+static const unsigned int sdhi0_data1_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(3, 2),
+};
+
+static const unsigned int sdhi0_data1_mux[] = {
+	SD0_DAT0_MARK,
+};
+
+static const unsigned int sdhi0_data4_pins[] = {
+	/* D[0:3] */
+	RCAR_GP_PIN(3, 2), RCAR_GP_PIN(3, 3),
+	RCAR_GP_PIN(3, 4), RCAR_GP_PIN(3, 5),
+};
+
+static const unsigned int sdhi0_data4_mux[] = {
+	SD0_DAT0_MARK, SD0_DAT1_MARK,
+	SD0_DAT2_MARK, SD0_DAT3_MARK,
+};
+
+static const unsigned int sdhi0_ctrl_pins[] = {
+	/* CLK, CMD */
+	RCAR_GP_PIN(3, 0), RCAR_GP_PIN(3, 1),
+};
+
+static const unsigned int sdhi0_ctrl_mux[] = {
+	SD0_CLK_MARK, SD0_CMD_MARK,
+};
+
+static const unsigned int sdhi0_cd_pins[] = {
+	/* CD */
+	RCAR_GP_PIN(3, 12),
+};
+
+static const unsigned int sdhi0_cd_mux[] = {
+	SD0_CD_MARK,
+};
+
+static const unsigned int sdhi0_wp_pins[] = {
+	/* WP */
+	RCAR_GP_PIN(3, 13),
+};
+
+static const unsigned int sdhi0_wp_mux[] = {
+	SD0_WP_MARK,
+};
+
+/* - SDHI1 ------------------------------------------------------------------ */
+static const unsigned int sdhi1_data1_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(3, 8),
+};
+
+static const unsigned int sdhi1_data1_mux[] = {
+	SD1_DAT0_MARK,
+};
+
+static const unsigned int sdhi1_data4_pins[] = {
+	/* D[0:3] */
+	RCAR_GP_PIN(3, 8),  RCAR_GP_PIN(3, 9),
+	RCAR_GP_PIN(3, 10), RCAR_GP_PIN(3, 11),
+};
+
+static const unsigned int sdhi1_data4_mux[] = {
+	SD1_DAT0_MARK, SD1_DAT1_MARK,
+	SD1_DAT2_MARK, SD1_DAT3_MARK,
+};
+
+static const unsigned int sdhi1_ctrl_pins[] = {
+	/* CLK, CMD */
+	RCAR_GP_PIN(3, 6), RCAR_GP_PIN(3, 7),
+};
+
+static const unsigned int sdhi1_ctrl_mux[] = {
+	SD1_CLK_MARK, SD1_CMD_MARK,
+};
+
+static const unsigned int sdhi1_cd_pins[] = {
+	/* CD */
+	RCAR_GP_PIN(3, 14),
+};
+
+static const unsigned int sdhi1_cd_mux[] = {
+	SD1_CD_MARK,
+};
+
+static const unsigned int sdhi1_wp_pins[] = {
+	/* WP */
+	RCAR_GP_PIN(3, 15),
+};
+
+static const unsigned int sdhi1_wp_mux[] = {
+	SD1_WP_MARK,
+};
+
+/* - SDHI2 ------------------------------------------------------------------ */
+static const unsigned int sdhi2_data1_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(4, 2),
+};
+
+static const unsigned int sdhi2_data1_mux[] = {
+	SD2_DAT0_MARK,
+};
+
+static const unsigned int sdhi2_data4_pins[] = {
+	/* D[0:3] */
+	RCAR_GP_PIN(4, 2), RCAR_GP_PIN(4, 3),
+	RCAR_GP_PIN(4, 4), RCAR_GP_PIN(4, 5),
+};
+
+static const unsigned int sdhi2_data4_mux[] = {
+	SD2_DAT0_MARK, SD2_DAT1_MARK,
+	SD2_DAT2_MARK, SD2_DAT3_MARK,
+};
+
+static const unsigned int sdhi2_data8_pins[] = {
+	/* D[0:7] */
+	RCAR_GP_PIN(4, 2),  RCAR_GP_PIN(4, 3),
+	RCAR_GP_PIN(4, 4),  RCAR_GP_PIN(4, 5),
+	RCAR_GP_PIN(3, 8),  RCAR_GP_PIN(3, 9),
+	RCAR_GP_PIN(3, 10), RCAR_GP_PIN(3, 11),
+};
+
+static const unsigned int sdhi2_data8_mux[] = {
+	SD2_DAT0_MARK, SD2_DAT1_MARK,
+	SD2_DAT2_MARK, SD2_DAT3_MARK,
+	SD2_DAT4_MARK, SD2_DAT5_MARK,
+	SD2_DAT6_MARK, SD2_DAT7_MARK,
+};
+
+static const unsigned int sdhi2_ctrl_pins[] = {
+	/* CLK, CMD */
+	RCAR_GP_PIN(4, 0), RCAR_GP_PIN(4, 1),
+};
+
+static const unsigned int sdhi2_ctrl_mux[] = {
+	SD2_CLK_MARK, SD2_CMD_MARK,
+};
+
+static const unsigned int sdhi2_cd_a_pins[] = {
+	/* CD */
+	RCAR_GP_PIN(4, 13),
+};
+
+static const unsigned int sdhi2_cd_a_mux[] = {
+	SD2_CD_A_MARK,
+};
+
+static const unsigned int sdhi2_cd_b_pins[] = {
+	/* CD */
+	RCAR_GP_PIN(5, 10),
+};
+
+static const unsigned int sdhi2_cd_b_mux[] = {
+	SD2_CD_B_MARK,
+};
+
+static const unsigned int sdhi2_wp_a_pins[] = {
+	/* WP */
+	RCAR_GP_PIN(4, 14),
+};
+
+static const unsigned int sdhi2_wp_a_mux[] = {
+	SD2_WP_A_MARK,
+};
+
+static const unsigned int sdhi2_wp_b_pins[] = {
+	/* WP */
+	RCAR_GP_PIN(5, 11),
+};
+
+static const unsigned int sdhi2_wp_b_mux[] = {
+	SD2_WP_B_MARK,
+};
+
+static const unsigned int sdhi2_ds_pins[] = {
+	/* DS */
+	RCAR_GP_PIN(4, 6),
+};
+
+static const unsigned int sdhi2_ds_mux[] = {
+	SD2_DS_MARK,
+};
+
+/* - SDHI3 ------------------------------------------------------------------ */
+static const unsigned int sdhi3_data1_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(4, 9),
+};
+
+static const unsigned int sdhi3_data1_mux[] = {
+	SD3_DAT0_MARK,
+};
+
+static const unsigned int sdhi3_data4_pins[] = {
+	/* D[0:3] */
+	RCAR_GP_PIN(4, 9),  RCAR_GP_PIN(4, 10),
+	RCAR_GP_PIN(4, 11), RCAR_GP_PIN(4, 12),
+};
+
+static const unsigned int sdhi3_data4_mux[] = {
+	SD3_DAT0_MARK, SD3_DAT1_MARK,
+	SD3_DAT2_MARK, SD3_DAT3_MARK,
+};
+
+static const unsigned int sdhi3_data8_pins[] = {
+	/* D[0:7] */
+	RCAR_GP_PIN(4, 9),  RCAR_GP_PIN(4, 10),
+	RCAR_GP_PIN(4, 11), RCAR_GP_PIN(4, 12),
+	RCAR_GP_PIN(4, 13), RCAR_GP_PIN(4, 14),
+	RCAR_GP_PIN(4, 15), RCAR_GP_PIN(4, 16),
+};
+
+static const unsigned int sdhi3_data8_mux[] = {
+	SD3_DAT0_MARK, SD3_DAT1_MARK,
+	SD3_DAT2_MARK, SD3_DAT3_MARK,
+	SD3_DAT4_MARK, SD3_DAT5_MARK,
+	SD3_DAT6_MARK, SD3_DAT7_MARK,
+};
+
+static const unsigned int sdhi3_ctrl_pins[] = {
+	/* CLK, CMD */
+	RCAR_GP_PIN(4, 7), RCAR_GP_PIN(4, 8),
+};
+
+static const unsigned int sdhi3_ctrl_mux[] = {
+	SD3_CLK_MARK, SD3_CMD_MARK,
+};
+
+static const unsigned int sdhi3_cd_pins[] = {
+	/* CD */
+	RCAR_GP_PIN(4, 15),
+};
+
+static const unsigned int sdhi3_cd_mux[] = {
+	SD3_CD_MARK,
+};
+
+static const unsigned int sdhi3_wp_pins[] = {
+	/* WP */
+	RCAR_GP_PIN(4, 16),
+};
+
+static const unsigned int sdhi3_wp_mux[] = {
+	SD3_WP_MARK,
+};
+
+static const unsigned int sdhi3_ds_pins[] = {
+	/* DS */
+	RCAR_GP_PIN(4, 17),
+};
+
+static const unsigned int sdhi3_ds_mux[] = {
+	SD3_DS_MARK,
+};
+
 /* - USB0 ------------------------------------------------------------------- */
 static const unsigned int usb0_pins[] = {
 	/* PWEN, OVC */
@@ -1959,12 +3161,139 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(avb_avtp_capture_a),
 	SH_PFC_PIN_GROUP(avb_avtp_match_b),
 	SH_PFC_PIN_GROUP(avb_avtp_capture_b),
+	SH_PFC_PIN_GROUP(du_rgb666),
+	SH_PFC_PIN_GROUP(du_rgb888),
+	SH_PFC_PIN_GROUP(du_clk_out_0),
+	SH_PFC_PIN_GROUP(du_clk_out_1),
+	SH_PFC_PIN_GROUP(du_sync),
+	SH_PFC_PIN_GROUP(du_oddf),
+	SH_PFC_PIN_GROUP(du_cde),
+	SH_PFC_PIN_GROUP(du_disp),
+	SH_PFC_PIN_GROUP(i2c1_a),
+	SH_PFC_PIN_GROUP(i2c1_b),
+	SH_PFC_PIN_GROUP(i2c2_a),
+	SH_PFC_PIN_GROUP(i2c2_b),
+	SH_PFC_PIN_GROUP(i2c6_a),
+	SH_PFC_PIN_GROUP(i2c6_b),
+	SH_PFC_PIN_GROUP(i2c6_c),
 	SH_PFC_PIN_GROUP(intc_ex_irq0),
 	SH_PFC_PIN_GROUP(intc_ex_irq1),
 	SH_PFC_PIN_GROUP(intc_ex_irq2),
 	SH_PFC_PIN_GROUP(intc_ex_irq3),
 	SH_PFC_PIN_GROUP(intc_ex_irq4),
 	SH_PFC_PIN_GROUP(intc_ex_irq5),
+	SH_PFC_PIN_GROUP(msiof0_clk),
+	SH_PFC_PIN_GROUP(msiof0_sync),
+	SH_PFC_PIN_GROUP(msiof0_ss1),
+	SH_PFC_PIN_GROUP(msiof0_ss2),
+	SH_PFC_PIN_GROUP(msiof0_txd),
+	SH_PFC_PIN_GROUP(msiof0_rxd),
+	SH_PFC_PIN_GROUP(msiof1_clk_a),
+	SH_PFC_PIN_GROUP(msiof1_sync_a),
+	SH_PFC_PIN_GROUP(msiof1_ss1_a),
+	SH_PFC_PIN_GROUP(msiof1_ss2_a),
+	SH_PFC_PIN_GROUP(msiof1_txd_a),
+	SH_PFC_PIN_GROUP(msiof1_rxd_a),
+	SH_PFC_PIN_GROUP(msiof1_clk_b),
+	SH_PFC_PIN_GROUP(msiof1_sync_b),
+	SH_PFC_PIN_GROUP(msiof1_ss1_b),
+	SH_PFC_PIN_GROUP(msiof1_ss2_b),
+	SH_PFC_PIN_GROUP(msiof1_txd_b),
+	SH_PFC_PIN_GROUP(msiof1_rxd_b),
+	SH_PFC_PIN_GROUP(msiof1_clk_c),
+	SH_PFC_PIN_GROUP(msiof1_sync_c),
+	SH_PFC_PIN_GROUP(msiof1_ss1_c),
+	SH_PFC_PIN_GROUP(msiof1_ss2_c),
+	SH_PFC_PIN_GROUP(msiof1_txd_c),
+	SH_PFC_PIN_GROUP(msiof1_rxd_c),
+	SH_PFC_PIN_GROUP(msiof1_clk_d),
+	SH_PFC_PIN_GROUP(msiof1_sync_d),
+	SH_PFC_PIN_GROUP(msiof1_ss1_d),
+	SH_PFC_PIN_GROUP(msiof1_ss2_d),
+	SH_PFC_PIN_GROUP(msiof1_txd_d),
+	SH_PFC_PIN_GROUP(msiof1_rxd_d),
+	SH_PFC_PIN_GROUP(msiof1_clk_e),
+	SH_PFC_PIN_GROUP(msiof1_sync_e),
+	SH_PFC_PIN_GROUP(msiof1_ss1_e),
+	SH_PFC_PIN_GROUP(msiof1_ss2_e),
+	SH_PFC_PIN_GROUP(msiof1_txd_e),
+	SH_PFC_PIN_GROUP(msiof1_rxd_e),
+	SH_PFC_PIN_GROUP(msiof1_clk_f),
+	SH_PFC_PIN_GROUP(msiof1_sync_f),
+	SH_PFC_PIN_GROUP(msiof1_ss1_f),
+	SH_PFC_PIN_GROUP(msiof1_ss2_f),
+	SH_PFC_PIN_GROUP(msiof1_txd_f),
+	SH_PFC_PIN_GROUP(msiof1_rxd_f),
+	SH_PFC_PIN_GROUP(msiof1_clk_g),
+	SH_PFC_PIN_GROUP(msiof1_sync_g),
+	SH_PFC_PIN_GROUP(msiof1_ss1_g),
+	SH_PFC_PIN_GROUP(msiof1_ss2_g),
+	SH_PFC_PIN_GROUP(msiof1_txd_g),
+	SH_PFC_PIN_GROUP(msiof1_rxd_g),
+	SH_PFC_PIN_GROUP(msiof2_clk_a),
+	SH_PFC_PIN_GROUP(msiof2_sync_a),
+	SH_PFC_PIN_GROUP(msiof2_ss1_a),
+	SH_PFC_PIN_GROUP(msiof2_ss2_a),
+	SH_PFC_PIN_GROUP(msiof2_txd_a),
+	SH_PFC_PIN_GROUP(msiof2_rxd_a),
+	SH_PFC_PIN_GROUP(msiof2_clk_b),
+	SH_PFC_PIN_GROUP(msiof2_sync_b),
+	SH_PFC_PIN_GROUP(msiof2_ss1_b),
+	SH_PFC_PIN_GROUP(msiof2_ss2_b),
+	SH_PFC_PIN_GROUP(msiof2_txd_b),
+	SH_PFC_PIN_GROUP(msiof2_rxd_b),
+	SH_PFC_PIN_GROUP(msiof2_clk_c),
+	SH_PFC_PIN_GROUP(msiof2_sync_c),
+	SH_PFC_PIN_GROUP(msiof2_ss1_c),
+	SH_PFC_PIN_GROUP(msiof2_ss2_c),
+	SH_PFC_PIN_GROUP(msiof2_txd_c),
+	SH_PFC_PIN_GROUP(msiof2_rxd_c),
+	SH_PFC_PIN_GROUP(msiof2_clk_d),
+	SH_PFC_PIN_GROUP(msiof2_sync_d),
+	SH_PFC_PIN_GROUP(msiof2_ss1_d),
+	SH_PFC_PIN_GROUP(msiof2_ss2_d),
+	SH_PFC_PIN_GROUP(msiof2_txd_d),
+	SH_PFC_PIN_GROUP(msiof2_rxd_d),
+	SH_PFC_PIN_GROUP(msiof3_clk_a),
+	SH_PFC_PIN_GROUP(msiof3_sync_a),
+	SH_PFC_PIN_GROUP(msiof3_ss1_a),
+	SH_PFC_PIN_GROUP(msiof3_ss2_a),
+	SH_PFC_PIN_GROUP(msiof3_txd_a),
+	SH_PFC_PIN_GROUP(msiof3_rxd_a),
+	SH_PFC_PIN_GROUP(msiof3_clk_b),
+	SH_PFC_PIN_GROUP(msiof3_sync_b),
+	SH_PFC_PIN_GROUP(msiof3_ss1_b),
+	SH_PFC_PIN_GROUP(msiof3_ss2_b),
+	SH_PFC_PIN_GROUP(msiof3_txd_b),
+	SH_PFC_PIN_GROUP(msiof3_rxd_b),
+	SH_PFC_PIN_GROUP(msiof3_clk_c),
+	SH_PFC_PIN_GROUP(msiof3_sync_c),
+	SH_PFC_PIN_GROUP(msiof3_txd_c),
+	SH_PFC_PIN_GROUP(msiof3_rxd_c),
+	SH_PFC_PIN_GROUP(msiof3_clk_d),
+	SH_PFC_PIN_GROUP(msiof3_sync_d),
+	SH_PFC_PIN_GROUP(msiof3_ss1_d),
+	SH_PFC_PIN_GROUP(msiof3_txd_d),
+	SH_PFC_PIN_GROUP(msiof3_rxd_d),
+	SH_PFC_PIN_GROUP(msiof3_clk_e),
+	SH_PFC_PIN_GROUP(msiof3_sync_e),
+	SH_PFC_PIN_GROUP(msiof3_ss1_e),
+	SH_PFC_PIN_GROUP(msiof3_ss2_e),
+	SH_PFC_PIN_GROUP(msiof3_txd_e),
+	SH_PFC_PIN_GROUP(msiof3_rxd_e),
+	SH_PFC_PIN_GROUP(pwm0),
+	SH_PFC_PIN_GROUP(pwm1_a),
+	SH_PFC_PIN_GROUP(pwm1_b),
+	SH_PFC_PIN_GROUP(pwm2_a),
+	SH_PFC_PIN_GROUP(pwm2_b),
+	SH_PFC_PIN_GROUP(pwm3_a),
+	SH_PFC_PIN_GROUP(pwm3_b),
+	SH_PFC_PIN_GROUP(pwm4_a),
+	SH_PFC_PIN_GROUP(pwm4_b),
+	SH_PFC_PIN_GROUP(pwm5_a),
+	SH_PFC_PIN_GROUP(pwm5_b),
+	SH_PFC_PIN_GROUP(pwm6_a),
+	SH_PFC_PIN_GROUP(pwm6_b),
 	SH_PFC_PIN_GROUP(scif0_data),
 	SH_PFC_PIN_GROUP(scif0_clk),
 	SH_PFC_PIN_GROUP(scif0_ctrl),
@@ -1994,6 +3323,32 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(scif5_clk_b),
 	SH_PFC_PIN_GROUP(scif_clk_a),
 	SH_PFC_PIN_GROUP(scif_clk_b),
+	SH_PFC_PIN_GROUP(sdhi0_data1),
+	SH_PFC_PIN_GROUP(sdhi0_data4),
+	SH_PFC_PIN_GROUP(sdhi0_ctrl),
+	SH_PFC_PIN_GROUP(sdhi0_cd),
+	SH_PFC_PIN_GROUP(sdhi0_wp),
+	SH_PFC_PIN_GROUP(sdhi1_data1),
+	SH_PFC_PIN_GROUP(sdhi1_data4),
+	SH_PFC_PIN_GROUP(sdhi1_ctrl),
+	SH_PFC_PIN_GROUP(sdhi1_cd),
+	SH_PFC_PIN_GROUP(sdhi1_wp),
+	SH_PFC_PIN_GROUP(sdhi2_data1),
+	SH_PFC_PIN_GROUP(sdhi2_data4),
+	SH_PFC_PIN_GROUP(sdhi2_data8),
+	SH_PFC_PIN_GROUP(sdhi2_ctrl),
+	SH_PFC_PIN_GROUP(sdhi2_cd_a),
+	SH_PFC_PIN_GROUP(sdhi2_wp_a),
+	SH_PFC_PIN_GROUP(sdhi2_cd_b),
+	SH_PFC_PIN_GROUP(sdhi2_wp_b),
+	SH_PFC_PIN_GROUP(sdhi2_ds),
+	SH_PFC_PIN_GROUP(sdhi3_data1),
+	SH_PFC_PIN_GROUP(sdhi3_data4),
+	SH_PFC_PIN_GROUP(sdhi3_data8),
+	SH_PFC_PIN_GROUP(sdhi3_ctrl),
+	SH_PFC_PIN_GROUP(sdhi3_cd),
+	SH_PFC_PIN_GROUP(sdhi3_wp),
+	SH_PFC_PIN_GROUP(sdhi3_ds),
 	SH_PFC_PIN_GROUP(usb0),
 	SH_PFC_PIN_GROUP(usb1),
 	SH_PFC_PIN_GROUP(usb30),
@@ -2013,6 +3368,33 @@ static const char * const avb_groups[] = {
 	"avb_avtp_capture_b",
 };
 
+static const char * const du_groups[] = {
+	"du_rgb666",
+	"du_rgb888",
+	"du_clk_out_0",
+	"du_clk_out_1",
+	"du_sync",
+	"du_oddf",
+	"du_cde",
+	"du_disp",
+};
+
+static const char * const i2c1_groups[] = {
+	"i2c1_a",
+	"i2c1_b",
+};
+
+static const char * const i2c2_groups[] = {
+	"i2c2_a",
+	"i2c2_b",
+};
+
+static const char * const i2c6_groups[] = {
+	"i2c6_a",
+	"i2c6_b",
+	"i2c6_c",
+};
+
 static const char * const intc_ex_groups[] = {
 	"intc_ex_irq0",
 	"intc_ex_irq1",
@@ -2022,6 +3404,151 @@ static const char * const intc_ex_groups[] = {
 	"intc_ex_irq5",
 };
 
+static const char * const msiof0_groups[] = {
+	"msiof0_clk",
+	"msiof0_sync",
+	"msiof0_ss1",
+	"msiof0_ss2",
+	"msiof0_txd",
+	"msiof0_rxd",
+};
+
+static const char * const msiof1_groups[] = {
+	"msiof1_clk_a",
+	"msiof1_sync_a",
+	"msiof1_ss1_a",
+	"msiof1_ss2_a",
+	"msiof1_txd_a",
+	"msiof1_rxd_a",
+	"msiof1_clk_b",
+	"msiof1_sync_b",
+	"msiof1_ss1_b",
+	"msiof1_ss2_b",
+	"msiof1_txd_b",
+	"msiof1_rxd_b",
+	"msiof1_clk_c",
+	"msiof1_sync_c",
+	"msiof1_ss1_c",
+	"msiof1_ss2_c",
+	"msiof1_txd_c",
+	"msiof1_rxd_c",
+	"msiof1_clk_d",
+	"msiof1_sync_d",
+	"msiof1_ss1_d",
+	"msiof1_ss2_d",
+	"msiof1_txd_d",
+	"msiof1_rxd_d",
+	"msiof1_clk_e",
+	"msiof1_sync_e",
+	"msiof1_ss1_e",
+	"msiof1_ss2_e",
+	"msiof1_txd_e",
+	"msiof1_rxd_e",
+	"msiof1_clk_f",
+	"msiof1_sync_f",
+	"msiof1_ss1_f",
+	"msiof1_ss2_f",
+	"msiof1_txd_f",
+	"msiof1_rxd_f",
+	"msiof1_clk_g",
+	"msiof1_sync_g",
+	"msiof1_ss1_g",
+	"msiof1_ss2_g",
+	"msiof1_txd_g",
+	"msiof1_rxd_g",
+};
+
+static const char * const msiof2_groups[] = {
+	"msiof2_clk_a",
+	"msiof2_sync_a",
+	"msiof2_ss1_a",
+	"msiof2_ss2_a",
+	"msiof2_txd_a",
+	"msiof2_rxd_a",
+	"msiof2_clk_b",
+	"msiof2_sync_b",
+	"msiof2_ss1_b",
+	"msiof2_ss2_b",
+	"msiof2_txd_b",
+	"msiof2_rxd_b",
+	"msiof2_clk_c",
+	"msiof2_sync_c",
+	"msiof2_ss1_c",
+	"msiof2_ss2_c",
+	"msiof2_txd_c",
+	"msiof2_rxd_c",
+	"msiof2_clk_d",
+	"msiof2_sync_d",
+	"msiof2_ss1_d",
+	"msiof2_ss2_d",
+	"msiof2_txd_d",
+	"msiof2_rxd_d",
+};
+
+static const char * const msiof3_groups[] = {
+	"msiof3_clk_a",
+	"msiof3_sync_a",
+	"msiof3_ss1_a",
+	"msiof3_ss2_a",
+	"msiof3_txd_a",
+	"msiof3_rxd_a",
+	"msiof3_clk_b",
+	"msiof3_sync_b",
+	"msiof3_ss1_b",
+	"msiof3_ss2_b",
+	"msiof3_txd_b",
+	"msiof3_rxd_b",
+	"msiof3_clk_c",
+	"msiof3_sync_c",
+	"msiof3_txd_c",
+	"msiof3_rxd_c",
+	"msiof3_clk_d",
+	"msiof3_sync_d",
+	"msiof3_ss1_d",
+	"msiof3_txd_d",
+	"msiof3_rxd_d",
+	"msiof3_clk_e",
+	"msiof3_sync_e",
+	"msiof3_ss1_e",
+	"msiof3_ss2_e",
+	"msiof3_txd_e",
+	"msiof3_rxd_e",
+};
+
+static const char * const pwm0_groups[] = {
+	"pwm0",
+};
+
+static const char * const pwm1_groups[] = {
+	"pwm1_a",
+	"pwm1_b",
+};
+
+static const char * const pwm2_groups[] = {
+	"pwm2_a",
+	"pwm2_b",
+};
+
+static const char * const pwm3_groups[] = {
+	"pwm3_a",
+	"pwm3_b",
+};
+
+static const char * const pwm4_groups[] = {
+	"pwm4_a",
+	"pwm4_b",
+};
+
+static const char * const pwm5_groups[] = {
+	"pwm5_a",
+	"pwm5_b",
+};
+
+static const char * const pwm6_groups[] = {
+	"pwm6_a",
+	"pwm6_b",
+};
+
 static const char * const scif0_groups[] = {
 	"scif0_data",
 	"scif0_clk",
@@ -2071,6 +3598,44 @@ static const char * const scif_clk_groups[] = {
 	"scif_clk_b",
 };
 
+static const char * const sdhi0_groups[] = {
+	"sdhi0_data1",
+	"sdhi0_data4",
+	"sdhi0_ctrl",
+	"sdhi0_cd",
+	"sdhi0_wp",
+};
+
+static const char * const sdhi1_groups[] = {
+	"sdhi1_data1",
+	"sdhi1_data4",
+	"sdhi1_ctrl",
+	"sdhi1_cd",
+	"sdhi1_wp",
+};
+
+static const char * const sdhi2_groups[] = {
+	"sdhi2_data1",
+	"sdhi2_data4",
+	"sdhi2_data8",
+	"sdhi2_ctrl",
+	"sdhi2_cd_a",
+	"sdhi2_wp_a",
+	"sdhi2_cd_b",
+	"sdhi2_wp_b",
+	"sdhi2_ds",
+};
+
+static const char * const sdhi3_groups[] = {
+	"sdhi3_data1",
+	"sdhi3_data4",
+	"sdhi3_data8",
+	"sdhi3_ctrl",
+	"sdhi3_cd",
+	"sdhi3_wp",
+	"sdhi3_ds",
+};
+
 static const char * const usb0_groups[] = {
 	"usb0",
 };
@@ -2085,7 +3650,22 @@ static const char * const usb30_groups[] = {
 
 static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(avb),
+	SH_PFC_FUNCTION(du),
+	SH_PFC_FUNCTION(i2c1),
+	SH_PFC_FUNCTION(i2c2),
+	SH_PFC_FUNCTION(i2c6),
 	SH_PFC_FUNCTION(intc_ex),
+	SH_PFC_FUNCTION(msiof0),
+	SH_PFC_FUNCTION(msiof1),
+	SH_PFC_FUNCTION(msiof2),
+	SH_PFC_FUNCTION(msiof3),
+	SH_PFC_FUNCTION(pwm0),
+	SH_PFC_FUNCTION(pwm1),
+	SH_PFC_FUNCTION(pwm2),
+	SH_PFC_FUNCTION(pwm3),
+	SH_PFC_FUNCTION(pwm4),
+	SH_PFC_FUNCTION(pwm5),
+	SH_PFC_FUNCTION(pwm6),
 	SH_PFC_FUNCTION(scif0),
 	SH_PFC_FUNCTION(scif1),
 	SH_PFC_FUNCTION(scif2),
@@ -2093,6 +3673,10 @@ static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(scif4),
 	SH_PFC_FUNCTION(scif5),
 	SH_PFC_FUNCTION(scif_clk),
+	SH_PFC_FUNCTION(sdhi0),
+	SH_PFC_FUNCTION(sdhi1),
+	SH_PFC_FUNCTION(sdhi2),
+	SH_PFC_FUNCTION(sdhi3),
 	SH_PFC_FUNCTION(usb0),
 	SH_PFC_FUNCTION(usb1),
 	SH_PFC_FUNCTION(usb30),
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77970.c b/drivers/pinctrl/sh-pfc/pfc-r8a77970.c
index b1bb7263532b..b02caf316711 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a77970.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77970.c
@@ -21,13 +21,15 @@
 #include "core.h"
 #include "sh_pfc.h"
 
+#define CFG_FLAGS SH_PFC_PIN_CFG_DRIVE_STRENGTH
+
 #define CPU_ALL_PORT(fn, sfx)						\
-	PORT_GP_CFG_22(0, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_28(1, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_17(2, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_17(3, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_6(4, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_15(5, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH)
+	PORT_GP_CFG_22(0, fn, sfx, CFG_FLAGS | SH_PFC_PIN_CFG_IO_VOLTAGE), \
+	PORT_GP_CFG_28(1, fn, sfx, CFG_FLAGS),				\
+	PORT_GP_CFG_17(2, fn, sfx, CFG_FLAGS | SH_PFC_PIN_CFG_IO_VOLTAGE), \
+	PORT_GP_CFG_17(3, fn, sfx, CFG_FLAGS | SH_PFC_PIN_CFG_IO_VOLTAGE), \
+	PORT_GP_CFG_6(4,  fn, sfx, CFG_FLAGS),				\
+	PORT_GP_CFG_15(5, fn, sfx, CFG_FLAGS)
 /*
  * F_() : just information
  * FM() : macro for FN_xxx / xxx_MARK
@@ -2382,18 +2384,31 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	{ },
 };
 
+enum ioctrl_regs {
+	IOCTRL30,
+	IOCTRL31,
+	IOCTRL32,
+};
+
+static const struct pinmux_ioctrl_reg pinmux_ioctrl_regs[] = {
+	[IOCTRL30] = { 0xe6060380 },
+	[IOCTRL31] = { 0xe6060384 },
+	[IOCTRL32] = { 0xe6060388 },
+	{ /* sentinel */ },
+};
+
 static int r8a77970_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin,
 				   u32 *pocctrl)
 {
 	int bit = pin & 0x1f;
 
-	*pocctrl = 0xe6060380;
+	*pocctrl = pinmux_ioctrl_regs[IOCTRL30].reg;
 	if (pin >= RCAR_GP_PIN(0, 0) && pin <= RCAR_GP_PIN(0, 21))
 		return bit;
 	if (pin >= RCAR_GP_PIN(2, 0) && pin <= RCAR_GP_PIN(2, 9))
 		return bit + 22;
 
-	*pocctrl += 4;
+	*pocctrl = pinmux_ioctrl_regs[IOCTRL31].reg;
 	if (pin >= RCAR_GP_PIN(2, 10) && pin <= RCAR_GP_PIN(2, 16))
 		return bit - 10;
 	if (pin >= RCAR_GP_PIN(3, 0) && pin <= RCAR_GP_PIN(3, 16))
@@ -2421,6 +2436,7 @@ const struct sh_pfc_soc_info r8a77970_pinmux_info = {
 	.nr_functions = ARRAY_SIZE(pinmux_functions),
 
 	.cfg_regs = pinmux_config_regs,
+	.ioctrl_regs = pinmux_ioctrl_regs,
 
 	.pinmux_data = pinmux_data,
 	.pinmux_data_size = ARRAY_SIZE(pinmux_data),
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77980.c b/drivers/pinctrl/sh-pfc/pfc-r8a77980.c
index 84c8f1c2f1d1..3f6967331f64 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a77980.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77980.c
@@ -19,10 +19,10 @@
 #include "sh_pfc.h"
 
 #define CPU_ALL_PORT(fn, sfx)	\
-	PORT_GP_22(0, fn, sfx),	\
+	PORT_GP_CFG_22(0, fn, sfx, SH_PFC_PIN_CFG_IO_VOLTAGE),	\
 	PORT_GP_28(1, fn, sfx),	\
-	PORT_GP_30(2, fn, sfx),	\
-	PORT_GP_17(3, fn, sfx),	\
+	PORT_GP_CFG_30(2, fn, sfx, SH_PFC_PIN_CFG_IO_VOLTAGE),	\
+	PORT_GP_CFG_17(3, fn, sfx, SH_PFC_PIN_CFG_IO_VOLTAGE), \
 	PORT_GP_25(4, fn, sfx),	\
 	PORT_GP_15(5, fn, sfx)
 
@@ -2779,8 +2779,53 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	{ },
 };
 
+enum ioctrl_regs {
+	IOCTRL30,
+	IOCTRL31,
+	IOCTRL32,
+	IOCTRL33,
+};
+
+static const struct pinmux_ioctrl_reg pinmux_ioctrl_regs[] = {
+	[IOCTRL30] = { 0xe6060380, },
+	[IOCTRL31] = { 0xe6060384, },
+	[IOCTRL32] = { 0xe6060388, },
+	[IOCTRL33] = { 0xe606038c, },
+	{ /* sentinel */ },
+};
+
+static int r8a77980_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin,
+				   u32 *pocctrl)
+{
+	int bit = pin & 0x1f;
+
+	*pocctrl = pinmux_ioctrl_regs[IOCTRL30].reg;
+	if (pin >= RCAR_GP_PIN(0, 0) && pin <= RCAR_GP_PIN(0, 21))
+		return bit;
+	else if (pin >= RCAR_GP_PIN(2, 0) && pin <= RCAR_GP_PIN(2, 9))
+		return bit + 22;
+
+	*pocctrl = pinmux_ioctrl_regs[IOCTRL31].reg;
+	if (pin >= RCAR_GP_PIN(2, 10) && pin <= RCAR_GP_PIN(2, 16))
+		return bit - 10;
+	if ((pin >= RCAR_GP_PIN(2, 17) && pin <= RCAR_GP_PIN(2, 24)) ||
+	    (pin >= RCAR_GP_PIN(3,  0) && pin <= RCAR_GP_PIN(3, 16)))
+		return bit + 7;
+
+	*pocctrl = pinmux_ioctrl_regs[IOCTRL32].reg;
+	if (pin >= RCAR_GP_PIN(2, 25) && pin <= RCAR_GP_PIN(2, 29))
+		return pin - 25;
+
+	return -EINVAL;
+}
+
+static const struct sh_pfc_soc_operations pinmux_ops = {
+	.pin_to_pocctrl = r8a77980_pin_to_pocctrl,
+};
+
 const struct sh_pfc_soc_info r8a77980_pinmux_info = {
 	.name = "r8a77980_pfc",
+	.ops = &pinmux_ops,
 	.unlock_reg = 0xe6060000, /* PMMR */
 
 	.function = { PINMUX_FUNCTION_BEGIN, PINMUX_FUNCTION_END },
@@ -2793,6 +2838,7 @@ const struct sh_pfc_soc_info r8a77980_pinmux_info = {
 	.nr_functions = ARRAY_SIZE(pinmux_functions),
 
 	.cfg_regs = pinmux_config_regs,
+	.ioctrl_regs = pinmux_ioctrl_regs,
 
 	.pinmux_data = pinmux_data,
 	.pinmux_data_size = ARRAY_SIZE(pinmux_data),
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77990.c b/drivers/pinctrl/sh-pfc/pfc-r8a77990.c
new file mode 100644
index 000000000000..a68fd658aada
--- /dev/null
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77990.c
@@ -0,0 +1,2695 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * R8A77990 processor support - PFC hardware block.
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ *
+ * This file is based on the drivers/pinctrl/sh-pfc/pfc-r8a7796.c
+ *
+ * R8A7796 processor support - PFC hardware block.
+ *
+ * Copyright (C) 2016-2017 Renesas Electronics Corp.
+ */
+
+#include <linux/kernel.h>
+
+#include "core.h"
+#include "sh_pfc.h"
+
+#define CFG_FLAGS (SH_PFC_PIN_CFG_PULL_UP | \
+		   SH_PFC_PIN_CFG_PULL_DOWN)
+
+#define CPU_ALL_PORT(fn, sfx) \
+	PORT_GP_CFG_18(0, fn, sfx, CFG_FLAGS), \
+	PORT_GP_CFG_23(1, fn, sfx, CFG_FLAGS), \
+	PORT_GP_CFG_26(2, fn, sfx, CFG_FLAGS), \
+	PORT_GP_CFG_16(3, fn, sfx, CFG_FLAGS), \
+	PORT_GP_CFG_11(4, fn, sfx, CFG_FLAGS), \
+	PORT_GP_CFG_20(5, fn, sfx, CFG_FLAGS), \
+	PORT_GP_CFG_18(6, fn, sfx, CFG_FLAGS)
+/*
+ * F_() : just information
+ * FM() : macro for FN_xxx / xxx_MARK
+ */
+
+/* GPSR0 */
+#define GPSR0_17	F_(SDA4,		IP7_27_24)
+#define GPSR0_16	F_(SCL4,		IP7_23_20)
+#define GPSR0_15	F_(D15,			IP7_19_16)
+#define GPSR0_14	F_(D14,			IP7_15_12)
+#define GPSR0_13	F_(D13,			IP7_11_8)
+#define GPSR0_12	F_(D12,			IP7_7_4)
+#define GPSR0_11	F_(D11,			IP7_3_0)
+#define GPSR0_10	F_(D10,			IP6_31_28)
+#define GPSR0_9		F_(D9,			IP6_27_24)
+#define GPSR0_8		F_(D8,			IP6_23_20)
+#define GPSR0_7		F_(D7,			IP6_19_16)
+#define GPSR0_6		F_(D6,			IP6_15_12)
+#define GPSR0_5		F_(D5,			IP6_11_8)
+#define GPSR0_4		F_(D4,			IP6_7_4)
+#define GPSR0_3		F_(D3,			IP6_3_0)
+#define GPSR0_2		F_(D2,			IP5_31_28)
+#define GPSR0_1		F_(D1,			IP5_27_24)
+#define GPSR0_0		F_(D0,			IP5_23_20)
+
+/* GPSR1 */
+#define GPSR1_22	F_(WE0_N,		IP5_19_16)
+#define GPSR1_21	F_(CS0_N,		IP5_15_12)
+#define GPSR1_20	FM(CLKOUT)
+#define GPSR1_19	F_(A19,			IP5_11_8)
+#define GPSR1_18	F_(A18,			IP5_7_4)
+#define GPSR1_17	F_(A17,			IP5_3_0)
+#define GPSR1_16	F_(A16,			IP4_31_28)
+#define GPSR1_15	F_(A15,			IP4_27_24)
+#define GPSR1_14	F_(A14,			IP4_23_20)
+#define GPSR1_13	F_(A13,			IP4_19_16)
+#define GPSR1_12	F_(A12,			IP4_15_12)
+#define GPSR1_11	F_(A11,			IP4_11_8)
+#define GPSR1_10	F_(A10,			IP4_7_4)
+#define GPSR1_9		F_(A9,			IP4_3_0)
+#define GPSR1_8		F_(A8,			IP3_31_28)
+#define GPSR1_7		F_(A7,			IP3_27_24)
+#define GPSR1_6		F_(A6,			IP3_23_20)
+#define GPSR1_5		F_(A5,			IP3_19_16)
+#define GPSR1_4		F_(A4,			IP3_15_12)
+#define GPSR1_3		F_(A3,			IP3_11_8)
+#define GPSR1_2		F_(A2,			IP3_7_4)
+#define GPSR1_1		F_(A1,			IP3_3_0)
+#define GPSR1_0		F_(A0,			IP2_31_28)
+
+/* GPSR2 */
+#define GPSR2_25	F_(EX_WAIT0,		IP2_27_24)
+#define GPSR2_24	F_(RD_WR_N,		IP2_23_20)
+#define GPSR2_23	F_(RD_N,		IP2_19_16)
+#define GPSR2_22	F_(BS_N,		IP2_15_12)
+#define GPSR2_21	FM(AVB_PHY_INT)
+#define GPSR2_20	F_(AVB_TXCREFCLK,	IP2_3_0)
+#define GPSR2_19	FM(AVB_RD3)
+#define GPSR2_18	F_(AVB_RD2,		IP1_31_28)
+#define GPSR2_17	F_(AVB_RD1,		IP1_27_24)
+#define GPSR2_16	F_(AVB_RD0,		IP1_23_20)
+#define GPSR2_15	FM(AVB_RXC)
+#define GPSR2_14	FM(AVB_RX_CTL)
+#define GPSR2_13	F_(RPC_RESET_N,		IP1_19_16)
+#define GPSR2_12	F_(RPC_INT_N,		IP1_15_12)
+#define GPSR2_11	F_(QSPI1_SSL,		IP1_11_8)
+#define GPSR2_10	F_(QSPI1_IO3,		IP1_7_4)
+#define GPSR2_9		F_(QSPI1_IO2,		IP1_3_0)
+#define GPSR2_8		F_(QSPI1_MISO_IO1,	IP0_31_28)
+#define GPSR2_7		F_(QSPI1_MOSI_IO0,	IP0_27_24)
+#define GPSR2_6		F_(QSPI1_SPCLK,		IP0_23_20)
+#define GPSR2_5		FM(QSPI0_SSL)
+#define GPSR2_4		F_(QSPI0_IO3,		IP0_19_16)
+#define GPSR2_3		F_(QSPI0_IO2,		IP0_15_12)
+#define GPSR2_2		F_(QSPI0_MISO_IO1,	IP0_11_8)
+#define GPSR2_1		F_(QSPI0_MOSI_IO0,	IP0_7_4)
+#define GPSR2_0		F_(QSPI0_SPCLK,		IP0_3_0)
+
+/* GPSR3 */
+#define GPSR3_15	F_(SD1_WP,		IP11_7_4)
+#define GPSR3_14	F_(SD1_CD,		IP11_3_0)
+#define GPSR3_13	F_(SD0_WP,		IP10_31_28)
+#define GPSR3_12	F_(SD0_CD,		IP10_27_24)
+#define GPSR3_11	F_(SD1_DAT3,		IP9_11_8)
+#define GPSR3_10	F_(SD1_DAT2,		IP9_7_4)
+#define GPSR3_9		F_(SD1_DAT1,		IP9_3_0)
+#define GPSR3_8		F_(SD1_DAT0,		IP8_31_28)
+#define GPSR3_7		F_(SD1_CMD,		IP8_27_24)
+#define GPSR3_6		F_(SD1_CLK,		IP8_23_20)
+#define GPSR3_5		F_(SD0_DAT3,		IP8_19_16)
+#define GPSR3_4		F_(SD0_DAT2,		IP8_15_12)
+#define GPSR3_3		F_(SD0_DAT1,		IP8_11_8)
+#define GPSR3_2		F_(SD0_DAT0,		IP8_7_4)
+#define GPSR3_1		F_(SD0_CMD,		IP8_3_0)
+#define GPSR3_0		F_(SD0_CLK,		IP7_31_28)
+
+/* GPSR4 */
+#define GPSR4_10	F_(SD3_DS,		IP10_23_20)
+#define GPSR4_9		F_(SD3_DAT7,		IP10_19_16)
+#define GPSR4_8		F_(SD3_DAT6,		IP10_15_12)
+#define GPSR4_7		F_(SD3_DAT5,		IP10_11_8)
+#define GPSR4_6		F_(SD3_DAT4,		IP10_7_4)
+#define GPSR4_5		F_(SD3_DAT3,		IP10_3_0)
+#define GPSR4_4		F_(SD3_DAT2,		IP9_31_28)
+#define GPSR4_3		F_(SD3_DAT1,		IP9_27_24)
+#define GPSR4_2		F_(SD3_DAT0,		IP9_23_20)
+#define GPSR4_1		F_(SD3_CMD,		IP9_19_16)
+#define GPSR4_0		F_(SD3_CLK,		IP9_15_12)
+
+/* GPSR5 */
+#define GPSR5_19	F_(MLB_DAT,		IP13_23_20)
+#define GPSR5_18	F_(MLB_SIG,		IP13_19_16)
+#define GPSR5_17	F_(MLB_CLK,		IP13_15_12)
+#define GPSR5_16	F_(SSI_SDATA9,		IP13_11_8)
+#define GPSR5_15	F_(MSIOF0_SS2,		IP13_7_4)
+#define GPSR5_14	F_(MSIOF0_SS1,		IP13_3_0)
+#define GPSR5_13	F_(MSIOF0_SYNC,		IP12_31_28)
+#define GPSR5_12	F_(MSIOF0_TXD,		IP12_27_24)
+#define GPSR5_11	F_(MSIOF0_RXD,		IP12_23_20)
+#define GPSR5_10	F_(MSIOF0_SCK,		IP12_19_16)
+#define GPSR5_9		F_(RX2_A,		IP12_15_12)
+#define GPSR5_8		F_(TX2_A,		IP12_11_8)
+#define GPSR5_7		F_(SCK2_A,		IP12_7_4)
+#define GPSR5_6		F_(TX1,			IP12_3_0)
+#define GPSR5_5		F_(RX1,			IP11_31_28)
+#define GPSR5_4		F_(RTS0_N_TANS_A,	IP11_23_20)
+#define GPSR5_3		F_(CTS0_N_A,		IP11_19_16)
+#define GPSR5_2		F_(TX0_A,		IP11_15_12)
+#define GPSR5_1		F_(RX0_A,		IP11_11_8)
+#define GPSR5_0		F_(SCK0_A,		IP11_27_24)
+
+/* GPSR6 */
+#define GPSR6_17	F_(USB30_PWEN,		IP15_27_24)
+#define GPSR6_16	F_(SSI_SDATA6,		IP15_19_16)
+#define GPSR6_15	F_(SSI_WS6,		IP15_15_12)
+#define GPSR6_14	F_(SSI_SCK6,		IP15_11_8)
+#define GPSR6_13	F_(SSI_SDATA5,		IP15_7_4)
+#define GPSR6_12	F_(SSI_WS5,		IP15_3_0)
+#define GPSR6_11	F_(SSI_SCK5,		IP14_31_28)
+#define GPSR6_10	F_(SSI_SDATA4,		IP14_27_24)
+#define GPSR6_9		F_(USB30_OVC,		IP15_31_28)
+#define GPSR6_8		F_(AUDIO_CLKA,		IP15_23_20)
+#define GPSR6_7		F_(SSI_SDATA3,		IP14_23_20)
+#define GPSR6_6		F_(SSI_WS349,		IP14_19_16)
+#define GPSR6_5		F_(SSI_SCK349,		IP14_15_12)
+#define GPSR6_4		F_(SSI_SDATA2,		IP14_11_8)
+#define GPSR6_3		F_(SSI_SDATA1,		IP14_7_4)
+#define GPSR6_2		F_(SSI_SDATA0,		IP14_3_0)
+#define GPSR6_1		F_(SSI_WS01239,		IP13_31_28)
+#define GPSR6_0		F_(SSI_SCK01239,	IP13_27_24)
+
+/* IPSRx */		/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */		/* 6 */		/* 7 */		/* 8 */		/* 9 - F */
+#define IP0_3_0		FM(QSPI0_SPCLK)		FM(HSCK4_A)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_7_4		FM(QSPI0_MOSI_IO0)	FM(HCTS4_N_A)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_11_8	FM(QSPI0_MISO_IO1)	FM(HRTS4_N_A)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_15_12	FM(QSPI0_IO2)		FM(HTX4_A)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_19_16	FM(QSPI0_IO3)		FM(HRX4_A)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_23_20	FM(QSPI1_SPCLK)		FM(RIF2_CLK_A)		FM(HSCK4_B)		FM(VI4_DATA0_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_27_24	FM(QSPI1_MOSI_IO0)	FM(RIF2_SYNC_A)		FM(HTX4_B)		FM(VI4_DATA1_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_31_28	FM(QSPI1_MISO_IO1)	FM(RIF2_D0_A)		FM(HRX4_B)		FM(VI4_DATA2_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_3_0		FM(QSPI1_IO2)		FM(RIF2_D1_A)		FM(HTX3_C)		FM(VI4_DATA3_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_7_4		FM(QSPI1_IO3)		FM(RIF3_CLK_A)		FM(HRX3_C)		FM(VI4_DATA4_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_11_8	FM(QSPI1_SSL)		FM(RIF3_SYNC_A)		FM(HSCK3_C)		FM(VI4_DATA5_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_15_12	FM(RPC_INT_N)		FM(RIF3_D0_A)		FM(HCTS3_N_C)		FM(VI4_DATA6_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_19_16	FM(RPC_RESET_N)		FM(RIF3_D1_A)		FM(HRTS3_N_C)		FM(VI4_DATA7_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_23_20	FM(AVB_RD0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_27_24	FM(AVB_RD1)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_31_28	FM(AVB_RD2)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_3_0		FM(AVB_TXCREFCLK)	F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_7_4		FM(AVB_MDIO)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_11_8	FM(AVB_MDC)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_15_12	FM(BS_N)		FM(PWM0_A)		FM(AVB_MAGIC)		FM(VI4_CLK)		F_(0, 0)		FM(TX3_C)	F_(0, 0)	FM(VI5_CLK_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_19_16	FM(RD_N)		FM(PWM1_A)		FM(AVB_LINK)		FM(VI4_FIELD)		F_(0, 0)		FM(RX3_C)	FM(FSCLKST2_N_A) FM(VI5_DATA0_B) F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_23_20	FM(RD_WR_N)		FM(SCL7_A)		FM(AVB_AVTP_MATCH_A)	FM(VI4_VSYNC_N)		FM(TX5_B)		FM(SCK3_C)	FM(PWM5_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_27_24	FM(EX_WAIT0)		FM(SDA7_A)		FM(AVB_AVTP_CAPTURE_A)	FM(VI4_HSYNC_N)		FM(RX5_B)		FM(PWM6_A)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_31_28	FM(A0)			FM(IRQ0)		FM(PWM2_A)		FM(MSIOF3_SS1_B)	FM(VI5_CLK_A)		FM(DU_CDE)	FM(HRX3_D)	FM(IERX)	FM(QSTB_QHE)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_3_0		FM(A1)			FM(IRQ1)		FM(PWM3_A)		FM(DU_DOTCLKIN1)	FM(VI5_DATA0_A)		FM(DU_DISP_CDE) FM(SDA6_B)	FM(IETX)	FM(QCPV_QDE)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_7_4		FM(A2)			FM(IRQ2)		FM(AVB_AVTP_PPS)	FM(VI4_CLKENB)		FM(VI5_DATA1_A)		FM(DU_DISP)	FM(SCL6_B)	F_(0, 0)	FM(QSTVB_QVE)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_11_8	FM(A3)			FM(CTS4_N_A)		FM(PWM4_A)		FM(VI4_DATA12)		F_(0, 0)		FM(DU_DOTCLKOUT0) FM(HTX3_D)	FM(IECLK)	FM(LCDOUT12)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_15_12	FM(A4)			FM(RTS4_N_TANS_A)	FM(MSIOF3_SYNC_B)	FM(VI4_DATA8)		FM(PWM2_B)		FM(DU_DG4)	FM(RIF2_CLK_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_19_16	FM(A5)			FM(SCK4_A)		FM(MSIOF3_SCK_B)	FM(VI4_DATA9)		FM(PWM3_B)		F_(0, 0)	FM(RIF2_SYNC_B)	F_(0, 0)	FM(QPOLA)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_23_20	FM(A6)			FM(RX4_A)		FM(MSIOF3_RXD_B)	FM(VI4_DATA10)		F_(0, 0)		F_(0, 0)	FM(RIF2_D0_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_27_24	FM(A7)			FM(TX4_A)		FM(MSIOF3_TXD_B)	FM(VI4_DATA11)		F_(0, 0)		F_(0, 0)	FM(RIF2_D1_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_31_28	FM(A8)			FM(SDA6_A)		FM(RX3_B)		FM(HRX4_C)		FM(VI5_HSYNC_N_A)	FM(DU_HSYNC)	FM(VI4_DATA0_B)	F_(0, 0)	FM(QSTH_QHS)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+/* IPSRx */		/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */		/* 6 */		/* 7 */		/* 8 */		/* 9 - F */
+#define IP4_3_0		FM(A9)			FM(TX5_A)		FM(IRQ3)		FM(VI4_DATA16)		FM(VI5_VSYNC_N_A)	FM(DU_DG7)	F_(0, 0)	F_(0, 0)	FM(LCDOUT15)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_7_4		FM(A10)			FM(IRQ4)		FM(MSIOF2_SYNC_B)	FM(VI4_DATA13)		FM(VI5_FIELD_A)		FM(DU_DG5)	FM(FSCLKST2_N_B) F_(0, 0)	FM(LCDOUT13)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_11_8	FM(A11)			FM(SCL6_A)		FM(TX3_B)		FM(HTX4_C)		F_(0, 0)		FM(DU_VSYNC)	FM(VI4_DATA1_B)	F_(0, 0)	FM(QSTVA_QVS)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_15_12	FM(A12)			FM(RX5_A)		FM(MSIOF2_SS2_B)	FM(VI4_DATA17)		FM(VI5_DATA3_A)		FM(DU_DG6)	F_(0, 0)	F_(0, 0)	FM(LCDOUT14)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_19_16	FM(A13)			FM(SCK5_A)		FM(MSIOF2_SCK_B)	FM(VI4_DATA14)		FM(HRX4_D)		FM(DU_DB2)	F_(0, 0)	F_(0, 0)	FM(LCDOUT2)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_23_20	FM(A14)			FM(MSIOF1_SS1)		FM(MSIOF2_RXD_B)	FM(VI4_DATA15)		FM(HTX4_D)		FM(DU_DB3)	F_(0, 0)	F_(0, 0)	FM(LCDOUT3)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_27_24	FM(A15)			FM(MSIOF1_SS2)		FM(MSIOF2_TXD_B)	FM(VI4_DATA18)		FM(VI5_DATA4_A)		FM(DU_DB4)	F_(0, 0)	F_(0, 0)	FM(LCDOUT4)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_31_28	FM(A16)			FM(MSIOF1_SYNC)		FM(MSIOF2_SS1_B)	FM(VI4_DATA19)		FM(VI5_DATA5_A)		FM(DU_DB5)	F_(0, 0)	F_(0, 0)	FM(LCDOUT5)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_3_0		FM(A17)			FM(MSIOF1_RXD)		F_(0, 0)		FM(VI4_DATA20)		FM(VI5_DATA6_A)		FM(DU_DB6)	F_(0, 0)	F_(0, 0)	FM(LCDOUT6)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_7_4		FM(A18)			FM(MSIOF1_TXD)		F_(0, 0)		FM(VI4_DATA21)		FM(VI5_DATA7_A)		FM(DU_DB0)	F_(0, 0)	FM(HRX4_E)	FM(LCDOUT0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_11_8	FM(A19)			FM(MSIOF1_SCK)		F_(0, 0)		FM(VI4_DATA22)		FM(VI5_DATA2_A)		FM(DU_DB1)	F_(0, 0)	FM(HTX4_E)	FM(LCDOUT1)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_15_12	FM(CS0_N)		FM(SCL5)		F_(0, 0)		F_(0, 0)		F_(0, 0)		FM(DU_DR0)	FM(VI4_DATA2_B)	F_(0, 0)	FM(LCDOUT16)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_19_16	FM(WE0_N)		FM(SDA5)		F_(0, 0)		F_(0, 0)		F_(0, 0)		FM(DU_DR1)	FM(VI4_DATA3_B)	F_(0, 0)	FM(LCDOUT17)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_23_20	FM(D0)			FM(MSIOF3_SCK_A)	F_(0, 0)		F_(0, 0)		F_(0, 0)		FM(DU_DR2)	FM(CTS4_N_C)	F_(0, 0)	FM(LCDOUT18)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_27_24	FM(D1)			FM(MSIOF3_SYNC_A)	FM(SCK3_A)		FM(VI4_DATA23)		FM(VI5_CLKENB_A)	FM(DU_DB7)	FM(RTS4_N_TANS_C) F_(0, 0)	FM(LCDOUT7)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_31_28	FM(D2)			FM(MSIOF3_RXD_A)	FM(RX5_C)		F_(0, 0)		FM(VI5_DATA14_A)	FM(DU_DR3)	FM(RX4_C)	F_(0, 0)	FM(LCDOUT19)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_3_0		FM(D3)			FM(MSIOF3_TXD_A)	FM(TX5_C)		F_(0, 0)		FM(VI5_DATA15_A)	FM(DU_DR4)	FM(TX4_C)	F_(0, 0)	FM(LCDOUT20)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_7_4		FM(D4)			FM(CANFD1_TX)		FM(HSCK3_B)		FM(CAN1_TX)		FM(RTS3_N_TANS_A)	FM(MSIOF3_SS2_A) F_(0, 0)	FM(VI5_DATA1_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_11_8	FM(D5)			FM(RX3_A)		FM(HRX3_B)		F_(0, 0)		F_(0, 0)		FM(DU_DR5)	FM(VI4_DATA4_B)	F_(0, 0)	FM(LCDOUT21)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_15_12	FM(D6)			FM(TX3_A)		FM(HTX3_B)		F_(0, 0)		F_(0, 0)		FM(DU_DR6)	FM(VI4_DATA5_B)	F_(0, 0)	FM(LCDOUT22)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_19_16	FM(D7)			FM(CANFD1_RX)		FM(IRQ5)		FM(CAN1_RX)		FM(CTS3_N_A)		F_(0, 0)	F_(0, 0)	FM(VI5_DATA2_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_23_20	FM(D8)			FM(MSIOF2_SCK_A)	FM(SCK4_B)		F_(0, 0)		FM(VI5_DATA12_A)	FM(DU_DR7)	FM(RIF3_CLK_B)	FM(HCTS3_N_E)	FM(LCDOUT23)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_27_24	FM(D9)			FM(MSIOF2_SYNC_A)	F_(0, 0)		F_(0, 0)		FM(VI5_DATA10_A)	FM(DU_DG0)	FM(RIF3_SYNC_B)	FM(HRX3_E)	FM(LCDOUT8)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_31_28	FM(D10)			FM(MSIOF2_RXD_A)	F_(0, 0)		F_(0, 0)		FM(VI5_DATA13_A)	FM(DU_DG1)	FM(RIF3_D0_B)	FM(HTX3_E)	FM(LCDOUT9)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_3_0		FM(D11)			FM(MSIOF2_TXD_A)	F_(0, 0)		F_(0, 0)		FM(VI5_DATA11_A)	FM(DU_DG2)	FM(RIF3_D1_B)	FM(HRTS3_N_E)	FM(LCDOUT10)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_7_4		FM(D12)			FM(CANFD0_TX)		FM(TX4_B)		FM(CAN0_TX)		FM(VI5_DATA8_A)		F_(0, 0)	F_(0, 0)	FM(VI5_DATA3_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_11_8	FM(D13)			FM(CANFD0_RX)		FM(RX4_B)		FM(CAN0_RX)		FM(VI5_DATA9_A)		FM(SCL7_B)	F_(0, 0)	FM(VI5_DATA4_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_15_12	FM(D14)			FM(CAN_CLK)		FM(HRX3_A)		FM(MSIOF2_SS2_A)	F_(0, 0)		FM(SDA7_B)	F_(0, 0)	FM(VI5_DATA5_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_19_16	FM(D15)			FM(MSIOF2_SS1_A)	FM(HTX3_A)		FM(MSIOF3_SS1_A)	F_(0, 0)		FM(DU_DG3)	F_(0, 0)	F_(0, 0)	FM(LCDOUT11)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_23_20	FM(SCL4)		FM(CS1_N_A26)		F_(0, 0)		F_(0, 0)		F_(0, 0)		FM(DU_DOTCLKIN0) FM(VI4_DATA6_B) FM(VI5_DATA6_B) FM(QCLK)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_27_24	FM(SDA4)		FM(WE1_N)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	FM(VI4_DATA7_B)	FM(VI5_DATA7_B)	FM(QPOLB)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_31_28	FM(SD0_CLK)		FM(NFDATA8)		FM(SCL1_C)		FM(HSCK1_B)		FM(SDA2_E)		FM(FMCLK_B)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+/* IPSRx */		/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */		/* 6 */		/* 7 */		/* 8 */		/* 9 - F */
+#define IP8_3_0		FM(SD0_CMD)		FM(NFDATA9)		F_(0, 0)		FM(HRX1_B)		F_(0, 0)		FM(SPEEDIN_B)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_7_4		FM(SD0_DAT0)		FM(NFDATA10)		F_(0, 0)		FM(HTX1_B)		F_(0, 0)		FM(REMOCON_B)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_11_8	FM(SD0_DAT1)		FM(NFDATA11)		FM(SDA2_C)		FM(HCTS1_N_B)		F_(0, 0)		FM(FMIN_B)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_15_12	FM(SD0_DAT2)		FM(NFDATA12)		FM(SCL2_C)		FM(HRTS1_N_B)		F_(0, 0)		FM(BPFCLK_B)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_19_16	FM(SD0_DAT3)		FM(NFDATA13)		FM(SDA1_C)		FM(SCL2_E)		FM(SPEEDIN_C)		FM(REMOCON_C)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_23_20	FM(SD1_CLK)		FM(NFDATA14_B)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_27_24	FM(SD1_CMD)		FM(NFDATA15_B)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_31_28	FM(SD1_DAT0)		FM(NFWP_N_B)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_3_0		FM(SD1_DAT1)		FM(NFCE_N_B)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_7_4		FM(SD1_DAT2)		FM(NFALE_B)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_11_8	FM(SD1_DAT3)		FM(NFRB_N_B)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_15_12	FM(SD3_CLK)		FM(NFWE_N)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_19_16	FM(SD3_CMD)		FM(NFRE_N)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_23_20	FM(SD3_DAT0)		FM(NFDATA0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_27_24	FM(SD3_DAT1)		FM(NFDATA1)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_31_28	FM(SD3_DAT2)		FM(NFDATA2)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_3_0	FM(SD3_DAT3)		FM(NFDATA3)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_7_4	FM(SD3_DAT4)		FM(NFDATA4)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_11_8	FM(SD3_DAT5)		FM(NFDATA5)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_15_12	FM(SD3_DAT6)		FM(NFDATA6)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_19_16	FM(SD3_DAT7)		FM(NFDATA7)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_23_20	FM(SD3_DS)		FM(NFCLE)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_27_24	FM(SD0_CD)		FM(NFALE_A)		FM(SD3_CD)		FM(RIF0_CLK_B)		FM(SCL2_B)		FM(TCLK1_A)	FM(SSI_SCK2_B)	FM(TS_SCK0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_31_28	FM(SD0_WP)		FM(NFRB_N_A)		FM(SD3_WP)		FM(RIF0_D0_B)		FM(SDA2_B)		FM(TCLK2_A)	FM(SSI_WS2_B)	FM(TS_SDAT0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_3_0	FM(SD1_CD)		FM(NFCE_N_A)		FM(SSI_SCK1)		FM(RIF0_D1_B)		F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(TS_SDEN0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_7_4	FM(SD1_WP)		FM(NFWP_N_A)		FM(SSI_WS1)		FM(RIF0_SYNC_B)		F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(TS_SPSYNC0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_11_8	FM(RX0_A)		FM(HRX1_A)		FM(SSI_SCK2_A)		FM(RIF1_SYNC)		F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(TS_SCK1)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_15_12	FM(TX0_A)		FM(HTX1_A)		FM(SSI_WS2_A)		FM(RIF1_D0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(TS_SDAT1)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_19_16	FM(CTS0_N_A)		FM(NFDATA14_A)		FM(AUDIO_CLKOUT_A)	FM(RIF1_D1)		FM(SCIF_CLK_A)		FM(FMCLK_A)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_23_20	FM(RTS0_N_TANS_A)	FM(NFDATA15_A)		FM(AUDIO_CLKOUT1_A)	FM(RIF1_CLK)		FM(SCL2_A)		FM(FMIN_A)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_27_24	FM(SCK0_A)		FM(HSCK1_A)		FM(USB3HS0_ID)		FM(RTS1_N_TANS)		FM(SDA2_A)		FM(FMCLK_C)	F_(0, 0)	F_(0, 0)	FM(USB1_ID)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_31_28	FM(RX1)			FM(HRX2_B)		FM(SSI_SCK9_B)		FM(AUDIO_CLKOUT1_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+/* IPSRx */		/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */		/* 6 */		/* 7 */		/* 8 */		/* 9 - F */
+#define IP12_3_0	FM(TX1)			FM(HTX2_B)		FM(SSI_WS9_B)		FM(AUDIO_CLKOUT3_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_7_4	FM(SCK2_A)		FM(HSCK0_A)		FM(AUDIO_CLKB_A)	FM(CTS1_N)		FM(RIF0_CLK_A)		FM(REMOCON_A)	FM(SCIF_CLK_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_11_8	FM(TX2_A)		FM(HRX0_A)		FM(AUDIO_CLKOUT2_A)	F_(0, 0)		FM(SCL1_A)		F_(0, 0)	FM(FSO_CFE_0_N_A) FM(TS_SDEN1)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_15_12	FM(RX2_A)		FM(HTX0_A)		FM(AUDIO_CLKOUT3_A)	F_(0, 0)		FM(SDA1_A)		F_(0, 0)	FM(FSO_CFE_1_N_A) FM(TS_SPSYNC1) F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_19_16	FM(MSIOF0_SCK)		F_(0, 0)		FM(SSI_SCK78)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_23_20	FM(MSIOF0_RXD)		F_(0, 0)		FM(SSI_WS78)		F_(0, 0)		F_(0, 0)		FM(TX2_B)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_27_24	FM(MSIOF0_TXD)		F_(0, 0)		FM(SSI_SDATA7)		F_(0, 0)		F_(0, 0)		FM(RX2_B)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_31_28	FM(MSIOF0_SYNC)		FM(AUDIO_CLKOUT_B)	FM(SSI_SDATA8)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_3_0	FM(MSIOF0_SS1)		FM(HRX2_A)		FM(SSI_SCK4)		FM(HCTS0_N_A)		FM(BPFCLK_C)		FM(SPEEDIN_A)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_7_4	FM(MSIOF0_SS2)		FM(HTX2_A)		FM(SSI_WS4)		FM(HRTS0_N_A)		FM(FMIN_C)		FM(BPFCLK_A)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_11_8	FM(SSI_SDATA9)		F_(0, 0)		FM(AUDIO_CLKC_A)	FM(SCK1)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_15_12	FM(MLB_CLK)		FM(RX0_B)		F_(0, 0)		FM(RIF0_D0_A)		FM(SCL1_B)		FM(TCLK1_B)	F_(0, 0)	F_(0, 0)	FM(SIM0_RST_A)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_19_16	FM(MLB_SIG)		FM(SCK0_B)		F_(0, 0)		FM(RIF0_D1_A)		FM(SDA1_B)		FM(TCLK2_B)	F_(0, 0)	F_(0, 0)	FM(SIM0_D_A)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_23_20	FM(MLB_DAT)		FM(TX0_B)		F_(0, 0)		FM(RIF0_SYNC_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	FM(SIM0_CLK_A)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_27_24	FM(SSI_SCK01239)	F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_31_28	FM(SSI_WS01239)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_3_0	FM(SSI_SDATA0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_7_4	FM(SSI_SDATA1)		FM(AUDIO_CLKC_B)	F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	FM(PWM0_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_11_8	FM(SSI_SDATA2)		FM(AUDIO_CLKOUT2_B)	FM(SSI_SCK9_A)		F_(0, 0)		F_(0, 0)		F_(0, 0)	FM(PWM1_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_15_12	FM(SSI_SCK349)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	FM(PWM2_C)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_19_16	FM(SSI_WS349)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	FM(PWM3_C)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_23_20	FM(SSI_SDATA3)		FM(AUDIO_CLKOUT1_C)	FM(AUDIO_CLKB_B)	F_(0, 0)		F_(0, 0)		F_(0, 0)	FM(PWM4_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_27_24	FM(SSI_SDATA4)		F_(0, 0)		FM(SSI_WS9_A)		F_(0, 0)		F_(0, 0)		F_(0, 0)	FM(PWM5_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_31_28	FM(SSI_SCK5)		FM(HRX0_B)		F_(0, 0)		FM(USB0_PWEN_B)		FM(SCL2_D)		F_(0, 0)	FM(PWM6_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_3_0	FM(SSI_WS5)		FM(HTX0_B)		F_(0, 0)		FM(USB0_OVC_B)		FM(SDA2_D)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_7_4	FM(SSI_SDATA5)		FM(HSCK0_B)		FM(AUDIO_CLKB_C)	FM(TPU0TO0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_11_8	FM(SSI_SCK6)		FM(HSCK2_A)		FM(AUDIO_CLKC_C)	FM(TPU0TO1)		F_(0, 0)		F_(0, 0)	FM(FSO_CFE_0_N_B) F_(0, 0)	FM(SIM0_RST_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_15_12	FM(SSI_WS6)		FM(HCTS2_N_A)		FM(AUDIO_CLKOUT2_C)	FM(TPU0TO2)		FM(SDA1_D)		F_(0, 0)	FM(FSO_CFE_1_N_B) F_(0, 0)	FM(SIM0_D_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_19_16	FM(SSI_SDATA6)		FM(HRTS2_N_A)		FM(AUDIO_CLKOUT3_C)	FM(TPU0TO3)		FM(SCL1_D)		F_(0, 0)	FM(FSO_TOE_N_B)	F_(0, 0)	FM(SIM0_CLK_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_23_20	FM(AUDIO_CLKA)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_27_24	FM(USB30_PWEN)		FM(USB0_PWEN_A)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_31_28	FM(USB30_OVC)		FM(USB0_OVC_A)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)	FM(FSO_TOE_N_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+#define PINMUX_GPSR	\
+\
+													 \
+													 \
+													 \
+													 \
+													 \
+													 \
+				GPSR2_25								 \
+				GPSR2_24								 \
+				GPSR2_23								 \
+		GPSR1_22	GPSR2_22								 \
+		GPSR1_21	GPSR2_21								 \
+		GPSR1_20	GPSR2_20								 \
+		GPSR1_19	GPSR2_19					GPSR5_19		 \
+		GPSR1_18	GPSR2_18					GPSR5_18		 \
+GPSR0_17	GPSR1_17	GPSR2_17					GPSR5_17	GPSR6_17 \
+GPSR0_16	GPSR1_16	GPSR2_16					GPSR5_16	GPSR6_16 \
+GPSR0_15	GPSR1_15	GPSR2_15	GPSR3_15			GPSR5_15	GPSR6_15 \
+GPSR0_14	GPSR1_14	GPSR2_14	GPSR3_14			GPSR5_14	GPSR6_14 \
+GPSR0_13	GPSR1_13	GPSR2_13	GPSR3_13			GPSR5_13	GPSR6_13 \
+GPSR0_12	GPSR1_12	GPSR2_12	GPSR3_12			GPSR5_12	GPSR6_12 \
+GPSR0_11	GPSR1_11	GPSR2_11	GPSR3_11			GPSR5_11	GPSR6_11 \
+GPSR0_10	GPSR1_10	GPSR2_10	GPSR3_10	GPSR4_10	GPSR5_10	GPSR6_10 \
+GPSR0_9		GPSR1_9		GPSR2_9		GPSR3_9		GPSR4_9		GPSR5_9		GPSR6_9 \
+GPSR0_8		GPSR1_8		GPSR2_8		GPSR3_8		GPSR4_8		GPSR5_8		GPSR6_8 \
+GPSR0_7		GPSR1_7		GPSR2_7		GPSR3_7		GPSR4_7		GPSR5_7		GPSR6_7 \
+GPSR0_6		GPSR1_6		GPSR2_6		GPSR3_6		GPSR4_6		GPSR5_6		GPSR6_6 \
+GPSR0_5		GPSR1_5		GPSR2_5		GPSR3_5		GPSR4_5		GPSR5_5		GPSR6_5 \
+GPSR0_4		GPSR1_4		GPSR2_4		GPSR3_4		GPSR4_4		GPSR5_4		GPSR6_4 \
+GPSR0_3		GPSR1_3		GPSR2_3		GPSR3_3		GPSR4_3		GPSR5_3		GPSR6_3 \
+GPSR0_2		GPSR1_2		GPSR2_2		GPSR3_2		GPSR4_2		GPSR5_2		GPSR6_2 \
+GPSR0_1		GPSR1_1		GPSR2_1		GPSR3_1		GPSR4_1		GPSR5_1		GPSR6_1 \
+GPSR0_0		GPSR1_0		GPSR2_0		GPSR3_0		GPSR4_0		GPSR5_0		GPSR6_0
+
+#define PINMUX_IPSR				\
+\
+FM(IP0_3_0)	IP0_3_0		FM(IP1_3_0)	IP1_3_0		FM(IP2_3_0)	IP2_3_0		FM(IP3_3_0)	IP3_3_0 \
+FM(IP0_7_4)	IP0_7_4		FM(IP1_7_4)	IP1_7_4		FM(IP2_7_4)	IP2_7_4		FM(IP3_7_4)	IP3_7_4 \
+FM(IP0_11_8)	IP0_11_8	FM(IP1_11_8)	IP1_11_8	FM(IP2_11_8)	IP2_11_8	FM(IP3_11_8)	IP3_11_8 \
+FM(IP0_15_12)	IP0_15_12	FM(IP1_15_12)	IP1_15_12	FM(IP2_15_12)	IP2_15_12	FM(IP3_15_12)	IP3_15_12 \
+FM(IP0_19_16)	IP0_19_16	FM(IP1_19_16)	IP1_19_16	FM(IP2_19_16)	IP2_19_16	FM(IP3_19_16)	IP3_19_16 \
+FM(IP0_23_20)	IP0_23_20	FM(IP1_23_20)	IP1_23_20	FM(IP2_23_20)	IP2_23_20	FM(IP3_23_20)	IP3_23_20 \
+FM(IP0_27_24)	IP0_27_24	FM(IP1_27_24)	IP1_27_24	FM(IP2_27_24)	IP2_27_24	FM(IP3_27_24)	IP3_27_24 \
+FM(IP0_31_28)	IP0_31_28	FM(IP1_31_28)	IP1_31_28	FM(IP2_31_28)	IP2_31_28	FM(IP3_31_28)	IP3_31_28 \
+\
+FM(IP4_3_0)	IP4_3_0		FM(IP5_3_0)	IP5_3_0		FM(IP6_3_0)	IP6_3_0		FM(IP7_3_0)	IP7_3_0 \
+FM(IP4_7_4)	IP4_7_4		FM(IP5_7_4)	IP5_7_4		FM(IP6_7_4)	IP6_7_4		FM(IP7_7_4)	IP7_7_4 \
+FM(IP4_11_8)	IP4_11_8	FM(IP5_11_8)	IP5_11_8	FM(IP6_11_8)	IP6_11_8	FM(IP7_11_8)	IP7_11_8 \
+FM(IP4_15_12)	IP4_15_12	FM(IP5_15_12)	IP5_15_12	FM(IP6_15_12)	IP6_15_12	FM(IP7_15_12)	IP7_15_12 \
+FM(IP4_19_16)	IP4_19_16	FM(IP5_19_16)	IP5_19_16	FM(IP6_19_16)	IP6_19_16	FM(IP7_19_16)	IP7_19_16 \
+FM(IP4_23_20)	IP4_23_20	FM(IP5_23_20)	IP5_23_20	FM(IP6_23_20)	IP6_23_20	FM(IP7_23_20)	IP7_23_20 \
+FM(IP4_27_24)	IP4_27_24	FM(IP5_27_24)	IP5_27_24	FM(IP6_27_24)	IP6_27_24	FM(IP7_27_24)	IP7_27_24 \
+FM(IP4_31_28)	IP4_31_28	FM(IP5_31_28)	IP5_31_28	FM(IP6_31_28)	IP6_31_28	FM(IP7_31_28)	IP7_31_28 \
+\
+FM(IP8_3_0)	IP8_3_0		FM(IP9_3_0)	IP9_3_0		FM(IP10_3_0)	IP10_3_0	FM(IP11_3_0)	IP11_3_0 \
+FM(IP8_7_4)	IP8_7_4		FM(IP9_7_4)	IP9_7_4		FM(IP10_7_4)	IP10_7_4	FM(IP11_7_4)	IP11_7_4 \
+FM(IP8_11_8)	IP8_11_8	FM(IP9_11_8)	IP9_11_8	FM(IP10_11_8)	IP10_11_8	FM(IP11_11_8)	IP11_11_8 \
+FM(IP8_15_12)	IP8_15_12	FM(IP9_15_12)	IP9_15_12	FM(IP10_15_12)	IP10_15_12	FM(IP11_15_12)	IP11_15_12 \
+FM(IP8_19_16)	IP8_19_16	FM(IP9_19_16)	IP9_19_16	FM(IP10_19_16)	IP10_19_16	FM(IP11_19_16)	IP11_19_16 \
+FM(IP8_23_20)	IP8_23_20	FM(IP9_23_20)	IP9_23_20	FM(IP10_23_20)	IP10_23_20	FM(IP11_23_20)	IP11_23_20 \
+FM(IP8_27_24)	IP8_27_24	FM(IP9_27_24)	IP9_27_24	FM(IP10_27_24)	IP10_27_24	FM(IP11_27_24)	IP11_27_24 \
+FM(IP8_31_28)	IP8_31_28	FM(IP9_31_28)	IP9_31_28	FM(IP10_31_28)	IP10_31_28	FM(IP11_31_28)	IP11_31_28 \
+\
+FM(IP12_3_0)	IP12_3_0	FM(IP13_3_0)	IP13_3_0	FM(IP14_3_0)	IP14_3_0	FM(IP15_3_0)	IP15_3_0 \
+FM(IP12_7_4)	IP12_7_4	FM(IP13_7_4)	IP13_7_4	FM(IP14_7_4)	IP14_7_4	FM(IP15_7_4)	IP15_7_4 \
+FM(IP12_11_8)	IP12_11_8	FM(IP13_11_8)	IP13_11_8	FM(IP14_11_8)	IP14_11_8	FM(IP15_11_8)	IP15_11_8 \
+FM(IP12_15_12)	IP12_15_12	FM(IP13_15_12)	IP13_15_12	FM(IP14_15_12)	IP14_15_12	FM(IP15_15_12)	IP15_15_12 \
+FM(IP12_19_16)	IP12_19_16	FM(IP13_19_16)	IP13_19_16	FM(IP14_19_16)	IP14_19_16	FM(IP15_19_16)	IP15_19_16 \
+FM(IP12_23_20)	IP12_23_20	FM(IP13_23_20)	IP13_23_20	FM(IP14_23_20)	IP14_23_20	FM(IP15_23_20)	IP15_23_20 \
+FM(IP12_27_24)	IP12_27_24	FM(IP13_27_24)	IP13_27_24	FM(IP14_27_24)	IP14_27_24	FM(IP15_27_24)	IP15_27_24 \
+FM(IP12_31_28)	IP12_31_28	FM(IP13_31_28)	IP13_31_28	FM(IP14_31_28)	IP14_31_28	FM(IP15_31_28)	IP15_31_28
+
+/* MOD_SEL0 */			/* 0 */				/* 1 */				/* 2 */				/* 3 */			/* 4 */			/* 5 */		/* 6 */		/* 7 */
+#define MOD_SEL0_30_29		FM(SEL_ADGB_0)			FM(SEL_ADGB_1)			FM(SEL_ADGB_2)			F_(0, 0)
+#define MOD_SEL0_28		FM(SEL_DRIF0_0)			FM(SEL_DRIF0_1)
+#define MOD_SEL0_27_26		FM(SEL_FM_0)			FM(SEL_FM_1)			FM(SEL_FM_2)			F_(0, 0)
+#define MOD_SEL0_25		FM(SEL_FSO_0)			FM(SEL_FSO_1)
+#define MOD_SEL0_24		FM(SEL_HSCIF0_0)		FM(SEL_HSCIF0_1)
+#define MOD_SEL0_23		FM(SEL_HSCIF1_0)		FM(SEL_HSCIF1_1)
+#define MOD_SEL0_22		FM(SEL_HSCIF2_0)		FM(SEL_HSCIF2_1)
+#define MOD_SEL0_21_20		FM(SEL_I2C1_0)			FM(SEL_I2C1_1)			FM(SEL_I2C1_2)			FM(SEL_I2C1_3)		FM(SEL_I2C1_4)		F_(0, 0)	F_(0, 0)	F_(0, 0)
+#define MOD_SEL0_19_18_17	FM(SEL_I2C2_0)			FM(SEL_I2C2_1)			FM(SEL_I2C2_2)			FM(SEL_I2C2_3)		FM(SEL_I2C2_4)		F_(0, 0)	F_(0, 0)	F_(0, 0)
+#define MOD_SEL0_16		FM(SEL_NDFC_0)			FM(SEL_NDFC_1)
+#define MOD_SEL0_15		FM(SEL_PWM0_0)			FM(SEL_PWM0_1)
+#define MOD_SEL0_14		FM(SEL_PWM1_0)			FM(SEL_PWM1_1)
+#define MOD_SEL0_13_12		FM(SEL_PWM2_0)			FM(SEL_PWM2_1)			FM(SEL_PWM2_2)			F_(0, 0)
+#define MOD_SEL0_11_10		FM(SEL_PWM3_0)			FM(SEL_PWM3_1)			FM(SEL_PWM3_2)			F_(0, 0)
+#define MOD_SEL0_9		FM(SEL_PWM4_0)			FM(SEL_PWM4_1)
+#define MOD_SEL0_8		FM(SEL_PWM5_0)			FM(SEL_PWM5_1)
+#define MOD_SEL0_7		FM(SEL_PWM6_0)			FM(SEL_PWM6_1)
+#define MOD_SEL0_6_5		FM(SEL_REMOCON_0)		FM(SEL_REMOCON_1)		FM(SEL_REMOCON_2)		F_(0, 0)
+#define MOD_SEL0_4		FM(SEL_SCIF_0)			FM(SEL_SCIF_1)
+#define MOD_SEL0_3		FM(SEL_SCIF0_0)			FM(SEL_SCIF0_1)
+#define MOD_SEL0_2		FM(SEL_SCIF2_0)			FM(SEL_SCIF2_1)
+#define MOD_SEL0_1_0		FM(SEL_SPEED_PULSE_IF_0)	FM(SEL_SPEED_PULSE_IF_1)	FM(SEL_SPEED_PULSE_IF_2)	F_(0, 0)
+
+/* MOD_SEL1 */			/* 0 */				/* 1 */				/* 2 */				/* 3 */			/* 4 */			/* 5 */		/* 6 */		/* 7 */
+#define MOD_SEL1_31		FM(SEL_SIMCARD_0)		FM(SEL_SIMCARD_1)
+#define MOD_SEL1_30		FM(SEL_SSI2_0)			FM(SEL_SSI2_1)
+#define MOD_SEL1_29		FM(SEL_TIMER_TMU_0)		FM(SEL_TIMER_TMU_1)
+#define MOD_SEL1_28		FM(SEL_USB_20_CH0_0)		FM(SEL_USB_20_CH0_1)
+#define MOD_SEL1_26		FM(SEL_DRIF2_0)			FM(SEL_DRIF2_1)
+#define MOD_SEL1_25		FM(SEL_DRIF3_0)			FM(SEL_DRIF3_1)
+#define MOD_SEL1_24_23_22	FM(SEL_HSCIF3_0)		FM(SEL_HSCIF3_1)		FM(SEL_HSCIF3_2)		FM(SEL_HSCIF3_3)	FM(SEL_HSCIF3_4)	F_(0, 0)	F_(0, 0)	F_(0, 0)
+#define MOD_SEL1_21_20_19	FM(SEL_HSCIF4_0)		FM(SEL_HSCIF4_1)		FM(SEL_HSCIF4_2)		FM(SEL_HSCIF4_3)	FM(SEL_HSCIF4_4)	F_(0, 0)	F_(0, 0)	F_(0, 0)
+#define MOD_SEL1_18		FM(SEL_I2C6_0)			FM(SEL_I2C6_1)
+#define MOD_SEL1_17		FM(SEL_I2C7_0)			FM(SEL_I2C7_1)
+#define MOD_SEL1_16		FM(SEL_MSIOF2_0)		FM(SEL_MSIOF2_1)
+#define MOD_SEL1_15		FM(SEL_MSIOF3_0)		FM(SEL_MSIOF3_1)
+#define MOD_SEL1_14_13		FM(SEL_SCIF3_0)			FM(SEL_SCIF3_1)			FM(SEL_SCIF3_2)			F_(0, 0)
+#define MOD_SEL1_12_11		FM(SEL_SCIF4_0)			FM(SEL_SCIF4_1)			FM(SEL_SCIF4_2)			F_(0, 0)
+#define MOD_SEL1_10_9		FM(SEL_SCIF5_0)			FM(SEL_SCIF5_1)			FM(SEL_SCIF5_2)			F_(0, 0)
+#define MOD_SEL1_8		FM(SEL_VIN4_0)			FM(SEL_VIN4_1)
+#define MOD_SEL1_7		FM(SEL_VIN5_0)			FM(SEL_VIN5_1)
+#define MOD_SEL1_6_5		FM(SEL_ADGC_0)			FM(SEL_ADGC_1)			FM(SEL_ADGC_2)			F_(0, 0)
+#define MOD_SEL1_4		FM(SEL_SSI9_0)			FM(SEL_SSI9_1)
+
+#define PINMUX_MOD_SELS	\
+\
+			MOD_SEL1_31 \
+MOD_SEL0_30_29		MOD_SEL1_30 \
+			MOD_SEL1_29 \
+MOD_SEL0_28		MOD_SEL1_28 \
+MOD_SEL0_27_26 \
+			MOD_SEL1_26 \
+MOD_SEL0_25		MOD_SEL1_25 \
+MOD_SEL0_24		MOD_SEL1_24_23_22 \
+MOD_SEL0_23 \
+MOD_SEL0_22 \
+MOD_SEL0_21_20		MOD_SEL1_21_20_19 \
+MOD_SEL0_19_18_17	MOD_SEL1_18 \
+			MOD_SEL1_17 \
+MOD_SEL0_16		MOD_SEL1_16 \
+MOD_SEL0_15		MOD_SEL1_15 \
+MOD_SEL0_14		MOD_SEL1_14_13 \
+MOD_SEL0_13_12 \
+			MOD_SEL1_12_11 \
+MOD_SEL0_11_10 \
+			MOD_SEL1_10_9 \
+MOD_SEL0_9 \
+MOD_SEL0_8		MOD_SEL1_8 \
+MOD_SEL0_7		MOD_SEL1_7 \
+MOD_SEL0_6_5		MOD_SEL1_6_5 \
+MOD_SEL0_4		MOD_SEL1_4 \
+MOD_SEL0_3 \
+MOD_SEL0_2 \
+MOD_SEL0_1_0
+
+/*
+ * These pins are not able to be muxed but have other properties
+ * that can be set, such as pull-up/pull-down enable.
+ */
+#define PINMUX_STATIC \
+	FM(AVB_TX_CTL) FM(AVB_TXC) FM(AVB_TD0) FM(AVB_TD1) FM(AVB_TD2) \
+	FM(AVB_TD3) \
+	FM(PRESETOUT_N) FM(FSCLKST_N) FM(TRST_N) FM(TCK) FM(TMS) FM(TDI) \
+	FM(ASEBRK) \
+	FM(MLB_REF)
+
+enum {
+	PINMUX_RESERVED = 0,
+
+	PINMUX_DATA_BEGIN,
+	GP_ALL(DATA),
+	PINMUX_DATA_END,
+
+#define F_(x, y)
+#define FM(x)	FN_##x,
+	PINMUX_FUNCTION_BEGIN,
+	GP_ALL(FN),
+	PINMUX_GPSR
+	PINMUX_IPSR
+	PINMUX_MOD_SELS
+	PINMUX_FUNCTION_END,
+#undef F_
+#undef FM
+
+#define F_(x, y)
+#define FM(x)	x##_MARK,
+	PINMUX_MARK_BEGIN,
+	PINMUX_GPSR
+	PINMUX_IPSR
+	PINMUX_MOD_SELS
+	PINMUX_STATIC
+	PINMUX_MARK_END,
+#undef F_
+#undef FM
+};
+
+static const u16 pinmux_data[] = {
+	PINMUX_DATA_GP_ALL(),
+
+	PINMUX_SINGLE(CLKOUT),
+	PINMUX_SINGLE(AVB_PHY_INT),
+	PINMUX_SINGLE(AVB_RD3),
+	PINMUX_SINGLE(AVB_RXC),
+	PINMUX_SINGLE(AVB_RX_CTL),
+	PINMUX_SINGLE(QSPI0_SSL),
+
+	/* IPSR0 */
+	PINMUX_IPSR_GPSR(IP0_3_0,		QSPI0_SPCLK),
+	PINMUX_IPSR_MSEL(IP0_3_0,		HSCK4_A,	SEL_HSCIF4_0),
+
+	PINMUX_IPSR_GPSR(IP0_7_4,		QSPI0_MOSI_IO0),
+	PINMUX_IPSR_MSEL(IP0_7_4,		HCTS4_N_A,	SEL_HSCIF4_0),
+
+	PINMUX_IPSR_GPSR(IP0_11_8,		QSPI0_MISO_IO1),
+	PINMUX_IPSR_MSEL(IP0_11_8,		HRTS4_N_A,	SEL_HSCIF4_0),
+
+	PINMUX_IPSR_GPSR(IP0_15_12,		QSPI0_IO2),
+	PINMUX_IPSR_GPSR(IP0_15_12,		HTX4_A),
+
+	PINMUX_IPSR_GPSR(IP0_19_16,		QSPI0_IO3),
+	PINMUX_IPSR_MSEL(IP0_19_16,		HRX4_A,		SEL_HSCIF4_0),
+
+	PINMUX_IPSR_GPSR(IP0_23_20,		QSPI1_SPCLK),
+	PINMUX_IPSR_MSEL(IP0_23_20,		RIF2_CLK_A,	SEL_DRIF2_0),
+	PINMUX_IPSR_MSEL(IP0_23_20,		HSCK4_B,	SEL_HSCIF4_1),
+	PINMUX_IPSR_MSEL(IP0_23_20,		VI4_DATA0_A,	SEL_VIN4_0),
+
+	PINMUX_IPSR_GPSR(IP0_27_24,		QSPI1_MOSI_IO0),
+	PINMUX_IPSR_MSEL(IP0_27_24,		RIF2_SYNC_A,	SEL_DRIF2_0),
+	PINMUX_IPSR_GPSR(IP0_27_24,		HTX4_B),
+	PINMUX_IPSR_MSEL(IP0_27_24,		VI4_DATA1_A,	SEL_VIN4_0),
+
+	PINMUX_IPSR_GPSR(IP0_31_28,		QSPI1_MISO_IO1),
+	PINMUX_IPSR_MSEL(IP0_31_28,		RIF2_D0_A,	SEL_DRIF2_0),
+	PINMUX_IPSR_MSEL(IP0_31_28,		HRX4_B,		SEL_HSCIF4_1),
+	PINMUX_IPSR_MSEL(IP0_31_28,		VI4_DATA2_A,	SEL_VIN4_0),
+
+	/* IPSR1 */
+	PINMUX_IPSR_GPSR(IP1_3_0,		QSPI1_IO2),
+	PINMUX_IPSR_MSEL(IP1_3_0,		RIF2_D1_A,	SEL_DRIF2_0),
+	PINMUX_IPSR_GPSR(IP1_3_0,		HTX3_C),
+	PINMUX_IPSR_MSEL(IP1_3_0,		VI4_DATA3_A,	SEL_VIN4_0),
+
+	PINMUX_IPSR_GPSR(IP1_7_4,		QSPI1_IO3),
+	PINMUX_IPSR_MSEL(IP1_7_4,		RIF3_CLK_A,	SEL_DRIF3_0),
+	PINMUX_IPSR_MSEL(IP1_7_4,		HRX3_C,		SEL_HSCIF3_2),
+	PINMUX_IPSR_MSEL(IP1_7_4,		VI4_DATA4_A,	SEL_VIN4_0),
+
+	PINMUX_IPSR_GPSR(IP1_11_8,		QSPI1_SSL),
+	PINMUX_IPSR_MSEL(IP1_11_8,		RIF3_SYNC_A,	SEL_DRIF3_0),
+	PINMUX_IPSR_MSEL(IP1_11_8,		HSCK3_C,	SEL_HSCIF3_2),
+	PINMUX_IPSR_MSEL(IP1_11_8,		VI4_DATA5_A,	SEL_VIN4_0),
+
+	PINMUX_IPSR_GPSR(IP1_15_12,		RPC_INT_N),
+	PINMUX_IPSR_MSEL(IP1_15_12,		RIF3_D0_A,	SEL_DRIF3_0),
+	PINMUX_IPSR_MSEL(IP1_15_12,		HCTS3_N_C,	SEL_HSCIF3_2),
+	PINMUX_IPSR_MSEL(IP1_15_12,		VI4_DATA6_A,	SEL_VIN4_0),
+
+	PINMUX_IPSR_GPSR(IP1_19_16,		RPC_RESET_N),
+	PINMUX_IPSR_MSEL(IP1_19_16,		RIF3_D1_A,	SEL_DRIF3_0),
+	PINMUX_IPSR_MSEL(IP1_19_16,		HRTS3_N_C,	SEL_HSCIF3_2),
+	PINMUX_IPSR_MSEL(IP1_19_16,		VI4_DATA7_A,	SEL_VIN4_0),
+
+	PINMUX_IPSR_GPSR(IP1_23_20,		AVB_RD0),
+
+	PINMUX_IPSR_GPSR(IP1_27_24,		AVB_RD1),
+
+	PINMUX_IPSR_GPSR(IP1_31_28,		AVB_RD2),
+
+	/* IPSR2 */
+	PINMUX_IPSR_GPSR(IP2_3_0,		AVB_TXCREFCLK),
+
+	PINMUX_IPSR_GPSR(IP2_7_4,		AVB_MDIO),
+
+	PINMUX_IPSR_GPSR(IP2_11_8,		AVB_MDC),
+
+	PINMUX_IPSR_GPSR(IP2_15_12,		BS_N),
+	PINMUX_IPSR_MSEL(IP2_15_12,		PWM0_A,		SEL_PWM0_0),
+	PINMUX_IPSR_GPSR(IP2_15_12,		AVB_MAGIC),
+	PINMUX_IPSR_GPSR(IP2_15_12,		VI4_CLK),
+	PINMUX_IPSR_GPSR(IP2_15_12,		TX3_C),
+	PINMUX_IPSR_MSEL(IP2_15_12,		VI5_CLK_B,	SEL_VIN5_1),
+
+	PINMUX_IPSR_GPSR(IP2_19_16,		RD_N),
+	PINMUX_IPSR_MSEL(IP2_19_16,		PWM1_A,		SEL_PWM1_0),
+	PINMUX_IPSR_GPSR(IP2_19_16,		AVB_LINK),
+	PINMUX_IPSR_GPSR(IP2_19_16,		VI4_FIELD),
+	PINMUX_IPSR_MSEL(IP2_19_16,		RX3_C,		SEL_SCIF3_2),
+	PINMUX_IPSR_GPSR(IP2_19_16,		FSCLKST2_N_A),
+	PINMUX_IPSR_MSEL(IP2_19_16,		VI5_DATA0_B,	SEL_VIN5_1),
+
+	PINMUX_IPSR_GPSR(IP2_23_20,		RD_WR_N),
+	PINMUX_IPSR_MSEL(IP2_23_20,		SCL7_A,		SEL_I2C7_0),
+	PINMUX_IPSR_GPSR(IP2_23_20,		AVB_AVTP_MATCH_A),
+	PINMUX_IPSR_GPSR(IP2_23_20,		VI4_VSYNC_N),
+	PINMUX_IPSR_GPSR(IP2_23_20,		TX5_B),
+	PINMUX_IPSR_MSEL(IP2_23_20,		SCK3_C,		SEL_SCIF3_2),
+	PINMUX_IPSR_MSEL(IP2_23_20,		PWM5_A,		SEL_PWM5_0),
+
+	PINMUX_IPSR_GPSR(IP2_27_24,		EX_WAIT0),
+	PINMUX_IPSR_MSEL(IP2_27_24,		SDA7_A,		SEL_I2C7_0),
+	PINMUX_IPSR_GPSR(IP2_27_24,		AVB_AVTP_CAPTURE_A),
+	PINMUX_IPSR_GPSR(IP2_27_24,		VI4_HSYNC_N),
+	PINMUX_IPSR_MSEL(IP2_27_24,		RX5_B,		SEL_SCIF5_1),
+	PINMUX_IPSR_MSEL(IP2_27_24,		PWM6_A,		SEL_PWM6_0),
+
+	PINMUX_IPSR_GPSR(IP2_31_28,		A0),
+	PINMUX_IPSR_GPSR(IP2_31_28,		IRQ0),
+	PINMUX_IPSR_MSEL(IP2_31_28,		PWM2_A,		SEL_PWM2_0),
+	PINMUX_IPSR_MSEL(IP2_31_28,		MSIOF3_SS1_B,	SEL_MSIOF3_1),
+	PINMUX_IPSR_MSEL(IP2_31_28,		VI5_CLK_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP2_31_28,		DU_CDE),
+	PINMUX_IPSR_MSEL(IP2_31_28,		HRX3_D,		SEL_HSCIF3_3),
+	PINMUX_IPSR_GPSR(IP2_31_28,		IERX),
+	PINMUX_IPSR_GPSR(IP2_31_28,		QSTB_QHE),
+
+	/* IPSR3 */
+	PINMUX_IPSR_GPSR(IP3_3_0,		A1),
+	PINMUX_IPSR_GPSR(IP3_3_0,		IRQ1),
+	PINMUX_IPSR_MSEL(IP3_3_0,		PWM3_A,		SEL_PWM3_0),
+	PINMUX_IPSR_GPSR(IP3_3_0,		DU_DOTCLKIN1),
+	PINMUX_IPSR_MSEL(IP3_3_0,		VI5_DATA0_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP3_3_0,		DU_DISP_CDE),
+	PINMUX_IPSR_MSEL(IP3_3_0,		SDA6_B,		SEL_I2C6_1),
+	PINMUX_IPSR_GPSR(IP3_3_0,		IETX),
+	PINMUX_IPSR_GPSR(IP3_3_0,		QCPV_QDE),
+
+	PINMUX_IPSR_GPSR(IP3_7_4,		A2),
+	PINMUX_IPSR_GPSR(IP3_7_4,		IRQ2),
+	PINMUX_IPSR_GPSR(IP3_7_4,		AVB_AVTP_PPS),
+	PINMUX_IPSR_GPSR(IP3_7_4,		VI4_CLKENB),
+	PINMUX_IPSR_MSEL(IP3_7_4,		VI5_DATA1_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP3_7_4,		DU_DISP),
+	PINMUX_IPSR_MSEL(IP3_7_4,		SCL6_B,		SEL_I2C6_1),
+	PINMUX_IPSR_GPSR(IP3_7_4,		QSTVB_QVE),
+
+	PINMUX_IPSR_GPSR(IP3_11_8,		A3),
+	PINMUX_IPSR_MSEL(IP3_11_8,		CTS4_N_A,	SEL_SCIF4_0),
+	PINMUX_IPSR_MSEL(IP3_11_8,		PWM4_A,		SEL_PWM4_0),
+	PINMUX_IPSR_GPSR(IP3_11_8,		VI4_DATA12),
+	PINMUX_IPSR_GPSR(IP3_11_8,		DU_DOTCLKOUT0),
+	PINMUX_IPSR_GPSR(IP3_11_8,		HTX3_D),
+	PINMUX_IPSR_GPSR(IP3_11_8,		IECLK),
+	PINMUX_IPSR_GPSR(IP3_11_8,		LCDOUT12),
+
+	PINMUX_IPSR_GPSR(IP3_15_12,		A4),
+	PINMUX_IPSR_MSEL(IP3_15_12,		RTS4_N_TANS_A,	SEL_SCIF4_0),
+	PINMUX_IPSR_MSEL(IP3_15_12,		MSIOF3_SYNC_B,	SEL_MSIOF3_1),
+	PINMUX_IPSR_GPSR(IP3_15_12,		VI4_DATA8),
+	PINMUX_IPSR_MSEL(IP3_15_12,		PWM2_B,		SEL_PWM2_1),
+	PINMUX_IPSR_GPSR(IP3_15_12,		DU_DG4),
+	PINMUX_IPSR_MSEL(IP3_15_12,		RIF2_CLK_B,	SEL_DRIF2_1),
+
+	PINMUX_IPSR_GPSR(IP3_19_16,		A5),
+	PINMUX_IPSR_MSEL(IP3_19_16,		SCK4_A,		SEL_SCIF4_0),
+	PINMUX_IPSR_MSEL(IP3_19_16,		MSIOF3_SCK_B,	SEL_MSIOF3_1),
+	PINMUX_IPSR_GPSR(IP3_19_16,		VI4_DATA9),
+	PINMUX_IPSR_MSEL(IP3_19_16,		PWM3_B,		SEL_PWM3_1),
+	PINMUX_IPSR_MSEL(IP3_19_16,		RIF2_SYNC_B,	SEL_DRIF2_1),
+	PINMUX_IPSR_GPSR(IP3_19_16,		QPOLA),
+
+	PINMUX_IPSR_GPSR(IP3_23_20,		A6),
+	PINMUX_IPSR_MSEL(IP3_23_20,		RX4_A,		SEL_SCIF4_0),
+	PINMUX_IPSR_MSEL(IP3_23_20,		MSIOF3_RXD_B,	SEL_MSIOF3_1),
+	PINMUX_IPSR_GPSR(IP3_23_20,		VI4_DATA10),
+	PINMUX_IPSR_MSEL(IP3_23_20,		RIF2_D0_B,	SEL_DRIF2_1),
+
+	PINMUX_IPSR_GPSR(IP3_27_24,		A7),
+	PINMUX_IPSR_GPSR(IP3_27_24,		TX4_A),
+	PINMUX_IPSR_GPSR(IP3_27_24,		MSIOF3_TXD_B),
+	PINMUX_IPSR_GPSR(IP3_27_24,		VI4_DATA11),
+	PINMUX_IPSR_MSEL(IP3_27_24,		RIF2_D1_B,	SEL_DRIF2_1),
+
+	PINMUX_IPSR_GPSR(IP3_31_28,		A8),
+	PINMUX_IPSR_MSEL(IP3_31_28,		SDA6_A,		SEL_I2C6_0),
+	PINMUX_IPSR_MSEL(IP3_31_28,		RX3_B,		SEL_SCIF3_1),
+	PINMUX_IPSR_MSEL(IP3_31_28,		HRX4_C,		SEL_HSCIF4_2),
+	PINMUX_IPSR_MSEL(IP3_31_28,		VI5_HSYNC_N_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP3_31_28,		DU_HSYNC),
+	PINMUX_IPSR_MSEL(IP3_31_28,		VI4_DATA0_B,	SEL_VIN4_1),
+	PINMUX_IPSR_GPSR(IP3_31_28,		QSTH_QHS),
+
+	/* IPSR4 */
+	PINMUX_IPSR_GPSR(IP4_3_0,		A9),
+	PINMUX_IPSR_GPSR(IP4_3_0,		TX5_A),
+	PINMUX_IPSR_GPSR(IP4_3_0,		IRQ3),
+	PINMUX_IPSR_GPSR(IP4_3_0,		VI4_DATA16),
+	PINMUX_IPSR_MSEL(IP4_3_0,		VI5_VSYNC_N_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP4_3_0,		DU_DG7),
+	PINMUX_IPSR_GPSR(IP4_3_0,		LCDOUT15),
+
+	PINMUX_IPSR_GPSR(IP4_7_4,		A10),
+	PINMUX_IPSR_GPSR(IP4_7_4,		IRQ4),
+	PINMUX_IPSR_MSEL(IP4_7_4,		MSIOF2_SYNC_B,	SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP4_7_4,		VI4_DATA13),
+	PINMUX_IPSR_MSEL(IP4_7_4,		VI5_FIELD_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP4_7_4,		DU_DG5),
+	PINMUX_IPSR_GPSR(IP4_7_4,		FSCLKST2_N_B),
+	PINMUX_IPSR_GPSR(IP4_7_4,		LCDOUT13),
+
+	PINMUX_IPSR_GPSR(IP4_11_8,		A11),
+	PINMUX_IPSR_MSEL(IP4_11_8,		SCL6_A,		SEL_I2C6_0),
+	PINMUX_IPSR_GPSR(IP4_11_8,		TX3_B),
+	PINMUX_IPSR_GPSR(IP4_11_8,		HTX4_C),
+	PINMUX_IPSR_GPSR(IP4_11_8,		DU_VSYNC),
+	PINMUX_IPSR_MSEL(IP4_11_8,		VI4_DATA1_B,	SEL_VIN4_1),
+	PINMUX_IPSR_GPSR(IP4_11_8,		QSTVA_QVS),
+
+	PINMUX_IPSR_GPSR(IP4_15_12,		A12),
+	PINMUX_IPSR_MSEL(IP4_15_12,		RX5_A,		SEL_SCIF5_0),
+	PINMUX_IPSR_GPSR(IP4_15_12,		MSIOF2_SS2_B),
+	PINMUX_IPSR_GPSR(IP4_15_12,		VI4_DATA17),
+	PINMUX_IPSR_MSEL(IP4_15_12,		VI5_DATA3_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP4_15_12,		DU_DG6),
+	PINMUX_IPSR_GPSR(IP4_15_12,		LCDOUT14),
+
+	PINMUX_IPSR_GPSR(IP4_19_16,		A13),
+	PINMUX_IPSR_MSEL(IP4_19_16,		SCK5_A,		SEL_SCIF5_0),
+	PINMUX_IPSR_MSEL(IP4_19_16,		MSIOF2_SCK_B,	SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP4_19_16,		VI4_DATA14),
+	PINMUX_IPSR_MSEL(IP4_19_16,		HRX4_D,		SEL_HSCIF4_3),
+	PINMUX_IPSR_GPSR(IP4_19_16,		DU_DB2),
+	PINMUX_IPSR_GPSR(IP4_19_16,		LCDOUT2),
+
+	PINMUX_IPSR_GPSR(IP4_23_20,		A14),
+	PINMUX_IPSR_GPSR(IP4_23_20,		MSIOF1_SS1),
+	PINMUX_IPSR_MSEL(IP4_23_20,		MSIOF2_RXD_B,	SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP4_23_20,		VI4_DATA15),
+	PINMUX_IPSR_GPSR(IP4_23_20,		HTX4_D),
+	PINMUX_IPSR_GPSR(IP4_23_20,		DU_DB3),
+	PINMUX_IPSR_GPSR(IP4_23_20,		LCDOUT3),
+
+	PINMUX_IPSR_GPSR(IP4_27_24,		A15),
+	PINMUX_IPSR_GPSR(IP4_27_24,		MSIOF1_SS2),
+	PINMUX_IPSR_GPSR(IP4_27_24,		MSIOF2_TXD_B),
+	PINMUX_IPSR_GPSR(IP4_27_24,		VI4_DATA18),
+	PINMUX_IPSR_MSEL(IP4_27_24,		VI5_DATA4_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP4_27_24,		DU_DB4),
+	PINMUX_IPSR_GPSR(IP4_27_24,		LCDOUT4),
+
+	PINMUX_IPSR_GPSR(IP4_31_28,		A16),
+	PINMUX_IPSR_GPSR(IP4_31_28,		MSIOF1_SYNC),
+	PINMUX_IPSR_GPSR(IP4_31_28,		MSIOF2_SS1_B),
+	PINMUX_IPSR_GPSR(IP4_31_28,		VI4_DATA19),
+	PINMUX_IPSR_MSEL(IP4_31_28,		VI5_DATA5_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP4_31_28,		DU_DB5),
+	PINMUX_IPSR_GPSR(IP4_31_28,		LCDOUT5),
+
+	/* IPSR5 */
+	PINMUX_IPSR_GPSR(IP5_3_0,		A17),
+	PINMUX_IPSR_GPSR(IP5_3_0,		MSIOF1_RXD),
+	PINMUX_IPSR_GPSR(IP5_3_0,		VI4_DATA20),
+	PINMUX_IPSR_MSEL(IP5_3_0,		VI5_DATA6_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP5_3_0,		DU_DB6),
+	PINMUX_IPSR_GPSR(IP5_3_0,		LCDOUT6),
+
+	PINMUX_IPSR_GPSR(IP5_7_4,		A18),
+	PINMUX_IPSR_GPSR(IP5_7_4,		MSIOF1_TXD),
+	PINMUX_IPSR_GPSR(IP5_7_4,		VI4_DATA21),
+	PINMUX_IPSR_MSEL(IP5_7_4,		VI5_DATA7_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP5_7_4,		DU_DB0),
+	PINMUX_IPSR_MSEL(IP5_7_4,		HRX4_E,		SEL_HSCIF4_4),
+	PINMUX_IPSR_GPSR(IP5_7_4,		LCDOUT0),
+
+	PINMUX_IPSR_GPSR(IP5_11_8,		A19),
+	PINMUX_IPSR_GPSR(IP5_11_8,		MSIOF1_SCK),
+	PINMUX_IPSR_GPSR(IP5_11_8,		VI4_DATA22),
+	PINMUX_IPSR_MSEL(IP5_11_8,		VI5_DATA2_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP5_11_8,		DU_DB1),
+	PINMUX_IPSR_GPSR(IP5_11_8,		HTX4_E),
+	PINMUX_IPSR_GPSR(IP5_11_8,		LCDOUT1),
+
+	PINMUX_IPSR_GPSR(IP5_15_12,		CS0_N),
+	PINMUX_IPSR_GPSR(IP5_15_12,		SCL5),
+	PINMUX_IPSR_GPSR(IP5_15_12,		DU_DR0),
+	PINMUX_IPSR_MSEL(IP5_15_12,		VI4_DATA2_B,	SEL_VIN4_1),
+	PINMUX_IPSR_GPSR(IP5_15_12,		LCDOUT16),
+
+	PINMUX_IPSR_GPSR(IP5_19_16,		WE0_N),
+	PINMUX_IPSR_GPSR(IP5_19_16,		SDA5),
+	PINMUX_IPSR_GPSR(IP5_19_16,		DU_DR1),
+	PINMUX_IPSR_MSEL(IP5_19_16,		VI4_DATA3_B,	SEL_VIN4_1),
+	PINMUX_IPSR_GPSR(IP5_19_16,		LCDOUT17),
+
+	PINMUX_IPSR_GPSR(IP5_23_20,		D0),
+	PINMUX_IPSR_MSEL(IP5_23_20,		MSIOF3_SCK_A,	SEL_MSIOF3_0),
+	PINMUX_IPSR_GPSR(IP5_23_20,		DU_DR2),
+	PINMUX_IPSR_MSEL(IP5_23_20,		CTS4_N_C,	SEL_SCIF4_2),
+	PINMUX_IPSR_GPSR(IP5_23_20,		LCDOUT18),
+
+	PINMUX_IPSR_GPSR(IP5_27_24,		D1),
+	PINMUX_IPSR_MSEL(IP5_27_24,		MSIOF3_SYNC_A,	SEL_MSIOF3_0),
+	PINMUX_IPSR_MSEL(IP5_27_24,		SCK3_A,		SEL_SCIF3_0),
+	PINMUX_IPSR_GPSR(IP5_27_24,		VI4_DATA23),
+	PINMUX_IPSR_MSEL(IP5_27_24,		VI5_CLKENB_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP5_27_24,		DU_DB7),
+	PINMUX_IPSR_MSEL(IP5_27_24,		RTS4_N_TANS_C,	SEL_SCIF4_2),
+	PINMUX_IPSR_GPSR(IP5_27_24,		LCDOUT7),
+
+	PINMUX_IPSR_GPSR(IP5_31_28,		D2),
+	PINMUX_IPSR_MSEL(IP5_31_28,		MSIOF3_RXD_A,	SEL_MSIOF3_0),
+	PINMUX_IPSR_MSEL(IP5_31_28,		RX5_C,		SEL_SCIF5_2),
+	PINMUX_IPSR_MSEL(IP5_31_28,		VI5_DATA14_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP5_31_28,		DU_DR3),
+	PINMUX_IPSR_MSEL(IP5_31_28,		RX4_C,		SEL_SCIF4_2),
+	PINMUX_IPSR_GPSR(IP5_31_28,		LCDOUT19),
+
+	/* IPSR6 */
+	PINMUX_IPSR_GPSR(IP6_3_0,		D3),
+	PINMUX_IPSR_GPSR(IP6_3_0,		MSIOF3_TXD_A),
+	PINMUX_IPSR_GPSR(IP6_3_0,		TX5_C),
+	PINMUX_IPSR_MSEL(IP6_3_0,		VI5_DATA15_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP6_3_0,		DU_DR4),
+	PINMUX_IPSR_GPSR(IP6_3_0,		TX4_C),
+	PINMUX_IPSR_GPSR(IP6_3_0,		LCDOUT20),
+
+	PINMUX_IPSR_GPSR(IP6_7_4,		D4),
+	PINMUX_IPSR_GPSR(IP6_7_4,		CANFD1_TX),
+	PINMUX_IPSR_MSEL(IP6_7_4,		HSCK3_B,	SEL_HSCIF3_1),
+	PINMUX_IPSR_GPSR(IP6_7_4,		CAN1_TX),
+	PINMUX_IPSR_MSEL(IP6_7_4,		RTS3_N_TANS_A,	SEL_SCIF3_0),
+	PINMUX_IPSR_GPSR(IP6_7_4,		MSIOF3_SS2_A),
+	PINMUX_IPSR_MSEL(IP6_7_4,		VI5_DATA1_B,	SEL_VIN5_1),
+
+	PINMUX_IPSR_GPSR(IP6_11_8,		D5),
+	PINMUX_IPSR_MSEL(IP6_11_8,		RX3_A,		SEL_SCIF3_0),
+	PINMUX_IPSR_MSEL(IP6_11_8,		HRX3_B,		SEL_HSCIF3_1),
+	PINMUX_IPSR_GPSR(IP6_11_8,		DU_DR5),
+	PINMUX_IPSR_MSEL(IP6_11_8,		VI4_DATA4_B,	SEL_VIN4_1),
+	PINMUX_IPSR_GPSR(IP6_11_8,		LCDOUT21),
+
+	PINMUX_IPSR_GPSR(IP6_15_12,		D6),
+	PINMUX_IPSR_GPSR(IP6_15_12,		TX3_A),
+	PINMUX_IPSR_GPSR(IP6_15_12,		HTX3_B),
+	PINMUX_IPSR_GPSR(IP6_15_12,		DU_DR6),
+	PINMUX_IPSR_MSEL(IP6_15_12,		VI4_DATA5_B,	SEL_VIN4_1),
+	PINMUX_IPSR_GPSR(IP6_15_12,		LCDOUT22),
+
+	PINMUX_IPSR_GPSR(IP6_19_16,		D7),
+	PINMUX_IPSR_GPSR(IP6_19_16,		CANFD1_RX),
+	PINMUX_IPSR_GPSR(IP6_19_16,		IRQ5),
+	PINMUX_IPSR_GPSR(IP6_19_16,		CAN1_RX),
+	PINMUX_IPSR_MSEL(IP6_19_16,		CTS3_N_A,	SEL_SCIF3_0),
+	PINMUX_IPSR_MSEL(IP6_19_16,		VI5_DATA2_B,	SEL_VIN5_1),
+
+	PINMUX_IPSR_GPSR(IP6_23_20,		D8),
+	PINMUX_IPSR_MSEL(IP6_23_20,		MSIOF2_SCK_A,	SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP6_23_20,		SCK4_B,		SEL_SCIF4_1),
+	PINMUX_IPSR_MSEL(IP6_23_20,		VI5_DATA12_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP6_23_20,		DU_DR7),
+	PINMUX_IPSR_MSEL(IP6_23_20,		RIF3_CLK_B,	SEL_DRIF3_1),
+	PINMUX_IPSR_MSEL(IP6_23_20,		HCTS3_N_E,	SEL_HSCIF3_4),
+	PINMUX_IPSR_GPSR(IP6_23_20,		LCDOUT23),
+
+	PINMUX_IPSR_GPSR(IP6_27_24,		D9),
+	PINMUX_IPSR_MSEL(IP6_27_24,		MSIOF2_SYNC_A,	SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP6_27_24,		VI5_DATA10_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP6_27_24,		DU_DG0),
+	PINMUX_IPSR_MSEL(IP6_27_24,		RIF3_SYNC_B,	SEL_DRIF3_1),
+	PINMUX_IPSR_MSEL(IP6_27_24,		HRX3_E,		SEL_HSCIF3_4),
+	PINMUX_IPSR_GPSR(IP6_27_24,		LCDOUT8),
+
+	PINMUX_IPSR_GPSR(IP6_31_28,		D10),
+	PINMUX_IPSR_MSEL(IP6_31_28,		MSIOF2_RXD_A,	SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP6_31_28,		VI5_DATA13_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP6_31_28,		DU_DG1),
+	PINMUX_IPSR_MSEL(IP6_31_28,		RIF3_D0_B,	SEL_DRIF3_1),
+	PINMUX_IPSR_GPSR(IP6_31_28,		HTX3_E),
+	PINMUX_IPSR_GPSR(IP6_31_28,		LCDOUT9),
+
+	/* IPSR7 */
+	PINMUX_IPSR_GPSR(IP7_3_0,		D11),
+	PINMUX_IPSR_GPSR(IP7_3_0,		MSIOF2_TXD_A),
+	PINMUX_IPSR_MSEL(IP7_3_0,		VI5_DATA11_A,	SEL_VIN5_0),
+	PINMUX_IPSR_GPSR(IP7_3_0,		DU_DG2),
+	PINMUX_IPSR_MSEL(IP7_3_0,		RIF3_D1_B,	SEL_DRIF3_1),
+	PINMUX_IPSR_MSEL(IP7_3_0,		HRTS3_N_E,	SEL_HSCIF3_4),
+	PINMUX_IPSR_GPSR(IP7_3_0,		LCDOUT10),
+
+	PINMUX_IPSR_GPSR(IP7_7_4,		D12),
+	PINMUX_IPSR_GPSR(IP7_7_4,		CANFD0_TX),
+	PINMUX_IPSR_GPSR(IP7_7_4,		TX4_B),
+	PINMUX_IPSR_GPSR(IP7_7_4,		CAN0_TX),
+	PINMUX_IPSR_MSEL(IP7_7_4,		VI5_DATA8_A,	SEL_VIN5_0),
+	PINMUX_IPSR_MSEL(IP7_7_4,		VI5_DATA3_B,	SEL_VIN5_1),
+
+	PINMUX_IPSR_GPSR(IP7_11_8,		D13),
+	PINMUX_IPSR_GPSR(IP7_11_8,		CANFD0_RX),
+	PINMUX_IPSR_MSEL(IP7_11_8,		RX4_B,		SEL_SCIF4_1),
+	PINMUX_IPSR_GPSR(IP7_11_8,		CAN0_RX),
+	PINMUX_IPSR_MSEL(IP7_11_8,		VI5_DATA9_A,	SEL_VIN5_0),
+	PINMUX_IPSR_MSEL(IP7_11_8,		SCL7_B,		SEL_I2C7_1),
+	PINMUX_IPSR_MSEL(IP7_11_8,		VI5_DATA4_B,	SEL_VIN5_1),
+
+	PINMUX_IPSR_GPSR(IP7_15_12,		D14),
+	PINMUX_IPSR_GPSR(IP7_15_12,		CAN_CLK),
+	PINMUX_IPSR_MSEL(IP7_15_12,		HRX3_A,		SEL_HSCIF3_0),
+	PINMUX_IPSR_GPSR(IP7_15_12,		MSIOF2_SS2_A),
+	PINMUX_IPSR_MSEL(IP7_15_12,		SDA7_B,		SEL_I2C7_1),
+	PINMUX_IPSR_MSEL(IP7_15_12,		VI5_DATA5_B,	SEL_VIN5_1),
+
+	PINMUX_IPSR_GPSR(IP7_19_16,		D15),
+	PINMUX_IPSR_GPSR(IP7_19_16,		MSIOF2_SS1_A),
+	PINMUX_IPSR_GPSR(IP7_19_16,		HTX3_A),
+	PINMUX_IPSR_GPSR(IP7_19_16,		MSIOF3_SS1_A),
+	PINMUX_IPSR_GPSR(IP7_19_16,		DU_DG3),
+	PINMUX_IPSR_GPSR(IP7_19_16,		LCDOUT11),
+
+	PINMUX_IPSR_GPSR(IP7_23_20,		SCL4),
+	PINMUX_IPSR_GPSR(IP7_23_20,		CS1_N_A26),
+	PINMUX_IPSR_GPSR(IP7_23_20,		DU_DOTCLKIN0),
+	PINMUX_IPSR_MSEL(IP7_23_20,		VI4_DATA6_B,	SEL_VIN4_1),
+	PINMUX_IPSR_MSEL(IP7_23_20,		VI5_DATA6_B,	SEL_VIN5_1),
+	PINMUX_IPSR_GPSR(IP7_23_20,		QCLK),
+
+	PINMUX_IPSR_GPSR(IP7_27_24,		SDA4),
+	PINMUX_IPSR_GPSR(IP7_27_24,		WE1_N),
+	PINMUX_IPSR_MSEL(IP7_27_24,		VI4_DATA7_B,	SEL_VIN4_1),
+	PINMUX_IPSR_MSEL(IP7_27_24,		VI5_DATA7_B,	SEL_VIN5_1),
+	PINMUX_IPSR_GPSR(IP7_27_24,		QPOLB),
+
+	PINMUX_IPSR_GPSR(IP7_31_28,		SD0_CLK),
+	PINMUX_IPSR_GPSR(IP7_31_28,		NFDATA8),
+	PINMUX_IPSR_MSEL(IP7_31_28,		SCL1_C,		SEL_I2C1_2),
+	PINMUX_IPSR_MSEL(IP7_31_28,		HSCK1_B,	SEL_HSCIF1_1),
+	PINMUX_IPSR_MSEL(IP7_31_28,		SDA2_E,		SEL_I2C2_4),
+	PINMUX_IPSR_MSEL(IP7_31_28,		FMCLK_B,	SEL_FM_1),
+
+	/* IPSR8 */
+	PINMUX_IPSR_GPSR(IP8_3_0,		SD0_CMD),
+	PINMUX_IPSR_GPSR(IP8_3_0,		NFDATA9),
+	PINMUX_IPSR_MSEL(IP8_3_0,		HRX1_B,		SEL_HSCIF1_1),
+	PINMUX_IPSR_MSEL(IP8_3_0,		SPEEDIN_B,	SEL_SPEED_PULSE_IF_1),
+
+	PINMUX_IPSR_GPSR(IP8_7_4,		SD0_DAT0),
+	PINMUX_IPSR_GPSR(IP8_7_4,		NFDATA10),
+	PINMUX_IPSR_GPSR(IP8_7_4,		HTX1_B),
+	PINMUX_IPSR_MSEL(IP8_7_4,		REMOCON_B,	SEL_REMOCON_1),
+
+	PINMUX_IPSR_GPSR(IP8_11_8,		SD0_DAT1),
+	PINMUX_IPSR_GPSR(IP8_11_8,		NFDATA11),
+	PINMUX_IPSR_MSEL(IP8_11_8,		SDA2_C,		SEL_I2C2_2),
+	PINMUX_IPSR_MSEL(IP8_11_8,		HCTS1_N_B,	SEL_HSCIF1_1),
+	PINMUX_IPSR_MSEL(IP8_11_8,		FMIN_B,		SEL_FM_1),
+
+	PINMUX_IPSR_GPSR(IP8_15_12,		SD0_DAT2),
+	PINMUX_IPSR_GPSR(IP8_15_12,		NFDATA12),
+	PINMUX_IPSR_MSEL(IP8_15_12,		SCL2_C,		SEL_I2C2_2),
+	PINMUX_IPSR_MSEL(IP8_15_12,		HRTS1_N_B,	SEL_HSCIF1_1),
+	PINMUX_IPSR_GPSR(IP8_15_12,		BPFCLK_B),
+
+	PINMUX_IPSR_GPSR(IP8_19_16,		SD0_DAT3),
+	PINMUX_IPSR_GPSR(IP8_19_16,		NFDATA13),
+	PINMUX_IPSR_MSEL(IP8_19_16,		SDA1_C,		SEL_I2C1_2),
+	PINMUX_IPSR_MSEL(IP8_19_16,		SCL2_E,		SEL_I2C2_4),
+	PINMUX_IPSR_MSEL(IP8_19_16,		SPEEDIN_C,	SEL_SPEED_PULSE_IF_2),
+	PINMUX_IPSR_MSEL(IP8_19_16,		REMOCON_C,	SEL_REMOCON_2),
+
+	PINMUX_IPSR_GPSR(IP8_23_20,		SD1_CLK),
+	PINMUX_IPSR_MSEL(IP8_23_20,		NFDATA14_B,	SEL_NDFC_1),
+
+	PINMUX_IPSR_GPSR(IP8_27_24,		SD1_CMD),
+	PINMUX_IPSR_MSEL(IP8_27_24,		NFDATA15_B,	SEL_NDFC_1),
+
+	PINMUX_IPSR_GPSR(IP8_31_28,		SD1_DAT0),
+	PINMUX_IPSR_MSEL(IP8_31_28,		NFWP_N_B,	SEL_NDFC_1),
+
+	/* IPSR9 */
+	PINMUX_IPSR_GPSR(IP9_3_0,		SD1_DAT1),
+	PINMUX_IPSR_MSEL(IP9_3_0,		NFCE_N_B,	SEL_NDFC_1),
+
+	PINMUX_IPSR_GPSR(IP9_7_4,		SD1_DAT2),
+	PINMUX_IPSR_MSEL(IP9_7_4,		NFALE_B,	SEL_NDFC_1),
+
+	PINMUX_IPSR_GPSR(IP9_11_8,		SD1_DAT3),
+	PINMUX_IPSR_MSEL(IP9_11_8,		NFRB_N_B,	SEL_NDFC_1),
+
+	PINMUX_IPSR_GPSR(IP9_15_12,		SD3_CLK),
+	PINMUX_IPSR_GPSR(IP9_15_12,		NFWE_N),
+
+	PINMUX_IPSR_GPSR(IP9_19_16,		SD3_CMD),
+	PINMUX_IPSR_GPSR(IP9_19_16,		NFRE_N),
+
+	PINMUX_IPSR_GPSR(IP9_23_20,		SD3_DAT0),
+	PINMUX_IPSR_GPSR(IP9_23_20,		NFDATA0),
+
+	PINMUX_IPSR_GPSR(IP9_27_24,		SD3_DAT1),
+	PINMUX_IPSR_GPSR(IP9_27_24,		NFDATA1),
+
+	PINMUX_IPSR_GPSR(IP9_31_28,		SD3_DAT2),
+	PINMUX_IPSR_GPSR(IP9_31_28,		NFDATA2),
+
+	/* IPSR10 */
+	PINMUX_IPSR_GPSR(IP10_3_0,		SD3_DAT3),
+	PINMUX_IPSR_GPSR(IP10_3_0,		NFDATA3),
+
+	PINMUX_IPSR_GPSR(IP10_7_4,		SD3_DAT4),
+	PINMUX_IPSR_GPSR(IP10_7_4,		NFDATA4),
+
+	PINMUX_IPSR_GPSR(IP10_11_8,		SD3_DAT5),
+	PINMUX_IPSR_GPSR(IP10_11_8,		NFDATA5),
+
+	PINMUX_IPSR_GPSR(IP10_15_12,		SD3_DAT6),
+	PINMUX_IPSR_GPSR(IP10_15_12,		NFDATA6),
+
+	PINMUX_IPSR_GPSR(IP10_19_16,		SD3_DAT7),
+	PINMUX_IPSR_GPSR(IP10_19_16,		NFDATA7),
+
+	PINMUX_IPSR_GPSR(IP10_23_20,		SD3_DS),
+	PINMUX_IPSR_GPSR(IP10_23_20,		NFCLE),
+
+	PINMUX_IPSR_GPSR(IP10_27_24,		SD0_CD),
+	PINMUX_IPSR_GPSR(IP10_27_24,		NFALE_A),
+	PINMUX_IPSR_GPSR(IP10_27_24,		SD3_CD),
+	PINMUX_IPSR_MSEL(IP10_27_24,		RIF0_CLK_B,	SEL_DRIF0_1),
+	PINMUX_IPSR_MSEL(IP10_27_24,		SCL2_B,		SEL_I2C2_1),
+	PINMUX_IPSR_MSEL(IP10_27_24,		TCLK1_A,	SEL_TIMER_TMU_0),
+	PINMUX_IPSR_MSEL(IP10_27_24,		SSI_SCK2_B,	SEL_SSI2_1),
+	PINMUX_IPSR_GPSR(IP10_27_24,		TS_SCK0),
+
+	PINMUX_IPSR_GPSR(IP10_31_28,		SD0_WP),
+	PINMUX_IPSR_GPSR(IP10_31_28,		NFRB_N_A),
+	PINMUX_IPSR_GPSR(IP10_31_28,		SD3_WP),
+	PINMUX_IPSR_MSEL(IP10_31_28,		RIF0_D0_B,	SEL_DRIF0_1),
+	PINMUX_IPSR_MSEL(IP10_31_28,		SDA2_B,		SEL_I2C2_1),
+	PINMUX_IPSR_MSEL(IP10_31_28,		TCLK2_A,	SEL_TIMER_TMU_0),
+	PINMUX_IPSR_MSEL(IP10_31_28,		SSI_WS2_B,	SEL_SSI2_1),
+	PINMUX_IPSR_GPSR(IP10_31_28,		TS_SDAT0),
+
+	/* IPSR11 */
+	PINMUX_IPSR_GPSR(IP11_3_0,		SD1_CD),
+	PINMUX_IPSR_MSEL(IP11_3_0,		NFCE_N_A,	SEL_NDFC_0),
+	PINMUX_IPSR_GPSR(IP11_3_0,		SSI_SCK1),
+	PINMUX_IPSR_MSEL(IP11_3_0,		RIF0_D1_B,	SEL_DRIF0_1),
+	PINMUX_IPSR_GPSR(IP11_3_0,		TS_SDEN0),
+
+	PINMUX_IPSR_GPSR(IP11_7_4,		SD1_WP),
+	PINMUX_IPSR_MSEL(IP11_7_4,		NFWP_N_A,	SEL_NDFC_0),
+	PINMUX_IPSR_GPSR(IP11_7_4,		SSI_WS1),
+	PINMUX_IPSR_MSEL(IP11_7_4,		RIF0_SYNC_B,	SEL_DRIF0_1),
+	PINMUX_IPSR_GPSR(IP11_7_4,		TS_SPSYNC0),
+
+	PINMUX_IPSR_MSEL(IP11_11_8,		RX0_A,		SEL_SCIF0_0),
+	PINMUX_IPSR_MSEL(IP11_11_8,		HRX1_A,		SEL_HSCIF1_0),
+	PINMUX_IPSR_MSEL(IP11_11_8,		SSI_SCK2_A,	SEL_SSI2_0),
+	PINMUX_IPSR_GPSR(IP11_11_8,		RIF1_SYNC),
+	PINMUX_IPSR_GPSR(IP11_11_8,		TS_SCK1),
+
+	PINMUX_IPSR_GPSR(IP11_15_12,		TX0_A),
+	PINMUX_IPSR_GPSR(IP11_15_12,		HTX1_A),
+	PINMUX_IPSR_MSEL(IP11_15_12,		SSI_WS2_A,	SEL_SSI2_0),
+	PINMUX_IPSR_GPSR(IP11_15_12,		RIF1_D0),
+	PINMUX_IPSR_GPSR(IP11_15_12,		TS_SDAT1),
+
+	PINMUX_IPSR_MSEL(IP11_19_16,		CTS0_N_A,	SEL_SCIF0_0),
+	PINMUX_IPSR_MSEL(IP11_19_16,		NFDATA14_A,	SEL_NDFC_0),
+	PINMUX_IPSR_GPSR(IP11_19_16,		AUDIO_CLKOUT_A),
+	PINMUX_IPSR_GPSR(IP11_19_16,		RIF1_D1),
+	PINMUX_IPSR_MSEL(IP11_19_16,		SCIF_CLK_A,	SEL_SCIF_0),
+	PINMUX_IPSR_MSEL(IP11_19_16,		FMCLK_A,	SEL_FM_0),
+
+	PINMUX_IPSR_MSEL(IP11_23_20,		RTS0_N_TANS_A,	SEL_SCIF0_0),
+	PINMUX_IPSR_MSEL(IP11_23_20,		NFDATA15_A,	SEL_NDFC_0),
+	PINMUX_IPSR_GPSR(IP11_23_20,		AUDIO_CLKOUT1_A),
+	PINMUX_IPSR_GPSR(IP11_23_20,		RIF1_CLK),
+	PINMUX_IPSR_MSEL(IP11_23_20,		SCL2_A,		SEL_I2C2_0),
+	PINMUX_IPSR_MSEL(IP11_23_20,		FMIN_A,		SEL_FM_0),
+
+	PINMUX_IPSR_MSEL(IP11_27_24,		SCK0_A,		SEL_SCIF0_0),
+	PINMUX_IPSR_MSEL(IP11_27_24,		HSCK1_A,	SEL_HSCIF1_0),
+	PINMUX_IPSR_GPSR(IP11_27_24,		USB3HS0_ID),
+	PINMUX_IPSR_GPSR(IP11_27_24,		RTS1_N_TANS),
+	PINMUX_IPSR_MSEL(IP11_27_24,		SDA2_A,		SEL_I2C2_0),
+	PINMUX_IPSR_MSEL(IP11_27_24,		FMCLK_C,	SEL_FM_2),
+	PINMUX_IPSR_GPSR(IP11_27_24,		USB1_ID),
+
+	PINMUX_IPSR_GPSR(IP11_31_28,		RX1),
+	PINMUX_IPSR_MSEL(IP11_31_28,		HRX2_B,		SEL_HSCIF2_1),
+	PINMUX_IPSR_MSEL(IP11_31_28,		SSI_SCK9_B,	SEL_SSI9_1),
+	PINMUX_IPSR_GPSR(IP11_31_28,		AUDIO_CLKOUT1_B),
+
+	/* IPSR12 */
+	PINMUX_IPSR_GPSR(IP12_3_0,		TX1),
+	PINMUX_IPSR_GPSR(IP12_3_0,		HTX2_B),
+	PINMUX_IPSR_MSEL(IP12_3_0,		SSI_WS9_B,	SEL_SSI9_1),
+	PINMUX_IPSR_GPSR(IP12_3_0,		AUDIO_CLKOUT3_B),
+
+	PINMUX_IPSR_GPSR(IP12_7_4,		SCK2_A),
+	PINMUX_IPSR_MSEL(IP12_7_4,		HSCK0_A,	SEL_HSCIF0_0),
+	PINMUX_IPSR_MSEL(IP12_7_4,		AUDIO_CLKB_A,	SEL_ADGB_0),
+	PINMUX_IPSR_GPSR(IP12_7_4,		CTS1_N),
+	PINMUX_IPSR_MSEL(IP12_7_4,		RIF0_CLK_A,	SEL_DRIF0_0),
+	PINMUX_IPSR_MSEL(IP12_7_4,		REMOCON_A,	SEL_REMOCON_0),
+	PINMUX_IPSR_MSEL(IP12_7_4,		SCIF_CLK_B,	SEL_SCIF_1),
+
+	PINMUX_IPSR_GPSR(IP12_11_8,		TX2_A),
+	PINMUX_IPSR_MSEL(IP12_11_8,		HRX0_A,		SEL_HSCIF0_0),
+	PINMUX_IPSR_GPSR(IP12_11_8,		AUDIO_CLKOUT2_A),
+	PINMUX_IPSR_MSEL(IP12_11_8,		SCL1_A,		SEL_I2C1_0),
+	PINMUX_IPSR_MSEL(IP12_11_8,		FSO_CFE_0_N_A,	SEL_FSO_0),
+	PINMUX_IPSR_GPSR(IP12_11_8,		TS_SDEN1),
+
+	PINMUX_IPSR_GPSR(IP12_15_12,		RX2_A),
+	PINMUX_IPSR_GPSR(IP12_15_12,		HTX0_A),
+	PINMUX_IPSR_GPSR(IP12_15_12,		AUDIO_CLKOUT3_A),
+	PINMUX_IPSR_MSEL(IP12_15_12,		SDA1_A,		SEL_I2C1_0),
+	PINMUX_IPSR_MSEL(IP12_15_12,		FSO_CFE_1_N_A,	SEL_FSO_0),
+	PINMUX_IPSR_GPSR(IP12_15_12,		TS_SPSYNC1),
+
+	PINMUX_IPSR_GPSR(IP12_19_16,		MSIOF0_SCK),
+	PINMUX_IPSR_GPSR(IP12_19_16,		SSI_SCK78),
+
+	PINMUX_IPSR_GPSR(IP12_23_20,		MSIOF0_RXD),
+	PINMUX_IPSR_GPSR(IP12_23_20,		SSI_WS78),
+	PINMUX_IPSR_GPSR(IP12_23_20,		TX2_B),
+
+	PINMUX_IPSR_GPSR(IP12_27_24,		MSIOF0_TXD),
+	PINMUX_IPSR_GPSR(IP12_27_24,		SSI_SDATA7),
+	PINMUX_IPSR_GPSR(IP12_27_24,		RX2_B),
+
+	PINMUX_IPSR_GPSR(IP12_31_28,		MSIOF0_SYNC),
+	PINMUX_IPSR_GPSR(IP12_31_28,		AUDIO_CLKOUT_B),
+	PINMUX_IPSR_GPSR(IP12_31_28,		SSI_SDATA8),
+
+	/* IPSR13 */
+	PINMUX_IPSR_GPSR(IP13_3_0,		MSIOF0_SS1),
+	PINMUX_IPSR_MSEL(IP13_3_0,		HRX2_A,		SEL_HSCIF2_0),
+	PINMUX_IPSR_GPSR(IP13_3_0,		SSI_SCK4),
+	PINMUX_IPSR_MSEL(IP13_3_0,		HCTS0_N_A,	SEL_HSCIF0_0),
+	PINMUX_IPSR_GPSR(IP13_3_0,		BPFCLK_C),
+	PINMUX_IPSR_MSEL(IP13_3_0,		SPEEDIN_A,	SEL_SPEED_PULSE_IF_0),
+
+	PINMUX_IPSR_GPSR(IP13_7_4,		MSIOF0_SS2),
+	PINMUX_IPSR_GPSR(IP13_7_4,		HTX2_A),
+	PINMUX_IPSR_GPSR(IP13_7_4,		SSI_WS4),
+	PINMUX_IPSR_MSEL(IP13_7_4,		HRTS0_N_A,	SEL_HSCIF0_0),
+	PINMUX_IPSR_MSEL(IP13_7_4,		FMIN_C,		SEL_FM_2),
+	PINMUX_IPSR_GPSR(IP13_7_4,		BPFCLK_A),
+
+	PINMUX_IPSR_GPSR(IP13_11_8,		SSI_SDATA9),
+	PINMUX_IPSR_MSEL(IP13_11_8,		AUDIO_CLKC_A,	SEL_ADGC_0),
+	PINMUX_IPSR_GPSR(IP13_11_8,		SCK1),
+
+	PINMUX_IPSR_GPSR(IP13_15_12,		MLB_CLK),
+	PINMUX_IPSR_MSEL(IP13_15_12,		RX0_B,		SEL_SCIF0_1),
+	PINMUX_IPSR_MSEL(IP13_15_12,		RIF0_D0_A,	SEL_DRIF0_0),
+	PINMUX_IPSR_MSEL(IP13_15_12,		SCL1_B,		SEL_I2C1_1),
+	PINMUX_IPSR_MSEL(IP13_15_12,		TCLK1_B,	SEL_TIMER_TMU_1),
+	PINMUX_IPSR_GPSR(IP13_15_12,		SIM0_RST_A),
+
+	PINMUX_IPSR_GPSR(IP13_19_16,		MLB_SIG),
+	PINMUX_IPSR_MSEL(IP13_19_16,		SCK0_B,		SEL_SCIF0_1),
+	PINMUX_IPSR_MSEL(IP13_19_16,		RIF0_D1_A,	SEL_DRIF0_0),
+	PINMUX_IPSR_MSEL(IP13_19_16,		SDA1_B,		SEL_I2C1_1),
+	PINMUX_IPSR_MSEL(IP13_19_16,		TCLK2_B,	SEL_TIMER_TMU_1),
+	PINMUX_IPSR_MSEL(IP13_19_16,		SIM0_D_A,	SEL_SIMCARD_0),
+
+	PINMUX_IPSR_GPSR(IP13_23_20,		MLB_DAT),
+	PINMUX_IPSR_GPSR(IP13_23_20,		TX0_B),
+	PINMUX_IPSR_MSEL(IP13_23_20,		RIF0_SYNC_A,	SEL_DRIF0_0),
+	PINMUX_IPSR_GPSR(IP13_23_20,		SIM0_CLK_A),
+
+	PINMUX_IPSR_GPSR(IP13_27_24,		SSI_SCK01239),
+
+	PINMUX_IPSR_GPSR(IP13_31_28,		SSI_WS01239),
+
+	/* IPSR14 */
+	PINMUX_IPSR_GPSR(IP14_3_0,		SSI_SDATA0),
+
+	PINMUX_IPSR_GPSR(IP14_7_4,		SSI_SDATA1),
+	PINMUX_IPSR_MSEL(IP14_7_4,		AUDIO_CLKC_B,	SEL_ADGC_1),
+	PINMUX_IPSR_MSEL(IP14_7_4,		PWM0_B,		SEL_PWM0_1),
+
+	PINMUX_IPSR_GPSR(IP14_11_8,		SSI_SDATA2),
+	PINMUX_IPSR_GPSR(IP14_11_8,		AUDIO_CLKOUT2_B),
+	PINMUX_IPSR_MSEL(IP14_11_8,		SSI_SCK9_A,	SEL_SSI9_0),
+	PINMUX_IPSR_MSEL(IP14_11_8,		PWM1_B,		SEL_PWM1_1),
+
+	PINMUX_IPSR_GPSR(IP14_15_12,		SSI_SCK349),
+	PINMUX_IPSR_MSEL(IP14_15_12,		PWM2_C,		SEL_PWM2_2),
+
+	PINMUX_IPSR_GPSR(IP14_19_16,		SSI_WS349),
+	PINMUX_IPSR_MSEL(IP14_19_16,		PWM3_C,		SEL_PWM3_2),
+
+	PINMUX_IPSR_GPSR(IP14_23_20,		SSI_SDATA3),
+	PINMUX_IPSR_GPSR(IP14_23_20,		AUDIO_CLKOUT1_C),
+	PINMUX_IPSR_MSEL(IP14_23_20,		AUDIO_CLKB_B,	SEL_ADGB_1),
+	PINMUX_IPSR_MSEL(IP14_23_20,		PWM4_B,		SEL_PWM4_1),
+
+	PINMUX_IPSR_GPSR(IP14_27_24,		SSI_SDATA4),
+	PINMUX_IPSR_MSEL(IP14_27_24,		SSI_WS9_A,	SEL_SSI9_0),
+	PINMUX_IPSR_MSEL(IP14_27_24,		PWM5_B,		SEL_PWM5_1),
+
+	PINMUX_IPSR_GPSR(IP14_31_28,		SSI_SCK5),
+	PINMUX_IPSR_MSEL(IP14_31_28,		HRX0_B,		SEL_HSCIF0_1),
+	PINMUX_IPSR_GPSR(IP14_31_28,		USB0_PWEN_B),
+	PINMUX_IPSR_MSEL(IP14_31_28,		SCL2_D,		SEL_I2C2_3),
+	PINMUX_IPSR_MSEL(IP14_31_28,		PWM6_B,		SEL_PWM6_1),
+
+	/* IPSR15 */
+	PINMUX_IPSR_GPSR(IP15_3_0,		SSI_WS5),
+	PINMUX_IPSR_GPSR(IP15_3_0,		HTX0_B),
+	PINMUX_IPSR_MSEL(IP15_3_0,		USB0_OVC_B,	SEL_USB_20_CH0_1),
+	PINMUX_IPSR_MSEL(IP15_3_0,		SDA2_D,		SEL_I2C2_3),
+
+	PINMUX_IPSR_GPSR(IP15_7_4,		SSI_SDATA5),
+	PINMUX_IPSR_MSEL(IP15_7_4,		HSCK0_B,	SEL_HSCIF0_1),
+	PINMUX_IPSR_MSEL(IP15_7_4,		AUDIO_CLKB_C,	SEL_ADGB_2),
+	PINMUX_IPSR_GPSR(IP15_7_4,		TPU0TO0),
+
+	PINMUX_IPSR_GPSR(IP15_11_8,		SSI_SCK6),
+	PINMUX_IPSR_MSEL(IP15_11_8,		HSCK2_A,	SEL_HSCIF2_0),
+	PINMUX_IPSR_MSEL(IP15_11_8,		AUDIO_CLKC_C,	SEL_ADGC_2),
+	PINMUX_IPSR_GPSR(IP15_11_8,		TPU0TO1),
+	PINMUX_IPSR_MSEL(IP15_11_8,		FSO_CFE_0_N_B,	SEL_FSO_1),
+	PINMUX_IPSR_GPSR(IP15_11_8,		SIM0_RST_B),
+
+	PINMUX_IPSR_GPSR(IP15_15_12,		SSI_WS6),
+	PINMUX_IPSR_MSEL(IP15_15_12,		HCTS2_N_A,	SEL_HSCIF2_0),
+	PINMUX_IPSR_GPSR(IP15_15_12,		AUDIO_CLKOUT2_C),
+	PINMUX_IPSR_GPSR(IP15_15_12,		TPU0TO2),
+	PINMUX_IPSR_MSEL(IP15_15_12,		SDA1_D,		SEL_I2C1_3),
+	PINMUX_IPSR_MSEL(IP15_15_12,		FSO_CFE_1_N_B,	SEL_FSO_1),
+	PINMUX_IPSR_MSEL(IP15_15_12,		SIM0_D_B,	SEL_SIMCARD_1),
+
+	PINMUX_IPSR_GPSR(IP15_19_16,		SSI_SDATA6),
+	PINMUX_IPSR_MSEL(IP15_19_16,		HRTS2_N_A,	SEL_HSCIF2_0),
+	PINMUX_IPSR_GPSR(IP15_19_16,		AUDIO_CLKOUT3_C),
+	PINMUX_IPSR_GPSR(IP15_19_16,		TPU0TO3),
+	PINMUX_IPSR_MSEL(IP15_19_16,		SCL1_D,		SEL_I2C1_3),
+	PINMUX_IPSR_MSEL(IP15_19_16,		FSO_TOE_N_B,	SEL_FSO_1),
+	PINMUX_IPSR_GPSR(IP15_19_16,		SIM0_CLK_B),
+
+	PINMUX_IPSR_GPSR(IP15_23_20,		AUDIO_CLKA),
+
+	PINMUX_IPSR_GPSR(IP15_27_24,		USB30_PWEN),
+	PINMUX_IPSR_GPSR(IP15_27_24,		USB0_PWEN_A),
+
+	PINMUX_IPSR_GPSR(IP15_31_28,		USB30_OVC),
+	PINMUX_IPSR_MSEL(IP15_31_28,		USB0_OVC_A,	SEL_USB_20_CH0_0),
+
+/*
+ * Static pins can not be muxed between different functions but
+ * still need mark entries in the pinmux list. Add each static
+ * pin to the list without an associated function. The sh-pfc
+ * core will do the right thing and skip trying to mux the pin
+ * while still applying configuration to it.
+ */
+#define FM(x)   PINMUX_DATA(x##_MARK, 0),
+	PINMUX_STATIC
+#undef FM
+};
+
+/*
+ * R8A77990 has 7 banks with 32 GPIOs in each => 224 GPIOs.
+ * Physical layout rows: A - AE, cols: 1 - 25.
+ */
+#define ROW_GROUP_A(r) ('Z' - 'A' + 1 + (r))
+#define PIN_NUMBER(r, c) (((r) - 'A') * 25 + (c) + 300)
+#define PIN_A_NUMBER(r, c) PIN_NUMBER(ROW_GROUP_A(r), c)
+#define PIN_NONE U16_MAX
+
+static const struct sh_pfc_pin pinmux_pins[] = {
+	PINMUX_GPIO_GP_ALL(),
+
+	/*
+	 * Pins not associated with a GPIO port.
+	 *
+	 * The pin positions are different between different R8A77990
+	 * packages, all that is needed for the pfc driver is a unique
+	 * number for each pin. To this end use the pin layout from
+	 * R8A77990 to calculate a unique number for each pin.
+	 */
+	SH_PFC_PIN_NAMED_CFG('F',  1, TRST_N,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('F',  3, TMS,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('F',  4, TCK,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('G',  2, TDI,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('G',  3, FSCLKST_N,	CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('H',  1, ASEBRK,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('N',  1, AVB_TXC,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('N',  2, AVB_TD0,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('N',  3, AVB_TD1,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('N',  5, AVB_TD2,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('N',  6, AVB_TD3,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('P',  3, AVB_TX_CTL,	CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('P',  4, AVB_MDIO,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('P',  5, AVB_MDC,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG('T', 21, MLB_REF,		CFG_FLAGS),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('D'), 3, PRESETOUT_N, CFG_FLAGS),
+};
+
+/* - EtherAVB --------------------------------------------------------------- */
+static const unsigned int avb_link_pins[] = {
+	/* AVB_LINK */
+	RCAR_GP_PIN(2, 23),
+};
+
+static const unsigned int avb_link_mux[] = {
+	AVB_LINK_MARK,
+};
+
+static const unsigned int avb_magic_pins[] = {
+	/* AVB_MAGIC */
+	RCAR_GP_PIN(2, 22),
+};
+
+static const unsigned int avb_magic_mux[] = {
+	AVB_MAGIC_MARK,
+};
+
+static const unsigned int avb_phy_int_pins[] = {
+	/* AVB_PHY_INT */
+	RCAR_GP_PIN(2, 21),
+};
+
+static const unsigned int avb_phy_int_mux[] = {
+	AVB_PHY_INT_MARK,
+};
+
+static const unsigned int avb_mii_pins[] = {
+	/*
+	 * AVB_RX_CTL, AVB_RXC, AVB_RD0,
+	 * AVB_RD1, AVB_RD2, AVB_RD3,
+	 * AVB_TXCREFCLK
+	 */
+	RCAR_GP_PIN(2, 14), RCAR_GP_PIN(2, 15), RCAR_GP_PIN(2, 16),
+	RCAR_GP_PIN(2, 17), RCAR_GP_PIN(2, 18), RCAR_GP_PIN(2, 19),
+	RCAR_GP_PIN(2, 20),
+};
+
+static const unsigned int avb_mii_mux[] = {
+	AVB_RX_CTL_MARK, AVB_RXC_MARK, AVB_RD0_MARK,
+	AVB_RD1_MARK, AVB_RD2_MARK, AVB_RD3_MARK,
+	AVB_TXCREFCLK_MARK,
+};
+
+static const unsigned int avb_avtp_pps_pins[] = {
+	/* AVB_AVTP_PPS */
+	RCAR_GP_PIN(1, 2),
+};
+
+static const unsigned int avb_avtp_pps_mux[] = {
+	AVB_AVTP_PPS_MARK,
+};
+
+static const unsigned int avb_avtp_match_a_pins[] = {
+	/* AVB_AVTP_MATCH_A */
+	RCAR_GP_PIN(2, 24),
+};
+
+static const unsigned int avb_avtp_match_a_mux[] = {
+	AVB_AVTP_MATCH_A_MARK,
+};
+
+static const unsigned int avb_avtp_capture_a_pins[] = {
+	/* AVB_AVTP_CAPTURE_A */
+	RCAR_GP_PIN(2, 25),
+};
+
+static const unsigned int avb_avtp_capture_a_mux[] = {
+	AVB_AVTP_CAPTURE_A_MARK,
+};
+
+/* - I2C -------------------------------------------------------------------- */
+static const unsigned int i2c1_a_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(5, 8), RCAR_GP_PIN(5, 9),
+};
+
+static const unsigned int i2c1_a_mux[] = {
+	SCL1_A_MARK, SDA1_A_MARK,
+};
+
+static const unsigned int i2c1_b_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(5, 17), RCAR_GP_PIN(5, 18),
+};
+
+static const unsigned int i2c1_b_mux[] = {
+	SCL1_B_MARK, SDA1_B_MARK,
+};
+
+static const unsigned int i2c1_c_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(3, 0), RCAR_GP_PIN(3, 5),
+};
+
+static const unsigned int i2c1_c_mux[] = {
+	SCL1_C_MARK, SDA1_C_MARK,
+};
+
+static const unsigned int i2c1_d_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(6, 16), RCAR_GP_PIN(6, 15),
+};
+
+static const unsigned int i2c1_d_mux[] = {
+	SCL1_D_MARK, SDA1_D_MARK,
+};
+
+static const unsigned int i2c2_a_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(5, 4), RCAR_GP_PIN(5, 0),
+};
+
+static const unsigned int i2c2_a_mux[] = {
+	SCL2_A_MARK, SDA2_A_MARK,
+};
+
+static const unsigned int i2c2_b_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(3, 12), RCAR_GP_PIN(3, 13),
+};
+
+static const unsigned int i2c2_b_mux[] = {
+	SCL2_B_MARK, SDA2_B_MARK,
+};
+
+static const unsigned int i2c2_c_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(3, 4), RCAR_GP_PIN(3, 3),
+};
+
+static const unsigned int i2c2_c_mux[] = {
+	SCL2_C_MARK, SDA2_C_MARK,
+};
+
+static const unsigned int i2c2_d_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(6, 11), RCAR_GP_PIN(6, 12),
+};
+
+static const unsigned int i2c2_d_mux[] = {
+	SCL2_D_MARK, SDA2_D_MARK,
+};
+
+static const unsigned int i2c2_e_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(3, 5), RCAR_GP_PIN(3, 0),
+};
+
+static const unsigned int i2c2_e_mux[] = {
+	SCL2_E_MARK, SDA2_E_MARK,
+};
+
+static const unsigned int i2c4_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(0, 16), RCAR_GP_PIN(0, 17),
+};
+
+static const unsigned int i2c4_mux[] = {
+	SCL4_MARK, SDA4_MARK,
+};
+
+static const unsigned int i2c5_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(1, 21), RCAR_GP_PIN(1, 22),
+};
+
+static const unsigned int i2c5_mux[] = {
+	SCL5_MARK, SDA5_MARK,
+};
+
+static const unsigned int i2c6_a_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(1, 11), RCAR_GP_PIN(1, 8),
+};
+
+static const unsigned int i2c6_a_mux[] = {
+	SCL6_A_MARK, SDA6_A_MARK,
+};
+
+static const unsigned int i2c6_b_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(1, 2), RCAR_GP_PIN(1, 1),
+};
+
+static const unsigned int i2c6_b_mux[] = {
+	SCL6_B_MARK, SDA6_B_MARK,
+};
+
+static const unsigned int i2c7_a_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(2, 24), RCAR_GP_PIN(2, 25),
+};
+
+static const unsigned int i2c7_a_mux[] = {
+	SCL7_A_MARK, SDA7_A_MARK,
+};
+
+static const unsigned int i2c7_b_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(0, 13), RCAR_GP_PIN(0, 14),
+};
+
+static const unsigned int i2c7_b_mux[] = {
+	SCL7_B_MARK, SDA7_B_MARK,
+};
+
+/* - SCIF0 ------------------------------------------------------------------ */
+static const unsigned int scif0_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 1), RCAR_GP_PIN(5, 2),
+};
+
+static const unsigned int scif0_data_a_mux[] = {
+	RX0_A_MARK, TX0_A_MARK,
+};
+
+static const unsigned int scif0_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 0),
+};
+
+static const unsigned int scif0_clk_a_mux[] = {
+	SCK0_A_MARK,
+};
+
+static const unsigned int scif0_ctrl_a_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(5, 4), RCAR_GP_PIN(5, 3),
+};
+
+static const unsigned int scif0_ctrl_a_mux[] = {
+	RTS0_N_TANS_A_MARK, CTS0_N_A_MARK,
+};
+
+static const unsigned int scif0_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 17), RCAR_GP_PIN(5, 19),
+};
+
+static const unsigned int scif0_data_b_mux[] = {
+	RX0_B_MARK, TX0_B_MARK,
+};
+
+static const unsigned int scif0_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 18),
+};
+
+static const unsigned int scif0_clk_b_mux[] = {
+	SCK0_B_MARK,
+};
+
+/* - SCIF1 ------------------------------------------------------------------ */
+static const unsigned int scif1_data_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 5), RCAR_GP_PIN(5, 6),
+};
+
+static const unsigned int scif1_data_mux[] = {
+	RX1_MARK, TX1_MARK,
+};
+
+static const unsigned int scif1_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 16),
+};
+
+static const unsigned int scif1_clk_mux[] = {
+	SCK1_MARK,
+};
+
+static const unsigned int scif1_ctrl_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(5, 0), RCAR_GP_PIN(5, 7),
+};
+
+static const unsigned int scif1_ctrl_mux[] = {
+	RTS1_N_TANS_MARK, CTS1_N_MARK,
+};
+
+/* - SCIF2 ------------------------------------------------------------------ */
+static const unsigned int scif2_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 9), RCAR_GP_PIN(5, 8),
+};
+
+static const unsigned int scif2_data_a_mux[] = {
+	RX2_A_MARK, TX2_A_MARK,
+};
+
+static const unsigned int scif2_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 7),
+};
+
+static const unsigned int scif2_clk_a_mux[] = {
+	SCK2_A_MARK,
+};
+
+static const unsigned int scif2_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 12), RCAR_GP_PIN(5, 11),
+};
+
+static const unsigned int scif2_data_b_mux[] = {
+	RX2_B_MARK, TX2_B_MARK,
+};
+
+/* - SCIF3 ------------------------------------------------------------------ */
+static const unsigned int scif3_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(0, 5), RCAR_GP_PIN(0, 6),
+};
+
+static const unsigned int scif3_data_a_mux[] = {
+	RX3_A_MARK, TX3_A_MARK,
+};
+
+static const unsigned int scif3_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(0, 1),
+};
+
+static const unsigned int scif3_clk_a_mux[] = {
+	SCK3_A_MARK,
+};
+
+static const unsigned int scif3_ctrl_a_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(0, 4), RCAR_GP_PIN(0, 7),
+};
+
+static const unsigned int scif3_ctrl_a_mux[] = {
+	RTS3_N_TANS_A_MARK, CTS3_N_A_MARK,
+};
+
+static const unsigned int scif3_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 8), RCAR_GP_PIN(1, 11),
+};
+
+static const unsigned int scif3_data_b_mux[] = {
+	RX3_B_MARK, TX3_B_MARK,
+};
+
+static const unsigned int scif3_data_c_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 23), RCAR_GP_PIN(2, 22),
+};
+
+static const unsigned int scif3_data_c_mux[] = {
+	RX3_C_MARK, TX3_C_MARK,
+};
+
+static const unsigned int scif3_clk_c_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(2, 24),
+};
+
+static const unsigned int scif3_clk_c_mux[] = {
+	SCK3_C_MARK,
+};
+
+/* - SCIF4 ------------------------------------------------------------------ */
+static const unsigned int scif4_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 6), RCAR_GP_PIN(1, 7),
+};
+
+static const unsigned int scif4_data_a_mux[] = {
+	RX4_A_MARK, TX4_A_MARK,
+};
+
+static const unsigned int scif4_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 5),
+};
+
+static const unsigned int scif4_clk_a_mux[] = {
+	SCK4_A_MARK,
+};
+
+static const unsigned int scif4_ctrl_a_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(1, 4), RCAR_GP_PIN(1, 3),
+};
+
+static const unsigned int scif4_ctrl_a_mux[] = {
+	RTS4_N_TANS_A_MARK, CTS4_N_A_MARK,
+};
+
+static const unsigned int scif4_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(0, 13), RCAR_GP_PIN(0, 12),
+};
+
+static const unsigned int scif4_data_b_mux[] = {
+	RX4_B_MARK, TX4_B_MARK,
+};
+
+static const unsigned int scif4_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(0, 8),
+};
+
+static const unsigned int scif4_clk_b_mux[] = {
+	SCK4_B_MARK,
+};
+
+static const unsigned int scif4_data_c_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(0, 2), RCAR_GP_PIN(0, 3),
+};
+
+static const unsigned int scif4_data_c_mux[] = {
+	RX4_C_MARK, TX4_C_MARK,
+};
+
+static const unsigned int scif4_ctrl_c_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(0, 1), RCAR_GP_PIN(0, 0),
+};
+
+static const unsigned int scif4_ctrl_c_mux[] = {
+	RTS4_N_TANS_C_MARK, CTS4_N_C_MARK,
+};
+
+/* - SCIF5 ------------------------------------------------------------------ */
+static const unsigned int scif5_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 12), RCAR_GP_PIN(1, 9),
+};
+
+static const unsigned int scif5_data_a_mux[] = {
+	RX5_A_MARK, TX5_A_MARK,
+};
+
+static const unsigned int scif5_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 13),
+};
+
+static const unsigned int scif5_clk_a_mux[] = {
+	SCK5_A_MARK,
+};
+
+static const unsigned int scif5_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 25), RCAR_GP_PIN(2, 24),
+};
+
+static const unsigned int scif5_data_b_mux[] = {
+	RX5_B_MARK, TX5_B_MARK,
+};
+
+static const unsigned int scif5_data_c_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(0, 2), RCAR_GP_PIN(0, 3),
+};
+
+static const unsigned int scif5_data_c_mux[] = {
+	RX5_C_MARK, TX5_C_MARK,
+};
+
+/* - SCIF Clock ------------------------------------------------------------- */
+static const unsigned int scif_clk_a_pins[] = {
+	/* SCIF_CLK */
+	RCAR_GP_PIN(5, 3),
+};
+
+static const unsigned int scif_clk_a_mux[] = {
+	SCIF_CLK_A_MARK,
+};
+
+static const unsigned int scif_clk_b_pins[] = {
+	/* SCIF_CLK */
+	RCAR_GP_PIN(5, 7),
+};
+
+static const unsigned int scif_clk_b_mux[] = {
+	SCIF_CLK_B_MARK,
+};
+
+static const struct sh_pfc_pin_group pinmux_groups[] = {
+	SH_PFC_PIN_GROUP(avb_link),
+	SH_PFC_PIN_GROUP(avb_magic),
+	SH_PFC_PIN_GROUP(avb_phy_int),
+	SH_PFC_PIN_GROUP(avb_mii),
+	SH_PFC_PIN_GROUP(avb_avtp_pps),
+	SH_PFC_PIN_GROUP(avb_avtp_match_a),
+	SH_PFC_PIN_GROUP(avb_avtp_capture_a),
+	SH_PFC_PIN_GROUP(i2c1_a),
+	SH_PFC_PIN_GROUP(i2c1_b),
+	SH_PFC_PIN_GROUP(i2c1_c),
+	SH_PFC_PIN_GROUP(i2c1_d),
+	SH_PFC_PIN_GROUP(i2c2_a),
+	SH_PFC_PIN_GROUP(i2c2_b),
+	SH_PFC_PIN_GROUP(i2c2_c),
+	SH_PFC_PIN_GROUP(i2c2_d),
+	SH_PFC_PIN_GROUP(i2c2_e),
+	SH_PFC_PIN_GROUP(i2c4),
+	SH_PFC_PIN_GROUP(i2c5),
+	SH_PFC_PIN_GROUP(i2c6_a),
+	SH_PFC_PIN_GROUP(i2c6_b),
+	SH_PFC_PIN_GROUP(i2c7_a),
+	SH_PFC_PIN_GROUP(i2c7_b),
+	SH_PFC_PIN_GROUP(scif0_data_a),
+	SH_PFC_PIN_GROUP(scif0_clk_a),
+	SH_PFC_PIN_GROUP(scif0_ctrl_a),
+	SH_PFC_PIN_GROUP(scif0_data_b),
+	SH_PFC_PIN_GROUP(scif0_clk_b),
+	SH_PFC_PIN_GROUP(scif1_data),
+	SH_PFC_PIN_GROUP(scif1_clk),
+	SH_PFC_PIN_GROUP(scif1_ctrl),
+	SH_PFC_PIN_GROUP(scif2_data_a),
+	SH_PFC_PIN_GROUP(scif2_clk_a),
+	SH_PFC_PIN_GROUP(scif2_data_b),
+	SH_PFC_PIN_GROUP(scif3_data_a),
+	SH_PFC_PIN_GROUP(scif3_clk_a),
+	SH_PFC_PIN_GROUP(scif3_ctrl_a),
+	SH_PFC_PIN_GROUP(scif3_data_b),
+	SH_PFC_PIN_GROUP(scif3_data_c),
+	SH_PFC_PIN_GROUP(scif3_clk_c),
+	SH_PFC_PIN_GROUP(scif4_data_a),
+	SH_PFC_PIN_GROUP(scif4_clk_a),
+	SH_PFC_PIN_GROUP(scif4_ctrl_a),
+	SH_PFC_PIN_GROUP(scif4_data_b),
+	SH_PFC_PIN_GROUP(scif4_clk_b),
+	SH_PFC_PIN_GROUP(scif4_data_c),
+	SH_PFC_PIN_GROUP(scif4_ctrl_c),
+	SH_PFC_PIN_GROUP(scif5_data_a),
+	SH_PFC_PIN_GROUP(scif5_clk_a),
+	SH_PFC_PIN_GROUP(scif5_data_b),
+	SH_PFC_PIN_GROUP(scif5_data_c),
+	SH_PFC_PIN_GROUP(scif_clk_a),
+	SH_PFC_PIN_GROUP(scif_clk_b),
+};
+
+static const char * const avb_groups[] = {
+	"avb_link",
+	"avb_magic",
+	"avb_phy_int",
+	"avb_mii",
+	"avb_avtp_pps",
+	"avb_avtp_match_a",
+	"avb_avtp_capture_a",
+};
+
+static const char * const i2c1_groups[] = {
+	"i2c1_a",
+	"i2c1_b",
+	"i2c1_c",
+	"i2c1_d",
+};
+
+static const char * const i2c2_groups[] = {
+	"i2c2_a",
+	"i2c2_b",
+	"i2c2_c",
+	"i2c2_d",
+	"i2c2_e",
+};
+
+static const char * const i2c4_groups[] = {
+	"i2c4",
+};
+
+static const char * const i2c5_groups[] = {
+	"i2c5",
+};
+
+static const char * const i2c6_groups[] = {
+	"i2c6_a",
+	"i2c6_b",
+};
+
+static const char * const i2c7_groups[] = {
+	"i2c7_a",
+	"i2c7_b",
+};
+
+static const char * const scif0_groups[] = {
+	"scif0_data_a",
+	"scif0_clk_a",
+	"scif0_ctrl_a",
+	"scif0_data_b",
+	"scif0_clk_b",
+};
+
+static const char * const scif1_groups[] = {
+	"scif1_data",
+	"scif1_clk",
+	"scif1_ctrl",
+};
+
+static const char * const scif2_groups[] = {
+	"scif2_data_a",
+	"scif2_clk_a",
+	"scif2_data_b",
+};
+
+static const char * const scif3_groups[] = {
+	"scif3_data_a",
+	"scif3_clk_a",
+	"scif3_ctrl_a",
+	"scif3_data_b",
+	"scif3_data_c",
+	"scif3_clk_c",
+};
+
+static const char * const scif4_groups[] = {
+	"scif4_data_a",
+	"scif4_clk_a",
+	"scif4_ctrl_a",
+	"scif4_data_b",
+	"scif4_clk_b",
+	"scif4_data_c",
+	"scif4_ctrl_c",
+};
+
+static const char * const scif5_groups[] = {
+	"scif5_data_a",
+	"scif5_clk_a",
+	"scif5_data_b",
+	"scif5_data_c",
+};
+
+static const char * const scif_clk_groups[] = {
+	"scif_clk_a",
+	"scif_clk_b",
+};
+
+static const struct sh_pfc_function pinmux_functions[] = {
+	SH_PFC_FUNCTION(avb),
+	SH_PFC_FUNCTION(i2c1),
+	SH_PFC_FUNCTION(i2c2),
+	SH_PFC_FUNCTION(i2c4),
+	SH_PFC_FUNCTION(i2c5),
+	SH_PFC_FUNCTION(i2c6),
+	SH_PFC_FUNCTION(i2c7),
+	SH_PFC_FUNCTION(scif0),
+	SH_PFC_FUNCTION(scif1),
+	SH_PFC_FUNCTION(scif2),
+	SH_PFC_FUNCTION(scif3),
+	SH_PFC_FUNCTION(scif4),
+	SH_PFC_FUNCTION(scif5),
+	SH_PFC_FUNCTION(scif_clk),
+};
+
+static const struct pinmux_cfg_reg pinmux_config_regs[] = {
+#define F_(x, y)	FN_##y
+#define FM(x)		FN_##x
+	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_0_17_FN,	GPSR0_17,
+		GP_0_16_FN,	GPSR0_16,
+		GP_0_15_FN,	GPSR0_15,
+		GP_0_14_FN,	GPSR0_14,
+		GP_0_13_FN,	GPSR0_13,
+		GP_0_12_FN,	GPSR0_12,
+		GP_0_11_FN,	GPSR0_11,
+		GP_0_10_FN,	GPSR0_10,
+		GP_0_9_FN,	GPSR0_9,
+		GP_0_8_FN,	GPSR0_8,
+		GP_0_7_FN,	GPSR0_7,
+		GP_0_6_FN,	GPSR0_6,
+		GP_0_5_FN,	GPSR0_5,
+		GP_0_4_FN,	GPSR0_4,
+		GP_0_3_FN,	GPSR0_3,
+		GP_0_2_FN,	GPSR0_2,
+		GP_0_1_FN,	GPSR0_1,
+		GP_0_0_FN,	GPSR0_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_1_22_FN,	GPSR1_22,
+		GP_1_21_FN,	GPSR1_21,
+		GP_1_20_FN,	GPSR1_20,
+		GP_1_19_FN,	GPSR1_19,
+		GP_1_18_FN,	GPSR1_18,
+		GP_1_17_FN,	GPSR1_17,
+		GP_1_16_FN,	GPSR1_16,
+		GP_1_15_FN,	GPSR1_15,
+		GP_1_14_FN,	GPSR1_14,
+		GP_1_13_FN,	GPSR1_13,
+		GP_1_12_FN,	GPSR1_12,
+		GP_1_11_FN,	GPSR1_11,
+		GP_1_10_FN,	GPSR1_10,
+		GP_1_9_FN,	GPSR1_9,
+		GP_1_8_FN,	GPSR1_8,
+		GP_1_7_FN,	GPSR1_7,
+		GP_1_6_FN,	GPSR1_6,
+		GP_1_5_FN,	GPSR1_5,
+		GP_1_4_FN,	GPSR1_4,
+		GP_1_3_FN,	GPSR1_3,
+		GP_1_2_FN,	GPSR1_2,
+		GP_1_1_FN,	GPSR1_1,
+		GP_1_0_FN,	GPSR1_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_2_25_FN,	GPSR2_25,
+		GP_2_24_FN,	GPSR2_24,
+		GP_2_23_FN,	GPSR2_23,
+		GP_2_22_FN,	GPSR2_22,
+		GP_2_21_FN,	GPSR2_21,
+		GP_2_20_FN,	GPSR2_20,
+		GP_2_19_FN,	GPSR2_19,
+		GP_2_18_FN,	GPSR2_18,
+		GP_2_17_FN,	GPSR2_17,
+		GP_2_16_FN,	GPSR2_16,
+		GP_2_15_FN,	GPSR2_15,
+		GP_2_14_FN,	GPSR2_14,
+		GP_2_13_FN,	GPSR2_13,
+		GP_2_12_FN,	GPSR2_12,
+		GP_2_11_FN,	GPSR2_11,
+		GP_2_10_FN,	GPSR2_10,
+		GP_2_9_FN,	GPSR2_9,
+		GP_2_8_FN,	GPSR2_8,
+		GP_2_7_FN,	GPSR2_7,
+		GP_2_6_FN,	GPSR2_6,
+		GP_2_5_FN,	GPSR2_5,
+		GP_2_4_FN,	GPSR2_4,
+		GP_2_3_FN,	GPSR2_3,
+		GP_2_2_FN,	GPSR2_2,
+		GP_2_1_FN,	GPSR2_1,
+		GP_2_0_FN,	GPSR2_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_3_15_FN,	GPSR3_15,
+		GP_3_14_FN,	GPSR3_14,
+		GP_3_13_FN,	GPSR3_13,
+		GP_3_12_FN,	GPSR3_12,
+		GP_3_11_FN,	GPSR3_11,
+		GP_3_10_FN,	GPSR3_10,
+		GP_3_9_FN,	GPSR3_9,
+		GP_3_8_FN,	GPSR3_8,
+		GP_3_7_FN,	GPSR3_7,
+		GP_3_6_FN,	GPSR3_6,
+		GP_3_5_FN,	GPSR3_5,
+		GP_3_4_FN,	GPSR3_4,
+		GP_3_3_FN,	GPSR3_3,
+		GP_3_2_FN,	GPSR3_2,
+		GP_3_1_FN,	GPSR3_1,
+		GP_3_0_FN,	GPSR3_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_4_10_FN,	GPSR4_10,
+		GP_4_9_FN,	GPSR4_9,
+		GP_4_8_FN,	GPSR4_8,
+		GP_4_7_FN,	GPSR4_7,
+		GP_4_6_FN,	GPSR4_6,
+		GP_4_5_FN,	GPSR4_5,
+		GP_4_4_FN,	GPSR4_4,
+		GP_4_3_FN,	GPSR4_3,
+		GP_4_2_FN,	GPSR4_2,
+		GP_4_1_FN,	GPSR4_1,
+		GP_4_0_FN,	GPSR4_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_5_19_FN,	GPSR5_19,
+		GP_5_18_FN,	GPSR5_18,
+		GP_5_17_FN,	GPSR5_17,
+		GP_5_16_FN,	GPSR5_16,
+		GP_5_15_FN,	GPSR5_15,
+		GP_5_14_FN,	GPSR5_14,
+		GP_5_13_FN,	GPSR5_13,
+		GP_5_12_FN,	GPSR5_12,
+		GP_5_11_FN,	GPSR5_11,
+		GP_5_10_FN,	GPSR5_10,
+		GP_5_9_FN,	GPSR5_9,
+		GP_5_8_FN,	GPSR5_8,
+		GP_5_7_FN,	GPSR5_7,
+		GP_5_6_FN,	GPSR5_6,
+		GP_5_5_FN,	GPSR5_5,
+		GP_5_4_FN,	GPSR5_4,
+		GP_5_3_FN,	GPSR5_3,
+		GP_5_2_FN,	GPSR5_2,
+		GP_5_1_FN,	GPSR5_1,
+		GP_5_0_FN,	GPSR5_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_6_17_FN,	GPSR6_17,
+		GP_6_16_FN,	GPSR6_16,
+		GP_6_15_FN,	GPSR6_15,
+		GP_6_14_FN,	GPSR6_14,
+		GP_6_13_FN,	GPSR6_13,
+		GP_6_12_FN,	GPSR6_12,
+		GP_6_11_FN,	GPSR6_11,
+		GP_6_10_FN,	GPSR6_10,
+		GP_6_9_FN,	GPSR6_9,
+		GP_6_8_FN,	GPSR6_8,
+		GP_6_7_FN,	GPSR6_7,
+		GP_6_6_FN,	GPSR6_6,
+		GP_6_5_FN,	GPSR6_5,
+		GP_6_4_FN,	GPSR6_4,
+		GP_6_3_FN,	GPSR6_3,
+		GP_6_2_FN,	GPSR6_2,
+		GP_6_1_FN,	GPSR6_1,
+		GP_6_0_FN,	GPSR6_0, }
+	},
+#undef F_
+#undef FM
+
+#define F_(x, y)	x,
+#define FM(x)		FN_##x,
+	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4) {
+		IP0_31_28
+		IP0_27_24
+		IP0_23_20
+		IP0_19_16
+		IP0_15_12
+		IP0_11_8
+		IP0_7_4
+		IP0_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4) {
+		IP1_31_28
+		IP1_27_24
+		IP1_23_20
+		IP1_19_16
+		IP1_15_12
+		IP1_11_8
+		IP1_7_4
+		IP1_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4) {
+		IP2_31_28
+		IP2_27_24
+		IP2_23_20
+		IP2_19_16
+		IP2_15_12
+		IP2_11_8
+		IP2_7_4
+		IP2_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4) {
+		IP3_31_28
+		IP3_27_24
+		IP3_23_20
+		IP3_19_16
+		IP3_15_12
+		IP3_11_8
+		IP3_7_4
+		IP3_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4) {
+		IP4_31_28
+		IP4_27_24
+		IP4_23_20
+		IP4_19_16
+		IP4_15_12
+		IP4_11_8
+		IP4_7_4
+		IP4_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4) {
+		IP5_31_28
+		IP5_27_24
+		IP5_23_20
+		IP5_19_16
+		IP5_15_12
+		IP5_11_8
+		IP5_7_4
+		IP5_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4) {
+		IP6_31_28
+		IP6_27_24
+		IP6_23_20
+		IP6_19_16
+		IP6_15_12
+		IP6_11_8
+		IP6_7_4
+		IP6_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4) {
+		IP7_31_28
+		IP7_27_24
+		IP7_23_20
+		IP7_19_16
+		IP7_15_12
+		IP7_11_8
+		IP7_7_4
+		IP7_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4) {
+		IP8_31_28
+		IP8_27_24
+		IP8_23_20
+		IP8_19_16
+		IP8_15_12
+		IP8_11_8
+		IP8_7_4
+		IP8_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4) {
+		IP9_31_28
+		IP9_27_24
+		IP9_23_20
+		IP9_19_16
+		IP9_15_12
+		IP9_11_8
+		IP9_7_4
+		IP9_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4) {
+		IP10_31_28
+		IP10_27_24
+		IP10_23_20
+		IP10_19_16
+		IP10_15_12
+		IP10_11_8
+		IP10_7_4
+		IP10_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4) {
+		IP11_31_28
+		IP11_27_24
+		IP11_23_20
+		IP11_19_16
+		IP11_15_12
+		IP11_11_8
+		IP11_7_4
+		IP11_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4) {
+		IP12_31_28
+		IP12_27_24
+		IP12_23_20
+		IP12_19_16
+		IP12_15_12
+		IP12_11_8
+		IP12_7_4
+		IP12_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4) {
+		IP13_31_28
+		IP13_27_24
+		IP13_23_20
+		IP13_19_16
+		IP13_15_12
+		IP13_11_8
+		IP13_7_4
+		IP13_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4) {
+		IP14_31_28
+		IP14_27_24
+		IP14_23_20
+		IP14_19_16
+		IP14_15_12
+		IP14_11_8
+		IP14_7_4
+		IP14_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4) {
+		IP15_31_28
+		IP15_27_24
+		IP15_23_20
+		IP15_19_16
+		IP15_15_12
+		IP15_11_8
+		IP15_7_4
+		IP15_3_0 }
+	},
+#undef F_
+#undef FM
+
+#define F_(x, y)	x,
+#define FM(x)		FN_##x,
+	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xe6060500, 32,
+			     1, 2, 1, 2, 1, 1, 1, 1, 2, 3, 1,
+			     1, 1, 2, 2, 1, 1, 1, 2, 1, 1, 1, 2) {
+		/* RESERVED 31 */
+		0, 0,
+		MOD_SEL0_30_29
+		MOD_SEL0_28
+		MOD_SEL0_27_26
+		MOD_SEL0_25
+		MOD_SEL0_24
+		MOD_SEL0_23
+		MOD_SEL0_22
+		MOD_SEL0_21_20
+		MOD_SEL0_19_18_17
+		MOD_SEL0_16
+		MOD_SEL0_15
+		MOD_SEL0_14
+		MOD_SEL0_13_12
+		MOD_SEL0_11_10
+		MOD_SEL0_9
+		MOD_SEL0_8
+		MOD_SEL0_7
+		MOD_SEL0_6_5
+		MOD_SEL0_4
+		MOD_SEL0_3
+		MOD_SEL0_2
+		MOD_SEL0_1_0 }
+	},
+	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xe6060504, 32,
+			     1, 1, 1, 1, 1, 1, 1, 3, 3, 1, 1, 1,
+			     1, 2, 2, 2, 1, 1, 2, 1, 4) {
+		MOD_SEL1_31
+		MOD_SEL1_30
+		MOD_SEL1_29
+		MOD_SEL1_28
+		/* RESERVED 27 */
+		0, 0,
+		MOD_SEL1_26
+		MOD_SEL1_25
+		MOD_SEL1_24_23_22
+		MOD_SEL1_21_20_19
+		MOD_SEL1_18
+		MOD_SEL1_17
+		MOD_SEL1_16
+		MOD_SEL1_15
+		MOD_SEL1_14_13
+		MOD_SEL1_12_11
+		MOD_SEL1_10_9
+		MOD_SEL1_8
+		MOD_SEL1_7
+		MOD_SEL1_6_5
+		MOD_SEL1_4
+		/* RESERVED 3, 2, 1, 0  */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+	},
+	{ },
+};
+
+static const struct pinmux_bias_reg pinmux_bias_regs[] = {
+	{ PINMUX_BIAS_REG("PUEN0", 0xe6060400, "PUD0", 0xe6060440) {
+		 [0] = RCAR_GP_PIN(2, 23),	/* RD# */
+		 [1] = RCAR_GP_PIN(2, 22),	/* BS# */
+		 [2] = RCAR_GP_PIN(2, 21),	/* AVB_PHY_INT */
+		 [3] = PIN_NUMBER('P', 5),	/* AVB_MDC */
+		 [4] = PIN_NUMBER('P', 4),	/* AVB_MDIO */
+		 [5] = RCAR_GP_PIN(2, 20),	/* AVB_TXCREFCLK */
+		 [6] = PIN_NUMBER('N', 6),	/* AVB_TD3 */
+		 [7] = PIN_NUMBER('N', 5),	/* AVB_TD2 */
+		 [8] = PIN_NUMBER('N', 3),	/* AVB_TD1 */
+		 [9] = PIN_NUMBER('N', 2),	/* AVB_TD0 */
+		[10] = PIN_NUMBER('N', 1),	/* AVB_TXC */
+		[11] = PIN_NUMBER('P', 3),	/* AVB_TX_CTL */
+		[12] = RCAR_GP_PIN(2, 19),	/* AVB_RD3 */
+		[13] = RCAR_GP_PIN(2, 18),	/* AVB_RD2 */
+		[14] = RCAR_GP_PIN(2, 17),	/* AVB_RD1 */
+		[15] = RCAR_GP_PIN(2, 16),	/* AVB_RD0 */
+		[16] = RCAR_GP_PIN(2, 15),	/* AVB_RXC */
+		[17] = RCAR_GP_PIN(2, 14),	/* AVB_RX_CTL */
+		[18] = RCAR_GP_PIN(2, 13),	/* RPC_RESET# */
+		[19] = RCAR_GP_PIN(2, 12),	/* RPC_INT# */
+		[20] = RCAR_GP_PIN(2, 11),	/* QSPI1_SSL */
+		[21] = RCAR_GP_PIN(2, 10),	/* QSPI1_IO3 */
+		[22] = RCAR_GP_PIN(2,  9),	/* QSPI1_IO2 */
+		[23] = RCAR_GP_PIN(2,  8),	/* QSPI1_MISO/IO1 */
+		[24] = RCAR_GP_PIN(2,  7),	/* QSPI1_MOSI/IO0 */
+		[25] = RCAR_GP_PIN(2,  6),	/* QSPI1_SPCLK */
+		[26] = RCAR_GP_PIN(2,  5),	/* QSPI0_SSL */
+		[27] = RCAR_GP_PIN(2,  4),	/* QSPI0_IO3 */
+		[28] = RCAR_GP_PIN(2,  3),	/* QSPI0_IO2 */
+		[29] = RCAR_GP_PIN(2,  2),	/* QSPI0_MISO/IO1 */
+		[30] = RCAR_GP_PIN(2,  1),	/* QSPI0_MOSI/IO0 */
+		[31] = RCAR_GP_PIN(2,  0),	/* QSPI0_SPCLK */
+	} },
+	{ PINMUX_BIAS_REG("PUEN1", 0xe6060404, "PUD1", 0xe6060444) {
+		 [0] = RCAR_GP_PIN(0,  4),	/* D4 */
+		 [1] = RCAR_GP_PIN(0,  3),	/* D3 */
+		 [2] = RCAR_GP_PIN(0,  2),	/* D2 */
+		 [3] = RCAR_GP_PIN(0,  1),	/* D1 */
+		 [4] = RCAR_GP_PIN(0,  0),	/* D0 */
+		 [5] = RCAR_GP_PIN(1, 22),	/* WE0# */
+		 [6] = RCAR_GP_PIN(1, 21),	/* CS0# */
+		 [7] = RCAR_GP_PIN(1, 20),	/* CLKOUT */
+		 [8] = RCAR_GP_PIN(1, 19),	/* A19 */
+		 [9] = RCAR_GP_PIN(1, 18),	/* A18 */
+		[10] = RCAR_GP_PIN(1, 17),	/* A17 */
+		[11] = RCAR_GP_PIN(1, 16),	/* A16 */
+		[12] = RCAR_GP_PIN(1, 15),	/* A15 */
+		[13] = RCAR_GP_PIN(1, 14),	/* A14 */
+		[14] = RCAR_GP_PIN(1, 13),	/* A13 */
+		[15] = RCAR_GP_PIN(1, 12),	/* A12 */
+		[16] = RCAR_GP_PIN(1, 11),	/* A11 */
+		[17] = RCAR_GP_PIN(1, 10),	/* A10 */
+		[18] = RCAR_GP_PIN(1,  9),	/* A9 */
+		[19] = RCAR_GP_PIN(1,  8),	/* A8 */
+		[20] = RCAR_GP_PIN(1,  7),	/* A7 */
+		[21] = RCAR_GP_PIN(1,  6),	/* A6 */
+		[22] = RCAR_GP_PIN(1,  5),	/* A5 */
+		[23] = RCAR_GP_PIN(1,  4),	/* A4 */
+		[24] = RCAR_GP_PIN(1,  3),	/* A3 */
+		[25] = RCAR_GP_PIN(1,  2),	/* A2 */
+		[26] = RCAR_GP_PIN(1,  1),	/* A1 */
+		[27] = RCAR_GP_PIN(1,  0),	/* A0 */
+		[28] = PIN_NONE,
+		[29] = PIN_NONE,
+		[30] = RCAR_GP_PIN(2, 25),	/* PUEN_EX_WAIT0 */
+		[31] = RCAR_GP_PIN(2, 24),	/* PUEN_RD/WR# */
+	} },
+	{ PINMUX_BIAS_REG("PUEN2", 0xe6060408, "PUD2", 0xe6060448) {
+		 [0] = RCAR_GP_PIN(3,  1),	/* SD0_CMD */
+		 [1] = RCAR_GP_PIN(3,  0),	/* SD0_CLK */
+		 [2] = PIN_NUMBER('H', 1),	/* ASEBRK */
+		 [3] = PIN_NONE,
+		 [4] = PIN_NUMBER('G', 2),	/* TDI */
+		 [5] = PIN_NUMBER('F', 3),	/* TMS */
+		 [6] = PIN_NUMBER('F', 4),	/* TCK */
+		 [7] = PIN_NUMBER('F', 1),	/* TRST# */
+		 [8] = PIN_NONE,
+		 [9] = PIN_NONE,
+		[10] = PIN_NONE,
+		[11] = PIN_NONE,
+		[12] = PIN_NONE,
+		[13] = PIN_NONE,
+		[14] = PIN_NONE,
+		[15] = PIN_NUMBER('G', 3),	/* FSCLKST# */
+		[16] = RCAR_GP_PIN(0, 17),	/* SDA4 */
+		[17] = RCAR_GP_PIN(0, 16),	/* SCL4 */
+		[18] = PIN_NONE,
+		[19] = PIN_NONE,
+		[20] = PIN_A_NUMBER('D', 3),	/* PRESETOUT# */
+		[21] = RCAR_GP_PIN(0, 15),	/* D15 */
+		[22] = RCAR_GP_PIN(0, 14),	/* D14 */
+		[23] = RCAR_GP_PIN(0, 13),	/* D13 */
+		[24] = RCAR_GP_PIN(0, 12),	/* D12 */
+		[25] = RCAR_GP_PIN(0, 11),	/* D11 */
+		[26] = RCAR_GP_PIN(0, 10),	/* D10 */
+		[27] = RCAR_GP_PIN(0,  9),	/* D9 */
+		[28] = RCAR_GP_PIN(0,  8),	/* D8 */
+		[29] = RCAR_GP_PIN(0,  7),	/* D7 */
+		[30] = RCAR_GP_PIN(0,  6),	/* D6 */
+		[31] = RCAR_GP_PIN(0,  5),	/* D5 */
+	} },
+	{ PINMUX_BIAS_REG("PUEN3", 0xe606040c, "PUD3", 0xe606044c) {
+		 [0] = RCAR_GP_PIN(5,  0),	/* SCK0_A */
+		 [1] = RCAR_GP_PIN(5,  4),	/* RTS0#/TANS_A */
+		 [2] = RCAR_GP_PIN(5,  3),	/* CTS0#_A */
+		 [3] = RCAR_GP_PIN(5,  2),	/* TX0_A */
+		 [4] = RCAR_GP_PIN(5,  1),	/* RX0_A */
+		 [5] = PIN_NONE,
+		 [6] = PIN_NONE,
+		 [7] = RCAR_GP_PIN(3, 15),	/* SD1_WP */
+		 [8] = RCAR_GP_PIN(3, 14),	/* SD1_CD */
+		 [9] = RCAR_GP_PIN(3, 13),	/* SD0_WP */
+		[10] = RCAR_GP_PIN(3, 12),	/* SD0_CD */
+		[11] = RCAR_GP_PIN(4, 10),	/* SD3_DS */
+		[12] = RCAR_GP_PIN(4,  9),	/* SD3_DAT7 */
+		[13] = RCAR_GP_PIN(4,  8),	/* SD3_DAT6 */
+		[14] = RCAR_GP_PIN(4,  7),	/* SD3_DAT5 */
+		[15] = RCAR_GP_PIN(4,  6),	/* SD3_DAT4 */
+		[16] = RCAR_GP_PIN(4,  5),	/* SD3_DAT3 */
+		[17] = RCAR_GP_PIN(4,  4),	/* SD3_DAT2 */
+		[18] = RCAR_GP_PIN(4,  3),	/* SD3_DAT1 */
+		[19] = RCAR_GP_PIN(4,  2),	/* SD3_DAT0 */
+		[20] = RCAR_GP_PIN(4,  1),	/* SD3_CMD */
+		[21] = RCAR_GP_PIN(4,  0),	/* SD3_CLK */
+		[22] = RCAR_GP_PIN(3, 11),	/* SD1_DAT3 */
+		[23] = RCAR_GP_PIN(3, 10),	/* SD1_DAT2 */
+		[24] = RCAR_GP_PIN(3,  9),	/* SD1_DAT1 */
+		[25] = RCAR_GP_PIN(3,  8),	/* SD1_DAT0 */
+		[26] = RCAR_GP_PIN(3,  7),	/* SD1_CMD */
+		[27] = RCAR_GP_PIN(3,  6),	/* SD1_CLK */
+		[28] = RCAR_GP_PIN(3,  5),	/* SD0_DAT3 */
+		[29] = RCAR_GP_PIN(3,  4),	/* SD0_DAT2 */
+		[30] = RCAR_GP_PIN(3,  3),	/* SD0_DAT1 */
+		[31] = RCAR_GP_PIN(3,  2),	/* SD0_DAT0 */
+	} },
+	{ PINMUX_BIAS_REG("PUEN4", 0xe6060410, "PUD4", 0xe6060450) {
+		 [0] = RCAR_GP_PIN(6,  8),	/* AUDIO_CLKA */
+		 [1] = RCAR_GP_PIN(6, 16),	/* SSI_SDATA6 */
+		 [2] = RCAR_GP_PIN(6, 15),	/* SSI_WS6 */
+		 [3] = RCAR_GP_PIN(6, 14),	/* SSI_SCK6 */
+		 [4] = RCAR_GP_PIN(6, 13),	/* SSI_SDATA5 */
+		 [5] = RCAR_GP_PIN(6, 12),	/* SSI_WS5 */
+		 [6] = RCAR_GP_PIN(6, 11),	/* SSI_SCK5 */
+		 [7] = RCAR_GP_PIN(6, 10),	/* SSI_SDATA4 */
+		 [8] = RCAR_GP_PIN(6,  7),	/* SSI_SDATA3 */
+		 [9] = RCAR_GP_PIN(6,  6),	/* SSI_WS349 */
+		[10] = RCAR_GP_PIN(6,  5),	/* SSI_SCK349 */
+		[11] = RCAR_GP_PIN(6,  4),	/* SSI_SDATA2 */
+		[12] = RCAR_GP_PIN(6,  3),	/* SSI_SDATA1 */
+		[13] = RCAR_GP_PIN(6,  2),	/* SSI_SDATA0 */
+		[14] = RCAR_GP_PIN(6,  1),	/* SSI_WS01239 */
+		[15] = RCAR_GP_PIN(6,  0),	/* SSI_SCK01239 */
+		[16] = PIN_NUMBER('T', 21),	/* MLB_REF */
+		[17] = RCAR_GP_PIN(5, 19),	/* MLB_DAT */
+		[18] = RCAR_GP_PIN(5, 18),	/* MLB_SIG */
+		[19] = RCAR_GP_PIN(5, 17),	/* MLB_CLK */
+		[20] = RCAR_GP_PIN(5, 16),	/* SSI_SDATA9 */
+		[21] = RCAR_GP_PIN(5, 15),	/* MSIOF0_SS2 */
+		[22] = RCAR_GP_PIN(5, 14),	/* MSIOF0_SS1 */
+		[23] = RCAR_GP_PIN(5, 13),	/* MSIOF0_SYNC */
+		[24] = RCAR_GP_PIN(5, 12),	/* MSIOF0_TXD */
+		[25] = RCAR_GP_PIN(5, 11),	/* MSIOF0_RXD */
+		[26] = RCAR_GP_PIN(5, 10),	/* MSIOF0_SCK */
+		[27] = RCAR_GP_PIN(5,  9),	/* RX2_A */
+		[28] = RCAR_GP_PIN(5,  8),	/* TX2_A */
+		[29] = RCAR_GP_PIN(5,  7),	/* SCK2_A */
+		[30] = RCAR_GP_PIN(5,  6),	/* TX1 */
+		[31] = RCAR_GP_PIN(5,  5),	/* RX1 */
+	} },
+	{ PINMUX_BIAS_REG("PUEN5", 0xe6060414, "PUD5", 0xe6060454) {
+		 [0] = PIN_NONE,
+		 [1] = PIN_NONE,
+		 [2] = PIN_NONE,
+		 [3] = PIN_NONE,
+		 [4] = PIN_NONE,
+		 [5] = PIN_NONE,
+		 [6] = PIN_NONE,
+		 [7] = PIN_NONE,
+		 [8] = PIN_NONE,
+		 [9] = PIN_NONE,
+		[10] = PIN_NONE,
+		[11] = PIN_NONE,
+		[12] = PIN_NONE,
+		[13] = PIN_NONE,
+		[14] = PIN_NONE,
+		[15] = PIN_NONE,
+		[16] = PIN_NONE,
+		[17] = PIN_NONE,
+		[18] = PIN_NONE,
+		[19] = PIN_NONE,
+		[20] = PIN_NONE,
+		[21] = PIN_NONE,
+		[22] = PIN_NONE,
+		[23] = PIN_NONE,
+		[24] = PIN_NONE,
+		[25] = PIN_NONE,
+		[26] = PIN_NONE,
+		[27] = PIN_NONE,
+		[28] = PIN_NONE,
+		[29] = PIN_NONE,
+		[30] = RCAR_GP_PIN(6,  9),	/* PUEN_USB30_OVC */
+		[31] = RCAR_GP_PIN(6, 17),	/* PUEN_USB30_PWEN */
+	} },
+	{ /* sentinel */ },
+};
+
+static unsigned int r8a77990_pinmux_get_bias(struct sh_pfc *pfc,
+					     unsigned int pin)
+{
+	const struct pinmux_bias_reg *reg;
+	unsigned int bit;
+
+	reg = sh_pfc_pin_to_bias_reg(pfc, pin, &bit);
+	if (!reg)
+		return PIN_CONFIG_BIAS_DISABLE;
+
+	if (!(sh_pfc_read(pfc, reg->puen) & BIT(bit)))
+		return PIN_CONFIG_BIAS_DISABLE;
+	else if (sh_pfc_read(pfc, reg->pud) & BIT(bit))
+		return PIN_CONFIG_BIAS_PULL_UP;
+	else
+		return PIN_CONFIG_BIAS_PULL_DOWN;
+}
+
+static void r8a77990_pinmux_set_bias(struct sh_pfc *pfc, unsigned int pin,
+				     unsigned int bias)
+{
+	const struct pinmux_bias_reg *reg;
+	u32 enable, updown;
+	unsigned int bit;
+
+	reg = sh_pfc_pin_to_bias_reg(pfc, pin, &bit);
+	if (!reg)
+		return;
+
+	enable = sh_pfc_read(pfc, reg->puen) & ~BIT(bit);
+	if (bias != PIN_CONFIG_BIAS_DISABLE)
+		enable |= BIT(bit);
+
+	updown = sh_pfc_read(pfc, reg->pud) & ~BIT(bit);
+	if (bias == PIN_CONFIG_BIAS_PULL_UP)
+		updown |= BIT(bit);
+
+	sh_pfc_write(pfc, reg->pud, updown);
+	sh_pfc_write(pfc, reg->puen, enable);
+}
+
+static const struct sh_pfc_soc_operations r8a77990_pinmux_ops = {
+	.get_bias = r8a77990_pinmux_get_bias,
+	.set_bias = r8a77990_pinmux_set_bias,
+};
+
+const struct sh_pfc_soc_info r8a77990_pinmux_info = {
+	.name = "r8a77990_pfc",
+	.ops = &r8a77990_pinmux_ops,
+	.unlock_reg = 0xe6060000, /* PMMR */
+
+	.function = { PINMUX_FUNCTION_BEGIN, PINMUX_FUNCTION_END },
+
+	.pins = pinmux_pins,
+	.nr_pins = ARRAY_SIZE(pinmux_pins),
+	.groups = pinmux_groups,
+	.nr_groups = ARRAY_SIZE(pinmux_groups),
+	.functions = pinmux_functions,
+	.nr_functions = ARRAY_SIZE(pinmux_functions),
+
+	.cfg_regs = pinmux_config_regs,
+	.bias_regs = pinmux_bias_regs,
+
+	.pinmux_data = pinmux_data,
+	.pinmux_data_size = ARRAY_SIZE(pinmux_data),
+};
diff --git a/drivers/pinctrl/sh-pfc/sh_pfc.h b/drivers/pinctrl/sh-pfc/sh_pfc.h
index 7fad897cd9f5..3d0b31636d6d 100644
--- a/drivers/pinctrl/sh-pfc/sh_pfc.h
+++ b/drivers/pinctrl/sh-pfc/sh_pfc.h
@@ -274,6 +274,7 @@ extern const struct sh_pfc_soc_info r8a73a4_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7740_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7743_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7745_pinmux_info;
+extern const struct sh_pfc_soc_info r8a77470_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7778_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7779_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7790_pinmux_info;
@@ -287,6 +288,7 @@ extern const struct sh_pfc_soc_info r8a7796_pinmux_info;
 extern const struct sh_pfc_soc_info r8a77965_pinmux_info;
 extern const struct sh_pfc_soc_info r8a77970_pinmux_info;
 extern const struct sh_pfc_soc_info r8a77980_pinmux_info;
+extern const struct sh_pfc_soc_info r8a77990_pinmux_info;
 extern const struct sh_pfc_soc_info r8a77995_pinmux_info;
 extern const struct sh_pfc_soc_info sh7203_pinmux_info;
 extern const struct sh_pfc_soc_info sh7264_pinmux_info;
@@ -415,9 +417,13 @@ extern const struct sh_pfc_soc_info shx3_pinmux_info;
 	PORT_GP_CFG_1(bank, 9,  fn, sfx, cfg)
 #define PORT_GP_10(bank, fn, sfx)	PORT_GP_CFG_10(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_12(bank, fn, sfx, cfg)				\
+#define PORT_GP_CFG_11(bank, fn, sfx, cfg)				\
 	PORT_GP_CFG_10(bank, fn, sfx, cfg),				\
-	PORT_GP_CFG_1(bank, 10, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 10, fn, sfx, cfg)
+#define PORT_GP_11(bank, fn, sfx)	PORT_GP_CFG_11(bank, fn, sfx, 0)
+
+#define PORT_GP_CFG_12(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_11(bank, fn, sfx, cfg),				\
 	PORT_GP_CFG_1(bank, 11, fn, sfx, cfg)
 #define PORT_GP_12(bank, fn, sfx)	PORT_GP_CFG_12(bank, fn, sfx, 0)
 
diff --git a/drivers/pinctrl/sunxi/Kconfig b/drivers/pinctrl/sunxi/Kconfig
index 5de1f63b07bb..95282cda6cee 100644
--- a/drivers/pinctrl/sunxi/Kconfig
+++ b/drivers/pinctrl/sunxi/Kconfig
@@ -81,4 +81,8 @@ config PINCTRL_SUN50I_H6
 	def_bool ARM64 && ARCH_SUNXI
 	select PINCTRL_SUNXI
 
+config PINCTRL_SUN50I_H6_R
+	def_bool ARM64 && ARCH_SUNXI
+	select PINCTRL_SUNXI
+
 endif
diff --git a/drivers/pinctrl/sunxi/Makefile b/drivers/pinctrl/sunxi/Makefile
index 3c4aec6611e9..adb8443aa55c 100644
--- a/drivers/pinctrl/sunxi/Makefile
+++ b/drivers/pinctrl/sunxi/Makefile
@@ -19,5 +19,6 @@ obj-$(CONFIG_PINCTRL_SUN8I_H3_R)	+= pinctrl-sun8i-h3-r.o
 obj-$(CONFIG_PINCTRL_SUN8I_V3S)		+= pinctrl-sun8i-v3s.o
 obj-$(CONFIG_PINCTRL_SUN50I_H5)		+= pinctrl-sun50i-h5.o
 obj-$(CONFIG_PINCTRL_SUN50I_H6)		+= pinctrl-sun50i-h6.o
+obj-$(CONFIG_PINCTRL_SUN50I_H6_R)	+= pinctrl-sun50i-h6-r.o
 obj-$(CONFIG_PINCTRL_SUN9I_A80)		+= pinctrl-sun9i-a80.o
 obj-$(CONFIG_PINCTRL_SUN9I_A80_R)	+= pinctrl-sun9i-a80-r.o
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun50i-h6-r.c b/drivers/pinctrl/sunxi/pinctrl-sun50i-h6-r.c
new file mode 100644
index 000000000000..4557e18d5989
--- /dev/null
+++ b/drivers/pinctrl/sunxi/pinctrl-sun50i-h6-r.c
@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Allwinner H6 R_PIO pin controller driver
+ *
+ * Copyright (C) 2017 Icenowy Zheng <icenowy@aosc.io>
+ *
+ * Based on pinctrl-sun6i-a31-r.c, which is:
+ *   Copyright (C) 2014 Boris Brezillon
+ *   Boris Brezillon <boris.brezillon@free-electrons.com>
+ *   Copyright (C) 2014 Maxime Ripard
+ *   Maxime Ripard <maxime.ripard@free-electrons.com>
+ */
+
+#include <linux/init.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/reset.h>
+
+#include "pinctrl-sunxi.h"
+
+static const struct sunxi_desc_pin sun50i_h6_r_pins[] = {
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 0),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x3, "s_i2c"),		/* SCK */
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 0)),	/* PL_EINT0 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 1),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x3, "s_i2c"),		/* SDA */
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 1)),	/* PL_EINT1 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 2),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "s_uart"),	/* TX */
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 2)),	/* PL_EINT2 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 3),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "s_uart"),	/* RX */
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 3)),	/* PL_EINT3 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 4),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "s_jtag"),	/* MS */
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 4)),	/* PL_EINT4 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 5),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "s_jtag"),	/* CK */
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 5)),	/* PL_EINT5 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 6),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "s_jtag"),	/* DO */
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 6)),	/* PL_EINT6 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 7),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "s_jtag"),	/* DI */
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 7)),	/* PL_EINT7 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 8),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "s_pwm"),
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 8)),	/* PL_EINT8 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 9),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "s_cir_rx"),
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 9)),	/* PL_EINT9 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 10),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "s_w1"),
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 10)),	/* PL_EINT10 */
+	/* Hole */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(M, 0),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 1, 0)),	/* PM_EINT0 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(M, 1),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 1, 1)),	/* PM_EINT1 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(M, 2),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 1, 2),	/* PM_EINT2 */
+		  SUNXI_FUNCTION(0x3, "1wire")),
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(M, 3),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 1, 3)),	/* PM_EINT3 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(M, 4),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION_IRQ_BANK(0x6, 1, 4)),	/* PM_EINT4 */
+};
+
+static const struct sunxi_pinctrl_desc sun50i_h6_r_pinctrl_data = {
+	.pins = sun50i_h6_r_pins,
+	.npins = ARRAY_SIZE(sun50i_h6_r_pins),
+	.pin_base = PL_BASE,
+	.irq_banks = 2,
+};
+
+static int sun50i_h6_r_pinctrl_probe(struct platform_device *pdev)
+{
+	return sunxi_pinctrl_init(pdev,
+				  &sun50i_h6_r_pinctrl_data);
+}
+
+static const struct of_device_id sun50i_h6_r_pinctrl_match[] = {
+	{ .compatible = "allwinner,sun50i-h6-r-pinctrl", },
+	{}
+};
+
+static struct platform_driver sun50i_h6_r_pinctrl_driver = {
+	.probe	= sun50i_h6_r_pinctrl_probe,
+	.driver	= {
+		.name		= "sun50i-h6-r-pinctrl",
+		.of_match_table	= sun50i_h6_r_pinctrl_match,
+	},
+};
+builtin_platform_driver(sun50i_h6_r_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
index 020d6d84639c..25e80a5370ca 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
@@ -12,12 +12,12 @@
 
 #include <linux/io.h>
 #include <linux/clk.h>
-#include <linux/clk-provider.h>
 #include <linux/gpio/driver.h>
 #include <linux/irqdomain.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/export.h>
 #include <linux/of.h>
+#include <linux/of_clk.h>
 #include <linux/of_address.h>
 #include <linux/of_device.h>
 #include <linux/of_irq.h>
@@ -1361,7 +1361,7 @@ int sunxi_pinctrl_init_with_variant(struct platform_device *pdev,
 			goto gpiochip_error;
 	}
 
-	ret = of_count_phandle_with_args(node, "clocks", "#clock-cells");
+	ret = of_clk_get_parent_count(node);
 	clk = devm_clk_get(&pdev->dev, ret == 1 ? NULL : "apb");
 	if (IS_ERR(clk)) {
 		ret = PTR_ERR(clk);
diff --git a/drivers/pinctrl/tegra/pinctrl-tegra.c b/drivers/pinctrl/tegra/pinctrl-tegra.c
index 72c718e66ebb..49c7c1499bc3 100644
--- a/drivers/pinctrl/tegra/pinctrl-tegra.c
+++ b/drivers/pinctrl/tegra/pinctrl-tegra.c
@@ -33,17 +33,6 @@
 #include "../pinctrl-utils.h"
 #include "pinctrl-tegra.h"
 
-struct tegra_pmx {
-	struct device *dev;
-	struct pinctrl_dev *pctl;
-
-	const struct tegra_pinctrl_soc_data *soc;
-	const char **group_pins;
-
-	int nbanks;
-	void __iomem **regs;
-};
-
 static inline u32 pmx_readl(struct tegra_pmx *pmx, u32 bank, u32 reg)
 {
 	return readl(pmx->regs[bank] + reg);
diff --git a/drivers/pinctrl/tegra/pinctrl-tegra.h b/drivers/pinctrl/tegra/pinctrl-tegra.h
index 33b17cb1471e..aa33c20766c4 100644
--- a/drivers/pinctrl/tegra/pinctrl-tegra.h
+++ b/drivers/pinctrl/tegra/pinctrl-tegra.h
@@ -16,6 +16,17 @@
 #ifndef __PINMUX_TEGRA_H__
 #define __PINMUX_TEGRA_H__
 
+struct tegra_pmx {
+	struct device *dev;
+	struct pinctrl_dev *pctl;
+
+	const struct tegra_pinctrl_soc_data *soc;
+	const char **group_pins;
+
+	int nbanks;
+	void __iomem **regs;
+};
+
 enum tegra_pinconf_param {
 	/* argument: tegra_pinconf_pull */
 	TEGRA_PINCONF_PARAM_PULL,
diff --git a/drivers/pinctrl/tegra/pinctrl-tegra20.c b/drivers/pinctrl/tegra/pinctrl-tegra20.c
index 7e38ee9bae78..b6dd939d32cc 100644
--- a/drivers/pinctrl/tegra/pinctrl-tegra20.c
+++ b/drivers/pinctrl/tegra/pinctrl-tegra20.c
@@ -19,6 +19,7 @@
  * more details.
  */
 
+#include <linux/clk-provider.h>
 #include <linux/init.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
@@ -2231,9 +2232,36 @@ static const struct tegra_pinctrl_soc_data tegra20_pinctrl = {
 	.drvtype_in_mux = false,
 };
 
+static const char *cdev1_parents[] = {
+	"dev1_osc_div", "pll_a_out0", "pll_m_out1", "audio",
+};
+
+static const char *cdev2_parents[] = {
+	"dev2_osc_div", "hclk", "pclk", "pll_p_out4",
+};
+
+static void tegra20_pinctrl_register_clock_muxes(struct platform_device *pdev)
+{
+	struct tegra_pmx *pmx = platform_get_drvdata(pdev);
+
+	clk_register_mux(NULL, "cdev1_mux", cdev1_parents, 4, 0,
+			 pmx->regs[1] + 0x8, 2, 2, CLK_MUX_READ_ONLY, NULL);
+
+	clk_register_mux(NULL, "cdev2_mux", cdev2_parents, 4, 0,
+			 pmx->regs[1] + 0x8, 4, 2, CLK_MUX_READ_ONLY, NULL);
+}
+
 static int tegra20_pinctrl_probe(struct platform_device *pdev)
 {
-	return tegra_pinctrl_probe(pdev, &tegra20_pinctrl);
+	int err;
+
+	err = tegra_pinctrl_probe(pdev, &tegra20_pinctrl);
+	if (err)
+		return err;
+
+	tegra20_pinctrl_register_clock_muxes(pdev);
+
+	return 0;
 }
 
 static const struct of_device_id tegra20_pinctrl_of_match[] = {
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld11.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld11.c
index 0976fbfecd50..58825f68b58b 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld11.c
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld11.c
@@ -481,6 +481,31 @@ static const int emmc_dat8_muxvals[] = {0, 0, 0, 0};
 static const unsigned ether_rmii_pins[] = {6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 					   16, 17};
 static const int ether_rmii_muxvals[] = {4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4};
+static const unsigned hscin0_ci_pins[] = {102, 103, 104, 105, 106, 107, 108,
+					  109, 110, 111, 112};
+static const int hscin0_ci_muxvals[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+static const unsigned hscin0_p_pins[] = {102, 103, 104, 105, 106, 107, 108, 109,
+					 110, 111, 112};
+static const int hscin0_p_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+static const unsigned hscin0_s_pins[] = {116, 117, 118, 119};
+static const int hscin0_s_muxvals[] = {3, 3, 3, 3};
+static const unsigned hscin1_p_pins[] = {124, 125, 126, 127, 128, 129, 130, 131,
+					 132, 133, 134};
+static const int hscin1_p_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+static const unsigned hscin1_s_pins[] = {120, 121, 122, 123};
+static const int hscin1_s_muxvals[] = {3, 3, 3, 3};
+static const unsigned hscin2_s_pins[] = {124, 125, 126, 127};
+static const int hscin2_s_muxvals[] = {3, 3, 3, 3};
+static const unsigned hscout0_ci_pins[] = {113, 114, 115, 116, 117, 118, 119,
+					   120, 121, 122, 123};
+static const int hscout0_ci_muxvals[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+static const unsigned hscout0_p_pins[] = {113, 114, 115, 116, 117, 118, 119,
+					  120, 121, 122, 123};
+static const int hscout0_p_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+static const unsigned hscout0_s_pins[] = {116, 117, 118, 119};
+static const int hscout0_s_muxvals[] = {4, 4, 4, 4};
+static const unsigned hscout1_s_pins[] = {120, 121, 122, 123};
+static const int hscout1_s_muxvals[] = {4, 4, 4, 4};
 static const unsigned i2c0_pins[] = {63, 64};
 static const int i2c0_muxvals[] = {0, 0};
 static const unsigned i2c1_pins[] = {65, 66};
@@ -556,6 +581,16 @@ static const struct uniphier_pinctrl_group uniphier_ld11_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(emmc),
 	UNIPHIER_PINCTRL_GROUP(emmc_dat8),
 	UNIPHIER_PINCTRL_GROUP(ether_rmii),
+	UNIPHIER_PINCTRL_GROUP(hscin0_ci),
+	UNIPHIER_PINCTRL_GROUP(hscin0_p),
+	UNIPHIER_PINCTRL_GROUP(hscin0_s),
+	UNIPHIER_PINCTRL_GROUP(hscin1_p),
+	UNIPHIER_PINCTRL_GROUP(hscin1_s),
+	UNIPHIER_PINCTRL_GROUP(hscin2_s),
+	UNIPHIER_PINCTRL_GROUP(hscout0_ci),
+	UNIPHIER_PINCTRL_GROUP(hscout0_p),
+	UNIPHIER_PINCTRL_GROUP(hscout0_s),
+	UNIPHIER_PINCTRL_GROUP(hscout1_s),
 	UNIPHIER_PINCTRL_GROUP(i2c0),
 	UNIPHIER_PINCTRL_GROUP(i2c1),
 	UNIPHIER_PINCTRL_GROUP(i2c3),
@@ -583,6 +618,15 @@ static const char * const aout1_groups[] = {"aout1"};
 static const char * const aoutiec1_groups[] = {"aoutiec1"};
 static const char * const emmc_groups[] = {"emmc", "emmc_dat8"};
 static const char * const ether_rmii_groups[] = {"ether_rmii"};
+static const char * const hscin0_groups[] = {"hscin0_ci",
+					     "hscin0_p",
+					     "hscin0_s"};
+static const char * const hscin1_groups[] = {"hscin1_p", "hscin1_s"};
+static const char * const hscin2_groups[] = {"hscin2_s"};
+static const char * const hscout0_groups[] = {"hscout0_ci",
+					      "hscout0_p",
+					      "hscout0_s"};
+static const char * const hscout1_groups[] = {"hscout1_s"};
 static const char * const i2c0_groups[] = {"i2c0"};
 static const char * const i2c1_groups[] = {"i2c1"};
 static const char * const i2c3_groups[] = {"i2c3"};
@@ -603,6 +647,11 @@ static const struct uniphier_pinmux_function uniphier_ld11_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(aoutiec1),
 	UNIPHIER_PINMUX_FUNCTION(emmc),
 	UNIPHIER_PINMUX_FUNCTION(ether_rmii),
+	UNIPHIER_PINMUX_FUNCTION(hscin0),
+	UNIPHIER_PINMUX_FUNCTION(hscin1),
+	UNIPHIER_PINMUX_FUNCTION(hscin2),
+	UNIPHIER_PINMUX_FUNCTION(hscout0),
+	UNIPHIER_PINMUX_FUNCTION(hscout1),
 	UNIPHIER_PINMUX_FUNCTION(i2c0),
 	UNIPHIER_PINMUX_FUNCTION(i2c1),
 	UNIPHIER_PINMUX_FUNCTION(i2c3),
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c
index bf8f0c3bea5e..9f449b35e300 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c
@@ -566,6 +566,33 @@ static const int ether_rgmii_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 static const unsigned ether_rmii_pins[] = {30, 31, 32, 33, 34, 35, 36, 37, 39,
 					   41, 42, 45};
 static const int ether_rmii_muxvals[] = {0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1};
+static const unsigned hscin0_ci_pins[] = {102, 103, 104, 105, 106, 107, 108,
+					  109, 110, 111, 112};
+static const int hscin0_ci_muxvals[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+static const unsigned hscin0_p_pins[] = {102, 103, 104, 105, 106, 107, 108, 109,
+					 110, 111, 112};
+static const int hscin0_p_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+static const unsigned hscin0_s_pins[] = {116, 117, 118, 119};
+static const int hscin0_s_muxvals[] = {3, 3, 3, 3};
+static const unsigned hscin1_p_pins[] = {124, 125, 126, 127, 128, 129, 130, 131,
+					 132, 133, 134};
+static const int hscin1_p_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+static const unsigned hscin1_s_pins[] = {120, 121, 122, 123};
+static const int hscin1_s_muxvals[] = {3, 3, 3, 3};
+static const unsigned hscin2_s_pins[] = {124, 125, 126, 127};
+static const int hscin2_s_muxvals[] = {3, 3, 3, 3};
+static const unsigned hscin3_s_pins[] = {129, 130, 131, 132};
+static const int hscin3_s_muxvals[] = {3, 3, 3, 3};
+static const unsigned hscout0_ci_pins[] = {113, 114, 115, 116, 117, 118, 119,
+					   120, 121, 122, 123};
+static const int hscout0_ci_muxvals[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+static const unsigned hscout0_p_pins[] = {113, 114, 115, 116, 117, 118, 119,
+					  120, 121, 122, 123};
+static const int hscout0_p_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+static const unsigned hscout0_s_pins[] = {116, 117, 118, 119};
+static const int hscout0_s_muxvals[] = {4, 4, 4, 4};
+static const unsigned hscout1_s_pins[] = {120, 121, 122, 123};
+static const int hscout1_s_muxvals[] = {4, 4, 4, 4};
 static const unsigned i2c0_pins[] = {63, 64};
 static const int i2c0_muxvals[] = {0, 0};
 static const unsigned i2c1_pins[] = {65, 66};
@@ -641,6 +668,17 @@ static const struct uniphier_pinctrl_group uniphier_ld20_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(emmc_dat8),
 	UNIPHIER_PINCTRL_GROUP(ether_rgmii),
 	UNIPHIER_PINCTRL_GROUP(ether_rmii),
+	UNIPHIER_PINCTRL_GROUP(hscin0_ci),
+	UNIPHIER_PINCTRL_GROUP(hscin0_p),
+	UNIPHIER_PINCTRL_GROUP(hscin0_s),
+	UNIPHIER_PINCTRL_GROUP(hscin1_p),
+	UNIPHIER_PINCTRL_GROUP(hscin1_s),
+	UNIPHIER_PINCTRL_GROUP(hscin2_s),
+	UNIPHIER_PINCTRL_GROUP(hscin3_s),
+	UNIPHIER_PINCTRL_GROUP(hscout0_ci),
+	UNIPHIER_PINCTRL_GROUP(hscout0_p),
+	UNIPHIER_PINCTRL_GROUP(hscout0_s),
+	UNIPHIER_PINCTRL_GROUP(hscout1_s),
 	UNIPHIER_PINCTRL_GROUP(i2c0),
 	UNIPHIER_PINCTRL_GROUP(i2c1),
 	UNIPHIER_PINCTRL_GROUP(i2c3),
@@ -668,6 +706,16 @@ static const char * const aoutiec1_groups[] = {"aoutiec1"};
 static const char * const emmc_groups[] = {"emmc", "emmc_dat8"};
 static const char * const ether_rgmii_groups[] = {"ether_rgmii"};
 static const char * const ether_rmii_groups[] = {"ether_rmii"};
+static const char * const hscin0_groups[] = {"hscin0_ci",
+					     "hscin0_p",
+					     "hscin0_s"};
+static const char * const hscin1_groups[] = {"hscin1_p", "hscin1_s"};
+static const char * const hscin2_groups[] = {"hscin2_s"};
+static const char * const hscin3_groups[] = {"hscin3_s"};
+static const char * const hscout0_groups[] = {"hscout0_ci",
+					      "hscout0_p",
+					      "hscout0_s"};
+static const char * const hscout1_groups[] = {"hscout1_s"};
 static const char * const i2c0_groups[] = {"i2c0"};
 static const char * const i2c1_groups[] = {"i2c1"};
 static const char * const i2c3_groups[] = {"i2c3"};
@@ -691,6 +739,12 @@ static const struct uniphier_pinmux_function uniphier_ld20_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(emmc),
 	UNIPHIER_PINMUX_FUNCTION(ether_rgmii),
 	UNIPHIER_PINMUX_FUNCTION(ether_rmii),
+	UNIPHIER_PINMUX_FUNCTION(hscin0),
+	UNIPHIER_PINMUX_FUNCTION(hscin1),
+	UNIPHIER_PINMUX_FUNCTION(hscin2),
+	UNIPHIER_PINMUX_FUNCTION(hscin3),
+	UNIPHIER_PINMUX_FUNCTION(hscout0),
+	UNIPHIER_PINMUX_FUNCTION(hscout1),
 	UNIPHIER_PINMUX_FUNCTION(i2c0),
 	UNIPHIER_PINMUX_FUNCTION(i2c1),
 	UNIPHIER_PINMUX_FUNCTION(i2c3),
diff --git a/drivers/platform/chrome/Kconfig b/drivers/platform/chrome/Kconfig
index e728a96cabfd..cb0df9eb3e0f 100644
--- a/drivers/platform/chrome/Kconfig
+++ b/drivers/platform/chrome/Kconfig
@@ -38,6 +38,17 @@ config CHROMEOS_PSTORE
 	  If you have a supported Chromebook, choose Y or M here.
 	  The module will be called chromeos_pstore.
 
+config CHROMEOS_TBMC
+	tristate "ChromeOS Tablet Switch Controller"
+	depends on ACPI
+	depends on INPUT
+	help
+	  This option adds a driver for the tablet switch on
+	  select Chrome OS systems.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called chromeos_tbmc.
+
 config CROS_EC_CTL
         tristate
 
diff --git a/drivers/platform/chrome/Makefile b/drivers/platform/chrome/Makefile
index ff3b369911f0..e44c37a63fa9 100644
--- a/drivers/platform/chrome/Makefile
+++ b/drivers/platform/chrome/Makefile
@@ -2,6 +2,7 @@
 
 obj-$(CONFIG_CHROMEOS_LAPTOP)		+= chromeos_laptop.o
 obj-$(CONFIG_CHROMEOS_PSTORE)		+= chromeos_pstore.o
+obj-$(CONFIG_CHROMEOS_TBMC)		+= chromeos_tbmc.o
 cros_ec_ctl-objs			:= cros_ec_sysfs.o cros_ec_lightbar.o \
 					   cros_ec_vbc.o cros_ec_debugfs.o
 obj-$(CONFIG_CROS_EC_CTL)		+= cros_ec_ctl.o
diff --git a/drivers/platform/chrome/chromeos_laptop.c b/drivers/platform/chrome/chromeos_laptop.c
index 5c47f451e43b..24326eecd787 100644
--- a/drivers/platform/chrome/chromeos_laptop.c
+++ b/drivers/platform/chrome/chromeos_laptop.c
@@ -6,6 +6,7 @@
 
 #define pr_fmt(fmt)		KBUILD_MODNAME ": " fmt
 
+#include <linux/acpi.h>
 #include <linux/dmi.h>
 #include <linux/i2c.h>
 #include <linux/input.h>
@@ -54,6 +55,11 @@ struct i2c_peripheral {
 	struct i2c_client *client;
 };
 
+struct acpi_peripheral {
+	char hid[ACPI_ID_LEN];
+	const struct property_entry *properties;
+};
+
 struct chromeos_laptop {
 	/*
 	 * Note that we can't mark this pointer as const because
@@ -61,6 +67,9 @@ struct chromeos_laptop {
 	 */
 	struct i2c_peripheral *i2c_peripherals;
 	unsigned int num_i2c_peripherals;
+
+	const struct acpi_peripheral *acpi_peripherals;
+	unsigned int num_acpi_peripherals;
 };
 
 static const struct chromeos_laptop *cros_laptop;
@@ -148,6 +157,38 @@ static void chromeos_laptop_check_adapter(struct i2c_adapter *adapter)
 	}
 }
 
+static bool chromeos_laptop_adjust_client(struct i2c_client *client)
+{
+	const struct acpi_peripheral *acpi_dev;
+	struct acpi_device_id acpi_ids[2] = { };
+	int i;
+	int error;
+
+	if (!has_acpi_companion(&client->dev))
+		return false;
+
+	for (i = 0; i < cros_laptop->num_acpi_peripherals; i++) {
+		acpi_dev = &cros_laptop->acpi_peripherals[i];
+
+		memcpy(acpi_ids[0].id, acpi_dev->hid, ACPI_ID_LEN);
+
+		if (acpi_match_device(acpi_ids, &client->dev)) {
+			error = device_add_properties(&client->dev,
+						      acpi_dev->properties);
+			if (error) {
+				dev_err(&client->dev,
+					"failed to add properties: %d\n",
+					error);
+				break;
+			}
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
 static void chromeos_laptop_detach_i2c_client(struct i2c_client *client)
 {
 	struct i2c_peripheral *i2c_dev;
@@ -170,6 +211,8 @@ static int chromeos_laptop_i2c_notifier_call(struct notifier_block *nb,
 	case BUS_NOTIFY_ADD_DEVICE:
 		if (dev->type == &i2c_adapter_type)
 			chromeos_laptop_check_adapter(to_i2c_adapter(dev));
+		else if (dev->type == &i2c_client_type)
+			chromeos_laptop_adjust_client(to_i2c_client(dev));
 		break;
 
 	case BUS_NOTIFY_REMOVED_DEVICE:
@@ -191,6 +234,12 @@ static const struct chromeos_laptop _name __initconst = {		\
 	.num_i2c_peripherals	= ARRAY_SIZE(_name##_peripherals),	\
 }
 
+#define DECLARE_ACPI_CROS_LAPTOP(_name)					\
+static const struct chromeos_laptop _name __initconst = {		\
+	.acpi_peripherals	= _name##_peripherals,			\
+	.num_acpi_peripherals	= ARRAY_SIZE(_name##_peripherals),	\
+}
+
 static struct i2c_peripheral samsung_series_5_550_peripherals[] __initdata = {
 	/* Touchpad. */
 	{
@@ -234,16 +283,25 @@ static const int chromebook_pixel_tp_keys[] __initconst = {
 
 static const struct property_entry
 chromebook_pixel_trackpad_props[] __initconst = {
+	PROPERTY_ENTRY_STRING("compatible", "atmel,maxtouch"),
 	PROPERTY_ENTRY_U32_ARRAY("linux,gpio-keymap", chromebook_pixel_tp_keys),
 	{ }
 };
 
+static const struct property_entry
+chromebook_atmel_touchscreen_props[] __initconst = {
+	PROPERTY_ENTRY_STRING("compatible", "atmel,maxtouch"),
+	{ }
+};
+
 static struct i2c_peripheral chromebook_pixel_peripherals[] __initdata = {
 	/* Touch Screen. */
 	{
 		.board_info	= {
 			I2C_BOARD_INFO("atmel_mxt_ts",
 					ATMEL_TS_I2C_ADDR),
+			.properties	=
+				chromebook_atmel_touchscreen_props,
 			.flags		= I2C_CLIENT_WAKE,
 		},
 		.dmi_name	= "touchscreen",
@@ -354,6 +412,8 @@ static struct i2c_peripheral acer_c720_peripherals[] __initdata = {
 		.board_info	= {
 			I2C_BOARD_INFO("atmel_mxt_ts",
 					ATMEL_TS_I2C_ADDR),
+			.properties	=
+				chromebook_atmel_touchscreen_props,
 			.flags		= I2C_CLIENT_WAKE,
 		},
 		.dmi_name	= "touchscreen",
@@ -419,6 +479,47 @@ static struct i2c_peripheral cr48_peripherals[] __initdata = {
 };
 DECLARE_CROS_LAPTOP(cr48);
 
+static const u32 samus_touchpad_buttons[] __initconst = {
+	KEY_RESERVED,
+	KEY_RESERVED,
+	KEY_RESERVED,
+	BTN_LEFT
+};
+
+static const struct property_entry samus_trackpad_props[] __initconst = {
+	PROPERTY_ENTRY_STRING("compatible", "atmel,maxtouch"),
+	PROPERTY_ENTRY_U32_ARRAY("linux,gpio-keymap", samus_touchpad_buttons),
+	{ }
+};
+
+static struct acpi_peripheral samus_peripherals[] __initdata = {
+	/* Touchpad */
+	{
+		.hid		= "ATML0000",
+		.properties	= samus_trackpad_props,
+	},
+	/* Touchsceen */
+	{
+		.hid		= "ATML0001",
+		.properties	= chromebook_atmel_touchscreen_props,
+	},
+};
+DECLARE_ACPI_CROS_LAPTOP(samus);
+
+static struct acpi_peripheral generic_atmel_peripherals[] __initdata = {
+	/* Touchpad */
+	{
+		.hid		= "ATML0000",
+		.properties	= chromebook_pixel_trackpad_props,
+	},
+	/* Touchsceen */
+	{
+		.hid		= "ATML0001",
+		.properties	= chromebook_atmel_touchscreen_props,
+	},
+};
+DECLARE_ACPI_CROS_LAPTOP(generic_atmel);
+
 static const struct dmi_system_id chromeos_laptop_dmi_table[] __initconst = {
 	{
 		.ident = "Samsung Series 5 550",
@@ -502,17 +603,72 @@ static const struct dmi_system_id chromeos_laptop_dmi_table[] __initconst = {
 		},
 		.driver_data = (void *)&cr48,
 	},
+	/* Devices with peripherals incompletely described in ACPI */
+	{
+		.ident = "Chromebook Pro",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Google"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Caroline"),
+		},
+		.driver_data = (void *)&samus,
+	},
+	{
+		.ident = "Google Pixel 2 (2015)",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "GOOGLE"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Samus"),
+		},
+		.driver_data = (void *)&samus,
+	},
+	{
+		.ident = "Samsung Chromebook 3",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "GOOGLE"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Celes"),
+		},
+		.driver_data = (void *)&samus,
+	},
+	{
+		/*
+		 * Other Chromebooks with Atmel touch controllers:
+		 * - Winky (touchpad)
+		 * - Clapper, Expresso, Rambi, Glimmer (touchscreen)
+		 */
+		.ident = "Other Chromebook",
+		.matches = {
+			/*
+			 * This will match all Google devices, not only devices
+			 * with Atmel, but we will validate that the device
+			 * actually has matching peripherals.
+			 */
+			DMI_MATCH(DMI_SYS_VENDOR, "GOOGLE"),
+		},
+		.driver_data = (void *)&generic_atmel,
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(dmi, chromeos_laptop_dmi_table);
 
-static int __init chromeos_laptop_scan_adapter(struct device *dev, void *data)
+static int __init chromeos_laptop_scan_peripherals(struct device *dev, void *data)
 {
-	struct i2c_adapter *adapter;
+	int error;
 
-	adapter = i2c_verify_adapter(dev);
-	if (adapter)
-		chromeos_laptop_check_adapter(adapter);
+	if (dev->type == &i2c_adapter_type) {
+		chromeos_laptop_check_adapter(to_i2c_adapter(dev));
+	} else if (dev->type == &i2c_client_type) {
+		if (chromeos_laptop_adjust_client(to_i2c_client(dev))) {
+			/*
+			 * Now that we have needed properties re-trigger
+			 * driver probe in case driver was initialized
+			 * earlier and probe failed.
+			 */
+			error = device_attach(dev);
+			if (error < 0)
+				dev_warn(dev,
+					 "%s: device_attach() failed: %d\n",
+					 __func__, error);
+		}
+	}
 
 	return 0;
 }
@@ -556,27 +712,24 @@ static int __init chromeos_laptop_setup_irq(struct i2c_peripheral *i2c_dev)
 	return 0;
 }
 
-static struct chromeos_laptop * __init
-chromeos_laptop_prepare(const struct chromeos_laptop *src)
+static int __init
+chromeos_laptop_prepare_i2c_peripherals(struct chromeos_laptop *cros_laptop,
+					const struct chromeos_laptop *src)
 {
-	struct chromeos_laptop *cros_laptop;
 	struct i2c_peripheral *i2c_dev;
 	struct i2c_board_info *info;
-	int error;
 	int i;
+	int error;
 
-	cros_laptop = kzalloc(sizeof(*cros_laptop), GFP_KERNEL);
-	if (!cros_laptop)
-		return ERR_PTR(-ENOMEM);
+	if (!src->num_i2c_peripherals)
+		return 0;
 
 	cros_laptop->i2c_peripherals = kmemdup(src->i2c_peripherals,
 					       src->num_i2c_peripherals *
 						sizeof(*src->i2c_peripherals),
 					       GFP_KERNEL);
-	if (!cros_laptop->i2c_peripherals) {
-		error = -ENOMEM;
-		goto err_free_cros_laptop;
-	}
+	if (!cros_laptop->i2c_peripherals)
+		return -ENOMEM;
 
 	cros_laptop->num_i2c_peripherals = src->num_i2c_peripherals;
 
@@ -586,7 +739,7 @@ chromeos_laptop_prepare(const struct chromeos_laptop *src)
 
 		error = chromeos_laptop_setup_irq(i2c_dev);
 		if (error)
-			goto err_destroy_cros_peripherals;
+			goto err_out;
 
 		/* We need to deep-copy properties */
 		if (info->properties) {
@@ -594,14 +747,14 @@ chromeos_laptop_prepare(const struct chromeos_laptop *src)
 				property_entries_dup(info->properties);
 			if (IS_ERR(info->properties)) {
 				error = PTR_ERR(info->properties);
-				goto err_destroy_cros_peripherals;
+				goto err_out;
 			}
 		}
 	}
 
-	return cros_laptop;
+	return 0;
 
-err_destroy_cros_peripherals:
+err_out:
 	while (--i >= 0) {
 		i2c_dev = &cros_laptop->i2c_peripherals[i];
 		info = &i2c_dev->board_info;
@@ -609,13 +762,74 @@ chromeos_laptop_prepare(const struct chromeos_laptop *src)
 			property_entries_free(info->properties);
 	}
 	kfree(cros_laptop->i2c_peripherals);
-err_free_cros_laptop:
-	kfree(cros_laptop);
-	return ERR_PTR(error);
+	return error;
+}
+
+static int __init
+chromeos_laptop_prepare_acpi_peripherals(struct chromeos_laptop *cros_laptop,
+					const struct chromeos_laptop *src)
+{
+	struct acpi_peripheral *acpi_peripherals;
+	struct acpi_peripheral *acpi_dev;
+	const struct acpi_peripheral *src_dev;
+	int n_peripherals = 0;
+	int i;
+	int error;
+
+	for (i = 0; i < src->num_acpi_peripherals; i++) {
+		if (acpi_dev_present(src->acpi_peripherals[i].hid, NULL, -1))
+			n_peripherals++;
+	}
+
+	if (!n_peripherals)
+		return 0;
+
+	acpi_peripherals = kcalloc(n_peripherals,
+				   sizeof(*src->acpi_peripherals),
+				   GFP_KERNEL);
+	if (!acpi_peripherals)
+		return -ENOMEM;
+
+	acpi_dev = acpi_peripherals;
+	for (i = 0; i < src->num_acpi_peripherals; i++) {
+		src_dev = &src->acpi_peripherals[i];
+		if (!acpi_dev_present(src_dev->hid, NULL, -1))
+			continue;
+
+		*acpi_dev = *src_dev;
+
+		/* We need to deep-copy properties */
+		if (src_dev->properties) {
+			acpi_dev->properties =
+				property_entries_dup(src_dev->properties);
+			if (IS_ERR(acpi_dev->properties)) {
+				error = PTR_ERR(acpi_dev->properties);
+				goto err_out;
+			}
+		}
+
+		acpi_dev++;
+	}
+
+	cros_laptop->acpi_peripherals = acpi_peripherals;
+	cros_laptop->num_acpi_peripherals = n_peripherals;
+
+	return 0;
+
+err_out:
+	while (--i >= 0) {
+		acpi_dev = &acpi_peripherals[i];
+		if (acpi_dev->properties)
+			property_entries_free(acpi_dev->properties);
+	}
+
+	kfree(acpi_peripherals);
+	return error;
 }
 
 static void chromeos_laptop_destroy(const struct chromeos_laptop *cros_laptop)
 {
+	const struct acpi_peripheral *acpi_dev;
 	struct i2c_peripheral *i2c_dev;
 	struct i2c_board_info *info;
 	int i;
@@ -631,10 +845,41 @@ static void chromeos_laptop_destroy(const struct chromeos_laptop *cros_laptop)
 			property_entries_free(info->properties);
 	}
 
+	for (i = 0; i < cros_laptop->num_acpi_peripherals; i++) {
+		acpi_dev = &cros_laptop->acpi_peripherals[i];
+
+		if (acpi_dev->properties)
+			property_entries_free(acpi_dev->properties);
+	}
+
 	kfree(cros_laptop->i2c_peripherals);
+	kfree(cros_laptop->acpi_peripherals);
 	kfree(cros_laptop);
 }
 
+static struct chromeos_laptop * __init
+chromeos_laptop_prepare(const struct chromeos_laptop *src)
+{
+	struct chromeos_laptop *cros_laptop;
+	int error;
+
+	cros_laptop = kzalloc(sizeof(*cros_laptop), GFP_KERNEL);
+	if (!cros_laptop)
+		return ERR_PTR(-ENOMEM);
+
+	error = chromeos_laptop_prepare_i2c_peripherals(cros_laptop, src);
+	if (!error)
+		error = chromeos_laptop_prepare_acpi_peripherals(cros_laptop,
+								 src);
+
+	if (error) {
+		chromeos_laptop_destroy(cros_laptop);
+		return ERR_PTR(error);
+	}
+
+	return cros_laptop;
+}
+
 static int __init chromeos_laptop_init(void)
 {
 	const struct dmi_system_id *dmi_id;
@@ -652,21 +897,33 @@ static int __init chromeos_laptop_init(void)
 	if (IS_ERR(cros_laptop))
 		return PTR_ERR(cros_laptop);
 
+	if (!cros_laptop->num_i2c_peripherals &&
+	    !cros_laptop->num_acpi_peripherals) {
+		pr_debug("no relevant devices detected\n");
+		error = -ENODEV;
+		goto err_destroy_cros_laptop;
+	}
+
 	error = bus_register_notifier(&i2c_bus_type,
 				      &chromeos_laptop_i2c_notifier);
 	if (error) {
-		pr_err("failed to register i2c bus notifier: %d\n", error);
-		chromeos_laptop_destroy(cros_laptop);
-		return error;
+		pr_err("failed to register i2c bus notifier: %d\n",
+		       error);
+		goto err_destroy_cros_laptop;
 	}
 
 	/*
-	 * Scan adapters that have been registered before we installed
-	 * the notifier to make sure we do not miss any devices.
+	 * Scan adapters that have been registered and clients that have
+	 * been created before we installed the notifier to make sure
+	 * we do not miss any devices.
 	 */
-	i2c_for_each_dev(NULL, chromeos_laptop_scan_adapter);
+	i2c_for_each_dev(NULL, chromeos_laptop_scan_peripherals);
 
 	return 0;
+
+err_destroy_cros_laptop:
+	chromeos_laptop_destroy(cros_laptop);
+	return error;
 }
 
 static void __exit chromeos_laptop_exit(void)
diff --git a/drivers/platform/chrome/chromeos_tbmc.c b/drivers/platform/chrome/chromeos_tbmc.c
new file mode 100644
index 000000000000..b935df6a9694
--- /dev/null
+++ b/drivers/platform/chrome/chromeos_tbmc.c
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Driver to detect Tablet Mode for ChromeOS convertible.
+//
+// Copyright (C) 2017 Google, Inc.
+// Author: Gwendal Grignou <gwendal@chromium.org>
+
+#include <linux/acpi.h>
+#include <linux/input.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/printk.h>
+
+#define DRV_NAME "chromeos_tbmc"
+#define ACPI_DRV_NAME "GOOG0006"
+
+static int chromeos_tbmc_query_switch(struct acpi_device *adev,
+				     struct input_dev *idev)
+{
+	unsigned long long state;
+	acpi_status status;
+
+	status = acpi_evaluate_integer(adev->handle, "TBMC", NULL, &state);
+	if (ACPI_FAILURE(status))
+		return -ENODEV;
+
+	/* input layer checks if event is redundant */
+	input_report_switch(idev, SW_TABLET_MODE, state);
+	input_sync(idev);
+
+	return 0;
+}
+
+static __maybe_unused int chromeos_tbmc_resume(struct device *dev)
+{
+	struct acpi_device *adev = to_acpi_device(dev);
+
+	return chromeos_tbmc_query_switch(adev, adev->driver_data);
+}
+
+static void chromeos_tbmc_notify(struct acpi_device *adev, u32 event)
+{
+	switch (event) {
+	case 0x80:
+		chromeos_tbmc_query_switch(adev, adev->driver_data);
+		break;
+	default:
+		dev_err(&adev->dev, "Unexpected event: 0x%08X\n", event);
+	}
+}
+
+static int chromeos_tbmc_open(struct input_dev *idev)
+{
+	struct acpi_device *adev = input_get_drvdata(idev);
+
+	return chromeos_tbmc_query_switch(adev, idev);
+}
+
+static int chromeos_tbmc_add(struct acpi_device *adev)
+{
+	struct input_dev *idev;
+	struct device *dev = &adev->dev;
+	int ret;
+
+	idev = devm_input_allocate_device(dev);
+	if (!idev)
+		return -ENOMEM;
+
+	idev->name = "Tablet Mode Switch";
+	idev->phys = acpi_device_hid(adev);
+
+	idev->id.bustype = BUS_HOST;
+	idev->id.version = 1;
+	idev->id.product = 0;
+	idev->open = chromeos_tbmc_open;
+
+	input_set_drvdata(idev, adev);
+	adev->driver_data = idev;
+
+	input_set_capability(idev, EV_SW, SW_TABLET_MODE);
+	ret = input_register_device(idev);
+	if (ret) {
+		dev_err(dev, "cannot register input device\n");
+		return ret;
+	}
+	return 0;
+}
+
+static const struct acpi_device_id chromeos_tbmc_acpi_device_ids[] = {
+	{ ACPI_DRV_NAME, 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, chromeos_tbmc_acpi_device_ids);
+
+static const SIMPLE_DEV_PM_OPS(chromeos_tbmc_pm_ops, NULL,
+		chromeos_tbmc_resume);
+
+static struct acpi_driver chromeos_tbmc_driver = {
+	.name = DRV_NAME,
+	.class = DRV_NAME,
+	.ids = chromeos_tbmc_acpi_device_ids,
+	.ops = {
+		.add = chromeos_tbmc_add,
+		.notify = chromeos_tbmc_notify,
+	},
+	.drv.pm = &chromeos_tbmc_pm_ops,
+};
+
+module_acpi_driver(chromeos_tbmc_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("ChromeOS ACPI tablet switch driver");
diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c
index cc265ed8deb7..c62ee8e610a0 100644
--- a/drivers/platform/chrome/cros_ec_debugfs.c
+++ b/drivers/platform/chrome/cros_ec_debugfs.c
@@ -470,3 +470,23 @@ void cros_ec_debugfs_remove(struct cros_ec_dev *ec)
 	cros_ec_cleanup_console_log(ec->debug_info);
 }
 EXPORT_SYMBOL(cros_ec_debugfs_remove);
+
+void cros_ec_debugfs_suspend(struct cros_ec_dev *ec)
+{
+	/*
+	 * cros_ec_debugfs_init() failures are non-fatal; it's also possible
+	 * that we initted things but decided that console log wasn't supported.
+	 * We'll use the same set of checks that cros_ec_debugfs_remove() +
+	 * cros_ec_cleanup_console_log() end up using to handle those cases.
+	 */
+	if (ec->debug_info && ec->debug_info->log_buffer.buf)
+		cancel_delayed_work_sync(&ec->debug_info->log_poll_work);
+}
+EXPORT_SYMBOL(cros_ec_debugfs_suspend);
+
+void cros_ec_debugfs_resume(struct cros_ec_dev *ec)
+{
+	if (ec->debug_info && ec->debug_info->log_buffer.buf)
+		schedule_delayed_work(&ec->debug_info->log_poll_work, 0);
+}
+EXPORT_SYMBOL(cros_ec_debugfs_resume);
diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c
index 6ea79d495aa2..68193bb53383 100644
--- a/drivers/platform/chrome/cros_ec_lightbar.c
+++ b/drivers/platform/chrome/cros_ec_lightbar.c
@@ -170,8 +170,7 @@ static ssize_t version_show(struct device *dev,
 			    struct device_attribute *attr, char *buf)
 {
 	uint32_t version = 0, flags = 0;
-	struct cros_ec_dev *ec = container_of(dev,
-					      struct cros_ec_dev, class_dev);
+	struct cros_ec_dev *ec = to_cros_ec_dev(dev);
 	int ret;
 
 	ret = lb_throttle();
@@ -193,8 +192,7 @@ static ssize_t brightness_store(struct device *dev,
 	struct cros_ec_command *msg;
 	int ret;
 	unsigned int val;
-	struct cros_ec_dev *ec = container_of(dev,
-					      struct cros_ec_dev, class_dev);
+	struct cros_ec_dev *ec = to_cros_ec_dev(dev);
 
 	if (kstrtouint(buf, 0, &val))
 		return -EINVAL;
@@ -238,8 +236,7 @@ static ssize_t led_rgb_store(struct device *dev, struct device_attribute *attr,
 {
 	struct ec_params_lightbar *param;
 	struct cros_ec_command *msg;
-	struct cros_ec_dev *ec = container_of(dev,
-					      struct cros_ec_dev, class_dev);
+	struct cros_ec_dev *ec = to_cros_ec_dev(dev);
 	unsigned int val[4];
 	int ret, i = 0, j = 0, ok = 0;
 
@@ -311,8 +308,7 @@ static ssize_t sequence_show(struct device *dev,
 	struct ec_response_lightbar *resp;
 	struct cros_ec_command *msg;
 	int ret;
-	struct cros_ec_dev *ec = container_of(dev,
-					      struct cros_ec_dev, class_dev);
+	struct cros_ec_dev *ec = to_cros_ec_dev(dev);
 
 	msg = alloc_lightbar_cmd_msg(ec);
 	if (!msg)
@@ -439,8 +435,7 @@ static ssize_t sequence_store(struct device *dev, struct device_attribute *attr,
 	struct cros_ec_command *msg;
 	unsigned int num;
 	int ret, len;
-	struct cros_ec_dev *ec = container_of(dev,
-					      struct cros_ec_dev, class_dev);
+	struct cros_ec_dev *ec = to_cros_ec_dev(dev);
 
 	for (len = 0; len < count; len++)
 		if (!isalnum(buf[len]))
@@ -488,8 +483,7 @@ static ssize_t program_store(struct device *dev, struct device_attribute *attr,
 	int extra_bytes, max_size, ret;
 	struct ec_params_lightbar *param;
 	struct cros_ec_command *msg;
-	struct cros_ec_dev *ec = container_of(dev, struct cros_ec_dev,
-					      class_dev);
+	struct cros_ec_dev *ec = to_cros_ec_dev(dev);
 
 	/*
 	 * We might need to reject the program for size reasons. The EC
@@ -599,8 +593,7 @@ static umode_t cros_ec_lightbar_attrs_are_visible(struct kobject *kobj,
 						  struct attribute *a, int n)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
-	struct cros_ec_dev *ec = container_of(dev,
-					      struct cros_ec_dev, class_dev);
+	struct cros_ec_dev *ec = to_cros_ec_dev(dev);
 	struct platform_device *pdev = to_platform_device(ec->dev);
 	struct cros_ec_platform *pdata = pdev->dev.platform_data;
 	int is_cros_ec;
diff --git a/drivers/platform/chrome/cros_ec_lpc.c b/drivers/platform/chrome/cros_ec_lpc.c
index 3682e1539251..31c8b8c49e45 100644
--- a/drivers/platform/chrome/cros_ec_lpc.c
+++ b/drivers/platform/chrome/cros_ec_lpc.c
@@ -435,7 +435,13 @@ static int __init cros_ec_lpc_init(void)
 	int ret;
 	acpi_status status;
 
-	if (!dmi_check_system(cros_ec_lpc_dmi_table)) {
+	status = acpi_get_devices(ACPI_DRV_NAME, cros_ec_lpc_parse_device,
+				  &cros_ec_lpc_acpi_device_found, NULL);
+	if (ACPI_FAILURE(status))
+		pr_warn(DRV_NAME ": Looking for %s failed\n", ACPI_DRV_NAME);
+
+	if (!cros_ec_lpc_acpi_device_found &&
+	    !dmi_check_system(cros_ec_lpc_dmi_table)) {
 		pr_err(DRV_NAME ": unsupported system.\n");
 		return -ENODEV;
 	}
@@ -450,11 +456,6 @@ static int __init cros_ec_lpc_init(void)
 		return ret;
 	}
 
-	status = acpi_get_devices(ACPI_DRV_NAME, cros_ec_lpc_parse_device,
-				  &cros_ec_lpc_acpi_device_found, NULL);
-	if (ACPI_FAILURE(status))
-		pr_warn(DRV_NAME ": Looking for %s failed\n", ACPI_DRV_NAME);
-
 	if (!cros_ec_lpc_acpi_device_found) {
 		/* Register the device, and it'll get hooked up automatically */
 		ret = platform_device_register(&cros_ec_lpc_device);
diff --git a/drivers/platform/chrome/cros_ec_sysfs.c b/drivers/platform/chrome/cros_ec_sysfs.c
index 5a6db3fe213a..f34a50121064 100644
--- a/drivers/platform/chrome/cros_ec_sysfs.c
+++ b/drivers/platform/chrome/cros_ec_sysfs.c
@@ -34,8 +34,6 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 
-#define to_cros_ec_dev(dev)  container_of(dev, struct cros_ec_dev, class_dev)
-
 /* Accessor functions */
 
 static ssize_t reboot_show(struct device *dev,
diff --git a/drivers/platform/chrome/cros_ec_vbc.c b/drivers/platform/chrome/cros_ec_vbc.c
index 6d38e6b08334..5356f26bc022 100644
--- a/drivers/platform/chrome/cros_ec_vbc.c
+++ b/drivers/platform/chrome/cros_ec_vbc.c
@@ -29,8 +29,7 @@ static ssize_t vboot_context_read(struct file *filp, struct kobject *kobj,
 				  loff_t pos, size_t count)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
-	struct cros_ec_dev *ec = container_of(dev, struct cros_ec_dev,
-					      class_dev);
+	struct cros_ec_dev *ec = to_cros_ec_dev(dev);
 	struct cros_ec_device *ecdev = ec->ec_dev;
 	struct ec_params_vbnvcontext *params;
 	struct cros_ec_command *msg;
@@ -70,8 +69,7 @@ static ssize_t vboot_context_write(struct file *filp, struct kobject *kobj,
 				   loff_t pos, size_t count)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
-	struct cros_ec_dev *ec = container_of(dev, struct cros_ec_dev,
-					      class_dev);
+	struct cros_ec_dev *ec = to_cros_ec_dev(dev);
 	struct cros_ec_device *ecdev = ec->ec_dev;
 	struct ec_params_vbnvcontext *params;
 	struct cros_ec_command *msg;
@@ -111,8 +109,7 @@ static umode_t cros_ec_vbc_is_visible(struct kobject *kobj,
 				      struct bin_attribute *a, int n)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
-	struct cros_ec_dev *ec = container_of(dev, struct cros_ec_dev,
-					      class_dev);
+	struct cros_ec_dev *ec = to_cros_ec_dev(dev);
 	struct device_node *np = ec->ec_dev->dev->of_node;
 
 	if (IS_ENABLED(CONFIG_OF) && np) {
diff --git a/drivers/platform/mellanox/mlxreg-hotplug.c b/drivers/platform/mellanox/mlxreg-hotplug.c
index ea9e7f4479ca..b56953ae1ab8 100644
--- a/drivers/platform/mellanox/mlxreg-hotplug.c
+++ b/drivers/platform/mellanox/mlxreg-hotplug.c
@@ -59,9 +59,11 @@
 /**
  * struct mlxreg_hotplug_priv_data - platform private data:
  * @irq: platform device interrupt number;
+ * @dev: basic device;
  * @pdev: platform device;
  * @plat: platform data;
- * @dwork: delayed work template;
+ * @regmap: register map handle;
+ * @dwork_irq: delayed work template;
  * @lock: spin lock;
  * @hwmon: hwmon device;
  * @mlxreg_hotplug_attr: sysfs attributes array;
@@ -71,6 +73,7 @@
  * @cell: location of top aggregation interrupt register;
  * @mask: top aggregation interrupt common mask;
  * @aggr_cache: last value of aggregation register status;
+ * @after_probe: flag indication probing completion;
  */
 struct mlxreg_hotplug_priv_data {
 	int irq;
@@ -79,7 +82,6 @@ struct mlxreg_hotplug_priv_data {
 	struct mlxreg_hotplug_platform_data *plat;
 	struct regmap *regmap;
 	struct delayed_work dwork_irq;
-	struct delayed_work dwork;
 	spinlock_t lock; /* sync with interrupt */
 	struct device *hwmon;
 	struct attribute *mlxreg_hotplug_attr[MLXREG_HOTPLUG_ATTRS_MAX + 1];
diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig
index 566644bb496a..f27cb186437d 100644
--- a/drivers/platform/x86/Kconfig
+++ b/drivers/platform/x86/Kconfig
@@ -866,6 +866,7 @@ config ACPI_CMPC
 config INTEL_CHT_INT33FE
 	tristate "Intel Cherry Trail ACPI INT33FE Driver"
 	depends on X86 && ACPI && I2C && REGULATOR
+	depends on CHARGER_BQ24190=y || (CHARGER_BQ24190=m && m)
 	---help---
 	  This driver add support for the INT33FE ACPI device found on
 	  some Intel Cherry Trail devices.
@@ -877,8 +878,7 @@ config INTEL_CHT_INT33FE
 	  i2c drivers for these chips can bind to the them.
 
 	  If you enable this driver it is advised to also select
-	  CONFIG_TYPEC_FUSB302=m, CONFIG_CHARGER_BQ24190=m and
-	  CONFIG_BATTERY_MAX17042=m.
+	  CONFIG_TYPEC_FUSB302=m and CONFIG_BATTERY_MAX17042=m.
 
 config INTEL_INT0002_VGPIO
 	tristate "Intel ACPI INT0002 Virtual GPIO driver"
diff --git a/drivers/platform/x86/acer-wmi.c b/drivers/platform/x86/acer-wmi.c
index 1be71f956d5c..8952173dd380 100644
--- a/drivers/platform/x86/acer-wmi.c
+++ b/drivers/platform/x86/acer-wmi.c
@@ -129,6 +129,7 @@ static const struct key_entry acer_wmi_keymap[] __initconst = {
 	{KE_IGNORE, 0x83, {KEY_TOUCHPAD_TOGGLE} },
 	{KE_KEY, 0x85, {KEY_TOUCHPAD_TOGGLE} },
 	{KE_KEY, 0x86, {KEY_WLAN} },
+	{KE_KEY, 0x87, {KEY_POWER} },
 	{KE_END, 0}
 };
 
diff --git a/drivers/platform/x86/apple-gmux.c b/drivers/platform/x86/apple-gmux.c
index 7c4eb86c851e..fd2ffebc868f 100644
--- a/drivers/platform/x86/apple-gmux.c
+++ b/drivers/platform/x86/apple-gmux.c
@@ -495,7 +495,7 @@ static int gmux_set_power_state(enum vga_switcheroo_client_id id,
 	return gmux_set_discrete_state(apple_gmux_data, state);
 }
 
-static int gmux_get_client_id(struct pci_dev *pdev)
+static enum vga_switcheroo_client_id gmux_get_client_id(struct pci_dev *pdev)
 {
 	/*
 	 * Early Macbook Pros with switchable graphics use nvidia
diff --git a/drivers/platform/x86/asus-laptop.c b/drivers/platform/x86/asus-laptop.c
index c4768be24ba9..700c48ddfa7c 100644
--- a/drivers/platform/x86/asus-laptop.c
+++ b/drivers/platform/x86/asus-laptop.c
@@ -1593,8 +1593,7 @@ static umode_t asus_sysfs_is_visible(struct kobject *kobj,
 				    int idx)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
-	struct platform_device *pdev = to_platform_device(dev);
-	struct asus_laptop *asus = platform_get_drvdata(pdev);
+	struct asus_laptop *asus = dev_get_drvdata(dev);
 	acpi_handle handle = asus->handle;
 	bool supported;
 
diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c
index a32c5c00e0e7..ef87e78ca772 100644
--- a/drivers/platform/x86/asus-wmi.c
+++ b/drivers/platform/x86/asus-wmi.c
@@ -1862,8 +1862,7 @@ static umode_t asus_sysfs_is_visible(struct kobject *kobj,
 				    struct attribute *attr, int idx)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
-	struct platform_device *pdev = to_platform_device(dev);
-	struct asus_wmi *asus = platform_get_drvdata(pdev);
+	struct asus_wmi *asus = dev_get_drvdata(dev);
 	bool ok = true;
 	int devid = -1;
 
diff --git a/drivers/platform/x86/dell-smbios-base.c b/drivers/platform/x86/dell-smbios-base.c
index 33fb2a20458a..9dc282ed5a9e 100644
--- a/drivers/platform/x86/dell-smbios-base.c
+++ b/drivers/platform/x86/dell-smbios-base.c
@@ -555,11 +555,10 @@ static void free_group(struct platform_device *pdev)
 
 static int __init dell_smbios_init(void)
 {
-	const struct dmi_device *valid;
 	int ret, wmi, smm;
 
-	valid = dmi_find_device(DMI_DEV_TYPE_OEM_STRING, "Dell System", NULL);
-	if (!valid) {
+	if (!dmi_find_device(DMI_DEV_TYPE_OEM_STRING, "Dell System", NULL) &&
+	    !dmi_find_device(DMI_DEV_TYPE_OEM_STRING, "www.dell.com", NULL)) {
 		pr_err("Unable to run on non-Dell system\n");
 		return -ENODEV;
 	}
diff --git a/drivers/platform/x86/dell-wmi.c b/drivers/platform/x86/dell-wmi.c
index 8d102195a392..7c918e373ac2 100644
--- a/drivers/platform/x86/dell-wmi.c
+++ b/drivers/platform/x86/dell-wmi.c
@@ -233,7 +233,7 @@ static const u16 bios_to_linux_keycode[256] = {
 	[18]	= KEY_PROG1,
 	[19]	= KEY_BRIGHTNESSDOWN,
 	[20]	= KEY_BRIGHTNESSUP,
-	[21]	= KEY_UNKNOWN,
+	[21]	= KEY_BRIGHTNESS_AUTO,
 	[22]	= KEY_KBDILLUMTOGGLE,
 	[23]	= KEY_UNKNOWN,
 	[24]	= KEY_SWITCHVIDEOMODE,
diff --git a/drivers/platform/x86/fujitsu-laptop.c b/drivers/platform/x86/fujitsu-laptop.c
index cd95b6f3a064..6afeaece2f50 100644
--- a/drivers/platform/x86/fujitsu-laptop.c
+++ b/drivers/platform/x86/fujitsu-laptop.c
@@ -91,6 +91,9 @@
 #define FLAG_RFKILL			BIT(5)
 #define FLAG_LID			BIT(8)
 #define FLAG_DOCK			BIT(9)
+#define FLAG_TOUCHPAD_TOGGLE		BIT(26)
+#define FLAG_MICMUTE			BIT(29)
+#define FLAG_SOFTKEYS			(FLAG_RFKILL | FLAG_TOUCHPAD_TOGGLE | FLAG_MICMUTE)
 
 /* FUNC interface - LED control */
 #define FUNC_LED_OFF			BIT(0)
@@ -456,14 +459,15 @@ static void acpi_fujitsu_bl_notify(struct acpi_device *device, u32 event)
 /* ACPI device for hotkey handling */
 
 static const struct key_entry keymap_default[] = {
-	{ KE_KEY, KEY1_CODE, { KEY_PROG1 } },
-	{ KE_KEY, KEY2_CODE, { KEY_PROG2 } },
-	{ KE_KEY, KEY3_CODE, { KEY_PROG3 } },
-	{ KE_KEY, KEY4_CODE, { KEY_PROG4 } },
-	{ KE_KEY, KEY5_CODE, { KEY_RFKILL } },
-	{ KE_KEY, BIT(5),    { KEY_RFKILL } },
-	{ KE_KEY, BIT(26),   { KEY_TOUCHPAD_TOGGLE } },
-	{ KE_KEY, BIT(29),   { KEY_MICMUTE } },
+	{ KE_KEY, KEY1_CODE,            { KEY_PROG1 } },
+	{ KE_KEY, KEY2_CODE,            { KEY_PROG2 } },
+	{ KE_KEY, KEY3_CODE,            { KEY_PROG3 } },
+	{ KE_KEY, KEY4_CODE,            { KEY_PROG4 } },
+	{ KE_KEY, KEY5_CODE,            { KEY_RFKILL } },
+	/* Soft keys read from status flags */
+	{ KE_KEY, FLAG_RFKILL,          { KEY_RFKILL } },
+	{ KE_KEY, FLAG_TOUCHPAD_TOGGLE, { KEY_TOUCHPAD_TOGGLE } },
+	{ KE_KEY, FLAG_MICMUTE,         { KEY_MICMUTE } },
 	{ KE_END, 0 }
 };
 
@@ -903,7 +907,8 @@ static void acpi_fujitsu_laptop_release(struct acpi_device *device)
 static void acpi_fujitsu_laptop_notify(struct acpi_device *device, u32 event)
 {
 	struct fujitsu_laptop *priv = acpi_driver_data(device);
-	int scancode, i = 0, ret;
+	unsigned long flags;
+	int scancode, i = 0;
 	unsigned int irb;
 
 	if (event != ACPI_FUJITSU_NOTIFY_CODE) {
@@ -930,21 +935,17 @@ static void acpi_fujitsu_laptop_notify(struct acpi_device *device, u32 event)
 					 "Unknown GIRB result [%x]\n", irb);
 	}
 
-	/* On some models (first seen on the Skylake-based Lifebook
-	 * E736/E746/E756), the touchpad toggle hotkey (Fn+F4) is
-	 * handled in software; its state is queried using FUNC_FLAGS
+	/*
+	 * First seen on the Skylake-based Lifebook E736/E746/E756), the
+	 * touchpad toggle hotkey (Fn+F4) is handled in software. Other models
+	 * have since added additional "soft keys". These are reported in the
+	 * status flags queried using FUNC_FLAGS.
 	 */
-	if (priv->flags_supported & (BIT(5) | BIT(26) | BIT(29))) {
-		ret = call_fext_func(device, FUNC_FLAGS, 0x1, 0x0, 0x0);
-		if (ret & BIT(5))
-			sparse_keymap_report_event(priv->input,
-						   BIT(5), 1, true);
-		if (ret & BIT(26))
-			sparse_keymap_report_event(priv->input,
-						   BIT(26), 1, true);
-		if (ret & BIT(29))
-			sparse_keymap_report_event(priv->input,
-						   BIT(29), 1, true);
+	if (priv->flags_supported & (FLAG_SOFTKEYS)) {
+		flags = call_fext_func(device, FUNC_FLAGS, 0x1, 0x0, 0x0);
+		flags &= (FLAG_SOFTKEYS);
+		for_each_set_bit(i, &flags, BITS_PER_LONG)
+			sparse_keymap_report_event(priv->input, BIT(i), 1, true);
 	}
 }
 
diff --git a/drivers/platform/x86/ideapad-laptop.c b/drivers/platform/x86/ideapad-laptop.c
index 535199c9e6bc..2b3663af1967 100644
--- a/drivers/platform/x86/ideapad-laptop.c
+++ b/drivers/platform/x86/ideapad-laptop.c
@@ -43,6 +43,7 @@
 #define IDEAPAD_RFKILL_DEV_NUM	(3)
 
 #define BM_CONSERVATION_BIT (5)
+#define HA_FNLOCK_BIT       (10)
 
 #define CFG_BT_BIT	(16)
 #define CFG_3G_BIT	(17)
@@ -59,6 +60,8 @@ static const char *const ideapad_wmi_fnesc_events[] = {
 enum {
 	BMCMD_CONSERVATION_ON = 3,
 	BMCMD_CONSERVATION_OFF = 5,
+	HACMD_FNLOCK_ON = 0xe,
+	HACMD_FNLOCK_OFF = 0xf,
 };
 
 enum {
@@ -139,11 +142,11 @@ static int method_gbmd(acpi_handle handle, unsigned long *ret)
 	return result;
 }
 
-static int method_sbmc(acpi_handle handle, int cmd)
+static int method_int1(acpi_handle handle, char *method, int cmd)
 {
 	acpi_status status;
 
-	status = acpi_execute_simple_method(handle, "SBMC", cmd);
+	status = acpi_execute_simple_method(handle, method, cmd);
 	return ACPI_FAILURE(status) ? -1 : 0;
 }
 
@@ -487,7 +490,7 @@ static ssize_t conservation_mode_store(struct device *dev,
 	if (ret)
 		return ret;
 
-	ret = method_sbmc(priv->adev->handle, state ?
+	ret = method_int1(priv->adev->handle, "SBMC", state ?
 					      BMCMD_CONSERVATION_ON :
 					      BMCMD_CONSERVATION_OFF);
 	if (ret < 0)
@@ -497,11 +500,51 @@ static ssize_t conservation_mode_store(struct device *dev,
 
 static DEVICE_ATTR_RW(conservation_mode);
 
+static ssize_t fn_lock_show(struct device *dev,
+			    struct device_attribute *attr,
+			    char *buf)
+{
+	struct ideapad_private *priv = dev_get_drvdata(dev);
+	unsigned long result;
+	int hals;
+	int fail = read_method_int(priv->adev->handle, "HALS", &hals);
+
+	if (fail)
+		return sprintf(buf, "-1\n");
+
+	result = hals;
+	return sprintf(buf, "%u\n", test_bit(HA_FNLOCK_BIT, &result));
+}
+
+static ssize_t fn_lock_store(struct device *dev,
+			     struct device_attribute *attr,
+			     const char *buf, size_t count)
+{
+	struct ideapad_private *priv = dev_get_drvdata(dev);
+	bool state;
+	int ret;
+
+	ret = kstrtobool(buf, &state);
+	if (ret)
+		return ret;
+
+	ret = method_int1(priv->adev->handle, "SALS", state ?
+			  HACMD_FNLOCK_ON :
+			  HACMD_FNLOCK_OFF);
+	if (ret < 0)
+		return -EIO;
+	return count;
+}
+
+static DEVICE_ATTR_RW(fn_lock);
+
+
 static struct attribute *ideapad_attributes[] = {
 	&dev_attr_camera_power.attr,
 	&dev_attr_fan_mode.attr,
 	&dev_attr_touchpad.attr,
 	&dev_attr_conservation_mode.attr,
+	&dev_attr_fn_lock.attr,
 	NULL
 };
 
@@ -522,6 +565,9 @@ static umode_t ideapad_is_visible(struct kobject *kobj,
 	} else if (attr == &dev_attr_conservation_mode.attr) {
 		supported = acpi_has_method(priv->adev->handle, "GBMD") &&
 			    acpi_has_method(priv->adev->handle, "SBMC");
+	} else if (attr == &dev_attr_fn_lock.attr) {
+		supported = acpi_has_method(priv->adev->handle, "HALS") &&
+			acpi_has_method(priv->adev->handle, "SALS");
 	} else
 		supported = true;
 
@@ -1079,6 +1125,13 @@ static const struct dmi_system_id no_hw_rfkill_list[] = {
 			DMI_MATCH(DMI_PRODUCT_VERSION, "Lenovo ideapad Y700-17ISK"),
 		},
 	},
+	{
+		.ident = "Lenovo ideapad MIIX 720-12IKB",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_VERSION, "MIIX 720-12IKB"),
+		},
+	},
 	{
 		.ident = "Lenovo Legion Y520-15IKBN",
 		.matches = {
diff --git a/drivers/platform/x86/intel_scu_ipc.c b/drivers/platform/x86/intel_scu_ipc.c
index 2c85f75e32b0..75c8fef7a482 100644
--- a/drivers/platform/x86/intel_scu_ipc.c
+++ b/drivers/platform/x86/intel_scu_ipc.c
@@ -584,11 +584,11 @@ int intel_scu_ipc_i2c_cntrl(u32 addr, u32 *data)
 	if (cmd == IPC_I2C_READ) {
 		writel(addr, scu->i2c_base + IPC_I2C_CNTRL_ADDR);
 		/* Write not getting updated without delay */
-		mdelay(1);
+		usleep_range(1000, 2000);
 		*data = readl(scu->i2c_base + I2C_DATA_ADDR);
 	} else if (cmd == IPC_I2C_WRITE) {
 		writel(*data, scu->i2c_base + I2C_DATA_ADDR);
-		mdelay(1);
+		usleep_range(1000, 2000);
 		writel(addr, scu->i2c_base + IPC_I2C_CNTRL_ADDR);
 	} else {
 		dev_err(scu->dev,
diff --git a/drivers/platform/x86/samsung-laptop.c b/drivers/platform/x86/samsung-laptop.c
index 03305e0b89ff..7b160ee98115 100644
--- a/drivers/platform/x86/samsung-laptop.c
+++ b/drivers/platform/x86/samsung-laptop.c
@@ -1216,8 +1216,7 @@ static umode_t samsung_sysfs_is_visible(struct kobject *kobj,
 					struct attribute *attr, int idx)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
-	struct platform_device *pdev = to_platform_device(dev);
-	struct samsung_laptop *samsung = platform_get_drvdata(pdev);
+	struct samsung_laptop *samsung = dev_get_drvdata(dev);
 	bool ok = true;
 
 	if (attr == &dev_attr_performance_level.attr)
diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c
index da1ca4856ea1..a0e9ce0d85b9 100644
--- a/drivers/platform/x86/thinkpad_acpi.c
+++ b/drivers/platform/x86/thinkpad_acpi.c
@@ -212,7 +212,12 @@ enum tpacpi_hkey_event_t {
 	TP_HKEY_EV_ALARM_BAT_XHOT	= 0x6012, /* battery critically hot */
 	TP_HKEY_EV_ALARM_SENSOR_HOT	= 0x6021, /* sensor too hot */
 	TP_HKEY_EV_ALARM_SENSOR_XHOT	= 0x6022, /* sensor critically hot */
-	TP_HKEY_EV_THM_TABLE_CHANGED	= 0x6030, /* thermal table changed */
+	TP_HKEY_EV_THM_TABLE_CHANGED	= 0x6030, /* windows; thermal table changed */
+	TP_HKEY_EV_THM_CSM_COMPLETED    = 0x6032, /* windows; thermal control set
+						   * command completed. Related to
+						   * AML DYTC */
+	TP_HKEY_EV_THM_TRANSFM_CHANGED  = 0x60F0, /* windows; thermal transformation
+						   * changed. Related to AML GMTS */
 
 	/* AC-related events */
 	TP_HKEY_EV_AC_CHANGED		= 0x6040, /* AC status changed */
@@ -4034,15 +4039,23 @@ static bool hotkey_notify_6xxx(const u32 hkey,
 				 bool *send_acpi_ev,
 				 bool *ignore_acpi_ev)
 {
-	bool known = true;
-
 	/* 0x6000-0x6FFF: thermal alarms/notices and keyboard events */
 	*send_acpi_ev = true;
 	*ignore_acpi_ev = false;
 
 	switch (hkey) {
 	case TP_HKEY_EV_THM_TABLE_CHANGED:
-		pr_info("EC reports that Thermal Table has changed\n");
+		pr_debug("EC reports: Thermal Table has changed\n");
+		/* recommended action: do nothing, we don't have
+		 * Lenovo ATM information */
+		return true;
+	case TP_HKEY_EV_THM_CSM_COMPLETED:
+		pr_debug("EC reports: Thermal Control Command set completed (DYTC)\n");
+		/* recommended action: do nothing, we don't have
+		 * Lenovo ATM information */
+		return true;
+	case TP_HKEY_EV_THM_TRANSFM_CHANGED:
+		pr_debug("EC reports: Thermal Transformation changed (GMTS)\n");
 		/* recommended action: do nothing, we don't have
 		 * Lenovo ATM information */
 		return true;
@@ -4083,7 +4096,7 @@ static bool hotkey_notify_6xxx(const u32 hkey,
 		tpacpi_input_send_tabletsw();
 		hotkey_tablet_mode_notify_change();
 		*send_acpi_ev = false;
-		break;
+		return true;
 
 	case TP_HKEY_EV_PALM_DETECTED:
 	case TP_HKEY_EV_PALM_UNDETECTED:
@@ -4092,13 +4105,12 @@ static bool hotkey_notify_6xxx(const u32 hkey,
 		return true;
 
 	default:
-		pr_warn("unknown possible thermal alarm or keyboard event received\n");
-		known = false;
+		/* report simply as unknown, no sensor dump */
+		return false;
 	}
 
 	thermal_dump_all_sensors();
-
-	return known;
+	return true;
 }
 
 static void hotkey_notify(struct ibm_struct *ibm, u32 event)
diff --git a/drivers/platform/x86/toshiba_acpi.c b/drivers/platform/x86/toshiba_acpi.c
index e8d058c5ef21..eef76bfa5d73 100644
--- a/drivers/platform/x86/toshiba_acpi.c
+++ b/drivers/platform/x86/toshiba_acpi.c
@@ -1689,19 +1689,6 @@ static int version_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int version_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, version_proc_show, PDE_DATA(inode));
-}
-
-static const struct file_operations version_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= version_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 /*
  * Proc and module init
  */
@@ -1722,8 +1709,8 @@ static void create_toshiba_proc_entries(struct toshiba_acpi_dev *dev)
 	if (dev->hotkey_dev)
 		proc_create_data("keys", S_IRUGO | S_IWUSR, toshiba_proc_dir,
 				 &keys_proc_fops, dev);
-	proc_create_data("version", S_IRUGO, toshiba_proc_dir,
-			 &version_proc_fops, dev);
+	proc_create_single_data("version", S_IRUGO, toshiba_proc_dir,
+			version_proc_show, dev);
 }
 
 static void remove_toshiba_proc_entries(struct toshiba_acpi_dev *dev)
diff --git a/drivers/pnp/isapnp/Kconfig b/drivers/pnp/isapnp/Kconfig
index f1ef36673ad4..a1af146d2d90 100644
--- a/drivers/pnp/isapnp/Kconfig
+++ b/drivers/pnp/isapnp/Kconfig
@@ -3,7 +3,7 @@
 #
 config ISAPNP
 	bool "ISA Plug and Play support"
-	depends on ISA
+	depends on ISA || COMPILE_TEST
 	help
 	  Say Y here if you would like support for ISA Plug and Play devices.
 	  Some information is in <file:Documentation/isapnp.txt>.
diff --git a/drivers/pnp/pnpbios/proc.c b/drivers/pnp/pnpbios/proc.c
index 7d4aca7948dd..fe1c8f5d9af0 100644
--- a/drivers/pnp/pnpbios/proc.c
+++ b/drivers/pnp/pnpbios/proc.c
@@ -47,19 +47,6 @@ static int pnpconfig_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int pnpconfig_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, pnpconfig_proc_show, NULL);
-}
-
-static const struct file_operations pnpconfig_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= pnpconfig_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int escd_info_proc_show(struct seq_file *m, void *v)
 {
 	struct escd_info_struc escd;
@@ -74,19 +61,6 @@ static int escd_info_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int escd_info_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, escd_info_proc_show, NULL);
-}
-
-static const struct file_operations escd_info_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= escd_info_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 #define MAX_SANE_ESCD_SIZE (32*1024)
 static int escd_proc_show(struct seq_file *m, void *v)
 {
@@ -129,19 +103,6 @@ static int escd_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int escd_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, escd_proc_show, NULL);
-}
-
-static const struct file_operations escd_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= escd_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int pnp_legacyres_proc_show(struct seq_file *m, void *v)
 {
 	void *buf;
@@ -159,19 +120,6 @@ static int pnp_legacyres_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int pnp_legacyres_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, pnp_legacyres_proc_show, NULL);
-}
-
-static const struct file_operations pnp_legacyres_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= pnp_legacyres_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int pnp_devices_proc_show(struct seq_file *m, void *v)
 {
 	struct pnp_bios_node *node;
@@ -202,19 +150,6 @@ static int pnp_devices_proc_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int pnp_devices_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, pnp_devices_proc_show, NULL);
-}
-
-static const struct file_operations pnp_devices_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= pnp_devices_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 static int pnpbios_proc_show(struct seq_file *m, void *v)
 {
 	void *data = m->private;
@@ -318,12 +253,13 @@ int __init pnpbios_proc_init(void)
 	proc_pnp_boot = proc_mkdir("boot", proc_pnp);
 	if (!proc_pnp_boot)
 		return -EIO;
-	proc_create("devices", 0, proc_pnp, &pnp_devices_proc_fops);
-	proc_create("configuration_info", 0, proc_pnp, &pnpconfig_proc_fops);
-	proc_create("escd_info", 0, proc_pnp, &escd_info_proc_fops);
-	proc_create("escd", S_IRUSR, proc_pnp, &escd_proc_fops);
-	proc_create("legacy_device_resources", 0, proc_pnp, &pnp_legacyres_proc_fops);
-
+	proc_create_single("devices", 0, proc_pnp, pnp_devices_proc_show);
+	proc_create_single("configuration_info", 0, proc_pnp,
+			pnpconfig_proc_show);
+	proc_create_single("escd_info", 0, proc_pnp, escd_info_proc_show);
+	proc_create_single("escd", S_IRUSR, proc_pnp, escd_proc_show);
+	proc_create_single("legacy_device_resources", 0, proc_pnp,
+			pnp_legacyres_proc_show);
 	return 0;
 }
 
diff --git a/drivers/power/avs/rockchip-io-domain.c b/drivers/power/avs/rockchip-io-domain.c
index ed2b109ae8fc..d6a5e6bf5f12 100644
--- a/drivers/power/avs/rockchip-io-domain.c
+++ b/drivers/power/avs/rockchip-io-domain.c
@@ -39,6 +39,10 @@
 #define MAX_VOLTAGE_1_8		1980000
 #define MAX_VOLTAGE_3_3		3600000
 
+#define PX30_IO_VSEL			0x180
+#define PX30_IO_VSEL_VCCIO6_SRC		BIT(0)
+#define PX30_IO_VSEL_VCCIO6_SUPPLY_NUM	1
+
 #define RK3288_SOC_CON2			0x24c
 #define RK3288_SOC_CON2_FLASH0		BIT(7)
 #define RK3288_SOC_FLASH_SUPPLY_NUM	2
@@ -151,6 +155,25 @@ static int rockchip_iodomain_notify(struct notifier_block *nb,
 	return NOTIFY_OK;
 }
 
+static void px30_iodomain_init(struct rockchip_iodomain *iod)
+{
+	int ret;
+	u32 val;
+
+	/* if no VCCIO0 supply we should leave things alone */
+	if (!iod->supplies[PX30_IO_VSEL_VCCIO6_SUPPLY_NUM].reg)
+		return;
+
+	/*
+	 * set vccio0 iodomain to also use this framework
+	 * instead of a special gpio.
+	 */
+	val = PX30_IO_VSEL_VCCIO6_SRC | (PX30_IO_VSEL_VCCIO6_SRC << 16);
+	ret = regmap_write(iod->grf, PX30_IO_VSEL, val);
+	if (ret < 0)
+		dev_warn(iod->dev, "couldn't update vccio0 ctrl\n");
+}
+
 static void rk3288_iodomain_init(struct rockchip_iodomain *iod)
 {
 	int ret;
@@ -227,6 +250,43 @@ static void rk3399_pmu_iodomain_init(struct rockchip_iodomain *iod)
 		dev_warn(iod->dev, "couldn't update pmu io iodomain ctrl\n");
 }
 
+static const struct rockchip_iodomain_soc_data soc_data_px30 = {
+	.grf_offset = 0x180,
+	.supply_names = {
+		NULL,
+		"vccio6",
+		"vccio1",
+		"vccio2",
+		"vccio3",
+		"vccio4",
+		"vccio5",
+		"vccio-oscgpi",
+	},
+	.init = px30_iodomain_init,
+};
+
+static const struct rockchip_iodomain_soc_data soc_data_px30_pmu = {
+	.grf_offset = 0x100,
+	.supply_names = {
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		"pmuio1",
+		"pmuio2",
+	},
+};
+
 /*
  * On the rk3188 the io-domains are handled by a shared register with the
  * lower 8 bits being still being continuing drive-strength settings.
@@ -380,6 +440,14 @@ static const struct rockchip_iodomain_soc_data soc_data_rv1108_pmu = {
 };
 
 static const struct of_device_id rockchip_iodomain_match[] = {
+	{
+		.compatible = "rockchip,px30-io-voltage-domain",
+		.data = (void *)&soc_data_px30
+	},
+	{
+		.compatible = "rockchip,px30-pmu-io-voltage-domain",
+		.data = (void *)&soc_data_px30_pmu
+	},
 	{
 		.compatible = "rockchip,rk3188-io-voltage-domain",
 		.data = &soc_data_rk3188
diff --git a/drivers/power/reset/gpio-poweroff.c b/drivers/power/reset/gpio-poweroff.c
index 6273ad3b411d..38206c39b3bf 100644
--- a/drivers/power/reset/gpio-poweroff.c
+++ b/drivers/power/reset/gpio-poweroff.c
@@ -35,11 +35,11 @@ static void gpio_poweroff_do_poweroff(void)
 	gpiod_direction_output(reset_gpio, 1);
 	mdelay(100);
 	/* drive inactive, also active->inactive edge */
-	gpiod_set_value(reset_gpio, 0);
+	gpiod_set_value_cansleep(reset_gpio, 0);
 	mdelay(100);
 
 	/* drive it active, also inactive->active edge */
-	gpiod_set_value(reset_gpio, 1);
+	gpiod_set_value_cansleep(reset_gpio, 1);
 
 	/* give it some time */
 	mdelay(timeout);
diff --git a/drivers/power/supply/ab8500_bmdata.c b/drivers/power/supply/ab8500_bmdata.c
index 4a7ed50d1dc5..7b2b69916f48 100644
--- a/drivers/power/supply/ab8500_bmdata.c
+++ b/drivers/power/supply/ab8500_bmdata.c
@@ -430,13 +430,6 @@ static const struct abx500_maxim_parameters ab8500_maxi_params = {
 	.charger_curr_step = 100,
 };
 
-static const struct abx500_maxim_parameters abx540_maxi_params = {
-	.ena_maxi = true,
-	.chg_curr = 3000,
-	.wait_cycles = 10,
-	.charger_curr_step = 200,
-};
-
 static const struct abx500_bm_charger_parameters chg = {
 	.usb_volt_max		= 5500,
 	.usb_curr_max		= 1500,
@@ -453,17 +446,6 @@ static int ab8500_charge_output_curr_map[] = {
         900,    1000,   1100,   1200,   1300,   1400,   1500,   1500,
 };
 
-static int ab8540_charge_output_curr_map[] = {
-        0,      0,      0,      75,     100,    125,    150,    175,
-        200,    225,    250,    275,    300,    325,    350,    375,
-        400,    425,    450,    475,    500,    525,    550,    575,
-        600,    625,    650,    675,    700,    725,    750,    775,
-        800,    825,    850,    875,    900,    925,    950,    975,
-        1000,   1025,   1050,   1075,   1100,   1125,   1150,   1175,
-        1200,   1225,   1250,   1275,   1300,   1325,   1350,   1375,
-        1400,   1425,   1450,   1500,   1600,   1700,   1900,   2000,
-};
-
 /*
  * This array maps the raw hex value to charger input current used by the
  * AB8500 values
@@ -473,17 +455,6 @@ static int ab8500_charge_input_curr_map[] = {
         700,    800,    900,    1000,   1100,   1300,   1400,   1500,
 };
 
-static int ab8540_charge_input_curr_map[] = {
-        25,     50,     75,     100,    125,    150,    175,    200,
-        225,    250,    275,    300,    325,    350,    375,    400,
-        425,    450,    475,    500,    525,    550,    575,    600,
-        625,    650,    675,    700,    725,    750,    775,    800,
-        825,    850,    875,    900,    925,    950,    975,    1000,
-        1025,   1050,   1075,   1100,   1125,   1150,   1175,   1200,
-        1225,   1250,   1275,   1300,   1325,   1350,   1375,   1400,
-        1425,   1450,   1475,   1500,   1500,   1500,   1500,   1500,
-};
-
 struct abx500_bm_data ab8500_bm_data = {
 	.temp_under             = 3,
 	.temp_low               = 8,
@@ -518,40 +489,6 @@ struct abx500_bm_data ab8500_bm_data = {
         .n_chg_in_curr          = ARRAY_SIZE(ab8500_charge_input_curr_map),
 };
 
-struct abx500_bm_data ab8540_bm_data = {
-        .temp_under             = 3,
-        .temp_low               = 8,
-        .temp_high              = 43,
-        .temp_over              = 48,
-        .main_safety_tmr_h      = 4,
-        .temp_interval_chg      = 20,
-        .temp_interval_nochg    = 120,
-        .usb_safety_tmr_h       = 4,
-        .bkup_bat_v             = BUP_VCH_SEL_2P6V,
-        .bkup_bat_i             = BUP_ICH_SEL_150UA,
-        .no_maintenance         = false,
-        .capacity_scaling       = false,
-        .adc_therm              = ABx500_ADC_THERM_BATCTRL,
-        .chg_unknown_bat        = false,
-        .enable_overshoot       = false,
-        .fg_res                 = 100,
-        .cap_levels             = &cap_levels,
-        .bat_type               = bat_type_thermistor,
-        .n_btypes               = ARRAY_SIZE(bat_type_thermistor),
-        .batt_id                = 0,
-        .interval_charging      = 5,
-        .interval_not_charging  = 120,
-        .temp_hysteresis        = 3,
-        .gnd_lift_resistance    = 0,
-        .maxi                   = &abx540_maxi_params,
-        .chg_params             = &chg,
-        .fg_params              = &fg,
-        .chg_output_curr        = ab8540_charge_output_curr_map,
-        .n_chg_out_curr         = ARRAY_SIZE(ab8540_charge_output_curr_map),
-        .chg_input_curr         = ab8540_charge_input_curr_map,
-        .n_chg_in_curr          = ARRAY_SIZE(ab8540_charge_input_curr_map),
-};
-
 int ab8500_bm_of_probe(struct device *dev,
 		       struct device_node *np,
 		       struct abx500_bm_data *bm)
diff --git a/drivers/power/supply/ab8500_btemp.c b/drivers/power/supply/ab8500_btemp.c
index f7a35ebfbab2..708fd58cd62b 100644
--- a/drivers/power/supply/ab8500_btemp.c
+++ b/drivers/power/supply/ab8500_btemp.c
@@ -214,22 +214,10 @@ static int ab8500_btemp_curr_source_enable(struct ab8500_btemp *di,
 	/* Only do this for batteries with internal NTC */
 	if (di->bm->adc_therm == ABx500_ADC_THERM_BATCTRL && enable) {
 
-		if (is_ab8540(di->parent)) {
-			if (di->curr_source == BTEMP_BATCTRL_CURR_SRC_60UA)
-				curr = BAT_CTRL_60U_ENA;
-			else
-				curr = BAT_CTRL_120U_ENA;
-		} else if (is_ab9540(di->parent) || is_ab8505(di->parent)) {
-			if (di->curr_source == BTEMP_BATCTRL_CURR_SRC_16UA)
-				curr = BAT_CTRL_16U_ENA;
-			else
-				curr = BAT_CTRL_18U_ENA;
-		} else {
-			if (di->curr_source == BTEMP_BATCTRL_CURR_SRC_7UA)
-				curr = BAT_CTRL_7U_ENA;
-			else
-				curr = BAT_CTRL_20U_ENA;
-		}
+		if (di->curr_source == BTEMP_BATCTRL_CURR_SRC_7UA)
+			curr = BAT_CTRL_7U_ENA;
+		else
+			curr = BAT_CTRL_20U_ENA;
 
 		dev_dbg(di->dev, "Set BATCTRL %duA\n", di->curr_source);
 
@@ -260,28 +248,12 @@ static int ab8500_btemp_curr_source_enable(struct ab8500_btemp *di,
 	} else if (di->bm->adc_therm == ABx500_ADC_THERM_BATCTRL && !enable) {
 		dev_dbg(di->dev, "Disable BATCTRL curr source\n");
 
-		if (is_ab8540(di->parent)) {
-			/* Write 0 to the curr bits */
-			ret = abx500_mask_and_set_register_interruptible(
-				di->dev,
-				AB8500_CHARGER, AB8500_BAT_CTRL_CURRENT_SOURCE,
-				BAT_CTRL_60U_ENA | BAT_CTRL_120U_ENA,
-				~(BAT_CTRL_60U_ENA | BAT_CTRL_120U_ENA));
-		} else if (is_ab9540(di->parent) || is_ab8505(di->parent)) {
-			/* Write 0 to the curr bits */
-			ret = abx500_mask_and_set_register_interruptible(
-				di->dev,
-				AB8500_CHARGER, AB8500_BAT_CTRL_CURRENT_SOURCE,
-				BAT_CTRL_16U_ENA | BAT_CTRL_18U_ENA,
-				~(BAT_CTRL_16U_ENA | BAT_CTRL_18U_ENA));
-		} else {
-			/* Write 0 to the curr bits */
-			ret = abx500_mask_and_set_register_interruptible(
-				di->dev,
-				AB8500_CHARGER, AB8500_BAT_CTRL_CURRENT_SOURCE,
-				BAT_CTRL_7U_ENA | BAT_CTRL_20U_ENA,
-				~(BAT_CTRL_7U_ENA | BAT_CTRL_20U_ENA));
-		}
+		/* Write 0 to the curr bits */
+		ret = abx500_mask_and_set_register_interruptible(
+			di->dev,
+			AB8500_CHARGER, AB8500_BAT_CTRL_CURRENT_SOURCE,
+			BAT_CTRL_7U_ENA | BAT_CTRL_20U_ENA,
+			~(BAT_CTRL_7U_ENA | BAT_CTRL_20U_ENA));
 
 		if (ret) {
 			dev_err(di->dev, "%s failed disabling current source\n",
@@ -324,25 +296,11 @@ static int ab8500_btemp_curr_source_enable(struct ab8500_btemp *di,
 	 * if we got an error above
 	 */
 disable_curr_source:
-	if (is_ab8540(di->parent)) {
-		/* Write 0 to the curr bits */
-		ret = abx500_mask_and_set_register_interruptible(di->dev,
-			AB8500_CHARGER, AB8500_BAT_CTRL_CURRENT_SOURCE,
-			BAT_CTRL_60U_ENA | BAT_CTRL_120U_ENA,
-			~(BAT_CTRL_60U_ENA | BAT_CTRL_120U_ENA));
-	} else if (is_ab9540(di->parent) || is_ab8505(di->parent)) {
-		/* Write 0 to the curr bits */
-		ret = abx500_mask_and_set_register_interruptible(di->dev,
-			AB8500_CHARGER, AB8500_BAT_CTRL_CURRENT_SOURCE,
-			BAT_CTRL_16U_ENA | BAT_CTRL_18U_ENA,
-			~(BAT_CTRL_16U_ENA | BAT_CTRL_18U_ENA));
-	} else {
-		/* Write 0 to the curr bits */
-		ret = abx500_mask_and_set_register_interruptible(di->dev,
-			AB8500_CHARGER, AB8500_BAT_CTRL_CURRENT_SOURCE,
-			BAT_CTRL_7U_ENA | BAT_CTRL_20U_ENA,
-			~(BAT_CTRL_7U_ENA | BAT_CTRL_20U_ENA));
-	}
+	/* Write 0 to the curr bits */
+	ret = abx500_mask_and_set_register_interruptible(di->dev,
+		AB8500_CHARGER, AB8500_BAT_CTRL_CURRENT_SOURCE,
+		BAT_CTRL_7U_ENA | BAT_CTRL_20U_ENA,
+		~(BAT_CTRL_7U_ENA | BAT_CTRL_20U_ENA));
 
 	if (ret) {
 		dev_err(di->dev, "%s failed disabling current source\n",
@@ -556,13 +514,8 @@ static int ab8500_btemp_id(struct ab8500_btemp *di)
 {
 	int res;
 	u8 i;
-	if (is_ab8540(di->parent))
-		di->curr_source = BTEMP_BATCTRL_CURR_SRC_60UA;
-	else if (is_ab9540(di->parent) || is_ab8505(di->parent))
-		di->curr_source = BTEMP_BATCTRL_CURR_SRC_16UA;
-	else
-		di->curr_source = BTEMP_BATCTRL_CURR_SRC_7UA;
 
+	di->curr_source = BTEMP_BATCTRL_CURR_SRC_7UA;
 	di->bm->batt_id = BATTERY_UNKNOWN;
 
 	res =  ab8500_btemp_get_batctrl_res(di);
@@ -600,18 +553,8 @@ static int ab8500_btemp_id(struct ab8500_btemp *di)
 	 */
 	if (di->bm->adc_therm == ABx500_ADC_THERM_BATCTRL &&
 	    di->bm->batt_id == 1) {
-		if (is_ab8540(di->parent)) {
-			dev_dbg(di->dev,
-				"Set BATCTRL current source to 60uA\n");
-			di->curr_source = BTEMP_BATCTRL_CURR_SRC_60UA;
-		} else if (is_ab9540(di->parent) || is_ab8505(di->parent)) {
-			dev_dbg(di->dev,
-				"Set BATCTRL current source to 16uA\n");
-			di->curr_source = BTEMP_BATCTRL_CURR_SRC_16UA;
-		} else {
-			dev_dbg(di->dev, "Set BATCTRL current source to 20uA\n");
-			di->curr_source = BTEMP_BATCTRL_CURR_SRC_20UA;
-		}
+		dev_dbg(di->dev, "Set BATCTRL current source to 20uA\n");
+		di->curr_source = BTEMP_BATCTRL_CURR_SRC_20UA;
 	}
 
 	return di->bm->batt_id;
diff --git a/drivers/power/supply/ab8500_charger.c b/drivers/power/supply/ab8500_charger.c
index 5a76c6d343de..ba0e97284ae5 100644
--- a/drivers/power/supply/ab8500_charger.c
+++ b/drivers/power/supply/ab8500_charger.c
@@ -58,9 +58,7 @@
 
 #define MAIN_CH_INPUT_CURR_SHIFT	4
 #define VBUS_IN_CURR_LIM_SHIFT		4
-#define AB8540_VBUS_IN_CURR_LIM_SHIFT	2
 #define AUTO_VBUS_IN_CURR_LIM_SHIFT	4
-#define AB8540_AUTO_VBUS_IN_CURR_MASK	0x3F
 #define VBUS_IN_CURR_LIM_RETRY_SET_TIME	30 /* seconds */
 
 #define LED_INDICATOR_PWM_ENA		0x01
@@ -1138,10 +1136,7 @@ static int ab8500_charger_set_current(struct ab8500_charger *di,
 			no_stepping = true;
 		break;
 	case AB8500_USBCH_IPT_CRNTLVL_REG:
-		if (is_ab8540(di->parent))
-			shift_value = AB8540_VBUS_IN_CURR_LIM_SHIFT;
-		else
-			shift_value = VBUS_IN_CURR_LIM_SHIFT;
+		shift_value = VBUS_IN_CURR_LIM_SHIFT;
 		prev_curr_index = (reg_value >> shift_value);
 		curr_index = ab8500_vbus_in_curr_to_regval(di, ich);
 		step_udelay = STEP_UDELAY * 100;
@@ -1865,67 +1860,6 @@ static int ab8500_charger_update_charger_current(struct ux500_charger *charger,
 	return ret;
 }
 
-/**
- * ab8540_charger_power_path_enable() - enable usb power path mode
- * @charger:	pointer to the ux500_charger structure
- * @enable:	enable/disable flag
- *
- * Enable or disable the power path for usb mode
- * Returns error code in case of failure else 0(on success)
- */
-static int ab8540_charger_power_path_enable(struct ux500_charger *charger,
-		bool enable)
-{
-	int ret;
-	struct ab8500_charger *di;
-
-	if (charger->psy->desc->type == POWER_SUPPLY_TYPE_USB)
-		di = to_ab8500_charger_usb_device_info(charger);
-	else
-		return -ENXIO;
-
-	ret = abx500_mask_and_set_register_interruptible(di->dev,
-				AB8500_CHARGER, AB8540_USB_PP_MODE_REG,
-				BUS_POWER_PATH_MODE_ENA, enable);
-	if (ret) {
-		dev_err(di->dev, "%s write failed\n", __func__);
-		return ret;
-	}
-
-	return ret;
-}
-
-
-/**
- * ab8540_charger_usb_pre_chg_enable() - enable usb pre change
- * @charger:	pointer to the ux500_charger structure
- * @enable:	enable/disable flag
- *
- * Enable or disable the pre-chage for usb mode
- * Returns error code in case of failure else 0(on success)
- */
-static int ab8540_charger_usb_pre_chg_enable(struct ux500_charger *charger,
-		bool enable)
-{
-	int ret;
-	struct ab8500_charger *di;
-
-	if (charger->psy->desc->type == POWER_SUPPLY_TYPE_USB)
-		di = to_ab8500_charger_usb_device_info(charger);
-	else
-		return -ENXIO;
-
-	ret = abx500_mask_and_set_register_interruptible(di->dev,
-				AB8500_CHARGER, AB8540_USB_PP_CHR_REG,
-				BUS_POWER_PATH_PRECHG_ENA, enable);
-	if (ret) {
-		dev_err(di->dev, "%s write failed\n", __func__);
-		return ret;
-	}
-
-	return ret;
-}
-
 static int ab8500_charger_get_ext_psy_data(struct device *dev, void *data)
 {
 	struct power_supply *psy;
@@ -2704,23 +2638,15 @@ static void ab8500_charger_vbus_drop_end_work(struct work_struct *work)
 	abx500_set_register_interruptible(di->dev,
 				  AB8500_CHARGER, AB8500_CHARGER_CTRL, 0x01);
 
-	if (is_ab8540(di->parent))
-		ret = abx500_get_register_interruptible(di->dev, AB8500_CHARGER,
-				AB8540_CH_USBCH_STAT3_REG, &reg_value);
-	else
-		ret = abx500_get_register_interruptible(di->dev, AB8500_CHARGER,
-				AB8500_CH_USBCH_STAT2_REG, &reg_value);
+	ret = abx500_get_register_interruptible(di->dev, AB8500_CHARGER,
+			AB8500_CH_USBCH_STAT2_REG, &reg_value);
 	if (ret < 0) {
 		dev_err(di->dev, "%s read failed\n", __func__);
 		return;
 	}
 
-	if (is_ab8540(di->parent))
-		curr = di->bm->chg_input_curr[
-			reg_value & AB8540_AUTO_VBUS_IN_CURR_MASK];
-	else
-		curr = di->bm->chg_input_curr[
-			reg_value >> AUTO_VBUS_IN_CURR_LIM_SHIFT];
+	curr = di->bm->chg_input_curr[
+		reg_value >> AUTO_VBUS_IN_CURR_LIM_SHIFT];
 
 	if (di->max_usb_in_curr.calculated_max != curr) {
 		/* USB source is collapsing */
@@ -3097,14 +3023,9 @@ static int ab8500_charger_init_hw_registers(struct ab8500_charger *di)
 			goto out;
 		}
 
-		if (is_ab8540(di->parent))
-			ret = abx500_set_register_interruptible(di->dev,
-				AB8500_CHARGER, AB8500_CH_OPT_CRNTLVL_MAX_REG,
-				CH_OP_CUR_LVL_2P);
-		else
-			ret = abx500_set_register_interruptible(di->dev,
-				AB8500_CHARGER, AB8500_CH_OPT_CRNTLVL_MAX_REG,
-				CH_OP_CUR_LVL_1P6);
+		ret = abx500_set_register_interruptible(di->dev,
+			AB8500_CHARGER, AB8500_CH_OPT_CRNTLVL_MAX_REG,
+			CH_OP_CUR_LVL_1P6);
 		if (ret) {
 			dev_err(di->dev,
 				"failed to set CH_OPT_CRNTLVL_MAX_REG\n");
@@ -3112,8 +3033,7 @@ static int ab8500_charger_init_hw_registers(struct ab8500_charger *di)
 		}
 	}
 
-	if (is_ab9540_2p0(di->parent) || is_ab9540_3p0(di->parent)
-	 || is_ab8505_2p0(di->parent) || is_ab8540(di->parent))
+	if (is_ab8505_2p0(di->parent))
 		ret = abx500_mask_and_set_register_interruptible(di->dev,
 			AB8500_CHARGER,
 			AB8500_USBCH_CTRL2_REG,
@@ -3205,17 +3125,6 @@ static int ab8500_charger_init_hw_registers(struct ab8500_charger *di)
 		dev_err(di->dev, "failed to setup backup battery charging\n");
 		goto out;
 	}
-	if (is_ab8540(di->parent)) {
-		ret = abx500_set_register_interruptible(di->dev,
-			AB8500_RTC,
-			AB8500_RTC_CTRL1_REG,
-			bup_vch_range | vbup33_vrtcn);
-		if (ret) {
-			dev_err(di->dev,
-				"failed to setup backup battery charging\n");
-			goto out;
-		}
-	}
 
 	/* Enable backup battery charging */
 	ret = abx500_mask_and_set_register_interruptible(di->dev,
@@ -3226,25 +3135,6 @@ static int ab8500_charger_init_hw_registers(struct ab8500_charger *di)
 		goto out;
 	}
 
-	if (is_ab8540(di->parent)) {
-		ret = abx500_mask_and_set_register_interruptible(di->dev,
-			AB8500_CHARGER, AB8540_USB_PP_MODE_REG,
-			BUS_VSYS_VOL_SELECT_MASK, BUS_VSYS_VOL_SELECT_3P6V);
-		if (ret) {
-			dev_err(di->dev,
-				"failed to setup usb power path vsys voltage\n");
-			goto out;
-		}
-		ret = abx500_mask_and_set_register_interruptible(di->dev,
-			AB8500_CHARGER, AB8540_USB_PP_CHR_REG,
-			BUS_PP_PRECHG_CURRENT_MASK, 0);
-		if (ret) {
-			dev_err(di->dev,
-				"failed to setup usb power path precharge current\n");
-			goto out;
-		}
-	}
-
 out:
 	return ret;
 }
@@ -3529,8 +3419,6 @@ static int ab8500_charger_probe(struct platform_device *pdev)
 	di->usb_chg.ops.check_enable = &ab8500_charger_usb_check_enable;
 	di->usb_chg.ops.kick_wd = &ab8500_charger_watchdog_kick;
 	di->usb_chg.ops.update_curr = &ab8500_charger_update_charger_current;
-	di->usb_chg.ops.pp_enable = &ab8540_charger_power_path_enable;
-	di->usb_chg.ops.pre_chg_enable = &ab8540_charger_usb_pre_chg_enable;
 	di->usb_chg.max_out_volt = ab8500_charger_voltage_map[
 		ARRAY_SIZE(ab8500_charger_voltage_map) - 1];
 	di->usb_chg.max_out_curr =
@@ -3538,7 +3426,6 @@ static int ab8500_charger_probe(struct platform_device *pdev)
 	di->usb_chg.wdt_refresh = CHG_WD_INTERVAL;
 	di->usb_chg.enabled = di->bm->usb_enabled;
 	di->usb_chg.external = false;
-	di->usb_chg.power_path = di->bm->usb_power_path;
 	di->usb_state.usb_current = -1;
 
 	/* Create a work queue for the charger */
diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c
index c569f82a0071..d9c6c7bedd85 100644
--- a/drivers/power/supply/ab8500_fg.c
+++ b/drivers/power/supply/ab8500_fg.c
@@ -1408,7 +1408,7 @@ static void ab8500_fg_charge_state_to(struct ab8500_fg *di,
 static void ab8500_fg_discharge_state_to(struct ab8500_fg *di,
 	enum ab8500_fg_discharge_state new_state)
 {
-	dev_dbg(di->dev, "Disharge state from %d [%s] to %d [%s]\n",
+	dev_dbg(di->dev, "Discharge state from %d [%s] to %d [%s]\n",
 		di->discharge_state,
 		discharge_state[di->discharge_state],
 		new_state,
@@ -2326,9 +2326,7 @@ static int ab8500_fg_init_hw_registers(struct ab8500_fg *di)
 		goto out;
 	}
 
-	if (((is_ab8505(di->parent) || is_ab9540(di->parent)) &&
-			abx500_get_chip_id(di->dev) >= AB8500_CUT2P0)
-			|| is_ab8540(di->parent)) {
+	if (is_ab8505(di->parent)) {
 		ret = abx500_set_register_interruptible(di->dev, AB8500_RTC,
 			AB8505_RTC_PCUT_MAX_TIME_REG, di->bm->fg_params->pcut_max_time);
 
@@ -2915,9 +2913,7 @@ static int ab8500_fg_sysfs_psy_create_attrs(struct ab8500_fg *di)
 {
 	unsigned int i;
 
-	if (((is_ab8505(di->parent) || is_ab9540(di->parent)) &&
-	     abx500_get_chip_id(di->dev) >= AB8500_CUT2P0)
-	    || is_ab8540(di->parent)) {
+	if (is_ab8505(di->parent)) {
 		for (i = 0; i < ARRAY_SIZE(ab8505_fg_sysfs_psy_attrs); i++)
 			if (device_create_file(&di->fg_psy->dev,
 					       &ab8505_fg_sysfs_psy_attrs[i]))
@@ -2937,9 +2933,7 @@ static void ab8500_fg_sysfs_psy_remove_attrs(struct ab8500_fg *di)
 {
 	unsigned int i;
 
-	if (((is_ab8505(di->parent) || is_ab9540(di->parent)) &&
-	     abx500_get_chip_id(di->dev) >= AB8500_CUT2P0)
-	    || is_ab8540(di->parent)) {
+	if (is_ab8505(di->parent)) {
 		for (i = 0; i < ARRAY_SIZE(ab8505_fg_sysfs_psy_attrs); i++)
 			(void)device_remove_file(&di->fg_psy->dev,
 						 &ab8505_fg_sysfs_psy_attrs[i]);
diff --git a/drivers/power/supply/abx500_chargalg.c b/drivers/power/supply/abx500_chargalg.c
index a4411d6bbc96..947709cdd14e 100644
--- a/drivers/power/supply/abx500_chargalg.c
+++ b/drivers/power/supply/abx500_chargalg.c
@@ -44,9 +44,6 @@
 /* Five minutes expressed in seconds */
 #define FIVE_MINUTES_IN_SECONDS        300
 
-/* Plus margin for the low battery threshold */
-#define BAT_PLUS_MARGIN                (100)
-
 #define CHARGALG_CURR_STEP_LOW		0
 #define CHARGALG_CURR_STEP_HIGH	100
 
@@ -101,7 +98,6 @@ enum abx500_chargalg_states {
 	STATE_HW_TEMP_PROTECT_INIT,
 	STATE_HW_TEMP_PROTECT,
 	STATE_NORMAL_INIT,
-	STATE_USB_PP_PRE_CHARGE,
 	STATE_NORMAL,
 	STATE_WAIT_FOR_RECHARGE_INIT,
 	STATE_WAIT_FOR_RECHARGE,
@@ -133,7 +129,6 @@ static const char *states[] = {
 	"HW_TEMP_PROTECT_INIT",
 	"HW_TEMP_PROTECT",
 	"NORMAL_INIT",
-	"USB_PP_PRE_CHARGE",
 	"NORMAL",
 	"WAIT_FOR_RECHARGE_INIT",
 	"WAIT_FOR_RECHARGE",
@@ -603,37 +598,6 @@ static int abx500_chargalg_usb_en(struct abx500_chargalg *di, int enable,
 	return di->usb_chg->ops.enable(di->usb_chg, enable, vset, iset);
 }
 
- /**
- * ab8540_chargalg_usb_pp_en() - Enable/ disable USB power path
- * @di:                pointer to the abx500_chargalg structure
- * @enable:    power path enable/disable
- *
- * The USB power path will be enable/ disable
- */
-static int ab8540_chargalg_usb_pp_en(struct abx500_chargalg *di, bool enable)
-{
-	if (!di->usb_chg || !di->usb_chg->ops.pp_enable)
-		return -ENXIO;
-
-	return di->usb_chg->ops.pp_enable(di->usb_chg, enable);
-}
-
-/**
- * ab8540_chargalg_usb_pre_chg_en() - Enable/ disable USB pre-charge
- * @di:                pointer to the abx500_chargalg structure
- * @enable:    USB pre-charge enable/disable
- *
- * The USB USB pre-charge will be enable/ disable
- */
-static int ab8540_chargalg_usb_pre_chg_en(struct abx500_chargalg *di,
-					  bool enable)
-{
-	if (!di->usb_chg || !di->usb_chg->ops.pre_chg_enable)
-		return -ENXIO;
-
-	return di->usb_chg->ops.pre_chg_enable(di->usb_chg, enable);
-}
-
 /**
  * abx500_chargalg_update_chg_curr() - Update charger current
  * @di:		pointer to the abx500_chargalg structure
@@ -833,9 +797,6 @@ static void abx500_chargalg_end_of_charge(struct abx500_chargalg *di)
 		di->batt_data.avg_curr > 0) {
 		if (++di->eoc_cnt >= EOC_COND_CNT) {
 			di->eoc_cnt = 0;
-			if ((di->chg_info.charger_type & USB_CHG) &&
-			   (di->usb_chg->power_path))
-				ab8540_chargalg_usb_pp_en(di, true);
 			di->charge_status = POWER_SUPPLY_STATUS_FULL;
 			di->maintenance_chg = true;
 			dev_dbg(di->dev, "EOC reached!\n");
@@ -1536,22 +1497,6 @@ static void abx500_chargalg_algorithm(struct abx500_chargalg *di)
 		break;
 
 	case STATE_NORMAL_INIT:
-		if ((di->chg_info.charger_type & USB_CHG) &&
-				di->usb_chg->power_path) {
-			if (di->batt_data.volt >
-			    (di->bm->fg_params->lowbat_threshold +
-			     BAT_PLUS_MARGIN)) {
-				ab8540_chargalg_usb_pre_chg_en(di, false);
-				ab8540_chargalg_usb_pp_en(di, false);
-			} else {
-				ab8540_chargalg_usb_pp_en(di, true);
-				ab8540_chargalg_usb_pre_chg_en(di, true);
-				abx500_chargalg_state_to(di,
-					STATE_USB_PP_PRE_CHARGE);
-				break;
-			}
-		}
-
 		if (di->curr_status.curr_step == CHARGALG_CURR_STEP_LOW)
 			abx500_chargalg_stop_charging(di);
 		else {
@@ -1575,13 +1520,6 @@ static void abx500_chargalg_algorithm(struct abx500_chargalg *di)
 
 		break;
 
-	case STATE_USB_PP_PRE_CHARGE:
-		if (di->batt_data.volt >
-			(di->bm->fg_params->lowbat_threshold +
-			BAT_PLUS_MARGIN))
-			abx500_chargalg_state_to(di, STATE_NORMAL_INIT);
-		break;
-
 	case STATE_NORMAL:
 		handle_maxim_chg_curr(di);
 		if (di->charge_status == POWER_SUPPLY_STATUS_FULL &&
diff --git a/drivers/power/supply/axp288_charger.c b/drivers/power/supply/axp288_charger.c
index 9bfbde15b07d..6e1bc14c3304 100644
--- a/drivers/power/supply/axp288_charger.c
+++ b/drivers/power/supply/axp288_charger.c
@@ -88,6 +88,8 @@
 #define CHRG_VBUS_ILIM_2000MA		0x4	/* 2000mA */
 #define CHRG_VBUS_ILIM_2500MA		0x5	/* 2500mA */
 #define CHRG_VBUS_ILIM_3000MA		0x6	/* 3000mA */
+#define CHRG_VBUS_ILIM_3500MA		0x7	/* 3500mA */
+#define CHRG_VBUS_ILIM_4000MA		0x8	/* 4000mA */
 
 #define CHRG_VLTFC_0C			0xA5	/* 0 DegC */
 #define CHRG_VHTFC_45C			0x1F	/* 45 DegC */
@@ -223,9 +225,11 @@ static int axp288_charger_get_vbus_inlmt(struct axp288_chrg_info *info)
 		return 2500000;
 	case CHRG_VBUS_ILIM_3000MA:
 		return 3000000;
+	case CHRG_VBUS_ILIM_3500MA:
+		return 3500000;
 	default:
-		dev_warn(&info->pdev->dev, "Unknown ilim reg val: %d\n", val);
-		return 0;
+		/* All b1xxx values map to 4000 mA */
+		return 4000000;
 	}
 }
 
@@ -235,7 +239,11 @@ static inline int axp288_charger_set_vbus_inlmt(struct axp288_chrg_info *info,
 	int ret;
 	u8 reg_val;
 
-	if (inlmt >= 3000000)
+	if (inlmt >= 4000000)
+		reg_val = CHRG_VBUS_ILIM_4000MA << CHRG_VBUS_ILIM_BIT_POS;
+	else if (inlmt >= 3500000)
+		reg_val = CHRG_VBUS_ILIM_3500MA << CHRG_VBUS_ILIM_BIT_POS;
+	else if (inlmt >= 3000000)
 		reg_val = CHRG_VBUS_ILIM_3000MA << CHRG_VBUS_ILIM_BIT_POS;
 	else if (inlmt >= 2500000)
 		reg_val = CHRG_VBUS_ILIM_2500MA << CHRG_VBUS_ILIM_BIT_POS;
@@ -739,6 +747,18 @@ static int axp288_charger_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct axp20x_dev *axp20x = dev_get_drvdata(pdev->dev.parent);
 	struct power_supply_config charger_cfg = {};
+	unsigned int val;
+
+	/*
+	 * On some devices the fuelgauge and charger parts of the axp288 are
+	 * not used, check that the fuelgauge is enabled (CC_CTRL != 0).
+	 */
+	ret = regmap_read(axp20x->regmap, AXP20X_CC_CTRL, &val);
+	if (ret < 0)
+		return ret;
+	if (val == 0)
+		return -ENODEV;
+
 	info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL);
 	if (!info)
 		return -ENOMEM;
diff --git a/drivers/power/supply/axp288_fuel_gauge.c b/drivers/power/supply/axp288_fuel_gauge.c
index fd8f0b2210bc..084c8ba9749d 100644
--- a/drivers/power/supply/axp288_fuel_gauge.c
+++ b/drivers/power/supply/axp288_fuel_gauge.c
@@ -24,7 +24,6 @@
 #include <linux/regmap.h>
 #include <linux/jiffies.h>
 #include <linux/interrupt.h>
-#include <linux/workqueue.h>
 #include <linux/mfd/axp20x.h>
 #include <linux/platform_device.h>
 #include <linux/power_supply.h>
@@ -88,7 +87,6 @@
 #define FG_LOW_CAP_CRIT_THR			4   /* 4 perc */
 #define FG_LOW_CAP_SHDN_THR			0   /* 0 perc */
 
-#define STATUS_MON_DELAY_JIFFIES    (HZ * 60)   /*60 sec */
 #define NR_RETRY_CNT    3
 #define DEV_NAME	"axp288_fuel_gauge"
 
@@ -128,7 +126,6 @@ struct axp288_fg_info {
 	struct mutex lock;
 	int status;
 	int max_volt;
-	struct delayed_work status_monitor;
 	struct dentry *debug_file;
 };
 
@@ -592,16 +589,6 @@ static int fuel_gauge_property_is_writeable(struct power_supply *psy,
 	return ret;
 }
 
-static void fuel_gauge_status_monitor(struct work_struct *work)
-{
-	struct axp288_fg_info *info = container_of(work,
-		struct axp288_fg_info, status_monitor.work);
-
-	fuel_gauge_get_status(info);
-	power_supply_changed(info->bat);
-	schedule_delayed_work(&info->status_monitor, STATUS_MON_DELAY_JIFFIES);
-}
-
 static irqreturn_t fuel_gauge_thread_handler(int irq, void *dev)
 {
 	struct axp288_fg_info *info = dev;
@@ -754,10 +741,21 @@ static int axp288_fuel_gauge_probe(struct platform_device *pdev)
 		[BAT_D_CURR] = "axp288-chrg-d-curr",
 		[BAT_VOLT] = "axp288-batt-volt",
 	};
+	unsigned int val;
 
 	if (dmi_check_system(axp288_fuel_gauge_blacklist))
 		return -ENODEV;
 
+	/*
+	 * On some devices the fuelgauge and charger parts of the axp288 are
+	 * not used, check that the fuelgauge is enabled (CC_CTRL != 0).
+	 */
+	ret = regmap_read(axp20x->regmap, AXP20X_CC_CTRL, &val);
+	if (ret < 0)
+		return ret;
+	if (val == 0)
+		return -ENODEV;
+
 	info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL);
 	if (!info)
 		return -ENOMEM;
@@ -770,7 +768,6 @@ static int axp288_fuel_gauge_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, info);
 
 	mutex_init(&info->lock);
-	INIT_DELAYED_WORK(&info->status_monitor, fuel_gauge_status_monitor);
 
 	for (i = 0; i < IIO_CHANNEL_NUM; i++) {
 		/*
@@ -830,7 +827,6 @@ static int axp288_fuel_gauge_probe(struct platform_device *pdev)
 
 	fuel_gauge_create_debugfs(info);
 	fuel_gauge_init_irq(info);
-	schedule_delayed_work(&info->status_monitor, STATUS_MON_DELAY_JIFFIES);
 
 	return 0;
 
@@ -853,7 +849,6 @@ static int axp288_fuel_gauge_remove(struct platform_device *pdev)
 	struct axp288_fg_info *info = platform_get_drvdata(pdev);
 	int i;
 
-	cancel_delayed_work_sync(&info->status_monitor);
 	power_supply_unregister(info->bat);
 	fuel_gauge_remove_debugfs(info);
 
diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c
index 7ce60519b1bc..d44ed8e17c47 100644
--- a/drivers/power/supply/bq27xxx_battery.c
+++ b/drivers/power/supply/bq27xxx_battery.c
@@ -432,6 +432,7 @@ static u8
 		BQ27XXX_DM_REG_ROWS,
 	};
 #define bq27425_regs bq27421_regs
+#define bq27426_regs bq27421_regs
 #define bq27441_regs bq27421_regs
 #define bq27621_regs bq27421_regs
 
@@ -664,6 +665,7 @@ static enum power_supply_property bq27421_props[] = {
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
 #define bq27425_props bq27421_props
+#define bq27426_props bq27421_props
 #define bq27441_props bq27421_props
 #define bq27621_props bq27421_props
 
@@ -734,6 +736,12 @@ static struct bq27xxx_dm_reg bq27425_dm_regs[] = {
 	[BQ27XXX_DM_TERMINATE_VOLTAGE] = { 82, 18, 2, 2800,  3700 },
 };
 
+static struct bq27xxx_dm_reg bq27426_dm_regs[] = {
+	[BQ27XXX_DM_DESIGN_CAPACITY]   = { 82,  6, 2,    0,  8000 },
+	[BQ27XXX_DM_DESIGN_ENERGY]     = { 82,  8, 2,    0, 32767 },
+	[BQ27XXX_DM_TERMINATE_VOLTAGE] = { 82, 10, 2, 2500,  3700 },
+};
+
 #if 0 /* not yet tested */
 #define bq27441_dm_regs bq27421_dm_regs
 #else
@@ -795,6 +803,7 @@ static struct {
 	[BQ27545]   = BQ27XXX_DATA(bq27545,   0x04143672, BQ27XXX_O_OTDC),
 	[BQ27421]   = BQ27XXX_DATA(bq27421,   0x80008000, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP | BQ27XXX_O_RAM),
 	[BQ27425]   = BQ27XXX_DATA(bq27425,   0x04143672, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP),
+	[BQ27426]   = BQ27XXX_DATA(bq27426,   0x80008000, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP | BQ27XXX_O_RAM),
 	[BQ27441]   = BQ27XXX_DATA(bq27441,   0x80008000, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP | BQ27XXX_O_RAM),
 	[BQ27621]   = BQ27XXX_DATA(bq27621,   0x80008000, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP | BQ27XXX_O_RAM),
 };
diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c
index 6b25e5f2337e..40069128ad44 100644
--- a/drivers/power/supply/bq27xxx_battery_i2c.c
+++ b/drivers/power/supply/bq27xxx_battery_i2c.c
@@ -249,6 +249,7 @@ static const struct i2c_device_id bq27xxx_i2c_id_table[] = {
 	{ "bq27545", BQ27545 },
 	{ "bq27421", BQ27421 },
 	{ "bq27425", BQ27425 },
+	{ "bq27426", BQ27426 },
 	{ "bq27441", BQ27441 },
 	{ "bq27621", BQ27621 },
 	{},
@@ -280,6 +281,7 @@ static const struct of_device_id bq27xxx_battery_i2c_of_match_table[] = {
 	{ .compatible = "ti,bq27545" },
 	{ .compatible = "ti,bq27421" },
 	{ .compatible = "ti,bq27425" },
+	{ .compatible = "ti,bq27426" },
 	{ .compatible = "ti,bq27441" },
 	{ .compatible = "ti,bq27621" },
 	{},
diff --git a/drivers/power/supply/charger-manager.c b/drivers/power/supply/charger-manager.c
index 1de4b4493824..2a50b4654793 100644
--- a/drivers/power/supply/charger-manager.c
+++ b/drivers/power/supply/charger-manager.c
@@ -1700,8 +1700,9 @@ static int charger_manager_probe(struct platform_device *pdev)
 		power_supply_put(psy);
 	}
 
-	if (desc->polling_interval_ms == 0 ||
-	    msecs_to_jiffies(desc->polling_interval_ms) <= CM_JIFFIES_SMALL) {
+	if (cm->desc->polling_mode != CM_POLL_DISABLE &&
+	    (desc->polling_interval_ms == 0 ||
+	     msecs_to_jiffies(desc->polling_interval_ms) <= CM_JIFFIES_SMALL)) {
 		dev_err(&pdev->dev, "polling_interval_ms is too small\n");
 		return -EINVAL;
 	}
diff --git a/drivers/power/supply/gpio-charger.c b/drivers/power/supply/gpio-charger.c
index bd2468ca6b63..c3f2a9479468 100644
--- a/drivers/power/supply/gpio-charger.c
+++ b/drivers/power/supply/gpio-charger.c
@@ -212,8 +212,7 @@ static int gpio_charger_suspend(struct device *dev)
 
 static int gpio_charger_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct gpio_charger *gpio_charger = platform_get_drvdata(pdev);
+	struct gpio_charger *gpio_charger = dev_get_drvdata(dev);
 
 	if (device_may_wakeup(dev) && gpio_charger->wakeup_enabled)
 		disable_irq_wake(gpio_charger->irq);
diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c
index feac7b066e6c..f57ab0a27301 100644
--- a/drivers/power/supply/power_supply_core.c
+++ b/drivers/power/supply/power_supply_core.c
@@ -19,6 +19,7 @@
 #include <linux/err.h>
 #include <linux/of.h>
 #include <linux/power_supply.h>
+#include <linux/property.h>
 #include <linux/thermal.h>
 #include "power_supply.h"
 
@@ -843,12 +844,21 @@ __power_supply_register(struct device *parent,
 {
 	struct device *dev;
 	struct power_supply *psy;
-	int rc;
+	int i, rc;
 
 	if (!parent)
 		pr_warn("%s: Expected proper parent device for '%s'\n",
 			__func__, desc->name);
 
+	if (!desc || !desc->name || !desc->properties || !desc->num_properties)
+		return ERR_PTR(-EINVAL);
+
+	for (i = 0; i < desc->num_properties; ++i) {
+		if ((desc->properties[i] == POWER_SUPPLY_PROP_USB_TYPE) &&
+		    (!desc->usb_types || !desc->num_usb_types))
+			return ERR_PTR(-EINVAL);
+	}
+
 	psy = kzalloc(sizeof(*psy), GFP_KERNEL);
 	if (!psy)
 		return ERR_PTR(-ENOMEM);
@@ -865,7 +875,8 @@ __power_supply_register(struct device *parent,
 	psy->desc = desc;
 	if (cfg) {
 		psy->drv_data = cfg->drv_data;
-		psy->of_node = cfg->of_node;
+		psy->of_node =
+			cfg->fwnode ? to_of_node(cfg->fwnode) : cfg->of_node;
 		psy->supplied_to = cfg->supplied_to;
 		psy->num_supplicants = cfg->num_supplicants;
 	}
diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c
index 5204f115970f..6170ed8b6854 100644
--- a/drivers/power/supply/power_supply_sysfs.c
+++ b/drivers/power/supply/power_supply_sysfs.c
@@ -46,6 +46,11 @@ static const char * const power_supply_type_text[] = {
 	"USB_PD", "USB_PD_DRP", "BrickID"
 };
 
+static const char * const power_supply_usb_type_text[] = {
+	"Unknown", "SDP", "DCP", "CDP", "ACA", "C",
+	"PD", "PD_DRP", "PD_PPS", "BrickID"
+};
+
 static const char * const power_supply_status_text[] = {
 	"Unknown", "Charging", "Discharging", "Not charging", "Full"
 };
@@ -73,18 +78,53 @@ static const char * const power_supply_scope_text[] = {
 	"Unknown", "System", "Device"
 };
 
+static ssize_t power_supply_show_usb_type(struct device *dev,
+					  enum power_supply_usb_type *usb_types,
+					  ssize_t num_usb_types,
+					  union power_supply_propval *value,
+					  char *buf)
+{
+	enum power_supply_usb_type usb_type;
+	ssize_t count = 0;
+	bool match = false;
+	int i;
+
+	for (i = 0; i < num_usb_types; ++i) {
+		usb_type = usb_types[i];
+
+		if (value->intval == usb_type) {
+			count += sprintf(buf + count, "[%s] ",
+					 power_supply_usb_type_text[usb_type]);
+			match = true;
+		} else {
+			count += sprintf(buf + count, "%s ",
+					 power_supply_usb_type_text[usb_type]);
+		}
+	}
+
+	if (!match) {
+		dev_warn(dev, "driver reporting unsupported connected type\n");
+		return -EINVAL;
+	}
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+
 static ssize_t power_supply_show_property(struct device *dev,
 					  struct device_attribute *attr,
 					  char *buf) {
-	ssize_t ret = 0;
+	ssize_t ret;
 	struct power_supply *psy = dev_get_drvdata(dev);
-	const ptrdiff_t off = attr - power_supply_attrs;
+	enum power_supply_property psp = attr - power_supply_attrs;
 	union power_supply_propval value;
 
-	if (off == POWER_SUPPLY_PROP_TYPE) {
+	if (psp == POWER_SUPPLY_PROP_TYPE) {
 		value.intval = psy->desc->type;
 	} else {
-		ret = power_supply_get_property(psy, off, &value);
+		ret = power_supply_get_property(psy, psp, &value);
 
 		if (ret < 0) {
 			if (ret == -ENODATA)
@@ -97,31 +137,48 @@ static ssize_t power_supply_show_property(struct device *dev,
 		}
 	}
 
-	if (off == POWER_SUPPLY_PROP_STATUS)
-		return sprintf(buf, "%s\n",
-			       power_supply_status_text[value.intval]);
-	else if (off == POWER_SUPPLY_PROP_CHARGE_TYPE)
-		return sprintf(buf, "%s\n",
-			       power_supply_charge_type_text[value.intval]);
-	else if (off == POWER_SUPPLY_PROP_HEALTH)
-		return sprintf(buf, "%s\n",
-			       power_supply_health_text[value.intval]);
-	else if (off == POWER_SUPPLY_PROP_TECHNOLOGY)
-		return sprintf(buf, "%s\n",
-			       power_supply_technology_text[value.intval]);
-	else if (off == POWER_SUPPLY_PROP_CAPACITY_LEVEL)
-		return sprintf(buf, "%s\n",
-			       power_supply_capacity_level_text[value.intval]);
-	else if (off == POWER_SUPPLY_PROP_TYPE)
-		return sprintf(buf, "%s\n",
-			       power_supply_type_text[value.intval]);
-	else if (off == POWER_SUPPLY_PROP_SCOPE)
-		return sprintf(buf, "%s\n",
-			       power_supply_scope_text[value.intval]);
-	else if (off >= POWER_SUPPLY_PROP_MODEL_NAME)
-		return sprintf(buf, "%s\n", value.strval);
-
-	return sprintf(buf, "%d\n", value.intval);
+	switch (psp) {
+	case POWER_SUPPLY_PROP_STATUS:
+		ret = sprintf(buf, "%s\n",
+			      power_supply_status_text[value.intval]);
+		break;
+	case POWER_SUPPLY_PROP_CHARGE_TYPE:
+		ret = sprintf(buf, "%s\n",
+			      power_supply_charge_type_text[value.intval]);
+		break;
+	case POWER_SUPPLY_PROP_HEALTH:
+		ret = sprintf(buf, "%s\n",
+			      power_supply_health_text[value.intval]);
+		break;
+	case POWER_SUPPLY_PROP_TECHNOLOGY:
+		ret = sprintf(buf, "%s\n",
+			      power_supply_technology_text[value.intval]);
+		break;
+	case POWER_SUPPLY_PROP_CAPACITY_LEVEL:
+		ret = sprintf(buf, "%s\n",
+			      power_supply_capacity_level_text[value.intval]);
+		break;
+	case POWER_SUPPLY_PROP_TYPE:
+		ret = sprintf(buf, "%s\n",
+			      power_supply_type_text[value.intval]);
+		break;
+	case POWER_SUPPLY_PROP_USB_TYPE:
+		ret = power_supply_show_usb_type(dev, psy->desc->usb_types,
+						 psy->desc->num_usb_types,
+						 &value, buf);
+		break;
+	case POWER_SUPPLY_PROP_SCOPE:
+		ret = sprintf(buf, "%s\n",
+			      power_supply_scope_text[value.intval]);
+		break;
+	case POWER_SUPPLY_PROP_MODEL_NAME ... POWER_SUPPLY_PROP_SERIAL_NUMBER:
+		ret = sprintf(buf, "%s\n", value.strval);
+		break;
+	default:
+		ret = sprintf(buf, "%d\n", value.intval);
+	}
+
+	return ret;
 }
 
 static ssize_t power_supply_store_property(struct device *dev,
@@ -129,11 +186,10 @@ static ssize_t power_supply_store_property(struct device *dev,
 					   const char *buf, size_t count) {
 	ssize_t ret;
 	struct power_supply *psy = dev_get_drvdata(dev);
-	const ptrdiff_t off = attr - power_supply_attrs;
+	enum power_supply_property psp = attr - power_supply_attrs;
 	union power_supply_propval value;
 
-	/* maybe it is a enum property? */
-	switch (off) {
+	switch (psp) {
 	case POWER_SUPPLY_PROP_STATUS:
 		ret = sysfs_match_string(power_supply_status_text, buf);
 		break;
@@ -172,7 +228,7 @@ static ssize_t power_supply_store_property(struct device *dev,
 
 	value.intval = ret;
 
-	ret = power_supply_set_property(psy, off, &value);
+	ret = power_supply_set_property(psy, psp, &value);
 	if (ret < 0)
 		return ret;
 
@@ -241,6 +297,7 @@ static struct device_attribute power_supply_attrs[] = {
 	POWER_SUPPLY_ATTR(time_to_full_now),
 	POWER_SUPPLY_ATTR(time_to_full_avg),
 	POWER_SUPPLY_ATTR(type),
+	POWER_SUPPLY_ATTR(usb_type),
 	POWER_SUPPLY_ATTR(scope),
 	POWER_SUPPLY_ATTR(precharge_current),
 	POWER_SUPPLY_ATTR(charge_term_current),
diff --git a/drivers/power/supply/s3c_adc_battery.c b/drivers/power/supply/s3c_adc_battery.c
index 0ffe5cd3abf6..3d00b35cafc9 100644
--- a/drivers/power/supply/s3c_adc_battery.c
+++ b/drivers/power/supply/s3c_adc_battery.c
@@ -293,6 +293,7 @@ static int s3c_adc_bat_probe(struct platform_device *pdev)
 {
 	struct s3c_adc_client	*client;
 	struct s3c_adc_bat_pdata *pdata = pdev->dev.platform_data;
+	struct power_supply_config psy_cfg = {};
 	int ret;
 
 	client = s3c_adc_register(pdev, NULL, NULL, 0);
@@ -309,14 +310,15 @@ static int s3c_adc_bat_probe(struct platform_device *pdev)
 	main_bat.cur_value = -1;
 	main_bat.cable_plugged = 0;
 	main_bat.status = POWER_SUPPLY_STATUS_DISCHARGING;
+	psy_cfg.drv_data = &main_bat;
 
-	main_bat.psy = power_supply_register(&pdev->dev, &main_bat_desc, NULL);
+	main_bat.psy = power_supply_register(&pdev->dev, &main_bat_desc, &psy_cfg);
 	if (IS_ERR(main_bat.psy)) {
 		ret = PTR_ERR(main_bat.psy);
 		goto err_reg_main;
 	}
 	if (pdata->backup_volt_mult) {
-		const struct power_supply_config psy_cfg
+		const struct power_supply_config backup_psy_cfg
 						= { .drv_data = &backup_bat, };
 
 		backup_bat.client = client;
@@ -324,7 +326,7 @@ static int s3c_adc_bat_probe(struct platform_device *pdev)
 		backup_bat.volt_value = -1;
 		backup_bat.psy = power_supply_register(&pdev->dev,
 						       &backup_bat_desc,
-						       &psy_cfg);
+						       &backup_psy_cfg);
 		if (IS_ERR(backup_bat.psy)) {
 			ret = PTR_ERR(backup_bat.psy);
 			goto err_reg_backup;
diff --git a/drivers/ptp/Kconfig b/drivers/ptp/Kconfig
index a21ad10d613c..474c988d2e95 100644
--- a/drivers/ptp/Kconfig
+++ b/drivers/ptp/Kconfig
@@ -41,19 +41,19 @@ config PTP_1588_CLOCK_DTE
 	  To compile this driver as a module, choose M here: the module
 	  will be called ptp_dte.
 
-config PTP_1588_CLOCK_GIANFAR
-	tristate "Freescale eTSEC as PTP clock"
+config PTP_1588_CLOCK_QORIQ
+	tristate "Freescale QorIQ 1588 timer as PTP clock"
 	depends on GIANFAR
 	depends on PTP_1588_CLOCK
 	default y
 	help
-	  This driver adds support for using the eTSEC as a PTP
-	  clock. This clock is only useful if your PTP programs are
-	  getting hardware time stamps on the PTP Ethernet packets
-	  using the SO_TIMESTAMPING API.
+	  This driver adds support for using the Freescale QorIQ 1588
+	  timer as a PTP clock. This clock is only useful if your PTP
+	  programs are getting hardware time stamps on the PTP Ethernet
+	  packets using the SO_TIMESTAMPING API.
 
 	  To compile this driver as a module, choose M here: the module
-	  will be called gianfar_ptp.
+	  will be called ptp_qoriq.
 
 config PTP_1588_CLOCK_IXP46X
 	tristate "Intel IXP46x as PTP clock"
diff --git a/drivers/ptp/Makefile b/drivers/ptp/Makefile
index fd28207f5379..19efa9cfa950 100644
--- a/drivers/ptp/Makefile
+++ b/drivers/ptp/Makefile
@@ -9,3 +9,4 @@ obj-$(CONFIG_PTP_1588_CLOCK_DTE)	+= ptp_dte.o
 obj-$(CONFIG_PTP_1588_CLOCK_IXP46X)	+= ptp_ixp46x.o
 obj-$(CONFIG_PTP_1588_CLOCK_PCH)	+= ptp_pch.o
 obj-$(CONFIG_PTP_1588_CLOCK_KVM)	+= ptp_kvm.o
+obj-$(CONFIG_PTP_1588_CLOCK_QORIQ)	+= ptp_qoriq.o
diff --git a/drivers/ptp/ptp_pch.c b/drivers/ptp/ptp_pch.c
index b3285175f20f..78ccf936d356 100644
--- a/drivers/ptp/ptp_pch.c
+++ b/drivers/ptp/ptp_pch.c
@@ -452,7 +452,6 @@ static int ptp_pch_adjtime(struct ptp_clock_info *ptp, s64 delta)
 static int ptp_pch_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts)
 {
 	u64 ns;
-	u32 remainder;
 	unsigned long flags;
 	struct pch_dev *pch_dev = container_of(ptp, struct pch_dev, caps);
 	struct pch_ts_regs __iomem *regs = pch_dev->regs;
@@ -461,8 +460,7 @@ static int ptp_pch_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts)
 	ns = pch_systime_read(regs);
 	spin_unlock_irqrestore(&pch_dev->register_lock, flags);
 
-	ts->tv_sec = div_u64_rem(ns, 1000000000, &remainder);
-	ts->tv_nsec = remainder;
+	*ts = ns_to_timespec64(ns);
 	return 0;
 }
 
@@ -474,8 +472,7 @@ static int ptp_pch_settime(struct ptp_clock_info *ptp,
 	struct pch_dev *pch_dev = container_of(ptp, struct pch_dev, caps);
 	struct pch_ts_regs __iomem *regs = pch_dev->regs;
 
-	ns = ts->tv_sec * 1000000000ULL;
-	ns += ts->tv_nsec;
+	ns = timespec64_to_ns(ts);
 
 	spin_lock_irqsave(&pch_dev->register_lock, flags);
 	pch_systime_write(regs, ns);
diff --git a/drivers/ptp/ptp_qoriq.c b/drivers/ptp/ptp_qoriq.c
new file mode 100644
index 000000000000..1468a1642b49
--- /dev/null
+++ b/drivers/ptp/ptp_qoriq.c
@@ -0,0 +1,454 @@
+/*
+ * PTP 1588 clock for Freescale QorIQ 1588 timer
+ *
+ * Copyright (C) 2010 OMICRON electronics GmbH
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/device.h>
+#include <linux/hrtimer.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/timex.h>
+#include <linux/slab.h>
+
+#include <linux/fsl/ptp_qoriq.h>
+
+/*
+ * Register access functions
+ */
+
+/* Caller must hold qoriq_ptp->lock. */
+static u64 tmr_cnt_read(struct qoriq_ptp *qoriq_ptp)
+{
+	u64 ns;
+	u32 lo, hi;
+
+	lo = qoriq_read(&qoriq_ptp->regs->tmr_cnt_l);
+	hi = qoriq_read(&qoriq_ptp->regs->tmr_cnt_h);
+	ns = ((u64) hi) << 32;
+	ns |= lo;
+	return ns;
+}
+
+/* Caller must hold qoriq_ptp->lock. */
+static void tmr_cnt_write(struct qoriq_ptp *qoriq_ptp, u64 ns)
+{
+	u32 hi = ns >> 32;
+	u32 lo = ns & 0xffffffff;
+
+	qoriq_write(&qoriq_ptp->regs->tmr_cnt_l, lo);
+	qoriq_write(&qoriq_ptp->regs->tmr_cnt_h, hi);
+}
+
+/* Caller must hold qoriq_ptp->lock. */
+static void set_alarm(struct qoriq_ptp *qoriq_ptp)
+{
+	u64 ns;
+	u32 lo, hi;
+
+	ns = tmr_cnt_read(qoriq_ptp) + 1500000000ULL;
+	ns = div_u64(ns, 1000000000UL) * 1000000000ULL;
+	ns -= qoriq_ptp->tclk_period;
+	hi = ns >> 32;
+	lo = ns & 0xffffffff;
+	qoriq_write(&qoriq_ptp->regs->tmr_alarm1_l, lo);
+	qoriq_write(&qoriq_ptp->regs->tmr_alarm1_h, hi);
+}
+
+/* Caller must hold qoriq_ptp->lock. */
+static void set_fipers(struct qoriq_ptp *qoriq_ptp)
+{
+	set_alarm(qoriq_ptp);
+	qoriq_write(&qoriq_ptp->regs->tmr_fiper1, qoriq_ptp->tmr_fiper1);
+	qoriq_write(&qoriq_ptp->regs->tmr_fiper2, qoriq_ptp->tmr_fiper2);
+}
+
+/*
+ * Interrupt service routine
+ */
+
+static irqreturn_t isr(int irq, void *priv)
+{
+	struct qoriq_ptp *qoriq_ptp = priv;
+	struct ptp_clock_event event;
+	u64 ns;
+	u32 ack = 0, lo, hi, mask, val;
+
+	val = qoriq_read(&qoriq_ptp->regs->tmr_tevent);
+
+	if (val & ETS1) {
+		ack |= ETS1;
+		hi = qoriq_read(&qoriq_ptp->regs->tmr_etts1_h);
+		lo = qoriq_read(&qoriq_ptp->regs->tmr_etts1_l);
+		event.type = PTP_CLOCK_EXTTS;
+		event.index = 0;
+		event.timestamp = ((u64) hi) << 32;
+		event.timestamp |= lo;
+		ptp_clock_event(qoriq_ptp->clock, &event);
+	}
+
+	if (val & ETS2) {
+		ack |= ETS2;
+		hi = qoriq_read(&qoriq_ptp->regs->tmr_etts2_h);
+		lo = qoriq_read(&qoriq_ptp->regs->tmr_etts2_l);
+		event.type = PTP_CLOCK_EXTTS;
+		event.index = 1;
+		event.timestamp = ((u64) hi) << 32;
+		event.timestamp |= lo;
+		ptp_clock_event(qoriq_ptp->clock, &event);
+	}
+
+	if (val & ALM2) {
+		ack |= ALM2;
+		if (qoriq_ptp->alarm_value) {
+			event.type = PTP_CLOCK_ALARM;
+			event.index = 0;
+			event.timestamp = qoriq_ptp->alarm_value;
+			ptp_clock_event(qoriq_ptp->clock, &event);
+		}
+		if (qoriq_ptp->alarm_interval) {
+			ns = qoriq_ptp->alarm_value + qoriq_ptp->alarm_interval;
+			hi = ns >> 32;
+			lo = ns & 0xffffffff;
+			spin_lock(&qoriq_ptp->lock);
+			qoriq_write(&qoriq_ptp->regs->tmr_alarm2_l, lo);
+			qoriq_write(&qoriq_ptp->regs->tmr_alarm2_h, hi);
+			spin_unlock(&qoriq_ptp->lock);
+			qoriq_ptp->alarm_value = ns;
+		} else {
+			qoriq_write(&qoriq_ptp->regs->tmr_tevent, ALM2);
+			spin_lock(&qoriq_ptp->lock);
+			mask = qoriq_read(&qoriq_ptp->regs->tmr_temask);
+			mask &= ~ALM2EN;
+			qoriq_write(&qoriq_ptp->regs->tmr_temask, mask);
+			spin_unlock(&qoriq_ptp->lock);
+			qoriq_ptp->alarm_value = 0;
+			qoriq_ptp->alarm_interval = 0;
+		}
+	}
+
+	if (val & PP1) {
+		ack |= PP1;
+		event.type = PTP_CLOCK_PPS;
+		ptp_clock_event(qoriq_ptp->clock, &event);
+	}
+
+	if (ack) {
+		qoriq_write(&qoriq_ptp->regs->tmr_tevent, ack);
+		return IRQ_HANDLED;
+	} else
+		return IRQ_NONE;
+}
+
+/*
+ * PTP clock operations
+ */
+
+static int ptp_qoriq_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
+{
+	u64 adj, diff;
+	u32 tmr_add;
+	int neg_adj = 0;
+	struct qoriq_ptp *qoriq_ptp = container_of(ptp, struct qoriq_ptp, caps);
+
+	if (scaled_ppm < 0) {
+		neg_adj = 1;
+		scaled_ppm = -scaled_ppm;
+	}
+	tmr_add = qoriq_ptp->tmr_add;
+	adj = tmr_add;
+
+	/* calculate diff as adj*(scaled_ppm/65536)/1000000
+	 * and round() to the nearest integer
+	 */
+	adj *= scaled_ppm;
+	diff = div_u64(adj, 8000000);
+	diff = (diff >> 13) + ((diff >> 12) & 1);
+
+	tmr_add = neg_adj ? tmr_add - diff : tmr_add + diff;
+
+	qoriq_write(&qoriq_ptp->regs->tmr_add, tmr_add);
+
+	return 0;
+}
+
+static int ptp_qoriq_adjtime(struct ptp_clock_info *ptp, s64 delta)
+{
+	s64 now;
+	unsigned long flags;
+	struct qoriq_ptp *qoriq_ptp = container_of(ptp, struct qoriq_ptp, caps);
+
+	spin_lock_irqsave(&qoriq_ptp->lock, flags);
+
+	now = tmr_cnt_read(qoriq_ptp);
+	now += delta;
+	tmr_cnt_write(qoriq_ptp, now);
+	set_fipers(qoriq_ptp);
+
+	spin_unlock_irqrestore(&qoriq_ptp->lock, flags);
+
+	return 0;
+}
+
+static int ptp_qoriq_gettime(struct ptp_clock_info *ptp,
+			       struct timespec64 *ts)
+{
+	u64 ns;
+	unsigned long flags;
+	struct qoriq_ptp *qoriq_ptp = container_of(ptp, struct qoriq_ptp, caps);
+
+	spin_lock_irqsave(&qoriq_ptp->lock, flags);
+
+	ns = tmr_cnt_read(qoriq_ptp);
+
+	spin_unlock_irqrestore(&qoriq_ptp->lock, flags);
+
+	*ts = ns_to_timespec64(ns);
+
+	return 0;
+}
+
+static int ptp_qoriq_settime(struct ptp_clock_info *ptp,
+			       const struct timespec64 *ts)
+{
+	u64 ns;
+	unsigned long flags;
+	struct qoriq_ptp *qoriq_ptp = container_of(ptp, struct qoriq_ptp, caps);
+
+	ns = timespec64_to_ns(ts);
+
+	spin_lock_irqsave(&qoriq_ptp->lock, flags);
+
+	tmr_cnt_write(qoriq_ptp, ns);
+	set_fipers(qoriq_ptp);
+
+	spin_unlock_irqrestore(&qoriq_ptp->lock, flags);
+
+	return 0;
+}
+
+static int ptp_qoriq_enable(struct ptp_clock_info *ptp,
+			      struct ptp_clock_request *rq, int on)
+{
+	struct qoriq_ptp *qoriq_ptp = container_of(ptp, struct qoriq_ptp, caps);
+	unsigned long flags;
+	u32 bit, mask;
+
+	switch (rq->type) {
+	case PTP_CLK_REQ_EXTTS:
+		switch (rq->extts.index) {
+		case 0:
+			bit = ETS1EN;
+			break;
+		case 1:
+			bit = ETS2EN;
+			break;
+		default:
+			return -EINVAL;
+		}
+		spin_lock_irqsave(&qoriq_ptp->lock, flags);
+		mask = qoriq_read(&qoriq_ptp->regs->tmr_temask);
+		if (on)
+			mask |= bit;
+		else
+			mask &= ~bit;
+		qoriq_write(&qoriq_ptp->regs->tmr_temask, mask);
+		spin_unlock_irqrestore(&qoriq_ptp->lock, flags);
+		return 0;
+
+	case PTP_CLK_REQ_PPS:
+		spin_lock_irqsave(&qoriq_ptp->lock, flags);
+		mask = qoriq_read(&qoriq_ptp->regs->tmr_temask);
+		if (on)
+			mask |= PP1EN;
+		else
+			mask &= ~PP1EN;
+		qoriq_write(&qoriq_ptp->regs->tmr_temask, mask);
+		spin_unlock_irqrestore(&qoriq_ptp->lock, flags);
+		return 0;
+
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static const struct ptp_clock_info ptp_qoriq_caps = {
+	.owner		= THIS_MODULE,
+	.name		= "qoriq ptp clock",
+	.max_adj	= 512000,
+	.n_alarm	= 0,
+	.n_ext_ts	= N_EXT_TS,
+	.n_per_out	= 0,
+	.n_pins		= 0,
+	.pps		= 1,
+	.adjfine	= ptp_qoriq_adjfine,
+	.adjtime	= ptp_qoriq_adjtime,
+	.gettime64	= ptp_qoriq_gettime,
+	.settime64	= ptp_qoriq_settime,
+	.enable		= ptp_qoriq_enable,
+};
+
+static int qoriq_ptp_probe(struct platform_device *dev)
+{
+	struct device_node *node = dev->dev.of_node;
+	struct qoriq_ptp *qoriq_ptp;
+	struct timespec64 now;
+	int err = -ENOMEM;
+	u32 tmr_ctrl;
+	unsigned long flags;
+
+	qoriq_ptp = kzalloc(sizeof(*qoriq_ptp), GFP_KERNEL);
+	if (!qoriq_ptp)
+		goto no_memory;
+
+	err = -ENODEV;
+
+	qoriq_ptp->caps = ptp_qoriq_caps;
+
+	if (of_property_read_u32(node, "fsl,cksel", &qoriq_ptp->cksel))
+		qoriq_ptp->cksel = DEFAULT_CKSEL;
+
+	if (of_property_read_u32(node,
+				 "fsl,tclk-period", &qoriq_ptp->tclk_period) ||
+	    of_property_read_u32(node,
+				 "fsl,tmr-prsc", &qoriq_ptp->tmr_prsc) ||
+	    of_property_read_u32(node,
+				 "fsl,tmr-add", &qoriq_ptp->tmr_add) ||
+	    of_property_read_u32(node,
+				 "fsl,tmr-fiper1", &qoriq_ptp->tmr_fiper1) ||
+	    of_property_read_u32(node,
+				 "fsl,tmr-fiper2", &qoriq_ptp->tmr_fiper2) ||
+	    of_property_read_u32(node,
+				 "fsl,max-adj", &qoriq_ptp->caps.max_adj)) {
+		pr_err("device tree node missing required elements\n");
+		goto no_node;
+	}
+
+	qoriq_ptp->irq = platform_get_irq(dev, 0);
+
+	if (qoriq_ptp->irq < 0) {
+		pr_err("irq not in device tree\n");
+		goto no_node;
+	}
+	if (request_irq(qoriq_ptp->irq, isr, 0, DRIVER, qoriq_ptp)) {
+		pr_err("request_irq failed\n");
+		goto no_node;
+	}
+
+	qoriq_ptp->rsrc = platform_get_resource(dev, IORESOURCE_MEM, 0);
+	if (!qoriq_ptp->rsrc) {
+		pr_err("no resource\n");
+		goto no_resource;
+	}
+	if (request_resource(&iomem_resource, qoriq_ptp->rsrc)) {
+		pr_err("resource busy\n");
+		goto no_resource;
+	}
+
+	spin_lock_init(&qoriq_ptp->lock);
+
+	qoriq_ptp->regs = ioremap(qoriq_ptp->rsrc->start,
+				resource_size(qoriq_ptp->rsrc));
+	if (!qoriq_ptp->regs) {
+		pr_err("ioremap ptp registers failed\n");
+		goto no_ioremap;
+	}
+	getnstimeofday64(&now);
+	ptp_qoriq_settime(&qoriq_ptp->caps, &now);
+
+	tmr_ctrl =
+	  (qoriq_ptp->tclk_period & TCLK_PERIOD_MASK) << TCLK_PERIOD_SHIFT |
+	  (qoriq_ptp->cksel & CKSEL_MASK) << CKSEL_SHIFT;
+
+	spin_lock_irqsave(&qoriq_ptp->lock, flags);
+
+	qoriq_write(&qoriq_ptp->regs->tmr_ctrl,   tmr_ctrl);
+	qoriq_write(&qoriq_ptp->regs->tmr_add,    qoriq_ptp->tmr_add);
+	qoriq_write(&qoriq_ptp->regs->tmr_prsc,   qoriq_ptp->tmr_prsc);
+	qoriq_write(&qoriq_ptp->regs->tmr_fiper1, qoriq_ptp->tmr_fiper1);
+	qoriq_write(&qoriq_ptp->regs->tmr_fiper2, qoriq_ptp->tmr_fiper2);
+	set_alarm(qoriq_ptp);
+	qoriq_write(&qoriq_ptp->regs->tmr_ctrl,   tmr_ctrl|FIPERST|RTPE|TE|FRD);
+
+	spin_unlock_irqrestore(&qoriq_ptp->lock, flags);
+
+	qoriq_ptp->clock = ptp_clock_register(&qoriq_ptp->caps, &dev->dev);
+	if (IS_ERR(qoriq_ptp->clock)) {
+		err = PTR_ERR(qoriq_ptp->clock);
+		goto no_clock;
+	}
+	qoriq_ptp->phc_index = ptp_clock_index(qoriq_ptp->clock);
+
+	platform_set_drvdata(dev, qoriq_ptp);
+
+	return 0;
+
+no_clock:
+	iounmap(qoriq_ptp->regs);
+no_ioremap:
+	release_resource(qoriq_ptp->rsrc);
+no_resource:
+	free_irq(qoriq_ptp->irq, qoriq_ptp);
+no_node:
+	kfree(qoriq_ptp);
+no_memory:
+	return err;
+}
+
+static int qoriq_ptp_remove(struct platform_device *dev)
+{
+	struct qoriq_ptp *qoriq_ptp = platform_get_drvdata(dev);
+
+	qoriq_write(&qoriq_ptp->regs->tmr_temask, 0);
+	qoriq_write(&qoriq_ptp->regs->tmr_ctrl,   0);
+
+	ptp_clock_unregister(qoriq_ptp->clock);
+	iounmap(qoriq_ptp->regs);
+	release_resource(qoriq_ptp->rsrc);
+	free_irq(qoriq_ptp->irq, qoriq_ptp);
+	kfree(qoriq_ptp);
+
+	return 0;
+}
+
+static const struct of_device_id match_table[] = {
+	{ .compatible = "fsl,etsec-ptp" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, match_table);
+
+static struct platform_driver qoriq_ptp_driver = {
+	.driver = {
+		.name		= "ptp_qoriq",
+		.of_match_table	= match_table,
+	},
+	.probe       = qoriq_ptp_probe,
+	.remove      = qoriq_ptp_remove,
+};
+
+module_platform_driver(qoriq_ptp_driver);
+
+MODULE_AUTHOR("Richard Cochran <richardcochran@gmail.com>");
+MODULE_DESCRIPTION("PTP clock for Freescale QorIQ 1588 timer");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pwm/pwm-atmel-tcb.c b/drivers/pwm/pwm-atmel-tcb.c
index 4fb1be246c44..0d0f8376bc35 100644
--- a/drivers/pwm/pwm-atmel-tcb.c
+++ b/drivers/pwm/pwm-atmel-tcb.c
@@ -460,8 +460,7 @@ MODULE_DEVICE_TABLE(of, atmel_tcb_pwm_dt_ids);
 #ifdef CONFIG_PM_SLEEP
 static int atmel_tcb_pwm_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct atmel_tcb_pwm_chip *tcbpwm = platform_get_drvdata(pdev);
+	struct atmel_tcb_pwm_chip *tcbpwm = dev_get_drvdata(dev);
 	void __iomem *base = tcbpwm->tc->regs;
 	int i;
 
@@ -478,8 +477,7 @@ static int atmel_tcb_pwm_suspend(struct device *dev)
 
 static int atmel_tcb_pwm_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct atmel_tcb_pwm_chip *tcbpwm = platform_get_drvdata(pdev);
+	struct atmel_tcb_pwm_chip *tcbpwm = dev_get_drvdata(dev);
 	void __iomem *base = tcbpwm->tc->regs;
 	int i;
 
diff --git a/drivers/pwm/pwm-meson.c b/drivers/pwm/pwm-meson.c
index 0767deba8e62..822860b4801a 100644
--- a/drivers/pwm/pwm-meson.c
+++ b/drivers/pwm/pwm-meson.c
@@ -541,8 +541,8 @@ static int meson_pwm_probe(struct platform_device *pdev)
 	meson->data = of_device_get_match_data(&pdev->dev);
 	meson->inverter_mask = BIT(meson->chip.npwm) - 1;
 
-	channels = devm_kcalloc(&pdev->dev, meson->chip.npwm, sizeof(*meson),
-				GFP_KERNEL);
+	channels = devm_kcalloc(&pdev->dev, meson->chip.npwm,
+				sizeof(*channels), GFP_KERNEL);
 	if (!channels)
 		return -ENOMEM;
 
diff --git a/drivers/pwm/pwm-rcar.c b/drivers/pwm/pwm-rcar.c
index 91d11f2e2fef..748f614d5375 100644
--- a/drivers/pwm/pwm-rcar.c
+++ b/drivers/pwm/pwm-rcar.c
@@ -261,8 +261,7 @@ MODULE_DEVICE_TABLE(of, rcar_pwm_of_table);
 #ifdef CONFIG_PM_SLEEP
 static struct pwm_device *rcar_pwm_dev_to_pwm_dev(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rcar_pwm_chip *rcar_pwm = platform_get_drvdata(pdev);
+	struct rcar_pwm_chip *rcar_pwm = dev_get_drvdata(dev);
 	struct pwm_chip *chip = &rcar_pwm->chip;
 
 	return &chip->pwms[0];
diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c
index 2708212933f7..60bfc07c4912 100644
--- a/drivers/pwm/pwm-stm32.c
+++ b/drivers/pwm/pwm-stm32.c
@@ -8,6 +8,7 @@
  *             pwm-atmel.c from Bo Shen
  */
 
+#include <linux/bitfield.h>
 #include <linux/mfd/stm32-timers.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -25,6 +26,7 @@ struct stm32_pwm {
 	struct regmap *regmap;
 	u32 max_arr;
 	bool have_complementary_output;
+	u32 capture[4] ____cacheline_aligned; /* DMA'able buffer */
 };
 
 struct stm32_breakinput {
@@ -62,6 +64,258 @@ static int write_ccrx(struct stm32_pwm *dev, int ch, u32 value)
 	return -EINVAL;
 }
 
+#define TIM_CCER_CC12P (TIM_CCER_CC1P | TIM_CCER_CC2P)
+#define TIM_CCER_CC12E (TIM_CCER_CC1E | TIM_CCER_CC2E)
+#define TIM_CCER_CC34P (TIM_CCER_CC3P | TIM_CCER_CC4P)
+#define TIM_CCER_CC34E (TIM_CCER_CC3E | TIM_CCER_CC4E)
+
+/*
+ * Capture using PWM input mode:
+ *                              ___          ___
+ * TI[1, 2, 3 or 4]: ........._|   |________|
+ *                             ^0  ^1       ^2
+ *                              .   .        .
+ *                              .   .        XXXXX
+ *                              .   .   XXXXX     |
+ *                              .  XXXXX     .    |
+ *                            XXXXX .        .    |
+ * COUNTER:        ______XXXXX  .   .        .    |_XXX
+ *                 start^       .   .        .        ^stop
+ *                      .       .   .        .
+ *                      v       v   .        v
+ *                                  v
+ * CCR1/CCR3:       tx..........t0...........t2
+ * CCR2/CCR4:       tx..............t1.........
+ *
+ * DMA burst transfer:          |            |
+ *                              v            v
+ * DMA buffer:                  { t0, tx }   { t2, t1 }
+ * DMA done:                                 ^
+ *
+ * 0: IC1/3 snapchot on rising edge: counter value -> CCR1/CCR3
+ *    + DMA transfer CCR[1/3] & CCR[2/4] values (t0, tx: doesn't care)
+ * 1: IC2/4 snapchot on falling edge: counter value -> CCR2/CCR4
+ * 2: IC1/3 snapchot on rising edge: counter value -> CCR1/CCR3
+ *    + DMA transfer CCR[1/3] & CCR[2/4] values (t2, t1)
+ *
+ * DMA done, compute:
+ * - Period     = t2 - t0
+ * - Duty cycle = t1 - t0
+ */
+static int stm32_pwm_raw_capture(struct stm32_pwm *priv, struct pwm_device *pwm,
+				 unsigned long tmo_ms, u32 *raw_prd,
+				 u32 *raw_dty)
+{
+	struct device *parent = priv->chip.dev->parent;
+	enum stm32_timers_dmas dma_id;
+	u32 ccen, ccr;
+	int ret;
+
+	/* Ensure registers have been updated, enable counter and capture */
+	regmap_update_bits(priv->regmap, TIM_EGR, TIM_EGR_UG, TIM_EGR_UG);
+	regmap_update_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN, TIM_CR1_CEN);
+
+	/* Use cc1 or cc3 DMA resp for PWM input channels 1 & 2 or 3 & 4 */
+	dma_id = pwm->hwpwm < 2 ? STM32_TIMERS_DMA_CH1 : STM32_TIMERS_DMA_CH3;
+	ccen = pwm->hwpwm < 2 ? TIM_CCER_CC12E : TIM_CCER_CC34E;
+	ccr = pwm->hwpwm < 2 ? TIM_CCR1 : TIM_CCR3;
+	regmap_update_bits(priv->regmap, TIM_CCER, ccen, ccen);
+
+	/*
+	 * Timer DMA burst mode. Request 2 registers, 2 bursts, to get both
+	 * CCR1 & CCR2 (or CCR3 & CCR4) on each capture event.
+	 * We'll get two capture snapchots: { CCR1, CCR2 }, { CCR1, CCR2 }
+	 * or { CCR3, CCR4 }, { CCR3, CCR4 }
+	 */
+	ret = stm32_timers_dma_burst_read(parent, priv->capture, dma_id, ccr, 2,
+					  2, tmo_ms);
+	if (ret)
+		goto stop;
+
+	/* Period: t2 - t0 (take care of counter overflow) */
+	if (priv->capture[0] <= priv->capture[2])
+		*raw_prd = priv->capture[2] - priv->capture[0];
+	else
+		*raw_prd = priv->max_arr - priv->capture[0] + priv->capture[2];
+
+	/* Duty cycle capture requires at least two capture units */
+	if (pwm->chip->npwm < 2)
+		*raw_dty = 0;
+	else if (priv->capture[0] <= priv->capture[3])
+		*raw_dty = priv->capture[3] - priv->capture[0];
+	else
+		*raw_dty = priv->max_arr - priv->capture[0] + priv->capture[3];
+
+	if (*raw_dty > *raw_prd) {
+		/*
+		 * Race beetween PWM input and DMA: it may happen
+		 * falling edge triggers new capture on TI2/4 before DMA
+		 * had a chance to read CCR2/4. It means capture[1]
+		 * contains period + duty_cycle. So, subtract period.
+		 */
+		*raw_dty -= *raw_prd;
+	}
+
+stop:
+	regmap_update_bits(priv->regmap, TIM_CCER, ccen, 0);
+	regmap_update_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN, 0);
+
+	return ret;
+}
+
+static int stm32_pwm_capture(struct pwm_chip *chip, struct pwm_device *pwm,
+			     struct pwm_capture *result, unsigned long tmo_ms)
+{
+	struct stm32_pwm *priv = to_stm32_pwm_dev(chip);
+	unsigned long long prd, div, dty;
+	unsigned long rate;
+	unsigned int psc = 0, icpsc, scale;
+	u32 raw_prd, raw_dty;
+	int ret = 0;
+
+	mutex_lock(&priv->lock);
+
+	if (active_channels(priv)) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	ret = clk_enable(priv->clk);
+	if (ret) {
+		dev_err(priv->chip.dev, "failed to enable counter clock\n");
+		goto unlock;
+	}
+
+	rate = clk_get_rate(priv->clk);
+	if (!rate) {
+		ret = -EINVAL;
+		goto clk_dis;
+	}
+
+	/* prescaler: fit timeout window provided by upper layer */
+	div = (unsigned long long)rate * (unsigned long long)tmo_ms;
+	do_div(div, MSEC_PER_SEC);
+	prd = div;
+	while ((div > priv->max_arr) && (psc < MAX_TIM_PSC)) {
+		psc++;
+		div = prd;
+		do_div(div, psc + 1);
+	}
+	regmap_write(priv->regmap, TIM_ARR, priv->max_arr);
+	regmap_write(priv->regmap, TIM_PSC, psc);
+
+	/* Map TI1 or TI2 PWM input to IC1 & IC2 (or TI3/4 to IC3 & IC4) */
+	regmap_update_bits(priv->regmap,
+			   pwm->hwpwm < 2 ? TIM_CCMR1 : TIM_CCMR2,
+			   TIM_CCMR_CC1S | TIM_CCMR_CC2S, pwm->hwpwm & 0x1 ?
+			   TIM_CCMR_CC1S_TI2 | TIM_CCMR_CC2S_TI2 :
+			   TIM_CCMR_CC1S_TI1 | TIM_CCMR_CC2S_TI1);
+
+	/* Capture period on IC1/3 rising edge, duty cycle on IC2/4 falling. */
+	regmap_update_bits(priv->regmap, TIM_CCER, pwm->hwpwm < 2 ?
+			   TIM_CCER_CC12P : TIM_CCER_CC34P, pwm->hwpwm < 2 ?
+			   TIM_CCER_CC2P : TIM_CCER_CC4P);
+
+	ret = stm32_pwm_raw_capture(priv, pwm, tmo_ms, &raw_prd, &raw_dty);
+	if (ret)
+		goto stop;
+
+	/*
+	 * Got a capture. Try to improve accuracy at high rates:
+	 * - decrease counter clock prescaler, scale up to max rate.
+	 * - use input prescaler, capture once every /2 /4 or /8 edges.
+	 */
+	if (raw_prd) {
+		u32 max_arr = priv->max_arr - 0x1000; /* arbitrary margin */
+
+		scale = max_arr / min(max_arr, raw_prd);
+	} else {
+		scale = priv->max_arr; /* bellow resolution, use max scale */
+	}
+
+	if (psc && scale > 1) {
+		/* 2nd measure with new scale */
+		psc /= scale;
+		regmap_write(priv->regmap, TIM_PSC, psc);
+		ret = stm32_pwm_raw_capture(priv, pwm, tmo_ms, &raw_prd,
+					    &raw_dty);
+		if (ret)
+			goto stop;
+	}
+
+	/* Compute intermediate period not to exceed timeout at low rates */
+	prd = (unsigned long long)raw_prd * (psc + 1) * NSEC_PER_SEC;
+	do_div(prd, rate);
+
+	for (icpsc = 0; icpsc < MAX_TIM_ICPSC ; icpsc++) {
+		/* input prescaler: also keep arbitrary margin */
+		if (raw_prd >= (priv->max_arr - 0x1000) >> (icpsc + 1))
+			break;
+		if (prd >= (tmo_ms * NSEC_PER_MSEC) >> (icpsc + 2))
+			break;
+	}
+
+	if (!icpsc)
+		goto done;
+
+	/* Last chance to improve period accuracy, using input prescaler */
+	regmap_update_bits(priv->regmap,
+			   pwm->hwpwm < 2 ? TIM_CCMR1 : TIM_CCMR2,
+			   TIM_CCMR_IC1PSC | TIM_CCMR_IC2PSC,
+			   FIELD_PREP(TIM_CCMR_IC1PSC, icpsc) |
+			   FIELD_PREP(TIM_CCMR_IC2PSC, icpsc));
+
+	ret = stm32_pwm_raw_capture(priv, pwm, tmo_ms, &raw_prd, &raw_dty);
+	if (ret)
+		goto stop;
+
+	if (raw_dty >= (raw_prd >> icpsc)) {
+		/*
+		 * We may fall here using input prescaler, when input
+		 * capture starts on high side (before falling edge).
+		 * Example with icpsc to capture on each 4 events:
+		 *
+		 *       start   1st capture                     2nd capture
+		 *         v     v                               v
+		 *         ___   _____   _____   _____   _____   ____
+		 * TI1..4     |__|    |__|    |__|    |__|    |__|
+		 *            v  v    .  .    .  .    .       v  v
+		 * icpsc1/3:  .  0    .  1    .  2    .  3    .  0
+		 * icpsc2/4:  0       1       2       3       0
+		 *            v  v                            v  v
+		 * CCR1/3  ......t0..............................t2
+		 * CCR2/4  ..t1..............................t1'...
+		 *               .                            .  .
+		 * Capture0:     .<----------------------------->.
+		 * Capture1:     .<-------------------------->.  .
+		 *               .                            .  .
+		 * Period:       .<------>                    .  .
+		 * Low side:                                  .<>.
+		 *
+		 * Result:
+		 * - Period = Capture0 / icpsc
+		 * - Duty = Period - Low side = Period - (Capture0 - Capture1)
+		 */
+		raw_dty = (raw_prd >> icpsc) - (raw_prd - raw_dty);
+	}
+
+done:
+	prd = (unsigned long long)raw_prd * (psc + 1) * NSEC_PER_SEC;
+	result->period = DIV_ROUND_UP_ULL(prd, rate << icpsc);
+	dty = (unsigned long long)raw_dty * (psc + 1) * NSEC_PER_SEC;
+	result->duty_cycle = DIV_ROUND_UP_ULL(dty, rate);
+stop:
+	regmap_write(priv->regmap, TIM_CCER, 0);
+	regmap_write(priv->regmap, pwm->hwpwm < 2 ? TIM_CCMR1 : TIM_CCMR2, 0);
+	regmap_write(priv->regmap, TIM_PSC, 0);
+clk_dis:
+	clk_disable(priv->clk);
+unlock:
+	mutex_unlock(&priv->lock);
+
+	return ret;
+}
+
 static int stm32_pwm_config(struct stm32_pwm *priv, int ch,
 			    int duty_ns, int period_ns)
 {
@@ -230,6 +484,9 @@ static int stm32_pwm_apply_locked(struct pwm_chip *chip, struct pwm_device *pwm,
 static const struct pwm_ops stm32pwm_ops = {
 	.owner = THIS_MODULE,
 	.apply = stm32_pwm_apply_locked,
+#if IS_ENABLED(CONFIG_DMA_ENGINE)
+	.capture = stm32_pwm_capture,
+#endif
 };
 
 static int stm32_pwm_set_breakinput(struct stm32_pwm *priv,
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index 097f61784a7d..5dbccf5f3037 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -180,6 +180,17 @@ config REGULATOR_BCM590XX
 	  BCM590xx PMUs. This will enable support for the software
 	  controllable LDO/Switching regulators.
 
+config REGULATOR_BD71837
+	tristate "ROHM BD71837 Power Regulator"
+	depends on MFD_BD71837
+	help
+	  This driver supports voltage regulators on ROHM BD71837 PMIC.
+	  This will enable support for the software controllable buck
+	  and LDO regulators.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called bd71837-regulator.
+
 config REGULATOR_BD9571MWV
 	tristate "ROHM BD9571MWV Regulators"
 	depends on MFD_BD9571MWV
@@ -801,6 +812,13 @@ config REGULATOR_STW481X_VMMC
 	  This driver supports the internal VMMC regulator in the STw481x
 	  PMIC chips.
 
+config REGULATOR_SY8106A
+	tristate "Silergy SY8106A regulator"
+	depends on I2C && (OF || COMPILE_TEST)
+	select REGMAP_I2C
+	help
+	  This driver supports SY8106A single output regulator.
+
 config REGULATOR_TPS51632
 	tristate "TI TPS51632 Power Regulator"
 	depends on I2C
diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile
index 590674fbecd7..bd818ceb7c72 100644
--- a/drivers/regulator/Makefile
+++ b/drivers/regulator/Makefile
@@ -27,6 +27,7 @@ obj-$(CONFIG_REGULATOR_AS3711) += as3711-regulator.o
 obj-$(CONFIG_REGULATOR_AS3722) += as3722-regulator.o
 obj-$(CONFIG_REGULATOR_AXP20X) += axp20x-regulator.o
 obj-$(CONFIG_REGULATOR_BCM590XX) += bcm590xx-regulator.o
+obj-$(CONFIG_REGULATOR_BD71837) += bd71837-regulator.o
 obj-$(CONFIG_REGULATOR_BD9571MWV) += bd9571mwv-regulator.o
 obj-$(CONFIG_REGULATOR_DA903X)	+= da903x.o
 obj-$(CONFIG_REGULATOR_DA9052)	+= da9052-regulator.o
@@ -100,6 +101,7 @@ obj-$(CONFIG_REGULATOR_SC2731) += sc2731-regulator.o
 obj-$(CONFIG_REGULATOR_SKY81452) += sky81452-regulator.o
 obj-$(CONFIG_REGULATOR_STM32_VREFBUF) += stm32-vrefbuf.o
 obj-$(CONFIG_REGULATOR_STW481X_VMMC) += stw481x-vmmc.o
+obj-$(CONFIG_REGULATOR_SY8106A) += sy8106a-regulator.o
 obj-$(CONFIG_REGULATOR_TI_ABB) += ti-abb-regulator.o
 obj-$(CONFIG_REGULATOR_TPS6105X) += tps6105x-regulator.o
 obj-$(CONFIG_REGULATOR_TPS62360) += tps62360-regulator.o
@@ -125,5 +127,4 @@ obj-$(CONFIG_REGULATOR_WM8350) += wm8350-regulator.o
 obj-$(CONFIG_REGULATOR_WM8400) += wm8400-regulator.o
 obj-$(CONFIG_REGULATOR_WM8994) += wm8994-regulator.o
 
-
 ccflags-$(CONFIG_REGULATOR_DEBUG) += -DDEBUG
diff --git a/drivers/regulator/ab8500.c b/drivers/regulator/ab8500.c
index 0f97514e3474..83dba3fbfe0c 100644
--- a/drivers/regulator/ab8500.c
+++ b/drivers/regulator/ab8500.c
@@ -132,33 +132,6 @@ static const unsigned int ldo_vaux56_voltages[] = {
 	2790000,
 };
 
-static const unsigned int ldo_vaux3_ab8540_voltages[] = {
-	1200000,
-	1500000,
-	1800000,
-	2100000,
-	2500000,
-	2750000,
-	2790000,
-	2910000,
-	3050000,
-};
-
-static const unsigned int ldo_vaux56_ab8540_voltages[] = {
-	750000, 760000, 770000, 780000, 790000, 800000,
-	810000, 820000, 830000, 840000, 850000, 860000,
-	870000, 880000, 890000, 900000, 910000, 920000,
-	930000, 940000, 950000, 960000, 970000, 980000,
-	990000, 1000000, 1010000, 1020000, 1030000,
-	1040000, 1050000, 1060000, 1070000, 1080000,
-	1090000, 1100000, 1110000, 1120000, 1130000,
-	1140000, 1150000, 1160000, 1170000, 1180000,
-	1190000, 1200000, 1210000, 1220000, 1230000,
-	1240000, 1250000, 1260000, 1270000, 1280000,
-	1290000, 1300000, 1310000, 1320000, 1330000,
-	1340000, 1350000, 1360000, 1800000, 2790000,
-};
-
 static const unsigned int ldo_vintcore_voltages[] = {
 	1200000,
 	1225000,
@@ -232,8 +205,6 @@ static const unsigned int ldo_vdmic_voltages[] = {
 static DEFINE_MUTEX(shared_mode_mutex);
 static struct ab8500_shared_mode ldo_anamic1_shared;
 static struct ab8500_shared_mode ldo_anamic2_shared;
-static struct ab8500_shared_mode ab8540_ldo_anamic1_shared;
-static struct ab8500_shared_mode ab8540_ldo_anamic2_shared;
 
 static int ab8500_regulator_enable(struct regulator_dev *rdev)
 {
@@ -507,53 +478,6 @@ static int ab8500_regulator_get_voltage_sel(struct regulator_dev *rdev)
 	return (regval & info->voltage_mask) >> voltage_shift;
 }
 
-static int ab8540_aux3_regulator_get_voltage_sel(struct regulator_dev *rdev)
-{
-	int ret, voltage_shift;
-	struct ab8500_regulator_info *info = rdev_get_drvdata(rdev);
-	u8 regval, regval_expand;
-
-	if (info == NULL) {
-		dev_err(rdev_get_dev(rdev), "regulator info null pointer\n");
-		return -EINVAL;
-	}
-
-	ret = abx500_get_register_interruptible(info->dev,
-			info->expand_register.voltage_bank,
-			info->expand_register.voltage_reg, &regval_expand);
-	if (ret < 0) {
-		dev_err(rdev_get_dev(rdev),
-			"couldn't read voltage expand reg for regulator\n");
-		return ret;
-	}
-
-	dev_vdbg(rdev_get_dev(rdev),
-		 "%s-get_voltage expand (bank, reg, mask, value): 0x%x, 0x%x, 0x%x, 0x%x\n",
-		 info->desc.name, info->expand_register.voltage_bank,
-		 info->expand_register.voltage_reg,
-		 info->expand_register.voltage_mask, regval_expand);
-
-	if (regval_expand & info->expand_register.voltage_mask)
-		return info->expand_register.voltage_limit;
-
-	ret = abx500_get_register_interruptible(info->dev,
-			info->voltage_bank, info->voltage_reg, &regval);
-	if (ret < 0) {
-		dev_err(rdev_get_dev(rdev),
-			"couldn't read voltage reg for regulator\n");
-		return ret;
-	}
-
-	dev_vdbg(rdev_get_dev(rdev),
-		 "%s-get_voltage (bank, reg, mask, value): 0x%x, 0x%x, 0x%x, 0x%x\n",
-		 info->desc.name, info->voltage_bank, info->voltage_reg,
-		 info->voltage_mask, regval);
-
-	voltage_shift = ffs(info->voltage_mask) - 1;
-
-	return (regval & info->voltage_mask) >> voltage_shift;
-}
-
 static int ab8500_regulator_set_voltage_sel(struct regulator_dev *rdev,
 					    unsigned selector)
 {
@@ -586,61 +510,6 @@ static int ab8500_regulator_set_voltage_sel(struct regulator_dev *rdev,
 	return ret;
 }
 
-static int ab8540_aux3_regulator_set_voltage_sel(struct regulator_dev *rdev,
-						unsigned selector)
-{
-	int ret;
-	struct ab8500_regulator_info *info = rdev_get_drvdata(rdev);
-	u8 regval, regval_expand;
-
-	if (info == NULL) {
-		dev_err(rdev_get_dev(rdev), "regulator info null pointer\n");
-		return -EINVAL;
-	}
-
-	if (selector < info->expand_register.voltage_limit) {
-		int voltage_shift = ffs(info->voltage_mask) - 1;
-
-		regval = (u8)selector << voltage_shift;
-		ret = abx500_mask_and_set_register_interruptible(info->dev,
-					info->voltage_bank, info->voltage_reg,
-					info->voltage_mask, regval);
-		if (ret < 0) {
-			dev_err(rdev_get_dev(rdev),
-				"couldn't set voltage reg for regulator\n");
-			return ret;
-		}
-
-		dev_vdbg(rdev_get_dev(rdev),
-			 "%s-set_voltage (bank, reg, mask, value): 0x%x, 0x%x, 0x%x, 0x%x\n",
-			 info->desc.name, info->voltage_bank, info->voltage_reg,
-			 info->voltage_mask, regval);
-
-		regval_expand = 0;
-	} else {
-		regval_expand = info->expand_register.voltage_mask;
-	}
-
-	ret = abx500_mask_and_set_register_interruptible(info->dev,
-				info->expand_register.voltage_bank,
-				info->expand_register.voltage_reg,
-				info->expand_register.voltage_mask,
-				regval_expand);
-	if (ret < 0) {
-		dev_err(rdev_get_dev(rdev),
-			"couldn't set expand voltage reg for regulator\n");
-		return ret;
-	}
-
-	dev_vdbg(rdev_get_dev(rdev),
-		 "%s-set_voltage expand (bank, reg, mask, value): 0x%x, 0x%x, 0x%x, 0x%x\n",
-		 info->desc.name, info->expand_register.voltage_bank,
-		 info->expand_register.voltage_reg,
-		 info->expand_register.voltage_mask, regval_expand);
-
-	return 0;
-}
-
 static struct regulator_ops ab8500_regulator_volt_mode_ops = {
 	.enable			= ab8500_regulator_enable,
 	.disable		= ab8500_regulator_disable,
@@ -653,18 +522,6 @@ static struct regulator_ops ab8500_regulator_volt_mode_ops = {
 	.list_voltage		= regulator_list_voltage_table,
 };
 
-static struct regulator_ops ab8540_aux3_regulator_volt_mode_ops = {
-	.enable		= ab8500_regulator_enable,
-	.disable	= ab8500_regulator_disable,
-	.get_optimum_mode	= ab8500_regulator_get_optimum_mode,
-	.set_mode	= ab8500_regulator_set_mode,
-	.get_mode	= ab8500_regulator_get_mode,
-	.is_enabled	= ab8500_regulator_is_enabled,
-	.get_voltage_sel = ab8540_aux3_regulator_get_voltage_sel,
-	.set_voltage_sel = ab8540_aux3_regulator_set_voltage_sel,
-	.list_voltage	= regulator_list_voltage_table,
-};
-
 static struct regulator_ops ab8500_regulator_volt_ops = {
 	.enable		= ab8500_regulator_enable,
 	.disable	= ab8500_regulator_disable,
@@ -1217,1156 +1074,118 @@ static struct ab8500_regulator_info
 	},
 };
 
-/* AB9540 regulator information */
-static struct ab8500_regulator_info
-		ab9540_regulator_info[AB9540_NUM_REGULATORS] = {
+static struct ab8500_shared_mode ldo_anamic1_shared = {
+	.shared_regulator = &ab8505_regulator_info[AB8505_LDO_ANAMIC2],
+};
+
+static struct ab8500_shared_mode ldo_anamic2_shared = {
+	.shared_regulator = &ab8505_regulator_info[AB8505_LDO_ANAMIC1],
+};
+
+struct ab8500_reg_init {
+	u8 bank;
+	u8 addr;
+	u8 mask;
+};
+
+#define REG_INIT(_id, _bank, _addr, _mask)	\
+	[_id] = {				\
+		.bank = _bank,			\
+		.addr = _addr,			\
+		.mask = _mask,			\
+	}
+
+/* AB8500 register init */
+static struct ab8500_reg_init ab8500_reg_init[] = {
 	/*
-	 * Variable Voltage Regulators
-	 *   name, min mV, max mV,
-	 *   update bank, reg, mask, enable val
-	 *   volt bank, reg, mask
+	 * 0x30, VanaRequestCtrl
+	 * 0xc0, VextSupply1RequestCtrl
 	 */
-	[AB9540_LDO_AUX1] = {
-		.desc = {
-			.name		= "LDO-AUX1",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB9540_LDO_AUX1,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vauxn_voltages),
-			.volt_table	= ldo_vauxn_voltages,
-		},
-		.load_lp_uA		= 5000,
-		.update_bank		= 0x04,
-		.update_reg		= 0x09,
-		.update_mask		= 0x03,
-		.update_val		= 0x01,
-		.update_val_idle	= 0x03,
-		.update_val_normal	= 0x01,
-		.voltage_bank		= 0x04,
-		.voltage_reg		= 0x1f,
-		.voltage_mask		= 0x0f,
-	},
-	[AB9540_LDO_AUX2] = {
-		.desc = {
-			.name		= "LDO-AUX2",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB9540_LDO_AUX2,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vauxn_voltages),
-			.volt_table	= ldo_vauxn_voltages,
-		},
-		.load_lp_uA		= 5000,
-		.update_bank		= 0x04,
-		.update_reg		= 0x09,
-		.update_mask		= 0x0c,
-		.update_val		= 0x04,
-		.update_val_idle	= 0x0c,
-		.update_val_normal	= 0x04,
-		.voltage_bank		= 0x04,
-		.voltage_reg		= 0x20,
-		.voltage_mask		= 0x0f,
-	},
-	[AB9540_LDO_AUX3] = {
-		.desc = {
-			.name		= "LDO-AUX3",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB9540_LDO_AUX3,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vaux3_voltages),
-			.volt_table	= ldo_vaux3_voltages,
-		},
-		.load_lp_uA		= 5000,
-		.update_bank		= 0x04,
-		.update_reg		= 0x0a,
-		.update_mask		= 0x03,
-		.update_val		= 0x01,
-		.update_val_idle	= 0x03,
-		.update_val_normal	= 0x01,
-		.voltage_bank		= 0x04,
-		.voltage_reg		= 0x21,
-		.voltage_mask		= 0x07,
-	},
-	[AB9540_LDO_AUX4] = {
-		.desc = {
-			.name		= "LDO-AUX4",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB9540_LDO_AUX4,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vauxn_voltages),
-			.volt_table	= ldo_vauxn_voltages,
-		},
-		.load_lp_uA		= 5000,
-		/* values for Vaux4Regu register */
-		.update_bank		= 0x04,
-		.update_reg		= 0x2e,
-		.update_mask		= 0x03,
-		.update_val		= 0x01,
-		.update_val_idle	= 0x03,
-		.update_val_normal	= 0x01,
-		/* values for Vaux4SEL register */
-		.voltage_bank		= 0x04,
-		.voltage_reg		= 0x2f,
-		.voltage_mask		= 0x0f,
-	},
-	[AB9540_LDO_INTCORE] = {
-		.desc = {
-			.name		= "LDO-INTCORE",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB9540_LDO_INTCORE,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vintcore_voltages),
-			.volt_table	= ldo_vintcore_voltages,
-		},
-		.load_lp_uA		= 5000,
-		.update_bank		= 0x03,
-		.update_reg		= 0x80,
-		.update_mask		= 0x44,
-		.update_val		= 0x44,
-		.update_val_idle	= 0x44,
-		.update_val_normal	= 0x04,
-		.voltage_bank		= 0x03,
-		.voltage_reg		= 0x80,
-		.voltage_mask		= 0x38,
-	},
-
+	REG_INIT(AB8500_REGUREQUESTCTRL2,	0x03, 0x04, 0xf0),
 	/*
-	 * Fixed Voltage Regulators
-	 *   name, fixed mV,
-	 *   update bank, reg, mask, enable val
+	 * 0x03, VextSupply2RequestCtrl
+	 * 0x0c, VextSupply3RequestCtrl
+	 * 0x30, Vaux1RequestCtrl
+	 * 0xc0, Vaux2RequestCtrl
 	 */
-	[AB9540_LDO_TVOUT] = {
-		.desc = {
-			.name		= "LDO-TVOUT",
-			.ops		= &ab8500_regulator_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB9540_LDO_TVOUT,
-			.owner		= THIS_MODULE,
-			.n_voltages	= 1,
-			.volt_table	= fixed_2000000_voltage,
-			.enable_time	= 10000,
-		},
-		.load_lp_uA		= 1000,
-		.update_bank		= 0x03,
-		.update_reg		= 0x80,
-		.update_mask		= 0x82,
-		.update_val		= 0x02,
-		.update_val_idle	= 0x82,
-		.update_val_normal	= 0x02,
-	},
-	[AB9540_LDO_USB] = {
-		.desc = {
-			.name           = "LDO-USB",
-			.ops            = &ab8500_regulator_ops,
-			.type           = REGULATOR_VOLTAGE,
-			.id             = AB9540_LDO_USB,
-			.owner          = THIS_MODULE,
-			.n_voltages     = 1,
-			.volt_table	= fixed_3300000_voltage,
-		},
-		.update_bank            = 0x03,
-		.update_reg             = 0x82,
-		.update_mask            = 0x03,
-		.update_val		= 0x01,
-		.update_val_idle	= 0x03,
-		.update_val_normal	= 0x01,
-	},
-	[AB9540_LDO_AUDIO] = {
-		.desc = {
-			.name		= "LDO-AUDIO",
-			.ops		= &ab8500_regulator_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB9540_LDO_AUDIO,
-			.owner		= THIS_MODULE,
-			.n_voltages	= 1,
-			.volt_table	= fixed_2000000_voltage,
-		},
-		.update_bank		= 0x03,
-		.update_reg		= 0x83,
-		.update_mask		= 0x02,
-		.update_val		= 0x02,
-	},
-	[AB9540_LDO_ANAMIC1] = {
-		.desc = {
-			.name		= "LDO-ANAMIC1",
-			.ops		= &ab8500_regulator_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB9540_LDO_ANAMIC1,
-			.owner		= THIS_MODULE,
-			.n_voltages	= 1,
-			.volt_table	= fixed_2050000_voltage,
-		},
-		.update_bank		= 0x03,
-		.update_reg		= 0x83,
-		.update_mask		= 0x08,
-		.update_val		= 0x08,
-	},
-	[AB9540_LDO_ANAMIC2] = {
-		.desc = {
-			.name		= "LDO-ANAMIC2",
-			.ops		= &ab8500_regulator_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB9540_LDO_ANAMIC2,
-			.owner		= THIS_MODULE,
-			.n_voltages	= 1,
-			.volt_table	= fixed_2050000_voltage,
-		},
-		.update_bank		= 0x03,
-		.update_reg		= 0x83,
-		.update_mask		= 0x10,
-		.update_val		= 0x10,
-	},
-	[AB9540_LDO_DMIC] = {
-		.desc = {
-			.name		= "LDO-DMIC",
-			.ops		= &ab8500_regulator_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB9540_LDO_DMIC,
-			.owner		= THIS_MODULE,
-			.n_voltages	= 1,
-			.volt_table	= fixed_1800000_voltage,
-		},
-		.update_bank		= 0x03,
-		.update_reg		= 0x83,
-		.update_mask		= 0x04,
-		.update_val		= 0x04,
-	},
-
-	/*
-	 * Regulators with fixed voltage and normal/idle modes
-	 */
-	[AB9540_LDO_ANA] = {
-		.desc = {
-			.name		= "LDO-ANA",
-			.ops		= &ab8500_regulator_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB9540_LDO_ANA,
-			.owner		= THIS_MODULE,
-			.n_voltages	= 1,
-			.volt_table	= fixed_1200000_voltage,
-		},
-		.load_lp_uA		= 1000,
-		.update_bank		= 0x04,
-		.update_reg		= 0x06,
-		.update_mask		= 0x0c,
-		.update_val		= 0x08,
-		.update_val_idle	= 0x0c,
-		.update_val_normal	= 0x08,
-	},
-};
-
-/* AB8540 regulator information */
-static struct ab8500_regulator_info
-		ab8540_regulator_info[AB8540_NUM_REGULATORS] = {
-	/*
-	 * Variable Voltage Regulators
-	 *   name, min mV, max mV,
-	 *   update bank, reg, mask, enable val
-	 *   volt bank, reg, mask
-	 */
-	[AB8540_LDO_AUX1] = {
-		.desc = {
-			.name		= "LDO-AUX1",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_AUX1,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vauxn_voltages),
-			.volt_table	= ldo_vauxn_voltages,
-		},
-		.load_lp_uA		= 5000,
-		.update_bank		= 0x04,
-		.update_reg		= 0x09,
-		.update_mask		= 0x03,
-		.update_val		= 0x01,
-		.update_val_idle	= 0x03,
-		.update_val_normal	= 0x01,
-		.voltage_bank		= 0x04,
-		.voltage_reg		= 0x1f,
-		.voltage_mask		= 0x0f,
-	},
-	[AB8540_LDO_AUX2] = {
-		.desc = {
-			.name		= "LDO-AUX2",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_AUX2,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vauxn_voltages),
-			.volt_table	= ldo_vauxn_voltages,
-		},
-		.load_lp_uA		= 5000,
-		.update_bank		= 0x04,
-		.update_reg		= 0x09,
-		.update_mask		= 0x0c,
-		.update_val		= 0x04,
-		.update_val_idle	= 0x0c,
-		.update_val_normal	= 0x04,
-		.voltage_bank		= 0x04,
-		.voltage_reg		= 0x20,
-		.voltage_mask		= 0x0f,
-	},
-	[AB8540_LDO_AUX3] = {
-		.desc = {
-			.name		= "LDO-AUX3",
-			.ops		= &ab8540_aux3_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_AUX3,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vaux3_ab8540_voltages),
-			.volt_table	= ldo_vaux3_ab8540_voltages,
-		},
-		.load_lp_uA		= 5000,
-		.update_bank		= 0x04,
-		.update_reg		= 0x0a,
-		.update_mask		= 0x03,
-		.update_val		= 0x01,
-		.update_val_idle	= 0x03,
-		.update_val_normal	= 0x01,
-		.voltage_bank		= 0x04,
-		.voltage_reg		= 0x21,
-		.voltage_mask		= 0x07,
-		.expand_register = {
-			.voltage_limit		= 8,
-			.voltage_bank		= 0x04,
-			.voltage_reg		= 0x01,
-			.voltage_mask		= 0x10,
-		}
-	},
-	[AB8540_LDO_AUX4] = {
-		.desc = {
-			.name		= "LDO-AUX4",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_AUX4,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vauxn_voltages),
-			.volt_table	= ldo_vauxn_voltages,
-		},
-		.load_lp_uA		= 5000,
-		/* values for Vaux4Regu register */
-		.update_bank		= 0x04,
-		.update_reg		= 0x2e,
-		.update_mask		= 0x03,
-		.update_val		= 0x01,
-		.update_val_idle	= 0x03,
-		.update_val_normal	= 0x01,
-		/* values for Vaux4SEL register */
-		.voltage_bank		= 0x04,
-		.voltage_reg		= 0x2f,
-		.voltage_mask		= 0x0f,
-	},
-	[AB8540_LDO_AUX5] = {
-		.desc = {
-			.name		= "LDO-AUX5",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_AUX5,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vaux56_ab8540_voltages),
-			.volt_table	= ldo_vaux56_ab8540_voltages,
-		},
-		.load_lp_uA		= 20000,
-		/* values for Vaux5Regu register */
-		.update_bank		= 0x04,
-		.update_reg		= 0x32,
-		.update_mask		= 0x03,
-		.update_val		= 0x01,
-		.update_val_idle	= 0x03,
-		.update_val_normal	= 0x01,
-		/* values for Vaux5SEL register */
-		.voltage_bank		= 0x04,
-		.voltage_reg		= 0x33,
-		.voltage_mask		= 0x3f,
-	},
-	[AB8540_LDO_AUX6] = {
-		.desc = {
-			.name		= "LDO-AUX6",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_AUX6,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vaux56_ab8540_voltages),
-			.volt_table	= ldo_vaux56_ab8540_voltages,
-		},
-		.load_lp_uA		= 20000,
-		/* values for Vaux6Regu register */
-		.update_bank		= 0x04,
-		.update_reg		= 0x35,
-		.update_mask		= 0x03,
-		.update_val		= 0x01,
-		.update_val_idle	= 0x03,
-		.update_val_normal	= 0x01,
-		/* values for Vaux6SEL register */
-		.voltage_bank		= 0x04,
-		.voltage_reg		= 0x36,
-		.voltage_mask		= 0x3f,
-	},
-	[AB8540_LDO_INTCORE] = {
-		.desc = {
-			.name		= "LDO-INTCORE",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_INTCORE,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vintcore_voltages),
-			.volt_table	= ldo_vintcore_voltages,
-		},
-		.load_lp_uA		= 5000,
-		.update_bank		= 0x03,
-		.update_reg		= 0x80,
-		.update_mask		= 0x44,
-		.update_val		= 0x44,
-		.update_val_idle	= 0x44,
-		.update_val_normal	= 0x04,
-		.voltage_bank		= 0x03,
-		.voltage_reg		= 0x80,
-		.voltage_mask		= 0x38,
-	},
-
-	/*
-	 * Fixed Voltage Regulators
-	 *   name, fixed mV,
-	 *   update bank, reg, mask, enable val
-	 */
-	[AB8540_LDO_TVOUT] = {
-		.desc = {
-			.name		= "LDO-TVOUT",
-			.ops		= &ab8500_regulator_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_TVOUT,
-			.owner		= THIS_MODULE,
-			.n_voltages	= 1,
-			.volt_table     = fixed_2000000_voltage,
-			.enable_time	= 10000,
-		},
-		.load_lp_uA		= 1000,
-		.update_bank		= 0x03,
-		.update_reg		= 0x80,
-		.update_mask		= 0x82,
-		.update_val		= 0x02,
-		.update_val_idle	= 0x82,
-		.update_val_normal	= 0x02,
-	},
-	[AB8540_LDO_AUDIO] = {
-		.desc = {
-			.name		= "LDO-AUDIO",
-			.ops		= &ab8500_regulator_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_AUDIO,
-			.owner		= THIS_MODULE,
-			.n_voltages	= 1,
-			.volt_table	= fixed_2000000_voltage,
-		},
-		.update_bank		= 0x03,
-		.update_reg		= 0x83,
-		.update_mask		= 0x02,
-		.update_val		= 0x02,
-	},
-	[AB8540_LDO_ANAMIC1] = {
-		.desc = {
-			.name		= "LDO-ANAMIC1",
-			.ops		= &ab8500_regulator_anamic_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_ANAMIC1,
-			.owner		= THIS_MODULE,
-			.n_voltages	= 1,
-			.volt_table	= fixed_2050000_voltage,
-		},
-		.shared_mode		= &ab8540_ldo_anamic1_shared,
-		.update_bank		= 0x03,
-		.update_reg		= 0x83,
-		.update_mask		= 0x08,
-		.update_val		= 0x08,
-		.mode_bank		= 0x03,
-		.mode_reg		= 0x83,
-		.mode_mask		= 0x20,
-		.mode_val_idle		= 0x20,
-		.mode_val_normal	= 0x00,
-	},
-	[AB8540_LDO_ANAMIC2] = {
-		.desc = {
-			.name		= "LDO-ANAMIC2",
-			.ops		= &ab8500_regulator_anamic_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_ANAMIC2,
-			.owner		= THIS_MODULE,
-			.n_voltages	= 1,
-			.volt_table	= fixed_2050000_voltage,
-		},
-		.shared_mode		= &ab8540_ldo_anamic2_shared,
-		.update_bank		= 0x03,
-		.update_reg		= 0x83,
-		.update_mask		= 0x10,
-		.update_val		= 0x10,
-		.mode_bank		= 0x03,
-		.mode_reg		= 0x83,
-		.mode_mask		= 0x20,
-		.mode_val_idle		= 0x20,
-		.mode_val_normal	= 0x00,
-	},
-	[AB8540_LDO_DMIC] = {
-		.desc = {
-			.name		= "LDO-DMIC",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_DMIC,
-			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vdmic_voltages),
-			.volt_table	= ldo_vdmic_voltages,
-		},
-		.load_lp_uA		= 1000,
-		.update_bank		= 0x03,
-		.update_reg		= 0x83,
-		.update_mask		= 0x04,
-		.update_val		= 0x04,
-		.voltage_bank		= 0x03,
-		.voltage_reg		= 0x83,
-		.voltage_mask		= 0xc0,
-	},
-
-	/*
-	 * Regulators with fixed voltage and normal/idle modes
-	 */
-	[AB8540_LDO_ANA] = {
-		.desc = {
-			.name		= "LDO-ANA",
-			.ops		= &ab8500_regulator_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_ANA,
-			.owner		= THIS_MODULE,
-			.n_voltages	= 1,
-			.volt_table     = fixed_1200000_voltage,
-		},
-		.load_lp_uA		= 1000,
-		.update_bank		= 0x04,
-		.update_reg		= 0x06,
-		.update_mask		= 0x0c,
-		.update_val		= 0x04,
-		.update_val_idle	= 0x0c,
-		.update_val_normal	= 0x04,
-	},
-	[AB8540_LDO_SDIO] = {
-		.desc = {
-			.name		= "LDO-SDIO",
-			.ops		= &ab8500_regulator_volt_mode_ops,
-			.type		= REGULATOR_VOLTAGE,
-			.id		= AB8540_LDO_SDIO,
-			.owner		= THIS_MODULE,
-			.n_voltages 	= ARRAY_SIZE(ldo_sdio_voltages),
-			.volt_table	= ldo_sdio_voltages,
-		},
-		.load_lp_uA		= 5000,
-		.update_bank		= 0x03,
-		.update_reg		= 0x88,
-		.update_mask		= 0x30,
-		.update_val		= 0x10,
-		.update_val_idle	= 0x30,
-		.update_val_normal	= 0x10,
-		.voltage_bank		= 0x03,
-		.voltage_reg		= 0x88,
-		.voltage_mask		= 0x07,
-	},
-};
-
-static struct ab8500_shared_mode ldo_anamic1_shared = {
-	.shared_regulator = &ab8505_regulator_info[AB8505_LDO_ANAMIC2],
-};
-
-static struct ab8500_shared_mode ldo_anamic2_shared = {
-	.shared_regulator = &ab8505_regulator_info[AB8505_LDO_ANAMIC1],
-};
-
-static struct ab8500_shared_mode ab8540_ldo_anamic1_shared = {
-	.shared_regulator = &ab8540_regulator_info[AB8540_LDO_ANAMIC2],
-};
-
-static struct ab8500_shared_mode ab8540_ldo_anamic2_shared = {
-	.shared_regulator = &ab8540_regulator_info[AB8540_LDO_ANAMIC1],
-};
-
-struct ab8500_reg_init {
-	u8 bank;
-	u8 addr;
-	u8 mask;
-};
-
-#define REG_INIT(_id, _bank, _addr, _mask)	\
-	[_id] = {				\
-		.bank = _bank,			\
-		.addr = _addr,			\
-		.mask = _mask,			\
-	}
-
-/* AB8500 register init */
-static struct ab8500_reg_init ab8500_reg_init[] = {
-	/*
-	 * 0x30, VanaRequestCtrl
-	 * 0xc0, VextSupply1RequestCtrl
-	 */
-	REG_INIT(AB8500_REGUREQUESTCTRL2,	0x03, 0x04, 0xf0),
-	/*
-	 * 0x03, VextSupply2RequestCtrl
-	 * 0x0c, VextSupply3RequestCtrl
-	 * 0x30, Vaux1RequestCtrl
-	 * 0xc0, Vaux2RequestCtrl
-	 */
-	REG_INIT(AB8500_REGUREQUESTCTRL3,	0x03, 0x05, 0xff),
-	/*
-	 * 0x03, Vaux3RequestCtrl
-	 * 0x04, SwHPReq
-	 */
-	REG_INIT(AB8500_REGUREQUESTCTRL4,	0x03, 0x06, 0x07),
-	/*
-	 * 0x08, VanaSysClkReq1HPValid
-	 * 0x20, Vaux1SysClkReq1HPValid
-	 * 0x40, Vaux2SysClkReq1HPValid
-	 * 0x80, Vaux3SysClkReq1HPValid
-	 */
-	REG_INIT(AB8500_REGUSYSCLKREQ1HPVALID1,	0x03, 0x07, 0xe8),
-	/*
-	 * 0x10, VextSupply1SysClkReq1HPValid
-	 * 0x20, VextSupply2SysClkReq1HPValid
-	 * 0x40, VextSupply3SysClkReq1HPValid
-	 */
-	REG_INIT(AB8500_REGUSYSCLKREQ1HPVALID2,	0x03, 0x08, 0x70),
-	/*
-	 * 0x08, VanaHwHPReq1Valid
-	 * 0x20, Vaux1HwHPReq1Valid
-	 * 0x40, Vaux2HwHPReq1Valid
-	 * 0x80, Vaux3HwHPReq1Valid
-	 */
-	REG_INIT(AB8500_REGUHWHPREQ1VALID1,	0x03, 0x09, 0xe8),
-	/*
-	 * 0x01, VextSupply1HwHPReq1Valid
-	 * 0x02, VextSupply2HwHPReq1Valid
-	 * 0x04, VextSupply3HwHPReq1Valid
-	 */
-	REG_INIT(AB8500_REGUHWHPREQ1VALID2,	0x03, 0x0a, 0x07),
-	/*
-	 * 0x08, VanaHwHPReq2Valid
-	 * 0x20, Vaux1HwHPReq2Valid
-	 * 0x40, Vaux2HwHPReq2Valid
-	 * 0x80, Vaux3HwHPReq2Valid
-	 */
-	REG_INIT(AB8500_REGUHWHPREQ2VALID1,	0x03, 0x0b, 0xe8),
-	/*
-	 * 0x01, VextSupply1HwHPReq2Valid
-	 * 0x02, VextSupply2HwHPReq2Valid
-	 * 0x04, VextSupply3HwHPReq2Valid
-	 */
-	REG_INIT(AB8500_REGUHWHPREQ2VALID2,	0x03, 0x0c, 0x07),
-	/*
-	 * 0x20, VanaSwHPReqValid
-	 * 0x80, Vaux1SwHPReqValid
-	 */
-	REG_INIT(AB8500_REGUSWHPREQVALID1,	0x03, 0x0d, 0xa0),
-	/*
-	 * 0x01, Vaux2SwHPReqValid
-	 * 0x02, Vaux3SwHPReqValid
-	 * 0x04, VextSupply1SwHPReqValid
-	 * 0x08, VextSupply2SwHPReqValid
-	 * 0x10, VextSupply3SwHPReqValid
-	 */
-	REG_INIT(AB8500_REGUSWHPREQVALID2,	0x03, 0x0e, 0x1f),
-	/*
-	 * 0x02, SysClkReq2Valid1
-	 * 0x04, SysClkReq3Valid1
-	 * 0x08, SysClkReq4Valid1
-	 * 0x10, SysClkReq5Valid1
-	 * 0x20, SysClkReq6Valid1
-	 * 0x40, SysClkReq7Valid1
-	 * 0x80, SysClkReq8Valid1
-	 */
-	REG_INIT(AB8500_REGUSYSCLKREQVALID1,	0x03, 0x0f, 0xfe),
-	/*
-	 * 0x02, SysClkReq2Valid2
-	 * 0x04, SysClkReq3Valid2
-	 * 0x08, SysClkReq4Valid2
-	 * 0x10, SysClkReq5Valid2
-	 * 0x20, SysClkReq6Valid2
-	 * 0x40, SysClkReq7Valid2
-	 * 0x80, SysClkReq8Valid2
-	 */
-	REG_INIT(AB8500_REGUSYSCLKREQVALID2,	0x03, 0x10, 0xfe),
-	/*
-	 * 0x02, VTVoutEna
-	 * 0x04, Vintcore12Ena
-	 * 0x38, Vintcore12Sel
-	 * 0x40, Vintcore12LP
-	 * 0x80, VTVoutLP
-	 */
-	REG_INIT(AB8500_REGUMISC1,		0x03, 0x80, 0xfe),
-	/*
-	 * 0x02, VaudioEna
-	 * 0x04, VdmicEna
-	 * 0x08, Vamic1Ena
-	 * 0x10, Vamic2Ena
-	 */
-	REG_INIT(AB8500_VAUDIOSUPPLY,		0x03, 0x83, 0x1e),
-	/*
-	 * 0x01, Vamic1_dzout
-	 * 0x02, Vamic2_dzout
-	 */
-	REG_INIT(AB8500_REGUCTRL1VAMIC,		0x03, 0x84, 0x03),
-	/*
-	 * 0x03, VpllRegu (NOTE! PRCMU register bits)
-	 * 0x0c, VanaRegu
-	 */
-	REG_INIT(AB8500_VPLLVANAREGU,		0x04, 0x06, 0x0f),
-	/*
-	 * 0x01, VrefDDREna
-	 * 0x02, VrefDDRSleepMode
-	 */
-	REG_INIT(AB8500_VREFDDR,		0x04, 0x07, 0x03),
-	/*
-	 * 0x03, VextSupply1Regu
-	 * 0x0c, VextSupply2Regu
-	 * 0x30, VextSupply3Regu
-	 * 0x40, ExtSupply2Bypass
-	 * 0x80, ExtSupply3Bypass
-	 */
-	REG_INIT(AB8500_EXTSUPPLYREGU,		0x04, 0x08, 0xff),
-	/*
-	 * 0x03, Vaux1Regu
-	 * 0x0c, Vaux2Regu
-	 */
-	REG_INIT(AB8500_VAUX12REGU,		0x04, 0x09, 0x0f),
-	/*
-	 * 0x03, Vaux3Regu
-	 */
-	REG_INIT(AB8500_VRF1VAUX3REGU,		0x04, 0x0a, 0x03),
-	/*
-	 * 0x0f, Vaux1Sel
-	 */
-	REG_INIT(AB8500_VAUX1SEL,		0x04, 0x1f, 0x0f),
-	/*
-	 * 0x0f, Vaux2Sel
-	 */
-	REG_INIT(AB8500_VAUX2SEL,		0x04, 0x20, 0x0f),
-	/*
-	 * 0x07, Vaux3Sel
-	 */
-	REG_INIT(AB8500_VRF1VAUX3SEL,		0x04, 0x21, 0x07),
-	/*
-	 * 0x01, VextSupply12LP
-	 */
-	REG_INIT(AB8500_REGUCTRL2SPARE,		0x04, 0x22, 0x01),
-	/*
-	 * 0x04, Vaux1Disch
-	 * 0x08, Vaux2Disch
-	 * 0x10, Vaux3Disch
-	 * 0x20, Vintcore12Disch
-	 * 0x40, VTVoutDisch
-	 * 0x80, VaudioDisch
-	 */
-	REG_INIT(AB8500_REGUCTRLDISCH,		0x04, 0x43, 0xfc),
-	/*
-	 * 0x02, VanaDisch
-	 * 0x04, VdmicPullDownEna
-	 * 0x10, VdmicDisch
-	 */
-	REG_INIT(AB8500_REGUCTRLDISCH2,		0x04, 0x44, 0x16),
-};
-
-/* AB8505 register init */
-static struct ab8500_reg_init ab8505_reg_init[] = {
-	/*
-	 * 0x03, VarmRequestCtrl
-	 * 0x0c, VsmpsCRequestCtrl
-	 * 0x30, VsmpsARequestCtrl
-	 * 0xc0, VsmpsBRequestCtrl
-	 */
-	REG_INIT(AB8505_REGUREQUESTCTRL1,	0x03, 0x03, 0xff),
-	/*
-	 * 0x03, VsafeRequestCtrl
-	 * 0x0c, VpllRequestCtrl
-	 * 0x30, VanaRequestCtrl
-	 */
-	REG_INIT(AB8505_REGUREQUESTCTRL2,	0x03, 0x04, 0x3f),
-	/*
-	 * 0x30, Vaux1RequestCtrl
-	 * 0xc0, Vaux2RequestCtrl
-	 */
-	REG_INIT(AB8505_REGUREQUESTCTRL3,	0x03, 0x05, 0xf0),
-	/*
-	 * 0x03, Vaux3RequestCtrl
-	 * 0x04, SwHPReq
-	 */
-	REG_INIT(AB8505_REGUREQUESTCTRL4,	0x03, 0x06, 0x07),
-	/*
-	 * 0x01, VsmpsASysClkReq1HPValid
-	 * 0x02, VsmpsBSysClkReq1HPValid
-	 * 0x04, VsafeSysClkReq1HPValid
-	 * 0x08, VanaSysClkReq1HPValid
-	 * 0x10, VpllSysClkReq1HPValid
-	 * 0x20, Vaux1SysClkReq1HPValid
-	 * 0x40, Vaux2SysClkReq1HPValid
-	 * 0x80, Vaux3SysClkReq1HPValid
-	 */
-	REG_INIT(AB8505_REGUSYSCLKREQ1HPVALID1,	0x03, 0x07, 0xff),
-	/*
-	 * 0x01, VsmpsCSysClkReq1HPValid
-	 * 0x02, VarmSysClkReq1HPValid
-	 * 0x04, VbbSysClkReq1HPValid
-	 * 0x08, VsmpsMSysClkReq1HPValid
-	 */
-	REG_INIT(AB8505_REGUSYSCLKREQ1HPVALID2,	0x03, 0x08, 0x0f),
-	/*
-	 * 0x01, VsmpsAHwHPReq1Valid
-	 * 0x02, VsmpsBHwHPReq1Valid
-	 * 0x04, VsafeHwHPReq1Valid
-	 * 0x08, VanaHwHPReq1Valid
-	 * 0x10, VpllHwHPReq1Valid
-	 * 0x20, Vaux1HwHPReq1Valid
-	 * 0x40, Vaux2HwHPReq1Valid
-	 * 0x80, Vaux3HwHPReq1Valid
-	 */
-	REG_INIT(AB8505_REGUHWHPREQ1VALID1,	0x03, 0x09, 0xff),
-	/*
-	 * 0x08, VsmpsMHwHPReq1Valid
-	 */
-	REG_INIT(AB8505_REGUHWHPREQ1VALID2,	0x03, 0x0a, 0x08),
-	/*
-	 * 0x01, VsmpsAHwHPReq2Valid
-	 * 0x02, VsmpsBHwHPReq2Valid
-	 * 0x04, VsafeHwHPReq2Valid
-	 * 0x08, VanaHwHPReq2Valid
-	 * 0x10, VpllHwHPReq2Valid
-	 * 0x20, Vaux1HwHPReq2Valid
-	 * 0x40, Vaux2HwHPReq2Valid
-	 * 0x80, Vaux3HwHPReq2Valid
-	 */
-	REG_INIT(AB8505_REGUHWHPREQ2VALID1,	0x03, 0x0b, 0xff),
-	/*
-	 * 0x08, VsmpsMHwHPReq2Valid
-	 */
-	REG_INIT(AB8505_REGUHWHPREQ2VALID2,	0x03, 0x0c, 0x08),
-	/*
-	 * 0x01, VsmpsCSwHPReqValid
-	 * 0x02, VarmSwHPReqValid
-	 * 0x04, VsmpsASwHPReqValid
-	 * 0x08, VsmpsBSwHPReqValid
-	 * 0x10, VsafeSwHPReqValid
-	 * 0x20, VanaSwHPReqValid
-	 * 0x40, VpllSwHPReqValid
-	 * 0x80, Vaux1SwHPReqValid
-	 */
-	REG_INIT(AB8505_REGUSWHPREQVALID1,	0x03, 0x0d, 0xff),
-	/*
-	 * 0x01, Vaux2SwHPReqValid
-	 * 0x02, Vaux3SwHPReqValid
-	 * 0x20, VsmpsMSwHPReqValid
-	 */
-	REG_INIT(AB8505_REGUSWHPREQVALID2,	0x03, 0x0e, 0x23),
-	/*
-	 * 0x02, SysClkReq2Valid1
-	 * 0x04, SysClkReq3Valid1
-	 * 0x08, SysClkReq4Valid1
-	 */
-	REG_INIT(AB8505_REGUSYSCLKREQVALID1,	0x03, 0x0f, 0x0e),
-	/*
-	 * 0x02, SysClkReq2Valid2
-	 * 0x04, SysClkReq3Valid2
-	 * 0x08, SysClkReq4Valid2
-	 */
-	REG_INIT(AB8505_REGUSYSCLKREQVALID2,	0x03, 0x10, 0x0e),
-	/*
-	 * 0x01, Vaux4SwHPReqValid
-	 * 0x02, Vaux4HwHPReq2Valid
-	 * 0x04, Vaux4HwHPReq1Valid
-	 * 0x08, Vaux4SysClkReq1HPValid
-	 */
-	REG_INIT(AB8505_REGUVAUX4REQVALID,	0x03, 0x11, 0x0f),
-	/*
-	 * 0x02, VadcEna
-	 * 0x04, VintCore12Ena
-	 * 0x38, VintCore12Sel
-	 * 0x40, VintCore12LP
-	 * 0x80, VadcLP
-	 */
-	REG_INIT(AB8505_REGUMISC1,		0x03, 0x80, 0xfe),
-	/*
-	 * 0x02, VaudioEna
-	 * 0x04, VdmicEna
-	 * 0x08, Vamic1Ena
-	 * 0x10, Vamic2Ena
-	 */
-	REG_INIT(AB8505_VAUDIOSUPPLY,		0x03, 0x83, 0x1e),
-	/*
-	 * 0x01, Vamic1_dzout
-	 * 0x02, Vamic2_dzout
-	 */
-	REG_INIT(AB8505_REGUCTRL1VAMIC,		0x03, 0x84, 0x03),
-	/*
-	 * 0x03, VsmpsARegu
-	 * 0x0c, VsmpsASelCtrl
-	 * 0x10, VsmpsAAutoMode
-	 * 0x20, VsmpsAPWMMode
-	 */
-	REG_INIT(AB8505_VSMPSAREGU,		0x04, 0x03, 0x3f),
-	/*
-	 * 0x03, VsmpsBRegu
-	 * 0x0c, VsmpsBSelCtrl
-	 * 0x10, VsmpsBAutoMode
-	 * 0x20, VsmpsBPWMMode
-	 */
-	REG_INIT(AB8505_VSMPSBREGU,		0x04, 0x04, 0x3f),
-	/*
-	 * 0x03, VsafeRegu
-	 * 0x0c, VsafeSelCtrl
-	 * 0x10, VsafeAutoMode
-	 * 0x20, VsafePWMMode
-	 */
-	REG_INIT(AB8505_VSAFEREGU,		0x04, 0x05, 0x3f),
-	/*
-	 * 0x03, VpllRegu (NOTE! PRCMU register bits)
-	 * 0x0c, VanaRegu
-	 */
-	REG_INIT(AB8505_VPLLVANAREGU,		0x04, 0x06, 0x0f),
-	/*
-	 * 0x03, VextSupply1Regu
-	 * 0x0c, VextSupply2Regu
-	 * 0x30, VextSupply3Regu
-	 * 0x40, ExtSupply2Bypass
-	 * 0x80, ExtSupply3Bypass
-	 */
-	REG_INIT(AB8505_EXTSUPPLYREGU,		0x04, 0x08, 0xff),
-	/*
-	 * 0x03, Vaux1Regu
-	 * 0x0c, Vaux2Regu
-	 */
-	REG_INIT(AB8505_VAUX12REGU,		0x04, 0x09, 0x0f),
-	/*
-	 * 0x0f, Vaux3Regu
-	 */
-	REG_INIT(AB8505_VRF1VAUX3REGU,		0x04, 0x0a, 0x0f),
-	/*
-	 * 0x3f, VsmpsASel1
-	 */
-	REG_INIT(AB8505_VSMPSASEL1,		0x04, 0x13, 0x3f),
-	/*
-	 * 0x3f, VsmpsASel2
-	 */
-	REG_INIT(AB8505_VSMPSASEL2,		0x04, 0x14, 0x3f),
-	/*
-	 * 0x3f, VsmpsASel3
-	 */
-	REG_INIT(AB8505_VSMPSASEL3,		0x04, 0x15, 0x3f),
-	/*
-	 * 0x3f, VsmpsBSel1
-	 */
-	REG_INIT(AB8505_VSMPSBSEL1,		0x04, 0x17, 0x3f),
-	/*
-	 * 0x3f, VsmpsBSel2
-	 */
-	REG_INIT(AB8505_VSMPSBSEL2,		0x04, 0x18, 0x3f),
-	/*
-	 * 0x3f, VsmpsBSel3
-	 */
-	REG_INIT(AB8505_VSMPSBSEL3,		0x04, 0x19, 0x3f),
-	/*
-	 * 0x7f, VsafeSel1
-	 */
-	REG_INIT(AB8505_VSAFESEL1,		0x04, 0x1b, 0x7f),
-	/*
-	 * 0x3f, VsafeSel2
-	 */
-	REG_INIT(AB8505_VSAFESEL2,		0x04, 0x1c, 0x7f),
-	/*
-	 * 0x3f, VsafeSel3
-	 */
-	REG_INIT(AB8505_VSAFESEL3,		0x04, 0x1d, 0x7f),
-	/*
-	 * 0x0f, Vaux1Sel
-	 */
-	REG_INIT(AB8505_VAUX1SEL,		0x04, 0x1f, 0x0f),
-	/*
-	 * 0x0f, Vaux2Sel
-	 */
-	REG_INIT(AB8505_VAUX2SEL,		0x04, 0x20, 0x0f),
-	/*
-	 * 0x07, Vaux3Sel
-	 * 0x30, VRF1Sel
-	 */
-	REG_INIT(AB8505_VRF1VAUX3SEL,		0x04, 0x21, 0x37),
-	/*
-	 * 0x03, Vaux4RequestCtrl
-	 */
-	REG_INIT(AB8505_VAUX4REQCTRL,		0x04, 0x2d, 0x03),
-	/*
-	 * 0x03, Vaux4Regu
-	 */
-	REG_INIT(AB8505_VAUX4REGU,		0x04, 0x2e, 0x03),
-	/*
-	 * 0x0f, Vaux4Sel
-	 */
-	REG_INIT(AB8505_VAUX4SEL,		0x04, 0x2f, 0x0f),
-	/*
-	 * 0x04, Vaux1Disch
-	 * 0x08, Vaux2Disch
-	 * 0x10, Vaux3Disch
-	 * 0x20, Vintcore12Disch
-	 * 0x40, VTVoutDisch
-	 * 0x80, VaudioDisch
-	 */
-	REG_INIT(AB8505_REGUCTRLDISCH,		0x04, 0x43, 0xfc),
-	/*
-	 * 0x02, VanaDisch
-	 * 0x04, VdmicPullDownEna
-	 * 0x10, VdmicDisch
-	 */
-	REG_INIT(AB8505_REGUCTRLDISCH2,		0x04, 0x44, 0x16),
-	/*
-	 * 0x01, Vaux4Disch
-	 */
-	REG_INIT(AB8505_REGUCTRLDISCH3,		0x04, 0x48, 0x01),
-	/*
-	 * 0x07, Vaux5Sel
-	 * 0x08, Vaux5LP
-	 * 0x10, Vaux5Ena
-	 * 0x20, Vaux5Disch
-	 * 0x40, Vaux5DisSfst
-	 * 0x80, Vaux5DisPulld
-	 */
-	REG_INIT(AB8505_CTRLVAUX5,		0x01, 0x55, 0xff),
-	/*
-	 * 0x07, Vaux6Sel
-	 * 0x08, Vaux6LP
-	 * 0x10, Vaux6Ena
-	 * 0x80, Vaux6DisPulld
-	 */
-	REG_INIT(AB8505_CTRLVAUX6,		0x01, 0x56, 0x9f),
-};
-
-/* AB9540 register init */
-static struct ab8500_reg_init ab9540_reg_init[] = {
-	/*
-	 * 0x03, VarmRequestCtrl
-	 * 0x0c, VapeRequestCtrl
-	 * 0x30, Vsmps1RequestCtrl
-	 * 0xc0, Vsmps2RequestCtrl
-	 */
-	REG_INIT(AB9540_REGUREQUESTCTRL1,	0x03, 0x03, 0xff),
-	/*
-	 * 0x03, Vsmps3RequestCtrl
-	 * 0x0c, VpllRequestCtrl
-	 * 0x30, VanaRequestCtrl
-	 * 0xc0, VextSupply1RequestCtrl
-	 */
-	REG_INIT(AB9540_REGUREQUESTCTRL2,	0x03, 0x04, 0xff),
-	/*
-	 * 0x03, VextSupply2RequestCtrl
-	 * 0x0c, VextSupply3RequestCtrl
-	 * 0x30, Vaux1RequestCtrl
-	 * 0xc0, Vaux2RequestCtrl
-	 */
-	REG_INIT(AB9540_REGUREQUESTCTRL3,	0x03, 0x05, 0xff),
+	REG_INIT(AB8500_REGUREQUESTCTRL3,	0x03, 0x05, 0xff),
 	/*
 	 * 0x03, Vaux3RequestCtrl
 	 * 0x04, SwHPReq
 	 */
-	REG_INIT(AB9540_REGUREQUESTCTRL4,	0x03, 0x06, 0x07),
+	REG_INIT(AB8500_REGUREQUESTCTRL4,	0x03, 0x06, 0x07),
 	/*
-	 * 0x01, Vsmps1SysClkReq1HPValid
-	 * 0x02, Vsmps2SysClkReq1HPValid
-	 * 0x04, Vsmps3SysClkReq1HPValid
 	 * 0x08, VanaSysClkReq1HPValid
-	 * 0x10, VpllSysClkReq1HPValid
 	 * 0x20, Vaux1SysClkReq1HPValid
 	 * 0x40, Vaux2SysClkReq1HPValid
 	 * 0x80, Vaux3SysClkReq1HPValid
 	 */
-	REG_INIT(AB9540_REGUSYSCLKREQ1HPVALID1,	0x03, 0x07, 0xff),
+	REG_INIT(AB8500_REGUSYSCLKREQ1HPVALID1,	0x03, 0x07, 0xe8),
 	/*
-	 * 0x01, VapeSysClkReq1HPValid
-	 * 0x02, VarmSysClkReq1HPValid
-	 * 0x04, VbbSysClkReq1HPValid
-	 * 0x08, VmodSysClkReq1HPValid
 	 * 0x10, VextSupply1SysClkReq1HPValid
 	 * 0x20, VextSupply2SysClkReq1HPValid
 	 * 0x40, VextSupply3SysClkReq1HPValid
 	 */
-	REG_INIT(AB9540_REGUSYSCLKREQ1HPVALID2,	0x03, 0x08, 0x7f),
+	REG_INIT(AB8500_REGUSYSCLKREQ1HPVALID2,	0x03, 0x08, 0x70),
 	/*
-	 * 0x01, Vsmps1HwHPReq1Valid
-	 * 0x02, Vsmps2HwHPReq1Valid
-	 * 0x04, Vsmps3HwHPReq1Valid
 	 * 0x08, VanaHwHPReq1Valid
-	 * 0x10, VpllHwHPReq1Valid
 	 * 0x20, Vaux1HwHPReq1Valid
 	 * 0x40, Vaux2HwHPReq1Valid
 	 * 0x80, Vaux3HwHPReq1Valid
 	 */
-	REG_INIT(AB9540_REGUHWHPREQ1VALID1,	0x03, 0x09, 0xff),
+	REG_INIT(AB8500_REGUHWHPREQ1VALID1,	0x03, 0x09, 0xe8),
 	/*
 	 * 0x01, VextSupply1HwHPReq1Valid
 	 * 0x02, VextSupply2HwHPReq1Valid
 	 * 0x04, VextSupply3HwHPReq1Valid
-	 * 0x08, VmodHwHPReq1Valid
 	 */
-	REG_INIT(AB9540_REGUHWHPREQ1VALID2,	0x03, 0x0a, 0x0f),
+	REG_INIT(AB8500_REGUHWHPREQ1VALID2,	0x03, 0x0a, 0x07),
 	/*
-	 * 0x01, Vsmps1HwHPReq2Valid
-	 * 0x02, Vsmps2HwHPReq2Valid
-	 * 0x03, Vsmps3HwHPReq2Valid
 	 * 0x08, VanaHwHPReq2Valid
-	 * 0x10, VpllHwHPReq2Valid
 	 * 0x20, Vaux1HwHPReq2Valid
 	 * 0x40, Vaux2HwHPReq2Valid
 	 * 0x80, Vaux3HwHPReq2Valid
 	 */
-	REG_INIT(AB9540_REGUHWHPREQ2VALID1,	0x03, 0x0b, 0xff),
+	REG_INIT(AB8500_REGUHWHPREQ2VALID1,	0x03, 0x0b, 0xe8),
 	/*
 	 * 0x01, VextSupply1HwHPReq2Valid
 	 * 0x02, VextSupply2HwHPReq2Valid
 	 * 0x04, VextSupply3HwHPReq2Valid
-	 * 0x08, VmodHwHPReq2Valid
 	 */
-	REG_INIT(AB9540_REGUHWHPREQ2VALID2,	0x03, 0x0c, 0x0f),
+	REG_INIT(AB8500_REGUHWHPREQ2VALID2,	0x03, 0x0c, 0x07),
 	/*
-	 * 0x01, VapeSwHPReqValid
-	 * 0x02, VarmSwHPReqValid
-	 * 0x04, Vsmps1SwHPReqValid
-	 * 0x08, Vsmps2SwHPReqValid
-	 * 0x10, Vsmps3SwHPReqValid
 	 * 0x20, VanaSwHPReqValid
-	 * 0x40, VpllSwHPReqValid
 	 * 0x80, Vaux1SwHPReqValid
 	 */
-	REG_INIT(AB9540_REGUSWHPREQVALID1,	0x03, 0x0d, 0xff),
+	REG_INIT(AB8500_REGUSWHPREQVALID1,	0x03, 0x0d, 0xa0),
 	/*
 	 * 0x01, Vaux2SwHPReqValid
 	 * 0x02, Vaux3SwHPReqValid
 	 * 0x04, VextSupply1SwHPReqValid
 	 * 0x08, VextSupply2SwHPReqValid
 	 * 0x10, VextSupply3SwHPReqValid
-	 * 0x20, VmodSwHPReqValid
 	 */
-	REG_INIT(AB9540_REGUSWHPREQVALID2,	0x03, 0x0e, 0x3f),
+	REG_INIT(AB8500_REGUSWHPREQVALID2,	0x03, 0x0e, 0x1f),
 	/*
 	 * 0x02, SysClkReq2Valid1
-	 * ...
+	 * 0x04, SysClkReq3Valid1
+	 * 0x08, SysClkReq4Valid1
+	 * 0x10, SysClkReq5Valid1
+	 * 0x20, SysClkReq6Valid1
+	 * 0x40, SysClkReq7Valid1
 	 * 0x80, SysClkReq8Valid1
 	 */
-	REG_INIT(AB9540_REGUSYSCLKREQVALID1,	0x03, 0x0f, 0xfe),
+	REG_INIT(AB8500_REGUSYSCLKREQVALID1,	0x03, 0x0f, 0xfe),
 	/*
 	 * 0x02, SysClkReq2Valid2
-	 * ...
+	 * 0x04, SysClkReq3Valid2
+	 * 0x08, SysClkReq4Valid2
+	 * 0x10, SysClkReq5Valid2
+	 * 0x20, SysClkReq6Valid2
+	 * 0x40, SysClkReq7Valid2
 	 * 0x80, SysClkReq8Valid2
 	 */
-	REG_INIT(AB9540_REGUSYSCLKREQVALID2,	0x03, 0x10, 0xfe),
-	/*
-	 * 0x01, Vaux4SwHPReqValid
-	 * 0x02, Vaux4HwHPReq2Valid
-	 * 0x04, Vaux4HwHPReq1Valid
-	 * 0x08, Vaux4SysClkReq1HPValid
-	 */
-	REG_INIT(AB9540_REGUVAUX4REQVALID,	0x03, 0x11, 0x0f),
+	REG_INIT(AB8500_REGUSYSCLKREQVALID2,	0x03, 0x10, 0xfe),
 	/*
 	 * 0x02, VTVoutEna
 	 * 0x04, Vintcore12Ena
@@ -2374,44 +1193,29 @@ static struct ab8500_reg_init ab9540_reg_init[] = {
 	 * 0x40, Vintcore12LP
 	 * 0x80, VTVoutLP
 	 */
-	REG_INIT(AB9540_REGUMISC1,		0x03, 0x80, 0xfe),
+	REG_INIT(AB8500_REGUMISC1,		0x03, 0x80, 0xfe),
 	/*
 	 * 0x02, VaudioEna
 	 * 0x04, VdmicEna
 	 * 0x08, Vamic1Ena
 	 * 0x10, Vamic2Ena
 	 */
-	REG_INIT(AB9540_VAUDIOSUPPLY,		0x03, 0x83, 0x1e),
+	REG_INIT(AB8500_VAUDIOSUPPLY,		0x03, 0x83, 0x1e),
 	/*
 	 * 0x01, Vamic1_dzout
 	 * 0x02, Vamic2_dzout
 	 */
-	REG_INIT(AB9540_REGUCTRL1VAMIC,		0x03, 0x84, 0x03),
-	/*
-	 * 0x03, Vsmps1Regu
-	 * 0x0c, Vsmps1SelCtrl
-	 * 0x10, Vsmps1AutoMode
-	 * 0x20, Vsmps1PWMMode
-	 */
-	REG_INIT(AB9540_VSMPS1REGU,		0x04, 0x03, 0x3f),
-	/*
-	 * 0x03, Vsmps2Regu
-	 * 0x0c, Vsmps2SelCtrl
-	 * 0x10, Vsmps2AutoMode
-	 * 0x20, Vsmps2PWMMode
-	 */
-	REG_INIT(AB9540_VSMPS2REGU,		0x04, 0x04, 0x3f),
+	REG_INIT(AB8500_REGUCTRL1VAMIC,		0x03, 0x84, 0x03),
 	/*
-	 * 0x03, Vsmps3Regu
-	 * 0x0c, Vsmps3SelCtrl
-	 * NOTE! PRCMU register
+	 * 0x03, VpllRegu (NOTE! PRCMU register bits)
+	 * 0x0c, VanaRegu
 	 */
-	REG_INIT(AB9540_VSMPS3REGU,		0x04, 0x05, 0x0f),
+	REG_INIT(AB8500_VPLLVANAREGU,		0x04, 0x06, 0x0f),
 	/*
-	 * 0x03, VpllRegu
-	 * 0x0c, VanaRegu
+	 * 0x01, VrefDDREna
+	 * 0x02, VrefDDRSleepMode
 	 */
-	REG_INIT(AB9540_VPLLVANAREGU,		0x04, 0x06, 0x0f),
+	REG_INIT(AB8500_VREFDDR,		0x04, 0x07, 0x03),
 	/*
 	 * 0x03, VextSupply1Regu
 	 * 0x0c, VextSupply2Regu
@@ -2419,83 +1223,33 @@ static struct ab8500_reg_init ab9540_reg_init[] = {
 	 * 0x40, ExtSupply2Bypass
 	 * 0x80, ExtSupply3Bypass
 	 */
-	REG_INIT(AB9540_EXTSUPPLYREGU,		0x04, 0x08, 0xff),
+	REG_INIT(AB8500_EXTSUPPLYREGU,		0x04, 0x08, 0xff),
 	/*
 	 * 0x03, Vaux1Regu
 	 * 0x0c, Vaux2Regu
 	 */
-	REG_INIT(AB9540_VAUX12REGU,		0x04, 0x09, 0x0f),
+	REG_INIT(AB8500_VAUX12REGU,		0x04, 0x09, 0x0f),
 	/*
-	 * 0x0c, Vrf1Regu
 	 * 0x03, Vaux3Regu
 	 */
-	REG_INIT(AB9540_VRF1VAUX3REGU,		0x04, 0x0a, 0x0f),
-	/*
-	 * 0x3f, Vsmps1Sel1
-	 */
-	REG_INIT(AB9540_VSMPS1SEL1,		0x04, 0x13, 0x3f),
-	/*
-	 * 0x3f, Vsmps1Sel2
-	 */
-	REG_INIT(AB9540_VSMPS1SEL2,		0x04, 0x14, 0x3f),
-	/*
-	 * 0x3f, Vsmps1Sel3
-	 */
-	REG_INIT(AB9540_VSMPS1SEL3,		0x04, 0x15, 0x3f),
-	/*
-	 * 0x3f, Vsmps2Sel1
-	 */
-	REG_INIT(AB9540_VSMPS2SEL1,		0x04, 0x17, 0x3f),
-	/*
-	 * 0x3f, Vsmps2Sel2
-	 */
-	REG_INIT(AB9540_VSMPS2SEL2,		0x04, 0x18, 0x3f),
-	/*
-	 * 0x3f, Vsmps2Sel3
-	 */
-	REG_INIT(AB9540_VSMPS2SEL3,		0x04, 0x19, 0x3f),
-	/*
-	 * 0x7f, Vsmps3Sel1
-	 * NOTE! PRCMU register
-	 */
-	REG_INIT(AB9540_VSMPS3SEL1,             0x04, 0x1b, 0x7f),
-	/*
-	 * 0x7f, Vsmps3Sel2
-	 * NOTE! PRCMU register
-	 */
-	REG_INIT(AB9540_VSMPS3SEL2,             0x04, 0x1c, 0x7f),
+	REG_INIT(AB8500_VRF1VAUX3REGU,		0x04, 0x0a, 0x03),
 	/*
 	 * 0x0f, Vaux1Sel
 	 */
-	REG_INIT(AB9540_VAUX1SEL,		0x04, 0x1f, 0x0f),
+	REG_INIT(AB8500_VAUX1SEL,		0x04, 0x1f, 0x0f),
 	/*
 	 * 0x0f, Vaux2Sel
 	 */
-	REG_INIT(AB9540_VAUX2SEL,		0x04, 0x20, 0x0f),
+	REG_INIT(AB8500_VAUX2SEL,		0x04, 0x20, 0x0f),
 	/*
 	 * 0x07, Vaux3Sel
-	 * 0x30, Vrf1Sel
 	 */
-	REG_INIT(AB9540_VRF1VAUX3SEL,		0x04, 0x21, 0x37),
+	REG_INIT(AB8500_VRF1VAUX3SEL,		0x04, 0x21, 0x07),
 	/*
 	 * 0x01, VextSupply12LP
 	 */
-	REG_INIT(AB9540_REGUCTRL2SPARE,		0x04, 0x22, 0x01),
-	/*
-	 * 0x03, Vaux4RequestCtrl
-	 */
-	REG_INIT(AB9540_VAUX4REQCTRL,		0x04, 0x2d, 0x03),
-	/*
-	 * 0x03, Vaux4Regu
-	 */
-	REG_INIT(AB9540_VAUX4REGU,		0x04, 0x2e, 0x03),
-	/*
-	 * 0x08, Vaux4Sel
-	 */
-	REG_INIT(AB9540_VAUX4SEL,		0x04, 0x2f, 0x0f),
+	REG_INIT(AB8500_REGUCTRL2SPARE,		0x04, 0x22, 0x01),
 	/*
-	 * 0x01, VpllDisch
-	 * 0x02, Vrf1Disch
 	 * 0x04, Vaux1Disch
 	 * 0x08, Vaux2Disch
 	 * 0x10, Vaux3Disch
@@ -2503,243 +1257,170 @@ static struct ab8500_reg_init ab9540_reg_init[] = {
 	 * 0x40, VTVoutDisch
 	 * 0x80, VaudioDisch
 	 */
-	REG_INIT(AB9540_REGUCTRLDISCH,		0x04, 0x43, 0xff),
+	REG_INIT(AB8500_REGUCTRLDISCH,		0x04, 0x43, 0xfc),
 	/*
-	 * 0x01, VsimDisch
 	 * 0x02, VanaDisch
 	 * 0x04, VdmicPullDownEna
-	 * 0x08, VpllPullDownEna
 	 * 0x10, VdmicDisch
 	 */
-	REG_INIT(AB9540_REGUCTRLDISCH2,		0x04, 0x44, 0x1f),
-	/*
-	 * 0x01, Vaux4Disch
-	 */
-	REG_INIT(AB9540_REGUCTRLDISCH3,		0x04, 0x48, 0x01),
+	REG_INIT(AB8500_REGUCTRLDISCH2,		0x04, 0x44, 0x16),
 };
 
-/* AB8540 register init */
-static struct ab8500_reg_init ab8540_reg_init[] = {
-	/*
-	 * 0x01, VSimSycClkReq1Valid
-	 * 0x02, VSimSycClkReq2Valid
-	 * 0x04, VSimSycClkReq3Valid
-	 * 0x08, VSimSycClkReq4Valid
-	 * 0x10, VSimSycClkReq5Valid
-	 * 0x20, VSimSycClkReq6Valid
-	 * 0x40, VSimSycClkReq7Valid
-	 * 0x80, VSimSycClkReq8Valid
-	 */
-	REG_INIT(AB8540_VSIMSYSCLKCTRL,		0x02, 0x33, 0xff),
+/* AB8505 register init */
+static struct ab8500_reg_init ab8505_reg_init[] = {
 	/*
 	 * 0x03, VarmRequestCtrl
-	 * 0x0c, VapeRequestCtrl
-	 * 0x30, Vsmps1RequestCtrl
-	 * 0xc0, Vsmps2RequestCtrl
+	 * 0x0c, VsmpsCRequestCtrl
+	 * 0x30, VsmpsARequestCtrl
+	 * 0xc0, VsmpsBRequestCtrl
 	 */
-	REG_INIT(AB8540_REGUREQUESTCTRL1,	0x03, 0x03, 0xff),
+	REG_INIT(AB8505_REGUREQUESTCTRL1,	0x03, 0x03, 0xff),
 	/*
-	 * 0x03, Vsmps3RequestCtrl
+	 * 0x03, VsafeRequestCtrl
 	 * 0x0c, VpllRequestCtrl
 	 * 0x30, VanaRequestCtrl
-	 * 0xc0, VextSupply1RequestCtrl
 	 */
-	REG_INIT(AB8540_REGUREQUESTCTRL2,	0x03, 0x04, 0xff),
+	REG_INIT(AB8505_REGUREQUESTCTRL2,	0x03, 0x04, 0x3f),
 	/*
-	 * 0x03, VextSupply2RequestCtrl
-	 * 0x0c, VextSupply3RequestCtrl
 	 * 0x30, Vaux1RequestCtrl
 	 * 0xc0, Vaux2RequestCtrl
 	 */
-	REG_INIT(AB8540_REGUREQUESTCTRL3,	0x03, 0x05, 0xff),
+	REG_INIT(AB8505_REGUREQUESTCTRL3,	0x03, 0x05, 0xf0),
 	/*
 	 * 0x03, Vaux3RequestCtrl
 	 * 0x04, SwHPReq
 	 */
-	REG_INIT(AB8540_REGUREQUESTCTRL4,	0x03, 0x06, 0x07),
+	REG_INIT(AB8505_REGUREQUESTCTRL4,	0x03, 0x06, 0x07),
 	/*
-	 * 0x01, Vsmps1SysClkReq1HPValid
-	 * 0x02, Vsmps2SysClkReq1HPValid
-	 * 0x04, Vsmps3SysClkReq1HPValid
+	 * 0x01, VsmpsASysClkReq1HPValid
+	 * 0x02, VsmpsBSysClkReq1HPValid
+	 * 0x04, VsafeSysClkReq1HPValid
 	 * 0x08, VanaSysClkReq1HPValid
 	 * 0x10, VpllSysClkReq1HPValid
 	 * 0x20, Vaux1SysClkReq1HPValid
 	 * 0x40, Vaux2SysClkReq1HPValid
 	 * 0x80, Vaux3SysClkReq1HPValid
 	 */
-	REG_INIT(AB8540_REGUSYSCLKREQ1HPVALID1,	0x03, 0x07, 0xff),
+	REG_INIT(AB8505_REGUSYSCLKREQ1HPVALID1,	0x03, 0x07, 0xff),
 	/*
-	 * 0x01, VapeSysClkReq1HPValid
+	 * 0x01, VsmpsCSysClkReq1HPValid
 	 * 0x02, VarmSysClkReq1HPValid
 	 * 0x04, VbbSysClkReq1HPValid
-	 * 0x10, VextSupply1SysClkReq1HPValid
-	 * 0x20, VextSupply2SysClkReq1HPValid
-	 * 0x40, VextSupply3SysClkReq1HPValid
+	 * 0x08, VsmpsMSysClkReq1HPValid
 	 */
-	REG_INIT(AB8540_REGUSYSCLKREQ1HPVALID2,	0x03, 0x08, 0x77),
+	REG_INIT(AB8505_REGUSYSCLKREQ1HPVALID2,	0x03, 0x08, 0x0f),
 	/*
-	 * 0x01, Vsmps1HwHPReq1Valid
-	 * 0x02, Vsmps2HwHPReq1Valid
-	 * 0x04, Vsmps3HwHPReq1Valid
+	 * 0x01, VsmpsAHwHPReq1Valid
+	 * 0x02, VsmpsBHwHPReq1Valid
+	 * 0x04, VsafeHwHPReq1Valid
 	 * 0x08, VanaHwHPReq1Valid
 	 * 0x10, VpllHwHPReq1Valid
 	 * 0x20, Vaux1HwHPReq1Valid
 	 * 0x40, Vaux2HwHPReq1Valid
 	 * 0x80, Vaux3HwHPReq1Valid
 	 */
-	REG_INIT(AB8540_REGUHWHPREQ1VALID1,	0x03, 0x09, 0xff),
+	REG_INIT(AB8505_REGUHWHPREQ1VALID1,	0x03, 0x09, 0xff),
 	/*
-	 * 0x01, VextSupply1HwHPReq1Valid
-	 * 0x02, VextSupply2HwHPReq1Valid
-	 * 0x04, VextSupply3HwHPReq1Valid
+	 * 0x08, VsmpsMHwHPReq1Valid
 	 */
-	REG_INIT(AB8540_REGUHWHPREQ1VALID2,	0x03, 0x0a, 0x07),
+	REG_INIT(AB8505_REGUHWHPREQ1VALID2,	0x03, 0x0a, 0x08),
 	/*
-	 * 0x01, Vsmps1HwHPReq2Valid
-	 * 0x02, Vsmps2HwHPReq2Valid
-	 * 0x03, Vsmps3HwHPReq2Valid
+	 * 0x01, VsmpsAHwHPReq2Valid
+	 * 0x02, VsmpsBHwHPReq2Valid
+	 * 0x04, VsafeHwHPReq2Valid
 	 * 0x08, VanaHwHPReq2Valid
 	 * 0x10, VpllHwHPReq2Valid
 	 * 0x20, Vaux1HwHPReq2Valid
 	 * 0x40, Vaux2HwHPReq2Valid
 	 * 0x80, Vaux3HwHPReq2Valid
 	 */
-	REG_INIT(AB8540_REGUHWHPREQ2VALID1,	0x03, 0x0b, 0xff),
+	REG_INIT(AB8505_REGUHWHPREQ2VALID1,	0x03, 0x0b, 0xff),
 	/*
-	 * 0x01, VextSupply1HwHPReq2Valid
-	 * 0x02, VextSupply2HwHPReq2Valid
-	 * 0x04, VextSupply3HwHPReq2Valid
+	 * 0x08, VsmpsMHwHPReq2Valid
 	 */
-	REG_INIT(AB8540_REGUHWHPREQ2VALID2,	0x03, 0x0c, 0x07),
+	REG_INIT(AB8505_REGUHWHPREQ2VALID2,	0x03, 0x0c, 0x08),
 	/*
-	 * 0x01, VapeSwHPReqValid
+	 * 0x01, VsmpsCSwHPReqValid
 	 * 0x02, VarmSwHPReqValid
-	 * 0x04, Vsmps1SwHPReqValid
-	 * 0x08, Vsmps2SwHPReqValid
-	 * 0x10, Vsmps3SwHPReqValid
+	 * 0x04, VsmpsASwHPReqValid
+	 * 0x08, VsmpsBSwHPReqValid
+	 * 0x10, VsafeSwHPReqValid
 	 * 0x20, VanaSwHPReqValid
 	 * 0x40, VpllSwHPReqValid
 	 * 0x80, Vaux1SwHPReqValid
 	 */
-	REG_INIT(AB8540_REGUSWHPREQVALID1,	0x03, 0x0d, 0xff),
+	REG_INIT(AB8505_REGUSWHPREQVALID1,	0x03, 0x0d, 0xff),
 	/*
 	 * 0x01, Vaux2SwHPReqValid
 	 * 0x02, Vaux3SwHPReqValid
-	 * 0x04, VextSupply1SwHPReqValid
-	 * 0x08, VextSupply2SwHPReqValid
-	 * 0x10, VextSupply3SwHPReqValid
+	 * 0x20, VsmpsMSwHPReqValid
 	 */
-	REG_INIT(AB8540_REGUSWHPREQVALID2,	0x03, 0x0e, 0x1f),
+	REG_INIT(AB8505_REGUSWHPREQVALID2,	0x03, 0x0e, 0x23),
 	/*
 	 * 0x02, SysClkReq2Valid1
-	 * ...
-	 * 0x80, SysClkReq8Valid1
+	 * 0x04, SysClkReq3Valid1
+	 * 0x08, SysClkReq4Valid1
 	 */
-	REG_INIT(AB8540_REGUSYSCLKREQVALID1,	0x03, 0x0f, 0xff),
+	REG_INIT(AB8505_REGUSYSCLKREQVALID1,	0x03, 0x0f, 0x0e),
 	/*
 	 * 0x02, SysClkReq2Valid2
-	 * ...
-	 * 0x80, SysClkReq8Valid2
+	 * 0x04, SysClkReq3Valid2
+	 * 0x08, SysClkReq4Valid2
 	 */
-	REG_INIT(AB8540_REGUSYSCLKREQVALID2,	0x03, 0x10, 0xff),
+	REG_INIT(AB8505_REGUSYSCLKREQVALID2,	0x03, 0x10, 0x0e),
 	/*
 	 * 0x01, Vaux4SwHPReqValid
 	 * 0x02, Vaux4HwHPReq2Valid
 	 * 0x04, Vaux4HwHPReq1Valid
 	 * 0x08, Vaux4SysClkReq1HPValid
 	 */
-	REG_INIT(AB8540_REGUVAUX4REQVALID,	0x03, 0x11, 0x0f),
-	/*
-	 * 0x01, Vaux5SwHPReqValid
-	 * 0x02, Vaux5HwHPReq2Valid
-	 * 0x04, Vaux5HwHPReq1Valid
-	 * 0x08, Vaux5SysClkReq1HPValid
-	 */
-	REG_INIT(AB8540_REGUVAUX5REQVALID,	0x03, 0x12, 0x0f),
-	/*
-	 * 0x01, Vaux6SwHPReqValid
-	 * 0x02, Vaux6HwHPReq2Valid
-	 * 0x04, Vaux6HwHPReq1Valid
-	 * 0x08, Vaux6SysClkReq1HPValid
-	 */
-	REG_INIT(AB8540_REGUVAUX6REQVALID,	0x03, 0x13, 0x0f),
-	/*
-	 * 0x01, VclkbSwHPReqValid
-	 * 0x02, VclkbHwHPReq2Valid
-	 * 0x04, VclkbHwHPReq1Valid
-	 * 0x08, VclkbSysClkReq1HPValid
-	 */
-	REG_INIT(AB8540_REGUVCLKBREQVALID,	0x03, 0x14, 0x0f),
-	/*
-	 * 0x01, Vrf1SwHPReqValid
-	 * 0x02, Vrf1HwHPReq2Valid
-	 * 0x04, Vrf1HwHPReq1Valid
-	 * 0x08, Vrf1SysClkReq1HPValid
-	 */
-	REG_INIT(AB8540_REGUVRF1REQVALID,	0x03, 0x15, 0x0f),
+	REG_INIT(AB8505_REGUVAUX4REQVALID,	0x03, 0x11, 0x0f),
 	/*
-	 * 0x02, VTVoutEna
-	 * 0x04, Vintcore12Ena
-	 * 0x38, Vintcore12Sel
-	 * 0x40, Vintcore12LP
-	 * 0x80, VTVoutLP
+	 * 0x02, VadcEna
+	 * 0x04, VintCore12Ena
+	 * 0x38, VintCore12Sel
+	 * 0x40, VintCore12LP
+	 * 0x80, VadcLP
 	 */
-	REG_INIT(AB8540_REGUMISC1,		0x03, 0x80, 0xfe),
+	REG_INIT(AB8505_REGUMISC1,		0x03, 0x80, 0xfe),
 	/*
 	 * 0x02, VaudioEna
 	 * 0x04, VdmicEna
 	 * 0x08, Vamic1Ena
 	 * 0x10, Vamic2Ena
-	 * 0x20, Vamic12LP
-	 * 0xC0, VdmicSel
 	 */
-	REG_INIT(AB8540_VAUDIOSUPPLY,		0x03, 0x83, 0xfe),
+	REG_INIT(AB8505_VAUDIOSUPPLY,		0x03, 0x83, 0x1e),
 	/*
 	 * 0x01, Vamic1_dzout
 	 * 0x02, Vamic2_dzout
 	 */
-	REG_INIT(AB8540_REGUCTRL1VAMIC,		0x03, 0x84, 0x03),
-	/*
-	 * 0x07, VHSICSel
-	 * 0x08, VHSICOffState
-	 * 0x10, VHSIEna
-	 * 0x20, VHSICLP
-	 */
-	REG_INIT(AB8540_VHSIC,			0x03, 0x87, 0x3f),
-	/*
-	 * 0x07, VSDIOSel
-	 * 0x08, VSDIOOffState
-	 * 0x10, VSDIOEna
-	 * 0x20, VSDIOLP
-	 */
-	REG_INIT(AB8540_VSDIO,			0x03, 0x88, 0x3f),
+	REG_INIT(AB8505_REGUCTRL1VAMIC,		0x03, 0x84, 0x03),
 	/*
-	 * 0x03, Vsmps1Regu
-	 * 0x0c, Vsmps1SelCtrl
-	 * 0x10, Vsmps1AutoMode
-	 * 0x20, Vsmps1PWMMode
+	 * 0x03, VsmpsARegu
+	 * 0x0c, VsmpsASelCtrl
+	 * 0x10, VsmpsAAutoMode
+	 * 0x20, VsmpsAPWMMode
 	 */
-	REG_INIT(AB8540_VSMPS1REGU,		0x04, 0x03, 0x3f),
+	REG_INIT(AB8505_VSMPSAREGU,		0x04, 0x03, 0x3f),
 	/*
-	 * 0x03, Vsmps2Regu
-	 * 0x0c, Vsmps2SelCtrl
-	 * 0x10, Vsmps2AutoMode
-	 * 0x20, Vsmps2PWMMode
+	 * 0x03, VsmpsBRegu
+	 * 0x0c, VsmpsBSelCtrl
+	 * 0x10, VsmpsBAutoMode
+	 * 0x20, VsmpsBPWMMode
 	 */
-	REG_INIT(AB8540_VSMPS2REGU,		0x04, 0x04, 0x3f),
+	REG_INIT(AB8505_VSMPSBREGU,		0x04, 0x04, 0x3f),
 	/*
-	 * 0x03, Vsmps3Regu
-	 * 0x0c, Vsmps3SelCtrl
-	 * 0x10, Vsmps3AutoMode
-	 * 0x20, Vsmps3PWMMode
-	 * NOTE! PRCMU register
+	 * 0x03, VsafeRegu
+	 * 0x0c, VsafeSelCtrl
+	 * 0x10, VsafeAutoMode
+	 * 0x20, VsafePWMMode
 	 */
-	REG_INIT(AB8540_VSMPS3REGU,		0x04, 0x05, 0x0f),
+	REG_INIT(AB8505_VSAFEREGU,		0x04, 0x05, 0x3f),
 	/*
-	 * 0x03, VpllRegu
+	 * 0x03, VpllRegu (NOTE! PRCMU register bits)
 	 * 0x0c, VanaRegu
 	 */
-	REG_INIT(AB8540_VPLLVANAREGU,		0x04, 0x06, 0x0f),
+	REG_INIT(AB8505_VPLLVANAREGU,		0x04, 0x06, 0x0f),
 	/*
 	 * 0x03, VextSupply1Regu
 	 * 0x0c, VextSupply2Regu
@@ -2747,128 +1428,78 @@ static struct ab8500_reg_init ab8540_reg_init[] = {
 	 * 0x40, ExtSupply2Bypass
 	 * 0x80, ExtSupply3Bypass
 	 */
-	REG_INIT(AB8540_EXTSUPPLYREGU,		0x04, 0x08, 0xff),
+	REG_INIT(AB8505_EXTSUPPLYREGU,		0x04, 0x08, 0xff),
 	/*
 	 * 0x03, Vaux1Regu
 	 * 0x0c, Vaux2Regu
 	 */
-	REG_INIT(AB8540_VAUX12REGU,		0x04, 0x09, 0x0f),
+	REG_INIT(AB8505_VAUX12REGU,		0x04, 0x09, 0x0f),
 	/*
-	 * 0x0c, VRF1Regu
-	 * 0x03, Vaux3Regu
+	 * 0x0f, Vaux3Regu
+	 */
+	REG_INIT(AB8505_VRF1VAUX3REGU,		0x04, 0x0a, 0x0f),
+	/*
+	 * 0x3f, VsmpsASel1
 	 */
-	REG_INIT(AB8540_VRF1VAUX3REGU,		0x04, 0x0a, 0x0f),
+	REG_INIT(AB8505_VSMPSASEL1,		0x04, 0x13, 0x3f),
 	/*
-	 * 0x3f, Vsmps1Sel1
+	 * 0x3f, VsmpsASel2
 	 */
-	REG_INIT(AB8540_VSMPS1SEL1,		0x04, 0x13, 0x3f),
+	REG_INIT(AB8505_VSMPSASEL2,		0x04, 0x14, 0x3f),
 	/*
-	 * 0x3f, Vsmps1Sel2
+	 * 0x3f, VsmpsASel3
 	 */
-	REG_INIT(AB8540_VSMPS1SEL2,		0x04, 0x14, 0x3f),
+	REG_INIT(AB8505_VSMPSASEL3,		0x04, 0x15, 0x3f),
 	/*
-	 * 0x3f, Vsmps1Sel3
+	 * 0x3f, VsmpsBSel1
 	 */
-	REG_INIT(AB8540_VSMPS1SEL3,		0x04, 0x15, 0x3f),
+	REG_INIT(AB8505_VSMPSBSEL1,		0x04, 0x17, 0x3f),
 	/*
-	 * 0x3f, Vsmps2Sel1
+	 * 0x3f, VsmpsBSel2
 	 */
-	REG_INIT(AB8540_VSMPS2SEL1,		0x04, 0x17, 0x3f),
+	REG_INIT(AB8505_VSMPSBSEL2,		0x04, 0x18, 0x3f),
 	/*
-	 * 0x3f, Vsmps2Sel2
+	 * 0x3f, VsmpsBSel3
 	 */
-	REG_INIT(AB8540_VSMPS2SEL2,		0x04, 0x18, 0x3f),
+	REG_INIT(AB8505_VSMPSBSEL3,		0x04, 0x19, 0x3f),
 	/*
-	 * 0x3f, Vsmps2Sel3
+	 * 0x7f, VsafeSel1
 	 */
-	REG_INIT(AB8540_VSMPS2SEL3,		0x04, 0x19, 0x3f),
+	REG_INIT(AB8505_VSAFESEL1,		0x04, 0x1b, 0x7f),
 	/*
-	 * 0x7f, Vsmps3Sel1
-	 * NOTE! PRCMU register
+	 * 0x3f, VsafeSel2
 	 */
-	REG_INIT(AB8540_VSMPS3SEL1,             0x04, 0x1b, 0x7f),
+	REG_INIT(AB8505_VSAFESEL2,		0x04, 0x1c, 0x7f),
 	/*
-	 * 0x7f, Vsmps3Sel2
-	 * NOTE! PRCMU register
+	 * 0x3f, VsafeSel3
 	 */
-	REG_INIT(AB8540_VSMPS3SEL2,             0x04, 0x1c, 0x7f),
+	REG_INIT(AB8505_VSAFESEL3,		0x04, 0x1d, 0x7f),
 	/*
 	 * 0x0f, Vaux1Sel
 	 */
-	REG_INIT(AB8540_VAUX1SEL,		0x04, 0x1f, 0x0f),
+	REG_INIT(AB8505_VAUX1SEL,		0x04, 0x1f, 0x0f),
 	/*
 	 * 0x0f, Vaux2Sel
 	 */
-	REG_INIT(AB8540_VAUX2SEL,		0x04, 0x20, 0x0f),
+	REG_INIT(AB8505_VAUX2SEL,		0x04, 0x20, 0x0f),
 	/*
 	 * 0x07, Vaux3Sel
-	 * 0x70, Vrf1Sel
-	 */
-	REG_INIT(AB8540_VRF1VAUX3SEL,		0x04, 0x21, 0x77),
-	/*
-	 * 0x01, VextSupply12LP
-	 */
-	REG_INIT(AB8540_REGUCTRL2SPARE,		0x04, 0x22, 0x01),
-	/*
-	 * 0x07, Vanasel
-	 * 0x30, Vpllsel
+	 * 0x30, VRF1Sel
 	 */
-	REG_INIT(AB8540_VANAVPLLSEL,		0x04, 0x29, 0x37),
+	REG_INIT(AB8505_VRF1VAUX3SEL,		0x04, 0x21, 0x37),
 	/*
 	 * 0x03, Vaux4RequestCtrl
 	 */
-	REG_INIT(AB8540_VAUX4REQCTRL,		0x04, 0x2d, 0x03),
+	REG_INIT(AB8505_VAUX4REQCTRL,		0x04, 0x2d, 0x03),
 	/*
 	 * 0x03, Vaux4Regu
 	 */
-	REG_INIT(AB8540_VAUX4REGU,		0x04, 0x2e, 0x03),
+	REG_INIT(AB8505_VAUX4REGU,		0x04, 0x2e, 0x03),
 	/*
 	 * 0x0f, Vaux4Sel
 	 */
-	REG_INIT(AB8540_VAUX4SEL,		0x04, 0x2f, 0x0f),
-	/*
-	 * 0x03, Vaux5RequestCtrl
-	 */
-	REG_INIT(AB8540_VAUX5REQCTRL,		0x04, 0x31, 0x03),
-	/*
-	 * 0x03, Vaux5Regu
-	 */
-	REG_INIT(AB8540_VAUX5REGU,		0x04, 0x32, 0x03),
-	/*
-	 * 0x3f, Vaux5Sel
-	 */
-	REG_INIT(AB8540_VAUX5SEL,		0x04, 0x33, 0x3f),
-	/*
-	 * 0x03, Vaux6RequestCtrl
-	 */
-	REG_INIT(AB8540_VAUX6REQCTRL,		0x04, 0x34, 0x03),
-	/*
-	 * 0x03, Vaux6Regu
-	 */
-	REG_INIT(AB8540_VAUX6REGU,		0x04, 0x35, 0x03),
-	/*
-	 * 0x3f, Vaux6Sel
-	 */
-	REG_INIT(AB8540_VAUX6SEL,		0x04, 0x36, 0x3f),
-	/*
-	 * 0x03, VCLKBRequestCtrl
-	 */
-	REG_INIT(AB8540_VCLKBREQCTRL,		0x04, 0x37, 0x03),
-	/*
-	 * 0x03, VCLKBRegu
-	 */
-	REG_INIT(AB8540_VCLKBREGU,		0x04, 0x38, 0x03),
-	/*
-	 * 0x07, VCLKBSel
-	 */
-	REG_INIT(AB8540_VCLKBSEL,		0x04, 0x39, 0x07),
-	/*
-	 * 0x03, Vrf1RequestCtrl
-	 */
-	REG_INIT(AB8540_VRF1REQCTRL,		0x04, 0x3a, 0x03),
+	REG_INIT(AB8505_VAUX4SEL,		0x04, 0x2f, 0x0f),
 	/*
-	 * 0x01, VpllDisch
-	 * 0x02, Vrf1Disch
 	 * 0x04, Vaux1Disch
 	 * 0x08, Vaux2Disch
 	 * 0x10, Vaux3Disch
@@ -2876,24 +1507,33 @@ static struct ab8500_reg_init ab8540_reg_init[] = {
 	 * 0x40, VTVoutDisch
 	 * 0x80, VaudioDisch
 	 */
-	REG_INIT(AB8540_REGUCTRLDISCH,		0x04, 0x43, 0xff),
+	REG_INIT(AB8505_REGUCTRLDISCH,		0x04, 0x43, 0xfc),
 	/*
 	 * 0x02, VanaDisch
 	 * 0x04, VdmicPullDownEna
-	 * 0x08, VpllPullDownEna
 	 * 0x10, VdmicDisch
 	 */
-	REG_INIT(AB8540_REGUCTRLDISCH2,		0x04, 0x44, 0x1e),
+	REG_INIT(AB8505_REGUCTRLDISCH2,		0x04, 0x44, 0x16),
 	/*
 	 * 0x01, Vaux4Disch
 	 */
-	REG_INIT(AB8540_REGUCTRLDISCH3,		0x04, 0x48, 0x01),
+	REG_INIT(AB8505_REGUCTRLDISCH3,		0x04, 0x48, 0x01),
+	/*
+	 * 0x07, Vaux5Sel
+	 * 0x08, Vaux5LP
+	 * 0x10, Vaux5Ena
+	 * 0x20, Vaux5Disch
+	 * 0x40, Vaux5DisSfst
+	 * 0x80, Vaux5DisPulld
+	 */
+	REG_INIT(AB8505_CTRLVAUX5,		0x01, 0x55, 0xff),
 	/*
-	 * 0x01, Vaux5Disch
-	 * 0x02, Vaux6Disch
-	 * 0x04, VCLKBDisch
+	 * 0x07, Vaux6Sel
+	 * 0x08, Vaux6LP
+	 * 0x10, Vaux6Ena
+	 * 0x80, Vaux6DisPulld
 	 */
-	REG_INIT(AB8540_REGUCTRLDISCH4,		0x04, 0x49, 0x07),
+	REG_INIT(AB8505_CTRLVAUX6,		0x01, 0x56, 0x9f),
 };
 
 static struct of_regulator_match ab8500_regulator_match[] = {
@@ -2925,37 +1565,6 @@ static struct of_regulator_match ab8505_regulator_match[] = {
 	{ .name	= "ab8500_ldo_ana",     .driver_data = (void *) AB8505_LDO_ANA, },
 };
 
-static struct of_regulator_match ab8540_regulator_match[] = {
-	{ .name	= "ab8500_ldo_aux1",    .driver_data = (void *) AB8540_LDO_AUX1, },
-	{ .name	= "ab8500_ldo_aux2",    .driver_data = (void *) AB8540_LDO_AUX2, },
-	{ .name	= "ab8500_ldo_aux3",    .driver_data = (void *) AB8540_LDO_AUX3, },
-	{ .name	= "ab8500_ldo_aux4",    .driver_data = (void *) AB8540_LDO_AUX4, },
-	{ .name	= "ab8500_ldo_aux5",    .driver_data = (void *) AB8540_LDO_AUX5, },
-	{ .name	= "ab8500_ldo_aux6",    .driver_data = (void *) AB8540_LDO_AUX6, },
-	{ .name	= "ab8500_ldo_intcore", .driver_data = (void *) AB8540_LDO_INTCORE, },
-	{ .name	= "ab8500_ldo_tvout",   .driver_data = (void *) AB8540_LDO_TVOUT, },
-	{ .name = "ab8500_ldo_audio",   .driver_data = (void *) AB8540_LDO_AUDIO, },
-	{ .name	= "ab8500_ldo_anamic1", .driver_data = (void *) AB8540_LDO_ANAMIC1, },
-	{ .name	= "ab8500_ldo_anamic2", .driver_data = (void *) AB8540_LDO_ANAMIC2, },
-	{ .name	= "ab8500_ldo_dmic",    .driver_data = (void *) AB8540_LDO_DMIC, },
-	{ .name	= "ab8500_ldo_ana",     .driver_data = (void *) AB8540_LDO_ANA, },
-	{ .name = "ab8500_ldo_sdio",    .driver_data = (void *) AB8540_LDO_SDIO, },
-};
-
-static struct of_regulator_match ab9540_regulator_match[] = {
-	{ .name	= "ab8500_ldo_aux1",    .driver_data = (void *) AB9540_LDO_AUX1, },
-	{ .name	= "ab8500_ldo_aux2",    .driver_data = (void *) AB9540_LDO_AUX2, },
-	{ .name	= "ab8500_ldo_aux3",    .driver_data = (void *) AB9540_LDO_AUX3, },
-	{ .name	= "ab8500_ldo_aux4",    .driver_data = (void *) AB9540_LDO_AUX4, },
-	{ .name	= "ab8500_ldo_intcore", .driver_data = (void *) AB9540_LDO_INTCORE, },
-	{ .name	= "ab8500_ldo_tvout",   .driver_data = (void *) AB9540_LDO_TVOUT, },
-	{ .name = "ab8500_ldo_audio",   .driver_data = (void *) AB9540_LDO_AUDIO, },
-	{ .name	= "ab8500_ldo_anamic1", .driver_data = (void *) AB9540_LDO_ANAMIC1, },
-	{ .name	= "ab8500_ldo_anamic2", .driver_data = (void *) AB9540_LDO_ANAMIC2, },
-	{ .name	= "ab8500_ldo_dmic",    .driver_data = (void *) AB9540_LDO_DMIC, },
-	{ .name	= "ab8500_ldo_ana",     .driver_data = (void *) AB9540_LDO_ANA, },
-};
-
 static struct {
 	struct ab8500_regulator_info *info;
 	int info_size;
@@ -2967,27 +1576,13 @@ static struct {
 
 static void abx500_get_regulator_info(struct ab8500 *ab8500)
 {
-	if (is_ab9540(ab8500)) {
-		abx500_regulator.info = ab9540_regulator_info;
-		abx500_regulator.info_size = ARRAY_SIZE(ab9540_regulator_info);
-		abx500_regulator.init = ab9540_reg_init;
-		abx500_regulator.init_size = AB9540_NUM_REGULATOR_REGISTERS;
-		abx500_regulator.match = ab9540_regulator_match;
-		abx500_regulator.match_size = ARRAY_SIZE(ab9540_regulator_match);
-	} else if (is_ab8505(ab8500)) {
+	if (is_ab8505(ab8500)) {
 		abx500_regulator.info = ab8505_regulator_info;
 		abx500_regulator.info_size = ARRAY_SIZE(ab8505_regulator_info);
 		abx500_regulator.init = ab8505_reg_init;
 		abx500_regulator.init_size = AB8505_NUM_REGULATOR_REGISTERS;
 		abx500_regulator.match = ab8505_regulator_match;
 		abx500_regulator.match_size = ARRAY_SIZE(ab8505_regulator_match);
-	} else if (is_ab8540(ab8500)) {
-		abx500_regulator.info = ab8540_regulator_info;
-		abx500_regulator.info_size = ARRAY_SIZE(ab8540_regulator_info);
-		abx500_regulator.init = ab8540_reg_init;
-		abx500_regulator.init_size = AB8540_NUM_REGULATOR_REGISTERS;
-		abx500_regulator.match = ab8540_regulator_match;
-		abx500_regulator.match_size = ARRAY_SIZE(ab8540_regulator_match);
 	} else {
 		abx500_regulator.info = ab8500_regulator_info;
 		abx500_regulator.info_size = ARRAY_SIZE(ab8500_regulator_info);
diff --git a/drivers/regulator/anatop-regulator.c b/drivers/regulator/anatop-regulator.c
index 7d6478e6a503..d9d8155ed8cb 100644
--- a/drivers/regulator/anatop-regulator.c
+++ b/drivers/regulator/anatop-regulator.c
@@ -1,22 +1,6 @@
-/*
- * Copyright (C) 2011 Freescale Semiconductor, Inc. All Rights Reserved.
- */
-
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (C) 2011 Freescale Semiconductor, Inc. All Rights Reserved.
 
 #include <linux/slab.h>
 #include <linux/device.h>
diff --git a/drivers/regulator/arizona-ldo1.c b/drivers/regulator/arizona-ldo1.c
index 96fddfff5dc4..f6d6a4ad9e8a 100644
--- a/drivers/regulator/arizona-ldo1.c
+++ b/drivers/regulator/arizona-ldo1.c
@@ -17,12 +17,11 @@
 #include <linux/bitops.h>
 #include <linux/err.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/driver.h>
 #include <linux/regulator/machine.h>
 #include <linux/regulator/of_regulator.h>
-#include <linux/gpio.h>
 #include <linux/slab.h>
 
 #include <linux/regulator/arizona-ldo1.h>
@@ -198,16 +197,6 @@ static int arizona_ldo1_of_get_pdata(struct arizona_ldo1_pdata *pdata,
 	struct device_node *init_node, *dcvdd_node;
 	struct regulator_init_data *init_data;
 
-	pdata->ldoena = of_get_named_gpio(np, "wlf,ldoena", 0);
-	if (pdata->ldoena < 0) {
-		dev_warn(config->dev,
-			 "LDOENA GPIO property missing/malformed: %d\n",
-			 pdata->ldoena);
-		pdata->ldoena = 0;
-	} else {
-		config->ena_gpio_initialized = true;
-	}
-
 	init_node = of_get_child_by_name(np, "ldo1");
 	dcvdd_node = of_parse_phandle(np, "DCVDD-supply", 0);
 
@@ -264,7 +253,11 @@ static int arizona_ldo1_common_init(struct platform_device *pdev,
 		}
 	}
 
-	config.ena_gpio = pdata->ldoena;
+	/* We assume that high output = regulator off */
+	config.ena_gpiod = devm_gpiod_get_optional(&pdev->dev, "wlf,ldoena",
+						   GPIOD_OUT_HIGH);
+	if (IS_ERR(config.ena_gpiod))
+		return PTR_ERR(config.ena_gpiod);
 
 	if (pdata->init_data)
 		config.init_data = pdata->init_data;
diff --git a/drivers/regulator/axp20x-regulator.c b/drivers/regulator/axp20x-regulator.c
index 181622b2813d..91b8ff8bac15 100644
--- a/drivers/regulator/axp20x-regulator.c
+++ b/drivers/regulator/axp20x-regulator.c
@@ -721,6 +721,8 @@ static int axp20x_regulator_probe(struct platform_device *pdev)
 	case AXP803_ID:
 		regulators = axp803_regulators;
 		nregulators = AXP803_REG_ID_MAX;
+		drivevbus = of_property_read_bool(pdev->dev.parent->of_node,
+						  "x-powers,drive-vbus-en");
 		break;
 	case AXP806_ID:
 		regulators = axp806_regulators;
diff --git a/drivers/regulator/bd71837-regulator.c b/drivers/regulator/bd71837-regulator.c
new file mode 100644
index 000000000000..6eae4d0432a2
--- /dev/null
+++ b/drivers/regulator/bd71837-regulator.c
@@ -0,0 +1,640 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2018 ROHM Semiconductors
+// bd71837-regulator.c ROHM BD71837MWV regulator driver
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/gpio.h>
+#include <linux/mfd/bd71837.h>
+#include <linux/regulator/of_regulator.h>
+
+struct bd71837_pmic {
+	struct regulator_desc descs[BD71837_REGULATOR_CNT];
+	struct bd71837 *mfd;
+	struct platform_device *pdev;
+	struct regulator_dev *rdev[BD71837_REGULATOR_CNT];
+};
+
+/*
+ * BUCK1/2/3/4
+ * BUCK1RAMPRATE[1:0] BUCK1 DVS ramp rate setting
+ * 00: 10.00mV/usec 10mV 1uS
+ * 01: 5.00mV/usec	10mV 2uS
+ * 10: 2.50mV/usec	10mV 4uS
+ * 11: 1.25mV/usec	10mV 8uS
+ */
+static int bd71837_buck1234_set_ramp_delay(struct regulator_dev *rdev,
+					   int ramp_delay)
+{
+	struct bd71837_pmic *pmic = rdev_get_drvdata(rdev);
+	struct bd71837 *mfd = pmic->mfd;
+	int id = rdev->desc->id;
+	unsigned int ramp_value = BUCK_RAMPRATE_10P00MV;
+
+	dev_dbg(&(pmic->pdev->dev), "Buck[%d] Set Ramp = %d\n", id + 1,
+		ramp_delay);
+	switch (ramp_delay) {
+	case 1 ... 1250:
+		ramp_value = BUCK_RAMPRATE_1P25MV;
+		break;
+	case 1251 ... 2500:
+		ramp_value = BUCK_RAMPRATE_2P50MV;
+		break;
+	case 2501 ... 5000:
+		ramp_value = BUCK_RAMPRATE_5P00MV;
+		break;
+	case 5001 ... 10000:
+		ramp_value = BUCK_RAMPRATE_10P00MV;
+		break;
+	default:
+		ramp_value = BUCK_RAMPRATE_10P00MV;
+		dev_err(&pmic->pdev->dev,
+			"%s: ramp_delay: %d not supported, setting 10000mV//us\n",
+			rdev->desc->name, ramp_delay);
+	}
+
+	return regmap_update_bits(mfd->regmap, BD71837_REG_BUCK1_CTRL + id,
+				  BUCK_RAMPRATE_MASK, ramp_value << 6);
+}
+
+/* Bucks 1 to 4 support DVS. PWM mode is used when voltage is changed.
+ * Bucks 5 to 8 and LDOs can use PFM and must be disabled when voltage
+ * is changed. Hence we return -EBUSY for these if voltage is changed
+ * when BUCK/LDO is enabled.
+ */
+static int bd71837_set_voltage_sel_restricted(struct regulator_dev *rdev,
+						    unsigned int sel)
+{
+	int ret;
+
+	ret = regulator_is_enabled_regmap(rdev);
+	if (!ret)
+		ret = regulator_set_voltage_sel_regmap(rdev, sel);
+	else if (ret == 1)
+		ret = -EBUSY;
+	return ret;
+}
+
+static struct regulator_ops bd71837_ldo_regulator_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_linear_range,
+	.set_voltage_sel = bd71837_set_voltage_sel_restricted,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+};
+
+static struct regulator_ops bd71837_ldo_regulator_nolinear_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_table,
+	.set_voltage_sel = bd71837_set_voltage_sel_restricted,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+};
+
+static struct regulator_ops bd71837_buck_regulator_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_linear_range,
+	.set_voltage_sel = bd71837_set_voltage_sel_restricted,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+};
+
+static struct regulator_ops bd71837_buck_regulator_nolinear_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_table,
+	.set_voltage_sel = bd71837_set_voltage_sel_restricted,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+};
+
+static struct regulator_ops bd71837_buck1234_regulator_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_linear_range,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+	.set_ramp_delay = bd71837_buck1234_set_ramp_delay,
+};
+
+/*
+ * BUCK1/2/3/4
+ * 0.70 to 1.30V (10mV step)
+ */
+static const struct regulator_linear_range bd71837_buck1234_voltage_ranges[] = {
+	REGULATOR_LINEAR_RANGE(700000, 0x00, 0x3C, 10000),
+	REGULATOR_LINEAR_RANGE(1300000, 0x3D, 0x3F, 0),
+};
+
+/*
+ * BUCK5
+ * 0.9V to 1.35V ()
+ */
+static const struct regulator_linear_range bd71837_buck5_voltage_ranges[] = {
+	REGULATOR_LINEAR_RANGE(700000, 0x00, 0x03, 100000),
+	REGULATOR_LINEAR_RANGE(1050000, 0x04, 0x05, 50000),
+	REGULATOR_LINEAR_RANGE(1200000, 0x06, 0x07, 150000),
+};
+
+/*
+ * BUCK6
+ * 3.0V to 3.3V (step 100mV)
+ */
+static const struct regulator_linear_range bd71837_buck6_voltage_ranges[] = {
+	REGULATOR_LINEAR_RANGE(3000000, 0x00, 0x03, 100000),
+};
+
+/*
+ * BUCK7
+ * 000 = 1.605V
+ * 001 = 1.695V
+ * 010 = 1.755V
+ * 011 = 1.8V (Initial)
+ * 100 = 1.845V
+ * 101 = 1.905V
+ * 110 = 1.95V
+ * 111 = 1.995V
+ */
+static const unsigned int buck_7_volts[] = {
+	1605000, 1695000, 1755000, 1800000, 1845000, 1905000, 1950000, 1995000
+};
+
+/*
+ * BUCK8
+ * 0.8V to 1.40V (step 10mV)
+ */
+static const struct regulator_linear_range bd71837_buck8_voltage_ranges[] = {
+	REGULATOR_LINEAR_RANGE(800000, 0x00, 0x3C, 10000),
+	REGULATOR_LINEAR_RANGE(1400000, 0x3D, 0x3F, 0),
+};
+
+/*
+ * LDO1
+ * 3.0 to 3.3V (100mV step)
+ */
+static const struct regulator_linear_range bd71837_ldo1_voltage_ranges[] = {
+	REGULATOR_LINEAR_RANGE(3000000, 0x00, 0x03, 100000),
+};
+
+/*
+ * LDO2
+ * 0.8 or 0.9V
+ */
+const unsigned int ldo_2_volts[] = {
+	900000, 800000
+};
+
+/*
+ * LDO3
+ * 1.8 to 3.3V (100mV step)
+ */
+static const struct regulator_linear_range bd71837_ldo3_voltage_ranges[] = {
+	REGULATOR_LINEAR_RANGE(1800000, 0x00, 0x0F, 100000),
+};
+
+/*
+ * LDO4
+ * 0.9 to 1.8V (100mV step)
+ */
+static const struct regulator_linear_range bd71837_ldo4_voltage_ranges[] = {
+	REGULATOR_LINEAR_RANGE(900000, 0x00, 0x09, 100000),
+	REGULATOR_LINEAR_RANGE(1800000, 0x0A, 0x0F, 0),
+};
+
+/*
+ * LDO5
+ * 1.8 to 3.3V (100mV step)
+ */
+static const struct regulator_linear_range bd71837_ldo5_voltage_ranges[] = {
+	REGULATOR_LINEAR_RANGE(1800000, 0x00, 0x0F, 100000),
+};
+
+/*
+ * LDO6
+ * 0.9 to 1.8V (100mV step)
+ */
+static const struct regulator_linear_range bd71837_ldo6_voltage_ranges[] = {
+	REGULATOR_LINEAR_RANGE(900000, 0x00, 0x09, 100000),
+	REGULATOR_LINEAR_RANGE(1800000, 0x0A, 0x0F, 0),
+};
+
+/*
+ * LDO7
+ * 1.8 to 3.3V (100mV step)
+ */
+static const struct regulator_linear_range bd71837_ldo7_voltage_ranges[] = {
+	REGULATOR_LINEAR_RANGE(1800000, 0x00, 0x0F, 100000),
+};
+
+static const struct regulator_desc bd71837_regulators[] = {
+	{
+		.name = "buck1",
+		.of_match = of_match_ptr("BUCK1"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_BUCK1,
+		.ops = &bd71837_buck1234_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_BUCK1_VOLTAGE_NUM,
+		.linear_ranges = bd71837_buck1234_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_buck1234_voltage_ranges),
+		.vsel_reg = BD71837_REG_BUCK1_VOLT_RUN,
+		.vsel_mask = BUCK1_RUN_MASK,
+		.enable_reg = BD71837_REG_BUCK1_CTRL,
+		.enable_mask = BD71837_BUCK_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "buck2",
+		.of_match = of_match_ptr("BUCK2"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_BUCK2,
+		.ops = &bd71837_buck1234_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_BUCK2_VOLTAGE_NUM,
+		.linear_ranges = bd71837_buck1234_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_buck1234_voltage_ranges),
+		.vsel_reg = BD71837_REG_BUCK2_VOLT_RUN,
+		.vsel_mask = BUCK2_RUN_MASK,
+		.enable_reg = BD71837_REG_BUCK2_CTRL,
+		.enable_mask = BD71837_BUCK_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "buck3",
+		.of_match = of_match_ptr("BUCK3"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_BUCK3,
+		.ops = &bd71837_buck1234_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_BUCK3_VOLTAGE_NUM,
+		.linear_ranges = bd71837_buck1234_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_buck1234_voltage_ranges),
+		.vsel_reg = BD71837_REG_BUCK3_VOLT_RUN,
+		.vsel_mask = BUCK3_RUN_MASK,
+		.enable_reg = BD71837_REG_BUCK3_CTRL,
+		.enable_mask = BD71837_BUCK_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "buck4",
+		.of_match = of_match_ptr("BUCK4"),
+		.regulators_node = of_match_ptr("regulators"),
+			.id = BD71837_BUCK4,
+		.ops = &bd71837_buck1234_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_BUCK4_VOLTAGE_NUM,
+		.linear_ranges = bd71837_buck1234_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_buck1234_voltage_ranges),
+		.vsel_reg = BD71837_REG_BUCK4_VOLT_RUN,
+		.vsel_mask = BUCK4_RUN_MASK,
+		.enable_reg = BD71837_REG_BUCK4_CTRL,
+		.enable_mask = BD71837_BUCK_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "buck5",
+		.of_match = of_match_ptr("BUCK5"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_BUCK5,
+		.ops = &bd71837_buck_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_BUCK5_VOLTAGE_NUM,
+		.linear_ranges = bd71837_buck5_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_buck5_voltage_ranges),
+		.vsel_reg = BD71837_REG_BUCK5_VOLT,
+		.vsel_mask = BUCK5_MASK,
+		.enable_reg = BD71837_REG_BUCK5_CTRL,
+		.enable_mask = BD71837_BUCK_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "buck6",
+		.of_match = of_match_ptr("BUCK6"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_BUCK6,
+		.ops = &bd71837_buck_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_BUCK6_VOLTAGE_NUM,
+		.linear_ranges = bd71837_buck6_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_buck6_voltage_ranges),
+		.vsel_reg = BD71837_REG_BUCK6_VOLT,
+		.vsel_mask = BUCK6_MASK,
+		.enable_reg = BD71837_REG_BUCK6_CTRL,
+		.enable_mask = BD71837_BUCK_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "buck7",
+		.of_match = of_match_ptr("BUCK7"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_BUCK7,
+		.ops = &bd71837_buck_regulator_nolinear_ops,
+		.type = REGULATOR_VOLTAGE,
+		.volt_table = &buck_7_volts[0],
+		.n_voltages = ARRAY_SIZE(buck_7_volts),
+		.vsel_reg = BD71837_REG_BUCK7_VOLT,
+		.vsel_mask = BUCK7_MASK,
+		.enable_reg = BD71837_REG_BUCK7_CTRL,
+		.enable_mask = BD71837_BUCK_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "buck8",
+		.of_match = of_match_ptr("BUCK8"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_BUCK8,
+		.ops = &bd71837_buck_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_BUCK8_VOLTAGE_NUM,
+		.linear_ranges = bd71837_buck8_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_buck8_voltage_ranges),
+		.vsel_reg = BD71837_REG_BUCK8_VOLT,
+		.vsel_mask = BUCK8_MASK,
+		.enable_reg = BD71837_REG_BUCK8_CTRL,
+		.enable_mask = BD71837_BUCK_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "ldo1",
+		.of_match = of_match_ptr("LDO1"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_LDO1,
+		.ops = &bd71837_ldo_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_LDO1_VOLTAGE_NUM,
+		.linear_ranges = bd71837_ldo1_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_ldo1_voltage_ranges),
+		.vsel_reg = BD71837_REG_LDO1_VOLT,
+		.vsel_mask = LDO1_MASK,
+		.enable_reg = BD71837_REG_LDO1_VOLT,
+		.enable_mask = BD71837_LDO_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "ldo2",
+		.of_match = of_match_ptr("LDO2"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_LDO2,
+		.ops = &bd71837_ldo_regulator_nolinear_ops,
+		.type = REGULATOR_VOLTAGE,
+		.volt_table = &ldo_2_volts[0],
+		.vsel_reg = BD71837_REG_LDO2_VOLT,
+		.vsel_mask = LDO2_MASK,
+		.n_voltages = ARRAY_SIZE(ldo_2_volts),
+		.n_voltages = BD71837_LDO2_VOLTAGE_NUM,
+		.enable_reg = BD71837_REG_LDO2_VOLT,
+		.enable_mask = BD71837_LDO_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "ldo3",
+		.of_match = of_match_ptr("LDO3"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_LDO3,
+		.ops = &bd71837_ldo_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_LDO3_VOLTAGE_NUM,
+		.linear_ranges = bd71837_ldo3_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_ldo3_voltage_ranges),
+		.vsel_reg = BD71837_REG_LDO3_VOLT,
+		.vsel_mask = LDO3_MASK,
+		.enable_reg = BD71837_REG_LDO3_VOLT,
+		.enable_mask = BD71837_LDO_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "ldo4",
+		.of_match = of_match_ptr("LDO4"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_LDO4,
+		.ops = &bd71837_ldo_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_LDO4_VOLTAGE_NUM,
+		.linear_ranges = bd71837_ldo4_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_ldo4_voltage_ranges),
+		.vsel_reg = BD71837_REG_LDO4_VOLT,
+		.vsel_mask = LDO4_MASK,
+		.enable_reg = BD71837_REG_LDO4_VOLT,
+		.enable_mask = BD71837_LDO_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "ldo5",
+		.of_match = of_match_ptr("LDO5"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_LDO5,
+		.ops = &bd71837_ldo_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_LDO5_VOLTAGE_NUM,
+		.linear_ranges = bd71837_ldo5_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_ldo5_voltage_ranges),
+		/* LDO5 is supplied by buck6 */
+		.supply_name = "buck6",
+		.vsel_reg = BD71837_REG_LDO5_VOLT,
+		.vsel_mask = LDO5_MASK,
+		.enable_reg = BD71837_REG_LDO5_VOLT,
+		.enable_mask = BD71837_LDO_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "ldo6",
+		.of_match = of_match_ptr("LDO6"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_LDO6,
+		.ops = &bd71837_ldo_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_LDO6_VOLTAGE_NUM,
+		.linear_ranges = bd71837_ldo6_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_ldo6_voltage_ranges),
+		/* LDO6 is supplied by buck7 */
+		.supply_name = "buck7",
+		.vsel_reg = BD71837_REG_LDO6_VOLT,
+		.vsel_mask = LDO6_MASK,
+		.enable_reg = BD71837_REG_LDO6_VOLT,
+		.enable_mask = BD71837_LDO_EN,
+		.owner = THIS_MODULE,
+	},
+	{
+		.name = "ldo7",
+		.of_match = of_match_ptr("LDO7"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = BD71837_LDO7,
+		.ops = &bd71837_ldo_regulator_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = BD71837_LDO7_VOLTAGE_NUM,
+		.linear_ranges = bd71837_ldo7_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(bd71837_ldo7_voltage_ranges),
+		.vsel_reg = BD71837_REG_LDO7_VOLT,
+		.vsel_mask = LDO7_MASK,
+		.enable_reg = BD71837_REG_LDO7_VOLT,
+		.enable_mask = BD71837_LDO_EN,
+		.owner = THIS_MODULE,
+	},
+};
+
+struct reg_init {
+	unsigned int reg;
+	unsigned int mask;
+};
+
+static int bd71837_probe(struct platform_device *pdev)
+{
+	struct bd71837_pmic *pmic;
+	struct bd71837_board *pdata;
+	struct regulator_config config = { 0 };
+	struct reg_init pmic_regulator_inits[] = {
+		{
+			.reg = BD71837_REG_BUCK1_CTRL,
+			.mask = BD71837_BUCK_SEL,
+		}, {
+			.reg = BD71837_REG_BUCK2_CTRL,
+			.mask = BD71837_BUCK_SEL,
+		}, {
+			.reg = BD71837_REG_BUCK3_CTRL,
+			.mask = BD71837_BUCK_SEL,
+		}, {
+			.reg = BD71837_REG_BUCK4_CTRL,
+			.mask = BD71837_BUCK_SEL,
+		}, {
+			.reg = BD71837_REG_BUCK5_CTRL,
+			.mask = BD71837_BUCK_SEL,
+		}, {
+			.reg = BD71837_REG_BUCK6_CTRL,
+			.mask = BD71837_BUCK_SEL,
+		}, {
+			.reg = BD71837_REG_BUCK7_CTRL,
+			.mask = BD71837_BUCK_SEL,
+		}, {
+			.reg = BD71837_REG_BUCK8_CTRL,
+			.mask = BD71837_BUCK_SEL,
+		}, {
+			.reg = BD71837_REG_LDO1_VOLT,
+			.mask = BD71837_LDO_SEL,
+		}, {
+			.reg = BD71837_REG_LDO2_VOLT,
+			.mask = BD71837_LDO_SEL,
+		}, {
+			.reg = BD71837_REG_LDO3_VOLT,
+			.mask = BD71837_LDO_SEL,
+		}, {
+			.reg = BD71837_REG_LDO4_VOLT,
+			.mask = BD71837_LDO_SEL,
+		}, {
+			.reg = BD71837_REG_LDO5_VOLT,
+			.mask = BD71837_LDO_SEL,
+		}, {
+			.reg = BD71837_REG_LDO6_VOLT,
+			.mask = BD71837_LDO_SEL,
+		}, {
+			.reg = BD71837_REG_LDO7_VOLT,
+			.mask = BD71837_LDO_SEL,
+		}
+	};
+
+	int i, err;
+
+	pmic = devm_kzalloc(&pdev->dev, sizeof(struct bd71837_pmic),
+			    GFP_KERNEL);
+	if (!pmic)
+		return -ENOMEM;
+
+	memcpy(pmic->descs, bd71837_regulators, sizeof(pmic->descs));
+
+	pmic->pdev = pdev;
+	pmic->mfd = dev_get_drvdata(pdev->dev.parent);
+
+	if (!pmic->mfd) {
+		dev_err(&pdev->dev, "No MFD driver data\n");
+		err = -EINVAL;
+		goto err;
+	}
+	platform_set_drvdata(pdev, pmic);
+	pdata = dev_get_platdata(pmic->mfd->dev);
+
+	/* Register LOCK release */
+	err = regmap_update_bits(pmic->mfd->regmap, BD71837_REG_REGLOCK,
+				 (REGLOCK_PWRSEQ | REGLOCK_VREG), 0);
+	if (err) {
+		dev_err(&pmic->pdev->dev, "Failed to unlock PMIC (%d)\n", err);
+		goto err;
+	} else {
+		dev_dbg(&pmic->pdev->dev, "%s: Unlocked lock register 0x%x\n",
+			__func__, BD71837_REG_REGLOCK);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(pmic_regulator_inits); i++) {
+
+		struct regulator_desc *desc;
+		struct regulator_dev *rdev;
+
+		desc = &pmic->descs[i];
+
+		if (pdata)
+			config.init_data = pdata->init_data[i];
+
+		config.dev = pdev->dev.parent;
+		config.driver_data = pmic;
+		config.regmap = pmic->mfd->regmap;
+
+		rdev = devm_regulator_register(&pdev->dev, desc, &config);
+		if (IS_ERR(rdev)) {
+			dev_err(pmic->mfd->dev,
+				"failed to register %s regulator\n",
+				desc->name);
+			err = PTR_ERR(rdev);
+			goto err;
+		}
+		/* Regulator register gets the regulator constraints and
+		 * applies them (set_machine_constraints). This should have
+		 * turned the control register(s) to correct values and we
+		 * can now switch the control from PMIC state machine to the
+		 * register interface
+		 */
+		err = regmap_update_bits(pmic->mfd->regmap,
+					 pmic_regulator_inits[i].reg,
+					 pmic_regulator_inits[i].mask,
+					 0xFFFFFFFF);
+		if (err) {
+			dev_err(&pmic->pdev->dev,
+				"Failed to write BUCK/LDO SEL bit for (%s)\n",
+				desc->name);
+			goto err;
+		}
+
+		pmic->rdev[i] = rdev;
+	}
+
+	return 0;
+
+err:
+	return err;
+}
+
+static struct platform_driver bd71837_regulator = {
+	.driver = {
+		.name = "bd71837-pmic",
+		.owner = THIS_MODULE,
+	},
+	.probe = bd71837_probe,
+};
+
+module_platform_driver(bd71837_regulator);
+
+MODULE_AUTHOR("Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>");
+MODULE_DESCRIPTION("BD71837 voltage regulator driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/bd9571mwv-regulator.c b/drivers/regulator/bd9571mwv-regulator.c
index c67a83d53c4c..be574eb444eb 100644
--- a/drivers/regulator/bd9571mwv-regulator.c
+++ b/drivers/regulator/bd9571mwv-regulator.c
@@ -24,6 +24,18 @@
 
 #include <linux/mfd/bd9571mwv.h>
 
+struct bd9571mwv_reg {
+	struct bd9571mwv *bd;
+
+	/* DDR Backup Power */
+	u8 bkup_mode_cnt_keepon;	/* from "rohm,ddr-backup-power" */
+	u8 bkup_mode_cnt_saved;
+
+	/* Power switch type */
+	bool rstbmode_level;
+	bool rstbmode_pulse;
+};
+
 enum bd9571mwv_regulators { VD09, VD18, VD25, VD33, DVFS };
 
 #define BD9571MWV_REG(_name, _of, _id, _ops, _vr, _vm, _nv, _min, _step, _lmin)\
@@ -131,14 +143,99 @@ static struct regulator_desc regulators[] = {
 		      0x80, 600000, 10000, 0x3c),
 };
 
+#ifdef CONFIG_PM_SLEEP
+static int bd9571mwv_bkup_mode_read(struct bd9571mwv *bd, unsigned int *mode)
+{
+	int ret;
+
+	ret = regmap_read(bd->regmap, BD9571MWV_BKUP_MODE_CNT, mode);
+	if (ret) {
+		dev_err(bd->dev, "failed to read backup mode (%d)\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int bd9571mwv_bkup_mode_write(struct bd9571mwv *bd, unsigned int mode)
+{
+	int ret;
+
+	ret = regmap_write(bd->regmap, BD9571MWV_BKUP_MODE_CNT, mode);
+	if (ret) {
+		dev_err(bd->dev, "failed to configure backup mode 0x%x (%d)\n",
+			mode, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int bd9571mwv_suspend(struct device *dev)
+{
+	struct bd9571mwv_reg *bdreg = dev_get_drvdata(dev);
+	unsigned int mode;
+	int ret;
+
+	if (!device_may_wakeup(dev))
+		return 0;
+
+	/* Save DDR Backup Mode */
+	ret = bd9571mwv_bkup_mode_read(bdreg->bd, &mode);
+	if (ret)
+		return ret;
+
+	bdreg->bkup_mode_cnt_saved = mode;
+
+	if (!bdreg->rstbmode_pulse)
+		return 0;
+
+	/* Enable DDR Backup Mode */
+	mode &= ~BD9571MWV_BKUP_MODE_CNT_KEEPON_MASK;
+	mode |= bdreg->bkup_mode_cnt_keepon;
+
+	if (mode != bdreg->bkup_mode_cnt_saved)
+		return bd9571mwv_bkup_mode_write(bdreg->bd, mode);
+
+	return 0;
+}
+
+static int bd9571mwv_resume(struct device *dev)
+{
+	struct bd9571mwv_reg *bdreg = dev_get_drvdata(dev);
+
+	if (!device_may_wakeup(dev))
+		return 0;
+
+	/* Restore DDR Backup Mode */
+	return bd9571mwv_bkup_mode_write(bdreg->bd, bdreg->bkup_mode_cnt_saved);
+}
+
+static const struct dev_pm_ops bd9571mwv_pm  = {
+	SET_SYSTEM_SLEEP_PM_OPS(bd9571mwv_suspend, bd9571mwv_resume)
+};
+
+#define DEV_PM_OPS	&bd9571mwv_pm
+#else
+#define DEV_PM_OPS	NULL
+#endif /* CONFIG_PM_SLEEP */
+
 static int bd9571mwv_regulator_probe(struct platform_device *pdev)
 {
 	struct bd9571mwv *bd = dev_get_drvdata(pdev->dev.parent);
 	struct regulator_config config = { };
+	struct bd9571mwv_reg *bdreg;
 	struct regulator_dev *rdev;
+	unsigned int val;
 	int i;
 
-	platform_set_drvdata(pdev, bd);
+	bdreg = devm_kzalloc(&pdev->dev, sizeof(*bdreg), GFP_KERNEL);
+	if (!bdreg)
+		return -ENOMEM;
+
+	bdreg->bd = bd;
+
+	platform_set_drvdata(pdev, bdreg);
 
 	config.dev = &pdev->dev;
 	config.dev->of_node = bd->dev->of_node;
@@ -155,6 +252,33 @@ static int bd9571mwv_regulator_probe(struct platform_device *pdev)
 		}
 	}
 
+	val = 0;
+	of_property_read_u32(bd->dev->of_node, "rohm,ddr-backup-power", &val);
+	if (val & ~BD9571MWV_BKUP_MODE_CNT_KEEPON_MASK) {
+		dev_err(bd->dev, "invalid %s mode %u\n",
+			"rohm,ddr-backup-power", val);
+		return -EINVAL;
+	}
+	bdreg->bkup_mode_cnt_keepon = val;
+
+	bdreg->rstbmode_level = of_property_read_bool(bd->dev->of_node,
+						      "rohm,rstbmode-level");
+	bdreg->rstbmode_pulse = of_property_read_bool(bd->dev->of_node,
+						      "rohm,rstbmode-pulse");
+	if (bdreg->rstbmode_level && bdreg->rstbmode_pulse) {
+		dev_err(bd->dev, "only one rohm,rstbmode-* may be specified");
+		return -EINVAL;
+	}
+
+	if (bdreg->bkup_mode_cnt_keepon) {
+		device_set_wakeup_capable(&pdev->dev, true);
+		/*
+		 * Wakeup is enabled by default in pulse mode, but needs
+		 * explicit user setup in level mode.
+		 */
+		device_set_wakeup_enable(&pdev->dev, bdreg->rstbmode_pulse);
+	}
+
 	return 0;
 }
 
@@ -167,6 +291,7 @@ MODULE_DEVICE_TABLE(platform, bd9571mwv_regulator_id_table);
 static struct platform_driver bd9571mwv_regulator_driver = {
 	.driver = {
 		.name = "bd9571mwv-regulator",
+		.pm = DEV_PM_OPS,
 	},
 	.probe = bd9571mwv_regulator_probe,
 	.id_table = bd9571mwv_regulator_id_table,
diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c
index d4803460a557..6ed568b96c0e 100644
--- a/drivers/regulator/core.c
+++ b/drivers/regulator/core.c
@@ -146,6 +146,56 @@ static inline struct regulator_dev *rdev_get_supply(struct regulator_dev *rdev)
 	return NULL;
 }
 
+/**
+ * regulator_lock_nested - lock a single regulator
+ * @rdev:		regulator source
+ * @subclass:		mutex subclass used for lockdep
+ *
+ * This function can be called many times by one task on
+ * a single regulator and its mutex will be locked only
+ * once. If a task, which is calling this function is other
+ * than the one, which initially locked the mutex, it will
+ * wait on mutex.
+ */
+static void regulator_lock_nested(struct regulator_dev *rdev,
+				  unsigned int subclass)
+{
+	if (!mutex_trylock(&rdev->mutex)) {
+		if (rdev->mutex_owner == current) {
+			rdev->ref_cnt++;
+			return;
+		}
+		mutex_lock_nested(&rdev->mutex, subclass);
+	}
+
+	rdev->ref_cnt = 1;
+	rdev->mutex_owner = current;
+}
+
+static inline void regulator_lock(struct regulator_dev *rdev)
+{
+	regulator_lock_nested(rdev, 0);
+}
+
+/**
+ * regulator_unlock - unlock a single regulator
+ * @rdev:		regulator_source
+ *
+ * This function unlocks the mutex when the
+ * reference counter reaches 0.
+ */
+static void regulator_unlock(struct regulator_dev *rdev)
+{
+	if (rdev->ref_cnt != 0) {
+		rdev->ref_cnt--;
+
+		if (!rdev->ref_cnt) {
+			rdev->mutex_owner = NULL;
+			mutex_unlock(&rdev->mutex);
+		}
+	}
+}
+
 /**
  * regulator_lock_supply - lock a regulator and its supplies
  * @rdev:         regulator source
@@ -155,7 +205,7 @@ static void regulator_lock_supply(struct regulator_dev *rdev)
 	int i;
 
 	for (i = 0; rdev; rdev = rdev_get_supply(rdev), i++)
-		mutex_lock_nested(&rdev->mutex, i);
+		regulator_lock_nested(rdev, i);
 }
 
 /**
@@ -167,7 +217,7 @@ static void regulator_unlock_supply(struct regulator_dev *rdev)
 	struct regulator *supply;
 
 	while (1) {
-		mutex_unlock(&rdev->mutex);
+		regulator_unlock(rdev);
 		supply = rdev->supply;
 
 		if (!rdev->supply)
@@ -350,9 +400,9 @@ static ssize_t regulator_uV_show(struct device *dev,
 	struct regulator_dev *rdev = dev_get_drvdata(dev);
 	ssize_t ret;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 	ret = sprintf(buf, "%d\n", _regulator_get_voltage(rdev));
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 
 	return ret;
 }
@@ -416,9 +466,9 @@ static ssize_t regulator_state_show(struct device *dev,
 	struct regulator_dev *rdev = dev_get_drvdata(dev);
 	ssize_t ret;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 	ret = regulator_print_state(buf, _regulator_is_enabled(rdev));
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 
 	return ret;
 }
@@ -526,10 +576,10 @@ static ssize_t regulator_total_uA_show(struct device *dev,
 	struct regulator *regulator;
 	int uA = 0;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 	list_for_each_entry(regulator, &rdev->consumer_list, list)
 		uA += regulator->uA_load;
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 	return sprintf(buf, "%d\n", uA);
 }
 static DEVICE_ATTR(requested_microamps, 0444, regulator_total_uA_show, NULL);
@@ -886,6 +936,18 @@ static int machine_constraints_voltage(struct regulator_dev *rdev,
 	    rdev->constraints->min_uV && rdev->constraints->max_uV) {
 		int target_min, target_max;
 		int current_uV = _regulator_get_voltage(rdev);
+
+		if (current_uV == -ENOTRECOVERABLE) {
+			/* This regulator can't be read and must be initted */
+			rdev_info(rdev, "Setting %d-%duV\n",
+				  rdev->constraints->min_uV,
+				  rdev->constraints->max_uV);
+			_regulator_do_set_voltage(rdev,
+						  rdev->constraints->min_uV,
+						  rdev->constraints->max_uV);
+			current_uV = _regulator_get_voltage(rdev);
+		}
+
 		if (current_uV < 0) {
 			rdev_err(rdev,
 				 "failed to get the current voltage(%d)\n",
@@ -1321,7 +1383,7 @@ static struct regulator *create_regulator(struct regulator_dev *rdev,
 	if (regulator == NULL)
 		return NULL;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 	regulator->rdev = rdev;
 	list_add(&regulator->list, &rdev->consumer_list);
 
@@ -1376,12 +1438,12 @@ static struct regulator *create_regulator(struct regulator_dev *rdev,
 	    _regulator_is_enabled(rdev))
 		regulator->always_on = true;
 
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 	return regulator;
 overflow_err:
 	list_del(&regulator->list);
 	kfree(regulator);
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 	return NULL;
 }
 
@@ -1770,13 +1832,13 @@ static void _regulator_put(struct regulator *regulator)
 	/* remove any sysfs entries */
 	if (regulator->dev)
 		sysfs_remove_link(&rdev->dev.kobj, regulator->supply_name);
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 	list_del(&regulator->list);
 
 	rdev->open_count--;
 	rdev->exclusive = 0;
 	put_device(&rdev->dev);
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 
 	kfree_const(regulator->supply_name);
 	kfree(regulator);
@@ -2384,7 +2446,7 @@ static void regulator_disable_work(struct work_struct *work)
 						  disable_work.work);
 	int count, i, ret;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 
 	BUG_ON(!rdev->deferred_disables);
 
@@ -2405,7 +2467,7 @@ static void regulator_disable_work(struct work_struct *work)
 			rdev_err(rdev, "Deferred disable failed: %d\n", ret);
 	}
 
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 
 	if (rdev->supply) {
 		for (i = 0; i < count; i++) {
@@ -2440,11 +2502,11 @@ int regulator_disable_deferred(struct regulator *regulator, int ms)
 	if (!ms)
 		return regulator_disable(regulator);
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 	rdev->deferred_disables++;
 	mod_delayed_work(system_power_efficient_wq, &rdev->disable_work,
 			 msecs_to_jiffies(ms));
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 
 	return 0;
 }
@@ -2476,10 +2538,10 @@ static int _regulator_list_voltage(struct regulator_dev *rdev,
 		if (selector >= rdev->desc->n_voltages)
 			return -EINVAL;
 		if (lock)
-			mutex_lock(&rdev->mutex);
+			regulator_lock(rdev);
 		ret = ops->list_voltage(rdev, selector);
 		if (lock)
-			mutex_unlock(&rdev->mutex);
+			regulator_unlock(rdev);
 	} else if (rdev->is_switch && rdev->supply) {
 		ret = _regulator_list_voltage(rdev->supply->rdev,
 					      selector, lock);
@@ -3252,7 +3314,7 @@ int regulator_sync_voltage(struct regulator *regulator)
 	struct regulator_voltage *voltage = &regulator->voltage[PM_SUSPEND_ON];
 	int ret, min_uV, max_uV;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 
 	if (!rdev->desc->ops->set_voltage &&
 	    !rdev->desc->ops->set_voltage_sel) {
@@ -3281,7 +3343,7 @@ int regulator_sync_voltage(struct regulator *regulator)
 	ret = _regulator_do_set_voltage(rdev, min_uV, max_uV);
 
 out:
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(regulator_sync_voltage);
@@ -3374,7 +3436,7 @@ int regulator_set_current_limit(struct regulator *regulator,
 	struct regulator_dev *rdev = regulator->rdev;
 	int ret;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 
 	/* sanity check */
 	if (!rdev->desc->ops->set_current_limit) {
@@ -3389,7 +3451,7 @@ int regulator_set_current_limit(struct regulator *regulator,
 
 	ret = rdev->desc->ops->set_current_limit(rdev, min_uA, max_uA);
 out:
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(regulator_set_current_limit);
@@ -3398,7 +3460,7 @@ static int _regulator_get_current_limit(struct regulator_dev *rdev)
 {
 	int ret;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 
 	/* sanity check */
 	if (!rdev->desc->ops->get_current_limit) {
@@ -3408,7 +3470,7 @@ static int _regulator_get_current_limit(struct regulator_dev *rdev)
 
 	ret = rdev->desc->ops->get_current_limit(rdev);
 out:
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 	return ret;
 }
 
@@ -3444,7 +3506,7 @@ int regulator_set_mode(struct regulator *regulator, unsigned int mode)
 	int ret;
 	int regulator_curr_mode;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 
 	/* sanity check */
 	if (!rdev->desc->ops->set_mode) {
@@ -3468,7 +3530,7 @@ int regulator_set_mode(struct regulator *regulator, unsigned int mode)
 
 	ret = rdev->desc->ops->set_mode(rdev, mode);
 out:
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(regulator_set_mode);
@@ -3477,7 +3539,7 @@ static unsigned int _regulator_get_mode(struct regulator_dev *rdev)
 {
 	int ret;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 
 	/* sanity check */
 	if (!rdev->desc->ops->get_mode) {
@@ -3487,7 +3549,7 @@ static unsigned int _regulator_get_mode(struct regulator_dev *rdev)
 
 	ret = rdev->desc->ops->get_mode(rdev);
 out:
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 	return ret;
 }
 
@@ -3508,7 +3570,7 @@ static int _regulator_get_error_flags(struct regulator_dev *rdev,
 {
 	int ret;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 
 	/* sanity check */
 	if (!rdev->desc->ops->get_error_flags) {
@@ -3518,7 +3580,7 @@ static int _regulator_get_error_flags(struct regulator_dev *rdev,
 
 	ret = rdev->desc->ops->get_error_flags(rdev, flags);
 out:
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 	return ret;
 }
 
@@ -3567,10 +3629,10 @@ int regulator_set_load(struct regulator *regulator, int uA_load)
 	struct regulator_dev *rdev = regulator->rdev;
 	int ret;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 	regulator->uA_load = uA_load;
 	ret = drms_uA_update(rdev);
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 
 	return ret;
 }
@@ -3598,7 +3660,7 @@ int regulator_allow_bypass(struct regulator *regulator, bool enable)
 	if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_BYPASS))
 		return 0;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 
 	if (enable && !regulator->bypass) {
 		rdev->bypass_count++;
@@ -3622,7 +3684,7 @@ int regulator_allow_bypass(struct regulator *regulator, bool enable)
 	if (ret == 0)
 		regulator->bypass = enable;
 
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 
 	return ret;
 }
@@ -4067,6 +4129,96 @@ static int regulator_register_resolve_supply(struct device *dev, void *data)
 	return 0;
 }
 
+static int regulator_fill_coupling_array(struct regulator_dev *rdev)
+{
+	struct coupling_desc *c_desc = &rdev->coupling_desc;
+	int n_coupled = c_desc->n_coupled;
+	struct regulator_dev *c_rdev;
+	int i;
+
+	for (i = 1; i < n_coupled; i++) {
+		/* already resolved */
+		if (c_desc->coupled_rdevs[i])
+			continue;
+
+		c_rdev = of_parse_coupled_regulator(rdev, i - 1);
+
+		if (c_rdev) {
+			c_desc->coupled_rdevs[i] = c_rdev;
+			c_desc->n_resolved++;
+		}
+	}
+
+	if (rdev->coupling_desc.n_resolved < n_coupled)
+		return -1;
+	else
+		return 0;
+}
+
+static int regulator_register_fill_coupling_array(struct device *dev,
+						  void *data)
+{
+	struct regulator_dev *rdev = dev_to_rdev(dev);
+
+	if (!IS_ENABLED(CONFIG_OF))
+		return 0;
+
+	if (regulator_fill_coupling_array(rdev))
+		rdev_dbg(rdev, "unable to resolve coupling\n");
+
+	return 0;
+}
+
+static int regulator_resolve_coupling(struct regulator_dev *rdev)
+{
+	int n_phandles;
+
+	if (!IS_ENABLED(CONFIG_OF))
+		n_phandles = 0;
+	else
+		n_phandles = of_get_n_coupled(rdev);
+
+	if (n_phandles + 1 > MAX_COUPLED) {
+		rdev_err(rdev, "too many regulators coupled\n");
+		return -EPERM;
+	}
+
+	/*
+	 * Every regulator should always have coupling descriptor filled with
+	 * at least pointer to itself.
+	 */
+	rdev->coupling_desc.coupled_rdevs[0] = rdev;
+	rdev->coupling_desc.n_coupled = n_phandles + 1;
+	rdev->coupling_desc.n_resolved++;
+
+	/* regulator isn't coupled */
+	if (n_phandles == 0)
+		return 0;
+
+	/* regulator, which can't change its voltage, can't be coupled */
+	if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE)) {
+		rdev_err(rdev, "voltage operation not allowed\n");
+		return -EPERM;
+	}
+
+	if (rdev->constraints->max_spread <= 0) {
+		rdev_err(rdev, "wrong max_spread value\n");
+		return -EPERM;
+	}
+
+	if (!of_check_coupling_data(rdev))
+		return -EPERM;
+
+	/*
+	 * After everything has been checked, try to fill rdevs array
+	 * with pointers to regulators parsed from device tree. If some
+	 * regulators are not registered yet, retry in late init call
+	 */
+	regulator_fill_coupling_array(rdev);
+
+	return 0;
+}
+
 /**
  * regulator_register - register regulator
  * @regulator_desc: regulator to register
@@ -4200,6 +4352,13 @@ regulator_register(const struct regulator_desc *regulator_desc,
 	if (ret < 0)
 		goto wash;
 
+	mutex_lock(&regulator_list_mutex);
+	ret = regulator_resolve_coupling(rdev);
+	mutex_unlock(&regulator_list_mutex);
+
+	if (ret != 0)
+		goto wash;
+
 	/* add consumers devices */
 	if (init_data) {
 		mutex_lock(&regulator_list_mutex);
@@ -4288,9 +4447,9 @@ static int _regulator_suspend_late(struct device *dev, void *data)
 	suspend_state_t *state = data;
 	int ret;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 	ret = suspend_set_state(rdev, *state);
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 
 	return ret;
 }
@@ -4320,14 +4479,14 @@ static int _regulator_resume_early(struct device *dev, void *data)
 	if (rstate == NULL)
 		return 0;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 
 	if (rdev->desc->ops->resume_early &&
 	    (rstate->enabled == ENABLE_IN_SUSPEND ||
 	     rstate->enabled == DISABLE_IN_SUSPEND))
 		ret = rdev->desc->ops->resume_early(rdev);
 
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 
 	return ret;
 }
@@ -4629,7 +4788,7 @@ static int __init regulator_late_cleanup(struct device *dev, void *data)
 	if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_STATUS))
 		return 0;
 
-	mutex_lock(&rdev->mutex);
+	regulator_lock(rdev);
 
 	if (rdev->use_count)
 		goto unlock;
@@ -4660,7 +4819,7 @@ static int __init regulator_late_cleanup(struct device *dev, void *data)
 	}
 
 unlock:
-	mutex_unlock(&rdev->mutex);
+	regulator_unlock(rdev);
 
 	return 0;
 }
@@ -4694,6 +4853,9 @@ static int __init regulator_init_complete(void)
 	class_for_each_device(&regulator_class, NULL, NULL,
 			      regulator_late_cleanup);
 
+	class_for_each_device(&regulator_class, NULL, NULL,
+			      regulator_register_fill_coupling_array);
+
 	return 0;
 }
 late_initcall_sync(regulator_init_complete);
diff --git a/drivers/regulator/cpcap-regulator.c b/drivers/regulator/cpcap-regulator.c
index f541b80f1b54..bd910fe123d9 100644
--- a/drivers/regulator/cpcap-regulator.c
+++ b/drivers/regulator/cpcap-regulator.c
@@ -222,7 +222,7 @@ static unsigned int cpcap_map_mode(unsigned int mode)
 	case CPCAP_BIT_AUDIO_LOW_PWR:
 		return REGULATOR_MODE_STANDBY;
 	default:
-		return -EINVAL;
+		return REGULATOR_MODE_INVALID;
 	}
 }
 
diff --git a/drivers/regulator/fixed-helper.c b/drivers/regulator/fixed-helper.c
index 777fac6fb4cb..2c6098e6f4bc 100644
--- a/drivers/regulator/fixed-helper.c
+++ b/drivers/regulator/fixed-helper.c
@@ -43,7 +43,6 @@ struct platform_device *regulator_register_always_on(int id, const char *name,
 	}
 
 	data->cfg.microvolts = uv;
-	data->cfg.gpio = -EINVAL;
 	data->cfg.enabled_at_boot = 1;
 	data->cfg.init_data = &data->init_data;
 
diff --git a/drivers/regulator/fixed.c b/drivers/regulator/fixed.c
index 988a7472c2ab..1142f195529b 100644
--- a/drivers/regulator/fixed.c
+++ b/drivers/regulator/fixed.c
@@ -24,10 +24,9 @@
 #include <linux/platform_device.h>
 #include <linux/regulator/driver.h>
 #include <linux/regulator/fixed.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/slab.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/regulator/of_regulator.h>
 #include <linux/regulator/machine.h>
 
@@ -78,10 +77,6 @@ of_get_fixed_voltage_config(struct device *dev,
 	if (init_data->constraints.boot_on)
 		config->enabled_at_boot = true;
 
-	config->gpio = of_get_named_gpio(np, "gpio", 0);
-	if ((config->gpio < 0) && (config->gpio != -ENOENT))
-		return ERR_PTR(config->gpio);
-
 	of_property_read_u32(np, "startup-delay-us", &config->startup_delay);
 
 	config->enable_high = of_property_read_bool(np, "enable-active-high");
@@ -102,6 +97,7 @@ static int reg_fixed_voltage_probe(struct platform_device *pdev)
 	struct fixed_voltage_config *config;
 	struct fixed_voltage_data *drvdata;
 	struct regulator_config cfg = { };
+	enum gpiod_flags gflags;
 	int ret;
 
 	drvdata = devm_kzalloc(&pdev->dev, sizeof(struct fixed_voltage_data),
@@ -150,25 +146,28 @@ static int reg_fixed_voltage_probe(struct platform_device *pdev)
 
 	drvdata->desc.fixed_uV = config->microvolts;
 
-	if (gpio_is_valid(config->gpio)) {
-		cfg.ena_gpio = config->gpio;
-		if (pdev->dev.of_node)
-			cfg.ena_gpio_initialized = true;
-	}
 	cfg.ena_gpio_invert = !config->enable_high;
 	if (config->enabled_at_boot) {
 		if (config->enable_high)
-			cfg.ena_gpio_flags |= GPIOF_OUT_INIT_HIGH;
+			gflags = GPIOD_OUT_HIGH;
 		else
-			cfg.ena_gpio_flags |= GPIOF_OUT_INIT_LOW;
+			gflags = GPIOD_OUT_LOW;
 	} else {
 		if (config->enable_high)
-			cfg.ena_gpio_flags |= GPIOF_OUT_INIT_LOW;
+			gflags = GPIOD_OUT_LOW;
 		else
-			cfg.ena_gpio_flags |= GPIOF_OUT_INIT_HIGH;
+			gflags = GPIOD_OUT_HIGH;
 	}
-	if (config->gpio_is_open_drain)
-		cfg.ena_gpio_flags |= GPIOF_OPEN_DRAIN;
+	if (config->gpio_is_open_drain) {
+		if (gflags == GPIOD_OUT_HIGH)
+			gflags = GPIOD_OUT_HIGH_OPEN_DRAIN;
+		else
+			gflags = GPIOD_OUT_LOW_OPEN_DRAIN;
+	}
+
+	cfg.ena_gpiod = devm_gpiod_get_optional(&pdev->dev, NULL, gflags);
+	if (IS_ERR(cfg.ena_gpiod))
+		return PTR_ERR(cfg.ena_gpiod);
 
 	cfg.dev = &pdev->dev;
 	cfg.init_data = config->init_data;
diff --git a/drivers/regulator/gpio-regulator.c b/drivers/regulator/gpio-regulator.c
index a86b8997bb54..9d6094c4d71c 100644
--- a/drivers/regulator/gpio-regulator.c
+++ b/drivers/regulator/gpio-regulator.c
@@ -31,6 +31,7 @@
 #include <linux/regulator/of_regulator.h>
 #include <linux/regulator/gpio-regulator.h>
 #include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/slab.h>
 #include <linux/of.h>
 #include <linux/of_gpio.h>
@@ -161,10 +162,6 @@ of_get_gpio_regulator_config(struct device *dev, struct device_node *np,
 
 	of_property_read_u32(np, "startup-delay-us", &config->startup_delay);
 
-	config->enable_gpio = of_get_named_gpio(np, "enable-gpio", 0);
-	if (config->enable_gpio < 0 && config->enable_gpio != -ENOENT)
-		return ERR_PTR(config->enable_gpio);
-
 	/* Fetch GPIOs. - optional property*/
 	ret = of_gpio_count(np);
 	if ((ret < 0) && (ret != -ENOENT))
@@ -255,6 +252,7 @@ static int gpio_regulator_probe(struct platform_device *pdev)
 	struct device_node *np = pdev->dev.of_node;
 	struct gpio_regulator_data *drvdata;
 	struct regulator_config cfg = { };
+	enum gpiod_flags gflags;
 	int ptr, ret, state;
 
 	drvdata = devm_kzalloc(&pdev->dev, sizeof(struct gpio_regulator_data),
@@ -340,21 +338,22 @@ static int gpio_regulator_probe(struct platform_device *pdev)
 	cfg.driver_data = drvdata;
 	cfg.of_node = np;
 
-	if (gpio_is_valid(config->enable_gpio)) {
-		cfg.ena_gpio = config->enable_gpio;
-		cfg.ena_gpio_initialized = true;
-	}
 	cfg.ena_gpio_invert = !config->enable_high;
 	if (config->enabled_at_boot) {
 		if (config->enable_high)
-			cfg.ena_gpio_flags |= GPIOF_OUT_INIT_HIGH;
+			gflags = GPIOD_OUT_HIGH;
 		else
-			cfg.ena_gpio_flags |= GPIOF_OUT_INIT_LOW;
+			gflags = GPIOD_OUT_LOW;
 	} else {
 		if (config->enable_high)
-			cfg.ena_gpio_flags |= GPIOF_OUT_INIT_LOW;
+			gflags = GPIOD_OUT_LOW;
 		else
-			cfg.ena_gpio_flags |= GPIOF_OUT_INIT_HIGH;
+			gflags = GPIOD_OUT_HIGH;
+	}
+	cfg.ena_gpiod = devm_gpiod_get_optional(&pdev->dev, "enable", gflags);
+	if (IS_ERR(cfg.ena_gpiod)) {
+		ret = PTR_ERR(cfg.ena_gpiod);
+		goto err_stategpio;
 	}
 
 	drvdata->dev = regulator_register(&drvdata->desc, &cfg);
diff --git a/drivers/regulator/internal.h b/drivers/regulator/internal.h
index abfd56e8c78a..943926a156f2 100644
--- a/drivers/regulator/internal.h
+++ b/drivers/regulator/internal.h
@@ -56,14 +56,27 @@ static inline struct regulator_dev *dev_to_rdev(struct device *dev)
 	return container_of(dev, struct regulator_dev, dev);
 }
 
-struct regulator_dev *of_find_regulator_by_node(struct device_node *np);
-
 #ifdef CONFIG_OF
+struct regulator_dev *of_find_regulator_by_node(struct device_node *np);
 struct regulator_init_data *regulator_of_get_init_data(struct device *dev,
 			         const struct regulator_desc *desc,
 				 struct regulator_config *config,
 				 struct device_node **node);
+
+struct regulator_dev *of_parse_coupled_regulator(struct regulator_dev *rdev,
+						 int index);
+
+int of_get_n_coupled(struct regulator_dev *rdev);
+
+bool of_check_coupling_data(struct regulator_dev *rdev);
+
 #else
+static inline struct regulator_dev *
+of_find_regulator_by_node(struct device_node *np)
+{
+	return NULL;
+}
+
 static inline struct regulator_init_data *
 regulator_of_get_init_data(struct device *dev,
 			   const struct regulator_desc *desc,
@@ -72,8 +85,25 @@ regulator_of_get_init_data(struct device *dev,
 {
 	return NULL;
 }
-#endif
 
+static inline struct regulator_dev *
+of_parse_coupled_regulator(struct regulator_dev *rdev,
+			   int index)
+{
+	return NULL;
+}
+
+static inline int of_get_n_coupled(struct regulator_dev *rdev)
+{
+	return 0;
+}
+
+static inline bool of_check_coupling_data(struct regulator_dev *rdev)
+{
+	return false;
+}
+
+#endif
 enum regulator_get_type {
 	NORMAL_GET,
 	EXCLUSIVE_GET,
@@ -83,5 +113,4 @@ enum regulator_get_type {
 
 struct regulator *_regulator_get(struct device *dev, const char *id,
 				 enum regulator_get_type get_type);
-
 #endif
diff --git a/drivers/regulator/lm363x-regulator.c b/drivers/regulator/lm363x-regulator.c
index ce5f7d9ad475..b615a413ca9f 100644
--- a/drivers/regulator/lm363x-regulator.c
+++ b/drivers/regulator/lm363x-regulator.c
@@ -16,7 +16,7 @@
 #include <linux/mfd/ti-lmu-register.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/driver.h>
 #include <linux/regulator/of_regulator.h>
@@ -219,7 +219,7 @@ static const struct regulator_desc lm363x_regulator_desc[] = {
 	},
 };
 
-static int lm363x_regulator_of_get_enable_gpio(struct device_node *np, int id)
+static struct gpio_desc *lm363x_regulator_of_get_enable_gpio(struct device *dev, int id)
 {
 	/*
 	 * Check LCM_EN1/2_GPIO is configured.
@@ -227,11 +227,11 @@ static int lm363x_regulator_of_get_enable_gpio(struct device_node *np, int id)
 	 */
 	switch (id) {
 	case LM3632_LDO_POS:
-		return of_get_named_gpio(np, "enable-gpios", 0);
+		return devm_gpiod_get_index_optional(dev, "enable", 0, GPIOD_OUT_LOW);
 	case LM3632_LDO_NEG:
-		return of_get_named_gpio(np, "enable-gpios", 1);
+		return devm_gpiod_get_index_optional(dev, "enable", 1, GPIOD_OUT_LOW);
 	default:
-		return -EINVAL;
+		return NULL;
 	}
 }
 
@@ -243,7 +243,8 @@ static int lm363x_regulator_probe(struct platform_device *pdev)
 	struct regulator_dev *rdev;
 	struct device *dev = &pdev->dev;
 	int id = pdev->id;
-	int ret, ena_gpio;
+	struct gpio_desc *gpiod;
+	int ret;
 
 	cfg.dev = dev;
 	cfg.regmap = regmap;
@@ -252,10 +253,9 @@ static int lm363x_regulator_probe(struct platform_device *pdev)
 	 * LM3632 LDOs can be controlled by external pin.
 	 * Register update is required if the pin is used.
 	 */
-	ena_gpio = lm363x_regulator_of_get_enable_gpio(dev->of_node, id);
-	if (gpio_is_valid(ena_gpio)) {
-		cfg.ena_gpio = ena_gpio;
-		cfg.ena_gpio_flags = GPIOF_OUT_INIT_LOW;
+	gpiod = lm363x_regulator_of_get_enable_gpio(dev, id);
+	if (gpiod) {
+		cfg.ena_gpiod = gpiod;
 
 		ret = regmap_update_bits(regmap, LM3632_REG_BIAS_CONFIG,
 					 LM3632_EXT_EN_MASK,
diff --git a/drivers/regulator/lp87565-regulator.c b/drivers/regulator/lp87565-regulator.c
index cfdbe294fb6a..c192357d1dea 100644
--- a/drivers/regulator/lp87565-regulator.c
+++ b/drivers/regulator/lp87565-regulator.c
@@ -95,6 +95,10 @@ static int lp87565_buck_set_ramp_delay(struct regulator_dev *rdev,
 
 	rdev->constraints->ramp_delay = lp87565_buck_ramp_delay[reg];
 
+	/* Conservatively give a 15% margin */
+	rdev->constraints->ramp_delay =
+				rdev->constraints->ramp_delay * 85 / 100;
+
 	return 0;
 }
 
@@ -154,32 +158,33 @@ static const struct lp87565_regulator regulators[] = {
 	LP87565_REGULATOR("BUCK0", LP87565_BUCK_0, "buck0", lp87565_buck_ops,
 			  256, LP87565_REG_BUCK0_VOUT, LP87565_BUCK_VSET,
 			  LP87565_REG_BUCK0_CTRL_1,
-			  LP87565_BUCK_CTRL_1_EN, 3800,
+			  LP87565_BUCK_CTRL_1_EN, 3230,
 			  buck0_1_2_3_ranges, LP87565_REG_BUCK0_CTRL_2),
 	LP87565_REGULATOR("BUCK1", LP87565_BUCK_1, "buck1", lp87565_buck_ops,
 			  256, LP87565_REG_BUCK1_VOUT, LP87565_BUCK_VSET,
 			  LP87565_REG_BUCK1_CTRL_1,
-			  LP87565_BUCK_CTRL_1_EN, 3800,
+			  LP87565_BUCK_CTRL_1_EN, 3230,
 			  buck0_1_2_3_ranges, LP87565_REG_BUCK1_CTRL_2),
 	LP87565_REGULATOR("BUCK2", LP87565_BUCK_2, "buck2", lp87565_buck_ops,
 			  256, LP87565_REG_BUCK2_VOUT, LP87565_BUCK_VSET,
 			  LP87565_REG_BUCK2_CTRL_1,
-			  LP87565_BUCK_CTRL_1_EN, 3800,
+			  LP87565_BUCK_CTRL_1_EN, 3230,
 			  buck0_1_2_3_ranges, LP87565_REG_BUCK2_CTRL_2),
 	LP87565_REGULATOR("BUCK3", LP87565_BUCK_3, "buck3", lp87565_buck_ops,
 			  256, LP87565_REG_BUCK3_VOUT, LP87565_BUCK_VSET,
 			  LP87565_REG_BUCK3_CTRL_1,
-			  LP87565_BUCK_CTRL_1_EN, 3800,
+			  LP87565_BUCK_CTRL_1_EN, 3230,
 			  buck0_1_2_3_ranges, LP87565_REG_BUCK3_CTRL_2),
 	LP87565_REGULATOR("BUCK10", LP87565_BUCK_10, "buck10", lp87565_buck_ops,
 			  256, LP87565_REG_BUCK0_VOUT, LP87565_BUCK_VSET,
 			  LP87565_REG_BUCK0_CTRL_1,
-			  LP87565_BUCK_CTRL_1_EN, 3800,
+			  LP87565_BUCK_CTRL_1_EN |
+			  LP87565_BUCK_CTRL_1_FPWM_MP_0_2, 3230,
 			  buck0_1_2_3_ranges, LP87565_REG_BUCK0_CTRL_2),
 	LP87565_REGULATOR("BUCK23", LP87565_BUCK_23, "buck23", lp87565_buck_ops,
 			  256, LP87565_REG_BUCK2_VOUT, LP87565_BUCK_VSET,
 			  LP87565_REG_BUCK2_CTRL_1,
-			  LP87565_BUCK_CTRL_1_EN, 3800,
+			  LP87565_BUCK_CTRL_1_EN, 3230,
 			  buck0_1_2_3_ranges, LP87565_REG_BUCK2_CTRL_2),
 };
 
diff --git a/drivers/regulator/lp8788-ldo.c b/drivers/regulator/lp8788-ldo.c
index cbfd35873575..f2347474a106 100644
--- a/drivers/regulator/lp8788-ldo.c
+++ b/drivers/regulator/lp8788-ldo.c
@@ -16,7 +16,7 @@
 #include <linux/err.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/driver.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/mfd/lp8788.h>
 
 /* register address */
@@ -85,8 +85,6 @@
 #define LP8788_STARTUP_TIME_S		3
 
 #define ENABLE_TIME_USEC		32
-#define ENABLE				GPIOF_OUT_INIT_HIGH
-#define DISABLE				GPIOF_OUT_INIT_LOW
 
 enum lp8788_ldo_id {
 	DLDO1,
@@ -117,7 +115,7 @@ struct lp8788_ldo {
 	struct lp8788 *lp;
 	struct regulator_desc *desc;
 	struct regulator_dev *regulator;
-	struct lp8788_ldo_enable_pin *en_pin;
+	struct gpio_desc *ena_gpiod;
 };
 
 /* DLDO 1, 2, 3, 9 voltage table */
@@ -469,7 +467,6 @@ static int lp8788_config_ldo_enable_mode(struct platform_device *pdev,
 					enum lp8788_ldo_id id)
 {
 	struct lp8788 *lp = ldo->lp;
-	struct lp8788_platform_data *pdata = lp->pdata;
 	enum lp8788_ext_ldo_en_id enable_id;
 	u8 en_mask[] = {
 		[EN_ALDO1]   = LP8788_EN_SEL_ALDO1_M,
@@ -504,11 +501,18 @@ static int lp8788_config_ldo_enable_mode(struct platform_device *pdev,
 		return 0;
 	}
 
-	/* if no platform data for ldo pin, then set default enable mode */
-	if (!pdata || !pdata->ldo_pin || !pdata->ldo_pin[enable_id])
+	/* FIXME: check default mode for GPIO here: high or low? */
+	ldo->ena_gpiod = devm_gpiod_get_index_optional(&pdev->dev,
+						       "enable",
+						       enable_id,
+						       GPIOD_OUT_HIGH);
+	if (IS_ERR(ldo->ena_gpiod))
+		return PTR_ERR(ldo->ena_gpiod);
+
+	/* if no GPIO for ldo pin, then set default enable mode */
+	if (!ldo->ena_gpiod)
 		goto set_default_ldo_enable_mode;
 
-	ldo->en_pin = pdata->ldo_pin[enable_id];
 	return 0;
 
 set_default_ldo_enable_mode:
@@ -533,10 +537,8 @@ static int lp8788_dldo_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	if (ldo->en_pin) {
-		cfg.ena_gpio = ldo->en_pin->gpio;
-		cfg.ena_gpio_flags = ldo->en_pin->init_state;
-	}
+	if (ldo->ena_gpiod)
+		cfg.ena_gpiod = ldo->ena_gpiod;
 
 	cfg.dev = pdev->dev.parent;
 	cfg.init_data = lp->pdata ? lp->pdata->dldo_data[id] : NULL;
@@ -582,10 +584,8 @@ static int lp8788_aldo_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	if (ldo->en_pin) {
-		cfg.ena_gpio = ldo->en_pin->gpio;
-		cfg.ena_gpio_flags = ldo->en_pin->init_state;
-	}
+	if (ldo->ena_gpiod)
+		cfg.ena_gpiod = ldo->ena_gpiod;
 
 	cfg.dev = pdev->dev.parent;
 	cfg.init_data = lp->pdata ? lp->pdata->aldo_data[id] : NULL;
diff --git a/drivers/regulator/ltc3676.c b/drivers/regulator/ltc3676.c
index 662ee05ea44d..9dec1609ff66 100644
--- a/drivers/regulator/ltc3676.c
+++ b/drivers/regulator/ltc3676.c
@@ -52,6 +52,7 @@
 #define LTC3676_CLIRQ     0x1F
 
 #define LTC3676_DVBxA_REF_SELECT	BIT(5)
+#define LTC3676_DVBxB_PGOOD_MASK	BIT(5)
 
 #define LTC3676_IRQSTAT_PGOOD_TIMEOUT	BIT(3)
 #define LTC3676_IRQSTAT_UNDERVOLT_WARN	BIT(4)
@@ -123,6 +124,23 @@ static int ltc3676_set_suspend_mode(struct regulator_dev *rdev,
 				  mask, val);
 }
 
+static int ltc3676_set_voltage_sel(struct regulator_dev *rdev, unsigned selector)
+{
+	struct ltc3676 *ltc3676 = rdev_get_drvdata(rdev);
+	struct device *dev = ltc3676->dev;
+	int ret, dcdc = rdev_get_id(rdev);
+
+	dev_dbg(dev, "%s id=%d selector=%d\n", __func__, dcdc, selector);
+
+	ret = regmap_update_bits(ltc3676->regmap, rdev->desc->vsel_reg + 1,
+				 LTC3676_DVBxB_PGOOD_MASK,
+				 LTC3676_DVBxB_PGOOD_MASK);
+	if (ret)
+		return ret;
+
+	return regulator_set_voltage_sel_regmap(rdev, selector);
+}
+
 static inline unsigned int ltc3676_scale(unsigned int uV, u32 r1, u32 r2)
 {
 	uint64_t tmp;
@@ -166,7 +184,7 @@ static const struct regulator_ops ltc3676_linear_regulator_ops = {
 	.disable = regulator_disable_regmap,
 	.is_enabled = regulator_is_enabled_regmap,
 	.list_voltage = regulator_list_voltage_linear,
-	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.set_voltage_sel = ltc3676_set_voltage_sel,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
 	.set_suspend_voltage = ltc3676_set_suspend_voltage,
 	.set_suspend_mode = ltc3676_set_suspend_mode,
diff --git a/drivers/regulator/max8952.c b/drivers/regulator/max8952.c
index 1096546c05e9..f1e77ed5dfec 100644
--- a/drivers/regulator/max8952.c
+++ b/drivers/regulator/max8952.c
@@ -27,6 +27,7 @@
 #include <linux/regulator/driver.h>
 #include <linux/regulator/max8952.h>
 #include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/io.h>
 #include <linux/of.h>
 #include <linux/of_gpio.h>
@@ -148,7 +149,6 @@ static struct max8952_platform_data *max8952_parse_dt(struct device *dev)
 
 	pd->gpio_vid0 = of_get_named_gpio(np, "max8952,vid-gpios", 0);
 	pd->gpio_vid1 = of_get_named_gpio(np, "max8952,vid-gpios", 1);
-	pd->gpio_en = of_get_named_gpio(np, "max8952,en-gpio", 0);
 
 	if (of_property_read_u32(np, "max8952,default-mode", &pd->default_mode))
 		dev_warn(dev, "Default mode not specified, assuming 0\n");
@@ -197,6 +197,8 @@ static int max8952_pmic_probe(struct i2c_client *client,
 	struct regulator_config config = { };
 	struct max8952_data *max8952;
 	struct regulator_dev *rdev;
+	struct gpio_desc *gpiod;
+	enum gpiod_flags gflags;
 
 	int ret = 0, err = 0;
 
@@ -224,11 +226,17 @@ static int max8952_pmic_probe(struct i2c_client *client,
 	config.driver_data = max8952;
 	config.of_node = client->dev.of_node;
 
-	config.ena_gpio = pdata->gpio_en;
-	if (client->dev.of_node)
-		config.ena_gpio_initialized = true;
 	if (pdata->reg_data->constraints.boot_on)
-		config.ena_gpio_flags |= GPIOF_OUT_INIT_HIGH;
+		gflags = GPIOD_OUT_HIGH;
+	else
+		gflags = GPIOD_OUT_LOW;
+	gpiod = devm_gpiod_get_optional(&client->dev,
+					"max8952,en",
+					gflags);
+	if (IS_ERR(gpiod))
+		return PTR_ERR(gpiod);
+	if (gpiod)
+		config.ena_gpiod = gpiod;
 
 	rdev = devm_regulator_register(&client->dev, &regulator, &config);
 	if (IS_ERR(rdev)) {
diff --git a/drivers/regulator/max8973-regulator.c b/drivers/regulator/max8973-regulator.c
index e0c747aa9f85..7cd493ec6315 100644
--- a/drivers/regulator/max8973-regulator.c
+++ b/drivers/regulator/max8973-regulator.c
@@ -34,6 +34,7 @@
 #include <linux/regulator/max8973-regulator.h>
 #include <linux/regulator/of_regulator.h>
 #include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/of_gpio.h>
 #include <linux/i2c.h>
 #include <linux/slab.h>
@@ -114,7 +115,6 @@ struct max8973_chip {
 	struct regulator_desc desc;
 	struct regmap *regmap;
 	bool enable_external_control;
-	int enable_gpio;
 	int dvs_gpio;
 	int lru_index[MAX8973_MAX_VOUT_REG];
 	int curr_vout_val[MAX8973_MAX_VOUT_REG];
@@ -567,7 +567,6 @@ static struct max8973_regulator_platform_data *max8973_parse_dt(
 
 	pdata->enable_ext_control = of_property_read_bool(np,
 						"maxim,externally-enable");
-	pdata->enable_gpio = of_get_named_gpio(np, "maxim,enable-gpio", 0);
 	pdata->dvs_gpio = of_get_named_gpio(np, "maxim,dvs-gpio", 0);
 
 	ret = of_property_read_u32(np, "maxim,dvs-default-state", &pval);
@@ -633,6 +632,8 @@ static int max8973_probe(struct i2c_client *client,
 	struct max8973_chip *max;
 	bool pdata_from_dt = false;
 	unsigned int chip_id;
+	struct gpio_desc *gpiod;
+	enum gpiod_flags gflags;
 	int ret;
 
 	pdata = dev_get_platdata(&client->dev);
@@ -647,8 +648,7 @@ static int max8973_probe(struct i2c_client *client,
 		return -EIO;
 	}
 
-	if ((pdata->dvs_gpio == -EPROBE_DEFER) ||
-		(pdata->enable_gpio == -EPROBE_DEFER))
+	if (pdata->dvs_gpio == -EPROBE_DEFER)
 		return -EPROBE_DEFER;
 
 	max = devm_kzalloc(&client->dev, sizeof(*max), GFP_KERNEL);
@@ -696,15 +696,11 @@ static int max8973_probe(struct i2c_client *client,
 	max->desc.n_voltages = MAX8973_BUCK_N_VOLTAGE;
 
 	max->dvs_gpio = (pdata->dvs_gpio) ? pdata->dvs_gpio : -EINVAL;
-	max->enable_gpio = (pdata->enable_gpio) ? pdata->enable_gpio : -EINVAL;
 	max->enable_external_control = pdata->enable_ext_control;
 	max->curr_gpio_val = pdata->dvs_def_state;
 	max->curr_vout_reg = MAX8973_VOUT + pdata->dvs_def_state;
 	max->junction_temp_warning = pdata->junction_temp_warning;
 
-	if (gpio_is_valid(max->enable_gpio))
-		max->enable_external_control = true;
-
 	max->lru_index[0] = max->curr_vout_reg;
 
 	if (gpio_is_valid(max->dvs_gpio)) {
@@ -757,27 +753,35 @@ static int max8973_probe(struct i2c_client *client,
 			break;
 		}
 
-		if (gpio_is_valid(max->enable_gpio)) {
-			config.ena_gpio_flags = GPIOF_OUT_INIT_LOW;
-			if (ridata && (ridata->constraints.always_on ||
-					ridata->constraints.boot_on))
-				config.ena_gpio_flags = GPIOF_OUT_INIT_HIGH;
-			config.ena_gpio = max->enable_gpio;
+		if (ridata && (ridata->constraints.always_on ||
+			       ridata->constraints.boot_on))
+			gflags = GPIOD_OUT_HIGH;
+		else
+			gflags = GPIOD_OUT_LOW;
+		gpiod = devm_gpiod_get_optional(&client->dev,
+						"maxim,enable",
+						gflags);
+		if (IS_ERR(gpiod))
+			return PTR_ERR(gpiod);
+		if (gpiod) {
+			config.ena_gpiod = gpiod;
+			max->enable_external_control = true;
 		}
+
 		break;
 
 	case MAX77621:
-		if (gpio_is_valid(max->enable_gpio)) {
-			ret = devm_gpio_request_one(&client->dev,
-					max->enable_gpio, GPIOF_OUT_INIT_HIGH,
-					"max8973-en-gpio");
-			if (ret) {
-				dev_err(&client->dev,
-					"gpio_request for gpio %d failed: %d\n",
-					max->enable_gpio, ret);
-				return ret;
-			}
-		}
+		/*
+		 * We do not let the core switch this regulator on/off,
+		 * we just leave it on.
+		 */
+		gpiod = devm_gpiod_get_optional(&client->dev,
+						"maxim,enable",
+						GPIOD_OUT_HIGH);
+		if (IS_ERR(gpiod))
+			return PTR_ERR(gpiod);
+		if (gpiod)
+			max->enable_external_control = true;
 
 		max->desc.enable_reg = MAX8973_VOUT;
 		max->desc.enable_mask = MAX8973_VOUT_ENABLE;
diff --git a/drivers/regulator/max8998.c b/drivers/regulator/max8998.c
index 3027e7ce100b..6a2b61c012b5 100644
--- a/drivers/regulator/max8998.c
+++ b/drivers/regulator/max8998.c
@@ -309,8 +309,7 @@ static int max8998_set_voltage_buck_sel(struct regulator_dev *rdev,
 					unsigned selector)
 {
 	struct max8998_data *max8998 = rdev_get_drvdata(rdev);
-	struct max8998_platform_data *pdata =
-		dev_get_platdata(max8998->iodev->dev);
+	struct max8998_platform_data *pdata = max8998->iodev->pdata;
 	struct i2c_client *i2c = max8998->iodev->i2c;
 	int buck = rdev_get_id(rdev);
 	int reg, shift = 0, mask, ret, j;
diff --git a/drivers/regulator/mc13783-regulator.c b/drivers/regulator/mc13783-regulator.c
index fe4c7d677f9c..3db9ebc76b79 100644
--- a/drivers/regulator/mc13783-regulator.c
+++ b/drivers/regulator/mc13783-regulator.c
@@ -1,14 +1,10 @@
-/*
- * Regulator Driver for Freescale MC13783 PMIC
- *
- * Copyright 2010 Yong Shen <yong.shen@linaro.org>
- * Copyright (C) 2008 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
- * Copyright 2009 Alberto Panizzo <maramaopercheseimorto@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Regulator Driver for Freescale MC13783 PMIC
+//
+// Copyright 2010 Yong Shen <yong.shen@linaro.org>
+// Copyright (C) 2008 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
+// Copyright 2009 Alberto Panizzo <maramaopercheseimorto@gmail.com>
 
 #include <linux/mfd/mc13783.h>
 #include <linux/regulator/machine.h>
diff --git a/drivers/regulator/mc13892-regulator.c b/drivers/regulator/mc13892-regulator.c
index 0d17c9206816..ab309ab693ba 100644
--- a/drivers/regulator/mc13892-regulator.c
+++ b/drivers/regulator/mc13892-regulator.c
@@ -1,14 +1,10 @@
-/*
- * Regulator Driver for Freescale MC13892 PMIC
- *
- * Copyright 2010 Yong Shen <yong.shen@linaro.org>
- *
- * Based on draft driver from Arnaud Patard <arnaud.patard@rtp-net.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Regulator Driver for Freescale MC13892 PMIC
+//
+// Copyright 2010 Yong Shen <yong.shen@linaro.org>
+//
+// Based on draft driver from Arnaud Patard <arnaud.patard@rtp-net.org>
 
 #include <linux/mfd/mc13892.h>
 #include <linux/regulator/machine.h>
diff --git a/drivers/regulator/mc13xxx-regulator-core.c b/drivers/regulator/mc13xxx-regulator-core.c
index 0281c31ae2ed..41271aeea63e 100644
--- a/drivers/regulator/mc13xxx-regulator-core.c
+++ b/drivers/regulator/mc13xxx-regulator-core.c
@@ -1,19 +1,15 @@
-/*
- * Regulator Driver for Freescale MC13xxx PMIC
- *
- * Copyright 2010 Yong Shen <yong.shen@linaro.org>
- *
- * Based on mc13783 regulator driver :
- * Copyright (C) 2008 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
- * Copyright 2009 Alberto Panizzo <maramaopercheseimorto@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * Regs infos taken from mc13xxx drivers from freescale and mc13xxx.pdf file
- * from freescale
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Regulator Driver for Freescale MC13xxx PMIC
+//
+// Copyright 2010 Yong Shen <yong.shen@linaro.org>
+//
+// Based on mc13783 regulator driver :
+// Copyright (C) 2008 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
+// Copyright 2009 Alberto Panizzo <maramaopercheseimorto@gmail.com>
+//
+// Regs infos taken from mc13xxx drivers from freescale and mc13xxx.pdf file
+// from freescale
 
 #include <linux/mfd/mc13xxx.h>
 #include <linux/regulator/machine.h>
diff --git a/drivers/regulator/of_regulator.c b/drivers/regulator/of_regulator.c
index f47264fa1940..638f17d4c848 100644
--- a/drivers/regulator/of_regulator.c
+++ b/drivers/regulator/of_regulator.c
@@ -31,7 +31,8 @@ static void of_get_regulation_constraints(struct device_node *np,
 	struct regulation_constraints *constraints = &(*init_data)->constraints;
 	struct regulator_state *suspend_state;
 	struct device_node *suspend_np;
-	int ret, i;
+	unsigned int mode;
+	int ret, i, len;
 	u32 pval;
 
 	constraints->name = of_get_property(np, "regulator-name", NULL);
@@ -124,20 +125,51 @@ static void of_get_regulation_constraints(struct device_node *np,
 
 	if (!of_property_read_u32(np, "regulator-initial-mode", &pval)) {
 		if (desc && desc->of_map_mode) {
-			ret = desc->of_map_mode(pval);
-			if (ret == -EINVAL)
+			mode = desc->of_map_mode(pval);
+			if (mode == REGULATOR_MODE_INVALID)
 				pr_err("%s: invalid mode %u\n", np->name, pval);
 			else
-				constraints->initial_mode = ret;
+				constraints->initial_mode = mode;
 		} else {
 			pr_warn("%s: mapping for mode %d not defined\n",
 				np->name, pval);
 		}
 	}
 
+	len = of_property_count_elems_of_size(np, "regulator-allowed-modes",
+						sizeof(u32));
+	if (len > 0) {
+		if (desc && desc->of_map_mode) {
+			for (i = 0; i < len; i++) {
+				ret = of_property_read_u32_index(np,
+					"regulator-allowed-modes", i, &pval);
+				if (ret) {
+					pr_err("%s: couldn't read allowed modes index %d, ret=%d\n",
+						np->name, i, ret);
+					break;
+				}
+				mode = desc->of_map_mode(pval);
+				if (mode == REGULATOR_MODE_INVALID)
+					pr_err("%s: invalid regulator-allowed-modes element %u\n",
+						np->name, pval);
+				else
+					constraints->valid_modes_mask |= mode;
+			}
+			if (constraints->valid_modes_mask)
+				constraints->valid_ops_mask
+					|= REGULATOR_CHANGE_MODE;
+		} else {
+			pr_warn("%s: mode mapping not defined\n", np->name);
+		}
+	}
+
 	if (!of_property_read_u32(np, "regulator-system-load", &pval))
 		constraints->system_load = pval;
 
+	if (!of_property_read_u32(np, "regulator-coupled-max-spread",
+				  &pval))
+		constraints->max_spread = pval;
+
 	constraints->over_current_protection = of_property_read_bool(np,
 					"regulator-over-current-protection");
 
@@ -163,12 +195,12 @@ static void of_get_regulation_constraints(struct device_node *np,
 		if (!of_property_read_u32(suspend_np, "regulator-mode",
 					  &pval)) {
 			if (desc && desc->of_map_mode) {
-				ret = desc->of_map_mode(pval);
-				if (ret == -EINVAL)
+				mode = desc->of_map_mode(pval);
+				if (mode == REGULATOR_MODE_INVALID)
 					pr_err("%s: invalid mode %u\n",
 					       np->name, pval);
 				else
-					suspend_state->mode = ret;
+					suspend_state->mode = mode;
 			} else {
 				pr_warn("%s: mapping for mode %d not defined\n",
 					np->name, pval);
@@ -407,3 +439,150 @@ struct regulator_dev *of_find_regulator_by_node(struct device_node *np)
 
 	return dev ? dev_to_rdev(dev) : NULL;
 }
+
+/*
+ * Returns number of regulators coupled with rdev.
+ */
+int of_get_n_coupled(struct regulator_dev *rdev)
+{
+	struct device_node *node = rdev->dev.of_node;
+	int n_phandles;
+
+	n_phandles = of_count_phandle_with_args(node,
+						"regulator-coupled-with",
+						NULL);
+
+	return (n_phandles > 0) ? n_phandles : 0;
+}
+
+/* Looks for "to_find" device_node in src's "regulator-coupled-with" property */
+static bool of_coupling_find_node(struct device_node *src,
+				  struct device_node *to_find)
+{
+	int n_phandles, i;
+	bool found = false;
+
+	n_phandles = of_count_phandle_with_args(src,
+						"regulator-coupled-with",
+						NULL);
+
+	for (i = 0; i < n_phandles; i++) {
+		struct device_node *tmp = of_parse_phandle(src,
+					   "regulator-coupled-with", i);
+
+		if (!tmp)
+			break;
+
+		/* found */
+		if (tmp == to_find)
+			found = true;
+
+		of_node_put(tmp);
+
+		if (found)
+			break;
+	}
+
+	return found;
+}
+
+/**
+ * of_check_coupling_data - Parse rdev's coupling properties and check data
+ *			    consistency
+ * @rdev - pointer to regulator_dev whose data is checked
+ *
+ * Function checks if all the following conditions are met:
+ * - rdev's max_spread is greater than 0
+ * - all coupled regulators have the same max_spread
+ * - all coupled regulators have the same number of regulator_dev phandles
+ * - all regulators are linked to each other
+ *
+ * Returns true if all conditions are met.
+ */
+bool of_check_coupling_data(struct regulator_dev *rdev)
+{
+	int max_spread = rdev->constraints->max_spread;
+	struct device_node *node = rdev->dev.of_node;
+	int n_phandles = of_get_n_coupled(rdev);
+	struct device_node *c_node;
+	int i;
+	bool ret = true;
+
+	if (max_spread <= 0) {
+		dev_err(&rdev->dev, "max_spread value invalid\n");
+		return false;
+	}
+
+	/* iterate over rdev's phandles */
+	for (i = 0; i < n_phandles; i++) {
+		int c_max_spread, c_n_phandles;
+
+		c_node = of_parse_phandle(node,
+					  "regulator-coupled-with", i);
+
+		if (!c_node)
+			ret = false;
+
+		c_n_phandles = of_count_phandle_with_args(c_node,
+							  "regulator-coupled-with",
+							  NULL);
+
+		if (c_n_phandles != n_phandles) {
+			dev_err(&rdev->dev, "number of couped reg phandles mismatch\n");
+			ret = false;
+			goto clean;
+		}
+
+		if (of_property_read_u32(c_node, "regulator-coupled-max-spread",
+					 &c_max_spread)) {
+			ret = false;
+			goto clean;
+		}
+
+		if (c_max_spread != max_spread) {
+			dev_err(&rdev->dev,
+				"coupled regulators max_spread mismatch\n");
+			ret = false;
+			goto clean;
+		}
+
+		if (!of_coupling_find_node(c_node, node)) {
+			dev_err(&rdev->dev, "missing 2-way linking for coupled regulators\n");
+			ret = false;
+		}
+
+clean:
+		of_node_put(c_node);
+		if (!ret)
+			break;
+	}
+
+	return ret;
+}
+
+/**
+ * of_parse_coupled regulator - Get regulator_dev pointer from rdev's property
+ * @rdev: Pointer to regulator_dev, whose DTS is used as a source to parse
+ *	  "regulator-coupled-with" property
+ * @index: Index in phandles array
+ *
+ * Returns the regulator_dev pointer parsed from DTS. If it has not been yet
+ * registered, returns NULL
+ */
+struct regulator_dev *of_parse_coupled_regulator(struct regulator_dev *rdev,
+						 int index)
+{
+	struct device_node *node = rdev->dev.of_node;
+	struct device_node *c_node;
+	struct regulator_dev *c_rdev;
+
+	c_node = of_parse_phandle(node, "regulator-coupled-with", index);
+	if (!c_node)
+		return NULL;
+
+	c_rdev = of_find_regulator_by_node(c_node);
+
+	of_node_put(c_node);
+
+	return c_rdev;
+}
diff --git a/drivers/regulator/pfuze100-regulator.c b/drivers/regulator/pfuze100-regulator.c
index 63922a2167e5..8d9dbcc775ea 100644
--- a/drivers/regulator/pfuze100-regulator.c
+++ b/drivers/regulator/pfuze100-regulator.c
@@ -1,20 +1,7 @@
-/*
- * Copyright (C) 2011-2013 Freescale Semiconductor, Inc. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (C) 2011-2013 Freescale Semiconductor, Inc. All Rights Reserved.
+
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/init.h>
@@ -86,6 +73,13 @@ static const int pfuze100_coin[] = {
 	2500000, 2700000, 2800000, 2900000, 3000000, 3100000, 3200000, 3300000,
 };
 
+static const int pfuze3000_sw1a[] = {
+	700000, 725000, 750000, 775000, 800000, 825000, 850000, 875000,
+	900000, 925000, 950000, 975000, 1000000, 1025000, 1050000, 1075000,
+	1100000, 1125000, 1150000, 1175000, 1200000, 1225000, 1250000, 1275000,
+	1300000, 1325000, 1350000, 1375000, 1400000, 1425000, 1800000, 3300000,
+};
+
 static const int pfuze3000_sw2lo[] = {
 	1500000, 1550000, 1600000, 1650000, 1700000, 1750000, 1800000, 1850000,
 };
@@ -148,6 +142,9 @@ static const struct regulator_ops pfuze100_fixed_regulator_ops = {
 };
 
 static const struct regulator_ops pfuze100_sw_regulator_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
 	.list_voltage = regulator_list_voltage_linear,
 	.set_voltage_sel = regulator_set_voltage_sel_regmap,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
@@ -158,6 +155,7 @@ static const struct regulator_ops pfuze100_sw_regulator_ops = {
 static const struct regulator_ops pfuze100_swb_regulator_ops = {
 	.enable = regulator_enable_regmap,
 	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
 	.list_voltage = regulator_list_voltage_table,
 	.map_voltage = regulator_map_voltage_ascend,
 	.set_voltage_sel = regulator_set_voltage_sel_regmap,
@@ -193,6 +191,11 @@ static const struct regulator_ops pfuze100_swb_regulator_ops = {
 			.uV_step = (step),	\
 			.vsel_reg = (base) + PFUZE100_VOL_OFFSET,	\
 			.vsel_mask = 0x3f,	\
+			.enable_reg = (base) + PFUZE100_MODE_OFFSET,	\
+			.enable_val = 0xc,	\
+			.disable_val = 0x0,	\
+			.enable_mask = 0xf,	\
+			.enable_time = 500,	\
 		},	\
 		.stby_reg = (base) + PFUZE100_STANDBY_OFFSET,	\
 		.stby_mask = 0x3f,	\
@@ -343,7 +346,7 @@ static struct pfuze_regulator pfuze200_regulators[] = {
 };
 
 static struct pfuze_regulator pfuze3000_regulators[] = {
-	PFUZE100_SW_REG(PFUZE3000, SW1A, PFUZE100_SW1ABVOL, 700000, 1475000, 25000),
+	PFUZE100_SWB_REG(PFUZE3000, SW1A, PFUZE100_SW1ABVOL, 0x1f, pfuze3000_sw1a),
 	PFUZE100_SW_REG(PFUZE3000, SW1B, PFUZE100_SW1CVOL, 700000, 1475000, 25000),
 	PFUZE100_SWB_REG(PFUZE3000, SW2, PFUZE100_SW2VOL, 0x7, pfuze3000_sw2lo),
 	PFUZE3000_SW3_REG(PFUZE3000, SW3, PFUZE100_SW3AVOL, 900000, 1650000, 50000),
@@ -648,7 +651,6 @@ static int pfuze100_regulator_probe(struct i2c_client *client,
 		config.init_data = init_data;
 		config.driver_data = pfuze_chip;
 		config.of_node = match_of_node(i);
-		config.ena_gpio = -EINVAL;
 
 		pfuze_chip->regulators[i] =
 			devm_regulator_register(&client->dev, desc, &config);
diff --git a/drivers/regulator/qcom_spmi-regulator.c b/drivers/regulator/qcom_spmi-regulator.c
index 63c7a0c17777..9817f1a75342 100644
--- a/drivers/regulator/qcom_spmi-regulator.c
+++ b/drivers/regulator/qcom_spmi-regulator.c
@@ -25,6 +25,8 @@
 #include <linux/regulator/driver.h>
 #include <linux/regmap.h>
 #include <linux/list.h>
+#include <linux/mfd/syscon.h>
+#include <linux/io.h>
 
 /* Pin control enable input pins. */
 #define SPMI_REGULATOR_PIN_CTRL_ENABLE_NONE		0x00
@@ -181,6 +183,23 @@ enum spmi_boost_byp_registers {
 	SPMI_BOOST_BYP_REG_CURRENT_LIMIT	= 0x4b,
 };
 
+enum spmi_saw3_registers {
+	SAW3_SECURE				= 0x00,
+	SAW3_ID					= 0x04,
+	SAW3_SPM_STS				= 0x0C,
+	SAW3_AVS_STS				= 0x10,
+	SAW3_PMIC_STS				= 0x14,
+	SAW3_RST				= 0x18,
+	SAW3_VCTL				= 0x1C,
+	SAW3_AVS_CTL				= 0x20,
+	SAW3_AVS_LIMIT				= 0x24,
+	SAW3_AVS_DLY				= 0x28,
+	SAW3_AVS_HYSTERESIS			= 0x2C,
+	SAW3_SPM_STS2				= 0x38,
+	SAW3_SPM_PMIC_DATA_3			= 0x4C,
+	SAW3_VERSION				= 0xFD0,
+};
+
 /* Used for indexing into ctrl_reg.  These are offets from 0x40 */
 enum spmi_common_control_register_index {
 	SPMI_COMMON_IDX_VOLTAGE_RANGE		= 0,
@@ -1035,6 +1054,89 @@ static irqreturn_t spmi_regulator_vs_ocp_isr(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+#define SAW3_VCTL_DATA_MASK	0xFF
+#define SAW3_VCTL_CLEAR_MASK	0x700FF
+#define SAW3_AVS_CTL_EN_MASK	0x1
+#define SAW3_AVS_CTL_TGGL_MASK	0x8000000
+#define SAW3_AVS_CTL_CLEAR_MASK	0x7efc00
+
+static struct regmap *saw_regmap = NULL;
+
+static void spmi_saw_set_vdd(void *data)
+{
+	u32 vctl, data3, avs_ctl, pmic_sts;
+	bool avs_enabled = false;
+	unsigned long timeout;
+	u8 voltage_sel = *(u8 *)data;
+
+	regmap_read(saw_regmap, SAW3_AVS_CTL, &avs_ctl);
+	regmap_read(saw_regmap, SAW3_VCTL, &vctl);
+	regmap_read(saw_regmap, SAW3_SPM_PMIC_DATA_3, &data3);
+
+	/* select the band */
+	vctl &= ~SAW3_VCTL_CLEAR_MASK;
+	vctl |= (u32)voltage_sel;
+
+	data3 &= ~SAW3_VCTL_CLEAR_MASK;
+	data3 |= (u32)voltage_sel;
+
+	/* If AVS is enabled, switch it off during the voltage change */
+	avs_enabled = SAW3_AVS_CTL_EN_MASK & avs_ctl;
+	if (avs_enabled) {
+		avs_ctl &= ~SAW3_AVS_CTL_TGGL_MASK;
+		regmap_write(saw_regmap, SAW3_AVS_CTL, avs_ctl);
+	}
+
+	regmap_write(saw_regmap, SAW3_RST, 1);
+	regmap_write(saw_regmap, SAW3_VCTL, vctl);
+	regmap_write(saw_regmap, SAW3_SPM_PMIC_DATA_3, data3);
+
+	timeout = jiffies + usecs_to_jiffies(100);
+	do {
+		regmap_read(saw_regmap, SAW3_PMIC_STS, &pmic_sts);
+		pmic_sts &= SAW3_VCTL_DATA_MASK;
+		if (pmic_sts == (u32)voltage_sel)
+			break;
+
+		cpu_relax();
+
+	} while (time_before(jiffies, timeout));
+
+	/* After successful voltage change, switch the AVS back on */
+	if (avs_enabled) {
+		pmic_sts &= 0x3f;
+		avs_ctl &= ~SAW3_AVS_CTL_CLEAR_MASK;
+		avs_ctl |= ((pmic_sts - 4) << 10);
+		avs_ctl |= (pmic_sts << 17);
+		avs_ctl |= SAW3_AVS_CTL_TGGL_MASK;
+		regmap_write(saw_regmap, SAW3_AVS_CTL, avs_ctl);
+	}
+}
+
+static int
+spmi_regulator_saw_set_voltage(struct regulator_dev *rdev, unsigned selector)
+{
+	struct spmi_regulator *vreg = rdev_get_drvdata(rdev);
+	int ret;
+	u8 range_sel, voltage_sel;
+
+	ret = spmi_sw_selector_to_hw(vreg, selector, &range_sel, &voltage_sel);
+	if (ret)
+		return ret;
+
+	if (0 != range_sel) {
+		dev_dbg(&rdev->dev, "range_sel = %02X voltage_sel = %02X", \
+			range_sel, voltage_sel);
+		return -EINVAL;
+	}
+
+	/* Always do the SAW register writes on the first CPU */
+	return smp_call_function_single(0, spmi_saw_set_vdd, \
+					&voltage_sel, true);
+}
+
+static struct regulator_ops spmi_saw_ops = {};
+
 static struct regulator_ops spmi_smps_ops = {
 	.enable			= regulator_enable_regmap,
 	.disable		= regulator_disable_regmap,
@@ -1250,6 +1352,7 @@ static int spmi_regulator_match(struct spmi_regulator *vreg, u16 force_type)
 	}
 	dig_major_rev	= version[SPMI_COMMON_REG_DIG_MAJOR_REV
 					- SPMI_COMMON_REG_DIG_MAJOR_REV];
+
 	if (!force_type) {
 		type		= version[SPMI_COMMON_REG_TYPE -
 					  SPMI_COMMON_REG_DIG_MAJOR_REV];
@@ -1648,7 +1751,9 @@ static int qcom_spmi_regulator_probe(struct platform_device *pdev)
 	struct regmap *regmap;
 	const char *name;
 	struct device *dev = &pdev->dev;
-	int ret;
+	struct device_node *node = pdev->dev.of_node;
+	struct device_node *syscon;
+	int ret, lenp;
 	struct list_head *vreg_list;
 
 	vreg_list = devm_kzalloc(dev, sizeof(*vreg_list), GFP_KERNEL);
@@ -1665,7 +1770,22 @@ static int qcom_spmi_regulator_probe(struct platform_device *pdev)
 	if (!match)
 		return -ENODEV;
 
+	if (of_find_property(node, "qcom,saw-reg", &lenp)) {
+		syscon = of_parse_phandle(node, "qcom,saw-reg", 0);
+		saw_regmap = syscon_node_to_regmap(syscon);
+		of_node_put(syscon);
+		if (IS_ERR(regmap))
+			dev_err(dev, "ERROR reading SAW regmap\n");
+	}
+
 	for (reg = match->data; reg->name; reg++) {
+
+		if (saw_regmap && \
+		    of_find_property(of_find_node_by_name(node, reg->name), \
+				     "qcom,saw-slave", &lenp)) {
+			continue;
+		}
+
 		vreg = devm_kzalloc(dev, sizeof(*vreg), GFP_KERNEL);
 		if (!vreg)
 			return -ENOMEM;
@@ -1673,7 +1793,6 @@ static int qcom_spmi_regulator_probe(struct platform_device *pdev)
 		vreg->dev = dev;
 		vreg->base = reg->base;
 		vreg->regmap = regmap;
-
 		if (reg->ocp) {
 			vreg->ocp_irq = platform_get_irq_byname(pdev, reg->ocp);
 			if (vreg->ocp_irq < 0) {
@@ -1681,7 +1800,6 @@ static int qcom_spmi_regulator_probe(struct platform_device *pdev)
 				goto err;
 			}
 		}
-
 		vreg->desc.id = -1;
 		vreg->desc.owner = THIS_MODULE;
 		vreg->desc.type = REGULATOR_VOLTAGE;
@@ -1698,6 +1816,15 @@ static int qcom_spmi_regulator_probe(struct platform_device *pdev)
 		if (ret)
 			continue;
 
+		if (saw_regmap && \
+		    of_find_property(of_find_node_by_name(node, reg->name), \
+				     "qcom,saw-leader", &lenp)) {
+			spmi_saw_ops = *(vreg->desc.ops);
+			spmi_saw_ops.set_voltage_sel = \
+				spmi_regulator_saw_set_voltage;
+			vreg->desc.ops = &spmi_saw_ops;
+		}
+
 		config.dev = dev;
 		config.driver_data = vreg;
 		config.regmap = regmap;
diff --git a/drivers/regulator/s5m8767.c b/drivers/regulator/s5m8767.c
index 4836947e1521..b8443a360646 100644
--- a/drivers/regulator/s5m8767.c
+++ b/drivers/regulator/s5m8767.c
@@ -13,6 +13,7 @@
 
 #include <linux/err.h>
 #include <linux/of_gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/driver.h>
@@ -459,15 +460,14 @@ static void s5m8767_regulator_config_ext_control(struct s5m8767_info *s5m8767,
 		return;
 	}
 
-	if (!gpio_is_valid(rdata->ext_control_gpio)) {
+	if (!rdata->ext_control_gpiod) {
 		dev_warn(s5m8767->dev,
 				"ext-control for %s: GPIO not valid, ignoring\n",
-				rdata->reg_node->name);
+			 rdata->reg_node->name);
 		return;
 	}
 
-	config->ena_gpio = rdata->ext_control_gpio;
-	config->ena_gpio_flags = GPIOF_OUT_INIT_HIGH;
+	config->ena_gpiod = rdata->ext_control_gpiod;
 }
 
 /*
@@ -577,8 +577,14 @@ static int s5m8767_pmic_dt_parse_pdata(struct platform_device *pdev,
 			continue;
 		}
 
-		rdata->ext_control_gpio = of_get_named_gpio(reg_np,
-			"s5m8767,pmic-ext-control-gpios", 0);
+		rdata->ext_control_gpiod = devm_gpiod_get_from_of_node(&pdev->dev,
+								       reg_np,
+								       "s5m8767,pmic-ext-control-gpios",
+								       0,
+								       GPIOD_OUT_HIGH,
+								       "s5m8767");
+		if (IS_ERR(rdata->ext_control_gpiod))
+			return PTR_ERR(rdata->ext_control_gpiod);
 
 		rdata->id = i;
 		rdata->initdata = of_get_regulator_init_data(
@@ -954,10 +960,8 @@ static int s5m8767_pmic_probe(struct platform_device *pdev)
 		config.driver_data = s5m8767;
 		config.regmap = iodev->regmap_pmic;
 		config.of_node = pdata->regulators[i].reg_node;
-		config.ena_gpio = -EINVAL;
-		config.ena_gpio_flags = 0;
-		config.ena_gpio_initialized = true;
-		if (gpio_is_valid(pdata->regulators[i].ext_control_gpio))
+		config.ena_gpiod = NULL;
+		if (pdata->regulators[i].ext_control_gpiod)
 			s5m8767_regulator_config_ext_control(s5m8767,
 					&pdata->regulators[i], &config);
 
@@ -970,7 +974,7 @@ static int s5m8767_pmic_probe(struct platform_device *pdev)
 			return ret;
 		}
 
-		if (gpio_is_valid(pdata->regulators[i].ext_control_gpio)) {
+		if (pdata->regulators[i].ext_control_gpiod) {
 			ret = s5m8767_enable_ext_control(s5m8767, rdev);
 			if (ret < 0) {
 				dev_err(s5m8767->dev,
diff --git a/drivers/regulator/sy8106a-regulator.c b/drivers/regulator/sy8106a-regulator.c
new file mode 100644
index 000000000000..65fbd1f0b612
--- /dev/null
+++ b/drivers/regulator/sy8106a-regulator.c
@@ -0,0 +1,167 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// sy8106a-regulator.c - Regulator device driver for SY8106A
+//
+// Copyright (C) 2016 Ondřej Jirman <megous@megous.com>
+// Copyright (c) 2017-2018 Icenowy Zheng <icenowy@aosc.io>
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/of_regulator.h>
+
+#define SY8106A_REG_VOUT1_SEL		0x01
+#define SY8106A_REG_VOUT_COM		0x02
+#define SY8106A_REG_VOUT1_SEL_MASK	0x7f
+#define SY8106A_DISABLE_REG		BIT(0)
+/*
+ * The I2C controlled voltage will only work when this bit is set; otherwise
+ * it will behave like a fixed regulator.
+ */
+#define SY8106A_GO_BIT			BIT(7)
+
+struct sy8106a {
+	struct regulator_dev *rdev;
+	struct regmap *regmap;
+	u32 fixed_voltage;
+};
+
+static const struct regmap_config sy8106a_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+};
+
+static const struct regulator_ops sy8106a_ops = {
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.list_voltage = regulator_list_voltage_linear,
+	/* Enabling/disabling the regulator is not yet implemented */
+};
+
+/* Default limits measured in millivolts */
+#define SY8106A_MIN_MV		680
+#define SY8106A_MAX_MV		1950
+#define SY8106A_STEP_MV		10
+
+static const struct regulator_desc sy8106a_reg = {
+	.name = "SY8106A",
+	.id = 0,
+	.ops = &sy8106a_ops,
+	.type = REGULATOR_VOLTAGE,
+	.n_voltages = ((SY8106A_MAX_MV - SY8106A_MIN_MV) / SY8106A_STEP_MV) + 1,
+	.min_uV = (SY8106A_MIN_MV * 1000),
+	.uV_step = (SY8106A_STEP_MV * 1000),
+	.vsel_reg = SY8106A_REG_VOUT1_SEL,
+	.vsel_mask = SY8106A_REG_VOUT1_SEL_MASK,
+	/*
+	 * This ramp_delay is a conservative default value which works on
+	 * H3/H5 boards VDD-CPUX situations.
+	 */
+	.ramp_delay = 200,
+	.owner = THIS_MODULE,
+};
+
+/*
+ * I2C driver interface functions
+ */
+static int sy8106a_i2c_probe(struct i2c_client *i2c,
+			    const struct i2c_device_id *id)
+{
+	struct sy8106a *chip;
+	struct device *dev = &i2c->dev;
+	struct regulator_dev *rdev = NULL;
+	struct regulator_config config = { };
+	unsigned int reg, vsel;
+	int error;
+
+	chip = devm_kzalloc(&i2c->dev, sizeof(struct sy8106a), GFP_KERNEL);
+	if (!chip)
+		return -ENOMEM;
+
+	error = of_property_read_u32(dev->of_node, "silergy,fixed-microvolt",
+				     &chip->fixed_voltage);
+	if (error)
+		return error;
+
+	if (chip->fixed_voltage < SY8106A_MIN_MV * 1000 ||
+	    chip->fixed_voltage > SY8106A_MAX_MV * 1000)
+		return -EINVAL;
+
+	chip->regmap = devm_regmap_init_i2c(i2c, &sy8106a_regmap_config);
+	if (IS_ERR(chip->regmap)) {
+		error = PTR_ERR(chip->regmap);
+		dev_err(dev, "Failed to allocate register map: %d\n", error);
+		return error;
+	}
+
+	config.dev = &i2c->dev;
+	config.regmap = chip->regmap;
+	config.driver_data = chip;
+
+	config.of_node = dev->of_node;
+	config.init_data = of_get_regulator_init_data(dev, dev->of_node,
+						      &sy8106a_reg);
+
+	if (!config.init_data)
+		return -ENOMEM;
+
+	/* Ensure GO_BIT is enabled when probing */
+	error = regmap_read(chip->regmap, SY8106A_REG_VOUT1_SEL, &reg);
+	if (error)
+		return error;
+
+	if (!(reg & SY8106A_GO_BIT)) {
+		vsel = (chip->fixed_voltage / 1000 - SY8106A_MIN_MV) /
+		       SY8106A_STEP_MV;
+
+		error = regmap_write(chip->regmap, SY8106A_REG_VOUT1_SEL,
+				     vsel | SY8106A_GO_BIT);
+		if (error)
+			return error;
+	}
+
+	/* Probe regulator */
+	rdev = devm_regulator_register(&i2c->dev, &sy8106a_reg, &config);
+	if (IS_ERR(rdev)) {
+		error = PTR_ERR(rdev);
+		dev_err(&i2c->dev, "Failed to register SY8106A regulator: %d\n", error);
+		return error;
+	}
+
+	chip->rdev = rdev;
+
+	i2c_set_clientdata(i2c, chip);
+
+	return 0;
+}
+
+static const struct of_device_id sy8106a_i2c_of_match[] = {
+	{ .compatible = "silergy,sy8106a" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, sy8106a_i2c_of_match);
+
+static const struct i2c_device_id sy8106a_i2c_id[] = {
+	{ "sy8106a", 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(i2c, sy8106a_i2c_id);
+
+static struct i2c_driver sy8106a_regulator_driver = {
+	.driver = {
+		.name = "sy8106a",
+		.of_match_table	= of_match_ptr(sy8106a_i2c_of_match),
+	},
+	.probe = sy8106a_i2c_probe,
+	.id_table = sy8106a_i2c_id,
+};
+
+module_i2c_driver(sy8106a_regulator_driver);
+
+MODULE_AUTHOR("Ondřej Jirman <megous@megous.com>");
+MODULE_AUTHOR("Icenowy Zheng <icenowy@aosc.io>");
+MODULE_DESCRIPTION("Regulator device driver for Silergy SY8106A");
+MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/tps65090-regulator.c b/drivers/regulator/tps65090-regulator.c
index 395f35dc8cdb..2d398fa3b720 100644
--- a/drivers/regulator/tps65090-regulator.c
+++ b/drivers/regulator/tps65090-regulator.c
@@ -19,8 +19,8 @@
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/init.h>
-#include <linux/gpio.h>
-#include <linux/of_gpio.h>
+#include <linux/of.h>
+#include <linux/gpio/consumer.h>
 #include <linux/slab.h>
 #include <linux/err.h>
 #include <linux/platform_device.h>
@@ -300,26 +300,6 @@ static int tps65090_regulator_disable_ext_control(
 	return tps65090_config_ext_control(ri, false);
 }
 
-static void tps65090_configure_regulator_config(
-		struct tps65090_regulator_plat_data *tps_pdata,
-		struct regulator_config *config)
-{
-	if (gpio_is_valid(tps_pdata->gpio)) {
-		int gpio_flag = GPIOF_OUT_INIT_LOW;
-
-		if (tps_pdata->reg_init_data->constraints.always_on ||
-				tps_pdata->reg_init_data->constraints.boot_on)
-			gpio_flag = GPIOF_OUT_INIT_HIGH;
-
-		config->ena_gpio = tps_pdata->gpio;
-		config->ena_gpio_initialized = true;
-		config->ena_gpio_flags = gpio_flag;
-	} else {
-		config->ena_gpio = -EINVAL;
-		config->ena_gpio_initialized = false;
-	}
-}
-
 #ifdef CONFIG_OF
 static struct of_regulator_match tps65090_matches[] = {
 	{ .name = "dcdc1", },
@@ -385,9 +365,26 @@ static struct tps65090_platform_data *tps65090_parse_dt_reg_data(
 		rpdata->enable_ext_control = of_property_read_bool(
 					tps65090_matches[idx].of_node,
 					"ti,enable-ext-control");
-		if (rpdata->enable_ext_control)
-			rpdata->gpio = of_get_named_gpio(np,
-					"dcdc-ext-control-gpios", 0);
+		if (rpdata->enable_ext_control) {
+			enum gpiod_flags gflags;
+
+			if (ri_data->constraints.always_on ||
+			    ri_data->constraints.boot_on)
+				gflags = GPIOD_OUT_HIGH;
+			else
+				gflags = GPIOD_OUT_LOW;
+
+			rpdata->gpiod = devm_gpiod_get_from_of_node(&pdev->dev,
+								    tps65090_matches[idx].of_node,
+								    "dcdc-ext-control-gpios", 0,
+								    gflags,
+								    "tps65090");
+			if (IS_ERR(rpdata->gpiod))
+				return ERR_CAST(rpdata->gpiod);
+			if (!rpdata->gpiod)
+				dev_err(&pdev->dev,
+					"could not find DCDC external control GPIO\n");
+		}
 
 		if (of_property_read_u32(tps65090_matches[idx].of_node,
 					 "ti,overcurrent-wait",
@@ -455,8 +452,7 @@ static int tps65090_regulator_probe(struct platform_device *pdev)
 		 */
 		if (tps_pdata && is_dcdc(num) && tps_pdata->reg_init_data) {
 			if (tps_pdata->enable_ext_control) {
-				tps65090_configure_regulator_config(
-						tps_pdata, &config);
+				config.ena_gpiod = tps_pdata->gpiod;
 				ri->desc->ops = &tps65090_ext_control_ops;
 			} else {
 				ret = tps65090_regulator_disable_ext_control(
diff --git a/drivers/regulator/tps6586x-regulator.c b/drivers/regulator/tps6586x-regulator.c
index 9e9d22038017..ba3dae7b2d2d 100644
--- a/drivers/regulator/tps6586x-regulator.c
+++ b/drivers/regulator/tps6586x-regulator.c
@@ -342,6 +342,7 @@ static struct tps6586x_regulator *find_regulator_info(int id, int version)
 
 	switch (version) {
 	case TPS658623:
+	case TPS658624:
 		table = tps658623_regulator;
 		num = ARRAY_SIZE(tps658623_regulator);
 		break;
diff --git a/drivers/regulator/twl-regulator.c b/drivers/regulator/twl-regulator.c
index a4456db5849d..884c7505ed91 100644
--- a/drivers/regulator/twl-regulator.c
+++ b/drivers/regulator/twl-regulator.c
@@ -274,7 +274,7 @@ static inline unsigned int twl4030reg_map_mode(unsigned int mode)
 	case RES_STATE_SLEEP:
 		return REGULATOR_MODE_STANDBY;
 	default:
-		return -EINVAL;
+		return REGULATOR_MODE_INVALID;
 	}
 }
 
diff --git a/drivers/regulator/wm8350-regulator.c b/drivers/regulator/wm8350-regulator.c
index da9106bd2109..8ad11b074b49 100644
--- a/drivers/regulator/wm8350-regulator.c
+++ b/drivers/regulator/wm8350-regulator.c
@@ -911,6 +911,7 @@ static unsigned int get_mode(int uA, const struct wm8350_dcdc_efficiency *eff)
 	while (eff[i].uA_load_min != -1) {
 		if (uA >= eff[i].uA_load_min && uA <= eff[i].uA_load_max)
 			return eff[i].mode;
+		i++;
 	}
 	return REGULATOR_MODE_NORMAL;
 }
diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
index 027274008b08..cd1c168fd188 100644
--- a/drivers/remoteproc/Kconfig
+++ b/drivers/remoteproc/Kconfig
@@ -24,7 +24,6 @@ config IMX_REMOTEPROC
 
 config OMAP_REMOTEPROC
 	tristate "OMAP remoteproc support"
-	depends on HAS_DMA
 	depends on ARCH_OMAP4 || SOC_OMAP5
 	depends on OMAP_IOMMU
 	select MAILBOX
diff --git a/drivers/remoteproc/da8xx_remoteproc.c b/drivers/remoteproc/da8xx_remoteproc.c
index bf3b9034c319..b668e32996e2 100644
--- a/drivers/remoteproc/da8xx_remoteproc.c
+++ b/drivers/remoteproc/da8xx_remoteproc.c
@@ -25,7 +25,7 @@
 #include "remoteproc_internal.h"
 
 static char *da8xx_fw_name;
-module_param(da8xx_fw_name, charp, S_IRUGO);
+module_param(da8xx_fw_name, charp, 0444);
 MODULE_PARM_DESC(da8xx_fw_name,
 		 "Name of DSP firmware file in /lib/firmware (if not specified defaults to 'rproc-dsp-fw')");
 
@@ -138,6 +138,7 @@ static int da8xx_rproc_start(struct rproc *rproc)
 	struct device *dev = rproc->dev.parent;
 	struct da8xx_rproc *drproc = (struct da8xx_rproc *)rproc->priv;
 	struct clk *dsp_clk = drproc->dsp_clk;
+	int ret;
 
 	/* hw requires the start (boot) address be on 1KB boundary */
 	if (rproc->bootaddr & 0x3ff) {
@@ -148,7 +149,12 @@ static int da8xx_rproc_start(struct rproc *rproc)
 
 	writel(rproc->bootaddr, drproc->bootreg);
 
-	clk_enable(dsp_clk);
+	ret = clk_prepare_enable(dsp_clk);
+	if (ret) {
+		dev_err(dev, "clk_prepare_enable() failed: %d\n", ret);
+		return ret;
+	}
+
 	davinci_clk_reset_deassert(dsp_clk);
 
 	return 0;
@@ -159,7 +165,7 @@ static int da8xx_rproc_stop(struct rproc *rproc)
 	struct da8xx_rproc *drproc = rproc->priv;
 
 	davinci_clk_reset_assert(drproc->dsp_clk);
-	clk_disable(drproc->dsp_clk);
+	clk_disable_unprepare(drproc->dsp_clk);
 
 	return 0;
 }
diff --git a/drivers/remoteproc/qcom_q6v5_pil.c b/drivers/remoteproc/qcom_q6v5_pil.c
index cbbafdcaaecb..2bf8e7c49f2a 100644
--- a/drivers/remoteproc/qcom_q6v5_pil.c
+++ b/drivers/remoteproc/qcom_q6v5_pil.c
@@ -57,6 +57,8 @@
 #define RMB_PMI_META_DATA_REG		0x10
 #define RMB_PMI_CODE_START_REG		0x14
 #define RMB_PMI_CODE_LENGTH_REG		0x18
+#define RMB_MBA_MSS_STATUS		0x40
+#define RMB_MBA_ALT_RESET		0x44
 
 #define RMB_CMD_META_DATA_READY		0x1
 #define RMB_CMD_LOAD_READY		0x2
@@ -104,6 +106,13 @@
 #define QDSP6SS_XO_CBCR		0x0038
 #define QDSP6SS_ACC_OVERRIDE_VAL		0x20
 
+/* QDSP6v65 parameters */
+#define QDSP6SS_SLEEP                   0x3C
+#define QDSP6SS_BOOT_CORE_START         0x400
+#define QDSP6SS_BOOT_CMD                0x404
+#define SLEEP_CHECK_MAX_LOOPS           200
+#define BOOT_FSM_TIMEOUT                10000
+
 struct reg_info {
 	struct regulator *reg;
 	int uV;
@@ -121,9 +130,11 @@ struct rproc_hexagon_res {
 	struct qcom_mss_reg_res *proxy_supply;
 	struct qcom_mss_reg_res *active_supply;
 	char **proxy_clk_names;
+	char **reset_clk_names;
 	char **active_clk_names;
 	int version;
 	bool need_mem_protection;
+	bool has_alt_reset;
 };
 
 struct q6v5 {
@@ -143,9 +154,15 @@ struct q6v5 {
 	struct qcom_smem_state *state;
 	unsigned stop_bit;
 
+	int handover_irq;
+
+	bool proxy_unvoted;
+
 	struct clk *active_clks[8];
+	struct clk *reset_clks[4];
 	struct clk *proxy_clks[4];
 	int active_clk_count;
+	int reset_clk_count;
 	int proxy_clk_count;
 
 	struct reg_info active_regs[1];
@@ -166,10 +183,12 @@ struct q6v5 {
 	void *mpss_region;
 	size_t mpss_size;
 
+	struct qcom_rproc_glink glink_subdev;
 	struct qcom_rproc_subdev smd_subdev;
 	struct qcom_rproc_ssr ssr_subdev;
 	struct qcom_sysmon *sysmon;
 	bool need_mem_protection;
+	bool has_alt_reset;
 	int mpss_perm;
 	int mba_perm;
 	int version;
@@ -179,6 +198,7 @@ enum {
 	MSS_MSM8916,
 	MSS_MSM8974,
 	MSS_MSM8996,
+	MSS_SDM845,
 };
 
 static int q6v5_regulator_init(struct device *dev, struct reg_info *regs,
@@ -333,6 +353,29 @@ static int q6v5_load(struct rproc *rproc, const struct firmware *fw)
 	return 0;
 }
 
+static int q6v5_reset_assert(struct q6v5 *qproc)
+{
+	if (qproc->has_alt_reset)
+		return reset_control_reset(qproc->mss_restart);
+	else
+		return reset_control_assert(qproc->mss_restart);
+}
+
+static int q6v5_reset_deassert(struct q6v5 *qproc)
+{
+	int ret;
+
+	if (qproc->has_alt_reset) {
+		writel(1, qproc->rmb_base + RMB_MBA_ALT_RESET);
+		ret = reset_control_reset(qproc->mss_restart);
+		writel(0, qproc->rmb_base + RMB_MBA_ALT_RESET);
+	} else {
+		ret = reset_control_deassert(qproc->mss_restart);
+	}
+
+	return ret;
+}
+
 static int q6v5_rmb_pbl_wait(struct q6v5 *qproc, int ms)
 {
 	unsigned long timeout;
@@ -385,8 +428,35 @@ static int q6v5proc_reset(struct q6v5 *qproc)
 	int ret;
 	int i;
 
+	if (qproc->version == MSS_SDM845) {
+		val = readl(qproc->reg_base + QDSP6SS_SLEEP);
+		val |= 0x1;
+		writel(val, qproc->reg_base + QDSP6SS_SLEEP);
 
-	if (qproc->version == MSS_MSM8996) {
+		ret = readl_poll_timeout(qproc->reg_base + QDSP6SS_SLEEP,
+					 val, !(val & BIT(31)), 1,
+					 SLEEP_CHECK_MAX_LOOPS);
+		if (ret) {
+			dev_err(qproc->dev, "QDSP6SS Sleep clock timed out\n");
+			return -ETIMEDOUT;
+		}
+
+		/* De-assert QDSP6 stop core */
+		writel(1, qproc->reg_base + QDSP6SS_BOOT_CORE_START);
+		/* Trigger boot FSM */
+		writel(1, qproc->reg_base + QDSP6SS_BOOT_CMD);
+
+		ret = readl_poll_timeout(qproc->rmb_base + RMB_MBA_MSS_STATUS,
+				val, (val & BIT(0)) != 0, 10, BOOT_FSM_TIMEOUT);
+		if (ret) {
+			dev_err(qproc->dev, "Boot FSM failed to complete.\n");
+			/* Reset the modem so that boot FSM is in reset state */
+			q6v5_reset_deassert(qproc);
+			return ret;
+		}
+
+		goto pbl_wait;
+	} else if (qproc->version == MSS_MSM8996) {
 		/* Override the ACC value if required */
 		writel(QDSP6SS_ACC_OVERRIDE_VAL,
 		       qproc->reg_base + QDSP6SS_STRAP_ACC);
@@ -494,6 +564,7 @@ static int q6v5proc_reset(struct q6v5 *qproc)
 	val &= ~Q6SS_STOP_CORE;
 	writel(val, qproc->reg_base + QDSP6SS_RESET_REG);
 
+pbl_wait:
 	/* Wait for PBL status */
 	ret = q6v5_rmb_pbl_wait(qproc, 1000);
 	if (ret == -ETIMEDOUT) {
@@ -615,7 +686,7 @@ static int q6v5_mpss_load(struct q6v5 *qproc)
 	struct elf32_hdr *ehdr;
 	phys_addr_t mpss_reloc;
 	phys_addr_t boot_addr;
-	phys_addr_t min_addr = (phys_addr_t)ULLONG_MAX;
+	phys_addr_t min_addr = PHYS_ADDR_MAX;
 	phys_addr_t max_addr = 0;
 	bool relocate = false;
 	char seg_name[10];
@@ -727,11 +798,15 @@ static int q6v5_start(struct rproc *rproc)
 	int xfermemop_ret;
 	int ret;
 
+	qproc->proxy_unvoted = false;
+
+	enable_irq(qproc->handover_irq);
+
 	ret = q6v5_regulator_enable(qproc, qproc->proxy_regs,
 				    qproc->proxy_reg_count);
 	if (ret) {
 		dev_err(qproc->dev, "failed to enable proxy supplies\n");
-		return ret;
+		goto disable_irqs;
 	}
 
 	ret = q6v5_clk_enable(qproc->dev, qproc->proxy_clks,
@@ -747,12 +822,20 @@ static int q6v5_start(struct rproc *rproc)
 		dev_err(qproc->dev, "failed to enable supplies\n");
 		goto disable_proxy_clk;
 	}
-	ret = reset_control_deassert(qproc->mss_restart);
+
+	ret = q6v5_clk_enable(qproc->dev, qproc->reset_clks,
+			      qproc->reset_clk_count);
 	if (ret) {
-		dev_err(qproc->dev, "failed to deassert mss restart\n");
+		dev_err(qproc->dev, "failed to enable reset clocks\n");
 		goto disable_vdd;
 	}
 
+	ret = q6v5_reset_deassert(qproc);
+	if (ret) {
+		dev_err(qproc->dev, "failed to deassert mss restart\n");
+		goto disable_reset_clks;
+	}
+
 	ret = q6v5_clk_enable(qproc->dev, qproc->active_clks,
 			      qproc->active_clk_count);
 	if (ret) {
@@ -761,13 +844,11 @@ static int q6v5_start(struct rproc *rproc)
 	}
 
 	/* Assign MBA image access in DDR to q6 */
-	xfermemop_ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, true,
-						qproc->mba_phys,
-						qproc->mba_size);
-	if (xfermemop_ret) {
+	ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, true,
+				      qproc->mba_phys, qproc->mba_size);
+	if (ret) {
 		dev_err(qproc->dev,
-			"assigning Q6 access to mba memory failed: %d\n",
-			xfermemop_ret);
+			"assigning Q6 access to mba memory failed: %d\n", ret);
 		goto disable_active_clks;
 	}
 
@@ -810,11 +891,6 @@ static int q6v5_start(struct rproc *rproc)
 			"Failed to reclaim mba buffer system may become unstable\n");
 	qproc->running = true;
 
-	q6v5_clk_disable(qproc->dev, qproc->proxy_clks,
-			 qproc->proxy_clk_count);
-	q6v5_regulator_disable(qproc, qproc->proxy_regs,
-			       qproc->proxy_reg_count);
-
 	return 0;
 
 reclaim_mpss:
@@ -842,7 +918,10 @@ static int q6v5_start(struct rproc *rproc)
 			 qproc->active_clk_count);
 
 assert_reset:
-	reset_control_assert(qproc->mss_restart);
+	q6v5_reset_assert(qproc);
+disable_reset_clks:
+	q6v5_clk_disable(qproc->dev, qproc->reset_clks,
+			 qproc->reset_clk_count);
 disable_vdd:
 	q6v5_regulator_disable(qproc, qproc->active_regs,
 			       qproc->active_reg_count);
@@ -853,6 +932,9 @@ static int q6v5_start(struct rproc *rproc)
 	q6v5_regulator_disable(qproc, qproc->proxy_regs,
 			       qproc->proxy_reg_count);
 
+disable_irqs:
+	disable_irq(qproc->handover_irq);
+
 	return ret;
 }
 
@@ -892,7 +974,19 @@ static int q6v5_stop(struct rproc *rproc)
 				      qproc->mpss_phys, qproc->mpss_size);
 	WARN_ON(ret);
 
-	reset_control_assert(qproc->mss_restart);
+	q6v5_reset_assert(qproc);
+
+	disable_irq(qproc->handover_irq);
+
+	if (!qproc->proxy_unvoted) {
+		q6v5_clk_disable(qproc->dev, qproc->proxy_clks,
+				 qproc->proxy_clk_count);
+		q6v5_regulator_disable(qproc, qproc->proxy_regs,
+				       qproc->proxy_reg_count);
+	}
+
+	q6v5_clk_disable(qproc->dev, qproc->reset_clks,
+			 qproc->reset_clk_count);
 	q6v5_clk_disable(qproc->dev, qproc->active_clks,
 			 qproc->active_clk_count);
 	q6v5_regulator_disable(qproc, qproc->active_regs,
@@ -960,7 +1054,7 @@ static irqreturn_t q6v5_fatal_interrupt(int irq, void *dev)
 	return IRQ_HANDLED;
 }
 
-static irqreturn_t q6v5_handover_interrupt(int irq, void *dev)
+static irqreturn_t q6v5_ready_interrupt(int irq, void *dev)
 {
 	struct q6v5 *qproc = dev;
 
@@ -968,6 +1062,20 @@ static irqreturn_t q6v5_handover_interrupt(int irq, void *dev)
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t q6v5_handover_interrupt(int irq, void *dev)
+{
+	struct q6v5 *qproc = dev;
+
+	q6v5_clk_disable(qproc->dev, qproc->proxy_clks,
+			 qproc->proxy_clk_count);
+	q6v5_regulator_disable(qproc, qproc->proxy_regs,
+			       qproc->proxy_reg_count);
+
+	qproc->proxy_unvoted = true;
+
+	return IRQ_HANDLED;
+}
+
 static irqreturn_t q6v5_stop_ack_interrupt(int irq, void *dev)
 {
 	struct q6v5 *qproc = dev;
@@ -1051,22 +1159,23 @@ static int q6v5_request_irq(struct q6v5 *qproc,
 			     const char *name,
 			     irq_handler_t thread_fn)
 {
+	int irq;
 	int ret;
 
-	ret = platform_get_irq_byname(pdev, name);
-	if (ret < 0) {
+	irq = platform_get_irq_byname(pdev, name);
+	if (irq < 0) {
 		dev_err(&pdev->dev, "no %s IRQ defined\n", name);
-		return ret;
+		return irq;
 	}
 
-	ret = devm_request_threaded_irq(&pdev->dev, ret,
+	ret = devm_request_threaded_irq(&pdev->dev, irq,
 					NULL, thread_fn,
 					IRQF_TRIGGER_RISING | IRQF_ONESHOT,
 					"q6v5", qproc);
 	if (ret)
 		dev_err(&pdev->dev, "request %s IRQ failed\n", name);
 
-	return ret;
+	return ret ? : irq;
 }
 
 static int q6v5_alloc_memory_region(struct q6v5 *qproc)
@@ -1157,6 +1266,14 @@ static int q6v5_probe(struct platform_device *pdev)
 	}
 	qproc->proxy_clk_count = ret;
 
+	ret = q6v5_init_clocks(&pdev->dev, qproc->reset_clks,
+			       desc->reset_clk_names);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Failed to get reset clocks.\n");
+		goto free_rproc;
+	}
+	qproc->reset_clk_count = ret;
+
 	ret = q6v5_init_clocks(&pdev->dev, qproc->active_clks,
 			       desc->active_clk_names);
 	if (ret < 0) {
@@ -1186,6 +1303,7 @@ static int q6v5_probe(struct platform_device *pdev)
 		goto free_rproc;
 
 	qproc->version = desc->version;
+	qproc->has_alt_reset = desc->has_alt_reset;
 	qproc->need_mem_protection = desc->need_mem_protection;
 	ret = q6v5_request_irq(qproc, pdev, "wdog", q6v5_wdog_interrupt);
 	if (ret < 0)
@@ -1195,9 +1313,15 @@ static int q6v5_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto free_rproc;
 
+	ret = q6v5_request_irq(qproc, pdev, "ready", q6v5_ready_interrupt);
+	if (ret < 0)
+		goto free_rproc;
+
 	ret = q6v5_request_irq(qproc, pdev, "handover", q6v5_handover_interrupt);
 	if (ret < 0)
 		goto free_rproc;
+	qproc->handover_irq = ret;
+	disable_irq(qproc->handover_irq);
 
 	ret = q6v5_request_irq(qproc, pdev, "stop-ack", q6v5_stop_ack_interrupt);
 	if (ret < 0)
@@ -1210,6 +1334,7 @@ static int q6v5_probe(struct platform_device *pdev)
 	}
 	qproc->mpss_perm = BIT(QCOM_SCM_VMID_HLOS);
 	qproc->mba_perm = BIT(QCOM_SCM_VMID_HLOS);
+	qcom_add_glink_subdev(rproc, &qproc->glink_subdev);
 	qcom_add_smd_subdev(rproc, &qproc->smd_subdev);
 	qcom_add_ssr_subdev(rproc, &qproc->ssr_subdev, "mpss");
 	qproc->sysmon = qcom_add_sysmon_subdev(rproc, "modem", 0x12);
@@ -1233,6 +1358,7 @@ static int q6v5_remove(struct platform_device *pdev)
 	rproc_del(qproc->rproc);
 
 	qcom_remove_sysmon_subdev(qproc->sysmon);
+	qcom_remove_glink_subdev(qproc->rproc, &qproc->glink_subdev);
 	qcom_remove_smd_subdev(qproc->rproc, &qproc->smd_subdev);
 	qcom_remove_ssr_subdev(qproc->rproc, &qproc->ssr_subdev);
 	rproc_free(qproc->rproc);
@@ -1240,6 +1366,31 @@ static int q6v5_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct rproc_hexagon_res sdm845_mss = {
+	.hexagon_mba_image = "mba.mbn",
+	.proxy_clk_names = (char*[]){
+			"xo",
+			"axis2",
+			"prng",
+			NULL
+	},
+	.reset_clk_names = (char*[]){
+			"iface",
+			"snoc_axi",
+			NULL
+	},
+	.active_clk_names = (char*[]){
+			"bus",
+			"mem",
+			"gpll0_mss",
+			"mnoc_axi",
+			NULL
+	},
+	.need_mem_protection = true,
+	.has_alt_reset = true,
+	.version = MSS_SDM845,
+};
+
 static const struct rproc_hexagon_res msm8996_mss = {
 	.hexagon_mba_image = "mba.mbn",
 	.proxy_clk_names = (char*[]){
@@ -1255,6 +1406,7 @@ static const struct rproc_hexagon_res msm8996_mss = {
 			NULL
 	},
 	.need_mem_protection = true,
+	.has_alt_reset = false,
 	.version = MSS_MSM8996,
 };
 
@@ -1286,6 +1438,7 @@ static const struct rproc_hexagon_res msm8916_mss = {
 		NULL
 	},
 	.need_mem_protection = false,
+	.has_alt_reset = false,
 	.version = MSS_MSM8916,
 };
 
@@ -1325,6 +1478,7 @@ static const struct rproc_hexagon_res msm8974_mss = {
 		NULL
 	},
 	.need_mem_protection = false,
+	.has_alt_reset = false,
 	.version = MSS_MSM8974,
 };
 
@@ -1333,6 +1487,7 @@ static const struct of_device_id q6v5_of_match[] = {
 	{ .compatible = "qcom,msm8916-mss-pil", .data = &msm8916_mss},
 	{ .compatible = "qcom,msm8974-mss-pil", .data = &msm8974_mss},
 	{ .compatible = "qcom,msm8996-mss-pil", .data = &msm8996_mss},
+	{ .compatible = "qcom,sdm845-mss-pil", .data = &sdm845_mss},
 	{ },
 };
 MODULE_DEVICE_TABLE(of, q6v5_of_match);
diff --git a/drivers/reset/reset-uniphier.c b/drivers/reset/reset-uniphier.c
index ac18f2f27881..e9030ff1bf2f 100644
--- a/drivers/reset/reset-uniphier.c
+++ b/drivers/reset/reset-uniphier.c
@@ -63,6 +63,9 @@ static const struct uniphier_reset_data uniphier_pro4_sys_reset_data[] = {
 	UNIPHIER_RESETX(12, 0x2000, 6),		/* GIO (Ether, SATA, USB3) */
 	UNIPHIER_RESETX(14, 0x2000, 17),	/* USB30 */
 	UNIPHIER_RESETX(15, 0x2004, 17),	/* USB31 */
+	UNIPHIER_RESETX(28, 0x2000, 18),	/* SATA0 */
+	UNIPHIER_RESETX(29, 0x2004, 18),	/* SATA1 */
+	UNIPHIER_RESETX(30, 0x2000, 19),	/* SATA-PHY */
 	UNIPHIER_RESETX(40, 0x2000, 13),	/* AIO */
 	UNIPHIER_RESET_END,
 };
@@ -73,6 +76,7 @@ static const struct uniphier_reset_data uniphier_pro5_sys_reset_data[] = {
 	UNIPHIER_RESETX(12, 0x2000, 6),		/* GIO (PCIe, USB3) */
 	UNIPHIER_RESETX(14, 0x2000, 17),	/* USB30 */
 	UNIPHIER_RESETX(15, 0x2004, 17),	/* USB31 */
+	UNIPHIER_RESETX(24, 0x2008, 2),		/* PCIe */
 	UNIPHIER_RESETX(40, 0x2000, 13),	/* AIO */
 	UNIPHIER_RESET_END,
 };
@@ -89,7 +93,7 @@ static const struct uniphier_reset_data uniphier_pxs2_sys_reset_data[] = {
 	UNIPHIER_RESETX(20, 0x2014, 5),		/* USB31-PHY0 */
 	UNIPHIER_RESETX(21, 0x2014, 1),		/* USB31-PHY1 */
 	UNIPHIER_RESETX(28, 0x2014, 12),	/* SATA */
-	UNIPHIER_RESET(29, 0x2014, 8),		/* SATA-PHY (active high) */
+	UNIPHIER_RESET(30, 0x2014, 8),		/* SATA-PHY (active high) */
 	UNIPHIER_RESETX(40, 0x2000, 13),	/* AIO */
 	UNIPHIER_RESET_END,
 };
@@ -99,6 +103,7 @@ static const struct uniphier_reset_data uniphier_ld11_sys_reset_data[] = {
 	UNIPHIER_RESETX(4, 0x200c, 2),		/* eMMC */
 	UNIPHIER_RESETX(6, 0x200c, 6),		/* Ether */
 	UNIPHIER_RESETX(8, 0x200c, 8),		/* STDMAC (HSC, MIO) */
+	UNIPHIER_RESETX(9, 0x200c, 9),		/* HSC */
 	UNIPHIER_RESETX(40, 0x2008, 0),		/* AIO */
 	UNIPHIER_RESETX(41, 0x2008, 1),		/* EVEA */
 	UNIPHIER_RESETX(42, 0x2010, 2),		/* EXIV */
@@ -110,11 +115,13 @@ static const struct uniphier_reset_data uniphier_ld20_sys_reset_data[] = {
 	UNIPHIER_RESETX(4, 0x200c, 2),		/* eMMC */
 	UNIPHIER_RESETX(6, 0x200c, 6),		/* Ether */
 	UNIPHIER_RESETX(8, 0x200c, 8),		/* STDMAC (HSC) */
+	UNIPHIER_RESETX(9, 0x200c, 9),		/* HSC */
 	UNIPHIER_RESETX(14, 0x200c, 5),		/* USB30 */
 	UNIPHIER_RESETX(16, 0x200c, 12),	/* USB30-PHY0 */
 	UNIPHIER_RESETX(17, 0x200c, 13),	/* USB30-PHY1 */
 	UNIPHIER_RESETX(18, 0x200c, 14),	/* USB30-PHY2 */
 	UNIPHIER_RESETX(19, 0x200c, 15),	/* USB30-PHY3 */
+	UNIPHIER_RESETX(24, 0x200c, 4),		/* PCIe */
 	UNIPHIER_RESETX(40, 0x2008, 0),		/* AIO */
 	UNIPHIER_RESETX(41, 0x2008, 1),		/* EVEA */
 	UNIPHIER_RESETX(42, 0x2010, 2),		/* EXIV */
@@ -134,6 +141,10 @@ static const struct uniphier_reset_data uniphier_pxs3_sys_reset_data[] = {
 	UNIPHIER_RESETX(18, 0x200c, 20),	/* USB30-PHY2 */
 	UNIPHIER_RESETX(20, 0x200c, 17),	/* USB31-PHY0 */
 	UNIPHIER_RESETX(21, 0x200c, 19),	/* USB31-PHY1 */
+	UNIPHIER_RESETX(24, 0x200c, 3),		/* PCIe */
+	UNIPHIER_RESETX(28, 0x200c, 7),		/* SATA0 */
+	UNIPHIER_RESETX(29, 0x200c, 8),		/* SATA1 */
+	UNIPHIER_RESETX(30, 0x200c, 21),	/* SATA-PHY */
 	UNIPHIER_RESET_END,
 };
 
diff --git a/drivers/rpmsg/Kconfig b/drivers/rpmsg/Kconfig
index 65a9f6b892f0..9b14090e3603 100644
--- a/drivers/rpmsg/Kconfig
+++ b/drivers/rpmsg/Kconfig
@@ -39,6 +39,7 @@ config RPMSG_QCOM_GLINK_SMEM
 
 config RPMSG_QCOM_SMD
 	tristate "Qualcomm Shared Memory Driver (SMD)"
+	depends on MAILBOX
 	depends on QCOM_SMEM
 	select RPMSG
 	help
diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c
index 5ce9bf7b897d..b07e9a1a9d22 100644
--- a/drivers/rpmsg/qcom_smd.c
+++ b/drivers/rpmsg/qcom_smd.c
@@ -14,6 +14,7 @@
 
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/mailbox_client.h>
 #include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/of_irq.h>
@@ -107,6 +108,8 @@ static const struct {
  * @ipc_regmap:		regmap handle holding the outgoing ipc register
  * @ipc_offset:		offset within @ipc_regmap of the register for ipc
  * @ipc_bit:		bit in the register at @ipc_offset of @ipc_regmap
+ * @mbox_client:	mailbox client handle
+ * @mbox_chan:		apcs ipc mailbox channel handle
  * @channels:		list of all channels detected on this edge
  * @channels_lock:	guard for modifications of @channels
  * @allocated:		array of bitmaps representing already allocated channels
@@ -129,6 +132,9 @@ struct qcom_smd_edge {
 	int ipc_offset;
 	int ipc_bit;
 
+	struct mbox_client mbox_client;
+	struct mbox_chan *mbox_chan;
+
 	struct list_head channels;
 	spinlock_t channels_lock;
 
@@ -366,7 +372,17 @@ static void qcom_smd_signal_channel(struct qcom_smd_channel *channel)
 {
 	struct qcom_smd_edge *edge = channel->edge;
 
-	regmap_write(edge->ipc_regmap, edge->ipc_offset, BIT(edge->ipc_bit));
+	if (edge->mbox_chan) {
+		/*
+		 * We can ignore a failing mbox_send_message() as the only
+		 * possible cause is that the FIFO in the framework is full of
+		 * other writes to the same bit.
+		 */
+		mbox_send_message(edge->mbox_chan, NULL);
+		mbox_client_txdone(edge->mbox_chan, 0);
+	} else {
+		regmap_write(edge->ipc_regmap, edge->ipc_offset, BIT(edge->ipc_bit));
+	}
 }
 
 /*
@@ -1326,27 +1342,37 @@ static int qcom_smd_parse_edge(struct device *dev,
 	key = "qcom,remote-pid";
 	of_property_read_u32(node, key, &edge->remote_pid);
 
-	syscon_np = of_parse_phandle(node, "qcom,ipc", 0);
-	if (!syscon_np) {
-		dev_err(dev, "no qcom,ipc node\n");
-		return -ENODEV;
-	}
+	edge->mbox_client.dev = dev;
+	edge->mbox_client.knows_txdone = true;
+	edge->mbox_chan = mbox_request_channel(&edge->mbox_client, 0);
+	if (IS_ERR(edge->mbox_chan)) {
+		if (PTR_ERR(edge->mbox_chan) != -ENODEV)
+			return PTR_ERR(edge->mbox_chan);
 
-	edge->ipc_regmap = syscon_node_to_regmap(syscon_np);
-	if (IS_ERR(edge->ipc_regmap))
-		return PTR_ERR(edge->ipc_regmap);
+		edge->mbox_chan = NULL;
 
-	key = "qcom,ipc";
-	ret = of_property_read_u32_index(node, key, 1, &edge->ipc_offset);
-	if (ret < 0) {
-		dev_err(dev, "no offset in %s\n", key);
-		return -EINVAL;
-	}
+		syscon_np = of_parse_phandle(node, "qcom,ipc", 0);
+		if (!syscon_np) {
+			dev_err(dev, "no qcom,ipc node\n");
+			return -ENODEV;
+		}
 
-	ret = of_property_read_u32_index(node, key, 2, &edge->ipc_bit);
-	if (ret < 0) {
-		dev_err(dev, "no bit in %s\n", key);
-		return -EINVAL;
+		edge->ipc_regmap = syscon_node_to_regmap(syscon_np);
+		if (IS_ERR(edge->ipc_regmap))
+			return PTR_ERR(edge->ipc_regmap);
+
+		key = "qcom,ipc";
+		ret = of_property_read_u32_index(node, key, 1, &edge->ipc_offset);
+		if (ret < 0) {
+			dev_err(dev, "no offset in %s\n", key);
+			return -EINVAL;
+		}
+
+		ret = of_property_read_u32_index(node, key, 2, &edge->ipc_bit);
+		if (ret < 0) {
+			dev_err(dev, "no bit in %s\n", key);
+			return -EINVAL;
+		}
 	}
 
 	ret = of_property_read_string(node, "label", &edge->name);
@@ -1453,6 +1479,9 @@ struct qcom_smd_edge *qcom_smd_register_edge(struct device *parent,
 	return edge;
 
 unregister_dev:
+	if (!IS_ERR_OR_NULL(edge->mbox_chan))
+		mbox_free_channel(edge->mbox_chan);
+
 	device_unregister(&edge->dev);
 	return ERR_PTR(ret);
 }
@@ -1481,6 +1510,7 @@ int qcom_smd_unregister_edge(struct qcom_smd_edge *edge)
 	if (ret)
 		dev_warn(&edge->dev, "can't remove smd device: %d\n", ret);
 
+	mbox_free_channel(edge->mbox_chan);
 	device_unregister(&edge->dev);
 
 	return 0;
diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c
index 920a02f0462c..bea7c74e12eb 100644
--- a/drivers/rpmsg/rpmsg_core.c
+++ b/drivers/rpmsg/rpmsg_core.c
@@ -333,11 +333,49 @@ field##_show(struct device *dev,					\
 }									\
 static DEVICE_ATTR_RO(field);
 
+#define rpmsg_string_attr(field, member)				\
+static ssize_t								\
+field##_store(struct device *dev, struct device_attribute *attr,	\
+	      const char *buf, size_t sz)				\
+{									\
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);		\
+	char *new, *old;						\
+									\
+	new = kstrndup(buf, sz, GFP_KERNEL);				\
+	if (!new)							\
+		return -ENOMEM;						\
+	new[strcspn(new, "\n")] = '\0';					\
+									\
+	device_lock(dev);						\
+	old = rpdev->member;						\
+	if (strlen(new)) {						\
+		rpdev->member = new;					\
+	} else {							\
+		kfree(new);						\
+		rpdev->member = NULL;					\
+	}								\
+	device_unlock(dev);						\
+									\
+	kfree(old);							\
+									\
+	return sz;							\
+}									\
+static ssize_t								\
+field##_show(struct device *dev,					\
+	     struct device_attribute *attr, char *buf)			\
+{									\
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);		\
+									\
+	return sprintf(buf, "%s\n", rpdev->member);			\
+}									\
+static DEVICE_ATTR_RW(field)
+
 /* for more info, see Documentation/ABI/testing/sysfs-bus-rpmsg */
 rpmsg_show_attr(name, id.name, "%s\n");
 rpmsg_show_attr(src, src, "0x%x\n");
 rpmsg_show_attr(dst, dst, "0x%x\n");
 rpmsg_show_attr(announce, announce ? "true" : "false", "%s\n");
+rpmsg_string_attr(driver_override, driver_override);
 
 static ssize_t modalias_show(struct device *dev,
 			     struct device_attribute *attr, char *buf)
@@ -359,6 +397,7 @@ static struct attribute *rpmsg_dev_attrs[] = {
 	&dev_attr_dst.attr,
 	&dev_attr_src.attr,
 	&dev_attr_announce.attr,
+	&dev_attr_driver_override.attr,
 	NULL,
 };
 ATTRIBUTE_GROUPS(rpmsg_dev);
diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig
index 59e6dede3db3..a2ba5db36145 100644
--- a/drivers/rtc/Kconfig
+++ b/drivers/rtc/Kconfig
@@ -1613,7 +1613,7 @@ config RTC_DRV_JZ4740
 	  If you say yes here you get support for the Ingenic JZ47xx SoCs RTC
 	  controllers.
 
-	  This driver can also be buillt as a module. If so, the module
+	  This driver can also be built as a module. If so, the module
 	  will be called rtc-jz4740.
 
 config RTC_DRV_LPC24XX
diff --git a/drivers/rtc/nvmem.c b/drivers/rtc/nvmem.c
index 17ec4c8d0fad..36ab183c42f1 100644
--- a/drivers/rtc/nvmem.c
+++ b/drivers/rtc/nvmem.c
@@ -94,7 +94,7 @@ int rtc_nvmem_register(struct rtc_device *rtc,
 	nvmem_config->dev = rtc->dev.parent;
 	nvmem_config->owner = rtc->owner;
 	rtc->nvmem = nvmem_register(nvmem_config);
-	if (IS_ERR_OR_NULL(rtc->nvmem))
+	if (IS_ERR(rtc->nvmem))
 		return PTR_ERR(rtc->nvmem);
 
 	/* Register the old ABI */
diff --git a/drivers/rtc/rtc-88pm80x.c b/drivers/rtc/rtc-88pm80x.c
index 6cbafefa80a2..cab293cb2bf0 100644
--- a/drivers/rtc/rtc-88pm80x.c
+++ b/drivers/rtc/rtc-88pm80x.c
@@ -52,10 +52,8 @@ struct pm80x_rtc_info {
 	struct regmap *map;
 	struct rtc_device *rtc_dev;
 	struct device *dev;
-	struct delayed_work calib_work;
 
 	int irq;
-	int vrtc;
 };
 
 static irqreturn_t rtc_update_handler(int irq, void *data)
@@ -100,13 +98,13 @@ static void rtc_next_alarm_time(struct rtc_time *next, struct rtc_time *now,
 	next->tm_min = alrm->tm_min;
 	next->tm_sec = alrm->tm_sec;
 
-	rtc_tm_to_time(now, &now_time);
-	rtc_tm_to_time(next, &next_time);
+	now_time = rtc_tm_to_time64(now);
+	next_time = rtc_tm_to_time64(next);
 
 	if (next_time < now_time) {
 		/* Advance one day */
 		next_time += 60 * 60 * 24;
-		rtc_time_to_tm(next_time, next);
+		rtc_time64_to_tm(next_time, next);
 	}
 }
 
@@ -125,7 +123,7 @@ static int pm80x_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	ticks = base + data;
 	dev_dbg(info->dev, "get base:0x%lx, RO count:0x%lx, ticks:0x%lx\n",
 		base, data, ticks);
-	rtc_time_to_tm(ticks, tm);
+	rtc_time64_to_tm(ticks, tm);
 	return 0;
 }
 
@@ -134,13 +132,8 @@ static int pm80x_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	struct pm80x_rtc_info *info = dev_get_drvdata(dev);
 	unsigned char buf[4];
 	unsigned long ticks, base, data;
-	if (tm->tm_year > 206) {
-		dev_dbg(info->dev,
-			"Set time %d out of range. Please set time between 1970 to 2106.\n",
-			1900 + tm->tm_year);
-		return -EINVAL;
-	}
-	rtc_tm_to_time(tm, &ticks);
+
+	ticks = rtc_tm_to_time64(tm);
 
 	/* load 32-bit read-only counter */
 	regmap_raw_read(info->map, PM800_RTC_COUNTER1, buf, 4);
@@ -174,7 +167,7 @@ static int pm80x_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	dev_dbg(info->dev, "get base:0x%lx, RO count:0x%lx, ticks:0x%lx\n",
 		base, data, ticks);
 
-	rtc_time_to_tm(ticks, &alrm->time);
+	rtc_time64_to_tm(ticks, &alrm->time);
 	regmap_read(info->map, PM800_RTC_CONTROL, &ret);
 	alrm->enabled = (ret & PM800_ALARM1_EN) ? 1 : 0;
 	alrm->pending = (ret & (PM800_ALARM | PM800_ALARM_WAKEUP)) ? 1 : 0;
@@ -202,11 +195,11 @@ static int pm80x_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	dev_dbg(info->dev, "get base:0x%lx, RO count:0x%lx, ticks:0x%lx\n",
 		base, data, ticks);
 
-	rtc_time_to_tm(ticks, &now_tm);
+	rtc_time64_to_tm(ticks, &now_tm);
 	dev_dbg(info->dev, "%s, now time : %lu\n", __func__, ticks);
 	rtc_next_alarm_time(&alarm_tm, &now_tm, &alrm->time);
 	/* get new ticks for alarm in 24 hours */
-	rtc_tm_to_time(&alarm_tm, &ticks);
+	ticks = rtc_tm_to_time64(&alarm_tm);
 	dev_dbg(info->dev, "%s, alarm time: %lu\n", __func__, ticks);
 	data = ticks - base;
 
@@ -254,8 +247,6 @@ static int pm80x_rtc_probe(struct platform_device *pdev)
 	struct pm80x_rtc_pdata *pdata = dev_get_platdata(&pdev->dev);
 	struct pm80x_rtc_info *info;
 	struct device_node *node = pdev->dev.of_node;
-	struct rtc_time tm;
-	unsigned long ticks = 0;
 	int ret;
 
 	if (!pdata && !node) {
@@ -294,6 +285,10 @@ static int pm80x_rtc_probe(struct platform_device *pdev)
 	info->dev = &pdev->dev;
 	dev_set_drvdata(&pdev->dev, info);
 
+	info->rtc_dev = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(info->rtc_dev))
+		return PTR_ERR(info->rtc_dev);
+
 	ret = pm80x_request_irq(chip, info->irq, rtc_update_handler,
 				IRQF_ONESHOT, "rtc", info);
 	if (ret < 0) {
@@ -302,30 +297,11 @@ static int pm80x_rtc_probe(struct platform_device *pdev)
 		goto out;
 	}
 
-	ret = pm80x_rtc_read_time(&pdev->dev, &tm);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "Failed to read initial time.\n");
-		goto out_rtc;
-	}
-	if ((tm.tm_year < 70) || (tm.tm_year > 138)) {
-		tm.tm_year = 70;
-		tm.tm_mon = 0;
-		tm.tm_mday = 1;
-		tm.tm_hour = 0;
-		tm.tm_min = 0;
-		tm.tm_sec = 0;
-		ret = pm80x_rtc_set_time(&pdev->dev, &tm);
-		if (ret < 0) {
-			dev_err(&pdev->dev, "Failed to set initial time.\n");
-			goto out_rtc;
-		}
-	}
-	rtc_tm_to_time(&tm, &ticks);
+	info->rtc_dev->ops = &pm80x_rtc_ops;
+	info->rtc_dev->range_max = U32_MAX;
 
-	info->rtc_dev = devm_rtc_device_register(&pdev->dev, "88pm80x-rtc",
-					    &pm80x_rtc_ops, THIS_MODULE);
-	if (IS_ERR(info->rtc_dev)) {
-		ret = PTR_ERR(info->rtc_dev);
+	ret = rtc_register_device(info->rtc_dev);
+	if (ret) {
 		dev_err(&pdev->dev, "Failed to register RTC device: %d\n", ret);
 		goto out_rtc;
 	}
diff --git a/drivers/rtc/rtc-ab-b5ze-s3.c b/drivers/rtc/rtc-ab-b5ze-s3.c
index 8dc451932446..2233601761ac 100644
--- a/drivers/rtc/rtc-ab-b5ze-s3.c
+++ b/drivers/rtc/rtc-ab-b5ze-s3.c
@@ -265,15 +265,6 @@ static int abb5zes3_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	u8 regs[ABB5ZES3_REG_RTC_SC + ABB5ZES3_RTC_SEC_LEN];
 	int ret;
 
-	/*
-	 * Year register is 8-bit wide and bcd-coded, i.e records values
-	 * between 0 and 99. tm_year is an offset from 1900 and we are
-	 * interested in the 2000-2099 range, so any value less than 100
-	 * is invalid.
-	 */
-	if (tm->tm_year < 100)
-		return -EINVAL;
-
 	regs[ABB5ZES3_REG_RTC_SC] = bin2bcd(tm->tm_sec); /* MSB=0 clears OSC */
 	regs[ABB5ZES3_REG_RTC_MN] = bin2bcd(tm->tm_min);
 	regs[ABB5ZES3_REG_RTC_HR] = bin2bcd(tm->tm_hour); /* 24-hour format */
@@ -925,6 +916,14 @@ static int abb5zes3_probe(struct i2c_client *client,
 	if (ret)
 		goto err;
 
+	data->rtc = devm_rtc_allocate_device(dev);
+	ret = PTR_ERR_OR_ZERO(data->rtc);
+	if (ret) {
+		dev_err(dev, "%s: unable to allocate RTC device (%d)\n",
+			__func__, ret);
+		goto err;
+	}
+
 	if (client->irq > 0) {
 		ret = devm_request_threaded_irq(dev, client->irq, NULL,
 						_abb5zes3_rtc_interrupt,
@@ -942,14 +941,9 @@ static int abb5zes3_probe(struct i2c_client *client,
 		}
 	}
 
-	data->rtc = devm_rtc_device_register(dev, DRV_NAME, &rtc_ops,
-					     THIS_MODULE);
-	ret = PTR_ERR_OR_ZERO(data->rtc);
-	if (ret) {
-		dev_err(dev, "%s: unable to register RTC device (%d)\n",
-			__func__, ret);
-		goto err;
-	}
+	data->rtc->ops = &rtc_ops;
+	data->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000;
+	data->rtc->range_max = RTC_TIMESTAMP_END_2099;
 
 	/* Enable battery low detection interrupt if battery not already low */
 	if (!data->battery_low && data->irq) {
@@ -961,6 +955,8 @@ static int abb5zes3_probe(struct i2c_client *client,
 		}
 	}
 
+	ret = rtc_register_device(data->rtc);
+
 err:
 	if (ret && data && data->irq)
 		device_init_wakeup(dev, false);
diff --git a/drivers/rtc/rtc-at91rm9200.c b/drivers/rtc/rtc-at91rm9200.c
index de81ecedd571..caa71d04e989 100644
--- a/drivers/rtc/rtc-at91rm9200.c
+++ b/drivers/rtc/rtc-at91rm9200.c
@@ -440,6 +440,8 @@ static int __init at91_rtc_probe(struct platform_device *pdev)
 		device_init_wakeup(&pdev->dev, 1);
 
 	rtc->ops = &at91_rtc_ops;
+	rtc->range_min = RTC_TIMESTAMP_BEGIN_1900;
+	rtc->range_max = RTC_TIMESTAMP_END_2099;
 	ret = rtc_register_device(rtc);
 	if (ret)
 		goto err_clk;
diff --git a/drivers/rtc/rtc-bq4802.c b/drivers/rtc/rtc-bq4802.c
index bd170cb3361c..d768f6747961 100644
--- a/drivers/rtc/rtc-bq4802.c
+++ b/drivers/rtc/rtc-bq4802.c
@@ -48,8 +48,7 @@ static void bq4802_write_mem(struct bq4802 *p, int off, u8 val)
 
 static int bq4802_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct bq4802 *p = platform_get_drvdata(pdev);
+	struct bq4802 *p = dev_get_drvdata(dev);
 	unsigned long flags;
 	unsigned int century;
 	u8 val;
@@ -91,8 +90,7 @@ static int bq4802_read_time(struct device *dev, struct rtc_time *tm)
 
 static int bq4802_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct bq4802 *p = platform_get_drvdata(pdev);
+	struct bq4802 *p = dev_get_drvdata(dev);
 	u8 sec, min, hrs, day, mon, yrs, century, val;
 	unsigned long flags;
 	unsigned int year;
diff --git a/drivers/rtc/rtc-brcmstb-waketimer.c b/drivers/rtc/rtc-brcmstb-waketimer.c
index bdd6674a1054..f4010a75f2be 100644
--- a/drivers/rtc/rtc-brcmstb-waketimer.c
+++ b/drivers/rtc/rtc-brcmstb-waketimer.c
@@ -145,9 +145,6 @@ static int brcmstb_waketmr_settime(struct device *dev,
 
 	sec = rtc_tm_to_time64(tm);
 
-	if (sec > U32_MAX || sec < 0)
-		return -EINVAL;
-
 	writel_relaxed(sec, timer->base + BRCMSTB_WKTMR_COUNTER);
 
 	return 0;
@@ -184,9 +181,6 @@ static int brcmstb_waketmr_setalarm(struct device *dev,
 	else
 		sec = 0;
 
-	if (sec > U32_MAX || sec < 0)
-		return -EINVAL;
-
 	brcmstb_waketmr_set_alarm(timer, sec);
 
 	return 0;
@@ -229,6 +223,10 @@ static int brcmstb_waketmr_probe(struct platform_device *pdev)
 	if (IS_ERR(timer->base))
 		return PTR_ERR(timer->base);
 
+	timer->rtc = devm_rtc_allocate_device(dev);
+	if (IS_ERR(timer->rtc))
+		return PTR_ERR(timer->rtc);
+
 	/*
 	 * Set wakeup capability before requesting wakeup interrupt, so we can
 	 * process boot-time "wakeups" (e.g., from S5 soft-off)
@@ -261,11 +259,12 @@ static int brcmstb_waketmr_probe(struct platform_device *pdev)
 	timer->reboot_notifier.notifier_call = brcmstb_waketmr_reboot;
 	register_reboot_notifier(&timer->reboot_notifier);
 
-	timer->rtc = rtc_device_register("brcmstb-waketmr", dev,
-					 &brcmstb_waketmr_ops, THIS_MODULE);
-	if (IS_ERR(timer->rtc)) {
+	timer->rtc->ops = &brcmstb_waketmr_ops;
+	timer->rtc->range_max = U32_MAX;
+
+	ret = rtc_register_device(timer->rtc);
+	if (ret) {
 		dev_err(dev, "unable to register device\n");
-		ret = PTR_ERR(timer->rtc);
 		goto err_notifier;
 	}
 
@@ -288,7 +287,6 @@ static int brcmstb_waketmr_remove(struct platform_device *pdev)
 	struct brcmstb_waketmr *timer = dev_get_drvdata(&pdev->dev);
 
 	unregister_reboot_notifier(&timer->reboot_notifier);
-	rtc_device_unregister(timer->rtc);
 
 	return 0;
 }
diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c
index 1b3738a11702..cd3a2411bc2f 100644
--- a/drivers/rtc/rtc-cmos.c
+++ b/drivers/rtc/rtc-cmos.c
@@ -43,11 +43,24 @@
 #include <linux/of_platform.h>
 #ifdef CONFIG_X86
 #include <asm/i8259.h>
+#include <asm/processor.h>
+#include <linux/dmi.h>
 #endif
 
 /* this is for "generic access to PC-style RTC" using CMOS_READ/CMOS_WRITE */
 #include <linux/mc146818rtc.h>
 
+/*
+ * Use ACPI SCI to replace HPET interrupt for RTC Alarm event
+ *
+ * If cleared, ACPI SCI is only used to wake up the system from suspend
+ *
+ * If set, ACPI SCI is used to handle UIE/AIE and system wakeup
+ */
+
+static bool use_acpi_alarm;
+module_param(use_acpi_alarm, bool, 0444);
+
 struct cmos_rtc {
 	struct rtc_device	*rtc;
 	struct device		*dev;
@@ -153,6 +166,12 @@ static inline int hpet_unregister_irq_handler(irq_handler_t handler)
 
 #endif
 
+/* Don't use HPET for RTC Alarm event if ACPI Fixed event is used */
+static int use_hpet_alarm(void)
+{
+	return is_hpet_enabled() && !use_acpi_alarm;
+}
+
 /*----------------------------------------------------------------*/
 
 #ifdef RTC_PORT
@@ -298,7 +317,7 @@ static void cmos_checkintr(struct cmos_rtc *cmos, unsigned char rtc_control)
 	 */
 	rtc_intr = CMOS_READ(RTC_INTR_FLAGS);
 
-	if (is_hpet_enabled())
+	if (use_hpet_alarm())
 		return;
 
 	rtc_intr &= (rtc_control & RTC_IRQMASK) | RTC_IRQF;
@@ -318,7 +337,13 @@ static void cmos_irq_enable(struct cmos_rtc *cmos, unsigned char mask)
 
 	rtc_control |= mask;
 	CMOS_WRITE(rtc_control, RTC_CONTROL);
-	hpet_set_rtc_irq_bit(mask);
+	if (use_hpet_alarm())
+		hpet_set_rtc_irq_bit(mask);
+
+	if ((mask & RTC_AIE) && use_acpi_alarm) {
+		if (cmos->wake_on)
+			cmos->wake_on(cmos->dev);
+	}
 
 	cmos_checkintr(cmos, rtc_control);
 }
@@ -330,7 +355,13 @@ static void cmos_irq_disable(struct cmos_rtc *cmos, unsigned char mask)
 	rtc_control = CMOS_READ(RTC_CONTROL);
 	rtc_control &= ~mask;
 	CMOS_WRITE(rtc_control, RTC_CONTROL);
-	hpet_mask_rtc_irq_bit(mask);
+	if (use_hpet_alarm())
+		hpet_mask_rtc_irq_bit(mask);
+
+	if ((mask & RTC_AIE) && use_acpi_alarm) {
+		if (cmos->wake_off)
+			cmos->wake_off(cmos->dev);
+	}
 
 	cmos_checkintr(cmos, rtc_control);
 }
@@ -448,10 +479,14 @@ static int cmos_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 			CMOS_WRITE(mon, cmos->mon_alrm);
 	}
 
-	/* FIXME the HPET alarm glue currently ignores day_alrm
-	 * and mon_alrm ...
-	 */
-	hpet_set_alarm_time(t->time.tm_hour, t->time.tm_min, t->time.tm_sec);
+	if (use_hpet_alarm()) {
+		/*
+		 * FIXME the HPET alarm glue currently ignores day_alrm
+		 * and mon_alrm ...
+		 */
+		hpet_set_alarm_time(t->time.tm_hour, t->time.tm_min,
+				    t->time.tm_sec);
+	}
 
 	if (t->enabled)
 		cmos_irq_enable(cmos, RTC_AIE);
@@ -508,7 +543,7 @@ static int cmos_procfs(struct device *dev, struct seq_file *seq)
 		   "batt_status\t: %s\n",
 		   (rtc_control & RTC_PIE) ? "yes" : "no",
 		   (rtc_control & RTC_UIE) ? "yes" : "no",
-		   is_hpet_enabled() ? "yes" : "no",
+		   use_hpet_alarm() ? "yes" : "no",
 		   // (rtc_control & RTC_SQWE) ? "yes" : "no",
 		   (rtc_control & RTC_DM_BINARY) ? "no" : "yes",
 		   (rtc_control & RTC_DST_EN) ? "yes" : "no",
@@ -614,7 +649,7 @@ static irqreturn_t cmos_interrupt(int irq, void *p)
 	 */
 	irqstat = CMOS_READ(RTC_INTR_FLAGS);
 	rtc_control = CMOS_READ(RTC_CONTROL);
-	if (is_hpet_enabled())
+	if (use_hpet_alarm())
 		irqstat = (unsigned long)irq & 0xF0;
 
 	/* If we were suspended, RTC_CONTROL may not be accurate since the
@@ -633,7 +668,8 @@ static irqreturn_t cmos_interrupt(int irq, void *p)
 		cmos_rtc.suspend_ctrl &= ~RTC_AIE;
 		rtc_control &= ~RTC_AIE;
 		CMOS_WRITE(rtc_control, RTC_CONTROL);
-		hpet_mask_rtc_irq_bit(RTC_AIE);
+		if (use_hpet_alarm())
+			hpet_mask_rtc_irq_bit(RTC_AIE);
 		CMOS_READ(RTC_INTR_FLAGS);
 	}
 	spin_unlock(&rtc_lock);
@@ -762,7 +798,8 @@ cmos_do_probe(struct device *dev, struct resource *ports, int rtc_irq)
 		 * need to do something about other clock frequencies.
 		 */
 		cmos_rtc.rtc->irq_freq = 1024;
-		hpet_set_periodic_freq(cmos_rtc.rtc->irq_freq);
+		if (use_hpet_alarm())
+			hpet_set_periodic_freq(cmos_rtc.rtc->irq_freq);
 		CMOS_WRITE(RTC_REF_CLCK_32KHZ | 0x06, RTC_FREQ_SELECT);
 	}
 
@@ -780,12 +817,13 @@ cmos_do_probe(struct device *dev, struct resource *ports, int rtc_irq)
 		goto cleanup1;
 	}
 
-	hpet_rtc_timer_init();
+	if (use_hpet_alarm())
+		hpet_rtc_timer_init();
 
 	if (is_valid_irq(rtc_irq)) {
 		irq_handler_t rtc_cmos_int_handler;
 
-		if (is_hpet_enabled()) {
+		if (use_hpet_alarm()) {
 			rtc_cmos_int_handler = hpet_rtc_interrupt;
 			retval = hpet_register_irq_handler(cmos_interrupt);
 			if (retval) {
@@ -824,7 +862,7 @@ cmos_do_probe(struct device *dev, struct resource *ports, int rtc_irq)
 		 "alarms up to one day",
 		 cmos_rtc.century ? ", y3k" : "",
 		 nvmem_cfg.size,
-		 is_hpet_enabled() ? ", hpet irqs" : "");
+		 use_hpet_alarm() ? ", hpet irqs" : "");
 
 	return 0;
 
@@ -858,7 +896,8 @@ static void cmos_do_remove(struct device *dev)
 
 	if (is_valid_irq(cmos->irq)) {
 		free_irq(cmos->irq, cmos->rtc);
-		hpet_unregister_irq_handler(cmos_interrupt);
+		if (use_hpet_alarm())
+			hpet_unregister_irq_handler(cmos_interrupt);
 	}
 
 	cmos->rtc = NULL;
@@ -935,13 +974,13 @@ static int cmos_suspend(struct device *dev)
 			mask = RTC_IRQMASK;
 		tmp &= ~mask;
 		CMOS_WRITE(tmp, RTC_CONTROL);
-		hpet_mask_rtc_irq_bit(mask);
-
+		if (use_hpet_alarm())
+			hpet_mask_rtc_irq_bit(mask);
 		cmos_checkintr(cmos, tmp);
 	}
 	spin_unlock_irq(&rtc_lock);
 
-	if (tmp & RTC_AIE) {
+	if ((tmp & RTC_AIE) && !use_acpi_alarm) {
 		cmos->enabled_wake = 1;
 		if (cmos->wake_on)
 			cmos->wake_on(dev);
@@ -976,8 +1015,26 @@ static void cmos_check_wkalrm(struct device *dev)
 {
 	struct cmos_rtc *cmos = dev_get_drvdata(dev);
 	struct rtc_wkalrm current_alarm;
+	time64_t t_now;
 	time64_t t_current_expires;
 	time64_t t_saved_expires;
+	struct rtc_time now;
+
+	/* Check if we have RTC Alarm armed */
+	if (!(cmos->suspend_ctrl & RTC_AIE))
+		return;
+
+	cmos_read_time(dev, &now);
+	t_now = rtc_tm_to_time64(&now);
+
+	/*
+	 * ACPI RTC wake event is cleared after resume from STR,
+	 * ACK the rtc irq here
+	 */
+	if (t_now >= cmos->alarm_expires && use_acpi_alarm) {
+		cmos_interrupt(0, (void *)cmos->rtc);
+		return;
+	}
 
 	cmos_read_alarm(dev, &current_alarm);
 	t_current_expires = rtc_tm_to_time64(&current_alarm.time);
@@ -996,7 +1053,7 @@ static int __maybe_unused cmos_resume(struct device *dev)
 	struct cmos_rtc	*cmos = dev_get_drvdata(dev);
 	unsigned char tmp;
 
-	if (cmos->enabled_wake) {
+	if (cmos->enabled_wake && !use_acpi_alarm) {
 		if (cmos->wake_off)
 			cmos->wake_off(dev);
 		else
@@ -1014,16 +1071,17 @@ static int __maybe_unused cmos_resume(struct device *dev)
 	if (tmp & RTC_IRQMASK) {
 		unsigned char	mask;
 
-		if (device_may_wakeup(dev))
+		if (device_may_wakeup(dev) && use_hpet_alarm())
 			hpet_rtc_timer_init();
 
 		do {
 			CMOS_WRITE(tmp, RTC_CONTROL);
-			hpet_set_rtc_irq_bit(tmp & RTC_IRQMASK);
+			if (use_hpet_alarm())
+				hpet_set_rtc_irq_bit(tmp & RTC_IRQMASK);
 
 			mask = CMOS_READ(RTC_INTR_FLAGS);
 			mask &= (tmp & RTC_IRQMASK) | RTC_IRQF;
-			if (!is_hpet_enabled() || !is_intr(mask))
+			if (!use_hpet_alarm() || !is_intr(mask))
 				break;
 
 			/* force one-shot behavior if HPET blocked
@@ -1068,16 +1126,27 @@ static u32 rtc_handler(void *context)
 	unsigned char rtc_intr;
 	unsigned long flags;
 
-	spin_lock_irqsave(&rtc_lock, flags);
-	if (cmos_rtc.suspend_ctrl)
-		rtc_control = CMOS_READ(RTC_CONTROL);
-	if (rtc_control & RTC_AIE) {
-		cmos_rtc.suspend_ctrl &= ~RTC_AIE;
-		CMOS_WRITE(rtc_control, RTC_CONTROL);
-		rtc_intr = CMOS_READ(RTC_INTR_FLAGS);
-		rtc_update_irq(cmos->rtc, 1, rtc_intr);
+
+	/*
+	 * Always update rtc irq when ACPI is used as RTC Alarm.
+	 * Or else, ACPI SCI is enabled during suspend/resume only,
+	 * update rtc irq in that case.
+	 */
+	if (use_acpi_alarm)
+		cmos_interrupt(0, (void *)cmos->rtc);
+	else {
+		/* Fix me: can we use cmos_interrupt() here as well? */
+		spin_lock_irqsave(&rtc_lock, flags);
+		if (cmos_rtc.suspend_ctrl)
+			rtc_control = CMOS_READ(RTC_CONTROL);
+		if (rtc_control & RTC_AIE) {
+			cmos_rtc.suspend_ctrl &= ~RTC_AIE;
+			CMOS_WRITE(rtc_control, RTC_CONTROL);
+			rtc_intr = CMOS_READ(RTC_INTR_FLAGS);
+			rtc_update_irq(cmos->rtc, 1, rtc_intr);
+		}
+		spin_unlock_irqrestore(&rtc_lock, flags);
 	}
-	spin_unlock_irqrestore(&rtc_lock, flags);
 
 	pm_wakeup_hard_event(dev);
 	acpi_clear_event(ACPI_EVENT_RTC);
@@ -1107,6 +1176,28 @@ static void rtc_wake_off(struct device *dev)
 	acpi_disable_event(ACPI_EVENT_RTC, 0);
 }
 
+#ifdef CONFIG_X86
+/* Enable use_acpi_alarm mode for Intel platforms no earlier than 2015 */
+static void use_acpi_alarm_quirks(void)
+{
+	int year;
+
+	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
+		return;
+
+	if (!(acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0))
+		return;
+
+	if (!is_hpet_enabled())
+		return;
+
+	if (dmi_get_date(DMI_BIOS_DATE, &year, NULL, NULL) && year >= 2015)
+		use_acpi_alarm = true;
+}
+#else
+static inline void use_acpi_alarm_quirks(void) { }
+#endif
+
 /* Every ACPI platform has a mc146818 compatible "cmos rtc".  Here we find
  * its device node and pass extra config data.  This helps its driver use
  * capabilities that the now-obsolete mc146818 didn't have, and informs it
@@ -1119,6 +1210,8 @@ static void cmos_wake_setup(struct device *dev)
 	if (acpi_disabled)
 		return;
 
+	use_acpi_alarm_quirks();
+
 	rtc_wake_setup(dev);
 	acpi_rtc_info.wake_on = rtc_wake_on;
 	acpi_rtc_info.wake_off = rtc_wake_off;
diff --git a/drivers/rtc/rtc-ds1216.c b/drivers/rtc/rtc-ds1216.c
index 5f158715fb4c..50fabe1cd286 100644
--- a/drivers/rtc/rtc-ds1216.c
+++ b/drivers/rtc/rtc-ds1216.c
@@ -76,8 +76,7 @@ static void ds1216_switch_ds_to_clock(u8 __iomem *ioaddr)
 
 static int ds1216_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct ds1216_priv *priv = platform_get_drvdata(pdev);
+	struct ds1216_priv *priv = dev_get_drvdata(dev);
 	struct ds1216_regs regs;
 
 	ds1216_switch_ds_to_clock(priv->ioaddr);
@@ -104,8 +103,7 @@ static int ds1216_rtc_read_time(struct device *dev, struct rtc_time *tm)
 
 static int ds1216_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct ds1216_priv *priv = platform_get_drvdata(pdev);
+	struct ds1216_priv *priv = dev_get_drvdata(dev);
 	struct ds1216_regs regs;
 
 	ds1216_switch_ds_to_clock(priv->ioaddr);
diff --git a/drivers/rtc/rtc-ds1307.c b/drivers/rtc/rtc-ds1307.c
index a13e59edff53..e9ec4160d7f6 100644
--- a/drivers/rtc/rtc-ds1307.c
+++ b/drivers/rtc/rtc-ds1307.c
@@ -201,6 +201,7 @@ static const struct chip_desc chips[last_ds_type] = {
 		.century_reg	= DS1307_REG_HOUR,
 		.century_enable_bit = DS1340_BIT_CENTURY_EN,
 		.century_bit	= DS1340_BIT_CENTURY,
+		.do_trickle_setup = &do_trickle_setup_ds1339,
 		.trickle_charger_reg = 0x08,
 	},
 	[ds_1341] = {
@@ -1371,6 +1372,7 @@ static void ds1307_clks_register(struct ds1307 *ds1307)
 static const struct regmap_config regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
+	.max_register = 0x9,
 };
 
 static int ds1307_probe(struct i2c_client *client,
diff --git a/drivers/rtc/rtc-ds1511.c b/drivers/rtc/rtc-ds1511.c
index a7d5ca428d68..b8b6e51c0461 100644
--- a/drivers/rtc/rtc-ds1511.c
+++ b/drivers/rtc/rtc-ds1511.c
@@ -314,8 +314,7 @@ ds1511_rtc_update_alarm(struct rtc_plat_data *pdata)
 static int
 ds1511_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	if (pdata->irq <= 0)
 		return -EINVAL;
@@ -334,8 +333,7 @@ ds1511_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 static int
 ds1511_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	if (pdata->irq <= 0)
 		return -EINVAL;
@@ -373,8 +371,7 @@ ds1511_interrupt(int irq, void *dev_id)
 
 static int ds1511_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	if (pdata->irq <= 0)
 		return -EINVAL;
diff --git a/drivers/rtc/rtc-ds1553.c b/drivers/rtc/rtc-ds1553.c
index 2441b9a2b366..34af7a802f43 100644
--- a/drivers/rtc/rtc-ds1553.c
+++ b/drivers/rtc/rtc-ds1553.c
@@ -73,8 +73,7 @@ struct rtc_plat_data {
 
 static int ds1553_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr;
 	u8 century;
 
@@ -98,8 +97,7 @@ static int ds1553_rtc_set_time(struct device *dev, struct rtc_time *tm)
 
 static int ds1553_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr;
 	unsigned int year, month, day, hour, minute, second, week;
 	unsigned int century;
@@ -155,8 +153,7 @@ static void ds1553_rtc_update_alarm(struct rtc_plat_data *pdata)
 
 static int ds1553_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	if (pdata->irq <= 0)
 		return -EINVAL;
@@ -172,8 +169,7 @@ static int ds1553_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 static int ds1553_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	if (pdata->irq <= 0)
 		return -EINVAL;
@@ -208,8 +204,7 @@ static irqreturn_t ds1553_rtc_interrupt(int irq, void *dev_id)
 
 static int ds1553_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	if (pdata->irq <= 0)
 		return -EINVAL;
diff --git a/drivers/rtc/rtc-ds1685.c b/drivers/rtc/rtc-ds1685.c
index 1a39829d2b40..f0f8011dce3d 100644
--- a/drivers/rtc/rtc-ds1685.c
+++ b/drivers/rtc/rtc-ds1685.c
@@ -267,8 +267,7 @@ ds1685_rtc_get_ssn(struct ds1685_priv *rtc, u8 *ssn)
 static int
 ds1685_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct ds1685_priv *rtc = platform_get_drvdata(pdev);
+	struct ds1685_priv *rtc = dev_get_drvdata(dev);
 	u8 ctrlb, century;
 	u8 seconds, minutes, hours, wday, mday, month, years;
 
@@ -317,8 +316,7 @@ ds1685_rtc_read_time(struct device *dev, struct rtc_time *tm)
 static int
 ds1685_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct ds1685_priv *rtc = platform_get_drvdata(pdev);
+	struct ds1685_priv *rtc = dev_get_drvdata(dev);
 	u8 ctrlb, seconds, minutes, hours, wday, mday, month, years, century;
 
 	/* Fetch the time info from rtc_time. */
@@ -394,8 +392,7 @@ ds1685_rtc_set_time(struct device *dev, struct rtc_time *tm)
 static int
 ds1685_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct ds1685_priv *rtc = platform_get_drvdata(pdev);
+	struct ds1685_priv *rtc = dev_get_drvdata(dev);
 	u8 seconds, minutes, hours, mday, ctrlb, ctrlc;
 	int ret;
 
@@ -453,8 +450,7 @@ ds1685_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 static int
 ds1685_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct ds1685_priv *rtc = platform_get_drvdata(pdev);
+	struct ds1685_priv *rtc = dev_get_drvdata(dev);
 	u8 ctrlb, seconds, minutes, hours, mday;
 	int ret;
 
@@ -1119,8 +1115,7 @@ static ssize_t
 ds1685_rtc_sysfs_battery_show(struct device *dev,
 			      struct device_attribute *attr, char *buf)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct ds1685_priv *rtc = platform_get_drvdata(pdev);
+	struct ds1685_priv *rtc = dev_get_drvdata(dev);
 	u8 ctrld;
 
 	ctrld = rtc->read(rtc, RTC_CTRL_D);
@@ -1140,8 +1135,7 @@ static ssize_t
 ds1685_rtc_sysfs_auxbatt_show(struct device *dev,
 			      struct device_attribute *attr, char *buf)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct ds1685_priv *rtc = platform_get_drvdata(pdev);
+	struct ds1685_priv *rtc = dev_get_drvdata(dev);
 	u8 ctrl4a;
 
 	ds1685_rtc_switch_to_bank1(rtc);
@@ -1163,8 +1157,7 @@ static ssize_t
 ds1685_rtc_sysfs_serial_show(struct device *dev,
 			     struct device_attribute *attr, char *buf)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct ds1685_priv *rtc = platform_get_drvdata(pdev);
+	struct ds1685_priv *rtc = dev_get_drvdata(dev);
 	u8 ssn[8];
 
 	ds1685_rtc_switch_to_bank1(rtc);
diff --git a/drivers/rtc/rtc-ds1742.c b/drivers/rtc/rtc-ds1742.c
index 2d781180e968..5a4c2c5e86fe 100644
--- a/drivers/rtc/rtc-ds1742.c
+++ b/drivers/rtc/rtc-ds1742.c
@@ -58,8 +58,7 @@ struct rtc_plat_data {
 
 static int ds1742_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr_rtc;
 	u8 century;
 
@@ -83,8 +82,7 @@ static int ds1742_rtc_set_time(struct device *dev, struct rtc_time *tm)
 
 static int ds1742_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr_rtc;
 	unsigned int year, month, day, hour, minute, second, week;
 	unsigned int century;
@@ -154,8 +152,6 @@ static int ds1742_rtc_probe(struct platform_device *pdev)
 	int ret = 0;
 	struct nvmem_config nvmem_cfg = {
 		.name = "ds1742_nvram",
-		.word_size = 1,
-		.stride = 1,
 		.reg_read = ds1742_nvram_read,
 		.reg_write = ds1742_nvram_write,
 	};
diff --git a/drivers/rtc/rtc-lpc32xx.c b/drivers/rtc/rtc-lpc32xx.c
index 3ba87239aacc..910e600275b9 100644
--- a/drivers/rtc/rtc-lpc32xx.c
+++ b/drivers/rtc/rtc-lpc32xx.c
@@ -294,11 +294,10 @@ static int lpc32xx_rtc_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM
 static int lpc32xx_rtc_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct lpc32xx_rtc *rtc = platform_get_drvdata(pdev);
+	struct lpc32xx_rtc *rtc = dev_get_drvdata(dev);
 
 	if (rtc->irq >= 0) {
-		if (device_may_wakeup(&pdev->dev))
+		if (device_may_wakeup(dev))
 			enable_irq_wake(rtc->irq);
 		else
 			disable_irq_wake(rtc->irq);
@@ -309,10 +308,9 @@ static int lpc32xx_rtc_suspend(struct device *dev)
 
 static int lpc32xx_rtc_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct lpc32xx_rtc *rtc = platform_get_drvdata(pdev);
+	struct lpc32xx_rtc *rtc = dev_get_drvdata(dev);
 
-	if (rtc->irq >= 0 && device_may_wakeup(&pdev->dev))
+	if (rtc->irq >= 0 && device_may_wakeup(dev))
 		disable_irq_wake(rtc->irq);
 
 	return 0;
@@ -321,8 +319,7 @@ static int lpc32xx_rtc_resume(struct device *dev)
 /* Unconditionally disable the alarm */
 static int lpc32xx_rtc_freeze(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct lpc32xx_rtc *rtc = platform_get_drvdata(pdev);
+	struct lpc32xx_rtc *rtc = dev_get_drvdata(dev);
 
 	spin_lock_irq(&rtc->lock);
 
@@ -337,8 +334,7 @@ static int lpc32xx_rtc_freeze(struct device *dev)
 
 static int lpc32xx_rtc_thaw(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct lpc32xx_rtc *rtc = platform_get_drvdata(pdev);
+	struct lpc32xx_rtc *rtc = dev_get_drvdata(dev);
 
 	if (rtc->alarm_enabled) {
 		spin_lock_irq(&rtc->lock);
diff --git a/drivers/rtc/rtc-ls1x.c b/drivers/rtc/rtc-ls1x.c
index 045af1135e48..f4c248655edd 100644
--- a/drivers/rtc/rtc-ls1x.c
+++ b/drivers/rtc/rtc-ls1x.c
@@ -87,16 +87,17 @@
 
 static int ls1x_rtc_read_time(struct device *dev, struct rtc_time *rtm)
 {
-	unsigned long v, t;
+	unsigned long v;
+	time64_t t;
 
 	v = readl(SYS_TOYREAD0);
 	t = readl(SYS_TOYREAD1);
 
 	memset(rtm, 0, sizeof(struct rtc_time));
-	t  = mktime((t & LS1X_YEAR_MASK), ls1x_get_month(v),
+	t  = mktime64((t & LS1X_YEAR_MASK), ls1x_get_month(v),
 			ls1x_get_day(v), ls1x_get_hour(v),
 			ls1x_get_min(v), ls1x_get_sec(v));
-	rtc_time_to_tm(t, rtm);
+	rtc_time64_to_tm(t, rtm);
 
 	return 0;
 }
@@ -147,15 +148,13 @@ static int ls1x_rtc_probe(struct platform_device *pdev)
 {
 	struct rtc_device *rtcdev;
 	unsigned long v;
-	int ret;
 
 	v = readl(SYS_COUNTER_CNTRL);
 	if (!(v & RTC_CNTR_OK)) {
 		dev_err(&pdev->dev, "rtc counters not working\n");
-		ret = -ENODEV;
-		goto err;
+		return -ENODEV;
 	}
-	ret = -ETIMEDOUT;
+
 	/* set to 1 HZ if needed */
 	if (readl(SYS_TOYTRIM) != 32767) {
 		v = 0x100000;
@@ -164,7 +163,7 @@ static int ls1x_rtc_probe(struct platform_device *pdev)
 
 		if (!v) {
 			dev_err(&pdev->dev, "time out\n");
-			goto err;
+			return -ETIMEDOUT;
 		}
 		writel(32767, SYS_TOYTRIM);
 	}
@@ -172,17 +171,16 @@ static int ls1x_rtc_probe(struct platform_device *pdev)
 	while (readl(SYS_COUNTER_CNTRL) & SYS_CNTRL_TTS)
 		usleep_range(1000, 3000);
 
-	rtcdev = devm_rtc_device_register(&pdev->dev, "ls1x-rtc",
-					&ls1x_rtc_ops , THIS_MODULE);
-	if (IS_ERR(rtcdev)) {
-		ret = PTR_ERR(rtcdev);
-		goto err;
-	}
+	rtcdev = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtcdev))
+		return PTR_ERR(rtcdev);
 
 	platform_set_drvdata(pdev, rtcdev);
-	return 0;
-err:
-	return ret;
+	rtcdev->ops = &ls1x_rtc_ops;
+	rtcdev->range_min = RTC_TIMESTAMP_BEGIN_1900;
+	rtcdev->range_max = RTC_TIMESTAMP_END_2099;
+
+	return rtc_register_device(rtcdev);
 }
 
 static struct platform_driver  ls1x_rtc_driver = {
diff --git a/drivers/rtc/rtc-m48t59.c b/drivers/rtc/rtc-m48t59.c
index 216fac62c888..1053a406b3aa 100644
--- a/drivers/rtc/rtc-m48t59.c
+++ b/drivers/rtc/rtc-m48t59.c
@@ -47,8 +47,7 @@ struct m48t59_private {
 static void
 m48t59_mem_writeb(struct device *dev, u32 ofs, u8 val)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct m48t59_private *m48t59 = platform_get_drvdata(pdev);
+	struct m48t59_private *m48t59 = dev_get_drvdata(dev);
 
 	writeb(val, m48t59->ioaddr+ofs);
 }
@@ -56,8 +55,7 @@ m48t59_mem_writeb(struct device *dev, u32 ofs, u8 val)
 static u8
 m48t59_mem_readb(struct device *dev, u32 ofs)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct m48t59_private *m48t59 = platform_get_drvdata(pdev);
+	struct m48t59_private *m48t59 = dev_get_drvdata(dev);
 
 	return readb(m48t59->ioaddr+ofs);
 }
@@ -67,9 +65,8 @@ m48t59_mem_readb(struct device *dev, u32 ofs)
  */
 static int m48t59_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct m48t59_plat_data *pdata = dev_get_platdata(&pdev->dev);
-	struct m48t59_private *m48t59 = platform_get_drvdata(pdev);
+	struct m48t59_plat_data *pdata = dev_get_platdata(dev);
+	struct m48t59_private *m48t59 = dev_get_drvdata(dev);
 	unsigned long flags;
 	u8 val;
 
@@ -110,9 +107,8 @@ static int m48t59_rtc_read_time(struct device *dev, struct rtc_time *tm)
 
 static int m48t59_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct m48t59_plat_data *pdata = dev_get_platdata(&pdev->dev);
-	struct m48t59_private *m48t59 = platform_get_drvdata(pdev);
+	struct m48t59_plat_data *pdata = dev_get_platdata(dev);
+	struct m48t59_private *m48t59 = dev_get_drvdata(dev);
 	unsigned long flags;
 	u8 val = 0;
 	int year = tm->tm_year;
@@ -157,9 +153,8 @@ static int m48t59_rtc_set_time(struct device *dev, struct rtc_time *tm)
  */
 static int m48t59_rtc_readalarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct m48t59_plat_data *pdata = dev_get_platdata(&pdev->dev);
-	struct m48t59_private *m48t59 = platform_get_drvdata(pdev);
+	struct m48t59_plat_data *pdata = dev_get_platdata(dev);
+	struct m48t59_private *m48t59 = dev_get_drvdata(dev);
 	struct rtc_time *tm = &alrm->time;
 	unsigned long flags;
 	u8 val;
@@ -204,9 +199,8 @@ static int m48t59_rtc_readalarm(struct device *dev, struct rtc_wkalrm *alrm)
  */
 static int m48t59_rtc_setalarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct m48t59_plat_data *pdata = dev_get_platdata(&pdev->dev);
-	struct m48t59_private *m48t59 = platform_get_drvdata(pdev);
+	struct m48t59_plat_data *pdata = dev_get_platdata(dev);
+	struct m48t59_private *m48t59 = dev_get_drvdata(dev);
 	struct rtc_time *tm = &alrm->time;
 	u8 mday, hour, min, sec;
 	unsigned long flags;
@@ -265,9 +259,8 @@ static int m48t59_rtc_setalarm(struct device *dev, struct rtc_wkalrm *alrm)
  */
 static int m48t59_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct m48t59_plat_data *pdata = dev_get_platdata(&pdev->dev);
-	struct m48t59_private *m48t59 = platform_get_drvdata(pdev);
+	struct m48t59_plat_data *pdata = dev_get_platdata(dev);
+	struct m48t59_private *m48t59 = dev_get_drvdata(dev);
 	unsigned long flags;
 
 	spin_lock_irqsave(&m48t59->lock, flags);
@@ -282,9 +275,8 @@ static int m48t59_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 
 static int m48t59_rtc_proc(struct device *dev, struct seq_file *seq)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct m48t59_plat_data *pdata = dev_get_platdata(&pdev->dev);
-	struct m48t59_private *m48t59 = platform_get_drvdata(pdev);
+	struct m48t59_plat_data *pdata = dev_get_platdata(dev);
+	struct m48t59_private *m48t59 = dev_get_drvdata(dev);
 	unsigned long flags;
 	u8 val;
 
@@ -303,9 +295,8 @@ static int m48t59_rtc_proc(struct device *dev, struct seq_file *seq)
 static irqreturn_t m48t59_rtc_interrupt(int irq, void *dev_id)
 {
 	struct device *dev = (struct device *)dev_id;
-	struct platform_device *pdev = to_platform_device(dev);
-	struct m48t59_plat_data *pdata = dev_get_platdata(&pdev->dev);
-	struct m48t59_private *m48t59 = platform_get_drvdata(pdev);
+	struct m48t59_plat_data *pdata = dev_get_platdata(dev);
+	struct m48t59_private *m48t59 = dev_get_drvdata(dev);
 	u8 event;
 
 	spin_lock(&m48t59->lock);
diff --git a/drivers/rtc/rtc-mt6397.c b/drivers/rtc/rtc-mt6397.c
index 1a61fa56f3ad..385f8303bb41 100644
--- a/drivers/rtc/rtc-mt6397.c
+++ b/drivers/rtc/rtc-mt6397.c
@@ -322,10 +322,9 @@ static int mtk_rtc_probe(struct platform_device *pdev)
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	rtc->addr_base = res->start;
 
-	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	rtc->irq = irq_create_mapping(mt6397_chip->irq_domain, res->start);
-	if (rtc->irq <= 0)
-		return -EINVAL;
+	rtc->irq = platform_get_irq(pdev, 0);
+	if (rtc->irq < 0)
+		return rtc->irq;
 
 	rtc->regmap = mt6397_chip->regmap;
 	rtc->dev = &pdev->dev;
diff --git a/drivers/rtc/rtc-mv.c b/drivers/rtc/rtc-mv.c
index bc52dbb0c0e2..4b198b3778d3 100644
--- a/drivers/rtc/rtc-mv.c
+++ b/drivers/rtc/rtc-mv.c
@@ -176,8 +176,7 @@ static int mv_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm)
 
 static int mv_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr;
 
 	if (pdata->irq < 0)
diff --git a/drivers/rtc/rtc-mxc.c b/drivers/rtc/rtc-mxc.c
index bce427d202ee..878c6ee82901 100644
--- a/drivers/rtc/rtc-mxc.c
+++ b/drivers/rtc/rtc-mxc.c
@@ -1,13 +1,6 @@
-/*
- * Copyright 2004-2008 Freescale Semiconductor, Inc. All Rights Reserved.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2004-2008 Freescale Semiconductor, Inc. All Rights Reserved.
 
 #include <linux/io.h>
 #include <linux/rtc.h>
@@ -109,8 +102,7 @@ static inline int is_imx1_rtc(struct rtc_plat_data *data)
  */
 static time64_t get_alarm_or_time(struct device *dev, int time_alarm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr;
 	u32 day = 0, hr = 0, min = 0, sec = 0, hr_min = 0;
 
@@ -139,8 +131,7 @@ static time64_t get_alarm_or_time(struct device *dev, int time_alarm)
 static void set_alarm_or_time(struct device *dev, int time_alarm, time64_t time)
 {
 	u32 tod, day, hr, min, sec, temp;
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr;
 
 	day = div_s64_rem(time, 86400, &tod);
@@ -176,8 +167,7 @@ static void set_alarm_or_time(struct device *dev, int time_alarm, time64_t time)
 static void rtc_update_alarm(struct device *dev, struct rtc_time *alrm)
 {
 	time64_t time;
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr;
 
 	time = rtc_tm_to_time64(alrm);
@@ -190,8 +180,7 @@ static void rtc_update_alarm(struct device *dev, struct rtc_time *alrm)
 static void mxc_rtc_irq_enable(struct device *dev, unsigned int bit,
 				unsigned int enabled)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr;
 	u32 reg;
 
@@ -266,8 +255,7 @@ static int mxc_rtc_read_time(struct device *dev, struct rtc_time *tm)
  */
 static int mxc_rtc_set_mmss(struct device *dev, time64_t time)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	/*
 	 * TTC_DAYR register is 9-bit in MX1 SoC, save time and day of year only
@@ -295,8 +283,7 @@ static int mxc_rtc_set_mmss(struct device *dev, time64_t time)
  */
 static int mxc_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr;
 
 	rtc_time64_to_tm(get_alarm_or_time(dev, MXC_RTC_ALARM), &alrm->time);
@@ -310,8 +297,7 @@ static int mxc_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
  */
 static int mxc_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	rtc_update_alarm(dev, &alrm->time);
 
diff --git a/drivers/rtc/rtc-mxc_v2.c b/drivers/rtc/rtc-mxc_v2.c
index 9e14efb990b2..c75f26dc8fcc 100644
--- a/drivers/rtc/rtc-mxc_v2.c
+++ b/drivers/rtc/rtc-mxc_v2.c
@@ -165,11 +165,6 @@ static int mxc_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	time64_t time = rtc_tm_to_time64(tm);
 	int ret;
 
-	if (time > U32_MAX) {
-		dev_err(dev, "RTC exceeded by %llus\n", time - U32_MAX);
-		return -EINVAL;
-	}
-
 	ret = mxc_rtc_lock(pdata);
 	if (ret)
 		return ret;
@@ -198,7 +193,7 @@ static int mxc_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	if (ret)
 		return ret;
 
-	rtc_time_to_tm(readl(ioaddr + SRTC_LPSAR), &alrm->time);
+	rtc_time64_to_tm(readl(ioaddr + SRTC_LPSAR), &alrm->time);
 	alrm->pending = !!(readl(ioaddr + SRTC_LPSR) & SRTC_LPSR_ALP);
 	return mxc_rtc_unlock(pdata);
 }
@@ -248,11 +243,6 @@ static int mxc_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	if (ret)
 		return ret;
 
-	if (time > U32_MAX) {
-		dev_err(dev, "Hopefully I am out of service by then :-(\n");
-		return -EINVAL;
-	}
-
 	writel((u32)time, pdata->ioaddr + SRTC_LPSAR);
 
 	/* clear alarm interrupt status bit */
@@ -343,6 +333,13 @@ static int mxc_rtc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	pdata->rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(pdata->rtc))
+		return PTR_ERR(pdata->rtc);
+
+	pdata->rtc->ops = &mxc_rtc_ops;
+	pdata->rtc->range_max = U32_MAX;
+
 	clk_disable(pdata->clk);
 	platform_set_drvdata(pdev, pdata);
 	ret =
@@ -354,15 +351,11 @@ static int mxc_rtc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	pdata->rtc =
-	    devm_rtc_device_register(&pdev->dev, pdev->name, &mxc_rtc_ops,
-				     THIS_MODULE);
-	if (IS_ERR(pdata->rtc)) {
+	ret = rtc_register_device(pdata->rtc);
+	if (ret < 0)
 		clk_unprepare(pdata->clk);
-		return PTR_ERR(pdata->rtc);
-	}
 
-	return 0;
+	return ret;
 }
 
 static int mxc_rtc_remove(struct platform_device *pdev)
diff --git a/drivers/rtc/rtc-pcap.c b/drivers/rtc/rtc-pcap.c
index c05f524ba9af..f176cb9d0dbc 100644
--- a/drivers/rtc/rtc-pcap.c
+++ b/drivers/rtc/rtc-pcap.c
@@ -43,8 +43,7 @@ static irqreturn_t pcap_rtc_irq(int irq, void *_pcap_rtc)
 
 static int pcap_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct pcap_rtc *pcap_rtc = platform_get_drvdata(pdev);
+	struct pcap_rtc *pcap_rtc = dev_get_drvdata(dev);
 	struct rtc_time *tm = &alrm->time;
 	unsigned long secs;
 	u32 tod;	/* time of day, seconds since midnight */
@@ -63,8 +62,7 @@ static int pcap_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 static int pcap_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct pcap_rtc *pcap_rtc = platform_get_drvdata(pdev);
+	struct pcap_rtc *pcap_rtc = dev_get_drvdata(dev);
 	struct rtc_time *tm = &alrm->time;
 	unsigned long secs;
 	u32 tod, days;
@@ -82,8 +80,7 @@ static int pcap_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 static int pcap_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct pcap_rtc *pcap_rtc = platform_get_drvdata(pdev);
+	struct pcap_rtc *pcap_rtc = dev_get_drvdata(dev);
 	unsigned long secs;
 	u32 tod, days;
 
@@ -100,8 +97,7 @@ static int pcap_rtc_read_time(struct device *dev, struct rtc_time *tm)
 
 static int pcap_rtc_set_mmss(struct device *dev, unsigned long secs)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct pcap_rtc *pcap_rtc = platform_get_drvdata(pdev);
+	struct pcap_rtc *pcap_rtc = dev_get_drvdata(dev);
 	u32 tod, days;
 
 	tod = secs % SEC_PER_DAY;
@@ -115,8 +111,7 @@ static int pcap_rtc_set_mmss(struct device *dev, unsigned long secs)
 
 static int pcap_rtc_irq_enable(struct device *dev, int pirq, unsigned int en)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct pcap_rtc *pcap_rtc = platform_get_drvdata(pdev);
+	struct pcap_rtc *pcap_rtc = dev_get_drvdata(dev);
 
 	if (en)
 		enable_irq(pcap_to_irq(pcap_rtc->pcap, pirq));
diff --git a/drivers/rtc/rtc-proc.c b/drivers/rtc/rtc-proc.c
index 31e7e23cc5be..a9dd9218fae2 100644
--- a/drivers/rtc/rtc-proc.c
+++ b/drivers/rtc/rtc-proc.c
@@ -107,40 +107,11 @@ static int rtc_proc_show(struct seq_file *seq, void *offset)
 	return 0;
 }
 
-static int rtc_proc_open(struct inode *inode, struct file *file)
-{
-	int ret;
-	struct rtc_device *rtc = PDE_DATA(inode);
-
-	if (!try_module_get(rtc->owner))
-		return -ENODEV;
-
-	ret = single_open(file, rtc_proc_show, rtc);
-	if (ret)
-		module_put(rtc->owner);
-	return ret;
-}
-
-static int rtc_proc_release(struct inode *inode, struct file *file)
-{
-	int res = single_release(inode, file);
-	struct rtc_device *rtc = PDE_DATA(inode);
-
-	module_put(rtc->owner);
-	return res;
-}
-
-static const struct file_operations rtc_proc_fops = {
-	.open		= rtc_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= rtc_proc_release,
-};
-
 void rtc_proc_add_device(struct rtc_device *rtc)
 {
 	if (is_rtc_hctosys(rtc))
-		proc_create_data("driver/rtc", 0, NULL, &rtc_proc_fops, rtc);
+		proc_create_single_data("driver/rtc", 0, NULL, rtc_proc_show,
+				rtc);
 }
 
 void rtc_proc_del_device(struct rtc_device *rtc)
diff --git a/drivers/rtc/rtc-rx8581.c b/drivers/rtc/rtc-rx8581.c
index 32caadf912ca..eac882169744 100644
--- a/drivers/rtc/rtc-rx8581.c
+++ b/drivers/rtc/rtc-rx8581.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/i2c.h>
 #include <linux/bcd.h>
+#include <linux/regmap.h>
 #include <linux/rtc.h>
 #include <linux/log2.h>
 
@@ -51,52 +52,20 @@
 #define RX8581_CTRL_RESET	0x01 /* RESET bit */
 
 struct rx8581 {
-	struct i2c_client	*client;
+	struct regmap		*regmap;
 	struct rtc_device	*rtc;
-	s32 (*read_block_data)(const struct i2c_client *client, u8 command,
-				u8 length, u8 *values);
-	s32 (*write_block_data)(const struct i2c_client *client, u8 command,
-				u8 length, const u8 *values);
 };
 
-static struct i2c_driver rx8581_driver;
-
-static int rx8581_read_block_data(const struct i2c_client *client, u8 command,
-					u8 length, u8 *values)
-{
-	s32 i, data;
-
-	for (i = 0; i < length; i++) {
-		data = i2c_smbus_read_byte_data(client, command + i);
-		if (data < 0)
-			return data;
-		values[i] = data;
-	}
-	return i;
-}
-
-static int rx8581_write_block_data(const struct i2c_client *client, u8 command,
-					u8 length, const u8 *values)
-{
-	s32 i, ret;
-
-	for (i = 0; i < length; i++) {
-		ret = i2c_smbus_write_byte_data(client, command + i,
-						values[i]);
-		if (ret < 0)
-			return ret;
-	}
-	return length;
-}
-
 /*
  * In the routines that deal directly with the rx8581 hardware, we use
  * rtc_time -- month 0-11, hour 0-23, yr = calendar year-epoch.
  */
-static int rx8581_get_datetime(struct i2c_client *client, struct rtc_time *tm)
+static int rx8581_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
+	struct i2c_client *client = to_i2c_client(dev);
 	unsigned char date[7];
-	int data, err;
+	unsigned int data;
+	int err;
 	struct rx8581 *rx8581 = i2c_get_clientdata(client);
 
 	/* First we ensure that the "update flag" is not set, we read the
@@ -104,45 +73,38 @@ static int rx8581_get_datetime(struct i2c_client *client, struct rtc_time *tm)
 	 * has been set, we know that the time has changed during the read so
 	 * we repeat the whole process again.
 	 */
-	data = i2c_smbus_read_byte_data(client, RX8581_REG_FLAG);
-	if (data < 0) {
-		dev_err(&client->dev, "Unable to read device flags\n");
-		return -EIO;
+	err = regmap_read(rx8581->regmap, RX8581_REG_FLAG, &data);
+	if (err < 0)
+		return err;
+
+	if (data & RX8581_FLAG_VLF) {
+		dev_warn(dev,
+			 "low voltage detected, date/time is not reliable.\n");
+		return -EINVAL;
 	}
 
 	do {
 		/* If update flag set, clear it */
 		if (data & RX8581_FLAG_UF) {
-			err = i2c_smbus_write_byte_data(client,
-				RX8581_REG_FLAG, (data & ~RX8581_FLAG_UF));
-			if (err != 0) {
-				dev_err(&client->dev, "Unable to write device flags\n");
-				return -EIO;
-			}
+			err = regmap_write(rx8581->regmap, RX8581_REG_FLAG,
+					  data & ~RX8581_FLAG_UF);
+			if (err < 0)
+				return err;
 		}
 
 		/* Now read time and date */
-		err = rx8581->read_block_data(client, RX8581_REG_SC,
-			7, date);
-		if (err < 0) {
-			dev_err(&client->dev, "Unable to read date\n");
-			return -EIO;
-		}
+		err = regmap_bulk_read(rx8581->regmap, RX8581_REG_SC, date,
+				       sizeof(date));
+		if (err < 0)
+			return err;
 
 		/* Check flag register */
-		data = i2c_smbus_read_byte_data(client, RX8581_REG_FLAG);
-		if (data < 0) {
-			dev_err(&client->dev, "Unable to read device flags\n");
-			return -EIO;
-		}
+		err = regmap_read(rx8581->regmap, RX8581_REG_FLAG, &data);
+		if (err < 0)
+			return err;
 	} while (data & RX8581_FLAG_UF);
 
-	if (data & RX8581_FLAG_VLF)
-		dev_info(&client->dev,
-			"low voltage detected, date/time is not reliable.\n");
-
-	dev_dbg(&client->dev,
-		"%s: raw data is sec=%02x, min=%02x, hr=%02x, "
+	dev_dbg(dev, "%s: raw data is sec=%02x, min=%02x, hr=%02x, "
 		"wday=%02x, mday=%02x, mon=%02x, year=%02x\n",
 		__func__,
 		date[0], date[1], date[2], date[3], date[4], date[5], date[6]);
@@ -153,12 +115,9 @@ static int rx8581_get_datetime(struct i2c_client *client, struct rtc_time *tm)
 	tm->tm_wday = ilog2(date[RX8581_REG_DW] & 0x7F);
 	tm->tm_mday = bcd2bin(date[RX8581_REG_DM] & 0x3F);
 	tm->tm_mon = bcd2bin(date[RX8581_REG_MO] & 0x1F) - 1; /* rtc mn 1-12 */
-	tm->tm_year = bcd2bin(date[RX8581_REG_YR]);
-	if (tm->tm_year < 70)
-		tm->tm_year += 100;	/* assume we are in 1970...2069 */
-
+	tm->tm_year = bcd2bin(date[RX8581_REG_YR]) + 100;
 
-	dev_dbg(&client->dev, "%s: tm is secs=%d, mins=%d, hours=%d, "
+	dev_dbg(dev, "%s: tm is secs=%d, mins=%d, hours=%d, "
 		"mday=%d, mon=%d, year=%d, wday=%d\n",
 		__func__,
 		tm->tm_sec, tm->tm_min, tm->tm_hour,
@@ -167,13 +126,14 @@ static int rx8581_get_datetime(struct i2c_client *client, struct rtc_time *tm)
 	return 0;
 }
 
-static int rx8581_set_datetime(struct i2c_client *client, struct rtc_time *tm)
+static int rx8581_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	int data, err;
+	struct i2c_client *client = to_i2c_client(dev);
+	int err;
 	unsigned char buf[7];
 	struct rx8581 *rx8581 = i2c_get_clientdata(client);
 
-	dev_dbg(&client->dev, "%s: secs=%d, mins=%d, hours=%d, "
+	dev_dbg(dev, "%s: secs=%d, mins=%d, hours=%d, "
 		"mday=%d, mon=%d, year=%d, wday=%d\n",
 		__func__,
 		tm->tm_sec, tm->tm_min, tm->tm_hour,
@@ -190,69 +150,30 @@ static int rx8581_set_datetime(struct i2c_client *client, struct rtc_time *tm)
 	buf[RX8581_REG_MO] = bin2bcd(tm->tm_mon + 1);
 
 	/* year and century */
-	buf[RX8581_REG_YR] = bin2bcd(tm->tm_year % 100);
+	buf[RX8581_REG_YR] = bin2bcd(tm->tm_year - 100);
 	buf[RX8581_REG_DW] = (0x1 << tm->tm_wday);
 
 	/* Stop the clock */
-	data = i2c_smbus_read_byte_data(client, RX8581_REG_CTRL);
-	if (data < 0) {
-		dev_err(&client->dev, "Unable to read control register\n");
-		return -EIO;
-	}
-
-	err = i2c_smbus_write_byte_data(client, RX8581_REG_CTRL,
-		(data | RX8581_CTRL_STOP));
-	if (err < 0) {
-		dev_err(&client->dev, "Unable to write control register\n");
-		return -EIO;
-	}
+	err = regmap_update_bits(rx8581->regmap, RX8581_REG_CTRL,
+				 RX8581_CTRL_STOP, RX8581_CTRL_STOP);
+	if (err < 0)
+		return err;
 
 	/* write register's data */
-	err = rx8581->write_block_data(client, RX8581_REG_SC, 7, buf);
-	if (err < 0) {
-		dev_err(&client->dev, "Unable to write to date registers\n");
-		return -EIO;
-	}
+	err = regmap_bulk_write(rx8581->regmap, RX8581_REG_SC,
+				buf, sizeof(buf));
+	if (err < 0)
+		return err;
 
 	/* get VLF and clear it */
-	data = i2c_smbus_read_byte_data(client, RX8581_REG_FLAG);
-	if (data < 0) {
-		dev_err(&client->dev, "Unable to read flag register\n");
-		return -EIO;
-	}
-
-	err = i2c_smbus_write_byte_data(client, RX8581_REG_FLAG,
-		(data & ~(RX8581_FLAG_VLF)));
-	if (err != 0) {
-		dev_err(&client->dev, "Unable to write flag register\n");
-		return -EIO;
-	}
+	err = regmap_update_bits(rx8581->regmap, RX8581_REG_FLAG,
+				 RX8581_FLAG_VLF, 0);
+	if (err < 0)
+		return err;
 
 	/* Restart the clock */
-	data = i2c_smbus_read_byte_data(client, RX8581_REG_CTRL);
-	if (data < 0) {
-		dev_err(&client->dev, "Unable to read control register\n");
-		return -EIO;
-	}
-
-	err = i2c_smbus_write_byte_data(client, RX8581_REG_CTRL,
-		(data & ~(RX8581_CTRL_STOP)));
-	if (err != 0) {
-		dev_err(&client->dev, "Unable to write control register\n");
-		return -EIO;
-	}
-
-	return 0;
-}
-
-static int rx8581_rtc_read_time(struct device *dev, struct rtc_time *tm)
-{
-	return rx8581_get_datetime(to_i2c_client(dev), tm);
-}
-
-static int rx8581_rtc_set_time(struct device *dev, struct rtc_time *tm)
-{
-	return rx8581_set_datetime(to_i2c_client(dev), tm);
+	return regmap_update_bits(rx8581->regmap, RX8581_REG_CTRL,
+				 RX8581_CTRL_STOP, 0);
 }
 
 static const struct rtc_class_ops rx8581_rtc_ops = {
@@ -264,38 +185,35 @@ static int rx8581_probe(struct i2c_client *client,
 			const struct i2c_device_id *id)
 {
 	struct rx8581	  *rx8581;
+	static const struct regmap_config config = {
+		.reg_bits = 8,
+		.val_bits = 8,
+		.max_register = 0xf,
+	};
 
 	dev_dbg(&client->dev, "%s\n", __func__);
 
-	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA)
-		&& !i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_I2C_BLOCK))
-		return -EIO;
-
 	rx8581 = devm_kzalloc(&client->dev, sizeof(struct rx8581), GFP_KERNEL);
 	if (!rx8581)
 		return -ENOMEM;
 
 	i2c_set_clientdata(client, rx8581);
-	rx8581->client = client;
-
-	if (i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_I2C_BLOCK)) {
-		rx8581->read_block_data = i2c_smbus_read_i2c_block_data;
-		rx8581->write_block_data = i2c_smbus_write_i2c_block_data;
-	} else {
-		rx8581->read_block_data = rx8581_read_block_data;
-		rx8581->write_block_data = rx8581_write_block_data;
-	}
 
-	rx8581->rtc = devm_rtc_device_register(&client->dev,
-		rx8581_driver.driver.name, &rx8581_rtc_ops, THIS_MODULE);
+	rx8581->regmap = devm_regmap_init_i2c(client, &config);
+	if (IS_ERR(rx8581->regmap))
+		return PTR_ERR(rx8581->regmap);
 
-	if (IS_ERR(rx8581->rtc)) {
-		dev_err(&client->dev,
-			"unable to register the class device\n");
+	rx8581->rtc = devm_rtc_allocate_device(&client->dev);
+	if (IS_ERR(rx8581->rtc))
 		return PTR_ERR(rx8581->rtc);
-	}
 
-	return 0;
+	rx8581->rtc->ops = &rx8581_rtc_ops;
+	rx8581->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000;
+	rx8581->rtc->range_max = RTC_TIMESTAMP_END_2099;
+	rx8581->rtc->start_secs = 0;
+	rx8581->rtc->set_start_time = true;
+
+	return rtc_register_device(rx8581->rtc);
 }
 
 static const struct i2c_device_id rx8581_id[] = {
diff --git a/drivers/rtc/rtc-sc27xx.c b/drivers/rtc/rtc-sc27xx.c
index 00d87d138984..6a3876edd8e0 100644
--- a/drivers/rtc/rtc-sc27xx.c
+++ b/drivers/rtc/rtc-sc27xx.c
@@ -600,6 +600,10 @@ static int sprd_rtc_probe(struct platform_device *pdev)
 		return rtc->irq;
 	}
 
+	rtc->rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtc->rtc))
+		return PTR_ERR(rtc->rtc);
+
 	rtc->dev = &pdev->dev;
 	platform_set_drvdata(pdev, rtc);
 
@@ -626,10 +630,14 @@ static int sprd_rtc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	rtc->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
-					    &sprd_rtc_ops, THIS_MODULE);
-	if (IS_ERR(rtc->rtc))
-		return PTR_ERR(rtc->rtc);
+	rtc->rtc->ops = &sprd_rtc_ops;
+	rtc->rtc->range_min = 0;
+	rtc->rtc->range_max = 5662310399LL;
+	ret = rtc_register_device(rtc->rtc);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to register rtc device\n");
+		return ret;
+	}
 
 	device_init_wakeup(&pdev->dev, 1);
 	return 0;
diff --git a/drivers/rtc/rtc-sh.c b/drivers/rtc/rtc-sh.c
index 4e8ab370ce63..4f98543d1ea5 100644
--- a/drivers/rtc/rtc-sh.c
+++ b/drivers/rtc/rtc-sh.c
@@ -359,8 +359,7 @@ static int sh_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 
 static int sh_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct sh_rtc *rtc = platform_get_drvdata(pdev);
+	struct sh_rtc *rtc = dev_get_drvdata(dev);
 	unsigned int sec128, sec2, yr, yr100, cf_bit;
 
 	do {
@@ -419,8 +418,7 @@ static int sh_rtc_read_time(struct device *dev, struct rtc_time *tm)
 
 static int sh_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct sh_rtc *rtc = platform_get_drvdata(pdev);
+	struct sh_rtc *rtc = dev_get_drvdata(dev);
 	unsigned int tmp;
 	int year;
 
@@ -475,8 +473,7 @@ static inline int sh_rtc_read_alarm_value(struct sh_rtc *rtc, int reg_off)
 
 static int sh_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *wkalrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct sh_rtc *rtc = platform_get_drvdata(pdev);
+	struct sh_rtc *rtc = dev_get_drvdata(dev);
 	struct rtc_time *tm = &wkalrm->time;
 
 	spin_lock_irq(&rtc->lock);
@@ -509,8 +506,7 @@ static inline void sh_rtc_write_alarm_value(struct sh_rtc *rtc,
 
 static int sh_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *wkalrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct sh_rtc *rtc = platform_get_drvdata(pdev);
+	struct sh_rtc *rtc = dev_get_drvdata(dev);
 	unsigned int rcr1;
 	struct rtc_time *tm = &wkalrm->time;
 	int mon;
@@ -723,8 +719,7 @@ static int __exit sh_rtc_remove(struct platform_device *pdev)
 
 static void sh_rtc_set_irq_wake(struct device *dev, int enabled)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct sh_rtc *rtc = platform_get_drvdata(pdev);
+	struct sh_rtc *rtc = dev_get_drvdata(dev);
 
 	irq_set_irq_wake(rtc->periodic_irq, enabled);
 
diff --git a/drivers/rtc/rtc-snvs.c b/drivers/rtc/rtc-snvs.c
index 9af591d5223c..8a75cc3af6e7 100644
--- a/drivers/rtc/rtc-snvs.c
+++ b/drivers/rtc/rtc-snvs.c
@@ -1,13 +1,6 @@
-/*
- * Copyright (C) 2011-2012 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (C) 2011-2012 Freescale Semiconductor, Inc.
 
 #include <linux/init.h>
 #include <linux/io.h>
diff --git a/drivers/rtc/rtc-st-lpc.c b/drivers/rtc/rtc-st-lpc.c
index d5222667f892..bee75ca7ff79 100644
--- a/drivers/rtc/rtc-st-lpc.c
+++ b/drivers/rtc/rtc-st-lpc.c
@@ -212,6 +212,10 @@ static int st_rtc_probe(struct platform_device *pdev)
 	if (!rtc)
 		return -ENOMEM;
 
+	rtc->rtc_dev = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtc->rtc_dev))
+		return PTR_ERR(rtc->rtc_dev);
+
 	spin_lock_init(&rtc->lock);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -253,26 +257,19 @@ static int st_rtc_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, rtc);
 
-	rtc->rtc_dev = rtc_device_register("st-lpc-rtc", &pdev->dev,
-					   &st_rtc_ops, THIS_MODULE);
-	if (IS_ERR(rtc->rtc_dev)) {
+	rtc->rtc_dev->ops = &st_rtc_ops;
+	rtc->rtc_dev->range_max = U64_MAX;
+	do_div(rtc->rtc_dev->range_max, rtc->clkrate);
+
+	ret = rtc_register_device(rtc->rtc_dev);
+	if (ret) {
 		clk_disable_unprepare(rtc->clk);
-		return PTR_ERR(rtc->rtc_dev);
+		return ret;
 	}
 
 	return 0;
 }
 
-static int st_rtc_remove(struct platform_device *pdev)
-{
-	struct st_rtc *rtc = platform_get_drvdata(pdev);
-
-	if (likely(rtc->rtc_dev))
-		rtc_device_unregister(rtc->rtc_dev);
-
-	return 0;
-}
-
 #ifdef CONFIG_PM_SLEEP
 static int st_rtc_suspend(struct device *dev)
 {
@@ -325,7 +322,6 @@ static struct platform_driver st_rtc_platform_driver = {
 		.of_match_table = st_rtc_match,
 	},
 	.probe = st_rtc_probe,
-	.remove = st_rtc_remove,
 };
 
 module_platform_driver(st_rtc_platform_driver);
diff --git a/drivers/rtc/rtc-stk17ta8.c b/drivers/rtc/rtc-stk17ta8.c
index e70b78d17a98..fccbecbb2c98 100644
--- a/drivers/rtc/rtc-stk17ta8.c
+++ b/drivers/rtc/rtc-stk17ta8.c
@@ -74,8 +74,7 @@ struct rtc_plat_data {
 
 static int stk17ta8_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr;
 	u8 flags;
 
@@ -97,8 +96,7 @@ static int stk17ta8_rtc_set_time(struct device *dev, struct rtc_time *tm)
 
 static int stk17ta8_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 	void __iomem *ioaddr = pdata->ioaddr;
 	unsigned int year, month, day, hour, minute, second, week;
 	unsigned int century;
@@ -163,8 +161,7 @@ static void stk17ta8_rtc_update_alarm(struct rtc_plat_data *pdata)
 
 static int stk17ta8_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	if (pdata->irq <= 0)
 		return -EINVAL;
@@ -180,8 +177,7 @@ static int stk17ta8_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 static int stk17ta8_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	if (pdata->irq <= 0)
 		return -EINVAL;
@@ -217,8 +213,7 @@ static irqreturn_t stk17ta8_rtc_interrupt(int irq, void *dev_id)
 static int stk17ta8_rtc_alarm_irq_enable(struct device *dev,
 	unsigned int enabled)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
+	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	if (pdata->irq <= 0)
 		return -EINVAL;
diff --git a/drivers/rtc/rtc-stm32.c b/drivers/rtc/rtc-stm32.c
index f25dabe8fd02..c5908cfea234 100644
--- a/drivers/rtc/rtc-stm32.c
+++ b/drivers/rtc/rtc-stm32.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) STMicroelectronics SA 2017
- * Author:  Amelie Delaunay <amelie.delaunay@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
+ * Copyright (C) STMicroelectronics 2017
+ * Author:  Amelie Delaunay <amelie.delaunay@st.com>
  */
 
 #include <linux/bcd.h>
@@ -11,20 +11,12 @@
 #include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/of_device.h>
+#include <linux/pm_wakeirq.h>
 #include <linux/regmap.h>
 #include <linux/rtc.h>
 
 #define DRIVER_NAME "stm32_rtc"
 
-/* STM32 RTC registers */
-#define STM32_RTC_TR		0x00
-#define STM32_RTC_DR		0x04
-#define STM32_RTC_CR		0x08
-#define STM32_RTC_ISR		0x0C
-#define STM32_RTC_PRER		0x10
-#define STM32_RTC_ALRMAR	0x1C
-#define STM32_RTC_WPR		0x24
-
 /* STM32_RTC_TR bit fields  */
 #define STM32_RTC_TR_SEC_SHIFT		0
 #define STM32_RTC_TR_SEC		GENMASK(6, 0)
@@ -48,7 +40,7 @@
 #define STM32_RTC_CR_ALRAE		BIT(8)
 #define STM32_RTC_CR_ALRAIE		BIT(12)
 
-/* STM32_RTC_ISR bit fields */
+/* STM32_RTC_ISR/STM32_RTC_ICSR bit fields */
 #define STM32_RTC_ISR_ALRAWF		BIT(0)
 #define STM32_RTC_ISR_INITS		BIT(4)
 #define STM32_RTC_ISR_RSF		BIT(5)
@@ -80,52 +72,87 @@
 #define STM32_RTC_ALRMXR_WDAY		GENMASK(27, 24)
 #define STM32_RTC_ALRMXR_DATE_MASK	BIT(31)
 
+/* STM32_RTC_SR/_SCR bit fields */
+#define STM32_RTC_SR_ALRA		BIT(0)
+
+/* STM32_RTC_VERR bit fields */
+#define STM32_RTC_VERR_MINREV_SHIFT	0
+#define STM32_RTC_VERR_MINREV		GENMASK(3, 0)
+#define STM32_RTC_VERR_MAJREV_SHIFT	4
+#define STM32_RTC_VERR_MAJREV		GENMASK(7, 4)
+
 /* STM32_RTC_WPR key constants */
 #define RTC_WPR_1ST_KEY			0xCA
 #define RTC_WPR_2ND_KEY			0x53
 #define RTC_WPR_WRONG_KEY		0xFF
 
-/*
- * RTC registers are protected against parasitic write access.
- * PWR_CR_DBP bit must be set to enable write access to RTC registers.
- */
-/* STM32_PWR_CR */
-#define PWR_CR				0x00
-/* STM32_PWR_CR bit field */
-#define PWR_CR_DBP			BIT(8)
+/* Max STM32 RTC register offset is 0x3FC */
+#define UNDEF_REG			0xFFFF
+
+struct stm32_rtc;
+
+struct stm32_rtc_registers {
+	u16 tr;
+	u16 dr;
+	u16 cr;
+	u16 isr;
+	u16 prer;
+	u16 alrmar;
+	u16 wpr;
+	u16 sr;
+	u16 scr;
+	u16 verr;
+};
+
+struct stm32_rtc_events {
+	u32 alra;
+};
 
 struct stm32_rtc_data {
+	const struct stm32_rtc_registers regs;
+	const struct stm32_rtc_events events;
+	void (*clear_events)(struct stm32_rtc *rtc, unsigned int flags);
 	bool has_pclk;
+	bool need_dbp;
+	bool has_wakeirq;
 };
 
 struct stm32_rtc {
 	struct rtc_device *rtc_dev;
 	void __iomem *base;
 	struct regmap *dbp;
-	struct stm32_rtc_data *data;
+	unsigned int dbp_reg;
+	unsigned int dbp_mask;
 	struct clk *pclk;
 	struct clk *rtc_ck;
+	const struct stm32_rtc_data *data;
 	int irq_alarm;
+	int wakeirq_alarm;
 };
 
 static void stm32_rtc_wpr_unlock(struct stm32_rtc *rtc)
 {
-	writel_relaxed(RTC_WPR_1ST_KEY, rtc->base + STM32_RTC_WPR);
-	writel_relaxed(RTC_WPR_2ND_KEY, rtc->base + STM32_RTC_WPR);
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
+
+	writel_relaxed(RTC_WPR_1ST_KEY, rtc->base + regs->wpr);
+	writel_relaxed(RTC_WPR_2ND_KEY, rtc->base + regs->wpr);
 }
 
 static void stm32_rtc_wpr_lock(struct stm32_rtc *rtc)
 {
-	writel_relaxed(RTC_WPR_WRONG_KEY, rtc->base + STM32_RTC_WPR);
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
+
+	writel_relaxed(RTC_WPR_WRONG_KEY, rtc->base + regs->wpr);
 }
 
 static int stm32_rtc_enter_init_mode(struct stm32_rtc *rtc)
 {
-	unsigned int isr = readl_relaxed(rtc->base + STM32_RTC_ISR);
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
+	unsigned int isr = readl_relaxed(rtc->base + regs->isr);
 
 	if (!(isr & STM32_RTC_ISR_INITF)) {
 		isr |= STM32_RTC_ISR_INIT;
-		writel_relaxed(isr, rtc->base + STM32_RTC_ISR);
+		writel_relaxed(isr, rtc->base + regs->isr);
 
 		/*
 		 * It takes around 2 rtc_ck clock cycles to enter in
@@ -134,7 +161,7 @@ static int stm32_rtc_enter_init_mode(struct stm32_rtc *rtc)
 		 * 1MHz, we poll every 10 us with a timeout of 100ms.
 		 */
 		return readl_relaxed_poll_timeout_atomic(
-					rtc->base + STM32_RTC_ISR,
+					rtc->base + regs->isr,
 					isr, (isr & STM32_RTC_ISR_INITF),
 					10, 100000);
 	}
@@ -144,40 +171,50 @@ static int stm32_rtc_enter_init_mode(struct stm32_rtc *rtc)
 
 static void stm32_rtc_exit_init_mode(struct stm32_rtc *rtc)
 {
-	unsigned int isr = readl_relaxed(rtc->base + STM32_RTC_ISR);
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
+	unsigned int isr = readl_relaxed(rtc->base + regs->isr);
 
 	isr &= ~STM32_RTC_ISR_INIT;
-	writel_relaxed(isr, rtc->base + STM32_RTC_ISR);
+	writel_relaxed(isr, rtc->base + regs->isr);
 }
 
 static int stm32_rtc_wait_sync(struct stm32_rtc *rtc)
 {
-	unsigned int isr = readl_relaxed(rtc->base + STM32_RTC_ISR);
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
+	unsigned int isr = readl_relaxed(rtc->base + regs->isr);
 
 	isr &= ~STM32_RTC_ISR_RSF;
-	writel_relaxed(isr, rtc->base + STM32_RTC_ISR);
+	writel_relaxed(isr, rtc->base + regs->isr);
 
 	/*
 	 * Wait for RSF to be set to ensure the calendar registers are
 	 * synchronised, it takes around 2 rtc_ck clock cycles
 	 */
-	return readl_relaxed_poll_timeout_atomic(rtc->base + STM32_RTC_ISR,
+	return readl_relaxed_poll_timeout_atomic(rtc->base + regs->isr,
 						 isr,
 						 (isr & STM32_RTC_ISR_RSF),
 						 10, 100000);
 }
 
+static void stm32_rtc_clear_event_flags(struct stm32_rtc *rtc,
+					unsigned int flags)
+{
+	rtc->data->clear_events(rtc, flags);
+}
+
 static irqreturn_t stm32_rtc_alarm_irq(int irq, void *dev_id)
 {
 	struct stm32_rtc *rtc = (struct stm32_rtc *)dev_id;
-	unsigned int isr, cr;
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
+	const struct stm32_rtc_events *evts = &rtc->data->events;
+	unsigned int status, cr;
 
 	mutex_lock(&rtc->rtc_dev->ops_lock);
 
-	isr = readl_relaxed(rtc->base + STM32_RTC_ISR);
-	cr = readl_relaxed(rtc->base + STM32_RTC_CR);
+	status = readl_relaxed(rtc->base + regs->sr);
+	cr = readl_relaxed(rtc->base + regs->cr);
 
-	if ((isr & STM32_RTC_ISR_ALRAF) &&
+	if ((status & evts->alra) &&
 	    (cr & STM32_RTC_CR_ALRAIE)) {
 		/* Alarm A flag - Alarm interrupt */
 		dev_dbg(&rtc->rtc_dev->dev, "Alarm occurred\n");
@@ -185,9 +222,8 @@ static irqreturn_t stm32_rtc_alarm_irq(int irq, void *dev_id)
 		/* Pass event to the kernel */
 		rtc_update_irq(rtc->rtc_dev, 1, RTC_IRQF | RTC_AF);
 
-		/* Clear event flag, otherwise new events won't be received */
-		writel_relaxed(isr & ~STM32_RTC_ISR_ALRAF,
-			       rtc->base + STM32_RTC_ISR);
+		/* Clear event flags, otherwise new events won't be received */
+		stm32_rtc_clear_event_flags(rtc, evts->alra);
 	}
 
 	mutex_unlock(&rtc->rtc_dev->ops_lock);
@@ -234,11 +270,12 @@ static void bcd2tm(struct rtc_time *tm)
 static int stm32_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
 	struct stm32_rtc *rtc = dev_get_drvdata(dev);
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
 	unsigned int tr, dr;
 
 	/* Time and Date in BCD format */
-	tr = readl_relaxed(rtc->base + STM32_RTC_TR);
-	dr = readl_relaxed(rtc->base + STM32_RTC_DR);
+	tr = readl_relaxed(rtc->base + regs->tr);
+	dr = readl_relaxed(rtc->base + regs->dr);
 
 	tm->tm_sec = (tr & STM32_RTC_TR_SEC) >> STM32_RTC_TR_SEC_SHIFT;
 	tm->tm_min = (tr & STM32_RTC_TR_MIN) >> STM32_RTC_TR_MIN_SHIFT;
@@ -259,6 +296,7 @@ static int stm32_rtc_read_time(struct device *dev, struct rtc_time *tm)
 static int stm32_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct stm32_rtc *rtc = dev_get_drvdata(dev);
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
 	unsigned int tr, dr;
 	int ret = 0;
 
@@ -283,8 +321,8 @@ static int stm32_rtc_set_time(struct device *dev, struct rtc_time *tm)
 		goto end;
 	}
 
-	writel_relaxed(tr, rtc->base + STM32_RTC_TR);
-	writel_relaxed(dr, rtc->base + STM32_RTC_DR);
+	writel_relaxed(tr, rtc->base + regs->tr);
+	writel_relaxed(dr, rtc->base + regs->dr);
 
 	stm32_rtc_exit_init_mode(rtc);
 
@@ -298,12 +336,14 @@ static int stm32_rtc_set_time(struct device *dev, struct rtc_time *tm)
 static int stm32_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
 	struct stm32_rtc *rtc = dev_get_drvdata(dev);
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
+	const struct stm32_rtc_events *evts = &rtc->data->events;
 	struct rtc_time *tm = &alrm->time;
-	unsigned int alrmar, cr, isr;
+	unsigned int alrmar, cr, status;
 
-	alrmar = readl_relaxed(rtc->base + STM32_RTC_ALRMAR);
-	cr = readl_relaxed(rtc->base + STM32_RTC_CR);
-	isr = readl_relaxed(rtc->base + STM32_RTC_ISR);
+	alrmar = readl_relaxed(rtc->base + regs->alrmar);
+	cr = readl_relaxed(rtc->base + regs->cr);
+	status = readl_relaxed(rtc->base + regs->sr);
 
 	if (alrmar & STM32_RTC_ALRMXR_DATE_MASK) {
 		/*
@@ -356,7 +396,7 @@ static int stm32_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	bcd2tm(tm);
 
 	alrm->enabled = (cr & STM32_RTC_CR_ALRAE) ? 1 : 0;
-	alrm->pending = (isr & STM32_RTC_ISR_ALRAF) ? 1 : 0;
+	alrm->pending = (status & evts->alra) ? 1 : 0;
 
 	return 0;
 }
@@ -364,9 +404,11 @@ static int stm32_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 static int stm32_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 {
 	struct stm32_rtc *rtc = dev_get_drvdata(dev);
-	unsigned int isr, cr;
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
+	const struct stm32_rtc_events *evts = &rtc->data->events;
+	unsigned int cr;
 
-	cr = readl_relaxed(rtc->base + STM32_RTC_CR);
+	cr = readl_relaxed(rtc->base + regs->cr);
 
 	stm32_rtc_wpr_unlock(rtc);
 
@@ -375,12 +417,10 @@ static int stm32_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 		cr |= (STM32_RTC_CR_ALRAIE | STM32_RTC_CR_ALRAE);
 	else
 		cr &= ~(STM32_RTC_CR_ALRAIE | STM32_RTC_CR_ALRAE);
-	writel_relaxed(cr, rtc->base + STM32_RTC_CR);
+	writel_relaxed(cr, rtc->base + regs->cr);
 
-	/* Clear event flag, otherwise new events won't be received */
-	isr = readl_relaxed(rtc->base + STM32_RTC_ISR);
-	isr &= ~STM32_RTC_ISR_ALRAF;
-	writel_relaxed(isr, rtc->base + STM32_RTC_ISR);
+	/* Clear event flags, otherwise new events won't be received */
+	stm32_rtc_clear_event_flags(rtc, evts->alra);
 
 	stm32_rtc_wpr_lock(rtc);
 
@@ -389,9 +429,10 @@ static int stm32_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 
 static int stm32_rtc_valid_alrm(struct stm32_rtc *rtc, struct rtc_time *tm)
 {
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
 	int cur_day, cur_mon, cur_year, cur_hour, cur_min, cur_sec;
-	unsigned int dr = readl_relaxed(rtc->base + STM32_RTC_DR);
-	unsigned int tr = readl_relaxed(rtc->base + STM32_RTC_TR);
+	unsigned int dr = readl_relaxed(rtc->base + regs->dr);
+	unsigned int tr = readl_relaxed(rtc->base + regs->tr);
 
 	cur_day = (dr & STM32_RTC_DR_DATE) >> STM32_RTC_DR_DATE_SHIFT;
 	cur_mon = (dr & STM32_RTC_DR_MONTH) >> STM32_RTC_DR_MONTH_SHIFT;
@@ -425,6 +466,7 @@ static int stm32_rtc_valid_alrm(struct stm32_rtc *rtc, struct rtc_time *tm)
 static int stm32_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
 	struct stm32_rtc *rtc = dev_get_drvdata(dev);
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
 	struct rtc_time *tm = &alrm->time;
 	unsigned int cr, isr, alrmar;
 	int ret = 0;
@@ -456,15 +498,15 @@ static int stm32_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	stm32_rtc_wpr_unlock(rtc);
 
 	/* Disable Alarm */
-	cr = readl_relaxed(rtc->base + STM32_RTC_CR);
+	cr = readl_relaxed(rtc->base + regs->cr);
 	cr &= ~STM32_RTC_CR_ALRAE;
-	writel_relaxed(cr, rtc->base + STM32_RTC_CR);
+	writel_relaxed(cr, rtc->base + regs->cr);
 
 	/*
 	 * Poll Alarm write flag to be sure that Alarm update is allowed: it
 	 * takes around 2 rtc_ck clock cycles
 	 */
-	ret = readl_relaxed_poll_timeout_atomic(rtc->base + STM32_RTC_ISR,
+	ret = readl_relaxed_poll_timeout_atomic(rtc->base + regs->isr,
 						isr,
 						(isr & STM32_RTC_ISR_ALRAWF),
 						10, 100000);
@@ -475,7 +517,7 @@ static int stm32_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	}
 
 	/* Write to Alarm register */
-	writel_relaxed(alrmar, rtc->base + STM32_RTC_ALRMAR);
+	writel_relaxed(alrmar, rtc->base + regs->alrmar);
 
 	if (alrm->enabled)
 		stm32_rtc_alarm_irq_enable(dev, 1);
@@ -496,17 +538,95 @@ static const struct rtc_class_ops stm32_rtc_ops = {
 	.alarm_irq_enable = stm32_rtc_alarm_irq_enable,
 };
 
+static void stm32_rtc_clear_events(struct stm32_rtc *rtc,
+				   unsigned int flags)
+{
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
+
+	/* Flags are cleared by writing 0 in RTC_ISR */
+	writel_relaxed(readl_relaxed(rtc->base + regs->isr) & ~flags,
+		       rtc->base + regs->isr);
+}
+
 static const struct stm32_rtc_data stm32_rtc_data = {
 	.has_pclk = false,
+	.need_dbp = true,
+	.has_wakeirq = false,
+	.regs = {
+		.tr = 0x00,
+		.dr = 0x04,
+		.cr = 0x08,
+		.isr = 0x0C,
+		.prer = 0x10,
+		.alrmar = 0x1C,
+		.wpr = 0x24,
+		.sr = 0x0C, /* set to ISR offset to ease alarm management */
+		.scr = UNDEF_REG,
+		.verr = UNDEF_REG,
+	},
+	.events = {
+		.alra = STM32_RTC_ISR_ALRAF,
+	},
+	.clear_events = stm32_rtc_clear_events,
 };
 
 static const struct stm32_rtc_data stm32h7_rtc_data = {
 	.has_pclk = true,
+	.need_dbp = true,
+	.has_wakeirq = false,
+	.regs = {
+		.tr = 0x00,
+		.dr = 0x04,
+		.cr = 0x08,
+		.isr = 0x0C,
+		.prer = 0x10,
+		.alrmar = 0x1C,
+		.wpr = 0x24,
+		.sr = 0x0C, /* set to ISR offset to ease alarm management */
+		.scr = UNDEF_REG,
+		.verr = UNDEF_REG,
+	},
+	.events = {
+		.alra = STM32_RTC_ISR_ALRAF,
+	},
+	.clear_events = stm32_rtc_clear_events,
+};
+
+static void stm32mp1_rtc_clear_events(struct stm32_rtc *rtc,
+				      unsigned int flags)
+{
+	struct stm32_rtc_registers regs = rtc->data->regs;
+
+	/* Flags are cleared by writing 1 in RTC_SCR */
+	writel_relaxed(flags, rtc->base + regs.scr);
+}
+
+static const struct stm32_rtc_data stm32mp1_data = {
+	.has_pclk = true,
+	.need_dbp = false,
+	.has_wakeirq = true,
+	.regs = {
+		.tr = 0x00,
+		.dr = 0x04,
+		.cr = 0x18,
+		.isr = 0x0C, /* named RTC_ICSR on stm32mp1 */
+		.prer = 0x10,
+		.alrmar = 0x40,
+		.wpr = 0x24,
+		.sr = 0x50,
+		.scr = 0x5C,
+		.verr = 0x3F4,
+	},
+	.events = {
+		.alra = STM32_RTC_SR_ALRA,
+	},
+	.clear_events = stm32mp1_rtc_clear_events,
 };
 
 static const struct of_device_id stm32_rtc_of_match[] = {
 	{ .compatible = "st,stm32-rtc", .data = &stm32_rtc_data },
 	{ .compatible = "st,stm32h7-rtc", .data = &stm32h7_rtc_data },
+	{ .compatible = "st,stm32mp1-rtc", .data = &stm32mp1_data },
 	{}
 };
 MODULE_DEVICE_TABLE(of, stm32_rtc_of_match);
@@ -514,6 +634,7 @@ MODULE_DEVICE_TABLE(of, stm32_rtc_of_match);
 static int stm32_rtc_init(struct platform_device *pdev,
 			  struct stm32_rtc *rtc)
 {
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
 	unsigned int prer, pred_a, pred_s, pred_a_max, pred_s_max, cr;
 	unsigned int rate;
 	int ret = 0;
@@ -554,14 +675,14 @@ static int stm32_rtc_init(struct platform_device *pdev,
 	}
 
 	prer = (pred_s << STM32_RTC_PRER_PRED_S_SHIFT) & STM32_RTC_PRER_PRED_S;
-	writel_relaxed(prer, rtc->base + STM32_RTC_PRER);
+	writel_relaxed(prer, rtc->base + regs->prer);
 	prer |= (pred_a << STM32_RTC_PRER_PRED_A_SHIFT) & STM32_RTC_PRER_PRED_A;
-	writel_relaxed(prer, rtc->base + STM32_RTC_PRER);
+	writel_relaxed(prer, rtc->base + regs->prer);
 
 	/* Force 24h time format */
-	cr = readl_relaxed(rtc->base + STM32_RTC_CR);
+	cr = readl_relaxed(rtc->base + regs->cr);
 	cr &= ~STM32_RTC_CR_FMT;
-	writel_relaxed(cr, rtc->base + STM32_RTC_CR);
+	writel_relaxed(cr, rtc->base + regs->cr);
 
 	stm32_rtc_exit_init_mode(rtc);
 
@@ -575,8 +696,8 @@ static int stm32_rtc_init(struct platform_device *pdev,
 static int stm32_rtc_probe(struct platform_device *pdev)
 {
 	struct stm32_rtc *rtc;
+	const struct stm32_rtc_registers *regs;
 	struct resource *res;
-	const struct of_device_id *match;
 	int ret;
 
 	rtc = devm_kzalloc(&pdev->dev, sizeof(*rtc), GFP_KERNEL);
@@ -588,15 +709,32 @@ static int stm32_rtc_probe(struct platform_device *pdev)
 	if (IS_ERR(rtc->base))
 		return PTR_ERR(rtc->base);
 
-	rtc->dbp = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
-						   "st,syscfg");
-	if (IS_ERR(rtc->dbp)) {
-		dev_err(&pdev->dev, "no st,syscfg\n");
-		return PTR_ERR(rtc->dbp);
-	}
+	rtc->data = (struct stm32_rtc_data *)
+		    of_device_get_match_data(&pdev->dev);
+	regs = &rtc->data->regs;
+
+	if (rtc->data->need_dbp) {
+		rtc->dbp = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
+							   "st,syscfg");
+		if (IS_ERR(rtc->dbp)) {
+			dev_err(&pdev->dev, "no st,syscfg\n");
+			return PTR_ERR(rtc->dbp);
+		}
+
+		ret = of_property_read_u32_index(pdev->dev.of_node, "st,syscfg",
+						 1, &rtc->dbp_reg);
+		if (ret) {
+			dev_err(&pdev->dev, "can't read DBP register offset\n");
+			return ret;
+		}
 
-	match = of_match_device(stm32_rtc_of_match, &pdev->dev);
-	rtc->data = (struct stm32_rtc_data *)match->data;
+		ret = of_property_read_u32_index(pdev->dev.of_node, "st,syscfg",
+						 2, &rtc->dbp_mask);
+		if (ret) {
+			dev_err(&pdev->dev, "can't read DBP register mask\n");
+			return ret;
+		}
+	}
 
 	if (!rtc->data->has_pclk) {
 		rtc->pclk = NULL;
@@ -624,11 +762,13 @@ static int stm32_rtc_probe(struct platform_device *pdev)
 	if (ret)
 		goto err;
 
-	regmap_update_bits(rtc->dbp, PWR_CR, PWR_CR_DBP, PWR_CR_DBP);
+	if (rtc->data->need_dbp)
+		regmap_update_bits(rtc->dbp, rtc->dbp_reg,
+				   rtc->dbp_mask, rtc->dbp_mask);
 
 	/*
 	 * After a system reset, RTC_ISR.INITS flag can be read to check if
-	 * the calendar has been initalized or not. INITS flag is reset by a
+	 * the calendar has been initialized or not. INITS flag is reset by a
 	 * power-on reset (no vbat, no power-supply). It is not reset if
 	 * rtc_ck parent clock has changed (so RTC prescalers need to be
 	 * changed). That's why we cannot rely on this flag to know if RTC
@@ -645,15 +785,22 @@ static int stm32_rtc_probe(struct platform_device *pdev)
 		goto err;
 	}
 
-	platform_set_drvdata(pdev, rtc);
-
 	ret = device_init_wakeup(&pdev->dev, true);
+	if (rtc->data->has_wakeirq) {
+		rtc->wakeirq_alarm = platform_get_irq(pdev, 1);
+		if (rtc->wakeirq_alarm <= 0)
+			ret = rtc->wakeirq_alarm;
+		else
+			ret = dev_pm_set_dedicated_wake_irq(&pdev->dev,
+							    rtc->wakeirq_alarm);
+	}
 	if (ret)
-		dev_warn(&pdev->dev,
-			 "alarm won't be able to wake up the system");
+		dev_warn(&pdev->dev, "alarm can't wake up the system: %d", ret);
+
+	platform_set_drvdata(pdev, rtc);
 
 	rtc->rtc_dev = devm_rtc_device_register(&pdev->dev, pdev->name,
-			&stm32_rtc_ops, THIS_MODULE);
+						&stm32_rtc_ops, THIS_MODULE);
 	if (IS_ERR(rtc->rtc_dev)) {
 		ret = PTR_ERR(rtc->rtc_dev);
 		dev_err(&pdev->dev, "rtc device registration failed, err=%d\n",
@@ -663,8 +810,7 @@ static int stm32_rtc_probe(struct platform_device *pdev)
 
 	/* Handle RTC alarm interrupts */
 	ret = devm_request_threaded_irq(&pdev->dev, rtc->irq_alarm, NULL,
-					stm32_rtc_alarm_irq,
-					IRQF_TRIGGER_RISING | IRQF_ONESHOT,
+					stm32_rtc_alarm_irq, IRQF_ONESHOT,
 					pdev->name, rtc);
 	if (ret) {
 		dev_err(&pdev->dev, "IRQ%d (alarm interrupt) already claimed\n",
@@ -676,17 +822,27 @@ static int stm32_rtc_probe(struct platform_device *pdev)
 	 * If INITS flag is reset (calendar year field set to 0x00), calendar
 	 * must be initialized
 	 */
-	if (!(readl_relaxed(rtc->base + STM32_RTC_ISR) & STM32_RTC_ISR_INITS))
+	if (!(readl_relaxed(rtc->base + regs->isr) & STM32_RTC_ISR_INITS))
 		dev_warn(&pdev->dev, "Date/Time must be initialized\n");
 
+	if (regs->verr != UNDEF_REG) {
+		u32 ver = readl_relaxed(rtc->base + regs->verr);
+
+		dev_info(&pdev->dev, "registered rev:%d.%d\n",
+			 (ver >> STM32_RTC_VERR_MAJREV_SHIFT) & 0xF,
+			 (ver >> STM32_RTC_VERR_MINREV_SHIFT) & 0xF);
+	}
+
 	return 0;
 err:
 	if (rtc->data->has_pclk)
 		clk_disable_unprepare(rtc->pclk);
 	clk_disable_unprepare(rtc->rtc_ck);
 
-	regmap_update_bits(rtc->dbp, PWR_CR, PWR_CR_DBP, 0);
+	if (rtc->data->need_dbp)
+		regmap_update_bits(rtc->dbp, rtc->dbp_reg, rtc->dbp_mask, 0);
 
+	dev_pm_clear_wake_irq(&pdev->dev);
 	device_init_wakeup(&pdev->dev, false);
 
 	return ret;
@@ -695,22 +851,25 @@ static int stm32_rtc_probe(struct platform_device *pdev)
 static int stm32_rtc_remove(struct platform_device *pdev)
 {
 	struct stm32_rtc *rtc = platform_get_drvdata(pdev);
+	const struct stm32_rtc_registers *regs = &rtc->data->regs;
 	unsigned int cr;
 
 	/* Disable interrupts */
 	stm32_rtc_wpr_unlock(rtc);
-	cr = readl_relaxed(rtc->base + STM32_RTC_CR);
+	cr = readl_relaxed(rtc->base + regs->cr);
 	cr &= ~STM32_RTC_CR_ALRAIE;
-	writel_relaxed(cr, rtc->base + STM32_RTC_CR);
+	writel_relaxed(cr, rtc->base + regs->cr);
 	stm32_rtc_wpr_lock(rtc);
 
 	clk_disable_unprepare(rtc->rtc_ck);
 	if (rtc->data->has_pclk)
 		clk_disable_unprepare(rtc->pclk);
 
-	/* Enable backup domain write protection */
-	regmap_update_bits(rtc->dbp, PWR_CR, PWR_CR_DBP, 0);
+	/* Enable backup domain write protection if needed */
+	if (rtc->data->need_dbp)
+		regmap_update_bits(rtc->dbp, rtc->dbp_reg, rtc->dbp_mask, 0);
 
+	dev_pm_clear_wake_irq(&pdev->dev);
 	device_init_wakeup(&pdev->dev, false);
 
 	return 0;
diff --git a/drivers/rtc/rtc-test.c b/drivers/rtc/rtc-test.c
index 3a2da4c892d6..a0d1571c4af6 100644
--- a/drivers/rtc/rtc-test.c
+++ b/drivers/rtc/rtc-test.c
@@ -13,10 +13,6 @@
 #include <linux/rtc.h>
 #include <linux/platform_device.h>
 
-static int test_mmss64;
-module_param(test_mmss64, int, 0644);
-MODULE_PARM_DESC(test_mmss64, "Test struct rtc_class_ops.set_mmss64().");
-
 static struct platform_device *test0 = NULL, *test1 = NULL;
 
 static int test_rtc_read_alarm(struct device *dev,
@@ -44,12 +40,6 @@ static int test_rtc_set_mmss64(struct device *dev, time64_t secs)
 	return 0;
 }
 
-static int test_rtc_set_mmss(struct device *dev, unsigned long secs)
-{
-	dev_info(dev, "%s, secs = %lu\n", __func__, secs);
-	return 0;
-}
-
 static int test_rtc_proc(struct device *dev, struct seq_file *seq)
 {
 	struct platform_device *plat_dev = to_platform_device(dev);
@@ -65,12 +55,12 @@ static int test_rtc_alarm_irq_enable(struct device *dev, unsigned int enable)
 	return 0;
 }
 
-static struct rtc_class_ops test_rtc_ops = {
+static const struct rtc_class_ops test_rtc_ops = {
 	.proc = test_rtc_proc,
 	.read_time = test_rtc_read_time,
 	.read_alarm = test_rtc_read_alarm,
 	.set_alarm = test_rtc_set_alarm,
-	.set_mmss = test_rtc_set_mmss,
+	.set_mmss64 = test_rtc_set_mmss64,
 	.alarm_irq_enable = test_rtc_alarm_irq_enable,
 };
 
@@ -84,8 +74,7 @@ static ssize_t test_irq_store(struct device *dev,
 				const char *buf, size_t count)
 {
 	int retval;
-	struct platform_device *plat_dev = to_platform_device(dev);
-	struct rtc_device *rtc = platform_get_drvdata(plat_dev);
+	struct rtc_device *rtc = dev_get_drvdata(dev);
 
 	retval = count;
 	if (strncmp(buf, "tick", 4) == 0 && rtc->pie_enabled)
@@ -111,11 +100,6 @@ static int test_probe(struct platform_device *plat_dev)
 	int err;
 	struct rtc_device *rtc;
 
-	if (test_mmss64) {
-		test_rtc_ops.set_mmss64 = test_rtc_set_mmss64;
-		test_rtc_ops.set_mmss = NULL;
-	}
-
 	rtc = devm_rtc_device_register(&plat_dev->dev, "test",
 				&test_rtc_ops, THIS_MODULE);
 	if (IS_ERR(rtc)) {
diff --git a/drivers/rtc/rtc-tps6586x.c b/drivers/rtc/rtc-tps6586x.c
index d7785ae0a2b4..d6434e514a52 100644
--- a/drivers/rtc/rtc-tps6586x.c
+++ b/drivers/rtc/rtc-tps6586x.c
@@ -58,7 +58,6 @@ struct tps6586x_rtc {
 	struct rtc_device	*rtc;
 	int			irq;
 	bool			irq_en;
-	unsigned long long	epoch_start;
 };
 
 static inline struct device *to_tps6586x_dev(struct device *dev)
@@ -68,10 +67,9 @@ static inline struct device *to_tps6586x_dev(struct device *dev)
 
 static int tps6586x_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct tps6586x_rtc *rtc = dev_get_drvdata(dev);
 	struct device *tps_dev = to_tps6586x_dev(dev);
 	unsigned long long ticks = 0;
-	unsigned long seconds;
+	time64_t seconds;
 	u8 buff[6];
 	int ret;
 	int i;
@@ -88,26 +86,20 @@ static int tps6586x_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	}
 
 	seconds = ticks >> 10;
-	seconds += rtc->epoch_start;
-	rtc_time_to_tm(seconds, tm);
+	rtc_time64_to_tm(seconds, tm);
+
 	return 0;
 }
 
 static int tps6586x_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct tps6586x_rtc *rtc = dev_get_drvdata(dev);
 	struct device *tps_dev = to_tps6586x_dev(dev);
 	unsigned long long ticks;
-	unsigned long seconds;
+	time64_t seconds;
 	u8 buff[5];
 	int ret;
 
-	rtc_tm_to_time(tm, &seconds);
-	if (seconds < rtc->epoch_start) {
-		dev_err(dev, "requested time unsupported\n");
-		return -EINVAL;
-	}
-	seconds -= rtc->epoch_start;
+	seconds = rtc_tm_to_time64(tm);
 
 	ticks = (unsigned long long)seconds << 10;
 	buff[0] = (ticks >> 32) & 0xff;
@@ -155,9 +147,8 @@ static int tps6586x_rtc_alarm_irq_enable(struct device *dev,
 
 static int tps6586x_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct tps6586x_rtc *rtc = dev_get_drvdata(dev);
 	struct device *tps_dev = to_tps6586x_dev(dev);
-	unsigned long seconds;
+	time64_t seconds;
 	unsigned long ticks;
 	unsigned long rtc_current_time;
 	unsigned long long rticks = 0;
@@ -166,12 +157,7 @@ static int tps6586x_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	int ret;
 	int i;
 
-	rtc_tm_to_time(&alrm->time, &seconds);
-
-	if (alrm->enabled && (seconds < rtc->epoch_start)) {
-		dev_err(dev, "can't set alarm to requested time\n");
-		return -EINVAL;
-	}
+	seconds = rtc_tm_to_time64(&alrm->time);
 
 	ret = tps6586x_rtc_alarm_irq_enable(dev, alrm->enabled);
 	if (ret < 0) {
@@ -179,7 +165,6 @@ static int tps6586x_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 		return ret;
 	}
 
-	seconds -= rtc->epoch_start;
 	ret = tps6586x_reads(tps_dev, RTC_COUNT4_DUMMYREAD,
 			sizeof(rbuff), rbuff);
 	if (ret < 0) {
@@ -210,10 +195,9 @@ static int tps6586x_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 static int tps6586x_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct tps6586x_rtc *rtc = dev_get_drvdata(dev);
 	struct device *tps_dev = to_tps6586x_dev(dev);
 	unsigned long ticks;
-	unsigned long seconds;
+	time64_t seconds;
 	u8 buff[3];
 	int ret;
 
@@ -225,9 +209,8 @@ static int tps6586x_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 	ticks = (buff[0] << 16) | (buff[1] << 8) | buff[2];
 	seconds = ticks >> 10;
-	seconds += rtc->epoch_start;
 
-	rtc_time_to_tm(seconds, &alrm->time);
+	rtc_time64_to_tm(seconds, &alrm->time);
 	return 0;
 }
 
@@ -260,9 +243,6 @@ static int tps6586x_rtc_probe(struct platform_device *pdev)
 	rtc->dev = &pdev->dev;
 	rtc->irq = platform_get_irq(pdev, 0);
 
-	/* Set epoch start as 00:00:00:01:01:2009 */
-	rtc->epoch_start = mktime(2009, 1, 1, 0, 0, 0);
-
 	/* 1 kHz tick mode, enable tick counting */
 	ret = tps6586x_update(tps_dev, RTC_CTRL,
 		RTC_ENABLE | OSC_SRC_SEL |
@@ -276,14 +256,18 @@ static int tps6586x_rtc_probe(struct platform_device *pdev)
 	device_init_wakeup(&pdev->dev, 1);
 
 	platform_set_drvdata(pdev, rtc);
-	rtc->rtc = devm_rtc_device_register(&pdev->dev, dev_name(&pdev->dev),
-				       &tps6586x_rtc_ops, THIS_MODULE);
+	rtc->rtc = devm_rtc_allocate_device(&pdev->dev);
 	if (IS_ERR(rtc->rtc)) {
 		ret = PTR_ERR(rtc->rtc);
-		dev_err(&pdev->dev, "RTC device register: ret %d\n", ret);
+		dev_err(&pdev->dev, "RTC allocate device: ret %d\n", ret);
 		goto fail_rtc_register;
 	}
 
+	rtc->rtc->ops = &tps6586x_rtc_ops;
+	rtc->rtc->range_max = (1ULL << 30) - 1; /* 30-bit seconds */
+	rtc->rtc->start_secs = mktime64(2009, 1, 1, 0, 0, 0);
+	rtc->rtc->set_start_time = true;
+
 	ret = devm_request_threaded_irq(&pdev->dev, rtc->irq, NULL,
 				tps6586x_rtc_irq,
 				IRQF_ONESHOT,
@@ -294,6 +278,13 @@ static int tps6586x_rtc_probe(struct platform_device *pdev)
 		goto fail_rtc_register;
 	}
 	disable_irq(rtc->irq);
+
+	ret = rtc_register_device(rtc->rtc);
+	if (ret) {
+		dev_err(&pdev->dev, "RTC device register: ret %d\n", ret);
+		goto fail_rtc_register;
+	}
+
 	return 0;
 
 fail_rtc_register:
diff --git a/drivers/rtc/rtc-tps65910.c b/drivers/rtc/rtc-tps65910.c
index d0244d7979fc..a9bbd022aeef 100644
--- a/drivers/rtc/rtc-tps65910.c
+++ b/drivers/rtc/rtc-tps65910.c
@@ -380,6 +380,10 @@ static int tps65910_rtc_probe(struct platform_device *pdev)
 	if (!tps_rtc)
 		return -ENOMEM;
 
+	tps_rtc->rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(tps_rtc->rtc))
+		return PTR_ERR(tps_rtc->rtc);
+
 	/* Clear pending interrupts */
 	ret = regmap_read(tps65910->regmap, TPS65910_RTC_STATUS, &rtc_reg);
 	if (ret < 0)
@@ -421,10 +425,12 @@ static int tps65910_rtc_probe(struct platform_device *pdev)
 	tps_rtc->irq = irq;
 	device_set_wakeup_capable(&pdev->dev, 1);
 
-	tps_rtc->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
-		&tps65910_rtc_ops, THIS_MODULE);
-	if (IS_ERR(tps_rtc->rtc)) {
-		ret = PTR_ERR(tps_rtc->rtc);
+	tps_rtc->rtc->ops = &tps65910_rtc_ops;
+	tps_rtc->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000;
+	tps_rtc->rtc->range_max = RTC_TIMESTAMP_END_2099;
+
+	ret = rtc_register_device(tps_rtc->rtc);
+	if (ret) {
 		dev_err(&pdev->dev, "RTC device register: err %d\n", ret);
 		return ret;
 	}
@@ -432,17 +438,6 @@ static int tps65910_rtc_probe(struct platform_device *pdev)
 	return 0;
 }
 
-/*
- * Disable tps65910 RTC interrupts.
- * Sets status flag to free.
- */
-static int tps65910_rtc_remove(struct platform_device *pdev)
-{
-	tps65910_rtc_alarm_irq_enable(&pdev->dev, 0);
-
-	return 0;
-}
-
 #ifdef CONFIG_PM_SLEEP
 static int tps65910_rtc_suspend(struct device *dev)
 {
@@ -468,7 +463,6 @@ static SIMPLE_DEV_PM_OPS(tps65910_rtc_pm_ops, tps65910_rtc_suspend,
 
 static struct platform_driver tps65910_rtc_driver = {
 	.probe		= tps65910_rtc_probe,
-	.remove		= tps65910_rtc_remove,
 	.driver		= {
 		.name	= "tps65910-rtc",
 		.pm	= &tps65910_rtc_pm_ops,
diff --git a/drivers/rtc/rtc-vr41xx.c b/drivers/rtc/rtc-vr41xx.c
index 7ce22967fd16..70f013e692b0 100644
--- a/drivers/rtc/rtc-vr41xx.c
+++ b/drivers/rtc/rtc-vr41xx.c
@@ -88,7 +88,7 @@ static unsigned int alarm_enabled;
 static int aie_irq;
 static int pie_irq;
 
-static inline unsigned long read_elapsed_second(void)
+static inline time64_t read_elapsed_second(void)
 {
 
 	unsigned long first_low, first_mid, first_high;
@@ -105,10 +105,10 @@ static inline unsigned long read_elapsed_second(void)
 	} while (first_low != second_low || first_mid != second_mid ||
 		 first_high != second_high);
 
-	return (first_high << 17) | (first_mid << 1) | (first_low >> 15);
+	return ((u64)first_high << 17) | (first_mid << 1) | (first_low >> 15);
 }
 
-static inline void write_elapsed_second(unsigned long sec)
+static inline void write_elapsed_second(time64_t sec)
 {
 	spin_lock_irq(&rtc_lock);
 
@@ -121,22 +121,22 @@ static inline void write_elapsed_second(unsigned long sec)
 
 static int vr41xx_rtc_read_time(struct device *dev, struct rtc_time *time)
 {
-	unsigned long epoch_sec, elapsed_sec;
+	time64_t epoch_sec, elapsed_sec;
 
-	epoch_sec = mktime(epoch, 1, 1, 0, 0, 0);
+	epoch_sec = mktime64(epoch, 1, 1, 0, 0, 0);
 	elapsed_sec = read_elapsed_second();
 
-	rtc_time_to_tm(epoch_sec + elapsed_sec, time);
+	rtc_time64_to_tm(epoch_sec + elapsed_sec, time);
 
 	return 0;
 }
 
 static int vr41xx_rtc_set_time(struct device *dev, struct rtc_time *time)
 {
-	unsigned long epoch_sec, current_sec;
+	time64_t epoch_sec, current_sec;
 
-	epoch_sec = mktime(epoch, 1, 1, 0, 0, 0);
-	current_sec = mktime(time->tm_year + 1900, time->tm_mon + 1, time->tm_mday,
+	epoch_sec = mktime64(epoch, 1, 1, 0, 0, 0);
+	current_sec = mktime64(time->tm_year + 1900, time->tm_mon + 1, time->tm_mday,
 			     time->tm_hour, time->tm_min, time->tm_sec);
 
 	write_elapsed_second(current_sec - epoch_sec);
@@ -165,11 +165,11 @@ static int vr41xx_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *wkalrm)
 
 static int vr41xx_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *wkalrm)
 {
-	unsigned long alarm_sec;
+	time64_t alarm_sec;
 	struct rtc_time *time = &wkalrm->time;
 
-	alarm_sec = mktime(time->tm_year + 1900, time->tm_mon + 1, time->tm_mday,
-			   time->tm_hour, time->tm_min, time->tm_sec);
+	alarm_sec = mktime64(time->tm_year + 1900, time->tm_mon + 1, time->tm_mday,
+			     time->tm_hour, time->tm_min, time->tm_sec);
 
 	spin_lock_irq(&rtc_lock);
 
@@ -292,13 +292,16 @@ static int rtc_probe(struct platform_device *pdev)
 		goto err_rtc1_iounmap;
 	}
 
-	rtc = devm_rtc_device_register(&pdev->dev, rtc_name, &vr41xx_rtc_ops,
-					THIS_MODULE);
+	rtc = devm_rtc_allocate_device(&pdev->dev);
 	if (IS_ERR(rtc)) {
 		retval = PTR_ERR(rtc);
 		goto err_iounmap_all;
 	}
 
+	rtc->ops = &vr41xx_rtc_ops;
+
+	/* 48-bit counter at 32.768 kHz */
+	rtc->range_max = (1ULL << 33) - 1;
 	rtc->max_user_freq = MAX_PERIODIC_RATE;
 
 	spin_lock_irq(&rtc_lock);
@@ -340,6 +343,10 @@ static int rtc_probe(struct platform_device *pdev)
 
 	dev_info(&pdev->dev, "Real Time Clock of NEC VR4100 series\n");
 
+	retval = rtc_register_device(rtc);
+	if (retval)
+		goto err_iounmap_all;
+
 	return 0;
 
 err_iounmap_all:
diff --git a/drivers/rtc/rtc-zynqmp.c b/drivers/rtc/rtc-zynqmp.c
index fba994dc31eb..c532bd13fbe5 100644
--- a/drivers/rtc/rtc-zynqmp.c
+++ b/drivers/rtc/rtc-zynqmp.c
@@ -278,10 +278,9 @@ static int xlnx_rtc_remove(struct platform_device *pdev)
 
 static int __maybe_unused xlnx_rtc_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct xlnx_rtc_dev *xrtcdev = platform_get_drvdata(pdev);
+	struct xlnx_rtc_dev *xrtcdev = dev_get_drvdata(dev);
 
-	if (device_may_wakeup(&pdev->dev))
+	if (device_may_wakeup(dev))
 		enable_irq_wake(xrtcdev->alarm_irq);
 	else
 		xlnx_rtc_alarm_irq_enable(dev, 0);
@@ -291,10 +290,9 @@ static int __maybe_unused xlnx_rtc_suspend(struct device *dev)
 
 static int __maybe_unused xlnx_rtc_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct xlnx_rtc_dev *xrtcdev = platform_get_drvdata(pdev);
+	struct xlnx_rtc_dev *xrtcdev = dev_get_drvdata(dev);
 
-	if (device_may_wakeup(&pdev->dev))
+	if (device_may_wakeup(dev))
 		disable_irq_wake(xrtcdev->alarm_irq);
 	else
 		xlnx_rtc_alarm_irq_enable(dev, 1);
diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c
index 04143c08bd6e..73cce3ecb97f 100644
--- a/drivers/s390/block/dasd.c
+++ b/drivers/s390/block/dasd.c
@@ -2569,14 +2569,11 @@ EXPORT_SYMBOL(dasd_sleep_on_immediatly);
  * Cancellation of a request is an asynchronous operation! The calling
  * function has to wait until the request is properly returned via callback.
  */
-int dasd_cancel_req(struct dasd_ccw_req *cqr)
+static int __dasd_cancel_req(struct dasd_ccw_req *cqr)
 {
 	struct dasd_device *device = cqr->startdev;
-	unsigned long flags;
-	int rc;
+	int rc = 0;
 
-	rc = 0;
-	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
 	switch (cqr->status) {
 	case DASD_CQR_QUEUED:
 		/* request was not started - just set to cleared */
@@ -2596,11 +2593,21 @@ int dasd_cancel_req(struct dasd_ccw_req *cqr)
 	default: /* already finished or clear pending - do nothing */
 		break;
 	}
-	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
 	dasd_schedule_device_bh(device);
 	return rc;
 }
-EXPORT_SYMBOL(dasd_cancel_req);
+
+int dasd_cancel_req(struct dasd_ccw_req *cqr)
+{
+	struct dasd_device *device = cqr->startdev;
+	unsigned long flags;
+	int rc;
+
+	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
+	rc = __dasd_cancel_req(cqr);
+	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
+	return rc;
+}
 
 /*
  * SECTION: Operations of the dasd_block layer.
@@ -3034,7 +3041,8 @@ static blk_status_t do_dasd_request(struct blk_mq_hw_ctx *hctx,
 	cqr->callback_data = req;
 	cqr->status = DASD_CQR_FILLED;
 	cqr->dq = dq;
-	req->completion_data = cqr;
+	*((struct dasd_ccw_req **) blk_mq_rq_to_pdu(req)) = cqr;
+
 	blk_mq_start_request(req);
 	spin_lock(&block->queue_lock);
 	list_add_tail(&cqr->blocklist, &block->ccw_queue);
@@ -3053,19 +3061,20 @@ static blk_status_t do_dasd_request(struct blk_mq_hw_ctx *hctx,
  *
  * Return values:
  * BLK_EH_RESET_TIMER if the request should be left running
- * BLK_EH_NOT_HANDLED if the request is handled or terminated
+ * BLK_EH_DONE if the request is handled or terminated
  *		      by the driver.
  */
 enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved)
 {
-	struct dasd_ccw_req *cqr = req->completion_data;
 	struct dasd_block *block = req->q->queuedata;
 	struct dasd_device *device;
+	struct dasd_ccw_req *cqr;
 	unsigned long flags;
 	int rc = 0;
 
+	cqr = *((struct dasd_ccw_req **) blk_mq_rq_to_pdu(req));
 	if (!cqr)
-		return BLK_EH_NOT_HANDLED;
+		return BLK_EH_DONE;
 
 	spin_lock_irqsave(&cqr->dq->lock, flags);
 	device = cqr->startdev ? cqr->startdev : block->base;
@@ -3082,12 +3091,10 @@ enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved)
 	cqr->retries = -1;
 	cqr->intrc = -ETIMEDOUT;
 	if (cqr->status >= DASD_CQR_QUEUED) {
-		spin_unlock(get_ccwdev_lock(device->cdev));
-		rc = dasd_cancel_req(cqr);
+		rc = __dasd_cancel_req(cqr);
 	} else if (cqr->status == DASD_CQR_FILLED ||
 		   cqr->status == DASD_CQR_NEED_ERP) {
 		cqr->status = DASD_CQR_TERMINATED;
-		spin_unlock(get_ccwdev_lock(device->cdev));
 	} else if (cqr->status == DASD_CQR_IN_ERP) {
 		struct dasd_ccw_req *searchcqr, *nextcqr, *tmpcqr;
 
@@ -3102,9 +3109,7 @@ enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved)
 			searchcqr->retries = -1;
 			searchcqr->intrc = -ETIMEDOUT;
 			if (searchcqr->status >= DASD_CQR_QUEUED) {
-				spin_unlock(get_ccwdev_lock(device->cdev));
-				rc = dasd_cancel_req(searchcqr);
-				spin_lock(get_ccwdev_lock(device->cdev));
+				rc = __dasd_cancel_req(searchcqr);
 			} else if ((searchcqr->status == DASD_CQR_FILLED) ||
 				   (searchcqr->status == DASD_CQR_NEED_ERP)) {
 				searchcqr->status = DASD_CQR_TERMINATED;
@@ -3118,13 +3123,13 @@ enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved)
 			}
 			break;
 		}
-		spin_unlock(get_ccwdev_lock(device->cdev));
 	}
+	spin_unlock(get_ccwdev_lock(device->cdev));
 	dasd_schedule_block_bh(block);
 	spin_unlock(&block->queue_lock);
 	spin_unlock_irqrestore(&cqr->dq->lock, flags);
 
-	return rc ? BLK_EH_RESET_TIMER : BLK_EH_NOT_HANDLED;
+	return rc ? BLK_EH_RESET_TIMER : BLK_EH_DONE;
 }
 
 static int dasd_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
@@ -3169,6 +3174,7 @@ static int dasd_alloc_queue(struct dasd_block *block)
 	int rc;
 
 	block->tag_set.ops = &dasd_mq_ops;
+	block->tag_set.cmd_size = sizeof(struct dasd_ccw_req *);
 	block->tag_set.nr_hw_queues = DASD_NR_HW_QUEUES;
 	block->tag_set.queue_depth = DASD_MAX_LCU_DEV * DASD_REQ_PER_DEV;
 	block->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
diff --git a/drivers/s390/block/dasd_ioctl.c b/drivers/s390/block/dasd_ioctl.c
index 7bdc6aaa0ba3..2016e0ed5865 100644
--- a/drivers/s390/block/dasd_ioctl.c
+++ b/drivers/s390/block/dasd_ioctl.c
@@ -18,7 +18,6 @@
 #include <linux/fs.h>
 #include <linux/blkpg.h>
 #include <linux/slab.h>
-#include <asm/compat.h>
 #include <asm/ccwdev.h>
 #include <asm/schid.h>
 #include <asm/cmb.h>
diff --git a/drivers/s390/block/dasd_proc.c b/drivers/s390/block/dasd_proc.c
index c33788a829c3..5cb80c645489 100644
--- a/drivers/s390/block/dasd_proc.c
+++ b/drivers/s390/block/dasd_proc.c
@@ -131,19 +131,6 @@ static const struct seq_operations dasd_devices_seq_ops = {
 	.show		= dasd_devices_show,
 };
 
-static int dasd_devices_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &dasd_devices_seq_ops);
-}
-
-static const struct file_operations dasd_devices_file_ops = {
-	.owner		= THIS_MODULE,
-	.open		= dasd_devices_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
 #ifdef CONFIG_DASD_PROFILE
 static int dasd_stats_all_block_on(void)
 {
@@ -352,10 +339,10 @@ dasd_proc_init(void)
 	dasd_proc_root_entry = proc_mkdir("dasd", NULL);
 	if (!dasd_proc_root_entry)
 		goto out_nodasd;
-	dasd_devices_entry = proc_create("devices",
+	dasd_devices_entry = proc_create_seq("devices",
 					 S_IFREG | S_IRUGO | S_IWUSR,
 					 dasd_proc_root_entry,
-					 &dasd_devices_file_ops);
+					 &dasd_devices_seq_ops);
 	if (!dasd_devices_entry)
 		goto out_nodevices;
 	dasd_statistics_entry = proc_create("statistics",
diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c
index 0a312e450207..29024492b8ed 100644
--- a/drivers/s390/block/dcssblk.c
+++ b/drivers/s390/block/dcssblk.c
@@ -51,9 +51,16 @@ static size_t dcssblk_dax_copy_from_iter(struct dax_device *dax_dev,
 	return copy_from_iter(addr, bytes, i);
 }
 
+static size_t dcssblk_dax_copy_to_iter(struct dax_device *dax_dev,
+		pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
+{
+	return copy_to_iter(addr, bytes, i);
+}
+
 static const struct dax_operations dcssblk_dax_ops = {
 	.direct_access = dcssblk_dax_direct_access,
 	.copy_from_iter = dcssblk_dax_copy_from_iter,
+	.copy_to_iter = dcssblk_dax_copy_to_iter,
 };
 
 struct dcssblk_dev_info {
diff --git a/drivers/s390/char/Makefile b/drivers/s390/char/Makefile
index d049e2d74484..0a4c13e1e76e 100644
--- a/drivers/s390/char/Makefile
+++ b/drivers/s390/char/Makefile
@@ -54,3 +54,6 @@ obj-$(CONFIG_CRASH_DUMP) += sclp_sdias.o zcore.o
 
 hmcdrv-objs := hmcdrv_mod.o hmcdrv_dev.o hmcdrv_ftp.o hmcdrv_cache.o diag_ftp.o sclp_ftp.o
 obj-$(CONFIG_HMC_DRV) += hmcdrv.o
+
+chkbss := sclp_early_core.o
+include $(srctree)/arch/s390/scripts/Makefile.chkbss
diff --git a/drivers/s390/char/fs3270.c b/drivers/s390/char/fs3270.c
index 61822480a2a0..16a4e8528bbc 100644
--- a/drivers/s390/char/fs3270.c
+++ b/drivers/s390/char/fs3270.c
@@ -19,7 +19,6 @@
 #include <linux/slab.h>
 #include <linux/types.h>
 
-#include <asm/compat.h>
 #include <asm/ccwdev.h>
 #include <asm/cio.h>
 #include <asm/ebcdic.h>
diff --git a/drivers/s390/char/sclp_ctl.c b/drivers/s390/char/sclp_ctl.c
index a78cea0c3a09..248b5db3eaa8 100644
--- a/drivers/s390/char/sclp_ctl.c
+++ b/drivers/s390/char/sclp_ctl.c
@@ -14,7 +14,6 @@
 #include <linux/init.h>
 #include <linux/ioctl.h>
 #include <linux/fs.h>
-#include <asm/compat.h>
 #include <asm/sclp_ctl.h>
 #include <asm/sclp.h>
 
diff --git a/drivers/s390/char/tape_proc.c b/drivers/s390/char/tape_proc.c
index faae30476f4b..32a14ee31c6b 100644
--- a/drivers/s390/char/tape_proc.c
+++ b/drivers/s390/char/tape_proc.c
@@ -105,29 +105,14 @@ static const struct seq_operations tape_proc_seq = {
 	.show		= tape_proc_show,
 };
 
-static int tape_proc_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &tape_proc_seq);
-}
-
-static const struct file_operations tape_proc_ops =
-{
-	.owner		= THIS_MODULE,
-	.open		= tape_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= seq_release,
-};
-
 /*
  * Initialize procfs stuff on startup
  */
 void
 tape_proc_init(void)
 {
-	tape_proc_devices =
-		proc_create("tapedevices", S_IFREG | S_IRUGO | S_IWUSR, NULL,
-			    &tape_proc_ops);
+	tape_proc_devices = proc_create_seq("tapedevices",
+			S_IFREG | S_IRUGO | S_IWUSR, NULL,  &tape_proc_seq);
 	if (tape_proc_devices == NULL) {
 		return;
 	}
diff --git a/drivers/s390/char/vmcp.c b/drivers/s390/char/vmcp.c
index 17e411c57576..948ce82a7725 100644
--- a/drivers/s390/char/vmcp.c
+++ b/drivers/s390/char/vmcp.c
@@ -23,7 +23,6 @@
 #include <linux/mutex.h>
 #include <linux/cma.h>
 #include <linux/mm.h>
-#include <asm/compat.h>
 #include <asm/cpcmd.h>
 #include <asm/debug.h>
 #include <asm/vmcp.h>
diff --git a/drivers/s390/cio/ccwgroup.c b/drivers/s390/cio/ccwgroup.c
index 5535312602af..db9c854088bc 100644
--- a/drivers/s390/cio/ccwgroup.c
+++ b/drivers/s390/cio/ccwgroup.c
@@ -561,6 +561,12 @@ static struct bus_type ccwgroup_bus_type = {
 	.pm = &ccwgroup_pm_ops,
 };
 
+bool dev_is_ccwgroup(struct device *dev)
+{
+	return dev->bus == &ccwgroup_bus_type;
+}
+EXPORT_SYMBOL(dev_is_ccwgroup);
+
 /**
  * ccwgroup_driver_register() - register a ccw group driver
  * @cdriver: driver to be registered
diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c
index 0015729d917d..8d9f36625ba5 100644
--- a/drivers/s390/cio/chsc_sch.c
+++ b/drivers/s390/cio/chsc_sch.c
@@ -16,7 +16,6 @@
 #include <linux/miscdevice.h>
 #include <linux/kernel_stat.h>
 
-#include <asm/compat.h>
 #include <asm/cio.h>
 #include <asm/chsc.h>
 #include <asm/isc.h>
diff --git a/drivers/s390/cio/device_ops.c b/drivers/s390/cio/device_ops.c
index aecfebb74157..4435ae0b3027 100644
--- a/drivers/s390/cio/device_ops.c
+++ b/drivers/s390/cio/device_ops.c
@@ -472,6 +472,36 @@ struct channel_path_desc_fmt0 *ccw_device_get_chp_desc(struct ccw_device *cdev,
 	return chp_get_chp_desc(chpid);
 }
 
+/**
+ * ccw_device_get_util_str() - return newly allocated utility strings
+ * @cdev: device to obtain the utility strings for
+ * @chp_idx: index of the channel path
+ *
+ * On success return a newly allocated copy of the utility strings
+ * associated with the given channel path. Return %NULL on error.
+ */
+u8 *ccw_device_get_util_str(struct ccw_device *cdev, int chp_idx)
+{
+	struct subchannel *sch = to_subchannel(cdev->dev.parent);
+	struct channel_path *chp;
+	struct chp_id chpid;
+	u8 *util_str;
+
+	chp_id_init(&chpid);
+	chpid.id = sch->schib.pmcw.chpid[chp_idx];
+	chp = chpid_to_chp(chpid);
+
+	util_str = kmalloc(sizeof(chp->desc_fmt3.util_str), GFP_KERNEL);
+	if (!util_str)
+		return NULL;
+
+	mutex_lock(&chp->lock);
+	memcpy(util_str, chp->desc_fmt3.util_str, sizeof(chp->desc_fmt3.util_str));
+	mutex_unlock(&chp->lock);
+
+	return util_str;
+}
+
 /**
  * ccw_device_get_id() - obtain a ccw device id
  * @cdev: device to obtain the id for
@@ -682,3 +712,4 @@ EXPORT_SYMBOL(ccw_device_start_key);
 EXPORT_SYMBOL(ccw_device_get_ciw);
 EXPORT_SYMBOL(ccw_device_get_path_mask);
 EXPORT_SYMBOL_GPL(ccw_device_get_chp_desc);
+EXPORT_SYMBOL_GPL(ccw_device_get_util_str);
diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h
index 02184cf35834..6a273c5ebca5 100644
--- a/drivers/s390/crypto/ap_bus.h
+++ b/drivers/s390/crypto/ap_bus.h
@@ -198,11 +198,18 @@ struct ap_message {
  */
 static inline void ap_init_message(struct ap_message *ap_msg)
 {
-	ap_msg->psmid = 0;
-	ap_msg->length = 0;
-	ap_msg->rc = 0;
-	ap_msg->special = 0;
-	ap_msg->receive = NULL;
+	memset(ap_msg, 0, sizeof(*ap_msg));
+}
+
+/**
+ * ap_release_message() - Release ap_message.
+ * Releases all memory used internal within the ap_message struct
+ * Currently this is the message and private field.
+ */
+static inline void ap_release_message(struct ap_message *ap_msg)
+{
+	kzfree(ap_msg->message);
+	kzfree(ap_msg->private);
 }
 
 #define for_each_ap_card(_ac) \
diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c
index 5efd84862ccb..febcdb5135bf 100644
--- a/drivers/s390/crypto/zcrypt_api.c
+++ b/drivers/s390/crypto/zcrypt_api.c
@@ -371,6 +371,7 @@ long zcrypt_send_cprb(struct ica_xcRB *xcRB)
 
 	trace_s390_zcrypt_req(xcRB, TB_ZSECSENDCPRB);
 
+	ap_init_message(&ap_msg);
 	rc = get_cprb_fc(xcRB, &ap_msg, &func_code, &domain);
 	if (rc)
 		goto out;
@@ -425,6 +426,7 @@ long zcrypt_send_cprb(struct ica_xcRB *xcRB)
 	spin_unlock(&zcrypt_list_lock);
 
 out:
+	ap_release_message(&ap_msg);
 	trace_s390_zcrypt_rep(xcRB, func_code, rc,
 			      AP_QID_CARD(qid), AP_QID_QUEUE(qid));
 	return rc;
@@ -468,6 +470,8 @@ static long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb)
 
 	trace_s390_zcrypt_req(xcrb, TP_ZSENDEP11CPRB);
 
+	ap_init_message(&ap_msg);
+
 	target_num = (unsigned short) xcrb->targets_num;
 
 	/* empty list indicates autoselect (all available targets) */
@@ -485,7 +489,7 @@ static long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb)
 		if (copy_from_user(targets, uptr,
 				   target_num * sizeof(*targets))) {
 			rc = -EFAULT;
-			goto out;
+			goto out_free;
 		}
 	}
 
@@ -542,6 +546,7 @@ static long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb)
 out_free:
 	kfree(targets);
 out:
+	ap_release_message(&ap_msg);
 	trace_s390_zcrypt_rep(xcrb, func_code, rc,
 			      AP_QID_CARD(qid), AP_QID_QUEUE(qid));
 	return rc;
@@ -559,6 +564,7 @@ static long zcrypt_rng(char *buffer)
 
 	trace_s390_zcrypt_req(buffer, TP_HWRNGCPRB);
 
+	ap_init_message(&ap_msg);
 	rc = get_rng_fc(&ap_msg, &func_code, &domain);
 	if (rc)
 		goto out;
@@ -589,8 +595,10 @@ static long zcrypt_rng(char *buffer)
 	pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, weight);
 	spin_unlock(&zcrypt_list_lock);
 
-	if (!pref_zq)
-		return -ENODEV;
+	if (!pref_zq) {
+		rc = -ENODEV;
+		goto out;
+	}
 
 	qid = pref_zq->queue->qid;
 	rc = pref_zq->ops->rng(pref_zq, buffer, &ap_msg);
@@ -600,6 +608,7 @@ static long zcrypt_rng(char *buffer)
 	spin_unlock(&zcrypt_list_lock);
 
 out:
+	ap_release_message(&ap_msg);
 	trace_s390_zcrypt_rep(buffer, func_code, rc,
 			      AP_QID_CARD(qid), AP_QID_QUEUE(qid));
 	return rc;
diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c
index f54bef4a928e..97d4bacbc442 100644
--- a/drivers/s390/crypto/zcrypt_msgtype6.c
+++ b/drivers/s390/crypto/zcrypt_msgtype6.c
@@ -1084,6 +1084,13 @@ static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_queue *zq,
 	return rc;
 }
 
+/**
+ * Fetch function code from cprb.
+ * Extracting the fc requires to copy the cprb from userspace.
+ * So this function allocates memory and needs an ap_msg prepared
+ * by the caller with ap_init_message(). Also the caller has to
+ * make sure ap_release_message() is always called even on failure.
+ */
 unsigned int get_cprb_fc(struct ica_xcRB *xcRB,
 				struct ap_message *ap_msg,
 				unsigned int *func_code, unsigned short **dom)
@@ -1091,9 +1098,7 @@ unsigned int get_cprb_fc(struct ica_xcRB *xcRB,
 	struct response_type resp_type = {
 		.type = PCIXCC_RESPONSE_TYPE_XCRB,
 	};
-	int rc;
 
-	ap_init_message(ap_msg);
 	ap_msg->message = kmalloc(MSGTYPE06_MAX_MSG_SIZE, GFP_KERNEL);
 	if (!ap_msg->message)
 		return -ENOMEM;
@@ -1101,17 +1106,10 @@ unsigned int get_cprb_fc(struct ica_xcRB *xcRB,
 	ap_msg->psmid = (((unsigned long long) current->pid) << 32) +
 				atomic_inc_return(&zcrypt_step);
 	ap_msg->private = kmalloc(sizeof(resp_type), GFP_KERNEL);
-	if (!ap_msg->private) {
-		kzfree(ap_msg->message);
+	if (!ap_msg->private)
 		return -ENOMEM;
-	}
 	memcpy(ap_msg->private, &resp_type, sizeof(resp_type));
-	rc = XCRB_msg_to_type6CPRB_msgX(ap_msg, xcRB, func_code, dom);
-	if (rc) {
-		kzfree(ap_msg->message);
-		kzfree(ap_msg->private);
-	}
-	return rc;
+	return XCRB_msg_to_type6CPRB_msgX(ap_msg, xcRB, func_code, dom);
 }
 
 /**
@@ -1139,11 +1137,16 @@ static long zcrypt_msgtype6_send_cprb(struct zcrypt_queue *zq,
 		/* Signal pending. */
 		ap_cancel_message(zq->queue, ap_msg);
 
-	kzfree(ap_msg->message);
-	kzfree(ap_msg->private);
 	return rc;
 }
 
+/**
+ * Fetch function code from ep11 cprb.
+ * Extracting the fc requires to copy the ep11 cprb from userspace.
+ * So this function allocates memory and needs an ap_msg prepared
+ * by the caller with ap_init_message(). Also the caller has to
+ * make sure ap_release_message() is always called even on failure.
+ */
 unsigned int get_ep11cprb_fc(struct ep11_urb *xcrb,
 				    struct ap_message *ap_msg,
 				    unsigned int *func_code)
@@ -1151,9 +1154,7 @@ unsigned int get_ep11cprb_fc(struct ep11_urb *xcrb,
 	struct response_type resp_type = {
 		.type = PCIXCC_RESPONSE_TYPE_EP11,
 	};
-	int rc;
 
-	ap_init_message(ap_msg);
 	ap_msg->message = kmalloc(MSGTYPE06_MAX_MSG_SIZE, GFP_KERNEL);
 	if (!ap_msg->message)
 		return -ENOMEM;
@@ -1161,17 +1162,10 @@ unsigned int get_ep11cprb_fc(struct ep11_urb *xcrb,
 	ap_msg->psmid = (((unsigned long long) current->pid) << 32) +
 				atomic_inc_return(&zcrypt_step);
 	ap_msg->private = kmalloc(sizeof(resp_type), GFP_KERNEL);
-	if (!ap_msg->private) {
-		kzfree(ap_msg->message);
+	if (!ap_msg->private)
 		return -ENOMEM;
-	}
 	memcpy(ap_msg->private, &resp_type, sizeof(resp_type));
-	rc = xcrb_msg_to_type6_ep11cprb_msgx(ap_msg, xcrb, func_code);
-	if (rc) {
-		kzfree(ap_msg->message);
-		kzfree(ap_msg->private);
-	}
-	return rc;
+	return xcrb_msg_to_type6_ep11cprb_msgx(ap_msg, xcrb, func_code);
 }
 
 /**
@@ -1246,8 +1240,6 @@ static long zcrypt_msgtype6_send_ep11_cprb(struct zcrypt_queue *zq,
 		/* Signal pending. */
 		ap_cancel_message(zq->queue, ap_msg);
 
-	kzfree(ap_msg->message);
-	kzfree(ap_msg->private);
 	return rc;
 }
 
@@ -1258,7 +1250,6 @@ unsigned int get_rng_fc(struct ap_message *ap_msg, int *func_code,
 		.type = PCIXCC_RESPONSE_TYPE_XCRB,
 	};
 
-	ap_init_message(ap_msg);
 	ap_msg->message = kmalloc(MSGTYPE06_MAX_MSG_SIZE, GFP_KERNEL);
 	if (!ap_msg->message)
 		return -ENOMEM;
@@ -1266,10 +1257,8 @@ unsigned int get_rng_fc(struct ap_message *ap_msg, int *func_code,
 	ap_msg->psmid = (((unsigned long long) current->pid) << 32) +
 				atomic_inc_return(&zcrypt_step);
 	ap_msg->private = kmalloc(sizeof(resp_type), GFP_KERNEL);
-	if (!ap_msg->private) {
-		kzfree(ap_msg->message);
+	if (!ap_msg->private)
 		return -ENOMEM;
-	}
 	memcpy(ap_msg->private, &resp_type, sizeof(resp_type));
 
 	rng_type6CPRB_msgX(ap_msg, ZCRYPT_RNG_BUFFER_SIZE, domain);
@@ -1313,8 +1302,6 @@ static long zcrypt_msgtype6_rng(struct zcrypt_queue *zq,
 		/* Signal pending. */
 		ap_cancel_message(zq->queue, ap_msg);
 
-	kzfree(ap_msg->message);
-	kzfree(ap_msg->private);
 	return rc;
 }
 
diff --git a/drivers/s390/net/lcs.c b/drivers/s390/net/lcs.c
index 0ee8f33efb54..2d9fe7e4ee40 100644
--- a/drivers/s390/net/lcs.c
+++ b/drivers/s390/net/lcs.c
@@ -1928,6 +1928,8 @@ lcs_portno_store (struct device *dev, struct device_attribute *attr, const char
 		return -EINVAL;
         /* TODO: sanity checks */
         card->portno = value;
+	if (card->dev)
+		card->dev->dev_port = card->portno;
 
         return count;
 
@@ -2158,6 +2160,7 @@ lcs_new_device(struct ccwgroup_device *ccwgdev)
 	card->dev = dev;
 	card->dev->ml_priv = card;
 	card->dev->netdev_ops = &lcs_netdev_ops;
+	card->dev->dev_port = card->portno;
 	memcpy(card->dev->dev_addr, card->mac, LCS_MAC_LENGTH);
 #ifdef CONFIG_IP_MULTICAST
 	if (!lcs_check_multicast_support(card))
diff --git a/drivers/s390/net/qeth_core.h b/drivers/s390/net/qeth_core.h
index 78b98b3e7efa..2a5fec55bf60 100644
--- a/drivers/s390/net/qeth_core.h
+++ b/drivers/s390/net/qeth_core.h
@@ -148,6 +148,7 @@ struct qeth_perf_stats {
 	unsigned int tx_csum;
 	unsigned int tx_lin;
 	unsigned int tx_linfail;
+	unsigned int rx_csum;
 };
 
 /* Routing stuff */
@@ -712,9 +713,6 @@ enum qeth_discipline_id {
 
 struct qeth_discipline {
 	const struct device_type *devtype;
-	void (*start_poll)(struct ccw_device *, int, unsigned long);
-	qdio_handler_t *input_handler;
-	qdio_handler_t *output_handler;
 	int (*process_rx_buffer)(struct qeth_card *card, int budget, int *done);
 	int (*recover)(void *ptr);
 	int (*setup) (struct ccwgroup_device *);
@@ -780,9 +778,9 @@ struct qeth_card {
 	struct qeth_card_options options;
 
 	wait_queue_head_t wait_q;
-	spinlock_t vlanlock;
 	spinlock_t mclock;
 	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
+	struct mutex vid_list_mutex;		/* vid_list */
 	struct list_head vid_list;
 	DECLARE_HASHTABLE(mac_htable, 4);
 	DECLARE_HASHTABLE(ip_htable, 4);
@@ -867,6 +865,32 @@ static inline int qeth_get_ip_version(struct sk_buff *skb)
 	}
 }
 
+static inline void qeth_rx_csum(struct qeth_card *card, struct sk_buff *skb,
+				u8 flags)
+{
+	if ((card->dev->features & NETIF_F_RXCSUM) &&
+	    (flags & QETH_HDR_EXT_CSUM_TRANSP_REQ)) {
+		skb->ip_summed = CHECKSUM_UNNECESSARY;
+		if (card->options.performance_stats)
+			card->perf_stats.rx_csum++;
+	} else {
+		skb->ip_summed = CHECKSUM_NONE;
+	}
+}
+
+static inline void qeth_tx_csum(struct sk_buff *skb, u8 *flags, int ipv)
+{
+	*flags |= QETH_HDR_EXT_CSUM_TRANSP_REQ;
+	if ((ipv == 4 && ip_hdr(skb)->protocol == IPPROTO_UDP) ||
+	    (ipv == 6 && ipv6_hdr(skb)->nexthdr == IPPROTO_UDP))
+		*flags |= QETH_HDR_EXT_UDP;
+	if (ipv == 4) {
+		/* some HW requires combined L3+L4 csum offload: */
+		*flags |= QETH_HDR_EXT_CSUM_HDR_REQ;
+		ip_hdr(skb)->check = 0;
+	}
+}
+
 static inline void qeth_put_buffer_pool_entry(struct qeth_card *card,
 		struct qeth_buffer_pool_entry *entry)
 {
@@ -879,6 +903,27 @@ static inline int qeth_is_diagass_supported(struct qeth_card *card,
 	return card->info.diagass_support & (__u32)cmd;
 }
 
+int qeth_send_simple_setassparms_prot(struct qeth_card *card,
+				      enum qeth_ipa_funcs ipa_func,
+				      u16 cmd_code, long data,
+				      enum qeth_prot_versions prot);
+/* IPv4 variant */
+static inline int qeth_send_simple_setassparms(struct qeth_card *card,
+					       enum qeth_ipa_funcs ipa_func,
+					       u16 cmd_code, long data)
+{
+	return qeth_send_simple_setassparms_prot(card, ipa_func, cmd_code,
+						 data, QETH_PROT_IPV4);
+}
+
+static inline int qeth_send_simple_setassparms_v6(struct qeth_card *card,
+						  enum qeth_ipa_funcs ipa_func,
+						  u16 cmd_code, long data)
+{
+	return qeth_send_simple_setassparms_prot(card, ipa_func, cmd_code,
+						 data, QETH_PROT_IPV6);
+}
+
 extern struct qeth_discipline qeth_l2_discipline;
 extern struct qeth_discipline qeth_l3_discipline;
 extern const struct attribute_group *qeth_generic_attr_groups[];
@@ -921,13 +966,7 @@ struct sk_buff *qeth_core_get_next_skb(struct qeth_card *,
 		struct qeth_qdio_buffer *, struct qdio_buffer_element **, int *,
 		struct qeth_hdr **);
 void qeth_schedule_recovery(struct qeth_card *);
-void qeth_qdio_start_poll(struct ccw_device *, int, unsigned long);
 int qeth_poll(struct napi_struct *napi, int budget);
-void qeth_qdio_input_handler(struct ccw_device *,
-		unsigned int, unsigned int, int,
-		int, unsigned long);
-void qeth_qdio_output_handler(struct ccw_device *, unsigned int,
-			int, int, int, unsigned long);
 void qeth_clear_ipacmd_list(struct qeth_card *);
 int qeth_qdio_clear_card(struct qeth_card *, int);
 void qeth_clear_working_pool_list(struct qeth_card *);
@@ -979,8 +1018,6 @@ int qeth_hw_trap(struct qeth_card *, enum qeth_diags_trap_action);
 int qeth_query_ipassists(struct qeth_card *, enum qeth_prot_versions prot);
 void qeth_trace_features(struct qeth_card *);
 void qeth_close_dev(struct qeth_card *);
-int qeth_send_simple_setassparms(struct qeth_card *, enum qeth_ipa_funcs,
-				 __u16, long);
 int qeth_send_setassparms(struct qeth_card *, struct qeth_cmd_buffer *, __u16,
 			  long,
 			  int (*reply_cb)(struct qeth_card *,
diff --git a/drivers/s390/net/qeth_core_main.c b/drivers/s390/net/qeth_core_main.c
index dffd820731f2..9f28b6f2efc4 100644
--- a/drivers/s390/net/qeth_core_main.c
+++ b/drivers/s390/net/qeth_core_main.c
@@ -10,6 +10,7 @@
 #define KMSG_COMPONENT "qeth"
 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 
+#include <linux/compat.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/string.h>
@@ -32,7 +33,6 @@
 #include <asm/chpid.h>
 #include <asm/io.h>
 #include <asm/sysinfo.h>
-#include <asm/compat.h>
 #include <asm/diag.h>
 #include <asm/cio.h>
 #include <asm/ccwdev.h>
@@ -1467,13 +1467,13 @@ static int qeth_setup_card(struct qeth_card *card)
 	card->lan_online = 0;
 	card->read_or_write_problem = 0;
 	card->dev = NULL;
-	spin_lock_init(&card->vlanlock);
 	spin_lock_init(&card->mclock);
 	spin_lock_init(&card->lock);
 	spin_lock_init(&card->ip_lock);
 	spin_lock_init(&card->thread_mask_lock);
 	mutex_init(&card->conf_mutex);
 	mutex_init(&card->discipline_mutex);
+	mutex_init(&card->vid_list_mutex);
 	card->thread_start_mask = 0;
 	card->thread_allowed_mask = 0;
 	card->thread_running_mask = 0;
@@ -3588,15 +3588,14 @@ static void qeth_check_outbound_queue(struct qeth_qdio_out_q *queue)
 	}
 }
 
-void qeth_qdio_start_poll(struct ccw_device *ccwdev, int queue,
-		unsigned long card_ptr)
+static void qeth_qdio_start_poll(struct ccw_device *ccwdev, int queue,
+				 unsigned long card_ptr)
 {
 	struct qeth_card *card = (struct qeth_card *)card_ptr;
 
 	if (card->dev && (card->dev->flags & IFF_UP))
 		napi_schedule(&card->napi);
 }
-EXPORT_SYMBOL_GPL(qeth_qdio_start_poll);
 
 int qeth_configure_cq(struct qeth_card *card, enum qeth_cq cq)
 {
@@ -3698,9 +3697,10 @@ static void qeth_qdio_cq_handler(struct qeth_card *card,
 	return;
 }
 
-void qeth_qdio_input_handler(struct ccw_device *ccwdev, unsigned int qdio_err,
-		unsigned int queue, int first_elem, int count,
-		unsigned long card_ptr)
+static void qeth_qdio_input_handler(struct ccw_device *ccwdev,
+				    unsigned int qdio_err, int queue,
+				    int first_elem, int count,
+				    unsigned long card_ptr)
 {
 	struct qeth_card *card = (struct qeth_card *)card_ptr;
 
@@ -3711,14 +3711,12 @@ void qeth_qdio_input_handler(struct ccw_device *ccwdev, unsigned int qdio_err,
 		qeth_qdio_cq_handler(card, qdio_err, queue, first_elem, count);
 	else if (qdio_err)
 		qeth_schedule_recovery(card);
-
-
 }
-EXPORT_SYMBOL_GPL(qeth_qdio_input_handler);
 
-void qeth_qdio_output_handler(struct ccw_device *ccwdev,
-		unsigned int qdio_error, int __queue, int first_element,
-		int count, unsigned long card_ptr)
+static void qeth_qdio_output_handler(struct ccw_device *ccwdev,
+				     unsigned int qdio_error, int __queue,
+				     int first_element, int count,
+				     unsigned long card_ptr)
 {
 	struct qeth_card *card        = (struct qeth_card *) card_ptr;
 	struct qeth_qdio_out_q *queue = card->qdio.out_qs[__queue];
@@ -3787,7 +3785,6 @@ void qeth_qdio_output_handler(struct ccw_device *ccwdev,
 		card->perf_stats.outbound_handler_time += qeth_get_micros() -
 			card->perf_stats.outbound_handler_start_time;
 }
-EXPORT_SYMBOL_GPL(qeth_qdio_output_handler);
 
 /* We cannot use outbound queue 3 for unicast packets on HiperSockets */
 static inline int qeth_cut_iqd_prio(struct qeth_card *card, int queue_num)
@@ -4995,7 +4992,7 @@ static int qeth_qdio_establish(struct qeth_card *card)
 		goto out_free_in_sbals;
 	}
 	for (i = 0; i < card->qdio.no_in_queues; ++i)
-		queue_start_poll[i] = card->discipline->start_poll;
+		queue_start_poll[i] = qeth_qdio_start_poll;
 
 	qeth_qdio_establish_cq(card, in_sbal_ptrs, queue_start_poll);
 
@@ -5019,8 +5016,8 @@ static int qeth_qdio_establish(struct qeth_card *card)
 	init_data.qib_param_field        = qib_param_field;
 	init_data.no_input_qs            = card->qdio.no_in_queues;
 	init_data.no_output_qs           = card->qdio.no_out_queues;
-	init_data.input_handler 	 = card->discipline->input_handler;
-	init_data.output_handler	 = card->discipline->output_handler;
+	init_data.input_handler		 = qeth_qdio_input_handler;
+	init_data.output_handler	 = qeth_qdio_output_handler;
 	init_data.queue_start_poll_array = queue_start_poll;
 	init_data.int_parm               = (unsigned long) card;
 	init_data.input_sbal_addr_array  = (void **) in_sbal_ptrs;
@@ -5204,6 +5201,11 @@ int qeth_core_hardsetup_card(struct qeth_card *card)
 	rc = qeth_query_ipassists(card, QETH_PROT_IPV4);
 	if (rc == -ENOMEM)
 		goto out;
+	if (qeth_is_supported(card, IPA_IPV6)) {
+		rc = qeth_query_ipassists(card, QETH_PROT_IPV6);
+		if (rc == -ENOMEM)
+			goto out;
+	}
 	if (qeth_is_supported(card, IPA_SETADAPTERPARMS)) {
 		rc = qeth_query_setadapterparms(card);
 		if (rc < 0) {
@@ -5511,26 +5513,26 @@ int qeth_send_setassparms(struct qeth_card *card,
 }
 EXPORT_SYMBOL_GPL(qeth_send_setassparms);
 
-int qeth_send_simple_setassparms(struct qeth_card *card,
-				 enum qeth_ipa_funcs ipa_func,
-				 __u16 cmd_code, long data)
+int qeth_send_simple_setassparms_prot(struct qeth_card *card,
+				      enum qeth_ipa_funcs ipa_func,
+				      u16 cmd_code, long data,
+				      enum qeth_prot_versions prot)
 {
 	int rc;
 	int length = 0;
 	struct qeth_cmd_buffer *iob;
 
-	QETH_CARD_TEXT(card, 4, "simassp4");
+	QETH_CARD_TEXT_(card, 4, "simassp%i", prot);
 	if (data)
 		length = sizeof(__u32);
-	iob = qeth_get_setassparms_cmd(card, ipa_func, cmd_code,
-				       length, QETH_PROT_IPV4);
+	iob = qeth_get_setassparms_cmd(card, ipa_func, cmd_code, length, prot);
 	if (!iob)
 		return -ENOMEM;
 	rc = qeth_send_setassparms(card, iob, length, data,
 				   qeth_setassparms_cb, NULL);
 	return rc;
 }
-EXPORT_SYMBOL_GPL(qeth_send_simple_setassparms);
+EXPORT_SYMBOL_GPL(qeth_send_simple_setassparms_prot);
 
 static void qeth_unregister_dbf_views(void)
 {
@@ -6008,7 +6010,8 @@ static struct {
 	{"tx lin"},
 	{"tx linfail"},
 	{"cq handler count"},
-	{"cq handler time"}
+	{"cq handler time"},
+	{"rx csum"}
 };
 
 int qeth_core_get_sset_count(struct net_device *dev, int stringset)
@@ -6070,6 +6073,7 @@ void qeth_core_get_ethtool_stats(struct net_device *dev,
 	data[35] = card->perf_stats.tx_linfail;
 	data[36] = card->perf_stats.cq_cnt;
 	data[37] = card->perf_stats.cq_time;
+	data[38] = card->perf_stats.rx_csum;
 }
 EXPORT_SYMBOL_GPL(qeth_core_get_ethtool_stats);
 
@@ -6326,14 +6330,15 @@ static int qeth_ipa_checksum_run_cmd_cb(struct qeth_card *card,
 static int qeth_ipa_checksum_run_cmd(struct qeth_card *card,
 				     enum qeth_ipa_funcs ipa_func,
 				     __u16 cmd_code, long data,
-				     struct qeth_checksum_cmd *chksum_cb)
+				     struct qeth_checksum_cmd *chksum_cb,
+				     enum qeth_prot_versions prot)
 {
 	struct qeth_cmd_buffer *iob;
 	int rc = -ENOMEM;
 
 	QETH_CARD_TEXT(card, 4, "chkdocmd");
 	iob = qeth_get_setassparms_cmd(card, ipa_func, cmd_code,
-				       sizeof(__u32), QETH_PROT_IPV4);
+				       sizeof(__u32), prot);
 	if (iob)
 		rc = qeth_send_setassparms(card, iob, sizeof(__u32), data,
 					   qeth_ipa_checksum_run_cmd_cb,
@@ -6341,16 +6346,17 @@ static int qeth_ipa_checksum_run_cmd(struct qeth_card *card,
 	return rc;
 }
 
-static int qeth_send_checksum_on(struct qeth_card *card, int cstype)
+static int qeth_send_checksum_on(struct qeth_card *card, int cstype,
+				 enum qeth_prot_versions prot)
 {
-	const __u32 required_features = QETH_IPA_CHECKSUM_IP_HDR |
-					QETH_IPA_CHECKSUM_UDP |
-					QETH_IPA_CHECKSUM_TCP;
+	u32 required_features = QETH_IPA_CHECKSUM_UDP | QETH_IPA_CHECKSUM_TCP;
 	struct qeth_checksum_cmd chksum_cb;
 	int rc;
 
+	if (prot == QETH_PROT_IPV4)
+		required_features |= QETH_IPA_CHECKSUM_IP_HDR;
 	rc = qeth_ipa_checksum_run_cmd(card, cstype, IPA_CMD_ASS_START, 0,
-				       &chksum_cb);
+				       &chksum_cb, prot);
 	if (!rc) {
 		if ((required_features & chksum_cb.supported) !=
 		    required_features)
@@ -6362,37 +6368,42 @@ static int qeth_send_checksum_on(struct qeth_card *card, int cstype)
 				 QETH_CARD_IFNAME(card));
 	}
 	if (rc) {
-		qeth_send_simple_setassparms(card, cstype, IPA_CMD_ASS_STOP, 0);
+		qeth_send_simple_setassparms_prot(card, cstype,
+						  IPA_CMD_ASS_STOP, 0, prot);
 		dev_warn(&card->gdev->dev,
-			 "Starting HW checksumming for %s failed, using SW checksumming\n",
-			 QETH_CARD_IFNAME(card));
+			 "Starting HW IPv%d checksumming for %s failed, using SW checksumming\n",
+			 prot, QETH_CARD_IFNAME(card));
 		return rc;
 	}
 	rc = qeth_ipa_checksum_run_cmd(card, cstype, IPA_CMD_ASS_ENABLE,
-				       chksum_cb.supported, &chksum_cb);
+				       chksum_cb.supported, &chksum_cb,
+				       prot);
 	if (!rc) {
 		if ((required_features & chksum_cb.enabled) !=
 		    required_features)
 			rc = -EIO;
 	}
 	if (rc) {
-		qeth_send_simple_setassparms(card, cstype, IPA_CMD_ASS_STOP, 0);
+		qeth_send_simple_setassparms_prot(card, cstype,
+						  IPA_CMD_ASS_STOP, 0, prot);
 		dev_warn(&card->gdev->dev,
-			 "Enabling HW checksumming for %s failed, using SW checksumming\n",
-			 QETH_CARD_IFNAME(card));
+			 "Enabling HW IPv%d checksumming for %s failed, using SW checksumming\n",
+			 prot, QETH_CARD_IFNAME(card));
 		return rc;
 	}
 
-	dev_info(&card->gdev->dev, "HW Checksumming (%sbound) enabled\n",
-		 cstype == IPA_INBOUND_CHECKSUM ? "in" : "out");
+	dev_info(&card->gdev->dev, "HW Checksumming (%sbound IPv%d) enabled\n",
+		 cstype == IPA_INBOUND_CHECKSUM ? "in" : "out", prot);
 	return 0;
 }
 
-static int qeth_set_ipa_csum(struct qeth_card *card, int on, int cstype)
+static int qeth_set_ipa_csum(struct qeth_card *card, bool on, int cstype,
+			     enum qeth_prot_versions prot)
 {
-	int rc = (on) ? qeth_send_checksum_on(card, cstype)
-		      : qeth_send_simple_setassparms(card, cstype,
-						     IPA_CMD_ASS_STOP, 0);
+	int rc = (on) ? qeth_send_checksum_on(card, cstype, prot)
+		      : qeth_send_simple_setassparms_prot(card, cstype,
+							  IPA_CMD_ASS_STOP, 0,
+							  prot);
 	return rc ? -EIO : 0;
 }
 
@@ -6419,8 +6430,31 @@ static int qeth_set_ipa_tso(struct qeth_card *card, int on)
 	return rc;
 }
 
-#define QETH_HW_FEATURES (NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_TSO)
+static int qeth_set_ipa_rx_csum(struct qeth_card *card, bool on)
+{
+	int rc_ipv4 = (on) ? -EOPNOTSUPP : 0;
+	int rc_ipv6;
+
+	if (qeth_is_supported(card, IPA_INBOUND_CHECKSUM))
+		rc_ipv4 = qeth_set_ipa_csum(card, on, IPA_INBOUND_CHECKSUM,
+					    QETH_PROT_IPV4);
+	if (!qeth_is_supported6(card, IPA_INBOUND_CHECKSUM_V6))
+		/* no/one Offload Assist available, so the rc is trivial */
+		return rc_ipv4;
 
+	rc_ipv6 = qeth_set_ipa_csum(card, on, IPA_INBOUND_CHECKSUM,
+				    QETH_PROT_IPV6);
+
+	if (on)
+		/* enable: success if any Assist is active */
+		return (rc_ipv6) ? rc_ipv4 : 0;
+
+	/* disable: failure if any Assist is still active */
+	return (rc_ipv6) ? rc_ipv6 : rc_ipv4;
+}
+
+#define QETH_HW_FEATURES (NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_TSO | \
+			  NETIF_F_IPV6_CSUM)
 /**
  * qeth_recover_features() - Restore device features after recovery
  * @dev:	the recovering net_device
@@ -6455,16 +6489,19 @@ int qeth_set_features(struct net_device *dev, netdev_features_t features)
 	QETH_DBF_HEX(SETUP, 2, &features, sizeof(features));
 
 	if ((changed & NETIF_F_IP_CSUM)) {
-		rc = qeth_set_ipa_csum(card,
-				       features & NETIF_F_IP_CSUM ? 1 : 0,
-				       IPA_OUTBOUND_CHECKSUM);
+		rc = qeth_set_ipa_csum(card, features & NETIF_F_IP_CSUM,
+				       IPA_OUTBOUND_CHECKSUM, QETH_PROT_IPV4);
 		if (rc)
 			changed ^= NETIF_F_IP_CSUM;
 	}
-	if ((changed & NETIF_F_RXCSUM)) {
-		rc = qeth_set_ipa_csum(card,
-					features & NETIF_F_RXCSUM ? 1 : 0,
-					IPA_INBOUND_CHECKSUM);
+	if (changed & NETIF_F_IPV6_CSUM) {
+		rc = qeth_set_ipa_csum(card, features & NETIF_F_IPV6_CSUM,
+				       IPA_OUTBOUND_CHECKSUM, QETH_PROT_IPV6);
+		if (rc)
+			changed ^= NETIF_F_IPV6_CSUM;
+	}
+	if (changed & NETIF_F_RXCSUM) {
+		rc = qeth_set_ipa_rx_csum(card, features & NETIF_F_RXCSUM);
 		if (rc)
 			changed ^= NETIF_F_RXCSUM;
 	}
@@ -6491,7 +6528,10 @@ netdev_features_t qeth_fix_features(struct net_device *dev,
 	QETH_DBF_TEXT(SETUP, 2, "fixfeat");
 	if (!qeth_is_supported(card, IPA_OUTBOUND_CHECKSUM))
 		features &= ~NETIF_F_IP_CSUM;
-	if (!qeth_is_supported(card, IPA_INBOUND_CHECKSUM))
+	if (!qeth_is_supported6(card, IPA_OUTBOUND_CHECKSUM_V6))
+		features &= ~NETIF_F_IPV6_CSUM;
+	if (!qeth_is_supported(card, IPA_INBOUND_CHECKSUM) &&
+	    !qeth_is_supported6(card, IPA_INBOUND_CHECKSUM_V6))
 		features &= ~NETIF_F_RXCSUM;
 	if (!qeth_is_supported(card, IPA_OUTBOUND_TSO))
 		features &= ~NETIF_F_TSO;
diff --git a/drivers/s390/net/qeth_core_mpc.h b/drivers/s390/net/qeth_core_mpc.h
index f4d1ec0b8f5a..878e62f35169 100644
--- a/drivers/s390/net/qeth_core_mpc.h
+++ b/drivers/s390/net/qeth_core_mpc.h
@@ -246,6 +246,8 @@ enum qeth_ipa_funcs {
 	IPA_QUERY_ARP_ASSIST	= 0x00040000L,
 	IPA_INBOUND_TSO         = 0x00080000L,
 	IPA_OUTBOUND_TSO        = 0x00100000L,
+	IPA_INBOUND_CHECKSUM_V6 = 0x00400000L,
+	IPA_OUTBOUND_CHECKSUM_V6 = 0x00800000L,
 };
 
 /* SETIP/DELIP IPA Command: ***************************************************/
diff --git a/drivers/s390/net/qeth_core_sys.c b/drivers/s390/net/qeth_core_sys.c
index ae81534de912..c3f18afb368b 100644
--- a/drivers/s390/net/qeth_core_sys.c
+++ b/drivers/s390/net/qeth_core_sys.c
@@ -144,6 +144,8 @@ static ssize_t qeth_dev_portno_store(struct device *dev,
 		goto out;
 	}
 	card->info.portno = portno;
+	if (card->dev)
+		card->dev->dev_port = portno;
 out:
 	mutex_unlock(&card->conf_mutex);
 	return rc ? rc : count;
diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c
index b8079f2a65b3..a7cb37da6a21 100644
--- a/drivers/s390/net/qeth_l2_main.c
+++ b/drivers/s390/net/qeth_l2_main.c
@@ -17,7 +17,6 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/etherdevice.h>
-#include <linux/ip.h>
 #include <linux/list.h>
 #include <linux/hash.h>
 #include <linux/hashtable.h>
@@ -195,23 +194,6 @@ static int qeth_l2_get_cast_type(struct qeth_card *card, struct sk_buff *skb)
 	return RTN_UNSPEC;
 }
 
-static void qeth_l2_hdr_csum(struct qeth_card *card, struct qeth_hdr *hdr,
-			     struct sk_buff *skb)
-{
-	struct iphdr *iph = ip_hdr(skb);
-
-	/* tcph->check contains already the pseudo hdr checksum
-	 * so just set the header flags
-	 */
-	if (iph->protocol == IPPROTO_UDP)
-		hdr->hdr.l2.flags[1] |= QETH_HDR_EXT_UDP;
-	hdr->hdr.l2.flags[1] |= QETH_HDR_EXT_CSUM_TRANSP_REQ |
-		QETH_HDR_EXT_CSUM_HDR_REQ;
-	iph->check = 0;
-	if (card->options.performance_stats)
-		card->perf_stats.tx_csum++;
-}
-
 static void qeth_l2_fill_header(struct qeth_hdr *hdr, struct sk_buff *skb,
 				int cast_type, unsigned int data_len)
 {
@@ -297,12 +279,13 @@ static int qeth_l2_send_setdelvlan(struct qeth_card *card, __u16 i,
 static void qeth_l2_process_vlans(struct qeth_card *card)
 {
 	struct qeth_vlan_vid *id;
+
 	QETH_CARD_TEXT(card, 3, "L2prcvln");
-	spin_lock_bh(&card->vlanlock);
+	mutex_lock(&card->vid_list_mutex);
 	list_for_each_entry(id, &card->vid_list, list) {
 		qeth_l2_send_setdelvlan(card, id->vid, IPA_CMD_SETVLAN);
 	}
-	spin_unlock_bh(&card->vlanlock);
+	mutex_unlock(&card->vid_list_mutex);
 }
 
 static int qeth_l2_vlan_rx_add_vid(struct net_device *dev,
@@ -319,7 +302,7 @@ static int qeth_l2_vlan_rx_add_vid(struct net_device *dev,
 		QETH_CARD_TEXT(card, 3, "aidREC");
 		return 0;
 	}
-	id = kmalloc(sizeof(struct qeth_vlan_vid), GFP_ATOMIC);
+	id = kmalloc(sizeof(*id), GFP_KERNEL);
 	if (id) {
 		id->vid = vid;
 		rc = qeth_l2_send_setdelvlan(card, vid, IPA_CMD_SETVLAN);
@@ -327,9 +310,9 @@ static int qeth_l2_vlan_rx_add_vid(struct net_device *dev,
 			kfree(id);
 			return rc;
 		}
-		spin_lock_bh(&card->vlanlock);
+		mutex_lock(&card->vid_list_mutex);
 		list_add_tail(&id->list, &card->vid_list);
-		spin_unlock_bh(&card->vlanlock);
+		mutex_unlock(&card->vid_list_mutex);
 	} else {
 		return -ENOMEM;
 	}
@@ -348,7 +331,7 @@ static int qeth_l2_vlan_rx_kill_vid(struct net_device *dev,
 		QETH_CARD_TEXT(card, 3, "kidREC");
 		return 0;
 	}
-	spin_lock_bh(&card->vlanlock);
+	mutex_lock(&card->vid_list_mutex);
 	list_for_each_entry(id, &card->vid_list, list) {
 		if (id->vid == vid) {
 			list_del(&id->list);
@@ -356,7 +339,7 @@ static int qeth_l2_vlan_rx_kill_vid(struct net_device *dev,
 			break;
 		}
 	}
-	spin_unlock_bh(&card->vlanlock);
+	mutex_unlock(&card->vid_list_mutex);
 	if (tmpid) {
 		rc = qeth_l2_send_setdelvlan(card, vid, IPA_CMD_DELVLAN);
 		kfree(tmpid);
@@ -423,15 +406,7 @@ static int qeth_l2_process_inbound_buffer(struct qeth_card *card,
 		switch (hdr->hdr.l2.id) {
 		case QETH_HEADER_TYPE_LAYER2:
 			skb->protocol = eth_type_trans(skb, skb->dev);
-			if ((card->dev->features & NETIF_F_RXCSUM)
-			   && ((hdr->hdr.l2.flags[1] &
-				(QETH_HDR_EXT_CSUM_HDR_REQ |
-				   QETH_HDR_EXT_CSUM_TRANSP_REQ)) ==
-				(QETH_HDR_EXT_CSUM_HDR_REQ |
-				   QETH_HDR_EXT_CSUM_TRANSP_REQ)))
-				skb->ip_summed = CHECKSUM_UNNECESSARY;
-			else
-				skb->ip_summed = CHECKSUM_NONE;
+			qeth_rx_csum(card, skb, hdr->hdr.l2.flags[1]);
 			if (skb->protocol == htons(ETH_P_802_2))
 				*((__u32 *)skb->cb) = ++card->seqno.pkt_seqno;
 			len = skb->len;
@@ -464,7 +439,6 @@ static int qeth_l2_process_inbound_buffer(struct qeth_card *card,
 static int qeth_l2_request_initial_mac(struct qeth_card *card)
 {
 	int rc = 0;
-	char vendor_pre[] = {0x02, 0x00, 0x00};
 
 	QETH_DBF_TEXT(SETUP, 2, "l2reqmac");
 	QETH_DBF_TEXT_(SETUP, 2, "doL2%s", CARD_BUS_ID(card));
@@ -484,16 +458,20 @@ static int qeth_l2_request_initial_mac(struct qeth_card *card)
 	    card->info.type == QETH_CARD_TYPE_OSX ||
 	    card->info.guestlan) {
 		rc = qeth_setadpparms_change_macaddr(card);
-		if (rc) {
-			QETH_DBF_MESSAGE(2, "couldn't get MAC address on "
-				"device %s: x%x\n", CARD_BUS_ID(card), rc);
-			QETH_DBF_TEXT_(SETUP, 2, "1err%04x", rc);
-			return rc;
-		}
-	} else {
-		eth_random_addr(card->dev->dev_addr);
-		memcpy(card->dev->dev_addr, vendor_pre, 3);
+		if (!rc)
+			goto out;
+		QETH_DBF_MESSAGE(2, "READ_MAC Assist failed on device %s: x%x\n",
+				 CARD_BUS_ID(card), rc);
+		QETH_DBF_TEXT_(SETUP, 2, "1err%04x", rc);
+		/* fall back once more: */
 	}
+
+	/* some devices don't support a custom MAC address: */
+	if (card->info.type == QETH_CARD_TYPE_OSM ||
+	    card->info.type == QETH_CARD_TYPE_OSX)
+		return (rc) ? rc : -EADDRNOTAVAIL;
+	eth_hw_addr_random(card->dev);
+
 out:
 	QETH_DBF_HEX(SETUP, 2, card->dev->dev_addr, card->dev->addr_len);
 	return 0;
@@ -685,7 +663,8 @@ static int qeth_l2_xmit_iqd(struct qeth_card *card, struct sk_buff *skb,
 }
 
 static int qeth_l2_xmit_osa(struct qeth_card *card, struct sk_buff *skb,
-			    struct qeth_qdio_out_q *queue, int cast_type)
+			    struct qeth_qdio_out_q *queue, int cast_type,
+			    int ipv)
 {
 	int push_len = sizeof(struct qeth_hdr);
 	unsigned int elements, nr_frags;
@@ -723,8 +702,11 @@ static int qeth_l2_xmit_osa(struct qeth_card *card, struct sk_buff *skb,
 		hdr_elements = 1;
 	}
 	qeth_l2_fill_header(hdr, skb, cast_type, skb->len - push_len);
-	if (skb->ip_summed == CHECKSUM_PARTIAL)
-		qeth_l2_hdr_csum(card, hdr, skb);
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
+		qeth_tx_csum(skb, &hdr->hdr.l2.flags[1], ipv);
+		if (card->options.performance_stats)
+			card->perf_stats.tx_csum++;
+	}
 
 	elements = qeth_get_elements_no(card, skb, hdr_elements, 0);
 	if (!elements) {
@@ -776,6 +758,7 @@ static netdev_tx_t qeth_l2_hard_start_xmit(struct sk_buff *skb,
 {
 	struct qeth_card *card = dev->ml_priv;
 	int cast_type = qeth_l2_get_cast_type(card, skb);
+	int ipv = qeth_get_ip_version(skb);
 	struct qeth_qdio_out_q *queue;
 	int tx_bytes = skb->len;
 	int rc;
@@ -783,7 +766,7 @@ static netdev_tx_t qeth_l2_hard_start_xmit(struct sk_buff *skb,
 	if (card->qdio.do_prio_queueing || (cast_type &&
 					card->info.is_multicast_different))
 		queue = card->qdio.out_qs[qeth_get_priority_queue(card, skb,
-					qeth_get_ip_version(skb), cast_type)];
+					ipv, cast_type)];
 	else
 		queue = card->qdio.out_qs[card->qdio.default_out_queue];
 
@@ -806,7 +789,7 @@ static netdev_tx_t qeth_l2_hard_start_xmit(struct sk_buff *skb,
 		rc = qeth_l2_xmit_iqd(card, skb, queue, cast_type);
 		break;
 	default:
-		rc = qeth_l2_xmit_osa(card, skb, queue, cast_type);
+		rc = qeth_l2_xmit_osa(card, skb, queue, cast_type, ipv);
 	}
 
 	if (!rc) {
@@ -983,6 +966,7 @@ static int qeth_l2_setup_netdev(struct qeth_card *card)
 	card->dev->mtu = card->info.initial_mtu;
 	card->dev->min_mtu = 64;
 	card->dev->max_mtu = ETH_MAX_MTU;
+	card->dev->dev_port = card->info.portno;
 	card->dev->netdev_ops = &qeth_l2_netdev_ops;
 	if (card->info.type == QETH_CARD_TYPE_OSN) {
 		card->dev->ethtool_ops = &qeth_l2_osn_ops;
@@ -1011,10 +995,15 @@ static int qeth_l2_setup_netdev(struct qeth_card *card)
 			card->dev->hw_features |= NETIF_F_IP_CSUM;
 			card->dev->vlan_features |= NETIF_F_IP_CSUM;
 		}
-		if (qeth_is_supported(card, IPA_INBOUND_CHECKSUM)) {
-			card->dev->hw_features |= NETIF_F_RXCSUM;
-			card->dev->vlan_features |= NETIF_F_RXCSUM;
-		}
+	}
+	if (qeth_is_supported6(card, IPA_OUTBOUND_CHECKSUM_V6)) {
+		card->dev->hw_features |= NETIF_F_IPV6_CSUM;
+		card->dev->vlan_features |= NETIF_F_IPV6_CSUM;
+	}
+	if (qeth_is_supported(card, IPA_INBOUND_CHECKSUM) ||
+	    qeth_is_supported6(card, IPA_INBOUND_CHECKSUM_V6)) {
+		card->dev->hw_features |= NETIF_F_RXCSUM;
+		card->dev->vlan_features |= NETIF_F_RXCSUM;
 	}
 
 	card->info.broadcast_capable = 1;
@@ -1315,9 +1304,6 @@ static int qeth_l2_control_event(struct qeth_card *card,
 
 struct qeth_discipline qeth_l2_discipline = {
 	.devtype = &qeth_l2_devtype,
-	.start_poll = qeth_qdio_start_poll,
-	.input_handler = (qdio_handler_t *) qeth_qdio_input_handler,
-	.output_handler = (qdio_handler_t *) qeth_qdio_output_handler,
 	.process_rx_buffer = qeth_l2_process_inbound_buffer,
 	.recover = qeth_l2_recover,
 	.setup = qeth_l2_probe_device,
diff --git a/drivers/s390/net/qeth_l3_main.c b/drivers/s390/net/qeth_l3_main.c
index c1a16a74aa83..e7fa479adf47 100644
--- a/drivers/s390/net/qeth_l3_main.c
+++ b/drivers/s390/net/qeth_l3_main.c
@@ -735,22 +735,6 @@ static int qeth_l3_setadapter_parms(struct qeth_card *card)
 	return rc;
 }
 
-static int qeth_l3_send_simple_setassparms_ipv6(struct qeth_card *card,
-		enum qeth_ipa_funcs ipa_func, __u16 cmd_code)
-{
-	int rc;
-	struct qeth_cmd_buffer *iob;
-
-	QETH_CARD_TEXT(card, 4, "simassp6");
-	iob = qeth_get_setassparms_cmd(card, ipa_func, cmd_code,
-				       0, QETH_PROT_IPV6);
-	if (!iob)
-		return -ENOMEM;
-	rc = qeth_send_setassparms(card, iob, 0, 0,
-				   qeth_setassparms_cb, NULL);
-	return rc;
-}
-
 static int qeth_l3_start_ipa_arp_processing(struct qeth_card *card)
 {
 	int rc;
@@ -851,14 +835,6 @@ static int qeth_l3_softsetup_ipv6(struct qeth_card *card)
 
 	QETH_CARD_TEXT(card, 3, "softipv6");
 
-	rc = qeth_query_ipassists(card, QETH_PROT_IPV6);
-	if (rc) {
-		dev_err(&card->gdev->dev,
-			"Activating IPv6 support for %s failed\n",
-			QETH_CARD_IFNAME(card));
-		return rc;
-	}
-
 	if (card->info.type == QETH_CARD_TYPE_IQD)
 		goto out;
 
@@ -870,16 +846,16 @@ static int qeth_l3_softsetup_ipv6(struct qeth_card *card)
 			QETH_CARD_IFNAME(card));
 		return rc;
 	}
-	rc = qeth_l3_send_simple_setassparms_ipv6(card, IPA_IPV6,
-					       IPA_CMD_ASS_START);
+	rc = qeth_send_simple_setassparms_v6(card, IPA_IPV6,
+					     IPA_CMD_ASS_START, 0);
 	if (rc) {
 		dev_err(&card->gdev->dev,
 			"Activating IPv6 support for %s failed\n",
 			 QETH_CARD_IFNAME(card));
 		return rc;
 	}
-	rc = qeth_l3_send_simple_setassparms_ipv6(card, IPA_PASSTHRU,
-					       IPA_CMD_ASS_START);
+	rc = qeth_send_simple_setassparms_v6(card, IPA_PASSTHRU,
+					     IPA_CMD_ASS_START, 0);
 	if (rc) {
 		dev_warn(&card->gdev->dev,
 			"Enabling the passthrough mode for %s failed\n",
@@ -1293,91 +1269,6 @@ static void qeth_l3_add_multicast_ipv6(struct qeth_card *card)
 	in6_dev_put(in6_dev);
 }
 
-static void qeth_l3_free_vlan_addresses4(struct qeth_card *card,
-			unsigned short vid)
-{
-	struct in_device *in_dev;
-	struct in_ifaddr *ifa;
-	struct qeth_ipaddr *addr;
-	struct net_device *netdev;
-
-	QETH_CARD_TEXT(card, 4, "frvaddr4");
-
-	netdev = __vlan_find_dev_deep_rcu(card->dev, htons(ETH_P_8021Q), vid);
-	if (!netdev)
-		return;
-	in_dev = in_dev_get(netdev);
-	if (!in_dev)
-		return;
-
-	addr = qeth_l3_get_addr_buffer(QETH_PROT_IPV4);
-	if (!addr)
-		goto out;
-
-	spin_lock_bh(&card->ip_lock);
-
-	for (ifa = in_dev->ifa_list; ifa; ifa = ifa->ifa_next) {
-		addr->u.a4.addr = be32_to_cpu(ifa->ifa_address);
-		addr->u.a4.mask = be32_to_cpu(ifa->ifa_mask);
-		addr->type = QETH_IP_TYPE_NORMAL;
-		qeth_l3_delete_ip(card, addr);
-	}
-
-	spin_unlock_bh(&card->ip_lock);
-
-	kfree(addr);
-out:
-	in_dev_put(in_dev);
-}
-
-static void qeth_l3_free_vlan_addresses6(struct qeth_card *card,
-					 unsigned short vid)
-{
-	struct inet6_dev *in6_dev;
-	struct inet6_ifaddr *ifa;
-	struct qeth_ipaddr *addr;
-	struct net_device *netdev;
-
-	QETH_CARD_TEXT(card, 4, "frvaddr6");
-
-	netdev = __vlan_find_dev_deep_rcu(card->dev, htons(ETH_P_8021Q), vid);
-	if (!netdev)
-		return;
-
-	in6_dev = in6_dev_get(netdev);
-	if (!in6_dev)
-		return;
-
-	addr = qeth_l3_get_addr_buffer(QETH_PROT_IPV6);
-	if (!addr)
-		goto out;
-
-	spin_lock_bh(&card->ip_lock);
-
-	list_for_each_entry(ifa, &in6_dev->addr_list, if_list) {
-		memcpy(&addr->u.a6.addr, &ifa->addr,
-		       sizeof(struct in6_addr));
-		addr->u.a6.pfxlen = ifa->prefix_len;
-		addr->type = QETH_IP_TYPE_NORMAL;
-		qeth_l3_delete_ip(card, addr);
-	}
-
-	spin_unlock_bh(&card->ip_lock);
-
-	kfree(addr);
-out:
-	in6_dev_put(in6_dev);
-}
-
-static void qeth_l3_free_vlan_addresses(struct qeth_card *card,
-			unsigned short vid)
-{
-	rcu_read_lock();
-	qeth_l3_free_vlan_addresses4(card, vid);
-	qeth_l3_free_vlan_addresses6(card, vid);
-	rcu_read_unlock();
-}
-
 static int qeth_l3_vlan_rx_add_vid(struct net_device *dev,
 				   __be16 proto, u16 vid)
 {
@@ -1398,8 +1289,6 @@ static int qeth_l3_vlan_rx_kill_vid(struct net_device *dev,
 		QETH_CARD_TEXT(card, 3, "kidREC");
 		return 0;
 	}
-	/* unregister IP addresses of vlan device */
-	qeth_l3_free_vlan_addresses(card, vid);
 	clear_bit(vid, card->active_vlans);
 	qeth_l3_set_rx_mode(dev);
 	return 0;
@@ -1454,17 +1343,7 @@ static void qeth_l3_rebuild_skb(struct qeth_card *card, struct sk_buff *skb,
 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag);
 	}
 
-	if (card->dev->features & NETIF_F_RXCSUM) {
-		if ((hdr->hdr.l3.ext_flags &
-		    (QETH_HDR_EXT_CSUM_HDR_REQ |
-		     QETH_HDR_EXT_CSUM_TRANSP_REQ)) ==
-		    (QETH_HDR_EXT_CSUM_HDR_REQ |
-		     QETH_HDR_EXT_CSUM_TRANSP_REQ))
-			skb->ip_summed = CHECKSUM_UNNECESSARY;
-		else
-			skb->ip_summed = CHECKSUM_NONE;
-	} else
-		skb->ip_summed = CHECKSUM_NONE;
+	qeth_rx_csum(card, skb, hdr->hdr.l3.ext_flags);
 }
 
 static int qeth_l3_process_inbound_buffer(struct qeth_card *card,
@@ -2210,23 +2089,6 @@ static void qeth_l3_fill_header(struct qeth_card *card, struct qeth_hdr *hdr,
 	rcu_read_unlock();
 }
 
-static void qeth_l3_hdr_csum(struct qeth_card *card, struct qeth_hdr *hdr,
-			     struct sk_buff *skb)
-{
-	struct iphdr *iph = ip_hdr(skb);
-
-	/* tcph->check contains already the pseudo hdr checksum
-	 * so just set the header flags
-	 */
-	if (iph->protocol == IPPROTO_UDP)
-		hdr->hdr.l3.ext_flags |= QETH_HDR_EXT_UDP;
-	hdr->hdr.l3.ext_flags |= QETH_HDR_EXT_CSUM_TRANSP_REQ |
-		QETH_HDR_EXT_CSUM_HDR_REQ;
-	iph->check = 0;
-	if (card->options.performance_stats)
-		card->perf_stats.tx_csum++;
-}
-
 static void qeth_tso_fill_header(struct qeth_card *card,
 		struct qeth_hdr *qhdr, struct sk_buff *skb)
 {
@@ -2418,8 +2280,11 @@ static netdev_tx_t qeth_l3_hard_start_xmit(struct sk_buff *skb,
 			}
 		}
 
-		if (skb->ip_summed == CHECKSUM_PARTIAL)
-			qeth_l3_hdr_csum(card, hdr, new_skb);
+		if (new_skb->ip_summed == CHECKSUM_PARTIAL) {
+			qeth_tx_csum(new_skb, &hdr->hdr.l3.ext_flags, ipv);
+			if (card->options.performance_stats)
+				card->perf_stats.tx_csum++;
+		}
 	}
 
 	elements = use_tso ?
@@ -2620,28 +2485,32 @@ static int qeth_l3_setup_netdev(struct qeth_card *card)
 		    (card->info.link_type == QETH_LINK_TYPE_HSTR)) {
 			pr_info("qeth_l3: ignoring TR device\n");
 			return -ENODEV;
-		} else {
-			card->dev = alloc_etherdev(0);
-			if (!card->dev)
-				return -ENODEV;
-			card->dev->netdev_ops = &qeth_l3_osa_netdev_ops;
-
-			/*IPv6 address autoconfiguration stuff*/
-			qeth_l3_get_unique_id(card);
-			if (!(card->info.unique_id & UNIQUE_ID_NOT_BY_CARD))
-				card->dev->dev_id = card->info.unique_id &
-							 0xffff;
-
-			card->dev->hw_features |= NETIF_F_SG;
-			card->dev->vlan_features |= NETIF_F_SG;
-
-			if (!card->info.guestlan) {
-				card->dev->features |= NETIF_F_SG;
-				card->dev->hw_features |= NETIF_F_TSO |
-					NETIF_F_RXCSUM | NETIF_F_IP_CSUM;
-				card->dev->vlan_features |= NETIF_F_TSO |
-					NETIF_F_RXCSUM | NETIF_F_IP_CSUM;
-			}
+		}
+
+		card->dev = alloc_etherdev(0);
+		if (!card->dev)
+			return -ENODEV;
+		card->dev->netdev_ops = &qeth_l3_osa_netdev_ops;
+
+		/*IPv6 address autoconfiguration stuff*/
+		qeth_l3_get_unique_id(card);
+		if (!(card->info.unique_id & UNIQUE_ID_NOT_BY_CARD))
+			card->dev->dev_id = card->info.unique_id & 0xffff;
+
+		card->dev->hw_features |= NETIF_F_SG;
+		card->dev->vlan_features |= NETIF_F_SG;
+
+		if (!card->info.guestlan) {
+			card->dev->features |= NETIF_F_SG;
+			card->dev->hw_features |= NETIF_F_TSO |
+				NETIF_F_RXCSUM | NETIF_F_IP_CSUM;
+			card->dev->vlan_features |= NETIF_F_TSO |
+				NETIF_F_RXCSUM | NETIF_F_IP_CSUM;
+		}
+
+		if (qeth_is_supported6(card, IPA_OUTBOUND_CHECKSUM_V6)) {
+			card->dev->hw_features |= NETIF_F_IPV6_CSUM;
+			card->dev->vlan_features |= NETIF_F_IPV6_CSUM;
 		}
 	} else if (card->info.type == QETH_CARD_TYPE_IQD) {
 		card->dev = alloc_netdev(0, "hsi%d", NET_NAME_UNKNOWN,
@@ -2663,6 +2532,7 @@ static int qeth_l3_setup_netdev(struct qeth_card *card)
 	card->dev->mtu = card->info.initial_mtu;
 	card->dev->min_mtu = 64;
 	card->dev->max_mtu = ETH_MAX_MTU;
+	card->dev->dev_port = card->info.portno;
 	card->dev->ethtool_ops = &qeth_l3_ethtool_ops;
 	card->dev->features |=	NETIF_F_HW_VLAN_CTAG_TX |
 				NETIF_F_HW_VLAN_CTAG_RX |
@@ -2960,9 +2830,6 @@ static int qeth_l3_control_event(struct qeth_card *card,
 
 struct qeth_discipline qeth_l3_discipline = {
 	.devtype = &qeth_l3_devtype,
-	.start_poll = qeth_qdio_start_poll,
-	.input_handler = (qdio_handler_t *) qeth_qdio_input_handler,
-	.output_handler = (qdio_handler_t *) qeth_qdio_output_handler,
 	.process_rx_buffer = qeth_l3_process_inbound_buffer,
 	.recover = qeth_l3_recover,
 	.setup = qeth_l3_probe_device,
diff --git a/drivers/s390/scsi/zfcp_dbf.c b/drivers/s390/scsi/zfcp_dbf.c
index 18c4f933e8b9..3b368fcf13f4 100644
--- a/drivers/s390/scsi/zfcp_dbf.c
+++ b/drivers/s390/scsi/zfcp_dbf.c
@@ -285,6 +285,8 @@ void zfcp_dbf_rec_trig(char *tag, struct zfcp_adapter *adapter,
 	struct list_head *entry;
 	unsigned long flags;
 
+	lockdep_assert_held(&adapter->erp_lock);
+
 	if (unlikely(!debug_level_enabled(dbf->rec, level)))
 		return;
 
@@ -599,16 +601,18 @@ void zfcp_dbf_san_in_els(char *tag, struct zfcp_fsf_req *fsf)
 }
 
 /**
- * zfcp_dbf_scsi - trace event for scsi commands
- * @tag: identifier for event
- * @sc: pointer to struct scsi_cmnd
- * @fsf: pointer to struct zfcp_fsf_req
+ * zfcp_dbf_scsi_common() - Common trace event helper for scsi.
+ * @tag: Identifier for event.
+ * @level: trace level of event.
+ * @sdev: Pointer to SCSI device as context for this event.
+ * @sc: Pointer to SCSI command, or NULL with task management function (TMF).
+ * @fsf: Pointer to FSF request, or NULL.
  */
-void zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *sc,
-		   struct zfcp_fsf_req *fsf)
+void zfcp_dbf_scsi_common(char *tag, int level, struct scsi_device *sdev,
+			  struct scsi_cmnd *sc, struct zfcp_fsf_req *fsf)
 {
 	struct zfcp_adapter *adapter =
-		(struct zfcp_adapter *) sc->device->host->hostdata[0];
+		(struct zfcp_adapter *) sdev->host->hostdata[0];
 	struct zfcp_dbf *dbf = adapter->dbf;
 	struct zfcp_dbf_scsi *rec = &dbf->scsi_buf;
 	struct fcp_resp_with_ext *fcp_rsp;
@@ -620,16 +624,28 @@ void zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *sc,
 
 	memcpy(rec->tag, tag, ZFCP_DBF_TAG_LEN);
 	rec->id = ZFCP_DBF_SCSI_CMND;
-	rec->scsi_result = sc->result;
-	rec->scsi_retries = sc->retries;
-	rec->scsi_allowed = sc->allowed;
-	rec->scsi_id = sc->device->id;
-	rec->scsi_lun = (u32)sc->device->lun;
-	rec->scsi_lun_64_hi = (u32)(sc->device->lun >> 32);
-	rec->host_scribble = (unsigned long)sc->host_scribble;
-
-	memcpy(rec->scsi_opcode, sc->cmnd,
-	       min((int)sc->cmd_len, ZFCP_DBF_SCSI_OPCODE));
+	if (sc) {
+		rec->scsi_result = sc->result;
+		rec->scsi_retries = sc->retries;
+		rec->scsi_allowed = sc->allowed;
+		rec->scsi_id = sc->device->id;
+		rec->scsi_lun = (u32)sc->device->lun;
+		rec->scsi_lun_64_hi = (u32)(sc->device->lun >> 32);
+		rec->host_scribble = (unsigned long)sc->host_scribble;
+
+		memcpy(rec->scsi_opcode, sc->cmnd,
+		       min_t(int, sc->cmd_len, ZFCP_DBF_SCSI_OPCODE));
+	} else {
+		rec->scsi_result = ~0;
+		rec->scsi_retries = ~0;
+		rec->scsi_allowed = ~0;
+		rec->scsi_id = sdev->id;
+		rec->scsi_lun = (u32)sdev->lun;
+		rec->scsi_lun_64_hi = (u32)(sdev->lun >> 32);
+		rec->host_scribble = ~0;
+
+		memset(rec->scsi_opcode, 0xff, ZFCP_DBF_SCSI_OPCODE);
+	}
 
 	if (fsf) {
 		rec->fsf_req_id = fsf->req_id;
@@ -664,6 +680,46 @@ void zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *sc,
 	spin_unlock_irqrestore(&dbf->scsi_lock, flags);
 }
 
+/**
+ * zfcp_dbf_scsi_eh() - Trace event for special cases of scsi_eh callbacks.
+ * @tag: Identifier for event.
+ * @adapter: Pointer to zfcp adapter as context for this event.
+ * @scsi_id: SCSI ID/target to indicate scope of task management function (TMF).
+ * @ret: Return value of calling function.
+ *
+ * This SCSI trace variant does not depend on any of:
+ * scsi_cmnd, zfcp_fsf_req, scsi_device.
+ */
+void zfcp_dbf_scsi_eh(char *tag, struct zfcp_adapter *adapter,
+		      unsigned int scsi_id, int ret)
+{
+	struct zfcp_dbf *dbf = adapter->dbf;
+	struct zfcp_dbf_scsi *rec = &dbf->scsi_buf;
+	unsigned long flags;
+	static int const level = 1;
+
+	if (unlikely(!debug_level_enabled(adapter->dbf->scsi, level)))
+		return;
+
+	spin_lock_irqsave(&dbf->scsi_lock, flags);
+	memset(rec, 0, sizeof(*rec));
+
+	memcpy(rec->tag, tag, ZFCP_DBF_TAG_LEN);
+	rec->id = ZFCP_DBF_SCSI_CMND;
+	rec->scsi_result = ret; /* re-use field, int is 4 bytes and fits */
+	rec->scsi_retries = ~0;
+	rec->scsi_allowed = ~0;
+	rec->fcp_rsp_info = ~0;
+	rec->scsi_id = scsi_id;
+	rec->scsi_lun = (u32)ZFCP_DBF_INVALID_LUN;
+	rec->scsi_lun_64_hi = (u32)(ZFCP_DBF_INVALID_LUN >> 32);
+	rec->host_scribble = ~0;
+	memset(rec->scsi_opcode, 0xff, ZFCP_DBF_SCSI_OPCODE);
+
+	debug_event(dbf->scsi, level, rec, sizeof(*rec));
+	spin_unlock_irqrestore(&dbf->scsi_lock, flags);
+}
+
 static debug_info_t *zfcp_dbf_reg(const char *name, int size, int rec_size)
 {
 	struct debug_info *d;
diff --git a/drivers/s390/scsi/zfcp_dbf.h b/drivers/s390/scsi/zfcp_dbf.h
index e2a973cd2573..d116c07ed77a 100644
--- a/drivers/s390/scsi/zfcp_dbf.h
+++ b/drivers/s390/scsi/zfcp_dbf.h
@@ -359,7 +359,7 @@ void _zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *scmd,
 					scmd->device->host->hostdata[0];
 
 	if (debug_level_enabled(adapter->dbf->scsi, level))
-		zfcp_dbf_scsi(tag, level, scmd, req);
+		zfcp_dbf_scsi_common(tag, level, scmd->device, scmd, req);
 }
 
 /**
@@ -402,16 +402,23 @@ void zfcp_dbf_scsi_abort(char *tag, struct scsi_cmnd *scmd,
 }
 
 /**
- * zfcp_dbf_scsi_devreset - trace event for Logical Unit or Target Reset
- * @tag: tag indicating success or failure of reset operation
- * @scmnd: SCSI command which caused this error recovery
- * @flag: indicates type of reset (Target Reset, Logical Unit Reset)
+ * zfcp_dbf_scsi_devreset() - Trace event for Logical Unit or Target Reset.
+ * @tag: Tag indicating success or failure of reset operation.
+ * @sdev: Pointer to SCSI device as context for this event.
+ * @flag: Indicates type of reset (Target Reset, Logical Unit Reset).
+ * @fsf_req: Pointer to FSF request representing the TMF, or NULL.
  */
 static inline
-void zfcp_dbf_scsi_devreset(char *tag, struct scsi_cmnd *scmnd, u8 flag,
+void zfcp_dbf_scsi_devreset(char *tag, struct scsi_device *sdev, u8 flag,
 			    struct zfcp_fsf_req *fsf_req)
 {
+	struct zfcp_adapter *adapter = (struct zfcp_adapter *)
+					sdev->host->hostdata[0];
 	char tmp_tag[ZFCP_DBF_TAG_LEN];
+	static int const level = 1;
+
+	if (unlikely(!debug_level_enabled(adapter->dbf->scsi, level)))
+		return;
 
 	if (flag == FCP_TMF_TGT_RESET)
 		memcpy(tmp_tag, "tr_", 3);
@@ -419,7 +426,7 @@ void zfcp_dbf_scsi_devreset(char *tag, struct scsi_cmnd *scmnd, u8 flag,
 		memcpy(tmp_tag, "lr_", 3);
 
 	memcpy(&tmp_tag[3], tag, 4);
-	_zfcp_dbf_scsi(tmp_tag, 1, scmnd, fsf_req);
+	zfcp_dbf_scsi_common(tmp_tag, level, sdev, NULL, fsf_req);
 }
 
 /**
diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c
index 1d91a32db08e..e7e6b63905e2 100644
--- a/drivers/s390/scsi/zfcp_erp.c
+++ b/drivers/s390/scsi/zfcp_erp.c
@@ -19,7 +19,6 @@
 enum zfcp_erp_act_flags {
 	ZFCP_STATUS_ERP_TIMEDOUT	= 0x10000000,
 	ZFCP_STATUS_ERP_CLOSE_ONLY	= 0x01000000,
-	ZFCP_STATUS_ERP_DISMISSING	= 0x00100000,
 	ZFCP_STATUS_ERP_DISMISSED	= 0x00200000,
 	ZFCP_STATUS_ERP_LOWMEM		= 0x00400000,
 	ZFCP_STATUS_ERP_NO_REF		= 0x00800000,
@@ -27,7 +26,6 @@ enum zfcp_erp_act_flags {
 
 enum zfcp_erp_steps {
 	ZFCP_ERP_STEP_UNINITIALIZED	= 0x0000,
-	ZFCP_ERP_STEP_FSF_XCONFIG	= 0x0001,
 	ZFCP_ERP_STEP_PHYS_PORT_CLOSING	= 0x0010,
 	ZFCP_ERP_STEP_PORT_CLOSING	= 0x0100,
 	ZFCP_ERP_STEP_PORT_OPENING	= 0x0800,
@@ -35,16 +33,28 @@ enum zfcp_erp_steps {
 	ZFCP_ERP_STEP_LUN_OPENING	= 0x2000,
 };
 
+/**
+ * enum zfcp_erp_act_type - Type of ERP action object.
+ * @ZFCP_ERP_ACTION_REOPEN_LUN: LUN recovery.
+ * @ZFCP_ERP_ACTION_REOPEN_PORT: Port recovery.
+ * @ZFCP_ERP_ACTION_REOPEN_PORT_FORCED: Forced port recovery.
+ * @ZFCP_ERP_ACTION_REOPEN_ADAPTER: Adapter recovery.
+ * @ZFCP_ERP_ACTION_NONE: Eyecatcher pseudo flag to bitwise or-combine with
+ *			  either of the first four enum values.
+ *			  Used to indicate that an ERP action could not be
+ *			  set up despite a detected need for some recovery.
+ * @ZFCP_ERP_ACTION_FAILED: Eyecatcher pseudo flag to bitwise or-combine with
+ *			    either of the first four enum values.
+ *			    Used to indicate that ERP not needed because
+ *			    the object has ZFCP_STATUS_COMMON_ERP_FAILED.
+ */
 enum zfcp_erp_act_type {
 	ZFCP_ERP_ACTION_REOPEN_LUN         = 1,
 	ZFCP_ERP_ACTION_REOPEN_PORT	   = 2,
 	ZFCP_ERP_ACTION_REOPEN_PORT_FORCED = 3,
 	ZFCP_ERP_ACTION_REOPEN_ADAPTER     = 4,
-};
-
-enum zfcp_erp_act_state {
-	ZFCP_ERP_ACTION_RUNNING = 1,
-	ZFCP_ERP_ACTION_READY   = 2,
+	ZFCP_ERP_ACTION_NONE		   = 0xc0,
+	ZFCP_ERP_ACTION_FAILED		   = 0xe0,
 };
 
 enum zfcp_erp_act_result {
@@ -62,14 +72,14 @@ static void zfcp_erp_adapter_block(struct zfcp_adapter *adapter, int mask)
 				       ZFCP_STATUS_COMMON_UNBLOCKED | mask);
 }
 
-static int zfcp_erp_action_exists(struct zfcp_erp_action *act)
+static bool zfcp_erp_action_is_running(struct zfcp_erp_action *act)
 {
 	struct zfcp_erp_action *curr_act;
 
 	list_for_each_entry(curr_act, &act->adapter->erp_running_head, list)
 		if (act == curr_act)
-			return ZFCP_ERP_ACTION_RUNNING;
-	return 0;
+			return true;
+	return false;
 }
 
 static void zfcp_erp_action_ready(struct zfcp_erp_action *act)
@@ -85,7 +95,7 @@ static void zfcp_erp_action_ready(struct zfcp_erp_action *act)
 static void zfcp_erp_action_dismiss(struct zfcp_erp_action *act)
 {
 	act->status |= ZFCP_STATUS_ERP_DISMISSED;
-	if (zfcp_erp_action_exists(act) == ZFCP_ERP_ACTION_RUNNING)
+	if (zfcp_erp_action_is_running(act))
 		zfcp_erp_action_ready(act);
 }
 
@@ -126,6 +136,49 @@ static void zfcp_erp_action_dismiss_adapter(struct zfcp_adapter *adapter)
 	}
 }
 
+static int zfcp_erp_handle_failed(int want, struct zfcp_adapter *adapter,
+				  struct zfcp_port *port,
+				  struct scsi_device *sdev)
+{
+	int need = want;
+	struct zfcp_scsi_dev *zsdev;
+
+	switch (want) {
+	case ZFCP_ERP_ACTION_REOPEN_LUN:
+		zsdev = sdev_to_zfcp(sdev);
+		if (atomic_read(&zsdev->status) & ZFCP_STATUS_COMMON_ERP_FAILED)
+			need = 0;
+		break;
+	case ZFCP_ERP_ACTION_REOPEN_PORT_FORCED:
+		if (atomic_read(&port->status) & ZFCP_STATUS_COMMON_ERP_FAILED)
+			need = 0;
+		break;
+	case ZFCP_ERP_ACTION_REOPEN_PORT:
+		if (atomic_read(&port->status) &
+		    ZFCP_STATUS_COMMON_ERP_FAILED) {
+			need = 0;
+			/* ensure propagation of failed status to new devices */
+			zfcp_erp_set_port_status(
+				port, ZFCP_STATUS_COMMON_ERP_FAILED);
+		}
+		break;
+	case ZFCP_ERP_ACTION_REOPEN_ADAPTER:
+		if (atomic_read(&adapter->status) &
+		    ZFCP_STATUS_COMMON_ERP_FAILED) {
+			need = 0;
+			/* ensure propagation of failed status to new devices */
+			zfcp_erp_set_adapter_status(
+				adapter, ZFCP_STATUS_COMMON_ERP_FAILED);
+		}
+		break;
+	default:
+		need = 0;
+		break;
+	}
+
+	return need;
+}
+
 static int zfcp_erp_required_act(int want, struct zfcp_adapter *adapter,
 				 struct zfcp_port *port,
 				 struct scsi_device *sdev)
@@ -241,48 +294,70 @@ static struct zfcp_erp_action *zfcp_erp_setup_act(int need, u32 act_status,
 	return erp_action;
 }
 
-static int zfcp_erp_action_enqueue(int want, struct zfcp_adapter *adapter,
-				   struct zfcp_port *port,
-				   struct scsi_device *sdev,
-				   char *id, u32 act_status)
+static void zfcp_erp_action_enqueue(int want, struct zfcp_adapter *adapter,
+				    struct zfcp_port *port,
+				    struct scsi_device *sdev,
+				    char *id, u32 act_status)
 {
-	int retval = 1, need;
+	int need;
 	struct zfcp_erp_action *act;
 
-	if (!adapter->erp_thread)
-		return -EIO;
+	need = zfcp_erp_handle_failed(want, adapter, port, sdev);
+	if (!need) {
+		need = ZFCP_ERP_ACTION_FAILED; /* marker for trace */
+		goto out;
+	}
+
+	if (!adapter->erp_thread) {
+		need = ZFCP_ERP_ACTION_NONE; /* marker for trace */
+		goto out;
+	}
 
 	need = zfcp_erp_required_act(want, adapter, port, sdev);
 	if (!need)
 		goto out;
 
 	act = zfcp_erp_setup_act(need, act_status, adapter, port, sdev);
-	if (!act)
+	if (!act) {
+		need |= ZFCP_ERP_ACTION_NONE; /* marker for trace */
 		goto out;
+	}
 	atomic_or(ZFCP_STATUS_ADAPTER_ERP_PENDING, &adapter->status);
 	++adapter->erp_total_count;
 	list_add_tail(&act->list, &adapter->erp_ready_head);
 	wake_up(&adapter->erp_ready_wq);
-	retval = 0;
  out:
 	zfcp_dbf_rec_trig(id, adapter, port, sdev, want, need);
-	return retval;
 }
 
-static int _zfcp_erp_adapter_reopen(struct zfcp_adapter *adapter,
+void zfcp_erp_port_forced_no_port_dbf(char *id, struct zfcp_adapter *adapter,
+				      u64 port_name, u32 port_id)
+{
+	unsigned long flags;
+	static /* don't waste stack */ struct zfcp_port tmpport;
+
+	write_lock_irqsave(&adapter->erp_lock, flags);
+	/* Stand-in zfcp port with fields just good enough for
+	 * zfcp_dbf_rec_trig() and zfcp_dbf_set_common().
+	 * Under lock because tmpport is static.
+	 */
+	atomic_set(&tmpport.status, -1); /* unknown */
+	tmpport.wwpn = port_name;
+	tmpport.d_id = port_id;
+	zfcp_dbf_rec_trig(id, adapter, &tmpport, NULL,
+			  ZFCP_ERP_ACTION_REOPEN_PORT_FORCED,
+			  ZFCP_ERP_ACTION_NONE);
+	write_unlock_irqrestore(&adapter->erp_lock, flags);
+}
+
+static void _zfcp_erp_adapter_reopen(struct zfcp_adapter *adapter,
 				    int clear_mask, char *id)
 {
 	zfcp_erp_adapter_block(adapter, clear_mask);
 	zfcp_scsi_schedule_rports_block(adapter);
 
-	/* ensure propagation of failed status to new devices */
-	if (atomic_read(&adapter->status) & ZFCP_STATUS_COMMON_ERP_FAILED) {
-		zfcp_erp_set_adapter_status(adapter,
-					    ZFCP_STATUS_COMMON_ERP_FAILED);
-		return -EIO;
-	}
-	return zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_ADAPTER,
-				       adapter, NULL, NULL, id, 0);
+	zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_ADAPTER,
+				adapter, NULL, NULL, id, 0);
 }
 
 /**
@@ -299,12 +374,8 @@ void zfcp_erp_adapter_reopen(struct zfcp_adapter *adapter, int clear, char *id)
 	zfcp_scsi_schedule_rports_block(adapter);
 
 	write_lock_irqsave(&adapter->erp_lock, flags);
-	if (atomic_read(&adapter->status) & ZFCP_STATUS_COMMON_ERP_FAILED)
-		zfcp_erp_set_adapter_status(adapter,
-					    ZFCP_STATUS_COMMON_ERP_FAILED);
-	else
-		zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_ADAPTER, adapter,
-					NULL, NULL, id, 0);
+	zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_ADAPTER, adapter,
+				NULL, NULL, id, 0);
 	write_unlock_irqrestore(&adapter->erp_lock, flags);
 }
 
@@ -345,9 +416,6 @@ static void _zfcp_erp_port_forced_reopen(struct zfcp_port *port, int clear,
 	zfcp_erp_port_block(port, clear);
 	zfcp_scsi_schedule_rport_block(port);
 
-	if (atomic_read(&port->status) & ZFCP_STATUS_COMMON_ERP_FAILED)
-		return;
-
 	zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_PORT_FORCED,
 				port->adapter, port, NULL, id, 0);
 }
@@ -368,19 +436,13 @@ void zfcp_erp_port_forced_reopen(struct zfcp_port *port, int clear, char *id)
 	write_unlock_irqrestore(&adapter->erp_lock, flags);
 }
 
-static int _zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id)
+static void _zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id)
 {
 	zfcp_erp_port_block(port, clear);
 	zfcp_scsi_schedule_rport_block(port);
 
-	if (atomic_read(&port->status) & ZFCP_STATUS_COMMON_ERP_FAILED) {
-		/* ensure propagation of failed status to new devices */
-		zfcp_erp_set_port_status(port, ZFCP_STATUS_COMMON_ERP_FAILED);
-		return -EIO;
-	}
-
-	return zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_PORT,
-				       port->adapter, port, NULL, id, 0);
+	zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_PORT,
+				port->adapter, port, NULL, id, 0);
 }
 
 /**
@@ -388,20 +450,15 @@ static int _zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id)
  * @port: port to recover
  * @clear_mask: flags in port status to be cleared
  * @id: Id for debug trace event.
- *
- * Returns 0 if recovery has been triggered, < 0 if not.
  */
-int zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id)
+void zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id)
 {
-	int retval;
 	unsigned long flags;
 	struct zfcp_adapter *adapter = port->adapter;
 
 	write_lock_irqsave(&adapter->erp_lock, flags);
-	retval = _zfcp_erp_port_reopen(port, clear, id);
+	_zfcp_erp_port_reopen(port, clear, id);
 	write_unlock_irqrestore(&adapter->erp_lock, flags);
-
-	return retval;
 }
 
 static void zfcp_erp_lun_block(struct scsi_device *sdev, int clear_mask)
@@ -418,9 +475,6 @@ static void _zfcp_erp_lun_reopen(struct scsi_device *sdev, int clear, char *id,
 
 	zfcp_erp_lun_block(sdev, clear);
 
-	if (atomic_read(&zfcp_sdev->status) & ZFCP_STATUS_COMMON_ERP_FAILED)
-		return;
-
 	zfcp_erp_action_enqueue(ZFCP_ERP_ACTION_REOPEN_LUN, adapter,
 				zfcp_sdev->port, sdev, id, act_status);
 }
@@ -482,21 +536,23 @@ void zfcp_erp_lun_shutdown_wait(struct scsi_device *sdev, char *id)
 	zfcp_erp_wait(adapter);
 }
 
-static int status_change_set(unsigned long mask, atomic_t *status)
+static int zfcp_erp_status_change_set(unsigned long mask, atomic_t *status)
 {
 	return (atomic_read(status) ^ mask) & mask;
 }
 
 static void zfcp_erp_adapter_unblock(struct zfcp_adapter *adapter)
 {
-	if (status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED, &adapter->status))
+	if (zfcp_erp_status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED,
+				       &adapter->status))
 		zfcp_dbf_rec_run("eraubl1", &adapter->erp_action);
 	atomic_or(ZFCP_STATUS_COMMON_UNBLOCKED, &adapter->status);
 }
 
 static void zfcp_erp_port_unblock(struct zfcp_port *port)
 {
-	if (status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED, &port->status))
+	if (zfcp_erp_status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED,
+				       &port->status))
 		zfcp_dbf_rec_run("erpubl1", &port->erp_action);
 	atomic_or(ZFCP_STATUS_COMMON_UNBLOCKED, &port->status);
 }
@@ -505,7 +561,8 @@ static void zfcp_erp_lun_unblock(struct scsi_device *sdev)
 {
 	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev);
 
-	if (status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED, &zfcp_sdev->status))
+	if (zfcp_erp_status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED,
+				       &zfcp_sdev->status))
 		zfcp_dbf_rec_run("erlubl1", &sdev_to_zfcp(sdev)->erp_action);
 	atomic_or(ZFCP_STATUS_COMMON_UNBLOCKED, &zfcp_sdev->status);
 }
@@ -553,7 +610,7 @@ void zfcp_erp_notify(struct zfcp_erp_action *erp_action, unsigned long set_mask)
 	unsigned long flags;
 
 	write_lock_irqsave(&adapter->erp_lock, flags);
-	if (zfcp_erp_action_exists(erp_action) == ZFCP_ERP_ACTION_RUNNING) {
+	if (zfcp_erp_action_is_running(erp_action)) {
 		erp_action->status |= set_mask;
 		zfcp_erp_action_ready(erp_action);
 	}
@@ -1634,3 +1691,14 @@ void zfcp_erp_clear_lun_status(struct scsi_device *sdev, u32 mask)
 		atomic_set(&zfcp_sdev->erp_counter, 0);
 }
 
+/**
+ * zfcp_erp_adapter_reset_sync() - Really reopen adapter and wait.
+ * @adapter: Pointer to zfcp_adapter to reopen.
+ * @id: Trace tag string of length %ZFCP_DBF_TAG_LEN.
+ */
+void zfcp_erp_adapter_reset_sync(struct zfcp_adapter *adapter, char *id)
+{
+	zfcp_erp_set_adapter_status(adapter, ZFCP_STATUS_COMMON_RUNNING);
+	zfcp_erp_adapter_reopen(adapter, ZFCP_STATUS_COMMON_ERP_FAILED, id);
+	zfcp_erp_wait(adapter);
+}
diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h
index e5eed8aac0ce..bd0c5a9f04cb 100644
--- a/drivers/s390/scsi/zfcp_ext.h
+++ b/drivers/s390/scsi/zfcp_ext.h
@@ -50,17 +50,23 @@ extern void zfcp_dbf_hba_basic(char *, struct zfcp_adapter *);
 extern void zfcp_dbf_san_req(char *, struct zfcp_fsf_req *, u32);
 extern void zfcp_dbf_san_res(char *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_san_in_els(char *, struct zfcp_fsf_req *);
-extern void zfcp_dbf_scsi(char *, int, struct scsi_cmnd *,
-			  struct zfcp_fsf_req *);
+extern void zfcp_dbf_scsi_common(char *tag, int level, struct scsi_device *sdev,
+				 struct scsi_cmnd *sc,
+				 struct zfcp_fsf_req *fsf);
+extern void zfcp_dbf_scsi_eh(char *tag, struct zfcp_adapter *adapter,
+			     unsigned int scsi_id, int ret);
 
 /* zfcp_erp.c */
 extern void zfcp_erp_set_adapter_status(struct zfcp_adapter *, u32);
 extern void zfcp_erp_clear_adapter_status(struct zfcp_adapter *, u32);
+extern void zfcp_erp_port_forced_no_port_dbf(char *id,
+					     struct zfcp_adapter *adapter,
+					     u64 port_name, u32 port_id);
 extern void zfcp_erp_adapter_reopen(struct zfcp_adapter *, int, char *);
 extern void zfcp_erp_adapter_shutdown(struct zfcp_adapter *, int, char *);
 extern void zfcp_erp_set_port_status(struct zfcp_port *, u32);
 extern void zfcp_erp_clear_port_status(struct zfcp_port *, u32);
-extern int  zfcp_erp_port_reopen(struct zfcp_port *, int, char *);
+extern void zfcp_erp_port_reopen(struct zfcp_port *port, int clear, char *id);
 extern void zfcp_erp_port_shutdown(struct zfcp_port *, int, char *);
 extern void zfcp_erp_port_forced_reopen(struct zfcp_port *, int, char *);
 extern void zfcp_erp_set_lun_status(struct scsi_device *, u32);
@@ -73,6 +79,7 @@ extern void zfcp_erp_thread_kill(struct zfcp_adapter *);
 extern void zfcp_erp_wait(struct zfcp_adapter *);
 extern void zfcp_erp_notify(struct zfcp_erp_action *, unsigned long);
 extern void zfcp_erp_timeout_handler(struct timer_list *t);
+extern void zfcp_erp_adapter_reset_sync(struct zfcp_adapter *adapter, char *id);
 
 /* zfcp_fc.c */
 extern struct kmem_cache *zfcp_fc_req_cache;
@@ -120,7 +127,8 @@ extern int zfcp_fsf_send_els(struct zfcp_adapter *, u32,
 			     struct zfcp_fsf_ct_els *, unsigned int);
 extern int zfcp_fsf_fcp_cmnd(struct scsi_cmnd *);
 extern void zfcp_fsf_req_free(struct zfcp_fsf_req *);
-extern struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_cmnd *, u8);
+extern struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_device *sdev,
+						   u8 tm_flags);
 extern struct zfcp_fsf_req *zfcp_fsf_abort_fcp_cmnd(struct scsi_cmnd *);
 extern void zfcp_fsf_reqid_check(struct zfcp_qdio *, int);
 
diff --git a/drivers/s390/scsi/zfcp_fc.c b/drivers/s390/scsi/zfcp_fc.c
index 6162cf57a20a..f6c415d6ef48 100644
--- a/drivers/s390/scsi/zfcp_fc.c
+++ b/drivers/s390/scsi/zfcp_fc.c
@@ -111,11 +111,10 @@ void zfcp_fc_post_event(struct work_struct *work)
 
 	list_for_each_entry_safe(event, tmp, &tmp_lh, list) {
 		fc_host_post_event(adapter->scsi_host, fc_get_event_number(),
-				event->code, event->data);
+				   event->code, event->data);
 		list_del(&event->list);
 		kfree(event);
 	}
-
 }
 
 /**
@@ -126,7 +125,7 @@ void zfcp_fc_post_event(struct work_struct *work)
  * @event_data: The event data (e.g. n_port page in case of els)
  */
 void zfcp_fc_enqueue_event(struct zfcp_adapter *adapter,
-			enum fc_host_event_code event_code, u32 event_data)
+			   enum fc_host_event_code event_code, u32 event_data)
 {
 	struct zfcp_fc_event *event;
 
@@ -425,6 +424,7 @@ void zfcp_fc_port_did_lookup(struct work_struct *work)
 	struct zfcp_port *port = container_of(work, struct zfcp_port,
 					      gid_pn_work);
 
+	set_worker_desc("zgidpn%16llx", port->wwpn); /* < WORKER_DESC_LEN=24 */
 	ret = zfcp_fc_ns_gid_pn(port);
 	if (ret) {
 		/* could not issue gid_pn for some reason */
@@ -559,6 +559,7 @@ void zfcp_fc_link_test_work(struct work_struct *work)
 		container_of(work, struct zfcp_port, test_link_work);
 	int retval;
 
+	set_worker_desc("zadisc%16llx", port->wwpn); /* < WORKER_DESC_LEN=24 */
 	get_device(&port->dev);
 	port->rport_task = RPORT_DEL;
 	zfcp_scsi_rport_work(&port->rport_work);
@@ -596,7 +597,7 @@ void zfcp_fc_test_link(struct zfcp_port *port)
 		put_device(&port->dev);
 }
 
-static struct zfcp_fc_req *zfcp_alloc_sg_env(int buf_num)
+static struct zfcp_fc_req *zfcp_fc_alloc_sg_env(int buf_num)
 {
 	struct zfcp_fc_req *fc_req;
 
@@ -748,7 +749,7 @@ void zfcp_fc_scan_ports(struct work_struct *work)
 	if (zfcp_fc_wka_port_get(&adapter->gs->ds))
 		return;
 
-	fc_req = zfcp_alloc_sg_env(buf_num);
+	fc_req = zfcp_fc_alloc_sg_env(buf_num);
 	if (!fc_req)
 		goto out;
 
diff --git a/drivers/s390/scsi/zfcp_fc.h b/drivers/s390/scsi/zfcp_fc.h
index 6a397ddaadf0..3cd74729cfb9 100644
--- a/drivers/s390/scsi/zfcp_fc.h
+++ b/drivers/s390/scsi/zfcp_fc.h
@@ -207,21 +207,14 @@ struct zfcp_fc_wka_ports {
  * zfcp_fc_scsi_to_fcp - setup FCP command with data from scsi_cmnd
  * @fcp: fcp_cmnd to setup
  * @scsi: scsi_cmnd where to get LUN, task attributes/flags and CDB
- * @tm: task management flags to setup task management command
  */
 static inline
-void zfcp_fc_scsi_to_fcp(struct fcp_cmnd *fcp, struct scsi_cmnd *scsi,
-			 u8 tm_flags)
+void zfcp_fc_scsi_to_fcp(struct fcp_cmnd *fcp, struct scsi_cmnd *scsi)
 {
 	u32 datalen;
 
 	int_to_scsilun(scsi->device->lun, (struct scsi_lun *) &fcp->fc_lun);
 
-	if (unlikely(tm_flags)) {
-		fcp->fc_tm_flags = tm_flags;
-		return;
-	}
-
 	fcp->fc_pri_ta = FCP_PTA_SIMPLE;
 
 	if (scsi->sc_data_direction == DMA_FROM_DEVICE)
@@ -240,6 +233,19 @@ void zfcp_fc_scsi_to_fcp(struct fcp_cmnd *fcp, struct scsi_cmnd *scsi,
 	}
 }
 
+/**
+ * zfcp_fc_fcp_tm() - Setup FCP command as task management command.
+ * @fcp: Pointer to FCP_CMND IU to set up.
+ * @dev: Pointer to SCSI_device where to send the task management command.
+ * @tm_flags: Task management flags to setup tm command.
+ */
+static inline
+void zfcp_fc_fcp_tm(struct fcp_cmnd *fcp, struct scsi_device *dev, u8 tm_flags)
+{
+	int_to_scsilun(dev->lun, (struct scsi_lun *) &fcp->fc_lun);
+	fcp->fc_tm_flags = tm_flags;
+}
+
 /**
  * zfcp_fc_evap_fcp_rsp - evaluate FCP RSP IU and update scsi_cmnd accordingly
  * @fcp_rsp: FCP RSP IU to evaluate
diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c
index b12cb81ad8a2..3c86e27f094d 100644
--- a/drivers/s390/scsi/zfcp_fsf.c
+++ b/drivers/s390/scsi/zfcp_fsf.c
@@ -4,7 +4,7 @@
  *
  * Implementation of FSF commands.
  *
- * Copyright IBM Corp. 2002, 2017
+ * Copyright IBM Corp. 2002, 2018
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -437,6 +437,9 @@ void zfcp_fsf_req_dismiss_all(struct zfcp_adapter *adapter)
 #define ZFCP_FSF_PORTSPEED_10GBIT	(1 <<  3)
 #define ZFCP_FSF_PORTSPEED_8GBIT	(1 <<  4)
 #define ZFCP_FSF_PORTSPEED_16GBIT	(1 <<  5)
+#define ZFCP_FSF_PORTSPEED_32GBIT	(1 <<  6)
+#define ZFCP_FSF_PORTSPEED_64GBIT	(1 <<  7)
+#define ZFCP_FSF_PORTSPEED_128GBIT	(1 <<  8)
 #define ZFCP_FSF_PORTSPEED_NOT_NEGOTIATED (1 << 15)
 
 static u32 zfcp_fsf_convert_portspeed(u32 fsf_speed)
@@ -454,6 +457,12 @@ static u32 zfcp_fsf_convert_portspeed(u32 fsf_speed)
 		fdmi_speed |= FC_PORTSPEED_8GBIT;
 	if (fsf_speed & ZFCP_FSF_PORTSPEED_16GBIT)
 		fdmi_speed |= FC_PORTSPEED_16GBIT;
+	if (fsf_speed & ZFCP_FSF_PORTSPEED_32GBIT)
+		fdmi_speed |= FC_PORTSPEED_32GBIT;
+	if (fsf_speed & ZFCP_FSF_PORTSPEED_64GBIT)
+		fdmi_speed |= FC_PORTSPEED_64GBIT;
+	if (fsf_speed & ZFCP_FSF_PORTSPEED_128GBIT)
+		fdmi_speed |= FC_PORTSPEED_128GBIT;
 	if (fsf_speed & ZFCP_FSF_PORTSPEED_NOT_NEGOTIATED)
 		fdmi_speed |= FC_PORTSPEED_NOT_NEGOTIATED;
 	return fdmi_speed;
@@ -662,7 +671,7 @@ static struct zfcp_fsf_req *zfcp_fsf_alloc(mempool_t *pool)
 	return req;
 }
 
-static struct fsf_qtcb *zfcp_qtcb_alloc(mempool_t *pool)
+static struct fsf_qtcb *zfcp_fsf_qtcb_alloc(mempool_t *pool)
 {
 	struct fsf_qtcb *qtcb;
 
@@ -701,9 +710,10 @@ static struct zfcp_fsf_req *zfcp_fsf_req_create(struct zfcp_qdio *qdio,
 
 	if (likely(fsf_cmd != FSF_QTCB_UNSOLICITED_STATUS)) {
 		if (likely(pool))
-			req->qtcb = zfcp_qtcb_alloc(adapter->pool.qtcb_pool);
+			req->qtcb = zfcp_fsf_qtcb_alloc(
+				adapter->pool.qtcb_pool);
 		else
-			req->qtcb = zfcp_qtcb_alloc(NULL);
+			req->qtcb = zfcp_fsf_qtcb_alloc(NULL);
 
 		if (unlikely(!req->qtcb)) {
 			zfcp_fsf_req_free(req);
@@ -2036,10 +2046,14 @@ static void zfcp_fsf_req_trace(struct zfcp_fsf_req *req, struct scsi_cmnd *scsi)
 			    sizeof(blktrc));
 }
 
-static void zfcp_fsf_fcp_handler_common(struct zfcp_fsf_req *req)
+/**
+ * zfcp_fsf_fcp_handler_common() - FCP response handler common to I/O and TMF.
+ * @req: Pointer to FSF request.
+ * @sdev: Pointer to SCSI device as request context.
+ */
+static void zfcp_fsf_fcp_handler_common(struct zfcp_fsf_req *req,
+					struct scsi_device *sdev)
 {
-	struct scsi_cmnd *scmnd = req->data;
-	struct scsi_device *sdev = scmnd->device;
 	struct zfcp_scsi_dev *zfcp_sdev;
 	struct fsf_qtcb_header *header = &req->qtcb->header;
 
@@ -2051,7 +2065,7 @@ static void zfcp_fsf_fcp_handler_common(struct zfcp_fsf_req *req)
 	switch (header->fsf_status) {
 	case FSF_HANDLE_MISMATCH:
 	case FSF_PORT_HANDLE_NOT_VALID:
-		zfcp_erp_adapter_reopen(zfcp_sdev->port->adapter, 0, "fssfch1");
+		zfcp_erp_adapter_reopen(req->adapter, 0, "fssfch1");
 		req->status |= ZFCP_STATUS_FSFREQ_ERROR;
 		break;
 	case FSF_FCPLUN_NOT_VALID:
@@ -2069,8 +2083,7 @@ static void zfcp_fsf_fcp_handler_common(struct zfcp_fsf_req *req)
 			req->qtcb->bottom.io.data_direction,
 			(unsigned long long)zfcp_scsi_dev_lun(sdev),
 			(unsigned long long)zfcp_sdev->port->wwpn);
-		zfcp_erp_adapter_shutdown(zfcp_sdev->port->adapter, 0,
-					  "fssfch3");
+		zfcp_erp_adapter_shutdown(req->adapter, 0, "fssfch3");
 		req->status |= ZFCP_STATUS_FSFREQ_ERROR;
 		break;
 	case FSF_CMND_LENGTH_NOT_VALID:
@@ -2080,8 +2093,7 @@ static void zfcp_fsf_fcp_handler_common(struct zfcp_fsf_req *req)
 			req->qtcb->bottom.io.fcp_cmnd_length,
 			(unsigned long long)zfcp_scsi_dev_lun(sdev),
 			(unsigned long long)zfcp_sdev->port->wwpn);
-		zfcp_erp_adapter_shutdown(zfcp_sdev->port->adapter, 0,
-					  "fssfch4");
+		zfcp_erp_adapter_shutdown(req->adapter, 0, "fssfch4");
 		req->status |= ZFCP_STATUS_FSFREQ_ERROR;
 		break;
 	case FSF_PORT_BOXED:
@@ -2120,7 +2132,7 @@ static void zfcp_fsf_fcp_cmnd_handler(struct zfcp_fsf_req *req)
 		return;
 	}
 
-	zfcp_fsf_fcp_handler_common(req);
+	zfcp_fsf_fcp_handler_common(req, scpnt->device);
 
 	if (unlikely(req->status & ZFCP_STATUS_FSFREQ_ERROR)) {
 		set_host_byte(scpnt, DID_TRANSPORT_DISRUPTED);
@@ -2258,7 +2270,7 @@ int zfcp_fsf_fcp_cmnd(struct scsi_cmnd *scsi_cmnd)
 
 	BUILD_BUG_ON(sizeof(struct fcp_cmnd) > FSF_FCP_CMND_SIZE);
 	fcp_cmnd = &req->qtcb->bottom.io.fcp_cmnd.iu;
-	zfcp_fc_scsi_to_fcp(fcp_cmnd, scsi_cmnd, 0);
+	zfcp_fc_scsi_to_fcp(fcp_cmnd, scsi_cmnd);
 
 	if ((scsi_get_prot_op(scsi_cmnd) != SCSI_PROT_NORMAL) &&
 	    scsi_prot_sg_count(scsi_cmnd)) {
@@ -2297,10 +2309,11 @@ int zfcp_fsf_fcp_cmnd(struct scsi_cmnd *scsi_cmnd)
 
 static void zfcp_fsf_fcp_task_mgmt_handler(struct zfcp_fsf_req *req)
 {
+	struct scsi_device *sdev = req->data;
 	struct fcp_resp_with_ext *fcp_rsp;
 	struct fcp_resp_rsp_info *rsp_info;
 
-	zfcp_fsf_fcp_handler_common(req);
+	zfcp_fsf_fcp_handler_common(req, sdev);
 
 	fcp_rsp = &req->qtcb->bottom.io.fcp_rsp.iu;
 	rsp_info = (struct fcp_resp_rsp_info *) &fcp_rsp[1];
@@ -2311,17 +2324,18 @@ static void zfcp_fsf_fcp_task_mgmt_handler(struct zfcp_fsf_req *req)
 }
 
 /**
- * zfcp_fsf_fcp_task_mgmt - send SCSI task management command
- * @scmnd: SCSI command to send the task management command for
- * @tm_flags: unsigned byte for task management flags
- * Returns: on success pointer to struct fsf_req, NULL otherwise
+ * zfcp_fsf_fcp_task_mgmt() - Send SCSI task management command (TMF).
+ * @sdev: Pointer to SCSI device to send the task management command to.
+ * @tm_flags: Unsigned byte for task management flags.
+ *
+ * Return: On success pointer to struct zfcp_fsf_req, %NULL otherwise.
  */
-struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_cmnd *scmnd,
+struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_device *sdev,
 					    u8 tm_flags)
 {
 	struct zfcp_fsf_req *req = NULL;
 	struct fcp_cmnd *fcp_cmnd;
-	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(scmnd->device);
+	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev);
 	struct zfcp_qdio *qdio = zfcp_sdev->port->adapter->qdio;
 
 	if (unlikely(!(atomic_read(&zfcp_sdev->status) &
@@ -2341,7 +2355,8 @@ struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_cmnd *scmnd,
 		goto out;
 	}
 
-	req->data = scmnd;
+	req->data = sdev;
+
 	req->handler = zfcp_fsf_fcp_task_mgmt_handler;
 	req->qtcb->header.lun_handle = zfcp_sdev->lun_handle;
 	req->qtcb->header.port_handle = zfcp_sdev->port->handle;
@@ -2352,7 +2367,7 @@ struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_cmnd *scmnd,
 	zfcp_qdio_set_sbale_last(qdio, &req->qdio_req);
 
 	fcp_cmnd = &req->qtcb->bottom.io.fcp_cmnd.iu;
-	zfcp_fc_scsi_to_fcp(fcp_cmnd, scmnd, tm_flags);
+	zfcp_fc_fcp_tm(fcp_cmnd, sdev, tm_flags);
 
 	zfcp_fsf_start_timer(req, ZFCP_SCSI_ER_TIMEOUT);
 	if (!zfcp_fsf_req_send(req))
diff --git a/drivers/s390/scsi/zfcp_fsf.h b/drivers/s390/scsi/zfcp_fsf.h
index 4baca67aba6d..535628b92f0a 100644
--- a/drivers/s390/scsi/zfcp_fsf.h
+++ b/drivers/s390/scsi/zfcp_fsf.h
@@ -4,7 +4,7 @@
  *
  * Interface to the FSF support functions.
  *
- * Copyright IBM Corp. 2002, 2017
+ * Copyright IBM Corp. 2002, 2018
  */
 
 #ifndef FSF_H
@@ -356,7 +356,7 @@ struct fsf_qtcb_bottom_config {
 	u32 adapter_features;
 	u32 connection_features;
 	u32 fc_topology;
-	u32 fc_link_speed;
+	u32 fc_link_speed;	/* one of ZFCP_FSF_PORTSPEED_* */
 	u32 adapter_type;
 	u8 res0;
 	u8 peer_d_id[3];
@@ -382,7 +382,7 @@ struct fsf_qtcb_bottom_port {
 	u32 class_of_service;	/* should be 0x00000006 for class 2 and 3 */
 	u8 supported_fc4_types[32]; /* should be 0x00000100 for scsi fcp */
 	u8 active_fc4_types[32];
-	u32 supported_speed;	/* 0x0001 for 1 GBit/s or 0x0002 for 2 GBit/s */
+	u32 supported_speed;	/* any combination of ZFCP_FSF_PORTSPEED_* */
 	u32 maximum_frame_size;	/* fixed value of 2112 */
 	u64 seconds_since_last_reset;
 	u64 tx_frames;
diff --git a/drivers/s390/scsi/zfcp_scsi.c b/drivers/s390/scsi/zfcp_scsi.c
index 22f9562f415c..a8efcb330bc1 100644
--- a/drivers/s390/scsi/zfcp_scsi.c
+++ b/drivers/s390/scsi/zfcp_scsi.c
@@ -181,6 +181,7 @@ static int zfcp_scsi_eh_abort_handler(struct scsi_cmnd *scpnt)
 		if (abrt_req)
 			break;
 
+		zfcp_dbf_scsi_abort("abrt_wt", scpnt, NULL);
 		zfcp_erp_wait(adapter);
 		ret = fc_block_scsi_eh(scpnt);
 		if (ret) {
@@ -264,44 +265,52 @@ static void zfcp_scsi_forget_cmnds(struct zfcp_scsi_dev *zsdev, u8 tm_flags)
 	write_unlock_irqrestore(&adapter->abort_lock, flags);
 }
 
-static int zfcp_task_mgmt_function(struct scsi_cmnd *scpnt, u8 tm_flags)
+/**
+ * zfcp_scsi_task_mgmt_function() - Send a task management function (sync).
+ * @sdev: Pointer to SCSI device to send the task management command to.
+ * @tm_flags: Task management flags,
+ *	      here we only handle %FCP_TMF_TGT_RESET or %FCP_TMF_LUN_RESET.
+ */
+static int zfcp_scsi_task_mgmt_function(struct scsi_device *sdev, u8 tm_flags)
 {
-	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(scpnt->device);
+	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev);
 	struct zfcp_adapter *adapter = zfcp_sdev->port->adapter;
+	struct fc_rport *rport = starget_to_rport(scsi_target(sdev));
 	struct zfcp_fsf_req *fsf_req = NULL;
 	int retval = SUCCESS, ret;
 	int retry = 3;
 
 	while (retry--) {
-		fsf_req = zfcp_fsf_fcp_task_mgmt(scpnt, tm_flags);
+		fsf_req = zfcp_fsf_fcp_task_mgmt(sdev, tm_flags);
 		if (fsf_req)
 			break;
 
+		zfcp_dbf_scsi_devreset("wait", sdev, tm_flags, NULL);
 		zfcp_erp_wait(adapter);
-		ret = fc_block_scsi_eh(scpnt);
+		ret = fc_block_rport(rport);
 		if (ret) {
-			zfcp_dbf_scsi_devreset("fiof", scpnt, tm_flags, NULL);
+			zfcp_dbf_scsi_devreset("fiof", sdev, tm_flags, NULL);
 			return ret;
 		}
 
 		if (!(atomic_read(&adapter->status) &
 		      ZFCP_STATUS_COMMON_RUNNING)) {
-			zfcp_dbf_scsi_devreset("nres", scpnt, tm_flags, NULL);
+			zfcp_dbf_scsi_devreset("nres", sdev, tm_flags, NULL);
 			return SUCCESS;
 		}
 	}
 	if (!fsf_req) {
-		zfcp_dbf_scsi_devreset("reqf", scpnt, tm_flags, NULL);
+		zfcp_dbf_scsi_devreset("reqf", sdev, tm_flags, NULL);
 		return FAILED;
 	}
 
 	wait_for_completion(&fsf_req->completion);
 
 	if (fsf_req->status & ZFCP_STATUS_FSFREQ_TMFUNCFAILED) {
-		zfcp_dbf_scsi_devreset("fail", scpnt, tm_flags, fsf_req);
+		zfcp_dbf_scsi_devreset("fail", sdev, tm_flags, fsf_req);
 		retval = FAILED;
 	} else {
-		zfcp_dbf_scsi_devreset("okay", scpnt, tm_flags, fsf_req);
+		zfcp_dbf_scsi_devreset("okay", sdev, tm_flags, fsf_req);
 		zfcp_scsi_forget_cmnds(zfcp_sdev, tm_flags);
 	}
 
@@ -311,27 +320,81 @@ static int zfcp_task_mgmt_function(struct scsi_cmnd *scpnt, u8 tm_flags)
 
 static int zfcp_scsi_eh_device_reset_handler(struct scsi_cmnd *scpnt)
 {
-	return zfcp_task_mgmt_function(scpnt, FCP_TMF_LUN_RESET);
+	struct scsi_device *sdev = scpnt->device;
+
+	return zfcp_scsi_task_mgmt_function(sdev, FCP_TMF_LUN_RESET);
 }
 
 static int zfcp_scsi_eh_target_reset_handler(struct scsi_cmnd *scpnt)
 {
-	return zfcp_task_mgmt_function(scpnt, FCP_TMF_TGT_RESET);
+	struct scsi_target *starget = scsi_target(scpnt->device);
+	struct fc_rport *rport = starget_to_rport(starget);
+	struct Scsi_Host *shost = rport_to_shost(rport);
+	struct scsi_device *sdev = NULL, *tmp_sdev;
+	struct zfcp_adapter *adapter =
+		(struct zfcp_adapter *)shost->hostdata[0];
+	int ret;
+
+	shost_for_each_device(tmp_sdev, shost) {
+		if (tmp_sdev->id == starget->id) {
+			sdev = tmp_sdev;
+			break;
+		}
+	}
+	if (!sdev) {
+		ret = FAILED;
+		zfcp_dbf_scsi_eh("tr_nosd", adapter, starget->id, ret);
+		return ret;
+	}
+
+	ret = zfcp_scsi_task_mgmt_function(sdev, FCP_TMF_TGT_RESET);
+
+	/* release reference from above shost_for_each_device */
+	if (sdev)
+		scsi_device_put(tmp_sdev);
+
+	return ret;
 }
 
 static int zfcp_scsi_eh_host_reset_handler(struct scsi_cmnd *scpnt)
 {
 	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(scpnt->device);
 	struct zfcp_adapter *adapter = zfcp_sdev->port->adapter;
-	int ret;
+	int ret = SUCCESS, fc_ret;
 
 	zfcp_erp_adapter_reopen(adapter, 0, "schrh_1");
 	zfcp_erp_wait(adapter);
-	ret = fc_block_scsi_eh(scpnt);
-	if (ret)
+	fc_ret = fc_block_scsi_eh(scpnt);
+	if (fc_ret)
+		ret = fc_ret;
+
+	zfcp_dbf_scsi_eh("schrh_r", adapter, ~0, ret);
+	return ret;
+}
+
+/**
+ * zfcp_scsi_sysfs_host_reset() - Support scsi_host sysfs attribute host_reset.
+ * @shost: Pointer to Scsi_Host to perform action on.
+ * @reset_type: We support %SCSI_ADAPTER_RESET but not %SCSI_FIRMWARE_RESET.
+ *
+ * Return: 0 on %SCSI_ADAPTER_RESET, -%EOPNOTSUPP otherwise.
+ *
+ * This is similar to zfcp_sysfs_adapter_failed_store().
+ */
+static int zfcp_scsi_sysfs_host_reset(struct Scsi_Host *shost, int reset_type)
+{
+	struct zfcp_adapter *adapter =
+		(struct zfcp_adapter *)shost->hostdata[0];
+	int ret = 0;
+
+	if (reset_type != SCSI_ADAPTER_RESET) {
+		ret = -EOPNOTSUPP;
+		zfcp_dbf_scsi_eh("scshr_n", adapter, ~0, ret);
 		return ret;
+	}
 
-	return SUCCESS;
+	zfcp_erp_adapter_reset_sync(adapter, "scshr_y");
+	return ret;
 }
 
 struct scsi_transport_template *zfcp_scsi_transport_template;
@@ -349,6 +412,7 @@ static struct scsi_host_template zfcp_scsi_host_template = {
 	.slave_configure	 = zfcp_scsi_slave_configure,
 	.slave_destroy		 = zfcp_scsi_slave_destroy,
 	.change_queue_depth	 = scsi_change_queue_depth,
+	.host_reset		 = zfcp_scsi_sysfs_host_reset,
 	.proc_name		 = "zfcp",
 	.can_queue		 = 4096,
 	.this_id		 = -1,
@@ -363,6 +427,7 @@ static struct scsi_host_template zfcp_scsi_host_template = {
 	.shost_attrs		 = zfcp_sysfs_shost_attrs,
 	.sdev_attrs		 = zfcp_sysfs_sdev_attrs,
 	.track_queue_depth	 = 1,
+	.supported_mode		 = MODE_INITIATOR,
 };
 
 /**
@@ -430,7 +495,7 @@ void zfcp_scsi_adapter_unregister(struct zfcp_adapter *adapter)
 }
 
 static struct fc_host_statistics*
-zfcp_init_fc_host_stats(struct zfcp_adapter *adapter)
+zfcp_scsi_init_fc_host_stats(struct zfcp_adapter *adapter)
 {
 	struct fc_host_statistics *fc_stats;
 
@@ -444,9 +509,9 @@ zfcp_init_fc_host_stats(struct zfcp_adapter *adapter)
 	return adapter->fc_stats;
 }
 
-static void zfcp_adjust_fc_host_stats(struct fc_host_statistics *fc_stats,
-				      struct fsf_qtcb_bottom_port *data,
-				      struct fsf_qtcb_bottom_port *old)
+static void zfcp_scsi_adjust_fc_host_stats(struct fc_host_statistics *fc_stats,
+					   struct fsf_qtcb_bottom_port *data,
+					   struct fsf_qtcb_bottom_port *old)
 {
 	fc_stats->seconds_since_last_reset =
 		data->seconds_since_last_reset - old->seconds_since_last_reset;
@@ -477,8 +542,8 @@ static void zfcp_adjust_fc_host_stats(struct fc_host_statistics *fc_stats,
 	fc_stats->fcp_output_megabytes = data->output_mb - old->output_mb;
 }
 
-static void zfcp_set_fc_host_stats(struct fc_host_statistics *fc_stats,
-				   struct fsf_qtcb_bottom_port *data)
+static void zfcp_scsi_set_fc_host_stats(struct fc_host_statistics *fc_stats,
+					struct fsf_qtcb_bottom_port *data)
 {
 	fc_stats->seconds_since_last_reset = data->seconds_since_last_reset;
 	fc_stats->tx_frames = data->tx_frames;
@@ -502,7 +567,8 @@ static void zfcp_set_fc_host_stats(struct fc_host_statistics *fc_stats,
 	fc_stats->fcp_output_megabytes = data->output_mb;
 }
 
-static struct fc_host_statistics *zfcp_get_fc_host_stats(struct Scsi_Host *host)
+static struct fc_host_statistics *
+zfcp_scsi_get_fc_host_stats(struct Scsi_Host *host)
 {
 	struct zfcp_adapter *adapter;
 	struct fc_host_statistics *fc_stats;
@@ -510,7 +576,7 @@ static struct fc_host_statistics *zfcp_get_fc_host_stats(struct Scsi_Host *host)
 	int ret;
 
 	adapter = (struct zfcp_adapter *)host->hostdata[0];
-	fc_stats = zfcp_init_fc_host_stats(adapter);
+	fc_stats = zfcp_scsi_init_fc_host_stats(adapter);
 	if (!fc_stats)
 		return NULL;
 
@@ -527,16 +593,16 @@ static struct fc_host_statistics *zfcp_get_fc_host_stats(struct Scsi_Host *host)
 	if (adapter->stats_reset &&
 	    ((jiffies/HZ - adapter->stats_reset) <
 	     data->seconds_since_last_reset))
-		zfcp_adjust_fc_host_stats(fc_stats, data,
-					  adapter->stats_reset_data);
+		zfcp_scsi_adjust_fc_host_stats(fc_stats, data,
+					       adapter->stats_reset_data);
 	else
-		zfcp_set_fc_host_stats(fc_stats, data);
+		zfcp_scsi_set_fc_host_stats(fc_stats, data);
 
 	kfree(data);
 	return fc_stats;
 }
 
-static void zfcp_reset_fc_host_stats(struct Scsi_Host *shost)
+static void zfcp_scsi_reset_fc_host_stats(struct Scsi_Host *shost)
 {
 	struct zfcp_adapter *adapter;
 	struct fsf_qtcb_bottom_port *data;
@@ -558,7 +624,7 @@ static void zfcp_reset_fc_host_stats(struct Scsi_Host *shost)
 	}
 }
 
-static void zfcp_get_host_port_state(struct Scsi_Host *shost)
+static void zfcp_scsi_get_host_port_state(struct Scsi_Host *shost)
 {
 	struct zfcp_adapter *adapter =
 		(struct zfcp_adapter *)shost->hostdata[0];
@@ -575,7 +641,8 @@ static void zfcp_get_host_port_state(struct Scsi_Host *shost)
 		fc_host_port_state(shost) = FC_PORTSTATE_UNKNOWN;
 }
 
-static void zfcp_set_rport_dev_loss_tmo(struct fc_rport *rport, u32 timeout)
+static void zfcp_scsi_set_rport_dev_loss_tmo(struct fc_rport *rport,
+					     u32 timeout)
 {
 	rport->dev_loss_tmo = timeout;
 }
@@ -602,6 +669,11 @@ static void zfcp_scsi_terminate_rport_io(struct fc_rport *rport)
 	if (port) {
 		zfcp_erp_port_forced_reopen(port, 0, "sctrpi1");
 		put_device(&port->dev);
+	} else {
+		zfcp_erp_port_forced_no_port_dbf(
+			"sctrpin", adapter,
+			rport->port_name /* zfcp_scsi_rport_register */,
+			rport->port_id /* zfcp_scsi_rport_register */);
 	}
 }
 
@@ -687,6 +759,9 @@ void zfcp_scsi_rport_work(struct work_struct *work)
 	struct zfcp_port *port = container_of(work, struct zfcp_port,
 					      rport_work);
 
+	set_worker_desc("zrp%c-%16llx",
+			(port->rport_task == RPORT_ADD) ? 'a' : 'd',
+			port->wwpn); /* < WORKER_DESC_LEN=24 */
 	while (port->rport_task) {
 		if (port->rport_task == RPORT_ADD) {
 			port->rport_task = RPORT_NONE;
@@ -761,10 +836,10 @@ struct fc_function_template zfcp_transport_functions = {
 	.show_host_supported_speeds = 1,
 	.show_host_maxframe_size = 1,
 	.show_host_serial_number = 1,
-	.get_fc_host_stats = zfcp_get_fc_host_stats,
-	.reset_fc_host_stats = zfcp_reset_fc_host_stats,
-	.set_rport_dev_loss_tmo = zfcp_set_rport_dev_loss_tmo,
-	.get_host_port_state = zfcp_get_host_port_state,
+	.get_fc_host_stats = zfcp_scsi_get_fc_host_stats,
+	.reset_fc_host_stats = zfcp_scsi_reset_fc_host_stats,
+	.set_rport_dev_loss_tmo = zfcp_scsi_set_rport_dev_loss_tmo,
+	.get_host_port_state = zfcp_scsi_get_host_port_state,
 	.terminate_rport_io = zfcp_scsi_terminate_rport_io,
 	.show_host_port_state = 1,
 	.show_host_active_fc4s = 1,
diff --git a/drivers/s390/scsi/zfcp_sysfs.c b/drivers/s390/scsi/zfcp_sysfs.c
index 3ac823f2540f..b277be6f7611 100644
--- a/drivers/s390/scsi/zfcp_sysfs.c
+++ b/drivers/s390/scsi/zfcp_sysfs.c
@@ -200,10 +200,7 @@ static ssize_t zfcp_sysfs_adapter_failed_store(struct device *dev,
 		goto out;
 	}
 
-	zfcp_erp_set_adapter_status(adapter, ZFCP_STATUS_COMMON_RUNNING);
-	zfcp_erp_adapter_reopen(adapter, ZFCP_STATUS_COMMON_ERP_FAILED,
-				"syafai2");
-	zfcp_erp_wait(adapter);
+	zfcp_erp_adapter_reset_sync(adapter, "syafai2");
 out:
 	zfcp_ccw_adapter_put(adapter);
 	return retval ? retval : (ssize_t) count;
diff --git a/drivers/sbus/char/Kconfig b/drivers/sbus/char/Kconfig
index bf3c5f735614..89edd13fd572 100644
--- a/drivers/sbus/char/Kconfig
+++ b/drivers/sbus/char/Kconfig
@@ -28,13 +28,6 @@ config TADPOLE_TS102_UCTRL
 	  events, and can also notice the attachment/detachment of external
 	  monitors and mice.
 
-config SUN_JSFLASH
-	tristate "JavaStation OS Flash SIMM"
-	depends on SPARC32
-	help
-	  If you say Y here, you will be able to boot from your JavaStation's
-	  Flash memory.
-
 config BBC_I2C
 	tristate "UltraSPARC-III bootbus i2c controller driver"
 	depends on PCI && SPARC64
diff --git a/drivers/sbus/char/Makefile b/drivers/sbus/char/Makefile
index 8c48ed96683f..44347c918f6b 100644
--- a/drivers/sbus/char/Makefile
+++ b/drivers/sbus/char/Makefile
@@ -15,6 +15,5 @@ obj-$(CONFIG_DISPLAY7SEG)		+= display7seg.o
 obj-$(CONFIG_OBP_FLASH)			+= flash.o
 obj-$(CONFIG_SUN_OPENPROMIO)		+= openprom.o
 obj-$(CONFIG_TADPOLE_TS102_UCTRL)	+= uctrl.o
-obj-$(CONFIG_SUN_JSFLASH)		+= jsflash.o
 obj-$(CONFIG_BBC_I2C)			+= bbc.o
 obj-$(CONFIG_ORACLE_DAX) 		+= oradax.o
diff --git a/drivers/sbus/char/jsflash.c b/drivers/sbus/char/jsflash.c
deleted file mode 100644
index 14f377ac1280..000000000000
--- a/drivers/sbus/char/jsflash.c
+++ /dev/null
@@ -1,658 +0,0 @@
-/*
- * drivers/sbus/char/jsflash.c
- *
- *  Copyright (C) 1991, 1992  Linus Torvalds	(drivers/char/mem.c)
- *  Copyright (C) 1997  Eddie C. Dost		(drivers/sbus/char/flash.c)
- *  Copyright (C) 1997-2000 Pavel Machek <pavel@ucw.cz>   (drivers/block/nbd.c)
- *  Copyright (C) 1999-2000 Pete Zaitcev
- *
- * This driver is used to program OS into a Flash SIMM on
- * Krups and Espresso platforms.
- *
- * TODO: do not allow erase/programming if file systems are mounted.
- * TODO: Erase/program both banks of a 8MB SIMM.
- *
- * It is anticipated that programming an OS Flash will be a routine
- * procedure. In the same time it is exceedingly dangerous because
- * a user can program its OBP flash with OS image and effectively
- * kill the machine.
- *
- * This driver uses an interface different from Eddie's flash.c
- * as a silly safeguard.
- *
- * XXX The flash.c manipulates page caching characteristics in a certain
- * dubious way; also it assumes that remap_pfn_range() can remap
- * PCI bus locations, which may be false. ioremap() must be used
- * instead. We should discuss this.
- */
-
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/types.h>
-#include <linux/errno.h>
-#include <linux/miscdevice.h>
-#include <linux/fcntl.h>
-#include <linux/poll.h>
-#include <linux/init.h>
-#include <linux/string.h>
-#include <linux/genhd.h>
-#include <linux/blkdev.h>
-#include <linux/uaccess.h>
-#include <asm/pgtable.h>
-#include <asm/io.h>
-#include <asm/pcic.h>
-#include <asm/oplib.h>
-
-#include <asm/jsflash.h>		/* ioctl arguments. <linux/> ?? */
-#define JSFIDSZ		(sizeof(struct jsflash_ident_arg))
-#define JSFPRGSZ	(sizeof(struct jsflash_program_arg))
-
-/*
- * Our device numbers have no business in system headers.
- * The only thing a user knows is the device name /dev/jsflash.
- *
- * Block devices are laid out like this:
- *   minor+0	- Bootstrap, for 8MB SIMM 0x20400000[0x800000]
- *   minor+1	- Filesystem to mount, normally 0x20400400[0x7ffc00]
- *   minor+2	- Whole flash area for any case... 0x20000000[0x01000000]
- * Total 3 minors per flash device.
- *
- * It is easier to have static size vectors, so we define
- * a total minor range JSF_MAX, which must cover all minors.
- */
-/* character device */
-#define JSF_MINOR	178	/* 178 is registered with hpa */
-/* block device */
-#define JSF_MAX		 3	/* 3 minors wasted total so far. */
-#define JSF_NPART	 3	/* 3 minors per flash device */
-#define JSF_PART_BITS	 2	/* 2 bits of minors to cover JSF_NPART */
-#define JSF_PART_MASK	 0x3	/* 2 bits mask */
-
-static DEFINE_MUTEX(jsf_mutex);
-
-/*
- * Access functions.
- * We could ioremap(), but it's easier this way.
- */
-static unsigned int jsf_inl(unsigned long addr)
-{
-	unsigned long retval;
-
-	__asm__ __volatile__("lda [%1] %2, %0\n\t" :
-				"=r" (retval) :
-				"r" (addr), "i" (ASI_M_BYPASS));
-        return retval;
-}
-
-static void jsf_outl(unsigned long addr, __u32 data)
-{
-
-	__asm__ __volatile__("sta %0, [%1] %2\n\t" : :
-				"r" (data), "r" (addr), "i" (ASI_M_BYPASS) :
-				"memory");
-}
-
-/*
- * soft carrier
- */
-
-struct jsfd_part {
-	unsigned long dbase;
-	unsigned long dsize;
-};
-
-struct jsflash {
-	unsigned long base;
-	unsigned long size;
-	unsigned long busy;		/* In use? */
-	struct jsflash_ident_arg id;
-	/* int mbase; */		/* Minor base, typically zero */
-	struct jsfd_part dv[JSF_NPART];
-};
-
-/*
- * We do not map normal memory or obio as a safety precaution.
- * But offsets are real, for ease of userland programming.
- */
-#define JSF_BASE_TOP	0x30000000
-#define JSF_BASE_ALL	0x20000000
-
-#define JSF_BASE_JK	0x20400000
-
-/*
- */
-static struct gendisk *jsfd_disk[JSF_MAX];
-
-/*
- * Let's pretend we may have several of these...
- */
-static struct jsflash jsf0;
-
-/*
- * Wait for AMD to finish its embedded algorithm.
- * We use the Toggle bit DQ6 (0x40) because it does not
- * depend on the data value as /DATA bit DQ7 does.
- *
- * XXX Do we need any timeout here? So far it never hanged, beware broken hw.
- */
-static void jsf_wait(unsigned long p) {
-	unsigned int x1, x2;
-
-	for (;;) {
-		x1 = jsf_inl(p);
-		x2 = jsf_inl(p);
-		if ((x1 & 0x40404040) == (x2 & 0x40404040)) return;
-	}
-}
-
-/*
- * Programming will only work if Flash is clean,
- * we leave it to the programmer application.
- *
- * AMD must be programmed one byte at a time;
- * thus, Simple Tech SIMM must be written 4 bytes at a time.
- *
- * Write waits for the chip to become ready after the write
- * was finished. This is done so that application would read
- * consistent data after the write is done.
- */
-static void jsf_write4(unsigned long fa, u32 data) {
-
-	jsf_outl(fa, 0xAAAAAAAA);		/* Unlock 1 Write 1 */
-	jsf_outl(fa, 0x55555555);		/* Unlock 1 Write 2 */
-	jsf_outl(fa, 0xA0A0A0A0);		/* Byte Program */
-	jsf_outl(fa, data);
-
-	jsf_wait(fa);
-}
-
-/*
- */
-static void jsfd_read(char *buf, unsigned long p, size_t togo) {
-	union byte4 {
-		char s[4];
-		unsigned int n;
-	} b;
-
-	while (togo >= 4) {
-		togo -= 4;
-		b.n = jsf_inl(p);
-		memcpy(buf, b.s, 4);
-		p += 4;
-		buf += 4;
-	}
-}
-
-static int jsfd_queue;
-
-static struct request *jsfd_next_request(void)
-{
-	struct request_queue *q;
-	struct request *rq;
-	int old_pos = jsfd_queue;
-
-	do {
-		q = jsfd_disk[jsfd_queue]->queue;
-		if (++jsfd_queue == JSF_MAX)
-			jsfd_queue = 0;
-		if (q) {
-			rq = blk_fetch_request(q);
-			if (rq)
-				return rq;
-		}
-	} while (jsfd_queue != old_pos);
-
-	return NULL;
-}
-
-static void jsfd_request(void)
-{
-	struct request *req;
-
-	req = jsfd_next_request();
-	while (req) {
-		struct jsfd_part *jdp = req->rq_disk->private_data;
-		unsigned long offset = blk_rq_pos(req) << 9;
-		size_t len = blk_rq_cur_bytes(req);
-		blk_status_t err = BLK_STS_IOERR;
-
-		if ((offset + len) > jdp->dsize)
-			goto end;
-
-		if (rq_data_dir(req) != READ) {
-			printk(KERN_ERR "jsfd: write\n");
-			goto end;
-		}
-
-		if ((jdp->dbase & 0xff000000) != 0x20000000) {
-			printk(KERN_ERR "jsfd: bad base %x\n", (int)jdp->dbase);
-			goto end;
-		}
-
-		jsfd_read(bio_data(req->bio), jdp->dbase + offset, len);
-		err = BLK_STS_OK;
-	end:
-		if (!__blk_end_request_cur(req, err))
-			req = jsfd_next_request();
-	}
-}
-
-static void jsfd_do_request(struct request_queue *q)
-{
-	jsfd_request();
-}
-
-/*
- * The memory devices use the full 32/64 bits of the offset, and so we cannot
- * check against negative addresses: they are ok. The return value is weird,
- * though, in that case (0).
- *
- * also note that seeking relative to the "end of file" isn't supported:
- * it has no meaning, so it returns -EINVAL.
- */
-static loff_t jsf_lseek(struct file * file, loff_t offset, int orig)
-{
-	loff_t ret;
-
-	mutex_lock(&jsf_mutex);
-	switch (orig) {
-		case 0:
-			file->f_pos = offset;
-			ret = file->f_pos;
-			break;
-		case 1:
-			file->f_pos += offset;
-			ret = file->f_pos;
-			break;
-		default:
-			ret = -EINVAL;
-	}
-	mutex_unlock(&jsf_mutex);
-	return ret;
-}
-
-/*
- * OS SIMM Cannot be read in other size but a 32bits word.
- */
-static ssize_t jsf_read(struct file * file, char __user * buf, 
-    size_t togo, loff_t *ppos)
-{
-	unsigned long p = *ppos;
-	char __user *tmp = buf;
-
-	union byte4 {
-		char s[4];
-		unsigned int n;
-	} b;
-
-	if (p < JSF_BASE_ALL || p >= JSF_BASE_TOP) {
-		return 0;
-	}
-
-	if ((p + togo) < p	/* wrap */
-	   || (p + togo) >= JSF_BASE_TOP) {
-		togo = JSF_BASE_TOP - p;
-	}
-
-	if (p < JSF_BASE_ALL && togo != 0) {
-#if 0 /* __bzero XXX */
-		size_t x = JSF_BASE_ALL - p;
-		if (x > togo) x = togo;
-		clear_user(tmp, x);
-		tmp += x;
-		p += x;
-		togo -= x;
-#else
-		/*
-		 * Implementation of clear_user() calls __bzero
-		 * without regard to modversions,
-		 * so we cannot build a module.
-		 */
-		return 0;
-#endif
-	}
-
-	while (togo >= 4) {
-		togo -= 4;
-		b.n = jsf_inl(p);
-		if (copy_to_user(tmp, b.s, 4))
-			return -EFAULT;
-		tmp += 4;
-		p += 4;
-	}
-
-	/*
-	 * XXX Small togo may remain if 1 byte is ordered.
-	 * It would be nice if we did a word size read and unpacked it.
-	 */
-
-	*ppos = p;
-	return tmp-buf;
-}
-
-static ssize_t jsf_write(struct file * file, const char __user * buf,
-    size_t count, loff_t *ppos)
-{
-	return -ENOSPC;
-}
-
-/*
- */
-static int jsf_ioctl_erase(unsigned long arg)
-{
-	unsigned long p;
-
-	/* p = jsf0.base;	hits wrong bank */
-	p = 0x20400000;
-
-	jsf_outl(p, 0xAAAAAAAA);		/* Unlock 1 Write 1 */
-	jsf_outl(p, 0x55555555);		/* Unlock 1 Write 2 */
-	jsf_outl(p, 0x80808080);		/* Erase setup */
-	jsf_outl(p, 0xAAAAAAAA);		/* Unlock 2 Write 1 */
-	jsf_outl(p, 0x55555555);		/* Unlock 2 Write 2 */
-	jsf_outl(p, 0x10101010);		/* Chip erase */
-
-#if 0
-	/*
-	 * This code is ok, except that counter based timeout
-	 * has no place in this world. Let's just drop timeouts...
-	 */
-	{
-		int i;
-		__u32 x;
-		for (i = 0; i < 1000000; i++) {
-			x = jsf_inl(p);
-			if ((x & 0x80808080) == 0x80808080) break;
-		}
-		if ((x & 0x80808080) != 0x80808080) {
-			printk("jsf0: erase timeout with 0x%08x\n", x);
-		} else {
-			printk("jsf0: erase done with 0x%08x\n", x);
-		}
-	}
-#else
-	jsf_wait(p);
-#endif
-
-	return 0;
-}
-
-/*
- * Program a block of flash.
- * Very simple because we can do it byte by byte anyway.
- */
-static int jsf_ioctl_program(void __user *arg)
-{
-	struct jsflash_program_arg abuf;
-	char __user *uptr;
-	unsigned long p;
-	unsigned int togo;
-	union {
-		unsigned int n;
-		char s[4];
-	} b;
-
-	if (copy_from_user(&abuf, arg, JSFPRGSZ))
-		return -EFAULT; 
-	p = abuf.off;
-	togo = abuf.size;
-	if ((togo & 3) || (p & 3)) return -EINVAL;
-
-	uptr = (char __user *) (unsigned long) abuf.data;
-	while (togo != 0) {
-		togo -= 4;
-		if (copy_from_user(&b.s[0], uptr, 4))
-			return -EFAULT;
-		jsf_write4(p, b.n);
-		p += 4;
-		uptr += 4;
-	}
-
-	return 0;
-}
-
-static long jsf_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
-{
-	mutex_lock(&jsf_mutex);
-	int error = -ENOTTY;
-	void __user *argp = (void __user *)arg;
-
-	if (!capable(CAP_SYS_ADMIN)) {
-		mutex_unlock(&jsf_mutex);
-		return -EPERM;
-	}
-	switch (cmd) {
-	case JSFLASH_IDENT:
-		if (copy_to_user(argp, &jsf0.id, JSFIDSZ)) {
-			mutex_unlock(&jsf_mutex);
-			return -EFAULT;
-		}
-		break;
-	case JSFLASH_ERASE:
-		error = jsf_ioctl_erase(arg);
-		break;
-	case JSFLASH_PROGRAM:
-		error = jsf_ioctl_program(argp);
-		break;
-	}
-
-	mutex_unlock(&jsf_mutex);
-	return error;
-}
-
-static int jsf_mmap(struct file * file, struct vm_area_struct * vma)
-{
-	return -ENXIO;
-}
-
-static int jsf_open(struct inode * inode, struct file * filp)
-{
-	mutex_lock(&jsf_mutex);
-	if (jsf0.base == 0) {
-		mutex_unlock(&jsf_mutex);
-		return -ENXIO;
-	}
-	if (test_and_set_bit(0, (void *)&jsf0.busy) != 0) {
-		mutex_unlock(&jsf_mutex);
-		return -EBUSY;
-	}
-
-	mutex_unlock(&jsf_mutex);
-	return 0;	/* XXX What security? */
-}
-
-static int jsf_release(struct inode *inode, struct file *file)
-{
-	jsf0.busy = 0;
-	return 0;
-}
-
-static const struct file_operations jsf_fops = {
-	.owner =	THIS_MODULE,
-	.llseek =	jsf_lseek,
-	.read =		jsf_read,
-	.write =	jsf_write,
-	.unlocked_ioctl =	jsf_ioctl,
-	.mmap =		jsf_mmap,
-	.open =		jsf_open,
-	.release =	jsf_release,
-};
-
-static struct miscdevice jsf_dev = { JSF_MINOR, "jsflash", &jsf_fops };
-
-static const struct block_device_operations jsfd_fops = {
-	.owner =	THIS_MODULE,
-};
-
-static int jsflash_init(void)
-{
-	int rc;
-	struct jsflash *jsf;
-	phandle node;
-	char banner[128];
-	struct linux_prom_registers reg0;
-
-	node = prom_getchild(prom_root_node);
-	node = prom_searchsiblings(node, "flash-memory");
-	if (node != 0 && (s32)node != -1) {
-		if (prom_getproperty(node, "reg",
-		    (char *)&reg0, sizeof(reg0)) == -1) {
-			printk("jsflash: no \"reg\" property\n");
-			return -ENXIO;
-		}
-		if (reg0.which_io != 0) {
-			printk("jsflash: bus number nonzero: 0x%x:%x\n",
-			    reg0.which_io, reg0.phys_addr);
-			return -ENXIO;
-		}
-		/*
-		 * Flash may be somewhere else, for instance on Ebus.
-		 * So, don't do the following check for IIep flash space.
-		 */
-#if 0
-		if ((reg0.phys_addr >> 24) != 0x20) {
-			printk("jsflash: suspicious address: 0x%x:%x\n",
-			    reg0.which_io, reg0.phys_addr);
-			return -ENXIO;
-		}
-#endif
-		if ((int)reg0.reg_size <= 0) {
-			printk("jsflash: bad size 0x%x\n", (int)reg0.reg_size);
-			return -ENXIO;
-		}
-	} else {
-		/* XXX Remove this code once PROLL ID12 got widespread */
-		printk("jsflash: no /flash-memory node, use PROLL >= 12\n");
-		prom_getproperty(prom_root_node, "banner-name", banner, 128);
-		if (strcmp (banner, "JavaStation-NC") != 0 &&
-		    strcmp (banner, "JavaStation-E") != 0) {
-			return -ENXIO;
-		}
-		reg0.which_io = 0;
-		reg0.phys_addr = 0x20400000;
-		reg0.reg_size  = 0x00800000;
-	}
-
-	/* Let us be really paranoid for modifications to probing code. */
-	if (sparc_cpu_model != sun4m) {
-		/* We must be on sun4m because we use MMU Bypass ASI. */
-		return -ENXIO;
-	}
-
-	if (jsf0.base == 0) {
-		jsf = &jsf0;
-
-		jsf->base = reg0.phys_addr;
-		jsf->size = reg0.reg_size;
-
-		/* XXX Redo the userland interface. */
-		jsf->id.off = JSF_BASE_ALL;
-		jsf->id.size = 0x01000000;	/* 16M - all segments */
-		strcpy(jsf->id.name, "Krups_all");
-
-		jsf->dv[0].dbase = jsf->base;
-		jsf->dv[0].dsize = jsf->size;
-		jsf->dv[1].dbase = jsf->base + 1024;
-		jsf->dv[1].dsize = jsf->size - 1024;
-		jsf->dv[2].dbase = JSF_BASE_ALL;
-		jsf->dv[2].dsize = 0x01000000;
-
-		printk("Espresso Flash @0x%lx [%d MB]\n", jsf->base,
-		    (int) (jsf->size / (1024*1024)));
-	}
-
-	if ((rc = misc_register(&jsf_dev)) != 0) {
-		printk(KERN_ERR "jsf: unable to get misc minor %d\n",
-		    JSF_MINOR);
-		jsf0.base = 0;
-		return rc;
-	}
-
-	return 0;
-}
-
-static int jsfd_init(void)
-{
-	static DEFINE_SPINLOCK(lock);
-	struct jsflash *jsf;
-	struct jsfd_part *jdp;
-	int err;
-	int i;
-
-	if (jsf0.base == 0)
-		return -ENXIO;
-
-	err = -ENOMEM;
-	for (i = 0; i < JSF_MAX; i++) {
-		struct gendisk *disk = alloc_disk(1);
-		if (!disk)
-			goto out;
-		disk->queue = blk_init_queue(jsfd_do_request, &lock);
-		if (!disk->queue) {
-			put_disk(disk);
-			goto out;
-		}
-		blk_queue_bounce_limit(disk->queue, BLK_BOUNCE_HIGH);
-		jsfd_disk[i] = disk;
-	}
-
-	if (register_blkdev(JSFD_MAJOR, "jsfd")) {
-		err = -EIO;
-		goto out;
-	}
-
-	for (i = 0; i < JSF_MAX; i++) {
-		struct gendisk *disk = jsfd_disk[i];
-		if ((i & JSF_PART_MASK) >= JSF_NPART) continue;
-		jsf = &jsf0;	/* actually, &jsfv[i >> JSF_PART_BITS] */
-		jdp = &jsf->dv[i&JSF_PART_MASK];
-
-		disk->major = JSFD_MAJOR;
-		disk->first_minor = i;
-		sprintf(disk->disk_name, "jsfd%d", i);
-		disk->fops = &jsfd_fops;
-		set_capacity(disk, jdp->dsize >> 9);
-		disk->private_data = jdp;
-		add_disk(disk);
-		set_disk_ro(disk, 1);
-	}
-	return 0;
-out:
-	while (i--)
-		put_disk(jsfd_disk[i]);
-	return err;
-}
-
-MODULE_LICENSE("GPL");
-
-static int __init jsflash_init_module(void) {
-	int rc;
-
-	if ((rc = jsflash_init()) == 0) {
-		jsfd_init();
-		return 0;
-	}
-	return rc;
-}
-
-static void __exit jsflash_cleanup_module(void)
-{
-	int i;
-
-	for (i = 0; i < JSF_MAX; i++) {
-		if ((i & JSF_PART_MASK) >= JSF_NPART) continue;
-		del_gendisk(jsfd_disk[i]);
-		blk_cleanup_queue(jsfd_disk[i]->queue);
-		put_disk(jsfd_disk[i]);
-	}
-	if (jsf0.busy)
-		printk("jsf0: cleaning busy unit\n");
-	jsf0.base = 0;
-	jsf0.busy = 0;
-
-	misc_deregister(&jsf_dev);
-	unregister_blkdev(JSFD_MAJOR, "jsfd");
-}
-
-module_init(jsflash_init_module);
-module_exit(jsflash_cleanup_module);
diff --git a/drivers/scsi/3w-9xxx.c b/drivers/scsi/3w-9xxx.c
index b42c9c479d4b..99ba4a770406 100644
--- a/drivers/scsi/3w-9xxx.c
+++ b/drivers/scsi/3w-9xxx.c
@@ -882,6 +882,11 @@ static int twa_chrdev_open(struct inode *inode, struct file *file)
 	unsigned int minor_number;
 	int retval = TW_IOCTL_ERROR_OS_ENODEV;
 
+	if (!capable(CAP_SYS_ADMIN)) {
+		retval = -EACCES;
+		goto out;
+	}
+
 	minor_number = iminor(inode);
 	if (minor_number >= twa_device_extension_count)
 		goto out;
diff --git a/drivers/scsi/3w-xxxx.c b/drivers/scsi/3w-xxxx.c
index 33261b690774..f6179e3d6953 100644
--- a/drivers/scsi/3w-xxxx.c
+++ b/drivers/scsi/3w-xxxx.c
@@ -1033,6 +1033,9 @@ static int tw_chrdev_open(struct inode *inode, struct file *file)
 
 	dprintk(KERN_WARNING "3w-xxxx: tw_ioctl_open()\n");
 
+	if (!capable(CAP_SYS_ADMIN))
+		return -EACCES;
+
 	minor_number = iminor(inode);
 	if (minor_number >= tw_device_extension_count)
 		return -ENODEV;
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index 11e89e56b865..35c909bbf8ba 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -1351,6 +1351,20 @@ config SCSI_ZORRO7XX
 	      accelerator card for the Amiga 1200,
 	    - the SCSI controller on the GVP Turbo 040/060 accelerator.
 
+config SCSI_ZORRO_ESP
+	tristate "Zorro ESP SCSI support"
+	depends on ZORRO && SCSI
+	select SCSI_SPI_ATTRS
+	help
+	  Support for various NCR53C9x (ESP) based SCSI controllers on Zorro
+	  expansion boards for the Amiga.
+	  This includes:
+	    - the Phase5 Blizzard 1230 II and IV SCSI controllers,
+	    - the Phase5 Blizzard 2060 SCSI controller,
+	    - the Phase5 Blizzard Cyberstorm and Cyberstorm II SCSI
+	      controllers,
+	    - the Fastlane Zorro III SCSI controller.
+
 config ATARI_SCSI
 	tristate "Atari native SCSI support"
 	depends on ATARI && SCSI
diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile
index 56c940394729..80aca2456353 100644
--- a/drivers/scsi/Makefile
+++ b/drivers/scsi/Makefile
@@ -48,6 +48,7 @@ obj-$(CONFIG_INFINIBAND_ISER) 	+= libiscsi.o
 obj-$(CONFIG_ISCSI_BOOT_SYSFS)	+= iscsi_boot_sysfs.o
 obj-$(CONFIG_SCSI_A4000T)	+= 53c700.o	a4000t.o
 obj-$(CONFIG_SCSI_ZORRO7XX)	+= 53c700.o	zorro7xx.o
+obj-$(CONFIG_SCSI_ZORRO_ESP)	+= esp_scsi.o	zorro_esp.o
 obj-$(CONFIG_A3000_SCSI)	+= a3000.o	wd33c93.o
 obj-$(CONFIG_A2091_SCSI)	+= a2091.o	wd33c93.o
 obj-$(CONFIG_GVP11_SCSI)	+= gvp11.o	wd33c93.o
@@ -189,7 +190,7 @@ $(obj)/53c700.o $(MODVERDIR)/$(obj)/53c700.ver: $(obj)/53c700_d.h
 $(obj)/scsi_sysfs.o: $(obj)/scsi_devinfo_tbl.c
 
 quiet_cmd_bflags = GEN     $@
-	cmd_bflags = sed -n 's/.*BLIST_\([A-Z0-9_]*\) *.*/BLIST_FLAG_NAME(\1),/p' $< > $@
+	cmd_bflags = sed -n 's/.*define *BLIST_\([A-Z0-9_]*\) *.*/BLIST_FLAG_NAME(\1),/p' $< > $@
 
 $(obj)/scsi_devinfo_tbl.c: include/scsi/scsi_devinfo.h
 	$(call if_changed,bflags)
diff --git a/drivers/scsi/a100u2w.c b/drivers/scsi/a100u2w.c
index 8086bd0ac9fd..b2942ec3d455 100644
--- a/drivers/scsi/a100u2w.c
+++ b/drivers/scsi/a100u2w.c
@@ -1222,19 +1222,8 @@ static struct pci_driver inia100_pci_driver = {
 	.remove		= inia100_remove_one,
 };
 
-static int __init inia100_init(void)
-{
-	return pci_register_driver(&inia100_pci_driver);
-}
-
-static void __exit inia100_exit(void)
-{
-	pci_unregister_driver(&inia100_pci_driver);
-}
+module_pci_driver(inia100_pci_driver);
 
 MODULE_DESCRIPTION("Initio A100U2W SCSI driver");
 MODULE_AUTHOR("Initio Corporation");
 MODULE_LICENSE("Dual BSD/GPL");
-
-module_init(inia100_init);
-module_exit(inia100_exit);
diff --git a/drivers/scsi/am53c974.c b/drivers/scsi/am53c974.c
index beea30e5a34a..d81ca66e24d6 100644
--- a/drivers/scsi/am53c974.c
+++ b/drivers/scsi/am53c974.c
@@ -556,15 +556,7 @@ static struct pci_driver am53c974_driver = {
 	.remove         = pci_esp_remove_one,
 };
 
-static int __init am53c974_module_init(void)
-{
-	return pci_register_driver(&am53c974_driver);
-}
-
-static void __exit am53c974_module_exit(void)
-{
-	pci_unregister_driver(&am53c974_driver);
-}
+module_pci_driver(am53c974_driver);
 
 MODULE_DESCRIPTION("AM53C974 SCSI driver");
 MODULE_AUTHOR("Hannes Reinecke <hare@suse.de>");
@@ -577,6 +569,3 @@ MODULE_PARM_DESC(am53c974_debug, "Enable debugging");
 
 module_param(am53c974_fenab, bool, 0444);
 MODULE_PARM_DESC(am53c974_fenab, "Enable 24-bit DMA transfer sizes");
-
-module_init(am53c974_module_init);
-module_exit(am53c974_module_exit);
diff --git a/drivers/scsi/csiostor/csio_hw.c b/drivers/scsi/csiostor/csio_hw.c
index 96bbb82c826d..a10cf25ee7f9 100644
--- a/drivers/scsi/csiostor/csio_hw.c
+++ b/drivers/scsi/csiostor/csio_hw.c
@@ -1500,8 +1500,8 @@ fw_port_cap32_t fwcaps16_to_caps32(fw_port_cap16_t caps16)
 	CAP16_TO_CAP32(FC_RX);
 	CAP16_TO_CAP32(FC_TX);
 	CAP16_TO_CAP32(ANEG);
-	CAP16_TO_CAP32(MDIX);
 	CAP16_TO_CAP32(MDIAUTO);
+	CAP16_TO_CAP32(MDISTRAIGHT);
 	CAP16_TO_CAP32(FEC_RS);
 	CAP16_TO_CAP32(FEC_BASER_RS);
 	CAP16_TO_CAP32(802_3_PAUSE);
diff --git a/drivers/scsi/cxlflash/Kconfig b/drivers/scsi/cxlflash/Kconfig
index a011c5dbf214..f1b17e3efb3f 100644
--- a/drivers/scsi/cxlflash/Kconfig
+++ b/drivers/scsi/cxlflash/Kconfig
@@ -4,7 +4,7 @@
 
 config CXLFLASH
 	tristate "Support for IBM CAPI Flash"
-	depends on PCI && SCSI && CXL && EEH
+	depends on PCI && SCSI && (CXL || OCXL) && EEH
 	select IRQ_POLL
 	default m
 	help
diff --git a/drivers/scsi/cxlflash/Makefile b/drivers/scsi/cxlflash/Makefile
index 7ec3f6b55dde..283377d8f6fb 100644
--- a/drivers/scsi/cxlflash/Makefile
+++ b/drivers/scsi/cxlflash/Makefile
@@ -1,2 +1,4 @@
 obj-$(CONFIG_CXLFLASH) += cxlflash.o
-cxlflash-y += main.o superpipe.o lunmgt.o vlun.o cxl_hw.o
+cxlflash-y += main.o superpipe.o lunmgt.o vlun.o
+cxlflash-$(CONFIG_CXL) += cxl_hw.o
+cxlflash-$(CONFIG_OCXL) += ocxl_hw.o
diff --git a/drivers/scsi/cxlflash/backend.h b/drivers/scsi/cxlflash/backend.h
index 339e42b03c49..55638d19c2fd 100644
--- a/drivers/scsi/cxlflash/backend.h
+++ b/drivers/scsi/cxlflash/backend.h
@@ -12,30 +12,41 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#ifndef _CXLFLASH_BACKEND_H
+#define _CXLFLASH_BACKEND_H
+
 extern const struct cxlflash_backend_ops cxlflash_cxl_ops;
+extern const struct cxlflash_backend_ops cxlflash_ocxl_ops;
 
 struct cxlflash_backend_ops {
 	struct module *module;
-	void __iomem * (*psa_map)(void *);
-	void (*psa_unmap)(void __iomem *);
-	int (*process_element)(void *);
-	int (*map_afu_irq)(void *, int, irq_handler_t, void *, char *);
-	void (*unmap_afu_irq)(void *, int, void *);
-	int (*start_context)(void *);
-	int (*stop_context)(void *);
-	int (*afu_reset)(void *);
-	void (*set_master)(void *);
-	void * (*get_context)(struct pci_dev *, void *);
-	void * (*dev_context_init)(struct pci_dev *, void *);
-	int (*release_context)(void *);
-	void (*perst_reloads_same_image)(void *, bool);
-	ssize_t (*read_adapter_vpd)(struct pci_dev *, void *, size_t);
-	int (*allocate_afu_irqs)(void *, int);
-	void (*free_afu_irqs)(void *);
-	void * (*create_afu)(struct pci_dev *);
-	struct file * (*get_fd)(void *, struct file_operations *, int *);
-	void * (*fops_get_context)(struct file *);
-	int (*start_work)(void *, u64);
-	int (*fd_mmap)(struct file *, struct vm_area_struct *);
-	int (*fd_release)(struct inode *, struct file *);
+	void __iomem * (*psa_map)(void *ctx_cookie);
+	void (*psa_unmap)(void __iomem *addr);
+	int (*process_element)(void *ctx_cookie);
+	int (*map_afu_irq)(void *ctx_cookie, int num, irq_handler_t handler,
+			   void *cookie, char *name);
+	void (*unmap_afu_irq)(void *ctx_cookie, int num, void *cookie);
+	u64 (*get_irq_objhndl)(void *ctx_cookie, int irq);
+	int (*start_context)(void *ctx_cookie);
+	int (*stop_context)(void *ctx_cookie);
+	int (*afu_reset)(void *ctx_cookie);
+	void (*set_master)(void *ctx_cookie);
+	void * (*get_context)(struct pci_dev *dev, void *afu_cookie);
+	void * (*dev_context_init)(struct pci_dev *dev, void *afu_cookie);
+	int (*release_context)(void *ctx_cookie);
+	void (*perst_reloads_same_image)(void *afu_cookie, bool image);
+	ssize_t (*read_adapter_vpd)(struct pci_dev *dev, void *buf,
+				    size_t count);
+	int (*allocate_afu_irqs)(void *ctx_cookie, int num);
+	void (*free_afu_irqs)(void *ctx_cookie);
+	void * (*create_afu)(struct pci_dev *dev);
+	void (*destroy_afu)(void *afu_cookie);
+	struct file * (*get_fd)(void *ctx_cookie, struct file_operations *fops,
+				int *fd);
+	void * (*fops_get_context)(struct file *file);
+	int (*start_work)(void *ctx_cookie, u64 irqs);
+	int (*fd_mmap)(struct file *file, struct vm_area_struct *vm);
+	int (*fd_release)(struct inode *inode, struct file *file);
 };
+
+#endif /* _CXLFLASH_BACKEND_H */
diff --git a/drivers/scsi/cxlflash/common.h b/drivers/scsi/cxlflash/common.h
index 102fd26ca886..8908a20065c8 100644
--- a/drivers/scsi/cxlflash/common.h
+++ b/drivers/scsi/cxlflash/common.h
@@ -211,6 +211,7 @@ struct hwq {
 	struct sisl_ctrl_map __iomem *ctrl_map;		/* MC control map */
 	ctx_hndl_t ctx_hndl;	/* master's context handle */
 	u32 index;		/* Index of this hwq */
+	int num_irqs;		/* Number of interrupts requested for context */
 	struct list_head pending_cmds;	/* Commands pending completion */
 
 	atomic_t hsq_credits;
@@ -223,6 +224,7 @@ struct hwq {
 	u64 *hrrq_end;
 	u64 *hrrq_curr;
 	bool toggle;
+	bool hrrq_online;
 
 	s64 room;
 
@@ -231,13 +233,14 @@ struct hwq {
 
 struct afu {
 	struct hwq hwqs[CXLFLASH_MAX_HWQS];
-	int (*send_cmd)(struct afu *, struct afu_cmd *);
-	int (*context_reset)(struct hwq *);
+	int (*send_cmd)(struct afu *afu, struct afu_cmd *cmd);
+	int (*context_reset)(struct hwq *hwq);
 
 	/* AFU HW */
 	struct cxlflash_afu_map __iomem *afu_map;	/* entire MMIO map */
 
 	atomic_t cmds_active;	/* Number of currently active AFU commands */
+	struct mutex sync_active;	/* Mutex to serialize AFU commands */
 	u64 hb;
 	u32 internal_lun;	/* User-desired LUN mode for this AFU */
 
@@ -272,6 +275,11 @@ static inline bool afu_has_cap(struct afu *afu, u64 cap)
 	return afu_cap & cap;
 }
 
+static inline bool afu_is_ocxl_lisn(struct afu *afu)
+{
+	return afu_has_cap(afu, SISL_INTVER_CAP_OCXL_LISN);
+}
+
 static inline bool afu_is_afu_debug(struct afu *afu)
 {
 	return afu_has_cap(afu, SISL_INTVER_CAP_AFU_DEBUG);
diff --git a/drivers/scsi/cxlflash/cxl_hw.c b/drivers/scsi/cxlflash/cxl_hw.c
index db1cadad5c5d..b42da88386bd 100644
--- a/drivers/scsi/cxlflash/cxl_hw.c
+++ b/drivers/scsi/cxlflash/cxl_hw.c
@@ -49,6 +49,12 @@ static void cxlflash_unmap_afu_irq(void *ctx_cookie, int num, void *cookie)
 	cxl_unmap_afu_irq(ctx_cookie, num, cookie);
 }
 
+static u64 cxlflash_get_irq_objhndl(void *ctx_cookie, int irq)
+{
+	/* Dummy fop for cxl */
+	return 0;
+}
+
 static int cxlflash_start_context(void *ctx_cookie)
 {
 	return cxl_start_context(ctx_cookie, 0, NULL);
@@ -110,6 +116,11 @@ static void *cxlflash_create_afu(struct pci_dev *dev)
 	return cxl_pci_to_afu(dev);
 }
 
+static void cxlflash_destroy_afu(void *afu)
+{
+	/* Dummy fop for cxl */
+}
+
 static struct file *cxlflash_get_fd(void *ctx_cookie,
 				    struct file_operations *fops, int *fd)
 {
@@ -148,6 +159,7 @@ const struct cxlflash_backend_ops cxlflash_cxl_ops = {
 	.process_element	= cxlflash_process_element,
 	.map_afu_irq		= cxlflash_map_afu_irq,
 	.unmap_afu_irq		= cxlflash_unmap_afu_irq,
+	.get_irq_objhndl	= cxlflash_get_irq_objhndl,
 	.start_context		= cxlflash_start_context,
 	.stop_context		= cxlflash_stop_context,
 	.afu_reset		= cxlflash_afu_reset,
@@ -160,6 +172,7 @@ const struct cxlflash_backend_ops cxlflash_cxl_ops = {
 	.allocate_afu_irqs	= cxlflash_allocate_afu_irqs,
 	.free_afu_irqs		= cxlflash_free_afu_irqs,
 	.create_afu		= cxlflash_create_afu,
+	.destroy_afu		= cxlflash_destroy_afu,
 	.get_fd			= cxlflash_get_fd,
 	.fops_get_context	= cxlflash_fops_get_context,
 	.start_work		= cxlflash_start_work,
diff --git a/drivers/scsi/cxlflash/lunmgt.c b/drivers/scsi/cxlflash/lunmgt.c
index 4d232e271af6..edea1255fdab 100644
--- a/drivers/scsi/cxlflash/lunmgt.c
+++ b/drivers/scsi/cxlflash/lunmgt.c
@@ -12,9 +12,11 @@
  * 2 of the License, or (at your option) any later version.
  */
 
-#include <misc/cxl.h>
 #include <asm/unaligned.h>
 
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+
 #include <scsi/scsi_host.h>
 #include <uapi/scsi/cxlflash_ioctl.h>
 
diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c
index d8fe7ab870b8..6637116529aa 100644
--- a/drivers/scsi/cxlflash/main.c
+++ b/drivers/scsi/cxlflash/main.c
@@ -19,8 +19,6 @@
 
 #include <asm/unaligned.h>
 
-#include <misc/cxl.h>
-
 #include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_host.h>
 #include <uapi/scsi/cxlflash_ioctl.h>
@@ -339,8 +337,8 @@ static int send_cmd_ioarrin(struct afu *afu, struct afu_cmd *cmd)
 	writeq_be((u64)&cmd->rcb, &hwq->host_map->ioarrin);
 out:
 	spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
-	dev_dbg(dev, "%s: cmd=%p len=%u ea=%016llx rc=%d\n", __func__,
-		cmd, cmd->rcb.data_len, cmd->rcb.data_ea, rc);
+	dev_dbg_ratelimited(dev, "%s: cmd=%p len=%u ea=%016llx rc=%d\n",
+		__func__, cmd, cmd->rcb.data_len, cmd->rcb.data_ea, rc);
 	return rc;
 }
 
@@ -473,6 +471,7 @@ static int send_tmf(struct cxlflash_cfg *cfg, struct scsi_device *sdev,
 	struct afu_cmd *cmd = NULL;
 	struct device *dev = &cfg->dev->dev;
 	struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
+	bool needs_deletion = false;
 	char *buf = NULL;
 	ulong lock_flags;
 	int rc = 0;
@@ -527,6 +526,7 @@ static int send_tmf(struct cxlflash_cfg *cfg, struct scsi_device *sdev,
 	if (!to) {
 		dev_err(dev, "%s: TMF timed out\n", __func__);
 		rc = -ETIMEDOUT;
+		needs_deletion = true;
 	} else if (cmd->cmd_aborted) {
 		dev_err(dev, "%s: TMF aborted\n", __func__);
 		rc = -EAGAIN;
@@ -537,6 +537,12 @@ static int send_tmf(struct cxlflash_cfg *cfg, struct scsi_device *sdev,
 	}
 	cfg->tmf_active = false;
 	spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags);
+
+	if (needs_deletion) {
+		spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
+		list_del(&cmd->list);
+		spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
+	}
 out:
 	kfree(buf);
 	return rc;
@@ -608,6 +614,7 @@ static int cxlflash_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scp)
 		rc = 0;
 		goto out;
 	default:
+		atomic_inc(&afu->cmds_active);
 		break;
 	}
 
@@ -633,6 +640,7 @@ static int cxlflash_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scp)
 	memcpy(cmd->rcb.cdb, scp->cmnd, sizeof(cmd->rcb.cdb));
 
 	rc = afu->send_cmd(afu, cmd);
+	atomic_dec(&afu->cmds_active);
 out:
 	return rc;
 }
@@ -793,6 +801,10 @@ static void term_mc(struct cxlflash_cfg *cfg, u32 index)
 		WARN_ON(cfg->ops->release_context(hwq->ctx_cookie));
 	hwq->ctx_cookie = NULL;
 
+	spin_lock_irqsave(&hwq->hrrq_slock, lock_flags);
+	hwq->hrrq_online = false;
+	spin_unlock_irqrestore(&hwq->hrrq_slock, lock_flags);
+
 	spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
 	flush_pending_cmds(hwq);
 	spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
@@ -946,9 +958,9 @@ static void cxlflash_remove(struct pci_dev *pdev)
 		return;
 	}
 
-	/* If a Task Management Function is active, wait for it to complete
-	 * before continuing with remove.
-	 */
+	/* Yield to running recovery threads before continuing with remove */
+	wait_event(cfg->reset_waitq, cfg->state != STATE_RESET &&
+				     cfg->state != STATE_PROBING);
 	spin_lock_irqsave(&cfg->tmf_slock, lock_flags);
 	if (cfg->tmf_active)
 		wait_event_interruptible_lock_irq(cfg->tmf_waitq,
@@ -971,6 +983,7 @@ static void cxlflash_remove(struct pci_dev *pdev)
 	case INIT_STATE_AFU:
 		term_afu(cfg);
 	case INIT_STATE_PCI:
+		cfg->ops->destroy_afu(cfg->afu_cookie);
 		pci_disable_device(pdev);
 	case INIT_STATE_NONE:
 		free_mem(cfg);
@@ -1303,7 +1316,10 @@ static void afu_err_intr_init(struct afu *afu)
 	for (i = 0; i < afu->num_hwqs; i++) {
 		hwq = get_hwq(afu, i);
 
-		writeq_be(SISL_MSI_SYNC_ERROR, &hwq->host_map->ctx_ctrl);
+		reg = readq_be(&hwq->host_map->ctx_ctrl);
+		WARN_ON((reg & SISL_CTX_CTRL_LISN_MASK) != 0);
+		reg |= SISL_MSI_SYNC_ERROR;
+		writeq_be(reg, &hwq->host_map->ctx_ctrl);
 		writeq_be(SISL_ISTATUS_MASK, &hwq->host_map->intr_mask);
 	}
 }
@@ -1463,6 +1479,12 @@ static irqreturn_t cxlflash_rrq_irq(int irq, void *data)
 
 	spin_lock_irqsave(&hwq->hrrq_slock, hrrq_flags);
 
+	/* Silently drop spurious interrupts when queue is not online */
+	if (!hwq->hrrq_online) {
+		spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags);
+		return IRQ_HANDLED;
+	}
+
 	if (afu_is_irqpoll_enabled(afu)) {
 		irq_poll_sched(&hwq->irqpoll);
 		spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags);
@@ -1752,6 +1774,8 @@ static int init_global(struct cxlflash_cfg *cfg)
 	u64 wwpn[MAX_FC_PORTS];	/* wwpn of AFU ports */
 	int i = 0, num_ports = 0;
 	int rc = 0;
+	int j;
+	void *ctx;
 	u64 reg;
 
 	rc = read_vpd(cfg, &wwpn[0]);
@@ -1767,6 +1791,7 @@ static int init_global(struct cxlflash_cfg *cfg)
 
 		writeq_be((u64) hwq->hrrq_start, &hmap->rrq_start);
 		writeq_be((u64) hwq->hrrq_end, &hmap->rrq_end);
+		hwq->hrrq_online = true;
 
 		if (afu_is_sq_cmd_mode(afu)) {
 			writeq_be((u64)hwq->hsq_start, &hmap->sq_start);
@@ -1812,6 +1837,25 @@ static int init_global(struct cxlflash_cfg *cfg)
 		msleep(100);
 	}
 
+	if (afu_is_ocxl_lisn(afu)) {
+		/* Set up the LISN effective address for each master */
+		for (i = 0; i < afu->num_hwqs; i++) {
+			hwq = get_hwq(afu, i);
+			ctx = hwq->ctx_cookie;
+
+			for (j = 0; j < hwq->num_irqs; j++) {
+				reg = cfg->ops->get_irq_objhndl(ctx, j);
+				writeq_be(reg, &hwq->ctrl_map->lisn_ea[j]);
+			}
+
+			reg = hwq->ctx_hndl;
+			writeq_be(SISL_LISN_PASID(reg, reg),
+				  &hwq->ctrl_map->lisn_pasid[0]);
+			writeq_be(SISL_LISN_PASID(0UL, reg),
+				  &hwq->ctrl_map->lisn_pasid[1]);
+		}
+	}
+
 	/* Set up master's own CTX_CAP to allow real mode, host translation */
 	/* tables, afu cmds and read/write GSCSI cmds. */
 	/* First, unlock ctx_cap write by reading mbox */
@@ -1911,7 +1955,7 @@ static enum undo_level init_intr(struct cxlflash_cfg *cfg,
 	int rc = 0;
 	enum undo_level level = UNDO_NOOP;
 	bool is_primary_hwq = (hwq->index == PRIMARY_HWQ);
-	int num_irqs = is_primary_hwq ? 3 : 2;
+	int num_irqs = hwq->num_irqs;
 
 	rc = cfg->ops->allocate_afu_irqs(ctx, num_irqs);
 	if (unlikely(rc)) {
@@ -1965,16 +2009,20 @@ static int init_mc(struct cxlflash_cfg *cfg, u32 index)
 	struct device *dev = &cfg->dev->dev;
 	struct hwq *hwq = get_hwq(cfg->afu, index);
 	int rc = 0;
+	int num_irqs;
 	enum undo_level level;
 
 	hwq->afu = cfg->afu;
 	hwq->index = index;
 	INIT_LIST_HEAD(&hwq->pending_cmds);
 
-	if (index == PRIMARY_HWQ)
+	if (index == PRIMARY_HWQ) {
 		ctx = cfg->ops->get_context(cfg->dev, cfg->afu_cookie);
-	else
+		num_irqs = 3;
+	} else {
 		ctx = cfg->ops->dev_context_init(cfg->dev, cfg->afu_cookie);
+		num_irqs = 2;
+	}
 	if (IS_ERR_OR_NULL(ctx)) {
 		rc = -ENOMEM;
 		goto err1;
@@ -1982,6 +2030,7 @@ static int init_mc(struct cxlflash_cfg *cfg, u32 index)
 
 	WARN_ON(hwq->ctx_cookie);
 	hwq->ctx_cookie = ctx;
+	hwq->num_irqs = num_irqs;
 
 	/* Set it up as a master with the CXL */
 	cfg->ops->set_master(ctx);
@@ -2075,6 +2124,7 @@ static int init_afu(struct cxlflash_cfg *cfg)
 
 	cfg->ops->perst_reloads_same_image(cfg->afu_cookie, true);
 
+	mutex_init(&afu->sync_active);
 	afu->num_hwqs = afu->desired_hwqs;
 	for (i = 0; i < afu->num_hwqs; i++) {
 		rc = init_mc(cfg, i);
@@ -2254,10 +2304,10 @@ static int send_afu_cmd(struct afu *afu, struct sisl_ioarcb *rcb)
 	struct device *dev = &cfg->dev->dev;
 	struct afu_cmd *cmd = NULL;
 	struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
+	ulong lock_flags;
 	char *buf = NULL;
 	int rc = 0;
 	int nretry = 0;
-	static DEFINE_MUTEX(sync_active);
 
 	if (cfg->state != STATE_NORMAL) {
 		dev_dbg(dev, "%s: Sync not required state=%u\n",
@@ -2265,7 +2315,7 @@ static int send_afu_cmd(struct afu *afu, struct sisl_ioarcb *rcb)
 		return 0;
 	}
 
-	mutex_lock(&sync_active);
+	mutex_lock(&afu->sync_active);
 	atomic_inc(&afu->cmds_active);
 	buf = kmalloc(sizeof(*cmd) + __alignof__(*cmd) - 1, GFP_KERNEL);
 	if (unlikely(!buf)) {
@@ -2299,6 +2349,11 @@ static int send_afu_cmd(struct afu *afu, struct sisl_ioarcb *rcb)
 	case -ETIMEDOUT:
 		rc = afu->context_reset(hwq);
 		if (rc) {
+			/* Delete the command from pending_cmds list */
+			spin_lock_irqsave(&hwq->hsq_slock, lock_flags);
+			list_del(&cmd->list);
+			spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags);
+
 			cxlflash_schedule_async_reset(cfg);
 			break;
 		}
@@ -2315,7 +2370,7 @@ static int send_afu_cmd(struct afu *afu, struct sisl_ioarcb *rcb)
 		*rcb->ioasa = cmd->sa;
 out:
 	atomic_dec(&afu->cmds_active);
-	mutex_unlock(&sync_active);
+	mutex_unlock(&afu->sync_active);
 	kfree(buf);
 	dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
 	return rc;
@@ -3138,7 +3193,8 @@ static struct dev_dependent_vals dev_corsa_vals = { CXLFLASH_MAX_SECTORS,
 static struct dev_dependent_vals dev_flash_gt_vals = { CXLFLASH_MAX_SECTORS,
 					CXLFLASH_NOTIFY_SHUTDOWN };
 static struct dev_dependent_vals dev_briard_vals = { CXLFLASH_MAX_SECTORS,
-					CXLFLASH_NOTIFY_SHUTDOWN };
+					(CXLFLASH_NOTIFY_SHUTDOWN |
+					CXLFLASH_OCXL_DEV) };
 
 /*
  * PCI device binding table
@@ -3649,8 +3705,9 @@ static int cxlflash_probe(struct pci_dev *pdev,
 
 	cfg->init_state = INIT_STATE_NONE;
 	cfg->dev = pdev;
-	cfg->ops = &cxlflash_cxl_ops;
 	cfg->cxl_fops = cxlflash_cxl_fops;
+	cfg->ops = cxlflash_assign_ops(ddv);
+	WARN_ON_ONCE(!cfg->ops);
 
 	/*
 	 * Promoted LUNs move to the top of the LUN table. The rest stay on
@@ -3681,8 +3738,6 @@ static int cxlflash_probe(struct pci_dev *pdev,
 
 	pci_set_drvdata(pdev, cfg);
 
-	cfg->afu_cookie = cfg->ops->create_afu(pdev);
-
 	rc = init_pci(cfg);
 	if (rc) {
 		dev_err(dev, "%s: init_pci failed rc=%d\n", __func__, rc);
@@ -3690,6 +3745,12 @@ static int cxlflash_probe(struct pci_dev *pdev,
 	}
 	cfg->init_state = INIT_STATE_PCI;
 
+	cfg->afu_cookie = cfg->ops->create_afu(pdev);
+	if (unlikely(!cfg->afu_cookie)) {
+		dev_err(dev, "%s: create_afu failed\n", __func__);
+		goto out_remove;
+	}
+
 	rc = init_afu(cfg);
 	if (rc && !wq_has_sleeper(&cfg->reset_waitq)) {
 		dev_err(dev, "%s: init_afu failed rc=%d\n", __func__, rc);
diff --git a/drivers/scsi/cxlflash/main.h b/drivers/scsi/cxlflash/main.h
index ba0108a7a9c2..2a3977823812 100644
--- a/drivers/scsi/cxlflash/main.h
+++ b/drivers/scsi/cxlflash/main.h
@@ -20,6 +20,8 @@
 #include <scsi/scsi.h>
 #include <scsi/scsi_device.h>
 
+#include "backend.h"
+
 #define CXLFLASH_NAME		"cxlflash"
 #define CXLFLASH_ADAPTER_NAME	"IBM POWER CXL Flash Adapter"
 #define CXLFLASH_MAX_ADAPTERS	32
@@ -97,8 +99,27 @@ struct dev_dependent_vals {
 	u64 flags;
 #define CXLFLASH_NOTIFY_SHUTDOWN	0x0000000000000001ULL
 #define CXLFLASH_WWPN_VPD_REQUIRED	0x0000000000000002ULL
+#define CXLFLASH_OCXL_DEV		0x0000000000000004ULL
 };
 
+static inline const struct cxlflash_backend_ops *
+cxlflash_assign_ops(struct dev_dependent_vals *ddv)
+{
+	const struct cxlflash_backend_ops *ops = NULL;
+
+#ifdef CONFIG_OCXL
+	if (ddv->flags & CXLFLASH_OCXL_DEV)
+		ops = &cxlflash_ocxl_ops;
+#endif
+
+#ifdef CONFIG_CXL
+	if (!(ddv->flags & CXLFLASH_OCXL_DEV))
+		ops = &cxlflash_cxl_ops;
+#endif
+
+	return ops;
+}
+
 struct asyc_intr_info {
 	u64 status;
 	char *desc;
diff --git a/drivers/scsi/cxlflash/ocxl_hw.c b/drivers/scsi/cxlflash/ocxl_hw.c
new file mode 100644
index 000000000000..0a95b5f25380
--- /dev/null
+++ b/drivers/scsi/cxlflash/ocxl_hw.c
@@ -0,0 +1,1436 @@
+/*
+ * CXL Flash Device Driver
+ *
+ * Written by: Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
+ *             Uma Krishnan <ukrishn@linux.vnet.ibm.com>, IBM Corporation
+ *
+ * Copyright (C) 2018 IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include <linux/file.h>
+#include <linux/idr.h>
+#include <linux/module.h>
+#include <linux/mount.h>
+#include <linux/poll.h>
+#include <linux/sched/signal.h>
+
+#include <misc/ocxl.h>
+
+#include <uapi/misc/cxl.h>
+
+#include "backend.h"
+#include "ocxl_hw.h"
+
+/*
+ * Pseudo-filesystem to allocate inodes.
+ */
+
+#define OCXLFLASH_FS_MAGIC      0x1697698f
+
+static int ocxlflash_fs_cnt;
+static struct vfsmount *ocxlflash_vfs_mount;
+
+static const struct dentry_operations ocxlflash_fs_dops = {
+	.d_dname	= simple_dname,
+};
+
+/*
+ * ocxlflash_fs_mount() - mount the pseudo-filesystem
+ * @fs_type:	File system type.
+ * @flags:	Flags for the filesystem.
+ * @dev_name:	Device name associated with the filesystem.
+ * @data:	Data pointer.
+ *
+ * Return: pointer to the directory entry structure
+ */
+static struct dentry *ocxlflash_fs_mount(struct file_system_type *fs_type,
+					 int flags, const char *dev_name,
+					 void *data)
+{
+	return mount_pseudo(fs_type, "ocxlflash:", NULL, &ocxlflash_fs_dops,
+			    OCXLFLASH_FS_MAGIC);
+}
+
+static struct file_system_type ocxlflash_fs_type = {
+	.name		= "ocxlflash",
+	.owner		= THIS_MODULE,
+	.mount		= ocxlflash_fs_mount,
+	.kill_sb	= kill_anon_super,
+};
+
+/*
+ * ocxlflash_release_mapping() - release the memory mapping
+ * @ctx:	Context whose mapping is to be released.
+ */
+static void ocxlflash_release_mapping(struct ocxlflash_context *ctx)
+{
+	if (ctx->mapping)
+		simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt);
+	ctx->mapping = NULL;
+}
+
+/*
+ * ocxlflash_getfile() - allocate pseudo filesystem, inode, and the file
+ * @dev:	Generic device of the host.
+ * @name:	Name of the pseudo filesystem.
+ * @fops:	File operations.
+ * @priv:	Private data.
+ * @flags:	Flags for the file.
+ *
+ * Return: pointer to the file on success, ERR_PTR on failure
+ */
+static struct file *ocxlflash_getfile(struct device *dev, const char *name,
+				      const struct file_operations *fops,
+				      void *priv, int flags)
+{
+	struct qstr this;
+	struct path path;
+	struct file *file;
+	struct inode *inode = NULL;
+	int rc;
+
+	if (fops->owner && !try_module_get(fops->owner)) {
+		dev_err(dev, "%s: Owner does not exist\n", __func__);
+		rc = -ENOENT;
+		goto err1;
+	}
+
+	rc = simple_pin_fs(&ocxlflash_fs_type, &ocxlflash_vfs_mount,
+			   &ocxlflash_fs_cnt);
+	if (unlikely(rc < 0)) {
+		dev_err(dev, "%s: Cannot mount ocxlflash pseudofs rc=%d\n",
+			__func__, rc);
+		goto err2;
+	}
+
+	inode = alloc_anon_inode(ocxlflash_vfs_mount->mnt_sb);
+	if (IS_ERR(inode)) {
+		rc = PTR_ERR(inode);
+		dev_err(dev, "%s: alloc_anon_inode failed rc=%d\n",
+			__func__, rc);
+		goto err3;
+	}
+
+	this.name = name;
+	this.len = strlen(name);
+	this.hash = 0;
+	path.dentry = d_alloc_pseudo(ocxlflash_vfs_mount->mnt_sb, &this);
+	if (!path.dentry) {
+		dev_err(dev, "%s: d_alloc_pseudo failed\n", __func__);
+		rc = -ENOMEM;
+		goto err4;
+	}
+
+	path.mnt = mntget(ocxlflash_vfs_mount);
+	d_instantiate(path.dentry, inode);
+
+	file = alloc_file(&path, OPEN_FMODE(flags), fops);
+	if (IS_ERR(file)) {
+		rc = PTR_ERR(file);
+		dev_err(dev, "%s: alloc_file failed rc=%d\n",
+			__func__, rc);
+		goto err5;
+	}
+
+	file->f_flags = flags & (O_ACCMODE | O_NONBLOCK);
+	file->private_data = priv;
+out:
+	return file;
+err5:
+	path_put(&path);
+err4:
+	iput(inode);
+err3:
+	simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt);
+err2:
+	module_put(fops->owner);
+err1:
+	file = ERR_PTR(rc);
+	goto out;
+}
+
+/**
+ * ocxlflash_psa_map() - map the process specific MMIO space
+ * @ctx_cookie:	Adapter context for which the mapping needs to be done.
+ *
+ * Return: MMIO pointer of the mapped region
+ */
+static void __iomem *ocxlflash_psa_map(void *ctx_cookie)
+{
+	struct ocxlflash_context *ctx = ctx_cookie;
+	struct device *dev = ctx->hw_afu->dev;
+
+	mutex_lock(&ctx->state_mutex);
+	if (ctx->state != STARTED) {
+		dev_err(dev, "%s: Context not started, state=%d\n", __func__,
+			ctx->state);
+		mutex_unlock(&ctx->state_mutex);
+		return NULL;
+	}
+	mutex_unlock(&ctx->state_mutex);
+
+	return ioremap(ctx->psn_phys, ctx->psn_size);
+}
+
+/**
+ * ocxlflash_psa_unmap() - unmap the process specific MMIO space
+ * @addr:	MMIO pointer to unmap.
+ */
+static void ocxlflash_psa_unmap(void __iomem *addr)
+{
+	iounmap(addr);
+}
+
+/**
+ * ocxlflash_process_element() - get process element of the adapter context
+ * @ctx_cookie:	Adapter context associated with the process element.
+ *
+ * Return: process element of the adapter context
+ */
+static int ocxlflash_process_element(void *ctx_cookie)
+{
+	struct ocxlflash_context *ctx = ctx_cookie;
+
+	return ctx->pe;
+}
+
+/**
+ * afu_map_irq() - map the interrupt of the adapter context
+ * @flags:	Flags.
+ * @ctx:	Adapter context.
+ * @num:	Per-context AFU interrupt number.
+ * @handler:	Interrupt handler to register.
+ * @cookie:	Interrupt handler private data.
+ * @name:	Name of the interrupt.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num,
+		       irq_handler_t handler, void *cookie, char *name)
+{
+	struct ocxl_hw_afu *afu = ctx->hw_afu;
+	struct device *dev = afu->dev;
+	struct ocxlflash_irqs *irq;
+	void __iomem *vtrig;
+	u32 virq;
+	int rc = 0;
+
+	if (num < 0 || num >= ctx->num_irqs) {
+		dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num);
+		rc = -ENOENT;
+		goto out;
+	}
+
+	irq = &ctx->irqs[num];
+	virq = irq_create_mapping(NULL, irq->hwirq);
+	if (unlikely(!virq)) {
+		dev_err(dev, "%s: irq_create_mapping failed\n", __func__);
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	rc = request_irq(virq, handler, 0, name, cookie);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: request_irq failed rc=%d\n", __func__, rc);
+		goto err1;
+	}
+
+	vtrig = ioremap(irq->ptrig, PAGE_SIZE);
+	if (unlikely(!vtrig)) {
+		dev_err(dev, "%s: Trigger page mapping failed\n", __func__);
+		rc = -ENOMEM;
+		goto err2;
+	}
+
+	irq->virq = virq;
+	irq->vtrig = vtrig;
+out:
+	return rc;
+err2:
+	free_irq(virq, cookie);
+err1:
+	irq_dispose_mapping(virq);
+	goto out;
+}
+
+/**
+ * ocxlflash_map_afu_irq() - map the interrupt of the adapter context
+ * @ctx_cookie:	Adapter context.
+ * @num:	Per-context AFU interrupt number.
+ * @handler:	Interrupt handler to register.
+ * @cookie:	Interrupt handler private data.
+ * @name:	Name of the interrupt.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_map_afu_irq(void *ctx_cookie, int num,
+				 irq_handler_t handler, void *cookie,
+				 char *name)
+{
+	return afu_map_irq(0, ctx_cookie, num, handler, cookie, name);
+}
+
+/**
+ * afu_unmap_irq() - unmap the interrupt
+ * @flags:	Flags.
+ * @ctx:	Adapter context.
+ * @num:	Per-context AFU interrupt number.
+ * @cookie:	Interrupt handler private data.
+ */
+static void afu_unmap_irq(u64 flags, struct ocxlflash_context *ctx, int num,
+			  void *cookie)
+{
+	struct ocxl_hw_afu *afu = ctx->hw_afu;
+	struct device *dev = afu->dev;
+	struct ocxlflash_irqs *irq;
+
+	if (num < 0 || num >= ctx->num_irqs) {
+		dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num);
+		return;
+	}
+
+	irq = &ctx->irqs[num];
+	if (irq->vtrig)
+		iounmap(irq->vtrig);
+
+	if (irq_find_mapping(NULL, irq->hwirq)) {
+		free_irq(irq->virq, cookie);
+		irq_dispose_mapping(irq->virq);
+	}
+
+	memset(irq, 0, sizeof(*irq));
+}
+
+/**
+ * ocxlflash_unmap_afu_irq() - unmap the interrupt
+ * @ctx_cookie:	Adapter context.
+ * @num:	Per-context AFU interrupt number.
+ * @cookie:	Interrupt handler private data.
+ */
+static void ocxlflash_unmap_afu_irq(void *ctx_cookie, int num, void *cookie)
+{
+	return afu_unmap_irq(0, ctx_cookie, num, cookie);
+}
+
+/**
+ * ocxlflash_get_irq_objhndl() - get the object handle for an interrupt
+ * @ctx_cookie:	Context associated with the interrupt.
+ * @irq:	Interrupt number.
+ *
+ * Return: effective address of the mapped region
+ */
+static u64 ocxlflash_get_irq_objhndl(void *ctx_cookie, int irq)
+{
+	struct ocxlflash_context *ctx = ctx_cookie;
+
+	if (irq < 0 || irq >= ctx->num_irqs)
+		return 0;
+
+	return (__force u64)ctx->irqs[irq].vtrig;
+}
+
+/**
+ * ocxlflash_xsl_fault() - callback when translation error is triggered
+ * @data:	Private data provided at callback registration, the context.
+ * @addr:	Address that triggered the error.
+ * @dsisr:	Value of dsisr register.
+ */
+static void ocxlflash_xsl_fault(void *data, u64 addr, u64 dsisr)
+{
+	struct ocxlflash_context *ctx = data;
+
+	spin_lock(&ctx->slock);
+	ctx->fault_addr = addr;
+	ctx->fault_dsisr = dsisr;
+	ctx->pending_fault = true;
+	spin_unlock(&ctx->slock);
+
+	wake_up_all(&ctx->wq);
+}
+
+/**
+ * start_context() - local routine to start a context
+ * @ctx:	Adapter context to be started.
+ *
+ * Assign the context specific MMIO space, add and enable the PE.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int start_context(struct ocxlflash_context *ctx)
+{
+	struct ocxl_hw_afu *afu = ctx->hw_afu;
+	struct ocxl_afu_config *acfg = &afu->acfg;
+	void *link_token = afu->link_token;
+	struct device *dev = afu->dev;
+	bool master = ctx->master;
+	struct mm_struct *mm;
+	int rc = 0;
+	u32 pid;
+
+	mutex_lock(&ctx->state_mutex);
+	if (ctx->state != OPENED) {
+		dev_err(dev, "%s: Context state invalid, state=%d\n",
+			__func__, ctx->state);
+		rc = -EINVAL;
+		goto out;
+	}
+
+	if (master) {
+		ctx->psn_size = acfg->global_mmio_size;
+		ctx->psn_phys = afu->gmmio_phys;
+	} else {
+		ctx->psn_size = acfg->pp_mmio_stride;
+		ctx->psn_phys = afu->ppmmio_phys + (ctx->pe * ctx->psn_size);
+	}
+
+	/* pid and mm not set for master contexts */
+	if (master) {
+		pid = 0;
+		mm = NULL;
+	} else {
+		pid = current->mm->context.id;
+		mm = current->mm;
+	}
+
+	rc = ocxl_link_add_pe(link_token, ctx->pe, pid, 0, 0, mm,
+			      ocxlflash_xsl_fault, ctx);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: ocxl_link_add_pe failed rc=%d\n",
+			__func__, rc);
+		goto out;
+	}
+
+	ctx->state = STARTED;
+out:
+	mutex_unlock(&ctx->state_mutex);
+	return rc;
+}
+
+/**
+ * ocxlflash_start_context() - start a kernel context
+ * @ctx_cookie:	Adapter context to be started.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_start_context(void *ctx_cookie)
+{
+	struct ocxlflash_context *ctx = ctx_cookie;
+
+	return start_context(ctx);
+}
+
+/**
+ * ocxlflash_stop_context() - stop a context
+ * @ctx_cookie:	Adapter context to be stopped.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_stop_context(void *ctx_cookie)
+{
+	struct ocxlflash_context *ctx = ctx_cookie;
+	struct ocxl_hw_afu *afu = ctx->hw_afu;
+	struct ocxl_afu_config *acfg = &afu->acfg;
+	struct pci_dev *pdev = afu->pdev;
+	struct device *dev = afu->dev;
+	enum ocxlflash_ctx_state state;
+	int rc = 0;
+
+	mutex_lock(&ctx->state_mutex);
+	state = ctx->state;
+	ctx->state = CLOSED;
+	mutex_unlock(&ctx->state_mutex);
+	if (state != STARTED)
+		goto out;
+
+	rc = ocxl_config_terminate_pasid(pdev, acfg->dvsec_afu_control_pos,
+					 ctx->pe);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: ocxl_config_terminate_pasid failed rc=%d\n",
+			__func__, rc);
+		/* If EBUSY, PE could be referenced in future by the AFU */
+		if (rc == -EBUSY)
+			goto out;
+	}
+
+	rc = ocxl_link_remove_pe(afu->link_token, ctx->pe);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: ocxl_link_remove_pe failed rc=%d\n",
+			__func__, rc);
+		goto out;
+	}
+out:
+	return rc;
+}
+
+/**
+ * ocxlflash_afu_reset() - reset the AFU
+ * @ctx_cookie:	Adapter context.
+ */
+static int ocxlflash_afu_reset(void *ctx_cookie)
+{
+	struct ocxlflash_context *ctx = ctx_cookie;
+	struct device *dev = ctx->hw_afu->dev;
+
+	/* Pending implementation from OCXL transport services */
+	dev_err_once(dev, "%s: afu_reset() fop not supported\n", __func__);
+
+	/* Silently return success until it is implemented */
+	return 0;
+}
+
+/**
+ * ocxlflash_set_master() - sets the context as master
+ * @ctx_cookie:	Adapter context to set as master.
+ */
+static void ocxlflash_set_master(void *ctx_cookie)
+{
+	struct ocxlflash_context *ctx = ctx_cookie;
+
+	ctx->master = true;
+}
+
+/**
+ * ocxlflash_get_context() - obtains the context associated with the host
+ * @pdev:	PCI device associated with the host.
+ * @afu_cookie:	Hardware AFU associated with the host.
+ *
+ * Return: returns the pointer to host adapter context
+ */
+static void *ocxlflash_get_context(struct pci_dev *pdev, void *afu_cookie)
+{
+	struct ocxl_hw_afu *afu = afu_cookie;
+
+	return afu->ocxl_ctx;
+}
+
+/**
+ * ocxlflash_dev_context_init() - allocate and initialize an adapter context
+ * @pdev:	PCI device associated with the host.
+ * @afu_cookie:	Hardware AFU associated with the host.
+ *
+ * Return: returns the adapter context on success, ERR_PTR on failure
+ */
+static void *ocxlflash_dev_context_init(struct pci_dev *pdev, void *afu_cookie)
+{
+	struct ocxl_hw_afu *afu = afu_cookie;
+	struct device *dev = afu->dev;
+	struct ocxlflash_context *ctx;
+	int rc;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (unlikely(!ctx)) {
+		dev_err(dev, "%s: Context allocation failed\n", __func__);
+		rc = -ENOMEM;
+		goto err1;
+	}
+
+	idr_preload(GFP_KERNEL);
+	rc = idr_alloc(&afu->idr, ctx, 0, afu->max_pasid, GFP_NOWAIT);
+	idr_preload_end();
+	if (unlikely(rc < 0)) {
+		dev_err(dev, "%s: idr_alloc failed rc=%d\n", __func__, rc);
+		goto err2;
+	}
+
+	spin_lock_init(&ctx->slock);
+	init_waitqueue_head(&ctx->wq);
+	mutex_init(&ctx->state_mutex);
+
+	ctx->state = OPENED;
+	ctx->pe = rc;
+	ctx->master = false;
+	ctx->mapping = NULL;
+	ctx->hw_afu = afu;
+	ctx->irq_bitmap = 0;
+	ctx->pending_irq = false;
+	ctx->pending_fault = false;
+out:
+	return ctx;
+err2:
+	kfree(ctx);
+err1:
+	ctx = ERR_PTR(rc);
+	goto out;
+}
+
+/**
+ * ocxlflash_release_context() - releases an adapter context
+ * @ctx_cookie:	Adapter context to be released.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_release_context(void *ctx_cookie)
+{
+	struct ocxlflash_context *ctx = ctx_cookie;
+	struct device *dev;
+	int rc = 0;
+
+	if (!ctx)
+		goto out;
+
+	dev = ctx->hw_afu->dev;
+	mutex_lock(&ctx->state_mutex);
+	if (ctx->state >= STARTED) {
+		dev_err(dev, "%s: Context in use, state=%d\n", __func__,
+			ctx->state);
+		mutex_unlock(&ctx->state_mutex);
+		rc = -EBUSY;
+		goto out;
+	}
+	mutex_unlock(&ctx->state_mutex);
+
+	idr_remove(&ctx->hw_afu->idr, ctx->pe);
+	ocxlflash_release_mapping(ctx);
+	kfree(ctx);
+out:
+	return rc;
+}
+
+/**
+ * ocxlflash_perst_reloads_same_image() - sets the image reload policy
+ * @afu_cookie:	Hardware AFU associated with the host.
+ * @image:	Whether to load the same image on PERST.
+ */
+static void ocxlflash_perst_reloads_same_image(void *afu_cookie, bool image)
+{
+	struct ocxl_hw_afu *afu = afu_cookie;
+
+	afu->perst_same_image = image;
+}
+
+/**
+ * ocxlflash_read_adapter_vpd() - reads the adapter VPD
+ * @pdev:	PCI device associated with the host.
+ * @buf:	Buffer to get the VPD data.
+ * @count:	Size of buffer (maximum bytes that can be read).
+ *
+ * Return: size of VPD on success, -errno on failure
+ */
+static ssize_t ocxlflash_read_adapter_vpd(struct pci_dev *pdev, void *buf,
+					  size_t count)
+{
+	return pci_read_vpd(pdev, 0, count, buf);
+}
+
+/**
+ * free_afu_irqs() - internal service to free interrupts
+ * @ctx:	Adapter context.
+ */
+static void free_afu_irqs(struct ocxlflash_context *ctx)
+{
+	struct ocxl_hw_afu *afu = ctx->hw_afu;
+	struct device *dev = afu->dev;
+	int i;
+
+	if (!ctx->irqs) {
+		dev_err(dev, "%s: Interrupts not allocated\n", __func__);
+		return;
+	}
+
+	for (i = ctx->num_irqs; i >= 0; i--)
+		ocxl_link_free_irq(afu->link_token, ctx->irqs[i].hwirq);
+
+	kfree(ctx->irqs);
+	ctx->irqs = NULL;
+}
+
+/**
+ * alloc_afu_irqs() - internal service to allocate interrupts
+ * @ctx:	Context associated with the request.
+ * @num:	Number of interrupts requested.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num)
+{
+	struct ocxl_hw_afu *afu = ctx->hw_afu;
+	struct device *dev = afu->dev;
+	struct ocxlflash_irqs *irqs;
+	u64 addr;
+	int rc = 0;
+	int hwirq;
+	int i;
+
+	if (ctx->irqs) {
+		dev_err(dev, "%s: Interrupts already allocated\n", __func__);
+		rc = -EEXIST;
+		goto out;
+	}
+
+	if (num > OCXL_MAX_IRQS) {
+		dev_err(dev, "%s: Too many interrupts num=%d\n", __func__, num);
+		rc = -EINVAL;
+		goto out;
+	}
+
+	irqs = kcalloc(num, sizeof(*irqs), GFP_KERNEL);
+	if (unlikely(!irqs)) {
+		dev_err(dev, "%s: Context irqs allocation failed\n", __func__);
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	for (i = 0; i < num; i++) {
+		rc = ocxl_link_irq_alloc(afu->link_token, &hwirq, &addr);
+		if (unlikely(rc)) {
+			dev_err(dev, "%s: ocxl_link_irq_alloc failed rc=%d\n",
+				__func__, rc);
+			goto err;
+		}
+
+		irqs[i].hwirq = hwirq;
+		irqs[i].ptrig = addr;
+	}
+
+	ctx->irqs = irqs;
+	ctx->num_irqs = num;
+out:
+	return rc;
+err:
+	for (i = i-1; i >= 0; i--)
+		ocxl_link_free_irq(afu->link_token, irqs[i].hwirq);
+	kfree(irqs);
+	goto out;
+}
+
+/**
+ * ocxlflash_allocate_afu_irqs() - allocates the requested number of interrupts
+ * @ctx_cookie:	Context associated with the request.
+ * @num:	Number of interrupts requested.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_allocate_afu_irqs(void *ctx_cookie, int num)
+{
+	return alloc_afu_irqs(ctx_cookie, num);
+}
+
+/**
+ * ocxlflash_free_afu_irqs() - frees the interrupts of an adapter context
+ * @ctx_cookie:	Adapter context.
+ */
+static void ocxlflash_free_afu_irqs(void *ctx_cookie)
+{
+	free_afu_irqs(ctx_cookie);
+}
+
+/**
+ * ocxlflash_unconfig_afu() - unconfigure the AFU
+ * @afu: AFU associated with the host.
+ */
+static void ocxlflash_unconfig_afu(struct ocxl_hw_afu *afu)
+{
+	if (afu->gmmio_virt) {
+		iounmap(afu->gmmio_virt);
+		afu->gmmio_virt = NULL;
+	}
+}
+
+/**
+ * ocxlflash_destroy_afu() - destroy the AFU structure
+ * @afu_cookie:	AFU to be freed.
+ */
+static void ocxlflash_destroy_afu(void *afu_cookie)
+{
+	struct ocxl_hw_afu *afu = afu_cookie;
+	int pos;
+
+	if (!afu)
+		return;
+
+	ocxlflash_release_context(afu->ocxl_ctx);
+	idr_destroy(&afu->idr);
+
+	/* Disable the AFU */
+	pos = afu->acfg.dvsec_afu_control_pos;
+	ocxl_config_set_afu_state(afu->pdev, pos, 0);
+
+	ocxlflash_unconfig_afu(afu);
+	kfree(afu);
+}
+
+/**
+ * ocxlflash_config_fn() - configure the host function
+ * @pdev:	PCI device associated with the host.
+ * @afu:	AFU associated with the host.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_config_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
+{
+	struct ocxl_fn_config *fcfg = &afu->fcfg;
+	struct device *dev = &pdev->dev;
+	u16 base, enabled, supported;
+	int rc = 0;
+
+	/* Read DVSEC config of the function */
+	rc = ocxl_config_read_function(pdev, fcfg);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: ocxl_config_read_function failed rc=%d\n",
+			__func__, rc);
+		goto out;
+	}
+
+	/* Check if function has AFUs defined, only 1 per function supported */
+	if (fcfg->max_afu_index >= 0) {
+		afu->is_present = true;
+		if (fcfg->max_afu_index != 0)
+			dev_warn(dev, "%s: Unexpected AFU index value %d\n",
+				 __func__, fcfg->max_afu_index);
+	}
+
+	rc = ocxl_config_get_actag_info(pdev, &base, &enabled, &supported);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: ocxl_config_get_actag_info failed rc=%d\n",
+			__func__, rc);
+		goto out;
+	}
+
+	afu->fn_actag_base = base;
+	afu->fn_actag_enabled = enabled;
+
+	ocxl_config_set_actag(pdev, fcfg->dvsec_function_pos, base, enabled);
+	dev_dbg(dev, "%s: Function acTag range base=%u enabled=%u\n",
+		__func__, base, enabled);
+
+	rc = ocxl_link_setup(pdev, 0, &afu->link_token);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: ocxl_link_setup failed rc=%d\n",
+			__func__, rc);
+		goto out;
+	}
+
+	rc = ocxl_config_set_TL(pdev, fcfg->dvsec_tl_pos);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: ocxl_config_set_TL failed rc=%d\n",
+			__func__, rc);
+		goto err;
+	}
+out:
+	return rc;
+err:
+	ocxl_link_release(pdev, afu->link_token);
+	goto out;
+}
+
+/**
+ * ocxlflash_unconfig_fn() - unconfigure the host function
+ * @pdev:	PCI device associated with the host.
+ * @afu:	AFU associated with the host.
+ */
+static void ocxlflash_unconfig_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
+{
+	ocxl_link_release(pdev, afu->link_token);
+}
+
+/**
+ * ocxlflash_map_mmio() - map the AFU MMIO space
+ * @afu: AFU associated with the host.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_map_mmio(struct ocxl_hw_afu *afu)
+{
+	struct ocxl_afu_config *acfg = &afu->acfg;
+	struct pci_dev *pdev = afu->pdev;
+	struct device *dev = afu->dev;
+	phys_addr_t gmmio, ppmmio;
+	int rc = 0;
+
+	rc = pci_request_region(pdev, acfg->global_mmio_bar, "ocxlflash");
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: pci_request_region for global failed rc=%d\n",
+			__func__, rc);
+		goto out;
+	}
+	gmmio = pci_resource_start(pdev, acfg->global_mmio_bar);
+	gmmio += acfg->global_mmio_offset;
+
+	rc = pci_request_region(pdev, acfg->pp_mmio_bar, "ocxlflash");
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: pci_request_region for pp bar failed rc=%d\n",
+			__func__, rc);
+		goto err1;
+	}
+	ppmmio = pci_resource_start(pdev, acfg->pp_mmio_bar);
+	ppmmio += acfg->pp_mmio_offset;
+
+	afu->gmmio_virt = ioremap(gmmio, acfg->global_mmio_size);
+	if (unlikely(!afu->gmmio_virt)) {
+		dev_err(dev, "%s: MMIO mapping failed\n", __func__);
+		rc = -ENOMEM;
+		goto err2;
+	}
+
+	afu->gmmio_phys = gmmio;
+	afu->ppmmio_phys = ppmmio;
+out:
+	return rc;
+err2:
+	pci_release_region(pdev, acfg->pp_mmio_bar);
+err1:
+	pci_release_region(pdev, acfg->global_mmio_bar);
+	goto out;
+}
+
+/**
+ * ocxlflash_config_afu() - configure the host AFU
+ * @pdev:	PCI device associated with the host.
+ * @afu:	AFU associated with the host.
+ *
+ * Must be called _after_ host function configuration.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_config_afu(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
+{
+	struct ocxl_afu_config *acfg = &afu->acfg;
+	struct ocxl_fn_config *fcfg = &afu->fcfg;
+	struct device *dev = &pdev->dev;
+	int count;
+	int base;
+	int pos;
+	int rc = 0;
+
+	/* This HW AFU function does not have any AFUs defined */
+	if (!afu->is_present)
+		goto out;
+
+	/* Read AFU config at index 0 */
+	rc = ocxl_config_read_afu(pdev, fcfg, acfg, 0);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: ocxl_config_read_afu failed rc=%d\n",
+			__func__, rc);
+		goto out;
+	}
+
+	/* Only one AFU per function is supported, so actag_base is same */
+	base = afu->fn_actag_base;
+	count = min_t(int, acfg->actag_supported, afu->fn_actag_enabled);
+	pos = acfg->dvsec_afu_control_pos;
+
+	ocxl_config_set_afu_actag(pdev, pos, base, count);
+	dev_dbg(dev, "%s: acTag base=%d enabled=%d\n", __func__, base, count);
+	afu->afu_actag_base = base;
+	afu->afu_actag_enabled = count;
+	afu->max_pasid = 1 << acfg->pasid_supported_log;
+
+	ocxl_config_set_afu_pasid(pdev, pos, 0, acfg->pasid_supported_log);
+
+	rc = ocxlflash_map_mmio(afu);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: ocxlflash_map_mmio failed rc=%d\n",
+			__func__, rc);
+		goto out;
+	}
+
+	/* Enable the AFU */
+	ocxl_config_set_afu_state(pdev, acfg->dvsec_afu_control_pos, 1);
+out:
+	return rc;
+}
+
+/**
+ * ocxlflash_create_afu() - create the AFU for OCXL
+ * @pdev:	PCI device associated with the host.
+ *
+ * Return: AFU on success, NULL on failure
+ */
+static void *ocxlflash_create_afu(struct pci_dev *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct ocxlflash_context *ctx;
+	struct ocxl_hw_afu *afu;
+	int rc;
+
+	afu = kzalloc(sizeof(*afu), GFP_KERNEL);
+	if (unlikely(!afu)) {
+		dev_err(dev, "%s: HW AFU allocation failed\n", __func__);
+		goto out;
+	}
+
+	afu->pdev = pdev;
+	afu->dev = dev;
+	idr_init(&afu->idr);
+
+	rc = ocxlflash_config_fn(pdev, afu);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: Function configuration failed rc=%d\n",
+			__func__, rc);
+		goto err1;
+	}
+
+	rc = ocxlflash_config_afu(pdev, afu);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: AFU configuration failed rc=%d\n",
+			__func__, rc);
+		goto err2;
+	}
+
+	ctx = ocxlflash_dev_context_init(pdev, afu);
+	if (IS_ERR(ctx)) {
+		rc = PTR_ERR(ctx);
+		dev_err(dev, "%s: ocxlflash_dev_context_init failed rc=%d\n",
+			__func__, rc);
+		goto err3;
+	}
+
+	afu->ocxl_ctx = ctx;
+out:
+	return afu;
+err3:
+	ocxlflash_unconfig_afu(afu);
+err2:
+	ocxlflash_unconfig_fn(pdev, afu);
+err1:
+	idr_destroy(&afu->idr);
+	kfree(afu);
+	afu = NULL;
+	goto out;
+}
+
+/**
+ * ctx_event_pending() - check for any event pending on the context
+ * @ctx:	Context to be checked.
+ *
+ * Return: true if there is an event pending, false if none pending
+ */
+static inline bool ctx_event_pending(struct ocxlflash_context *ctx)
+{
+	if (ctx->pending_irq || ctx->pending_fault)
+		return true;
+
+	return false;
+}
+
+/**
+ * afu_poll() - poll the AFU for events on the context
+ * @file:	File associated with the adapter context.
+ * @poll:	Poll structure from the user.
+ *
+ * Return: poll mask
+ */
+static unsigned int afu_poll(struct file *file, struct poll_table_struct *poll)
+{
+	struct ocxlflash_context *ctx = file->private_data;
+	struct device *dev = ctx->hw_afu->dev;
+	ulong lock_flags;
+	int mask = 0;
+
+	poll_wait(file, &ctx->wq, poll);
+
+	spin_lock_irqsave(&ctx->slock, lock_flags);
+	if (ctx_event_pending(ctx))
+		mask |= POLLIN | POLLRDNORM;
+	else if (ctx->state == CLOSED)
+		mask |= POLLERR;
+	spin_unlock_irqrestore(&ctx->slock, lock_flags);
+
+	dev_dbg(dev, "%s: Poll wait completed for pe %i mask %i\n",
+		__func__, ctx->pe, mask);
+
+	return mask;
+}
+
+/**
+ * afu_read() - perform a read on the context for any event
+ * @file:	File associated with the adapter context.
+ * @buf:	Buffer to receive the data.
+ * @count:	Size of buffer (maximum bytes that can be read).
+ * @off:	Offset.
+ *
+ * Return: size of the data read on success, -errno on failure
+ */
+static ssize_t afu_read(struct file *file, char __user *buf, size_t count,
+			loff_t *off)
+{
+	struct ocxlflash_context *ctx = file->private_data;
+	struct device *dev = ctx->hw_afu->dev;
+	struct cxl_event event;
+	ulong lock_flags;
+	ssize_t esize;
+	ssize_t rc;
+	int bit;
+	DEFINE_WAIT(event_wait);
+
+	if (*off != 0) {
+		dev_err(dev, "%s: Non-zero offset not supported, off=%lld\n",
+			__func__, *off);
+		rc = -EINVAL;
+		goto out;
+	}
+
+	spin_lock_irqsave(&ctx->slock, lock_flags);
+
+	for (;;) {
+		prepare_to_wait(&ctx->wq, &event_wait, TASK_INTERRUPTIBLE);
+
+		if (ctx_event_pending(ctx) || (ctx->state == CLOSED))
+			break;
+
+		if (file->f_flags & O_NONBLOCK) {
+			dev_err(dev, "%s: File cannot be blocked on I/O\n",
+				__func__);
+			rc = -EAGAIN;
+			goto err;
+		}
+
+		if (signal_pending(current)) {
+			dev_err(dev, "%s: Signal pending on the process\n",
+				__func__);
+			rc = -ERESTARTSYS;
+			goto err;
+		}
+
+		spin_unlock_irqrestore(&ctx->slock, lock_flags);
+		schedule();
+		spin_lock_irqsave(&ctx->slock, lock_flags);
+	}
+
+	finish_wait(&ctx->wq, &event_wait);
+
+	memset(&event, 0, sizeof(event));
+	event.header.process_element = ctx->pe;
+	event.header.size = sizeof(struct cxl_event_header);
+	if (ctx->pending_irq) {
+		esize = sizeof(struct cxl_event_afu_interrupt);
+		event.header.size += esize;
+		event.header.type = CXL_EVENT_AFU_INTERRUPT;
+
+		bit = find_first_bit(&ctx->irq_bitmap, ctx->num_irqs);
+		clear_bit(bit, &ctx->irq_bitmap);
+		event.irq.irq = bit + 1;
+		if (bitmap_empty(&ctx->irq_bitmap, ctx->num_irqs))
+			ctx->pending_irq = false;
+	} else if (ctx->pending_fault) {
+		event.header.size += sizeof(struct cxl_event_data_storage);
+		event.header.type = CXL_EVENT_DATA_STORAGE;
+		event.fault.addr = ctx->fault_addr;
+		event.fault.dsisr = ctx->fault_dsisr;
+		ctx->pending_fault = false;
+	}
+
+	spin_unlock_irqrestore(&ctx->slock, lock_flags);
+
+	if (copy_to_user(buf, &event, event.header.size)) {
+		dev_err(dev, "%s: copy_to_user failed\n", __func__);
+		rc = -EFAULT;
+		goto out;
+	}
+
+	rc = event.header.size;
+out:
+	return rc;
+err:
+	finish_wait(&ctx->wq, &event_wait);
+	spin_unlock_irqrestore(&ctx->slock, lock_flags);
+	goto out;
+}
+
+/**
+ * afu_release() - release and free the context
+ * @inode:	File inode pointer.
+ * @file:	File associated with the context.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int afu_release(struct inode *inode, struct file *file)
+{
+	struct ocxlflash_context *ctx = file->private_data;
+	int i;
+
+	/* Unmap and free the interrupts associated with the context */
+	for (i = ctx->num_irqs; i >= 0; i--)
+		afu_unmap_irq(0, ctx, i, ctx);
+	free_afu_irqs(ctx);
+
+	return ocxlflash_release_context(ctx);
+}
+
+/**
+ * ocxlflash_mmap_fault() - mmap fault handler
+ * @vmf:	VM fault associated with current fault.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_mmap_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct ocxlflash_context *ctx = vma->vm_file->private_data;
+	struct device *dev = ctx->hw_afu->dev;
+	u64 mmio_area, offset;
+
+	offset = vmf->pgoff << PAGE_SHIFT;
+	if (offset >= ctx->psn_size)
+		return VM_FAULT_SIGBUS;
+
+	mutex_lock(&ctx->state_mutex);
+	if (ctx->state != STARTED) {
+		dev_err(dev, "%s: Context not started, state=%d\n",
+			__func__, ctx->state);
+		mutex_unlock(&ctx->state_mutex);
+		return VM_FAULT_SIGBUS;
+	}
+	mutex_unlock(&ctx->state_mutex);
+
+	mmio_area = ctx->psn_phys;
+	mmio_area += offset;
+
+	vm_insert_pfn(vma, vmf->address, mmio_area >> PAGE_SHIFT);
+	return VM_FAULT_NOPAGE;
+}
+
+static const struct vm_operations_struct ocxlflash_vmops = {
+	.fault = ocxlflash_mmap_fault,
+};
+
+/**
+ * afu_mmap() - map the fault handler operations
+ * @file:	File associated with the context.
+ * @vma:	VM area associated with mapping.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int afu_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct ocxlflash_context *ctx = file->private_data;
+
+	if ((vma_pages(vma) + vma->vm_pgoff) >
+	    (ctx->psn_size >> PAGE_SHIFT))
+		return -EINVAL;
+
+	vma->vm_flags |= VM_IO | VM_PFNMAP;
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	vma->vm_ops = &ocxlflash_vmops;
+	return 0;
+}
+
+static const struct file_operations ocxl_afu_fops = {
+	.owner		= THIS_MODULE,
+	.poll		= afu_poll,
+	.read		= afu_read,
+	.release	= afu_release,
+	.mmap		= afu_mmap,
+};
+
+#define PATCH_FOPS(NAME)						\
+	do { if (!fops->NAME) fops->NAME = ocxl_afu_fops.NAME; } while (0)
+
+/**
+ * ocxlflash_get_fd() - get file descriptor for an adapter context
+ * @ctx_cookie:	Adapter context.
+ * @fops:	File operations to be associated.
+ * @fd:		File descriptor to be returned back.
+ *
+ * Return: pointer to the file on success, ERR_PTR on failure
+ */
+static struct file *ocxlflash_get_fd(void *ctx_cookie,
+				     struct file_operations *fops, int *fd)
+{
+	struct ocxlflash_context *ctx = ctx_cookie;
+	struct device *dev = ctx->hw_afu->dev;
+	struct file *file;
+	int flags, fdtmp;
+	int rc = 0;
+	char *name = NULL;
+
+	/* Only allow one fd per context */
+	if (ctx->mapping) {
+		dev_err(dev, "%s: Context is already mapped to an fd\n",
+			__func__);
+		rc = -EEXIST;
+		goto err1;
+	}
+
+	flags = O_RDWR | O_CLOEXEC;
+
+	/* This code is similar to anon_inode_getfd() */
+	rc = get_unused_fd_flags(flags);
+	if (unlikely(rc < 0)) {
+		dev_err(dev, "%s: get_unused_fd_flags failed rc=%d\n",
+			__func__, rc);
+		goto err1;
+	}
+	fdtmp = rc;
+
+	/* Patch the file ops that are not defined */
+	if (fops) {
+		PATCH_FOPS(poll);
+		PATCH_FOPS(read);
+		PATCH_FOPS(release);
+		PATCH_FOPS(mmap);
+	} else /* Use default ops */
+		fops = (struct file_operations *)&ocxl_afu_fops;
+
+	name = kasprintf(GFP_KERNEL, "ocxlflash:%d", ctx->pe);
+	file = ocxlflash_getfile(dev, name, fops, ctx, flags);
+	kfree(name);
+	if (IS_ERR(file)) {
+		rc = PTR_ERR(file);
+		dev_err(dev, "%s: ocxlflash_getfile failed rc=%d\n",
+			__func__, rc);
+		goto err2;
+	}
+
+	ctx->mapping = file->f_mapping;
+	*fd = fdtmp;
+out:
+	return file;
+err2:
+	put_unused_fd(fdtmp);
+err1:
+	file = ERR_PTR(rc);
+	goto out;
+}
+
+/**
+ * ocxlflash_fops_get_context() - get the context associated with the file
+ * @file:	File associated with the adapter context.
+ *
+ * Return: pointer to the context
+ */
+static void *ocxlflash_fops_get_context(struct file *file)
+{
+	return file->private_data;
+}
+
+/**
+ * ocxlflash_afu_irq() - interrupt handler for user contexts
+ * @irq:	Interrupt number.
+ * @data:	Private data provided at interrupt registration, the context.
+ *
+ * Return: Always return IRQ_HANDLED.
+ */
+static irqreturn_t ocxlflash_afu_irq(int irq, void *data)
+{
+	struct ocxlflash_context *ctx = data;
+	struct device *dev = ctx->hw_afu->dev;
+	int i;
+
+	dev_dbg(dev, "%s: Interrupt raised for pe %i virq %i\n",
+		__func__, ctx->pe, irq);
+
+	for (i = 0; i < ctx->num_irqs; i++) {
+		if (ctx->irqs[i].virq == irq)
+			break;
+	}
+	if (unlikely(i >= ctx->num_irqs)) {
+		dev_err(dev, "%s: Received AFU IRQ out of range\n", __func__);
+		goto out;
+	}
+
+	spin_lock(&ctx->slock);
+	set_bit(i - 1, &ctx->irq_bitmap);
+	ctx->pending_irq = true;
+	spin_unlock(&ctx->slock);
+
+	wake_up_all(&ctx->wq);
+out:
+	return IRQ_HANDLED;
+}
+
+/**
+ * ocxlflash_start_work() - start a user context
+ * @ctx_cookie:	Context to be started.
+ * @num_irqs:	Number of interrupts requested.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_start_work(void *ctx_cookie, u64 num_irqs)
+{
+	struct ocxlflash_context *ctx = ctx_cookie;
+	struct ocxl_hw_afu *afu = ctx->hw_afu;
+	struct device *dev = afu->dev;
+	char *name;
+	int rc = 0;
+	int i;
+
+	rc = alloc_afu_irqs(ctx, num_irqs);
+	if (unlikely(rc < 0)) {
+		dev_err(dev, "%s: alloc_afu_irqs failed rc=%d\n", __func__, rc);
+		goto out;
+	}
+
+	for (i = 0; i < num_irqs; i++) {
+		name = kasprintf(GFP_KERNEL, "ocxlflash-%s-pe%i-%i",
+				 dev_name(dev), ctx->pe, i);
+		rc = afu_map_irq(0, ctx, i, ocxlflash_afu_irq, ctx, name);
+		kfree(name);
+		if (unlikely(rc < 0)) {
+			dev_err(dev, "%s: afu_map_irq failed rc=%d\n",
+				__func__, rc);
+			goto err;
+		}
+	}
+
+	rc = start_context(ctx);
+	if (unlikely(rc)) {
+		dev_err(dev, "%s: start_context failed rc=%d\n", __func__, rc);
+		goto err;
+	}
+out:
+	return rc;
+err:
+	for (i = i-1; i >= 0; i--)
+		afu_unmap_irq(0, ctx, i, ctx);
+	free_afu_irqs(ctx);
+	goto out;
+};
+
+/**
+ * ocxlflash_fd_mmap() - mmap handler for adapter file descriptor
+ * @file:	File installed with adapter file descriptor.
+ * @vma:	VM area associated with mapping.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_fd_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	return afu_mmap(file, vma);
+}
+
+/**
+ * ocxlflash_fd_release() - release the context associated with the file
+ * @inode:	File inode pointer.
+ * @file:	File associated with the adapter context.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int ocxlflash_fd_release(struct inode *inode, struct file *file)
+{
+	return afu_release(inode, file);
+}
+
+/* Backend ops to ocxlflash services */
+const struct cxlflash_backend_ops cxlflash_ocxl_ops = {
+	.module			= THIS_MODULE,
+	.psa_map		= ocxlflash_psa_map,
+	.psa_unmap		= ocxlflash_psa_unmap,
+	.process_element	= ocxlflash_process_element,
+	.map_afu_irq		= ocxlflash_map_afu_irq,
+	.unmap_afu_irq		= ocxlflash_unmap_afu_irq,
+	.get_irq_objhndl	= ocxlflash_get_irq_objhndl,
+	.start_context		= ocxlflash_start_context,
+	.stop_context		= ocxlflash_stop_context,
+	.afu_reset		= ocxlflash_afu_reset,
+	.set_master		= ocxlflash_set_master,
+	.get_context		= ocxlflash_get_context,
+	.dev_context_init	= ocxlflash_dev_context_init,
+	.release_context	= ocxlflash_release_context,
+	.perst_reloads_same_image = ocxlflash_perst_reloads_same_image,
+	.read_adapter_vpd	= ocxlflash_read_adapter_vpd,
+	.allocate_afu_irqs	= ocxlflash_allocate_afu_irqs,
+	.free_afu_irqs		= ocxlflash_free_afu_irqs,
+	.create_afu		= ocxlflash_create_afu,
+	.destroy_afu		= ocxlflash_destroy_afu,
+	.get_fd			= ocxlflash_get_fd,
+	.fops_get_context	= ocxlflash_fops_get_context,
+	.start_work		= ocxlflash_start_work,
+	.fd_mmap		= ocxlflash_fd_mmap,
+	.fd_release		= ocxlflash_fd_release,
+};
diff --git a/drivers/scsi/cxlflash/ocxl_hw.h b/drivers/scsi/cxlflash/ocxl_hw.h
new file mode 100644
index 000000000000..9270d35c4620
--- /dev/null
+++ b/drivers/scsi/cxlflash/ocxl_hw.h
@@ -0,0 +1,77 @@
+/*
+ * CXL Flash Device Driver
+ *
+ * Written by: Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
+ *	       Uma Krishnan <ukrishn@linux.vnet.ibm.com>, IBM Corporation
+ *
+ * Copyright (C) 2018 IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#define OCXL_MAX_IRQS	4	/* Max interrupts per process */
+
+struct ocxlflash_irqs {
+	int hwirq;
+	u32 virq;
+	u64 ptrig;
+	void __iomem *vtrig;
+};
+
+/* OCXL hardware AFU associated with the host */
+struct ocxl_hw_afu {
+	struct ocxlflash_context *ocxl_ctx; /* Host context */
+	struct pci_dev *pdev;		/* PCI device */
+	struct device *dev;		/* Generic device */
+	bool perst_same_image;		/* Same image loaded on perst */
+
+	struct ocxl_fn_config fcfg;	/* DVSEC config of the function */
+	struct ocxl_afu_config acfg;	/* AFU configuration data */
+
+	int fn_actag_base;		/* Function acTag base */
+	int fn_actag_enabled;		/* Function acTag number enabled */
+	int afu_actag_base;		/* AFU acTag base */
+	int afu_actag_enabled;		/* AFU acTag number enabled */
+
+	phys_addr_t ppmmio_phys;	/* Per process MMIO space */
+	phys_addr_t gmmio_phys;		/* Global AFU MMIO space */
+	void __iomem *gmmio_virt;	/* Global MMIO map */
+
+	void *link_token;		/* Link token for the SPA */
+	struct idr idr;			/* IDR to manage contexts */
+	int max_pasid;			/* Maximum number of contexts */
+	bool is_present;		/* Function has AFUs defined */
+};
+
+enum ocxlflash_ctx_state {
+	CLOSED,
+	OPENED,
+	STARTED
+};
+
+struct ocxlflash_context {
+	struct ocxl_hw_afu *hw_afu;	/* HW AFU back pointer */
+	struct address_space *mapping;	/* Mapping for pseudo filesystem */
+	bool master;			/* Whether this is a master context */
+	int pe;				/* Process element */
+
+	phys_addr_t psn_phys;		/* Process mapping */
+	u64 psn_size;			/* Process mapping size */
+
+	spinlock_t slock;		/* Protects irq/fault/event updates */
+	wait_queue_head_t wq;		/* Wait queue for poll and interrupts */
+	struct mutex state_mutex;	/* Mutex to update context state */
+	enum ocxlflash_ctx_state state;	/* Context state */
+
+	struct ocxlflash_irqs *irqs;	/* Pointer to array of structures */
+	int num_irqs;			/* Number of interrupts */
+	bool pending_irq;		/* Pending interrupt on the context */
+	ulong irq_bitmap;		/* Bits indicating pending irq num */
+
+	u64 fault_addr;			/* Address that triggered the fault */
+	u64 fault_dsisr;		/* Value of dsisr register at fault */
+	bool pending_fault;		/* Pending translation fault */
+};
diff --git a/drivers/scsi/cxlflash/sislite.h b/drivers/scsi/cxlflash/sislite.h
index bedf1ce2f33c..874abce35ab4 100644
--- a/drivers/scsi/cxlflash/sislite.h
+++ b/drivers/scsi/cxlflash/sislite.h
@@ -258,23 +258,30 @@ struct sisl_host_map {
 				 * exit since there is no way to tell which
 				 * command caused the error.
 				 */
-#define SISL_ISTATUS_PERM_ERR_CMDROOM    0x0010ULL	/* b59, user error */
-#define SISL_ISTATUS_PERM_ERR_RCB_READ   0x0008ULL	/* b60, user error */
-#define SISL_ISTATUS_PERM_ERR_SA_WRITE   0x0004ULL	/* b61, user error */
-#define SISL_ISTATUS_PERM_ERR_RRQ_WRITE  0x0002ULL	/* b62, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_3_EA		0x0400ULL /* b53, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_2_EA		0x0200ULL /* b54, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_1_EA		0x0100ULL /* b55, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_3_PASID	0x0080ULL /* b56, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_2_PASID	0x0040ULL /* b57, user error */
+#define SISL_ISTATUS_PERM_ERR_LISN_1_PASID	0x0020ULL /* b58, user error */
+#define SISL_ISTATUS_PERM_ERR_CMDROOM		0x0010ULL /* b59, user error */
+#define SISL_ISTATUS_PERM_ERR_RCB_READ		0x0008ULL /* b60, user error */
+#define SISL_ISTATUS_PERM_ERR_SA_WRITE		0x0004ULL /* b61, user error */
+#define SISL_ISTATUS_PERM_ERR_RRQ_WRITE		0x0002ULL /* b62, user error */
 	/* Page in wait accessing RCB/IOASA/RRQ is reported in b63.
 	 * Same error in data/LXT/RHT access is reported via IOASA.
 	 */
-#define SISL_ISTATUS_TEMP_ERR_PAGEIN     0x0001ULL	/* b63, can be generated
-							 * only when AFU auto
-							 * retry is disabled.
-							 * If user can determine
-							 * the command that
-							 * caused the error, it
-							 * can be retried.
-							 */
-#define SISL_ISTATUS_UNMASK  (0x001FULL)	/* 1 means unmasked */
-#define SISL_ISTATUS_MASK    ~(SISL_ISTATUS_UNMASK)	/* 1 means masked */
+#define SISL_ISTATUS_TEMP_ERR_PAGEIN		0x0001ULL /* b63, can only be
+							   * generated when AFU
+							   * auto retry is
+							   * disabled. If user
+							   * can determine the
+							   * command that caused
+							   * the error, it can
+							   * be retried.
+							   */
+#define SISL_ISTATUS_UNMASK	(0x07FFULL)		/* 1 means unmasked */
+#define SISL_ISTATUS_MASK	~(SISL_ISTATUS_UNMASK)	/* 1 means masked */
 
 	__be64 intr_clear;
 	__be64 intr_mask;
@@ -284,6 +291,7 @@ struct sisl_host_map {
 	__be64 cmd_room;
 	__be64 ctx_ctrl;	/* least significant byte or b56:63 is LISN# */
 #define SISL_CTX_CTRL_UNMAP_SECTOR	0x8000000000000000ULL /* b0 */
+#define SISL_CTX_CTRL_LISN_MASK		(0xFFULL)
 	__be64 mbox_w;		/* restricted use */
 	__be64 sq_start;	/* Submission Queue (R/W): write sequence and */
 	__be64 sq_end;		/* inclusion semantics are the same as RRQ    */
@@ -309,6 +317,10 @@ struct sisl_ctrl_map {
 #define SISL_CTX_CAP_WRITE_CMD         0x0000000000000002ULL /* afu_rc 0x21 */
 #define SISL_CTX_CAP_READ_CMD          0x0000000000000001ULL /* afu_rc 0x21 */
 	__be64 mbox_r;
+	__be64 lisn_pasid[2];
+	/* pasid _a arg must be ULL */
+#define SISL_LISN_PASID(_a, _b)	(((_a) << 32) | (_b))
+	__be64 lisn_ea[3];
 };
 
 /* single copy global regs */
@@ -415,6 +427,7 @@ struct sisl_global_regs {
 #define SISL_INTVER_CAP_RESERVED_CMD_MODE_B	0x100000000000ULL
 #define SISL_INTVER_CAP_LUN_PROVISION		0x080000000000ULL
 #define SISL_INTVER_CAP_AFU_DEBUG		0x040000000000ULL
+#define SISL_INTVER_CAP_OCXL_LISN		0x020000000000ULL
 };
 
 #define CXLFLASH_NUM_FC_PORTS_PER_BANK	2	/* fixed # of ports per bank */
diff --git a/drivers/scsi/cxlflash/superpipe.c b/drivers/scsi/cxlflash/superpipe.c
index 2fe79df5c73c..e489d89cbb45 100644
--- a/drivers/scsi/cxlflash/superpipe.c
+++ b/drivers/scsi/cxlflash/superpipe.c
@@ -14,8 +14,9 @@
 
 #include <linux/delay.h>
 #include <linux/file.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
 #include <linux/syscalls.h>
-#include <misc/cxl.h>
 #include <asm/unaligned.h>
 
 #include <scsi/scsi.h>
@@ -269,6 +270,7 @@ static int afu_attach(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)
 	int rc = 0;
 	struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
 	u64 val;
+	int i;
 
 	/* Unlock cap and restrict user to read/write cmds in translated mode */
 	readq_be(&ctrl_map->mbox_r);
@@ -282,6 +284,19 @@ static int afu_attach(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)
 		goto out;
 	}
 
+	if (afu_is_ocxl_lisn(afu)) {
+		/* Set up the LISN effective address for each interrupt */
+		for (i = 0; i < ctxi->irqs; i++) {
+			val = cfg->ops->get_irq_objhndl(ctxi->ctx, i);
+			writeq_be(val, &ctrl_map->lisn_ea[i]);
+		}
+
+		/* Use primary HWQ PASID as identifier for all interrupts */
+		val = hwq->ctx_hndl;
+		writeq_be(SISL_LISN_PASID(val, val), &ctrl_map->lisn_pasid[0]);
+		writeq_be(SISL_LISN_PASID(0UL, val), &ctrl_map->lisn_pasid[1]);
+	}
+
 	/* Set up MMIO registers pointing to the RHT */
 	writeq_be((u64)ctxi->rht_start, &ctrl_map->rht_start);
 	val = SISL_RHT_CNT_ID((u64)MAX_RHT_PER_CONTEXT, (u64)(hwq->ctx_hndl));
@@ -974,6 +989,10 @@ static int cxlflash_disk_detach(struct scsi_device *sdev,
  * theoretically never occur), every call into this routine results
  * in a complete freeing of a context.
  *
+ * Detaching the LUN is typically an ioctl() operation and the underlying
+ * code assumes that ioctl_rwsem has been acquired as a reader. To support
+ * that design point, the semaphore is acquired and released around detach.
+ *
  * Return: 0 on success
  */
 static int cxlflash_cxl_release(struct inode *inode, struct file *file)
@@ -1012,9 +1031,11 @@ static int cxlflash_cxl_release(struct inode *inode, struct file *file)
 
 	dev_dbg(dev, "%s: close for ctxid=%d\n", __func__, ctxid);
 
+	down_read(&cfg->ioctl_rwsem);
 	detach.context_id = ctxi->ctxid;
 	list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)
 		_cxlflash_disk_detach(lun_access->sdev, ctxi, &detach);
+	up_read(&cfg->ioctl_rwsem);
 out_release:
 	cfg->ops->fd_release(inode, file);
 out:
diff --git a/drivers/scsi/cxlflash/vlun.c b/drivers/scsi/cxlflash/vlun.c
index 5deef57a7834..66e445a17d6c 100644
--- a/drivers/scsi/cxlflash/vlun.c
+++ b/drivers/scsi/cxlflash/vlun.c
@@ -12,8 +12,9 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#include <linux/interrupt.h>
+#include <linux/pci.h>
 #include <linux/syscalls.h>
-#include <misc/cxl.h>
 #include <asm/unaligned.h>
 #include <asm/bitsperlong.h>
 
diff --git a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c
index 5ceea8da7bb6..37de8fb186d7 100644
--- a/drivers/scsi/dpt_i2o.c
+++ b/drivers/scsi/dpt_i2o.c
@@ -1706,7 +1706,7 @@ static int adpt_i2o_passthru(adpt_hba* pHba, u32 __user *arg)
 	u32 reply_size = 0;
 	u32 __user *user_msg = arg;
 	u32 __user * user_reply = NULL;
-	void *sg_list[pHba->sg_tablesize];
+	void **sg_list = NULL;
 	u32 sg_offset = 0;
 	u32 sg_count = 0;
 	int sg_index = 0;
@@ -1748,19 +1748,23 @@ static int adpt_i2o_passthru(adpt_hba* pHba, u32 __user *arg)
 	msg[2] = 0x40000000; // IOCTL context
 	msg[3] = adpt_ioctl_to_context(pHba, reply);
 	if (msg[3] == (u32)-1) {
-		kfree(reply);
-		return -EBUSY;
+		rcode = -EBUSY;
+		goto free;
 	}
 
-	memset(sg_list,0, sizeof(sg_list[0])*pHba->sg_tablesize);
+	sg_list = kcalloc(pHba->sg_tablesize, sizeof(*sg_list), GFP_KERNEL);
+	if (!sg_list) {
+		rcode = -ENOMEM;
+		goto free;
+	}
 	if(sg_offset) {
 		// TODO add 64 bit API
 		struct sg_simple_element *sg =  (struct sg_simple_element*) (msg+sg_offset);
 		sg_count = (size - sg_offset*4) / sizeof(struct sg_simple_element);
 		if (sg_count > pHba->sg_tablesize){
 			printk(KERN_DEBUG"%s:IOCTL SG List too large (%u)\n", pHba->name,sg_count);
-			kfree (reply);
-			return -EINVAL;
+			rcode = -EINVAL;
+			goto free;
 		}
 
 		for(i = 0; i < sg_count; i++) {
@@ -1879,7 +1883,6 @@ static int adpt_i2o_passthru(adpt_hba* pHba, u32 __user *arg)
 	if (rcode != -ETIME && rcode != -EINTR) {
 		struct sg_simple_element *sg =
 				(struct sg_simple_element*) (msg +sg_offset);
-		kfree (reply);
 		while(sg_index) {
 			if(sg_list[--sg_index]) {
 				dma_free_coherent(&pHba->pDev->dev,
@@ -1889,6 +1892,10 @@ static int adpt_i2o_passthru(adpt_hba* pHba, u32 __user *arg)
 			}
 		}
 	}
+
+free:
+	kfree(sg_list);
+	kfree(reply);
 	return rcode;
 }
 
diff --git a/drivers/scsi/esas2r/esas2r_init.c b/drivers/scsi/esas2r/esas2r_init.c
index 9dffcb28c9b7..9db645dde35e 100644
--- a/drivers/scsi/esas2r/esas2r_init.c
+++ b/drivers/scsi/esas2r/esas2r_init.c
@@ -1202,8 +1202,6 @@ static bool esas2r_format_init_msg(struct esas2r_adapter *a,
 	case ESAS2R_INIT_MSG_START:
 	case ESAS2R_INIT_MSG_REINIT:
 	{
-		struct timeval now;
-		do_gettimeofday(&now);
 		esas2r_hdebug("CFG init");
 		esas2r_build_cfg_req(a,
 				     rq,
@@ -1212,7 +1210,8 @@ static bool esas2r_format_init_msg(struct esas2r_adapter *a,
 				     NULL);
 		ci = (struct atto_vda_cfg_init *)&rq->vrq->cfg.data.init;
 		ci->sgl_page_size = cpu_to_le32(sgl_page_size);
-		ci->epoch_time = cpu_to_le32(now.tv_sec);
+		/* firmware interface overflows in y2106 */
+		ci->epoch_time = cpu_to_le32(ktime_get_real_seconds());
 		rq->flags |= RF_FAILURE_OK;
 		a->init_msg = ESAS2R_INIT_MSG_INIT;
 		break;
diff --git a/drivers/scsi/esas2r/esas2r_ioctl.c b/drivers/scsi/esas2r/esas2r_ioctl.c
index 97623002908f..34bcc8c04ff4 100644
--- a/drivers/scsi/esas2r/esas2r_ioctl.c
+++ b/drivers/scsi/esas2r/esas2r_ioctl.c
@@ -1849,7 +1849,7 @@ int esas2r_read_vda(struct esas2r_adapter *a, char *buf, long off, int count)
 		/* allocate a request */
 		rq = esas2r_alloc_request(a);
 		if (rq == NULL) {
-			esas2r_debug("esas2r_read_vda: out of requestss");
+			esas2r_debug("esas2r_read_vda: out of requests");
 			return -EBUSY;
 		}
 
diff --git a/drivers/scsi/esas2r/esas2r_main.c b/drivers/scsi/esas2r/esas2r_main.c
index e07eac5be087..c07118617d89 100644
--- a/drivers/scsi/esas2r/esas2r_main.c
+++ b/drivers/scsi/esas2r/esas2r_main.c
@@ -283,7 +283,7 @@ MODULE_PARM_DESC(num_requests,
 int num_ae_requests = 4;
 module_param(num_ae_requests, int, 0);
 MODULE_PARM_DESC(num_ae_requests,
-		 "Number of VDA asynchromous event requests.  Default 4.");
+		 "Number of VDA asynchronous event requests.  Default 4.");
 
 int cmd_per_lun = ESAS2R_DEFAULT_CMD_PER_LUN;
 module_param(cmd_per_lun, int, 0);
diff --git a/drivers/scsi/gdth.c b/drivers/scsi/gdth.c
index c35f05c4c6bb..85604795d8ee 100644
--- a/drivers/scsi/gdth.c
+++ b/drivers/scsi/gdth.c
@@ -3882,7 +3882,7 @@ static enum blk_eh_timer_return gdth_timed_out(struct scsi_cmnd *scp)
 	struct gdth_cmndinfo *cmndinfo = gdth_cmnd_priv(scp);
 	u8 b, t;
 	unsigned long flags;
-	enum blk_eh_timer_return retval = BLK_EH_NOT_HANDLED;
+	enum blk_eh_timer_return retval = BLK_EH_DONE;
 
 	TRACE(("%s() cmd 0x%x\n", scp->cmnd[0], __func__));
 	b = scp->device->channel;
diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h
index d1153e8e846b..7052a5d45f7f 100644
--- a/drivers/scsi/hisi_sas/hisi_sas.h
+++ b/drivers/scsi/hisi_sas/hisi_sas.h
@@ -136,12 +136,14 @@ struct hisi_sas_phy {
 	struct hisi_sas_port	*port;
 	struct asd_sas_phy	sas_phy;
 	struct sas_identify	identify;
+	struct completion *reset_completion;
+	spinlock_t lock;
 	u64		port_id; /* from hw */
-	u64		dev_sas_addr;
 	u64		frame_rcvd_size;
 	u8		frame_rcvd[32];
 	u8		phy_attached;
-	u8		reserved[3];
+	u8		in_reset;
+	u8		reserved[2];
 	u32		phy_type;
 	enum sas_linkrate	minimum_linkrate;
 	enum sas_linkrate	maximum_linkrate;
@@ -162,7 +164,7 @@ struct hisi_sas_cq {
 
 struct hisi_sas_dq {
 	struct hisi_hba *hisi_hba;
-	struct hisi_sas_slot	*slot_prep;
+	struct list_head list;
 	spinlock_t lock;
 	int	wr_point;
 	int	id;
@@ -174,15 +176,22 @@ struct hisi_sas_device {
 	struct completion *completion;
 	struct hisi_sas_dq	*dq;
 	struct list_head	list;
-	u64 attached_phy;
 	enum sas_device_type	dev_type;
 	int device_id;
 	int sata_idx;
 	u8 dev_status;
 };
 
+struct hisi_sas_tmf_task {
+	int force_phy;
+	int phy_id;
+	u8 tmf;
+	u16 tag_of_task_to_be_managed;
+};
+
 struct hisi_sas_slot {
 	struct list_head entry;
+	struct list_head delivery;
 	struct sas_task *task;
 	struct hisi_sas_port	*port;
 	u64	n_elem;
@@ -192,17 +201,15 @@ struct hisi_sas_slot {
 	int	cmplt_queue_slot;
 	int	idx;
 	int	abort;
+	int	ready;
 	void	*buf;
 	dma_addr_t buf_dma;
 	void	*cmd_hdr;
 	dma_addr_t cmd_hdr_dma;
 	struct work_struct abort_slot;
 	struct timer_list internal_abort_timer;
-};
-
-struct hisi_sas_tmf_task {
-	u8 tmf;
-	u16 tag_of_task_to_be_managed;
+	bool is_internal;
+	struct hisi_sas_tmf_task *tmf;
 };
 
 struct hisi_sas_hw {
@@ -215,14 +222,13 @@ struct hisi_sas_hw {
 	void (*sl_notify)(struct hisi_hba *hisi_hba, int phy_no);
 	int (*get_free_slot)(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq);
 	void (*start_delivery)(struct hisi_sas_dq *dq);
-	int (*prep_ssp)(struct hisi_hba *hisi_hba,
-			struct hisi_sas_slot *slot, int is_tmf,
-			struct hisi_sas_tmf_task *tmf);
-	int (*prep_smp)(struct hisi_hba *hisi_hba,
+	void (*prep_ssp)(struct hisi_hba *hisi_hba,
+			struct hisi_sas_slot *slot);
+	void (*prep_smp)(struct hisi_hba *hisi_hba,
 			struct hisi_sas_slot *slot);
-	int (*prep_stp)(struct hisi_hba *hisi_hba,
+	void (*prep_stp)(struct hisi_hba *hisi_hba,
 			struct hisi_sas_slot *slot);
-	int (*prep_abort)(struct hisi_hba *hisi_hba,
+	void (*prep_abort)(struct hisi_hba *hisi_hba,
 			  struct hisi_sas_slot *slot,
 			  int device_id, int abort_flag, int tag_to_abort);
 	int (*slot_complete)(struct hisi_hba *hisi_hba,
@@ -245,8 +251,11 @@ struct hisi_sas_hw {
 	u32 (*get_phys_state)(struct hisi_hba *hisi_hba);
 	int (*write_gpio)(struct hisi_hba *hisi_hba, u8 reg_type,
 				u8 reg_index, u8 reg_count, u8 *write_data);
+	void (*wait_cmds_complete_timeout)(struct hisi_hba *hisi_hba,
+					   int delay_ms, int timeout_ms);
 	int max_command_entries;
 	int complete_hdr_size;
+	struct scsi_host_template *sht;
 };
 
 struct hisi_hba {
@@ -273,6 +282,8 @@ struct hisi_hba {
 	struct workqueue_struct *wq;
 
 	int slot_index_count;
+	int last_slot_index;
+	int last_dev_id;
 	unsigned long *slot_index_tags;
 	unsigned long reject_stp_links_msk;
 
@@ -411,7 +422,7 @@ struct hisi_sas_command_table_ssp {
 	union {
 		struct {
 			struct ssp_command_iu task;
-			u32 prot[6];
+			u32 prot[7];
 		};
 		struct ssp_tmf_iu ssp_task;
 		struct xfer_rdy_iu xfer_rdy;
@@ -437,10 +448,7 @@ struct hisi_sas_slot_buf_table {
 };
 
 extern struct scsi_transport_template *hisi_sas_stt;
-extern struct scsi_host_template *hisi_sas_sht;
-
 extern void hisi_sas_stop_phys(struct hisi_hba *hisi_hba);
-extern void hisi_sas_init_add(struct hisi_hba *hisi_hba);
 extern int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost);
 extern void hisi_sas_free(struct hisi_hba *hisi_hba);
 extern u8 hisi_sas_get_ata_protocol(struct host_to_dev_fis *fis,
@@ -454,6 +462,11 @@ extern int hisi_sas_probe(struct platform_device *pdev,
 			  const struct hisi_sas_hw *ops);
 extern int hisi_sas_remove(struct platform_device *pdev);
 
+extern int hisi_sas_slave_configure(struct scsi_device *sdev);
+extern int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time);
+extern void hisi_sas_scan_start(struct Scsi_Host *shost);
+extern struct device_attribute *host_attrs[];
+extern int hisi_sas_host_reset(struct Scsi_Host *shost, int reset_type);
 extern void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy);
 extern void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba,
 				    struct sas_task *task,
@@ -465,4 +478,5 @@ extern void hisi_sas_kill_tasklets(struct hisi_hba *hisi_hba);
 extern bool hisi_sas_notify_phy_event(struct hisi_sas_phy *phy,
 				enum hisi_sas_phy_event event);
 extern void hisi_sas_release_tasks(struct hisi_hba *hisi_hba);
+extern u8 hisi_sas_get_prog_phy_linkrate_mask(enum sas_linkrate max);
 #endif
diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c
index 49c1fa643803..6f562974f8f6 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_main.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_main.c
@@ -24,6 +24,9 @@ hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,
 static int hisi_sas_softreset_ata_disk(struct domain_device *device);
 static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
 				void *funcdata);
+static void hisi_sas_release_task(struct hisi_hba *hisi_hba,
+				  struct domain_device *device);
+static void hisi_sas_dev_gone(struct domain_device *device);
 
 u8 hisi_sas_get_ata_protocol(struct host_to_dev_fis *fis, int direction)
 {
@@ -78,22 +81,23 @@ u8 hisi_sas_get_ata_protocol(struct host_to_dev_fis *fis, int direction)
 	case ATA_CMD_STANDBYNOW1:
 	case ATA_CMD_ZAC_MGMT_OUT:
 		return HISI_SAS_SATA_PROTOCOL_NONDATA;
+
+	case ATA_CMD_SET_MAX:
+		switch (fis->features) {
+		case ATA_SET_MAX_PASSWD:
+		case ATA_SET_MAX_LOCK:
+			return HISI_SAS_SATA_PROTOCOL_PIO;
+
+		case ATA_SET_MAX_PASSWD_DMA:
+		case ATA_SET_MAX_UNLOCK_DMA:
+			return HISI_SAS_SATA_PROTOCOL_DMA;
+
+		default:
+			return HISI_SAS_SATA_PROTOCOL_NONDATA;
+		}
+
 	default:
 	{
-		if (fis->command == ATA_CMD_SET_MAX) {
-			switch (fis->features) {
-			case ATA_SET_MAX_PASSWD:
-			case ATA_SET_MAX_LOCK:
-				return HISI_SAS_SATA_PROTOCOL_PIO;
-
-			case ATA_SET_MAX_PASSWD_DMA:
-			case ATA_SET_MAX_UNLOCK_DMA:
-				return HISI_SAS_SATA_PROTOCOL_DMA;
-
-			default:
-				return HISI_SAS_SATA_PROTOCOL_NONDATA;
-			}
-		}
 		if (direction == DMA_NONE)
 			return HISI_SAS_SATA_PROTOCOL_NONDATA;
 		return HISI_SAS_SATA_PROTOCOL_PIO;
@@ -134,6 +138,22 @@ int hisi_sas_get_ncq_tag(struct sas_task *task, u32 *tag)
 }
 EXPORT_SYMBOL_GPL(hisi_sas_get_ncq_tag);
 
+/*
+ * This function assumes linkrate mask fits in 8 bits, which it
+ * does for all HW versions supported.
+ */
+u8 hisi_sas_get_prog_phy_linkrate_mask(enum sas_linkrate max)
+{
+	u16 rate = 0;
+	int i;
+
+	max -= SAS_LINK_RATE_1_5_GBPS;
+	for (i = 0; i <= max; i++)
+		rate |= 1 << (i * 2);
+	return rate;
+}
+EXPORT_SYMBOL_GPL(hisi_sas_get_prog_phy_linkrate_mask);
+
 static struct hisi_hba *dev_to_hisi_hba(struct domain_device *device)
 {
 	return device->port->ha->lldd_ha;
@@ -178,11 +198,18 @@ static int hisi_sas_slot_index_alloc(struct hisi_hba *hisi_hba, int *slot_idx)
 	unsigned int index;
 	void *bitmap = hisi_hba->slot_index_tags;
 
-	index = find_first_zero_bit(bitmap, hisi_hba->slot_index_count);
-	if (index >= hisi_hba->slot_index_count)
-		return -SAS_QUEUE_FULL;
+	index = find_next_zero_bit(bitmap, hisi_hba->slot_index_count,
+			hisi_hba->last_slot_index + 1);
+	if (index >= hisi_hba->slot_index_count) {
+		index = find_next_zero_bit(bitmap, hisi_hba->slot_index_count,
+					   0);
+		if (index >= hisi_hba->slot_index_count)
+			return -SAS_QUEUE_FULL;
+	}
 	hisi_sas_slot_index_set(hisi_hba, index);
 	*slot_idx = index;
+	hisi_hba->last_slot_index = index;
+
 	return 0;
 }
 
@@ -197,6 +224,8 @@ static void hisi_sas_slot_index_init(struct hisi_hba *hisi_hba)
 void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task,
 			     struct hisi_sas_slot *slot)
 {
+	struct hisi_sas_dq *dq = &hisi_hba->dq[slot->dlvry_queue];
+	unsigned long flags;
 
 	if (task) {
 		struct device *dev = hisi_hba->dev;
@@ -216,40 +245,43 @@ void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task,
 	if (slot->buf)
 		dma_pool_free(hisi_hba->buffer_pool, slot->buf, slot->buf_dma);
 
+	spin_lock_irqsave(&dq->lock, flags);
 	list_del_init(&slot->entry);
+	spin_unlock_irqrestore(&dq->lock, flags);
 	slot->buf = NULL;
 	slot->task = NULL;
 	slot->port = NULL;
+	spin_lock_irqsave(&hisi_hba->lock, flags);
 	hisi_sas_slot_index_free(hisi_hba, slot->idx);
+	spin_unlock_irqrestore(&hisi_hba->lock, flags);
 
 	/* slot memory is fully zeroed when it is reused */
 }
 EXPORT_SYMBOL_GPL(hisi_sas_slot_task_free);
 
-static int hisi_sas_task_prep_smp(struct hisi_hba *hisi_hba,
+static void hisi_sas_task_prep_smp(struct hisi_hba *hisi_hba,
 				  struct hisi_sas_slot *slot)
 {
-	return hisi_hba->hw->prep_smp(hisi_hba, slot);
+	hisi_hba->hw->prep_smp(hisi_hba, slot);
 }
 
-static int hisi_sas_task_prep_ssp(struct hisi_hba *hisi_hba,
-				  struct hisi_sas_slot *slot, int is_tmf,
-				  struct hisi_sas_tmf_task *tmf)
+static void hisi_sas_task_prep_ssp(struct hisi_hba *hisi_hba,
+				  struct hisi_sas_slot *slot)
 {
-	return hisi_hba->hw->prep_ssp(hisi_hba, slot, is_tmf, tmf);
+	hisi_hba->hw->prep_ssp(hisi_hba, slot);
 }
 
-static int hisi_sas_task_prep_ata(struct hisi_hba *hisi_hba,
+static void hisi_sas_task_prep_ata(struct hisi_hba *hisi_hba,
 				  struct hisi_sas_slot *slot)
 {
-	return hisi_hba->hw->prep_stp(hisi_hba, slot);
+	hisi_hba->hw->prep_stp(hisi_hba, slot);
 }
 
-static int hisi_sas_task_prep_abort(struct hisi_hba *hisi_hba,
+static void hisi_sas_task_prep_abort(struct hisi_hba *hisi_hba,
 		struct hisi_sas_slot *slot,
 		int device_id, int abort_flag, int tag_to_abort)
 {
-	return hisi_hba->hw->prep_abort(hisi_hba, slot,
+	hisi_hba->hw->prep_abort(hisi_hba, slot,
 			device_id, abort_flag, tag_to_abort);
 }
 
@@ -269,7 +301,6 @@ static void hisi_sas_slot_abort(struct work_struct *work)
 	struct scsi_lun lun;
 	struct device *dev = hisi_hba->dev;
 	int tag = abort_slot->idx;
-	unsigned long flags;
 
 	if (!(task->task_proto & SAS_PROTOCOL_SSP)) {
 		dev_err(dev, "cannot abort slot for non-ssp task\n");
@@ -283,27 +314,29 @@ static void hisi_sas_slot_abort(struct work_struct *work)
 	hisi_sas_debug_issue_ssp_tmf(task->dev, lun.scsi_lun, &tmf_task);
 out:
 	/* Do cleanup for this task */
-	spin_lock_irqsave(&hisi_hba->lock, flags);
 	hisi_sas_slot_task_free(hisi_hba, task, abort_slot);
-	spin_unlock_irqrestore(&hisi_hba->lock, flags);
 	if (task->task_done)
 		task->task_done(task);
 }
 
-static int hisi_sas_task_prep(struct sas_task *task, struct hisi_sas_dq
-		*dq, int is_tmf, struct hisi_sas_tmf_task *tmf,
-		int *pass)
+static int hisi_sas_task_prep(struct sas_task *task,
+			      struct hisi_sas_dq **dq_pointer,
+			      bool is_tmf, struct hisi_sas_tmf_task *tmf,
+			      int *pass)
 {
-	struct hisi_hba *hisi_hba = dq->hisi_hba;
 	struct domain_device *device = task->dev;
+	struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 	struct hisi_sas_device *sas_dev = device->lldd_dev;
 	struct hisi_sas_port *port;
 	struct hisi_sas_slot *slot;
 	struct hisi_sas_cmd_hdr	*cmd_hdr_base;
 	struct asd_sas_port *sas_port = device->port;
 	struct device *dev = hisi_hba->dev;
-	int dlvry_queue_slot, dlvry_queue, n_elem = 0, rc, slot_idx;
-	unsigned long flags;
+	int dlvry_queue_slot, dlvry_queue, rc, slot_idx;
+	int n_elem = 0, n_elem_req = 0, n_elem_resp = 0;
+	unsigned long flags, flags_dq;
+	struct hisi_sas_dq *dq;
+	int wr_q_index;
 
 	if (!sas_port) {
 		struct task_status_struct *ts = &task->task_status;
@@ -330,6 +363,8 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_sas_dq
 		return -ECOMM;
 	}
 
+	*dq_pointer = dq = sas_dev->dq;
+
 	port = to_hisi_sas_port(sas_port);
 	if (port && !port->port_attached) {
 		dev_info(dev, "task prep: %s port%d not attach device\n",
@@ -341,6 +376,8 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_sas_dq
 	}
 
 	if (!sas_protocol_ata(task->task_proto)) {
+		unsigned int req_len, resp_len;
+
 		if (task->num_scatter) {
 			n_elem = dma_map_sg(dev, task->scatter,
 					    task->num_scatter, task->data_dir);
@@ -348,31 +385,74 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_sas_dq
 				rc = -ENOMEM;
 				goto prep_out;
 			}
+		} else if (task->task_proto & SAS_PROTOCOL_SMP) {
+			n_elem_req = dma_map_sg(dev, &task->smp_task.smp_req,
+						1, DMA_TO_DEVICE);
+			if (!n_elem_req) {
+				rc = -ENOMEM;
+				goto prep_out;
+			}
+			req_len = sg_dma_len(&task->smp_task.smp_req);
+			if (req_len & 0x3) {
+				rc = -EINVAL;
+				goto err_out_dma_unmap;
+			}
+			n_elem_resp = dma_map_sg(dev, &task->smp_task.smp_resp,
+						 1, DMA_FROM_DEVICE);
+			if (!n_elem_resp) {
+				rc = -ENOMEM;
+				goto err_out_dma_unmap;
+			}
+			resp_len = sg_dma_len(&task->smp_task.smp_resp);
+			if (resp_len & 0x3) {
+				rc = -EINVAL;
+				goto err_out_dma_unmap;
+			}
 		}
 	} else
 		n_elem = task->num_scatter;
 
+	if (n_elem > HISI_SAS_SGE_PAGE_CNT) {
+		dev_err(dev, "task prep: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT",
+			n_elem);
+		rc = -EINVAL;
+		goto err_out_dma_unmap;
+	}
+
 	spin_lock_irqsave(&hisi_hba->lock, flags);
 	if (hisi_hba->hw->slot_index_alloc)
 		rc = hisi_hba->hw->slot_index_alloc(hisi_hba, &slot_idx,
 						    device);
 	else
 		rc = hisi_sas_slot_index_alloc(hisi_hba, &slot_idx);
-	if (rc) {
-		spin_unlock_irqrestore(&hisi_hba->lock, flags);
-		goto err_out;
-	}
 	spin_unlock_irqrestore(&hisi_hba->lock, flags);
-
-	rc = hisi_hba->hw->get_free_slot(hisi_hba, dq);
 	if (rc)
-		goto err_out_tag;
+		goto err_out_dma_unmap;
 
-	dlvry_queue = dq->id;
-	dlvry_queue_slot = dq->wr_point;
 	slot = &hisi_hba->slot_info[slot_idx];
 	memset(slot, 0, sizeof(struct hisi_sas_slot));
 
+	slot->buf = dma_pool_alloc(hisi_hba->buffer_pool,
+				   GFP_ATOMIC, &slot->buf_dma);
+	if (!slot->buf) {
+		rc = -ENOMEM;
+		goto err_out_tag;
+	}
+
+	spin_lock_irqsave(&dq->lock, flags_dq);
+	wr_q_index = hisi_hba->hw->get_free_slot(hisi_hba, dq);
+	if (wr_q_index < 0) {
+		spin_unlock_irqrestore(&dq->lock, flags_dq);
+		rc = -EAGAIN;
+		goto err_out_buf;
+	}
+
+	list_add_tail(&slot->delivery, &dq->list);
+	spin_unlock_irqrestore(&dq->lock, flags_dq);
+
+	dlvry_queue = dq->id;
+	dlvry_queue_slot = wr_q_index;
+
 	slot->idx = slot_idx;
 	slot->n_elem = n_elem;
 	slot->dlvry_queue = dlvry_queue;
@@ -381,99 +461,94 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_sas_dq
 	slot->cmd_hdr = &cmd_hdr_base[dlvry_queue_slot];
 	slot->task = task;
 	slot->port = port;
+	slot->tmf = tmf;
+	slot->is_internal = is_tmf;
 	task->lldd_task = slot;
 	INIT_WORK(&slot->abort_slot, hisi_sas_slot_abort);
 
-	slot->buf = dma_pool_alloc(hisi_hba->buffer_pool,
-				   GFP_ATOMIC, &slot->buf_dma);
-	if (!slot->buf) {
-		rc = -ENOMEM;
-		goto err_out_slot_buf;
-	}
 	memset(slot->cmd_hdr, 0, sizeof(struct hisi_sas_cmd_hdr));
 	memset(hisi_sas_cmd_hdr_addr_mem(slot), 0, HISI_SAS_COMMAND_TABLE_SZ);
 	memset(hisi_sas_status_buf_addr_mem(slot), 0, HISI_SAS_STATUS_BUF_SZ);
 
 	switch (task->task_proto) {
 	case SAS_PROTOCOL_SMP:
-		rc = hisi_sas_task_prep_smp(hisi_hba, slot);
+		hisi_sas_task_prep_smp(hisi_hba, slot);
 		break;
 	case SAS_PROTOCOL_SSP:
-		rc = hisi_sas_task_prep_ssp(hisi_hba, slot, is_tmf, tmf);
+		hisi_sas_task_prep_ssp(hisi_hba, slot);
 		break;
 	case SAS_PROTOCOL_SATA:
 	case SAS_PROTOCOL_STP:
 	case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
-		rc = hisi_sas_task_prep_ata(hisi_hba, slot);
+		hisi_sas_task_prep_ata(hisi_hba, slot);
 		break;
 	default:
 		dev_err(dev, "task prep: unknown/unsupported proto (0x%x)\n",
 			task->task_proto);
-		rc = -EINVAL;
 		break;
 	}
 
-	if (rc) {
-		dev_err(dev, "task prep: rc = 0x%x\n", rc);
-		goto err_out_buf;
-	}
-
-	spin_lock_irqsave(&hisi_hba->lock, flags);
+	spin_lock_irqsave(&dq->lock, flags);
 	list_add_tail(&slot->entry, &sas_dev->list);
-	spin_unlock_irqrestore(&hisi_hba->lock, flags);
+	spin_unlock_irqrestore(&dq->lock, flags);
 	spin_lock_irqsave(&task->task_state_lock, flags);
 	task->task_state_flags |= SAS_TASK_AT_INITIATOR;
 	spin_unlock_irqrestore(&task->task_state_lock, flags);
 
-	dq->slot_prep = slot;
 	++(*pass);
+	slot->ready = 1;
 
 	return 0;
 
 err_out_buf:
 	dma_pool_free(hisi_hba->buffer_pool, slot->buf,
-		slot->buf_dma);
-err_out_slot_buf:
-	/* Nothing to be done */
+		      slot->buf_dma);
 err_out_tag:
 	spin_lock_irqsave(&hisi_hba->lock, flags);
 	hisi_sas_slot_index_free(hisi_hba, slot_idx);
 	spin_unlock_irqrestore(&hisi_hba->lock, flags);
-err_out:
-	dev_err(dev, "task prep: failed[%d]!\n", rc);
-	if (!sas_protocol_ata(task->task_proto))
-		if (n_elem)
-			dma_unmap_sg(dev, task->scatter,
-				     task->num_scatter,
-				     task->data_dir);
+err_out_dma_unmap:
+	if (!sas_protocol_ata(task->task_proto)) {
+		if (task->num_scatter) {
+			dma_unmap_sg(dev, task->scatter, task->num_scatter,
+			     task->data_dir);
+		} else if (task->task_proto & SAS_PROTOCOL_SMP) {
+			if (n_elem_req)
+				dma_unmap_sg(dev, &task->smp_task.smp_req,
+					     1, DMA_TO_DEVICE);
+			if (n_elem_resp)
+				dma_unmap_sg(dev, &task->smp_task.smp_resp,
+					     1, DMA_FROM_DEVICE);
+		}
+	}
 prep_out:
+	dev_err(dev, "task prep: failed[%d]!\n", rc);
 	return rc;
 }
 
 static int hisi_sas_task_exec(struct sas_task *task, gfp_t gfp_flags,
-			      int is_tmf, struct hisi_sas_tmf_task *tmf)
+			      bool is_tmf, struct hisi_sas_tmf_task *tmf)
 {
 	u32 rc;
 	u32 pass = 0;
 	unsigned long flags;
 	struct hisi_hba *hisi_hba = dev_to_hisi_hba(task->dev);
 	struct device *dev = hisi_hba->dev;
-	struct domain_device *device = task->dev;
-	struct hisi_sas_device *sas_dev = device->lldd_dev;
-	struct hisi_sas_dq *dq = sas_dev->dq;
+	struct hisi_sas_dq *dq = NULL;
 
 	if (unlikely(test_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags)))
 		return -EINVAL;
 
 	/* protect task_prep and start_delivery sequence */
-	spin_lock_irqsave(&dq->lock, flags);
-	rc = hisi_sas_task_prep(task, dq, is_tmf, tmf, &pass);
+	rc = hisi_sas_task_prep(task, &dq, is_tmf, tmf, &pass);
 	if (rc)
 		dev_err(dev, "task exec: failed[%d]!\n", rc);
 
-	if (likely(pass))
+	if (likely(pass)) {
+		spin_lock_irqsave(&dq->lock, flags);
 		hisi_hba->hw->start_delivery(dq);
-	spin_unlock_irqrestore(&dq->lock, flags);
+		spin_unlock_irqrestore(&dq->lock, flags);
+	}
 
 	return rc;
 }
@@ -524,10 +599,12 @@ static struct hisi_sas_device *hisi_sas_alloc_dev(struct domain_device *device)
 	struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 	struct hisi_sas_device *sas_dev = NULL;
 	unsigned long flags;
+	int last = hisi_hba->last_dev_id;
+	int first = (hisi_hba->last_dev_id + 1) % HISI_SAS_MAX_DEVICES;
 	int i;
 
 	spin_lock_irqsave(&hisi_hba->lock, flags);
-	for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
+	for (i = first; i != last; i %= HISI_SAS_MAX_DEVICES) {
 		if (hisi_hba->devices[i].dev_type == SAS_PHY_UNUSED) {
 			int queue = i % hisi_hba->queue_count;
 			struct hisi_sas_dq *dq = &hisi_hba->dq[queue];
@@ -542,18 +619,57 @@ static struct hisi_sas_device *hisi_sas_alloc_dev(struct domain_device *device)
 			INIT_LIST_HEAD(&hisi_hba->devices[i].list);
 			break;
 		}
+		i++;
 	}
+	hisi_hba->last_dev_id = i;
 	spin_unlock_irqrestore(&hisi_hba->lock, flags);
 
 	return sas_dev;
 }
 
+#define HISI_SAS_SRST_ATA_DISK_CNT 3
+static int hisi_sas_init_device(struct domain_device *device)
+{
+	int rc = TMF_RESP_FUNC_COMPLETE;
+	struct scsi_lun lun;
+	struct hisi_sas_tmf_task tmf_task;
+	int retry = HISI_SAS_SRST_ATA_DISK_CNT;
+	struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
+
+	switch (device->dev_type) {
+	case SAS_END_DEVICE:
+		int_to_scsilun(0, &lun);
+
+		tmf_task.tmf = TMF_CLEAR_TASK_SET;
+		rc = hisi_sas_debug_issue_ssp_tmf(device, lun.scsi_lun,
+						  &tmf_task);
+		if (rc == TMF_RESP_FUNC_COMPLETE)
+			hisi_sas_release_task(hisi_hba, device);
+		break;
+	case SAS_SATA_DEV:
+	case SAS_SATA_PM:
+	case SAS_SATA_PM_PORT:
+	case SAS_SATA_PENDING:
+		while (retry-- > 0) {
+			rc = hisi_sas_softreset_ata_disk(device);
+			if (!rc)
+				break;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return rc;
+}
+
 static int hisi_sas_dev_found(struct domain_device *device)
 {
 	struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 	struct domain_device *parent_dev = device->parent;
 	struct hisi_sas_device *sas_dev;
 	struct device *dev = hisi_hba->dev;
+	int rc;
 
 	if (hisi_hba->hw->alloc_dev)
 		sas_dev = hisi_hba->hw->alloc_dev(device);
@@ -576,10 +692,8 @@ static int hisi_sas_dev_found(struct domain_device *device)
 		for (phy_no = 0; phy_no < phy_num; phy_no++) {
 			phy = &parent_dev->ex_dev.ex_phy[phy_no];
 			if (SAS_ADDR(phy->attached_sas_addr) ==
-				SAS_ADDR(device->sas_addr)) {
-				sas_dev->attached_phy = phy_no;
+				SAS_ADDR(device->sas_addr))
 				break;
-			}
 		}
 
 		if (phy_no == phy_num) {
@@ -587,17 +701,25 @@ static int hisi_sas_dev_found(struct domain_device *device)
 				 "dev:%016llx at ex:%016llx\n",
 				 SAS_ADDR(device->sas_addr),
 				 SAS_ADDR(parent_dev->sas_addr));
-			return -EINVAL;
+			rc = -EINVAL;
+			goto err_out;
 		}
 	}
 
 	dev_info(dev, "dev[%d:%x] found\n",
 		sas_dev->device_id, sas_dev->dev_type);
 
+	rc = hisi_sas_init_device(device);
+	if (rc)
+		goto err_out;
 	return 0;
+
+err_out:
+	hisi_sas_dev_gone(device);
+	return rc;
 }
 
-static int hisi_sas_slave_configure(struct scsi_device *sdev)
+int hisi_sas_slave_configure(struct scsi_device *sdev)
 {
 	struct domain_device *dev = sdev_to_domain_dev(sdev);
 	int ret = sas_slave_configure(sdev);
@@ -609,15 +731,17 @@ static int hisi_sas_slave_configure(struct scsi_device *sdev)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(hisi_sas_slave_configure);
 
-static void hisi_sas_scan_start(struct Scsi_Host *shost)
+void hisi_sas_scan_start(struct Scsi_Host *shost)
 {
 	struct hisi_hba *hisi_hba = shost_priv(shost);
 
 	hisi_hba->hw->phys_init(hisi_hba);
 }
+EXPORT_SYMBOL_GPL(hisi_sas_scan_start);
 
-static int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time)
+int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time)
 {
 	struct hisi_hba *hisi_hba = shost_priv(shost);
 	struct sas_ha_struct *sha = &hisi_hba->sha;
@@ -629,6 +753,7 @@ static int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time)
 	sas_drain_work(sha);
 	return 1;
 }
+EXPORT_SYMBOL_GPL(hisi_sas_scan_finished);
 
 static void hisi_sas_phyup_work(struct work_struct *work)
 {
@@ -803,6 +928,33 @@ static int hisi_sas_queue_command(struct sas_task *task, gfp_t gfp_flags)
 	return hisi_sas_task_exec(task, gfp_flags, 0, NULL);
 }
 
+static void hisi_sas_phy_set_linkrate(struct hisi_hba *hisi_hba, int phy_no,
+			struct sas_phy_linkrates *r)
+{
+	struct sas_phy_linkrates _r;
+
+	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
+	struct asd_sas_phy *sas_phy = &phy->sas_phy;
+	enum sas_linkrate min, max;
+
+	if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
+		max = sas_phy->phy->maximum_linkrate;
+		min = r->minimum_linkrate;
+	} else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
+		max = r->maximum_linkrate;
+		min = sas_phy->phy->minimum_linkrate;
+	} else
+		return;
+
+	_r.maximum_linkrate = max;
+	_r.minimum_linkrate = min;
+
+	hisi_hba->hw->phy_disable(hisi_hba, phy_no);
+	msleep(100);
+	hisi_hba->hw->phy_set_linkrate(hisi_hba, phy_no, &_r);
+	hisi_hba->hw->phy_start(hisi_hba, phy_no);
+}
+
 static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
 				void *funcdata)
 {
@@ -826,7 +978,7 @@ static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
 		break;
 
 	case PHY_FUNC_SET_LINK_RATE:
-		hisi_hba->hw->phy_set_linkrate(hisi_hba, phy_no, funcdata);
+		hisi_sas_phy_set_linkrate(hisi_hba, phy_no, funcdata);
 		break;
 	case PHY_FUNC_GET_EVENTS:
 		if (hisi_hba->hw->get_events) {
@@ -990,7 +1142,6 @@ static int hisi_sas_softreset_ata_disk(struct domain_device *device)
 	struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 	struct device *dev = hisi_hba->dev;
 	int s = sizeof(struct host_to_dev_fis);
-	unsigned long flags;
 
 	ata_for_each_link(link, ap, EDGE) {
 		int pmp = sata_srst_pmp(link);
@@ -1015,11 +1166,8 @@ static int hisi_sas_softreset_ata_disk(struct domain_device *device)
 		dev_err(dev, "ata disk reset failed\n");
 	}
 
-	if (rc == TMF_RESP_FUNC_COMPLETE) {
-		spin_lock_irqsave(&hisi_hba->lock, flags);
+	if (rc == TMF_RESP_FUNC_COMPLETE)
 		hisi_sas_release_task(hisi_hba, device);
-		spin_unlock_irqrestore(&hisi_hba->lock, flags);
-	}
 
 	return rc;
 }
@@ -1111,12 +1259,103 @@ static void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 old_state,
 	}
 }
 
+static void hisi_sas_reset_init_all_devices(struct hisi_hba *hisi_hba)
+{
+	struct hisi_sas_device *sas_dev;
+	struct domain_device *device;
+	int i;
+
+	for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
+		sas_dev = &hisi_hba->devices[i];
+		device = sas_dev->sas_device;
+
+		if ((sas_dev->dev_type == SAS_PHY_UNUSED) || !device)
+			continue;
+
+		hisi_sas_init_device(device);
+	}
+}
+
+static void hisi_sas_send_ata_reset_each_phy(struct hisi_hba *hisi_hba,
+					     struct asd_sas_port *sas_port,
+					     struct domain_device *device)
+{
+	struct hisi_sas_tmf_task tmf_task = { .force_phy = 1 };
+	struct ata_port *ap = device->sata_dev.ap;
+	struct device *dev = hisi_hba->dev;
+	int s = sizeof(struct host_to_dev_fis);
+	int rc = TMF_RESP_FUNC_FAILED;
+	struct asd_sas_phy *sas_phy;
+	struct ata_link *link;
+	u8 fis[20] = {0};
+	u32 state;
+
+	state = hisi_hba->hw->get_phys_state(hisi_hba);
+	list_for_each_entry(sas_phy, &sas_port->phy_list, port_phy_el) {
+		if (!(state & BIT(sas_phy->id)))
+			continue;
+
+		ata_for_each_link(link, ap, EDGE) {
+			int pmp = sata_srst_pmp(link);
+
+			tmf_task.phy_id = sas_phy->id;
+			hisi_sas_fill_ata_reset_cmd(link->device, 1, pmp, fis);
+			rc = hisi_sas_exec_internal_tmf_task(device, fis, s,
+							     &tmf_task);
+			if (rc != TMF_RESP_FUNC_COMPLETE) {
+				dev_err(dev, "phy%d ata reset failed rc=%d\n",
+					sas_phy->id, rc);
+				break;
+			}
+		}
+	}
+}
+
+static void hisi_sas_terminate_stp_reject(struct hisi_hba *hisi_hba)
+{
+	struct device *dev = hisi_hba->dev;
+	int port_no, rc, i;
+
+	for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
+		struct hisi_sas_device *sas_dev = &hisi_hba->devices[i];
+		struct domain_device *device = sas_dev->sas_device;
+
+		if ((sas_dev->dev_type == SAS_PHY_UNUSED) || !device)
+			continue;
+
+		rc = hisi_sas_internal_task_abort(hisi_hba, device,
+						  HISI_SAS_INT_ABT_DEV, 0);
+		if (rc < 0)
+			dev_err(dev, "STP reject: abort dev failed %d\n", rc);
+	}
+
+	for (port_no = 0; port_no < hisi_hba->n_phy; port_no++) {
+		struct hisi_sas_port *port = &hisi_hba->port[port_no];
+		struct asd_sas_port *sas_port = &port->sas_port;
+		struct domain_device *port_dev = sas_port->port_dev;
+		struct domain_device *device;
+
+		if (!port_dev || !DEV_IS_EXPANDER(port_dev->dev_type))
+			continue;
+
+		/* Try to find a SATA device */
+		list_for_each_entry(device, &sas_port->dev_list,
+				    dev_list_node) {
+			if (dev_is_sata(device)) {
+				hisi_sas_send_ata_reset_each_phy(hisi_hba,
+								 sas_port,
+								 device);
+				break;
+			}
+		}
+	}
+}
+
 static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba)
 {
 	struct device *dev = hisi_hba->dev;
 	struct Scsi_Host *shost = hisi_hba->shost;
 	u32 old_state, state;
-	unsigned long flags;
 	int rc;
 
 	if (!hisi_hba->hw->soft_reset)
@@ -1129,6 +1368,11 @@ static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba)
 	old_state = hisi_hba->hw->get_phys_state(hisi_hba);
 
 	scsi_block_requests(shost);
+	hisi_hba->hw->wait_cmds_complete_timeout(hisi_hba, 100, 5000);
+
+	if (timer_pending(&hisi_hba->timer))
+		del_timer_sync(&hisi_hba->timer);
+
 	set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
 	rc = hisi_hba->hw->soft_reset(hisi_hba);
 	if (rc) {
@@ -1137,9 +1381,6 @@ static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba)
 		scsi_unblock_requests(shost);
 		goto out;
 	}
-	spin_lock_irqsave(&hisi_hba->lock, flags);
-	hisi_sas_release_tasks(hisi_hba);
-	spin_unlock_irqrestore(&hisi_hba->lock, flags);
 
 	clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
 
@@ -1147,6 +1388,10 @@ static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba)
 	hisi_hba->hw->phys_init(hisi_hba);
 	msleep(1000);
 	hisi_sas_refresh_port_id(hisi_hba);
+
+	if (hisi_hba->reject_stp_links_msk)
+		hisi_sas_terminate_stp_reject(hisi_hba);
+	hisi_sas_reset_init_all_devices(hisi_hba);
 	scsi_unblock_requests(shost);
 
 	state = hisi_hba->hw->get_phys_state(hisi_hba);
@@ -1165,20 +1410,25 @@ static int hisi_sas_abort_task(struct sas_task *task)
 	struct hisi_sas_tmf_task tmf_task;
 	struct domain_device *device = task->dev;
 	struct hisi_sas_device *sas_dev = device->lldd_dev;
-	struct hisi_hba *hisi_hba = dev_to_hisi_hba(task->dev);
-	struct device *dev = hisi_hba->dev;
+	struct hisi_hba *hisi_hba;
+	struct device *dev;
 	int rc = TMF_RESP_FUNC_FAILED;
 	unsigned long flags;
 
-	if (!sas_dev) {
-		dev_warn(dev, "Device has been removed\n");
+	if (!sas_dev)
 		return TMF_RESP_FUNC_FAILED;
-	}
 
+	hisi_hba = dev_to_hisi_hba(task->dev);
+	dev = hisi_hba->dev;
+
+	spin_lock_irqsave(&task->task_state_lock, flags);
 	if (task->task_state_flags & SAS_TASK_STATE_DONE) {
+		spin_unlock_irqrestore(&task->task_state_lock, flags);
 		rc = TMF_RESP_FUNC_COMPLETE;
 		goto out;
 	}
+	task->task_state_flags |= SAS_TASK_STATE_ABORTED;
+	spin_unlock_irqrestore(&task->task_state_lock, flags);
 
 	sas_dev->dev_status = HISI_SAS_DEV_EH;
 	if (task->lldd_task && task->task_proto & SAS_PROTOCOL_SSP) {
@@ -1209,11 +1459,8 @@ static int hisi_sas_abort_task(struct sas_task *task)
 		 * will have already been completed
 		 */
 		if (rc == TMF_RESP_FUNC_COMPLETE && rc2 != TMF_RESP_FUNC_SUCC) {
-			if (task->lldd_task) {
-				spin_lock_irqsave(&hisi_hba->lock, flags);
+			if (task->lldd_task)
 				hisi_sas_do_release_task(hisi_hba, task, slot);
-				spin_unlock_irqrestore(&hisi_hba->lock, flags);
-			}
 		}
 	} else if (task->task_proto & SAS_PROTOCOL_SATA ||
 		task->task_proto & SAS_PROTOCOL_STP) {
@@ -1235,11 +1482,8 @@ static int hisi_sas_abort_task(struct sas_task *task)
 		rc = hisi_sas_internal_task_abort(hisi_hba, device,
 			     HISI_SAS_INT_ABT_CMD, tag);
 		if (((rc < 0) || (rc == TMF_RESP_FUNC_FAILED)) &&
-					task->lldd_task) {
-			spin_lock_irqsave(&hisi_hba->lock, flags);
+					task->lldd_task)
 			hisi_sas_do_release_task(hisi_hba, task, slot);
-			spin_unlock_irqrestore(&hisi_hba->lock, flags);
-		}
 	}
 
 out:
@@ -1254,7 +1498,6 @@ static int hisi_sas_abort_task_set(struct domain_device *device, u8 *lun)
 	struct device *dev = hisi_hba->dev;
 	struct hisi_sas_tmf_task tmf_task;
 	int rc = TMF_RESP_FUNC_FAILED;
-	unsigned long flags;
 
 	rc = hisi_sas_internal_task_abort(hisi_hba, device,
 					HISI_SAS_INT_ABT_DEV, 0);
@@ -1267,11 +1510,8 @@ static int hisi_sas_abort_task_set(struct domain_device *device, u8 *lun)
 	tmf_task.tmf = TMF_ABORT_TASK_SET;
 	rc = hisi_sas_debug_issue_ssp_tmf(device, lun, &tmf_task);
 
-	if (rc == TMF_RESP_FUNC_COMPLETE) {
-		spin_lock_irqsave(&hisi_hba->lock, flags);
+	if (rc == TMF_RESP_FUNC_COMPLETE)
 		hisi_sas_release_task(hisi_hba, device);
-		spin_unlock_irqrestore(&hisi_hba->lock, flags);
-	}
 
 	return rc;
 }
@@ -1289,12 +1529,39 @@ static int hisi_sas_clear_aca(struct domain_device *device, u8 *lun)
 
 static int hisi_sas_debug_I_T_nexus_reset(struct domain_device *device)
 {
-	struct sas_phy *phy = sas_get_local_phy(device);
+	struct sas_phy *local_phy = sas_get_local_phy(device);
 	int rc, reset_type = (device->dev_type == SAS_SATA_DEV ||
 			(device->tproto & SAS_PROTOCOL_STP)) ? 0 : 1;
-	rc = sas_phy_reset(phy, reset_type);
-	sas_put_local_phy(phy);
-	msleep(2000);
+	struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
+	struct sas_ha_struct *sas_ha = &hisi_hba->sha;
+	struct asd_sas_phy *sas_phy = sas_ha->sas_phy[local_phy->number];
+	struct hisi_sas_phy *phy = container_of(sas_phy,
+			struct hisi_sas_phy, sas_phy);
+	DECLARE_COMPLETION_ONSTACK(phyreset);
+
+	if (scsi_is_sas_phy_local(local_phy)) {
+		phy->in_reset = 1;
+		phy->reset_completion = &phyreset;
+	}
+
+	rc = sas_phy_reset(local_phy, reset_type);
+	sas_put_local_phy(local_phy);
+
+	if (scsi_is_sas_phy_local(local_phy)) {
+		int ret = wait_for_completion_timeout(&phyreset, 2 * HZ);
+		unsigned long flags;
+
+		spin_lock_irqsave(&phy->lock, flags);
+		phy->reset_completion = NULL;
+		phy->in_reset = 0;
+		spin_unlock_irqrestore(&phy->lock, flags);
+
+		/* report PHY down if timed out */
+		if (!ret)
+			hisi_sas_phy_down(hisi_hba, sas_phy->id, 0);
+	} else
+		msleep(2000);
+
 	return rc;
 }
 
@@ -1304,7 +1571,6 @@ static int hisi_sas_I_T_nexus_reset(struct domain_device *device)
 	struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 	struct device *dev = hisi_hba->dev;
 	int rc = TMF_RESP_FUNC_FAILED;
-	unsigned long flags;
 
 	if (sas_dev->dev_status != HISI_SAS_DEV_EH)
 		return TMF_RESP_FUNC_FAILED;
@@ -1320,11 +1586,9 @@ static int hisi_sas_I_T_nexus_reset(struct domain_device *device)
 
 	rc = hisi_sas_debug_I_T_nexus_reset(device);
 
-	if ((rc == TMF_RESP_FUNC_COMPLETE) || (rc == -ENODEV)) {
-		spin_lock_irqsave(&hisi_hba->lock, flags);
+	if ((rc == TMF_RESP_FUNC_COMPLETE) || (rc == -ENODEV))
 		hisi_sas_release_task(hisi_hba, device);
-		spin_unlock_irqrestore(&hisi_hba->lock, flags);
-	}
+
 	return rc;
 }
 
@@ -1333,7 +1597,6 @@ static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun)
 	struct hisi_sas_device *sas_dev = device->lldd_dev;
 	struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 	struct device *dev = hisi_hba->dev;
-	unsigned long flags;
 	int rc = TMF_RESP_FUNC_FAILED;
 
 	sas_dev->dev_status = HISI_SAS_DEV_EH;
@@ -1353,11 +1616,8 @@ static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun)
 
 		rc = sas_phy_reset(phy, 1);
 
-		if (rc == 0) {
-			spin_lock_irqsave(&hisi_hba->lock, flags);
+		if (rc == 0)
 			hisi_sas_release_task(hisi_hba, device);
-			spin_unlock_irqrestore(&hisi_hba->lock, flags);
-		}
 		sas_put_local_phy(phy);
 	} else {
 		struct hisi_sas_tmf_task tmf_task = { .tmf =  TMF_LU_RESET };
@@ -1371,11 +1631,8 @@ static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun)
 		hisi_sas_dereg_device(hisi_hba, device);
 
 		rc = hisi_sas_debug_issue_ssp_tmf(device, lun, &tmf_task);
-		if (rc == TMF_RESP_FUNC_COMPLETE) {
-			spin_lock_irqsave(&hisi_hba->lock, flags);
+		if (rc == TMF_RESP_FUNC_COMPLETE)
 			hisi_sas_release_task(hisi_hba, device);
-			spin_unlock_irqrestore(&hisi_hba->lock, flags);
-		}
 	}
 out:
 	if (rc != TMF_RESP_FUNC_COMPLETE)
@@ -1445,7 +1702,8 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id,
 	struct hisi_sas_cmd_hdr *cmd_hdr_base;
 	struct hisi_sas_dq *dq = sas_dev->dq;
 	int dlvry_queue_slot, dlvry_queue, n_elem = 0, rc, slot_idx;
-	unsigned long flags, flags_dq;
+	unsigned long flags, flags_dq = 0;
+	int wr_q_index;
 
 	if (unlikely(test_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags)))
 		return -EINVAL;
@@ -1464,16 +1722,28 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id,
 	}
 	spin_unlock_irqrestore(&hisi_hba->lock, flags);
 
-	spin_lock_irqsave(&dq->lock, flags_dq);
-	rc = hisi_hba->hw->get_free_slot(hisi_hba, dq);
-	if (rc)
+	slot = &hisi_hba->slot_info[slot_idx];
+	memset(slot, 0, sizeof(struct hisi_sas_slot));
+
+	slot->buf = dma_pool_alloc(hisi_hba->buffer_pool,
+			GFP_ATOMIC, &slot->buf_dma);
+	if (!slot->buf) {
+		rc = -ENOMEM;
 		goto err_out_tag;
+	}
 
-	dlvry_queue = dq->id;
-	dlvry_queue_slot = dq->wr_point;
+	spin_lock_irqsave(&dq->lock, flags_dq);
+	wr_q_index = hisi_hba->hw->get_free_slot(hisi_hba, dq);
+	if (wr_q_index < 0) {
+		spin_unlock_irqrestore(&dq->lock, flags_dq);
+		rc = -EAGAIN;
+		goto err_out_buf;
+	}
+	list_add_tail(&slot->delivery, &dq->list);
+	spin_unlock_irqrestore(&dq->lock, flags_dq);
 
-	slot = &hisi_hba->slot_info[slot_idx];
-	memset(slot, 0, sizeof(struct hisi_sas_slot));
+	dlvry_queue = dq->id;
+	dlvry_queue_slot = wr_q_index;
 
 	slot->idx = slot_idx;
 	slot->n_elem = n_elem;
@@ -1483,47 +1753,36 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id,
 	slot->cmd_hdr = &cmd_hdr_base[dlvry_queue_slot];
 	slot->task = task;
 	slot->port = port;
+	slot->is_internal = true;
 	task->lldd_task = slot;
 
-	slot->buf = dma_pool_alloc(hisi_hba->buffer_pool,
-			GFP_ATOMIC, &slot->buf_dma);
-	if (!slot->buf) {
-		rc = -ENOMEM;
-		goto err_out_tag;
-	}
-
 	memset(slot->cmd_hdr, 0, sizeof(struct hisi_sas_cmd_hdr));
 	memset(hisi_sas_cmd_hdr_addr_mem(slot), 0, HISI_SAS_COMMAND_TABLE_SZ);
 	memset(hisi_sas_status_buf_addr_mem(slot), 0, HISI_SAS_STATUS_BUF_SZ);
 
-	rc = hisi_sas_task_prep_abort(hisi_hba, slot, device_id,
+	hisi_sas_task_prep_abort(hisi_hba, slot, device_id,
 				      abort_flag, task_tag);
-	if (rc)
-		goto err_out_buf;
 
-	spin_lock_irqsave(&hisi_hba->lock, flags);
-	list_add_tail(&slot->entry, &sas_dev->list);
-	spin_unlock_irqrestore(&hisi_hba->lock, flags);
 	spin_lock_irqsave(&task->task_state_lock, flags);
 	task->task_state_flags |= SAS_TASK_AT_INITIATOR;
 	spin_unlock_irqrestore(&task->task_state_lock, flags);
 
-	dq->slot_prep = slot;
-
+	slot->ready = 1;
 	/* send abort command to the chip */
+	spin_lock_irqsave(&dq->lock, flags);
+	list_add_tail(&slot->entry, &sas_dev->list);
 	hisi_hba->hw->start_delivery(dq);
-	spin_unlock_irqrestore(&dq->lock, flags_dq);
+	spin_unlock_irqrestore(&dq->lock, flags);
 
 	return 0;
 
 err_out_buf:
 	dma_pool_free(hisi_hba->buffer_pool, slot->buf,
-		slot->buf_dma);
+		      slot->buf_dma);
 err_out_tag:
 	spin_lock_irqsave(&hisi_hba->lock, flags);
 	hisi_sas_slot_index_free(hisi_hba, slot_idx);
 	spin_unlock_irqrestore(&hisi_hba->lock, flags);
-	spin_unlock_irqrestore(&dq->lock, flags_dq);
 err_out:
 	dev_err(dev, "internal abort task prep: failed[%d]!\n", rc);
 
@@ -1651,6 +1910,7 @@ void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy)
 	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 	struct asd_sas_phy *sas_phy = &phy->sas_phy;
 	struct sas_ha_struct *sas_ha = &hisi_hba->sha;
+	struct device *dev = hisi_hba->dev;
 
 	if (rdy) {
 		/* Phy down but ready */
@@ -1659,6 +1919,10 @@ void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy)
 	} else {
 		struct hisi_sas_port *port  = phy->port;
 
+		if (phy->in_reset) {
+			dev_info(dev, "ignore flutter phy%d down\n", phy_no);
+			return;
+		}
 		/* Phy down and not ready */
 		sas_ha->notify_phy_event(sas_phy, PHYE_LOSS_OF_SIGNAL);
 		sas_phy_disconnected(sas_phy);
@@ -1693,34 +1957,11 @@ EXPORT_SYMBOL_GPL(hisi_sas_kill_tasklets);
 struct scsi_transport_template *hisi_sas_stt;
 EXPORT_SYMBOL_GPL(hisi_sas_stt);
 
-static struct device_attribute *host_attrs[] = {
+struct device_attribute *host_attrs[] = {
 	&dev_attr_phy_event_threshold,
 	NULL,
 };
-
-static struct scsi_host_template _hisi_sas_sht = {
-	.module			= THIS_MODULE,
-	.name			= DRV_NAME,
-	.queuecommand		= sas_queuecommand,
-	.target_alloc		= sas_target_alloc,
-	.slave_configure	= hisi_sas_slave_configure,
-	.scan_finished		= hisi_sas_scan_finished,
-	.scan_start		= hisi_sas_scan_start,
-	.change_queue_depth	= sas_change_queue_depth,
-	.bios_param		= sas_bios_param,
-	.can_queue		= 1,
-	.this_id		= -1,
-	.sg_tablesize		= SG_ALL,
-	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
-	.use_clustering		= ENABLE_CLUSTERING,
-	.eh_device_reset_handler = sas_eh_device_reset_handler,
-	.eh_target_reset_handler = sas_eh_target_reset_handler,
-	.target_destroy		= sas_target_destroy,
-	.ioctl			= sas_ioctl,
-	.shost_attrs		= host_attrs,
-};
-struct scsi_host_template *hisi_sas_sht = &_hisi_sas_sht;
-EXPORT_SYMBOL_GPL(hisi_sas_sht);
+EXPORT_SYMBOL_GPL(host_attrs);
 
 static struct sas_domain_function_template hisi_sas_transport_ops = {
 	.lldd_dev_found		= hisi_sas_dev_found,
@@ -1798,6 +2039,7 @@ int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost)
 
 		/* Delivery queue structure */
 		spin_lock_init(&dq->lock);
+		INIT_LIST_HEAD(&dq->list);
 		dq->id = i;
 		dq->hisi_hba = hisi_hba;
 
@@ -1822,13 +2064,11 @@ int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost)
 		goto err_out;
 
 	s = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_itct);
-	hisi_hba->itct = dma_alloc_coherent(dev, s, &hisi_hba->itct_dma,
+	hisi_hba->itct = dma_zalloc_coherent(dev, s, &hisi_hba->itct_dma,
 					    GFP_KERNEL);
 	if (!hisi_hba->itct)
 		goto err_out;
 
-	memset(hisi_hba->itct, 0, s);
-
 	hisi_hba->slot_info = devm_kcalloc(dev, max_command_entries,
 					   sizeof(struct hisi_sas_slot),
 					   GFP_KERNEL);
@@ -2031,7 +2271,7 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev,
 	struct hisi_hba *hisi_hba;
 	struct device *dev = &pdev->dev;
 
-	shost = scsi_host_alloc(hisi_sas_sht, sizeof(*hisi_hba));
+	shost = scsi_host_alloc(hw->sht, sizeof(*hisi_hba));
 	if (!shost) {
 		dev_err(dev, "scsi host alloc failed\n");
 		return NULL;
@@ -2080,19 +2320,8 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev,
 	return NULL;
 }
 
-void hisi_sas_init_add(struct hisi_hba *hisi_hba)
-{
-	int i;
-
-	for (i = 0; i < hisi_hba->n_phy; i++)
-		memcpy(&hisi_hba->phy[i].dev_sas_addr,
-		       hisi_hba->sas_addr,
-		       SAS_ADDR_SIZE);
-}
-EXPORT_SYMBOL_GPL(hisi_sas_init_add);
-
 int hisi_sas_probe(struct platform_device *pdev,
-			 const struct hisi_sas_hw *hw)
+		   const struct hisi_sas_hw *hw)
 {
 	struct Scsi_Host *shost;
 	struct hisi_hba *hisi_hba;
@@ -2144,8 +2373,6 @@ int hisi_sas_probe(struct platform_device *pdev,
 		sha->sas_port[i] = &hisi_hba->port[i].sas_port;
 	}
 
-	hisi_sas_init_add(hisi_hba);
-
 	rc = scsi_add_host(shost, &pdev->dev);
 	if (rc)
 		goto err_out_ha;
@@ -2177,6 +2404,9 @@ int hisi_sas_remove(struct platform_device *pdev)
 	struct hisi_hba *hisi_hba = sha->lldd_ha;
 	struct Scsi_Host *shost = sha->core.shost;
 
+	if (timer_pending(&hisi_hba->timer))
+		del_timer(&hisi_hba->timer);
+
 	sas_unregister_ha(sha);
 	sas_remove_host(sha->core.shost);
 
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
index 84a0ccc4daf5..89ab18c1959c 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
@@ -855,39 +855,12 @@ static enum sas_linkrate phy_get_max_linkrate_v1_hw(void)
 static void phy_set_linkrate_v1_hw(struct hisi_hba *hisi_hba, int phy_no,
 		struct sas_phy_linkrates *r)
 {
-	u32 prog_phy_link_rate =
-		hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
-	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
-	struct asd_sas_phy *sas_phy = &phy->sas_phy;
-	int i;
-	enum sas_linkrate min, max;
-	u32 rate_mask = 0;
-
-	if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-		max = sas_phy->phy->maximum_linkrate;
-		min = r->minimum_linkrate;
-	} else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-		max = r->maximum_linkrate;
-		min = sas_phy->phy->minimum_linkrate;
-	} else
-		return;
-
-	sas_phy->phy->maximum_linkrate = max;
-	sas_phy->phy->minimum_linkrate = min;
-
-	max -= SAS_LINK_RATE_1_5_GBPS;
+	enum sas_linkrate max = r->maximum_linkrate;
+	u32 prog_phy_link_rate = 0x800;
 
-	for (i = 0; i <= max; i++)
-		rate_mask |= 1 << (i * 2);
-
-	prog_phy_link_rate &= ~0xff;
-	prog_phy_link_rate |= rate_mask;
-
-	disable_phy_v1_hw(hisi_hba, phy_no);
-	msleep(100);
+	prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
 	hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
-			prog_phy_link_rate);
-	start_phy_v1_hw(hisi_hba, phy_no);
+			     prog_phy_link_rate);
 }
 
 static int get_wideport_bitmap_v1_hw(struct hisi_hba *hisi_hba, int port_id)
@@ -921,37 +894,45 @@ get_free_slot_v1_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
 		return -EAGAIN;
 	}
 
-	return 0;
+	dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS;
+
+	return w;
 }
 
+/* DQ lock must be taken here */
 static void start_delivery_v1_hw(struct hisi_sas_dq *dq)
 {
 	struct hisi_hba *hisi_hba = dq->hisi_hba;
-	int dlvry_queue = dq->slot_prep->dlvry_queue;
-	int dlvry_queue_slot = dq->slot_prep->dlvry_queue_slot;
+	struct hisi_sas_slot *s, *s1;
+	struct list_head *dq_list;
+	int dlvry_queue = dq->id;
+	int wp, count = 0;
+
+	dq_list = &dq->list;
+	list_for_each_entry_safe(s, s1, &dq->list, delivery) {
+		if (!s->ready)
+			break;
+		count++;
+		wp = (s->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
+		list_del(&s->delivery);
+	}
 
-	dq->wr_point = ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS;
-	hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14),
-			 dq->wr_point);
+	if (!count)
+		return;
+
+	hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), wp);
 }
 
-static int prep_prd_sge_v1_hw(struct hisi_hba *hisi_hba,
+static void prep_prd_sge_v1_hw(struct hisi_hba *hisi_hba,
 			      struct hisi_sas_slot *slot,
 			      struct hisi_sas_cmd_hdr *hdr,
 			      struct scatterlist *scatter,
 			      int n_elem)
 {
 	struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
-	struct device *dev = hisi_hba->dev;
 	struct scatterlist *sg;
 	int i;
 
-	if (n_elem > HISI_SAS_SGE_PAGE_CNT) {
-		dev_err(dev, "prd err: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT",
-			n_elem);
-		return -EINVAL;
-	}
-
 	for_each_sg(scatter, sg, n_elem, i) {
 		struct hisi_sas_sge *entry = &sge_page->sge[i];
 
@@ -964,48 +945,25 @@ static int prep_prd_sge_v1_hw(struct hisi_hba *hisi_hba,
 	hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
 
 	hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
-
-	return 0;
 }
 
-static int prep_smp_v1_hw(struct hisi_hba *hisi_hba,
+static void prep_smp_v1_hw(struct hisi_hba *hisi_hba,
 			  struct hisi_sas_slot *slot)
 {
 	struct sas_task *task = slot->task;
 	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 	struct domain_device *device = task->dev;
-	struct device *dev = hisi_hba->dev;
 	struct hisi_sas_port *port = slot->port;
-	struct scatterlist *sg_req, *sg_resp;
+	struct scatterlist *sg_req;
 	struct hisi_sas_device *sas_dev = device->lldd_dev;
 	dma_addr_t req_dma_addr;
-	unsigned int req_len, resp_len;
-	int elem, rc;
+	unsigned int req_len;
 
-	/*
-	* DMA-map SMP request, response buffers
-	*/
 	/* req */
 	sg_req = &task->smp_task.smp_req;
-	elem = dma_map_sg(dev, sg_req, 1, DMA_TO_DEVICE);
-	if (!elem)
-		return -ENOMEM;
 	req_len = sg_dma_len(sg_req);
 	req_dma_addr = sg_dma_address(sg_req);
 
-	/* resp */
-	sg_resp = &task->smp_task.smp_resp;
-	elem = dma_map_sg(dev, sg_resp, 1, DMA_FROM_DEVICE);
-	if (!elem) {
-		rc = -ENOMEM;
-		goto err_out_req;
-	}
-	resp_len = sg_dma_len(sg_resp);
-	if ((req_len & 0x3) || (resp_len & 0x3)) {
-		rc = -EINVAL;
-		goto err_out_resp;
-	}
-
 	/* create header */
 	/* dw0 */
 	hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
@@ -1025,21 +983,10 @@ static int prep_smp_v1_hw(struct hisi_hba *hisi_hba,
 
 	hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
 	hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
-
-	return 0;
-
-err_out_resp:
-	dma_unmap_sg(dev, &slot->task->smp_task.smp_resp, 1,
-		     DMA_FROM_DEVICE);
-err_out_req:
-	dma_unmap_sg(dev, &slot->task->smp_task.smp_req, 1,
-		     DMA_TO_DEVICE);
-	return rc;
 }
 
-static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
-			  struct hisi_sas_slot *slot, int is_tmf,
-			  struct hisi_sas_tmf_task *tmf)
+static void prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
+			  struct hisi_sas_slot *slot)
 {
 	struct sas_task *task = slot->task;
 	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
@@ -1048,7 +995,8 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
 	struct hisi_sas_port *port = slot->port;
 	struct sas_ssp_task *ssp_task = &task->ssp_task;
 	struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
-	int has_data = 0, rc, priority = is_tmf;
+	struct hisi_sas_tmf_task *tmf = slot->tmf;
+	int has_data = 0, priority = !!tmf;
 	u8 *buf_cmd, fburst = 0;
 	u32 dw1, dw2;
 
@@ -1062,7 +1010,7 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
 
 	dw1 = 1 << CMD_HDR_VERIFY_DTL_OFF;
 
-	if (is_tmf) {
+	if (tmf) {
 		dw1 |= 3 << CMD_HDR_SSP_FRAME_TYPE_OFF;
 	} else {
 		switch (scsi_cmnd->sc_data_direction) {
@@ -1083,7 +1031,7 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
 	dw1 |= sas_dev->device_id << CMD_HDR_DEVICE_ID_OFF;
 	hdr->dw1 = cpu_to_le32(dw1);
 
-	if (is_tmf) {
+	if (tmf) {
 		dw2 = ((sizeof(struct ssp_tmf_iu) +
 			sizeof(struct ssp_frame_hdr)+3)/4) <<
 			CMD_HDR_CFL_OFF;
@@ -1097,12 +1045,9 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
 
 	hdr->transfer_tags = cpu_to_le32(slot->idx << CMD_HDR_IPTT_OFF);
 
-	if (has_data) {
-		rc = prep_prd_sge_v1_hw(hisi_hba, slot, hdr, task->scatter,
+	if (has_data)
+		prep_prd_sge_v1_hw(hisi_hba, slot, hdr, task->scatter,
 					slot->n_elem);
-		if (rc)
-			return rc;
-	}
 
 	hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
 	hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
@@ -1117,7 +1062,7 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
 	hdr->dw2 = cpu_to_le32(dw2);
 
 	memcpy(buf_cmd, &task->ssp_task.LUN, 8);
-	if (!is_tmf) {
+	if (!tmf) {
 		buf_cmd[9] = fburst | task->ssp_task.task_attr |
 				(task->ssp_task.task_prio << 3);
 		memcpy(buf_cmd + 12, task->ssp_task.cmd->cmnd,
@@ -1136,8 +1081,6 @@ static int prep_ssp_v1_hw(struct hisi_hba *hisi_hba,
 			break;
 		}
 	}
-
-	return 0;
 }
 
 /* by default, task resp is complete */
@@ -1430,6 +1373,7 @@ static irqreturn_t int_phyup_v1_hw(int irq_no, void *p)
 	u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
 	struct sas_identify_frame *id = (struct sas_identify_frame *)frame_rcvd;
 	irqreturn_t res = IRQ_HANDLED;
+	unsigned long flags;
 
 	irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2);
 	if (!(irq_value & CHL_INT2_SL_PHY_ENA_MSK)) {
@@ -1483,6 +1427,13 @@ static irqreturn_t int_phyup_v1_hw(int irq_no, void *p)
 			SAS_PROTOCOL_SMP;
 	hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
 
+	spin_lock_irqsave(&phy->lock, flags);
+	if (phy->reset_completion) {
+		phy->in_reset = 0;
+		complete(phy->reset_completion);
+	}
+	spin_unlock_irqrestore(&phy->lock, flags);
+
 end:
 	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2,
 			     CHL_INT2_SL_PHY_ENA_MSK);
@@ -1845,6 +1796,28 @@ static int hisi_sas_v1_init(struct hisi_hba *hisi_hba)
 	return 0;
 }
 
+static struct scsi_host_template sht_v1_hw = {
+	.name			= DRV_NAME,
+	.module			= THIS_MODULE,
+	.queuecommand		= sas_queuecommand,
+	.target_alloc		= sas_target_alloc,
+	.slave_configure	= hisi_sas_slave_configure,
+	.scan_finished		= hisi_sas_scan_finished,
+	.scan_start		= hisi_sas_scan_start,
+	.change_queue_depth	= sas_change_queue_depth,
+	.bios_param		= sas_bios_param,
+	.can_queue		= 1,
+	.this_id		= -1,
+	.sg_tablesize		= SG_ALL,
+	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
+	.use_clustering		= ENABLE_CLUSTERING,
+	.eh_device_reset_handler = sas_eh_device_reset_handler,
+	.eh_target_reset_handler = sas_eh_target_reset_handler,
+	.target_destroy		= sas_target_destroy,
+	.ioctl			= sas_ioctl,
+	.shost_attrs		= host_attrs,
+};
+
 static const struct hisi_sas_hw hisi_sas_v1_hw = {
 	.hw_init = hisi_sas_v1_init,
 	.setup_itct = setup_itct_v1_hw,
@@ -1864,6 +1837,7 @@ static const struct hisi_sas_hw hisi_sas_v1_hw = {
 	.get_wideport_bitmap = get_wideport_bitmap_v1_hw,
 	.max_command_entries = HISI_SAS_COMMAND_ENTRIES_V1_HW,
 	.complete_hdr_size = sizeof(struct hisi_sas_complete_v1_hdr),
+	.sht = &sht_v1_hw,
 };
 
 static int hisi_sas_v1_probe(struct platform_device *pdev)
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
index f89fb9a49ea9..213c530e63f2 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
@@ -144,6 +144,7 @@
 #define SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF	19
 #define SAS_ECC_INTR_MSK		0x1ec
 #define HGC_ERR_STAT_EN			0x238
+#define CQE_SEND_CNT			0x248
 #define DLVRY_Q_0_BASE_ADDR_LO		0x260
 #define DLVRY_Q_0_BASE_ADDR_HI		0x264
 #define DLVRY_Q_0_DEPTH			0x268
@@ -295,6 +296,10 @@
 #define CMD_HDR_RESP_REPORT_MSK		(0x1 << CMD_HDR_RESP_REPORT_OFF)
 #define CMD_HDR_TLR_CTRL_OFF		6
 #define CMD_HDR_TLR_CTRL_MSK		(0x3 << CMD_HDR_TLR_CTRL_OFF)
+#define CMD_HDR_PHY_ID_OFF		8
+#define CMD_HDR_PHY_ID_MSK		(0x1ff << CMD_HDR_PHY_ID_OFF)
+#define CMD_HDR_FORCE_PHY_OFF		17
+#define CMD_HDR_FORCE_PHY_MSK		(0x1 << CMD_HDR_FORCE_PHY_OFF)
 #define CMD_HDR_PORT_OFF		18
 #define CMD_HDR_PORT_MSK		(0xf << CMD_HDR_PORT_OFF)
 #define CMD_HDR_PRIORITY_OFF		27
@@ -1216,7 +1221,22 @@ static void init_reg_v2_hw(struct hisi_hba *hisi_hba)
 	}
 
 	for (i = 0; i < hisi_hba->n_phy; i++) {
-		hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE, 0x855);
+		struct hisi_sas_phy *phy = &hisi_hba->phy[i];
+		struct asd_sas_phy *sas_phy = &phy->sas_phy;
+		u32 prog_phy_link_rate = 0x800;
+
+		if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate <
+				SAS_LINK_RATE_1_5_GBPS)) {
+			prog_phy_link_rate = 0x855;
+		} else {
+			enum sas_linkrate max = sas_phy->phy->maximum_linkrate;
+
+			prog_phy_link_rate =
+				hisi_sas_get_prog_phy_linkrate_mask(max) |
+				0x800;
+		}
+		hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE,
+			prog_phy_link_rate);
 		hisi_sas_phy_write32(hisi_hba, i, SAS_PHY_CTRL, sas_phy_ctrl);
 		hisi_sas_phy_write32(hisi_hba, i, SL_TOUT_CFG, 0x7d7d7d7d);
 		hisi_sas_phy_write32(hisi_hba, i, SL_CONTROL, 0x0);
@@ -1585,39 +1605,12 @@ static enum sas_linkrate phy_get_max_linkrate_v2_hw(void)
 static void phy_set_linkrate_v2_hw(struct hisi_hba *hisi_hba, int phy_no,
 		struct sas_phy_linkrates *r)
 {
-	u32 prog_phy_link_rate =
-		hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
-	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
-	struct asd_sas_phy *sas_phy = &phy->sas_phy;
-	int i;
-	enum sas_linkrate min, max;
-	u32 rate_mask = 0;
-
-	if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-		max = sas_phy->phy->maximum_linkrate;
-		min = r->minimum_linkrate;
-	} else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-		max = r->maximum_linkrate;
-		min = sas_phy->phy->minimum_linkrate;
-	} else
-		return;
-
-	sas_phy->phy->maximum_linkrate = max;
-	sas_phy->phy->minimum_linkrate = min;
-
-	max -= SAS_LINK_RATE_1_5_GBPS;
-
-	for (i = 0; i <= max; i++)
-		rate_mask |= 1 << (i * 2);
-
-	prog_phy_link_rate &= ~0xff;
-	prog_phy_link_rate |= rate_mask;
+	enum sas_linkrate max = r->maximum_linkrate;
+	u32 prog_phy_link_rate = 0x800;
 
-	disable_phy_v2_hw(hisi_hba, phy_no);
-	msleep(100);
+	prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
 	hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
-			prog_phy_link_rate);
-	start_phy_v2_hw(hisi_hba, phy_no);
+			     prog_phy_link_rate);
 }
 
 static int get_wideport_bitmap_v2_hw(struct hisi_hba *hisi_hba, int port_id)
@@ -1658,42 +1651,50 @@ get_free_slot_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
 	r = hisi_sas_read32_relaxed(hisi_hba,
 				DLVRY_Q_0_RD_PTR + (queue * 0x14));
 	if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
-		dev_warn(dev, "full queue=%d r=%d w=%d\n\n",
+		dev_warn(dev, "full queue=%d r=%d w=%d\n",
 				queue, r, w);
 		return -EAGAIN;
 	}
 
-	return 0;
+	dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS;
+
+	return w;
 }
 
+/* DQ lock must be taken here */
 static void start_delivery_v2_hw(struct hisi_sas_dq *dq)
 {
 	struct hisi_hba *hisi_hba = dq->hisi_hba;
-	int dlvry_queue = dq->slot_prep->dlvry_queue;
-	int dlvry_queue_slot = dq->slot_prep->dlvry_queue_slot;
+	struct hisi_sas_slot *s, *s1;
+	struct list_head *dq_list;
+	int dlvry_queue = dq->id;
+	int wp, count = 0;
+
+	dq_list = &dq->list;
+	list_for_each_entry_safe(s, s1, &dq->list, delivery) {
+		if (!s->ready)
+			break;
+		count++;
+		wp = (s->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
+		list_del(&s->delivery);
+	}
+
+	if (!count)
+		return;
 
-	dq->wr_point = ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS;
-	hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14),
-			 dq->wr_point);
+	hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), wp);
 }
 
-static int prep_prd_sge_v2_hw(struct hisi_hba *hisi_hba,
+static void prep_prd_sge_v2_hw(struct hisi_hba *hisi_hba,
 			      struct hisi_sas_slot *slot,
 			      struct hisi_sas_cmd_hdr *hdr,
 			      struct scatterlist *scatter,
 			      int n_elem)
 {
 	struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
-	struct device *dev = hisi_hba->dev;
 	struct scatterlist *sg;
 	int i;
 
-	if (n_elem > HISI_SAS_SGE_PAGE_CNT) {
-		dev_err(dev, "prd err: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT",
-			n_elem);
-		return -EINVAL;
-	}
-
 	for_each_sg(scatter, sg, n_elem, i) {
 		struct hisi_sas_sge *entry = &sge_page->sge[i];
 
@@ -1706,47 +1707,24 @@ static int prep_prd_sge_v2_hw(struct hisi_hba *hisi_hba,
 	hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
 
 	hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
-
-	return 0;
 }
 
-static int prep_smp_v2_hw(struct hisi_hba *hisi_hba,
+static void prep_smp_v2_hw(struct hisi_hba *hisi_hba,
 			  struct hisi_sas_slot *slot)
 {
 	struct sas_task *task = slot->task;
 	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 	struct domain_device *device = task->dev;
-	struct device *dev = hisi_hba->dev;
 	struct hisi_sas_port *port = slot->port;
-	struct scatterlist *sg_req, *sg_resp;
+	struct scatterlist *sg_req;
 	struct hisi_sas_device *sas_dev = device->lldd_dev;
 	dma_addr_t req_dma_addr;
-	unsigned int req_len, resp_len;
-	int elem, rc;
+	unsigned int req_len;
 
-	/*
-	* DMA-map SMP request, response buffers
-	*/
 	/* req */
 	sg_req = &task->smp_task.smp_req;
-	elem = dma_map_sg(dev, sg_req, 1, DMA_TO_DEVICE);
-	if (!elem)
-		return -ENOMEM;
-	req_len = sg_dma_len(sg_req);
 	req_dma_addr = sg_dma_address(sg_req);
-
-	/* resp */
-	sg_resp = &task->smp_task.smp_resp;
-	elem = dma_map_sg(dev, sg_resp, 1, DMA_FROM_DEVICE);
-	if (!elem) {
-		rc = -ENOMEM;
-		goto err_out_req;
-	}
-	resp_len = sg_dma_len(sg_resp);
-	if ((req_len & 0x3) || (resp_len & 0x3)) {
-		rc = -EINVAL;
-		goto err_out_resp;
-	}
+	req_len = sg_dma_len(&task->smp_task.smp_req);
 
 	/* create header */
 	/* dw0 */
@@ -1768,21 +1746,10 @@ static int prep_smp_v2_hw(struct hisi_hba *hisi_hba,
 
 	hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
 	hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
-
-	return 0;
-
-err_out_resp:
-	dma_unmap_sg(dev, &slot->task->smp_task.smp_resp, 1,
-		     DMA_FROM_DEVICE);
-err_out_req:
-	dma_unmap_sg(dev, &slot->task->smp_task.smp_req, 1,
-		     DMA_TO_DEVICE);
-	return rc;
 }
 
-static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
-			  struct hisi_sas_slot *slot, int is_tmf,
-			  struct hisi_sas_tmf_task *tmf)
+static void prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
+			  struct hisi_sas_slot *slot)
 {
 	struct sas_task *task = slot->task;
 	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
@@ -1791,7 +1758,8 @@ static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
 	struct hisi_sas_port *port = slot->port;
 	struct sas_ssp_task *ssp_task = &task->ssp_task;
 	struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
-	int has_data = 0, rc, priority = is_tmf;
+	struct hisi_sas_tmf_task *tmf = slot->tmf;
+	int has_data = 0, priority = !!tmf;
 	u8 *buf_cmd;
 	u32 dw1 = 0, dw2 = 0;
 
@@ -1802,7 +1770,7 @@ static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
 			       (1 << CMD_HDR_CMD_OFF)); /* ssp */
 
 	dw1 = 1 << CMD_HDR_VDTL_OFF;
-	if (is_tmf) {
+	if (tmf) {
 		dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
 		dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
 	} else {
@@ -1833,12 +1801,9 @@ static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
 
 	hdr->transfer_tags = cpu_to_le32(slot->idx);
 
-	if (has_data) {
-		rc = prep_prd_sge_v2_hw(hisi_hba, slot, hdr, task->scatter,
+	if (has_data)
+		prep_prd_sge_v2_hw(hisi_hba, slot, hdr, task->scatter,
 					slot->n_elem);
-		if (rc)
-			return rc;
-	}
 
 	hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
 	hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
@@ -1848,7 +1813,7 @@ static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
 		sizeof(struct ssp_frame_hdr);
 
 	memcpy(buf_cmd, &task->ssp_task.LUN, 8);
-	if (!is_tmf) {
+	if (!tmf) {
 		buf_cmd[9] = task->ssp_task.task_attr |
 				(task->ssp_task.task_prio << 3);
 		memcpy(buf_cmd + 12, task->ssp_task.cmd->cmnd,
@@ -1867,8 +1832,6 @@ static int prep_ssp_v2_hw(struct hisi_hba *hisi_hba,
 			break;
 		}
 	}
-
-	return 0;
 }
 
 #define TRANS_TX_ERR	0
@@ -2380,23 +2343,24 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	struct device *dev = hisi_hba->dev;
 	struct task_status_struct *ts;
 	struct domain_device *device;
+	struct sas_ha_struct *ha;
 	enum exec_status sts;
 	struct hisi_sas_complete_v2_hdr *complete_queue =
 			hisi_hba->complete_hdr[slot->cmplt_queue];
 	struct hisi_sas_complete_v2_hdr *complete_hdr =
 			&complete_queue[slot->cmplt_queue_slot];
 	unsigned long flags;
-	int aborted;
+	bool is_internal = slot->is_internal;
 
 	if (unlikely(!task || !task->lldd_task || !task->dev))
 		return -EINVAL;
 
 	ts = &task->task_status;
 	device = task->dev;
+	ha = device->port->ha;
 	sas_dev = device->lldd_dev;
 
 	spin_lock_irqsave(&task->task_state_lock, flags);
-	aborted = task->task_state_flags & SAS_TASK_STATE_ABORTED;
 	task->task_state_flags &=
 		~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
 	spin_unlock_irqrestore(&task->task_state_lock, flags);
@@ -2404,15 +2368,6 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	memset(ts, 0, sizeof(*ts));
 	ts->resp = SAS_TASK_COMPLETE;
 
-	if (unlikely(aborted)) {
-		dev_dbg(dev, "slot_complete: task(%p) aborted\n", task);
-		ts->stat = SAS_ABORTED_TASK;
-		spin_lock_irqsave(&hisi_hba->lock, flags);
-		hisi_sas_slot_task_free(hisi_hba, task, slot);
-		spin_unlock_irqrestore(&hisi_hba->lock, flags);
-		return ts->stat;
-	}
-
 	if (unlikely(!sas_dev)) {
 		dev_dbg(dev, "slot complete: port has no device\n");
 		ts->stat = SAS_PHY_DOWN;
@@ -2459,10 +2414,10 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 			slot_err_v2_hw(hisi_hba, task, slot, 2);
 
 		if (ts->stat != SAS_DATA_UNDERRUN)
-			dev_info(dev, "erroneous completion iptt=%d task=%p "
+			dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d "
 				"CQ hdr: 0x%x 0x%x 0x%x 0x%x "
 				"Error info: 0x%x 0x%x 0x%x 0x%x\n",
-				slot->idx, task,
+				slot->idx, task, sas_dev->device_id,
 				complete_hdr->dw0, complete_hdr->dw1,
 				complete_hdr->act, complete_hdr->dw3,
 				error_info[0], error_info[1],
@@ -2523,13 +2478,27 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	}
 
 out:
+	hisi_sas_slot_task_free(hisi_hba, task, slot);
+	sts = ts->stat;
 	spin_lock_irqsave(&task->task_state_lock, flags);
+	if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
+		spin_unlock_irqrestore(&task->task_state_lock, flags);
+		dev_info(dev, "slot complete: task(%p) aborted\n", task);
+		return SAS_ABORTED_TASK;
+	}
 	task->task_state_flags |= SAS_TASK_STATE_DONE;
 	spin_unlock_irqrestore(&task->task_state_lock, flags);
-	spin_lock_irqsave(&hisi_hba->lock, flags);
-	hisi_sas_slot_task_free(hisi_hba, task, slot);
-	spin_unlock_irqrestore(&hisi_hba->lock, flags);
-	sts = ts->stat;
+
+	if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
+		spin_lock_irqsave(&device->done_lock, flags);
+		if (test_bit(SAS_HA_FROZEN, &ha->state)) {
+			spin_unlock_irqrestore(&device->done_lock, flags);
+			dev_info(dev, "slot complete: task(%p) ignored\n ",
+				 task);
+			return sts;
+		}
+		spin_unlock_irqrestore(&device->done_lock, flags);
+	}
 
 	if (task->task_done)
 		task->task_done(task);
@@ -2537,7 +2506,7 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	return sts;
 }
 
-static int prep_ata_v2_hw(struct hisi_hba *hisi_hba,
+static void prep_ata_v2_hw(struct hisi_hba *hisi_hba,
 			  struct hisi_sas_slot *slot)
 {
 	struct sas_task *task = slot->task;
@@ -2547,8 +2516,9 @@ static int prep_ata_v2_hw(struct hisi_hba *hisi_hba,
 	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 	struct asd_sas_port *sas_port = device->port;
 	struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
+	struct hisi_sas_tmf_task *tmf = slot->tmf;
 	u8 *buf_cmd;
-	int has_data = 0, rc = 0, hdr_tag = 0;
+	int has_data = 0, hdr_tag = 0;
 	u32 dw1 = 0, dw2 = 0;
 
 	/* create header */
@@ -2559,6 +2529,12 @@ static int prep_ata_v2_hw(struct hisi_hba *hisi_hba,
 	else
 		hdr->dw0 |= cpu_to_le32(4 << CMD_HDR_CMD_OFF);
 
+	if (tmf && tmf->force_phy) {
+		hdr->dw0 |= CMD_HDR_FORCE_PHY_MSK;
+		hdr->dw0 |= cpu_to_le32((1 << tmf->phy_id)
+				<< CMD_HDR_PHY_ID_OFF);
+	}
+
 	/* dw1 */
 	switch (task->data_dir) {
 	case DMA_TO_DEVICE:
@@ -2596,12 +2572,9 @@ static int prep_ata_v2_hw(struct hisi_hba *hisi_hba,
 	/* dw3 */
 	hdr->transfer_tags = cpu_to_le32(slot->idx);
 
-	if (has_data) {
-		rc = prep_prd_sge_v2_hw(hisi_hba, slot, hdr, task->scatter,
+	if (has_data)
+		prep_prd_sge_v2_hw(hisi_hba, slot, hdr, task->scatter,
 					slot->n_elem);
-		if (rc)
-			return rc;
-	}
 
 	hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
 	hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
@@ -2613,8 +2586,6 @@ static int prep_ata_v2_hw(struct hisi_hba *hisi_hba,
 		task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
 	/* fill in command FIS */
 	memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
-
-	return 0;
 }
 
 static void hisi_sas_internal_abort_quirk_timeout(struct timer_list *t)
@@ -2651,7 +2622,7 @@ static void hisi_sas_internal_abort_quirk_timeout(struct timer_list *t)
 	}
 }
 
-static int prep_abort_v2_hw(struct hisi_hba *hisi_hba,
+static void prep_abort_v2_hw(struct hisi_hba *hisi_hba,
 		struct hisi_sas_slot *slot,
 		int device_id, int abort_flag, int tag_to_abort)
 {
@@ -2679,8 +2650,6 @@ static int prep_abort_v2_hw(struct hisi_hba *hisi_hba,
 	/* dw7 */
 	hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF);
 	hdr->transfer_tags = cpu_to_le32(slot->idx);
-
-	return 0;
 }
 
 static int phy_up_v2_hw(int phy_no, struct hisi_hba *hisi_hba)
@@ -2692,6 +2661,7 @@ static int phy_up_v2_hw(int phy_no, struct hisi_hba *hisi_hba)
 	struct device *dev = hisi_hba->dev;
 	u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
 	struct sas_identify_frame *id = (struct sas_identify_frame *)frame_rcvd;
+	unsigned long flags;
 
 	hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1);
 
@@ -2744,6 +2714,12 @@ static int phy_up_v2_hw(int phy_no, struct hisi_hba *hisi_hba)
 			set_link_timer_quirk(hisi_hba);
 	}
 	hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
+	spin_lock_irqsave(&phy->lock, flags);
+	if (phy->reset_completion) {
+		phy->in_reset = 0;
+		complete(phy->reset_completion);
+	}
+	spin_unlock_irqrestore(&phy->lock, flags);
 
 end:
 	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
@@ -3151,14 +3127,12 @@ static void cq_tasklet_v2_hw(unsigned long val)
 	struct hisi_sas_complete_v2_hdr *complete_queue;
 	u32 rd_point = cq->rd_point, wr_point, dev_id;
 	int queue = cq->id;
-	struct hisi_sas_dq *dq = &hisi_hba->dq[queue];
 
 	if (unlikely(hisi_hba->reject_stp_links_msk))
 		phys_try_accept_stp_links_v2_hw(hisi_hba);
 
 	complete_queue = hisi_hba->complete_hdr[queue];
 
-	spin_lock(&dq->lock);
 	wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
 				   (0x14 * queue));
 
@@ -3208,7 +3182,6 @@ static void cq_tasklet_v2_hw(unsigned long val)
 	/* update rd_point */
 	cq->rd_point = rd_point;
 	hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
-	spin_unlock(&dq->lock);
 }
 
 static irqreturn_t cq_interrupt_v2_hw(int irq_no, void *p)
@@ -3235,6 +3208,7 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p)
 	u32 ent_tmp, ent_msk, ent_int, port_id, link_rate, hard_phy_linkrate;
 	irqreturn_t res = IRQ_HANDLED;
 	u8 attached_sas_addr[SAS_ADDR_SIZE] = {0};
+	unsigned long flags;
 	int phy_no, offset;
 
 	phy_no = sas_phy->id;
@@ -3295,6 +3269,7 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p)
 	sas_phy->oob_mode = SATA_OOB_MODE;
 	/* Make up some unique SAS address */
 	attached_sas_addr[0] = 0x50;
+	attached_sas_addr[6] = hisi_hba->shost->host_no;
 	attached_sas_addr[7] = phy_no;
 	memcpy(sas_phy->attached_sas_addr, attached_sas_addr, SAS_ADDR_SIZE);
 	memcpy(sas_phy->frame_rcvd, fis, sizeof(struct dev_to_host_fis));
@@ -3308,6 +3283,12 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p)
 	phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
 	hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
 
+	spin_lock_irqsave(&phy->lock, flags);
+	if (phy->reset_completion) {
+		phy->in_reset = 0;
+		complete(phy->reset_completion);
+	}
+	spin_unlock_irqrestore(&phy->lock, flags);
 end:
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC1 + offset, ent_tmp);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1 + offset, ent_msk);
@@ -3546,6 +3527,46 @@ static int write_gpio_v2_hw(struct hisi_hba *hisi_hba, u8 reg_type,
 	return 0;
 }
 
+static void wait_cmds_complete_timeout_v2_hw(struct hisi_hba *hisi_hba,
+					     int delay_ms, int timeout_ms)
+{
+	struct device *dev = hisi_hba->dev;
+	int entries, entries_old = 0, time;
+
+	for (time = 0; time < timeout_ms; time += delay_ms) {
+		entries = hisi_sas_read32(hisi_hba, CQE_SEND_CNT);
+		if (entries == entries_old)
+			break;
+
+		entries_old = entries;
+		msleep(delay_ms);
+	}
+
+	dev_dbg(dev, "wait commands complete %dms\n", time);
+}
+
+static struct scsi_host_template sht_v2_hw = {
+	.name			= DRV_NAME,
+	.module			= THIS_MODULE,
+	.queuecommand		= sas_queuecommand,
+	.target_alloc		= sas_target_alloc,
+	.slave_configure	= hisi_sas_slave_configure,
+	.scan_finished		= hisi_sas_scan_finished,
+	.scan_start		= hisi_sas_scan_start,
+	.change_queue_depth	= sas_change_queue_depth,
+	.bios_param		= sas_bios_param,
+	.can_queue		= 1,
+	.this_id		= -1,
+	.sg_tablesize		= SG_ALL,
+	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
+	.use_clustering		= ENABLE_CLUSTERING,
+	.eh_device_reset_handler = sas_eh_device_reset_handler,
+	.eh_target_reset_handler = sas_eh_target_reset_handler,
+	.target_destroy		= sas_target_destroy,
+	.ioctl			= sas_ioctl,
+	.shost_attrs		= host_attrs,
+};
+
 static const struct hisi_sas_hw hisi_sas_v2_hw = {
 	.hw_init = hisi_sas_v2_init,
 	.setup_itct = setup_itct_v2_hw,
@@ -3574,6 +3595,8 @@ static const struct hisi_sas_hw hisi_sas_v2_hw = {
 	.soft_reset = soft_reset_v2_hw,
 	.get_phys_state = get_phys_state_v2_hw,
 	.write_gpio = write_gpio_v2_hw,
+	.wait_cmds_complete_timeout = wait_cmds_complete_timeout_v2_hw,
+	.sht = &sht_v2_hw,
 };
 
 static int hisi_sas_v2_probe(struct platform_device *pdev)
@@ -3598,9 +3621,6 @@ static int hisi_sas_v2_remove(struct platform_device *pdev)
 	struct sas_ha_struct *sha = platform_get_drvdata(pdev);
 	struct hisi_hba *hisi_hba = sha->lldd_ha;
 
-	if (timer_pending(&hisi_hba->timer))
-		del_timer(&hisi_hba->timer);
-
 	hisi_sas_kill_tasklets(hisi_hba);
 
 	return hisi_sas_remove(pdev);
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
index 6f3e5ba6b472..9f1e2d03f914 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
@@ -92,6 +92,7 @@
 #define SAS_ECC_INTR			0x1e8
 #define SAS_ECC_INTR_MSK		0x1ec
 #define HGC_ERR_STAT_EN			0x238
+#define CQE_SEND_CNT			0x248
 #define DLVRY_Q_0_BASE_ADDR_LO		0x260
 #define DLVRY_Q_0_BASE_ADDR_HI		0x264
 #define DLVRY_Q_0_DEPTH			0x268
@@ -106,6 +107,11 @@
 #define COMPL_Q_0_RD_PTR		0x4f0
 #define AWQOS_AWCACHE_CFG	0xc84
 #define ARQOS_ARCACHE_CFG	0xc88
+#define HILINK_ERR_DFX		0xe04
+#define SAS_GPIO_CFG_0		0x1000
+#define SAS_GPIO_CFG_1		0x1004
+#define SAS_GPIO_TX_0_1	0x1040
+#define SAS_CFG_DRIVE_VLD	0x1070
 
 /* phy registers requiring init */
 #define PORT_BASE			(0x2000)
@@ -167,6 +173,7 @@
 #define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF	22
 #define CHL_INT2			(PORT_BASE + 0x1bc)
 #define CHL_INT2_SL_IDAF_TOUT_CONF_OFF	0
+#define CHL_INT2_RX_INVLD_DW_OFF	30
 #define CHL_INT2_STP_LINK_TIMEOUT_OFF	31
 #define CHL_INT0_MSK			(PORT_BASE + 0x1c0)
 #define CHL_INT1_MSK			(PORT_BASE + 0x1c4)
@@ -216,6 +223,9 @@
 #define SAS_RAS_INTR1			(RAS_BASE + 0x04)
 #define SAS_RAS_INTR0_MASK		(RAS_BASE + 0x08)
 #define SAS_RAS_INTR1_MASK		(RAS_BASE + 0x0c)
+#define CFG_SAS_RAS_INTR_MASK		(RAS_BASE + 0x1c)
+#define SAS_RAS_INTR2			(RAS_BASE + 0x20)
+#define SAS_RAS_INTR2_MASK		(RAS_BASE + 0x24)
 
 /* HW dma structures */
 /* Delivery queue header */
@@ -348,10 +358,11 @@ struct hisi_sas_err_record_v3 {
 #define DIR_TO_DEVICE 2
 #define DIR_RESERVED 3
 
-#define CMD_IS_UNCONSTRAINT(cmd) \
-	((cmd == ATA_CMD_READ_LOG_EXT) || \
-	(cmd == ATA_CMD_READ_LOG_DMA_EXT) || \
-	(cmd == ATA_CMD_DEV_RESET))
+#define FIS_CMD_IS_UNCONSTRAINED(fis) \
+	((fis.command == ATA_CMD_READ_LOG_EXT) || \
+	(fis.command == ATA_CMD_READ_LOG_DMA_EXT) || \
+	((fis.command == ATA_CMD_DEV_RESET) && \
+	((fis.control & ATA_SRST) != 0)))
 
 static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off)
 {
@@ -390,8 +401,23 @@ static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
 	return readl(regs);
 }
 
+#define hisi_sas_read32_poll_timeout(off, val, cond, delay_us,		\
+				     timeout_us)			\
+({									\
+	void __iomem *regs = hisi_hba->regs + off;			\
+	readl_poll_timeout(regs, val, cond, delay_us, timeout_us);	\
+})
+
+#define hisi_sas_read32_poll_timeout_atomic(off, val, cond, delay_us,	\
+					    timeout_us)			\
+({									\
+	void __iomem *regs = hisi_hba->regs + off;			\
+	readl_poll_timeout_atomic(regs, val, cond, delay_us, timeout_us);\
+})
+
 static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 {
+	struct pci_dev *pdev = hisi_hba->pci_dev;
 	int i;
 
 	/* Global registers init */
@@ -409,7 +435,10 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
-	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xfffe20ff);
+	if (pdev->revision >= 0x21)
+		hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffff7fff);
+	else
+		hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xfffe20ff);
 	hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
 	hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
 	hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
@@ -422,13 +451,33 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 	hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1);
 
 	for (i = 0; i < hisi_hba->n_phy; i++) {
-		hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE, 0x855);
+		struct hisi_sas_phy *phy = &hisi_hba->phy[i];
+		struct asd_sas_phy *sas_phy = &phy->sas_phy;
+		u32 prog_phy_link_rate = 0x800;
+
+		if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate <
+				SAS_LINK_RATE_1_5_GBPS)) {
+			prog_phy_link_rate = 0x855;
+		} else {
+			enum sas_linkrate max = sas_phy->phy->maximum_linkrate;
+
+			prog_phy_link_rate =
+				hisi_sas_get_prog_phy_linkrate_mask(max) |
+				0x800;
+		}
+		hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE,
+			prog_phy_link_rate);
 		hisi_sas_phy_write32(hisi_hba, i, SAS_RX_TRAIN_TIMER, 0x13e80);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
 		hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
-		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xff87ffff);
+		if (pdev->revision >= 0x21)
+			hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
+					0xffffffff);
+		else
+			hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
+					0xff87ffff);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
 		hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
 		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
@@ -503,6 +552,16 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 	/* RAS registers init */
 	hisi_sas_write32(hisi_hba, SAS_RAS_INTR0_MASK, 0x0);
 	hisi_sas_write32(hisi_hba, SAS_RAS_INTR1_MASK, 0x0);
+	hisi_sas_write32(hisi_hba, SAS_RAS_INTR2_MASK, 0x0);
+	hisi_sas_write32(hisi_hba, CFG_SAS_RAS_INTR_MASK, 0x0);
+
+	/* LED registers init */
+	hisi_sas_write32(hisi_hba, SAS_CFG_DRIVE_VLD, 0x80000ff);
+	hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1, 0x80808080);
+	hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1 + 0x4, 0x80808080);
+	/* Configure blink generator rate A to 1Hz and B to 4Hz */
+	hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_1, 0x121700);
+	hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_0, 0x800000);
 }
 
 static void config_phy_opt_mode_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
@@ -654,8 +713,8 @@ static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
 	udelay(50);
 
 	/* Ensure axi bus idle */
-	ret = readl_poll_timeout(hisi_hba->regs + AXI_CFG, val, !val,
-			20000, 1000000);
+	ret = hisi_sas_read32_poll_timeout(AXI_CFG, val, !val,
+					   20000, 1000000);
 	if (ret) {
 		dev_err(dev, "axi bus is not idle, ret = %d!\n", ret);
 		return -EIO;
@@ -794,42 +853,49 @@ get_free_slot_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
 	r = hisi_sas_read32_relaxed(hisi_hba,
 				DLVRY_Q_0_RD_PTR + (queue * 0x14));
 	if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
-		dev_warn(dev, "full queue=%d r=%d w=%d\n\n",
+		dev_warn(dev, "full queue=%d r=%d w=%d\n",
 				queue, r, w);
 		return -EAGAIN;
 	}
 
-	return 0;
+	dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS;
+
+	return w;
 }
 
 static void start_delivery_v3_hw(struct hisi_sas_dq *dq)
 {
 	struct hisi_hba *hisi_hba = dq->hisi_hba;
-	int dlvry_queue = dq->slot_prep->dlvry_queue;
-	int dlvry_queue_slot = dq->slot_prep->dlvry_queue_slot;
+	struct hisi_sas_slot *s, *s1;
+	struct list_head *dq_list;
+	int dlvry_queue = dq->id;
+	int wp, count = 0;
+
+	dq_list = &dq->list;
+	list_for_each_entry_safe(s, s1, &dq->list, delivery) {
+		if (!s->ready)
+			break;
+		count++;
+		wp = (s->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
+		list_del(&s->delivery);
+	}
+
+	if (!count)
+		return;
 
-	dq->wr_point = ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS;
-	hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14),
-			 dq->wr_point);
+	hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), wp);
 }
 
-static int prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
+static void prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
 			      struct hisi_sas_slot *slot,
 			      struct hisi_sas_cmd_hdr *hdr,
 			      struct scatterlist *scatter,
 			      int n_elem)
 {
 	struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
-	struct device *dev = hisi_hba->dev;
 	struct scatterlist *sg;
 	int i;
 
-	if (n_elem > HISI_SAS_SGE_PAGE_CNT) {
-		dev_err(dev, "prd err: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT",
-			n_elem);
-		return -EINVAL;
-	}
-
 	for_each_sg(scatter, sg, n_elem, i) {
 		struct hisi_sas_sge *entry = &sge_page->sge[i];
 
@@ -842,13 +908,10 @@ static int prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
 	hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
 
 	hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
-
-	return 0;
 }
 
-static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
-			  struct hisi_sas_slot *slot, int is_tmf,
-			  struct hisi_sas_tmf_task *tmf)
+static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
+			  struct hisi_sas_slot *slot)
 {
 	struct sas_task *task = slot->task;
 	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
@@ -857,7 +920,8 @@ static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
 	struct hisi_sas_port *port = slot->port;
 	struct sas_ssp_task *ssp_task = &task->ssp_task;
 	struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
-	int has_data = 0, rc, priority = is_tmf;
+	struct hisi_sas_tmf_task *tmf = slot->tmf;
+	int has_data = 0, priority = !!tmf;
 	u8 *buf_cmd;
 	u32 dw1 = 0, dw2 = 0;
 
@@ -868,7 +932,7 @@ static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
 			       (1 << CMD_HDR_CMD_OFF)); /* ssp */
 
 	dw1 = 1 << CMD_HDR_VDTL_OFF;
-	if (is_tmf) {
+	if (tmf) {
 		dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
 		dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
 	} else {
@@ -898,12 +962,9 @@ static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
 	hdr->dw2 = cpu_to_le32(dw2);
 	hdr->transfer_tags = cpu_to_le32(slot->idx);
 
-	if (has_data) {
-		rc = prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
+	if (has_data)
+		prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
 					slot->n_elem);
-		if (rc)
-			return rc;
-	}
 
 	hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
 	hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
@@ -913,7 +974,7 @@ static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
 		sizeof(struct ssp_frame_hdr);
 
 	memcpy(buf_cmd, &task->ssp_task.LUN, 8);
-	if (!is_tmf) {
+	if (!tmf) {
 		buf_cmd[9] = ssp_task->task_attr | (ssp_task->task_prio << 3);
 		memcpy(buf_cmd + 12, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
 	} else {
@@ -930,48 +991,25 @@ static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
 			break;
 		}
 	}
-
-	return 0;
 }
 
-static int prep_smp_v3_hw(struct hisi_hba *hisi_hba,
+static void prep_smp_v3_hw(struct hisi_hba *hisi_hba,
 			  struct hisi_sas_slot *slot)
 {
 	struct sas_task *task = slot->task;
 	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 	struct domain_device *device = task->dev;
-	struct device *dev = hisi_hba->dev;
 	struct hisi_sas_port *port = slot->port;
-	struct scatterlist *sg_req, *sg_resp;
+	struct scatterlist *sg_req;
 	struct hisi_sas_device *sas_dev = device->lldd_dev;
 	dma_addr_t req_dma_addr;
-	unsigned int req_len, resp_len;
-	int elem, rc;
+	unsigned int req_len;
 
-	/*
-	 * DMA-map SMP request, response buffers
-	 */
 	/* req */
 	sg_req = &task->smp_task.smp_req;
-	elem = dma_map_sg(dev, sg_req, 1, DMA_TO_DEVICE);
-	if (!elem)
-		return -ENOMEM;
 	req_len = sg_dma_len(sg_req);
 	req_dma_addr = sg_dma_address(sg_req);
 
-	/* resp */
-	sg_resp = &task->smp_task.smp_resp;
-	elem = dma_map_sg(dev, sg_resp, 1, DMA_FROM_DEVICE);
-	if (!elem) {
-		rc = -ENOMEM;
-		goto err_out_req;
-	}
-	resp_len = sg_dma_len(sg_resp);
-	if ((req_len & 0x3) || (resp_len & 0x3)) {
-		rc = -EINVAL;
-		goto err_out_resp;
-	}
-
 	/* create header */
 	/* dw0 */
 	hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
@@ -993,18 +1031,9 @@ static int prep_smp_v3_hw(struct hisi_hba *hisi_hba,
 	hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
 	hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
 
-	return 0;
-
-err_out_resp:
-	dma_unmap_sg(dev, &slot->task->smp_task.smp_resp, 1,
-		     DMA_FROM_DEVICE);
-err_out_req:
-	dma_unmap_sg(dev, &slot->task->smp_task.smp_req, 1,
-		     DMA_TO_DEVICE);
-	return rc;
 }
 
-static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
+static void prep_ata_v3_hw(struct hisi_hba *hisi_hba,
 			  struct hisi_sas_slot *slot)
 {
 	struct sas_task *task = slot->task;
@@ -1015,7 +1044,7 @@ static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
 	struct asd_sas_port *sas_port = device->port;
 	struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
 	u8 *buf_cmd;
-	int has_data = 0, rc = 0, hdr_tag = 0;
+	int has_data = 0, hdr_tag = 0;
 	u32 dw1 = 0, dw2 = 0;
 
 	hdr->dw0 = cpu_to_le32(port->id << CMD_HDR_PORT_OFF);
@@ -1046,7 +1075,7 @@ static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
 		<< CMD_HDR_FRAME_TYPE_OFF;
 	dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
 
-	if (CMD_IS_UNCONSTRAINT(task->ata_task.fis.command))
+	if (FIS_CMD_IS_UNCONSTRAINED(task->ata_task.fis))
 		dw1 |= 1 << CMD_HDR_UNCON_CMD_OFF;
 
 	hdr->dw1 = cpu_to_le32(dw1);
@@ -1064,12 +1093,9 @@ static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
 	/* dw3 */
 	hdr->transfer_tags = cpu_to_le32(slot->idx);
 
-	if (has_data) {
-		rc = prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
+	if (has_data)
+		prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
 					slot->n_elem);
-		if (rc)
-			return rc;
-	}
 
 	hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
 	hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
@@ -1081,11 +1107,9 @@ static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
 		task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
 	/* fill in command FIS */
 	memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
-
-	return 0;
 }
 
-static int prep_abort_v3_hw(struct hisi_hba *hisi_hba,
+static void prep_abort_v3_hw(struct hisi_hba *hisi_hba,
 		struct hisi_sas_slot *slot,
 		int device_id, int abort_flag, int tag_to_abort)
 {
@@ -1110,7 +1134,6 @@ static int prep_abort_v3_hw(struct hisi_hba *hisi_hba,
 	hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF);
 	hdr->transfer_tags = cpu_to_le32(slot->idx);
 
-	return 0;
 }
 
 static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
@@ -1120,6 +1143,7 @@ static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
 	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 	struct asd_sas_phy *sas_phy = &phy->sas_phy;
 	struct device *dev = hisi_hba->dev;
+	unsigned long flags;
 
 	hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1);
 
@@ -1188,6 +1212,12 @@ static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
 	phy->phy_attached = 1;
 	hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
 	res = IRQ_HANDLED;
+	spin_lock_irqsave(&phy->lock, flags);
+	if (phy->reset_completion) {
+		phy->in_reset = 0;
+		complete(phy->reset_completion);
+	}
+	spin_unlock_irqrestore(&phy->lock, flags);
 end:
 	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
 			     CHL_INT0_SL_PHY_ENABLE_MSK);
@@ -1301,14 +1331,10 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
 {
 	struct hisi_hba *hisi_hba = p;
 	struct device *dev = hisi_hba->dev;
-	u32 ent_msk, ent_tmp, irq_msk;
+	struct pci_dev *pci_dev = hisi_hba->pci_dev;
+	u32 irq_msk;
 	int phy_no = 0;
 
-	ent_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
-	ent_tmp = ent_msk;
-	ent_msk |= ENT_INT_SRC_MSK3_ENT95_MSK_MSK;
-	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, ent_msk);
-
 	irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
 				& 0xeeeeeeee;
 
@@ -1319,6 +1345,13 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
 						     CHL_INT1);
 		u32 irq_value2 = hisi_sas_phy_read32(hisi_hba, phy_no,
 						     CHL_INT2);
+		u32 irq_msk1 = hisi_sas_phy_read32(hisi_hba, phy_no,
+							CHL_INT1_MSK);
+		u32 irq_msk2 = hisi_sas_phy_read32(hisi_hba, phy_no,
+							CHL_INT2_MSK);
+
+		irq_value1 &= ~irq_msk1;
+		irq_value2 &= ~irq_msk2;
 
 		if ((irq_msk & (4 << (phy_no * 4))) &&
 						irq_value1) {
@@ -1364,8 +1397,28 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
 
 			hisi_sas_phy_write32(hisi_hba, phy_no,
 					     CHL_INT2, irq_value2);
-		}
 
+			if ((irq_value2 & BIT(CHL_INT2_RX_INVLD_DW_OFF)) &&
+			    (pci_dev->revision == 0x20)) {
+				u32 reg_value;
+				int rc;
+
+				rc = hisi_sas_read32_poll_timeout_atomic(
+					HILINK_ERR_DFX, reg_value,
+					!((reg_value >> 8) & BIT(phy_no)),
+					1000, 10000);
+				if (rc) {
+					disable_phy_v3_hw(hisi_hba, phy_no);
+					hisi_sas_phy_write32(hisi_hba, phy_no,
+						CHL_INT2,
+						BIT(CHL_INT2_RX_INVLD_DW_OFF));
+					hisi_sas_phy_read32(hisi_hba, phy_no,
+						ERR_CNT_INVLD_DW);
+					mdelay(1);
+					enable_phy_v3_hw(hisi_hba, phy_no);
+				}
+			}
+		}
 
 		if (irq_msk & (2 << (phy_no * 4)) && irq_value0) {
 			hisi_sas_phy_write32(hisi_hba, phy_no,
@@ -1378,8 +1431,6 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
 		phy_no++;
 	}
 
-	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, ent_tmp);
-
 	return IRQ_HANDLED;
 }
 
@@ -1448,6 +1499,7 @@ static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk | 0x1df00);
 
 	irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
+	irq_value &= ~irq_msk;
 
 	for (i = 0; i < ARRAY_SIZE(fatal_axi_error); i++) {
 		const struct hisi_sas_hw_error *error = &fatal_axi_error[i];
@@ -1549,37 +1601,30 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	struct device *dev = hisi_hba->dev;
 	struct task_status_struct *ts;
 	struct domain_device *device;
+	struct sas_ha_struct *ha;
 	enum exec_status sts;
 	struct hisi_sas_complete_v3_hdr *complete_queue =
 			hisi_hba->complete_hdr[slot->cmplt_queue];
 	struct hisi_sas_complete_v3_hdr *complete_hdr =
 			&complete_queue[slot->cmplt_queue_slot];
-	int aborted;
 	unsigned long flags;
+	bool is_internal = slot->is_internal;
 
 	if (unlikely(!task || !task->lldd_task || !task->dev))
 		return -EINVAL;
 
 	ts = &task->task_status;
 	device = task->dev;
+	ha = device->port->ha;
 	sas_dev = device->lldd_dev;
 
 	spin_lock_irqsave(&task->task_state_lock, flags);
-	aborted = task->task_state_flags & SAS_TASK_STATE_ABORTED;
 	task->task_state_flags &=
 		~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
 	spin_unlock_irqrestore(&task->task_state_lock, flags);
 
 	memset(ts, 0, sizeof(*ts));
 	ts->resp = SAS_TASK_COMPLETE;
-	if (unlikely(aborted)) {
-		dev_dbg(dev, "slot complete: task(%p) aborted\n", task);
-		ts->stat = SAS_ABORTED_TASK;
-		spin_lock_irqsave(&hisi_hba->lock, flags);
-		hisi_sas_slot_task_free(hisi_hba, task, slot);
-		spin_unlock_irqrestore(&hisi_hba->lock, flags);
-		return ts->stat;
-	}
 
 	if (unlikely(!sas_dev)) {
 		dev_dbg(dev, "slot complete: port has not device\n");
@@ -1619,10 +1664,10 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 
 		slot_err_v3_hw(hisi_hba, task, slot);
 		if (ts->stat != SAS_DATA_UNDERRUN)
-			dev_info(dev, "erroneous completion iptt=%d task=%p "
+			dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d "
 				"CQ hdr: 0x%x 0x%x 0x%x 0x%x "
 				"Error info: 0x%x 0x%x 0x%x 0x%x\n",
-				slot->idx, task,
+				slot->idx, task, sas_dev->device_id,
 				complete_hdr->dw0, complete_hdr->dw1,
 				complete_hdr->act, complete_hdr->dw3,
 				error_info[0], error_info[1],
@@ -1677,13 +1722,27 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	}
 
 out:
+	hisi_sas_slot_task_free(hisi_hba, task, slot);
+	sts = ts->stat;
 	spin_lock_irqsave(&task->task_state_lock, flags);
+	if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
+		spin_unlock_irqrestore(&task->task_state_lock, flags);
+		dev_info(dev, "slot complete: task(%p) aborted\n", task);
+		return SAS_ABORTED_TASK;
+	}
 	task->task_state_flags |= SAS_TASK_STATE_DONE;
 	spin_unlock_irqrestore(&task->task_state_lock, flags);
-	spin_lock_irqsave(&hisi_hba->lock, flags);
-	hisi_sas_slot_task_free(hisi_hba, task, slot);
-	spin_unlock_irqrestore(&hisi_hba->lock, flags);
-	sts = ts->stat;
+
+	if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
+		spin_lock_irqsave(&device->done_lock, flags);
+		if (test_bit(SAS_HA_FROZEN, &ha->state)) {
+			spin_unlock_irqrestore(&device->done_lock, flags);
+			dev_info(dev, "slot complete: task(%p) ignored\n ",
+				 task);
+			return sts;
+		}
+		spin_unlock_irqrestore(&device->done_lock, flags);
+	}
 
 	if (task->task_done)
 		task->task_done(task);
@@ -1699,25 +1758,27 @@ static void cq_tasklet_v3_hw(unsigned long val)
 	struct hisi_sas_complete_v3_hdr *complete_queue;
 	u32 rd_point = cq->rd_point, wr_point;
 	int queue = cq->id;
-	struct hisi_sas_dq *dq = &hisi_hba->dq[queue];
 
 	complete_queue = hisi_hba->complete_hdr[queue];
 
-	spin_lock(&dq->lock);
 	wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
 				   (0x14 * queue));
 
 	while (rd_point != wr_point) {
 		struct hisi_sas_complete_v3_hdr *complete_hdr;
+		struct device *dev = hisi_hba->dev;
 		int iptt;
 
 		complete_hdr = &complete_queue[rd_point];
 
 		iptt = (complete_hdr->dw1) & CMPLT_HDR_IPTT_MSK;
-		slot = &hisi_hba->slot_info[iptt];
-		slot->cmplt_queue_slot = rd_point;
-		slot->cmplt_queue = queue;
-		slot_complete_v3_hw(hisi_hba, slot);
+		if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) {
+			slot = &hisi_hba->slot_info[iptt];
+			slot->cmplt_queue_slot = rd_point;
+			slot->cmplt_queue = queue;
+			slot_complete_v3_hw(hisi_hba, slot);
+		} else
+			dev_err(dev, "IPTT %d is invalid, discard it.\n", iptt);
 
 		if (++rd_point >= HISI_SAS_QUEUE_SLOTS)
 			rd_point = 0;
@@ -1726,7 +1787,6 @@ static void cq_tasklet_v3_hw(unsigned long val)
 	/* update rd_point */
 	cq->rd_point = rd_point;
 	hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
-	spin_unlock(&dq->lock);
 }
 
 static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
@@ -1839,39 +1899,12 @@ static int hisi_sas_v3_init(struct hisi_hba *hisi_hba)
 static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no,
 		struct sas_phy_linkrates *r)
 {
-	u32 prog_phy_link_rate =
-		hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
-	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
-	struct asd_sas_phy *sas_phy = &phy->sas_phy;
-	int i;
-	enum sas_linkrate min, max;
-	u32 rate_mask = 0;
-
-	if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-		max = sas_phy->phy->maximum_linkrate;
-		min = r->minimum_linkrate;
-	} else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
-		max = r->maximum_linkrate;
-		min = sas_phy->phy->minimum_linkrate;
-	} else
-		return;
-
-	sas_phy->phy->maximum_linkrate = max;
-	sas_phy->phy->minimum_linkrate = min;
-
-	max -= SAS_LINK_RATE_1_5_GBPS;
-
-	for (i = 0; i <= max; i++)
-		rate_mask |= 1 << (i * 2);
-
-	prog_phy_link_rate &= ~0xff;
-	prog_phy_link_rate |= rate_mask;
+	enum sas_linkrate max = r->maximum_linkrate;
+	u32 prog_phy_link_rate = 0x800;
 
-	disable_phy_v3_hw(hisi_hba, phy_no);
-	msleep(100);
+	prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
 	hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
-			prog_phy_link_rate);
-	start_phy_v3_hw(hisi_hba, phy_no);
+			     prog_phy_link_rate);
 }
 
 static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba)
@@ -1948,8 +1981,9 @@ static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
 	hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE + AM_CTRL_GLOBAL, 0x1);
 
 	/* wait until bus idle */
-	rc = readl_poll_timeout(hisi_hba->regs + AXI_MASTER_CFG_BASE +
-		AM_CURR_TRANS_RETURN, status, status == 0x3, 10, 100);
+	rc = hisi_sas_read32_poll_timeout(AXI_MASTER_CFG_BASE +
+					  AM_CURR_TRANS_RETURN, status,
+					  status == 0x3, 10, 100);
 	if (rc) {
 		dev_err(dev, "axi bus is not idle, rc = %d\n", rc);
 		return rc;
@@ -1960,6 +1994,75 @@ static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
 	return hw_init_v3_hw(hisi_hba);
 }
 
+static int write_gpio_v3_hw(struct hisi_hba *hisi_hba, u8 reg_type,
+			u8 reg_index, u8 reg_count, u8 *write_data)
+{
+	struct device *dev = hisi_hba->dev;
+	u32 *data = (u32 *)write_data;
+	int i;
+
+	switch (reg_type) {
+	case SAS_GPIO_REG_TX:
+		if ((reg_index + reg_count) > ((hisi_hba->n_phy + 3) / 4)) {
+			dev_err(dev, "write gpio: invalid reg range[%d, %d]\n",
+				reg_index, reg_index + reg_count - 1);
+			return -EINVAL;
+		}
+
+		for (i = 0; i < reg_count; i++)
+			hisi_sas_write32(hisi_hba,
+					 SAS_GPIO_TX_0_1 + (reg_index + i) * 4,
+					 data[i]);
+		break;
+	default:
+		dev_err(dev, "write gpio: unsupported or bad reg type %d\n",
+				reg_type);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void wait_cmds_complete_timeout_v3_hw(struct hisi_hba *hisi_hba,
+					     int delay_ms, int timeout_ms)
+{
+	struct device *dev = hisi_hba->dev;
+	int entries, entries_old = 0, time;
+
+	for (time = 0; time < timeout_ms; time += delay_ms) {
+		entries = hisi_sas_read32(hisi_hba, CQE_SEND_CNT);
+		if (entries == entries_old)
+			break;
+
+		entries_old = entries;
+		msleep(delay_ms);
+	}
+
+	dev_dbg(dev, "wait commands complete %dms\n", time);
+}
+
+static struct scsi_host_template sht_v3_hw = {
+	.name			= DRV_NAME,
+	.module			= THIS_MODULE,
+	.queuecommand		= sas_queuecommand,
+	.target_alloc		= sas_target_alloc,
+	.slave_configure	= hisi_sas_slave_configure,
+	.scan_finished		= hisi_sas_scan_finished,
+	.scan_start		= hisi_sas_scan_start,
+	.change_queue_depth	= sas_change_queue_depth,
+	.bios_param		= sas_bios_param,
+	.can_queue		= 1,
+	.this_id		= -1,
+	.sg_tablesize		= SG_ALL,
+	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
+	.use_clustering		= ENABLE_CLUSTERING,
+	.eh_device_reset_handler = sas_eh_device_reset_handler,
+	.eh_target_reset_handler = sas_eh_target_reset_handler,
+	.target_destroy		= sas_target_destroy,
+	.ioctl			= sas_ioctl,
+	.shost_attrs		= host_attrs,
+};
+
 static const struct hisi_sas_hw hisi_sas_v3_hw = {
 	.hw_init = hisi_sas_v3_init,
 	.setup_itct = setup_itct_v3_hw,
@@ -1985,6 +2088,8 @@ static const struct hisi_sas_hw hisi_sas_v3_hw = {
 	.soft_reset = soft_reset_v3_hw,
 	.get_phys_state = get_phys_state_v3_hw,
 	.get_events = phy_get_events_v3_hw,
+	.write_gpio = write_gpio_v3_hw,
+	.wait_cmds_complete_timeout = wait_cmds_complete_timeout_v3_hw,
 };
 
 static struct Scsi_Host *
@@ -1994,7 +2099,7 @@ hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
 	struct hisi_hba *hisi_hba;
 	struct device *dev = &pdev->dev;
 
-	shost = scsi_host_alloc(hisi_sas_sht, sizeof(*hisi_hba));
+	shost = scsi_host_alloc(&sht_v3_hw, sizeof(*hisi_hba));
 	if (!shost) {
 		dev_err(dev, "shost alloc failed\n");
 		return NULL;
@@ -2108,8 +2213,6 @@ hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		sha->sas_port[i] = &hisi_hba->port[i].sas_port;
 	}
 
-	hisi_sas_init_add(hisi_hba);
-
 	rc = scsi_add_host(shost, dev);
 	if (rc)
 		goto err_out_ha;
@@ -2161,6 +2264,9 @@ static void hisi_sas_v3_remove(struct pci_dev *pdev)
 	struct hisi_hba *hisi_hba = sha->lldd_ha;
 	struct Scsi_Host *shost = sha->core.shost;
 
+	if (timer_pending(&hisi_hba->timer))
+		del_timer(&hisi_hba->timer);
+
 	sas_unregister_ha(sha);
 	sas_remove_host(sha->core.shost);
 
@@ -2222,6 +2328,29 @@ static const struct hisi_sas_hw_error sas_ras_intr1_nfe[] = {
 	{ .irq_msk = BIT(31), .msg = "DMAC7_RX_POISON" },
 };
 
+static const struct hisi_sas_hw_error sas_ras_intr2_nfe[] = {
+	{ .irq_msk = BIT(0), .msg = "DMAC0_AXI_BUS_ERR" },
+	{ .irq_msk = BIT(1), .msg = "DMAC1_AXI_BUS_ERR" },
+	{ .irq_msk = BIT(2), .msg = "DMAC2_AXI_BUS_ERR" },
+	{ .irq_msk = BIT(3), .msg = "DMAC3_AXI_BUS_ERR" },
+	{ .irq_msk = BIT(4), .msg = "DMAC4_AXI_BUS_ERR" },
+	{ .irq_msk = BIT(5), .msg = "DMAC5_AXI_BUS_ERR" },
+	{ .irq_msk = BIT(6), .msg = "DMAC6_AXI_BUS_ERR" },
+	{ .irq_msk = BIT(7), .msg = "DMAC7_AXI_BUS_ERR" },
+	{ .irq_msk = BIT(8), .msg = "DMAC0_FIFO_OMIT_ERR" },
+	{ .irq_msk = BIT(9), .msg = "DMAC1_FIFO_OMIT_ERR" },
+	{ .irq_msk = BIT(10), .msg = "DMAC2_FIFO_OMIT_ERR" },
+	{ .irq_msk = BIT(11), .msg = "DMAC3_FIFO_OMIT_ERR" },
+	{ .irq_msk = BIT(12), .msg = "DMAC4_FIFO_OMIT_ERR" },
+	{ .irq_msk = BIT(13), .msg = "DMAC5_FIFO_OMIT_ERR" },
+	{ .irq_msk = BIT(14), .msg = "DMAC6_FIFO_OMIT_ERR" },
+	{ .irq_msk = BIT(15), .msg = "DMAC7_FIFO_OMIT_ERR" },
+	{ .irq_msk = BIT(16), .msg = "HGC_RLSE_SLOT_UNMATCH" },
+	{ .irq_msk = BIT(17), .msg = "HGC_LM_ADD_FCH_LIST_ERR" },
+	{ .irq_msk = BIT(18), .msg = "HGC_AXI_BUS_ERR" },
+	{ .irq_msk = BIT(19), .msg = "HGC_FIFO_OMIT_ERR" },
+};
+
 static bool process_non_fatal_error_v3_hw(struct hisi_hba *hisi_hba)
 {
 	struct device *dev = hisi_hba->dev;
@@ -2252,6 +2381,17 @@ static bool process_non_fatal_error_v3_hw(struct hisi_hba *hisi_hba)
 	}
 	hisi_sas_write32(hisi_hba, SAS_RAS_INTR1, irq_value);
 
+	irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR2);
+	for (i = 0; i < ARRAY_SIZE(sas_ras_intr2_nfe); i++) {
+		ras_error = &sas_ras_intr2_nfe[i];
+		if (ras_error->irq_msk & irq_value) {
+			dev_warn(dev, "SAS_RAS_INTR2: %s(irq_value=0x%x) found.\n",
+					ras_error->msg, irq_value);
+			need_reset = true;
+		}
+	}
+	hisi_sas_write32(hisi_hba, SAS_RAS_INTR2, irq_value);
+
 	return need_reset;
 }
 
@@ -2307,7 +2447,6 @@ static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
 	u32 device_state, status;
 	int rc;
 	u32 reg_val;
-	unsigned long flags;
 
 	if (!pdev->pm_cap) {
 		dev_err(dev, "PCI PM not supported\n");
@@ -2332,8 +2471,9 @@ static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
 		AM_CTRL_GLOBAL, reg_val);
 
 	/* wait until bus idle */
-	rc = readl_poll_timeout(hisi_hba->regs + AXI_MASTER_CFG_BASE +
-		AM_CURR_TRANS_RETURN, status, status == 0x3, 10, 100);
+	rc = hisi_sas_read32_poll_timeout(AXI_MASTER_CFG_BASE +
+					  AM_CURR_TRANS_RETURN, status,
+					  status == 0x3, 10, 100);
 	if (rc) {
 		dev_err(dev, "axi bus is not idle, rc = %d\n", rc);
 		clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
@@ -2351,9 +2491,7 @@ static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
 	pci_disable_device(pdev);
 	pci_set_power_state(pdev, device_state);
 
-	spin_lock_irqsave(&hisi_hba->lock, flags);
 	hisi_sas_release_tasks(hisi_hba);
-	spin_unlock_irqrestore(&hisi_hba->lock, flags);
 
 	sas_suspend_ha(sha);
 	return 0;
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index dda1a64ab89c..6615ad8754b8 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -435,6 +435,8 @@ struct ipr_error_table_t ipr_error_table[] = {
 	"4080: IOA exceeded maximum operating temperature"},
 	{0x060B8000, 0, IPR_DEFAULT_LOG_LEVEL,
 	"4085: Service required"},
+	{0x060B8100, 0, IPR_DEFAULT_LOG_LEVEL,
+	"4086: SAS Adapter Hardware Configuration Error"},
 	{0x06288000, 0, IPR_DEFAULT_LOG_LEVEL,
 	"3140: Device bus not ready to ready transition"},
 	{0x06290000, 0, IPR_DEFAULT_LOG_LEVEL,
diff --git a/drivers/scsi/ips.c b/drivers/scsi/ips.c
index e3c8857741a1..bd6ac6b5980a 100644
--- a/drivers/scsi/ips.c
+++ b/drivers/scsi/ips.c
@@ -291,7 +291,7 @@ static void ips_freescb(ips_ha_t *, ips_scb_t *);
 static void ips_setup_funclist(ips_ha_t *);
 static void ips_statinit(ips_ha_t *);
 static void ips_statinit_memio(ips_ha_t *);
-static void ips_fix_ffdc_time(ips_ha_t *, ips_scb_t *, time_t);
+static void ips_fix_ffdc_time(ips_ha_t *, ips_scb_t *, time64_t);
 static void ips_ffdc_reset(ips_ha_t *, int);
 static void ips_ffdc_time(ips_ha_t *);
 static uint32_t ips_statupd_copperhead(ips_ha_t *);
@@ -985,10 +985,7 @@ static int __ips_eh_reset(struct scsi_cmnd *SC)
 
 	/* FFDC */
 	if (le32_to_cpu(ha->subsys->param[3]) & 0x300000) {
-		struct timeval tv;
-
-		do_gettimeofday(&tv);
-		ha->last_ffdc = tv.tv_sec;
+		ha->last_ffdc = ktime_get_real_seconds();
 		ha->reset_count++;
 		ips_ffdc_reset(ha, IPS_INTR_IORL);
 	}
@@ -2392,7 +2389,6 @@ static int
 ips_hainit(ips_ha_t * ha)
 {
 	int i;
-	struct timeval tv;
 
 	METHOD_TRACE("ips_hainit", 1);
 
@@ -2407,8 +2403,7 @@ ips_hainit(ips_ha_t * ha)
 
 	/* Send FFDC */
 	ha->reset_count = 1;
-	do_gettimeofday(&tv);
-	ha->last_ffdc = tv.tv_sec;
+	ha->last_ffdc = ktime_get_real_seconds();
 	ips_ffdc_reset(ha, IPS_INTR_IORL);
 
 	if (!ips_read_config(ha, IPS_INTR_IORL)) {
@@ -2548,12 +2543,9 @@ ips_next(ips_ha_t * ha, int intr)
 
 	if ((ha->subsys->param[3] & 0x300000)
 	    && (ha->scb_activelist.count == 0)) {
-		struct timeval tv;
-
-		do_gettimeofday(&tv);
-
-		if (tv.tv_sec - ha->last_ffdc > IPS_SECS_8HOURS) {
-			ha->last_ffdc = tv.tv_sec;
+		time64_t now = ktime_get_real_seconds();
+		if (now - ha->last_ffdc > IPS_SECS_8HOURS) {
+			ha->last_ffdc = now;
 			ips_ffdc_time(ha);
 		}
 	}
@@ -5988,59 +5980,21 @@ ips_ffdc_time(ips_ha_t * ha)
 /*                                                                          */
 /****************************************************************************/
 static void
-ips_fix_ffdc_time(ips_ha_t * ha, ips_scb_t * scb, time_t current_time)
+ips_fix_ffdc_time(ips_ha_t * ha, ips_scb_t * scb, time64_t current_time)
 {
-	long days;
-	long rem;
-	int i;
-	int year;
-	int yleap;
-	int year_lengths[2] = { IPS_DAYS_NORMAL_YEAR, IPS_DAYS_LEAP_YEAR };
-	int month_lengths[12][2] = { {31, 31},
-	{28, 29},
-	{31, 31},
-	{30, 30},
-	{31, 31},
-	{30, 30},
-	{31, 31},
-	{31, 31},
-	{30, 30},
-	{31, 31},
-	{30, 30},
-	{31, 31}
-	};
+	struct tm tm;
 
 	METHOD_TRACE("ips_fix_ffdc_time", 1);
 
-	days = current_time / IPS_SECS_DAY;
-	rem = current_time % IPS_SECS_DAY;
-
-	scb->cmd.ffdc.hour = (rem / IPS_SECS_HOUR);
-	rem = rem % IPS_SECS_HOUR;
-	scb->cmd.ffdc.minute = (rem / IPS_SECS_MIN);
-	scb->cmd.ffdc.second = (rem % IPS_SECS_MIN);
-
-	year = IPS_EPOCH_YEAR;
-	while (days < 0 || days >= year_lengths[yleap = IPS_IS_LEAP_YEAR(year)]) {
-		int newy;
-
-		newy = year + (days / IPS_DAYS_NORMAL_YEAR);
-		if (days < 0)
-			--newy;
-		days -= (newy - year) * IPS_DAYS_NORMAL_YEAR +
-		    IPS_NUM_LEAP_YEARS_THROUGH(newy - 1) -
-		    IPS_NUM_LEAP_YEARS_THROUGH(year - 1);
-		year = newy;
-	}
-
-	scb->cmd.ffdc.yearH = year / 100;
-	scb->cmd.ffdc.yearL = year % 100;
-
-	for (i = 0; days >= month_lengths[i][yleap]; ++i)
-		days -= month_lengths[i][yleap];
+	time64_to_tm(current_time, 0, &tm);
 
-	scb->cmd.ffdc.month = i + 1;
-	scb->cmd.ffdc.day = days + 1;
+	scb->cmd.ffdc.hour   = tm.tm_hour;
+	scb->cmd.ffdc.minute = tm.tm_min;
+	scb->cmd.ffdc.second = tm.tm_sec;
+	scb->cmd.ffdc.yearH  = (tm.tm_year + 1900) / 100;
+	scb->cmd.ffdc.yearL  = tm.tm_year % 100;
+	scb->cmd.ffdc.month  = tm.tm_mon + 1;
+	scb->cmd.ffdc.day    = tm.tm_mday;
 }
 
 /****************************************************************************
diff --git a/drivers/scsi/ips.h b/drivers/scsi/ips.h
index 366be3b2f9b4..db546171e97f 100644
--- a/drivers/scsi/ips.h
+++ b/drivers/scsi/ips.h
@@ -402,16 +402,7 @@
    #define IPS_BIOS_HEADER             0xC0
 
    /* time oriented stuff */
-   #define IPS_IS_LEAP_YEAR(y)           (((y % 4 == 0) && ((y % 100 != 0) || (y % 400 == 0))) ? 1 : 0)
-   #define IPS_NUM_LEAP_YEARS_THROUGH(y) ((y) / 4 - (y) / 100 + (y) / 400)
-
-   #define IPS_SECS_MIN                 60
-   #define IPS_SECS_HOUR                3600
    #define IPS_SECS_8HOURS              28800
-   #define IPS_SECS_DAY                 86400
-   #define IPS_DAYS_NORMAL_YEAR         365
-   #define IPS_DAYS_LEAP_YEAR           366
-   #define IPS_EPOCH_YEAR               1970
 
    /*
     * Scsi_Host Template
@@ -1054,7 +1045,7 @@ typedef struct ips_ha {
    uint8_t            active;
    int                ioctl_reset;        /* IOCTL Requested Reset Flag */
    uint16_t           reset_count;        /* number of resets           */
-   time_t             last_ffdc;          /* last time we sent ffdc info*/
+   time64_t           last_ffdc;          /* last time we sent ffdc info*/
    uint8_t            slot_num;           /* PCI Slot Number            */
    int                ioctl_len;          /* size of ioctl buffer       */
    dma_addr_t         ioctl_busaddr;      /* dma address of ioctl buffer*/
diff --git a/drivers/scsi/isci/init.c b/drivers/scsi/isci/init.c
index 922e3e56c90d..05cf4daf8788 100644
--- a/drivers/scsi/isci/init.c
+++ b/drivers/scsi/isci/init.c
@@ -433,9 +433,6 @@ static enum sci_status sci_user_parameters_set(struct isci_host *ihost,
 		      (u->max_speed_generation > SCIC_SDS_PARM_NO_SPEED)))
 			return SCI_FAILURE_INVALID_PARAMETER_VALUE;
 
-		if (u->in_connection_align_insertion_frequency < 3)
-			return SCI_FAILURE_INVALID_PARAMETER_VALUE;
-
 		if ((u->in_connection_align_insertion_frequency < 3) ||
 		    (u->align_insertion_frequency == 0) ||
 		    (u->notify_enable_spin_up_insertion_frequency == 0))
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index 2ba4b68fdb73..b025a0b74341 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -962,7 +962,6 @@ static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev)
 	if (conn->datadgst_en)
 		sdev->request_queue->backing_dev_info->capabilities
 			|= BDI_CAP_STABLE_WRITES;
-	blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
 	blk_queue_dma_alignment(sdev->request_queue, 0);
 	return 0;
 }
diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index 15a2fef51e38..71bdc0b52cf9 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -1963,7 +1963,7 @@ static int iscsi_has_ping_timed_out(struct iscsi_conn *conn)
 
 enum blk_eh_timer_return iscsi_eh_cmd_timed_out(struct scsi_cmnd *sc)
 {
-	enum blk_eh_timer_return rc = BLK_EH_NOT_HANDLED;
+	enum blk_eh_timer_return rc = BLK_EH_DONE;
 	struct iscsi_task *task = NULL, *running_task;
 	struct iscsi_cls_session *cls_session;
 	struct iscsi_session *session;
@@ -1982,7 +1982,7 @@ enum blk_eh_timer_return iscsi_eh_cmd_timed_out(struct scsi_cmnd *sc)
 		 * Raced with completion. Blk layer has taken ownership
 		 * so let timeout code complete it now.
 		 */
-		rc = BLK_EH_HANDLED;
+		rc = BLK_EH_DONE;
 		goto done;
 	}
 
@@ -1997,7 +1997,7 @@ enum blk_eh_timer_return iscsi_eh_cmd_timed_out(struct scsi_cmnd *sc)
 		if (unlikely(system_state != SYSTEM_RUNNING)) {
 			sc->result = DID_NO_CONNECT << 16;
 			ISCSI_DBG_EH(session, "sc on shutdown, handled\n");
-			rc = BLK_EH_HANDLED;
+			rc = BLK_EH_DONE;
 			goto done;
 		}
 		/*
diff --git a/drivers/scsi/libsas/sas_ata.c b/drivers/scsi/libsas/sas_ata.c
index 0cc1567eacc1..ff1d612f6fb9 100644
--- a/drivers/scsi/libsas/sas_ata.c
+++ b/drivers/scsi/libsas/sas_ata.c
@@ -577,6 +577,11 @@ int sas_ata_init(struct domain_device *found_dev)
 		ata_sas_port_destroy(ap);
 		return rc;
 	}
+	rc = ata_sas_tport_add(found_dev->sata_dev.ata_host.dev, ap);
+	if (rc) {
+		ata_sas_port_destroy(ap);
+		return rc;
+	}
 	found_dev->sata_dev.ap = ap;
 
 	return 0;
diff --git a/drivers/scsi/libsas/sas_discover.c b/drivers/scsi/libsas/sas_discover.c
index a0fa7ef3a071..1ffca28fe6a8 100644
--- a/drivers/scsi/libsas/sas_discover.c
+++ b/drivers/scsi/libsas/sas_discover.c
@@ -314,6 +314,7 @@ void sas_free_device(struct kref *kref)
 		kfree(dev->ex_dev.ex_phy);
 
 	if (dev_is_sata(dev) && dev->sata_dev.ap) {
+		ata_sas_tport_delete(dev->sata_dev.ap);
 		ata_sas_port_destroy(dev->sata_dev.ap);
 		dev->sata_dev.ap = NULL;
 	}
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index 6c0d351c0d0d..20b249a649dd 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -64,8 +64,6 @@ struct lpfc_sli2_slim;
 #define LPFC_IOCB_LIST_CNT	2250	/* list of IOCBs for fast-path usage. */
 #define LPFC_Q_RAMP_UP_INTERVAL 120     /* lun q_depth ramp up interval */
 #define LPFC_VNAME_LEN		100	/* vport symbolic name length */
-#define LPFC_TGTQ_INTERVAL	40000	/* Min amount of time between tgt
-					   queue depth change in millisecs */
 #define LPFC_TGTQ_RAMPUP_PCENT	5	/* Target queue rampup in percentage */
 #define LPFC_MIN_TGT_QDEPTH	10
 #define LPFC_MAX_TGT_QDEPTH	0xFFFF
@@ -784,6 +782,7 @@ struct lpfc_hba {
 	uint32_t cfg_nvme_oas;
 	uint32_t cfg_nvme_embed_cmd;
 	uint32_t cfg_nvme_io_channel;
+	uint32_t cfg_nvmet_mrq_post;
 	uint32_t cfg_nvmet_mrq;
 	uint32_t cfg_enable_nvmet;
 	uint32_t cfg_nvme_enable_fb;
@@ -922,12 +921,6 @@ struct lpfc_hba {
 	atomic_t fc4ScsiOutputRequests;
 	atomic_t fc4ScsiControlRequests;
 	atomic_t fc4ScsiIoCmpls;
-	atomic_t fc4NvmeInputRequests;
-	atomic_t fc4NvmeOutputRequests;
-	atomic_t fc4NvmeControlRequests;
-	atomic_t fc4NvmeIoCmpls;
-	atomic_t fc4NvmeLsRequests;
-	atomic_t fc4NvmeLsCmpls;
 
 	uint64_t bg_guard_err_cnt;
 	uint64_t bg_apptag_err_cnt;
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index 2ac1d21c553f..729d343861f4 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -149,10 +149,14 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_nvmet_tgtport *tgtp;
 	struct nvme_fc_local_port *localport;
 	struct lpfc_nvme_lport *lport;
+	struct lpfc_nvme_rport *rport;
 	struct lpfc_nodelist *ndlp;
 	struct nvme_fc_remote_port *nrport;
-	uint64_t data1, data2, data3, tot;
+	struct lpfc_nvme_ctrl_stat *cstat;
+	uint64_t data1, data2, data3;
+	uint64_t totin, totout, tot;
 	char *statep;
+	int i;
 	int len = 0;
 
 	if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) {
@@ -293,6 +297,13 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
 	len = snprintf(buf, PAGE_SIZE, "NVME Initiator Enabled\n");
 
 	spin_lock_irq(shost->host_lock);
+	len += snprintf(buf + len, PAGE_SIZE - len,
+			"XRI Dist lpfc%d Total %d NVME %d SCSI %d ELS %d\n",
+			phba->brd_no,
+			phba->sli4_hba.max_cfg_param.max_xri,
+			phba->sli4_hba.nvme_xri_max,
+			phba->sli4_hba.scsi_xri_max,
+			lpfc_sli4_get_els_iocb_cnt(phba));
 
 	/* Port state is only one of two values for now. */
 	if (localport->port_id)
@@ -309,11 +320,14 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
 			localport->port_id, statep);
 
 	list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) {
-		if (!ndlp->nrport)
+		rport = lpfc_ndlp_get_nrport(ndlp);
+		if (!rport)
 			continue;
 
 		/* local short-hand pointer. */
-		nrport = ndlp->nrport->remoteport;
+		nrport = rport->remoteport;
+		if (!nrport)
+			continue;
 
 		/* Port state is only one of two values for now. */
 		switch (nrport->port_state) {
@@ -364,11 +378,14 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
 	}
 	spin_unlock_irq(shost->host_lock);
 
+	if (!lport)
+		return len;
+
 	len += snprintf(buf + len, PAGE_SIZE - len, "\nNVME Statistics\n");
 	len += snprintf(buf+len, PAGE_SIZE-len,
 			"LS: Xmt %010x Cmpl %010x Abort %08x\n",
-			atomic_read(&phba->fc4NvmeLsRequests),
-			atomic_read(&phba->fc4NvmeLsCmpls),
+			atomic_read(&lport->fc4NvmeLsRequests),
+			atomic_read(&lport->fc4NvmeLsCmpls),
 			atomic_read(&lport->xmt_ls_abort));
 
 	len += snprintf(buf + len, PAGE_SIZE - len,
@@ -377,27 +394,32 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
 			atomic_read(&lport->cmpl_ls_xb),
 			atomic_read(&lport->cmpl_ls_err));
 
-	tot = atomic_read(&phba->fc4NvmeIoCmpls);
-	data1 = atomic_read(&phba->fc4NvmeInputRequests);
-	data2 = atomic_read(&phba->fc4NvmeOutputRequests);
-	data3 = atomic_read(&phba->fc4NvmeControlRequests);
+	totin = 0;
+	totout = 0;
+	for (i = 0; i < phba->cfg_nvme_io_channel; i++) {
+		cstat = &lport->cstat[i];
+		tot = atomic_read(&cstat->fc4NvmeIoCmpls);
+		totin += tot;
+		data1 = atomic_read(&cstat->fc4NvmeInputRequests);
+		data2 = atomic_read(&cstat->fc4NvmeOutputRequests);
+		data3 = atomic_read(&cstat->fc4NvmeControlRequests);
+		totout += (data1 + data2 + data3);
+	}
 	len += snprintf(buf+len, PAGE_SIZE-len,
-			"FCP: Rd %016llx Wr %016llx IO %016llx\n",
-			data1, data2, data3);
+			"Total FCP Cmpl %016llx Issue %016llx "
+			"OutIO %016llx\n",
+			totin, totout, totout - totin);
 
 	len += snprintf(buf+len, PAGE_SIZE-len,
-			"    noxri %08x nondlp %08x qdepth %08x "
-			"wqerr %08x\n",
+			"      abort %08x noxri %08x nondlp %08x qdepth %08x "
+			"wqerr %08x err %08x\n",
+			atomic_read(&lport->xmt_fcp_abort),
 			atomic_read(&lport->xmt_fcp_noxri),
 			atomic_read(&lport->xmt_fcp_bad_ndlp),
 			atomic_read(&lport->xmt_fcp_qdepth),
+			atomic_read(&lport->xmt_fcp_err),
 			atomic_read(&lport->xmt_fcp_wqerr));
 
-	len += snprintf(buf + len, PAGE_SIZE - len,
-			"    Cmpl %016llx Outstanding %016llx Abort %08x\n",
-			tot, ((data1 + data2 + data3) - tot),
-			atomic_read(&lport->xmt_fcp_abort));
-
 	len += snprintf(buf + len, PAGE_SIZE - len,
 			"FCP CMPL: xb %08x Err %08x\n",
 			atomic_read(&lport->cmpl_fcp_xb),
@@ -3280,6 +3302,9 @@ lpfc_update_rport_devloss_tmo(struct lpfc_vport *vport)
 {
 	struct Scsi_Host  *shost;
 	struct lpfc_nodelist  *ndlp;
+#if (IS_ENABLED(CONFIG_NVME_FC))
+	struct lpfc_nvme_rport *rport;
+#endif
 
 	shost = lpfc_shost_from_vport(vport);
 	spin_lock_irq(shost->host_lock);
@@ -3289,8 +3314,9 @@ lpfc_update_rport_devloss_tmo(struct lpfc_vport *vport)
 		if (ndlp->rport)
 			ndlp->rport->dev_loss_tmo = vport->cfg_devloss_tmo;
 #if (IS_ENABLED(CONFIG_NVME_FC))
-		if (ndlp->nrport)
-			nvme_fc_set_remoteport_devloss(ndlp->nrport->remoteport,
+		rport = lpfc_ndlp_get_nrport(ndlp);
+		if (rport)
+			nvme_fc_set_remoteport_devloss(rport->remoteport,
 						       vport->cfg_devloss_tmo);
 #endif
 	}
@@ -3413,6 +3439,15 @@ LPFC_ATTR_R(nvmet_mrq,
 	    LPFC_NVMET_MRQ_AUTO, LPFC_NVMET_MRQ_AUTO, LPFC_NVMET_MRQ_MAX,
 	    "Specify number of RQ pairs for processing NVMET cmds");
 
+/*
+ * lpfc_nvmet_mrq_post: Specify number of RQ buffer to initially post
+ * to each NVMET RQ. Range 64 to 2048, default is 512.
+ */
+LPFC_ATTR_R(nvmet_mrq_post,
+	    LPFC_NVMET_RQE_DEF_POST, LPFC_NVMET_RQE_MIN_POST,
+	    LPFC_NVMET_RQE_DEF_COUNT,
+	    "Specify number of RQ buffers to initially post");
+
 /*
  * lpfc_enable_fc4_type: Defines what FC4 types are supported.
  * Supported Values:  1 - register just FCP
@@ -3469,8 +3504,49 @@ LPFC_VPORT_ATTR_R(lun_queue_depth, 30, 1, 512,
 # tgt_queue_depth:  This parameter is used to limit the number of outstanding
 # commands per target port. Value range is [10,65535]. Default value is 65535.
 */
-LPFC_VPORT_ATTR_RW(tgt_queue_depth, 65535, 10, 65535,
-		   "Max number of FCP commands we can queue to a specific target port");
+static uint lpfc_tgt_queue_depth = LPFC_MAX_TGT_QDEPTH;
+module_param(lpfc_tgt_queue_depth, uint, 0444);
+MODULE_PARM_DESC(lpfc_tgt_queue_depth, "Set max Target queue depth");
+lpfc_vport_param_show(tgt_queue_depth);
+lpfc_vport_param_init(tgt_queue_depth, LPFC_MAX_TGT_QDEPTH,
+		      LPFC_MIN_TGT_QDEPTH, LPFC_MAX_TGT_QDEPTH);
+
+/**
+ * lpfc_tgt_queue_depth_store: Sets an attribute value.
+ * @phba: pointer the the adapter structure.
+ * @val: integer attribute value.
+ *
+ * Description: Sets the parameter to the new value.
+ *
+ * Returns:
+ * zero on success
+ * -EINVAL if val is invalid
+ */
+static int
+lpfc_tgt_queue_depth_set(struct lpfc_vport *vport, uint val)
+{
+	struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
+	struct lpfc_nodelist *ndlp;
+
+	if (!lpfc_rangecheck(val, LPFC_MIN_TGT_QDEPTH, LPFC_MAX_TGT_QDEPTH))
+		return -EINVAL;
+
+	if (val == vport->cfg_tgt_queue_depth)
+		return 0;
+
+	spin_lock_irq(shost->host_lock);
+	vport->cfg_tgt_queue_depth = val;
+
+	/* Next loop thru nodelist and change cmd_qdepth */
+	list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp)
+		ndlp->cmd_qdepth = vport->cfg_tgt_queue_depth;
+
+	spin_unlock_irq(shost->host_lock);
+	return 0;
+}
+
+lpfc_vport_param_store(tgt_queue_depth);
+static DEVICE_ATTR_RW(lpfc_tgt_queue_depth);
 
 /*
 # hba_queue_depth:  This parameter is used to limit the number of outstanding
@@ -5302,6 +5378,7 @@ struct device_attribute *lpfc_hba_attrs[] = {
 	&dev_attr_lpfc_suppress_rsp,
 	&dev_attr_lpfc_nvme_io_channel,
 	&dev_attr_lpfc_nvmet_mrq,
+	&dev_attr_lpfc_nvmet_mrq_post,
 	&dev_attr_lpfc_nvme_enable_fb,
 	&dev_attr_lpfc_nvmet_fb_size,
 	&dev_attr_lpfc_enable_bg,
@@ -6352,6 +6429,7 @@ lpfc_get_cfgparam(struct lpfc_hba *phba)
 
 	lpfc_enable_fc4_type_init(phba, lpfc_enable_fc4_type);
 	lpfc_nvmet_mrq_init(phba, lpfc_nvmet_mrq);
+	lpfc_nvmet_mrq_post_init(phba, lpfc_nvmet_mrq_post);
 
 	/* Initialize first burst. Target vs Initiator are different. */
 	lpfc_nvme_enable_fb_init(phba, lpfc_nvme_enable_fb);
diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c
index 0f174ca80f67..edb1a18a6414 100644
--- a/drivers/scsi/lpfc/lpfc_bsg.c
+++ b/drivers/scsi/lpfc/lpfc_bsg.c
@@ -3621,7 +3621,7 @@ lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 		bsg_reply->result = 0;
 
 		lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
-				"2937 SLI_CONFIG ext-buffer maibox command "
+				"2937 SLI_CONFIG ext-buffer mailbox command "
 				"(x%x/x%x) complete bsg job done, bsize:%d\n",
 				phba->mbox_ext_buf_ctx.nembType,
 				phba->mbox_ext_buf_ctx.mboxType, size);
@@ -3632,7 +3632,7 @@ lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 					phba->mbox_ext_buf_ctx.mbx_dmabuf, 0);
 	} else {
 		lpfc_printf_log(phba, KERN_ERR, LOG_LIBDFC,
-				"2938 SLI_CONFIG ext-buffer maibox "
+				"2938 SLI_CONFIG ext-buffer mailbox "
 				"command (x%x/x%x) failure, rc:x%x\n",
 				phba->mbox_ext_buf_ctx.nembType,
 				phba->mbox_ext_buf_ctx.mboxType, rc);
@@ -3666,7 +3666,7 @@ lpfc_bsg_issue_read_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 		pmboxq->u.mb.mbxStatus = MBXERR_ERROR;
 
 	lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
-			"2939 SLI_CONFIG ext-buffer rd maibox command "
+			"2939 SLI_CONFIG ext-buffer rd mailbox command "
 			"complete, ctxState:x%x, mbxStatus:x%x\n",
 			phba->mbox_ext_buf_ctx.state, pmboxq->u.mb.mbxStatus);
 
@@ -3706,7 +3706,7 @@ lpfc_bsg_issue_write_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 		pmboxq->u.mb.mbxStatus = MBXERR_ERROR;
 
 	lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
-			"2940 SLI_CONFIG ext-buffer wr maibox command "
+			"2940 SLI_CONFIG ext-buffer wr mailbox command "
 			"complete, ctxState:x%x, mbxStatus:x%x\n",
 			phba->mbox_ext_buf_ctx.state, pmboxq->u.mb.mbxStatus);
 
@@ -3988,12 +3988,12 @@ lpfc_bsg_sli_cfg_read_cmd_ext(struct lpfc_hba *phba, struct bsg_job *job,
 	if ((rc == MBX_SUCCESS) || (rc == MBX_BUSY)) {
 		lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
 				"2947 Issued SLI_CONFIG ext-buffer "
-				"maibox command, rc:x%x\n", rc);
+				"mailbox command, rc:x%x\n", rc);
 		return SLI_CONFIG_HANDLED;
 	}
 	lpfc_printf_log(phba, KERN_ERR, LOG_LIBDFC,
 			"2948 Failed to issue SLI_CONFIG ext-buffer "
-			"maibox command, rc:x%x\n", rc);
+			"mailbox command, rc:x%x\n", rc);
 	rc = -EPIPE;
 
 job_error:
@@ -4147,12 +4147,12 @@ lpfc_bsg_sli_cfg_write_cmd_ext(struct lpfc_hba *phba, struct bsg_job *job,
 		if ((rc == MBX_SUCCESS) || (rc == MBX_BUSY)) {
 			lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
 					"2955 Issued SLI_CONFIG ext-buffer "
-					"maibox command, rc:x%x\n", rc);
+					"mailbox command, rc:x%x\n", rc);
 			return SLI_CONFIG_HANDLED;
 		}
 		lpfc_printf_log(phba, KERN_ERR, LOG_LIBDFC,
 				"2956 Failed to issue SLI_CONFIG ext-buffer "
-				"maibox command, rc:x%x\n", rc);
+				"mailbox command, rc:x%x\n", rc);
 		rc = -EPIPE;
 		goto job_error;
 	}
@@ -4492,12 +4492,12 @@ lpfc_bsg_write_ebuf_set(struct lpfc_hba *phba, struct bsg_job *job,
 		if ((rc == MBX_SUCCESS) || (rc == MBX_BUSY)) {
 			lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
 					"2969 Issued SLI_CONFIG ext-buffer "
-					"maibox command, rc:x%x\n", rc);
+					"mailbox command, rc:x%x\n", rc);
 			return SLI_CONFIG_HANDLED;
 		}
 		lpfc_printf_log(phba, KERN_ERR, LOG_LIBDFC,
 				"2970 Failed to issue SLI_CONFIG ext-buffer "
-				"maibox command, rc:x%x\n", rc);
+				"mailbox command, rc:x%x\n", rc);
 		rc = -EPIPE;
 		goto job_error;
 	}
diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c
index 0617c8ea88c6..d4a200ae5a6f 100644
--- a/drivers/scsi/lpfc/lpfc_ct.c
+++ b/drivers/scsi/lpfc/lpfc_ct.c
@@ -471,6 +471,11 @@ lpfc_prep_node_fc4type(struct lpfc_vport *vport, uint32_t Did, uint8_t fc4_type)
 				"Parse GID_FTrsp: did:x%x flg:x%x x%x",
 				Did, ndlp->nlp_flag, vport->fc_flag);
 
+			/* Don't assume the rport is always the previous
+			 * FC4 type.
+			 */
+			ndlp->nlp_fc4_type &= ~(NLP_FC4_FCP | NLP_FC4_NVME);
+
 			/* By default, the driver expects to support FCP FC4 */
 			if (fc4_type == FC_TYPE_FCP)
 				ndlp->nlp_fc4_type |= NLP_FC4_FCP;
@@ -691,6 +696,11 @@ lpfc_cmpl_ct_cmd_gid_ft(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		vport->fc_flag &= ~FC_RSCN_DEFERRED;
 		spin_unlock_irq(shost->host_lock);
 
+		/* This is a GID_FT completing so the gidft_inp counter was
+		 * incremented before the GID_FT was issued to the wire.
+		 */
+		vport->gidft_inp--;
+
 		/*
 		 * Skip processing the NS response
 		 * Re-issue the NS cmd
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c
index fb0dc2aeed91..9df0c051349f 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.c
+++ b/drivers/scsi/lpfc/lpfc_debugfs.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2007-2015 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -544,7 +544,7 @@ static int
 lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 {
 	int len = 0;
-	int cnt;
+	int i, iocnt, outio, cnt;
 	struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
 	struct lpfc_hba  *phba = vport->phba;
 	struct lpfc_nodelist *ndlp;
@@ -552,12 +552,15 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 	struct nvme_fc_local_port *localport;
 	struct lpfc_nvmet_tgtport *tgtp;
 	struct nvme_fc_remote_port *nrport;
+	struct lpfc_nvme_rport *rport;
 
 	cnt = (LPFC_NODELIST_SIZE / LPFC_NODELIST_ENTRY_SIZE);
+	outio = 0;
 
 	len += snprintf(buf+len, size-len, "\nFCP Nodelist Entries ...\n");
 	spin_lock_irq(shost->host_lock);
 	list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) {
+		iocnt = 0;
 		if (!cnt) {
 			len +=  snprintf(buf+len, size-len,
 				"Missing Nodelist Entries\n");
@@ -585,9 +588,11 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 			break;
 		case NLP_STE_UNMAPPED_NODE:
 			statep = "UNMAP ";
+			iocnt = 1;
 			break;
 		case NLP_STE_MAPPED_NODE:
 			statep = "MAPPED";
+			iocnt = 1;
 			break;
 		case NLP_STE_NPR_NODE:
 			statep = "NPR   ";
@@ -614,8 +619,10 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 			len += snprintf(buf+len, size-len, "UNKNOWN_TYPE ");
 		if (ndlp->nlp_type & NLP_FC_NODE)
 			len += snprintf(buf+len, size-len, "FC_NODE ");
-		if (ndlp->nlp_type & NLP_FABRIC)
+		if (ndlp->nlp_type & NLP_FABRIC) {
 			len += snprintf(buf+len, size-len, "FABRIC ");
+			iocnt = 0;
+		}
 		if (ndlp->nlp_type & NLP_FCP_TARGET)
 			len += snprintf(buf+len, size-len, "FCP_TGT sid:%d ",
 				ndlp->nlp_sid);
@@ -632,10 +639,20 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 			ndlp->nlp_usg_map);
 		len += snprintf(buf+len, size-len, "refcnt:%x",
 			kref_read(&ndlp->kref));
+		if (iocnt) {
+			i = atomic_read(&ndlp->cmd_pending);
+			len += snprintf(buf + len, size - len,
+					" OutIO:x%x Qdepth x%x",
+					i, ndlp->cmd_qdepth);
+			outio += i;
+		}
 		len +=  snprintf(buf+len, size-len, "\n");
 	}
 	spin_unlock_irq(shost->host_lock);
 
+	len += snprintf(buf + len, size - len,
+			"\nOutstanding IO x%x\n",  outio);
+
 	if (phba->nvmet_support && phba->targetport && (vport == phba->pport)) {
 		tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
 		len += snprintf(buf + len, size - len,
@@ -679,10 +696,13 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 	len += snprintf(buf + len, size - len, "\tRport List:\n");
 	list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) {
 		/* local short-hand pointer. */
-		if (!ndlp->nrport)
+		rport = lpfc_ndlp_get_nrport(ndlp);
+		if (!rport)
 			continue;
 
-		nrport = ndlp->nrport->remoteport;
+		nrport = rport->remoteport;
+		if (!nrport)
+			continue;
 
 		/* Port state is only one of two values for now. */
 		switch (nrport->port_state) {
@@ -751,10 +771,12 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 	struct lpfc_nvmet_tgtport *tgtp;
 	struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
 	struct nvme_fc_local_port *localport;
+	struct lpfc_nvme_ctrl_stat *cstat;
 	struct lpfc_nvme_lport *lport;
-	uint64_t tot, data1, data2, data3;
+	uint64_t data1, data2, data3;
+	uint64_t tot, totin, totout;
+	int cnt, i, maxch;
 	int len = 0;
-	int cnt;
 
 	if (phba->nvmet_support) {
 		if (!phba->targetport)
@@ -880,33 +902,52 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 		if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
 			return len;
 
+		localport = vport->localport;
+		if (!localport)
+			return len;
+		lport = (struct lpfc_nvme_lport *)localport->private;
+		if (!lport)
+			return len;
+
 		len += snprintf(buf + len, size - len,
 				"\nNVME Lport Statistics\n");
 
 		len += snprintf(buf + len, size - len,
 				"LS: Xmt %016x Cmpl %016x\n",
-				atomic_read(&phba->fc4NvmeLsRequests),
-				atomic_read(&phba->fc4NvmeLsCmpls));
-
-		tot = atomic_read(&phba->fc4NvmeIoCmpls);
-		data1 = atomic_read(&phba->fc4NvmeInputRequests);
-		data2 = atomic_read(&phba->fc4NvmeOutputRequests);
-		data3 = atomic_read(&phba->fc4NvmeControlRequests);
+				atomic_read(&lport->fc4NvmeLsRequests),
+				atomic_read(&lport->fc4NvmeLsCmpls));
 
-		len += snprintf(buf + len, size - len,
-				"FCP: Rd %016llx Wr %016llx IO %016llx\n",
-				data1, data2, data3);
-
-		len += snprintf(buf + len, size - len,
-				"   Cmpl %016llx Outstanding %016llx\n",
-				tot, (data1 + data2 + data3) - tot);
+		if (phba->cfg_nvme_io_channel < 32)
+			maxch = phba->cfg_nvme_io_channel;
+		else
+			maxch = 32;
+		totin = 0;
+		totout = 0;
+		for (i = 0; i < phba->cfg_nvme_io_channel; i++) {
+			cstat = &lport->cstat[i];
+			tot = atomic_read(&cstat->fc4NvmeIoCmpls);
+			totin += tot;
+			data1 = atomic_read(&cstat->fc4NvmeInputRequests);
+			data2 = atomic_read(&cstat->fc4NvmeOutputRequests);
+			data3 = atomic_read(&cstat->fc4NvmeControlRequests);
+			totout += (data1 + data2 + data3);
+
+			/* Limit to 32, debugfs display buffer limitation */
+			if (i >= 32)
+				continue;
 
-		localport = vport->localport;
-		if (!localport)
-			return len;
-		lport = (struct lpfc_nvme_lport *)localport->private;
-		if (!lport)
-			return len;
+			len += snprintf(buf + len, PAGE_SIZE - len,
+					"FCP (%d): Rd %016llx Wr %016llx "
+					"IO %016llx ",
+					i, data1, data2, data3);
+			len += snprintf(buf + len, PAGE_SIZE - len,
+					"Cmpl %016llx OutIO %016llx\n",
+					tot, ((data1 + data2 + data3) - tot));
+		}
+		len += snprintf(buf + len, PAGE_SIZE - len,
+				"Total FCP Cmpl %016llx Issue %016llx "
+				"OutIO %016llx\n",
+				totin, totout, totout - totin);
 
 		len += snprintf(buf + len, size - len,
 				"LS Xmt Err: Abrt %08x Err %08x  "
@@ -918,11 +959,12 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 
 		len += snprintf(buf + len, size - len,
 				"FCP Xmt Err: noxri %06x nondlp %06x "
-				"qdepth %06x wqerr %06x Abrt %06x\n",
+				"qdepth %06x wqerr %06x err %06x Abrt %06x\n",
 				atomic_read(&lport->xmt_fcp_noxri),
 				atomic_read(&lport->xmt_fcp_bad_ndlp),
 				atomic_read(&lport->xmt_fcp_qdepth),
 				atomic_read(&lport->xmt_fcp_wqerr),
+				atomic_read(&lport->xmt_fcp_err),
 				atomic_read(&lport->xmt_fcp_abort));
 
 		len += snprintf(buf + len, size - len,
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index 74895e62aaea..6d84a10fef07 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -6268,7 +6268,6 @@ lpfc_els_handle_rscn(struct lpfc_vport *vport)
 		 * flush the RSCN.  Otherwise, the outstanding requests
 		 * need to complete.
 		 */
-		vport->gidft_inp = 0;
 		if (lpfc_issue_gidft(vport) > 0)
 			return 1;
 	} else {
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index 3e7712cd6c9a..2fef54fab86d 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -708,8 +708,7 @@ lpfc_work_done(struct lpfc_hba *phba)
 								HA_RXMASK));
 			}
 		}
-		if ((phba->sli_rev == LPFC_SLI_REV4) &&
-				 (!list_empty(&pring->txq)))
+		if (phba->sli_rev == LPFC_SLI_REV4)
 			lpfc_drain_txq(phba);
 		/*
 		 * Turn on Ring interrupts
@@ -3876,10 +3875,6 @@ int
 lpfc_issue_gidft(struct lpfc_vport *vport)
 {
 	struct lpfc_hba *phba = vport->phba;
-	struct lpfc_nodelist *ndlp;
-
-	list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp)
-		ndlp->nlp_fc4_type &= ~(NLP_FC4_FCP | NLP_FC4_NVME);
 
 	/* Good status, issue CT Request to NameServer */
 	if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) ||
diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h
index 98b80559c215..f43f0bacb77a 100644
--- a/drivers/scsi/lpfc/lpfc_hw4.h
+++ b/drivers/scsi/lpfc/lpfc_hw4.h
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2009-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -566,6 +566,7 @@ struct lpfc_register {
 
 /* The following BAR0 register sets are defined for if_type 0 and 2 UCNAs. */
 #define LPFC_SLI_INTF			0x0058
+#define LPFC_SLI_ASIC_VER		0x009C
 
 #define LPFC_CTL_PORT_SEM_OFFSET	0x400
 #define lpfc_port_smphr_perr_SHIFT	31
@@ -3912,6 +3913,7 @@ struct lpfc_acqe_link {
 #define LPFC_ASYNC_LINK_FAULT_NONE	0x0
 #define LPFC_ASYNC_LINK_FAULT_LOCAL	0x1
 #define LPFC_ASYNC_LINK_FAULT_REMOTE	0x2
+#define LPFC_ASYNC_LINK_FAULT_LR_LRR	0x3
 #define lpfc_acqe_logical_link_speed_SHIFT	16
 #define lpfc_acqe_logical_link_speed_MASK	0x0000FFFF
 #define lpfc_acqe_logical_link_speed_WORD	word1
@@ -4616,6 +4618,9 @@ union lpfc_wqe128 {
 	struct send_frame_wqe send_frame;
 };
 
+#define MAGIC_NUMER_G6 0xFEAA0003
+#define MAGIC_NUMER_G7 0xFEAA0005
+
 struct lpfc_grp_hdr {
 	uint32_t size;
 	uint32_t magic_number;
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 7887468c71b4..7ae343b14630 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -1266,6 +1266,9 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba)
 	uint64_t tot, data1, data2, data3;
 	struct lpfc_nvmet_tgtport *tgtp;
 	struct lpfc_register reg_data;
+	struct nvme_fc_local_port *localport;
+	struct lpfc_nvme_lport *lport;
+	struct lpfc_nvme_ctrl_stat *cstat;
 	void __iomem *eqdreg = phba->sli4_hba.u.if_type2.EQDregaddr;
 
 	vports = lpfc_create_vport_work_array(phba);
@@ -1299,14 +1302,25 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba)
 				tot += atomic_read(&tgtp->xmt_fcp_release);
 				tot = atomic_read(&tgtp->rcv_fcp_cmd_in) - tot;
 			} else {
-				tot = atomic_read(&phba->fc4NvmeIoCmpls);
-				data1 = atomic_read(
-					&phba->fc4NvmeInputRequests);
-				data2 = atomic_read(
-					&phba->fc4NvmeOutputRequests);
-				data3 = atomic_read(
-					&phba->fc4NvmeControlRequests);
-				tot =  (data1 + data2 + data3) - tot;
+				localport = phba->pport->localport;
+				if (!localport || !localport->private)
+					goto skip_eqdelay;
+				lport = (struct lpfc_nvme_lport *)
+					localport->private;
+				tot = 0;
+				for (i = 0;
+					i < phba->cfg_nvme_io_channel; i++) {
+					cstat = &lport->cstat[i];
+					data1 = atomic_read(
+						&cstat->fc4NvmeInputRequests);
+					data2 = atomic_read(
+						&cstat->fc4NvmeOutputRequests);
+					data3 = atomic_read(
+						&cstat->fc4NvmeControlRequests);
+					tot += (data1 + data2 + data3);
+					tot -= atomic_read(
+						&cstat->fc4NvmeIoCmpls);
+				}
 			}
 		}
 
@@ -4265,32 +4279,24 @@ lpfc_sli4_fcf_redisc_wait_tmo(struct timer_list *t)
  * @phba: pointer to lpfc hba data structure.
  * @acqe_link: pointer to the async link completion queue entry.
  *
- * This routine is to parse the SLI4 link-attention link fault code and
- * translate it into the base driver's read link attention mailbox command
- * status.
- *
- * Return: Link-attention status in terms of base driver's coding.
+ * This routine is to parse the SLI4 link-attention link fault code.
  **/
-static uint16_t
+static void
 lpfc_sli4_parse_latt_fault(struct lpfc_hba *phba,
 			   struct lpfc_acqe_link *acqe_link)
 {
-	uint16_t latt_fault;
-
 	switch (bf_get(lpfc_acqe_link_fault, acqe_link)) {
 	case LPFC_ASYNC_LINK_FAULT_NONE:
 	case LPFC_ASYNC_LINK_FAULT_LOCAL:
 	case LPFC_ASYNC_LINK_FAULT_REMOTE:
-		latt_fault = 0;
+	case LPFC_ASYNC_LINK_FAULT_LR_LRR:
 		break;
 	default:
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-				"0398 Invalid link fault code: x%x\n",
+				"0398 Unknown link fault code: x%x\n",
 				bf_get(lpfc_acqe_link_fault, acqe_link));
-		latt_fault = MBXERR_ERROR;
 		break;
 	}
-	return latt_fault;
 }
 
 /**
@@ -4565,9 +4571,12 @@ lpfc_sli4_async_link_evt(struct lpfc_hba *phba,
 	 * the READ_TOPOLOGY completion routine to continue without actually
 	 * sending the READ_TOPOLOGY mailbox command to the port.
 	 */
-	/* Parse and translate status field */
+	/* Initialize completion status */
 	mb = &pmb->u.mb;
-	mb->mbxStatus = lpfc_sli4_parse_latt_fault(phba, acqe_link);
+	mb->mbxStatus = MBX_SUCCESS;
+
+	/* Parse port fault information field */
+	lpfc_sli4_parse_latt_fault(phba, acqe_link);
 
 	/* Parse and translate link attention fields */
 	la = (struct lpfc_mbx_read_top *) &pmb->u.mb.un.varReadTop;
@@ -4695,10 +4704,12 @@ lpfc_sli4_async_fc_evt(struct lpfc_hba *phba, struct lpfc_acqe_fc_la *acqe_fc)
 			break;
 		}
 
-		/* Parse and translate status field */
+		/* Initialize completion status */
 		mb = &pmb->u.mb;
-		mb->mbxStatus = lpfc_sli4_parse_latt_fault(phba,
-							   (void *)acqe_fc);
+		mb->mbxStatus = MBX_SUCCESS;
+
+		/* Parse port fault information field */
+		lpfc_sli4_parse_latt_fault(phba, (void *)acqe_fc);
 
 		/* Parse and translate link attention fields */
 		la = (struct lpfc_mbx_read_top *)&pmb->u.mb.un.varReadTop;
@@ -5103,7 +5114,7 @@ lpfc_sli4_async_fip_evt(struct lpfc_hba *phba,
 		if (rc) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_FIP |
 					LOG_DISCOVERY,
-					"2772 Issue FCF rediscover mabilbox "
+					"2772 Issue FCF rediscover mailbox "
 					"command failed, fail through to FCF "
 					"dead event\n");
 			spin_lock_irq(&phba->hbalock);
@@ -5195,7 +5206,7 @@ lpfc_sli4_async_fip_evt(struct lpfc_hba *phba,
 				lpfc_printf_log(phba, KERN_ERR, LOG_FIP |
 						LOG_DISCOVERY,
 						"2774 Issue FCF rediscover "
-						"mabilbox command failed, "
+						"mailbox command failed, "
 						"through to CVL event\n");
 				spin_lock_irq(&phba->hbalock);
 				phba->fcf.fcf_flag &= ~FCF_ACVL_DISC;
@@ -5839,6 +5850,8 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 	int fof_vectors = 0;
 	int extra;
 	uint64_t wwn;
+	u32 if_type;
+	u32 if_fam;
 
 	phba->sli4_hba.num_online_cpu = num_online_cpus();
 	phba->sli4_hba.num_present_cpu = lpfc_present_cpu;
@@ -6160,15 +6173,28 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 	 */
 	rc = lpfc_get_sli4_parameters(phba, mboxq);
 	if (rc) {
+		if_type = bf_get(lpfc_sli_intf_if_type,
+				 &phba->sli4_hba.sli_intf);
+		if_fam = bf_get(lpfc_sli_intf_sli_family,
+				&phba->sli4_hba.sli_intf);
 		if (phba->sli4_hba.extents_in_use &&
 		    phba->sli4_hba.rpi_hdrs_in_use) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"2999 Unsupported SLI4 Parameters "
 				"Extents and RPI headers enabled.\n");
+			if (if_type == LPFC_SLI_INTF_IF_TYPE_0 &&
+			    if_fam ==  LPFC_SLI_INTF_FAMILY_BE2) {
+				mempool_free(mboxq, phba->mbox_mem_pool);
+				rc = -EIO;
+				goto out_free_bsmbx;
+			}
+		}
+		if (!(if_type == LPFC_SLI_INTF_IF_TYPE_0 &&
+		      if_fam == LPFC_SLI_INTF_FAMILY_BE2)) {
+			mempool_free(mboxq, phba->mbox_mem_pool);
+			rc = -EIO;
+			goto out_free_bsmbx;
 		}
-		mempool_free(mboxq, phba->mbox_mem_pool);
-		rc = -EIO;
-		goto out_free_bsmbx;
 	}
 
 	mempool_free(mboxq, phba->mbox_mem_pool);
@@ -6406,8 +6432,11 @@ lpfc_setup_driver_resource_phase2(struct lpfc_hba *phba)
 		return error;
 	}
 
-	/* workqueue for deferred irq use */
-	phba->wq = alloc_workqueue("lpfc_wq", WQ_MEM_RECLAIM, 0);
+	/* The lpfc_wq workqueue for deferred irq use, is only used for SLI4 */
+	if (phba->sli_rev == LPFC_SLI_REV4)
+		phba->wq = alloc_workqueue("lpfc_wq", WQ_MEM_RECLAIM, 0);
+	else
+		phba->wq = NULL;
 
 	return 0;
 }
@@ -6430,7 +6459,8 @@ lpfc_unset_driver_resource_phase2(struct lpfc_hba *phba)
 	}
 
 	/* Stop kernel worker thread */
-	kthread_stop(phba->worker_thread);
+	if (phba->worker_thread)
+		kthread_stop(phba->worker_thread);
 }
 
 /**
@@ -6895,12 +6925,6 @@ lpfc_create_shost(struct lpfc_hba *phba)
 	atomic_set(&phba->fc4ScsiOutputRequests, 0);
 	atomic_set(&phba->fc4ScsiControlRequests, 0);
 	atomic_set(&phba->fc4ScsiIoCmpls, 0);
-	atomic_set(&phba->fc4NvmeInputRequests, 0);
-	atomic_set(&phba->fc4NvmeOutputRequests, 0);
-	atomic_set(&phba->fc4NvmeControlRequests, 0);
-	atomic_set(&phba->fc4NvmeIoCmpls, 0);
-	atomic_set(&phba->fc4NvmeLsRequests, 0);
-	atomic_set(&phba->fc4NvmeLsCmpls, 0);
 	vport = lpfc_create_port(phba, phba->brd_no, &phba->pcidev->dev);
 	if (!vport)
 		return -ENODEV;
@@ -7781,6 +7805,40 @@ lpfc_sli4_read_config(struct lpfc_hba *phba)
 				phba->sli4_hba.max_cfg_param.max_wq,
 				phba->sli4_hba.max_cfg_param.max_rq);
 
+		/*
+		 * Calculate NVME queue resources based on how
+		 * many WQ/CQs are available.
+		 */
+		if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
+			length = phba->sli4_hba.max_cfg_param.max_wq;
+			if (phba->sli4_hba.max_cfg_param.max_cq <
+			    phba->sli4_hba.max_cfg_param.max_wq)
+				length = phba->sli4_hba.max_cfg_param.max_cq;
+
+			/*
+			 * Whats left after this can go toward NVME.
+			 * The minus 6 accounts for ELS, NVME LS, MBOX
+			 * fof plus a couple extra. When configured for
+			 * NVMET, FCP io channel WQs are not created.
+			 */
+			length -= 6;
+			if (!phba->nvmet_support)
+				length -= phba->cfg_fcp_io_channel;
+
+			if (phba->cfg_nvme_io_channel > length) {
+				lpfc_printf_log(
+					phba, KERN_ERR, LOG_SLI,
+					"2005 Reducing NVME IO channel to %d: "
+					"WQ %d CQ %d NVMEIO %d FCPIO %d\n",
+					length,
+					phba->sli4_hba.max_cfg_param.max_wq,
+					phba->sli4_hba.max_cfg_param.max_cq,
+					phba->cfg_nvme_io_channel,
+					phba->cfg_fcp_io_channel);
+
+				phba->cfg_nvme_io_channel = length;
+			}
+		}
 	}
 
 	if (rc)
@@ -10533,6 +10591,7 @@ lpfc_get_sli4_parameters(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 	struct lpfc_pc_sli4_params *sli4_params;
 	uint32_t mbox_tmo;
 	int length;
+	bool exp_wqcq_pages = true;
 	struct lpfc_sli4_parameters *mbx_sli4_parameters;
 
 	/*
@@ -10659,8 +10718,15 @@ lpfc_get_sli4_parameters(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 			phba->nvme_support, phba->nvme_embed_pbde,
 			phba->cfg_nvme_embed_cmd, phba->cfg_suppress_rsp);
 
+	if ((bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) ==
+	    LPFC_SLI_INTF_IF_TYPE_2) &&
+	    (bf_get(lpfc_sli_intf_sli_family, &phba->sli4_hba.sli_intf) ==
+		 LPFC_SLI_INTF_FAMILY_LNCR_A0))
+		exp_wqcq_pages = false;
+
 	if ((bf_get(cfg_cqpsize, mbx_sli4_parameters) & LPFC_CQ_16K_PAGE_SZ) &&
 	    (bf_get(cfg_wqpsize, mbx_sli4_parameters) & LPFC_WQ_16K_PAGE_SZ) &&
+	    exp_wqcq_pages &&
 	    (sli4_params->wqsize & LPFC_WQ_SZ128_SUPPORT))
 		phba->enab_exp_wqcq_pages = 1;
 	else
@@ -11322,7 +11388,11 @@ lpfc_log_write_firmware_error(struct lpfc_hba *phba, uint32_t offset,
 	uint32_t magic_number, uint32_t ftype, uint32_t fid, uint32_t fsize,
 	const struct firmware *fw)
 {
-	if (offset == ADD_STATUS_FW_NOT_SUPPORTED)
+	if ((offset == ADD_STATUS_FW_NOT_SUPPORTED) ||
+	    (phba->pcidev->device == PCI_DEVICE_ID_LANCER_G6_FC &&
+	     magic_number != MAGIC_NUMER_G6) ||
+	    (phba->pcidev->device == PCI_DEVICE_ID_LANCER_G7_FC &&
+	     magic_number != MAGIC_NUMER_G7))
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 			"3030 This firmware version is not supported on "
 			"this HBA model. Device:%x Magic:%x Type:%x "
@@ -11719,6 +11789,7 @@ lpfc_pci_remove_one_s4(struct pci_dev *pdev)
 	lpfc_nvme_free(phba);
 	lpfc_free_iocb_list(phba);
 
+	lpfc_unset_driver_resource_phase2(phba);
 	lpfc_sli4_driver_resource_unset(phba);
 
 	/* Unmap adapter Control and Doorbell registers */
diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c
index 022060636ae1..1a803975bcbc 100644
--- a/drivers/scsi/lpfc/lpfc_nportdisc.c
+++ b/drivers/scsi/lpfc/lpfc_nportdisc.c
@@ -1936,31 +1936,14 @@ lpfc_cmpl_prli_prli_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 			goto out;
 		}
 
-		/* When the rport rejected the FCP PRLI as unsupported.
-		 * This should only happen in Pt2Pt so an NVME PRLI
-		 * should be outstanding still.
-		 */
-		if (npr && ndlp->nlp_flag & NLP_FCP_PRLI_RJT) {
+		/* Adjust the nlp_type accordingly if the PRLI failed */
+		if (npr)
 			ndlp->nlp_fc4_type &= ~NLP_FC4_FCP;
-			goto out_err;
-		}
-
-		/* The LS Req had some error.  Don't let this be a
-		 * target.
-		 */
-		if ((ndlp->fc4_prli_sent == 1) &&
-		    (ndlp->nlp_state == NLP_STE_PRLI_ISSUE) &&
-		    (ndlp->nlp_type & (NLP_FCP_TARGET | NLP_FCP_INITIATOR)))
-			/* The FCP PRLI completed successfully but
-			 * the NVME PRLI failed.  Since they are sent in
-			 * succession, allow the FCP to complete.
-			 */
-			goto out_err;
+		if (nvpr)
+			ndlp->nlp_fc4_type &= ~NLP_FC4_NVME;
 
-		ndlp->nlp_prev_state = NLP_STE_PRLI_ISSUE;
-		ndlp->nlp_type |= NLP_FCP_INITIATOR;
-		lpfc_nlp_set_state(vport, ndlp, NLP_STE_UNMAPPED_NODE);
-		return ndlp->nlp_state;
+		/* We can't set the DSM state till BOTH PRLIs complete */
+		goto out_err;
 	}
 
 	if (npr && (npr->acceptRspCode == PRLI_REQ_EXECUTED) &&
@@ -1999,6 +1982,12 @@ lpfc_cmpl_prli_prli_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 			if (bf_get_be32(prli_disc, nvpr))
 				ndlp->nlp_type |= NLP_NVME_DISCOVERY;
 
+			/* This node is an NVME target.  Adjust the command
+			 * queue depth on this node to not exceed the available
+			 * xris.
+			 */
+			ndlp->cmd_qdepth = phba->sli4_hba.nvme_xri_max;
+
 			/*
 			 * If prli_fba is set, the Target supports FirstBurst.
 			 * If prli_fb_sz is 0, the FirstBurst size is unlimited,
diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c
index 378dca40ca20..76a5a99605aa 100644
--- a/drivers/scsi/lpfc/lpfc_nvme.c
+++ b/drivers/scsi/lpfc/lpfc_nvme.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -334,7 +334,14 @@ lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
 			"6146 remoteport delete of remoteport %p\n",
 			remoteport);
 	spin_lock_irq(&vport->phba->hbalock);
-	ndlp->nrport = NULL;
+
+	/* The register rebind might have occurred before the delete
+	 * downcall.  Guard against this race.
+	 */
+	if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) {
+		ndlp->nrport = NULL;
+		ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
+	}
 	spin_unlock_irq(&vport->phba->hbalock);
 
 	/* Remove original register reference. The host transport
@@ -357,15 +364,19 @@ lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 	struct lpfc_dmabuf *buf_ptr;
 	struct lpfc_nodelist *ndlp;
 
-	atomic_inc(&vport->phba->fc4NvmeLsCmpls);
-
 	pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2;
 	status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
-	if (status) {
+
+	if (vport->localport) {
 		lport = (struct lpfc_nvme_lport *)vport->localport->private;
-		if (bf_get(lpfc_wcqe_c_xb, wcqe))
-			atomic_inc(&lport->cmpl_ls_xb);
-		atomic_inc(&lport->cmpl_ls_err);
+		if (lport) {
+			atomic_inc(&lport->fc4NvmeLsCmpls);
+			if (status) {
+				if (bf_get(lpfc_wcqe_c_xb, wcqe))
+					atomic_inc(&lport->cmpl_ls_xb);
+				atomic_inc(&lport->cmpl_ls_err);
+			}
+		}
 	}
 
 	ndlp = (struct lpfc_nodelist *)cmdwqe->context1;
@@ -570,6 +581,9 @@ lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
 
 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
 	rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
+	if (unlikely(!lport) || unlikely(!rport))
+		return -EINVAL;
+
 	vport = lport->vport;
 
 	if (vport->load_flag & FC_UNLOADING)
@@ -639,7 +653,7 @@ lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
 			 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
 			 &pnvme_lsreq->rspdma);
 
-	atomic_inc(&vport->phba->fc4NvmeLsRequests);
+	atomic_inc(&lport->fc4NvmeLsRequests);
 
 	/* Hardcode the wait to 30 seconds.  Connections are failing otherwise.
 	 * This code allows it all to work.
@@ -690,6 +704,8 @@ lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
 	struct lpfc_iocbq *wqe, *next_wqe;
 
 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
+	if (unlikely(!lport))
+		return;
 	vport = lport->vport;
 	phba = vport->phba;
 
@@ -949,28 +965,48 @@ lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
 	struct lpfc_nodelist *ndlp;
 	struct lpfc_nvme_fcpreq_priv *freqpriv;
 	struct lpfc_nvme_lport *lport;
+	struct lpfc_nvme_ctrl_stat *cstat;
 	unsigned long flags;
-	uint32_t code, status;
+	uint32_t code, status, idx;
 	uint16_t cid, sqhd, data;
 	uint32_t *ptr;
 
 	/* Sanity check on return of outstanding command */
 	if (!lpfc_ncmd || !lpfc_ncmd->nvmeCmd || !lpfc_ncmd->nrport) {
+		if (!lpfc_ncmd) {
+			lpfc_printf_vlog(vport, KERN_ERR,
+					 LOG_NODE | LOG_NVME_IOERR,
+					 "6071 Null lpfc_ncmd pointer. No "
+					 "release, skip completion\n");
+			return;
+		}
+
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
-				 "6071 Completion pointers bad on wqe %p.\n",
-				 wcqe);
+				 "6066 Missing cmpl ptrs: lpfc_ncmd %p, "
+				 "nvmeCmd %p nrport %p\n",
+				 lpfc_ncmd, lpfc_ncmd->nvmeCmd,
+				 lpfc_ncmd->nrport);
+
+		/* Release the lpfc_ncmd regardless of the missing elements. */
+		lpfc_release_nvme_buf(phba, lpfc_ncmd);
 		return;
 	}
-	atomic_inc(&phba->fc4NvmeIoCmpls);
-
 	nCmd = lpfc_ncmd->nvmeCmd;
 	rport = lpfc_ncmd->nrport;
 	status = bf_get(lpfc_wcqe_c_status, wcqe);
-	if (status) {
+
+	if (vport->localport) {
 		lport = (struct lpfc_nvme_lport *)vport->localport->private;
-		if (bf_get(lpfc_wcqe_c_xb, wcqe))
-			atomic_inc(&lport->cmpl_fcp_xb);
-		atomic_inc(&lport->cmpl_fcp_err);
+		if (lport) {
+			idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
+			cstat = &lport->cstat[idx];
+			atomic_inc(&cstat->fc4NvmeIoCmpls);
+			if (status) {
+				if (bf_get(lpfc_wcqe_c_xb, wcqe))
+					atomic_inc(&lport->cmpl_fcp_xb);
+				atomic_inc(&lport->cmpl_fcp_err);
+			}
+		}
 	}
 
 	lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
@@ -1163,7 +1199,8 @@ lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
 static int
 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
 		      struct lpfc_nvme_buf *lpfc_ncmd,
-		      struct lpfc_nodelist *pnode)
+		      struct lpfc_nodelist *pnode,
+		      struct lpfc_nvme_ctrl_stat *cstat)
 {
 	struct lpfc_hba *phba = vport->phba;
 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
@@ -1201,7 +1238,7 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
 			} else {
 				wqe->fcp_iwrite.initial_xfer_len = 0;
 			}
-			atomic_inc(&phba->fc4NvmeOutputRequests);
+			atomic_inc(&cstat->fc4NvmeOutputRequests);
 		} else {
 			/* From the iread template, initialize words 7 - 11 */
 			memcpy(&wqe->words[7],
@@ -1214,13 +1251,13 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
 			/* Word 5 */
 			wqe->fcp_iread.rsrvd5 = 0;
 
-			atomic_inc(&phba->fc4NvmeInputRequests);
+			atomic_inc(&cstat->fc4NvmeInputRequests);
 		}
 	} else {
 		/* From the icmnd template, initialize words 4 - 11 */
 		memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
 		       sizeof(uint32_t) * 8);
-		atomic_inc(&phba->fc4NvmeControlRequests);
+		atomic_inc(&cstat->fc4NvmeControlRequests);
 	}
 	/*
 	 * Finish initializing those WQE fields that are independent
@@ -1400,7 +1437,9 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 {
 	int ret = 0;
 	int expedite = 0;
+	int idx;
 	struct lpfc_nvme_lport *lport;
+	struct lpfc_nvme_ctrl_stat *cstat;
 	struct lpfc_vport *vport;
 	struct lpfc_hba *phba;
 	struct lpfc_nodelist *ndlp;
@@ -1425,9 +1464,10 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 	vport = lport->vport;
 
 	if (unlikely(!hw_queue_handle)) {
-		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
-				 "6129 Fail Abort, NULL hw_queue_handle\n");
-		ret = -EINVAL;
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
+				 "6117 Fail IO, NULL hw_queue_handle\n");
+		atomic_inc(&lport->xmt_fcp_err);
+		ret = -EBUSY;
 		goto out_fail;
 	}
 
@@ -1439,12 +1479,18 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 	}
 
 	if (vport->load_flag & FC_UNLOADING) {
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
+				 "6124 Fail IO, Driver unload\n");
+		atomic_inc(&lport->xmt_fcp_err);
 		ret = -ENODEV;
 		goto out_fail;
 	}
 
 	freqpriv = pnvme_fcreq->private;
 	if (unlikely(!freqpriv)) {
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
+				 "6158 Fail IO, NULL request data\n");
+		atomic_inc(&lport->xmt_fcp_err);
 		ret = -EINVAL;
 		goto out_fail;
 	}
@@ -1462,32 +1508,26 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 	 */
 	ndlp = rport->ndlp;
 	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
-		lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
-				 "6053 rport %p, ndlp %p, DID x%06x "
-				 "ndlp not ready.\n",
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
+				 "6053 Fail IO, ndlp not ready: rport %p "
+				  "ndlp %p, DID x%06x\n",
 				 rport, ndlp, pnvme_rport->port_id);
-
-		ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
-		if (!ndlp) {
-			lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
-					 "6066 Missing node for DID %x\n",
-					 pnvme_rport->port_id);
-			atomic_inc(&lport->xmt_fcp_bad_ndlp);
-			ret = -ENODEV;
-			goto out_fail;
-		}
+		atomic_inc(&lport->xmt_fcp_err);
+		ret = -EBUSY;
+		goto out_fail;
 	}
 
 	/* The remote node has to be a mapped target or it's an error. */
 	if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
 	    (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
-		lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
-				 "6036 rport %p, DID x%06x not ready for "
-				 "IO. State x%x, Type x%x\n",
-				 rport, pnvme_rport->port_id,
-				 ndlp->nlp_state, ndlp->nlp_type);
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
+				 "6036 Fail IO, DID x%06x not ready for "
+				 "IO. State x%x, Type x%x Flg x%x\n",
+				 pnvme_rport->port_id,
+				 ndlp->nlp_state, ndlp->nlp_type,
+				 ndlp->upcall_flags);
 		atomic_inc(&lport->xmt_fcp_bad_ndlp);
-		ret = -ENODEV;
+		ret = -EBUSY;
 		goto out_fail;
 
 	}
@@ -1508,6 +1548,12 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 	 */
 	if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
 	    !expedite) {
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
+				 "6174 Fail IO, ndlp qdepth exceeded: "
+				 "idx %d DID %x pend %d qdepth %d\n",
+				 lpfc_queue_info->index, ndlp->nlp_DID,
+				 atomic_read(&ndlp->cmd_pending),
+				 ndlp->cmd_qdepth);
 		atomic_inc(&lport->xmt_fcp_qdepth);
 		ret = -EBUSY;
 		goto out_fail;
@@ -1517,8 +1563,9 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 	if (lpfc_ncmd == NULL) {
 		atomic_inc(&lport->xmt_fcp_noxri);
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
-				 "6065 driver's buffer pool is empty, "
-				 "IO failed\n");
+				 "6065 Fail IO, driver buffer pool is empty: "
+				 "idx %d DID %x\n",
+				 lpfc_queue_info->index, ndlp->nlp_DID);
 		ret = -EBUSY;
 		goto out_fail;
 	}
@@ -1543,15 +1590,6 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 	lpfc_ncmd->ndlp = ndlp;
 	lpfc_ncmd->start_time = jiffies;
 
-	lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp);
-	ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
-	if (ret) {
-		ret = -ENOMEM;
-		goto out_free_nvme_buf;
-	}
-
-	atomic_inc(&ndlp->cmd_pending);
-
 	/*
 	 * Issue the IO on the WQ indicated by index in the hw_queue_handle.
 	 * This identfier was create in our hardware queue create callback
@@ -1560,7 +1598,23 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 	 * index to use and that they have affinitized a CPU to this hardware
 	 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
 	 */
-	lpfc_ncmd->cur_iocbq.hba_wqidx = lpfc_queue_info->index;
+	idx = lpfc_queue_info->index;
+	lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
+	cstat = &lport->cstat[idx];
+
+	lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
+	ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
+	if (ret) {
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
+				 "6175 Fail IO, Prep DMA: "
+				 "idx %d DID %x\n",
+				 lpfc_queue_info->index, ndlp->nlp_DID);
+		atomic_inc(&lport->xmt_fcp_err);
+		ret = -ENOMEM;
+		goto out_free_nvme_buf;
+	}
+
+	atomic_inc(&ndlp->cmd_pending);
 
 	lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
 			 lpfc_ncmd->cur_iocbq.sli4_xritag,
@@ -1571,7 +1625,7 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 		atomic_inc(&lport->xmt_fcp_wqerr);
 		atomic_dec(&ndlp->cmd_pending);
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
-				 "6113 FCP could not issue WQE err %x "
+				 "6113 Fail IO, Could not issue WQE err %x "
 				 "sid: x%x did: x%x oxid: x%x\n",
 				 ret, vport->fc_myDID, ndlp->nlp_DID,
 				 lpfc_ncmd->cur_iocbq.sli4_xritag);
@@ -1605,11 +1659,11 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
  out_free_nvme_buf:
 	if (lpfc_ncmd->nvmeCmd->sg_cnt) {
 		if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
-			atomic_dec(&phba->fc4NvmeOutputRequests);
+			atomic_dec(&cstat->fc4NvmeOutputRequests);
 		else
-			atomic_dec(&phba->fc4NvmeInputRequests);
+			atomic_dec(&cstat->fc4NvmeInputRequests);
 	} else
-		atomic_dec(&phba->fc4NvmeControlRequests);
+		atomic_dec(&cstat->fc4NvmeControlRequests);
 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
  out_fail:
 	return ret;
@@ -2390,7 +2444,8 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
 	struct nvme_fc_port_info nfcp_info;
 	struct nvme_fc_local_port *localport;
 	struct lpfc_nvme_lport *lport;
-	int len;
+	struct lpfc_nvme_ctrl_stat *cstat;
+	int len, i;
 
 	/* Initialize this localport instance.  The vport wwn usage ensures
 	 * that NPIV is accounted for.
@@ -2414,6 +2469,11 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
 	lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
 	lpfc_nvme_template.max_hw_queues = phba->cfg_nvme_io_channel;
 
+	cstat = kmalloc((sizeof(struct lpfc_nvme_ctrl_stat) *
+			phba->cfg_nvme_io_channel), GFP_KERNEL);
+	if (!cstat)
+		return -ENOMEM;
+
 	/* localport is allocated from the stack, but the registration
 	 * call allocates heap memory as well as the private area.
 	 */
@@ -2436,11 +2496,13 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
 		lport = (struct lpfc_nvme_lport *)localport->private;
 		vport->localport = localport;
 		lport->vport = vport;
+		lport->cstat = cstat;
 		vport->nvmei_support = 1;
 
 		atomic_set(&lport->xmt_fcp_noxri, 0);
 		atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
 		atomic_set(&lport->xmt_fcp_qdepth, 0);
+		atomic_set(&lport->xmt_fcp_err, 0);
 		atomic_set(&lport->xmt_fcp_wqerr, 0);
 		atomic_set(&lport->xmt_fcp_abort, 0);
 		atomic_set(&lport->xmt_ls_abort, 0);
@@ -2449,6 +2511,16 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
 		atomic_set(&lport->cmpl_fcp_err, 0);
 		atomic_set(&lport->cmpl_ls_xb, 0);
 		atomic_set(&lport->cmpl_ls_err, 0);
+		atomic_set(&lport->fc4NvmeLsRequests, 0);
+		atomic_set(&lport->fc4NvmeLsCmpls, 0);
+
+		for (i = 0; i < phba->cfg_nvme_io_channel; i++) {
+			cstat = &lport->cstat[i];
+			atomic_set(&cstat->fc4NvmeInputRequests, 0);
+			atomic_set(&cstat->fc4NvmeOutputRequests, 0);
+			atomic_set(&cstat->fc4NvmeControlRequests, 0);
+			atomic_set(&cstat->fc4NvmeIoCmpls, 0);
+		}
 
 		/* Don't post more new bufs if repost already recovered
 		 * the nvme sgls.
@@ -2458,6 +2530,8 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
 						 phba->sli4_hba.nvme_xri_max);
 			vport->phba->total_nvme_bufs += len;
 		}
+	} else {
+		kfree(cstat);
 	}
 
 	return ret;
@@ -2520,6 +2594,7 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
 #if (IS_ENABLED(CONFIG_NVME_FC))
 	struct nvme_fc_local_port *localport;
 	struct lpfc_nvme_lport *lport;
+	struct lpfc_nvme_ctrl_stat *cstat;
 	int ret;
 
 	if (vport->nvmei_support == 0)
@@ -2528,6 +2603,7 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
 	localport = vport->localport;
 	vport->localport = NULL;
 	lport = (struct lpfc_nvme_lport *)localport->private;
+	cstat = lport->cstat;
 
 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
 			 "6011 Destroying NVME localport %p\n",
@@ -2543,6 +2619,7 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
 	 * indefinitely or succeeds
 	 */
 	lpfc_nvme_lport_unreg_wait(vport, lport);
+	kfree(cstat);
 
 	/* Regardless of the unregister upcall response, clear
 	 * nvmei_support.  All rports are unregistered and the
@@ -2607,6 +2684,7 @@ lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 	struct nvme_fc_local_port *localport;
 	struct lpfc_nvme_lport *lport;
 	struct lpfc_nvme_rport *rport;
+	struct lpfc_nvme_rport *oldrport;
 	struct nvme_fc_remote_port *remote_port;
 	struct nvme_fc_port_info rpinfo;
 	struct lpfc_nodelist *prev_ndlp;
@@ -2639,7 +2717,9 @@ lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 
 	rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
 	rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
-	if (!ndlp->nrport)
+
+	oldrport = lpfc_ndlp_get_nrport(ndlp);
+	if (!oldrport)
 		lpfc_nlp_get(ndlp);
 
 	ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
@@ -2648,9 +2728,15 @@ lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 		 * a resume of the existing rport.  Else this is a
 		 * new rport.
 		 */
+		/* Guard against an unregister/reregister
+		 * race that leaves the WAIT flag set.
+		 */
+		spin_lock_irq(&vport->phba->hbalock);
+		ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
+		spin_unlock_irq(&vport->phba->hbalock);
 		rport = remote_port->private;
-		if (ndlp->nrport) {
-			if (ndlp->nrport == remote_port->private) {
+		if (oldrport) {
+			if (oldrport == remote_port->private) {
 				/* Same remoteport.  Just reuse. */
 				lpfc_printf_vlog(ndlp->vport, KERN_INFO,
 						 LOG_NVME_DISC,
@@ -2674,11 +2760,20 @@ lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 			 */
 			spin_lock_irq(&vport->phba->hbalock);
 			ndlp->nrport = NULL;
+			ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
 			spin_unlock_irq(&vport->phba->hbalock);
 			rport->ndlp = NULL;
 			rport->remoteport = NULL;
-			if (prev_ndlp)
-				lpfc_nlp_put(ndlp);
+
+			/* Reference only removed if previous NDLP is no longer
+			 * active. It might be just a swap and removing the
+			 * reference would cause a premature cleanup.
+			 */
+			if (prev_ndlp && prev_ndlp != ndlp) {
+				if ((!NLP_CHK_NODE_ACT(prev_ndlp)) ||
+				    (!prev_ndlp->nrport))
+					lpfc_nlp_put(prev_ndlp);
+			}
 		}
 
 		/* Clean bind the rport to the ndlp. */
@@ -2746,7 +2841,7 @@ lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 	if (!lport)
 		goto input_err;
 
-	rport = ndlp->nrport;
+	rport = lpfc_ndlp_get_nrport(ndlp);
 	if (!rport)
 		goto input_err;
 
@@ -2767,6 +2862,15 @@ lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 		 * The transport will update it.
 		 */
 		ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG;
+
+		/* Don't let the host nvme transport keep sending keep-alives
+		 * on this remoteport. Vport is unloading, no recovery. The
+		 * return values is ignored.  The upcall is a courtesy to the
+		 * transport.
+		 */
+		if (vport->load_flag & FC_UNLOADING)
+			(void)nvme_fc_set_remoteport_devloss(remoteport, 0);
+
 		ret = nvme_fc_unregister_remoteport(remoteport);
 		if (ret != 0) {
 			lpfc_nlp_put(ndlp);
diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h
index 9216653e0441..04bd463dd043 100644
--- a/drivers/scsi/lpfc/lpfc_nvme.h
+++ b/drivers/scsi/lpfc/lpfc_nvme.h
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -30,21 +30,36 @@
 #define LPFC_NVME_FB_SHIFT		9
 #define LPFC_NVME_MAX_FB		(1 << 20)	/* 1M */
 
+#define lpfc_ndlp_get_nrport(ndlp)					\
+	((!ndlp->nrport || (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG))	\
+	? NULL : ndlp->nrport)
+
 struct lpfc_nvme_qhandle {
 	uint32_t index;		/* WQ index to use */
 	uint32_t qidx;		/* queue index passed to create */
 	uint32_t cpu_id;	/* current cpu id at time of create */
 };
 
+struct lpfc_nvme_ctrl_stat {
+	atomic_t fc4NvmeInputRequests;
+	atomic_t fc4NvmeOutputRequests;
+	atomic_t fc4NvmeControlRequests;
+	atomic_t fc4NvmeIoCmpls;
+};
+
 /* Declare nvme-based local and remote port definitions. */
 struct lpfc_nvme_lport {
 	struct lpfc_vport *vport;
 	struct completion lport_unreg_done;
 	/* Add stats counters here */
+	struct lpfc_nvme_ctrl_stat *cstat;
+	atomic_t fc4NvmeLsRequests;
+	atomic_t fc4NvmeLsCmpls;
 	atomic_t xmt_fcp_noxri;
 	atomic_t xmt_fcp_bad_ndlp;
 	atomic_t xmt_fcp_qdepth;
 	atomic_t xmt_fcp_wqerr;
+	atomic_t xmt_fcp_err;
 	atomic_t xmt_fcp_abort;
 	atomic_t xmt_ls_abort;
 	atomic_t xmt_ls_err;
diff --git a/drivers/scsi/lpfc/lpfc_nvmet.h b/drivers/scsi/lpfc/lpfc_nvmet.h
index c1bcef3f103c..81f520abfd64 100644
--- a/drivers/scsi/lpfc/lpfc_nvmet.h
+++ b/drivers/scsi/lpfc/lpfc_nvmet.h
@@ -22,8 +22,10 @@
  ********************************************************************/
 
 #define LPFC_NVMET_DEFAULT_SEGS		(64 + 1)	/* 256K IOs */
-#define LPFC_NVMET_RQE_DEF_COUNT	512
-#define LPFC_NVMET_SUCCESS_LEN	12
+#define LPFC_NVMET_RQE_MIN_POST		128
+#define LPFC_NVMET_RQE_DEF_POST		512
+#define LPFC_NVMET_RQE_DEF_COUNT	2048
+#define LPFC_NVMET_SUCCESS_LEN		12
 
 #define LPFC_NVMET_MRQ_OFF		0xffff
 #define LPFC_NVMET_MRQ_AUTO		0
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index 050f04418f5f..a94fb9f8bb44 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -1021,7 +1021,7 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
 		if (lpfc_test_rrq_active(phba, ndlp,
 					 lpfc_cmd->cur_iocbq.sli4_lxritag))
 			continue;
-		list_del(&lpfc_cmd->list);
+		list_del_init(&lpfc_cmd->list);
 		found = 1;
 		break;
 	}
@@ -1036,7 +1036,7 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
 			if (lpfc_test_rrq_active(
 				phba, ndlp, lpfc_cmd->cur_iocbq.sli4_lxritag))
 				continue;
-			list_del(&lpfc_cmd->list);
+			list_del_init(&lpfc_cmd->list);
 			found = 1;
 			break;
 		}
@@ -3983,9 +3983,6 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
 	}
 #endif
 
-	if (pnode && NLP_CHK_NODE_ACT(pnode))
-		atomic_dec(&pnode->cmd_pending);
-
 	if (lpfc_cmd->status) {
 		if (lpfc_cmd->status == IOSTAT_LOCAL_REJECT &&
 		    (lpfc_cmd->result & IOERR_DRVR_MASK))
@@ -4125,6 +4122,7 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
 		msecs_to_jiffies(vport->cfg_max_scsicmpl_time))) {
 		spin_lock_irqsave(shost->host_lock, flags);
 		if (pnode && NLP_CHK_NODE_ACT(pnode)) {
+			atomic_dec(&pnode->cmd_pending);
 			if (pnode->cmd_qdepth >
 				atomic_read(&pnode->cmd_pending) &&
 				(atomic_read(&pnode->cmd_pending) >
@@ -4138,16 +4136,8 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
 		}
 		spin_unlock_irqrestore(shost->host_lock, flags);
 	} else if (pnode && NLP_CHK_NODE_ACT(pnode)) {
-		if ((pnode->cmd_qdepth != vport->cfg_tgt_queue_depth) &&
-		    time_after(jiffies, pnode->last_change_time +
-			      msecs_to_jiffies(LPFC_TGTQ_INTERVAL))) {
-			spin_lock_irqsave(shost->host_lock, flags);
-			pnode->cmd_qdepth = vport->cfg_tgt_queue_depth;
-			pnode->last_change_time = jiffies;
-			spin_unlock_irqrestore(shost->host_lock, flags);
-		}
+		atomic_dec(&pnode->cmd_pending);
 	}
-
 	lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
 
 	spin_lock_irqsave(&phba->hbalock, flags);
@@ -4591,6 +4581,8 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
 				 ndlp->nlp_portname.u.wwn[7]);
 		goto out_tgt_busy;
 	}
+	atomic_inc(&ndlp->cmd_pending);
+
 	lpfc_cmd = lpfc_get_scsi_buf(phba, ndlp);
 	if (lpfc_cmd == NULL) {
 		lpfc_rampdown_queue_depth(phba);
@@ -4643,11 +4635,9 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
 
 	lpfc_scsi_prep_cmnd(vport, lpfc_cmd, ndlp);
 
-	atomic_inc(&ndlp->cmd_pending);
 	err = lpfc_sli_issue_iocb(phba, LPFC_FCP_RING,
 				  &lpfc_cmd->cur_iocbq, SLI_IOCB_RET_IOCB);
 	if (err) {
-		atomic_dec(&ndlp->cmd_pending);
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
 				 "3376 FCP could not issue IOCB err %x"
 				 "FCP cmd x%x <%d/%llu> "
@@ -4691,6 +4681,7 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
 	lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
 	lpfc_release_scsi_buf(phba, lpfc_cmd);
  out_host_busy:
+	atomic_dec(&ndlp->cmd_pending);
 	return SCSI_MLQUEUE_HOST_BUSY;
 
  out_tgt_busy:
@@ -4725,7 +4716,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd)
 	int ret = SUCCESS, status = 0;
 	struct lpfc_sli_ring *pring_s4;
 	int ret_val;
-	unsigned long flags, iflags;
+	unsigned long flags;
 	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(waitq);
 
 	status = fc_block_scsi_eh(cmnd);
@@ -4825,16 +4816,16 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd)
 	abtsiocb->iocb_cmpl = lpfc_sli_abort_fcp_cmpl;
 	abtsiocb->vport = vport;
 	if (phba->sli_rev == LPFC_SLI_REV4) {
-		pring_s4 = lpfc_sli4_calc_ring(phba, iocb);
+		pring_s4 = lpfc_sli4_calc_ring(phba, abtsiocb);
 		if (pring_s4 == NULL) {
 			ret = FAILED;
 			goto out_unlock;
 		}
 		/* Note: both hbalock and ring_lock must be set here */
-		spin_lock_irqsave(&pring_s4->ring_lock, iflags);
+		spin_lock(&pring_s4->ring_lock);
 		ret_val = __lpfc_sli_issue_iocb(phba, pring_s4->ringno,
 						abtsiocb, 0);
-		spin_unlock_irqrestore(&pring_s4->ring_lock, iflags);
+		spin_unlock(&pring_s4->ring_lock);
 	} else {
 		ret_val = __lpfc_sli_issue_iocb(phba, LPFC_FCP_RING,
 						abtsiocb, 0);
diff --git a/drivers/scsi/lpfc/lpfc_scsi.h b/drivers/scsi/lpfc/lpfc_scsi.h
index 8e38e0204c47..c38e4da71f5f 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.h
+++ b/drivers/scsi/lpfc/lpfc_scsi.h
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index cb17e2b2be81..4b70d53acb72 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -96,6 +96,34 @@ lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq)
 	return &iocbq->iocb;
 }
 
+#if defined(CONFIG_64BIT) && defined(__LITTLE_ENDIAN)
+/**
+ * lpfc_sli4_pcimem_bcopy - SLI4 memory copy function
+ * @srcp: Source memory pointer.
+ * @destp: Destination memory pointer.
+ * @cnt: Number of words required to be copied.
+ *       Must be a multiple of sizeof(uint64_t)
+ *
+ * This function is used for copying data between driver memory
+ * and the SLI WQ. This function also changes the endianness
+ * of each word if native endianness is different from SLI
+ * endianness. This function can be called with or without
+ * lock.
+ **/
+void
+lpfc_sli4_pcimem_bcopy(void *srcp, void *destp, uint32_t cnt)
+{
+	uint64_t *src = srcp;
+	uint64_t *dest = destp;
+	int i;
+
+	for (i = 0; i < (int)cnt; i += sizeof(uint64_t))
+		*dest++ = *src++;
+}
+#else
+#define lpfc_sli4_pcimem_bcopy(a, b, c) lpfc_sli_pcimem_bcopy(a, b, c)
+#endif
+
 /**
  * lpfc_sli4_wq_put - Put a Work Queue Entry on an Work Queue
  * @q: The Work Queue to operate on.
@@ -137,7 +165,7 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe)
 		bf_set(wqe_wqec, &wqe->generic.wqe_com, 0);
 	if (q->phba->sli3_options & LPFC_SLI4_PHWQ_ENABLED)
 		bf_set(wqe_wqid, &wqe->generic.wqe_com, q->queue_id);
-	lpfc_sli_pcimem_bcopy(wqe, temp_wqe, q->entry_size);
+	lpfc_sli4_pcimem_bcopy(wqe, temp_wqe, q->entry_size);
 	if (q->dpp_enable && q->phba->cfg_enable_dpp) {
 		/* write to DPP aperture taking advatage of Combined Writes */
 		tmp = (uint8_t *)temp_wqe;
@@ -240,7 +268,7 @@ lpfc_sli4_mq_put(struct lpfc_queue *q, struct lpfc_mqe *mqe)
 	/* If the host has not yet processed the next entry then we are done */
 	if (((q->host_index + 1) % q->entry_count) == q->hba_index)
 		return -ENOMEM;
-	lpfc_sli_pcimem_bcopy(mqe, temp_mqe, q->entry_size);
+	lpfc_sli4_pcimem_bcopy(mqe, temp_mqe, q->entry_size);
 	/* Save off the mailbox pointer for completion */
 	q->phba->mbox = (MAILBOX_t *)temp_mqe;
 
@@ -663,8 +691,8 @@ lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq,
 	/* If the host has not yet processed the next entry then we are done */
 	if (((hq_put_index + 1) % hq->entry_count) == hq->hba_index)
 		return -EBUSY;
-	lpfc_sli_pcimem_bcopy(hrqe, temp_hrqe, hq->entry_size);
-	lpfc_sli_pcimem_bcopy(drqe, temp_drqe, dq->entry_size);
+	lpfc_sli4_pcimem_bcopy(hrqe, temp_hrqe, hq->entry_size);
+	lpfc_sli4_pcimem_bcopy(drqe, temp_drqe, dq->entry_size);
 
 	/* Update the host index to point to the next slot */
 	hq->host_index = ((hq_put_index + 1) % hq->entry_count);
@@ -7199,7 +7227,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)
 			lpfc_post_rq_buffer(
 				phba, phba->sli4_hba.nvmet_mrq_hdr[i],
 				phba->sli4_hba.nvmet_mrq_data[i],
-				LPFC_NVMET_RQE_DEF_COUNT, i);
+				phba->cfg_nvmet_mrq_post, i);
 		}
 	}
 
@@ -8185,8 +8213,8 @@ lpfc_sli4_post_sync_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 	 */
 	mbx_cmnd = bf_get(lpfc_mqe_command, mb);
 	memset(phba->sli4_hba.bmbx.avirt, 0, sizeof(struct lpfc_bmbx_create));
-	lpfc_sli_pcimem_bcopy(mb, phba->sli4_hba.bmbx.avirt,
-			      sizeof(struct lpfc_mqe));
+	lpfc_sli4_pcimem_bcopy(mb, phba->sli4_hba.bmbx.avirt,
+			       sizeof(struct lpfc_mqe));
 
 	/* Post the high mailbox dma address to the port and wait for ready. */
 	dma_address = &phba->sli4_hba.bmbx.dma_address;
@@ -8210,11 +8238,11 @@ lpfc_sli4_post_sync_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 	 * If so, update the mailbox status so that the upper layers
 	 * can complete the request normally.
 	 */
-	lpfc_sli_pcimem_bcopy(phba->sli4_hba.bmbx.avirt, mb,
-			      sizeof(struct lpfc_mqe));
+	lpfc_sli4_pcimem_bcopy(phba->sli4_hba.bmbx.avirt, mb,
+			       sizeof(struct lpfc_mqe));
 	mbox_rgn = (struct lpfc_bmbx_create *) phba->sli4_hba.bmbx.avirt;
-	lpfc_sli_pcimem_bcopy(&mbox_rgn->mcqe, &mboxq->mcqe,
-			      sizeof(struct lpfc_mcqe));
+	lpfc_sli4_pcimem_bcopy(&mbox_rgn->mcqe, &mboxq->mcqe,
+			       sizeof(struct lpfc_mcqe));
 	mcqe_status = bf_get(lpfc_mcqe_status, &mbox_rgn->mcqe);
 	/*
 	 * When the CQE status indicates a failure and the mailbox status
@@ -11300,11 +11328,11 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring,
 	unsigned long iflags;
 	struct lpfc_sli_ring *pring_s4;
 
-	spin_lock_irq(&phba->hbalock);
+	spin_lock_irqsave(&phba->hbalock, iflags);
 
 	/* all I/Os are in process of being flushed */
 	if (phba->hba_flag & HBA_FCP_IOQ_FLUSH) {
-		spin_unlock_irq(&phba->hbalock);
+		spin_unlock_irqrestore(&phba->hbalock, iflags);
 		return 0;
 	}
 	sum = 0;
@@ -11366,14 +11394,14 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring,
 		iocbq->iocb_flag |= LPFC_DRIVER_ABORTED;
 
 		if (phba->sli_rev == LPFC_SLI_REV4) {
-			pring_s4 = lpfc_sli4_calc_ring(phba, iocbq);
-			if (pring_s4 == NULL)
+			pring_s4 = lpfc_sli4_calc_ring(phba, abtsiocbq);
+			if (!pring_s4)
 				continue;
 			/* Note: both hbalock and ring_lock must be set here */
-			spin_lock_irqsave(&pring_s4->ring_lock, iflags);
+			spin_lock(&pring_s4->ring_lock);
 			ret_val = __lpfc_sli_issue_iocb(phba, pring_s4->ringno,
 							abtsiocbq, 0);
-			spin_unlock_irqrestore(&pring_s4->ring_lock, iflags);
+			spin_unlock(&pring_s4->ring_lock);
 		} else {
 			ret_val = __lpfc_sli_issue_iocb(phba, pring->ringno,
 							abtsiocbq, 0);
@@ -11385,7 +11413,7 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring,
 		else
 			sum++;
 	}
-	spin_unlock_irq(&phba->hbalock);
+	spin_unlock_irqrestore(&phba->hbalock, iflags);
 	return sum;
 }
 
@@ -12830,7 +12858,7 @@ lpfc_sli4_sp_handle_mbox_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe)
 
 	/* Move mbox data to caller's mailbox region, do endian swapping */
 	if (pmb->mbox_cmpl && mbox)
-		lpfc_sli_pcimem_bcopy(mbox, mqe, sizeof(struct lpfc_mqe));
+		lpfc_sli4_pcimem_bcopy(mbox, mqe, sizeof(struct lpfc_mqe));
 
 	/*
 	 * For mcqe errors, conditionally move a modified error code to
@@ -12913,7 +12941,7 @@ lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_cqe *cqe)
 	bool workposted;
 
 	/* Copy the mailbox MCQE and convert endian order as needed */
-	lpfc_sli_pcimem_bcopy(cqe, &mcqe, sizeof(struct lpfc_mcqe));
+	lpfc_sli4_pcimem_bcopy(cqe, &mcqe, sizeof(struct lpfc_mcqe));
 
 	/* Invoke the proper event handling routine */
 	if (!bf_get(lpfc_trailer_async, &mcqe))
@@ -12944,6 +12972,17 @@ lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
 	int txcmplq_cnt = 0;
 	int fcp_txcmplq_cnt = 0;
 
+	/* Check for response status */
+	if (unlikely(bf_get(lpfc_wcqe_c_status, wcqe))) {
+		/* Log the error status */
+		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
+				"0357 ELS CQE error: status=x%x: "
+				"CQE: %08x %08x %08x %08x\n",
+				bf_get(lpfc_wcqe_c_status, wcqe),
+				wcqe->word0, wcqe->total_data_placed,
+				wcqe->parameter, wcqe->word3);
+	}
+
 	/* Get an irspiocbq for later ELS response processing use */
 	irspiocbq = lpfc_sli_get_iocbq(phba);
 	if (!irspiocbq) {
@@ -13173,7 +13212,7 @@ lpfc_sli4_sp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
 	bool workposted = false;
 
 	/* Copy the work queue CQE and convert endian order if needed */
-	lpfc_sli_pcimem_bcopy(cqe, &cqevt, sizeof(struct lpfc_cqe));
+	lpfc_sli4_pcimem_bcopy(cqe, &cqevt, sizeof(struct lpfc_cqe));
 
 	/* Check and process for different type of WCQE and dispatch */
 	switch (bf_get(lpfc_cqe_code, &cqevt)) {
@@ -13364,14 +13403,12 @@ lpfc_sli4_fp_handle_fcp_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
 			phba->lpfc_rampdown_queue_depth(phba);
 
 		/* Log the error status */
-		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
-				"0373 FCP complete error: status=x%x, "
-				"hw_status=x%x, total_data_specified=%d, "
-				"parameter=x%x, word3=x%x\n",
+		lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
+				"0373 FCP CQE error: status=x%x: "
+				"CQE: %08x %08x %08x %08x\n",
 				bf_get(lpfc_wcqe_c_status, wcqe),
-				bf_get(lpfc_wcqe_c_hw_status, wcqe),
-				wcqe->total_data_placed, wcqe->parameter,
-				wcqe->word3);
+				wcqe->word0, wcqe->total_data_placed,
+				wcqe->parameter, wcqe->word3);
 	}
 
 	/* Look up the FCP command IOCB and create pseudo response IOCB */
@@ -13581,7 +13618,7 @@ lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
 	bool workposted = false;
 
 	/* Copy the work queue CQE and convert endian order if needed */
-	lpfc_sli_pcimem_bcopy(cqe, &wcqe, sizeof(struct lpfc_cqe));
+	lpfc_sli4_pcimem_bcopy(cqe, &wcqe, sizeof(struct lpfc_cqe));
 
 	/* Check and process for different type of WCQE and dispatch */
 	switch (bf_get(lpfc_wcqe_c_code, &wcqe)) {
@@ -19032,9 +19069,22 @@ lpfc_drain_txq(struct lpfc_hba *phba)
 	struct lpfc_sglq *sglq;
 	union lpfc_wqe128 wqe;
 	uint32_t txq_cnt = 0;
+	struct lpfc_queue *wq;
 
-	pring = lpfc_phba_elsring(phba);
-	if (unlikely(!pring))
+	if (phba->link_flag & LS_MDS_LOOPBACK) {
+		/* MDS WQE are posted only to first WQ*/
+		wq = phba->sli4_hba.fcp_wq[0];
+		if (unlikely(!wq))
+			return 0;
+		pring = wq->pring;
+	} else {
+		wq = phba->sli4_hba.els_wq;
+		if (unlikely(!wq))
+			return 0;
+		pring = lpfc_phba_elsring(phba);
+	}
+
+	if (unlikely(!pring) || list_empty(&pring->txq))
 		return 0;
 
 	spin_lock_irqsave(&pring->ring_lock, iflags);
@@ -19075,7 +19125,7 @@ lpfc_drain_txq(struct lpfc_hba *phba)
 			fail_msg = "to convert bpl to sgl";
 		else if (lpfc_sli4_iocb2wqe(phba, piocbq, &wqe))
 			fail_msg = "to convert iocb to wqe";
-		else if (lpfc_sli4_wq_put(phba->sli4_hba.els_wq, &wqe))
+		else if (lpfc_sli4_wq_put(wq, &wqe))
 			fail_msg = " - Wq is full";
 		else
 			lpfc_sli_ringtxcmpl_put(phba, pring, piocbq);
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index e8b089abbfb3..18c23afcf46b 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -2,7 +2,7 @@
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
  * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
- * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.broadcom.com                                                *
@@ -20,7 +20,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "12.0.0.1"
+#define LPFC_DRIVER_VERSION "12.0.0.4"
 #define LPFC_DRIVER_NAME		"lpfc"
 
 /* Used for SLI 2/3 */
diff --git a/drivers/scsi/megaraid.c b/drivers/scsi/megaraid.c
index 7195cff51d4c..3b3767e240d8 100644
--- a/drivers/scsi/megaraid.c
+++ b/drivers/scsi/megaraid.c
@@ -2731,53 +2731,6 @@ proc_show_rdrv_40(struct seq_file *m, void *v)
 	return proc_show_rdrv(m, m->private, 30, 39);
 }
 
-
-/*
- * seq_file wrappers for procfile show routines.
- */
-static int mega_proc_open(struct inode *inode, struct file *file)
-{
-	adapter_t *adapter = proc_get_parent_data(inode);
-	int (*show)(struct seq_file *, void *) = PDE_DATA(inode);
-
-	return single_open(file, show, adapter);
-}
-
-static const struct file_operations mega_proc_fops = {
-	.open		= mega_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-/*
- * Table of proc files we need to create.
- */
-struct mega_proc_file {
-	const char *name;
-	unsigned short ptr_offset;
-	int (*show) (struct seq_file *m, void *v);
-};
-
-static const struct mega_proc_file mega_proc_files[] = {
-	{ "config",	      offsetof(adapter_t, proc_read), proc_show_config },
-	{ "stat",	      offsetof(adapter_t, proc_stat), proc_show_stat },
-	{ "mailbox",	      offsetof(adapter_t, proc_mbox), proc_show_mbox },
-#if MEGA_HAVE_ENH_PROC
-	{ "rebuild-rate",     offsetof(adapter_t, proc_rr), proc_show_rebuild_rate },
-	{ "battery-status",   offsetof(adapter_t, proc_battery), proc_show_battery },
-	{ "diskdrives-ch0",   offsetof(adapter_t, proc_pdrvstat[0]), proc_show_pdrv_ch0 },
-	{ "diskdrives-ch1",   offsetof(adapter_t, proc_pdrvstat[1]), proc_show_pdrv_ch1 },
-	{ "diskdrives-ch2",   offsetof(adapter_t, proc_pdrvstat[2]), proc_show_pdrv_ch2 },
-	{ "diskdrives-ch3",   offsetof(adapter_t, proc_pdrvstat[3]), proc_show_pdrv_ch3 },
-	{ "raiddrives-0-9",   offsetof(adapter_t, proc_rdrvstat[0]), proc_show_rdrv_10 },
-	{ "raiddrives-10-19", offsetof(adapter_t, proc_rdrvstat[1]), proc_show_rdrv_20 },
-	{ "raiddrives-20-29", offsetof(adapter_t, proc_rdrvstat[2]), proc_show_rdrv_30 },
-	{ "raiddrives-30-39", offsetof(adapter_t, proc_rdrvstat[3]), proc_show_rdrv_40 },
-#endif
-	{ NULL }
-};
-
 /**
  * mega_create_proc_entry()
  * @index - index in soft state array
@@ -2788,31 +2741,45 @@ static const struct mega_proc_file mega_proc_files[] = {
 static void
 mega_create_proc_entry(int index, struct proc_dir_entry *parent)
 {
-	const struct mega_proc_file *f;
-	adapter_t	*adapter = hba_soft_state[index];
-	struct proc_dir_entry	*dir, *de, **ppde;
-	u8		string[16];
+	adapter_t *adapter = hba_soft_state[index];
+	struct proc_dir_entry *dir;
+	u8 string[16];
 
 	sprintf(string, "hba%d", adapter->host->host_no);
-
-	dir = adapter->controller_proc_dir_entry =
-		proc_mkdir_data(string, 0, parent, adapter);
-	if(!dir) {
+	dir = proc_mkdir_data(string, 0, parent, adapter);
+	if (!dir) {
 		dev_warn(&adapter->dev->dev, "proc_mkdir failed\n");
 		return;
 	}
 
-	for (f = mega_proc_files; f->name; f++) {
-		de = proc_create_data(f->name, S_IRUSR, dir, &mega_proc_fops,
-				      f->show);
-		if (!de) {
-			dev_warn(&adapter->dev->dev, "proc_create failed\n");
-			return;
-		}
-
-		ppde = (void *)adapter + f->ptr_offset;
-		*ppde = de;
-	}
+	proc_create_single_data("config", S_IRUSR, dir,
+			proc_show_config, adapter);
+	proc_create_single_data("stat", S_IRUSR, dir,
+			proc_show_stat, adapter);
+	proc_create_single_data("mailbox", S_IRUSR, dir,
+			proc_show_mbox, adapter);
+#if MEGA_HAVE_ENH_PROC
+	proc_create_single_data("rebuild-rate", S_IRUSR, dir,
+			proc_show_rebuild_rate, adapter);
+	proc_create_single_data("battery-status", S_IRUSR, dir,
+			proc_show_battery, adapter);
+	proc_create_single_data("diskdrives-ch0", S_IRUSR, dir,
+			proc_show_pdrv_ch0, adapter);
+	proc_create_single_data("diskdrives-ch1", S_IRUSR, dir,
+			proc_show_pdrv_ch1, adapter);
+	proc_create_single_data("diskdrives-ch2", S_IRUSR, dir,
+			proc_show_pdrv_ch2, adapter);
+	proc_create_single_data("diskdrives-ch3", S_IRUSR, dir,
+			proc_show_pdrv_ch3, adapter);
+	proc_create_single_data("raiddrives-0-9", S_IRUSR, dir,
+			proc_show_rdrv_10, adapter);
+	proc_create_single_data("raiddrives-10-19", S_IRUSR, dir,
+			proc_show_rdrv_20, adapter);
+	proc_create_single_data("raiddrives-20-29", S_IRUSR, dir,
+			proc_show_rdrv_30, adapter);
+	proc_create_single_data("raiddrives-30-39", S_IRUSR, dir,
+			proc_show_rdrv_40, adapter);
+#endif
 }
 
 #else
@@ -4199,6 +4166,9 @@ megaraid_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	int irq, i, j;
 	int error = -ENODEV;
 
+	if (hba_count >= MAX_CONTROLLERS)
+		goto out;
+
 	if (pci_enable_device(pdev))
 		goto out;
 	pci_set_master(pdev);
@@ -4580,6 +4550,7 @@ megaraid_remove_one(struct pci_dev *pdev)
 {
 	struct Scsi_Host *host = pci_get_drvdata(pdev);
 	adapter_t *adapter = (adapter_t *)host->hostdata;
+	char buf[12] = { 0 };
 
 	scsi_remove_host(host);
 
@@ -4594,44 +4565,8 @@ megaraid_remove_one(struct pci_dev *pdev)
 
 	mega_free_sgl(adapter);
 
-#ifdef CONFIG_PROC_FS
-	if (adapter->controller_proc_dir_entry) {
-		remove_proc_entry("stat", adapter->controller_proc_dir_entry);
-		remove_proc_entry("config",
-				adapter->controller_proc_dir_entry);
-		remove_proc_entry("mailbox",
-				adapter->controller_proc_dir_entry);
-#if MEGA_HAVE_ENH_PROC
-		remove_proc_entry("rebuild-rate",
-				adapter->controller_proc_dir_entry);
-		remove_proc_entry("battery-status",
-				adapter->controller_proc_dir_entry);
-
-		remove_proc_entry("diskdrives-ch0",
-				adapter->controller_proc_dir_entry);
-		remove_proc_entry("diskdrives-ch1",
-				adapter->controller_proc_dir_entry);
-		remove_proc_entry("diskdrives-ch2",
-				adapter->controller_proc_dir_entry);
-		remove_proc_entry("diskdrives-ch3",
-				adapter->controller_proc_dir_entry);
-
-		remove_proc_entry("raiddrives-0-9",
-				adapter->controller_proc_dir_entry);
-		remove_proc_entry("raiddrives-10-19",
-				adapter->controller_proc_dir_entry);
-		remove_proc_entry("raiddrives-20-29",
-				adapter->controller_proc_dir_entry);
-		remove_proc_entry("raiddrives-30-39",
-				adapter->controller_proc_dir_entry);
-#endif
-		{
-			char	buf[12] = { 0 };
-			sprintf(buf, "hba%d", adapter->host->host_no);
-			remove_proc_entry(buf, mega_proc_dir_entry);
-		}
-	}
-#endif
+	sprintf(buf, "hba%d", adapter->host->host_no);
+	remove_proc_subtree(buf, mega_proc_dir_entry);
 
 	pci_free_consistent(adapter->dev, MEGA_BUFFER_SIZE,
 			adapter->mega_buffer, adapter->buf_dma_handle);
diff --git a/drivers/scsi/megaraid.h b/drivers/scsi/megaraid.h
index 21eba2fd465a..18e85d9267ff 100644
--- a/drivers/scsi/megaraid.h
+++ b/drivers/scsi/megaraid.h
@@ -814,18 +814,6 @@ typedef struct {
 
 #ifdef CONFIG_PROC_FS
 	struct proc_dir_entry	*controller_proc_dir_entry;
-	struct proc_dir_entry	*proc_read;
-	struct proc_dir_entry	*proc_stat;
-	struct proc_dir_entry	*proc_mbox;
-
-#if MEGA_HAVE_ENH_PROC
-	struct proc_dir_entry	*proc_rr;
-	struct proc_dir_entry	*proc_battery;
-#define MAX_PROC_CHANNELS	4
-	struct proc_dir_entry	*proc_pdrvstat[MAX_PROC_CHANNELS];
-	struct proc_dir_entry	*proc_rdrvstat[MAX_PROC_CHANNELS];
-#endif
-
 #endif
 
 	int	has_64bit_addr;		/* are we using 64-bit addressing */
diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h
index 27fab8235ea5..75dc25f78336 100644
--- a/drivers/scsi/megaraid/megaraid_sas.h
+++ b/drivers/scsi/megaraid/megaraid_sas.h
@@ -35,8 +35,8 @@
 /*
  * MegaRAID SAS Driver meta data
  */
-#define MEGASAS_VERSION				"07.704.04.00-rc1"
-#define MEGASAS_RELDATE				"December 7, 2017"
+#define MEGASAS_VERSION				"07.705.02.00-rc1"
+#define MEGASAS_RELDATE				"April 4, 2018"
 
 /*
  * Device IDs
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index b89c6e6c0589..c5d0c4bd71d2 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -92,7 +92,7 @@ MODULE_PARM_DESC(resetwaittime, "Wait time in seconds after I/O timeout "
 
 int smp_affinity_enable = 1;
 module_param(smp_affinity_enable, int, S_IRUGO);
-MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disbale Default: enable(1)");
+MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)");
 
 int rdpq_enable = 1;
 module_param(rdpq_enable, int, S_IRUGO);
@@ -2224,9 +2224,9 @@ static int megasas_get_ld_vf_affiliation_111(struct megasas_instance *instance,
 			       sizeof(struct MR_LD_VF_AFFILIATION_111));
 	else {
 		new_affiliation_111 =
-			pci_alloc_consistent(instance->pdev,
-					     sizeof(struct MR_LD_VF_AFFILIATION_111),
-					     &new_affiliation_111_h);
+			pci_zalloc_consistent(instance->pdev,
+					      sizeof(struct MR_LD_VF_AFFILIATION_111),
+					      &new_affiliation_111_h);
 		if (!new_affiliation_111) {
 			dev_printk(KERN_DEBUG, &instance->pdev->dev, "SR-IOV: Couldn't allocate "
 			       "memory for new affiliation for scsi%d\n",
@@ -2234,8 +2234,6 @@ static int megasas_get_ld_vf_affiliation_111(struct megasas_instance *instance,
 			megasas_return_cmd(instance, cmd);
 			return -ENOMEM;
 		}
-		memset(new_affiliation_111, 0,
-		       sizeof(struct MR_LD_VF_AFFILIATION_111));
 	}
 
 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
@@ -2333,10 +2331,10 @@ static int megasas_get_ld_vf_affiliation_12(struct megasas_instance *instance,
 		       sizeof(struct MR_LD_VF_AFFILIATION));
 	else {
 		new_affiliation =
-			pci_alloc_consistent(instance->pdev,
-					     (MAX_LOGICAL_DRIVES + 1) *
-					     sizeof(struct MR_LD_VF_AFFILIATION),
-					     &new_affiliation_h);
+			pci_zalloc_consistent(instance->pdev,
+					      (MAX_LOGICAL_DRIVES + 1) *
+					      sizeof(struct MR_LD_VF_AFFILIATION),
+					      &new_affiliation_h);
 		if (!new_affiliation) {
 			dev_printk(KERN_DEBUG, &instance->pdev->dev, "SR-IOV: Couldn't allocate "
 			       "memory for new affiliation for scsi%d\n",
@@ -2344,8 +2342,6 @@ static int megasas_get_ld_vf_affiliation_12(struct megasas_instance *instance,
 			megasas_return_cmd(instance, cmd);
 			return -ENOMEM;
 		}
-		memset(new_affiliation, 0, (MAX_LOGICAL_DRIVES + 1) *
-		       sizeof(struct MR_LD_VF_AFFILIATION));
 	}
 
 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
@@ -2772,7 +2768,7 @@ blk_eh_timer_return megasas_reset_timer(struct scsi_cmnd *scmd)
 
 	if (time_after(jiffies, scmd->jiffies_at_alloc +
 				(scmd_timeout * 2) * HZ)) {
-		return BLK_EH_NOT_HANDLED;
+		return BLK_EH_DONE;
 	}
 
 	instance = (struct megasas_instance *)scmd->device->host->hostdata;
@@ -5636,16 +5632,15 @@ megasas_get_seq_num(struct megasas_instance *instance,
 	}
 
 	dcmd = &cmd->frame->dcmd;
-	el_info = pci_alloc_consistent(instance->pdev,
-				       sizeof(struct megasas_evt_log_info),
-				       &el_info_h);
+	el_info = pci_zalloc_consistent(instance->pdev,
+					sizeof(struct megasas_evt_log_info),
+					&el_info_h);
 
 	if (!el_info) {
 		megasas_return_cmd(instance, cmd);
 		return -ENOMEM;
 	}
 
-	memset(el_info, 0, sizeof(*el_info));
 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
 
 	dcmd->cmd = MFI_CMD_DCMD;
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index f4d988dd1e9d..98a7a090b75e 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -684,15 +684,14 @@ megasas_alloc_rdpq_fusion(struct megasas_instance *instance)
 	array_size = sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) *
 		     MAX_MSIX_QUEUES_FUSION;
 
-	fusion->rdpq_virt = pci_alloc_consistent(instance->pdev, array_size,
-						 &fusion->rdpq_phys);
+	fusion->rdpq_virt = pci_zalloc_consistent(instance->pdev, array_size,
+						  &fusion->rdpq_phys);
 	if (!fusion->rdpq_virt) {
 		dev_err(&instance->pdev->dev,
 			"Failed from %s %d\n",  __func__, __LINE__);
 		return -ENOMEM;
 	}
 
-	memset(fusion->rdpq_virt, 0, array_size);
 	msix_count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
 
 	fusion->reply_frames_desc_pool = dma_pool_create("mr_rdpq",
@@ -2981,6 +2980,9 @@ megasas_build_syspd_fusion(struct megasas_instance *instance,
 		pRAID_Context->timeout_value = cpu_to_le16(os_timeout_value);
 		pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
 	} else {
+		if (os_timeout_value)
+			os_timeout_value++;
+
 		/* system pd Fast Path */
 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
 		timeout_limit = (scmd->device->type == TYPE_DISK) ?
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2.h b/drivers/scsi/mpt3sas/mpi/mpi2.h
index b015c30d2c32..1e45268a78fc 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2.h
@@ -9,7 +9,7 @@
  *                 scatter/gather formats.
  * Creation Date:  June 21, 2006
  *
- * mpi2.h Version:  02.00.48
+ *  mpi2.h Version:  02.00.50
  *
  * NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
  *       prefix are for use only on MPI v2.5 products, and must not be used
@@ -114,6 +114,8 @@
  * 09-02-16  02.00.46  Bumped MPI2_HEADER_VERSION_UNIT.
  * 11-23-16  02.00.47  Bumped MPI2_HEADER_VERSION_UNIT.
  * 02-03-17  02.00.48  Bumped MPI2_HEADER_VERSION_UNIT.
+ * 06-13-17  02.00.49  Bumped MPI2_HEADER_VERSION_UNIT.
+ * 09-29-17  02.00.50  Bumped MPI2_HEADER_VERSION_UNIT.
  * --------------------------------------------------------------------------
  */
 
@@ -152,8 +154,9 @@
 					MPI26_VERSION_MINOR)
 #define MPI2_VERSION_02_06		    (0x0206)
 
-/*Unit and Dev versioning for this MPI header set */
-#define MPI2_HEADER_VERSION_UNIT            (0x30)
+
+/* Unit and Dev versioning for this MPI header set */
+#define MPI2_HEADER_VERSION_UNIT            (0x32)
 #define MPI2_HEADER_VERSION_DEV             (0x00)
 #define MPI2_HEADER_VERSION_UNIT_MASK       (0xFF00)
 #define MPI2_HEADER_VERSION_UNIT_SHIFT      (8)
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
index 0ad88deb3176..5122920a961a 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
@@ -7,7 +7,7 @@
  *         Title:  MPI Configuration messages and pages
  * Creation Date:  November 10, 2006
  *
- *   mpi2_cnfg.h Version:  02.00.40
+ *    mpi2_cnfg.h Version:  02.00.42
  *
  * NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
  *       prefix are for use only on MPI v2.5 products, and must not be used
@@ -219,6 +219,18 @@
  *                     Added ChassisSlot field to SAS Enclosure Page 0.
  *                     Added ChassisSlot Valid bit (bit 5) to the Flags field
  *                     in SAS Enclosure Page 0.
+ * 06-13-17  02.00.41  Added MPI26_MFGPAGE_DEVID_SAS3816 and
+ *                     MPI26_MFGPAGE_DEVID_SAS3916 defines.
+ *                     Removed MPI26_MFGPAGE_DEVID_SAS4008 define.
+ *                     Added MPI26_PCIEIOUNIT1_LINKFLAGS_SRNS_EN define.
+ *                     Renamed PI26_PCIEIOUNIT1_LINKFLAGS_EN_SRIS to
+ *                     PI26_PCIEIOUNIT1_LINKFLAGS_SRIS_EN.
+ *                     Renamed MPI26_PCIEIOUNIT1_LINKFLAGS_DIS_SRIS to
+ *                     MPI26_PCIEIOUNIT1_LINKFLAGS_DIS_SEPARATE_REFCLK.
+ * 09-29-17  02.00.42  Added ControllerResetTO field to PCIe Device Page 2.
+ *                     Added NOIOB field to PCIe Device Page 2.
+ *                     Added MPI26_PCIEDEV2_CAP_DATA_BLK_ALIGN_AND_GRAN to
+ *                     the Capabilities field of PCIe Device Page 2.
  * --------------------------------------------------------------------------
  */
 
@@ -556,7 +568,8 @@ typedef struct _MPI2_CONFIG_REPLY {
 #define MPI26_MFGPAGE_DEVID_SAS3616                 (0x00D1)
 #define MPI26_MFGPAGE_DEVID_SAS3708                 (0x00D2)
 
-#define MPI26_MFGPAGE_DEVID_SAS4008                 (0x00A1)
+#define MPI26_MFGPAGE_DEVID_SAS3816                 (0x00A1)
+#define MPI26_MFGPAGE_DEVID_SAS3916                 (0x00A0)
 
 
 /*Manufacturing Page 0 */
@@ -3864,20 +3877,25 @@ typedef struct _MPI26_CONFIG_PAGE_PCIEDEV_0 {
 typedef struct _MPI26_CONFIG_PAGE_PCIEDEV_2 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER	Header;	/*0x00 */
 	U16	DevHandle;		/*0x08 */
-	U16	Reserved1;		/*0x0A */
-	U32	MaximumDataTransferSize;/*0x0C */
+	U8	ControllerResetTO;		/* 0x0A */
+	U8	Reserved1;		/* 0x0B */
+	U32	MaximumDataTransferSize;	/*0x0C */
 	U32	Capabilities;		/*0x10 */
-	U32	Reserved2;		/*0x14 */
+	U16	NOIOB;		/* 0x14 */
+	U16	Reserved2;		/* 0x16 */
 } MPI26_CONFIG_PAGE_PCIEDEV_2, *PTR_MPI26_CONFIG_PAGE_PCIEDEV_2,
 	Mpi26PCIeDevicePage2_t, *pMpi26PCIeDevicePage2_t;
 
-#define MPI26_PCIEDEVICE2_PAGEVERSION       (0x00)
+#define MPI26_PCIEDEVICE2_PAGEVERSION       (0x01)
 
 /*defines for PCIe Device Page 2 Capabilities field */
+#define MPI26_PCIEDEV2_CAP_DATA_BLK_ALIGN_AND_GRAN     (0x00000008)
 #define MPI26_PCIEDEV2_CAP_SGL_FORMAT                  (0x00000004)
 #define MPI26_PCIEDEV2_CAP_BIT_BUCKET_SUPPORT          (0x00000002)
 #define MPI26_PCIEDEV2_CAP_SGL_SUPPORT                 (0x00000001)
 
+/* Defines for the NOIOB field */
+#define MPI26_PCIEDEV2_NOIOB_UNSUPPORTED                (0x0000)
 
 /****************************************************************************
 *  PCIe Link Config Pages (MPI v2.6 and later)
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_init.h b/drivers/scsi/mpt3sas/mpi/mpi2_init.h
index 948a3ba682d7..6213ce6791ac 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2_init.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2_init.h
@@ -75,7 +75,7 @@
 
 typedef struct _MPI2_SCSI_IO_CDB_EEDP32 {
 	U8 CDB[20];		/*0x00 */
-	U32 PrimaryReferenceTag;	/*0x14 */
+	__be32 PrimaryReferenceTag;	/*0x14 */
 	U16 PrimaryApplicationTag;	/*0x18 */
 	U16 PrimaryApplicationTagMask;	/*0x1A */
 	U32 TransferLength;	/*0x1C */
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h b/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h
index cc2aff7aa67b..1faec3a93e69 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h
@@ -7,7 +7,7 @@
  *         Title:  MPI IOC, Port, Event, FW Download, and FW Upload messages
  * Creation Date:  October 11, 2006
  *
- * mpi2_ioc.h Version:  02.00.32
+ * mpi2_ioc.h Version:  02.00.34
  *
  * NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
  *       prefix are for use only on MPI v2.5 products, and must not be used
@@ -167,6 +167,10 @@
  * 02-02-17   02.00.32 Added MPI2_FW_DOWNLOAD_ITYPE_CBB_BACKUP.
  *                     Added MPI25_EVENT_DATA_ACTIVE_CABLE_EXCEPT and related
  *                     defines for the ReasonCode field.
+ * 06-13-17   02.00.33 Added MPI2_FW_DOWNLOAD_ITYPE_CPLD.
+ * 09-29-17   02.00.34 Added MPI26_EVENT_PCIDEV_STAT_RC_PCIE_HOT_RESET_FAILED
+ *                     to the ReasonCode field in PCIe Device Status Change
+ *                     Event Data.
  * --------------------------------------------------------------------------
  */
 
@@ -1182,6 +1186,7 @@ typedef struct _MPI26_EVENT_DATA_PCIE_DEVICE_STATUS_CHANGE {
 #define MPI26_EVENT_PCIDEV_STAT_RC_CMP_INTERNAL_DEV_RESET               (0x0E)
 #define MPI26_EVENT_PCIDEV_STAT_RC_CMP_TASK_ABORT_INTERNAL              (0x0F)
 #define MPI26_EVENT_PCIDEV_STAT_RC_DEV_INIT_FAILURE                     (0x10)
+#define MPI26_EVENT_PCIDEV_STAT_RC_PCIE_HOT_RESET_FAILED                (0x11)
 
 
 /*PCIe Enumeration Event data (MPI v2.6 and later) */
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index 61f93a134956..bf04fa90f433 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -87,7 +87,7 @@ MODULE_PARM_DESC(msix_disable, " disable msix routed interrupts (default=0)");
 
 static int smp_affinity_enable = 1;
 module_param(smp_affinity_enable, int, S_IRUGO);
-MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disbale Default: enable(1)");
+MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)");
 
 static int max_msix_vectors = -1;
 module_param(max_msix_vectors, int, 0);
@@ -297,12 +297,15 @@ static void *
 _base_get_chain_buffer_dma_to_chain_buffer(struct MPT3SAS_ADAPTER *ioc,
 		dma_addr_t chain_buffer_dma)
 {
-	u16 index;
-
-	for (index = 0; index < ioc->chain_depth; index++) {
-		if (ioc->chain_lookup[index].chain_buffer_dma ==
-				chain_buffer_dma)
-			return ioc->chain_lookup[index].chain_buffer;
+	u16 index, j;
+	struct chain_tracker *ct;
+
+	for (index = 0; index < ioc->scsiio_depth; index++) {
+		for (j = 0; j < ioc->chains_needed_per_io; j++) {
+			ct = &ioc->chain_lookup[index].chains_per_smid[j];
+			if (ct && ct->chain_buffer_dma == chain_buffer_dma)
+				return ct->chain_buffer;
+		}
 	}
 	pr_info(MPT3SAS_FMT
 	    "Provided chain_buffer_dma address is not in the lookup list\n",
@@ -394,13 +397,14 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
 	buff_ptr_phys = buffer_iomem_phys;
 	WARN_ON(buff_ptr_phys > U32_MAX);
 
-	if (sgel->FlagsLength &
+	if (le32_to_cpu(sgel->FlagsLength) &
 			(MPI2_SGE_FLAGS_HOST_TO_IOC << MPI2_SGE_FLAGS_SHIFT))
 		is_write = 1;
 
 	for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) {
 
-		sgl_flags = (sgel->FlagsLength >> MPI2_SGE_FLAGS_SHIFT);
+		sgl_flags =
+		    (le32_to_cpu(sgel->FlagsLength) >> MPI2_SGE_FLAGS_SHIFT);
 
 		switch (sgl_flags & MPI2_SGE_FLAGS_ELEMENT_MASK) {
 		case MPI2_SGE_FLAGS_CHAIN_ELEMENT:
@@ -411,7 +415,7 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
 			 */
 			sgel_next =
 				_base_get_chain_buffer_dma_to_chain_buffer(ioc,
-						sgel->Address);
+						le32_to_cpu(sgel->Address));
 			if (sgel_next == NULL)
 				return;
 			/*
@@ -426,7 +430,8 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
 			dst_addr_phys = _base_get_chain_phys(ioc,
 						smid, sge_chain_count);
 			WARN_ON(dst_addr_phys > U32_MAX);
-			sgel->Address = (u32)dst_addr_phys;
+			sgel->Address =
+				cpu_to_le32(lower_32_bits(dst_addr_phys));
 			sgel = sgel_next;
 			sge_chain_count++;
 			break;
@@ -435,22 +440,28 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
 				if (is_scsiio_req) {
 					_base_clone_to_sys_mem(buff_ptr,
 					    sg_virt(sg_scmd),
-					    (sgel->FlagsLength & 0x00ffffff));
+					    (le32_to_cpu(sgel->FlagsLength) &
+					    0x00ffffff));
 					/*
 					 * FIXME: this relies on a a zero
 					 * PCI mem_offset.
 					 */
-					sgel->Address = (u32)buff_ptr_phys;
+					sgel->Address =
+					    cpu_to_le32((u32)buff_ptr_phys);
 				} else {
 					_base_clone_to_sys_mem(buff_ptr,
 					    ioc->config_vaddr,
-					    (sgel->FlagsLength & 0x00ffffff));
-					sgel->Address = (u32)buff_ptr_phys;
+					    (le32_to_cpu(sgel->FlagsLength) &
+					    0x00ffffff));
+					sgel->Address =
+					    cpu_to_le32((u32)buff_ptr_phys);
 				}
 			}
-			buff_ptr += (sgel->FlagsLength & 0x00ffffff);
-			buff_ptr_phys += (sgel->FlagsLength & 0x00ffffff);
-			if ((sgel->FlagsLength &
+			buff_ptr += (le32_to_cpu(sgel->FlagsLength) &
+			    0x00ffffff);
+			buff_ptr_phys += (le32_to_cpu(sgel->FlagsLength) &
+			    0x00ffffff);
+			if ((le32_to_cpu(sgel->FlagsLength) &
 			    (MPI2_SGE_FLAGS_END_OF_BUFFER
 					<< MPI2_SGE_FLAGS_SHIFT)))
 				goto eob_clone_chain;
@@ -1019,6 +1030,9 @@ _base_display_event_data(struct MPT3SAS_ADAPTER *ioc,
 	case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION:
 		desc = "Cable Event";
 		break;
+	case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
+		desc = "SAS Device Discovery Error";
+		break;
 	case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
 		desc = "PCIE Device Status Change";
 		break;
@@ -1433,7 +1447,7 @@ _base_interrupt(int irq, void *bus_id)
 				    cpu_to_le32(reply);
 				if (ioc->is_mcpu_endpoint)
 					_base_clone_reply_to_sys_mem(ioc,
-						cpu_to_le32(reply),
+						reply,
 						ioc->reply_free_host_index);
 				writel(ioc->reply_free_host_index,
 				    &ioc->chip->ReplyFreeHostIndex);
@@ -1671,7 +1685,8 @@ _base_add_sg_single_64(void *paddr, u32 flags_length, dma_addr_t dma_addr)
  * @ioc: per adapter object
  * @scmd: SCSI commands of the IO request
  *
- * Returns chain tracker(from ioc->free_chain_list)
+ * Returns chain tracker from chain_lookup table using key as
+ * smid and smid's chain_offset.
  */
 static struct chain_tracker *
 _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc,
@@ -1679,20 +1694,15 @@ _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc,
 {
 	struct chain_tracker *chain_req;
 	struct scsiio_tracker *st = scsi_cmd_priv(scmd);
-	unsigned long flags;
+	u16 smid = st->smid;
+	u8 chain_offset =
+	   atomic_read(&ioc->chain_lookup[smid - 1].chain_offset);
 
-	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
-	if (list_empty(&ioc->free_chain_list)) {
-		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
-		dfailprintk(ioc, pr_warn(MPT3SAS_FMT
-			"chain buffers not available\n", ioc->name));
+	if (chain_offset == ioc->chains_needed_per_io)
 		return NULL;
-	}
-	chain_req = list_entry(ioc->free_chain_list.next,
-	    struct chain_tracker, tracker_list);
-	list_del_init(&chain_req->tracker_list);
-	list_add_tail(&chain_req->tracker_list, &st->chain_list);
-	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
+
+	chain_req = &ioc->chain_lookup[smid - 1].chains_per_smid[chain_offset];
+	atomic_inc(&ioc->chain_lookup[smid - 1].chain_offset);
 	return chain_req;
 }
 
@@ -3044,7 +3054,7 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
 
 		for (i = 0; i < ioc->combined_reply_index_count; i++) {
 			ioc->replyPostRegisterIndex[i] = (resource_size_t *)
-			     ((u8 *)&ioc->chip->Doorbell +
+			     ((u8 __force *)&ioc->chip->Doorbell +
 			     MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET +
 			     (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET));
 		}
@@ -3273,13 +3283,7 @@ void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc,
 		return;
 	st->cb_idx = 0xFF;
 	st->direct_io = 0;
-	if (!list_empty(&st->chain_list)) {
-		unsigned long flags;
-
-		spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
-		list_splice_init(&st->chain_list, &ioc->free_chain_list);
-		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
-	}
+	atomic_set(&ioc->chain_lookup[st->smid - 1].chain_offset, 0);
 }
 
 /**
@@ -3339,7 +3343,7 @@ _base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr,
 					spinlock_t *writeq_lock)
 {
 	unsigned long flags;
-	__u64 data_out = cpu_to_le64(b);
+	__u64 data_out = b;
 
 	spin_lock_irqsave(writeq_lock, flags);
 	writel((u32)(data_out), addr);
@@ -3362,7 +3366,7 @@ _base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr,
 static inline void
 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
 {
-	writeq(cpu_to_le64(b), addr);
+	writeq(b, addr);
 }
 #else
 static inline void
@@ -3389,7 +3393,7 @@ _base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
 	__le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
 
 	_clone_sg_entries(ioc, (void *) mfp, smid);
-	mpi_req_iomem = (void *)ioc->chip +
+	mpi_req_iomem = (void __force *)ioc->chip +
 			MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
 	_base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
 					ioc->request_sz);
@@ -3473,7 +3477,8 @@ mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 
 		request_hdr = (MPI2RequestHeader_t *)mfp;
 		/* TBD 256 is offset within sys register. */
-		mpi_req_iomem = (void *)ioc->chip + MPI_FRAME_START_OFFSET
+		mpi_req_iomem = (void __force *)ioc->chip
+					+ MPI_FRAME_START_OFFSET
 					+ (smid * ioc->request_sz);
 		_base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
 							ioc->request_sz);
@@ -3542,7 +3547,7 @@ mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)
 
 		_clone_sg_entries(ioc, (void *) mfp, smid);
 		/* TBD 256 is offset within sys register */
-		mpi_req_iomem = (void *)ioc->chip +
+		mpi_req_iomem = (void __force *)ioc->chip +
 			MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
 		_base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
 							ioc->request_sz);
@@ -3822,6 +3827,105 @@ _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc)
 	}
 }
 
+/**
+ * _base_display_fwpkg_version - sends FWUpload request to pull FWPkg
+ *				version from FW Image Header.
+ * @ioc: per adapter object
+ *
+ * Returns 0 for success, non-zero for failure.
+ */
+	static int
+_base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc)
+{
+	Mpi2FWImageHeader_t *FWImgHdr;
+	Mpi25FWUploadRequest_t *mpi_request;
+	Mpi2FWUploadReply_t mpi_reply;
+	int r = 0;
+	void *fwpkg_data = NULL;
+	dma_addr_t fwpkg_data_dma;
+	u16 smid, ioc_status;
+	size_t data_length;
+
+	dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
+				__func__));
+
+	if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
+		pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
+				ioc->name, __func__);
+		return -EAGAIN;
+	}
+
+	data_length = sizeof(Mpi2FWImageHeader_t);
+	fwpkg_data = pci_alloc_consistent(ioc->pdev, data_length,
+			&fwpkg_data_dma);
+	if (!fwpkg_data) {
+		pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
+				ioc->name, __FILE__, __LINE__, __func__);
+		return -ENOMEM;
+	}
+
+	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
+	if (!smid) {
+		pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
+				ioc->name, __func__);
+		r = -EAGAIN;
+		goto out;
+	}
+
+	ioc->base_cmds.status = MPT3_CMD_PENDING;
+	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
+	ioc->base_cmds.smid = smid;
+	memset(mpi_request, 0, sizeof(Mpi25FWUploadRequest_t));
+	mpi_request->Function = MPI2_FUNCTION_FW_UPLOAD;
+	mpi_request->ImageType = MPI2_FW_UPLOAD_ITYPE_FW_FLASH;
+	mpi_request->ImageSize = cpu_to_le32(data_length);
+	ioc->build_sg(ioc, &mpi_request->SGL, 0, 0, fwpkg_data_dma,
+			data_length);
+	init_completion(&ioc->base_cmds.done);
+	mpt3sas_base_put_smid_default(ioc, smid);
+	/* Wait for 15 seconds */
+	wait_for_completion_timeout(&ioc->base_cmds.done,
+			FW_IMG_HDR_READ_TIMEOUT*HZ);
+	pr_info(MPT3SAS_FMT "%s: complete\n",
+			ioc->name, __func__);
+	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
+		pr_err(MPT3SAS_FMT "%s: timeout\n",
+				ioc->name, __func__);
+		_debug_dump_mf(mpi_request,
+				sizeof(Mpi25FWUploadRequest_t)/4);
+		r = -ETIME;
+	} else {
+		memset(&mpi_reply, 0, sizeof(Mpi2FWUploadReply_t));
+		if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) {
+			memcpy(&mpi_reply, ioc->base_cmds.reply,
+					sizeof(Mpi2FWUploadReply_t));
+			ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
+						MPI2_IOCSTATUS_MASK;
+			if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
+				FWImgHdr = (Mpi2FWImageHeader_t *)fwpkg_data;
+				if (FWImgHdr->PackageVersion.Word) {
+					pr_info(MPT3SAS_FMT "FW Package Version"
+					"(%02d.%02d.%02d.%02d)\n",
+					ioc->name,
+					FWImgHdr->PackageVersion.Struct.Major,
+					FWImgHdr->PackageVersion.Struct.Minor,
+					FWImgHdr->PackageVersion.Struct.Unit,
+					FWImgHdr->PackageVersion.Struct.Dev);
+				}
+			} else {
+				_debug_dump_mf(&mpi_reply,
+						sizeof(Mpi2FWUploadReply_t)/4);
+			}
+		}
+	}
+	ioc->base_cmds.status = MPT3_CMD_NOT_USED;
+out:
+	if (fwpkg_data)
+		pci_free_consistent(ioc->pdev, data_length, fwpkg_data,
+				fwpkg_data_dma);
+	return r;
+}
+
 /**
  * _base_display_ioc_capabilities - Disply IOC's capabilities.
  * @ioc: per adapter object
@@ -4038,6 +4142,7 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
 	Mpi2ConfigReply_t mpi_reply;
 	u32 iounit_pg1_flags;
 
+	ioc->nvme_abort_timeout = 30;
 	mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply, &ioc->manu_pg0);
 	if (ioc->ir_firmware)
 		mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply,
@@ -4056,6 +4161,18 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
 		mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply,
 		    &ioc->manu_pg11);
 	}
+	if (ioc->manu_pg11.AddlFlags2 & NVME_TASK_MNGT_CUSTOM_MASK)
+		ioc->tm_custom_handling = 1;
+	else {
+		ioc->tm_custom_handling = 0;
+		if (ioc->manu_pg11.NVMeAbortTO < NVME_TASK_ABORT_MIN_TIMEOUT)
+			ioc->nvme_abort_timeout = NVME_TASK_ABORT_MIN_TIMEOUT;
+		else if (ioc->manu_pg11.NVMeAbortTO >
+					NVME_TASK_ABORT_MAX_TIMEOUT)
+			ioc->nvme_abort_timeout = NVME_TASK_ABORT_MAX_TIMEOUT;
+		else
+			ioc->nvme_abort_timeout = ioc->manu_pg11.NVMeAbortTO;
+	}
 
 	mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2);
 	mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3);
@@ -4084,6 +4201,27 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
 		ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors;
 }
 
+/**
+ * mpt3sas_free_enclosure_list - release memory
+ * @ioc: per adapter object
+ *
+ * Free memory allocated during encloure add.
+ *
+ * Return nothing.
+ */
+void
+mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc)
+{
+	struct _enclosure_node *enclosure_dev, *enclosure_dev_next;
+
+	/* Free enclosure list */
+	list_for_each_entry_safe(enclosure_dev,
+			enclosure_dev_next, &ioc->enclosure_list, list) {
+		list_del(&enclosure_dev->list);
+		kfree(enclosure_dev);
+	}
+}
+
 /**
  * _base_release_memory_pools - release memory
  * @ioc: per adapter object
@@ -4096,6 +4234,8 @@ static void
 _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 {
 	int i = 0;
+	int j = 0;
+	struct chain_tracker *ct;
 	struct reply_post_struct *rps;
 
 	dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
@@ -4153,7 +4293,14 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 			}
 		} while (ioc->rdpq_array_enable &&
 			   (++i < ioc->reply_queue_count));
-
+		if (ioc->reply_post_free_array &&
+			ioc->rdpq_array_enable) {
+			dma_pool_free(ioc->reply_post_free_array_dma_pool,
+				ioc->reply_post_free_array,
+				ioc->reply_post_free_array_dma);
+			ioc->reply_post_free_array = NULL;
+		}
+		dma_pool_destroy(ioc->reply_post_free_array_dma_pool);
 		dma_pool_destroy(ioc->reply_post_free_dma_pool);
 		kfree(ioc->reply_post);
 	}
@@ -4179,18 +4326,48 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	kfree(ioc->hpr_lookup);
 	kfree(ioc->internal_lookup);
 	if (ioc->chain_lookup) {
-		for (i = 0; i < ioc->chain_depth; i++) {
-			if (ioc->chain_lookup[i].chain_buffer)
-				dma_pool_free(ioc->chain_dma_pool,
-				    ioc->chain_lookup[i].chain_buffer,
-				    ioc->chain_lookup[i].chain_buffer_dma);
+		for (i = 0; i < ioc->scsiio_depth; i++) {
+			for (j = ioc->chains_per_prp_buffer;
+			    j < ioc->chains_needed_per_io; j++) {
+				ct = &ioc->chain_lookup[i].chains_per_smid[j];
+				if (ct && ct->chain_buffer)
+					dma_pool_free(ioc->chain_dma_pool,
+						ct->chain_buffer,
+						ct->chain_buffer_dma);
+			}
+			kfree(ioc->chain_lookup[i].chains_per_smid);
 		}
 		dma_pool_destroy(ioc->chain_dma_pool);
-		free_pages((ulong)ioc->chain_lookup, ioc->chain_pages);
+		kfree(ioc->chain_lookup);
 		ioc->chain_lookup = NULL;
 	}
 }
 
+/**
+ * is_MSB_are_same - checks whether all reply queues in a set are
+ *	having same upper 32bits in their base memory address.
+ * @reply_pool_start_address: Base address of a reply queue set
+ * @pool_sz: Size of single Reply Descriptor Post Queues pool size
+ *
+ * Returns 1 if reply queues in a set have a same upper 32bits
+ * in their base memory address,
+ * else 0
+ */
+
+static int
+is_MSB_are_same(long reply_pool_start_address, u32 pool_sz)
+{
+	long reply_pool_end_address;
+
+	reply_pool_end_address = reply_pool_start_address + pool_sz;
+
+	if (upper_32_bits(reply_pool_start_address) ==
+		upper_32_bits(reply_pool_end_address))
+		return 1;
+	else
+		return 0;
+}
+
 /**
  * _base_allocate_memory_pools - allocate start of day memory pools
  * @ioc: per adapter object
@@ -4203,12 +4380,13 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	struct mpt3sas_facts *facts;
 	u16 max_sge_elements;
 	u16 chains_needed_per_io;
-	u32 sz, total_sz, reply_post_free_sz;
+	u32 sz, total_sz, reply_post_free_sz, reply_post_free_array_sz;
 	u32 retry_sz;
 	u16 max_request_credit, nvme_blocks_needed;
 	unsigned short sg_tablesize;
 	u16 sge_size;
-	int i;
+	int i, j;
+	struct chain_tracker *ct;
 
 	dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
 	    __func__));
@@ -4489,37 +4667,23 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 		ioc->name, ioc->request, ioc->scsiio_depth));
 
 	ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH);
-	sz = ioc->chain_depth * sizeof(struct chain_tracker);
-	ioc->chain_pages = get_order(sz);
-	ioc->chain_lookup = (struct chain_tracker *)__get_free_pages(
-	    GFP_KERNEL, ioc->chain_pages);
+	sz = ioc->scsiio_depth * sizeof(struct chain_lookup);
+	ioc->chain_lookup = kzalloc(sz, GFP_KERNEL);
 	if (!ioc->chain_lookup) {
-		pr_err(MPT3SAS_FMT "chain_lookup: __get_free_pages failed\n",
-			ioc->name);
+		pr_err(MPT3SAS_FMT "chain_lookup: __get_free_pages "
+				"failed\n", ioc->name);
 		goto out;
 	}
-	ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
-	    ioc->chain_segment_sz, 16, 0);
-	if (!ioc->chain_dma_pool) {
-		pr_err(MPT3SAS_FMT "chain_dma_pool: dma_pool_create failed\n",
-			ioc->name);
-		goto out;
-	}
-	for (i = 0; i < ioc->chain_depth; i++) {
-		ioc->chain_lookup[i].chain_buffer = dma_pool_alloc(
-		    ioc->chain_dma_pool , GFP_KERNEL,
-		    &ioc->chain_lookup[i].chain_buffer_dma);
-		if (!ioc->chain_lookup[i].chain_buffer) {
-			ioc->chain_depth = i;
-			goto chain_done;
+
+	sz = ioc->chains_needed_per_io * sizeof(struct chain_tracker);
+	for (i = 0; i < ioc->scsiio_depth; i++) {
+		ioc->chain_lookup[i].chains_per_smid = kzalloc(sz, GFP_KERNEL);
+		if (!ioc->chain_lookup[i].chains_per_smid) {
+			pr_err(MPT3SAS_FMT "chain_lookup: "
+					" kzalloc failed\n", ioc->name);
+			goto out;
 		}
-		total_sz += ioc->chain_segment_sz;
 	}
- chain_done:
-	dinitprintk(ioc, pr_info(MPT3SAS_FMT
-		"chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
-		ioc->name, ioc->chain_depth, ioc->chain_segment_sz,
-		((ioc->chain_depth *  ioc->chain_segment_sz))/1024));
 
 	/* initialize hi-priority queue smid's */
 	ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth,
@@ -4561,6 +4725,7 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	 * be required for NVMe PRP's, only each set of NVMe blocks will be
 	 * contiguous, so a new set is allocated for each possible I/O.
 	 */
+	ioc->chains_per_prp_buffer = 0;
 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
 		nvme_blocks_needed =
 			(ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1;
@@ -4583,6 +4748,11 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 			    ioc->name);
 			goto out;
 		}
+
+		ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz;
+		ioc->chains_per_prp_buffer = min(ioc->chains_per_prp_buffer,
+						ioc->chains_needed_per_io);
+
 		for (i = 0; i < ioc->scsiio_depth; i++) {
 			ioc->pcie_sg_lookup[i].pcie_sgl = dma_pool_alloc(
 				ioc->pcie_sgl_dma_pool, GFP_KERNEL,
@@ -4593,13 +4763,55 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 				    ioc->name);
 				goto out;
 			}
+			for (j = 0; j < ioc->chains_per_prp_buffer; j++) {
+				ct = &ioc->chain_lookup[i].chains_per_smid[j];
+				ct->chain_buffer =
+				    ioc->pcie_sg_lookup[i].pcie_sgl +
+				    (j * ioc->chain_segment_sz);
+				ct->chain_buffer_dma =
+				    ioc->pcie_sg_lookup[i].pcie_sgl_dma +
+				    (j * ioc->chain_segment_sz);
+			}
 		}
 
 		dinitprintk(ioc, pr_info(MPT3SAS_FMT "PCIe sgl pool depth(%d), "
 			"element_size(%d), pool_size(%d kB)\n", ioc->name,
 			ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024));
+		dinitprintk(ioc, pr_info(MPT3SAS_FMT "Number of chains can "
+		    "fit in a PRP page(%d)\n", ioc->name,
+		    ioc->chains_per_prp_buffer));
 		total_sz += sz * ioc->scsiio_depth;
 	}
+
+	ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
+	    ioc->chain_segment_sz, 16, 0);
+	if (!ioc->chain_dma_pool) {
+		pr_err(MPT3SAS_FMT "chain_dma_pool: dma_pool_create failed\n",
+			ioc->name);
+		goto out;
+	}
+	for (i = 0; i < ioc->scsiio_depth; i++) {
+		for (j = ioc->chains_per_prp_buffer;
+				j < ioc->chains_needed_per_io; j++) {
+			ct = &ioc->chain_lookup[i].chains_per_smid[j];
+			ct->chain_buffer = dma_pool_alloc(
+					ioc->chain_dma_pool, GFP_KERNEL,
+					&ct->chain_buffer_dma);
+			if (!ct->chain_buffer) {
+				pr_err(MPT3SAS_FMT "chain_lookup: "
+				" pci_pool_alloc failed\n", ioc->name);
+				_base_release_memory_pools(ioc);
+				goto out;
+			}
+		}
+		total_sz += ioc->chain_segment_sz;
+	}
+
+	dinitprintk(ioc, pr_info(MPT3SAS_FMT
+		"chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
+		ioc->name, ioc->chain_depth, ioc->chain_segment_sz,
+		((ioc->chain_depth *  ioc->chain_segment_sz))/1024));
+
 	/* sense buffers, 4 byte align */
 	sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;
 	ioc->sense_dma_pool = dma_pool_create("sense pool", &ioc->pdev->dev, sz,
@@ -4616,6 +4828,37 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 		    ioc->name);
 		goto out;
 	}
+	/* sense buffer requires to be in same 4 gb region.
+	 * Below function will check the same.
+	 * In case of failure, new pci pool will be created with updated
+	 * alignment. Older allocation and pool will be destroyed.
+	 * Alignment will be used such a way that next allocation if
+	 * success, will always meet same 4gb region requirement.
+	 * Actual requirement is not alignment, but we need start and end of
+	 * DMA address must have same upper 32 bit address.
+	 */
+	if (!is_MSB_are_same((long)ioc->sense, sz)) {
+		//Release Sense pool & Reallocate
+		dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
+		dma_pool_destroy(ioc->sense_dma_pool);
+		ioc->sense = NULL;
+
+		ioc->sense_dma_pool =
+			dma_pool_create("sense pool", &ioc->pdev->dev, sz,
+						roundup_pow_of_two(sz), 0);
+		if (!ioc->sense_dma_pool) {
+			pr_err(MPT3SAS_FMT "sense pool: pci_pool_create failed\n",
+					ioc->name);
+			goto out;
+		}
+		ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, GFP_KERNEL,
+				&ioc->sense_dma);
+		if (!ioc->sense) {
+			pr_err(MPT3SAS_FMT "sense pool: pci_pool_alloc failed\n",
+					ioc->name);
+			goto out;
+		}
+	}
 	dinitprintk(ioc, pr_info(MPT3SAS_FMT
 	    "sense pool(0x%p): depth(%d), element_size(%d), pool_size"
 	    "(%d kB)\n", ioc->name, ioc->sense, ioc->scsiio_depth,
@@ -4675,6 +4918,28 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 		ioc->name, (unsigned long long)ioc->reply_free_dma));
 	total_sz += sz;
 
+	if (ioc->rdpq_array_enable) {
+		reply_post_free_array_sz = ioc->reply_queue_count *
+		    sizeof(Mpi2IOCInitRDPQArrayEntry);
+		ioc->reply_post_free_array_dma_pool =
+		    dma_pool_create("reply_post_free_array pool",
+		    &ioc->pdev->dev, reply_post_free_array_sz, 16, 0);
+		if (!ioc->reply_post_free_array_dma_pool) {
+			dinitprintk(ioc,
+			    pr_info(MPT3SAS_FMT "reply_post_free_array pool: "
+			    "dma_pool_create failed\n", ioc->name));
+			goto out;
+		}
+		ioc->reply_post_free_array =
+		    dma_pool_alloc(ioc->reply_post_free_array_dma_pool,
+		    GFP_KERNEL, &ioc->reply_post_free_array_dma);
+		if (!ioc->reply_post_free_array) {
+			dinitprintk(ioc,
+			    pr_info(MPT3SAS_FMT "reply_post_free_array pool: "
+			    "dma_pool_alloc failed\n", ioc->name));
+			goto out;
+		}
+	}
 	ioc->config_page_sz = 512;
 	ioc->config_page = pci_alloc_consistent(ioc->pdev,
 	    ioc->config_page_sz, &ioc->config_page_dma);
@@ -5002,7 +5267,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
 
 	/* send message 32-bits at a time */
 	for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) {
-		writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell);
+		writel((u32)(request[i]), &ioc->chip->Doorbell);
 		if ((_base_wait_for_doorbell_ack(ioc, 5)))
 			failed = 1;
 	}
@@ -5023,7 +5288,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
 	}
 
 	/* read the first two 16-bits, it gives the total length of the reply */
-	reply[0] = le16_to_cpu(readl(&ioc->chip->Doorbell)
+	reply[0] = (u16)(readl(&ioc->chip->Doorbell)
 	    & MPI2_DOORBELL_DATA_MASK);
 	writel(0, &ioc->chip->HostInterruptStatus);
 	if ((_base_wait_for_doorbell_int(ioc, 5))) {
@@ -5032,7 +5297,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
 			ioc->name, __LINE__);
 		return -EFAULT;
 	}
-	reply[1] = le16_to_cpu(readl(&ioc->chip->Doorbell)
+	reply[1] = (u16)(readl(&ioc->chip->Doorbell)
 	    & MPI2_DOORBELL_DATA_MASK);
 	writel(0, &ioc->chip->HostInterruptStatus);
 
@@ -5046,7 +5311,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
 		if (i >=  reply_bytes/2) /* overflow case */
 			readl(&ioc->chip->Doorbell);
 		else
-			reply[i] = le16_to_cpu(readl(&ioc->chip->Doorbell)
+			reply[i] = (u16)(readl(&ioc->chip->Doorbell)
 			    & MPI2_DOORBELL_DATA_MASK);
 		writel(0, &ioc->chip->HostInterruptStatus);
 	}
@@ -5481,8 +5746,6 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
 	ktime_t current_time;
 	u16 ioc_status;
 	u32 reply_post_free_array_sz = 0;
-	Mpi2IOCInitRDPQArrayEntry *reply_post_free_array = NULL;
-	dma_addr_t reply_post_free_array_dma;
 
 	dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
 	    __func__));
@@ -5516,23 +5779,14 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
 	if (ioc->rdpq_array_enable) {
 		reply_post_free_array_sz = ioc->reply_queue_count *
 		    sizeof(Mpi2IOCInitRDPQArrayEntry);
-		reply_post_free_array = pci_alloc_consistent(ioc->pdev,
-			reply_post_free_array_sz, &reply_post_free_array_dma);
-		if (!reply_post_free_array) {
-			pr_err(MPT3SAS_FMT
-			"reply_post_free_array: pci_alloc_consistent failed\n",
-			ioc->name);
-			r = -ENOMEM;
-			goto out;
-		}
-		memset(reply_post_free_array, 0, reply_post_free_array_sz);
+		memset(ioc->reply_post_free_array, 0, reply_post_free_array_sz);
 		for (i = 0; i < ioc->reply_queue_count; i++)
-			reply_post_free_array[i].RDPQBaseAddress =
+			ioc->reply_post_free_array[i].RDPQBaseAddress =
 			    cpu_to_le64(
 				(u64)ioc->reply_post[i].reply_post_free_dma);
 		mpi_request.MsgFlags = MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE;
 		mpi_request.ReplyDescriptorPostQueueAddress =
-		    cpu_to_le64((u64)reply_post_free_array_dma);
+		    cpu_to_le64((u64)ioc->reply_post_free_array_dma);
 	} else {
 		mpi_request.ReplyDescriptorPostQueueAddress =
 		    cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma);
@@ -5562,7 +5816,7 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
 	if (r != 0) {
 		pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
 		    ioc->name, __func__, r);
-		goto out;
+		return r;
 	}
 
 	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
@@ -5572,11 +5826,6 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
 		r = -EIO;
 	}
 
-out:
-	if (reply_post_free_array)
-		pci_free_consistent(ioc->pdev, reply_post_free_array_sz,
-				    reply_post_free_array,
-				    reply_post_free_array_dma);
 	return r;
 }
 
@@ -6157,12 +6406,6 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
 		    &ioc->internal_free_list);
 	}
 
-	/* chain pool */
-	INIT_LIST_HEAD(&ioc->free_chain_list);
-	for (i = 0; i < ioc->chain_depth; i++)
-		list_add_tail(&ioc->chain_lookup[i].tracker_list,
-		    &ioc->free_chain_list);
-
 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
 
 	/* initialize Reply Free Queue */
@@ -6172,7 +6415,7 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
 		ioc->reply_free[i] = cpu_to_le32(reply_address);
 		if (ioc->is_mcpu_endpoint)
 			_base_clone_reply_to_sys_mem(ioc,
-					(__le32)reply_address, i);
+					reply_address, i);
 	}
 
 	/* initialize reply queues */
@@ -6230,12 +6473,18 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
  skip_init_reply_post_host_index:
 
 	_base_unmask_interrupts(ioc);
+
+	if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
+		r = _base_display_fwpkg_version(ioc);
+		if (r)
+			return r;
+	}
+
+	_base_static_config_pages(ioc);
 	r = _base_event_notification(ioc);
 	if (r)
 		return r;
 
-	_base_static_config_pages(ioc);
-
 	if (ioc->is_driver_loading) {
 
 		if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier
@@ -6492,6 +6741,7 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 	_base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED);
 	_base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD);
 	_base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
+	_base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
 	if (ioc->hba_mpi_version_belonged == MPI26_VERSION) {
 		if (ioc->is_gen35_ioc) {
 			_base_unmask_events(ioc,
@@ -6558,6 +6808,7 @@ mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc)
 	mpt3sas_base_stop_watchdog(ioc);
 	mpt3sas_base_free_resources(ioc);
 	_base_release_memory_pools(ioc);
+	mpt3sas_free_enclosure_list(ioc);
 	pci_set_drvdata(ioc->pdev, NULL);
 	kfree(ioc->cpu_msix_table);
 	if (ioc->is_warpdrive)
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
index ae36d8fb2f2b..f02974c0be4a 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
@@ -74,8 +74,8 @@
 #define MPT3SAS_DRIVER_NAME		"mpt3sas"
 #define MPT3SAS_AUTHOR "Avago Technologies <MPT-FusionLinux.pdl@avagotech.com>"
 #define MPT3SAS_DESCRIPTION	"LSI MPT Fusion SAS 3.0 Device Driver"
-#define MPT3SAS_DRIVER_VERSION		"17.100.00.00"
-#define MPT3SAS_MAJOR_VERSION		17
+#define MPT3SAS_DRIVER_VERSION		"25.100.00.00"
+#define MPT3SAS_MAJOR_VERSION		25
 #define MPT3SAS_MINOR_VERSION		100
 #define MPT3SAS_BUILD_VERSION		0
 #define MPT3SAS_RELEASE_VERSION	00
@@ -138,6 +138,7 @@
 #define MAX_CHAIN_ELEMT_SZ		16
 #define DEFAULT_NUM_FWCHAIN_ELEMTS	8
 
+#define FW_IMG_HDR_READ_TIMEOUT	15
 /*
  * NVMe defines
  */
@@ -145,8 +146,12 @@
 #define	NVME_CMD_PRP1_OFFSET		24	/* PRP1 offset in NVMe cmd */
 #define	NVME_CMD_PRP2_OFFSET		32	/* PRP2 offset in NVMe cmd */
 #define	NVME_ERROR_RESPONSE_SIZE	16	/* Max NVME Error Response */
+#define NVME_TASK_ABORT_MIN_TIMEOUT	6
+#define NVME_TASK_ABORT_MAX_TIMEOUT	60
+#define NVME_TASK_MNGT_CUSTOM_MASK	(0x0010)
 #define	NVME_PRP_PAGE_SIZE		4096	/* Page size */
 
+
 /*
  * reset phases
  */
@@ -362,7 +367,15 @@ struct Mpi2ManufacturingPage11_t {
 	u8	EEDPTagMode;			/* 09h */
 	u8	Reserved3;			/* 0Ah */
 	u8	Reserved4;			/* 0Bh */
-	__le32	Reserved5[23];			/* 0Ch-60h*/
+	__le32	Reserved5[8];			/* 0Ch-2Ch */
+	u16	AddlFlags2;			/* 2Ch */
+	u8	AddlFlags3;			/* 2Eh */
+	u8	Reserved6;			/* 2Fh */
+	__le32	Reserved7[7];			/* 30h - 4Bh */
+	u8	NVMeAbortTO;			/* 4Ch */
+	u8	Reserved8;			/* 4Dh */
+	u16	Reserved9;			/* 4Eh */
+	__le32	Reserved10[4];			/* 50h - 60h */
 };
 
 /**
@@ -572,6 +585,7 @@ struct _pcie_device {
 	u8	enclosure_level;
 	u8	connector_name[4];
 	u8	*serial_number;
+	u8	reset_timeout;
 	struct kref refcount;
 };
 /**
@@ -741,6 +755,17 @@ struct _sas_node {
 	struct list_head sas_port_list;
 };
 
+
+/**
+ * struct _enclosure_node - enclosure information
+ * @list: list of enclosures
+ * @pg0: enclosure pg0;
+ */
+struct _enclosure_node {
+	struct list_head list;
+	Mpi2SasEnclosurePage0_t pg0;
+};
+
 /**
  * enum reset_type - reset state
  * @FORCE_BIG_HAMMER: issue diagnostic reset
@@ -770,7 +795,11 @@ struct pcie_sg_list {
 struct chain_tracker {
 	void *chain_buffer;
 	dma_addr_t chain_buffer_dma;
-	struct list_head tracker_list;
+};
+
+struct chain_lookup {
+	struct chain_tracker *chains_per_smid;
+	atomic_t	chain_offset;
 };
 
 /**
@@ -829,8 +858,8 @@ struct _sc_list {
  */
 struct _event_ack_list {
 	struct list_head list;
-	u16     Event;
-	u32     EventContext;
+	U16     Event;
+	U32     EventContext;
 };
 
 /**
@@ -1009,6 +1038,7 @@ typedef void (*MPT3SAS_FLUSH_RUNNING_CMDS)(struct MPT3SAS_ADAPTER *ioc);
  * @iounit_pg8: static iounit page 8
  * @sas_hba: sas host object
  * @sas_expander_list: expander object list
+ * @enclosure_list: enclosure object list
  * @sas_node_lock:
  * @sas_device_list: sas device object list
  * @sas_device_init_list: sas device object list (used only at init time)
@@ -1194,6 +1224,10 @@ struct MPT3SAS_ADAPTER {
 	void		*event_log;
 	u32		event_masks[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];
 
+	u8		tm_custom_handling;
+	u8		nvme_abort_timeout;
+
+
 	/* static config pages */
 	struct mpt3sas_facts facts;
 	struct mpt3sas_port_facts *pfacts;
@@ -1214,6 +1248,7 @@ struct MPT3SAS_ADAPTER {
 	/* sas hba, expander, and device list */
 	struct _sas_node sas_hba;
 	struct list_head sas_expander_list;
+	struct list_head enclosure_list;
 	spinlock_t	sas_node_lock;
 	struct list_head sas_device_list;
 	struct list_head sas_device_init_list;
@@ -1261,7 +1296,7 @@ struct MPT3SAS_ADAPTER {
 	u32		page_size;
 
 	/* chain */
-	struct chain_tracker *chain_lookup;
+	struct chain_lookup *chain_lookup;
 	struct list_head free_chain_list;
 	struct dma_pool *chain_dma_pool;
 	ulong		chain_pages;
@@ -1315,6 +1350,9 @@ struct MPT3SAS_ADAPTER {
 	u8		rdpq_array_enable;
 	u8		rdpq_array_enable_assigned;
 	struct dma_pool *reply_post_free_dma_pool;
+	struct dma_pool *reply_post_free_array_dma_pool;
+	Mpi2IOCInitRDPQArrayEntry *reply_post_free_array;
+	dma_addr_t reply_post_free_array_dma;
 	u8		reply_queue_count;
 	struct list_head reply_queue_list;
 
@@ -1384,6 +1422,7 @@ int mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc);
 void mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc);
 int mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc);
 void mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc);
+void mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc);
 int mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc,
 	enum reset_type type);
 
@@ -1451,10 +1490,11 @@ u8 mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,
 	u32 reply);
 void mpt3sas_scsih_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase);
 
-int mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
-	u64 lun, u8 type, u16 smid_task, u16 msix_task, ulong timeout);
+int mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, u64 lun,
+	u8 type, u16 smid_task, u16 msix_task, u8 timeout, u8 tr_method);
 int mpt3sas_scsih_issue_locked_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
-	u64 lun, u8 type, u16 smid_task, u16 msix_task, ulong timeout);
+	u64 lun, u8 type, u16 smid_task, u16 msix_task,
+	u8 timeout, u8 tr_method);
 
 void mpt3sas_scsih_set_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle);
 void mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle);
diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.c b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
index d3cb387ba9f4..3269ef43f07e 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_ctl.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
@@ -297,7 +297,7 @@ mpt3sas_ctl_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
 			nvme_error_reply =
 			    (Mpi26NVMeEncapsulatedErrorReply_t *)mpi_reply;
 			sz = min_t(u32, NVME_ERROR_RESPONSE_SIZE,
-			    le32_to_cpu(nvme_error_reply->ErrorResponseCount));
+			    le16_to_cpu(nvme_error_reply->ErrorResponseCount));
 			sense_data = mpt3sas_base_get_sense_buffer(ioc, smid);
 			memcpy(ioc->ctl_cmds.sense, sense_data, sz);
 		}
@@ -644,9 +644,10 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 	MPI2RequestHeader_t *mpi_request = NULL, *request;
 	MPI2DefaultReply_t *mpi_reply;
 	Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request = NULL;
+	struct _pcie_device *pcie_device = NULL;
 	u32 ioc_state;
 	u16 smid;
-	unsigned long timeout;
+	u8 timeout;
 	u8 issue_reset;
 	u32 sz, sz_arg;
 	void *psge;
@@ -659,6 +660,7 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 	long ret;
 	u16 wait_state_count;
 	u16 device_handle = MPT3SAS_INVALID_DEVICE_HANDLE;
+	u8 tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
 
 	issue_reset = 0;
 
@@ -803,12 +805,13 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 		 * Build the PRPs and set direction bits.
 		 * Send the request.
 		 */
-		nvme_encap_request->ErrorResponseBaseAddress = ioc->sense_dma &
-		    0xFFFFFFFF00000000;
+		nvme_encap_request->ErrorResponseBaseAddress =
+		    cpu_to_le64(ioc->sense_dma & 0xFFFFFFFF00000000UL);
 		nvme_encap_request->ErrorResponseBaseAddress |=
-		    (U64)mpt3sas_base_get_sense_buffer_dma(ioc, smid);
+		   cpu_to_le64(le32_to_cpu(
+		   mpt3sas_base_get_sense_buffer_dma(ioc, smid)));
 		nvme_encap_request->ErrorResponseAllocationLength =
-						NVME_ERROR_RESPONSE_SIZE;
+					cpu_to_le16(NVME_ERROR_RESPONSE_SIZE);
 		memset(ioc->ctl_cmds.sense, 0, NVME_ERROR_RESPONSE_SIZE);
 		ioc->build_nvme_prp(ioc, smid, nvme_encap_request,
 		    data_out_dma, data_out_sz, data_in_dma, data_in_sz);
@@ -1073,14 +1076,26 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 				ioc->name,
 				le16_to_cpu(mpi_request->FunctionDependent1));
 			mpt3sas_halt_firmware(ioc);
-			mpt3sas_scsih_issue_locked_tm(ioc,
-			    le16_to_cpu(mpi_request->FunctionDependent1), 0,
-			    MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 0, 30);
+			pcie_device = mpt3sas_get_pdev_by_handle(ioc,
+				le16_to_cpu(mpi_request->FunctionDependent1));
+			if (pcie_device && (!ioc->tm_custom_handling))
+				mpt3sas_scsih_issue_locked_tm(ioc,
+				  le16_to_cpu(mpi_request->FunctionDependent1),
+				  0, MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0,
+				  0, pcie_device->reset_timeout,
+				  tr_method);
+			else
+				mpt3sas_scsih_issue_locked_tm(ioc,
+				  le16_to_cpu(mpi_request->FunctionDependent1),
+				  0, MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0,
+				  0, 30, MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET);
 		} else
 			mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 	}
 
  out:
+	if (pcie_device)
+		pcie_device_put(pcie_device);
 
 	/* free memory associated with sg buffers */
 	if (data_in)
diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.h b/drivers/scsi/mpt3sas/mpt3sas_ctl.h
index a44046cff0f3..18b46faef6f1 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_ctl.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.h
@@ -184,7 +184,7 @@ struct mpt3_ioctl_iocinfo {
 
 
 /* number of event log entries */
-#define MPT3SAS_CTL_EVENT_LOG_SIZE (50)
+#define MPT3SAS_CTL_EVENT_LOG_SIZE (200)
 
 /**
  * struct mpt3_ioctl_eventquery - query event count and type
diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
index 8cd3782fab49..b8d131a455d0 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
@@ -157,8 +157,8 @@ MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=7 ");
 
 
 /* raid transport support */
-struct raid_template *mpt3sas_raid_template;
-struct raid_template *mpt2sas_raid_template;
+static struct raid_template *mpt3sas_raid_template;
+static struct raid_template *mpt2sas_raid_template;
 
 
 /**
@@ -1088,7 +1088,7 @@ _scsih_pcie_device_remove(struct MPT3SAS_ADAPTER *ioc,
 		pcie_device->slot);
 	if (pcie_device->connector_name[0] != '\0')
 		pr_info(MPT3SAS_FMT
-			"removing enclosure level(0x%04x), connector name( %s)\n",
+		    "removing enclosure level(0x%04x), connector name( %s)\n",
 			ioc->name, pcie_device->enclosure_level,
 			pcie_device->connector_name);
 
@@ -1361,6 +1361,30 @@ mpt3sas_scsih_expander_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	return r;
 }
 
+/**
+ * mpt3sas_scsih_enclosure_find_by_handle - exclosure device search
+ * @ioc: per adapter object
+ * @handle: enclosure handle (assigned by firmware)
+ * Context: Calling function should acquire ioc->sas_device_lock
+ *
+ * This searches for enclosure device based on handle, then returns the
+ * enclosure object.
+ */
+static struct _enclosure_node *
+mpt3sas_scsih_enclosure_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
+{
+	struct _enclosure_node *enclosure_dev, *r;
+
+	r = NULL;
+	list_for_each_entry(enclosure_dev, &ioc->enclosure_list, list) {
+		if (le16_to_cpu(enclosure_dev->pg0.EnclosureHandle) != handle)
+			continue;
+		r = enclosure_dev;
+		goto out;
+	}
+out:
+	return r;
+}
 /**
  * mpt3sas_scsih_expander_find_by_sas_address - expander device search
  * @ioc: per adapter object
@@ -2608,6 +2632,7 @@ mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
  * @smid_task: smid assigned to the task
  * @msix_task: MSIX table index supplied by the OS
  * @timeout: timeout in seconds
+ * @tr_method: Target Reset Method
  * Context: user
  *
  * A generic API for sending task management requests to firmware.
@@ -2618,8 +2643,8 @@ mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
  * Return SUCCESS or FAILED.
  */
 int
-mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
-	u64 lun, u8 type, u16 smid_task, u16 msix_task, ulong timeout)
+mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, u64 lun,
+	u8 type, u16 smid_task, u16 msix_task, u8 timeout, u8 tr_method)
 {
 	Mpi2SCSITaskManagementRequest_t *mpi_request;
 	Mpi2SCSITaskManagementReply_t *mpi_reply;
@@ -2665,8 +2690,8 @@ mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
 	}
 
 	dtmprintk(ioc, pr_info(MPT3SAS_FMT
-		"sending tm: handle(0x%04x), task_type(0x%02x), smid(%d)\n",
-		ioc->name, handle, type, smid_task));
+		"sending tm: handle(0x%04x), task_type(0x%02x), smid(%d), timeout(%d), tr_method(0x%x)\n",
+		ioc->name, handle, type, smid_task, timeout, tr_method));
 	ioc->tm_cmds.status = MPT3_CMD_PENDING;
 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
 	ioc->tm_cmds.smid = smid;
@@ -2675,6 +2700,7 @@ mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
 	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
 	mpi_request->DevHandle = cpu_to_le16(handle);
 	mpi_request->TaskType = type;
+	mpi_request->MsgFlags = tr_method;
 	mpi_request->TaskMID = cpu_to_le16(smid_task);
 	int_to_scsilun(lun, (struct scsi_lun *)mpi_request->LUN);
 	mpt3sas_scsih_set_tm_flag(ioc, handle);
@@ -2721,13 +2747,14 @@ mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
 }
 
 int mpt3sas_scsih_issue_locked_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
-	u64 lun, u8 type, u16 smid_task, u16 msix_task, ulong timeout)
+		u64 lun, u8 type, u16 smid_task, u16 msix_task,
+		u8 timeout, u8 tr_method)
 {
 	int ret;
 
 	mutex_lock(&ioc->tm_cmds.mutex);
 	ret = mpt3sas_scsih_issue_tm(ioc, handle, lun, type, smid_task,
-			msix_task, timeout);
+			msix_task, timeout, tr_method);
 	mutex_unlock(&ioc->tm_cmds.mutex);
 
 	return ret;
@@ -2830,6 +2857,8 @@ scsih_abort(struct scsi_cmnd *scmd)
 	u16 handle;
 	int r;
 
+	u8 timeout = 30;
+	struct _pcie_device *pcie_device = NULL;
 	sdev_printk(KERN_INFO, scmd->device,
 		"attempting task abort! scmd(%p)\n", scmd);
 	_scsih_tm_display_info(ioc, scmd);
@@ -2864,15 +2893,20 @@ scsih_abort(struct scsi_cmnd *scmd)
 	mpt3sas_halt_firmware(ioc);
 
 	handle = sas_device_priv_data->sas_target->handle;
+	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
+	if (pcie_device && (!ioc->tm_custom_handling))
+		timeout = ioc->nvme_abort_timeout;
 	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->lun,
 		MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
-		st->smid, st->msix_io, 30);
+		st->smid, st->msix_io, timeout, 0);
 	/* Command must be cleared after abort */
 	if (r == SUCCESS && st->cb_idx != 0xFF)
 		r = FAILED;
  out:
 	sdev_printk(KERN_INFO, scmd->device, "task abort: %s scmd(%p)\n",
 	    ((r == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
+	if (pcie_device)
+		pcie_device_put(pcie_device);
 	return r;
 }
 
@@ -2888,7 +2922,10 @@ scsih_dev_reset(struct scsi_cmnd *scmd)
 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
 	struct MPT3SAS_DEVICE *sas_device_priv_data;
 	struct _sas_device *sas_device = NULL;
+	struct _pcie_device *pcie_device = NULL;
 	u16	handle;
+	u8	tr_method = 0;
+	u8	tr_timeout = 30;
 	int r;
 
 	struct scsi_target *starget = scmd->device->sdev_target;
@@ -2926,8 +2963,16 @@ scsih_dev_reset(struct scsi_cmnd *scmd)
 		goto out;
 	}
 
+	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
+
+	if (pcie_device && (!ioc->tm_custom_handling)) {
+		tr_timeout = pcie_device->reset_timeout;
+		tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
+	} else
+		tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
 	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->lun,
-		MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET, 0, 0, 30);
+		MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET, 0, 0,
+		tr_timeout, tr_method);
 	/* Check for busy commands after reset */
 	if (r == SUCCESS && atomic_read(&scmd->device->device_busy))
 		r = FAILED;
@@ -2937,6 +2982,8 @@ scsih_dev_reset(struct scsi_cmnd *scmd)
 
 	if (sas_device)
 		sas_device_put(sas_device);
+	if (pcie_device)
+		pcie_device_put(pcie_device);
 
 	return r;
 }
@@ -2953,7 +3000,10 @@ scsih_target_reset(struct scsi_cmnd *scmd)
 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
 	struct MPT3SAS_DEVICE *sas_device_priv_data;
 	struct _sas_device *sas_device = NULL;
+	struct _pcie_device *pcie_device = NULL;
 	u16	handle;
+	u8	tr_method = 0;
+	u8	tr_timeout = 30;
 	int r;
 	struct scsi_target *starget = scmd->device->sdev_target;
 	struct MPT3SAS_TARGET *target_priv_data = starget->hostdata;
@@ -2990,8 +3040,16 @@ scsih_target_reset(struct scsi_cmnd *scmd)
 		goto out;
 	}
 
+	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
+
+	if (pcie_device && (!ioc->tm_custom_handling)) {
+		tr_timeout = pcie_device->reset_timeout;
+		tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
+	} else
+		tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
 	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, 0,
-		MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 0, 30);
+		MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 0,
+	    tr_timeout, tr_method);
 	/* Check for busy commands after reset */
 	if (r == SUCCESS && atomic_read(&starget->target_busy))
 		r = FAILED;
@@ -3001,7 +3059,8 @@ scsih_target_reset(struct scsi_cmnd *scmd)
 
 	if (sas_device)
 		sas_device_put(sas_device);
-
+	if (pcie_device)
+		pcie_device_put(pcie_device);
 	return r;
 }
 
@@ -3535,6 +3594,7 @@ _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	unsigned long flags;
 	struct _tr_list *delayed_tr;
 	u32 ioc_state;
+	u8 tr_method = 0;
 
 	if (ioc->pci_error_recovery) {
 		dewtprintk(ioc, pr_info(MPT3SAS_FMT
@@ -3577,6 +3637,11 @@ _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 			sas_address = pcie_device->wwid;
 		}
 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
+		if (pcie_device && (!ioc->tm_custom_handling))
+			tr_method =
+			    MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
+		else
+			tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
 	}
 	if (sas_target_priv_data) {
 		dewtprintk(ioc, pr_info(MPT3SAS_FMT
@@ -3640,6 +3705,7 @@ _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
 	mpi_request->DevHandle = cpu_to_le16(handle);
 	mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
+	mpi_request->MsgFlags = tr_method;
 	set_bit(handle, ioc->device_remove_in_progress);
 	mpt3sas_base_put_smid_hi_priority(ioc, smid, 0);
 	mpt3sas_trigger_master(ioc, MASTER_TRIGGER_DEVICE_REMOVAL);
@@ -3680,11 +3746,7 @@ _scsih_tm_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
 	u32 ioc_state;
 	struct _sc_list *delayed_sc;
 
-	if (ioc->remove_host) {
-		dewtprintk(ioc, pr_info(MPT3SAS_FMT
-			"%s: host has been removed\n", __func__, ioc->name));
-		return 1;
-	} else if (ioc->pci_error_recovery) {
+	if (ioc->pci_error_recovery) {
 		dewtprintk(ioc, pr_info(MPT3SAS_FMT
 			"%s: host in pci error recovery\n", __func__,
 			ioc->name));
@@ -3725,7 +3787,7 @@ _scsih_tm_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
 		if (!delayed_sc)
 			return _scsih_check_for_pending_tm(ioc, smid);
 		INIT_LIST_HEAD(&delayed_sc->list);
-		delayed_sc->handle = mpi_request_tm->DevHandle;
+		delayed_sc->handle = le16_to_cpu(mpi_request_tm->DevHandle);
 		list_add_tail(&delayed_sc->list, &ioc->delayed_sc_list);
 		dewtprintk(ioc, pr_info(MPT3SAS_FMT
 		    "DELAYED:sc:handle(0x%04x), (open)\n",
@@ -3806,8 +3868,7 @@ _scsih_tm_tr_volume_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	u16 smid;
 	struct _tr_list *delayed_tr;
 
-	if (ioc->shost_recovery || ioc->remove_host ||
-	    ioc->pci_error_recovery) {
+	if (ioc->pci_error_recovery) {
 		dewtprintk(ioc, pr_info(MPT3SAS_FMT
 			"%s: host reset in progress!\n",
 			__func__, ioc->name));
@@ -3860,8 +3921,7 @@ _scsih_tm_volume_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 	Mpi2SCSITaskManagementReply_t *mpi_reply =
 	    mpt3sas_base_get_reply_virt_addr(ioc, reply);
 
-	if (ioc->shost_recovery || ioc->remove_host ||
-	    ioc->pci_error_recovery) {
+	if (ioc->shost_recovery || ioc->pci_error_recovery) {
 		dewtprintk(ioc, pr_info(MPT3SAS_FMT
 			"%s: host reset in progress!\n",
 			__func__, ioc->name));
@@ -3903,8 +3963,8 @@ _scsih_tm_volume_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
  * Context - processed in interrupt context.
  */
 static void
-_scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 event,
-				u32 event_context)
+_scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, U16 event,
+				U32 event_context)
 {
 	Mpi2EventAckRequest_t *ack_request;
 	int i = smid - ioc->internal_smid;
@@ -3979,13 +4039,13 @@ _scsih_issue_delayed_sas_io_unit_ctrl(struct MPT3SAS_ADAPTER *ioc,
 
 	dewtprintk(ioc, pr_info(MPT3SAS_FMT
 	    "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
-	    ioc->name, le16_to_cpu(handle), smid,
+	    ioc->name, handle, smid,
 	    ioc->tm_sas_control_cb_idx));
 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
 	memset(mpi_request, 0, sizeof(Mpi2SasIoUnitControlRequest_t));
 	mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
 	mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
-	mpi_request->DevHandle = handle;
+	mpi_request->DevHandle = cpu_to_le16(handle);
 	mpt3sas_base_put_smid_default(ioc, smid);
 }
 
@@ -5618,10 +5678,10 @@ static int
 _scsih_expander_add(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 {
 	struct _sas_node *sas_expander;
+	struct _enclosure_node *enclosure_dev;
 	Mpi2ConfigReply_t mpi_reply;
 	Mpi2ExpanderPage0_t expander_pg0;
 	Mpi2ExpanderPage1_t expander_pg1;
-	Mpi2SasEnclosurePage0_t enclosure_pg0;
 	u32 ioc_status;
 	u16 parent_handle;
 	u64 sas_address, sas_address_parent = 0;
@@ -5743,11 +5803,12 @@ _scsih_expander_add(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	}
 
 	if (sas_expander->enclosure_handle) {
-		if (!(mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
-		    &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
-		   sas_expander->enclosure_handle)))
+		enclosure_dev =
+			mpt3sas_scsih_enclosure_find_by_handle(ioc,
+						sas_expander->enclosure_handle);
+		if (enclosure_dev)
 			sas_expander->enclosure_logical_id =
-			    le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
+			    le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
 	}
 
 	_scsih_expander_node_add(ioc, sas_expander);
@@ -5890,52 +5951,6 @@ _scsih_check_access_status(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
 	return rc;
 }
 
-/**
- * _scsih_get_enclosure_logicalid_chassis_slot - get device's
- *			EnclosureLogicalID and ChassisSlot information.
- * @ioc: per adapter object
- * @sas_device_pg0: SAS device page0
- * @sas_device: per sas device object
- *
- * Returns nothing.
- */
-static void
-_scsih_get_enclosure_logicalid_chassis_slot(struct MPT3SAS_ADAPTER *ioc,
-	Mpi2SasDevicePage0_t *sas_device_pg0, struct _sas_device *sas_device)
-{
-	Mpi2ConfigReply_t mpi_reply;
-	Mpi2SasEnclosurePage0_t enclosure_pg0;
-
-	if (!sas_device_pg0 || !sas_device)
-		return;
-
-	sas_device->enclosure_handle =
-	    le16_to_cpu(sas_device_pg0->EnclosureHandle);
-	sas_device->is_chassis_slot_valid = 0;
-
-	if (!le16_to_cpu(sas_device_pg0->EnclosureHandle))
-		return;
-
-	if (mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
-	    &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
-	    le16_to_cpu(sas_device_pg0->EnclosureHandle))) {
-		pr_err(MPT3SAS_FMT
-		    "Enclosure Pg0 read failed for handle(0x%04x)\n",
-		    ioc->name, le16_to_cpu(sas_device_pg0->EnclosureHandle));
-		return;
-	}
-
-	sas_device->enclosure_logical_id =
-	    le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
-
-	if (le16_to_cpu(enclosure_pg0.Flags) &
-	    MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
-		sas_device->is_chassis_slot_valid = 1;
-		sas_device->chassis_slot = enclosure_pg0.ChassisSlot;
-	}
-}
-
-
 /**
  * _scsih_check_device - checking device responsiveness
  * @ioc: per adapter object
@@ -5953,6 +5968,7 @@ _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
 	Mpi2ConfigReply_t mpi_reply;
 	Mpi2SasDevicePage0_t sas_device_pg0;
 	struct _sas_device *sas_device;
+	struct _enclosure_node *enclosure_dev = NULL;
 	u32 ioc_status;
 	unsigned long flags;
 	u64 sas_address;
@@ -6007,8 +6023,21 @@ _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
 			sas_device->connector_name[0] = '\0';
 		}
 
-		_scsih_get_enclosure_logicalid_chassis_slot(ioc,
-		    &sas_device_pg0, sas_device);
+		sas_device->enclosure_handle =
+				le16_to_cpu(sas_device_pg0.EnclosureHandle);
+		sas_device->is_chassis_slot_valid = 0;
+		enclosure_dev = mpt3sas_scsih_enclosure_find_by_handle(ioc,
+						sas_device->enclosure_handle);
+		if (enclosure_dev) {
+			sas_device->enclosure_logical_id =
+			    le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
+			if (le16_to_cpu(enclosure_dev->pg0.Flags) &
+			    MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
+				sas_device->is_chassis_slot_valid = 1;
+				sas_device->chassis_slot =
+					enclosure_dev->pg0.ChassisSlot;
+			}
+		}
 	}
 
 	/* check if device is present */
@@ -6055,12 +6084,11 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
 {
 	Mpi2ConfigReply_t mpi_reply;
 	Mpi2SasDevicePage0_t sas_device_pg0;
-	Mpi2SasEnclosurePage0_t enclosure_pg0;
 	struct _sas_device *sas_device;
+	struct _enclosure_node *enclosure_dev = NULL;
 	u32 ioc_status;
 	u64 sas_address;
 	u32 device_info;
-	int encl_pg0_rc = -1;
 
 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
 	    MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
@@ -6106,12 +6134,12 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
 	}
 
 	if (sas_device_pg0.EnclosureHandle) {
-		encl_pg0_rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
-		    &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
-		    sas_device_pg0.EnclosureHandle);
-		if (encl_pg0_rc)
-			pr_info(MPT3SAS_FMT
-			    "Enclosure Pg0 read failed for handle(0x%04x)\n",
+		enclosure_dev =
+			mpt3sas_scsih_enclosure_find_by_handle(ioc,
+			    le16_to_cpu(sas_device_pg0.EnclosureHandle));
+		if (enclosure_dev == NULL)
+			pr_info(MPT3SAS_FMT "Enclosure handle(0x%04x)"
+			    "doesn't match with enclosure device!\n",
 			    ioc->name, sas_device_pg0.EnclosureHandle);
 	}
 
@@ -6152,18 +6180,16 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
 		sas_device->enclosure_level = 0;
 		sas_device->connector_name[0] = '\0';
 	}
-
-	/* get enclosure_logical_id & chassis_slot */
+	/* get enclosure_logical_id & chassis_slot*/
 	sas_device->is_chassis_slot_valid = 0;
-	if (encl_pg0_rc == 0) {
+	if (enclosure_dev) {
 		sas_device->enclosure_logical_id =
-		    le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
-
-		if (le16_to_cpu(enclosure_pg0.Flags) &
+		    le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
+		if (le16_to_cpu(enclosure_dev->pg0.Flags) &
 		    MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
 			sas_device->is_chassis_slot_valid = 1;
 			sas_device->chassis_slot =
-			    enclosure_pg0.ChassisSlot;
+					enclosure_dev->pg0.ChassisSlot;
 		}
 	}
 
@@ -6845,8 +6871,8 @@ _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	Mpi26PCIeDevicePage0_t pcie_device_pg0;
 	Mpi26PCIeDevicePage2_t pcie_device_pg2;
 	Mpi2ConfigReply_t mpi_reply;
-	Mpi2SasEnclosurePage0_t enclosure_pg0;
 	struct _pcie_device *pcie_device;
+	struct _enclosure_node *enclosure_dev;
 	u32 pcie_device_type;
 	u32 ioc_status;
 	u64 wwid;
@@ -6917,7 +6943,7 @@ _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	if (pcie_device->enclosure_handle != 0)
 		pcie_device->slot = le16_to_cpu(pcie_device_pg0.Slot);
 
-	if (le16_to_cpu(pcie_device_pg0.Flags) &
+	if (le32_to_cpu(pcie_device_pg0.Flags) &
 	    MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) {
 		pcie_device->enclosure_level = pcie_device_pg0.EnclosureLevel;
 		memcpy(&pcie_device->connector_name[0],
@@ -6928,13 +6954,14 @@ _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	}
 
 	/* get enclosure_logical_id */
-	if (pcie_device->enclosure_handle &&
-		!(mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
-			&enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
-			pcie_device->enclosure_handle)))
-		pcie_device->enclosure_logical_id =
-			le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
-
+	if (pcie_device->enclosure_handle) {
+		enclosure_dev =
+			mpt3sas_scsih_enclosure_find_by_handle(ioc,
+						pcie_device->enclosure_handle);
+		if (enclosure_dev)
+			pcie_device->enclosure_logical_id =
+			    le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
+	}
 	/* TODO -- Add device name once FW supports it */
 	if (mpt3sas_config_get_pcie_device_pg2(ioc, &mpi_reply,
 		&pcie_device_pg2, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle)) {
@@ -6953,6 +6980,11 @@ _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	}
 	pcie_device->nvme_mdts =
 		le32_to_cpu(pcie_device_pg2.MaximumDataTransferSize);
+	if (pcie_device_pg2.ControllerResetTO)
+		pcie_device->reset_timeout =
+			pcie_device_pg2.ControllerResetTO;
+	else
+		pcie_device->reset_timeout = 30;
 
 	if (ioc->wait_for_discovery_to_complete)
 		_scsih_pcie_device_init_add(ioc, pcie_device);
@@ -7205,6 +7237,9 @@ _scsih_pcie_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
 	case MPI26_EVENT_PCIDEV_STAT_RC_ASYNC_NOTIFICATION:
 		reason_str = "internal async notification";
 		break;
+	case MPI26_EVENT_PCIDEV_STAT_RC_PCIE_HOT_RESET_FAILED:
+		reason_str = "pcie hot reset failed";
+		break;
 	default:
 		reason_str = "unknown reason";
 		break;
@@ -7320,10 +7355,60 @@ static void
 _scsih_sas_enclosure_dev_status_change_event(struct MPT3SAS_ADAPTER *ioc,
 	struct fw_event_work *fw_event)
 {
+	Mpi2ConfigReply_t mpi_reply;
+	struct _enclosure_node *enclosure_dev = NULL;
+	Mpi2EventDataSasEnclDevStatusChange_t *event_data =
+		(Mpi2EventDataSasEnclDevStatusChange_t *)fw_event->event_data;
+	int rc;
+	u16 enclosure_handle = le16_to_cpu(event_data->EnclosureHandle);
+
 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
 		_scsih_sas_enclosure_dev_status_change_event_debug(ioc,
 		     (Mpi2EventDataSasEnclDevStatusChange_t *)
 		     fw_event->event_data);
+	if (ioc->shost_recovery)
+		return;
+
+	if (enclosure_handle)
+		enclosure_dev =
+			mpt3sas_scsih_enclosure_find_by_handle(ioc,
+						enclosure_handle);
+	switch (event_data->ReasonCode) {
+	case MPI2_EVENT_SAS_ENCL_RC_ADDED:
+		if (!enclosure_dev) {
+			enclosure_dev =
+				kzalloc(sizeof(struct _enclosure_node),
+					GFP_KERNEL);
+			if (!enclosure_dev) {
+				pr_info(MPT3SAS_FMT
+					"failure at %s:%d/%s()!\n", ioc->name,
+					__FILE__, __LINE__, __func__);
+				return;
+			}
+			rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
+				&enclosure_dev->pg0,
+				MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
+				enclosure_handle);
+
+			if (rc || (le16_to_cpu(mpi_reply.IOCStatus) &
+						MPI2_IOCSTATUS_MASK)) {
+				kfree(enclosure_dev);
+				return;
+			}
+
+			list_add_tail(&enclosure_dev->list,
+							&ioc->enclosure_list);
+		}
+		break;
+	case MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING:
+		if (enclosure_dev) {
+			list_del(&enclosure_dev->list);
+			kfree(enclosure_dev);
+		}
+		break;
+	default:
+		break;
+	}
 }
 
 /**
@@ -7409,7 +7494,7 @@ _scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,
 		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
 		r = mpt3sas_scsih_issue_tm(ioc, handle, lun,
 			MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK, st->smid,
-			st->msix_io, 30);
+			st->msix_io, 30, 0);
 		if (r == FAILED) {
 			sdev_printk(KERN_WARNING, sdev,
 			    "mpt3sas_scsih_issue_tm: FAILED when sending "
@@ -7450,7 +7535,7 @@ _scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,
 
 		r = mpt3sas_scsih_issue_tm(ioc, handle, sdev->lun,
 			MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK, st->smid,
-			st->msix_io, 30);
+			st->msix_io, 30, 0);
 		if (r == FAILED || st->cb_idx != 0xFF) {
 			sdev_printk(KERN_WARNING, sdev,
 			    "mpt3sas_scsih_issue_tm: ABORT_TASK: FAILED : "
@@ -7526,6 +7611,44 @@ _scsih_sas_discovery_event(struct MPT3SAS_ADAPTER *ioc,
 	}
 }
 
+/**
+ * _scsih_sas_device_discovery_error_event - display SAS device discovery error
+ *						events
+ * @ioc: per adapter object
+ * @fw_event: The fw_event_work object
+ * Context: user.
+ *
+ * Return nothing.
+ */
+static void
+_scsih_sas_device_discovery_error_event(struct MPT3SAS_ADAPTER *ioc,
+	struct fw_event_work *fw_event)
+{
+	Mpi25EventDataSasDeviceDiscoveryError_t *event_data =
+		(Mpi25EventDataSasDeviceDiscoveryError_t *)fw_event->event_data;
+
+	switch (event_data->ReasonCode) {
+	case MPI25_EVENT_SAS_DISC_ERR_SMP_FAILED:
+		pr_warn(MPT3SAS_FMT "SMP command sent to the expander"
+			"(handle:0x%04x, sas_address:0x%016llx,"
+			"physical_port:0x%02x) has failed",
+			ioc->name, le16_to_cpu(event_data->DevHandle),
+			(unsigned long long)le64_to_cpu(event_data->SASAddress),
+			event_data->PhysicalPort);
+		break;
+	case MPI25_EVENT_SAS_DISC_ERR_SMP_TIMEOUT:
+		pr_warn(MPT3SAS_FMT "SMP command sent to the expander"
+			"(handle:0x%04x, sas_address:0x%016llx,"
+			"physical_port:0x%02x) has timed out",
+			ioc->name, le16_to_cpu(event_data->DevHandle),
+			(unsigned long long)le64_to_cpu(event_data->SASAddress),
+			event_data->PhysicalPort);
+		break;
+	default:
+		break;
+	}
+}
+
 /**
  * _scsih_pcie_enumeration_event - handle enumeration events
  * @ioc: per adapter object
@@ -8360,12 +8483,23 @@ Mpi2SasDevicePage0_t *sas_device_pg0)
 	struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
 	struct scsi_target *starget;
 	struct _sas_device *sas_device = NULL;
+	struct _enclosure_node *enclosure_dev = NULL;
 	unsigned long flags;
 
+	if (sas_device_pg0->EnclosureHandle) {
+		enclosure_dev =
+			mpt3sas_scsih_enclosure_find_by_handle(ioc,
+				le16_to_cpu(sas_device_pg0->EnclosureHandle));
+		if (enclosure_dev == NULL)
+			pr_info(MPT3SAS_FMT "Enclosure handle(0x%04x)"
+			    "doesn't match with enclosure device!\n",
+			    ioc->name, sas_device_pg0->EnclosureHandle);
+	}
 	spin_lock_irqsave(&ioc->sas_device_lock, flags);
 	list_for_each_entry(sas_device, &ioc->sas_device_list, list) {
-		if ((sas_device->sas_address == sas_device_pg0->SASAddress) &&
-			(sas_device->slot == sas_device_pg0->Slot)) {
+		if ((sas_device->sas_address == le64_to_cpu(
+		    sas_device_pg0->SASAddress)) && (sas_device->slot ==
+		    le16_to_cpu(sas_device_pg0->Slot))) {
 			sas_device->responding = 1;
 			starget = sas_device->starget;
 			if (starget && starget->hostdata) {
@@ -8377,7 +8511,7 @@ Mpi2SasDevicePage0_t *sas_device_pg0)
 			if (starget) {
 				starget_printk(KERN_INFO, starget,
 				    "handle(0x%04x), sas_addr(0x%016llx)\n",
-				    sas_device_pg0->DevHandle,
+				    le16_to_cpu(sas_device_pg0->DevHandle),
 				    (unsigned long long)
 				    sas_device->sas_address);
 
@@ -8389,7 +8523,7 @@ Mpi2SasDevicePage0_t *sas_device_pg0)
 					 sas_device->enclosure_logical_id,
 					 sas_device->slot);
 			}
-			if (sas_device_pg0->Flags &
+			if (le16_to_cpu(sas_device_pg0->Flags) &
 			      MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
 				sas_device->enclosure_level =
 				   sas_device_pg0->EnclosureLevel;
@@ -8400,17 +8534,30 @@ Mpi2SasDevicePage0_t *sas_device_pg0)
 				sas_device->connector_name[0] = '\0';
 			}
 
-			_scsih_get_enclosure_logicalid_chassis_slot(ioc,
-			    sas_device_pg0, sas_device);
+			sas_device->enclosure_handle =
+				le16_to_cpu(sas_device_pg0->EnclosureHandle);
+			sas_device->is_chassis_slot_valid = 0;
+			if (enclosure_dev) {
+				sas_device->enclosure_logical_id = le64_to_cpu(
+					enclosure_dev->pg0.EnclosureLogicalID);
+				if (le16_to_cpu(enclosure_dev->pg0.Flags) &
+				    MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
+					sas_device->is_chassis_slot_valid = 1;
+					sas_device->chassis_slot =
+						enclosure_dev->pg0.ChassisSlot;
+				}
+			}
 
-			if (sas_device->handle == sas_device_pg0->DevHandle)
+			if (sas_device->handle == le16_to_cpu(
+			    sas_device_pg0->DevHandle))
 				goto out;
 			pr_info("\thandle changed from(0x%04x)!!!\n",
 			    sas_device->handle);
-			sas_device->handle = sas_device_pg0->DevHandle;
+			sas_device->handle = le16_to_cpu(
+			    sas_device_pg0->DevHandle);
 			if (sas_target_priv_data)
 				sas_target_priv_data->handle =
-					sas_device_pg0->DevHandle;
+				    le16_to_cpu(sas_device_pg0->DevHandle);
 			goto out;
 		}
 	}
@@ -8418,6 +8565,52 @@ Mpi2SasDevicePage0_t *sas_device_pg0)
 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 }
 
+/**
+ * _scsih_create_enclosure_list_after_reset - Free Existing list,
+ *	And create enclosure list by scanning all Enclosure Page(0)s
+ * @ioc: per adapter object
+ *
+ * Return nothing.
+ */
+static void
+_scsih_create_enclosure_list_after_reset(struct MPT3SAS_ADAPTER *ioc)
+{
+	struct _enclosure_node *enclosure_dev;
+	Mpi2ConfigReply_t mpi_reply;
+	u16 enclosure_handle;
+	int rc;
+
+	/* Free existing enclosure list */
+	mpt3sas_free_enclosure_list(ioc);
+
+	/* Re constructing enclosure list after reset*/
+	enclosure_handle = 0xFFFF;
+	do {
+		enclosure_dev =
+			kzalloc(sizeof(struct _enclosure_node), GFP_KERNEL);
+		if (!enclosure_dev) {
+			pr_err(MPT3SAS_FMT
+				"failure at %s:%d/%s()!\n", ioc->name,
+				__FILE__, __LINE__, __func__);
+			return;
+		}
+		rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
+				&enclosure_dev->pg0,
+				MPI2_SAS_ENCLOS_PGAD_FORM_GET_NEXT_HANDLE,
+				enclosure_handle);
+
+		if (rc || (le16_to_cpu(mpi_reply.IOCStatus) &
+						MPI2_IOCSTATUS_MASK)) {
+			kfree(enclosure_dev);
+			return;
+		}
+		list_add_tail(&enclosure_dev->list,
+						&ioc->enclosure_list);
+		enclosure_handle =
+			le16_to_cpu(enclosure_dev->pg0.EnclosureHandle);
+	} while (1);
+}
+
 /**
  * _scsih_search_responding_sas_devices -
  * @ioc: per adapter object
@@ -8449,15 +8642,10 @@ _scsih_search_responding_sas_devices(struct MPT3SAS_ADAPTER *ioc)
 		    MPI2_IOCSTATUS_MASK;
 		if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
 			break;
-		handle = sas_device_pg0.DevHandle =
-				le16_to_cpu(sas_device_pg0.DevHandle);
+		handle = le16_to_cpu(sas_device_pg0.DevHandle);
 		device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
 		if (!(_scsih_is_end_device(device_info)))
 			continue;
-		sas_device_pg0.SASAddress =
-				le64_to_cpu(sas_device_pg0.SASAddress);
-		sas_device_pg0.Slot = le16_to_cpu(sas_device_pg0.Slot);
-		sas_device_pg0.Flags = le16_to_cpu(sas_device_pg0.Flags);
 		_scsih_mark_responding_sas_device(ioc, &sas_device_pg0);
 	}
 
@@ -8487,8 +8675,9 @@ _scsih_mark_responding_pcie_device(struct MPT3SAS_ADAPTER *ioc,
 
 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
 	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {
-		if ((pcie_device->wwid == pcie_device_pg0->WWID) &&
-		    (pcie_device->slot == pcie_device_pg0->Slot)) {
+		if ((pcie_device->wwid == le64_to_cpu(pcie_device_pg0->WWID))
+		    && (pcie_device->slot == le16_to_cpu(
+		    pcie_device_pg0->Slot))) {
 			pcie_device->responding = 1;
 			starget = pcie_device->starget;
 			if (starget && starget->hostdata) {
@@ -8523,14 +8712,16 @@ _scsih_mark_responding_pcie_device(struct MPT3SAS_ADAPTER *ioc,
 				pcie_device->connector_name[0] = '\0';
 			}
 
-			if (pcie_device->handle == pcie_device_pg0->DevHandle)
+			if (pcie_device->handle == le16_to_cpu(
+			    pcie_device_pg0->DevHandle))
 				goto out;
 			pr_info("\thandle changed from(0x%04x)!!!\n",
 			    pcie_device->handle);
-			pcie_device->handle = pcie_device_pg0->DevHandle;
+			pcie_device->handle = le16_to_cpu(
+			    pcie_device_pg0->DevHandle);
 			if (sas_target_priv_data)
 				sas_target_priv_data->handle =
-				    pcie_device_pg0->DevHandle;
+				    le16_to_cpu(pcie_device_pg0->DevHandle);
 			goto out;
 		}
 	}
@@ -8579,10 +8770,6 @@ _scsih_search_responding_pcie_devices(struct MPT3SAS_ADAPTER *ioc)
 		device_info = le32_to_cpu(pcie_device_pg0.DeviceInfo);
 		if (!(_scsih_is_nvme_device(device_info)))
 			continue;
-		pcie_device_pg0.WWID = le64_to_cpu(pcie_device_pg0.WWID),
-		pcie_device_pg0.Slot = le16_to_cpu(pcie_device_pg0.Slot);
-		pcie_device_pg0.Flags = le32_to_cpu(pcie_device_pg0.Flags);
-		pcie_device_pg0.DevHandle = handle;
 		_scsih_mark_responding_pcie_device(ioc, &pcie_device_pg0);
 	}
 out:
@@ -8736,22 +8923,16 @@ _scsih_mark_responding_expander(struct MPT3SAS_ADAPTER *ioc,
 {
 	struct _sas_node *sas_expander = NULL;
 	unsigned long flags;
-	int i, encl_pg0_rc = -1;
-	Mpi2ConfigReply_t mpi_reply;
-	Mpi2SasEnclosurePage0_t enclosure_pg0;
+	int i;
+	struct _enclosure_node *enclosure_dev = NULL;
 	u16 handle = le16_to_cpu(expander_pg0->DevHandle);
+	u16 enclosure_handle = le16_to_cpu(expander_pg0->EnclosureHandle);
 	u64 sas_address = le64_to_cpu(expander_pg0->SASAddress);
 
-	if (le16_to_cpu(expander_pg0->EnclosureHandle)) {
-		encl_pg0_rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
-		    &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
-		    le16_to_cpu(expander_pg0->EnclosureHandle));
-		if (encl_pg0_rc)
-			pr_info(MPT3SAS_FMT
-			    "Enclosure Pg0 read failed for handle(0x%04x)\n",
-			    ioc->name,
-			    le16_to_cpu(expander_pg0->EnclosureHandle));
-	}
+	if (enclosure_handle)
+		enclosure_dev =
+			mpt3sas_scsih_enclosure_find_by_handle(ioc,
+							enclosure_handle);
 
 	spin_lock_irqsave(&ioc->sas_node_lock, flags);
 	list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {
@@ -8759,12 +8940,12 @@ _scsih_mark_responding_expander(struct MPT3SAS_ADAPTER *ioc,
 			continue;
 		sas_expander->responding = 1;
 
-		if (!encl_pg0_rc)
+		if (enclosure_dev) {
 			sas_expander->enclosure_logical_id =
-			    le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
-
-		sas_expander->enclosure_handle =
-		    le16_to_cpu(expander_pg0->EnclosureHandle);
+			    le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
+			sas_expander->enclosure_handle =
+			    le16_to_cpu(expander_pg0->EnclosureHandle);
+		}
 
 		if (sas_expander->handle == handle)
 			goto out;
@@ -9286,6 +9467,7 @@ mpt3sas_scsih_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase)
 		if ((!ioc->is_driver_loading) && !(disable_discovery > 0 &&
 		    !ioc->sas_hba.num_phys)) {
 			_scsih_prep_device_scan(ioc);
+			_scsih_create_enclosure_list_after_reset(ioc);
 			_scsih_search_responding_sas_devices(ioc);
 			_scsih_search_responding_pcie_devices(ioc);
 			_scsih_search_responding_raid_devices(ioc);
@@ -9356,6 +9538,9 @@ _mpt3sas_fw_work(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)
 	case MPI2_EVENT_SAS_DISCOVERY:
 		_scsih_sas_discovery_event(ioc, fw_event);
 		break;
+	case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
+		_scsih_sas_device_discovery_error_event(ioc, fw_event);
+		break;
 	case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
 		_scsih_sas_broadcast_primitive_event(ioc, fw_event);
 		break;
@@ -9433,8 +9618,8 @@ mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,
 	u16 sz;
 	Mpi26EventDataActiveCableExcept_t *ActiveCableEventData;
 
-	/* events turned off due to host reset or driver unloading */
-	if (ioc->remove_host || ioc->pci_error_recovery)
+	/* events turned off due to host reset */
+	if (ioc->pci_error_recovery)
 		return 1;
 
 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
@@ -9540,6 +9725,7 @@ mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,
 	case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
 	case MPI2_EVENT_IR_OPERATION_STATUS:
 	case MPI2_EVENT_SAS_DISCOVERY:
+	case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
 	case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
 	case MPI2_EVENT_IR_PHYSICAL_DISK:
 	case MPI2_EVENT_PCIE_ENUMERATION:
@@ -10513,6 +10699,7 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	INIT_LIST_HEAD(&ioc->sas_device_list);
 	INIT_LIST_HEAD(&ioc->sas_device_init_list);
 	INIT_LIST_HEAD(&ioc->sas_expander_list);
+	INIT_LIST_HEAD(&ioc->enclosure_list);
 	INIT_LIST_HEAD(&ioc->pcie_device_list);
 	INIT_LIST_HEAD(&ioc->pcie_device_init_list);
 	INIT_LIST_HEAD(&ioc->fw_event_list);
@@ -11100,10 +11287,10 @@ _mpt3sas_exit(void)
 	pr_info("mpt3sas version %s unloading\n",
 				MPT3SAS_DRIVER_VERSION);
 
-	pci_unregister_driver(&mpt3sas_driver);
-
 	mpt3sas_ctl_exit(hbas_to_enumerate);
 
+	pci_unregister_driver(&mpt3sas_driver);
+
 	scsih_exit();
 }
 
diff --git a/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c b/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c
index 6bfcee4757e0..45aa94915cbf 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c
@@ -177,7 +177,8 @@ mpt3sas_init_warpdrive_properties(struct MPT3SAS_ADAPTER *ioc,
 		if (mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,
 		    &pd_pg0, MPI2_PHYSDISK_PGAD_FORM_PHYSDISKNUM,
 		    vol_pg0->PhysDisk[count].PhysDiskNum) ||
-		    pd_pg0.DevHandle == MPT3SAS_INVALID_DEVICE_HANDLE) {
+		    le16_to_cpu(pd_pg0.DevHandle) ==
+		    MPT3SAS_INVALID_DEVICE_HANDLE) {
 			pr_info(MPT3SAS_FMT "WarpDrive : Direct IO is "
 			    "disabled for the drive with handle(0x%04x) member"
 			    "handle retrieval failed for member number=%d\n",
diff --git a/drivers/scsi/mvumi.c b/drivers/scsi/mvumi.c
index fe97401ad192..b3cd9a6b1d30 100644
--- a/drivers/scsi/mvumi.c
+++ b/drivers/scsi/mvumi.c
@@ -2155,7 +2155,7 @@ static enum blk_eh_timer_return mvumi_timed_out(struct scsi_cmnd *scmd)
 	mvumi_return_cmd(mhba, cmd);
 	spin_unlock_irqrestore(mhba->shost->host_lock, flags);
 
-	return BLK_EH_NOT_HANDLED;
+	return BLK_EH_DONE;
 }
 
 static int
@@ -2693,22 +2693,4 @@ static struct pci_driver mvumi_pci_driver = {
 #endif
 };
 
-/**
- * mvumi_init - Driver load entry point
- */
-static int __init mvumi_init(void)
-{
-	return pci_register_driver(&mvumi_pci_driver);
-}
-
-/**
- * mvumi_exit - Driver unload entry point
- */
-static void __exit mvumi_exit(void)
-{
-
-	pci_unregister_driver(&mvumi_pci_driver);
-}
-
-module_init(mvumi_init);
-module_exit(mvumi_exit);
+module_pci_driver(mvumi_pci_driver);
diff --git a/drivers/scsi/osd/osd_initiator.c b/drivers/scsi/osd/osd_initiator.c
index e18877177f1b..67b14576fff2 100644
--- a/drivers/scsi/osd/osd_initiator.c
+++ b/drivers/scsi/osd/osd_initiator.c
@@ -99,7 +99,7 @@ static int _osd_get_print_system_info(struct osd_dev *od,
 	int nelem = ARRAY_SIZE(get_attrs), a = 0;
 	int ret;
 
-	or = osd_start_request(od, GFP_KERNEL);
+	or = osd_start_request(od);
 	if (!or)
 		return -ENOMEM;
 
@@ -409,16 +409,15 @@ static void _osd_request_free(struct osd_request *or)
 	kfree(or);
 }
 
-struct osd_request *osd_start_request(struct osd_dev *dev, gfp_t gfp)
+struct osd_request *osd_start_request(struct osd_dev *dev)
 {
 	struct osd_request *or;
 
-	or = _osd_request_alloc(gfp);
+	or = _osd_request_alloc(GFP_KERNEL);
 	if (!or)
 		return NULL;
 
 	or->osd_dev = dev;
-	or->alloc_flags = gfp;
 	or->timeout = dev->def_timeout;
 	or->retries = OSD_REQ_RETRIES;
 
@@ -546,7 +545,7 @@ static int _osd_realloc_seg(struct osd_request *or,
 	if (seg->alloc_size >= max_bytes)
 		return 0;
 
-	buff = krealloc(seg->buff, max_bytes, or->alloc_flags);
+	buff = krealloc(seg->buff, max_bytes, GFP_KERNEL);
 	if (!buff) {
 		OSD_ERR("Failed to Realloc %d-bytes was-%d\n", max_bytes,
 			seg->alloc_size);
@@ -728,7 +727,7 @@ static int _osd_req_list_objects(struct osd_request *or,
 		_osd_req_encode_olist(or, list);
 
 	WARN_ON(or->in.bio);
-	bio = bio_map_kern(q, list, len, or->alloc_flags);
+	bio = bio_map_kern(q, list, len, GFP_KERNEL);
 	if (IS_ERR(bio)) {
 		OSD_ERR("!!! Failed to allocate list_objects BIO\n");
 		return PTR_ERR(bio);
@@ -1190,14 +1189,14 @@ static int _req_append_segment(struct osd_request *or,
 			pad_buff = io->pad_buff;
 
 		ret = blk_rq_map_kern(io->req->q, io->req, pad_buff, padding,
-				       or->alloc_flags);
+				       GFP_KERNEL);
 		if (ret)
 			return ret;
 		io->total_bytes += padding;
 	}
 
 	ret = blk_rq_map_kern(io->req->q, io->req, seg->buff, seg->total_bytes,
-			       or->alloc_flags);
+			       GFP_KERNEL);
 	if (ret)
 		return ret;
 
@@ -1564,14 +1563,14 @@ static int _osd_req_finalize_data_integrity(struct osd_request *or,
  * osd_finalize_request and helpers
  */
 static struct request *_make_request(struct request_queue *q, bool has_write,
-			      struct _osd_io_info *oii, gfp_t flags)
+			      struct _osd_io_info *oii)
 {
 	struct request *req;
 	struct bio *bio = oii->bio;
 	int ret;
 
 	req = blk_get_request(q, has_write ? REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN,
-			flags);
+			0);
 	if (IS_ERR(req))
 		return req;
 
@@ -1589,13 +1588,12 @@ static struct request *_make_request(struct request_queue *q, bool has_write,
 static int _init_blk_request(struct osd_request *or,
 	bool has_in, bool has_out)
 {
-	gfp_t flags = or->alloc_flags;
 	struct scsi_device *scsi_device = or->osd_dev->scsi_device;
 	struct request_queue *q = scsi_device->request_queue;
 	struct request *req;
 	int ret;
 
-	req = _make_request(q, has_out, has_out ? &or->out : &or->in, flags);
+	req = _make_request(q, has_out, has_out ? &or->out : &or->in);
 	if (IS_ERR(req)) {
 		ret = PTR_ERR(req);
 		goto out;
@@ -1611,7 +1609,7 @@ static int _init_blk_request(struct osd_request *or,
 		or->out.req = req;
 		if (has_in) {
 			/* allocate bidi request */
-			req = _make_request(q, false, &or->in, flags);
+			req = _make_request(q, false, &or->in);
 			if (IS_ERR(req)) {
 				OSD_DEBUG("blk_get_request for bidi failed\n");
 				ret = PTR_ERR(req);
@@ -1842,14 +1840,14 @@ int osd_req_decode_sense_full(struct osd_request *or,
 		case osd_sense_response_integrity_check:
 		{
 			struct osd_sense_response_integrity_check_descriptor
-				*osricd = cur_descriptor;
-			const unsigned len =
-					  sizeof(osricd->integrity_check_value);
-			char key_dump[len*4 + 2]; /* 2nibbles+space+ASCII */
-
-			hex_dump_to_buffer(osricd->integrity_check_value, len,
-				       32, 1, key_dump, sizeof(key_dump), true);
-			OSD_SENSE_PRINT2("response_integrity [%s]\n", key_dump);
+				*d = cur_descriptor;
+			/* 2nibbles+space+ASCII */
+			char dump[sizeof(d->integrity_check_value) * 4 + 2];
+
+			hex_dump_to_buffer(d->integrity_check_value,
+					sizeof(d->integrity_check_value),
+					32, 1, dump, sizeof(dump), true);
+			OSD_SENSE_PRINT2("response_integrity [%s]\n", dump);
 		}
 		case osd_sense_attribute_identification:
 		{
diff --git a/drivers/scsi/osst.c b/drivers/scsi/osst.c
index 20ec1c01dbd5..2bbe797f8c3d 100644
--- a/drivers/scsi/osst.c
+++ b/drivers/scsi/osst.c
@@ -368,7 +368,7 @@ static int osst_execute(struct osst_request *SRpnt, const unsigned char *cmd,
 	int write = (data_direction == DMA_TO_DEVICE);
 
 	req = blk_get_request(SRpnt->stp->device->request_queue,
-			write ? REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, GFP_KERNEL);
+			write ? REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, 0);
 	if (IS_ERR(req))
 		return DRIVER_ERROR << 24;
 
diff --git a/drivers/scsi/pcmcia/aha152x_stub.c b/drivers/scsi/pcmcia/aha152x_stub.c
index 7d1609fa233c..df82a349e969 100644
--- a/drivers/scsi/pcmcia/aha152x_stub.c
+++ b/drivers/scsi/pcmcia/aha152x_stub.c
@@ -220,16 +220,4 @@ static struct pcmcia_driver aha152x_cs_driver = {
 	.id_table       = aha152x_ids,
 	.resume		= aha152x_resume,
 };
-
-static int __init init_aha152x_cs(void)
-{
-	return pcmcia_register_driver(&aha152x_cs_driver);
-}
-
-static void __exit exit_aha152x_cs(void)
-{
-	pcmcia_unregister_driver(&aha152x_cs_driver);
-}
-
-module_init(init_aha152x_cs);
-module_exit(exit_aha152x_cs);
+module_pcmcia_driver(aha152x_cs_driver);
diff --git a/drivers/scsi/pcmcia/nsp_cs.c b/drivers/scsi/pcmcia/nsp_cs.c
index 5fb6eefc6541..f3230494a8c9 100644
--- a/drivers/scsi/pcmcia/nsp_cs.c
+++ b/drivers/scsi/pcmcia/nsp_cs.c
@@ -1742,19 +1742,6 @@ static struct pcmcia_driver nsp_driver = {
 	.suspend	= nsp_cs_suspend,
 	.resume		= nsp_cs_resume,
 };
-
-static int __init nsp_cs_init(void)
-{
-	return pcmcia_register_driver(&nsp_driver);
-}
-
-static void __exit nsp_cs_exit(void)
-{
-	pcmcia_unregister_driver(&nsp_driver);
-}
-
-
-module_init(nsp_cs_init)
-module_exit(nsp_cs_exit)
+module_pcmcia_driver(nsp_driver);
 
 /* end */
diff --git a/drivers/scsi/pcmcia/nsp_cs.h b/drivers/scsi/pcmcia/nsp_cs.h
index afd64f0adc4b..ea5122f3396d 100644
--- a/drivers/scsi/pcmcia/nsp_cs.h
+++ b/drivers/scsi/pcmcia/nsp_cs.h
@@ -326,10 +326,6 @@ static struct Scsi_Host *nsp_detect(struct scsi_host_template *sht);
 /* Interrupt handler */
 //static irqreturn_t nspintr(int irq, void *dev_id);
 
-/* Module entry point*/
-static int  __init nsp_cs_init(void);
-static void __exit nsp_cs_exit(void);
-
 /* Debug */
 #ifdef NSP_DEBUG
 static void show_command (struct scsi_cmnd *SCpnt);
diff --git a/drivers/scsi/pcmcia/qlogic_stub.c b/drivers/scsi/pcmcia/qlogic_stub.c
index 0556054764dc..d4e7e36254b0 100644
--- a/drivers/scsi/pcmcia/qlogic_stub.c
+++ b/drivers/scsi/pcmcia/qlogic_stub.c
@@ -300,18 +300,7 @@ static struct pcmcia_driver qlogic_cs_driver = {
 	.resume		= qlogic_resume,
 };
 
-static int __init init_qlogic_cs(void)
-{
-	return pcmcia_register_driver(&qlogic_cs_driver);
-}
-
-static void __exit exit_qlogic_cs(void)
-{
-	pcmcia_unregister_driver(&qlogic_cs_driver);
-}
-
 MODULE_AUTHOR("Tom Zerucha, Michael Griffith");
 MODULE_DESCRIPTION("Driver for the PCMCIA Qlogic FAS SCSI controllers");
 MODULE_LICENSE("GPL");
-module_init(init_qlogic_cs);
-module_exit(exit_qlogic_cs);
+module_pcmcia_driver(qlogic_cs_driver);
diff --git a/drivers/scsi/pcmcia/sym53c500_cs.c b/drivers/scsi/pcmcia/sym53c500_cs.c
index 20011c8afbb5..a3b63bea0e50 100644
--- a/drivers/scsi/pcmcia/sym53c500_cs.c
+++ b/drivers/scsi/pcmcia/sym53c500_cs.c
@@ -880,18 +880,4 @@ static struct pcmcia_driver sym53c500_cs_driver = {
 	.id_table       = sym53c500_ids,
 	.resume		= sym53c500_resume,
 };
-
-static int __init
-init_sym53c500_cs(void)
-{
-	return pcmcia_register_driver(&sym53c500_cs_driver);
-}
-
-static void __exit
-exit_sym53c500_cs(void)
-{
-	pcmcia_unregister_driver(&sym53c500_cs_driver);
-}
-
-module_init(init_sym53c500_cs);
-module_exit(exit_sym53c500_cs);
+module_pcmcia_driver(sym53c500_cs_driver);
diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c
index db88a8e7ee0e..4dd6cad330e8 100644
--- a/drivers/scsi/pm8001/pm8001_hwi.c
+++ b/drivers/scsi/pm8001/pm8001_hwi.c
@@ -3607,7 +3607,7 @@ int pm8001_mpi_reg_resp(struct pm8001_hba_info *pm8001_ha, void *piomb)
 		break;
 	default:
 		PM8001_MSG_DBG(pm8001_ha,
-		 pm8001_printk("DEVREG_FAILURE_DEVICE_TYPE_NOT_UNSORPORTED\n"));
+			pm8001_printk("DEVREG_FAILURE_DEVICE_TYPE_NOT_SUPPORTED\n"));
 		break;
 	}
 	complete(pm8001_dev->dcompletion);
diff --git a/drivers/scsi/qedf/drv_fcoe_fw_funcs.c b/drivers/scsi/qedf/drv_fcoe_fw_funcs.c
index a980ef756a67..5bd10b534c99 100644
--- a/drivers/scsi/qedf/drv_fcoe_fw_funcs.c
+++ b/drivers/scsi/qedf/drv_fcoe_fw_funcs.c
@@ -1,5 +1,5 @@
 /* QLogic FCoE Offload Driver
- * Copyright (c) 2016-2017 Cavium Inc.
+ * Copyright (c) 2016-2018 Cavium Inc.
  *
  * This software is available under the terms of the GNU General Public License
  * (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/drv_fcoe_fw_funcs.h b/drivers/scsi/qedf/drv_fcoe_fw_funcs.h
index b5c236efd465..42fde55ac735 100644
--- a/drivers/scsi/qedf/drv_fcoe_fw_funcs.h
+++ b/drivers/scsi/qedf/drv_fcoe_fw_funcs.h
@@ -1,5 +1,5 @@
 /* QLogic FCoE Offload Driver
- * Copyright (c) 2016-2017 Cavium Inc.
+ * Copyright (c) 2016-2018 Cavium Inc.
  *
  * This software is available under the terms of the GNU General Public License
  * (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/drv_scsi_fw_funcs.c b/drivers/scsi/qedf/drv_scsi_fw_funcs.c
index 5d5095e3d96d..29a55257224f 100644
--- a/drivers/scsi/qedf/drv_scsi_fw_funcs.c
+++ b/drivers/scsi/qedf/drv_scsi_fw_funcs.c
@@ -1,5 +1,5 @@
 /* QLogic FCoE Offload Driver
- * Copyright (c) 2016-2017 Cavium Inc.
+ * Copyright (c) 2016-2018 Cavium Inc.
  *
  * This software is available under the terms of the GNU General Public License
  * (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/drv_scsi_fw_funcs.h b/drivers/scsi/qedf/drv_scsi_fw_funcs.h
index 8fbe6e4d0b4f..bf102204fe56 100644
--- a/drivers/scsi/qedf/drv_scsi_fw_funcs.h
+++ b/drivers/scsi/qedf/drv_scsi_fw_funcs.h
@@ -1,5 +1,5 @@
 /* QLogic FCoE Offload Driver
- * Copyright (c) 2016-2017 Cavium Inc.
+ * Copyright (c) 2016-2018 Cavium Inc.
  *
  * This software is available under the terms of the GNU General Public License
  * (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/qedf.h b/drivers/scsi/qedf/qedf.h
index c105a2e48ac1..2c78d8fb9122 100644
--- a/drivers/scsi/qedf/qedf.h
+++ b/drivers/scsi/qedf/qedf.h
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -180,6 +180,7 @@ struct qedf_rport {
 	spinlock_t rport_lock;
 #define QEDF_RPORT_SESSION_READY 1
 #define QEDF_RPORT_UPLOADING_CONNECTION	2
+#define QEDF_RPORT_IN_RESET 3
 	unsigned long flags;
 	unsigned long retry_delay_timestamp;
 	struct fc_rport *rport;
@@ -300,6 +301,7 @@ struct qedf_ctx {
 #define QEDF_FALLBACK_VLAN	1002
 #define QEDF_DEFAULT_PRIO	3
 	int vlan_id;
+	u8 prio;
 	struct qed_dev *cdev;
 	struct qed_dev_fcoe_info dev_info;
 	struct qed_int_info int_info;
@@ -365,6 +367,7 @@ struct qedf_ctx {
 #define QEDF_IO_WORK_MIN		64
 	mempool_t *io_mempool;
 	struct workqueue_struct *dpc_wq;
+	struct delayed_work grcdump_work;
 
 	u32 slow_sge_ios;
 	u32 fast_sge_ios;
@@ -383,11 +386,16 @@ struct qedf_ctx {
 	u32 flogi_failed;
 
 	/* Used for fc statistics */
+	struct mutex stats_mutex;
 	u64 input_requests;
 	u64 output_requests;
 	u64 control_requests;
 	u64 packet_aborts;
 	u64 alloc_failures;
+	u8 lun_resets;
+	u8 target_resets;
+	u8 task_set_fulls;
+	u8 busy;
 };
 
 struct io_bdt {
@@ -496,7 +504,10 @@ extern int qedf_post_io_req(struct qedf_rport *fcport,
 extern void qedf_process_seq_cleanup_compl(struct qedf_ctx *qedf,
 	struct fcoe_cqe *cqe, struct qedf_ioreq *io_req);
 extern int qedf_send_flogi(struct qedf_ctx *qedf);
+extern void qedf_get_protocol_tlv_data(void *dev, void *data);
 extern void qedf_fp_io_handler(struct work_struct *work);
+extern void qedf_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data);
+extern void qedf_wq_grcdump(struct work_struct *work);
 
 #define FCOE_WORD_TO_BYTE  4
 #define QEDF_MAX_TASK_NUM	0xFFFF
diff --git a/drivers/scsi/qedf/qedf_attr.c b/drivers/scsi/qedf/qedf_attr.c
index fa6727685627..0487b7237104 100644
--- a/drivers/scsi/qedf/qedf_attr.c
+++ b/drivers/scsi/qedf/qedf_attr.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/qedf_dbg.c b/drivers/scsi/qedf/qedf_dbg.c
index bd1cef25a900..f2397ee9ba69 100644
--- a/drivers/scsi/qedf/qedf_dbg.c
+++ b/drivers/scsi/qedf/qedf_dbg.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -147,7 +147,7 @@ qedf_get_grc_dump(struct qed_dev *cdev, const struct qed_common_ops *common,
 	if (!*buf)
 		return -EINVAL;
 
-	return common->dbg_grc(cdev, *buf, grcsize);
+	return common->dbg_all_data(cdev, *buf);
 }
 
 void
diff --git a/drivers/scsi/qedf/qedf_dbg.h b/drivers/scsi/qedf/qedf_dbg.h
index 77c27e888969..dd0109653aa3 100644
--- a/drivers/scsi/qedf/qedf_dbg.h
+++ b/drivers/scsi/qedf/qedf_dbg.h
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/qedf_debugfs.c b/drivers/scsi/qedf/qedf_debugfs.c
index c539a7ae3a7e..c29c162a494f 100644
--- a/drivers/scsi/qedf/qedf_debugfs.c
+++ b/drivers/scsi/qedf/qedf_debugfs.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 QLogic Corporation
+ *  Copyright (c) 2016-2018 QLogic Corporation
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -439,7 +439,6 @@ qedf_dbg_offload_stats_open(struct inode *inode, struct file *file)
 	return single_open(file, qedf_offload_stats_show, qedf);
 }
 
-
 const struct file_operations qedf_dbg_fops[] = {
 	qedf_dbg_fileops(qedf, fp_int),
 	qedf_dbg_fileops_seq(qedf, io_trace),
diff --git a/drivers/scsi/qedf/qedf_els.c b/drivers/scsi/qedf/qedf_els.c
index aa22b11436ba..04f0c4d2e256 100644
--- a/drivers/scsi/qedf/qedf_els.c
+++ b/drivers/scsi/qedf/qedf_els.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -14,8 +14,8 @@ static int qedf_initiate_els(struct qedf_rport *fcport, unsigned int op,
 	void (*cb_func)(struct qedf_els_cb_arg *cb_arg),
 	struct qedf_els_cb_arg *cb_arg, uint32_t timer_msec)
 {
-	struct qedf_ctx *qedf = fcport->qedf;
-	struct fc_lport *lport = qedf->lport;
+	struct qedf_ctx *qedf;
+	struct fc_lport *lport;
 	struct qedf_ioreq *els_req;
 	struct qedf_mp_req *mp_req;
 	struct fc_frame_header *fc_hdr;
@@ -29,6 +29,15 @@ static int qedf_initiate_els(struct qedf_rport *fcport, unsigned int op,
 	unsigned long flags;
 	u16 sqe_idx;
 
+	if (!fcport) {
+		QEDF_ERR(NULL, "fcport is NULL");
+		rc = -EINVAL;
+		goto els_err;
+	}
+
+	qedf = fcport->qedf;
+	lport = qedf->lport;
+
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Sending ELS\n");
 
 	rc = fc_remote_port_chkready(fcport->rport);
@@ -201,6 +210,14 @@ static void qedf_rrq_compl(struct qedf_els_cb_arg *cb_arg)
 		kref_put(&orig_io_req->refcount, qedf_release_cmd);
 
 out_free:
+	/*
+	 * Release a reference to the rrq request if we timed out as the
+	 * rrq completion handler is called directly from the timeout handler
+	 * and not from els_compl where the reference would have normally been
+	 * released.
+	 */
+	if (rrq_req->event == QEDF_IOREQ_EV_ELS_TMO)
+		kref_put(&rrq_req->refcount, qedf_release_cmd);
 	kfree(cb_arg);
 }
 
@@ -322,6 +339,17 @@ void qedf_restart_rport(struct qedf_rport *fcport)
 	if (!fcport)
 		return;
 
+	if (test_bit(QEDF_RPORT_IN_RESET, &fcport->flags) ||
+	    !test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) ||
+	    test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
+		QEDF_ERR(&(fcport->qedf->dbg_ctx), "fcport %p already in reset or not offloaded.\n",
+		    fcport);
+		return;
+	}
+
+	/* Set that we are now in reset */
+	set_bit(QEDF_RPORT_IN_RESET, &fcport->flags);
+
 	rdata = fcport->rdata;
 	if (rdata) {
 		lport = fcport->qedf->lport;
@@ -334,6 +362,7 @@ void qedf_restart_rport(struct qedf_rport *fcport)
 		if (rdata)
 			fc_rport_login(rdata);
 	}
+	clear_bit(QEDF_RPORT_IN_RESET, &fcport->flags);
 }
 
 static void qedf_l2_els_compl(struct qedf_els_cb_arg *cb_arg)
diff --git a/drivers/scsi/qedf/qedf_fip.c b/drivers/scsi/qedf/qedf_fip.c
index 773558fc0697..3fd3af799b3d 100644
--- a/drivers/scsi/qedf/qedf_fip.c
+++ b/drivers/scsi/qedf/qedf_fip.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -23,6 +23,7 @@ void qedf_fcoe_send_vlan_req(struct qedf_ctx *qedf)
 	struct fip_vlan *vlan;
 #define MY_FIP_ALL_FCF_MACS        ((__u8[6]) { 1, 0x10, 0x18, 1, 0, 2 })
 	static u8 my_fcoe_all_fcfs[ETH_ALEN] = MY_FIP_ALL_FCF_MACS;
+	unsigned long flags = 0;
 
 	skb = dev_alloc_skb(sizeof(struct fip_vlan));
 	if (!skb)
@@ -65,7 +66,9 @@ void qedf_fcoe_send_vlan_req(struct qedf_ctx *qedf)
 		kfree_skb(skb);
 		return;
 	}
-	qed_ops->ll2->start_xmit(qedf->cdev, skb);
+
+	set_bit(QED_LL2_XMIT_FLAGS_FIP_DISCOVERY, &flags);
+	qed_ops->ll2->start_xmit(qedf->cdev, skb, flags);
 }
 
 static void qedf_fcoe_process_vlan_resp(struct qedf_ctx *qedf,
@@ -134,12 +137,12 @@ void qedf_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb)
 
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2, "FIP frame send: "
 	    "dest=%pM op=%x sub=%x vlan=%04x.", eth_hdr->h_dest, op, sub,
-	    ntohs(vlan_tci));
+	    vlan_tci);
 	if (qedf_dump_frames)
 		print_hex_dump(KERN_WARNING, "fip ", DUMP_PREFIX_OFFSET, 16, 1,
 		    skb->data, skb->len, false);
 
-	qed_ops->ll2->start_xmit(qedf->cdev, skb);
+	qed_ops->ll2->start_xmit(qedf->cdev, skb, 0);
 }
 
 /* Process incoming FIP frames. */
@@ -181,6 +184,7 @@ void qedf_fip_recv(struct qedf_ctx *qedf, struct sk_buff *skb)
 			QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
 			    "Dropping CVL since FCF has not been selected "
 			    "yet.");
+			kfree_skb(skb);
 			return;
 		}
 
diff --git a/drivers/scsi/qedf/qedf_hsi.h b/drivers/scsi/qedf/qedf_hsi.h
index 503c1ae3ccd0..f6f634e48d69 100644
--- a/drivers/scsi/qedf/qedf_hsi.h
+++ b/drivers/scsi/qedf/qedf_hsi.h
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
diff --git a/drivers/scsi/qedf/qedf_io.c b/drivers/scsi/qedf/qedf_io.c
index 50a50c4249d0..6bbc38b1b465 100644
--- a/drivers/scsi/qedf/qedf_io.c
+++ b/drivers/scsi/qedf/qedf_io.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -23,12 +23,31 @@ static void qedf_cmd_timeout(struct work_struct *work)
 
 	struct qedf_ioreq *io_req =
 	    container_of(work, struct qedf_ioreq, timeout_work.work);
-	struct qedf_ctx *qedf = io_req->fcport->qedf;
-	struct qedf_rport *fcport = io_req->fcport;
+	struct qedf_ctx *qedf;
+	struct qedf_rport *fcport;
 	u8 op = 0;
 
+	if (io_req == NULL) {
+		QEDF_INFO(NULL, QEDF_LOG_IO, "io_req is NULL.\n");
+		return;
+	}
+
+	fcport = io_req->fcport;
+	if (io_req->fcport == NULL) {
+		QEDF_INFO(NULL, QEDF_LOG_IO,  "fcport is NULL.\n");
+		return;
+	}
+
+	qedf = fcport->qedf;
+
 	switch (io_req->cmd_type) {
 	case QEDF_ABTS:
+		if (qedf == NULL) {
+			QEDF_INFO(NULL, QEDF_LOG_IO, "qedf is NULL for xid=0x%x.\n",
+			    io_req->xid);
+			return;
+		}
+
 		QEDF_ERR((&qedf->dbg_ctx), "ABTS timeout, xid=0x%x.\n",
 		    io_req->xid);
 		/* Cleanup timed out ABTS */
@@ -931,6 +950,15 @@ qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
 		return 0;
 	}
 
+	if (!qedf->pdev->msix_enabled) {
+		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
+		    "Completing sc_cmd=%p DID_NO_CONNECT as MSI-X is not enabled.\n",
+		    sc_cmd);
+		sc_cmd->result = DID_NO_CONNECT << 16;
+		sc_cmd->scsi_done(sc_cmd);
+		return 0;
+	}
+
 	rval = fc_remote_port_chkready(rport);
 	if (rval) {
 		sc_cmd->result = rval;
@@ -1200,6 +1228,12 @@ void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 					fcport->retry_delay_timestamp =
 					    jiffies + (qualifier * HZ / 10);
 				}
+				/* Record stats */
+				if (io_req->cdb_status ==
+				    SAM_STAT_TASK_SET_FULL)
+					qedf->task_set_fulls++;
+				else
+					qedf->busy++;
 			}
 		}
 		if (io_req->fcp_resid)
@@ -1414,6 +1448,12 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
 	if (!fcport)
 		return;
 
+	/* Check that fcport is still offloaded */
+	if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+		QEDF_ERR(NULL, "fcport is no longer offloaded.\n");
+		return;
+	}
+
 	qedf = fcport->qedf;
 	cmd_mgr = qedf->cmd_mgr;
 
@@ -1430,8 +1470,8 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
 			rc = kref_get_unless_zero(&io_req->refcount);
 			if (!rc) {
 				QEDF_ERR(&(qedf->dbg_ctx),
-				    "Could not get kref for io_req=0x%p.\n",
-				    io_req);
+				    "Could not get kref for ELS io_req=0x%p xid=0x%x.\n",
+				    io_req, io_req->xid);
 				continue;
 			}
 			qedf_flush_els_req(qedf, io_req);
@@ -1442,6 +1482,31 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
 			goto free_cmd;
 		}
 
+		if (io_req->cmd_type == QEDF_ABTS) {
+			rc = kref_get_unless_zero(&io_req->refcount);
+			if (!rc) {
+				QEDF_ERR(&(qedf->dbg_ctx),
+				    "Could not get kref for abort io_req=0x%p xid=0x%x.\n",
+				    io_req, io_req->xid);
+				continue;
+			}
+			QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+			    "Flushing abort xid=0x%x.\n", io_req->xid);
+
+			clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
+
+			if (io_req->sc_cmd) {
+				if (io_req->return_scsi_cmd_on_abts)
+					qedf_scsi_done(qedf, io_req, DID_ERROR);
+			}
+
+			/* Notify eh_abort handler that ABTS is complete */
+			complete(&io_req->abts_done);
+			kref_put(&io_req->refcount, qedf_release_cmd);
+
+			goto free_cmd;
+		}
+
 		if (!io_req->sc_cmd)
 			continue;
 		if (lun > 0) {
@@ -1457,7 +1522,7 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
 		rc = kref_get_unless_zero(&io_req->refcount);
 		if (!rc) {
 			QEDF_ERR(&(qedf->dbg_ctx), "Could not get kref for "
-			    "io_req=0x%p\n", io_req);
+			    "io_req=0x%p xid=0x%x\n", io_req, io_req->xid);
 			continue;
 		}
 		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
@@ -1519,6 +1584,21 @@ int qedf_initiate_abts(struct qedf_ioreq *io_req, bool return_scsi_cmd_on_abts)
 		goto abts_err;
 	}
 
+	if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
+		QEDF_ERR(&qedf->dbg_ctx, "fcport is uploading.\n");
+		rc = 1;
+		goto out;
+	}
+
+	if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
+	    test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
+	    test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
+		QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
+			  "cleanup or abort processing or already "
+			  "completed.\n", io_req->xid);
+		rc = 1;
+		goto out;
+	}
 
 	kref_get(&io_req->refcount);
 
@@ -1558,6 +1638,7 @@ int qedf_initiate_abts(struct qedf_ioreq *io_req, bool return_scsi_cmd_on_abts)
 	 * task at the firmware.
 	 */
 	qedf_initiate_cleanup(io_req, return_scsi_cmd_on_abts);
+out:
 	return rc;
 }
 
@@ -1866,6 +1947,11 @@ static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
 		goto reset_tmf_err;
 	}
 
+	if (tm_flags == FCP_TMF_LUN_RESET)
+		qedf->lun_resets++;
+	else if (tm_flags == FCP_TMF_TGT_RESET)
+		qedf->target_resets++;
+
 	/* Initialize rest of io_req fields */
 	io_req->sc_cmd = sc_cmd;
 	io_req->fcport = fcport;
diff --git a/drivers/scsi/qedf/qedf_main.c b/drivers/scsi/qedf/qedf_main.c
index 284ccb566b19..90394cef0f41 100644
--- a/drivers/scsi/qedf/qedf_main.c
+++ b/drivers/scsi/qedf/qedf_main.c
@@ -1,6 +1,6 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
@@ -44,20 +44,20 @@ module_param_named(debug, qedf_debug, uint, S_IRUGO);
 MODULE_PARM_DESC(debug, " Debug mask. Pass '1' to enable default debugging"
 	" mask");
 
-static uint qedf_fipvlan_retries = 30;
+static uint qedf_fipvlan_retries = 60;
 module_param_named(fipvlan_retries, qedf_fipvlan_retries, int, S_IRUGO);
 MODULE_PARM_DESC(fipvlan_retries, " Number of FIP VLAN requests to attempt "
-	"before giving up (default 30)");
+	"before giving up (default 60)");
 
 static uint qedf_fallback_vlan = QEDF_FALLBACK_VLAN;
 module_param_named(fallback_vlan, qedf_fallback_vlan, int, S_IRUGO);
 MODULE_PARM_DESC(fallback_vlan, " VLAN ID to try if fip vlan request fails "
 	"(default 1002).");
 
-static uint qedf_default_prio = QEDF_DEFAULT_PRIO;
+static int qedf_default_prio = -1;
 module_param_named(default_prio, qedf_default_prio, int, S_IRUGO);
-MODULE_PARM_DESC(default_prio, " Default 802.1q priority for FIP and FCoE"
-	" traffic (default 3).");
+MODULE_PARM_DESC(default_prio, " Override 802.1q priority for FIP and FCoE"
+	" traffic (value between 0 and 7, default 3).");
 
 uint qedf_dump_frames;
 module_param_named(dump_frames, qedf_dump_frames, int, S_IRUGO | S_IWUSR);
@@ -89,6 +89,11 @@ module_param_named(retry_delay, qedf_retry_delay, bool, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(retry_delay, " Enable/disable handling of FCP_RSP IU retry "
 	"delay handling (default off).");
 
+static bool qedf_dcbx_no_wait;
+module_param_named(dcbx_no_wait, qedf_dcbx_no_wait, bool, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(dcbx_no_wait, " Do not wait for DCBX convergence to start "
+	"sending FIP VLAN requests on link up (Default: off).");
+
 static uint qedf_dp_module;
 module_param_named(dp_module, qedf_dp_module, uint, S_IRUGO);
 MODULE_PARM_DESC(dp_module, " bit flags control for verbose printk passed "
@@ -109,9 +114,9 @@ static struct kmem_cache *qedf_io_work_cache;
 void qedf_set_vlan_id(struct qedf_ctx *qedf, int vlan_id)
 {
 	qedf->vlan_id = vlan_id;
-	qedf->vlan_id |= qedf_default_prio << VLAN_PRIO_SHIFT;
+	qedf->vlan_id |= qedf->prio << VLAN_PRIO_SHIFT;
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Setting vlan_id=%04x "
-		   "prio=%d.\n", vlan_id, qedf_default_prio);
+		   "prio=%d.\n", vlan_id, qedf->prio);
 }
 
 /* Returns true if we have a valid vlan, false otherwise */
@@ -480,6 +485,11 @@ static void qedf_link_update(void *dev, struct qed_link_output *link)
 	struct qedf_ctx *qedf = (struct qedf_ctx *)dev;
 
 	if (link->link_up) {
+		if (atomic_read(&qedf->link_state) == QEDF_LINK_UP) {
+			QEDF_INFO((&qedf->dbg_ctx), QEDF_LOG_DISC,
+			    "Ignoring link up event as link is already up.\n");
+			return;
+		}
 		QEDF_ERR(&(qedf->dbg_ctx), "LINK UP (%d GB/s).\n",
 		    link->speed / 1000);
 
@@ -489,7 +499,8 @@ static void qedf_link_update(void *dev, struct qed_link_output *link)
 		atomic_set(&qedf->link_state, QEDF_LINK_UP);
 		qedf_update_link_speed(qedf, link);
 
-		if (atomic_read(&qedf->dcbx) == QEDF_DCBX_DONE) {
+		if (atomic_read(&qedf->dcbx) == QEDF_DCBX_DONE ||
+		    qedf_dcbx_no_wait) {
 			QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
 			     "DCBx done.\n");
 			if (atomic_read(&qedf->link_down_tmo_valid) > 0)
@@ -515,7 +526,7 @@ static void qedf_link_update(void *dev, struct qed_link_output *link)
 			    "Starting link down tmo.\n");
 			atomic_set(&qedf->link_down_tmo_valid, 1);
 		}
-		qedf->vlan_id  = 0;
+		qedf->vlan_id = 0;
 		qedf_update_link_speed(qedf, link);
 		queue_delayed_work(qedf->link_update_wq, &qedf->link_update,
 		    qedf_link_down_tmo * HZ);
@@ -526,6 +537,7 @@ static void qedf_link_update(void *dev, struct qed_link_output *link)
 static void qedf_dcbx_handler(void *dev, struct qed_dcbx_get *get, u32 mib_type)
 {
 	struct qedf_ctx *qedf = (struct qedf_ctx *)dev;
+	u8 tmp_prio;
 
 	QEDF_ERR(&(qedf->dbg_ctx), "DCBx event valid=%d enabled=%d fcoe "
 	    "prio=%d.\n", get->operational.valid, get->operational.enabled,
@@ -541,7 +553,26 @@ static void qedf_dcbx_handler(void *dev, struct qed_dcbx_get *get, u32 mib_type)
 
 		atomic_set(&qedf->dcbx, QEDF_DCBX_DONE);
 
-		if (atomic_read(&qedf->link_state) == QEDF_LINK_UP) {
+		/*
+		 * Set the 8021q priority in the following manner:
+		 *
+		 * 1. If a modparam is set use that
+		 * 2. If the value is not between 0..7 use the default
+		 * 3. Use the priority we get from the DCBX app tag
+		 */
+		tmp_prio = get->operational.app_prio.fcoe;
+		if (qedf_default_prio > -1)
+			qedf->prio = qedf_default_prio;
+		else if (tmp_prio < 0 || tmp_prio > 7) {
+			QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+			    "FIP/FCoE prio %d out of range, setting to %d.\n",
+			    tmp_prio, QEDF_DEFAULT_PRIO);
+			qedf->prio = QEDF_DEFAULT_PRIO;
+		} else
+			qedf->prio = tmp_prio;
+
+		if (atomic_read(&qedf->link_state) == QEDF_LINK_UP &&
+		    !qedf_dcbx_no_wait) {
 			if (atomic_read(&qedf->link_down_tmo_valid) > 0)
 				queue_delayed_work(qedf->link_update_wq,
 				    &qedf->link_recovery, 0);
@@ -566,6 +597,8 @@ static struct qed_fcoe_cb_ops qedf_cb_ops = {
 	{
 		.link_update = qedf_link_update,
 		.dcbx_aen = qedf_dcbx_handler,
+		.get_generic_tlv_data = qedf_get_generic_tlv_data,
+		.get_protocol_tlv_data = qedf_get_protocol_tlv_data,
 	}
 };
 
@@ -612,16 +645,6 @@ static int qedf_eh_abort(struct scsi_cmnd *sc_cmd)
 		goto out;
 	}
 
-	if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
-	    test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
-	    test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
-		QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
-			  "cleanup or abort processing or already "
-			  "completed.\n", io_req->xid);
-		rc = SUCCESS;
-		goto out;
-	}
-
 	QEDF_ERR(&(qedf->dbg_ctx), "Aborting io_req sc_cmd=%p xid=0x%x "
 		  "fp_idx=%d.\n", sc_cmd, io_req->xid, io_req->fp_idx);
 
@@ -703,7 +726,6 @@ static void qedf_ctx_soft_reset(struct fc_lport *lport)
 
 	/* For host reset, essentially do a soft link up/down */
 	atomic_set(&qedf->link_state, QEDF_LINK_DOWN);
-	atomic_set(&qedf->dcbx, QEDF_DCBX_PENDING);
 	queue_delayed_work(qedf->link_update_wq, &qedf->link_update,
 	    0);
 	qedf_wait_for_upload(qedf);
@@ -718,6 +740,22 @@ static int qedf_eh_host_reset(struct scsi_cmnd *sc_cmd)
 {
 	struct fc_lport *lport;
 	struct qedf_ctx *qedf;
+	struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
+	struct fc_rport_libfc_priv *rp = rport->dd_data;
+	struct qedf_rport *fcport = (struct qedf_rport *)&rp[1];
+	int rval;
+
+	rval = fc_remote_port_chkready(rport);
+
+	if (rval) {
+		QEDF_ERR(NULL, "device_reset rport not ready\n");
+		return FAILED;
+	}
+
+	if (fcport == NULL) {
+		QEDF_ERR(NULL, "device_reset: rport is NULL\n");
+		return FAILED;
+	}
 
 	lport = shost_priv(sc_cmd->device->host);
 	qedf = lport_priv(lport);
@@ -994,7 +1032,7 @@ static int qedf_xmit(struct fc_lport *lport, struct fc_frame *fp)
 	if (qedf_dump_frames)
 		print_hex_dump(KERN_WARNING, "fcoe: ", DUMP_PREFIX_OFFSET, 16,
 		    1, skb->data, skb->len, false);
-	qed_ops->ll2->start_xmit(qedf->cdev, skb);
+	qed_ops->ll2->start_xmit(qedf->cdev, skb, 0);
 
 	return 0;
 }
@@ -1107,7 +1145,7 @@ static int qedf_offload_connection(struct qedf_ctx *qedf,
 	conn_info.vlan_tag = qedf->vlan_id <<
 	    FCOE_CONN_OFFLOAD_RAMROD_DATA_VLAN_ID_SHIFT;
 	conn_info.vlan_tag |=
-	    qedf_default_prio << FCOE_CONN_OFFLOAD_RAMROD_DATA_PRIORITY_SHIFT;
+	    qedf->prio << FCOE_CONN_OFFLOAD_RAMROD_DATA_PRIORITY_SHIFT;
 	conn_info.flags |= (FCOE_CONN_OFFLOAD_RAMROD_DATA_B_VLAN_FLAG_MASK <<
 	    FCOE_CONN_OFFLOAD_RAMROD_DATA_B_VLAN_FLAG_SHIFT);
 
@@ -1647,6 +1685,15 @@ static int qedf_vport_destroy(struct fc_vport *vport)
 	struct Scsi_Host *shost = vport_to_shost(vport);
 	struct fc_lport *n_port = shost_priv(shost);
 	struct fc_lport *vn_port = vport->dd_data;
+	struct qedf_ctx *qedf = lport_priv(vn_port);
+
+	if (!qedf) {
+		QEDF_ERR(NULL, "qedf is NULL.\n");
+		goto out;
+	}
+
+	/* Set unloading bit on vport qedf_ctx to prevent more I/O */
+	set_bit(QEDF_UNLOADING, &qedf->flags);
 
 	mutex_lock(&n_port->lp_mutex);
 	list_del(&vn_port->list);
@@ -1673,6 +1720,7 @@ static int qedf_vport_destroy(struct fc_vport *vport)
 	if (vn_port->host)
 		scsi_host_put(vn_port->host);
 
+out:
 	return 0;
 }
 
@@ -1746,6 +1794,8 @@ static struct fc_host_statistics *qedf_fc_get_host_stats(struct Scsi_Host
 		goto out;
 	}
 
+	mutex_lock(&qedf->stats_mutex);
+
 	/* Query firmware for offload stats */
 	qed_ops->get_stats(qedf->cdev, fw_fcoe_stats);
 
@@ -1779,6 +1829,7 @@ static struct fc_host_statistics *qedf_fc_get_host_stats(struct Scsi_Host
 	qedf_stats->fcp_packet_aborts += qedf->packet_aborts;
 	qedf_stats->fcp_frame_alloc_failures += qedf->alloc_failures;
 
+	mutex_unlock(&qedf->stats_mutex);
 	kfree(fw_fcoe_stats);
 out:
 	return qedf_stats;
@@ -2104,7 +2155,8 @@ static int qedf_setup_int(struct qedf_ctx *qedf)
 	    QEDF_SIMD_HANDLER_NUM, qedf_simd_int_handler);
 	qedf->int_info.used_cnt = 1;
 
-	return 0;
+	QEDF_ERR(&qedf->dbg_ctx, "Only MSI-X supported. Failing probe.\n");
+	return -EINVAL;
 }
 
 /* Main function for libfc frame reception */
@@ -2190,6 +2242,7 @@ static void qedf_recv_frame(struct qedf_ctx *qedf,
 	if (ntoh24(&dest_mac[3]) != ntoh24(fh->fh_d_id)) {
 		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
 		    "FC frame d_id mismatch with MAC %pM.\n", dest_mac);
+		kfree_skb(skb);
 		return;
 	}
 
@@ -2948,6 +3001,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 		qedf->stop_io_on_error = false;
 		pci_set_drvdata(pdev, qedf);
 		init_completion(&qedf->fipvlan_compl);
+		mutex_init(&qedf->stats_mutex);
 
 		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_INFO,
 		   "QLogic FastLinQ FCoE Module qedf %s, "
@@ -2977,8 +3031,17 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 	qedf->link_update_wq = create_workqueue(host_buf);
 	INIT_DELAYED_WORK(&qedf->link_update, qedf_handle_link_update);
 	INIT_DELAYED_WORK(&qedf->link_recovery, qedf_link_recovery);
-
+	INIT_DELAYED_WORK(&qedf->grcdump_work, qedf_wq_grcdump);
 	qedf->fipvlan_retries = qedf_fipvlan_retries;
+	/* Set a default prio in case DCBX doesn't converge */
+	if (qedf_default_prio > -1) {
+		/*
+		 * This is the case where we pass a modparam in so we want to
+		 * honor it even if dcbx doesn't converge.
+		 */
+		qedf->prio = qedf_default_prio;
+	} else
+		qedf->prio = QEDF_DEFAULT_PRIO;
 
 	/*
 	 * Common probe. Takes care of basic hardware init and pci_*
@@ -3208,7 +3271,8 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 	 * unload process.
 	 */
 	if (mode != QEDF_MODE_RECOVERY) {
-		qedf->grcdump_size = qed_ops->common->dbg_grc_size(qedf->cdev);
+		qedf->grcdump_size =
+		    qed_ops->common->dbg_all_data_size(qedf->cdev);
 		if (qedf->grcdump_size) {
 			rc = qedf_alloc_grc_dump_buf(&qedf->grcdump,
 			    qedf->grcdump_size);
@@ -3392,6 +3456,113 @@ static void qedf_remove(struct pci_dev *pdev)
 	__qedf_remove(pdev, QEDF_MODE_NORMAL);
 }
 
+void qedf_wq_grcdump(struct work_struct *work)
+{
+	struct qedf_ctx *qedf =
+	    container_of(work, struct qedf_ctx, grcdump_work.work);
+
+	QEDF_ERR(&(qedf->dbg_ctx), "Collecting GRC dump.\n");
+	qedf_capture_grc_dump(qedf);
+}
+
+/*
+ * Protocol TLV handler
+ */
+void qedf_get_protocol_tlv_data(void *dev, void *data)
+{
+	struct qedf_ctx *qedf = dev;
+	struct qed_mfw_tlv_fcoe *fcoe = data;
+	struct fc_lport *lport = qedf->lport;
+	struct Scsi_Host *host = lport->host;
+	struct fc_host_attrs *fc_host = shost_to_fc_host(host);
+	struct fc_host_statistics *hst;
+
+	/* Force a refresh of the fc_host stats including offload stats */
+	hst = qedf_fc_get_host_stats(host);
+
+	fcoe->qos_pri_set = true;
+	fcoe->qos_pri = 3; /* Hard coded to 3 in driver */
+
+	fcoe->ra_tov_set = true;
+	fcoe->ra_tov = lport->r_a_tov;
+
+	fcoe->ed_tov_set = true;
+	fcoe->ed_tov = lport->e_d_tov;
+
+	fcoe->npiv_state_set = true;
+	fcoe->npiv_state = 1; /* NPIV always enabled */
+
+	fcoe->num_npiv_ids_set = true;
+	fcoe->num_npiv_ids = fc_host->npiv_vports_inuse;
+
+	/* Certain attributes we only want to set if we've selected an FCF */
+	if (qedf->ctlr.sel_fcf) {
+		fcoe->switch_name_set = true;
+		u64_to_wwn(qedf->ctlr.sel_fcf->switch_name, fcoe->switch_name);
+	}
+
+	fcoe->port_state_set = true;
+	/* For qedf we're either link down or fabric attach */
+	if (lport->link_up)
+		fcoe->port_state = QED_MFW_TLV_PORT_STATE_FABRIC;
+	else
+		fcoe->port_state = QED_MFW_TLV_PORT_STATE_OFFLINE;
+
+	fcoe->link_failures_set = true;
+	fcoe->link_failures = (u16)hst->link_failure_count;
+
+	fcoe->fcoe_txq_depth_set = true;
+	fcoe->fcoe_rxq_depth_set = true;
+	fcoe->fcoe_rxq_depth = FCOE_PARAMS_NUM_TASKS;
+	fcoe->fcoe_txq_depth = FCOE_PARAMS_NUM_TASKS;
+
+	fcoe->fcoe_rx_frames_set = true;
+	fcoe->fcoe_rx_frames = hst->rx_frames;
+
+	fcoe->fcoe_tx_frames_set = true;
+	fcoe->fcoe_tx_frames = hst->tx_frames;
+
+	fcoe->fcoe_rx_bytes_set = true;
+	fcoe->fcoe_rx_bytes = hst->fcp_input_megabytes * 1000000;
+
+	fcoe->fcoe_tx_bytes_set = true;
+	fcoe->fcoe_tx_bytes = hst->fcp_output_megabytes * 1000000;
+
+	fcoe->crc_count_set = true;
+	fcoe->crc_count = hst->invalid_crc_count;
+
+	fcoe->tx_abts_set = true;
+	fcoe->tx_abts = hst->fcp_packet_aborts;
+
+	fcoe->tx_lun_rst_set = true;
+	fcoe->tx_lun_rst = qedf->lun_resets;
+
+	fcoe->abort_task_sets_set = true;
+	fcoe->abort_task_sets = qedf->packet_aborts;
+
+	fcoe->scsi_busy_set = true;
+	fcoe->scsi_busy = qedf->busy;
+
+	fcoe->scsi_tsk_full_set = true;
+	fcoe->scsi_tsk_full = qedf->task_set_fulls;
+}
+
+/* Generic TLV data callback */
+void qedf_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data)
+{
+	struct qedf_ctx *qedf;
+
+	if (!dev) {
+		QEDF_INFO(NULL, QEDF_LOG_EVT,
+			  "dev is NULL so ignoring get_generic_tlv_data request.\n");
+		return;
+	}
+	qedf = (struct qedf_ctx *)dev;
+
+	memset(data, 0, sizeof(struct qed_generic_tlvs));
+	ether_addr_copy(data->mac[0], qedf->mac);
+}
+
 /*
  * Module Init/Remove
  */
@@ -3404,6 +3575,17 @@ static int __init qedf_init(void)
 	if (qedf_debug == QEDF_LOG_DEFAULT)
 		qedf_debug = QEDF_DEFAULT_LOG_MASK;
 
+	/*
+	 * Check that default prio for FIP/FCoE traffic is between 0..7 if a
+	 * value has been set
+	 */
+	if (qedf_default_prio > -1)
+		if (qedf_default_prio > 7) {
+			qedf_default_prio = QEDF_DEFAULT_PRIO;
+			QEDF_ERR(NULL, "FCoE/FIP priority out of range, resetting to %d.\n",
+			    QEDF_DEFAULT_PRIO);
+		}
+
 	/* Print driver banner */
 	QEDF_INFO(NULL, QEDF_LOG_INFO, "%s v%s.\n", QEDF_DESCR,
 		   QEDF_VERSION);
diff --git a/drivers/scsi/qedf/qedf_version.h b/drivers/scsi/qedf/qedf_version.h
index c2478056356a..9455faacd5de 100644
--- a/drivers/scsi/qedf/qedf_version.h
+++ b/drivers/scsi/qedf/qedf_version.h
@@ -1,15 +1,15 @@
 /*
  *  QLogic FCoE Offload Driver
- *  Copyright (c) 2016-2017 Cavium Inc.
+ *  Copyright (c) 2016-2018 Cavium Inc.
  *
  *  This software is available under the terms of the GNU General Public License
  *  (GPL) Version 2, available from the file COPYING in the main directory of
  *  this source tree.
  */
 
-#define QEDF_VERSION		"8.33.0.20"
+#define QEDF_VERSION		"8.33.16.20"
 #define QEDF_DRIVER_MAJOR_VER		8
 #define QEDF_DRIVER_MINOR_VER		33
-#define QEDF_DRIVER_REV_VER		0
+#define QEDF_DRIVER_REV_VER		16
 #define QEDF_DRIVER_ENG_VER		20
 
diff --git a/drivers/scsi/qedi/qedi.h b/drivers/scsi/qedi/qedi.h
index b8b22ce60ecc..fc3babc15fa3 100644
--- a/drivers/scsi/qedi/qedi.h
+++ b/drivers/scsi/qedi/qedi.h
@@ -353,6 +353,9 @@ struct qedi_ctx {
 #define IPV6_LEN	41
 #define IPV4_LEN	17
 	struct iscsi_boot_kset *boot_kset;
+
+	/* Used for iscsi statistics */
+	struct mutex stats_lock;
 };
 
 struct qedi_work {
diff --git a/drivers/scsi/qedi/qedi_iscsi.c b/drivers/scsi/qedi/qedi_iscsi.c
index 7ec7f6e00fb8..ff21aa1fd939 100644
--- a/drivers/scsi/qedi/qedi_iscsi.c
+++ b/drivers/scsi/qedi/qedi_iscsi.c
@@ -1150,7 +1150,7 @@ static int qedi_data_avail(struct qedi_ctx *qedi, u16 vlanid)
 	if (vlanid)
 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid);
 
-	rc = qedi_ops->ll2->start_xmit(cdev, skb);
+	rc = qedi_ops->ll2->start_xmit(cdev, skb, 0);
 	if (rc) {
 		QEDI_ERR(&qedi->dbg_ctx, "ll2 start_xmit returned %d\n",
 			 rc);
diff --git a/drivers/scsi/qedi/qedi_iscsi.h b/drivers/scsi/qedi/qedi_iscsi.h
index ea1315189922..11260776212f 100644
--- a/drivers/scsi/qedi/qedi_iscsi.h
+++ b/drivers/scsi/qedi/qedi_iscsi.h
@@ -223,6 +223,12 @@ struct qedi_work_map {
 	struct work_struct *ptr_tmf_work;
 };
 
+struct qedi_boot_target {
+	char ip_addr[64];
+	char iscsi_name[255];
+	u32 ipv6_en;
+};
+
 #define qedi_set_itt(task_id, itt) ((u32)(((task_id) & 0xffff) | ((itt) << 16)))
 #define qedi_get_itt(cqe) (cqe.iscsi_hdr.cmd.itt >> 16)
 
diff --git a/drivers/scsi/qedi/qedi_main.c b/drivers/scsi/qedi/qedi_main.c
index 4da3592aec0f..32ee7f62fef9 100644
--- a/drivers/scsi/qedi/qedi_main.c
+++ b/drivers/scsi/qedi/qedi_main.c
@@ -55,6 +55,7 @@ static void qedi_free_global_queues(struct qedi_ctx *qedi);
 static struct qedi_cmd *qedi_get_cmd_from_tid(struct qedi_ctx *qedi, u32 tid);
 static void qedi_reset_uio_rings(struct qedi_uio_dev *udev);
 static void qedi_ll2_free_skbs(struct qedi_ctx *qedi);
+static struct nvm_iscsi_block *qedi_get_nvram_block(struct qedi_ctx *qedi);
 
 static int qedi_iscsi_event_cb(void *context, u8 fw_event_code, void *fw_handle)
 {
@@ -879,6 +880,201 @@ static void qedi_free_iscsi_pf_param(struct qedi_ctx *qedi)
 	kfree(qedi->global_queues);
 }
 
+static void qedi_get_boot_tgt_info(struct nvm_iscsi_block *block,
+				   struct qedi_boot_target *tgt, u8 index)
+{
+	u32 ipv6_en;
+
+	ipv6_en = !!(block->generic.ctrl_flags &
+		     NVM_ISCSI_CFG_GEN_IPV6_ENABLED);
+
+	snprintf(tgt->iscsi_name, NVM_ISCSI_CFG_ISCSI_NAME_MAX_LEN, "%s\n",
+		 block->target[index].target_name.byte);
+
+	tgt->ipv6_en = ipv6_en;
+
+	if (ipv6_en)
+		snprintf(tgt->ip_addr, IPV6_LEN, "%pI6\n",
+			 block->target[index].ipv6_addr.byte);
+	else
+		snprintf(tgt->ip_addr, IPV4_LEN, "%pI4\n",
+			 block->target[index].ipv4_addr.byte);
+}
+
+static int qedi_find_boot_info(struct qedi_ctx *qedi,
+			       struct qed_mfw_tlv_iscsi *iscsi,
+			       struct nvm_iscsi_block *block)
+{
+	struct qedi_boot_target *pri_tgt = NULL, *sec_tgt = NULL;
+	u32 pri_ctrl_flags = 0, sec_ctrl_flags = 0, found = 0;
+	struct iscsi_cls_session *cls_sess;
+	struct iscsi_cls_conn *cls_conn;
+	struct qedi_conn *qedi_conn;
+	struct iscsi_session *sess;
+	struct iscsi_conn *conn;
+	char ep_ip_addr[64];
+	int i, ret = 0;
+
+	pri_ctrl_flags = !!(block->target[0].ctrl_flags &
+					NVM_ISCSI_CFG_TARGET_ENABLED);
+	if (pri_ctrl_flags) {
+		pri_tgt = kzalloc(sizeof(*pri_tgt), GFP_KERNEL);
+		if (!pri_tgt)
+			return -1;
+		qedi_get_boot_tgt_info(block, pri_tgt, 0);
+	}
+
+	sec_ctrl_flags = !!(block->target[1].ctrl_flags &
+					NVM_ISCSI_CFG_TARGET_ENABLED);
+	if (sec_ctrl_flags) {
+		sec_tgt = kzalloc(sizeof(*sec_tgt), GFP_KERNEL);
+		if (!sec_tgt) {
+			ret = -1;
+			goto free_tgt;
+		}
+		qedi_get_boot_tgt_info(block, sec_tgt, 1);
+	}
+
+	for (i = 0; i < qedi->max_active_conns; i++) {
+		qedi_conn = qedi_get_conn_from_id(qedi, i);
+		if (!qedi_conn)
+			continue;
+
+		if (qedi_conn->ep->ip_type == TCP_IPV4)
+			snprintf(ep_ip_addr, IPV4_LEN, "%pI4\n",
+				 qedi_conn->ep->dst_addr);
+		else
+			snprintf(ep_ip_addr, IPV6_LEN, "%pI6\n",
+				 qedi_conn->ep->dst_addr);
+
+		cls_conn = qedi_conn->cls_conn;
+		conn = cls_conn->dd_data;
+		cls_sess = iscsi_conn_to_session(cls_conn);
+		sess = cls_sess->dd_data;
+
+		if (pri_ctrl_flags) {
+			if (!strcmp(pri_tgt->iscsi_name, sess->targetname) &&
+			    !strcmp(pri_tgt->ip_addr, ep_ip_addr)) {
+				found = 1;
+				break;
+			}
+		}
+
+		if (sec_ctrl_flags) {
+			if (!strcmp(sec_tgt->iscsi_name, sess->targetname) &&
+			    !strcmp(sec_tgt->ip_addr, ep_ip_addr)) {
+				found = 1;
+				break;
+			}
+		}
+	}
+
+	if (found) {
+		if (conn->hdrdgst_en) {
+			iscsi->header_digest_set = true;
+			iscsi->header_digest = 1;
+		}
+
+		if (conn->datadgst_en) {
+			iscsi->data_digest_set = true;
+			iscsi->data_digest = 1;
+		}
+		iscsi->boot_taget_portal_set = true;
+		iscsi->boot_taget_portal = sess->tpgt;
+
+	} else {
+		ret = -1;
+	}
+
+	if (sec_ctrl_flags)
+		kfree(sec_tgt);
+free_tgt:
+	if (pri_ctrl_flags)
+		kfree(pri_tgt);
+
+	return ret;
+}
+
+static void qedi_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data)
+{
+	struct qedi_ctx *qedi;
+
+	if (!dev) {
+		QEDI_INFO(NULL, QEDI_LOG_EVT,
+			  "dev is NULL so ignoring get_generic_tlv_data request.\n");
+		return;
+	}
+	qedi = (struct qedi_ctx *)dev;
+
+	memset(data, 0, sizeof(struct qed_generic_tlvs));
+	ether_addr_copy(data->mac[0], qedi->mac);
+}
+
+/*
+ * Protocol TLV handler
+ */
+static void qedi_get_protocol_tlv_data(void *dev, void *data)
+{
+	struct qed_mfw_tlv_iscsi *iscsi = data;
+	struct qed_iscsi_stats *fw_iscsi_stats;
+	struct nvm_iscsi_block *block = NULL;
+	u32 chap_en = 0, mchap_en = 0;
+	struct qedi_ctx *qedi = dev;
+	int rval = 0;
+
+	fw_iscsi_stats = kmalloc(sizeof(*fw_iscsi_stats), GFP_KERNEL);
+	if (!fw_iscsi_stats) {
+		QEDI_ERR(&qedi->dbg_ctx,
+			 "Could not allocate memory for fw_iscsi_stats.\n");
+		goto exit_get_data;
+	}
+
+	mutex_lock(&qedi->stats_lock);
+	/* Query firmware for offload stats */
+	qedi_ops->get_stats(qedi->cdev, fw_iscsi_stats);
+	mutex_unlock(&qedi->stats_lock);
+
+	iscsi->rx_frames_set = true;
+	iscsi->rx_frames = fw_iscsi_stats->iscsi_rx_packet_cnt;
+	iscsi->rx_bytes_set = true;
+	iscsi->rx_bytes = fw_iscsi_stats->iscsi_rx_bytes_cnt;
+	iscsi->tx_frames_set = true;
+	iscsi->tx_frames = fw_iscsi_stats->iscsi_tx_packet_cnt;
+	iscsi->tx_bytes_set = true;
+	iscsi->tx_bytes = fw_iscsi_stats->iscsi_tx_bytes_cnt;
+	iscsi->frame_size_set = true;
+	iscsi->frame_size = qedi->ll2_mtu;
+	block = qedi_get_nvram_block(qedi);
+	if (block) {
+		chap_en = !!(block->generic.ctrl_flags &
+			     NVM_ISCSI_CFG_GEN_CHAP_ENABLED);
+		mchap_en = !!(block->generic.ctrl_flags &
+			      NVM_ISCSI_CFG_GEN_CHAP_MUTUAL_ENABLED);
+
+		iscsi->auth_method_set = (chap_en || mchap_en) ? true : false;
+		iscsi->auth_method = 1;
+		if (chap_en)
+			iscsi->auth_method = 2;
+		if (mchap_en)
+			iscsi->auth_method = 3;
+
+		iscsi->tx_desc_size_set = true;
+		iscsi->tx_desc_size = QEDI_SQ_SIZE;
+		iscsi->rx_desc_size_set = true;
+		iscsi->rx_desc_size = QEDI_CQ_SIZE;
+
+		/* tpgt, hdr digest, data digest */
+		rval = qedi_find_boot_info(qedi, iscsi, block);
+		if (rval)
+			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
+				  "Boot target not set");
+	}
+
+	kfree(fw_iscsi_stats);
+exit_get_data:
+	return;
+}
+
 static void qedi_link_update(void *dev, struct qed_link_output *link)
 {
 	struct qedi_ctx *qedi = (struct qedi_ctx *)dev;
@@ -896,6 +1092,8 @@ static void qedi_link_update(void *dev, struct qed_link_output *link)
 static struct qed_iscsi_cb_ops qedi_cb_ops = {
 	{
 		.link_update =		qedi_link_update,
+		.get_protocol_tlv_data = qedi_get_protocol_tlv_data,
+		.get_generic_tlv_data = qedi_get_generic_tlv_data,
 	}
 };
 
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index eb2ec1fb07cb..9442e18aef6f 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -2279,8 +2279,6 @@ enum discovery_state {
 	DSC_LOGIN_PEND,
 	DSC_LOGIN_FAILED,
 	DSC_GPDB,
-	DSC_GFPN_ID,
-	DSC_GPSC,
 	DSC_UPD_FCPORT,
 	DSC_LOGIN_COMPLETE,
 	DSC_ADISC,
@@ -2346,6 +2344,7 @@ typedef struct fc_port {
 	unsigned int login_succ:1;
 	unsigned int query:1;
 	unsigned int id_changed:1;
+	unsigned int rscn_rcvd:1;
 
 	struct work_struct nvme_del_work;
 	struct completion nvme_del_done;
@@ -3226,6 +3225,7 @@ enum qla_work_type {
 	QLA_EVT_GNNID,
 	QLA_EVT_GFPNID,
 	QLA_EVT_SP_RETRY,
+	QLA_EVT_IIDMA,
 };
 
 
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 3c4c84ed0f0f..f68eb6096559 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -116,7 +116,8 @@ extern int qla2x00_post_async_prlo_work(struct scsi_qla_host *, fc_port_t *,
     uint16_t *);
 extern int qla2x00_post_async_prlo_done_work(struct scsi_qla_host *,
     fc_port_t *, uint16_t *);
-
+int qla_post_iidma_work(struct scsi_qla_host *vha, fc_port_t *fcport);
+void qla_do_iidma_work(struct scsi_qla_host *vha, fc_port_t *fcport);
 /*
  * Global Data in qla_os.c source file.
  */
diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c
index 9e914f9c3ffb..4bc2b66b299f 100644
--- a/drivers/scsi/qla2xxx/qla_gs.c
+++ b/drivers/scsi/qla2xxx/qla_gs.c
@@ -3175,7 +3175,6 @@ int qla24xx_async_gidpn(scsi_qla_host_t *vha, fc_port_t *fcport)
 
 done_free_sp:
 	sp->free(sp);
-	fcport->flags &= ~FCF_ASYNC_SENT;
 done:
 	fcport->flags &= ~FCF_ASYNC_ACTIVE;
 	return rval;
@@ -3239,7 +3238,7 @@ void qla24xx_handle_gpsc_event(scsi_qla_host_t *vha, struct event_arg *ea)
 		return;
 	}
 
-	qla24xx_post_upd_fcport_work(vha, ea->fcport);
+	qla_post_iidma_work(vha, fcport);
 }
 
 static void qla24xx_async_gpsc_sp_done(void *s, int res)
@@ -3257,8 +3256,6 @@ static void qla24xx_async_gpsc_sp_done(void *s, int res)
 	    "Async done-%s res %x, WWPN %8phC \n",
 	    sp->name, res, fcport->port_name);
 
-	fcport->flags &= ~(FCF_ASYNC_SENT | FCF_ASYNC_ACTIVE);
-
 	if (res == (DID_ERROR << 16)) {
 		/* entry status error */
 		goto done;
@@ -3327,7 +3324,6 @@ int qla24xx_async_gpsc(scsi_qla_host_t *vha, fc_port_t *fcport)
 	if (!sp)
 		goto done;
 
-	fcport->flags |= FCF_ASYNC_SENT;
 	sp->type = SRB_CT_PTHRU_CMD;
 	sp->name = "gpsc";
 	sp->gen1 = fcport->rscn_gen;
@@ -3862,6 +3858,7 @@ void qla24xx_async_gnnft_done(scsi_qla_host_t *vha, srb_t *sp)
 	bool found;
 	struct fab_scan_rp *rp;
 	unsigned long flags;
+	u8 recheck = 0;
 
 	ql_dbg(ql_dbg_disc, vha, 0xffff,
 	    "%s enter\n", __func__);
@@ -3914,8 +3911,8 @@ void qla24xx_async_gnnft_done(scsi_qla_host_t *vha, srb_t *sp)
 		list_for_each_entry(fcport, &vha->vp_fcports, list) {
 			if (memcmp(rp->port_name, fcport->port_name, WWN_SIZE))
 				continue;
+			fcport->rscn_rcvd = 0;
 			fcport->scan_state = QLA_FCPORT_FOUND;
-			fcport->d_id.b24 = rp->id.b24;
 			found = true;
 			/*
 			 * If device was not a fabric device before.
@@ -3923,7 +3920,10 @@ void qla24xx_async_gnnft_done(scsi_qla_host_t *vha, srb_t *sp)
 			if ((fcport->flags & FCF_FABRIC_DEVICE) == 0) {
 				qla2x00_clear_loop_id(fcport);
 				fcport->flags |= FCF_FABRIC_DEVICE;
+			} else if (fcport->d_id.b24 != rp->id.b24) {
+				qlt_schedule_sess_for_deletion(fcport);
 			}
+			fcport->d_id.b24 = rp->id.b24;
 			break;
 		}
 
@@ -3940,10 +3940,13 @@ void qla24xx_async_gnnft_done(scsi_qla_host_t *vha, srb_t *sp)
 	 * Logout all previous fabric dev marked lost, except FCP2 devices.
 	 */
 	list_for_each_entry(fcport, &vha->vp_fcports, list) {
-		if ((fcport->flags & FCF_FABRIC_DEVICE) == 0)
+		if ((fcport->flags & FCF_FABRIC_DEVICE) == 0) {
+			fcport->rscn_rcvd = 0;
 			continue;
+		}
 
 		if (fcport->scan_state != QLA_FCPORT_FOUND) {
+			fcport->rscn_rcvd = 0;
 			if ((qla_dual_mode_enabled(vha) ||
 				qla_ini_mode_enabled(vha)) &&
 			    atomic_read(&fcport->state) == FCS_ONLINE) {
@@ -3961,15 +3964,31 @@ void qla24xx_async_gnnft_done(scsi_qla_host_t *vha, srb_t *sp)
 					continue;
 				}
 			}
-		} else
-			qla24xx_fcport_handle_login(vha, fcport);
+		} else {
+			if (fcport->rscn_rcvd ||
+			    fcport->disc_state != DSC_LOGIN_COMPLETE) {
+				fcport->rscn_rcvd = 0;
+				qla24xx_fcport_handle_login(vha, fcport);
+			}
+		}
 	}
 
+	recheck = 1;
 out:
 	qla24xx_sp_unmap(vha, sp);
 	spin_lock_irqsave(&vha->work_lock, flags);
 	vha->scan.scan_flags &= ~SF_SCANNING;
 	spin_unlock_irqrestore(&vha->work_lock, flags);
+
+	if (recheck) {
+		list_for_each_entry(fcport, &vha->vp_fcports, list) {
+			if (fcport->rscn_rcvd) {
+				set_bit(LOCAL_LOOP_UPDATE, &vha->dpc_flags);
+				set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
+				break;
+			}
+		}
+	}
 }
 
 static void qla2x00_find_free_fcp_nvme_slot(struct scsi_qla_host *vha,
@@ -4532,7 +4551,6 @@ int qla24xx_async_gnnid(scsi_qla_host_t *vha, fc_port_t *fcport)
 
 done_free_sp:
 	sp->free(sp);
-	fcport->flags &= ~FCF_ASYNC_SENT;
 done:
 	return rval;
 }
@@ -4594,7 +4612,6 @@ static void qla2x00_async_gfpnid_sp_done(void *s, int res)
 	struct event_arg ea;
 	u64 wwn;
 
-	fcport->flags &= ~FCF_ASYNC_SENT;
 	wwn = wwn_to_u64(fpn);
 	if (wwn)
 		memcpy(fcport->fabric_port_name, fpn, WWN_SIZE);
@@ -4623,12 +4640,10 @@ int qla24xx_async_gfpnid(scsi_qla_host_t *vha, fc_port_t *fcport)
 	if (!vha->flags.online || (fcport->flags & FCF_ASYNC_SENT))
 		return rval;
 
-	fcport->disc_state = DSC_GFPN_ID;
 	sp = qla2x00_get_sp(vha, fcport, GFP_ATOMIC);
 	if (!sp)
 		goto done;
 
-	fcport->flags |= FCF_ASYNC_SENT;
 	sp->type = SRB_CT_PTHRU_CMD;
 	sp->name = "gfpnid";
 	sp->gen1 = fcport->rscn_gen;
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index 8f55dd44adae..1aa3720ea2ed 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -1021,30 +1021,11 @@ void __qla24xx_handle_gpdb_event(scsi_qla_host_t *vha, struct event_arg *ea)
 		vha->fcport_count++;
 		ea->fcport->login_succ = 1;
 
-		if (!IS_IIDMA_CAPABLE(vha->hw) ||
-		    !vha->hw->flags.gpsc_supported) {
-			ql_dbg(ql_dbg_disc, vha, 0x20d6,
-			    "%s %d %8phC post upd_fcport fcp_cnt %d\n",
-			    __func__, __LINE__,  ea->fcport->port_name,
-			    vha->fcport_count);
-
-			qla24xx_post_upd_fcport_work(vha, ea->fcport);
-		} else {
-			if (ea->fcport->id_changed) {
-				ea->fcport->id_changed = 0;
-				ql_dbg(ql_dbg_disc, vha, 0x20d7,
-				    "%s %d %8phC post gfpnid fcp_cnt %d\n",
-				    __func__, __LINE__, ea->fcport->port_name,
-				    vha->fcport_count);
-				qla24xx_post_gfpnid_work(vha, ea->fcport);
-			} else {
-				ql_dbg(ql_dbg_disc, vha, 0x20d7,
-				    "%s %d %8phC post gpsc fcp_cnt %d\n",
-				    __func__, __LINE__, ea->fcport->port_name,
-				    vha->fcport_count);
-				qla24xx_post_gpsc_work(vha, ea->fcport);
-			}
-		}
+		ql_dbg(ql_dbg_disc, vha, 0x20d6,
+		    "%s %d %8phC post upd_fcport fcp_cnt %d\n",
+		    __func__, __LINE__,  ea->fcport->port_name,
+		    vha->fcport_count);
+		qla24xx_post_upd_fcport_work(vha, ea->fcport);
 	} else if (ea->fcport->login_succ) {
 		/*
 		 * We have an existing session. A late RSCN delivery
@@ -1167,9 +1148,6 @@ int qla24xx_fcport_handle_login(struct scsi_qla_host *vha, fc_port_t *fcport)
 	    fcport->login_gen, fcport->login_retry,
 	    fcport->loop_id, fcport->scan_state);
 
-	if (fcport->login_retry == 0)
-		return 0;
-
 	if (fcport->scan_state != QLA_FCPORT_FOUND)
 		return 0;
 
@@ -1194,7 +1172,8 @@ int qla24xx_fcport_handle_login(struct scsi_qla_host *vha, fc_port_t *fcport)
 		return 0;
 	}
 
-	fcport->login_retry--;
+	if (fcport->login_retry > 0)
+		fcport->login_retry--;
 
 	switch (fcport->disc_state) {
 	case DSC_DELETED:
@@ -1350,20 +1329,7 @@ void qla2x00_fcport_event_handler(scsi_qla_host_t *vha, struct event_arg *ea)
 	fc_port_t *f, *tf;
 	uint32_t id = 0, mask, rid;
 	unsigned long flags;
-
-	switch (ea->event) {
-	case FCME_RSCN:
-	case FCME_GIDPN_DONE:
-	case FCME_GPSC_DONE:
-	case FCME_GPNID_DONE:
-	case FCME_GNNID_DONE:
-		if (test_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags) ||
-		    test_bit(LOOP_RESYNC_ACTIVE, &vha->dpc_flags))
-			return;
-		break;
-	default:
-		break;
-	}
+	fc_port_t *fcport;
 
 	switch (ea->event) {
 	case FCME_RELOGIN:
@@ -1377,6 +1343,11 @@ void qla2x00_fcport_event_handler(scsi_qla_host_t *vha, struct event_arg *ea)
 			return;
 		switch (ea->id.b.rsvd_1) {
 		case RSCN_PORT_ADDR:
+			fcport = qla2x00_find_fcport_by_nportid
+				(vha, &ea->id, 1);
+			if (fcport)
+				fcport->rscn_rcvd = 1;
+
 			spin_lock_irqsave(&vha->work_lock, flags);
 			if (vha->scan.scan_flags == 0) {
 				ql_dbg(ql_dbg_disc, vha, 0xffff,
@@ -4532,7 +4503,6 @@ qla2x00_alloc_fcport(scsi_qla_host_t *vha, gfp_t flags)
 	fcport->fw_login_state = DSC_LS_PORT_UNAVAIL;
 	fcport->deleted = QLA_SESS_DELETED;
 	fcport->login_retry = vha->hw->login_retry_count;
-	fcport->login_retry = 5;
 	fcport->logout_on_delete = 1;
 
 	if (!fcport->ct_desc.ct_sns) {
@@ -5054,6 +5024,24 @@ qla2x00_iidma_fcport(scsi_qla_host_t *vha, fc_port_t *fcport)
 	}
 }
 
+void qla_do_iidma_work(struct scsi_qla_host *vha, fc_port_t *fcport)
+{
+	qla2x00_iidma_fcport(vha, fcport);
+	qla24xx_update_fcport_fcp_prio(vha, fcport);
+}
+
+int qla_post_iidma_work(struct scsi_qla_host *vha, fc_port_t *fcport)
+{
+	struct qla_work_evt *e;
+
+	e = qla2x00_alloc_work(vha, QLA_EVT_IIDMA);
+	if (!e)
+		return QLA_FUNCTION_FAILED;
+
+	e->u.fcport.fcport = fcport;
+	return qla2x00_post_work(vha, e);
+}
+
 /* qla2x00_reg_remote_port is reserved for Initiator Mode only.*/
 static void
 qla2x00_reg_remote_port(scsi_qla_host_t *vha, fc_port_t *fcport)
@@ -5122,13 +5110,14 @@ qla2x00_update_fcport(scsi_qla_host_t *vha, fc_port_t *fcport)
 
 	if (IS_QLAFX00(vha->hw)) {
 		qla2x00_set_fcport_state(fcport, FCS_ONLINE);
-		goto reg_port;
+	} else {
+		fcport->login_retry = 0;
+		fcport->flags &= ~(FCF_LOGIN_NEEDED | FCF_ASYNC_SENT);
+		fcport->disc_state = DSC_LOGIN_COMPLETE;
+		fcport->deleted = 0;
+		fcport->logout_on_delete = 1;
+		qla2x00_set_fcport_state(fcport, FCS_ONLINE);
 	}
-	fcport->login_retry = 0;
-	fcport->flags &= ~(FCF_LOGIN_NEEDED | FCF_ASYNC_SENT);
-	fcport->disc_state = DSC_LOGIN_COMPLETE;
-	fcport->deleted = 0;
-	fcport->logout_on_delete = 1;
 
 	qla2x00_set_fcport_state(fcport, FCS_ONLINE);
 	qla2x00_iidma_fcport(vha, fcport);
@@ -5140,7 +5129,6 @@ qla2x00_update_fcport(scsi_qla_host_t *vha, fc_port_t *fcport)
 
 	qla24xx_update_fcport_fcp_prio(vha, fcport);
 
-reg_port:
 	switch (vha->host->active_mode) {
 	case MODE_INITIATOR:
 		qla2x00_reg_remote_port(vha, fcport);
@@ -5159,6 +5147,23 @@ qla2x00_update_fcport(scsi_qla_host_t *vha, fc_port_t *fcport)
 	default:
 		break;
 	}
+
+	if (IS_IIDMA_CAPABLE(vha->hw) && vha->hw->flags.gpsc_supported) {
+		if (fcport->id_changed) {
+			fcport->id_changed = 0;
+			ql_dbg(ql_dbg_disc, vha, 0x20d7,
+			    "%s %d %8phC post gfpnid fcp_cnt %d\n",
+			    __func__, __LINE__, fcport->port_name,
+			    vha->fcport_count);
+			qla24xx_post_gfpnid_work(vha, fcport);
+		} else {
+			ql_dbg(ql_dbg_disc, vha, 0x20d7,
+			    "%s %d %8phC post gpsc fcp_cnt %d\n",
+			    __func__, __LINE__, fcport->port_name,
+			    vha->fcport_count);
+			qla24xx_post_gpsc_work(vha, fcport);
+		}
+	}
 }
 
 /*
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 15eaa6dded04..817c18a8e84d 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -5063,6 +5063,10 @@ qla2x00_do_work(struct scsi_qla_host *vha)
 			break;
 		case QLA_EVT_SP_RETRY:
 			qla_sp_retry(vha, e);
+			break;
+		case QLA_EVT_IIDMA:
+			qla_do_iidma_work(vha, e->u.fcport.fcport);
+			break;
 		}
 		if (e->flags & QLA_EVT_FLAG_FREE)
 			kfree(e);
diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c
index 025dc2d3f3de..b85c833099ff 100644
--- a/drivers/scsi/qla2xxx/qla_target.c
+++ b/drivers/scsi/qla2xxx/qla_target.c
@@ -601,24 +601,18 @@ void qla2x00_async_nack_sp_done(void *s, int res)
 
 			vha->fcport_count++;
 
-			if (!IS_IIDMA_CAPABLE(vha->hw) ||
-			    !vha->hw->flags.gpsc_supported) {
-				ql_dbg(ql_dbg_disc, vha, 0x20f3,
-				    "%s %d %8phC post upd_fcport fcp_cnt %d\n",
-				    __func__, __LINE__,
-				    sp->fcport->port_name,
-				    vha->fcport_count);
-				sp->fcport->disc_state = DSC_UPD_FCPORT;
-				qla24xx_post_upd_fcport_work(vha, sp->fcport);
-			} else {
-				ql_dbg(ql_dbg_disc, vha, 0x20f5,
-				    "%s %d %8phC post gpsc fcp_cnt %d\n",
-				    __func__, __LINE__,
-				    sp->fcport->port_name,
-				    vha->fcport_count);
-
-				qla24xx_post_gpsc_work(vha, sp->fcport);
-			}
+			ql_dbg(ql_dbg_disc, vha, 0x20f3,
+			    "%s %d %8phC post upd_fcport fcp_cnt %d\n",
+			    __func__, __LINE__,
+			    sp->fcport->port_name,
+			    vha->fcport_count);
+			sp->fcport->disc_state = DSC_UPD_FCPORT;
+			qla24xx_post_upd_fcport_work(vha, sp->fcport);
+		} else {
+			sp->fcport->login_retry = 0;
+			sp->fcport->disc_state = DSC_LOGIN_COMPLETE;
+			sp->fcport->deleted = 0;
+			sp->fcport->logout_on_delete = 1;
 		}
 		break;
 
@@ -1930,13 +1924,84 @@ static void abort_cmds_for_lun(struct scsi_qla_host *vha,
 	spin_unlock_irqrestore(&vha->cmd_list_lock, flags);
 }
 
+static struct qla_qpair_hint *qlt_find_qphint(struct scsi_qla_host *vha,
+    uint64_t unpacked_lun)
+{
+	struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
+	struct qla_qpair_hint *h = NULL;
+
+	if (vha->flags.qpairs_available) {
+		h = btree_lookup64(&tgt->lun_qpair_map, unpacked_lun);
+		if (!h)
+			h = &tgt->qphints[0];
+	} else {
+		h = &tgt->qphints[0];
+	}
+
+	return h;
+}
+
+static void qlt_do_tmr_work(struct work_struct *work)
+{
+	struct qla_tgt_mgmt_cmd *mcmd =
+		container_of(work, struct qla_tgt_mgmt_cmd, work);
+	struct qla_hw_data *ha = mcmd->vha->hw;
+	int rc = EIO;
+	uint32_t tag;
+	unsigned long flags;
+
+	switch (mcmd->tmr_func) {
+	case QLA_TGT_ABTS:
+		tag = mcmd->orig_iocb.abts.exchange_addr_to_abort;
+		break;
+	default:
+		tag = 0;
+		break;
+	}
+
+	rc = ha->tgt.tgt_ops->handle_tmr(mcmd, mcmd->unpacked_lun,
+	    mcmd->tmr_func, tag);
+
+	if (rc != 0) {
+		spin_lock_irqsave(mcmd->qpair->qp_lock_ptr, flags);
+		switch (mcmd->tmr_func) {
+		case QLA_TGT_ABTS:
+			qlt_24xx_send_abts_resp(mcmd->qpair,
+			    &mcmd->orig_iocb.abts,
+			    FCP_TMF_REJECTED, false);
+			break;
+		case QLA_TGT_LUN_RESET:
+		case QLA_TGT_CLEAR_TS:
+		case QLA_TGT_ABORT_TS:
+		case QLA_TGT_CLEAR_ACA:
+		case QLA_TGT_TARGET_RESET:
+			qlt_send_busy(mcmd->qpair, &mcmd->orig_iocb.atio,
+			    qla_sam_status);
+			break;
+
+		case QLA_TGT_ABORT_ALL:
+		case QLA_TGT_NEXUS_LOSS_SESS:
+		case QLA_TGT_NEXUS_LOSS:
+			qlt_send_notify_ack(mcmd->qpair,
+			    &mcmd->orig_iocb.imm_ntfy, 0, 0, 0, 0, 0, 0);
+			break;
+		}
+		spin_unlock_irqrestore(mcmd->qpair->qp_lock_ptr, flags);
+
+		ql_dbg(ql_dbg_tgt_mgt, mcmd->vha, 0xf052,
+		    "qla_target(%d):  tgt_ops->handle_tmr() failed: %d\n",
+		    mcmd->vha->vp_idx, rc);
+		mempool_free(mcmd, qla_tgt_mgmt_cmd_mempool);
+	}
+}
+
 /* ha->hardware_lock supposed to be held on entry */
 static int __qlt_24xx_handle_abts(struct scsi_qla_host *vha,
 	struct abts_recv_from_24xx *abts, struct fc_port *sess)
 {
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_tgt_mgmt_cmd *mcmd;
-	int rc;
+	struct qla_qpair_hint *h = &vha->vha_tgt.qla_tgt->qphints[0];
 
 	if (abort_cmd_for_tag(vha, abts->exchange_addr_to_abort)) {
 		/* send TASK_ABORT response immediately */
@@ -1961,23 +2026,29 @@ static int __qlt_24xx_handle_abts(struct scsi_qla_host *vha,
 	memcpy(&mcmd->orig_iocb.abts, abts, sizeof(mcmd->orig_iocb.abts));
 	mcmd->reset_count = ha->base_qpair->chip_reset;
 	mcmd->tmr_func = QLA_TGT_ABTS;
-	mcmd->qpair = ha->base_qpair;
+	mcmd->qpair = h->qpair;
 	mcmd->vha = vha;
 
 	/*
 	 * LUN is looked up by target-core internally based on the passed
 	 * abts->exchange_addr_to_abort tag.
 	 */
-	rc = ha->tgt.tgt_ops->handle_tmr(mcmd, 0, mcmd->tmr_func,
-	    abts->exchange_addr_to_abort);
-	if (rc != 0) {
-		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf052,
-		    "qla_target(%d):  tgt_ops->handle_tmr()"
-		    " failed: %d", vha->vp_idx, rc);
-		mempool_free(mcmd, qla_tgt_mgmt_cmd_mempool);
-		return -EFAULT;
+	mcmd->se_cmd.cpuid = h->cpuid;
+
+	if (ha->tgt.tgt_ops->find_cmd_by_tag) {
+		struct qla_tgt_cmd *abort_cmd;
+
+		abort_cmd = ha->tgt.tgt_ops->find_cmd_by_tag(sess,
+		    abts->exchange_addr_to_abort);
+		if (abort_cmd && abort_cmd->qpair) {
+			mcmd->qpair = abort_cmd->qpair;
+			mcmd->se_cmd.cpuid = abort_cmd->se_cmd.cpuid;
+		}
 	}
 
+	INIT_WORK(&mcmd->work, qlt_do_tmr_work);
+	queue_work_on(mcmd->se_cmd.cpuid, qla_tgt_wq, &mcmd->work);
+
 	return 0;
 }
 
@@ -3556,13 +3627,6 @@ static int __qlt_send_term_exchange(struct qla_qpair *qpair,
 	temp = be16_to_cpu(atio->u.isp24.fcp_hdr.ox_id);
 	ctio24->u.status1.ox_id = cpu_to_le16(temp);
 
-	/* Most likely, it isn't needed */
-	ctio24->u.status1.residual = get_unaligned((uint32_t *)
-	    &atio->u.isp24.fcp_cmnd.add_cdb[
-	    atio->u.isp24.fcp_cmnd.add_cdb_len]);
-	if (ctio24->u.status1.residual != 0)
-		ctio24->u.status1.scsi_status |= SS_RESIDUAL_UNDER;
-
 	/* Memory Barrier */
 	wmb();
 	if (qpair->reqq_start_iocbs)
@@ -4057,9 +4121,7 @@ static void __qlt_do_work(struct qla_tgt_cmd *cmd)
 
 	fcp_task_attr = qlt_get_fcp_task_attr(vha,
 	    atio->u.isp24.fcp_cmnd.task_attr);
-	data_length = be32_to_cpu(get_unaligned((uint32_t *)
-	    &atio->u.isp24.fcp_cmnd.add_cdb[
-	    atio->u.isp24.fcp_cmnd.add_cdb_len]));
+	data_length = get_datalen_for_atio(atio);
 
 	ret = ha->tgt.tgt_ops->handle_cmd(vha, cmd, cdb, data_length,
 				          fcp_task_attr, data_dir, bidi);
@@ -4335,7 +4397,7 @@ static int qlt_issue_task_mgmt(struct fc_port *sess, u64 lun,
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_tgt_mgmt_cmd *mcmd;
 	struct atio_from_isp *a = (struct atio_from_isp *)iocb;
-	int res;
+	struct qla_qpair_hint *h = &vha->vha_tgt.qla_tgt->qphints[0];
 
 	mcmd = mempool_alloc(qla_tgt_mgmt_cmd_mempool, GFP_ATOMIC);
 	if (!mcmd) {
@@ -4355,24 +4417,36 @@ static int qlt_issue_task_mgmt(struct fc_port *sess, u64 lun,
 	mcmd->tmr_func = fn;
 	mcmd->flags = flags;
 	mcmd->reset_count = ha->base_qpair->chip_reset;
-	mcmd->qpair = ha->base_qpair;
+	mcmd->qpair = h->qpair;
 	mcmd->vha = vha;
+	mcmd->se_cmd.cpuid = h->cpuid;
+	mcmd->unpacked_lun = lun;
 
 	switch (fn) {
 	case QLA_TGT_LUN_RESET:
-	    abort_cmds_for_lun(vha, lun, a->u.isp24.fcp_hdr.s_id);
-	    break;
-	}
+	case QLA_TGT_CLEAR_TS:
+	case QLA_TGT_ABORT_TS:
+		abort_cmds_for_lun(vha, lun, a->u.isp24.fcp_hdr.s_id);
+		/* drop through */
+	case QLA_TGT_CLEAR_ACA:
+		h = qlt_find_qphint(vha, mcmd->unpacked_lun);
+		mcmd->qpair = h->qpair;
+		mcmd->se_cmd.cpuid = h->cpuid;
+		break;
 
-	res = ha->tgt.tgt_ops->handle_tmr(mcmd, lun, mcmd->tmr_func, 0);
-	if (res != 0) {
-		ql_dbg(ql_dbg_tgt_tmr, vha, 0x1000b,
-		    "qla_target(%d): tgt.tgt_ops->handle_tmr() failed: %d\n",
-		    sess->vha->vp_idx, res);
-		mempool_free(mcmd, qla_tgt_mgmt_cmd_mempool);
-		return -EFAULT;
+	case QLA_TGT_TARGET_RESET:
+	case QLA_TGT_NEXUS_LOSS_SESS:
+	case QLA_TGT_NEXUS_LOSS:
+	case QLA_TGT_ABORT_ALL:
+	default:
+		/* no-op */
+		break;
 	}
 
+	INIT_WORK(&mcmd->work, qlt_do_tmr_work);
+	queue_work_on(mcmd->se_cmd.cpuid, qla_tgt_wq,
+	    &mcmd->work);
+
 	return 0;
 }
 
@@ -4841,7 +4915,6 @@ static int qlt_24xx_handle_els(struct scsi_qla_host *vha,
 			switch (sess->disc_state) {
 			case DSC_LOGIN_PEND:
 			case DSC_GPDB:
-			case DSC_GPSC:
 			case DSC_UPD_FCPORT:
 			case DSC_LOGIN_COMPLETE:
 			case DSC_ADISC:
@@ -5113,8 +5186,6 @@ static void qlt_handle_imm_notify(struct scsi_qla_host *vha,
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf038,
 		    "qla_target(%d): Immediate notify task %x\n",
 		    vha->vp_idx, iocb->u.isp2x.task_flags);
-		if (qlt_handle_task_mgmt(vha, iocb) == 0)
-			send_notify_ack = 0;
 		break;
 
 	case IMM_NTFY_ELS:
@@ -5147,10 +5218,15 @@ static int __qlt_send_busy(struct qla_qpair *qpair,
 	struct fc_port *sess = NULL;
 	unsigned long flags;
 	u16 temp;
+	port_id_t id;
+
+	id.b.al_pa = atio->u.isp24.fcp_hdr.s_id[2];
+	id.b.area = atio->u.isp24.fcp_hdr.s_id[1];
+	id.b.domain = atio->u.isp24.fcp_hdr.s_id[0];
+	id.b.rsvd_1 = 0;
 
 	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
-	sess = ha->tgt.tgt_ops->find_sess_by_s_id(vha,
-	    atio->u.isp24.fcp_hdr.s_id);
+	sess = qla2x00_find_fcport_by_nportid(vha, &id, 1);
 	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 	if (!sess) {
 		qlt_send_term_exchange(qpair, NULL, atio, 1, 0);
@@ -5189,6 +5265,12 @@ static int __qlt_send_busy(struct qla_qpair *qpair,
 	 */
 	ctio24->u.status1.ox_id = swab16(atio->u.isp24.fcp_hdr.ox_id);
 	ctio24->u.status1.scsi_status = cpu_to_le16(status);
+
+	ctio24->u.status1.residual = get_datalen_for_atio(atio);
+
+	if (ctio24->u.status1.residual != 0)
+		ctio24->u.status1.scsi_status |= SS_RESIDUAL_UNDER;
+
 	/* Memory Barrier */
 	wmb();
 	if (qpair->reqq_start_iocbs)
diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h
index 728ce74358e7..fecf96f0225c 100644
--- a/drivers/scsi/qla2xxx/qla_target.h
+++ b/drivers/scsi/qla2xxx/qla_target.h
@@ -682,7 +682,7 @@ struct qla_tgt_cmd;
  * target module (tcm_qla2xxx).
  */
 struct qla_tgt_func_tmpl {
-
+	struct qla_tgt_cmd *(*find_cmd_by_tag)(struct fc_port *, uint64_t);
 	int (*handle_cmd)(struct scsi_qla_host *, struct qla_tgt_cmd *,
 			unsigned char *, uint32_t, int, int, int);
 	void (*handle_data)(struct qla_tgt_cmd *);
@@ -966,6 +966,8 @@ struct qla_tgt_mgmt_cmd {
 	unsigned int flags;
 	uint32_t reset_count;
 #define QLA24XX_MGMT_SEND_NACK	1
+	struct work_struct work;
+	uint64_t unpacked_lun;
 	union {
 		struct atio_from_isp atio;
 		struct imm_ntfy_from_isp imm_ntfy;
diff --git a/drivers/scsi/qla2xxx/qla_version.h b/drivers/scsi/qla2xxx/qla_version.h
index 0c55d7057280..1ad7582220c3 100644
--- a/drivers/scsi/qla2xxx/qla_version.h
+++ b/drivers/scsi/qla2xxx/qla_version.h
@@ -7,7 +7,7 @@
 /*
  * Driver version
  */
-#define QLA2XXX_VERSION      "10.00.00.06-k"
+#define QLA2XXX_VERSION      "10.00.00.07-k"
 
 #define QLA_DRIVER_MAJOR_VER	10
 #define QLA_DRIVER_MINOR_VER	0
diff --git a/drivers/scsi/qla2xxx/tcm_qla2xxx.c b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
index aadfeaac3898..0c2e82af9c0a 100644
--- a/drivers/scsi/qla2xxx/tcm_qla2xxx.c
+++ b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
@@ -48,7 +48,6 @@
 #include "tcm_qla2xxx.h"
 
 static struct workqueue_struct *tcm_qla2xxx_free_wq;
-static struct workqueue_struct *tcm_qla2xxx_cmd_wq;
 
 /*
  * Parse WWN.
@@ -630,6 +629,32 @@ static int tcm_qla2xxx_handle_tmr(struct qla_tgt_mgmt_cmd *mcmd, u64 lun,
 	    transl_tmr_func, GFP_ATOMIC, tag, flags);
 }
 
+static struct qla_tgt_cmd *tcm_qla2xxx_find_cmd_by_tag(struct fc_port *sess,
+    uint64_t tag)
+{
+	struct qla_tgt_cmd *cmd = NULL;
+	struct se_cmd *secmd;
+	unsigned long flags;
+
+	if (!sess->se_sess)
+		return NULL;
+
+	spin_lock_irqsave(&sess->se_sess->sess_cmd_lock, flags);
+	list_for_each_entry(secmd, &sess->se_sess->sess_cmd_list, se_cmd_list) {
+		/* skip task management functions, including tmr->task_cmd */
+		if (secmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
+			continue;
+
+		if (secmd->tag == tag) {
+			cmd = container_of(secmd, struct qla_tgt_cmd, se_cmd);
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&sess->se_sess->sess_cmd_lock, flags);
+
+	return cmd;
+}
+
 static int tcm_qla2xxx_queue_data_in(struct se_cmd *se_cmd)
 {
 	struct qla_tgt_cmd *cmd = container_of(se_cmd,
@@ -1608,6 +1633,7 @@ static void tcm_qla2xxx_update_sess(struct fc_port *sess, port_id_t s_id,
  * Calls into tcm_qla2xxx used by qla2xxx LLD I/O path.
  */
 static struct qla_tgt_func_tmpl tcm_qla2xxx_template = {
+	.find_cmd_by_tag	= tcm_qla2xxx_find_cmd_by_tag,
 	.handle_cmd		= tcm_qla2xxx_handle_cmd,
 	.handle_data		= tcm_qla2xxx_handle_data,
 	.handle_tmr		= tcm_qla2xxx_handle_tmr,
@@ -1976,16 +2002,8 @@ static int tcm_qla2xxx_register_configfs(void)
 		goto out_fabric_npiv;
 	}
 
-	tcm_qla2xxx_cmd_wq = alloc_workqueue("tcm_qla2xxx_cmd", 0, 0);
-	if (!tcm_qla2xxx_cmd_wq) {
-		ret = -ENOMEM;
-		goto out_free_wq;
-	}
-
 	return 0;
 
-out_free_wq:
-	destroy_workqueue(tcm_qla2xxx_free_wq);
 out_fabric_npiv:
 	target_unregister_template(&tcm_qla2xxx_npiv_ops);
 out_fabric:
@@ -1995,7 +2013,6 @@ static int tcm_qla2xxx_register_configfs(void)
 
 static void tcm_qla2xxx_deregister_configfs(void)
 {
-	destroy_workqueue(tcm_qla2xxx_cmd_wq);
 	destroy_workqueue(tcm_qla2xxx_free_wq);
 
 	target_unregister_template(&tcm_qla2xxx_ops);
diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
index 94c14ce94da2..0e13349dce57 100644
--- a/drivers/scsi/qla4xxx/ql4_os.c
+++ b/drivers/scsi/qla4xxx/ql4_os.c
@@ -1848,7 +1848,7 @@ static enum blk_eh_timer_return qla4xxx_eh_cmd_timed_out(struct scsi_cmnd *sc)
 	struct iscsi_cls_session *session;
 	struct iscsi_session *sess;
 	unsigned long flags;
-	enum blk_eh_timer_return ret = BLK_EH_NOT_HANDLED;
+	enum blk_eh_timer_return ret = BLK_EH_DONE;
 
 	session = starget_to_session(scsi_target(sc->device));
 	sess = session->dd_data;
diff --git a/drivers/scsi/qlogicpti.c b/drivers/scsi/qlogicpti.c
index cec9a14982e6..8578e566ab41 100644
--- a/drivers/scsi/qlogicpti.c
+++ b/drivers/scsi/qlogicpti.c
@@ -1385,6 +1385,9 @@ static int qpti_sbus_probe(struct platform_device *op)
 			  qpti->req_cpu, qpti->req_dvma);
 #undef QSIZE
 
+fail_free_irq:
+	free_irq(qpti->irq, qpti);
+
 fail_unmap_regs:
 	of_iounmap(&op->resource[0], qpti->qregs,
 		   resource_size(&op->resource[0]));
@@ -1392,9 +1395,6 @@ static int qpti_sbus_probe(struct platform_device *op)
 		of_iounmap(&op->resource[0], qpti->sreg,
 			   sizeof(unsigned char));
 
-fail_free_irq:
-	free_irq(qpti->irq, qpti);
-
 fail_unlink:
 	scsi_host_put(host);
 
diff --git a/drivers/scsi/scsi_debugfs.c b/drivers/scsi/scsi_debugfs.c
index b784002ef0bd..c5a8756384bc 100644
--- a/drivers/scsi/scsi_debugfs.c
+++ b/drivers/scsi/scsi_debugfs.c
@@ -4,7 +4,7 @@
 #include <scsi/scsi_dbg.h>
 #include "scsi_debugfs.h"
 
-#define SCSI_CMD_FLAG_NAME(name) [ilog2(SCMD_##name)] = #name
+#define SCSI_CMD_FLAG_NAME(name)[const_ilog2(SCMD_##name)] = #name
 static const char *const scsi_cmd_flags[] = {
 	SCSI_CMD_FLAG_NAME(TAGGED),
 	SCSI_CMD_FLAG_NAME(UNCHECKED_ISA_DMA),
diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c
index dd107dc4db0e..c4cbfd07b916 100644
--- a/drivers/scsi/scsi_devinfo.c
+++ b/drivers/scsi/scsi_devinfo.c
@@ -161,15 +161,16 @@ static struct {
 	{"DGC", "RAID", NULL, BLIST_SPARSELUN},	/* EMC CLARiiON, storage on LUN 0 */
 	{"DGC", "DISK", NULL, BLIST_SPARSELUN},	/* EMC CLARiiON, no storage on LUN 0 */
 	{"EMC",  "Invista", "*", BLIST_SPARSELUN | BLIST_LARGELUN},
-	{"EMC", "SYMMETRIX", NULL, BLIST_SPARSELUN | BLIST_LARGELUN | BLIST_REPORTLUN2},
+	{"EMC", "SYMMETRIX", NULL, BLIST_SPARSELUN | BLIST_LARGELUN |
+	 BLIST_REPORTLUN2 | BLIST_RETRY_ITF},
 	{"EMULEX", "MD21/S2     ESDI", NULL, BLIST_SINGLELUN},
 	{"easyRAID", "16P", NULL, BLIST_NOREPORTLUN},
 	{"easyRAID", "X6P", NULL, BLIST_NOREPORTLUN},
 	{"easyRAID", "F8", NULL, BLIST_NOREPORTLUN},
 	{"FSC", "CentricStor", "*", BLIST_SPARSELUN | BLIST_LARGELUN},
+	{"FUJITSU", "ETERNUS_DXM", "*", BLIST_RETRY_ASC_C1},
 	{"Generic", "USB SD Reader", "1.00", BLIST_FORCELUN | BLIST_INQUIRY_36},
-	{"Generic", "USB Storage-SMC", "0180", BLIST_FORCELUN | BLIST_INQUIRY_36},
-	{"Generic", "USB Storage-SMC", "0207", BLIST_FORCELUN | BLIST_INQUIRY_36},
+	{"Generic", "USB Storage-SMC", NULL, BLIST_FORCELUN | BLIST_INQUIRY_36}, /* FW: 0180 and 0207 */
 	{"HITACHI", "DF400", "*", BLIST_REPORTLUN2},
 	{"HITACHI", "DF500", "*", BLIST_REPORTLUN2},
 	{"HITACHI", "DISK-SUBSYSTEM", "*", BLIST_REPORTLUN2},
@@ -361,8 +362,22 @@ int scsi_dev_info_list_add_keyed(int compatible, char *vendor, char *model,
 	scsi_strcpy_devinfo("model", devinfo->model, sizeof(devinfo->model),
 			    model, compatible);
 
-	if (strflags)
-		flags = (__force blist_flags_t)simple_strtoul(strflags, NULL, 0);
+	if (strflags) {
+		unsigned long long val;
+		int ret = kstrtoull(strflags, 0, &val);
+
+		if (ret != 0) {
+			kfree(devinfo);
+			return ret;
+		}
+		flags = (__force blist_flags_t)val;
+	}
+	if (flags & __BLIST_UNUSED_MASK) {
+		pr_err("scsi_devinfo (%s:%s): unsupported flags 0x%llx",
+		       vendor, model, flags & __BLIST_UNUSED_MASK);
+		kfree(devinfo);
+		return -EINVAL;
+	}
 	devinfo->flags = flags;
 	devinfo->compatible = compatible;
 
@@ -615,7 +630,7 @@ static int devinfo_seq_show(struct seq_file *m, void *v)
 	    devinfo_table->name)
 		seq_printf(m, "[%s]:\n", devinfo_table->name);
 
-	seq_printf(m, "'%.8s' '%.16s' 0x%x\n",
+	seq_printf(m, "'%.8s' '%.16s' 0x%llx\n",
 		   devinfo->vendor, devinfo->model, devinfo->flags);
 	return 0;
 }
@@ -734,9 +749,9 @@ MODULE_PARM_DESC(dev_flags,
 	 " list entries for vendor and model with an integer value of flags"
 	 " to the scsi device info list");
 
-module_param_named(default_dev_flags, scsi_default_dev_flags, int, S_IRUGO|S_IWUSR);
+module_param_named(default_dev_flags, scsi_default_dev_flags, ullong, 0644);
 MODULE_PARM_DESC(default_dev_flags,
-		 "scsi default device flag integer value");
+		 "scsi default device flag uint64_t value");
 
 /**
  * scsi_exit_devinfo - remove /proc/scsi/device_info & the scsi_dev_info_list
diff --git a/drivers/scsi/scsi_dh.c b/drivers/scsi/scsi_dh.c
index 188f30572aa1..5a58cbf3a75d 100644
--- a/drivers/scsi/scsi_dh.c
+++ b/drivers/scsi/scsi_dh.c
@@ -58,7 +58,10 @@ static const struct scsi_dh_blist scsi_dh_blist[] = {
 	{"IBM", "3526",			"rdac", },
 	{"IBM", "3542",			"rdac", },
 	{"IBM", "3552",			"rdac", },
-	{"SGI", "TP9",			"rdac", },
+	{"SGI", "TP9300",		"rdac", },
+	{"SGI", "TP9400",		"rdac", },
+	{"SGI", "TP9500",		"rdac", },
+	{"SGI", "TP9700",		"rdac", },
 	{"SGI", "IS",			"rdac", },
 	{"STK", "OPENstorage",		"rdac", },
 	{"STK", "FLEXLINE 380",		"rdac", },
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index 946039117bf4..8932ae81a15a 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -38,6 +38,7 @@
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_ioctl.h>
 #include <scsi/scsi_dh.h>
+#include <scsi/scsi_devinfo.h>
 #include <scsi/sg.h>
 
 #include "scsi_priv.h"
@@ -282,7 +283,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd)
 enum blk_eh_timer_return scsi_times_out(struct request *req)
 {
 	struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(req);
-	enum blk_eh_timer_return rtn = BLK_EH_NOT_HANDLED;
+	enum blk_eh_timer_return rtn = BLK_EH_DONE;
 	struct Scsi_Host *host = scmd->device->host;
 
 	trace_scsi_dispatch_cmd_timeout(scmd);
@@ -294,7 +295,7 @@ enum blk_eh_timer_return scsi_times_out(struct request *req)
 	if (host->hostt->eh_timed_out)
 		rtn = host->hostt->eh_timed_out(scmd);
 
-	if (rtn == BLK_EH_NOT_HANDLED) {
+	if (rtn == BLK_EH_DONE) {
 		if (scsi_abort_command(scmd) != SUCCESS) {
 			set_host_byte(scmd, DID_TIME_OUT);
 			scsi_eh_scmd_add(scmd);
@@ -525,6 +526,12 @@ int scsi_check_sense(struct scsi_cmnd *scmd)
 		if (sshdr.asc == 0x10) /* DIF */
 			return SUCCESS;
 
+		if (sshdr.asc == 0x44 && sdev->sdev_bflags & BLIST_RETRY_ITF)
+			return ADD_TO_MLQUEUE;
+		if (sshdr.asc == 0xc1 && sshdr.ascq == 0x01 &&
+		    sdev->sdev_bflags & BLIST_RETRY_ASC_C1)
+			return ADD_TO_MLQUEUE;
+
 		return NEEDS_RETRY;
 	case NOT_READY:
 	case UNIT_ATTENTION:
@@ -1933,11 +1940,7 @@ static void scsi_eh_lock_door(struct scsi_device *sdev)
 	struct request *req;
 	struct scsi_request *rq;
 
-	/*
-	 * blk_get_request with GFP_KERNEL (__GFP_RECLAIM) sleeps until a
-	 * request becomes available
-	 */
-	req = blk_get_request(sdev->request_queue, REQ_OP_SCSI_IN, GFP_KERNEL);
+	req = blk_get_request(sdev->request_queue, REQ_OP_SCSI_IN, 0);
 	if (IS_ERR(req))
 		return;
 	rq = scsi_req(req);
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index e9b4f279d29c..41e9ac9fc138 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -265,7 +265,7 @@ int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
 	struct scsi_request *rq;
 	int ret = DRIVER_ERROR << 24;
 
-	req = blk_get_request_flags(sdev->request_queue,
+	req = blk_get_request(sdev->request_queue,
 			data_direction == DMA_TO_DEVICE ?
 			REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, BLK_MQ_REQ_PREEMPT);
 	if (IS_ERR(req))
@@ -273,7 +273,7 @@ int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
 	rq = scsi_req(req);
 
 	if (bufflen &&	blk_rq_map_kern(sdev->request_queue, req,
-					buffer, bufflen, __GFP_RECLAIM))
+					buffer, bufflen, GFP_NOIO))
 		goto out;
 
 	rq->cmd_len = COMMAND_SIZE(cmd[0]);
@@ -985,6 +985,10 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
 				case 0x08: /* Long write in progress */
 				case 0x09: /* self test in progress */
 				case 0x14: /* space allocation in progress */
+				case 0x1a: /* start stop unit in progress */
+				case 0x1b: /* sanitize in progress */
+				case 0x1d: /* configuration in progress */
+				case 0x24: /* depopulation in progress */
 					action = ACTION_DELAYED_RETRY;
 					break;
 				default:
@@ -2149,27 +2153,6 @@ static int scsi_map_queues(struct blk_mq_tag_set *set)
 	return blk_mq_map_queues(set);
 }
 
-static u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
-{
-	struct device *host_dev;
-	u64 bounce_limit = 0xffffffff;
-
-	if (shost->unchecked_isa_dma)
-		return BLK_BOUNCE_ISA;
-	/*
-	 * Platforms with virtual-DMA translation
-	 * hardware have no practical limit.
-	 */
-	if (!PCI_DMA_BUS_IS_PHYS)
-		return BLK_BOUNCE_ANY;
-
-	host_dev = scsi_get_device(shost);
-	if (host_dev && host_dev->dma_mask)
-		bounce_limit = (u64)dma_max_pfn(host_dev) << PAGE_SHIFT;
-
-	return bounce_limit;
-}
-
 void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q)
 {
 	struct device *dev = shost->dma_dev;
@@ -2189,7 +2172,8 @@ void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q)
 	}
 
 	blk_queue_max_hw_sectors(q, shost->max_sectors);
-	blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
+	if (shost->unchecked_isa_dma)
+		blk_queue_bounce_limit(q, BLK_BOUNCE_ISA);
 	blk_queue_segment_boundary(q, shost->dma_boundary);
 	dma_set_seg_boundary(dev, shost->dma_boundary);
 
diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c
index 1e36c9a9ad17..7943b762c12d 100644
--- a/drivers/scsi/scsi_sysfs.c
+++ b/drivers/scsi/scsi_sysfs.c
@@ -968,7 +968,7 @@ sdev_show_wwid(struct device *dev, struct device_attribute *attr,
 static DEVICE_ATTR(wwid, S_IRUGO, sdev_show_wwid, NULL);
 
 #define BLIST_FLAG_NAME(name)					\
-	[ilog2((__force unsigned int)BLIST_##name)] = #name
+	[const_ilog2((__force __u64)BLIST_##name)] = #name
 static const char *const sdev_bflags_name[] = {
 #include "scsi_devinfo_tbl.c"
 };
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index be3be0f9cb2d..1da3d71e9f61 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -2087,7 +2087,7 @@ fc_eh_timed_out(struct scsi_cmnd *scmd)
 	if (rport->port_state == FC_PORTSTATE_BLOCKED)
 		return BLK_EH_RESET_TIMER;
 
-	return BLK_EH_NOT_HANDLED;
+	return BLK_EH_DONE;
 }
 EXPORT_SYMBOL(fc_eh_timed_out);
 
@@ -3591,10 +3591,9 @@ fc_bsg_job_timeout(struct request *req)
 	}
 
 	/* the blk_end_sync_io() doesn't check the error */
-	if (!inflight)
-		return BLK_EH_NOT_HANDLED;
-	else
-		return BLK_EH_HANDLED;
+	if (inflight)
+		blk_mq_complete_request(req);
+	return BLK_EH_DONE;
 }
 
 /**
@@ -3781,8 +3780,7 @@ fc_bsg_hostadd(struct Scsi_Host *shost, struct fc_host_attrs *fc_host)
 	snprintf(bsg_name, sizeof(bsg_name),
 		 "fc_host%d", shost->host_no);
 
-	q = bsg_setup_queue(dev, bsg_name, fc_bsg_dispatch, i->f->dd_bsg_size,
-			NULL);
+	q = bsg_setup_queue(dev, bsg_name, fc_bsg_dispatch, i->f->dd_bsg_size);
 	if (IS_ERR(q)) {
 		dev_err(dev,
 			"fc_host%d: bsg interface failed to initialize - setup queue\n",
@@ -3827,8 +3825,8 @@ fc_bsg_rportadd(struct Scsi_Host *shost, struct fc_rport *rport)
 	if (!i->f->bsg_request)
 		return -ENOTSUPP;
 
-	q = bsg_setup_queue(dev, NULL, fc_bsg_dispatch, i->f->dd_bsg_size,
-			NULL);
+	q = bsg_setup_queue(dev, dev_name(dev), fc_bsg_dispatch,
+			i->f->dd_bsg_size);
 	if (IS_ERR(q)) {
 		dev_err(dev, "failed to setup bsg queue\n");
 		return PTR_ERR(q);
diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c
index 65f6c94f2e9b..6fd2fe210fc3 100644
--- a/drivers/scsi/scsi_transport_iscsi.c
+++ b/drivers/scsi/scsi_transport_iscsi.c
@@ -1542,7 +1542,7 @@ iscsi_bsg_host_add(struct Scsi_Host *shost, struct iscsi_cls_host *ihost)
 		return -ENOTSUPP;
 
 	snprintf(bsg_name, sizeof(bsg_name), "iscsi_host%d", shost->host_no);
-	q = bsg_setup_queue(dev, bsg_name, iscsi_bsg_host_dispatch, 0, NULL);
+	q = bsg_setup_queue(dev, bsg_name, iscsi_bsg_host_dispatch, 0);
 	if (IS_ERR(q)) {
 		shost_printk(KERN_ERR, shost, "bsg interface failed to "
 			     "initialize - no request queue\n");
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index 08acbabfae07..0cd16e80b019 100644
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -187,16 +187,6 @@ static int sas_smp_dispatch(struct bsg_job *job)
 	return 0;
 }
 
-static void sas_host_release(struct device *dev)
-{
-	struct Scsi_Host *shost = dev_to_shost(dev);
-	struct sas_host_attrs *sas_host = to_sas_host_attrs(shost);
-	struct request_queue *q = sas_host->q;
-
-	if (q)
-		blk_cleanup_queue(q);
-}
-
 static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy)
 {
 	struct request_queue *q;
@@ -208,7 +198,7 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy)
 
 	if (rphy) {
 		q = bsg_setup_queue(&rphy->dev, dev_name(&rphy->dev),
-				sas_smp_dispatch, 0, NULL);
+				sas_smp_dispatch, 0);
 		if (IS_ERR(q))
 			return PTR_ERR(q);
 		rphy->q = q;
@@ -217,16 +207,12 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy)
 
 		snprintf(name, sizeof(name), "sas_host%d", shost->host_no);
 		q = bsg_setup_queue(&shost->shost_gendev, name,
-				sas_smp_dispatch, 0, sas_host_release);
+				sas_smp_dispatch, 0);
 		if (IS_ERR(q))
 			return PTR_ERR(q);
 		to_sas_host_attrs(shost)->q = q;
 	}
 
-	/*
-	 * by default assume old behaviour and bounce for any highmem page
-	 */
-	blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH);
 	blk_queue_flag_set(QUEUE_FLAG_BIDI, q);
 	return 0;
 }
@@ -260,8 +246,11 @@ static int sas_host_remove(struct transport_container *tc, struct device *dev,
 	struct Scsi_Host *shost = dev_to_shost(dev);
 	struct request_queue *q = to_sas_host_attrs(shost)->q;
 
-	if (q)
+	if (q) {
 		bsg_unregister_queue(q);
+		blk_cleanup_queue(q);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/scsi/scsi_transport_srp.c b/drivers/scsi/scsi_transport_srp.c
index 36f6190931bc..4e46fdb2d7c9 100644
--- a/drivers/scsi/scsi_transport_srp.c
+++ b/drivers/scsi/scsi_transport_srp.c
@@ -51,6 +51,8 @@ struct srp_internal {
 	struct transport_container rport_attr_cont;
 };
 
+static int scsi_is_srp_rport(const struct device *dev);
+
 #define to_srp_internal(tmpl) container_of(tmpl, struct srp_internal, t)
 
 #define	dev_to_rport(d)	container_of(d, struct srp_rport, dev)
@@ -60,9 +62,24 @@ static inline struct Scsi_Host *rport_to_shost(struct srp_rport *r)
 	return dev_to_shost(r->dev.parent);
 }
 
+static int find_child_rport(struct device *dev, void *data)
+{
+	struct device **child = data;
+
+	if (scsi_is_srp_rport(dev)) {
+		WARN_ON_ONCE(*child);
+		*child = dev;
+	}
+	return 0;
+}
+
 static inline struct srp_rport *shost_to_rport(struct Scsi_Host *shost)
 {
-	return transport_class_to_srp_rport(&shost->shost_gendev);
+	struct device *child = NULL;
+
+	WARN_ON_ONCE(device_for_each_child(&shost->shost_gendev, &child,
+					   find_child_rport) < 0);
+	return child ? dev_to_rport(child) : NULL;
 }
 
 /**
@@ -587,7 +604,7 @@ EXPORT_SYMBOL(srp_reconnect_rport);
  *
  * If a timeout occurs while an rport is in the blocked state, ask the SCSI
  * EH to continue waiting (BLK_EH_RESET_TIMER). Otherwise let the SCSI core
- * handle the timeout (BLK_EH_NOT_HANDLED).
+ * handle the timeout (BLK_EH_DONE).
  *
  * Note: This function is called from soft-IRQ context and with the request
  * queue lock held.
@@ -600,9 +617,10 @@ enum blk_eh_timer_return srp_timed_out(struct scsi_cmnd *scmd)
 	struct srp_rport *rport = shost_to_rport(shost);
 
 	pr_debug("timeout for sdev %s\n", dev_name(&sdev->sdev_gendev));
-	return rport->fast_io_fail_tmo < 0 && rport->dev_loss_tmo < 0 &&
+	return rport && rport->fast_io_fail_tmo < 0 &&
+		rport->dev_loss_tmo < 0 &&
 		i->f->reset_timer_if_blocked && scsi_device_blocked(sdev) ?
-		BLK_EH_RESET_TIMER : BLK_EH_NOT_HANDLED;
+		BLK_EH_RESET_TIMER : BLK_EH_DONE;
 }
 EXPORT_SYMBOL(srp_timed_out);
 
diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h
index 0d663b5e45bb..392c7d078ae3 100644
--- a/drivers/scsi/sd.h
+++ b/drivers/scsi/sd.h
@@ -74,12 +74,12 @@ struct scsi_disk {
 	struct gendisk	*disk;
 	struct opal_dev *opal_dev;
 #ifdef CONFIG_BLK_DEV_ZONED
-	unsigned int	nr_zones;
-	unsigned int	zone_blocks;
-	unsigned int	zone_shift;
-	unsigned int	zones_optimal_open;
-	unsigned int	zones_optimal_nonseq;
-	unsigned int	zones_max_open;
+	u32		nr_zones;
+	u32		zone_blocks;
+	u32		zone_shift;
+	u32		zones_optimal_open;
+	u32		zones_optimal_nonseq;
+	u32		zones_max_open;
 #endif
 	atomic_t	openers;
 	sector_t	capacity;	/* size in logical blocks */
diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c
index 210407cd2341..323e3dc4bc59 100644
--- a/drivers/scsi/sd_zbc.c
+++ b/drivers/scsi/sd_zbc.c
@@ -299,16 +299,6 @@ void sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes,
 	case REQ_OP_WRITE:
 	case REQ_OP_WRITE_ZEROES:
 	case REQ_OP_WRITE_SAME:
-
-		if (result &&
-		    sshdr->sense_key == ILLEGAL_REQUEST &&
-		    sshdr->asc == 0x21)
-			/*
-			 * INVALID ADDRESS FOR WRITE error: It is unlikely that
-			 * retrying write requests failed with any kind of
-			 * alignement error will result in success. So don't.
-			 */
-			cmd->allowed = 0;
 		break;
 
 	case REQ_OP_ZONE_REPORT:
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index 5c40d809830f..573763908562 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -66,7 +66,6 @@ static int sg_version_num = 30536;	/* 2 digits for each component */
 static char *sg_version_date = "20140603";
 
 static int sg_proc_init(void);
-static void sg_proc_cleanup(void);
 #endif
 
 #define SG_ALLOW_DIO_DEF 0
@@ -1192,7 +1191,7 @@ sg_fasync(int fd, struct file *filp, int mode)
 	return fasync_helper(fd, filp, mode, &sfp->async_qp);
 }
 
-static int
+static vm_fault_t
 sg_vma_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
@@ -1661,7 +1660,7 @@ static void __exit
 exit_sg(void)
 {
 #ifdef CONFIG_SCSI_PROC_FS
-	sg_proc_cleanup();
+	remove_proc_subtree("scsi/sg", NULL);
 #endif				/* CONFIG_SCSI_PROC_FS */
 	scsi_unregister_interface(&sg_interface);
 	class_destroy(sg_sysfs_class);
@@ -1715,7 +1714,7 @@ sg_start_req(Sg_request *srp, unsigned char *cmd)
 	 * does not sleep except under memory pressure.
 	 */
 	rq = blk_get_request(q, hp->dxfer_direction == SG_DXFER_TO_DEV ?
-			REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, GFP_KERNEL);
+			REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, 0);
 	if (IS_ERR(rq)) {
 		kfree(long_cmdp);
 		return PTR_ERR(rq);
@@ -2274,11 +2273,6 @@ sg_get_dev(int dev)
 }
 
 #ifdef CONFIG_SCSI_PROC_FS
-
-static struct proc_dir_entry *sg_proc_sgp = NULL;
-
-static char sg_proc_sg_dirname[] = "scsi/sg";
-
 static int sg_proc_seq_show_int(struct seq_file *s, void *v);
 
 static int sg_proc_single_open_adio(struct inode *inode, struct file *file);
@@ -2306,37 +2300,11 @@ static const struct file_operations dressz_fops = {
 };
 
 static int sg_proc_seq_show_version(struct seq_file *s, void *v);
-static int sg_proc_single_open_version(struct inode *inode, struct file *file);
-static const struct file_operations version_fops = {
-	.owner = THIS_MODULE,
-	.open = sg_proc_single_open_version,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-
 static int sg_proc_seq_show_devhdr(struct seq_file *s, void *v);
-static int sg_proc_single_open_devhdr(struct inode *inode, struct file *file);
-static const struct file_operations devhdr_fops = {
-	.owner = THIS_MODULE,
-	.open = sg_proc_single_open_devhdr,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-
 static int sg_proc_seq_show_dev(struct seq_file *s, void *v);
-static int sg_proc_open_dev(struct inode *inode, struct file *file);
 static void * dev_seq_start(struct seq_file *s, loff_t *pos);
 static void * dev_seq_next(struct seq_file *s, void *v, loff_t *pos);
 static void dev_seq_stop(struct seq_file *s, void *v);
-static const struct file_operations dev_fops = {
-	.owner = THIS_MODULE,
-	.open = sg_proc_open_dev,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = seq_release,
-};
 static const struct seq_operations dev_seq_ops = {
 	.start = dev_seq_start,
 	.next  = dev_seq_next,
@@ -2345,14 +2313,6 @@ static const struct seq_operations dev_seq_ops = {
 };
 
 static int sg_proc_seq_show_devstrs(struct seq_file *s, void *v);
-static int sg_proc_open_devstrs(struct inode *inode, struct file *file);
-static const struct file_operations devstrs_fops = {
-	.owner = THIS_MODULE,
-	.open = sg_proc_open_devstrs,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = seq_release,
-};
 static const struct seq_operations devstrs_seq_ops = {
 	.start = dev_seq_start,
 	.next  = dev_seq_next,
@@ -2361,14 +2321,6 @@ static const struct seq_operations devstrs_seq_ops = {
 };
 
 static int sg_proc_seq_show_debug(struct seq_file *s, void *v);
-static int sg_proc_open_debug(struct inode *inode, struct file *file);
-static const struct file_operations debug_fops = {
-	.owner = THIS_MODULE,
-	.open = sg_proc_open_debug,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = seq_release,
-};
 static const struct seq_operations debug_seq_ops = {
 	.start = dev_seq_start,
 	.next  = dev_seq_next,
@@ -2376,50 +2328,23 @@ static const struct seq_operations debug_seq_ops = {
 	.show  = sg_proc_seq_show_debug,
 };
 
-
-struct sg_proc_leaf {
-	const char * name;
-	const struct file_operations * fops;
-};
-
-static const struct sg_proc_leaf sg_proc_leaf_arr[] = {
-	{"allow_dio", &adio_fops},
-	{"debug", &debug_fops},
-	{"def_reserved_size", &dressz_fops},
-	{"device_hdr", &devhdr_fops},
-	{"devices", &dev_fops},
-	{"device_strs", &devstrs_fops},
-	{"version", &version_fops}
-};
-
 static int
 sg_proc_init(void)
 {
-	int num_leaves = ARRAY_SIZE(sg_proc_leaf_arr);
-	int k;
+	struct proc_dir_entry *p;
 
-	sg_proc_sgp = proc_mkdir(sg_proc_sg_dirname, NULL);
-	if (!sg_proc_sgp)
+	p = proc_mkdir("scsi/sg", NULL);
+	if (!p)
 		return 1;
-	for (k = 0; k < num_leaves; ++k) {
-		const struct sg_proc_leaf *leaf = &sg_proc_leaf_arr[k];
-		umode_t mask = leaf->fops->write ? S_IRUGO | S_IWUSR : S_IRUGO;
-		proc_create(leaf->name, mask, sg_proc_sgp, leaf->fops);
-	}
-	return 0;
-}
 
-static void
-sg_proc_cleanup(void)
-{
-	int k;
-	int num_leaves = ARRAY_SIZE(sg_proc_leaf_arr);
-
-	if (!sg_proc_sgp)
-		return;
-	for (k = 0; k < num_leaves; ++k)
-		remove_proc_entry(sg_proc_leaf_arr[k].name, sg_proc_sgp);
-	remove_proc_entry(sg_proc_sg_dirname, NULL);
+	proc_create("allow_dio", S_IRUGO | S_IWUSR, p, &adio_fops);
+	proc_create_seq("debug", S_IRUGO, p, &debug_seq_ops);
+	proc_create("def_reserved_size", S_IRUGO | S_IWUSR, p, &dressz_fops);
+	proc_create_single("device_hdr", S_IRUGO, p, sg_proc_seq_show_devhdr);
+	proc_create_seq("devices", S_IRUGO, p, &dev_seq_ops);
+	proc_create_seq("device_strs", S_IRUGO, p, &devstrs_seq_ops);
+	proc_create_single("version", S_IRUGO, p, sg_proc_seq_show_version);
+	return 0;
 }
 
 
@@ -2482,22 +2407,12 @@ static int sg_proc_seq_show_version(struct seq_file *s, void *v)
 	return 0;
 }
 
-static int sg_proc_single_open_version(struct inode *inode, struct file *file)
-{
-	return single_open(file, sg_proc_seq_show_version, NULL);
-}
-
 static int sg_proc_seq_show_devhdr(struct seq_file *s, void *v)
 {
 	seq_puts(s, "host\tchan\tid\tlun\ttype\topens\tqdepth\tbusy\tonline\n");
 	return 0;
 }
 
-static int sg_proc_single_open_devhdr(struct inode *inode, struct file *file)
-{
-	return single_open(file, sg_proc_seq_show_devhdr, NULL);
-}
-
 struct sg_proc_deviter {
 	loff_t	index;
 	size_t	max;
@@ -2531,11 +2446,6 @@ static void dev_seq_stop(struct seq_file *s, void *v)
 	kfree(s->private);
 }
 
-static int sg_proc_open_dev(struct inode *inode, struct file *file)
-{
-        return seq_open(file, &dev_seq_ops);
-}
-
 static int sg_proc_seq_show_dev(struct seq_file *s, void *v)
 {
 	struct sg_proc_deviter * it = (struct sg_proc_deviter *) v;
@@ -2562,11 +2472,6 @@ static int sg_proc_seq_show_dev(struct seq_file *s, void *v)
 	return 0;
 }
 
-static int sg_proc_open_devstrs(struct inode *inode, struct file *file)
-{
-        return seq_open(file, &devstrs_seq_ops);
-}
-
 static int sg_proc_seq_show_devstrs(struct seq_file *s, void *v)
 {
 	struct sg_proc_deviter * it = (struct sg_proc_deviter *) v;
@@ -2650,11 +2555,6 @@ static void sg_proc_debug_helper(struct seq_file *s, Sg_device * sdp)
 	}
 }
 
-static int sg_proc_open_debug(struct inode *inode, struct file *file)
-{
-        return seq_open(file, &debug_seq_ops);
-}
-
 static int sg_proc_seq_show_debug(struct seq_file *s, void *v)
 {
 	struct sg_proc_deviter * it = (struct sg_proc_deviter *) v;
diff --git a/drivers/scsi/snic/snic_scsi.c b/drivers/scsi/snic/snic_scsi.c
index d8a376b7882d..d9b2e46424aa 100644
--- a/drivers/scsi/snic/snic_scsi.c
+++ b/drivers/scsi/snic/snic_scsi.c
@@ -47,10 +47,10 @@ static const char * const snic_req_state_str[] = {
 	[SNIC_IOREQ_NOT_INITED]	= "SNIC_IOREQ_NOT_INITED",
 	[SNIC_IOREQ_PENDING]	= "SNIC_IOREQ_PENDING",
 	[SNIC_IOREQ_ABTS_PENDING] = "SNIC_IOREQ_ABTS_PENDING",
-	[SNIC_IOREQ_ABTS_COMPLETE] = "SNIC_IOREQ_ABTS_COMPELTE",
+	[SNIC_IOREQ_ABTS_COMPLETE] = "SNIC_IOREQ_ABTS_COMPLETE",
 	[SNIC_IOREQ_LR_PENDING]	= "SNIC_IOREQ_LR_PENDING",
-	[SNIC_IOREQ_LR_COMPLETE] = "SNIC_IOREQ_LR_COMPELTE",
-	[SNIC_IOREQ_COMPLETE]	= "SNIC_IOREQ_CMD_COMPELTE",
+	[SNIC_IOREQ_LR_COMPLETE] = "SNIC_IOREQ_LR_COMPLETE",
+	[SNIC_IOREQ_COMPLETE]	= "SNIC_IOREQ_CMD_COMPLETE",
 };
 
 /* snic cmd status strings */
diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c
index 6c399480783d..c9e27e752c25 100644
--- a/drivers/scsi/st.c
+++ b/drivers/scsi/st.c
@@ -545,7 +545,7 @@ static int st_scsi_execute(struct st_request *SRpnt, const unsigned char *cmd,
 
 	req = blk_get_request(SRpnt->stp->device->request_queue,
 			data_direction == DMA_TO_DEVICE ?
-			REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, GFP_KERNEL);
+			REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, 0);
 	if (IS_ERR(req))
 		return DRIVER_ERROR << 24;
 	rq = scsi_req(req);
@@ -3878,7 +3878,7 @@ static struct st_buffer *new_tape_buffer(int need_dma, int max_sg)
 {
 	struct st_buffer *tb;
 
-	tb = kzalloc(sizeof(struct st_buffer), GFP_ATOMIC);
+	tb = kzalloc(sizeof(struct st_buffer), GFP_KERNEL);
 	if (!tb) {
 		printk(KERN_NOTICE "st: Can't allocate new tape buffer.\n");
 		return NULL;
@@ -3889,7 +3889,7 @@ static struct st_buffer *new_tape_buffer(int need_dma, int max_sg)
 	tb->buffer_size = 0;
 
 	tb->reserved_pages = kzalloc(max_sg * sizeof(struct page *),
-				     GFP_ATOMIC);
+				     GFP_KERNEL);
 	if (!tb->reserved_pages) {
 		kfree(tb);
 		return NULL;
@@ -4290,7 +4290,7 @@ static int st_probe(struct device *dev)
 		goto out_buffer_free;
 	}
 
-	tpnt = kzalloc(sizeof(struct scsi_tape), GFP_ATOMIC);
+	tpnt = kzalloc(sizeof(struct scsi_tape), GFP_KERNEL);
 	if (tpnt == NULL) {
 		sdev_printk(KERN_ERR, SDp,
 			    "st: Can't allocate device descriptor.\n");
diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c
index a2ec0bc9e9fa..33a4a4dad324 100644
--- a/drivers/scsi/storvsc_drv.c
+++ b/drivers/scsi/storvsc_drv.c
@@ -395,6 +395,12 @@ MODULE_PARM_DESC(storvsc_ringbuffer_size, "Ring buffer size (bytes)");
 
 module_param(storvsc_vcpus_per_sub_channel, int, S_IRUGO);
 MODULE_PARM_DESC(storvsc_vcpus_per_sub_channel, "Ratio of VCPUs to subchannels");
+
+static int ring_avail_percent_lowater = 10;
+module_param(ring_avail_percent_lowater, int, S_IRUGO);
+MODULE_PARM_DESC(ring_avail_percent_lowater,
+		"Select a channel if available ring size > this in percent");
+
 /*
  * Timeout in seconds for all devices managed by this driver.
  */
@@ -1241,7 +1247,7 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device,
 {
 	u16 slot = 0;
 	u16 hash_qnum;
-	struct cpumask alloced_mask;
+	const struct cpumask *node_mask;
 	int num_channels, tgt_cpu;
 
 	if (stor_device->num_sc == 0)
@@ -1257,10 +1263,13 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device,
 	 * III. Mapping is persistent.
 	 */
 
-	cpumask_and(&alloced_mask, &stor_device->alloced_cpus,
-		    cpumask_of_node(cpu_to_node(q_num)));
+	node_mask = cpumask_of_node(cpu_to_node(q_num));
 
-	num_channels = cpumask_weight(&alloced_mask);
+	num_channels = 0;
+	for_each_cpu(tgt_cpu, &stor_device->alloced_cpus) {
+		if (cpumask_test_cpu(tgt_cpu, node_mask))
+			num_channels++;
+	}
 	if (num_channels == 0)
 		return stor_device->device->channel;
 
@@ -1268,7 +1277,9 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device,
 	while (hash_qnum >= num_channels)
 		hash_qnum -= num_channels;
 
-	for_each_cpu(tgt_cpu, &alloced_mask) {
+	for_each_cpu(tgt_cpu, &stor_device->alloced_cpus) {
+		if (!cpumask_test_cpu(tgt_cpu, node_mask))
+			continue;
 		if (slot == hash_qnum)
 			break;
 		slot++;
@@ -1285,9 +1296,9 @@ static int storvsc_do_io(struct hv_device *device,
 {
 	struct storvsc_device *stor_device;
 	struct vstor_packet *vstor_packet;
-	struct vmbus_channel *outgoing_channel;
+	struct vmbus_channel *outgoing_channel, *channel;
 	int ret = 0;
-	struct cpumask alloced_mask;
+	const struct cpumask *node_mask;
 	int tgt_cpu;
 
 	vstor_packet = &request->vstor_packet;
@@ -1301,22 +1312,52 @@ static int storvsc_do_io(struct hv_device *device,
 	/*
 	 * Select an an appropriate channel to send the request out.
 	 */
-
 	if (stor_device->stor_chns[q_num] != NULL) {
 		outgoing_channel = stor_device->stor_chns[q_num];
-		if (outgoing_channel->target_cpu == smp_processor_id()) {
+		if (outgoing_channel->target_cpu == q_num) {
 			/*
 			 * Ideally, we want to pick a different channel if
 			 * available on the same NUMA node.
 			 */
-			cpumask_and(&alloced_mask, &stor_device->alloced_cpus,
-				    cpumask_of_node(cpu_to_node(q_num)));
-			for_each_cpu_wrap(tgt_cpu, &alloced_mask,
-					outgoing_channel->target_cpu + 1) {
-				if (tgt_cpu != outgoing_channel->target_cpu) {
-					outgoing_channel =
-					stor_device->stor_chns[tgt_cpu];
-					break;
+			node_mask = cpumask_of_node(cpu_to_node(q_num));
+			for_each_cpu_wrap(tgt_cpu,
+				 &stor_device->alloced_cpus, q_num + 1) {
+				if (!cpumask_test_cpu(tgt_cpu, node_mask))
+					continue;
+				if (tgt_cpu == q_num)
+					continue;
+				channel = stor_device->stor_chns[tgt_cpu];
+				if (hv_get_avail_to_write_percent(
+							&channel->outbound)
+						> ring_avail_percent_lowater) {
+					outgoing_channel = channel;
+					goto found_channel;
+				}
+			}
+
+			/*
+			 * All the other channels on the same NUMA node are
+			 * busy. Try to use the channel on the current CPU
+			 */
+			if (hv_get_avail_to_write_percent(
+						&outgoing_channel->outbound)
+					> ring_avail_percent_lowater)
+				goto found_channel;
+
+			/*
+			 * If we reach here, all the channels on the current
+			 * NUMA node are busy. Try to find a channel in
+			 * other NUMA nodes
+			 */
+			for_each_cpu(tgt_cpu, &stor_device->alloced_cpus) {
+				if (cpumask_test_cpu(tgt_cpu, node_mask))
+					continue;
+				channel = stor_device->stor_chns[tgt_cpu];
+				if (hv_get_avail_to_write_percent(
+							&channel->outbound)
+						> ring_avail_percent_lowater) {
+					outgoing_channel = channel;
+					goto found_channel;
 				}
 			}
 		}
@@ -1324,7 +1365,7 @@ static int storvsc_do_io(struct hv_device *device,
 		outgoing_channel = get_og_chn(stor_device, q_num);
 	}
 
-
+found_channel:
 	vstor_packet->flags |= REQUEST_COMPLETION_FLAG;
 
 	vstor_packet->vm_srb.length = (sizeof(struct vmscsi_request) -
@@ -1382,9 +1423,6 @@ static int storvsc_device_alloc(struct scsi_device *sdevice)
 
 static int storvsc_device_configure(struct scsi_device *sdevice)
 {
-
-	blk_queue_bounce_limit(sdevice->request_queue, BLK_BOUNCE_ANY);
-
 	blk_queue_rq_timeout(sdevice->request_queue, (storvsc_timeout * HZ));
 
 	/* Ensure there are no gaps in presented sgls */
@@ -1732,8 +1770,9 @@ static int storvsc_probe(struct hv_device *device,
 				(num_cpus - 1) / storvsc_vcpus_per_sub_channel;
 	}
 
-	scsi_driver.can_queue = (max_outstanding_req_per_channel *
-				 (max_sub_channels + 1));
+	scsi_driver.can_queue = max_outstanding_req_per_channel *
+				(max_sub_channels + 1) *
+				(100 - ring_avail_percent_lowater) / 100;
 
 	host = scsi_host_alloc(&scsi_driver,
 			       sizeof(struct hv_host_device));
diff --git a/drivers/scsi/ufs/ufs-qcom.c b/drivers/scsi/ufs/ufs-qcom.c
index 2b38db2eeafa..221820a7c78b 100644
--- a/drivers/scsi/ufs/ufs-qcom.c
+++ b/drivers/scsi/ufs/ufs-qcom.c
@@ -1098,7 +1098,7 @@ static void ufs_qcom_advertise_quirks(struct ufs_hba *hba)
 		hba->quirks |= UFSHCD_QUIRK_BROKEN_LCC;
 	}
 
-	if (host->hw_ver.major >= 0x2) {
+	if (host->hw_ver.major == 0x2) {
 		hba->quirks |= UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION;
 
 		if (!ufs_qcom_cap_qunipro(host))
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
index 00e79057f870..3a811c5f70ba 100644
--- a/drivers/scsi/ufs/ufshcd.c
+++ b/drivers/scsi/ufs/ufshcd.c
@@ -233,8 +233,6 @@ static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
 static irqreturn_t ufshcd_intr(int irq, void *__hba);
-static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
-		struct ufs_pa_layer_attr *desired_pwr_mode);
 static int ufshcd_change_power_mode(struct ufs_hba *hba,
 			     struct ufs_pa_layer_attr *pwr_mode);
 static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
@@ -266,6 +264,18 @@ static inline void ufshcd_disable_irq(struct ufs_hba *hba)
 	}
 }
 
+static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
+{
+	if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
+		scsi_unblock_requests(hba->host);
+}
+
+static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
+{
+	if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
+		scsi_block_requests(hba->host);
+}
+
 /* replace non-printable or non-ASCII characters with spaces */
 static inline void ufshcd_remove_non_printable(char *val)
 {
@@ -675,7 +685,24 @@ static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot)
  */
 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
 {
-	ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
+	if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
+		ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
+	else
+		ufshcd_writel(hba, ~(1 << pos),
+				REG_UTP_TRANSFER_REQ_LIST_CLEAR);
+}
+
+/**
+ * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
+ * @hba: per adapter instance
+ * @pos: position of the bit to be cleared
+ */
+static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
+{
+	if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
+		ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
+	else
+		ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 }
 
 /**
@@ -1091,12 +1118,12 @@ static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
 	 * make sure that there are no outstanding requests when
 	 * clock scaling is in progress
 	 */
-	scsi_block_requests(hba->host);
+	ufshcd_scsi_block_requests(hba);
 	down_write(&hba->clk_scaling_lock);
 	if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
 		ret = -EBUSY;
 		up_write(&hba->clk_scaling_lock);
-		scsi_unblock_requests(hba->host);
+		ufshcd_scsi_unblock_requests(hba);
 	}
 
 	return ret;
@@ -1105,7 +1132,7 @@ static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
 {
 	up_write(&hba->clk_scaling_lock);
-	scsi_unblock_requests(hba->host);
+	ufshcd_scsi_unblock_requests(hba);
 }
 
 /**
@@ -1200,16 +1227,13 @@ static int ufshcd_devfreq_target(struct device *dev,
 	struct ufs_hba *hba = dev_get_drvdata(dev);
 	ktime_t start;
 	bool scale_up, sched_clk_scaling_suspend_work = false;
+	struct list_head *clk_list = &hba->clk_list_head;
+	struct ufs_clk_info *clki;
 	unsigned long irq_flags;
 
 	if (!ufshcd_is_clkscaling_supported(hba))
 		return -EINVAL;
 
-	if ((*freq > 0) && (*freq < UINT_MAX)) {
-		dev_err(hba->dev, "%s: invalid freq = %lu\n", __func__, *freq);
-		return -EINVAL;
-	}
-
 	spin_lock_irqsave(hba->host->host_lock, irq_flags);
 	if (ufshcd_eh_in_progress(hba)) {
 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
@@ -1219,7 +1243,13 @@ static int ufshcd_devfreq_target(struct device *dev,
 	if (!hba->clk_scaling.active_reqs)
 		sched_clk_scaling_suspend_work = true;
 
-	scale_up = (*freq == UINT_MAX) ? true : false;
+	if (list_empty(clk_list)) {
+		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
+		goto out;
+	}
+
+	clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
+	scale_up = (*freq == clki->max_freq) ? true : false;
 	if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
 		ret = 0;
@@ -1287,6 +1317,55 @@ static struct devfreq_dev_profile ufs_devfreq_profile = {
 	.get_dev_status	= ufshcd_devfreq_get_dev_status,
 };
 
+static int ufshcd_devfreq_init(struct ufs_hba *hba)
+{
+	struct list_head *clk_list = &hba->clk_list_head;
+	struct ufs_clk_info *clki;
+	struct devfreq *devfreq;
+	int ret;
+
+	/* Skip devfreq if we don't have any clocks in the list */
+	if (list_empty(clk_list))
+		return 0;
+
+	clki = list_first_entry(clk_list, struct ufs_clk_info, list);
+	dev_pm_opp_add(hba->dev, clki->min_freq, 0);
+	dev_pm_opp_add(hba->dev, clki->max_freq, 0);
+
+	devfreq = devfreq_add_device(hba->dev,
+			&ufs_devfreq_profile,
+			DEVFREQ_GOV_SIMPLE_ONDEMAND,
+			NULL);
+	if (IS_ERR(devfreq)) {
+		ret = PTR_ERR(devfreq);
+		dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
+
+		dev_pm_opp_remove(hba->dev, clki->min_freq);
+		dev_pm_opp_remove(hba->dev, clki->max_freq);
+		return ret;
+	}
+
+	hba->devfreq = devfreq;
+
+	return 0;
+}
+
+static void ufshcd_devfreq_remove(struct ufs_hba *hba)
+{
+	struct list_head *clk_list = &hba->clk_list_head;
+	struct ufs_clk_info *clki;
+
+	if (!hba->devfreq)
+		return;
+
+	devfreq_remove_device(hba->devfreq);
+	hba->devfreq = NULL;
+
+	clki = list_first_entry(clk_list, struct ufs_clk_info, list);
+	dev_pm_opp_remove(hba->dev, clki->min_freq);
+	dev_pm_opp_remove(hba->dev, clki->max_freq);
+}
+
 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
 {
 	unsigned long flags;
@@ -1425,7 +1504,7 @@ static void ufshcd_ungate_work(struct work_struct *work)
 		hba->clk_gating.is_suspended = false;
 	}
 unblock_reqs:
-	scsi_unblock_requests(hba->host);
+	ufshcd_scsi_unblock_requests(hba);
 }
 
 /**
@@ -1481,11 +1560,12 @@ int ufshcd_hold(struct ufs_hba *hba, bool async)
 		 * work and to enable clocks.
 		 */
 	case CLKS_OFF:
-		scsi_block_requests(hba->host);
+		ufshcd_scsi_block_requests(hba);
 		hba->clk_gating.state = REQ_CLKS_ON;
 		trace_ufshcd_clk_gating(dev_name(hba->dev),
 					hba->clk_gating.state);
-		schedule_work(&hba->clk_gating.ungate_work);
+		queue_work(hba->clk_gating.clk_gating_workq,
+			   &hba->clk_gating.ungate_work);
 		/*
 		 * fall through to check if we should wait for this
 		 * work to be done or not.
@@ -1671,6 +1751,8 @@ static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
 
 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
 {
+	char wq_name[sizeof("ufs_clk_gating_00")];
+
 	if (!ufshcd_is_clkgating_allowed(hba))
 		return;
 
@@ -1678,6 +1760,11 @@ static void ufshcd_init_clk_gating(struct ufs_hba *hba)
 	INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
 	INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
 
+	snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
+		 hba->host->host_no);
+	hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
+							   WQ_MEM_RECLAIM);
+
 	hba->clk_gating.is_enabled = true;
 
 	hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
@@ -1705,6 +1792,7 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
 	device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
 	cancel_work_sync(&hba->clk_gating.ungate_work);
 	cancel_delayed_work_sync(&hba->clk_gating.gate_work);
+	destroy_workqueue(hba->clk_gating.clk_gating_workq);
 }
 
 /* Must be called with host lock acquired */
@@ -3383,6 +3471,52 @@ static int ufshcd_dme_link_startup(struct ufs_hba *hba)
 			"dme-link-startup: error code %d\n", ret);
 	return ret;
 }
+/**
+ * ufshcd_dme_reset - UIC command for DME_RESET
+ * @hba: per adapter instance
+ *
+ * DME_RESET command is issued in order to reset UniPro stack.
+ * This function now deal with cold reset.
+ *
+ * Returns 0 on success, non-zero value on failure
+ */
+static int ufshcd_dme_reset(struct ufs_hba *hba)
+{
+	struct uic_command uic_cmd = {0};
+	int ret;
+
+	uic_cmd.command = UIC_CMD_DME_RESET;
+
+	ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
+	if (ret)
+		dev_err(hba->dev,
+			"dme-reset: error code %d\n", ret);
+
+	return ret;
+}
+
+/**
+ * ufshcd_dme_enable - UIC command for DME_ENABLE
+ * @hba: per adapter instance
+ *
+ * DME_ENABLE command is issued in order to enable UniPro stack.
+ *
+ * Returns 0 on success, non-zero value on failure
+ */
+static int ufshcd_dme_enable(struct ufs_hba *hba)
+{
+	struct uic_command uic_cmd = {0};
+	int ret;
+
+	uic_cmd.command = UIC_CMD_DME_ENABLE;
+
+	ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
+	if (ret)
+		dev_err(hba->dev,
+			"dme-reset: error code %d\n", ret);
+
+	return ret;
+}
 
 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
 {
@@ -3906,7 +4040,7 @@ static int ufshcd_change_power_mode(struct ufs_hba *hba,
  * @hba: per-adapter instance
  * @desired_pwr_mode: desired power configuration
  */
-static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
+int ufshcd_config_pwr_mode(struct ufs_hba *hba,
 		struct ufs_pa_layer_attr *desired_pwr_mode)
 {
 	struct ufs_pa_layer_attr final_params = { 0 };
@@ -3924,6 +4058,7 @@ static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
 
 /**
  * ufshcd_complete_dev_init() - checks device readiness
@@ -4041,7 +4176,7 @@ static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
 }
 
 /**
- * ufshcd_hba_enable - initialize the controller
+ * ufshcd_hba_execute_hce - initialize the controller
  * @hba: per adapter instance
  *
  * The controller resets itself and controller firmware initialization
@@ -4050,7 +4185,7 @@ static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
  *
  * Returns 0 on success, non-zero value on failure
  */
-static int ufshcd_hba_enable(struct ufs_hba *hba)
+static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
 {
 	int retry;
 
@@ -4105,6 +4240,31 @@ static int ufshcd_hba_enable(struct ufs_hba *hba)
 	return 0;
 }
 
+static int ufshcd_hba_enable(struct ufs_hba *hba)
+{
+	int ret;
+
+	if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
+		ufshcd_set_link_off(hba);
+		ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
+
+		/* enable UIC related interrupts */
+		ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
+		ret = ufshcd_dme_reset(hba);
+		if (!ret) {
+			ret = ufshcd_dme_enable(hba);
+			if (!ret)
+				ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
+			if (ret)
+				dev_err(hba->dev,
+					"Host controller enable failed with non-hce\n");
+		}
+	} else {
+		ret = ufshcd_hba_execute_hce(hba);
+	}
+
+	return ret;
+}
 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
 {
 	int tx_lanes, i, err = 0;
@@ -4678,7 +4838,8 @@ static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
 	 * false interrupt if device completes another request after resetting
 	 * aggregation and before reading the DB.
 	 */
-	if (ufshcd_is_intr_aggr_allowed(hba))
+	if (ufshcd_is_intr_aggr_allowed(hba) &&
+	    !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
 		ufshcd_reset_intr_aggr(hba);
 
 	tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
@@ -4969,6 +5130,7 @@ static void ufshcd_exception_event_handler(struct work_struct *work)
 	hba = container_of(work, struct ufs_hba, eeh_work);
 
 	pm_runtime_get_sync(hba->dev);
+	scsi_block_requests(hba->host);
 	err = ufshcd_get_ee_status(hba, &status);
 	if (err) {
 		dev_err(hba->dev, "%s: failed to get exception status %d\n",
@@ -4982,6 +5144,7 @@ static void ufshcd_exception_event_handler(struct work_struct *work)
 		ufshcd_bkops_exception_event_handler(hba);
 
 out:
+	scsi_unblock_requests(hba->host);
 	pm_runtime_put_sync(hba->dev);
 	return;
 }
@@ -5192,7 +5355,7 @@ static void ufshcd_err_handler(struct work_struct *work)
 
 out:
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
-	scsi_unblock_requests(hba->host);
+	ufshcd_scsi_unblock_requests(hba);
 	ufshcd_release(hba);
 	pm_runtime_put_sync(hba->dev);
 }
@@ -5294,7 +5457,7 @@ static void ufshcd_check_errors(struct ufs_hba *hba)
 		/* handle fatal errors only when link is functional */
 		if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
 			/* block commands from scsi mid-layer */
-			scsi_block_requests(hba->host);
+			ufshcd_scsi_block_requests(hba);
 
 			hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED;
 
@@ -5371,19 +5534,30 @@ static irqreturn_t ufshcd_intr(int irq, void *__hba)
 	u32 intr_status, enabled_intr_status;
 	irqreturn_t retval = IRQ_NONE;
 	struct ufs_hba *hba = __hba;
+	int retries = hba->nutrs;
 
 	spin_lock(hba->host->host_lock);
 	intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
-	enabled_intr_status =
-		intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
 
-	if (intr_status)
-		ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
+	/*
+	 * There could be max of hba->nutrs reqs in flight and in worst case
+	 * if the reqs get finished 1 by 1 after the interrupt status is
+	 * read, make sure we handle them by checking the interrupt status
+	 * again in a loop until we process all of the reqs before returning.
+	 */
+	do {
+		enabled_intr_status =
+			intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
+		if (intr_status)
+			ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
+		if (enabled_intr_status) {
+			ufshcd_sl_intr(hba, enabled_intr_status);
+			retval = IRQ_HANDLED;
+		}
+
+		intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
+	} while (intr_status && --retries);
 
-	if (enabled_intr_status) {
-		ufshcd_sl_intr(hba, enabled_intr_status);
-		retval = IRQ_HANDLED;
-	}
 	spin_unlock(hba->host->host_lock);
 	return retval;
 }
@@ -5398,7 +5572,7 @@ static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
 		goto out;
 
 	spin_lock_irqsave(hba->host->host_lock, flags);
-	ufshcd_writel(hba, ~(1 << tag), REG_UTP_TASK_REQ_LIST_CLEAR);
+	ufshcd_utmrl_clear(hba, tag);
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
 
 	/* poll for max. 1 sec to clear door bell register by h/w */
@@ -5958,14 +6132,18 @@ static void ufshcd_init_icc_levels(struct ufs_hba *hba)
 {
 	int ret;
 	int buff_len = hba->desc_size.pwr_desc;
-	u8 desc_buf[hba->desc_size.pwr_desc];
+	u8 *desc_buf;
+
+	desc_buf = kmalloc(buff_len, GFP_KERNEL);
+	if (!desc_buf)
+		return;
 
 	ret = ufshcd_read_power_desc(hba, desc_buf, buff_len);
 	if (ret) {
 		dev_err(hba->dev,
 			"%s: Failed reading power descriptor.len = %d ret = %d",
 			__func__, buff_len, ret);
-		return;
+		goto out;
 	}
 
 	hba->init_prefetch_data.icc_level =
@@ -5983,6 +6161,8 @@ static void ufshcd_init_icc_levels(struct ufs_hba *hba)
 			"%s: Failed configuring bActiveICCLevel = %d ret = %d",
 			__func__, hba->init_prefetch_data.icc_level , ret);
 
+out:
+	kfree(desc_buf);
 }
 
 /**
@@ -6052,9 +6232,17 @@ static int ufs_get_device_desc(struct ufs_hba *hba,
 			       struct ufs_dev_desc *dev_desc)
 {
 	int err;
+	size_t buff_len;
 	u8 model_index;
-	u8 str_desc_buf[QUERY_DESC_MAX_SIZE + 1] = {0};
-	u8 desc_buf[hba->desc_size.dev_desc];
+	u8 *desc_buf;
+
+	buff_len = max_t(size_t, hba->desc_size.dev_desc,
+			 QUERY_DESC_MAX_SIZE + 1);
+	desc_buf = kmalloc(buff_len, GFP_KERNEL);
+	if (!desc_buf) {
+		err = -ENOMEM;
+		goto out;
+	}
 
 	err = ufshcd_read_device_desc(hba, desc_buf, hba->desc_size.dev_desc);
 	if (err) {
@@ -6072,7 +6260,10 @@ static int ufs_get_device_desc(struct ufs_hba *hba,
 
 	model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
 
-	err = ufshcd_read_string_desc(hba, model_index, str_desc_buf,
+	/* Zero-pad entire buffer for string termination. */
+	memset(desc_buf, 0, buff_len);
+
+	err = ufshcd_read_string_desc(hba, model_index, desc_buf,
 				      QUERY_DESC_MAX_SIZE, true/*ASCII*/);
 	if (err) {
 		dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
@@ -6080,15 +6271,16 @@ static int ufs_get_device_desc(struct ufs_hba *hba,
 		goto out;
 	}
 
-	str_desc_buf[QUERY_DESC_MAX_SIZE] = '\0';
-	strlcpy(dev_desc->model, (str_desc_buf + QUERY_DESC_HDR_SIZE),
-		min_t(u8, str_desc_buf[QUERY_DESC_LENGTH_OFFSET],
+	desc_buf[QUERY_DESC_MAX_SIZE] = '\0';
+	strlcpy(dev_desc->model, (desc_buf + QUERY_DESC_HDR_SIZE),
+		min_t(u8, desc_buf[QUERY_DESC_LENGTH_OFFSET],
 		      MAX_MODEL_LEN));
 
 	/* Null terminate the model string */
 	dev_desc->model[MAX_MODEL_LEN] = '\0';
 
 out:
+	kfree(desc_buf);
 	return err;
 }
 
@@ -6439,16 +6631,9 @@ static int ufshcd_probe_hba(struct ufs_hba *hba)
 				sizeof(struct ufs_pa_layer_attr));
 			hba->clk_scaling.saved_pwr_info.is_valid = true;
 			if (!hba->devfreq) {
-				hba->devfreq = devm_devfreq_add_device(hba->dev,
-							&ufs_devfreq_profile,
-							"simple_ondemand",
-							NULL);
-				if (IS_ERR(hba->devfreq)) {
-					ret = PTR_ERR(hba->devfreq);
-					dev_err(hba->dev, "Unable to register with devfreq %d\n",
-							ret);
+				ret = ufshcd_devfreq_init(hba);
+				if (ret)
 					goto out;
-				}
 			}
 			hba->clk_scaling.is_allowed = true;
 		}
@@ -6497,12 +6682,12 @@ static enum blk_eh_timer_return ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
 	bool found = false;
 
 	if (!scmd || !scmd->device || !scmd->device->host)
-		return BLK_EH_NOT_HANDLED;
+		return BLK_EH_DONE;
 
 	host = scmd->device->host;
 	hba = shost_priv(host);
 	if (!hba)
-		return BLK_EH_NOT_HANDLED;
+		return BLK_EH_DONE;
 
 	spin_lock_irqsave(host->host_lock, flags);
 
@@ -6520,7 +6705,7 @@ static enum blk_eh_timer_return ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
 	 * SCSI command was not actually dispatched to UFS driver, otherwise
 	 * let SCSI layer handle the error as usual.
 	 */
-	return found ? BLK_EH_NOT_HANDLED : BLK_EH_RESET_TIMER;
+	return found ? BLK_EH_DONE : BLK_EH_RESET_TIMER;
 }
 
 static const struct attribute_group *ufshcd_driver_groups[] = {
@@ -6799,9 +6984,16 @@ static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
 	if (list_empty(head))
 		goto out;
 
-	ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
-	if (ret)
-		return ret;
+	/*
+	 * vendor specific setup_clocks ops may depend on clocks managed by
+	 * this standard driver hence call the vendor specific setup_clocks
+	 * before disabling the clocks managed here.
+	 */
+	if (!on) {
+		ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
+		if (ret)
+			return ret;
+	}
 
 	list_for_each_entry(clki, head, list) {
 		if (!IS_ERR_OR_NULL(clki->clk)) {
@@ -6825,9 +7017,16 @@ static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
 		}
 	}
 
-	ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
-	if (ret)
-		return ret;
+	/*
+	 * vendor specific setup_clocks ops may depend on clocks managed by
+	 * this standard driver hence call the vendor specific setup_clocks
+	 * after enabling the clocks managed here.
+	 */
+	if (on) {
+		ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
+		if (ret)
+			return ret;
+	}
 
 out:
 	if (ret) {
@@ -6992,6 +7191,7 @@ static void ufshcd_hba_exit(struct ufs_hba *hba)
 			if (hba->devfreq)
 				ufshcd_suspend_clkscaling(hba);
 			destroy_workqueue(hba->clk_scaling.workq);
+			ufshcd_devfreq_remove(hba);
 		}
 		ufshcd_setup_clocks(hba, false);
 		ufshcd_setup_hba_vreg(hba, false);
@@ -7904,7 +8104,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
 
 	/* Hold auto suspend until async scan completes */
 	pm_runtime_get_sync(dev);
-
+	atomic_set(&hba->scsi_block_reqs_cnt, 0);
 	/*
 	 * We are assuming that device wasn't put in sleep/power-down
 	 * state exclusively during the boot stage before kernel.
diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h
index 8110dcd04d22..f51758f1e5cc 100644
--- a/drivers/scsi/ufs/ufshcd.h
+++ b/drivers/scsi/ufs/ufshcd.h
@@ -362,6 +362,7 @@ struct ufs_clk_gating {
 	struct device_attribute enable_attr;
 	bool is_enabled;
 	int active_reqs;
+	struct workqueue_struct *clk_gating_workq;
 };
 
 struct ufs_saved_pwr_info {
@@ -499,6 +500,7 @@ struct ufs_stats {
  * @urgent_bkops_lvl: keeps track of urgent bkops level for device
  * @is_urgent_bkops_lvl_checked: keeps track if the urgent bkops level for
  *  device is known or not.
+ * @scsi_block_reqs_cnt: reference counting for scsi block requests
  */
 struct ufs_hba {
 	void __iomem *mmio_base;
@@ -595,6 +597,22 @@ struct ufs_hba {
 	 */
 	#define UFSHCD_QUIRK_PRDT_BYTE_GRAN			0x80
 
+	/*
+	 * Clear handling for transfer/task request list is just opposite.
+	 */
+	#define UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR		0x100
+
+	/*
+	 * This quirk needs to be enabled if host controller doesn't allow
+	 * that the interrupt aggregation timer and counter are reset by s/w.
+	 */
+	#define UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR		0x200
+
+	/*
+	 * This quirks needs to be enabled if host controller cannot be
+	 * enabled via HCE register.
+	 */
+	#define UFSHCI_QUIRK_BROKEN_HCE				0x400
 	unsigned int quirks;	/* Deviations from standard UFSHCI spec. */
 
 	/* Device deviations from standard UFS device spec. */
@@ -683,6 +701,7 @@ struct ufs_hba {
 
 	struct rw_semaphore clk_scaling_lock;
 	struct ufs_desc_size desc_size;
+	atomic_t scsi_block_reqs_cnt;
 };
 
 /* Returns true if clocks can be gated. Otherwise false */
@@ -789,6 +808,8 @@ extern int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
 			       u8 attr_set, u32 mib_val, u8 peer);
 extern int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
 			       u32 *mib_val, u8 peer);
+extern int ufshcd_config_pwr_mode(struct ufs_hba *hba,
+			struct ufs_pa_layer_attr *desired_pwr_mode);
 
 /* UIC command interfaces for DME primitives */
 #define DME_LOCAL	0
diff --git a/drivers/scsi/wd719x.c b/drivers/scsi/wd719x.c
index 2ba2b7b47f41..974bfb3f30f4 100644
--- a/drivers/scsi/wd719x.c
+++ b/drivers/scsi/wd719x.c
@@ -978,18 +978,7 @@ static struct pci_driver wd719x_pci_driver = {
 	.remove =	wd719x_pci_remove,
 };
 
-static int __init wd719x_init(void)
-{
-	return pci_register_driver(&wd719x_pci_driver);
-}
-
-static void __exit wd719x_exit(void)
-{
-	pci_unregister_driver(&wd719x_pci_driver);
-}
-
-module_init(wd719x_init);
-module_exit(wd719x_exit);
+module_pci_driver(wd719x_pci_driver);
 
 MODULE_DESCRIPTION("Western Digital WD7193/7197/7296 SCSI driver");
 MODULE_AUTHOR("Ondrej Zary, Aaron Dewell, Juergen Gaertner");
diff --git a/drivers/scsi/zorro_esp.c b/drivers/scsi/zorro_esp.c
new file mode 100644
index 000000000000..bb70882e6b56
--- /dev/null
+++ b/drivers/scsi/zorro_esp.c
@@ -0,0 +1,1172 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESP front-end for Amiga ZORRO SCSI systems.
+ *
+ * Copyright (C) 1996 Jesper Skov (jskov@cygnus.co.uk)
+ *
+ * Copyright (C) 2011,2018 Michael Schmitz (schmitz@debian.org) for
+ *               migration to ESP SCSI core
+ *
+ * Copyright (C) 2013 Tuomas Vainikka (tuomas.vainikka@aalto.fi) for
+ *               Blizzard 1230 DMA and probe function fixes
+ *
+ * Copyright (C) 2017 Finn Thain for PIO code from Mac ESP driver adapted here
+ */
+/*
+ * ZORRO bus code from:
+ */
+/*
+ * Detection routine for the NCR53c710 based Amiga SCSI Controllers for Linux.
+ *		Amiga MacroSystemUS WarpEngine SCSI controller.
+ *		Amiga Technologies/DKB A4091 SCSI controller.
+ *
+ * Written 1997 by Alan Hourihane <alanh@fairlite.demon.co.uk>
+ * plus modifications of the 53c7xx.c driver to support the Amiga.
+ *
+ * Rewritten to use 53c700.c by Kars de Jong <jongk@linux-m68k.org>
+ */
+
+#define pr_fmt(fmt)        KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/dma-mapping.h>
+#include <linux/scatterlist.h>
+#include <linux/delay.h>
+#include <linux/zorro.h>
+#include <linux/slab.h>
+
+#include <asm/page.h>
+#include <asm/pgtable.h>
+#include <asm/cacheflush.h>
+#include <asm/amigahw.h>
+#include <asm/amigaints.h>
+
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_transport_spi.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_tcq.h>
+
+#include "esp_scsi.h"
+
+MODULE_AUTHOR("Michael Schmitz <schmitz@debian.org>");
+MODULE_DESCRIPTION("Amiga Zorro NCR5C9x (ESP) driver");
+MODULE_LICENSE("GPL");
+
+/* per-board register layout definitions */
+
+/* Blizzard 1230 DMA interface */
+
+struct blz1230_dma_registers {
+	unsigned char dma_addr;		/* DMA address      [0x0000] */
+	unsigned char dmapad2[0x7fff];
+	unsigned char dma_latch;	/* DMA latch        [0x8000] */
+};
+
+/* Blizzard 1230II DMA interface */
+
+struct blz1230II_dma_registers {
+	unsigned char dma_addr;		/* DMA address      [0x0000] */
+	unsigned char dmapad2[0xf];
+	unsigned char dma_latch;	/* DMA latch        [0x0010] */
+};
+
+/* Blizzard 2060 DMA interface */
+
+struct blz2060_dma_registers {
+	unsigned char dma_led_ctrl;	/* DMA led control   [0x000] */
+	unsigned char dmapad1[0x0f];
+	unsigned char dma_addr0;	/* DMA address (MSB) [0x010] */
+	unsigned char dmapad2[0x03];
+	unsigned char dma_addr1;	/* DMA address       [0x014] */
+	unsigned char dmapad3[0x03];
+	unsigned char dma_addr2;	/* DMA address       [0x018] */
+	unsigned char dmapad4[0x03];
+	unsigned char dma_addr3;	/* DMA address (LSB) [0x01c] */
+};
+
+/* DMA control bits */
+#define DMA_WRITE 0x80000000
+
+/* Cyberstorm DMA interface */
+
+struct cyber_dma_registers {
+	unsigned char dma_addr0;	/* DMA address (MSB) [0x000] */
+	unsigned char dmapad1[1];
+	unsigned char dma_addr1;	/* DMA address       [0x002] */
+	unsigned char dmapad2[1];
+	unsigned char dma_addr2;	/* DMA address       [0x004] */
+	unsigned char dmapad3[1];
+	unsigned char dma_addr3;	/* DMA address (LSB) [0x006] */
+	unsigned char dmapad4[0x3fb];
+	unsigned char cond_reg;		/* DMA cond    (ro)  [0x402] */
+#define ctrl_reg  cond_reg		/* DMA control (wo)  [0x402] */
+};
+
+/* DMA control bits */
+#define CYBER_DMA_WRITE  0x40	/* DMA direction. 1 = write */
+#define CYBER_DMA_Z3     0x20	/* 16 (Z2) or 32 (CHIP/Z3) bit DMA transfer */
+
+/* DMA status bits */
+#define CYBER_DMA_HNDL_INTR 0x80	/* DMA IRQ pending? */
+
+/* The CyberStorm II DMA interface */
+struct cyberII_dma_registers {
+	unsigned char cond_reg;		/* DMA cond    (ro)  [0x000] */
+#define ctrl_reg  cond_reg		/* DMA control (wo)  [0x000] */
+	unsigned char dmapad4[0x3f];
+	unsigned char dma_addr0;	/* DMA address (MSB) [0x040] */
+	unsigned char dmapad1[3];
+	unsigned char dma_addr1;	/* DMA address       [0x044] */
+	unsigned char dmapad2[3];
+	unsigned char dma_addr2;	/* DMA address       [0x048] */
+	unsigned char dmapad3[3];
+	unsigned char dma_addr3;	/* DMA address (LSB) [0x04c] */
+};
+
+/* Fastlane DMA interface */
+
+struct fastlane_dma_registers {
+	unsigned char cond_reg;		/* DMA status  (ro) [0x0000] */
+#define ctrl_reg  cond_reg		/* DMA control (wo) [0x0000] */
+	char dmapad1[0x3f];
+	unsigned char clear_strobe;	/* DMA clear   (wo) [0x0040] */
+};
+
+/*
+ * The controller registers can be found in the Z2 config area at these
+ * offsets:
+ */
+#define FASTLANE_ESP_ADDR	0x1000001
+
+/* DMA status bits */
+#define FASTLANE_DMA_MINT	0x80
+#define FASTLANE_DMA_IACT	0x40
+#define FASTLANE_DMA_CREQ	0x20
+
+/* DMA control bits */
+#define FASTLANE_DMA_FCODE	0xa0
+#define FASTLANE_DMA_MASK	0xf3
+#define FASTLANE_DMA_WRITE	0x08	/* 1 = write */
+#define FASTLANE_DMA_ENABLE	0x04	/* Enable DMA */
+#define FASTLANE_DMA_EDI	0x02	/* Enable DMA IRQ ? */
+#define FASTLANE_DMA_ESI	0x01	/* Enable SCSI IRQ */
+
+/*
+ * private data used for driver
+ */
+struct zorro_esp_priv {
+	struct esp *esp;		/* our ESP instance - for Scsi_host* */
+	void __iomem *board_base;	/* virtual address (Zorro III board) */
+	int error;			/* PIO error flag */
+	int zorro3;			/* board is Zorro III */
+	unsigned char ctrl_data;	/* shadow copy of ctrl_reg */
+};
+
+/*
+ * On all implementations except for the Oktagon, padding between ESP
+ * registers is three bytes.
+ * On Oktagon, it is one byte - use a different accessor there.
+ *
+ * Oktagon needs PDMA - currently unsupported!
+ */
+
+static void zorro_esp_write8(struct esp *esp, u8 val, unsigned long reg)
+{
+	writeb(val, esp->regs + (reg * 4UL));
+}
+
+static u8 zorro_esp_read8(struct esp *esp, unsigned long reg)
+{
+	return readb(esp->regs + (reg * 4UL));
+}
+
+static dma_addr_t zorro_esp_map_single(struct esp *esp, void *buf,
+				      size_t sz, int dir)
+{
+	return dma_map_single(esp->dev, buf, sz, dir);
+}
+
+static int zorro_esp_map_sg(struct esp *esp, struct scatterlist *sg,
+				  int num_sg, int dir)
+{
+	return dma_map_sg(esp->dev, sg, num_sg, dir);
+}
+
+static void zorro_esp_unmap_single(struct esp *esp, dma_addr_t addr,
+				  size_t sz, int dir)
+{
+	dma_unmap_single(esp->dev, addr, sz, dir);
+}
+
+static void zorro_esp_unmap_sg(struct esp *esp, struct scatterlist *sg,
+			      int num_sg, int dir)
+{
+	dma_unmap_sg(esp->dev, sg, num_sg, dir);
+}
+
+static int zorro_esp_irq_pending(struct esp *esp)
+{
+	/* check ESP status register; DMA has no status reg. */
+	if (zorro_esp_read8(esp, ESP_STATUS) & ESP_STAT_INTR)
+		return 1;
+
+	return 0;
+}
+
+static int cyber_esp_irq_pending(struct esp *esp)
+{
+	struct cyber_dma_registers __iomem *dregs = esp->dma_regs;
+	unsigned char dma_status = readb(&dregs->cond_reg);
+
+	/* It's important to check the DMA IRQ bit in the correct way! */
+	return ((zorro_esp_read8(esp, ESP_STATUS) & ESP_STAT_INTR) &&
+		(dma_status & CYBER_DMA_HNDL_INTR));
+}
+
+static int fastlane_esp_irq_pending(struct esp *esp)
+{
+	struct fastlane_dma_registers __iomem *dregs = esp->dma_regs;
+	unsigned char dma_status;
+
+	dma_status = readb(&dregs->cond_reg);
+
+	if (dma_status & FASTLANE_DMA_IACT)
+		return 0;	/* not our IRQ */
+
+	/* Return non-zero if ESP requested IRQ */
+	return (
+	   (dma_status & FASTLANE_DMA_CREQ) &&
+	   (!(dma_status & FASTLANE_DMA_MINT)) &&
+	   (zorro_esp_read8(esp, ESP_STATUS) & ESP_STAT_INTR));
+}
+
+static u32 zorro_esp_dma_length_limit(struct esp *esp, u32 dma_addr,
+					u32 dma_len)
+{
+	return dma_len > 0xFFFFFF ? 0xFFFFFF : dma_len;
+}
+
+static void zorro_esp_reset_dma(struct esp *esp)
+{
+	/* nothing to do here */
+}
+
+static void zorro_esp_dma_drain(struct esp *esp)
+{
+	/* nothing to do here */
+}
+
+static void zorro_esp_dma_invalidate(struct esp *esp)
+{
+	/* nothing to do here */
+}
+
+static void fastlane_esp_dma_invalidate(struct esp *esp)
+{
+	struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+	struct fastlane_dma_registers __iomem *dregs = esp->dma_regs;
+	unsigned char *ctrl_data = &zep->ctrl_data;
+
+	*ctrl_data = (*ctrl_data & FASTLANE_DMA_MASK);
+	writeb(0, &dregs->clear_strobe);
+	z_writel(0, zep->board_base);
+}
+
+/*
+ * Programmed IO routines follow.
+ */
+
+static inline unsigned int zorro_esp_wait_for_fifo(struct esp *esp)
+{
+	int i = 500000;
+
+	do {
+		unsigned int fbytes = zorro_esp_read8(esp, ESP_FFLAGS)
+							& ESP_FF_FBYTES;
+
+		if (fbytes)
+			return fbytes;
+
+		udelay(2);
+	} while (--i);
+
+	pr_err("FIFO is empty (sreg %02x)\n",
+	       zorro_esp_read8(esp, ESP_STATUS));
+	return 0;
+}
+
+static inline int zorro_esp_wait_for_intr(struct esp *esp)
+{
+	struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+	int i = 500000;
+
+	do {
+		esp->sreg = zorro_esp_read8(esp, ESP_STATUS);
+		if (esp->sreg & ESP_STAT_INTR)
+			return 0;
+
+		udelay(2);
+	} while (--i);
+
+	pr_err("IRQ timeout (sreg %02x)\n", esp->sreg);
+	zep->error = 1;
+	return 1;
+}
+
+/*
+ * PIO macros as used in mac_esp.c.
+ * Note that addr and fifo arguments are local-scope variables declared
+ * in zorro_esp_send_pio_cmd(), the macros are only used in that function,
+ * and addr and fifo are referenced in each use of the macros so there
+ * is no need to pass them as macro parameters.
+ */
+#define ZORRO_ESP_PIO_LOOP(operands, reg1) \
+	asm volatile ( \
+	     "1:     moveb " operands "\n" \
+	     "       subqw #1,%1       \n" \
+	     "       jbne 1b           \n" \
+	     : "+a" (addr), "+r" (reg1) \
+	     : "a" (fifo));
+
+#define ZORRO_ESP_PIO_FILL(operands, reg1) \
+	asm volatile ( \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       moveb " operands "\n" \
+	     "       subqw #8,%1       \n" \
+	     "       subqw #8,%1       \n" \
+	     : "+a" (addr), "+r" (reg1) \
+	     : "a" (fifo));
+
+#define ZORRO_ESP_FIFO_SIZE 16
+
+static void zorro_esp_send_pio_cmd(struct esp *esp, u32 addr, u32 esp_count,
+				 u32 dma_count, int write, u8 cmd)
+{
+	struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+	u8 __iomem *fifo = esp->regs + ESP_FDATA * 16;
+	u8 phase = esp->sreg & ESP_STAT_PMASK;
+
+	cmd &= ~ESP_CMD_DMA;
+
+	if (write) {
+		u8 *dst = (u8 *)addr;
+		u8 mask = ~(phase == ESP_MIP ? ESP_INTR_FDONE : ESP_INTR_BSERV);
+
+		scsi_esp_cmd(esp, cmd);
+
+		while (1) {
+			if (!zorro_esp_wait_for_fifo(esp))
+				break;
+
+			*dst++ = zorro_esp_read8(esp, ESP_FDATA);
+			--esp_count;
+
+			if (!esp_count)
+				break;
+
+			if (zorro_esp_wait_for_intr(esp))
+				break;
+
+			if ((esp->sreg & ESP_STAT_PMASK) != phase)
+				break;
+
+			esp->ireg = zorro_esp_read8(esp, ESP_INTRPT);
+			if (esp->ireg & mask) {
+				zep->error = 1;
+				break;
+			}
+
+			if (phase == ESP_MIP)
+				scsi_esp_cmd(esp, ESP_CMD_MOK);
+
+			scsi_esp_cmd(esp, ESP_CMD_TI);
+		}
+	} else {	/* unused, as long as we only handle MIP here */
+		scsi_esp_cmd(esp, ESP_CMD_FLUSH);
+
+		if (esp_count >= ZORRO_ESP_FIFO_SIZE)
+			ZORRO_ESP_PIO_FILL("%0@+,%2@", esp_count)
+		else
+			ZORRO_ESP_PIO_LOOP("%0@+,%2@", esp_count)
+
+		scsi_esp_cmd(esp, cmd);
+
+		while (esp_count) {
+			unsigned int n;
+
+			if (zorro_esp_wait_for_intr(esp))
+				break;
+
+			if ((esp->sreg & ESP_STAT_PMASK) != phase)
+				break;
+
+			esp->ireg = zorro_esp_read8(esp, ESP_INTRPT);
+			if (esp->ireg & ~ESP_INTR_BSERV) {
+				zep->error = 1;
+				break;
+			}
+
+			n = ZORRO_ESP_FIFO_SIZE -
+			    (zorro_esp_read8(esp, ESP_FFLAGS) & ESP_FF_FBYTES);
+			if (n > esp_count)
+				n = esp_count;
+
+			if (n == ZORRO_ESP_FIFO_SIZE)
+				ZORRO_ESP_PIO_FILL("%0@+,%2@", esp_count)
+			else {
+				esp_count -= n;
+				ZORRO_ESP_PIO_LOOP("%0@+,%2@", n)
+			}
+
+			scsi_esp_cmd(esp, ESP_CMD_TI);
+		}
+	}
+}
+
+/* Blizzard 1230/60 SCSI-IV DMA */
+
+static void zorro_esp_send_blz1230_dma_cmd(struct esp *esp, u32 addr,
+			u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+	struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+	struct blz1230_dma_registers __iomem *dregs = esp->dma_regs;
+	u8 phase = esp->sreg & ESP_STAT_PMASK;
+
+	zep->error = 0;
+	/*
+	 * Use PIO if transferring message bytes to esp->command_block_dma.
+	 * PIO requires a virtual address, so substitute esp->command_block
+	 * for addr.
+	 */
+	if (phase == ESP_MIP && addr == esp->command_block_dma) {
+		zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+					esp_count, dma_count, write, cmd);
+		return;
+	}
+
+	if (write)
+		/* DMA receive */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_FROM_DEVICE);
+	else
+		/* DMA send */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_TO_DEVICE);
+
+	addr >>= 1;
+	if (write)
+		addr &= ~(DMA_WRITE);
+	else
+		addr |= DMA_WRITE;
+
+	writeb((addr >> 24) & 0xff, &dregs->dma_latch);
+	writeb((addr >> 24) & 0xff, &dregs->dma_addr);
+	writeb((addr >> 16) & 0xff, &dregs->dma_addr);
+	writeb((addr >>  8) & 0xff, &dregs->dma_addr);
+	writeb(addr & 0xff, &dregs->dma_addr);
+
+	scsi_esp_cmd(esp, ESP_CMD_DMA);
+	zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+	zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+	zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+	scsi_esp_cmd(esp, cmd);
+}
+
+/* Blizzard 1230-II DMA */
+
+static void zorro_esp_send_blz1230II_dma_cmd(struct esp *esp, u32 addr,
+			u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+	struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+	struct blz1230II_dma_registers __iomem *dregs = esp->dma_regs;
+	u8 phase = esp->sreg & ESP_STAT_PMASK;
+
+	zep->error = 0;
+	/* Use PIO if transferring message bytes to esp->command_block_dma */
+	if (phase == ESP_MIP && addr == esp->command_block_dma) {
+		zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+					esp_count, dma_count, write, cmd);
+		return;
+	}
+
+	if (write)
+		/* DMA receive */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_FROM_DEVICE);
+	else
+		/* DMA send */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_TO_DEVICE);
+
+	addr >>= 1;
+	if (write)
+		addr &= ~(DMA_WRITE);
+	else
+		addr |= DMA_WRITE;
+
+	writeb((addr >> 24) & 0xff, &dregs->dma_latch);
+	writeb((addr >> 16) & 0xff, &dregs->dma_addr);
+	writeb((addr >>  8) & 0xff, &dregs->dma_addr);
+	writeb(addr & 0xff, &dregs->dma_addr);
+
+	scsi_esp_cmd(esp, ESP_CMD_DMA);
+	zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+	zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+	zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+	scsi_esp_cmd(esp, cmd);
+}
+
+/* Blizzard 2060 DMA */
+
+static void zorro_esp_send_blz2060_dma_cmd(struct esp *esp, u32 addr,
+			u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+	struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+	struct blz2060_dma_registers __iomem *dregs = esp->dma_regs;
+	u8 phase = esp->sreg & ESP_STAT_PMASK;
+
+	zep->error = 0;
+	/* Use PIO if transferring message bytes to esp->command_block_dma */
+	if (phase == ESP_MIP && addr == esp->command_block_dma) {
+		zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+					esp_count, dma_count, write, cmd);
+		return;
+	}
+
+	if (write)
+		/* DMA receive */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_FROM_DEVICE);
+	else
+		/* DMA send */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_TO_DEVICE);
+
+	addr >>= 1;
+	if (write)
+		addr &= ~(DMA_WRITE);
+	else
+		addr |= DMA_WRITE;
+
+	writeb(addr & 0xff, &dregs->dma_addr3);
+	writeb((addr >>  8) & 0xff, &dregs->dma_addr2);
+	writeb((addr >> 16) & 0xff, &dregs->dma_addr1);
+	writeb((addr >> 24) & 0xff, &dregs->dma_addr0);
+
+	scsi_esp_cmd(esp, ESP_CMD_DMA);
+	zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+	zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+	zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+	scsi_esp_cmd(esp, cmd);
+}
+
+/* Cyberstorm I DMA */
+
+static void zorro_esp_send_cyber_dma_cmd(struct esp *esp, u32 addr,
+			u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+	struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+	struct cyber_dma_registers __iomem *dregs = esp->dma_regs;
+	u8 phase = esp->sreg & ESP_STAT_PMASK;
+	unsigned char *ctrl_data = &zep->ctrl_data;
+
+	zep->error = 0;
+	/* Use PIO if transferring message bytes to esp->command_block_dma */
+	if (phase == ESP_MIP && addr == esp->command_block_dma) {
+		zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+					esp_count, dma_count, write, cmd);
+		return;
+	}
+
+	zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+	zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+	zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+	if (write) {
+		/* DMA receive */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_FROM_DEVICE);
+		addr &= ~(1);
+	} else {
+		/* DMA send */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_TO_DEVICE);
+		addr |= 1;
+	}
+
+	writeb((addr >> 24) & 0xff, &dregs->dma_addr0);
+	writeb((addr >> 16) & 0xff, &dregs->dma_addr1);
+	writeb((addr >>  8) & 0xff, &dregs->dma_addr2);
+	writeb(addr & 0xff, &dregs->dma_addr3);
+
+	if (write)
+		*ctrl_data &= ~(CYBER_DMA_WRITE);
+	else
+		*ctrl_data |= CYBER_DMA_WRITE;
+
+	*ctrl_data &= ~(CYBER_DMA_Z3);	/* Z2, do 16 bit DMA */
+
+	writeb(*ctrl_data, &dregs->ctrl_reg);
+
+	scsi_esp_cmd(esp, cmd);
+}
+
+/* Cyberstorm II DMA */
+
+static void zorro_esp_send_cyberII_dma_cmd(struct esp *esp, u32 addr,
+			u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+	struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+	struct cyberII_dma_registers __iomem *dregs = esp->dma_regs;
+	u8 phase = esp->sreg & ESP_STAT_PMASK;
+
+	zep->error = 0;
+	/* Use PIO if transferring message bytes to esp->command_block_dma */
+	if (phase == ESP_MIP && addr == esp->command_block_dma) {
+		zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+					esp_count, dma_count, write, cmd);
+		return;
+	}
+
+	zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+	zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+	zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+	if (write) {
+		/* DMA receive */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_FROM_DEVICE);
+		addr &= ~(1);
+	} else {
+		/* DMA send */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_TO_DEVICE);
+		addr |= 1;
+	}
+
+	writeb((addr >> 24) & 0xff, &dregs->dma_addr0);
+	writeb((addr >> 16) & 0xff, &dregs->dma_addr1);
+	writeb((addr >>  8) & 0xff, &dregs->dma_addr2);
+	writeb(addr & 0xff, &dregs->dma_addr3);
+
+	scsi_esp_cmd(esp, cmd);
+}
+
+/* Fastlane DMA */
+
+static void zorro_esp_send_fastlane_dma_cmd(struct esp *esp, u32 addr,
+			u32 esp_count, u32 dma_count, int write, u8 cmd)
+{
+	struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+	struct fastlane_dma_registers __iomem *dregs = esp->dma_regs;
+	u8 phase = esp->sreg & ESP_STAT_PMASK;
+	unsigned char *ctrl_data = &zep->ctrl_data;
+
+	zep->error = 0;
+	/* Use PIO if transferring message bytes to esp->command_block_dma */
+	if (phase == ESP_MIP && addr == esp->command_block_dma) {
+		zorro_esp_send_pio_cmd(esp, (u32) esp->command_block,
+					esp_count, dma_count, write, cmd);
+		return;
+	}
+
+	zorro_esp_write8(esp, (esp_count >> 0) & 0xff, ESP_TCLOW);
+	zorro_esp_write8(esp, (esp_count >> 8) & 0xff, ESP_TCMED);
+	zorro_esp_write8(esp, (esp_count >> 16) & 0xff, ESP_TCHI);
+
+	if (write) {
+		/* DMA receive */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_FROM_DEVICE);
+		addr &= ~(1);
+	} else {
+		/* DMA send */
+		dma_sync_single_for_device(esp->dev, addr, esp_count,
+				DMA_TO_DEVICE);
+		addr |= 1;
+	}
+
+	writeb(0, &dregs->clear_strobe);
+	z_writel(addr, ((addr & 0x00ffffff) + zep->board_base));
+
+	if (write) {
+		*ctrl_data = (*ctrl_data & FASTLANE_DMA_MASK) |
+				FASTLANE_DMA_ENABLE;
+	} else {
+		*ctrl_data = ((*ctrl_data & FASTLANE_DMA_MASK) |
+				FASTLANE_DMA_ENABLE |
+				FASTLANE_DMA_WRITE);
+	}
+
+	writeb(*ctrl_data, &dregs->ctrl_reg);
+
+	scsi_esp_cmd(esp, cmd);
+}
+
+static int zorro_esp_dma_error(struct esp *esp)
+{
+	struct zorro_esp_priv *zep = dev_get_drvdata(esp->dev);
+
+	/* check for error in case we've been doing PIO */
+	if (zep->error == 1)
+		return 1;
+
+	/* do nothing - there seems to be no way to check for DMA errors */
+	return 0;
+}
+
+/* per-board ESP driver ops */
+
+static const struct esp_driver_ops blz1230_esp_ops = {
+	.esp_write8		= zorro_esp_write8,
+	.esp_read8		= zorro_esp_read8,
+	.map_single		= zorro_esp_map_single,
+	.map_sg			= zorro_esp_map_sg,
+	.unmap_single		= zorro_esp_unmap_single,
+	.unmap_sg		= zorro_esp_unmap_sg,
+	.irq_pending		= zorro_esp_irq_pending,
+	.dma_length_limit	= zorro_esp_dma_length_limit,
+	.reset_dma		= zorro_esp_reset_dma,
+	.dma_drain		= zorro_esp_dma_drain,
+	.dma_invalidate		= zorro_esp_dma_invalidate,
+	.send_dma_cmd		= zorro_esp_send_blz1230_dma_cmd,
+	.dma_error		= zorro_esp_dma_error,
+};
+
+static const struct esp_driver_ops blz1230II_esp_ops = {
+	.esp_write8		= zorro_esp_write8,
+	.esp_read8		= zorro_esp_read8,
+	.map_single		= zorro_esp_map_single,
+	.map_sg			= zorro_esp_map_sg,
+	.unmap_single		= zorro_esp_unmap_single,
+	.unmap_sg		= zorro_esp_unmap_sg,
+	.irq_pending		= zorro_esp_irq_pending,
+	.dma_length_limit	= zorro_esp_dma_length_limit,
+	.reset_dma		= zorro_esp_reset_dma,
+	.dma_drain		= zorro_esp_dma_drain,
+	.dma_invalidate		= zorro_esp_dma_invalidate,
+	.send_dma_cmd		= zorro_esp_send_blz1230II_dma_cmd,
+	.dma_error		= zorro_esp_dma_error,
+};
+
+static const struct esp_driver_ops blz2060_esp_ops = {
+	.esp_write8		= zorro_esp_write8,
+	.esp_read8		= zorro_esp_read8,
+	.map_single		= zorro_esp_map_single,
+	.map_sg			= zorro_esp_map_sg,
+	.unmap_single		= zorro_esp_unmap_single,
+	.unmap_sg		= zorro_esp_unmap_sg,
+	.irq_pending		= zorro_esp_irq_pending,
+	.dma_length_limit	= zorro_esp_dma_length_limit,
+	.reset_dma		= zorro_esp_reset_dma,
+	.dma_drain		= zorro_esp_dma_drain,
+	.dma_invalidate		= zorro_esp_dma_invalidate,
+	.send_dma_cmd		= zorro_esp_send_blz2060_dma_cmd,
+	.dma_error		= zorro_esp_dma_error,
+};
+
+static const struct esp_driver_ops cyber_esp_ops = {
+	.esp_write8		= zorro_esp_write8,
+	.esp_read8		= zorro_esp_read8,
+	.map_single		= zorro_esp_map_single,
+	.map_sg			= zorro_esp_map_sg,
+	.unmap_single		= zorro_esp_unmap_single,
+	.unmap_sg		= zorro_esp_unmap_sg,
+	.irq_pending		= cyber_esp_irq_pending,
+	.dma_length_limit	= zorro_esp_dma_length_limit,
+	.reset_dma		= zorro_esp_reset_dma,
+	.dma_drain		= zorro_esp_dma_drain,
+	.dma_invalidate		= zorro_esp_dma_invalidate,
+	.send_dma_cmd		= zorro_esp_send_cyber_dma_cmd,
+	.dma_error		= zorro_esp_dma_error,
+};
+
+static const struct esp_driver_ops cyberII_esp_ops = {
+	.esp_write8		= zorro_esp_write8,
+	.esp_read8		= zorro_esp_read8,
+	.map_single		= zorro_esp_map_single,
+	.map_sg			= zorro_esp_map_sg,
+	.unmap_single		= zorro_esp_unmap_single,
+	.unmap_sg		= zorro_esp_unmap_sg,
+	.irq_pending		= zorro_esp_irq_pending,
+	.dma_length_limit	= zorro_esp_dma_length_limit,
+	.reset_dma		= zorro_esp_reset_dma,
+	.dma_drain		= zorro_esp_dma_drain,
+	.dma_invalidate		= zorro_esp_dma_invalidate,
+	.send_dma_cmd		= zorro_esp_send_cyberII_dma_cmd,
+	.dma_error		= zorro_esp_dma_error,
+};
+
+static const struct esp_driver_ops fastlane_esp_ops = {
+	.esp_write8		= zorro_esp_write8,
+	.esp_read8		= zorro_esp_read8,
+	.map_single		= zorro_esp_map_single,
+	.map_sg			= zorro_esp_map_sg,
+	.unmap_single		= zorro_esp_unmap_single,
+	.unmap_sg		= zorro_esp_unmap_sg,
+	.irq_pending		= fastlane_esp_irq_pending,
+	.dma_length_limit	= zorro_esp_dma_length_limit,
+	.reset_dma		= zorro_esp_reset_dma,
+	.dma_drain		= zorro_esp_dma_drain,
+	.dma_invalidate		= fastlane_esp_dma_invalidate,
+	.send_dma_cmd		= zorro_esp_send_fastlane_dma_cmd,
+	.dma_error		= zorro_esp_dma_error,
+};
+
+/* Zorro driver config data */
+
+struct zorro_driver_data {
+	const char *name;
+	unsigned long offset;
+	unsigned long dma_offset;
+	int absolute;	/* offset is absolute address */
+	int scsi_option;
+	const struct esp_driver_ops *esp_ops;
+};
+
+/* board types */
+
+enum {
+	ZORRO_BLZ1230,
+	ZORRO_BLZ1230II,
+	ZORRO_BLZ2060,
+	ZORRO_CYBER,
+	ZORRO_CYBERII,
+	ZORRO_FASTLANE,
+};
+
+/* per-board config data */
+
+static const struct zorro_driver_data zorro_esp_boards[] = {
+	[ZORRO_BLZ1230] = {
+				.name		= "Blizzard 1230",
+				.offset		= 0x8000,
+				.dma_offset	= 0x10000,
+				.scsi_option	= 1,
+				.esp_ops	= &blz1230_esp_ops,
+	},
+	[ZORRO_BLZ1230II] = {
+				.name		= "Blizzard 1230II",
+				.offset		= 0x10000,
+				.dma_offset	= 0x10021,
+				.scsi_option	= 1,
+				.esp_ops	= &blz1230II_esp_ops,
+	},
+	[ZORRO_BLZ2060] = {
+				.name		= "Blizzard 2060",
+				.offset		= 0x1ff00,
+				.dma_offset	= 0x1ffe0,
+				.esp_ops	= &blz2060_esp_ops,
+	},
+	[ZORRO_CYBER] = {
+				.name		= "CyberStormI",
+				.offset		= 0xf400,
+				.dma_offset	= 0xf800,
+				.esp_ops	= &cyber_esp_ops,
+	},
+	[ZORRO_CYBERII] = {
+				.name		= "CyberStormII",
+				.offset		= 0x1ff03,
+				.dma_offset	= 0x1ff43,
+				.scsi_option	= 1,
+				.esp_ops	= &cyberII_esp_ops,
+	},
+	[ZORRO_FASTLANE] = {
+				.name		= "Fastlane",
+				.offset		= 0x1000001,
+				.dma_offset	= 0x1000041,
+				.esp_ops	= &fastlane_esp_ops,
+	},
+};
+
+static const struct zorro_device_id zorro_esp_zorro_tbl[] = {
+	{	/* Blizzard 1230 IV */
+		.id = ZORRO_ID(PHASE5, 0x11, 0),
+		.driver_data = ZORRO_BLZ1230,
+	},
+	{	/* Blizzard 1230 II (Zorro II) or Fastlane (Zorro III) */
+		.id = ZORRO_ID(PHASE5, 0x0B, 0),
+		.driver_data = ZORRO_BLZ1230II,
+	},
+	{	/* Blizzard 2060 */
+		.id = ZORRO_ID(PHASE5, 0x18, 0),
+		.driver_data = ZORRO_BLZ2060,
+	},
+	{	/* Cyberstorm */
+		.id = ZORRO_ID(PHASE5, 0x0C, 0),
+		.driver_data = ZORRO_CYBER,
+	},
+	{	/* Cyberstorm II */
+		.id = ZORRO_ID(PHASE5, 0x19, 0),
+		.driver_data = ZORRO_CYBERII,
+	},
+	{ 0 }
+};
+MODULE_DEVICE_TABLE(zorro, zorro_esp_zorro_tbl);
+
+static int zorro_esp_probe(struct zorro_dev *z,
+				       const struct zorro_device_id *ent)
+{
+	struct scsi_host_template *tpnt = &scsi_esp_template;
+	struct Scsi_Host *host;
+	struct esp *esp;
+	const struct zorro_driver_data *zdd;
+	struct zorro_esp_priv *zep;
+	unsigned long board, ioaddr, dmaaddr;
+	int err;
+
+	board = zorro_resource_start(z);
+	zdd = &zorro_esp_boards[ent->driver_data];
+
+	pr_info("%s found at address 0x%lx.\n", zdd->name, board);
+
+	zep = kzalloc(sizeof(*zep), GFP_KERNEL);
+	if (!zep) {
+		pr_err("Can't allocate device private data!\n");
+		return -ENOMEM;
+	}
+
+	/* let's figure out whether we have a Zorro II or Zorro III board */
+	if ((z->rom.er_Type & ERT_TYPEMASK) == ERT_ZORROIII) {
+		if (board > 0xffffff)
+			zep->zorro3 = 1;
+	} else {
+		/*
+		 * Even though most of these boards identify as Zorro II,
+		 * they are in fact CPU expansion slot boards and have full
+		 * access to all of memory. Fix up DMA bitmask here.
+		 */
+		z->dev.coherent_dma_mask = DMA_BIT_MASK(32);
+	}
+
+	/*
+	 * If Zorro III and ID matches Fastlane, our device table entry
+	 * contains data for the Blizzard 1230 II board which does share the
+	 * same ID. Fix up device table entry here.
+	 * TODO: Some Cyberstom060 boards also share this ID but would need
+	 * to use the Cyberstorm I driver data ... we catch this by checking
+	 * for presence of ESP chip later, but don't try to fix up yet.
+	 */
+	if (zep->zorro3 && ent->driver_data == ZORRO_BLZ1230II) {
+		pr_info("%s at address 0x%lx is Fastlane Z3, fixing data!\n",
+			zdd->name, board);
+		zdd = &zorro_esp_boards[ZORRO_FASTLANE];
+	}
+
+	if (zdd->absolute) {
+		ioaddr  = zdd->offset;
+		dmaaddr = zdd->dma_offset;
+	} else {
+		ioaddr  = board + zdd->offset;
+		dmaaddr = board + zdd->dma_offset;
+	}
+
+	if (!zorro_request_device(z, zdd->name)) {
+		pr_err("cannot reserve region 0x%lx, abort\n",
+		       board);
+		err = -EBUSY;
+		goto fail_free_zep;
+	}
+
+	host = scsi_host_alloc(tpnt, sizeof(struct esp));
+
+	if (!host) {
+		pr_err("No host detected; board configuration problem?\n");
+		err = -ENOMEM;
+		goto fail_release_device;
+	}
+
+	host->base		= ioaddr;
+	host->this_id		= 7;
+
+	esp			= shost_priv(host);
+	esp->host		= host;
+	esp->dev		= &z->dev;
+
+	esp->scsi_id		= host->this_id;
+	esp->scsi_id_mask	= (1 << esp->scsi_id);
+
+	esp->cfreq = 40000000;
+
+	zep->esp = esp;
+
+	dev_set_drvdata(esp->dev, zep);
+
+	/* additional setup required for Fastlane */
+	if (zep->zorro3 && ent->driver_data == ZORRO_BLZ1230II) {
+		/* map full address space up to ESP base for DMA */
+		zep->board_base = ioremap_nocache(board,
+						FASTLANE_ESP_ADDR-1);
+		if (!zep->board_base) {
+			pr_err("Cannot allocate board address space\n");
+			err = -ENOMEM;
+			goto fail_free_host;
+		}
+		/* initialize DMA control shadow register */
+		zep->ctrl_data = (FASTLANE_DMA_FCODE |
+				  FASTLANE_DMA_EDI | FASTLANE_DMA_ESI);
+	}
+
+	esp->ops = zdd->esp_ops;
+
+	if (ioaddr > 0xffffff)
+		esp->regs = ioremap_nocache(ioaddr, 0x20);
+	else
+		/* ZorroII address space remapped nocache by early startup */
+		esp->regs = ZTWO_VADDR(ioaddr);
+
+	if (!esp->regs) {
+		err = -ENOMEM;
+		goto fail_unmap_fastlane;
+	}
+
+	/* Check whether a Blizzard 12x0 or CyberstormII really has SCSI */
+	if (zdd->scsi_option) {
+		zorro_esp_write8(esp, (ESP_CONFIG1_PENABLE | 7), ESP_CFG1);
+		if (zorro_esp_read8(esp, ESP_CFG1) != (ESP_CONFIG1_PENABLE|7)) {
+			err = -ENODEV;
+			goto fail_unmap_regs;
+		}
+	}
+
+	if (zep->zorro3) {
+		/*
+		 * Only Fastlane Z3 for now - add switch for correct struct
+		 * dma_registers size if adding any more
+		 */
+		esp->dma_regs = ioremap_nocache(dmaaddr,
+				sizeof(struct fastlane_dma_registers));
+	} else
+		/* ZorroII address space remapped nocache by early startup */
+		esp->dma_regs = ZTWO_VADDR(dmaaddr);
+
+	if (!esp->dma_regs) {
+		err = -ENOMEM;
+		goto fail_unmap_regs;
+	}
+
+	esp->command_block = dma_alloc_coherent(esp->dev, 16,
+						&esp->command_block_dma,
+						GFP_KERNEL);
+
+	if (!esp->command_block) {
+		err = -ENOMEM;
+		goto fail_unmap_dma_regs;
+	}
+
+	host->irq = IRQ_AMIGA_PORTS;
+	err = request_irq(host->irq, scsi_esp_intr, IRQF_SHARED,
+			  "Amiga Zorro ESP", esp);
+	if (err < 0) {
+		err = -ENODEV;
+		goto fail_free_command_block;
+	}
+
+	/* register the chip */
+	err = scsi_esp_register(esp, &z->dev);
+
+	if (err) {
+		err = -ENOMEM;
+		goto fail_free_irq;
+	}
+
+	return 0;
+
+fail_free_irq:
+	free_irq(host->irq, esp);
+
+fail_free_command_block:
+	dma_free_coherent(esp->dev, 16,
+			  esp->command_block,
+			  esp->command_block_dma);
+
+fail_unmap_dma_regs:
+	if (zep->zorro3)
+		iounmap(esp->dma_regs);
+
+fail_unmap_regs:
+	if (ioaddr > 0xffffff)
+		iounmap(esp->regs);
+
+fail_unmap_fastlane:
+	if (zep->zorro3)
+		iounmap(zep->board_base);
+
+fail_free_host:
+	scsi_host_put(host);
+
+fail_release_device:
+	zorro_release_device(z);
+
+fail_free_zep:
+	kfree(zep);
+
+	return err;
+}
+
+static void zorro_esp_remove(struct zorro_dev *z)
+{
+	struct zorro_esp_priv *zep = dev_get_drvdata(&z->dev);
+	struct esp *esp	= zep->esp;
+	struct Scsi_Host *host = esp->host;
+
+	scsi_esp_unregister(esp);
+
+	free_irq(host->irq, esp);
+	dma_free_coherent(esp->dev, 16,
+			  esp->command_block,
+			  esp->command_block_dma);
+
+	if (zep->zorro3) {
+		iounmap(zep->board_base);
+		iounmap(esp->dma_regs);
+	}
+
+	if (host->base > 0xffffff)
+		iounmap(esp->regs);
+
+	scsi_host_put(host);
+
+	zorro_release_device(z);
+
+	kfree(zep);
+}
+
+static struct zorro_driver zorro_esp_driver = {
+	.name	  = KBUILD_MODNAME,
+	.id_table = zorro_esp_zorro_tbl,
+	.probe	  = zorro_esp_probe,
+	.remove	  = zorro_esp_remove,
+};
+
+static int __init zorro_esp_scsi_init(void)
+{
+	return zorro_register_driver(&zorro_esp_driver);
+}
+
+static void __exit zorro_esp_scsi_exit(void)
+{
+	zorro_unregister_driver(&zorro_esp_driver);
+}
+
+module_init(zorro_esp_scsi_init);
+module_exit(zorro_esp_scsi_exit);
diff --git a/drivers/slimbus/qcom-ctrl.c b/drivers/slimbus/qcom-ctrl.c
index ffb46f915334..bb36a8fbc9b1 100644
--- a/drivers/slimbus/qcom-ctrl.c
+++ b/drivers/slimbus/qcom-ctrl.c
@@ -439,13 +439,12 @@ static int slim_get_current_rxbuf(struct qcom_slim_ctrl *ctrl, void *buf)
 static void qcom_slim_rxwq(struct work_struct *work)
 {
 	u8 buf[SLIM_MSGQ_BUF_LEN];
-	u8 mc, mt, len;
+	u8 mc, mt;
 	int ret;
 	struct qcom_slim_ctrl *ctrl = container_of(work, struct qcom_slim_ctrl,
 						 wd);
 
 	while ((slim_get_current_rxbuf(ctrl, buf)) != -ENODATA) {
-		len = SLIM_HEADER_GET_RL(buf[0]);
 		mt = SLIM_HEADER_GET_MT(buf[0]);
 		mc = SLIM_HEADER_GET_MC(buf[1]);
 		if (mt == SLIM_MSG_MT_CORE &&
diff --git a/drivers/soc/Makefile b/drivers/soc/Makefile
index 40523577bdaa..113e884697fd 100644
--- a/drivers/soc/Makefile
+++ b/drivers/soc/Makefile
@@ -14,7 +14,7 @@ obj-$(CONFIG_ARCH_MXC)		+= imx/
 obj-$(CONFIG_SOC_XWAY)		+= lantiq/
 obj-y				+= mediatek/
 obj-$(CONFIG_ARCH_MESON)	+= amlogic/
-obj-$(CONFIG_ARCH_QCOM)		+= qcom/
+obj-y				+= qcom/
 obj-y				+= renesas/
 obj-$(CONFIG_ARCH_ROCKCHIP)	+= rockchip/
 obj-$(CONFIG_SOC_SAMSUNG)	+= samsung/
diff --git a/drivers/soc/imx/gpc.c b/drivers/soc/imx/gpc.c
index c4d35f32af8d..32f0748fd067 100644
--- a/drivers/soc/imx/gpc.c
+++ b/drivers/soc/imx/gpc.c
@@ -443,17 +443,25 @@ static int imx_gpc_probe(struct platform_device *pdev)
 			if (domain_index >= of_id_data->num_domains)
 				continue;
 
-			domain = &imx_gpc_domains[domain_index];
-			domain->regmap = regmap;
-			domain->ipg_rate_mhz = ipg_rate_mhz;
-
 			pd_pdev = platform_device_alloc("imx-pgc-power-domain",
 							domain_index);
 			if (!pd_pdev) {
 				of_node_put(np);
 				return -ENOMEM;
 			}
-			pd_pdev->dev.platform_data = domain;
+
+			ret = platform_device_add_data(pd_pdev,
+						       &imx_gpc_domains[domain_index],
+						       sizeof(imx_gpc_domains[domain_index]));
+			if (ret) {
+				platform_device_put(pd_pdev);
+				of_node_put(np);
+				return ret;
+			}
+			domain = pd_pdev->dev.platform_data;
+			domain->regmap = regmap;
+			domain->ipg_rate_mhz = ipg_rate_mhz;
+
 			pd_pdev->dev.parent = &pdev->dev;
 			pd_pdev->dev.of_node = np;
 
diff --git a/drivers/soc/imx/gpcv2.c b/drivers/soc/imx/gpcv2.c
index afc7ecc3c187..f4e3bd40c72e 100644
--- a/drivers/soc/imx/gpcv2.c
+++ b/drivers/soc/imx/gpcv2.c
@@ -155,7 +155,7 @@ static int imx7_gpc_pu_pgc_sw_pdn_req(struct generic_pm_domain *genpd)
 	return imx7_gpc_pu_pgc_sw_pxx_req(genpd, false);
 }
 
-static struct imx7_pgc_domain imx7_pgc_domains[] = {
+static const struct imx7_pgc_domain imx7_pgc_domains[] = {
 	[IMX7_POWER_DOMAIN_MIPI_PHY] = {
 		.genpd = {
 			.name      = "mipi-phy",
@@ -321,11 +321,6 @@ static int imx_gpcv2_probe(struct platform_device *pdev)
 			continue;
 		}
 
-		domain = &imx7_pgc_domains[domain_index];
-		domain->regmap = regmap;
-		domain->genpd.power_on  = imx7_gpc_pu_pgc_sw_pup_req;
-		domain->genpd.power_off = imx7_gpc_pu_pgc_sw_pdn_req;
-
 		pd_pdev = platform_device_alloc("imx7-pgc-domain",
 						domain_index);
 		if (!pd_pdev) {
@@ -334,7 +329,20 @@ static int imx_gpcv2_probe(struct platform_device *pdev)
 			return -ENOMEM;
 		}
 
-		pd_pdev->dev.platform_data = domain;
+		ret = platform_device_add_data(pd_pdev,
+					       &imx7_pgc_domains[domain_index],
+					       sizeof(imx7_pgc_domains[domain_index]));
+		if (ret) {
+			platform_device_put(pd_pdev);
+			of_node_put(np);
+			return ret;
+		}
+
+		domain = pd_pdev->dev.platform_data;
+		domain->regmap = regmap;
+		domain->genpd.power_on  = imx7_gpc_pu_pgc_sw_pup_req;
+		domain->genpd.power_off = imx7_gpc_pu_pgc_sw_pdn_req;
+
 		pd_pdev->dev.parent = dev;
 		pd_pdev->dev.of_node = np;
 
diff --git a/drivers/soc/lantiq/gphy.c b/drivers/soc/lantiq/gphy.c
index 8d8659463b3e..feeb17cebc25 100644
--- a/drivers/soc/lantiq/gphy.c
+++ b/drivers/soc/lantiq/gphy.c
@@ -30,7 +30,6 @@ struct xway_gphy_priv {
 	struct clk *gphy_clk_gate;
 	struct reset_control *gphy_reset;
 	struct reset_control *gphy_reset2;
-	struct notifier_block gphy_reboot_nb;
 	void __iomem *membase;
 	char *fw_name;
 };
@@ -64,24 +63,6 @@ static const struct of_device_id xway_gphy_match[] = {
 };
 MODULE_DEVICE_TABLE(of, xway_gphy_match);
 
-static struct xway_gphy_priv *to_xway_gphy_priv(struct notifier_block *nb)
-{
-	return container_of(nb, struct xway_gphy_priv, gphy_reboot_nb);
-}
-
-static int xway_gphy_reboot_notify(struct notifier_block *reboot_nb,
-				   unsigned long code, void *unused)
-{
-	struct xway_gphy_priv *priv = to_xway_gphy_priv(reboot_nb);
-
-	if (priv) {
-		reset_control_assert(priv->gphy_reset);
-		reset_control_assert(priv->gphy_reset2);
-	}
-
-	return NOTIFY_DONE;
-}
-
 static int xway_gphy_load(struct device *dev, struct xway_gphy_priv *priv,
 			  dma_addr_t *dev_addr)
 {
@@ -205,14 +186,6 @@ static int xway_gphy_probe(struct platform_device *pdev)
 	reset_control_deassert(priv->gphy_reset);
 	reset_control_deassert(priv->gphy_reset2);
 
-	/* assert the gphy reset because it can hang after a reboot: */
-	priv->gphy_reboot_nb.notifier_call = xway_gphy_reboot_notify;
-	priv->gphy_reboot_nb.priority = -1;
-
-	ret = register_reboot_notifier(&priv->gphy_reboot_nb);
-	if (ret)
-		dev_warn(dev, "Failed to register reboot notifier\n");
-
 	platform_set_drvdata(pdev, priv);
 
 	return ret;
@@ -220,21 +193,12 @@ static int xway_gphy_probe(struct platform_device *pdev)
 
 static int xway_gphy_remove(struct platform_device *pdev)
 {
-	struct device *dev = &pdev->dev;
 	struct xway_gphy_priv *priv = platform_get_drvdata(pdev);
-	int ret;
-
-	reset_control_assert(priv->gphy_reset);
-	reset_control_assert(priv->gphy_reset2);
 
 	iowrite32be(0, priv->membase);
 
 	clk_disable_unprepare(priv->gphy_clk_gate);
 
-	ret = unregister_reboot_notifier(&priv->gphy_reboot_nb);
-	if (ret)
-		dev_warn(dev, "Failed to unregister reboot notifier\n");
-
 	return 0;
 }
 
diff --git a/drivers/soc/mediatek/mtk-infracfg.c b/drivers/soc/mediatek/mtk-infracfg.c
index 8c310de01e93..958861c9e6ee 100644
--- a/drivers/soc/mediatek/mtk-infracfg.c
+++ b/drivers/soc/mediatek/mtk-infracfg.c
@@ -17,6 +17,9 @@
 #include <linux/soc/mediatek/infracfg.h>
 #include <asm/processor.h>
 
+#define MTK_POLL_DELAY_US   10
+#define MTK_POLL_TIMEOUT    (jiffies_to_usecs(HZ))
+
 #define INFRA_TOPAXI_PROTECTEN		0x0220
 #define INFRA_TOPAXI_PROTECTSTA1	0x0228
 #define INFRA_TOPAXI_PROTECTEN_SET	0x0260
@@ -37,7 +40,6 @@
 int mtk_infracfg_set_bus_protection(struct regmap *infracfg, u32 mask,
 		bool reg_update)
 {
-	unsigned long expired;
 	u32 val;
 	int ret;
 
@@ -47,22 +49,11 @@ int mtk_infracfg_set_bus_protection(struct regmap *infracfg, u32 mask,
 	else
 		regmap_write(infracfg, INFRA_TOPAXI_PROTECTEN_SET, mask);
 
-	expired = jiffies + HZ;
-
-	while (1) {
-		ret = regmap_read(infracfg, INFRA_TOPAXI_PROTECTSTA1, &val);
-		if (ret)
-			return ret;
-
-		if ((val & mask) == mask)
-			break;
+	ret = regmap_read_poll_timeout(infracfg, INFRA_TOPAXI_PROTECTSTA1,
+				       val, (val & mask) == mask,
+				       MTK_POLL_DELAY_US, MTK_POLL_TIMEOUT);
 
-		cpu_relax();
-		if (time_after(jiffies, expired))
-			return -EIO;
-	}
-
-	return 0;
+	return ret;
 }
 
 /**
@@ -80,30 +71,17 @@ int mtk_infracfg_set_bus_protection(struct regmap *infracfg, u32 mask,
 int mtk_infracfg_clear_bus_protection(struct regmap *infracfg, u32 mask,
 		bool reg_update)
 {
-	unsigned long expired;
 	int ret;
+	u32 val;
 
 	if (reg_update)
 		regmap_update_bits(infracfg, INFRA_TOPAXI_PROTECTEN, mask, 0);
 	else
 		regmap_write(infracfg, INFRA_TOPAXI_PROTECTEN_CLR, mask);
 
-	expired = jiffies + HZ;
-
-	while (1) {
-		u32 val;
-
-		ret = regmap_read(infracfg, INFRA_TOPAXI_PROTECTSTA1, &val);
-		if (ret)
-			return ret;
-
-		if (!(val & mask))
-			break;
-
-		cpu_relax();
-		if (time_after(jiffies, expired))
-			return -EIO;
-	}
+	ret = regmap_read_poll_timeout(infracfg, INFRA_TOPAXI_PROTECTSTA1,
+				       val, !(val & mask),
+				       MTK_POLL_DELAY_US, MTK_POLL_TIMEOUT);
 
-	return 0;
+	return ret;
 }
diff --git a/drivers/soc/mediatek/mtk-pmic-wrap.c b/drivers/soc/mediatek/mtk-pmic-wrap.c
index e9e054a15b7d..2afae64061d8 100644
--- a/drivers/soc/mediatek/mtk-pmic-wrap.c
+++ b/drivers/soc/mediatek/mtk-pmic-wrap.c
@@ -1458,19 +1458,12 @@ static int pwrap_probe(struct platform_device *pdev)
 	int ret, irq;
 	struct pmic_wrapper *wrp;
 	struct device_node *np = pdev->dev.of_node;
-	const struct of_device_id *of_id =
-		of_match_device(of_pwrap_match_tbl, &pdev->dev);
 	const struct of_device_id *of_slave_id = NULL;
 	struct resource *res;
 
-	if (!of_id) {
-		dev_err(&pdev->dev, "Error: No device match found\n");
-		return -ENODEV;
-	}
+	if (np->child)
+		of_slave_id = of_match_node(of_slave_match_tbl, np->child);
 
-	if (pdev->dev.of_node->child)
-		of_slave_id = of_match_node(of_slave_match_tbl,
-					    pdev->dev.of_node->child);
 	if (!of_slave_id) {
 		dev_dbg(&pdev->dev, "slave pmic should be defined in dts\n");
 		return -EINVAL;
@@ -1482,7 +1475,7 @@ static int pwrap_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, wrp);
 
-	wrp->master = of_id->data;
+	wrp->master = of_device_get_match_data(&pdev->dev);
 	wrp->slave = of_slave_id->data;
 	wrp->dev = &pdev->dev;
 
diff --git a/drivers/soc/mediatek/mtk-scpsys.c b/drivers/soc/mediatek/mtk-scpsys.c
index d762a46d434f..128e3dd3186d 100644
--- a/drivers/soc/mediatek/mtk-scpsys.c
+++ b/drivers/soc/mediatek/mtk-scpsys.c
@@ -13,6 +13,7 @@
 #include <linux/clk.h>
 #include <linux/init.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/mfd/syscon.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
@@ -27,6 +28,13 @@
 #include <dt-bindings/power/mt7623a-power.h>
 #include <dt-bindings/power/mt8173-power.h>
 
+#define MTK_POLL_DELAY_US   10
+#define MTK_POLL_TIMEOUT    (jiffies_to_usecs(HZ))
+
+#define MTK_SCPD_ACTIVE_WAKEUP		BIT(0)
+#define MTK_SCPD_FWAIT_SRAM		BIT(1)
+#define MTK_SCPD_CAPS(_scpd, _x)	((_scpd)->data->caps & (_x))
+
 #define SPM_VDE_PWR_CON			0x0210
 #define SPM_MFG_PWR_CON			0x0214
 #define SPM_VEN_PWR_CON			0x0230
@@ -116,7 +124,7 @@ struct scp_domain_data {
 	u32 sram_pdn_ack_bits;
 	u32 bus_prot_mask;
 	enum clk_id clk_id[MAX_CLKS];
-	bool active_wakeup;
+	u8 caps;
 };
 
 struct scp;
@@ -184,12 +192,10 @@ static int scpsys_power_on(struct generic_pm_domain *genpd)
 {
 	struct scp_domain *scpd = container_of(genpd, struct scp_domain, genpd);
 	struct scp *scp = scpd->scp;
-	unsigned long timeout;
-	bool expired;
 	void __iomem *ctl_addr = scp->base + scpd->data->ctl_offs;
-	u32 sram_pdn_ack = scpd->data->sram_pdn_ack_bits;
+	u32 pdn_ack = scpd->data->sram_pdn_ack_bits;
 	u32 val;
-	int ret;
+	int ret, tmp;
 	int i;
 
 	if (scpd->supply) {
@@ -215,23 +221,10 @@ static int scpsys_power_on(struct generic_pm_domain *genpd)
 	writel(val, ctl_addr);
 
 	/* wait until PWR_ACK = 1 */
-	timeout = jiffies + HZ;
-	expired = false;
-	while (1) {
-		ret = scpsys_domain_is_on(scpd);
-		if (ret > 0)
-			break;
-
-		if (expired) {
-			ret = -ETIMEDOUT;
-			goto err_pwr_ack;
-		}
-
-		cpu_relax();
-
-		if (time_after(jiffies, timeout))
-			expired = true;
-	}
+	ret = readx_poll_timeout(scpsys_domain_is_on, scpd, tmp, tmp > 0,
+				 MTK_POLL_DELAY_US, MTK_POLL_TIMEOUT);
+	if (ret < 0)
+		goto err_pwr_ack;
 
 	val &= ~PWR_CLK_DIS_BIT;
 	writel(val, ctl_addr);
@@ -245,20 +238,20 @@ static int scpsys_power_on(struct generic_pm_domain *genpd)
 	val &= ~scpd->data->sram_pdn_bits;
 	writel(val, ctl_addr);
 
-	/* wait until SRAM_PDN_ACK all 0 */
-	timeout = jiffies + HZ;
-	expired = false;
-	while (sram_pdn_ack && (readl(ctl_addr) & sram_pdn_ack)) {
+	/* Either wait until SRAM_PDN_ACK all 0 or have a force wait */
+	if (MTK_SCPD_CAPS(scpd, MTK_SCPD_FWAIT_SRAM)) {
+		/*
+		 * Currently, MTK_SCPD_FWAIT_SRAM is necessary only for
+		 * MT7622_POWER_DOMAIN_WB and thus just a trivial setup is
+		 * applied here.
+		 */
+		usleep_range(12000, 12100);
 
-		if (expired) {
-			ret = -ETIMEDOUT;
+	} else {
+		ret = readl_poll_timeout(ctl_addr, tmp, (tmp & pdn_ack) == 0,
+					 MTK_POLL_DELAY_US, MTK_POLL_TIMEOUT);
+		if (ret < 0)
 			goto err_pwr_ack;
-		}
-
-		cpu_relax();
-
-		if (time_after(jiffies, timeout))
-			expired = true;
 	}
 
 	if (scpd->data->bus_prot_mask) {
@@ -289,12 +282,10 @@ static int scpsys_power_off(struct generic_pm_domain *genpd)
 {
 	struct scp_domain *scpd = container_of(genpd, struct scp_domain, genpd);
 	struct scp *scp = scpd->scp;
-	unsigned long timeout;
-	bool expired;
 	void __iomem *ctl_addr = scp->base + scpd->data->ctl_offs;
 	u32 pdn_ack = scpd->data->sram_pdn_ack_bits;
 	u32 val;
-	int ret;
+	int ret, tmp;
 	int i;
 
 	if (scpd->data->bus_prot_mask) {
@@ -310,19 +301,10 @@ static int scpsys_power_off(struct generic_pm_domain *genpd)
 	writel(val, ctl_addr);
 
 	/* wait until SRAM_PDN_ACK all 1 */
-	timeout = jiffies + HZ;
-	expired = false;
-	while (pdn_ack && (readl(ctl_addr) & pdn_ack) != pdn_ack) {
-		if (expired) {
-			ret = -ETIMEDOUT;
-			goto out;
-		}
-
-		cpu_relax();
-
-		if (time_after(jiffies, timeout))
-			expired = true;
-	}
+	ret = readl_poll_timeout(ctl_addr, tmp, (tmp & pdn_ack) == pdn_ack,
+				 MTK_POLL_DELAY_US, MTK_POLL_TIMEOUT);
+	if (ret < 0)
+		goto out;
 
 	val |= PWR_ISO_BIT;
 	writel(val, ctl_addr);
@@ -340,23 +322,10 @@ static int scpsys_power_off(struct generic_pm_domain *genpd)
 	writel(val, ctl_addr);
 
 	/* wait until PWR_ACK = 0 */
-	timeout = jiffies + HZ;
-	expired = false;
-	while (1) {
-		ret = scpsys_domain_is_on(scpd);
-		if (ret == 0)
-			break;
-
-		if (expired) {
-			ret = -ETIMEDOUT;
-			goto out;
-		}
-
-		cpu_relax();
-
-		if (time_after(jiffies, timeout))
-			expired = true;
-	}
+	ret = readx_poll_timeout(scpsys_domain_is_on, scpd, tmp, tmp == 0,
+				 MTK_POLL_DELAY_US, MTK_POLL_TIMEOUT);
+	if (ret < 0)
+		goto out;
 
 	for (i = 0; i < MAX_CLKS && scpd->clk[i]; i++)
 		clk_disable_unprepare(scpd->clk[i]);
@@ -469,7 +438,7 @@ static struct scp *init_scp(struct platform_device *pdev,
 		genpd->name = data->name;
 		genpd->power_off = scpsys_power_off;
 		genpd->power_on = scpsys_power_on;
-		if (scpd->data->active_wakeup)
+		if (MTK_SCPD_CAPS(scpd, MTK_SCPD_ACTIVE_WAKEUP))
 			genpd->flags |= GENPD_FLAG_ACTIVE_WAKEUP;
 	}
 
@@ -522,7 +491,7 @@ static const struct scp_domain_data scp_domain_data_mt2701[] = {
 		.bus_prot_mask = MT2701_TOP_AXI_PROT_EN_CONN_M |
 				 MT2701_TOP_AXI_PROT_EN_CONN_S,
 		.clk_id = {CLK_NONE},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2701_POWER_DOMAIN_DISP] = {
 		.name = "disp",
@@ -531,7 +500,7 @@ static const struct scp_domain_data scp_domain_data_mt2701[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.clk_id = {CLK_MM},
 		.bus_prot_mask = MT2701_TOP_AXI_PROT_EN_MM_M0,
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2701_POWER_DOMAIN_MFG] = {
 		.name = "mfg",
@@ -540,7 +509,7 @@ static const struct scp_domain_data scp_domain_data_mt2701[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.sram_pdn_ack_bits = GENMASK(12, 12),
 		.clk_id = {CLK_MFG},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2701_POWER_DOMAIN_VDEC] = {
 		.name = "vdec",
@@ -549,7 +518,7 @@ static const struct scp_domain_data scp_domain_data_mt2701[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.sram_pdn_ack_bits = GENMASK(12, 12),
 		.clk_id = {CLK_MM},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2701_POWER_DOMAIN_ISP] = {
 		.name = "isp",
@@ -558,7 +527,7 @@ static const struct scp_domain_data scp_domain_data_mt2701[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.sram_pdn_ack_bits = GENMASK(13, 12),
 		.clk_id = {CLK_MM},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2701_POWER_DOMAIN_BDP] = {
 		.name = "bdp",
@@ -566,7 +535,7 @@ static const struct scp_domain_data scp_domain_data_mt2701[] = {
 		.ctl_offs = SPM_BDP_PWR_CON,
 		.sram_pdn_bits = GENMASK(11, 8),
 		.clk_id = {CLK_NONE},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2701_POWER_DOMAIN_ETH] = {
 		.name = "eth",
@@ -575,7 +544,7 @@ static const struct scp_domain_data scp_domain_data_mt2701[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.sram_pdn_ack_bits = GENMASK(15, 12),
 		.clk_id = {CLK_ETHIF},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2701_POWER_DOMAIN_HIF] = {
 		.name = "hif",
@@ -584,14 +553,14 @@ static const struct scp_domain_data scp_domain_data_mt2701[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.sram_pdn_ack_bits = GENMASK(15, 12),
 		.clk_id = {CLK_ETHIF},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2701_POWER_DOMAIN_IFR_MSC] = {
 		.name = "ifr_msc",
 		.sta_mask = PWR_STATUS_IFR_MSC,
 		.ctl_offs = SPM_IFR_MSC_PWR_CON,
 		.clk_id = {CLK_NONE},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 };
 
@@ -606,7 +575,7 @@ static const struct scp_domain_data scp_domain_data_mt2712[] = {
 		.sram_pdn_bits = GENMASK(8, 8),
 		.sram_pdn_ack_bits = GENMASK(12, 12),
 		.clk_id = {CLK_MM},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2712_POWER_DOMAIN_VDEC] = {
 		.name = "vdec",
@@ -615,7 +584,7 @@ static const struct scp_domain_data scp_domain_data_mt2712[] = {
 		.sram_pdn_bits = GENMASK(8, 8),
 		.sram_pdn_ack_bits = GENMASK(12, 12),
 		.clk_id = {CLK_MM, CLK_VDEC},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2712_POWER_DOMAIN_VENC] = {
 		.name = "venc",
@@ -624,7 +593,7 @@ static const struct scp_domain_data scp_domain_data_mt2712[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.sram_pdn_ack_bits = GENMASK(15, 12),
 		.clk_id = {CLK_MM, CLK_VENC, CLK_JPGDEC},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2712_POWER_DOMAIN_ISP] = {
 		.name = "isp",
@@ -633,7 +602,7 @@ static const struct scp_domain_data scp_domain_data_mt2712[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.sram_pdn_ack_bits = GENMASK(13, 12),
 		.clk_id = {CLK_MM},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2712_POWER_DOMAIN_AUDIO] = {
 		.name = "audio",
@@ -642,7 +611,7 @@ static const struct scp_domain_data scp_domain_data_mt2712[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.sram_pdn_ack_bits = GENMASK(15, 12),
 		.clk_id = {CLK_AUDIO},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2712_POWER_DOMAIN_USB] = {
 		.name = "usb",
@@ -651,7 +620,7 @@ static const struct scp_domain_data scp_domain_data_mt2712[] = {
 		.sram_pdn_bits = GENMASK(10, 8),
 		.sram_pdn_ack_bits = GENMASK(14, 12),
 		.clk_id = {CLK_NONE},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2712_POWER_DOMAIN_USB2] = {
 		.name = "usb2",
@@ -660,7 +629,7 @@ static const struct scp_domain_data scp_domain_data_mt2712[] = {
 		.sram_pdn_bits = GENMASK(10, 8),
 		.sram_pdn_ack_bits = GENMASK(14, 12),
 		.clk_id = {CLK_NONE},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2712_POWER_DOMAIN_MFG] = {
 		.name = "mfg",
@@ -670,7 +639,7 @@ static const struct scp_domain_data scp_domain_data_mt2712[] = {
 		.sram_pdn_ack_bits = GENMASK(16, 16),
 		.clk_id = {CLK_MFG},
 		.bus_prot_mask = BIT(14) | BIT(21) | BIT(23),
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2712_POWER_DOMAIN_MFG_SC1] = {
 		.name = "mfg_sc1",
@@ -679,7 +648,7 @@ static const struct scp_domain_data scp_domain_data_mt2712[] = {
 		.sram_pdn_bits = GENMASK(8, 8),
 		.sram_pdn_ack_bits = GENMASK(16, 16),
 		.clk_id = {CLK_NONE},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2712_POWER_DOMAIN_MFG_SC2] = {
 		.name = "mfg_sc2",
@@ -688,7 +657,7 @@ static const struct scp_domain_data scp_domain_data_mt2712[] = {
 		.sram_pdn_bits = GENMASK(8, 8),
 		.sram_pdn_ack_bits = GENMASK(16, 16),
 		.clk_id = {CLK_NONE},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT2712_POWER_DOMAIN_MFG_SC3] = {
 		.name = "mfg_sc3",
@@ -697,7 +666,7 @@ static const struct scp_domain_data scp_domain_data_mt2712[] = {
 		.sram_pdn_bits = GENMASK(8, 8),
 		.sram_pdn_ack_bits = GENMASK(16, 16),
 		.clk_id = {CLK_NONE},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 };
 
@@ -797,7 +766,7 @@ static const struct scp_domain_data scp_domain_data_mt7622[] = {
 		.sram_pdn_ack_bits = GENMASK(15, 12),
 		.clk_id = {CLK_NONE},
 		.bus_prot_mask = MT7622_TOP_AXI_PROT_EN_ETHSYS,
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT7622_POWER_DOMAIN_HIF0] = {
 		.name = "hif0",
@@ -807,7 +776,7 @@ static const struct scp_domain_data scp_domain_data_mt7622[] = {
 		.sram_pdn_ack_bits = GENMASK(15, 12),
 		.clk_id = {CLK_HIFSEL},
 		.bus_prot_mask = MT7622_TOP_AXI_PROT_EN_HIF0,
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT7622_POWER_DOMAIN_HIF1] = {
 		.name = "hif1",
@@ -817,7 +786,7 @@ static const struct scp_domain_data scp_domain_data_mt7622[] = {
 		.sram_pdn_ack_bits = GENMASK(15, 12),
 		.clk_id = {CLK_HIFSEL},
 		.bus_prot_mask = MT7622_TOP_AXI_PROT_EN_HIF1,
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT7622_POWER_DOMAIN_WB] = {
 		.name = "wb",
@@ -827,7 +796,7 @@ static const struct scp_domain_data scp_domain_data_mt7622[] = {
 		.sram_pdn_ack_bits = 0,
 		.clk_id = {CLK_NONE},
 		.bus_prot_mask = MT7622_TOP_AXI_PROT_EN_WB,
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP | MTK_SCPD_FWAIT_SRAM,
 	},
 };
 
@@ -843,7 +812,7 @@ static const struct scp_domain_data scp_domain_data_mt7623a[] = {
 		.bus_prot_mask = MT2701_TOP_AXI_PROT_EN_CONN_M |
 				 MT2701_TOP_AXI_PROT_EN_CONN_S,
 		.clk_id = {CLK_NONE},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT7623A_POWER_DOMAIN_ETH] = {
 		.name = "eth",
@@ -852,7 +821,7 @@ static const struct scp_domain_data scp_domain_data_mt7623a[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.sram_pdn_ack_bits = GENMASK(15, 12),
 		.clk_id = {CLK_ETHIF},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT7623A_POWER_DOMAIN_HIF] = {
 		.name = "hif",
@@ -861,14 +830,14 @@ static const struct scp_domain_data scp_domain_data_mt7623a[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.sram_pdn_ack_bits = GENMASK(15, 12),
 		.clk_id = {CLK_ETHIF},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT7623A_POWER_DOMAIN_IFR_MSC] = {
 		.name = "ifr_msc",
 		.sta_mask = PWR_STATUS_IFR_MSC,
 		.ctl_offs = SPM_IFR_MSC_PWR_CON,
 		.clk_id = {CLK_NONE},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 };
 
@@ -934,7 +903,7 @@ static const struct scp_domain_data scp_domain_data_mt8173[] = {
 		.sram_pdn_bits = GENMASK(11, 8),
 		.sram_pdn_ack_bits = GENMASK(15, 12),
 		.clk_id = {CLK_NONE},
-		.active_wakeup = true,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
 	},
 	[MT8173_POWER_DOMAIN_MFG_ASYNC] = {
 		.name = "mfg_async",
@@ -1067,15 +1036,13 @@ static const struct of_device_id of_scpsys_match_tbl[] = {
 
 static int scpsys_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *match;
 	const struct scp_subdomain *sd;
 	const struct scp_soc_data *soc;
 	struct scp *scp;
 	struct genpd_onecell_data *pd_data;
 	int i, ret;
 
-	match = of_match_device(of_scpsys_match_tbl, &pdev->dev);
-	soc = (const struct scp_soc_data *)match->data;
+	soc = of_device_get_match_data(&pdev->dev);
 
 	scp = init_scp(pdev, soc->domains, soc->num_domains, &soc->regs,
 			soc->bus_prot_reg_update);
diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig
index 5c4535b545cc..9dc02f390ba3 100644
--- a/drivers/soc/qcom/Kconfig
+++ b/drivers/soc/qcom/Kconfig
@@ -3,6 +3,24 @@
 #
 menu "Qualcomm SoC drivers"
 
+config QCOM_COMMAND_DB
+	bool "Qualcomm Command DB"
+	depends on (ARCH_QCOM && OF) || COMPILE_TEST
+	help
+	  Command DB queries shared memory by key string for shared system
+	  resources. Platform drivers that require to set state of a shared
+	  resource on a RPM-hardened platform must use this database to get
+	  SoC specific identifier and information for the shared resources.
+
+config QCOM_GENI_SE
+	tristate "QCOM GENI Serial Engine Driver"
+	depends on ARCH_QCOM || COMPILE_TEST
+	help
+	  This driver is used to manage Generic Interface (GENI) firmware based
+	  Qualcomm Technologies, Inc. Universal Peripheral (QUP) Wrapper. This
+	  driver is also used to manage the common aspects of multiple Serial
+	  Engines present in the QUP.
+
 config QCOM_GLINK_SSR
 	tristate "Qualcomm Glink SSR driver"
 	depends on RPMSG
@@ -108,4 +126,13 @@ config QCOM_WCNSS_CTRL
 	  Client driver for the WCNSS_CTRL SMD channel, used to download nv
 	  firmware to a newly booted WCNSS chip.
 
+config QCOM_APR
+	tristate "Qualcomm APR Bus (Asynchronous Packet Router)"
+	depends on ARCH_QCOM
+	depends on RPMSG
+	help
+          Enable APR IPC protocol support between
+          application processor and QDSP6. APR is
+          used by audio driver to configure QDSP6
+          ASM, ADM and AFE modules.
 endmenu
diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile
index dcebf2814e6d..19dcf957cb3a 100644
--- a/drivers/soc/qcom/Makefile
+++ b/drivers/soc/qcom/Makefile
@@ -1,4 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_QCOM_GENI_SE) +=	qcom-geni-se.o
+obj-$(CONFIG_QCOM_COMMAND_DB) += cmd-db.o
 obj-$(CONFIG_QCOM_GLINK_SSR) +=	glink_ssr.o
 obj-$(CONFIG_QCOM_GSBI)	+=	qcom_gsbi.o
 obj-$(CONFIG_QCOM_MDT_LOADER)	+= mdt_loader.o
@@ -12,3 +14,4 @@ obj-$(CONFIG_QCOM_SMEM_STATE) += smem_state.o
 obj-$(CONFIG_QCOM_SMP2P)	+= smp2p.o
 obj-$(CONFIG_QCOM_SMSM)	+= smsm.o
 obj-$(CONFIG_QCOM_WCNSS_CTRL) += wcnss_ctrl.o
+obj-$(CONFIG_QCOM_APR) += apr.o
diff --git a/drivers/soc/qcom/apr.c b/drivers/soc/qcom/apr.c
new file mode 100644
index 000000000000..57af8a537332
--- /dev/null
+++ b/drivers/soc/qcom/apr.c
@@ -0,0 +1,378 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2011-2017, The Linux Foundation. All rights reserved.
+// Copyright (c) 2018, Linaro Limited
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/spinlock.h>
+#include <linux/idr.h>
+#include <linux/slab.h>
+#include <linux/of_device.h>
+#include <linux/soc/qcom/apr.h>
+#include <linux/rpmsg.h>
+#include <linux/of.h>
+
+struct apr {
+	struct rpmsg_endpoint *ch;
+	struct device *dev;
+	spinlock_t svcs_lock;
+	struct idr svcs_idr;
+	int dest_domain_id;
+};
+
+/**
+ * apr_send_pkt() - Send a apr message from apr device
+ *
+ * @adev: Pointer to previously registered apr device.
+ * @pkt: Pointer to apr packet to send
+ *
+ * Return: Will be an negative on packet size on success.
+ */
+int apr_send_pkt(struct apr_device *adev, struct apr_pkt *pkt)
+{
+	struct apr *apr = dev_get_drvdata(adev->dev.parent);
+	struct apr_hdr *hdr;
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&adev->lock, flags);
+
+	hdr = &pkt->hdr;
+	hdr->src_domain = APR_DOMAIN_APPS;
+	hdr->src_svc = adev->svc_id;
+	hdr->dest_domain = adev->domain_id;
+	hdr->dest_svc = adev->svc_id;
+
+	ret = rpmsg_trysend(apr->ch, pkt, hdr->pkt_size);
+	spin_unlock_irqrestore(&adev->lock, flags);
+
+	return ret ? ret : hdr->pkt_size;
+}
+EXPORT_SYMBOL_GPL(apr_send_pkt);
+
+static void apr_dev_release(struct device *dev)
+{
+	struct apr_device *adev = to_apr_device(dev);
+
+	kfree(adev);
+}
+
+static int apr_callback(struct rpmsg_device *rpdev, void *buf,
+				  int len, void *priv, u32 addr)
+{
+	struct apr *apr = dev_get_drvdata(&rpdev->dev);
+	uint16_t hdr_size, msg_type, ver, svc_id;
+	struct apr_device *svc = NULL;
+	struct apr_driver *adrv = NULL;
+	struct apr_resp_pkt resp;
+	struct apr_hdr *hdr;
+	unsigned long flags;
+
+	if (len <= APR_HDR_SIZE) {
+		dev_err(apr->dev, "APR: Improper apr pkt received:%p %d\n",
+			buf, len);
+		return -EINVAL;
+	}
+
+	hdr = buf;
+	ver = APR_HDR_FIELD_VER(hdr->hdr_field);
+	if (ver > APR_PKT_VER + 1)
+		return -EINVAL;
+
+	hdr_size = APR_HDR_FIELD_SIZE_BYTES(hdr->hdr_field);
+	if (hdr_size < APR_HDR_SIZE) {
+		dev_err(apr->dev, "APR: Wrong hdr size:%d\n", hdr_size);
+		return -EINVAL;
+	}
+
+	if (hdr->pkt_size < APR_HDR_SIZE || hdr->pkt_size != len) {
+		dev_err(apr->dev, "APR: Wrong paket size\n");
+		return -EINVAL;
+	}
+
+	msg_type = APR_HDR_FIELD_MT(hdr->hdr_field);
+	if (msg_type >= APR_MSG_TYPE_MAX) {
+		dev_err(apr->dev, "APR: Wrong message type: %d\n", msg_type);
+		return -EINVAL;
+	}
+
+	if (hdr->src_domain >= APR_DOMAIN_MAX ||
+			hdr->dest_domain >= APR_DOMAIN_MAX ||
+			hdr->src_svc >= APR_SVC_MAX ||
+			hdr->dest_svc >= APR_SVC_MAX) {
+		dev_err(apr->dev, "APR: Wrong APR header\n");
+		return -EINVAL;
+	}
+
+	svc_id = hdr->dest_svc;
+	spin_lock_irqsave(&apr->svcs_lock, flags);
+	svc = idr_find(&apr->svcs_idr, svc_id);
+	if (svc && svc->dev.driver)
+		adrv = to_apr_driver(svc->dev.driver);
+	spin_unlock_irqrestore(&apr->svcs_lock, flags);
+
+	if (!adrv) {
+		dev_err(apr->dev, "APR: service is not registered\n");
+		return -EINVAL;
+	}
+
+	resp.hdr = *hdr;
+	resp.payload_size = hdr->pkt_size - hdr_size;
+
+	/*
+	 * NOTE: hdr_size is not same as APR_HDR_SIZE as remote can include
+	 * optional headers in to apr_hdr which should be ignored
+	 */
+	if (resp.payload_size > 0)
+		resp.payload = buf + hdr_size;
+
+	adrv->callback(svc, &resp);
+
+	return 0;
+}
+
+static int apr_device_match(struct device *dev, struct device_driver *drv)
+{
+	struct apr_device *adev = to_apr_device(dev);
+	struct apr_driver *adrv = to_apr_driver(drv);
+	const struct apr_device_id *id = adrv->id_table;
+
+	/* Attempt an OF style match first */
+	if (of_driver_match_device(dev, drv))
+		return 1;
+
+	if (!id)
+		return 0;
+
+	while (id->domain_id != 0 || id->svc_id != 0) {
+		if (id->domain_id == adev->domain_id &&
+		    id->svc_id == adev->svc_id)
+			return 1;
+		id++;
+	}
+
+	return 0;
+}
+
+static int apr_device_probe(struct device *dev)
+{
+	struct apr_device *adev = to_apr_device(dev);
+	struct apr_driver *adrv = to_apr_driver(dev->driver);
+
+	return adrv->probe(adev);
+}
+
+static int apr_device_remove(struct device *dev)
+{
+	struct apr_device *adev = to_apr_device(dev);
+	struct apr_driver *adrv;
+	struct apr *apr = dev_get_drvdata(adev->dev.parent);
+
+	if (dev->driver) {
+		adrv = to_apr_driver(dev->driver);
+		if (adrv->remove)
+			adrv->remove(adev);
+		spin_lock(&apr->svcs_lock);
+		idr_remove(&apr->svcs_idr, adev->svc_id);
+		spin_unlock(&apr->svcs_lock);
+	}
+
+	return 0;
+}
+
+static int apr_uevent(struct device *dev, struct kobj_uevent_env *env)
+{
+	struct apr_device *adev = to_apr_device(dev);
+	int ret;
+
+	ret = of_device_uevent_modalias(dev, env);
+	if (ret != -ENODEV)
+		return ret;
+
+	return add_uevent_var(env, "MODALIAS=apr:%s", adev->name);
+}
+
+struct bus_type aprbus = {
+	.name		= "aprbus",
+	.match		= apr_device_match,
+	.probe		= apr_device_probe,
+	.uevent		= apr_uevent,
+	.remove		= apr_device_remove,
+};
+EXPORT_SYMBOL_GPL(aprbus);
+
+static int apr_add_device(struct device *dev, struct device_node *np,
+			  const struct apr_device_id *id)
+{
+	struct apr *apr = dev_get_drvdata(dev);
+	struct apr_device *adev = NULL;
+	int ret;
+
+	adev = kzalloc(sizeof(*adev), GFP_KERNEL);
+	if (!adev)
+		return -ENOMEM;
+
+	spin_lock_init(&adev->lock);
+
+	adev->svc_id = id->svc_id;
+	adev->domain_id = id->domain_id;
+	adev->version = id->svc_version;
+	if (np)
+		strncpy(adev->name, np->name, APR_NAME_SIZE);
+	else
+		strncpy(adev->name, id->name, APR_NAME_SIZE);
+
+	dev_set_name(&adev->dev, "aprsvc:%s:%x:%x", adev->name,
+		     id->domain_id, id->svc_id);
+
+	adev->dev.bus = &aprbus;
+	adev->dev.parent = dev;
+	adev->dev.of_node = np;
+	adev->dev.release = apr_dev_release;
+	adev->dev.driver = NULL;
+
+	spin_lock(&apr->svcs_lock);
+	idr_alloc(&apr->svcs_idr, adev, id->svc_id,
+		  id->svc_id + 1, GFP_ATOMIC);
+	spin_unlock(&apr->svcs_lock);
+
+	dev_info(dev, "Adding APR dev: %s\n", dev_name(&adev->dev));
+
+	ret = device_register(&adev->dev);
+	if (ret) {
+		dev_err(dev, "device_register failed: %d\n", ret);
+		put_device(&adev->dev);
+	}
+
+	return ret;
+}
+
+static void of_register_apr_devices(struct device *dev)
+{
+	struct apr *apr = dev_get_drvdata(dev);
+	struct device_node *node;
+
+	for_each_child_of_node(dev->of_node, node) {
+		struct apr_device_id id = { {0} };
+
+		if (of_property_read_u32(node, "reg", &id.svc_id))
+			continue;
+
+		id.domain_id = apr->dest_domain_id;
+
+		if (apr_add_device(dev, node, &id))
+			dev_err(dev, "Failed to add apr %d svc\n", id.svc_id);
+	}
+}
+
+static int apr_probe(struct rpmsg_device *rpdev)
+{
+	struct device *dev = &rpdev->dev;
+	struct apr *apr;
+	int ret;
+
+	apr = devm_kzalloc(dev, sizeof(*apr), GFP_KERNEL);
+	if (!apr)
+		return -ENOMEM;
+
+	ret = of_property_read_u32(dev->of_node, "reg", &apr->dest_domain_id);
+	if (ret) {
+		dev_err(dev, "APR Domain ID not specified in DT\n");
+		return ret;
+	}
+
+	dev_set_drvdata(dev, apr);
+	apr->ch = rpdev->ept;
+	apr->dev = dev;
+	spin_lock_init(&apr->svcs_lock);
+	idr_init(&apr->svcs_idr);
+	of_register_apr_devices(dev);
+
+	return 0;
+}
+
+static int apr_remove_device(struct device *dev, void *null)
+{
+	struct apr_device *adev = to_apr_device(dev);
+
+	device_unregister(&adev->dev);
+
+	return 0;
+}
+
+static void apr_remove(struct rpmsg_device *rpdev)
+{
+	device_for_each_child(&rpdev->dev, NULL, apr_remove_device);
+}
+
+/*
+ * __apr_driver_register() - Client driver registration with aprbus
+ *
+ * @drv:Client driver to be associated with client-device.
+ * @owner: owning module/driver
+ *
+ * This API will register the client driver with the aprbus
+ * It is called from the driver's module-init function.
+ */
+int __apr_driver_register(struct apr_driver *drv, struct module *owner)
+{
+	drv->driver.bus = &aprbus;
+	drv->driver.owner = owner;
+
+	return driver_register(&drv->driver);
+}
+EXPORT_SYMBOL_GPL(__apr_driver_register);
+
+/*
+ * apr_driver_unregister() - Undo effect of apr_driver_register
+ *
+ * @drv: Client driver to be unregistered
+ */
+void apr_driver_unregister(struct apr_driver *drv)
+{
+	driver_unregister(&drv->driver);
+}
+EXPORT_SYMBOL_GPL(apr_driver_unregister);
+
+static const struct of_device_id apr_of_match[] = {
+	{ .compatible = "qcom,apr"},
+	{ .compatible = "qcom,apr-v2"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, apr_of_match);
+
+static struct rpmsg_driver apr_driver = {
+	.probe = apr_probe,
+	.remove = apr_remove,
+	.callback = apr_callback,
+	.drv = {
+		.name = "qcom,apr",
+		.of_match_table = apr_of_match,
+	},
+};
+
+static int __init apr_init(void)
+{
+	int ret;
+
+	ret = bus_register(&aprbus);
+	if (!ret)
+		ret = register_rpmsg_driver(&apr_driver);
+	else
+		bus_unregister(&aprbus);
+
+	return ret;
+}
+
+static void __exit apr_exit(void)
+{
+	bus_unregister(&aprbus);
+	unregister_rpmsg_driver(&apr_driver);
+}
+
+subsys_initcall(apr_init);
+module_exit(apr_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Qualcomm APR Bus");
diff --git a/drivers/soc/qcom/cmd-db.c b/drivers/soc/qcom/cmd-db.c
new file mode 100644
index 000000000000..a6f646295f06
--- /dev/null
+++ b/drivers/soc/qcom/cmd-db.c
@@ -0,0 +1,317 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. */
+
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+
+#include <soc/qcom/cmd-db.h>
+
+#define NUM_PRIORITY		2
+#define MAX_SLV_ID		8
+#define SLAVE_ID_MASK		0x7
+#define SLAVE_ID_SHIFT		16
+
+/**
+ * struct entry_header: header for each entry in cmddb
+ *
+ * @id: resource's identifier
+ * @priority: unused
+ * @addr: the address of the resource
+ * @len: length of the data
+ * @offset: offset from :@data_offset, start of the data
+ */
+struct entry_header {
+	u8 id[8];
+	__le32 priority[NUM_PRIORITY];
+	__le32 addr;
+	__le16 len;
+	__le16 offset;
+};
+
+/**
+ * struct rsc_hdr: resource header information
+ *
+ * @slv_id: id for the resource
+ * @header_offset: entry's header at offset from the end of the cmd_db_header
+ * @data_offset: entry's data at offset from the end of the cmd_db_header
+ * @cnt: number of entries for HW type
+ * @version: MSB is major, LSB is minor
+ * @reserved: reserved for future use.
+ */
+struct rsc_hdr {
+	__le16 slv_id;
+	__le16 header_offset;
+	__le16 data_offset;
+	__le16 cnt;
+	__le16 version;
+	__le16 reserved[3];
+};
+
+/**
+ * struct cmd_db_header: The DB header information
+ *
+ * @version: The cmd db version
+ * @magic: constant expected in the database
+ * @header: array of resources
+ * @checksum: checksum for the header. Unused.
+ * @reserved: reserved memory
+ * @data: driver specific data
+ */
+struct cmd_db_header {
+	__le32 version;
+	u8 magic[4];
+	struct rsc_hdr header[MAX_SLV_ID];
+	__le32 checksum;
+	__le32 reserved;
+	u8 data[];
+};
+
+/**
+ * DOC: Description of the Command DB database.
+ *
+ * At the start of the command DB memory is the cmd_db_header structure.
+ * The cmd_db_header holds the version, checksum, magic key as well as an
+ * array for header for each slave (depicted by the rsc_header). Each h/w
+ * based accelerator is a 'slave' (shared resource) and has slave id indicating
+ * the type of accelerator. The rsc_header is the header for such individual
+ * slaves of a given type. The entries for each of these slaves begin at the
+ * rsc_hdr.header_offset. In addition each slave could have auxiliary data
+ * that may be needed by the driver. The data for the slave starts at the
+ * entry_header.offset to the location pointed to by the rsc_hdr.data_offset.
+ *
+ * Drivers have a stringified key to a slave/resource. They can query the slave
+ * information and get the slave id and the auxiliary data and the length of the
+ * data. Using this information, they can format the request to be sent to the
+ * h/w accelerator and request a resource state.
+ */
+
+static const u8 CMD_DB_MAGIC[] = { 0xdb, 0x30, 0x03, 0x0c };
+
+static bool cmd_db_magic_matches(const struct cmd_db_header *header)
+{
+	const u8 *magic = header->magic;
+
+	return memcmp(magic, CMD_DB_MAGIC, ARRAY_SIZE(CMD_DB_MAGIC)) == 0;
+}
+
+static struct cmd_db_header *cmd_db_header;
+
+
+static inline void *rsc_to_entry_header(struct rsc_hdr *hdr)
+{
+	u16 offset = le16_to_cpu(hdr->header_offset);
+
+	return cmd_db_header->data + offset;
+}
+
+static inline void *
+rsc_offset(struct rsc_hdr *hdr, struct entry_header *ent)
+{
+	u16 offset = le16_to_cpu(hdr->data_offset);
+	u16 loffset = le16_to_cpu(ent->offset);
+
+	return cmd_db_header->data + offset + loffset;
+}
+
+/**
+ * cmd_db_ready - Indicates if command DB is available
+ *
+ * Return: 0 on success, errno otherwise
+ */
+int cmd_db_ready(void)
+{
+	if (cmd_db_header == NULL)
+		return -EPROBE_DEFER;
+	else if (!cmd_db_magic_matches(cmd_db_header))
+		return -EINVAL;
+
+	return 0;
+}
+EXPORT_SYMBOL(cmd_db_ready);
+
+static int cmd_db_get_header(const char *id, struct entry_header *eh,
+			     struct rsc_hdr *rh)
+{
+	struct rsc_hdr *rsc_hdr;
+	struct entry_header *ent;
+	int ret, i, j;
+	u8 query[8];
+
+	ret = cmd_db_ready();
+	if (ret)
+		return ret;
+
+	if (!eh || !rh)
+		return -EINVAL;
+
+	/* Pad out query string to same length as in DB */
+	strncpy(query, id, sizeof(query));
+
+	for (i = 0; i < MAX_SLV_ID; i++) {
+		rsc_hdr = &cmd_db_header->header[i];
+		if (!rsc_hdr->slv_id)
+			break;
+
+		ent = rsc_to_entry_header(rsc_hdr);
+		for (j = 0; j < le16_to_cpu(rsc_hdr->cnt); j++, ent++) {
+			if (memcmp(ent->id, query, sizeof(ent->id)) == 0)
+				break;
+		}
+
+		if (j < le16_to_cpu(rsc_hdr->cnt)) {
+			memcpy(eh, ent, sizeof(*ent));
+			memcpy(rh, rsc_hdr, sizeof(*rh));
+			return 0;
+		}
+	}
+
+	return -ENODEV;
+}
+
+/**
+ * cmd_db_read_addr() - Query command db for resource id address.
+ *
+ * @id: resource id to query for address
+ *
+ * Return: resource address on success, 0 on error
+ *
+ * This is used to retrieve resource address based on resource
+ * id.
+ */
+u32 cmd_db_read_addr(const char *id)
+{
+	int ret;
+	struct entry_header ent;
+	struct rsc_hdr rsc_hdr;
+
+	ret = cmd_db_get_header(id, &ent, &rsc_hdr);
+
+	return ret < 0 ? 0 : le32_to_cpu(ent.addr);
+}
+EXPORT_SYMBOL(cmd_db_read_addr);
+
+/**
+ * cmd_db_read_aux_data() - Query command db for aux data.
+ *
+ *  @id: Resource to retrieve AUX Data on.
+ *  @data: Data buffer to copy returned aux data to. Returns size on NULL
+ *  @len: Caller provides size of data buffer passed in.
+ *
+ *  Return: size of data on success, errno otherwise
+ */
+int cmd_db_read_aux_data(const char *id, u8 *data, size_t len)
+{
+	int ret;
+	struct entry_header ent;
+	struct rsc_hdr rsc_hdr;
+	u16 ent_len;
+
+	if (!data)
+		return -EINVAL;
+
+	ret = cmd_db_get_header(id, &ent, &rsc_hdr);
+	if (ret)
+		return ret;
+
+	ent_len = le16_to_cpu(ent.len);
+	if (len < ent_len)
+		return -EINVAL;
+
+	len = min_t(u16, ent_len, len);
+	memcpy(data, rsc_offset(&rsc_hdr, &ent), len);
+
+	return len;
+}
+EXPORT_SYMBOL(cmd_db_read_aux_data);
+
+/**
+ * cmd_db_read_aux_data_len - Get the length of the auxiliary data stored in DB.
+ *
+ * @id: Resource to retrieve AUX Data.
+ *
+ * Return: size on success, 0 on error
+ */
+size_t cmd_db_read_aux_data_len(const char *id)
+{
+	int ret;
+	struct entry_header ent;
+	struct rsc_hdr rsc_hdr;
+
+	ret = cmd_db_get_header(id, &ent, &rsc_hdr);
+
+	return ret < 0 ? 0 : le16_to_cpu(ent.len);
+}
+EXPORT_SYMBOL(cmd_db_read_aux_data_len);
+
+/**
+ * cmd_db_read_slave_id - Get the slave ID for a given resource address
+ *
+ * @id: Resource id to query the DB for version
+ *
+ * Return: cmd_db_hw_type enum on success, CMD_DB_HW_INVALID on error
+ */
+enum cmd_db_hw_type cmd_db_read_slave_id(const char *id)
+{
+	int ret;
+	struct entry_header ent;
+	struct rsc_hdr rsc_hdr;
+	u32 addr;
+
+	ret = cmd_db_get_header(id, &ent, &rsc_hdr);
+	if (ret < 0)
+		return CMD_DB_HW_INVALID;
+
+	addr = le32_to_cpu(ent.addr);
+	return (addr >> SLAVE_ID_SHIFT) & SLAVE_ID_MASK;
+}
+EXPORT_SYMBOL(cmd_db_read_slave_id);
+
+static int cmd_db_dev_probe(struct platform_device *pdev)
+{
+	struct reserved_mem *rmem;
+	int ret = 0;
+
+	rmem = of_reserved_mem_lookup(pdev->dev.of_node);
+	if (!rmem) {
+		dev_err(&pdev->dev, "failed to acquire memory region\n");
+		return -EINVAL;
+	}
+
+	cmd_db_header = memremap(rmem->base, rmem->size, MEMREMAP_WB);
+	if (IS_ERR_OR_NULL(cmd_db_header)) {
+		ret = PTR_ERR(cmd_db_header);
+		cmd_db_header = NULL;
+		return ret;
+	}
+
+	if (!cmd_db_magic_matches(cmd_db_header)) {
+		dev_err(&pdev->dev, "Invalid Command DB Magic\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id cmd_db_match_table[] = {
+	{ .compatible = "qcom,cmd-db" },
+	{ },
+};
+
+static struct platform_driver cmd_db_dev_driver = {
+	.probe  = cmd_db_dev_probe,
+	.driver = {
+		   .name = "cmd-db",
+		   .of_match_table = cmd_db_match_table,
+	},
+};
+
+static int __init cmd_db_device_init(void)
+{
+	return platform_driver_register(&cmd_db_dev_driver);
+}
+arch_initcall(cmd_db_device_init);
diff --git a/drivers/soc/qcom/mdt_loader.c b/drivers/soc/qcom/mdt_loader.c
index 17b314d9a148..dc09d7ac905f 100644
--- a/drivers/soc/qcom/mdt_loader.c
+++ b/drivers/soc/qcom/mdt_loader.c
@@ -50,7 +50,7 @@ ssize_t qcom_mdt_get_size(const struct firmware *fw)
 	const struct elf32_phdr *phdrs;
 	const struct elf32_phdr *phdr;
 	const struct elf32_hdr *ehdr;
-	phys_addr_t min_addr = (phys_addr_t)ULLONG_MAX;
+	phys_addr_t min_addr = PHYS_ADDR_MAX;
 	phys_addr_t max_addr = 0;
 	int i;
 
@@ -97,7 +97,7 @@ int qcom_mdt_load(struct device *dev, const struct firmware *fw,
 	const struct elf32_hdr *ehdr;
 	const struct firmware *seg_fw;
 	phys_addr_t mem_reloc;
-	phys_addr_t min_addr = (phys_addr_t)ULLONG_MAX;
+	phys_addr_t min_addr = PHYS_ADDR_MAX;
 	phys_addr_t max_addr = 0;
 	size_t fw_name_len;
 	ssize_t offset;
diff --git a/drivers/soc/qcom/qcom-geni-se.c b/drivers/soc/qcom/qcom-geni-se.c
new file mode 100644
index 000000000000..feed3db21c10
--- /dev/null
+++ b/drivers/soc/qcom/qcom-geni-se.c
@@ -0,0 +1,748 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2017-2018, The Linux Foundation. All rights reserved.
+
+#include <linux/clk.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/platform_device.h>
+#include <linux/qcom-geni-se.h>
+
+/**
+ * DOC: Overview
+ *
+ * Generic Interface (GENI) Serial Engine (SE) Wrapper driver is introduced
+ * to manage GENI firmware based Qualcomm Universal Peripheral (QUP) Wrapper
+ * controller. QUP Wrapper is designed to support various serial bus protocols
+ * like UART, SPI, I2C, I3C, etc.
+ */
+
+/**
+ * DOC: Hardware description
+ *
+ * GENI based QUP is a highly-flexible and programmable module for supporting
+ * a wide range of serial interfaces like UART, SPI, I2C, I3C, etc. A single
+ * QUP module can provide upto 8 serial interfaces, using its internal
+ * serial engines. The actual configuration is determined by the target
+ * platform configuration. The protocol supported by each interface is
+ * determined by the firmware loaded to the serial engine. Each SE consists
+ * of a DMA Engine and GENI sub modules which enable serial engines to
+ * support FIFO and DMA modes of operation.
+ *
+ *
+ *                      +-----------------------------------------+
+ *                      |QUP Wrapper                              |
+ *                      |         +----------------------------+  |
+ *   --QUP & SE Clocks-->         | Serial Engine N            |  +-IO------>
+ *                      |         | ...                        |  | Interface
+ *   <---Clock Perf.----+    +----+-----------------------+    |  |
+ *     State Interface  |    | Serial Engine 1            |    |  |
+ *                      |    |                            |    |  |
+ *                      |    |                            |    |  |
+ *   <--------AHB------->    |                            |    |  |
+ *                      |    |                            +----+  |
+ *                      |    |                            |       |
+ *                      |    |                            |       |
+ *   <------SE IRQ------+    +----------------------------+       |
+ *                      |                                         |
+ *                      +-----------------------------------------+
+ *
+ *                         Figure 1: GENI based QUP Wrapper
+ *
+ * The GENI submodules include primary and secondary sequencers which are
+ * used to drive TX & RX operations. On serial interfaces that operate using
+ * master-slave model, primary sequencer drives both TX & RX operations. On
+ * serial interfaces that operate using peer-to-peer model, primary sequencer
+ * drives TX operation and secondary sequencer drives RX operation.
+ */
+
+/**
+ * DOC: Software description
+ *
+ * GENI SE Wrapper driver is structured into 2 parts:
+ *
+ * geni_wrapper represents QUP Wrapper controller. This part of the driver
+ * manages QUP Wrapper information such as hardware version, clock
+ * performance table that is common to all the internal serial engines.
+ *
+ * geni_se represents serial engine. This part of the driver manages serial
+ * engine information such as clocks, containing QUP Wrapper, etc. This part
+ * of driver also supports operations (eg. initialize the concerned serial
+ * engine, select between FIFO and DMA mode of operation etc.) that are
+ * common to all the serial engines and are independent of serial interfaces.
+ */
+
+#define MAX_CLK_PERF_LEVEL 32
+#define NUM_AHB_CLKS 2
+
+/**
+ * @struct geni_wrapper - Data structure to represent the QUP Wrapper Core
+ * @dev:		Device pointer of the QUP wrapper core
+ * @base:		Base address of this instance of QUP wrapper core
+ * @ahb_clks:		Handle to the primary & secondary AHB clocks
+ */
+struct geni_wrapper {
+	struct device *dev;
+	void __iomem *base;
+	struct clk_bulk_data ahb_clks[NUM_AHB_CLKS];
+};
+
+#define QUP_HW_VER_REG			0x4
+
+/* Common SE registers */
+#define GENI_INIT_CFG_REVISION		0x0
+#define GENI_S_INIT_CFG_REVISION	0x4
+#define GENI_OUTPUT_CTRL		0x24
+#define GENI_CGC_CTRL			0x28
+#define GENI_CLK_CTRL_RO		0x60
+#define GENI_IF_DISABLE_RO		0x64
+#define GENI_FW_S_REVISION_RO		0x6c
+#define SE_GENI_BYTE_GRAN		0x254
+#define SE_GENI_TX_PACKING_CFG0		0x260
+#define SE_GENI_TX_PACKING_CFG1		0x264
+#define SE_GENI_RX_PACKING_CFG0		0x284
+#define SE_GENI_RX_PACKING_CFG1		0x288
+#define SE_GENI_M_GP_LENGTH		0x910
+#define SE_GENI_S_GP_LENGTH		0x914
+#define SE_DMA_TX_PTR_L			0xc30
+#define SE_DMA_TX_PTR_H			0xc34
+#define SE_DMA_TX_ATTR			0xc38
+#define SE_DMA_TX_LEN			0xc3c
+#define SE_DMA_TX_IRQ_EN		0xc48
+#define SE_DMA_TX_IRQ_EN_SET		0xc4c
+#define SE_DMA_TX_IRQ_EN_CLR		0xc50
+#define SE_DMA_TX_LEN_IN		0xc54
+#define SE_DMA_TX_MAX_BURST		0xc5c
+#define SE_DMA_RX_PTR_L			0xd30
+#define SE_DMA_RX_PTR_H			0xd34
+#define SE_DMA_RX_ATTR			0xd38
+#define SE_DMA_RX_LEN			0xd3c
+#define SE_DMA_RX_IRQ_EN		0xd48
+#define SE_DMA_RX_IRQ_EN_SET		0xd4c
+#define SE_DMA_RX_IRQ_EN_CLR		0xd50
+#define SE_DMA_RX_LEN_IN		0xd54
+#define SE_DMA_RX_MAX_BURST		0xd5c
+#define SE_DMA_RX_FLUSH			0xd60
+#define SE_GSI_EVENT_EN			0xe18
+#define SE_IRQ_EN			0xe1c
+#define SE_DMA_GENERAL_CFG		0xe30
+
+/* GENI_OUTPUT_CTRL fields */
+#define DEFAULT_IO_OUTPUT_CTRL_MSK	GENMASK(6, 0)
+
+/* GENI_CGC_CTRL fields */
+#define CFG_AHB_CLK_CGC_ON		BIT(0)
+#define CFG_AHB_WR_ACLK_CGC_ON		BIT(1)
+#define DATA_AHB_CLK_CGC_ON		BIT(2)
+#define SCLK_CGC_ON			BIT(3)
+#define TX_CLK_CGC_ON			BIT(4)
+#define RX_CLK_CGC_ON			BIT(5)
+#define EXT_CLK_CGC_ON			BIT(6)
+#define PROG_RAM_HCLK_OFF		BIT(8)
+#define PROG_RAM_SCLK_OFF		BIT(9)
+#define DEFAULT_CGC_EN			GENMASK(6, 0)
+
+/* SE_GSI_EVENT_EN fields */
+#define DMA_RX_EVENT_EN			BIT(0)
+#define DMA_TX_EVENT_EN			BIT(1)
+#define GENI_M_EVENT_EN			BIT(2)
+#define GENI_S_EVENT_EN			BIT(3)
+
+/* SE_IRQ_EN fields */
+#define DMA_RX_IRQ_EN			BIT(0)
+#define DMA_TX_IRQ_EN			BIT(1)
+#define GENI_M_IRQ_EN			BIT(2)
+#define GENI_S_IRQ_EN			BIT(3)
+
+/* SE_DMA_GENERAL_CFG */
+#define DMA_RX_CLK_CGC_ON		BIT(0)
+#define DMA_TX_CLK_CGC_ON		BIT(1)
+#define DMA_AHB_SLV_CFG_ON		BIT(2)
+#define AHB_SEC_SLV_CLK_CGC_ON		BIT(3)
+#define DUMMY_RX_NON_BUFFERABLE		BIT(4)
+#define RX_DMA_ZERO_PADDING_EN		BIT(5)
+#define RX_DMA_IRQ_DELAY_MSK		GENMASK(8, 6)
+#define RX_DMA_IRQ_DELAY_SHFT		6
+
+/**
+ * geni_se_get_qup_hw_version() - Read the QUP wrapper Hardware version
+ * @se:	Pointer to the corresponding serial engine.
+ *
+ * Return: Hardware Version of the wrapper.
+ */
+u32 geni_se_get_qup_hw_version(struct geni_se *se)
+{
+	struct geni_wrapper *wrapper = se->wrapper;
+
+	return readl_relaxed(wrapper->base + QUP_HW_VER_REG);
+}
+EXPORT_SYMBOL(geni_se_get_qup_hw_version);
+
+static void geni_se_io_set_mode(void __iomem *base)
+{
+	u32 val;
+
+	val = readl_relaxed(base + SE_IRQ_EN);
+	val |= GENI_M_IRQ_EN | GENI_S_IRQ_EN;
+	val |= DMA_TX_IRQ_EN | DMA_RX_IRQ_EN;
+	writel_relaxed(val, base + SE_IRQ_EN);
+
+	val = readl_relaxed(base + SE_GENI_DMA_MODE_EN);
+	val &= ~GENI_DMA_MODE_EN;
+	writel_relaxed(val, base + SE_GENI_DMA_MODE_EN);
+
+	writel_relaxed(0, base + SE_GSI_EVENT_EN);
+}
+
+static void geni_se_io_init(void __iomem *base)
+{
+	u32 val;
+
+	val = readl_relaxed(base + GENI_CGC_CTRL);
+	val |= DEFAULT_CGC_EN;
+	writel_relaxed(val, base + GENI_CGC_CTRL);
+
+	val = readl_relaxed(base + SE_DMA_GENERAL_CFG);
+	val |= AHB_SEC_SLV_CLK_CGC_ON | DMA_AHB_SLV_CFG_ON;
+	val |= DMA_TX_CLK_CGC_ON | DMA_RX_CLK_CGC_ON;
+	writel_relaxed(val, base + SE_DMA_GENERAL_CFG);
+
+	writel_relaxed(DEFAULT_IO_OUTPUT_CTRL_MSK, base + GENI_OUTPUT_CTRL);
+	writel_relaxed(FORCE_DEFAULT, base + GENI_FORCE_DEFAULT_REG);
+}
+
+/**
+ * geni_se_init() - Initialize the GENI serial engine
+ * @se:		Pointer to the concerned serial engine.
+ * @rx_wm:	Receive watermark, in units of FIFO words.
+ * @rx_rfr_wm:	Ready-for-receive watermark, in units of FIFO words.
+ *
+ * This function is used to initialize the GENI serial engine, configure
+ * receive watermark and ready-for-receive watermarks.
+ */
+void geni_se_init(struct geni_se *se, u32 rx_wm, u32 rx_rfr)
+{
+	u32 val;
+
+	geni_se_io_init(se->base);
+	geni_se_io_set_mode(se->base);
+
+	writel_relaxed(rx_wm, se->base + SE_GENI_RX_WATERMARK_REG);
+	writel_relaxed(rx_rfr, se->base + SE_GENI_RX_RFR_WATERMARK_REG);
+
+	val = readl_relaxed(se->base + SE_GENI_M_IRQ_EN);
+	val |= M_COMMON_GENI_M_IRQ_EN;
+	writel_relaxed(val, se->base + SE_GENI_M_IRQ_EN);
+
+	val = readl_relaxed(se->base + SE_GENI_S_IRQ_EN);
+	val |= S_COMMON_GENI_S_IRQ_EN;
+	writel_relaxed(val, se->base + SE_GENI_S_IRQ_EN);
+}
+EXPORT_SYMBOL(geni_se_init);
+
+static void geni_se_select_fifo_mode(struct geni_se *se)
+{
+	u32 proto = geni_se_read_proto(se);
+	u32 val;
+
+	writel_relaxed(0, se->base + SE_GSI_EVENT_EN);
+	writel_relaxed(0xffffffff, se->base + SE_GENI_M_IRQ_CLEAR);
+	writel_relaxed(0xffffffff, se->base + SE_GENI_S_IRQ_CLEAR);
+	writel_relaxed(0xffffffff, se->base + SE_DMA_TX_IRQ_CLR);
+	writel_relaxed(0xffffffff, se->base + SE_DMA_RX_IRQ_CLR);
+	writel_relaxed(0xffffffff, se->base + SE_IRQ_EN);
+
+	val = readl_relaxed(se->base + SE_GENI_M_IRQ_EN);
+	if (proto != GENI_SE_UART) {
+		val |= M_CMD_DONE_EN | M_TX_FIFO_WATERMARK_EN;
+		val |= M_RX_FIFO_WATERMARK_EN | M_RX_FIFO_LAST_EN;
+	}
+	writel_relaxed(val, se->base + SE_GENI_M_IRQ_EN);
+
+	val = readl_relaxed(se->base + SE_GENI_S_IRQ_EN);
+	if (proto != GENI_SE_UART)
+		val |= S_CMD_DONE_EN;
+	writel_relaxed(val, se->base + SE_GENI_S_IRQ_EN);
+
+	val = readl_relaxed(se->base + SE_GENI_DMA_MODE_EN);
+	val &= ~GENI_DMA_MODE_EN;
+	writel_relaxed(val, se->base + SE_GENI_DMA_MODE_EN);
+}
+
+static void geni_se_select_dma_mode(struct geni_se *se)
+{
+	u32 val;
+
+	writel_relaxed(0, se->base + SE_GSI_EVENT_EN);
+	writel_relaxed(0xffffffff, se->base + SE_GENI_M_IRQ_CLEAR);
+	writel_relaxed(0xffffffff, se->base + SE_GENI_S_IRQ_CLEAR);
+	writel_relaxed(0xffffffff, se->base + SE_DMA_TX_IRQ_CLR);
+	writel_relaxed(0xffffffff, se->base + SE_DMA_RX_IRQ_CLR);
+	writel_relaxed(0xffffffff, se->base + SE_IRQ_EN);
+
+	val = readl_relaxed(se->base + SE_GENI_DMA_MODE_EN);
+	val |= GENI_DMA_MODE_EN;
+	writel_relaxed(val, se->base + SE_GENI_DMA_MODE_EN);
+}
+
+/**
+ * geni_se_select_mode() - Select the serial engine transfer mode
+ * @se:		Pointer to the concerned serial engine.
+ * @mode:	Transfer mode to be selected.
+ */
+void geni_se_select_mode(struct geni_se *se, enum geni_se_xfer_mode mode)
+{
+	WARN_ON(mode != GENI_SE_FIFO && mode != GENI_SE_DMA);
+
+	switch (mode) {
+	case GENI_SE_FIFO:
+		geni_se_select_fifo_mode(se);
+		break;
+	case GENI_SE_DMA:
+		geni_se_select_dma_mode(se);
+		break;
+	case GENI_SE_INVALID:
+	default:
+		break;
+	}
+}
+EXPORT_SYMBOL(geni_se_select_mode);
+
+/**
+ * DOC: Overview
+ *
+ * GENI FIFO packing is highly configurable. TX/RX packing/unpacking consist
+ * of up to 4 operations, each operation represented by 4 configuration vectors
+ * of 10 bits programmed in GENI_TX_PACKING_CFG0 and GENI_TX_PACKING_CFG1 for
+ * TX FIFO and in GENI_RX_PACKING_CFG0 and GENI_RX_PACKING_CFG1 for RX FIFO.
+ * Refer to below examples for detailed bit-field description.
+ *
+ * Example 1: word_size = 7, packing_mode = 4 x 8, msb_to_lsb = 1
+ *
+ *        +-----------+-------+-------+-------+-------+
+ *        |           | vec_0 | vec_1 | vec_2 | vec_3 |
+ *        +-----------+-------+-------+-------+-------+
+ *        | start     | 0x6   | 0xe   | 0x16  | 0x1e  |
+ *        | direction | 1     | 1     | 1     | 1     |
+ *        | length    | 6     | 6     | 6     | 6     |
+ *        | stop      | 0     | 0     | 0     | 1     |
+ *        +-----------+-------+-------+-------+-------+
+ *
+ * Example 2: word_size = 15, packing_mode = 2 x 16, msb_to_lsb = 0
+ *
+ *        +-----------+-------+-------+-------+-------+
+ *        |           | vec_0 | vec_1 | vec_2 | vec_3 |
+ *        +-----------+-------+-------+-------+-------+
+ *        | start     | 0x0   | 0x8   | 0x10  | 0x18  |
+ *        | direction | 0     | 0     | 0     | 0     |
+ *        | length    | 7     | 6     | 7     | 6     |
+ *        | stop      | 0     | 0     | 0     | 1     |
+ *        +-----------+-------+-------+-------+-------+
+ *
+ * Example 3: word_size = 23, packing_mode = 1 x 32, msb_to_lsb = 1
+ *
+ *        +-----------+-------+-------+-------+-------+
+ *        |           | vec_0 | vec_1 | vec_2 | vec_3 |
+ *        +-----------+-------+-------+-------+-------+
+ *        | start     | 0x16  | 0xe   | 0x6   | 0x0   |
+ *        | direction | 1     | 1     | 1     | 1     |
+ *        | length    | 7     | 7     | 6     | 0     |
+ *        | stop      | 0     | 0     | 1     | 0     |
+ *        +-----------+-------+-------+-------+-------+
+ *
+ */
+
+#define NUM_PACKING_VECTORS 4
+#define PACKING_START_SHIFT 5
+#define PACKING_DIR_SHIFT 4
+#define PACKING_LEN_SHIFT 1
+#define PACKING_STOP_BIT BIT(0)
+#define PACKING_VECTOR_SHIFT 10
+/**
+ * geni_se_config_packing() - Packing configuration of the serial engine
+ * @se:		Pointer to the concerned serial engine
+ * @bpw:	Bits of data per transfer word.
+ * @pack_words:	Number of words per fifo element.
+ * @msb_to_lsb:	Transfer from MSB to LSB or vice-versa.
+ * @tx_cfg:	Flag to configure the TX Packing.
+ * @rx_cfg:	Flag to configure the RX Packing.
+ *
+ * This function is used to configure the packing rules for the current
+ * transfer.
+ */
+void geni_se_config_packing(struct geni_se *se, int bpw, int pack_words,
+			    bool msb_to_lsb, bool tx_cfg, bool rx_cfg)
+{
+	u32 cfg0, cfg1, cfg[NUM_PACKING_VECTORS] = {0};
+	int len;
+	int temp_bpw = bpw;
+	int idx_start = msb_to_lsb ? bpw - 1 : 0;
+	int idx = idx_start;
+	int idx_delta = msb_to_lsb ? -BITS_PER_BYTE : BITS_PER_BYTE;
+	int ceil_bpw = ALIGN(bpw, BITS_PER_BYTE);
+	int iter = (ceil_bpw * pack_words) / BITS_PER_BYTE;
+	int i;
+
+	if (iter <= 0 || iter > NUM_PACKING_VECTORS)
+		return;
+
+	for (i = 0; i < iter; i++) {
+		len = min_t(int, temp_bpw, BITS_PER_BYTE) - 1;
+		cfg[i] = idx << PACKING_START_SHIFT;
+		cfg[i] |= msb_to_lsb << PACKING_DIR_SHIFT;
+		cfg[i] |= len << PACKING_LEN_SHIFT;
+
+		if (temp_bpw <= BITS_PER_BYTE) {
+			idx = ((i + 1) * BITS_PER_BYTE) + idx_start;
+			temp_bpw = bpw;
+		} else {
+			idx = idx + idx_delta;
+			temp_bpw = temp_bpw - BITS_PER_BYTE;
+		}
+	}
+	cfg[iter - 1] |= PACKING_STOP_BIT;
+	cfg0 = cfg[0] | (cfg[1] << PACKING_VECTOR_SHIFT);
+	cfg1 = cfg[2] | (cfg[3] << PACKING_VECTOR_SHIFT);
+
+	if (tx_cfg) {
+		writel_relaxed(cfg0, se->base + SE_GENI_TX_PACKING_CFG0);
+		writel_relaxed(cfg1, se->base + SE_GENI_TX_PACKING_CFG1);
+	}
+	if (rx_cfg) {
+		writel_relaxed(cfg0, se->base + SE_GENI_RX_PACKING_CFG0);
+		writel_relaxed(cfg1, se->base + SE_GENI_RX_PACKING_CFG1);
+	}
+
+	/*
+	 * Number of protocol words in each FIFO entry
+	 * 0 - 4x8, four words in each entry, max word size of 8 bits
+	 * 1 - 2x16, two words in each entry, max word size of 16 bits
+	 * 2 - 1x32, one word in each entry, max word size of 32 bits
+	 * 3 - undefined
+	 */
+	if (pack_words || bpw == 32)
+		writel_relaxed(bpw / 16, se->base + SE_GENI_BYTE_GRAN);
+}
+EXPORT_SYMBOL(geni_se_config_packing);
+
+static void geni_se_clks_off(struct geni_se *se)
+{
+	struct geni_wrapper *wrapper = se->wrapper;
+
+	clk_disable_unprepare(se->clk);
+	clk_bulk_disable_unprepare(ARRAY_SIZE(wrapper->ahb_clks),
+						wrapper->ahb_clks);
+}
+
+/**
+ * geni_se_resources_off() - Turn off resources associated with the serial
+ *                           engine
+ * @se:	Pointer to the concerned serial engine.
+ *
+ * Return: 0 on success, standard Linux error codes on failure/error.
+ */
+int geni_se_resources_off(struct geni_se *se)
+{
+	int ret;
+
+	ret = pinctrl_pm_select_sleep_state(se->dev);
+	if (ret)
+		return ret;
+
+	geni_se_clks_off(se);
+	return 0;
+}
+EXPORT_SYMBOL(geni_se_resources_off);
+
+static int geni_se_clks_on(struct geni_se *se)
+{
+	int ret;
+	struct geni_wrapper *wrapper = se->wrapper;
+
+	ret = clk_bulk_prepare_enable(ARRAY_SIZE(wrapper->ahb_clks),
+						wrapper->ahb_clks);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(se->clk);
+	if (ret)
+		clk_bulk_disable_unprepare(ARRAY_SIZE(wrapper->ahb_clks),
+							wrapper->ahb_clks);
+	return ret;
+}
+
+/**
+ * geni_se_resources_on() - Turn on resources associated with the serial
+ *                          engine
+ * @se:	Pointer to the concerned serial engine.
+ *
+ * Return: 0 on success, standard Linux error codes on failure/error.
+ */
+int geni_se_resources_on(struct geni_se *se)
+{
+	int ret;
+
+	ret = geni_se_clks_on(se);
+	if (ret)
+		return ret;
+
+	ret = pinctrl_pm_select_default_state(se->dev);
+	if (ret)
+		geni_se_clks_off(se);
+
+	return ret;
+}
+EXPORT_SYMBOL(geni_se_resources_on);
+
+/**
+ * geni_se_clk_tbl_get() - Get the clock table to program DFS
+ * @se:		Pointer to the concerned serial engine.
+ * @tbl:	Table in which the output is returned.
+ *
+ * This function is called by the protocol drivers to determine the different
+ * clock frequencies supported by serial engine core clock. The protocol
+ * drivers use the output to determine the clock frequency index to be
+ * programmed into DFS.
+ *
+ * Return: number of valid performance levels in the table on success,
+ *	   standard Linux error codes on failure.
+ */
+int geni_se_clk_tbl_get(struct geni_se *se, unsigned long **tbl)
+{
+	unsigned long freq = 0;
+	int i;
+
+	if (se->clk_perf_tbl) {
+		*tbl = se->clk_perf_tbl;
+		return se->num_clk_levels;
+	}
+
+	se->clk_perf_tbl = devm_kcalloc(se->dev, MAX_CLK_PERF_LEVEL,
+					sizeof(*se->clk_perf_tbl),
+					GFP_KERNEL);
+	if (!se->clk_perf_tbl)
+		return -ENOMEM;
+
+	for (i = 0; i < MAX_CLK_PERF_LEVEL; i++) {
+		freq = clk_round_rate(se->clk, freq + 1);
+		if (!freq || freq == se->clk_perf_tbl[i - 1])
+			break;
+		se->clk_perf_tbl[i] = freq;
+	}
+	se->num_clk_levels = i;
+	*tbl = se->clk_perf_tbl;
+	return se->num_clk_levels;
+}
+EXPORT_SYMBOL(geni_se_clk_tbl_get);
+
+/**
+ * geni_se_clk_freq_match() - Get the matching or closest SE clock frequency
+ * @se:		Pointer to the concerned serial engine.
+ * @req_freq:	Requested clock frequency.
+ * @index:	Index of the resultant frequency in the table.
+ * @res_freq:	Resultant frequency which matches or is closer to the
+ *		requested frequency.
+ * @exact:	Flag to indicate exact multiple requirement of the requested
+ *		frequency.
+ *
+ * This function is called by the protocol drivers to determine the matching
+ * or exact multiple of the requested frequency, as provided by the serial
+ * engine clock in order to meet the performance requirements. If there is
+ * no matching or exact multiple of the requested frequency found, then it
+ * selects the closest floor frequency, if exact flag is not set.
+ *
+ * Return: 0 on success, standard Linux error codes on failure.
+ */
+int geni_se_clk_freq_match(struct geni_se *se, unsigned long req_freq,
+			   unsigned int *index, unsigned long *res_freq,
+			   bool exact)
+{
+	unsigned long *tbl;
+	int num_clk_levels;
+	int i;
+
+	num_clk_levels = geni_se_clk_tbl_get(se, &tbl);
+	if (num_clk_levels < 0)
+		return num_clk_levels;
+
+	if (num_clk_levels == 0)
+		return -EINVAL;
+
+	*res_freq = 0;
+	for (i = 0; i < num_clk_levels; i++) {
+		if (!(tbl[i] % req_freq)) {
+			*index = i;
+			*res_freq = tbl[i];
+			return 0;
+		}
+
+		if (!(*res_freq) || ((tbl[i] > *res_freq) &&
+				     (tbl[i] < req_freq))) {
+			*index = i;
+			*res_freq = tbl[i];
+		}
+	}
+
+	if (exact)
+		return -EINVAL;
+
+	return 0;
+}
+EXPORT_SYMBOL(geni_se_clk_freq_match);
+
+#define GENI_SE_DMA_DONE_EN BIT(0)
+#define GENI_SE_DMA_EOT_EN BIT(1)
+#define GENI_SE_DMA_AHB_ERR_EN BIT(2)
+#define GENI_SE_DMA_EOT_BUF BIT(0)
+/**
+ * geni_se_tx_dma_prep() - Prepare the serial engine for TX DMA transfer
+ * @se:			Pointer to the concerned serial engine.
+ * @buf:		Pointer to the TX buffer.
+ * @len:		Length of the TX buffer.
+ * @iova:		Pointer to store the mapped DMA address.
+ *
+ * This function is used to prepare the buffers for DMA TX.
+ *
+ * Return: 0 on success, standard Linux error codes on failure.
+ */
+int geni_se_tx_dma_prep(struct geni_se *se, void *buf, size_t len,
+			dma_addr_t *iova)
+{
+	struct geni_wrapper *wrapper = se->wrapper;
+	u32 val;
+
+	*iova = dma_map_single(wrapper->dev, buf, len, DMA_TO_DEVICE);
+	if (dma_mapping_error(wrapper->dev, *iova))
+		return -EIO;
+
+	val = GENI_SE_DMA_DONE_EN;
+	val |= GENI_SE_DMA_EOT_EN;
+	val |= GENI_SE_DMA_AHB_ERR_EN;
+	writel_relaxed(val, se->base + SE_DMA_TX_IRQ_EN_SET);
+	writel_relaxed(lower_32_bits(*iova), se->base + SE_DMA_TX_PTR_L);
+	writel_relaxed(upper_32_bits(*iova), se->base + SE_DMA_TX_PTR_H);
+	writel_relaxed(GENI_SE_DMA_EOT_BUF, se->base + SE_DMA_TX_ATTR);
+	writel_relaxed(len, se->base + SE_DMA_TX_LEN);
+	return 0;
+}
+EXPORT_SYMBOL(geni_se_tx_dma_prep);
+
+/**
+ * geni_se_rx_dma_prep() - Prepare the serial engine for RX DMA transfer
+ * @se:			Pointer to the concerned serial engine.
+ * @buf:		Pointer to the RX buffer.
+ * @len:		Length of the RX buffer.
+ * @iova:		Pointer to store the mapped DMA address.
+ *
+ * This function is used to prepare the buffers for DMA RX.
+ *
+ * Return: 0 on success, standard Linux error codes on failure.
+ */
+int geni_se_rx_dma_prep(struct geni_se *se, void *buf, size_t len,
+			dma_addr_t *iova)
+{
+	struct geni_wrapper *wrapper = se->wrapper;
+	u32 val;
+
+	*iova = dma_map_single(wrapper->dev, buf, len, DMA_FROM_DEVICE);
+	if (dma_mapping_error(wrapper->dev, *iova))
+		return -EIO;
+
+	val = GENI_SE_DMA_DONE_EN;
+	val |= GENI_SE_DMA_EOT_EN;
+	val |= GENI_SE_DMA_AHB_ERR_EN;
+	writel_relaxed(val, se->base + SE_DMA_RX_IRQ_EN_SET);
+	writel_relaxed(lower_32_bits(*iova), se->base + SE_DMA_RX_PTR_L);
+	writel_relaxed(upper_32_bits(*iova), se->base + SE_DMA_RX_PTR_H);
+	/* RX does not have EOT buffer type bit. So just reset RX_ATTR */
+	writel_relaxed(0, se->base + SE_DMA_RX_ATTR);
+	writel_relaxed(len, se->base + SE_DMA_RX_LEN);
+	return 0;
+}
+EXPORT_SYMBOL(geni_se_rx_dma_prep);
+
+/**
+ * geni_se_tx_dma_unprep() - Unprepare the serial engine after TX DMA transfer
+ * @se:			Pointer to the concerned serial engine.
+ * @iova:		DMA address of the TX buffer.
+ * @len:		Length of the TX buffer.
+ *
+ * This function is used to unprepare the DMA buffers after DMA TX.
+ */
+void geni_se_tx_dma_unprep(struct geni_se *se, dma_addr_t iova, size_t len)
+{
+	struct geni_wrapper *wrapper = se->wrapper;
+
+	if (iova && !dma_mapping_error(wrapper->dev, iova))
+		dma_unmap_single(wrapper->dev, iova, len, DMA_TO_DEVICE);
+}
+EXPORT_SYMBOL(geni_se_tx_dma_unprep);
+
+/**
+ * geni_se_rx_dma_unprep() - Unprepare the serial engine after RX DMA transfer
+ * @se:			Pointer to the concerned serial engine.
+ * @iova:		DMA address of the RX buffer.
+ * @len:		Length of the RX buffer.
+ *
+ * This function is used to unprepare the DMA buffers after DMA RX.
+ */
+void geni_se_rx_dma_unprep(struct geni_se *se, dma_addr_t iova, size_t len)
+{
+	struct geni_wrapper *wrapper = se->wrapper;
+
+	if (iova && !dma_mapping_error(wrapper->dev, iova))
+		dma_unmap_single(wrapper->dev, iova, len, DMA_FROM_DEVICE);
+}
+EXPORT_SYMBOL(geni_se_rx_dma_unprep);
+
+static int geni_se_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	struct geni_wrapper *wrapper;
+	int ret;
+
+	wrapper = devm_kzalloc(dev, sizeof(*wrapper), GFP_KERNEL);
+	if (!wrapper)
+		return -ENOMEM;
+
+	wrapper->dev = dev;
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	wrapper->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(wrapper->base))
+		return PTR_ERR(wrapper->base);
+
+	wrapper->ahb_clks[0].id = "m-ahb";
+	wrapper->ahb_clks[1].id = "s-ahb";
+	ret = devm_clk_bulk_get(dev, NUM_AHB_CLKS, wrapper->ahb_clks);
+	if (ret) {
+		dev_err(dev, "Err getting AHB clks %d\n", ret);
+		return ret;
+	}
+
+	dev_set_drvdata(dev, wrapper);
+	dev_dbg(dev, "GENI SE Driver probed\n");
+	return devm_of_platform_populate(dev);
+}
+
+static const struct of_device_id geni_se_dt_match[] = {
+	{ .compatible = "qcom,geni-se-qup", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, geni_se_dt_match);
+
+static struct platform_driver geni_se_driver = {
+	.driver = {
+		.name = "geni_se_qup",
+		.of_match_table = geni_se_dt_match,
+	},
+	.probe = geni_se_probe,
+};
+module_platform_driver(geni_se_driver);
+
+MODULE_DESCRIPTION("GENI Serial Engine Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/soc/qcom/qmi_interface.c b/drivers/soc/qcom/qmi_interface.c
index 321982277697..938ca41c56cd 100644
--- a/drivers/soc/qcom/qmi_interface.c
+++ b/drivers/soc/qcom/qmi_interface.c
@@ -639,10 +639,11 @@ int qmi_handle_init(struct qmi_handle *qmi, size_t recv_buf_size,
 	if (ops)
 		qmi->ops = *ops;
 
+	/* Make room for the header */
+	recv_buf_size += sizeof(struct qmi_header);
+	/* Must also be sufficient to hold a control packet */
 	if (recv_buf_size < sizeof(struct qrtr_ctrl_pkt))
 		recv_buf_size = sizeof(struct qrtr_ctrl_pkt);
-	else
-		recv_buf_size += sizeof(struct qmi_header);
 
 	qmi->recv_buf_size = recv_buf_size;
 	qmi->recv_buf = kzalloc(recv_buf_size, GFP_KERNEL);
diff --git a/drivers/soc/qcom/smd-rpm.c b/drivers/soc/qcom/smd-rpm.c
index c2346752b3ea..93517ed83355 100644
--- a/drivers/soc/qcom/smd-rpm.c
+++ b/drivers/soc/qcom/smd-rpm.c
@@ -226,6 +226,7 @@ static const struct of_device_id qcom_smd_rpm_of_match[] = {
 	{ .compatible = "qcom,rpm-msm8916" },
 	{ .compatible = "qcom,rpm-msm8974" },
 	{ .compatible = "qcom,rpm-msm8996" },
+	{ .compatible = "qcom,rpm-msm8998" },
 	{}
 };
 MODULE_DEVICE_TABLE(of, qcom_smd_rpm_of_match);
diff --git a/drivers/soc/qcom/smem.c b/drivers/soc/qcom/smem.c
index 0b94d62fad2b..70b2ee80d6bd 100644
--- a/drivers/soc/qcom/smem.c
+++ b/drivers/soc/qcom/smem.c
@@ -280,7 +280,7 @@ struct qcom_smem {
 	struct smem_region regions[0];
 };
 
-static struct smem_private_entry *
+static void *
 phdr_to_last_uncached_entry(struct smem_partition_header *phdr)
 {
 	void *p = phdr;
@@ -288,15 +288,18 @@ phdr_to_last_uncached_entry(struct smem_partition_header *phdr)
 	return p + le32_to_cpu(phdr->offset_free_uncached);
 }
 
-static void *phdr_to_first_cached_entry(struct smem_partition_header *phdr,
+static struct smem_private_entry *
+phdr_to_first_cached_entry(struct smem_partition_header *phdr,
 					size_t cacheline)
 {
 	void *p = phdr;
+	struct smem_private_entry *e;
 
-	return p + le32_to_cpu(phdr->size) - ALIGN(sizeof(*phdr), cacheline);
+	return p + le32_to_cpu(phdr->size) - ALIGN(sizeof(*e), cacheline);
 }
 
-static void *phdr_to_last_cached_entry(struct smem_partition_header *phdr)
+static void *
+phdr_to_last_cached_entry(struct smem_partition_header *phdr)
 {
 	void *p = phdr;
 
@@ -361,14 +364,14 @@ static int qcom_smem_alloc_private(struct qcom_smem *smem,
 	end = phdr_to_last_uncached_entry(phdr);
 	cached = phdr_to_last_cached_entry(phdr);
 
-	while (hdr < end) {
-		if (hdr->canary != SMEM_PRIVATE_CANARY) {
-			dev_err(smem->dev,
-				"Found invalid canary in hosts %d:%d partition\n",
-				phdr->host0, phdr->host1);
-			return -EINVAL;
-		}
+	if (smem->global_partition) {
+		dev_err(smem->dev, "Already found the global partition\n");
+		return -EINVAL;
+	}
 
+	while (hdr < end) {
+		if (hdr->canary != SMEM_PRIVATE_CANARY)
+			goto bad_canary;
 		if (le16_to_cpu(hdr->item) == item)
 			return -EEXIST;
 
@@ -377,7 +380,7 @@ static int qcom_smem_alloc_private(struct qcom_smem *smem,
 
 	/* Check that we don't grow into the cached region */
 	alloc_size = sizeof(*hdr) + ALIGN(size, 8);
-	if ((void *)hdr + alloc_size >= cached) {
+	if ((void *)hdr + alloc_size > cached) {
 		dev_err(smem->dev, "Out of memory\n");
 		return -ENOSPC;
 	}
@@ -397,6 +400,11 @@ static int qcom_smem_alloc_private(struct qcom_smem *smem,
 	le32_add_cpu(&phdr->offset_free_uncached, alloc_size);
 
 	return 0;
+bad_canary:
+	dev_err(smem->dev, "Found invalid canary in hosts %hu:%hu partition\n",
+		le16_to_cpu(phdr->host0), le16_to_cpu(phdr->host1));
+
+	return -EINVAL;
 }
 
 static int qcom_smem_alloc_global(struct qcom_smem *smem,
@@ -560,8 +568,8 @@ static void *qcom_smem_get_private(struct qcom_smem *smem,
 	return ERR_PTR(-ENOENT);
 
 invalid_canary:
-	dev_err(smem->dev, "Found invalid canary in hosts %d:%d partition\n",
-			phdr->host0, phdr->host1);
+	dev_err(smem->dev, "Found invalid canary in hosts %hu:%hu partition\n",
+			le16_to_cpu(phdr->host0), le16_to_cpu(phdr->host1));
 
 	return ERR_PTR(-EINVAL);
 }
@@ -647,6 +655,33 @@ int qcom_smem_get_free_space(unsigned host)
 }
 EXPORT_SYMBOL(qcom_smem_get_free_space);
 
+/**
+ * qcom_smem_virt_to_phys() - return the physical address associated
+ * with an smem item pointer (previously returned by qcom_smem_get()
+ * @p:	the virtual address to convert
+ *
+ * Returns 0 if the pointer provided is not within any smem region.
+ */
+phys_addr_t qcom_smem_virt_to_phys(void *p)
+{
+	unsigned i;
+
+	for (i = 0; i < __smem->num_regions; i++) {
+		struct smem_region *region = &__smem->regions[i];
+
+		if (p < region->virt_base)
+			continue;
+		if (p < region->virt_base + region->size) {
+			u64 offset = p - region->virt_base;
+
+			return (phys_addr_t)region->aux_base + offset;
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(qcom_smem_virt_to_phys);
+
 static int qcom_smem_get_sbl_version(struct qcom_smem *smem)
 {
 	struct smem_header *header;
@@ -695,9 +730,10 @@ static u32 qcom_smem_get_item_count(struct qcom_smem *smem)
 static int qcom_smem_set_global_partition(struct qcom_smem *smem)
 {
 	struct smem_partition_header *header;
-	struct smem_ptable_entry *entry = NULL;
+	struct smem_ptable_entry *entry;
 	struct smem_ptable *ptable;
 	u32 host0, host1, size;
+	bool found = false;
 	int i;
 
 	ptable = qcom_smem_get_ptable(smem);
@@ -709,11 +745,13 @@ static int qcom_smem_set_global_partition(struct qcom_smem *smem)
 		host0 = le16_to_cpu(entry->host0);
 		host1 = le16_to_cpu(entry->host1);
 
-		if (host0 == SMEM_GLOBAL_HOST && host0 == host1)
+		if (host0 == SMEM_GLOBAL_HOST && host0 == host1) {
+			found = true;
 			break;
+		}
 	}
 
-	if (!entry) {
+	if (!found) {
 		dev_err(smem->dev, "Missing entry for global partition\n");
 		return -EINVAL;
 	}
@@ -723,11 +761,6 @@ static int qcom_smem_set_global_partition(struct qcom_smem *smem)
 		return -EINVAL;
 	}
 
-	if (smem->global_partition) {
-		dev_err(smem->dev, "Already found the global partition\n");
-		return -EINVAL;
-	}
-
 	header = smem->regions[0].virt_base + le32_to_cpu(entry->offset);
 	host0 = le16_to_cpu(header->host0);
 	host1 = le16_to_cpu(header->host1);
diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig
index 3bbe6114a420..1d824cbd462d 100644
--- a/drivers/soc/renesas/Kconfig
+++ b/drivers/soc/renesas/Kconfig
@@ -4,9 +4,11 @@ config SOC_RENESAS
 	select SOC_BUS
 	select RST_RCAR if ARCH_RCAR_GEN1 || ARCH_RCAR_GEN2 || \
 			   ARCH_R8A7795 || ARCH_R8A7796 || ARCH_R8A77965 || \
-			   ARCH_R8A77970 || ARCH_R8A77980 || ARCH_R8A77995
+			   ARCH_R8A77970 || ARCH_R8A77980 || ARCH_R8A77990 || \
+			   ARCH_R8A77995
 	select SYSC_R8A7743 if ARCH_R8A7743
 	select SYSC_R8A7745 if ARCH_R8A7745
+	select SYSC_R8A77470 if ARCH_R8A77470
 	select SYSC_R8A7779 if ARCH_R8A7779
 	select SYSC_R8A7790 if ARCH_R8A7790
 	select SYSC_R8A7791 if ARCH_R8A7791 || ARCH_R8A7793
@@ -17,6 +19,7 @@ config SOC_RENESAS
 	select SYSC_R8A77965 if ARCH_R8A77965
 	select SYSC_R8A77970 if ARCH_R8A77970
 	select SYSC_R8A77980 if ARCH_R8A77980
+	select SYSC_R8A77990 if ARCH_R8A77990
 	select SYSC_R8A77995 if ARCH_R8A77995
 
 if SOC_RENESAS
@@ -30,6 +33,10 @@ config SYSC_R8A7745
 	bool "RZ/G1E System Controller support" if COMPILE_TEST
 	select SYSC_RCAR
 
+config SYSC_R8A77470
+	bool "RZ/G1C System Controller support" if COMPILE_TEST
+	select SYSC_RCAR
+
 config SYSC_R8A7779
 	bool "R-Car H1 System Controller support" if COMPILE_TEST
 	select SYSC_RCAR
@@ -70,6 +77,10 @@ config SYSC_R8A77980
 	bool "R-Car V3H System Controller support" if COMPILE_TEST
 	select SYSC_RCAR
 
+config SYSC_R8A77990
+	bool "R-Car E3 System Controller support" if COMPILE_TEST
+	select SYSC_RCAR
+
 config SYSC_R8A77995
 	bool "R-Car D3 System Controller support" if COMPILE_TEST
 	select SYSC_RCAR
diff --git a/drivers/soc/renesas/Makefile b/drivers/soc/renesas/Makefile
index ccb5ec57a262..7dc0f20d7907 100644
--- a/drivers/soc/renesas/Makefile
+++ b/drivers/soc/renesas/Makefile
@@ -5,6 +5,7 @@ obj-$(CONFIG_SOC_RENESAS)	+= renesas-soc.o
 # SoC
 obj-$(CONFIG_SYSC_R8A7743)	+= r8a7743-sysc.o
 obj-$(CONFIG_SYSC_R8A7745)	+= r8a7745-sysc.o
+obj-$(CONFIG_SYSC_R8A77470)	+= r8a77470-sysc.o
 obj-$(CONFIG_SYSC_R8A7779)	+= r8a7779-sysc.o
 obj-$(CONFIG_SYSC_R8A7790)	+= r8a7790-sysc.o
 obj-$(CONFIG_SYSC_R8A7791)	+= r8a7791-sysc.o
@@ -15,6 +16,7 @@ obj-$(CONFIG_SYSC_R8A7796)	+= r8a7796-sysc.o
 obj-$(CONFIG_SYSC_R8A77965)	+= r8a77965-sysc.o
 obj-$(CONFIG_SYSC_R8A77970)	+= r8a77970-sysc.o
 obj-$(CONFIG_SYSC_R8A77980)	+= r8a77980-sysc.o
+obj-$(CONFIG_SYSC_R8A77990)	+= r8a77990-sysc.o
 obj-$(CONFIG_SYSC_R8A77995)	+= r8a77995-sysc.o
 
 # Family
diff --git a/drivers/soc/renesas/r8a77470-sysc.c b/drivers/soc/renesas/r8a77470-sysc.c
new file mode 100644
index 000000000000..cfa015e208ef
--- /dev/null
+++ b/drivers/soc/renesas/r8a77470-sysc.c
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Renesas RZ/G1C System Controller
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ */
+
+#include <linux/bug.h>
+#include <linux/kernel.h>
+
+#include <dt-bindings/power/r8a77470-sysc.h>
+
+#include "rcar-sysc.h"
+
+static const struct rcar_sysc_area r8a77470_areas[] __initconst = {
+	{ "always-on",	    0, 0, R8A77470_PD_ALWAYS_ON, -1, PD_ALWAYS_ON },
+	{ "ca7-scu",	0x100, 0, R8A77470_PD_CA7_SCU,	R8A77470_PD_ALWAYS_ON,
+	  PD_SCU },
+	{ "ca7-cpu0",	0x1c0, 0, R8A77470_PD_CA7_CPU0,	R8A77470_PD_CA7_SCU,
+	  PD_CPU_NOCR },
+	{ "ca7-cpu1",	0x1c0, 1, R8A77470_PD_CA7_CPU1,	R8A77470_PD_CA7_SCU,
+	  PD_CPU_NOCR },
+	{ "sgx",	 0xc0, 0, R8A77470_PD_SGX, R8A77470_PD_ALWAYS_ON },
+};
+
+const struct rcar_sysc_info r8a77470_sysc_info __initconst = {
+	.areas = r8a77470_areas,
+	.num_areas = ARRAY_SIZE(r8a77470_areas),
+};
diff --git a/drivers/soc/renesas/r8a77990-sysc.c b/drivers/soc/renesas/r8a77990-sysc.c
new file mode 100644
index 000000000000..15579ebc5ed2
--- /dev/null
+++ b/drivers/soc/renesas/r8a77990-sysc.c
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Renesas R-Car E3 System Controller
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ */
+
+#include <linux/bug.h>
+#include <linux/kernel.h>
+#include <linux/sys_soc.h>
+
+#include <dt-bindings/power/r8a77990-sysc.h>
+
+#include "rcar-sysc.h"
+
+static struct rcar_sysc_area r8a77990_areas[] __initdata = {
+	{ "always-on",	    0, 0, R8A77990_PD_ALWAYS_ON, -1, PD_ALWAYS_ON },
+	{ "ca53-scu",	0x140, 0, R8A77990_PD_CA53_SCU,  R8A77990_PD_ALWAYS_ON,
+	  PD_SCU },
+	{ "ca53-cpu0",	0x200, 0, R8A77990_PD_CA53_CPU0, R8A77990_PD_CA53_SCU,
+	  PD_CPU_NOCR },
+	{ "ca53-cpu1",	0x200, 1, R8A77990_PD_CA53_CPU1, R8A77990_PD_CA53_SCU,
+	  PD_CPU_NOCR },
+	{ "cr7",	0x240, 0, R8A77990_PD_CR7,	R8A77990_PD_ALWAYS_ON },
+	{ "a3vc",	0x380, 0, R8A77990_PD_A3VC,	R8A77990_PD_ALWAYS_ON },
+	{ "a2vc1",	0x3c0, 1, R8A77990_PD_A2VC1,	R8A77990_PD_A3VC },
+	{ "3dg-a",	0x100, 0, R8A77990_PD_3DG_A,	R8A77990_PD_ALWAYS_ON },
+	{ "3dg-b",	0x100, 1, R8A77990_PD_3DG_B,	R8A77990_PD_3DG_A },
+};
+
+static void __init rcar_sysc_fix_parent(struct rcar_sysc_area *areas,
+					unsigned int num_areas, u8 id,
+					int new_parent)
+{
+	unsigned int i;
+
+	for (i = 0; i < num_areas; i++)
+		if (areas[i].isr_bit == id) {
+			areas[i].parent = new_parent;
+			return;
+		}
+}
+
+/* Fixups for R-Car E3 ES1.0 revision */
+static const struct soc_device_attribute r8a77990[] __initconst = {
+	{ .soc_id = "r8a77990", .revision = "ES1.0" },
+	{ /* sentinel */ }
+};
+
+static int __init r8a77990_sysc_init(void)
+{
+	if (soc_device_match(r8a77990)) {
+		rcar_sysc_fix_parent(r8a77990_areas,
+				     ARRAY_SIZE(r8a77990_areas),
+				     R8A77990_PD_3DG_A, R8A77990_PD_3DG_B);
+		rcar_sysc_fix_parent(r8a77990_areas,
+				     ARRAY_SIZE(r8a77990_areas),
+				     R8A77990_PD_3DG_B, R8A77990_PD_ALWAYS_ON);
+	}
+
+	return 0;
+}
+
+const struct rcar_sysc_info r8a77990_sysc_info __initconst = {
+	.init = r8a77990_sysc_init,
+	.areas = r8a77990_areas,
+	.num_areas = ARRAY_SIZE(r8a77990_areas),
+};
diff --git a/drivers/soc/renesas/r8a77995-sysc.c b/drivers/soc/renesas/r8a77995-sysc.c
index f718429cab02..1b2ef415bbe1 100644
--- a/drivers/soc/renesas/r8a77995-sysc.c
+++ b/drivers/soc/renesas/r8a77995-sysc.c
@@ -10,13 +10,12 @@
 
 #include <linux/bug.h>
 #include <linux/kernel.h>
-#include <linux/sys_soc.h>
 
 #include <dt-bindings/power/r8a77995-sysc.h>
 
 #include "rcar-sysc.h"
 
-static struct rcar_sysc_area r8a77995_areas[] __initdata = {
+static const struct rcar_sysc_area r8a77995_areas[] __initconst = {
 	{ "always-on",     0, 0, R8A77995_PD_ALWAYS_ON, -1, PD_ALWAYS_ON },
 	{ "ca53-scu",  0x140, 0, R8A77995_PD_CA53_SCU,  R8A77995_PD_ALWAYS_ON,
 	  PD_SCU },
diff --git a/drivers/soc/renesas/rcar-rst.c b/drivers/soc/renesas/rcar-rst.c
index 8e9cb7996ab0..d9c1034e70e9 100644
--- a/drivers/soc/renesas/rcar-rst.c
+++ b/drivers/soc/renesas/rcar-rst.c
@@ -44,6 +44,7 @@ static const struct of_device_id rcar_rst_matches[] __initconst = {
 	/* RZ/G is handled like R-Car Gen2 */
 	{ .compatible = "renesas,r8a7743-rst", .data = &rcar_rst_gen2 },
 	{ .compatible = "renesas,r8a7745-rst", .data = &rcar_rst_gen2 },
+	{ .compatible = "renesas,r8a77470-rst", .data = &rcar_rst_gen2 },
 	/* R-Car Gen1 */
 	{ .compatible = "renesas,r8a7778-reset-wdt", .data = &rcar_rst_gen1 },
 	{ .compatible = "renesas,r8a7779-reset-wdt", .data = &rcar_rst_gen1 },
@@ -59,6 +60,7 @@ static const struct of_device_id rcar_rst_matches[] __initconst = {
 	{ .compatible = "renesas,r8a77965-rst", .data = &rcar_rst_gen3 },
 	{ .compatible = "renesas,r8a77970-rst", .data = &rcar_rst_gen3 },
 	{ .compatible = "renesas,r8a77980-rst", .data = &rcar_rst_gen3 },
+	{ .compatible = "renesas,r8a77990-rst", .data = &rcar_rst_gen3 },
 	{ .compatible = "renesas,r8a77995-rst", .data = &rcar_rst_gen3 },
 	{ /* sentinel */ }
 };
diff --git a/drivers/soc/renesas/rcar-sysc.c b/drivers/soc/renesas/rcar-sysc.c
index faf20e719361..95120acc4d80 100644
--- a/drivers/soc/renesas/rcar-sysc.c
+++ b/drivers/soc/renesas/rcar-sysc.c
@@ -261,6 +261,9 @@ static const struct of_device_id rcar_sysc_matches[] __initconst = {
 #ifdef CONFIG_SYSC_R8A7745
 	{ .compatible = "renesas,r8a7745-sysc", .data = &r8a7745_sysc_info },
 #endif
+#ifdef CONFIG_SYSC_R8A77470
+	{ .compatible = "renesas,r8a77470-sysc", .data = &r8a77470_sysc_info },
+#endif
 #ifdef CONFIG_SYSC_R8A7779
 	{ .compatible = "renesas,r8a7779-sysc", .data = &r8a7779_sysc_info },
 #endif
@@ -293,6 +296,9 @@ static const struct of_device_id rcar_sysc_matches[] __initconst = {
 #ifdef CONFIG_SYSC_R8A77980
 	{ .compatible = "renesas,r8a77980-sysc", .data = &r8a77980_sysc_info },
 #endif
+#ifdef CONFIG_SYSC_R8A77990
+	{ .compatible = "renesas,r8a77990-sysc", .data = &r8a77990_sysc_info },
+#endif
 #ifdef CONFIG_SYSC_R8A77995
 	{ .compatible = "renesas,r8a77995-sysc", .data = &r8a77995_sysc_info },
 #endif
diff --git a/drivers/soc/renesas/rcar-sysc.h b/drivers/soc/renesas/rcar-sysc.h
index dcdc9ec8eba7..a22e7cf25e30 100644
--- a/drivers/soc/renesas/rcar-sysc.h
+++ b/drivers/soc/renesas/rcar-sysc.h
@@ -51,6 +51,7 @@ struct rcar_sysc_info {
 
 extern const struct rcar_sysc_info r8a7743_sysc_info;
 extern const struct rcar_sysc_info r8a7745_sysc_info;
+extern const struct rcar_sysc_info r8a77470_sysc_info;
 extern const struct rcar_sysc_info r8a7779_sysc_info;
 extern const struct rcar_sysc_info r8a7790_sysc_info;
 extern const struct rcar_sysc_info r8a7791_sysc_info;
@@ -61,6 +62,7 @@ extern const struct rcar_sysc_info r8a7796_sysc_info;
 extern const struct rcar_sysc_info r8a77965_sysc_info;
 extern const struct rcar_sysc_info r8a77970_sysc_info;
 extern const struct rcar_sysc_info r8a77980_sysc_info;
+extern const struct rcar_sysc_info r8a77990_sysc_info;
 extern const struct rcar_sysc_info r8a77995_sysc_info;
 
 
diff --git a/drivers/soc/renesas/renesas-soc.c b/drivers/soc/renesas/renesas-soc.c
index ea71c413c926..d44d0e687ab8 100644
--- a/drivers/soc/renesas/renesas-soc.c
+++ b/drivers/soc/renesas/renesas-soc.c
@@ -100,6 +100,11 @@ static const struct renesas_soc soc_rz_g1e __initconst __maybe_unused = {
 	.id	= 0x4c,
 };
 
+static const struct renesas_soc soc_rz_g1c __initconst __maybe_unused = {
+	.family	= &fam_rzg,
+	.id	= 0x53,
+};
+
 static const struct renesas_soc soc_rcar_m1a __initconst __maybe_unused = {
 	.family	= &fam_rcar_gen1,
 };
@@ -159,6 +164,11 @@ static const struct renesas_soc soc_rcar_v3h __initconst __maybe_unused = {
 	.id	= 0x56,
 };
 
+static const struct renesas_soc soc_rcar_e3 __initconst __maybe_unused = {
+	.family	= &fam_rcar_gen3,
+	.id	= 0x57,
+};
+
 static const struct renesas_soc soc_rcar_d3 __initconst __maybe_unused = {
 	.family	= &fam_rcar_gen3,
 	.id	= 0x58,
@@ -192,6 +202,9 @@ static const struct of_device_id renesas_socs[] __initconst = {
 #ifdef CONFIG_ARCH_R8A7745
 	{ .compatible = "renesas,r8a7745",	.data = &soc_rz_g1e },
 #endif
+#ifdef CONFIG_ARCH_R8A77470
+	{ .compatible = "renesas,r8a77470",	.data = &soc_rz_g1c },
+#endif
 #ifdef CONFIG_ARCH_R8A7778
 	{ .compatible = "renesas,r8a7778",	.data = &soc_rcar_m1a },
 #endif
@@ -228,6 +241,9 @@ static const struct of_device_id renesas_socs[] __initconst = {
 #ifdef CONFIG_ARCH_R8A77980
 	{ .compatible = "renesas,r8a77980",	.data = &soc_rcar_v3h },
 #endif
+#ifdef CONFIG_ARCH_R8A77990
+	{ .compatible = "renesas,r8a77990",	.data = &soc_rcar_e3 },
+#endif
 #ifdef CONFIG_ARCH_R8A77995
 	{ .compatible = "renesas,r8a77995",	.data = &soc_rcar_d3 },
 #endif
diff --git a/drivers/soc/rockchip/pm_domains.c b/drivers/soc/rockchip/pm_domains.c
index 53efc386b1ad..78dc5adb875f 100644
--- a/drivers/soc/rockchip/pm_domains.c
+++ b/drivers/soc/rockchip/pm_domains.c
@@ -14,10 +14,15 @@
 #include <linux/pm_clock.h>
 #include <linux/pm_domain.h>
 #include <linux/of_address.h>
+#include <linux/of_clk.h>
 #include <linux/of_platform.h>
 #include <linux/clk.h>
 #include <linux/regmap.h>
 #include <linux/mfd/syscon.h>
+#include <dt-bindings/power/px30-power.h>
+#include <dt-bindings/power/rk3036-power.h>
+#include <dt-bindings/power/rk3128-power.h>
+#include <dt-bindings/power/rk3228-power.h>
 #include <dt-bindings/power/rk3288-power.h>
 #include <dt-bindings/power/rk3328-power.h>
 #include <dt-bindings/power/rk3366-power.h>
@@ -103,6 +108,18 @@ struct rockchip_pmu {
 	.active_wakeup = wakeup,			\
 }
 
+#define DOMAIN_RK3036(req, ack, idle, wakeup)		\
+{							\
+	.req_mask = (req >= 0) ? BIT(req) : 0,		\
+	.req_w_mask = (req >= 0) ?  BIT(req + 16) : 0,	\
+	.ack_mask = (ack >= 0) ? BIT(ack) : 0,		\
+	.idle_mask = (idle >= 0) ? BIT(idle) : 0,	\
+	.active_wakeup = wakeup,			\
+}
+
+#define DOMAIN_PX30(pwr, status, req, wakeup)		\
+	DOMAIN_M(pwr, status, req, (req) + 16, req, wakeup)
+
 #define DOMAIN_RK3288(pwr, status, req, wakeup)		\
 	DOMAIN(pwr, status, req, req, (req) + 16, wakeup)
 
@@ -255,7 +272,7 @@ static void rockchip_do_pmu_set_power_domain(struct rockchip_pm_domain *pd,
 		return;
 	else if (pd->info->pwr_w_mask)
 		regmap_write(pmu->regmap, pmu->info->pwr_offset,
-			     on ? pd->info->pwr_mask :
+			     on ? pd->info->pwr_w_mask :
 			     (pd->info->pwr_mask | pd->info->pwr_w_mask));
 	else
 		regmap_update_bits(pmu->regmap, pmu->info->pwr_offset,
@@ -400,8 +417,7 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu,
 	pd->info = pd_info;
 	pd->pmu = pmu;
 
-	pd->num_clks = of_count_phandle_with_args(node, "clocks",
-						  "#clock-cells");
+	pd->num_clks = of_clk_get_parent_count(node);
 	if (pd->num_clks > 0) {
 		pd->clks = devm_kcalloc(pmu->dev, pd->num_clks,
 					sizeof(*pd->clks), GFP_KERNEL);
@@ -701,6 +717,49 @@ static int rockchip_pm_domain_probe(struct platform_device *pdev)
 	return error;
 }
 
+static const struct rockchip_domain_info px30_pm_domains[] = {
+	[PX30_PD_USB]		= DOMAIN_PX30(5, 5, 10, false),
+	[PX30_PD_SDCARD]	= DOMAIN_PX30(8, 8, 9, false),
+	[PX30_PD_GMAC]		= DOMAIN_PX30(10, 10, 6, false),
+	[PX30_PD_MMC_NAND]	= DOMAIN_PX30(11, 11, 5, false),
+	[PX30_PD_VPU]		= DOMAIN_PX30(12, 12, 14, false),
+	[PX30_PD_VO]		= DOMAIN_PX30(13, 13, 7, false),
+	[PX30_PD_VI]		= DOMAIN_PX30(14, 14, 8, false),
+	[PX30_PD_GPU]		= DOMAIN_PX30(15, 15, 2, false),
+};
+
+static const struct rockchip_domain_info rk3036_pm_domains[] = {
+	[RK3036_PD_MSCH]	= DOMAIN_RK3036(14, 23, 30, true),
+	[RK3036_PD_CORE]	= DOMAIN_RK3036(13, 17, 24, false),
+	[RK3036_PD_PERI]	= DOMAIN_RK3036(12, 18, 25, false),
+	[RK3036_PD_VIO]		= DOMAIN_RK3036(11, 19, 26, false),
+	[RK3036_PD_VPU]		= DOMAIN_RK3036(10, 20, 27, false),
+	[RK3036_PD_GPU]		= DOMAIN_RK3036(9, 21, 28, false),
+	[RK3036_PD_SYS]		= DOMAIN_RK3036(8, 22, 29, false),
+};
+
+static const struct rockchip_domain_info rk3128_pm_domains[] = {
+	[RK3128_PD_CORE]	= DOMAIN_RK3288(0, 0, 4, false),
+	[RK3128_PD_MSCH]	= DOMAIN_RK3288(-1, -1, 6, true),
+	[RK3128_PD_VIO]		= DOMAIN_RK3288(3, 3, 2, false),
+	[RK3128_PD_VIDEO]	= DOMAIN_RK3288(2, 2, 1, false),
+	[RK3128_PD_GPU]		= DOMAIN_RK3288(1, 1, 3, false),
+};
+
+static const struct rockchip_domain_info rk3228_pm_domains[] = {
+	[RK3228_PD_CORE]	= DOMAIN_RK3036(0, 0, 16, true),
+	[RK3228_PD_MSCH]	= DOMAIN_RK3036(1, 1, 17, true),
+	[RK3228_PD_BUS]		= DOMAIN_RK3036(2, 2, 18, true),
+	[RK3228_PD_SYS]		= DOMAIN_RK3036(3, 3, 19, true),
+	[RK3228_PD_VIO]		= DOMAIN_RK3036(4, 4, 20, false),
+	[RK3228_PD_VOP]		= DOMAIN_RK3036(5, 5, 21, false),
+	[RK3228_PD_VPU]		= DOMAIN_RK3036(6, 6, 22, false),
+	[RK3228_PD_RKVDEC]	= DOMAIN_RK3036(7, 7, 23, false),
+	[RK3228_PD_GPU]		= DOMAIN_RK3036(8, 8, 24, false),
+	[RK3228_PD_PERI]	= DOMAIN_RK3036(9, 9, 25, true),
+	[RK3228_PD_GMAC]	= DOMAIN_RK3036(10, 10, 26, false),
+};
+
 static const struct rockchip_domain_info rk3288_pm_domains[] = {
 	[RK3288_PD_VIO]		= DOMAIN_RK3288(7, 7, 4, false),
 	[RK3288_PD_HEVC]	= DOMAIN_RK3288(14, 10, 9, false),
@@ -768,6 +827,46 @@ static const struct rockchip_domain_info rk3399_pm_domains[] = {
 	[RK3399_PD_SDIOAUDIO]	= DOMAIN_RK3399(31, 31, 29, true),
 };
 
+static const struct rockchip_pmu_info px30_pmu = {
+	.pwr_offset = 0x18,
+	.status_offset = 0x20,
+	.req_offset = 0x64,
+	.idle_offset = 0x6c,
+	.ack_offset = 0x6c,
+
+	.num_domains = ARRAY_SIZE(px30_pm_domains),
+	.domain_info = px30_pm_domains,
+};
+
+static const struct rockchip_pmu_info rk3036_pmu = {
+	.req_offset = 0x148,
+	.idle_offset = 0x14c,
+	.ack_offset = 0x14c,
+
+	.num_domains = ARRAY_SIZE(rk3036_pm_domains),
+	.domain_info = rk3036_pm_domains,
+};
+
+static const struct rockchip_pmu_info rk3128_pmu = {
+	.pwr_offset = 0x04,
+	.status_offset = 0x08,
+	.req_offset = 0x0c,
+	.idle_offset = 0x10,
+	.ack_offset = 0x10,
+
+	.num_domains = ARRAY_SIZE(rk3128_pm_domains),
+	.domain_info = rk3128_pm_domains,
+};
+
+static const struct rockchip_pmu_info rk3228_pmu = {
+	.req_offset = 0x40c,
+	.idle_offset = 0x488,
+	.ack_offset = 0x488,
+
+	.num_domains = ARRAY_SIZE(rk3228_pm_domains),
+	.domain_info = rk3228_pm_domains,
+};
+
 static const struct rockchip_pmu_info rk3288_pmu = {
 	.pwr_offset = 0x08,
 	.status_offset = 0x0c,
@@ -842,6 +941,22 @@ static const struct rockchip_pmu_info rk3399_pmu = {
 };
 
 static const struct of_device_id rockchip_pm_domain_dt_match[] = {
+	{
+		.compatible = "rockchip,px30-power-controller",
+		.data = (void *)&px30_pmu,
+	},
+	{
+		.compatible = "rockchip,rk3036-power-controller",
+		.data = (void *)&rk3036_pmu,
+	},
+	{
+		.compatible = "rockchip,rk3128-power-controller",
+		.data = (void *)&rk3128_pmu,
+	},
+	{
+		.compatible = "rockchip,rk3228-power-controller",
+		.data = (void *)&rk3228_pmu,
+	},
 	{
 		.compatible = "rockchip,rk3288-power-controller",
 		.data = (void *)&rk3288_pmu,
diff --git a/drivers/soc/samsung/pm_domains.c b/drivers/soc/samsung/pm_domains.c
index caf45cf7aa8e..ab8582971bfc 100644
--- a/drivers/soc/samsung/pm_domains.c
+++ b/drivers/soc/samsung/pm_domains.c
@@ -13,14 +13,11 @@
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/pm_domain.h>
-#include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
 #include <linux/sched.h>
 
-#define MAX_CLK_PER_DOMAIN	4
-
 struct exynos_pm_domain_config {
 	/* Value for LOCAL_PWR_CFG and STATUS fields for each domain */
 	u32 local_pwr_cfg;
@@ -33,10 +30,6 @@ struct exynos_pm_domain {
 	void __iomem *base;
 	bool is_off;
 	struct generic_pm_domain pd;
-	struct clk *oscclk;
-	struct clk *clk[MAX_CLK_PER_DOMAIN];
-	struct clk *pclk[MAX_CLK_PER_DOMAIN];
-	struct clk *asb_clk[MAX_CLK_PER_DOMAIN];
 	u32 local_pwr_cfg;
 };
 
@@ -46,29 +39,10 @@ static int exynos_pd_power(struct generic_pm_domain *domain, bool power_on)
 	void __iomem *base;
 	u32 timeout, pwr;
 	char *op;
-	int i;
 
 	pd = container_of(domain, struct exynos_pm_domain, pd);
 	base = pd->base;
 
-	for (i = 0; i < MAX_CLK_PER_DOMAIN; i++) {
-		if (IS_ERR(pd->asb_clk[i]))
-			break;
-		clk_prepare_enable(pd->asb_clk[i]);
-	}
-
-	/* Set oscclk before powering off a domain*/
-	if (!power_on) {
-		for (i = 0; i < MAX_CLK_PER_DOMAIN; i++) {
-			if (IS_ERR(pd->clk[i]))
-				break;
-			pd->pclk[i] = clk_get_parent(pd->clk[i]);
-			if (clk_set_parent(pd->clk[i], pd->oscclk))
-				pr_err("%s: error setting oscclk as parent to clock %d\n",
-						domain->name, i);
-		}
-	}
-
 	pwr = power_on ? pd->local_pwr_cfg : 0;
 	writel_relaxed(pwr, base);
 
@@ -86,26 +60,6 @@ static int exynos_pd_power(struct generic_pm_domain *domain, bool power_on)
 		usleep_range(80, 100);
 	}
 
-	/* Restore clocks after powering on a domain*/
-	if (power_on) {
-		for (i = 0; i < MAX_CLK_PER_DOMAIN; i++) {
-			if (IS_ERR(pd->clk[i]))
-				break;
-
-			if (IS_ERR(pd->pclk[i]))
-				continue; /* Skip on first power up */
-			if (clk_set_parent(pd->clk[i], pd->pclk[i]))
-				pr_err("%s: error setting parent to clock%d\n",
-						domain->name, i);
-		}
-	}
-
-	for (i = 0; i < MAX_CLK_PER_DOMAIN; i++) {
-		if (IS_ERR(pd->asb_clk[i]))
-			break;
-		clk_disable_unprepare(pd->asb_clk[i]);
-	}
-
 	return 0;
 }
 
@@ -147,12 +101,6 @@ static __init const char *exynos_get_domain_name(struct device_node *node)
 	return kstrdup_const(name, GFP_KERNEL);
 }
 
-static const char *soc_force_no_clk[] = {
-	"samsung,exynos5250-clock",
-	"samsung,exynos5420-clock",
-	"samsung,exynos5800-clock",
-};
-
 static __init int exynos4_pm_init_power_domain(void)
 {
 	struct device_node *np;
@@ -161,7 +109,7 @@ static __init int exynos4_pm_init_power_domain(void)
 	for_each_matching_node_and_match(np, exynos_pm_domain_of_match, &match) {
 		const struct exynos_pm_domain_config *pm_domain_cfg;
 		struct exynos_pm_domain *pd;
-		int on, i;
+		int on;
 
 		pm_domain_cfg = match->data;
 
@@ -189,42 +137,6 @@ static __init int exynos4_pm_init_power_domain(void)
 		pd->pd.power_on = exynos_pd_power_on;
 		pd->local_pwr_cfg = pm_domain_cfg->local_pwr_cfg;
 
-		for (i = 0; i < ARRAY_SIZE(soc_force_no_clk); i++)
-			if (of_find_compatible_node(NULL, NULL,
-						    soc_force_no_clk[i]))
-				goto no_clk;
-
-		for (i = 0; i < MAX_CLK_PER_DOMAIN; i++) {
-			char clk_name[8];
-
-			snprintf(clk_name, sizeof(clk_name), "asb%d", i);
-			pd->asb_clk[i] = of_clk_get_by_name(np, clk_name);
-			if (IS_ERR(pd->asb_clk[i]))
-				break;
-		}
-
-		pd->oscclk = of_clk_get_by_name(np, "oscclk");
-		if (IS_ERR(pd->oscclk))
-			goto no_clk;
-
-		for (i = 0; i < MAX_CLK_PER_DOMAIN; i++) {
-			char clk_name[8];
-
-			snprintf(clk_name, sizeof(clk_name), "clk%d", i);
-			pd->clk[i] = of_clk_get_by_name(np, clk_name);
-			if (IS_ERR(pd->clk[i]))
-				break;
-			/*
-			 * Skip setting parent on first power up.
-			 * The parent at this time may not be useful at all.
-			 */
-			pd->pclk[i] = ERR_PTR(-EINVAL);
-		}
-
-		if (IS_ERR(pd->clk[0]))
-			clk_put(pd->oscclk);
-
-no_clk:
 		on = readl_relaxed(pd->base + 0x4) & pd->local_pwr_cfg;
 
 		pm_genpd_init(&pd->pd, NULL, !on);
diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c
index d9fcdb592b39..2d6f3fcf3211 100644
--- a/drivers/soc/tegra/pmc.c
+++ b/drivers/soc/tegra/pmc.c
@@ -31,6 +31,7 @@
 #include <linux/iopoll.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/of_clk.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
@@ -559,22 +560,28 @@ EXPORT_SYMBOL(tegra_powergate_remove_clamping);
 int tegra_powergate_sequence_power_up(unsigned int id, struct clk *clk,
 				      struct reset_control *rst)
 {
-	struct tegra_powergate pg;
+	struct tegra_powergate *pg;
 	int err;
 
 	if (!tegra_powergate_is_available(id))
 		return -EINVAL;
 
-	pg.id = id;
-	pg.clks = &clk;
-	pg.num_clks = 1;
-	pg.reset = rst;
-	pg.pmc = pmc;
+	pg = kzalloc(sizeof(*pg), GFP_KERNEL);
+	if (!pg)
+		return -ENOMEM;
 
-	err = tegra_powergate_power_up(&pg, false);
+	pg->id = id;
+	pg->clks = &clk;
+	pg->num_clks = 1;
+	pg->reset = rst;
+	pg->pmc = pmc;
+
+	err = tegra_powergate_power_up(pg, false);
 	if (err)
 		pr_err("failed to turn on partition %d: %d\n", id, err);
 
+	kfree(pg);
+
 	return err;
 }
 EXPORT_SYMBOL(tegra_powergate_sequence_power_up);
@@ -725,7 +732,7 @@ static int tegra_powergate_of_get_clks(struct tegra_powergate *pg,
 	unsigned int i, count;
 	int err;
 
-	count = of_count_phandle_with_args(np, "clocks", "#clock-cells");
+	count = of_clk_get_parent_count(np);
 	if (count == 0)
 		return -ENODEV;
 
diff --git a/drivers/soc/ti/knav_dma.c b/drivers/soc/ti/knav_dma.c
index 026182d3b27c..224d7ddeeb76 100644
--- a/drivers/soc/ti/knav_dma.c
+++ b/drivers/soc/ti/knav_dma.c
@@ -134,6 +134,13 @@ struct knav_dma_chan {
 
 static struct knav_dma_pool_device *kdev;
 
+static bool device_ready;
+bool knav_dma_device_ready(void)
+{
+	return device_ready;
+}
+EXPORT_SYMBOL_GPL(knav_dma_device_ready);
+
 static bool check_config(struct knav_dma_chan *chan, struct knav_dma_cfg *cfg)
 {
 	if (!memcmp(&chan->cfg, cfg, sizeof(*cfg)))
@@ -773,6 +780,7 @@ static int knav_dma_probe(struct platform_device *pdev)
 	debugfs_create_file("knav_dma", S_IFREG | S_IRUGO, NULL, NULL,
 			    &knav_dma_debug_ops);
 
+	device_ready = true;
 	return ret;
 }
 
diff --git a/drivers/soc/ti/knav_qmss.h b/drivers/soc/ti/knav_qmss.h
index 905b974d1bdc..3efc47e82973 100644
--- a/drivers/soc/ti/knav_qmss.h
+++ b/drivers/soc/ti/knav_qmss.h
@@ -19,6 +19,8 @@
 #ifndef __KNAV_QMSS_H__
 #define __KNAV_QMSS_H__
 
+#include <linux/percpu.h>
+
 #define THRESH_GTE	BIT(7)
 #define THRESH_LT	0
 
@@ -162,11 +164,11 @@ struct knav_qmgr_info {
  * notifies:			notifier counts
  */
 struct knav_queue_stats {
-	atomic_t	 pushes;
-	atomic_t	 pops;
-	atomic_t	 push_errors;
-	atomic_t	 pop_errors;
-	atomic_t	 notifies;
+	unsigned int pushes;
+	unsigned int pops;
+	unsigned int push_errors;
+	unsigned int pop_errors;
+	unsigned int notifies;
 };
 
 /**
@@ -283,7 +285,7 @@ struct knav_queue_inst {
 struct knav_queue {
 	struct knav_reg_queue __iomem	*reg_push, *reg_pop, *reg_peek;
 	struct knav_queue_inst		*inst;
-	struct knav_queue_stats	stats;
+	struct knav_queue_stats __percpu	*stats;
 	knav_queue_notify_fn		notifier_fn;
 	void				*notifier_fn_arg;
 	atomic_t			notifier_enabled;
@@ -292,6 +294,11 @@ struct knav_queue {
 	struct list_head		list;
 };
 
+enum qmss_version {
+	QMSS,
+	QMSS_66AK2G,
+};
+
 struct knav_device {
 	struct device				*dev;
 	unsigned				base_id;
@@ -305,6 +312,7 @@ struct knav_device {
 	struct list_head			pools;
 	struct list_head			pdsps;
 	struct list_head			qmgrs;
+	enum qmss_version			version;
 };
 
 struct knav_range_ops {
diff --git a/drivers/soc/ti/knav_qmss_queue.c b/drivers/soc/ti/knav_qmss_queue.c
index 77d6b5c03aae..6755f2af5619 100644
--- a/drivers/soc/ti/knav_qmss_queue.c
+++ b/drivers/soc/ti/knav_qmss_queue.c
@@ -42,6 +42,15 @@ static DEFINE_MUTEX(knav_dev_lock);
 #define KNAV_QUEUE_PUSH_REG_INDEX	4
 #define KNAV_QUEUE_POP_REG_INDEX	5
 
+/* Queue manager register indices in DTS for QMSS in K2G NAVSS.
+ * There are no status and vbusm push registers on this version
+ * of QMSS. Push registers are same as pop, So all indices above 1
+ * are to be re-defined
+ */
+#define KNAV_L_QUEUE_CONFIG_REG_INDEX	1
+#define KNAV_L_QUEUE_REGION_REG_INDEX	2
+#define KNAV_L_QUEUE_PUSH_REG_INDEX	3
+
 /* PDSP register indices in DTS */
 #define KNAV_QUEUE_PDSP_IRAM_REG_INDEX	0
 #define KNAV_QUEUE_PDSP_REGS_REG_INDEX	1
@@ -65,6 +74,13 @@ static DEFINE_MUTEX(knav_dev_lock);
  */
 const char *knav_acc_firmwares[] = {"ks2_qmss_pdsp_acc48.bin"};
 
+static bool device_ready;
+bool knav_qmss_device_ready(void)
+{
+	return device_ready;
+}
+EXPORT_SYMBOL_GPL(knav_qmss_device_ready);
+
 /**
  * knav_queue_notify: qmss queue notfier call
  *
@@ -83,7 +99,7 @@ void knav_queue_notify(struct knav_queue_inst *inst)
 			continue;
 		if (WARN_ON(!qh->notifier_fn))
 			continue;
-		atomic_inc(&qh->stats.notifies);
+		this_cpu_inc(qh->stats->notifies);
 		qh->notifier_fn(qh->notifier_fn_arg);
 	}
 	rcu_read_unlock();
@@ -214,6 +230,12 @@ static struct knav_queue *__knav_queue_open(struct knav_queue_inst *inst,
 	if (!qh)
 		return ERR_PTR(-ENOMEM);
 
+	qh->stats = alloc_percpu(struct knav_queue_stats);
+	if (!qh->stats) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
 	qh->flags = flags;
 	qh->inst = inst;
 	id = inst->id - inst->qmgr->start_queue;
@@ -229,13 +251,17 @@ static struct knav_queue *__knav_queue_open(struct knav_queue_inst *inst,
 		if (range->ops && range->ops->open_queue)
 			ret = range->ops->open_queue(range, inst, flags);
 
-		if (ret) {
-			devm_kfree(inst->kdev->dev, qh);
-			return ERR_PTR(ret);
-		}
+		if (ret)
+			goto err;
 	}
 	list_add_tail_rcu(&qh->list, &inst->handles);
 	return qh;
+
+err:
+	if (qh->stats)
+		free_percpu(qh->stats);
+	devm_kfree(inst->kdev->dev, qh);
+	return ERR_PTR(ret);
 }
 
 static struct knav_queue *
@@ -411,6 +437,12 @@ static void knav_queue_debug_show_instance(struct seq_file *s,
 {
 	struct knav_device *kdev = inst->kdev;
 	struct knav_queue *qh;
+	int cpu = 0;
+	int pushes = 0;
+	int pops = 0;
+	int push_errors = 0;
+	int pop_errors = 0;
+	int notifies = 0;
 
 	if (!knav_queue_is_busy(inst))
 		return;
@@ -418,19 +450,22 @@ static void knav_queue_debug_show_instance(struct seq_file *s,
 	seq_printf(s, "\tqueue id %d (%s)\n",
 		   kdev->base_id + inst->id, inst->name);
 	for_each_handle_rcu(qh, inst) {
-		seq_printf(s, "\t\thandle %p: ", qh);
-		seq_printf(s, "pushes %8d, ",
-			   atomic_read(&qh->stats.pushes));
-		seq_printf(s, "pops %8d, ",
-			   atomic_read(&qh->stats.pops));
-		seq_printf(s, "count %8d, ",
-			   knav_queue_get_count(qh));
-		seq_printf(s, "notifies %8d, ",
-			   atomic_read(&qh->stats.notifies));
-		seq_printf(s, "push errors %8d, ",
-			   atomic_read(&qh->stats.push_errors));
-		seq_printf(s, "pop errors %8d\n",
-			   atomic_read(&qh->stats.pop_errors));
+		for_each_possible_cpu(cpu) {
+			pushes += per_cpu_ptr(qh->stats, cpu)->pushes;
+			pops += per_cpu_ptr(qh->stats, cpu)->pops;
+			push_errors += per_cpu_ptr(qh->stats, cpu)->push_errors;
+			pop_errors += per_cpu_ptr(qh->stats, cpu)->pop_errors;
+			notifies += per_cpu_ptr(qh->stats, cpu)->notifies;
+		}
+
+		seq_printf(s, "\t\thandle %p: pushes %8d, pops %8d, count %8d, notifies %8d, push errors %8d, pop errors %8d\n",
+				qh,
+				pushes,
+				pops,
+				knav_queue_get_count(qh),
+				notifies,
+				push_errors,
+				pop_errors);
 	}
 }
 
@@ -547,6 +582,7 @@ void knav_queue_close(void *qhandle)
 		if (range->ops && range->ops->close_queue)
 			range->ops->close_queue(range, inst);
 	}
+	free_percpu(qh->stats);
 	devm_kfree(inst->kdev->dev, qh);
 }
 EXPORT_SYMBOL_GPL(knav_queue_close);
@@ -620,7 +656,7 @@ int knav_queue_push(void *qhandle, dma_addr_t dma,
 	val = (u32)dma | ((size / 16) - 1);
 	writel_relaxed(val, &qh->reg_push[0].ptr_size_thresh);
 
-	atomic_inc(&qh->stats.pushes);
+	this_cpu_inc(qh->stats->pushes);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(knav_queue_push);
@@ -658,7 +694,7 @@ dma_addr_t knav_queue_pop(void *qhandle, unsigned *size)
 	if (size)
 		*size = ((val & DESC_SIZE_MASK) + 1) * 16;
 
-	atomic_inc(&qh->stats.pops);
+	this_cpu_inc(qh->stats->pops);
 	return dma;
 }
 EXPORT_SYMBOL_GPL(knav_queue_pop);
@@ -1169,8 +1205,12 @@ static int knav_queue_setup_link_ram(struct knav_device *kdev)
 		dev_dbg(kdev->dev, "linkram0: dma:%pad, virt:%p, size:%x\n",
 			&block->dma, block->virt, block->size);
 		writel_relaxed((u32)block->dma, &qmgr->reg_config->link_ram_base0);
-		writel_relaxed(block->size, &qmgr->reg_config->link_ram_size0);
-
+		if (kdev->version == QMSS_66AK2G)
+			writel_relaxed(block->size,
+				       &qmgr->reg_config->link_ram_size0);
+		else
+			writel_relaxed(block->size - 1,
+				       &qmgr->reg_config->link_ram_size0);
 		block++;
 		if (!block->size)
 			continue;
@@ -1387,42 +1427,64 @@ static int knav_queue_init_qmgrs(struct knav_device *kdev,
 		qmgr->reg_peek =
 			knav_queue_map_reg(kdev, child,
 					   KNAV_QUEUE_PEEK_REG_INDEX);
-		qmgr->reg_status =
-			knav_queue_map_reg(kdev, child,
-					   KNAV_QUEUE_STATUS_REG_INDEX);
+
+		if (kdev->version == QMSS) {
+			qmgr->reg_status =
+				knav_queue_map_reg(kdev, child,
+						   KNAV_QUEUE_STATUS_REG_INDEX);
+		}
+
 		qmgr->reg_config =
 			knav_queue_map_reg(kdev, child,
+					   (kdev->version == QMSS_66AK2G) ?
+					   KNAV_L_QUEUE_CONFIG_REG_INDEX :
 					   KNAV_QUEUE_CONFIG_REG_INDEX);
 		qmgr->reg_region =
 			knav_queue_map_reg(kdev, child,
+					   (kdev->version == QMSS_66AK2G) ?
+					   KNAV_L_QUEUE_REGION_REG_INDEX :
 					   KNAV_QUEUE_REGION_REG_INDEX);
+
 		qmgr->reg_push =
 			knav_queue_map_reg(kdev, child,
-					   KNAV_QUEUE_PUSH_REG_INDEX);
-		qmgr->reg_pop =
-			knav_queue_map_reg(kdev, child,
-					   KNAV_QUEUE_POP_REG_INDEX);
+					   (kdev->version == QMSS_66AK2G) ?
+					    KNAV_L_QUEUE_PUSH_REG_INDEX :
+					    KNAV_QUEUE_PUSH_REG_INDEX);
+
+		if (kdev->version == QMSS) {
+			qmgr->reg_pop =
+				knav_queue_map_reg(kdev, child,
+						   KNAV_QUEUE_POP_REG_INDEX);
+		}
 
-		if (IS_ERR(qmgr->reg_peek) || IS_ERR(qmgr->reg_status) ||
+		if (IS_ERR(qmgr->reg_peek) ||
+		    ((kdev->version == QMSS) &&
+		    (IS_ERR(qmgr->reg_status) || IS_ERR(qmgr->reg_pop))) ||
 		    IS_ERR(qmgr->reg_config) || IS_ERR(qmgr->reg_region) ||
-		    IS_ERR(qmgr->reg_push) || IS_ERR(qmgr->reg_pop)) {
+		    IS_ERR(qmgr->reg_push)) {
 			dev_err(dev, "failed to map qmgr regs\n");
+			if (kdev->version == QMSS) {
+				if (!IS_ERR(qmgr->reg_status))
+					devm_iounmap(dev, qmgr->reg_status);
+				if (!IS_ERR(qmgr->reg_pop))
+					devm_iounmap(dev, qmgr->reg_pop);
+			}
 			if (!IS_ERR(qmgr->reg_peek))
 				devm_iounmap(dev, qmgr->reg_peek);
-			if (!IS_ERR(qmgr->reg_status))
-				devm_iounmap(dev, qmgr->reg_status);
 			if (!IS_ERR(qmgr->reg_config))
 				devm_iounmap(dev, qmgr->reg_config);
 			if (!IS_ERR(qmgr->reg_region))
 				devm_iounmap(dev, qmgr->reg_region);
 			if (!IS_ERR(qmgr->reg_push))
 				devm_iounmap(dev, qmgr->reg_push);
-			if (!IS_ERR(qmgr->reg_pop))
-				devm_iounmap(dev, qmgr->reg_pop);
 			devm_kfree(dev, qmgr);
 			continue;
 		}
 
+		/* Use same push register for pop as well */
+		if (kdev->version == QMSS_66AK2G)
+			qmgr->reg_pop = qmgr->reg_push;
+
 		list_add_tail(&qmgr->list, &kdev->qmgrs);
 		dev_info(dev, "added qmgr start queue %d, num of queues %d, reg_peek %p, reg_status %p, reg_config %p, reg_region %p, reg_push %p, reg_pop %p\n",
 			 qmgr->start_queue, qmgr->num_queues,
@@ -1681,10 +1743,24 @@ static int knav_queue_init_queues(struct knav_device *kdev)
 	return 0;
 }
 
+/* Match table for of_platform binding */
+static const struct of_device_id keystone_qmss_of_match[] = {
+	{
+		.compatible = "ti,keystone-navigator-qmss",
+	},
+	{
+		.compatible = "ti,66ak2g-navss-qm",
+		.data	= (void *)QMSS_66AK2G,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, keystone_qmss_of_match);
+
 static int knav_queue_probe(struct platform_device *pdev)
 {
 	struct device_node *node = pdev->dev.of_node;
 	struct device_node *qmgrs, *queue_pools, *regions, *pdsps;
+	const struct of_device_id *match;
 	struct device *dev = &pdev->dev;
 	u32 temp[2];
 	int ret;
@@ -1700,6 +1776,10 @@ static int knav_queue_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	}
 
+	match = of_match_device(of_match_ptr(keystone_qmss_of_match), dev);
+	if (match && match->data)
+		kdev->version = QMSS_66AK2G;
+
 	platform_set_drvdata(pdev, kdev);
 	kdev->dev = dev;
 	INIT_LIST_HEAD(&kdev->queue_ranges);
@@ -1796,6 +1876,7 @@ static int knav_queue_probe(struct platform_device *pdev)
 
 	debugfs_create_file("qmss", S_IFREG | S_IRUGO, NULL, NULL,
 			    &knav_queue_debug_ops);
+	device_ready = true;
 	return 0;
 
 err:
@@ -1815,13 +1896,6 @@ static int knav_queue_remove(struct platform_device *pdev)
 	return 0;
 }
 
-/* Match table for of_platform binding */
-static struct of_device_id keystone_qmss_of_match[] = {
-	{ .compatible = "ti,keystone-navigator-qmss", },
-	{},
-};
-MODULE_DEVICE_TABLE(of, keystone_qmss_of_match);
-
 static struct platform_driver keystone_qmss_driver = {
 	.probe		= knav_queue_probe,
 	.remove		= knav_queue_remove,
diff --git a/drivers/soundwire/Kconfig b/drivers/soundwire/Kconfig
index b46084b4b1f8..19c8efb9a5ee 100644
--- a/drivers/soundwire/Kconfig
+++ b/drivers/soundwire/Kconfig
@@ -27,7 +27,7 @@ config SOUNDWIRE_INTEL
 	tristate "Intel SoundWire Master driver"
 	select SOUNDWIRE_CADENCE
 	select SOUNDWIRE_BUS
-	depends on X86 && ACPI
+	depends on X86 && ACPI && SND_SOC
 	---help---
 	  SoundWire Intel Master driver.
 	  If you have an Intel platform which has a SoundWire Master then
diff --git a/drivers/soundwire/Makefile b/drivers/soundwire/Makefile
index e1a74c5692aa..5817beaca0e1 100644
--- a/drivers/soundwire/Makefile
+++ b/drivers/soundwire/Makefile
@@ -3,7 +3,7 @@
 #
 
 #Bus Objs
-soundwire-bus-objs := bus_type.o bus.o slave.o mipi_disco.o
+soundwire-bus-objs := bus_type.o bus.o slave.o mipi_disco.o stream.o
 obj-$(CONFIG_SOUNDWIRE_BUS) += soundwire-bus.o
 
 #Cadence Objs
diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c
index d6dc8e7a8614..dcc0ff9f0c22 100644
--- a/drivers/soundwire/bus.c
+++ b/drivers/soundwire/bus.c
@@ -17,6 +17,7 @@
  */
 int sdw_add_bus_master(struct sdw_bus *bus)
 {
+	struct sdw_master_prop *prop = NULL;
 	int ret;
 
 	if (!bus->dev) {
@@ -32,6 +33,7 @@ int sdw_add_bus_master(struct sdw_bus *bus)
 	mutex_init(&bus->msg_lock);
 	mutex_init(&bus->bus_lock);
 	INIT_LIST_HEAD(&bus->slaves);
+	INIT_LIST_HEAD(&bus->m_rt_list);
 
 	if (bus->ops->read_prop) {
 		ret = bus->ops->read_prop(bus);
@@ -77,6 +79,21 @@ int sdw_add_bus_master(struct sdw_bus *bus)
 		return ret;
 	}
 
+	/*
+	 * Initialize clock values based on Master properties. The max
+	 * frequency is read from max_freq property. Current assumption
+	 * is that the bus will start at highest clock frequency when
+	 * powered on.
+	 *
+	 * Default active bank will be 0 as out of reset the Slaves have
+	 * to start with bank 0 (Table 40 of Spec)
+	 */
+	prop = &bus->prop;
+	bus->params.max_dr_freq = prop->max_freq * SDW_DOUBLE_RATE_FACTOR;
+	bus->params.curr_dr_freq = bus->params.max_dr_freq;
+	bus->params.curr_bank = SDW_BANK0;
+	bus->params.next_bank = SDW_BANK1;
+
 	return 0;
 }
 EXPORT_SYMBOL(sdw_add_bus_master);
@@ -576,6 +593,32 @@ static void sdw_modify_slave_status(struct sdw_slave *slave,
 	mutex_unlock(&slave->bus->bus_lock);
 }
 
+int sdw_configure_dpn_intr(struct sdw_slave *slave,
+			int port, bool enable, int mask)
+{
+	u32 addr;
+	int ret;
+	u8 val = 0;
+
+	addr = SDW_DPN_INTMASK(port);
+
+	/* Set/Clear port ready interrupt mask */
+	if (enable) {
+		val |= mask;
+		val |= SDW_DPN_INT_PORT_READY;
+	} else {
+		val &= ~(mask);
+		val &= ~SDW_DPN_INT_PORT_READY;
+	}
+
+	ret = sdw_update(slave, addr, (mask | SDW_DPN_INT_PORT_READY), val);
+	if (ret < 0)
+		dev_err(slave->bus->dev,
+				"SDW_DPN_INTMASK write failed:%d", val);
+
+	return ret;
+}
+
 static int sdw_initialize_slave(struct sdw_slave *slave)
 {
 	struct sdw_slave_prop *prop = &slave->prop;
diff --git a/drivers/soundwire/bus.h b/drivers/soundwire/bus.h
index 345c34d697e9..3b15c4e25a3a 100644
--- a/drivers/soundwire/bus.h
+++ b/drivers/soundwire/bus.h
@@ -45,6 +45,78 @@ struct sdw_msg {
 	bool page;
 };
 
+#define SDW_DOUBLE_RATE_FACTOR		2
+
+extern int rows[SDW_FRAME_ROWS];
+extern int cols[SDW_FRAME_COLS];
+
+/**
+ * sdw_port_runtime: Runtime port parameters for Master or Slave
+ *
+ * @num: Port number. For audio streams, valid port number ranges from
+ * [1,14]
+ * @ch_mask: Channel mask
+ * @transport_params: Transport parameters
+ * @port_params: Port parameters
+ * @port_node: List node for Master or Slave port_list
+ *
+ * SoundWire spec has no mention of ports for Master interface but the
+ * concept is logically extended.
+ */
+struct sdw_port_runtime {
+	int num;
+	int ch_mask;
+	struct sdw_transport_params transport_params;
+	struct sdw_port_params port_params;
+	struct list_head port_node;
+};
+
+/**
+ * sdw_slave_runtime: Runtime Stream parameters for Slave
+ *
+ * @slave: Slave handle
+ * @direction: Data direction for Slave
+ * @ch_count: Number of channels handled by the Slave for
+ * this stream
+ * @m_rt_node: sdw_master_runtime list node
+ * @port_list: List of Slave Ports configured for this stream
+ */
+struct sdw_slave_runtime {
+	struct sdw_slave *slave;
+	enum sdw_data_direction direction;
+	unsigned int ch_count;
+	struct list_head m_rt_node;
+	struct list_head port_list;
+};
+
+/**
+ * sdw_master_runtime: Runtime stream parameters for Master
+ *
+ * @bus: Bus handle
+ * @stream: Stream runtime handle
+ * @direction: Data direction for Master
+ * @ch_count: Number of channels handled by the Master for
+ * this stream, can be zero.
+ * @slave_rt_list: Slave runtime list
+ * @port_list: List of Master Ports configured for this stream, can be zero.
+ * @bus_node: sdw_bus m_rt_list node
+ */
+struct sdw_master_runtime {
+	struct sdw_bus *bus;
+	struct sdw_stream_runtime *stream;
+	enum sdw_data_direction direction;
+	unsigned int ch_count;
+	struct list_head slave_rt_list;
+	struct list_head port_list;
+	struct list_head bus_node;
+};
+
+struct sdw_dpn_prop *sdw_get_slave_dpn_prop(struct sdw_slave *slave,
+				enum sdw_data_direction direction,
+				unsigned int port_num);
+int sdw_configure_dpn_intr(struct sdw_slave *slave, int port,
+					bool enable, int mask);
+
 int sdw_transfer(struct sdw_bus *bus, struct sdw_msg *msg);
 int sdw_transfer_defer(struct sdw_bus *bus, struct sdw_msg *msg,
 				struct sdw_defer *defer);
diff --git a/drivers/soundwire/bus_type.c b/drivers/soundwire/bus_type.c
index d5f3a70c06b0..283b2832728e 100644
--- a/drivers/soundwire/bus_type.c
+++ b/drivers/soundwire/bus_type.c
@@ -83,17 +83,16 @@ static int sdw_drv_probe(struct device *dev)
 	 * attach to power domain but don't turn on (last arg)
 	 */
 	ret = dev_pm_domain_attach(dev, false);
-	if (ret != -EPROBE_DEFER) {
-		ret = drv->probe(slave, id);
-		if (ret) {
-			dev_err(dev, "Probe of %s failed: %d\n", drv->name, ret);
-			dev_pm_domain_detach(dev, false);
-		}
-	}
-
 	if (ret)
 		return ret;
 
+	ret = drv->probe(slave, id);
+	if (ret) {
+		dev_err(dev, "Probe of %s failed: %d\n", drv->name, ret);
+		dev_pm_domain_detach(dev, false);
+		return ret;
+	}
+
 	/* device is probed so let's read the properties now */
 	if (slave->ops && slave->ops->read_prop)
 		slave->ops->read_prop(slave);
diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c
index 3a9b1462039b..cb6a331f448a 100644
--- a/drivers/soundwire/cadence_master.c
+++ b/drivers/soundwire/cadence_master.c
@@ -13,6 +13,8 @@
 #include <linux/mod_devicetable.h>
 #include <linux/soundwire/sdw_registers.h>
 #include <linux/soundwire/sdw.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
 #include "bus.h"
 #include "cadence_master.h"
 
@@ -396,7 +398,7 @@ static int cdns_prep_msg(struct sdw_cdns *cdns, struct sdw_msg *msg, int *cmd)
 	return 0;
 }
 
-static enum sdw_command_response
+enum sdw_command_response
 cdns_xfer_msg(struct sdw_bus *bus, struct sdw_msg *msg)
 {
 	struct sdw_cdns *cdns = bus_to_cdns(bus);
@@ -422,8 +424,9 @@ cdns_xfer_msg(struct sdw_bus *bus, struct sdw_msg *msg)
 exit:
 	return ret;
 }
+EXPORT_SYMBOL(cdns_xfer_msg);
 
-static enum sdw_command_response
+enum sdw_command_response
 cdns_xfer_msg_defer(struct sdw_bus *bus,
 		struct sdw_msg *msg, struct sdw_defer *defer)
 {
@@ -443,8 +446,9 @@ cdns_xfer_msg_defer(struct sdw_bus *bus,
 
 	return _cdns_xfer_msg(cdns, msg, cmd, 0, msg->len, true);
 }
+EXPORT_SYMBOL(cdns_xfer_msg_defer);
 
-static enum sdw_command_response
+enum sdw_command_response
 cdns_reset_page_addr(struct sdw_bus *bus, unsigned int dev_num)
 {
 	struct sdw_cdns *cdns = bus_to_cdns(bus);
@@ -456,6 +460,7 @@ cdns_reset_page_addr(struct sdw_bus *bus, unsigned int dev_num)
 
 	return cdns_program_scp_addr(cdns, &msg);
 }
+EXPORT_SYMBOL(cdns_reset_page_addr);
 
 /*
  * IRQ handling
@@ -666,6 +671,120 @@ int sdw_cdns_enable_interrupt(struct sdw_cdns *cdns)
 }
 EXPORT_SYMBOL(sdw_cdns_enable_interrupt);
 
+static int cdns_allocate_pdi(struct sdw_cdns *cdns,
+			struct sdw_cdns_pdi **stream,
+			u32 num, u32 pdi_offset)
+{
+	struct sdw_cdns_pdi *pdi;
+	int i;
+
+	if (!num)
+		return 0;
+
+	pdi = devm_kcalloc(cdns->dev, num, sizeof(*pdi), GFP_KERNEL);
+	if (!pdi)
+		return -ENOMEM;
+
+	for (i = 0; i < num; i++) {
+		pdi[i].num = i + pdi_offset;
+		pdi[i].assigned = false;
+	}
+
+	*stream = pdi;
+	return 0;
+}
+
+/**
+ * sdw_cdns_pdi_init() - PDI initialization routine
+ *
+ * @cdns: Cadence instance
+ * @config: Stream configurations
+ */
+int sdw_cdns_pdi_init(struct sdw_cdns *cdns,
+			struct sdw_cdns_stream_config config)
+{
+	struct sdw_cdns_streams *stream;
+	int offset, i, ret;
+
+	cdns->pcm.num_bd = config.pcm_bd;
+	cdns->pcm.num_in = config.pcm_in;
+	cdns->pcm.num_out = config.pcm_out;
+	cdns->pdm.num_bd = config.pdm_bd;
+	cdns->pdm.num_in = config.pdm_in;
+	cdns->pdm.num_out = config.pdm_out;
+
+	/* Allocate PDIs for PCMs */
+	stream = &cdns->pcm;
+
+	/* First two PDIs are reserved for bulk transfers */
+	stream->num_bd -= CDNS_PCM_PDI_OFFSET;
+	offset = CDNS_PCM_PDI_OFFSET;
+
+	ret = cdns_allocate_pdi(cdns, &stream->bd,
+				stream->num_bd, offset);
+	if (ret)
+		return ret;
+
+	offset += stream->num_bd;
+
+	ret = cdns_allocate_pdi(cdns, &stream->in,
+				stream->num_in, offset);
+	if (ret)
+		return ret;
+
+	offset += stream->num_in;
+
+	ret = cdns_allocate_pdi(cdns, &stream->out,
+				stream->num_out, offset);
+	if (ret)
+		return ret;
+
+	/* Update total number of PCM PDIs */
+	stream->num_pdi = stream->num_bd + stream->num_in + stream->num_out;
+	cdns->num_ports = stream->num_pdi;
+
+	/* Allocate PDIs for PDMs */
+	stream = &cdns->pdm;
+	offset = CDNS_PDM_PDI_OFFSET;
+	ret = cdns_allocate_pdi(cdns, &stream->bd,
+				stream->num_bd, offset);
+	if (ret)
+		return ret;
+
+	offset += stream->num_bd;
+
+	ret = cdns_allocate_pdi(cdns, &stream->in,
+				stream->num_in, offset);
+	if (ret)
+		return ret;
+
+	offset += stream->num_in;
+
+	ret = cdns_allocate_pdi(cdns, &stream->out,
+				stream->num_out, offset);
+	if (ret)
+		return ret;
+
+	/* Update total number of PDM PDIs */
+	stream->num_pdi = stream->num_bd + stream->num_in + stream->num_out;
+	cdns->num_ports += stream->num_pdi;
+
+	cdns->ports = devm_kcalloc(cdns->dev, cdns->num_ports,
+				sizeof(*cdns->ports), GFP_KERNEL);
+	if (!cdns->ports) {
+		ret = -ENOMEM;
+		return ret;
+	}
+
+	for (i = 0; i < cdns->num_ports; i++) {
+		cdns->ports[i].assigned = false;
+		cdns->ports[i].num = i + 1; /* Port 0 reserved for bulk */
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(sdw_cdns_pdi_init);
+
 /**
  * sdw_cdns_init() - Cadence initialization
  * @cdns: Cadence instance
@@ -727,13 +846,133 @@ int sdw_cdns_init(struct sdw_cdns *cdns)
 }
 EXPORT_SYMBOL(sdw_cdns_init);
 
-struct sdw_master_ops sdw_cdns_master_ops = {
-	.read_prop = sdw_master_read_prop,
-	.xfer_msg = cdns_xfer_msg,
-	.xfer_msg_defer = cdns_xfer_msg_defer,
-	.reset_page_addr = cdns_reset_page_addr,
+int cdns_bus_conf(struct sdw_bus *bus, struct sdw_bus_params *params)
+{
+	struct sdw_cdns *cdns = bus_to_cdns(bus);
+	int mcp_clkctrl_off, mcp_clkctrl;
+	int divider;
+
+	if (!params->curr_dr_freq) {
+		dev_err(cdns->dev, "NULL curr_dr_freq");
+		return -EINVAL;
+	}
+
+	divider	= (params->max_dr_freq / params->curr_dr_freq) - 1;
+
+	if (params->next_bank)
+		mcp_clkctrl_off = CDNS_MCP_CLK_CTRL1;
+	else
+		mcp_clkctrl_off = CDNS_MCP_CLK_CTRL0;
+
+	mcp_clkctrl = cdns_readl(cdns, mcp_clkctrl_off);
+	mcp_clkctrl |= divider;
+	cdns_writel(cdns, mcp_clkctrl_off, mcp_clkctrl);
+
+	return 0;
+}
+EXPORT_SYMBOL(cdns_bus_conf);
+
+static int cdns_port_params(struct sdw_bus *bus,
+		struct sdw_port_params *p_params, unsigned int bank)
+{
+	struct sdw_cdns *cdns = bus_to_cdns(bus);
+	int dpn_config = 0, dpn_config_off;
+
+	if (bank)
+		dpn_config_off = CDNS_DPN_B1_CONFIG(p_params->num);
+	else
+		dpn_config_off = CDNS_DPN_B0_CONFIG(p_params->num);
+
+	dpn_config = cdns_readl(cdns, dpn_config_off);
+
+	dpn_config |= ((p_params->bps - 1) <<
+				SDW_REG_SHIFT(CDNS_DPN_CONFIG_WL));
+	dpn_config |= (p_params->flow_mode <<
+				SDW_REG_SHIFT(CDNS_DPN_CONFIG_PORT_FLOW));
+	dpn_config |= (p_params->data_mode <<
+				SDW_REG_SHIFT(CDNS_DPN_CONFIG_PORT_DAT));
+
+	cdns_writel(cdns, dpn_config_off, dpn_config);
+
+	return 0;
+}
+
+static int cdns_transport_params(struct sdw_bus *bus,
+			struct sdw_transport_params *t_params,
+			enum sdw_reg_bank bank)
+{
+	struct sdw_cdns *cdns = bus_to_cdns(bus);
+	int dpn_offsetctrl = 0, dpn_offsetctrl_off;
+	int dpn_config = 0, dpn_config_off;
+	int dpn_hctrl = 0, dpn_hctrl_off;
+	int num = t_params->port_num;
+	int dpn_samplectrl_off;
+
+	/*
+	 * Note: Only full data port is supported on the Master side for
+	 * both PCM and PDM ports.
+	 */
+
+	if (bank) {
+		dpn_config_off = CDNS_DPN_B1_CONFIG(num);
+		dpn_samplectrl_off = CDNS_DPN_B1_SAMPLE_CTRL(num);
+		dpn_hctrl_off = CDNS_DPN_B1_HCTRL(num);
+		dpn_offsetctrl_off = CDNS_DPN_B1_OFFSET_CTRL(num);
+	} else {
+		dpn_config_off = CDNS_DPN_B0_CONFIG(num);
+		dpn_samplectrl_off = CDNS_DPN_B0_SAMPLE_CTRL(num);
+		dpn_hctrl_off = CDNS_DPN_B0_HCTRL(num);
+		dpn_offsetctrl_off = CDNS_DPN_B0_OFFSET_CTRL(num);
+	}
+
+	dpn_config = cdns_readl(cdns, dpn_config_off);
+
+	dpn_config |= (t_params->blk_grp_ctrl <<
+				SDW_REG_SHIFT(CDNS_DPN_CONFIG_BGC));
+	dpn_config |= (t_params->blk_pkg_mode <<
+				SDW_REG_SHIFT(CDNS_DPN_CONFIG_BPM));
+	cdns_writel(cdns, dpn_config_off, dpn_config);
+
+	dpn_offsetctrl |= (t_params->offset1 <<
+				SDW_REG_SHIFT(CDNS_DPN_OFFSET_CTRL_1));
+	dpn_offsetctrl |= (t_params->offset2 <<
+				SDW_REG_SHIFT(CDNS_DPN_OFFSET_CTRL_2));
+	cdns_writel(cdns, dpn_offsetctrl_off,  dpn_offsetctrl);
+
+	dpn_hctrl |= (t_params->hstart <<
+				SDW_REG_SHIFT(CDNS_DPN_HCTRL_HSTART));
+	dpn_hctrl |= (t_params->hstop << SDW_REG_SHIFT(CDNS_DPN_HCTRL_HSTOP));
+	dpn_hctrl |= (t_params->lane_ctrl <<
+				SDW_REG_SHIFT(CDNS_DPN_HCTRL_LCTRL));
+
+	cdns_writel(cdns, dpn_hctrl_off, dpn_hctrl);
+	cdns_writel(cdns, dpn_samplectrl_off, (t_params->sample_interval - 1));
+
+	return 0;
+}
+
+static int cdns_port_enable(struct sdw_bus *bus,
+		struct sdw_enable_ch *enable_ch, unsigned int bank)
+{
+	struct sdw_cdns *cdns = bus_to_cdns(bus);
+	int dpn_chnen_off, ch_mask;
+
+	if (bank)
+		dpn_chnen_off = CDNS_DPN_B1_CH_EN(enable_ch->port_num);
+	else
+		dpn_chnen_off = CDNS_DPN_B0_CH_EN(enable_ch->port_num);
+
+	ch_mask = enable_ch->ch_mask * enable_ch->enable;
+	cdns_writel(cdns, dpn_chnen_off, ch_mask);
+
+	return 0;
+}
+
+static const struct sdw_master_port_ops cdns_port_ops = {
+	.dpn_set_port_params = cdns_port_params,
+	.dpn_set_port_transport_params = cdns_transport_params,
+	.dpn_port_enable_ch = cdns_port_enable,
 };
-EXPORT_SYMBOL(sdw_cdns_master_ops);
 
 /**
  * sdw_cdns_probe() - Cadence probe routine
@@ -742,10 +981,204 @@ EXPORT_SYMBOL(sdw_cdns_master_ops);
 int sdw_cdns_probe(struct sdw_cdns *cdns)
 {
 	init_completion(&cdns->tx_complete);
+	cdns->bus.port_ops = &cdns_port_ops;
 
 	return 0;
 }
 EXPORT_SYMBOL(sdw_cdns_probe);
 
+int cdns_set_sdw_stream(struct snd_soc_dai *dai,
+		void *stream, bool pcm, int direction)
+{
+	struct sdw_cdns *cdns = snd_soc_dai_get_drvdata(dai);
+	struct sdw_cdns_dma_data *dma;
+
+	dma = kzalloc(sizeof(*dma), GFP_KERNEL);
+	if (!dma)
+		return -ENOMEM;
+
+	if (pcm)
+		dma->stream_type = SDW_STREAM_PCM;
+	else
+		dma->stream_type = SDW_STREAM_PDM;
+
+	dma->bus = &cdns->bus;
+	dma->link_id = cdns->instance;
+
+	dma->stream = stream;
+
+	if (direction == SNDRV_PCM_STREAM_PLAYBACK)
+		dai->playback_dma_data = dma;
+	else
+		dai->capture_dma_data = dma;
+
+	return 0;
+}
+EXPORT_SYMBOL(cdns_set_sdw_stream);
+
+/**
+ * cdns_find_pdi() - Find a free PDI
+ *
+ * @cdns: Cadence instance
+ * @num: Number of PDIs
+ * @pdi: PDI instances
+ *
+ * Find and return a free PDI for a given PDI array
+ */
+static struct sdw_cdns_pdi *cdns_find_pdi(struct sdw_cdns *cdns,
+		unsigned int num, struct sdw_cdns_pdi *pdi)
+{
+	int i;
+
+	for (i = 0; i < num; i++) {
+		if (pdi[i].assigned == true)
+			continue;
+		pdi[i].assigned = true;
+		return &pdi[i];
+	}
+
+	return NULL;
+}
+
+/**
+ * sdw_cdns_config_stream: Configure a stream
+ *
+ * @cdns: Cadence instance
+ * @port: Cadence data port
+ * @ch: Channel count
+ * @dir: Data direction
+ * @pdi: PDI to be used
+ */
+void sdw_cdns_config_stream(struct sdw_cdns *cdns,
+				struct sdw_cdns_port *port,
+				u32 ch, u32 dir, struct sdw_cdns_pdi *pdi)
+{
+	u32 offset, val = 0;
+
+	if (dir == SDW_DATA_DIR_RX)
+		val = CDNS_PORTCTRL_DIRN;
+
+	offset = CDNS_PORTCTRL + port->num * CDNS_PORT_OFFSET;
+	cdns_updatel(cdns, offset, CDNS_PORTCTRL_DIRN, val);
+
+	val = port->num;
+	val |= ((1 << ch) - 1) << SDW_REG_SHIFT(CDNS_PDI_CONFIG_CHANNEL);
+	cdns_writel(cdns, CDNS_PDI_CONFIG(pdi->num), val);
+}
+EXPORT_SYMBOL(sdw_cdns_config_stream);
+
+/**
+ * cdns_get_num_pdi() - Get number of PDIs required
+ *
+ * @cdns: Cadence instance
+ * @pdi: PDI to be used
+ * @num: Number of PDIs
+ * @ch_count: Channel count
+ */
+static int cdns_get_num_pdi(struct sdw_cdns *cdns,
+		struct sdw_cdns_pdi *pdi,
+		unsigned int num, u32 ch_count)
+{
+	int i, pdis = 0;
+
+	for (i = 0; i < num; i++) {
+		if (pdi[i].assigned == true)
+			continue;
+
+		if (pdi[i].ch_count < ch_count)
+			ch_count -= pdi[i].ch_count;
+		else
+			ch_count = 0;
+
+		pdis++;
+
+		if (!ch_count)
+			break;
+	}
+
+	if (ch_count)
+		return 0;
+
+	return pdis;
+}
+
+/**
+ * sdw_cdns_get_stream() - Get stream information
+ *
+ * @cdns: Cadence instance
+ * @stream: Stream to be allocated
+ * @ch: Channel count
+ * @dir: Data direction
+ */
+int sdw_cdns_get_stream(struct sdw_cdns *cdns,
+			struct sdw_cdns_streams *stream,
+			u32 ch, u32 dir)
+{
+	int pdis = 0;
+
+	if (dir == SDW_DATA_DIR_RX)
+		pdis = cdns_get_num_pdi(cdns, stream->in, stream->num_in, ch);
+	else
+		pdis = cdns_get_num_pdi(cdns, stream->out, stream->num_out, ch);
+
+	/* check if we found PDI, else find in bi-directional */
+	if (!pdis)
+		pdis = cdns_get_num_pdi(cdns, stream->bd, stream->num_bd, ch);
+
+	return pdis;
+}
+EXPORT_SYMBOL(sdw_cdns_get_stream);
+
+/**
+ * sdw_cdns_alloc_stream() - Allocate a stream
+ *
+ * @cdns: Cadence instance
+ * @stream: Stream to be allocated
+ * @port: Cadence data port
+ * @ch: Channel count
+ * @dir: Data direction
+ */
+int sdw_cdns_alloc_stream(struct sdw_cdns *cdns,
+			struct sdw_cdns_streams *stream,
+			struct sdw_cdns_port *port, u32 ch, u32 dir)
+{
+	struct sdw_cdns_pdi *pdi = NULL;
+
+	if (dir == SDW_DATA_DIR_RX)
+		pdi = cdns_find_pdi(cdns, stream->num_in, stream->in);
+	else
+		pdi = cdns_find_pdi(cdns, stream->num_out, stream->out);
+
+	/* check if we found a PDI, else find in bi-directional */
+	if (!pdi)
+		pdi = cdns_find_pdi(cdns, stream->num_bd, stream->bd);
+
+	if (!pdi)
+		return -EIO;
+
+	port->pdi = pdi;
+	pdi->l_ch_num = 0;
+	pdi->h_ch_num = ch - 1;
+	pdi->dir = dir;
+	pdi->ch_count = ch;
+
+	return 0;
+}
+EXPORT_SYMBOL(sdw_cdns_alloc_stream);
+
+void sdw_cdns_shutdown(struct snd_pcm_substream *substream,
+					struct snd_soc_dai *dai)
+{
+	struct sdw_cdns_dma_data *dma;
+
+	dma = snd_soc_dai_get_dma_data(dai, substream);
+	if (!dma)
+		return;
+
+	snd_soc_dai_set_dma_data(dai, substream, NULL);
+	kfree(dma);
+}
+EXPORT_SYMBOL(sdw_cdns_shutdown);
+
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("Cadence Soundwire Library");
diff --git a/drivers/soundwire/cadence_master.h b/drivers/soundwire/cadence_master.h
index beaf6c9804eb..eb902b19c5a4 100644
--- a/drivers/soundwire/cadence_master.h
+++ b/drivers/soundwire/cadence_master.h
@@ -1,9 +1,116 @@
 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
 // Copyright(c) 2015-17 Intel Corporation.
+#include <sound/soc.h>
 
 #ifndef __SDW_CADENCE_H
 #define __SDW_CADENCE_H
 
+/**
+ * struct sdw_cdns_pdi: PDI (Physical Data Interface) instance
+ *
+ * @assigned: pdi assigned
+ * @num: pdi number
+ * @intel_alh_id: link identifier
+ * @l_ch_num: low channel for PDI
+ * @h_ch_num: high channel for PDI
+ * @ch_count: total channel count for PDI
+ * @dir: data direction
+ * @type: stream type, PDM or PCM
+ */
+struct sdw_cdns_pdi {
+	bool assigned;
+	int num;
+	int intel_alh_id;
+	int l_ch_num;
+	int h_ch_num;
+	int ch_count;
+	enum sdw_data_direction dir;
+	enum sdw_stream_type type;
+};
+
+/**
+ * struct sdw_cdns_port: Cadence port structure
+ *
+ * @num: port number
+ * @assigned: port assigned
+ * @ch: channel count
+ * @direction: data port direction
+ * @pdi: pdi for this port
+ */
+struct sdw_cdns_port {
+	unsigned int num;
+	bool assigned;
+	unsigned int ch;
+	enum sdw_data_direction direction;
+	struct sdw_cdns_pdi *pdi;
+};
+
+/**
+ * struct sdw_cdns_streams: Cadence stream data structure
+ *
+ * @num_bd: number of bidirectional streams
+ * @num_in: number of input streams
+ * @num_out: number of output streams
+ * @num_ch_bd: number of bidirectional stream channels
+ * @num_ch_bd: number of input stream channels
+ * @num_ch_bd: number of output stream channels
+ * @num_pdi: total number of PDIs
+ * @bd: bidirectional streams
+ * @in: input streams
+ * @out: output streams
+ */
+struct sdw_cdns_streams {
+	unsigned int num_bd;
+	unsigned int num_in;
+	unsigned int num_out;
+	unsigned int num_ch_bd;
+	unsigned int num_ch_in;
+	unsigned int num_ch_out;
+	unsigned int num_pdi;
+	struct sdw_cdns_pdi *bd;
+	struct sdw_cdns_pdi *in;
+	struct sdw_cdns_pdi *out;
+};
+
+/**
+ * struct sdw_cdns_stream_config: stream configuration
+ *
+ * @pcm_bd: number of bidirectional PCM streams supported
+ * @pcm_in: number of input PCM streams supported
+ * @pcm_out: number of output PCM streams supported
+ * @pdm_bd: number of bidirectional PDM streams supported
+ * @pdm_in: number of input PDM streams supported
+ * @pdm_out: number of output PDM streams supported
+ */
+struct sdw_cdns_stream_config {
+	unsigned int pcm_bd;
+	unsigned int pcm_in;
+	unsigned int pcm_out;
+	unsigned int pdm_bd;
+	unsigned int pdm_in;
+	unsigned int pdm_out;
+};
+
+/**
+ * struct sdw_cdns_dma_data: Cadence DMA data
+ *
+ * @name: SoundWire stream name
+ * @nr_ports: Number of ports
+ * @port: Ports
+ * @bus: Bus handle
+ * @stream_type: Stream type
+ * @link_id: Master link id
+ */
+struct sdw_cdns_dma_data {
+	char *name;
+	struct sdw_stream_runtime *stream;
+	int nr_ports;
+	struct sdw_cdns_port **port;
+	struct sdw_bus *bus;
+	enum sdw_stream_type stream_type;
+	int link_id;
+};
+
 /**
  * struct sdw_cdns - Cadence driver context
  * @dev: Linux device
@@ -12,6 +119,10 @@
  * @response_buf: SoundWire response buffer
  * @tx_complete: Tx completion
  * @defer: Defer pointer
+ * @ports: Data ports
+ * @num_ports: Total number of data ports
+ * @pcm: PCM streams
+ * @pdm: PDM streams
  * @registers: Cadence registers
  * @link_up: Link status
  * @msg_count: Messages sent on bus
@@ -25,6 +136,12 @@ struct sdw_cdns {
 	struct completion tx_complete;
 	struct sdw_defer *defer;
 
+	struct sdw_cdns_port *ports;
+	int num_ports;
+
+	struct sdw_cdns_streams pcm;
+	struct sdw_cdns_streams pdm;
+
 	void __iomem *registers;
 
 	bool link_up;
@@ -42,7 +159,41 @@ irqreturn_t sdw_cdns_irq(int irq, void *dev_id);
 irqreturn_t sdw_cdns_thread(int irq, void *dev_id);
 
 int sdw_cdns_init(struct sdw_cdns *cdns);
+int sdw_cdns_pdi_init(struct sdw_cdns *cdns,
+			struct sdw_cdns_stream_config config);
 int sdw_cdns_enable_interrupt(struct sdw_cdns *cdns);
 
+int sdw_cdns_get_stream(struct sdw_cdns *cdns,
+			struct sdw_cdns_streams *stream,
+			u32 ch, u32 dir);
+int sdw_cdns_alloc_stream(struct sdw_cdns *cdns,
+			struct sdw_cdns_streams *stream,
+			struct sdw_cdns_port *port, u32 ch, u32 dir);
+void sdw_cdns_config_stream(struct sdw_cdns *cdns, struct sdw_cdns_port *port,
+			u32 ch, u32 dir, struct sdw_cdns_pdi *pdi);
+
+void sdw_cdns_shutdown(struct snd_pcm_substream *substream,
+				struct snd_soc_dai *dai);
+int sdw_cdns_pcm_set_stream(struct snd_soc_dai *dai,
+				void *stream, int direction);
+int sdw_cdns_pdm_set_stream(struct snd_soc_dai *dai,
+				void *stream, int direction);
+
+enum sdw_command_response
+cdns_reset_page_addr(struct sdw_bus *bus, unsigned int dev_num);
+
+enum sdw_command_response
+cdns_xfer_msg(struct sdw_bus *bus, struct sdw_msg *msg);
+
+enum sdw_command_response
+cdns_xfer_msg_defer(struct sdw_bus *bus,
+		struct sdw_msg *msg, struct sdw_defer *defer);
+
+enum sdw_command_response
+cdns_reset_page_addr(struct sdw_bus *bus, unsigned int dev_num);
+
+int cdns_bus_conf(struct sdw_bus *bus, struct sdw_bus_params *params);
 
+int cdns_set_sdw_stream(struct snd_soc_dai *dai,
+		void *stream, bool pcm, int direction);
 #endif /* __SDW_CADENCE_H */
diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c
index 86a7bd1fc912..0a8990e758f9 100644
--- a/drivers/soundwire/intel.c
+++ b/drivers/soundwire/intel.c
@@ -9,6 +9,8 @@
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
 #include <linux/soundwire/sdw_registers.h>
 #include <linux/soundwire/sdw.h>
 #include <linux/soundwire/sdw_intel.h>
@@ -85,6 +87,12 @@
 #define SDW_ALH_STRMZCFG_DMAT		GENMASK(7, 0)
 #define SDW_ALH_STRMZCFG_CHN		GENMASK(19, 16)
 
+enum intel_pdi_type {
+	INTEL_PDI_IN = 0,
+	INTEL_PDI_OUT = 1,
+	INTEL_PDI_BD = 2,
+};
+
 struct sdw_intel {
 	struct sdw_cdns cdns;
 	int instance;
@@ -234,6 +242,490 @@ static int intel_shim_init(struct sdw_intel *sdw)
 	return ret;
 }
 
+/*
+ * PDI routines
+ */
+static void intel_pdi_init(struct sdw_intel *sdw,
+			struct sdw_cdns_stream_config *config)
+{
+	void __iomem *shim = sdw->res->shim;
+	unsigned int link_id = sdw->instance;
+	int pcm_cap, pdm_cap;
+
+	/* PCM Stream Capability */
+	pcm_cap = intel_readw(shim, SDW_SHIM_PCMSCAP(link_id));
+
+	config->pcm_bd = (pcm_cap & SDW_SHIM_PCMSCAP_BSS) >>
+					SDW_REG_SHIFT(SDW_SHIM_PCMSCAP_BSS);
+	config->pcm_in = (pcm_cap & SDW_SHIM_PCMSCAP_ISS) >>
+					SDW_REG_SHIFT(SDW_SHIM_PCMSCAP_ISS);
+	config->pcm_out = (pcm_cap & SDW_SHIM_PCMSCAP_OSS) >>
+					SDW_REG_SHIFT(SDW_SHIM_PCMSCAP_OSS);
+
+	/* PDM Stream Capability */
+	pdm_cap = intel_readw(shim, SDW_SHIM_PDMSCAP(link_id));
+
+	config->pdm_bd = (pdm_cap & SDW_SHIM_PDMSCAP_BSS) >>
+					SDW_REG_SHIFT(SDW_SHIM_PDMSCAP_BSS);
+	config->pdm_in = (pdm_cap & SDW_SHIM_PDMSCAP_ISS) >>
+					SDW_REG_SHIFT(SDW_SHIM_PDMSCAP_ISS);
+	config->pdm_out = (pdm_cap & SDW_SHIM_PDMSCAP_OSS) >>
+					SDW_REG_SHIFT(SDW_SHIM_PDMSCAP_OSS);
+}
+
+static int
+intel_pdi_get_ch_cap(struct sdw_intel *sdw, unsigned int pdi_num, bool pcm)
+{
+	void __iomem *shim = sdw->res->shim;
+	unsigned int link_id = sdw->instance;
+	int count;
+
+	if (pcm) {
+		count = intel_readw(shim, SDW_SHIM_PCMSYCHC(link_id, pdi_num));
+	} else {
+		count = intel_readw(shim, SDW_SHIM_PDMSCAP(link_id));
+		count = ((count & SDW_SHIM_PDMSCAP_CPSS) >>
+					SDW_REG_SHIFT(SDW_SHIM_PDMSCAP_CPSS));
+	}
+
+	/* zero based values for channel count in register */
+	count++;
+
+	return count;
+}
+
+static int intel_pdi_get_ch_update(struct sdw_intel *sdw,
+				struct sdw_cdns_pdi *pdi,
+				unsigned int num_pdi,
+				unsigned int *num_ch, bool pcm)
+{
+	int i, ch_count = 0;
+
+	for (i = 0; i < num_pdi; i++) {
+		pdi->ch_count = intel_pdi_get_ch_cap(sdw, pdi->num, pcm);
+		ch_count += pdi->ch_count;
+		pdi++;
+	}
+
+	*num_ch = ch_count;
+	return 0;
+}
+
+static int intel_pdi_stream_ch_update(struct sdw_intel *sdw,
+				struct sdw_cdns_streams *stream, bool pcm)
+{
+	intel_pdi_get_ch_update(sdw, stream->bd, stream->num_bd,
+			&stream->num_ch_bd, pcm);
+
+	intel_pdi_get_ch_update(sdw, stream->in, stream->num_in,
+			&stream->num_ch_in, pcm);
+
+	intel_pdi_get_ch_update(sdw, stream->out, stream->num_out,
+			&stream->num_ch_out, pcm);
+
+	return 0;
+}
+
+static int intel_pdi_ch_update(struct sdw_intel *sdw)
+{
+	/* First update PCM streams followed by PDM streams */
+	intel_pdi_stream_ch_update(sdw, &sdw->cdns.pcm, true);
+	intel_pdi_stream_ch_update(sdw, &sdw->cdns.pdm, false);
+
+	return 0;
+}
+
+static void
+intel_pdi_shim_configure(struct sdw_intel *sdw, struct sdw_cdns_pdi *pdi)
+{
+	void __iomem *shim = sdw->res->shim;
+	unsigned int link_id = sdw->instance;
+	int pdi_conf = 0;
+
+	pdi->intel_alh_id = (link_id * 16) + pdi->num + 5;
+
+	/*
+	 * Program stream parameters to stream SHIM register
+	 * This is applicable for PCM stream only.
+	 */
+	if (pdi->type != SDW_STREAM_PCM)
+		return;
+
+	if (pdi->dir == SDW_DATA_DIR_RX)
+		pdi_conf |= SDW_SHIM_PCMSYCM_DIR;
+	else
+		pdi_conf &= ~(SDW_SHIM_PCMSYCM_DIR);
+
+	pdi_conf |= (pdi->intel_alh_id <<
+			SDW_REG_SHIFT(SDW_SHIM_PCMSYCM_STREAM));
+	pdi_conf |= (pdi->l_ch_num << SDW_REG_SHIFT(SDW_SHIM_PCMSYCM_LCHN));
+	pdi_conf |= (pdi->h_ch_num << SDW_REG_SHIFT(SDW_SHIM_PCMSYCM_HCHN));
+
+	intel_writew(shim, SDW_SHIM_PCMSYCHM(link_id, pdi->num), pdi_conf);
+}
+
+static void
+intel_pdi_alh_configure(struct sdw_intel *sdw, struct sdw_cdns_pdi *pdi)
+{
+	void __iomem *alh = sdw->res->alh;
+	unsigned int link_id = sdw->instance;
+	unsigned int conf;
+
+	pdi->intel_alh_id = (link_id * 16) + pdi->num + 5;
+
+	/* Program Stream config ALH register */
+	conf = intel_readl(alh, SDW_ALH_STRMZCFG(pdi->intel_alh_id));
+
+	conf |= (SDW_ALH_STRMZCFG_DMAT_VAL <<
+			SDW_REG_SHIFT(SDW_ALH_STRMZCFG_DMAT));
+
+	conf |= ((pdi->ch_count - 1) <<
+			SDW_REG_SHIFT(SDW_ALH_STRMZCFG_CHN));
+
+	intel_writel(alh, SDW_ALH_STRMZCFG(pdi->intel_alh_id), conf);
+}
+
+static int intel_config_stream(struct sdw_intel *sdw,
+			struct snd_pcm_substream *substream,
+			struct snd_soc_dai *dai,
+			struct snd_pcm_hw_params *hw_params, int link_id)
+{
+	if (sdw->res->ops && sdw->res->ops->config_stream)
+		return sdw->res->ops->config_stream(sdw->res->arg,
+				substream, dai, hw_params, link_id);
+
+	return -EIO;
+}
+
+/*
+ * DAI routines
+ */
+
+static struct sdw_cdns_port *intel_alloc_port(struct sdw_intel *sdw,
+				u32 ch, u32 dir, bool pcm)
+{
+	struct sdw_cdns *cdns = &sdw->cdns;
+	struct sdw_cdns_port *port = NULL;
+	int i, ret = 0;
+
+	for (i = 0; i < cdns->num_ports; i++) {
+		if (cdns->ports[i].assigned == true)
+			continue;
+
+		port = &cdns->ports[i];
+		port->assigned = true;
+		port->direction = dir;
+		port->ch = ch;
+		break;
+	}
+
+	if (!port) {
+		dev_err(cdns->dev, "Unable to find a free port\n");
+		return NULL;
+	}
+
+	if (pcm) {
+		ret = sdw_cdns_alloc_stream(cdns, &cdns->pcm, port, ch, dir);
+		if (ret)
+			goto out;
+
+		intel_pdi_shim_configure(sdw, port->pdi);
+		sdw_cdns_config_stream(cdns, port, ch, dir, port->pdi);
+
+		intel_pdi_alh_configure(sdw, port->pdi);
+
+	} else {
+		ret = sdw_cdns_alloc_stream(cdns, &cdns->pdm, port, ch, dir);
+	}
+
+out:
+	if (ret) {
+		port->assigned = false;
+		port = NULL;
+	}
+
+	return port;
+}
+
+static void intel_port_cleanup(struct sdw_cdns_dma_data *dma)
+{
+	int i;
+
+	for (i = 0; i < dma->nr_ports; i++) {
+		if (dma->port[i]) {
+			dma->port[i]->pdi->assigned = false;
+			dma->port[i]->pdi = NULL;
+			dma->port[i]->assigned = false;
+			dma->port[i] = NULL;
+		}
+	}
+}
+
+static int intel_hw_params(struct snd_pcm_substream *substream,
+				struct snd_pcm_hw_params *params,
+				struct snd_soc_dai *dai)
+{
+	struct sdw_cdns *cdns = snd_soc_dai_get_drvdata(dai);
+	struct sdw_intel *sdw = cdns_to_intel(cdns);
+	struct sdw_cdns_dma_data *dma;
+	struct sdw_stream_config sconfig;
+	struct sdw_port_config *pconfig;
+	int ret, i, ch, dir;
+	bool pcm = true;
+
+	dma = snd_soc_dai_get_dma_data(dai, substream);
+	if (!dma)
+		return -EIO;
+
+	ch = params_channels(params);
+	if (substream->stream == SNDRV_PCM_STREAM_CAPTURE)
+		dir = SDW_DATA_DIR_RX;
+	else
+		dir = SDW_DATA_DIR_TX;
+
+	if (dma->stream_type == SDW_STREAM_PDM) {
+		/* TODO: Check whether PDM decimator is already in use */
+		dma->nr_ports = sdw_cdns_get_stream(cdns, &cdns->pdm, ch, dir);
+		pcm = false;
+	} else {
+		dma->nr_ports = sdw_cdns_get_stream(cdns, &cdns->pcm, ch, dir);
+	}
+
+	if (!dma->nr_ports) {
+		dev_err(dai->dev, "ports/resources not available");
+		return -EINVAL;
+	}
+
+	dma->port = kcalloc(dma->nr_ports, sizeof(*dma->port), GFP_KERNEL);
+	if (!dma->port)
+		return -ENOMEM;
+
+	for (i = 0; i < dma->nr_ports; i++) {
+		dma->port[i] = intel_alloc_port(sdw, ch, dir, pcm);
+		if (!dma->port[i]) {
+			ret = -EINVAL;
+			goto port_error;
+		}
+	}
+
+	/* Inform DSP about PDI stream number */
+	for (i = 0; i < dma->nr_ports; i++) {
+		ret = intel_config_stream(sdw, substream, dai, params,
+				dma->port[i]->pdi->intel_alh_id);
+		if (ret)
+			goto port_error;
+	}
+
+	sconfig.direction = dir;
+	sconfig.ch_count = ch;
+	sconfig.frame_rate = params_rate(params);
+	sconfig.type = dma->stream_type;
+
+	if (dma->stream_type == SDW_STREAM_PDM) {
+		sconfig.frame_rate *= 50;
+		sconfig.bps = 1;
+	} else {
+		sconfig.bps = snd_pcm_format_width(params_format(params));
+	}
+
+	/* Port configuration */
+	pconfig = kcalloc(dma->nr_ports, sizeof(*pconfig), GFP_KERNEL);
+	if (!pconfig) {
+		ret =  -ENOMEM;
+		goto port_error;
+	}
+
+	for (i = 0; i < dma->nr_ports; i++) {
+		pconfig[i].num = dma->port[i]->num;
+		pconfig[i].ch_mask = (1 << ch) - 1;
+	}
+
+	ret = sdw_stream_add_master(&cdns->bus, &sconfig,
+				pconfig, dma->nr_ports, dma->stream);
+	if (ret) {
+		dev_err(cdns->dev, "add master to stream failed:%d", ret);
+		goto stream_error;
+	}
+
+	kfree(pconfig);
+	return ret;
+
+stream_error:
+	kfree(pconfig);
+port_error:
+	intel_port_cleanup(dma);
+	kfree(dma->port);
+	return ret;
+}
+
+static int
+intel_hw_free(struct snd_pcm_substream *substream, struct snd_soc_dai *dai)
+{
+	struct sdw_cdns *cdns = snd_soc_dai_get_drvdata(dai);
+	struct sdw_cdns_dma_data *dma;
+	int ret;
+
+	dma = snd_soc_dai_get_dma_data(dai, substream);
+	if (!dma)
+		return -EIO;
+
+	ret = sdw_stream_remove_master(&cdns->bus, dma->stream);
+	if (ret < 0)
+		dev_err(dai->dev, "remove master from stream %s failed: %d",
+							dma->stream->name, ret);
+
+	intel_port_cleanup(dma);
+	kfree(dma->port);
+	return ret;
+}
+
+static int intel_pcm_set_sdw_stream(struct snd_soc_dai *dai,
+					void *stream, int direction)
+{
+	return cdns_set_sdw_stream(dai, stream, true, direction);
+}
+
+static int intel_pdm_set_sdw_stream(struct snd_soc_dai *dai,
+					void *stream, int direction)
+{
+	return cdns_set_sdw_stream(dai, stream, false, direction);
+}
+
+static struct snd_soc_dai_ops intel_pcm_dai_ops = {
+	.hw_params = intel_hw_params,
+	.hw_free = intel_hw_free,
+	.shutdown = sdw_cdns_shutdown,
+	.set_sdw_stream = intel_pcm_set_sdw_stream,
+};
+
+static struct snd_soc_dai_ops intel_pdm_dai_ops = {
+	.hw_params = intel_hw_params,
+	.hw_free = intel_hw_free,
+	.shutdown = sdw_cdns_shutdown,
+	.set_sdw_stream = intel_pdm_set_sdw_stream,
+};
+
+static const struct snd_soc_component_driver dai_component = {
+	.name           = "soundwire",
+};
+
+static int intel_create_dai(struct sdw_cdns *cdns,
+			struct snd_soc_dai_driver *dais,
+			enum intel_pdi_type type,
+			u32 num, u32 off, u32 max_ch, bool pcm)
+{
+	int i;
+
+	if (num == 0)
+		return 0;
+
+	 /* TODO: Read supported rates/formats from hardware */
+	for (i = off; i < (off + num); i++) {
+		dais[i].name = kasprintf(GFP_KERNEL, "SDW%d Pin%d",
+					cdns->instance, i);
+		if (!dais[i].name)
+			return -ENOMEM;
+
+		if (type == INTEL_PDI_BD || type == INTEL_PDI_OUT) {
+			dais[i].playback.stream_name = kasprintf(GFP_KERNEL,
+							"SDW%d Tx%d",
+							cdns->instance, i);
+			if (!dais[i].playback.stream_name) {
+				kfree(dais[i].name);
+				return -ENOMEM;
+			}
+
+			dais[i].playback.channels_min = 1;
+			dais[i].playback.channels_max = max_ch;
+			dais[i].playback.rates = SNDRV_PCM_RATE_48000;
+			dais[i].playback.formats = SNDRV_PCM_FMTBIT_S16_LE;
+		}
+
+		if (type == INTEL_PDI_BD || type == INTEL_PDI_IN) {
+			dais[i].capture.stream_name = kasprintf(GFP_KERNEL,
+							"SDW%d Rx%d",
+							cdns->instance, i);
+			if (!dais[i].capture.stream_name) {
+				kfree(dais[i].name);
+				kfree(dais[i].playback.stream_name);
+				return -ENOMEM;
+			}
+
+			dais[i].playback.channels_min = 1;
+			dais[i].playback.channels_max = max_ch;
+			dais[i].capture.rates = SNDRV_PCM_RATE_48000;
+			dais[i].capture.formats = SNDRV_PCM_FMTBIT_S16_LE;
+		}
+
+		dais[i].id = SDW_DAI_ID_RANGE_START + i;
+
+		if (pcm)
+			dais[i].ops = &intel_pcm_dai_ops;
+		else
+			dais[i].ops = &intel_pdm_dai_ops;
+	}
+
+	return 0;
+}
+
+static int intel_register_dai(struct sdw_intel *sdw)
+{
+	struct sdw_cdns *cdns = &sdw->cdns;
+	struct sdw_cdns_streams *stream;
+	struct snd_soc_dai_driver *dais;
+	int num_dai, ret, off = 0;
+
+	/* DAIs are created based on total number of PDIs supported */
+	num_dai = cdns->pcm.num_pdi + cdns->pdm.num_pdi;
+
+	dais = devm_kcalloc(cdns->dev, num_dai, sizeof(*dais), GFP_KERNEL);
+	if (!dais)
+		return -ENOMEM;
+
+	/* Create PCM DAIs */
+	stream = &cdns->pcm;
+
+	ret = intel_create_dai(cdns, dais, INTEL_PDI_IN,
+			stream->num_in, off, stream->num_ch_in, true);
+	if (ret)
+		return ret;
+
+	off += cdns->pcm.num_in;
+	ret = intel_create_dai(cdns, dais, INTEL_PDI_OUT,
+			cdns->pcm.num_out, off, stream->num_ch_out, true);
+	if (ret)
+		return ret;
+
+	off += cdns->pcm.num_out;
+	ret = intel_create_dai(cdns, dais, INTEL_PDI_BD,
+			cdns->pcm.num_bd, off, stream->num_ch_bd, true);
+	if (ret)
+		return ret;
+
+	/* Create PDM DAIs */
+	stream = &cdns->pdm;
+	off += cdns->pcm.num_bd;
+	ret = intel_create_dai(cdns, dais, INTEL_PDI_IN,
+			cdns->pdm.num_in, off, stream->num_ch_in, false);
+	if (ret)
+		return ret;
+
+	off += cdns->pdm.num_in;
+	ret = intel_create_dai(cdns, dais, INTEL_PDI_OUT,
+			cdns->pdm.num_out, off, stream->num_ch_out, false);
+	if (ret)
+		return ret;
+
+	off += cdns->pdm.num_bd;
+	ret = intel_create_dai(cdns, dais, INTEL_PDI_BD,
+			cdns->pdm.num_bd, off, stream->num_ch_bd, false);
+	if (ret)
+		return ret;
+
+	return snd_soc_register_component(cdns->dev, &dai_component,
+				dais, num_dai);
+}
+
 static int intel_prop_read(struct sdw_bus *bus)
 {
 	/* Initialize with default handler to read all DisCo properties */
@@ -252,11 +744,20 @@ static int intel_prop_read(struct sdw_bus *bus)
 	return 0;
 }
 
+static struct sdw_master_ops sdw_intel_ops = {
+	.read_prop = sdw_master_read_prop,
+	.xfer_msg = cdns_xfer_msg,
+	.xfer_msg_defer = cdns_xfer_msg_defer,
+	.reset_page_addr = cdns_reset_page_addr,
+	.set_bus_conf = cdns_bus_conf,
+};
+
 /*
  * probe and init
  */
 static int intel_probe(struct platform_device *pdev)
 {
+	struct sdw_cdns_stream_config config;
 	struct sdw_intel *sdw;
 	int ret;
 
@@ -276,8 +777,11 @@ static int intel_probe(struct platform_device *pdev)
 	sdw_cdns_probe(&sdw->cdns);
 
 	/* Set property read ops */
-	sdw_cdns_master_ops.read_prop = intel_prop_read;
-	sdw->cdns.bus.ops = &sdw_cdns_master_ops;
+	sdw_intel_ops.read_prop = intel_prop_read;
+	sdw->cdns.bus.ops = &sdw_intel_ops;
+
+	sdw_intel_ops.read_prop = intel_prop_read;
+	sdw->cdns.bus.ops = &sdw_intel_ops;
 
 	platform_set_drvdata(pdev, sdw);
 
@@ -296,9 +800,15 @@ static int intel_probe(struct platform_device *pdev)
 		goto err_init;
 
 	ret = sdw_cdns_enable_interrupt(&sdw->cdns);
+
+	/* Read the PDI config and initialize cadence PDI */
+	intel_pdi_init(sdw, &config);
+	ret = sdw_cdns_pdi_init(&sdw->cdns, config);
 	if (ret)
 		goto err_init;
 
+	intel_pdi_ch_update(sdw);
+
 	/* Acquire IRQ */
 	ret = request_threaded_irq(sdw->res->irq, sdw_cdns_irq,
 			sdw_cdns_thread, IRQF_SHARED, KBUILD_MODNAME,
@@ -309,8 +819,18 @@ static int intel_probe(struct platform_device *pdev)
 		goto err_init;
 	}
 
+	/* Register DAIs */
+	ret = intel_register_dai(sdw);
+	if (ret) {
+		dev_err(sdw->cdns.dev, "DAI registration failed: %d", ret);
+		snd_soc_unregister_component(sdw->cdns.dev);
+		goto err_dai;
+	}
+
 	return 0;
 
+err_dai:
+	free_irq(sdw->res->irq, sdw);
 err_init:
 	sdw_delete_bus_master(&sdw->cdns.bus);
 err_master_reg:
@@ -324,6 +844,7 @@ static int intel_remove(struct platform_device *pdev)
 	sdw = platform_get_drvdata(pdev);
 
 	free_irq(sdw->res->irq, sdw);
+	snd_soc_unregister_component(sdw->cdns.dev);
 	sdw_delete_bus_master(&sdw->cdns.bus);
 
 	return 0;
diff --git a/drivers/soundwire/intel.h b/drivers/soundwire/intel.h
index ffa30d9535a2..c1a5bac6212e 100644
--- a/drivers/soundwire/intel.h
+++ b/drivers/soundwire/intel.h
@@ -10,6 +10,8 @@
  * @shim: Audio shim pointer
  * @alh: ALH (Audio Link Hub) pointer
  * @irq: Interrupt line
+ * @ops: Shim callback ops
+ * @arg: Shim callback ops argument
  *
  * This is set as pdata for each link instance.
  */
@@ -18,6 +20,8 @@ struct sdw_intel_link_res {
 	void __iomem *shim;
 	void __iomem *alh;
 	int irq;
+	const struct sdw_intel_ops *ops;
+	void *arg;
 };
 
 #endif /* __SDW_INTEL_LOCAL_H */
diff --git a/drivers/soundwire/intel_init.c b/drivers/soundwire/intel_init.c
index 6f2bb99526f2..d1ea6b4d0ad3 100644
--- a/drivers/soundwire/intel_init.c
+++ b/drivers/soundwire/intel_init.c
@@ -111,6 +111,9 @@ static struct sdw_intel_ctx
 		link->res.shim = res->mmio_base + SDW_SHIM_BASE;
 		link->res.alh = res->mmio_base + SDW_ALH_BASE;
 
+		link->res.ops = res->ops;
+		link->res.arg = res->arg;
+
 		memset(&pdevinfo, 0, sizeof(pdevinfo));
 
 		pdevinfo.parent = res->parent;
diff --git a/drivers/soundwire/stream.c b/drivers/soundwire/stream.c
new file mode 100644
index 000000000000..8974a0fcda1b
--- /dev/null
+++ b/drivers/soundwire/stream.c
@@ -0,0 +1,1479 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+// Copyright(c) 2015-18 Intel Corporation.
+
+/*
+ *  stream.c - SoundWire Bus stream operations.
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/slab.h>
+#include <linux/soundwire/sdw_registers.h>
+#include <linux/soundwire/sdw.h>
+#include "bus.h"
+
+/*
+ * Array of supported rows and columns as per MIPI SoundWire Specification 1.1
+ *
+ * The rows are arranged as per the array index value programmed
+ * in register. The index 15 has dummy value 0 in order to fill hole.
+ */
+int rows[SDW_FRAME_ROWS] = {48, 50, 60, 64, 75, 80, 125, 147,
+			96, 100, 120, 128, 150, 160, 250, 0,
+			192, 200, 240, 256, 72, 144, 90, 180};
+
+int cols[SDW_FRAME_COLS] = {2, 4, 6, 8, 10, 12, 14, 16};
+
+static int sdw_find_col_index(int col)
+{
+	int i;
+
+	for (i = 0; i < SDW_FRAME_COLS; i++) {
+		if (cols[i] == col)
+			return i;
+	}
+
+	pr_warn("Requested column not found, selecting lowest column no: 2\n");
+	return 0;
+}
+
+static int sdw_find_row_index(int row)
+{
+	int i;
+
+	for (i = 0; i < SDW_FRAME_ROWS; i++) {
+		if (rows[i] == row)
+			return i;
+	}
+
+	pr_warn("Requested row not found, selecting lowest row no: 48\n");
+	return 0;
+}
+static int _sdw_program_slave_port_params(struct sdw_bus *bus,
+				struct sdw_slave *slave,
+				struct sdw_transport_params *t_params,
+				enum sdw_dpn_type type)
+{
+	u32 addr1, addr2, addr3, addr4;
+	int ret;
+	u16 wbuf;
+
+	if (bus->params.next_bank) {
+		addr1 = SDW_DPN_OFFSETCTRL2_B1(t_params->port_num);
+		addr2 = SDW_DPN_BLOCKCTRL3_B1(t_params->port_num);
+		addr3 = SDW_DPN_SAMPLECTRL2_B1(t_params->port_num);
+		addr4 = SDW_DPN_HCTRL_B1(t_params->port_num);
+	} else {
+		addr1 = SDW_DPN_OFFSETCTRL2_B0(t_params->port_num);
+		addr2 = SDW_DPN_BLOCKCTRL3_B0(t_params->port_num);
+		addr3 = SDW_DPN_SAMPLECTRL2_B0(t_params->port_num);
+		addr4 = SDW_DPN_HCTRL_B0(t_params->port_num);
+	}
+
+	/* Program DPN_OffsetCtrl2 registers */
+	ret = sdw_write(slave, addr1, t_params->offset2);
+	if (ret < 0) {
+		dev_err(bus->dev, "DPN_OffsetCtrl2 register write failed");
+		return ret;
+	}
+
+	/* Program DPN_BlockCtrl3 register */
+	ret = sdw_write(slave, addr2, t_params->blk_pkg_mode);
+	if (ret < 0) {
+		dev_err(bus->dev, "DPN_BlockCtrl3 register write failed");
+		return ret;
+	}
+
+	/*
+	 * Data ports are FULL, SIMPLE and REDUCED. This function handles
+	 * FULL and REDUCED only and and beyond this point only FULL is
+	 * handled, so bail out if we are not FULL data port type
+	 */
+	if (type != SDW_DPN_FULL)
+		return ret;
+
+	/* Program DPN_SampleCtrl2 register */
+	wbuf = (t_params->sample_interval - 1);
+	wbuf &= SDW_DPN_SAMPLECTRL_HIGH;
+	wbuf >>= SDW_REG_SHIFT(SDW_DPN_SAMPLECTRL_HIGH);
+
+	ret = sdw_write(slave, addr3, wbuf);
+	if (ret < 0) {
+		dev_err(bus->dev, "DPN_SampleCtrl2 register write failed");
+		return ret;
+	}
+
+	/* Program DPN_HCtrl register */
+	wbuf = t_params->hstart;
+	wbuf <<= SDW_REG_SHIFT(SDW_DPN_HCTRL_HSTART);
+	wbuf |= t_params->hstop;
+
+	ret = sdw_write(slave, addr4, wbuf);
+	if (ret < 0)
+		dev_err(bus->dev, "DPN_HCtrl register write failed");
+
+	return ret;
+}
+
+static int sdw_program_slave_port_params(struct sdw_bus *bus,
+			struct sdw_slave_runtime *s_rt,
+			struct sdw_port_runtime *p_rt)
+{
+	struct sdw_transport_params *t_params = &p_rt->transport_params;
+	struct sdw_port_params *p_params = &p_rt->port_params;
+	struct sdw_slave_prop *slave_prop = &s_rt->slave->prop;
+	u32 addr1, addr2, addr3, addr4, addr5, addr6;
+	struct sdw_dpn_prop *dpn_prop;
+	int ret;
+	u8 wbuf;
+
+	dpn_prop = sdw_get_slave_dpn_prop(s_rt->slave,
+					s_rt->direction,
+					t_params->port_num);
+	if (!dpn_prop)
+		return -EINVAL;
+
+	addr1 = SDW_DPN_PORTCTRL(t_params->port_num);
+	addr2 = SDW_DPN_BLOCKCTRL1(t_params->port_num);
+
+	if (bus->params.next_bank) {
+		addr3 = SDW_DPN_SAMPLECTRL1_B1(t_params->port_num);
+		addr4 = SDW_DPN_OFFSETCTRL1_B1(t_params->port_num);
+		addr5 = SDW_DPN_BLOCKCTRL2_B1(t_params->port_num);
+		addr6 = SDW_DPN_LANECTRL_B1(t_params->port_num);
+
+	} else {
+		addr3 = SDW_DPN_SAMPLECTRL1_B0(t_params->port_num);
+		addr4 = SDW_DPN_OFFSETCTRL1_B0(t_params->port_num);
+		addr5 = SDW_DPN_BLOCKCTRL2_B0(t_params->port_num);
+		addr6 = SDW_DPN_LANECTRL_B0(t_params->port_num);
+	}
+
+	/* Program DPN_PortCtrl register */
+	wbuf = p_params->data_mode << SDW_REG_SHIFT(SDW_DPN_PORTCTRL_DATAMODE);
+	wbuf |= p_params->flow_mode;
+
+	ret = sdw_update(s_rt->slave, addr1, 0xF, wbuf);
+	if (ret < 0) {
+		dev_err(&s_rt->slave->dev,
+			"DPN_PortCtrl register write failed for port %d",
+			t_params->port_num);
+		return ret;
+	}
+
+	/* Program DPN_BlockCtrl1 register */
+	ret = sdw_write(s_rt->slave, addr2, (p_params->bps - 1));
+	if (ret < 0) {
+		dev_err(&s_rt->slave->dev,
+			"DPN_BlockCtrl1 register write failed for port %d",
+			t_params->port_num);
+		return ret;
+	}
+
+	/* Program DPN_SampleCtrl1 register */
+	wbuf = (t_params->sample_interval - 1) & SDW_DPN_SAMPLECTRL_LOW;
+	ret = sdw_write(s_rt->slave, addr3, wbuf);
+	if (ret < 0) {
+		dev_err(&s_rt->slave->dev,
+			"DPN_SampleCtrl1 register write failed for port %d",
+			t_params->port_num);
+		return ret;
+	}
+
+	/* Program DPN_OffsetCtrl1 registers */
+	ret = sdw_write(s_rt->slave, addr4, t_params->offset1);
+	if (ret < 0) {
+		dev_err(&s_rt->slave->dev,
+			"DPN_OffsetCtrl1 register write failed for port %d",
+			t_params->port_num);
+		return ret;
+	}
+
+	/* Program DPN_BlockCtrl2 register*/
+	if (t_params->blk_grp_ctrl_valid) {
+		ret = sdw_write(s_rt->slave, addr5, t_params->blk_grp_ctrl);
+		if (ret < 0) {
+			dev_err(&s_rt->slave->dev,
+				"DPN_BlockCtrl2 reg write failed for port %d",
+				t_params->port_num);
+			return ret;
+		}
+	}
+
+	/* program DPN_LaneCtrl register */
+	if (slave_prop->lane_control_support) {
+		ret = sdw_write(s_rt->slave, addr6, t_params->lane_ctrl);
+		if (ret < 0) {
+			dev_err(&s_rt->slave->dev,
+				"DPN_LaneCtrl register write failed for port %d",
+				t_params->port_num);
+			return ret;
+		}
+	}
+
+	if (dpn_prop->type != SDW_DPN_SIMPLE) {
+		ret = _sdw_program_slave_port_params(bus, s_rt->slave,
+						t_params, dpn_prop->type);
+		if (ret < 0)
+			dev_err(&s_rt->slave->dev,
+				"Transport reg write failed for port: %d",
+				t_params->port_num);
+	}
+
+	return ret;
+}
+
+static int sdw_program_master_port_params(struct sdw_bus *bus,
+		struct sdw_port_runtime *p_rt)
+{
+	int ret;
+
+	/*
+	 * we need to set transport and port parameters for the port.
+	 * Transport parameters refers to the smaple interval, offsets and
+	 * hstart/stop etc of the data. Port parameters refers to word
+	 * length, flow mode etc of the port
+	 */
+	ret = bus->port_ops->dpn_set_port_transport_params(bus,
+					&p_rt->transport_params,
+					bus->params.next_bank);
+	if (ret < 0)
+		return ret;
+
+	return bus->port_ops->dpn_set_port_params(bus,
+				&p_rt->port_params,
+				bus->params.next_bank);
+}
+
+/**
+ * sdw_program_port_params() - Programs transport parameters of Master(s)
+ * and Slave(s)
+ *
+ * @m_rt: Master stream runtime
+ */
+static int sdw_program_port_params(struct sdw_master_runtime *m_rt)
+{
+	struct sdw_slave_runtime *s_rt = NULL;
+	struct sdw_bus *bus = m_rt->bus;
+	struct sdw_port_runtime *p_rt;
+	int ret = 0;
+
+	/* Program transport & port parameters for Slave(s) */
+	list_for_each_entry(s_rt, &m_rt->slave_rt_list, m_rt_node) {
+		list_for_each_entry(p_rt, &s_rt->port_list, port_node) {
+			ret = sdw_program_slave_port_params(bus, s_rt, p_rt);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
+	/* Program transport & port parameters for Master(s) */
+	list_for_each_entry(p_rt, &m_rt->port_list, port_node) {
+		ret = sdw_program_master_port_params(bus, p_rt);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * sdw_enable_disable_slave_ports: Enable/disable slave data port
+ *
+ * @bus: bus instance
+ * @s_rt: slave runtime
+ * @p_rt: port runtime
+ * @en: enable or disable operation
+ *
+ * This function only sets the enable/disable bits in the relevant bank, the
+ * actual enable/disable is done with a bank switch
+ */
+static int sdw_enable_disable_slave_ports(struct sdw_bus *bus,
+				struct sdw_slave_runtime *s_rt,
+				struct sdw_port_runtime *p_rt, bool en)
+{
+	struct sdw_transport_params *t_params = &p_rt->transport_params;
+	u32 addr;
+	int ret;
+
+	if (bus->params.next_bank)
+		addr = SDW_DPN_CHANNELEN_B1(p_rt->num);
+	else
+		addr = SDW_DPN_CHANNELEN_B0(p_rt->num);
+
+	/*
+	 * Since bus doesn't support sharing a port across two streams,
+	 * it is safe to reset this register
+	 */
+	if (en)
+		ret = sdw_update(s_rt->slave, addr, 0xFF, p_rt->ch_mask);
+	else
+		ret = sdw_update(s_rt->slave, addr, 0xFF, 0x0);
+
+	if (ret < 0)
+		dev_err(&s_rt->slave->dev,
+			"Slave chn_en reg write failed:%d port:%d",
+			ret, t_params->port_num);
+
+	return ret;
+}
+
+static int sdw_enable_disable_master_ports(struct sdw_master_runtime *m_rt,
+			struct sdw_port_runtime *p_rt, bool en)
+{
+	struct sdw_transport_params *t_params = &p_rt->transport_params;
+	struct sdw_bus *bus = m_rt->bus;
+	struct sdw_enable_ch enable_ch;
+	int ret = 0;
+
+	enable_ch.port_num = p_rt->num;
+	enable_ch.ch_mask = p_rt->ch_mask;
+	enable_ch.enable = en;
+
+	/* Perform Master port channel(s) enable/disable */
+	if (bus->port_ops->dpn_port_enable_ch) {
+		ret = bus->port_ops->dpn_port_enable_ch(bus,
+				&enable_ch, bus->params.next_bank);
+		if (ret < 0) {
+			dev_err(bus->dev,
+				"Master chn_en write failed:%d port:%d",
+				ret, t_params->port_num);
+			return ret;
+		}
+	} else {
+		dev_err(bus->dev,
+			"dpn_port_enable_ch not supported, %s failed\n",
+			en ? "enable" : "disable");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * sdw_enable_disable_ports() - Enable/disable port(s) for Master and
+ * Slave(s)
+ *
+ * @m_rt: Master stream runtime
+ * @en: mode (enable/disable)
+ */
+static int sdw_enable_disable_ports(struct sdw_master_runtime *m_rt, bool en)
+{
+	struct sdw_port_runtime *s_port, *m_port;
+	struct sdw_slave_runtime *s_rt = NULL;
+	int ret = 0;
+
+	/* Enable/Disable Slave port(s) */
+	list_for_each_entry(s_rt, &m_rt->slave_rt_list, m_rt_node) {
+		list_for_each_entry(s_port, &s_rt->port_list, port_node) {
+			ret = sdw_enable_disable_slave_ports(m_rt->bus, s_rt,
+							s_port, en);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
+	/* Enable/Disable Master port(s) */
+	list_for_each_entry(m_port, &m_rt->port_list, port_node) {
+		ret = sdw_enable_disable_master_ports(m_rt, m_port, en);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int sdw_do_port_prep(struct sdw_slave_runtime *s_rt,
+		struct sdw_prepare_ch prep_ch, enum sdw_port_prep_ops cmd)
+{
+	const struct sdw_slave_ops *ops = s_rt->slave->ops;
+	int ret;
+
+	if (ops->port_prep) {
+		ret = ops->port_prep(s_rt->slave, &prep_ch, cmd);
+		if (ret < 0) {
+			dev_err(&s_rt->slave->dev,
+				"Slave Port Prep cmd %d failed: %d", cmd, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int sdw_prep_deprep_slave_ports(struct sdw_bus *bus,
+			struct sdw_slave_runtime *s_rt,
+			struct sdw_port_runtime *p_rt, bool prep)
+{
+	struct completion *port_ready = NULL;
+	struct sdw_dpn_prop *dpn_prop;
+	struct sdw_prepare_ch prep_ch;
+	unsigned int time_left;
+	bool intr = false;
+	int ret = 0, val;
+	u32 addr;
+
+	prep_ch.num = p_rt->num;
+	prep_ch.ch_mask = p_rt->ch_mask;
+
+	dpn_prop = sdw_get_slave_dpn_prop(s_rt->slave,
+					s_rt->direction,
+					prep_ch.num);
+	if (!dpn_prop) {
+		dev_err(bus->dev,
+			"Slave Port:%d properties not found", prep_ch.num);
+		return -EINVAL;
+	}
+
+	prep_ch.prepare = prep;
+
+	prep_ch.bank = bus->params.next_bank;
+
+	if (dpn_prop->device_interrupts || !dpn_prop->simple_ch_prep_sm)
+		intr = true;
+
+	/*
+	 * Enable interrupt before Port prepare.
+	 * For Port de-prepare, it is assumed that port
+	 * was prepared earlier
+	 */
+	if (prep && intr) {
+		ret = sdw_configure_dpn_intr(s_rt->slave, p_rt->num, prep,
+						dpn_prop->device_interrupts);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* Inform slave about the impending port prepare */
+	sdw_do_port_prep(s_rt, prep_ch, SDW_OPS_PORT_PRE_PREP);
+
+	/* Prepare Slave port implementing CP_SM */
+	if (!dpn_prop->simple_ch_prep_sm) {
+		addr = SDW_DPN_PREPARECTRL(p_rt->num);
+
+		if (prep)
+			ret = sdw_update(s_rt->slave, addr,
+					0xFF, p_rt->ch_mask);
+		else
+			ret = sdw_update(s_rt->slave, addr, 0xFF, 0x0);
+
+		if (ret < 0) {
+			dev_err(&s_rt->slave->dev,
+				"Slave prep_ctrl reg write failed");
+			return ret;
+		}
+
+		/* Wait for completion on port ready */
+		port_ready = &s_rt->slave->port_ready[prep_ch.num];
+		time_left = wait_for_completion_timeout(port_ready,
+				msecs_to_jiffies(dpn_prop->ch_prep_timeout));
+
+		val = sdw_read(s_rt->slave, SDW_DPN_PREPARESTATUS(p_rt->num));
+		val &= p_rt->ch_mask;
+		if (!time_left || val) {
+			dev_err(&s_rt->slave->dev,
+				"Chn prep failed for port:%d", prep_ch.num);
+			return -ETIMEDOUT;
+		}
+	}
+
+	/* Inform slaves about ports prepared */
+	sdw_do_port_prep(s_rt, prep_ch, SDW_OPS_PORT_POST_PREP);
+
+	/* Disable interrupt after Port de-prepare */
+	if (!prep && intr)
+		ret = sdw_configure_dpn_intr(s_rt->slave, p_rt->num, prep,
+						dpn_prop->device_interrupts);
+
+	return ret;
+}
+
+static int sdw_prep_deprep_master_ports(struct sdw_master_runtime *m_rt,
+				struct sdw_port_runtime *p_rt, bool prep)
+{
+	struct sdw_transport_params *t_params = &p_rt->transport_params;
+	struct sdw_bus *bus = m_rt->bus;
+	const struct sdw_master_port_ops *ops = bus->port_ops;
+	struct sdw_prepare_ch prep_ch;
+	int ret = 0;
+
+	prep_ch.num = p_rt->num;
+	prep_ch.ch_mask = p_rt->ch_mask;
+	prep_ch.prepare = prep; /* Prepare/De-prepare */
+	prep_ch.bank = bus->params.next_bank;
+
+	/* Pre-prepare/Pre-deprepare port(s) */
+	if (ops->dpn_port_prep) {
+		ret = ops->dpn_port_prep(bus, &prep_ch);
+		if (ret < 0) {
+			dev_err(bus->dev, "Port prepare failed for port:%d",
+					t_params->port_num);
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+/**
+ * sdw_prep_deprep_ports() - Prepare/De-prepare port(s) for Master(s) and
+ * Slave(s)
+ *
+ * @m_rt: Master runtime handle
+ * @prep: Prepare or De-prepare
+ */
+static int sdw_prep_deprep_ports(struct sdw_master_runtime *m_rt, bool prep)
+{
+	struct sdw_slave_runtime *s_rt = NULL;
+	struct sdw_port_runtime *p_rt;
+	int ret = 0;
+
+	/* Prepare/De-prepare Slave port(s) */
+	list_for_each_entry(s_rt, &m_rt->slave_rt_list, m_rt_node) {
+		list_for_each_entry(p_rt, &s_rt->port_list, port_node) {
+			ret = sdw_prep_deprep_slave_ports(m_rt->bus, s_rt,
+							p_rt, prep);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
+	/* Prepare/De-prepare Master port(s) */
+	list_for_each_entry(p_rt, &m_rt->port_list, port_node) {
+		ret = sdw_prep_deprep_master_ports(m_rt, p_rt, prep);
+		if (ret < 0)
+			return ret;
+	}
+
+	return ret;
+}
+
+/**
+ * sdw_notify_config() - Notify bus configuration
+ *
+ * @m_rt: Master runtime handle
+ *
+ * This function notifies the Master(s) and Slave(s) of the
+ * new bus configuration.
+ */
+static int sdw_notify_config(struct sdw_master_runtime *m_rt)
+{
+	struct sdw_slave_runtime *s_rt;
+	struct sdw_bus *bus = m_rt->bus;
+	struct sdw_slave *slave;
+	int ret = 0;
+
+	if (bus->ops->set_bus_conf) {
+		ret = bus->ops->set_bus_conf(bus, &bus->params);
+		if (ret < 0)
+			return ret;
+	}
+
+	list_for_each_entry(s_rt, &m_rt->slave_rt_list, m_rt_node) {
+		slave = s_rt->slave;
+
+		if (slave->ops->bus_config) {
+			ret = slave->ops->bus_config(slave, &bus->params);
+			if (ret < 0)
+				dev_err(bus->dev, "Notify Slave: %d failed",
+								slave->dev_num);
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+/**
+ * sdw_program_params() - Program transport and port parameters for Master(s)
+ * and Slave(s)
+ *
+ * @bus: SDW bus instance
+ */
+static int sdw_program_params(struct sdw_bus *bus)
+{
+	struct sdw_master_runtime *m_rt = NULL;
+	int ret = 0;
+
+	list_for_each_entry(m_rt, &bus->m_rt_list, bus_node) {
+		ret = sdw_program_port_params(m_rt);
+		if (ret < 0) {
+			dev_err(bus->dev,
+				"Program transport params failed: %d", ret);
+			return ret;
+		}
+
+		ret = sdw_notify_config(m_rt);
+		if (ret < 0) {
+			dev_err(bus->dev, "Notify bus config failed: %d", ret);
+			return ret;
+		}
+
+		/* Enable port(s) on alternate bank for all active streams */
+		if (m_rt->stream->state != SDW_STREAM_ENABLED)
+			continue;
+
+		ret = sdw_enable_disable_ports(m_rt, true);
+		if (ret < 0) {
+			dev_err(bus->dev, "Enable channel failed: %d", ret);
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+static int sdw_bank_switch(struct sdw_bus *bus)
+{
+	int col_index, row_index;
+	struct sdw_msg *wr_msg;
+	u8 *wbuf = NULL;
+	int ret = 0;
+	u16 addr;
+
+	wr_msg = kzalloc(sizeof(*wr_msg), GFP_KERNEL);
+	if (!wr_msg)
+		return -ENOMEM;
+
+	wbuf = kzalloc(sizeof(*wbuf), GFP_KERNEL);
+	if (!wbuf) {
+		ret = -ENOMEM;
+		goto error_1;
+	}
+
+	/* Get row and column index to program register */
+	col_index = sdw_find_col_index(bus->params.col);
+	row_index = sdw_find_row_index(bus->params.row);
+	wbuf[0] = col_index | (row_index << 3);
+
+	if (bus->params.next_bank)
+		addr = SDW_SCP_FRAMECTRL_B1;
+	else
+		addr = SDW_SCP_FRAMECTRL_B0;
+
+	sdw_fill_msg(wr_msg, NULL, addr, 1, SDW_BROADCAST_DEV_NUM,
+					SDW_MSG_FLAG_WRITE, wbuf);
+	wr_msg->ssp_sync = true;
+
+	ret = sdw_transfer(bus, wr_msg);
+	if (ret < 0) {
+		dev_err(bus->dev, "Slave frame_ctrl reg write failed");
+		goto error;
+	}
+
+	kfree(wr_msg);
+	kfree(wbuf);
+	bus->defer_msg.msg = NULL;
+	bus->params.curr_bank = !bus->params.curr_bank;
+	bus->params.next_bank = !bus->params.next_bank;
+
+	return 0;
+
+error:
+	kfree(wbuf);
+error_1:
+	kfree(wr_msg);
+	return ret;
+}
+
+static int do_bank_switch(struct sdw_stream_runtime *stream)
+{
+	struct sdw_master_runtime *m_rt = stream->m_rt;
+	const struct sdw_master_ops *ops;
+	struct sdw_bus *bus = m_rt->bus;
+	int ret = 0;
+
+	ops = bus->ops;
+
+	/* Pre-bank switch */
+	if (ops->pre_bank_switch) {
+		ret = ops->pre_bank_switch(bus);
+		if (ret < 0) {
+			dev_err(bus->dev, "Pre bank switch op failed: %d", ret);
+			return ret;
+		}
+	}
+
+	/* Bank switch */
+	ret = sdw_bank_switch(bus);
+	if (ret < 0) {
+		dev_err(bus->dev, "Bank switch failed: %d", ret);
+		return ret;
+	}
+
+	/* Post-bank switch */
+	if (ops->post_bank_switch) {
+		ret = ops->post_bank_switch(bus);
+		if (ret < 0) {
+			dev_err(bus->dev,
+					"Post bank switch op failed: %d", ret);
+		}
+	}
+
+	return ret;
+}
+
+/**
+ * sdw_release_stream() - Free the assigned stream runtime
+ *
+ * @stream: SoundWire stream runtime
+ *
+ * sdw_release_stream should be called only once per stream
+ */
+void sdw_release_stream(struct sdw_stream_runtime *stream)
+{
+	kfree(stream);
+}
+EXPORT_SYMBOL(sdw_release_stream);
+
+/**
+ * sdw_alloc_stream() - Allocate and return stream runtime
+ *
+ * @stream_name: SoundWire stream name
+ *
+ * Allocates a SoundWire stream runtime instance.
+ * sdw_alloc_stream should be called only once per stream. Typically
+ * invoked from ALSA/ASoC machine/platform driver.
+ */
+struct sdw_stream_runtime *sdw_alloc_stream(char *stream_name)
+{
+	struct sdw_stream_runtime *stream;
+
+	stream = kzalloc(sizeof(*stream), GFP_KERNEL);
+	if (!stream)
+		return NULL;
+
+	stream->name = stream_name;
+	stream->state = SDW_STREAM_ALLOCATED;
+
+	return stream;
+}
+EXPORT_SYMBOL(sdw_alloc_stream);
+
+/**
+ * sdw_alloc_master_rt() - Allocates and initialize Master runtime handle
+ *
+ * @bus: SDW bus instance
+ * @stream_config: Stream configuration
+ * @stream: Stream runtime handle.
+ *
+ * This function is to be called with bus_lock held.
+ */
+static struct sdw_master_runtime
+*sdw_alloc_master_rt(struct sdw_bus *bus,
+			struct sdw_stream_config *stream_config,
+			struct sdw_stream_runtime *stream)
+{
+	struct sdw_master_runtime *m_rt;
+
+	m_rt = stream->m_rt;
+
+	/*
+	 * check if Master is already allocated (as a result of Slave adding
+	 * it first), if so skip allocation and go to configure
+	 */
+	if (m_rt)
+		goto stream_config;
+
+	m_rt = kzalloc(sizeof(*m_rt), GFP_KERNEL);
+	if (!m_rt)
+		return NULL;
+
+	/* Initialization of Master runtime handle */
+	INIT_LIST_HEAD(&m_rt->port_list);
+	INIT_LIST_HEAD(&m_rt->slave_rt_list);
+	stream->m_rt = m_rt;
+
+	list_add_tail(&m_rt->bus_node, &bus->m_rt_list);
+
+stream_config:
+	m_rt->ch_count = stream_config->ch_count;
+	m_rt->bus = bus;
+	m_rt->stream = stream;
+	m_rt->direction = stream_config->direction;
+
+	return m_rt;
+}
+
+/**
+ * sdw_alloc_slave_rt() - Allocate and initialize Slave runtime handle.
+ *
+ * @slave: Slave handle
+ * @stream_config: Stream configuration
+ * @stream: Stream runtime handle
+ *
+ * This function is to be called with bus_lock held.
+ */
+static struct sdw_slave_runtime
+*sdw_alloc_slave_rt(struct sdw_slave *slave,
+			struct sdw_stream_config *stream_config,
+			struct sdw_stream_runtime *stream)
+{
+	struct sdw_slave_runtime *s_rt = NULL;
+
+	s_rt = kzalloc(sizeof(*s_rt), GFP_KERNEL);
+	if (!s_rt)
+		return NULL;
+
+	INIT_LIST_HEAD(&s_rt->port_list);
+	s_rt->ch_count = stream_config->ch_count;
+	s_rt->direction = stream_config->direction;
+	s_rt->slave = slave;
+
+	return s_rt;
+}
+
+static void sdw_master_port_release(struct sdw_bus *bus,
+			struct sdw_master_runtime *m_rt)
+{
+	struct sdw_port_runtime *p_rt, *_p_rt;
+
+	list_for_each_entry_safe(p_rt, _p_rt,
+			&m_rt->port_list, port_node) {
+		list_del(&p_rt->port_node);
+		kfree(p_rt);
+	}
+}
+
+static void sdw_slave_port_release(struct sdw_bus *bus,
+			struct sdw_slave *slave,
+			struct sdw_stream_runtime *stream)
+{
+	struct sdw_port_runtime *p_rt, *_p_rt;
+	struct sdw_master_runtime *m_rt = stream->m_rt;
+	struct sdw_slave_runtime *s_rt;
+
+	list_for_each_entry(s_rt, &m_rt->slave_rt_list, m_rt_node) {
+		if (s_rt->slave != slave)
+			continue;
+
+		list_for_each_entry_safe(p_rt, _p_rt,
+				&s_rt->port_list, port_node) {
+			list_del(&p_rt->port_node);
+			kfree(p_rt);
+		}
+	}
+}
+
+/**
+ * sdw_release_slave_stream() - Free Slave(s) runtime handle
+ *
+ * @slave: Slave handle.
+ * @stream: Stream runtime handle.
+ *
+ * This function is to be called with bus_lock held.
+ */
+static void sdw_release_slave_stream(struct sdw_slave *slave,
+			struct sdw_stream_runtime *stream)
+{
+	struct sdw_slave_runtime *s_rt, *_s_rt;
+	struct sdw_master_runtime *m_rt = stream->m_rt;
+
+	/* Retrieve Slave runtime handle */
+	list_for_each_entry_safe(s_rt, _s_rt,
+			&m_rt->slave_rt_list, m_rt_node) {
+
+		if (s_rt->slave == slave) {
+			list_del(&s_rt->m_rt_node);
+			kfree(s_rt);
+			return;
+		}
+	}
+}
+
+/**
+ * sdw_release_master_stream() - Free Master runtime handle
+ *
+ * @stream: Stream runtime handle.
+ *
+ * This function is to be called with bus_lock held
+ * It frees the Master runtime handle and associated Slave(s) runtime
+ * handle. If this is called first then sdw_release_slave_stream() will have
+ * no effect as Slave(s) runtime handle would already be freed up.
+ */
+static void sdw_release_master_stream(struct sdw_stream_runtime *stream)
+{
+	struct sdw_master_runtime *m_rt = stream->m_rt;
+	struct sdw_slave_runtime *s_rt, *_s_rt;
+
+	list_for_each_entry_safe(s_rt, _s_rt,
+			&m_rt->slave_rt_list, m_rt_node)
+		sdw_stream_remove_slave(s_rt->slave, stream);
+
+	list_del(&m_rt->bus_node);
+}
+
+/**
+ * sdw_stream_remove_master() - Remove master from sdw_stream
+ *
+ * @bus: SDW Bus instance
+ * @stream: SoundWire stream
+ *
+ * This removes and frees port_rt and master_rt from a stream
+ */
+int sdw_stream_remove_master(struct sdw_bus *bus,
+		struct sdw_stream_runtime *stream)
+{
+	mutex_lock(&bus->bus_lock);
+
+	sdw_release_master_stream(stream);
+	sdw_master_port_release(bus, stream->m_rt);
+	stream->state = SDW_STREAM_RELEASED;
+	kfree(stream->m_rt);
+	stream->m_rt = NULL;
+
+	mutex_unlock(&bus->bus_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL(sdw_stream_remove_master);
+
+/**
+ * sdw_stream_remove_slave() - Remove slave from sdw_stream
+ *
+ * @slave: SDW Slave instance
+ * @stream: SoundWire stream
+ *
+ * This removes and frees port_rt and slave_rt from a stream
+ */
+int sdw_stream_remove_slave(struct sdw_slave *slave,
+		struct sdw_stream_runtime *stream)
+{
+	mutex_lock(&slave->bus->bus_lock);
+
+	sdw_slave_port_release(slave->bus, slave, stream);
+	sdw_release_slave_stream(slave, stream);
+
+	mutex_unlock(&slave->bus->bus_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL(sdw_stream_remove_slave);
+
+/**
+ * sdw_config_stream() - Configure the allocated stream
+ *
+ * @dev: SDW device
+ * @stream: SoundWire stream
+ * @stream_config: Stream configuration for audio stream
+ * @is_slave: is API called from Slave or Master
+ *
+ * This function is to be called with bus_lock held.
+ */
+static int sdw_config_stream(struct device *dev,
+		struct sdw_stream_runtime *stream,
+		struct sdw_stream_config *stream_config, bool is_slave)
+{
+	/*
+	 * Update the stream rate, channel and bps based on data
+	 * source. For more than one data source (multilink),
+	 * match the rate, bps, stream type and increment number of channels.
+	 *
+	 * If rate/bps is zero, it means the values are not set, so skip
+	 * comparison and allow the value to be set and stored in stream
+	 */
+	if (stream->params.rate &&
+			stream->params.rate != stream_config->frame_rate) {
+		dev_err(dev, "rate not matching, stream:%s", stream->name);
+		return -EINVAL;
+	}
+
+	if (stream->params.bps &&
+			stream->params.bps != stream_config->bps) {
+		dev_err(dev, "bps not matching, stream:%s", stream->name);
+		return -EINVAL;
+	}
+
+	stream->type = stream_config->type;
+	stream->params.rate = stream_config->frame_rate;
+	stream->params.bps = stream_config->bps;
+
+	/* TODO: Update this check during Device-device support */
+	if (is_slave)
+		stream->params.ch_count += stream_config->ch_count;
+
+	return 0;
+}
+
+static int sdw_is_valid_port_range(struct device *dev,
+				struct sdw_port_runtime *p_rt)
+{
+	if (!SDW_VALID_PORT_RANGE(p_rt->num)) {
+		dev_err(dev,
+			"SoundWire: Invalid port number :%d", p_rt->num);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static struct sdw_port_runtime *sdw_port_alloc(struct device *dev,
+				struct sdw_port_config *port_config,
+				int port_index)
+{
+	struct sdw_port_runtime *p_rt;
+
+	p_rt = kzalloc(sizeof(*p_rt), GFP_KERNEL);
+	if (!p_rt)
+		return NULL;
+
+	p_rt->ch_mask = port_config[port_index].ch_mask;
+	p_rt->num = port_config[port_index].num;
+
+	return p_rt;
+}
+
+static int sdw_master_port_config(struct sdw_bus *bus,
+			struct sdw_master_runtime *m_rt,
+			struct sdw_port_config *port_config,
+			unsigned int num_ports)
+{
+	struct sdw_port_runtime *p_rt;
+	int i;
+
+	/* Iterate for number of ports to perform initialization */
+	for (i = 0; i < num_ports; i++) {
+		p_rt = sdw_port_alloc(bus->dev, port_config, i);
+		if (!p_rt)
+			return -ENOMEM;
+
+		/*
+		 * TODO: Check port capabilities for requested
+		 * configuration (audio mode support)
+		 */
+
+		list_add_tail(&p_rt->port_node, &m_rt->port_list);
+	}
+
+	return 0;
+}
+
+static int sdw_slave_port_config(struct sdw_slave *slave,
+			struct sdw_slave_runtime *s_rt,
+			struct sdw_port_config *port_config,
+			unsigned int num_config)
+{
+	struct sdw_port_runtime *p_rt;
+	int i, ret;
+
+	/* Iterate for number of ports to perform initialization */
+	for (i = 0; i < num_config; i++) {
+		p_rt = sdw_port_alloc(&slave->dev, port_config, i);
+		if (!p_rt)
+			return -ENOMEM;
+
+		/*
+		 * TODO: Check valid port range as defined by DisCo/
+		 * slave
+		 */
+		ret = sdw_is_valid_port_range(&slave->dev, p_rt);
+		if (ret < 0) {
+			kfree(p_rt);
+			return ret;
+		}
+
+		/*
+		 * TODO: Check port capabilities for requested
+		 * configuration (audio mode support)
+		 */
+
+		list_add_tail(&p_rt->port_node, &s_rt->port_list);
+	}
+
+	return 0;
+}
+
+/**
+ * sdw_stream_add_master() - Allocate and add master runtime to a stream
+ *
+ * @bus: SDW Bus instance
+ * @stream_config: Stream configuration for audio stream
+ * @port_config: Port configuration for audio stream
+ * @num_ports: Number of ports
+ * @stream: SoundWire stream
+ */
+int sdw_stream_add_master(struct sdw_bus *bus,
+		struct sdw_stream_config *stream_config,
+		struct sdw_port_config *port_config,
+		unsigned int num_ports,
+		struct sdw_stream_runtime *stream)
+{
+	struct sdw_master_runtime *m_rt = NULL;
+	int ret;
+
+	mutex_lock(&bus->bus_lock);
+
+	m_rt = sdw_alloc_master_rt(bus, stream_config, stream);
+	if (!m_rt) {
+		dev_err(bus->dev,
+				"Master runtime config failed for stream:%s",
+				stream->name);
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	ret = sdw_config_stream(bus->dev, stream, stream_config, false);
+	if (ret)
+		goto stream_error;
+
+	ret = sdw_master_port_config(bus, m_rt, port_config, num_ports);
+	if (ret)
+		goto stream_error;
+
+	stream->state = SDW_STREAM_CONFIGURED;
+
+stream_error:
+	sdw_release_master_stream(stream);
+error:
+	mutex_unlock(&bus->bus_lock);
+	return ret;
+}
+EXPORT_SYMBOL(sdw_stream_add_master);
+
+/**
+ * sdw_stream_add_slave() - Allocate and add master/slave runtime to a stream
+ *
+ * @slave: SDW Slave instance
+ * @stream_config: Stream configuration for audio stream
+ * @stream: SoundWire stream
+ * @port_config: Port configuration for audio stream
+ * @num_ports: Number of ports
+ */
+int sdw_stream_add_slave(struct sdw_slave *slave,
+		struct sdw_stream_config *stream_config,
+		struct sdw_port_config *port_config,
+		unsigned int num_ports,
+		struct sdw_stream_runtime *stream)
+{
+	struct sdw_slave_runtime *s_rt;
+	struct sdw_master_runtime *m_rt;
+	int ret;
+
+	mutex_lock(&slave->bus->bus_lock);
+
+	/*
+	 * If this API is invoked by Slave first then m_rt is not valid.
+	 * So, allocate m_rt and add Slave to it.
+	 */
+	m_rt = sdw_alloc_master_rt(slave->bus, stream_config, stream);
+	if (!m_rt) {
+		dev_err(&slave->dev,
+				"alloc master runtime failed for stream:%s",
+				stream->name);
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	s_rt = sdw_alloc_slave_rt(slave, stream_config, stream);
+	if (!s_rt) {
+		dev_err(&slave->dev,
+				"Slave runtime config failed for stream:%s",
+				stream->name);
+		ret = -ENOMEM;
+		goto stream_error;
+	}
+
+	ret = sdw_config_stream(&slave->dev, stream, stream_config, true);
+	if (ret)
+		goto stream_error;
+
+	list_add_tail(&s_rt->m_rt_node, &m_rt->slave_rt_list);
+
+	ret = sdw_slave_port_config(slave, s_rt, port_config, num_ports);
+	if (ret)
+		goto stream_error;
+
+	stream->state = SDW_STREAM_CONFIGURED;
+	goto error;
+
+stream_error:
+	/*
+	 * we hit error so cleanup the stream, release all Slave(s) and
+	 * Master runtime
+	 */
+	sdw_release_master_stream(stream);
+error:
+	mutex_unlock(&slave->bus->bus_lock);
+	return ret;
+}
+EXPORT_SYMBOL(sdw_stream_add_slave);
+
+/**
+ * sdw_get_slave_dpn_prop() - Get Slave port capabilities
+ *
+ * @slave: Slave handle
+ * @direction: Data direction.
+ * @port_num: Port number
+ */
+struct sdw_dpn_prop *sdw_get_slave_dpn_prop(struct sdw_slave *slave,
+				enum sdw_data_direction direction,
+				unsigned int port_num)
+{
+	struct sdw_dpn_prop *dpn_prop;
+	u8 num_ports;
+	int i;
+
+	if (direction == SDW_DATA_DIR_TX) {
+		num_ports = hweight32(slave->prop.source_ports);
+		dpn_prop = slave->prop.src_dpn_prop;
+	} else {
+		num_ports = hweight32(slave->prop.sink_ports);
+		dpn_prop = slave->prop.sink_dpn_prop;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		dpn_prop = &dpn_prop[i];
+
+		if (dpn_prop->num == port_num)
+			return &dpn_prop[i];
+	}
+
+	return NULL;
+}
+
+static int _sdw_prepare_stream(struct sdw_stream_runtime *stream)
+{
+	struct sdw_master_runtime *m_rt = stream->m_rt;
+	struct sdw_bus *bus = m_rt->bus;
+	struct sdw_master_prop *prop = NULL;
+	struct sdw_bus_params params;
+	int ret;
+
+	prop = &bus->prop;
+	memcpy(&params, &bus->params, sizeof(params));
+
+	/* TODO: Support Asynchronous mode */
+	if ((prop->max_freq % stream->params.rate) != 0) {
+		dev_err(bus->dev, "Async mode not supported");
+		return -EINVAL;
+	}
+
+	/* Increment cumulative bus bandwidth */
+	/* TODO: Update this during Device-Device support */
+	bus->params.bandwidth += m_rt->stream->params.rate *
+		m_rt->ch_count * m_rt->stream->params.bps;
+
+	/* Program params */
+	ret = sdw_program_params(bus);
+	if (ret < 0) {
+		dev_err(bus->dev, "Program params failed: %d", ret);
+		goto restore_params;
+	}
+
+	ret = do_bank_switch(stream);
+	if (ret < 0) {
+		dev_err(bus->dev, "Bank switch failed: %d", ret);
+		goto restore_params;
+	}
+
+	/* Prepare port(s) on the new clock configuration */
+	ret = sdw_prep_deprep_ports(m_rt, true);
+	if (ret < 0) {
+		dev_err(bus->dev, "Prepare port(s) failed ret = %d",
+				ret);
+		return ret;
+	}
+
+	stream->state = SDW_STREAM_PREPARED;
+
+	return ret;
+
+restore_params:
+	memcpy(&bus->params, &params, sizeof(params));
+	return ret;
+}
+
+/**
+ * sdw_prepare_stream() - Prepare SoundWire stream
+ *
+ * @stream: Soundwire stream
+ *
+ * Documentation/soundwire/stream.txt explains this API in detail
+ */
+int sdw_prepare_stream(struct sdw_stream_runtime *stream)
+{
+	int ret = 0;
+
+	if (!stream) {
+		pr_err("SoundWire: Handle not found for stream");
+		return -EINVAL;
+	}
+
+	mutex_lock(&stream->m_rt->bus->bus_lock);
+
+	ret = _sdw_prepare_stream(stream);
+	if (ret < 0)
+		pr_err("Prepare for stream:%s failed: %d", stream->name, ret);
+
+	mutex_unlock(&stream->m_rt->bus->bus_lock);
+	return ret;
+}
+EXPORT_SYMBOL(sdw_prepare_stream);
+
+static int _sdw_enable_stream(struct sdw_stream_runtime *stream)
+{
+	struct sdw_master_runtime *m_rt = stream->m_rt;
+	struct sdw_bus *bus = m_rt->bus;
+	int ret;
+
+	/* Program params */
+	ret = sdw_program_params(bus);
+	if (ret < 0) {
+		dev_err(bus->dev, "Program params failed: %d", ret);
+		return ret;
+	}
+
+	/* Enable port(s) */
+	ret = sdw_enable_disable_ports(m_rt, true);
+	if (ret < 0) {
+		dev_err(bus->dev, "Enable port(s) failed ret: %d", ret);
+		return ret;
+	}
+
+	ret = do_bank_switch(stream);
+	if (ret < 0) {
+		dev_err(bus->dev, "Bank switch failed: %d", ret);
+		return ret;
+	}
+
+	stream->state = SDW_STREAM_ENABLED;
+	return 0;
+}
+
+/**
+ * sdw_enable_stream() - Enable SoundWire stream
+ *
+ * @stream: Soundwire stream
+ *
+ * Documentation/soundwire/stream.txt explains this API in detail
+ */
+int sdw_enable_stream(struct sdw_stream_runtime *stream)
+{
+	int ret = 0;
+
+	if (!stream) {
+		pr_err("SoundWire: Handle not found for stream");
+		return -EINVAL;
+	}
+
+	mutex_lock(&stream->m_rt->bus->bus_lock);
+
+	ret = _sdw_enable_stream(stream);
+	if (ret < 0)
+		pr_err("Enable for stream:%s failed: %d", stream->name, ret);
+
+	mutex_unlock(&stream->m_rt->bus->bus_lock);
+	return ret;
+}
+EXPORT_SYMBOL(sdw_enable_stream);
+
+static int _sdw_disable_stream(struct sdw_stream_runtime *stream)
+{
+	struct sdw_master_runtime *m_rt = stream->m_rt;
+	struct sdw_bus *bus = m_rt->bus;
+	int ret;
+
+	/* Disable port(s) */
+	ret = sdw_enable_disable_ports(m_rt, false);
+	if (ret < 0) {
+		dev_err(bus->dev, "Disable port(s) failed: %d", ret);
+		return ret;
+	}
+
+	stream->state = SDW_STREAM_DISABLED;
+
+	/* Program params */
+	ret = sdw_program_params(bus);
+	if (ret < 0) {
+		dev_err(bus->dev, "Program params failed: %d", ret);
+		return ret;
+	}
+
+	return do_bank_switch(stream);
+}
+
+/**
+ * sdw_disable_stream() - Disable SoundWire stream
+ *
+ * @stream: Soundwire stream
+ *
+ * Documentation/soundwire/stream.txt explains this API in detail
+ */
+int sdw_disable_stream(struct sdw_stream_runtime *stream)
+{
+	int ret = 0;
+
+	if (!stream) {
+		pr_err("SoundWire: Handle not found for stream");
+		return -EINVAL;
+	}
+
+	mutex_lock(&stream->m_rt->bus->bus_lock);
+
+	ret = _sdw_disable_stream(stream);
+	if (ret < 0)
+		pr_err("Disable for stream:%s failed: %d", stream->name, ret);
+
+	mutex_unlock(&stream->m_rt->bus->bus_lock);
+	return ret;
+}
+EXPORT_SYMBOL(sdw_disable_stream);
+
+static int _sdw_deprepare_stream(struct sdw_stream_runtime *stream)
+{
+	struct sdw_master_runtime *m_rt = stream->m_rt;
+	struct sdw_bus *bus = m_rt->bus;
+	int ret = 0;
+
+	/* De-prepare port(s) */
+	ret = sdw_prep_deprep_ports(m_rt, false);
+	if (ret < 0) {
+		dev_err(bus->dev, "De-prepare port(s) failed: %d", ret);
+		return ret;
+	}
+
+	stream->state = SDW_STREAM_DEPREPARED;
+
+	/* TODO: Update this during Device-Device support */
+	bus->params.bandwidth -= m_rt->stream->params.rate *
+		m_rt->ch_count * m_rt->stream->params.bps;
+
+	/* Program params */
+	ret = sdw_program_params(bus);
+	if (ret < 0) {
+		dev_err(bus->dev, "Program params failed: %d", ret);
+		return ret;
+	}
+
+	return do_bank_switch(stream);
+}
+
+/**
+ * sdw_deprepare_stream() - Deprepare SoundWire stream
+ *
+ * @stream: Soundwire stream
+ *
+ * Documentation/soundwire/stream.txt explains this API in detail
+ */
+int sdw_deprepare_stream(struct sdw_stream_runtime *stream)
+{
+	int ret = 0;
+
+	if (!stream) {
+		pr_err("SoundWire: Handle not found for stream");
+		return -EINVAL;
+	}
+
+	mutex_lock(&stream->m_rt->bus->bus_lock);
+
+	ret = _sdw_deprepare_stream(stream);
+	if (ret < 0)
+		pr_err("De-prepare for stream:%d failed: %d", ret, ret);
+
+	mutex_unlock(&stream->m_rt->bus->bus_lock);
+	return ret;
+}
+EXPORT_SYMBOL(sdw_deprepare_stream);
diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index 2d4146ce2f1b..e62ac3289bc1 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -47,6 +47,13 @@ config SPI_MASTER
 
 if SPI_MASTER
 
+config SPI_MEM
+	bool "SPI memory extension"
+	help
+	  Enable this option if you want to enable the SPI memory extension.
+	  This extension is meant to simplify interaction with SPI memories
+	  by providing an high-level interface to send memory-like commands.
+
 comment "SPI Master Controller Drivers"
 
 config SPI_ALTERA
@@ -71,7 +78,6 @@ config SPI_ARMADA_3700
 
 config SPI_ATMEL
 	tristate "Atmel SPI Controller"
-	depends on HAS_DMA
 	depends on ARCH_AT91 || COMPILE_TEST
 	help
 	  This selects a driver for the Atmel SPI Controller, present on
@@ -115,14 +121,6 @@ config SPI_BCM2835AUX
 	  "universal SPI master", and the regular SPI controller.
 	  This driver is for the universal/auxiliary SPI controller.
 
-config SPI_BCM53XX
-	tristate "Broadcom BCM53xx SPI controller"
-	depends on ARCH_BCM_5301X
-	depends on BCMA_POSSIBLE
-	select BCMA
-	help
-          Enable support for the SPI controller on Broadcom BCM53xx ARM SoCs.
-
 config SPI_BCM63XX
 	tristate "Broadcom BCM63xx SPI controller"
 	depends on BCM63XX || COMPILE_TEST
@@ -233,7 +231,6 @@ config SPI_EFM32
 
 config SPI_EP93XX
 	tristate "Cirrus Logic EP93xx SPI controller"
-	depends on HAS_DMA
 	depends on ARCH_EP93XX || COMPILE_TEST
 	help
 	  This enables using the Cirrus EP93xx SPI controller in master
@@ -355,7 +352,6 @@ config SPI_FSL_SPI
 config SPI_FSL_DSPI
 	tristate "Freescale DSPI controller"
 	select REGMAP_MMIO
-	depends on HAS_DMA
 	depends on SOC_VF610 || SOC_LS1021A || ARCH_LAYERSCAPE || M5441x || COMPILE_TEST
 	help
 	  This enables support for the Freescale DSPI controller in master
@@ -431,7 +427,6 @@ config SPI_OMAP_UWIRE
 
 config SPI_OMAP24XX
 	tristate "McSPI driver for OMAP"
-	depends on HAS_DMA
 	depends on ARCH_OMAP2PLUS || COMPILE_TEST
 	select SG_SPLIT
 	help
@@ -440,7 +435,6 @@ config SPI_OMAP24XX
 
 config SPI_TI_QSPI
 	tristate "DRA7xxx QSPI controller support"
-	depends on HAS_DMA
 	depends on ARCH_OMAP2PLUS || COMPILE_TEST
 	help
 	  QSPI master controller for DRA7xxx used for flash devices.
@@ -469,7 +463,6 @@ config SPI_PIC32
 config SPI_PIC32_SQI
 	tristate "Microchip PIC32 Quad SPI driver"
 	depends on MACH_PIC32 || COMPILE_TEST
-	depends on HAS_DMA
 	help
 	  SPI driver for PIC32 Quad SPI controller.
 
@@ -572,7 +565,7 @@ config SPI_SC18IS602
 
 config SPI_SH_MSIOF
 	tristate "SuperH MSIOF SPI controller"
-	depends on HAVE_CLK && HAS_DMA
+	depends on HAVE_CLK
 	depends on ARCH_SHMOBILE || ARCH_RENESAS || COMPILE_TEST
 	help
 	  SPI driver for SuperH and SH Mobile MSIOF blocks.
@@ -650,7 +643,7 @@ config SPI_MXS
 config SPI_TEGRA114
 	tristate "NVIDIA Tegra114 SPI Controller"
 	depends on (ARCH_TEGRA && TEGRA20_APB_DMA) || COMPILE_TEST
-	depends on RESET_CONTROLLER && HAS_DMA
+	depends on RESET_CONTROLLER
 	help
 	  SPI driver for NVIDIA Tegra114 SPI Controller interface. This controller
 	  is different than the older SoCs SPI controller and also register interface
@@ -668,7 +661,7 @@ config SPI_TEGRA20_SFLASH
 config SPI_TEGRA20_SLINK
 	tristate "Nvidia Tegra20/Tegra30 SLINK Controller"
 	depends on (ARCH_TEGRA && TEGRA20_APB_DMA) || COMPILE_TEST
-	depends on RESET_CONTROLLER && HAS_DMA
+	depends on RESET_CONTROLLER
 	help
 	  SPI driver for Nvidia Tegra20/Tegra30 SLINK Controller interface.
 
diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile
index b935f10eb961..cb1f4378b87c 100644
--- a/drivers/spi/Makefile
+++ b/drivers/spi/Makefile
@@ -8,6 +8,7 @@ ccflags-$(CONFIG_SPI_DEBUG) := -DDEBUG
 # small core, mostly translating board-specific
 # config declarations into driver model code
 obj-$(CONFIG_SPI_MASTER)		+= spi.o
+obj-$(CONFIG_SPI_MEM)			+= spi-mem.o
 obj-$(CONFIG_SPI_SPIDEV)		+= spidev.o
 obj-$(CONFIG_SPI_LOOPBACK_TEST)		+= spi-loopback-test.o
 
@@ -20,7 +21,6 @@ obj-$(CONFIG_SPI_AU1550)		+= spi-au1550.o
 obj-$(CONFIG_SPI_AXI_SPI_ENGINE)	+= spi-axi-spi-engine.o
 obj-$(CONFIG_SPI_BCM2835)		+= spi-bcm2835.o
 obj-$(CONFIG_SPI_BCM2835AUX)		+= spi-bcm2835aux.o
-obj-$(CONFIG_SPI_BCM53XX)		+= spi-bcm53xx.o
 obj-$(CONFIG_SPI_BCM63XX)		+= spi-bcm63xx.o
 obj-$(CONFIG_SPI_BCM63XX_HSSPI)		+= spi-bcm63xx-hsspi.o
 obj-$(CONFIG_SPI_BCM_QSPI)		+= spi-iproc-qspi.o spi-brcmstb-qspi.o spi-bcm-qspi.o
diff --git a/drivers/spi/internals.h b/drivers/spi/internals.h
new file mode 100644
index 000000000000..4a28a8395552
--- /dev/null
+++ b/drivers/spi/internals.h
@@ -0,0 +1,43 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (C) 2018 Exceet Electronics GmbH
+ * Copyright (C) 2018 Bootlin
+ *
+ * Author: Boris Brezillon <boris.brezillon@bootlin.com>
+ *
+ * Helpers needed by the spi or spi-mem logic. Should not be used outside of
+ * spi-mem.c and spi.c.
+ */
+
+#ifndef __LINUX_SPI_INTERNALS_H
+#define __LINUX_SPI_INTERNALS_H
+
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/scatterlist.h>
+#include <linux/spi/spi.h>
+
+void spi_flush_queue(struct spi_controller *ctrl);
+
+#ifdef CONFIG_HAS_DMA
+int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
+		struct sg_table *sgt, void *buf, size_t len,
+		enum dma_data_direction dir);
+void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev,
+		   struct sg_table *sgt, enum dma_data_direction dir);
+#else /* !CONFIG_HAS_DMA */
+static inline int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
+			      struct sg_table *sgt, void *buf, size_t len,
+			      enum dma_data_direction dir)
+{
+	return -EINVAL;
+}
+
+static inline void spi_unmap_buf(struct spi_controller *ctlr,
+				 struct device *dev, struct sg_table *sgt,
+				 enum dma_data_direction dir)
+{
+}
+#endif /* CONFIG_HAS_DMA */
+
+#endif /* __LINUX_SPI_INTERNALS_H */
diff --git a/drivers/spi/spi-bcm-qspi.c b/drivers/spi/spi-bcm-qspi.c
index 6573152ce893..8612525fa4e3 100644
--- a/drivers/spi/spi-bcm-qspi.c
+++ b/drivers/spi/spi-bcm-qspi.c
@@ -30,6 +30,7 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
 #include <linux/sysfs.h>
 #include <linux/types.h>
 #include "spi-bcm-qspi.h"
@@ -215,10 +216,10 @@ struct bcm_qspi {
 	int bspi_maj_rev;
 	int bspi_min_rev;
 	int bspi_enabled;
-	struct spi_flash_read_message *bspi_rf_msg;
-	u32 bspi_rf_msg_idx;
-	u32 bspi_rf_msg_len;
-	u32 bspi_rf_msg_status;
+	const struct spi_mem_op *bspi_rf_op;
+	u32 bspi_rf_op_idx;
+	u32 bspi_rf_op_len;
+	u32 bspi_rf_op_status;
 	struct bcm_xfer_mode xfer_mode;
 	u32 s3_strap_override_ctrl;
 	bool bspi_mode;
@@ -313,26 +314,26 @@ static inline void bcm_qspi_bspi_lr_clear(struct bcm_qspi *qspi)
 
 static void bcm_qspi_bspi_lr_data_read(struct bcm_qspi *qspi)
 {
-	u32 *buf = (u32 *)qspi->bspi_rf_msg->buf;
+	u32 *buf = (u32 *)qspi->bspi_rf_op->data.buf.in;
 	u32 data = 0;
 
-	dev_dbg(&qspi->pdev->dev, "xfer %p rx %p rxlen %d\n", qspi->bspi_rf_msg,
-		qspi->bspi_rf_msg->buf, qspi->bspi_rf_msg_len);
+	dev_dbg(&qspi->pdev->dev, "xfer %p rx %p rxlen %d\n", qspi->bspi_rf_op,
+		qspi->bspi_rf_op->data.buf.in, qspi->bspi_rf_op_len);
 	while (!bcm_qspi_bspi_lr_is_fifo_empty(qspi)) {
 		data = bcm_qspi_bspi_lr_read_fifo(qspi);
-		if (likely(qspi->bspi_rf_msg_len >= 4) &&
+		if (likely(qspi->bspi_rf_op_len >= 4) &&
 		    IS_ALIGNED((uintptr_t)buf, 4)) {
-			buf[qspi->bspi_rf_msg_idx++] = data;
-			qspi->bspi_rf_msg_len -= 4;
+			buf[qspi->bspi_rf_op_idx++] = data;
+			qspi->bspi_rf_op_len -= 4;
 		} else {
 			/* Read out remaining bytes, make sure*/
-			u8 *cbuf = (u8 *)&buf[qspi->bspi_rf_msg_idx];
+			u8 *cbuf = (u8 *)&buf[qspi->bspi_rf_op_idx];
 
 			data = cpu_to_le32(data);
-			while (qspi->bspi_rf_msg_len) {
+			while (qspi->bspi_rf_op_len) {
 				*cbuf++ = (u8)data;
 				data >>= 8;
-				qspi->bspi_rf_msg_len--;
+				qspi->bspi_rf_op_len--;
 			}
 		}
 	}
@@ -349,14 +350,12 @@ static void bcm_qspi_bspi_set_xfer_params(struct bcm_qspi *qspi, u8 cmd_byte,
 }
 
 static int bcm_qspi_bspi_set_flex_mode(struct bcm_qspi *qspi,
-				       struct spi_flash_read_message *msg,
-				       int hp)
+				       const struct spi_mem_op *op, int hp)
 {
 	int bpc = 0, bpp = 0;
-	u8 command = msg->read_opcode;
-	int width  = msg->data_nbits ? msg->data_nbits : SPI_NBITS_SINGLE;
-	int addrlen = msg->addr_width;
-	int addr_nbits = msg->addr_nbits ? msg->addr_nbits : SPI_NBITS_SINGLE;
+	u8 command = op->cmd.opcode;
+	int width  = op->cmd.buswidth ? op->cmd.buswidth : SPI_NBITS_SINGLE;
+	int addrlen = op->addr.nbytes * 8;
 	int flex_mode = 1;
 
 	dev_dbg(&qspi->pdev->dev, "set flex mode w %x addrlen %x hp %d\n",
@@ -365,7 +364,7 @@ static int bcm_qspi_bspi_set_flex_mode(struct bcm_qspi *qspi,
 	if (addrlen == BSPI_ADDRLEN_4BYTES)
 		bpp = BSPI_BPP_ADDR_SELECT_MASK;
 
-	bpp |= msg->dummy_bytes * (8/addr_nbits);
+	bpp |= (op->dummy.nbytes * 8) / op->dummy.buswidth;
 
 	switch (width) {
 	case SPI_NBITS_SINGLE:
@@ -397,11 +396,10 @@ static int bcm_qspi_bspi_set_flex_mode(struct bcm_qspi *qspi,
 }
 
 static int bcm_qspi_bspi_set_override(struct bcm_qspi *qspi,
-				      struct spi_flash_read_message *msg,
-				      int hp)
+				      const struct spi_mem_op *op, int hp)
 {
-	int width = msg->data_nbits ? msg->data_nbits : SPI_NBITS_SINGLE;
-	int addrlen = msg->addr_width;
+	int width = op->data.buswidth ? op->data.buswidth : SPI_NBITS_SINGLE;
+	int addrlen = op->addr.nbytes;
 	u32 data = bcm_qspi_read(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL);
 
 	dev_dbg(&qspi->pdev->dev, "set override mode w %x addrlen %x hp %d\n",
@@ -437,17 +435,17 @@ static int bcm_qspi_bspi_set_override(struct bcm_qspi *qspi,
 	/* set the override mode */
 	data |=	BSPI_STRAP_OVERRIDE_CTRL_OVERRIDE;
 	bcm_qspi_write(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL, data);
-	bcm_qspi_bspi_set_xfer_params(qspi, msg->read_opcode, 0, 0, 0);
+	bcm_qspi_bspi_set_xfer_params(qspi, op->cmd.opcode, 0, 0, 0);
 
 	return 0;
 }
 
 static int bcm_qspi_bspi_set_mode(struct bcm_qspi *qspi,
-				  struct spi_flash_read_message *msg, int hp)
+				  const struct spi_mem_op *op, int hp)
 {
 	int error = 0;
-	int width = msg->data_nbits ? msg->data_nbits : SPI_NBITS_SINGLE;
-	int addrlen = msg->addr_width;
+	int width = op->data.buswidth ? op->data.buswidth : SPI_NBITS_SINGLE;
+	int addrlen = op->addr.nbytes;
 
 	/* default mode */
 	qspi->xfer_mode.flex_mode = true;
@@ -460,12 +458,12 @@ static int bcm_qspi_bspi_set_mode(struct bcm_qspi *qspi,
 		if (val & mask || qspi->s3_strap_override_ctrl & mask) {
 			qspi->xfer_mode.flex_mode = false;
 			bcm_qspi_write(qspi, BSPI, BSPI_FLEX_MODE_ENABLE, 0);
-			error = bcm_qspi_bspi_set_override(qspi, msg, hp);
+			error = bcm_qspi_bspi_set_override(qspi, op, hp);
 		}
 	}
 
 	if (qspi->xfer_mode.flex_mode)
-		error = bcm_qspi_bspi_set_flex_mode(qspi, msg, hp);
+		error = bcm_qspi_bspi_set_flex_mode(qspi, op, hp);
 
 	if (error) {
 		dev_warn(&qspi->pdev->dev,
@@ -802,19 +800,20 @@ static int write_to_hw(struct bcm_qspi *qspi, struct spi_device *spi)
 	return slot;
 }
 
-static int bcm_qspi_bspi_flash_read(struct spi_device *spi,
-				    struct spi_flash_read_message *msg)
+static int bcm_qspi_bspi_exec_mem_op(struct spi_device *spi,
+				     const struct spi_mem_op *op)
 {
 	struct bcm_qspi *qspi = spi_master_get_devdata(spi->master);
-	u32 addr = 0, len, rdlen, len_words;
+	u32 addr = 0, len, rdlen, len_words, from = 0;
 	int ret = 0;
 	unsigned long timeo = msecs_to_jiffies(100);
 	struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;
 
 	if (bcm_qspi_bspi_ver_three(qspi))
-		if (msg->addr_width == BSPI_ADDRLEN_4BYTES)
+		if (op->addr.nbytes == BSPI_ADDRLEN_4BYTES)
 			return -EIO;
 
+	from = op->addr.val;
 	bcm_qspi_chip_select(qspi, spi->chip_select);
 	bcm_qspi_write(qspi, MSPI, MSPI_WRITE_LOCK, 0);
 
@@ -823,15 +822,15 @@ static int bcm_qspi_bspi_flash_read(struct spi_device *spi,
 	 * the upper address byte to bspi
 	 */
 	if (bcm_qspi_bspi_ver_three(qspi) == false) {
-		addr = msg->from & 0xff000000;
+		addr = from & 0xff000000;
 		bcm_qspi_write(qspi, BSPI,
 			       BSPI_BSPI_FLASH_UPPER_ADDR_BYTE, addr);
 	}
 
 	if (!qspi->xfer_mode.flex_mode)
-		addr = msg->from;
+		addr = from;
 	else
-		addr = msg->from & 0x00ffffff;
+		addr = from & 0x00ffffff;
 
 	if (bcm_qspi_bspi_ver_three(qspi) == true)
 		addr = (addr + 0xc00000) & 0xffffff;
@@ -840,8 +839,8 @@ static int bcm_qspi_bspi_flash_read(struct spi_device *spi,
 	 * read into the entire buffer by breaking the reads
 	 * into RAF buffer read lengths
 	 */
-	len = msg->len;
-	qspi->bspi_rf_msg_idx = 0;
+	len = op->data.nbytes;
+	qspi->bspi_rf_op_idx = 0;
 
 	do {
 		if (len > BSPI_READ_LENGTH)
@@ -852,9 +851,9 @@ static int bcm_qspi_bspi_flash_read(struct spi_device *spi,
 		reinit_completion(&qspi->bspi_done);
 		bcm_qspi_enable_bspi(qspi);
 		len_words = (rdlen + 3) >> 2;
-		qspi->bspi_rf_msg = msg;
-		qspi->bspi_rf_msg_status = 0;
-		qspi->bspi_rf_msg_len = rdlen;
+		qspi->bspi_rf_op = op;
+		qspi->bspi_rf_op_status = 0;
+		qspi->bspi_rf_op_len = rdlen;
 		dev_dbg(&qspi->pdev->dev,
 			"bspi xfr addr 0x%x len 0x%x", addr, rdlen);
 		bcm_qspi_write(qspi, BSPI, BSPI_RAF_START_ADDR, addr);
@@ -879,7 +878,6 @@ static int bcm_qspi_bspi_flash_read(struct spi_device *spi,
 		}
 
 		/* set msg return length */
-		msg->retlen += rdlen;
 		addr += rdlen;
 		len -= rdlen;
 	} while (len);
@@ -914,61 +912,63 @@ static int bcm_qspi_transfer_one(struct spi_master *master,
 	return 0;
 }
 
-static int bcm_qspi_mspi_flash_read(struct spi_device *spi,
-				    struct spi_flash_read_message *msg)
+static int bcm_qspi_mspi_exec_mem_op(struct spi_device *spi,
+				     const struct spi_mem_op *op)
 {
-	struct bcm_qspi *qspi = spi_master_get_devdata(spi->master);
+	struct spi_master *master = spi->master;
+	struct bcm_qspi *qspi = spi_master_get_devdata(master);
 	struct spi_transfer t[2];
-	u8 cmd[6];
-	int ret;
+	u8 cmd[6] = { };
+	int ret, i;
 
 	memset(cmd, 0, sizeof(cmd));
 	memset(t, 0, sizeof(t));
 
 	/* tx */
 	/* opcode is in cmd[0] */
-	cmd[0] = msg->read_opcode;
-	cmd[1] = msg->from >> (msg->addr_width * 8 -  8);
-	cmd[2] = msg->from >> (msg->addr_width * 8 - 16);
-	cmd[3] = msg->from >> (msg->addr_width * 8 - 24);
-	cmd[4] = msg->from >> (msg->addr_width * 8 - 32);
+	cmd[0] = op->cmd.opcode;
+	for (i = 0; i < op->addr.nbytes; i++)
+		cmd[1 + i] = op->addr.val >> (8 * (op->addr.nbytes - i - 1));
+
 	t[0].tx_buf = cmd;
-	t[0].len = msg->addr_width + msg->dummy_bytes + 1;
+	t[0].len = op->addr.nbytes + op->dummy.nbytes + 1;
 	t[0].bits_per_word = spi->bits_per_word;
-	t[0].tx_nbits = msg->opcode_nbits;
+	t[0].tx_nbits = op->cmd.buswidth;
 	/* lets mspi know that this is not last transfer */
 	qspi->trans_pos.mspi_last_trans = false;
-	ret = bcm_qspi_transfer_one(spi->master, spi, &t[0]);
+	ret = bcm_qspi_transfer_one(master, spi, &t[0]);
 
 	/* rx */
 	qspi->trans_pos.mspi_last_trans = true;
 	if (!ret) {
 		/* rx */
-		t[1].rx_buf = msg->buf;
-		t[1].len = msg->len;
-		t[1].rx_nbits =  msg->data_nbits;
+		t[1].rx_buf = op->data.buf.in;
+		t[1].len = op->data.nbytes;
+		t[1].rx_nbits =  op->data.buswidth;
 		t[1].bits_per_word = spi->bits_per_word;
-		ret = bcm_qspi_transfer_one(spi->master, spi, &t[1]);
+		ret = bcm_qspi_transfer_one(master, spi, &t[1]);
 	}
 
-	if (!ret)
-		msg->retlen = msg->len;
-
 	return ret;
 }
 
-static int bcm_qspi_flash_read(struct spi_device *spi,
-			       struct spi_flash_read_message *msg)
+static int bcm_qspi_exec_mem_op(struct spi_mem *mem,
+				const struct spi_mem_op *op)
 {
+	struct spi_device *spi = mem->spi;
 	struct bcm_qspi *qspi = spi_master_get_devdata(spi->master);
 	int ret = 0;
 	bool mspi_read = false;
-	u32 addr, len;
+	u32 addr = 0, len;
 	u_char *buf;
 
-	buf = msg->buf;
-	addr = msg->from;
-	len = msg->len;
+	if (!op->data.nbytes || !op->addr.nbytes || op->addr.nbytes > 4 ||
+	    op->data.dir != SPI_MEM_DATA_IN)
+		return -ENOTSUPP;
+
+	buf = op->data.buf.in;
+	addr = op->addr.val;
+	len = op->data.nbytes;
 
 	if (bcm_qspi_bspi_ver_three(qspi) == true) {
 		/*
@@ -990,12 +990,12 @@ static int bcm_qspi_flash_read(struct spi_device *spi,
 		mspi_read = true;
 
 	if (mspi_read)
-		return bcm_qspi_mspi_flash_read(spi, msg);
+		return bcm_qspi_mspi_exec_mem_op(spi, op);
 
-	ret = bcm_qspi_bspi_set_mode(qspi, msg, -1);
+	ret = bcm_qspi_bspi_set_mode(qspi, op, -1);
 
 	if (!ret)
-		ret = bcm_qspi_bspi_flash_read(spi, msg);
+		ret = bcm_qspi_bspi_exec_mem_op(spi, op);
 
 	return ret;
 }
@@ -1034,10 +1034,10 @@ static irqreturn_t bcm_qspi_bspi_lr_l2_isr(int irq, void *dev_id)
 	struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;
 	u32 status = qspi_dev_id->irqp->mask;
 
-	if (qspi->bspi_enabled && qspi->bspi_rf_msg) {
+	if (qspi->bspi_enabled && qspi->bspi_rf_op) {
 		bcm_qspi_bspi_lr_data_read(qspi);
-		if (qspi->bspi_rf_msg_len == 0) {
-			qspi->bspi_rf_msg = NULL;
+		if (qspi->bspi_rf_op_len == 0) {
+			qspi->bspi_rf_op = NULL;
 			if (qspi->soc_intc) {
 				/* disable soc BSPI interrupt */
 				soc_intc->bcm_qspi_int_set(soc_intc, BSPI_DONE,
@@ -1046,7 +1046,7 @@ static irqreturn_t bcm_qspi_bspi_lr_l2_isr(int irq, void *dev_id)
 				status = INTR_BSPI_LR_SESSION_DONE_MASK;
 			}
 
-			if (qspi->bspi_rf_msg_status)
+			if (qspi->bspi_rf_op_status)
 				bcm_qspi_bspi_lr_clear(qspi);
 			else
 				bcm_qspi_bspi_flush_prefetch_buffers(qspi);
@@ -1058,7 +1058,7 @@ static irqreturn_t bcm_qspi_bspi_lr_l2_isr(int irq, void *dev_id)
 	}
 
 	status &= INTR_BSPI_LR_SESSION_DONE_MASK;
-	if (qspi->bspi_enabled && status && qspi->bspi_rf_msg_len == 0)
+	if (qspi->bspi_enabled && status && qspi->bspi_rf_op_len == 0)
 		complete(&qspi->bspi_done);
 
 	return IRQ_HANDLED;
@@ -1071,7 +1071,7 @@ static irqreturn_t bcm_qspi_bspi_lr_err_l2_isr(int irq, void *dev_id)
 	struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;
 
 	dev_err(&qspi->pdev->dev, "BSPI INT error\n");
-	qspi->bspi_rf_msg_status = -EIO;
+	qspi->bspi_rf_op_status = -EIO;
 	if (qspi->soc_intc)
 		/* clear soc interrupt */
 		soc_intc->bcm_qspi_int_ack(soc_intc, BSPI_ERR);
@@ -1194,6 +1194,10 @@ static void bcm_qspi_hw_uninit(struct bcm_qspi *qspi)
 
 }
 
+static const struct spi_controller_mem_ops bcm_qspi_mem_ops = {
+	.exec_op = bcm_qspi_exec_mem_op,
+};
+
 static const struct of_device_id bcm_qspi_of_match[] = {
 	{ .compatible = "brcm,spi-bcm-qspi" },
 	{},
@@ -1236,7 +1240,7 @@ int bcm_qspi_probe(struct platform_device *pdev,
 	master->mode_bits = SPI_CPHA | SPI_CPOL | SPI_RX_DUAL | SPI_RX_QUAD;
 	master->setup = bcm_qspi_setup;
 	master->transfer_one = bcm_qspi_transfer_one;
-	master->spi_flash_read = bcm_qspi_flash_read;
+	master->mem_ops = &bcm_qspi_mem_ops;
 	master->cleanup = bcm_qspi_cleanup;
 	master->dev.of_node = dev->of_node;
 	master->num_chipselect = NUM_CHIPSELECT;
diff --git a/drivers/spi/spi-bcm53xx.c b/drivers/spi/spi-bcm53xx.c
deleted file mode 100644
index d02ceb7a29d1..000000000000
--- a/drivers/spi/spi-bcm53xx.c
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Copyright (C) 2014-2016 Rafał Miłecki <rafal@milecki.pl>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#define pr_fmt(fmt)		KBUILD_MODNAME ": " fmt
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-#include <linux/bcma/bcma.h>
-#include <linux/spi/spi.h>
-
-#include "spi-bcm53xx.h"
-
-#define BCM53XXSPI_MAX_SPI_BAUD	13500000	/* 216 MHz? */
-#define BCM53XXSPI_FLASH_WINDOW	SZ_32M
-
-/* The longest observed required wait was 19 ms */
-#define BCM53XXSPI_SPE_TIMEOUT_MS	80
-
-struct bcm53xxspi {
-	struct bcma_device *core;
-	struct spi_master *master;
-	void __iomem *mmio_base;
-	bool bspi;				/* Boot SPI mode with memory mapping */
-};
-
-static inline u32 bcm53xxspi_read(struct bcm53xxspi *b53spi, u16 offset)
-{
-	return bcma_read32(b53spi->core, offset);
-}
-
-static inline void bcm53xxspi_write(struct bcm53xxspi *b53spi, u16 offset,
-				    u32 value)
-{
-	bcma_write32(b53spi->core, offset, value);
-}
-
-static void bcm53xxspi_disable_bspi(struct bcm53xxspi *b53spi)
-{
-	struct device *dev = &b53spi->core->dev;
-	unsigned long deadline;
-	u32 tmp;
-
-	if (!b53spi->bspi)
-		return;
-
-	tmp = bcm53xxspi_read(b53spi, B53SPI_BSPI_MAST_N_BOOT_CTRL);
-	if (tmp & 0x1)
-		return;
-
-	deadline = jiffies + usecs_to_jiffies(200);
-	do {
-		tmp = bcm53xxspi_read(b53spi, B53SPI_BSPI_BUSY_STATUS);
-		if (!(tmp & 0x1)) {
-			bcm53xxspi_write(b53spi, B53SPI_BSPI_MAST_N_BOOT_CTRL,
-					 0x1);
-			ndelay(200);
-			b53spi->bspi = false;
-			return;
-		}
-		udelay(1);
-	} while (!time_after_eq(jiffies, deadline));
-
-	dev_warn(dev, "Timeout disabling BSPI\n");
-}
-
-static void bcm53xxspi_enable_bspi(struct bcm53xxspi *b53spi)
-{
-	u32 tmp;
-
-	if (b53spi->bspi)
-		return;
-
-	tmp = bcm53xxspi_read(b53spi, B53SPI_BSPI_MAST_N_BOOT_CTRL);
-	if (!(tmp & 0x1))
-		return;
-
-	bcm53xxspi_write(b53spi, B53SPI_BSPI_MAST_N_BOOT_CTRL, 0x0);
-	b53spi->bspi = true;
-}
-
-static inline unsigned int bcm53xxspi_calc_timeout(size_t len)
-{
-	/* Do some magic calculation based on length and buad. Add 10% and 1. */
-	return (len * 9000 / BCM53XXSPI_MAX_SPI_BAUD * 110 / 100) + 1;
-}
-
-static int bcm53xxspi_wait(struct bcm53xxspi *b53spi, unsigned int timeout_ms)
-{
-	unsigned long deadline;
-	u32 tmp;
-
-	/* SPE bit has to be 0 before we read MSPI STATUS */
-	deadline = jiffies + msecs_to_jiffies(BCM53XXSPI_SPE_TIMEOUT_MS);
-	do {
-		tmp = bcm53xxspi_read(b53spi, B53SPI_MSPI_SPCR2);
-		if (!(tmp & B53SPI_MSPI_SPCR2_SPE))
-			break;
-		udelay(5);
-	} while (!time_after_eq(jiffies, deadline));
-
-	if (tmp & B53SPI_MSPI_SPCR2_SPE)
-		goto spi_timeout;
-
-	/* Check status */
-	deadline = jiffies + msecs_to_jiffies(timeout_ms);
-	do {
-		tmp = bcm53xxspi_read(b53spi, B53SPI_MSPI_MSPI_STATUS);
-		if (tmp & B53SPI_MSPI_MSPI_STATUS_SPIF) {
-			bcm53xxspi_write(b53spi, B53SPI_MSPI_MSPI_STATUS, 0);
-			return 0;
-		}
-
-		cpu_relax();
-		udelay(100);
-	} while (!time_after_eq(jiffies, deadline));
-
-spi_timeout:
-	bcm53xxspi_write(b53spi, B53SPI_MSPI_MSPI_STATUS, 0);
-
-	pr_err("Timeout waiting for SPI to be ready!\n");
-
-	return -EBUSY;
-}
-
-static void bcm53xxspi_buf_write(struct bcm53xxspi *b53spi, u8 *w_buf,
-				 size_t len, bool cont)
-{
-	u32 tmp;
-	int i;
-
-	for (i = 0; i < len; i++) {
-		/* Transmit Register File MSB */
-		bcm53xxspi_write(b53spi, B53SPI_MSPI_TXRAM + 4 * (i * 2),
-				 (unsigned int)w_buf[i]);
-	}
-
-	for (i = 0; i < len; i++) {
-		tmp = B53SPI_CDRAM_CONT | B53SPI_CDRAM_PCS_DISABLE_ALL |
-		      B53SPI_CDRAM_PCS_DSCK;
-		if (!cont && i == len - 1)
-			tmp &= ~B53SPI_CDRAM_CONT;
-		tmp &= ~0x1;
-		/* Command Register File */
-		bcm53xxspi_write(b53spi, B53SPI_MSPI_CDRAM + 4 * i, tmp);
-	}
-
-	/* Set queue pointers */
-	bcm53xxspi_write(b53spi, B53SPI_MSPI_NEWQP, 0);
-	bcm53xxspi_write(b53spi, B53SPI_MSPI_ENDQP, len - 1);
-
-	if (cont)
-		bcm53xxspi_write(b53spi, B53SPI_MSPI_WRITE_LOCK, 1);
-
-	/* Start SPI transfer */
-	tmp = bcm53xxspi_read(b53spi, B53SPI_MSPI_SPCR2);
-	tmp |= B53SPI_MSPI_SPCR2_SPE;
-	if (cont)
-		tmp |= B53SPI_MSPI_SPCR2_CONT_AFTER_CMD;
-	bcm53xxspi_write(b53spi, B53SPI_MSPI_SPCR2, tmp);
-
-	/* Wait for SPI to finish */
-	bcm53xxspi_wait(b53spi, bcm53xxspi_calc_timeout(len));
-
-	if (!cont)
-		bcm53xxspi_write(b53spi, B53SPI_MSPI_WRITE_LOCK, 0);
-}
-
-static void bcm53xxspi_buf_read(struct bcm53xxspi *b53spi, u8 *r_buf,
-				size_t len, bool cont)
-{
-	u32 tmp;
-	int i;
-
-	for (i = 0; i < len; i++) {
-		tmp = B53SPI_CDRAM_CONT | B53SPI_CDRAM_PCS_DISABLE_ALL |
-		      B53SPI_CDRAM_PCS_DSCK;
-		if (!cont && i == len - 1)
-			tmp &= ~B53SPI_CDRAM_CONT;
-		tmp &= ~0x1;
-		/* Command Register File */
-		bcm53xxspi_write(b53spi, B53SPI_MSPI_CDRAM + 4 * i, tmp);
-	}
-
-	/* Set queue pointers */
-	bcm53xxspi_write(b53spi, B53SPI_MSPI_NEWQP, 0);
-	bcm53xxspi_write(b53spi, B53SPI_MSPI_ENDQP, len - 1);
-
-	if (cont)
-		bcm53xxspi_write(b53spi, B53SPI_MSPI_WRITE_LOCK, 1);
-
-	/* Start SPI transfer */
-	tmp = bcm53xxspi_read(b53spi, B53SPI_MSPI_SPCR2);
-	tmp |= B53SPI_MSPI_SPCR2_SPE;
-	if (cont)
-		tmp |= B53SPI_MSPI_SPCR2_CONT_AFTER_CMD;
-	bcm53xxspi_write(b53spi, B53SPI_MSPI_SPCR2, tmp);
-
-	/* Wait for SPI to finish */
-	bcm53xxspi_wait(b53spi, bcm53xxspi_calc_timeout(len));
-
-	if (!cont)
-		bcm53xxspi_write(b53spi, B53SPI_MSPI_WRITE_LOCK, 0);
-
-	for (i = 0; i < len; ++i) {
-		u16 reg = B53SPI_MSPI_RXRAM + 4 * (1 + i * 2);
-
-		/* Data stored in the transmit register file LSB */
-		r_buf[i] = (u8)bcm53xxspi_read(b53spi, reg);
-	}
-}
-
-static int bcm53xxspi_transfer_one(struct spi_master *master,
-				   struct spi_device *spi,
-				   struct spi_transfer *t)
-{
-	struct bcm53xxspi *b53spi = spi_master_get_devdata(master);
-	u8 *buf;
-	size_t left;
-
-	bcm53xxspi_disable_bspi(b53spi);
-
-	if (t->tx_buf) {
-		buf = (u8 *)t->tx_buf;
-		left = t->len;
-		while (left) {
-			size_t to_write = min_t(size_t, 16, left);
-			bool cont = !spi_transfer_is_last(master, t) ||
-				    left - to_write > 0;
-
-			bcm53xxspi_buf_write(b53spi, buf, to_write, cont);
-			left -= to_write;
-			buf += to_write;
-		}
-	}
-
-	if (t->rx_buf) {
-		buf = (u8 *)t->rx_buf;
-		left = t->len;
-		while (left) {
-			size_t to_read = min_t(size_t, 16, left);
-			bool cont = !spi_transfer_is_last(master, t) ||
-				    left - to_read > 0;
-
-			bcm53xxspi_buf_read(b53spi, buf, to_read, cont);
-			left -= to_read;
-			buf += to_read;
-		}
-	}
-
-	return 0;
-}
-
-static int bcm53xxspi_flash_read(struct spi_device *spi,
-				 struct spi_flash_read_message *msg)
-{
-	struct bcm53xxspi *b53spi = spi_master_get_devdata(spi->master);
-	int ret = 0;
-
-	if (msg->from + msg->len > BCM53XXSPI_FLASH_WINDOW)
-		return -EINVAL;
-
-	bcm53xxspi_enable_bspi(b53spi);
-	memcpy_fromio(msg->buf, b53spi->mmio_base + msg->from, msg->len);
-	msg->retlen = msg->len;
-
-	return ret;
-}
-
-/**************************************************
- * BCMA
- **************************************************/
-
-static const struct bcma_device_id bcm53xxspi_bcma_tbl[] = {
-	BCMA_CORE(BCMA_MANUF_BCM, BCMA_CORE_NS_QSPI, BCMA_ANY_REV, BCMA_ANY_CLASS),
-	{},
-};
-MODULE_DEVICE_TABLE(bcma, bcm53xxspi_bcma_tbl);
-
-static int bcm53xxspi_bcma_probe(struct bcma_device *core)
-{
-	struct device *dev = &core->dev;
-	struct bcm53xxspi *b53spi;
-	struct spi_master *master;
-	int err;
-
-	if (core->bus->drv_cc.core->id.rev != 42) {
-		pr_err("SPI on SoC with unsupported ChipCommon rev\n");
-		return -ENOTSUPP;
-	}
-
-	master = spi_alloc_master(dev, sizeof(*b53spi));
-	if (!master)
-		return -ENOMEM;
-
-	b53spi = spi_master_get_devdata(master);
-	b53spi->master = master;
-	b53spi->core = core;
-
-	if (core->addr_s[0])
-		b53spi->mmio_base = devm_ioremap(dev, core->addr_s[0],
-						 BCM53XXSPI_FLASH_WINDOW);
-	b53spi->bspi = true;
-	bcm53xxspi_disable_bspi(b53spi);
-
-	master->dev.of_node = dev->of_node;
-	master->transfer_one = bcm53xxspi_transfer_one;
-	if (b53spi->mmio_base)
-		master->spi_flash_read = bcm53xxspi_flash_read;
-
-	bcma_set_drvdata(core, b53spi);
-
-	err = devm_spi_register_master(dev, master);
-	if (err) {
-		spi_master_put(master);
-		bcma_set_drvdata(core, NULL);
-		return err;
-	}
-
-	return 0;
-}
-
-static struct bcma_driver bcm53xxspi_bcma_driver = {
-	.name		= KBUILD_MODNAME,
-	.id_table	= bcm53xxspi_bcma_tbl,
-	.probe		= bcm53xxspi_bcma_probe,
-};
-
-/**************************************************
- * Init & exit
- **************************************************/
-
-static int __init bcm53xxspi_module_init(void)
-{
-	int err = 0;
-
-	err = bcma_driver_register(&bcm53xxspi_bcma_driver);
-	if (err)
-		pr_err("Failed to register bcma driver: %d\n", err);
-
-	return err;
-}
-
-static void __exit bcm53xxspi_module_exit(void)
-{
-	bcma_driver_unregister(&bcm53xxspi_bcma_driver);
-}
-
-module_init(bcm53xxspi_module_init);
-module_exit(bcm53xxspi_module_exit);
-
-MODULE_DESCRIPTION("Broadcom BCM53xx SPI Controller driver");
-MODULE_AUTHOR("Rafał Miłecki <zajec5@gmail.com>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/spi/spi-bcm53xx.h b/drivers/spi/spi-bcm53xx.h
deleted file mode 100644
index 03e3442086ec..000000000000
--- a/drivers/spi/spi-bcm53xx.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef SPI_BCM53XX_H
-#define SPI_BCM53XX_H
-
-#define B53SPI_BSPI_REVISION_ID			0x000
-#define B53SPI_BSPI_SCRATCH			0x004
-#define B53SPI_BSPI_MAST_N_BOOT_CTRL		0x008
-#define B53SPI_BSPI_BUSY_STATUS			0x00c
-#define B53SPI_BSPI_INTR_STATUS			0x010
-#define B53SPI_BSPI_B0_STATUS			0x014
-#define B53SPI_BSPI_B0_CTRL			0x018
-#define B53SPI_BSPI_B1_STATUS			0x01c
-#define B53SPI_BSPI_B1_CTRL			0x020
-#define B53SPI_BSPI_STRAP_OVERRIDE_CTRL		0x024
-#define B53SPI_BSPI_FLEX_MODE_ENABLE		0x028
-#define B53SPI_BSPI_BITS_PER_CYCLE		0x02c
-#define B53SPI_BSPI_BITS_PER_PHASE		0x030
-#define B53SPI_BSPI_CMD_AND_MODE_BYTE		0x034
-#define B53SPI_BSPI_BSPI_FLASH_UPPER_ADDR_BYTE	0x038
-#define B53SPI_BSPI_BSPI_XOR_VALUE		0x03c
-#define B53SPI_BSPI_BSPI_XOR_ENABLE		0x040
-#define B53SPI_BSPI_BSPI_PIO_MODE_ENABLE	0x044
-#define B53SPI_BSPI_BSPI_PIO_IODIR		0x048
-#define B53SPI_BSPI_BSPI_PIO_DATA		0x04c
-
-/* RAF */
-#define B53SPI_RAF_START_ADDR			0x100
-#define B53SPI_RAF_NUM_WORDS			0x104
-#define B53SPI_RAF_CTRL				0x108
-#define B53SPI_RAF_FULLNESS			0x10c
-#define B53SPI_RAF_WATERMARK			0x110
-#define B53SPI_RAF_STATUS			0x114
-#define B53SPI_RAF_READ_DATA			0x118
-#define B53SPI_RAF_WORD_CNT			0x11c
-#define B53SPI_RAF_CURR_ADDR			0x120
-
-/* MSPI */
-#define B53SPI_MSPI_SPCR0_LSB			0x200
-#define B53SPI_MSPI_SPCR0_MSB			0x204
-#define B53SPI_MSPI_SPCR1_LSB			0x208
-#define B53SPI_MSPI_SPCR1_MSB			0x20c
-#define B53SPI_MSPI_NEWQP			0x210
-#define B53SPI_MSPI_ENDQP			0x214
-#define B53SPI_MSPI_SPCR2			0x218
-#define  B53SPI_MSPI_SPCR2_SPE			0x00000040
-#define  B53SPI_MSPI_SPCR2_CONT_AFTER_CMD	0x00000080
-#define B53SPI_MSPI_MSPI_STATUS			0x220
-#define  B53SPI_MSPI_MSPI_STATUS_SPIF		0x00000001
-#define B53SPI_MSPI_CPTQP			0x224
-#define B53SPI_MSPI_TXRAM			0x240 /* 32 registers, up to 0x2b8 */
-#define B53SPI_MSPI_RXRAM			0x2c0 /* 32 registers, up to 0x33c */
-#define B53SPI_MSPI_CDRAM			0x340 /* 16 registers, up to 0x37c */
-#define  B53SPI_CDRAM_PCS_PCS0			0x00000001
-#define  B53SPI_CDRAM_PCS_PCS1			0x00000002
-#define  B53SPI_CDRAM_PCS_PCS2			0x00000004
-#define  B53SPI_CDRAM_PCS_PCS3			0x00000008
-#define  B53SPI_CDRAM_PCS_DISABLE_ALL		0x0000000f
-#define  B53SPI_CDRAM_PCS_DSCK			0x00000010
-#define  B53SPI_CDRAM_BITSE			0x00000040
-#define  B53SPI_CDRAM_CONT			0x00000080
-#define B53SPI_MSPI_WRITE_LOCK			0x380
-#define B53SPI_MSPI_DISABLE_FLUSH_GEN		0x384
-
-/* Interrupt */
-#define B53SPI_INTR_RAF_LR_FULLNESS_REACHED	0x3a0
-#define B53SPI_INTR_RAF_LR_TRUNCATED		0x3a4
-#define B53SPI_INTR_RAF_LR_IMPATIENT		0x3a8
-#define B53SPI_INTR_RAF_LR_SESSION_DONE		0x3ac
-#define B53SPI_INTR_RAF_LR_OVERREAD		0x3b0
-#define B53SPI_INTR_MSPI_DONE			0x3b4
-#define B53SPI_INTR_MSPI_HALT_SET_TRANSACTION_DONE	0x3b8
-
-#endif /* SPI_BCM53XX_H */
diff --git a/drivers/spi/spi-bcm63xx-hsspi.c b/drivers/spi/spi-bcm63xx-hsspi.c
index cbcba614b253..c23849f7aa7b 100644
--- a/drivers/spi/spi-bcm63xx-hsspi.c
+++ b/drivers/spi/spi-bcm63xx-hsspi.c
@@ -352,22 +352,31 @@ static int bcm63xx_hsspi_probe(struct platform_device *pdev)
 	if (IS_ERR(clk))
 		return PTR_ERR(clk);
 
+	ret = clk_prepare_enable(clk);
+	if (ret)
+		return ret;
+
 	rate = clk_get_rate(clk);
 	if (!rate) {
 		struct clk *pll_clk = devm_clk_get(dev, "pll");
 
-		if (IS_ERR(pll_clk))
-			return PTR_ERR(pll_clk);
+		if (IS_ERR(pll_clk)) {
+			ret = PTR_ERR(pll_clk);
+			goto out_disable_clk;
+		}
+
+		ret = clk_prepare_enable(pll_clk);
+		if (ret)
+			goto out_disable_clk;
 
 		rate = clk_get_rate(pll_clk);
-		if (!rate)
-			return -EINVAL;
+		clk_disable_unprepare(pll_clk);
+		if (!rate) {
+			ret = -EINVAL;
+			goto out_disable_clk;
+		}
 	}
 
-	ret = clk_prepare_enable(clk);
-	if (ret)
-		return ret;
-
 	master = spi_alloc_master(&pdev->dev, sizeof(*bs));
 	if (!master) {
 		ret = -ENOMEM;
diff --git a/drivers/spi/spi-cadence.c b/drivers/spi/spi-cadence.c
index 4a001634023e..f3dad6fcdc35 100644
--- a/drivers/spi/spi-cadence.c
+++ b/drivers/spi/spi-cadence.c
@@ -694,8 +694,7 @@ static int cdns_spi_remove(struct platform_device *pdev)
  */
 static int __maybe_unused cdns_spi_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct spi_master *master = platform_get_drvdata(pdev);
+	struct spi_master *master = dev_get_drvdata(dev);
 
 	return spi_master_suspend(master);
 }
@@ -710,8 +709,7 @@ static int __maybe_unused cdns_spi_suspend(struct device *dev)
  */
 static int __maybe_unused cdns_spi_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct spi_master *master = platform_get_drvdata(pdev);
+	struct spi_master *master = dev_get_drvdata(dev);
 	struct cdns_spi *xspi = spi_master_get_devdata(master);
 
 	cdns_spi_init_hw(xspi);
diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c
index cb3c73007ca1..e6d5cc6ab108 100644
--- a/drivers/spi/spi-fsl-lpspi.c
+++ b/drivers/spi/spi-fsl-lpspi.c
@@ -1,19 +1,8 @@
-/*
- * Freescale i.MX7ULP LPSPI driver
- *
- * Copyright 2016 Freescale Semiconductor, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Freescale i.MX7ULP LPSPI driver
+//
+// Copyright 2016 Freescale Semiconductor, Inc.
 
 #include <linux/clk.h>
 #include <linux/completion.h>
diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c
index a056ee88a960..866246f21041 100644
--- a/drivers/spi/spi-imx.c
+++ b/drivers/spi/spi-imx.c
@@ -1,22 +1,6 @@
-/*
- * Copyright 2004-2007 Freescale Semiconductor, Inc. All Rights Reserved.
- * Copyright (C) 2008 Juergen Beisert
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the
- * Free Software Foundation
- * 51 Franklin Street, Fifth Floor
- * Boston, MA  02110-1301, USA.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2004-2007 Freescale Semiconductor, Inc. All Rights Reserved.
+// Copyright (C) 2008 Juergen Beisert
 
 #include <linux/clk.h>
 #include <linux/completion.h>
diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c
new file mode 100644
index 000000000000..990770dfa5cf
--- /dev/null
+++ b/drivers/spi/spi-mem.c
@@ -0,0 +1,410 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2018 Exceet Electronics GmbH
+ * Copyright (C) 2018 Bootlin
+ *
+ * Author: Boris Brezillon <boris.brezillon@bootlin.com>
+ */
+#include <linux/dmaengine.h>
+#include <linux/pm_runtime.h>
+#include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
+
+#include "internals.h"
+
+/**
+ * spi_controller_dma_map_mem_op_data() - DMA-map the buffer attached to a
+ *					  memory operation
+ * @ctlr: the SPI controller requesting this dma_map()
+ * @op: the memory operation containing the buffer to map
+ * @sgt: a pointer to a non-initialized sg_table that will be filled by this
+ *	 function
+ *
+ * Some controllers might want to do DMA on the data buffer embedded in @op.
+ * This helper prepares everything for you and provides a ready-to-use
+ * sg_table. This function is not intended to be called from spi drivers.
+ * Only SPI controller drivers should use it.
+ * Note that the caller must ensure the memory region pointed by
+ * op->data.buf.{in,out} is DMA-able before calling this function.
+ *
+ * Return: 0 in case of success, a negative error code otherwise.
+ */
+int spi_controller_dma_map_mem_op_data(struct spi_controller *ctlr,
+				       const struct spi_mem_op *op,
+				       struct sg_table *sgt)
+{
+	struct device *dmadev;
+
+	if (!op->data.nbytes)
+		return -EINVAL;
+
+	if (op->data.dir == SPI_MEM_DATA_OUT && ctlr->dma_tx)
+		dmadev = ctlr->dma_tx->device->dev;
+	else if (op->data.dir == SPI_MEM_DATA_IN && ctlr->dma_rx)
+		dmadev = ctlr->dma_rx->device->dev;
+	else
+		dmadev = ctlr->dev.parent;
+
+	if (!dmadev)
+		return -EINVAL;
+
+	return spi_map_buf(ctlr, dmadev, sgt, op->data.buf.in, op->data.nbytes,
+			   op->data.dir == SPI_MEM_DATA_IN ?
+			   DMA_FROM_DEVICE : DMA_TO_DEVICE);
+}
+EXPORT_SYMBOL_GPL(spi_controller_dma_map_mem_op_data);
+
+/**
+ * spi_controller_dma_unmap_mem_op_data() - DMA-unmap the buffer attached to a
+ *					    memory operation
+ * @ctlr: the SPI controller requesting this dma_unmap()
+ * @op: the memory operation containing the buffer to unmap
+ * @sgt: a pointer to an sg_table previously initialized by
+ *	 spi_controller_dma_map_mem_op_data()
+ *
+ * Some controllers might want to do DMA on the data buffer embedded in @op.
+ * This helper prepares things so that the CPU can access the
+ * op->data.buf.{in,out} buffer again.
+ *
+ * This function is not intended to be called from SPI drivers. Only SPI
+ * controller drivers should use it.
+ *
+ * This function should be called after the DMA operation has finished and is
+ * only valid if the previous spi_controller_dma_map_mem_op_data() call
+ * returned 0.
+ *
+ * Return: 0 in case of success, a negative error code otherwise.
+ */
+void spi_controller_dma_unmap_mem_op_data(struct spi_controller *ctlr,
+					  const struct spi_mem_op *op,
+					  struct sg_table *sgt)
+{
+	struct device *dmadev;
+
+	if (!op->data.nbytes)
+		return;
+
+	if (op->data.dir == SPI_MEM_DATA_OUT && ctlr->dma_tx)
+		dmadev = ctlr->dma_tx->device->dev;
+	else if (op->data.dir == SPI_MEM_DATA_IN && ctlr->dma_rx)
+		dmadev = ctlr->dma_rx->device->dev;
+	else
+		dmadev = ctlr->dev.parent;
+
+	spi_unmap_buf(ctlr, dmadev, sgt,
+		      op->data.dir == SPI_MEM_DATA_IN ?
+		      DMA_FROM_DEVICE : DMA_TO_DEVICE);
+}
+EXPORT_SYMBOL_GPL(spi_controller_dma_unmap_mem_op_data);
+
+static int spi_check_buswidth_req(struct spi_mem *mem, u8 buswidth, bool tx)
+{
+	u32 mode = mem->spi->mode;
+
+	switch (buswidth) {
+	case 1:
+		return 0;
+
+	case 2:
+		if ((tx && (mode & (SPI_TX_DUAL | SPI_TX_QUAD))) ||
+		    (!tx && (mode & (SPI_RX_DUAL | SPI_RX_QUAD))))
+			return 0;
+
+		break;
+
+	case 4:
+		if ((tx && (mode & SPI_TX_QUAD)) ||
+		    (!tx && (mode & SPI_RX_QUAD)))
+			return 0;
+
+		break;
+
+	default:
+		break;
+	}
+
+	return -ENOTSUPP;
+}
+
+static bool spi_mem_default_supports_op(struct spi_mem *mem,
+					const struct spi_mem_op *op)
+{
+	if (spi_check_buswidth_req(mem, op->cmd.buswidth, true))
+		return false;
+
+	if (op->addr.nbytes &&
+	    spi_check_buswidth_req(mem, op->addr.buswidth, true))
+		return false;
+
+	if (op->dummy.nbytes &&
+	    spi_check_buswidth_req(mem, op->dummy.buswidth, true))
+		return false;
+
+	if (op->data.nbytes &&
+	    spi_check_buswidth_req(mem, op->data.buswidth,
+				   op->data.dir == SPI_MEM_DATA_OUT))
+		return false;
+
+	return true;
+}
+EXPORT_SYMBOL_GPL(spi_mem_default_supports_op);
+
+/**
+ * spi_mem_supports_op() - Check if a memory device and the controller it is
+ *			   connected to support a specific memory operation
+ * @mem: the SPI memory
+ * @op: the memory operation to check
+ *
+ * Some controllers are only supporting Single or Dual IOs, others might only
+ * support specific opcodes, or it can even be that the controller and device
+ * both support Quad IOs but the hardware prevents you from using it because
+ * only 2 IO lines are connected.
+ *
+ * This function checks whether a specific operation is supported.
+ *
+ * Return: true if @op is supported, false otherwise.
+ */
+bool spi_mem_supports_op(struct spi_mem *mem, const struct spi_mem_op *op)
+{
+	struct spi_controller *ctlr = mem->spi->controller;
+
+	if (ctlr->mem_ops && ctlr->mem_ops->supports_op)
+		return ctlr->mem_ops->supports_op(mem, op);
+
+	return spi_mem_default_supports_op(mem, op);
+}
+EXPORT_SYMBOL_GPL(spi_mem_supports_op);
+
+/**
+ * spi_mem_exec_op() - Execute a memory operation
+ * @mem: the SPI memory
+ * @op: the memory operation to execute
+ *
+ * Executes a memory operation.
+ *
+ * This function first checks that @op is supported and then tries to execute
+ * it.
+ *
+ * Return: 0 in case of success, a negative error code otherwise.
+ */
+int spi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
+{
+	unsigned int tmpbufsize, xferpos = 0, totalxferlen = 0;
+	struct spi_controller *ctlr = mem->spi->controller;
+	struct spi_transfer xfers[4] = { };
+	struct spi_message msg;
+	u8 *tmpbuf;
+	int ret;
+
+	if (!spi_mem_supports_op(mem, op))
+		return -ENOTSUPP;
+
+	if (ctlr->mem_ops) {
+		/*
+		 * Flush the message queue before executing our SPI memory
+		 * operation to prevent preemption of regular SPI transfers.
+		 */
+		spi_flush_queue(ctlr);
+
+		if (ctlr->auto_runtime_pm) {
+			ret = pm_runtime_get_sync(ctlr->dev.parent);
+			if (ret < 0) {
+				dev_err(&ctlr->dev,
+					"Failed to power device: %d\n",
+					ret);
+				return ret;
+			}
+		}
+
+		mutex_lock(&ctlr->bus_lock_mutex);
+		mutex_lock(&ctlr->io_mutex);
+		ret = ctlr->mem_ops->exec_op(mem, op);
+		mutex_unlock(&ctlr->io_mutex);
+		mutex_unlock(&ctlr->bus_lock_mutex);
+
+		if (ctlr->auto_runtime_pm)
+			pm_runtime_put(ctlr->dev.parent);
+
+		/*
+		 * Some controllers only optimize specific paths (typically the
+		 * read path) and expect the core to use the regular SPI
+		 * interface in other cases.
+		 */
+		if (!ret || ret != -ENOTSUPP)
+			return ret;
+	}
+
+	tmpbufsize = sizeof(op->cmd.opcode) + op->addr.nbytes +
+		     op->dummy.nbytes;
+
+	/*
+	 * Allocate a buffer to transmit the CMD, ADDR cycles with kmalloc() so
+	 * we're guaranteed that this buffer is DMA-able, as required by the
+	 * SPI layer.
+	 */
+	tmpbuf = kzalloc(tmpbufsize, GFP_KERNEL | GFP_DMA);
+	if (!tmpbuf)
+		return -ENOMEM;
+
+	spi_message_init(&msg);
+
+	tmpbuf[0] = op->cmd.opcode;
+	xfers[xferpos].tx_buf = tmpbuf;
+	xfers[xferpos].len = sizeof(op->cmd.opcode);
+	xfers[xferpos].tx_nbits = op->cmd.buswidth;
+	spi_message_add_tail(&xfers[xferpos], &msg);
+	xferpos++;
+	totalxferlen++;
+
+	if (op->addr.nbytes) {
+		int i;
+
+		for (i = 0; i < op->addr.nbytes; i++)
+			tmpbuf[i + 1] = op->addr.val >>
+					(8 * (op->addr.nbytes - i - 1));
+
+		xfers[xferpos].tx_buf = tmpbuf + 1;
+		xfers[xferpos].len = op->addr.nbytes;
+		xfers[xferpos].tx_nbits = op->addr.buswidth;
+		spi_message_add_tail(&xfers[xferpos], &msg);
+		xferpos++;
+		totalxferlen += op->addr.nbytes;
+	}
+
+	if (op->dummy.nbytes) {
+		memset(tmpbuf + op->addr.nbytes + 1, 0xff, op->dummy.nbytes);
+		xfers[xferpos].tx_buf = tmpbuf + op->addr.nbytes + 1;
+		xfers[xferpos].len = op->dummy.nbytes;
+		xfers[xferpos].tx_nbits = op->dummy.buswidth;
+		spi_message_add_tail(&xfers[xferpos], &msg);
+		xferpos++;
+		totalxferlen += op->dummy.nbytes;
+	}
+
+	if (op->data.nbytes) {
+		if (op->data.dir == SPI_MEM_DATA_IN) {
+			xfers[xferpos].rx_buf = op->data.buf.in;
+			xfers[xferpos].rx_nbits = op->data.buswidth;
+		} else {
+			xfers[xferpos].tx_buf = op->data.buf.out;
+			xfers[xferpos].tx_nbits = op->data.buswidth;
+		}
+
+		xfers[xferpos].len = op->data.nbytes;
+		spi_message_add_tail(&xfers[xferpos], &msg);
+		xferpos++;
+		totalxferlen += op->data.nbytes;
+	}
+
+	ret = spi_sync(mem->spi, &msg);
+
+	kfree(tmpbuf);
+
+	if (ret)
+		return ret;
+
+	if (msg.actual_length != totalxferlen)
+		return -EIO;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(spi_mem_exec_op);
+
+/**
+ * spi_mem_adjust_op_size() - Adjust the data size of a SPI mem operation to
+ *			      match controller limitations
+ * @mem: the SPI memory
+ * @op: the operation to adjust
+ *
+ * Some controllers have FIFO limitations and must split a data transfer
+ * operation into multiple ones, others require a specific alignment for
+ * optimized accesses. This function allows SPI mem drivers to split a single
+ * operation into multiple sub-operations when required.
+ *
+ * Return: a negative error code if the controller can't properly adjust @op,
+ *	   0 otherwise. Note that @op->data.nbytes will be updated if @op
+ *	   can't be handled in a single step.
+ */
+int spi_mem_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
+{
+	struct spi_controller *ctlr = mem->spi->controller;
+
+	if (ctlr->mem_ops && ctlr->mem_ops->adjust_op_size)
+		return ctlr->mem_ops->adjust_op_size(mem, op);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(spi_mem_adjust_op_size);
+
+static inline struct spi_mem_driver *to_spi_mem_drv(struct device_driver *drv)
+{
+	return container_of(drv, struct spi_mem_driver, spidrv.driver);
+}
+
+static int spi_mem_probe(struct spi_device *spi)
+{
+	struct spi_mem_driver *memdrv = to_spi_mem_drv(spi->dev.driver);
+	struct spi_mem *mem;
+
+	mem = devm_kzalloc(&spi->dev, sizeof(*mem), GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	mem->spi = spi;
+	spi_set_drvdata(spi, mem);
+
+	return memdrv->probe(mem);
+}
+
+static int spi_mem_remove(struct spi_device *spi)
+{
+	struct spi_mem_driver *memdrv = to_spi_mem_drv(spi->dev.driver);
+	struct spi_mem *mem = spi_get_drvdata(spi);
+
+	if (memdrv->remove)
+		return memdrv->remove(mem);
+
+	return 0;
+}
+
+static void spi_mem_shutdown(struct spi_device *spi)
+{
+	struct spi_mem_driver *memdrv = to_spi_mem_drv(spi->dev.driver);
+	struct spi_mem *mem = spi_get_drvdata(spi);
+
+	if (memdrv->shutdown)
+		memdrv->shutdown(mem);
+}
+
+/**
+ * spi_mem_driver_register_with_owner() - Register a SPI memory driver
+ * @memdrv: the SPI memory driver to register
+ * @owner: the owner of this driver
+ *
+ * Registers a SPI memory driver.
+ *
+ * Return: 0 in case of success, a negative error core otherwise.
+ */
+
+int spi_mem_driver_register_with_owner(struct spi_mem_driver *memdrv,
+				       struct module *owner)
+{
+	memdrv->spidrv.probe = spi_mem_probe;
+	memdrv->spidrv.remove = spi_mem_remove;
+	memdrv->spidrv.shutdown = spi_mem_shutdown;
+
+	return __spi_register_driver(owner, &memdrv->spidrv);
+}
+EXPORT_SYMBOL_GPL(spi_mem_driver_register_with_owner);
+
+/**
+ * spi_mem_driver_unregister_with_owner() - Unregister a SPI memory driver
+ * @memdrv: the SPI memory driver to unregister
+ *
+ * Unregisters a SPI memory driver.
+ */
+void spi_mem_driver_unregister(struct spi_mem_driver *memdrv)
+{
+	spi_unregister_driver(&memdrv->spidrv);
+}
+EXPORT_SYMBOL_GPL(spi_mem_driver_unregister);
diff --git a/drivers/spi/spi-meson-spicc.c b/drivers/spi/spi-meson-spicc.c
index 5c82910e3480..7fe4488ace57 100644
--- a/drivers/spi/spi-meson-spicc.c
+++ b/drivers/spi/spi-meson-spicc.c
@@ -574,10 +574,15 @@ static int meson_spicc_probe(struct platform_device *pdev)
 		master->max_speed_hz = rate >> 2;
 
 	ret = devm_spi_register_master(&pdev->dev, master);
-	if (!ret)
-		return 0;
+	if (ret) {
+		dev_err(&pdev->dev, "spi master registration failed\n");
+		goto out_clk;
+	}
 
-	dev_err(&pdev->dev, "spi master registration failed\n");
+	return 0;
+
+out_clk:
+	clk_disable_unprepare(spicc->core);
 
 out_master:
 	spi_master_put(master);
diff --git a/drivers/spi/spi-mpc52xx.c b/drivers/spi/spi-mpc52xx.c
index e8b59ce4dc3a..0e55784a3ad9 100644
--- a/drivers/spi/spi-mpc52xx.c
+++ b/drivers/spi/spi-mpc52xx.c
@@ -447,7 +447,7 @@ static int mpc52xx_spi_probe(struct platform_device *op)
 
 		for (i = 0; i < ms->gpio_cs_count; i++) {
 			gpio_cs = of_get_gpio(op->dev.of_node, i);
-			if (gpio_cs < 0) {
+			if (!gpio_is_valid(gpio_cs)) {
 				dev_err(&op->dev,
 					"could not parse the gpio field in oftree\n");
 				rc = -ENODEV;
diff --git a/drivers/spi/spi-mxs.c b/drivers/spi/spi-mxs.c
index 3d216b950b41..6ac95a2a21ce 100644
--- a/drivers/spi/spi-mxs.c
+++ b/drivers/spi/spi-mxs.c
@@ -1,32 +1,22 @@
-/*
- * Freescale MXS SPI master driver
- *
- * Copyright 2012 DENX Software Engineering, GmbH.
- * Copyright 2012 Freescale Semiconductor, Inc.
- * Copyright 2008 Embedded Alley Solutions, Inc All Rights Reserved.
- *
- * Rework and transition to new API by:
- * Marek Vasut <marex@denx.de>
- *
- * Based on previous attempt by:
- * Fabio Estevam <fabio.estevam@freescale.com>
- *
- * Based on code from U-Boot bootloader by:
- * Marek Vasut <marex@denx.de>
- *
- * Based on spi-stmp.c, which is:
- * Author: Dmitry Pervushin <dimka@embeddedalley.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Freescale MXS SPI master driver
+//
+// Copyright 2012 DENX Software Engineering, GmbH.
+// Copyright 2012 Freescale Semiconductor, Inc.
+// Copyright 2008 Embedded Alley Solutions, Inc All Rights Reserved.
+//
+// Rework and transition to new API by:
+// Marek Vasut <marex@denx.de>
+//
+// Based on previous attempt by:
+// Fabio Estevam <fabio.estevam@freescale.com>
+//
+// Based on code from U-Boot bootloader by:
+// Marek Vasut <marex@denx.de>
+//
+// Based on spi-stmp.c, which is:
+// Author: Dmitry Pervushin <dimka@embeddedalley.com>
 
 #include <linux/kernel.h>
 #include <linux/ioport.h>
diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c
index 9bf64e6eca9b..6c628a54e946 100644
--- a/drivers/spi/spi-omap2-mcspi.c
+++ b/drivers/spi/spi-omap2-mcspi.c
@@ -255,6 +255,7 @@ static void omap2_mcspi_set_cs(struct spi_device *spi, bool enable)
 	if (spi->controller_state) {
 		int err = pm_runtime_get_sync(mcspi->dev);
 		if (err < 0) {
+			pm_runtime_put_noidle(mcspi->dev);
 			dev_err(mcspi->dev, "failed to get sync: %d\n", err);
 			return;
 		}
@@ -350,20 +351,6 @@ static void omap2_mcspi_set_fifo(const struct spi_device *spi,
 	mcspi->fifo_depth = 0;
 }
 
-static void omap2_mcspi_restore_ctx(struct omap2_mcspi *mcspi)
-{
-	struct spi_master	*spi_cntrl = mcspi->master;
-	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
-	struct omap2_mcspi_cs	*cs;
-
-	/* McSPI: context restore */
-	mcspi_write_reg(spi_cntrl, OMAP2_MCSPI_MODULCTRL, ctx->modulctrl);
-	mcspi_write_reg(spi_cntrl, OMAP2_MCSPI_WAKEUPENABLE, ctx->wakeupenable);
-
-	list_for_each_entry(cs, &ctx->cs, node)
-		writel_relaxed(cs->chconf0, cs->base + OMAP2_MCSPI_CHCONF0);
-}
-
 static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit)
 {
 	unsigned long timeout;
@@ -1065,8 +1052,11 @@ static int omap2_mcspi_setup(struct spi_device *spi)
 	}
 
 	ret = pm_runtime_get_sync(mcspi->dev);
-	if (ret < 0)
+	if (ret < 0) {
+		pm_runtime_put_noidle(mcspi->dev);
+
 		return ret;
+	}
 
 	ret = omap2_mcspi_setup_transfer(spi, NULL);
 	pm_runtime_mark_last_busy(mcspi->dev);
@@ -1284,8 +1274,11 @@ static int omap2_mcspi_master_setup(struct omap2_mcspi *mcspi)
 	int			ret = 0;
 
 	ret = pm_runtime_get_sync(mcspi->dev);
-	if (ret < 0)
+	if (ret < 0) {
+		pm_runtime_put_noidle(mcspi->dev);
+
 		return ret;
+	}
 
 	mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE,
 			OMAP2_MCSPI_WAKEUPENABLE_WKEN);
@@ -1297,14 +1290,39 @@ static int omap2_mcspi_master_setup(struct omap2_mcspi *mcspi)
 	return 0;
 }
 
+/*
+ * When SPI wake up from off-mode, CS is in activate state. If it was in
+ * inactive state when driver was suspend, then force it to inactive state at
+ * wake up.
+ */
 static int omap_mcspi_runtime_resume(struct device *dev)
 {
-	struct omap2_mcspi	*mcspi;
-	struct spi_master	*master;
+	struct spi_master *master = dev_get_drvdata(dev);
+	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
+	struct omap2_mcspi_regs *ctx = &mcspi->ctx;
+	struct omap2_mcspi_cs *cs;
 
-	master = dev_get_drvdata(dev);
-	mcspi = spi_master_get_devdata(master);
-	omap2_mcspi_restore_ctx(mcspi);
+	/* McSPI: context restore */
+	mcspi_write_reg(master, OMAP2_MCSPI_MODULCTRL, ctx->modulctrl);
+	mcspi_write_reg(master, OMAP2_MCSPI_WAKEUPENABLE, ctx->wakeupenable);
+
+	list_for_each_entry(cs, &ctx->cs, node) {
+		/*
+		 * We need to toggle CS state for OMAP take this
+		 * change in account.
+		 */
+		if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE) == 0) {
+			cs->chconf0 |= OMAP2_MCSPI_CHCONF_FORCE;
+			writel_relaxed(cs->chconf0,
+				       cs->base + OMAP2_MCSPI_CHCONF0);
+			cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
+			writel_relaxed(cs->chconf0,
+				       cs->base + OMAP2_MCSPI_CHCONF0);
+		} else {
+			writel_relaxed(cs->chconf0,
+				       cs->base + OMAP2_MCSPI_CHCONF0);
+		}
+	}
 
 	return 0;
 }
@@ -1447,50 +1465,33 @@ static int omap2_mcspi_remove(struct platform_device *pdev)
 MODULE_ALIAS("platform:omap2_mcspi");
 
 #ifdef	CONFIG_SUSPEND
-/*
- * When SPI wake up from off-mode, CS is in activate state. If it was in
- * unactive state when driver was suspend, then force it to unactive state at
- * wake up.
- */
-static int omap2_mcspi_resume(struct device *dev)
+static int omap2_mcspi_suspend_noirq(struct device *dev)
 {
-	struct spi_master	*master = dev_get_drvdata(dev);
-	struct omap2_mcspi	*mcspi = spi_master_get_devdata(master);
-	struct omap2_mcspi_regs	*ctx = &mcspi->ctx;
-	struct omap2_mcspi_cs	*cs;
-
-	pm_runtime_get_sync(mcspi->dev);
-	list_for_each_entry(cs, &ctx->cs, node) {
-		if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE) == 0) {
-			/*
-			 * We need to toggle CS state for OMAP take this
-			 * change in account.
-			 */
-			cs->chconf0 |= OMAP2_MCSPI_CHCONF_FORCE;
-			writel_relaxed(cs->chconf0, cs->base + OMAP2_MCSPI_CHCONF0);
-			cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
-			writel_relaxed(cs->chconf0, cs->base + OMAP2_MCSPI_CHCONF0);
-		}
-	}
-	pm_runtime_mark_last_busy(mcspi->dev);
-	pm_runtime_put_autosuspend(mcspi->dev);
-
-	return pinctrl_pm_select_default_state(dev);
+	return pinctrl_pm_select_sleep_state(dev);
 }
 
-static int omap2_mcspi_suspend(struct device *dev)
+static int omap2_mcspi_resume_noirq(struct device *dev)
 {
-	return pinctrl_pm_select_sleep_state(dev);
+	struct spi_master *master = dev_get_drvdata(dev);
+	struct omap2_mcspi *mcspi = spi_master_get_devdata(master);
+	int error;
+
+	error = pinctrl_pm_select_default_state(dev);
+	if (error)
+		dev_warn(mcspi->dev, "%s: failed to set pins: %i\n",
+			 __func__, error);
+
+	return 0;
 }
 
 #else
-#define omap2_mcspi_suspend	NULL
-#define	omap2_mcspi_resume	NULL
+#define omap2_mcspi_suspend_noirq	NULL
+#define omap2_mcspi_resume_noirq	NULL
 #endif
 
 static const struct dev_pm_ops omap2_mcspi_pm_ops = {
-	.resume = omap2_mcspi_resume,
-	.suspend = omap2_mcspi_suspend,
+	.suspend_noirq = omap2_mcspi_suspend_noirq,
+	.resume_noirq = omap2_mcspi_resume_noirq,
 	.runtime_resume	= omap_mcspi_runtime_resume,
 };
 
diff --git a/drivers/spi/spi-pxa2xx-dma.c b/drivers/spi/spi-pxa2xx-dma.c
index 3d7f66080c57..2fa7f4b43492 100644
--- a/drivers/spi/spi-pxa2xx-dma.c
+++ b/drivers/spi/spi-pxa2xx-dma.c
@@ -51,19 +51,15 @@ static void pxa2xx_spi_dma_transfer_complete(struct driver_data *drv_data,
 		if (!pxa25x_ssp_comp(drv_data))
 			pxa2xx_spi_write(drv_data, SSTO, 0);
 
-		if (!error) {
-			msg->actual_length += drv_data->len;
-			msg->state = pxa2xx_spi_next_transfer(drv_data);
-		} else {
+		if (error) {
 			/* In case we got an error we disable the SSP now */
 			pxa2xx_spi_write(drv_data, SSCR0,
 					 pxa2xx_spi_read(drv_data, SSCR0)
 					 & ~SSCR0_SSE);
-
-			msg->state = ERROR_STATE;
+			msg->status = -EIO;
 		}
 
-		tasklet_schedule(&drv_data->pump_transfers);
+		spi_finalize_current_transfer(drv_data->master);
 	}
 }
 
@@ -74,11 +70,11 @@ static void pxa2xx_spi_dma_callback(void *data)
 
 static struct dma_async_tx_descriptor *
 pxa2xx_spi_dma_prepare_one(struct driver_data *drv_data,
-			   enum dma_transfer_direction dir)
+			   enum dma_transfer_direction dir,
+			   struct spi_transfer *xfer)
 {
 	struct chip_data *chip =
 		spi_get_ctldata(drv_data->master->cur_msg->spi);
-	struct spi_transfer *xfer = drv_data->cur_transfer;
 	enum dma_slave_buswidth width;
 	struct dma_slave_config cfg;
 	struct dma_chan *chan;
@@ -144,12 +140,13 @@ irqreturn_t pxa2xx_spi_dma_transfer(struct driver_data *drv_data)
 	return IRQ_NONE;
 }
 
-int pxa2xx_spi_dma_prepare(struct driver_data *drv_data, u32 dma_burst)
+int pxa2xx_spi_dma_prepare(struct driver_data *drv_data,
+			   struct spi_transfer *xfer)
 {
 	struct dma_async_tx_descriptor *tx_desc, *rx_desc;
 	int err;
 
-	tx_desc = pxa2xx_spi_dma_prepare_one(drv_data, DMA_MEM_TO_DEV);
+	tx_desc = pxa2xx_spi_dma_prepare_one(drv_data, DMA_MEM_TO_DEV, xfer);
 	if (!tx_desc) {
 		dev_err(&drv_data->pdev->dev,
 			"failed to get DMA TX descriptor\n");
@@ -157,7 +154,7 @@ int pxa2xx_spi_dma_prepare(struct driver_data *drv_data, u32 dma_burst)
 		goto err_tx;
 	}
 
-	rx_desc = pxa2xx_spi_dma_prepare_one(drv_data, DMA_DEV_TO_MEM);
+	rx_desc = pxa2xx_spi_dma_prepare_one(drv_data, DMA_DEV_TO_MEM, xfer);
 	if (!rx_desc) {
 		dev_err(&drv_data->pdev->dev,
 			"failed to get DMA RX descriptor\n");
@@ -187,6 +184,13 @@ void pxa2xx_spi_dma_start(struct driver_data *drv_data)
 	atomic_set(&drv_data->dma_running, 1);
 }
 
+void pxa2xx_spi_dma_stop(struct driver_data *drv_data)
+{
+	atomic_set(&drv_data->dma_running, 0);
+	dmaengine_terminate_sync(drv_data->master->dma_rx);
+	dmaengine_terminate_sync(drv_data->master->dma_tx);
+}
+
 int pxa2xx_spi_dma_setup(struct driver_data *drv_data)
 {
 	struct pxa2xx_spi_master *pdata = drv_data->master_info;
diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c
index 82dcb88fcfba..0b2d60d30f69 100644
--- a/drivers/spi/spi-pxa2xx.c
+++ b/drivers/spi/spi-pxa2xx.c
@@ -340,9 +340,11 @@ static void lpss_ssp_setup(struct driver_data *drv_data)
 	}
 }
 
-static void lpss_ssp_select_cs(struct driver_data *drv_data,
+static void lpss_ssp_select_cs(struct spi_device *spi,
 			       const struct lpss_config *config)
 {
+	struct driver_data *drv_data =
+		spi_controller_get_devdata(spi->controller);
 	u32 value, cs;
 
 	if (!config->cs_sel_mask)
@@ -350,7 +352,7 @@ static void lpss_ssp_select_cs(struct driver_data *drv_data,
 
 	value = __lpss_ssp_read_priv(drv_data, config->reg_cs_ctrl);
 
-	cs = drv_data->master->cur_msg->spi->chip_select;
+	cs = spi->chip_select;
 	cs <<= config->cs_sel_shift;
 	if (cs != (value & config->cs_sel_mask)) {
 		/*
@@ -369,15 +371,17 @@ static void lpss_ssp_select_cs(struct driver_data *drv_data,
 	}
 }
 
-static void lpss_ssp_cs_control(struct driver_data *drv_data, bool enable)
+static void lpss_ssp_cs_control(struct spi_device *spi, bool enable)
 {
+	struct driver_data *drv_data =
+		spi_controller_get_devdata(spi->controller);
 	const struct lpss_config *config;
 	u32 value;
 
 	config = lpss_get_config(drv_data);
 
 	if (enable)
-		lpss_ssp_select_cs(drv_data, config);
+		lpss_ssp_select_cs(spi, config);
 
 	value = __lpss_ssp_read_priv(drv_data, config->reg_cs_ctrl);
 	if (enable)
@@ -387,10 +391,11 @@ static void lpss_ssp_cs_control(struct driver_data *drv_data, bool enable)
 	__lpss_ssp_write_priv(drv_data, config->reg_cs_ctrl, value);
 }
 
-static void cs_assert(struct driver_data *drv_data)
+static void cs_assert(struct spi_device *spi)
 {
-	struct chip_data *chip =
-		spi_get_ctldata(drv_data->master->cur_msg->spi);
+	struct chip_data *chip = spi_get_ctldata(spi);
+	struct driver_data *drv_data =
+		spi_controller_get_devdata(spi->controller);
 
 	if (drv_data->ssp_type == CE4100_SSP) {
 		pxa2xx_spi_write(drv_data, SSSR, chip->frm);
@@ -408,13 +413,14 @@ static void cs_assert(struct driver_data *drv_data)
 	}
 
 	if (is_lpss_ssp(drv_data))
-		lpss_ssp_cs_control(drv_data, true);
+		lpss_ssp_cs_control(spi, true);
 }
 
-static void cs_deassert(struct driver_data *drv_data)
+static void cs_deassert(struct spi_device *spi)
 {
-	struct chip_data *chip =
-		spi_get_ctldata(drv_data->master->cur_msg->spi);
+	struct chip_data *chip = spi_get_ctldata(spi);
+	struct driver_data *drv_data =
+		spi_controller_get_devdata(spi->controller);
 	unsigned long timeout;
 
 	if (drv_data->ssp_type == CE4100_SSP)
@@ -437,7 +443,15 @@ static void cs_deassert(struct driver_data *drv_data)
 	}
 
 	if (is_lpss_ssp(drv_data))
-		lpss_ssp_cs_control(drv_data, false);
+		lpss_ssp_cs_control(spi, false);
+}
+
+static void pxa2xx_spi_set_cs(struct spi_device *spi, bool level)
+{
+	if (level)
+		cs_deassert(spi);
+	else
+		cs_assert(spi);
 }
 
 int pxa2xx_spi_flush(struct driver_data *drv_data)
@@ -549,70 +563,6 @@ static int u32_reader(struct driver_data *drv_data)
 	return drv_data->rx == drv_data->rx_end;
 }
 
-void *pxa2xx_spi_next_transfer(struct driver_data *drv_data)
-{
-	struct spi_message *msg = drv_data->master->cur_msg;
-	struct spi_transfer *trans = drv_data->cur_transfer;
-
-	/* Move to next transfer */
-	if (trans->transfer_list.next != &msg->transfers) {
-		drv_data->cur_transfer =
-			list_entry(trans->transfer_list.next,
-					struct spi_transfer,
-					transfer_list);
-		return RUNNING_STATE;
-	} else
-		return DONE_STATE;
-}
-
-/* caller already set message->status; dma and pio irqs are blocked */
-static void giveback(struct driver_data *drv_data)
-{
-	struct spi_transfer* last_transfer;
-	struct spi_message *msg;
-
-	msg = drv_data->master->cur_msg;
-	drv_data->cur_transfer = NULL;
-
-	last_transfer = list_last_entry(&msg->transfers, struct spi_transfer,
-					transfer_list);
-
-	/* Delay if requested before any change in chip select */
-	if (last_transfer->delay_usecs)
-		udelay(last_transfer->delay_usecs);
-
-	/* Drop chip select UNLESS cs_change is true or we are returning
-	 * a message with an error, or next message is for another chip
-	 */
-	if (!last_transfer->cs_change)
-		cs_deassert(drv_data);
-	else {
-		struct spi_message *next_msg;
-
-		/* Holding of cs was hinted, but we need to make sure
-		 * the next message is for the same chip.  Don't waste
-		 * time with the following tests unless this was hinted.
-		 *
-		 * We cannot postpone this until pump_messages, because
-		 * after calling msg->complete (below) the driver that
-		 * sent the current message could be unloaded, which
-		 * could invalidate the cs_control() callback...
-		 */
-
-		/* get a pointer to the next message, if any */
-		next_msg = spi_get_next_queued_message(drv_data->master);
-
-		/* see if the next and current messages point
-		 * to the same chip
-		 */
-		if ((next_msg && next_msg->spi != msg->spi) ||
-		    msg->state == ERROR_STATE)
-			cs_deassert(drv_data);
-	}
-
-	spi_finalize_current_message(drv_data->master);
-}
-
 static void reset_sccr1(struct driver_data *drv_data)
 {
 	struct chip_data *chip =
@@ -648,8 +598,8 @@ static void int_error_stop(struct driver_data *drv_data, const char* msg)
 
 	dev_err(&drv_data->pdev->dev, "%s\n", msg);
 
-	drv_data->master->cur_msg->state = ERROR_STATE;
-	tasklet_schedule(&drv_data->pump_transfers);
+	drv_data->master->cur_msg->status = -EIO;
+	spi_finalize_current_transfer(drv_data->master);
 }
 
 static void int_transfer_complete(struct driver_data *drv_data)
@@ -660,19 +610,7 @@ static void int_transfer_complete(struct driver_data *drv_data)
 	if (!pxa25x_ssp_comp(drv_data))
 		pxa2xx_spi_write(drv_data, SSTO, 0);
 
-	/* Update total byte transferred return count actual bytes read */
-	drv_data->master->cur_msg->actual_length += drv_data->len -
-				(drv_data->rx_end - drv_data->rx);
-
-	/* Transfer delays and chip select release are
-	 * handled in pump_transfers or giveback
-	 */
-
-	/* Move to next transfer */
-	drv_data->master->cur_msg->state = pxa2xx_spi_next_transfer(drv_data);
-
-	/* Schedule transfer tasklet */
-	tasklet_schedule(&drv_data->pump_transfers);
+	spi_finalize_current_transfer(drv_data->master);
 }
 
 static irqreturn_t interrupt_transfer(struct driver_data *drv_data)
@@ -973,17 +911,16 @@ static bool pxa2xx_spi_can_dma(struct spi_controller *master,
 	       xfer->len >= chip->dma_burst_size;
 }
 
-static void pump_transfers(unsigned long data)
+static int pxa2xx_spi_transfer_one(struct spi_controller *master,
+				   struct spi_device *spi,
+				   struct spi_transfer *transfer)
 {
-	struct driver_data *drv_data = (struct driver_data *)data;
-	struct spi_controller *master = drv_data->master;
+	struct driver_data *drv_data = spi_controller_get_devdata(master);
 	struct spi_message *message = master->cur_msg;
 	struct chip_data *chip = spi_get_ctldata(message->spi);
 	u32 dma_thresh = chip->dma_threshold;
 	u32 dma_burst = chip->dma_burst_size;
 	u32 change_mask = pxa2xx_spi_get_ssrc1_change_mask(drv_data);
-	struct spi_transfer *transfer;
-	struct spi_transfer *previous;
 	u32 clk_div;
 	u8 bits;
 	u32 speed;
@@ -992,36 +929,6 @@ static void pump_transfers(unsigned long data)
 	int err;
 	int dma_mapped;
 
-	/* Get current state information */
-	transfer = drv_data->cur_transfer;
-
-	/* Handle for abort */
-	if (message->state == ERROR_STATE) {
-		message->status = -EIO;
-		giveback(drv_data);
-		return;
-	}
-
-	/* Handle end of message */
-	if (message->state == DONE_STATE) {
-		message->status = 0;
-		giveback(drv_data);
-		return;
-	}
-
-	/* Delay if requested at end of transfer before CS change */
-	if (message->state == RUNNING_STATE) {
-		previous = list_entry(transfer->transfer_list.prev,
-					struct spi_transfer,
-					transfer_list);
-		if (previous->delay_usecs)
-			udelay(previous->delay_usecs);
-
-		/* Drop chip select only if cs_change is requested */
-		if (previous->cs_change)
-			cs_deassert(drv_data);
-	}
-
 	/* Check if we can DMA this transfer */
 	if (transfer->len > MAX_DMA_LEN && chip->enable_dma) {
 
@@ -1029,34 +936,27 @@ static void pump_transfers(unsigned long data)
 		if (message->is_dma_mapped
 				|| transfer->rx_dma || transfer->tx_dma) {
 			dev_err(&drv_data->pdev->dev,
-				"pump_transfers: mapped transfer length of "
-				"%u is greater than %d\n",
+				"Mapped transfer length of %u is greater than %d\n",
 				transfer->len, MAX_DMA_LEN);
-			message->status = -EINVAL;
-			giveback(drv_data);
-			return;
+			return -EINVAL;
 		}
 
 		/* warn ... we force this to PIO mode */
 		dev_warn_ratelimited(&message->spi->dev,
-				     "pump_transfers: DMA disabled for transfer length %ld "
-				     "greater than %d\n",
-				     (long)drv_data->len, MAX_DMA_LEN);
+				     "DMA disabled for transfer length %ld greater than %d\n",
+				     (long)transfer->len, MAX_DMA_LEN);
 	}
 
 	/* Setup the transfer state based on the type of transfer */
 	if (pxa2xx_spi_flush(drv_data) == 0) {
-		dev_err(&drv_data->pdev->dev, "pump_transfers: flush failed\n");
-		message->status = -EIO;
-		giveback(drv_data);
-		return;
+		dev_err(&drv_data->pdev->dev, "Flush failed\n");
+		return -EIO;
 	}
 	drv_data->n_bytes = chip->n_bytes;
 	drv_data->tx = (void *)transfer->tx_buf;
 	drv_data->tx_end = drv_data->tx + transfer->len;
 	drv_data->rx = transfer->rx_buf;
 	drv_data->rx_end = drv_data->rx + transfer->len;
-	drv_data->len = transfer->len;
 	drv_data->write = drv_data->tx ? chip->write : null_writer;
 	drv_data->read = drv_data->rx ? chip->read : null_reader;
 
@@ -1095,11 +995,9 @@ static void pump_transfers(unsigned long data)
 						bits, &dma_burst,
 						&dma_thresh))
 			dev_warn_ratelimited(&message->spi->dev,
-					     "pump_transfers: DMA burst size reduced to match bits_per_word\n");
+					     "DMA burst size reduced to match bits_per_word\n");
 	}
 
-	message->state = RUNNING_STATE;
-
 	dma_mapped = master->can_dma &&
 		     master->can_dma(master, message->spi, transfer) &&
 		     master->cur_msg_mapped;
@@ -1108,12 +1006,9 @@ static void pump_transfers(unsigned long data)
 		/* Ensure we have the correct interrupt handler */
 		drv_data->transfer_handler = pxa2xx_spi_dma_transfer;
 
-		err = pxa2xx_spi_dma_prepare(drv_data, dma_burst);
-		if (err) {
-			message->status = err;
-			giveback(drv_data);
-			return;
-		}
+		err = pxa2xx_spi_dma_prepare(drv_data, transfer);
+		if (err)
+			return err;
 
 		/* Clear status and start DMA engine */
 		cr1 = chip->cr1 | dma_thresh | drv_data->dma_cr1;
@@ -1175,27 +1070,40 @@ static void pump_transfers(unsigned long data)
 			pxa2xx_spi_write(drv_data, SSTO, chip->timeout);
 	}
 
-	cs_assert(drv_data);
-
-	/* after chip select, release the data by enabling service
-	 * requests and interrupts, without changing any mode bits */
+	/*
+	 * Release the data by enabling service requests and interrupts,
+	 * without changing any mode bits
+	 */
 	pxa2xx_spi_write(drv_data, SSCR1, cr1);
+
+	return 1;
 }
 
-static int pxa2xx_spi_transfer_one_message(struct spi_controller *master,
-					   struct spi_message *msg)
+static void pxa2xx_spi_handle_err(struct spi_controller *master,
+				 struct spi_message *msg)
 {
 	struct driver_data *drv_data = spi_controller_get_devdata(master);
 
-	/* Initial message state*/
-	msg->state = START_STATE;
-	drv_data->cur_transfer = list_entry(msg->transfers.next,
-						struct spi_transfer,
-						transfer_list);
+	/* Disable the SSP */
+	pxa2xx_spi_write(drv_data, SSCR0,
+			 pxa2xx_spi_read(drv_data, SSCR0) & ~SSCR0_SSE);
+	/* Clear and disable interrupts and service requests */
+	write_SSSR_CS(drv_data, drv_data->clear_sr);
+	pxa2xx_spi_write(drv_data, SSCR1,
+			 pxa2xx_spi_read(drv_data, SSCR1)
+			 & ~(drv_data->int_cr1 | drv_data->dma_cr1));
+	if (!pxa25x_ssp_comp(drv_data))
+		pxa2xx_spi_write(drv_data, SSTO, 0);
 
-	/* Mark as busy and launch transfers */
-	tasklet_schedule(&drv_data->pump_transfers);
-	return 0;
+	/*
+	 * Stop the DMA if running. Note DMA callback handler may have unset
+	 * the dma_running already, which is fine as stopping is not needed
+	 * then but we shouldn't rely this flag for anything else than
+	 * stopping. For instance to differentiate between PIO and DMA
+	 * transfers.
+	 */
+	if (atomic_read(&drv_data->dma_running))
+		pxa2xx_spi_dma_stop(drv_data);
 }
 
 static int pxa2xx_spi_unprepare_transfer(struct spi_controller *master)
@@ -1651,7 +1559,9 @@ static int pxa2xx_spi_probe(struct platform_device *pdev)
 	master->dma_alignment = DMA_ALIGNMENT;
 	master->cleanup = cleanup;
 	master->setup = setup;
-	master->transfer_one_message = pxa2xx_spi_transfer_one_message;
+	master->set_cs = pxa2xx_spi_set_cs;
+	master->transfer_one = pxa2xx_spi_transfer_one;
+	master->handle_err = pxa2xx_spi_handle_err;
 	master->unprepare_transfer_hardware = pxa2xx_spi_unprepare_transfer;
 	master->fw_translate_cs = pxa2xx_spi_fw_translate_cs;
 	master->auto_runtime_pm = true;
@@ -1702,7 +1612,9 @@ static int pxa2xx_spi_probe(struct platform_device *pdev)
 	}
 
 	/* Enable SOC clock */
-	clk_prepare_enable(ssp->clk);
+	status = clk_prepare_enable(ssp->clk);
+	if (status)
+		goto out_error_dma_irq_alloc;
 
 	master->max_speed_hz = clk_get_rate(ssp->clk);
 
@@ -1787,9 +1699,6 @@ static int pxa2xx_spi_probe(struct platform_device *pdev)
 		}
 	}
 
-	tasklet_init(&drv_data->pump_transfers, pump_transfers,
-		     (unsigned long)drv_data);
-
 	pm_runtime_set_autosuspend_delay(&pdev->dev, 50);
 	pm_runtime_use_autosuspend(&pdev->dev);
 	pm_runtime_set_active(&pdev->dev);
@@ -1809,6 +1718,8 @@ static int pxa2xx_spi_probe(struct platform_device *pdev)
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 	clk_disable_unprepare(ssp->clk);
+
+out_error_dma_irq_alloc:
 	pxa2xx_spi_dma_release(drv_data);
 	free_irq(ssp->irq, drv_data);
 
@@ -1882,8 +1793,11 @@ static int pxa2xx_spi_resume(struct device *dev)
 	int status;
 
 	/* Enable the SSP clock */
-	if (!pm_runtime_suspended(dev))
-		clk_prepare_enable(ssp->clk);
+	if (!pm_runtime_suspended(dev)) {
+		status = clk_prepare_enable(ssp->clk);
+		if (status)
+			return status;
+	}
 
 	/* Restore LPSS private register bits */
 	if (is_lpss_ssp(drv_data))
@@ -1912,9 +1826,10 @@ static int pxa2xx_spi_runtime_suspend(struct device *dev)
 static int pxa2xx_spi_runtime_resume(struct device *dev)
 {
 	struct driver_data *drv_data = dev_get_drvdata(dev);
+	int status;
 
-	clk_prepare_enable(drv_data->ssp->clk);
-	return 0;
+	status = clk_prepare_enable(drv_data->ssp->clk);
+	return status;
 }
 #endif
 
diff --git a/drivers/spi/spi-pxa2xx.h b/drivers/spi/spi-pxa2xx.h
index 0ae7defd3492..513c53aaeab2 100644
--- a/drivers/spi/spi-pxa2xx.h
+++ b/drivers/spi/spi-pxa2xx.h
@@ -46,15 +46,10 @@ struct driver_data {
 	u32 clear_sr;
 	u32 mask_sr;
 
-	/* Message Transfer pump */
-	struct tasklet_struct pump_transfers;
-
 	/* DMA engine support */
 	atomic_t dma_running;
 
-	/* Current message transfer state info */
-	struct spi_transfer *cur_transfer;
-	size_t len;
+	/* Current transfer state info */
 	void *tx;
 	void *tx_end;
 	void *rx;
@@ -104,11 +99,6 @@ static  inline void pxa2xx_spi_write(const struct driver_data *drv_data,
 	__raw_writel(val, drv_data->ioaddr + reg);
 }
 
-#define START_STATE ((void *)0)
-#define RUNNING_STATE ((void *)1)
-#define DONE_STATE ((void *)2)
-#define ERROR_STATE ((void *)-1)
-
 #define DMA_ALIGNMENT		8
 
 static inline int pxa25x_ssp_comp(struct driver_data *drv_data)
@@ -133,14 +123,15 @@ static inline void write_SSSR_CS(struct driver_data *drv_data, u32 val)
 }
 
 extern int pxa2xx_spi_flush(struct driver_data *drv_data);
-extern void *pxa2xx_spi_next_transfer(struct driver_data *drv_data);
 
 #define MAX_DMA_LEN		SZ_64K
 #define DEFAULT_DMA_CR1		(SSCR1_TSRE | SSCR1_RSRE | SSCR1_TRAIL)
 
 extern irqreturn_t pxa2xx_spi_dma_transfer(struct driver_data *drv_data);
-extern int pxa2xx_spi_dma_prepare(struct driver_data *drv_data, u32 dma_burst);
+extern int pxa2xx_spi_dma_prepare(struct driver_data *drv_data,
+				  struct spi_transfer *xfer);
 extern void pxa2xx_spi_dma_start(struct driver_data *drv_data);
+extern void pxa2xx_spi_dma_stop(struct driver_data *drv_data);
 extern int pxa2xx_spi_dma_setup(struct driver_data *drv_data);
 extern void pxa2xx_spi_dma_release(struct driver_data *drv_data);
 extern int pxa2xx_spi_set_dma_burst_and_threshold(struct chip_data *chip,
diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c
index baa3a9fa2638..7b7151ec14c8 100644
--- a/drivers/spi/spi-s3c64xx.c
+++ b/drivers/spi/spi-s3c64xx.c
@@ -28,15 +28,15 @@
 
 #define S3C64XX_SPI_CH_CFG		0x00
 #define S3C64XX_SPI_CLK_CFG		0x04
-#define S3C64XX_SPI_MODE_CFG	0x08
-#define S3C64XX_SPI_SLAVE_SEL	0x0C
+#define S3C64XX_SPI_MODE_CFG		0x08
+#define S3C64XX_SPI_SLAVE_SEL		0x0C
 #define S3C64XX_SPI_INT_EN		0x10
 #define S3C64XX_SPI_STATUS		0x14
 #define S3C64XX_SPI_TX_DATA		0x18
 #define S3C64XX_SPI_RX_DATA		0x1C
-#define S3C64XX_SPI_PACKET_CNT	0x20
-#define S3C64XX_SPI_PENDING_CLR	0x24
-#define S3C64XX_SPI_SWAP_CFG	0x28
+#define S3C64XX_SPI_PACKET_CNT		0x20
+#define S3C64XX_SPI_PENDING_CLR		0x24
+#define S3C64XX_SPI_SWAP_CFG		0x28
 #define S3C64XX_SPI_FB_CLK		0x2C
 
 #define S3C64XX_SPI_CH_HS_EN		(1<<6)	/* High Speed Enable */
@@ -77,9 +77,9 @@
 #define S3C64XX_SPI_INT_TX_FIFORDY_EN		(1<<0)
 
 #define S3C64XX_SPI_ST_RX_OVERRUN_ERR		(1<<5)
-#define S3C64XX_SPI_ST_RX_UNDERRUN_ERR	(1<<4)
+#define S3C64XX_SPI_ST_RX_UNDERRUN_ERR		(1<<4)
 #define S3C64XX_SPI_ST_TX_OVERRUN_ERR		(1<<3)
-#define S3C64XX_SPI_ST_TX_UNDERRUN_ERR	(1<<2)
+#define S3C64XX_SPI_ST_TX_UNDERRUN_ERR		(1<<2)
 #define S3C64XX_SPI_ST_RX_FIFORDY		(1<<1)
 #define S3C64XX_SPI_ST_TX_FIFORDY		(1<<0)
 
@@ -100,7 +100,7 @@
 #define S3C64XX_SPI_SWAP_TX_BIT			(1<<1)
 #define S3C64XX_SPI_SWAP_TX_EN			(1<<0)
 
-#define S3C64XX_SPI_FBCLK_MSK		(3<<0)
+#define S3C64XX_SPI_FBCLK_MSK			(3<<0)
 
 #define FIFO_LVL_MASK(i) ((i)->port_conf->fifo_lvl_mask[i->port_id])
 #define S3C64XX_SPI_ST_TX_DONE(v, i) (((v) & \
@@ -156,7 +156,6 @@ struct s3c64xx_spi_port_config {
  * @ioclk: Pointer to the i/o clock between master and slave
  * @master: Pointer to the SPI Protocol master.
  * @cntrlr_info: Platform specific data for the controller this driver manages.
- * @tgl_spi: Pointer to the last CS left untoggled by the cs_change hint.
  * @lock: Controller specific lock.
  * @state: Set of FLAGS to indicate status.
  * @rx_dmach: Controller's DMA channel for Rx.
@@ -177,7 +176,6 @@ struct s3c64xx_spi_driver_data {
 	struct platform_device          *pdev;
 	struct spi_master               *master;
 	struct s3c64xx_spi_info  *cntrlr_info;
-	struct spi_device               *tgl_spi;
 	spinlock_t                      lock;
 	unsigned long                   sfr_start;
 	struct completion               xfer_completion;
@@ -190,7 +188,7 @@ struct s3c64xx_spi_driver_data {
 	unsigned int			port_id;
 };
 
-static void flush_fifo(struct s3c64xx_spi_driver_data *sdd)
+static void s3c64xx_flush_fifo(struct s3c64xx_spi_driver_data *sdd)
 {
 	void __iomem *regs = sdd->regs;
 	unsigned long loops;
@@ -350,9 +348,8 @@ static bool s3c64xx_spi_can_dma(struct spi_master *master,
 	return xfer->len > (FIFO_LVL_MASK(sdd) >> 1) + 1;
 }
 
-static void enable_datapath(struct s3c64xx_spi_driver_data *sdd,
-				struct spi_device *spi,
-				struct spi_transfer *xfer, int dma_mode)
+static void s3c64xx_enable_datapath(struct s3c64xx_spi_driver_data *sdd,
+				    struct spi_transfer *xfer, int dma_mode)
 {
 	void __iomem *regs = sdd->regs;
 	u32 modecfg, chcfg;
@@ -442,8 +439,8 @@ static u32 s3c64xx_spi_wait_for_timeout(struct s3c64xx_spi_driver_data *sdd,
 	return RX_FIFO_LVL(status, sdd);
 }
 
-static int wait_for_dma(struct s3c64xx_spi_driver_data *sdd,
-			struct spi_transfer *xfer)
+static int s3c64xx_wait_for_dma(struct s3c64xx_spi_driver_data *sdd,
+				struct spi_transfer *xfer)
 {
 	void __iomem *regs = sdd->regs;
 	unsigned long val;
@@ -485,8 +482,8 @@ static int wait_for_dma(struct s3c64xx_spi_driver_data *sdd,
 	return 0;
 }
 
-static int wait_for_pio(struct s3c64xx_spi_driver_data *sdd,
-			struct spi_transfer *xfer)
+static int s3c64xx_wait_for_pio(struct s3c64xx_spi_driver_data *sdd,
+				struct spi_transfer *xfer)
 {
 	void __iomem *regs = sdd->regs;
 	unsigned long val;
@@ -505,6 +502,8 @@ static int wait_for_pio(struct s3c64xx_spi_driver_data *sdd,
 		status = readl(regs + S3C64XX_SPI_STATUS);
 	} while (RX_FIFO_LVL(status, sdd) < xfer->len && --val);
 
+	if (!val)
+		return -EIO;
 
 	/* If it was only Tx */
 	if (!xfer->rx_buf) {
@@ -635,11 +634,15 @@ static int s3c64xx_spi_transfer_one(struct spi_master *master,
 				    struct spi_transfer *xfer)
 {
 	struct s3c64xx_spi_driver_data *sdd = spi_master_get_devdata(master);
+	const unsigned int fifo_len = (FIFO_LVL_MASK(sdd) >> 1) + 1;
+	const void *tx_buf = NULL;
+	void *rx_buf = NULL;
+	int target_len = 0, origin_len = 0;
+	int use_dma = 0;
 	int status;
 	u32 speed;
 	u8 bpw;
 	unsigned long flags;
-	int use_dma;
 
 	reinit_completion(&sdd->xfer_completion);
 
@@ -654,48 +657,77 @@ static int s3c64xx_spi_transfer_one(struct spi_master *master,
 		s3c64xx_spi_config(sdd);
 	}
 
-	/* Polling method for xfers not bigger than FIFO capacity */
-	use_dma = 0;
-	if (!is_polling(sdd) &&
-	    (sdd->rx_dma.ch && sdd->tx_dma.ch &&
-	     (xfer->len > ((FIFO_LVL_MASK(sdd) >> 1) + 1))))
+	if (!is_polling(sdd) && (xfer->len > fifo_len) &&
+	    sdd->rx_dma.ch && sdd->tx_dma.ch) {
 		use_dma = 1;
 
-	spin_lock_irqsave(&sdd->lock, flags);
+	} else if (is_polling(sdd) && xfer->len > fifo_len) {
+		tx_buf = xfer->tx_buf;
+		rx_buf = xfer->rx_buf;
+		origin_len = xfer->len;
 
-	/* Pending only which is to be done */
-	sdd->state &= ~RXBUSY;
-	sdd->state &= ~TXBUSY;
+		target_len = xfer->len;
+		if (xfer->len > fifo_len)
+			xfer->len = fifo_len;
+	}
 
-	enable_datapath(sdd, spi, xfer, use_dma);
+	do {
+		spin_lock_irqsave(&sdd->lock, flags);
 
-	/* Start the signals */
-	s3c64xx_spi_set_cs(spi, true);
+		/* Pending only which is to be done */
+		sdd->state &= ~RXBUSY;
+		sdd->state &= ~TXBUSY;
 
-	spin_unlock_irqrestore(&sdd->lock, flags);
+		s3c64xx_enable_datapath(sdd, xfer, use_dma);
 
-	if (use_dma)
-		status = wait_for_dma(sdd, xfer);
-	else
-		status = wait_for_pio(sdd, xfer);
-
-	if (status) {
-		dev_err(&spi->dev, "I/O Error: rx-%d tx-%d res:rx-%c tx-%c len-%d\n",
-			xfer->rx_buf ? 1 : 0, xfer->tx_buf ? 1 : 0,
-			(sdd->state & RXBUSY) ? 'f' : 'p',
-			(sdd->state & TXBUSY) ? 'f' : 'p',
-			xfer->len);
-
-		if (use_dma) {
-			if (xfer->tx_buf != NULL
-			    && (sdd->state & TXBUSY))
-				dmaengine_terminate_all(sdd->tx_dma.ch);
-			if (xfer->rx_buf != NULL
-			    && (sdd->state & RXBUSY))
-				dmaengine_terminate_all(sdd->rx_dma.ch);
+		/* Start the signals */
+		s3c64xx_spi_set_cs(spi, true);
+
+		spin_unlock_irqrestore(&sdd->lock, flags);
+
+		if (use_dma)
+			status = s3c64xx_wait_for_dma(sdd, xfer);
+		else
+			status = s3c64xx_wait_for_pio(sdd, xfer);
+
+		if (status) {
+			dev_err(&spi->dev,
+				"I/O Error: rx-%d tx-%d res:rx-%c tx-%c len-%d\n",
+				xfer->rx_buf ? 1 : 0, xfer->tx_buf ? 1 : 0,
+				(sdd->state & RXBUSY) ? 'f' : 'p',
+				(sdd->state & TXBUSY) ? 'f' : 'p',
+				xfer->len);
+
+			if (use_dma) {
+				if (xfer->tx_buf && (sdd->state & TXBUSY))
+					dmaengine_terminate_all(sdd->tx_dma.ch);
+				if (xfer->rx_buf && (sdd->state & RXBUSY))
+					dmaengine_terminate_all(sdd->rx_dma.ch);
+			}
+		} else {
+			s3c64xx_flush_fifo(sdd);
 		}
-	} else {
-		flush_fifo(sdd);
+		if (target_len > 0) {
+			target_len -= xfer->len;
+
+			if (xfer->tx_buf)
+				xfer->tx_buf += xfer->len;
+
+			if (xfer->rx_buf)
+				xfer->rx_buf += xfer->len;
+
+			if (target_len > fifo_len)
+				xfer->len = fifo_len;
+			else
+				xfer->len = target_len;
+		}
+	} while (target_len > 0);
+
+	if (origin_len) {
+		/* Restore original xfer buffers and length */
+		xfer->tx_buf = tx_buf;
+		xfer->rx_buf = rx_buf;
+		xfer->len = origin_len;
 	}
 
 	return status;
@@ -891,7 +923,7 @@ static irqreturn_t s3c64xx_spi_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static void s3c64xx_spi_hwinit(struct s3c64xx_spi_driver_data *sdd, int channel)
+static void s3c64xx_spi_hwinit(struct s3c64xx_spi_driver_data *sdd)
 {
 	struct s3c64xx_spi_info *sci = sdd->cntrlr_info;
 	void __iomem *regs = sdd->regs;
@@ -929,7 +961,7 @@ static void s3c64xx_spi_hwinit(struct s3c64xx_spi_driver_data *sdd, int channel)
 	val |= (S3C64XX_SPI_TRAILCNT << S3C64XX_SPI_TRAILCNT_OFF);
 	writel(val, regs + S3C64XX_SPI_MODE_CFG);
 
-	flush_fifo(sdd);
+	s3c64xx_flush_fifo(sdd);
 }
 
 #ifdef CONFIG_OF
@@ -1145,7 +1177,7 @@ static int s3c64xx_spi_probe(struct platform_device *pdev)
 	pm_runtime_get_sync(&pdev->dev);
 
 	/* Setup Deufult Mode */
-	s3c64xx_spi_hwinit(sdd, sdd->port_id);
+	s3c64xx_spi_hwinit(sdd);
 
 	spin_lock_init(&sdd->lock);
 	init_completion(&sdd->xfer_completion);
@@ -1260,8 +1292,6 @@ static int s3c64xx_spi_resume(struct device *dev)
 	if (ret < 0)
 		return ret;
 
-	s3c64xx_spi_hwinit(sdd, sdd->port_id);
-
 	return spi_master_resume(master);
 }
 #endif /* CONFIG_PM_SLEEP */
@@ -1299,6 +1329,8 @@ static int s3c64xx_spi_runtime_resume(struct device *dev)
 	if (ret != 0)
 		goto err_disable_src_clk;
 
+	s3c64xx_spi_hwinit(sdd);
+
 	return 0;
 
 err_disable_src_clk:
@@ -1344,15 +1376,6 @@ static struct s3c64xx_spi_port_config exynos4_spi_port_config = {
 	.clk_from_cmu	= true,
 };
 
-static struct s3c64xx_spi_port_config exynos5440_spi_port_config = {
-	.fifo_lvl_mask	= { 0x1ff },
-	.rx_lvl_offset	= 15,
-	.tx_st_done	= 25,
-	.high_speed	= true,
-	.clk_from_cmu	= true,
-	.quirks		= S3C64XX_SPI_QUIRK_POLL,
-};
-
 static struct s3c64xx_spi_port_config exynos7_spi_port_config = {
 	.fifo_lvl_mask	= { 0x1ff, 0x7F, 0x7F, 0x7F, 0x7F, 0x1ff},
 	.rx_lvl_offset	= 15,
@@ -1396,9 +1419,6 @@ static const struct of_device_id s3c64xx_spi_dt_match[] = {
 	{ .compatible = "samsung,exynos4210-spi",
 			.data = (void *)&exynos4_spi_port_config,
 	},
-	{ .compatible = "samsung,exynos5440-spi",
-			.data = (void *)&exynos5440_spi_port_config,
-	},
 	{ .compatible = "samsung,exynos7-spi",
 			.data = (void *)&exynos7_spi_port_config,
 	},
diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c
index 8171eedbfc90..0e74cbf9929d 100644
--- a/drivers/spi/spi-sh-msiof.c
+++ b/drivers/spi/spi-sh-msiof.c
@@ -39,7 +39,7 @@ struct sh_msiof_chipdata {
 	u16 tx_fifo_size;
 	u16 rx_fifo_size;
 	u16 master_flags;
-	u16 min_div;
+	u16 min_div_pow;
 };
 
 struct sh_msiof_spi_priv {
@@ -51,7 +51,7 @@ struct sh_msiof_spi_priv {
 	struct completion done;
 	unsigned int tx_fifo_size;
 	unsigned int rx_fifo_size;
-	unsigned int min_div;
+	unsigned int min_div_pow;
 	void *tx_dma_page;
 	void *rx_dma_page;
 	dma_addr_t tx_dma_addr;
@@ -249,43 +249,46 @@ static irqreturn_t sh_msiof_spi_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static struct {
-	unsigned short div;
-	unsigned short brdv;
-} const sh_msiof_spi_div_table[] = {
-	{ 1,	SCR_BRDV_DIV_1 },
-	{ 2,	SCR_BRDV_DIV_2 },
-	{ 4,	SCR_BRDV_DIV_4 },
-	{ 8,	SCR_BRDV_DIV_8 },
-	{ 16,	SCR_BRDV_DIV_16 },
-	{ 32,	SCR_BRDV_DIV_32 },
+static const u32 sh_msiof_spi_div_array[] = {
+	SCR_BRDV_DIV_1, SCR_BRDV_DIV_2,	 SCR_BRDV_DIV_4,
+	SCR_BRDV_DIV_8,	SCR_BRDV_DIV_16, SCR_BRDV_DIV_32,
 };
 
 static void sh_msiof_spi_set_clk_regs(struct sh_msiof_spi_priv *p,
 				      unsigned long parent_rate, u32 spi_hz)
 {
-	unsigned long div = 1024;
+	unsigned long div;
 	u32 brps, scr;
-	size_t k;
+	unsigned int div_pow = p->min_div_pow;
 
-	if (!WARN_ON(!spi_hz || !parent_rate))
-		div = DIV_ROUND_UP(parent_rate, spi_hz);
-
-	div = max_t(unsigned long, div, p->min_div);
+	if (!spi_hz || !parent_rate) {
+		WARN(1, "Invalid clock rate parameters %lu and %u\n",
+		     parent_rate, spi_hz);
+		return;
+	}
 
-	for (k = 0; k < ARRAY_SIZE(sh_msiof_spi_div_table); k++) {
-		brps = DIV_ROUND_UP(div, sh_msiof_spi_div_table[k].div);
+	div = DIV_ROUND_UP(parent_rate, spi_hz);
+	if (div <= 1024) {
 		/* SCR_BRDV_DIV_1 is valid only if BRPS is x 1/1 or x 1/2 */
-		if (sh_msiof_spi_div_table[k].div == 1 && brps > 2)
-			continue;
-		if (brps <= 32) /* max of brdv is 32 */
-			break;
-	}
+		if (!div_pow && div <= 32 && div > 2)
+			div_pow = 1;
+
+		if (div_pow)
+			brps = (div + 1) >> div_pow;
+		else
+			brps = div;
 
-	k = min_t(int, k, ARRAY_SIZE(sh_msiof_spi_div_table) - 1);
-	brps = min_t(int, brps, 32);
+		for (; brps > 32; div_pow++)
+			brps = (brps + 1) >> 1;
+	} else {
+		/* Set transfer rate composite divisor to 2^5 * 32 = 1024 */
+		dev_err(&p->pdev->dev,
+			"Requested SPI transfer rate %d is too low\n", spi_hz);
+		div_pow = 5;
+		brps = 32;
+	}
 
-	scr = sh_msiof_spi_div_table[k].brdv | SCR_BRPS(brps);
+	scr = sh_msiof_spi_div_array[div_pow] | SCR_BRPS(brps);
 	sh_msiof_write(p, TSCR, scr);
 	if (!(p->master->flags & SPI_MASTER_MUST_TX))
 		sh_msiof_write(p, RSCR, scr);
@@ -564,14 +567,16 @@ static int sh_msiof_spi_setup(struct spi_device *spi)
 
 	/* Configure native chip select mode/polarity early */
 	clr = MDR1_SYNCMD_MASK;
-	set = MDR1_TRMD | TMDR1_PCON | MDR1_SYNCMD_SPI;
+	set = MDR1_SYNCMD_SPI;
 	if (spi->mode & SPI_CS_HIGH)
 		clr |= BIT(MDR1_SYNCAC_SHIFT);
 	else
 		set |= BIT(MDR1_SYNCAC_SHIFT);
 	pm_runtime_get_sync(&p->pdev->dev);
 	tmp = sh_msiof_read(p, TMDR1) & ~clr;
-	sh_msiof_write(p, TMDR1, tmp | set);
+	sh_msiof_write(p, TMDR1, tmp | set | MDR1_TRMD | TMDR1_PCON);
+	tmp = sh_msiof_read(p, RMDR1) & ~clr;
+	sh_msiof_write(p, RMDR1, tmp | set);
 	pm_runtime_put(&p->pdev->dev);
 	p->native_cs_high = spi->mode & SPI_CS_HIGH;
 	p->native_cs_inited = true;
@@ -1041,21 +1046,21 @@ static const struct sh_msiof_chipdata sh_data = {
 	.tx_fifo_size = 64,
 	.rx_fifo_size = 64,
 	.master_flags = 0,
-	.min_div = 1,
+	.min_div_pow = 0,
 };
 
 static const struct sh_msiof_chipdata rcar_gen2_data = {
 	.tx_fifo_size = 64,
 	.rx_fifo_size = 64,
 	.master_flags = SPI_MASTER_MUST_TX,
-	.min_div = 1,
+	.min_div_pow = 0,
 };
 
 static const struct sh_msiof_chipdata rcar_gen3_data = {
 	.tx_fifo_size = 64,
 	.rx_fifo_size = 64,
 	.master_flags = SPI_MASTER_MUST_TX,
-	.min_div = 2,
+	.min_div_pow = 1,
 };
 
 static const struct of_device_id sh_msiof_match[] = {
@@ -1319,7 +1324,7 @@ static int sh_msiof_spi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, p);
 	p->master = master;
 	p->info = info;
-	p->min_div = chipdata->min_div;
+	p->min_div_pow = chipdata->min_div_pow;
 
 	init_completion(&p->done);
 
diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c
index ba9743fa2326..ad1e55d3d5d5 100644
--- a/drivers/spi/spi-stm32.c
+++ b/drivers/spi/spi-stm32.c
@@ -1129,7 +1129,7 @@ static int stm32_spi_probe(struct platform_device *pdev)
 	if (!spi->clk_rate) {
 		dev_err(&pdev->dev, "clk rate = 0\n");
 		ret = -EINVAL;
-		goto err_master_put;
+		goto err_clk_disable;
 	}
 
 	spi->rst = devm_reset_control_get_exclusive(&pdev->dev, NULL);
diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c
index c24d9b45a27c..5f19016bbf10 100644
--- a/drivers/spi/spi-ti-qspi.c
+++ b/drivers/spi/spi-ti-qspi.c
@@ -36,6 +36,7 @@
 #include <linux/sizes.h>
 
 #include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
 
 struct ti_qspi_regs {
 	u32 clkctrl;
@@ -50,6 +51,7 @@ struct ti_qspi {
 	struct spi_master	*master;
 	void __iomem            *base;
 	void __iomem            *mmap_base;
+	size_t			mmap_size;
 	struct regmap		*ctrl_base;
 	unsigned int		ctrl_reg;
 	struct clk		*fclk;
@@ -434,12 +436,10 @@ static int ti_qspi_dma_xfer(struct ti_qspi *qspi, dma_addr_t dma_dst,
 	return 0;
 }
 
-static int ti_qspi_dma_bounce_buffer(struct ti_qspi *qspi,
-				     struct spi_flash_read_message *msg)
+static int ti_qspi_dma_bounce_buffer(struct ti_qspi *qspi, loff_t offs,
+				     void *to, size_t readsize)
 {
-	size_t readsize = msg->len;
-	void *to = msg->buf;
-	dma_addr_t dma_src = qspi->mmap_phys_base + msg->from;
+	dma_addr_t dma_src = qspi->mmap_phys_base + offs;
 	int ret = 0;
 
 	/*
@@ -507,13 +507,14 @@ static void ti_qspi_disable_memory_map(struct spi_device *spi)
 	qspi->mmap_enabled = false;
 }
 
-static void ti_qspi_setup_mmap_read(struct spi_device *spi,
-				    struct spi_flash_read_message *msg)
+static void ti_qspi_setup_mmap_read(struct spi_device *spi, u8 opcode,
+				    u8 data_nbits, u8 addr_width,
+				    u8 dummy_bytes)
 {
 	struct ti_qspi  *qspi = spi_master_get_devdata(spi->master);
-	u32 memval = msg->read_opcode;
+	u32 memval = opcode;
 
-	switch (msg->data_nbits) {
+	switch (data_nbits) {
 	case SPI_NBITS_QUAD:
 		memval |= QSPI_SETUP_RD_QUAD;
 		break;
@@ -524,48 +525,64 @@ static void ti_qspi_setup_mmap_read(struct spi_device *spi,
 		memval |= QSPI_SETUP_RD_NORMAL;
 		break;
 	}
-	memval |= ((msg->addr_width - 1) << QSPI_SETUP_ADDR_SHIFT |
-		   msg->dummy_bytes << QSPI_SETUP_DUMMY_SHIFT);
+	memval |= ((addr_width - 1) << QSPI_SETUP_ADDR_SHIFT |
+		   dummy_bytes << QSPI_SETUP_DUMMY_SHIFT);
 	ti_qspi_write(qspi, memval,
 		      QSPI_SPI_SETUP_REG(spi->chip_select));
 }
 
-static bool ti_qspi_spi_flash_can_dma(struct spi_device *spi,
-				      struct spi_flash_read_message *msg)
+static int ti_qspi_exec_mem_op(struct spi_mem *mem,
+			       const struct spi_mem_op *op)
 {
-	return virt_addr_valid(msg->buf);
-}
-
-static int ti_qspi_spi_flash_read(struct spi_device *spi,
-				  struct spi_flash_read_message *msg)
-{
-	struct ti_qspi *qspi = spi_master_get_devdata(spi->master);
+	struct ti_qspi *qspi = spi_master_get_devdata(mem->spi->master);
+	u32 from = 0;
 	int ret = 0;
 
+	/* Only optimize read path. */
+	if (!op->data.nbytes || op->data.dir != SPI_MEM_DATA_IN ||
+	    !op->addr.nbytes || op->addr.nbytes > 4)
+		return -ENOTSUPP;
+
+	/* Address exceeds MMIO window size, fall back to regular mode. */
+	from = op->addr.val;
+	if (from + op->data.nbytes > qspi->mmap_size)
+		return -ENOTSUPP;
+
 	mutex_lock(&qspi->list_lock);
 
 	if (!qspi->mmap_enabled)
-		ti_qspi_enable_memory_map(spi);
-	ti_qspi_setup_mmap_read(spi, msg);
+		ti_qspi_enable_memory_map(mem->spi);
+	ti_qspi_setup_mmap_read(mem->spi, op->cmd.opcode, op->data.buswidth,
+				op->addr.nbytes, op->dummy.nbytes);
 
 	if (qspi->rx_chan) {
-		if (msg->cur_msg_mapped)
-			ret = ti_qspi_dma_xfer_sg(qspi, msg->rx_sg, msg->from);
-		else
-			ret = ti_qspi_dma_bounce_buffer(qspi, msg);
-		if (ret)
-			goto err_unlock;
+		struct sg_table sgt;
+
+		if (virt_addr_valid(op->data.buf.in) &&
+		    !spi_controller_dma_map_mem_op_data(mem->spi->master, op,
+							&sgt)) {
+			ret = ti_qspi_dma_xfer_sg(qspi, sgt, from);
+			spi_controller_dma_unmap_mem_op_data(mem->spi->master,
+							     op, &sgt);
+		} else {
+			ret = ti_qspi_dma_bounce_buffer(qspi, from,
+							op->data.buf.in,
+							op->data.nbytes);
+		}
 	} else {
-		memcpy_fromio(msg->buf, qspi->mmap_base + msg->from, msg->len);
+		memcpy_fromio(op->data.buf.in, qspi->mmap_base + from,
+			      op->data.nbytes);
 	}
-	msg->retlen = msg->len;
 
-err_unlock:
 	mutex_unlock(&qspi->list_lock);
 
 	return ret;
 }
 
+static const struct spi_controller_mem_ops ti_qspi_mem_ops = {
+	.exec_op = ti_qspi_exec_mem_op,
+};
+
 static int ti_qspi_start_transfer_one(struct spi_master *master,
 		struct spi_message *m)
 {
@@ -672,7 +689,7 @@ static int ti_qspi_probe(struct platform_device *pdev)
 	master->dev.of_node = pdev->dev.of_node;
 	master->bits_per_word_mask = SPI_BPW_MASK(32) | SPI_BPW_MASK(16) |
 				     SPI_BPW_MASK(8);
-	master->spi_flash_read = ti_qspi_spi_flash_read;
+	master->mem_ops = &ti_qspi_mem_ops;
 
 	if (!of_property_read_u32(np, "num-cs", &num_cs))
 		master->num_chipselect = num_cs;
@@ -702,6 +719,9 @@ static int ti_qspi_probe(struct platform_device *pdev)
 		}
 	}
 
+	if (res_mmap)
+		qspi->mmap_size = resource_size(res_mmap);
+
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
 		dev_err(&pdev->dev, "no irq resource?\n");
@@ -770,7 +790,6 @@ static int ti_qspi_probe(struct platform_device *pdev)
 		dma_release_channel(qspi->rx_chan);
 		goto no_dma;
 	}
-	master->spi_flash_can_dma = ti_qspi_spi_flash_can_dma;
 	master->dma_rx = qspi->rx_chan;
 	init_completion(&qspi->transfer_complete);
 	if (res_mmap)
@@ -784,7 +803,7 @@ static int ti_qspi_probe(struct platform_device *pdev)
 				 "mmap failed with error %ld using PIO mode\n",
 				 PTR_ERR(qspi->mmap_base));
 			qspi->mmap_base = NULL;
-			master->spi_flash_read = NULL;
+			master->mem_ops = NULL;
 		}
 	}
 	qspi->mmap_enabled = false;
diff --git a/drivers/spi/spi-zynqmp-gqspi.c b/drivers/spi/spi-zynqmp-gqspi.c
index 18aeaceee286..cc4d31033494 100644
--- a/drivers/spi/spi-zynqmp-gqspi.c
+++ b/drivers/spi/spi-zynqmp-gqspi.c
@@ -20,6 +20,7 @@
 #include <linux/of_irq.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/spi/spi.h>
 #include <linux/spinlock.h>
 #include <linux/workqueue.h>
@@ -135,6 +136,7 @@
 #define GQSPI_DMA_UNALIGN		0x3
 #define GQSPI_DEFAULT_NUM_CS	1	/* Default number of chip selects */
 
+#define SPI_AUTOSUSPEND_TIMEOUT		3000
 enum mode_type {GQSPI_MODE_IO, GQSPI_MODE_DMA};
 
 /**
@@ -356,21 +358,9 @@ static void zynqmp_qspi_copy_read_data(struct zynqmp_qspi *xqspi,
 static int zynqmp_prepare_transfer_hardware(struct spi_master *master)
 {
 	struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
-	int ret;
-
-	ret = clk_enable(xqspi->refclk);
-	if (ret)
-		return ret;
-
-	ret = clk_enable(xqspi->pclk);
-	if (ret)
-		goto clk_err;
 
 	zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, GQSPI_EN_MASK);
 	return 0;
-clk_err:
-	clk_disable(xqspi->refclk);
-	return ret;
 }
 
 /**
@@ -387,8 +377,6 @@ static int zynqmp_unprepare_transfer_hardware(struct spi_master *master)
 	struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
 
 	zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, 0x0);
-	clk_disable(xqspi->refclk);
-	clk_disable(xqspi->pclk);
 	return 0;
 }
 
@@ -918,8 +906,7 @@ static int zynqmp_qspi_start_transfer(struct spi_master *master,
  */
 static int __maybe_unused zynqmp_qspi_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct spi_master *master = platform_get_drvdata(pdev);
+	struct spi_master *master = dev_get_drvdata(dev);
 
 	spi_master_suspend(master);
 
@@ -939,8 +926,7 @@ static int __maybe_unused zynqmp_qspi_suspend(struct device *dev)
  */
 static int __maybe_unused zynqmp_qspi_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct spi_master *master = platform_get_drvdata(pdev);
+	struct spi_master *master = dev_get_drvdata(dev);
 	struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
 	int ret = 0;
 
@@ -959,11 +945,67 @@ static int __maybe_unused zynqmp_qspi_resume(struct device *dev)
 
 	spi_master_resume(master);
 
+	clk_disable(xqspi->refclk);
+	clk_disable(xqspi->pclk);
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(zynqmp_qspi_dev_pm_ops, zynqmp_qspi_suspend,
-			 zynqmp_qspi_resume);
+/**
+ * zynqmp_runtime_suspend - Runtime suspend method for the SPI driver
+ * @dev:	Address of the platform_device structure
+ *
+ * This function disables the clocks
+ *
+ * Return:	Always 0
+ */
+static int __maybe_unused zynqmp_runtime_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct spi_master *master = platform_get_drvdata(pdev);
+	struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
+
+	clk_disable(xqspi->refclk);
+	clk_disable(xqspi->pclk);
+
+	return 0;
+}
+
+/**
+ * zynqmp_runtime_resume - Runtime resume method for the SPI driver
+ * @dev:	Address of the platform_device structure
+ *
+ * This function enables the clocks
+ *
+ * Return:	0 on success and error value on error
+ */
+static int __maybe_unused zynqmp_runtime_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct spi_master *master = platform_get_drvdata(pdev);
+	struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
+	int ret;
+
+	ret = clk_enable(xqspi->pclk);
+	if (ret) {
+		dev_err(dev, "Cannot enable APB clock.\n");
+		return ret;
+	}
+
+	ret = clk_enable(xqspi->refclk);
+	if (ret) {
+		dev_err(dev, "Cannot enable device clock.\n");
+		clk_disable(xqspi->pclk);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct dev_pm_ops zynqmp_qspi_dev_pm_ops = {
+	SET_RUNTIME_PM_OPS(zynqmp_runtime_suspend,
+			   zynqmp_runtime_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(zynqmp_qspi_suspend, zynqmp_qspi_resume)
+};
 
 /**
  * zynqmp_qspi_probe:	Probe method for the QSPI driver
@@ -1023,9 +1065,15 @@ static int zynqmp_qspi_probe(struct platform_device *pdev)
 		goto clk_dis_pclk;
 	}
 
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
+	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
 	/* QSPI controller initializations */
 	zynqmp_qspi_init_hw(xqspi);
 
+	pm_runtime_mark_last_busy(&pdev->dev);
+	pm_runtime_put_autosuspend(&pdev->dev);
 	xqspi->irq = platform_get_irq(pdev, 0);
 	if (xqspi->irq <= 0) {
 		ret = -ENXIO;
@@ -1063,6 +1111,8 @@ static int zynqmp_qspi_probe(struct platform_device *pdev)
 	return 0;
 
 clk_dis_all:
+	pm_runtime_set_suspended(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
 	clk_disable_unprepare(xqspi->refclk);
 clk_dis_pclk:
 	clk_disable_unprepare(xqspi->pclk);
@@ -1090,6 +1140,8 @@ static int zynqmp_qspi_remove(struct platform_device *pdev)
 	zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, 0x0);
 	clk_disable_unprepare(xqspi->refclk);
 	clk_disable_unprepare(xqspi->pclk);
+	pm_runtime_set_suspended(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
 
 	spi_unregister_master(master);
 
diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c
index 7b213faa0a2b..efc624f9e490 100644
--- a/drivers/spi/spi.c
+++ b/drivers/spi/spi.c
@@ -28,6 +28,7 @@
 #include <linux/slab.h>
 #include <linux/mod_devicetable.h>
 #include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
 #include <linux/of_gpio.h>
 #include <linux/pm_runtime.h>
 #include <linux/pm_domain.h>
@@ -46,6 +47,8 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/spi.h>
 
+#include "internals.h"
+
 static DEFINE_IDR(spi_master_idr);
 
 static void spidev_release(struct device *dev)
@@ -356,11 +359,12 @@ static int spi_drv_probe(struct device *dev)
 	}
 
 	ret = dev_pm_domain_attach(dev, true);
-	if (ret != -EPROBE_DEFER) {
-		ret = sdrv->probe(spi);
-		if (ret)
-			dev_pm_domain_detach(dev, true);
-	}
+	if (ret)
+		return ret;
+
+	ret = sdrv->probe(spi);
+	if (ret)
+		dev_pm_domain_detach(dev, true);
 
 	return ret;
 }
@@ -740,9 +744,9 @@ static void spi_set_cs(struct spi_device *spi, bool enable)
 }
 
 #ifdef CONFIG_HAS_DMA
-static int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
-		       struct sg_table *sgt, void *buf, size_t len,
-		       enum dma_data_direction dir)
+int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
+		struct sg_table *sgt, void *buf, size_t len,
+		enum dma_data_direction dir)
 {
 	const bool vmalloced_buf = is_vmalloc_addr(buf);
 	unsigned int max_seg_size = dma_get_max_seg_size(dev);
@@ -821,8 +825,8 @@ static int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
 	return 0;
 }
 
-static void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev,
-			  struct sg_table *sgt, enum dma_data_direction dir)
+void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev,
+		   struct sg_table *sgt, enum dma_data_direction dir)
 {
 	if (sgt->orig_nents) {
 		dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir);
@@ -907,19 +911,6 @@ static int __spi_unmap_msg(struct spi_controller *ctlr, struct spi_message *msg)
 	return 0;
 }
 #else /* !CONFIG_HAS_DMA */
-static inline int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
-			      struct sg_table *sgt, void *buf, size_t len,
-			      enum dma_data_direction dir)
-{
-	return -EINVAL;
-}
-
-static inline void spi_unmap_buf(struct spi_controller *ctlr,
-				 struct device *dev, struct sg_table *sgt,
-				 enum dma_data_direction dir)
-{
-}
-
 static inline int __spi_map_msg(struct spi_controller *ctlr,
 				struct spi_message *msg)
 {
@@ -1222,6 +1213,7 @@ static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread)
 	if (!was_busy && ctlr->auto_runtime_pm) {
 		ret = pm_runtime_get_sync(ctlr->dev.parent);
 		if (ret < 0) {
+			pm_runtime_put_noidle(ctlr->dev.parent);
 			dev_err(&ctlr->dev, "Failed to power device: %d\n",
 				ret);
 			mutex_unlock(&ctlr->io_mutex);
@@ -1533,6 +1525,22 @@ static int spi_controller_initialize_queue(struct spi_controller *ctlr)
 	return ret;
 }
 
+/**
+ * spi_flush_queue - Send all pending messages in the queue from the callers'
+ *		     context
+ * @ctlr: controller to process queue for
+ *
+ * This should be used when one wants to ensure all pending messages have been
+ * sent before doing something. Is used by the spi-mem code to make sure SPI
+ * memory operations do not preempt regular SPI transfers that have been queued
+ * before the spi-mem operation.
+ */
+void spi_flush_queue(struct spi_controller *ctlr)
+{
+	if (ctlr->transfer == spi_queued_transfer)
+		__spi_pump_messages(ctlr, false);
+}
+
 /*-------------------------------------------------------------------------*/
 
 #if defined(CONFIG_OF)
@@ -2063,6 +2071,26 @@ static int of_spi_register_master(struct spi_controller *ctlr)
 }
 #endif
 
+static int spi_controller_check_ops(struct spi_controller *ctlr)
+{
+	/*
+	 * The controller may implement only the high-level SPI-memory like
+	 * operations if it does not support regular SPI transfers, and this is
+	 * valid use case.
+	 * If ->mem_ops is NULL, we request that at least one of the
+	 * ->transfer_xxx() method be implemented.
+	 */
+	if (ctlr->mem_ops) {
+		if (!ctlr->mem_ops->exec_op)
+			return -EINVAL;
+	} else if (!ctlr->transfer && !ctlr->transfer_one &&
+		   !ctlr->transfer_one_message) {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 /**
  * spi_register_controller - register SPI master or slave controller
  * @ctlr: initialized master, originally from spi_alloc_master() or
@@ -2096,6 +2124,14 @@ int spi_register_controller(struct spi_controller *ctlr)
 	if (!dev)
 		return -ENODEV;
 
+	/*
+	 * Make sure all necessary hooks are implemented before registering
+	 * the SPI controller.
+	 */
+	status = spi_controller_check_ops(ctlr);
+	if (status)
+		return status;
+
 	if (!spi_controller_is_slave(ctlr)) {
 		status = of_spi_register_master(ctlr);
 		if (status)
@@ -2161,10 +2197,14 @@ int spi_register_controller(struct spi_controller *ctlr)
 			spi_controller_is_slave(ctlr) ? "slave" : "master",
 			dev_name(&ctlr->dev));
 
-	/* If we're using a queued driver, start the queue */
-	if (ctlr->transfer)
+	/*
+	 * If we're using a queued driver, start the queue. Note that we don't
+	 * need the queueing logic if the driver is only supporting high-level
+	 * memory operations.
+	 */
+	if (ctlr->transfer) {
 		dev_info(dev, "controller is unqueued, this is deprecated\n");
-	else {
+	} else if (ctlr->transfer_one || ctlr->transfer_one_message) {
 		status = spi_controller_initialize_queue(ctlr);
 		if (status) {
 			device_del(&ctlr->dev);
@@ -2894,6 +2934,13 @@ static int __spi_async(struct spi_device *spi, struct spi_message *message)
 {
 	struct spi_controller *ctlr = spi->controller;
 
+	/*
+	 * Some controllers do not support doing regular SPI transfers. Return
+	 * ENOTSUPP when this is the case.
+	 */
+	if (!ctlr->transfer)
+		return -ENOTSUPP;
+
 	message->spi = spi;
 
 	SPI_STATISTICS_INCREMENT_FIELD(&ctlr->statistics, spi_async);
@@ -3010,63 +3057,6 @@ int spi_async_locked(struct spi_device *spi, struct spi_message *message)
 }
 EXPORT_SYMBOL_GPL(spi_async_locked);
 
-
-int spi_flash_read(struct spi_device *spi,
-		   struct spi_flash_read_message *msg)
-
-{
-	struct spi_controller *master = spi->controller;
-	struct device *rx_dev = NULL;
-	int ret;
-
-	if ((msg->opcode_nbits == SPI_NBITS_DUAL ||
-	     msg->addr_nbits == SPI_NBITS_DUAL) &&
-	    !(spi->mode & (SPI_TX_DUAL | SPI_TX_QUAD)))
-		return -EINVAL;
-	if ((msg->opcode_nbits == SPI_NBITS_QUAD ||
-	     msg->addr_nbits == SPI_NBITS_QUAD) &&
-	    !(spi->mode & SPI_TX_QUAD))
-		return -EINVAL;
-	if (msg->data_nbits == SPI_NBITS_DUAL &&
-	    !(spi->mode & (SPI_RX_DUAL | SPI_RX_QUAD)))
-		return -EINVAL;
-	if (msg->data_nbits == SPI_NBITS_QUAD &&
-	    !(spi->mode &  SPI_RX_QUAD))
-		return -EINVAL;
-
-	if (master->auto_runtime_pm) {
-		ret = pm_runtime_get_sync(master->dev.parent);
-		if (ret < 0) {
-			dev_err(&master->dev, "Failed to power device: %d\n",
-				ret);
-			return ret;
-		}
-	}
-
-	mutex_lock(&master->bus_lock_mutex);
-	mutex_lock(&master->io_mutex);
-	if (master->dma_rx && master->spi_flash_can_dma(spi, msg)) {
-		rx_dev = master->dma_rx->device->dev;
-		ret = spi_map_buf(master, rx_dev, &msg->rx_sg,
-				  msg->buf, msg->len,
-				  DMA_FROM_DEVICE);
-		if (!ret)
-			msg->cur_msg_mapped = true;
-	}
-	ret = master->spi_flash_read(spi, msg);
-	if (msg->cur_msg_mapped)
-		spi_unmap_buf(master, rx_dev, &msg->rx_sg,
-			      DMA_FROM_DEVICE);
-	mutex_unlock(&master->io_mutex);
-	mutex_unlock(&master->bus_lock_mutex);
-
-	if (master->auto_runtime_pm)
-		pm_runtime_put(master->dev.parent);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(spi_flash_read);
-
 /*-------------------------------------------------------------------------*/
 
 /* Utility methods for SPI protocol drivers, layered on
diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig
index 71a50b99caff..17c5587805f5 100644
--- a/drivers/staging/android/Kconfig
+++ b/drivers/staging/android/Kconfig
@@ -6,7 +6,7 @@ config ASHMEM
 	bool "Enable the Anonymous Shared Memory Subsystem"
 	default n
 	depends on SHMEM
-	---help---
+	help
 	  The ashmem subsystem is a new shared memory allocator, similar to
 	  POSIX SHM but with different behavior and sporting a simpler
 	  file-based API.
@@ -14,6 +14,15 @@ config ASHMEM
 	  It is, in theory, a good memory allocator for low-memory devices,
 	  because it can discard shared memory units when under memory pressure.
 
+config ANDROID_VSOC
+	tristate "Android Virtual SoC support"
+	default n
+	depends on PCI_MSI
+	help
+	  This option adds support for the Virtual SoC driver needed to boot
+	  a 'cuttlefish' Android image inside QEmu. The driver interacts with
+	  a QEmu ivshmem device. If built as a module, it will be called vsoc.
+
 source "drivers/staging/android/ion/Kconfig"
 
 endif # if ANDROID
diff --git a/drivers/staging/android/Makefile b/drivers/staging/android/Makefile
index 7cf1564a49a5..90e6154f11a4 100644
--- a/drivers/staging/android/Makefile
+++ b/drivers/staging/android/Makefile
@@ -3,3 +3,4 @@ ccflags-y += -I$(src)			# needed for trace events
 obj-y					+= ion/
 
 obj-$(CONFIG_ASHMEM)			+= ashmem.o
+obj-$(CONFIG_ANDROID_VSOC)		+= vsoc.o
diff --git a/drivers/staging/android/TODO b/drivers/staging/android/TODO
index 687e0eac85bf..fbf015cc6d62 100644
--- a/drivers/staging/android/TODO
+++ b/drivers/staging/android/TODO
@@ -11,5 +11,14 @@ ion/
  - Split /dev/ion up into multiple nodes (e.g. /dev/ion/heap0)
  - Better test framework (integration with VGEM was suggested)
 
+vsoc.c, uapi/vsoc_shm.h
+ - The current driver uses the same wait queue for all of the futexes in a
+   region. This will cause false wakeups in regions with a large number of
+   waiting threads. We should eventually use multiple queues and select the
+   queue based on the region.
+ - Add debugfs support for examining the permissions of regions.
+ - Remove VSOC_WAIT_FOR_INCOMING_INTERRUPT ioctl. This functionality has been
+   superseded by the futex and is there for legacy reasons.
+
 Please send patches to Greg Kroah-Hartman <greg@kroah.com> and Cc:
 Arve Hjønnevåg <arve@android.com> and Riley Andrews <riandrews@android.com>
diff --git a/drivers/staging/android/ashmem.h b/drivers/staging/android/ashmem.h
index 60d7208f110a..1a478173cd21 100644
--- a/drivers/staging/android/ashmem.h
+++ b/drivers/staging/android/ashmem.h
@@ -1,13 +1,9 @@
-// SPDX-License-Identifier: (GPL-2.0 OR Apache-2.0)
+/* SPDX-License-Identifier: GPL-2.0 OR Apache-2.0 */
 /*
  * include/linux/ashmem.h
  *
  * Copyright 2008 Google Inc.
  * Author: Robert Love
- *
- * This file is dual licensed.  It may be redistributed and/or modified
- * under the terms of the Apache 2.0 License OR version 2 of the GNU
- * General Public License.
  */
 
 #ifndef _LINUX_ASHMEM_H
diff --git a/drivers/staging/android/ion/Kconfig b/drivers/staging/android/ion/Kconfig
index 898e9a834ccc..c16dd16afe6a 100644
--- a/drivers/staging/android/ion/Kconfig
+++ b/drivers/staging/android/ion/Kconfig
@@ -3,7 +3,7 @@ menuconfig ION
 	depends on HAVE_MEMBLOCK && HAS_DMA && MMU
 	select GENERIC_ALLOCATOR
 	select DMA_SHARED_BUFFER
-	---help---
+	help
 	  Choose this option to enable the ION Memory Manager,
 	  used by Android to efficiently allocate buffers
 	  from userspace that can be shared between drivers.
diff --git a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c
index e74db7902549..9d1109e43ed4 100644
--- a/drivers/staging/android/ion/ion.c
+++ b/drivers/staging/android/ion/ion.c
@@ -74,6 +74,8 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
 
 	buffer->heap = heap;
 	buffer->flags = flags;
+	buffer->dev = dev;
+	buffer->size = len;
 
 	ret = heap->ops->allocate(heap, buffer, len, flags);
 
@@ -93,11 +95,6 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
 		goto err1;
 	}
 
-	buffer->dev = dev;
-	buffer->size = len;
-
-	buffer->dev = dev;
-	buffer->size = len;
 	INIT_LIST_HEAD(&buffer->attachments);
 	mutex_init(&buffer->lock);
 	mutex_lock(&dev->buffer_lock);
@@ -114,8 +111,11 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
 
 void ion_buffer_destroy(struct ion_buffer *buffer)
 {
-	if (WARN_ON(buffer->kmap_cnt > 0))
+	if (buffer->kmap_cnt > 0) {
+		pr_warn_once("%s: buffer still mapped in the kernel\n",
+			     __func__);
 		buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
+	}
 	buffer->heap->ops->free(buffer);
 	kfree(buffer);
 }
@@ -318,6 +318,7 @@ static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
 	struct ion_buffer *buffer = dmabuf->priv;
 	void *vaddr;
 	struct ion_dma_buf_attachment *a;
+	int ret = 0;
 
 	/*
 	 * TODO: Move this elsewhere because we don't always need a vaddr
@@ -325,6 +326,10 @@ static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
 	if (buffer->heap->ops->map_kernel) {
 		mutex_lock(&buffer->lock);
 		vaddr = ion_buffer_kmap_get(buffer);
+		if (IS_ERR(vaddr)) {
+			ret = PTR_ERR(vaddr);
+			goto unlock;
+		}
 		mutex_unlock(&buffer->lock);
 	}
 
@@ -333,9 +338,10 @@ static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
 		dma_sync_sg_for_cpu(a->dev, a->table->sgl, a->table->nents,
 				    direction);
 	}
-	mutex_unlock(&buffer->lock);
 
-	return 0;
+unlock:
+	mutex_unlock(&buffer->lock);
+	return ret;
 }
 
 static int ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
diff --git a/drivers/staging/android/ion/ion.h b/drivers/staging/android/ion/ion.h
index ea0897812780..16cbd38a7160 100644
--- a/drivers/staging/android/ion/ion.h
+++ b/drivers/staging/android/ion/ion.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * drivers/staging/android/ion/ion.h
  *
diff --git a/drivers/staging/android/ion/ion_page_pool.c b/drivers/staging/android/ion/ion_page_pool.c
index db8f61446917..9bc56eb48d2a 100644
--- a/drivers/staging/android/ion/ion_page_pool.c
+++ b/drivers/staging/android/ion/ion_page_pool.c
@@ -32,6 +32,9 @@ static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page)
 		list_add_tail(&page->lru, &pool->low_items);
 		pool->low_count++;
 	}
+
+	mod_node_page_state(page_pgdat(page), NR_INDIRECTLY_RECLAIMABLE_BYTES,
+			    (1 << (PAGE_SHIFT + pool->order)));
 	mutex_unlock(&pool->mutex);
 }
 
@@ -50,6 +53,8 @@ static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high)
 	}
 
 	list_del(&page->lru);
+	mod_node_page_state(page_pgdat(page), NR_INDIRECTLY_RECLAIMABLE_BYTES,
+			    -(1 << (PAGE_SHIFT + pool->order)));
 	return page;
 }
 
diff --git a/drivers/staging/android/uapi/ashmem.h b/drivers/staging/android/uapi/ashmem.h
index 5b531af6820e..5442e0019dcd 100644
--- a/drivers/staging/android/uapi/ashmem.h
+++ b/drivers/staging/android/uapi/ashmem.h
@@ -1,13 +1,9 @@
-// SPDX-License-Identifier: (GPL-2.0 OR Apache-2.0)
+/* SPDX-License-Identifier: GPL-2.0 OR Apache-2.0 */
 /*
  * drivers/staging/android/uapi/ashmem.h
  *
  * Copyright 2008 Google Inc.
  * Author: Robert Love
- *
- * This file is dual licensed.  It may be redistributed and/or modified
- * under the terms of the Apache 2.0 License OR version 2 of the GNU
- * General Public License.
  */
 
 #ifndef _UAPI_LINUX_ASHMEM_H
diff --git a/drivers/staging/android/uapi/ion.h b/drivers/staging/android/uapi/ion.h
index 825d3e95ccd3..5d7009884c13 100644
--- a/drivers/staging/android/uapi/ion.h
+++ b/drivers/staging/android/uapi/ion.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * drivers/staging/android/uapi/ion.h
  *
diff --git a/drivers/staging/android/uapi/vsoc_shm.h b/drivers/staging/android/uapi/vsoc_shm.h
new file mode 100644
index 000000000000..6291fb24efb2
--- /dev/null
+++ b/drivers/staging/android/uapi/vsoc_shm.h
@@ -0,0 +1,295 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2017 Google, Inc.
+ *
+ */
+
+#ifndef _UAPI_LINUX_VSOC_SHM_H
+#define _UAPI_LINUX_VSOC_SHM_H
+
+#include <linux/types.h>
+
+/**
+ * A permission is a token that permits a receiver to read and/or write an area
+ * of memory within a Vsoc region.
+ *
+ * An fd_scoped permission grants both read and write access, and can be
+ * attached to a file description (see open(2)).
+ * Ownership of the area can then be shared by passing a file descriptor
+ * among processes.
+ *
+ * begin_offset and end_offset define the area of memory that is controlled by
+ * the permission. owner_offset points to a word, also in shared memory, that
+ * controls ownership of the area.
+ *
+ * ownership of the region expires when the associated file description is
+ * released.
+ *
+ * At most one permission can be attached to each file description.
+ *
+ * This is useful when implementing HALs like gralloc that scope and pass
+ * ownership of shared resources via file descriptors.
+ *
+ * The caller is responsibe for doing any fencing.
+ *
+ * The calling process will normally identify a currently free area of
+ * memory. It will construct a proposed fd_scoped_permission_arg structure:
+ *
+ *   begin_offset and end_offset describe the area being claimed
+ *
+ *   owner_offset points to the location in shared memory that indicates the
+ *   owner of the area.
+ *
+ *   owned_value is the value that will be stored in owner_offset iff the
+ *   permission can be granted. It must be different than VSOC_REGION_FREE.
+ *
+ * Two fd_scoped_permission structures are compatible if they vary only by
+ * their owned_value fields.
+ *
+ * The driver ensures that, for any group of simultaneous callers proposing
+ * compatible fd_scoped_permissions, it will accept exactly one of the
+ * propopsals. The other callers will get a failure with errno of EAGAIN.
+ *
+ * A process receiving a file descriptor can identify the region being
+ * granted using the VSOC_GET_FD_SCOPED_PERMISSION ioctl.
+ */
+struct fd_scoped_permission {
+	__u32 begin_offset;
+	__u32 end_offset;
+	__u32 owner_offset;
+	__u32 owned_value;
+};
+
+/*
+ * This value represents a free area of memory. The driver expects to see this
+ * value at owner_offset when creating a permission otherwise it will not do it,
+ * and will write this value back once the permission is no longer needed.
+ */
+#define VSOC_REGION_FREE ((__u32)0)
+
+/**
+ * ioctl argument for VSOC_CREATE_FD_SCOPE_PERMISSION
+ */
+struct fd_scoped_permission_arg {
+	struct fd_scoped_permission perm;
+	__s32 managed_region_fd;
+};
+
+#define VSOC_NODE_FREE ((__u32)0)
+
+/*
+ * Describes a signal table in shared memory. Each non-zero entry in the
+ * table indicates that the receiver should signal the futex at the given
+ * offset. Offsets are relative to the region, not the shared memory window.
+ *
+ * interrupt_signalled_offset is used to reliably signal interrupts across the
+ * vmm boundary. There are two roles: transmitter and receiver. For example,
+ * in the host_to_guest_signal_table the host is the transmitter and the
+ * guest is the receiver. The protocol is as follows:
+ *
+ * 1. The transmitter should convert the offset of the futex to an offset
+ *    in the signal table [0, (1 << num_nodes_lg2))
+ *    The transmitter can choose any appropriate hashing algorithm, including
+ *    hash = futex_offset & ((1 << num_nodes_lg2) - 1)
+ *
+ * 3. The transmitter should atomically compare and swap futex_offset with 0
+ *    at hash. There are 3 possible outcomes
+ *      a. The swap fails because the futex_offset is already in the table.
+ *         The transmitter should stop.
+ *      b. Some other offset is in the table. This is a hash collision. The
+ *         transmitter should move to another table slot and try again. One
+ *         possible algorithm:
+ *         hash = (hash + 1) & ((1 << num_nodes_lg2) - 1)
+ *      c. The swap worked. Continue below.
+ *
+ * 3. The transmitter atomically swaps 1 with the value at the
+ *    interrupt_signalled_offset. There are two outcomes:
+ *      a. The prior value was 1. In this case an interrupt has already been
+ *         posted. The transmitter is done.
+ *      b. The prior value was 0, indicating that the receiver may be sleeping.
+ *         The transmitter will issue an interrupt.
+ *
+ * 4. On waking the receiver immediately exchanges a 0 with the
+ *    interrupt_signalled_offset. If it receives a 0 then this a spurious
+ *    interrupt. That may occasionally happen in the current protocol, but
+ *    should be rare.
+ *
+ * 5. The receiver scans the signal table by atomicaly exchanging 0 at each
+ *    location. If a non-zero offset is returned from the exchange the
+ *    receiver wakes all sleepers at the given offset:
+ *      futex((int*)(region_base + old_value), FUTEX_WAKE, MAX_INT);
+ *
+ * 6. The receiver thread then does a conditional wait, waking immediately
+ *    if the value at interrupt_signalled_offset is non-zero. This catches cases
+ *    here additional  signals were posted while the table was being scanned.
+ *    On the guest the wait is handled via the VSOC_WAIT_FOR_INCOMING_INTERRUPT
+ *    ioctl.
+ */
+struct vsoc_signal_table_layout {
+	/* log_2(Number of signal table entries) */
+	__u32 num_nodes_lg2;
+	/*
+	 * Offset to the first signal table entry relative to the start of the
+	 * region
+	 */
+	__u32 futex_uaddr_table_offset;
+	/*
+	 * Offset to an atomic_t / atomic uint32_t. A non-zero value indicates
+	 * that one or more offsets are currently posted in the table.
+	 * semi-unique access to an entry in the table
+	 */
+	__u32 interrupt_signalled_offset;
+};
+
+#define VSOC_REGION_WHOLE ((__s32)0)
+#define VSOC_DEVICE_NAME_SZ 16
+
+/**
+ * Each HAL would (usually) talk to a single device region
+ * Mulitple entities care about these regions:
+ * - The ivshmem_server will populate the regions in shared memory
+ * - The guest kernel will read the region, create minor device nodes, and
+ *   allow interested parties to register for FUTEX_WAKE events in the region
+ * - HALs will access via the minor device nodes published by the guest kernel
+ * - Host side processes will access the region via the ivshmem_server:
+ *   1. Pass name to ivshmem_server at a UNIX socket
+ *   2. ivshmemserver will reply with 2 fds:
+ *     - host->guest doorbell fd
+ *     - guest->host doorbell fd
+ *     - fd for the shared memory region
+ *     - region offset
+ *   3. Start a futex receiver thread on the doorbell fd pointed at the
+ *      signal_nodes
+ */
+struct vsoc_device_region {
+	__u16 current_version;
+	__u16 min_compatible_version;
+	__u32 region_begin_offset;
+	__u32 region_end_offset;
+	__u32 offset_of_region_data;
+	struct vsoc_signal_table_layout guest_to_host_signal_table;
+	struct vsoc_signal_table_layout host_to_guest_signal_table;
+	/* Name of the device. Must always be terminated with a '\0', so
+	 * the longest supported device name is 15 characters.
+	 */
+	char device_name[VSOC_DEVICE_NAME_SZ];
+	/* There are two ways that permissions to access regions are handled:
+	 *   - When subdivided_by is VSOC_REGION_WHOLE, any process that can
+	 *     open the device node for the region gains complete access to it.
+	 *   - When subdivided is set processes that open the region cannot
+	 *     access it. Access to a sub-region must be established by invoking
+	 *     the VSOC_CREATE_FD_SCOPE_PERMISSION ioctl on the region
+	 *     referenced in subdivided_by, providing a fileinstance
+	 *     (represented by a fd) opened on this region.
+	 */
+	__u32 managed_by;
+};
+
+/*
+ * The vsoc layout descriptor.
+ * The first 4K should be reserved for the shm header and region descriptors.
+ * The regions should be page aligned.
+ */
+
+struct vsoc_shm_layout_descriptor {
+	__u16 major_version;
+	__u16 minor_version;
+
+	/* size of the shm. This may be redundant but nice to have */
+	__u32 size;
+
+	/* number of shared memory regions */
+	__u32 region_count;
+
+	/* The offset to the start of region descriptors */
+	__u32 vsoc_region_desc_offset;
+};
+
+/*
+ * This specifies the current version that should be stored in
+ * vsoc_shm_layout_descriptor.major_version and
+ * vsoc_shm_layout_descriptor.minor_version.
+ * It should be updated only if the vsoc_device_region and
+ * vsoc_shm_layout_descriptor structures have changed.
+ * Versioning within each region is transferred
+ * via the min_compatible_version and current_version fields in
+ * vsoc_device_region. The driver does not consult these fields: they are left
+ * for the HALs and host processes and will change independently of the layout
+ * version.
+ */
+#define CURRENT_VSOC_LAYOUT_MAJOR_VERSION 2
+#define CURRENT_VSOC_LAYOUT_MINOR_VERSION 0
+
+#define VSOC_CREATE_FD_SCOPED_PERMISSION \
+	_IOW(0xF5, 0, struct fd_scoped_permission)
+#define VSOC_GET_FD_SCOPED_PERMISSION _IOR(0xF5, 1, struct fd_scoped_permission)
+
+/*
+ * This is used to signal the host to scan the guest_to_host_signal_table
+ * for new futexes to wake. This sends an interrupt if one is not already
+ * in flight.
+ */
+#define VSOC_MAYBE_SEND_INTERRUPT_TO_HOST _IO(0xF5, 2)
+
+/*
+ * When this returns the guest will scan host_to_guest_signal_table to
+ * check for new futexes to wake.
+ */
+/* TODO(ghartman): Consider moving this to the bottom half */
+#define VSOC_WAIT_FOR_INCOMING_INTERRUPT _IO(0xF5, 3)
+
+/*
+ * Guest HALs will use this to retrieve the region description after
+ * opening their device node.
+ */
+#define VSOC_DESCRIBE_REGION _IOR(0xF5, 4, struct vsoc_device_region)
+
+/*
+ * Wake any threads that may be waiting for a host interrupt on this region.
+ * This is mostly used during shutdown.
+ */
+#define VSOC_SELF_INTERRUPT _IO(0xF5, 5)
+
+/*
+ * This is used to signal the host to scan the guest_to_host_signal_table
+ * for new futexes to wake. This sends an interrupt unconditionally.
+ */
+#define VSOC_SEND_INTERRUPT_TO_HOST _IO(0xF5, 6)
+
+enum wait_types {
+	VSOC_WAIT_UNDEFINED = 0,
+	VSOC_WAIT_IF_EQUAL = 1,
+	VSOC_WAIT_IF_EQUAL_TIMEOUT = 2
+};
+
+/*
+ * Wait for a condition to be true
+ *
+ * Note, this is sized and aligned so the 32 bit and 64 bit layouts are
+ * identical.
+ */
+struct vsoc_cond_wait {
+	/* Input: Offset of the 32 bit word to check */
+	__u32 offset;
+	/* Input: Value that will be compared with the offset */
+	__u32 value;
+	/* Monotonic time to wake at in seconds */
+	__u64 wake_time_sec;
+	/* Input: Monotonic time to wait in nanoseconds */
+	__u32 wake_time_nsec;
+	/* Input: Type of wait */
+	__u32 wait_type;
+	/* Output: Number of times the thread woke before returning. */
+	__u32 wakes;
+	/* Ensure that we're 8-byte aligned and 8 byte length for 32/64 bit
+	 * compatibility.
+	 */
+	__u32 reserved_1;
+};
+
+#define VSOC_COND_WAIT _IOWR(0xF5, 7, struct vsoc_cond_wait)
+
+/* Wake any local threads waiting at the offset given in arg */
+#define VSOC_COND_WAKE _IO(0xF5, 8)
+
+#endif /* _UAPI_LINUX_VSOC_SHM_H */
diff --git a/drivers/staging/android/vsoc.c b/drivers/staging/android/vsoc.c
new file mode 100644
index 000000000000..806beda1040b
--- /dev/null
+++ b/drivers/staging/android/vsoc.c
@@ -0,0 +1,1152 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * drivers/android/staging/vsoc.c
+ *
+ * Android Virtual System on a Chip (VSoC) driver
+ *
+ * Copyright (C) 2017 Google, Inc.
+ *
+ * Author: ghartman@google.com
+ *
+ * Based on drivers/char/kvm_ivshmem.c - driver for KVM Inter-VM shared memory
+ *         Copyright 2009 Cam Macdonell <cam@cs.ualberta.ca>
+ *
+ * Based on cirrusfb.c and 8139cp.c:
+ *   Copyright 1999-2001 Jeff Garzik
+ *   Copyright 2001-2004 Jeff Garzik
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/freezer.h>
+#include <linux/futex.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/proc_fs.h>
+#include <linux/sched.h>
+#include <linux/syscalls.h>
+#include <linux/uaccess.h>
+#include <linux/interrupt.h>
+#include <linux/mutex.h>
+#include <linux/cdev.h>
+#include <linux/file.h>
+#include "uapi/vsoc_shm.h"
+
+#define VSOC_DEV_NAME "vsoc"
+
+/*
+ * Description of the ivshmem-doorbell PCI device used by QEmu. These
+ * constants follow docs/specs/ivshmem-spec.txt, which can be found in
+ * the QEmu repository. This was last reconciled with the version that
+ * came out with 2.8
+ */
+
+/*
+ * These constants are determined KVM Inter-VM shared memory device
+ * register offsets
+ */
+enum {
+	INTR_MASK = 0x00,	/* Interrupt Mask */
+	INTR_STATUS = 0x04,	/* Interrupt Status */
+	IV_POSITION = 0x08,	/* VM ID */
+	DOORBELL = 0x0c,	/* Doorbell */
+};
+
+static const int REGISTER_BAR;  /* Equal to 0 */
+static const int MAX_REGISTER_BAR_LEN = 0x100;
+/*
+ * The MSI-x BAR is not used directly.
+ *
+ * static const int MSI_X_BAR = 1;
+ */
+static const int SHARED_MEMORY_BAR = 2;
+
+struct vsoc_region_data {
+	char name[VSOC_DEVICE_NAME_SZ + 1];
+	wait_queue_head_t interrupt_wait_queue;
+	/* TODO(b/73664181): Use multiple futex wait queues */
+	wait_queue_head_t futex_wait_queue;
+	/* Flag indicating that an interrupt has been signalled by the host. */
+	atomic_t *incoming_signalled;
+	/* Flag indicating the guest has signalled the host. */
+	atomic_t *outgoing_signalled;
+	bool irq_requested;
+	bool device_created;
+};
+
+struct vsoc_device {
+	/* Kernel virtual address of REGISTER_BAR. */
+	void __iomem *regs;
+	/* Physical address of SHARED_MEMORY_BAR. */
+	phys_addr_t shm_phys_start;
+	/* Kernel virtual address of SHARED_MEMORY_BAR. */
+	void __iomem *kernel_mapped_shm;
+	/* Size of the entire shared memory window in bytes. */
+	size_t shm_size;
+	/*
+	 * Pointer to the virtual address of the shared memory layout structure.
+	 * This is probably identical to kernel_mapped_shm, but saving this
+	 * here saves a lot of annoying casts.
+	 */
+	struct vsoc_shm_layout_descriptor *layout;
+	/*
+	 * Points to a table of region descriptors in the kernel's virtual
+	 * address space. Calculated from
+	 * vsoc_shm_layout_descriptor.vsoc_region_desc_offset
+	 */
+	struct vsoc_device_region *regions;
+	/* Head of a list of permissions that have been granted. */
+	struct list_head permissions;
+	struct pci_dev *dev;
+	/* Per-region (and therefore per-interrupt) information. */
+	struct vsoc_region_data *regions_data;
+	/*
+	 * Table of msi-x entries. This has to be separated from struct
+	 * vsoc_region_data because the kernel deals with them as an array.
+	 */
+	struct msix_entry *msix_entries;
+	/* Mutex that protectes the permission list */
+	struct mutex mtx;
+	/* Major number assigned by the kernel */
+	int major;
+	/* Character device assigned by the kernel */
+	struct cdev cdev;
+	/* Device class assigned by the kernel */
+	struct class *class;
+	/*
+	 * Flags that indicate what we've initialized. These are used to do an
+	 * orderly cleanup of the device.
+	 */
+	bool enabled_device;
+	bool requested_regions;
+	bool cdev_added;
+	bool class_added;
+	bool msix_enabled;
+};
+
+static struct vsoc_device vsoc_dev;
+
+/*
+ * TODO(ghartman): Add a /sys filesystem entry that summarizes the permissions.
+ */
+
+struct fd_scoped_permission_node {
+	struct fd_scoped_permission permission;
+	struct list_head list;
+};
+
+struct vsoc_private_data {
+	struct fd_scoped_permission_node *fd_scoped_permission_node;
+};
+
+static long vsoc_ioctl(struct file *, unsigned int, unsigned long);
+static int vsoc_mmap(struct file *, struct vm_area_struct *);
+static int vsoc_open(struct inode *, struct file *);
+static int vsoc_release(struct inode *, struct file *);
+static ssize_t vsoc_read(struct file *, char __user *, size_t, loff_t *);
+static ssize_t vsoc_write(struct file *, const char __user *, size_t, loff_t *);
+static loff_t vsoc_lseek(struct file *filp, loff_t offset, int origin);
+static int
+do_create_fd_scoped_permission(struct vsoc_device_region *region_p,
+			       struct fd_scoped_permission_node *np,
+			       struct fd_scoped_permission_arg __user *arg);
+static void
+do_destroy_fd_scoped_permission(struct vsoc_device_region *owner_region_p,
+				struct fd_scoped_permission *perm);
+static long do_vsoc_describe_region(struct file *,
+				    struct vsoc_device_region __user *);
+static ssize_t vsoc_get_area(struct file *filp, __u32 *perm_off);
+
+/**
+ * Validate arguments on entry points to the driver.
+ */
+inline int vsoc_validate_inode(struct inode *inode)
+{
+	if (iminor(inode) >= vsoc_dev.layout->region_count) {
+		dev_err(&vsoc_dev.dev->dev,
+			"describe_region: invalid region %d\n", iminor(inode));
+		return -ENODEV;
+	}
+	return 0;
+}
+
+inline int vsoc_validate_filep(struct file *filp)
+{
+	int ret = vsoc_validate_inode(file_inode(filp));
+
+	if (ret)
+		return ret;
+	if (!filp->private_data) {
+		dev_err(&vsoc_dev.dev->dev,
+			"No private data on fd, region %d\n",
+			iminor(file_inode(filp)));
+		return -EBADFD;
+	}
+	return 0;
+}
+
+/* Converts from shared memory offset to virtual address */
+static inline void *shm_off_to_virtual_addr(__u32 offset)
+{
+	return (void __force *)vsoc_dev.kernel_mapped_shm + offset;
+}
+
+/* Converts from shared memory offset to physical address */
+static inline phys_addr_t shm_off_to_phys_addr(__u32 offset)
+{
+	return vsoc_dev.shm_phys_start + offset;
+}
+
+/**
+ * Convenience functions to obtain the region from the inode or file.
+ * Dangerous to call before validating the inode/file.
+ */
+static
+inline struct vsoc_device_region *vsoc_region_from_inode(struct inode *inode)
+{
+	return &vsoc_dev.regions[iminor(inode)];
+}
+
+static
+inline struct vsoc_device_region *vsoc_region_from_filep(struct file *inode)
+{
+	return vsoc_region_from_inode(file_inode(inode));
+}
+
+static inline uint32_t vsoc_device_region_size(struct vsoc_device_region *r)
+{
+	return r->region_end_offset - r->region_begin_offset;
+}
+
+static const struct file_operations vsoc_ops = {
+	.owner = THIS_MODULE,
+	.open = vsoc_open,
+	.mmap = vsoc_mmap,
+	.read = vsoc_read,
+	.unlocked_ioctl = vsoc_ioctl,
+	.compat_ioctl = vsoc_ioctl,
+	.write = vsoc_write,
+	.llseek = vsoc_lseek,
+	.release = vsoc_release,
+};
+
+static struct pci_device_id vsoc_id_table[] = {
+	{0x1af4, 0x1110, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
+	{0},
+};
+
+MODULE_DEVICE_TABLE(pci, vsoc_id_table);
+
+static void vsoc_remove_device(struct pci_dev *pdev);
+static int vsoc_probe_device(struct pci_dev *pdev,
+			     const struct pci_device_id *ent);
+
+static struct pci_driver vsoc_pci_driver = {
+	.name = "vsoc",
+	.id_table = vsoc_id_table,
+	.probe = vsoc_probe_device,
+	.remove = vsoc_remove_device,
+};
+
+static int
+do_create_fd_scoped_permission(struct vsoc_device_region *region_p,
+			       struct fd_scoped_permission_node *np,
+			       struct fd_scoped_permission_arg __user *arg)
+{
+	struct file *managed_filp;
+	s32 managed_fd;
+	atomic_t *owner_ptr = NULL;
+	struct vsoc_device_region *managed_region_p;
+
+	if (copy_from_user(&np->permission, &arg->perm, sizeof(*np)) ||
+	    copy_from_user(&managed_fd,
+			   &arg->managed_region_fd, sizeof(managed_fd))) {
+		return -EFAULT;
+	}
+	managed_filp = fdget(managed_fd).file;
+	/* Check that it's a valid fd, */
+	if (!managed_filp || vsoc_validate_filep(managed_filp))
+		return -EPERM;
+	/* EEXIST if the given fd already has a permission. */
+	if (((struct vsoc_private_data *)managed_filp->private_data)->
+	    fd_scoped_permission_node)
+		return -EEXIST;
+	managed_region_p = vsoc_region_from_filep(managed_filp);
+	/* Check that the provided region is managed by this one */
+	if (&vsoc_dev.regions[managed_region_p->managed_by] != region_p)
+		return -EPERM;
+	/* The area must be well formed and have non-zero size */
+	if (np->permission.begin_offset >= np->permission.end_offset)
+		return -EINVAL;
+	/* The area must fit in the memory window */
+	if (np->permission.end_offset >
+	    vsoc_device_region_size(managed_region_p))
+		return -ERANGE;
+	/* The area must be in the region data section */
+	if (np->permission.begin_offset <
+	    managed_region_p->offset_of_region_data)
+		return -ERANGE;
+	/* The area must be page aligned */
+	if (!PAGE_ALIGNED(np->permission.begin_offset) ||
+	    !PAGE_ALIGNED(np->permission.end_offset))
+		return -EINVAL;
+	/* Owner offset must be naturally aligned in the window */
+	if (np->permission.owner_offset &
+	    (sizeof(np->permission.owner_offset) - 1))
+		return -EINVAL;
+	/* The owner flag must reside in the owner memory */
+	if (np->permission.owner_offset + sizeof(np->permission.owner_offset) >
+	    vsoc_device_region_size(region_p))
+		return -ERANGE;
+	/* The owner flag must reside in the data section */
+	if (np->permission.owner_offset < region_p->offset_of_region_data)
+		return -EINVAL;
+	/* The owner value must change to claim the memory */
+	if (np->permission.owned_value == VSOC_REGION_FREE)
+		return -EINVAL;
+	owner_ptr =
+	    (atomic_t *)shm_off_to_virtual_addr(region_p->region_begin_offset +
+						np->permission.owner_offset);
+	/* We've already verified that this is in the shared memory window, so
+	 * it should be safe to write to this address.
+	 */
+	if (atomic_cmpxchg(owner_ptr,
+			   VSOC_REGION_FREE,
+			   np->permission.owned_value) != VSOC_REGION_FREE) {
+		return -EBUSY;
+	}
+	((struct vsoc_private_data *)managed_filp->private_data)->
+	    fd_scoped_permission_node = np;
+	/* The file offset needs to be adjusted if the calling
+	 * process did any read/write operations on the fd
+	 * before creating the permission.
+	 */
+	if (managed_filp->f_pos) {
+		if (managed_filp->f_pos > np->permission.end_offset) {
+			/* If the offset is beyond the permission end, set it
+			 * to the end.
+			 */
+			managed_filp->f_pos = np->permission.end_offset;
+		} else {
+			/* If the offset is within the permission interval
+			 * keep it there otherwise reset it to zero.
+			 */
+			if (managed_filp->f_pos < np->permission.begin_offset) {
+				managed_filp->f_pos = 0;
+			} else {
+				managed_filp->f_pos -=
+				    np->permission.begin_offset;
+			}
+		}
+	}
+	return 0;
+}
+
+static void
+do_destroy_fd_scoped_permission_node(struct vsoc_device_region *owner_region_p,
+				     struct fd_scoped_permission_node *node)
+{
+	if (node) {
+		do_destroy_fd_scoped_permission(owner_region_p,
+						&node->permission);
+		mutex_lock(&vsoc_dev.mtx);
+		list_del(&node->list);
+		mutex_unlock(&vsoc_dev.mtx);
+		kfree(node);
+	}
+}
+
+static void
+do_destroy_fd_scoped_permission(struct vsoc_device_region *owner_region_p,
+				struct fd_scoped_permission *perm)
+{
+	atomic_t *owner_ptr = NULL;
+	int prev = 0;
+
+	if (!perm)
+		return;
+	owner_ptr = (atomic_t *)shm_off_to_virtual_addr
+		(owner_region_p->region_begin_offset + perm->owner_offset);
+	prev = atomic_xchg(owner_ptr, VSOC_REGION_FREE);
+	if (prev != perm->owned_value)
+		dev_err(&vsoc_dev.dev->dev,
+			"%x-%x: owner (%s) %x: expected to be %x was %x",
+			perm->begin_offset, perm->end_offset,
+			owner_region_p->device_name, perm->owner_offset,
+			perm->owned_value, prev);
+}
+
+static long do_vsoc_describe_region(struct file *filp,
+				    struct vsoc_device_region __user *dest)
+{
+	struct vsoc_device_region *region_p;
+	int retval = vsoc_validate_filep(filp);
+
+	if (retval)
+		return retval;
+	region_p = vsoc_region_from_filep(filp);
+	if (copy_to_user(dest, region_p, sizeof(*region_p)))
+		return -EFAULT;
+	return 0;
+}
+
+/**
+ * Implements the inner logic of cond_wait. Copies to and from userspace are
+ * done in the helper function below.
+ */
+static int handle_vsoc_cond_wait(struct file *filp, struct vsoc_cond_wait *arg)
+{
+	DEFINE_WAIT(wait);
+	u32 region_number = iminor(file_inode(filp));
+	struct vsoc_region_data *data = vsoc_dev.regions_data + region_number;
+	struct hrtimer_sleeper timeout, *to = NULL;
+	int ret = 0;
+	struct vsoc_device_region *region_p = vsoc_region_from_filep(filp);
+	atomic_t *address = NULL;
+	struct timespec ts;
+
+	/* Ensure that the offset is aligned */
+	if (arg->offset & (sizeof(uint32_t) - 1))
+		return -EADDRNOTAVAIL;
+	/* Ensure that the offset is within shared memory */
+	if (((uint64_t)arg->offset) + region_p->region_begin_offset +
+	    sizeof(uint32_t) > region_p->region_end_offset)
+		return -E2BIG;
+	address = shm_off_to_virtual_addr(region_p->region_begin_offset +
+					  arg->offset);
+
+	/* Ensure that the type of wait is valid */
+	switch (arg->wait_type) {
+	case VSOC_WAIT_IF_EQUAL:
+		break;
+	case VSOC_WAIT_IF_EQUAL_TIMEOUT:
+		to = &timeout;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (to) {
+		/* Copy the user-supplied timesec into the kernel structure.
+		 * We do things this way to flatten differences between 32 bit
+		 * and 64 bit timespecs.
+		 */
+		ts.tv_sec = arg->wake_time_sec;
+		ts.tv_nsec = arg->wake_time_nsec;
+
+		if (!timespec_valid(&ts))
+			return -EINVAL;
+		hrtimer_init_on_stack(&to->timer, CLOCK_MONOTONIC,
+				      HRTIMER_MODE_ABS);
+		hrtimer_set_expires_range_ns(&to->timer, timespec_to_ktime(ts),
+					     current->timer_slack_ns);
+
+		hrtimer_init_sleeper(to, current);
+	}
+
+	while (1) {
+		prepare_to_wait(&data->futex_wait_queue, &wait,
+				TASK_INTERRUPTIBLE);
+		/*
+		 * Check the sentinel value after prepare_to_wait. If the value
+		 * changes after this check the writer will call signal,
+		 * changing the task state from INTERRUPTIBLE to RUNNING. That
+		 * will ensure that schedule() will eventually schedule this
+		 * task.
+		 */
+		if (atomic_read(address) != arg->value) {
+			ret = 0;
+			break;
+		}
+		if (to) {
+			hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
+			if (likely(to->task))
+				freezable_schedule();
+			hrtimer_cancel(&to->timer);
+			if (!to->task) {
+				ret = -ETIMEDOUT;
+				break;
+			}
+		} else {
+			freezable_schedule();
+		}
+		/* Count the number of times that we woke up. This is useful
+		 * for unit testing.
+		 */
+		++arg->wakes;
+		if (signal_pending(current)) {
+			ret = -EINTR;
+			break;
+		}
+	}
+	finish_wait(&data->futex_wait_queue, &wait);
+	if (to)
+		destroy_hrtimer_on_stack(&to->timer);
+	return ret;
+}
+
+/**
+ * Handles the details of copying from/to userspace to ensure that the copies
+ * happen on all of the return paths of cond_wait.
+ */
+static int do_vsoc_cond_wait(struct file *filp,
+			     struct vsoc_cond_wait __user *untrusted_in)
+{
+	struct vsoc_cond_wait arg;
+	int rval = 0;
+
+	if (copy_from_user(&arg, untrusted_in, sizeof(arg)))
+		return -EFAULT;
+	/* wakes is an out parameter. Initialize it to something sensible. */
+	arg.wakes = 0;
+	rval = handle_vsoc_cond_wait(filp, &arg);
+	if (copy_to_user(untrusted_in, &arg, sizeof(arg)))
+		return -EFAULT;
+	return rval;
+}
+
+static int do_vsoc_cond_wake(struct file *filp, uint32_t offset)
+{
+	struct vsoc_device_region *region_p = vsoc_region_from_filep(filp);
+	u32 region_number = iminor(file_inode(filp));
+	struct vsoc_region_data *data = vsoc_dev.regions_data + region_number;
+	/* Ensure that the offset is aligned */
+	if (offset & (sizeof(uint32_t) - 1))
+		return -EADDRNOTAVAIL;
+	/* Ensure that the offset is within shared memory */
+	if (((uint64_t)offset) + region_p->region_begin_offset +
+	    sizeof(uint32_t) > region_p->region_end_offset)
+		return -E2BIG;
+	/*
+	 * TODO(b/73664181): Use multiple futex wait queues.
+	 * We need to wake every sleeper when the condition changes. Typically
+	 * only a single thread will be waiting on the condition, but there
+	 * are exceptions. The worst case is about 10 threads.
+	 */
+	wake_up_interruptible_all(&data->futex_wait_queue);
+	return 0;
+}
+
+static long vsoc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+	int rv = 0;
+	struct vsoc_device_region *region_p;
+	u32 reg_num;
+	struct vsoc_region_data *reg_data;
+	int retval = vsoc_validate_filep(filp);
+
+	if (retval)
+		return retval;
+	region_p = vsoc_region_from_filep(filp);
+	reg_num = iminor(file_inode(filp));
+	reg_data = vsoc_dev.regions_data + reg_num;
+	switch (cmd) {
+	case VSOC_CREATE_FD_SCOPED_PERMISSION:
+		{
+			struct fd_scoped_permission_node *node = NULL;
+
+			node = kzalloc(sizeof(*node), GFP_KERNEL);
+			/* We can't allocate memory for the permission */
+			if (!node)
+				return -ENOMEM;
+			INIT_LIST_HEAD(&node->list);
+			rv = do_create_fd_scoped_permission
+				(region_p,
+				 node,
+				 (struct fd_scoped_permission_arg __user *)arg);
+			if (!rv) {
+				mutex_lock(&vsoc_dev.mtx);
+				list_add(&node->list, &vsoc_dev.permissions);
+				mutex_unlock(&vsoc_dev.mtx);
+			} else {
+				kfree(node);
+				return rv;
+			}
+		}
+		break;
+
+	case VSOC_GET_FD_SCOPED_PERMISSION:
+		{
+			struct fd_scoped_permission_node *node =
+			    ((struct vsoc_private_data *)filp->private_data)->
+			    fd_scoped_permission_node;
+			if (!node)
+				return -ENOENT;
+			if (copy_to_user
+			    ((struct fd_scoped_permission __user *)arg,
+			     &node->permission, sizeof(node->permission)))
+				return -EFAULT;
+		}
+		break;
+
+	case VSOC_MAYBE_SEND_INTERRUPT_TO_HOST:
+		if (!atomic_xchg(reg_data->outgoing_signalled, 1)) {
+			writel(reg_num, vsoc_dev.regs + DOORBELL);
+			return 0;
+		} else {
+			return -EBUSY;
+		}
+		break;
+
+	case VSOC_SEND_INTERRUPT_TO_HOST:
+		writel(reg_num, vsoc_dev.regs + DOORBELL);
+		return 0;
+	case VSOC_WAIT_FOR_INCOMING_INTERRUPT:
+		wait_event_interruptible
+			(reg_data->interrupt_wait_queue,
+			 (atomic_read(reg_data->incoming_signalled) != 0));
+		break;
+
+	case VSOC_DESCRIBE_REGION:
+		return do_vsoc_describe_region
+			(filp,
+			 (struct vsoc_device_region __user *)arg);
+
+	case VSOC_SELF_INTERRUPT:
+		atomic_set(reg_data->incoming_signalled, 1);
+		wake_up_interruptible(&reg_data->interrupt_wait_queue);
+		break;
+
+	case VSOC_COND_WAIT:
+		return do_vsoc_cond_wait(filp,
+					 (struct vsoc_cond_wait __user *)arg);
+	case VSOC_COND_WAKE:
+		return do_vsoc_cond_wake(filp, arg);
+
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static ssize_t vsoc_read(struct file *filp, char __user *buffer, size_t len,
+			 loff_t *poffset)
+{
+	__u32 area_off;
+	const void *area_p;
+	ssize_t area_len;
+	int retval = vsoc_validate_filep(filp);
+
+	if (retval)
+		return retval;
+	area_len = vsoc_get_area(filp, &area_off);
+	area_p = shm_off_to_virtual_addr(area_off);
+	area_p += *poffset;
+	area_len -= *poffset;
+	if (area_len <= 0)
+		return 0;
+	if (area_len < len)
+		len = area_len;
+	if (copy_to_user(buffer, area_p, len))
+		return -EFAULT;
+	*poffset += len;
+	return len;
+}
+
+static loff_t vsoc_lseek(struct file *filp, loff_t offset, int origin)
+{
+	ssize_t area_len = 0;
+	int retval = vsoc_validate_filep(filp);
+
+	if (retval)
+		return retval;
+	area_len = vsoc_get_area(filp, NULL);
+	switch (origin) {
+	case SEEK_SET:
+		break;
+
+	case SEEK_CUR:
+		if (offset > 0 && offset + filp->f_pos < 0)
+			return -EOVERFLOW;
+		offset += filp->f_pos;
+		break;
+
+	case SEEK_END:
+		if (offset > 0 && offset + area_len < 0)
+			return -EOVERFLOW;
+		offset += area_len;
+		break;
+
+	case SEEK_DATA:
+		if (offset >= area_len)
+			return -EINVAL;
+		if (offset < 0)
+			offset = 0;
+		break;
+
+	case SEEK_HOLE:
+		/* Next hole is always the end of the region, unless offset is
+		 * beyond that
+		 */
+		if (offset < area_len)
+			offset = area_len;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	if (offset < 0 || offset > area_len)
+		return -EINVAL;
+	filp->f_pos = offset;
+
+	return offset;
+}
+
+static ssize_t vsoc_write(struct file *filp, const char __user *buffer,
+			  size_t len, loff_t *poffset)
+{
+	__u32 area_off;
+	void *area_p;
+	ssize_t area_len;
+	int retval = vsoc_validate_filep(filp);
+
+	if (retval)
+		return retval;
+	area_len = vsoc_get_area(filp, &area_off);
+	area_p = shm_off_to_virtual_addr(area_off);
+	area_p += *poffset;
+	area_len -= *poffset;
+	if (area_len <= 0)
+		return 0;
+	if (area_len < len)
+		len = area_len;
+	if (copy_from_user(area_p, buffer, len))
+		return -EFAULT;
+	*poffset += len;
+	return len;
+}
+
+static irqreturn_t vsoc_interrupt(int irq, void *region_data_v)
+{
+	struct vsoc_region_data *region_data =
+	    (struct vsoc_region_data *)region_data_v;
+	int reg_num = region_data - vsoc_dev.regions_data;
+
+	if (unlikely(!region_data))
+		return IRQ_NONE;
+
+	if (unlikely(reg_num < 0 ||
+		     reg_num >= vsoc_dev.layout->region_count)) {
+		dev_err(&vsoc_dev.dev->dev,
+			"invalid irq @%p reg_num=0x%04x\n",
+			region_data, reg_num);
+		return IRQ_NONE;
+	}
+	if (unlikely(vsoc_dev.regions_data + reg_num != region_data)) {
+		dev_err(&vsoc_dev.dev->dev,
+			"irq not aligned @%p reg_num=0x%04x\n",
+			region_data, reg_num);
+		return IRQ_NONE;
+	}
+	wake_up_interruptible(&region_data->interrupt_wait_queue);
+	return IRQ_HANDLED;
+}
+
+static int vsoc_probe_device(struct pci_dev *pdev,
+			     const struct pci_device_id *ent)
+{
+	int result;
+	int i;
+	resource_size_t reg_size;
+	dev_t devt;
+
+	vsoc_dev.dev = pdev;
+	result = pci_enable_device(pdev);
+	if (result) {
+		dev_err(&pdev->dev,
+			"pci_enable_device failed %s: error %d\n",
+			pci_name(pdev), result);
+		return result;
+	}
+	vsoc_dev.enabled_device = true;
+	result = pci_request_regions(pdev, "vsoc");
+	if (result < 0) {
+		dev_err(&pdev->dev, "pci_request_regions failed\n");
+		vsoc_remove_device(pdev);
+		return -EBUSY;
+	}
+	vsoc_dev.requested_regions = true;
+	/* Set up the control registers in BAR 0 */
+	reg_size = pci_resource_len(pdev, REGISTER_BAR);
+	if (reg_size > MAX_REGISTER_BAR_LEN)
+		vsoc_dev.regs =
+		    pci_iomap(pdev, REGISTER_BAR, MAX_REGISTER_BAR_LEN);
+	else
+		vsoc_dev.regs = pci_iomap(pdev, REGISTER_BAR, reg_size);
+
+	if (!vsoc_dev.regs) {
+		dev_err(&pdev->dev,
+			"cannot map registers of size %zu\n",
+		       (size_t)reg_size);
+		vsoc_remove_device(pdev);
+		return -EBUSY;
+	}
+
+	/* Map the shared memory in BAR 2 */
+	vsoc_dev.shm_phys_start = pci_resource_start(pdev, SHARED_MEMORY_BAR);
+	vsoc_dev.shm_size = pci_resource_len(pdev, SHARED_MEMORY_BAR);
+
+	dev_info(&pdev->dev, "shared memory @ DMA %pa size=0x%zx\n",
+		 &vsoc_dev.shm_phys_start, vsoc_dev.shm_size);
+	vsoc_dev.kernel_mapped_shm = pci_iomap_wc(pdev, SHARED_MEMORY_BAR, 0);
+	if (!vsoc_dev.kernel_mapped_shm) {
+		dev_err(&vsoc_dev.dev->dev, "cannot iomap region\n");
+		vsoc_remove_device(pdev);
+		return -EBUSY;
+	}
+
+	vsoc_dev.layout = (struct vsoc_shm_layout_descriptor __force *)
+				vsoc_dev.kernel_mapped_shm;
+	dev_info(&pdev->dev, "major_version: %d\n",
+		 vsoc_dev.layout->major_version);
+	dev_info(&pdev->dev, "minor_version: %d\n",
+		 vsoc_dev.layout->minor_version);
+	dev_info(&pdev->dev, "size: 0x%x\n", vsoc_dev.layout->size);
+	dev_info(&pdev->dev, "regions: %d\n", vsoc_dev.layout->region_count);
+	if (vsoc_dev.layout->major_version !=
+	    CURRENT_VSOC_LAYOUT_MAJOR_VERSION) {
+		dev_err(&vsoc_dev.dev->dev,
+			"driver supports only major_version %d\n",
+			CURRENT_VSOC_LAYOUT_MAJOR_VERSION);
+		vsoc_remove_device(pdev);
+		return -EBUSY;
+	}
+	result = alloc_chrdev_region(&devt, 0, vsoc_dev.layout->region_count,
+				     VSOC_DEV_NAME);
+	if (result) {
+		dev_err(&vsoc_dev.dev->dev, "alloc_chrdev_region failed\n");
+		vsoc_remove_device(pdev);
+		return -EBUSY;
+	}
+	vsoc_dev.major = MAJOR(devt);
+	cdev_init(&vsoc_dev.cdev, &vsoc_ops);
+	vsoc_dev.cdev.owner = THIS_MODULE;
+	result = cdev_add(&vsoc_dev.cdev, devt, vsoc_dev.layout->region_count);
+	if (result) {
+		dev_err(&vsoc_dev.dev->dev, "cdev_add error\n");
+		vsoc_remove_device(pdev);
+		return -EBUSY;
+	}
+	vsoc_dev.cdev_added = true;
+	vsoc_dev.class = class_create(THIS_MODULE, VSOC_DEV_NAME);
+	if (IS_ERR(vsoc_dev.class)) {
+		dev_err(&vsoc_dev.dev->dev, "class_create failed\n");
+		vsoc_remove_device(pdev);
+		return PTR_ERR(vsoc_dev.class);
+	}
+	vsoc_dev.class_added = true;
+	vsoc_dev.regions = (struct vsoc_device_region __force *)
+		((void *)vsoc_dev.layout +
+		 vsoc_dev.layout->vsoc_region_desc_offset);
+	vsoc_dev.msix_entries =
+		kcalloc(vsoc_dev.layout->region_count,
+			sizeof(vsoc_dev.msix_entries[0]), GFP_KERNEL);
+	if (!vsoc_dev.msix_entries) {
+		dev_err(&vsoc_dev.dev->dev,
+			"unable to allocate msix_entries\n");
+		vsoc_remove_device(pdev);
+		return -ENOSPC;
+	}
+	vsoc_dev.regions_data =
+		kcalloc(vsoc_dev.layout->region_count,
+			sizeof(vsoc_dev.regions_data[0]), GFP_KERNEL);
+	if (!vsoc_dev.regions_data) {
+		dev_err(&vsoc_dev.dev->dev,
+			"unable to allocate regions' data\n");
+		vsoc_remove_device(pdev);
+		return -ENOSPC;
+	}
+	for (i = 0; i < vsoc_dev.layout->region_count; ++i)
+		vsoc_dev.msix_entries[i].entry = i;
+
+	result = pci_enable_msix_exact(vsoc_dev.dev, vsoc_dev.msix_entries,
+				       vsoc_dev.layout->region_count);
+	if (result) {
+		dev_info(&pdev->dev, "pci_enable_msix failed: %d\n", result);
+		vsoc_remove_device(pdev);
+		return -ENOSPC;
+	}
+	/* Check that all regions are well formed */
+	for (i = 0; i < vsoc_dev.layout->region_count; ++i) {
+		const struct vsoc_device_region *region = vsoc_dev.regions + i;
+
+		if (!PAGE_ALIGNED(region->region_begin_offset) ||
+		    !PAGE_ALIGNED(region->region_end_offset)) {
+			dev_err(&vsoc_dev.dev->dev,
+				"region %d not aligned (%x:%x)", i,
+				region->region_begin_offset,
+				region->region_end_offset);
+			vsoc_remove_device(pdev);
+			return -EFAULT;
+		}
+		if (region->region_begin_offset >= region->region_end_offset ||
+		    region->region_end_offset > vsoc_dev.shm_size) {
+			dev_err(&vsoc_dev.dev->dev,
+				"region %d offsets are wrong: %x %x %zx",
+				i, region->region_begin_offset,
+				region->region_end_offset, vsoc_dev.shm_size);
+			vsoc_remove_device(pdev);
+			return -EFAULT;
+		}
+		if (region->managed_by >= vsoc_dev.layout->region_count) {
+			dev_err(&vsoc_dev.dev->dev,
+				"region %d has invalid owner: %u",
+				i, region->managed_by);
+			vsoc_remove_device(pdev);
+			return -EFAULT;
+		}
+	}
+	vsoc_dev.msix_enabled = true;
+	for (i = 0; i < vsoc_dev.layout->region_count; ++i) {
+		const struct vsoc_device_region *region = vsoc_dev.regions + i;
+		size_t name_sz = sizeof(vsoc_dev.regions_data[i].name) - 1;
+		const struct vsoc_signal_table_layout *h_to_g_signal_table =
+			&region->host_to_guest_signal_table;
+		const struct vsoc_signal_table_layout *g_to_h_signal_table =
+			&region->guest_to_host_signal_table;
+
+		vsoc_dev.regions_data[i].name[name_sz] = '\0';
+		memcpy(vsoc_dev.regions_data[i].name, region->device_name,
+		       name_sz);
+		dev_info(&pdev->dev, "region %d name=%s\n",
+			 i, vsoc_dev.regions_data[i].name);
+		init_waitqueue_head
+			(&vsoc_dev.regions_data[i].interrupt_wait_queue);
+		init_waitqueue_head(&vsoc_dev.regions_data[i].futex_wait_queue);
+		vsoc_dev.regions_data[i].incoming_signalled =
+			shm_off_to_virtual_addr(region->region_begin_offset) +
+			h_to_g_signal_table->interrupt_signalled_offset;
+		vsoc_dev.regions_data[i].outgoing_signalled =
+			shm_off_to_virtual_addr(region->region_begin_offset) +
+			g_to_h_signal_table->interrupt_signalled_offset;
+		result = request_irq(vsoc_dev.msix_entries[i].vector,
+				     vsoc_interrupt, 0,
+				     vsoc_dev.regions_data[i].name,
+				     vsoc_dev.regions_data + i);
+		if (result) {
+			dev_info(&pdev->dev,
+				 "request_irq failed irq=%d vector=%d\n",
+				i, vsoc_dev.msix_entries[i].vector);
+			vsoc_remove_device(pdev);
+			return -ENOSPC;
+		}
+		vsoc_dev.regions_data[i].irq_requested = true;
+		if (!device_create(vsoc_dev.class, NULL,
+				   MKDEV(vsoc_dev.major, i),
+				   NULL, vsoc_dev.regions_data[i].name)) {
+			dev_err(&vsoc_dev.dev->dev, "device_create failed\n");
+			vsoc_remove_device(pdev);
+			return -EBUSY;
+		}
+		vsoc_dev.regions_data[i].device_created = true;
+	}
+	return 0;
+}
+
+/*
+ * This should undo all of the allocations in the probe function in reverse
+ * order.
+ *
+ * Notes:
+ *
+ *   The device may have been partially initialized, so double check
+ *   that the allocations happened.
+ *
+ *   This function may be called multiple times, so mark resources as freed
+ *   as they are deallocated.
+ */
+static void vsoc_remove_device(struct pci_dev *pdev)
+{
+	int i;
+	/*
+	 * pdev is the first thing to be set on probe and the last thing
+	 * to be cleared here. If it's NULL then there is no cleanup.
+	 */
+	if (!pdev || !vsoc_dev.dev)
+		return;
+	dev_info(&pdev->dev, "remove_device\n");
+	if (vsoc_dev.regions_data) {
+		for (i = 0; i < vsoc_dev.layout->region_count; ++i) {
+			if (vsoc_dev.regions_data[i].device_created) {
+				device_destroy(vsoc_dev.class,
+					       MKDEV(vsoc_dev.major, i));
+				vsoc_dev.regions_data[i].device_created = false;
+			}
+			if (vsoc_dev.regions_data[i].irq_requested)
+				free_irq(vsoc_dev.msix_entries[i].vector, NULL);
+			vsoc_dev.regions_data[i].irq_requested = false;
+		}
+		kfree(vsoc_dev.regions_data);
+		vsoc_dev.regions_data = NULL;
+	}
+	if (vsoc_dev.msix_enabled) {
+		pci_disable_msix(pdev);
+		vsoc_dev.msix_enabled = false;
+	}
+	kfree(vsoc_dev.msix_entries);
+	vsoc_dev.msix_entries = NULL;
+	vsoc_dev.regions = NULL;
+	if (vsoc_dev.class_added) {
+		class_destroy(vsoc_dev.class);
+		vsoc_dev.class_added = false;
+	}
+	if (vsoc_dev.cdev_added) {
+		cdev_del(&vsoc_dev.cdev);
+		vsoc_dev.cdev_added = false;
+	}
+	if (vsoc_dev.major && vsoc_dev.layout) {
+		unregister_chrdev_region(MKDEV(vsoc_dev.major, 0),
+					 vsoc_dev.layout->region_count);
+		vsoc_dev.major = 0;
+	}
+	vsoc_dev.layout = NULL;
+	if (vsoc_dev.kernel_mapped_shm) {
+		pci_iounmap(pdev, vsoc_dev.kernel_mapped_shm);
+		vsoc_dev.kernel_mapped_shm = NULL;
+	}
+	if (vsoc_dev.regs) {
+		pci_iounmap(pdev, vsoc_dev.regs);
+		vsoc_dev.regs = NULL;
+	}
+	if (vsoc_dev.requested_regions) {
+		pci_release_regions(pdev);
+		vsoc_dev.requested_regions = false;
+	}
+	if (vsoc_dev.enabled_device) {
+		pci_disable_device(pdev);
+		vsoc_dev.enabled_device = false;
+	}
+	/* Do this last: it indicates that the device is not initialized. */
+	vsoc_dev.dev = NULL;
+}
+
+static void __exit vsoc_cleanup_module(void)
+{
+	vsoc_remove_device(vsoc_dev.dev);
+	pci_unregister_driver(&vsoc_pci_driver);
+}
+
+static int __init vsoc_init_module(void)
+{
+	int err = -ENOMEM;
+
+	INIT_LIST_HEAD(&vsoc_dev.permissions);
+	mutex_init(&vsoc_dev.mtx);
+
+	err = pci_register_driver(&vsoc_pci_driver);
+	if (err < 0)
+		return err;
+	return 0;
+}
+
+static int vsoc_open(struct inode *inode, struct file *filp)
+{
+	/* Can't use vsoc_validate_filep because filp is still incomplete */
+	int ret = vsoc_validate_inode(inode);
+
+	if (ret)
+		return ret;
+	filp->private_data =
+		kzalloc(sizeof(struct vsoc_private_data), GFP_KERNEL);
+	if (!filp->private_data)
+		return -ENOMEM;
+	return 0;
+}
+
+static int vsoc_release(struct inode *inode, struct file *filp)
+{
+	struct vsoc_private_data *private_data = NULL;
+	struct fd_scoped_permission_node *node = NULL;
+	struct vsoc_device_region *owner_region_p = NULL;
+	int retval = vsoc_validate_filep(filp);
+
+	if (retval)
+		return retval;
+	private_data = (struct vsoc_private_data *)filp->private_data;
+	if (!private_data)
+		return 0;
+
+	node = private_data->fd_scoped_permission_node;
+	if (node) {
+		owner_region_p = vsoc_region_from_inode(inode);
+		if (owner_region_p->managed_by != VSOC_REGION_WHOLE) {
+			owner_region_p =
+			    &vsoc_dev.regions[owner_region_p->managed_by];
+		}
+		do_destroy_fd_scoped_permission_node(owner_region_p, node);
+		private_data->fd_scoped_permission_node = NULL;
+	}
+	kfree(private_data);
+	filp->private_data = NULL;
+
+	return 0;
+}
+
+/*
+ * Returns the device relative offset and length of the area specified by the
+ * fd scoped permission. If there is no fd scoped permission set, a default
+ * permission covering the entire region is assumed, unless the region is owned
+ * by another one, in which case the default is a permission with zero size.
+ */
+static ssize_t vsoc_get_area(struct file *filp, __u32 *area_offset)
+{
+	__u32 off = 0;
+	ssize_t length = 0;
+	struct vsoc_device_region *region_p;
+	struct fd_scoped_permission *perm;
+
+	region_p = vsoc_region_from_filep(filp);
+	off = region_p->region_begin_offset;
+	perm = &((struct vsoc_private_data *)filp->private_data)->
+		fd_scoped_permission_node->permission;
+	if (perm) {
+		off += perm->begin_offset;
+		length = perm->end_offset - perm->begin_offset;
+	} else if (region_p->managed_by == VSOC_REGION_WHOLE) {
+		/* No permission set and the regions is not owned by another,
+		 * default to full region access.
+		 */
+		length = vsoc_device_region_size(region_p);
+	} else {
+		/* return zero length, access is denied. */
+		length = 0;
+	}
+	if (area_offset)
+		*area_offset = off;
+	return length;
+}
+
+static int vsoc_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	unsigned long len = vma->vm_end - vma->vm_start;
+	__u32 area_off;
+	phys_addr_t mem_off;
+	ssize_t area_len;
+	int retval = vsoc_validate_filep(filp);
+
+	if (retval)
+		return retval;
+	area_len = vsoc_get_area(filp, &area_off);
+	/* Add the requested offset */
+	area_off += (vma->vm_pgoff << PAGE_SHIFT);
+	area_len -= (vma->vm_pgoff << PAGE_SHIFT);
+	if (area_len < len)
+		return -EINVAL;
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	mem_off = shm_off_to_phys_addr(area_off);
+	if (io_remap_pfn_range(vma, vma->vm_start, mem_off >> PAGE_SHIFT,
+			       len, vma->vm_page_prot))
+		return -EAGAIN;
+	return 0;
+}
+
+module_init(vsoc_init_module);
+module_exit(vsoc_cleanup_module);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Greg Hartman <ghartman@google.com>");
+MODULE_DESCRIPTION("VSoC interpretation of QEmu's ivshmem device");
+MODULE_VERSION("1.0");
diff --git a/drivers/staging/board/armadillo800eva.c b/drivers/staging/board/armadillo800eva.c
index 4de4fd06eebc..962cc0c79988 100644
--- a/drivers/staging/board/armadillo800eva.c
+++ b/drivers/staging/board/armadillo800eva.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Staging board support for Armadillo 800 eva.
  * Enable not-yet-DT-capable devices here.
@@ -6,15 +7,6 @@
  *
  * Copyright (C) 2012 Renesas Solutions Corp.
  * Copyright (C) 2012 Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/dma-mapping.h>
diff --git a/drivers/staging/board/board.c b/drivers/staging/board/board.c
index 86dc41101610..cb6feb34dd40 100644
--- a/drivers/staging/board/board.c
+++ b/drivers/staging/board/board.c
@@ -1,10 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2014 Magnus Damm
  * Copyright (C) 2015 Glider bvba
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file "COPYING" in the main directory of this archive
- * for more details.
  */
 
 #define pr_fmt(fmt)	"board_staging: "  fmt
diff --git a/drivers/staging/comedi/comedi_compat32.c b/drivers/staging/comedi/comedi_compat32.c
index 97fb9388bc22..fa9d239474ee 100644
--- a/drivers/staging/comedi/comedi_compat32.c
+++ b/drivers/staging/comedi/comedi_compat32.c
@@ -34,7 +34,7 @@
 struct comedi32_chaninfo_struct {
 	unsigned int subdev;
 	compat_uptr_t maxdata_list;	/* 32-bit 'unsigned int *' */
-	compat_uptr_t flaglist;		/* 32-bit 'unsigned int *' */
+	compat_uptr_t flaglist;	/* 32-bit 'unsigned int *' */
 	compat_uptr_t rangelist;	/* 32-bit 'unsigned int *' */
 	unsigned int unused[4];
 };
@@ -57,16 +57,16 @@ struct comedi32_cmd_struct {
 	unsigned int scan_end_arg;
 	unsigned int stop_src;
 	unsigned int stop_arg;
-	compat_uptr_t chanlist;		/* 32-bit 'unsigned int *' */
+	compat_uptr_t chanlist;	/* 32-bit 'unsigned int *' */
 	unsigned int chanlist_len;
-	compat_uptr_t data;		/* 32-bit 'short *' */
+	compat_uptr_t data;	/* 32-bit 'short *' */
 	unsigned int data_len;
 };
 
 struct comedi32_insn_struct {
 	unsigned int insn;
 	unsigned int n;
-	compat_uptr_t data;		/* 32-bit 'unsigned int *' */
+	compat_uptr_t data;	/* 32-bit 'unsigned int *' */
 	unsigned int subdev;
 	unsigned int chanspec;
 	unsigned int unused[3];
@@ -74,7 +74,7 @@ struct comedi32_insn_struct {
 
 struct comedi32_insnlist_struct {
 	unsigned int n_insns;
-	compat_uptr_t insns;		/* 32-bit 'struct comedi_insn *' */
+	compat_uptr_t insns;	/* 32-bit 'struct comedi_insn *' */
 };
 
 /* Handle translated ioctl. */
@@ -194,7 +194,7 @@ static int get_compat_cmd(struct comedi_cmd __user *cmd,
 	err |= __put_user(temp.uint, &cmd->stop_arg);
 	err |= __get_user(temp.uptr, &cmd32->chanlist);
 	err |= __put_user((unsigned int __force *)compat_ptr(temp.uptr),
-			&cmd->chanlist);
+			  &cmd->chanlist);
 	err |= __get_user(temp.uint, &cmd32->chanlist_len);
 	err |= __put_user(temp.uint, &cmd->chanlist_len);
 	err |= __get_user(temp.uptr, &cmd32->data);
diff --git a/drivers/staging/comedi/comedidev.h b/drivers/staging/comedi/comedidev.h
index f3474a4ba69e..c54ac94d89d2 100644
--- a/drivers/staging/comedi/comedidev.h
+++ b/drivers/staging/comedi/comedidev.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * comedidev.h
  * header file for kernel-only structures, variables, and constants
diff --git a/drivers/staging/comedi/drivers/cb_pcidas64.c b/drivers/staging/comedi/drivers/cb_pcidas64.c
index fdd81c3beb51..631a703b345d 100644
--- a/drivers/staging/comedi/drivers/cb_pcidas64.c
+++ b/drivers/staging/comedi/drivers/cb_pcidas64.c
@@ -2268,14 +2268,14 @@ static inline unsigned int dma_transfer_size(struct comedi_device *dev)
 }
 
 static u32 ai_convert_counter_6xxx(const struct comedi_device *dev,
-					const struct comedi_cmd *cmd)
+				   const struct comedi_cmd *cmd)
 {
 	/* supposed to load counter with desired divisor minus 3 */
 	return cmd->convert_arg / TIMER_BASE - 3;
 }
 
 static u32 ai_scan_counter_6xxx(struct comedi_device *dev,
-				     struct comedi_cmd *cmd)
+				struct comedi_cmd *cmd)
 {
 	u32 count;
 
@@ -2296,7 +2296,7 @@ static u32 ai_scan_counter_6xxx(struct comedi_device *dev,
 }
 
 static u32 ai_convert_counter_4020(struct comedi_device *dev,
-					struct comedi_cmd *cmd)
+				   struct comedi_cmd *cmd)
 {
 	struct pcidas64_private *devpriv = dev->private;
 	unsigned int divisor;
diff --git a/drivers/staging/comedi/drivers/serial2002.c b/drivers/staging/comedi/drivers/serial2002.c
index b3f3b4a201af..5471b2212a62 100644
--- a/drivers/staging/comedi/drivers/serial2002.c
+++ b/drivers/staging/comedi/drivers/serial2002.c
@@ -113,7 +113,7 @@ static void serial2002_tty_read_poll_wait(struct file *f, int timeout)
 		long elapsed;
 		__poll_t mask;
 
-		mask = f->f_op->poll(f, &table.pt);
+		mask = vfs_poll(f, &table.pt);
 		if (mask & (EPOLLRDNORM | EPOLLRDBAND | EPOLLIN |
 			    EPOLLHUP | EPOLLERR)) {
 			break;
@@ -136,7 +136,7 @@ static int serial2002_tty_read(struct file *f, int timeout)
 
 	result = -1;
 	if (!IS_ERR(f)) {
-		if (f->f_op->poll) {
+		if (file_can_poll(f)) {
 			serial2002_tty_read_poll_wait(f, timeout);
 
 			if (kernel_read(f, &ch, 1, &pos) == 1)
diff --git a/drivers/staging/comedi/proc.c b/drivers/staging/comedi/proc.c
index 50d38938ac6f..8bc8e42beb90 100644
--- a/drivers/staging/comedi/proc.c
+++ b/drivers/staging/comedi/proc.c
@@ -62,25 +62,9 @@ static int comedi_read(struct seq_file *m, void *v)
 	return 0;
 }
 
-/*
- * seq_file wrappers for procfile show routines.
- */
-static int comedi_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, comedi_read, NULL);
-}
-
-static const struct file_operations comedi_proc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= comedi_proc_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
 void __init comedi_proc_init(void)
 {
-	if (!proc_create("comedi", 0444, NULL, &comedi_proc_fops))
+	if (!proc_create_single("comedi", 0444, NULL, comedi_read))
 		pr_warn("comedi: unable to create proc entry\n");
 }
 
diff --git a/drivers/staging/emxx_udc/Kconfig b/drivers/staging/emxx_udc/Kconfig
index d7577096fb25..e50e72218364 100644
--- a/drivers/staging/emxx_udc/Kconfig
+++ b/drivers/staging/emxx_udc/Kconfig
@@ -1,6 +1,6 @@
 config USB_EMXX
 	tristate "EMXX USB Function Device Controller"
- 	depends on USB_GADGET && (ARCH_SHMOBILE || (ARM && COMPILE_TEST))
+	depends on USB_GADGET && (ARCH_RENESAS || (ARM && COMPILE_TEST))
 	help
 	   The Emma Mobile series of SoCs from Renesas Electronics and
 	   former NEC Electronics include USB Function hardware.
diff --git a/drivers/staging/emxx_udc/TODO b/drivers/staging/emxx_udc/TODO
index 1319379beb7e..471529a470c7 100644
--- a/drivers/staging/emxx_udc/TODO
+++ b/drivers/staging/emxx_udc/TODO
@@ -1,4 +1,6 @@
 * add clock framework support (platform device with CCF needs special care)
 * break out board-specific VBUS GPIO to work with multiplatform
+* convert VBUS GPIO to use GPIO descriptors from <linux/gpio/consumer.h>
+  and stop using the old GPIO API
 * DT bindings
 * move driver into drivers/usb/gadget/
diff --git a/drivers/staging/fbtft/TODO b/drivers/staging/fbtft/TODO
new file mode 100644
index 000000000000..7e64c7e438f0
--- /dev/null
+++ b/drivers/staging/fbtft/TODO
@@ -0,0 +1,4 @@
+* convert all uses of the old GPIO API from <linux/gpio.h> to the
+  GPIO descriptor API in <linux/gpio/consumer.h> and look up GPIO
+  lines from device tree, ACPI or board files, board files should
+  use <linux/gpio/machine.h>
diff --git a/drivers/staging/fbtft/fbtft-bus.c b/drivers/staging/fbtft/fbtft-bus.c
index a263bce260c9..871b307d83cb 100644
--- a/drivers/staging/fbtft/fbtft-bus.c
+++ b/drivers/staging/fbtft/fbtft-bus.c
@@ -22,10 +22,13 @@ void func(struct fbtft_par *par, int len, ...)                                \
 	if (unlikely(par->debug & DEBUG_WRITE_REGISTER)) {                    \
 		va_start(args, len);                                          \
 		for (i = 0; i < len; i++) {                                   \
-			buf[i] = modifier((data_type)va_arg(args, unsigned int)); \
+			buf[i] = modifier((data_type)va_arg(args,             \
+							    unsigned int));   \
 		}                                                             \
 		va_end(args);                                                 \
-		fbtft_par_dbg_hex(DEBUG_WRITE_REGISTER, par, par->info->device, buffer_type, buf, len, "%s: ", __func__); \
+		fbtft_par_dbg_hex(DEBUG_WRITE_REGISTER, par,                  \
+				  par->info->device, buffer_type, buf, len,   \
+				  "%s: ", __func__);                          \
 	}                                                                     \
 									      \
 	va_start(args, len);                                                  \
@@ -37,7 +40,8 @@ void func(struct fbtft_par *par, int len, ...)                                \
 	}                                                                     \
 									      \
 	*buf = modifier((data_type)va_arg(args, unsigned int));               \
-	ret = fbtft_write_buf_dc(par, par->buf, sizeof(data_type) + offset, 0); \
+	ret = fbtft_write_buf_dc(par, par->buf, sizeof(data_type) + offset,   \
+				 0);                                          \
 	if (ret < 0)							      \
 		goto out;						      \
 	len--;                                                                \
@@ -48,7 +52,8 @@ void func(struct fbtft_par *par, int len, ...)                                \
 	if (len) {                                                            \
 		i = len;                                                      \
 		while (i--)						      \
-			*buf++ = modifier((data_type)va_arg(args, unsigned int)); \
+			*buf++ = modifier((data_type)va_arg(args,             \
+							    unsigned int));   \
 		fbtft_write_buf_dc(par, par->buf,			      \
 				   len * (sizeof(data_type) + offset), 1);    \
 	}                                                                     \
diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c
index 0e36b66ae5f7..731e47149af8 100644
--- a/drivers/staging/fbtft/fbtft-core.c
+++ b/drivers/staging/fbtft/fbtft-core.c
@@ -246,7 +246,7 @@ static int fbtft_request_gpios_dt(struct fbtft_par *par)
 static int fbtft_backlight_update_status(struct backlight_device *bd)
 {
 	struct fbtft_par *par = bl_get_data(bd);
-	bool polarity = !!(bd->props.state & BL_CORE_DRIVER1);
+	bool polarity = par->polarity;
 
 	fbtft_par_dbg(DEBUG_BACKLIGHT, par,
 		"%s: polarity=%d, power=%d, fb_blank=%d\n",
@@ -296,7 +296,7 @@ void fbtft_register_backlight(struct fbtft_par *par)
 	/* Assume backlight is off, get polarity from current state of pin */
 	bl_props.power = FB_BLANK_POWERDOWN;
 	if (!gpio_get_value(par->gpio.led[0]))
-		bl_props.state |= BL_CORE_DRIVER1;
+		par->polarity = true;
 
 	bd = backlight_device_register(dev_driver_string(par->info->device),
 				       par->info->device, par,
diff --git a/drivers/staging/fbtft/fbtft.h b/drivers/staging/fbtft/fbtft.h
index e19e64e0d094..c7cb4a7896f4 100644
--- a/drivers/staging/fbtft/fbtft.h
+++ b/drivers/staging/fbtft/fbtft.h
@@ -229,6 +229,7 @@ struct fbtft_par {
 	ktime_t update_time;
 	bool bgr;
 	void *extra;
+	bool polarity;
 };
 
 #define NUMARGS(...)  (sizeof((int[]){__VA_ARGS__})/sizeof(int))
diff --git a/drivers/staging/fsl-dpaa2/Kconfig b/drivers/staging/fsl-dpaa2/Kconfig
index bbb7af551696..a4c4b83ddc9c 100644
--- a/drivers/staging/fsl-dpaa2/Kconfig
+++ b/drivers/staging/fsl-dpaa2/Kconfig
@@ -24,3 +24,11 @@ config FSL_DPAA2_ETHSW
 	---help---
 	Driver for Freescale DPAA2 Ethernet Switch. Select
 	BRIDGE to have support for bridge tools.
+
+config FSL_DPAA2_PTP_CLOCK
+	tristate "Freescale DPAA2 PTP Clock"
+	depends on FSL_DPAA2_ETH && POSIX_TIMERS
+	select PTP_1588_CLOCK
+	help
+	  This driver adds support for using the DPAA2 1588 timer module
+	  as a PTP clock.
diff --git a/drivers/staging/fsl-dpaa2/Makefile b/drivers/staging/fsl-dpaa2/Makefile
index 6cfd76b29970..9c7062945758 100644
--- a/drivers/staging/fsl-dpaa2/Makefile
+++ b/drivers/staging/fsl-dpaa2/Makefile
@@ -2,5 +2,6 @@
 # Freescale DataPath Acceleration Architecture Gen2 (DPAA2) drivers
 #
 
-obj-$(CONFIG_FSL_DPAA2_ETH)	+= ethernet/
-obj-$(CONFIG_FSL_DPAA2_ETHSW)	+= ethsw/
+obj-$(CONFIG_FSL_DPAA2_ETH)		+= ethernet/
+obj-$(CONFIG_FSL_DPAA2_ETHSW)		+= ethsw/
+obj-$(CONFIG_FSL_DPAA2_PTP_CLOCK)	+= rtc/
diff --git a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c
index 553678d8b2eb..396371728aa1 100644
--- a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c
+++ b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c
@@ -38,8 +38,11 @@
 #include <linux/msi.h>
 #include <linux/kthread.h>
 #include <linux/iommu.h>
-
+#include <linux/net_tstamp.h>
 #include <linux/fsl/mc.h>
+
+#include <net/sock.h>
+
 #include "dpaa2-eth.h"
 
 /* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files
@@ -275,6 +278,18 @@ static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv,
 
 	prefetch(skb->data);
 
+	/* Get the timestamp value */
+	if (priv->rx_tstamp) {
+		struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
+		__le64 *ts = dpaa2_get_ts(vaddr, false);
+		u64 ns;
+
+		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
+
+		ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts);
+		shhwtstamps->hwtstamp = ns_to_ktime(ns);
+	}
+
 	/* Check if we need to validate the L4 csum */
 	if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) {
 		status = le32_to_cpu(fas->status);
@@ -334,6 +349,28 @@ static int consume_frames(struct dpaa2_eth_channel *ch)
 	return cleaned;
 }
 
+/* Configure the egress frame annotation for timestamp update */
+static void enable_tx_tstamp(struct dpaa2_fd *fd, void *buf_start)
+{
+	struct dpaa2_faead *faead;
+	u32 ctrl, frc;
+
+	/* Mark the egress frame annotation area as valid */
+	frc = dpaa2_fd_get_frc(fd);
+	dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV);
+
+	/* Set hardware annotation size */
+	ctrl = dpaa2_fd_get_ctrl(fd);
+	dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL);
+
+	/* enable UPD (update prepanded data) bit in FAEAD field of
+	 * hardware frame annotation area
+	 */
+	ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD;
+	faead = dpaa2_get_faead(buf_start, true);
+	faead->ctrl = cpu_to_le32(ctrl);
+}
+
 /* Create a frame descriptor based on a fragmented skb */
 static int build_sg_fd(struct dpaa2_eth_priv *priv,
 		       struct sk_buff *skb,
@@ -420,6 +457,9 @@ static int build_sg_fd(struct dpaa2_eth_priv *priv,
 	dpaa2_fd_set_len(fd, skb->len);
 	dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_PTA | DPAA2_FD_CTRL_PTV1);
 
+	if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
+		enable_tx_tstamp(fd, sgt_buf);
+
 	return 0;
 
 dma_map_single_failed:
@@ -470,6 +510,9 @@ static int build_single_fd(struct dpaa2_eth_priv *priv,
 	dpaa2_fd_set_format(fd, dpaa2_fd_single);
 	dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_PTA | DPAA2_FD_CTRL_PTV1);
 
+	if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
+		enable_tx_tstamp(fd, buffer_start);
+
 	return 0;
 }
 
@@ -520,6 +563,19 @@ static void free_tx_fd(const struct dpaa2_eth_priv *priv,
 		return;
 	}
 
+	/* Get the timestamp value */
+	if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
+		struct skb_shared_hwtstamps shhwtstamps;
+		__le64 *ts = dpaa2_get_ts(skbh, true);
+		u64 ns;
+
+		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
+
+		ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts);
+		shhwtstamps.hwtstamp = ns_to_ktime(ns);
+		skb_tstamp_tx(skb, &shhwtstamps);
+	}
+
 	/* Free SGT buffer allocated on tx */
 	if (fd_format != dpaa2_fd_single)
 		skb_free_frag(skbh);
@@ -552,6 +608,10 @@ static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev)
 			goto err_alloc_headroom;
 		}
 		percpu_extras->tx_reallocs++;
+
+		if (skb->sk)
+			skb_set_owner_w(ns, skb->sk);
+
 		dev_kfree_skb(skb);
 		skb = ns;
 	}
@@ -1365,6 +1425,45 @@ static int dpaa2_eth_set_features(struct net_device *net_dev,
 	return 0;
 }
 
+static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
+{
+	struct dpaa2_eth_priv *priv = netdev_priv(dev);
+	struct hwtstamp_config config;
+
+	if (copy_from_user(&config, rq->ifr_data, sizeof(config)))
+		return -EFAULT;
+
+	switch (config.tx_type) {
+	case HWTSTAMP_TX_OFF:
+		priv->tx_tstamp = false;
+		break;
+	case HWTSTAMP_TX_ON:
+		priv->tx_tstamp = true;
+		break;
+	default:
+		return -ERANGE;
+	}
+
+	if (config.rx_filter == HWTSTAMP_FILTER_NONE) {
+		priv->rx_tstamp = false;
+	} else {
+		priv->rx_tstamp = true;
+		/* TS is set for all frame types, not only those requested */
+		config.rx_filter = HWTSTAMP_FILTER_ALL;
+	}
+
+	return copy_to_user(rq->ifr_data, &config, sizeof(config)) ?
+			-EFAULT : 0;
+}
+
+static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
+{
+	if (cmd == SIOCSHWTSTAMP)
+		return dpaa2_eth_ts_ioctl(dev, rq, cmd);
+
+	return -EINVAL;
+}
+
 static const struct net_device_ops dpaa2_eth_ops = {
 	.ndo_open = dpaa2_eth_open,
 	.ndo_start_xmit = dpaa2_eth_tx,
@@ -1375,6 +1474,7 @@ static const struct net_device_ops dpaa2_eth_ops = {
 	.ndo_change_mtu = dpaa2_eth_change_mtu,
 	.ndo_set_rx_mode = dpaa2_eth_set_rx_mode,
 	.ndo_set_features = dpaa2_eth_set_features,
+	.ndo_do_ioctl = dpaa2_eth_ioctl,
 };
 
 static void cdan_cb(struct dpaa2_io_notification_ctx *ctx)
@@ -1770,7 +1870,9 @@ static int set_buffer_layout(struct dpaa2_eth_priv *priv)
 
 	/* tx buffer */
 	buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE;
-	buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE;
+	buf_layout.pass_timestamp = true;
+	buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE |
+			     DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
 	err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
 				     DPNI_QUEUE_TX, &buf_layout);
 	if (err) {
@@ -1779,7 +1881,7 @@ static int set_buffer_layout(struct dpaa2_eth_priv *priv)
 	}
 
 	/* tx-confirm buffer */
-	buf_layout.options = 0;
+	buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
 	err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
 				     DPNI_QUEUE_TX_CONFIRM, &buf_layout);
 	if (err) {
@@ -1810,7 +1912,8 @@ static int set_buffer_layout(struct dpaa2_eth_priv *priv)
 	buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT |
 			     DPNI_BUF_LAYOUT_OPT_FRAME_STATUS |
 			     DPNI_BUF_LAYOUT_OPT_DATA_ALIGN |
-			     DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM;
+			     DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM |
+			     DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
 	err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
 				     DPNI_QUEUE_RX, &buf_layout);
 	if (err) {
diff --git a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.h b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.h
index 54cea2fc6e58..905a4e6be8fa 100644
--- a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.h
+++ b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.h
@@ -88,8 +88,12 @@
 #define DPAA2_ETH_SKB_SIZE \
 	(DPAA2_ETH_RX_BUF_SIZE + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
 
-/* Hardware annotation area in RX  buffers */
+/* Hardware annotation area in RX/TX buffers */
 #define DPAA2_ETH_RX_HWA_SIZE		64
+#define DPAA2_ETH_TX_HWA_SIZE		128
+
+/* PTP nominal frequency 1GHz */
+#define DPAA2_PTP_CLK_PERIOD_NS		1
 
 /* Due to a limitation in WRIOP 1.0.0, the RX buffer data must be aligned
  * to 256B. For newer revisions, the requirement is only for 64B alignment
@@ -135,6 +139,7 @@ struct dpaa2_eth_swa {
 /* Annotation bits in FD CTRL */
 #define DPAA2_FD_CTRL_PTA		0x00800000
 #define DPAA2_FD_CTRL_PTV1		0x00400000
+#define DPAA2_FD_CTRL_ASAL		0x00020000	/* ASAL = 128B */
 
 /* Frame annotation status */
 struct dpaa2_fas {
@@ -150,6 +155,23 @@ struct dpaa2_fas {
 #define DPAA2_FAS_OFFSET		0
 #define DPAA2_FAS_SIZE			(sizeof(struct dpaa2_fas))
 
+/* Timestamp is located in the next 8 bytes of the buffer's
+ * hardware annotation area
+ */
+#define DPAA2_TS_OFFSET			0x8
+
+/* Frame annotation egress action descriptor */
+#define DPAA2_FAEAD_OFFSET		0x58
+
+struct dpaa2_faead {
+	__le32 conf_fqid;
+	__le32 ctrl;
+};
+
+#define DPAA2_FAEAD_A2V			0x20000000
+#define DPAA2_FAEAD_UPDV		0x00001000
+#define DPAA2_FAEAD_UPD			0x00000010
+
 /* Accessors for the hardware annotation fields that we use */
 static inline void *dpaa2_get_hwa(void *buf_addr, bool swa)
 {
@@ -161,6 +183,16 @@ static inline struct dpaa2_fas *dpaa2_get_fas(void *buf_addr, bool swa)
 	return dpaa2_get_hwa(buf_addr, swa) + DPAA2_FAS_OFFSET;
 }
 
+static inline __le64 *dpaa2_get_ts(void *buf_addr, bool swa)
+{
+	return dpaa2_get_hwa(buf_addr, swa) + DPAA2_TS_OFFSET;
+}
+
+static inline struct dpaa2_faead *dpaa2_get_faead(void *buf_addr, bool swa)
+{
+	return dpaa2_get_hwa(buf_addr, swa) + DPAA2_FAEAD_OFFSET;
+}
+
 /* Error and status bits in the frame annotation status word */
 /* Debug frame, otherwise supposed to be discarded */
 #define DPAA2_FAS_DISC			0x80000000
@@ -319,6 +351,9 @@ struct dpaa2_eth_priv {
 	u16 bpid;
 	struct iommu_domain *iommu_domain;
 
+	bool tx_tstamp; /* Tx timestamping enabled */
+	bool rx_tstamp; /* Rx timestamping enabled */
+
 	u16 tx_qdid;
 	u16 rx_buf_align;
 	struct fsl_mc_io *mc_io;
@@ -355,6 +390,7 @@ struct dpaa2_eth_priv {
 
 extern const struct ethtool_ops dpaa2_ethtool_ops;
 extern const char dpaa2_eth_drv_version[];
+extern int dpaa2_phc_index;
 
 static inline int dpaa2_eth_cmp_dpni_ver(struct dpaa2_eth_priv *priv,
 					 u16 ver_major, u16 ver_minor)
@@ -377,10 +413,19 @@ static inline
 unsigned int dpaa2_eth_needed_headroom(struct dpaa2_eth_priv *priv,
 				       struct sk_buff *skb)
 {
+	unsigned int headroom = DPAA2_ETH_SWA_SIZE;
+
+	/* For non-linear skbs we have no headroom requirement, as we build a
+	 * SG frame with a newly allocated SGT buffer
+	 */
 	if (skb_is_nonlinear(skb))
 		return 0;
 
-	return DPAA2_ETH_SWA_SIZE;
+	/* If we have Tx timestamping, need 128B hardware annotation */
+	if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
+		headroom += DPAA2_ETH_TX_HWA_SIZE;
+
+	return headroom;
 }
 
 /* Extra headroom space requested to hardware, in order to make sure there's
diff --git a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-ethtool.c b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-ethtool.c
index bfc8b64169ca..1ae779ae8c99 100644
--- a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-ethtool.c
+++ b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-ethtool.c
@@ -30,6 +30,8 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <linux/net_tstamp.h>
+
 #include "dpni.h"	/* DPNI_LINK_OPT_* */
 #include "dpaa2-eth.h"
 
@@ -274,6 +276,26 @@ static int dpaa2_eth_get_rxnfc(struct net_device *net_dev,
 	return 0;
 }
 
+int dpaa2_phc_index = -1;
+EXPORT_SYMBOL(dpaa2_phc_index);
+
+static int dpaa2_eth_get_ts_info(struct net_device *dev,
+				 struct ethtool_ts_info *info)
+{
+	info->so_timestamping = SOF_TIMESTAMPING_TX_HARDWARE |
+				SOF_TIMESTAMPING_RX_HARDWARE |
+				SOF_TIMESTAMPING_RAW_HARDWARE;
+
+	info->phc_index = dpaa2_phc_index;
+
+	info->tx_types = (1 << HWTSTAMP_TX_OFF) |
+			 (1 << HWTSTAMP_TX_ON);
+
+	info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) |
+			   (1 << HWTSTAMP_FILTER_ALL);
+	return 0;
+}
+
 const struct ethtool_ops dpaa2_ethtool_ops = {
 	.get_drvinfo = dpaa2_eth_get_drvinfo,
 	.get_link = ethtool_op_get_link,
@@ -283,4 +305,5 @@ const struct ethtool_ops dpaa2_ethtool_ops = {
 	.get_ethtool_stats = dpaa2_eth_get_ethtool_stats,
 	.get_strings = dpaa2_eth_get_strings,
 	.get_rxnfc = dpaa2_eth_get_rxnfc,
+	.get_ts_info = dpaa2_eth_get_ts_info,
 };
diff --git a/drivers/staging/fsl-dpaa2/ethsw/dpsw-cmd.h b/drivers/staging/fsl-dpaa2/ethsw/dpsw-cmd.h
index 1c203e6e8035..da744f2b0ee6 100644
--- a/drivers/staging/fsl-dpaa2/ethsw/dpsw-cmd.h
+++ b/drivers/staging/fsl-dpaa2/ethsw/dpsw-cmd.h
@@ -49,6 +49,8 @@
 #define DPSW_CMDID_IF_SET_FLOODING          DPSW_CMD_ID(0x047)
 #define DPSW_CMDID_IF_SET_BROADCAST         DPSW_CMD_ID(0x048)
 
+#define DPSW_CMDID_IF_GET_TCI               DPSW_CMD_ID(0x04A)
+
 #define DPSW_CMDID_IF_SET_LINK_CFG          DPSW_CMD_ID(0x04C)
 
 #define DPSW_CMDID_VLAN_ADD                 DPSW_CMD_ID(0x060)
@@ -206,6 +208,17 @@ struct dpsw_cmd_if_set_tci {
 	__le16 conf;
 };
 
+struct dpsw_cmd_if_get_tci {
+	__le16 if_id;
+};
+
+struct dpsw_rsp_if_get_tci {
+	__le16 pad;
+	__le16 vlan_id;
+	u8 dei;
+	u8 pcp;
+};
+
 #define DPSW_STATE_SHIFT	0
 #define DPSW_STATE_SIZE		4
 
diff --git a/drivers/staging/fsl-dpaa2/ethsw/dpsw.c b/drivers/staging/fsl-dpaa2/ethsw/dpsw.c
index 9b9bc604b461..cabed77b445d 100644
--- a/drivers/staging/fsl-dpaa2/ethsw/dpsw.c
+++ b/drivers/staging/fsl-dpaa2/ethsw/dpsw.c
@@ -528,6 +528,48 @@ int dpsw_if_set_tci(struct fsl_mc_io *mc_io,
 	return mc_send_command(mc_io, &cmd);
 }
 
+/**
+ * dpsw_if_get_tci() - Get default VLAN Tag Control Information (TCI)
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPSW object
+ * @if_id:	Interface Identifier
+ * @cfg:	Tag Control Information Configuration
+ *
+ * Return:	Completion status. '0' on Success; Error code otherwise.
+ */
+int dpsw_if_get_tci(struct fsl_mc_io *mc_io,
+		    u32 cmd_flags,
+		    u16 token,
+		    u16 if_id,
+		    struct dpsw_tci_cfg *cfg)
+{
+	struct fsl_mc_command cmd = { 0 };
+	struct dpsw_cmd_if_get_tci *cmd_params;
+	struct dpsw_rsp_if_get_tci *rsp_params;
+	int err;
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPSW_CMDID_IF_GET_TCI,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dpsw_cmd_if_get_tci *)cmd.params;
+	cmd_params->if_id = cpu_to_le16(if_id);
+
+	/* send command to mc*/
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	/* retrieve response parameters */
+	rsp_params = (struct dpsw_rsp_if_get_tci *)cmd.params;
+	cfg->pcp = rsp_params->pcp;
+	cfg->dei = rsp_params->dei;
+	cfg->vlan_id = le16_to_cpu(rsp_params->vlan_id);
+
+	return 0;
+}
+
 /**
  * dpsw_if_set_stp() - Function sets Spanning Tree Protocol (STP) state.
  * @mc_io:	Pointer to MC portal's I/O object
diff --git a/drivers/staging/fsl-dpaa2/ethsw/dpsw.h b/drivers/staging/fsl-dpaa2/ethsw/dpsw.h
index 3335adde0193..82f80c409ec3 100644
--- a/drivers/staging/fsl-dpaa2/ethsw/dpsw.h
+++ b/drivers/staging/fsl-dpaa2/ethsw/dpsw.h
@@ -306,6 +306,12 @@ int dpsw_if_set_tci(struct fsl_mc_io *mc_io,
 		    u16 if_id,
 		    const struct dpsw_tci_cfg *cfg);
 
+int dpsw_if_get_tci(struct fsl_mc_io *mc_io,
+		    u32 cmd_flags,
+		    u16 token,
+		    u16 if_id,
+		    struct dpsw_tci_cfg *cfg);
+
 /**
  * enum dpsw_stp_state - Spanning Tree Protocol (STP) states
  * @DPSW_STP_STATE_BLOCKING: Blocking state
diff --git a/drivers/staging/fsl-dpaa2/ethsw/ethsw.c b/drivers/staging/fsl-dpaa2/ethsw/ethsw.c
index c723a04bc3d6..1c4fe64026d0 100644
--- a/drivers/staging/fsl-dpaa2/ethsw/ethsw.c
+++ b/drivers/staging/fsl-dpaa2/ethsw/ethsw.c
@@ -50,14 +50,23 @@ static int ethsw_add_vlan(struct ethsw_core *ethsw, u16 vid)
 	return 0;
 }
 
-static int ethsw_port_set_tci(struct ethsw_port_priv *port_priv,
-			      struct dpsw_tci_cfg *tci_cfg)
+static int ethsw_port_set_pvid(struct ethsw_port_priv *port_priv, u16 pvid)
 {
 	struct ethsw_core *ethsw = port_priv->ethsw_data;
 	struct net_device *netdev = port_priv->netdev;
+	struct dpsw_tci_cfg tci_cfg = { 0 };
 	bool is_oper;
 	int err, ret;
 
+	err = dpsw_if_get_tci(ethsw->mc_io, 0, ethsw->dpsw_handle,
+			      port_priv->idx, &tci_cfg);
+	if (err) {
+		netdev_err(netdev, "dpsw_if_get_tci err %d\n", err);
+		return err;
+	}
+
+	tci_cfg.vlan_id = pvid;
+
 	/* Interface needs to be down to change PVID */
 	is_oper = netif_oper_up(netdev);
 	if (is_oper) {
@@ -71,17 +80,16 @@ static int ethsw_port_set_tci(struct ethsw_port_priv *port_priv,
 	}
 
 	err = dpsw_if_set_tci(ethsw->mc_io, 0, ethsw->dpsw_handle,
-			      port_priv->idx, tci_cfg);
+			      port_priv->idx, &tci_cfg);
 	if (err) {
 		netdev_err(netdev, "dpsw_if_set_tci err %d\n", err);
 		goto set_tci_error;
 	}
 
 	/* Delete previous PVID info and mark the new one */
-	if (port_priv->pvid)
-		port_priv->vlans[port_priv->pvid] &= ~ETHSW_VLAN_PVID;
-	port_priv->vlans[tci_cfg->vlan_id] |= ETHSW_VLAN_PVID;
-	port_priv->pvid = tci_cfg->vlan_id;
+	port_priv->vlans[port_priv->pvid] &= ~ETHSW_VLAN_PVID;
+	port_priv->vlans[pvid] |= ETHSW_VLAN_PVID;
+	port_priv->pvid = pvid;
 
 set_tci_error:
 	if (is_oper) {
@@ -133,13 +141,7 @@ static int ethsw_port_add_vlan(struct ethsw_port_priv *port_priv,
 	}
 
 	if (flags & BRIDGE_VLAN_INFO_PVID) {
-		struct dpsw_tci_cfg tci_cfg = {
-			.pcp = 0,
-			.dei = 0,
-			.vlan_id = vid,
-		};
-
-		err = ethsw_port_set_tci(port_priv, &tci_cfg);
+		err = ethsw_port_set_pvid(port_priv, vid);
 		if (err)
 			return err;
 	}
@@ -616,10 +618,8 @@ static void ethsw_teardown_irqs(struct fsl_mc_device *sw_dev)
 {
 	struct device *dev = &sw_dev->dev;
 	struct ethsw_core *ethsw = dev_get_drvdata(dev);
-	struct fsl_mc_device_irq *irq;
 	int err;
 
-	irq = sw_dev->irqs[DPSW_IRQ_INDEX_IF];
 	err = dpsw_set_irq_enable(ethsw->mc_io, 0, ethsw->dpsw_handle,
 				  DPSW_IRQ_INDEX_IF, 0);
 	if (err)
@@ -819,9 +819,7 @@ static int ethsw_port_del_vlan(struct ethsw_port_priv *port_priv, u16 vid)
 		return -ENOENT;
 
 	if (port_priv->vlans[vid] & ETHSW_VLAN_PVID) {
-		struct dpsw_tci_cfg tci_cfg = { 0 };
-
-		err = ethsw_port_set_tci(port_priv, &tci_cfg);
+		err = ethsw_port_set_pvid(port_priv, 0);
 		if (err)
 			return err;
 	}
@@ -1254,7 +1252,6 @@ static int ethsw_port_init(struct ethsw_port_priv *port_priv, u16 port)
 	const char def_mcast[ETH_ALEN] = {0x01, 0x00, 0x5e, 0x00, 0x00, 0x01};
 	struct net_device *netdev = port_priv->netdev;
 	struct ethsw_core *ethsw = port_priv->ethsw_data;
-	struct dpsw_tci_cfg tci_cfg = {0};
 	struct dpsw_vlan_if_cfg vcfg;
 	int err;
 
@@ -1272,7 +1269,7 @@ static int ethsw_port_init(struct ethsw_port_priv *port_priv, u16 port)
 		return err;
 	}
 
-	err = ethsw_port_set_tci(port_priv, &tci_cfg);
+	err = ethsw_port_set_pvid(port_priv, 0);
 	if (err)
 		return err;
 
diff --git a/drivers/staging/fsl-dpaa2/rtc/Makefile b/drivers/staging/fsl-dpaa2/rtc/Makefile
new file mode 100644
index 000000000000..5468da071163
--- /dev/null
+++ b/drivers/staging/fsl-dpaa2/rtc/Makefile
@@ -0,0 +1,7 @@
+#
+# Makefile for the Freescale DPAA2 PTP clock
+#
+
+obj-$(CONFIG_FSL_DPAA2_PTP_CLOCK) += dpaa2-rtc.o
+
+dpaa2-rtc-objs := rtc.o dprtc.o
diff --git a/drivers/staging/fsl-dpaa2/rtc/dprtc-cmd.h b/drivers/staging/fsl-dpaa2/rtc/dprtc-cmd.h
new file mode 100644
index 000000000000..db6a473430cc
--- /dev/null
+++ b/drivers/staging/fsl-dpaa2/rtc/dprtc-cmd.h
@@ -0,0 +1,137 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
+ * Copyright 2016-2018 NXP
+ */
+
+#ifndef _FSL_DPRTC_CMD_H
+#define _FSL_DPRTC_CMD_H
+
+/* DPRTC Version */
+#define DPRTC_VER_MAJOR			2
+#define DPRTC_VER_MINOR			0
+
+/* Command versioning */
+#define DPRTC_CMD_BASE_VERSION		1
+#define DPRTC_CMD_ID_OFFSET		4
+
+#define DPRTC_CMD(id)	(((id) << DPRTC_CMD_ID_OFFSET) | DPRTC_CMD_BASE_VERSION)
+
+/* Command IDs */
+#define DPRTC_CMDID_CLOSE			DPRTC_CMD(0x800)
+#define DPRTC_CMDID_OPEN			DPRTC_CMD(0x810)
+#define DPRTC_CMDID_CREATE			DPRTC_CMD(0x910)
+#define DPRTC_CMDID_DESTROY			DPRTC_CMD(0x990)
+#define DPRTC_CMDID_GET_API_VERSION		DPRTC_CMD(0xa10)
+
+#define DPRTC_CMDID_ENABLE			DPRTC_CMD(0x002)
+#define DPRTC_CMDID_DISABLE			DPRTC_CMD(0x003)
+#define DPRTC_CMDID_GET_ATTR			DPRTC_CMD(0x004)
+#define DPRTC_CMDID_RESET			DPRTC_CMD(0x005)
+#define DPRTC_CMDID_IS_ENABLED			DPRTC_CMD(0x006)
+
+#define DPRTC_CMDID_SET_IRQ_ENABLE		DPRTC_CMD(0x012)
+#define DPRTC_CMDID_GET_IRQ_ENABLE		DPRTC_CMD(0x013)
+#define DPRTC_CMDID_SET_IRQ_MASK		DPRTC_CMD(0x014)
+#define DPRTC_CMDID_GET_IRQ_MASK		DPRTC_CMD(0x015)
+#define DPRTC_CMDID_GET_IRQ_STATUS		DPRTC_CMD(0x016)
+#define DPRTC_CMDID_CLEAR_IRQ_STATUS		DPRTC_CMD(0x017)
+
+#define DPRTC_CMDID_SET_CLOCK_OFFSET		DPRTC_CMD(0x1d0)
+#define DPRTC_CMDID_SET_FREQ_COMPENSATION	DPRTC_CMD(0x1d1)
+#define DPRTC_CMDID_GET_FREQ_COMPENSATION	DPRTC_CMD(0x1d2)
+#define DPRTC_CMDID_GET_TIME			DPRTC_CMD(0x1d3)
+#define DPRTC_CMDID_SET_TIME			DPRTC_CMD(0x1d4)
+#define DPRTC_CMDID_SET_ALARM			DPRTC_CMD(0x1d5)
+#define DPRTC_CMDID_SET_PERIODIC_PULSE		DPRTC_CMD(0x1d6)
+#define DPRTC_CMDID_CLEAR_PERIODIC_PULSE	DPRTC_CMD(0x1d7)
+#define DPRTC_CMDID_SET_EXT_TRIGGER		DPRTC_CMD(0x1d8)
+#define DPRTC_CMDID_CLEAR_EXT_TRIGGER		DPRTC_CMD(0x1d9)
+#define DPRTC_CMDID_GET_EXT_TRIGGER_TIMESTAMP	DPRTC_CMD(0x1dA)
+
+/* Macros for accessing command fields smaller than 1byte */
+#define DPRTC_MASK(field)        \
+	GENMASK(DPRTC_##field##_SHIFT + DPRTC_##field##_SIZE - 1, \
+		DPRTC_##field##_SHIFT)
+#define dprtc_get_field(var, field)      \
+	(((var) & DPRTC_MASK(field)) >> DPRTC_##field##_SHIFT)
+
+#pragma pack(push, 1)
+struct dprtc_cmd_open {
+	__le32 dprtc_id;
+};
+
+struct dprtc_cmd_destroy {
+	__le32 object_id;
+};
+
+#define DPRTC_ENABLE_SHIFT	0
+#define DPRTC_ENABLE_SIZE	1
+
+struct dprtc_rsp_is_enabled {
+	u8 en;
+};
+
+struct dprtc_cmd_get_irq {
+	__le32 pad;
+	u8 irq_index;
+};
+
+struct dprtc_cmd_set_irq_enable {
+	u8 en;
+	u8 pad[3];
+	u8 irq_index;
+};
+
+struct dprtc_rsp_get_irq_enable {
+	u8 en;
+};
+
+struct dprtc_cmd_set_irq_mask {
+	__le32 mask;
+	u8 irq_index;
+};
+
+struct dprtc_rsp_get_irq_mask {
+	__le32 mask;
+};
+
+struct dprtc_cmd_get_irq_status {
+	__le32 status;
+	u8 irq_index;
+};
+
+struct dprtc_rsp_get_irq_status {
+	__le32 status;
+};
+
+struct dprtc_cmd_clear_irq_status {
+	__le32 status;
+	u8 irq_index;
+};
+
+struct dprtc_rsp_get_attributes {
+	__le32 pad;
+	__le32 id;
+};
+
+struct dprtc_cmd_set_clock_offset {
+	__le64 offset;
+};
+
+struct dprtc_get_freq_compensation {
+	__le32 freq_compensation;
+};
+
+struct dprtc_time {
+	__le64 time;
+};
+
+struct dprtc_rsp_get_api_version {
+	__le16 major;
+	__le16 minor;
+};
+
+#pragma pack(pop)
+
+#endif /* _FSL_DPRTC_CMD_H */
diff --git a/drivers/staging/fsl-dpaa2/rtc/dprtc.c b/drivers/staging/fsl-dpaa2/rtc/dprtc.c
new file mode 100644
index 000000000000..68ae6ffefbf5
--- /dev/null
+++ b/drivers/staging/fsl-dpaa2/rtc/dprtc.c
@@ -0,0 +1,701 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
+ * Copyright 2016-2018 NXP
+ */
+
+#include <linux/fsl/mc.h>
+
+#include "dprtc.h"
+#include "dprtc-cmd.h"
+
+/**
+ * dprtc_open() - Open a control session for the specified object.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @dprtc_id:	DPRTC unique ID
+ * @token:	Returned token; use in subsequent API calls
+ *
+ * This function can be used to open a control session for an
+ * already created object; an object may have been declared in
+ * the DPL or by calling the dprtc_create function.
+ * This function returns a unique authentication token,
+ * associated with the specific object ID and the specific MC
+ * portal; this token must be used in all subsequent commands for
+ * this specific object
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_open(struct fsl_mc_io *mc_io,
+	       u32 cmd_flags,
+	       int dprtc_id,
+	       u16 *token)
+{
+	struct dprtc_cmd_open *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+	int err;
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_OPEN,
+					  cmd_flags,
+					  0);
+	cmd_params = (struct dprtc_cmd_open *)cmd.params;
+	cmd_params->dprtc_id = cpu_to_le32(dprtc_id);
+
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	*token = mc_cmd_hdr_read_token(&cmd);
+
+	return 0;
+}
+
+/**
+ * dprtc_close() - Close the control session of the object
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ *
+ * After this function is called, no further operations are
+ * allowed on the object without opening a new control session.
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_close(struct fsl_mc_io *mc_io,
+		u32 cmd_flags,
+		u16 token)
+{
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_CLOSE, cmd_flags,
+					  token);
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_create() - Create the DPRTC object.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @dprc_token:	Parent container token; '0' for default container
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @cfg:	Configuration structure
+ * @obj_id:	Returned object id
+ *
+ * Create the DPRTC object, allocate required resources and
+ * perform required initialization.
+ *
+ * The function accepts an authentication token of a parent
+ * container that this object should be assigned to. The token
+ * can be '0' so the object will be assigned to the default container.
+ * The newly created object can be opened with the returned
+ * object id and using the container's associated tokens and MC portals.
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_create(struct fsl_mc_io *mc_io,
+		 u16 dprc_token,
+		 u32 cmd_flags,
+		 const struct dprtc_cfg *cfg,
+		 u32 *obj_id)
+{
+	struct fsl_mc_command cmd = { 0 };
+	int err;
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_CREATE,
+					  cmd_flags,
+					  dprc_token);
+
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	*obj_id = mc_cmd_read_object_id(&cmd);
+
+	return 0;
+}
+
+/**
+ * dprtc_destroy() - Destroy the DPRTC object and release all its resources.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @dprc_token: Parent container token; '0' for default container
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @object_id:	The object id; it must be a valid id within the container that
+ * created this object;
+ *
+ * The function accepts the authentication token of the parent container that
+ * created the object (not the one that currently owns the object). The object
+ * is searched within parent using the provided 'object_id'.
+ * All tokens to the object must be closed before calling destroy.
+ *
+ * Return:	'0' on Success; error code otherwise.
+ */
+int dprtc_destroy(struct fsl_mc_io *mc_io,
+		  u16 dprc_token,
+		  u32 cmd_flags,
+		  u32 object_id)
+{
+	struct dprtc_cmd_destroy *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_DESTROY,
+					  cmd_flags,
+					  dprc_token);
+	cmd_params = (struct dprtc_cmd_destroy *)cmd.params;
+	cmd_params->object_id = cpu_to_le32(object_id);
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_enable() - Enable the DPRTC.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_enable(struct fsl_mc_io *mc_io,
+		 u32 cmd_flags,
+		 u16 token)
+{
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_ENABLE, cmd_flags,
+					  token);
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_disable() - Disable the DPRTC.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_disable(struct fsl_mc_io *mc_io,
+		  u32 cmd_flags,
+		  u16 token)
+{
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_DISABLE,
+					  cmd_flags,
+					  token);
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_is_enabled() - Check if the DPRTC is enabled.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @en:		Returns '1' if object is enabled; '0' otherwise
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_is_enabled(struct fsl_mc_io *mc_io,
+		     u32 cmd_flags,
+		     u16 token,
+		     int *en)
+{
+	struct dprtc_rsp_is_enabled *rsp_params;
+	struct fsl_mc_command cmd = { 0 };
+	int err;
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_IS_ENABLED, cmd_flags,
+					  token);
+
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	rsp_params = (struct dprtc_rsp_is_enabled *)cmd.params;
+	*en = dprtc_get_field(rsp_params->en, ENABLE);
+
+	return 0;
+}
+
+/**
+ * dprtc_reset() - Reset the DPRTC, returns the object to initial state.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_reset(struct fsl_mc_io *mc_io,
+		u32 cmd_flags,
+		u16 token)
+{
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_RESET,
+					  cmd_flags,
+					  token);
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_set_irq_enable() - Set overall interrupt state.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @irq_index:	The interrupt index to configure
+ * @en:		Interrupt state - enable = 1, disable = 0
+ *
+ * Allows GPP software to control when interrupts are generated.
+ * Each interrupt can have up to 32 causes.  The enable/disable control's the
+ * overall interrupt state. if the interrupt is disabled no causes will cause
+ * an interrupt.
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_set_irq_enable(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 u8 irq_index,
+			 u8 en)
+{
+	struct dprtc_cmd_set_irq_enable *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_SET_IRQ_ENABLE,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dprtc_cmd_set_irq_enable *)cmd.params;
+	cmd_params->irq_index = irq_index;
+	cmd_params->en = en;
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_get_irq_enable() - Get overall interrupt state
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @irq_index:	The interrupt index to configure
+ * @en:		Returned interrupt state - enable = 1, disable = 0
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_get_irq_enable(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 u8 irq_index,
+			 u8 *en)
+{
+	struct dprtc_rsp_get_irq_enable *rsp_params;
+	struct dprtc_cmd_get_irq *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+	int err;
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_GET_IRQ_ENABLE,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dprtc_cmd_get_irq *)cmd.params;
+	cmd_params->irq_index = irq_index;
+
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	rsp_params = (struct dprtc_rsp_get_irq_enable *)cmd.params;
+	*en = rsp_params->en;
+
+	return 0;
+}
+
+/**
+ * dprtc_set_irq_mask() - Set interrupt mask.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @irq_index:	The interrupt index to configure
+ * @mask:	Event mask to trigger interrupt;
+ *		each bit:
+ *			0 = ignore event
+ *			1 = consider event for asserting IRQ
+ *
+ * Every interrupt can have up to 32 causes and the interrupt model supports
+ * masking/unmasking each cause independently
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_set_irq_mask(struct fsl_mc_io *mc_io,
+		       u32 cmd_flags,
+		       u16 token,
+		       u8 irq_index,
+		       u32 mask)
+{
+	struct dprtc_cmd_set_irq_mask *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_SET_IRQ_MASK,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dprtc_cmd_set_irq_mask *)cmd.params;
+	cmd_params->mask = cpu_to_le32(mask);
+	cmd_params->irq_index = irq_index;
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_get_irq_mask() - Get interrupt mask.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @irq_index:	The interrupt index to configure
+ * @mask:	Returned event mask to trigger interrupt
+ *
+ * Every interrupt can have up to 32 causes and the interrupt model supports
+ * masking/unmasking each cause independently
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_get_irq_mask(struct fsl_mc_io *mc_io,
+		       u32 cmd_flags,
+		       u16 token,
+		       u8 irq_index,
+		       u32 *mask)
+{
+	struct dprtc_rsp_get_irq_mask *rsp_params;
+	struct dprtc_cmd_get_irq *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+	int err;
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_GET_IRQ_MASK,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dprtc_cmd_get_irq *)cmd.params;
+	cmd_params->irq_index = irq_index;
+
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	rsp_params = (struct dprtc_rsp_get_irq_mask *)cmd.params;
+	*mask = le32_to_cpu(rsp_params->mask);
+
+	return 0;
+}
+
+/**
+ * dprtc_get_irq_status() - Get the current status of any pending interrupts.
+ *
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @irq_index:	The interrupt index to configure
+ * @status:	Returned interrupts status - one bit per cause:
+ *			0 = no interrupt pending
+ *			1 = interrupt pending
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_get_irq_status(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 u8 irq_index,
+			 u32 *status)
+{
+	struct dprtc_cmd_get_irq_status *cmd_params;
+	struct dprtc_rsp_get_irq_status *rsp_params;
+	struct fsl_mc_command cmd = { 0 };
+	int err;
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_GET_IRQ_STATUS,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dprtc_cmd_get_irq_status *)cmd.params;
+	cmd_params->status = cpu_to_le32(*status);
+	cmd_params->irq_index = irq_index;
+
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	rsp_params = (struct dprtc_rsp_get_irq_status *)cmd.params;
+	*status = le32_to_cpu(rsp_params->status);
+
+	return 0;
+}
+
+/**
+ * dprtc_clear_irq_status() - Clear a pending interrupt's status
+ *
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @irq_index:	The interrupt index to configure
+ * @status:	Bits to clear (W1C) - one bit per cause:
+ *			0 = don't change
+ *			1 = clear status bit
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_clear_irq_status(struct fsl_mc_io *mc_io,
+			   u32 cmd_flags,
+			   u16 token,
+			   u8 irq_index,
+			   u32 status)
+{
+	struct dprtc_cmd_clear_irq_status *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_CLEAR_IRQ_STATUS,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dprtc_cmd_clear_irq_status *)cmd.params;
+	cmd_params->irq_index = irq_index;
+	cmd_params->status = cpu_to_le32(status);
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_get_attributes - Retrieve DPRTC attributes.
+ *
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @attr:	Returned object's attributes
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_get_attributes(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 struct dprtc_attr *attr)
+{
+	struct dprtc_rsp_get_attributes *rsp_params;
+	struct fsl_mc_command cmd = { 0 };
+	int err;
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_GET_ATTR,
+					  cmd_flags,
+					  token);
+
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	rsp_params = (struct dprtc_rsp_get_attributes *)cmd.params;
+	attr->id = le32_to_cpu(rsp_params->id);
+
+	return 0;
+}
+
+/**
+ * dprtc_set_clock_offset() - Sets the clock's offset
+ * (usually relative to another clock).
+ *
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @offset:	New clock offset (in nanoseconds).
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_set_clock_offset(struct fsl_mc_io *mc_io,
+			   u32 cmd_flags,
+			   u16 token,
+			   int64_t offset)
+{
+	struct dprtc_cmd_set_clock_offset *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_SET_CLOCK_OFFSET,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dprtc_cmd_set_clock_offset *)cmd.params;
+	cmd_params->offset = cpu_to_le64(offset);
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_set_freq_compensation() - Sets a new frequency compensation value.
+ *
+ * @mc_io:		Pointer to MC portal's I/O object
+ * @cmd_flags:		Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:		Token of DPRTC object
+ * @freq_compensation:	The new frequency compensation value to set.
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_set_freq_compensation(struct fsl_mc_io *mc_io,
+				u32 cmd_flags,
+				u16 token,
+				u32 freq_compensation)
+{
+	struct dprtc_get_freq_compensation *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_SET_FREQ_COMPENSATION,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dprtc_get_freq_compensation *)cmd.params;
+	cmd_params->freq_compensation = cpu_to_le32(freq_compensation);
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_get_freq_compensation() - Retrieves the frequency compensation value
+ *
+ * @mc_io:		Pointer to MC portal's I/O object
+ * @cmd_flags:		Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:		Token of DPRTC object
+ * @freq_compensation:	Frequency compensation value
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_get_freq_compensation(struct fsl_mc_io *mc_io,
+				u32 cmd_flags,
+				u16 token,
+				u32 *freq_compensation)
+{
+	struct dprtc_get_freq_compensation *rsp_params;
+	struct fsl_mc_command cmd = { 0 };
+	int err;
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_GET_FREQ_COMPENSATION,
+					  cmd_flags,
+					  token);
+
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	rsp_params = (struct dprtc_get_freq_compensation *)cmd.params;
+	*freq_compensation = le32_to_cpu(rsp_params->freq_compensation);
+
+	return 0;
+}
+
+/**
+ * dprtc_get_time() - Returns the current RTC time.
+ *
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @time:	Current RTC time.
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_get_time(struct fsl_mc_io *mc_io,
+		   u32 cmd_flags,
+		   u16 token,
+		   uint64_t *time)
+{
+	struct dprtc_time *rsp_params;
+	struct fsl_mc_command cmd = { 0 };
+	int err;
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_GET_TIME,
+					  cmd_flags,
+					  token);
+
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	rsp_params = (struct dprtc_time *)cmd.params;
+	*time = le64_to_cpu(rsp_params->time);
+
+	return 0;
+}
+
+/**
+ * dprtc_set_time() - Updates current RTC time.
+ *
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @time:	New RTC time.
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_set_time(struct fsl_mc_io *mc_io,
+		   u32 cmd_flags,
+		   u16 token,
+		   uint64_t time)
+{
+	struct dprtc_time *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_SET_TIME,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dprtc_time *)cmd.params;
+	cmd_params->time = cpu_to_le64(time);
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_set_alarm() - Defines and sets alarm.
+ *
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRTC object
+ * @time:	In nanoseconds, the time when the alarm
+ *			should go off - must be a multiple of
+ *			1 microsecond
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprtc_set_alarm(struct fsl_mc_io *mc_io,
+		    u32 cmd_flags,
+		    u16 token, uint64_t time)
+{
+	struct dprtc_time *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_SET_ALARM,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dprtc_time *)cmd.params;
+	cmd_params->time = cpu_to_le64(time);
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dprtc_get_api_version() - Get Data Path Real Time Counter API version
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @major_ver:	Major version of data path real time counter API
+ * @minor_ver:	Minor version of data path real time counter API
+ *
+ * Return:  '0' on Success; Error code otherwise.
+ */
+int dprtc_get_api_version(struct fsl_mc_io *mc_io,
+			  u32 cmd_flags,
+			  u16 *major_ver,
+			  u16 *minor_ver)
+{
+	struct dprtc_rsp_get_api_version *rsp_params;
+	struct fsl_mc_command cmd = { 0 };
+	int err;
+
+	cmd.header = mc_encode_cmd_header(DPRTC_CMDID_GET_API_VERSION,
+					  cmd_flags,
+					  0);
+
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	rsp_params = (struct dprtc_rsp_get_api_version *)cmd.params;
+	*major_ver = le16_to_cpu(rsp_params->major);
+	*minor_ver = le16_to_cpu(rsp_params->minor);
+
+	return 0;
+}
diff --git a/drivers/staging/fsl-dpaa2/rtc/dprtc.h b/drivers/staging/fsl-dpaa2/rtc/dprtc.h
new file mode 100644
index 000000000000..08f7c7bebbca
--- /dev/null
+++ b/drivers/staging/fsl-dpaa2/rtc/dprtc.h
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
+ * Copyright 2016-2018 NXP
+ */
+
+#ifndef __FSL_DPRTC_H
+#define __FSL_DPRTC_H
+
+/* Data Path Real Time Counter API
+ * Contains initialization APIs and runtime control APIs for RTC
+ */
+
+struct fsl_mc_io;
+
+/**
+ * Number of irq's
+ */
+#define DPRTC_MAX_IRQ_NUM			1
+#define DPRTC_IRQ_INDEX				0
+
+/**
+ * Interrupt event masks:
+ */
+
+/**
+ * Interrupt event mask indicating alarm event had occurred
+ */
+#define DPRTC_EVENT_ALARM			0x40000000
+/**
+ * Interrupt event mask indicating periodic pulse event had occurred
+ */
+#define DPRTC_EVENT_PPS				0x08000000
+
+int dprtc_open(struct fsl_mc_io *mc_io,
+	       u32 cmd_flags,
+	       int dprtc_id,
+	       u16 *token);
+
+int dprtc_close(struct fsl_mc_io *mc_io,
+		u32 cmd_flags,
+		u16 token);
+
+/**
+ * struct dprtc_cfg - Structure representing DPRTC configuration
+ * @options:	place holder
+ */
+struct dprtc_cfg {
+	u32 options;
+};
+
+int dprtc_create(struct fsl_mc_io *mc_io,
+		 u16 dprc_token,
+		 u32 cmd_flags,
+		 const struct dprtc_cfg *cfg,
+		 u32 *obj_id);
+
+int dprtc_destroy(struct fsl_mc_io *mc_io,
+		  u16 dprc_token,
+		  u32 cmd_flags,
+		  u32 object_id);
+
+int dprtc_enable(struct fsl_mc_io *mc_io,
+		 u32 cmd_flags,
+		 u16 token);
+
+int dprtc_disable(struct fsl_mc_io *mc_io,
+		  u32 cmd_flags,
+		  u16 token);
+
+int dprtc_is_enabled(struct fsl_mc_io *mc_io,
+		     u32 cmd_flags,
+		     u16 token,
+		     int *en);
+
+int dprtc_reset(struct fsl_mc_io *mc_io,
+		u32 cmd_flags,
+		u16 token);
+
+int dprtc_set_clock_offset(struct fsl_mc_io *mc_io,
+			   u32 cmd_flags,
+			   u16 token,
+			   int64_t offset);
+
+int dprtc_set_freq_compensation(struct fsl_mc_io *mc_io,
+				u32 cmd_flags,
+				u16 token,
+				u32 freq_compensation);
+
+int dprtc_get_freq_compensation(struct fsl_mc_io *mc_io,
+				u32 cmd_flags,
+				u16 token,
+				u32 *freq_compensation);
+
+int dprtc_get_time(struct fsl_mc_io *mc_io,
+		   u32 cmd_flags,
+		   u16 token,
+		   uint64_t *time);
+
+int dprtc_set_time(struct fsl_mc_io *mc_io,
+		   u32 cmd_flags,
+		   u16 token,
+		   uint64_t time);
+
+int dprtc_set_alarm(struct fsl_mc_io *mc_io,
+		    u32 cmd_flags,
+		    u16 token,
+		    uint64_t time);
+
+int dprtc_set_irq_enable(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 u8 irq_index,
+			 u8 en);
+
+int dprtc_get_irq_enable(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 u8 irq_index,
+			 u8 *en);
+
+int dprtc_set_irq_mask(struct fsl_mc_io *mc_io,
+		       u32 cmd_flags,
+		       u16 token,
+		       u8 irq_index,
+		       u32 mask);
+
+int dprtc_get_irq_mask(struct fsl_mc_io *mc_io,
+		       u32 cmd_flags,
+		       u16 token,
+		       u8 irq_index,
+		       u32 *mask);
+
+int dprtc_get_irq_status(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 u8 irq_index,
+			 u32 *status);
+
+int dprtc_clear_irq_status(struct fsl_mc_io *mc_io,
+			   u32 cmd_flags,
+			   u16 token,
+			   u8 irq_index,
+			   u32 status);
+
+/**
+ * struct dprtc_attr - Structure representing DPRTC attributes
+ * @id:		DPRTC object ID
+ */
+struct dprtc_attr {
+	int id;
+};
+
+int dprtc_get_attributes(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 struct dprtc_attr *attr);
+
+int dprtc_get_api_version(struct fsl_mc_io *mc_io,
+			  u32 cmd_flags,
+			  u16 *major_ver,
+			  u16 *minor_ver);
+
+#endif /* __FSL_DPRTC_H */
diff --git a/drivers/staging/fsl-dpaa2/rtc/rtc.c b/drivers/staging/fsl-dpaa2/rtc/rtc.c
new file mode 100644
index 000000000000..0d52cb85441f
--- /dev/null
+++ b/drivers/staging/fsl-dpaa2/rtc/rtc.c
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
+ * Copyright 2016-2018 NXP
+ */
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/ptp_clock_kernel.h>
+#include <linux/fsl/mc.h>
+
+#include "rtc.h"
+
+struct ptp_dpaa2_priv {
+	struct fsl_mc_device *rtc_mc_dev;
+	struct ptp_clock *clock;
+	struct ptp_clock_info caps;
+	u32 freq_comp;
+};
+
+/* PTP clock operations */
+static int ptp_dpaa2_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
+{
+	struct ptp_dpaa2_priv *ptp_dpaa2 =
+		container_of(ptp, struct ptp_dpaa2_priv, caps);
+	struct fsl_mc_device *mc_dev = ptp_dpaa2->rtc_mc_dev;
+	struct device *dev = &mc_dev->dev;
+	u64 adj;
+	u32 diff, tmr_add;
+	int neg_adj = 0;
+	int err = 0;
+
+	if (ppb < 0) {
+		neg_adj = 1;
+		ppb = -ppb;
+	}
+
+	tmr_add = ptp_dpaa2->freq_comp;
+	adj = tmr_add;
+	adj *= ppb;
+	diff = div_u64(adj, 1000000000ULL);
+
+	tmr_add = neg_adj ? tmr_add - diff : tmr_add + diff;
+
+	err = dprtc_set_freq_compensation(mc_dev->mc_io, 0,
+					  mc_dev->mc_handle, tmr_add);
+	if (err)
+		dev_err(dev, "dprtc_set_freq_compensation err %d\n", err);
+	return 0;
+}
+
+static int ptp_dpaa2_adjtime(struct ptp_clock_info *ptp, s64 delta)
+{
+	struct ptp_dpaa2_priv *ptp_dpaa2 =
+		container_of(ptp, struct ptp_dpaa2_priv, caps);
+	struct fsl_mc_device *mc_dev = ptp_dpaa2->rtc_mc_dev;
+	struct device *dev = &mc_dev->dev;
+	s64 now;
+	int err = 0;
+
+	err = dprtc_get_time(mc_dev->mc_io, 0, mc_dev->mc_handle, &now);
+	if (err) {
+		dev_err(dev, "dprtc_get_time err %d\n", err);
+		return 0;
+	}
+
+	now += delta;
+
+	err = dprtc_set_time(mc_dev->mc_io, 0, mc_dev->mc_handle, now);
+	if (err) {
+		dev_err(dev, "dprtc_set_time err %d\n", err);
+		return 0;
+	}
+	return 0;
+}
+
+static int ptp_dpaa2_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts)
+{
+	struct ptp_dpaa2_priv *ptp_dpaa2 =
+		container_of(ptp, struct ptp_dpaa2_priv, caps);
+	struct fsl_mc_device *mc_dev = ptp_dpaa2->rtc_mc_dev;
+	struct device *dev = &mc_dev->dev;
+	u64 ns;
+	u32 remainder;
+	int err = 0;
+
+	err = dprtc_get_time(mc_dev->mc_io, 0, mc_dev->mc_handle, &ns);
+	if (err) {
+		dev_err(dev, "dprtc_get_time err %d\n", err);
+		return 0;
+	}
+
+	ts->tv_sec = div_u64_rem(ns, 1000000000, &remainder);
+	ts->tv_nsec = remainder;
+	return 0;
+}
+
+static int ptp_dpaa2_settime(struct ptp_clock_info *ptp,
+			     const struct timespec64 *ts)
+{
+	struct ptp_dpaa2_priv *ptp_dpaa2 =
+		container_of(ptp, struct ptp_dpaa2_priv, caps);
+	struct fsl_mc_device *mc_dev = ptp_dpaa2->rtc_mc_dev;
+	struct device *dev = &mc_dev->dev;
+	u64 ns;
+	int err = 0;
+
+	ns = ts->tv_sec * 1000000000ULL;
+	ns += ts->tv_nsec;
+
+	err = dprtc_set_time(mc_dev->mc_io, 0, mc_dev->mc_handle, ns);
+	if (err)
+		dev_err(dev, "dprtc_set_time err %d\n", err);
+	return 0;
+}
+
+static struct ptp_clock_info ptp_dpaa2_caps = {
+	.owner		= THIS_MODULE,
+	.name		= "DPAA2 PTP Clock",
+	.max_adj	= 512000,
+	.n_alarm	= 2,
+	.n_ext_ts	= 2,
+	.n_per_out	= 3,
+	.n_pins		= 0,
+	.pps		= 1,
+	.adjfreq	= ptp_dpaa2_adjfreq,
+	.adjtime	= ptp_dpaa2_adjtime,
+	.gettime64	= ptp_dpaa2_gettime,
+	.settime64	= ptp_dpaa2_settime,
+};
+
+static int rtc_probe(struct fsl_mc_device *mc_dev)
+{
+	struct device *dev = &mc_dev->dev;
+	struct ptp_dpaa2_priv *ptp_dpaa2;
+	u32 tmr_add = 0;
+	int err;
+
+	ptp_dpaa2 = kzalloc(sizeof(*ptp_dpaa2), GFP_KERNEL);
+	if (!ptp_dpaa2)
+		return -ENOMEM;
+
+	err = fsl_mc_portal_allocate(mc_dev, 0, &mc_dev->mc_io);
+	if (err) {
+		dev_err(dev, "fsl_mc_portal_allocate err %d\n", err);
+		goto err_exit;
+	}
+
+	err = dprtc_open(mc_dev->mc_io, 0, mc_dev->obj_desc.id,
+			 &mc_dev->mc_handle);
+	if (err) {
+		dev_err(dev, "dprtc_open err %d\n", err);
+		goto err_free_mcp;
+	}
+
+	ptp_dpaa2->rtc_mc_dev = mc_dev;
+
+	err = dprtc_get_freq_compensation(mc_dev->mc_io, 0,
+					  mc_dev->mc_handle, &tmr_add);
+	if (err) {
+		dev_err(dev, "dprtc_get_freq_compensation err %d\n", err);
+		goto err_close;
+	}
+
+	ptp_dpaa2->freq_comp = tmr_add;
+	ptp_dpaa2->caps = ptp_dpaa2_caps;
+
+	ptp_dpaa2->clock = ptp_clock_register(&ptp_dpaa2->caps, dev);
+	if (IS_ERR(ptp_dpaa2->clock)) {
+		err = PTR_ERR(ptp_dpaa2->clock);
+		goto err_close;
+	}
+
+	dpaa2_phc_index = ptp_clock_index(ptp_dpaa2->clock);
+
+	dev_set_drvdata(dev, ptp_dpaa2);
+
+	return 0;
+
+err_close:
+	dprtc_close(mc_dev->mc_io, 0, mc_dev->mc_handle);
+err_free_mcp:
+	fsl_mc_portal_free(mc_dev->mc_io);
+err_exit:
+	kfree(ptp_dpaa2);
+	dev_set_drvdata(dev, NULL);
+	return err;
+}
+
+static int rtc_remove(struct fsl_mc_device *mc_dev)
+{
+	struct ptp_dpaa2_priv *ptp_dpaa2;
+	struct device *dev = &mc_dev->dev;
+
+	ptp_dpaa2 = dev_get_drvdata(dev);
+	ptp_clock_unregister(ptp_dpaa2->clock);
+
+	dprtc_close(mc_dev->mc_io, 0, mc_dev->mc_handle);
+	fsl_mc_portal_free(mc_dev->mc_io);
+
+	kfree(ptp_dpaa2);
+	dev_set_drvdata(dev, NULL);
+
+	return 0;
+}
+
+static const struct fsl_mc_device_id rtc_match_id_table[] = {
+	{
+		.vendor = FSL_MC_VENDOR_FREESCALE,
+		.obj_type = "dprtc",
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(fslmc, rtc_match_id_table);
+
+static struct fsl_mc_driver rtc_drv = {
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.owner = THIS_MODULE,
+	},
+	.probe = rtc_probe,
+	.remove = rtc_remove,
+	.match_id_table = rtc_match_id_table,
+};
+
+module_fsl_mc_driver(rtc_drv);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("DPAA2 PTP Clock Driver");
diff --git a/drivers/staging/fsl-dpaa2/rtc/rtc.h b/drivers/staging/fsl-dpaa2/rtc/rtc.h
new file mode 100644
index 000000000000..ff2e177395d4
--- /dev/null
+++ b/drivers/staging/fsl-dpaa2/rtc/rtc.h
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018 NXP
+ */
+
+#ifndef __RTC_H
+#define __RTC_H
+
+#include "dprtc.h"
+#include "dprtc-cmd.h"
+
+extern int dpaa2_phc_index;
+
+#endif
diff --git a/drivers/staging/fsl-mc/bus/dpio/dpio-driver.c b/drivers/staging/fsl-mc/bus/dpio/dpio-driver.c
index 182b38412a82..11a90a90d827 100644
--- a/drivers/staging/fsl-mc/bus/dpio/dpio-driver.c
+++ b/drivers/staging/fsl-mc/bus/dpio/dpio-driver.c
@@ -13,6 +13,7 @@
 #include <linux/msi.h>
 #include <linux/dma-mapping.h>
 #include <linux/delay.h>
+#include <linux/io.h>
 
 #include <linux/fsl/mc.h>
 #include "../../include/dpaa2-io.h"
@@ -100,7 +101,7 @@ static int dpaa2_dpio_probe(struct fsl_mc_device *dpio_dev)
 	if (err) {
 		dev_dbg(dev, "MC portal allocation failed\n");
 		err = -EPROBE_DEFER;
-		goto err_mcportal;
+		goto err_priv_alloc;
 	}
 
 	err = dpio_open(dpio_dev->mc_io, 0, dpio_dev->obj_desc.id,
@@ -146,10 +147,22 @@ static int dpaa2_dpio_probe(struct fsl_mc_device *dpio_dev)
 	 * Set the CENA regs to be the cache inhibited area of the portal to
 	 * avoid coherency issues if a user migrates to another core.
 	 */
-	desc.regs_cena = ioremap_wc(dpio_dev->regions[1].start,
-		resource_size(&dpio_dev->regions[1]));
-	desc.regs_cinh = ioremap(dpio_dev->regions[1].start,
-		resource_size(&dpio_dev->regions[1]));
+	desc.regs_cena = devm_memremap(dev, dpio_dev->regions[1].start,
+				       resource_size(&dpio_dev->regions[1]),
+				       MEMREMAP_WC);
+	if (IS_ERR(desc.regs_cena)) {
+		dev_err(dev, "devm_memremap failed\n");
+		err = PTR_ERR(desc.regs_cena);
+		goto err_allocate_irqs;
+	}
+
+	desc.regs_cinh = devm_ioremap(dev, dpio_dev->regions[1].start,
+				      resource_size(&dpio_dev->regions[1]));
+	if (!desc.regs_cinh) {
+		err = -ENOMEM;
+		dev_err(dev, "devm_ioremap failed\n");
+		goto err_allocate_irqs;
+	}
 
 	err = fsl_mc_allocate_irqs(dpio_dev);
 	if (err) {
@@ -164,6 +177,7 @@ static int dpaa2_dpio_probe(struct fsl_mc_device *dpio_dev)
 	priv->io = dpaa2_io_create(&desc);
 	if (!priv->io) {
 		dev_err(dev, "dpaa2_io_create failed\n");
+		err = -ENOMEM;
 		goto err_dpaa2_io_create;
 	}
 
@@ -185,8 +199,6 @@ static int dpaa2_dpio_probe(struct fsl_mc_device *dpio_dev)
 	dpio_close(dpio_dev->mc_io, 0, dpio_dev->mc_handle);
 err_open:
 	fsl_mc_portal_free(dpio_dev->mc_io);
-err_mcportal:
-	dev_set_drvdata(dev, NULL);
 err_priv_alloc:
 	return err;
 }
@@ -230,8 +242,6 @@ static int dpaa2_dpio_remove(struct fsl_mc_device *dpio_dev)
 
 	fsl_mc_portal_free(dpio_dev->mc_io);
 
-	dev_set_drvdata(dev, NULL);
-
 	return 0;
 
 err_open:
diff --git a/drivers/staging/fsl-mc/bus/dpio/qbman-portal.h b/drivers/staging/fsl-mc/bus/dpio/qbman-portal.h
index 4488a445b709..69db3c818742 100644
--- a/drivers/staging/fsl-mc/bus/dpio/qbman-portal.h
+++ b/drivers/staging/fsl-mc/bus/dpio/qbman-portal.h
@@ -15,7 +15,7 @@ struct qbman_swp;
 /* qbman software portal descriptor structure */
 struct qbman_swp_desc {
 	void *cena_bar; /* Cache-enabled portal base address */
-	void *cinh_bar; /* Cache-inhibited portal base address */
+	void __iomem *cinh_bar; /* Cache-inhibited portal base address */
 	u32 qman_version;
 };
 
@@ -102,7 +102,7 @@ struct qbman_release_desc {
 /* portal data structure */
 struct qbman_swp {
 	const struct qbman_swp_desc *desc;
-	void __iomem *addr_cena;
+	void *addr_cena;
 	void __iomem *addr_cinh;
 
 	/* Management commands */
diff --git a/drivers/staging/fsl-mc/include/dpaa2-io.h b/drivers/staging/fsl-mc/include/dpaa2-io.h
index f71227d3df8d..ab51e40d11db 100644
--- a/drivers/staging/fsl-mc/include/dpaa2-io.h
+++ b/drivers/staging/fsl-mc/include/dpaa2-io.h
@@ -52,7 +52,7 @@ struct dpaa2_io_desc {
 	int has_8prio;
 	int cpu;
 	void *regs_cena;
-	void *regs_cinh;
+	void __iomem *regs_cinh;
 	int dpio_id;
 	u32 qman_version;
 };
diff --git a/drivers/staging/fwserial/fwserial.c b/drivers/staging/fwserial/fwserial.c
index e8bfe5520bc7..fa0dd425b454 100644
--- a/drivers/staging/fwserial/fwserial.c
+++ b/drivers/staging/fwserial/fwserial.c
@@ -1506,11 +1506,6 @@ static int fwtty_debugfs_peers_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int fwtty_proc_open(struct inode *inode, struct file *fp)
-{
-	return single_open(fp, fwtty_proc_show, NULL);
-}
-
 static int fwtty_stats_open(struct inode *inode, struct file *fp)
 {
 	return single_open(fp, fwtty_debugfs_stats_show, inode->i_private);
@@ -1537,14 +1532,6 @@ static const struct file_operations fwtty_peers_fops = {
 	.release =	single_release,
 };
 
-static const struct file_operations fwtty_proc_fops = {
-	.owner =        THIS_MODULE,
-	.open =         fwtty_proc_open,
-	.read =         seq_read,
-	.llseek =       seq_lseek,
-	.release =      single_release,
-};
-
 static const struct tty_port_operations fwtty_port_ops = {
 	.dtr_rts =		fwtty_port_dtr_rts,
 	.carrier_raised =	fwtty_port_carrier_raised,
@@ -1570,7 +1557,7 @@ static const struct tty_operations fwtty_ops = {
 	.tiocmget =		fwtty_tiocmget,
 	.tiocmset =		fwtty_tiocmset,
 	.get_icount =		fwtty_get_icount,
-	.proc_fops =		&fwtty_proc_fops,
+	.proc_show =		fwtty_proc_show,
 };
 
 static const struct tty_operations fwloop_ops = {
diff --git a/drivers/staging/gdm724x/gdm_endian.c b/drivers/staging/gdm724x/gdm_endian.c
index 4200391b1a97..ae39e59daf70 100644
--- a/drivers/staging/gdm724x/gdm_endian.c
+++ b/drivers/staging/gdm724x/gdm_endian.c
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #include <linux/kernel.h>
 #include "gdm_endian.h"
diff --git a/drivers/staging/gdm724x/gdm_endian.h b/drivers/staging/gdm724x/gdm_endian.h
index e58d29f868ba..f373dc3a19bf 100644
--- a/drivers/staging/gdm724x/gdm_endian.h
+++ b/drivers/staging/gdm724x/gdm_endian.h
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #ifndef __GDM_ENDIAN_H__
 #define __GDM_ENDIAN_H__
diff --git a/drivers/staging/gdm724x/gdm_lte.c b/drivers/staging/gdm724x/gdm_lte.c
index 4f3c518304f2..3c2aab7a921e 100644
--- a/drivers/staging/gdm724x/gdm_lte.c
+++ b/drivers/staging/gdm724x/gdm_lte.c
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
@@ -408,7 +398,7 @@ static s32 gdm_lte_tx_nic_type(struct net_device *dev, struct sk_buff *skb)
 	return nic_type;
 }
 
-static int gdm_lte_tx(struct sk_buff *skb, struct net_device *dev)
+static netdev_tx_t gdm_lte_tx(struct sk_buff *skb, struct net_device *dev)
 {
 	struct nic *nic = netdev_priv(dev);
 	u32 nic_type;
diff --git a/drivers/staging/gdm724x/gdm_lte.h b/drivers/staging/gdm724x/gdm_lte.h
index bad0855e4721..f2143a6e0e99 100644
--- a/drivers/staging/gdm724x/gdm_lte.h
+++ b/drivers/staging/gdm724x/gdm_lte.h
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #ifndef _GDM_LTE_H_
 #define _GDM_LTE_H_
diff --git a/drivers/staging/gdm724x/gdm_mux.c b/drivers/staging/gdm724x/gdm_mux.c
index 63921bad519e..e2a050ba6fbb 100644
--- a/drivers/staging/gdm724x/gdm_mux.c
+++ b/drivers/staging/gdm724x/gdm_mux.c
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
diff --git a/drivers/staging/gdm724x/gdm_mux.h b/drivers/staging/gdm724x/gdm_mux.h
index 0871b8feec55..51c22e3d8aeb 100644
--- a/drivers/staging/gdm724x/gdm_mux.h
+++ b/drivers/staging/gdm724x/gdm_mux.h
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #ifndef _GDM_MUX_H_
 #define _GDM_MUX_H_
diff --git a/drivers/staging/gdm724x/gdm_tty.c b/drivers/staging/gdm724x/gdm_tty.c
index 3cdebb81ba63..bf554f7c56ca 100644
--- a/drivers/staging/gdm724x/gdm_tty.c
+++ b/drivers/staging/gdm724x/gdm_tty.c
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
diff --git a/drivers/staging/gdm724x/gdm_tty.h b/drivers/staging/gdm724x/gdm_tty.h
index 195c5902989f..afec97ced476 100644
--- a/drivers/staging/gdm724x/gdm_tty.h
+++ b/drivers/staging/gdm724x/gdm_tty.h
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #ifndef _GDM_TTY_H_
 #define _GDM_TTY_H_
diff --git a/drivers/staging/gdm724x/gdm_usb.c b/drivers/staging/gdm724x/gdm_usb.c
index c95bad4a8615..0218782d1a08 100644
--- a/drivers/staging/gdm724x/gdm_usb.c
+++ b/drivers/staging/gdm724x/gdm_usb.c
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
diff --git a/drivers/staging/gdm724x/gdm_usb.h b/drivers/staging/gdm724x/gdm_usb.h
index 701038685e23..db689b091c4f 100644
--- a/drivers/staging/gdm724x/gdm_usb.h
+++ b/drivers/staging/gdm724x/gdm_usb.h
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #ifndef _GDM_USB_H_
 #define _GDM_USB_H_
diff --git a/drivers/staging/gdm724x/hci.h b/drivers/staging/gdm724x/hci.h
index 9a591b0db516..b30945daf3a5 100644
--- a/drivers/staging/gdm724x/hci.h
+++ b/drivers/staging/gdm724x/hci.h
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #ifndef _HCI_H_
 #define _HCI_H_
diff --git a/drivers/staging/gdm724x/hci_packet.h b/drivers/staging/gdm724x/hci_packet.h
index 22ce8b9477b6..83fbd2515467 100644
--- a/drivers/staging/gdm724x/hci_packet.h
+++ b/drivers/staging/gdm724x/hci_packet.h
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #ifndef _HCI_PACKET_H_
 #define _HCI_PACKET_H_
diff --git a/drivers/staging/gdm724x/netlink_k.c b/drivers/staging/gdm724x/netlink_k.c
index abe242505882..92440c3f055b 100644
--- a/drivers/staging/gdm724x/netlink_k.c
+++ b/drivers/staging/gdm724x/netlink_k.c
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
diff --git a/drivers/staging/gdm724x/netlink_k.h b/drivers/staging/gdm724x/netlink_k.h
index 5ebd73157f5a..c9e1d3b2d54f 100644
--- a/drivers/staging/gdm724x/netlink_k.h
+++ b/drivers/staging/gdm724x/netlink_k.h
@@ -1,15 +1,5 @@
-/*
- * Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2012 GCT Semiconductor, Inc. All rights reserved. */
 
 #ifndef _NETLINK_K_H
 #define _NETLINK_K_H
diff --git a/drivers/staging/goldfish/Kconfig b/drivers/staging/goldfish/Kconfig
index 4e094602437c..9165385df9de 100644
--- a/drivers/staging/goldfish/Kconfig
+++ b/drivers/staging/goldfish/Kconfig
@@ -1,13 +1,6 @@
 config GOLDFISH_AUDIO
 	tristate "Goldfish AVD Audio Device"
 	depends on GOLDFISH
-	---help---
+	help
 	  Emulated audio channel for the Goldfish Android Virtual Device
 
-config MTD_GOLDFISH_NAND
-	tristate "Goldfish NAND device"
-	depends on GOLDFISH
-	depends on MTD
-	help
-	  Drives the emulated NAND flash device on the Google Goldfish
-	  Android virtual device.
diff --git a/drivers/staging/goldfish/Makefile b/drivers/staging/goldfish/Makefile
index dec34ad58162..054eeb82151e 100644
--- a/drivers/staging/goldfish/Makefile
+++ b/drivers/staging/goldfish/Makefile
@@ -3,4 +3,3 @@
 #
 
 obj-$(CONFIG_GOLDFISH_AUDIO) += goldfish_audio.o
-obj-$(CONFIG_MTD_GOLDFISH_NAND)	+= goldfish_nand.o
diff --git a/drivers/staging/goldfish/goldfish_nand.c b/drivers/staging/goldfish/goldfish_nand.c
deleted file mode 100644
index f5e002ecba75..000000000000
--- a/drivers/staging/goldfish/goldfish_nand.c
+++ /dev/null
@@ -1,441 +0,0 @@
-/*
- * drivers/mtd/devices/goldfish_nand.c
- *
- * Copyright (C) 2007 Google, Inc.
- * Copyright (C) 2012 Intel, Inc.
- * Copyright (C) 2013 Intel, Inc.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- */
-
-#include <linux/io.h>
-#include <linux/device.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/ioport.h>
-#include <linux/vmalloc.h>
-#include <linux/mtd/mtd.h>
-#include <linux/platform_device.h>
-#include <linux/mutex.h>
-#include <linux/goldfish.h>
-#include <asm/div64.h>
-#include <linux/dma-mapping.h>
-
-#include "goldfish_nand_reg.h"
-
-struct goldfish_nand {
-	/* lock protects access to the device registers */
-	struct mutex            lock;
-	unsigned char __iomem  *base;
-	struct cmd_params       *cmd_params;
-	size_t                  mtd_count;
-	struct mtd_info         mtd[0];
-};
-
-static u32 goldfish_nand_cmd_with_params(struct mtd_info *mtd,
-					 enum nand_cmd cmd, u64 addr, u32 len,
-					 void *ptr, u32 *rv)
-{
-	u32 cmdp;
-	struct goldfish_nand *nand = mtd->priv;
-	struct cmd_params *cps = nand->cmd_params;
-	unsigned char __iomem  *base = nand->base;
-
-	if (!cps)
-		return -1;
-
-	switch (cmd) {
-	case NAND_CMD_ERASE:
-		cmdp = NAND_CMD_ERASE_WITH_PARAMS;
-		break;
-	case NAND_CMD_READ:
-		cmdp = NAND_CMD_READ_WITH_PARAMS;
-		break;
-	case NAND_CMD_WRITE:
-		cmdp = NAND_CMD_WRITE_WITH_PARAMS;
-		break;
-	default:
-		return -1;
-	}
-	cps->dev = mtd - nand->mtd;
-	cps->addr_high = (u32)(addr >> 32);
-	cps->addr_low = (u32)addr;
-	cps->transfer_size = len;
-	cps->data = (unsigned long)ptr;
-	writel(cmdp, base + NAND_COMMAND);
-	*rv = cps->result;
-	return 0;
-}
-
-static u32 goldfish_nand_cmd(struct mtd_info *mtd, enum nand_cmd cmd,
-			     u64 addr, u32 len, void *ptr)
-{
-	struct goldfish_nand *nand = mtd->priv;
-	u32 rv;
-	unsigned char __iomem  *base = nand->base;
-
-	mutex_lock(&nand->lock);
-	if (goldfish_nand_cmd_with_params(mtd, cmd, addr, len, ptr, &rv)) {
-		writel(mtd - nand->mtd, base + NAND_DEV);
-		writel((u32)(addr >> 32), base + NAND_ADDR_HIGH);
-		writel((u32)addr, base + NAND_ADDR_LOW);
-		writel(len, base + NAND_TRANSFER_SIZE);
-		gf_write_ptr(ptr, base + NAND_DATA, base + NAND_DATA_HIGH);
-		writel(cmd, base + NAND_COMMAND);
-		rv = readl(base + NAND_RESULT);
-	}
-	mutex_unlock(&nand->lock);
-	return rv;
-}
-
-static int goldfish_nand_erase(struct mtd_info *mtd, struct erase_info *instr)
-{
-	loff_t ofs = instr->addr;
-	u32 len = instr->len;
-	s32 rem;
-
-	if (ofs + len > mtd->size)
-		goto invalid_arg;
-	ofs = div_s64_rem(ofs, mtd->writesize, &rem);
-	if (rem)
-		goto invalid_arg;
-	ofs *= (mtd->writesize + mtd->oobsize);
-
-	if (len % mtd->writesize)
-		goto invalid_arg;
-	len = len / mtd->writesize * (mtd->writesize + mtd->oobsize);
-
-	if (goldfish_nand_cmd(mtd, NAND_CMD_ERASE, ofs, len, NULL) != len) {
-		pr_err("%s: erase failed, start %llx, len %x, dev_size %llx, erase_size %x\n",
-		       __func__, ofs, len, mtd->size, mtd->erasesize);
-		return -EIO;
-	}
-
-	return 0;
-
-invalid_arg:
-	pr_err("%s: invalid erase, start %llx, len %x, dev_size %llx, erase_size %x\n",
-	       __func__, ofs, len, mtd->size, mtd->erasesize);
-	return -EINVAL;
-}
-
-static int goldfish_nand_read_oob(struct mtd_info *mtd, loff_t ofs,
-				  struct mtd_oob_ops *ops)
-{
-	s32 rem;
-
-	if (ofs + ops->len > mtd->size)
-		goto invalid_arg;
-	if (ops->datbuf && ops->len && ops->len != mtd->writesize)
-		goto invalid_arg;
-	if (ops->ooblen + ops->ooboffs > mtd->oobsize)
-		goto invalid_arg;
-
-	ofs = div_s64_rem(ofs, mtd->writesize, &rem);
-	if (rem)
-		goto invalid_arg;
-	ofs *= (mtd->writesize + mtd->oobsize);
-
-	if (ops->datbuf)
-		ops->retlen = goldfish_nand_cmd(mtd, NAND_CMD_READ, ofs,
-						ops->len, ops->datbuf);
-	ofs += mtd->writesize + ops->ooboffs;
-	if (ops->oobbuf)
-		ops->oobretlen = goldfish_nand_cmd(mtd, NAND_CMD_READ, ofs,
-						   ops->ooblen, ops->oobbuf);
-	return 0;
-
-invalid_arg:
-	pr_err("%s: invalid read, start %llx, len %zx, ooblen %zx, dev_size %llx, write_size %x\n",
-	       __func__, ofs, ops->len, ops->ooblen, mtd->size, mtd->writesize);
-	return -EINVAL;
-}
-
-static int goldfish_nand_write_oob(struct mtd_info *mtd, loff_t ofs,
-				   struct mtd_oob_ops *ops)
-{
-	s32 rem;
-
-	if (ofs + ops->len > mtd->size)
-		goto invalid_arg;
-	if (ops->len && ops->len != mtd->writesize)
-		goto invalid_arg;
-	if (ops->ooblen + ops->ooboffs > mtd->oobsize)
-		goto invalid_arg;
-
-	ofs = div_s64_rem(ofs, mtd->writesize, &rem);
-	if (rem)
-		goto invalid_arg;
-	ofs *= (mtd->writesize + mtd->oobsize);
-
-	if (ops->datbuf)
-		ops->retlen = goldfish_nand_cmd(mtd, NAND_CMD_WRITE, ofs,
-						ops->len, ops->datbuf);
-	ofs += mtd->writesize + ops->ooboffs;
-	if (ops->oobbuf)
-		ops->oobretlen = goldfish_nand_cmd(mtd, NAND_CMD_WRITE, ofs,
-						   ops->ooblen, ops->oobbuf);
-	return 0;
-
-invalid_arg:
-	pr_err("%s: invalid write, start %llx, len %zx, ooblen %zx, dev_size %llx, write_size %x\n",
-	       __func__, ofs, ops->len, ops->ooblen, mtd->size, mtd->writesize);
-	return -EINVAL;
-}
-
-static int goldfish_nand_read(struct mtd_info *mtd, loff_t from, size_t len,
-			      size_t *retlen, u_char *buf)
-{
-	s32 rem;
-
-	if (from + len > mtd->size)
-		goto invalid_arg;
-
-	from = div_s64_rem(from, mtd->writesize, &rem);
-	if (rem)
-		goto invalid_arg;
-	from *= (mtd->writesize + mtd->oobsize);
-
-	*retlen = goldfish_nand_cmd(mtd, NAND_CMD_READ, from, len, buf);
-	return 0;
-
-invalid_arg:
-	pr_err("%s: invalid read, start %llx, len %zx, dev_size %llx, write_size %x\n",
-	       __func__, from, len, mtd->size, mtd->writesize);
-	return -EINVAL;
-}
-
-static int goldfish_nand_write(struct mtd_info *mtd, loff_t to, size_t len,
-			       size_t *retlen, const u_char *buf)
-{
-	s32 rem;
-
-	if (to + len > mtd->size)
-		goto invalid_arg;
-
-	to = div_s64_rem(to, mtd->writesize, &rem);
-	if (rem)
-		goto invalid_arg;
-	to *= (mtd->writesize + mtd->oobsize);
-
-	*retlen = goldfish_nand_cmd(mtd, NAND_CMD_WRITE, to, len, (void *)buf);
-	return 0;
-
-invalid_arg:
-	pr_err("%s: invalid write, start %llx, len %zx, dev_size %llx, write_size %x\n",
-	       __func__, to, len, mtd->size, mtd->writesize);
-	return -EINVAL;
-}
-
-static int goldfish_nand_block_isbad(struct mtd_info *mtd, loff_t ofs)
-{
-	s32 rem;
-
-	if (ofs >= mtd->size)
-		goto invalid_arg;
-
-	ofs = div_s64_rem(ofs, mtd->writesize, &rem);
-	if (rem)
-		goto invalid_arg;
-	ofs *= mtd->erasesize / mtd->writesize;
-	ofs *= (mtd->writesize + mtd->oobsize);
-
-	return goldfish_nand_cmd(mtd, NAND_CMD_BLOCK_BAD_GET, ofs, 0, NULL);
-
-invalid_arg:
-	pr_err("%s: invalid arg, ofs %llx, dev_size %llx, write_size %x\n",
-	       __func__, ofs, mtd->size, mtd->writesize);
-	return -EINVAL;
-}
-
-static int goldfish_nand_block_markbad(struct mtd_info *mtd, loff_t ofs)
-{
-	s32 rem;
-
-	if (ofs >= mtd->size)
-		goto invalid_arg;
-
-	ofs = div_s64_rem(ofs, mtd->writesize, &rem);
-	if (rem)
-		goto invalid_arg;
-	ofs *= mtd->erasesize / mtd->writesize;
-	ofs *= (mtd->writesize + mtd->oobsize);
-
-	if (goldfish_nand_cmd(mtd, NAND_CMD_BLOCK_BAD_SET, ofs, 0, NULL) != 1)
-		return -EIO;
-	return 0;
-
-invalid_arg:
-	pr_err("%s: invalid arg, ofs %llx, dev_size %llx, write_size %x\n",
-	       __func__, ofs, mtd->size, mtd->writesize);
-	return -EINVAL;
-}
-
-static int nand_setup_cmd_params(struct platform_device *pdev,
-				 struct goldfish_nand *nand)
-{
-	dma_addr_t dma_handle;
-	unsigned char __iomem  *base = nand->base;
-
-	nand->cmd_params = dmam_alloc_coherent(&pdev->dev,
-					       sizeof(struct cmd_params),
-					       &dma_handle, GFP_KERNEL);
-	if (!nand->cmd_params) {
-		dev_err(&pdev->dev, "allocate buffer failed\n");
-		return -ENOMEM;
-	}
-	writel((u32)((u64)dma_handle >> 32), base + NAND_CMD_PARAMS_ADDR_HIGH);
-	writel((u32)dma_handle, base + NAND_CMD_PARAMS_ADDR_LOW);
-	return 0;
-}
-
-static int goldfish_nand_init_device(struct platform_device *pdev,
-				     struct goldfish_nand *nand, int id)
-{
-	u32 name_len;
-	u32 result;
-	u32 flags;
-	unsigned char __iomem  *base = nand->base;
-	struct mtd_info *mtd = &nand->mtd[id];
-	char *name;
-
-	mutex_lock(&nand->lock);
-	writel(id, base + NAND_DEV);
-	flags = readl(base + NAND_DEV_FLAGS);
-	name_len = readl(base + NAND_DEV_NAME_LEN);
-	mtd->writesize = readl(base + NAND_DEV_PAGE_SIZE);
-	mtd->size = readl(base + NAND_DEV_SIZE_LOW);
-	mtd->size |= (u64)readl(base + NAND_DEV_SIZE_HIGH) << 32;
-	mtd->oobsize = readl(base + NAND_DEV_EXTRA_SIZE);
-	mtd->oobavail = mtd->oobsize;
-	mtd->erasesize = readl(base + NAND_DEV_ERASE_SIZE) /
-			(mtd->writesize + mtd->oobsize) * mtd->writesize;
-	mtd->size = div_s64(mtd->size, mtd->writesize + mtd->oobsize);
-	mtd->size *= mtd->writesize;
-	dev_dbg(&pdev->dev,
-		"goldfish nand dev%d: size %llx, page %d, extra %d, erase %d\n",
-		       id, mtd->size, mtd->writesize,
-		       mtd->oobsize, mtd->erasesize);
-	mutex_unlock(&nand->lock);
-
-	mtd->priv = nand;
-
-	name = devm_kzalloc(&pdev->dev, name_len + 1, GFP_KERNEL);
-	if (!name)
-		return -ENOMEM;
-	mtd->name = name;
-
-	result = goldfish_nand_cmd(mtd, NAND_CMD_GET_DEV_NAME, 0, name_len,
-				   name);
-	if (result != name_len) {
-		dev_err(&pdev->dev,
-			"%s: failed to get dev name %d != %d\n",
-			       __func__, result, name_len);
-		return -ENODEV;
-	}
-	((char *)mtd->name)[name_len] = '\0';
-
-	/* Setup the MTD structure */
-	mtd->type = MTD_NANDFLASH;
-	mtd->flags = MTD_CAP_NANDFLASH;
-	if (flags & NAND_DEV_FLAG_READ_ONLY)
-		mtd->flags &= ~MTD_WRITEABLE;
-	if (flags & NAND_DEV_FLAG_CMD_PARAMS_CAP)
-		nand_setup_cmd_params(pdev, nand);
-
-	mtd->owner = THIS_MODULE;
-	mtd->_erase = goldfish_nand_erase;
-	mtd->_read = goldfish_nand_read;
-	mtd->_write = goldfish_nand_write;
-	mtd->_read_oob = goldfish_nand_read_oob;
-	mtd->_write_oob = goldfish_nand_write_oob;
-	mtd->_block_isbad = goldfish_nand_block_isbad;
-	mtd->_block_markbad = goldfish_nand_block_markbad;
-
-	if (mtd_device_register(mtd, NULL, 0))
-		return -EIO;
-
-	return 0;
-}
-
-static int goldfish_nand_probe(struct platform_device *pdev)
-{
-	u32 num_dev;
-	int i;
-	int err;
-	u32 num_dev_working;
-	u32 version;
-	struct resource *r;
-	struct goldfish_nand *nand;
-	unsigned char __iomem  *base;
-
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!r)
-		return -ENODEV;
-
-	base = devm_ioremap(&pdev->dev, r->start, PAGE_SIZE);
-	if (!base)
-		return -ENOMEM;
-
-	version = readl(base + NAND_VERSION);
-	if (version != NAND_VERSION_CURRENT) {
-		dev_err(&pdev->dev,
-			"goldfish_nand_init: version mismatch, got %d, expected %d\n",
-				version, NAND_VERSION_CURRENT);
-		return -ENODEV;
-	}
-	num_dev = readl(base + NAND_NUM_DEV);
-	if (num_dev == 0)
-		return -ENODEV;
-
-	nand = devm_kzalloc(&pdev->dev, sizeof(*nand) +
-				sizeof(struct mtd_info) * num_dev, GFP_KERNEL);
-	if (!nand)
-		return -ENOMEM;
-
-	mutex_init(&nand->lock);
-	nand->base = base;
-	nand->mtd_count = num_dev;
-	platform_set_drvdata(pdev, nand);
-
-	num_dev_working = 0;
-	for (i = 0; i < num_dev; i++) {
-		err = goldfish_nand_init_device(pdev, nand, i);
-		if (err == 0)
-			num_dev_working++;
-	}
-	if (num_dev_working == 0)
-		return -ENODEV;
-	return 0;
-}
-
-static int goldfish_nand_remove(struct platform_device *pdev)
-{
-	struct goldfish_nand *nand = platform_get_drvdata(pdev);
-	int i;
-
-	for (i = 0; i < nand->mtd_count; i++) {
-		if (nand->mtd[i].name)
-			mtd_device_unregister(&nand->mtd[i]);
-	}
-	return 0;
-}
-
-static struct platform_driver goldfish_nand_driver = {
-	.probe		= goldfish_nand_probe,
-	.remove		= goldfish_nand_remove,
-	.driver = {
-		.name = "goldfish_nand"
-	}
-};
-
-module_platform_driver(goldfish_nand_driver);
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/goldfish/goldfish_nand_reg.h b/drivers/staging/goldfish/goldfish_nand_reg.h
deleted file mode 100644
index 43aeba3a4c8f..000000000000
--- a/drivers/staging/goldfish/goldfish_nand_reg.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * drivers/mtd/devices/goldfish_nand_reg.h
- *
- * Copyright (C) 2007 Google, Inc.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- */
-
-#ifndef GOLDFISH_NAND_REG_H
-#define GOLDFISH_NAND_REG_H
-
-enum nand_cmd {
-	/* Write device name for NAND_DEV to NAND_DATA (vaddr) */
-	NAND_CMD_GET_DEV_NAME,
-	NAND_CMD_READ,
-	NAND_CMD_WRITE,
-	NAND_CMD_ERASE,
-	/* NAND_RESULT is 1 if block is bad, 0 if it is not */
-	NAND_CMD_BLOCK_BAD_GET,
-	NAND_CMD_BLOCK_BAD_SET,
-	NAND_CMD_READ_WITH_PARAMS,
-	NAND_CMD_WRITE_WITH_PARAMS,
-	NAND_CMD_ERASE_WITH_PARAMS
-};
-
-enum nand_dev_flags {
-	NAND_DEV_FLAG_READ_ONLY = 0x00000001,
-	NAND_DEV_FLAG_CMD_PARAMS_CAP = 0x00000002,
-};
-
-#define NAND_VERSION_CURRENT (1)
-
-enum nand_reg {
-	/* Global */
-	NAND_VERSION        = 0x000,
-	NAND_NUM_DEV        = 0x004,
-	NAND_DEV            = 0x008,
-
-	/* Dev info */
-	NAND_DEV_FLAGS      = 0x010,
-	NAND_DEV_NAME_LEN   = 0x014,
-	NAND_DEV_PAGE_SIZE  = 0x018,
-	NAND_DEV_EXTRA_SIZE = 0x01c,
-	NAND_DEV_ERASE_SIZE = 0x020,
-	NAND_DEV_SIZE_LOW   = 0x028,
-	NAND_DEV_SIZE_HIGH  = 0x02c,
-
-	/* Command */
-	NAND_RESULT         = 0x040,
-	NAND_COMMAND        = 0x044,
-	NAND_DATA           = 0x048,
-	NAND_DATA_HIGH	    = 0x100,
-	NAND_TRANSFER_SIZE  = 0x04c,
-	NAND_ADDR_LOW       = 0x050,
-	NAND_ADDR_HIGH      = 0x054,
-	NAND_CMD_PARAMS_ADDR_LOW = 0x058,
-	NAND_CMD_PARAMS_ADDR_HIGH = 0x05c,
-};
-
-struct cmd_params {
-	u32 dev;
-	u32 addr_low;
-	u32 addr_high;
-	u32 transfer_size;
-	unsigned long data;
-	u32 result;
-};
-#endif
diff --git a/drivers/staging/greybus/TODO b/drivers/staging/greybus/TODO
new file mode 100644
index 000000000000..3b90a5711998
--- /dev/null
+++ b/drivers/staging/greybus/TODO
@@ -0,0 +1,5 @@
+* Convert all uses of the old GPIO API from <linux/gpio.h> to the
+  GPIO descriptor API in <linux/gpio/consumer.h> and look up GPIO
+  lines from device tree or ACPI.
+* Convert the GPIO driver to use the GPIO irqchip library
+  GPIOLIB_IRQCHIP instead of reimplementing the same.
diff --git a/drivers/staging/greybus/arche-platform.c b/drivers/staging/greybus/arche-platform.c
index 83254a72a7bb..4c36e88766e7 100644
--- a/drivers/staging/greybus/arche-platform.c
+++ b/drivers/staging/greybus/arche-platform.c
@@ -315,8 +315,7 @@ static ssize_t state_store(struct device *dev,
 			   struct device_attribute *attr,
 			   const char *buf, size_t count)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct arche_platform_drvdata *arche_pdata = platform_get_drvdata(pdev);
+	struct arche_platform_drvdata *arche_pdata = dev_get_drvdata(dev);
 	int ret = 0;
 
 	mutex_lock(&arche_pdata->platform_state_mutex);
@@ -448,7 +447,7 @@ static int arche_platform_probe(struct platform_device *pdev)
 	arche_pdata->svc_reset_gpio = of_get_named_gpio(np,
 							"svc,reset-gpio",
 							0);
-	if (arche_pdata->svc_reset_gpio < 0) {
+	if (!gpio_is_valid(arche_pdata->svc_reset_gpio)) {
 		dev_err(dev, "failed to get reset-gpio\n");
 		return arche_pdata->svc_reset_gpio;
 	}
@@ -468,7 +467,7 @@ static int arche_platform_probe(struct platform_device *pdev)
 	arche_pdata->svc_sysboot_gpio = of_get_named_gpio(np,
 							  "svc,sysboot-gpio",
 							  0);
-	if (arche_pdata->svc_sysboot_gpio < 0) {
+	if (!gpio_is_valid(arche_pdata->svc_sysboot_gpio)) {
 		dev_err(dev, "failed to get sysboot gpio\n");
 		return arche_pdata->svc_sysboot_gpio;
 	}
@@ -487,7 +486,7 @@ static int arche_platform_probe(struct platform_device *pdev)
 	arche_pdata->svc_refclk_req = of_get_named_gpio(np,
 							"svc,refclk-req-gpio",
 							0);
-	if (arche_pdata->svc_refclk_req < 0) {
+	if (!gpio_is_valid(arche_pdata->svc_refclk_req)) {
 		dev_err(dev, "failed to get svc clock-req gpio\n");
 		return arche_pdata->svc_refclk_req;
 	}
diff --git a/drivers/staging/greybus/audio_codec.h b/drivers/staging/greybus/audio_codec.h
index a1d5440552d4..4efd8b3ebe07 100644
--- a/drivers/staging/greybus/audio_codec.h
+++ b/drivers/staging/greybus/audio_codec.h
@@ -23,7 +23,10 @@ enum {
 	NUM_CODEC_DAIS,
 };
 
-/* device_type should be same as defined in audio.h (Android media layer) */
+/*
+ * device_type should be same as defined in audio.h
+ * (Android media layer)
+ */
 enum {
 	GBAUDIO_DEVICE_NONE                     = 0x0,
 	/* reserved bits */
diff --git a/drivers/staging/greybus/audio_topology.c b/drivers/staging/greybus/audio_topology.c
index de4b1b2b12f3..15e57f701630 100644
--- a/drivers/staging/greybus/audio_topology.c
+++ b/drivers/staging/greybus/audio_topology.c
@@ -996,7 +996,7 @@ static int gbaudio_tplg_create_widget(struct gbaudio_module_info *module,
 
 	ret = gbaudio_validate_kcontrol_count(w);
 	if (ret) {
-		dev_err(module->dev, "Inavlid kcontrol count=%d for %s\n",
+		dev_err(module->dev, "Invalid kcontrol count=%d for %s\n",
 			w->ncontrols, w->name);
 		return ret;
 	}
diff --git a/drivers/staging/greybus/svc.c b/drivers/staging/greybus/svc.c
index a874fed761a1..a2bb7e1a3db3 100644
--- a/drivers/staging/greybus/svc.c
+++ b/drivers/staging/greybus/svc.c
@@ -1137,7 +1137,6 @@ static int gb_svc_intf_reset_recv(struct gb_operation *op)
 	struct gb_svc *svc = gb_connection_get_data(op->connection);
 	struct gb_message *request = op->request;
 	struct gb_svc_intf_reset_request *reset;
-	u8 intf_id;
 
 	if (request->payload_size < sizeof(*reset)) {
 		dev_warn(&svc->dev, "short reset request received (%zu < %zu)\n",
@@ -1146,8 +1145,6 @@ static int gb_svc_intf_reset_recv(struct gb_operation *op)
 	}
 	reset = request->payload;
 
-	intf_id = reset->intf_id;
-
 	/* FIXME Reset the interface here */
 
 	return 0;
diff --git a/drivers/staging/iio/Kconfig b/drivers/staging/iio/Kconfig
index bd9445956511..aee2335a25a1 100644
--- a/drivers/staging/iio/Kconfig
+++ b/drivers/staging/iio/Kconfig
@@ -11,7 +11,6 @@ source "drivers/staging/iio/cdc/Kconfig"
 source "drivers/staging/iio/frequency/Kconfig"
 source "drivers/staging/iio/gyro/Kconfig"
 source "drivers/staging/iio/impedance-analyzer/Kconfig"
-source "drivers/staging/iio/light/Kconfig"
 source "drivers/staging/iio/meter/Kconfig"
 source "drivers/staging/iio/resolver/Kconfig"
 
diff --git a/drivers/staging/iio/Makefile b/drivers/staging/iio/Makefile
index e99a375c07b9..c28d657497de 100644
--- a/drivers/staging/iio/Makefile
+++ b/drivers/staging/iio/Makefile
@@ -10,6 +10,5 @@ obj-y += cdc/
 obj-y += frequency/
 obj-y += gyro/
 obj-y += impedance-analyzer/
-obj-y += light/
 obj-y += meter/
 obj-y += resolver/
diff --git a/drivers/staging/iio/TODO b/drivers/staging/iio/TODO
index 4922402e2e98..1b8ebf2c1b69 100644
--- a/drivers/staging/iio/TODO
+++ b/drivers/staging/iio/TODO
@@ -1,4 +1,11 @@
-2016 10/09
+2018-04-15
+
+All affected drivers:
+Convert all uses of the old GPIO API from <linux/gpio.h> to the
+GPIO descriptor API in <linux/gpio/consumer.h> and look up GPIO
+lines from device tree, ACPI or board files, board files should
+use <linux/gpio/machine.h>.
+
 
 ADI Drivers:
 CC the device-drivers-devel@blackfin.uclinux.org mailing list when
diff --git a/drivers/staging/iio/accel/Kconfig b/drivers/staging/iio/accel/Kconfig
index c6b0f5eae7ab..befbbfe911c2 100644
--- a/drivers/staging/iio/accel/Kconfig
+++ b/drivers/staging/iio/accel/Kconfig
@@ -3,18 +3,6 @@
 #
 menu "Accelerometers"
 
-config ADIS16201
-	tristate "Analog Devices ADIS16201 Dual-Axis Digital Inclinometer and Accelerometer"
-	depends on SPI
-	select IIO_ADIS_LIB
-	select IIO_ADIS_LIB_BUFFER if IIO_BUFFER
-	help
-	  Say Y here to build support for Analog Devices adis16201 dual-axis
-	  digital inclinometer and accelerometer.
-
-	  To compile this driver as a module, say M here: the module will
-	  be called adis16201.
-
 config ADIS16203
 	tristate "Analog Devices ADIS16203 Programmable 360 Degrees Inclinometer"
 	depends on SPI
@@ -27,18 +15,6 @@ config ADIS16203
 	  To compile this driver as a module, say M here: the module will be
 	  called adis16203.
 
-config ADIS16209
-	tristate "Analog Devices ADIS16209 Dual-Axis Digital Inclinometer and Accelerometer"
-	depends on SPI
-	select IIO_ADIS_LIB
-	select IIO_ADIS_LIB_BUFFER if IIO_BUFFER
-	help
-	  Say Y here to build support for Analog Devices adis16209 dual-axis digital inclinometer
-	  and accelerometer.
-
-	  To compile this driver as a module, say M here: the module will be
-	  called adis16209.
-
 config ADIS16240
 	tristate "Analog Devices ADIS16240 Programmable Impact Sensor and Recorder"
 	depends on SPI
diff --git a/drivers/staging/iio/accel/Makefile b/drivers/staging/iio/accel/Makefile
index 5d8ad21a0dae..773212e0c859 100644
--- a/drivers/staging/iio/accel/Makefile
+++ b/drivers/staging/iio/accel/Makefile
@@ -2,7 +2,5 @@
 # Makefile for industrial I/O accelerometer drivers
 #
 
-obj-$(CONFIG_ADIS16201) += adis16201.o
 obj-$(CONFIG_ADIS16203) += adis16203.o
-obj-$(CONFIG_ADIS16209) += adis16209.o
 obj-$(CONFIG_ADIS16240) += adis16240.o
diff --git a/drivers/staging/iio/accel/adis16201.c b/drivers/staging/iio/accel/adis16201.c
deleted file mode 100644
index 0fae8aaf1cf4..000000000000
--- a/drivers/staging/iio/accel/adis16201.c
+++ /dev/null
@@ -1,385 +0,0 @@
-/*
- * ADIS16201 Dual-Axis Digital Inclinometer and Accelerometer
- *
- * Copyright 2010 Analog Devices Inc.
- *
- * Licensed under the GPL-2 or later.
- */
-
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/slab.h>
-#include <linux/spi/spi.h>
-#include <linux/sysfs.h>
-
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-#include <linux/iio/buffer.h>
-#include <linux/iio/imu/adis.h>
-
-#define ADIS16201_STARTUP_DELAY	220 /* ms */
-
-/* Flash memory write count */
-#define ADIS16201_FLASH_CNT      0x00
-
-/* Output, power supply */
-#define ADIS16201_SUPPLY_OUT     0x02
-
-/* Output, x-axis accelerometer */
-#define ADIS16201_XACCL_OUT      0x04
-
-/* Output, y-axis accelerometer */
-#define ADIS16201_YACCL_OUT      0x06
-
-/* Output, auxiliary ADC input */
-#define ADIS16201_AUX_ADC        0x08
-
-/* Output, temperature */
-#define ADIS16201_TEMP_OUT       0x0A
-
-/* Output, x-axis inclination */
-#define ADIS16201_XINCL_OUT      0x0C
-
-/* Output, y-axis inclination */
-#define ADIS16201_YINCL_OUT      0x0E
-
-/* Calibration, x-axis acceleration offset */
-#define ADIS16201_XACCL_OFFS     0x10
-
-/* Calibration, y-axis acceleration offset */
-#define ADIS16201_YACCL_OFFS     0x12
-
-/* x-axis acceleration scale factor */
-#define ADIS16201_XACCL_SCALE    0x14
-
-/* y-axis acceleration scale factor */
-#define ADIS16201_YACCL_SCALE    0x16
-
-/* Calibration, x-axis inclination offset */
-#define ADIS16201_XINCL_OFFS     0x18
-
-/* Calibration, y-axis inclination offset */
-#define ADIS16201_YINCL_OFFS     0x1A
-
-/* x-axis inclination scale factor */
-#define ADIS16201_XINCL_SCALE    0x1C
-
-/* y-axis inclination scale factor */
-#define ADIS16201_YINCL_SCALE    0x1E
-
-/* Alarm 1 amplitude threshold */
-#define ADIS16201_ALM_MAG1       0x20
-
-/* Alarm 2 amplitude threshold */
-#define ADIS16201_ALM_MAG2       0x22
-
-/* Alarm 1, sample period */
-#define ADIS16201_ALM_SMPL1      0x24
-
-/* Alarm 2, sample period */
-#define ADIS16201_ALM_SMPL2      0x26
-
-/* Alarm control */
-#define ADIS16201_ALM_CTRL       0x28
-
-/* Auxiliary DAC data */
-#define ADIS16201_AUX_DAC        0x30
-
-/* General-purpose digital input/output control */
-#define ADIS16201_GPIO_CTRL      0x32
-
-/* Miscellaneous control */
-#define ADIS16201_MSC_CTRL       0x34
-
-/* Internal sample period (rate) control */
-#define ADIS16201_SMPL_PRD       0x36
-
-/* Operation, filter configuration */
-#define ADIS16201_AVG_CNT        0x38
-
-/* Operation, sleep mode control */
-#define ADIS16201_SLP_CNT        0x3A
-
-/* Diagnostics, system status register */
-#define ADIS16201_DIAG_STAT      0x3C
-
-/* Operation, system command register */
-#define ADIS16201_GLOB_CMD       0x3E
-
-/* MSC_CTRL */
-
-/* Self-test enable */
-#define ADIS16201_MSC_CTRL_SELF_TEST_EN	        BIT(8)
-
-/* Data-ready enable: 1 = enabled, 0 = disabled */
-#define ADIS16201_MSC_CTRL_DATA_RDY_EN	        BIT(2)
-
-/* Data-ready polarity: 1 = active high, 0 = active low */
-#define ADIS16201_MSC_CTRL_ACTIVE_HIGH	        BIT(1)
-
-/* Data-ready line selection: 1 = DIO1, 0 = DIO0 */
-#define ADIS16201_MSC_CTRL_DATA_RDY_DIO1	BIT(0)
-
-/* DIAG_STAT */
-
-/* Alarm 2 status: 1 = alarm active, 0 = alarm inactive */
-#define ADIS16201_DIAG_STAT_ALARM2        BIT(9)
-
-/* Alarm 1 status: 1 = alarm active, 0 = alarm inactive */
-#define ADIS16201_DIAG_STAT_ALARM1        BIT(8)
-
-/* SPI communications failure */
-#define ADIS16201_DIAG_STAT_SPI_FAIL_BIT   3
-
-/* Flash update failure */
-#define ADIS16201_DIAG_STAT_FLASH_UPT_BIT  2
-
-/* Power supply above 3.625 V */
-#define ADIS16201_DIAG_STAT_POWER_HIGH_BIT 1
-
-/* Power supply below 3.15 V */
-#define ADIS16201_DIAG_STAT_POWER_LOW_BIT  0
-
-/* GLOB_CMD */
-
-#define ADIS16201_GLOB_CMD_SW_RESET	BIT(7)
-#define ADIS16201_GLOB_CMD_FACTORY_CAL	BIT(1)
-
-#define ADIS16201_ERROR_ACTIVE          BIT(14)
-
-enum adis16201_scan {
-	ADIS16201_SCAN_ACC_X,
-	ADIS16201_SCAN_ACC_Y,
-	ADIS16201_SCAN_INCLI_X,
-	ADIS16201_SCAN_INCLI_Y,
-	ADIS16201_SCAN_SUPPLY,
-	ADIS16201_SCAN_AUX_ADC,
-	ADIS16201_SCAN_TEMP,
-};
-
-static const u8 adis16201_addresses[] = {
-	[ADIS16201_SCAN_ACC_X] = ADIS16201_XACCL_OFFS,
-	[ADIS16201_SCAN_ACC_Y] = ADIS16201_YACCL_OFFS,
-	[ADIS16201_SCAN_INCLI_X] = ADIS16201_XINCL_OFFS,
-	[ADIS16201_SCAN_INCLI_Y] = ADIS16201_YINCL_OFFS,
-};
-
-static int adis16201_read_raw(struct iio_dev *indio_dev,
-			      struct iio_chan_spec const *chan,
-			      int *val, int *val2,
-			      long mask)
-{
-	struct adis *st = iio_priv(indio_dev);
-	int ret;
-	int bits;
-	u8 addr;
-	s16 val16;
-
-	switch (mask) {
-	case IIO_CHAN_INFO_RAW:
-		return adis_single_conversion(indio_dev, chan,
-				ADIS16201_ERROR_ACTIVE, val);
-	case IIO_CHAN_INFO_SCALE:
-		switch (chan->type) {
-		case IIO_VOLTAGE:
-			if (chan->channel == 0) {
-				*val = 1;
-				*val2 = 220000; /* 1.22 mV */
-			} else {
-				*val = 0;
-				*val2 = 610000; /* 0.610 mV */
-			}
-			return IIO_VAL_INT_PLUS_MICRO;
-		case IIO_TEMP:
-			*val = -470; /* 0.47 C */
-			*val2 = 0;
-			return IIO_VAL_INT_PLUS_MICRO;
-		case IIO_ACCEL:
-			*val = 0;
-			*val2 = IIO_G_TO_M_S_2(462400); /* 0.4624 mg */
-			return IIO_VAL_INT_PLUS_NANO;
-		case IIO_INCLI:
-			*val = 0;
-			*val2 = 100000; /* 0.1 degree */
-			return IIO_VAL_INT_PLUS_MICRO;
-		default:
-			return -EINVAL;
-		}
-		break;
-	case IIO_CHAN_INFO_OFFSET:
-		*val = 25000 / -470 - 1278; /* 25 C = 1278 */
-		return IIO_VAL_INT;
-	case IIO_CHAN_INFO_CALIBBIAS:
-		switch (chan->type) {
-		case IIO_ACCEL:
-			bits = 12;
-			break;
-		case IIO_INCLI:
-			bits = 9;
-			break;
-		default:
-			return -EINVAL;
-		}
-		addr = adis16201_addresses[chan->scan_index];
-		ret = adis_read_reg_16(st, addr, &val16);
-		if (ret)
-			return ret;
-		val16 &= (1 << bits) - 1;
-		val16 = (s16)(val16 << (16 - bits)) >> (16 - bits);
-		*val = val16;
-		return IIO_VAL_INT;
-	}
-
-	return -EINVAL;
-}
-
-static int adis16201_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
-{
-	struct adis *st = iio_priv(indio_dev);
-	int bits;
-	s16 val16;
-	u8 addr;
-
-	switch (mask) {
-	case IIO_CHAN_INFO_CALIBBIAS:
-		switch (chan->type) {
-		case IIO_ACCEL:
-			bits = 12;
-			break;
-		case IIO_INCLI:
-			bits = 9;
-			break;
-		default:
-			return -EINVAL;
-		}
-		val16 = val & ((1 << bits) - 1);
-		addr = adis16201_addresses[chan->scan_index];
-		return adis_write_reg_16(st, addr, val16);
-	}
-
-	return -EINVAL;
-}
-
-static const struct iio_chan_spec adis16201_channels[] = {
-	ADIS_SUPPLY_CHAN(ADIS16201_SUPPLY_OUT, ADIS16201_SCAN_SUPPLY, 0, 12),
-	ADIS_TEMP_CHAN(ADIS16201_TEMP_OUT, ADIS16201_SCAN_TEMP, 0, 12),
-	ADIS_ACCEL_CHAN(X, ADIS16201_XACCL_OUT, ADIS16201_SCAN_ACC_X,
-			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
-	ADIS_ACCEL_CHAN(Y, ADIS16201_YACCL_OUT, ADIS16201_SCAN_ACC_Y,
-			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
-	ADIS_AUX_ADC_CHAN(ADIS16201_AUX_ADC, ADIS16201_SCAN_AUX_ADC, 0, 12),
-	ADIS_INCLI_CHAN(X, ADIS16201_XINCL_OUT, ADIS16201_SCAN_INCLI_X,
-			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
-	ADIS_INCLI_CHAN(X, ADIS16201_YINCL_OUT, ADIS16201_SCAN_INCLI_Y,
-			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
-	IIO_CHAN_SOFT_TIMESTAMP(7)
-};
-
-static const struct iio_info adis16201_info = {
-	.read_raw = adis16201_read_raw,
-	.write_raw = adis16201_write_raw,
-	.update_scan_mode = adis_update_scan_mode,
-};
-
-static const char * const adis16201_status_error_msgs[] = {
-	[ADIS16201_DIAG_STAT_SPI_FAIL_BIT] = "SPI failure",
-	[ADIS16201_DIAG_STAT_FLASH_UPT_BIT] = "Flash update failed",
-	[ADIS16201_DIAG_STAT_POWER_HIGH_BIT] = "Power supply above 3.625V",
-	[ADIS16201_DIAG_STAT_POWER_LOW_BIT] = "Power supply below 3.15V",
-};
-
-static const struct adis_data adis16201_data = {
-	.read_delay = 20,
-	.msc_ctrl_reg = ADIS16201_MSC_CTRL,
-	.glob_cmd_reg = ADIS16201_GLOB_CMD,
-	.diag_stat_reg = ADIS16201_DIAG_STAT,
-
-	.self_test_mask = ADIS16201_MSC_CTRL_SELF_TEST_EN,
-	.self_test_no_autoclear = true,
-	.startup_delay = ADIS16201_STARTUP_DELAY,
-
-	.status_error_msgs = adis16201_status_error_msgs,
-	.status_error_mask = BIT(ADIS16201_DIAG_STAT_SPI_FAIL_BIT) |
-		BIT(ADIS16201_DIAG_STAT_FLASH_UPT_BIT) |
-		BIT(ADIS16201_DIAG_STAT_POWER_HIGH_BIT) |
-		BIT(ADIS16201_DIAG_STAT_POWER_LOW_BIT),
-};
-
-static int adis16201_probe(struct spi_device *spi)
-{
-	int ret;
-	struct adis *st;
-	struct iio_dev *indio_dev;
-
-	/* setup the industrialio driver allocated elements */
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev)
-		return -ENOMEM;
-
-	st = iio_priv(indio_dev);
-	/* this is only used for removal purposes */
-	spi_set_drvdata(spi, indio_dev);
-
-	indio_dev->name = spi->dev.driver->name;
-	indio_dev->dev.parent = &spi->dev;
-	indio_dev->info = &adis16201_info;
-
-	indio_dev->channels = adis16201_channels;
-	indio_dev->num_channels = ARRAY_SIZE(adis16201_channels);
-	indio_dev->modes = INDIO_DIRECT_MODE;
-
-	ret = adis_init(st, indio_dev, spi, &adis16201_data);
-	if (ret)
-		return ret;
-
-	ret = adis_setup_buffer_and_trigger(st, indio_dev, NULL);
-	if (ret)
-		return ret;
-
-	/* Get the device into a sane initial state */
-	ret = adis_initial_startup(st);
-	if (ret)
-		goto error_cleanup_buffer_trigger;
-
-	ret = iio_device_register(indio_dev);
-	if (ret < 0)
-		goto error_cleanup_buffer_trigger;
-
-	return 0;
-
-error_cleanup_buffer_trigger:
-	adis_cleanup_buffer_and_trigger(st, indio_dev);
-	return ret;
-}
-
-static int adis16201_remove(struct spi_device *spi)
-{
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
-	struct adis *st = iio_priv(indio_dev);
-
-	iio_device_unregister(indio_dev);
-	adis_cleanup_buffer_and_trigger(st, indio_dev);
-
-	return 0;
-}
-
-static struct spi_driver adis16201_driver = {
-	.driver = {
-		.name = "adis16201",
-	},
-	.probe = adis16201_probe,
-	.remove = adis16201_remove,
-};
-module_spi_driver(adis16201_driver);
-
-MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices ADIS16201 Dual-Axis Digital Inclinometer and Accelerometer");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("spi:adis16201");
diff --git a/drivers/staging/iio/accel/adis16209.c b/drivers/staging/iio/accel/adis16209.c
deleted file mode 100644
index 72a18cfe81ee..000000000000
--- a/drivers/staging/iio/accel/adis16209.c
+++ /dev/null
@@ -1,334 +0,0 @@
-/*
- * ADIS16209 Dual-Axis Digital Inclinometer and Accelerometer
- *
- * Copyright 2010 Analog Devices Inc.
- *
- * Licensed under the GPL-2 or later.
- */
-
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/list.h>
-#include <linux/module.h>
-#include <linux/spi/spi.h>
-#include <linux/slab.h>
-#include <linux/sysfs.h>
-
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-#include <linux/iio/buffer.h>
-#include <linux/iio/imu/adis.h>
-
-#define ADIS16209_STARTUP_DELAY_MS	220
-#define ADIS16209_FLASH_CNT_REG		0x00
-
-/* Data Output Register Definitions */
-#define ADIS16209_SUPPLY_OUT_REG	0x02
-#define ADIS16209_XACCL_OUT_REG		0x04
-#define ADIS16209_YACCL_OUT_REG		0x06
-/* Output, auxiliary ADC input */
-#define ADIS16209_AUX_ADC_REG		0x08
-/* Output, temperature */
-#define ADIS16209_TEMP_OUT_REG		0x0A
-/* Output, +/- 90 degrees X-axis inclination */
-#define ADIS16209_XINCL_OUT_REG		0x0C
-#define ADIS16209_YINCL_OUT_REG		0x0E
-/* Output, +/-180 vertical rotational position */
-#define ADIS16209_ROT_OUT_REG		0x10
-
-/*
- * Calibration Register Definitions.
- * Acceleration, inclination or rotation offset null.
- */
-#define ADIS16209_XACCL_NULL_REG	0x12
-#define ADIS16209_YACCL_NULL_REG	0x14
-#define ADIS16209_XINCL_NULL_REG	0x16
-#define ADIS16209_YINCL_NULL_REG	0x18
-#define ADIS16209_ROT_NULL_REG		0x1A
-
-/* Alarm Register Definitions */
-#define ADIS16209_ALM_MAG1_REG		0x20
-#define ADIS16209_ALM_MAG2_REG		0x22
-#define ADIS16209_ALM_SMPL1_REG		0x24
-#define ADIS16209_ALM_SMPL2_REG		0x26
-#define ADIS16209_ALM_CTRL_REG		0x28
-
-#define ADIS16209_AUX_DAC_REG		0x30
-#define ADIS16209_GPIO_CTRL_REG		0x32
-#define ADIS16209_SMPL_PRD_REG		0x36
-#define ADIS16209_AVG_CNT_REG		0x38
-#define ADIS16209_SLP_CNT_REG		0x3A
-
-#define ADIS16209_MSC_CTRL_REG			0x34
-#define  ADIS16209_MSC_CTRL_PWRUP_SELF_TEST	BIT(10)
-#define  ADIS16209_MSC_CTRL_SELF_TEST_EN	BIT(8)
-#define  ADIS16209_MSC_CTRL_DATA_RDY_EN		BIT(2)
-/* Data-ready polarity: 1 = active high, 0 = active low */
-#define  ADIS16209_MSC_CTRL_ACTIVE_HIGH		BIT(1)
-#define  ADIS16209_MSC_CTRL_DATA_RDY_DIO2	BIT(0)
-
-#define ADIS16209_STAT_REG			0x3C
-#define  ADIS16209_STAT_ALARM2			BIT(9)
-#define  ADIS16209_STAT_ALARM1			BIT(8)
-#define ADIS16209_STAT_SELFTEST_FAIL_BIT	5
-#define ADIS16209_STAT_SPI_FAIL_BIT		3
-#define ADIS16209_STAT_FLASH_UPT_FAIL_BIT	2
-/* Power supply above 3.625 V */
-#define ADIS16209_STAT_POWER_HIGH_BIT		1
-/* Power supply below 3.15 V */
-#define ADIS16209_STAT_POWER_LOW_BIT		0
-
-#define ADIS16209_CMD_REG			0x3E
-#define  ADIS16209_CMD_SW_RESET			BIT(7)
-#define  ADIS16209_CMD_CLEAR_STAT		BIT(4)
-#define  ADIS16209_CMD_FACTORY_CAL		BIT(1)
-
-#define ADIS16209_ERROR_ACTIVE			BIT(14)
-
-enum adis16209_scan {
-	ADIS16209_SCAN_SUPPLY,
-	ADIS16209_SCAN_ACC_X,
-	ADIS16209_SCAN_ACC_Y,
-	ADIS16209_SCAN_AUX_ADC,
-	ADIS16209_SCAN_TEMP,
-	ADIS16209_SCAN_INCLI_X,
-	ADIS16209_SCAN_INCLI_Y,
-	ADIS16209_SCAN_ROT,
-};
-
-static const u8 adis16209_addresses[8][1] = {
-	[ADIS16209_SCAN_SUPPLY] = { },
-	[ADIS16209_SCAN_AUX_ADC] = { },
-	[ADIS16209_SCAN_ACC_X] = { ADIS16209_XACCL_NULL_REG },
-	[ADIS16209_SCAN_ACC_Y] = { ADIS16209_YACCL_NULL_REG },
-	[ADIS16209_SCAN_INCLI_X] = { ADIS16209_XINCL_NULL_REG },
-	[ADIS16209_SCAN_INCLI_Y] = { ADIS16209_YINCL_NULL_REG },
-	[ADIS16209_SCAN_ROT] = { },
-	[ADIS16209_SCAN_TEMP] = { },
-};
-
-static int adis16209_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
-{
-	struct adis *st = iio_priv(indio_dev);
-	int bits;
-	s16 val16;
-	u8 addr;
-
-	switch (mask) {
-	case IIO_CHAN_INFO_CALIBBIAS:
-		switch (chan->type) {
-		case IIO_ACCEL:
-		case IIO_INCLI:
-			bits = 14;
-			break;
-		default:
-			return -EINVAL;
-		}
-		val16 = val & ((1 << bits) - 1);
-		addr = adis16209_addresses[chan->scan_index][0];
-		return adis_write_reg_16(st, addr, val16);
-	}
-	return -EINVAL;
-}
-
-static int adis16209_read_raw(struct iio_dev *indio_dev,
-			      struct iio_chan_spec const *chan,
-			      int *val, int *val2,
-			      long mask)
-{
-	struct adis *st = iio_priv(indio_dev);
-	int ret;
-	int bits;
-	u8 addr;
-	s16 val16;
-
-	switch (mask) {
-	case IIO_CHAN_INFO_RAW:
-		return adis_single_conversion(indio_dev, chan,
-			ADIS16209_ERROR_ACTIVE, val);
-	case IIO_CHAN_INFO_SCALE:
-		switch (chan->type) {
-		case IIO_VOLTAGE:
-			*val = 0;
-			switch (chan->channel) {
-			case 0:
-				*val2 = 305180; /* 0.30518 mV */
-				break;
-			case 1:
-				*val2 = 610500; /* 0.6105 mV */
-				break;
-			default:
-				return -EINVAL;
-			}
-			return IIO_VAL_INT_PLUS_MICRO;
-		case IIO_TEMP:
-			*val = -470;
-			*val2 = 0;
-			return IIO_VAL_INT_PLUS_MICRO;
-		case IIO_ACCEL:
-			/*
-			 * IIO base unit for sensitivity of accelerometer
-			 * is milli g.
-			 * 1 LSB represents 0.244 mg.
-			 */
-			*val = 0;
-			*val2 = IIO_G_TO_M_S_2(244140);
-			return IIO_VAL_INT_PLUS_NANO;
-		case IIO_INCLI:
-		case IIO_ROT:
-			/*
-			 * IIO base units for rotation are degrees.
-			 * 1 LSB represents 0.025 milli degrees.
-			 */
-			*val = 0;
-			*val2 = 25000;
-			return IIO_VAL_INT_PLUS_MICRO;
-		default:
-			return -EINVAL;
-		}
-		break;
-	case IIO_CHAN_INFO_OFFSET:
-		/*
-		 * The raw ADC value is 0x4FE when the temperature
-		 * is 25 degrees and the scale factor per milli
-		 * degree celcius is -470.
-		 */
-		*val = 25000 / -470 - 0x4FE;
-		return IIO_VAL_INT;
-	case IIO_CHAN_INFO_CALIBBIAS:
-		switch (chan->type) {
-		case IIO_ACCEL:
-			bits = 14;
-			break;
-		default:
-			return -EINVAL;
-		}
-		addr = adis16209_addresses[chan->scan_index][0];
-		ret = adis_read_reg_16(st, addr, &val16);
-		if (ret)
-			return ret;
-
-		*val = sign_extend32(val16, bits - 1);
-		return IIO_VAL_INT;
-	}
-	return -EINVAL;
-}
-
-static const struct iio_chan_spec adis16209_channels[] = {
-	ADIS_SUPPLY_CHAN(ADIS16209_SUPPLY_OUT_REG, ADIS16209_SCAN_SUPPLY,
-			 0, 14),
-	ADIS_TEMP_CHAN(ADIS16209_TEMP_OUT_REG, ADIS16209_SCAN_TEMP, 0, 12),
-	ADIS_ACCEL_CHAN(X, ADIS16209_XACCL_OUT_REG, ADIS16209_SCAN_ACC_X,
-			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
-	ADIS_ACCEL_CHAN(Y, ADIS16209_YACCL_OUT_REG, ADIS16209_SCAN_ACC_Y,
-			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
-	ADIS_AUX_ADC_CHAN(ADIS16209_AUX_ADC_REG, ADIS16209_SCAN_AUX_ADC, 0, 12),
-	ADIS_INCLI_CHAN(X, ADIS16209_XINCL_OUT_REG, ADIS16209_SCAN_INCLI_X,
-			0, 0, 14),
-	ADIS_INCLI_CHAN(Y, ADIS16209_YINCL_OUT_REG, ADIS16209_SCAN_INCLI_Y,
-			0, 0, 14),
-	ADIS_ROT_CHAN(X, ADIS16209_ROT_OUT_REG, ADIS16209_SCAN_ROT, 0, 0, 14),
-	IIO_CHAN_SOFT_TIMESTAMP(8)
-};
-
-static const struct iio_info adis16209_info = {
-	.read_raw = adis16209_read_raw,
-	.write_raw = adis16209_write_raw,
-	.update_scan_mode = adis_update_scan_mode,
-};
-
-static const char * const adis16209_status_error_msgs[] = {
-	[ADIS16209_STAT_SELFTEST_FAIL_BIT] = "Self test failure",
-	[ADIS16209_STAT_SPI_FAIL_BIT] = "SPI failure",
-	[ADIS16209_STAT_FLASH_UPT_FAIL_BIT] = "Flash update failed",
-	[ADIS16209_STAT_POWER_HIGH_BIT] = "Power supply above 3.625V",
-	[ADIS16209_STAT_POWER_LOW_BIT] = "Power supply below 3.15V",
-};
-
-static const struct adis_data adis16209_data = {
-	.read_delay = 30,
-	.msc_ctrl_reg = ADIS16209_MSC_CTRL_REG,
-	.glob_cmd_reg = ADIS16209_CMD_REG,
-	.diag_stat_reg = ADIS16209_STAT_REG,
-
-	.self_test_mask = ADIS16209_MSC_CTRL_SELF_TEST_EN,
-	.self_test_no_autoclear = true,
-	.startup_delay = ADIS16209_STARTUP_DELAY_MS,
-
-	.status_error_msgs = adis16209_status_error_msgs,
-	.status_error_mask = BIT(ADIS16209_STAT_SELFTEST_FAIL_BIT) |
-		BIT(ADIS16209_STAT_SPI_FAIL_BIT) |
-		BIT(ADIS16209_STAT_FLASH_UPT_FAIL_BIT) |
-		BIT(ADIS16209_STAT_POWER_HIGH_BIT) |
-		BIT(ADIS16209_STAT_POWER_LOW_BIT),
-};
-
-static int adis16209_probe(struct spi_device *spi)
-{
-	int ret;
-	struct adis *st;
-	struct iio_dev *indio_dev;
-
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev)
-		return -ENOMEM;
-	st = iio_priv(indio_dev);
-	spi_set_drvdata(spi, indio_dev);
-
-	indio_dev->name = spi->dev.driver->name;
-	indio_dev->dev.parent = &spi->dev;
-	indio_dev->info = &adis16209_info;
-	indio_dev->channels = adis16209_channels;
-	indio_dev->num_channels = ARRAY_SIZE(adis16209_channels);
-	indio_dev->modes = INDIO_DIRECT_MODE;
-
-	ret = adis_init(st, indio_dev, spi, &adis16209_data);
-	if (ret)
-		return ret;
-	ret = adis_setup_buffer_and_trigger(st, indio_dev, NULL);
-	if (ret)
-		return ret;
-
-	ret = adis_initial_startup(st);
-	if (ret)
-		goto error_cleanup_buffer_trigger;
-	ret = iio_device_register(indio_dev);
-	if (ret)
-		goto error_cleanup_buffer_trigger;
-
-	return 0;
-
-error_cleanup_buffer_trigger:
-	adis_cleanup_buffer_and_trigger(st, indio_dev);
-	return ret;
-}
-
-static int adis16209_remove(struct spi_device *spi)
-{
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
-	struct adis *st = iio_priv(indio_dev);
-
-	iio_device_unregister(indio_dev);
-	adis_cleanup_buffer_and_trigger(st, indio_dev);
-
-	return 0;
-}
-
-static struct spi_driver adis16209_driver = {
-	.driver = {
-		.name = "adis16209",
-	},
-	.probe = adis16209_probe,
-	.remove = adis16209_remove,
-};
-module_spi_driver(adis16209_driver);
-
-MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices ADIS16209 Dual-Axis Digital Inclinometer and Accelerometer");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("spi:adis16209");
diff --git a/drivers/staging/iio/adc/ad7606_par.c b/drivers/staging/iio/adc/ad7606_par.c
index 3eb6f8f312dd..a34c2a1d5373 100644
--- a/drivers/staging/iio/adc/ad7606_par.c
+++ b/drivers/staging/iio/adc/ad7606_par.c
@@ -18,8 +18,7 @@
 static int ad7606_par16_read_block(struct device *dev,
 				   int count, void *buf)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct ad7606_state *st = iio_priv(indio_dev);
 
 	insw((unsigned long)st->base_address, buf, count);
@@ -34,8 +33,7 @@ static const struct ad7606_bus_ops ad7606_par16_bops = {
 static int ad7606_par8_read_block(struct device *dev,
 				  int count, void *buf)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct ad7606_state *st = iio_priv(indio_dev);
 
 	insb((unsigned long)st->base_address, buf, count * 2);
diff --git a/drivers/staging/iio/adc/ad7780.c b/drivers/staging/iio/adc/ad7780.c
index a7797af579b9..16d72072c076 100644
--- a/drivers/staging/iio/adc/ad7780.c
+++ b/drivers/staging/iio/adc/ad7780.c
@@ -128,7 +128,7 @@ static const struct ad_sigma_delta_info ad7780_sigma_delta_info = {
 };
 
 #define AD7780_CHANNEL(bits, wordsize) \
-	AD_SD_CHANNEL(1, 0, 0, bits, 32, wordsize - bits)
+	AD_SD_CHANNEL_NO_SAMP_FREQ(1, 0, 0, bits, 32, wordsize - bits)
 
 static const struct ad7780_chip_info ad7780_chip_info_tbl[] = {
 	[ID_AD7170] = {
diff --git a/drivers/staging/iio/cdc/ad7746.c b/drivers/staging/iio/cdc/ad7746.c
index 4882dbc81c53..f53612a6461d 100644
--- a/drivers/staging/iio/cdc/ad7746.c
+++ b/drivers/staging/iio/cdc/ad7746.c
@@ -6,15 +6,15 @@
  * Licensed under the GPL-2.
  */
 
-#include <linux/interrupt.h>
+#include <linux/delay.h>
 #include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/sysfs.h>
 #include <linux/i2c.h>
-#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/slab.h>
 #include <linux/stat.h>
+#include <linux/sysfs.h>
 
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
@@ -220,8 +220,8 @@ static int ad7746_select_channel(struct iio_dev *indio_dev,
 				 struct iio_chan_spec const *chan)
 {
 	struct ad7746_chip_info *chip = iio_priv(indio_dev);
-	int ret, delay, idx;
 	u8 vt_setup, cap_setup;
+	int ret, delay, idx;
 
 	switch (chan->type) {
 	case IIO_CAPACITANCE:
@@ -289,8 +289,8 @@ static inline ssize_t ad7746_start_calib(struct device *dev,
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ad7746_chip_info *chip = iio_priv(indio_dev);
-	bool doit;
 	int ret, timeout = 10;
+	bool doit;
 
 	ret = strtobool(buf, &doit);
 	if (ret < 0)
@@ -410,8 +410,7 @@ static struct attribute *ad7746_attributes[] = {
 	&iio_dev_attr_in_capacitance1_calibbias_calibration.dev_attr.attr,
 	&iio_dev_attr_in_voltage0_calibscale_calibration.dev_attr.attr,
 	&iio_const_attr_in_voltage_sampling_frequency_available.dev_attr.attr,
-	&iio_const_attr_in_capacitance_sampling_frequency_available.
-	dev_attr.attr,
+	&iio_const_attr_in_capacitance_sampling_frequency_available.dev_attr.attr,
 	NULL,
 };
 
@@ -451,26 +450,26 @@ static int ad7746_write_raw(struct iio_dev *indio_dev,
 			goto out;
 		}
 
-		ret = i2c_smbus_write_word_data(chip->client, reg, swab16(val));
+		ret = i2c_smbus_write_word_swapped(chip->client, reg, val);
 		if (ret < 0)
 			goto out;
 
 		ret = 0;
 		break;
 	case IIO_CHAN_INFO_CALIBBIAS:
-		if ((val < 0) | (val > 0xFFFF)) {
+		if (val < 0 || val > 0xFFFF) {
 			ret = -EINVAL;
 			goto out;
 		}
-		ret = i2c_smbus_write_word_data(chip->client,
-				AD7746_REG_CAP_OFFH, swab16(val));
+		ret = i2c_smbus_write_word_swapped(chip->client,
+						   AD7746_REG_CAP_OFFH, val);
 		if (ret < 0)
 			goto out;
 
 		ret = 0;
 		break;
 	case IIO_CHAN_INFO_OFFSET:
-		if ((val < 0) | (val > 43008000)) { /* 21pF */
+		if (val < 0 || val > 43008000) { /* 21pF */
 			ret = -EINVAL;
 			goto out;
 		}
@@ -556,7 +555,8 @@ static int ad7746_read_raw(struct iio_dev *indio_dev,
 		/* Now read the actual register */
 
 		ret = i2c_smbus_read_i2c_block_data(chip->client,
-			chan->address >> 8, 3, &chip->data.d8[1]);
+						    chan->address >> 8, 3,
+						    &chip->data.d8[1]);
 
 		if (ret < 0)
 			goto out;
@@ -594,27 +594,27 @@ static int ad7746_read_raw(struct iio_dev *indio_dev,
 			goto out;
 		}
 
-		ret = i2c_smbus_read_word_data(chip->client, reg);
+		ret = i2c_smbus_read_word_swapped(chip->client, reg);
 		if (ret < 0)
 			goto out;
 		/* 1 + gain_val / 2^16 */
 		*val = 1;
-		*val2 = (15625 * swab16(ret)) / 1024;
+		*val2 = (15625 * ret) / 1024;
 
 		ret = IIO_VAL_INT_PLUS_MICRO;
 		break;
 	case IIO_CHAN_INFO_CALIBBIAS:
-		ret = i2c_smbus_read_word_data(chip->client,
-					       AD7746_REG_CAP_OFFH);
+		ret = i2c_smbus_read_word_swapped(chip->client,
+						  AD7746_REG_CAP_OFFH);
 		if (ret < 0)
 			goto out;
-		*val = swab16(ret);
+		*val = ret;
 
 		ret = IIO_VAL_INT;
 		break;
 	case IIO_CHAN_INFO_OFFSET:
 		*val = AD7746_CAPDAC_DACP(chip->capdac[chan->channel]
-			[chan->differential]) * 338646;
+					  [chan->differential]) * 338646;
 
 		ret = IIO_VAL_INT;
 		break;
@@ -680,8 +680,8 @@ static int ad7746_probe(struct i2c_client *client,
 	struct ad7746_platform_data *pdata = client->dev.platform_data;
 	struct ad7746_chip_info *chip;
 	struct iio_dev *indio_dev;
-	int ret = 0;
 	unsigned char regval = 0;
+	int ret = 0;
 
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*chip));
 	if (!indio_dev)
diff --git a/drivers/staging/iio/light/Kconfig b/drivers/staging/iio/light/Kconfig
deleted file mode 100644
index aacb0ae58c0e..000000000000
--- a/drivers/staging/iio/light/Kconfig
+++ /dev/null
@@ -1,14 +0,0 @@
-#
-# Light sensors
-#
-menu "Light sensors"
-
-config TSL2x7x
-	tristate "TAOS TSL/TMD2x71 and TSL/TMD2x72 Family of light and proximity sensors"
-	depends on I2C
-	help
-	 Support for: tsl2571, tsl2671, tmd2671, tsl2771, tmd2771, tsl2572, tsl2672,
-	 tmd2672, tsl2772, tmd2772 devices.
-	 Provides iio_events and direct access via sysfs.
-
-endmenu
diff --git a/drivers/staging/iio/light/Makefile b/drivers/staging/iio/light/Makefile
deleted file mode 100644
index ab8dc3a3d10b..000000000000
--- a/drivers/staging/iio/light/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-#
-# Makefile for industrial I/O Light sensors
-#
-
-obj-$(CONFIG_TSL2x7x)	+= tsl2x7x.o
diff --git a/drivers/staging/iio/light/tsl2x7x.c b/drivers/staging/iio/light/tsl2x7x.c
deleted file mode 100644
index 82681300e106..000000000000
--- a/drivers/staging/iio/light/tsl2x7x.c
+++ /dev/null
@@ -1,1889 +0,0 @@
-/*
- * Device driver for monitoring ambient light intensity in (lux)
- * and proximity detection (prox) within the TAOS TSL2X7X family of devices.
- *
- * Copyright (c) 2012, TAOS Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <linux/delay.h>
-#include <linux/errno.h>
-#include <linux/i2c.h>
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/slab.h>
-#include <linux/iio/events.h>
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-#include "tsl2x7x.h"
-
-/* Cal defs*/
-#define PROX_STAT_CAL			0
-#define PROX_STAT_SAMP			1
-#define MAX_SAMPLES_CAL			200
-
-/* TSL2X7X Device ID */
-#define TRITON_ID			0x00
-#define SWORDFISH_ID			0x30
-#define HALIBUT_ID			0x20
-
-/* Lux calculation constants */
-#define TSL2X7X_LUX_CALC_OVER_FLOW	65535
-
-/* TAOS Register definitions - note:
- * depending on device, some of these register are not used and the
- * register address is benign.
- */
-/* 2X7X register offsets */
-#define TSL2X7X_MAX_CONFIG_REG		16
-
-/* Device Registers and Masks */
-#define TSL2X7X_CNTRL			0x00
-#define TSL2X7X_ALS_TIME		0X01
-#define TSL2X7X_PRX_TIME		0x02
-#define TSL2X7X_WAIT_TIME		0x03
-#define TSL2X7X_ALS_MINTHRESHLO		0X04
-#define TSL2X7X_ALS_MINTHRESHHI		0X05
-#define TSL2X7X_ALS_MAXTHRESHLO		0X06
-#define TSL2X7X_ALS_MAXTHRESHHI		0X07
-#define TSL2X7X_PRX_MINTHRESHLO		0X08
-#define TSL2X7X_PRX_MINTHRESHHI		0X09
-#define TSL2X7X_PRX_MAXTHRESHLO		0X0A
-#define TSL2X7X_PRX_MAXTHRESHHI		0X0B
-#define TSL2X7X_PERSISTENCE		0x0C
-#define TSL2X7X_PRX_CONFIG		0x0D
-#define TSL2X7X_PRX_COUNT		0x0E
-#define TSL2X7X_GAIN			0x0F
-#define TSL2X7X_NOTUSED			0x10
-#define TSL2X7X_REVID			0x11
-#define TSL2X7X_CHIPID			0x12
-#define TSL2X7X_STATUS			0x13
-#define TSL2X7X_ALS_CHAN0LO		0x14
-#define TSL2X7X_ALS_CHAN0HI		0x15
-#define TSL2X7X_ALS_CHAN1LO		0x16
-#define TSL2X7X_ALS_CHAN1HI		0x17
-#define TSL2X7X_PRX_LO			0x18
-#define TSL2X7X_PRX_HI			0x19
-
-/* tsl2X7X cmd reg masks */
-#define TSL2X7X_CMD_REG			0x80
-#define TSL2X7X_CMD_SPL_FN		0x60
-
-#define TSL2X7X_CMD_PROX_INT_CLR	0X05
-#define TSL2X7X_CMD_ALS_INT_CLR		0x06
-#define TSL2X7X_CMD_PROXALS_INT_CLR	0X07
-
-/* tsl2X7X cntrl reg masks */
-#define TSL2X7X_CNTL_ADC_ENBL		0x02
-#define TSL2X7X_CNTL_PWR_ON		0x01
-
-/* tsl2X7X status reg masks */
-#define TSL2X7X_STA_ADC_VALID		0x01
-#define TSL2X7X_STA_PRX_VALID		0x02
-#define TSL2X7X_STA_ADC_PRX_VALID	(TSL2X7X_STA_ADC_VALID | \
-					 TSL2X7X_STA_PRX_VALID)
-#define TSL2X7X_STA_ALS_INTR		0x10
-#define TSL2X7X_STA_PRX_INTR		0x20
-
-/* tsl2X7X cntrl reg masks */
-#define TSL2X7X_CNTL_REG_CLEAR		0x00
-#define TSL2X7X_CNTL_PROX_INT_ENBL	0X20
-#define TSL2X7X_CNTL_ALS_INT_ENBL	0X10
-#define TSL2X7X_CNTL_WAIT_TMR_ENBL	0X08
-#define TSL2X7X_CNTL_PROX_DET_ENBL	0X04
-#define TSL2X7X_CNTL_PWRON		0x01
-#define TSL2X7X_CNTL_ALSPON_ENBL	0x03
-#define TSL2X7X_CNTL_INTALSPON_ENBL	0x13
-#define TSL2X7X_CNTL_PROXPON_ENBL	0x0F
-#define TSL2X7X_CNTL_INTPROXPON_ENBL	0x2F
-
-/*Prox diode to use */
-#define TSL2X7X_DIODE0			0x01
-#define TSL2X7X_DIODE1			0x02
-#define TSL2X7X_DIODE_BOTH		0x03
-
-/* LED Power */
-#define TSL2X7X_100_mA			0x00
-#define TSL2X7X_50_mA			0x01
-#define TSL2X7X_25_mA			0x02
-#define TSL2X7X_13_mA			0x03
-#define TSL2X7X_MAX_TIMER_CNT		0xFF
-
-#define TSL2X7X_MIN_ITIME		3
-
-/* TAOS txx2x7x Device family members */
-enum {
-	tsl2571,
-	tsl2671,
-	tmd2671,
-	tsl2771,
-	tmd2771,
-	tsl2572,
-	tsl2672,
-	tmd2672,
-	tsl2772,
-	tmd2772
-};
-
-enum {
-	TSL2X7X_CHIP_UNKNOWN = 0,
-	TSL2X7X_CHIP_WORKING = 1,
-	TSL2X7X_CHIP_SUSPENDED = 2
-};
-
-/* Per-device data */
-struct tsl2x7x_als_info {
-	u16 als_ch0;
-	u16 als_ch1;
-	u16 lux;
-};
-
-struct tsl2x7x_prox_stat {
-	int min;
-	int max;
-	int mean;
-	unsigned long stddev;
-};
-
-struct tsl2x7x_chip_info {
-	int chan_table_elements;
-	struct iio_chan_spec		channel[4];
-	const struct iio_info		*info;
-};
-
-struct tsl2X7X_chip {
-	kernel_ulong_t id;
-	struct mutex prox_mutex;
-	struct mutex als_mutex;
-	struct i2c_client *client;
-	u16 prox_data;
-	struct tsl2x7x_als_info als_cur_info;
-	struct tsl2x7x_settings settings;
-	struct tsl2X7X_platform_data *pdata;
-	int als_time_scale;
-	int als_saturation;
-	int tsl2x7x_chip_status;
-	u8 tsl2x7x_config[TSL2X7X_MAX_CONFIG_REG];
-	const struct tsl2x7x_chip_info	*chip_info;
-	const struct iio_info *info;
-	s64 event_timestamp;
-	/*
-	 * This structure is intentionally large to accommodate
-	 * updates via sysfs.
-	 * Sized to 9 = max 8 segments + 1 termination segment
-	 */
-	struct tsl2x7x_lux tsl2x7x_device_lux[TSL2X7X_MAX_LUX_TABLE_SIZE];
-};
-
-/* Different devices require different coefficents */
-static const struct tsl2x7x_lux tsl2x71_lux_table[TSL2X7X_DEF_LUX_TABLE_SZ] = {
-	{ 14461,   611,   1211 },
-	{ 18540,   352,    623 },
-	{     0,     0,      0 },
-};
-
-static const struct tsl2x7x_lux tmd2x71_lux_table[TSL2X7X_DEF_LUX_TABLE_SZ] = {
-	{ 11635,   115,    256 },
-	{ 15536,    87,    179 },
-	{     0,     0,      0 },
-};
-
-static const struct tsl2x7x_lux tsl2x72_lux_table[TSL2X7X_DEF_LUX_TABLE_SZ] = {
-	{ 14013,   466,   917 },
-	{ 18222,   310,   552 },
-	{     0,     0,     0 },
-};
-
-static const struct tsl2x7x_lux tmd2x72_lux_table[TSL2X7X_DEF_LUX_TABLE_SZ] = {
-	{ 13218,   130,   262 },
-	{ 17592,   92,    169 },
-	{     0,     0,     0 },
-};
-
-static const struct tsl2x7x_lux *tsl2x7x_default_lux_table_group[] = {
-	[tsl2571] =	tsl2x71_lux_table,
-	[tsl2671] =	tsl2x71_lux_table,
-	[tmd2671] =	tmd2x71_lux_table,
-	[tsl2771] =	tsl2x71_lux_table,
-	[tmd2771] =	tmd2x71_lux_table,
-	[tsl2572] =	tsl2x72_lux_table,
-	[tsl2672] =	tsl2x72_lux_table,
-	[tmd2672] =	tmd2x72_lux_table,
-	[tsl2772] =	tsl2x72_lux_table,
-	[tmd2772] =	tmd2x72_lux_table,
-};
-
-static const struct tsl2x7x_settings tsl2x7x_default_settings = {
-	.als_time = 219, /* 101 ms */
-	.als_gain = 0,
-	.prx_time = 254, /* 5.4 ms */
-	.prox_gain = 0,
-	.wait_time = 245,
-	.prox_config = 0,
-	.als_gain_trim = 1000,
-	.als_cal_target = 150,
-	.als_thresh_low = 200,
-	.als_thresh_high = 256,
-	.persistence = 255,
-	.interrupts_en = 0,
-	.prox_thres_low  = 0,
-	.prox_thres_high = 512,
-	.prox_max_samples_cal = 30,
-	.prox_pulse_count = 8,
-	.prox_diode = TSL2X7X_DIODE1,
-	.prox_power = TSL2X7X_100_mA
-};
-
-static const s16 tsl2x7x_als_gain[] = {
-	1,
-	8,
-	16,
-	120
-};
-
-static const s16 tsl2x7x_prx_gain[] = {
-	1,
-	2,
-	4,
-	8
-};
-
-/* Channel variations */
-enum {
-	ALS,
-	PRX,
-	ALSPRX,
-	PRX2,
-	ALSPRX2,
-};
-
-static const u8 device_channel_config[] = {
-	ALS,
-	PRX,
-	PRX,
-	ALSPRX,
-	ALSPRX,
-	ALS,
-	PRX2,
-	PRX2,
-	ALSPRX2,
-	ALSPRX2
-};
-
-static int tsl2x7x_clear_interrupts(struct tsl2X7X_chip *chip, int reg)
-{
-	int ret;
-
-	ret = i2c_smbus_write_byte(chip->client,
-				   TSL2X7X_CMD_REG | TSL2X7X_CMD_SPL_FN | reg);
-	if (ret < 0)
-		dev_err(&chip->client->dev,
-			"%s: failed to clear interrupt status %x: %d\n",
-			__func__, reg, ret);
-
-	return ret;
-}
-
-static int tsl2x7x_read_status(struct tsl2X7X_chip *chip)
-{
-	int ret;
-
-	ret = i2c_smbus_read_byte_data(chip->client,
-				       TSL2X7X_CMD_REG | TSL2X7X_STATUS);
-	if (ret < 0)
-		dev_err(&chip->client->dev,
-			"%s: failed to read STATUS register: %d\n", __func__,
-			ret);
-
-	return ret;
-}
-
-static int tsl2x7x_write_control_reg(struct tsl2X7X_chip *chip, u8 data)
-{
-	int ret;
-
-	ret = i2c_smbus_write_byte_data(chip->client,
-					TSL2X7X_CMD_REG | TSL2X7X_CNTRL, data);
-	if (ret < 0) {
-		dev_err(&chip->client->dev,
-			"%s: failed to write to control register %x: %d\n",
-			__func__, data, ret);
-	}
-
-	return ret;
-}
-
-/**
- * tsl2x7x_get_lux() - Reads and calculates current lux value.
- * @indio_dev:	pointer to IIO device
- *
- * The raw ch0 and ch1 values of the ambient light sensed in the last
- * integration cycle are read from the device.
- * Time scale factor array values are adjusted based on the integration time.
- * The raw values are multiplied by a scale factor, and device gain is obtained
- * using gain index. Limit checks are done next, then the ratio of a multiple
- * of ch1 value, to the ch0 value, is calculated. Array tsl2x7x_device_lux[]
- * is then scanned to find the first ratio value that is just above the ratio
- * we just calculated. The ch0 and ch1 multiplier constants in the array are
- * then used along with the time scale factor array values, to calculate the
- * lux.
- */
-static int tsl2x7x_get_lux(struct iio_dev *indio_dev)
-{
-	u16 ch0, ch1; /* separated ch0/ch1 data from device */
-	u32 lux; /* raw lux calculated from device data */
-	u64 lux64;
-	u32 ratio;
-	u8 buf[4];
-	struct tsl2x7x_lux *p;
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	int i, ret;
-	u32 ch0lux = 0;
-	u32 ch1lux = 0;
-
-	mutex_lock(&chip->als_mutex);
-
-	if (chip->tsl2x7x_chip_status != TSL2X7X_CHIP_WORKING) {
-		/* device is not enabled */
-		dev_err(&chip->client->dev, "%s: device is not enabled\n",
-			__func__);
-		ret = -EBUSY;
-		goto out_unlock;
-	}
-
-	ret = tsl2x7x_read_status(chip);
-	if (ret < 0)
-		goto out_unlock;
-
-	/* is data new & valid */
-	if (!(ret & TSL2X7X_STA_ADC_VALID)) {
-		dev_err(&chip->client->dev,
-			"%s: data not valid yet\n", __func__);
-		ret = chip->als_cur_info.lux; /* return LAST VALUE */
-		goto out_unlock;
-	}
-
-	for (i = 0; i < 4; i++) {
-		int reg = TSL2X7X_CMD_REG | (TSL2X7X_ALS_CHAN0LO + i);
-
-		ret = i2c_smbus_read_byte_data(chip->client, reg);
-		if (ret < 0) {
-			dev_err(&chip->client->dev,
-				"failed to read. err=%x\n", ret);
-			goto out_unlock;
-		}
-
-		buf[i] = ret;
-	}
-
-	ret = tsl2x7x_clear_interrupts(chip, TSL2X7X_CMD_ALS_INT_CLR);
-	if (ret < 0)
-		goto out_unlock;
-
-	/* extract ALS/lux data */
-	ch0 = le16_to_cpup((const __le16 *)&buf[0]);
-	ch1 = le16_to_cpup((const __le16 *)&buf[2]);
-
-	chip->als_cur_info.als_ch0 = ch0;
-	chip->als_cur_info.als_ch1 = ch1;
-
-	if (ch0 >= chip->als_saturation || ch1 >= chip->als_saturation) {
-		lux = TSL2X7X_LUX_CALC_OVER_FLOW;
-		goto return_max;
-	}
-
-	if (!ch0) {
-		/* have no data, so return LAST VALUE */
-		ret = chip->als_cur_info.lux;
-		goto out_unlock;
-	}
-	/* calculate ratio */
-	ratio = (ch1 << 15) / ch0;
-	/* convert to unscaled lux using the pointer to the table */
-	p = (struct tsl2x7x_lux *)chip->tsl2x7x_device_lux;
-	while (p->ratio != 0 && p->ratio < ratio)
-		p++;
-
-	if (p->ratio == 0) {
-		lux = 0;
-	} else {
-		lux = DIV_ROUND_UP(ch0 * p->ch0,
-				   tsl2x7x_als_gain[chip->settings.als_gain]) -
-		      DIV_ROUND_UP(ch1 * p->ch1,
-				   tsl2x7x_als_gain[chip->settings.als_gain]);
-	}
-
-	/* note: lux is 31 bit max at this point */
-	if (ch1lux > ch0lux) {
-		dev_dbg(&chip->client->dev, "ch1lux > ch0lux-return last value\n");
-		ret = chip->als_cur_info.lux;
-		goto out_unlock;
-	}
-
-	/* adjust for active time scale */
-	if (chip->als_time_scale == 0)
-		lux = 0;
-	else
-		lux = (lux + (chip->als_time_scale >> 1)) /
-			chip->als_time_scale;
-
-	/* adjust for active gain scale
-	 * The tsl2x7x_device_lux tables have a factor of 256 built-in.
-	 * User-specified gain provides a multiplier.
-	 * Apply user-specified gain before shifting right to retain precision.
-	 * Use 64 bits to avoid overflow on multiplication.
-	 * Then go back to 32 bits before division to avoid using div_u64().
-	 */
-
-	lux64 = lux;
-	lux64 = lux64 * chip->settings.als_gain_trim;
-	lux64 >>= 8;
-	lux = lux64;
-	lux = (lux + 500) / 1000;
-
-	if (lux > TSL2X7X_LUX_CALC_OVER_FLOW) /* check for overflow */
-		lux = TSL2X7X_LUX_CALC_OVER_FLOW;
-
-	/* Update the structure with the latest lux. */
-return_max:
-	chip->als_cur_info.lux = lux;
-	ret = lux;
-
-out_unlock:
-	mutex_unlock(&chip->als_mutex);
-
-	return ret;
-}
-
-/**
- * tsl2x7x_get_prox() - Reads proximity data registers and updates
- *                      chip->prox_data.
- *
- * @indio_dev:	pointer to IIO device
- */
-static int tsl2x7x_get_prox(struct iio_dev *indio_dev)
-{
-	int i;
-	int ret;
-	u8 chdata[2];
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-
-	mutex_lock(&chip->prox_mutex);
-
-	ret = tsl2x7x_read_status(chip);
-	if (ret < 0)
-		goto prox_poll_err;
-
-	switch (chip->id) {
-	case tsl2571:
-	case tsl2671:
-	case tmd2671:
-	case tsl2771:
-	case tmd2771:
-		if (!(ret & TSL2X7X_STA_ADC_VALID)) {
-			ret = -EINVAL;
-			goto prox_poll_err;
-		}
-		break;
-	case tsl2572:
-	case tsl2672:
-	case tmd2672:
-	case tsl2772:
-	case tmd2772:
-		if (!(ret & TSL2X7X_STA_PRX_VALID)) {
-			ret = -EINVAL;
-			goto prox_poll_err;
-		}
-		break;
-	}
-
-	for (i = 0; i < 2; i++) {
-		int reg = TSL2X7X_CMD_REG | (TSL2X7X_PRX_LO + i);
-
-		ret = i2c_smbus_read_byte_data(chip->client, reg);
-		if (ret < 0)
-			goto prox_poll_err;
-
-		chdata[i] = ret;
-	}
-
-	chip->prox_data = le16_to_cpup((const __le16 *)&chdata[0]);
-	ret = chip->prox_data;
-
-prox_poll_err:
-	mutex_unlock(&chip->prox_mutex);
-
-	return ret;
-}
-
-/**
- * tsl2x7x_defaults() - Populates the device nominal operating parameters
- *                      with those provided by a 'platform' data struct or
- *                      with prefined defaults.
- *
- * @chip:               pointer to device structure.
- */
-static void tsl2x7x_defaults(struct tsl2X7X_chip *chip)
-{
-	/* If Operational settings defined elsewhere.. */
-	if (chip->pdata && chip->pdata->platform_default_settings)
-		memcpy(&chip->settings, chip->pdata->platform_default_settings,
-		       sizeof(tsl2x7x_default_settings));
-	else
-		memcpy(&chip->settings, &tsl2x7x_default_settings,
-		       sizeof(tsl2x7x_default_settings));
-
-	/* Load up the proper lux table. */
-	if (chip->pdata && chip->pdata->platform_lux_table[0].ratio != 0)
-		memcpy(chip->tsl2x7x_device_lux,
-		       chip->pdata->platform_lux_table,
-		       sizeof(chip->pdata->platform_lux_table));
-	else
-		memcpy(chip->tsl2x7x_device_lux,
-		       tsl2x7x_default_lux_table_group[chip->id],
-		       TSL2X7X_DEFAULT_TABLE_BYTES);
-}
-
-/**
- * tsl2x7x_als_calibrate() -	Obtain single reading and calculate
- *                              the als_gain_trim.
- *
- * @indio_dev:	pointer to IIO device
- */
-static int tsl2x7x_als_calibrate(struct iio_dev *indio_dev)
-{
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	int ret, lux_val;
-
-	ret = i2c_smbus_read_byte_data(chip->client,
-				       TSL2X7X_CMD_REG | TSL2X7X_CNTRL);
-	if (ret < 0) {
-		dev_err(&chip->client->dev,
-			"%s: failed to read from the CNTRL register\n",
-			__func__);
-		return ret;
-	}
-
-	if ((ret & (TSL2X7X_CNTL_ADC_ENBL | TSL2X7X_CNTL_PWR_ON))
-			!= (TSL2X7X_CNTL_ADC_ENBL | TSL2X7X_CNTL_PWR_ON)) {
-		dev_err(&chip->client->dev,
-			"%s: Device is not powered on and/or ADC is not enabled\n",
-			__func__);
-		return -EINVAL;
-	} else if ((ret & TSL2X7X_STA_ADC_VALID) != TSL2X7X_STA_ADC_VALID) {
-		dev_err(&chip->client->dev,
-			"%s: The two ADC channels have not completed an integration cycle\n",
-			__func__);
-		return -ENODATA;
-	}
-
-	lux_val = tsl2x7x_get_lux(indio_dev);
-	if (lux_val < 0) {
-		dev_err(&chip->client->dev,
-			"%s: failed to get lux\n", __func__);
-		return lux_val;
-	}
-
-	ret = (chip->settings.als_cal_target * chip->settings.als_gain_trim) /
-			lux_val;
-	if (ret < 250 || ret > 4000)
-		return -ERANGE;
-
-	chip->settings.als_gain_trim = ret;
-	dev_info(&chip->client->dev,
-		 "%s als_calibrate completed\n", chip->client->name);
-
-	return ret;
-}
-
-static int tsl2x7x_chip_on(struct iio_dev *indio_dev)
-{
-	int i;
-	int ret = 0;
-	u8 *dev_reg;
-	int als_count;
-	int als_time;
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	u8 reg_val = 0;
-
-	/* Non calculated parameters */
-	chip->tsl2x7x_config[TSL2X7X_PRX_TIME] = chip->settings.prx_time;
-	chip->tsl2x7x_config[TSL2X7X_WAIT_TIME] = chip->settings.wait_time;
-	chip->tsl2x7x_config[TSL2X7X_PRX_CONFIG] = chip->settings.prox_config;
-
-	chip->tsl2x7x_config[TSL2X7X_ALS_MINTHRESHLO] =
-		(chip->settings.als_thresh_low) & 0xFF;
-	chip->tsl2x7x_config[TSL2X7X_ALS_MINTHRESHHI] =
-		(chip->settings.als_thresh_low >> 8) & 0xFF;
-	chip->tsl2x7x_config[TSL2X7X_ALS_MAXTHRESHLO] =
-		(chip->settings.als_thresh_high) & 0xFF;
-	chip->tsl2x7x_config[TSL2X7X_ALS_MAXTHRESHHI] =
-		(chip->settings.als_thresh_high >> 8) & 0xFF;
-	chip->tsl2x7x_config[TSL2X7X_PERSISTENCE] = chip->settings.persistence;
-
-	chip->tsl2x7x_config[TSL2X7X_PRX_COUNT] =
-			chip->settings.prox_pulse_count;
-	chip->tsl2x7x_config[TSL2X7X_PRX_MINTHRESHLO] =
-			(chip->settings.prox_thres_low) & 0xFF;
-	chip->tsl2x7x_config[TSL2X7X_PRX_MINTHRESHHI] =
-			(chip->settings.prox_thres_low >> 8) & 0xFF;
-	chip->tsl2x7x_config[TSL2X7X_PRX_MAXTHRESHLO] =
-			(chip->settings.prox_thres_high) & 0xFF;
-	chip->tsl2x7x_config[TSL2X7X_PRX_MAXTHRESHHI] =
-			(chip->settings.prox_thres_high >> 8) & 0xFF;
-
-	/* and make sure we're not already on */
-	if (chip->tsl2x7x_chip_status == TSL2X7X_CHIP_WORKING) {
-		/* if forcing a register update - turn off, then on */
-		dev_info(&chip->client->dev, "device is already enabled\n");
-		return -EINVAL;
-	}
-
-	/* determine als integration register */
-	als_count = (chip->settings.als_time * 100 + 135) / 270;
-	if (!als_count)
-		als_count = 1; /* ensure at least one cycle */
-
-	/* convert back to time (encompasses overrides) */
-	als_time = (als_count * 27 + 5) / 10;
-	chip->tsl2x7x_config[TSL2X7X_ALS_TIME] = 256 - als_count;
-
-	/* Set the gain based on tsl2x7x_settings struct */
-	chip->tsl2x7x_config[TSL2X7X_GAIN] =
-		(chip->settings.als_gain & 0xFF) |
-		((chip->settings.prox_gain & 0xFF) << 2) |
-		(chip->settings.prox_diode << 4) |
-		(chip->settings.prox_power << 6);
-
-	/* set chip struct re scaling and saturation */
-	chip->als_saturation = als_count * 922; /* 90% of full scale */
-	chip->als_time_scale = (als_time + 25) / 50;
-
-	/*
-	 * TSL2X7X Specific power-on / adc enable sequence
-	 * Power on the device 1st.
-	 */
-	ret = tsl2x7x_write_control_reg(chip, TSL2X7X_CNTL_PWR_ON);
-	if (ret < 0)
-		return ret;
-
-	/*
-	 * Use the following shadow copy for our delay before enabling ADC.
-	 * Write all the registers.
-	 */
-	for (i = 0, dev_reg = chip->tsl2x7x_config;
-			i < TSL2X7X_MAX_CONFIG_REG; i++) {
-		ret = i2c_smbus_write_byte_data(chip->client,
-						TSL2X7X_CMD_REG + i,
-						*dev_reg++);
-		if (ret < 0) {
-			dev_err(&chip->client->dev,
-				"failed on write to reg %d.\n", i);
-			return ret;
-		}
-	}
-
-	/* Power-on settling time */
-	usleep_range(3000, 3500);
-
-	/*
-	 * NOW enable the ADC
-	 * initialize the desired mode of operation
-	 */
-	ret = tsl2x7x_write_control_reg(chip,
-					TSL2X7X_CNTL_PWR_ON |
-					TSL2X7X_CNTL_ADC_ENBL |
-					TSL2X7X_CNTL_PROX_DET_ENBL);
-	if (ret < 0)
-		return ret;
-
-	chip->tsl2x7x_chip_status = TSL2X7X_CHIP_WORKING;
-
-	if (chip->settings.interrupts_en != 0) {
-		dev_info(&chip->client->dev, "Setting Up Interrupt(s)\n");
-
-		reg_val = TSL2X7X_CNTL_PWR_ON | TSL2X7X_CNTL_ADC_ENBL;
-		if (chip->settings.interrupts_en == 0x20 ||
-		    chip->settings.interrupts_en == 0x30)
-			reg_val |= TSL2X7X_CNTL_PROX_DET_ENBL;
-
-		reg_val |= chip->settings.interrupts_en;
-		ret = tsl2x7x_write_control_reg(chip, reg_val);
-		if (ret < 0)
-			return ret;
-
-		ret = tsl2x7x_clear_interrupts(chip,
-					       TSL2X7X_CMD_PROXALS_INT_CLR);
-		if (ret < 0)
-			return ret;
-	}
-
-	return ret;
-}
-
-static int tsl2x7x_chip_off(struct iio_dev *indio_dev)
-{
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-
-	/* turn device off */
-	chip->tsl2x7x_chip_status = TSL2X7X_CHIP_SUSPENDED;
-	return tsl2x7x_write_control_reg(chip, 0x00);
-}
-
-/**
- * tsl2x7x_invoke_change
- * @indio_dev:	pointer to IIO device
- *
- * Obtain and lock both ALS and PROX resources,
- * determine and save device state (On/Off),
- * cycle device to implement updated parameter,
- * put device back into proper state, and unlock
- * resource.
- */
-static int tsl2x7x_invoke_change(struct iio_dev *indio_dev)
-{
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	int device_status = chip->tsl2x7x_chip_status;
-	int ret;
-
-	mutex_lock(&chip->als_mutex);
-	mutex_lock(&chip->prox_mutex);
-
-	if (device_status == TSL2X7X_CHIP_WORKING) {
-		ret = tsl2x7x_chip_off(indio_dev);
-		if (ret < 0)
-			goto unlock;
-	}
-
-	ret = tsl2x7x_chip_on(indio_dev);
-
-unlock:
-	mutex_unlock(&chip->prox_mutex);
-	mutex_unlock(&chip->als_mutex);
-
-	return ret;
-}
-
-static void tsl2x7x_prox_calculate(int *data, int length,
-				   struct tsl2x7x_prox_stat *stat)
-{
-	int i;
-	int sample_sum;
-	int tmp;
-
-	if (!length)
-		length = 1;
-
-	sample_sum = 0;
-	stat->min = INT_MAX;
-	stat->max = INT_MIN;
-	for (i = 0; i < length; i++) {
-		sample_sum += data[i];
-		stat->min = min(stat->min, data[i]);
-		stat->max = max(stat->max, data[i]);
-	}
-
-	stat->mean = sample_sum / length;
-	sample_sum = 0;
-	for (i = 0; i < length; i++) {
-		tmp = data[i] - stat->mean;
-		sample_sum += tmp * tmp;
-	}
-	stat->stddev = int_sqrt((long)sample_sum / length);
-}
-
-/**
- * tsl2x7x_prox_cal() - Calculates std. and sets thresholds.
- * @indio_dev:	pointer to IIO device
- *
- * Calculates a standard deviation based on the samples,
- * and sets the threshold accordingly.
- */
-static int tsl2x7x_prox_cal(struct iio_dev *indio_dev)
-{
-	int prox_history[MAX_SAMPLES_CAL + 1];
-	int i, ret;
-	struct tsl2x7x_prox_stat prox_stat_data[2];
-	struct tsl2x7x_prox_stat *cal;
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	u8 tmp_irq_settings;
-	u8 current_state = chip->tsl2x7x_chip_status;
-
-	if (chip->settings.prox_max_samples_cal > MAX_SAMPLES_CAL) {
-		dev_err(&chip->client->dev,
-			"max prox samples cal is too big: %d\n",
-			chip->settings.prox_max_samples_cal);
-		chip->settings.prox_max_samples_cal = MAX_SAMPLES_CAL;
-	}
-
-	/* have to stop to change settings */
-	ret = tsl2x7x_chip_off(indio_dev);
-	if (ret < 0)
-		return ret;
-
-	/* Enable proximity detection save just in case prox not wanted yet*/
-	tmp_irq_settings = chip->settings.interrupts_en;
-	chip->settings.interrupts_en |= TSL2X7X_CNTL_PROX_INT_ENBL;
-
-	/*turn on device if not already on*/
-	ret = tsl2x7x_chip_on(indio_dev);
-	if (ret < 0)
-		return ret;
-
-	/*gather the samples*/
-	for (i = 0; i < chip->settings.prox_max_samples_cal; i++) {
-		usleep_range(15000, 17500);
-		ret = tsl2x7x_get_prox(indio_dev);
-		if (ret < 0)
-			return ret;
-		prox_history[i] = chip->prox_data;
-		dev_info(&chip->client->dev, "2 i=%d prox data= %d\n",
-			 i, chip->prox_data);
-	}
-
-	ret = tsl2x7x_chip_off(indio_dev);
-	if (ret < 0)
-		return ret;
-	cal = &prox_stat_data[PROX_STAT_CAL];
-	tsl2x7x_prox_calculate(prox_history,
-			       chip->settings.prox_max_samples_cal, cal);
-	chip->settings.prox_thres_high = (cal->max << 1) - cal->mean;
-
-	dev_info(&chip->client->dev, " cal min=%d mean=%d max=%d\n",
-		 cal->min, cal->mean, cal->max);
-	dev_info(&chip->client->dev,
-		 "%s proximity threshold set to %d\n",
-		 chip->client->name, chip->settings.prox_thres_high);
-
-	/* back to the way they were */
-	chip->settings.interrupts_en = tmp_irq_settings;
-	if (current_state == TSL2X7X_CHIP_WORKING) {
-		ret = tsl2x7x_chip_on(indio_dev);
-		if (ret < 0)
-			return ret;
-	}
-
-	return 0;
-}
-
-static ssize_t
-in_illuminance0_calibscale_available_show(struct device *dev,
-					  struct device_attribute *attr,
-					  char *buf)
-{
-	struct tsl2X7X_chip *chip = iio_priv(dev_to_iio_dev(dev));
-
-	switch (chip->id) {
-	case tsl2571:
-	case tsl2671:
-	case tmd2671:
-	case tsl2771:
-	case tmd2771:
-		return snprintf(buf, PAGE_SIZE, "%s\n", "1 8 16 128");
-	}
-
-	return snprintf(buf, PAGE_SIZE, "%s\n", "1 8 16 120");
-}
-
-static IIO_CONST_ATTR(in_proximity0_calibscale_available, "1 2 4 8");
-
-static IIO_CONST_ATTR(in_illuminance0_integration_time_available,
-		".00272 - .696");
-
-static ssize_t in_illuminance0_target_input_show(struct device *dev,
-						 struct device_attribute *attr,
-						 char *buf)
-{
-	struct tsl2X7X_chip *chip = iio_priv(dev_to_iio_dev(dev));
-
-	return snprintf(buf, PAGE_SIZE, "%d\n", chip->settings.als_cal_target);
-}
-
-static ssize_t in_illuminance0_target_input_store(struct device *dev,
-						  struct device_attribute *attr,
-						  const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	unsigned long value;
-	int ret;
-
-	if (kstrtoul(buf, 0, &value))
-		return -EINVAL;
-
-	if (value)
-		chip->settings.als_cal_target = value;
-
-	ret = tsl2x7x_invoke_change(indio_dev);
-	if (ret < 0)
-		return ret;
-
-	return len;
-}
-
-static ssize_t in_illuminance0_calibrate_store(struct device *dev,
-					       struct device_attribute *attr,
-					       const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	bool value;
-	int ret;
-
-	if (strtobool(buf, &value))
-		return -EINVAL;
-
-	if (value) {
-		ret = tsl2x7x_als_calibrate(indio_dev);
-		if (ret < 0)
-			return ret;
-	}
-
-	ret = tsl2x7x_invoke_change(indio_dev);
-	if (ret < 0)
-		return ret;
-
-	return len;
-}
-
-static ssize_t in_illuminance0_lux_table_show(struct device *dev,
-					      struct device_attribute *attr,
-					      char *buf)
-{
-	struct tsl2X7X_chip *chip = iio_priv(dev_to_iio_dev(dev));
-	int i = 0;
-	int offset = 0;
-
-	while (i < TSL2X7X_MAX_LUX_TABLE_SIZE) {
-		offset += snprintf(buf + offset, PAGE_SIZE, "%u,%u,%u,",
-			chip->tsl2x7x_device_lux[i].ratio,
-			chip->tsl2x7x_device_lux[i].ch0,
-			chip->tsl2x7x_device_lux[i].ch1);
-		if (chip->tsl2x7x_device_lux[i].ratio == 0) {
-			/*
-			 * We just printed the first "0" entry.
-			 * Now get rid of the extra "," and break.
-			 */
-			offset--;
-			break;
-		}
-		i++;
-	}
-
-	offset += snprintf(buf + offset, PAGE_SIZE, "\n");
-	return offset;
-}
-
-static ssize_t in_illuminance0_lux_table_store(struct device *dev,
-					       struct device_attribute *attr,
-					       const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	int value[ARRAY_SIZE(chip->tsl2x7x_device_lux) * 3 + 1];
-	int n, ret;
-
-	get_options(buf, ARRAY_SIZE(value), value);
-
-	/* We now have an array of ints starting at value[1], and
-	 * enumerated by value[0].
-	 * We expect each group of three ints is one table entry,
-	 * and the last table entry is all 0.
-	 */
-	n = value[0];
-	if ((n % 3) || n < 6 ||
-	    n > ((ARRAY_SIZE(chip->tsl2x7x_device_lux) - 1) * 3)) {
-		dev_info(dev, "LUX TABLE INPUT ERROR 1 Value[0]=%d\n", n);
-		return -EINVAL;
-	}
-
-	if ((value[(n - 2)] | value[(n - 1)] | value[n]) != 0) {
-		dev_info(dev, "LUX TABLE INPUT ERROR 2 Value[0]=%d\n", n);
-		return -EINVAL;
-	}
-
-	if (chip->tsl2x7x_chip_status == TSL2X7X_CHIP_WORKING) {
-		ret = tsl2x7x_chip_off(indio_dev);
-		if (ret < 0)
-			return ret;
-	}
-
-	/* Zero out the table */
-	memset(chip->tsl2x7x_device_lux, 0, sizeof(chip->tsl2x7x_device_lux));
-	memcpy(chip->tsl2x7x_device_lux, &value[1], (value[0] * 4));
-
-	ret = tsl2x7x_invoke_change(indio_dev);
-	if (ret < 0)
-		return ret;
-
-	return len;
-}
-
-static ssize_t in_proximity0_calibrate_store(struct device *dev,
-					     struct device_attribute *attr,
-					     const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	bool value;
-	int ret;
-
-	if (strtobool(buf, &value))
-		return -EINVAL;
-
-	if (value) {
-		ret = tsl2x7x_prox_cal(indio_dev);
-		if (ret < 0)
-			return ret;
-	}
-
-	ret = tsl2x7x_invoke_change(indio_dev);
-	if (ret < 0)
-		return ret;
-
-	return len;
-}
-
-static int tsl2x7x_read_interrupt_config(struct iio_dev *indio_dev,
-					 const struct iio_chan_spec *chan,
-					 enum iio_event_type type,
-					 enum iio_event_direction dir)
-{
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	int ret;
-
-	if (chan->type == IIO_INTENSITY)
-		ret = !!(chip->settings.interrupts_en & 0x10);
-	else
-		ret = !!(chip->settings.interrupts_en & 0x20);
-
-	return ret;
-}
-
-static int tsl2x7x_write_interrupt_config(struct iio_dev *indio_dev,
-					  const struct iio_chan_spec *chan,
-					  enum iio_event_type type,
-					  enum iio_event_direction dir,
-					  int val)
-{
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	int ret;
-
-	if (chan->type == IIO_INTENSITY) {
-		if (val)
-			chip->settings.interrupts_en |= 0x10;
-		else
-			chip->settings.interrupts_en &= 0x20;
-	} else {
-		if (val)
-			chip->settings.interrupts_en |= 0x20;
-		else
-			chip->settings.interrupts_en &= 0x10;
-	}
-
-	ret = tsl2x7x_invoke_change(indio_dev);
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-
-static int tsl2x7x_write_event_value(struct iio_dev *indio_dev,
-				     const struct iio_chan_spec *chan,
-				     enum iio_event_type type,
-				     enum iio_event_direction dir,
-				     enum iio_event_info info,
-				     int val, int val2)
-{
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	int ret = -EINVAL, y, z, filter_delay;
-	u8 time;
-
-	switch (info) {
-	case IIO_EV_INFO_VALUE:
-		if (chan->type == IIO_INTENSITY) {
-			switch (dir) {
-			case IIO_EV_DIR_RISING:
-				chip->settings.als_thresh_high = val;
-				ret = 0;
-				break;
-			case IIO_EV_DIR_FALLING:
-				chip->settings.als_thresh_low = val;
-				ret = 0;
-				break;
-			default:
-				break;
-			}
-		} else {
-			switch (dir) {
-			case IIO_EV_DIR_RISING:
-				chip->settings.prox_thres_high = val;
-				ret = 0;
-				break;
-			case IIO_EV_DIR_FALLING:
-				chip->settings.prox_thres_low = val;
-				ret = 0;
-				break;
-			default:
-				break;
-			}
-		}
-		break;
-	case IIO_EV_INFO_PERIOD:
-		if (chan->type == IIO_INTENSITY)
-			time = chip->settings.als_time;
-		else
-			time = chip->settings.prx_time;
-
-		y = (TSL2X7X_MAX_TIMER_CNT - time) + 1;
-		z = y * TSL2X7X_MIN_ITIME;
-
-		filter_delay = DIV_ROUND_UP((val * 1000) + val2, z);
-
-		if (chan->type == IIO_INTENSITY) {
-			chip->settings.persistence &= 0xF0;
-			chip->settings.persistence |=
-				(filter_delay & 0x0F);
-			dev_info(&chip->client->dev, "%s: ALS persistence = %d",
-				 __func__, filter_delay);
-		} else {
-			chip->settings.persistence &= 0x0F;
-			chip->settings.persistence |=
-				((filter_delay << 4) & 0xF0);
-			dev_info(&chip->client->dev,
-				 "%s: Proximity persistence = %d",
-				 __func__, filter_delay);
-		}
-		ret = 0;
-		break;
-	default:
-		break;
-	}
-
-	if (ret < 0)
-		return ret;
-
-	return tsl2x7x_invoke_change(indio_dev);
-}
-
-static int tsl2x7x_read_event_value(struct iio_dev *indio_dev,
-				    const struct iio_chan_spec *chan,
-				    enum iio_event_type type,
-				    enum iio_event_direction dir,
-				    enum iio_event_info info,
-				    int *val, int *val2)
-{
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	int ret = -EINVAL, filter_delay, mult;
-	u8 time;
-
-	switch (info) {
-	case IIO_EV_INFO_VALUE:
-		if (chan->type == IIO_INTENSITY) {
-			switch (dir) {
-			case IIO_EV_DIR_RISING:
-				*val = chip->settings.als_thresh_high;
-				ret = IIO_VAL_INT;
-				break;
-			case IIO_EV_DIR_FALLING:
-				*val = chip->settings.als_thresh_low;
-				ret = IIO_VAL_INT;
-				break;
-			default:
-				break;
-			}
-		} else {
-			switch (dir) {
-			case IIO_EV_DIR_RISING:
-				*val = chip->settings.prox_thres_high;
-				ret = IIO_VAL_INT;
-				break;
-			case IIO_EV_DIR_FALLING:
-				*val = chip->settings.prox_thres_low;
-				ret = IIO_VAL_INT;
-				break;
-			default:
-				break;
-			}
-		}
-		break;
-	case IIO_EV_INFO_PERIOD:
-		if (chan->type == IIO_INTENSITY) {
-			time = chip->settings.als_time;
-			mult = chip->settings.persistence & 0x0F;
-		} else {
-			time = chip->settings.prx_time;
-			mult = (chip->settings.persistence & 0xF0) >> 4;
-		}
-
-		/* Determine integration time */
-		*val = (TSL2X7X_MAX_TIMER_CNT - time) + 1;
-		*val2 = *val * TSL2X7X_MIN_ITIME;
-		filter_delay = *val2 * mult;
-		*val = filter_delay / 1000;
-		*val2 = filter_delay % 1000;
-		ret = IIO_VAL_INT_PLUS_MICRO;
-		break;
-	default:
-		break;
-	}
-
-	return ret;
-}
-
-static int tsl2x7x_read_raw(struct iio_dev *indio_dev,
-			    struct iio_chan_spec const *chan,
-			    int *val,
-			    int *val2,
-			    long mask)
-{
-	int ret = -EINVAL;
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-
-	switch (mask) {
-	case IIO_CHAN_INFO_PROCESSED:
-		switch (chan->type) {
-		case IIO_LIGHT:
-			tsl2x7x_get_lux(indio_dev);
-			*val = chip->als_cur_info.lux;
-			ret = IIO_VAL_INT;
-			break;
-		default:
-			return -EINVAL;
-		}
-		break;
-	case IIO_CHAN_INFO_RAW:
-		switch (chan->type) {
-		case IIO_INTENSITY:
-			tsl2x7x_get_lux(indio_dev);
-			if (chan->channel == 0)
-				*val = chip->als_cur_info.als_ch0;
-			else
-				*val = chip->als_cur_info.als_ch1;
-			ret = IIO_VAL_INT;
-			break;
-		case IIO_PROXIMITY:
-			tsl2x7x_get_prox(indio_dev);
-			*val = chip->prox_data;
-			ret = IIO_VAL_INT;
-			break;
-		default:
-			return -EINVAL;
-		}
-		break;
-	case IIO_CHAN_INFO_CALIBSCALE:
-		if (chan->type == IIO_LIGHT)
-			*val = tsl2x7x_als_gain[chip->settings.als_gain];
-		else
-			*val = tsl2x7x_prx_gain[chip->settings.prox_gain];
-		ret = IIO_VAL_INT;
-		break;
-	case IIO_CHAN_INFO_CALIBBIAS:
-		*val = chip->settings.als_gain_trim;
-		ret = IIO_VAL_INT;
-		break;
-	case IIO_CHAN_INFO_INT_TIME:
-		*val = (TSL2X7X_MAX_TIMER_CNT - chip->settings.als_time) + 1;
-		*val2 = ((*val * TSL2X7X_MIN_ITIME) % 1000) / 1000;
-		ret = IIO_VAL_INT_PLUS_MICRO;
-		break;
-	default:
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-static int tsl2x7x_write_raw(struct iio_dev *indio_dev,
-			     struct iio_chan_spec const *chan,
-			     int val,
-			     int val2,
-			     long mask)
-{
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-
-	switch (mask) {
-	case IIO_CHAN_INFO_CALIBSCALE:
-		if (chan->type == IIO_INTENSITY) {
-			switch (val) {
-			case 1:
-				chip->settings.als_gain = 0;
-				break;
-			case 8:
-				chip->settings.als_gain = 1;
-				break;
-			case 16:
-				chip->settings.als_gain = 2;
-				break;
-			case 120:
-				switch (chip->id) {
-				case tsl2572:
-				case tsl2672:
-				case tmd2672:
-				case tsl2772:
-				case tmd2772:
-					return -EINVAL;
-				}
-				chip->settings.als_gain = 3;
-				break;
-			case 128:
-				switch (chip->id) {
-				case tsl2571:
-				case tsl2671:
-				case tmd2671:
-				case tsl2771:
-				case tmd2771:
-					return -EINVAL;
-				}
-				chip->settings.als_gain = 3;
-				break;
-			default:
-				return -EINVAL;
-			}
-		} else {
-			switch (val) {
-			case 1:
-				chip->settings.prox_gain = 0;
-				break;
-			case 2:
-				chip->settings.prox_gain = 1;
-				break;
-			case 4:
-				chip->settings.prox_gain = 2;
-				break;
-			case 8:
-				chip->settings.prox_gain = 3;
-				break;
-			default:
-				return -EINVAL;
-			}
-		}
-		break;
-	case IIO_CHAN_INFO_CALIBBIAS:
-		chip->settings.als_gain_trim = val;
-		break;
-	case IIO_CHAN_INFO_INT_TIME:
-		chip->settings.als_time =
-			TSL2X7X_MAX_TIMER_CNT - (val2 / TSL2X7X_MIN_ITIME);
-
-		dev_info(&chip->client->dev, "%s: als time = %d",
-			 __func__, chip->settings.als_time);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return tsl2x7x_invoke_change(indio_dev);
-}
-
-static DEVICE_ATTR_RO(in_illuminance0_calibscale_available);
-
-static DEVICE_ATTR_RW(in_illuminance0_target_input);
-
-static DEVICE_ATTR_WO(in_illuminance0_calibrate);
-
-static DEVICE_ATTR_WO(in_proximity0_calibrate);
-
-static DEVICE_ATTR_RW(in_illuminance0_lux_table);
-
-/* Use the default register values to identify the Taos device */
-static int tsl2x7x_device_id(int *id, int target)
-{
-	switch (target) {
-	case tsl2571:
-	case tsl2671:
-	case tsl2771:
-		return (*id & 0xf0) == TRITON_ID;
-	case tmd2671:
-	case tmd2771:
-		return (*id & 0xf0) == HALIBUT_ID;
-	case tsl2572:
-	case tsl2672:
-	case tmd2672:
-	case tsl2772:
-	case tmd2772:
-		return (*id & 0xf0) == SWORDFISH_ID;
-	}
-
-	return -EINVAL;
-}
-
-static irqreturn_t tsl2x7x_event_handler(int irq, void *private)
-{
-	struct iio_dev *indio_dev = private;
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	s64 timestamp = iio_get_time_ns(indio_dev);
-	int ret;
-
-	ret = tsl2x7x_read_status(chip);
-	if (ret < 0)
-		return ret;
-
-	/* What type of interrupt do we need to process */
-	if (ret & TSL2X7X_STA_PRX_INTR) {
-		tsl2x7x_get_prox(indio_dev); /* freshen data for ABI */
-		iio_push_event(indio_dev,
-			       IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY,
-						    0,
-						    IIO_EV_TYPE_THRESH,
-						    IIO_EV_DIR_EITHER),
-						    timestamp);
-	}
-
-	if (ret & TSL2X7X_STA_ALS_INTR) {
-		tsl2x7x_get_lux(indio_dev); /* freshen data for ABI */
-		iio_push_event(indio_dev,
-			       IIO_UNMOD_EVENT_CODE(IIO_LIGHT,
-						    0,
-						    IIO_EV_TYPE_THRESH,
-						    IIO_EV_DIR_EITHER),
-			       timestamp);
-	}
-
-	ret = tsl2x7x_clear_interrupts(chip, TSL2X7X_CMD_PROXALS_INT_CLR);
-	if (ret < 0)
-		return ret;
-
-	return IRQ_HANDLED;
-}
-
-static struct attribute *tsl2x7x_ALS_device_attrs[] = {
-	&dev_attr_in_illuminance0_calibscale_available.attr,
-	&iio_const_attr_in_illuminance0_integration_time_available
-		.dev_attr.attr,
-	&dev_attr_in_illuminance0_target_input.attr,
-	&dev_attr_in_illuminance0_calibrate.attr,
-	&dev_attr_in_illuminance0_lux_table.attr,
-	NULL
-};
-
-static struct attribute *tsl2x7x_PRX_device_attrs[] = {
-	&dev_attr_in_proximity0_calibrate.attr,
-	NULL
-};
-
-static struct attribute *tsl2x7x_ALSPRX_device_attrs[] = {
-	&dev_attr_in_illuminance0_calibscale_available.attr,
-	&iio_const_attr_in_illuminance0_integration_time_available
-		.dev_attr.attr,
-	&dev_attr_in_illuminance0_target_input.attr,
-	&dev_attr_in_illuminance0_calibrate.attr,
-	&dev_attr_in_illuminance0_lux_table.attr,
-	NULL
-};
-
-static struct attribute *tsl2x7x_PRX2_device_attrs[] = {
-	&dev_attr_in_proximity0_calibrate.attr,
-	&iio_const_attr_in_proximity0_calibscale_available.dev_attr.attr,
-	NULL
-};
-
-static struct attribute *tsl2x7x_ALSPRX2_device_attrs[] = {
-	&dev_attr_in_illuminance0_calibscale_available.attr,
-	&iio_const_attr_in_illuminance0_integration_time_available
-		.dev_attr.attr,
-	&dev_attr_in_illuminance0_target_input.attr,
-	&dev_attr_in_illuminance0_calibrate.attr,
-	&dev_attr_in_illuminance0_lux_table.attr,
-	&dev_attr_in_proximity0_calibrate.attr,
-	&iio_const_attr_in_proximity0_calibscale_available.dev_attr.attr,
-	NULL
-};
-
-static const struct attribute_group tsl2X7X_device_attr_group_tbl[] = {
-	[ALS] = {
-		.attrs = tsl2x7x_ALS_device_attrs,
-	},
-	[PRX] = {
-		.attrs = tsl2x7x_PRX_device_attrs,
-	},
-	[ALSPRX] = {
-		.attrs = tsl2x7x_ALSPRX_device_attrs,
-	},
-	[PRX2] = {
-		.attrs = tsl2x7x_PRX2_device_attrs,
-	},
-	[ALSPRX2] = {
-		.attrs = tsl2x7x_ALSPRX2_device_attrs,
-	},
-};
-
-static const struct iio_info tsl2X7X_device_info[] = {
-	[ALS] = {
-		.attrs = &tsl2X7X_device_attr_group_tbl[ALS],
-		.read_raw = &tsl2x7x_read_raw,
-		.write_raw = &tsl2x7x_write_raw,
-		.read_event_value = &tsl2x7x_read_event_value,
-		.write_event_value = &tsl2x7x_write_event_value,
-		.read_event_config = &tsl2x7x_read_interrupt_config,
-		.write_event_config = &tsl2x7x_write_interrupt_config,
-	},
-	[PRX] = {
-		.attrs = &tsl2X7X_device_attr_group_tbl[PRX],
-		.read_raw = &tsl2x7x_read_raw,
-		.write_raw = &tsl2x7x_write_raw,
-		.read_event_value = &tsl2x7x_read_event_value,
-		.write_event_value = &tsl2x7x_write_event_value,
-		.read_event_config = &tsl2x7x_read_interrupt_config,
-		.write_event_config = &tsl2x7x_write_interrupt_config,
-	},
-	[ALSPRX] = {
-		.attrs = &tsl2X7X_device_attr_group_tbl[ALSPRX],
-		.read_raw = &tsl2x7x_read_raw,
-		.write_raw = &tsl2x7x_write_raw,
-		.read_event_value = &tsl2x7x_read_event_value,
-		.write_event_value = &tsl2x7x_write_event_value,
-		.read_event_config = &tsl2x7x_read_interrupt_config,
-		.write_event_config = &tsl2x7x_write_interrupt_config,
-	},
-	[PRX2] = {
-		.attrs = &tsl2X7X_device_attr_group_tbl[PRX2],
-		.read_raw = &tsl2x7x_read_raw,
-		.write_raw = &tsl2x7x_write_raw,
-		.read_event_value = &tsl2x7x_read_event_value,
-		.write_event_value = &tsl2x7x_write_event_value,
-		.read_event_config = &tsl2x7x_read_interrupt_config,
-		.write_event_config = &tsl2x7x_write_interrupt_config,
-	},
-	[ALSPRX2] = {
-		.attrs = &tsl2X7X_device_attr_group_tbl[ALSPRX2],
-		.read_raw = &tsl2x7x_read_raw,
-		.write_raw = &tsl2x7x_write_raw,
-		.read_event_value = &tsl2x7x_read_event_value,
-		.write_event_value = &tsl2x7x_write_event_value,
-		.read_event_config = &tsl2x7x_read_interrupt_config,
-		.write_event_config = &tsl2x7x_write_interrupt_config,
-	},
-};
-
-static const struct iio_event_spec tsl2x7x_events[] = {
-	{
-		.type = IIO_EV_TYPE_THRESH,
-		.dir = IIO_EV_DIR_RISING,
-		.mask_separate = BIT(IIO_EV_INFO_VALUE) |
-			BIT(IIO_EV_INFO_ENABLE),
-	}, {
-		.type = IIO_EV_TYPE_THRESH,
-		.dir = IIO_EV_DIR_FALLING,
-		.mask_separate = BIT(IIO_EV_INFO_VALUE) |
-			BIT(IIO_EV_INFO_ENABLE),
-	}, {
-		.type = IIO_EV_TYPE_THRESH,
-		.dir = IIO_EV_DIR_EITHER,
-		.mask_separate = BIT(IIO_EV_INFO_PERIOD),
-	},
-};
-
-static const struct tsl2x7x_chip_info tsl2x7x_chip_info_tbl[] = {
-	[ALS] = {
-		.channel = {
-			{
-			.type = IIO_LIGHT,
-			.indexed = 1,
-			.channel = 0,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED) |
-					      BIT(IIO_CHAN_INFO_INT_TIME),
-			}, {
-			.type = IIO_INTENSITY,
-			.indexed = 1,
-			.channel = 0,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
-				BIT(IIO_CHAN_INFO_CALIBSCALE) |
-				BIT(IIO_CHAN_INFO_CALIBBIAS),
-			.event_spec = tsl2x7x_events,
-			.num_event_specs = ARRAY_SIZE(tsl2x7x_events),
-			}, {
-			.type = IIO_INTENSITY,
-			.indexed = 1,
-			.channel = 1,
-			},
-		},
-	.chan_table_elements = 3,
-	.info = &tsl2X7X_device_info[ALS],
-	},
-	[PRX] = {
-		.channel = {
-			{
-			.type = IIO_PROXIMITY,
-			.indexed = 1,
-			.channel = 0,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
-			.event_spec = tsl2x7x_events,
-			.num_event_specs = ARRAY_SIZE(tsl2x7x_events),
-			},
-		},
-	.chan_table_elements = 1,
-	.info = &tsl2X7X_device_info[PRX],
-	},
-	[ALSPRX] = {
-		.channel = {
-			{
-			.type = IIO_LIGHT,
-			.indexed = 1,
-			.channel = 0,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED) |
-					      BIT(IIO_CHAN_INFO_INT_TIME),
-			}, {
-			.type = IIO_INTENSITY,
-			.indexed = 1,
-			.channel = 0,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
-				BIT(IIO_CHAN_INFO_CALIBSCALE) |
-				BIT(IIO_CHAN_INFO_CALIBBIAS),
-			.event_spec = tsl2x7x_events,
-			.num_event_specs = ARRAY_SIZE(tsl2x7x_events),
-			}, {
-			.type = IIO_INTENSITY,
-			.indexed = 1,
-			.channel = 1,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
-			}, {
-			.type = IIO_PROXIMITY,
-			.indexed = 1,
-			.channel = 0,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
-			.event_spec = tsl2x7x_events,
-			.num_event_specs = ARRAY_SIZE(tsl2x7x_events),
-			},
-		},
-	.chan_table_elements = 4,
-	.info = &tsl2X7X_device_info[ALSPRX],
-	},
-	[PRX2] = {
-		.channel = {
-			{
-			.type = IIO_PROXIMITY,
-			.indexed = 1,
-			.channel = 0,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
-				BIT(IIO_CHAN_INFO_CALIBSCALE),
-			.event_spec = tsl2x7x_events,
-			.num_event_specs = ARRAY_SIZE(tsl2x7x_events),
-			},
-		},
-	.chan_table_elements = 1,
-	.info = &tsl2X7X_device_info[PRX2],
-	},
-	[ALSPRX2] = {
-		.channel = {
-			{
-			.type = IIO_LIGHT,
-			.indexed = 1,
-			.channel = 0,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED) |
-					      BIT(IIO_CHAN_INFO_INT_TIME),
-			}, {
-			.type = IIO_INTENSITY,
-			.indexed = 1,
-			.channel = 0,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
-				BIT(IIO_CHAN_INFO_CALIBSCALE) |
-				BIT(IIO_CHAN_INFO_CALIBBIAS),
-			.event_spec = tsl2x7x_events,
-			.num_event_specs = ARRAY_SIZE(tsl2x7x_events),
-			}, {
-			.type = IIO_INTENSITY,
-			.indexed = 1,
-			.channel = 1,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
-			}, {
-			.type = IIO_PROXIMITY,
-			.indexed = 1,
-			.channel = 0,
-			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
-				BIT(IIO_CHAN_INFO_CALIBSCALE),
-			.event_spec = tsl2x7x_events,
-			.num_event_specs = ARRAY_SIZE(tsl2x7x_events),
-			},
-		},
-	.chan_table_elements = 4,
-	.info = &tsl2X7X_device_info[ALSPRX2],
-	},
-};
-
-static int tsl2x7x_probe(struct i2c_client *clientp,
-			 const struct i2c_device_id *id)
-{
-	int ret;
-	struct iio_dev *indio_dev;
-	struct tsl2X7X_chip *chip;
-
-	indio_dev = devm_iio_device_alloc(&clientp->dev, sizeof(*chip));
-	if (!indio_dev)
-		return -ENOMEM;
-
-	chip = iio_priv(indio_dev);
-	chip->client = clientp;
-	i2c_set_clientdata(clientp, indio_dev);
-
-	ret = i2c_smbus_read_byte_data(chip->client,
-				       TSL2X7X_CMD_REG | TSL2X7X_CHIPID);
-	if (ret < 0)
-		return ret;
-
-	if ((!tsl2x7x_device_id(&ret, id->driver_data)) ||
-	    (tsl2x7x_device_id(&ret, id->driver_data) == -EINVAL)) {
-		dev_info(&chip->client->dev,
-			 "%s: i2c device found does not match expected id\n",
-				__func__);
-		return -EINVAL;
-	}
-
-	ret = i2c_smbus_write_byte(clientp, TSL2X7X_CMD_REG | TSL2X7X_CNTRL);
-	if (ret < 0) {
-		dev_err(&clientp->dev, "write to cmd reg failed. err = %d\n",
-			ret);
-		return ret;
-	}
-
-	/*
-	 * ALS and PROX functions can be invoked via user space poll
-	 * or H/W interrupt. If busy return last sample.
-	 */
-	mutex_init(&chip->als_mutex);
-	mutex_init(&chip->prox_mutex);
-
-	chip->tsl2x7x_chip_status = TSL2X7X_CHIP_UNKNOWN;
-	chip->pdata = dev_get_platdata(&clientp->dev);
-	chip->id = id->driver_data;
-	chip->chip_info =
-		&tsl2x7x_chip_info_tbl[device_channel_config[id->driver_data]];
-
-	indio_dev->info = chip->chip_info->info;
-	indio_dev->dev.parent = &clientp->dev;
-	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->name = chip->client->name;
-	indio_dev->channels = chip->chip_info->channel;
-	indio_dev->num_channels = chip->chip_info->chan_table_elements;
-
-	if (clientp->irq) {
-		ret = devm_request_threaded_irq(&clientp->dev, clientp->irq,
-						NULL,
-						&tsl2x7x_event_handler,
-						IRQF_TRIGGER_RISING |
-						IRQF_ONESHOT,
-						"TSL2X7X_event",
-						indio_dev);
-		if (ret) {
-			dev_err(&clientp->dev,
-				"%s: irq request failed", __func__);
-			return ret;
-		}
-	}
-
-	/* Load up the defaults */
-	tsl2x7x_defaults(chip);
-	/* Make sure the chip is on */
-	tsl2x7x_chip_on(indio_dev);
-
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&clientp->dev,
-			"%s: iio registration failed\n", __func__);
-		return ret;
-	}
-
-	dev_info(&clientp->dev, "%s Light sensor found.\n", id->name);
-
-	return 0;
-}
-
-static int tsl2x7x_suspend(struct device *dev)
-{
-	struct iio_dev *indio_dev = dev_get_drvdata(dev);
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	int ret = 0;
-
-	if (chip->tsl2x7x_chip_status == TSL2X7X_CHIP_WORKING) {
-		ret = tsl2x7x_chip_off(indio_dev);
-		chip->tsl2x7x_chip_status = TSL2X7X_CHIP_SUSPENDED;
-	}
-
-	return ret;
-}
-
-static int tsl2x7x_resume(struct device *dev)
-{
-	struct iio_dev *indio_dev = dev_get_drvdata(dev);
-	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	int ret = 0;
-
-	if (chip->tsl2x7x_chip_status == TSL2X7X_CHIP_SUSPENDED)
-		ret = tsl2x7x_chip_on(indio_dev);
-
-	return ret;
-}
-
-static int tsl2x7x_remove(struct i2c_client *client)
-{
-	struct iio_dev *indio_dev = i2c_get_clientdata(client);
-
-	tsl2x7x_chip_off(indio_dev);
-
-	iio_device_unregister(indio_dev);
-
-	return 0;
-}
-
-static const struct i2c_device_id tsl2x7x_idtable[] = {
-	{ "tsl2571", tsl2571 },
-	{ "tsl2671", tsl2671 },
-	{ "tmd2671", tmd2671 },
-	{ "tsl2771", tsl2771 },
-	{ "tmd2771", tmd2771 },
-	{ "tsl2572", tsl2572 },
-	{ "tsl2672", tsl2672 },
-	{ "tmd2672", tmd2672 },
-	{ "tsl2772", tsl2772 },
-	{ "tmd2772", tmd2772 },
-	{}
-};
-
-MODULE_DEVICE_TABLE(i2c, tsl2x7x_idtable);
-
-static const struct of_device_id tsl2x7x_of_match[] = {
-	{ .compatible = "amstaos,tsl2571" },
-	{ .compatible = "amstaos,tsl2671" },
-	{ .compatible = "amstaos,tmd2671" },
-	{ .compatible = "amstaos,tsl2771" },
-	{ .compatible = "amstaos,tmd2771" },
-	{ .compatible = "amstaos,tsl2572" },
-	{ .compatible = "amstaos,tsl2672" },
-	{ .compatible = "amstaos,tmd2672" },
-	{ .compatible = "amstaos,tsl2772" },
-	{ .compatible = "amstaos,tmd2772" },
-	{}
-};
-MODULE_DEVICE_TABLE(of, tsl2x7x_of_match);
-
-static const struct dev_pm_ops tsl2x7x_pm_ops = {
-	.suspend = tsl2x7x_suspend,
-	.resume  = tsl2x7x_resume,
-};
-
-/* Driver definition */
-static struct i2c_driver tsl2x7x_driver = {
-	.driver = {
-		.name = "tsl2x7x",
-		.of_match_table = tsl2x7x_of_match,
-		.pm = &tsl2x7x_pm_ops,
-	},
-	.id_table = tsl2x7x_idtable,
-	.probe = tsl2x7x_probe,
-	.remove = tsl2x7x_remove,
-};
-
-module_i2c_driver(tsl2x7x_driver);
-
-MODULE_AUTHOR("J. August Brenner<jbrenner@taosinc.com>");
-MODULE_DESCRIPTION("TAOS tsl2x7x ambient and proximity light sensor driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/iio/light/tsl2x7x.h b/drivers/staging/iio/light/tsl2x7x.h
deleted file mode 100644
index 28b0e7fdc9b8..000000000000
--- a/drivers/staging/iio/light/tsl2x7x.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Device driver for monitoring ambient light intensity (lux)
- * and proximity (prox) within the TAOS TSL2X7X family of devices.
- *
- * Copyright (c) 2012, TAOS Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA	02110-1301, USA.
- */
-
-#ifndef __TSL2X7X_H
-#define __TSL2X7X_H
-
-struct tsl2x7x_lux {
-	unsigned int ratio;
-	unsigned int ch0;
-	unsigned int ch1;
-};
-
-/* Max number of segments allowable in LUX table */
-#define TSL2X7X_MAX_LUX_TABLE_SIZE		9
-/* The default LUX tables all have 3 elements.  */
-#define TSL2X7X_DEF_LUX_TABLE_SZ		3
-#define TSL2X7X_DEFAULT_TABLE_BYTES (sizeof(struct tsl2x7x_lux) * \
-				     TSL2X7X_DEF_LUX_TABLE_SZ)
-
-/**
- * struct tsl2x7x_default_settings - power on defaults unless
- *                                   overridden by platform data.
- *  @als_time:              ALS Integration time - multiple of 50mS
- *  @als_gain:              Index into the ALS gain table.
- *  @als_gain_trim:         default gain trim to account for
- *                          aperture effects.
- *  @wait_time:             Time between PRX and ALS cycles
- *                          in 2.7 periods
- *  @prx_time:              5.2ms prox integration time -
- *                          decrease in 2.7ms periods
- *  @prx_gain:              Proximity gain index
- *  @prox_config:           Prox configuration filters.
- *  @als_cal_target:        Known external ALS reading for
- *                          calibration.
- *  @interrupts_en:         Enable/Disable - 0x00 = none, 0x10 = als,
- *                                           0x20 = prx,  0x30 = bth
- *  @persistence:           H/W Filters, Number of 'out of limits'
- *                          ADC readings PRX/ALS.
- *  @als_thresh_low:        CH0 'low' count to trigger interrupt.
- *  @als_thresh_high:       CH0 'high' count to trigger interrupt.
- *  @prox_thres_low:        Low threshold proximity detection.
- *  @prox_thres_high:       High threshold proximity detection
- *  @prox_pulse_count:      Number if proximity emitter pulses
- *  @prox_max_samples_cal:  Used for prox cal.
- */
-struct tsl2x7x_settings {
-	int als_time;
-	int als_gain;
-	int als_gain_trim;
-	int wait_time;
-	int prx_time;
-	int prox_gain;
-	int prox_config;
-	int als_cal_target;
-	u8  interrupts_en;
-	u8  persistence;
-	int als_thresh_low;
-	int als_thresh_high;
-	int prox_thres_low;
-	int prox_thres_high;
-	int prox_pulse_count;
-	int prox_max_samples_cal;
-	int prox_diode;
-	int prox_power;
-};
-
-/**
- * struct tsl2X7X_platform_data - Platform callback, glass and defaults
- * @platform_power:            Suspend/resume platform callback
- * @power_on:                  Power on callback
- * @power_off:                 Power off callback
- * @platform_lux_table:        Device specific glass coefficents
- * @platform_default_settings: Device specific power on defaults
- *
- */
-struct tsl2X7X_platform_data {
-	struct tsl2x7x_lux platform_lux_table[TSL2X7X_MAX_LUX_TABLE_SIZE];
-	struct tsl2x7x_settings *platform_default_settings;
-};
-
-#endif /* __TSL2X7X_H */
diff --git a/drivers/staging/iio/meter/Kconfig b/drivers/staging/iio/meter/Kconfig
index 64cd3704ec6e..e01eb8abcdce 100644
--- a/drivers/staging/iio/meter/Kconfig
+++ b/drivers/staging/iio/meter/Kconfig
@@ -3,48 +3,6 @@
 #
 menu "Active energy metering IC"
 
-config ADE7753
-	tristate "Analog Devices ADE7753/6 Single-Phase Multifunction Metering IC Driver"
-	depends on SPI
-	help
-	  Say yes here to build support for Analog Devices ADE7753 Single-Phase Multifunction
-	  Metering IC with di/dt Sensor Interface.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ade7753.
-
-config ADE7754
-	tristate "Analog Devices ADE7754 Polyphase Multifunction Energy Metering IC Driver"
-	depends on SPI
-	help
-	  Say yes here to build support for Analog Devices ADE7754 Polyphase
-	  Multifunction Energy Metering IC Driver.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ade7754.
-
-config ADE7758
-	tristate "Analog Devices ADE7758 Poly Phase Multifunction Energy Metering IC Driver"
-	depends on SPI
-	select IIO_TRIGGER if IIO_BUFFER
-	select IIO_KFIFO_BUF if IIO_BUFFER
-	help
-	  Say yes here to build support for Analog Devices ADE7758 Polyphase
-	  Multifunction Energy Metering IC with Per Phase Information Driver.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ade7758.
-
-config ADE7759
-	tristate "Analog Devices ADE7759 Active Energy Metering IC Driver"
-	depends on SPI
-	help
-	  Say yes here to build support for Analog Devices ADE7758 Active Energy
-	  Metering IC with di/dt Sensor Interface.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ade7759.
-
 config ADE7854
 	tristate "Analog Devices ADE7854/58/68/78 Polyphase Multifunction Energy Metering IC Driver"
 	depends on SPI || I2C
diff --git a/drivers/staging/iio/meter/Makefile b/drivers/staging/iio/meter/Makefile
index 19e7982f5563..ed4547e38f3a 100644
--- a/drivers/staging/iio/meter/Makefile
+++ b/drivers/staging/iio/meter/Makefile
@@ -3,14 +3,6 @@
 # Makefile for metering ic drivers
 #
 
-obj-$(CONFIG_ADE7753) += ade7753.o
-obj-$(CONFIG_ADE7754) += ade7754.o
-
-ade7758-y             := ade7758_core.o
-ade7758-$(CONFIG_IIO_BUFFER) += ade7758_ring.o ade7758_trigger.o
-obj-$(CONFIG_ADE7758) += ade7758.o
-
-obj-$(CONFIG_ADE7759) += ade7759.o
 obj-$(CONFIG_ADE7854) += ade7854.o
 obj-$(CONFIG_ADE7854_I2C) += ade7854-i2c.o
 obj-$(CONFIG_ADE7854_SPI) += ade7854-spi.o
diff --git a/drivers/staging/iio/meter/ade7753.c b/drivers/staging/iio/meter/ade7753.c
deleted file mode 100644
index 275e8dfff836..000000000000
--- a/drivers/staging/iio/meter/ade7753.c
+++ /dev/null
@@ -1,630 +0,0 @@
-/*
- * ADE7753 Single-Phase Multifunction Metering IC with di/dt Sensor Interface
- *
- * Copyright 2010 Analog Devices Inc.
- *
- * Licensed under the GPL-2 or later.
- */
-
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/kernel.h>
-#include <linux/list.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/slab.h>
-#include <linux/sysfs.h>
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-#include <linux/spi/spi.h>
-#include "meter.h"
-
-#define ADE7753_WAVEFORM   0x01
-#define ADE7753_AENERGY    0x02
-#define ADE7753_RAENERGY   0x03
-#define ADE7753_LAENERGY   0x04
-#define ADE7753_VAENERGY   0x05
-#define ADE7753_RVAENERGY  0x06
-#define ADE7753_LVAENERGY  0x07
-#define ADE7753_LVARENERGY 0x08
-#define ADE7753_MODE       0x09
-#define ADE7753_IRQEN      0x0A
-#define ADE7753_STATUS     0x0B
-#define ADE7753_RSTSTATUS  0x0C
-#define ADE7753_CH1OS      0x0D
-#define ADE7753_CH2OS      0x0E
-#define ADE7753_GAIN       0x0F
-#define ADE7753_PHCAL      0x10
-#define ADE7753_APOS       0x11
-#define ADE7753_WGAIN      0x12
-#define ADE7753_WDIV       0x13
-#define ADE7753_CFNUM      0x14
-#define ADE7753_CFDEN      0x15
-#define ADE7753_IRMS       0x16
-#define ADE7753_VRMS       0x17
-#define ADE7753_IRMSOS     0x18
-#define ADE7753_VRMSOS     0x19
-#define ADE7753_VAGAIN     0x1A
-#define ADE7753_VADIV      0x1B
-#define ADE7753_LINECYC    0x1C
-#define ADE7753_ZXTOUT     0x1D
-#define ADE7753_SAGCYC     0x1E
-#define ADE7753_SAGLVL     0x1F
-#define ADE7753_IPKLVL     0x20
-#define ADE7753_VPKLVL     0x21
-#define ADE7753_IPEAK      0x22
-#define ADE7753_RSTIPEAK   0x23
-#define ADE7753_VPEAK      0x24
-#define ADE7753_RSTVPEAK   0x25
-#define ADE7753_TEMP       0x26
-#define ADE7753_PERIOD     0x27
-#define ADE7753_TMODE      0x3D
-#define ADE7753_CHKSUM     0x3E
-#define ADE7753_DIEREV     0x3F
-
-#define ADE7753_READ_REG(a)    a
-#define ADE7753_WRITE_REG(a) ((a) | 0x80)
-
-#define ADE7753_MAX_TX    4
-#define ADE7753_MAX_RX    4
-#define ADE7753_STARTUP_DELAY 1000
-
-#define ADE7753_SPI_SLOW    (u32)(300 * 1000)
-#define ADE7753_SPI_BURST   (u32)(1000 * 1000)
-#define ADE7753_SPI_FAST    (u32)(2000 * 1000)
-
-/**
- * struct ade7753_state - device instance specific data
- * @us:         actual spi_device
- * @tx:         transmit buffer
- * @rx:         receive buffer
- * @buf_lock:       mutex to protect tx, rx and write frequency
- **/
-struct ade7753_state {
-	struct spi_device   *us;
-	struct mutex        buf_lock;
-	u8          tx[ADE7753_MAX_TX] ____cacheline_aligned;
-	u8          rx[ADE7753_MAX_RX];
-};
-
-static int ade7753_spi_write_reg_8(struct device *dev,
-				   u8 reg_address,
-				   u8 val)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7753_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7753_WRITE_REG(reg_address);
-	st->tx[1] = val;
-
-	ret = spi_write(st->us, st->tx, 2);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int __ade7753_spi_write_reg_16(struct device *dev, u8 reg_address,
-				      u16 value)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7753_state *st = iio_priv(indio_dev);
-
-	st->tx[0] = ADE7753_WRITE_REG(reg_address);
-	st->tx[1] = (value >> 8) & 0xFF;
-	st->tx[2] = value & 0xFF;
-
-	return spi_write(st->us, st->tx, 3);
-}
-
-static int ade7753_spi_write_reg_16(struct device *dev, u8 reg_address,
-				    u16 value)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7753_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->buf_lock);
-	ret = __ade7753_spi_write_reg_16(dev, reg_address, value);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7753_spi_read_reg_8(struct device *dev,
-				  u8 reg_address,
-				  u8 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7753_state *st = iio_priv(indio_dev);
-	ssize_t ret;
-
-	ret = spi_w8r8(st->us, ADE7753_READ_REG(reg_address));
-	if (ret < 0) {
-		dev_err(&st->us->dev, "problem when reading 8 bit register 0x%02X",
-			reg_address);
-		return ret;
-	}
-	*val = ret;
-
-	return 0;
-}
-
-static int ade7753_spi_read_reg_16(struct device *dev,
-				   u8 reg_address,
-				   u16 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7753_state *st = iio_priv(indio_dev);
-	ssize_t ret;
-
-	ret = spi_w8r16be(st->us, ADE7753_READ_REG(reg_address));
-	if (ret < 0) {
-		dev_err(&st->us->dev, "problem when reading 16 bit register 0x%02X",
-			reg_address);
-		return ret;
-	}
-
-	*val = ret;
-
-	return 0;
-}
-
-static int ade7753_spi_read_reg_24(struct device *dev,
-				   u8 reg_address,
-				   u32 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7753_state *st = iio_priv(indio_dev);
-	int ret;
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.bits_per_word = 8,
-			.len = 1,
-		}, {
-			.rx_buf = st->tx,
-			.bits_per_word = 8,
-			.len = 3,
-		}
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7753_READ_REG(reg_address);
-
-	ret = spi_sync_transfer(st->us, xfers, ARRAY_SIZE(xfers));
-	if (ret) {
-		dev_err(&st->us->dev, "problem when reading 24 bit register 0x%02X",
-			reg_address);
-		goto error_ret;
-	}
-	*val = (st->rx[0] << 16) | (st->rx[1] << 8) | st->rx[2];
-
-error_ret:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static ssize_t ade7753_read_8bit(struct device *dev,
-				 struct device_attribute *attr,
-				 char *buf)
-{
-	int ret;
-	u8 val;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7753_spi_read_reg_8(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7753_read_16bit(struct device *dev,
-				  struct device_attribute *attr,
-				  char *buf)
-{
-	int ret;
-	u16 val;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7753_spi_read_reg_16(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7753_read_24bit(struct device *dev,
-				  struct device_attribute *attr,
-				  char *buf)
-{
-	int ret;
-	u32 val;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7753_spi_read_reg_24(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7753_write_8bit(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf,
-				  size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	int ret;
-	u8 val;
-
-	ret = kstrtou8(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = ade7753_spi_write_reg_8(dev, this_attr->address, val);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static ssize_t ade7753_write_16bit(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf,
-				   size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	int ret;
-	u16 val;
-
-	ret = kstrtou16(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = ade7753_spi_write_reg_16(dev, this_attr->address, val);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static int ade7753_reset(struct device *dev)
-{
-	u16 val;
-	int ret;
-
-	ret = ade7753_spi_read_reg_16(dev, ADE7753_MODE, &val);
-	if (ret)
-		return ret;
-
-	val |= BIT(6); /* Software Chip Reset */
-
-	return ade7753_spi_write_reg_16(dev, ADE7753_MODE, val);
-}
-
-static IIO_DEV_ATTR_AENERGY(ade7753_read_24bit, ADE7753_AENERGY);
-static IIO_DEV_ATTR_LAENERGY(ade7753_read_24bit, ADE7753_LAENERGY);
-static IIO_DEV_ATTR_VAENERGY(ade7753_read_24bit, ADE7753_VAENERGY);
-static IIO_DEV_ATTR_LVAENERGY(ade7753_read_24bit, ADE7753_LVAENERGY);
-static IIO_DEV_ATTR_CFDEN(0644,
-		ade7753_read_16bit,
-		ade7753_write_16bit,
-		ADE7753_CFDEN);
-static IIO_DEV_ATTR_CFNUM(0644,
-		ade7753_read_8bit,
-		ade7753_write_8bit,
-		ADE7753_CFNUM);
-static IIO_DEV_ATTR_CHKSUM(ade7753_read_8bit, ADE7753_CHKSUM);
-static IIO_DEV_ATTR_PHCAL(0644,
-		ade7753_read_16bit,
-		ade7753_write_16bit,
-		ADE7753_PHCAL);
-static IIO_DEV_ATTR_APOS(0644,
-		ade7753_read_16bit,
-		ade7753_write_16bit,
-		ADE7753_APOS);
-static IIO_DEV_ATTR_SAGCYC(0644,
-		ade7753_read_8bit,
-		ade7753_write_8bit,
-		ADE7753_SAGCYC);
-static IIO_DEV_ATTR_SAGLVL(0644,
-		ade7753_read_8bit,
-		ade7753_write_8bit,
-		ADE7753_SAGLVL);
-static IIO_DEV_ATTR_LINECYC(0644,
-		ade7753_read_8bit,
-		ade7753_write_8bit,
-		ADE7753_LINECYC);
-static IIO_DEV_ATTR_WDIV(0644,
-		ade7753_read_8bit,
-		ade7753_write_8bit,
-		ADE7753_WDIV);
-static IIO_DEV_ATTR_IRMS(0644,
-		ade7753_read_24bit,
-		NULL,
-		ADE7753_IRMS);
-static IIO_DEV_ATTR_VRMS(0444,
-		ade7753_read_24bit,
-		NULL,
-		ADE7753_VRMS);
-static IIO_DEV_ATTR_IRMSOS(0644,
-		ade7753_read_16bit,
-		ade7753_write_16bit,
-		ADE7753_IRMSOS);
-static IIO_DEV_ATTR_VRMSOS(0644,
-		ade7753_read_16bit,
-		ade7753_write_16bit,
-		ADE7753_VRMSOS);
-static IIO_DEV_ATTR_WGAIN(0644,
-		ade7753_read_16bit,
-		ade7753_write_16bit,
-		ADE7753_WGAIN);
-static IIO_DEV_ATTR_VAGAIN(0644,
-		ade7753_read_16bit,
-		ade7753_write_16bit,
-		ADE7753_VAGAIN);
-static IIO_DEV_ATTR_PGA_GAIN(0644,
-		ade7753_read_16bit,
-		ade7753_write_16bit,
-		ADE7753_GAIN);
-static IIO_DEV_ATTR_IPKLVL(0644,
-		ade7753_read_8bit,
-		ade7753_write_8bit,
-		ADE7753_IPKLVL);
-static IIO_DEV_ATTR_VPKLVL(0644,
-		ade7753_read_8bit,
-		ade7753_write_8bit,
-		ADE7753_VPKLVL);
-static IIO_DEV_ATTR_IPEAK(0444,
-		ade7753_read_24bit,
-		NULL,
-		ADE7753_IPEAK);
-static IIO_DEV_ATTR_VPEAK(0444,
-		ade7753_read_24bit,
-		NULL,
-		ADE7753_VPEAK);
-static IIO_DEV_ATTR_VPERIOD(0444,
-		ade7753_read_16bit,
-		NULL,
-		ADE7753_PERIOD);
-
-static IIO_DEVICE_ATTR(choff_1, 0644,
-			ade7753_read_8bit,
-			ade7753_write_8bit,
-			ADE7753_CH1OS);
-
-static IIO_DEVICE_ATTR(choff_2, 0644,
-			ade7753_read_8bit,
-			ade7753_write_8bit,
-			ADE7753_CH2OS);
-
-static int ade7753_set_irq(struct device *dev, bool enable)
-{
-	int ret;
-	u8 irqen;
-
-	ret = ade7753_spi_read_reg_8(dev, ADE7753_IRQEN, &irqen);
-	if (ret)
-		goto error_ret;
-
-	if (enable)
-		irqen |= BIT(3); /* Enables an interrupt when a data is
-				  * present in the waveform register
-				  */
-	else
-		irqen &= ~BIT(3);
-
-	ret = ade7753_spi_write_reg_8(dev, ADE7753_IRQEN, irqen);
-
-error_ret:
-	return ret;
-}
-
-/* Power down the device */
-static int ade7753_stop_device(struct device *dev)
-{
-	u16 val;
-	int ret;
-
-	ret = ade7753_spi_read_reg_16(dev, ADE7753_MODE, &val);
-	if (ret)
-		return ret;
-
-	val |= BIT(4);  /* AD converters can be turned off */
-
-	return ade7753_spi_write_reg_16(dev, ADE7753_MODE, val);
-}
-
-static int ade7753_initial_setup(struct iio_dev *indio_dev)
-{
-	int ret;
-	struct device *dev = &indio_dev->dev;
-	struct ade7753_state *st = iio_priv(indio_dev);
-
-	/* use low spi speed for init */
-	st->us->mode = SPI_MODE_3;
-	spi_setup(st->us);
-
-	/* Disable IRQ */
-	ret = ade7753_set_irq(dev, false);
-	if (ret) {
-		dev_err(dev, "disable irq failed");
-		goto err_ret;
-	}
-
-	ade7753_reset(dev);
-	usleep_range(ADE7753_STARTUP_DELAY, ADE7753_STARTUP_DELAY + 100);
-
-err_ret:
-	return ret;
-}
-
-static ssize_t ade7753_read_frequency(struct device *dev,
-				      struct device_attribute *attr,
-				      char *buf)
-{
-	int ret;
-	u16 t;
-	int sps;
-
-	ret = ade7753_spi_read_reg_16(dev, ADE7753_MODE, &t);
-	if (ret)
-		return ret;
-
-	t = (t >> 11) & 0x3;
-	sps = 27900 / (1 + t);
-
-	return sprintf(buf, "%d\n", sps);
-}
-
-static ssize_t ade7753_write_frequency(struct device *dev,
-				       struct device_attribute *attr,
-				       const char *buf,
-				       size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7753_state *st = iio_priv(indio_dev);
-	u16 val;
-	int ret;
-	u16 reg, t;
-
-	ret = kstrtou16(buf, 10, &val);
-	if (ret)
-		return ret;
-	if (!val)
-		return -EINVAL;
-
-	mutex_lock(&st->buf_lock);
-
-	t = 27900 / val;
-	if (t > 0)
-		t--;
-
-	if (t > 1)
-		st->us->max_speed_hz = ADE7753_SPI_SLOW;
-	else
-		st->us->max_speed_hz = ADE7753_SPI_FAST;
-
-	ret = ade7753_spi_read_reg_16(dev, ADE7753_MODE, &reg);
-	if (ret)
-		goto out;
-
-	reg &= ~(3 << 11);
-	reg |= t << 11;
-
-	ret = __ade7753_spi_write_reg_16(dev, ADE7753_MODE, reg);
-
-out:
-	mutex_unlock(&st->buf_lock);
-
-	return ret ? ret : len;
-}
-
-static IIO_DEV_ATTR_TEMP_RAW(ade7753_read_8bit);
-static IIO_CONST_ATTR(in_temp_offset, "-25 C");
-static IIO_CONST_ATTR(in_temp_scale, "0.67 C");
-
-static IIO_DEV_ATTR_SAMP_FREQ(0644,
-		ade7753_read_frequency,
-		ade7753_write_frequency);
-
-static IIO_CONST_ATTR_SAMP_FREQ_AVAIL("27900 14000 7000 3500");
-
-static struct attribute *ade7753_attributes[] = {
-	&iio_dev_attr_in_temp_raw.dev_attr.attr,
-	&iio_const_attr_in_temp_offset.dev_attr.attr,
-	&iio_const_attr_in_temp_scale.dev_attr.attr,
-	&iio_dev_attr_sampling_frequency.dev_attr.attr,
-	&iio_const_attr_sampling_frequency_available.dev_attr.attr,
-	&iio_dev_attr_phcal.dev_attr.attr,
-	&iio_dev_attr_cfden.dev_attr.attr,
-	&iio_dev_attr_aenergy.dev_attr.attr,
-	&iio_dev_attr_laenergy.dev_attr.attr,
-	&iio_dev_attr_vaenergy.dev_attr.attr,
-	&iio_dev_attr_lvaenergy.dev_attr.attr,
-	&iio_dev_attr_cfnum.dev_attr.attr,
-	&iio_dev_attr_apos.dev_attr.attr,
-	&iio_dev_attr_sagcyc.dev_attr.attr,
-	&iio_dev_attr_saglvl.dev_attr.attr,
-	&iio_dev_attr_linecyc.dev_attr.attr,
-	&iio_dev_attr_chksum.dev_attr.attr,
-	&iio_dev_attr_pga_gain.dev_attr.attr,
-	&iio_dev_attr_wgain.dev_attr.attr,
-	&iio_dev_attr_choff_1.dev_attr.attr,
-	&iio_dev_attr_choff_2.dev_attr.attr,
-	&iio_dev_attr_wdiv.dev_attr.attr,
-	&iio_dev_attr_irms.dev_attr.attr,
-	&iio_dev_attr_vrms.dev_attr.attr,
-	&iio_dev_attr_irmsos.dev_attr.attr,
-	&iio_dev_attr_vrmsos.dev_attr.attr,
-	&iio_dev_attr_vagain.dev_attr.attr,
-	&iio_dev_attr_ipklvl.dev_attr.attr,
-	&iio_dev_attr_vpklvl.dev_attr.attr,
-	&iio_dev_attr_ipeak.dev_attr.attr,
-	&iio_dev_attr_vpeak.dev_attr.attr,
-	&iio_dev_attr_vperiod.dev_attr.attr,
-	NULL,
-};
-
-static const struct attribute_group ade7753_attribute_group = {
-	.attrs = ade7753_attributes,
-};
-
-static const struct iio_info ade7753_info = {
-	.attrs = &ade7753_attribute_group,
-};
-
-static int ade7753_probe(struct spi_device *spi)
-{
-	int ret;
-	struct ade7753_state *st;
-	struct iio_dev *indio_dev;
-
-	/* setup the industrialio driver allocated elements */
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev)
-		return -ENOMEM;
-	/* this is only used for removal purposes */
-	spi_set_drvdata(spi, indio_dev);
-
-	st = iio_priv(indio_dev);
-	st->us = spi;
-	mutex_init(&st->buf_lock);
-
-	indio_dev->name = spi->dev.driver->name;
-	indio_dev->dev.parent = &spi->dev;
-	indio_dev->info = &ade7753_info;
-	indio_dev->modes = INDIO_DIRECT_MODE;
-
-	/* Get the device into a sane initial state */
-	ret = ade7753_initial_setup(indio_dev);
-	if (ret)
-		return ret;
-
-	return iio_device_register(indio_dev);
-}
-
-static int ade7753_remove(struct spi_device *spi)
-{
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
-
-	iio_device_unregister(indio_dev);
-	ade7753_stop_device(&indio_dev->dev);
-
-	return 0;
-}
-
-static struct spi_driver ade7753_driver = {
-	.driver = {
-		.name = "ade7753",
-	},
-	.probe = ade7753_probe,
-	.remove = ade7753_remove,
-};
-module_spi_driver(ade7753_driver);
-
-MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices ADE7753/6 Single-Phase Multifunction Meter");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("spi:ade7753");
diff --git a/drivers/staging/iio/meter/ade7754.c b/drivers/staging/iio/meter/ade7754.c
deleted file mode 100644
index 9aa067736715..000000000000
--- a/drivers/staging/iio/meter/ade7754.c
+++ /dev/null
@@ -1,664 +0,0 @@
-/*
- * ADE7754 Polyphase Multifunction Energy Metering IC Driver
- *
- * Copyright 2010 Analog Devices Inc.
- *
- * Licensed under the GPL-2 or later.
- */
-
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/kernel.h>
-#include <linux/list.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/spi/spi.h>
-#include <linux/slab.h>
-#include <linux/sysfs.h>
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-#include "meter.h"
-
-#define ADE7754_AENERGY   0x01
-#define ADE7754_RAENERGY  0x02
-#define ADE7754_LAENERGY  0x03
-#define ADE7754_VAENERGY  0x04
-#define ADE7754_RVAENERGY 0x05
-#define ADE7754_LVAENERGY 0x06
-#define ADE7754_PERIOD    0x07
-#define ADE7754_TEMP      0x08
-#define ADE7754_WFORM     0x09
-#define ADE7754_OPMODE    0x0A
-#define ADE7754_MMODE     0x0B
-#define ADE7754_WAVMODE   0x0C
-#define ADE7754_WATMODE   0x0D
-#define ADE7754_VAMODE    0x0E
-#define ADE7754_IRQEN     0x0F
-#define ADE7754_STATUS    0x10
-#define ADE7754_RSTATUS   0x11
-#define ADE7754_ZXTOUT    0x12
-#define ADE7754_LINCYC    0x13
-#define ADE7754_SAGCYC    0x14
-#define ADE7754_SAGLVL    0x15
-#define ADE7754_VPEAK     0x16
-#define ADE7754_IPEAK     0x17
-#define ADE7754_GAIN      0x18
-#define ADE7754_AWG       0x19
-#define ADE7754_BWG       0x1A
-#define ADE7754_CWG       0x1B
-#define ADE7754_AVAG      0x1C
-#define ADE7754_BVAG      0x1D
-#define ADE7754_CVAG      0x1E
-#define ADE7754_APHCAL    0x1F
-#define ADE7754_BPHCAL    0x20
-#define ADE7754_CPHCAL    0x21
-#define ADE7754_AAPOS     0x22
-#define ADE7754_BAPOS     0x23
-#define ADE7754_CAPOS     0x24
-#define ADE7754_CFNUM     0x25
-#define ADE7754_CFDEN     0x26
-#define ADE7754_WDIV      0x27
-#define ADE7754_VADIV     0x28
-#define ADE7754_AIRMS     0x29
-#define ADE7754_BIRMS     0x2A
-#define ADE7754_CIRMS     0x2B
-#define ADE7754_AVRMS     0x2C
-#define ADE7754_BVRMS     0x2D
-#define ADE7754_CVRMS     0x2E
-#define ADE7754_AIRMSOS   0x2F
-#define ADE7754_BIRMSOS   0x30
-#define ADE7754_CIRMSOS   0x31
-#define ADE7754_AVRMSOS   0x32
-#define ADE7754_BVRMSOS   0x33
-#define ADE7754_CVRMSOS   0x34
-#define ADE7754_AAPGAIN   0x35
-#define ADE7754_BAPGAIN   0x36
-#define ADE7754_CAPGAIN   0x37
-#define ADE7754_AVGAIN    0x38
-#define ADE7754_BVGAIN    0x39
-#define ADE7754_CVGAIN    0x3A
-#define ADE7754_CHKSUM    0x3E
-#define ADE7754_VERSION   0x3F
-
-#define ADE7754_READ_REG(a)    a
-#define ADE7754_WRITE_REG(a) ((a) | 0x80)
-
-#define ADE7754_MAX_TX    4
-#define ADE7754_MAX_RX    4
-#define ADE7754_STARTUP_DELAY 1000
-
-#define ADE7754_SPI_SLOW	(u32)(300 * 1000)
-#define ADE7754_SPI_BURST	(u32)(1000 * 1000)
-#define ADE7754_SPI_FAST	(u32)(2000 * 1000)
-
-/**
- * struct ade7754_state - device instance specific data
- * @us:			actual spi_device
- * @buf_lock:		mutex to protect tx, rx and write frequency
- * @tx:			transmit buffer
- * @rx:			receive buffer
- **/
-struct ade7754_state {
-	struct spi_device	*us;
-	struct mutex		buf_lock;
-	u8			tx[ADE7754_MAX_TX] ____cacheline_aligned;
-	u8			rx[ADE7754_MAX_RX];
-};
-
-/* Unlocked version of ade7754_spi_write_reg_8 function */
-static int __ade7754_spi_write_reg_8(struct device *dev, u8 reg_address, u8 val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7754_state *st = iio_priv(indio_dev);
-
-	st->tx[0] = ADE7754_WRITE_REG(reg_address);
-	st->tx[1] = val;
-	return spi_write(st->us, st->tx, 2);
-}
-
-static int ade7754_spi_write_reg_8(struct device *dev, u8 reg_address, u8 val)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7754_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->buf_lock);
-	ret = __ade7754_spi_write_reg_8(dev, reg_address, val);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7754_spi_write_reg_16(struct device *dev,
-				    u8 reg_address, u16 val)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7754_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7754_WRITE_REG(reg_address);
-	st->tx[1] = (val >> 8) & 0xFF;
-	st->tx[2] = val & 0xFF;
-	ret = spi_write(st->us, st->tx, 3);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7754_spi_read_reg_8(struct device *dev, u8 reg_address, u8 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7754_state *st = iio_priv(indio_dev);
-	int ret;
-
-	ret = spi_w8r8(st->us, ADE7754_READ_REG(reg_address));
-	if (ret < 0) {
-		dev_err(&st->us->dev, "problem when reading 8 bit register 0x%02X",
-			reg_address);
-		return ret;
-	}
-	*val = ret;
-
-	return 0;
-}
-
-static int ade7754_spi_read_reg_16(struct device *dev,
-				   u8 reg_address, u16 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7754_state *st = iio_priv(indio_dev);
-	int ret;
-
-	ret = spi_w8r16be(st->us, ADE7754_READ_REG(reg_address));
-	if (ret < 0) {
-		dev_err(&st->us->dev, "problem when reading 16 bit register 0x%02X",
-			reg_address);
-		return ret;
-	}
-
-	*val = ret;
-
-	return 0;
-}
-
-static int ade7754_spi_read_reg_24(struct device *dev,
-				   u8 reg_address, u32 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7754_state *st = iio_priv(indio_dev);
-	int ret;
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.rx_buf = st->rx,
-			.bits_per_word = 8,
-			.len = 4,
-		},
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7754_READ_REG(reg_address);
-	st->tx[1] = 0;
-	st->tx[2] = 0;
-	st->tx[3] = 0;
-
-	ret = spi_sync_transfer(st->us, xfers, ARRAY_SIZE(xfers));
-	if (ret) {
-		dev_err(&st->us->dev, "problem when reading 24 bit register 0x%02X",
-			reg_address);
-		goto error_ret;
-	}
-	*val = (st->rx[1] << 16) | (st->rx[2] << 8) | st->rx[3];
-
-error_ret:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static ssize_t ade7754_read_8bit(struct device *dev,
-				 struct device_attribute *attr,
-				 char *buf)
-{
-	int ret;
-	u8 val = 0;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7754_spi_read_reg_8(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7754_read_16bit(struct device *dev,
-				  struct device_attribute *attr,
-				  char *buf)
-{
-	int ret;
-	u16 val = 0;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7754_spi_read_reg_16(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7754_read_24bit(struct device *dev,
-				  struct device_attribute *attr,
-				  char *buf)
-{
-	int ret;
-	u32 val = 0;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7754_spi_read_reg_24(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%u\n", val & 0xFFFFFF);
-}
-
-static ssize_t ade7754_write_8bit(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf,
-				  size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	int ret;
-	u8 val;
-
-	ret = kstrtou8(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = ade7754_spi_write_reg_8(dev, this_attr->address, val);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static ssize_t ade7754_write_16bit(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf,
-				   size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	int ret;
-	u16 val;
-
-	ret = kstrtou16(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = ade7754_spi_write_reg_16(dev, this_attr->address, val);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static int ade7754_reset(struct device *dev)
-{
-	int ret;
-	u8 val;
-
-	ret = ade7754_spi_read_reg_8(dev, ADE7754_OPMODE, &val);
-	if (ret < 0)
-		return ret;
-
-	val |= BIT(6); /* Software Chip Reset */
-	return ade7754_spi_write_reg_8(dev, ADE7754_OPMODE, val);
-}
-
-static IIO_DEV_ATTR_AENERGY(ade7754_read_24bit, ADE7754_AENERGY);
-static IIO_DEV_ATTR_LAENERGY(ade7754_read_24bit, ADE7754_LAENERGY);
-static IIO_DEV_ATTR_VAENERGY(ade7754_read_24bit, ADE7754_VAENERGY);
-static IIO_DEV_ATTR_LVAENERGY(ade7754_read_24bit, ADE7754_LVAENERGY);
-static IIO_DEV_ATTR_VPEAK(0644,
-		ade7754_read_8bit,
-		ade7754_write_8bit,
-		ADE7754_VPEAK);
-static IIO_DEV_ATTR_IPEAK(0644,
-		ade7754_read_8bit,
-		ade7754_write_8bit,
-		ADE7754_VPEAK);
-static IIO_DEV_ATTR_APHCAL(0644,
-		ade7754_read_8bit,
-		ade7754_write_8bit,
-		ADE7754_APHCAL);
-static IIO_DEV_ATTR_BPHCAL(0644,
-		ade7754_read_8bit,
-		ade7754_write_8bit,
-		ADE7754_BPHCAL);
-static IIO_DEV_ATTR_CPHCAL(0644,
-		ade7754_read_8bit,
-		ade7754_write_8bit,
-		ADE7754_CPHCAL);
-static IIO_DEV_ATTR_AAPOS(0644,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_AAPOS);
-static IIO_DEV_ATTR_BAPOS(0644,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_BAPOS);
-static IIO_DEV_ATTR_CAPOS(0644,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_CAPOS);
-static IIO_DEV_ATTR_WDIV(0644,
-		ade7754_read_8bit,
-		ade7754_write_8bit,
-		ADE7754_WDIV);
-static IIO_DEV_ATTR_VADIV(0644,
-		ade7754_read_8bit,
-		ade7754_write_8bit,
-		ADE7754_VADIV);
-static IIO_DEV_ATTR_CFNUM(0644,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_CFNUM);
-static IIO_DEV_ATTR_CFDEN(0644,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_CFDEN);
-static IIO_DEV_ATTR_ACTIVE_POWER_A_GAIN(0644,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_AAPGAIN);
-static IIO_DEV_ATTR_ACTIVE_POWER_B_GAIN(0644,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_BAPGAIN);
-static IIO_DEV_ATTR_ACTIVE_POWER_C_GAIN(0644,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_CAPGAIN);
-static IIO_DEV_ATTR_AIRMS(0444,
-		ade7754_read_24bit,
-		NULL,
-		ADE7754_AIRMS);
-static IIO_DEV_ATTR_BIRMS(0444,
-		ade7754_read_24bit,
-		NULL,
-		ADE7754_BIRMS);
-static IIO_DEV_ATTR_CIRMS(0444,
-		ade7754_read_24bit,
-		NULL,
-		ADE7754_CIRMS);
-static IIO_DEV_ATTR_AVRMS(0444,
-		ade7754_read_24bit,
-		NULL,
-		ADE7754_AVRMS);
-static IIO_DEV_ATTR_BVRMS(0444,
-		ade7754_read_24bit,
-		NULL,
-		ADE7754_BVRMS);
-static IIO_DEV_ATTR_CVRMS(0444,
-		ade7754_read_24bit,
-		NULL,
-		ADE7754_CVRMS);
-static IIO_DEV_ATTR_AIRMSOS(0444,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_AIRMSOS);
-static IIO_DEV_ATTR_BIRMSOS(0444,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_BIRMSOS);
-static IIO_DEV_ATTR_CIRMSOS(0444,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_CIRMSOS);
-static IIO_DEV_ATTR_AVRMSOS(0444,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_AVRMSOS);
-static IIO_DEV_ATTR_BVRMSOS(0444,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_BVRMSOS);
-static IIO_DEV_ATTR_CVRMSOS(0444,
-		ade7754_read_16bit,
-		ade7754_write_16bit,
-		ADE7754_CVRMSOS);
-
-static int ade7754_set_irq(struct device *dev, bool enable)
-{
-	int ret;
-	u16 irqen;
-
-	ret = ade7754_spi_read_reg_16(dev, ADE7754_IRQEN, &irqen);
-	if (ret)
-		return ret;
-
-	if (enable)
-		irqen |= BIT(14); /* Enables an interrupt when a data is
-				   * present in the waveform register
-				   */
-	else
-		irqen &= ~BIT(14);
-
-	return ade7754_spi_write_reg_16(dev, ADE7754_IRQEN, irqen);
-}
-
-/* Power down the device */
-static int ade7754_stop_device(struct device *dev)
-{
-	int ret;
-	u8 val;
-
-	ret = ade7754_spi_read_reg_8(dev, ADE7754_OPMODE, &val);
-	if (ret < 0) {
-		dev_err(dev, "unable to power down the device, error: %d",
-			ret);
-		return ret;
-	}
-
-	val |= 7 << 3;  /* ADE7754 powered down */
-	return ade7754_spi_write_reg_8(dev, ADE7754_OPMODE, val);
-}
-
-static int ade7754_initial_setup(struct iio_dev *indio_dev)
-{
-	int ret;
-	struct ade7754_state *st = iio_priv(indio_dev);
-	struct device *dev = &indio_dev->dev;
-
-	/* use low spi speed for init */
-	st->us->mode = SPI_MODE_3;
-	spi_setup(st->us);
-
-	/* Disable IRQ */
-	ret = ade7754_set_irq(dev, false);
-	if (ret) {
-		dev_err(dev, "disable irq failed");
-		goto err_ret;
-	}
-
-	ade7754_reset(dev);
-	usleep_range(ADE7754_STARTUP_DELAY, ADE7754_STARTUP_DELAY + 100);
-
-err_ret:
-	return ret;
-}
-
-static ssize_t ade7754_read_frequency(struct device *dev,
-				      struct device_attribute *attr,
-				      char *buf)
-{
-	int ret;
-	u8 t;
-	int sps;
-
-	ret = ade7754_spi_read_reg_8(dev, ADE7754_WAVMODE, &t);
-	if (ret)
-		return ret;
-
-	t = (t >> 3) & 0x3;
-	sps = 26000 / (1 + t);
-
-	return sprintf(buf, "%d\n", sps);
-}
-
-static ssize_t ade7754_write_frequency(struct device *dev,
-				       struct device_attribute *attr,
-				       const char *buf,
-				       size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7754_state *st = iio_priv(indio_dev);
-	u16 val;
-	int ret;
-	u8 reg, t;
-
-	ret = kstrtou16(buf, 10, &val);
-	if (ret)
-		return ret;
-	if (!val)
-		return -EINVAL;
-
-	mutex_lock(&st->buf_lock);
-
-	t = 26000 / val;
-	if (t > 0)
-		t--;
-
-	if (t > 1)
-		st->us->max_speed_hz = ADE7754_SPI_SLOW;
-	else
-		st->us->max_speed_hz = ADE7754_SPI_FAST;
-
-	ret = ade7754_spi_read_reg_8(dev, ADE7754_WAVMODE, &reg);
-	if (ret)
-		goto out;
-
-	reg &= ~(3 << 3);
-	reg |= t << 3;
-
-	ret = __ade7754_spi_write_reg_8(dev, ADE7754_WAVMODE, reg);
-
-out:
-	mutex_unlock(&st->buf_lock);
-
-	return ret ? ret : len;
-}
-static IIO_DEV_ATTR_TEMP_RAW(ade7754_read_8bit);
-static IIO_CONST_ATTR(in_temp_offset, "129 C");
-static IIO_CONST_ATTR(in_temp_scale, "4 C");
-
-static IIO_DEV_ATTR_SAMP_FREQ(0644,
-		ade7754_read_frequency,
-		ade7754_write_frequency);
-
-static IIO_CONST_ATTR_SAMP_FREQ_AVAIL("26000 13000 65000 33000");
-
-static struct attribute *ade7754_attributes[] = {
-	&iio_dev_attr_in_temp_raw.dev_attr.attr,
-	&iio_const_attr_in_temp_offset.dev_attr.attr,
-	&iio_const_attr_in_temp_scale.dev_attr.attr,
-	&iio_dev_attr_sampling_frequency.dev_attr.attr,
-	&iio_const_attr_sampling_frequency_available.dev_attr.attr,
-	&iio_dev_attr_aenergy.dev_attr.attr,
-	&iio_dev_attr_laenergy.dev_attr.attr,
-	&iio_dev_attr_vaenergy.dev_attr.attr,
-	&iio_dev_attr_lvaenergy.dev_attr.attr,
-	&iio_dev_attr_vpeak.dev_attr.attr,
-	&iio_dev_attr_ipeak.dev_attr.attr,
-	&iio_dev_attr_aphcal.dev_attr.attr,
-	&iio_dev_attr_bphcal.dev_attr.attr,
-	&iio_dev_attr_cphcal.dev_attr.attr,
-	&iio_dev_attr_aapos.dev_attr.attr,
-	&iio_dev_attr_bapos.dev_attr.attr,
-	&iio_dev_attr_capos.dev_attr.attr,
-	&iio_dev_attr_wdiv.dev_attr.attr,
-	&iio_dev_attr_vadiv.dev_attr.attr,
-	&iio_dev_attr_cfnum.dev_attr.attr,
-	&iio_dev_attr_cfden.dev_attr.attr,
-	&iio_dev_attr_active_power_a_gain.dev_attr.attr,
-	&iio_dev_attr_active_power_b_gain.dev_attr.attr,
-	&iio_dev_attr_active_power_c_gain.dev_attr.attr,
-	&iio_dev_attr_airms.dev_attr.attr,
-	&iio_dev_attr_birms.dev_attr.attr,
-	&iio_dev_attr_cirms.dev_attr.attr,
-	&iio_dev_attr_avrms.dev_attr.attr,
-	&iio_dev_attr_bvrms.dev_attr.attr,
-	&iio_dev_attr_cvrms.dev_attr.attr,
-	&iio_dev_attr_airmsos.dev_attr.attr,
-	&iio_dev_attr_birmsos.dev_attr.attr,
-	&iio_dev_attr_cirmsos.dev_attr.attr,
-	&iio_dev_attr_avrmsos.dev_attr.attr,
-	&iio_dev_attr_bvrmsos.dev_attr.attr,
-	&iio_dev_attr_cvrmsos.dev_attr.attr,
-	NULL,
-};
-
-static const struct attribute_group ade7754_attribute_group = {
-	.attrs = ade7754_attributes,
-};
-
-static const struct iio_info ade7754_info = {
-	.attrs = &ade7754_attribute_group,
-};
-
-static int ade7754_probe(struct spi_device *spi)
-{
-	int ret;
-	struct ade7754_state *st;
-	struct iio_dev *indio_dev;
-
-	/* setup the industrialio driver allocated elements */
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev)
-		return -ENOMEM;
-	/* this is only used for removal purposes */
-	spi_set_drvdata(spi, indio_dev);
-
-	st = iio_priv(indio_dev);
-	st->us = spi;
-	mutex_init(&st->buf_lock);
-
-	indio_dev->name = spi->dev.driver->name;
-	indio_dev->dev.parent = &spi->dev;
-	indio_dev->info = &ade7754_info;
-	indio_dev->modes = INDIO_DIRECT_MODE;
-
-	/* Get the device into a sane initial state */
-	ret = ade7754_initial_setup(indio_dev);
-	if (ret)
-		goto powerdown_on_error;
-	ret = iio_device_register(indio_dev);
-	if (ret)
-		goto powerdown_on_error;
-	return ret;
-
-powerdown_on_error:
-	ade7754_stop_device(&indio_dev->dev);
-	return ret;
-}
-
-static int ade7754_remove(struct spi_device *spi)
-{
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
-
-	iio_device_unregister(indio_dev);
-	ade7754_stop_device(&indio_dev->dev);
-
-	return 0;
-}
-
-static struct spi_driver ade7754_driver = {
-	.driver = {
-		.name = "ade7754",
-	},
-	.probe = ade7754_probe,
-	.remove = ade7754_remove,
-};
-module_spi_driver(ade7754_driver);
-
-MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices ADE7754 Polyphase Multifunction Energy Metering IC Driver");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("spi:ad7754");
diff --git a/drivers/staging/iio/meter/ade7758.h b/drivers/staging/iio/meter/ade7758.h
deleted file mode 100644
index 2de81b53e786..000000000000
--- a/drivers/staging/iio/meter/ade7758.h
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * ADE7758 Poly Phase Multifunction Energy Metering IC driver
- *
- * Copyright 2010-2011 Analog Devices Inc.
- *
- * Licensed under the GPL-2.
- */
-
-#ifndef _ADE7758_H
-#define _ADE7758_H
-
-#define ADE7758_AWATTHR   0x01
-#define ADE7758_BWATTHR   0x02
-#define ADE7758_CWATTHR   0x03
-#define ADE7758_AVARHR    0x04
-#define ADE7758_BVARHR    0x05
-#define ADE7758_CVARHR    0x06
-#define ADE7758_AVAHR     0x07
-#define ADE7758_BVAHR     0x08
-#define ADE7758_CVAHR     0x09
-#define ADE7758_AIRMS     0x0A
-#define ADE7758_BIRMS     0x0B
-#define ADE7758_CIRMS     0x0C
-#define ADE7758_AVRMS     0x0D
-#define ADE7758_BVRMS     0x0E
-#define ADE7758_CVRMS     0x0F
-#define ADE7758_FREQ      0x10
-#define ADE7758_TEMP      0x11
-#define ADE7758_WFORM     0x12
-#define ADE7758_OPMODE    0x13
-#define ADE7758_MMODE     0x14
-#define ADE7758_WAVMODE   0x15
-#define ADE7758_COMPMODE  0x16
-#define ADE7758_LCYCMODE  0x17
-#define ADE7758_MASK      0x18
-#define ADE7758_STATUS    0x19
-#define ADE7758_RSTATUS   0x1A
-#define ADE7758_ZXTOUT    0x1B
-#define ADE7758_LINECYC   0x1C
-#define ADE7758_SAGCYC    0x1D
-#define ADE7758_SAGLVL    0x1E
-#define ADE7758_VPINTLVL  0x1F
-#define ADE7758_IPINTLVL  0x20
-#define ADE7758_VPEAK     0x21
-#define ADE7758_IPEAK     0x22
-#define ADE7758_GAIN      0x23
-#define ADE7758_AVRMSGAIN 0x24
-#define ADE7758_BVRMSGAIN 0x25
-#define ADE7758_CVRMSGAIN 0x26
-#define ADE7758_AIGAIN    0x27
-#define ADE7758_BIGAIN    0x28
-#define ADE7758_CIGAIN    0x29
-#define ADE7758_AWG       0x2A
-#define ADE7758_BWG       0x2B
-#define ADE7758_CWG       0x2C
-#define ADE7758_AVARG     0x2D
-#define ADE7758_BVARG     0x2E
-#define ADE7758_CVARG     0x2F
-#define ADE7758_AVAG      0x30
-#define ADE7758_BVAG      0x31
-#define ADE7758_CVAG      0x32
-#define ADE7758_AVRMSOS   0x33
-#define ADE7758_BVRMSOS   0x34
-#define ADE7758_CVRMSOS   0x35
-#define ADE7758_AIRMSOS   0x36
-#define ADE7758_BIRMSOS   0x37
-#define ADE7758_CIRMSOS   0x38
-#define ADE7758_AWAITOS   0x39
-#define ADE7758_BWAITOS   0x3A
-#define ADE7758_CWAITOS   0x3B
-#define ADE7758_AVAROS    0x3C
-#define ADE7758_BVAROS    0x3D
-#define ADE7758_CVAROS    0x3E
-#define ADE7758_APHCAL    0x3F
-#define ADE7758_BPHCAL    0x40
-#define ADE7758_CPHCAL    0x41
-#define ADE7758_WDIV      0x42
-#define ADE7758_VADIV     0x44
-#define ADE7758_VARDIV    0x43
-#define ADE7758_APCFNUM   0x45
-#define ADE7758_APCFDEN   0x46
-#define ADE7758_VARCFNUM  0x47
-#define ADE7758_VARCFDEN  0x48
-#define ADE7758_CHKSUM    0x7E
-#define ADE7758_VERSION   0x7F
-
-#define ADE7758_READ_REG(a)    a
-#define ADE7758_WRITE_REG(a) ((a) | 0x80)
-
-#define ADE7758_MAX_TX    8
-#define ADE7758_MAX_RX    4
-#define ADE7758_STARTUP_DELAY 1000
-
-#define AD7758_NUM_WAVSEL	5
-#define AD7758_NUM_PHSEL	3
-#define AD7758_NUM_WAVESRC	(AD7758_NUM_WAVSEL * AD7758_NUM_PHSEL)
-
-#define AD7758_PHASE_A		0
-#define AD7758_PHASE_B		1
-#define AD7758_PHASE_C		2
-#define AD7758_CURRENT		0
-#define AD7758_VOLTAGE		1
-#define AD7758_ACT_PWR		2
-#define AD7758_REACT_PWR	3
-#define AD7758_APP_PWR		4
-#define AD7758_WT(p, w)		(((w) << 2) | (p))
-
-/**
- * struct ade7758_state - device instance specific data
- * @us:			actual spi_device
- * @trig:		data ready trigger registered with iio
- * @tx:			transmit buffer
- * @rx:			receive buffer
- * @buf_lock:		mutex to protect tx, rx, read and write frequency
- **/
-struct ade7758_state {
-	struct spi_device	*us;
-	struct iio_trigger	*trig;
-	u8			*tx;
-	u8			*rx;
-	struct mutex		buf_lock;
-	struct spi_transfer	ring_xfer[4];
-	struct spi_message	ring_msg;
-	/*
-	 * DMA (thus cache coherency maintenance) requires the
-	 * transfer buffers to live in their own cache lines.
-	 */
-	unsigned char		rx_buf[8] ____cacheline_aligned;
-	unsigned char		tx_buf[8];
-
-};
-
-#ifdef CONFIG_IIO_BUFFER
-/* At the moment triggers are only used for ring buffer
- * filling. This may change!
- */
-
-void ade7758_remove_trigger(struct iio_dev *indio_dev);
-int ade7758_probe_trigger(struct iio_dev *indio_dev);
-
-ssize_t ade7758_read_data_from_ring(struct device *dev,
-				    struct device_attribute *attr, char *buf);
-
-int ade7758_configure_ring(struct iio_dev *indio_dev);
-void ade7758_unconfigure_ring(struct iio_dev *indio_dev);
-
-int ade7758_set_irq(struct device *dev, bool enable);
-
-int ade7758_spi_write_reg_8(struct device *dev, u8 reg_address, u8 val);
-int ade7758_spi_read_reg_8(struct device *dev, u8 reg_address, u8 *val);
-
-#else /* CONFIG_IIO_BUFFER */
-
-static inline void ade7758_remove_trigger(struct iio_dev *indio_dev)
-{
-}
-
-static inline int ade7758_probe_trigger(struct iio_dev *indio_dev)
-{
-	return 0;
-}
-
-static int ade7758_configure_ring(struct iio_dev *indio_dev)
-{
-	return 0;
-}
-
-static inline void ade7758_unconfigure_ring(struct iio_dev *indio_dev)
-{
-}
-
-static inline int ade7758_initialize_ring(struct iio_ring_buffer *ring)
-{
-	return 0;
-}
-
-static inline void ade7758_uninitialize_ring(struct iio_dev *indio_dev)
-{
-}
-
-#endif /* CONFIG_IIO_BUFFER */
-
-#endif
diff --git a/drivers/staging/iio/meter/ade7758_core.c b/drivers/staging/iio/meter/ade7758_core.c
deleted file mode 100644
index 4e0dbf5c5705..000000000000
--- a/drivers/staging/iio/meter/ade7758_core.c
+++ /dev/null
@@ -1,955 +0,0 @@
-/*
- * ADE7758 Poly Phase Multifunction Energy Metering IC driver
- *
- * Copyright 2010-2011 Analog Devices Inc.
- *
- * Licensed under the GPL-2.
- */
-
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/delay.h>
-#include <linux/mutex.h>
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/spi/spi.h>
-#include <linux/slab.h>
-#include <linux/sysfs.h>
-#include <linux/list.h>
-#include <linux/module.h>
-
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-#include <linux/iio/buffer.h>
-#include "meter.h"
-#include "ade7758.h"
-
-static int __ade7758_spi_write_reg_8(struct device *dev, u8 reg_address, u8 val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7758_state *st = iio_priv(indio_dev);
-
-	st->tx[0] = ADE7758_WRITE_REG(reg_address);
-	st->tx[1] = val;
-
-	return spi_write(st->us, st->tx, 2);
-}
-
-int ade7758_spi_write_reg_8(struct device *dev, u8 reg_address, u8 val)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7758_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->buf_lock);
-	ret = __ade7758_spi_write_reg_8(dev, reg_address, val);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7758_spi_write_reg_16(struct device *dev, u8 reg_address,
-				    u16 value)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7758_state *st = iio_priv(indio_dev);
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.bits_per_word = 8,
-			.len = 3,
-		}
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7758_WRITE_REG(reg_address);
-	st->tx[1] = (value >> 8) & 0xFF;
-	st->tx[2] = value & 0xFF;
-
-	ret = spi_sync_transfer(st->us, xfers, ARRAY_SIZE(xfers));
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7758_spi_write_reg_24(struct device *dev, u8 reg_address,
-				    u32 value)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7758_state *st = iio_priv(indio_dev);
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.bits_per_word = 8,
-			.len = 4,
-		}
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7758_WRITE_REG(reg_address);
-	st->tx[1] = (value >> 16) & 0xFF;
-	st->tx[2] = (value >> 8) & 0xFF;
-	st->tx[3] = value & 0xFF;
-
-	ret = spi_sync_transfer(st->us, xfers, ARRAY_SIZE(xfers));
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int __ade7758_spi_read_reg_8(struct device *dev, u8 reg_address, u8 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7758_state *st = iio_priv(indio_dev);
-	int ret;
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.bits_per_word = 8,
-			.len = 1,
-			.delay_usecs = 4,
-		},
-		{
-			.tx_buf = &st->tx[1],
-			.rx_buf = st->rx,
-			.bits_per_word = 8,
-			.len = 1,
-		},
-	};
-
-	st->tx[0] = ADE7758_READ_REG(reg_address);
-	st->tx[1] = 0;
-
-	ret = spi_sync_transfer(st->us, xfers, ARRAY_SIZE(xfers));
-	if (ret) {
-		dev_err(&st->us->dev, "problem when reading 8 bit register 0x%02X",
-			reg_address);
-		goto error_ret;
-	}
-	*val = st->rx[0];
-
-error_ret:
-	return ret;
-}
-
-int ade7758_spi_read_reg_8(struct device *dev, u8 reg_address, u8 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7758_state *st = iio_priv(indio_dev);
-	int ret;
-
-	mutex_lock(&st->buf_lock);
-	ret = __ade7758_spi_read_reg_8(dev, reg_address, val);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7758_spi_read_reg_16(struct device *dev, u8 reg_address,
-				   u16 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7758_state *st = iio_priv(indio_dev);
-	int ret;
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.bits_per_word = 8,
-			.len = 1,
-			.delay_usecs = 4,
-		},
-		{
-			.tx_buf = &st->tx[1],
-			.rx_buf = st->rx,
-			.bits_per_word = 8,
-			.len = 2,
-		},
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7758_READ_REG(reg_address);
-	st->tx[1] = 0;
-	st->tx[2] = 0;
-
-	ret = spi_sync_transfer(st->us, xfers, ARRAY_SIZE(xfers));
-	if (ret) {
-		dev_err(&st->us->dev, "problem when reading 16 bit register 0x%02X",
-			reg_address);
-		goto error_ret;
-	}
-
-	*val = (st->rx[0] << 8) | st->rx[1];
-
-error_ret:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static int ade7758_spi_read_reg_24(struct device *dev, u8 reg_address,
-				   u32 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7758_state *st = iio_priv(indio_dev);
-	int ret;
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.bits_per_word = 8,
-			.len = 1,
-			.delay_usecs = 4,
-		},
-		{
-			.tx_buf = &st->tx[1],
-			.rx_buf = st->rx,
-			.bits_per_word = 8,
-			.len = 3,
-		},
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7758_READ_REG(reg_address);
-	st->tx[1] = 0;
-	st->tx[2] = 0;
-	st->tx[3] = 0;
-
-	ret = spi_sync_transfer(st->us, xfers, ARRAY_SIZE(xfers));
-	if (ret) {
-		dev_err(&st->us->dev, "problem when reading 24 bit register 0x%02X",
-			reg_address);
-		goto error_ret;
-	}
-	*val = (st->rx[0] << 16) | (st->rx[1] << 8) | st->rx[2];
-
-error_ret:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static ssize_t ade7758_read_8bit(struct device *dev,
-				 struct device_attribute *attr, char *buf)
-{
-	int ret;
-	u8 val = 0;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7758_spi_read_reg_8(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7758_read_16bit(struct device *dev,
-				  struct device_attribute *attr, char *buf)
-{
-	int ret;
-	u16 val = 0;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7758_spi_read_reg_16(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7758_read_24bit(struct device *dev,
-				  struct device_attribute *attr, char *buf)
-{
-	int ret;
-	u32 val = 0;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7758_spi_read_reg_24(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%u\n", val & 0xFFFFFF);
-}
-
-static ssize_t ade7758_write_8bit(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf, size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	int ret;
-	u8 val;
-
-	ret = kstrtou8(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = ade7758_spi_write_reg_8(dev, this_attr->address, val);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static ssize_t ade7758_write_16bit(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf, size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	int ret;
-	u16 val;
-
-	ret = kstrtou16(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = ade7758_spi_write_reg_16(dev, this_attr->address, val);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static int ade7758_reset(struct device *dev)
-{
-	int ret;
-	u8 val;
-
-	ret = ade7758_spi_read_reg_8(dev, ADE7758_OPMODE, &val);
-	if (ret < 0) {
-		dev_err(dev, "Failed to read opmode reg\n");
-		return ret;
-	}
-	val |= BIT(6); /* Software Chip Reset */
-	ret = ade7758_spi_write_reg_8(dev, ADE7758_OPMODE, val);
-	if (ret < 0)
-		dev_err(dev, "Failed to write opmode reg\n");
-	return ret;
-}
-
-static IIO_DEV_ATTR_VPEAK(0644,
-		ade7758_read_8bit,
-		ade7758_write_8bit,
-		ADE7758_VPEAK);
-static IIO_DEV_ATTR_IPEAK(0644,
-		ade7758_read_8bit,
-		ade7758_write_8bit,
-		ADE7758_VPEAK);
-static IIO_DEV_ATTR_APHCAL(0644,
-		ade7758_read_8bit,
-		ade7758_write_8bit,
-		ADE7758_APHCAL);
-static IIO_DEV_ATTR_BPHCAL(0644,
-		ade7758_read_8bit,
-		ade7758_write_8bit,
-		ADE7758_BPHCAL);
-static IIO_DEV_ATTR_CPHCAL(0644,
-		ade7758_read_8bit,
-		ade7758_write_8bit,
-		ADE7758_CPHCAL);
-static IIO_DEV_ATTR_WDIV(0644,
-		ade7758_read_8bit,
-		ade7758_write_8bit,
-		ADE7758_WDIV);
-static IIO_DEV_ATTR_VADIV(0644,
-		ade7758_read_8bit,
-		ade7758_write_8bit,
-		ADE7758_VADIV);
-static IIO_DEV_ATTR_AIRMS(0444,
-		ade7758_read_24bit,
-		NULL,
-		ADE7758_AIRMS);
-static IIO_DEV_ATTR_BIRMS(0444,
-		ade7758_read_24bit,
-		NULL,
-		ADE7758_BIRMS);
-static IIO_DEV_ATTR_CIRMS(0444,
-		ade7758_read_24bit,
-		NULL,
-		ADE7758_CIRMS);
-static IIO_DEV_ATTR_AVRMS(0444,
-		ade7758_read_24bit,
-		NULL,
-		ADE7758_AVRMS);
-static IIO_DEV_ATTR_BVRMS(0444,
-		ade7758_read_24bit,
-		NULL,
-		ADE7758_BVRMS);
-static IIO_DEV_ATTR_CVRMS(0444,
-		ade7758_read_24bit,
-		NULL,
-		ADE7758_CVRMS);
-static IIO_DEV_ATTR_AIRMSOS(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_AIRMSOS);
-static IIO_DEV_ATTR_BIRMSOS(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_BIRMSOS);
-static IIO_DEV_ATTR_CIRMSOS(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_CIRMSOS);
-static IIO_DEV_ATTR_AVRMSOS(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_AVRMSOS);
-static IIO_DEV_ATTR_BVRMSOS(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_BVRMSOS);
-static IIO_DEV_ATTR_CVRMSOS(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_CVRMSOS);
-static IIO_DEV_ATTR_AIGAIN(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_AIGAIN);
-static IIO_DEV_ATTR_BIGAIN(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_BIGAIN);
-static IIO_DEV_ATTR_CIGAIN(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_CIGAIN);
-static IIO_DEV_ATTR_AVRMSGAIN(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_AVRMSGAIN);
-static IIO_DEV_ATTR_BVRMSGAIN(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_BVRMSGAIN);
-static IIO_DEV_ATTR_CVRMSGAIN(0644,
-		ade7758_read_16bit,
-		ade7758_write_16bit,
-		ADE7758_CVRMSGAIN);
-
-int ade7758_set_irq(struct device *dev, bool enable)
-{
-	int ret;
-	u32 irqen;
-
-	ret = ade7758_spi_read_reg_24(dev, ADE7758_MASK, &irqen);
-	if (ret)
-		return ret;
-
-	if (enable)
-		irqen |= BIT(16); /* Enables an interrupt when a data is
-				   * present in the waveform register
-				   */
-	else
-		irqen &= ~BIT(16);
-
-	ret = ade7758_spi_write_reg_24(dev, ADE7758_MASK, irqen);
-
-	return ret;
-}
-
-/* Power down the device */
-static int ade7758_stop_device(struct device *dev)
-{
-	int ret;
-	u8 val;
-
-	ret = ade7758_spi_read_reg_8(dev, ADE7758_OPMODE, &val);
-	if (ret < 0) {
-		dev_err(dev, "Failed to read opmode reg\n");
-		return ret;
-	}
-	val |= 7 << 3;  /* ADE7758 powered down */
-	ret = ade7758_spi_write_reg_8(dev, ADE7758_OPMODE, val);
-	if (ret < 0)
-		dev_err(dev, "Failed to write opmode reg\n");
-	return ret;
-}
-
-static int ade7758_initial_setup(struct iio_dev *indio_dev)
-{
-	struct ade7758_state *st = iio_priv(indio_dev);
-	struct device *dev = &indio_dev->dev;
-	int ret;
-
-	/* use low spi speed for init */
-	st->us->mode = SPI_MODE_1;
-	spi_setup(st->us);
-
-	/* Disable IRQ */
-	ret = ade7758_set_irq(dev, false);
-	if (ret) {
-		dev_err(dev, "disable irq failed");
-		goto err_ret;
-	}
-
-	ade7758_reset(dev);
-	usleep_range(ADE7758_STARTUP_DELAY, ADE7758_STARTUP_DELAY + 100);
-
-err_ret:
-	return ret;
-}
-
-static int ade7758_read_samp_freq(struct device *dev, int *val)
-{
-	int ret;
-	u8 t;
-
-	ret = ade7758_spi_read_reg_8(dev, ADE7758_WAVMODE, &t);
-	if (ret)
-		return ret;
-
-	t = (t >> 5) & 0x3;
-	*val = 26040 / (1 << t);
-
-	return 0;
-}
-
-static int ade7758_write_samp_freq(struct device *dev, int val)
-{
-	int ret;
-	u8 reg, t;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7758_state *st = iio_priv(indio_dev);
-
-	switch (val) {
-	case 26040:
-		t = 0;
-		break;
-	case 13020:
-		t = 1;
-		break;
-	case 6510:
-		t = 2;
-		break;
-	case 3255:
-		t = 3;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	mutex_lock(&st->buf_lock);
-
-	ret = __ade7758_spi_read_reg_8(dev, ADE7758_WAVMODE, &reg);
-	if (ret)
-		goto out;
-
-	reg &= ~(5 << 3);
-	reg |= t << 5;
-
-	ret = __ade7758_spi_write_reg_8(dev, ADE7758_WAVMODE, reg);
-
-out:
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7758_read_raw(struct iio_dev *indio_dev,
-			    struct iio_chan_spec const *chan,
-			    int *val,
-			    int *val2,
-			    long mask)
-{
-	int ret;
-
-	switch (mask) {
-	case IIO_CHAN_INFO_SAMP_FREQ:
-
-		ret = ade7758_read_samp_freq(&indio_dev->dev, val);
-
-		return ret;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-static int ade7758_write_raw(struct iio_dev *indio_dev,
-			     struct iio_chan_spec const *chan,
-			     int val, int val2, long mask)
-{
-	int ret;
-
-	switch (mask) {
-	case IIO_CHAN_INFO_SAMP_FREQ:
-		if (val2)
-			return -EINVAL;
-
-		ret = ade7758_write_samp_freq(&indio_dev->dev, val);
-
-		return ret;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-static IIO_DEV_ATTR_TEMP_RAW(ade7758_read_8bit);
-static IIO_CONST_ATTR(in_temp_offset, "129 C");
-static IIO_CONST_ATTR(in_temp_scale, "4 C");
-
-static IIO_DEV_ATTR_AWATTHR(ade7758_read_16bit,
-		ADE7758_AWATTHR);
-static IIO_DEV_ATTR_BWATTHR(ade7758_read_16bit,
-		ADE7758_BWATTHR);
-static IIO_DEV_ATTR_CWATTHR(ade7758_read_16bit,
-		ADE7758_CWATTHR);
-static IIO_DEV_ATTR_AVARHR(ade7758_read_16bit,
-		ADE7758_AVARHR);
-static IIO_DEV_ATTR_BVARHR(ade7758_read_16bit,
-		ADE7758_BVARHR);
-static IIO_DEV_ATTR_CVARHR(ade7758_read_16bit,
-		ADE7758_CVARHR);
-static IIO_DEV_ATTR_AVAHR(ade7758_read_16bit,
-		ADE7758_AVAHR);
-static IIO_DEV_ATTR_BVAHR(ade7758_read_16bit,
-		ADE7758_BVAHR);
-static IIO_DEV_ATTR_CVAHR(ade7758_read_16bit,
-		ADE7758_CVAHR);
-
-static IIO_CONST_ATTR_SAMP_FREQ_AVAIL("26040 13020 6510 3255");
-
-static struct attribute *ade7758_attributes[] = {
-	&iio_dev_attr_in_temp_raw.dev_attr.attr,
-	&iio_const_attr_in_temp_offset.dev_attr.attr,
-	&iio_const_attr_in_temp_scale.dev_attr.attr,
-	&iio_const_attr_sampling_frequency_available.dev_attr.attr,
-	&iio_dev_attr_awatthr.dev_attr.attr,
-	&iio_dev_attr_bwatthr.dev_attr.attr,
-	&iio_dev_attr_cwatthr.dev_attr.attr,
-	&iio_dev_attr_avarhr.dev_attr.attr,
-	&iio_dev_attr_bvarhr.dev_attr.attr,
-	&iio_dev_attr_cvarhr.dev_attr.attr,
-	&iio_dev_attr_avahr.dev_attr.attr,
-	&iio_dev_attr_bvahr.dev_attr.attr,
-	&iio_dev_attr_cvahr.dev_attr.attr,
-	&iio_dev_attr_vpeak.dev_attr.attr,
-	&iio_dev_attr_ipeak.dev_attr.attr,
-	&iio_dev_attr_aphcal.dev_attr.attr,
-	&iio_dev_attr_bphcal.dev_attr.attr,
-	&iio_dev_attr_cphcal.dev_attr.attr,
-	&iio_dev_attr_wdiv.dev_attr.attr,
-	&iio_dev_attr_vadiv.dev_attr.attr,
-	&iio_dev_attr_airms.dev_attr.attr,
-	&iio_dev_attr_birms.dev_attr.attr,
-	&iio_dev_attr_cirms.dev_attr.attr,
-	&iio_dev_attr_avrms.dev_attr.attr,
-	&iio_dev_attr_bvrms.dev_attr.attr,
-	&iio_dev_attr_cvrms.dev_attr.attr,
-	&iio_dev_attr_aigain.dev_attr.attr,
-	&iio_dev_attr_bigain.dev_attr.attr,
-	&iio_dev_attr_cigain.dev_attr.attr,
-	&iio_dev_attr_avrmsgain.dev_attr.attr,
-	&iio_dev_attr_bvrmsgain.dev_attr.attr,
-	&iio_dev_attr_cvrmsgain.dev_attr.attr,
-	&iio_dev_attr_airmsos.dev_attr.attr,
-	&iio_dev_attr_birmsos.dev_attr.attr,
-	&iio_dev_attr_cirmsos.dev_attr.attr,
-	&iio_dev_attr_avrmsos.dev_attr.attr,
-	&iio_dev_attr_bvrmsos.dev_attr.attr,
-	&iio_dev_attr_cvrmsos.dev_attr.attr,
-	NULL,
-};
-
-static const struct attribute_group ade7758_attribute_group = {
-	.attrs = ade7758_attributes,
-};
-
-static const struct iio_chan_spec ade7758_channels[] = {
-	{
-		.type = IIO_VOLTAGE,
-		.indexed = 1,
-		.channel = 0,
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_A, AD7758_VOLTAGE),
-		.scan_index = 0,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_CURRENT,
-		.indexed = 1,
-		.channel = 0,
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_A, AD7758_CURRENT),
-		.scan_index = 1,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_POWER,
-		.indexed = 1,
-		.channel = 0,
-		.extend_name = "apparent",
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_A, AD7758_APP_PWR),
-		.scan_index = 2,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_POWER,
-		.indexed = 1,
-		.channel = 0,
-		.extend_name = "active",
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_A, AD7758_ACT_PWR),
-		.scan_index = 3,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_POWER,
-		.indexed = 1,
-		.channel = 0,
-		.extend_name = "reactive",
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_A, AD7758_REACT_PWR),
-		.scan_index = 4,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_VOLTAGE,
-		.indexed = 1,
-		.channel = 1,
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_B, AD7758_VOLTAGE),
-		.scan_index = 5,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_CURRENT,
-		.indexed = 1,
-		.channel = 1,
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_B, AD7758_CURRENT),
-		.scan_index = 6,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_POWER,
-		.indexed = 1,
-		.channel = 1,
-		.extend_name = "apparent",
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_B, AD7758_APP_PWR),
-		.scan_index = 7,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_POWER,
-		.indexed = 1,
-		.channel = 1,
-		.extend_name = "active",
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_B, AD7758_ACT_PWR),
-		.scan_index = 8,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_POWER,
-		.indexed = 1,
-		.channel = 1,
-		.extend_name = "reactive",
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_B, AD7758_REACT_PWR),
-		.scan_index = 9,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_VOLTAGE,
-		.indexed = 1,
-		.channel = 2,
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_C, AD7758_VOLTAGE),
-		.scan_index = 10,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_CURRENT,
-		.indexed = 1,
-		.channel = 2,
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_C, AD7758_CURRENT),
-		.scan_index = 11,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_POWER,
-		.indexed = 1,
-		.channel = 2,
-		.extend_name = "apparent",
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_C, AD7758_APP_PWR),
-		.scan_index = 12,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_POWER,
-		.indexed = 1,
-		.channel = 2,
-		.extend_name = "active",
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_C, AD7758_ACT_PWR),
-		.scan_index = 13,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	}, {
-		.type = IIO_POWER,
-		.indexed = 1,
-		.channel = 2,
-		.extend_name = "reactive",
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
-		.address = AD7758_WT(AD7758_PHASE_C, AD7758_REACT_PWR),
-		.scan_index = 14,
-		.scan_type = {
-			.sign = 's',
-			.realbits = 24,
-			.storagebits = 32,
-		},
-	},
-	IIO_CHAN_SOFT_TIMESTAMP(15),
-};
-
-static const struct iio_info ade7758_info = {
-	.attrs = &ade7758_attribute_group,
-	.read_raw = &ade7758_read_raw,
-	.write_raw = &ade7758_write_raw,
-};
-
-static int ade7758_probe(struct spi_device *spi)
-{
-	int ret;
-	struct ade7758_state *st;
-	struct iio_dev *indio_dev;
-
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev)
-		return -ENOMEM;
-
-	st = iio_priv(indio_dev);
-	/* this is only used for removal purposes */
-	spi_set_drvdata(spi, indio_dev);
-
-	/* Allocate the comms buffers */
-	st->rx = kcalloc(ADE7758_MAX_RX, sizeof(*st->rx), GFP_KERNEL);
-	if (!st->rx)
-		return -ENOMEM;
-	st->tx = kcalloc(ADE7758_MAX_TX, sizeof(*st->tx), GFP_KERNEL);
-	if (!st->tx) {
-		ret = -ENOMEM;
-		goto error_free_rx;
-	}
-	st->us = spi;
-	mutex_init(&st->buf_lock);
-
-	indio_dev->name = spi->dev.driver->name;
-	indio_dev->dev.parent = &spi->dev;
-	indio_dev->info = &ade7758_info;
-	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->channels = ade7758_channels;
-	indio_dev->num_channels = ARRAY_SIZE(ade7758_channels);
-
-	ret = ade7758_configure_ring(indio_dev);
-	if (ret)
-		goto error_free_tx;
-
-	/* Get the device into a sane initial state */
-	ret = ade7758_initial_setup(indio_dev);
-	if (ret)
-		goto error_unreg_ring_funcs;
-
-	if (spi->irq) {
-		ret = ade7758_probe_trigger(indio_dev);
-		if (ret)
-			goto error_unreg_ring_funcs;
-	}
-
-	ret = iio_device_register(indio_dev);
-	if (ret)
-		goto error_remove_trigger;
-
-	return 0;
-
-error_remove_trigger:
-	if (spi->irq)
-		ade7758_remove_trigger(indio_dev);
-error_unreg_ring_funcs:
-	ade7758_unconfigure_ring(indio_dev);
-error_free_tx:
-	kfree(st->tx);
-error_free_rx:
-	kfree(st->rx);
-	return ret;
-}
-
-static int ade7758_remove(struct spi_device *spi)
-{
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
-	struct ade7758_state *st = iio_priv(indio_dev);
-
-	iio_device_unregister(indio_dev);
-	ade7758_stop_device(&indio_dev->dev);
-	ade7758_remove_trigger(indio_dev);
-	ade7758_unconfigure_ring(indio_dev);
-	kfree(st->tx);
-	kfree(st->rx);
-
-	return 0;
-}
-
-static const struct spi_device_id ade7758_id[] = {
-	{"ade7758", 0},
-	{}
-};
-MODULE_DEVICE_TABLE(spi, ade7758_id);
-
-static struct spi_driver ade7758_driver = {
-	.driver = {
-		.name = "ade7758",
-	},
-	.probe = ade7758_probe,
-	.remove = ade7758_remove,
-	.id_table = ade7758_id,
-};
-module_spi_driver(ade7758_driver);
-
-MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices ADE7758 Polyphase Multifunction Energy Metering IC Driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/iio/meter/ade7758_ring.c b/drivers/staging/iio/meter/ade7758_ring.c
deleted file mode 100644
index 6d7444d6e880..000000000000
--- a/drivers/staging/iio/meter/ade7758_ring.c
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * ADE7758 Poly Phase Multifunction Energy Metering IC driver
- *
- * Copyright 2010-2011 Analog Devices Inc.
- *
- * Licensed under the GPL-2.
- */
-#include <linux/export.h>
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/spi/spi.h>
-#include <linux/slab.h>
-#include <asm/unaligned.h>
-
-#include <linux/iio/iio.h>
-#include <linux/iio/buffer.h>
-#include <linux/iio/kfifo_buf.h>
-#include <linux/iio/trigger_consumer.h>
-#include "ade7758.h"
-
-/**
- * ade7758_spi_read_burst() - read data registers
- * @indio_dev: the IIO device
- **/
-static int ade7758_spi_read_burst(struct iio_dev *indio_dev)
-{
-	struct ade7758_state *st = iio_priv(indio_dev);
-	int ret;
-
-	ret = spi_sync(st->us, &st->ring_msg);
-	if (ret)
-		dev_err(&st->us->dev, "problem when reading WFORM value\n");
-
-	return ret;
-}
-
-static int ade7758_write_waveform_type(struct device *dev, unsigned int type)
-{
-	int ret;
-	u8 reg;
-
-	ret = ade7758_spi_read_reg_8(dev, ADE7758_WAVMODE, &reg);
-	if (ret)
-		goto out;
-
-	reg &= ~0x1F;
-	reg |= type & 0x1F;
-
-	ret = ade7758_spi_write_reg_8(dev, ADE7758_WAVMODE, reg);
-out:
-	return ret;
-}
-
-/* Whilst this makes a lot of calls to iio_sw_ring functions - it is too device
- * specific to be rolled into the core.
- */
-static irqreturn_t ade7758_trigger_handler(int irq, void *p)
-{
-	struct iio_poll_func *pf = p;
-	struct iio_dev *indio_dev = pf->indio_dev;
-	struct ade7758_state *st = iio_priv(indio_dev);
-	s64 dat64[2];
-	u32 *dat32 = (u32 *)dat64;
-
-	if (!bitmap_empty(indio_dev->active_scan_mask, indio_dev->masklength))
-		if (ade7758_spi_read_burst(indio_dev) >= 0)
-			*dat32 = get_unaligned_be32(&st->rx_buf[5]) & 0xFFFFFF;
-
-	iio_push_to_buffers_with_timestamp(indio_dev, dat64, pf->timestamp);
-
-	iio_trigger_notify_done(indio_dev->trig);
-
-	return IRQ_HANDLED;
-}
-
-/**
- * ade7758_ring_preenable() setup the parameters of the ring before enabling
- *
- * The complex nature of the setting of the number of bytes per datum is due
- * to this driver currently ensuring that the timestamp is stored at an 8
- * byte boundary.
- **/
-static int ade7758_ring_preenable(struct iio_dev *indio_dev)
-{
-	unsigned int channel;
-
-	if (bitmap_empty(indio_dev->active_scan_mask, indio_dev->masklength))
-		return -EINVAL;
-
-	channel = find_first_bit(indio_dev->active_scan_mask,
-				 indio_dev->masklength);
-
-	ade7758_write_waveform_type(&indio_dev->dev,
-				    indio_dev->channels[channel].address);
-
-	return 0;
-}
-
-static const struct iio_buffer_setup_ops ade7758_ring_setup_ops = {
-	.preenable = &ade7758_ring_preenable,
-	.postenable = &iio_triggered_buffer_postenable,
-	.predisable = &iio_triggered_buffer_predisable,
-	.validate_scan_mask = &iio_validate_scan_mask_onehot,
-};
-
-void ade7758_unconfigure_ring(struct iio_dev *indio_dev)
-{
-	iio_dealloc_pollfunc(indio_dev->pollfunc);
-	iio_kfifo_free(indio_dev->buffer);
-}
-
-int ade7758_configure_ring(struct iio_dev *indio_dev)
-{
-	struct ade7758_state *st = iio_priv(indio_dev);
-	struct iio_buffer *buffer;
-	int ret = 0;
-
-	buffer = iio_kfifo_allocate();
-	if (!buffer)
-		return -ENOMEM;
-
-	iio_device_attach_buffer(indio_dev, buffer);
-
-	indio_dev->setup_ops = &ade7758_ring_setup_ops;
-
-	indio_dev->pollfunc = iio_alloc_pollfunc(&iio_pollfunc_store_time,
-						 &ade7758_trigger_handler,
-						 0,
-						 indio_dev,
-						 "ade7759_consumer%d",
-						 indio_dev->id);
-	if (!indio_dev->pollfunc) {
-		ret = -ENOMEM;
-		goto error_iio_kfifo_free;
-	}
-
-	indio_dev->modes |= INDIO_BUFFER_TRIGGERED;
-
-	st->tx_buf[0] = ADE7758_READ_REG(ADE7758_RSTATUS);
-	st->tx_buf[1] = 0;
-	st->tx_buf[2] = 0;
-	st->tx_buf[3] = 0;
-	st->tx_buf[4] = ADE7758_READ_REG(ADE7758_WFORM);
-	st->tx_buf[5] = 0;
-	st->tx_buf[6] = 0;
-	st->tx_buf[7] = 0;
-
-	/* build spi ring message */
-	st->ring_xfer[0].tx_buf = &st->tx_buf[0];
-	st->ring_xfer[0].len = 1;
-	st->ring_xfer[0].bits_per_word = 8;
-	st->ring_xfer[0].delay_usecs = 4;
-	st->ring_xfer[1].rx_buf = &st->rx_buf[1];
-	st->ring_xfer[1].len = 3;
-	st->ring_xfer[1].bits_per_word = 8;
-	st->ring_xfer[1].cs_change = 1;
-
-	st->ring_xfer[2].tx_buf = &st->tx_buf[4];
-	st->ring_xfer[2].len = 1;
-	st->ring_xfer[2].bits_per_word = 8;
-	st->ring_xfer[2].delay_usecs = 1;
-	st->ring_xfer[3].rx_buf = &st->rx_buf[5];
-	st->ring_xfer[3].len = 3;
-	st->ring_xfer[3].bits_per_word = 8;
-
-	spi_message_init(&st->ring_msg);
-	spi_message_add_tail(&st->ring_xfer[0], &st->ring_msg);
-	spi_message_add_tail(&st->ring_xfer[1], &st->ring_msg);
-	spi_message_add_tail(&st->ring_xfer[2], &st->ring_msg);
-	spi_message_add_tail(&st->ring_xfer[3], &st->ring_msg);
-
-	return 0;
-
-error_iio_kfifo_free:
-	iio_kfifo_free(indio_dev->buffer);
-	return ret;
-}
diff --git a/drivers/staging/iio/meter/ade7758_trigger.c b/drivers/staging/iio/meter/ade7758_trigger.c
deleted file mode 100644
index 4f6b338cffeb..000000000000
--- a/drivers/staging/iio/meter/ade7758_trigger.c
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * ADE7758 Poly Phase Multifunction Energy Metering IC driver
- *
- * Copyright 2010-2011 Analog Devices Inc.
- *
- * Licensed under the GPL-2.
- */
-
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/spi/spi.h>
-#include <linux/export.h>
-
-#include <linux/iio/iio.h>
-#include <linux/iio/trigger.h>
-#include "ade7758.h"
-
-/**
- * ade7758_data_rdy_trig_poll() the event handler for the data rdy trig
- **/
-static irqreturn_t ade7758_data_rdy_trig_poll(int irq, void *private)
-{
-	disable_irq_nosync(irq);
-	iio_trigger_poll(private);
-
-	return IRQ_HANDLED;
-}
-
-/**
- * ade7758_data_rdy_trigger_set_state() set datardy interrupt state
- **/
-static int ade7758_data_rdy_trigger_set_state(struct iio_trigger *trig,
-					      bool state)
-{
-	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig);
-
-	dev_dbg(&indio_dev->dev, "(%d)\n", state);
-	return ade7758_set_irq(&indio_dev->dev, state);
-}
-
-/**
- * ade7758_trig_try_reen() try renabling irq for data rdy trigger
- * @trig:	the datardy trigger
- **/
-static int ade7758_trig_try_reen(struct iio_trigger *trig)
-{
-	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig);
-	struct ade7758_state *st = iio_priv(indio_dev);
-
-	enable_irq(st->us->irq);
-	/* irq reenabled so success! */
-	return 0;
-}
-
-static const struct iio_trigger_ops ade7758_trigger_ops = {
-	.set_trigger_state = &ade7758_data_rdy_trigger_set_state,
-	.try_reenable = &ade7758_trig_try_reen,
-};
-
-int ade7758_probe_trigger(struct iio_dev *indio_dev)
-{
-	struct ade7758_state *st = iio_priv(indio_dev);
-	int ret;
-
-	st->trig = iio_trigger_alloc("%s-dev%d",
-				     spi_get_device_id(st->us)->name,
-				     indio_dev->id);
-	if (!st->trig) {
-		ret = -ENOMEM;
-		goto error_ret;
-	}
-
-	ret = request_irq(st->us->irq,
-			  ade7758_data_rdy_trig_poll,
-			  IRQF_TRIGGER_LOW,
-			  spi_get_device_id(st->us)->name,
-			  st->trig);
-	if (ret)
-		goto error_free_trig;
-
-	st->trig->dev.parent = &st->us->dev;
-	st->trig->ops = &ade7758_trigger_ops;
-	iio_trigger_set_drvdata(st->trig, indio_dev);
-	ret = iio_trigger_register(st->trig);
-
-	/* select default trigger */
-	indio_dev->trig = iio_trigger_get(st->trig);
-	if (ret)
-		goto error_free_irq;
-
-	return 0;
-
-error_free_irq:
-	free_irq(st->us->irq, st->trig);
-error_free_trig:
-	iio_trigger_free(st->trig);
-error_ret:
-	return ret;
-}
-
-void ade7758_remove_trigger(struct iio_dev *indio_dev)
-{
-	struct ade7758_state *st = iio_priv(indio_dev);
-
-	iio_trigger_unregister(st->trig);
-	free_irq(st->us->irq, st->trig);
-	iio_trigger_free(st->trig);
-}
diff --git a/drivers/staging/iio/meter/ade7759.c b/drivers/staging/iio/meter/ade7759.c
deleted file mode 100644
index c078b770fa53..000000000000
--- a/drivers/staging/iio/meter/ade7759.c
+++ /dev/null
@@ -1,558 +0,0 @@
-/*
- * ADE7759 Active Energy Metering IC with di/dt Sensor Interface Driver
- *
- * Copyright 2010 Analog Devices Inc.
- *
- * Licensed under the GPL-2 or later.
- */
-
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/delay.h>
-#include <linux/mutex.h>
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/spi/spi.h>
-#include <linux/slab.h>
-#include <linux/sysfs.h>
-#include <linux/list.h>
-#include <linux/module.h>
-
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-#include "meter.h"
-
-#define ADE7759_WAVEFORM  0x01
-#define ADE7759_AENERGY   0x02
-#define ADE7759_RSTENERGY 0x03
-#define ADE7759_STATUS    0x04
-#define ADE7759_RSTSTATUS 0x05
-#define ADE7759_MODE      0x06
-#define ADE7759_CFDEN     0x07
-#define ADE7759_CH1OS     0x08
-#define ADE7759_CH2OS     0x09
-#define ADE7759_GAIN      0x0A
-#define ADE7759_APGAIN    0x0B
-#define ADE7759_PHCAL     0x0C
-#define ADE7759_APOS      0x0D
-#define ADE7759_ZXTOUT    0x0E
-#define ADE7759_SAGCYC    0x0F
-#define ADE7759_IRQEN     0x10
-#define ADE7759_SAGLVL    0x11
-#define ADE7759_TEMP      0x12
-#define ADE7759_LINECYC   0x13
-#define ADE7759_LENERGY   0x14
-#define ADE7759_CFNUM     0x15
-#define ADE7759_CHKSUM    0x1E
-#define ADE7759_DIEREV    0x1F
-
-#define ADE7759_READ_REG(a)    a
-#define ADE7759_WRITE_REG(a) ((a) | 0x80)
-
-#define ADE7759_MAX_TX    6
-#define ADE7759_MAX_RX    6
-#define ADE7759_STARTUP_DELAY 1000
-
-#define ADE7759_SPI_SLOW	(u32)(300 * 1000)
-#define ADE7759_SPI_BURST	(u32)(1000 * 1000)
-#define ADE7759_SPI_FAST	(u32)(2000 * 1000)
-
-/**
- * struct ade7759_state - device instance specific data
- * @us:			actual spi_device
- * @buf_lock:		mutex to protect tx and rx and write frequency
- * @tx:			transmit buffer
- * @rx:			receive buffer
- **/
-struct ade7759_state {
-	struct spi_device	*us;
-	struct mutex		buf_lock;
-	u8			tx[ADE7759_MAX_TX] ____cacheline_aligned;
-	u8			rx[ADE7759_MAX_RX];
-};
-
-static int ade7759_spi_write_reg_8(struct device *dev,
-				   u8 reg_address,
-				   u8 val)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7759_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7759_WRITE_REG(reg_address);
-	st->tx[1] = val;
-
-	ret = spi_write(st->us, st->tx, 2);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-/*Unlocked version of ade7759_spi_write_reg_16 function */
-static int __ade7759_spi_write_reg_16(struct device *dev,
-				      u8 reg_address,
-				      u16 value)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7759_state *st = iio_priv(indio_dev);
-
-	st->tx[0] = ADE7759_WRITE_REG(reg_address);
-	st->tx[1] = (value >> 8) & 0xFF;
-	st->tx[2] = value & 0xFF;
-	return spi_write(st->us, st->tx, 3);
-}
-
-static int ade7759_spi_write_reg_16(struct device *dev,
-				    u8 reg_address,
-				    u16 value)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7759_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->buf_lock);
-	ret = __ade7759_spi_write_reg_16(dev, reg_address, value);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7759_spi_read_reg_8(struct device *dev,
-				  u8 reg_address,
-				  u8 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7759_state *st = iio_priv(indio_dev);
-	int ret;
-
-	ret = spi_w8r8(st->us, ADE7759_READ_REG(reg_address));
-	if (ret < 0) {
-		dev_err(&st->us->dev,
-			"problem when reading 8 bit register 0x%02X",
-			reg_address);
-		return ret;
-	}
-	*val = ret;
-
-	return 0;
-}
-
-static int ade7759_spi_read_reg_16(struct device *dev,
-				   u8 reg_address,
-				   u16 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7759_state *st = iio_priv(indio_dev);
-	int ret;
-
-	ret = spi_w8r16be(st->us, ADE7759_READ_REG(reg_address));
-	if (ret < 0) {
-		dev_err(&st->us->dev, "problem when reading 16 bit register 0x%02X",
-			reg_address);
-		return ret;
-	}
-
-	*val = ret;
-
-	return 0;
-}
-
-static int ade7759_spi_read_reg_40(struct device *dev,
-				   u8 reg_address,
-				   u64 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7759_state *st = iio_priv(indio_dev);
-	int ret;
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.rx_buf = st->rx,
-			.bits_per_word = 8,
-			.len = 6,
-		},
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7759_READ_REG(reg_address);
-	memset(&st->tx[1], 0, 5);
-
-	ret = spi_sync_transfer(st->us, xfers, ARRAY_SIZE(xfers));
-	if (ret) {
-		dev_err(&st->us->dev,
-			"problem when reading 40 bit register 0x%02X",
-			reg_address);
-		goto error_ret;
-	}
-	*val = ((u64)st->rx[1] << 32) | ((u64)st->rx[2] << 24) |
-		(st->rx[3] << 16) | (st->rx[4] << 8) | st->rx[5];
-
-error_ret:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static ssize_t ade7759_read_8bit(struct device *dev,
-				 struct device_attribute *attr,
-				 char *buf)
-{
-	int ret;
-	u8 val = 0;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7759_spi_read_reg_8(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7759_read_16bit(struct device *dev,
-				  struct device_attribute *attr,
-				  char *buf)
-{
-	int ret;
-	u16 val = 0;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7759_spi_read_reg_16(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%u\n", val);
-}
-
-static ssize_t ade7759_read_40bit(struct device *dev,
-				  struct device_attribute *attr,
-				  char *buf)
-{
-	int ret;
-	u64 val = 0;
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-
-	ret = ade7759_spi_read_reg_40(dev, this_attr->address, &val);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%llu\n", val);
-}
-
-static ssize_t ade7759_write_8bit(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf,
-				  size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	int ret;
-	u8 val;
-
-	ret = kstrtou8(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = ade7759_spi_write_reg_8(dev, this_attr->address, val);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static ssize_t ade7759_write_16bit(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf,
-				   size_t len)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	int ret;
-	u16 val;
-
-	ret = kstrtou16(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = ade7759_spi_write_reg_16(dev, this_attr->address, val);
-
-error_ret:
-	return ret ? ret : len;
-}
-
-static int ade7759_reset(struct device *dev)
-{
-	int ret;
-	u16 val;
-
-	ret = ade7759_spi_read_reg_16(dev, ADE7759_MODE, &val);
-	if (ret < 0)
-		return ret;
-
-	val |= BIT(6); /* Software Chip Reset */
-	return ade7759_spi_write_reg_16(dev,
-			ADE7759_MODE,
-			val);
-}
-
-static IIO_DEV_ATTR_AENERGY(ade7759_read_40bit, ADE7759_AENERGY);
-static IIO_DEV_ATTR_CFDEN(0644,
-		ade7759_read_16bit,
-		ade7759_write_16bit,
-		ADE7759_CFDEN);
-static IIO_DEV_ATTR_CFNUM(0644,
-		ade7759_read_8bit,
-		ade7759_write_8bit,
-		ADE7759_CFNUM);
-static IIO_DEV_ATTR_CHKSUM(ade7759_read_8bit, ADE7759_CHKSUM);
-static IIO_DEV_ATTR_PHCAL(0644,
-		ade7759_read_16bit,
-		ade7759_write_16bit,
-		ADE7759_PHCAL);
-static IIO_DEV_ATTR_APOS(0644,
-		ade7759_read_16bit,
-		ade7759_write_16bit,
-		ADE7759_APOS);
-static IIO_DEV_ATTR_SAGCYC(0644,
-		ade7759_read_8bit,
-		ade7759_write_8bit,
-		ADE7759_SAGCYC);
-static IIO_DEV_ATTR_SAGLVL(0644,
-		ade7759_read_8bit,
-		ade7759_write_8bit,
-		ADE7759_SAGLVL);
-static IIO_DEV_ATTR_LINECYC(0644,
-		ade7759_read_8bit,
-		ade7759_write_8bit,
-		ADE7759_LINECYC);
-static IIO_DEV_ATTR_LENERGY(ade7759_read_40bit, ADE7759_LENERGY);
-static IIO_DEV_ATTR_PGA_GAIN(0644,
-		ade7759_read_8bit,
-		ade7759_write_8bit,
-		ADE7759_GAIN);
-static IIO_DEV_ATTR_ACTIVE_POWER_GAIN(0644,
-		ade7759_read_16bit,
-		ade7759_write_16bit,
-		ADE7759_APGAIN);
-
-static IIO_DEVICE_ATTR(choff_1, 0644,
-			ade7759_read_8bit,
-			ade7759_write_8bit,
-			ADE7759_CH1OS);
-
-static IIO_DEVICE_ATTR(choff_2, 0644,
-			ade7759_read_8bit,
-			ade7759_write_8bit,
-			ADE7759_CH2OS);
-
-static int ade7759_set_irq(struct device *dev, bool enable)
-{
-	int ret;
-	u8 irqen;
-
-	ret = ade7759_spi_read_reg_8(dev, ADE7759_IRQEN, &irqen);
-	if (ret)
-		goto error_ret;
-
-	if (enable)
-		irqen |= BIT(3); /* Enables an interrupt when a data is
-				  * present in the waveform register
-				  */
-	else
-		irqen &= ~BIT(3);
-
-	ret = ade7759_spi_write_reg_8(dev, ADE7759_IRQEN, irqen);
-
-error_ret:
-	return ret;
-}
-
-/* Power down the device */
-static int ade7759_stop_device(struct device *dev)
-{
-	int ret;
-	u16 val;
-
-	ret = ade7759_spi_read_reg_16(dev, ADE7759_MODE, &val);
-	if (ret < 0) {
-		dev_err(dev, "unable to power down the device, error: %d\n",
-			ret);
-		return ret;
-	}
-
-	val |= BIT(4);  /* AD converters can be turned off */
-
-	return ade7759_spi_write_reg_16(dev, ADE7759_MODE, val);
-}
-
-static int ade7759_initial_setup(struct iio_dev *indio_dev)
-{
-	int ret;
-	struct ade7759_state *st = iio_priv(indio_dev);
-	struct device *dev = &indio_dev->dev;
-
-	/* use low spi speed for init */
-	st->us->mode = SPI_MODE_3;
-	spi_setup(st->us);
-
-	/* Disable IRQ */
-	ret = ade7759_set_irq(dev, false);
-	if (ret) {
-		dev_err(dev, "disable irq failed");
-		goto err_ret;
-	}
-
-	ade7759_reset(dev);
-	usleep_range(ADE7759_STARTUP_DELAY, ADE7759_STARTUP_DELAY + 100);
-
-err_ret:
-	return ret;
-}
-
-static ssize_t ade7759_read_frequency(struct device *dev,
-				      struct device_attribute *attr,
-				      char *buf)
-{
-	int ret;
-	u16 t;
-	int sps;
-
-	ret = ade7759_spi_read_reg_16(dev, ADE7759_MODE, &t);
-	if (ret)
-		return ret;
-
-	t = (t >> 3) & 0x3;
-	sps = 27900 / (1 + t);
-
-	return sprintf(buf, "%d\n", sps);
-}
-
-static ssize_t ade7759_write_frequency(struct device *dev,
-				       struct device_attribute *attr,
-				       const char *buf,
-				       size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7759_state *st = iio_priv(indio_dev);
-	u16 val;
-	int ret;
-	u16 reg, t;
-
-	ret = kstrtou16(buf, 10, &val);
-	if (ret)
-		return ret;
-	if (!val)
-		return -EINVAL;
-
-	mutex_lock(&st->buf_lock);
-
-	t = 27900 / val;
-	if (t > 0)
-		t--;
-
-	if (t > 1)
-		st->us->max_speed_hz = ADE7759_SPI_SLOW;
-	else
-		st->us->max_speed_hz = ADE7759_SPI_FAST;
-
-	ret = ade7759_spi_read_reg_16(dev, ADE7759_MODE, &reg);
-	if (ret)
-		goto out;
-
-	reg &= ~(3 << 13);
-	reg |= t << 13;
-
-	ret = __ade7759_spi_write_reg_16(dev, ADE7759_MODE, reg);
-
-out:
-	mutex_unlock(&st->buf_lock);
-
-	return ret ? ret : len;
-}
-static IIO_DEV_ATTR_TEMP_RAW(ade7759_read_8bit);
-static IIO_CONST_ATTR(in_temp_offset, "70 C");
-static IIO_CONST_ATTR(in_temp_scale, "1 C");
-
-static IIO_DEV_ATTR_SAMP_FREQ(0644,
-		ade7759_read_frequency,
-		ade7759_write_frequency);
-
-static IIO_CONST_ATTR_SAMP_FREQ_AVAIL("27900 14000 7000 3500");
-
-static struct attribute *ade7759_attributes[] = {
-	&iio_dev_attr_in_temp_raw.dev_attr.attr,
-	&iio_const_attr_in_temp_offset.dev_attr.attr,
-	&iio_const_attr_in_temp_scale.dev_attr.attr,
-	&iio_dev_attr_sampling_frequency.dev_attr.attr,
-	&iio_const_attr_sampling_frequency_available.dev_attr.attr,
-	&iio_dev_attr_phcal.dev_attr.attr,
-	&iio_dev_attr_cfden.dev_attr.attr,
-	&iio_dev_attr_aenergy.dev_attr.attr,
-	&iio_dev_attr_cfnum.dev_attr.attr,
-	&iio_dev_attr_apos.dev_attr.attr,
-	&iio_dev_attr_sagcyc.dev_attr.attr,
-	&iio_dev_attr_saglvl.dev_attr.attr,
-	&iio_dev_attr_linecyc.dev_attr.attr,
-	&iio_dev_attr_lenergy.dev_attr.attr,
-	&iio_dev_attr_chksum.dev_attr.attr,
-	&iio_dev_attr_pga_gain.dev_attr.attr,
-	&iio_dev_attr_active_power_gain.dev_attr.attr,
-	&iio_dev_attr_choff_1.dev_attr.attr,
-	&iio_dev_attr_choff_2.dev_attr.attr,
-	NULL,
-};
-
-static const struct attribute_group ade7759_attribute_group = {
-	.attrs = ade7759_attributes,
-};
-
-static const struct iio_info ade7759_info = {
-	.attrs = &ade7759_attribute_group,
-};
-
-static int ade7759_probe(struct spi_device *spi)
-{
-	int ret;
-	struct ade7759_state *st;
-	struct iio_dev *indio_dev;
-
-	/* setup the industrialio driver allocated elements */
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev)
-		return -ENOMEM;
-	/* this is only used for removal purposes */
-	spi_set_drvdata(spi, indio_dev);
-
-	st = iio_priv(indio_dev);
-	st->us = spi;
-	mutex_init(&st->buf_lock);
-	indio_dev->name = spi->dev.driver->name;
-	indio_dev->dev.parent = &spi->dev;
-	indio_dev->info = &ade7759_info;
-	indio_dev->modes = INDIO_DIRECT_MODE;
-
-	/* Get the device into a sane initial state */
-	ret = ade7759_initial_setup(indio_dev);
-	if (ret)
-		return ret;
-
-	return iio_device_register(indio_dev);
-}
-
-static int ade7759_remove(struct spi_device *spi)
-{
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
-
-	iio_device_unregister(indio_dev);
-	ade7759_stop_device(&indio_dev->dev);
-
-	return 0;
-}
-
-static struct spi_driver ade7759_driver = {
-	.driver = {
-		.name = "ade7759",
-	},
-	.probe = ade7759_probe,
-	.remove = ade7759_remove,
-};
-module_spi_driver(ade7759_driver);
-
-MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices ADE7759 Active Energy Metering IC Driver");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("spi:ad7759");
diff --git a/drivers/staging/iio/meter/ade7854-i2c.c b/drivers/staging/iio/meter/ade7854-i2c.c
index 317e4f0d8176..c3aa6ea9d036 100644
--- a/drivers/staging/iio/meter/ade7854-i2c.c
+++ b/drivers/staging/iio/meter/ade7854-i2c.c
@@ -15,91 +15,60 @@
 #include <linux/iio/iio.h>
 #include "ade7854.h"
 
-static int ade7854_i2c_write_reg_8(struct device *dev,
-				   u16 reg_address,
-				   u8 val)
+static int ade7854_i2c_write_reg(struct device *dev,
+				 u16 reg_address,
+				 u32 val,
+				 int bits)
 {
 	int ret;
+	int count;
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ade7854_state *st = iio_priv(indio_dev);
 
 	mutex_lock(&st->buf_lock);
 	st->tx[0] = (reg_address >> 8) & 0xFF;
 	st->tx[1] = reg_address & 0xFF;
-	st->tx[2] = val;
 
-	ret = i2c_master_send(st->i2c, st->tx, 3);
+	switch (bits) {
+	case 8:
+		st->tx[2] = val & 0xFF;
+		count = 3;
+		break;
+	case 16:
+		st->tx[2] = (val >> 8) & 0xFF;
+		st->tx[3] = val & 0xFF;
+		count = 4;
+		break;
+	case 24:
+		st->tx[2] = (val >> 16) & 0xFF;
+		st->tx[3] = (val >> 8) & 0xFF;
+		st->tx[4] = val & 0xFF;
+		count = 5;
+		break;
+	case 32:
+		st->tx[2] = (val >> 24) & 0xFF;
+		st->tx[3] = (val >> 16) & 0xFF;
+		st->tx[4] = (val >> 8) & 0xFF;
+		st->tx[5] = val & 0xFF;
+		count = 6;
+		break;
+	default:
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	ret = i2c_master_send(st->i2c, st->tx, count);
+
+unlock:
 	mutex_unlock(&st->buf_lock);
 
-	return ret;
+	return ret < 0 ? ret : 0;
 }
 
-static int ade7854_i2c_write_reg_16(struct device *dev,
-				    u16 reg_address,
-				    u16 val)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = (reg_address >> 8) & 0xFF;
-	st->tx[1] = reg_address & 0xFF;
-	st->tx[2] = (val >> 8) & 0xFF;
-	st->tx[3] = val & 0xFF;
-
-	ret = i2c_master_send(st->i2c, st->tx, 4);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7854_i2c_write_reg_24(struct device *dev,
-				    u16 reg_address,
-				    u32 val)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = (reg_address >> 8) & 0xFF;
-	st->tx[1] = reg_address & 0xFF;
-	st->tx[2] = (val >> 16) & 0xFF;
-	st->tx[3] = (val >> 8) & 0xFF;
-	st->tx[4] = val & 0xFF;
-
-	ret = i2c_master_send(st->i2c, st->tx, 5);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7854_i2c_write_reg_32(struct device *dev,
-				    u16 reg_address,
-				    u32 val)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = (reg_address >> 8) & 0xFF;
-	st->tx[1] = reg_address & 0xFF;
-	st->tx[2] = (val >> 24) & 0xFF;
-	st->tx[3] = (val >> 16) & 0xFF;
-	st->tx[4] = (val >> 8) & 0xFF;
-	st->tx[5] = val & 0xFF;
-
-	ret = i2c_master_send(st->i2c, st->tx, 6);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7854_i2c_read_reg_8(struct device *dev,
-				  u16 reg_address,
-				  u8 *val)
+static int ade7854_i2c_read_reg(struct device *dev,
+				u16 reg_address,
+				u32 *val,
+				int bits)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ade7854_state *st = iio_priv(indio_dev);
@@ -110,94 +79,33 @@ static int ade7854_i2c_read_reg_8(struct device *dev,
 	st->tx[1] = reg_address & 0xFF;
 
 	ret = i2c_master_send(st->i2c, st->tx, 2);
-	if (ret)
-		goto out;
-
-	ret = i2c_master_recv(st->i2c, st->rx, 1);
-	if (ret)
-		goto out;
-
-	*val = st->rx[0];
-out:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static int ade7854_i2c_read_reg_16(struct device *dev,
-				   u16 reg_address,
-				   u16 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	int ret;
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = (reg_address >> 8) & 0xFF;
-	st->tx[1] = reg_address & 0xFF;
-
-	ret = i2c_master_send(st->i2c, st->tx, 2);
-	if (ret)
-		goto out;
-
-	ret = i2c_master_recv(st->i2c, st->rx, 2);
-	if (ret)
-		goto out;
-
-	*val = (st->rx[0] << 8) | st->rx[1];
-out:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static int ade7854_i2c_read_reg_24(struct device *dev,
-				   u16 reg_address,
-				   u32 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	int ret;
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = (reg_address >> 8) & 0xFF;
-	st->tx[1] = reg_address & 0xFF;
-
-	ret = i2c_master_send(st->i2c, st->tx, 2);
-	if (ret)
-		goto out;
-
-	ret = i2c_master_recv(st->i2c, st->rx, 3);
-	if (ret)
-		goto out;
-
-	*val = (st->rx[0] << 16) | (st->rx[1] << 8) | st->rx[2];
-out:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static int ade7854_i2c_read_reg_32(struct device *dev,
-				   u16 reg_address,
-				   u32 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	int ret;
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = (reg_address >> 8) & 0xFF;
-	st->tx[1] = reg_address & 0xFF;
-
-	ret = i2c_master_send(st->i2c, st->tx, 2);
-	if (ret)
-		goto out;
-
-	ret = i2c_master_recv(st->i2c, st->rx, 3);
-	if (ret)
-		goto out;
-
-	*val = (st->rx[0] << 24) | (st->rx[1] << 16) |
-		(st->rx[2] << 8) | st->rx[3];
-out:
+	if (ret < 0)
+		goto unlock;
+
+	ret = i2c_master_recv(st->i2c, st->rx, bits);
+	if (ret < 0)
+		goto unlock;
+
+	switch (bits) {
+	case 8:
+		*val = st->rx[0];
+		break;
+	case 16:
+		*val = (st->rx[0] << 8) | st->rx[1];
+		break;
+	case 24:
+		*val = (st->rx[0] << 16) | (st->rx[1] << 8) | st->rx[2];
+		break;
+	case 32:
+		*val = (st->rx[0] << 24) | (st->rx[1] << 16) |
+			(st->rx[2] << 8) | st->rx[3];
+		break;
+	default:
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+unlock:
 	mutex_unlock(&st->buf_lock);
 	return ret;
 }
@@ -213,14 +121,8 @@ static int ade7854_i2c_probe(struct i2c_client *client,
 		return -ENOMEM;
 	st = iio_priv(indio_dev);
 	i2c_set_clientdata(client, indio_dev);
-	st->read_reg_8 = ade7854_i2c_read_reg_8;
-	st->read_reg_16 = ade7854_i2c_read_reg_16;
-	st->read_reg_24 = ade7854_i2c_read_reg_24;
-	st->read_reg_32 = ade7854_i2c_read_reg_32;
-	st->write_reg_8 = ade7854_i2c_write_reg_8;
-	st->write_reg_16 = ade7854_i2c_write_reg_16;
-	st->write_reg_24 = ade7854_i2c_write_reg_24;
-	st->write_reg_32 = ade7854_i2c_write_reg_32;
+	st->read_reg = ade7854_i2c_read_reg;
+	st->write_reg = ade7854_i2c_write_reg;
 	st->i2c = client;
 	st->irq = client->irq;
 
diff --git a/drivers/staging/iio/meter/ade7854-spi.c b/drivers/staging/iio/meter/ade7854-spi.c
index 4419b8f06197..fc9146757283 100644
--- a/drivers/staging/iio/meter/ade7854-spi.c
+++ b/drivers/staging/iio/meter/ade7854-spi.c
@@ -15,9 +15,10 @@
 #include <linux/iio/iio.h>
 #include "ade7854.h"
 
-static int ade7854_spi_write_reg_8(struct device *dev,
-				   u16 reg_address,
-				   u8 val)
+static int ade7854_spi_write_reg(struct device *dev,
+				 u16 reg_address,
+				 u32 val,
+				 int bits)
 {
 	int ret;
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
@@ -32,173 +33,44 @@ static int ade7854_spi_write_reg_8(struct device *dev,
 	st->tx[0] = ADE7854_WRITE_REG;
 	st->tx[1] = (reg_address >> 8) & 0xFF;
 	st->tx[2] = reg_address & 0xFF;
-	st->tx[3] = val & 0xFF;
-
-	ret = spi_sync_transfer(st->spi, &xfer, 1);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7854_spi_write_reg_16(struct device *dev,
-				    u16 reg_address,
-				    u16 val)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	struct spi_transfer xfer = {
-		.tx_buf = st->tx,
-		.bits_per_word = 8,
-		.len = 5,
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7854_WRITE_REG;
-	st->tx[1] = (reg_address >> 8) & 0xFF;
-	st->tx[2] = reg_address & 0xFF;
-	st->tx[3] = (val >> 8) & 0xFF;
-	st->tx[4] = val & 0xFF;
-
-	ret = spi_sync_transfer(st->spi, &xfer, 1);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7854_spi_write_reg_24(struct device *dev,
-				    u16 reg_address,
-				    u32 val)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	struct spi_transfer xfer = {
-		.tx_buf = st->tx,
-		.bits_per_word = 8,
-		.len = 6,
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7854_WRITE_REG;
-	st->tx[1] = (reg_address >> 8) & 0xFF;
-	st->tx[2] = reg_address & 0xFF;
-	st->tx[3] = (val >> 16) & 0xFF;
-	st->tx[4] = (val >> 8) & 0xFF;
-	st->tx[5] = val & 0xFF;
-
-	ret = spi_sync_transfer(st->spi, &xfer, 1);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7854_spi_write_reg_32(struct device *dev,
-				    u16 reg_address,
-				    u32 val)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	struct spi_transfer xfer = {
-		.tx_buf = st->tx,
-		.bits_per_word = 8,
-		.len = 7,
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7854_WRITE_REG;
-	st->tx[1] = (reg_address >> 8) & 0xFF;
-	st->tx[2] = reg_address & 0xFF;
-	st->tx[3] = (val >> 24) & 0xFF;
-	st->tx[4] = (val >> 16) & 0xFF;
-	st->tx[5] = (val >> 8) & 0xFF;
-	st->tx[6] = val & 0xFF;
-
-	ret = spi_sync_transfer(st->spi, &xfer, 1);
-	mutex_unlock(&st->buf_lock);
-
-	return ret;
-}
-
-static int ade7854_spi_read_reg_8(struct device *dev,
-				  u16 reg_address,
-				  u8 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	int ret;
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.bits_per_word = 8,
-			.len = 3,
-		}, {
-			.rx_buf = st->rx,
-			.bits_per_word = 8,
-			.len = 1,
-		}
-	};
-
-	mutex_lock(&st->buf_lock);
-
-	st->tx[0] = ADE7854_READ_REG;
-	st->tx[1] = (reg_address >> 8) & 0xFF;
-	st->tx[2] = reg_address & 0xFF;
-
-	ret = spi_sync_transfer(st->spi, xfers, ARRAY_SIZE(xfers));
-	if (ret) {
-		dev_err(&st->spi->dev, "problem when reading 8 bit register 0x%02X",
-			reg_address);
-		goto error_ret;
+	switch (bits) {
+	case 8:
+		st->tx[3] = val & 0xFF;
+		break;
+	case 16:
+		xfer.len = 5;
+		st->tx[3] = (val >> 8) & 0xFF;
+		st->tx[4] = val & 0xFF;
+		break;
+	case 24:
+		xfer.len = 6;
+		st->tx[3] = (val >> 16) & 0xFF;
+		st->tx[4] = (val >> 8) & 0xFF;
+		st->tx[5] = val & 0xFF;
+		break;
+	case 32:
+		xfer.len = 7;
+		st->tx[3] = (val >> 24) & 0xFF;
+		st->tx[4] = (val >> 16) & 0xFF;
+		st->tx[5] = (val >> 8) & 0xFF;
+		st->tx[6] = val & 0xFF;
+		break;
+	default:
+		ret = -EINVAL;
+		goto unlock;
 	}
-	*val = st->rx[0];
 
-error_ret:
+	ret = spi_sync_transfer(st->spi, &xfer, 1);
+unlock:
 	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static int ade7854_spi_read_reg_16(struct device *dev,
-				   u16 reg_address,
-				   u16 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	int ret;
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.bits_per_word = 8,
-			.len = 3,
-		}, {
-			.rx_buf = st->rx,
-			.bits_per_word = 8,
-			.len = 2,
-		}
-	};
 
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = ADE7854_READ_REG;
-	st->tx[1] = (reg_address >> 8) & 0xFF;
-	st->tx[2] = reg_address & 0xFF;
-
-	ret = spi_sync_transfer(st->spi, xfers, ARRAY_SIZE(xfers));
-	if (ret) {
-		dev_err(&st->spi->dev, "problem when reading 16 bit register 0x%02X",
-			reg_address);
-		goto error_ret;
-	}
-	*val = be16_to_cpup((const __be16 *)st->rx);
-
-error_ret:
-	mutex_unlock(&st->buf_lock);
 	return ret;
 }
 
-static int ade7854_spi_read_reg_24(struct device *dev,
-				   u16 reg_address,
-				   u32 *val)
+static int ade7854_spi_read_reg(struct device *dev,
+				u16 reg_address,
+				u32 *val,
+				int bits)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ade7854_state *st = iio_priv(indio_dev);
@@ -211,7 +83,7 @@ static int ade7854_spi_read_reg_24(struct device *dev,
 		}, {
 			.rx_buf = st->rx,
 			.bits_per_word = 8,
-			.len = 3,
+			.len = bits,
 		}
 	};
 
@@ -222,52 +94,28 @@ static int ade7854_spi_read_reg_24(struct device *dev,
 	st->tx[2] = reg_address & 0xFF;
 
 	ret = spi_sync_transfer(st->spi, xfers, ARRAY_SIZE(xfers));
-	if (ret) {
-		dev_err(&st->spi->dev, "problem when reading 24 bit register 0x%02X",
+	if (ret < 0) {
+		dev_err(&st->spi->dev, "problem when reading register 0x%02X",
 			reg_address);
-		goto error_ret;
+		goto unlock;
 	}
-	*val = (st->rx[0] << 16) | (st->rx[1] << 8) | st->rx[2];
-
-error_ret:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
-
-static int ade7854_spi_read_reg_32(struct device *dev,
-				   u16 reg_address,
-				   u32 *val)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ade7854_state *st = iio_priv(indio_dev);
-	int ret;
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = st->tx,
-			.bits_per_word = 8,
-			.len = 3,
-		}, {
-			.rx_buf = st->rx,
-			.bits_per_word = 8,
-			.len = 4,
-		}
-	};
-
-	mutex_lock(&st->buf_lock);
 
-	st->tx[0] = ADE7854_READ_REG;
-	st->tx[1] = (reg_address >> 8) & 0xFF;
-	st->tx[2] = reg_address & 0xFF;
-
-	ret = spi_sync_transfer(st->spi, xfers, ARRAY_SIZE(xfers));
-	if (ret) {
-		dev_err(&st->spi->dev, "problem when reading 32 bit register 0x%02X",
-			reg_address);
-		goto error_ret;
+	switch (bits) {
+	case 8:
+		*val = st->rx[0];
+		break;
+	case 16:
+		*val = be16_to_cpup((const __be16 *)st->rx);
+		break;
+	case 24:
+		*val = (st->rx[0] << 16) | (st->rx[1] << 8) | st->rx[2];
+		break;
+	case 32:
+		*val = be32_to_cpup((const __be32 *)st->rx);
+		break;
 	}
-	*val = be32_to_cpup((const __be32 *)st->rx);
 
-error_ret:
+unlock:
 	mutex_unlock(&st->buf_lock);
 	return ret;
 }
@@ -282,14 +130,8 @@ static int ade7854_spi_probe(struct spi_device *spi)
 		return -ENOMEM;
 	st = iio_priv(indio_dev);
 	spi_set_drvdata(spi, indio_dev);
-	st->read_reg_8 = ade7854_spi_read_reg_8;
-	st->read_reg_16 = ade7854_spi_read_reg_16;
-	st->read_reg_24 = ade7854_spi_read_reg_24;
-	st->read_reg_32 = ade7854_spi_read_reg_32;
-	st->write_reg_8 = ade7854_spi_write_reg_8;
-	st->write_reg_16 = ade7854_spi_write_reg_16;
-	st->write_reg_24 = ade7854_spi_write_reg_24;
-	st->write_reg_32 = ade7854_spi_write_reg_32;
+	st->read_reg = ade7854_spi_read_reg;
+	st->write_reg = ade7854_spi_write_reg;
 	st->irq = spi->irq;
 	st->spi = spi;
 
diff --git a/drivers/staging/iio/meter/ade7854.c b/drivers/staging/iio/meter/ade7854.c
index 90d07cdca4b8..029c3bf42d4d 100644
--- a/drivers/staging/iio/meter/ade7854.c
+++ b/drivers/staging/iio/meter/ade7854.c
@@ -27,13 +27,13 @@ static ssize_t ade7854_read_8bit(struct device *dev,
 				 char *buf)
 {
 	int ret;
-	u8 val = 0;
+	u32 val = 0;
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ade7854_state *st = iio_priv(indio_dev);
 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
 
-	ret = st->read_reg_8(dev, this_attr->address, &val);
-	if (ret)
+	ret = st->read_reg(dev, this_attr->address, &val, 8);
+	if (ret < 0)
 		return ret;
 
 	return sprintf(buf, "%u\n", val);
@@ -44,13 +44,13 @@ static ssize_t ade7854_read_16bit(struct device *dev,
 				  char *buf)
 {
 	int ret;
-	u16 val = 0;
+	u32 val = 0;
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ade7854_state *st = iio_priv(indio_dev);
 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
 
-	ret = st->read_reg_16(dev, this_attr->address, &val);
-	if (ret)
+	ret = st->read_reg(dev, this_attr->address, &val, 16);
+	if (ret < 0)
 		return ret;
 
 	return sprintf(buf, "%u\n", val);
@@ -66,8 +66,8 @@ static ssize_t ade7854_read_24bit(struct device *dev,
 	struct ade7854_state *st = iio_priv(indio_dev);
 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
 
-	ret = st->read_reg_24(dev, this_attr->address, &val);
-	if (ret)
+	ret = st->read_reg(dev, this_attr->address, &val, 24);
+	if (ret < 0)
 		return ret;
 
 	return sprintf(buf, "%u\n", val);
@@ -83,8 +83,8 @@ static ssize_t ade7854_read_32bit(struct device *dev,
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ade7854_state *st = iio_priv(indio_dev);
 
-	ret = st->read_reg_32(dev, this_attr->address, &val);
-	if (ret)
+	ret = st->read_reg(dev, this_attr->address, &val, 32);
+	if (ret < 0)
 		return ret;
 
 	return sprintf(buf, "%u\n", val);
@@ -105,7 +105,7 @@ static ssize_t ade7854_write_8bit(struct device *dev,
 	ret = kstrtou8(buf, 10, &val);
 	if (ret)
 		goto error_ret;
-	ret = st->write_reg_8(dev, this_attr->address, val);
+	ret = st->write_reg(dev, this_attr->address, val, 8);
 
 error_ret:
 	return ret ? ret : len;
@@ -126,7 +126,7 @@ static ssize_t ade7854_write_16bit(struct device *dev,
 	ret = kstrtou16(buf, 10, &val);
 	if (ret)
 		goto error_ret;
-	ret = st->write_reg_16(dev, this_attr->address, val);
+	ret = st->write_reg(dev, this_attr->address, val, 16);
 
 error_ret:
 	return ret ? ret : len;
@@ -147,7 +147,7 @@ static ssize_t ade7854_write_24bit(struct device *dev,
 	ret = kstrtou32(buf, 10, &val);
 	if (ret)
 		goto error_ret;
-	ret = st->write_reg_24(dev, this_attr->address, val);
+	ret = st->write_reg(dev, this_attr->address, val, 24);
 
 error_ret:
 	return ret ? ret : len;
@@ -168,7 +168,7 @@ static ssize_t ade7854_write_32bit(struct device *dev,
 	ret = kstrtou32(buf, 10, &val);
 	if (ret)
 		goto error_ret;
-	ret = st->write_reg_32(dev, this_attr->address, val);
+	ret = st->write_reg(dev, this_attr->address, val, 32);
 
 error_ret:
 	return ret ? ret : len;
@@ -178,12 +178,12 @@ static int ade7854_reset(struct device *dev)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ade7854_state *st = iio_priv(indio_dev);
-	u16 val;
+	u32 val;
 
-	st->read_reg_16(dev, ADE7854_CONFIG, &val);
+	st->read_reg(dev, ADE7854_CONFIG, &val, 16);
 	val |= BIT(7); /* Software Chip Reset */
 
-	return st->write_reg_16(dev, ADE7854_CONFIG, val);
+	return st->write_reg(dev, ADE7854_CONFIG, val, 16);
 }
 
 static IIO_DEV_ATTR_AIGAIN(0644,
@@ -415,8 +415,8 @@ static int ade7854_set_irq(struct device *dev, bool enable)
 	int ret;
 	u32 irqen;
 
-	ret = st->read_reg_32(dev, ADE7854_MASK0, &irqen);
-	if (ret)
+	ret = st->read_reg(dev, ADE7854_MASK0, &irqen, 32);
+	if (ret < 0)
 		return ret;
 
 	if (enable)
@@ -426,7 +426,7 @@ static int ade7854_set_irq(struct device *dev, bool enable)
 	else
 		irqen &= ~BIT(17);
 
-	return st->write_reg_32(dev, ADE7854_MASK0, irqen);
+	return st->write_reg(dev, ADE7854_MASK0, irqen, 32);
 }
 
 static int ade7854_initial_setup(struct iio_dev *indio_dev)
diff --git a/drivers/staging/iio/meter/ade7854.h b/drivers/staging/iio/meter/ade7854.h
index a82d38224cbd..a51e6e3183d3 100644
--- a/drivers/staging/iio/meter/ade7854.h
+++ b/drivers/staging/iio/meter/ade7854.h
@@ -145,7 +145,9 @@
 
 /**
  * struct ade7854_state - device instance specific data
- * @spi:			actual spi_device
+ * @spi:		actual spi_device
+ * @read_reg		Wrapper function for I2C and SPI read
+ * @write_reg		Wrapper function for I2C and SPI write
  * @indio_dev:		industrial I/O device structure
  * @buf_lock:		mutex to protect tx and rx
  * @tx:			transmit buffer
@@ -154,14 +156,10 @@
 struct ade7854_state {
 	struct spi_device *spi;
 	struct i2c_client *i2c;
-	int (*read_reg_8)(struct device *dev, u16 reg_address, u8 *val);
-	int (*read_reg_16)(struct device *dev, u16 reg_address, u16 *val);
-	int (*read_reg_24)(struct device *dev, u16 reg_address, u32 *val);
-	int (*read_reg_32)(struct device *dev, u16 reg_address, u32 *val);
-	int (*write_reg_8)(struct device *dev, u16 reg_address, u8 val);
-	int (*write_reg_16)(struct device *dev, u16 reg_address, u16 val);
-	int (*write_reg_24)(struct device *dev, u16 reg_address, u32 val);
-	int (*write_reg_32)(struct device *dev, u16 reg_address, u32 val);
+	int (*read_reg)(struct device *dev, u16 reg_address, u32 *val,
+			int bits);
+	int (*write_reg)(struct device *dev, u16 reg_address, u32 val,
+			 int bits);
 	int irq;
 	struct mutex buf_lock;
 	u8 tx[ADE7854_MAX_TX] ____cacheline_aligned;
diff --git a/drivers/staging/iio/resolver/Kconfig b/drivers/staging/iio/resolver/Kconfig
index 1c7e2860d6b7..6a469ee6101f 100644
--- a/drivers/staging/iio/resolver/Kconfig
+++ b/drivers/staging/iio/resolver/Kconfig
@@ -13,18 +13,6 @@ config AD2S90
 	  To compile this driver as a module, choose M here: the
 	  module will be called ad2s90.
 
-config AD2S1200
-	tristate "Analog Devices ad2s1200/ad2s1205 driver"
-	depends on SPI
-	depends on GPIOLIB || COMPILE_TEST
-	help
-	  Say yes here to build support for Analog Devices spi resolver
-	  to digital converters, ad2s1200 and ad2s1205, provides direct access
-	  via sysfs.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ad2s1200.
-
 config AD2S1210
 	tristate "Analog Devices ad2s1210 driver"
 	depends on SPI
diff --git a/drivers/staging/iio/resolver/Makefile b/drivers/staging/iio/resolver/Makefile
index 14375e444ebf..8d901dc7500b 100644
--- a/drivers/staging/iio/resolver/Makefile
+++ b/drivers/staging/iio/resolver/Makefile
@@ -3,5 +3,4 @@
 #
 
 obj-$(CONFIG_AD2S90) += ad2s90.o
-obj-$(CONFIG_AD2S1200) += ad2s1200.o
 obj-$(CONFIG_AD2S1210) += ad2s1210.o
diff --git a/drivers/staging/iio/resolver/ad2s1200.c b/drivers/staging/iio/resolver/ad2s1200.c
deleted file mode 100644
index aa62c64e9bc4..000000000000
--- a/drivers/staging/iio/resolver/ad2s1200.c
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * ad2s1200.c simple support for the ADI Resolver to Digital Converters:
- * AD2S1200/1205
- *
- * Copyright (c) 2010-2010 Analog Devices Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
-#include <linux/types.h>
-#include <linux/mutex.h>
-#include <linux/device.h>
-#include <linux/spi/spi.h>
-#include <linux/slab.h>
-#include <linux/sysfs.h>
-#include <linux/delay.h>
-#include <linux/gpio.h>
-#include <linux/module.h>
-#include <linux/bitops.h>
-
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-
-#define DRV_NAME "ad2s1200"
-
-/* input pin sample and rdvel is controlled by driver */
-#define AD2S1200_PN	2
-
-/* input clock on serial interface */
-#define AD2S1200_HZ	8192000
-/* clock period in nano second */
-#define AD2S1200_TSCLK	(1000000000 / AD2S1200_HZ)
-
-struct ad2s1200_state {
-	struct mutex lock;
-	struct spi_device *sdev;
-	int sample;
-	int rdvel;
-	u8 rx[2] ____cacheline_aligned;
-};
-
-static int ad2s1200_read_raw(struct iio_dev *indio_dev,
-			     struct iio_chan_spec const *chan,
-			     int *val,
-			     int *val2,
-			     long m)
-{
-	int ret = 0;
-	s16 vel;
-	struct ad2s1200_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->lock);
-	gpio_set_value(st->sample, 0);
-	/* delay (6 * AD2S1200_TSCLK + 20) nano seconds */
-	udelay(1);
-	gpio_set_value(st->sample, 1);
-	gpio_set_value(st->rdvel, !!(chan->type == IIO_ANGL));
-	ret = spi_read(st->sdev, st->rx, 2);
-	if (ret < 0) {
-		mutex_unlock(&st->lock);
-		return ret;
-	}
-
-	switch (chan->type) {
-	case IIO_ANGL:
-		*val = (((u16)(st->rx[0])) << 4) | ((st->rx[1] & 0xF0) >> 4);
-		break;
-	case IIO_ANGL_VEL:
-		vel = (((s16)(st->rx[0])) << 4) | ((st->rx[1] & 0xF0) >> 4);
-		vel = sign_extend32(vel, 11);
-		*val = vel;
-		break;
-	default:
-		mutex_unlock(&st->lock);
-		return -EINVAL;
-	}
-	/* delay (2 * AD2S1200_TSCLK + 20) ns for sample pulse */
-	udelay(1);
-	mutex_unlock(&st->lock);
-	return IIO_VAL_INT;
-}
-
-static const struct iio_chan_spec ad2s1200_channels[] = {
-	{
-		.type = IIO_ANGL,
-		.indexed = 1,
-		.channel = 0,
-		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
-	}, {
-		.type = IIO_ANGL_VEL,
-		.indexed = 1,
-		.channel = 0,
-		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
-	}
-};
-
-static const struct iio_info ad2s1200_info = {
-	.read_raw = ad2s1200_read_raw,
-};
-
-static int ad2s1200_probe(struct spi_device *spi)
-{
-	struct ad2s1200_state *st;
-	struct iio_dev *indio_dev;
-	int pn, ret = 0;
-	unsigned short *pins = spi->dev.platform_data;
-
-	for (pn = 0; pn < AD2S1200_PN; pn++) {
-		ret = devm_gpio_request_one(&spi->dev, pins[pn], GPIOF_DIR_OUT,
-					    DRV_NAME);
-		if (ret) {
-			dev_err(&spi->dev, "request gpio pin %d failed\n",
-				pins[pn]);
-			return ret;
-		}
-	}
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev)
-		return -ENOMEM;
-	spi_set_drvdata(spi, indio_dev);
-	st = iio_priv(indio_dev);
-	mutex_init(&st->lock);
-	st->sdev = spi;
-	st->sample = pins[0];
-	st->rdvel = pins[1];
-
-	indio_dev->dev.parent = &spi->dev;
-	indio_dev->info = &ad2s1200_info;
-	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->channels = ad2s1200_channels;
-	indio_dev->num_channels = ARRAY_SIZE(ad2s1200_channels);
-	indio_dev->name = spi_get_device_id(spi)->name;
-
-	ret = devm_iio_device_register(&spi->dev, indio_dev);
-	if (ret)
-		return ret;
-
-	spi->max_speed_hz = AD2S1200_HZ;
-	spi->mode = SPI_MODE_3;
-	spi_setup(spi);
-
-	return 0;
-}
-
-static const struct spi_device_id ad2s1200_id[] = {
-	{ "ad2s1200" },
-	{ "ad2s1205" },
-	{}
-};
-MODULE_DEVICE_TABLE(spi, ad2s1200_id);
-
-static struct spi_driver ad2s1200_driver = {
-	.driver = {
-		.name = DRV_NAME,
-	},
-	.probe = ad2s1200_probe,
-	.id_table = ad2s1200_id,
-};
-module_spi_driver(ad2s1200_driver);
-
-MODULE_AUTHOR("Graff Yang <graff.yang@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices AD2S1200/1205 Resolver to Digital SPI driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/ipx/af_ipx.c b/drivers/staging/ipx/af_ipx.c
index 5703dd176787..208b5c161631 100644
--- a/drivers/staging/ipx/af_ipx.c
+++ b/drivers/staging/ipx/af_ipx.c
@@ -1965,7 +1965,7 @@ static const struct proto_ops ipx_dgram_ops = {
 	.socketpair	= sock_no_socketpair,
 	.accept		= sock_no_accept,
 	.getname	= ipx_getname,
-	.poll		= datagram_poll,
+	.poll_mask	= datagram_poll_mask,
 	.ioctl		= ipx_ioctl,
 #ifdef CONFIG_COMPAT
 	.compat_ioctl	= ipx_compat_ioctl,
diff --git a/drivers/staging/ipx/ipx_proc.c b/drivers/staging/ipx/ipx_proc.c
index b9232e4e2ed4..360f0ad970de 100644
--- a/drivers/staging/ipx/ipx_proc.c
+++ b/drivers/staging/ipx/ipx_proc.c
@@ -244,42 +244,6 @@ static const struct seq_operations ipx_seq_socket_ops = {
 	.show   = ipx_seq_socket_show,
 };
 
-static int ipx_seq_route_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &ipx_seq_route_ops);
-}
-
-static int ipx_seq_interface_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &ipx_seq_interface_ops);
-}
-
-static int ipx_seq_socket_open(struct inode *inode, struct file *file)
-{
-	return seq_open(file, &ipx_seq_socket_ops);
-}
-
-static const struct file_operations ipx_seq_interface_fops = {
-	.open           = ipx_seq_interface_open,
-	.read           = seq_read,
-	.llseek         = seq_lseek,
-	.release        = seq_release,
-};
-
-static const struct file_operations ipx_seq_route_fops = {
-	.open           = ipx_seq_route_open,
-	.read           = seq_read,
-	.llseek         = seq_lseek,
-	.release        = seq_release,
-};
-
-static const struct file_operations ipx_seq_socket_fops = {
-	.open           = ipx_seq_socket_open,
-	.read           = seq_read,
-	.llseek         = seq_lseek,
-	.release        = seq_release,
-};
-
 static struct proc_dir_entry *ipx_proc_dir;
 
 int __init ipx_proc_init(void)
@@ -291,16 +255,17 @@ int __init ipx_proc_init(void)
 
 	if (!ipx_proc_dir)
 		goto out;
-	p = proc_create("interface", S_IRUGO,
-			ipx_proc_dir, &ipx_seq_interface_fops);
+	p = proc_create_seq("interface", S_IRUGO, ipx_proc_dir,
+			&ipx_seq_interface_ops);
 	if (!p)
 		goto out_interface;
 
-	p = proc_create("route", S_IRUGO, ipx_proc_dir, &ipx_seq_route_fops);
+	p = proc_create_seq("route", S_IRUGO, ipx_proc_dir, &ipx_seq_route_ops);
 	if (!p)
 		goto out_route;
 
-	p = proc_create("socket", S_IRUGO, ipx_proc_dir, &ipx_seq_socket_fops);
+	p = proc_create_seq("socket", S_IRUGO, ipx_proc_dir,
+			&ipx_seq_socket_ops);
 	if (!p)
 		goto out_socket;
 
diff --git a/drivers/staging/ks7010/eap_packet.h b/drivers/staging/ks7010/eap_packet.h
index 58c2a3dafca2..1eee774319ad 100644
--- a/drivers/staging/ks7010/eap_packet.h
+++ b/drivers/staging/ks7010/eap_packet.h
@@ -6,8 +6,6 @@
 #include <linux/bitops.h>
 #include <uapi/linux/if_ether.h>
 
-#define ETHER_HDR_SIZE 20
-
 struct ether_hdr {
 	unsigned char h_dest[ETH_ALEN];	/* destination eth addr */
 	unsigned char h_source[ETH_ALEN];	/* source ether addr    */
@@ -19,6 +17,8 @@ struct ether_hdr {
 	/* followed by length octets of data */
 } __packed;
 
+#define ETHER_HDR_SIZE sizeof(struct ether_hdr)
+
 struct ieee802_1x_hdr {
 	unsigned char version;
 	unsigned char type;
@@ -26,60 +26,14 @@ struct ieee802_1x_hdr {
 	/* followed by length octets of data */
 } __packed;
 
-#define EAPOL_VERSION 2
-
-enum { IEEE802_1X_TYPE_EAP_PACKET = 0,
+enum {
+	IEEE802_1X_TYPE_EAP_PACKET = 0,
 	IEEE802_1X_TYPE_EAPOL_START = 1,
 	IEEE802_1X_TYPE_EAPOL_LOGOFF = 2,
 	IEEE802_1X_TYPE_EAPOL_KEY = 3,
 	IEEE802_1X_TYPE_EAPOL_ENCAPSULATED_ASF_ALERT = 4
 };
 
-enum { EAPOL_KEY_TYPE_RC4 = 1, EAPOL_KEY_TYPE_RSN = 2,
-	EAPOL_KEY_TYPE_WPA = 254
-};
-
-#define IEEE8021X_REPLAY_COUNTER_LEN 8
-#define IEEE8021X_KEY_SIGN_LEN 16
-#define IEEE8021X_KEY_IV_LEN 16
-
-#define IEEE8021X_KEY_INDEX_FLAG 0x80
-#define IEEE8021X_KEY_INDEX_MASK 0x03
-
-struct ieee802_1x_eapol_key {
-	unsigned char type;
-	unsigned short key_length;
-	/*
-	 * does not repeat within the life of the keying material used to
-	 * encrypt the Key field; 64-bit NTP timestamp MAY be used here
-	 */
-	unsigned char replay_counter[IEEE8021X_REPLAY_COUNTER_LEN];
-	unsigned char key_iv[IEEE8021X_KEY_IV_LEN]; /* cryptographically random
-						     * number
-						     */
-	unsigned char key_index;	/*
-					 * key flag in the most significant bit:
-					 * 0 = broadcast (default key),
-					 * 1 = unicast (key mapping key);
-					 * key index is in the 7 least
-					 * significant bits
-					 */
-	/*
-	 * HMAC-MD5 message integrity check computed with MS-MPPE-Send-Key as
-	 * the key
-	 */
-	unsigned char key_signature[IEEE8021X_KEY_SIGN_LEN];
-
-	/*
-	 * followed by key: if packet body length = 44 + key length, then the
-	 * key field (of key_length bytes) contains the key in encrypted form;
-	 * if packet body length = 44, key field is absent and key_length
-	 * represents the number of least significant octets from
-	 * MS-MPPE-Send-Key attribute to be used as the keying material;
-	 * RC4 key used in encryption = Key-IV + MS-MPPE-Recv-Key
-	 */
-} __packed;
-
 #define WPA_NONCE_LEN 32
 #define WPA_REPLAY_COUNTER_LEN 8
 
@@ -113,26 +67,4 @@ struct wpa_eapol_key {
 #define WPA_KEY_INFO_REQUEST BIT(11)
 #define WPA_KEY_INFO_ENCR_KEY_DATA BIT(12)	/* IEEE 802.11i/RSN only */
 
-#define WPA_CAPABILITY_PREAUTH BIT(0)
-
-#define GENERIC_INFO_ELEM 0xdd
-#define RSN_INFO_ELEM 0x30
-
-enum {
-	REASON_UNSPECIFIED = 1,
-	REASON_DEAUTH_LEAVING = 3,
-	REASON_INVALID_IE = 13,
-	REASON_MICHAEL_MIC_FAILURE = 14,
-	REASON_4WAY_HANDSHAKE_TIMEOUT = 15,
-	REASON_GROUP_KEY_UPDATE_TIMEOUT = 16,
-	REASON_IE_IN_4WAY_DIFFERS = 17,
-	REASON_GROUP_CIPHER_NOT_VALID = 18,
-	REASON_PAIRWISE_CIPHER_NOT_VALID = 19,
-	REASON_AKMP_NOT_VALID = 20,
-	REASON_UNSUPPORTED_RSN_IE_VERSION = 21,
-	REASON_INVALID_RSN_IE_CAPAB = 22,
-	REASON_IEEE_802_1X_AUTH_FAILED = 23,
-	REASON_CIPHER_SUITE_REJECTED = 24
-};
-
 #endif /* EAP_PACKET_H */
diff --git a/drivers/staging/ks7010/ks7010_sdio.c b/drivers/staging/ks7010/ks7010_sdio.c
index b8f55a11ee1c..74551eb717fc 100644
--- a/drivers/staging/ks7010/ks7010_sdio.c
+++ b/drivers/staging/ks7010/ks7010_sdio.c
@@ -1,76 +1,126 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *   Driver for KeyStream, KS7010 based SDIO cards.
  *
  *   Copyright (C) 2006-2008 KeyStream Corp.
  *   Copyright (C) 2009 Renesas Technology Corp.
  *   Copyright (C) 2016 Sang Engineering, Wolfram Sang
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License version 2 as
- *   published by the Free Software Foundation.
  */
 
+#include <linux/atomic.h>
 #include <linux/firmware.h>
+#include <linux/jiffies.h>
 #include <linux/mmc/card.h>
 #include <linux/mmc/sdio_func.h>
+#include <linux/module.h>
 #include <linux/workqueue.h>
-#include <linux/atomic.h>
-
 #include "ks_wlan.h"
-#include "ks_wlan_ioctl.h"
 #include "ks_hostif.h"
-#include "ks7010_sdio.h"
 
-#define KS7010_FUNC_NUM 1
-#define KS7010_IO_BLOCK_SIZE 512
-#define KS7010_MAX_CLOCK 25000000
+#define ROM_FILE "ks7010sd.rom"
 
-static const struct sdio_device_id ks7010_sdio_ids[] = {
-	{SDIO_DEVICE(SDIO_VENDOR_ID_KS_CODE_A, SDIO_DEVICE_ID_KS_7010)},
-	{SDIO_DEVICE(SDIO_VENDOR_ID_KS_CODE_B, SDIO_DEVICE_ID_KS_7010)},
-	{ /* all zero */ }
+/*  SDIO KeyStream vendor and device */
+#define SDIO_VENDOR_ID_KS_CODE_A	0x005b
+#define SDIO_VENDOR_ID_KS_CODE_B	0x0023
+
+/* Older sources suggest earlier versions were named 7910 or 79xx */
+#define SDIO_DEVICE_ID_KS_7010		0x7910
+
+/* Read/Write Status Register */
+#define READ_STATUS_REG		0x000000
+#define WRITE_STATUS_REG	0x00000C
+enum reg_status_type {
+	REG_STATUS_BUSY,
+	REG_STATUS_IDLE
 };
-MODULE_DEVICE_TABLE(sdio, ks7010_sdio_ids);
 
-static inline void inc_txqhead(struct ks_wlan_private *priv)
-{
-	priv->tx_dev.qhead = (priv->tx_dev.qhead + 1) % TX_DEVICE_BUFF_SIZE;
-}
+/* Read Index Register */
+#define READ_INDEX_REG		0x000004
 
-static inline void inc_txqtail(struct ks_wlan_private *priv)
-{
-	priv->tx_dev.qtail = (priv->tx_dev.qtail + 1) % TX_DEVICE_BUFF_SIZE;
-}
+/* Read Data Size Register */
+#define READ_DATA_SIZE_REG	0x000008
 
-static inline unsigned int cnt_txqbody(struct ks_wlan_private *priv)
-{
-	unsigned int tx_cnt = priv->tx_dev.qtail - priv->tx_dev.qhead;
+/* Write Index Register */
+#define WRITE_INDEX_REG		0x000010
 
-	return (tx_cnt + TX_DEVICE_BUFF_SIZE) % TX_DEVICE_BUFF_SIZE;
-}
+/*
+ * Write Status/Read Data Size Register
+ * for network packet (less than 2048 bytes data)
+ */
+#define WSTATUS_RSIZE_REG	0x000014
+
+/* Write Status Register value */
+#define WSTATUS_MASK		0x80
+
+/* Read Data Size Register value [10:4] */
+#define RSIZE_MASK		0x7F
+
+/* ARM to SD interrupt Enable */
+#define INT_ENABLE_REG		0x000020
+/* ARM to SD interrupt Pending */
+#define INT_PENDING_REG		0x000024
+
+#define INT_GCR_B              BIT(7)
+#define INT_GCR_A              BIT(6)
+#define INT_WRITE_STATUS       BIT(5)
+#define INT_WRITE_INDEX        BIT(4)
+#define INT_WRITE_SIZE         BIT(3)
+#define INT_READ_STATUS        BIT(2)
+#define INT_READ_INDEX         BIT(1)
+#define INT_READ_SIZE          BIT(0)
+
+/* General Communication Register A */
+#define GCR_A_REG		0x000028
+enum gen_com_reg_a {
+	GCR_A_INIT,
+	GCR_A_REMAP,
+	GCR_A_RUN
+};
 
-static inline void inc_rxqhead(struct ks_wlan_private *priv)
-{
-	priv->rx_dev.qhead = (priv->rx_dev.qhead + 1) % RX_DEVICE_BUFF_SIZE;
-}
+/* General Communication Register B */
+#define GCR_B_REG		0x00002C
+enum gen_com_reg_b {
+	GCR_B_ACTIVE,
+	GCR_B_DOZE
+};
 
-static inline void inc_rxqtail(struct ks_wlan_private *priv)
-{
-	priv->rx_dev.qtail = (priv->rx_dev.qtail + 1) % RX_DEVICE_BUFF_SIZE;
-}
+/* Wakeup Register */
+#define WAKEUP_REG		0x008018
+#define WAKEUP_REQ		0x5a
+
+/* AHB Data Window  0x010000-0x01FFFF */
+#define DATA_WINDOW		0x010000
+#define WINDOW_SIZE		(64 * 1024)
 
-static inline unsigned int cnt_rxqbody(struct ks_wlan_private *priv)
+#define KS7010_IRAM_ADDRESS	0x06000000
+
+#define KS7010_IO_BLOCK_SIZE 512
+
+/**
+ * struct ks_sdio_card - SDIO device data.
+ *
+ * Structure is used as the &struct sdio_func private data.
+ *
+ * @func: Pointer to the SDIO function device.
+ * @priv: Pointer to the &struct net_device private data.
+ */
+struct ks_sdio_card {
+	struct sdio_func *func;
+	struct ks_wlan_private *priv;
+};
+
+static struct sdio_func *ks7010_to_func(struct ks_wlan_private *priv)
 {
-	unsigned int rx_cnt = priv->rx_dev.qtail - priv->rx_dev.qhead;
+	struct ks_sdio_card *ks_sdio = priv->if_hw;
 
-	return (rx_cnt + RX_DEVICE_BUFF_SIZE) % RX_DEVICE_BUFF_SIZE;
+	return ks_sdio->func;
 }
 
 /* Read single byte from device address into byte (CMD52) */
-static int ks7010_sdio_readb(struct ks_wlan_private *priv, unsigned int address,
-			     unsigned char *byte)
+static int ks7010_sdio_readb(struct ks_wlan_private *priv,
+			     u32 address, u8 *byte)
 {
-	struct sdio_func *func = priv->ks_sdio_card->func;
+	struct sdio_func *func = ks7010_to_func(priv);
 	int ret;
 
 	*byte = sdio_readb(func, address, &ret);
@@ -79,19 +129,19 @@ static int ks7010_sdio_readb(struct ks_wlan_private *priv, unsigned int address,
 }
 
 /* Read length bytes from device address into buffer (CMD53) */
-static int ks7010_sdio_read(struct ks_wlan_private *priv, unsigned int address,
-			    unsigned char *buffer, int length)
+static int ks7010_sdio_read(struct ks_wlan_private *priv, u32 address,
+			    u8 *buffer, unsigned int length)
 {
-	struct sdio_func *func = priv->ks_sdio_card->func;
+	struct sdio_func *func = ks7010_to_func(priv);
 
 	return sdio_memcpy_fromio(func, buffer, address, length);
 }
 
 /* Write single byte to device address (CMD52) */
 static int ks7010_sdio_writeb(struct ks_wlan_private *priv,
-			      unsigned int address, unsigned char byte)
+			      u32 address, u8 byte)
 {
-	struct sdio_func *func = priv->ks_sdio_card->func;
+	struct sdio_func *func = ks7010_to_func(priv);
 	int ret;
 
 	sdio_writeb(func, byte, address, &ret);
@@ -100,10 +150,10 @@ static int ks7010_sdio_writeb(struct ks_wlan_private *priv,
 }
 
 /* Write length bytes to device address from buffer (CMD53) */
-static int ks7010_sdio_write(struct ks_wlan_private *priv, unsigned int address,
-			     unsigned char *buffer, int length)
+static int ks7010_sdio_write(struct ks_wlan_private *priv, u32 address,
+			     u8 *buffer, unsigned int length)
 {
-	struct sdio_func *func = priv->ks_sdio_card->func;
+	struct sdio_func *func = ks7010_to_func(priv);
 
 	return sdio_memcpy_toio(func, address, buffer, length);
 }
@@ -116,9 +166,9 @@ static void ks_wlan_hw_sleep_doze_request(struct ks_wlan_private *priv)
 	atomic_set(&priv->sleepstatus.doze_request, 0);
 
 	if (atomic_read(&priv->sleepstatus.status) == 0) {
-		ret = ks7010_sdio_writeb(priv, GCR_B, GCR_B_DOZE);
+		ret = ks7010_sdio_writeb(priv, GCR_B_REG, GCR_B_DOZE);
 		if (ret) {
-			netdev_err(priv->net_dev, " error : GCR_B\n");
+			netdev_err(priv->net_dev, "write GCR_B_REG\n");
 			goto set_sleep_mode;
 		}
 		atomic_set(&priv->sleepstatus.status, 1);
@@ -137,9 +187,9 @@ static void ks_wlan_hw_sleep_wakeup_request(struct ks_wlan_private *priv)
 	atomic_set(&priv->sleepstatus.wakeup_request, 0);
 
 	if (atomic_read(&priv->sleepstatus.status) == 1) {
-		ret = ks7010_sdio_writeb(priv, WAKEUP, WAKEUP_REQ);
+		ret = ks7010_sdio_writeb(priv, WAKEUP_REG, WAKEUP_REQ);
 		if (ret) {
-			netdev_err(priv->net_dev, " error : WAKEUP\n");
+			netdev_err(priv->net_dev, "write WAKEUP_REG\n");
 			goto set_sleep_mode;
 		}
 		atomic_set(&priv->sleepstatus.status, 0);
@@ -156,9 +206,9 @@ void ks_wlan_hw_wakeup_request(struct ks_wlan_private *priv)
 	int ret;
 
 	if (atomic_read(&priv->psstatus.status) == PS_SNOOZE) {
-		ret = ks7010_sdio_writeb(priv, WAKEUP, WAKEUP_REQ);
+		ret = ks7010_sdio_writeb(priv, WAKEUP_REG, WAKEUP_REQ);
 		if (ret)
-			netdev_err(priv->net_dev, " error : WAKEUP\n");
+			netdev_err(priv->net_dev, "write WAKEUP_REG\n");
 
 		priv->last_wakeup = jiffies;
 		++priv->wakeup_count;
@@ -167,7 +217,7 @@ void ks_wlan_hw_wakeup_request(struct ks_wlan_private *priv)
 
 static void _ks_wlan_hw_power_save(struct ks_wlan_private *priv)
 {
-	unsigned char byte;
+	u8 byte;
 	int ret;
 
 	if (priv->reg.power_mgmt == POWER_MGMT_ACTIVE)
@@ -185,30 +235,35 @@ static void _ks_wlan_hw_power_save(struct ks_wlan_private *priv)
 	if (atomic_read(&priv->psstatus.status) == PS_SNOOZE)
 		return;
 
-	netdev_dbg(priv->net_dev, "\npsstatus.status=%d\npsstatus.confirm_wait=%d\npsstatus.snooze_guard=%d\ncnt_txqbody=%d\n",
+	netdev_dbg(priv->net_dev,
+		   "STATUS:\n"
+		   "- psstatus.status = %d\n"
+		   "- psstatus.confirm_wait = %d\n"
+		   "- psstatus.snooze_guard = %d\n"
+		   "- txq_count = %d\n",
 		   atomic_read(&priv->psstatus.status),
 		   atomic_read(&priv->psstatus.confirm_wait),
 		   atomic_read(&priv->psstatus.snooze_guard),
-		   cnt_txqbody(priv));
+		   txq_count(priv));
 
 	if (atomic_read(&priv->psstatus.confirm_wait) ||
 	    atomic_read(&priv->psstatus.snooze_guard) ||
-	    cnt_txqbody(priv)) {
+	    txq_has_space(priv)) {
 		queue_delayed_work(priv->wq, &priv->rw_dwork, 0);
 		return;
 	}
 
-	ret = ks7010_sdio_readb(priv, INT_PENDING, &byte);
+	ret = ks7010_sdio_readb(priv, INT_PENDING_REG, &byte);
 	if (ret) {
-		netdev_err(priv->net_dev, " error : INT_PENDING\n");
+		netdev_err(priv->net_dev, "read INT_PENDING_REG\n");
 		goto queue_delayed_work;
 	}
 	if (byte)
 		goto queue_delayed_work;
 
-	ret = ks7010_sdio_writeb(priv, GCR_B, GCR_B_DOZE);
+	ret = ks7010_sdio_writeb(priv, GCR_B_REG, GCR_B_DOZE);
 	if (ret) {
-		netdev_err(priv->net_dev, " error : GCR_B\n");
+		netdev_err(priv->net_dev, "write GCR_B_REG\n");
 		goto queue_delayed_work;
 	}
 	atomic_set(&priv->psstatus.status, PS_SNOOZE);
@@ -239,7 +294,7 @@ static int enqueue_txdev(struct ks_wlan_private *priv, unsigned char *p,
 		goto err_complete;
 	}
 
-	if ((TX_DEVICE_BUFF_SIZE - 1) <= cnt_txqbody(priv)) {
+	if ((TX_DEVICE_BUFF_SIZE - 1) <= txq_count(priv)) {
 		netdev_err(priv->net_dev, "tx buffer overflow\n");
 		ret = -EOVERFLOW;
 		goto err_complete;
@@ -263,7 +318,7 @@ static int enqueue_txdev(struct ks_wlan_private *priv, unsigned char *p,
 }
 
 /* write data */
-static int write_to_device(struct ks_wlan_private *priv, unsigned char *buffer,
+static int write_to_device(struct ks_wlan_private *priv, u8 *buffer,
 			   unsigned long size)
 {
 	struct hostif_hdr *hdr;
@@ -279,13 +334,13 @@ static int write_to_device(struct ks_wlan_private *priv, unsigned char *buffer,
 
 	ret = ks7010_sdio_write(priv, DATA_WINDOW, buffer, size);
 	if (ret) {
-		netdev_err(priv->net_dev, " write error : retval=%d\n", ret);
+		netdev_err(priv->net_dev, "write DATA_WINDOW\n");
 		return ret;
 	}
 
-	ret = ks7010_sdio_writeb(priv, WRITE_STATUS, REG_STATUS_BUSY);
+	ret = ks7010_sdio_writeb(priv, WRITE_STATUS_REG, REG_STATUS_BUSY);
 	if (ret) {
-		netdev_err(priv->net_dev, " error : WRITE_STATUS\n");
+		netdev_err(priv->net_dev, "write WRITE_STATUS_REG\n");
 		return ret;
 	}
 
@@ -297,7 +352,7 @@ static void tx_device_task(struct ks_wlan_private *priv)
 	struct tx_device_buffer *sp;
 	int ret;
 
-	if (cnt_txqbody(priv) <= 0 ||
+	if (!txq_has_space(priv) ||
 	    atomic_read(&priv->psstatus.status) == PS_SNOOZE)
 		return;
 
@@ -305,7 +360,8 @@ static void tx_device_task(struct ks_wlan_private *priv)
 	if (priv->dev_state >= DEVICE_STATE_BOOT) {
 		ret = write_to_device(priv, sp->sendp, sp->size);
 		if (ret) {
-			netdev_err(priv->net_dev, "write_to_device error !!(%d)\n", ret);
+			netdev_err(priv->net_dev,
+				   "write_to_device error !!(%d)\n", ret);
 			queue_delayed_work(priv->wq, &priv->rw_dwork, 1);
 			return;
 		}
@@ -315,7 +371,7 @@ static void tx_device_task(struct ks_wlan_private *priv)
 		(*sp->complete_handler)(priv, sp->skb);
 	inc_txqhead(priv);
 
-	if (cnt_txqbody(priv) > 0)
+	if (txq_has_space(priv))
 		queue_delayed_work(priv->wq, &priv->rw_dwork, 0);
 }
 
@@ -343,7 +399,7 @@ int ks_wlan_hw_tx(struct ks_wlan_private *priv, void *p, unsigned long size,
 	result = enqueue_txdev(priv, p, size, complete_handler, skb);
 	spin_unlock(&priv->tx_dev.tx_dev_lock);
 
-	if (cnt_txqbody(priv) > 0)
+	if (txq_has_space(priv))
 		queue_delayed_work(priv->wq, &priv->rw_dwork, 0);
 
 	return result;
@@ -354,25 +410,25 @@ static void rx_event_task(unsigned long dev)
 	struct ks_wlan_private *priv = (struct ks_wlan_private *)dev;
 	struct rx_device_buffer *rp;
 
-	if (cnt_rxqbody(priv) > 0 && priv->dev_state >= DEVICE_STATE_BOOT) {
+	if (rxq_has_space(priv) && priv->dev_state >= DEVICE_STATE_BOOT) {
 		rp = &priv->rx_dev.rx_dev_buff[priv->rx_dev.qhead];
 		hostif_receive(priv, rp->data, rp->size);
 		inc_rxqhead(priv);
 
-		if (cnt_rxqbody(priv) > 0)
+		if (rxq_has_space(priv))
 			tasklet_schedule(&priv->rx_bh_task);
 	}
 }
 
-static void ks_wlan_hw_rx(struct ks_wlan_private *priv, uint16_t size)
+static void ks_wlan_hw_rx(struct ks_wlan_private *priv, size_t size)
 {
 	int ret;
 	struct rx_device_buffer *rx_buffer;
 	struct hostif_hdr *hdr;
-	unsigned short event = 0;
+	u16 event = 0;
 
 	/* receive data */
-	if (cnt_rxqbody(priv) >= (RX_DEVICE_BUFF_SIZE - 1)) {
+	if (rxq_count(priv) >= (RX_DEVICE_BUFF_SIZE - 1)) {
 		netdev_err(priv->net_dev, "rx buffer overflow\n");
 		return;
 	}
@@ -390,9 +446,9 @@ static void ks_wlan_hw_rx(struct ks_wlan_private *priv, uint16_t size)
 				     DUMP_PREFIX_OFFSET,
 				     rx_buffer->data, 32);
 #endif
-		ret = ks7010_sdio_writeb(priv, READ_STATUS, REG_STATUS_IDLE);
+		ret = ks7010_sdio_writeb(priv, READ_STATUS_REG, REG_STATUS_IDLE);
 		if (ret)
-			netdev_err(priv->net_dev, " error : READ_STATUS\n");
+			netdev_err(priv->net_dev, "write READ_STATUS_REG\n");
 
 		/* length check fail */
 		return;
@@ -403,15 +459,13 @@ static void ks_wlan_hw_rx(struct ks_wlan_private *priv, uint16_t size)
 	event = le16_to_cpu(hdr->event);
 	inc_rxqtail(priv);
 
-	ret = ks7010_sdio_writeb(priv, READ_STATUS, REG_STATUS_IDLE);
+	ret = ks7010_sdio_writeb(priv, READ_STATUS_REG, REG_STATUS_IDLE);
 	if (ret)
-		netdev_err(priv->net_dev, " error : READ_STATUS\n");
+		netdev_err(priv->net_dev, "write READ_STATUS_REG\n");
 
-	if (atomic_read(&priv->psstatus.confirm_wait)) {
-		if (IS_HIF_CONF(event)) {
-			netdev_dbg(priv->net_dev, "IS_HIF_CONF true !!\n");
-			atomic_dec(&priv->psstatus.confirm_wait);
-		}
+	if (atomic_read(&priv->psstatus.confirm_wait) && is_hif_conf(event)) {
+		netdev_dbg(priv->net_dev, "IS_HIF_CONF true !!\n");
+		atomic_dec(&priv->psstatus.confirm_wait);
 	}
 
 	tasklet_schedule(&priv->rx_bh_task);
@@ -419,34 +473,33 @@ static void ks_wlan_hw_rx(struct ks_wlan_private *priv, uint16_t size)
 
 static void ks7010_rw_function(struct work_struct *work)
 {
-	struct ks_wlan_private *priv;
-	unsigned char byte;
+	struct ks_wlan_private *priv = container_of(work,
+						    struct ks_wlan_private,
+						    rw_dwork.work);
+	struct sdio_func *func = ks7010_to_func(priv);
+	u8 byte;
 	int ret;
 
-	priv = container_of(work, struct ks_wlan_private, rw_dwork.work);
-
 	/* wait after DOZE */
-	if (time_after(priv->last_doze + ((30 * HZ) / 1000), jiffies)) {
+	if (time_after(priv->last_doze + msecs_to_jiffies(30), jiffies)) {
 		netdev_dbg(priv->net_dev, "wait after DOZE\n");
 		queue_delayed_work(priv->wq, &priv->rw_dwork, 1);
 		return;
 	}
 
 	/* wait after WAKEUP */
-	while (time_after(priv->last_wakeup + ((30 * HZ) / 1000), jiffies)) {
+	while (time_after(priv->last_wakeup + msecs_to_jiffies(30), jiffies)) {
 		netdev_dbg(priv->net_dev, "wait after WAKEUP\n");
-		dev_info(&priv->ks_sdio_card->func->dev,
-			 "wake: %lu %lu\n",
-			 priv->last_wakeup + (30 * HZ) / 1000,
-				jiffies);
+		dev_info(&func->dev, "wake: %lu %lu\n",
+			 priv->last_wakeup + msecs_to_jiffies(30), jiffies);
 		msleep(30);
 	}
 
-	sdio_claim_host(priv->ks_sdio_card->func);
+	sdio_claim_host(func);
 
 	/* power save wakeup */
 	if (atomic_read(&priv->psstatus.status) == PS_SNOOZE) {
-		if (cnt_txqbody(priv) > 0) {
+		if (txq_has_space(priv)) {
 			ks_wlan_hw_wakeup_request(priv);
 			queue_delayed_work(priv->wq, &priv->rw_dwork, 1);
 		}
@@ -465,15 +518,15 @@ static void ks7010_rw_function(struct work_struct *work)
 	}
 
 	/* read (WriteStatus/ReadDataSize FN1:00_0014) */
-	ret = ks7010_sdio_readb(priv, WSTATUS_RSIZE, &byte);
+	ret = ks7010_sdio_readb(priv, WSTATUS_RSIZE_REG, &byte);
 	if (ret) {
-		netdev_err(priv->net_dev, " error : WSTATUS_RSIZE psstatus=%d\n",
+		netdev_err(priv->net_dev, "read WSTATUS_RSIZE_REG psstatus=%d\n",
 			   atomic_read(&priv->psstatus.status));
 		goto release_host;
 	}
 
 	if (byte & RSIZE_MASK) {	/* Read schedule */
-		ks_wlan_hw_rx(priv, (uint16_t)((byte & RSIZE_MASK) << 4));
+		ks_wlan_hw_rx(priv, (size_t)((byte & RSIZE_MASK) << 4));
 	}
 	if ((byte & WSTATUS_MASK))
 		tx_device_task(priv);
@@ -481,7 +534,7 @@ static void ks7010_rw_function(struct work_struct *work)
 	_ks_wlan_hw_power_save(priv);
 
 release_host:
-	sdio_release_host(priv->ks_sdio_card->func);
+	sdio_release_host(func);
 }
 
 static void ks_sdio_interrupt(struct sdio_func *func)
@@ -489,7 +542,7 @@ static void ks_sdio_interrupt(struct sdio_func *func)
 	int ret;
 	struct ks_sdio_card *card;
 	struct ks_wlan_private *priv;
-	unsigned char status, rsize, byte;
+	u8 status, rsize, byte;
 
 	card = sdio_get_drvdata(func);
 	priv = card->priv;
@@ -497,9 +550,9 @@ static void ks_sdio_interrupt(struct sdio_func *func)
 	if (priv->dev_state < DEVICE_STATE_BOOT)
 		goto queue_delayed_work;
 
-	ret = ks7010_sdio_readb(priv, INT_PENDING, &status);
+	ret = ks7010_sdio_readb(priv, INT_PENDING_REG, &status);
 	if (ret) {
-		netdev_err(priv->net_dev, "error : INT_PENDING\n");
+		netdev_err(priv->net_dev, "read INT_PENDING_REG\n");
 		goto queue_delayed_work;
 	}
 
@@ -510,9 +563,9 @@ static void ks_sdio_interrupt(struct sdio_func *func)
 	/* bit2 -> Read Status Busy  */
 	if (status & INT_GCR_B ||
 	    atomic_read(&priv->psstatus.status) == PS_SNOOZE) {
-		ret = ks7010_sdio_readb(priv, GCR_B, &byte);
+		ret = ks7010_sdio_readb(priv, GCR_B_REG, &byte);
 		if (ret) {
-			netdev_err(priv->net_dev, " error : GCR_B\n");
+			netdev_err(priv->net_dev, "read GCR_B_REG\n");
 			goto queue_delayed_work;
 		}
 		if (byte == GCR_B_ACTIVE) {
@@ -526,20 +579,21 @@ static void ks_sdio_interrupt(struct sdio_func *func)
 
 	do {
 		/* read (WriteStatus/ReadDataSize FN1:00_0014) */
-		ret = ks7010_sdio_readb(priv, WSTATUS_RSIZE, &byte);
+		ret = ks7010_sdio_readb(priv, WSTATUS_RSIZE_REG, &byte);
 		if (ret) {
-			netdev_err(priv->net_dev, " error : WSTATUS_RSIZE\n");
+			netdev_err(priv->net_dev, "read WSTATUS_RSIZE_REG\n");
 			goto queue_delayed_work;
 		}
 		rsize = byte & RSIZE_MASK;
 		if (rsize != 0)		/* Read schedule */
-			ks_wlan_hw_rx(priv, (uint16_t)(rsize << 4));
+			ks_wlan_hw_rx(priv, (size_t)(rsize << 4));
 
 		if (byte & WSTATUS_MASK) {
 			if (atomic_read(&priv->psstatus.status) == PS_SNOOZE) {
-				if (cnt_txqbody(priv)) {
+				if (txq_has_space(priv)) {
 					ks_wlan_hw_wakeup_request(priv);
-					queue_delayed_work(priv->wq, &priv->rw_dwork, 1);
+					queue_delayed_work(priv->wq,
+							   &priv->rw_dwork, 1);
 					return;
 				}
 			} else {
@@ -573,7 +627,7 @@ static void trx_device_exit(struct ks_wlan_private *priv)
 	struct tx_device_buffer *sp;
 
 	/* tx buffer clear */
-	while (cnt_txqbody(priv) > 0) {
+	while (txq_has_space(priv)) {
 		sp = &priv->tx_dev.tx_dev_buff[priv->tx_dev.qhead];
 		kfree(sp->sendp);
 		if (sp->complete_handler)	/* TX Complete */
@@ -589,16 +643,15 @@ static int ks7010_sdio_update_index(struct ks_wlan_private *priv, u32 index)
 	int ret;
 	unsigned char *data_buf;
 
-	data_buf = kmalloc(sizeof(u32), GFP_KERNEL);
+	data_buf = kmemdup(&index, sizeof(u32), GFP_KERNEL);
 	if (!data_buf)
 		return -ENOMEM;
 
-	memcpy(data_buf, &index, sizeof(index));
-	ret = ks7010_sdio_write(priv, WRITE_INDEX, data_buf, sizeof(index));
+	ret = ks7010_sdio_write(priv, WRITE_INDEX_REG, data_buf, sizeof(index));
 	if (ret)
 		goto err_free_data_buf;
 
-	ret = ks7010_sdio_write(priv, READ_INDEX, data_buf, sizeof(index));
+	ret = ks7010_sdio_write(priv, READ_INDEX_REG, data_buf, sizeof(index));
 	if (ret)
 		goto err_free_data_buf;
 
@@ -612,10 +665,10 @@ static int ks7010_sdio_update_index(struct ks_wlan_private *priv, u32 index)
 
 #define ROM_BUFF_SIZE (64 * 1024)
 static int ks7010_sdio_data_compare(struct ks_wlan_private *priv, u32 address,
-				    unsigned char *data, unsigned int size)
+				    u8 *data, unsigned int size)
 {
 	int ret;
-	unsigned char *read_buf;
+	u8 *read_buf;
 
 	read_buf = kmalloc(ROM_BUFF_SIZE, GFP_KERNEL);
 	if (!read_buf)
@@ -639,37 +692,22 @@ static int ks7010_sdio_data_compare(struct ks_wlan_private *priv, u32 address,
 	return ret;
 }
 
-static int ks7010_upload_firmware(struct ks_sdio_card *card)
+static int ks7010_copy_firmware(struct ks_wlan_private *priv,
+				const struct firmware *fw_entry)
 {
-	struct ks_wlan_private *priv = card->priv;
-	unsigned int size, offset, n = 0;
-	unsigned char *rom_buf;
-	unsigned char byte = 0;
-	int ret;
 	unsigned int length;
-	const struct firmware *fw_entry = NULL;
+	unsigned int size;
+	unsigned int offset;
+	unsigned int n = 0;
+	u8 *rom_buf;
+	int ret;
 
 	rom_buf = kmalloc(ROM_BUFF_SIZE, GFP_KERNEL);
 	if (!rom_buf)
 		return -ENOMEM;
 
-	sdio_claim_host(card->func);
-
-	/* Firmware running ? */
-	ret = ks7010_sdio_readb(priv, GCR_A, &byte);
-	if (byte == GCR_A_RUN) {
-		netdev_dbg(priv->net_dev, "MAC firmware running ...\n");
-		goto release_host_and_free;
-	}
-
-	ret = request_firmware(&fw_entry, ROM_FILE,
-			       &priv->ks_sdio_card->func->dev);
-	if (ret)
-		goto release_host_and_free;
-
 	length = fw_entry->size;
 
-	n = 0;
 	do {
 		if (length >= ROM_BUFF_SIZE) {
 			size = ROM_BUFF_SIZE;
@@ -680,33 +718,69 @@ static int ks7010_upload_firmware(struct ks_sdio_card *card)
 		}
 		if (size == 0)
 			break;
+
 		memcpy(rom_buf, fw_entry->data + n, size);
 
 		offset = n;
-		ret = ks7010_sdio_update_index(priv, KS7010_IRAM_ADDRESS + offset);
+		ret = ks7010_sdio_update_index(priv,
+					       KS7010_IRAM_ADDRESS + offset);
 		if (ret)
-			goto release_firmware;
+			goto free_rom_buf;
 
 		ret = ks7010_sdio_write(priv, DATA_WINDOW, rom_buf, size);
 		if (ret)
-			goto release_firmware;
+			goto free_rom_buf;
 
-		ret = ks7010_sdio_data_compare(priv, DATA_WINDOW, rom_buf, size);
+		ret = ks7010_sdio_data_compare(priv,
+					       DATA_WINDOW, rom_buf, size);
 		if (ret)
-			goto release_firmware;
+			goto free_rom_buf;
 
 		n += size;
 
 	} while (size);
 
-	ret = ks7010_sdio_writeb(priv, GCR_A, GCR_A_REMAP);
+	ret = ks7010_sdio_writeb(priv, GCR_A_REG, GCR_A_REMAP);
+
+free_rom_buf:
+	kfree(rom_buf);
+	return ret;
+}
+
+static int ks7010_upload_firmware(struct ks_sdio_card *card)
+{
+	struct ks_wlan_private *priv = card->priv;
+	struct sdio_func *func = ks7010_to_func(priv);
+	unsigned int n;
+	u8 byte = 0;
+	int ret;
+	const struct firmware *fw_entry = NULL;
+
+	sdio_claim_host(func);
+
+	/* Firmware running ? */
+	ret = ks7010_sdio_readb(priv, GCR_A_REG, &byte);
+	if (ret)
+		goto release_host;
+	if (byte == GCR_A_RUN) {
+		netdev_dbg(priv->net_dev, "MAC firmware running ...\n");
+		ret = -EBUSY;
+		goto release_host;
+	}
+
+	ret = request_firmware(&fw_entry, ROM_FILE,
+			       &func->dev);
+	if (ret)
+		goto release_host;
+
+	ret = ks7010_copy_firmware(priv, fw_entry);
 	if (ret)
 		goto release_firmware;
 
 	/* Firmware running check */
 	for (n = 0; n < 50; ++n) {
-		mdelay(10);	/* wait_ms(10); */
-		ret = ks7010_sdio_readb(priv, GCR_A, &byte);
+		usleep_range(10000, 11000);	/* wait_ms(10); */
+		ret = ks7010_sdio_readb(priv, GCR_A_REG, &byte);
 		if (ret)
 			goto release_firmware;
 
@@ -723,13 +797,29 @@ static int ks7010_upload_firmware(struct ks_sdio_card *card)
 
  release_firmware:
 	release_firmware(fw_entry);
- release_host_and_free:
-	sdio_release_host(card->func);
-	kfree(rom_buf);
+ release_host:
+	sdio_release_host(func);
 
 	return ret;
 }
 
+static void ks7010_sme_enqueue_events(struct ks_wlan_private *priv)
+{
+	static const u16 init_events[] = {
+		SME_GET_EEPROM_CKSUM, SME_STOP_REQUEST,
+		SME_RTS_THRESHOLD_REQUEST, SME_FRAGMENTATION_THRESHOLD_REQUEST,
+		SME_WEP_INDEX_REQUEST, SME_WEP_KEY1_REQUEST,
+		SME_WEP_KEY2_REQUEST, SME_WEP_KEY3_REQUEST,
+		SME_WEP_KEY4_REQUEST, SME_WEP_FLAG_REQUEST,
+		SME_RSN_ENABLED_REQUEST, SME_MODE_SET_REQUEST,
+		SME_START_REQUEST
+	};
+	int ev;
+
+	for (ev = 0; ev < ARRAY_SIZE(init_events); ev++)
+		hostif_sme_enqueue(priv, init_events[ev]);
+}
+
 static void ks7010_card_init(struct ks_wlan_private *priv)
 {
 	init_completion(&priv->confirm_wait);
@@ -745,24 +835,7 @@ static void ks7010_card_init(struct ks_wlan_private *priv)
 	if (priv->mac_address_valid && priv->version_size != 0)
 		priv->dev_state = DEVICE_STATE_PREINIT;
 
-	hostif_sme_enqueue(priv, SME_GET_EEPROM_CKSUM);
-
-	/* load initial wireless parameter */
-	hostif_sme_enqueue(priv, SME_STOP_REQUEST);
-
-	hostif_sme_enqueue(priv, SME_RTS_THRESHOLD_REQUEST);
-	hostif_sme_enqueue(priv, SME_FRAGMENTATION_THRESHOLD_REQUEST);
-
-	hostif_sme_enqueue(priv, SME_WEP_INDEX_REQUEST);
-	hostif_sme_enqueue(priv, SME_WEP_KEY1_REQUEST);
-	hostif_sme_enqueue(priv, SME_WEP_KEY2_REQUEST);
-	hostif_sme_enqueue(priv, SME_WEP_KEY3_REQUEST);
-	hostif_sme_enqueue(priv, SME_WEP_KEY4_REQUEST);
-
-	hostif_sme_enqueue(priv, SME_WEP_FLAG_REQUEST);
-	hostif_sme_enqueue(priv, SME_RSN_ENABLED_REQUEST);
-	hostif_sme_enqueue(priv, SME_MODE_SET_REQUEST);
-	hostif_sme_enqueue(priv, SME_START_REQUEST);
+	ks7010_sme_enqueue_events(priv);
 
 	if (!wait_for_completion_interruptible_timeout
 	    (&priv->confirm_wait, 5 * HZ)) {
@@ -802,18 +875,87 @@ static void ks7010_init_defaults(struct ks_wlan_private *priv)
 	priv->reg.rate_set.size = 12;
 }
 
+static int ks7010_sdio_setup_irqs(struct sdio_func *func)
+{
+	int ret;
+
+	/* interrupt disable */
+	sdio_writeb(func, 0, INT_ENABLE_REG, &ret);
+	if (ret)
+		goto irq_error;
+
+	sdio_writeb(func, 0xff, INT_PENDING_REG, &ret);
+	if (ret)
+		goto irq_error;
+
+	/* setup interrupt handler */
+	ret = sdio_claim_irq(func, ks_sdio_interrupt);
+
+irq_error:
+	return ret;
+}
+
+static void ks7010_sdio_init_irqs(struct sdio_func *func,
+				  struct ks_wlan_private *priv)
+{
+	u8 byte;
+	int ret;
+
+	/*
+	 * interrupt setting
+	 * clear Interrupt status write
+	 * (ARMtoSD_InterruptPending FN1:00_0024)
+	 */
+	sdio_claim_host(func);
+	ret = ks7010_sdio_writeb(priv, INT_PENDING_REG, 0xff);
+	sdio_release_host(func);
+	if (ret)
+		netdev_err(priv->net_dev, "write INT_PENDING_REG\n");
+
+	/* enable ks7010sdio interrupt */
+	byte = (INT_GCR_B | INT_READ_STATUS | INT_WRITE_STATUS);
+	sdio_claim_host(func);
+	ret = ks7010_sdio_writeb(priv, INT_ENABLE_REG, byte);
+	sdio_release_host(func);
+	if (ret)
+		netdev_err(priv->net_dev, "write INT_ENABLE_REG\n");
+}
+
+static void ks7010_private_init(struct ks_wlan_private *priv,
+				struct ks_sdio_card *card,
+				struct net_device *netdev)
+{
+	/* private memory initialize */
+	priv->if_hw = card;
+
+	priv->dev_state = DEVICE_STATE_PREBOOT;
+	priv->net_dev = netdev;
+	priv->firmware_version[0] = '\0';
+	priv->version_size = 0;
+	priv->last_doze = jiffies;
+	priv->last_wakeup = jiffies;
+	memset(&priv->nstats, 0, sizeof(priv->nstats));
+	memset(&priv->wstats, 0, sizeof(priv->wstats));
+
+	/* sleep mode */
+	atomic_set(&priv->sleepstatus.doze_request, 0);
+	atomic_set(&priv->sleepstatus.wakeup_request, 0);
+	atomic_set(&priv->sleepstatus.wakeup_request, 0);
+
+	trx_device_init(priv);
+	hostif_init(priv);
+	ks_wlan_net_start(netdev);
+	ks7010_init_defaults(priv);
+}
+
 static int ks7010_sdio_probe(struct sdio_func *func,
 			     const struct sdio_device_id *device)
 {
-	struct ks_wlan_private *priv;
+	struct ks_wlan_private *priv = NULL;
+	struct net_device *netdev = NULL;
 	struct ks_sdio_card *card;
-	struct net_device *netdev;
-	unsigned char byte;
 	int ret;
 
-	priv = NULL;
-	netdev = NULL;
-
 	card = kzalloc(sizeof(*card), GFP_KERNEL);
 	if (!card)
 		return -ENOMEM;
@@ -823,6 +965,9 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 	sdio_claim_host(func);
 
 	ret = sdio_set_block_size(func, KS7010_IO_BLOCK_SIZE);
+	if (ret)
+		goto err_free_card;
+
 	dev_dbg(&card->func->dev, "multi_block=%d sdio_set_block_size()=%d %d\n",
 		func->card->cccr.multi_block, func->cur_blksize, ret);
 
@@ -830,16 +975,7 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 	if (ret)
 		goto err_free_card;
 
-	/* interrupt disable */
-	sdio_writeb(func, 0, INT_ENABLE, &ret);
-	if (ret)
-		goto err_free_card;
-	sdio_writeb(func, 0xff, INT_PENDING, &ret);
-	if (ret)
-		goto err_disable_func;
-
-	/* setup interrupt handler */
-	ret = sdio_claim_irq(func, ks_sdio_interrupt);
+	ret = ks7010_sdio_setup_irqs(func);
 	if (ret)
 		goto err_disable_func;
 
@@ -853,70 +989,35 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 	/* private memory allocate */
 	netdev = alloc_etherdev(sizeof(*priv));
 	if (!netdev) {
-		dev_err(&card->func->dev, "ks7010 : Unable to alloc new net device\n");
+		dev_err(&card->func->dev, "Unable to alloc new net device\n");
 		goto err_release_irq;
 	}
-	if (dev_alloc_name(netdev, "wlan%d") < 0) {
-		dev_err(&card->func->dev,
-			"ks7010 :  Couldn't get name!\n");
+
+	ret = dev_alloc_name(netdev, "wlan%d");
+	if (ret < 0) {
+		dev_err(&card->func->dev, "Couldn't get name!\n");
 		goto err_free_netdev;
 	}
 
 	priv = netdev_priv(netdev);
 
 	card->priv = priv;
-	SET_NETDEV_DEV(netdev, &card->func->dev);	/* for create sysfs symlinks */
-
-	/* private memory initialize */
-	priv->ks_sdio_card = card;
-
-	priv->dev_state = DEVICE_STATE_PREBOOT;
-	priv->net_dev = netdev;
-	priv->firmware_version[0] = '\0';
-	priv->version_size = 0;
-	priv->last_doze = jiffies;
-	priv->last_wakeup = jiffies;
-	memset(&priv->nstats, 0, sizeof(priv->nstats));
-	memset(&priv->wstats, 0, sizeof(priv->wstats));
-
-	/* sleep mode */
-	atomic_set(&priv->sleepstatus.doze_request, 0);
-	atomic_set(&priv->sleepstatus.wakeup_request, 0);
-	atomic_set(&priv->sleepstatus.wakeup_request, 0);
+	SET_NETDEV_DEV(netdev, &card->func->dev);
 
-	trx_device_init(priv);
-	hostif_init(priv);
-	ks_wlan_net_start(netdev);
-
-	ks7010_init_defaults(priv);
+	ks7010_private_init(priv, card, netdev);
 
 	ret = ks7010_upload_firmware(card);
 	if (ret) {
 		netdev_err(priv->net_dev,
-			   "ks7010: firmware load failed !! return code = %d\n",
-			   ret);
+			   "firmware load failed !! ret = %d\n", ret);
 		goto err_free_netdev;
 	}
 
-	/* interrupt setting */
-	/* clear Interrupt status write (ARMtoSD_InterruptPending FN1:00_0024) */
-	sdio_claim_host(func);
-	ret = ks7010_sdio_writeb(priv, INT_PENDING, 0xff);
-	sdio_release_host(func);
-	if (ret)
-		netdev_err(priv->net_dev, " error : INT_PENDING\n");
-
-	/* enable ks7010sdio interrupt */
-	byte = (INT_GCR_B | INT_READ_STATUS | INT_WRITE_STATUS);
-	sdio_claim_host(func);
-	ret = ks7010_sdio_writeb(priv, INT_ENABLE, byte);
-	sdio_release_host(func);
-	if (ret)
-		netdev_err(priv->net_dev, " err : INT_ENABLE\n");
+	ks7010_sdio_init_irqs(func, priv);
 
 	priv->dev_state = DEVICE_STATE_BOOT;
 
-	priv->wq = create_workqueue("wq");
+	priv->wq = alloc_workqueue("wq", WQ_MEM_RECLAIM, 1);
 	if (!priv->wq) {
 		netdev_err(priv->net_dev, "create_workqueue failed !!\n");
 		goto err_free_netdev;
@@ -932,8 +1033,7 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 	return 0;
 
  err_free_netdev:
-	free_netdev(priv->net_dev);
-	card->priv = NULL;
+	free_netdev(netdev);
  err_release_irq:
 	sdio_claim_host(func);
 	sdio_release_irq(func);
@@ -950,25 +1050,22 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 /* send stop request to MAC */
 static int send_stop_request(struct sdio_func *func)
 {
-	struct hostif_stop_request_t *pp;
+	struct hostif_stop_request *pp;
 	struct ks_sdio_card *card;
 	size_t size;
 
 	card = sdio_get_drvdata(func);
 
 	pp = kzalloc(hif_align_size(sizeof(*pp)), GFP_KERNEL);
-	if (!pp) {
-		netdev_err(card->priv->net_dev, "allocate memory failed..\n");
+	if (!pp)
 		return -ENOMEM;
-	}
 
 	size = sizeof(*pp) - sizeof(pp->header.size);
-	pp->header.size = cpu_to_le16((uint16_t)size);
-	pp->header.event = cpu_to_le16((uint16_t)HIF_STOP_REQ);
+	pp->header.size = cpu_to_le16(size);
+	pp->header.event = cpu_to_le16(HIF_STOP_REQ);
 
 	sdio_claim_host(func);
-	write_to_device(card->priv, (unsigned char *)pp,
-			hif_align_size(sizeof(*pp)));
+	write_to_device(card->priv, (u8 *)pp, hif_align_size(sizeof(*pp)));
 	sdio_release_host(func);
 
 	kfree(pp);
@@ -987,43 +1084,50 @@ static void ks7010_sdio_remove(struct sdio_func *func)
 		return;
 
 	priv = card->priv;
-	if (priv) {
-		struct net_device *netdev = priv->net_dev;
+	if (!priv)
+		goto err_free_card;
 
-		ks_wlan_net_stop(netdev);
+	ks_wlan_net_stop(priv->net_dev);
 
-		/* interrupt disable */
-		sdio_claim_host(func);
-		sdio_writeb(func, 0, INT_ENABLE, &ret);
-		sdio_writeb(func, 0xff, INT_PENDING, &ret);
-		sdio_release_host(func);
+	/* interrupt disable */
+	sdio_claim_host(func);
+	sdio_writeb(func, 0, INT_ENABLE_REG, &ret);
+	sdio_writeb(func, 0xff, INT_PENDING_REG, &ret);
+	sdio_release_host(func);
 
-		ret = send_stop_request(func);
-		if (ret)	/* memory allocation failure */
-			return;
+	ret = send_stop_request(func);
+	if (ret)	/* memory allocation failure */
+		goto err_free_card;
 
-		if (priv->wq) {
-			flush_workqueue(priv->wq);
-			destroy_workqueue(priv->wq);
-		}
+	if (priv->wq) {
+		flush_workqueue(priv->wq);
+		destroy_workqueue(priv->wq);
+	}
 
-		hostif_exit(priv);
+	hostif_exit(priv);
 
-		unregister_netdev(netdev);
+	unregister_netdev(priv->net_dev);
 
-		trx_device_exit(priv);
-		free_netdev(priv->net_dev);
-		card->priv = NULL;
-	}
+	trx_device_exit(priv);
+	free_netdev(priv->net_dev);
+	card->priv = NULL;
 
 	sdio_claim_host(func);
 	sdio_release_irq(func);
 	sdio_disable_func(func);
 	sdio_release_host(func);
+err_free_card:
 	sdio_set_drvdata(func, NULL);
 	kfree(card);
 }
 
+static const struct sdio_device_id ks7010_sdio_ids[] = {
+	{SDIO_DEVICE(SDIO_VENDOR_ID_KS_CODE_A, SDIO_DEVICE_ID_KS_7010)},
+	{SDIO_DEVICE(SDIO_VENDOR_ID_KS_CODE_B, SDIO_DEVICE_ID_KS_7010)},
+	{ /* all zero */ }
+};
+MODULE_DEVICE_TABLE(sdio, ks7010_sdio_ids);
+
 static struct sdio_driver ks7010_sdio_driver = {
 	.name = "ks7010_sdio",
 	.id_table = ks7010_sdio_ids,
diff --git a/drivers/staging/ks7010/ks7010_sdio.h b/drivers/staging/ks7010/ks7010_sdio.h
deleted file mode 100644
index e4f56a11c888..000000000000
--- a/drivers/staging/ks7010/ks7010_sdio.h
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- *   Driver for KeyStream, KS7010 based SDIO cards.
- *
- *   Copyright (C) 2006-2008 KeyStream Corp.
- *   Copyright (C) 2009 Renesas Technology Corp.
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License version 2 as
- *   published by the Free Software Foundation.
- */
-#ifndef _KS7010_SDIO_H
-#define _KS7010_SDIO_H
-
-#ifdef	DEVICE_ALIGNMENT
-#undef	DEVICE_ALIGNMENT
-#endif
-#define DEVICE_ALIGNMENT 32
-
-/*  SDIO KeyStream vendor and device */
-#define SDIO_VENDOR_ID_KS_CODE_A	0x005b
-#define SDIO_VENDOR_ID_KS_CODE_B	0x0023
-/* Older sources suggest earlier versions were named 7910 or 79xx */
-#define SDIO_DEVICE_ID_KS_7010		0x7910
-
-/* Read/Write Status Register */
-#define READ_STATUS		0x000000
-#define WRITE_STATUS		0x00000C
-enum reg_status_type {
-	REG_STATUS_BUSY,
-	REG_STATUS_IDLE
-};
-
-/* Read Index Register */
-#define READ_INDEX		0x000004
-
-/* Read Data Size Register */
-#define READ_DATA_SIZE		0x000008
-
-/* Write Index Register */
-#define WRITE_INDEX		0x000010
-
-/* Write Status/Read Data Size Register
- * for network packet (less than 2048 bytes data)
- */
-#define WSTATUS_RSIZE		0x000014
-#define WSTATUS_MASK		0x80	/* Write Status Register value */
-#define RSIZE_MASK		0x7F	/* Read Data Size Register value [10:4] */
-
-/* ARM to SD interrupt Enable */
-#define INT_ENABLE		0x000020
-/* ARM to SD interrupt Pending */
-#define INT_PENDING		0x000024
-
-#define INT_GCR_B              BIT(7)
-#define INT_GCR_A              BIT(6)
-#define INT_WRITE_STATUS       BIT(5)
-#define INT_WRITE_INDEX        BIT(4)
-#define INT_WRITE_SIZE         BIT(3)
-#define INT_READ_STATUS        BIT(2)
-#define INT_READ_INDEX         BIT(1)
-#define INT_READ_SIZE          BIT(0)
-
-/* General Communication Register A */
-#define GCR_A			0x000028
-enum gen_com_reg_a {
-	GCR_A_INIT,
-	GCR_A_REMAP,
-	GCR_A_RUN
-};
-
-/* General Communication Register B */
-#define GCR_B			0x00002C
-enum gen_com_reg_b {
-	GCR_B_ACTIVE,
-	GCR_B_DOZE
-};
-
-/* Wakeup Register */
-#define WAKEUP			0x008018
-#define WAKEUP_REQ		0x5a
-
-/* AHB Data Window  0x010000-0x01FFFF */
-#define DATA_WINDOW		0x010000
-#define WINDOW_SIZE		(64 * 1024)
-
-#define KS7010_IRAM_ADDRESS	0x06000000
-
-/**
- * struct ks_sdio_card - SDIO device data.
- *
- * Structure is used as the &struct sdio_func private data.
- *
- * @func: Pointer to the SDIO function device.
- * @priv: Pointer to the &struct net_device private data.
- */
-struct ks_sdio_card {
-	struct sdio_func *func;
-	struct ks_wlan_private *priv;
-};
-
-/* Tx Device struct */
-#define	TX_DEVICE_BUFF_SIZE	1024
-
-/**
- * struct tx_device_buffer - Queue item for the tx queue.
- * @sendp: Pointer to the send request data.
- * @size: Size of @sendp data.
- * @complete_handler: Function called once data write to device is complete.
- * @arg1: First argument to @complete_handler.
- * @arg2: Second argument to @complete_handler.
- */
-struct tx_device_buffer {
-	unsigned char *sendp;
-	unsigned int size;
-	void (*complete_handler)(struct ks_wlan_private *priv,
-				 struct sk_buff *skb);
-	struct sk_buff *skb;
-};
-
-/**
- * struct tx_device - Tx buffer queue.
- * @tx_device_buffer: Queue buffer.
- * @qhead: Head of tx queue.
- * @qtail: Tail of tx queue.
- * @tx_dev_lock: Queue lock.
- */
-struct tx_device {
-	struct tx_device_buffer tx_dev_buff[TX_DEVICE_BUFF_SIZE];
-	unsigned int qhead;
-	unsigned int qtail;
-	spinlock_t tx_dev_lock;	/* protect access to the queue */
-};
-
-/* Rx Device struct */
-#define	RX_DATA_SIZE	(2 + 2 + 2347 + 1)
-#define	RX_DEVICE_BUFF_SIZE	32
-
-/**
- * struct rx_device_buffer - Queue item for the rx queue.
- * @data: rx data.
- * @size: Size of @data.
- */
-struct rx_device_buffer {
-	unsigned char data[RX_DATA_SIZE];
-	unsigned int size;
-};
-
-/**
- * struct rx_device - Rx buffer queue.
- * @rx_device_buffer: Queue buffer.
- * @qhead: Head of rx queue.
- * @qtail: Tail of rx queue.
- * @rx_dev_lock: Queue lock.
- */
-struct rx_device {
-	struct rx_device_buffer rx_dev_buff[RX_DEVICE_BUFF_SIZE];
-	unsigned int qhead;
-	unsigned int qtail;
-	spinlock_t rx_dev_lock;	/* protect access to the queue */
-};
-
-#define	ROM_FILE "ks7010sd.rom"
-
-#endif /* _KS7010_SDIO_H */
diff --git a/drivers/staging/ks7010/ks_hostif.c b/drivers/staging/ks7010/ks_hostif.c
index 676961cf4103..0ecffab52ec2 100644
--- a/drivers/staging/ks7010/ks_hostif.c
+++ b/drivers/staging/ks7010/ks_hostif.c
@@ -1,25 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *   Driver for KeyStream wireless LAN cards.
  *
  *   Copyright (C) 2005-2008 KeyStream Corp.
  *   Copyright (C) 2009 Renesas Technology Corp.
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License version 2 as
- *   published by the Free Software Foundation.
  */
 
-#include "ks_wlan.h"
-#include "ks_hostif.h"
+#include <linux/circ_buf.h>
+#include <linux/if_arp.h>
+#include <net/iw_handler.h>
+#include <uapi/linux/llc.h>
 #include "eap_packet.h"
+#include "ks_wlan.h"
 #include "michael_mic.h"
-
-#include <linux/etherdevice.h>
-#include <linux/if_ether.h>
-#include <linux/if_arp.h>
-
-/* Include Wireless Extension definition and check version */
-#include <net/iw_handler.h>	/* New driver API */
+#include "ks_hostif.h"
 
 static inline void inc_smeqhead(struct ks_wlan_private *priv)
 {
@@ -33,13 +27,10 @@ static inline void inc_smeqtail(struct ks_wlan_private *priv)
 
 static inline unsigned int cnt_smeqbody(struct ks_wlan_private *priv)
 {
-	unsigned int sme_cnt = priv->sme_i.qtail - priv->sme_i.qhead;
-
-	return (sme_cnt + SME_EVENT_BUFF_SIZE) % SME_EVENT_BUFF_SIZE;
+	return CIRC_CNT_TO_END(priv->sme_i.qhead, priv->sme_i.qtail,
+			       SME_EVENT_BUFF_SIZE);
 }
 
-#define KS_WLAN_MEM_FLAG (GFP_ATOMIC)
-
 static inline u8 get_byte(struct ks_wlan_private *priv)
 {
 	u8 data;
@@ -96,82 +87,59 @@ static void ks_wlan_hw_wakeup_task(struct work_struct *work)
 		tasklet_enable(&priv->sme_task);
 }
 
-static
-int ks_wlan_do_power_save(struct ks_wlan_private *priv)
+static void ks_wlan_do_power_save(struct ks_wlan_private *priv)
 {
 	if (is_connect_status(priv->connect_status))
 		hostif_sme_enqueue(priv, SME_POW_MNGMT_REQUEST);
 	else
 		priv->dev_state = DEVICE_STATE_READY;
-	return 0;
 }
 
 static
-int get_current_ap(struct ks_wlan_private *priv, struct link_ap_info_t *ap_info)
+int get_current_ap(struct ks_wlan_private *priv, struct link_ap_info *ap_info)
 {
-	struct local_ap_t *ap;
+	struct local_ap *ap;
 	union iwreq_data wrqu;
 	struct net_device *netdev = priv->net_dev;
+	u8 size;
 
 	ap = &priv->current_ap;
 
 	if (is_disconnect_status(priv->connect_status)) {
-		memset(ap, 0, sizeof(struct local_ap_t));
+		memset(ap, 0, sizeof(struct local_ap));
 		return -EPERM;
 	}
 
-	/* bssid */
-	memcpy(ap->bssid, ap_info->bssid, ETH_ALEN);
-	/* essid */
+	ether_addr_copy(ap->bssid, ap_info->bssid);
 	memcpy(ap->ssid.body, priv->reg.ssid.body,
 	       priv->reg.ssid.size);
 	ap->ssid.size = priv->reg.ssid.size;
-	/* rate_set */
 	memcpy(ap->rate_set.body, ap_info->rate_set.body,
 	       ap_info->rate_set.size);
 	ap->rate_set.size = ap_info->rate_set.size;
 	if (ap_info->ext_rate_set.size != 0) {
-		/* rate_set */
 		memcpy(&ap->rate_set.body[ap->rate_set.size],
 		       ap_info->ext_rate_set.body,
 		       ap_info->ext_rate_set.size);
 		ap->rate_set.size += ap_info->ext_rate_set.size;
 	}
-	/* channel */
 	ap->channel = ap_info->ds_parameter.channel;
-	/* rssi */
 	ap->rssi = ap_info->rssi;
-	/* sq */
 	ap->sq = ap_info->sq;
-	/* noise */
 	ap->noise = ap_info->noise;
-	/* capability */
 	ap->capability = le16_to_cpu(ap_info->capability);
-	/* rsn */
+	size = (ap_info->rsn.size <= RSN_IE_BODY_MAX) ?
+		ap_info->rsn.size : RSN_IE_BODY_MAX;
 	if ((ap_info->rsn_mode & RSN_MODE_WPA2) &&
 	    (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)) {
-		ap->rsn_ie.id = 0x30;
-		if (ap_info->rsn.size <= RSN_IE_BODY_MAX) {
-			ap->rsn_ie.size = ap_info->rsn.size;
-			memcpy(ap->rsn_ie.body, ap_info->rsn.body,
-			       ap_info->rsn.size);
-		} else {
-			ap->rsn_ie.size = RSN_IE_BODY_MAX;
-			memcpy(ap->rsn_ie.body, ap_info->rsn.body,
-			       RSN_IE_BODY_MAX);
-		}
+		ap->rsn_ie.id = RSN_INFO_ELEM_ID;
+		ap->rsn_ie.size = size;
+		memcpy(ap->rsn_ie.body, ap_info->rsn.body, size);
 	} else if ((ap_info->rsn_mode & RSN_MODE_WPA) &&
 		   (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA)) {
-		ap->wpa_ie.id = 0xdd;
-		if (ap_info->rsn.size <= RSN_IE_BODY_MAX) {
-			ap->wpa_ie.size = ap_info->rsn.size;
-			memcpy(ap->wpa_ie.body, ap_info->rsn.body,
-			       ap_info->rsn.size);
-		} else {
-			ap->wpa_ie.size = RSN_IE_BODY_MAX;
-			memcpy(ap->wpa_ie.body, ap_info->rsn.body,
-			       RSN_IE_BODY_MAX);
-		}
+		ap->wpa_ie.id = WPA_INFO_ELEM_ID;
+		ap->wpa_ie.size = size;
+		memcpy(ap->wpa_ie.body, ap_info->rsn.body, size);
 	} else {
 		ap->rsn_ie.id = 0;
 		ap->rsn_ie.size = 0;
@@ -183,20 +151,24 @@ int get_current_ap(struct ks_wlan_private *priv, struct link_ap_info_t *ap_info)
 	wrqu.data.flags = 0;
 	wrqu.ap_addr.sa_family = ARPHRD_ETHER;
 	if (is_connect_status(priv->connect_status)) {
-		memcpy(wrqu.ap_addr.sa_data,
-		       priv->current_ap.bssid, ETH_ALEN);
+		ether_addr_copy(wrqu.ap_addr.sa_data, priv->current_ap.bssid);
 		netdev_dbg(priv->net_dev,
-			   "IWEVENT: connect bssid=%pM\n", wrqu.ap_addr.sa_data);
+			   "IWEVENT: connect bssid=%pM\n",
+			   wrqu.ap_addr.sa_data);
 		wireless_send_event(netdev, SIOCGIWAP, &wrqu, NULL);
 	}
-	netdev_dbg(priv->net_dev, "    Link AP\n");
-	netdev_dbg(priv->net_dev, "    bssid=%02X:%02X:%02X:%02X:%02X:%02X\n"
-		   "    essid=%s\n"
-		   "    rate_set=%02X,%02X,%02X,%02X,%02X,%02X,%02X,%02X\n"
-		   "    channel=%d\n"
-		   "    rssi=%d\n"
-		   "    sq=%d\n"
-		   "    capability=%04X\n",
+	netdev_dbg(priv->net_dev, "Link AP\n"
+		   "- bssid=%02X:%02X:%02X:%02X:%02X:%02X\n"
+		   "- essid=%s\n"
+		   "- rate_set=%02X,%02X,%02X,%02X,%02X,%02X,%02X,%02X\n"
+		   "- channel=%d\n"
+		   "- rssi=%d\n"
+		   "- sq=%d\n"
+		   "- capability=%04X\n"
+		   "- rsn.mode=%d\n"
+		   "- rsn.size=%d\n"
+		   "- ext_rate_set_size=%d\n"
+		   "- rate_set_size=%d\n",
 		   ap->bssid[0], ap->bssid[1], ap->bssid[2],
 		   ap->bssid[3], ap->bssid[4], ap->bssid[5],
 		   &(ap->ssid.body[0]),
@@ -204,10 +176,8 @@ int get_current_ap(struct ks_wlan_private *priv, struct link_ap_info_t *ap_info)
 		   ap->rate_set.body[2], ap->rate_set.body[3],
 		   ap->rate_set.body[4], ap->rate_set.body[5],
 		   ap->rate_set.body[6], ap->rate_set.body[7],
-		   ap->channel, ap->rssi, ap->sq, ap->capability);
-	netdev_dbg(priv->net_dev, "    Link AP\n    rsn.mode=%d\n    rsn.size=%d\n",
-		   ap_info->rsn_mode, ap_info->rsn.size);
-	netdev_dbg(priv->net_dev, "    ext_rate_set_size=%d\n    rate_set_size=%d\n",
+		   ap->channel, ap->rssi, ap->sq, ap->capability,
+		   ap_info->rsn_mode, ap_info->rsn.size,
 		   ap_info->ext_rate_set.size, ap_info->rate_set.size);
 
 	return 0;
@@ -223,25 +193,19 @@ static u8 read_ie(unsigned char *bp, u8 max, u8 *body)
 
 
 static
-int get_ap_information(struct ks_wlan_private *priv, struct ap_info_t *ap_info,
-		       struct local_ap_t *ap)
+int get_ap_information(struct ks_wlan_private *priv, struct ap_info *ap_info,
+		       struct local_ap *ap)
 {
 	unsigned char *bp;
 	int bsize, offset;
 
-	memset(ap, 0, sizeof(struct local_ap_t));
+	memset(ap, 0, sizeof(struct local_ap));
 
-	/* bssid */
-	memcpy(ap->bssid, ap_info->bssid, ETH_ALEN);
-	/* rssi */
+	ether_addr_copy(ap->bssid, ap_info->bssid);
 	ap->rssi = ap_info->rssi;
-	/* sq */
 	ap->sq = ap_info->sq;
-	/* noise */
 	ap->noise = ap_info->noise;
-	/* capability */
 	ap->capability = le16_to_cpu(ap_info->capability);
-	/* channel */
 	ap->channel = ap_info->ch_info;
 
 	bp = ap_info->body;
@@ -269,8 +233,6 @@ int get_ap_information(struct ks_wlan_private *priv, struct ap_info_t *ap_info,
 				    (RATE_SET_MAX_SIZE - ap->rate_set.size);
 			}
 			break;
-		case WLAN_EID_DS_PARAMS:
-			break;
 		case WLAN_EID_RSN:
 			ap->rsn_ie.id = *bp;
 			ap->rsn_ie.size = read_ie(bp, RSN_IE_BODY_MAX,
@@ -284,7 +246,7 @@ int get_ap_information(struct ks_wlan_private *priv, struct ap_info_t *ap_info,
 							  ap->wpa_ie.body);
 			}
 			break;
-
+		case WLAN_EID_DS_PARAMS:
 		case WLAN_EID_FH_PARAMS:
 		case WLAN_EID_CF_PARAMS:
 		case WLAN_EID_TIM:
@@ -293,7 +255,8 @@ int get_ap_information(struct ks_wlan_private *priv, struct ap_info_t *ap_info,
 		case WLAN_EID_ERP_INFO:
 			break;
 		default:
-			netdev_err(priv->net_dev, "unknown Element ID=%d\n", *bp);
+			netdev_err(priv->net_dev,
+				   "unknown Element ID=%d\n", *bp);
 			break;
 		}
 
@@ -314,11 +277,11 @@ int hostif_data_indication_wpa(struct ks_wlan_private *priv,
 	unsigned char recv_mic[8];
 	char buf[128];
 	unsigned long now;
-	struct mic_failure_t *mic_failure;
-	struct michael_mic_t michael_mic;
+	struct mic_failure *mic_failure;
+	struct michael_mic michael_mic;
 	union iwreq_data wrqu;
 	unsigned int key_index = auth_type - 1;
-	struct wpa_key_t *key = &priv->wpa.key[key_index];
+	struct wpa_key *key = &priv->wpa.key[key_index];
 
 	eth_hdr = (struct ether_hdr *)(priv->rxp);
 	eth_proto = ntohs(eth_hdr->h_proto);
@@ -341,12 +304,9 @@ int hostif_data_indication_wpa(struct ks_wlan_private *priv,
 		memcpy(&recv_mic[0], (priv->rxp) + ((priv->rx_size) - 8), 8);
 		priv->rx_size = priv->rx_size - 8;
 		if (auth_type > 0 && auth_type < 4) {	/* auth_type check */
-			michael_mic_function(&michael_mic,
-					     (uint8_t *)key->rx_mic_key,
-					     (uint8_t *)priv->rxp,
-					     (int)priv->rx_size,
-					     (uint8_t)0,	/* priority */
-					     (uint8_t *)michael_mic.result);
+			michael_mic_function(&michael_mic, key->rx_mic_key,
+					     priv->rxp, priv->rx_size,
+					     0,	michael_mic.result);
 		}
 		if (memcmp(michael_mic.result, recv_mic, 8) != 0) {
 			now = jiffies;
@@ -363,8 +323,9 @@ int hostif_data_indication_wpa(struct ks_wlan_private *priv,
 			} else if (mic_failure->failure == 1) {
 				mic_failure->failure = 2;
 				mic_failure->counter =
-					(uint16_t)((now - mic_failure->last_failure_time) / HZ);
-				if (!mic_failure->counter)	/*  range 1-60 */
+					(u16)((now - mic_failure->last_failure_time) / HZ);
+				/*  range 1-60 */
+				if (!mic_failure->counter)
 					mic_failure->counter = 1;
 			}
 			priv->wpa.mic_failure.last_failure_time = now;
@@ -390,7 +351,7 @@ void hostif_data_indication(struct ks_wlan_private *priv)
 {
 	unsigned int rx_ind_size;	/* indicate data size */
 	struct sk_buff *skb;
-	unsigned short auth_type;
+	u16 auth_type;
 	unsigned char temp[256];
 	struct ether_hdr *eth_hdr;
 	unsigned short eth_proto;
@@ -411,7 +372,7 @@ void hostif_data_indication(struct ks_wlan_private *priv)
 	eth_proto = ntohs(eth_hdr->h_proto);
 
 	/* source address check */
-	if (memcmp(&priv->eth_addr[0], eth_hdr->h_source, ETH_ALEN) == 0) {
+	if (ether_addr_equal(&priv->eth_addr[0], eth_hdr->h_source)) {
 		netdev_err(priv->net_dev, "invalid : source is own mac address !!\n");
 		netdev_err(priv->net_dev,
 			   "eth_hdrernet->h_dest=%02X:%02X:%02X:%02X:%02X:%02X\n",
@@ -436,7 +397,7 @@ void hostif_data_indication(struct ks_wlan_private *priv)
 
 	/* check 13th byte at rx data */
 	switch (*(priv->rxp + 12)) {
-	case 0xAA:	/* SNAP */
+	case LLC_SAP_SNAP:
 		rx_ind_size = priv->rx_size - 6;
 		skb = dev_alloc_skb(rx_ind_size);
 		if (!skb) {
@@ -456,7 +417,7 @@ void hostif_data_indication(struct ks_wlan_private *priv)
 
 		aa1x_hdr = (struct ieee802_1x_hdr *)(priv->rxp + ETHER_HDR_SIZE);
 		break;
-	case 0xF0:	/* NETBEUI/NetBIOS */
+	case LLC_SAP_NETBEUI:
 		rx_ind_size = (priv->rx_size + 2);
 		skb = dev_alloc_skb(rx_ind_size);
 		if (!skb) {
@@ -506,13 +467,12 @@ void hostif_mib_get_confirm(struct ks_wlan_private *priv)
 	u16 mib_val_size;
 	u16 mib_val_type;
 
-	mib_status = get_dword(priv);	/* MIB status */
-	mib_attribute = get_dword(priv);	/* MIB atttibute */
-	mib_val_size = get_word(priv);	/* MIB value size */
-	mib_val_type = get_word(priv);	/* MIB value type */
+	mib_status = get_dword(priv);
+	mib_attribute = get_dword(priv);
+	mib_val_size = get_word(priv);
+	mib_val_type = get_word(priv);
 
 	if (mib_status) {
-		/* in case of error */
 		netdev_err(priv->net_dev, "attribute=%08X, status=%08X\n",
 			   mib_attribute, mib_status);
 		return;
@@ -520,22 +480,13 @@ void hostif_mib_get_confirm(struct ks_wlan_private *priv)
 
 	switch (mib_attribute) {
 	case DOT11_MAC_ADDRESS:
-		/* MAC address */
 		hostif_sme_enqueue(priv, SME_GET_MAC_ADDRESS);
-		memcpy(priv->eth_addr, priv->rxp, ETH_ALEN);
+		ether_addr_copy(priv->eth_addr, priv->rxp);
 		priv->mac_address_valid = true;
-		dev->dev_addr[0] = priv->eth_addr[0];
-		dev->dev_addr[1] = priv->eth_addr[1];
-		dev->dev_addr[2] = priv->eth_addr[2];
-		dev->dev_addr[3] = priv->eth_addr[3];
-		dev->dev_addr[4] = priv->eth_addr[4];
-		dev->dev_addr[5] = priv->eth_addr[5];
-		dev->dev_addr[6] = 0x00;
-		dev->dev_addr[7] = 0x00;
+		ether_addr_copy(dev->dev_addr, priv->eth_addr);
 		netdev_info(dev, "MAC ADDRESS = %pM\n", priv->eth_addr);
 		break;
 	case DOT11_PRODUCT_VERSION:
-		/* firmware version */
 		priv->version_size = priv->rx_size;
 		memcpy(priv->firmware_version, priv->rxp, priv->rx_size);
 		priv->firmware_version[priv->rx_size] = '\0';
@@ -553,18 +504,15 @@ void hostif_mib_get_confirm(struct ks_wlan_private *priv)
 		break;
 	case LOCAL_EEPROM_SUM:
 		memcpy(&priv->eeprom_sum, priv->rxp, sizeof(priv->eeprom_sum));
-		if (priv->eeprom_sum.type == 0) {
-			priv->eeprom_checksum = EEPROM_CHECKSUM_NONE;
-		} else if (priv->eeprom_sum.type == 1) {
-			if (priv->eeprom_sum.result == 0) {
-				priv->eeprom_checksum = EEPROM_NG;
-				netdev_info(dev, "LOCAL_EEPROM_SUM NG\n");
-			} else if (priv->eeprom_sum.result == 1) {
-				priv->eeprom_checksum = EEPROM_OK;
-			}
-		} else {
+		if (priv->eeprom_sum.type != 0 &&
+		    priv->eeprom_sum.type != 1) {
 			netdev_err(dev, "LOCAL_EEPROM_SUM error!\n");
+			return;
 		}
+		priv->eeprom_checksum = (priv->eeprom_sum.type == 0) ?
+					 EEPROM_CHECKSUM_NONE :
+					 (priv->eeprom_sum.result == 0) ?
+					 EEPROM_NG : EEPROM_OK;
 		break;
 	default:
 		netdev_err(priv->net_dev, "mib_attribute=%08x\n",
@@ -576,11 +524,11 @@ void hostif_mib_get_confirm(struct ks_wlan_private *priv)
 static
 void hostif_mib_set_confirm(struct ks_wlan_private *priv)
 {
-	u32 mib_status;	/* +04 MIB Status */
-	u32 mib_attribute;	/* +08 MIB attribute */
+	u32 mib_status;
+	u32 mib_attribute;
 
-	mib_status = get_dword(priv);	/* MIB Status */
-	mib_attribute = get_dword(priv);	/* MIB attribute */
+	mib_status = get_dword(priv);
+	mib_attribute = get_dword(priv);
 
 	if (mib_status) {
 		/* in case of error */
@@ -649,8 +597,6 @@ void hostif_mib_set_confirm(struct ks_wlan_private *priv)
 	case DOT11_RSN_CONFIG_AUTH_SUITE:
 		hostif_sme_enqueue(priv, SME_RSN_AUTH_CONFIRM);
 		break;
-	case DOT11_PMK_TSC:
-		break;
 	case DOT11_GMK1_TSC:
 		if (atomic_read(&priv->psstatus.snooze_guard))
 			atomic_set(&priv->psstatus.snooze_guard, 0);
@@ -659,16 +605,11 @@ void hostif_mib_set_confirm(struct ks_wlan_private *priv)
 		if (atomic_read(&priv->psstatus.snooze_guard))
 			atomic_set(&priv->psstatus.snooze_guard, 0);
 		break;
+	case DOT11_PMK_TSC:
 	case LOCAL_PMK:
-		break;
 	case LOCAL_GAIN:
-		break;
-#ifdef WPS
 	case LOCAL_WPS_ENABLE:
-		break;
 	case LOCAL_WPS_PROBE_REQ:
-		break;
-#endif /* WPS */
 	case LOCAL_REGION:
 	default:
 		break;
@@ -698,7 +639,6 @@ void hostif_sleep_confirm(struct ks_wlan_private *priv)
 static
 void hostif_start_confirm(struct ks_wlan_private *priv)
 {
-#ifdef WPS
 	union iwreq_data wrqu;
 
 	wrqu.data.length = 0;
@@ -708,7 +648,6 @@ void hostif_start_confirm(struct ks_wlan_private *priv)
 		eth_zero_addr(wrqu.ap_addr.sa_data);
 		wireless_send_event(priv->net_dev, SIOCGIWAP, &wrqu, NULL);
 	}
-#endif
 	netdev_dbg(priv->net_dev, " scan_ind_count=%d\n", priv->scan_ind_count);
 	hostif_sme_enqueue(priv, SME_START_CONFIRM);
 }
@@ -716,7 +655,7 @@ void hostif_start_confirm(struct ks_wlan_private *priv)
 static
 void hostif_connect_indication(struct ks_wlan_private *priv)
 {
-	unsigned short connect_code;
+	u16 connect_code;
 	unsigned int tmp = 0;
 	unsigned int old_status = priv->connect_status;
 	struct net_device *netdev = priv->net_dev;
@@ -725,13 +664,13 @@ void hostif_connect_indication(struct ks_wlan_private *priv)
 	connect_code = get_word(priv);
 
 	switch (connect_code) {
-	case RESULT_CONNECT:	/* connect */
+	case RESULT_CONNECT:
 		if (!(priv->connect_status & FORCE_DISCONNECT))
 			netif_carrier_on(netdev);
 		tmp = FORCE_DISCONNECT & priv->connect_status;
 		priv->connect_status = tmp + CONNECT_STATUS;
 		break;
-	case RESULT_DISCONNECT: /* disconnect */
+	case RESULT_DISCONNECT:
 		netif_carrier_off(netdev);
 		tmp = FORCE_DISCONNECT & priv->connect_status;
 		priv->connect_status = tmp + DISCONNECT_STATUS;
@@ -745,7 +684,7 @@ void hostif_connect_indication(struct ks_wlan_private *priv)
 		break;
 	}
 
-	get_current_ap(priv, (struct link_ap_info_t *)priv->rxp);
+	get_current_ap(priv, (struct link_ap_info *)priv->rxp);
 	if (is_connect_status(priv->connect_status) &&
 	    is_disconnect_status(old_status)) {
 		/* for power save */
@@ -771,10 +710,11 @@ static
 void hostif_scan_indication(struct ks_wlan_private *priv)
 {
 	int i;
-	struct ap_info_t *ap_info;
+	struct ap_info *ap_info;
 
-	netdev_dbg(priv->net_dev, "scan_ind_count = %d\n", priv->scan_ind_count);
-	ap_info = (struct ap_info_t *)(priv->rxp);
+	netdev_dbg(priv->net_dev,
+		   "scan_ind_count = %d\n", priv->scan_ind_count);
+	ap_info = (struct ap_info *)(priv->rxp);
 
 	if (priv->scan_ind_count) {
 		/* bssid check */
@@ -794,7 +734,7 @@ void hostif_scan_indication(struct ks_wlan_private *priv)
 	if (priv->scan_ind_count < LOCAL_APLIST_MAX + 1) {
 		netdev_dbg(priv->net_dev, " scan_ind_count=%d :: aplist.size=%d\n",
 			priv->scan_ind_count, priv->aplist.size);
-		get_ap_information(priv, (struct ap_info_t *)(priv->rxp),
+		get_ap_information(priv, (struct ap_info *)(priv->rxp),
 				   &(priv->aplist.ap[priv->scan_ind_count - 1]));
 		priv->aplist.size = priv->scan_ind_count;
 	} else {
@@ -863,8 +803,8 @@ void hostif_adhoc_set_confirm(struct ks_wlan_private *priv)
 static
 void hostif_associate_indication(struct ks_wlan_private *priv)
 {
-	struct association_request_t *assoc_req;
-	struct association_response_t *assoc_resp;
+	struct association_request *assoc_req;
+	struct association_response *assoc_resp;
 	unsigned char *pb;
 	union iwreq_data wrqu;
 	char buf[IW_CUSTOM_MAX];
@@ -874,8 +814,8 @@ void hostif_associate_indication(struct ks_wlan_private *priv)
 	static const char associnfo_leader0[] = "ASSOCINFO(ReqIEs=";
 	static const char associnfo_leader1[] = " RespIEs=";
 
-	assoc_req = (struct association_request_t *)(priv->rxp);
-	assoc_resp = (struct association_response_t *)(assoc_req + 1);
+	assoc_req = (struct association_request *)(priv->rxp);
+	assoc_resp = (struct association_response *)(assoc_req + 1);
 	pb = (unsigned char *)(assoc_resp + 1);
 
 	memset(&wrqu, 0, sizeof(wrqu));
@@ -905,13 +845,13 @@ void hostif_associate_indication(struct ks_wlan_private *priv)
 static
 void hostif_bss_scan_confirm(struct ks_wlan_private *priv)
 {
-	unsigned int result_code;
+	u32 result_code;
 	struct net_device *dev = priv->net_dev;
 	union iwreq_data wrqu;
 
 	result_code = get_dword(priv);
-	netdev_dbg(priv->net_dev, "result=%d :: scan_ind_count=%d\n", result_code,
-		   priv->scan_ind_count);
+	netdev_dbg(priv->net_dev, "result=%d :: scan_ind_count=%d\n",
+		   result_code, priv->scan_ind_count);
 
 	priv->sme_i.sme_flag &= ~SME_AP_SCAN;
 	hostif_sme_enqueue(priv, SME_BSS_SCAN_CONFIRM);
@@ -926,10 +866,10 @@ static
 void hostif_phy_information_confirm(struct ks_wlan_private *priv)
 {
 	struct iw_statistics *wstats = &priv->wstats;
-	unsigned char rssi, signal, noise;
-	unsigned char link_speed;
-	unsigned int transmitted_frame_count, received_fragment_count;
-	unsigned int failed_count, fcs_error_count;
+	u8 rssi, signal, noise;
+	u8 link_speed;
+	u32 transmitted_frame_count, received_fragment_count;
+	u32 failed_count, fcs_error_count;
 
 	rssi = get_byte(priv);
 	signal = get_byte(priv);
@@ -964,16 +904,17 @@ void hostif_phy_information_confirm(struct ks_wlan_private *priv)
 static
 void hostif_mic_failure_confirm(struct ks_wlan_private *priv)
 {
-	netdev_dbg(priv->net_dev, "mic_failure=%u\n", priv->wpa.mic_failure.failure);
+	netdev_dbg(priv->net_dev, "mic_failure=%u\n",
+		   priv->wpa.mic_failure.failure);
 	hostif_sme_enqueue(priv, SME_MIC_FAILURE_CONFIRM);
 }
 
 static
 void hostif_event_check(struct ks_wlan_private *priv)
 {
-	unsigned short event;
+	u16 event;
 
-	event = get_word(priv);	/* get event */
+	event = get_word(priv);
 	switch (event) {
 	case HIF_DATA_IND:
 		hostif_data_indication(priv);
@@ -1044,11 +985,11 @@ static void *hostif_generic_request(size_t size, int event)
 {
 	struct hostif_hdr *p;
 
-	p = kzalloc(hif_align_size(size), KS_WLAN_MEM_FLAG);
+	p = kzalloc(hif_align_size(size), GFP_ATOMIC);
 	if (!p)
 		return NULL;
 
-	p->size = cpu_to_le16((u16)(size - sizeof(p->size)));
+	p->size = cpu_to_le16(size - sizeof(p->size));
 	p->event = cpu_to_le16(event);
 
 	return p;
@@ -1057,15 +998,14 @@ static void *hostif_generic_request(size_t size, int event)
 int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *skb)
 {
 	unsigned int skb_len = 0;
-
 	unsigned char *buffer = NULL;
 	unsigned int length = 0;
-	struct hostif_data_request_t *pp;
+	struct hostif_data_request *pp;
 	unsigned char *p;
 	int result = 0;
 	unsigned short eth_proto;
 	struct ether_hdr *eth_hdr;
-	struct michael_mic_t michael_mic;
+	struct michael_mic michael_mic;
 	unsigned short keyinfo = 0;
 	struct ieee802_1x_hdr *aa1x_hdr;
 	struct wpa_eapol_key *eap_key;
@@ -1098,7 +1038,7 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *skb)
 	}
 
 	size = sizeof(*pp) + 6 + skb_len + 8;
-	pp = kmalloc(hif_align_size(size), KS_WLAN_MEM_FLAG);
+	pp = kmalloc(hif_align_size(size), GFP_ATOMIC);
 	if (!pp) {
 		ret = -ENOMEM;
 		goto err_kfree_skb;
@@ -1111,9 +1051,10 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *skb)
 
 	/* skb check */
 	eth = (struct ethhdr *)skb->data;
-	if (memcmp(&priv->eth_addr[0], eth->h_source, ETH_ALEN) != 0) {
-		netdev_err(priv->net_dev, "invalid mac address !!\n");
-		netdev_err(priv->net_dev, "ethernet->h_source=%pM\n", eth->h_source);
+	if (!ether_addr_equal(&priv->eth_addr[0], eth->h_source)) {
+		netdev_err(priv->net_dev,
+			   "Invalid mac address: ethernet->h_source=%pM\n",
+			   eth->h_source);
 		ret = -ENXIO;
 		goto err_kfree;
 	}
@@ -1128,7 +1069,6 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *skb)
 	/* EtherType/Length check */
 	if (*(buffer + 1) + (*buffer << 8) > 1500) {
 		/* ProtocolEAP = *(buffer+1) + (*buffer << 8); */
-		/* netdev_dbg(priv->net_dev, "Send [SNAP]Type %x\n",ProtocolEAP); */
 		/* SAP/CTL/OUI(6 byte) add */
 		*p++ = 0xAA;	/* DSAP */
 		*p++ = 0xAA;	/* SSAP */
@@ -1169,40 +1109,37 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *skb)
 		    priv->wpa.key[1].key_len == 0 &&
 		    priv->wpa.key[2].key_len == 0 &&
 		    priv->wpa.key[3].key_len == 0) {
-			pp->auth_type = cpu_to_le16((uint16_t)TYPE_AUTH);
+			pp->auth_type = cpu_to_le16(TYPE_AUTH);
 		} else {
 			if (priv->wpa.pairwise_suite == IW_AUTH_CIPHER_TKIP) {
 				michael_mic_function(&michael_mic,
-						     (uint8_t *)priv->wpa.key[0].tx_mic_key,
-						     (uint8_t *)&pp->data[0],
-						     (int)skb_len,
-						     (uint8_t)0,	/* priority */
-						     (uint8_t *)michael_mic.result);
+						     priv->wpa.key[0].tx_mic_key,
+						     &pp->data[0], skb_len,
+						     0,	michael_mic.result);
 				memcpy(p, michael_mic.result, 8);
 				length += 8;
 				skb_len += 8;
 				p += 8;
 				pp->auth_type =
-				    cpu_to_le16((uint16_t)TYPE_DATA);
+				    cpu_to_le16(TYPE_DATA);
 
 			} else if (priv->wpa.pairwise_suite ==
 				   IW_AUTH_CIPHER_CCMP) {
 				pp->auth_type =
-				    cpu_to_le16((uint16_t)TYPE_DATA);
+				    cpu_to_le16(TYPE_DATA);
 			}
 		}
 	} else {
 		if (eth_proto == ETH_P_PAE)
-			pp->auth_type = cpu_to_le16((uint16_t)TYPE_AUTH);
+			pp->auth_type = cpu_to_le16(TYPE_AUTH);
 		else
-			pp->auth_type = cpu_to_le16((uint16_t)TYPE_DATA);
+			pp->auth_type = cpu_to_le16(TYPE_DATA);
 	}
 
 	/* header value set */
 	pp->header.size =
-	    cpu_to_le16((uint16_t)
-			(sizeof(*pp) - sizeof(pp->header.size) + skb_len));
-	pp->header.event = cpu_to_le16((uint16_t)HIF_DATA_REQ);
+	    cpu_to_le16((sizeof(*pp) - sizeof(pp->header.size) + skb_len));
+	pp->header.event = cpu_to_le16(HIF_DATA_REQ);
 
 	/* tx request */
 	result = ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp) + skb_len),
@@ -1213,7 +1150,8 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *skb)
 	    priv->wpa.mic_failure.failure > 0) {
 		if (keyinfo & WPA_KEY_INFO_ERROR &&
 		    keyinfo & WPA_KEY_INFO_REQUEST) {
-			netdev_err(priv->net_dev, " MIC ERROR Report SET : %04X\n", keyinfo);
+			netdev_err(priv->net_dev,
+				   "MIC ERROR Report SET : %04X\n", keyinfo);
 			hostif_sme_enqueue(priv, SME_MIC_FAILURE_REQUEST);
 		}
 		if (priv->wpa.mic_failure.failure == 2)
@@ -1236,27 +1174,31 @@ static inline void ps_confirm_wait_inc(struct ks_wlan_private *priv)
 		atomic_inc(&priv->psstatus.confirm_wait);
 }
 
-static
-void hostif_mib_get_request(struct ks_wlan_private *priv,
-			    unsigned long mib_attribute)
+static inline void send_request_to_device(struct ks_wlan_private *priv,
+					  void *data, size_t size)
+{
+	ps_confirm_wait_inc(priv);
+	ks_wlan_hw_tx(priv, data, size, NULL, NULL);
+}
+
+static void hostif_mib_get_request(struct ks_wlan_private *priv,
+				   u32 mib_attribute)
 {
-	struct hostif_mib_get_request_t *pp;
+	struct hostif_mib_get_request *pp;
 
 	pp = hostif_generic_request(sizeof(*pp), HIF_MIB_GET_REQ);
 	if (!pp)
 		return;
 
-	pp->mib_attribute = cpu_to_le32((uint32_t)mib_attribute);
+	pp->mib_attribute = cpu_to_le32(mib_attribute);
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 }
 
-static
-void hostif_mib_set_request(struct ks_wlan_private *priv,
-			    unsigned long mib_attribute, unsigned short size,
-			    unsigned short type, void *vp)
+static void hostif_mib_set_request(struct ks_wlan_private *priv,
+				   enum mib_attribute attr,
+				   enum mib_data_type type,
+				   void *data, size_t size)
 {
 	struct hostif_mib_set_request_t *pp;
 
@@ -1267,30 +1209,52 @@ void hostif_mib_set_request(struct ks_wlan_private *priv,
 	if (!pp)
 		return;
 
-	pp->mib_attribute = cpu_to_le32((uint32_t)mib_attribute);
-	pp->mib_value.size = cpu_to_le16((uint16_t)size);
-	pp->mib_value.type = cpu_to_le16((uint16_t)type);
-	memcpy(&pp->mib_value.body, vp, size);
+	pp->mib_attribute = cpu_to_le32(attr);
+	pp->mib_value.size = cpu_to_le16(size);
+	pp->mib_value.type = cpu_to_le16(type);
+	memcpy(&pp->mib_value.body, data, size);
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp) + size), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp) + size));
+}
+
+static inline void hostif_mib_set_request_int(struct ks_wlan_private *priv,
+					      enum mib_attribute attr, int val)
+{
+	__le32 v = cpu_to_le32(val);
+	size_t size = sizeof(v);
+
+	hostif_mib_set_request(priv, attr, MIB_VALUE_TYPE_INT, &v, size);
+}
+
+static inline void hostif_mib_set_request_bool(struct ks_wlan_private *priv,
+					       enum mib_attribute attr,
+					       bool val)
+{
+	__le32 v = cpu_to_le32(val);
+	size_t size = sizeof(v);
+
+	hostif_mib_set_request(priv, attr, MIB_VALUE_TYPE_BOOL, &v, size);
+}
+
+static inline void hostif_mib_set_request_ostring(struct ks_wlan_private *priv,
+						  enum mib_attribute attr,
+						  void *data, size_t size)
+{
+	hostif_mib_set_request(priv, attr, MIB_VALUE_TYPE_OSTRING, data, size);
 }
 
 static
 void hostif_start_request(struct ks_wlan_private *priv, unsigned char mode)
 {
-	struct hostif_start_request_t *pp;
+	struct hostif_start_request *pp;
 
 	pp = hostif_generic_request(sizeof(*pp), HIF_START_REQ);
 	if (!pp)
 		return;
 
-	pp->mode = cpu_to_le16((uint16_t)mode);
+	pp->mode = cpu_to_le16(mode);
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 
 	priv->aplist.size = 0;
 	priv->scan_ind_count = 0;
@@ -1310,14 +1274,15 @@ static __le16 ks_wlan_cap(struct ks_wlan_private *priv)
 		capability &= ~(WLAN_CAPABILITY_DSSS_OFDM);
 	}
 
-	return cpu_to_le16((uint16_t)capability);
+	return cpu_to_le16(capability);
 }
 
-static void init_request(struct ks_wlan_private *priv, struct hostif_request_t *req)
+static void init_request(struct ks_wlan_private *priv,
+			 struct hostif_request *req)
 {
-	req->phy_type = cpu_to_le16((uint16_t)(priv->reg.phy_type));
-	req->cts_mode = cpu_to_le16((uint16_t)(priv->reg.cts_mode));
-	req->scan_type = cpu_to_le16((uint16_t)(priv->reg.scan_type));
+	req->phy_type = cpu_to_le16(priv->reg.phy_type);
+	req->cts_mode = cpu_to_le16(priv->reg.cts_mode);
+	req->scan_type = cpu_to_le16(priv->reg.scan_type);
 	req->rate_set.size = priv->reg.rate_set.size;
 	req->capability = ks_wlan_cap(priv);
 	memcpy(&req->rate_set.body[0], &priv->reg.rate_set.body[0],
@@ -1327,24 +1292,22 @@ static void init_request(struct ks_wlan_private *priv, struct hostif_request_t *
 static
 void hostif_ps_adhoc_set_request(struct ks_wlan_private *priv)
 {
-	struct hostif_ps_adhoc_set_request_t *pp;
+	struct hostif_ps_adhoc_set_request *pp;
 
 	pp = hostif_generic_request(sizeof(*pp), HIF_PS_ADH_SET_REQ);
 	if (!pp)
 		return;
 
 	init_request(priv, &pp->request);
-	pp->channel = cpu_to_le16((uint16_t)(priv->reg.channel));
+	pp->channel = cpu_to_le16(priv->reg.channel);
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 }
 
 static
 void hostif_infrastructure_set_request(struct ks_wlan_private *priv, int event)
 {
-	struct hostif_infrastructure_set_request_t *pp;
+	struct hostif_infrastructure_set_request *pp;
 
 	pp = hostif_generic_request(sizeof(*pp), event);
 	if (!pp)
@@ -1354,8 +1317,8 @@ void hostif_infrastructure_set_request(struct ks_wlan_private *priv, int event)
 	pp->ssid.size = priv->reg.ssid.size;
 	memcpy(&pp->ssid.body[0], &priv->reg.ssid.body[0], priv->reg.ssid.size);
 	pp->beacon_lost_count =
-	    cpu_to_le16((uint16_t)(priv->reg.beacon_lost_count));
-	pp->auth_type = cpu_to_le16((uint16_t)(priv->reg.authenticate_type));
+	    cpu_to_le16(priv->reg.beacon_lost_count);
+	pp->auth_type = cpu_to_le16(priv->reg.authenticate_type);
 
 	pp->channel_list.body[0] = 1;
 	pp->channel_list.body[1] = 8;
@@ -1377,34 +1340,30 @@ void hostif_infrastructure_set_request(struct ks_wlan_private *priv, int event)
 		pp->channel_list.size = 14;
 	}
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 }
 
 static
 void hostif_adhoc_set_request(struct ks_wlan_private *priv)
 {
-	struct hostif_adhoc_set_request_t *pp;
+	struct hostif_adhoc_set_request *pp;
 
 	pp = hostif_generic_request(sizeof(*pp), HIF_ADH_SET_REQ);
 	if (!pp)
 		return;
 
 	init_request(priv, &pp->request);
-	pp->channel = cpu_to_le16((uint16_t)(priv->reg.channel));
+	pp->channel = cpu_to_le16(priv->reg.channel);
 	pp->ssid.size = priv->reg.ssid.size;
 	memcpy(&pp->ssid.body[0], &priv->reg.ssid.body[0], priv->reg.ssid.size);
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 }
 
 static
 void hostif_adhoc_set2_request(struct ks_wlan_private *priv)
 {
-	struct hostif_adhoc_set2_request_t *pp;
+	struct hostif_adhoc_set2_request *pp;
 
 	pp = hostif_generic_request(sizeof(*pp), HIF_ADH_SET_REQ);
 	if (!pp)
@@ -1418,82 +1377,70 @@ void hostif_adhoc_set2_request(struct ks_wlan_private *priv)
 	pp->channel_list.size = 1;
 	memcpy(pp->bssid, priv->reg.bssid, ETH_ALEN);
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 }
 
 static
 void hostif_stop_request(struct ks_wlan_private *priv)
 {
-	struct hostif_stop_request_t *pp;
+	struct hostif_stop_request *pp;
 
 	pp = hostif_generic_request(sizeof(*pp), HIF_STOP_REQ);
 	if (!pp)
 		return;
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 }
 
 static
 void hostif_phy_information_request(struct ks_wlan_private *priv)
 {
-	struct hostif_phy_information_request_t *pp;
+	struct hostif_phy_information_request *pp;
 
 	pp = hostif_generic_request(sizeof(*pp), HIF_PHY_INFO_REQ);
 	if (!pp)
 		return;
 
 	if (priv->reg.phy_info_timer) {
-		pp->type = cpu_to_le16((uint16_t)TIME_TYPE);
-		pp->time = cpu_to_le16((uint16_t)(priv->reg.phy_info_timer));
+		pp->type = cpu_to_le16(TIME_TYPE);
+		pp->time = cpu_to_le16(priv->reg.phy_info_timer);
 	} else {
-		pp->type = cpu_to_le16((uint16_t)NORMAL_TYPE);
-		pp->time = cpu_to_le16((uint16_t)0);
+		pp->type = cpu_to_le16(NORMAL_TYPE);
+		pp->time = cpu_to_le16(0);
 	}
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 }
 
 static
 void hostif_power_mgmt_request(struct ks_wlan_private *priv,
-			       unsigned long mode, unsigned long wake_up,
-			       unsigned long receive_dtims)
+			       u32 mode, u32 wake_up, u32 receive_dtims)
 {
-	struct hostif_power_mgmt_request_t *pp;
+	struct hostif_power_mgmt_request *pp;
 
 	pp = hostif_generic_request(sizeof(*pp), HIF_POWER_MGMT_REQ);
 	if (!pp)
 		return;
 
-	pp->mode = cpu_to_le32((uint32_t)mode);
-	pp->wake_up = cpu_to_le32((uint32_t)wake_up);
-	pp->receive_dtims = cpu_to_le32((uint32_t)receive_dtims);
+	pp->mode = cpu_to_le32(mode);
+	pp->wake_up = cpu_to_le32(wake_up);
+	pp->receive_dtims = cpu_to_le32(receive_dtims);
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 }
 
 static
 void hostif_sleep_request(struct ks_wlan_private *priv,
 			  enum sleep_mode_type mode)
 {
-	struct hostif_sleep_request_t *pp;
+	struct hostif_sleep_request *pp;
 
 	if (mode == SLP_SLEEP) {
 		pp = hostif_generic_request(sizeof(*pp), HIF_SLEEP_REQ);
 		if (!pp)
 			return;
 
-		/* send to device request */
-		ps_confirm_wait_inc(priv);
-		ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL,
-			      NULL);
+		send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 	} else if (mode == SLP_ACTIVE) {
 		atomic_set(&priv->sleepstatus.wakeup_request, 1);
 		queue_delayed_work(priv->wq, &priv->rw_dwork, 1);
@@ -1505,10 +1452,10 @@ void hostif_sleep_request(struct ks_wlan_private *priv,
 
 static
 void hostif_bss_scan_request(struct ks_wlan_private *priv,
-			     unsigned long scan_type, uint8_t *scan_ssid,
-			     uint8_t scan_ssid_len)
+			     unsigned long scan_type, u8 *scan_ssid,
+			     u8 scan_ssid_len)
 {
-	struct hostif_bss_scan_request_t *pp;
+	struct hostif_bss_scan_request *pp;
 
 	pp = hostif_generic_request(sizeof(*pp), HIF_SCAN_REQ);
 	if (!pp)
@@ -1516,8 +1463,8 @@ void hostif_bss_scan_request(struct ks_wlan_private *priv,
 
 	pp->scan_type = scan_type;
 
-	pp->ch_time_min = cpu_to_le32((uint32_t)110);	/* default value */
-	pp->ch_time_max = cpu_to_le32((uint32_t)130);	/* default value */
+	pp->ch_time_min = cpu_to_le32(110);	/* default value */
+	pp->ch_time_max = cpu_to_le32(130);	/* default value */
 	pp->channel_list.body[0] = 1;
 	pp->channel_list.body[1] = 8;
 	pp->channel_list.body[2] = 2;
@@ -1545,9 +1492,7 @@ void hostif_bss_scan_request(struct ks_wlan_private *priv,
 		memcpy(&pp->ssid.body[0], scan_ssid, scan_ssid_len);
 	}
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 
 	priv->aplist.size = 0;
 	priv->scan_ind_count = 0;
@@ -1555,45 +1500,42 @@ void hostif_bss_scan_request(struct ks_wlan_private *priv,
 
 static
 void hostif_mic_failure_request(struct ks_wlan_private *priv,
-				unsigned short failure_count,
-				unsigned short timer)
+				u16 failure_count, u16 timer)
 {
-	struct hostif_mic_failure_request_t *pp;
+	struct hostif_mic_failure_request *pp;
 
 	pp = hostif_generic_request(sizeof(*pp), HIF_MIC_FAILURE_REQ);
 	if (!pp)
 		return;
 
-	pp->failure_count = cpu_to_le16((uint16_t)failure_count);
-	pp->timer = cpu_to_le16((uint16_t)timer);
+	pp->failure_count = cpu_to_le16(failure_count);
+	pp->timer = cpu_to_le16(timer);
 
-	/* send to device request */
-	ps_confirm_wait_inc(priv);
-	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL);
+	send_request_to_device(priv, pp, hif_align_size(sizeof(*pp)));
 }
 
 /* Device I/O Receive indicate */
 static void devio_rec_ind(struct ks_wlan_private *priv, unsigned char *p,
 			  unsigned int size)
 {
-	if (priv->device_open_status) {
-		spin_lock(&priv->dev_read_lock);	/* request spin lock */
-		priv->dev_data[atomic_read(&priv->rec_count)] = p;
-		priv->dev_size[atomic_read(&priv->rec_count)] = size;
-
-		if (atomic_read(&priv->event_count) != DEVICE_STOCK_COUNT) {
-			/* rx event count inc */
-			atomic_inc(&priv->event_count);
-		}
-		atomic_inc(&priv->rec_count);
-		if (atomic_read(&priv->rec_count) == DEVICE_STOCK_COUNT)
-			atomic_set(&priv->rec_count, 0);
+	if (!priv->is_device_open)
+		return;
 
-		wake_up_interruptible_all(&priv->devread_wait);
+	spin_lock(&priv->dev_read_lock);
+	priv->dev_data[atomic_read(&priv->rec_count)] = p;
+	priv->dev_size[atomic_read(&priv->rec_count)] = size;
 
-		/* release spin lock */
-		spin_unlock(&priv->dev_read_lock);
+	if (atomic_read(&priv->event_count) != DEVICE_STOCK_COUNT) {
+		/* rx event count inc */
+		atomic_inc(&priv->event_count);
 	}
+	atomic_inc(&priv->rec_count);
+	if (atomic_read(&priv->rec_count) == DEVICE_STOCK_COUNT)
+		atomic_set(&priv->rec_count, 0);
+
+	wake_up_interruptible_all(&priv->devread_wait);
+
+	spin_unlock(&priv->dev_read_lock);
 }
 
 void hostif_receive(struct ks_wlan_private *priv, unsigned char *p,
@@ -1604,247 +1546,183 @@ void hostif_receive(struct ks_wlan_private *priv, unsigned char *p,
 	priv->rxp = p;
 	priv->rx_size = size;
 
-	if (get_word(priv) == priv->rx_size) {	/* length check !! */
-		hostif_event_check(priv);	/* event check */
-	}
+	if (get_word(priv) == priv->rx_size)
+		hostif_event_check(priv);
 }
 
-static
-void hostif_sme_set_wep(struct ks_wlan_private *priv, int type)
+static void hostif_sme_set_wep(struct ks_wlan_private *priv, int type)
 {
-	__le32 val;
-
 	switch (type) {
 	case SME_WEP_INDEX_REQUEST:
-		val = cpu_to_le32((uint32_t)(priv->reg.wep_index));
-		hostif_mib_set_request(priv, DOT11_WEP_DEFAULT_KEY_ID,
-				       sizeof(val), MIB_VALUE_TYPE_INT, &val);
+		hostif_mib_set_request_int(priv, DOT11_WEP_DEFAULT_KEY_ID,
+					   priv->reg.wep_index);
 		break;
 	case SME_WEP_KEY1_REQUEST:
-		if (!priv->wpa.wpa_enabled)
-			hostif_mib_set_request(priv,
+		if (priv->wpa.wpa_enabled)
+			return;
+		hostif_mib_set_request_ostring(priv,
 					       DOT11_WEP_DEFAULT_KEY_VALUE1,
-					       priv->reg.wep_key[0].size,
-					       MIB_VALUE_TYPE_OSTRING,
-					       &priv->reg.wep_key[0].val[0]);
+					       &priv->reg.wep_key[0].val[0],
+					       priv->reg.wep_key[0].size);
 		break;
 	case SME_WEP_KEY2_REQUEST:
-		if (!priv->wpa.wpa_enabled)
-			hostif_mib_set_request(priv,
+		if (priv->wpa.wpa_enabled)
+			return;
+		hostif_mib_set_request_ostring(priv,
 					       DOT11_WEP_DEFAULT_KEY_VALUE2,
-					       priv->reg.wep_key[1].size,
-					       MIB_VALUE_TYPE_OSTRING,
-					       &priv->reg.wep_key[1].val[0]);
+					       &priv->reg.wep_key[1].val[0],
+					       priv->reg.wep_key[1].size);
 		break;
 	case SME_WEP_KEY3_REQUEST:
-		if (!priv->wpa.wpa_enabled)
-			hostif_mib_set_request(priv,
+		if (priv->wpa.wpa_enabled)
+			return;
+		hostif_mib_set_request_ostring(priv,
 					       DOT11_WEP_DEFAULT_KEY_VALUE3,
-					       priv->reg.wep_key[2].size,
-					       MIB_VALUE_TYPE_OSTRING,
-					       &priv->reg.wep_key[2].val[0]);
+					       &priv->reg.wep_key[2].val[0],
+					       priv->reg.wep_key[2].size);
 		break;
 	case SME_WEP_KEY4_REQUEST:
-		if (!priv->wpa.wpa_enabled)
-			hostif_mib_set_request(priv,
+		if (priv->wpa.wpa_enabled)
+			return;
+		hostif_mib_set_request_ostring(priv,
 					       DOT11_WEP_DEFAULT_KEY_VALUE4,
-					       priv->reg.wep_key[3].size,
-					       MIB_VALUE_TYPE_OSTRING,
-					       &priv->reg.wep_key[3].val[0]);
+					       &priv->reg.wep_key[3].val[0],
+					       priv->reg.wep_key[3].size);
 		break;
 	case SME_WEP_FLAG_REQUEST:
-		val = cpu_to_le32((uint32_t)(priv->reg.privacy_invoked));
-		hostif_mib_set_request(priv, DOT11_PRIVACY_INVOKED,
-				       sizeof(val), MIB_VALUE_TYPE_BOOL, &val);
+		hostif_mib_set_request_bool(priv, DOT11_PRIVACY_INVOKED,
+					    priv->reg.privacy_invoked);
 		break;
 	}
 }
 
-struct wpa_suite_t {
+struct wpa_suite {
 	__le16 size;
 	unsigned char suite[4][CIPHER_ID_LEN];
 } __packed;
 
-struct rsn_mode_t {
+struct rsn_mode {
 	__le32 rsn_mode;
 	__le16 rsn_capability;
 } __packed;
 
-static
-void hostif_sme_set_rsn(struct ks_wlan_private *priv, int type)
+static void hostif_sme_set_rsn(struct ks_wlan_private *priv, int type)
 {
-	struct wpa_suite_t wpa_suite;
-	struct rsn_mode_t rsn_mode;
-	__le32 val;
+	struct wpa_suite wpa_suite;
+	struct rsn_mode rsn_mode;
+	size_t size;
+	u32 mode;
+	const u8 *buf = NULL;
 
 	memset(&wpa_suite, 0, sizeof(wpa_suite));
 
 	switch (type) {
 	case SME_RSN_UCAST_REQUEST:
-		wpa_suite.size = cpu_to_le16((uint16_t)1);
+		wpa_suite.size = cpu_to_le16(1);
 		switch (priv->wpa.pairwise_suite) {
 		case IW_AUTH_CIPHER_NONE:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA2_NONE, CIPHER_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA_NONE, CIPHER_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				CIPHER_ID_WPA2_NONE : CIPHER_ID_WPA_NONE;
 			break;
 		case IW_AUTH_CIPHER_WEP40:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA2_WEP40, CIPHER_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA_WEP40, CIPHER_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				CIPHER_ID_WPA2_WEP40 : CIPHER_ID_WPA_WEP40;
 			break;
 		case IW_AUTH_CIPHER_TKIP:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA2_TKIP, CIPHER_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA_TKIP, CIPHER_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				CIPHER_ID_WPA2_TKIP : CIPHER_ID_WPA_TKIP;
 			break;
 		case IW_AUTH_CIPHER_CCMP:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA2_CCMP, CIPHER_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA_CCMP, CIPHER_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				CIPHER_ID_WPA2_CCMP : CIPHER_ID_WPA_CCMP;
 			break;
 		case IW_AUTH_CIPHER_WEP104:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA2_WEP104, CIPHER_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA_WEP104, CIPHER_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				CIPHER_ID_WPA2_WEP104 : CIPHER_ID_WPA_WEP104;
 			break;
 		}
 
-		hostif_mib_set_request(priv, DOT11_RSN_CONFIG_UNICAST_CIPHER,
-				       sizeof(wpa_suite.size) +
-				       CIPHER_ID_LEN *
-				       le16_to_cpu(wpa_suite.size),
-				       MIB_VALUE_TYPE_OSTRING, &wpa_suite);
+		if (buf)
+			memcpy(&wpa_suite.suite[0][0], buf, CIPHER_ID_LEN);
+		size = sizeof(wpa_suite.size) +
+		       (CIPHER_ID_LEN * le16_to_cpu(wpa_suite.size));
+		hostif_mib_set_request_ostring(priv,
+					       DOT11_RSN_CONFIG_UNICAST_CIPHER,
+					       &wpa_suite, size);
 		break;
 	case SME_RSN_MCAST_REQUEST:
 		switch (priv->wpa.group_suite) {
 		case IW_AUTH_CIPHER_NONE:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA2_NONE, CIPHER_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA_NONE, CIPHER_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				CIPHER_ID_WPA2_NONE : CIPHER_ID_WPA_NONE;
 			break;
 		case IW_AUTH_CIPHER_WEP40:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA2_WEP40, CIPHER_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA_WEP40, CIPHER_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				CIPHER_ID_WPA2_WEP40 : CIPHER_ID_WPA_WEP40;
 			break;
 		case IW_AUTH_CIPHER_TKIP:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA2_TKIP, CIPHER_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA_TKIP, CIPHER_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				CIPHER_ID_WPA2_TKIP : CIPHER_ID_WPA_TKIP;
 			break;
 		case IW_AUTH_CIPHER_CCMP:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA2_CCMP, CIPHER_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA_CCMP, CIPHER_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				CIPHER_ID_WPA2_CCMP : CIPHER_ID_WPA_CCMP;
 			break;
 		case IW_AUTH_CIPHER_WEP104:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA2_WEP104, CIPHER_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       CIPHER_ID_WPA_WEP104, CIPHER_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				CIPHER_ID_WPA2_WEP104 : CIPHER_ID_WPA_WEP104;
 			break;
 		}
-
-		hostif_mib_set_request(priv, DOT11_RSN_CONFIG_MULTICAST_CIPHER,
-				       CIPHER_ID_LEN, MIB_VALUE_TYPE_OSTRING,
-				       &wpa_suite.suite[0][0]);
+		if (buf)
+			memcpy(&wpa_suite.suite[0][0], buf, CIPHER_ID_LEN);
+		hostif_mib_set_request_ostring(priv,
+					       DOT11_RSN_CONFIG_MULTICAST_CIPHER,
+					       &wpa_suite.suite[0][0],
+					       CIPHER_ID_LEN);
 		break;
 	case SME_RSN_AUTH_REQUEST:
-		wpa_suite.size = cpu_to_le16((uint16_t)1);
+		wpa_suite.size = cpu_to_le16(1);
 		switch (priv->wpa.key_mgmt_suite) {
 		case IW_AUTH_KEY_MGMT_802_1X:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       KEY_MGMT_ID_WPA2_1X, KEY_MGMT_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       KEY_MGMT_ID_WPA_1X, KEY_MGMT_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				KEY_MGMT_ID_WPA2_1X : KEY_MGMT_ID_WPA_1X;
 			break;
 		case IW_AUTH_KEY_MGMT_PSK:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       KEY_MGMT_ID_WPA2_PSK, KEY_MGMT_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       KEY_MGMT_ID_WPA_PSK, KEY_MGMT_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				KEY_MGMT_ID_WPA2_PSK : KEY_MGMT_ID_WPA_PSK;
 			break;
 		case 0:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       KEY_MGMT_ID_WPA2_NONE, KEY_MGMT_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       KEY_MGMT_ID_WPA_NONE, KEY_MGMT_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				KEY_MGMT_ID_WPA2_NONE : KEY_MGMT_ID_WPA_NONE;
 			break;
 		case 4:
-			if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2)
-				memcpy(&wpa_suite.suite[0][0],
-				       KEY_MGMT_ID_WPA2_WPANONE,
-				       KEY_MGMT_ID_LEN);
-			else
-				memcpy(&wpa_suite.suite[0][0],
-				       KEY_MGMT_ID_WPA_WPANONE,
-				       KEY_MGMT_ID_LEN);
+			buf = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+				KEY_MGMT_ID_WPA2_WPANONE :
+				KEY_MGMT_ID_WPA_WPANONE;
 			break;
 		}
 
-		hostif_mib_set_request(priv, DOT11_RSN_CONFIG_AUTH_SUITE,
-				       sizeof(wpa_suite.size) +
-				       KEY_MGMT_ID_LEN *
-				       le16_to_cpu(wpa_suite.size),
-				       MIB_VALUE_TYPE_OSTRING, &wpa_suite);
+		if (buf)
+			memcpy(&wpa_suite.suite[0][0], buf, KEY_MGMT_ID_LEN);
+		size = sizeof(wpa_suite.size) +
+		       (KEY_MGMT_ID_LEN * le16_to_cpu(wpa_suite.size));
+		hostif_mib_set_request_ostring(priv,
+					       DOT11_RSN_CONFIG_AUTH_SUITE,
+					       &wpa_suite, size);
 		break;
 	case SME_RSN_ENABLED_REQUEST:
-		val = cpu_to_le32((uint32_t)(priv->wpa.rsn_enabled));
-		hostif_mib_set_request(priv, DOT11_RSN_ENABLED,
-				       sizeof(val), MIB_VALUE_TYPE_BOOL, &val);
+		hostif_mib_set_request_bool(priv, DOT11_RSN_ENABLED,
+					    priv->wpa.rsn_enabled);
 		break;
 	case SME_RSN_MODE_REQUEST:
-		if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) {
-			rsn_mode.rsn_mode =
-			    cpu_to_le32((uint32_t)RSN_MODE_WPA2);
-			rsn_mode.rsn_capability = cpu_to_le16((uint16_t)0);
-		} else if (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA) {
-			rsn_mode.rsn_mode =
-			    cpu_to_le32((uint32_t)RSN_MODE_WPA);
-			rsn_mode.rsn_capability = cpu_to_le16((uint16_t)0);
-		} else {
-			rsn_mode.rsn_mode =
-			    cpu_to_le32((uint32_t)RSN_MODE_NONE);
-			rsn_mode.rsn_capability = cpu_to_le16((uint16_t)0);
-		}
-		hostif_mib_set_request(priv, LOCAL_RSN_MODE, sizeof(rsn_mode),
-				       MIB_VALUE_TYPE_OSTRING, &rsn_mode);
+		mode = (priv->wpa.version == IW_AUTH_WPA_VERSION_WPA2) ?
+			RSN_MODE_WPA2 :
+			(priv->wpa.version == IW_AUTH_WPA_VERSION_WPA) ?
+			 RSN_MODE_WPA : RSN_MODE_NONE;
+		rsn_mode.rsn_mode = cpu_to_le32(mode);
+		rsn_mode.rsn_capability = cpu_to_le16(0);
+		hostif_mib_set_request_ostring(priv, LOCAL_RSN_MODE,
+					       &rsn_mode, sizeof(rsn_mode));
 		break;
 	}
 }
@@ -1884,7 +1762,7 @@ void hostif_sme_mode_setup(struct ks_wlan_private *priv)
 	/* rate mask by phy setting */
 	if (priv->reg.phy_type == D_11B_ONLY_MODE) {
 		for (i = 0; i < priv->reg.rate_set.size; i++) {
-			if (!IS_11B_RATE(priv->reg.rate_set.body[i]))
+			if (!is_11b_rate(priv->reg.rate_set.body[i]))
 				break;
 
 			if ((priv->reg.rate_set.body[i] & RATE_MASK) >= TX_RATE_5M) {
@@ -1897,10 +1775,10 @@ void hostif_sme_mode_setup(struct ks_wlan_private *priv)
 
 	} else {	/* D_11G_ONLY_MODE or D_11BG_COMPATIBLE_MODE */
 		for (i = 0; i < priv->reg.rate_set.size; i++) {
-			if (!IS_11BG_RATE(priv->reg.rate_set.body[i]))
+			if (!is_11bg_rate(priv->reg.rate_set.body[i]))
 				break;
 
-			if (IS_OFDM_EXT_RATE(priv->reg.rate_set.body[i])) {
+			if (is_ofdm_ext_rate(priv->reg.rate_set.body[i])) {
 				rate_octet[i] = priv->reg.rate_set.body[i] &
 						RATE_MASK;
 			} else {
@@ -1923,21 +1801,20 @@ void hostif_sme_mode_setup(struct ks_wlan_private *priv)
 
 	switch (priv->reg.operation_mode) {
 	case MODE_PSEUDO_ADHOC:
-		/* Pseudo Ad-Hoc mode */
 		hostif_ps_adhoc_set_request(priv);
 		break;
 	case MODE_INFRASTRUCTURE:
-		/* Infrastructure mode */
 		if (!is_valid_ether_addr((u8 *)priv->reg.bssid)) {
-			hostif_infrastructure_set_request(priv, HIF_INFRA_SET_REQ);
+			hostif_infrastructure_set_request(priv,
+							  HIF_INFRA_SET_REQ);
 		} else {
-			hostif_infrastructure_set_request(priv, HIF_INFRA_SET2_REQ);
+			hostif_infrastructure_set_request(priv,
+							  HIF_INFRA_SET2_REQ);
 			netdev_dbg(priv->net_dev,
 				   "Infra bssid = %pM\n", priv->reg.bssid);
 		}
 		break;
 	case MODE_ADHOC:
-		/* IEEE802.11 Ad-Hoc mode */
 		if (!is_valid_ether_addr((u8 *)priv->reg.bssid)) {
 			hostif_adhoc_set_request(priv);
 		} else {
@@ -1958,7 +1835,6 @@ void hostif_sme_multicast_set(struct ks_wlan_private *priv)
 	int mc_count;
 	struct netdev_hw_addr *ha;
 	char set_address[NIC_MAX_MCAST_LIST * ETH_ALEN];
-	__le32 filter_type;
 	int i = 0;
 
 	spin_lock(&priv->multicast_spin);
@@ -1966,156 +1842,117 @@ void hostif_sme_multicast_set(struct ks_wlan_private *priv)
 	memset(set_address, 0, NIC_MAX_MCAST_LIST * ETH_ALEN);
 
 	if (dev->flags & IFF_PROMISC) {
-		filter_type = cpu_to_le32((uint32_t)MCAST_FILTER_PROMISC);
-		hostif_mib_set_request(priv, LOCAL_MULTICAST_FILTER,
-				       sizeof(filter_type), MIB_VALUE_TYPE_BOOL,
-				       &filter_type);
+		hostif_mib_set_request_bool(priv, LOCAL_MULTICAST_FILTER,
+					    MCAST_FILTER_PROMISC);
 		goto spin_unlock;
 	}
 
 	if ((netdev_mc_count(dev) > NIC_MAX_MCAST_LIST) ||
 	    (dev->flags & IFF_ALLMULTI)) {
-		filter_type = cpu_to_le32((uint32_t)MCAST_FILTER_MCASTALL);
-		hostif_mib_set_request(priv, LOCAL_MULTICAST_FILTER,
-				       sizeof(filter_type), MIB_VALUE_TYPE_BOOL,
-				       &filter_type);
+		hostif_mib_set_request_bool(priv, LOCAL_MULTICAST_FILTER,
+					    MCAST_FILTER_MCASTALL);
 		goto spin_unlock;
 	}
 
 	if (priv->sme_i.sme_flag & SME_MULTICAST) {
 		mc_count = netdev_mc_count(dev);
 		netdev_for_each_mc_addr(ha, dev) {
-			memcpy(&set_address[i * ETH_ALEN], ha->addr, ETH_ALEN);
+			ether_addr_copy(&set_address[i * ETH_ALEN], ha->addr);
 			i++;
 		}
 		priv->sme_i.sme_flag &= ~SME_MULTICAST;
-		hostif_mib_set_request(priv, LOCAL_MULTICAST_ADDRESS,
-				       ETH_ALEN * mc_count,
-				       MIB_VALUE_TYPE_OSTRING,
-				       &set_address[0]);
+		hostif_mib_set_request_ostring(priv, LOCAL_MULTICAST_ADDRESS,
+					       &set_address[0],
+					       ETH_ALEN * mc_count);
 	} else {
-		filter_type = cpu_to_le32((uint32_t)MCAST_FILTER_MCAST);
 		priv->sme_i.sme_flag |= SME_MULTICAST;
-		hostif_mib_set_request(priv, LOCAL_MULTICAST_FILTER,
-				       sizeof(filter_type), MIB_VALUE_TYPE_BOOL,
-				       &filter_type);
+		hostif_mib_set_request_bool(priv, LOCAL_MULTICAST_FILTER,
+					    MCAST_FILTER_MCAST);
 	}
 
 spin_unlock:
 	spin_unlock(&priv->multicast_spin);
 }
 
-static
-void hostif_sme_power_mgmt_set(struct ks_wlan_private *priv)
+static void hostif_sme_power_mgmt_set(struct ks_wlan_private *priv)
 {
-	unsigned long mode, wake_up, receive_dtims;
+	u32 mode, wake_up, receive_dtims;
 
-	switch (priv->reg.power_mgmt) {
-	case POWER_MGMT_ACTIVE:
+	if (priv->reg.power_mgmt != POWER_MGMT_SAVE1 &&
+	    priv->reg.power_mgmt != POWER_MGMT_SAVE2) {
 		mode = POWER_ACTIVE;
 		wake_up = 0;
 		receive_dtims = 0;
-		break;
-	case POWER_MGMT_SAVE1:
-		if (priv->reg.operation_mode == MODE_INFRASTRUCTURE) {
-			mode = POWER_SAVE;
-			wake_up = 0;
-			receive_dtims = 0;
-		} else {
-			mode = POWER_ACTIVE;
-			wake_up = 0;
-			receive_dtims = 0;
-		}
-		break;
-	case POWER_MGMT_SAVE2:
-		if (priv->reg.operation_mode == MODE_INFRASTRUCTURE) {
-			mode = POWER_SAVE;
-			wake_up = 0;
-			receive_dtims = 1;
-		} else {
-			mode = POWER_ACTIVE;
-			wake_up = 0;
-			receive_dtims = 0;
-		}
-		break;
-	default:
-		mode = POWER_ACTIVE;
+	} else {
+		mode = (priv->reg.operation_mode == MODE_INFRASTRUCTURE) ?
+			POWER_SAVE : POWER_ACTIVE;
 		wake_up = 0;
-		receive_dtims = 0;
-		break;
+		receive_dtims = (priv->reg.operation_mode == MODE_INFRASTRUCTURE &&
+				 priv->reg.power_mgmt == POWER_MGMT_SAVE2);
 	}
+
 	hostif_power_mgmt_request(priv, mode, wake_up, receive_dtims);
 }
 
-static
-void hostif_sme_sleep_set(struct ks_wlan_private *priv)
+static void hostif_sme_sleep_set(struct ks_wlan_private *priv)
 {
-	switch (priv->sleep_mode) {
-	case SLP_SLEEP:
-		hostif_sleep_request(priv, priv->sleep_mode);
-		break;
-	case SLP_ACTIVE:
-		hostif_sleep_request(priv, priv->sleep_mode);
-		break;
-	default:
-		break;
-	}
+	if (priv->sleep_mode != SLP_SLEEP &&
+	    priv->sleep_mode != SLP_ACTIVE)
+		return;
+
+	hostif_sleep_request(priv, priv->sleep_mode);
 }
 
 static
 void hostif_sme_set_key(struct ks_wlan_private *priv, int type)
 {
-	__le32 val;
-
 	switch (type) {
 	case SME_SET_FLAG:
-		val = cpu_to_le32((uint32_t)(priv->reg.privacy_invoked));
-		hostif_mib_set_request(priv, DOT11_PRIVACY_INVOKED,
-				       sizeof(val), MIB_VALUE_TYPE_BOOL, &val);
+		hostif_mib_set_request_bool(priv, DOT11_PRIVACY_INVOKED,
+					    priv->reg.privacy_invoked);
 		break;
 	case SME_SET_TXKEY:
-		val = cpu_to_le32((uint32_t)(priv->wpa.txkey));
-		hostif_mib_set_request(priv, DOT11_WEP_DEFAULT_KEY_ID,
-				       sizeof(val), MIB_VALUE_TYPE_INT, &val);
+		hostif_mib_set_request_int(priv, DOT11_WEP_DEFAULT_KEY_ID,
+					   priv->wpa.txkey);
 		break;
 	case SME_SET_KEY1:
-		hostif_mib_set_request(priv, DOT11_WEP_DEFAULT_KEY_VALUE1,
-				       priv->wpa.key[0].key_len,
-				       MIB_VALUE_TYPE_OSTRING,
-				       &priv->wpa.key[0].key_val[0]);
+		hostif_mib_set_request_ostring(priv,
+					       DOT11_WEP_DEFAULT_KEY_VALUE1,
+					       &priv->wpa.key[0].key_val[0],
+					       priv->wpa.key[0].key_len);
 		break;
 	case SME_SET_KEY2:
-		hostif_mib_set_request(priv, DOT11_WEP_DEFAULT_KEY_VALUE2,
-				       priv->wpa.key[1].key_len,
-				       MIB_VALUE_TYPE_OSTRING,
-				       &priv->wpa.key[1].key_val[0]);
+		hostif_mib_set_request_ostring(priv,
+					       DOT11_WEP_DEFAULT_KEY_VALUE2,
+					       &priv->wpa.key[1].key_val[0],
+					       priv->wpa.key[1].key_len);
 		break;
 	case SME_SET_KEY3:
-		hostif_mib_set_request(priv, DOT11_WEP_DEFAULT_KEY_VALUE3,
-				       priv->wpa.key[2].key_len,
-				       MIB_VALUE_TYPE_OSTRING,
-				       &priv->wpa.key[2].key_val[0]);
+		hostif_mib_set_request_ostring(priv,
+					       DOT11_WEP_DEFAULT_KEY_VALUE3,
+					       &priv->wpa.key[2].key_val[0],
+					       priv->wpa.key[2].key_len);
 		break;
 	case SME_SET_KEY4:
-		hostif_mib_set_request(priv, DOT11_WEP_DEFAULT_KEY_VALUE4,
-				       priv->wpa.key[3].key_len,
-				       MIB_VALUE_TYPE_OSTRING,
-				       &priv->wpa.key[3].key_val[0]);
+		hostif_mib_set_request_ostring(priv,
+					       DOT11_WEP_DEFAULT_KEY_VALUE4,
+					       &priv->wpa.key[3].key_val[0],
+					       priv->wpa.key[3].key_len);
 		break;
 	case SME_SET_PMK_TSC:
-		hostif_mib_set_request(priv, DOT11_PMK_TSC,
-				       WPA_RX_SEQ_LEN, MIB_VALUE_TYPE_OSTRING,
-				       &priv->wpa.key[0].rx_seq[0]);
+		hostif_mib_set_request_ostring(priv, DOT11_PMK_TSC,
+					       &priv->wpa.key[0].rx_seq[0],
+					       WPA_RX_SEQ_LEN);
 		break;
 	case SME_SET_GMK1_TSC:
-		hostif_mib_set_request(priv, DOT11_GMK1_TSC,
-				       WPA_RX_SEQ_LEN, MIB_VALUE_TYPE_OSTRING,
-				       &priv->wpa.key[1].rx_seq[0]);
+		hostif_mib_set_request_ostring(priv, DOT11_GMK1_TSC,
+					       &priv->wpa.key[1].rx_seq[0],
+					       WPA_RX_SEQ_LEN);
 		break;
 	case SME_SET_GMK2_TSC:
-		hostif_mib_set_request(priv, DOT11_GMK2_TSC,
-				       WPA_RX_SEQ_LEN, MIB_VALUE_TYPE_OSTRING,
-				       &priv->wpa.key[2].rx_seq[0]);
+		hostif_mib_set_request_ostring(priv, DOT11_GMK2_TSC,
+					       &priv->wpa.key[2].rx_seq[0],
+					       WPA_RX_SEQ_LEN);
 		break;
 	}
 }
@@ -2123,38 +1960,34 @@ void hostif_sme_set_key(struct ks_wlan_private *priv, int type)
 static
 void hostif_sme_set_pmksa(struct ks_wlan_private *priv)
 {
-	struct pmk_cache_t {
+	struct pmk_cache {
 		__le16 size;
 		struct {
 			u8 bssid[ETH_ALEN];
 			u8 pmkid[IW_PMKID_LEN];
 		} __packed list[PMK_LIST_MAX];
 	} __packed pmkcache;
-	struct pmk_t *pmk;
-	int i;
+	struct pmk *pmk;
+	size_t size;
+	int i = 0;
 
-	i = 0;
 	list_for_each_entry(pmk, &priv->pmklist.head, list) {
-		if (i < PMK_LIST_MAX) {
-			memcpy(pmkcache.list[i].bssid, pmk->bssid, ETH_ALEN);
-			memcpy(pmkcache.list[i].pmkid, pmk->pmkid,
-			       IW_PMKID_LEN);
-			i++;
-		}
+		if (i >= PMK_LIST_MAX)
+			break;
+		ether_addr_copy(pmkcache.list[i].bssid, pmk->bssid);
+		memcpy(pmkcache.list[i].pmkid, pmk->pmkid, IW_PMKID_LEN);
+		i++;
 	}
-	pmkcache.size = cpu_to_le16((uint16_t)(priv->pmklist.size));
-	hostif_mib_set_request(priv, LOCAL_PMK,
-			       sizeof(priv->pmklist.size) + (ETH_ALEN +
-							     IW_PMKID_LEN) *
-			       (priv->pmklist.size), MIB_VALUE_TYPE_OSTRING,
-			       &pmkcache);
+	pmkcache.size = cpu_to_le16(priv->pmklist.size);
+	size = sizeof(priv->pmklist.size) +
+	       ((ETH_ALEN + IW_PMKID_LEN) * priv->pmklist.size);
+	hostif_mib_set_request_ostring(priv, LOCAL_PMK, &pmkcache, size);
 }
 
 /* execute sme */
-static
-void hostif_sme_execute(struct ks_wlan_private *priv, int event)
+static void hostif_sme_execute(struct ks_wlan_private *priv, int event)
 {
-	__le32 val;
+	u16 failure;
 
 	switch (event) {
 	case SME_START:
@@ -2165,9 +1998,8 @@ void hostif_sme_execute(struct ks_wlan_private *priv, int event)
 		hostif_sme_multicast_set(priv);
 		break;
 	case SME_MACADDRESS_SET_REQUEST:
-		hostif_mib_set_request(priv, LOCAL_CURRENTADDRESS, ETH_ALEN,
-				       MIB_VALUE_TYPE_OSTRING,
-				       &priv->eth_addr[0]);
+		hostif_mib_set_request_ostring(priv, LOCAL_CURRENTADDRESS,
+					       &priv->eth_addr[0], ETH_ALEN);
 		break;
 	case SME_BSS_SCAN_REQUEST:
 		hostif_bss_scan_request(priv, priv->reg.scan_type,
@@ -2180,18 +2012,15 @@ void hostif_sme_execute(struct ks_wlan_private *priv, int event)
 		hostif_phy_information_request(priv);
 		break;
 	case SME_MIC_FAILURE_REQUEST:
-		if (priv->wpa.mic_failure.failure == 1) {
-			hostif_mic_failure_request(priv,
-						   priv->wpa.mic_failure.failure - 1,
-						   0);
-		} else if (priv->wpa.mic_failure.failure == 2) {
-			hostif_mic_failure_request(priv,
-						   priv->wpa.mic_failure.failure - 1,
-						   priv->wpa.mic_failure.counter);
-		} else {
-			netdev_err(priv->net_dev, "SME_MIC_FAILURE_REQUEST: failure count=%u error?\n",
-				   priv->wpa.mic_failure.failure);
+		failure = priv->wpa.mic_failure.failure;
+		if (failure != 1 && failure != 2) {
+			netdev_err(priv->net_dev,
+				   "SME_MIC_FAILURE_REQUEST: failure count=%u error?\n",
+				   failure);
+			return;
 		}
+		hostif_mic_failure_request(priv, failure - 1, (failure == 1) ?
+					    0 : priv->wpa.mic_failure.counter);
 		break;
 	case SME_MIC_FAILURE_CONFIRM:
 		if (priv->wpa.mic_failure.failure == 2) {
@@ -2213,14 +2042,12 @@ void hostif_sme_execute(struct ks_wlan_private *priv, int event)
 		hostif_stop_request(priv);
 		break;
 	case SME_RTS_THRESHOLD_REQUEST:
-		val = cpu_to_le32((uint32_t)(priv->reg.rts));
-		hostif_mib_set_request(priv, DOT11_RTS_THRESHOLD,
-				       sizeof(val), MIB_VALUE_TYPE_INT, &val);
+		hostif_mib_set_request_int(priv, DOT11_RTS_THRESHOLD,
+					   priv->reg.rts);
 		break;
 	case SME_FRAGMENTATION_THRESHOLD_REQUEST:
-		val = cpu_to_le32((uint32_t)(priv->reg.fragment));
-		hostif_mib_set_request(priv, DOT11_FRAGMENTATION_THRESHOLD,
-				       sizeof(val), MIB_VALUE_TYPE_INT, &val);
+		hostif_mib_set_request_int(priv, DOT11_FRAGMENTATION_THRESHOLD,
+					   priv->reg.fragment);
 		break;
 	case SME_WEP_INDEX_REQUEST:
 	case SME_WEP_KEY1_REQUEST:
@@ -2251,26 +2078,20 @@ void hostif_sme_execute(struct ks_wlan_private *priv, int event)
 	case SME_SET_PMKSA:
 		hostif_sme_set_pmksa(priv);
 		break;
-#ifdef WPS
 	case SME_WPS_ENABLE_REQUEST:
-		hostif_mib_set_request(priv, LOCAL_WPS_ENABLE,
-				       sizeof(priv->wps.wps_enabled),
-				       MIB_VALUE_TYPE_INT,
-				       &priv->wps.wps_enabled);
+		hostif_mib_set_request_int(priv, LOCAL_WPS_ENABLE,
+					   priv->wps.wps_enabled);
 		break;
 	case SME_WPS_PROBE_REQUEST:
-		hostif_mib_set_request(priv, LOCAL_WPS_PROBE_REQ,
-				       priv->wps.ielen,
-				       MIB_VALUE_TYPE_OSTRING, priv->wps.ie);
+		hostif_mib_set_request_ostring(priv, LOCAL_WPS_PROBE_REQ,
+					       priv->wps.ie, priv->wps.ielen);
 		break;
-#endif /* WPS */
 	case SME_MODE_SET_REQUEST:
 		hostif_sme_mode_setup(priv);
 		break;
 	case SME_SET_GAIN:
-		hostif_mib_set_request(priv, LOCAL_GAIN,
-				       sizeof(priv->gain),
-				       MIB_VALUE_TYPE_OSTRING, &priv->gain);
+		hostif_mib_set_request_ostring(priv, LOCAL_GAIN,
+					       &priv->gain, sizeof(priv->gain));
 		break;
 	case SME_GET_GAIN:
 		hostif_mib_get_request(priv, LOCAL_GAIN);
@@ -2295,9 +2116,7 @@ void hostif_sme_execute(struct ks_wlan_private *priv, int event)
 		hostif_sme_sleep_set(priv);
 		break;
 	case SME_SET_REGION:
-		val = cpu_to_le32((uint32_t)(priv->region));
-		hostif_mib_set_request(priv, LOCAL_REGION,
-				       sizeof(val), MIB_VALUE_TYPE_INT, &val);
+		hostif_mib_set_request_int(priv, LOCAL_REGION, priv->region);
 		break;
 	case SME_MULTICAST_CONFIRM:
 	case SME_BSS_SCAN_CONFIRM:
@@ -2318,7 +2137,6 @@ void hostif_sme_execute(struct ks_wlan_private *priv, int event)
 	case SME_RSN_ENABLED_CONFIRM:
 	case SME_RSN_MODE_CONFIRM:
 	case SME_MODE_SET_CONFIRM:
-		break;
 	case SME_TERMINATE:
 	default:
 		break;
@@ -2343,7 +2161,7 @@ void hostif_sme_task(unsigned long dev)
 }
 
 /* send to Station Management Entity module */
-void hostif_sme_enqueue(struct ks_wlan_private *priv, unsigned short event)
+void hostif_sme_enqueue(struct ks_wlan_private *priv, u16 event)
 {
 	/* enqueue sme event */
 	if (cnt_smeqbody(priv) < (SME_EVENT_BUFF_SIZE - 1)) {
@@ -2359,7 +2177,8 @@ void hostif_sme_enqueue(struct ks_wlan_private *priv, unsigned short event)
 
 static inline void hostif_aplist_init(struct ks_wlan_private *priv)
 {
-	size_t size = LOCAL_APLIST_MAX * sizeof(struct local_ap_t);
+	size_t size = LOCAL_APLIST_MAX * sizeof(struct local_ap);
+
 	priv->aplist.size = 0;
 	memset(&priv->aplist.ap[0], 0, size);
 }
@@ -2384,7 +2203,7 @@ static inline void hostif_sme_init(struct ks_wlan_private *priv)
 static inline void hostif_wpa_init(struct ks_wlan_private *priv)
 {
 	memset(&priv->wpa, 0, sizeof(priv->wpa));
-	priv->wpa.rsn_enabled = 0;
+	priv->wpa.rsn_enabled = false;
 	priv->wpa.mic_failure.failure = 0;
 	priv->wpa.mic_failure.last_failure_time = 0;
 	priv->wpa.mic_failure.stop = 0;
diff --git a/drivers/staging/ks7010/ks_hostif.h b/drivers/staging/ks7010/ks_hostif.h
index 2f918b11b337..ca7dc8f5166c 100644
--- a/drivers/staging/ks7010/ks_hostif.h
+++ b/drivers/staging/ks7010/ks_hostif.h
@@ -1,12 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  *   Driver for KeyStream wireless LAN
  *
  *   Copyright (c) 2005-2008 KeyStream Corp.
  *   Copyright (C) 2009 Renesas Technology Corp.
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License version 2 as
- *   published by the Free Software Foundation.
  */
 
 #ifndef _KS_HOSTIF_H_
@@ -67,7 +64,7 @@ struct hostif_hdr {
 	__le16 event;
 } __packed;
 
-struct hostif_data_request_t {
+struct hostif_data_request {
 	struct hostif_hdr header;
 	__le16 auth_type;
 #define TYPE_DATA 0x0000
@@ -76,85 +73,127 @@ struct hostif_data_request_t {
 	u8 data[0];
 } __packed;
 
-struct hostif_data_indication_t {
-	struct hostif_hdr header;
-	__le16 auth_type;
-/* #define TYPE_DATA 0x0000 */
 #define TYPE_PMK1 0x0001
 #define TYPE_GMK1 0x0002
 #define TYPE_GMK2 0x0003
-	__le16 reserved;
-	u8 data[0];
-} __packed;
 
 #define CHANNEL_LIST_MAX_SIZE 14
-struct channel_list_t {
+struct channel_list {
 	u8 size;
 	u8 body[CHANNEL_LIST_MAX_SIZE];
 	u8 pad;
 } __packed;
 
-/* MIB Attribute */
-#define DOT11_MAC_ADDRESS                 0x21010100	/* MAC Address (R) */
-#define DOT11_PRODUCT_VERSION             0x31024100	/* FirmWare Version (R) */
-#define DOT11_RTS_THRESHOLD               0x21020100	/* RTS Threshold (R/W) */
-#define DOT11_FRAGMENTATION_THRESHOLD     0x21050100	/* Fragment Threshold (R/W) */
-#define DOT11_PRIVACY_INVOKED             0x15010100	/* WEP ON/OFF (W) */
-#define DOT11_WEP_DEFAULT_KEY_ID          0x15020100	/* WEP Index (W) */
-#define DOT11_WEP_DEFAULT_KEY_VALUE1      0x13020101	/* WEP Key#1(TKIP AES: PairwiseTemporalKey) (W) */
-#define DOT11_WEP_DEFAULT_KEY_VALUE2      0x13020102	/* WEP Key#2(TKIP AES: GroupKey1) (W) */
-#define DOT11_WEP_DEFAULT_KEY_VALUE3      0x13020103	/* WEP Key#3(TKIP AES: GroupKey2) (W) */
-#define DOT11_WEP_DEFAULT_KEY_VALUE4      0x13020104	/* WEP Key#4 (W) */
-#define DOT11_WEP_LIST                    0x13020100	/* WEP LIST */
-#define	DOT11_DESIRED_SSID		  0x11090100	/* SSID */
-#define	DOT11_CURRENT_CHANNEL		  0x45010100	/* channel set */
-#define	DOT11_OPERATION_RATE_SET	  0x11110100	/* rate set */
-
-#define LOCAL_AP_SEARCH_INTEAVAL          0xF1010100	/* AP search interval (R/W) */
-#define LOCAL_CURRENTADDRESS              0xF1050100	/* MAC Address change (W) */
-#define LOCAL_MULTICAST_ADDRESS           0xF1060100	/* Multicast Address (W) */
-#define LOCAL_MULTICAST_FILTER            0xF1060200	/* Multicast Address Filter enable/disable (W) */
-#define LOCAL_SEARCHED_AP_LIST            0xF1030100	/* AP list (R) */
-#define LOCAL_LINK_AP_STATUS              0xF1040100	/* Link AP status (R) */
-#define	LOCAL_PACKET_STATISTICS		  0xF1020100	/* tx,rx packets statistics */
-#define LOCAL_AP_SCAN_LIST_TYPE_SET	  0xF1030200	/* AP_SCAN_LIST_TYPE */
-
-#define DOT11_RSN_ENABLED                 0x15070100	/* WPA enable/disable (W) */
-#define LOCAL_RSN_MODE                    0x56010100	/* RSN mode WPA/WPA2 (W) */
-#define DOT11_RSN_CONFIG_MULTICAST_CIPHER 0x51040100	/* GroupKeyCipherSuite (W) */
-#define DOT11_RSN_CONFIG_UNICAST_CIPHER   0x52020100	/* PairwiseKeyCipherSuite (W) */
-#define DOT11_RSN_CONFIG_AUTH_SUITE       0x53020100	/* AuthenticationKeyManagementSuite (W) */
-#define DOT11_RSN_CONFIG_VERSION          0x51020100	/* RSN version (W) */
-#define LOCAL_RSN_CONFIG_ALL              0x5F010100	/* RSN CONFIG ALL (W) */
-#define DOT11_PMK_TSC                     0x55010100	/* PMK_TSC (W) */
-#define DOT11_GMK1_TSC                    0x55010101	/* GMK1_TSC (W) */
-#define DOT11_GMK2_TSC                    0x55010102	/* GMK2_TSC (W) */
-#define DOT11_GMK3_TSC                    0x55010103	/* GMK3_TSC */
-#define LOCAL_PMK                         0x58010100	/* Pairwise Master Key cache (W) */
-
-#define LOCAL_REGION                      0xF10A0100	/* Region setting */
-
-#ifdef WPS
-#define LOCAL_WPS_ENABLE                  0xF10B0100	/* WiFi Protected Setup */
-#define LOCAL_WPS_PROBE_REQ               0xF10C0100	/* WPS Probe Request */
-#endif /* WPS */
-
-#define LOCAL_GAIN                        0xF10D0100	/* Carrer sense threshold for demo ato show */
-#define LOCAL_EEPROM_SUM                  0xF10E0100	/* EEPROM checksum information */
-
-struct hostif_mib_get_request_t {
+/**
+ * enum mib_attribute - Management Information Base attribute
+ * Attribute value used for accessing and updating MIB
+ *
+ * @DOT11_MAC_ADDRESS: MAC Address (R)
+ * @DOT11_PRODUCT_VERSION: FirmWare Version (R)
+ * @DOT11_RTS_THRESHOLD: RTS Threshold (R/W)
+ * @DOT11_FRAGMENTATION_THRESHOLD: Fragment Threshold (R/W)
+ * @DOT11_PRIVACY_INVOKED: WEP ON/OFF (W)
+ * @DOT11_WEP_DEFAULT_KEY_ID: WEP Index (W)
+ * @DOT11_WEP_DEFAULT_KEY_VALUE1: WEP Key#1(TKIP AES: PairwiseTemporalKey) (W)
+ * @DOT11_WEP_DEFAULT_KEY_VALUE2: WEP Key#2(TKIP AES: GroupKey1) (W)
+ * @DOT11_WEP_DEFAULT_KEY_VALUE3: WEP Key#3(TKIP AES: GroupKey2) (W)
+ * @DOT11_WEP_DEFAULT_KEY_VALUE4: WEP Key#4 (W)
+ * @DOT11_WEP_LIST: WEP LIST
+ * @DOT11_DESIRED_SSID: SSID
+ * @DOT11_CURRENT_CHANNEL: channel set
+ * @DOT11_OPERATION_RATE_SET: rate set
+ * @LOCAL_AP_SEARCH_INTERVAL: AP search interval (R/W)
+ * @LOCAL_CURRENTADDRESS: MAC Address change (W)
+ * @LOCAL_MULTICAST_ADDRESS: Multicast Address (W)
+ * @LOCAL_MULTICAST_FILTER: Multicast Address Filter enable/disable (W)
+ * @LOCAL_SEARCHED_AP_LIST: AP list (R)
+ * @LOCAL_LINK_AP_STATUS: Link AP status (R)
+ * @LOCAL_PACKET_STATISTICS: tx,rx packets statistics
+ * @LOCAL_AP_SCAN_LIST_TYPE_SET: AP_SCAN_LIST_TYPE
+ * @DOT11_RSN_ENABLED: WPA enable/disable (W)
+ * @LOCAL_RSN_MODE: RSN mode WPA/WPA2 (W)
+ * @DOT11_RSN_CONFIG_MULTICAST_CIPHER: GroupKeyCipherSuite (W)
+ * @DOT11_RSN_CONFIG_UNICAST_CIPHER: PairwiseKeyCipherSuite (W)
+ * @DOT11_RSN_CONFIG_AUTH_SUITE: AuthenticationKeyManagementSuite (W)
+ * @DOT11_RSN_CONFIG_VERSION: RSN version (W)
+ * @LOCAL_RSN_CONFIG_ALL: RSN CONFIG ALL (W)
+ * @DOT11_PMK_TSC: PMK_TSC (W)
+ * @DOT11_GMK1_TSC: GMK1_TSC (W)
+ * @DOT11_GMK2_TSC: GMK2_TSC (W)
+ * @DOT11_GMK3_TSC: GMK3_TSC
+ * @LOCAL_PMK: Pairwise Master Key cache (W)
+ * @LOCAL_REGION: Region setting
+ * @LOCAL_WPS_ENABLE: WiFi Protected Setup
+ * @LOCAL_WPS_PROBE_REQ: WPS Probe Request
+ * @LOCAL_GAIN: Carrer sense threshold for demo ato show
+ * @LOCAL_EEPROM_SUM: EEPROM checksum information
+ */
+enum mib_attribute {
+	DOT11_MAC_ADDRESS                 = 0x21010100,
+	DOT11_PRODUCT_VERSION             = 0x31024100,
+	DOT11_RTS_THRESHOLD               = 0x21020100,
+	DOT11_FRAGMENTATION_THRESHOLD     = 0x21050100,
+	DOT11_PRIVACY_INVOKED             = 0x15010100,
+	DOT11_WEP_DEFAULT_KEY_ID          = 0x15020100,
+	DOT11_WEP_DEFAULT_KEY_VALUE1      = 0x13020101,
+	DOT11_WEP_DEFAULT_KEY_VALUE2      = 0x13020102,
+	DOT11_WEP_DEFAULT_KEY_VALUE3      = 0x13020103,
+	DOT11_WEP_DEFAULT_KEY_VALUE4      = 0x13020104,
+	DOT11_WEP_LIST                    = 0x13020100,
+	DOT11_DESIRED_SSID		  = 0x11090100,
+	DOT11_CURRENT_CHANNEL		  = 0x45010100,
+	DOT11_OPERATION_RATE_SET	  = 0x11110100,
+	LOCAL_AP_SEARCH_INTERVAL          = 0xF1010100,
+	LOCAL_CURRENTADDRESS              = 0xF1050100,
+	LOCAL_MULTICAST_ADDRESS           = 0xF1060100,
+	LOCAL_MULTICAST_FILTER            = 0xF1060200,
+	LOCAL_SEARCHED_AP_LIST            = 0xF1030100,
+	LOCAL_LINK_AP_STATUS              = 0xF1040100,
+	LOCAL_PACKET_STATISTICS		  = 0xF1020100,
+	LOCAL_AP_SCAN_LIST_TYPE_SET	  = 0xF1030200,
+	DOT11_RSN_ENABLED                 = 0x15070100,
+	LOCAL_RSN_MODE                    = 0x56010100,
+	DOT11_RSN_CONFIG_MULTICAST_CIPHER = 0x51040100,
+	DOT11_RSN_CONFIG_UNICAST_CIPHER   = 0x52020100,
+	DOT11_RSN_CONFIG_AUTH_SUITE       = 0x53020100,
+	DOT11_RSN_CONFIG_VERSION          = 0x51020100,
+	LOCAL_RSN_CONFIG_ALL              = 0x5F010100,
+	DOT11_PMK_TSC                     = 0x55010100,
+	DOT11_GMK1_TSC                    = 0x55010101,
+	DOT11_GMK2_TSC                    = 0x55010102,
+	DOT11_GMK3_TSC                    = 0x55010103,
+	LOCAL_PMK                         = 0x58010100,
+	LOCAL_REGION                      = 0xF10A0100,
+	LOCAL_WPS_ENABLE                  = 0xF10B0100,
+	LOCAL_WPS_PROBE_REQ               = 0xF10C0100,
+	LOCAL_GAIN                        = 0xF10D0100,
+	LOCAL_EEPROM_SUM                  = 0xF10E0100
+};
+
+struct hostif_mib_get_request {
 	struct hostif_hdr header;
 	__le32 mib_attribute;
 } __packed;
 
-struct hostif_mib_value_t {
+/**
+ * enum mib_data_type - Message Information Base data type.
+ * @MIB_VALUE_TYPE_NULL: NULL type
+ * @MIB_VALUE_TYPE_INT: INTEGER type
+ * @MIB_VALUE_TYPE_BOOL: BOOL type
+ * @MIB_VALUE_TYPE_COUNT32: unused
+ * @MIB_VALUE_TYPE_OSTRING: Chunk of memory
+ */
+enum mib_data_type {
+	MIB_VALUE_TYPE_NULL = 0,
+	MIB_VALUE_TYPE_INT,
+	MIB_VALUE_TYPE_BOOL,
+	MIB_VALUE_TYPE_COUNT32,
+	MIB_VALUE_TYPE_OSTRING
+};
+
+struct hostif_mib_value {
 	__le16 size;
 	__le16 type;
-#define MIB_VALUE_TYPE_NULL     0
-#define MIB_VALUE_TYPE_INT      1
-#define MIB_VALUE_TYPE_BOOL     2
-#define MIB_VALUE_TYPE_COUNT32  3
-#define MIB_VALUE_TYPE_OSTRING  4
 	u8 body[0];
 } __packed;
 
@@ -166,22 +205,16 @@ struct hostif_mib_get_confirm_t {
 #define MIB_READ_ONLY  2
 #define MIB_WRITE_ONLY 3
 	__le32 mib_attribute;
-	struct hostif_mib_value_t mib_value;
+	struct hostif_mib_value mib_value;
 } __packed;
 
 struct hostif_mib_set_request_t {
 	struct hostif_hdr header;
 	__le32 mib_attribute;
-	struct hostif_mib_value_t mib_value;
-} __packed;
-
-struct hostif_mib_set_confirm_t {
-	struct hostif_hdr header;
-	__le32 mib_status;
-	__le32 mib_attribute;
+	struct hostif_mib_value mib_value;
 } __packed;
 
-struct hostif_power_mgmt_request_t {
+struct hostif_power_mgmt_request {
 	struct hostif_hdr header;
 	__le32 mode;
 #define POWER_ACTIVE  1
@@ -206,12 +239,7 @@ enum power_mgmt_mode_type {
 /* #define	RESULT_ALREADY_RUNNING    3 */
 #define	RESULT_ALREADY_RUNNING    7
 
-struct hostif_power_mgmt_confirm_t {
-	struct hostif_hdr header;
-	__le16 result_code;
-} __packed;
-
-struct hostif_start_request_t {
+struct hostif_start_request {
 	struct hostif_hdr header;
 	__le16 mode;
 #define MODE_PSEUDO_ADHOC   0
@@ -220,43 +248,38 @@ struct hostif_start_request_t {
 #define MODE_ADHOC          3
 } __packed;
 
-struct hostif_start_confirm_t {
-	struct hostif_hdr header;
-	__le16 result_code;
-} __packed;
-
-struct ssid_t {
+struct ssid {
 	u8 size;
 	u8 body[IEEE80211_MAX_SSID_LEN];
 	u8 ssid_pad;
 } __packed;
 
 #define RATE_SET_MAX_SIZE 16
-struct rate_set8_t {
+struct rate_set8 {
 	u8 size;
 	u8 body[8];
 	u8 rate_pad;
 } __packed;
 
-struct fh_parms_t {
+struct fh_parms {
 	__le16 dwell_time;
 	u8 hop_set;
 	u8 hop_pattern;
 	u8 hop_index;
 } __packed;
 
-struct ds_parms_t {
+struct ds_parms {
 	u8 channel;
 } __packed;
 
-struct cf_parms_t {
+struct cf_parms {
 	u8 count;
 	u8 period;
 	__le16 max_duration;
 	__le16 dur_remaining;
 } __packed;
 
-struct ibss_parms_t {
+struct ibss_parms {
 	__le16 atim_window;
 } __packed;
 
@@ -270,13 +293,13 @@ struct erp_params_t {
 	u8 erp_info;
 } __packed;
 
-struct rate_set16_t {
+struct rate_set16 {
 	u8 size;
 	u8 body[16];
 	u8 rate_pad;
 } __packed;
 
-struct ap_info_t {
+struct ap_info {
 	u8 bssid[6];	/* +00 */
 	u8 rssi;	/* +06 */
 	u8 sq;	/* +07 */
@@ -291,7 +314,7 @@ struct ap_info_t {
 	/* +1032 */
 } __packed;
 
-struct link_ap_info_t {
+struct link_ap_info {
 	u8 bssid[6];	/* +00 */
 	u8 rssi;	/* +06 */
 	u8 sq;	/* +07 */
@@ -299,14 +322,14 @@ struct link_ap_info_t {
 	u8 pad0;	/* +09 */
 	__le16 beacon_period;	/* +10 */
 	__le16 capability;	/* +12 */
-	struct rate_set8_t rate_set;	/* +14 */
-	struct fh_parms_t fh_parameter;	/* +24 */
-	struct ds_parms_t ds_parameter;	/* +29 */
-	struct cf_parms_t cf_parameter;	/* +30 */
-	struct ibss_parms_t ibss_parameter;	/* +36 */
+	struct rate_set8 rate_set;	/* +14 */
+	struct fh_parms fh_parameter;	/* +24 */
+	struct ds_parms ds_parameter;	/* +29 */
+	struct cf_parms cf_parameter;	/* +30 */
+	struct ibss_parms ibss_parameter;	/* +36 */
 	struct erp_params_t erp_parameter;	/* +38 */
 	u8 pad1;	/* +39 */
-	struct rate_set8_t ext_rate_set;	/* +40 */
+	struct rate_set8 ext_rate_set;	/* +40 */
 	u8 DTIM_period;	/* +50 */
 	u8 rsn_mode;	/* +51 */
 #define RSN_MODE_NONE	0
@@ -318,23 +341,13 @@ struct link_ap_info_t {
 	} __packed rsn;
 } __packed;
 
-struct hostif_connect_indication_t {
-	struct hostif_hdr header;
-	__le16 connect_code;
 #define RESULT_CONNECT    0
 #define RESULT_DISCONNECT 1
-	struct link_ap_info_t link_ap_info;
-} __packed;
 
-struct hostif_stop_request_t {
+struct hostif_stop_request {
 	struct hostif_hdr header;
 } __packed;
 
-struct hostif_stop_confirm_t {
-	struct hostif_hdr header;
-	__le16 result_code;
-} __packed;
-
 #define D_11B_ONLY_MODE		0
 #define D_11G_ONLY_MODE		1
 #define D_11BG_COMPATIBLE_MODE	2
@@ -343,98 +356,78 @@ struct hostif_stop_confirm_t {
 #define CTS_MODE_FALSE	0
 #define CTS_MODE_TRUE	1
 
-struct hostif_request_t {
+struct hostif_request {
 	__le16 phy_type;
 	__le16 cts_mode;
 	__le16 scan_type;
 	__le16 capability;
-	struct rate_set16_t rate_set;
+	struct rate_set16 rate_set;
 } __packed;
 
 /**
- * struct hostif_ps_adhoc_set_request_t - pseudo adhoc mode
+ * struct hostif_ps_adhoc_set_request - pseudo adhoc mode
  * @capability: bit5  : preamble
  *              bit6  : pbcc - Not supported always 0
  *              bit10 : ShortSlotTime
  *              bit13 : DSSS-OFDM - Not supported always 0
  */
-struct hostif_ps_adhoc_set_request_t {
+struct hostif_ps_adhoc_set_request {
 	struct hostif_hdr header;
-	struct hostif_request_t request;
+	struct hostif_request request;
 	__le16 channel;
 } __packed;
 
-struct hostif_ps_adhoc_set_confirm_t {
-	struct hostif_hdr header;
-	__le16 result_code;
-} __packed;
-
 #define AUTH_TYPE_OPEN_SYSTEM 0
 #define AUTH_TYPE_SHARED_KEY  1
 
 /**
- * struct hostif_infrastructure_set_request_t
+ * struct hostif_infrastructure_set_request
  * @capability: bit5  : preamble
  *              bit6  : pbcc - Not supported always 0
  *              bit10 : ShortSlotTime
  *              bit13 : DSSS-OFDM - Not supported always 0
  */
-struct hostif_infrastructure_set_request_t {
+struct hostif_infrastructure_set_request {
 	struct hostif_hdr header;
-	struct hostif_request_t request;
-	struct ssid_t ssid;
+	struct hostif_request request;
+	struct ssid ssid;
 	__le16 beacon_lost_count;
 	__le16 auth_type;
-	struct channel_list_t channel_list;
+	struct channel_list channel_list;
 	u8 bssid[ETH_ALEN];
 } __packed;
 
-struct hostif_infrastructure_set_confirm_t {
-	struct hostif_hdr header;
-	__le16 result_code;
-} __packed;
-
 /**
- * struct hostif_adhoc_set_request_t
+ * struct hostif_adhoc_set_request
  * @capability: bit5  : preamble
  *              bit6  : pbcc - Not supported always 0
  *              bit10 : ShortSlotTime
  *              bit13 : DSSS-OFDM - Not supported always 0
  */
-struct hostif_adhoc_set_request_t {
+struct hostif_adhoc_set_request {
 	struct hostif_hdr header;
-	struct hostif_request_t request;
-	struct ssid_t ssid;
+	struct hostif_request request;
+	struct ssid ssid;
 	__le16 channel;
 } __packed;
 
 /**
- * struct hostif_adhoc_set2_request_t
+ * struct hostif_adhoc_set2_request
  * @capability: bit5  : preamble
  *              bit6  : pbcc - Not supported always 0
  *              bit10 : ShortSlotTime
  *              bit13 : DSSS-OFDM - Not supported always 0
  */
-struct hostif_adhoc_set2_request_t {
+struct hostif_adhoc_set2_request {
 	struct hostif_hdr header;
-	struct hostif_request_t request;
+	struct hostif_request request;
 	__le16 reserved;
-	struct ssid_t ssid;
-	struct channel_list_t channel_list;
+	struct ssid ssid;
+	struct channel_list channel_list;
 	u8 bssid[ETH_ALEN];
 } __packed;
 
-struct hostif_adhoc_set_confirm_t {
-	struct hostif_hdr header;
-	__le16 result_code;
-} __packed;
-
-struct last_associate_t {
-	u8 type;
-	u8 status;
-} __packed;
-
-struct association_request_t {
+struct association_request {
 	u8 type;
 	u8 pad;
 	__le16 capability;
@@ -443,7 +436,7 @@ struct association_request_t {
 	__le16 req_ies_size;
 } __packed;
 
-struct association_response_t {
+struct association_response {
 	u8 type;
 	u8 pad;
 	__le16 capability;
@@ -452,15 +445,7 @@ struct association_response_t {
 	__le16 resp_ies_size;
 } __packed;
 
-struct hostif_associate_indication_t {
-	struct hostif_hdr header;
-	struct association_request_t assoc_req;
-	struct association_response_t assoc_resp;
-	/* followed by (req_ies_size + resp_ies_size) octets of data */
-	/* reqIEs data *//* respIEs data */
-} __packed;
-
-struct hostif_bss_scan_request_t {
+struct hostif_bss_scan_request {
 	struct hostif_hdr header;
 	u8 scan_type;
 #define ACTIVE_SCAN  0
@@ -468,17 +453,11 @@ struct hostif_bss_scan_request_t {
 	u8 pad[3];
 	__le32 ch_time_min;
 	__le32 ch_time_max;
-	struct channel_list_t channel_list;
-	struct ssid_t ssid;
+	struct channel_list channel_list;
+	struct ssid ssid;
 } __packed;
 
-struct hostif_bss_scan_confirm_t {
-	struct hostif_hdr header;
-	__le16 result_code;
-	__le16 reserved;
-} __packed;
-
-struct hostif_phy_information_request_t {
+struct hostif_phy_information_request {
 	struct hostif_hdr header;
 	__le16 type;
 #define NORMAL_TYPE	0
@@ -486,43 +465,21 @@ struct hostif_phy_information_request_t {
 	__le16 time;	/* unit 100ms */
 } __packed;
 
-struct hostif_phy_information_confirm_t {
-	struct hostif_hdr header;
-	u8 rssi;
-	u8 sq;
-	u8 noise;
-	u8 link_speed;
-	__le32 tx_frame;
-	__le32 rx_frame;
-	__le32 tx_error;
-	__le32 rx_error;
-} __packed;
-
 enum sleep_mode_type {
 	SLP_ACTIVE,
 	SLP_SLEEP
 };
 
-struct hostif_sleep_request_t {
+struct hostif_sleep_request {
 	struct hostif_hdr header;
 } __packed;
 
-struct hostif_sleep_confirm_t {
-	struct hostif_hdr header;
-	__le16 result_code;
-} __packed;
-
-struct hostif_mic_failure_request_t {
+struct hostif_mic_failure_request {
 	struct hostif_hdr header;
 	__le16 failure_count;
 	__le16 timer;
 } __packed;
 
-struct hostif_mic_failure_confirm_t {
-	struct hostif_hdr header;
-	__le16 result_code;
-} __packed;
-
 #define BASIC_RATE	0x80
 #define RATE_MASK	0x7F
 
@@ -541,34 +498,54 @@ struct hostif_mic_failure_confirm_t {
 #define TX_RATE_FIXED		5
 
 /* 11b rate */
-#define TX_RATE_1M	(uint8_t)(10 / 5)	/* 11b 11g basic rate */
-#define TX_RATE_2M	(uint8_t)(20 / 5)	/* 11b 11g basic rate */
-#define TX_RATE_5M	(uint8_t)(55 / 5)	/* 11g basic rate */
-#define TX_RATE_11M	(uint8_t)(110 / 5)	/* 11g basic rate */
+#define TX_RATE_1M	(u8)(10 / 5)	/* 11b 11g basic rate */
+#define TX_RATE_2M	(u8)(20 / 5)	/* 11b 11g basic rate */
+#define TX_RATE_5M	(u8)(55 / 5)	/* 11g basic rate */
+#define TX_RATE_11M	(u8)(110 / 5)	/* 11g basic rate */
 
 /* 11g rate */
-#define TX_RATE_6M	(uint8_t)(60 / 5)	/* 11g basic rate */
-#define TX_RATE_12M	(uint8_t)(120 / 5)	/* 11g basic rate */
-#define TX_RATE_24M	(uint8_t)(240 / 5)	/* 11g basic rate */
-#define TX_RATE_9M	(uint8_t)(90 / 5)
-#define TX_RATE_18M	(uint8_t)(180 / 5)
-#define TX_RATE_36M	(uint8_t)(360 / 5)
-#define TX_RATE_48M	(uint8_t)(480 / 5)
-#define TX_RATE_54M	(uint8_t)(540 / 5)
-
-#define IS_11B_RATE(A) (((A & RATE_MASK) == TX_RATE_1M) || ((A & RATE_MASK) == TX_RATE_2M) || \
-			((A & RATE_MASK) == TX_RATE_5M) || ((A & RATE_MASK) == TX_RATE_11M))
+#define TX_RATE_6M	(u8)(60 / 5)	/* 11g basic rate */
+#define TX_RATE_12M	(u8)(120 / 5)	/* 11g basic rate */
+#define TX_RATE_24M	(u8)(240 / 5)	/* 11g basic rate */
+#define TX_RATE_9M	(u8)(90 / 5)
+#define TX_RATE_18M	(u8)(180 / 5)
+#define TX_RATE_36M	(u8)(360 / 5)
+#define TX_RATE_48M	(u8)(480 / 5)
+#define TX_RATE_54M	(u8)(540 / 5)
+
+static inline bool is_11b_rate(u8 rate)
+{
+	return (((rate & RATE_MASK) == TX_RATE_1M) ||
+		((rate & RATE_MASK) == TX_RATE_2M) ||
+		((rate & RATE_MASK) == TX_RATE_5M) ||
+		((rate & RATE_MASK) == TX_RATE_11M));
+}
 
-#define IS_OFDM_RATE(A) (((A & RATE_MASK) == TX_RATE_6M) || ((A & RATE_MASK) == TX_RATE_12M) || \
-			 ((A & RATE_MASK) == TX_RATE_24M) || ((A & RATE_MASK) == TX_RATE_9M) || \
-			 ((A & RATE_MASK) == TX_RATE_18M) || ((A & RATE_MASK) == TX_RATE_36M) || \
-			 ((A & RATE_MASK) == TX_RATE_48M) || ((A & RATE_MASK) == TX_RATE_54M))
+static inline bool is_ofdm_rate(u8 rate)
+{
+	return (((rate & RATE_MASK) == TX_RATE_6M)  ||
+		((rate & RATE_MASK) == TX_RATE_12M) ||
+		((rate & RATE_MASK) == TX_RATE_24M) ||
+		((rate & RATE_MASK) == TX_RATE_9M)  ||
+		((rate & RATE_MASK) == TX_RATE_18M) ||
+		((rate & RATE_MASK) == TX_RATE_36M) ||
+		((rate & RATE_MASK) == TX_RATE_48M) ||
+		((rate & RATE_MASK) == TX_RATE_54M));
+}
 
-#define IS_11BG_RATE(A) (IS_11B_RATE(A) || IS_OFDM_RATE(A))
+static inline bool is_11bg_rate(u8 rate)
+{
+	return (is_11b_rate(rate) || is_ofdm_rate(rate));
+}
 
-#define IS_OFDM_EXT_RATE(A) (((A & RATE_MASK) == TX_RATE_9M) || ((A & RATE_MASK) == TX_RATE_18M) || \
-			     ((A & RATE_MASK) == TX_RATE_36M) || ((A & RATE_MASK) == TX_RATE_48M) || \
-			     ((A & RATE_MASK) == TX_RATE_54M))
+static inline bool is_ofdm_ext_rate(u8 rate)
+{
+	return (((rate & RATE_MASK) == TX_RATE_9M)  ||
+		((rate & RATE_MASK) == TX_RATE_18M) ||
+		((rate & RATE_MASK) == TX_RATE_36M) ||
+		((rate & RATE_MASK) == TX_RATE_48M) ||
+		((rate & RATE_MASK) == TX_RATE_54M));
+}
 
 enum connect_status_type {
 	CONNECT_STATUS,
@@ -588,19 +565,25 @@ enum multicast_filter_type {
 
 #define NIC_MAX_MCAST_LIST 32
 
-/* macro function */
 #define HIF_EVENT_MASK 0xE800
-#define IS_HIF_IND(_EVENT)  ((_EVENT & HIF_EVENT_MASK) == 0xE800  && \
-			     ((_EVENT & ~HIF_EVENT_MASK) == 0x0001 || \
-			     (_EVENT & ~HIF_EVENT_MASK) == 0x0006 || \
-			     (_EVENT & ~HIF_EVENT_MASK) == 0x000C || \
-			     (_EVENT & ~HIF_EVENT_MASK) == 0x0011 || \
-			     (_EVENT & ~HIF_EVENT_MASK) == 0x0012))
-
-#define IS_HIF_CONF(_EVENT) ((_EVENT & HIF_EVENT_MASK) == 0xE800  && \
-			     (_EVENT & ~HIF_EVENT_MASK) > 0x0000  && \
-			     (_EVENT & ~HIF_EVENT_MASK) < 0x0012  && \
-			     !IS_HIF_IND(_EVENT))
+
+static inline bool is_hif_ind(unsigned short event)
+{
+	return (((event & HIF_EVENT_MASK) == HIF_EVENT_MASK) &&
+		(((event & ~HIF_EVENT_MASK) == 0x0001) ||
+		 ((event & ~HIF_EVENT_MASK) == 0x0006) ||
+		 ((event & ~HIF_EVENT_MASK) == 0x000C) ||
+		 ((event & ~HIF_EVENT_MASK) == 0x0011) ||
+		 ((event & ~HIF_EVENT_MASK) == 0x0012)));
+}
+
+static inline bool is_hif_conf(unsigned short event)
+{
+	return (((event & HIF_EVENT_MASK) == HIF_EVENT_MASK) &&
+		((event & ~HIF_EVENT_MASK) > 0x0000) &&
+		((event & ~HIF_EVENT_MASK) < 0x0012) &&
+		!is_hif_ind(event));
+}
 
 #ifdef __KERNEL__
 
@@ -610,7 +593,7 @@ enum multicast_filter_type {
 int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *skb);
 void hostif_receive(struct ks_wlan_private *priv, unsigned char *p,
 		    unsigned int size);
-void hostif_sme_enqueue(struct ks_wlan_private *priv, uint16_t event);
+void hostif_sme_enqueue(struct ks_wlan_private *priv, u16 event);
 int hostif_init(struct ks_wlan_private *priv);
 void hostif_exit(struct ks_wlan_private *priv);
 int ks_wlan_hw_tx(struct ks_wlan_private *priv, void *p, unsigned long size,
@@ -622,19 +605,11 @@ void send_packet_complete(struct ks_wlan_private *priv, struct sk_buff *skb);
 void ks_wlan_hw_wakeup_request(struct ks_wlan_private *priv);
 int ks_wlan_hw_power_save(struct ks_wlan_private *priv);
 
-static
-inline int hif_align_size(int size)
+#define KS7010_SIZE_ALIGNMENT	32
+
+static inline size_t hif_align_size(size_t size)
 {
-#ifdef	KS_ATOM
-	if (size < 1024)
-		size = 1024;
-#endif
-#ifdef	DEVICE_ALIGNMENT
-	return (size % DEVICE_ALIGNMENT) ? size + DEVICE_ALIGNMENT -
-	    (size % DEVICE_ALIGNMENT) : size;
-#else
-	return size;
-#endif
+	return ALIGN(size, KS7010_SIZE_ALIGNMENT);
 }
 
 #endif /* __KERNEL__ */
diff --git a/drivers/staging/ks7010/ks_wlan.h b/drivers/staging/ks7010/ks_wlan.h
index 1b7036c32d1c..7aaf8d780939 100644
--- a/drivers/staging/ks7010/ks_wlan.h
+++ b/drivers/staging/ks7010/ks_wlan.h
@@ -1,60 +1,42 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  *   Driver for KeyStream IEEE802.11 b/g wireless LAN cards.
  *
  *   Copyright (C) 2006-2008 KeyStream Corp.
  *   Copyright (C) 2009 Renesas Technology Corp.
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License version 2 as
- *   published by the Free Software Foundation.
  */
 
 #ifndef _KS_WLAN_H
 #define _KS_WLAN_H
 
-#define WPS
-
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-#include <linux/spinlock.h>	/* spinlock_t */
-#include <linux/sched.h>	/* wait_queue_head_t */
-#include <linux/types.h>	/* pid_t */
-#include <linux/netdevice.h>	/* struct net_device_stats,  struct sk_buff */
-#include <linux/etherdevice.h>
+#include <linux/atomic.h>
+#include <linux/circ_buf.h>
+#include <linux/completion.h>
+#include <linux/netdevice.h>
+#include <linux/sched.h>
+#include <linux/spinlock.h>
 #include <linux/wireless.h>
-#include <linux/atomic.h>	/* struct atomic_t */
-#include <linux/timer.h>	/* struct timer_list */
-#include <linux/string.h>
-#include <linux/completion.h>	/* struct completion */
-#include <linux/workqueue.h>
-
-#include <linux/io.h>
-
-#include "ks7010_sdio.h"
 
 struct ks_wlan_parameter {
-	u8 operation_mode;	/* Operation Mode */
-	u8 channel;	/*  Channel */
-	u8 tx_rate;	/*  Transmit Rate */
+	u8 operation_mode;
+	u8 channel;
+	u8 tx_rate;
 	struct {
 		u8 size;
 		u8 body[16];
 	} rate_set;
-	u8 bssid[ETH_ALEN];	/* BSSID */
+	u8 bssid[ETH_ALEN];
 	struct {
 		u8 size;
 		u8 body[32 + 1];
-	} ssid;	/*  SSID */
-	u8 preamble;	/*  Preamble */
+	} ssid;
+	u8 preamble;
 	u8 power_mgmt;
-	u32 scan_type;	/*  AP List Scan Type */
-#define BEACON_LOST_COUNT_MIN 0
+	u32 scan_type;
 #define BEACON_LOST_COUNT_MAX 65535
-	u32 beacon_lost_count;	/*  Beacon Lost Count */
-	u32 rts;	/*  RTS Threashold */
-	u32 fragment;	/*  Fragmentation Threshold */
+	u32 beacon_lost_count;
+	u32 rts;
+	u32 fragment;
 	u32 privacy_invoked;
 	u32 wep_index;
 	struct {
@@ -62,9 +44,9 @@ struct ks_wlan_parameter {
 		u8 val[13 * 2 + 1];
 	} wep_key[4];
 	u16 authenticate_type;
-	u16 phy_type;	/* 11b/11g/11bg mode type */
-	u16 cts_mode;	/* for 11g/11bg mode cts mode */
-	u16 phy_info_timer;	/* phy information timer */
+	u16 phy_type;
+	u16 cts_mode;
+	u16 phy_info_timer;
 };
 
 enum {
@@ -133,10 +115,8 @@ enum {
 	SME_RSN_AUTH_REQUEST,
 	SME_RSN_ENABLED_REQUEST,
 	SME_RSN_MODE_REQUEST,
-#ifdef WPS
 	SME_WPS_ENABLE_REQUEST,
 	SME_WPS_PROBE_REQUEST,
-#endif
 	SME_SET_GAIN,
 	SME_GET_GAIN,
 	SME_SLEEP_REQUEST,
@@ -174,7 +154,7 @@ enum {
 	SME_WEP_SET_CONFIRM,
 	SME_TERMINATE,
 
-	SME_EVENT_SIZE	/* end */
+	SME_EVENT_SIZE
 };
 
 /* SME Status */
@@ -196,29 +176,30 @@ struct sme_info {
 	unsigned long sme_flag;
 };
 
-struct hostt_t {
+struct hostt {
 	int buff[SME_EVENT_BUFF_SIZE];
 	unsigned int qhead;
 	unsigned int qtail;
 };
 
 #define RSN_IE_BODY_MAX 64
-struct rsn_ie_t {
+struct rsn_ie {
 	u8 id;	/* 0xdd = WPA or 0x30 = RSN */
 	u8 size;	/* max ? 255 ? */
 	u8 body[RSN_IE_BODY_MAX];
 } __packed;
 
-#ifdef WPS
+#define WPA_INFO_ELEM_ID 0xdd
+#define RSN_INFO_ELEM_ID 0x30
+
 #define WPS_IE_BODY_MAX 255
-struct wps_ie_t {
+struct wps_ie {
 	u8 id;	/* 221 'dd <len> 00 50 F2 04' */
 	u8 size;	/* max ? 255 ? */
 	u8 body[WPS_IE_BODY_MAX];
 } __packed;
-#endif /* WPS */
 
-struct local_ap_t {
+struct local_ap {
 	u8 bssid[6];
 	u8 rssi;
 	u8 sq;
@@ -235,28 +216,26 @@ struct local_ap_t {
 	u16 capability;
 	u8 channel;
 	u8 noise;
-	struct rsn_ie_t wpa_ie;
-	struct rsn_ie_t rsn_ie;
-#ifdef WPS
-	struct wps_ie_t wps_ie;
-#endif /* WPS */
+	struct rsn_ie wpa_ie;
+	struct rsn_ie rsn_ie;
+	struct wps_ie wps_ie;
 };
 
 #define LOCAL_APLIST_MAX 31
 #define LOCAL_CURRENT_AP LOCAL_APLIST_MAX
-struct local_aplist_t {
+struct local_aplist {
 	int size;
-	struct local_ap_t ap[LOCAL_APLIST_MAX + 1];
+	struct local_ap ap[LOCAL_APLIST_MAX + 1];
 };
 
-struct local_gain_t {
+struct local_gain {
 	u8 tx_mode;
 	u8 rx_mode;
 	u8 tx_gain;
 	u8 rx_gain;
 };
 
-struct local_eeprom_sum_t {
+struct local_eeprom_sum {
 	u8 type;
 	u8 result;
 };
@@ -278,33 +257,25 @@ enum {
 	PS_WAKEUP
 };
 
-struct power_save_status_t {
+struct power_save_status {
 	atomic_t status;	/* initialvalue 0 */
 	struct completion wakeup_wait;
 	atomic_t confirm_wait;
 	atomic_t snooze_guard;
 };
 
-struct sleep_status_t {
+struct sleep_status {
 	atomic_t status;	/* initialvalue 0 */
 	atomic_t doze_request;
 	atomic_t wakeup_request;
 };
 
 /* WPA */
-struct scan_ext_t {
+struct scan_ext {
 	unsigned int flag;
 	char ssid[IW_ESSID_MAX_SIZE + 1];
 };
 
-enum {
-	CIPHER_NONE,
-	CIPHER_WEP40,
-	CIPHER_TKIP,
-	CIPHER_CCMP,
-	CIPHER_WEP104
-};
-
 #define CIPHER_ID_WPA_NONE    "\x00\x50\xf2\x00"
 #define CIPHER_ID_WPA_WEP40   "\x00\x50\xf2\x01"
 #define CIPHER_ID_WPA_TKIP    "\x00\x50\xf2\x02"
@@ -339,7 +310,7 @@ enum {
 
 #define MIC_KEY_SIZE 8
 
-struct wpa_key_t {
+struct wpa_key {
 	u32 ext_flags;	/* IW_ENCODE_EXT_xxx */
 	u8 tx_seq[IW_ENCODE_SEQ_MAX_SIZE];	/* LSB first */
 	u8 rx_seq[IW_ENCODE_SEQ_MAX_SIZE];	/* LSB first */
@@ -357,50 +328,111 @@ struct wpa_key_t {
 #define WPA_KEY_INDEX_MAX 4
 #define WPA_RX_SEQ_LEN 6
 
-struct mic_failure_t {
+struct mic_failure {
 	u16 failure;	/* MIC Failure counter 0 or 1 or 2 */
 	u16 counter;	/* 1sec counter 0-60 */
 	u32 last_failure_time;
-	int stop;	/* stop flag */
+	int stop;
 };
 
-struct wpa_status_t {
+struct wpa_status {
 	int wpa_enabled;
-	unsigned int rsn_enabled;
+	bool rsn_enabled;
 	int version;
 	int pairwise_suite;	/* unicast cipher */
 	int group_suite;	/* multicast cipher */
-	int key_mgmt_suite;	/* authentication key management suite */
+	int key_mgmt_suite;
 	int auth_alg;
 	int txkey;
-	struct wpa_key_t key[WPA_KEY_INDEX_MAX];
-	struct scan_ext_t scan_ext;
-	struct mic_failure_t mic_failure;
+	struct wpa_key key[WPA_KEY_INDEX_MAX];
+	struct scan_ext scan_ext;
+	struct mic_failure mic_failure;
 };
 
 #include <linux/list.h>
 #define PMK_LIST_MAX 8
-struct pmk_list_t {
+struct pmk_list {
 	u16 size;
 	struct list_head head;
-	struct pmk_t {
+	struct pmk {
 		struct list_head list;
 		u8 bssid[ETH_ALEN];
 		u8 pmkid[IW_PMKID_LEN];
 	} pmk[PMK_LIST_MAX];
 };
 
-#ifdef WPS
-struct wps_status_t {
+struct wps_status {
 	int wps_enabled;
 	int ielen;
 	u8 ie[255];
 };
-#endif /* WPS */
+
+/* Tx Device struct */
+#define	TX_DEVICE_BUFF_SIZE	1024
+
+struct ks_wlan_private;
+
+/**
+ * struct tx_device_buffer - Queue item for the tx queue.
+ * @sendp: Pointer to the send request data.
+ * @size: Size of @sendp data.
+ * @complete_handler: Function called once data write to device is complete.
+ * @arg1: First argument to @complete_handler.
+ * @arg2: Second argument to @complete_handler.
+ */
+struct tx_device_buffer {
+	unsigned char *sendp;
+	unsigned int size;
+	void (*complete_handler)(struct ks_wlan_private *priv,
+				 struct sk_buff *skb);
+	struct sk_buff *skb;
+};
+
+/**
+ * struct tx_device - Tx buffer queue.
+ * @tx_device_buffer: Queue buffer.
+ * @qhead: Head of tx queue.
+ * @qtail: Tail of tx queue.
+ * @tx_dev_lock: Queue lock.
+ */
+struct tx_device {
+	struct tx_device_buffer tx_dev_buff[TX_DEVICE_BUFF_SIZE];
+	unsigned int qhead;
+	unsigned int qtail;
+	spinlock_t tx_dev_lock;	/* protect access to the queue */
+};
+
+/* Rx Device struct */
+#define	RX_DATA_SIZE	(2 + 2 + 2347 + 1)
+#define	RX_DEVICE_BUFF_SIZE	32
+
+/**
+ * struct rx_device_buffer - Queue item for the rx queue.
+ * @data: rx data.
+ * @size: Size of @data.
+ */
+struct rx_device_buffer {
+	unsigned char data[RX_DATA_SIZE];
+	unsigned int size;
+};
+
+/**
+ * struct rx_device - Rx buffer queue.
+ * @rx_device_buffer: Queue buffer.
+ * @qhead: Head of rx queue.
+ * @qtail: Tail of rx queue.
+ * @rx_dev_lock: Queue lock.
+ */
+struct rx_device {
+	struct rx_device_buffer rx_dev_buff[RX_DEVICE_BUFF_SIZE];
+	unsigned int qhead;
+	unsigned int qtail;
+	spinlock_t rx_dev_lock;	/* protect access to the queue */
+};
 
 struct ks_wlan_private {
 	/* hardware information */
-	struct ks_sdio_card *ks_sdio_card;
+	void *if_hw;
 	struct workqueue_struct *wq;
 	struct delayed_work rw_dwork;
 	struct tasklet_struct rx_bh_task;
@@ -423,12 +455,12 @@ struct ks_wlan_private {
 
 	unsigned char eth_addr[ETH_ALEN];
 
-	struct local_aplist_t aplist;
-	struct local_ap_t current_ap;
-	struct power_save_status_t psstatus;
-	struct sleep_status_t sleepstatus;
-	struct wpa_status_t wpa;
-	struct pmk_list_t pmklist;
+	struct local_aplist aplist;
+	struct local_ap current_ap;
+	struct power_save_status psstatus;
+	struct sleep_status sleepstatus;
+	struct wpa_status wpa;
+	struct pmk_list pmklist;
 	/* wireless parameter */
 	struct ks_wlan_parameter reg;
 	u8 current_rate;
@@ -443,7 +475,7 @@ struct ks_wlan_private {
 	unsigned int need_commit;	/* for ioctl */
 
 	/* DeviceIoControl */
-	int device_open_status;
+	bool is_device_open;
 	atomic_t event_count;
 	atomic_t rec_count;
 	int dev_count;
@@ -455,42 +487,78 @@ struct ks_wlan_private {
 	unsigned char firmware_version[128 + 1];
 	int version_size;
 
-	bool mac_address_valid;	/* Mac Address Status */
+	bool mac_address_valid;
 
 	int dev_state;
 
 	struct sk_buff *skb;
 	unsigned int cur_rx;	/* Index into the Rx buffer of next Rx pkt. */
-	/* spinlock_t lock; */
 #define FORCE_DISCONNECT    0x80000000
 #define CONNECT_STATUS_MASK 0x7FFFFFFF
-	u32 connect_status;	/* connect status */
-	int infra_status;	/* Infractructure status */
-
-	u8 data_buff[0x1000];
-
+	u32 connect_status;
+	int infra_status;
 	u8 scan_ssid_len;
 	u8 scan_ssid[IW_ESSID_MAX_SIZE + 1];
-	struct local_gain_t gain;
-#ifdef WPS
-	struct net_device *l2_dev;
-	int l2_fd;
-	struct wps_status_t wps;
-#endif /* WPS */
+	struct local_gain gain;
+	struct wps_status wps;
 	u8 sleep_mode;
 
 	u8 region;
-	struct local_eeprom_sum_t eeprom_sum;
+	struct local_eeprom_sum eeprom_sum;
 	u8 eeprom_checksum;
 
-	struct hostt_t hostt;
+	struct hostt hostt;
 
 	unsigned long last_doze;
 	unsigned long last_wakeup;
 
-	uint wakeup_count;	/* for detect wakeup loop */
+	unsigned int wakeup_count;	/* for detect wakeup loop */
 };
 
+static inline void inc_txqhead(struct ks_wlan_private *priv)
+{
+	priv->tx_dev.qhead = (priv->tx_dev.qhead + 1) % TX_DEVICE_BUFF_SIZE;
+}
+
+static inline void inc_txqtail(struct ks_wlan_private *priv)
+{
+	priv->tx_dev.qtail = (priv->tx_dev.qtail + 1) % TX_DEVICE_BUFF_SIZE;
+}
+
+static inline bool txq_has_space(struct ks_wlan_private *priv)
+{
+	return (CIRC_SPACE(priv->tx_dev.qhead, priv->tx_dev.qtail,
+			   TX_DEVICE_BUFF_SIZE) > 0);
+}
+
+static inline void inc_rxqhead(struct ks_wlan_private *priv)
+{
+	priv->rx_dev.qhead = (priv->rx_dev.qhead + 1) % RX_DEVICE_BUFF_SIZE;
+}
+
+static inline void inc_rxqtail(struct ks_wlan_private *priv)
+{
+	priv->rx_dev.qtail = (priv->rx_dev.qtail + 1) % RX_DEVICE_BUFF_SIZE;
+}
+
+static inline bool rxq_has_space(struct ks_wlan_private *priv)
+{
+	return (CIRC_SPACE(priv->rx_dev.qhead, priv->rx_dev.qtail,
+			   RX_DEVICE_BUFF_SIZE) > 0);
+}
+
+static inline unsigned int txq_count(struct ks_wlan_private *priv)
+{
+	return CIRC_CNT_TO_END(priv->tx_dev.qhead, priv->tx_dev.qtail,
+			       TX_DEVICE_BUFF_SIZE);
+}
+
+static inline unsigned int rxq_count(struct ks_wlan_private *priv)
+{
+	return CIRC_CNT_TO_END(priv->rx_dev.qhead, priv->rx_dev.qtail,
+			       RX_DEVICE_BUFF_SIZE);
+}
+
 int ks_wlan_net_start(struct net_device *dev);
 int ks_wlan_net_stop(struct net_device *dev);
 bool is_connect_status(u32 status);
diff --git a/drivers/staging/ks7010/ks_wlan_ioctl.h b/drivers/staging/ks7010/ks_wlan_ioctl.h
index 121e7cb808a2..97c7d95de411 100644
--- a/drivers/staging/ks7010/ks_wlan_ioctl.h
+++ b/drivers/staging/ks7010/ks_wlan_ioctl.h
@@ -1,12 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  *   Driver for KeyStream 11b/g wireless LAN
  *
  *   Copyright (c) 2005-2008 KeyStream Corp.
  *   Copyright (C) 2009 Renesas Technology Corp.
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License version 2 as
- *   published by the Free Software Foundation.
  */
 
 #ifndef _KS_WLAN_IOCTL_H
@@ -19,11 +16,9 @@
 /* former KS_WLAN_GET_DRIVER_VERSION	(SIOCIWFIRSTPRIV + 1) */
 /*					(SIOCIWFIRSTPRIV + 2) */
 #define KS_WLAN_GET_FIRM_VERSION	(SIOCIWFIRSTPRIV + 3)
-#ifdef WPS
 #define KS_WLAN_SET_WPS_ENABLE		(SIOCIWFIRSTPRIV + 4)
 #define KS_WLAN_GET_WPS_ENABLE		(SIOCIWFIRSTPRIV + 5)
 #define KS_WLAN_SET_WPS_PROBE_REQ	(SIOCIWFIRSTPRIV + 6)
-#endif
 #define KS_WLAN_GET_EEPROM_CKSUM	(SIOCIWFIRSTPRIV + 7)
 #define KS_WLAN_SET_PREAMBLE		(SIOCIWFIRSTPRIV + 8)
 #define KS_WLAN_GET_PREAMBLE		(SIOCIWFIRSTPRIV + 9)
diff --git a/drivers/staging/ks7010/ks_wlan_net.c b/drivers/staging/ks7010/ks_wlan_net.c
index 9078e13b0d4a..dc5459ae0b51 100644
--- a/drivers/staging/ks7010/ks_wlan_net.c
+++ b/drivers/staging/ks7010/ks_wlan_net.c
@@ -1,31 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *   Driver for KeyStream 11b/g wireless LAN
  *
  *   Copyright (C) 2005-2008 KeyStream Corp.
  *   Copyright (C) 2009 Renesas Technology Corp.
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License version 2 as
- *   published by the Free Software Foundation.
  */
 
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/compiler.h>
-#include <linux/init.h>
-#include <linux/ioport.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/if_arp.h>
-#include <linux/rtnetlink.h>
-#include <linux/delay.h>
+#include <linux/atomic.h>
 #include <linux/completion.h>
-#include <linux/mii.h>
-#include <linux/pci.h>
-#include <linux/ctype.h>
+#include <linux/if_arp.h>
+#include <linux/netdevice.h>
 #include <linux/timer.h>
-#include <linux/atomic.h>
-#include <linux/io.h>
 #include <linux/uaccess.h>
 
 static int wep_on_off;
@@ -43,7 +28,8 @@ static int wep_on_off;
 #include <net/iw_handler.h>	/* New driver API */
 
 /* Frequency list (map channels to frequencies) */
-static const long frequency_list[] = { 2412, 2417, 2422, 2427, 2432, 2437, 2442,
+static const long frequency_list[] = {
+	2412, 2417, 2422, 2427, 2432, 2437, 2442,
 	2447, 2452, 2457, 2462, 2467, 2472, 2484
 };
 
@@ -55,17 +41,6 @@ struct wep_key {
 	u8 key[16];	/* 40-bit and 104-bit keys */
 };
 
-/* Backward compatibility */
-#ifndef IW_ENCODE_NOKEY
-#define IW_ENCODE_NOKEY 0x0800	/* Key is write only, so not present */
-#define IW_ENCODE_MODE  (IW_ENCODE_DISABLED | IW_ENCODE_RESTRICTED | IW_ENCODE_OPEN)
-#endif /* IW_ENCODE_NOKEY */
-
-/* List of Wireless Handlers (new API) */
-static const struct iw_handler_def ks_wlan_handler_def;
-
-#define KSC_OPNOTSUPP	/* Operation Not Support */
-
 /*
  *	function prototypes
  */
@@ -73,7 +48,7 @@ static int ks_wlan_open(struct net_device *dev);
 static void ks_wlan_tx_timeout(struct net_device *dev);
 static int ks_wlan_start_xmit(struct sk_buff *skb, struct net_device *dev);
 static int ks_wlan_close(struct net_device *dev);
-static void ks_wlan_set_multicast_list(struct net_device *dev);
+static void ks_wlan_set_rx_mode(struct net_device *dev);
 static struct net_device_stats *ks_wlan_get_stats(struct net_device *dev);
 static int ks_wlan_set_mac_address(struct net_device *dev, void *addr);
 static int ks_wlan_netdev_ioctl(struct net_device *dev, struct ifreq *rq,
@@ -172,7 +147,8 @@ int ks_wlan_setup_parameter(struct ks_wlan_private *priv,
  */
 
 static int ks_wlan_get_name(struct net_device *dev,
-			    struct iw_request_info *info, char *cwrq,
+			    struct iw_request_info *info,
+			    union iwreq_data *cwrq,
 			    char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
@@ -182,20 +158,20 @@ static int ks_wlan_get_name(struct net_device *dev,
 
 	/* for SLEEP MODE */
 	if (priv->dev_state < DEVICE_STATE_READY)
-		strcpy(cwrq, "NOT READY!");
+		strcpy(cwrq->name, "NOT READY!");
 	else if (priv->reg.phy_type == D_11B_ONLY_MODE)
-		strcpy(cwrq, "IEEE 802.11b");
+		strcpy(cwrq->name, "IEEE 802.11b");
 	else if (priv->reg.phy_type == D_11G_ONLY_MODE)
-		strcpy(cwrq, "IEEE 802.11g");
+		strcpy(cwrq->name, "IEEE 802.11g");
 	else
-		strcpy(cwrq, "IEEE 802.11b/g");
+		strcpy(cwrq->name, "IEEE 802.11b/g");
 
 	return 0;
 }
 
 static int ks_wlan_set_freq(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_freq *fwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *fwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	int channel;
@@ -205,28 +181,28 @@ static int ks_wlan_set_freq(struct net_device *dev,
 
 	/* for SLEEP MODE */
 	/* If setting by frequency, convert to a channel */
-	if ((fwrq->e == 1) &&
-	    (fwrq->m >= (int)2.412e8) && (fwrq->m <= (int)2.487e8)) {
-		int f = fwrq->m / 100000;
+	if ((fwrq->freq.e == 1) &&
+	    (fwrq->freq.m >= (int)2.412e8) && (fwrq->freq.m <= (int)2.487e8)) {
+		int f = fwrq->freq.m / 100000;
 		int c = 0;
 
 		while ((c < 14) && (f != frequency_list[c]))
 			c++;
 		/* Hack to fall through... */
-		fwrq->e = 0;
-		fwrq->m = c + 1;
+		fwrq->freq.e = 0;
+		fwrq->freq.m = c + 1;
 	}
 	/* Setting by channel number */
-	if ((fwrq->m > 1000) || (fwrq->e > 0))
+	if ((fwrq->freq.m > 1000) || (fwrq->freq.e > 0))
 		return -EOPNOTSUPP;
 
-	channel = fwrq->m;
+	channel = fwrq->freq.m;
 	/* We should do a better check than that,
 	 * based on the card capability !!!
 	 */
 	if ((channel < 1) || (channel > 14)) {
 		netdev_dbg(dev, "%s: New channel value of %d is invalid!\n",
-			   dev->name, fwrq->m);
+			   dev->name, fwrq->freq.m);
 		return -EINVAL;
 	}
 
@@ -238,8 +214,8 @@ static int ks_wlan_set_freq(struct net_device *dev,
 }
 
 static int ks_wlan_get_freq(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_freq *fwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *fwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	int f;
@@ -253,15 +229,15 @@ static int ks_wlan_get_freq(struct net_device *dev,
 	else
 		f = (int)priv->reg.channel;
 
-	fwrq->m = frequency_list[f - 1] * 100000;
-	fwrq->e = 1;
+	fwrq->freq.m = frequency_list[f - 1] * 100000;
+	fwrq->freq.e = 1;
 
 	return 0;
 }
 
 static int ks_wlan_set_essid(struct net_device *dev,
 			     struct iw_request_info *info,
-			     struct iw_point *dwrq, char *extra)
+			     union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	size_t len;
@@ -271,12 +247,12 @@ static int ks_wlan_set_essid(struct net_device *dev,
 
 	/* for SLEEP MODE */
 	/* Check if we asked for `any' */
-	if (!dwrq->flags) {
+	if (!dwrq->essid.flags) {
 		/* Just send an empty SSID list */
 		memset(priv->reg.ssid.body, 0, sizeof(priv->reg.ssid.body));
 		priv->reg.ssid.size = 0;
 	} else {
-		len = dwrq->length;
+		len = dwrq->essid.length;
 		/* iwconfig uses nul termination in SSID.. */
 		if (len > 0 && extra[len - 1] == '\0')
 			len--;
@@ -300,7 +276,7 @@ static int ks_wlan_set_essid(struct net_device *dev,
 
 static int ks_wlan_get_essid(struct net_device *dev,
 			     struct iw_request_info *info,
-			     struct iw_point *dwrq, char *extra)
+			     union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
@@ -318,18 +294,18 @@ static int ks_wlan_get_essid(struct net_device *dev,
 		/* If none, we may want to get the one that was set */
 
 		/* Push it out ! */
-		dwrq->length = priv->reg.ssid.size;
-		dwrq->flags = 1;	/* active */
+		dwrq->essid.length = priv->reg.ssid.size;
+		dwrq->essid.flags = 1;	/* active */
 	} else {
-		dwrq->length = 0;
-		dwrq->flags = 0;	/* ANY */
+		dwrq->essid.length = 0;
+		dwrq->essid.flags = 0;	/* ANY */
 	}
 
 	return 0;
 }
 
 static int ks_wlan_set_wap(struct net_device *dev, struct iw_request_info *info,
-			   struct sockaddr *ap_addr, char *extra)
+			   union iwreq_data *awrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
@@ -337,18 +313,16 @@ static int ks_wlan_set_wap(struct net_device *dev, struct iw_request_info *info,
 		return -EPERM;
 
 	/* for SLEEP MODE */
-	if (priv->reg.operation_mode == MODE_ADHOC ||
-	    priv->reg.operation_mode == MODE_INFRASTRUCTURE) {
-		memcpy(priv->reg.bssid, &ap_addr->sa_data, ETH_ALEN);
-
-		if (is_valid_ether_addr((u8 *)priv->reg.bssid))
-			priv->need_commit |= SME_MODE_SET;
-
-	} else {
+	if (priv->reg.operation_mode != MODE_ADHOC &&
+	    priv->reg.operation_mode != MODE_INFRASTRUCTURE) {
 		eth_zero_addr(priv->reg.bssid);
 		return -EOPNOTSUPP;
 	}
 
+	ether_addr_copy(priv->reg.bssid, awrq->ap_addr.sa_data);
+	if (is_valid_ether_addr((u8 *)priv->reg.bssid))
+		priv->need_commit |= SME_MODE_SET;
+
 	netdev_dbg(dev, "bssid = %pM\n", priv->reg.bssid);
 
 	/* Write it to the card */
@@ -360,7 +334,7 @@ static int ks_wlan_set_wap(struct net_device *dev, struct iw_request_info *info,
 }
 
 static int ks_wlan_get_wap(struct net_device *dev, struct iw_request_info *info,
-			   struct sockaddr *awrq, char *extra)
+			   union iwreq_data *awrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
@@ -369,18 +343,18 @@ static int ks_wlan_get_wap(struct net_device *dev, struct iw_request_info *info,
 
 	/* for SLEEP MODE */
 	if (is_connect_status(priv->connect_status))
-		memcpy(awrq->sa_data, priv->current_ap.bssid, ETH_ALEN);
+		ether_addr_copy(awrq->ap_addr.sa_data, priv->current_ap.bssid);
 	else
-		eth_zero_addr(awrq->sa_data);
+		eth_zero_addr(awrq->ap_addr.sa_data);
 
-	awrq->sa_family = ARPHRD_ETHER;
+	awrq->ap_addr.sa_family = ARPHRD_ETHER;
 
 	return 0;
 }
 
 static int ks_wlan_set_nick(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_point *dwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
@@ -389,18 +363,18 @@ static int ks_wlan_set_nick(struct net_device *dev,
 
 	/* for SLEEP MODE */
 	/* Check the size of the string */
-	if (dwrq->length > 16 + 1)
+	if (dwrq->data.length > 16 + 1)
 		return -E2BIG;
 
 	memset(priv->nick, 0, sizeof(priv->nick));
-	memcpy(priv->nick, extra, dwrq->length);
+	memcpy(priv->nick, extra, dwrq->data.length);
 
 	return -EINPROGRESS;	/* Call commit handler */
 }
 
 static int ks_wlan_get_nick(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_point *dwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
@@ -410,14 +384,14 @@ static int ks_wlan_get_nick(struct net_device *dev,
 	/* for SLEEP MODE */
 	strncpy(extra, priv->nick, 16);
 	extra[16] = '\0';
-	dwrq->length = strlen(extra) + 1;
+	dwrq->data.length = strlen(extra) + 1;
 
 	return 0;
 }
 
 static int ks_wlan_set_rate(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_param *vwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *vwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	int i = 0;
@@ -427,17 +401,17 @@ static int ks_wlan_set_rate(struct net_device *dev,
 
 	/* for SLEEP MODE */
 	if (priv->reg.phy_type == D_11B_ONLY_MODE) {
-		if (vwrq->fixed == 1) {
-			switch (vwrq->value) {
+		if (vwrq->bitrate.fixed == 1) {
+			switch (vwrq->bitrate.value) {
 			case 11000000:
 			case 5500000:
 				priv->reg.rate_set.body[0] =
-				    (uint8_t)(vwrq->value / 500000);
+				    (u8)(vwrq->bitrate.value / 500000);
 				break;
 			case 2000000:
 			case 1000000:
 				priv->reg.rate_set.body[0] =
-				    ((uint8_t)(vwrq->value / 500000)) |
+				    ((u8)(vwrq->bitrate.value / 500000)) |
 				    BASIC_RATE;
 				break;
 			default:
@@ -446,8 +420,8 @@ static int ks_wlan_set_rate(struct net_device *dev,
 			priv->reg.tx_rate = TX_RATE_FIXED;
 			priv->reg.rate_set.size = 1;
 		} else {	/* vwrq->fixed == 0 */
-			if (vwrq->value > 0) {
-				switch (vwrq->value) {
+			if (vwrq->bitrate.value > 0) {
+				switch (vwrq->bitrate.value) {
 				case 11000000:
 					priv->reg.rate_set.body[3] =
 					    TX_RATE_11M;
@@ -484,15 +458,15 @@ static int ks_wlan_set_rate(struct net_device *dev,
 			}
 		}
 	} else {	/* D_11B_ONLY_MODE or  D_11BG_COMPATIBLE_MODE */
-		if (vwrq->fixed == 1) {
-			switch (vwrq->value) {
+		if (vwrq->bitrate.fixed == 1) {
+			switch (vwrq->bitrate.value) {
 			case 54000000:
 			case 48000000:
 			case 36000000:
 			case 18000000:
 			case 9000000:
 				priv->reg.rate_set.body[0] =
-				    (uint8_t)(vwrq->value / 500000);
+				    (u8)(vwrq->bitrate.value / 500000);
 				break;
 			case 24000000:
 			case 12000000:
@@ -502,7 +476,7 @@ static int ks_wlan_set_rate(struct net_device *dev,
 			case 2000000:
 			case 1000000:
 				priv->reg.rate_set.body[0] =
-				    ((uint8_t)(vwrq->value / 500000)) |
+				    ((u8)(vwrq->bitrate.value / 500000)) |
 				    BASIC_RATE;
 				break;
 			default:
@@ -511,8 +485,8 @@ static int ks_wlan_set_rate(struct net_device *dev,
 			priv->reg.tx_rate = TX_RATE_FIXED;
 			priv->reg.rate_set.size = 1;
 		} else {	/* vwrq->fixed == 0 */
-			if (vwrq->value > 0) {
-				switch (vwrq->value) {
+			if (vwrq->bitrate.value > 0) {
+				switch (vwrq->bitrate.value) {
 				case 54000000:
 					priv->reg.rate_set.body[11] =
 					    TX_RATE_54M;
@@ -534,7 +508,7 @@ static int ks_wlan_set_rate(struct net_device *dev,
 				case 11000000:
 				case 9000000:
 				case 6000000:
-					if (vwrq->value == 24000000) {
+					if (vwrq->bitrate.value == 24000000) {
 						priv->reg.rate_set.body[8] =
 						    TX_RATE_18M;
 						i++;
@@ -553,7 +527,7 @@ static int ks_wlan_set_rate(struct net_device *dev,
 						priv->reg.rate_set.body[3] =
 						    TX_RATE_11M | BASIC_RATE;
 						i++;
-					} else if (vwrq->value == 18000000) {
+					} else if (vwrq->bitrate.value == 18000000) {
 						priv->reg.rate_set.body[7] =
 						    TX_RATE_18M;
 						i++;
@@ -569,7 +543,7 @@ static int ks_wlan_set_rate(struct net_device *dev,
 						priv->reg.rate_set.body[3] =
 						    TX_RATE_11M | BASIC_RATE;
 						i++;
-					} else if (vwrq->value == 12000000) {
+					} else if (vwrq->bitrate.value == 12000000) {
 						priv->reg.rate_set.body[6] =
 						    TX_RATE_9M;
 						i++;
@@ -582,7 +556,7 @@ static int ks_wlan_set_rate(struct net_device *dev,
 						priv->reg.rate_set.body[3] =
 						    TX_RATE_11M | BASIC_RATE;
 						i++;
-					} else if (vwrq->value == 11000000) {
+					} else if (vwrq->bitrate.value == 11000000) {
 						priv->reg.rate_set.body[5] =
 						    TX_RATE_9M;
 						i++;
@@ -592,7 +566,7 @@ static int ks_wlan_set_rate(struct net_device *dev,
 						priv->reg.rate_set.body[3] =
 						    TX_RATE_11M | BASIC_RATE;
 						i++;
-					} else if (vwrq->value == 9000000) {
+					} else if (vwrq->bitrate.value == 9000000) {
 						priv->reg.rate_set.body[4] =
 						    TX_RATE_9M;
 						i++;
@@ -657,8 +631,8 @@ static int ks_wlan_set_rate(struct net_device *dev,
 }
 
 static int ks_wlan_get_rate(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_param *vwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *vwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
@@ -672,26 +646,23 @@ static int ks_wlan_get_rate(struct net_device *dev,
 	if (!atomic_read(&update_phyinfo))
 		ks_wlan_update_phy_information(priv);
 
-	vwrq->value = ((priv->current_rate) & RATE_MASK) * 500000;
-	if (priv->reg.tx_rate == TX_RATE_FIXED)
-		vwrq->fixed = 1;
-	else
-		vwrq->fixed = 0;
+	vwrq->bitrate.value = ((priv->current_rate) & RATE_MASK) * 500000;
+	vwrq->bitrate.fixed = (priv->reg.tx_rate == TX_RATE_FIXED) ? 1 : 0;
 
 	return 0;
 }
 
 static int ks_wlan_set_rts(struct net_device *dev, struct iw_request_info *info,
-			   struct iw_param *vwrq, char *extra)
+			   union iwreq_data *vwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
-	int rthr = vwrq->value;
+	int rthr = vwrq->rts.value;
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 
 	/* for SLEEP MODE */
-	if (vwrq->disabled)
+	if (vwrq->rts.disabled)
 		rthr = 2347;
 	if ((rthr < 0) || (rthr > 2347))
 		return -EINVAL;
@@ -703,7 +674,7 @@ static int ks_wlan_set_rts(struct net_device *dev, struct iw_request_info *info,
 }
 
 static int ks_wlan_get_rts(struct net_device *dev, struct iw_request_info *info,
-			   struct iw_param *vwrq, char *extra)
+			   union iwreq_data *vwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
@@ -711,25 +682,25 @@ static int ks_wlan_get_rts(struct net_device *dev, struct iw_request_info *info,
 		return -EPERM;
 
 	/* for SLEEP MODE */
-	vwrq->value = priv->reg.rts;
-	vwrq->disabled = (vwrq->value >= 2347);
-	vwrq->fixed = 1;
+	vwrq->rts.value = priv->reg.rts;
+	vwrq->rts.disabled = (vwrq->rts.value >= 2347);
+	vwrq->rts.fixed = 1;
 
 	return 0;
 }
 
 static int ks_wlan_set_frag(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_param *vwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *vwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
-	int fthr = vwrq->value;
+	int fthr = vwrq->frag.value;
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 
 	/* for SLEEP MODE */
-	if (vwrq->disabled)
+	if (vwrq->frag.disabled)
 		fthr = 2346;
 	if ((fthr < 256) || (fthr > 2346))
 		return -EINVAL;
@@ -742,8 +713,8 @@ static int ks_wlan_set_frag(struct net_device *dev,
 }
 
 static int ks_wlan_get_frag(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_param *vwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *vwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
@@ -751,134 +722,99 @@ static int ks_wlan_get_frag(struct net_device *dev,
 		return -EPERM;
 
 	/* for SLEEP MODE */
-	vwrq->value = priv->reg.fragment;
-	vwrq->disabled = (vwrq->value >= 2346);
-	vwrq->fixed = 1;
+	vwrq->frag.value = priv->reg.fragment;
+	vwrq->frag.disabled = (vwrq->frag.value >= 2346);
+	vwrq->frag.fixed = 1;
 
 	return 0;
 }
 
 static int ks_wlan_set_mode(struct net_device *dev,
-			    struct iw_request_info *info, __u32 *uwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *uwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 
-	/* for SLEEP MODE */
-	switch (*uwrq) {
-	case IW_MODE_ADHOC:
-		priv->reg.operation_mode = MODE_ADHOC;
-		priv->need_commit |= SME_MODE_SET;
-		break;
-	case IW_MODE_INFRA:
-		priv->reg.operation_mode = MODE_INFRASTRUCTURE;
-		priv->need_commit |= SME_MODE_SET;
-		break;
-	case IW_MODE_AUTO:
-	case IW_MODE_MASTER:
-	case IW_MODE_REPEAT:
-	case IW_MODE_SECOND:
-	case IW_MODE_MONITOR:
-	default:
+	if (uwrq->mode != IW_MODE_ADHOC &&
+	    uwrq->mode != IW_MODE_INFRA)
 		return -EINVAL;
-	}
+
+	priv->reg.operation_mode = (uwrq->mode == IW_MODE_ADHOC) ?
+				    MODE_ADHOC : MODE_INFRASTRUCTURE;
+	priv->need_commit |= SME_MODE_SET;
 
 	return -EINPROGRESS;	/* Call commit handler */
 }
 
 static int ks_wlan_get_mode(struct net_device *dev,
-			    struct iw_request_info *info, __u32 *uwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *uwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 
-	/* for SLEEP MODE */
 	/* If not managed, assume it's ad-hoc */
-	switch (priv->reg.operation_mode) {
-	case MODE_INFRASTRUCTURE:
-		*uwrq = IW_MODE_INFRA;
-		break;
-	case MODE_ADHOC:
-		*uwrq = IW_MODE_ADHOC;
-		break;
-	default:
-		*uwrq = IW_MODE_ADHOC;
-	}
+	uwrq->mode = (priv->reg.operation_mode == MODE_INFRASTRUCTURE) ?
+		      IW_MODE_INFRA : IW_MODE_ADHOC;
 
 	return 0;
 }
 
 static int ks_wlan_set_encode(struct net_device *dev,
 			      struct iw_request_info *info,
-			      struct iw_point *dwrq, char *extra)
+			      union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
-
+	struct iw_point *enc = &dwrq->encoding;
 	struct wep_key key;
-	int index = (dwrq->flags & IW_ENCODE_INDEX);
-	int current_index = priv->reg.wep_index;
-	int i;
+	int index = (enc->flags & IW_ENCODE_INDEX);
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 
+	if (enc->length > MAX_KEY_SIZE)
+		return -EINVAL;
+
 	/* for SLEEP MODE */
-	/* index check */
 	if ((index < 0) || (index > 4))
 		return -EINVAL;
-	else if (index == 0)
-		index = current_index;
-	else
-		index--;
+
+	index = (index == 0) ? priv->reg.wep_index : (index - 1);
 
 	/* Is WEP supported ? */
 	/* Basic checking: do we have a key to set ? */
-	if (dwrq->length > 0) {
-		if (dwrq->length > MAX_KEY_SIZE) {	/* Check the size of the key */
-			return -EINVAL;
-		}
-		if (dwrq->length > MIN_KEY_SIZE) {	/* Set the length */
-			key.len = MAX_KEY_SIZE;
-			priv->reg.privacy_invoked = 0x01;
-			priv->need_commit |= SME_WEP_FLAG;
-			wep_on_off = WEP_ON_128BIT;
-		} else {
-			if (dwrq->length > 0) {
-				key.len = MIN_KEY_SIZE;
-				priv->reg.privacy_invoked = 0x01;
-				priv->need_commit |= SME_WEP_FLAG;
-				wep_on_off = WEP_ON_64BIT;
-			} else {	/* Disable the key */
-				key.len = 0;
-			}
-		}
+	if (enc->length > 0) {
+		key.len = (enc->length > MIN_KEY_SIZE) ?
+			   MAX_KEY_SIZE : MIN_KEY_SIZE;
+		priv->reg.privacy_invoked = 0x01;
+		priv->need_commit |= SME_WEP_FLAG;
+		wep_on_off = (enc->length > MIN_KEY_SIZE) ?
+			      WEP_ON_128BIT : WEP_ON_64BIT;
 		/* Check if the key is not marked as invalid */
-		if (!(dwrq->flags & IW_ENCODE_NOKEY)) {
-			/* Cleanup */
-			memset(key.key, 0, MAX_KEY_SIZE);
-			/* Copy the key in the driver */
-			if (copy_from_user
-			    (key.key, dwrq->pointer, dwrq->length)) {
-				key.len = 0;
-				return -EFAULT;
-			}
-			/* Send the key to the card */
-			priv->reg.wep_key[index].size = key.len;
-			for (i = 0; i < (priv->reg.wep_key[index].size); i++)
-				priv->reg.wep_key[index].val[i] = key.key[i];
+		if (enc->flags & IW_ENCODE_NOKEY)
+			return 0;
 
-			priv->need_commit |= (SME_WEP_VAL1 << index);
-			priv->reg.wep_index = index;
-			priv->need_commit |= SME_WEP_INDEX;
+		/* Cleanup */
+		memset(key.key, 0, MAX_KEY_SIZE);
+		/* Copy the key in the driver */
+		if (copy_from_user(key.key, enc->pointer, enc->length)) {
+			key.len = 0;
+			return -EFAULT;
 		}
+		/* Send the key to the card */
+		priv->reg.wep_key[index].size = key.len;
+		memcpy(&priv->reg.wep_key[index].val[0], &key.key[0],
+		       priv->reg.wep_key[index].size);
+		priv->need_commit |= (SME_WEP_VAL1 << index);
+		priv->reg.wep_index = index;
+		priv->need_commit |= SME_WEP_INDEX;
 	} else {
-		if (dwrq->flags & IW_ENCODE_DISABLED) {
+		if (enc->flags & IW_ENCODE_DISABLED) {
 			priv->reg.wep_key[0].size = 0;
 			priv->reg.wep_key[1].size = 0;
 			priv->reg.wep_key[2].size = 0;
@@ -891,35 +827,29 @@ static int ks_wlan_set_encode(struct net_device *dev,
 			wep_on_off = WEP_OFF;
 			priv->need_commit |= SME_WEP_FLAG;
 		} else {
-			/* Do we want to just set the transmit key index ? */
-			if ((index >= 0) && (index < 4)) {
-				/* set_wep_key(priv, index, 0, 0, 1);   xxx */
-				if (priv->reg.wep_key[index].size != 0) {
-					priv->reg.wep_index = index;
-					priv->need_commit |= SME_WEP_INDEX;
-				} else {
-					return -EINVAL;
-				}
-			}
+			/* set_wep_key(priv, index, 0, 0, 1);   xxx */
+			if (priv->reg.wep_key[index].size == 0)
+				return -EINVAL;
+			priv->reg.wep_index = index;
+			priv->need_commit |= SME_WEP_INDEX;
 		}
 	}
 
 	/* Commit the changes if needed */
-	if (dwrq->flags & IW_ENCODE_MODE)
+	if (enc->flags & IW_ENCODE_MODE)
 		priv->need_commit |= SME_WEP_FLAG;
 
-	if (dwrq->flags & IW_ENCODE_OPEN) {
+	if (enc->flags & IW_ENCODE_OPEN) {
 		if (priv->reg.authenticate_type == AUTH_TYPE_SHARED_KEY)
 			priv->need_commit |= SME_MODE_SET;
 
 		priv->reg.authenticate_type = AUTH_TYPE_OPEN_SYSTEM;
-	} else if (dwrq->flags & IW_ENCODE_RESTRICTED) {
+	} else if (enc->flags & IW_ENCODE_RESTRICTED) {
 		if (priv->reg.authenticate_type == AUTH_TYPE_OPEN_SYSTEM)
 			priv->need_commit |= SME_MODE_SET;
 
 		priv->reg.authenticate_type = AUTH_TYPE_SHARED_KEY;
 	}
-//      return -EINPROGRESS;            /* Call commit handler */
 	if (priv->need_commit) {
 		ks_wlan_setup_parameter(priv, priv->need_commit);
 		priv->need_commit = 0;
@@ -929,104 +859,49 @@ static int ks_wlan_set_encode(struct net_device *dev,
 
 static int ks_wlan_get_encode(struct net_device *dev,
 			      struct iw_request_info *info,
-			      struct iw_point *dwrq, char *extra)
+			      union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
-	char zeros[16];
-	int index = (dwrq->flags & IW_ENCODE_INDEX) - 1;
+	struct iw_point *enc = &dwrq->encoding;
+	int index = (enc->flags & IW_ENCODE_INDEX) - 1;
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 
 	/* for SLEEP MODE */
-	dwrq->flags = IW_ENCODE_DISABLED;
+	enc->flags = IW_ENCODE_DISABLED;
 
 	/* Check encryption mode */
 	switch (priv->reg.authenticate_type) {
 	case AUTH_TYPE_OPEN_SYSTEM:
-		dwrq->flags = IW_ENCODE_OPEN;
+		enc->flags = IW_ENCODE_OPEN;
 		break;
 	case AUTH_TYPE_SHARED_KEY:
-		dwrq->flags = IW_ENCODE_RESTRICTED;
+		enc->flags = IW_ENCODE_RESTRICTED;
 		break;
 	}
 
-	memset(zeros, 0, sizeof(zeros));
-
 	/* Which key do we want ? -1 -> tx index */
 	if ((index < 0) || (index >= 4))
 		index = priv->reg.wep_index;
 	if (priv->reg.privacy_invoked) {
-		dwrq->flags &= ~IW_ENCODE_DISABLED;
+		enc->flags &= ~IW_ENCODE_DISABLED;
 		/* dwrq->flags |= IW_ENCODE_NOKEY; */
 	}
-	dwrq->flags |= index + 1;
+	enc->flags |= index + 1;
 	/* Copy the key to the user buffer */
-	if ((index >= 0) && (index < 4))
-		dwrq->length = priv->reg.wep_key[index].size;
-	if (dwrq->length > 16)
-		dwrq->length = 0;
-#if 1	/* IW_ENCODE_NOKEY; */
-	if (dwrq->length) {
-		if ((index >= 0) && (index < 4))
-			memcpy(extra, priv->reg.wep_key[index].val,
-			       dwrq->length);
-	} else {
-		memcpy(extra, zeros, dwrq->length);
+	if (index >= 0 && index < 4) {
+		enc->length = (priv->reg.wep_key[index].size <= 16) ?
+				priv->reg.wep_key[index].size : 0;
+		memcpy(extra, priv->reg.wep_key[index].val, enc->length);
 	}
-#endif
-	return 0;
-}
-
-#ifndef KSC_OPNOTSUPP
-static int ks_wlan_set_txpow(struct net_device *dev,
-			     struct iw_request_info *info,
-			     struct iw_param *vwrq, char *extra)
-{
-	return -EOPNOTSUPP;	/* Not Support */
-}
-
-static int ks_wlan_get_txpow(struct net_device *dev,
-			     struct iw_request_info *info,
-			     struct iw_param *vwrq, char *extra)
-{
-	if (priv->sleep_mode == SLP_SLEEP)
-		return -EPERM;
 
-	/* for SLEEP MODE */
-	/* Not Support */
-	vwrq->value = 0;
-	vwrq->disabled = (vwrq->value == 0);
-	vwrq->fixed = 1;
 	return 0;
 }
 
-static int ks_wlan_set_retry(struct net_device *dev,
-			     struct iw_request_info *info,
-			     struct iw_param *vwrq, char *extra)
-{
-	return -EOPNOTSUPP;	/* Not Support */
-}
-
-static int ks_wlan_get_retry(struct net_device *dev,
-			     struct iw_request_info *info,
-			     struct iw_param *vwrq, char *extra)
-{
-	if (priv->sleep_mode == SLP_SLEEP)
-		return -EPERM;
-
-	/* for SLEEP MODE */
-	/* Not Support */
-	vwrq->value = 0;
-	vwrq->disabled = (vwrq->value == 0);
-	vwrq->fixed = 1;
-	return 0;
-}
-#endif /* KSC_OPNOTSUPP */
-
 static int ks_wlan_get_range(struct net_device *dev,
 			     struct iw_request_info *info,
-			     struct iw_point *dwrq, char *extra)
+			     union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	struct iw_range *range = (struct iw_range *)extra;
@@ -1036,7 +911,7 @@ static int ks_wlan_get_range(struct net_device *dev,
 		return -EPERM;
 
 	/* for SLEEP MODE */
-	dwrq->length = sizeof(struct iw_range);
+	dwrq->data.length = sizeof(struct iw_range);
 	memset(range, 0, sizeof(*range));
 	range->min_nwid = 0x0000;
 	range->max_nwid = 0x0000;
@@ -1051,7 +926,8 @@ static int ks_wlan_get_range(struct net_device *dev,
 		range->freq[k++].e = 1;	/* Values in table in MHz -> * 10^5 * 10 */
 	}
 	range->num_frequency = k;
-	if (priv->reg.phy_type == D_11B_ONLY_MODE || priv->reg.phy_type == D_11BG_COMPATIBLE_MODE) {	/* channel 14 */
+	if (priv->reg.phy_type == D_11B_ONLY_MODE ||
+	    priv->reg.phy_type == D_11BG_COMPATIBLE_MODE) {	/* channel 14 */
 		range->freq[13].i = 14;	/* List index */
 		range->freq[13].m = frequency_list[13] * 100000;
 		range->freq[13].e = 1;	/* Values in table in MHz -> * 10^5 * 10 */
@@ -1156,20 +1032,19 @@ static int ks_wlan_get_range(struct net_device *dev,
 
 static int ks_wlan_set_power(struct net_device *dev,
 			     struct iw_request_info *info,
-			     struct iw_param *vwrq, char *extra)
+			     union iwreq_data *vwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 
-	if (vwrq->disabled) {
+	if (vwrq->power.disabled) {
 		priv->reg.power_mgmt = POWER_MGMT_ACTIVE;
 	} else {
-		if (priv->reg.operation_mode == MODE_INFRASTRUCTURE)
-			priv->reg.power_mgmt = POWER_MGMT_SAVE1;
-		else
+		if (priv->reg.operation_mode != MODE_INFRASTRUCTURE)
 			return -EINVAL;
+		priv->reg.power_mgmt = POWER_MGMT_SAVE1;
 	}
 
 	hostif_sme_enqueue(priv, SME_POW_MNGMT_REQUEST);
@@ -1179,76 +1054,50 @@ static int ks_wlan_set_power(struct net_device *dev,
 
 static int ks_wlan_get_power(struct net_device *dev,
 			     struct iw_request_info *info,
-			     struct iw_param *vwrq, char *extra)
+			     union iwreq_data *vwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 	/* for SLEEP MODE */
-	if (priv->reg.power_mgmt > 0)
-		vwrq->disabled = 0;
-	else
-		vwrq->disabled = 1;
+	vwrq->power.disabled = (priv->reg.power_mgmt <= 0);
 
 	return 0;
 }
 
 static int ks_wlan_get_iwstats(struct net_device *dev,
 			       struct iw_request_info *info,
-			       struct iw_quality *vwrq, char *extra)
+			       union iwreq_data *vwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 	/* for SLEEP MODE */
-	vwrq->qual = 0;	/* not supported */
-	vwrq->level = priv->wstats.qual.level;
-	vwrq->noise = 0;	/* not supported */
-	vwrq->updated = 0;
+	vwrq->qual.qual = 0;	/* not supported */
+	vwrq->qual.level = priv->wstats.qual.level;
+	vwrq->qual.noise = 0;	/* not supported */
+	vwrq->qual.updated = 0;
 
 	return 0;
 }
 
-#ifndef KSC_OPNOTSUPP
-
-static int ks_wlan_set_sens(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_param *vwrq,
-			    char *extra)
-{
-	return -EOPNOTSUPP;	/* Not Support */
-}
-
-static int ks_wlan_get_sens(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_param *vwrq,
-			    char *extra)
-{
-	/* Not Support */
-	vwrq->value = 0;
-	vwrq->disabled = (vwrq->value == 0);
-	vwrq->fixed = 1;
-	return 0;
-}
-#endif /* KSC_OPNOTSUPP */
-
 /* Note : this is deprecated in favor of IWSCAN */
 static int ks_wlan_get_aplist(struct net_device *dev,
 			      struct iw_request_info *info,
-			      struct iw_point *dwrq, char *extra)
+			      union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	struct sockaddr *address = (struct sockaddr *)extra;
 	struct iw_quality qual[LOCAL_APLIST_MAX];
-
 	int i;
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 	/* for SLEEP MODE */
 	for (i = 0; i < priv->aplist.size; i++) {
-		memcpy(address[i].sa_data, &(priv->aplist.ap[i].bssid[0]),
-		       ETH_ALEN);
+		ether_addr_copy(address[i].sa_data, priv->aplist.ap[i].bssid);
 		address[i].sa_family = ARPHRD_ETHER;
 		qual[i].level = 256 - priv->aplist.ap[i].rssi;
 		qual[i].qual = priv->aplist.ap[i].sq;
@@ -1256,11 +1105,11 @@ static int ks_wlan_get_aplist(struct net_device *dev,
 		qual[i].updated = 7;
 	}
 	if (i) {
-		dwrq->flags = 1;	/* Should be define'd */
+		dwrq->data.flags = 1;	/* Should be define'd */
 		memcpy(extra + sizeof(struct sockaddr) * i,
 		       &qual, sizeof(struct iw_quality) * i);
 	}
-	dwrq->length = i;
+	dwrq->data.length = i;
 
 	return 0;
 }
@@ -1294,6 +1143,33 @@ static int ks_wlan_set_scan(struct net_device *dev,
 	return 0;
 }
 
+static char *ks_wlan_add_leader_event(const char *rsn_leader, char *end_buf,
+				      char *current_ev, struct rsn_ie *rsn,
+				      struct iw_event *iwe,
+				      struct iw_request_info *info)
+{
+	char buffer[RSN_IE_BODY_MAX * 2 + 30];
+	char *pbuf;
+	int i;
+
+	pbuf = &buffer[0];
+	memset(iwe, 0, sizeof(*iwe));
+	iwe->cmd = IWEVCUSTOM;
+	memcpy(buffer, rsn_leader, sizeof(rsn_leader) - 1);
+	iwe->u.data.length += sizeof(rsn_leader) - 1;
+	pbuf += sizeof(rsn_leader) - 1;
+	pbuf += sprintf(pbuf, "%02x", rsn->id);
+	pbuf += sprintf(pbuf, "%02x", rsn->size);
+	iwe->u.data.length += 4;
+
+	for (i = 0; i < rsn->size; i++)
+		pbuf += sprintf(pbuf, "%02x", rsn->body[i]);
+
+	iwe->u.data.length += rsn->size * 2;
+
+	return iwe_stream_add_point(info, current_ev, end_buf, iwe, &buffer[0]);
+}
+
 /*
  * Translate scan data returned from the card to a card independent
  * format that the Wireless Tools will understand - Jean II
@@ -1301,25 +1177,22 @@ static int ks_wlan_set_scan(struct net_device *dev,
 static inline char *ks_wlan_translate_scan(struct net_device *dev,
 					   struct iw_request_info *info,
 					   char *current_ev, char *end_buf,
-					   struct local_ap_t *ap)
+					   struct local_ap *ap)
 {
 	/* struct ks_wlan_private *priv = (struct ks_wlan_private *)dev->priv; */
+	static const char rsn_leader[] = "rsn_ie=";
+	static const char wpa_leader[] = "wpa_ie=";
 	struct iw_event iwe;	/* Temporary buffer */
 	u16 capabilities;
 	char *current_val;	/* For rates */
 	int i;
-	static const char rsn_leader[] = "rsn_ie=";
-	static const char wpa_leader[] = "wpa_ie=";
-	char buf0[RSN_IE_BODY_MAX * 2 + 30];
-	char buf1[RSN_IE_BODY_MAX * 2 + 30];
-	char *pbuf;
+
 	/* First entry *MUST* be the AP MAC address */
 	iwe.cmd = SIOCGIWAP;
 	iwe.u.ap_addr.sa_family = ARPHRD_ETHER;
-	memcpy(iwe.u.ap_addr.sa_data, ap->bssid, ETH_ALEN);
-	current_ev =
-	    iwe_stream_add_event(info, current_ev, end_buf, &iwe,
-				 IW_EV_ADDR_LEN);
+	ether_addr_copy(iwe.u.ap_addr.sa_data, ap->bssid);
+	current_ev = iwe_stream_add_event(info, current_ev,
+					  end_buf, &iwe, IW_EV_ADDR_LEN);
 
 	/* Other entries will be displayed in the order we give them */
 
@@ -1329,21 +1202,17 @@ static inline char *ks_wlan_translate_scan(struct net_device *dev,
 		iwe.u.data.length = 32;
 	iwe.cmd = SIOCGIWESSID;
 	iwe.u.data.flags = 1;
-	current_ev =
-	    iwe_stream_add_point(info, current_ev, end_buf, &iwe,
-				 ap->ssid.body);
+	current_ev = iwe_stream_add_point(info, current_ev,
+					  end_buf, &iwe, ap->ssid.body);
 
 	/* Add mode */
 	iwe.cmd = SIOCGIWMODE;
 	capabilities = ap->capability;
 	if (capabilities & (WLAN_CAPABILITY_ESS | WLAN_CAPABILITY_IBSS)) {
-		if (capabilities & WLAN_CAPABILITY_ESS)
-			iwe.u.mode = IW_MODE_INFRA;
-		else
-			iwe.u.mode = IW_MODE_ADHOC;
-		current_ev =
-		    iwe_stream_add_event(info, current_ev, end_buf, &iwe,
-					 IW_EV_UINT_LEN);
+		iwe.u.mode = (capabilities & WLAN_CAPABILITY_ESS) ?
+			      IW_MODE_INFRA : IW_MODE_ADHOC;
+		current_ev = iwe_stream_add_event(info, current_ev,
+						  end_buf, &iwe, IW_EV_UINT_LEN);
 	}
 
 	/* Add frequency */
@@ -1351,32 +1220,29 @@ static inline char *ks_wlan_translate_scan(struct net_device *dev,
 	iwe.u.freq.m = ap->channel;
 	iwe.u.freq.m = frequency_list[iwe.u.freq.m - 1] * 100000;
 	iwe.u.freq.e = 1;
-	current_ev =
-	    iwe_stream_add_event(info, current_ev, end_buf, &iwe,
-				 IW_EV_FREQ_LEN);
+	current_ev = iwe_stream_add_event(info, current_ev,
+					  end_buf, &iwe, IW_EV_FREQ_LEN);
 
 	/* Add quality statistics */
 	iwe.cmd = IWEVQUAL;
 	iwe.u.qual.level = 256 - ap->rssi;
 	iwe.u.qual.qual = ap->sq;
 	iwe.u.qual.noise = 0;	/* invalid noise value */
-	current_ev =
-	    iwe_stream_add_event(info, current_ev, end_buf, &iwe,
-				 IW_EV_QUAL_LEN);
+	current_ev = iwe_stream_add_event(info, current_ev, end_buf,
+					  &iwe, IW_EV_QUAL_LEN);
 
 	/* Add encryption capability */
 	iwe.cmd = SIOCGIWENCODE;
-	if (capabilities & WLAN_CAPABILITY_PRIVACY)
-		iwe.u.data.flags = IW_ENCODE_ENABLED | IW_ENCODE_NOKEY;
-	else
-		iwe.u.data.flags = IW_ENCODE_DISABLED;
+	iwe.u.data.flags = (capabilities & WLAN_CAPABILITY_PRIVACY) ?
+			    (IW_ENCODE_ENABLED | IW_ENCODE_NOKEY) :
+			     IW_ENCODE_DISABLED;
 	iwe.u.data.length = 0;
-	current_ev =
-	    iwe_stream_add_point(info, current_ev, end_buf, &iwe,
-				 ap->ssid.body);
+	current_ev = iwe_stream_add_point(info, current_ev, end_buf,
+					  &iwe, ap->ssid.body);
 
-	/* Rate : stuffing multiple values in a single event require a bit
-	 * more of magic - Jean II
+	/*
+	 * Rate : stuffing multiple values in a single event
+	 * require a bit more of magic - Jean II
 	 */
 	current_val = current_ev + IW_EV_LCP_LEN;
 
@@ -1394,71 +1260,34 @@ static inline char *ks_wlan_translate_scan(struct net_device *dev,
 		/* Bit rate given in 500 kb/s units (+ 0x80) */
 		iwe.u.bitrate.value = ((ap->rate_set.body[i] & 0x7f) * 500000);
 		/* Add new value to event */
-		current_val =
-		    iwe_stream_add_value(info, current_ev, current_val, end_buf,
-					 &iwe, IW_EV_PARAM_LEN);
+		current_val = iwe_stream_add_value(info, current_ev,
+						   current_val, end_buf, &iwe,
+						   IW_EV_PARAM_LEN);
 	}
 	/* Check if we added any event */
 	if ((current_val - current_ev) > IW_EV_LCP_LEN)
 		current_ev = current_val;
 
-#define GENERIC_INFO_ELEM_ID 0xdd
-#define RSN_INFO_ELEM_ID 0x30
-	if (ap->rsn_ie.id == RSN_INFO_ELEM_ID && ap->rsn_ie.size != 0) {
-		pbuf = &buf0[0];
-		memset(&iwe, 0, sizeof(iwe));
-		iwe.cmd = IWEVCUSTOM;
-		memcpy(buf0, rsn_leader, sizeof(rsn_leader) - 1);
-		iwe.u.data.length += sizeof(rsn_leader) - 1;
-		pbuf += sizeof(rsn_leader) - 1;
-
-		pbuf += sprintf(pbuf, "%02x", ap->rsn_ie.id);
-		pbuf += sprintf(pbuf, "%02x", ap->rsn_ie.size);
-		iwe.u.data.length += 4;
-
-		for (i = 0; i < ap->rsn_ie.size; i++)
-			pbuf += sprintf(pbuf, "%02x", ap->rsn_ie.body[i]);
-		iwe.u.data.length += (ap->rsn_ie.size) * 2;
-
-		netdev_dbg(dev, "ap->rsn.size=%d\n", ap->rsn_ie.size);
-
-		current_ev =
-		    iwe_stream_add_point(info, current_ev, end_buf, &iwe,
-					 &buf0[0]);
-	}
-	if (ap->wpa_ie.id == GENERIC_INFO_ELEM_ID && ap->wpa_ie.size != 0) {
-		pbuf = &buf1[0];
-		memset(&iwe, 0, sizeof(iwe));
-		iwe.cmd = IWEVCUSTOM;
-		memcpy(buf1, wpa_leader, sizeof(wpa_leader) - 1);
-		iwe.u.data.length += sizeof(wpa_leader) - 1;
-		pbuf += sizeof(wpa_leader) - 1;
-
-		pbuf += sprintf(pbuf, "%02x", ap->wpa_ie.id);
-		pbuf += sprintf(pbuf, "%02x", ap->wpa_ie.size);
-		iwe.u.data.length += 4;
-
-		for (i = 0; i < ap->wpa_ie.size; i++)
-			pbuf += sprintf(pbuf, "%02x", ap->wpa_ie.body[i]);
-		iwe.u.data.length += (ap->wpa_ie.size) * 2;
-
-		netdev_dbg(dev, "ap->rsn.size=%d\n", ap->wpa_ie.size);
-		netdev_dbg(dev, "iwe.u.data.length=%d\n", iwe.u.data.length);
-
-		current_ev =
-		    iwe_stream_add_point(info, current_ev, end_buf, &iwe,
-					 &buf1[0]);
-	}
+	if (ap->rsn_ie.id == RSN_INFO_ELEM_ID && ap->rsn_ie.size != 0)
+		current_ev = ks_wlan_add_leader_event(rsn_leader, end_buf,
+						      current_ev, &ap->rsn_ie,
+						      &iwe, info);
+
+	if (ap->wpa_ie.id == WPA_INFO_ELEM_ID && ap->wpa_ie.size != 0)
+		current_ev = ks_wlan_add_leader_event(wpa_leader, end_buf,
+						      current_ev, &ap->wpa_ie,
+						      &iwe, info);
 
-	/* The other data in the scan result are not really
+	/*
+	 * The other data in the scan result are not really
 	 * interesting, so for now drop it - Jean II
 	 */
 	return current_ev;
 }
 
 static int ks_wlan_get_scan(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_point *dwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	int i;
@@ -1479,25 +1308,26 @@ static int ks_wlan_get_scan(struct net_device *dev,
 
 	/* Read and parse all entries */
 	for (i = 0; i < priv->aplist.size; i++) {
-		if ((extra + dwrq->length) - current_ev <= IW_EV_ADDR_LEN) {
-			dwrq->length = 0;
+		if ((extra + dwrq->data.length) - current_ev <= IW_EV_ADDR_LEN) {
+			dwrq->data.length = 0;
 			return -E2BIG;
 		}
 		/* Translate to WE format this entry */
 		current_ev = ks_wlan_translate_scan(dev, info, current_ev,
-						    extra + dwrq->length,
+						    extra + dwrq->data.length,
 						    &priv->aplist.ap[i]);
 	}
 	/* Length of data */
-	dwrq->length = (current_ev - extra);
-	dwrq->flags = 0;
+	dwrq->data.length = (current_ev - extra);
+	dwrq->data.flags = 0;
 
 	return 0;
 }
 
 /* called after a bunch of SET operations */
 static int ks_wlan_config_commit(struct net_device *dev,
-				 struct iw_request_info *info, void *zwrq,
+				 struct iw_request_info *info,
+				 union iwreq_data *zwrq,
 				 char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
@@ -1513,7 +1343,7 @@ static int ks_wlan_config_commit(struct net_device *dev,
 /* set association ie params */
 static int ks_wlan_set_genie(struct net_device *dev,
 			     struct iw_request_info *info,
-			     struct iw_point *dwrq, char *extra)
+			     union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
@@ -1526,11 +1356,12 @@ static int ks_wlan_set_genie(struct net_device *dev,
 
 static int ks_wlan_set_auth_mode(struct net_device *dev,
 				 struct iw_request_info *info,
-				 struct iw_param *vwrq, char *extra)
+				 union iwreq_data *vwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
-	int index = (vwrq->flags & IW_AUTH_INDEX);
-	int value = vwrq->value;
+	struct iw_param *param = &vwrq->param;
+	int index = (param->flags & IW_AUTH_INDEX);
+	int value = param->value;
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
@@ -1541,14 +1372,14 @@ static int ks_wlan_set_auth_mode(struct net_device *dev,
 		case IW_AUTH_WPA_VERSION_DISABLED:
 			priv->wpa.version = value;
 			if (priv->wpa.rsn_enabled)
-				priv->wpa.rsn_enabled = 0;
+				priv->wpa.rsn_enabled = false;
 			priv->need_commit |= SME_RSN;
 			break;
 		case IW_AUTH_WPA_VERSION_WPA:
 		case IW_AUTH_WPA_VERSION_WPA2:
 			priv->wpa.version = value;
 			if (!(priv->wpa.rsn_enabled))
-				priv->wpa.rsn_enabled = 1;
+				priv->wpa.rsn_enabled = true;
 			priv->need_commit |= SME_RSN;
 			break;
 		default:
@@ -1658,10 +1489,11 @@ static int ks_wlan_set_auth_mode(struct net_device *dev,
 
 static int ks_wlan_get_auth_mode(struct net_device *dev,
 				 struct iw_request_info *info,
-				 struct iw_param *vwrq, char *extra)
+				 union iwreq_data *vwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
-	int index = (vwrq->flags & IW_AUTH_INDEX);
+	struct iw_param *param = &vwrq->param;
+	int index = (param->flags & IW_AUTH_INDEX);
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
@@ -1670,22 +1502,22 @@ static int ks_wlan_get_auth_mode(struct net_device *dev,
 	/*  WPA (not used ?? wpa_supplicant) */
 	switch (index) {
 	case IW_AUTH_WPA_VERSION:
-		vwrq->value = priv->wpa.version;
+		param->value = priv->wpa.version;
 		break;
 	case IW_AUTH_CIPHER_PAIRWISE:
-		vwrq->value = priv->wpa.pairwise_suite;
+		param->value = priv->wpa.pairwise_suite;
 		break;
 	case IW_AUTH_CIPHER_GROUP:
-		vwrq->value = priv->wpa.group_suite;
+		param->value = priv->wpa.group_suite;
 		break;
 	case IW_AUTH_KEY_MGMT:
-		vwrq->value = priv->wpa.key_mgmt_suite;
+		param->value = priv->wpa.key_mgmt_suite;
 		break;
 	case IW_AUTH_80211_AUTH_ALG:
-		vwrq->value = priv->wpa.auth_alg;
+		param->value = priv->wpa.auth_alg;
 		break;
 	case IW_AUTH_WPA_ENABLED:
-		vwrq->value = priv->wpa.rsn_enabled;
+		param->value = priv->wpa.rsn_enabled;
 		break;
 	case IW_AUTH_RX_UNENCRYPTED_EAPOL:	/* OK??? */
 	case IW_AUTH_TKIP_COUNTERMEASURES:
@@ -1700,13 +1532,13 @@ static int ks_wlan_get_auth_mode(struct net_device *dev,
 /* set encoding token & mode (WPA)*/
 static int ks_wlan_set_encode_ext(struct net_device *dev,
 				  struct iw_request_info *info,
-				  struct iw_point *dwrq, char *extra)
+				  union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	struct iw_encode_ext *enc;
-	int index = dwrq->flags & IW_ENCODE_INDEX;
+	int index = dwrq->encoding.flags & IW_ENCODE_INDEX;
 	unsigned int commit = 0;
-	struct wpa_key_t *key;
+	struct wpa_key *key;
 
 	enc = (struct iw_encode_ext *)extra;
 	if (!enc)
@@ -1721,7 +1553,7 @@ static int ks_wlan_set_encode_ext(struct net_device *dev,
 	index--;
 	key = &priv->wpa.key[index];
 
-	if (dwrq->flags & IW_ENCODE_DISABLED)
+	if (dwrq->encoding.flags & IW_ENCODE_DISABLED)
 		key->key_len = 0;
 
 	key->ext_flags = enc->ext_flags;
@@ -1732,7 +1564,7 @@ static int ks_wlan_set_encode_ext(struct net_device *dev,
 		memcpy(&key->rx_seq[0], &enc->rx_seq[0], IW_ENCODE_SEQ_MAX_SIZE);
 	}
 
-	memcpy(&key->addr.sa_data[0], &enc->addr.sa_data[0], ETH_ALEN);
+	ether_addr_copy(&key->addr.sa_data[0], &enc->addr.sa_data[0]);
 
 	switch (enc->alg) {
 	case IW_ENCODE_ALG_NONE:
@@ -1793,7 +1625,7 @@ static int ks_wlan_set_encode_ext(struct net_device *dev,
 /* get encoding token & mode (WPA)*/
 static int ks_wlan_get_encode_ext(struct net_device *dev,
 				  struct iw_request_info *info,
-				  struct iw_point *dwrq, char *extra)
+				  union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
@@ -1813,12 +1645,12 @@ static int ks_wlan_get_encode_ext(struct net_device *dev,
 
 static int ks_wlan_set_pmksa(struct net_device *dev,
 			     struct iw_request_info *info,
-			     struct iw_point *dwrq, char *extra)
+			     union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	struct iw_pmksa *pmksa;
 	int i;
-	struct pmk_t *pmk;
+	struct pmk *pmk;
 	struct list_head *ptr;
 
 	if (priv->sleep_mode == SLP_SLEEP)
@@ -1832,68 +1664,65 @@ static int ks_wlan_set_pmksa(struct net_device *dev,
 
 	switch (pmksa->cmd) {
 	case IW_PMKSA_ADD:
-		if (list_empty(&priv->pmklist.head)) {	/* new list */
+		if (list_empty(&priv->pmklist.head)) {
 			for (i = 0; i < PMK_LIST_MAX; i++) {
 				pmk = &priv->pmklist.pmk[i];
-				if (memcmp("\x00\x00\x00\x00\x00\x00",
-					   pmk->bssid, ETH_ALEN) == 0)
-					break; /* loop */
+				if (is_zero_ether_addr(pmk->bssid))
+					break;
 			}
-			memcpy(pmk->bssid, pmksa->bssid.sa_data, ETH_ALEN);
+			ether_addr_copy(pmk->bssid, pmksa->bssid.sa_data);
 			memcpy(pmk->pmkid, pmksa->pmkid, IW_PMKID_LEN);
 			list_add(&pmk->list, &priv->pmklist.head);
 			priv->pmklist.size++;
-			break;	/* case */
+			break;
 		}
 		/* search cache data */
 		list_for_each(ptr, &priv->pmklist.head) {
-			pmk = list_entry(ptr, struct pmk_t, list);
-			if (memcmp(pmksa->bssid.sa_data, pmk->bssid, ETH_ALEN) == 0) {
+			pmk = list_entry(ptr, struct pmk, list);
+			if (ether_addr_equal(pmksa->bssid.sa_data, pmk->bssid)) {
 				memcpy(pmk->pmkid, pmksa->pmkid, IW_PMKID_LEN);
 				list_move(&pmk->list, &priv->pmklist.head);
-				break; /* list_for_each */
+				break;
 			}
 		}
-		if (ptr != &priv->pmklist.head)	/* not find address. */
-			break;	/* case */
-
-		if (priv->pmklist.size < PMK_LIST_MAX) {	/* new cache data */
+		/* not find address. */
+		if (ptr != &priv->pmklist.head)
+			break;
+		/* new cache data */
+		if (priv->pmklist.size < PMK_LIST_MAX) {
 			for (i = 0; i < PMK_LIST_MAX; i++) {
 				pmk = &priv->pmklist.pmk[i];
-				if (memcmp("\x00\x00\x00\x00\x00\x00",
-					   pmk->bssid, ETH_ALEN) == 0)
-					break; /* loop */
+				if (is_zero_ether_addr(pmk->bssid))
+					break;
 			}
-			memcpy(pmk->bssid, pmksa->bssid.sa_data, ETH_ALEN);
+			ether_addr_copy(pmk->bssid, pmksa->bssid.sa_data);
 			memcpy(pmk->pmkid, pmksa->pmkid, IW_PMKID_LEN);
 			list_add(&pmk->list, &priv->pmklist.head);
 			priv->pmklist.size++;
-		} else {	/* overwrite old cache data */
-			pmk = list_entry(priv->pmklist.head.prev, struct pmk_t,
+		} else { /* overwrite old cache data */
+			pmk = list_entry(priv->pmklist.head.prev, struct pmk,
 					 list);
-			memcpy(pmk->bssid, pmksa->bssid.sa_data, ETH_ALEN);
+			ether_addr_copy(pmk->bssid, pmksa->bssid.sa_data);
 			memcpy(pmk->pmkid, pmksa->pmkid, IW_PMKID_LEN);
 			list_move(&pmk->list, &priv->pmklist.head);
 		}
 		break;
 	case IW_PMKSA_REMOVE:
-		if (list_empty(&priv->pmklist.head)) {	/* list empty */
+		if (list_empty(&priv->pmklist.head))
 			return -EINVAL;
-		}
 		/* search cache data */
 		list_for_each(ptr, &priv->pmklist.head) {
-			pmk = list_entry(ptr, struct pmk_t, list);
-			if (memcmp(pmksa->bssid.sa_data, pmk->bssid, ETH_ALEN) == 0) {
+			pmk = list_entry(ptr, struct pmk, list);
+			if (ether_addr_equal(pmksa->bssid.sa_data, pmk->bssid)) {
 				eth_zero_addr(pmk->bssid);
 				memset(pmk->pmkid, 0, IW_PMKID_LEN);
 				list_del_init(&pmk->list);
 				break;
 			}
 		}
-		if (ptr == &priv->pmklist.head) {	/* not find address. */
+		/* not find address. */
+		if (ptr == &priv->pmklist.head)
 			return 0;
-		}
-
 		break;
 	case IW_PMKSA_FLUSH:
 		memset(&priv->pmklist, 0, sizeof(priv->pmklist));
@@ -1914,14 +1743,11 @@ static struct iw_statistics *ks_get_wireless_stats(struct net_device *dev)
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	struct iw_statistics *wstats = &priv->wstats;
 
-	if (!atomic_read(&update_phyinfo)) {
-		if (priv->dev_state < DEVICE_STATE_READY)
-			return NULL;	/* not finished initialize */
-		else
-			return wstats;
-	}
+	if (!atomic_read(&update_phyinfo))
+		return (priv->dev_state < DEVICE_STATE_READY) ? NULL : wstats;
 
-	/* Packets discarded in the wireless adapter due to wireless
+	/*
+	 * Packets discarded in the wireless adapter due to wireless
 	 * specific problems
 	 */
 	wstats->discard.nwid = 0;	/* Rx invalid nwid      */
@@ -1953,28 +1779,25 @@ static int ks_wlan_set_stop_request(struct net_device *dev,
 
 #include <linux/ieee80211.h>
 static int ks_wlan_set_mlme(struct net_device *dev,
-			    struct iw_request_info *info, struct iw_point *dwrq,
-			    char *extra)
+			    struct iw_request_info *info,
+			    union iwreq_data *dwrq, char *extra)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	struct iw_mlme *mlme = (struct iw_mlme *)extra;
-	__u32 mode;
+	__u32 mode = 1;
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 
-	/* for SLEEP MODE */
-	switch (mlme->cmd) {
-	case IW_MLME_DEAUTH:
-		if (mlme->reason_code == WLAN_REASON_MIC_FAILURE)
-			return 0;
-		/* fall through */
-	case IW_MLME_DISASSOC:
-		mode = 1;
-		return ks_wlan_set_stop_request(dev, NULL, &mode, NULL);
-	default:
-		return -EOPNOTSUPP;	/* Not Support */
-	}
+	if (mlme->cmd != IW_MLME_DEAUTH &&
+	    mlme->cmd != IW_MLME_DISASSOC)
+		return -EOPNOTSUPP;
+
+	if (mlme->cmd == IW_MLME_DEAUTH &&
+	    mlme->reason_code == WLAN_REASON_MIC_FAILURE)
+		return 0;
+
+	return ks_wlan_set_stop_request(dev, NULL, &mode, NULL);
 }
 
 static int ks_wlan_get_firmware_version(struct net_device *dev,
@@ -1998,14 +1821,10 @@ static int ks_wlan_set_preamble(struct net_device *dev,
 		return -EPERM;
 
 	/* for SLEEP MODE */
-	if (*uwrq == LONG_PREAMBLE) {	/* 0 */
-		priv->reg.preamble = LONG_PREAMBLE;
-	} else if (*uwrq == SHORT_PREAMBLE) {	/* 1 */
-		priv->reg.preamble = SHORT_PREAMBLE;
-	} else {
+	if (*uwrq != LONG_PREAMBLE && *uwrq != SHORT_PREAMBLE)
 		return -EINVAL;
-	}
 
+	priv->reg.preamble = *uwrq;
 	priv->need_commit |= SME_MODE_SET;
 	return -EINPROGRESS;	/* Call commit handler */
 }
@@ -2033,23 +1852,16 @@ static int ks_wlan_set_power_mgmt(struct net_device *dev,
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 
-	/* for SLEEP MODE */
-	if (*uwrq == POWER_MGMT_ACTIVE) {	/* 0 */
-		priv->reg.power_mgmt = POWER_MGMT_ACTIVE;
-	} else if (*uwrq == POWER_MGMT_SAVE1) {	/* 1 */
-		if (priv->reg.operation_mode == MODE_INFRASTRUCTURE)
-			priv->reg.power_mgmt = POWER_MGMT_SAVE1;
-		else
-			return -EINVAL;
-	} else if (*uwrq == POWER_MGMT_SAVE2) {	/* 2 */
-		if (priv->reg.operation_mode == MODE_INFRASTRUCTURE)
-			priv->reg.power_mgmt = POWER_MGMT_SAVE2;
-		else
-			return -EINVAL;
-	} else {
+	if (*uwrq != POWER_MGMT_ACTIVE &&
+	    *uwrq != POWER_MGMT_SAVE1 &&
+	    *uwrq != POWER_MGMT_SAVE2)
+		return -EINVAL;
+
+	if ((*uwrq == POWER_MGMT_SAVE1 || *uwrq == POWER_MGMT_SAVE2) &&
+	    (priv->reg.operation_mode != MODE_INFRASTRUCTURE))
 		return -EINVAL;
-	}
 
+	priv->reg.power_mgmt = *uwrq;
 	hostif_sme_enqueue(priv, SME_POW_MNGMT_REQUEST);
 
 	return 0;
@@ -2078,14 +1890,11 @@ static int ks_wlan_set_scan_type(struct net_device *dev,
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 	/* for SLEEP MODE */
-	if (*uwrq == ACTIVE_SCAN) {	/* 0 */
-		priv->reg.scan_type = ACTIVE_SCAN;
-	} else if (*uwrq == PASSIVE_SCAN) {	/* 1 */
-		priv->reg.scan_type = PASSIVE_SCAN;
-	} else {
+
+	if (*uwrq != ACTIVE_SCAN && *uwrq != PASSIVE_SCAN)
 		return -EINVAL;
-	}
 
+	priv->reg.scan_type = *uwrq;
 	return 0;
 }
 
@@ -2111,17 +1920,17 @@ static int ks_wlan_set_beacon_lost(struct net_device *dev,
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 	/* for SLEEP MODE */
-	if (*uwrq >= BEACON_LOST_COUNT_MIN && *uwrq <= BEACON_LOST_COUNT_MAX)
-		priv->reg.beacon_lost_count = *uwrq;
-	else
+	if (*uwrq > BEACON_LOST_COUNT_MAX)
 		return -EINVAL;
 
+	priv->reg.beacon_lost_count = *uwrq;
+
 	if (priv->reg.operation_mode == MODE_INFRASTRUCTURE) {
 		priv->need_commit |= SME_MODE_SET;
 		return -EINPROGRESS;	/* Call commit handler */
-	} else {
-		return 0;
 	}
+
+	return 0;
 }
 
 static int ks_wlan_get_beacon_lost(struct net_device *dev,
@@ -2145,17 +1954,14 @@ static int ks_wlan_set_phy_type(struct net_device *dev,
 
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	/* for SLEEP MODE */
-	if (*uwrq == D_11B_ONLY_MODE) {	/* 0 */
-		priv->reg.phy_type = D_11B_ONLY_MODE;
-	} else if (*uwrq == D_11G_ONLY_MODE) {	/* 1 */
-		priv->reg.phy_type = D_11G_ONLY_MODE;
-	} else if (*uwrq == D_11BG_COMPATIBLE_MODE) {	/* 2 */
-		priv->reg.phy_type = D_11BG_COMPATIBLE_MODE;
-	} else {
+
+	if (*uwrq != D_11B_ONLY_MODE &&
+	    *uwrq != D_11G_ONLY_MODE &&
+	    *uwrq != D_11BG_COMPATIBLE_MODE)
 		return -EINVAL;
-	}
 
+	/* for SLEEP MODE */
+	priv->reg.phy_type = *uwrq;
 	priv->need_commit |= SME_MODE_SET;
 	return -EINPROGRESS;	/* Call commit handler */
 }
@@ -2182,18 +1988,13 @@ static int ks_wlan_set_cts_mode(struct net_device *dev,
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 	/* for SLEEP MODE */
-	if (*uwrq == CTS_MODE_FALSE) {	/* 0 */
-		priv->reg.cts_mode = CTS_MODE_FALSE;
-	} else if (*uwrq == CTS_MODE_TRUE) {	/* 1 */
-		if (priv->reg.phy_type == D_11G_ONLY_MODE ||
-		    priv->reg.phy_type == D_11BG_COMPATIBLE_MODE) {
-			priv->reg.cts_mode = CTS_MODE_TRUE;
-		} else {
-			priv->reg.cts_mode = CTS_MODE_FALSE;
-		}
-	} else {
+	if (*uwrq != CTS_MODE_FALSE && *uwrq != CTS_MODE_TRUE)
 		return -EINVAL;
-	}
+
+	priv->reg.cts_mode = (*uwrq == CTS_MODE_FALSE) ? *uwrq :
+			      (priv->reg.phy_type == D_11G_ONLY_MODE ||
+			       priv->reg.phy_type == D_11BG_COMPATIBLE_MODE) ?
+			       *uwrq : !*uwrq;
 
 	priv->need_commit |= SME_MODE_SET;
 	return -EINPROGRESS;	/* Call commit handler */
@@ -2218,22 +2019,20 @@ static int ks_wlan_set_sleep_mode(struct net_device *dev,
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
-	if (*uwrq == SLP_SLEEP) {
-		priv->sleep_mode = *uwrq;
-		netdev_info(dev, "SET_SLEEP_MODE %d\n", priv->sleep_mode);
-
-		hostif_sme_enqueue(priv, SME_STOP_REQUEST);
-		hostif_sme_enqueue(priv, SME_SLEEP_REQUEST);
-
-	} else if (*uwrq == SLP_ACTIVE) {
-		priv->sleep_mode = *uwrq;
-		netdev_info(dev, "SET_SLEEP_MODE %d\n", priv->sleep_mode);
-		hostif_sme_enqueue(priv, SME_SLEEP_REQUEST);
-	} else {
+	if (*uwrq != SLP_SLEEP &&
+	    *uwrq != SLP_ACTIVE) {
 		netdev_err(dev, "SET_SLEEP_MODE %d error\n", *uwrq);
 		return -EINVAL;
 	}
 
+	priv->sleep_mode = *uwrq;
+	netdev_info(dev, "SET_SLEEP_MODE %d\n", priv->sleep_mode);
+
+	if (*uwrq == SLP_SLEEP)
+		hostif_sme_enqueue(priv, SME_STOP_REQUEST);
+
+	hostif_sme_enqueue(priv, SME_SLEEP_REQUEST);
+
 	return 0;
 }
 
@@ -2248,8 +2047,6 @@ static int ks_wlan_get_sleep_mode(struct net_device *dev,
 	return 0;
 }
 
-#ifdef WPS
-
 static int ks_wlan_set_wps_enable(struct net_device *dev,
 				  struct iw_request_info *info, __u32 *uwrq,
 				  char *extra)
@@ -2259,11 +2056,10 @@ static int ks_wlan_set_wps_enable(struct net_device *dev,
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 	/* for SLEEP MODE */
-	if (*uwrq == 0 || *uwrq == 1)
-		priv->wps.wps_enabled = *uwrq;
-	else
+	if (*uwrq != 0 && *uwrq != 1)
 		return -EINVAL;
 
+	priv->wps.wps_enabled = *uwrq;
 	hostif_sme_enqueue(priv, SME_WPS_ENABLE_REQUEST);
 
 	return 0;
@@ -2314,7 +2110,6 @@ static int ks_wlan_set_wps_probe_req(struct net_device *dev,
 
 	return 0;
 }
-#endif /* WPS */
 
 static int ks_wlan_set_tx_gain(struct net_device *dev,
 			       struct iw_request_info *info, __u32 *uwrq,
@@ -2325,16 +2120,11 @@ static int ks_wlan_set_tx_gain(struct net_device *dev,
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 	/* for SLEEP MODE */
-	if (*uwrq >= 0 && *uwrq <= 0xFF)	/* 0-255 */
-		priv->gain.tx_gain = (uint8_t)*uwrq;
-	else
+	if (*uwrq > 0xFF)
 		return -EINVAL;
 
-	if (priv->gain.tx_gain < 0xFF)
-		priv->gain.tx_mode = 1;
-	else
-		priv->gain.tx_mode = 0;
-
+	priv->gain.tx_gain = (u8)*uwrq;
+	priv->gain.tx_mode = (priv->gain.tx_gain < 0xFF) ? 1 : 0;
 	hostif_sme_enqueue(priv, SME_SET_GAIN);
 	return 0;
 }
@@ -2362,16 +2152,11 @@ static int ks_wlan_set_rx_gain(struct net_device *dev,
 	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
 	/* for SLEEP MODE */
-	if (*uwrq >= 0 && *uwrq <= 0xFF)	/* 0-255 */
-		priv->gain.rx_gain = (uint8_t)*uwrq;
-	else
+	if (*uwrq > 0xFF)
 		return -EINVAL;
 
-	if (priv->gain.rx_gain < 0xFF)
-		priv->gain.rx_mode = 1;
-	else
-		priv->gain.rx_mode = 0;
-
+	priv->gain.rx_gain = (u8)*uwrq;
+	priv->gain.rx_mode = (priv->gain.rx_gain < 0xFF) ? 1 : 0;
 	hostif_sme_enqueue(priv, SME_SET_GAIN);
 	return 0;
 }
@@ -2535,14 +2320,12 @@ static const struct iw_priv_args ks_wlan_private_args[] = {
 /*{ cmd, set_args, get_args, name[16] } */
 	{KS_WLAN_GET_FIRM_VERSION, IW_PRIV_TYPE_NONE,
 	 IW_PRIV_TYPE_CHAR | (128 + 1), "GetFirmwareVer"},
-#ifdef WPS
 	{KS_WLAN_SET_WPS_ENABLE, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
 	 IW_PRIV_TYPE_NONE, "SetWPSEnable"},
 	{KS_WLAN_GET_WPS_ENABLE, IW_PRIV_TYPE_NONE,
 	 IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "GetW"},
 	{KS_WLAN_SET_WPS_PROBE_REQ, IW_PRIV_TYPE_BYTE | 2047, IW_PRIV_TYPE_NONE,
 	 "SetWPSProbeReq"},
-#endif /* WPS */
 	{KS_WLAN_SET_PREAMBLE, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
 	 IW_PRIV_TYPE_NONE, "SetPreamble"},
 	{KS_WLAN_GET_PREAMBLE, IW_PRIV_TYPE_NONE,
@@ -2586,105 +2369,62 @@ static const struct iw_priv_args ks_wlan_private_args[] = {
 };
 
 static const iw_handler ks_wlan_handler[] = {
-	(iw_handler)ks_wlan_config_commit,	/* SIOCSIWCOMMIT */
-	(iw_handler)ks_wlan_get_name,	/* SIOCGIWNAME */
-	(iw_handler)NULL,	/* SIOCSIWNWID */
-	(iw_handler)NULL,	/* SIOCGIWNWID */
-	(iw_handler)ks_wlan_set_freq,	/* SIOCSIWFREQ */
-	(iw_handler)ks_wlan_get_freq,	/* SIOCGIWFREQ */
-	(iw_handler)ks_wlan_set_mode,	/* SIOCSIWMODE */
-	(iw_handler)ks_wlan_get_mode,	/* SIOCGIWMODE */
-#ifndef KSC_OPNOTSUPP
-	(iw_handler)ks_wlan_set_sens,	/* SIOCSIWSENS */
-	(iw_handler)ks_wlan_get_sens,	/* SIOCGIWSENS */
-#else /* KSC_OPNOTSUPP */
-	(iw_handler)NULL,	/* SIOCSIWSENS */
-	(iw_handler)NULL,	/* SIOCGIWSENS */
-#endif /* KSC_OPNOTSUPP */
-	(iw_handler)NULL,	/* SIOCSIWRANGE */
-	(iw_handler)ks_wlan_get_range,	/* SIOCGIWRANGE */
-	(iw_handler)NULL,	/* SIOCSIWPRIV */
-	(iw_handler)NULL,	/* SIOCGIWPRIV */
-	(iw_handler)NULL,	/* SIOCSIWSTATS */
-	(iw_handler)ks_wlan_get_iwstats,	/* SIOCGIWSTATS */
-	(iw_handler)NULL,	/* SIOCSIWSPY */
-	(iw_handler)NULL,	/* SIOCGIWSPY */
-	(iw_handler)NULL,	/* SIOCSIWTHRSPY */
-	(iw_handler)NULL,	/* SIOCGIWTHRSPY */
-	(iw_handler)ks_wlan_set_wap,	/* SIOCSIWAP */
-	(iw_handler)ks_wlan_get_wap,	/* SIOCGIWAP */
-//      (iw_handler)NULL,                      /* SIOCSIWMLME */
-	(iw_handler)ks_wlan_set_mlme,	/* SIOCSIWMLME */
-	(iw_handler)ks_wlan_get_aplist,	/* SIOCGIWAPLIST */
-	(iw_handler)ks_wlan_set_scan,	/* SIOCSIWSCAN */
-	(iw_handler)ks_wlan_get_scan,	/* SIOCGIWSCAN */
-	(iw_handler)ks_wlan_set_essid,	/* SIOCSIWESSID */
-	(iw_handler)ks_wlan_get_essid,	/* SIOCGIWESSID */
-	(iw_handler)ks_wlan_set_nick,	/* SIOCSIWNICKN */
-	(iw_handler)ks_wlan_get_nick,	/* SIOCGIWNICKN */
-	(iw_handler)NULL,	/* -- hole -- */
-	(iw_handler)NULL,	/* -- hole -- */
-	(iw_handler)ks_wlan_set_rate,	/* SIOCSIWRATE */
-	(iw_handler)ks_wlan_get_rate,	/* SIOCGIWRATE */
-	(iw_handler)ks_wlan_set_rts,	/* SIOCSIWRTS */
-	(iw_handler)ks_wlan_get_rts,	/* SIOCGIWRTS */
-	(iw_handler)ks_wlan_set_frag,	/* SIOCSIWFRAG */
-	(iw_handler)ks_wlan_get_frag,	/* SIOCGIWFRAG */
-#ifndef KSC_OPNOTSUPP
-	(iw_handler)ks_wlan_set_txpow,	/* SIOCSIWTXPOW */
-	(iw_handler)ks_wlan_get_txpow,	/* SIOCGIWTXPOW */
-	(iw_handler)ks_wlan_set_retry,	/* SIOCSIWRETRY */
-	(iw_handler)ks_wlan_get_retry,	/* SIOCGIWRETRY */
-#else /* KSC_OPNOTSUPP */
-	(iw_handler)NULL,	/* SIOCSIWTXPOW */
-	(iw_handler)NULL,	/* SIOCGIWTXPOW */
-	(iw_handler)NULL,	/* SIOCSIWRETRY */
-	(iw_handler)NULL,	/* SIOCGIWRETRY */
-#endif /* KSC_OPNOTSUPP */
-	(iw_handler)ks_wlan_set_encode,	/* SIOCSIWENCODE */
-	(iw_handler)ks_wlan_get_encode,	/* SIOCGIWENCODE */
-	(iw_handler)ks_wlan_set_power,	/* SIOCSIWPOWER */
-	(iw_handler)ks_wlan_get_power,	/* SIOCGIWPOWER */
-	(iw_handler)NULL,	/* -- hole -- */
-	(iw_handler)NULL,	/* -- hole -- */
-//      (iw_handler)NULL,                      /* SIOCSIWGENIE */
-	(iw_handler)ks_wlan_set_genie,	/* SIOCSIWGENIE */
-	(iw_handler)NULL,	/* SIOCGIWGENIE */
-	(iw_handler)ks_wlan_set_auth_mode,	/* SIOCSIWAUTH */
-	(iw_handler)ks_wlan_get_auth_mode,	/* SIOCGIWAUTH */
-	(iw_handler)ks_wlan_set_encode_ext,	/* SIOCSIWENCODEEXT */
-	(iw_handler)ks_wlan_get_encode_ext,	/* SIOCGIWENCODEEXT */
-	(iw_handler)ks_wlan_set_pmksa,	/* SIOCSIWPMKSA */
-	(iw_handler)NULL,	/* -- hole -- */
+	IW_HANDLER(SIOCSIWCOMMIT, ks_wlan_config_commit),
+	IW_HANDLER(SIOCGIWNAME, ks_wlan_get_name),
+	IW_HANDLER(SIOCSIWFREQ, ks_wlan_set_freq),
+	IW_HANDLER(SIOCGIWFREQ, ks_wlan_get_freq),
+	IW_HANDLER(SIOCSIWMODE, ks_wlan_set_mode),
+	IW_HANDLER(SIOCGIWMODE, ks_wlan_get_mode),
+	IW_HANDLER(SIOCGIWRANGE, ks_wlan_get_range),
+	IW_HANDLER(SIOCGIWSTATS, ks_wlan_get_iwstats),
+	IW_HANDLER(SIOCSIWAP, ks_wlan_set_wap),
+	IW_HANDLER(SIOCGIWAP, ks_wlan_get_wap),
+	IW_HANDLER(SIOCSIWMLME, ks_wlan_set_mlme),
+	IW_HANDLER(SIOCGIWAPLIST, ks_wlan_get_aplist),
+	IW_HANDLER(SIOCSIWSCAN, ks_wlan_set_scan),
+	IW_HANDLER(SIOCGIWSCAN, ks_wlan_get_scan),
+	IW_HANDLER(SIOCSIWESSID, ks_wlan_set_essid),
+	IW_HANDLER(SIOCGIWESSID, ks_wlan_get_essid),
+	IW_HANDLER(SIOCSIWNICKN, ks_wlan_set_nick),
+	IW_HANDLER(SIOCGIWNICKN, ks_wlan_get_nick),
+	IW_HANDLER(SIOCSIWRATE, ks_wlan_set_rate),
+	IW_HANDLER(SIOCGIWRATE, ks_wlan_get_rate),
+	IW_HANDLER(SIOCSIWRTS, ks_wlan_set_rts),
+	IW_HANDLER(SIOCGIWRTS, ks_wlan_get_rts),
+	IW_HANDLER(SIOCSIWFRAG, ks_wlan_set_frag),
+	IW_HANDLER(SIOCGIWFRAG, ks_wlan_get_frag),
+	IW_HANDLER(SIOCSIWENCODE, ks_wlan_set_encode),
+	IW_HANDLER(SIOCGIWENCODE, ks_wlan_get_encode),
+	IW_HANDLER(SIOCSIWPOWER, ks_wlan_set_power),
+	IW_HANDLER(SIOCGIWPOWER, ks_wlan_get_power),
+	IW_HANDLER(SIOCSIWGENIE, ks_wlan_set_genie),
+	IW_HANDLER(SIOCSIWAUTH, ks_wlan_set_auth_mode),
+	IW_HANDLER(SIOCGIWAUTH, ks_wlan_get_auth_mode),
+	IW_HANDLER(SIOCSIWENCODEEXT, ks_wlan_set_encode_ext),
+	IW_HANDLER(SIOCGIWENCODEEXT, ks_wlan_get_encode_ext),
+	IW_HANDLER(SIOCSIWPMKSA, ks_wlan_set_pmksa),
 };
 
 /* private_handler */
 static const iw_handler ks_wlan_private_handler[] = {
-	(iw_handler)NULL,	/*  0 */
-	(iw_handler)NULL,	/*  1, used to be: KS_WLAN_GET_DRIVER_VERSION */
-	(iw_handler)NULL,	/*  2 */
-	(iw_handler)ks_wlan_get_firmware_version,	/*  3 KS_WLAN_GET_FIRM_VERSION */
-#ifdef WPS
-	(iw_handler)ks_wlan_set_wps_enable,	/*  4 KS_WLAN_SET_WPS_ENABLE  */
-	(iw_handler)ks_wlan_get_wps_enable,	/*  5 KS_WLAN_GET_WPS_ENABLE  */
-	(iw_handler)ks_wlan_set_wps_probe_req,	/*  6 KS_WLAN_SET_WPS_PROBE_REQ */
-#else
-	(iw_handler)NULL,	/*  4 */
-	(iw_handler)NULL,	/*  5 */
-	(iw_handler)NULL,	/*  6 */
-#endif /* WPS */
-
-	(iw_handler)ks_wlan_get_eeprom_cksum,	/*  7 KS_WLAN_GET_CONNECT */
-	(iw_handler)ks_wlan_set_preamble,	/*  8 KS_WLAN_SET_PREAMBLE */
-	(iw_handler)ks_wlan_get_preamble,	/*  9 KS_WLAN_GET_PREAMBLE */
+	(iw_handler)NULL,			/* 0 */
+	(iw_handler)NULL,			/* 1, KS_WLAN_GET_DRIVER_VERSION */
+	(iw_handler)NULL,			/* 2 */
+	(iw_handler)ks_wlan_get_firmware_version,/* 3 KS_WLAN_GET_FIRM_VERSION */
+	(iw_handler)ks_wlan_set_wps_enable,	/* 4 KS_WLAN_SET_WPS_ENABLE */
+	(iw_handler)ks_wlan_get_wps_enable,	/* 5 KS_WLAN_GET_WPS_ENABLE */
+	(iw_handler)ks_wlan_set_wps_probe_req,	/* 6 KS_WLAN_SET_WPS_PROBE_REQ */
+	(iw_handler)ks_wlan_get_eeprom_cksum,	/* 7 KS_WLAN_GET_CONNECT */
+	(iw_handler)ks_wlan_set_preamble,	/* 8 KS_WLAN_SET_PREAMBLE */
+	(iw_handler)ks_wlan_get_preamble,	/* 9 KS_WLAN_GET_PREAMBLE */
 	(iw_handler)ks_wlan_set_power_mgmt,	/* 10 KS_WLAN_SET_POWER_SAVE */
 	(iw_handler)ks_wlan_get_power_mgmt,	/* 11 KS_WLAN_GET_POWER_SAVE */
 	(iw_handler)ks_wlan_set_scan_type,	/* 12 KS_WLAN_SET_SCAN_TYPE */
 	(iw_handler)ks_wlan_get_scan_type,	/* 13 KS_WLAN_GET_SCAN_TYPE */
 	(iw_handler)ks_wlan_set_rx_gain,	/* 14 KS_WLAN_SET_RX_GAIN */
 	(iw_handler)ks_wlan_get_rx_gain,	/* 15 KS_WLAN_GET_RX_GAIN */
-	(iw_handler)ks_wlan_hostt,	/* 16 KS_WLAN_HOSTT */
-	(iw_handler)NULL,	/* 17 */
+	(iw_handler)ks_wlan_hostt,		/* 16 KS_WLAN_HOSTT */
+	(iw_handler)NULL,			/* 17 */
 	(iw_handler)ks_wlan_set_beacon_lost,	/* 18 KS_WLAN_SET_BECAN_LOST */
 	(iw_handler)ks_wlan_get_beacon_lost,	/* 19 KS_WLAN_GET_BECAN_LOST */
 	(iw_handler)ks_wlan_set_tx_gain,	/* 20 KS_WLAN_SET_TX_GAIN */
@@ -2693,22 +2433,21 @@ static const iw_handler ks_wlan_private_handler[] = {
 	(iw_handler)ks_wlan_get_phy_type,	/* 23 KS_WLAN_GET_PHY_TYPE */
 	(iw_handler)ks_wlan_set_cts_mode,	/* 24 KS_WLAN_SET_CTS_MODE */
 	(iw_handler)ks_wlan_get_cts_mode,	/* 25 KS_WLAN_GET_CTS_MODE */
-	(iw_handler)NULL,	/* 26 */
-	(iw_handler)NULL,	/* 27 */
+	(iw_handler)NULL,			/* 26 */
+	(iw_handler)NULL,			/* 27 */
 	(iw_handler)ks_wlan_set_sleep_mode,	/* 28 KS_WLAN_SET_SLEEP_MODE */
 	(iw_handler)ks_wlan_get_sleep_mode,	/* 29 KS_WLAN_GET_SLEEP_MODE */
-	(iw_handler)NULL,	/* 30 */
-	(iw_handler)NULL,	/* 31 */
+	(iw_handler)NULL,			/* 30 */
+	(iw_handler)NULL,			/* 31 */
 };
 
 static const struct iw_handler_def ks_wlan_handler_def = {
-	.num_standard = sizeof(ks_wlan_handler) / sizeof(iw_handler),
-	.num_private = sizeof(ks_wlan_private_handler) / sizeof(iw_handler),
-	.num_private_args =
-	    sizeof(ks_wlan_private_args) / sizeof(struct iw_priv_args),
-	.standard = (iw_handler *)ks_wlan_handler,
-	.private = (iw_handler *)ks_wlan_private_handler,
-	.private_args = (struct iw_priv_args *)ks_wlan_private_args,
+	.num_standard = ARRAY_SIZE(ks_wlan_handler),
+	.num_private = ARRAY_SIZE(ks_wlan_private_handler),
+	.num_private_args = ARRAY_SIZE(ks_wlan_private_args),
+	.standard = ks_wlan_handler,
+	.private = ks_wlan_private_handler,
+	.private_args = ks_wlan_private_args,
 	.get_wireless_stats = ks_get_wireless_stats,
 };
 
@@ -2750,7 +2489,7 @@ int ks_wlan_set_mac_address(struct net_device *dev, void *addr)
 	if (netif_running(dev))
 		return -EBUSY;
 	memcpy(dev->dev_addr, mac_addr->sa_data, dev->addr_len);
-	memcpy(priv->eth_addr, mac_addr->sa_data, ETH_ALEN);
+	ether_addr_copy(priv->eth_addr, mac_addr->sa_data);
 
 	priv->mac_address_valid = false;
 	hostif_sme_enqueue(priv, SME_MACADDRESS_SET_REQUEST);
@@ -2818,7 +2557,7 @@ void send_packet_complete(struct ks_wlan_private *priv, struct sk_buff *skb)
  * This routine is not state sensitive and need not be SMP locked.
  */
 static
-void ks_wlan_set_multicast_list(struct net_device *dev)
+void ks_wlan_set_rx_mode(struct net_device *dev)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
@@ -2866,7 +2605,7 @@ static const struct net_device_ops ks_wlan_netdev_ops = {
 	.ndo_set_mac_address = ks_wlan_set_mac_address,
 	.ndo_get_stats = ks_wlan_get_stats,
 	.ndo_tx_timeout = ks_wlan_tx_timeout,
-	.ndo_set_rx_mode = ks_wlan_set_multicast_list,
+	.ndo_set_rx_mode = ks_wlan_set_rx_mode,
 };
 
 int ks_wlan_net_start(struct net_device *dev)
@@ -2876,24 +2615,15 @@ int ks_wlan_net_start(struct net_device *dev)
 
 	priv = netdev_priv(dev);
 	priv->mac_address_valid = false;
+	priv->is_device_open = true;
 	priv->need_commit = 0;
-
-	priv->device_open_status = 1;
-
 	/* phy information update timer */
 	atomic_set(&update_phyinfo, 0);
 	timer_setup(&update_phyinfo_timer, ks_wlan_update_phyinfo_timeout, 0);
 
 	/* dummy address set */
-	memcpy(priv->eth_addr, dummy_addr, ETH_ALEN);
-	dev->dev_addr[0] = priv->eth_addr[0];
-	dev->dev_addr[1] = priv->eth_addr[1];
-	dev->dev_addr[2] = priv->eth_addr[2];
-	dev->dev_addr[3] = priv->eth_addr[3];
-	dev->dev_addr[4] = priv->eth_addr[4];
-	dev->dev_addr[5] = priv->eth_addr[5];
-	dev->dev_addr[6] = 0x00;
-	dev->dev_addr[7] = 0x00;
+	ether_addr_copy(priv->eth_addr, dummy_addr);
+	ether_addr_copy(dev->dev_addr, priv->eth_addr);
 
 	/* The ks_wlan-specific entries in the device structure. */
 	dev->netdev_ops = &ks_wlan_netdev_ops;
@@ -2909,7 +2639,7 @@ int ks_wlan_net_stop(struct net_device *dev)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
-	priv->device_open_status = 0;
+	priv->is_device_open = false;
 	del_timer_sync(&update_phyinfo_timer);
 
 	if (netif_running(dev))
diff --git a/drivers/staging/ks7010/michael_mic.c b/drivers/staging/ks7010/michael_mic.c
index 292eae29c552..e6bd70846e98 100644
--- a/drivers/staging/ks7010/michael_mic.c
+++ b/drivers/staging/ks7010/michael_mic.c
@@ -1,30 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *   Driver for KeyStream wireless LAN
  *
  *   Copyright (C) 2005-2008 KeyStream Corp.
  *   Copyright (C) 2009 Renesas Technology Corp.
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License version 2 as
- *   published by the Free Software Foundation.
  */
 
-#include <linux/types.h>
-#include <linux/string.h>
-#include <linux/bitops.h>
 #include <asm/unaligned.h>
+#include <linux/bitops.h>
+#include <linux/string.h>
 #include "michael_mic.h"
 
 
 // Reset the state to the empty message.
-static inline void michael_clear(struct michael_mic_t *mic)
+static inline void michael_clear(struct michael_mic *mic)
 {
 	mic->l = mic->k0;
 	mic->r = mic->k1;
 	mic->m_bytes = 0;
 }
 
-static void michael_init(struct michael_mic_t *mic, u8 *key)
+static void michael_init(struct michael_mic *mic, u8 *key)
 {
 	// Set the key
 	mic->k0 = get_unaligned_le32(key);
@@ -34,20 +30,20 @@ static void michael_init(struct michael_mic_t *mic, u8 *key)
 	michael_clear(mic);
 }
 
-static inline void michael_block(struct michael_mic_t *mic)
+static inline void michael_block(struct michael_mic *mic)
 {
 	mic->r ^= rol32(mic->l, 17);
 	mic->l += mic->r;
 	mic->r ^= ((mic->l & 0xff00ff00) >> 8) |
 		  ((mic->l & 0x00ff00ff) << 8);
 	mic->l += mic->r;
-	mic->r ^= rol32(mic->l, 3);					\
+	mic->r ^= rol32(mic->l, 3);
 	mic->l += mic->r;
-	mic->r ^= ror32(mic->l, 2);					\
+	mic->r ^= ror32(mic->l, 2);
 	mic->l += mic->r;
 }
 
-static void michael_append(struct michael_mic_t *mic, u8 *src, int bytes)
+static void michael_append(struct michael_mic *mic, u8 *src, int bytes)
 {
 	int addlen;
 
@@ -81,7 +77,7 @@ static void michael_append(struct michael_mic_t *mic, u8 *src, int bytes)
 	}
 }
 
-static void michael_get_mic(struct michael_mic_t *mic, u8 *dst)
+static void michael_get_mic(struct michael_mic *mic, u8 *dst)
 {
 	u8 *data = mic->m;
 
@@ -110,8 +106,8 @@ static void michael_get_mic(struct michael_mic_t *mic, u8 *dst)
 	michael_clear(mic);
 }
 
-void michael_mic_function(struct michael_mic_t *mic, u8 *key,
-			  u8 *data, int len, u8 priority, u8 *result)
+void michael_mic_function(struct michael_mic *mic, u8 *key,
+			  u8 *data, unsigned int len, u8 priority, u8 *result)
 {
 	u8 pad_data[4] = { priority, 0, 0, 0 };
 	// Compute the MIC value
diff --git a/drivers/staging/ks7010/michael_mic.h b/drivers/staging/ks7010/michael_mic.h
index 894a8d4121a4..f0ac164b999b 100644
--- a/drivers/staging/ks7010/michael_mic.h
+++ b/drivers/staging/ks7010/michael_mic.h
@@ -1,16 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  *   Driver for KeyStream wireless LAN
  *
  *   Copyright (C) 2005-2008 KeyStream Corp.
  *   Copyright (C) 2009 Renesas Technology Corp.
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License version 2 as
- *   published by the Free Software Foundation.
  */
 
 /* MichaelMIC routine define */
-struct michael_mic_t {
+struct michael_mic {
 	u32 k0;	// Key
 	u32 k1;	// Key
 	u32 l;	// Current state
@@ -20,5 +17,5 @@ struct michael_mic_t {
 	u8 result[8];
 };
 
-void michael_mic_function(struct michael_mic_t *mic, u8 *key,
-			  u8 *data, int len, u8 priority, u8 *result);
+void michael_mic_function(struct michael_mic *mic, u8 *key,
+			  u8 *data, unsigned int len, u8 priority, u8 *result);
diff --git a/drivers/staging/lustre/README.txt b/drivers/staging/lustre/README.txt
index 0676243eea9e..783959240490 100644
--- a/drivers/staging/lustre/README.txt
+++ b/drivers/staging/lustre/README.txt
@@ -4,7 +4,7 @@ Lustre Parallel Filesystem Client
 The Lustre file system is an open-source, parallel file system
 that supports many requirements of leadership class HPC simulation
 environments.
-Born from from a research project at Carnegie Mellon University,
+Born from a research project at Carnegie Mellon University,
 the Lustre file system is a widely-used option in HPC.
 The Lustre file system provides a POSIX compliant file system interface,
 can scale to thousands of clients, petabytes of storage and
diff --git a/drivers/staging/lustre/TODO b/drivers/staging/lustre/TODO
index 94446487748a..5332cdb19bfa 100644
--- a/drivers/staging/lustre/TODO
+++ b/drivers/staging/lustre/TODO
@@ -12,7 +12,7 @@ addressed:
 * https://jira.hpdd.intel.com/browse/LU-9859
 *
 * Track all the cleanups and simplification of the libcfs module. Remove
-* functions the kernel provides. Possible intergrate some of the functionality
+* functions the kernel provides. Possibly integrate some of the functionality
 * into the kernel proper.
 *
 ******************************************************************************
diff --git a/drivers/staging/lustre/include/linux/libcfs/curproc.h b/drivers/staging/lustre/include/linux/libcfs/curproc.h
deleted file mode 100644
index 4702956805a6..000000000000
--- a/drivers/staging/lustre/include/linux/libcfs/curproc.h
+++ /dev/null
@@ -1,85 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- *
- * Copyright (c) 2011, 2012, Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * libcfs/include/libcfs/curproc.h
- *
- * Lustre curproc API declaration
- *
- * Author: Nikita Danilov <nikita@clusterfs.com>
- */
-
-#ifndef __LIBCFS_CURPROC_H__
-#define __LIBCFS_CURPROC_H__
-
-/*
- * Plus, platform-specific constant
- *
- * CFS_CURPROC_COMM_MAX,
- *
- * and opaque scalar type
- *
- * kernel_cap_t
- */
-
-/* check if task is running in compat mode.*/
-#define current_pid()		(current->pid)
-#define current_comm()		(current->comm)
-
-typedef u32 cfs_cap_t;
-
-#define CFS_CAP_FS_MASK (BIT(CAP_CHOWN) |		\
-			 BIT(CAP_DAC_OVERRIDE) |	\
-			 BIT(CAP_DAC_READ_SEARCH) |	\
-			 BIT(CAP_FOWNER) |		\
-			 BIT(CAP_FSETID) |		\
-			 BIT(CAP_LINUX_IMMUTABLE) | \
-			 BIT(CAP_SYS_ADMIN) |	\
-			 BIT(CAP_SYS_BOOT) |	\
-			 BIT(CAP_SYS_RESOURCE))
-
-static inline cfs_cap_t cfs_curproc_cap_pack(void)
-{
-	/* cfs_cap_t is only the first word of kernel_cap_t */
-	return (cfs_cap_t)(current_cap().cap[0]);
-}
-
-/* __LIBCFS_CURPROC_H__ */
-#endif
-/*
- * Local variables:
- * c-indentation-style: "K&R"
- * c-basic-offset: 8
- * tab-width: 8
- * fill-column: 80
- * scroll-step: 1
- * End:
- */
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs.h b/drivers/staging/lustre/include/linux/libcfs/libcfs.h
index 392793582956..edc7ed0dcb94 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs.h
@@ -34,102 +34,43 @@
 #ifndef __LIBCFS_LIBCFS_H__
 #define __LIBCFS_LIBCFS_H__
 
-#include <linux/gfp.h>
-#include <linux/list.h>
+#include <linux/notifier.h>
+#include <linux/workqueue.h>
+#include <linux/sysctl.h>
 
-#include <uapi/linux/lnet/libcfs_ioctl.h>
-#include <linux/libcfs/linux/libcfs.h>
 #include <linux/libcfs/libcfs_debug.h>
 #include <linux/libcfs/libcfs_private.h>
-#include <linux/libcfs/libcfs_cpu.h>
-#include <linux/libcfs/libcfs_prim.h>
-#include <linux/libcfs/libcfs_time.h>
-#include <linux/libcfs/libcfs_string.h>
-#include <linux/libcfs/libcfs_hash.h>
 #include <linux/libcfs/libcfs_fail.h>
-#include <linux/libcfs/curproc.h>
 
 #define LIBCFS_VERSION "0.7.0"
 
-#define LOWEST_BIT_SET(x)       ((x) & ~((x) - 1))
-
-/*
- * Lustre Error Checksum: calculates checksum
- * of Hex number by XORing each bit.
- */
-#define LERRCHKSUM(hexnum) (((hexnum) & 0xf) ^ ((hexnum) >> 4 & 0xf) ^ \
-			   ((hexnum) >> 8 & 0xf))
-
-/* need both kernel and user-land acceptor */
-#define LNET_ACCEPTOR_MIN_RESERVED_PORT    512
-#define LNET_ACCEPTOR_MAX_RESERVED_PORT    1023
-
-/* Block all signals except for the @sigs */
-static inline void cfs_block_sigsinv(unsigned long sigs, sigset_t *old)
-{
-	sigset_t new;
-
-	siginitsetinv(&new, sigs);
-	sigorsets(&new, &current->blocked, &new);
-	sigprocmask(SIG_BLOCK, &new, old);
-}
-
-static inline void
-cfs_restore_sigs(sigset_t *old)
-{
-	sigprocmask(SIG_SETMASK, old, NULL);
-}
-
-struct libcfs_ioctl_handler {
-	struct list_head item;
-	int (*handle_ioctl)(unsigned int cmd, struct libcfs_ioctl_hdr *hdr);
-};
-
-#define DECLARE_IOCTL_HANDLER(ident, func)			\
-	struct libcfs_ioctl_handler ident = {			\
-		.item		= LIST_HEAD_INIT(ident.item),	\
-		.handle_ioctl	= func				\
-	}
-
-int libcfs_register_ioctl(struct libcfs_ioctl_handler *hand);
-int libcfs_deregister_ioctl(struct libcfs_ioctl_handler *hand);
-
-int libcfs_ioctl_getdata(struct libcfs_ioctl_hdr **hdr_pp,
-			 const struct libcfs_ioctl_hdr __user *uparam);
-int libcfs_ioctl_data_adjust(struct libcfs_ioctl_data *data);
-int libcfs_ioctl(unsigned long cmd, void __user *arg);
-
-/* container_of depends on "likely" which is defined in libcfs_private.h */
-static inline void *__container_of(void *ptr, unsigned long shift)
+extern struct blocking_notifier_head libcfs_ioctl_list;
+static inline int notifier_from_ioctl_errno(int err)
 {
-	if (IS_ERR_OR_NULL(ptr))
-		return ptr;
-	return (char *)ptr - shift;
+	if (err == -EINVAL)
+		return NOTIFY_OK;
+	return notifier_from_errno(err) | NOTIFY_STOP_MASK;
 }
 
-#define container_of0(ptr, type, member) \
-	((type *)__container_of((void *)(ptr), offsetof(type, member)))
-
-#define _LIBCFS_H
-
-extern struct miscdevice libcfs_dev;
-/**
- * The path of debug log dump upcall script.
- */
-extern char lnet_debug_log_upcall[1024];
+int libcfs_setup(void);
 
 extern struct workqueue_struct *cfs_rehash_wq;
 
-struct lnet_debugfs_symlink_def {
-	char *name;
-	char *target;
-};
-
-void lustre_insert_debugfs(struct ctl_table *table,
-			   const struct lnet_debugfs_symlink_def *symlinks);
+void lustre_insert_debugfs(struct ctl_table *table);
 int lprocfs_call_handler(void *data, int write, loff_t *ppos,
 			 void __user *buffer, size_t *lenp,
 			 int (*handler)(void *data, int write, loff_t pos,
 					void __user *buffer, int len));
 
-#endif /* _LIBCFS_H */
+/*
+ * Memory
+ */
+#if BITS_PER_LONG == 32
+/* limit to lowmem on 32-bit systems */
+#define NUM_CACHEPAGES \
+	min(totalram_pages, 1UL << (30 - PAGE_SHIFT) * 3 / 4)
+#else
+#define NUM_CACHEPAGES totalram_pages
+#endif
+
+#endif /* __LIBCFS_LIBCFS_H__ */
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_cpu.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_cpu.h
index 61bce77fddd6..61641c41c492 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_cpu.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_cpu.h
@@ -59,9 +59,9 @@
  *   . NUMA allocators, CPU affinity threads are built over CPU partitions,
  *     instead of HW CPUs or HW nodes.
  *
- *   . By default, Lustre modules should refer to the global cfs_cpt_table,
+ *   . By default, Lustre modules should refer to the global cfs_cpt_tab,
  *     instead of accessing HW CPUs directly, so concurrency of Lustre can be
- *     configured by cpu_npartitions of the global cfs_cpt_table
+ *     configured by cpu_npartitions of the global cfs_cpt_tab
  *
  *   . If cpu_npartitions=1(all CPUs in one pool), lustre should work the
  *     same way as 2.2 or earlier versions
@@ -72,53 +72,53 @@
 #ifndef __LIBCFS_CPU_H__
 #define __LIBCFS_CPU_H__
 
+#include <linux/cpu.h>
+#include <linux/cpuset.h>
+#include <linux/topology.h>
+
 /* any CPU partition */
 #define CFS_CPT_ANY		(-1)
 
 #ifdef CONFIG_SMP
-/**
- * return cpumask of CPU partition \a cpt
- */
-cpumask_var_t *cfs_cpt_cpumask(struct cfs_cpt_table *cptab, int cpt);
-/**
- * print string information of cpt-table
- */
-int cfs_cpt_table_print(struct cfs_cpt_table *cptab, char *buf, int len);
-#else /* !CONFIG_SMP */
-struct cfs_cpt_table {
-	/* # of CPU partitions */
-	int			ctb_nparts;
-	/* cpu mask */
-	cpumask_t		ctb_mask;
-	/* node mask */
-	nodemask_t		ctb_nodemask;
-	/* version */
-	u64			ctb_version;
+/** virtual processing unit */
+struct cfs_cpu_partition {
+	/* CPUs mask for this partition */
+	cpumask_var_t			cpt_cpumask;
+	/* nodes mask for this partition */
+	nodemask_t			*cpt_nodemask;
+	/* spread rotor for NUMA allocator */
+	unsigned int			cpt_spread_rotor;
 };
 
-static inline cpumask_var_t *
-cfs_cpt_cpumask(struct cfs_cpt_table *cptab, int cpt)
-{
-	return NULL;
-}
 
-static inline int
-cfs_cpt_table_print(struct cfs_cpt_table *cptab, char *buf, int len)
-{
-	return 0;
-}
-#endif /* CONFIG_SMP */
+/** descriptor for CPU partitions */
+struct cfs_cpt_table {
+	/* version, reserved for hotplug */
+	unsigned int			ctb_version;
+	/* spread rotor for NUMA allocator */
+	unsigned int			ctb_spread_rotor;
+	/* # of CPU partitions */
+	unsigned int			ctb_nparts;
+	/* partitions tables */
+	struct cfs_cpu_partition	*ctb_parts;
+	/* shadow HW CPU to CPU partition ID */
+	int				*ctb_cpu2cpt;
+	/* all cpus in this partition table */
+	cpumask_var_t			ctb_cpumask;
+	/* all nodes in this partition table */
+	nodemask_t			*ctb_nodemask;
+};
 
-extern struct cfs_cpt_table	*cfs_cpt_table;
+extern struct cfs_cpt_table	*cfs_cpt_tab;
 
 /**
- * destroy a CPU partition table
+ * return cpumask of CPU partition \a cpt
  */
-void cfs_cpt_table_free(struct cfs_cpt_table *cptab);
+cpumask_var_t *cfs_cpt_cpumask(struct cfs_cpt_table *cptab, int cpt);
 /**
- * create a cfs_cpt_table with \a ncpt number of partitions
+ * print string information of cpt-table
  */
-struct cfs_cpt_table *cfs_cpt_table_alloc(unsigned int ncpt);
+int cfs_cpt_table_print(struct cfs_cpt_table *cptab, char *buf, int len);
 /**
  * return total number of CPU partitions in \a cptab
  */
@@ -204,6 +204,148 @@ int cfs_cpt_spread_node(struct cfs_cpt_table *cptab, int cpt);
  */
 int cfs_cpu_ht_nsiblings(int cpu);
 
+int  cfs_cpu_init(void);
+void cfs_cpu_fini(void);
+
+#else /* !CONFIG_SMP */
+struct cfs_cpt_table;
+#define cfs_cpt_tab ((struct cfs_cpt_table *)NULL)
+
+static inline cpumask_var_t *
+cfs_cpt_cpumask(struct cfs_cpt_table *cptab, int cpt)
+{
+	return NULL;
+}
+
+static inline int
+cfs_cpt_table_print(struct cfs_cpt_table *cptab, char *buf, int len)
+{
+	return 0;
+}
+static inline int
+cfs_cpt_number(struct cfs_cpt_table *cptab)
+{
+	return 1;
+}
+
+static inline int
+cfs_cpt_weight(struct cfs_cpt_table *cptab, int cpt)
+{
+	return 1;
+}
+
+static inline int
+cfs_cpt_online(struct cfs_cpt_table *cptab, int cpt)
+{
+	return 1;
+}
+
+static inline nodemask_t *
+cfs_cpt_nodemask(struct cfs_cpt_table *cptab, int cpt)
+{
+	return NULL;
+}
+
+static inline int
+cfs_cpt_set_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu)
+{
+	return 1;
+}
+
+static inline void
+cfs_cpt_unset_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu)
+{
+}
+
+static inline int
+cfs_cpt_set_cpumask(struct cfs_cpt_table *cptab, int cpt, cpumask_t *mask)
+{
+	return 1;
+}
+
+static inline void
+cfs_cpt_unset_cpumask(struct cfs_cpt_table *cptab, int cpt, cpumask_t *mask)
+{
+}
+
+static inline int
+cfs_cpt_set_node(struct cfs_cpt_table *cptab, int cpt, int node)
+{
+	return 1;
+}
+
+static inline void
+cfs_cpt_unset_node(struct cfs_cpt_table *cptab, int cpt, int node)
+{
+}
+
+static inline int
+cfs_cpt_set_nodemask(struct cfs_cpt_table *cptab, int cpt, nodemask_t *mask)
+{
+	return 1;
+}
+
+static inline void
+cfs_cpt_unset_nodemask(struct cfs_cpt_table *cptab, int cpt, nodemask_t *mask)
+{
+}
+
+static inline void
+cfs_cpt_clear(struct cfs_cpt_table *cptab, int cpt)
+{
+}
+
+static inline int
+cfs_cpt_spread_node(struct cfs_cpt_table *cptab, int cpt)
+{
+	return 0;
+}
+
+static inline int
+cfs_cpu_ht_nsiblings(int cpu)
+{
+	return 1;
+}
+
+static inline int
+cfs_cpt_current(struct cfs_cpt_table *cptab, int remap)
+{
+	return 0;
+}
+
+static inline int
+cfs_cpt_of_cpu(struct cfs_cpt_table *cptab, int cpu)
+{
+	return 0;
+}
+
+static inline int
+cfs_cpt_bind(struct cfs_cpt_table *cptab, int cpt)
+{
+	return 0;
+}
+
+static inline int
+cfs_cpu_init(void)
+{
+	return 0;
+}
+
+static inline void cfs_cpu_fini(void)
+{
+}
+
+#endif /* CONFIG_SMP */
+
+/**
+ * destroy a CPU partition table
+ */
+void cfs_cpt_table_free(struct cfs_cpt_table *cptab);
+/**
+ * create a cfs_cpt_table with \a ncpt number of partitions
+ */
+struct cfs_cpt_table *cfs_cpt_table_alloc(unsigned int ncpt);
+
 /*
  * allocate per-cpu-partition data, returned value is an array of pointers,
  * variable can be indexed by CPU ID.
@@ -289,7 +431,4 @@ void cfs_percpt_unlock(struct cfs_percpt_lock *pcl, int index);
 #define cfs_cpt_for_each(i, cptab)	\
 	for (i = 0; i < cfs_cpt_number(cptab); i++)
 
-int  cfs_cpu_init(void);
-void cfs_cpu_fini(void);
-
 #endif /* __LIBCFS_CPU_H__ */
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_crypto.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_crypto.h
index 3a72117140ed..176fae7319e3 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_crypto.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_crypto.h
@@ -29,6 +29,9 @@
 #ifndef _LIBCFS_CRYPTO_H
 #define _LIBCFS_CRYPTO_H
 
+#include <linux/string.h>
+struct page;
+
 struct cfs_crypto_hash_type {
 	char		*cht_name;      /*< hash algorithm name, equal to
 					 * format name for crypto api
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h
index 9290a19429e7..17534a76362a 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h
@@ -39,6 +39,7 @@
 #ifndef __LIBCFS_DEBUG_H__
 #define __LIBCFS_DEBUG_H__
 
+#include <linux/limits.h>
 #include <uapi/linux/lnet/libcfs_debug.h>
 
 /*
@@ -62,6 +63,38 @@ int libcfs_debug_str2mask(int *mask, const char *str, int is_subsys);
 extern unsigned int libcfs_catastrophe;
 extern unsigned int libcfs_panic_on_lbug;
 
+/* Enable debug-checks on stack size - except on x86_64 */
+#if !defined(__x86_64__)
+# ifdef __ia64__
+#  define CDEBUG_STACK() (THREAD_SIZE -				 \
+			  ((unsigned long)__builtin_dwarf_cfa() &       \
+			   (THREAD_SIZE - 1)))
+# else
+#  define CDEBUG_STACK() (THREAD_SIZE -				 \
+			  ((unsigned long)__builtin_frame_address(0) &  \
+			   (THREAD_SIZE - 1)))
+# endif /* __ia64__ */
+
+#define __CHECK_STACK(msgdata, mask, cdls)			      \
+do {								    \
+	if (unlikely(CDEBUG_STACK() > libcfs_stack)) {		  \
+		LIBCFS_DEBUG_MSG_DATA_INIT(msgdata, D_WARNING, NULL);   \
+		libcfs_stack = CDEBUG_STACK();			  \
+		libcfs_debug_msg(msgdata,			       \
+				 "maximum lustre stack %lu\n",	  \
+				 CDEBUG_STACK());		       \
+		(msgdata)->msg_mask = mask;			     \
+		(msgdata)->msg_cdls = cdls;			     \
+		dump_stack();					   \
+	      /*panic("LBUG");*/					\
+	}							       \
+} while (0)
+#define CFS_CHECK_STACK(msgdata, mask, cdls)  __CHECK_STACK(msgdata, mask, cdls)
+#else /* __x86_64__ */
+#define CFS_CHECK_STACK(msgdata, mask, cdls) do {} while (0)
+#define CDEBUG_STACK() (0L)
+#endif /* __x86_64__ */
+
 #ifndef DEBUG_SUBSYSTEM
 # define DEBUG_SUBSYSTEM S_UNDEFINED
 #endif
@@ -133,6 +166,13 @@ do {									\
 	__CDEBUG(&cdls, mask, format, ## __VA_ARGS__);			\
 } while (0)
 
+/*
+ * Lustre Error Checksum: calculates checksum
+ * of Hex number by XORing the nybbles.
+ */
+#define LERRCHKSUM(hexnum) (((hexnum) & 0xf) ^ ((hexnum) >> 4 & 0xf) ^ \
+			   ((hexnum) >> 8 & 0xf))
+
 #define CWARN(format, ...)	CDEBUG_LIMIT(D_WARNING, format, ## __VA_ARGS__)
 #define CERROR(format, ...)	CDEBUG_LIMIT(D_ERROR, format, ## __VA_ARGS__)
 #define CNETERR(format, a...)	CDEBUG_LIMIT(D_NETERROR, format, ## a)
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h
index d6fc3164e7e7..8074e390b4d1 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h
@@ -33,6 +33,9 @@
 #ifndef _LIBCFS_FAIL_H
 #define _LIBCFS_FAIL_H
 
+#include <linux/sched.h>
+#include <linux/wait.h>
+
 extern unsigned long cfs_fail_loc;
 extern unsigned int cfs_fail_val;
 extern int cfs_fail_err;
@@ -140,7 +143,7 @@ static inline int cfs_fail_timeout_set(u32 id, u32 value, int ms, int set)
 
 /* If id hit cfs_fail_loc, sleep for seconds or milliseconds */
 #define CFS_FAIL_TIMEOUT(id, secs) \
-	cfs_fail_timeout_set(id, 0, secs * 1000, CFS_FAIL_LOC_NOSET)
+	cfs_fail_timeout_set(id, 0, (secs) * 1000, CFS_FAIL_LOC_NOSET)
 
 #define CFS_FAIL_TIMEOUT_MS(id, ms) \
 	cfs_fail_timeout_set(id, 0, ms, CFS_FAIL_LOC_NOSET)
@@ -150,10 +153,10 @@ static inline int cfs_fail_timeout_set(u32 id, u32 value, int ms, int set)
  * sleep seconds or milliseconds
  */
 #define CFS_FAIL_TIMEOUT_ORSET(id, value, secs) \
-	cfs_fail_timeout_set(id, value, secs * 1000, CFS_FAIL_LOC_ORSET)
+	cfs_fail_timeout_set(id, value, (secs) * 1000, CFS_FAIL_LOC_ORSET)
 
 #define CFS_FAIL_TIMEOUT_RESET(id, value, secs) \
-	cfs_fail_timeout_set(id, value, secs * 1000, CFS_FAIL_LOC_RESET)
+	cfs_fail_timeout_set(id, value, (secs) * 1000, CFS_FAIL_LOC_RESET)
 
 #define CFS_FAIL_TIMEOUT_MS_ORSET(id, value, ms) \
 	cfs_fail_timeout_set(id, value, ms, CFS_FAIL_LOC_ORSET)
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_hash.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_hash.h
index 0506f1d45757..be315958a4b3 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_hash.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_hash.h
@@ -40,6 +40,9 @@
 #define __LIBCFS_HASH_H__
 
 #include <linux/hash.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/libcfs/libcfs.h>
 
 /*
  * Knuth recommends primes in approximately golden ratio to the maximum
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_prim.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_prim.h
deleted file mode 100644
index d4c5965c43b1..000000000000
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_prim.h
+++ /dev/null
@@ -1,82 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * libcfs/include/libcfs/libcfs_prim.h
- *
- * General primitives.
- *
- */
-
-#ifndef __LIBCFS_PRIM_H__
-#define __LIBCFS_PRIM_H__
-
-/*
- * Memory
- */
-#if BITS_PER_LONG == 32
-/* limit to lowmem on 32-bit systems */
-#define NUM_CACHEPAGES \
-	min(totalram_pages, 1UL << (30 - PAGE_SHIFT) * 3 / 4)
-#else
-#define NUM_CACHEPAGES totalram_pages
-#endif
-
-static inline unsigned int memory_pressure_get(void)
-{
-	return current->flags & PF_MEMALLOC;
-}
-
-static inline void memory_pressure_set(void)
-{
-	current->flags |= PF_MEMALLOC;
-}
-
-static inline void memory_pressure_clr(void)
-{
-	current->flags &= ~PF_MEMALLOC;
-}
-
-static inline int cfs_memory_pressure_get_and_set(void)
-{
-	int old = memory_pressure_get();
-
-	if (!old)
-		memory_pressure_set();
-	return old;
-}
-
-static inline void cfs_memory_pressure_restore(int old)
-{
-	if (old)
-		memory_pressure_set();
-	else
-		memory_pressure_clr();
-}
-#endif
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_string.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_string.h
index 66463477074a..cd7c3ccb2dc0 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_string.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_string.h
@@ -40,6 +40,8 @@
 #ifndef __LIBCFS_STRING_H__
 #define __LIBCFS_STRING_H__
 
+#include <linux/mm.h>
+
 /* libcfs_string.c */
 /* Convert a text string to a bitmask */
 int cfs_str2mask(const char *str, const char *(*bit2str)(int bit),
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_time.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_time.h
deleted file mode 100644
index c4f25be78268..000000000000
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_time.h
+++ /dev/null
@@ -1,79 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * libcfs/include/libcfs/libcfs_time.h
- *
- * Time functions.
- *
- */
-
-#ifndef __LIBCFS_TIME_H__
-#define __LIBCFS_TIME_H__
-/*
- * generic time manipulation functions.
- */
-
-static inline unsigned long cfs_time_add(unsigned long t, long d)
-{
-	return (unsigned long)(t + d);
-}
-
-static inline unsigned long cfs_time_sub(unsigned long t1, unsigned long t2)
-{
-	return (unsigned long)(t1 - t2);
-}
-
-static inline int cfs_time_after(unsigned long t1, unsigned long t2)
-{
-	return time_before(t2, t1);
-}
-
-static inline int cfs_time_aftereq(unsigned long t1, unsigned long t2)
-{
-	return time_before_eq(t2, t1);
-}
-
-static inline unsigned long cfs_time_shift(int seconds)
-{
-	return cfs_time_add(cfs_time_current(), seconds * HZ);
-}
-
-/*
- * return valid time-out based on user supplied one. Currently we only check
- * that time-out is not shorted than allowed.
- */
-static inline long cfs_timeout_cap(long timeout)
-{
-	if (timeout < CFS_TICK)
-		timeout = CFS_TICK;
-	return timeout;
-}
-
-#endif
diff --git a/drivers/staging/lustre/include/linux/libcfs/linux/libcfs.h b/drivers/staging/lustre/include/linux/libcfs/linux/libcfs.h
deleted file mode 100644
index 30e333af8d0d..000000000000
--- a/drivers/staging/lustre/include/linux/libcfs/linux/libcfs.h
+++ /dev/null
@@ -1,133 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- *
- * Copyright (c) 2012, Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- */
-
-#ifndef __LIBCFS_LINUX_LIBCFS_H__
-#define __LIBCFS_LINUX_LIBCFS_H__
-
-#ifndef __LIBCFS_LIBCFS_H__
-#error Do not #include this file directly. #include <linux/libcfs/libcfs.h> instead
-#endif
-
-#include <linux/bitops.h>
-#include <linux/compiler.h>
-#include <linux/ctype.h>
-#include <linux/errno.h>
-#include <linux/file.h>
-#include <linux/fs.h>
-#include <linux/highmem.h>
-#include <linux/interrupt.h>
-#include <linux/kallsyms.h>
-#include <linux/kernel.h>
-#include <linux/kmod.h>
-#include <linux/kthread.h>
-#include <linux/miscdevice.h>
-#include <linux/mm.h>
-#include <linux/mm_inline.h>
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/mutex.h>
-#include <linux/notifier.h>
-#include <linux/pagemap.h>
-#include <linux/random.h>
-#include <linux/rbtree.h>
-#include <linux/rwsem.h>
-#include <linux/scatterlist.h>
-#include <linux/sched.h>
-#include <linux/signal.h>
-#include <linux/slab.h>
-#include <linux/smp.h>
-#include <linux/stat.h>
-#include <linux/string.h>
-#include <linux/time.h>
-#include <linux/timer.h>
-#include <linux/types.h>
-#include <linux/unistd.h>
-#include <linux/vmalloc.h>
-#include <net/sock.h>
-#include <linux/atomic.h>
-#include <asm/div64.h>
-#include <linux/timex.h>
-#include <linux/uaccess.h>
-#include <stdarg.h>
-#include "linux-cpu.h"
-#include "linux-time.h"
-
-#define LUSTRE_TRACE_SIZE (THREAD_SIZE >> 5)
-
-#if !defined(__x86_64__)
-# ifdef __ia64__
-#  define CDEBUG_STACK() (THREAD_SIZE -				 \
-			  ((unsigned long)__builtin_dwarf_cfa() &       \
-			   (THREAD_SIZE - 1)))
-# else
-#  define CDEBUG_STACK() (THREAD_SIZE -				 \
-			  ((unsigned long)__builtin_frame_address(0) &  \
-			   (THREAD_SIZE - 1)))
-# endif /* __ia64__ */
-
-#define __CHECK_STACK(msgdata, mask, cdls)			      \
-do {								    \
-	if (unlikely(CDEBUG_STACK() > libcfs_stack)) {		  \
-		LIBCFS_DEBUG_MSG_DATA_INIT(msgdata, D_WARNING, NULL);   \
-		libcfs_stack = CDEBUG_STACK();			  \
-		libcfs_debug_msg(msgdata,			       \
-				 "maximum lustre stack %lu\n",	  \
-				 CDEBUG_STACK());		       \
-		(msgdata)->msg_mask = mask;			     \
-		(msgdata)->msg_cdls = cdls;			     \
-		dump_stack();					   \
-	      /*panic("LBUG");*/					\
-	}							       \
-} while (0)
-#define CFS_CHECK_STACK(msgdata, mask, cdls)  __CHECK_STACK(msgdata, mask, cdls)
-#else /* __x86_64__ */
-#define CFS_CHECK_STACK(msgdata, mask, cdls) do {} while (0)
-#define CDEBUG_STACK() (0L)
-#endif /* __x86_64__ */
-
-#define __current_nesting_level() (0)
-
-/**
- * Platform specific declarations for cfs_curproc API (libcfs/curproc.h)
- *
- * Implementation is in linux-curproc.c
- */
-#define CFS_CURPROC_COMM_MAX (sizeof((struct task_struct *)0)->comm)
-
-#include <linux/capability.h>
-
-#ifndef WITH_WATCHDOG
-#define WITH_WATCHDOG
-#endif
-
-#endif /* _LINUX_LIBCFS_H */
diff --git a/drivers/staging/lustre/include/linux/libcfs/linux/linux-cpu.h b/drivers/staging/lustre/include/linux/libcfs/linux/linux-cpu.h
deleted file mode 100644
index 6035376f2830..000000000000
--- a/drivers/staging/lustre/include/linux/libcfs/linux/linux-cpu.h
+++ /dev/null
@@ -1,78 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * libcfs/include/libcfs/linux/linux-cpu.h
- *
- * Basic library routines.
- *
- * Author: liang@whamcloud.com
- */
-
-#ifndef __LIBCFS_LINUX_CPU_H__
-#define __LIBCFS_LINUX_CPU_H__
-
-#ifndef __LIBCFS_LIBCFS_H__
-#error Do not #include this file directly. #include <linux/libcfs/libcfs.h> instead
-#endif
-
-#include <linux/cpu.h>
-#include <linux/cpuset.h>
-#include <linux/topology.h>
-
-#ifdef CONFIG_SMP
-
-#define HAVE_LIBCFS_CPT
-
-/** virtual processing unit */
-struct cfs_cpu_partition {
-	/* CPUs mask for this partition */
-	cpumask_var_t			cpt_cpumask;
-	/* nodes mask for this partition */
-	nodemask_t			*cpt_nodemask;
-	/* spread rotor for NUMA allocator */
-	unsigned int			cpt_spread_rotor;
-};
-
-/** descriptor for CPU partitions */
-struct cfs_cpt_table {
-	/* version, reserved for hotplug */
-	unsigned int			ctb_version;
-	/* spread rotor for NUMA allocator */
-	unsigned int			ctb_spread_rotor;
-	/* # of CPU partitions */
-	unsigned int			ctb_nparts;
-	/* partitions tables */
-	struct cfs_cpu_partition	*ctb_parts;
-	/* shadow HW CPU to CPU partition ID */
-	int				*ctb_cpu2cpt;
-	/* all cpus in this partition table */
-	cpumask_var_t			ctb_cpumask;
-	/* all nodes in this partition table */
-	nodemask_t			*ctb_nodemask;
-};
-
-#endif /* CONFIG_SMP */
-#endif /* __LIBCFS_LINUX_CPU_H__ */
diff --git a/drivers/staging/lustre/include/linux/libcfs/linux/linux-time.h b/drivers/staging/lustre/include/linux/libcfs/linux/linux-time.h
deleted file mode 100644
index 805cb326af86..000000000000
--- a/drivers/staging/lustre/include/linux/libcfs/linux/linux-time.h
+++ /dev/null
@@ -1,103 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * libcfs/include/libcfs/linux/linux-time.h
- *
- * Implementation of portable time API for Linux (kernel and user-level).
- *
- * Author: Nikita Danilov <nikita@clusterfs.com>
- */
-
-#ifndef __LIBCFS_LINUX_LINUX_TIME_H__
-#define __LIBCFS_LINUX_LINUX_TIME_H__
-
-#ifndef __LIBCFS_LIBCFS_H__
-#error Do not #include this file directly. #include <linux/libcfs/libcfs.h> instead
-#endif
-
-#define ONE_BILLION ((u_int64_t)1000000000)
-#define ONE_MILLION 1000000
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/time.h>
-#include <asm/div64.h>
-
-/*
- * post 2.5 kernels.
- */
-
-#include <linux/jiffies.h>
-
-/*
- * Generic kernel stuff
- */
-
-static inline unsigned long cfs_time_current(void)
-{
-	return jiffies;
-}
-
-static inline long cfs_duration_sec(long d)
-{
-	return d / msecs_to_jiffies(MSEC_PER_SEC);
-}
-
-#define cfs_time_current_64 get_jiffies_64
-
-static inline u64 cfs_time_add_64(u64 t, u64 d)
-{
-	return t + d;
-}
-
-static inline u64 cfs_time_shift_64(int seconds)
-{
-	return cfs_time_add_64(cfs_time_current_64(),
-			       seconds * HZ);
-}
-
-static inline int cfs_time_before_64(u64 t1, u64 t2)
-{
-	return (__s64)t2 - (__s64)t1 > 0;
-}
-
-static inline int cfs_time_beforeq_64(u64 t1, u64 t2)
-{
-	return (__s64)t2 - (__s64)t1 >= 0;
-}
-
-/*
- * One jiffy
- */
-#define CFS_TICK		(1)
-
-#define CFS_DURATION_T	  "%ld"
-
-#endif /* __LIBCFS_LINUX_LINUX_TIME_H__ */
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
index df4c72507a15..973c17a1c4a1 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
@@ -37,6 +37,10 @@
 #define __LNET_LIB_LNET_H__
 
 #include <linux/libcfs/libcfs.h>
+#include <linux/libcfs/libcfs_cpu.h>
+#include <linux/libcfs/libcfs_string.h>
+#include <net/sock.h>
+
 #include <linux/lnet/api.h>
 #include <linux/lnet/lib-types.h>
 #include <uapi/linux/lnet/lnet-dlc.h>
@@ -68,6 +72,10 @@ extern struct lnet the_lnet;	/* THE network */
 /** exclusive lock */
 #define LNET_LOCK_EX		CFS_PERCPT_LOCK_EX
 
+/* need both kernel and user-land acceptor */
+#define LNET_ACCEPTOR_MIN_RESERVED_PORT    512
+#define LNET_ACCEPTOR_MAX_RESERVED_PORT    1023
+
 static inline int lnet_is_route_alive(struct lnet_route *route)
 {
 	/* gateway is down */
diff --git a/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
index e45d828bfd1b..c1619f411d81 100644
--- a/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
@@ -53,7 +53,8 @@ struct lnet_ioctl_config_o2iblnd_tunables {
 	__u32 lnd_fmr_pool_size;
 	__u32 lnd_fmr_flush_trigger;
 	__u32 lnd_fmr_cache;
-	__u32 pad;
+	__u16 lnd_conns_per_peer;
+	__u16 pad;
 };
 
 struct lnet_ioctl_config_lnd_tunables {
diff --git a/drivers/staging/lustre/include/uapi/linux/lnet/lnetctl.h b/drivers/staging/lustre/include/uapi/linux/lnet/lnetctl.h
index d9da625d70de..cccb32dd28f2 100644
--- a/drivers/staging/lustre/include/uapi/linux/lnet/lnetctl.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/lnetctl.h
@@ -119,16 +119,5 @@ struct lnet_fault_stat {
 
 #define LNET_DEV_ID 0
 #define LNET_DEV_PATH "/dev/lnet"
-#define LNET_DEV_MAJOR 10
-#define LNET_DEV_MINOR 240
-#define OBD_DEV_ID 1
-#define OBD_DEV_NAME "obd"
-#define OBD_DEV_PATH "/dev/" OBD_DEV_NAME
-#define OBD_DEV_MAJOR 10
-#define OBD_DEV_MINOR 241
-#define SMFS_DEV_ID  2
-#define SMFS_DEV_PATH "/dev/snapdev"
-#define SMFS_DEV_MAJOR 10
-#define SMFS_DEV_MINOR 242
 
 #endif
diff --git a/drivers/staging/lustre/include/uapi/linux/lustre/lustre_fiemap.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_fiemap.h
index f5214dc33c60..d375a476f5ea 100644
--- a/drivers/staging/lustre/include/uapi/linux/lustre/lustre_fiemap.h
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_fiemap.h
@@ -51,7 +51,7 @@ static inline size_t fiemap_count_to_size(size_t extent_count)
 				       sizeof(struct fiemap_extent);
 }
 
-static inline unsigned fiemap_size_to_count(size_t array_size)
+static inline unsigned int fiemap_size_to_count(size_t array_size)
 {
 	return (array_size - sizeof(struct fiemap)) /
 		sizeof(struct fiemap_extent);
diff --git a/drivers/staging/lustre/include/uapi/linux/lustre/lustre_idl.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_idl.h
index aac98dbcf6e3..8a7301a76122 100644
--- a/drivers/staging/lustre/include/uapi/linux/lustre/lustre_idl.h
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_idl.h
@@ -1526,9 +1526,11 @@ struct mdt_rec_setattr {
 #define MDS_OPEN_HAS_OBJS    020000000000 /* Just set the EA the obj exist */
 #define MDS_OPEN_NORESTORE  0100000000000ULL /* Do not restore file at open */
 #define MDS_OPEN_NEWSTRIPE  0200000000000ULL /* New stripe needed (restripe or
-					      * hsm restore) */
+					      * hsm restore)
+					      */
 #define MDS_OPEN_VOLATILE   0400000000000ULL /* File is volatile = created
-						unlinked */
+					      * unlinked
+					      */
 #define MDS_OPEN_LEASE	   01000000000000ULL /* Open the file and grant lease
 					      * delegation, succeed if it's not
 					      * being opened with conflict mode.
diff --git a/drivers/staging/lustre/include/uapi/linux/lustre/lustre_ioctl.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_ioctl.h
index 9590864e0b50..6e4e109fb874 100644
--- a/drivers/staging/lustre/include/uapi/linux/lustre/lustre_ioctl.h
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_ioctl.h
@@ -51,8 +51,6 @@ enum md_echo_cmd {
 #define OBD_DEV_ID 1
 #define OBD_DEV_NAME "obd"
 #define OBD_DEV_PATH "/dev/" OBD_DEV_NAME
-#define OBD_DEV_MAJOR 10
-#define OBD_DEV_MINOR 241
 
 #define OBD_IOCTL_VERSION	0x00010004
 #define OBD_DEV_BY_DEVNAME	0xffffd0de
diff --git a/drivers/staging/lustre/include/uapi/linux/lustre/lustre_user.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_user.h
index 5e332e3af68a..69387f36d1f1 100644
--- a/drivers/staging/lustre/include/uapi/linux/lustre/lustre_user.h
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_user.h
@@ -405,7 +405,8 @@ static inline __u32 lov_user_md_size(__u16 stripes, __u32 lmm_magic)
 
 /* Compile with -D_LARGEFILE64_SOURCE or -D_GNU_SOURCE (or #define) to
  * use this.  It is unsafe to #define those values in this header as it
- * is possible the application has already #included <sys/stat.h>. */
+ * is possible the application has already #included <sys/stat.h>.
+ */
 #ifdef HAVE_LOV_USER_MDS_DATA
 #define lov_user_mds_data lov_user_mds_data_v1
 struct lov_user_mds_data_v1 {
@@ -468,7 +469,8 @@ struct ll_fid {
 	__u64 id;	 /* holds object id */
 	__u32 generation; /* holds object generation */
 	__u32 f_type;     /* holds object type or stripe idx when passing it to
-			   * OST for saving into EA. */
+			   * OST for saving into EA.
+			   */
 };
 
 #define UUID_MAX	40
diff --git a/drivers/staging/lustre/lnet/Kconfig b/drivers/staging/lustre/lnet/Kconfig
index f3b1ad4bd3dc..ad049e6f24e4 100644
--- a/drivers/staging/lustre/lnet/Kconfig
+++ b/drivers/staging/lustre/lnet/Kconfig
@@ -34,7 +34,7 @@ config LNET_SELFTEST
 
 config LNET_XPRT_IB
 	tristate "LNET infiniband support"
-	depends on LNET && PCI && INFINIBAND_ADDR_TRANS
+	depends on LNET && PCI && INFINIBAND && INFINIBAND_ADDR_TRANS
 	default LNET && INFINIBAND
 	help
 	  This option allows the LNET users to use infiniband as an
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
index 7ae2955c4db6..f0b4eb42bc1d 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1043,7 +1043,7 @@ static void kiblnd_query(struct lnet_ni *ni, lnet_nid_t nid,
 			 unsigned long *when)
 {
 	unsigned long last_alive = 0;
-	unsigned long now = cfs_time_current();
+	unsigned long now = jiffies;
 	rwlock_t *glock = &kiblnd_data.kib_global_lock;
 	struct kib_peer *peer;
 	unsigned long flags;
@@ -1068,7 +1068,7 @@ static void kiblnd_query(struct lnet_ni *ni, lnet_nid_t nid,
 
 	CDEBUG(D_NET, "Peer %s %p, alive %ld secs ago\n",
 	       libcfs_nid2str(nid), peer,
-	       last_alive ? cfs_duration_sec(now - last_alive) : -1);
+	       last_alive ? (now - last_alive) / HZ : -1);
 }
 
 static void kiblnd_free_pages(struct kib_pages *p)
@@ -1446,7 +1446,7 @@ static int kiblnd_create_fmr_pool(struct kib_fmr_poolset *fps,
 	if (rc)
 		goto out_fpo;
 
-	fpo->fpo_deadline = cfs_time_shift(IBLND_POOL_DEADLINE);
+	fpo->fpo_deadline = jiffies + IBLND_POOL_DEADLINE * HZ;
 	fpo->fpo_owner = fps;
 	*pp_fpo = fpo;
 
@@ -1522,7 +1522,7 @@ static int kiblnd_fmr_pool_is_idle(struct kib_fmr_pool *fpo, unsigned long now)
 		return 0;
 	if (fpo->fpo_failed)
 		return 1;
-	return cfs_time_aftereq(now, fpo->fpo_deadline);
+	return time_after_eq(now, fpo->fpo_deadline);
 }
 
 static int
@@ -1552,7 +1552,7 @@ void kiblnd_fmr_pool_unmap(struct kib_fmr *fmr, int status)
 	LIST_HEAD(zombies);
 	struct kib_fmr_pool *fpo = fmr->fmr_pool;
 	struct kib_fmr_poolset *fps;
-	unsigned long now = cfs_time_current();
+	unsigned long now = jiffies;
 	struct kib_fmr_pool *tmp;
 	int rc;
 
@@ -1619,7 +1619,7 @@ int kiblnd_fmr_pool_map(struct kib_fmr_poolset *fps, struct kib_tx *tx,
 	spin_lock(&fps->fps_lock);
 	version = fps->fps_version;
 	list_for_each_entry(fpo, &fps->fps_pool_list, fpo_list) {
-		fpo->fpo_deadline = cfs_time_shift(IBLND_POOL_DEADLINE);
+		fpo->fpo_deadline = jiffies + IBLND_POOL_DEADLINE * HZ;
 		fpo->fpo_map_count++;
 
 		if (fpo->fpo_is_fmr) {
@@ -1702,7 +1702,7 @@ int kiblnd_fmr_pool_map(struct kib_fmr_poolset *fps, struct kib_tx *tx,
 				return 0;
 			}
 			spin_unlock(&fps->fps_lock);
-			rc = -EBUSY;
+			rc = -EAGAIN;
 		}
 
 		spin_lock(&fps->fps_lock);
@@ -1726,7 +1726,7 @@ int kiblnd_fmr_pool_map(struct kib_fmr_poolset *fps, struct kib_tx *tx,
 		goto again;
 	}
 
-	if (time_before(cfs_time_current(), fps->fps_next_retry)) {
+	if (time_before(jiffies, fps->fps_next_retry)) {
 		/* someone failed recently */
 		spin_unlock(&fps->fps_lock);
 		return -EAGAIN;
@@ -1743,7 +1743,7 @@ int kiblnd_fmr_pool_map(struct kib_fmr_poolset *fps, struct kib_tx *tx,
 		fps->fps_version++;
 		list_add_tail(&fpo->fpo_list, &fps->fps_pool_list);
 	} else {
-		fps->fps_next_retry = cfs_time_shift(IBLND_POOL_RETRY);
+		fps->fps_next_retry = jiffies + IBLND_POOL_RETRY * HZ;
 	}
 	spin_unlock(&fps->fps_lock);
 
@@ -1764,7 +1764,7 @@ static void kiblnd_init_pool(struct kib_poolset *ps, struct kib_pool *pool, int
 
 	memset(pool, 0, sizeof(*pool));
 	INIT_LIST_HEAD(&pool->po_free_list);
-	pool->po_deadline = cfs_time_shift(IBLND_POOL_DEADLINE);
+	pool->po_deadline = jiffies + IBLND_POOL_DEADLINE * HZ;
 	pool->po_owner    = ps;
 	pool->po_size     = size;
 }
@@ -1850,7 +1850,7 @@ static int kiblnd_pool_is_idle(struct kib_pool *pool, unsigned long now)
 		return 0;
 	if (pool->po_failed)
 		return 1;
-	return cfs_time_aftereq(now, pool->po_deadline);
+	return time_after_eq(now, pool->po_deadline);
 }
 
 void kiblnd_pool_free_node(struct kib_pool *pool, struct list_head *node)
@@ -1858,7 +1858,7 @@ void kiblnd_pool_free_node(struct kib_pool *pool, struct list_head *node)
 	LIST_HEAD(zombies);
 	struct kib_poolset *ps = pool->po_owner;
 	struct kib_pool *tmp;
-	unsigned long now = cfs_time_current();
+	unsigned long now = jiffies;
 
 	spin_lock(&ps->ps_lock);
 
@@ -1899,7 +1899,7 @@ struct list_head *kiblnd_pool_alloc_node(struct kib_poolset *ps)
 			continue;
 
 		pool->po_allocated++;
-		pool->po_deadline = cfs_time_shift(IBLND_POOL_DEADLINE);
+		pool->po_deadline = jiffies + IBLND_POOL_DEADLINE * HZ;
 		node = pool->po_free_list.next;
 		list_del(node);
 
@@ -1927,7 +1927,7 @@ struct list_head *kiblnd_pool_alloc_node(struct kib_poolset *ps)
 		goto again;
 	}
 
-	if (time_before(cfs_time_current(), ps->ps_next_retry)) {
+	if (time_before(jiffies, ps->ps_next_retry)) {
 		/* someone failed recently */
 		spin_unlock(&ps->ps_lock);
 		return NULL;
@@ -1937,17 +1937,17 @@ struct list_head *kiblnd_pool_alloc_node(struct kib_poolset *ps)
 	spin_unlock(&ps->ps_lock);
 
 	CDEBUG(D_NET, "%s pool exhausted, allocate new pool\n", ps->ps_name);
-	time_before = cfs_time_current();
+	time_before = jiffies;
 	rc = ps->ps_pool_create(ps, ps->ps_pool_size, &pool);
 	CDEBUG(D_NET, "ps_pool_create took %lu HZ to complete",
-	       cfs_time_current() - time_before);
+	       jiffies - time_before);
 
 	spin_lock(&ps->ps_lock);
 	ps->ps_increasing = 0;
 	if (!rc) {
 		list_add_tail(&pool->po_list, &ps->ps_pool_list);
 	} else {
-		ps->ps_next_retry = cfs_time_shift(IBLND_POOL_RETRY);
+		ps->ps_next_retry = jiffies + IBLND_POOL_RETRY * HZ;
 		CERROR("Can't allocate new %s pool because out of memory\n",
 		       ps->ps_name);
 	}
@@ -2928,6 +2928,8 @@ static void __exit ko2iblnd_exit(void)
 
 static int __init ko2iblnd_init(void)
 {
+	int rc;
+
 	BUILD_BUG_ON(sizeof(struct kib_msg) > IBLND_MSG_SIZE);
 	BUILD_BUG_ON(offsetof(struct kib_msg,
 			  ibm_u.get.ibgm_rd.rd_frags[IBLND_MAX_RDMA_FRAGS])
@@ -2938,6 +2940,10 @@ static int __init ko2iblnd_init(void)
 
 	kiblnd_tunables_init();
 
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	lnet_register_lnd(&the_o2iblnd);
 
 	return 0;
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index b18911d09e9a..217503f125bc 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -64,7 +64,6 @@
 
 #define DEBUG_SUBSYSTEM S_LND
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/lnet/lib-lnet.h>
 
 #define IBLND_PEER_HASH_SIZE		101	/* # peer lists */
@@ -568,6 +567,8 @@ struct kib_peer {
 	lnet_nid_t       ibp_nid;         /* who's on the other end(s) */
 	struct lnet_ni	*ibp_ni;         /* LNet interface */
 	struct list_head ibp_conns;       /* all active connections */
+	struct kib_conn	*ibp_next_conn;  /* next connection to send on for
+					  * round robin */
 	struct list_head ibp_tx_queue;    /* msgs waiting for a conn */
 	__u64            ibp_incarnation; /* incarnation of peer */
 	/* when (in jiffies) I was last alive */
@@ -581,7 +582,7 @@ struct kib_peer {
 	/* current active connection attempts */
 	unsigned short		ibp_connecting;
 	/* reconnect this peer later */
-	unsigned short		ibp_reconnecting:1;
+	unsigned char		ibp_reconnecting;
 	/* counter of how many times we triggered a conn race */
 	unsigned char		ibp_races;
 	/* # consecutive reconnection attempts to this peer */
@@ -744,19 +745,28 @@ kiblnd_peer_active(struct kib_peer *peer)
 static inline struct kib_conn *
 kiblnd_get_conn_locked(struct kib_peer *peer)
 {
+	struct list_head *next;
+
 	LASSERT(!list_empty(&peer->ibp_conns));
 
-	/* just return the first connection */
-	return list_entry(peer->ibp_conns.next, struct kib_conn, ibc_list);
+	/* Advance to next connection, be sure to skip the head node */
+	if (!peer->ibp_next_conn ||
+	    peer->ibp_next_conn->ibc_list.next == &peer->ibp_conns)
+		next = peer->ibp_conns.next;
+	else
+		next = peer->ibp_next_conn->ibc_list.next;
+	peer->ibp_next_conn = list_entry(next, struct kib_conn, ibc_list);
+
+	return peer->ibp_next_conn;
 }
 
 static inline int
 kiblnd_send_keepalive(struct kib_conn *conn)
 {
 	return (*kiblnd_tunables.kib_keepalive > 0) &&
-		cfs_time_after(jiffies, conn->ibc_last_send +
-			       msecs_to_jiffies(*kiblnd_tunables.kib_keepalive *
-						MSEC_PER_SEC));
+		time_after(jiffies, conn->ibc_last_send +
+			   msecs_to_jiffies(*kiblnd_tunables.kib_keepalive *
+					    MSEC_PER_SEC));
 }
 
 static inline int
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
index 6690a6cd4e34..47eb8b4c28db 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -48,7 +48,7 @@ static int kiblnd_init_rdma(struct kib_conn *conn, struct kib_tx *tx, int type,
 			    __u64 dstcookie);
 static void kiblnd_queue_tx_locked(struct kib_tx *tx, struct kib_conn *conn);
 static void kiblnd_queue_tx(struct kib_tx *tx, struct kib_conn *conn);
-static void kiblnd_unmap_tx(struct lnet_ni *ni, struct kib_tx *tx);
+static void kiblnd_unmap_tx(struct kib_tx *tx);
 static void kiblnd_check_sends_locked(struct kib_conn *conn);
 
 static void
@@ -66,7 +66,7 @@ kiblnd_tx_done(struct lnet_ni *ni, struct kib_tx *tx)
 	LASSERT(!tx->tx_waiting);	      /* mustn't be awaiting peer response */
 	LASSERT(tx->tx_pool);
 
-	kiblnd_unmap_tx(ni, tx);
+	kiblnd_unmap_tx(tx);
 
 	/* tx may have up to 2 lnet msgs to finalise */
 	lntmsg[0] = tx->tx_lntmsg[0]; tx->tx_lntmsg[0] = NULL;
@@ -591,13 +591,9 @@ kiblnd_fmr_map_tx(struct kib_net *net, struct kib_tx *tx, struct kib_rdma_desc *
 	return 0;
 }
 
-static void kiblnd_unmap_tx(struct lnet_ni *ni, struct kib_tx *tx)
+static void kiblnd_unmap_tx(struct kib_tx *tx)
 {
-	struct kib_net *net = ni->ni_data;
-
-	LASSERT(net);
-
-	if (net->ibn_fmr_ps)
+	if (tx->fmr.fmr_pfmr || tx->fmr.fmr_frd)
 		kiblnd_fmr_pool_unmap(&tx->fmr, tx->tx_status);
 
 	if (tx->tx_nfrags) {
@@ -1250,7 +1246,6 @@ kiblnd_connect_peer(struct kib_peer *peer)
 
 	LASSERT(net);
 	LASSERT(peer->ibp_connecting > 0);
-	LASSERT(!peer->ibp_reconnecting);
 
 	cmid = kiblnd_rdma_create_id(kiblnd_cm_callback, peer, RDMA_PS_TCP,
 				     IB_QPT_RC);
@@ -1290,11 +1285,6 @@ kiblnd_connect_peer(struct kib_peer *peer)
 		goto failed2;
 	}
 
-	LASSERT(cmid->device);
-	CDEBUG(D_NET, "%s: connection bound to %s:%pI4h:%s\n",
-	       libcfs_nid2str(peer->ibp_nid), dev->ibd_ifname,
-	       &dev->ibd_ifip, cmid->device->name);
-
 	return;
 
  failed2:
@@ -1332,7 +1322,7 @@ kiblnd_reconnect_peer(struct kib_peer *peer)
 
 	LASSERT(!peer->ibp_accepting && !peer->ibp_connecting &&
 		list_empty(&peer->ibp_conns));
-	peer->ibp_reconnecting = 0;
+	peer->ibp_reconnecting--;
 
 	if (!kiblnd_peer_active(peer)) {
 		list_splice_init(&peer->ibp_tx_queue, &txs);
@@ -1365,6 +1355,8 @@ kiblnd_launch_tx(struct lnet_ni *ni, struct kib_tx *tx, lnet_nid_t nid)
 	rwlock_t *g_lock = &kiblnd_data.kib_global_lock;
 	unsigned long flags;
 	int rc;
+	int		   i;
+	struct lnet_ioctl_config_o2iblnd_tunables *tunables;
 
 	/*
 	 * If I get here, I've committed to send, so I complete the tx with
@@ -1461,7 +1453,8 @@ kiblnd_launch_tx(struct lnet_ni *ni, struct kib_tx *tx, lnet_nid_t nid)
 
 	/* Brand new peer */
 	LASSERT(!peer->ibp_connecting);
-	peer->ibp_connecting = 1;
+	tunables = &peer->ibp_ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
+	peer->ibp_connecting = tunables->lnd_conns_per_peer;
 
 	/* always called with a ref on ni, which prevents ni being shutdown */
 	LASSERT(!((struct kib_net *)ni->ni_data)->ibn_shutdown);
@@ -1474,7 +1467,8 @@ kiblnd_launch_tx(struct lnet_ni *ni, struct kib_tx *tx, lnet_nid_t nid)
 
 	write_unlock_irqrestore(g_lock, flags);
 
-	kiblnd_connect_peer(peer);
+	for (i = 0; i < tunables->lnd_conns_per_peer; i++)
+		kiblnd_connect_peer(peer);
 	kiblnd_peer_decref(peer);
 }
 
@@ -1853,8 +1847,8 @@ kiblnd_thread_fini(void)
 static void
 kiblnd_peer_alive(struct kib_peer *peer)
 {
-	/* This is racy, but everyone's only writing cfs_time_current() */
-	peer->ibp_last_alive = cfs_time_current();
+	/* This is racy, but everyone's only writing jiffies */
+	peer->ibp_last_alive = jiffies;
 	mb();
 }
 
@@ -1923,6 +1917,9 @@ kiblnd_close_conn_locked(struct kib_conn *conn, int error)
 	}
 
 	dev = ((struct kib_net *)peer->ibp_ni->ni_data)->ibn_dev;
+	if (peer->ibp_next_conn == conn)
+		/* clear next_conn so it won't be used */
+		peer->ibp_next_conn = NULL;
 	list_del(&conn->ibc_list);
 	/* connd (see below) takes over ibc_list's ref */
 
@@ -1968,13 +1965,14 @@ kiblnd_handle_early_rxs(struct kib_conn *conn)
 {
 	unsigned long flags;
 	struct kib_rx *rx;
-	struct kib_rx *tmp;
 
 	LASSERT(!in_interrupt());
 	LASSERT(conn->ibc_state >= IBLND_CONN_ESTABLISHED);
 
 	write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
-	list_for_each_entry_safe(rx, tmp, &conn->ibc_early_rxs, rx_list) {
+	while (!list_empty(&conn->ibc_early_rxs)) {
+		rx = list_entry(conn->ibc_early_rxs.next,
+				struct kib_rx, rx_list);
 		list_del(&rx->rx_list);
 		write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
 
@@ -2192,7 +2190,11 @@ kiblnd_connreq_done(struct kib_conn *conn, int status)
 	kiblnd_conn_addref(conn);
 	write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
 
-	/* Schedule blocked txs */
+	/* Schedule blocked txs
+	 * Note: if we are running with conns_per_peer > 1, these blocked
+	 * txs will all get scheduled to the first connection which gets
+	 * scheduled.  We won't be using round robin on this first batch.
+	 */
 	spin_lock(&conn->ibc_lock);
 	list_for_each_entry_safe(tx, tmp, &txs, tx_list) {
 		list_del(&tx->tx_list);
@@ -2561,7 +2563,6 @@ kiblnd_check_reconnect(struct kib_conn *conn, int version,
 
 	LASSERT(conn->ibc_state == IBLND_CONN_ACTIVE_CONNECT);
 	LASSERT(peer->ibp_connecting > 0);     /* 'conn' at least */
-	LASSERT(!peer->ibp_reconnecting);
 
 	if (cp) {
 		msg_size = cp->ibcp_max_msg_size;
@@ -2579,7 +2580,7 @@ kiblnd_check_reconnect(struct kib_conn *conn, int version,
 	 */
 	reconnect = (!list_empty(&peer->ibp_tx_queue) ||
 		     peer->ibp_version != version) &&
-		    peer->ibp_connecting == 1 &&
+		    peer->ibp_connecting &&
 		    !peer->ibp_accepting;
 	if (!reconnect) {
 		reason = "no need";
@@ -2640,7 +2641,7 @@ kiblnd_check_reconnect(struct kib_conn *conn, int version,
 	}
 
 	conn->ibc_reconnect = 1;
-	peer->ibp_reconnecting = 1;
+	peer->ibp_reconnecting++;
 	peer->ibp_version = version;
 	if (incarnation)
 		peer->ibp_incarnation = incarnation;
@@ -2996,8 +2997,19 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
 		} else {
 			rc = rdma_resolve_route(
 				cmid, *kiblnd_tunables.kib_timeout * 1000);
-			if (!rc)
+			if (!rc) {
+				struct kib_net *net = peer->ibp_ni->ni_data;
+				struct kib_dev *dev = net->ibn_dev;
+
+				CDEBUG(D_NET, "%s: connection bound to "\
+				       "%s:%pI4h:%s\n",
+				       libcfs_nid2str(peer->ibp_nid),
+				       dev->ibd_ifname,
+				       &dev->ibd_ifip, cmid->device->name);
+
 				return 0;
+			}
+
 			/* Can't initiate route resolution */
 			CERROR("Can't resolve route for %s: %d\n",
 			       libcfs_nid2str(peer->ibp_nid), rc);
@@ -3141,10 +3153,10 @@ kiblnd_check_txs_locked(struct kib_conn *conn, struct list_head *txs)
 			LASSERT(tx->tx_waiting || tx->tx_sending);
 		}
 
-		if (cfs_time_aftereq(jiffies, tx->tx_deadline)) {
+		if (time_after_eq(jiffies, tx->tx_deadline)) {
 			CERROR("Timed out tx: %s, %lu seconds\n",
 			       kiblnd_queue2str(conn, txs),
-			       cfs_duration_sec(jiffies - tx->tx_deadline));
+			       (jiffies - tx->tx_deadline) / HZ);
 			return 1;
 		}
 	}
@@ -3206,8 +3218,7 @@ kiblnd_check_conns(int idx)
 			if (timedout) {
 				CERROR("Timed out RDMA with %s (%lu): c: %u, oc: %u, rc: %u\n",
 				       libcfs_nid2str(peer->ibp_nid),
-				       cfs_duration_sec(cfs_time_current() -
-							peer->ibp_last_alive),
+				       (jiffies - peer->ibp_last_alive) / HZ,
 				       conn->ibc_credits,
 				       conn->ibc_outstanding_credits,
 				       conn->ibc_reserved_credits);
@@ -3681,7 +3692,7 @@ kiblnd_failover_thread(void *arg)
 
 		list_for_each_entry(dev, &kiblnd_data.kib_failed_devs,
 				    ibd_fail_list) {
-			if (time_before(cfs_time_current(),
+			if (time_before(jiffies,
 					dev->ibd_next_failover))
 				continue;
 			do_failover = 1;
@@ -3700,13 +3711,13 @@ kiblnd_failover_thread(void *arg)
 			LASSERT(dev->ibd_failover);
 			dev->ibd_failover = 0;
 			if (rc >= 0) { /* Device is OK or failover succeed */
-				dev->ibd_next_failover = cfs_time_shift(3);
+				dev->ibd_next_failover = jiffies + 3 * HZ;
 				continue;
 			}
 
 			/* failed to failover, retry later */
 			dev->ibd_next_failover =
-				cfs_time_shift(min(dev->ibd_failed_failover, 10));
+				jiffies + min(dev->ibd_failed_failover, 10) * HZ;
 			if (kiblnd_dev_can_failover(dev)) {
 				list_add_tail(&dev->ibd_fail_list,
 					      &kiblnd_data.kib_failed_devs);
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index b9235400bf1d..39d07926d603 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -57,6 +57,10 @@ static int nscheds;
 module_param(nscheds, int, 0444);
 MODULE_PARM_DESC(nscheds, "number of threads in each scheduler pool");
 
+static unsigned int conns_per_peer = 1;
+module_param(conns_per_peer, uint, 0444);
+MODULE_PARM_DESC(conns_per_peer, "number of connections per peer");
+
 /* NB: this value is shared by all CPTs, it can grow at runtime */
 static int ntx = 512;
 module_param(ntx, int, 0444);
@@ -271,6 +275,10 @@ int kiblnd_tunables_setup(struct lnet_ni *ni)
 		tunables->lnd_fmr_flush_trigger = fmr_flush_trigger;
 	if (!tunables->lnd_fmr_cache)
 		tunables->lnd_fmr_cache = fmr_cache;
+	if (!tunables->lnd_conns_per_peer) {
+		tunables->lnd_conns_per_peer = (conns_per_peer) ?
+			conns_per_peer : 1;
+	}
 
 	return 0;
 }
@@ -284,4 +292,5 @@ void kiblnd_tunables_init(void)
 	default_tunables.lnd_fmr_pool_size = fmr_pool_size;
 	default_tunables.lnd_fmr_flush_trigger = fmr_flush_trigger;
 	default_tunables.lnd_fmr_cache = fmr_cache;
+	default_tunables.lnd_conns_per_peer = conns_per_peer;
 }
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c
index 7086678e1c3e..f01b34ac1a53 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c
@@ -1276,7 +1276,7 @@ ksocknal_create_conn(struct lnet_ni *ni, struct ksock_route *route,
 	}
 
 	conn->ksnc_peer = peer;		 /* conn takes my ref on peer */
-	peer->ksnp_last_alive = cfs_time_current();
+	peer->ksnp_last_alive = jiffies;
 	peer->ksnp_send_keepalive = 0;
 	peer->ksnp_error = 0;
 
@@ -1284,10 +1284,10 @@ ksocknal_create_conn(struct lnet_ni *ni, struct ksock_route *route,
 	sched->kss_nconns++;
 	conn->ksnc_scheduler = sched;
 
-	conn->ksnc_tx_last_post = cfs_time_current();
+	conn->ksnc_tx_last_post = jiffies;
 	/* Set the deadline for the outgoing HELLO to drain */
 	conn->ksnc_tx_bufnob = sock->sk->sk_wmem_queued;
-	conn->ksnc_tx_deadline = cfs_time_shift(*ksocknal_tunables.ksnd_timeout);
+	conn->ksnc_tx_deadline = jiffies + *ksocknal_tunables.ksnd_timeout * HZ;
 	mb();   /* order with adding to peer's conn list */
 
 	list_add(&conn->ksnc_list, &peer->ksnp_conns);
@@ -1682,8 +1682,7 @@ ksocknal_destroy_conn(struct ksock_conn *conn)
 		       libcfs_id2str(conn->ksnc_peer->ksnp_id), conn->ksnc_type,
 		       &conn->ksnc_ipaddr, conn->ksnc_port,
 		       iov_iter_count(&conn->ksnc_rx_to), conn->ksnc_rx_nob_left,
-		       cfs_duration_sec(cfs_time_sub(cfs_time_current(),
-						     last_rcv)));
+		       (jiffies - last_rcv) / HZ);
 		lnet_finalize(conn->ksnc_peer->ksnp_ni,
 			      conn->ksnc_cookie, -EIO);
 		break;
@@ -1832,7 +1831,7 @@ ksocknal_query(struct lnet_ni *ni, lnet_nid_t nid, unsigned long *when)
 {
 	int connect = 1;
 	unsigned long last_alive = 0;
-	unsigned long now = cfs_time_current();
+	unsigned long now = jiffies;
 	struct ksock_peer *peer = NULL;
 	rwlock_t *glock = &ksocknal_data.ksnd_global_lock;
 	struct lnet_process_id id = {
@@ -1853,7 +1852,7 @@ ksocknal_query(struct lnet_ni *ni, lnet_nid_t nid, unsigned long *when)
 			if (bufnob < conn->ksnc_tx_bufnob) {
 				/* something got ACKed */
 				conn->ksnc_tx_deadline =
-					cfs_time_shift(*ksocknal_tunables.ksnd_timeout);
+					jiffies + *ksocknal_tunables.ksnd_timeout * HZ;
 				peer->ksnp_last_alive = now;
 				conn->ksnc_tx_bufnob = bufnob;
 			}
@@ -1871,7 +1870,7 @@ ksocknal_query(struct lnet_ni *ni, lnet_nid_t nid, unsigned long *when)
 
 	CDEBUG(D_NET, "Peer %s %p, alive %ld secs ago, connect %d\n",
 	       libcfs_nid2str(nid), peer,
-	       last_alive ? cfs_duration_sec(now - last_alive) : -1,
+	       last_alive ? (now - last_alive) / HZ : -1,
 	       connect);
 
 	if (!connect)
@@ -2904,6 +2903,10 @@ static int __init ksocklnd_init(void)
 	if (rc)
 		return rc;
 
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	lnet_register_lnd(&the_ksocklnd);
 
 	return 0;
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
index 570f54ed57b1..4e5c89a692a3 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
@@ -47,7 +47,6 @@
 #include <net/sock.h>
 #include <net/tcp.h>
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/lnet/lib-lnet.h>
 #include <linux/lnet/socklnd.h>
 
@@ -58,7 +57,7 @@
 #define SOCKNAL_PEER_HASH_SIZE  101   /* # peer lists */
 #define SOCKNAL_RESCHED         100   /* # scheduler loops before reschedule */
 #define SOCKNAL_INSANITY_RECONN 5000  /* connd is trying on reconn infinitely */
-#define SOCKNAL_ENOMEM_RETRY    CFS_TICK /* jiffies between retries */
+#define SOCKNAL_ENOMEM_RETRY    1     /* jiffies between retries */
 
 #define SOCKNAL_SINGLE_FRAG_TX  0     /* disable multi-fragment sends */
 #define SOCKNAL_SINGLE_FRAG_RX  0     /* disable multi-fragment receives */
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c
index 036fecbcede8..01b31a6bb588 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c
@@ -22,6 +22,7 @@
  *
  */
 
+#include <linux/sched/mm.h>
 #include "socklnd.h"
 
 struct ksock_tx *
@@ -221,8 +222,8 @@ ksocknal_transmit(struct ksock_conn *conn, struct ksock_tx *tx)
 			 * something got ACKed
 			 */
 			conn->ksnc_tx_deadline =
-				cfs_time_shift(*ksocknal_tunables.ksnd_timeout);
-			conn->ksnc_peer->ksnp_last_alive = cfs_time_current();
+				jiffies + *ksocknal_tunables.ksnd_timeout * HZ;
+			conn->ksnc_peer->ksnp_last_alive = jiffies;
 			conn->ksnc_tx_bufnob = bufnob;
 			mb();
 		}
@@ -267,9 +268,9 @@ ksocknal_recv_iter(struct ksock_conn *conn)
 	/* received something... */
 	nob = rc;
 
-	conn->ksnc_peer->ksnp_last_alive = cfs_time_current();
+	conn->ksnc_peer->ksnp_last_alive = jiffies;
 	conn->ksnc_rx_deadline =
-		cfs_time_shift(*ksocknal_tunables.ksnd_timeout);
+		jiffies + *ksocknal_tunables.ksnd_timeout * HZ;
 	mb();		       /* order with setting rx_started */
 	conn->ksnc_rx_started = 1;
 
@@ -405,7 +406,7 @@ ksocknal_check_zc_req(struct ksock_tx *tx)
 
 	/* ZC_REQ is going to be pinned to the peer */
 	tx->tx_deadline =
-		cfs_time_shift(*ksocknal_tunables.ksnd_timeout);
+		jiffies + *ksocknal_tunables.ksnd_timeout * HZ;
 
 	LASSERT(!tx->tx_msg.ksm_zc_cookies[0]);
 
@@ -481,8 +482,7 @@ ksocknal_process_transmit(struct ksock_conn *conn, struct ksock_tx *tx)
 		LASSERT(conn->ksnc_tx_scheduled);
 		list_add_tail(&conn->ksnc_tx_list,
 			      &ksocknal_data.ksnd_enomem_conns);
-		if (!cfs_time_aftereq(cfs_time_add(cfs_time_current(),
-						   SOCKNAL_ENOMEM_RETRY),
+		if (!time_after_eq(jiffies + SOCKNAL_ENOMEM_RETRY,
 				   ksocknal_data.ksnd_reaper_waketime))
 			wake_up(&ksocknal_data.ksnd_reaper_waitq);
 
@@ -591,7 +591,7 @@ ksocknal_find_conn_locked(struct ksock_peer *peer, struct ksock_tx *tx,
 		case SOCKNAL_MATCH_YES: /* typed connection */
 			if (!typed || tnob > nob ||
 			    (tnob == nob && *ksocknal_tunables.ksnd_round_robin &&
-			     cfs_time_after(typed->ksnc_tx_last_post, c->ksnc_tx_last_post))) {
+			     time_after(typed->ksnc_tx_last_post, c->ksnc_tx_last_post))) {
 				typed = c;
 				tnob  = nob;
 			}
@@ -600,7 +600,7 @@ ksocknal_find_conn_locked(struct ksock_peer *peer, struct ksock_tx *tx,
 		case SOCKNAL_MATCH_MAY: /* fallback connection */
 			if (!fallback || fnob > nob ||
 			    (fnob == nob && *ksocknal_tunables.ksnd_round_robin &&
-			     cfs_time_after(fallback->ksnc_tx_last_post, c->ksnc_tx_last_post))) {
+			     time_after(fallback->ksnc_tx_last_post, c->ksnc_tx_last_post))) {
 				fallback = c;
 				fnob = nob;
 			}
@@ -612,7 +612,7 @@ ksocknal_find_conn_locked(struct ksock_peer *peer, struct ksock_tx *tx,
 	conn = (typed) ? typed : fallback;
 
 	if (conn)
-		conn->ksnc_tx_last_post = cfs_time_current();
+		conn->ksnc_tx_last_post = jiffies;
 
 	return conn;
 }
@@ -678,9 +678,9 @@ ksocknal_queue_tx_locked(struct ksock_tx *tx, struct ksock_conn *conn)
 	if (list_empty(&conn->ksnc_tx_queue) && !bufnob) {
 		/* First packet starts the timeout */
 		conn->ksnc_tx_deadline =
-			cfs_time_shift(*ksocknal_tunables.ksnd_timeout);
+			jiffies + *ksocknal_tunables.ksnd_timeout * HZ;
 		if (conn->ksnc_tx_bufnob > 0) /* something got ACKed */
-			conn->ksnc_peer->ksnp_last_alive = cfs_time_current();
+			conn->ksnc_peer->ksnp_last_alive = jiffies;
 		conn->ksnc_tx_bufnob = 0;
 		mb(); /* order with adding to tx_queue */
 	}
@@ -728,7 +728,7 @@ ksocknal_queue_tx_locked(struct ksock_tx *tx, struct ksock_conn *conn)
 struct ksock_route *
 ksocknal_find_connectable_route_locked(struct ksock_peer *peer)
 {
-	unsigned long now = cfs_time_current();
+	unsigned long now = jiffies;
 	struct list_head *tmp;
 	struct ksock_route *route;
 
@@ -746,13 +746,13 @@ ksocknal_find_connectable_route_locked(struct ksock_peer *peer)
 			continue;
 
 		if (!(!route->ksnr_retry_interval || /* first attempt */
-		      cfs_time_aftereq(now, route->ksnr_timeout))) {
+		      time_after_eq(now, route->ksnr_timeout))) {
 			CDEBUG(D_NET,
 			       "Too soon to retry route %pI4h (cnted %d, interval %ld, %ld secs later)\n",
 			       &route->ksnr_ipaddr,
 			       route->ksnr_connected,
 			       route->ksnr_retry_interval,
-			       cfs_duration_sec(route->ksnr_timeout - now));
+			       (route->ksnr_timeout - now) / HZ);
 			continue;
 		}
 
@@ -859,7 +859,7 @@ ksocknal_launch_packet(struct lnet_ni *ni, struct ksock_tx *tx,
 	    ksocknal_find_connecting_route_locked(peer)) {
 		/* the message is going to be pinned to the peer */
 		tx->tx_deadline =
-			cfs_time_shift(*ksocknal_tunables.ksnd_timeout);
+			jiffies + *ksocknal_tunables.ksnd_timeout * HZ;
 
 		/* Queue the message until a connection is established */
 		list_add_tail(&tx->tx_list, &peer->ksnp_tx_queue);
@@ -877,7 +877,7 @@ ksocknal_launch_packet(struct lnet_ni *ni, struct ksock_tx *tx,
 int
 ksocknal_send(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg)
 {
-	int mpflag = 1;
+	unsigned int mpflag = 0;
 	int type = lntmsg->msg_type;
 	struct lnet_process_id target = lntmsg->msg_target;
 	unsigned int payload_niov = lntmsg->msg_niov;
@@ -910,13 +910,13 @@ ksocknal_send(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg)
 				     tx_frags.paged.kiov[payload_niov]);
 
 	if (lntmsg->msg_vmflush)
-		mpflag = cfs_memory_pressure_get_and_set();
+		mpflag = memalloc_noreclaim_save();
 	tx = ksocknal_alloc_tx(KSOCK_MSG_LNET, desc_size);
 	if (!tx) {
 		CERROR("Can't allocate tx desc type %d size %d\n",
 		       type, desc_size);
 		if (lntmsg->msg_vmflush)
-			cfs_memory_pressure_restore(mpflag);
+			memalloc_noreclaim_restore(mpflag);
 		return -ENOMEM;
 	}
 
@@ -950,8 +950,8 @@ ksocknal_send(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg)
 
 	/* The first fragment will be set later in pro_pack */
 	rc = ksocknal_launch_packet(ni, tx, target);
-	if (!mpflag)
-		cfs_memory_pressure_restore(mpflag);
+	if (mpflag)
+		memalloc_noreclaim_restore(mpflag);
 
 	if (!rc)
 		return 0;
@@ -1777,8 +1777,7 @@ ksocknal_connect(struct ksock_route *route)
 	int retry_later = 0;
 	int rc = 0;
 
-	deadline = cfs_time_add(cfs_time_current(),
-				*ksocknal_tunables.ksnd_timeout * HZ);
+	deadline = jiffies + *ksocknal_tunables.ksnd_timeout * HZ;
 
 	write_lock_bh(&ksocknal_data.ksnd_global_lock);
 
@@ -1825,7 +1824,7 @@ ksocknal_connect(struct ksock_route *route)
 
 		write_unlock_bh(&ksocknal_data.ksnd_global_lock);
 
-		if (cfs_time_aftereq(cfs_time_current(), deadline)) {
+		if (time_after_eq(jiffies, deadline)) {
 			rc = -ETIMEDOUT;
 			lnet_connect_console_error(rc, peer->ksnp_id.nid,
 						   route->ksnr_ipaddr,
@@ -1877,8 +1876,7 @@ ksocknal_connect(struct ksock_route *route)
 			 */
 			route->ksnr_retry_interval =
 				*ksocknal_tunables.ksnd_min_reconnectms * HZ / 1000;
-			route->ksnr_timeout = cfs_time_add(cfs_time_current(),
-							   route->ksnr_retry_interval);
+			route->ksnr_timeout = jiffies + route->ksnr_retry_interval;
 		}
 
 		ksocknal_launch_connection_locked(route);
@@ -1903,8 +1901,7 @@ ksocknal_connect(struct ksock_route *route)
 		    (long)*ksocknal_tunables.ksnd_max_reconnectms * HZ / 1000);
 
 	LASSERT(route->ksnr_retry_interval);
-	route->ksnr_timeout = cfs_time_add(cfs_time_current(),
-					   route->ksnr_retry_interval);
+	route->ksnr_timeout = jiffies + route->ksnr_retry_interval;
 
 	if (!list_empty(&peer->ksnp_tx_queue) &&
 	    !peer->ksnp_accepting &&
@@ -2050,13 +2047,13 @@ ksocknal_connd_get_route_locked(signed long *timeout_p)
 	struct ksock_route *route;
 	unsigned long now;
 
-	now = cfs_time_current();
+	now = jiffies;
 
 	/* connd_routes can contain both pending and ordinary routes */
 	list_for_each_entry(route, &ksocknal_data.ksnd_connd_routes,
 			    ksnr_connd_list) {
 		if (!route->ksnr_retry_interval ||
-		    cfs_time_aftereq(now, route->ksnr_timeout))
+		    time_after_eq(now, route->ksnr_timeout))
 			return route;
 
 		if (*timeout_p == MAX_SCHEDULE_TIMEOUT ||
@@ -2228,8 +2225,8 @@ ksocknal_find_timed_out_conn(struct ksock_peer *peer)
 		}
 
 		if (conn->ksnc_rx_started &&
-		    cfs_time_aftereq(cfs_time_current(),
-				     conn->ksnc_rx_deadline)) {
+		    time_after_eq(jiffies,
+				  conn->ksnc_rx_deadline)) {
 			/* Timed out incomplete incoming message */
 			ksocknal_conn_addref(conn);
 			CNETERR("Timeout receiving from %s (%pI4h:%d), state %d wanted %zd left %d\n",
@@ -2244,8 +2241,8 @@ ksocknal_find_timed_out_conn(struct ksock_peer *peer)
 
 		if ((!list_empty(&conn->ksnc_tx_queue) ||
 		     conn->ksnc_sock->sk->sk_wmem_queued) &&
-		    cfs_time_aftereq(cfs_time_current(),
-				     conn->ksnc_tx_deadline)) {
+		    time_after_eq(jiffies,
+				  conn->ksnc_tx_deadline)) {
 			/*
 			 * Timed out messages queued for sending or
 			 * buffered in the socket's send buffer
@@ -2272,8 +2269,8 @@ ksocknal_flush_stale_txs(struct ksock_peer *peer)
 	write_lock_bh(&ksocknal_data.ksnd_global_lock);
 
 	list_for_each_entry_safe(tx, tmp, &peer->ksnp_tx_queue, tx_list) {
-		if (!cfs_time_aftereq(cfs_time_current(),
-				      tx->tx_deadline))
+		if (!time_after_eq(jiffies,
+				   tx->tx_deadline))
 			break;
 
 		list_del(&tx->tx_list);
@@ -2301,19 +2298,18 @@ ksocknal_send_keepalive_locked(struct ksock_peer *peer)
 		return 0;
 
 	if (*ksocknal_tunables.ksnd_keepalive <= 0 ||
-	    time_before(cfs_time_current(),
-			cfs_time_add(peer->ksnp_last_alive,
-				     *ksocknal_tunables.ksnd_keepalive * HZ)))
+	    time_before(jiffies,
+			peer->ksnp_last_alive + *ksocknal_tunables.ksnd_keepalive * HZ))
 		return 0;
 
-	if (time_before(cfs_time_current(), peer->ksnp_send_keepalive))
+	if (time_before(jiffies, peer->ksnp_send_keepalive))
 		return 0;
 
 	/*
 	 * retry 10 secs later, so we wouldn't put pressure
 	 * on this peer if we failed to send keepalive this time
 	 */
-	peer->ksnp_send_keepalive = cfs_time_shift(10);
+	peer->ksnp_send_keepalive = jiffies + 10 * HZ;
 
 	conn = ksocknal_find_conn_locked(peer, NULL, 1);
 	if (conn) {
@@ -2400,8 +2396,8 @@ ksocknal_check_peer_timeouts(int idx)
 			tx = list_entry(peer->ksnp_tx_queue.next,
 					struct ksock_tx, tx_list);
 
-			if (cfs_time_aftereq(cfs_time_current(),
-					     tx->tx_deadline)) {
+			if (time_after_eq(jiffies,
+					  tx->tx_deadline)) {
 				ksocknal_peer_addref(peer);
 				read_unlock(&ksocknal_data.ksnd_global_lock);
 
@@ -2418,8 +2414,8 @@ ksocknal_check_peer_timeouts(int idx)
 		tx_stale = NULL;
 		spin_lock(&peer->ksnp_lock);
 		list_for_each_entry(tx, &peer->ksnp_zc_req_list, tx_zc_list) {
-			if (!cfs_time_aftereq(cfs_time_current(),
-					      tx->tx_deadline))
+			if (!time_after_eq(jiffies,
+					   tx->tx_deadline))
 				break;
 			/* ignore the TX if connection is being closed */
 			if (tx->tx_conn->ksnc_closing)
@@ -2444,7 +2440,7 @@ ksocknal_check_peer_timeouts(int idx)
 
 		CERROR("Total %d stale ZC_REQs for peer %s detected; the oldest(%p) timed out %ld secs ago, resid: %d, wmem: %d\n",
 		       n, libcfs_nid2str(peer->ksnp_id.nid), tx_stale,
-		       cfs_duration_sec(cfs_time_current() - deadline),
+		       (jiffies - deadline) / HZ,
 		       resid, conn->ksnc_sock->sk->sk_wmem_queued);
 
 		ksocknal_close_conn_and_siblings(conn, -ETIMEDOUT);
@@ -2466,7 +2462,7 @@ ksocknal_reaper(void *arg)
 	long timeout;
 	int i;
 	int peer_index = 0;
-	unsigned long deadline = cfs_time_current();
+	unsigned long deadline = jiffies;
 
 	INIT_LIST_HEAD(&enomem_conns);
 	init_waitqueue_entry(&wait, current);
@@ -2531,8 +2527,7 @@ ksocknal_reaper(void *arg)
 		}
 
 		/* careful with the jiffy wrap... */
-		while ((timeout = cfs_time_sub(deadline,
-					       cfs_time_current())) <= 0) {
+		while ((timeout = deadline - jiffies) <= 0) {
 			const int n = 4;
 			const int p = 1;
 			int chunk = ksocknal_data.ksnd_peer_hash_size;
@@ -2557,7 +2552,7 @@ ksocknal_reaper(void *arg)
 					     ksocknal_data.ksnd_peer_hash_size;
 			}
 
-			deadline = cfs_time_add(deadline, p * HZ);
+			deadline = deadline + p * HZ;
 		}
 
 		if (nenomem_conns) {
@@ -2568,8 +2563,7 @@ ksocknal_reaper(void *arg)
 			 */
 			timeout = SOCKNAL_ENOMEM_RETRY;
 		}
-		ksocknal_data.ksnd_reaper_waketime =
-			cfs_time_add(cfs_time_current(), timeout);
+		ksocknal_data.ksnd_reaper_waketime = jiffies + timeout;
 
 		set_current_state(TASK_INTERRUPTIBLE);
 		add_wait_queue(&ksocknal_data.ksnd_reaper_waitq, &wait);
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
index 7941cfa526bc..93a02cd6b6b5 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
@@ -31,6 +31,7 @@
  * Lustre is a trademark of Sun Microsystems, Inc.
  */
 
+#include <linux/highmem.h>
 #include "socklnd.h"
 
 int
diff --git a/drivers/staging/lustre/lnet/libcfs/Makefile b/drivers/staging/lustre/lnet/libcfs/Makefile
index b7dc7ac11cc5..6a1b232da495 100644
--- a/drivers/staging/lustre/lnet/libcfs/Makefile
+++ b/drivers/staging/lustre/lnet/libcfs/Makefile
@@ -4,16 +4,13 @@ subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
 
 obj-$(CONFIG_LNET) += libcfs.o
 
-libcfs-linux-objs := linux-tracefile.o linux-debug.o
-libcfs-linux-objs += linux-cpu.o
-libcfs-linux-objs += linux-module.o
-libcfs-linux-objs += linux-crypto.o
-libcfs-linux-objs += linux-crypto-adler.o
+libcfs-obj-y += linux-tracefile.o linux-debug.o
+libcfs-obj-y += linux-crypto.o
+libcfs-obj-y += linux-crypto-adler.o
 
-libcfs-linux-objs := $(addprefix linux/,$(libcfs-linux-objs))
+libcfs-obj-y += debug.o fail.o module.o tracefile.o
+libcfs-obj-y += libcfs_string.o hash.o
+libcfs-obj-$(CONFIG_SMP) += libcfs_cpu.o
+libcfs-obj-y += libcfs_mem.o libcfs_lock.o
 
-libcfs-all-objs := debug.o fail.o module.o tracefile.o \
-		   libcfs_string.o hash.o \
-		   libcfs_cpu.o libcfs_mem.o libcfs_lock.o
-
-libcfs-objs := $(libcfs-linux-objs) $(libcfs-all-objs)
+libcfs-objs := $(libcfs-obj-y)
diff --git a/drivers/staging/lustre/lnet/libcfs/debug.c b/drivers/staging/lustre/lnet/libcfs/debug.c
index 1371224a8cb9..06f694f6a28f 100644
--- a/drivers/staging/lustre/lnet/libcfs/debug.c
+++ b/drivers/staging/lustre/lnet/libcfs/debug.c
@@ -38,7 +38,10 @@
 
 # define DEBUG_SUBSYSTEM S_LNET
 
-#include <linux/libcfs/libcfs.h>
+#include <linux/module.h>
+#include <linux/ctype.h>
+#include <linux/libcfs/libcfs_string.h>
+#include <linux/kthread.h>
 #include "tracefile.h"
 
 static char debug_file_name[1024];
@@ -126,7 +129,7 @@ static int param_get_delay(char *buffer, const struct kernel_param *kp)
 {
 	unsigned int d = *(unsigned int *)kp->arg;
 
-	return sprintf(buffer, "%u", (unsigned int)cfs_duration_sec(d * 100));
+	return sprintf(buffer, "%u", (unsigned int)(d * 100) / HZ);
 }
 
 unsigned int libcfs_console_max_delay;
@@ -373,7 +376,7 @@ void libcfs_debug_dumplog(void)
 	add_wait_queue(&debug_ctlwq, &wait);
 
 	dumper = kthread_run(libcfs_debug_dumplog_thread,
-			     (void *)(long)current_pid(),
+			     (void *)(long)current->pid,
 			     "libcfs_debug_dumper");
 	set_current_state(TASK_INTERRUPTIBLE);
 	if (IS_ERR(dumper))
diff --git a/drivers/staging/lustre/lnet/libcfs/fail.c b/drivers/staging/lustre/lnet/libcfs/fail.c
index d3f1e866c6a7..bd86b3b5bc34 100644
--- a/drivers/staging/lustre/lnet/libcfs/fail.c
+++ b/drivers/staging/lustre/lnet/libcfs/fail.c
@@ -30,7 +30,11 @@
  * Lustre is a trademark of Oracle Corporation, Inc.
  */
 
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/module.h>
 #include <linux/libcfs/libcfs.h>
+#include <linux/random.h>
 
 unsigned long cfs_fail_loc;
 EXPORT_SYMBOL(cfs_fail_loc);
diff --git a/drivers/staging/lustre/lnet/libcfs/hash.c b/drivers/staging/lustre/lnet/libcfs/hash.c
index f7b3c9306456..48be66f0d654 100644
--- a/drivers/staging/lustre/lnet/libcfs/hash.c
+++ b/drivers/staging/lustre/lnet/libcfs/hash.c
@@ -105,8 +105,9 @@
  */
 #include <linux/seq_file.h>
 #include <linux/log2.h>
-
-#include <linux/libcfs/libcfs.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
+#include <linux/libcfs/libcfs_hash.h>
 
 #if CFS_HASH_DEBUG_LEVEL >= CFS_HASH_DEBUG_1
 static unsigned int warn_on_depth = 8;
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c b/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c
index 76291a350406..3d1cf457b286 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c
@@ -31,13 +31,58 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
+#include <linux/cpu.h>
+#include <linux/sched.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/cache.h>
+
+#include <linux/libcfs/libcfs_cpu.h>
+#include <linux/libcfs/libcfs_string.h>
 #include <linux/libcfs/libcfs.h>
 
 /** Global CPU partition table */
-struct cfs_cpt_table   *cfs_cpt_table __read_mostly;
-EXPORT_SYMBOL(cfs_cpt_table);
+struct cfs_cpt_table   *cfs_cpt_tab __read_mostly;
+EXPORT_SYMBOL(cfs_cpt_tab);
 
-#ifndef HAVE_LIBCFS_CPT
+/**
+ * modparam for setting number of partitions
+ *
+ *  0 : estimate best value based on cores or NUMA nodes
+ *  1 : disable multiple partitions
+ * >1 : specify number of partitions
+ */
+static int	cpu_npartitions;
+module_param(cpu_npartitions, int, 0444);
+MODULE_PARM_DESC(cpu_npartitions, "# of CPU partitions");
+
+/**
+ * modparam for setting CPU partitions patterns:
+ *
+ * i.e: "0[0,1,2,3] 1[4,5,6,7]", number before bracket is CPU partition ID,
+ *      number in bracket is processor ID (core or HT)
+ *
+ * i.e: "N 0[0,1] 1[2,3]" the first character 'N' means numbers in bracket
+ *       are NUMA node ID, number before bracket is CPU partition ID.
+ *
+ * i.e: "N", shortcut expression to create CPT from NUMA & CPU topology
+ *
+ * NB: If user specified cpu_pattern, cpu_npartitions will be ignored
+ */
+static char	*cpu_pattern = "N";
+module_param(cpu_pattern, charp, 0444);
+MODULE_PARM_DESC(cpu_pattern, "CPU partitions pattern");
+
+static struct cfs_cpt_data {
+	/* serialize hotplug etc */
+	spinlock_t		cpt_lock;
+	/* reserved for hotplug */
+	unsigned long		cpt_version;
+	/* mutex to protect cpt_cpumask */
+	struct mutex		cpt_mutex;
+	/* scratch buffer for set/unset_node */
+	cpumask_var_t		cpt_cpumask;
+} cpt_data;
 
 #define CFS_CPU_VERSION_MAGIC	   0xbabecafe
 
@@ -45,79 +90,221 @@ struct cfs_cpt_table *
 cfs_cpt_table_alloc(unsigned int ncpt)
 {
 	struct cfs_cpt_table *cptab;
+	int i;
 
-	if (ncpt != 1) {
-		CERROR("Can't support cpu partition number %d\n", ncpt);
+	cptab = kzalloc(sizeof(*cptab), GFP_NOFS);
+	if (!cptab)
 		return NULL;
-	}
 
-	cptab = kzalloc(sizeof(*cptab), GFP_NOFS);
-	if (cptab) {
-		cptab->ctb_version = CFS_CPU_VERSION_MAGIC;
-		node_set(0, cptab->ctb_nodemask);
-		cptab->ctb_nparts  = ncpt;
+	cptab->ctb_nparts = ncpt;
+
+	cptab->ctb_nodemask = kzalloc(sizeof(*cptab->ctb_nodemask),
+				      GFP_NOFS);
+	if (!zalloc_cpumask_var(&cptab->ctb_cpumask, GFP_NOFS) ||
+	    !cptab->ctb_nodemask)
+		goto failed;
+
+	cptab->ctb_cpu2cpt = kvmalloc_array(num_possible_cpus(),
+					    sizeof(cptab->ctb_cpu2cpt[0]),
+					    GFP_KERNEL);
+	if (!cptab->ctb_cpu2cpt)
+		goto failed;
+
+	memset(cptab->ctb_cpu2cpt, -1,
+	       num_possible_cpus() * sizeof(cptab->ctb_cpu2cpt[0]));
+
+	cptab->ctb_parts = kvmalloc_array(ncpt, sizeof(cptab->ctb_parts[0]),
+					  GFP_KERNEL);
+	if (!cptab->ctb_parts)
+		goto failed;
+
+	for (i = 0; i < ncpt; i++) {
+		struct cfs_cpu_partition *part = &cptab->ctb_parts[i];
+
+		part->cpt_nodemask = kzalloc(sizeof(*part->cpt_nodemask),
+					     GFP_NOFS);
+		if (!zalloc_cpumask_var(&part->cpt_cpumask, GFP_NOFS) ||
+		    !part->cpt_nodemask)
+			goto failed;
 	}
 
+	spin_lock(&cpt_data.cpt_lock);
+	/* Reserved for hotplug */
+	cptab->ctb_version = cpt_data.cpt_version;
+	spin_unlock(&cpt_data.cpt_lock);
+
 	return cptab;
+
+ failed:
+	cfs_cpt_table_free(cptab);
+	return NULL;
 }
 EXPORT_SYMBOL(cfs_cpt_table_alloc);
 
 void
 cfs_cpt_table_free(struct cfs_cpt_table *cptab)
 {
-	LASSERT(cptab->ctb_version == CFS_CPU_VERSION_MAGIC);
+	int i;
+
+	kvfree(cptab->ctb_cpu2cpt);
+
+	for (i = 0; cptab->ctb_parts && i < cptab->ctb_nparts; i++) {
+		struct cfs_cpu_partition *part = &cptab->ctb_parts[i];
+
+		kfree(part->cpt_nodemask);
+		free_cpumask_var(part->cpt_cpumask);
+	}
+
+	kvfree(cptab->ctb_parts);
+
+	kfree(cptab->ctb_nodemask);
+	free_cpumask_var(cptab->ctb_cpumask);
 
 	kfree(cptab);
 }
 EXPORT_SYMBOL(cfs_cpt_table_free);
 
-#ifdef CONFIG_SMP
 int
 cfs_cpt_table_print(struct cfs_cpt_table *cptab, char *buf, int len)
 {
-	int rc;
+	char *tmp = buf;
+	int rc = 0;
+	int i;
+	int j;
 
-	rc = snprintf(buf, len, "%d\t: %d\n", 0, 0);
-	len -= rc;
-	if (len <= 0)
-		return -EFBIG;
+	for (i = 0; i < cptab->ctb_nparts; i++) {
+		if (len > 0) {
+			rc = snprintf(tmp, len, "%d\t: ", i);
+			len -= rc;
+		}
 
-	return rc;
+		if (len <= 0) {
+			rc = -EFBIG;
+			goto out;
+		}
+
+		tmp += rc;
+		for_each_cpu(j, cptab->ctb_parts[i].cpt_cpumask) {
+			rc = snprintf(tmp, len, "%d ", j);
+			len -= rc;
+			if (len <= 0) {
+				rc = -EFBIG;
+				goto out;
+			}
+			tmp += rc;
+		}
+
+		*tmp = '\n';
+		tmp++;
+		len--;
+	}
+
+ out:
+	if (rc < 0)
+		return rc;
+
+	return tmp - buf;
 }
 EXPORT_SYMBOL(cfs_cpt_table_print);
-#endif /* CONFIG_SMP */
+
+static void
+cfs_node_to_cpumask(int node, cpumask_t *mask)
+{
+	const cpumask_t *tmp = cpumask_of_node(node);
+
+	if (tmp)
+		cpumask_copy(mask, tmp);
+	else
+		cpumask_clear(mask);
+}
 
 int
 cfs_cpt_number(struct cfs_cpt_table *cptab)
 {
-	return 1;
+	return cptab->ctb_nparts;
 }
 EXPORT_SYMBOL(cfs_cpt_number);
 
 int
 cfs_cpt_weight(struct cfs_cpt_table *cptab, int cpt)
 {
-	return 1;
+	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
+
+	return cpt == CFS_CPT_ANY ?
+	       cpumask_weight(cptab->ctb_cpumask) :
+	       cpumask_weight(cptab->ctb_parts[cpt].cpt_cpumask);
 }
 EXPORT_SYMBOL(cfs_cpt_weight);
 
 int
 cfs_cpt_online(struct cfs_cpt_table *cptab, int cpt)
 {
-	return 1;
+	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
+
+	return cpt == CFS_CPT_ANY ?
+	       cpumask_any_and(cptab->ctb_cpumask,
+			       cpu_online_mask) < nr_cpu_ids :
+	       cpumask_any_and(cptab->ctb_parts[cpt].cpt_cpumask,
+			       cpu_online_mask) < nr_cpu_ids;
 }
 EXPORT_SYMBOL(cfs_cpt_online);
 
+cpumask_var_t *
+cfs_cpt_cpumask(struct cfs_cpt_table *cptab, int cpt)
+{
+	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
+
+	return cpt == CFS_CPT_ANY ?
+	       &cptab->ctb_cpumask : &cptab->ctb_parts[cpt].cpt_cpumask;
+}
+EXPORT_SYMBOL(cfs_cpt_cpumask);
+
 nodemask_t *
 cfs_cpt_nodemask(struct cfs_cpt_table *cptab, int cpt)
 {
-	return &cptab->ctb_nodemask;
+	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
+
+	return cpt == CFS_CPT_ANY ?
+	       cptab->ctb_nodemask : cptab->ctb_parts[cpt].cpt_nodemask;
 }
 EXPORT_SYMBOL(cfs_cpt_nodemask);
 
 int
 cfs_cpt_set_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu)
 {
+	int node;
+
+	LASSERT(cpt >= 0 && cpt < cptab->ctb_nparts);
+
+	if (cpu < 0 || cpu >= nr_cpu_ids || !cpu_online(cpu)) {
+		CDEBUG(D_INFO, "CPU %d is invalid or it's offline\n", cpu);
+		return 0;
+	}
+
+	if (cptab->ctb_cpu2cpt[cpu] != -1) {
+		CDEBUG(D_INFO, "CPU %d is already in partition %d\n",
+		       cpu, cptab->ctb_cpu2cpt[cpu]);
+		return 0;
+	}
+
+	cptab->ctb_cpu2cpt[cpu] = cpt;
+
+	LASSERT(!cpumask_test_cpu(cpu, cptab->ctb_cpumask));
+	LASSERT(!cpumask_test_cpu(cpu, cptab->ctb_parts[cpt].cpt_cpumask));
+
+	cpumask_set_cpu(cpu, cptab->ctb_cpumask);
+	cpumask_set_cpu(cpu, cptab->ctb_parts[cpt].cpt_cpumask);
+
+	node = cpu_to_node(cpu);
+
+	/* first CPU of @node in this CPT table */
+	if (!node_isset(node, *cptab->ctb_nodemask))
+		node_set(node, *cptab->ctb_nodemask);
+
+	/* first CPU of @node in this partition */
+	if (!node_isset(node, *cptab->ctb_parts[cpt].cpt_nodemask))
+		node_set(node, *cptab->ctb_parts[cpt].cpt_nodemask);
+
 	return 1;
 }
 EXPORT_SYMBOL(cfs_cpt_set_cpu);
@@ -125,12 +312,80 @@ EXPORT_SYMBOL(cfs_cpt_set_cpu);
 void
 cfs_cpt_unset_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu)
 {
+	int node;
+	int i;
+
+	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
+
+	if (cpu < 0 || cpu >= nr_cpu_ids) {
+		CDEBUG(D_INFO, "Invalid CPU id %d\n", cpu);
+		return;
+	}
+
+	if (cpt == CFS_CPT_ANY) {
+		/* caller doesn't know the partition ID */
+		cpt = cptab->ctb_cpu2cpt[cpu];
+		if (cpt < 0) { /* not set in this CPT-table */
+			CDEBUG(D_INFO, "Try to unset cpu %d which is not in CPT-table %p\n",
+			       cpt, cptab);
+			return;
+		}
+
+	} else if (cpt != cptab->ctb_cpu2cpt[cpu]) {
+		CDEBUG(D_INFO,
+		       "CPU %d is not in cpu-partition %d\n", cpu, cpt);
+		return;
+	}
+
+	LASSERT(cpumask_test_cpu(cpu, cptab->ctb_parts[cpt].cpt_cpumask));
+	LASSERT(cpumask_test_cpu(cpu, cptab->ctb_cpumask));
+
+	cpumask_clear_cpu(cpu, cptab->ctb_parts[cpt].cpt_cpumask);
+	cpumask_clear_cpu(cpu, cptab->ctb_cpumask);
+	cptab->ctb_cpu2cpt[cpu] = -1;
+
+	node = cpu_to_node(cpu);
+
+	LASSERT(node_isset(node, *cptab->ctb_parts[cpt].cpt_nodemask));
+	LASSERT(node_isset(node, *cptab->ctb_nodemask));
+
+	for_each_cpu(i, cptab->ctb_parts[cpt].cpt_cpumask) {
+		/* this CPT has other CPU belonging to this node? */
+		if (cpu_to_node(i) == node)
+			break;
+	}
+
+	if (i >= nr_cpu_ids)
+		node_clear(node, *cptab->ctb_parts[cpt].cpt_nodemask);
+
+	for_each_cpu(i, cptab->ctb_cpumask) {
+		/* this CPT-table has other CPU belonging to this node? */
+		if (cpu_to_node(i) == node)
+			break;
+	}
+
+	if (i >= nr_cpu_ids)
+		node_clear(node, *cptab->ctb_nodemask);
 }
 EXPORT_SYMBOL(cfs_cpt_unset_cpu);
 
 int
 cfs_cpt_set_cpumask(struct cfs_cpt_table *cptab, int cpt, cpumask_t *mask)
 {
+	int i;
+
+	if (!cpumask_weight(mask) ||
+	    cpumask_any_and(mask, cpu_online_mask) >= nr_cpu_ids) {
+		CDEBUG(D_INFO, "No online CPU is found in the CPU mask for CPU partition %d\n",
+		       cpt);
+		return 0;
+	}
+
+	for_each_cpu(i, mask) {
+		if (!cfs_cpt_set_cpu(cptab, cpt, i))
+			return 0;
+	}
+
 	return 1;
 }
 EXPORT_SYMBOL(cfs_cpt_set_cpumask);
@@ -138,25 +393,65 @@ EXPORT_SYMBOL(cfs_cpt_set_cpumask);
 void
 cfs_cpt_unset_cpumask(struct cfs_cpt_table *cptab, int cpt, cpumask_t *mask)
 {
+	int i;
+
+	for_each_cpu(i, mask)
+		cfs_cpt_unset_cpu(cptab, cpt, i);
 }
 EXPORT_SYMBOL(cfs_cpt_unset_cpumask);
 
 int
 cfs_cpt_set_node(struct cfs_cpt_table *cptab, int cpt, int node)
 {
-	return 1;
+	int rc;
+
+	if (node < 0 || node >= MAX_NUMNODES) {
+		CDEBUG(D_INFO,
+		       "Invalid NUMA id %d for CPU partition %d\n", node, cpt);
+		return 0;
+	}
+
+	mutex_lock(&cpt_data.cpt_mutex);
+
+	cfs_node_to_cpumask(node, cpt_data.cpt_cpumask);
+
+	rc = cfs_cpt_set_cpumask(cptab, cpt, cpt_data.cpt_cpumask);
+
+	mutex_unlock(&cpt_data.cpt_mutex);
+
+	return rc;
 }
 EXPORT_SYMBOL(cfs_cpt_set_node);
 
 void
 cfs_cpt_unset_node(struct cfs_cpt_table *cptab, int cpt, int node)
 {
+	if (node < 0 || node >= MAX_NUMNODES) {
+		CDEBUG(D_INFO,
+		       "Invalid NUMA id %d for CPU partition %d\n", node, cpt);
+		return;
+	}
+
+	mutex_lock(&cpt_data.cpt_mutex);
+
+	cfs_node_to_cpumask(node, cpt_data.cpt_cpumask);
+
+	cfs_cpt_unset_cpumask(cptab, cpt, cpt_data.cpt_cpumask);
+
+	mutex_unlock(&cpt_data.cpt_mutex);
 }
 EXPORT_SYMBOL(cfs_cpt_unset_node);
 
 int
 cfs_cpt_set_nodemask(struct cfs_cpt_table *cptab, int cpt, nodemask_t *mask)
 {
+	int i;
+
+	for_each_node_mask(i, *mask) {
+		if (!cfs_cpt_set_node(cptab, cpt, i))
+			return 0;
+	}
+
 	return 1;
 }
 EXPORT_SYMBOL(cfs_cpt_set_nodemask);
@@ -164,65 +459,628 @@ EXPORT_SYMBOL(cfs_cpt_set_nodemask);
 void
 cfs_cpt_unset_nodemask(struct cfs_cpt_table *cptab, int cpt, nodemask_t *mask)
 {
+	int i;
+
+	for_each_node_mask(i, *mask)
+		cfs_cpt_unset_node(cptab, cpt, i);
 }
 EXPORT_SYMBOL(cfs_cpt_unset_nodemask);
 
 void
 cfs_cpt_clear(struct cfs_cpt_table *cptab, int cpt)
 {
+	int last;
+	int i;
+
+	if (cpt == CFS_CPT_ANY) {
+		last = cptab->ctb_nparts - 1;
+		cpt = 0;
+	} else {
+		last = cpt;
+	}
+
+	for (; cpt <= last; cpt++) {
+		for_each_cpu(i, cptab->ctb_parts[cpt].cpt_cpumask)
+			cfs_cpt_unset_cpu(cptab, cpt, i);
+	}
 }
 EXPORT_SYMBOL(cfs_cpt_clear);
 
 int
 cfs_cpt_spread_node(struct cfs_cpt_table *cptab, int cpt)
 {
+	nodemask_t *mask;
+	int weight;
+	int rotor;
+	int node;
+
+	/* convert CPU partition ID to HW node id */
+
+	if (cpt < 0 || cpt >= cptab->ctb_nparts) {
+		mask = cptab->ctb_nodemask;
+		rotor = cptab->ctb_spread_rotor++;
+	} else {
+		mask = cptab->ctb_parts[cpt].cpt_nodemask;
+		rotor = cptab->ctb_parts[cpt].cpt_spread_rotor++;
+	}
+
+	weight = nodes_weight(*mask);
+	LASSERT(weight > 0);
+
+	rotor %= weight;
+
+	for_each_node_mask(node, *mask) {
+		if (!rotor--)
+			return node;
+	}
+
+	LBUG();
 	return 0;
 }
 EXPORT_SYMBOL(cfs_cpt_spread_node);
 
-int
-cfs_cpu_ht_nsiblings(int cpu)
-{
-	return 1;
-}
-EXPORT_SYMBOL(cfs_cpu_ht_nsiblings);
-
 int
 cfs_cpt_current(struct cfs_cpt_table *cptab, int remap)
 {
-	return 0;
+	int cpu;
+	int cpt;
+
+	preempt_disable();
+	cpu = smp_processor_id();
+	cpt = cptab->ctb_cpu2cpt[cpu];
+
+	if (cpt < 0 && remap) {
+		/* don't return negative value for safety of upper layer,
+		 * instead we shadow the unknown cpu to a valid partition ID
+		 */
+		cpt = cpu % cptab->ctb_nparts;
+	}
+	preempt_enable();
+	return cpt;
 }
 EXPORT_SYMBOL(cfs_cpt_current);
 
 int
 cfs_cpt_of_cpu(struct cfs_cpt_table *cptab, int cpu)
 {
-	return 0;
+	LASSERT(cpu >= 0 && cpu < nr_cpu_ids);
+
+	return cptab->ctb_cpu2cpt[cpu];
 }
 EXPORT_SYMBOL(cfs_cpt_of_cpu);
 
 int
 cfs_cpt_bind(struct cfs_cpt_table *cptab, int cpt)
 {
+	cpumask_var_t *cpumask;
+	nodemask_t *nodemask;
+	int rc;
+	int i;
+
+	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
+
+	if (cpt == CFS_CPT_ANY) {
+		cpumask = &cptab->ctb_cpumask;
+		nodemask = cptab->ctb_nodemask;
+	} else {
+		cpumask = &cptab->ctb_parts[cpt].cpt_cpumask;
+		nodemask = cptab->ctb_parts[cpt].cpt_nodemask;
+	}
+
+	if (cpumask_any_and(*cpumask, cpu_online_mask) >= nr_cpu_ids) {
+		CERROR("No online CPU found in CPU partition %d, did someone do CPU hotplug on system? You might need to reload Lustre modules to keep system working well.\n",
+		       cpt);
+		return -EINVAL;
+	}
+
+	for_each_online_cpu(i) {
+		if (cpumask_test_cpu(i, *cpumask))
+			continue;
+
+		rc = set_cpus_allowed_ptr(current, *cpumask);
+		set_mems_allowed(*nodemask);
+		if (!rc)
+			schedule(); /* switch to allowed CPU */
+
+		return rc;
+	}
+
+	/* don't need to set affinity because all online CPUs are covered */
 	return 0;
 }
 EXPORT_SYMBOL(cfs_cpt_bind);
 
+/**
+ * Choose max to \a number CPUs from \a node and set them in \a cpt.
+ * We always prefer to choose CPU in the same core/socket.
+ */
+static int
+cfs_cpt_choose_ncpus(struct cfs_cpt_table *cptab, int cpt,
+		     cpumask_t *node, int number)
+{
+	cpumask_var_t socket;
+	cpumask_var_t core;
+	int rc = 0;
+	int cpu;
+
+	LASSERT(number > 0);
+
+	if (number >= cpumask_weight(node)) {
+		while (!cpumask_empty(node)) {
+			cpu = cpumask_first(node);
+
+			rc = cfs_cpt_set_cpu(cptab, cpt, cpu);
+			if (!rc)
+				return -EINVAL;
+			cpumask_clear_cpu(cpu, node);
+		}
+		return 0;
+	}
+
+	/*
+	 * Allocate scratch buffers
+	 * As we cannot initialize a cpumask_var_t, we need
+	 * to alloc both before we can risk trying to free either
+	 */
+	if (!zalloc_cpumask_var(&socket, GFP_NOFS))
+		rc = -ENOMEM;
+	if (!zalloc_cpumask_var(&core, GFP_NOFS))
+		rc = -ENOMEM;
+	if (rc)
+		goto out;
+
+	while (!cpumask_empty(node)) {
+		cpu = cpumask_first(node);
+
+		/* get cpumask for cores in the same socket */
+		cpumask_copy(socket, topology_core_cpumask(cpu));
+		cpumask_and(socket, socket, node);
+
+		LASSERT(!cpumask_empty(socket));
+
+		while (!cpumask_empty(socket)) {
+			int i;
+
+			/* get cpumask for hts in the same core */
+			cpumask_copy(core, topology_sibling_cpumask(cpu));
+			cpumask_and(core, core, node);
+
+			LASSERT(!cpumask_empty(core));
+
+			for_each_cpu(i, core) {
+				cpumask_clear_cpu(i, socket);
+				cpumask_clear_cpu(i, node);
+
+				rc = cfs_cpt_set_cpu(cptab, cpt, i);
+				if (!rc) {
+					rc = -EINVAL;
+					goto out;
+				}
+
+				if (!--number)
+					goto out;
+			}
+			cpu = cpumask_first(socket);
+		}
+	}
+
+out:
+	free_cpumask_var(socket);
+	free_cpumask_var(core);
+	return rc;
+}
+
+#define CPT_WEIGHT_MIN  4u
+
+static unsigned int
+cfs_cpt_num_estimate(void)
+{
+	unsigned int nnode = num_online_nodes();
+	unsigned int ncpu = num_online_cpus();
+	unsigned int ncpt;
+
+	if (ncpu <= CPT_WEIGHT_MIN) {
+		ncpt = 1;
+		goto out;
+	}
+
+	/* generate reasonable number of CPU partitions based on total number
+	 * of CPUs, Preferred N should be power2 and match this condition:
+	 * 2 * (N - 1)^2 < NCPUS <= 2 * N^2
+	 */
+	for (ncpt = 2; ncpu > 2 * ncpt * ncpt; ncpt <<= 1)
+		;
+
+	if (ncpt <= nnode) { /* fat numa system */
+		while (nnode > ncpt)
+			nnode >>= 1;
+
+	} else { /* ncpt > nnode */
+		while ((nnode << 1) <= ncpt)
+			nnode <<= 1;
+	}
+
+	ncpt = nnode;
+
+out:
+#if (BITS_PER_LONG == 32)
+	/* config many CPU partitions on 32-bit system could consume
+	 * too much memory
+	 */
+	ncpt = min(2U, ncpt);
+#endif
+	while (ncpu % ncpt)
+		ncpt--; /* worst case is 1 */
+
+	return ncpt;
+}
+
+static struct cfs_cpt_table *
+cfs_cpt_table_create(int ncpt)
+{
+	struct cfs_cpt_table *cptab = NULL;
+	cpumask_var_t mask;
+	int cpt = 0;
+	int num;
+	int rc;
+	int i;
+
+	rc = cfs_cpt_num_estimate();
+	if (ncpt <= 0)
+		ncpt = rc;
+
+	if (ncpt > num_online_cpus() || ncpt > 4 * rc) {
+		CWARN("CPU partition number %d is larger than suggested value (%d), your system may have performance issue or run out of memory while under pressure\n",
+		      ncpt, rc);
+	}
+
+	if (num_online_cpus() % ncpt) {
+		CERROR("CPU number %d is not multiple of cpu_npartition %d, please try different cpu_npartitions value or set pattern string by cpu_pattern=STRING\n",
+		       (int)num_online_cpus(), ncpt);
+		goto failed;
+	}
+
+	cptab = cfs_cpt_table_alloc(ncpt);
+	if (!cptab) {
+		CERROR("Failed to allocate CPU map(%d)\n", ncpt);
+		goto failed;
+	}
+
+	num = num_online_cpus() / ncpt;
+	if (!num) {
+		CERROR("CPU changed while setting CPU partition\n");
+		goto failed;
+	}
+
+	if (!zalloc_cpumask_var(&mask, GFP_NOFS)) {
+		CERROR("Failed to allocate scratch cpumask\n");
+		goto failed;
+	}
+
+	for_each_online_node(i) {
+		cfs_node_to_cpumask(i, mask);
+
+		while (!cpumask_empty(mask)) {
+			struct cfs_cpu_partition *part;
+			int n;
+
+			/*
+			 * Each emulated NUMA node has all allowed CPUs in
+			 * the mask.
+			 * End loop when all partitions have assigned CPUs.
+			 */
+			if (cpt == ncpt)
+				break;
+
+			part = &cptab->ctb_parts[cpt];
+
+			n = num - cpumask_weight(part->cpt_cpumask);
+			LASSERT(n > 0);
+
+			rc = cfs_cpt_choose_ncpus(cptab, cpt, mask, n);
+			if (rc < 0)
+				goto failed_mask;
+
+			LASSERT(num >= cpumask_weight(part->cpt_cpumask));
+			if (num == cpumask_weight(part->cpt_cpumask))
+				cpt++;
+		}
+	}
+
+	if (cpt != ncpt ||
+	    num != cpumask_weight(cptab->ctb_parts[ncpt - 1].cpt_cpumask)) {
+		CERROR("Expect %d(%d) CPU partitions but got %d(%d), CPU hotplug/unplug while setting?\n",
+		       cptab->ctb_nparts, num, cpt,
+		       cpumask_weight(cptab->ctb_parts[ncpt - 1].cpt_cpumask));
+		goto failed_mask;
+	}
+
+	free_cpumask_var(mask);
+
+	return cptab;
+
+ failed_mask:
+	free_cpumask_var(mask);
+ failed:
+	CERROR("Failed to setup CPU-partition-table with %d CPU-partitions, online HW nodes: %d, HW cpus: %d.\n",
+	       ncpt, num_online_nodes(), num_online_cpus());
+
+	if (cptab)
+		cfs_cpt_table_free(cptab);
+
+	return NULL;
+}
+
+static struct cfs_cpt_table *
+cfs_cpt_table_create_pattern(char *pattern)
+{
+	struct cfs_cpt_table *cptab;
+	char *str;
+	int node = 0;
+	int high;
+	int ncpt = 0;
+	int cpt;
+	int rc;
+	int c;
+	int i;
+
+	str = strim(pattern);
+	if (*str == 'n' || *str == 'N') {
+		pattern = str + 1;
+		if (*pattern != '\0') {
+			node = 1;
+		} else { /* shortcut to create CPT from NUMA & CPU topology */
+			node = -1;
+			ncpt = num_online_nodes();
+		}
+	}
+
+	if (!ncpt) { /* scanning bracket which is mark of partition */
+		for (str = pattern;; str++, ncpt++) {
+			str = strchr(str, '[');
+			if (!str)
+				break;
+		}
+	}
+
+	if (!ncpt ||
+	    (node && ncpt > num_online_nodes()) ||
+	    (!node && ncpt > num_online_cpus())) {
+		CERROR("Invalid pattern %s, or too many partitions %d\n",
+		       pattern, ncpt);
+		return NULL;
+	}
+
+	cptab = cfs_cpt_table_alloc(ncpt);
+	if (!cptab) {
+		CERROR("Failed to allocate cpu partition table\n");
+		return NULL;
+	}
+
+	if (node < 0) { /* shortcut to create CPT from NUMA & CPU topology */
+		cpt = 0;
+
+		for_each_online_node(i) {
+			if (cpt >= ncpt) {
+				CERROR("CPU changed while setting CPU partition table, %d/%d\n",
+				       cpt, ncpt);
+				goto failed;
+			}
+
+			rc = cfs_cpt_set_node(cptab, cpt++, i);
+			if (!rc)
+				goto failed;
+		}
+		return cptab;
+	}
+
+	high = node ? MAX_NUMNODES - 1 : nr_cpu_ids - 1;
+
+	for (str = strim(pattern), c = 0;; c++) {
+		struct cfs_range_expr *range;
+		struct cfs_expr_list *el;
+		char *bracket = strchr(str, '[');
+		int n;
+
+		if (!bracket) {
+			if (*str) {
+				CERROR("Invalid pattern %s\n", str);
+				goto failed;
+			}
+			if (c != ncpt) {
+				CERROR("expect %d partitions but found %d\n",
+				       ncpt, c);
+				goto failed;
+			}
+			break;
+		}
+
+		if (sscanf(str, "%d%n", &cpt, &n) < 1) {
+			CERROR("Invalid cpu pattern %s\n", str);
+			goto failed;
+		}
+
+		if (cpt < 0 || cpt >= ncpt) {
+			CERROR("Invalid partition id %d, total partitions %d\n",
+			       cpt, ncpt);
+			goto failed;
+		}
+
+		if (cfs_cpt_weight(cptab, cpt)) {
+			CERROR("Partition %d has already been set.\n", cpt);
+			goto failed;
+		}
+
+		str = strim(str + n);
+		if (str != bracket) {
+			CERROR("Invalid pattern %s\n", str);
+			goto failed;
+		}
+
+		bracket = strchr(str, ']');
+		if (!bracket) {
+			CERROR("missing right bracket for cpt %d, %s\n",
+			       cpt, str);
+			goto failed;
+		}
+
+		if (cfs_expr_list_parse(str, (bracket - str) + 1,
+					0, high, &el)) {
+			CERROR("Can't parse number range: %s\n", str);
+			goto failed;
+		}
+
+		list_for_each_entry(range, &el->el_exprs, re_link) {
+			for (i = range->re_lo; i <= range->re_hi; i++) {
+				if ((i - range->re_lo) % range->re_stride)
+					continue;
+
+				rc = node ? cfs_cpt_set_node(cptab, cpt, i) :
+					    cfs_cpt_set_cpu(cptab, cpt, i);
+				if (!rc) {
+					cfs_expr_list_free(el);
+					goto failed;
+				}
+			}
+		}
+
+		cfs_expr_list_free(el);
+
+		if (!cfs_cpt_online(cptab, cpt)) {
+			CERROR("No online CPU is found on partition %d\n", cpt);
+			goto failed;
+		}
+
+		str = strim(bracket + 1);
+	}
+
+	return cptab;
+
+ failed:
+	cfs_cpt_table_free(cptab);
+	return NULL;
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static enum cpuhp_state lustre_cpu_online;
+
+static void cfs_cpu_incr_cpt_version(void)
+{
+	spin_lock(&cpt_data.cpt_lock);
+	cpt_data.cpt_version++;
+	spin_unlock(&cpt_data.cpt_lock);
+}
+
+static int cfs_cpu_online(unsigned int cpu)
+{
+	cfs_cpu_incr_cpt_version();
+	return 0;
+}
+
+static int cfs_cpu_dead(unsigned int cpu)
+{
+	bool warn;
+
+	cfs_cpu_incr_cpt_version();
+
+	mutex_lock(&cpt_data.cpt_mutex);
+	/* if all HTs in a core are offline, it may break affinity */
+	cpumask_copy(cpt_data.cpt_cpumask, topology_sibling_cpumask(cpu));
+	warn = cpumask_any_and(cpt_data.cpt_cpumask,
+			       cpu_online_mask) >= nr_cpu_ids;
+	mutex_unlock(&cpt_data.cpt_mutex);
+	CDEBUG(warn ? D_WARNING : D_INFO,
+	       "Lustre: can't support CPU plug-out well now, performance and stability could be impacted [CPU %u]\n",
+	       cpu);
+	return 0;
+}
+#endif
+
 void
 cfs_cpu_fini(void)
 {
-	if (cfs_cpt_table) {
-		cfs_cpt_table_free(cfs_cpt_table);
-		cfs_cpt_table = NULL;
-	}
+	if (cfs_cpt_tab)
+		cfs_cpt_table_free(cfs_cpt_tab);
+
+#ifdef CONFIG_HOTPLUG_CPU
+	if (lustre_cpu_online > 0)
+		cpuhp_remove_state_nocalls(lustre_cpu_online);
+	cpuhp_remove_state_nocalls(CPUHP_LUSTRE_CFS_DEAD);
+#endif
+	free_cpumask_var(cpt_data.cpt_cpumask);
 }
 
 int
 cfs_cpu_init(void)
 {
-	cfs_cpt_table = cfs_cpt_table_alloc(1);
+	int ret = 0;
 
-	return cfs_cpt_table ? 0 : -1;
-}
+	LASSERT(!cfs_cpt_tab);
+
+	memset(&cpt_data, 0, sizeof(cpt_data));
+
+	if (!zalloc_cpumask_var(&cpt_data.cpt_cpumask, GFP_NOFS)) {
+		CERROR("Failed to allocate scratch buffer\n");
+		return -1;
+	}
+
+	spin_lock_init(&cpt_data.cpt_lock);
+	mutex_init(&cpt_data.cpt_mutex);
+
+#ifdef CONFIG_HOTPLUG_CPU
+	ret = cpuhp_setup_state_nocalls(CPUHP_LUSTRE_CFS_DEAD,
+					"staging/lustre/cfe:dead", NULL,
+					cfs_cpu_dead);
+	if (ret < 0)
+		goto failed;
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					"staging/lustre/cfe:online",
+					cfs_cpu_online, NULL);
+	if (ret < 0)
+		goto failed;
+	lustre_cpu_online = ret;
+#endif
+	ret = -EINVAL;
+
+	if (*cpu_pattern) {
+		char *cpu_pattern_dup = kstrdup(cpu_pattern, GFP_KERNEL);
+
+		if (!cpu_pattern_dup) {
+			CERROR("Failed to duplicate cpu_pattern\n");
+			goto failed;
+		}
 
-#endif /* HAVE_LIBCFS_CPT */
+		cfs_cpt_tab = cfs_cpt_table_create_pattern(cpu_pattern_dup);
+		kfree(cpu_pattern_dup);
+		if (!cfs_cpt_tab) {
+			CERROR("Failed to create cptab from pattern %s\n",
+			       cpu_pattern);
+			goto failed;
+		}
+
+	} else {
+		cfs_cpt_tab = cfs_cpt_table_create(cpu_npartitions);
+		if (!cfs_cpt_tab) {
+			CERROR("Failed to create ptable with npartitions %d\n",
+			       cpu_npartitions);
+			goto failed;
+		}
+	}
+
+	spin_lock(&cpt_data.cpt_lock);
+	if (cfs_cpt_tab->ctb_version != cpt_data.cpt_version) {
+		spin_unlock(&cpt_data.cpt_lock);
+		CERROR("CPU hotplug/unplug during setup\n");
+		goto failed;
+	}
+	spin_unlock(&cpt_data.cpt_lock);
+
+	LCONSOLE(0, "HW nodes: %d, HW CPU cores: %d, npartitions: %d\n",
+		 num_online_nodes(), num_online_cpus(),
+		 cfs_cpt_number(cfs_cpt_tab));
+	return 0;
+
+ failed:
+	cfs_cpu_fini();
+	return ret;
+}
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c b/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c
index 670ad5a34224..223505c37545 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c
@@ -28,7 +28,10 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
+#include <linux/module.h>
+#include <linux/slab.h>
 #include <linux/libcfs/libcfs.h>
+#include <linux/libcfs/libcfs_cpu.h>
 
 /** destroy cpu-partition lock, see libcfs_private.h for more detail */
 void
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c b/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c
index 7faed94994ea..2d533be9bb30 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c
@@ -29,7 +29,11 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include <linux/libcfs/libcfs.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/libcfs/libcfs_cpu.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
 
 struct cfs_var_array {
 	unsigned int		va_count;	/* # of buffers */
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_string.c b/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
index 442889a3d729..e1fb1263e3ae 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
@@ -37,7 +37,13 @@
  * Author: Nathan Rutman <nathan.rutman@sun.com>
  */
 
+#include <linux/ctype.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+#include <linux/module.h>
+#include <linux/slab.h>
 #include <linux/libcfs/libcfs.h>
+#include <linux/libcfs/libcfs_string.h>
 
 /* Convert a text string to a bitmask */
 int cfs_str2mask(const char *str, const char *(*bit2str)(int bit),
diff --git a/drivers/staging/lustre/lnet/libcfs/linux-crypto-adler.c b/drivers/staging/lustre/lnet/libcfs/linux-crypto-adler.c
new file mode 100644
index 000000000000..db81ed527452
--- /dev/null
+++ b/drivers/staging/lustre/lnet/libcfs/linux-crypto-adler.c
@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: GPL-2.0
+/* GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please  visit http://www.xyratex.com/contact if you need additional
+ * information or have any questions.
+ *
+ * GPL HEADER END
+ */
+
+/*
+ * Copyright 2012 Xyratex Technology Limited
+ */
+
+/*
+ * This is crypto api shash wrappers to zlib_adler32.
+ */
+
+#include <linux/module.h>
+#include <linux/zutil.h>
+#include <crypto/internal/hash.h>
+#include "linux-crypto.h"
+
+#define CHKSUM_BLOCK_SIZE	1
+#define CHKSUM_DIGEST_SIZE	4
+
+static int adler32_cra_init(struct crypto_tfm *tfm)
+{
+	u32 *key = crypto_tfm_ctx(tfm);
+
+	*key = 1;
+
+	return 0;
+}
+
+static int adler32_setkey(struct crypto_shash *hash, const u8 *key,
+			  unsigned int keylen)
+{
+	u32 *mctx = crypto_shash_ctx(hash);
+
+	if (keylen != sizeof(u32)) {
+		crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+	*mctx = *(u32 *)key;
+	return 0;
+}
+
+static int adler32_init(struct shash_desc *desc)
+{
+	u32 *mctx = crypto_shash_ctx(desc->tfm);
+	u32 *cksump = shash_desc_ctx(desc);
+
+	*cksump = *mctx;
+
+	return 0;
+}
+
+static int adler32_update(struct shash_desc *desc, const u8 *data,
+			  unsigned int len)
+{
+	u32 *cksump = shash_desc_ctx(desc);
+
+	*cksump = zlib_adler32(*cksump, data, len);
+	return 0;
+}
+
+static int __adler32_finup(u32 *cksump, const u8 *data, unsigned int len,
+			   u8 *out)
+{
+	*(u32 *)out = zlib_adler32(*cksump, data, len);
+	return 0;
+}
+
+static int adler32_finup(struct shash_desc *desc, const u8 *data,
+			 unsigned int len, u8 *out)
+{
+	return __adler32_finup(shash_desc_ctx(desc), data, len, out);
+}
+
+static int adler32_final(struct shash_desc *desc, u8 *out)
+{
+	u32 *cksump = shash_desc_ctx(desc);
+
+	*(u32 *)out = *cksump;
+	return 0;
+}
+
+static int adler32_digest(struct shash_desc *desc, const u8 *data,
+			  unsigned int len, u8 *out)
+{
+	return __adler32_finup(crypto_shash_ctx(desc->tfm), data, len,
+				    out);
+}
+
+static struct shash_alg alg = {
+	.setkey		= adler32_setkey,
+	.init		= adler32_init,
+	.update		= adler32_update,
+	.final		= adler32_final,
+	.finup		= adler32_finup,
+	.digest		= adler32_digest,
+	.descsize	= sizeof(u32),
+	.digestsize	= CHKSUM_DIGEST_SIZE,
+	.base		= {
+		.cra_name		= "adler32",
+		.cra_driver_name	= "adler32-zlib",
+		.cra_priority		= 100,
+		.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
+		.cra_blocksize		= CHKSUM_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(u32),
+		.cra_module		= THIS_MODULE,
+		.cra_init		= adler32_cra_init,
+	}
+};
+
+int cfs_crypto_adler32_register(void)
+{
+	return crypto_register_shash(&alg);
+}
+
+void cfs_crypto_adler32_unregister(void)
+{
+	crypto_unregister_shash(&alg);
+}
diff --git a/drivers/staging/lustre/lnet/libcfs/linux-crypto.c b/drivers/staging/lustre/lnet/libcfs/linux-crypto.c
new file mode 100644
index 000000000000..21ff9bf6da47
--- /dev/null
+++ b/drivers/staging/lustre/lnet/libcfs/linux-crypto.c
@@ -0,0 +1,447 @@
+// SPDX-License-Identifier: GPL-2.0
+/* GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please  visit http://www.xyratex.com/contact if you need additional
+ * information or have any questions.
+ *
+ * GPL HEADER END
+ */
+
+/*
+ * Copyright 2012 Xyratex Technology Limited
+ *
+ * Copyright (c) 2012, Intel Corporation.
+ */
+
+#include <crypto/hash.h>
+#include <linux/scatterlist.h>
+#include <linux/highmem.h>
+#include <linux/module.h>
+#include <linux/libcfs/libcfs_crypto.h>
+#include <linux/libcfs/libcfs.h>
+#include "linux-crypto.h"
+
+/**
+ *  Array of hash algorithm speed in MByte per second
+ */
+static int cfs_crypto_hash_speeds[CFS_HASH_ALG_MAX];
+
+/**
+ * Initialize the state descriptor for the specified hash algorithm.
+ *
+ * An internal routine to allocate the hash-specific state in \a req for
+ * use with cfs_crypto_hash_digest() to compute the hash of a single message,
+ * though possibly in multiple chunks.  The descriptor internal state should
+ * be freed with cfs_crypto_hash_final().
+ *
+ * \param[in]	  hash_alg	hash algorithm id (CFS_HASH_ALG_*)
+ * \param[out]	  type		pointer to the hash description in hash_types[]
+ *				array
+ * \param[in,out] req		hash state descriptor to be initialized
+ * \param[in]	  key		initial hash value/state, NULL to use default
+ *				value
+ * \param[in]	  key_len	length of \a key
+ *
+ * \retval			0 on success
+ * \retval			negative errno on failure
+ */
+static int cfs_crypto_hash_alloc(enum cfs_crypto_hash_alg hash_alg,
+				 const struct cfs_crypto_hash_type **type,
+				 struct ahash_request **req,
+				 unsigned char *key,
+				 unsigned int key_len)
+{
+	struct crypto_ahash *tfm;
+	int err = 0;
+
+	*type = cfs_crypto_hash_type(hash_alg);
+
+	if (!*type) {
+		CWARN("Unsupported hash algorithm id = %d, max id is %d\n",
+		      hash_alg, CFS_HASH_ALG_MAX);
+		return -EINVAL;
+	}
+	tfm = crypto_alloc_ahash((*type)->cht_name, 0, CRYPTO_ALG_ASYNC);
+
+	if (IS_ERR(tfm)) {
+		CDEBUG(D_INFO, "Failed to alloc crypto hash %s\n",
+		       (*type)->cht_name);
+		return PTR_ERR(tfm);
+	}
+
+	*req = ahash_request_alloc(tfm, GFP_KERNEL);
+	if (!*req) {
+		CDEBUG(D_INFO, "Failed to alloc ahash_request for %s\n",
+		       (*type)->cht_name);
+		crypto_free_ahash(tfm);
+		return -ENOMEM;
+	}
+
+	ahash_request_set_callback(*req, 0, NULL, NULL);
+
+	if (key)
+		err = crypto_ahash_setkey(tfm, key, key_len);
+	else if ((*type)->cht_key)
+		err = crypto_ahash_setkey(tfm,
+					  (unsigned char *)&((*type)->cht_key),
+					  (*type)->cht_size);
+
+	if (err) {
+		ahash_request_free(*req);
+		crypto_free_ahash(tfm);
+		return err;
+	}
+
+	CDEBUG(D_INFO, "Using crypto hash: %s (%s) speed %d MB/s\n",
+	       crypto_ahash_alg_name(tfm), crypto_ahash_driver_name(tfm),
+	       cfs_crypto_hash_speeds[hash_alg]);
+
+	err = crypto_ahash_init(*req);
+	if (err) {
+		ahash_request_free(*req);
+		crypto_free_ahash(tfm);
+	}
+	return err;
+}
+
+/**
+ * Calculate hash digest for the passed buffer.
+ *
+ * This should be used when computing the hash on a single contiguous buffer.
+ * It combines the hash initialization, computation, and cleanup.
+ *
+ * \param[in]	  hash_alg	id of hash algorithm (CFS_HASH_ALG_*)
+ * \param[in]	  buf		data buffer on which to compute hash
+ * \param[in]	  buf_len	length of \a buf in bytes
+ * \param[in]	  key		initial value/state for algorithm,
+ *				if \a key = NULL use default initial value
+ * \param[in]	  key_len	length of \a key in bytes
+ * \param[out]	  hash		pointer to computed hash value,
+ *				if \a hash = NULL then \a hash_len is to digest
+ *				size in bytes, retval -ENOSPC
+ * \param[in,out] hash_len	size of \a hash buffer
+ *
+ * \retval -EINVAL		\a buf, \a buf_len, \a hash_len,
+ *				\a hash_alg invalid
+ * \retval -ENOENT		\a hash_alg is unsupported
+ * \retval -ENOSPC		\a hash is NULL, or \a hash_len less than
+ *				digest size
+ * \retval			0 for success
+ * \retval			negative errno for other errors from lower
+ *				layers.
+ */
+int cfs_crypto_hash_digest(enum cfs_crypto_hash_alg hash_alg,
+			   const void *buf, unsigned int buf_len,
+			   unsigned char *key, unsigned int key_len,
+			   unsigned char *hash, unsigned int *hash_len)
+{
+	struct scatterlist sl;
+	struct ahash_request *req;
+	int err;
+	const struct cfs_crypto_hash_type *type;
+
+	if (!buf || !buf_len || !hash_len)
+		return -EINVAL;
+
+	err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len);
+	if (err)
+		return err;
+
+	if (!hash || *hash_len < type->cht_size) {
+		*hash_len = type->cht_size;
+		crypto_free_ahash(crypto_ahash_reqtfm(req));
+		ahash_request_free(req);
+		return -ENOSPC;
+	}
+	sg_init_one(&sl, buf, buf_len);
+
+	ahash_request_set_crypt(req, &sl, hash, sl.length);
+	err = crypto_ahash_digest(req);
+	crypto_free_ahash(crypto_ahash_reqtfm(req));
+	ahash_request_free(req);
+
+	return err;
+}
+EXPORT_SYMBOL(cfs_crypto_hash_digest);
+
+/**
+ * Allocate and initialize descriptor for hash algorithm.
+ *
+ * This should be used to initialize a hash descriptor for multiple calls
+ * to a single hash function when computing the hash across multiple
+ * separate buffers or pages using cfs_crypto_hash_update{,_page}().
+ *
+ * The hash descriptor should be freed with cfs_crypto_hash_final().
+ *
+ * \param[in] hash_alg	algorithm id (CFS_HASH_ALG_*)
+ * \param[in] key	initial value/state for algorithm, if \a key = NULL
+ *			use default initial value
+ * \param[in] key_len	length of \a key in bytes
+ *
+ * \retval		pointer to descriptor of hash instance
+ * \retval		ERR_PTR(errno) in case of error
+ */
+struct ahash_request *
+cfs_crypto_hash_init(enum cfs_crypto_hash_alg hash_alg,
+		     unsigned char *key, unsigned int key_len)
+{
+	struct ahash_request *req;
+	int err;
+	const struct cfs_crypto_hash_type *type;
+
+	err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len);
+
+	if (err)
+		return ERR_PTR(err);
+	return req;
+}
+EXPORT_SYMBOL(cfs_crypto_hash_init);
+
+/**
+ * Update hash digest computed on data within the given \a page
+ *
+ * \param[in] hreq	hash state descriptor
+ * \param[in] page	data page on which to compute the hash
+ * \param[in] offset	offset within \a page at which to start hash
+ * \param[in] len	length of data on which to compute hash
+ *
+ * \retval		0 for success
+ * \retval		negative errno on failure
+ */
+int cfs_crypto_hash_update_page(struct ahash_request *req,
+				struct page *page, unsigned int offset,
+				unsigned int len)
+{
+	struct scatterlist sl;
+
+	sg_init_table(&sl, 1);
+	sg_set_page(&sl, page, len, offset & ~PAGE_MASK);
+
+	ahash_request_set_crypt(req, &sl, NULL, sl.length);
+	return crypto_ahash_update(req);
+}
+EXPORT_SYMBOL(cfs_crypto_hash_update_page);
+
+/**
+ * Update hash digest computed on the specified data
+ *
+ * \param[in] req	hash state descriptor
+ * \param[in] buf	data buffer on which to compute the hash
+ * \param[in] buf_len	length of \buf on which to compute hash
+ *
+ * \retval		0 for success
+ * \retval		negative errno on failure
+ */
+int cfs_crypto_hash_update(struct ahash_request *req,
+			   const void *buf, unsigned int buf_len)
+{
+	struct scatterlist sl;
+
+	sg_init_one(&sl, buf, buf_len);
+
+	ahash_request_set_crypt(req, &sl, NULL, sl.length);
+	return crypto_ahash_update(req);
+}
+EXPORT_SYMBOL(cfs_crypto_hash_update);
+
+/**
+ * Finish hash calculation, copy hash digest to buffer, clean up hash descriptor
+ *
+ * \param[in]	  req		hash descriptor
+ * \param[out]	  hash		pointer to hash buffer to store hash digest
+ * \param[in,out] hash_len	pointer to hash buffer size, if \a req = NULL
+ *				only free \a req instead of computing the hash
+ *
+ * \retval	0 for success
+ * \retval	-EOVERFLOW if hash_len is too small for the hash digest
+ * \retval	negative errno for other errors from lower layers
+ */
+int cfs_crypto_hash_final(struct ahash_request *req,
+			  unsigned char *hash, unsigned int *hash_len)
+{
+	int err;
+	int size = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
+
+	if (!hash || !hash_len) {
+		err = 0;
+		goto free_ahash;
+	}
+	if (*hash_len < size) {
+		err = -EOVERFLOW;
+		goto free_ahash;
+	}
+
+	ahash_request_set_crypt(req, NULL, hash, 0);
+	err = crypto_ahash_final(req);
+	if (!err)
+		*hash_len = size;
+free_ahash:
+	crypto_free_ahash(crypto_ahash_reqtfm(req));
+	ahash_request_free(req);
+	return err;
+}
+EXPORT_SYMBOL(cfs_crypto_hash_final);
+
+/**
+ * Compute the speed of specified hash function
+ *
+ * Run a speed test on the given hash algorithm on buffer of the given size.
+ * The speed is stored internally in the cfs_crypto_hash_speeds[] array, and
+ * is available through the cfs_crypto_hash_speed() function.
+ *
+ * \param[in] hash_alg	hash algorithm id (CFS_HASH_ALG_*)
+ * \param[in] buf	data buffer on which to compute the hash
+ * \param[in] buf_len	length of \buf on which to compute hash
+ */
+static void cfs_crypto_performance_test(enum cfs_crypto_hash_alg hash_alg)
+{
+	int buf_len = max(PAGE_SIZE, 1048576UL);
+	void *buf;
+	unsigned long start, end;
+	int bcount, err = 0;
+	struct page *page;
+	unsigned char hash[CFS_CRYPTO_HASH_DIGESTSIZE_MAX];
+	unsigned int hash_len = sizeof(hash);
+
+	page = alloc_page(GFP_KERNEL);
+	if (!page) {
+		err = -ENOMEM;
+		goto out_err;
+	}
+
+	buf = kmap(page);
+	memset(buf, 0xAD, PAGE_SIZE);
+	kunmap(page);
+
+	for (start = jiffies, end = start + msecs_to_jiffies(MSEC_PER_SEC),
+	     bcount = 0; time_before(jiffies, end); bcount++) {
+		struct ahash_request *hdesc;
+		int i;
+
+		hdesc = cfs_crypto_hash_init(hash_alg, NULL, 0);
+		if (IS_ERR(hdesc)) {
+			err = PTR_ERR(hdesc);
+			break;
+		}
+
+		for (i = 0; i < buf_len / PAGE_SIZE; i++) {
+			err = cfs_crypto_hash_update_page(hdesc, page, 0,
+							  PAGE_SIZE);
+			if (err)
+				break;
+		}
+
+		err = cfs_crypto_hash_final(hdesc, hash, &hash_len);
+		if (err)
+			break;
+	}
+	end = jiffies;
+	__free_page(page);
+out_err:
+	if (err) {
+		cfs_crypto_hash_speeds[hash_alg] = err;
+		CDEBUG(D_INFO, "Crypto hash algorithm %s test error: rc = %d\n",
+		       cfs_crypto_hash_name(hash_alg), err);
+	} else {
+		unsigned long tmp;
+
+		tmp = ((bcount * buf_len / jiffies_to_msecs(end - start)) *
+		       1000) / (1024 * 1024);
+		cfs_crypto_hash_speeds[hash_alg] = (int)tmp;
+		CDEBUG(D_CONFIG, "Crypto hash algorithm %s speed = %d MB/s\n",
+		       cfs_crypto_hash_name(hash_alg),
+		       cfs_crypto_hash_speeds[hash_alg]);
+	}
+}
+
+/**
+ * hash speed in Mbytes per second for valid hash algorithm
+ *
+ * Return the performance of the specified \a hash_alg that was previously
+ * computed using cfs_crypto_performance_test().
+ *
+ * \param[in] hash_alg	hash algorithm id (CFS_HASH_ALG_*)
+ *
+ * \retval		positive speed of the hash function in MB/s
+ * \retval		-ENOENT if \a hash_alg is unsupported
+ * \retval		negative errno if \a hash_alg speed is unavailable
+ */
+int cfs_crypto_hash_speed(enum cfs_crypto_hash_alg hash_alg)
+{
+	if (hash_alg < CFS_HASH_ALG_MAX)
+		return cfs_crypto_hash_speeds[hash_alg];
+	return -ENOENT;
+}
+EXPORT_SYMBOL(cfs_crypto_hash_speed);
+
+/**
+ * Run the performance test for all hash algorithms.
+ *
+ * Run the cfs_crypto_performance_test() benchmark for all of the available
+ * hash functions using a 1MB buffer size.  This is a reasonable buffer size
+ * for Lustre RPCs, even if the actual RPC size is larger or smaller.
+ *
+ * Since the setup cost and computation speed of various hash algorithms is
+ * a function of the buffer size (and possibly internal contention of offload
+ * engines), this speed only represents an estimate of the actual speed under
+ * actual usage, but is reasonable for comparing available algorithms.
+ *
+ * The actual speeds are available via cfs_crypto_hash_speed() for later
+ * comparison.
+ *
+ * \retval	0 on success
+ * \retval	-ENOMEM if no memory is available for test buffer
+ */
+static int cfs_crypto_test_hashes(void)
+{
+	enum cfs_crypto_hash_alg hash_alg;
+
+	for (hash_alg = 0; hash_alg < CFS_HASH_ALG_MAX; hash_alg++)
+		cfs_crypto_performance_test(hash_alg);
+
+	return 0;
+}
+
+static int adler32;
+
+/**
+ * Register available hash functions
+ *
+ * \retval	0
+ */
+int cfs_crypto_register(void)
+{
+	request_module("crc32c");
+
+	if (cfs_crypto_adler32_register() == 0)
+		adler32 = 1;
+
+	/* check all algorithms and do performance test */
+	cfs_crypto_test_hashes();
+	return 0;
+}
+
+/**
+ * Unregister previously registered hash functions
+ */
+void cfs_crypto_unregister(void)
+{
+	if (adler32)
+		cfs_crypto_adler32_unregister();
+	adler32 = 0;
+}
diff --git a/drivers/staging/lustre/lnet/libcfs/linux-crypto.h b/drivers/staging/lustre/lnet/libcfs/linux-crypto.h
new file mode 100644
index 000000000000..5616e9ea1450
--- /dev/null
+++ b/drivers/staging/lustre/lnet/libcfs/linux-crypto.h
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please  visit http://www.xyratex.com/contact if you need additional
+ * information or have any questions.
+ *
+ * GPL HEADER END
+ */
+
+/**
+ * Functions for start/stop shash adler32 algorithm.
+ */
+int cfs_crypto_adler32_register(void);
+void cfs_crypto_adler32_unregister(void);
diff --git a/drivers/staging/lustre/lnet/libcfs/linux-debug.c b/drivers/staging/lustre/lnet/libcfs/linux-debug.c
new file mode 100644
index 000000000000..15ab849374c2
--- /dev/null
+++ b/drivers/staging/lustre/lnet/libcfs/linux-debug.c
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * Copyright (c) 2012, Intel Corporation.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ * Lustre is a trademark of Sun Microsystems, Inc.
+ *
+ * libcfs/libcfs/linux/linux-debug.c
+ *
+ * Author: Phil Schwan <phil@clusterfs.com>
+ */
+
+#include <linux/module.h>
+#include <linux/kmod.h>
+#include <linux/notifier.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/string.h>
+#include <linux/stat.h>
+#include <linux/errno.h>
+#include <linux/unistd.h>
+#include <linux/interrupt.h>
+#include <linux/completion.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+
+# define DEBUG_SUBSYSTEM S_LNET
+
+#include "tracefile.h"
+
+#include <linux/kallsyms.h>
+
+char lnet_debug_log_upcall[1024] = "/usr/lib/lustre/lnet_debug_log_upcall";
+
+/**
+ * Upcall function once a Lustre log has been dumped.
+ *
+ * \param file  path of the dumped log
+ */
+void libcfs_run_debug_log_upcall(char *file)
+{
+	char *argv[3];
+	int rc;
+	static const char * const envp[] = {
+		"HOME=/",
+		"PATH=/sbin:/bin:/usr/sbin:/usr/bin",
+		NULL
+	};
+
+	argv[0] = lnet_debug_log_upcall;
+
+	LASSERTF(file, "called on a null filename\n");
+	argv[1] = file; /* only need to pass the path of the file */
+
+	argv[2] = NULL;
+
+	rc = call_usermodehelper(argv[0], argv, (char **)envp, 1);
+	if (rc < 0 && rc != -ENOENT) {
+		CERROR("Error %d invoking LNET debug log upcall %s %s; check /sys/kernel/debug/lnet/debug_log_upcall\n",
+		       rc, argv[0], argv[1]);
+	} else {
+		CDEBUG(D_HA, "Invoked LNET debug log upcall %s %s\n",
+		       argv[0], argv[1]);
+	}
+}
+
+/* coverity[+kill] */
+void __noreturn lbug_with_loc(struct libcfs_debug_msg_data *msgdata)
+{
+	libcfs_catastrophe = 1;
+	libcfs_debug_msg(msgdata, "LBUG\n");
+
+	if (in_interrupt()) {
+		panic("LBUG in interrupt.\n");
+		/* not reached */
+	}
+
+	dump_stack();
+	if (!libcfs_panic_on_lbug)
+		libcfs_debug_dumplog();
+	if (libcfs_panic_on_lbug)
+		panic("LBUG");
+	set_current_state(TASK_UNINTERRUPTIBLE);
+	while (1)
+		schedule();
+}
+EXPORT_SYMBOL(lbug_with_loc);
+
+static int panic_notifier(struct notifier_block *self, unsigned long unused1,
+			  void *unused2)
+{
+	if (libcfs_panic_in_progress)
+		return 0;
+
+	libcfs_panic_in_progress = 1;
+	mb();
+
+	return 0;
+}
+
+static struct notifier_block libcfs_panic_notifier = {
+	.notifier_call	= panic_notifier,
+	.next		= NULL,
+	.priority	= 10000,
+};
+
+void libcfs_register_panic_notifier(void)
+{
+	atomic_notifier_chain_register(&panic_notifier_list,
+				       &libcfs_panic_notifier);
+}
+
+void libcfs_unregister_panic_notifier(void)
+{
+	atomic_notifier_chain_unregister(&panic_notifier_list,
+					 &libcfs_panic_notifier);
+}
diff --git a/drivers/staging/lustre/lnet/libcfs/linux-tracefile.c b/drivers/staging/lustre/lnet/libcfs/linux-tracefile.c
new file mode 100644
index 000000000000..347138409eba
--- /dev/null
+++ b/drivers/staging/lustre/lnet/libcfs/linux-tracefile.c
@@ -0,0 +1,258 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * Copyright (c) 2012, Intel Corporation.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ * Lustre is a trademark of Sun Microsystems, Inc.
+ */
+
+#define DEBUG_SUBSYSTEM S_LNET
+#define LUSTRE_TRACEFILE_PRIVATE
+
+#include <linux/slab.h>
+#include <linux/mm.h>
+#include "tracefile.h"
+
+/* percents to share the total debug memory for each type */
+static unsigned int pages_factor[CFS_TCD_TYPE_MAX] = {
+	80,  /* 80% pages for CFS_TCD_TYPE_PROC */
+	10,  /* 10% pages for CFS_TCD_TYPE_SOFTIRQ */
+	10   /* 10% pages for CFS_TCD_TYPE_IRQ */
+};
+
+char *cfs_trace_console_buffers[NR_CPUS][CFS_TCD_TYPE_MAX];
+
+static DECLARE_RWSEM(cfs_tracefile_sem);
+
+int cfs_tracefile_init_arch(void)
+{
+	int i;
+	int j;
+	struct cfs_trace_cpu_data *tcd;
+
+	/* initialize trace_data */
+	memset(cfs_trace_data, 0, sizeof(cfs_trace_data));
+	for (i = 0; i < CFS_TCD_TYPE_MAX; i++) {
+		cfs_trace_data[i] =
+			kmalloc_array(num_possible_cpus(),
+				      sizeof(union cfs_trace_data_union),
+				      GFP_KERNEL);
+		if (!cfs_trace_data[i])
+			goto out;
+	}
+
+	/* arch related info initialized */
+	cfs_tcd_for_each(tcd, i, j) {
+		spin_lock_init(&tcd->tcd_lock);
+		tcd->tcd_pages_factor = pages_factor[i];
+		tcd->tcd_type = i;
+		tcd->tcd_cpu = j;
+	}
+
+	for (i = 0; i < num_possible_cpus(); i++)
+		for (j = 0; j < 3; j++) {
+			cfs_trace_console_buffers[i][j] =
+				kmalloc(CFS_TRACE_CONSOLE_BUFFER_SIZE,
+					GFP_KERNEL);
+
+			if (!cfs_trace_console_buffers[i][j])
+				goto out;
+		}
+
+	return 0;
+
+out:
+	cfs_tracefile_fini_arch();
+	pr_err("lnet: Not enough memory\n");
+	return -ENOMEM;
+}
+
+void cfs_tracefile_fini_arch(void)
+{
+	int i;
+	int j;
+
+	for (i = 0; i < num_possible_cpus(); i++)
+		for (j = 0; j < 3; j++) {
+			kfree(cfs_trace_console_buffers[i][j]);
+			cfs_trace_console_buffers[i][j] = NULL;
+		}
+
+	for (i = 0; cfs_trace_data[i]; i++) {
+		kfree(cfs_trace_data[i]);
+		cfs_trace_data[i] = NULL;
+	}
+}
+
+void cfs_tracefile_read_lock(void)
+{
+	down_read(&cfs_tracefile_sem);
+}
+
+void cfs_tracefile_read_unlock(void)
+{
+	up_read(&cfs_tracefile_sem);
+}
+
+void cfs_tracefile_write_lock(void)
+{
+	down_write(&cfs_tracefile_sem);
+}
+
+void cfs_tracefile_write_unlock(void)
+{
+	up_write(&cfs_tracefile_sem);
+}
+
+enum cfs_trace_buf_type cfs_trace_buf_idx_get(void)
+{
+	if (in_irq())
+		return CFS_TCD_TYPE_IRQ;
+	if (in_softirq())
+		return CFS_TCD_TYPE_SOFTIRQ;
+	return CFS_TCD_TYPE_PROC;
+}
+
+/*
+ * The walking argument indicates the locking comes from all tcd types
+ * iterator and we must lock it and dissable local irqs to avoid deadlocks
+ * with other interrupt locks that might be happening. See LU-1311
+ * for details.
+ */
+int cfs_trace_lock_tcd(struct cfs_trace_cpu_data *tcd, int walking)
+	__acquires(&tcd->tc_lock)
+{
+	__LASSERT(tcd->tcd_type < CFS_TCD_TYPE_MAX);
+	if (tcd->tcd_type == CFS_TCD_TYPE_IRQ)
+		spin_lock_irqsave(&tcd->tcd_lock, tcd->tcd_lock_flags);
+	else if (tcd->tcd_type == CFS_TCD_TYPE_SOFTIRQ)
+		spin_lock_bh(&tcd->tcd_lock);
+	else if (unlikely(walking))
+		spin_lock_irq(&tcd->tcd_lock);
+	else
+		spin_lock(&tcd->tcd_lock);
+	return 1;
+}
+
+void cfs_trace_unlock_tcd(struct cfs_trace_cpu_data *tcd, int walking)
+	__releases(&tcd->tcd_lock)
+{
+	__LASSERT(tcd->tcd_type < CFS_TCD_TYPE_MAX);
+	if (tcd->tcd_type == CFS_TCD_TYPE_IRQ)
+		spin_unlock_irqrestore(&tcd->tcd_lock, tcd->tcd_lock_flags);
+	else if (tcd->tcd_type == CFS_TCD_TYPE_SOFTIRQ)
+		spin_unlock_bh(&tcd->tcd_lock);
+	else if (unlikely(walking))
+		spin_unlock_irq(&tcd->tcd_lock);
+	else
+		spin_unlock(&tcd->tcd_lock);
+}
+
+void
+cfs_set_ptldebug_header(struct ptldebug_header *header,
+			struct libcfs_debug_msg_data *msgdata,
+			unsigned long stack)
+{
+	struct timespec64 ts;
+
+	ktime_get_real_ts64(&ts);
+
+	header->ph_subsys = msgdata->msg_subsys;
+	header->ph_mask = msgdata->msg_mask;
+	header->ph_cpu_id = smp_processor_id();
+	header->ph_type = cfs_trace_buf_idx_get();
+	/* y2038 safe since all user space treats this as unsigned, but
+	 * will overflow in 2106
+	 */
+	header->ph_sec = (u32)ts.tv_sec;
+	header->ph_usec = ts.tv_nsec / NSEC_PER_USEC;
+	header->ph_stack = stack;
+	header->ph_pid = current->pid;
+	header->ph_line_num = msgdata->msg_line;
+	header->ph_extern_pid = 0;
+}
+
+static char *
+dbghdr_to_err_string(struct ptldebug_header *hdr)
+{
+	switch (hdr->ph_subsys) {
+	case S_LND:
+	case S_LNET:
+		return "LNetError";
+	default:
+		return "LustreError";
+	}
+}
+
+static char *
+dbghdr_to_info_string(struct ptldebug_header *hdr)
+{
+	switch (hdr->ph_subsys) {
+	case S_LND:
+	case S_LNET:
+		return "LNet";
+	default:
+		return "Lustre";
+	}
+}
+
+void cfs_print_to_console(struct ptldebug_header *hdr, int mask,
+			  const char *buf, int len, const char *file,
+			  const char *fn)
+{
+	char *prefix = "Lustre", *ptype = NULL;
+
+	if (mask & D_EMERG) {
+		prefix = dbghdr_to_err_string(hdr);
+		ptype = KERN_EMERG;
+	} else if (mask & D_ERROR) {
+		prefix = dbghdr_to_err_string(hdr);
+		ptype = KERN_ERR;
+	} else if (mask & D_WARNING) {
+		prefix = dbghdr_to_info_string(hdr);
+		ptype = KERN_WARNING;
+	} else if (mask & (D_CONSOLE | libcfs_printk)) {
+		prefix = dbghdr_to_info_string(hdr);
+		ptype = KERN_INFO;
+	}
+
+	if (mask & D_CONSOLE) {
+		pr_info("%s%s: %.*s", ptype, prefix, len, buf);
+	} else {
+		pr_info("%s%s: %d:%d:(%s:%d:%s()) %.*s", ptype, prefix,
+			hdr->ph_pid, hdr->ph_extern_pid, file,
+			hdr->ph_line_num, fn, len, buf);
+	}
+}
+
+int cfs_trace_max_debug_mb(void)
+{
+	int  total_mb = (totalram_pages >> (20 - PAGE_SHIFT));
+
+	return max(512, (total_mb * 80) / 100);
+}
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
deleted file mode 100644
index 388521e4e354..000000000000
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
+++ /dev/null
@@ -1,1079 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
- *
- * Copyright (c) 2012, 2015 Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * Author: liang@whamcloud.com
- */
-
-#define DEBUG_SUBSYSTEM S_LNET
-
-#include <linux/cpu.h>
-#include <linux/sched.h>
-#include <linux/libcfs/libcfs.h>
-
-#ifdef CONFIG_SMP
-
-/**
- * modparam for setting number of partitions
- *
- *  0 : estimate best value based on cores or NUMA nodes
- *  1 : disable multiple partitions
- * >1 : specify number of partitions
- */
-static int	cpu_npartitions;
-module_param(cpu_npartitions, int, 0444);
-MODULE_PARM_DESC(cpu_npartitions, "# of CPU partitions");
-
-/**
- * modparam for setting CPU partitions patterns:
- *
- * i.e: "0[0,1,2,3] 1[4,5,6,7]", number before bracket is CPU partition ID,
- *      number in bracket is processor ID (core or HT)
- *
- * i.e: "N 0[0,1] 1[2,3]" the first character 'N' means numbers in bracket
- *       are NUMA node ID, number before bracket is CPU partition ID.
- *
- * i.e: "N", shortcut expression to create CPT from NUMA & CPU topology
- *
- * NB: If user specified cpu_pattern, cpu_npartitions will be ignored
- */
-static char	*cpu_pattern = "N";
-module_param(cpu_pattern, charp, 0444);
-MODULE_PARM_DESC(cpu_pattern, "CPU partitions pattern");
-
-struct cfs_cpt_data {
-	/* serialize hotplug etc */
-	spinlock_t		cpt_lock;
-	/* reserved for hotplug */
-	unsigned long		cpt_version;
-	/* mutex to protect cpt_cpumask */
-	struct mutex		cpt_mutex;
-	/* scratch buffer for set/unset_node */
-	cpumask_var_t		cpt_cpumask;
-};
-
-static struct cfs_cpt_data	cpt_data;
-
-static void
-cfs_node_to_cpumask(int node, cpumask_t *mask)
-{
-	const cpumask_t *tmp = cpumask_of_node(node);
-
-	if (tmp)
-		cpumask_copy(mask, tmp);
-	else
-		cpumask_clear(mask);
-}
-
-void
-cfs_cpt_table_free(struct cfs_cpt_table *cptab)
-{
-	int i;
-
-	kvfree(cptab->ctb_cpu2cpt);
-
-	for (i = 0; cptab->ctb_parts && i < cptab->ctb_nparts; i++) {
-		struct cfs_cpu_partition *part = &cptab->ctb_parts[i];
-
-		kfree(part->cpt_nodemask);
-		free_cpumask_var(part->cpt_cpumask);
-	}
-
-	kvfree(cptab->ctb_parts);
-
-	kfree(cptab->ctb_nodemask);
-	free_cpumask_var(cptab->ctb_cpumask);
-
-	kfree(cptab);
-}
-EXPORT_SYMBOL(cfs_cpt_table_free);
-
-struct cfs_cpt_table *
-cfs_cpt_table_alloc(unsigned int ncpt)
-{
-	struct cfs_cpt_table *cptab;
-	int i;
-
-	cptab = kzalloc(sizeof(*cptab), GFP_NOFS);
-	if (!cptab)
-		return NULL;
-
-	cptab->ctb_nparts = ncpt;
-
-	cptab->ctb_nodemask = kzalloc(sizeof(*cptab->ctb_nodemask),
-				      GFP_NOFS);
-	if (!zalloc_cpumask_var(&cptab->ctb_cpumask, GFP_NOFS) ||
-	    !cptab->ctb_nodemask)
-		goto failed;
-
-	cptab->ctb_cpu2cpt = kvmalloc_array(num_possible_cpus(),
-					    sizeof(cptab->ctb_cpu2cpt[0]),
-					    GFP_KERNEL);
-	if (!cptab->ctb_cpu2cpt)
-		goto failed;
-
-	memset(cptab->ctb_cpu2cpt, -1,
-	       num_possible_cpus() * sizeof(cptab->ctb_cpu2cpt[0]));
-
-	cptab->ctb_parts = kvmalloc_array(ncpt, sizeof(cptab->ctb_parts[0]),
-					  GFP_KERNEL);
-	if (!cptab->ctb_parts)
-		goto failed;
-
-	for (i = 0; i < ncpt; i++) {
-		struct cfs_cpu_partition *part = &cptab->ctb_parts[i];
-
-		part->cpt_nodemask = kzalloc(sizeof(*part->cpt_nodemask),
-					     GFP_NOFS);
-		if (!zalloc_cpumask_var(&part->cpt_cpumask, GFP_NOFS) ||
-		    !part->cpt_nodemask)
-			goto failed;
-	}
-
-	spin_lock(&cpt_data.cpt_lock);
-	/* Reserved for hotplug */
-	cptab->ctb_version = cpt_data.cpt_version;
-	spin_unlock(&cpt_data.cpt_lock);
-
-	return cptab;
-
- failed:
-	cfs_cpt_table_free(cptab);
-	return NULL;
-}
-EXPORT_SYMBOL(cfs_cpt_table_alloc);
-
-int
-cfs_cpt_table_print(struct cfs_cpt_table *cptab, char *buf, int len)
-{
-	char *tmp = buf;
-	int rc = 0;
-	int i;
-	int j;
-
-	for (i = 0; i < cptab->ctb_nparts; i++) {
-		if (len > 0) {
-			rc = snprintf(tmp, len, "%d\t: ", i);
-			len -= rc;
-		}
-
-		if (len <= 0) {
-			rc = -EFBIG;
-			goto out;
-		}
-
-		tmp += rc;
-		for_each_cpu(j, cptab->ctb_parts[i].cpt_cpumask) {
-			rc = snprintf(tmp, len, "%d ", j);
-			len -= rc;
-			if (len <= 0) {
-				rc = -EFBIG;
-				goto out;
-			}
-			tmp += rc;
-		}
-
-		*tmp = '\n';
-		tmp++;
-		len--;
-	}
-
- out:
-	if (rc < 0)
-		return rc;
-
-	return tmp - buf;
-}
-EXPORT_SYMBOL(cfs_cpt_table_print);
-
-int
-cfs_cpt_number(struct cfs_cpt_table *cptab)
-{
-	return cptab->ctb_nparts;
-}
-EXPORT_SYMBOL(cfs_cpt_number);
-
-int
-cfs_cpt_weight(struct cfs_cpt_table *cptab, int cpt)
-{
-	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
-
-	return cpt == CFS_CPT_ANY ?
-	       cpumask_weight(cptab->ctb_cpumask) :
-	       cpumask_weight(cptab->ctb_parts[cpt].cpt_cpumask);
-}
-EXPORT_SYMBOL(cfs_cpt_weight);
-
-int
-cfs_cpt_online(struct cfs_cpt_table *cptab, int cpt)
-{
-	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
-
-	return cpt == CFS_CPT_ANY ?
-	       cpumask_any_and(cptab->ctb_cpumask,
-			       cpu_online_mask) < nr_cpu_ids :
-	       cpumask_any_and(cptab->ctb_parts[cpt].cpt_cpumask,
-			       cpu_online_mask) < nr_cpu_ids;
-}
-EXPORT_SYMBOL(cfs_cpt_online);
-
-cpumask_var_t *
-cfs_cpt_cpumask(struct cfs_cpt_table *cptab, int cpt)
-{
-	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
-
-	return cpt == CFS_CPT_ANY ?
-	       &cptab->ctb_cpumask : &cptab->ctb_parts[cpt].cpt_cpumask;
-}
-EXPORT_SYMBOL(cfs_cpt_cpumask);
-
-nodemask_t *
-cfs_cpt_nodemask(struct cfs_cpt_table *cptab, int cpt)
-{
-	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
-
-	return cpt == CFS_CPT_ANY ?
-	       cptab->ctb_nodemask : cptab->ctb_parts[cpt].cpt_nodemask;
-}
-EXPORT_SYMBOL(cfs_cpt_nodemask);
-
-int
-cfs_cpt_set_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu)
-{
-	int node;
-
-	LASSERT(cpt >= 0 && cpt < cptab->ctb_nparts);
-
-	if (cpu < 0 || cpu >= nr_cpu_ids || !cpu_online(cpu)) {
-		CDEBUG(D_INFO, "CPU %d is invalid or it's offline\n", cpu);
-		return 0;
-	}
-
-	if (cptab->ctb_cpu2cpt[cpu] != -1) {
-		CDEBUG(D_INFO, "CPU %d is already in partition %d\n",
-		       cpu, cptab->ctb_cpu2cpt[cpu]);
-		return 0;
-	}
-
-	cptab->ctb_cpu2cpt[cpu] = cpt;
-
-	LASSERT(!cpumask_test_cpu(cpu, cptab->ctb_cpumask));
-	LASSERT(!cpumask_test_cpu(cpu, cptab->ctb_parts[cpt].cpt_cpumask));
-
-	cpumask_set_cpu(cpu, cptab->ctb_cpumask);
-	cpumask_set_cpu(cpu, cptab->ctb_parts[cpt].cpt_cpumask);
-
-	node = cpu_to_node(cpu);
-
-	/* first CPU of @node in this CPT table */
-	if (!node_isset(node, *cptab->ctb_nodemask))
-		node_set(node, *cptab->ctb_nodemask);
-
-	/* first CPU of @node in this partition */
-	if (!node_isset(node, *cptab->ctb_parts[cpt].cpt_nodemask))
-		node_set(node, *cptab->ctb_parts[cpt].cpt_nodemask);
-
-	return 1;
-}
-EXPORT_SYMBOL(cfs_cpt_set_cpu);
-
-void
-cfs_cpt_unset_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu)
-{
-	int node;
-	int i;
-
-	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
-
-	if (cpu < 0 || cpu >= nr_cpu_ids) {
-		CDEBUG(D_INFO, "Invalid CPU id %d\n", cpu);
-		return;
-	}
-
-	if (cpt == CFS_CPT_ANY) {
-		/* caller doesn't know the partition ID */
-		cpt = cptab->ctb_cpu2cpt[cpu];
-		if (cpt < 0) { /* not set in this CPT-table */
-			CDEBUG(D_INFO, "Try to unset cpu %d which is not in CPT-table %p\n",
-			       cpt, cptab);
-			return;
-		}
-
-	} else if (cpt != cptab->ctb_cpu2cpt[cpu]) {
-		CDEBUG(D_INFO,
-		       "CPU %d is not in cpu-partition %d\n", cpu, cpt);
-		return;
-	}
-
-	LASSERT(cpumask_test_cpu(cpu, cptab->ctb_parts[cpt].cpt_cpumask));
-	LASSERT(cpumask_test_cpu(cpu, cptab->ctb_cpumask));
-
-	cpumask_clear_cpu(cpu, cptab->ctb_parts[cpt].cpt_cpumask);
-	cpumask_clear_cpu(cpu, cptab->ctb_cpumask);
-	cptab->ctb_cpu2cpt[cpu] = -1;
-
-	node = cpu_to_node(cpu);
-
-	LASSERT(node_isset(node, *cptab->ctb_parts[cpt].cpt_nodemask));
-	LASSERT(node_isset(node, *cptab->ctb_nodemask));
-
-	for_each_cpu(i, cptab->ctb_parts[cpt].cpt_cpumask) {
-		/* this CPT has other CPU belonging to this node? */
-		if (cpu_to_node(i) == node)
-			break;
-	}
-
-	if (i >= nr_cpu_ids)
-		node_clear(node, *cptab->ctb_parts[cpt].cpt_nodemask);
-
-	for_each_cpu(i, cptab->ctb_cpumask) {
-		/* this CPT-table has other CPU belonging to this node? */
-		if (cpu_to_node(i) == node)
-			break;
-	}
-
-	if (i >= nr_cpu_ids)
-		node_clear(node, *cptab->ctb_nodemask);
-}
-EXPORT_SYMBOL(cfs_cpt_unset_cpu);
-
-int
-cfs_cpt_set_cpumask(struct cfs_cpt_table *cptab, int cpt, cpumask_t *mask)
-{
-	int i;
-
-	if (!cpumask_weight(mask) ||
-	    cpumask_any_and(mask, cpu_online_mask) >= nr_cpu_ids) {
-		CDEBUG(D_INFO, "No online CPU is found in the CPU mask for CPU partition %d\n",
-		       cpt);
-		return 0;
-	}
-
-	for_each_cpu(i, mask) {
-		if (!cfs_cpt_set_cpu(cptab, cpt, i))
-			return 0;
-	}
-
-	return 1;
-}
-EXPORT_SYMBOL(cfs_cpt_set_cpumask);
-
-void
-cfs_cpt_unset_cpumask(struct cfs_cpt_table *cptab, int cpt, cpumask_t *mask)
-{
-	int i;
-
-	for_each_cpu(i, mask)
-		cfs_cpt_unset_cpu(cptab, cpt, i);
-}
-EXPORT_SYMBOL(cfs_cpt_unset_cpumask);
-
-int
-cfs_cpt_set_node(struct cfs_cpt_table *cptab, int cpt, int node)
-{
-	int rc;
-
-	if (node < 0 || node >= MAX_NUMNODES) {
-		CDEBUG(D_INFO,
-		       "Invalid NUMA id %d for CPU partition %d\n", node, cpt);
-		return 0;
-	}
-
-	mutex_lock(&cpt_data.cpt_mutex);
-
-	cfs_node_to_cpumask(node, cpt_data.cpt_cpumask);
-
-	rc = cfs_cpt_set_cpumask(cptab, cpt, cpt_data.cpt_cpumask);
-
-	mutex_unlock(&cpt_data.cpt_mutex);
-
-	return rc;
-}
-EXPORT_SYMBOL(cfs_cpt_set_node);
-
-void
-cfs_cpt_unset_node(struct cfs_cpt_table *cptab, int cpt, int node)
-{
-	if (node < 0 || node >= MAX_NUMNODES) {
-		CDEBUG(D_INFO,
-		       "Invalid NUMA id %d for CPU partition %d\n", node, cpt);
-		return;
-	}
-
-	mutex_lock(&cpt_data.cpt_mutex);
-
-	cfs_node_to_cpumask(node, cpt_data.cpt_cpumask);
-
-	cfs_cpt_unset_cpumask(cptab, cpt, cpt_data.cpt_cpumask);
-
-	mutex_unlock(&cpt_data.cpt_mutex);
-}
-EXPORT_SYMBOL(cfs_cpt_unset_node);
-
-int
-cfs_cpt_set_nodemask(struct cfs_cpt_table *cptab, int cpt, nodemask_t *mask)
-{
-	int i;
-
-	for_each_node_mask(i, *mask) {
-		if (!cfs_cpt_set_node(cptab, cpt, i))
-			return 0;
-	}
-
-	return 1;
-}
-EXPORT_SYMBOL(cfs_cpt_set_nodemask);
-
-void
-cfs_cpt_unset_nodemask(struct cfs_cpt_table *cptab, int cpt, nodemask_t *mask)
-{
-	int i;
-
-	for_each_node_mask(i, *mask)
-		cfs_cpt_unset_node(cptab, cpt, i);
-}
-EXPORT_SYMBOL(cfs_cpt_unset_nodemask);
-
-void
-cfs_cpt_clear(struct cfs_cpt_table *cptab, int cpt)
-{
-	int last;
-	int i;
-
-	if (cpt == CFS_CPT_ANY) {
-		last = cptab->ctb_nparts - 1;
-		cpt = 0;
-	} else {
-		last = cpt;
-	}
-
-	for (; cpt <= last; cpt++) {
-		for_each_cpu(i, cptab->ctb_parts[cpt].cpt_cpumask)
-			cfs_cpt_unset_cpu(cptab, cpt, i);
-	}
-}
-EXPORT_SYMBOL(cfs_cpt_clear);
-
-int
-cfs_cpt_spread_node(struct cfs_cpt_table *cptab, int cpt)
-{
-	nodemask_t *mask;
-	int weight;
-	int rotor;
-	int node;
-
-	/* convert CPU partition ID to HW node id */
-
-	if (cpt < 0 || cpt >= cptab->ctb_nparts) {
-		mask = cptab->ctb_nodemask;
-		rotor = cptab->ctb_spread_rotor++;
-	} else {
-		mask = cptab->ctb_parts[cpt].cpt_nodemask;
-		rotor = cptab->ctb_parts[cpt].cpt_spread_rotor++;
-	}
-
-	weight = nodes_weight(*mask);
-	LASSERT(weight > 0);
-
-	rotor %= weight;
-
-	for_each_node_mask(node, *mask) {
-		if (!rotor--)
-			return node;
-	}
-
-	LBUG();
-	return 0;
-}
-EXPORT_SYMBOL(cfs_cpt_spread_node);
-
-int
-cfs_cpt_current(struct cfs_cpt_table *cptab, int remap)
-{
-	int cpu;
-	int cpt;
-
-	preempt_disable();
-	cpu = smp_processor_id();
-	cpt = cptab->ctb_cpu2cpt[cpu];
-
-	if (cpt < 0 && remap) {
-		/* don't return negative value for safety of upper layer,
-		 * instead we shadow the unknown cpu to a valid partition ID
-		 */
-		cpt = cpu % cptab->ctb_nparts;
-	}
-	preempt_enable();
-	return cpt;
-}
-EXPORT_SYMBOL(cfs_cpt_current);
-
-int
-cfs_cpt_of_cpu(struct cfs_cpt_table *cptab, int cpu)
-{
-	LASSERT(cpu >= 0 && cpu < nr_cpu_ids);
-
-	return cptab->ctb_cpu2cpt[cpu];
-}
-EXPORT_SYMBOL(cfs_cpt_of_cpu);
-
-int
-cfs_cpt_bind(struct cfs_cpt_table *cptab, int cpt)
-{
-	cpumask_var_t *cpumask;
-	nodemask_t *nodemask;
-	int rc;
-	int i;
-
-	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
-
-	if (cpt == CFS_CPT_ANY) {
-		cpumask = &cptab->ctb_cpumask;
-		nodemask = cptab->ctb_nodemask;
-	} else {
-		cpumask = &cptab->ctb_parts[cpt].cpt_cpumask;
-		nodemask = cptab->ctb_parts[cpt].cpt_nodemask;
-	}
-
-	if (cpumask_any_and(*cpumask, cpu_online_mask) >= nr_cpu_ids) {
-		CERROR("No online CPU found in CPU partition %d, did someone do CPU hotplug on system? You might need to reload Lustre modules to keep system working well.\n",
-		       cpt);
-		return -EINVAL;
-	}
-
-	for_each_online_cpu(i) {
-		if (cpumask_test_cpu(i, *cpumask))
-			continue;
-
-		rc = set_cpus_allowed_ptr(current, *cpumask);
-		set_mems_allowed(*nodemask);
-		if (!rc)
-			schedule(); /* switch to allowed CPU */
-
-		return rc;
-	}
-
-	/* don't need to set affinity because all online CPUs are covered */
-	return 0;
-}
-EXPORT_SYMBOL(cfs_cpt_bind);
-
-/**
- * Choose max to \a number CPUs from \a node and set them in \a cpt.
- * We always prefer to choose CPU in the same core/socket.
- */
-static int
-cfs_cpt_choose_ncpus(struct cfs_cpt_table *cptab, int cpt,
-		     cpumask_t *node, int number)
-{
-	cpumask_var_t socket;
-	cpumask_var_t core;
-	int rc = 0;
-	int cpu;
-
-	LASSERT(number > 0);
-
-	if (number >= cpumask_weight(node)) {
-		while (!cpumask_empty(node)) {
-			cpu = cpumask_first(node);
-
-			rc = cfs_cpt_set_cpu(cptab, cpt, cpu);
-			if (!rc)
-				return -EINVAL;
-			cpumask_clear_cpu(cpu, node);
-		}
-		return 0;
-	}
-
-	/*
-	 * Allocate scratch buffers
-	 * As we cannot initialize a cpumask_var_t, we need
-	 * to alloc both before we can risk trying to free either
-	 */
-	if (!zalloc_cpumask_var(&socket, GFP_NOFS))
-		rc = -ENOMEM;
-	if (!zalloc_cpumask_var(&core, GFP_NOFS))
-		rc = -ENOMEM;
-	if (rc)
-		goto out;
-
-	while (!cpumask_empty(node)) {
-		cpu = cpumask_first(node);
-
-		/* get cpumask for cores in the same socket */
-		cpumask_copy(socket, topology_core_cpumask(cpu));
-		cpumask_and(socket, socket, node);
-
-		LASSERT(!cpumask_empty(socket));
-
-		while (!cpumask_empty(socket)) {
-			int i;
-
-			/* get cpumask for hts in the same core */
-			cpumask_copy(core, topology_sibling_cpumask(cpu));
-			cpumask_and(core, core, node);
-
-			LASSERT(!cpumask_empty(core));
-
-			for_each_cpu(i, core) {
-				cpumask_clear_cpu(i, socket);
-				cpumask_clear_cpu(i, node);
-
-				rc = cfs_cpt_set_cpu(cptab, cpt, i);
-				if (!rc) {
-					rc = -EINVAL;
-					goto out;
-				}
-
-				if (!--number)
-					goto out;
-			}
-			cpu = cpumask_first(socket);
-		}
-	}
-
-out:
-	free_cpumask_var(socket);
-	free_cpumask_var(core);
-	return rc;
-}
-
-#define CPT_WEIGHT_MIN  4u
-
-static unsigned int
-cfs_cpt_num_estimate(void)
-{
-	unsigned int nnode = num_online_nodes();
-	unsigned int ncpu = num_online_cpus();
-	unsigned int ncpt;
-
-	if (ncpu <= CPT_WEIGHT_MIN) {
-		ncpt = 1;
-		goto out;
-	}
-
-	/* generate reasonable number of CPU partitions based on total number
-	 * of CPUs, Preferred N should be power2 and match this condition:
-	 * 2 * (N - 1)^2 < NCPUS <= 2 * N^2
-	 */
-	for (ncpt = 2; ncpu > 2 * ncpt * ncpt; ncpt <<= 1)
-		;
-
-	if (ncpt <= nnode) { /* fat numa system */
-		while (nnode > ncpt)
-			nnode >>= 1;
-
-	} else { /* ncpt > nnode */
-		while ((nnode << 1) <= ncpt)
-			nnode <<= 1;
-	}
-
-	ncpt = nnode;
-
-out:
-#if (BITS_PER_LONG == 32)
-	/* config many CPU partitions on 32-bit system could consume
-	 * too much memory
-	 */
-	ncpt = min(2U, ncpt);
-#endif
-	while (ncpu % ncpt)
-		ncpt--; /* worst case is 1 */
-
-	return ncpt;
-}
-
-static struct cfs_cpt_table *
-cfs_cpt_table_create(int ncpt)
-{
-	struct cfs_cpt_table *cptab = NULL;
-	cpumask_var_t mask;
-	int cpt = 0;
-	int num;
-	int rc;
-	int i;
-
-	rc = cfs_cpt_num_estimate();
-	if (ncpt <= 0)
-		ncpt = rc;
-
-	if (ncpt > num_online_cpus() || ncpt > 4 * rc) {
-		CWARN("CPU partition number %d is larger than suggested value (%d), your system may have performance issue or run out of memory while under pressure\n",
-		      ncpt, rc);
-	}
-
-	if (num_online_cpus() % ncpt) {
-		CERROR("CPU number %d is not multiple of cpu_npartition %d, please try different cpu_npartitions value or set pattern string by cpu_pattern=STRING\n",
-		       (int)num_online_cpus(), ncpt);
-		goto failed;
-	}
-
-	cptab = cfs_cpt_table_alloc(ncpt);
-	if (!cptab) {
-		CERROR("Failed to allocate CPU map(%d)\n", ncpt);
-		goto failed;
-	}
-
-	num = num_online_cpus() / ncpt;
-	if (!num) {
-		CERROR("CPU changed while setting CPU partition\n");
-		goto failed;
-	}
-
-	if (!zalloc_cpumask_var(&mask, GFP_NOFS)) {
-		CERROR("Failed to allocate scratch cpumask\n");
-		goto failed;
-	}
-
-	for_each_online_node(i) {
-		cfs_node_to_cpumask(i, mask);
-
-		while (!cpumask_empty(mask)) {
-			struct cfs_cpu_partition *part;
-			int n;
-
-			/*
-			 * Each emulated NUMA node has all allowed CPUs in
-			 * the mask.
-			 * End loop when all partitions have assigned CPUs.
-			 */
-			if (cpt == ncpt)
-				break;
-
-			part = &cptab->ctb_parts[cpt];
-
-			n = num - cpumask_weight(part->cpt_cpumask);
-			LASSERT(n > 0);
-
-			rc = cfs_cpt_choose_ncpus(cptab, cpt, mask, n);
-			if (rc < 0)
-				goto failed_mask;
-
-			LASSERT(num >= cpumask_weight(part->cpt_cpumask));
-			if (num == cpumask_weight(part->cpt_cpumask))
-				cpt++;
-		}
-	}
-
-	if (cpt != ncpt ||
-	    num != cpumask_weight(cptab->ctb_parts[ncpt - 1].cpt_cpumask)) {
-		CERROR("Expect %d(%d) CPU partitions but got %d(%d), CPU hotplug/unplug while setting?\n",
-		       cptab->ctb_nparts, num, cpt,
-		       cpumask_weight(cptab->ctb_parts[ncpt - 1].cpt_cpumask));
-		goto failed_mask;
-	}
-
-	free_cpumask_var(mask);
-
-	return cptab;
-
- failed_mask:
-	free_cpumask_var(mask);
- failed:
-	CERROR("Failed to setup CPU-partition-table with %d CPU-partitions, online HW nodes: %d, HW cpus: %d.\n",
-	       ncpt, num_online_nodes(), num_online_cpus());
-
-	if (cptab)
-		cfs_cpt_table_free(cptab);
-
-	return NULL;
-}
-
-static struct cfs_cpt_table *
-cfs_cpt_table_create_pattern(char *pattern)
-{
-	struct cfs_cpt_table *cptab;
-	char *str;
-	int node = 0;
-	int high;
-	int ncpt = 0;
-	int cpt;
-	int rc;
-	int c;
-	int i;
-
-	str = strim(pattern);
-	if (*str == 'n' || *str == 'N') {
-		pattern = str + 1;
-		if (*pattern != '\0') {
-			node = 1;
-		} else { /* shortcut to create CPT from NUMA & CPU topology */
-			node = -1;
-			ncpt = num_online_nodes();
-		}
-	}
-
-	if (!ncpt) { /* scanning bracket which is mark of partition */
-		for (str = pattern;; str++, ncpt++) {
-			str = strchr(str, '[');
-			if (!str)
-				break;
-		}
-	}
-
-	if (!ncpt ||
-	    (node && ncpt > num_online_nodes()) ||
-	    (!node && ncpt > num_online_cpus())) {
-		CERROR("Invalid pattern %s, or too many partitions %d\n",
-		       pattern, ncpt);
-		return NULL;
-	}
-
-	cptab = cfs_cpt_table_alloc(ncpt);
-	if (!cptab) {
-		CERROR("Failed to allocate cpu partition table\n");
-		return NULL;
-	}
-
-	if (node < 0) { /* shortcut to create CPT from NUMA & CPU topology */
-		cpt = 0;
-
-		for_each_online_node(i) {
-			if (cpt >= ncpt) {
-				CERROR("CPU changed while setting CPU partition table, %d/%d\n",
-				       cpt, ncpt);
-				goto failed;
-			}
-
-			rc = cfs_cpt_set_node(cptab, cpt++, i);
-			if (!rc)
-				goto failed;
-		}
-		return cptab;
-	}
-
-	high = node ? MAX_NUMNODES - 1 : nr_cpu_ids - 1;
-
-	for (str = strim(pattern), c = 0;; c++) {
-		struct cfs_range_expr *range;
-		struct cfs_expr_list *el;
-		char *bracket = strchr(str, '[');
-		int n;
-
-		if (!bracket) {
-			if (*str) {
-				CERROR("Invalid pattern %s\n", str);
-				goto failed;
-			}
-			if (c != ncpt) {
-				CERROR("expect %d partitions but found %d\n",
-				       ncpt, c);
-				goto failed;
-			}
-			break;
-		}
-
-		if (sscanf(str, "%d%n", &cpt, &n) < 1) {
-			CERROR("Invalid cpu pattern %s\n", str);
-			goto failed;
-		}
-
-		if (cpt < 0 || cpt >= ncpt) {
-			CERROR("Invalid partition id %d, total partitions %d\n",
-			       cpt, ncpt);
-			goto failed;
-		}
-
-		if (cfs_cpt_weight(cptab, cpt)) {
-			CERROR("Partition %d has already been set.\n", cpt);
-			goto failed;
-		}
-
-		str = strim(str + n);
-		if (str != bracket) {
-			CERROR("Invalid pattern %s\n", str);
-			goto failed;
-		}
-
-		bracket = strchr(str, ']');
-		if (!bracket) {
-			CERROR("missing right bracket for cpt %d, %s\n",
-			       cpt, str);
-			goto failed;
-		}
-
-		if (cfs_expr_list_parse(str, (bracket - str) + 1,
-					0, high, &el)) {
-			CERROR("Can't parse number range: %s\n", str);
-			goto failed;
-		}
-
-		list_for_each_entry(range, &el->el_exprs, re_link) {
-			for (i = range->re_lo; i <= range->re_hi; i++) {
-				if ((i - range->re_lo) % range->re_stride)
-					continue;
-
-				rc = node ? cfs_cpt_set_node(cptab, cpt, i) :
-					    cfs_cpt_set_cpu(cptab, cpt, i);
-				if (!rc) {
-					cfs_expr_list_free(el);
-					goto failed;
-				}
-			}
-		}
-
-		cfs_expr_list_free(el);
-
-		if (!cfs_cpt_online(cptab, cpt)) {
-			CERROR("No online CPU is found on partition %d\n", cpt);
-			goto failed;
-		}
-
-		str = strim(bracket + 1);
-	}
-
-	return cptab;
-
- failed:
-	cfs_cpt_table_free(cptab);
-	return NULL;
-}
-
-#ifdef CONFIG_HOTPLUG_CPU
-static enum cpuhp_state lustre_cpu_online;
-
-static void cfs_cpu_incr_cpt_version(void)
-{
-	spin_lock(&cpt_data.cpt_lock);
-	cpt_data.cpt_version++;
-	spin_unlock(&cpt_data.cpt_lock);
-}
-
-static int cfs_cpu_online(unsigned int cpu)
-{
-	cfs_cpu_incr_cpt_version();
-	return 0;
-}
-
-static int cfs_cpu_dead(unsigned int cpu)
-{
-	bool warn;
-
-	cfs_cpu_incr_cpt_version();
-
-	mutex_lock(&cpt_data.cpt_mutex);
-	/* if all HTs in a core are offline, it may break affinity */
-	cpumask_copy(cpt_data.cpt_cpumask, topology_sibling_cpumask(cpu));
-	warn = cpumask_any_and(cpt_data.cpt_cpumask,
-			       cpu_online_mask) >= nr_cpu_ids;
-	mutex_unlock(&cpt_data.cpt_mutex);
-	CDEBUG(warn ? D_WARNING : D_INFO,
-	       "Lustre: can't support CPU plug-out well now, performance and stability could be impacted [CPU %u]\n",
-	       cpu);
-	return 0;
-}
-#endif
-
-void
-cfs_cpu_fini(void)
-{
-	if (cfs_cpt_table)
-		cfs_cpt_table_free(cfs_cpt_table);
-
-#ifdef CONFIG_HOTPLUG_CPU
-	if (lustre_cpu_online > 0)
-		cpuhp_remove_state_nocalls(lustre_cpu_online);
-	cpuhp_remove_state_nocalls(CPUHP_LUSTRE_CFS_DEAD);
-#endif
-	free_cpumask_var(cpt_data.cpt_cpumask);
-}
-
-int
-cfs_cpu_init(void)
-{
-	int ret = 0;
-
-	LASSERT(!cfs_cpt_table);
-
-	memset(&cpt_data, 0, sizeof(cpt_data));
-
-	if (!zalloc_cpumask_var(&cpt_data.cpt_cpumask, GFP_NOFS)) {
-		CERROR("Failed to allocate scratch buffer\n");
-		return -1;
-	}
-
-	spin_lock_init(&cpt_data.cpt_lock);
-	mutex_init(&cpt_data.cpt_mutex);
-
-#ifdef CONFIG_HOTPLUG_CPU
-	ret = cpuhp_setup_state_nocalls(CPUHP_LUSTRE_CFS_DEAD,
-					"staging/lustre/cfe:dead", NULL,
-					cfs_cpu_dead);
-	if (ret < 0)
-		goto failed;
-	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
-					"staging/lustre/cfe:online",
-					cfs_cpu_online, NULL);
-	if (ret < 0)
-		goto failed;
-	lustre_cpu_online = ret;
-#endif
-	ret = -EINVAL;
-
-	if (*cpu_pattern) {
-		char *cpu_pattern_dup = kstrdup(cpu_pattern, GFP_KERNEL);
-
-		if (!cpu_pattern_dup) {
-			CERROR("Failed to duplicate cpu_pattern\n");
-			goto failed;
-		}
-
-		cfs_cpt_table = cfs_cpt_table_create_pattern(cpu_pattern_dup);
-		kfree(cpu_pattern_dup);
-		if (!cfs_cpt_table) {
-			CERROR("Failed to create cptab from pattern %s\n",
-			       cpu_pattern);
-			goto failed;
-		}
-
-	} else {
-		cfs_cpt_table = cfs_cpt_table_create(cpu_npartitions);
-		if (!cfs_cpt_table) {
-			CERROR("Failed to create ptable with npartitions %d\n",
-			       cpu_npartitions);
-			goto failed;
-		}
-	}
-
-	spin_lock(&cpt_data.cpt_lock);
-	if (cfs_cpt_table->ctb_version != cpt_data.cpt_version) {
-		spin_unlock(&cpt_data.cpt_lock);
-		CERROR("CPU hotplug/unplug during setup\n");
-		goto failed;
-	}
-	spin_unlock(&cpt_data.cpt_lock);
-
-	LCONSOLE(0, "HW nodes: %d, HW CPU cores: %d, npartitions: %d\n",
-		 num_online_nodes(), num_online_cpus(),
-		 cfs_cpt_number(cfs_cpt_table));
-	return 0;
-
- failed:
-	cfs_cpu_fini();
-	return ret;
-}
-
-#endif
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto-adler.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto-adler.c
deleted file mode 100644
index db81ed527452..000000000000
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto-adler.c
+++ /dev/null
@@ -1,139 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see http://www.gnu.org/licenses
- *
- * Please  visit http://www.xyratex.com/contact if you need additional
- * information or have any questions.
- *
- * GPL HEADER END
- */
-
-/*
- * Copyright 2012 Xyratex Technology Limited
- */
-
-/*
- * This is crypto api shash wrappers to zlib_adler32.
- */
-
-#include <linux/module.h>
-#include <linux/zutil.h>
-#include <crypto/internal/hash.h>
-#include "linux-crypto.h"
-
-#define CHKSUM_BLOCK_SIZE	1
-#define CHKSUM_DIGEST_SIZE	4
-
-static int adler32_cra_init(struct crypto_tfm *tfm)
-{
-	u32 *key = crypto_tfm_ctx(tfm);
-
-	*key = 1;
-
-	return 0;
-}
-
-static int adler32_setkey(struct crypto_shash *hash, const u8 *key,
-			  unsigned int keylen)
-{
-	u32 *mctx = crypto_shash_ctx(hash);
-
-	if (keylen != sizeof(u32)) {
-		crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
-		return -EINVAL;
-	}
-	*mctx = *(u32 *)key;
-	return 0;
-}
-
-static int adler32_init(struct shash_desc *desc)
-{
-	u32 *mctx = crypto_shash_ctx(desc->tfm);
-	u32 *cksump = shash_desc_ctx(desc);
-
-	*cksump = *mctx;
-
-	return 0;
-}
-
-static int adler32_update(struct shash_desc *desc, const u8 *data,
-			  unsigned int len)
-{
-	u32 *cksump = shash_desc_ctx(desc);
-
-	*cksump = zlib_adler32(*cksump, data, len);
-	return 0;
-}
-
-static int __adler32_finup(u32 *cksump, const u8 *data, unsigned int len,
-			   u8 *out)
-{
-	*(u32 *)out = zlib_adler32(*cksump, data, len);
-	return 0;
-}
-
-static int adler32_finup(struct shash_desc *desc, const u8 *data,
-			 unsigned int len, u8 *out)
-{
-	return __adler32_finup(shash_desc_ctx(desc), data, len, out);
-}
-
-static int adler32_final(struct shash_desc *desc, u8 *out)
-{
-	u32 *cksump = shash_desc_ctx(desc);
-
-	*(u32 *)out = *cksump;
-	return 0;
-}
-
-static int adler32_digest(struct shash_desc *desc, const u8 *data,
-			  unsigned int len, u8 *out)
-{
-	return __adler32_finup(crypto_shash_ctx(desc->tfm), data, len,
-				    out);
-}
-
-static struct shash_alg alg = {
-	.setkey		= adler32_setkey,
-	.init		= adler32_init,
-	.update		= adler32_update,
-	.final		= adler32_final,
-	.finup		= adler32_finup,
-	.digest		= adler32_digest,
-	.descsize	= sizeof(u32),
-	.digestsize	= CHKSUM_DIGEST_SIZE,
-	.base		= {
-		.cra_name		= "adler32",
-		.cra_driver_name	= "adler32-zlib",
-		.cra_priority		= 100,
-		.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
-		.cra_blocksize		= CHKSUM_BLOCK_SIZE,
-		.cra_ctxsize		= sizeof(u32),
-		.cra_module		= THIS_MODULE,
-		.cra_init		= adler32_cra_init,
-	}
-};
-
-int cfs_crypto_adler32_register(void)
-{
-	return crypto_register_shash(&alg);
-}
-
-void cfs_crypto_adler32_unregister(void)
-{
-	crypto_unregister_shash(&alg);
-}
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c
deleted file mode 100644
index b55006264155..000000000000
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c
+++ /dev/null
@@ -1,443 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see http://www.gnu.org/licenses
- *
- * Please  visit http://www.xyratex.com/contact if you need additional
- * information or have any questions.
- *
- * GPL HEADER END
- */
-
-/*
- * Copyright 2012 Xyratex Technology Limited
- *
- * Copyright (c) 2012, Intel Corporation.
- */
-
-#include <crypto/hash.h>
-#include <linux/scatterlist.h>
-#include <linux/libcfs/libcfs.h>
-#include <linux/libcfs/libcfs_crypto.h>
-#include "linux-crypto.h"
-
-/**
- *  Array of hash algorithm speed in MByte per second
- */
-static int cfs_crypto_hash_speeds[CFS_HASH_ALG_MAX];
-
-/**
- * Initialize the state descriptor for the specified hash algorithm.
- *
- * An internal routine to allocate the hash-specific state in \a req for
- * use with cfs_crypto_hash_digest() to compute the hash of a single message,
- * though possibly in multiple chunks.  The descriptor internal state should
- * be freed with cfs_crypto_hash_final().
- *
- * \param[in]	  hash_alg	hash algorithm id (CFS_HASH_ALG_*)
- * \param[out]	  type		pointer to the hash description in hash_types[]
- *				array
- * \param[in,out] req		hash state descriptor to be initialized
- * \param[in]	  key		initial hash value/state, NULL to use default
- *				value
- * \param[in]	  key_len	length of \a key
- *
- * \retval			0 on success
- * \retval			negative errno on failure
- */
-static int cfs_crypto_hash_alloc(enum cfs_crypto_hash_alg hash_alg,
-				 const struct cfs_crypto_hash_type **type,
-				 struct ahash_request **req,
-				 unsigned char *key,
-				 unsigned int key_len)
-{
-	struct crypto_ahash *tfm;
-	int err = 0;
-
-	*type = cfs_crypto_hash_type(hash_alg);
-
-	if (!*type) {
-		CWARN("Unsupported hash algorithm id = %d, max id is %d\n",
-		      hash_alg, CFS_HASH_ALG_MAX);
-		return -EINVAL;
-	}
-	tfm = crypto_alloc_ahash((*type)->cht_name, 0, CRYPTO_ALG_ASYNC);
-
-	if (IS_ERR(tfm)) {
-		CDEBUG(D_INFO, "Failed to alloc crypto hash %s\n",
-		       (*type)->cht_name);
-		return PTR_ERR(tfm);
-	}
-
-	*req = ahash_request_alloc(tfm, GFP_KERNEL);
-	if (!*req) {
-		CDEBUG(D_INFO, "Failed to alloc ahash_request for %s\n",
-		       (*type)->cht_name);
-		crypto_free_ahash(tfm);
-		return -ENOMEM;
-	}
-
-	ahash_request_set_callback(*req, 0, NULL, NULL);
-
-	if (key)
-		err = crypto_ahash_setkey(tfm, key, key_len);
-	else if ((*type)->cht_key)
-		err = crypto_ahash_setkey(tfm,
-					  (unsigned char *)&((*type)->cht_key),
-					  (*type)->cht_size);
-
-	if (err) {
-		ahash_request_free(*req);
-		crypto_free_ahash(tfm);
-		return err;
-	}
-
-	CDEBUG(D_INFO, "Using crypto hash: %s (%s) speed %d MB/s\n",
-	       crypto_ahash_alg_name(tfm), crypto_ahash_driver_name(tfm),
-	       cfs_crypto_hash_speeds[hash_alg]);
-
-	err = crypto_ahash_init(*req);
-	if (err) {
-		ahash_request_free(*req);
-		crypto_free_ahash(tfm);
-	}
-	return err;
-}
-
-/**
- * Calculate hash digest for the passed buffer.
- *
- * This should be used when computing the hash on a single contiguous buffer.
- * It combines the hash initialization, computation, and cleanup.
- *
- * \param[in]	  hash_alg	id of hash algorithm (CFS_HASH_ALG_*)
- * \param[in]	  buf		data buffer on which to compute hash
- * \param[in]	  buf_len	length of \a buf in bytes
- * \param[in]	  key		initial value/state for algorithm,
- *				if \a key = NULL use default initial value
- * \param[in]	  key_len	length of \a key in bytes
- * \param[out]	  hash		pointer to computed hash value,
- *				if \a hash = NULL then \a hash_len is to digest
- *				size in bytes, retval -ENOSPC
- * \param[in,out] hash_len	size of \a hash buffer
- *
- * \retval -EINVAL		\a buf, \a buf_len, \a hash_len,
- *				\a hash_alg invalid
- * \retval -ENOENT		\a hash_alg is unsupported
- * \retval -ENOSPC		\a hash is NULL, or \a hash_len less than
- *				digest size
- * \retval			0 for success
- * \retval			negative errno for other errors from lower
- *				layers.
- */
-int cfs_crypto_hash_digest(enum cfs_crypto_hash_alg hash_alg,
-			   const void *buf, unsigned int buf_len,
-			   unsigned char *key, unsigned int key_len,
-			   unsigned char *hash, unsigned int *hash_len)
-{
-	struct scatterlist sl;
-	struct ahash_request *req;
-	int err;
-	const struct cfs_crypto_hash_type *type;
-
-	if (!buf || !buf_len || !hash_len)
-		return -EINVAL;
-
-	err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len);
-	if (err)
-		return err;
-
-	if (!hash || *hash_len < type->cht_size) {
-		*hash_len = type->cht_size;
-		crypto_free_ahash(crypto_ahash_reqtfm(req));
-		ahash_request_free(req);
-		return -ENOSPC;
-	}
-	sg_init_one(&sl, buf, buf_len);
-
-	ahash_request_set_crypt(req, &sl, hash, sl.length);
-	err = crypto_ahash_digest(req);
-	crypto_free_ahash(crypto_ahash_reqtfm(req));
-	ahash_request_free(req);
-
-	return err;
-}
-EXPORT_SYMBOL(cfs_crypto_hash_digest);
-
-/**
- * Allocate and initialize descriptor for hash algorithm.
- *
- * This should be used to initialize a hash descriptor for multiple calls
- * to a single hash function when computing the hash across multiple
- * separate buffers or pages using cfs_crypto_hash_update{,_page}().
- *
- * The hash descriptor should be freed with cfs_crypto_hash_final().
- *
- * \param[in] hash_alg	algorithm id (CFS_HASH_ALG_*)
- * \param[in] key	initial value/state for algorithm, if \a key = NULL
- *			use default initial value
- * \param[in] key_len	length of \a key in bytes
- *
- * \retval		pointer to descriptor of hash instance
- * \retval		ERR_PTR(errno) in case of error
- */
-struct ahash_request *
-cfs_crypto_hash_init(enum cfs_crypto_hash_alg hash_alg,
-		     unsigned char *key, unsigned int key_len)
-{
-	struct ahash_request *req;
-	int err;
-	const struct cfs_crypto_hash_type *type;
-
-	err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len);
-
-	if (err)
-		return ERR_PTR(err);
-	return req;
-}
-EXPORT_SYMBOL(cfs_crypto_hash_init);
-
-/**
- * Update hash digest computed on data within the given \a page
- *
- * \param[in] hreq	hash state descriptor
- * \param[in] page	data page on which to compute the hash
- * \param[in] offset	offset within \a page at which to start hash
- * \param[in] len	length of data on which to compute hash
- *
- * \retval		0 for success
- * \retval		negative errno on failure
- */
-int cfs_crypto_hash_update_page(struct ahash_request *req,
-				struct page *page, unsigned int offset,
-				unsigned int len)
-{
-	struct scatterlist sl;
-
-	sg_init_table(&sl, 1);
-	sg_set_page(&sl, page, len, offset & ~PAGE_MASK);
-
-	ahash_request_set_crypt(req, &sl, NULL, sl.length);
-	return crypto_ahash_update(req);
-}
-EXPORT_SYMBOL(cfs_crypto_hash_update_page);
-
-/**
- * Update hash digest computed on the specified data
- *
- * \param[in] req	hash state descriptor
- * \param[in] buf	data buffer on which to compute the hash
- * \param[in] buf_len	length of \buf on which to compute hash
- *
- * \retval		0 for success
- * \retval		negative errno on failure
- */
-int cfs_crypto_hash_update(struct ahash_request *req,
-			   const void *buf, unsigned int buf_len)
-{
-	struct scatterlist sl;
-
-	sg_init_one(&sl, buf, buf_len);
-
-	ahash_request_set_crypt(req, &sl, NULL, sl.length);
-	return crypto_ahash_update(req);
-}
-EXPORT_SYMBOL(cfs_crypto_hash_update);
-
-/**
- * Finish hash calculation, copy hash digest to buffer, clean up hash descriptor
- *
- * \param[in]	  req		hash descriptor
- * \param[out]	  hash		pointer to hash buffer to store hash digest
- * \param[in,out] hash_len	pointer to hash buffer size, if \a req = NULL
- *				only free \a req instead of computing the hash
- *
- * \retval	0 for success
- * \retval	-EOVERFLOW if hash_len is too small for the hash digest
- * \retval	negative errno for other errors from lower layers
- */
-int cfs_crypto_hash_final(struct ahash_request *req,
-			  unsigned char *hash, unsigned int *hash_len)
-{
-	int err;
-	int size = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
-
-	if (!hash || !hash_len) {
-		err = 0;
-		goto free_ahash;
-	}
-	if (*hash_len < size) {
-		err = -EOVERFLOW;
-		goto free_ahash;
-	}
-
-	ahash_request_set_crypt(req, NULL, hash, 0);
-	err = crypto_ahash_final(req);
-	if (!err)
-		*hash_len = size;
-free_ahash:
-	crypto_free_ahash(crypto_ahash_reqtfm(req));
-	ahash_request_free(req);
-	return err;
-}
-EXPORT_SYMBOL(cfs_crypto_hash_final);
-
-/**
- * Compute the speed of specified hash function
- *
- * Run a speed test on the given hash algorithm on buffer of the given size.
- * The speed is stored internally in the cfs_crypto_hash_speeds[] array, and
- * is available through the cfs_crypto_hash_speed() function.
- *
- * \param[in] hash_alg	hash algorithm id (CFS_HASH_ALG_*)
- * \param[in] buf	data buffer on which to compute the hash
- * \param[in] buf_len	length of \buf on which to compute hash
- */
-static void cfs_crypto_performance_test(enum cfs_crypto_hash_alg hash_alg)
-{
-	int buf_len = max(PAGE_SIZE, 1048576UL);
-	void *buf;
-	unsigned long start, end;
-	int bcount, err = 0;
-	struct page *page;
-	unsigned char hash[CFS_CRYPTO_HASH_DIGESTSIZE_MAX];
-	unsigned int hash_len = sizeof(hash);
-
-	page = alloc_page(GFP_KERNEL);
-	if (!page) {
-		err = -ENOMEM;
-		goto out_err;
-	}
-
-	buf = kmap(page);
-	memset(buf, 0xAD, PAGE_SIZE);
-	kunmap(page);
-
-	for (start = jiffies, end = start + msecs_to_jiffies(MSEC_PER_SEC),
-	     bcount = 0; time_before(jiffies, end); bcount++) {
-		struct ahash_request *hdesc;
-		int i;
-
-		hdesc = cfs_crypto_hash_init(hash_alg, NULL, 0);
-		if (IS_ERR(hdesc)) {
-			err = PTR_ERR(hdesc);
-			break;
-		}
-
-		for (i = 0; i < buf_len / PAGE_SIZE; i++) {
-			err = cfs_crypto_hash_update_page(hdesc, page, 0,
-							  PAGE_SIZE);
-			if (err)
-				break;
-		}
-
-		err = cfs_crypto_hash_final(hdesc, hash, &hash_len);
-		if (err)
-			break;
-	}
-	end = jiffies;
-	__free_page(page);
-out_err:
-	if (err) {
-		cfs_crypto_hash_speeds[hash_alg] = err;
-		CDEBUG(D_INFO, "Crypto hash algorithm %s test error: rc = %d\n",
-		       cfs_crypto_hash_name(hash_alg), err);
-	} else {
-		unsigned long tmp;
-
-		tmp = ((bcount * buf_len / jiffies_to_msecs(end - start)) *
-		       1000) / (1024 * 1024);
-		cfs_crypto_hash_speeds[hash_alg] = (int)tmp;
-		CDEBUG(D_CONFIG, "Crypto hash algorithm %s speed = %d MB/s\n",
-		       cfs_crypto_hash_name(hash_alg),
-		       cfs_crypto_hash_speeds[hash_alg]);
-	}
-}
-
-/**
- * hash speed in Mbytes per second for valid hash algorithm
- *
- * Return the performance of the specified \a hash_alg that was previously
- * computed using cfs_crypto_performance_test().
- *
- * \param[in] hash_alg	hash algorithm id (CFS_HASH_ALG_*)
- *
- * \retval		positive speed of the hash function in MB/s
- * \retval		-ENOENT if \a hash_alg is unsupported
- * \retval		negative errno if \a hash_alg speed is unavailable
- */
-int cfs_crypto_hash_speed(enum cfs_crypto_hash_alg hash_alg)
-{
-	if (hash_alg < CFS_HASH_ALG_MAX)
-		return cfs_crypto_hash_speeds[hash_alg];
-	return -ENOENT;
-}
-EXPORT_SYMBOL(cfs_crypto_hash_speed);
-
-/**
- * Run the performance test for all hash algorithms.
- *
- * Run the cfs_crypto_performance_test() benchmark for all of the available
- * hash functions using a 1MB buffer size.  This is a reasonable buffer size
- * for Lustre RPCs, even if the actual RPC size is larger or smaller.
- *
- * Since the setup cost and computation speed of various hash algorithms is
- * a function of the buffer size (and possibly internal contention of offload
- * engines), this speed only represents an estimate of the actual speed under
- * actual usage, but is reasonable for comparing available algorithms.
- *
- * The actual speeds are available via cfs_crypto_hash_speed() for later
- * comparison.
- *
- * \retval	0 on success
- * \retval	-ENOMEM if no memory is available for test buffer
- */
-static int cfs_crypto_test_hashes(void)
-{
-	enum cfs_crypto_hash_alg hash_alg;
-
-	for (hash_alg = 0; hash_alg < CFS_HASH_ALG_MAX; hash_alg++)
-		cfs_crypto_performance_test(hash_alg);
-
-	return 0;
-}
-
-static int adler32;
-
-/**
- * Register available hash functions
- *
- * \retval	0
- */
-int cfs_crypto_register(void)
-{
-	request_module("crc32c");
-
-	adler32 = cfs_crypto_adler32_register();
-
-	/* check all algorithms and do performance test */
-	cfs_crypto_test_hashes();
-	return 0;
-}
-
-/**
- * Unregister previously registered hash functions
- */
-void cfs_crypto_unregister(void)
-{
-	if (!adler32)
-		cfs_crypto_adler32_unregister();
-}
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.h b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.h
deleted file mode 100644
index 5616e9ea1450..000000000000
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.h
+++ /dev/null
@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see http://www.gnu.org/licenses
- *
- * Please  visit http://www.xyratex.com/contact if you need additional
- * information or have any questions.
- *
- * GPL HEADER END
- */
-
-/**
- * Functions for start/stop shash adler32 algorithm.
- */
-int cfs_crypto_adler32_register(void);
-void cfs_crypto_adler32_unregister(void);
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-debug.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-debug.c
deleted file mode 100644
index 0092166af258..000000000000
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-debug.c
+++ /dev/null
@@ -1,145 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- *
- * Copyright (c) 2012, Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * libcfs/libcfs/linux/linux-debug.c
- *
- * Author: Phil Schwan <phil@clusterfs.com>
- */
-
-#include <linux/module.h>
-#include <linux/kmod.h>
-#include <linux/notifier.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/string.h>
-#include <linux/stat.h>
-#include <linux/errno.h>
-#include <linux/unistd.h>
-#include <linux/interrupt.h>
-#include <linux/completion.h>
-#include <linux/fs.h>
-#include <linux/uaccess.h>
-#include <linux/miscdevice.h>
-
-# define DEBUG_SUBSYSTEM S_LNET
-
-#include <linux/libcfs/libcfs.h>
-
-#include "../tracefile.h"
-
-#include <linux/kallsyms.h>
-
-char lnet_debug_log_upcall[1024] = "/usr/lib/lustre/lnet_debug_log_upcall";
-
-/**
- * Upcall function once a Lustre log has been dumped.
- *
- * \param file  path of the dumped log
- */
-void libcfs_run_debug_log_upcall(char *file)
-{
-	char *argv[3];
-	int rc;
-	static const char * const envp[] = {
-		"HOME=/",
-		"PATH=/sbin:/bin:/usr/sbin:/usr/bin",
-		NULL
-	};
-
-	argv[0] = lnet_debug_log_upcall;
-
-	LASSERTF(file, "called on a null filename\n");
-	argv[1] = file; /* only need to pass the path of the file */
-
-	argv[2] = NULL;
-
-	rc = call_usermodehelper(argv[0], argv, (char **)envp, 1);
-	if (rc < 0 && rc != -ENOENT) {
-		CERROR("Error %d invoking LNET debug log upcall %s %s; check /sys/kernel/debug/lnet/debug_log_upcall\n",
-		       rc, argv[0], argv[1]);
-	} else {
-		CDEBUG(D_HA, "Invoked LNET debug log upcall %s %s\n",
-		       argv[0], argv[1]);
-	}
-}
-
-/* coverity[+kill] */
-void __noreturn lbug_with_loc(struct libcfs_debug_msg_data *msgdata)
-{
-	libcfs_catastrophe = 1;
-	libcfs_debug_msg(msgdata, "LBUG\n");
-
-	if (in_interrupt()) {
-		panic("LBUG in interrupt.\n");
-		/* not reached */
-	}
-
-	dump_stack();
-	if (!libcfs_panic_on_lbug)
-		libcfs_debug_dumplog();
-	if (libcfs_panic_on_lbug)
-		panic("LBUG");
-	set_current_state(TASK_UNINTERRUPTIBLE);
-	while (1)
-		schedule();
-}
-EXPORT_SYMBOL(lbug_with_loc);
-
-static int panic_notifier(struct notifier_block *self, unsigned long unused1,
-			  void *unused2)
-{
-	if (libcfs_panic_in_progress)
-		return 0;
-
-	libcfs_panic_in_progress = 1;
-	mb();
-
-	return 0;
-}
-
-static struct notifier_block libcfs_panic_notifier = {
-	.notifier_call	= panic_notifier,
-	.next		= NULL,
-	.priority	= 10000,
-};
-
-void libcfs_register_panic_notifier(void)
-{
-	atomic_notifier_chain_register(&panic_notifier_list,
-				       &libcfs_panic_notifier);
-}
-
-void libcfs_unregister_panic_notifier(void)
-{
-	atomic_notifier_chain_unregister(&panic_notifier_list,
-					 &libcfs_panic_notifier);
-}
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-module.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-module.c
deleted file mode 100644
index ddf625669bff..000000000000
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-module.c
+++ /dev/null
@@ -1,197 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- *
- * Copyright (c) 2012, Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- */
-
-#define DEBUG_SUBSYSTEM S_LNET
-
-#include <linux/libcfs/libcfs.h>
-
-#define LNET_MINOR 240
-
-static inline size_t libcfs_ioctl_packlen(struct libcfs_ioctl_data *data)
-{
-	size_t len = sizeof(*data);
-
-	len += cfs_size_round(data->ioc_inllen1);
-	len += cfs_size_round(data->ioc_inllen2);
-	return len;
-}
-
-static inline bool libcfs_ioctl_is_invalid(struct libcfs_ioctl_data *data)
-{
-	if (data->ioc_hdr.ioc_len > BIT(30)) {
-		CERROR("LIBCFS ioctl: ioc_len larger than 1<<30\n");
-		return true;
-	}
-	if (data->ioc_inllen1 > BIT(30)) {
-		CERROR("LIBCFS ioctl: ioc_inllen1 larger than 1<<30\n");
-		return true;
-	}
-	if (data->ioc_inllen2 > BIT(30)) {
-		CERROR("LIBCFS ioctl: ioc_inllen2 larger than 1<<30\n");
-		return true;
-	}
-	if (data->ioc_inlbuf1 && !data->ioc_inllen1) {
-		CERROR("LIBCFS ioctl: inlbuf1 pointer but 0 length\n");
-		return true;
-	}
-	if (data->ioc_inlbuf2 && !data->ioc_inllen2) {
-		CERROR("LIBCFS ioctl: inlbuf2 pointer but 0 length\n");
-		return true;
-	}
-	if (data->ioc_pbuf1 && !data->ioc_plen1) {
-		CERROR("LIBCFS ioctl: pbuf1 pointer but 0 length\n");
-		return true;
-	}
-	if (data->ioc_pbuf2 && !data->ioc_plen2) {
-		CERROR("LIBCFS ioctl: pbuf2 pointer but 0 length\n");
-		return true;
-	}
-	if (data->ioc_plen1 && !data->ioc_pbuf1) {
-		CERROR("LIBCFS ioctl: plen1 nonzero but no pbuf1 pointer\n");
-		return true;
-	}
-	if (data->ioc_plen2 && !data->ioc_pbuf2) {
-		CERROR("LIBCFS ioctl: plen2 nonzero but no pbuf2 pointer\n");
-		return true;
-	}
-	if ((u32)libcfs_ioctl_packlen(data) != data->ioc_hdr.ioc_len) {
-		CERROR("LIBCFS ioctl: packlen != ioc_len\n");
-		return true;
-	}
-	if (data->ioc_inllen1 &&
-	    data->ioc_bulk[data->ioc_inllen1 - 1] != '\0') {
-		CERROR("LIBCFS ioctl: inlbuf1 not 0 terminated\n");
-		return true;
-	}
-	if (data->ioc_inllen2 &&
-	    data->ioc_bulk[cfs_size_round(data->ioc_inllen1) +
-			   data->ioc_inllen2 - 1] != '\0') {
-		CERROR("LIBCFS ioctl: inlbuf2 not 0 terminated\n");
-		return true;
-	}
-	return false;
-}
-
-int libcfs_ioctl_data_adjust(struct libcfs_ioctl_data *data)
-{
-	if (libcfs_ioctl_is_invalid(data)) {
-		CERROR("libcfs ioctl: parameter not correctly formatted\n");
-		return -EINVAL;
-	}
-
-	if (data->ioc_inllen1)
-		data->ioc_inlbuf1 = &data->ioc_bulk[0];
-
-	if (data->ioc_inllen2)
-		data->ioc_inlbuf2 = &data->ioc_bulk[0] +
-			cfs_size_round(data->ioc_inllen1);
-
-	return 0;
-}
-
-int libcfs_ioctl_getdata(struct libcfs_ioctl_hdr **hdr_pp,
-			 const struct libcfs_ioctl_hdr __user *uhdr)
-{
-	struct libcfs_ioctl_hdr hdr;
-	int err;
-
-	if (copy_from_user(&hdr, uhdr, sizeof(hdr)))
-		return -EFAULT;
-
-	if (hdr.ioc_version != LIBCFS_IOCTL_VERSION &&
-	    hdr.ioc_version != LIBCFS_IOCTL_VERSION2) {
-		CERROR("libcfs ioctl: version mismatch expected %#x, got %#x\n",
-		       LIBCFS_IOCTL_VERSION, hdr.ioc_version);
-		return -EINVAL;
-	}
-
-	if (hdr.ioc_len < sizeof(hdr)) {
-		CERROR("libcfs ioctl: user buffer too small for ioctl\n");
-		return -EINVAL;
-	}
-
-	if (hdr.ioc_len > LIBCFS_IOC_DATA_MAX) {
-		CERROR("libcfs ioctl: user buffer is too large %d/%d\n",
-		       hdr.ioc_len, LIBCFS_IOC_DATA_MAX);
-		return -EINVAL;
-	}
-
-	*hdr_pp = kvmalloc(hdr.ioc_len, GFP_KERNEL);
-	if (!*hdr_pp)
-		return -ENOMEM;
-
-	if (copy_from_user(*hdr_pp, uhdr, hdr.ioc_len)) {
-		err = -EFAULT;
-		goto free;
-	}
-
-	if ((*hdr_pp)->ioc_version != hdr.ioc_version ||
-	    (*hdr_pp)->ioc_len != hdr.ioc_len) {
-		err = -EINVAL;
-		goto free;
-	}
-
-	return 0;
-
-free:
-	kvfree(*hdr_pp);
-	return err;
-}
-
-static long
-libcfs_psdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
-{
-	if (!capable(CAP_SYS_ADMIN))
-		return -EACCES;
-
-	if (_IOC_TYPE(cmd) != IOC_LIBCFS_TYPE ||
-	    _IOC_NR(cmd) < IOC_LIBCFS_MIN_NR  ||
-	    _IOC_NR(cmd) > IOC_LIBCFS_MAX_NR) {
-		CDEBUG(D_IOCTL, "invalid ioctl ( type %d, nr %d, size %d )\n",
-		       _IOC_TYPE(cmd), _IOC_NR(cmd), _IOC_SIZE(cmd));
-		return -EINVAL;
-	}
-
-	return libcfs_ioctl(cmd, (void __user *)arg);
-}
-
-static const struct file_operations libcfs_fops = {
-	.owner		= THIS_MODULE,
-	.unlocked_ioctl	= libcfs_psdev_ioctl,
-};
-
-struct miscdevice libcfs_dev = {
-	.minor = LNET_MINOR,
-	.name = "lnet",
-	.fops = &libcfs_fops,
-};
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-tracefile.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-tracefile.c
deleted file mode 100644
index 7928d7182634..000000000000
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-tracefile.c
+++ /dev/null
@@ -1,257 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- *
- * Copyright (c) 2012, Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- */
-
-#define DEBUG_SUBSYSTEM S_LNET
-#define LUSTRE_TRACEFILE_PRIVATE
-
-#include <linux/libcfs/libcfs.h>
-#include "../tracefile.h"
-
-/* percents to share the total debug memory for each type */
-static unsigned int pages_factor[CFS_TCD_TYPE_MAX] = {
-	80,  /* 80% pages for CFS_TCD_TYPE_PROC */
-	10,  /* 10% pages for CFS_TCD_TYPE_SOFTIRQ */
-	10   /* 10% pages for CFS_TCD_TYPE_IRQ */
-};
-
-char *cfs_trace_console_buffers[NR_CPUS][CFS_TCD_TYPE_MAX];
-
-static DECLARE_RWSEM(cfs_tracefile_sem);
-
-int cfs_tracefile_init_arch(void)
-{
-	int i;
-	int j;
-	struct cfs_trace_cpu_data *tcd;
-
-	/* initialize trace_data */
-	memset(cfs_trace_data, 0, sizeof(cfs_trace_data));
-	for (i = 0; i < CFS_TCD_TYPE_MAX; i++) {
-		cfs_trace_data[i] =
-			kmalloc_array(num_possible_cpus(),
-				      sizeof(union cfs_trace_data_union),
-				      GFP_KERNEL);
-		if (!cfs_trace_data[i])
-			goto out;
-	}
-
-	/* arch related info initialized */
-	cfs_tcd_for_each(tcd, i, j) {
-		spin_lock_init(&tcd->tcd_lock);
-		tcd->tcd_pages_factor = pages_factor[i];
-		tcd->tcd_type = i;
-		tcd->tcd_cpu = j;
-	}
-
-	for (i = 0; i < num_possible_cpus(); i++)
-		for (j = 0; j < 3; j++) {
-			cfs_trace_console_buffers[i][j] =
-				kmalloc(CFS_TRACE_CONSOLE_BUFFER_SIZE,
-					GFP_KERNEL);
-
-			if (!cfs_trace_console_buffers[i][j])
-				goto out;
-		}
-
-	return 0;
-
-out:
-	cfs_tracefile_fini_arch();
-	pr_err("lnet: Not enough memory\n");
-	return -ENOMEM;
-}
-
-void cfs_tracefile_fini_arch(void)
-{
-	int i;
-	int j;
-
-	for (i = 0; i < num_possible_cpus(); i++)
-		for (j = 0; j < 3; j++) {
-			kfree(cfs_trace_console_buffers[i][j]);
-			cfs_trace_console_buffers[i][j] = NULL;
-		}
-
-	for (i = 0; cfs_trace_data[i]; i++) {
-		kfree(cfs_trace_data[i]);
-		cfs_trace_data[i] = NULL;
-	}
-}
-
-void cfs_tracefile_read_lock(void)
-{
-	down_read(&cfs_tracefile_sem);
-}
-
-void cfs_tracefile_read_unlock(void)
-{
-	up_read(&cfs_tracefile_sem);
-}
-
-void cfs_tracefile_write_lock(void)
-{
-	down_write(&cfs_tracefile_sem);
-}
-
-void cfs_tracefile_write_unlock(void)
-{
-	up_write(&cfs_tracefile_sem);
-}
-
-enum cfs_trace_buf_type cfs_trace_buf_idx_get(void)
-{
-	if (in_irq())
-		return CFS_TCD_TYPE_IRQ;
-	if (in_softirq())
-		return CFS_TCD_TYPE_SOFTIRQ;
-	return CFS_TCD_TYPE_PROC;
-}
-
-/*
- * The walking argument indicates the locking comes from all tcd types
- * iterator and we must lock it and dissable local irqs to avoid deadlocks
- * with other interrupt locks that might be happening. See LU-1311
- * for details.
- */
-int cfs_trace_lock_tcd(struct cfs_trace_cpu_data *tcd, int walking)
-	__acquires(&tcd->tc_lock)
-{
-	__LASSERT(tcd->tcd_type < CFS_TCD_TYPE_MAX);
-	if (tcd->tcd_type == CFS_TCD_TYPE_IRQ)
-		spin_lock_irqsave(&tcd->tcd_lock, tcd->tcd_lock_flags);
-	else if (tcd->tcd_type == CFS_TCD_TYPE_SOFTIRQ)
-		spin_lock_bh(&tcd->tcd_lock);
-	else if (unlikely(walking))
-		spin_lock_irq(&tcd->tcd_lock);
-	else
-		spin_lock(&tcd->tcd_lock);
-	return 1;
-}
-
-void cfs_trace_unlock_tcd(struct cfs_trace_cpu_data *tcd, int walking)
-	__releases(&tcd->tcd_lock)
-{
-	__LASSERT(tcd->tcd_type < CFS_TCD_TYPE_MAX);
-	if (tcd->tcd_type == CFS_TCD_TYPE_IRQ)
-		spin_unlock_irqrestore(&tcd->tcd_lock, tcd->tcd_lock_flags);
-	else if (tcd->tcd_type == CFS_TCD_TYPE_SOFTIRQ)
-		spin_unlock_bh(&tcd->tcd_lock);
-	else if (unlikely(walking))
-		spin_unlock_irq(&tcd->tcd_lock);
-	else
-		spin_unlock(&tcd->tcd_lock);
-}
-
-void
-cfs_set_ptldebug_header(struct ptldebug_header *header,
-			struct libcfs_debug_msg_data *msgdata,
-			unsigned long stack)
-{
-	struct timespec64 ts;
-
-	ktime_get_real_ts64(&ts);
-
-	header->ph_subsys = msgdata->msg_subsys;
-	header->ph_mask = msgdata->msg_mask;
-	header->ph_cpu_id = smp_processor_id();
-	header->ph_type = cfs_trace_buf_idx_get();
-	/* y2038 safe since all user space treats this as unsigned, but
-	 * will overflow in 2106
-	 */
-	header->ph_sec = (u32)ts.tv_sec;
-	header->ph_usec = ts.tv_nsec / NSEC_PER_USEC;
-	header->ph_stack = stack;
-	header->ph_pid = current->pid;
-	header->ph_line_num = msgdata->msg_line;
-	header->ph_extern_pid = 0;
-}
-
-static char *
-dbghdr_to_err_string(struct ptldebug_header *hdr)
-{
-	switch (hdr->ph_subsys) {
-	case S_LND:
-	case S_LNET:
-		return "LNetError";
-	default:
-		return "LustreError";
-	}
-}
-
-static char *
-dbghdr_to_info_string(struct ptldebug_header *hdr)
-{
-	switch (hdr->ph_subsys) {
-	case S_LND:
-	case S_LNET:
-		return "LNet";
-	default:
-		return "Lustre";
-	}
-}
-
-void cfs_print_to_console(struct ptldebug_header *hdr, int mask,
-			  const char *buf, int len, const char *file,
-			  const char *fn)
-{
-	char *prefix = "Lustre", *ptype = NULL;
-
-	if (mask & D_EMERG) {
-		prefix = dbghdr_to_err_string(hdr);
-		ptype = KERN_EMERG;
-	} else if (mask & D_ERROR) {
-		prefix = dbghdr_to_err_string(hdr);
-		ptype = KERN_ERR;
-	} else if (mask & D_WARNING) {
-		prefix = dbghdr_to_info_string(hdr);
-		ptype = KERN_WARNING;
-	} else if (mask & (D_CONSOLE | libcfs_printk)) {
-		prefix = dbghdr_to_info_string(hdr);
-		ptype = KERN_INFO;
-	}
-
-	if (mask & D_CONSOLE) {
-		pr_info("%s%s: %.*s", ptype, prefix, len, buf);
-	} else {
-		pr_info("%s%s: %d:%d:(%s:%d:%s()) %.*s", ptype, prefix,
-			hdr->ph_pid, hdr->ph_extern_pid, file,
-			hdr->ph_line_num, fn, len, buf);
-	}
-}
-
-int cfs_trace_max_debug_mb(void)
-{
-	int  total_mb = (totalram_pages >> (20 - PAGE_SHIFT));
-
-	return max(512, (total_mb * 80) / 100);
-}
diff --git a/drivers/staging/lustre/lnet/libcfs/module.c b/drivers/staging/lustre/lnet/libcfs/module.c
index a03f924f1d7c..5dc7de9e6478 100644
--- a/drivers/staging/lustre/lnet/libcfs/module.c
+++ b/drivers/staging/lustre/lnet/libcfs/module.c
@@ -30,6 +30,7 @@
  * This file is part of Lustre, http://www.lustre.org/
  * Lustre is a trademark of Sun Microsystems, Inc.
  */
+#include <linux/miscdevice.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>
@@ -51,7 +52,6 @@
 
 # define DEBUG_SUBSYSTEM S_LNET
 
-#include <linux/libcfs/libcfs.h>
 #include <asm/div64.h>
 
 #include <linux/libcfs/libcfs_crypto.h>
@@ -59,42 +59,148 @@
 #include <uapi/linux/lnet/lnet-dlc.h>
 #include "tracefile.h"
 
+struct lnet_debugfs_symlink_def {
+	char *name;
+	char *target;
+};
+
 static struct dentry *lnet_debugfs_root;
 
-static DECLARE_RWSEM(ioctl_list_sem);
-static LIST_HEAD(ioctl_list);
+BLOCKING_NOTIFIER_HEAD(libcfs_ioctl_list);
+EXPORT_SYMBOL(libcfs_ioctl_list);
 
-int libcfs_register_ioctl(struct libcfs_ioctl_handler *hand)
+static inline size_t libcfs_ioctl_packlen(struct libcfs_ioctl_data *data)
 {
-	int rc = 0;
+	size_t len = sizeof(*data);
 
-	down_write(&ioctl_list_sem);
-	if (!list_empty(&hand->item))
-		rc = -EBUSY;
-	else
-		list_add_tail(&hand->item, &ioctl_list);
-	up_write(&ioctl_list_sem);
+	len += cfs_size_round(data->ioc_inllen1);
+	len += cfs_size_round(data->ioc_inllen2);
+	return len;
+}
 
-	return rc;
+static inline bool libcfs_ioctl_is_invalid(struct libcfs_ioctl_data *data)
+{
+	if (data->ioc_hdr.ioc_len > BIT(30)) {
+		CERROR("LIBCFS ioctl: ioc_len larger than 1<<30\n");
+		return true;
+	}
+	if (data->ioc_inllen1 > BIT(30)) {
+		CERROR("LIBCFS ioctl: ioc_inllen1 larger than 1<<30\n");
+		return true;
+	}
+	if (data->ioc_inllen2 > BIT(30)) {
+		CERROR("LIBCFS ioctl: ioc_inllen2 larger than 1<<30\n");
+		return true;
+	}
+	if (data->ioc_inlbuf1 && !data->ioc_inllen1) {
+		CERROR("LIBCFS ioctl: inlbuf1 pointer but 0 length\n");
+		return true;
+	}
+	if (data->ioc_inlbuf2 && !data->ioc_inllen2) {
+		CERROR("LIBCFS ioctl: inlbuf2 pointer but 0 length\n");
+		return true;
+	}
+	if (data->ioc_pbuf1 && !data->ioc_plen1) {
+		CERROR("LIBCFS ioctl: pbuf1 pointer but 0 length\n");
+		return true;
+	}
+	if (data->ioc_pbuf2 && !data->ioc_plen2) {
+		CERROR("LIBCFS ioctl: pbuf2 pointer but 0 length\n");
+		return true;
+	}
+	if (data->ioc_plen1 && !data->ioc_pbuf1) {
+		CERROR("LIBCFS ioctl: plen1 nonzero but no pbuf1 pointer\n");
+		return true;
+	}
+	if (data->ioc_plen2 && !data->ioc_pbuf2) {
+		CERROR("LIBCFS ioctl: plen2 nonzero but no pbuf2 pointer\n");
+		return true;
+	}
+	if ((u32)libcfs_ioctl_packlen(data) != data->ioc_hdr.ioc_len) {
+		CERROR("LIBCFS ioctl: packlen != ioc_len\n");
+		return true;
+	}
+	if (data->ioc_inllen1 &&
+	    data->ioc_bulk[data->ioc_inllen1 - 1] != '\0') {
+		CERROR("LIBCFS ioctl: inlbuf1 not 0 terminated\n");
+		return true;
+	}
+	if (data->ioc_inllen2 &&
+	    data->ioc_bulk[cfs_size_round(data->ioc_inllen1) +
+			   data->ioc_inllen2 - 1] != '\0') {
+		CERROR("LIBCFS ioctl: inlbuf2 not 0 terminated\n");
+		return true;
+	}
+	return false;
 }
-EXPORT_SYMBOL(libcfs_register_ioctl);
 
-int libcfs_deregister_ioctl(struct libcfs_ioctl_handler *hand)
+static int libcfs_ioctl_data_adjust(struct libcfs_ioctl_data *data)
 {
-	int rc = 0;
+	if (libcfs_ioctl_is_invalid(data)) {
+		CERROR("libcfs ioctl: parameter not correctly formatted\n");
+		return -EINVAL;
+	}
 
-	down_write(&ioctl_list_sem);
-	if (list_empty(&hand->item))
-		rc = -ENOENT;
-	else
-		list_del_init(&hand->item);
-	up_write(&ioctl_list_sem);
+	if (data->ioc_inllen1)
+		data->ioc_inlbuf1 = &data->ioc_bulk[0];
 
-	return rc;
+	if (data->ioc_inllen2)
+		data->ioc_inlbuf2 = &data->ioc_bulk[0] +
+			cfs_size_round(data->ioc_inllen1);
+
+	return 0;
 }
-EXPORT_SYMBOL(libcfs_deregister_ioctl);
 
-int libcfs_ioctl(unsigned long cmd, void __user *uparam)
+static int libcfs_ioctl_getdata(struct libcfs_ioctl_hdr **hdr_pp,
+				const struct libcfs_ioctl_hdr __user *uhdr)
+{
+	struct libcfs_ioctl_hdr hdr;
+	int err;
+
+	if (copy_from_user(&hdr, uhdr, sizeof(hdr)))
+		return -EFAULT;
+
+	if (hdr.ioc_version != LIBCFS_IOCTL_VERSION &&
+	    hdr.ioc_version != LIBCFS_IOCTL_VERSION2) {
+		CERROR("libcfs ioctl: version mismatch expected %#x, got %#x\n",
+		       LIBCFS_IOCTL_VERSION, hdr.ioc_version);
+		return -EINVAL;
+	}
+
+	if (hdr.ioc_len < sizeof(hdr)) {
+		CERROR("libcfs ioctl: user buffer too small for ioctl\n");
+		return -EINVAL;
+	}
+
+	if (hdr.ioc_len > LIBCFS_IOC_DATA_MAX) {
+		CERROR("libcfs ioctl: user buffer is too large %d/%d\n",
+		       hdr.ioc_len, LIBCFS_IOC_DATA_MAX);
+		return -EINVAL;
+	}
+
+	*hdr_pp = kvmalloc(hdr.ioc_len, GFP_KERNEL);
+	if (!*hdr_pp)
+		return -ENOMEM;
+
+	if (copy_from_user(*hdr_pp, uhdr, hdr.ioc_len)) {
+		err = -EFAULT;
+		goto free;
+	}
+
+	if ((*hdr_pp)->ioc_version != hdr.ioc_version ||
+	    (*hdr_pp)->ioc_len != hdr.ioc_len) {
+		err = -EINVAL;
+		goto free;
+	}
+
+	return 0;
+
+free:
+	kvfree(*hdr_pp);
+	return err;
+}
+
+static int libcfs_ioctl(unsigned long cmd, void __user *uparam)
 {
 	struct libcfs_ioctl_data *data = NULL;
 	struct libcfs_ioctl_hdr *hdr;
@@ -136,30 +242,54 @@ int libcfs_ioctl(unsigned long cmd, void __user *uparam)
 		libcfs_debug_mark_buffer(data->ioc_inlbuf1);
 		break;
 
-	default: {
-		struct libcfs_ioctl_handler *hand;
-
-		err = -EINVAL;
-		down_read(&ioctl_list_sem);
-		list_for_each_entry(hand, &ioctl_list, item) {
-			err = hand->handle_ioctl(cmd, hdr);
-			if (err == -EINVAL)
-				continue;
-
-			if (!err) {
-				if (copy_to_user(uparam, hdr, hdr->ioc_len))
-					err = -EFAULT;
-			}
-			break;
-		}
-		up_read(&ioctl_list_sem);
-		break; }
+	default:
+		err = blocking_notifier_call_chain(&libcfs_ioctl_list,
+						   cmd, hdr);
+		if (!(err & NOTIFY_STOP_MASK))
+			/* No-one claimed the ioctl */
+			err = -EINVAL;
+		else
+			err = notifier_to_errno(err);
+		if (!err)
+			if (copy_to_user(uparam, hdr, hdr->ioc_len))
+				err = -EFAULT;
+		break;
 	}
 out:
 	kvfree(hdr);
 	return err;
 }
 
+static long
+libcfs_psdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	if (!capable(CAP_SYS_ADMIN))
+		return -EACCES;
+
+	if (_IOC_TYPE(cmd) != IOC_LIBCFS_TYPE ||
+	    _IOC_NR(cmd) < IOC_LIBCFS_MIN_NR  ||
+	    _IOC_NR(cmd) > IOC_LIBCFS_MAX_NR) {
+		CDEBUG(D_IOCTL, "invalid ioctl ( type %d, nr %d, size %d )\n",
+		       _IOC_TYPE(cmd), _IOC_NR(cmd), _IOC_SIZE(cmd));
+		return -EINVAL;
+	}
+
+	return libcfs_ioctl(cmd, (void __user *)arg);
+}
+
+static const struct file_operations libcfs_fops = {
+	.owner		= THIS_MODULE,
+	.unlocked_ioctl	= libcfs_psdev_ioctl,
+};
+
+static struct miscdevice libcfs_dev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "lnet",
+	.fops = &libcfs_fops,
+};
+
+static int libcfs_dev_registered;
+
 int lprocfs_call_handler(void *data, int write, loff_t *ppos,
 			 void __user *buffer, size_t *lenp,
 			 int (*handler)(void *data, int write, loff_t pos,
@@ -299,14 +429,12 @@ static int __proc_cpt_table(void *data, int write,
 	if (write)
 		return -EPERM;
 
-	LASSERT(cfs_cpt_table);
-
 	while (1) {
 		buf = kzalloc(len, GFP_KERNEL);
 		if (!buf)
 			return -ENOMEM;
 
-		rc = cfs_cpt_table_print(cfs_cpt_table, buf, len);
+		rc = cfs_cpt_table_print(cfs_cpt_tab, buf, len);
 		if (rc >= 0)
 			break;
 
@@ -496,8 +624,7 @@ static const struct file_operations *lnet_debugfs_fops_select(umode_t mode)
 	return &lnet_debugfs_file_operations_rw;
 }
 
-void lustre_insert_debugfs(struct ctl_table *table,
-			   const struct lnet_debugfs_symlink_def *symlinks)
+void lustre_insert_debugfs(struct ctl_table *table)
 {
 	if (!lnet_debugfs_root)
 		lnet_debugfs_root = debugfs_create_dir("lnet", NULL);
@@ -506,19 +633,24 @@ void lustre_insert_debugfs(struct ctl_table *table,
 	if (IS_ERR_OR_NULL(lnet_debugfs_root))
 		return;
 
-	/* We don't save the dentry returned in next two calls, because
-	 * we don't call debugfs_remove() but rather remove_recursive()
+	/*
+	 * We don't save the dentry returned because we don't call
+	 * debugfs_remove() but rather remove_recursive()
 	 */
 	for (; table->procname; table++)
 		debugfs_create_file(table->procname, table->mode,
 				    lnet_debugfs_root, table,
 				    lnet_debugfs_fops_select(table->mode));
+}
+EXPORT_SYMBOL_GPL(lustre_insert_debugfs);
 
+static void lustre_insert_debugfs_links(
+	const struct lnet_debugfs_symlink_def *symlinks)
+{
 	for (; symlinks && symlinks->name; symlinks++)
 		debugfs_create_symlink(symlinks->name, lnet_debugfs_root,
 				       symlinks->target);
 }
-EXPORT_SYMBOL_GPL(lustre_insert_debugfs);
 
 static void lustre_remove_debugfs(void)
 {
@@ -527,49 +659,72 @@ static void lustre_remove_debugfs(void)
 	lnet_debugfs_root = NULL;
 }
 
-static int libcfs_init(void)
+static DEFINE_MUTEX(libcfs_startup);
+static int libcfs_active;
+
+int libcfs_setup(void)
 {
-	int rc;
+	int rc = -EINVAL;
+
+	mutex_lock(&libcfs_startup);
+	if (libcfs_active)
+		goto out;
+
+	if (!libcfs_dev_registered)
+		goto err;
 
 	rc = libcfs_debug_init(5 * 1024 * 1024);
 	if (rc < 0) {
 		pr_err("LustreError: libcfs_debug_init: %d\n", rc);
-		return rc;
+		goto err;
 	}
 
 	rc = cfs_cpu_init();
 	if (rc)
-		goto cleanup_debug;
-
-	rc = misc_register(&libcfs_dev);
-	if (rc) {
-		CERROR("misc_register: error %d\n", rc);
-		goto cleanup_cpu;
-	}
+		goto err;
 
 	cfs_rehash_wq = alloc_workqueue("cfs_rh", WQ_SYSFS, 4);
 	if (!cfs_rehash_wq) {
 		CERROR("Failed to start rehash workqueue.\n");
 		rc = -ENOMEM;
-		goto cleanup_deregister;
+		goto err;
 	}
 
 	rc = cfs_crypto_register();
 	if (rc) {
 		CERROR("cfs_crypto_register: error %d\n", rc);
-		goto cleanup_deregister;
+		goto err;
 	}
 
-	lustre_insert_debugfs(lnet_table, lnet_debugfs_symlinks);
+	lustre_insert_debugfs(lnet_table);
+	if (!IS_ERR_OR_NULL(lnet_debugfs_root))
+		lustre_insert_debugfs_links(lnet_debugfs_symlinks);
 
 	CDEBUG(D_OTHER, "portals setup OK\n");
+out:
+	libcfs_active = 1;
+	mutex_unlock(&libcfs_startup);
 	return 0;
- cleanup_deregister:
-	misc_deregister(&libcfs_dev);
-cleanup_cpu:
+err:
+	cfs_crypto_unregister();
+	if (cfs_rehash_wq)
+		destroy_workqueue(cfs_rehash_wq);
 	cfs_cpu_fini();
- cleanup_debug:
 	libcfs_debug_cleanup();
+	mutex_unlock(&libcfs_startup);
+	return rc;
+}
+EXPORT_SYMBOL(libcfs_setup);
+
+static int libcfs_init(void)
+{
+	int rc;
+
+	rc = misc_register(&libcfs_dev);
+	if (rc)
+		CERROR("misc_register: error %d\n", rc);
+	else
+		libcfs_dev_registered = 1;
 	return rc;
 }
 
@@ -579,14 +734,13 @@ static void libcfs_exit(void)
 
 	lustre_remove_debugfs();
 
-	if (cfs_rehash_wq) {
+	if (cfs_rehash_wq)
 		destroy_workqueue(cfs_rehash_wq);
-		cfs_rehash_wq = NULL;
-	}
 
 	cfs_crypto_unregister();
 
-	misc_deregister(&libcfs_dev);
+	if (libcfs_dev_registered)
+		misc_deregister(&libcfs_dev);
 
 	cfs_cpu_fini();
 
diff --git a/drivers/staging/lustre/lnet/libcfs/tracefile.c b/drivers/staging/lustre/lnet/libcfs/tracefile.c
index 4affca750bc5..7ca562e156f0 100644
--- a/drivers/staging/lustre/lnet/libcfs/tracefile.c
+++ b/drivers/staging/lustre/lnet/libcfs/tracefile.c
@@ -39,9 +39,15 @@
 #define DEBUG_SUBSYSTEM S_LNET
 #define LUSTRE_TRACEFILE_PRIVATE
 #define pr_fmt(fmt) "Lustre: " fmt
-#include "tracefile.h"
 
-#include <linux/libcfs/libcfs.h>
+#include <linux/ratelimit.h>
+#include <linux/highmem.h>
+#include <linux/ctype.h>
+#include <linux/kthread.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include "tracefile.h"
 
 /* XXX move things up to the top, comment */
 union cfs_trace_data_union (*cfs_trace_data[TCD_MAX_TYPES])[NR_CPUS] __cacheline_aligned;
@@ -114,7 +120,7 @@ static struct cfs_trace_page *cfs_tage_alloc(gfp_t gfp)
 	struct cfs_trace_page *tage;
 
 	/* My caller is trying to free memory */
-	if (!in_interrupt() && memory_pressure_get())
+	if (!in_interrupt() && (current->flags & PF_MEMALLOC))
 		return NULL;
 
 	/*
@@ -192,7 +198,8 @@ cfs_trace_get_tage_try(struct cfs_trace_cpu_data *tcd, unsigned long len)
 		} else {
 			tage = cfs_tage_alloc(GFP_ATOMIC);
 			if (unlikely(!tage)) {
-				if (!memory_pressure_get() || in_interrupt())
+				if (!(current->flags & PF_MEMALLOC) ||
+				    in_interrupt())
 					pr_warn_ratelimited("cannot allocate a tage (%ld)\n",
 							    tcd->tcd_cur_pages);
 				return NULL;
@@ -328,7 +335,7 @@ int libcfs_debug_vmsg2(struct libcfs_debug_msg_data *msgdata,
 		goto console;
 	}
 
-	depth = __current_nesting_level();
+	depth = 0;
 	known_size = strlen(file) + 1 + depth;
 	if (msgdata->msg_fn)
 		known_size += strlen(msgdata->msg_fn) + 1;
@@ -431,7 +438,7 @@ int libcfs_debug_vmsg2(struct libcfs_debug_msg_data *msgdata,
 	if (cdls) {
 		if (libcfs_console_ratelimit &&
 		    cdls->cdls_next &&		/* not first time ever */
-		    !cfs_time_after(cfs_time_current(), cdls->cdls_next)) {
+		    !time_after(jiffies, cdls->cdls_next)) {
 			/* skipping a console message */
 			cdls->cdls_count++;
 			if (tcd)
@@ -439,9 +446,9 @@ int libcfs_debug_vmsg2(struct libcfs_debug_msg_data *msgdata,
 			return 1;
 		}
 
-		if (cfs_time_after(cfs_time_current(),
-				   cdls->cdls_next + libcfs_console_max_delay +
-				   10 * HZ)) {
+		if (time_after(jiffies,
+			       cdls->cdls_next + libcfs_console_max_delay +
+			       10 * HZ)) {
 			/* last timeout was a long time ago */
 			cdls->cdls_delay /= libcfs_console_backoff * 4;
 		} else {
@@ -454,7 +461,7 @@ int libcfs_debug_vmsg2(struct libcfs_debug_msg_data *msgdata,
 			cdls->cdls_delay = libcfs_console_max_delay;
 
 		/* ensure cdls_next is never zero after it's been seen */
-		cdls->cdls_next = (cfs_time_current() + cdls->cdls_delay) | 1;
+		cdls->cdls_next = (jiffies + cdls->cdls_delay) | 1;
 	}
 
 	if (tcd) {
diff --git a/drivers/staging/lustre/lnet/libcfs/tracefile.h b/drivers/staging/lustre/lnet/libcfs/tracefile.h
index a29d6eb3a785..0608240d897f 100644
--- a/drivers/staging/lustre/lnet/libcfs/tracefile.h
+++ b/drivers/staging/lustre/lnet/libcfs/tracefile.h
@@ -34,6 +34,12 @@
 #ifndef __LIBCFS_TRACEFILE_H__
 #define __LIBCFS_TRACEFILE_H__
 
+#include <linux/spinlock.h>
+#include <linux/list.h>
+#include <linux/cache.h>
+#include <linux/threads.h>
+#include <linux/limits.h>
+#include <linux/smp.h>
 #include <linux/libcfs/libcfs.h>
 
 enum cfs_trace_buf_type {
@@ -49,6 +55,11 @@ enum cfs_trace_buf_type {
 extern char cfs_tracefile[TRACEFILE_NAME_SIZE];
 extern long long cfs_tracefile_size;
 
+/**
+ * The path of debug log dump upcall script.
+ */
+extern char lnet_debug_log_upcall[1024];
+
 void libcfs_run_debug_log_upcall(char *file);
 
 int  cfs_tracefile_init_arch(void);
diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
index 90266be0132d..f9ed6977056c 100644
--- a/drivers/staging/lustre/lnet/lnet/api-ni.c
+++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
@@ -1413,9 +1413,9 @@ int lnet_lib_init(void)
 
 	memset(&the_lnet, 0, sizeof(the_lnet));
 
-	/* refer to global cfs_cpt_table for now */
-	the_lnet.ln_cpt_table	= cfs_cpt_table;
-	the_lnet.ln_cpt_number	= cfs_cpt_number(cfs_cpt_table);
+	/* refer to global cfs_cpt_tab for now */
+	the_lnet.ln_cpt_table	= cfs_cpt_tab;
+	the_lnet.ln_cpt_number	= cfs_cpt_number(cfs_cpt_tab);
 
 	LASSERT(the_lnet.ln_cpt_number > 0);
 	if (the_lnet.ln_cpt_number > LNET_CPT_MAX) {
diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
index 0aea268a4f1c..55ecc1998b7e 100644
--- a/drivers/staging/lustre/lnet/lnet/config.c
+++ b/drivers/staging/lustre/lnet/lnet/config.c
@@ -34,6 +34,7 @@
 #define DEBUG_SUBSYSTEM S_LNET
 #include <linux/nsproxy.h>
 #include <net/net_namespace.h>
+#include <linux/ctype.h>
 #include <linux/lnet/lib-lnet.h>
 
 struct lnet_text_buf {	    /* tmp struct for parsing routes */
diff --git a/drivers/staging/lustre/lnet/lnet/lib-eq.c b/drivers/staging/lustre/lnet/lnet/lib-eq.c
index ea53b5cb3f72..c78e70373ab4 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-eq.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-eq.c
@@ -225,7 +225,7 @@ lnet_eq_enqueue_event(struct lnet_eq *eq, struct lnet_event *ev)
 	lnet_eq_wait_lock();
 	ev->sequence = eq->eq_enq_seq++;
 
-	LASSERT(eq->eq_size == LOWEST_BIT_SET(eq->eq_size));
+	LASSERT(is_power_of_2(eq->eq_size));
 	index = ev->sequence & (eq->eq_size - 1);
 
 	eq->eq_events[index] = *ev;
diff --git a/drivers/staging/lustre/lnet/lnet/lib-move.c b/drivers/staging/lustre/lnet/lnet/lib-move.c
index ed43b3f4b114..f8eaf8ff8d8d 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-move.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-move.c
@@ -501,7 +501,7 @@ lnet_ni_query_locked(struct lnet_ni *ni, struct lnet_peer *lp)
 	ni->ni_lnd->lnd_query(ni, lp->lp_nid, &last_alive);
 	lnet_net_lock(lp->lp_cpt);
 
-	lp->lp_last_query = cfs_time_current();
+	lp->lp_last_query = jiffies;
 
 	if (last_alive) /* NI has updated timestamp */
 		lp->lp_last_alive = last_alive;
@@ -520,12 +520,11 @@ lnet_peer_is_alive(struct lnet_peer *lp, unsigned long now)
 	 * ignore the initial assumed death (see lnet_peers_start_down()).
 	 */
 	if (!lp->lp_alive && lp->lp_alive_count > 0 &&
-	    cfs_time_aftereq(lp->lp_timestamp, lp->lp_last_alive))
+	    time_after_eq(lp->lp_timestamp, lp->lp_last_alive))
 		return 0;
 
-	deadline = cfs_time_add(lp->lp_last_alive,
-				lp->lp_ni->ni_peertimeout * HZ);
-	alive = cfs_time_after(deadline, now);
+	deadline = lp->lp_last_alive + lp->lp_ni->ni_peertimeout * HZ;
+	alive = time_after(deadline, now);
 
 	/* Update obsolete lp_alive except for routers assumed to be dead
 	 * initially, because router checker would update aliveness in this
@@ -545,7 +544,7 @@ lnet_peer_is_alive(struct lnet_peer *lp, unsigned long now)
 static int
 lnet_peer_alive_locked(struct lnet_peer *lp)
 {
-	unsigned long now = cfs_time_current();
+	unsigned long now = jiffies;
 
 	if (!lnet_peer_aliveness_enabled(lp))
 		return -ENODEV;
@@ -561,8 +560,7 @@ lnet_peer_alive_locked(struct lnet_peer *lp)
 		static const int lnet_queryinterval = 1;
 
 		unsigned long next_query =
-			   cfs_time_add(lp->lp_last_query,
-					lnet_queryinterval * HZ);
+			   lp->lp_last_query + lnet_queryinterval * HZ;
 
 		if (time_before(now, next_query)) {
 			if (lp->lp_alive)
@@ -2016,7 +2014,7 @@ LNetPut(lnet_nid_t self, struct lnet_handle_md mdh, enum lnet_ack_req ack,
 		       libcfs_id2str(target));
 		return -ENOMEM;
 	}
-	msg->msg_vmflush = !!memory_pressure_get();
+	msg->msg_vmflush = !!(current->flags & PF_MEMALLOC);
 
 	cpt = lnet_cpt_of_cookie(mdh.cookie);
 	lnet_res_lock(cpt);
diff --git a/drivers/staging/lustre/lnet/lnet/lib-socket.c b/drivers/staging/lustre/lnet/lnet/lib-socket.c
index 1bee667802b0..9b61260155f2 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-socket.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-socket.c
@@ -41,7 +41,6 @@
 #include <linux/syscalls.h>
 #include <net/sock.h>
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/lnet/lib-lnet.h>
 
 static int
diff --git a/drivers/staging/lustre/lnet/lnet/module.c b/drivers/staging/lustre/lnet/lnet/module.c
index c0c4723f72fd..9d06664f0c17 100644
--- a/drivers/staging/lustre/lnet/lnet/module.c
+++ b/drivers/staging/lustre/lnet/lnet/module.c
@@ -136,30 +136,37 @@ lnet_dyn_unconfigure(struct libcfs_ioctl_hdr *hdr)
 }
 
 static int
-lnet_ioctl(unsigned int cmd, struct libcfs_ioctl_hdr *hdr)
+lnet_ioctl(struct notifier_block *nb,
+	   unsigned long cmd, void *vdata)
 {
 	int rc;
+	struct libcfs_ioctl_hdr *hdr = vdata;
 
 	switch (cmd) {
 	case IOC_LIBCFS_CONFIGURE: {
 		struct libcfs_ioctl_data *data =
 			(struct libcfs_ioctl_data *)hdr;
 
-		if (data->ioc_hdr.ioc_len < sizeof(*data))
-			return -EINVAL;
-
-		the_lnet.ln_nis_from_mod_params = data->ioc_flags;
-		return lnet_configure(NULL);
+		if (data->ioc_hdr.ioc_len < sizeof(*data)) {
+			rc = -EINVAL;
+		} else {
+			the_lnet.ln_nis_from_mod_params = data->ioc_flags;
+			rc = lnet_configure(NULL);
+		}
+		break;
 	}
 
 	case IOC_LIBCFS_UNCONFIGURE:
-		return lnet_unconfigure();
+		rc = lnet_unconfigure();
+		break;
 
 	case IOC_LIBCFS_ADD_NET:
-		return lnet_dyn_configure(hdr);
+		rc = lnet_dyn_configure(hdr);
+		break;
 
 	case IOC_LIBCFS_DEL_NET:
-		return lnet_dyn_unconfigure(hdr);
+		rc = lnet_dyn_unconfigure(hdr);
+		break;
 
 	default:
 		/*
@@ -172,11 +179,14 @@ lnet_ioctl(unsigned int cmd, struct libcfs_ioctl_hdr *hdr)
 			rc = LNetCtl(cmd, hdr);
 			LNetNIFini();
 		}
-		return rc;
+		break;
 	}
+	return notifier_from_ioctl_errno(rc);
 }
 
-static DECLARE_IOCTL_HANDLER(lnet_ioctl_handler, lnet_ioctl);
+static struct notifier_block lnet_ioctl_handler = {
+	.notifier_call = lnet_ioctl,
+};
 
 static int __init lnet_init(void)
 {
@@ -184,13 +194,18 @@ static int __init lnet_init(void)
 
 	mutex_init(&lnet_config_mutex);
 
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	rc = lnet_lib_init();
 	if (rc) {
 		CERROR("lnet_lib_init: error %d\n", rc);
 		return rc;
 	}
 
-	rc = libcfs_register_ioctl(&lnet_ioctl_handler);
+	rc = blocking_notifier_chain_register(&libcfs_ioctl_list,
+					      &lnet_ioctl_handler);
 	LASSERT(!rc);
 
 	if (config_on_load) {
@@ -208,7 +223,8 @@ static void __exit lnet_exit(void)
 {
 	int rc;
 
-	rc = libcfs_deregister_ioctl(&lnet_ioctl_handler);
+	rc = blocking_notifier_chain_unregister(&libcfs_ioctl_list,
+						&lnet_ioctl_handler);
 	LASSERT(!rc);
 
 	lnet_lib_exit();
diff --git a/drivers/staging/lustre/lnet/lnet/net_fault.c b/drivers/staging/lustre/lnet/lnet/net_fault.c
index a63b7941d435..0066394b0bb0 100644
--- a/drivers/staging/lustre/lnet/lnet/net_fault.c
+++ b/drivers/staging/lustre/lnet/lnet/net_fault.c
@@ -169,9 +169,9 @@ lnet_drop_rule_add(struct lnet_fault_attr *attr)
 
 	rule->dr_attr = *attr;
 	if (attr->u.drop.da_interval) {
-		rule->dr_time_base = cfs_time_shift(attr->u.drop.da_interval);
-		rule->dr_drop_time = cfs_time_shift(
-			prandom_u32_max(attr->u.drop.da_interval));
+		rule->dr_time_base = jiffies + attr->u.drop.da_interval * HZ;
+		rule->dr_drop_time = jiffies +
+			prandom_u32_max(attr->u.drop.da_interval) * HZ;
 	} else {
 		rule->dr_drop_at = prandom_u32_max(attr->u.drop.da_rate);
 	}
@@ -279,9 +279,9 @@ lnet_drop_rule_reset(void)
 		if (attr->u.drop.da_rate) {
 			rule->dr_drop_at = prandom_u32_max(attr->u.drop.da_rate);
 		} else {
-			rule->dr_drop_time = cfs_time_shift(
-				prandom_u32_max(attr->u.drop.da_interval));
-			rule->dr_time_base = cfs_time_shift(attr->u.drop.da_interval);
+			rule->dr_drop_time = jiffies +
+				prandom_u32_max(attr->u.drop.da_interval) * HZ;
+			rule->dr_time_base = jiffies + attr->u.drop.da_interval * HZ;
 		}
 		spin_unlock(&rule->dr_lock);
 	}
@@ -306,12 +306,12 @@ drop_rule_match(struct lnet_drop_rule *rule, lnet_nid_t src,
 	/* match this rule, check drop rate now */
 	spin_lock(&rule->dr_lock);
 	if (rule->dr_drop_time) { /* time based drop */
-		unsigned long now = cfs_time_current();
+		unsigned long now = jiffies;
 
 		rule->dr_stat.fs_count++;
-		drop = cfs_time_aftereq(now, rule->dr_drop_time);
+		drop = time_after_eq(now, rule->dr_drop_time);
 		if (drop) {
-			if (cfs_time_after(now, rule->dr_time_base))
+			if (time_after(now, rule->dr_time_base))
 				rule->dr_time_base = now;
 
 			rule->dr_drop_time = rule->dr_time_base +
@@ -472,12 +472,12 @@ delay_rule_match(struct lnet_delay_rule *rule, lnet_nid_t src,
 	/* match this rule, check delay rate now */
 	spin_lock(&rule->dl_lock);
 	if (rule->dl_delay_time) { /* time based delay */
-		unsigned long now = cfs_time_current();
+		unsigned long now = jiffies;
 
 		rule->dl_stat.fs_count++;
-		delay = cfs_time_aftereq(now, rule->dl_delay_time);
+		delay = time_after_eq(now, rule->dl_delay_time);
 		if (delay) {
-			if (cfs_time_after(now, rule->dl_time_base))
+			if (time_after(now, rule->dl_time_base))
 				rule->dl_time_base = now;
 
 			rule->dl_delay_time = rule->dl_time_base +
@@ -513,7 +513,7 @@ delay_rule_match(struct lnet_delay_rule *rule, lnet_nid_t src,
 
 	list_add_tail(&msg->msg_list, &rule->dl_msg_list);
 	msg->msg_delay_send = round_timeout(
-			cfs_time_shift(attr->u.delay.la_latency));
+			jiffies + attr->u.delay.la_latency * HZ);
 	if (rule->dl_msg_send == -1) {
 		rule->dl_msg_send = msg->msg_delay_send;
 		mod_timer(&rule->dl_timer, rule->dl_msg_send);
@@ -562,7 +562,7 @@ delayed_msg_check(struct lnet_delay_rule *rule, bool all,
 {
 	struct lnet_msg *msg;
 	struct lnet_msg *tmp;
-	unsigned long now = cfs_time_current();
+	unsigned long now = jiffies;
 
 	if (!all && rule->dl_msg_send > now)
 		return;
@@ -767,9 +767,9 @@ lnet_delay_rule_add(struct lnet_fault_attr *attr)
 
 	rule->dl_attr = *attr;
 	if (attr->u.delay.la_interval) {
-		rule->dl_time_base = cfs_time_shift(attr->u.delay.la_interval);
-		rule->dl_delay_time = cfs_time_shift(
-			prandom_u32_max(attr->u.delay.la_interval));
+		rule->dl_time_base = jiffies + attr->u.delay.la_interval * HZ;
+		rule->dl_delay_time = jiffies + 
+			prandom_u32_max(attr->u.delay.la_interval) * HZ;
 	} else {
 		rule->dl_delay_at = prandom_u32_max(attr->u.delay.la_rate);
 	}
@@ -920,9 +920,9 @@ lnet_delay_rule_reset(void)
 			rule->dl_delay_at = prandom_u32_max(attr->u.delay.la_rate);
 		} else {
 			rule->dl_delay_time =
-				cfs_time_shift(prandom_u32_max(
-						       attr->u.delay.la_interval));
-			rule->dl_time_base = cfs_time_shift(attr->u.delay.la_interval);
+				jiffies + prandom_u32_max(
+					attr->u.delay.la_interval) * HZ;
+			rule->dl_time_base = jiffies + attr->u.delay.la_interval * HZ;
 		}
 		spin_unlock(&rule->dl_lock);
 	}
diff --git a/drivers/staging/lustre/lnet/lnet/nidstrings.c b/drivers/staging/lustre/lnet/lnet/nidstrings.c
index 3aba1421c741..0f6c3fa16c65 100644
--- a/drivers/staging/lustre/lnet/lnet/nidstrings.c
+++ b/drivers/staging/lustre/lnet/lnet/nidstrings.c
@@ -37,7 +37,10 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
+#include <linux/spinlock.h>
+#include <linux/slab.h>
 #include <linux/libcfs/libcfs.h>
+#include <linux/libcfs/libcfs_string.h>
 #include <uapi/linux/lnet/nidstr.h>
 
 /* max value for numeric network address */
diff --git a/drivers/staging/lustre/lnet/lnet/peer.c b/drivers/staging/lustre/lnet/lnet/peer.c
index 3d4caa609c83..58294149f7b2 100644
--- a/drivers/staging/lustre/lnet/lnet/peer.c
+++ b/drivers/staging/lustre/lnet/lnet/peer.c
@@ -315,7 +315,7 @@ lnet_nid2peer_locked(struct lnet_peer **lpp, lnet_nid_t nid, int cpt)
 	lp->lp_alive_count = 0;
 	lp->lp_timestamp = 0;
 	lp->lp_alive = !lnet_peers_start_down(); /* 1 bit!! */
-	lp->lp_last_alive = cfs_time_current(); /* assumes alive */
+	lp->lp_last_alive = jiffies; /* assumes alive */
 	lp->lp_last_query = 0; /* haven't asked NI yet */
 	lp->lp_ping_timestamp = 0;
 	lp->lp_ping_feats = LNET_PING_FEAT_INVAL;
diff --git a/drivers/staging/lustre/lnet/lnet/router.c b/drivers/staging/lustre/lnet/lnet/router.c
index a3c3f4959f46..6267d5e4bbd6 100644
--- a/drivers/staging/lustre/lnet/lnet/router.c
+++ b/drivers/staging/lustre/lnet/lnet/router.c
@@ -765,7 +765,7 @@ lnet_router_checker_event(struct lnet_event *event)
 	 * we ping alive routers to try to detect router death before
 	 * apps get burned).
 	 */
-	lnet_notify_locked(lp, 1, !event->status, cfs_time_current());
+	lnet_notify_locked(lp, 1, !event->status, jiffies);
 
 	/*
 	 * The router checker will wake up very shortly and do the
@@ -976,13 +976,13 @@ static void
 lnet_ping_router_locked(struct lnet_peer *rtr)
 {
 	struct lnet_rc_data *rcd = NULL;
-	unsigned long now = cfs_time_current();
+	unsigned long now = jiffies;
 	int secs;
 
 	lnet_peer_addref_locked(rtr);
 
 	if (rtr->lp_ping_deadline && /* ping timed out? */
-	    cfs_time_after(now, rtr->lp_ping_deadline))
+	    time_after(now, rtr->lp_ping_deadline))
 		lnet_notify_locked(rtr, 1, 0, now);
 
 	/* Run any outstanding notifications */
@@ -1010,8 +1010,7 @@ lnet_ping_router_locked(struct lnet_peer *rtr)
 	       rtr->lp_alive, rtr->lp_alive_count, rtr->lp_ping_timestamp);
 
 	if (secs && !rtr->lp_ping_notsent &&
-	    cfs_time_after(now, cfs_time_add(rtr->lp_ping_timestamp,
-					     secs * HZ))) {
+	    time_after(now, rtr->lp_ping_timestamp + secs * HZ)) {
 		int rc;
 		struct lnet_process_id id;
 		struct lnet_handle_md mdh;
@@ -1027,7 +1026,7 @@ lnet_ping_router_locked(struct lnet_peer *rtr)
 
 		if (!rtr->lp_ping_deadline) {
 			rtr->lp_ping_deadline =
-				cfs_time_shift(router_ping_timeout);
+				jiffies + router_ping_timeout * HZ;
 		}
 
 		lnet_net_unlock(rtr->lp_cpt);
@@ -1730,7 +1729,7 @@ int
 lnet_notify(struct lnet_ni *ni, lnet_nid_t nid, int alive, unsigned long when)
 {
 	struct lnet_peer *lp = NULL;
-	unsigned long now = cfs_time_current();
+	unsigned long now = jiffies;
 	int cpt = lnet_cpt_of_nid(nid);
 
 	LASSERT(!in_interrupt());
@@ -1749,11 +1748,11 @@ lnet_notify(struct lnet_ni *ni, lnet_nid_t nid, int alive, unsigned long when)
 	}
 
 	/* can't do predictions... */
-	if (cfs_time_after(when, now)) {
+	if (time_after(when, now)) {
 		CWARN("Ignoring prediction from %s of %s %s %ld seconds in the future\n",
 		      !ni ? "userspace" : libcfs_nid2str(ni->ni_nid),
 		      libcfs_nid2str(nid), alive ? "up" : "down",
-		      cfs_duration_sec(cfs_time_sub(when, now)));
+		      (when - now) / HZ);
 		return -EINVAL;
 	}
 
diff --git a/drivers/staging/lustre/lnet/lnet/router_proc.c b/drivers/staging/lustre/lnet/lnet/router_proc.c
index 1a71ffebc889..ae4b7f5953a0 100644
--- a/drivers/staging/lustre/lnet/lnet/router_proc.c
+++ b/drivers/staging/lustre/lnet/lnet/router_proc.c
@@ -20,7 +20,6 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/lnet/lib-lnet.h>
 
 /*
@@ -324,15 +323,14 @@ static int proc_lnet_routers(struct ctl_table *table, int write,
 
 		if (peer) {
 			lnet_nid_t nid = peer->lp_nid;
-			unsigned long now = cfs_time_current();
+			unsigned long now = jiffies;
 			unsigned long deadline = peer->lp_ping_deadline;
 			int nrefs = peer->lp_refcount;
 			int nrtrrefs = peer->lp_rtr_refcount;
 			int alive_cnt = peer->lp_alive_count;
 			int alive = peer->lp_alive;
 			int pingsent = !peer->lp_ping_notsent;
-			int last_ping = cfs_duration_sec(cfs_time_sub(now,
-						     peer->lp_ping_timestamp));
+			int last_ping = (now - peer->lp_ping_timestamp) / HZ;
 			int down_ni = 0;
 			struct lnet_route *rtr;
 
@@ -364,7 +362,7 @@ static int proc_lnet_routers(struct ctl_table *table, int write,
 					      nrefs, nrtrrefs, alive_cnt,
 					      alive ? "up" : "down", last_ping,
 					      pingsent,
-					      cfs_duration_sec(cfs_time_sub(deadline, now)),
+					      (deadline - now) / HZ,
 					      down_ni, libcfs_nid2str(nid));
 			LASSERT(tmpstr + tmpsiz - s > 0);
 		}
@@ -509,11 +507,11 @@ static int proc_lnet_peers(struct ctl_table *table, int write,
 				aliveness = peer->lp_alive ? "up" : "down";
 
 			if (lnet_peer_aliveness_enabled(peer)) {
-				unsigned long now = cfs_time_current();
+				unsigned long now = jiffies;
 				long delta;
 
-				delta = cfs_time_sub(now, peer->lp_last_alive);
-				lastalive = cfs_duration_sec(delta);
+				delta = now - peer->lp_last_alive;
+				lastalive = (delta) / HZ;
 
 				/* No need to mess up peers contents with
 				 * arbitrarily long integers - it suffices to
@@ -901,7 +899,7 @@ static struct ctl_table lnet_table[] = {
 
 void lnet_router_debugfs_init(void)
 {
-	lustre_insert_debugfs(lnet_table, NULL);
+	lustre_insert_debugfs(lnet_table);
 }
 
 void lnet_router_debugfs_fini(void)
diff --git a/drivers/staging/lustre/lnet/selftest/conctl.c b/drivers/staging/lustre/lnet/selftest/conctl.c
index a2d8092bdeb7..906d82d90c0c 100644
--- a/drivers/staging/lustre/lnet/selftest/conctl.c
+++ b/drivers/staging/lustre/lnet/selftest/conctl.c
@@ -37,7 +37,6 @@
  * Author: Liang Zhen <liangzhen@clusterfs.com>
  */
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/lnet/lib-lnet.h>
 #include <uapi/linux/lnet/lnetst.h>
 #include "console.h"
@@ -680,32 +679,34 @@ static int lst_test_add_ioctl(struct lstio_test_args *args)
 }
 
 int
-lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr)
+lstcon_ioctl_entry(struct notifier_block *nb,
+		   unsigned long cmd, void *vdata)
 {
-	char *buf;
+	struct libcfs_ioctl_hdr *hdr = vdata;
+	char *buf = NULL;
 	struct libcfs_ioctl_data *data;
 	int opc;
-	int rc;
+	int rc = -EINVAL;
 
 	if (cmd != IOC_LIBCFS_LNETST)
-		return -EINVAL;
+		goto err;
 
 	data = container_of(hdr, struct libcfs_ioctl_data, ioc_hdr);
 
 	opc = data->ioc_u32[0];
 
 	if (data->ioc_plen1 > PAGE_SIZE)
-		return -EINVAL;
+		goto err;
 
 	buf = kmalloc(data->ioc_plen1, GFP_KERNEL);
+	rc = -ENOMEM;
 	if (!buf)
-		return -ENOMEM;
+		goto err;
 
 	/* copy in parameter */
-	if (copy_from_user(buf, data->ioc_pbuf1, data->ioc_plen1)) {
-		kfree(buf);
-		return -EFAULT;
-	}
+	rc = -EFAULT;
+	if (copy_from_user(buf, data->ioc_pbuf1, data->ioc_plen1))
+		goto err;
 
 	mutex_lock(&console_session.ses_mutex);
 
@@ -785,6 +786,7 @@ lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr)
 		break;
 	default:
 		rc = -EINVAL;
+		goto out;
 	}
 
 	if (copy_to_user(data->ioc_pbuf2, &console_session.ses_trans_stat,
@@ -792,8 +794,8 @@ lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr)
 		rc = -EFAULT;
 out:
 	mutex_unlock(&console_session.ses_mutex);
-
+err:
 	kfree(buf);
 
-	return rc;
+	return notifier_from_ioctl_errno(rc);
 }
diff --git a/drivers/staging/lustre/lnet/selftest/conrpc.c b/drivers/staging/lustre/lnet/selftest/conrpc.c
index 6dcc966b293b..0dabade3d091 100644
--- a/drivers/staging/lustre/lnet/selftest/conrpc.c
+++ b/drivers/staging/lustre/lnet/selftest/conrpc.c
@@ -37,7 +37,6 @@
  * Author: Liang Zhen <liang@whamcloud.com>
  */
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/lnet/lib-lnet.h>
 #include "timer.h"
 #include "conrpc.h"
@@ -75,7 +74,7 @@ lstcon_rpc_done(struct srpc_client_rpc *rpc)
 		/* not aborted */
 		LASSERT(!crpc->crp_status);
 
-		crpc->crp_stamp = cfs_time_current();
+		crpc->crp_stamp = jiffies;
 		crpc->crp_status = rpc->crpc_status;
 	}
 
@@ -297,14 +296,14 @@ lstcon_rpc_trans_abort(struct lstcon_rpc_trans *trans, int error)
 		if (!crpc->crp_posted || /* not posted */
 		    crpc->crp_stamp) {	 /* rpc done or aborted already */
 			if (!crpc->crp_stamp) {
-				crpc->crp_stamp = cfs_time_current();
+				crpc->crp_stamp = jiffies;
 				crpc->crp_status = -EINTR;
 			}
 			spin_unlock(&rpc->crpc_lock);
 			continue;
 		}
 
-		crpc->crp_stamp = cfs_time_current();
+		crpc->crp_stamp = jiffies;
 		crpc->crp_status = error;
 
 		spin_unlock(&rpc->crpc_lock);
@@ -315,7 +314,7 @@ lstcon_rpc_trans_abort(struct lstcon_rpc_trans *trans, int error)
 			continue;
 
 		nd = crpc->crp_node;
-		if (cfs_time_after(nd->nd_stamp, crpc->crp_stamp))
+		if (time_after(nd->nd_stamp, crpc->crp_stamp))
 			continue;
 
 		nd->nd_stamp = crpc->crp_stamp;
@@ -404,7 +403,7 @@ lstcon_rpc_get_reply(struct lstcon_rpc *crpc, struct srpc_msg **msgpp)
 		crpc->crp_unpacked = 1;
 	}
 
-	if (cfs_time_after(nd->nd_stamp, crpc->crp_stamp))
+	if (time_after(nd->nd_stamp, crpc->crp_stamp))
 		return 0;
 
 	nd->nd_stamp = crpc->crp_stamp;
@@ -500,8 +499,8 @@ lstcon_rpc_trans_interpreter(struct lstcon_rpc_trans *trans,
 
 		nd = crpc->crp_node;
 
-		dur = (long)cfs_time_sub(crpc->crp_stamp,
-		      (unsigned long)console_session.ses_id.ses_stamp);
+		dur = (long)(crpc->crp_stamp -
+			     (unsigned long)console_session.ses_id.ses_stamp);
 		jiffies_to_timeval(dur, &tv);
 
 		if (copy_to_user(&ent->rpe_peer, &nd->nd_id,
diff --git a/drivers/staging/lustre/lnet/selftest/conrpc.h b/drivers/staging/lustre/lnet/selftest/conrpc.h
index 374a5f31ef6f..ce2f92d04838 100644
--- a/drivers/staging/lustre/lnet/selftest/conrpc.h
+++ b/drivers/staging/lustre/lnet/selftest/conrpc.h
@@ -40,7 +40,6 @@
 #ifndef __LST_CONRPC_H__
 #define __LST_CONRPC_H__
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/lnet/lib-types.h>
 #include <uapi/linux/lnet/lnetst.h>
 #include "rpc.h"
diff --git a/drivers/staging/lustre/lnet/selftest/console.c b/drivers/staging/lustre/lnet/selftest/console.c
index 1acd5cb324b1..3c1c1b5997e0 100644
--- a/drivers/staging/lustre/lnet/selftest/console.c
+++ b/drivers/staging/lustre/lnet/selftest/console.c
@@ -37,7 +37,6 @@
  * Author: Liang Zhen <liangzhen@clusterfs.com>
  */
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/lnet/lib-lnet.h>
 #include "console.h"
 #include "conrpc.h"
@@ -98,7 +97,7 @@ lstcon_node_find(struct lnet_process_id id, struct lstcon_node **ndpp,
 
 	ndl->ndl_node->nd_ref = 1;
 	ndl->ndl_node->nd_id = id;
-	ndl->ndl_node->nd_stamp = cfs_time_current();
+	ndl->ndl_node->nd_stamp = jiffies;
 	ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 	ndl->ndl_node->nd_timeout = 0;
 	memset(&ndl->ndl_node->nd_ping, 0, sizeof(struct lstcon_rpc));
@@ -1701,7 +1700,7 @@ lstcon_new_session_id(struct lst_sid *sid)
 
 	LNetGetId(1, &id);
 	sid->ses_nid = id.nid;
-	sid->ses_stamp = cfs_time_current();
+	sid->ses_stamp = jiffies;
 }
 
 int
@@ -1996,7 +1995,9 @@ static void lstcon_init_acceptor_service(void)
 	lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
 }
 
-static DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
+static struct notifier_block lstcon_ioctl_handler = {
+	.notifier_call = lstcon_ioctl_entry,
+};
 
 /* initialize console */
 int
@@ -2048,7 +2049,8 @@ lstcon_console_init(void)
 		goto out;
 	}
 
-	rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
+	rc = blocking_notifier_chain_register(&libcfs_ioctl_list,
+					      &lstcon_ioctl_handler);
 
 	if (!rc) {
 		lstcon_rpc_module_init();
@@ -2071,7 +2073,8 @@ lstcon_console_fini(void)
 {
 	int i;
 
-	libcfs_deregister_ioctl(&lstcon_ioctl_handler);
+	blocking_notifier_chain_unregister(&libcfs_ioctl_list,
+					   &lstcon_ioctl_handler);
 
 	mutex_lock(&console_session.ses_mutex);
 
diff --git a/drivers/staging/lustre/lnet/selftest/console.h b/drivers/staging/lustre/lnet/selftest/console.h
index 3933ed4cca93..2826205e36a1 100644
--- a/drivers/staging/lustre/lnet/selftest/console.h
+++ b/drivers/staging/lustre/lnet/selftest/console.h
@@ -40,7 +40,6 @@
 #ifndef __LST_CONSOLE_H__
 #define __LST_CONSOLE_H__
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/lnet/lib-types.h>
 #include <uapi/linux/lnet/lnetst.h>
 #include "selftest.h"
@@ -187,7 +186,8 @@ lstcon_id2hash(struct lnet_process_id id, struct list_head *hash)
 	return &hash[idx];
 }
 
-int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr);
+int lstcon_ioctl_entry(struct notifier_block *nb,
+		       unsigned long cmd, void *vdata);
 int lstcon_console_init(void);
 int lstcon_console_fini(void);
 int lstcon_session_match(struct lst_sid sid);
diff --git a/drivers/staging/lustre/lnet/selftest/framework.c b/drivers/staging/lustre/lnet/selftest/framework.c
index 0ca1e3a780ca..741af10560ad 100644
--- a/drivers/staging/lustre/lnet/selftest/framework.c
+++ b/drivers/staging/lustre/lnet/selftest/framework.c
@@ -272,7 +272,7 @@ sfw_init_session(struct sfw_session *sn, struct lst_sid sid,
 	sn->sn_id = sid;
 	sn->sn_features = features;
 	sn->sn_timeout = session_timeout;
-	sn->sn_started = cfs_time_current();
+	sn->sn_started = jiffies;
 
 	timer->stt_data = sn;
 	timer->stt_func = sfw_session_expired;
@@ -588,7 +588,7 @@ sfw_load_test(struct sfw_test_instance *tsi)
 
 	CDEBUG(D_NET, "Reserved %d buffers for test %s\n",
 	       nbuf * (srpc_serv_is_framework(svc) ?
-		       2 : cfs_cpt_number(cfs_cpt_table)), svc->sv_name);
+		       2 : cfs_cpt_number(cfs_cpt_tab)), svc->sv_name);
 	return 0;
 }
 
diff --git a/drivers/staging/lustre/lnet/selftest/module.c b/drivers/staging/lustre/lnet/selftest/module.c
index 7359aa56d9b3..9ba65320f748 100644
--- a/drivers/staging/lustre/lnet/selftest/module.c
+++ b/drivers/staging/lustre/lnet/selftest/module.c
@@ -89,9 +89,13 @@ static int
 lnet_selftest_init(void)
 {
 	int nscheds;
-	int rc = -ENOMEM;
+	int rc;
 	int i;
 
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	lst_serial_wq = alloc_ordered_workqueue("lst_s", 0);
 	if (!lst_serial_wq) {
 		CERROR("Failed to create serial WI scheduler for LST\n");
diff --git a/drivers/staging/lustre/lnet/selftest/selftest.h b/drivers/staging/lustre/lnet/selftest/selftest.h
index 05466b85e1c0..8737fa96b192 100644
--- a/drivers/staging/lustre/lnet/selftest/selftest.h
+++ b/drivers/staging/lustre/lnet/selftest/selftest.h
@@ -39,7 +39,6 @@
 
 #define LNET_ONLY
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/lnet/lib-lnet.h>
 #include <linux/lnet/lib-types.h>
 #include <uapi/linux/lnet/lnetst.h>
diff --git a/drivers/staging/lustre/lnet/selftest/timer.c b/drivers/staging/lustre/lnet/selftest/timer.c
index 1b2c5fc81358..582f252b3e12 100644
--- a/drivers/staging/lustre/lnet/selftest/timer.c
+++ b/drivers/staging/lustre/lnet/selftest/timer.c
@@ -155,9 +155,9 @@ stt_check_timers(unsigned long *last)
 
 	spin_lock(&stt_data.stt_lock);
 
-	while (cfs_time_aftereq(this_slot, *last)) {
+	while (time_after_eq(this_slot, *last)) {
 		expired += stt_expire_list(STTIMER_SLOT(this_slot), now);
-		this_slot = cfs_time_sub(this_slot, STTIMER_SLOTTIME);
+		this_slot = this_slot - STTIMER_SLOTTIME;
 	}
 
 	*last = now & STTIMER_SLOTTIMEMASK;
diff --git a/drivers/staging/lustre/lustre/fid/fid_internal.h b/drivers/staging/lustre/lustre/fid/fid_internal.h
index b7b8f900df8e..14569e969a31 100644
--- a/drivers/staging/lustre/lustre/fid/fid_internal.h
+++ b/drivers/staging/lustre/lustre/fid/fid_internal.h
@@ -38,7 +38,6 @@
 #define __FID_INTERNAL_H
 
 #include <uapi/linux/lustre/lustre_idl.h>
-#include <linux/libcfs/libcfs.h>
 
 /* Functions used internally in module. */
 
diff --git a/drivers/staging/lustre/lustre/fid/fid_lib.c b/drivers/staging/lustre/lustre/fid/fid_lib.c
index 9577da33e666..ac52b378c155 100644
--- a/drivers/staging/lustre/lustre/fid/fid_lib.c
+++ b/drivers/staging/lustre/lustre/fid/fid_lib.c
@@ -40,7 +40,6 @@
 
 #define DEBUG_SUBSYSTEM S_FID
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
 #include <lustre_fid.h>
 
diff --git a/drivers/staging/lustre/lustre/fid/fid_request.c b/drivers/staging/lustre/lustre/fid/fid_request.c
index 030680f37c79..c674652af03a 100644
--- a/drivers/staging/lustre/lustre/fid/fid_request.c
+++ b/drivers/staging/lustre/lustre/fid/fid_request.c
@@ -39,7 +39,6 @@
 
 #define DEBUG_SUBSYSTEM S_FID
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
 
 #include <obd.h>
@@ -419,6 +418,12 @@ EXPORT_SYMBOL(client_fid_fini);
 
 static int __init fid_init(void)
 {
+	int rc;
+
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	seq_debugfs_dir = ldebugfs_register(LUSTRE_SEQ_NAME,
 					    debugfs_lustre_root,
 					    NULL, NULL);
diff --git a/drivers/staging/lustre/lustre/fid/lproc_fid.c b/drivers/staging/lustre/lustre/fid/lproc_fid.c
index a1e5bf9f36ec..0aabf473c9bd 100644
--- a/drivers/staging/lustre/lustre/fid/lproc_fid.c
+++ b/drivers/staging/lustre/lustre/fid/lproc_fid.c
@@ -39,7 +39,6 @@
 
 #define DEBUG_SUBSYSTEM S_FID
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
 
 #include <obd.h>
diff --git a/drivers/staging/lustre/lustre/fld/fld_cache.c b/drivers/staging/lustre/lustre/fld/fld_cache.c
index 2d61ca4e51cf..a7415c9a1c28 100644
--- a/drivers/staging/lustre/lustre/fld/fld_cache.c
+++ b/drivers/staging/lustre/lustre/fld/fld_cache.c
@@ -40,7 +40,6 @@
 
 #define DEBUG_SUBSYSTEM S_FLD
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
 #include <asm/div64.h>
 
diff --git a/drivers/staging/lustre/lustre/fld/fld_internal.h b/drivers/staging/lustre/lustre/fld/fld_internal.h
index b5e3abaa508a..e1d6aaa5c2b4 100644
--- a/drivers/staging/lustre/lustre/fld/fld_internal.h
+++ b/drivers/staging/lustre/lustre/fld/fld_internal.h
@@ -59,7 +59,6 @@
 
 #include <uapi/linux/lustre/lustre_idl.h>
 
-#include <linux/libcfs/libcfs.h>
 #include <lustre_req_layout.h>
 #include <lustre_fld.h>
 
diff --git a/drivers/staging/lustre/lustre/fld/fld_request.c b/drivers/staging/lustre/lustre/fld/fld_request.c
index 068c364adda8..7b7ba93a4db6 100644
--- a/drivers/staging/lustre/lustre/fld/fld_request.c
+++ b/drivers/staging/lustre/lustre/fld/fld_request.c
@@ -39,7 +39,6 @@
 
 #define DEBUG_SUBSYSTEM S_FLD
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
 #include <asm/div64.h>
 
@@ -450,6 +449,12 @@ void fld_client_flush(struct lu_client_fld *fld)
 
 static int __init fld_init(void)
 {
+	int rc;
+
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	fld_debugfs_dir = ldebugfs_register(LUSTRE_FLD_NAME,
 					    debugfs_lustre_root,
 					    NULL, NULL);
diff --git a/drivers/staging/lustre/lustre/fld/lproc_fld.c b/drivers/staging/lustre/lustre/fld/lproc_fld.c
index 1a6a76110c3e..0bcfb26ef8aa 100644
--- a/drivers/staging/lustre/lustre/fld/lproc_fld.c
+++ b/drivers/staging/lustre/lustre/fld/lproc_fld.c
@@ -40,7 +40,6 @@
 
 #define DEBUG_SUBSYSTEM S_FLD
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
 
 #include <obd.h>
diff --git a/drivers/staging/lustre/lustre/include/cl_object.h b/drivers/staging/lustre/lustre/include/cl_object.h
index 341a145c3331..6f7b991be809 100644
--- a/drivers/staging/lustre/lustre/include/cl_object.h
+++ b/drivers/staging/lustre/lustre/include/cl_object.h
@@ -1941,7 +1941,7 @@ static inline int lu_device_is_cl(const struct lu_device *d)
 static inline struct cl_device *lu2cl_dev(const struct lu_device *d)
 {
 	LASSERT(!d || IS_ERR(d) || lu_device_is_cl(d));
-	return container_of0(d, struct cl_device, cd_lu_dev);
+	return container_of_safe(d, struct cl_device, cd_lu_dev);
 }
 
 static inline struct lu_device *cl2lu_dev(struct cl_device *d)
@@ -1952,13 +1952,13 @@ static inline struct lu_device *cl2lu_dev(struct cl_device *d)
 static inline struct cl_object *lu2cl(const struct lu_object *o)
 {
 	LASSERT(!o || IS_ERR(o) || lu_device_is_cl(o->lo_dev));
-	return container_of0(o, struct cl_object, co_lu);
+	return container_of_safe(o, struct cl_object, co_lu);
 }
 
 static inline const struct cl_object_conf *
 lu2cl_conf(const struct lu_object_conf *conf)
 {
-	return container_of0(conf, struct cl_object_conf, coc_lu);
+	return container_of_safe(conf, struct cl_object_conf, coc_lu);
 }
 
 static inline struct cl_object *cl_object_next(const struct cl_object *obj)
@@ -1969,12 +1969,12 @@ static inline struct cl_object *cl_object_next(const struct cl_object *obj)
 static inline struct cl_device *cl_object_device(const struct cl_object *o)
 {
 	LASSERT(!o || IS_ERR(o) || lu_device_is_cl(o->co_lu.lo_dev));
-	return container_of0(o->co_lu.lo_dev, struct cl_device, cd_lu_dev);
+	return container_of_safe(o->co_lu.lo_dev, struct cl_device, cd_lu_dev);
 }
 
 static inline struct cl_object_header *luh2coh(const struct lu_object_header *h)
 {
-	return container_of0(h, struct cl_object_header, coh_lu);
+	return container_of_safe(h, struct cl_object_header, coh_lu);
 }
 
 static inline struct cl_site *cl_object_site(const struct cl_object *obj)
diff --git a/drivers/staging/lustre/lustre/include/lprocfs_status.h b/drivers/staging/lustre/lustre/include/lprocfs_status.h
index 426e8f3c9809..1322ecffda13 100644
--- a/drivers/staging/lustre/lustre/include/lprocfs_status.h
+++ b/drivers/staging/lustre/lustre/include/lprocfs_status.h
@@ -43,8 +43,8 @@
 #include <linux/seq_file.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
+#include <linux/device.h>
 
-#include <linux/libcfs/libcfs.h>
 #include <uapi/linux/lustre/lustre_cfg.h>
 #include <uapi/linux/lustre/lustre_idl.h>
 
diff --git a/drivers/staging/lustre/lustre/include/lu_object.h b/drivers/staging/lustre/lustre/include/lu_object.h
index 35c7b582f36d..f29bbca5af65 100644
--- a/drivers/staging/lustre/lustre/include/lu_object.h
+++ b/drivers/staging/lustre/lustre/include/lu_object.h
@@ -549,31 +549,7 @@ struct lu_object_header {
 };
 
 struct fld;
-
-struct lu_site_bkt_data {
-	/**
-	 * number of object in this bucket on the lsb_lru list.
-	 */
-	long			lsb_lru_len;
-	/**
-	 * LRU list, updated on each access to object. Protected by
-	 * bucket lock of lu_site::ls_obj_hash.
-	 *
-	 * "Cold" end of LRU is lu_site::ls_lru.next. Accessed object are
-	 * moved to the lu_site::ls_lru.prev (this is due to the non-existence
-	 * of list_for_each_entry_safe_reverse()).
-	 */
-	struct list_head		lsb_lru;
-	/**
-	 * Wait-queue signaled when an object in this site is ultimately
-	 * destroyed (lu_object_free()). It is used by lu_object_find() to
-	 * wait before re-trying when object in the process of destruction is
-	 * found in the hash table.
-	 *
-	 * \see htable_lookup().
-	 */
-	wait_queue_head_t	       lsb_marche_funebre;
-};
+struct lu_site_bkt_data;
 
 enum {
 	LU_SS_CREATED	 = 0,
@@ -642,14 +618,8 @@ struct lu_site {
 	struct percpu_counter	 ls_lru_len_counter;
 };
 
-static inline struct lu_site_bkt_data *
-lu_site_bkt_from_fid(struct lu_site *site, struct lu_fid *fid)
-{
-	struct cfs_hash_bd bd;
-
-	cfs_hash_bd_get(site->ls_obj_hash, fid, &bd);
-	return cfs_hash_bd_extra_get(site->ls_obj_hash, &bd);
-}
+wait_queue_head_t *
+lu_site_wq_from_fid(struct lu_site *site, struct lu_fid *fid);
 
 static inline struct seq_server_site *lu_site2seq(const struct lu_site *s)
 {
@@ -745,15 +715,15 @@ struct lu_object *lu_object_find_slice(const struct lu_env *env,
 static inline struct lu_object *lu_object_top(struct lu_object_header *h)
 {
 	LASSERT(!list_empty(&h->loh_layers));
-	return container_of0(h->loh_layers.next, struct lu_object, lo_linkage);
+	return list_first_entry(&h->loh_layers, struct lu_object, lo_linkage);
 }
 
 /**
  * Next sub-object in the layering
  */
-static inline struct lu_object *lu_object_next(const struct lu_object *o)
+static inline const struct lu_object *lu_object_next(const struct lu_object *o)
 {
-	return container_of0(o->lo_linkage.next, struct lu_object, lo_linkage);
+	return list_next_entry(o, lo_linkage);
 }
 
 /**
diff --git a/drivers/staging/lustre/lustre/include/lustre_compat.h b/drivers/staging/lustre/lustre/include/lustre_compat.h
index 9f488e605083..3c6db0d632dc 100644
--- a/drivers/staging/lustre/lustre/include/lustre_compat.h
+++ b/drivers/staging/lustre/lustre/include/lustre_compat.h
@@ -37,6 +37,7 @@
 #include <linux/fs_struct.h>
 #include <linux/namei.h>
 #include <linux/cred.h>
+#include <linux/module.h>
 
 #include <lustre_patchless_compat.h>
 
diff --git a/drivers/staging/lustre/lustre/include/lustre_disk.h b/drivers/staging/lustre/lustre/include/lustre_disk.h
index 100e993ab00b..886e817644d6 100644
--- a/drivers/staging/lustre/lustre/include/lustre_disk.h
+++ b/drivers/staging/lustre/lustre/include/lustre_disk.h
@@ -48,7 +48,6 @@
 #include <asm/byteorder.h>
 #include <linux/types.h>
 #include <linux/backing-dev.h>
-#include <linux/libcfs/libcfs.h>
 
 /****************** persistent mount data *********************/
 
diff --git a/drivers/staging/lustre/lustre/include/lustre_dlm.h b/drivers/staging/lustre/lustre/include/lustre_dlm.h
index 239aa2b1268f..b3532adac31c 100644
--- a/drivers/staging/lustre/lustre/include/lustre_dlm.h
+++ b/drivers/staging/lustre/lustre/include/lustre_dlm.h
@@ -362,6 +362,9 @@ struct ldlm_namespace {
 	/** Flag indicating if namespace is on client instead of server */
 	enum ldlm_side		ns_client;
 
+	/** name of this namespace */
+	char			*ns_name;
+
 	/** Resource hash table for namespace. */
 	struct cfs_hash		*ns_rs_hash;
 
@@ -629,16 +632,6 @@ struct ldlm_lock {
 	 * Tree node for ldlm_extent.
 	 */
 	struct ldlm_interval	*l_tree_node;
-	/**
-	 * Per export hash of locks.
-	 * Protected by per-bucket exp->exp_lock_hash locks.
-	 */
-	struct hlist_node	l_exp_hash;
-	/**
-	 * Per export hash of flock locks.
-	 * Protected by per-bucket exp->exp_flock_hash locks.
-	 */
-	struct hlist_node	l_exp_flock_hash;
 	/**
 	 * Requested mode.
 	 * Protected by lr_lock.
@@ -888,7 +881,7 @@ static inline bool ldlm_has_layout(struct ldlm_lock *lock)
 static inline char *
 ldlm_ns_name(struct ldlm_namespace *ns)
 {
-	return ns->ns_rs_hash->hs_name;
+	return ns->ns_name;
 }
 
 static inline struct ldlm_namespace *
@@ -1200,7 +1193,7 @@ struct ldlm_resource *ldlm_resource_get(struct ldlm_namespace *ns,
 					struct ldlm_resource *parent,
 					const struct ldlm_res_id *,
 					enum ldlm_type type, int create);
-int ldlm_resource_putref(struct ldlm_resource *res);
+void ldlm_resource_putref(struct ldlm_resource *res);
 void ldlm_resource_add_lock(struct ldlm_resource *res,
 			    struct list_head *head,
 			    struct ldlm_lock *lock);
@@ -1227,7 +1220,6 @@ int ldlm_lock_change_resource(struct ldlm_namespace *, struct ldlm_lock *,
  * processing.
  * @{
  */
-int ldlm_completion_ast_async(struct ldlm_lock *lock, __u64 flags, void *data);
 int ldlm_completion_ast(struct ldlm_lock *lock, __u64 flags, void *data);
 /** @} ldlm_local_ast */
 
diff --git a/drivers/staging/lustre/lustre/include/lustre_export.h b/drivers/staging/lustre/lustre/include/lustre_export.h
index 40cd168ed2ea..79ad5aae86b9 100644
--- a/drivers/staging/lustre/lustre/include/lustre_export.h
+++ b/drivers/staging/lustre/lustre/include/lustre_export.h
@@ -89,7 +89,7 @@ struct obd_export {
 	struct list_head		exp_obd_chain;
 	/** work_struct for destruction of export */
 	struct work_struct	exp_zombie_work;
-	struct hlist_node	  exp_uuid_hash; /** uuid-export hash*/
+	struct rhash_head	exp_uuid_hash; /** uuid-export hash*/
 	/** Obd device of this export */
 	struct obd_device	*exp_obd;
 	/**
@@ -102,13 +102,6 @@ struct obd_export {
 	struct ptlrpc_connection *exp_connection;
 	/** Connection count value from last successful reconnect rpc */
 	__u32		     exp_conn_cnt;
-	/** Hash list of all ldlm locks granted on this export */
-	struct cfs_hash	       *exp_lock_hash;
-	/**
-	 * Hash list for Posix lock deadlock detection, added with
-	 * ldlm_lock::l_exp_flock_hash.
-	 */
-	struct cfs_hash	       *exp_flock_hash;
 	struct list_head		exp_outstanding_replies;
 	struct list_head		exp_uncommitted_replies;
 	spinlock_t		  exp_uncommitted_replies_lock;
diff --git a/drivers/staging/lustre/lustre/include/lustre_fld.h b/drivers/staging/lustre/lustre/include/lustre_fld.h
index 4055bbd24c55..f42122a4dfaa 100644
--- a/drivers/staging/lustre/lustre/include/lustre_fld.h
+++ b/drivers/staging/lustre/lustre/include/lustre_fld.h
@@ -40,7 +40,6 @@
  */
 
 #include <uapi/linux/lustre/lustre_idl.h>
-#include <linux/libcfs/libcfs.h>
 #include <seq_range.h>
 
 struct lu_client_fld;
diff --git a/drivers/staging/lustre/lustre/include/lustre_handles.h b/drivers/staging/lustre/lustre/include/lustre_handles.h
index c48c97362cf6..3556ce8d94e8 100644
--- a/drivers/staging/lustre/lustre/include/lustre_handles.h
+++ b/drivers/staging/lustre/lustre/include/lustre_handles.h
@@ -45,8 +45,6 @@
 #include <linux/spinlock.h>
 #include <linux/types.h>
 
-#include <linux/libcfs/libcfs.h>
-
 struct portals_handle_ops {
 	void (*hop_addref)(void *object);
 	void (*hop_free)(void *object, int size);
diff --git a/drivers/staging/lustre/lustre/include/lustre_import.h b/drivers/staging/lustre/lustre/include/lustre_import.h
index 1731048f1ff2..ac3805ead620 100644
--- a/drivers/staging/lustre/lustre/include/lustre_import.h
+++ b/drivers/staging/lustre/lustre/include/lustre_import.h
@@ -44,6 +44,7 @@
  * @{
  */
 
+#include <linux/libcfs/libcfs.h>
 #include <lustre_handles.h>
 #include <uapi/linux/lustre/lustre_idl.h>
 
diff --git a/drivers/staging/lustre/lustre/include/lustre_intent.h b/drivers/staging/lustre/lustre/include/lustre_intent.h
index 519e94fc089d..51e5c0e03872 100644
--- a/drivers/staging/lustre/lustre/include/lustre_intent.h
+++ b/drivers/staging/lustre/lustre/include/lustre_intent.h
@@ -34,6 +34,8 @@
 #ifndef LUSTRE_INTENT_H
 #define LUSTRE_INTENT_H
 
+#include <linux/types.h>
+
 /* intent IT_XXX are defined in lustre/include/obd.h */
 
 struct lookup_intent {
diff --git a/drivers/staging/lustre/lustre/include/lustre_lib.h b/drivers/staging/lustre/lustre/include/lustre_lib.h
index 0053eafc1c10..87748e9902a7 100644
--- a/drivers/staging/lustre/lustre/include/lustre_lib.h
+++ b/drivers/staging/lustre/lustre/include/lustre_lib.h
@@ -46,7 +46,6 @@
 #include <linux/sched/signal.h>
 #include <linux/signal.h>
 #include <linux/types.h>
-#include <linux/libcfs/libcfs.h>
 #include <uapi/linux/lustre/lustre_idl.h>
 #include <uapi/linux/lustre/lustre_ver.h>
 #include <uapi/linux/lustre/lustre_cfg.h>
@@ -94,31 +93,34 @@ static inline int l_fatal_signal_pending(struct task_struct *p)
  */
 #define l_wait_event_abortable(wq, condition)				\
 ({									\
-	sigset_t __old_blocked;						\
+	sigset_t __new_blocked, __old_blocked;				\
 	int __ret = 0;							\
-	cfs_block_sigsinv(LUSTRE_FATAL_SIGS, &__old_blocked);		\
+	siginitset(&__new_blocked, LUSTRE_FATAL_SIGS);			\
+	sigprocmask(SIG_BLOCK, &__new_blocked, &__old_blocked);		\
 	__ret = wait_event_interruptible(wq, condition);		\
-	cfs_restore_sigs(&__old_blocked);				\
+	sigprocmask(SIG_SETMASK, &__old_blocked, NULL);			\
 	__ret;								\
 })
 
 #define l_wait_event_abortable_timeout(wq, condition, timeout)		\
 ({									\
-	sigset_t __old_blocked;						\
+	sigset_t __new_blocked, __old_blocked;				\
 	int __ret = 0;							\
-	cfs_block_sigsinv(LUSTRE_FATAL_SIGS, &__old_blocked);		\
+	siginitset(&__new_blocked, LUSTRE_FATAL_SIGS);			\
+	sigprocmask(SIG_BLOCK, &__new_blocked, &__old_blocked);		\
 	__ret = wait_event_interruptible_timeout(wq, condition, timeout);\
-	cfs_restore_sigs(&__old_blocked);				\
+	sigprocmask(SIG_SETMASK, &__old_blocked, NULL);			\
 	__ret;								\
 })
 
 #define l_wait_event_abortable_exclusive(wq, condition)			\
 ({									\
-	sigset_t __old_blocked;						\
+	sigset_t __new_blocked, __old_blocked;				\
 	int __ret = 0;							\
-	cfs_block_sigsinv(LUSTRE_FATAL_SIGS, &__old_blocked);		\
+	siginitset(&__new_blocked, LUSTRE_FATAL_SIGS);			\
+	sigprocmask(SIG_BLOCK, &__new_blocked, &__old_blocked);		\
 	__ret = wait_event_interruptible_exclusive(wq, condition);	\
-	cfs_restore_sigs(&__old_blocked);				\
+	sigprocmask(SIG_SETMASK, &__old_blocked, NULL);			\
 	__ret;								\
 })
 #endif /* _LUSTRE_LIB_H */
diff --git a/drivers/staging/lustre/lustre/include/lustre_mds.h b/drivers/staging/lustre/lustre/include/lustre_mds.h
index 6937546f1d46..f665556556ec 100644
--- a/drivers/staging/lustre/lustre/include/lustre_mds.h
+++ b/drivers/staging/lustre/lustre/include/lustre_mds.h
@@ -45,7 +45,6 @@
  */
 
 #include <lustre_handles.h>
-#include <linux/libcfs/libcfs.h>
 #include <lustre_lib.h>
 #include <lustre_dlm.h>
 #include <lustre_export.h>
diff --git a/drivers/staging/lustre/lustre/include/lustre_net.h b/drivers/staging/lustre/lustre/include/lustre_net.h
index d35ae0cda8d2..35b43a77eb18 100644
--- a/drivers/staging/lustre/lustre/include/lustre_net.h
+++ b/drivers/staging/lustre/lustre/include/lustre_net.h
@@ -67,6 +67,8 @@
 #include <obd_support.h>
 #include <uapi/linux/lustre/lustre_ver.h>
 
+#include <linux/rhashtable.h>
+
 /* MD flags we _always_ use */
 #define PTLRPC_MD_OPTIONS  0
 
@@ -286,7 +288,7 @@ struct ptlrpc_replay_async_args {
  */
 struct ptlrpc_connection {
 	/** linkage for connections hash table */
-	struct hlist_node	c_hash;
+	struct rhash_head	c_hash;
 	/** Our own lnet nid for this connection */
 	lnet_nid_t	      c_self;
 	/** Remote side nid for this connection */
@@ -556,7 +558,7 @@ struct ptlrpc_cli_req {
 	/** optional time limit for send attempts */
 	long				 cr_delay_limit;
 	/** time request was first queued */
-	time_t				 cr_queued_time;
+	unsigned long			 cr_queued_time;
 	/** request sent timeval */
 	struct timespec64		 cr_sent_tv;
 	/** time for request really sent out */
@@ -2253,9 +2255,8 @@ static inline int ptlrpc_req_get_repsize(struct ptlrpc_request *req)
 static inline int ptlrpc_send_limit_expired(struct ptlrpc_request *req)
 {
 	if (req->rq_delay_limit != 0 &&
-	    time_before(cfs_time_add(req->rq_queued_time,
-				     req->rq_delay_limit * HZ),
-			cfs_time_current())) {
+	    time_before(req->rq_queued_time + req->rq_delay_limit * HZ,
+			jiffies)) {
 		return 1;
 	}
 	return 0;
diff --git a/drivers/staging/lustre/lustre/include/lustre_patchless_compat.h b/drivers/staging/lustre/lustre/include/lustre_patchless_compat.h
index ce28ed5c1ef8..298476ea7557 100644
--- a/drivers/staging/lustre/lustre/include/lustre_patchless_compat.h
+++ b/drivers/staging/lustre/lustre/include/lustre_patchless_compat.h
@@ -39,6 +39,7 @@
 #include <linux/list.h>
 #include <linux/mm.h>
 #include <linux/hash.h>
+#include <linux/pagemap.h>
 
 #define ll_delete_from_page_cache(page) delete_from_page_cache(page)
 
diff --git a/drivers/staging/lustre/lustre/include/lustre_sec.h b/drivers/staging/lustre/lustre/include/lustre_sec.h
index c5cb07acd0da..d35bcbc98831 100644
--- a/drivers/staging/lustre/lustre/include/lustre_sec.h
+++ b/drivers/staging/lustre/lustre/include/lustre_sec.h
@@ -34,6 +34,8 @@
 #ifndef _LUSTRE_SEC_H_
 #define _LUSTRE_SEC_H_
 
+#include <linux/module.h>
+
 /** \defgroup sptlrpc sptlrpc
  *
  * @{
diff --git a/drivers/staging/lustre/lustre/include/obd.h b/drivers/staging/lustre/lustre/include/obd.h
index f1233ca7d337..da99a0f02b6c 100644
--- a/drivers/staging/lustre/lustre/include/obd.h
+++ b/drivers/staging/lustre/lustre/include/obd.h
@@ -46,6 +46,8 @@
 #include <lustre_intent.h>
 #include <cl_object.h>
 
+#include <linux/rhashtable.h>
+
 #define MAX_OBD_DEVICES 8192
 
 struct osc_async_rc {
@@ -331,7 +333,7 @@ struct client_obd {
 	void		    *cl_writeback_work;
 	void			*cl_lru_work;
 	/* hash tables for osc_quota_info */
-	struct cfs_hash	      *cl_quota_hash[MAXQUOTAS];
+	struct rhashtable	cl_quota_hash[MAXQUOTAS];
 };
 
 #define obd2cli_tgt(obd) ((char *)(obd)->u.cli.cl_target_uuid.uuid)
@@ -383,7 +385,7 @@ struct lov_obd {
 	__u32		   lov_tgt_size;   /* size of tgts array */
 	int		     lov_connects;
 	int		     lov_pool_count;
-	struct cfs_hash	     *lov_pools_hash_body; /* used for key access */
+	struct rhashtable	lov_pools_hash_body; /* used for key access */
 	struct list_head	lov_pool_list; /* used for sequential access */
 	struct dentry		*lov_pool_debugfs_entry;
 	enum lustre_sec_part    lov_sp_me;
@@ -556,7 +558,7 @@ struct obd_device {
 	 */
 	unsigned long obd_recovery_expired:1;
 	/* uuid-export hash body */
-	struct cfs_hash	     *obd_uuid_hash;
+	struct rhashtable	obd_uuid_hash;
 	wait_queue_head_t	     obd_refcount_waitq;
 	struct list_head	      obd_exports;
 	struct list_head	      obd_unlinked_exports;
@@ -620,6 +622,9 @@ struct obd_device {
 	struct completion	obd_kobj_unregister;
 };
 
+int obd_uuid_add(struct obd_device *obd, struct obd_export *export);
+void obd_uuid_del(struct obd_device *obd, struct obd_export *export);
+
 /* get/set_info keys */
 #define KEY_ASYNC	       "async"
 #define KEY_CHANGELOG_CLEAR     "changelog_clear"
@@ -686,6 +691,16 @@ enum md_cli_flags {
 	CLI_MIGRATE	= BIT(4),
 };
 
+/**
+ * GETXATTR is not included as only a couple of fields in the reply body
+ * is filled, but not FID which is needed for common intent handling in
+ * mdc_finish_intent_lock()
+ */
+static inline bool it_has_reply_body(const struct lookup_intent *it)
+{
+	return it->it_op & (IT_OPEN | IT_UNLINK | IT_LOOKUP | IT_GETATTR);
+}
+
 struct md_op_data {
 	struct lu_fid	   op_fid1; /* operation fid1 (usually parent) */
 	struct lu_fid	   op_fid2; /* operation fid2 (usually child) */
@@ -702,7 +717,7 @@ struct md_op_data {
 	__u32		   op_suppgids[2];
 	__u32		   op_fsuid;
 	__u32		   op_fsgid;
-	cfs_cap_t	       op_cap;
+	kernel_cap_t	       op_cap;
 	void		   *op_data;
 	size_t			op_data_size;
 
@@ -897,10 +912,9 @@ struct md_ops {
 		     struct md_open_data *, struct ptlrpc_request **);
 	int (*create)(struct obd_export *, struct md_op_data *,
 		      const void *, size_t, umode_t, uid_t, gid_t,
-		      cfs_cap_t, __u64, struct ptlrpc_request **);
+		      kernel_cap_t, __u64, struct ptlrpc_request **);
 	int (*enqueue)(struct obd_export *, struct ldlm_enqueue_info *,
-		       const union ldlm_policy_data *,
-		       struct lookup_intent *, struct md_op_data *,
+		       const union ldlm_policy_data *, struct md_op_data *,
 		       struct lustre_handle *, __u64);
 	int (*getattr)(struct obd_export *, struct md_op_data *,
 		       struct ptlrpc_request **);
diff --git a/drivers/staging/lustre/lustre/include/obd_class.h b/drivers/staging/lustre/lustre/include/obd_class.h
index f24dd74ffa09..a3b1465121e5 100644
--- a/drivers/staging/lustre/lustre/include/obd_class.h
+++ b/drivers/staging/lustre/lustre/include/obd_class.h
@@ -913,7 +913,7 @@ static inline int obd_destroy_export(struct obd_export *exp)
 /*
  * @max_age is the oldest time in jiffies that we accept using a cached data.
  * If the cache is older than @max_age we will get a new value from the
- * target.  Use a value of "cfs_time_current() + HZ" to guarantee freshness.
+ * target.  Use a value of "jiffies + HZ" to guarantee freshness.
  */
 static inline int obd_statfs_async(struct obd_export *exp,
 				   struct obd_info *oinfo,
@@ -932,7 +932,7 @@ static inline int obd_statfs_async(struct obd_export *exp,
 
 	CDEBUG(D_SUPER, "%s: osfs %p age %llu, max_age %llu\n",
 	       obd->obd_name, &obd->obd_osfs, obd->obd_osfs_age, max_age);
-	if (cfs_time_before_64(obd->obd_osfs_age, max_age)) {
+	if (time_before64(obd->obd_osfs_age, max_age)) {
 		rc = OBP(obd, statfs_async)(exp, oinfo, max_age, rqset);
 	} else {
 		CDEBUG(D_SUPER,
@@ -975,7 +975,7 @@ static inline int obd_statfs_rqset(struct obd_export *exp,
 /*
  * @max_age is the oldest time in jiffies that we accept using a cached data.
  * If the cache is older than @max_age we will get a new value from the
- * target.  Use a value of "cfs_time_current() + HZ" to guarantee freshness.
+ * target.  Use a value of "jiffies + HZ" to guarantee freshness.
  */
 static inline int obd_statfs(const struct lu_env *env, struct obd_export *exp,
 			     struct obd_statfs *osfs, __u64 max_age,
@@ -992,12 +992,12 @@ static inline int obd_statfs(const struct lu_env *env, struct obd_export *exp,
 
 	CDEBUG(D_SUPER, "osfs %llu, max_age %llu\n",
 	       obd->obd_osfs_age, max_age);
-	if (cfs_time_before_64(obd->obd_osfs_age, max_age)) {
+	if (time_before64(obd->obd_osfs_age, max_age)) {
 		rc = OBP(obd, statfs)(env, exp, osfs, max_age, flags);
 		if (rc == 0) {
 			spin_lock(&obd->obd_osfs_lock);
 			memcpy(&obd->obd_osfs, osfs, sizeof(obd->obd_osfs));
-			obd->obd_osfs_age = cfs_time_current_64();
+			obd->obd_osfs_age = get_jiffies_64();
 			spin_unlock(&obd->obd_osfs_lock);
 		}
 	} else {
@@ -1226,7 +1226,7 @@ static inline int md_close(struct obd_export *exp, struct md_op_data *op_data,
 
 static inline int md_create(struct obd_export *exp, struct md_op_data *op_data,
 			    const void *data, size_t datalen, umode_t mode,
-			    uid_t uid, gid_t gid, cfs_cap_t cap_effective,
+			    uid_t uid, gid_t gid, kernel_cap_t cap_effective,
 			    __u64 rdev, struct ptlrpc_request **request)
 {
 	int rc;
@@ -1241,7 +1241,6 @@ static inline int md_create(struct obd_export *exp, struct md_op_data *op_data,
 static inline int md_enqueue(struct obd_export *exp,
 			     struct ldlm_enqueue_info *einfo,
 			     const union ldlm_policy_data *policy,
-			     struct lookup_intent *it,
 			     struct md_op_data *op_data,
 			     struct lustre_handle *lockh,
 			     __u64 extra_lock_flags)
@@ -1250,7 +1249,7 @@ static inline int md_enqueue(struct obd_export *exp,
 
 	EXP_CHECK_MD_OP(exp, enqueue);
 	EXP_MD_COUNTER_INCREMENT(exp, enqueue);
-	rc = MDP(exp->exp_obd, enqueue)(exp, einfo, policy, it, op_data, lockh,
+	rc = MDP(exp->exp_obd, enqueue)(exp, einfo, policy, op_data, lockh,
 					extra_lock_flags);
 	return rc;
 }
diff --git a/drivers/staging/lustre/lustre/include/obd_support.h b/drivers/staging/lustre/lustre/include/obd_support.h
index 8595091b8b86..9e41633823f7 100644
--- a/drivers/staging/lustre/lustre/include/obd_support.h
+++ b/drivers/staging/lustre/lustre/include/obd_support.h
@@ -37,7 +37,6 @@
 #include <linux/slab.h>
 #include <linux/sched/signal.h>
 
-#include <linux/libcfs/libcfs.h>
 #include <lustre_compat.h>
 #include <lprocfs_status.h>
 
@@ -61,33 +60,6 @@ extern atomic_long_t obd_dirty_transit_pages;
 extern char obd_jobid_var[];
 
 /* Some hash init argument constants */
-#define HASH_POOLS_BKT_BITS 3
-#define HASH_POOLS_CUR_BITS 3
-#define HASH_POOLS_MAX_BITS 7
-#define HASH_UUID_BKT_BITS 5
-#define HASH_UUID_CUR_BITS 7
-#define HASH_UUID_MAX_BITS 12
-#define HASH_NID_BKT_BITS 5
-#define HASH_NID_CUR_BITS 7
-#define HASH_NID_MAX_BITS 12
-#define HASH_NID_STATS_BKT_BITS 5
-#define HASH_NID_STATS_CUR_BITS 7
-#define HASH_NID_STATS_MAX_BITS 12
-#define HASH_LQE_BKT_BITS 5
-#define HASH_LQE_CUR_BITS 7
-#define HASH_LQE_MAX_BITS 12
-#define HASH_CONN_BKT_BITS 5
-#define HASH_CONN_CUR_BITS 5
-#define HASH_CONN_MAX_BITS 15
-#define HASH_EXP_LOCK_BKT_BITS  5
-#define HASH_EXP_LOCK_CUR_BITS  7
-#define HASH_EXP_LOCK_MAX_BITS  16
-#define HASH_CL_ENV_BKT_BITS    5
-#define HASH_CL_ENV_BITS	10
-#define HASH_JOB_STATS_BKT_BITS 5
-#define HASH_JOB_STATS_CUR_BITS 7
-#define HASH_JOB_STATS_MAX_BITS 12
-
 /* Timeout definitions */
 #define OBD_TIMEOUT_DEFAULT	     100
 /* Time to wait for all clients to reconnect during recovery (hard limit) */
diff --git a/drivers/staging/lustre/lustre/ldlm/l_lock.c b/drivers/staging/lustre/lustre/ldlm/l_lock.c
index 0662cec14b81..296259aa51e6 100644
--- a/drivers/staging/lustre/lustre/ldlm/l_lock.c
+++ b/drivers/staging/lustre/lustre/ldlm/l_lock.c
@@ -32,7 +32,6 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LDLM
-#include <linux/libcfs/libcfs.h>
 
 #include <lustre_dlm.h>
 #include <lustre_lib.h>
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c b/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
index 11b11b5f3216..4da23ade2bb3 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
@@ -47,7 +47,6 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LDLM
-#include <linux/libcfs/libcfs.h>
 #include <lustre_dlm.h>
 #include <obd_support.h>
 #include <obd.h>
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c b/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
index 411b540b96d9..94f3b1e49896 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
@@ -83,9 +83,6 @@ ldlm_flock_destroy(struct ldlm_lock *lock, enum ldlm_mode mode)
 	LDLM_DEBUG(lock, "%s(mode: %d)",
 		   __func__, mode);
 
-	/* Safe to not lock here, since it should be empty anyway */
-	LASSERT(hlist_unhashed(&lock->l_exp_flock_hash));
-
 	list_del_init(&lock->l_res_link);
 
 	/* client side - set a flag to prevent sending a CANCEL */
@@ -263,15 +260,9 @@ static int ldlm_process_flock_lock(struct ldlm_lock *req)
 		lock->l_policy_data.l_flock.start =
 			new->l_policy_data.l_flock.end + 1;
 		new2->l_conn_export = lock->l_conn_export;
-		if (lock->l_export) {
+		if (lock->l_export)
 			new2->l_export = class_export_lock_get(lock->l_export,
 							       new2);
-			if (new2->l_export->exp_lock_hash &&
-			    hlist_unhashed(&new2->l_exp_hash))
-				cfs_hash_add(new2->l_export->exp_lock_hash,
-					     &new2->l_remote_handle,
-					     &new2->l_exp_hash);
-		}
 		ldlm_lock_addref_internal_nolock(new2,
 						 lock->l_granted_mode);
 
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
index 9efd26ec59dd..0aa4f234a4f4 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
@@ -40,7 +40,6 @@
 
 #define DEBUG_SUBSYSTEM S_LDLM
 
-#include <linux/libcfs/libcfs.h>
 #include <obd.h>
 #include <obd_class.h>
 #include <lustre_dlm.h>
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
index 95bea351d21d..a644d133063b 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
@@ -38,7 +38,6 @@
 
 #define DEBUG_SUBSYSTEM S_LDLM
 
-#include <linux/libcfs/libcfs.h>
 #include <lustre_intent.h>
 #include <lustre_swab.h>
 #include <obd_class.h>
@@ -243,7 +242,7 @@ static void ldlm_lock_add_to_lru_nolock(struct ldlm_lock *lock)
 {
 	struct ldlm_namespace *ns = ldlm_lock_to_ns(lock);
 
-	lock->l_last_used = cfs_time_current();
+	lock->l_last_used = jiffies;
 	LASSERT(list_empty(&lock->l_lru));
 	LASSERT(lock->l_resource->lr_type != LDLM_FLOCK);
 	list_add_tail(&lock->l_lru, &ns->ns_unused_list);
@@ -318,18 +317,6 @@ static int ldlm_lock_destroy_internal(struct ldlm_lock *lock)
 	}
 	ldlm_set_destroyed(lock);
 
-	if (lock->l_export && lock->l_export->exp_lock_hash) {
-		/* NB: it's safe to call cfs_hash_del() even lock isn't
-		 * in exp_lock_hash.
-		 */
-		/* In the function below, .hs_keycmp resolves to
-		 * ldlm_export_lock_keycmp()
-		 */
-		/* coverity[overrun-buffer-val] */
-		cfs_hash_del(lock->l_export->exp_lock_hash,
-			     &lock->l_remote_handle, &lock->l_exp_hash);
-	}
-
 	ldlm_lock_remove_from_lru(lock);
 	class_handle_unhash(&lock->l_handle);
 
@@ -419,8 +406,6 @@ static struct ldlm_lock *ldlm_lock_new(struct ldlm_resource *resource)
 	lock->l_blocking_lock = NULL;
 	INIT_LIST_HEAD(&lock->l_sl_mode);
 	INIT_LIST_HEAD(&lock->l_sl_policy);
-	INIT_HLIST_NODE(&lock->l_exp_hash);
-	INIT_HLIST_NODE(&lock->l_exp_flock_hash);
 
 	lprocfs_counter_incr(ldlm_res_to_ns(resource)->ns_stats,
 			     LDLM_NSS_LOCKS);
@@ -1565,12 +1550,14 @@ struct ldlm_lock *ldlm_lock_create(struct ldlm_namespace *ns,
 		return ERR_CAST(res);
 
 	lock = ldlm_lock_new(res);
-	if (!lock)
+	if (!lock) {
+		ldlm_resource_putref(res);
 		return ERR_PTR(-ENOMEM);
+	}
 
 	lock->l_req_mode = mode;
 	lock->l_ast_data = data;
-	lock->l_pid = current_pid();
+	lock->l_pid = current->pid;
 	if (cbs) {
 		lock->l_blocking_ast = cbs->lcs_blocking;
 		lock->l_completion_ast = cbs->lcs_completion;
@@ -1609,6 +1596,8 @@ struct ldlm_lock *ldlm_lock_create(struct ldlm_namespace *ns,
 	return ERR_PTR(rc);
 }
 
+
+
 /**
  * Enqueue (request) a lock.
  * On the client this is called from ldlm_cli_enqueue_fini
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
index c772c68e5a49..b0a29f50c7d6 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
@@ -38,7 +38,8 @@
 
 #define DEBUG_SUBSYSTEM S_LDLM
 
-#include <linux/libcfs/libcfs.h>
+#include <linux/kthread.h>
+#include <linux/sched/mm.h>
 #include <lustre_dlm.h>
 #include <obd_class.h>
 #include <linux/list.h>
@@ -325,9 +326,8 @@ static void ldlm_handle_gl_callback(struct ptlrpc_request *req,
 	lock_res_and_lock(lock);
 	if (lock->l_granted_mode == LCK_PW &&
 	    !lock->l_readers && !lock->l_writers &&
-	    cfs_time_after(cfs_time_current(),
-			   cfs_time_add(lock->l_last_used,
-					10 * HZ))) {
+	    time_after(jiffies,
+		       lock->l_last_used + 10 * HZ)) {
 		unlock_res_and_lock(lock);
 		if (ldlm_bl_to_thread_lock(ns, NULL, lock))
 			ldlm_handle_bl_callback(ns, NULL, lock);
@@ -388,7 +388,7 @@ static inline void init_blwi(struct ldlm_bl_work_item *blwi,
 	init_completion(&blwi->blwi_comp);
 	INIT_LIST_HEAD(&blwi->blwi_head);
 
-	if (memory_pressure_get())
+	if (current->flags & PF_MEMALLOC)
 		blwi->blwi_mem_pressure = 1;
 
 	blwi->blwi_ns = ns;
@@ -777,12 +777,14 @@ static int ldlm_bl_thread_need_create(struct ldlm_bl_pool *blp,
 static int ldlm_bl_thread_blwi(struct ldlm_bl_pool *blp,
 			       struct ldlm_bl_work_item *blwi)
 {
+	unsigned int flags = 0;
+
 	if (!blwi->blwi_ns)
 		/* added by ldlm_cleanup() */
 		return LDLM_ITER_STOP;
 
 	if (blwi->blwi_mem_pressure)
-		memory_pressure_set();
+		flags = memalloc_noreclaim_save();
 
 	OBD_FAIL_TIMEOUT(OBD_FAIL_LDLM_PAUSE_CANCEL2, 4);
 
@@ -805,7 +807,7 @@ static int ldlm_bl_thread_blwi(struct ldlm_bl_pool *blp,
 					blwi->blwi_lock);
 	}
 	if (blwi->blwi_mem_pressure)
-		memory_pressure_clr();
+		memalloc_noreclaim_restore(flags);
 
 	if (blwi->blwi_flags & LCF_ASYNC)
 		kfree(blwi);
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_request.c b/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
index c3c9186b74ce..cdc52eed6d85 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
@@ -62,6 +62,7 @@
 #include <lustre_dlm.h>
 #include <obd_class.h>
 #include <obd.h>
+#include <linux/libcfs/libcfs_hash.h>
 
 #include "ldlm_internal.h"
 
@@ -116,9 +117,9 @@ static void ldlm_expired_completion_wait(struct ldlm_lock *lock, __u32 conn_cnt)
 			   (s64)lock->l_last_activity,
 			   (s64)(ktime_get_real_seconds() -
 				 lock->l_last_activity));
-		if (cfs_time_after(cfs_time_current(), next_dump)) {
+		if (time_after(jiffies, next_dump)) {
 			last_dump = next_dump;
-			next_dump = cfs_time_shift(300);
+			next_dump = jiffies + 300 * HZ;
 			ldlm_namespace_dump(D_DLMTRACE,
 					    ldlm_lock_to_ns(lock));
 			if (last_dump == 0)
@@ -191,29 +192,6 @@ static int ldlm_completion_tail(struct ldlm_lock *lock, void *data)
 	return result;
 }
 
-/**
- * Implementation of ->l_completion_ast() for a client, that doesn't wait
- * until lock is granted. Suitable for locks enqueued through ptlrpcd, of
- * other threads that cannot block for long.
- */
-int ldlm_completion_ast_async(struct ldlm_lock *lock, __u64 flags, void *data)
-{
-	if (flags == LDLM_FL_WAIT_NOREPROC) {
-		LDLM_DEBUG(lock, "client-side enqueue waiting on pending lock");
-		return 0;
-	}
-
-	if (!(flags & LDLM_FL_BLOCKED_MASK)) {
-		wake_up(&lock->l_waitq);
-		return ldlm_completion_tail(lock, data);
-	}
-
-	LDLM_DEBUG(lock,
-		   "client-side enqueue returned a blocked lock, going forward");
-	return 0;
-}
-EXPORT_SYMBOL(ldlm_completion_ast_async);
-
 /**
  * Generic LDLM "completion" AST. This is called in several cases:
  *
@@ -428,19 +406,7 @@ int ldlm_cli_enqueue_fini(struct obd_export *exp, struct ptlrpc_request *req,
 	cleanup_phase = 0;
 
 	lock_res_and_lock(lock);
-	/* Key change rehash lock in per-export hash with new key */
-	if (exp->exp_lock_hash) {
-		/* In the function below, .hs_keycmp resolves to
-		 * ldlm_export_lock_keycmp()
-		 */
-		/* coverity[overrun-buffer-val] */
-		cfs_hash_rehash_key(exp->exp_lock_hash,
-				    &lock->l_remote_handle,
-				    &reply->lock_handle,
-				    &lock->l_exp_hash);
-	} else {
-		lock->l_remote_handle = reply->lock_handle;
-	}
+	lock->l_remote_handle = reply->lock_handle;
 
 	*flags = ldlm_flags_from_wire(reply->lock_flags);
 	lock->l_flags |= ldlm_flags_from_wire(reply->lock_flags &
@@ -1196,7 +1162,7 @@ static enum ldlm_policy_res ldlm_cancel_lrur_policy(struct ldlm_namespace *ns,
 						    int unused, int added,
 						    int count)
 {
-	unsigned long cur = cfs_time_current();
+	unsigned long cur = jiffies;
 	struct ldlm_pool *pl = &ns->ns_pool;
 	__u64 slv, lvf, lv;
 	unsigned long la;
@@ -1211,13 +1177,12 @@ static enum ldlm_policy_res ldlm_cancel_lrur_policy(struct ldlm_namespace *ns,
 	 * Despite of the LV, It doesn't make sense to keep the lock which
 	 * is unused for ns_max_age time.
 	 */
-	if (cfs_time_after(cfs_time_current(),
-			   cfs_time_add(lock->l_last_used, ns->ns_max_age)))
+	if (time_after(jiffies, lock->l_last_used + ns->ns_max_age))
 		return LDLM_POLICY_CANCEL_LOCK;
 
 	slv = ldlm_pool_get_slv(pl);
 	lvf = ldlm_pool_get_lvf(pl);
-	la = cfs_duration_sec(cfs_time_sub(cur, lock->l_last_used));
+	la = (cur - lock->l_last_used) / HZ;
 	lv = lvf * la * unused;
 
 	/* Inform pool about current CLV to see it via debugfs. */
@@ -1268,8 +1233,7 @@ static enum ldlm_policy_res ldlm_cancel_aged_policy(struct ldlm_namespace *ns,
 						    int count)
 {
 	if ((added >= count) &&
-	    time_before(cfs_time_current(),
-			cfs_time_add(lock->l_last_used, ns->ns_max_age)))
+	    time_before(jiffies, lock->l_last_used + ns->ns_max_age))
 		return LDLM_POLICY_KEEP_LOCK;
 
 	return LDLM_POLICY_CANCEL_LOCK;
@@ -1415,7 +1379,7 @@ static int ldlm_prepare_lru_list(struct ldlm_namespace *ns,
 				continue;
 
 			last_use = lock->l_last_used;
-			if (last_use == cfs_time_current())
+			if (last_use == jiffies)
 				continue;
 
 			/* Somebody is already doing CANCEL. No need for this
@@ -1907,18 +1871,7 @@ static int replay_lock_interpret(const struct lu_env *env,
 
 	/* Key change rehash lock in per-export hash with new key */
 	exp = req->rq_export;
-	if (exp && exp->exp_lock_hash) {
-		/* In the function below, .hs_keycmp resolves to
-		 * ldlm_export_lock_keycmp()
-		 */
-		/* coverity[overrun-buffer-val] */
-		cfs_hash_rehash_key(exp->exp_lock_hash,
-				    &lock->l_remote_handle,
-				    &reply->lock_handle,
-				    &lock->l_exp_hash);
-	} else {
-		lock->l_remote_handle = reply->lock_handle;
-	}
+	lock->l_remote_handle = reply->lock_handle;
 
 	LDLM_DEBUG(lock, "replayed lock:");
 	ptlrpc_import_recovery_state_machine(req->rq_import);
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c b/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
index 4c44603ab6f9..28cd8398d4ce 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
@@ -41,6 +41,7 @@
 #include <lustre_fid.h>
 #include <obd_class.h>
 #include "ldlm_internal.h"
+#include <linux/libcfs/libcfs_hash.h>
 
 struct kmem_cache *ldlm_resource_slab, *ldlm_lock_slab;
 
@@ -688,6 +689,9 @@ struct ldlm_namespace *ldlm_namespace_new(struct obd_device *obd, char *name,
 	ns->ns_obd      = obd;
 	ns->ns_appetite = apt;
 	ns->ns_client   = client;
+	ns->ns_name     = kstrdup(name, GFP_KERNEL);
+	if (!ns->ns_name)
+		goto out_hash;
 
 	INIT_LIST_HEAD(&ns->ns_list_chain);
 	INIT_LIST_HEAD(&ns->ns_unused_list);
@@ -730,6 +734,7 @@ struct ldlm_namespace *ldlm_namespace_new(struct obd_device *obd, char *name,
 	ldlm_namespace_sysfs_unregister(ns);
 	ldlm_namespace_cleanup(ns, 0);
 out_hash:
+	kfree(ns->ns_name);
 	cfs_hash_putref(ns->ns_rs_hash);
 out_ns:
 	kfree(ns);
@@ -993,6 +998,7 @@ void ldlm_namespace_free_post(struct ldlm_namespace *ns)
 	ldlm_namespace_debugfs_unregister(ns);
 	ldlm_namespace_sysfs_unregister(ns);
 	cfs_hash_putref(ns->ns_rs_hash);
+	kfree(ns->ns_name);
 	/* Namespace \a ns should be not on list at this time, otherwise
 	 * this will cause issues related to using freed \a ns in poold
 	 * thread.
@@ -1195,6 +1201,7 @@ static void __ldlm_resource_putref_final(struct cfs_hash_bd *bd,
 					 struct ldlm_resource *res)
 {
 	struct ldlm_ns_bucket *nsb = res->lr_ns_bucket;
+	struct ldlm_namespace *ns = nsb->nsb_namespace;
 
 	if (!list_empty(&res->lr_granted)) {
 		ldlm_resource_dump(D_ERROR, res);
@@ -1206,15 +1213,18 @@ static void __ldlm_resource_putref_final(struct cfs_hash_bd *bd,
 		LBUG();
 	}
 
-	cfs_hash_bd_del_locked(nsb->nsb_namespace->ns_rs_hash,
+	cfs_hash_bd_del_locked(ns->ns_rs_hash,
 			       bd, &res->lr_hash);
 	lu_ref_fini(&res->lr_reference);
+	cfs_hash_bd_unlock(ns->ns_rs_hash, bd, 1);
+	if (ns->ns_lvbo && ns->ns_lvbo->lvbo_free)
+		ns->ns_lvbo->lvbo_free(res);
 	if (cfs_hash_bd_count_get(bd) == 0)
-		ldlm_namespace_put(nsb->nsb_namespace);
+		ldlm_namespace_put(ns);
+	kmem_cache_free(ldlm_resource_slab, res);
 }
 
-/* Returns 1 if the resource was freed, 0 if it remains. */
-int ldlm_resource_putref(struct ldlm_resource *res)
+void ldlm_resource_putref(struct ldlm_resource *res)
 {
 	struct ldlm_namespace *ns = ldlm_res_to_ns(res);
 	struct cfs_hash_bd   bd;
@@ -1224,15 +1234,8 @@ int ldlm_resource_putref(struct ldlm_resource *res)
 	       res, atomic_read(&res->lr_refcount) - 1);
 
 	cfs_hash_bd_get(ns->ns_rs_hash, &res->lr_name, &bd);
-	if (cfs_hash_bd_dec_and_lock(ns->ns_rs_hash, &bd, &res->lr_refcount)) {
+	if (cfs_hash_bd_dec_and_lock(ns->ns_rs_hash, &bd, &res->lr_refcount))
 		__ldlm_resource_putref_final(&bd, res);
-		cfs_hash_bd_unlock(ns->ns_rs_hash, &bd, 1);
-		if (ns->ns_lvbo && ns->ns_lvbo->lvbo_free)
-			ns->ns_lvbo->lvbo_free(res);
-		kmem_cache_free(ldlm_resource_slab, res);
-		return 1;
-	}
-	return 0;
 }
 EXPORT_SYMBOL(ldlm_resource_putref);
 
@@ -1319,14 +1322,14 @@ void ldlm_namespace_dump(int level, struct ldlm_namespace *ns)
 	CDEBUG(level, "--- Namespace: %s (rc: %d, side: client)\n",
 	       ldlm_ns_name(ns), atomic_read(&ns->ns_bref));
 
-	if (time_before(cfs_time_current(), ns->ns_next_dump))
+	if (time_before(jiffies, ns->ns_next_dump))
 		return;
 
 	cfs_hash_for_each_nolock(ns->ns_rs_hash,
 				 ldlm_res_hash_dump,
 				 (void *)(unsigned long)level, 0);
 	spin_lock(&ns->ns_lock);
-	ns->ns_next_dump = cfs_time_shift(10);
+	ns->ns_next_dump = jiffies + 10 * HZ;
 	spin_unlock(&ns->ns_lock);
 }
 
diff --git a/drivers/staging/lustre/lustre/llite/dir.c b/drivers/staging/lustre/lustre/llite/dir.c
index d10d27268323..688dddf3ca47 100644
--- a/drivers/staging/lustre/lustre/llite/dir.c
+++ b/drivers/staging/lustre/lustre/llite/dir.c
@@ -445,7 +445,7 @@ static int ll_dir_setdirstripe(struct inode *parent, struct lmv_user_md *lump,
 	err = md_create(sbi->ll_md_exp, op_data, lump, sizeof(*lump), mode,
 			from_kuid(&init_user_ns, current_fsuid()),
 			from_kgid(&init_user_ns, current_fsgid()),
-			cfs_curproc_cap_pack(), 0, &request);
+			current_cap(), 0, &request);
 	ll_finish_md_op_data(op_data);
 
 	err = ll_prep_inode(&inode, request, parent->i_sb, NULL);
@@ -1185,6 +1185,8 @@ static long ll_dir_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 		if (lumv1->lmm_magic == LOV_USER_MAGIC_V3) {
 			if (copy_from_user(&lumv3, lumv3p, sizeof(lumv3)))
 				return -EFAULT;
+			if (lumv3.lmm_magic != LOV_USER_MAGIC_V3)
+				return -EINVAL;
 		}
 
 		if (is_root_inode(inode))
diff --git a/drivers/staging/lustre/lustre/llite/file.c b/drivers/staging/lustre/lustre/llite/file.c
index ca5faea13b7e..a77cadcd9d48 100644
--- a/drivers/staging/lustre/lustre/llite/file.c
+++ b/drivers/staging/lustre/lustre/llite/file.c
@@ -2514,7 +2514,7 @@ ll_file_flock(struct file *file, int cmd, struct file_lock *file_lock)
 	       PFID(ll_inode2fid(inode)), flock.l_flock.pid, flags,
 	       einfo.ei_mode, flock.l_flock.start, flock.l_flock.end);
 
-	rc = md_enqueue(sbi->ll_md_exp, &einfo, &flock, NULL, op_data, &lockh,
+	rc = md_enqueue(sbi->ll_md_exp, &einfo, &flock, op_data, &lockh,
 			flags);
 
 	/* Restore the file lock type if not TEST lock. */
@@ -2527,7 +2527,7 @@ ll_file_flock(struct file *file, int cmd, struct file_lock *file_lock)
 
 	if (rc2 && file_lock->fl_type != F_UNLCK) {
 		einfo.ei_mode = LCK_NL;
-		md_enqueue(sbi->ll_md_exp, &einfo, &flock, NULL, op_data,
+		md_enqueue(sbi->ll_md_exp, &einfo, &flock, op_data,
 			   &lockh, flags);
 		rc = rc2;
 	}
@@ -3050,7 +3050,6 @@ int ll_inode_permission(struct inode *inode, int mask)
 	const struct cred *old_cred = NULL;
 	struct cred *cred = NULL;
 	bool squash_id = false;
-	cfs_cap_t cap;
 	int rc = 0;
 
 	if (mask & MAY_NOT_BLOCK)
@@ -3094,10 +3093,9 @@ int ll_inode_permission(struct inode *inode, int mask)
 
 		cred->fsuid = make_kuid(&init_user_ns, squash->rsi_uid);
 		cred->fsgid = make_kgid(&init_user_ns, squash->rsi_gid);
-		for (cap = 0; cap < sizeof(cfs_cap_t) * 8; cap++) {
-			if ((1 << cap) & CFS_CAP_FS_MASK)
-				cap_lower(cred->cap_effective, cap);
-		}
+		cred->cap_effective = cap_drop_nfsd_set(cred->cap_effective);
+		cred->cap_effective = cap_drop_fs_set(cred->cap_effective);
+
 		old_cred = override_creds(cred);
 	}
 
@@ -3474,12 +3472,7 @@ static int ll_layout_refresh_locked(struct inode *inode)
 	struct lookup_intent   it;
 	struct lustre_handle   lockh;
 	enum ldlm_mode	       mode;
-	struct ldlm_enqueue_info einfo = {
-		.ei_type = LDLM_IBITS,
-		.ei_mode = LCK_CR,
-		.ei_cb_bl = &ll_md_blocking_ast,
-		.ei_cb_cp = &ldlm_completion_ast,
-	};
+	struct ptlrpc_request *req;
 	int rc;
 
 again:
@@ -3503,13 +3496,13 @@ static int ll_layout_refresh_locked(struct inode *inode)
 	/* have to enqueue one */
 	memset(&it, 0, sizeof(it));
 	it.it_op = IT_LAYOUT;
-	lockh.cookie = 0ULL;
 
 	LDLM_DEBUG_NOLOCK("%s: requeue layout lock for file " DFID "(%p)",
 			  ll_get_fsname(inode->i_sb, NULL, 0),
 			  PFID(&lli->lli_fid), inode);
 
-	rc = md_enqueue(sbi->ll_md_exp, &einfo, NULL, &it, op_data, &lockh, 0);
+	rc = md_intent_lock(sbi->ll_md_exp, op_data, &it, &req,
+			    &ll_md_blocking_ast, 0);
 	ptlrpc_req_finished(it.it_request);
 	it.it_request = NULL;
 
@@ -3522,6 +3515,7 @@ static int ll_layout_refresh_locked(struct inode *inode)
 	if (rc == 0) {
 		/* set lock data in case this is a new lock */
 		ll_set_lock_data(sbi->ll_md_exp, inode, &it, NULL);
+		lockh.cookie = it.it_lock_handle;
 		rc = ll_layout_lock_set(&lockh, mode, inode);
 		if (rc == -EAGAIN)
 			goto again;
diff --git a/drivers/staging/lustre/lustre/llite/glimpse.c b/drivers/staging/lustre/lustre/llite/glimpse.c
index 3075358f3f08..ce0d51767da3 100644
--- a/drivers/staging/lustre/lustre/llite/glimpse.c
+++ b/drivers/staging/lustre/lustre/llite/glimpse.c
@@ -37,7 +37,6 @@
  *   Author: Oleg Drokin <oleg.drokin@sun.com>
  */
 
-#include <linux/libcfs/libcfs.h>
 #include <obd_class.h>
 #include <obd_support.h>
 #include <obd.h>
diff --git a/drivers/staging/lustre/lustre/llite/lcommon_cl.c b/drivers/staging/lustre/lustre/llite/lcommon_cl.c
index df5c0c0ae703..d7ea39ce0cb2 100644
--- a/drivers/staging/lustre/lustre/llite/lcommon_cl.c
+++ b/drivers/staging/lustre/lustre/llite/lcommon_cl.c
@@ -38,7 +38,6 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include <linux/libcfs/libcfs.h>
 #include <linux/fs.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
@@ -211,12 +210,12 @@ static void cl_object_put_last(struct lu_env *env, struct cl_object *obj)
 
 	if (unlikely(atomic_read(&header->loh_ref) != 1)) {
 		struct lu_site *site = obj->co_lu.lo_dev->ld_site;
-		struct lu_site_bkt_data *bkt;
+		wait_queue_head_t *wq;
 
-		bkt = lu_site_bkt_from_fid(site, &header->loh_fid);
+		wq = lu_site_wq_from_fid(site, &header->loh_fid);
 
 		init_waitqueue_entry(&waiter, current);
-		add_wait_queue(&bkt->lsb_marche_funebre, &waiter);
+		add_wait_queue(wq, &waiter);
 
 		while (1) {
 			set_current_state(TASK_UNINTERRUPTIBLE);
@@ -226,7 +225,7 @@ static void cl_object_put_last(struct lu_env *env, struct cl_object *obj)
 		}
 
 		set_current_state(TASK_RUNNING);
-		remove_wait_queue(&bkt->lsb_marche_funebre, &waiter);
+		remove_wait_queue(wq, &waiter);
 	}
 
 	cl_object_put(env, obj);
diff --git a/drivers/staging/lustre/lustre/llite/llite_internal.h b/drivers/staging/lustre/lustre/llite/llite_internal.h
index d46bcf71b273..379d88e20cd3 100644
--- a/drivers/staging/lustre/lustre/llite/llite_internal.h
+++ b/drivers/staging/lustre/lustre/llite/llite_internal.h
@@ -1110,7 +1110,7 @@ static inline int ll_glimpse_size(struct inode *inode)
 
 	down_read(&lli->lli_glimpse_sem);
 	rc = cl_glimpse_size(inode);
-	lli->lli_glimpse_time = cfs_time_current();
+	lli->lli_glimpse_time = jiffies;
 	up_read(&lli->lli_glimpse_sem);
 	return rc;
 }
@@ -1140,7 +1140,7 @@ dentry_may_statahead(struct inode *dir, struct dentry *dentry)
 		return false;
 
 	/* not the same process, don't statahead */
-	if (lli->lli_opendir_pid != current_pid())
+	if (lli->lli_opendir_pid != current->pid)
 		return false;
 
 	/*
diff --git a/drivers/staging/lustre/lustre/llite/llite_lib.c b/drivers/staging/lustre/lustre/llite/llite_lib.c
index e7500c53fafc..1bc0782feae3 100644
--- a/drivers/staging/lustre/lustre/llite/llite_lib.c
+++ b/drivers/staging/lustre/lustre/llite/llite_lib.c
@@ -41,6 +41,7 @@
 #include <linux/statfs.h>
 #include <linux/types.h>
 #include <linux/mm.h>
+#include <linux/random.h>
 
 #include <uapi/linux/lustre/lustre_ioctl.h>
 #include <lustre_ha.h>
@@ -257,7 +258,7 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt)
 	 * available
 	 */
 	err = obd_statfs(NULL, sbi->ll_md_exp, osfs,
-			 cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+			 get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 			 OBD_STATFS_FOR_MDT0);
 	if (err)
 		goto out_md_fid;
@@ -400,11 +401,11 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt)
 		LCONSOLE_ERROR_MSG(0x150,
 				   "An OST (dt %s) is performing recovery, of which this client is not a part.  Please wait for recovery to complete, abort, or time out.\n",
 				   dt);
-		goto out_md;
+		goto out_md_fid;
 	} else if (err) {
 		CERROR("%s: Cannot connect to %s: rc = %d\n",
 		       sbi->ll_dt_exp->exp_obd->obd_name, dt, err);
-		goto out_md;
+		goto out_md_fid;
 	}
 
 	sbi->ll_dt_exp->exp_connect_data = *data;
@@ -1675,7 +1676,7 @@ int ll_statfs(struct dentry *de, struct kstatfs *sfs)
 
 	/* Some amount of caching on the client is allowed */
 	rc = ll_statfs_internal(sb, &osfs,
-				cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+				get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 				0);
 	if (rc)
 		return rc;
@@ -2336,7 +2337,7 @@ struct md_op_data *ll_prep_md_op_data(struct md_op_data *op_data,
 	op_data->op_mod_time = ktime_get_real_seconds();
 	op_data->op_fsuid = from_kuid(&init_user_ns, current_fsuid());
 	op_data->op_fsgid = from_kgid(&init_user_ns, current_fsgid());
-	op_data->op_cap = cfs_curproc_cap_pack();
+	op_data->op_cap = current_cap();
 	if ((opc == LUSTRE_OPC_CREATE) && name &&
 	    filename_is_volatile(name, namelen, &op_data->op_mds))
 		op_data->op_bias |= MDS_CREATE_VOLATILE;
diff --git a/drivers/staging/lustre/lustre/llite/llite_mmap.c b/drivers/staging/lustre/lustre/llite/llite_mmap.c
index 214b07554e62..d7fb5533f707 100644
--- a/drivers/staging/lustre/lustre/llite/llite_mmap.c
+++ b/drivers/staging/lustre/lustre/llite/llite_mmap.c
@@ -152,7 +152,7 @@ static int ll_page_mkwrite0(struct vm_area_struct *vma, struct page *vmpage,
 	struct vvp_io	   *vio;
 	int		      result;
 	u16 refcheck;
-	sigset_t	     set;
+	sigset_t	     old, new;
 	struct inode	     *inode;
 	struct ll_inode_info     *lli;
 
@@ -177,14 +177,15 @@ static int ll_page_mkwrite0(struct vm_area_struct *vma, struct page *vmpage,
 	vio->u.fault.ft_vma    = vma;
 	vio->u.fault.ft_vmpage = vmpage;
 
-	cfs_block_sigsinv(sigmask(SIGKILL) | sigmask(SIGTERM), &set);
+	siginitsetinv(&new, sigmask(SIGKILL) | sigmask(SIGTERM));
+	sigprocmask(SIG_BLOCK, &new, &old);
 
 	inode = vvp_object_inode(io->ci_obj);
 	lli = ll_i2info(inode);
 
 	result = cl_io_loop(env, io);
 
-	cfs_restore_sigs(&set);
+	sigprocmask(SIG_SETMASK, &old, NULL);
 
 	if (result == 0) {
 		struct inode *inode = file_inode(vma->vm_file);
@@ -328,13 +329,14 @@ static int ll_fault(struct vm_fault *vmf)
 	int count = 0;
 	bool printed = false;
 	int result;
-	sigset_t set;
+	sigset_t old, new;
 
 	/* Only SIGKILL and SIGTERM are allowed for fault/nopage/mkwrite
 	 * so that it can be killed by admin but not cause segfault by
 	 * other signals.
 	 */
-	cfs_block_sigsinv(sigmask(SIGKILL) | sigmask(SIGTERM), &set);
+	siginitsetinv(&new, sigmask(SIGKILL) | sigmask(SIGTERM));
+	sigprocmask(SIG_BLOCK, &new, &old);
 
 restart:
 	result = ll_fault0(vmf->vma, vmf);
@@ -360,7 +362,7 @@ static int ll_fault(struct vm_fault *vmf)
 
 		result = VM_FAULT_LOCKED;
 	}
-	cfs_restore_sigs(&set);
+	sigprocmask(SIG_SETMASK, &old, NULL);
 	return result;
 }
 
diff --git a/drivers/staging/lustre/lustre/llite/llite_nfs.c b/drivers/staging/lustre/lustre/llite/llite_nfs.c
index a6a1d80c711a..14172688d55f 100644
--- a/drivers/staging/lustre/lustre/llite/llite_nfs.c
+++ b/drivers/staging/lustre/lustre/llite/llite_nfs.c
@@ -223,7 +223,7 @@ static int ll_nfs_get_name_filldir(struct dir_context *ctx, const char *name,
 	/* It is hack to access lde_fid for comparison with lgd_fid.
 	 * So the input 'name' must be part of the 'lu_dirent'.
 	 */
-	struct lu_dirent *lde = container_of0(name, struct lu_dirent, lde_name);
+	struct lu_dirent *lde = container_of((void*)name, struct lu_dirent, lde_name);
 	struct ll_getname_data *lgd =
 		container_of(ctx, struct ll_getname_data, ctx);
 	struct lu_fid fid;
diff --git a/drivers/staging/lustre/lustre/llite/lproc_llite.c b/drivers/staging/lustre/lustre/llite/lproc_llite.c
index 644bea2f9d37..164fe4d6b6b8 100644
--- a/drivers/staging/lustre/lustre/llite/lproc_llite.c
+++ b/drivers/staging/lustre/lustre/llite/lproc_llite.c
@@ -53,7 +53,7 @@ static ssize_t blocksize_show(struct kobject *kobj, struct attribute *attr,
 	int rc;
 
 	rc = ll_statfs_internal(sbi->ll_sb, &osfs,
-				cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+				get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 				OBD_STATFS_NODELAY);
 	if (!rc)
 		return sprintf(buf, "%u\n", osfs.os_bsize);
@@ -71,7 +71,7 @@ static ssize_t kbytestotal_show(struct kobject *kobj, struct attribute *attr,
 	int rc;
 
 	rc = ll_statfs_internal(sbi->ll_sb, &osfs,
-				cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+				get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 				OBD_STATFS_NODELAY);
 	if (!rc) {
 		__u32 blk_size = osfs.os_bsize >> 10;
@@ -96,7 +96,7 @@ static ssize_t kbytesfree_show(struct kobject *kobj, struct attribute *attr,
 	int rc;
 
 	rc = ll_statfs_internal(sbi->ll_sb, &osfs,
-				cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+				get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 				OBD_STATFS_NODELAY);
 	if (!rc) {
 		__u32 blk_size = osfs.os_bsize >> 10;
@@ -121,7 +121,7 @@ static ssize_t kbytesavail_show(struct kobject *kobj, struct attribute *attr,
 	int rc;
 
 	rc = ll_statfs_internal(sbi->ll_sb, &osfs,
-				cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+				get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 				OBD_STATFS_NODELAY);
 	if (!rc) {
 		__u32 blk_size = osfs.os_bsize >> 10;
@@ -146,7 +146,7 @@ static ssize_t filestotal_show(struct kobject *kobj, struct attribute *attr,
 	int rc;
 
 	rc = ll_statfs_internal(sbi->ll_sb, &osfs,
-				cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+				get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 				OBD_STATFS_NODELAY);
 	if (!rc)
 		return sprintf(buf, "%llu\n", osfs.os_files);
@@ -164,7 +164,7 @@ static ssize_t filesfree_show(struct kobject *kobj, struct attribute *attr,
 	int rc;
 
 	rc = ll_statfs_internal(sbi->ll_sb, &osfs,
-				cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+				get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 				OBD_STATFS_NODELAY);
 	if (!rc)
 		return sprintf(buf, "%llu\n", osfs.os_ffree);
diff --git a/drivers/staging/lustre/lustre/llite/namei.c b/drivers/staging/lustre/lustre/llite/namei.c
index 6c9ec462eb41..d5f6d20afe8c 100644
--- a/drivers/staging/lustre/lustre/llite/namei.c
+++ b/drivers/staging/lustre/lustre/llite/namei.c
@@ -442,11 +442,16 @@ struct dentry *ll_splice_alias(struct inode *inode, struct dentry *de)
 	} else {
 		struct dentry *new = d_splice_alias(inode, de);
 
+		if (IS_ERR(new))
+			CDEBUG(D_DENTRY,
+			       "splice inode %p as %pd gives error %lu\n",
+			       inode, de, PTR_ERR(new));
 		if (new)
 			de = new;
 	}
-	CDEBUG(D_DENTRY, "Add dentry %p inode %p refc %d flags %#x\n",
-	       de, d_inode(de), d_count(de), de->d_flags);
+	if (!IS_ERR(de))
+		CDEBUG(D_DENTRY, "Add dentry %p inode %p refc %d flags %#x\n",
+		       de, d_inode(de), d_count(de), de->d_flags);
 	return de;
 }
 
@@ -880,7 +885,7 @@ static int ll_new_node(struct inode *dir, struct dentry *dentry,
 	err = md_create(sbi->ll_md_exp, op_data, tgt, tgt_len, mode,
 			from_kuid(&init_user_ns, current_fsuid()),
 			from_kgid(&init_user_ns, current_fsgid()),
-			cfs_curproc_cap_pack(), rdev, &request);
+			current_cap(), rdev, &request);
 	ll_finish_md_op_data(op_data);
 	if (err < 0 && err != -EREMOTE)
 		goto err_exit;
diff --git a/drivers/staging/lustre/lustre/llite/range_lock.c b/drivers/staging/lustre/lustre/llite/range_lock.c
index cc9565f6bfe2..008a8874118d 100644
--- a/drivers/staging/lustre/lustre/llite/range_lock.c
+++ b/drivers/staging/lustre/lustre/llite/range_lock.c
@@ -36,6 +36,7 @@
  */
 #include "range_lock.h"
 #include <uapi/linux/lustre/lustre_idl.h>
+#include <linux/libcfs/libcfs.h>
 
 /**
  * Initialize a range lock tree
diff --git a/drivers/staging/lustre/lustre/llite/range_lock.h b/drivers/staging/lustre/lustre/llite/range_lock.h
index 38b2be4e378f..9ebac09160f2 100644
--- a/drivers/staging/lustre/lustre/llite/range_lock.h
+++ b/drivers/staging/lustre/lustre/llite/range_lock.h
@@ -37,7 +37,7 @@
 #ifndef _RANGE_LOCK_H
 #define _RANGE_LOCK_H
 
-#include <linux/libcfs/libcfs.h>
+#include <linux/spinlock.h>
 #include <interval_tree.h>
 
 struct range_lock {
diff --git a/drivers/staging/lustre/lustre/llite/statahead.c b/drivers/staging/lustre/lustre/llite/statahead.c
index 155ce3cf6f60..d864f5f36d85 100644
--- a/drivers/staging/lustre/lustre/llite/statahead.c
+++ b/drivers/staging/lustre/lustre/llite/statahead.c
@@ -523,7 +523,7 @@ static void ll_agl_trigger(struct inode *inode, struct ll_statahead_info *sai)
 	 *    affect the performance.
 	 */
 	if (lli->lli_glimpse_time != 0 &&
-	    time_before(cfs_time_shift(-1), lli->lli_glimpse_time)) {
+	    time_before(jiffies - 1 * HZ, lli->lli_glimpse_time)) {
 		up_write(&lli->lli_glimpse_sem);
 		lli->lli_agl_index = 0;
 		iput(inode);
@@ -535,7 +535,7 @@ static void ll_agl_trigger(struct inode *inode, struct ll_statahead_info *sai)
 
 	cl_agl(inode);
 	lli->lli_agl_index = 0;
-	lli->lli_glimpse_time = cfs_time_current();
+	lli->lli_glimpse_time = jiffies;
 	up_write(&lli->lli_glimpse_sem);
 
 	CDEBUG(D_READA, "Handled (init) async glimpse: inode= "
@@ -1075,7 +1075,7 @@ static int ll_statahead_thread(void *arg)
 			CDEBUG(D_READA, "Statahead for dir " DFID " hit ratio too low: hit/miss %llu/%llu, sent/replied %llu/%llu, stopping statahead thread: pid %d\n",
 			       PFID(&lli->lli_fid), sai->sai_hit,
 			       sai->sai_miss, sai->sai_sent,
-			       sai->sai_replied, current_pid());
+			       sai->sai_replied, current->pid);
 			break;
 		}
 	}
@@ -1147,7 +1147,7 @@ void ll_authorize_statahead(struct inode *dir, void *key)
 		 */
 		LASSERT(!lli->lli_opendir_pid);
 		lli->lli_opendir_key = key;
-		lli->lli_opendir_pid = current_pid();
+		lli->lli_opendir_pid = current->pid;
 		lli->lli_sa_enabled = 1;
 	}
 	spin_unlock(&lli->lli_sa_lock);
@@ -1506,7 +1506,7 @@ static int start_statahead_thread(struct inode *dir, struct dentry *dentry)
 	atomic_inc(&ll_i2sbi(parent->d_inode)->ll_sa_running);
 
 	CDEBUG(D_READA, "start statahead thread: [pid %d] [parent %pd]\n",
-	       current_pid(), parent);
+	       current->pid, parent);
 
 	task = kthread_create(ll_statahead_thread, parent, "ll_sa_%u",
 			      lli->lli_opendir_pid);
diff --git a/drivers/staging/lustre/lustre/llite/super25.c b/drivers/staging/lustre/lustre/llite/super25.c
index 861e7a60f408..d335f29556c2 100644
--- a/drivers/staging/lustre/lustre/llite/super25.c
+++ b/drivers/staging/lustre/lustre/llite/super25.c
@@ -90,6 +90,10 @@ static int __init lustre_init(void)
 	BUILD_BUG_ON(sizeof(LUSTRE_VOLATILE_HDR) !=
 		     LUSTRE_VOLATILE_HDR_LEN + 1);
 
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	/* print an address of _any_ initialized kernel symbol from this
 	 * module, to allow debugging with gdb that doesn't support data
 	 * symbols from modules.
diff --git a/drivers/staging/lustre/lustre/llite/vvp_dev.c b/drivers/staging/lustre/lustre/llite/vvp_dev.c
index 987c03b058e6..31dc3c0ade01 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_dev.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_dev.c
@@ -38,6 +38,7 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
+#include <linux/libcfs/libcfs_hash.h>
 #include <obd.h>
 #include "llite_internal.h"
 #include "vvp_internal.h"
@@ -390,6 +391,13 @@ struct vvp_pgcache_id {
 	struct lu_object_header *vpi_obj;
 };
 
+struct seq_private {
+	struct ll_sb_info	*sbi;
+	struct lu_env		*env;
+	u16			refcheck;
+	struct cl_object	*clob;
+};
+
 static void vvp_pgcache_id_unpack(loff_t pos, struct vvp_pgcache_id *id)
 {
 	BUILD_BUG_ON(sizeof(pos) != sizeof(__u64));
@@ -452,19 +460,20 @@ static struct cl_object *vvp_pgcache_obj(const struct lu_env *env,
 	return NULL;
 }
 
-static loff_t vvp_pgcache_find(const struct lu_env *env,
-			       struct lu_device *dev, loff_t pos)
+static struct page *vvp_pgcache_find(const struct lu_env *env,
+				     struct lu_device *dev,
+				     struct cl_object **clobp, loff_t *pos)
 {
 	struct cl_object     *clob;
 	struct lu_site       *site;
 	struct vvp_pgcache_id id;
 
 	site = dev->ld_site;
-	vvp_pgcache_id_unpack(pos, &id);
+	vvp_pgcache_id_unpack(*pos, &id);
 
 	while (1) {
 		if (id.vpi_bucket >= CFS_HASH_NHLIST(site->ls_obj_hash))
-			return ~0ULL;
+			return NULL;
 		clob = vvp_pgcache_obj(env, dev, &id);
 		if (clob) {
 			struct inode *inode = vvp_object_inode(clob);
@@ -476,20 +485,22 @@ static loff_t vvp_pgcache_find(const struct lu_env *env,
 			if (nr > 0) {
 				id.vpi_index = vmpage->index;
 				/* Cant support over 16T file */
-				nr = !(vmpage->index > 0xffffffff);
+				if (vmpage->index <= 0xffffffff) {
+					*clobp = clob;
+					*pos = vvp_pgcache_id_pack(&id);
+					return vmpage;
+				}
 				put_page(vmpage);
 			}
 
 			lu_object_ref_del(&clob->co_lu, "dump", current);
 			cl_object_put(env, clob);
-			if (nr > 0)
-				return vvp_pgcache_id_pack(&id);
 		}
 		/* to the next object. */
 		++id.vpi_depth;
 		id.vpi_depth &= 0xf;
 		if (id.vpi_depth == 0 && ++id.vpi_bucket == 0)
-			return ~0ULL;
+			return NULL;
 		id.vpi_index = 0;
 	}
 }
@@ -531,96 +542,53 @@ static void vvp_pgcache_page_show(const struct lu_env *env,
 
 static int vvp_pgcache_show(struct seq_file *f, void *v)
 {
-	loff_t		   pos;
-	struct ll_sb_info       *sbi;
-	struct cl_object	*clob;
-	struct lu_env	   *env;
-	struct vvp_pgcache_id    id;
-	u16 refcheck;
-	int		      result;
-
-	env = cl_env_get(&refcheck);
-	if (!IS_ERR(env)) {
-		pos = *(loff_t *)v;
-		vvp_pgcache_id_unpack(pos, &id);
-		sbi = f->private;
-		clob = vvp_pgcache_obj(env, &sbi->ll_cl->cd_lu_dev, &id);
-		if (clob) {
-			struct inode *inode = vvp_object_inode(clob);
-			struct cl_page *page = NULL;
-			struct page *vmpage;
-
-			result = find_get_pages_contig(inode->i_mapping,
-						       id.vpi_index, 1,
-						       &vmpage);
-			if (result > 0) {
-				lock_page(vmpage);
-				page = cl_vmpage_page(vmpage, clob);
-				unlock_page(vmpage);
-				put_page(vmpage);
-			}
-
-			seq_printf(f, "%8x@" DFID ": ", id.vpi_index,
-				   PFID(lu_object_fid(&clob->co_lu)));
-			if (page) {
-				vvp_pgcache_page_show(env, f, page);
-				cl_page_put(env, page);
-			} else {
-				seq_puts(f, "missing\n");
-			}
-			lu_object_ref_del(&clob->co_lu, "dump", current);
-			cl_object_put(env, clob);
-		} else {
-			seq_printf(f, "%llx missing\n", pos);
-		}
-		cl_env_put(env, &refcheck);
-		result = 0;
+	struct seq_private	*priv = f->private;
+	struct page		*vmpage = v;
+	struct cl_page		*page;
+
+	seq_printf(f, "%8lx@" DFID ": ", vmpage->index,
+		   PFID(lu_object_fid(&priv->clob->co_lu)));
+	lock_page(vmpage);
+	page = cl_vmpage_page(vmpage, priv->clob);
+	unlock_page(vmpage);
+	put_page(vmpage);
+
+	if (page) {
+		vvp_pgcache_page_show(priv->env, f, page);
+		cl_page_put(priv->env, page);
 	} else {
-		result = PTR_ERR(env);
+		seq_puts(f, "missing\n");
 	}
-	return result;
+	lu_object_ref_del(&priv->clob->co_lu, "dump", current);
+	cl_object_put(priv->env, priv->clob);
+
+	return 0;
 }
 
 static void *vvp_pgcache_start(struct seq_file *f, loff_t *pos)
 {
-	struct ll_sb_info *sbi;
-	struct lu_env     *env;
-	u16 refcheck;
+	struct seq_private	*priv = f->private;
+	struct page *ret;
 
-	sbi = f->private;
+	if (priv->sbi->ll_site->ls_obj_hash->hs_cur_bits >
+	    64 - PGC_OBJ_SHIFT)
+		ret = ERR_PTR(-EFBIG);
+	else
+		ret = vvp_pgcache_find(priv->env, &priv->sbi->ll_cl->cd_lu_dev,
+				       &priv->clob, pos);
 
-	env = cl_env_get(&refcheck);
-	if (!IS_ERR(env)) {
-		sbi = f->private;
-		if (sbi->ll_site->ls_obj_hash->hs_cur_bits >
-		    64 - PGC_OBJ_SHIFT) {
-			pos = ERR_PTR(-EFBIG);
-		} else {
-			*pos = vvp_pgcache_find(env, &sbi->ll_cl->cd_lu_dev,
-						*pos);
-			if (*pos == ~0ULL)
-				pos = NULL;
-		}
-		cl_env_put(env, &refcheck);
-	}
-	return pos;
+	return ret;
 }
 
 static void *vvp_pgcache_next(struct seq_file *f, void *v, loff_t *pos)
 {
-	struct ll_sb_info *sbi;
-	struct lu_env     *env;
-	u16 refcheck;
+	struct seq_private *priv = f->private;
+	struct page *ret;
 
-	env = cl_env_get(&refcheck);
-	if (!IS_ERR(env)) {
-		sbi = f->private;
-		*pos = vvp_pgcache_find(env, &sbi->ll_cl->cd_lu_dev, *pos + 1);
-		if (*pos == ~0ULL)
-			pos = NULL;
-		cl_env_put(env, &refcheck);
-	}
-	return pos;
+	*pos += 1;
+	ret = vvp_pgcache_find(priv->env, &priv->sbi->ll_cl->cd_lu_dev,
+			       &priv->clob, pos);
+	return ret;
 }
 
 static void vvp_pgcache_stop(struct seq_file *f, void *v)
@@ -637,23 +605,36 @@ static const struct seq_operations vvp_pgcache_ops = {
 
 static int vvp_dump_pgcache_seq_open(struct inode *inode, struct file *filp)
 {
-	struct seq_file *seq;
-	int rc;
+	struct seq_private *priv;
 
-	rc = seq_open(filp, &vvp_pgcache_ops);
-	if (rc)
-		return rc;
+	priv = __seq_open_private(filp, &vvp_pgcache_ops, sizeof(*priv));
+	if (!priv)
+		return -ENOMEM;
 
-	seq = filp->private_data;
-	seq->private = inode->i_private;
+	priv->sbi = inode->i_private;
+	priv->env = cl_env_get(&priv->refcheck);
+	if (IS_ERR(priv->env)) {
+		int err = PTR_ERR(priv->env);
 
+		seq_release_private(inode, filp);
+		return err;
+	}
 	return 0;
 }
 
+static int vvp_dump_pgcache_seq_release(struct inode *inode, struct file *file)
+{
+	struct seq_file *seq = file->private_data;
+	struct seq_private *priv = seq->private;
+
+	cl_env_put(priv->env, &priv->refcheck);
+	return seq_release_private(inode, file);
+}
+
 const struct file_operations vvp_dump_pgcache_file_ops = {
 	.owner   = THIS_MODULE,
 	.open    = vvp_dump_pgcache_seq_open,
 	.read    = seq_read,
 	.llseek	 = seq_lseek,
-	.release = seq_release,
+	.release = vvp_dump_pgcache_seq_release,
 };
diff --git a/drivers/staging/lustre/lustre/llite/vvp_internal.h b/drivers/staging/lustre/lustre/llite/vvp_internal.h
index 02ea5161d635..7d3abb43584a 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_internal.h
+++ b/drivers/staging/lustre/lustre/llite/vvp_internal.h
@@ -263,22 +263,22 @@ static inline struct lu_device *vvp2lu_dev(struct vvp_device *vdv)
 
 static inline struct vvp_device *lu2vvp_dev(const struct lu_device *d)
 {
-	return container_of0(d, struct vvp_device, vdv_cl.cd_lu_dev);
+	return container_of_safe(d, struct vvp_device, vdv_cl.cd_lu_dev);
 }
 
 static inline struct vvp_device *cl2vvp_dev(const struct cl_device *d)
 {
-	return container_of0(d, struct vvp_device, vdv_cl);
+	return container_of_safe(d, struct vvp_device, vdv_cl);
 }
 
 static inline struct vvp_object *cl2vvp(const struct cl_object *obj)
 {
-	return container_of0(obj, struct vvp_object, vob_cl);
+	return container_of_safe(obj, struct vvp_object, vob_cl);
 }
 
 static inline struct vvp_object *lu2vvp(const struct lu_object *obj)
 {
-	return container_of0(obj, struct vvp_object, vob_cl.co_lu);
+	return container_of_safe(obj, struct vvp_object, vob_cl.co_lu);
 }
 
 static inline struct inode *vvp_object_inode(const struct cl_object *obj)
diff --git a/drivers/staging/lustre/lustre/llite/vvp_object.c b/drivers/staging/lustre/lustre/llite/vvp_object.c
index 05ad3b322a29..b2cb51c8f7f4 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_object.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_object.c
@@ -37,8 +37,6 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include <linux/libcfs/libcfs.h>
-
 #include <obd.h>
 
 #include "llite_internal.h"
diff --git a/drivers/staging/lustre/lustre/llite/xattr.c b/drivers/staging/lustre/lustre/llite/xattr.c
index 2d78432963dc..1a597a640b6c 100644
--- a/drivers/staging/lustre/lustre/llite/xattr.c
+++ b/drivers/staging/lustre/lustre/llite/xattr.c
@@ -46,15 +46,16 @@
 
 const struct xattr_handler *get_xattr_type(const char *name)
 {
-	int i = 0;
+	int i;
 
-	while (ll_xattr_handlers[i]) {
-		size_t len = strlen(ll_xattr_handlers[i]->prefix);
+	for (i = 0; ll_xattr_handlers[i]; i++) {
+		const char *prefix = xattr_prefix(ll_xattr_handlers[i]);
+		size_t prefix_len = strlen(prefix);
 
-		if (!strncmp(ll_xattr_handlers[i]->prefix, name, len))
+		if (!strncmp(prefix, name, prefix_len))
 			return ll_xattr_handlers[i];
-		i++;
 	}
+
 	return NULL;
 }
 
@@ -81,20 +82,23 @@ static int xattr_type_filter(struct ll_sb_info *sbi,
 	return 0;
 }
 
-static int
-ll_xattr_set_common(const struct xattr_handler *handler,
-		    struct dentry *dentry, struct inode *inode,
-		    const char *name, const void *value, size_t size,
-		    int flags)
+static int ll_xattr_set_common(const struct xattr_handler *handler,
+			       struct dentry *dentry, struct inode *inode,
+			       const char *name, const void *value, size_t size,
+			       int flags)
 {
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
 	struct ptlrpc_request *req = NULL;
 	const char *pv = value;
 	char *fullname;
-	__u64 valid;
+	u64 valid;
 	int rc;
 
-	if (flags == XATTR_REPLACE) {
+	/* When setxattr() is called with a size of 0 the value is
+	 * unconditionally replaced by "". When removexattr() is
+	 * called we get a NULL value and XATTR_REPLACE for flags.
+	 */
+	if (!value && flags == XATTR_REPLACE) {
 		ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_REMOVEXATTR, 1);
 		valid = OBD_MD_FLXATTRRM;
 	} else {
@@ -117,11 +121,6 @@ ll_xattr_set_common(const struct xattr_handler *handler,
 	     (handler->flags == XATTR_LUSTRE_T && !strcmp(name, "lov"))))
 		return 0;
 
-	/* b15587: ignore security.capability xattr for now */
-	if ((handler->flags == XATTR_SECURITY_T &&
-	     !strcmp(name, "capability")))
-		return 0;
-
 	/* LU-549:  Disable security.selinux when selinux is disabled */
 	if (handler->flags == XATTR_SECURITY_T && !selinux_is_enabled() &&
 	    strcmp(name, "selinux") == 0)
@@ -141,12 +140,12 @@ ll_xattr_set_common(const struct xattr_handler *handler,
 			return -EPERM;
 	}
 
-	fullname = kasprintf(GFP_KERNEL, "%s%s\n", handler->prefix, name);
+	fullname = kasprintf(GFP_KERNEL, "%s%s", handler->prefix, name);
 	if (!fullname)
 		return -ENOMEM;
-	rc = md_setxattr(sbi->ll_md_exp, ll_inode2fid(inode),
-			 valid, fullname, pv, size, 0, flags,
-			 ll_i2suppgid(inode), &req);
+
+	rc = md_setxattr(sbi->ll_md_exp, ll_inode2fid(inode), valid, fullname,
+			 pv, size, 0, flags, ll_i2suppgid(inode), &req);
 	kfree(fullname);
 	if (rc) {
 		if (rc == -EOPNOTSUPP && handler->flags == XATTR_USER_T) {
@@ -191,6 +190,95 @@ static int get_hsm_state(struct inode *inode, u32 *hus_states)
 	return rc;
 }
 
+static int ll_adjust_lum(struct inode *inode, struct lov_user_md *lump)
+{
+	int rc = 0;
+
+	if (!lump)
+		return 0;
+
+	/* Attributes that are saved via getxattr will always have
+	 * the stripe_offset as 0.  Instead, the MDS should be
+	 * allowed to pick the starting OST index.   b=17846
+	 */
+	if (lump->lmm_stripe_offset == 0)
+		lump->lmm_stripe_offset = -1;
+
+	/* Avoid anyone directly setting the RELEASED flag. */
+	if (lump->lmm_pattern & LOV_PATTERN_F_RELEASED) {
+		/* Only if we have a released flag check if the file
+		 * was indeed archived.
+		 */
+		u32 state = HS_NONE;
+
+		rc = get_hsm_state(inode, &state);
+		if (rc)
+			return rc;
+
+		if (!(state & HS_ARCHIVED)) {
+			CDEBUG(D_VFSTRACE,
+			       "hus_states state = %x, pattern = %x\n",
+				state, lump->lmm_pattern);
+			/*
+			 * Here the state is: real file is not
+			 * archived but user is requesting to set
+			 * the RELEASED flag so we mask off the
+			 * released flag from the request
+			 */
+			lump->lmm_pattern ^= LOV_PATTERN_F_RELEASED;
+		}
+	}
+
+	return rc;
+}
+
+static int ll_setstripe_ea(struct dentry *dentry, struct lov_user_md *lump,
+			   size_t size)
+{
+	struct inode *inode = d_inode(dentry);
+	int rc = 0;
+
+	/*
+	 * It is possible to set an xattr to a "" value of zero size.
+	 * For this case we are going to treat it as a removal.
+	 */
+	if (!size && lump)
+		lump = NULL;
+
+	rc = ll_adjust_lum(inode, lump);
+	if (rc)
+		return rc;
+
+	if (lump && S_ISREG(inode->i_mode)) {
+		u64 it_flags = FMODE_WRITE;
+		ssize_t lum_size;
+
+		lum_size = ll_lov_user_md_size(lump);
+		if (lum_size < 0 || size < lum_size)
+			return -ERANGE;
+
+		rc = ll_lov_setstripe_ea_info(inode, dentry, it_flags, lump,
+					      lum_size);
+		/**
+		 * b=10667: ignore -EEXIST.
+		 * Silently eat error on setting trusted.lov/lustre.lov
+		 * attribute for platforms that added the default option
+		 * to copy all attributes in 'cp' command. Both rsync and
+		 * tar --xattrs also will try to set LOVEA for existing
+		 * files.
+		 */
+		if (rc == -EEXIST)
+			rc = 0;
+	} else if (S_ISDIR(inode->i_mode)) {
+		if (size != 0 && size < sizeof(struct lov_user_md))
+			return -EINVAL;
+
+		rc = ll_dir_setstripe(inode, lump, 0);
+	}
+
+	return rc;
+}
+
 static int ll_xattr_set(const struct xattr_handler *handler,
 			struct dentry *dentry, struct inode *inode,
 			const char *name, const void *value, size_t size,
@@ -202,76 +290,20 @@ static int ll_xattr_set(const struct xattr_handler *handler,
 	CDEBUG(D_VFSTRACE, "VFS Op:inode=" DFID "(%p), xattr %s\n",
 	       PFID(ll_inode2fid(inode)), inode, name);
 
+	/* lustre/trusted.lov.xxx would be passed through xattr API */
 	if (!strcmp(name, "lov")) {
-		struct lov_user_md *lump = (struct lov_user_md *)value;
 		int op_type = flags == XATTR_REPLACE ? LPROC_LL_REMOVEXATTR :
 						       LPROC_LL_SETXATTR;
-		int rc = 0;
 
 		ll_stats_ops_tally(ll_i2sbi(inode), op_type, 1);
 
-		if (size != 0 && size < sizeof(struct lov_user_md))
-			return -EINVAL;
-
-		/*
-		 * It is possible to set an xattr to a "" value of zero size.
-		 * For this case we are going to treat it as a removal.
-		 */
-		if (!size && lump)
-			lump = NULL;
-
-		/* Attributes that are saved via getxattr will always have
-		 * the stripe_offset as 0.  Instead, the MDS should be
-		 * allowed to pick the starting OST index.   b=17846
-		 */
-		if (lump && lump->lmm_stripe_offset == 0)
-			lump->lmm_stripe_offset = -1;
-
-		/* Avoid anyone directly setting the RELEASED flag. */
-		if (lump && (lump->lmm_pattern & LOV_PATTERN_F_RELEASED)) {
-			/* Only if we have a released flag check if the file
-			 * was indeed archived.
-			 */
-			u32 state = HS_NONE;
-
-			rc = get_hsm_state(inode, &state);
-			if (rc)
-				return rc;
-
-			if (!(state & HS_ARCHIVED)) {
-				CDEBUG(D_VFSTRACE,
-				       "hus_states state = %x, pattern = %x\n",
-				state, lump->lmm_pattern);
-				/*
-				 * Here the state is: real file is not
-				 * archived but user is requesting to set
-				 * the RELEASED flag so we mask off the
-				 * released flag from the request
-				 */
-				lump->lmm_pattern ^= LOV_PATTERN_F_RELEASED;
-			}
-		}
-
-		if (lump && S_ISREG(inode->i_mode)) {
-			__u64 it_flags = FMODE_WRITE;
-			int lum_size;
-
-			lum_size = ll_lov_user_md_size(lump);
-			if (lum_size < 0 || size < lum_size)
-				return 0; /* b=10667: ignore error */
-
-			rc = ll_lov_setstripe_ea_info(inode, dentry, it_flags,
-						      lump, lum_size);
-			/* b=10667: rc always be 0 here for now */
-			rc = 0;
-		} else if (S_ISDIR(inode->i_mode)) {
-			rc = ll_dir_setstripe(inode, lump, 0);
-		}
-
-		return rc;
-
+		return ll_setstripe_ea(dentry, (struct lov_user_md *)value,
+				       size);
 	} else if (!strcmp(name, "lma") || !strcmp(name, "link")) {
-		ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_SETXATTR, 1);
+		int op_type = flags == XATTR_REPLACE ? LPROC_LL_REMOVEXATTR :
+						       LPROC_LL_SETXATTR;
+
+		ll_stats_ops_tally(ll_i2sbi(inode), op_type, 1);
 		return 0;
 	}
 
@@ -279,9 +311,8 @@ static int ll_xattr_set(const struct xattr_handler *handler,
 				   flags);
 }
 
-int
-ll_xattr_list(struct inode *inode, const char *name, int type, void *buffer,
-	      size_t size, __u64 valid)
+int ll_xattr_list(struct inode *inode, const char *name, int type, void *buffer,
+		  size_t size, u64 valid)
 {
 	struct ll_inode_info *lli = ll_i2info(inode);
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
@@ -368,9 +399,6 @@ static int ll_xattr_get_common(const struct xattr_handler *handler,
 			       const char *name, void *buffer, size_t size)
 {
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
-#ifdef CONFIG_FS_POSIX_ACL
-	struct ll_inode_info *lli = ll_i2info(inode);
-#endif
 	char *fullname;
 	int rc;
 
@@ -383,10 +411,6 @@ static int ll_xattr_get_common(const struct xattr_handler *handler,
 	if (rc)
 		return rc;
 
-	/* b15587: ignore security.capability xattr for now */
-	if ((handler->flags == XATTR_SECURITY_T && !strcmp(name, "capability")))
-		return -ENODATA;
-
 	/* LU-549:  Disable security.selinux when selinux is disabled */
 	if (handler->flags == XATTR_SECURITY_T && !selinux_is_enabled() &&
 	    !strcmp(name, "selinux"))
@@ -398,6 +422,7 @@ static int ll_xattr_get_common(const struct xattr_handler *handler,
 	 * chance that cached ACL is uptodate.
 	 */
 	if (handler->flags == XATTR_ACL_ACCESS_T) {
+		struct ll_inode_info *lli = ll_i2info(inode);
 		struct posix_acl *acl;
 
 		spin_lock(&lli->lli_lock);
@@ -414,9 +439,10 @@ static int ll_xattr_get_common(const struct xattr_handler *handler,
 	if (handler->flags == XATTR_ACL_DEFAULT_T && !S_ISDIR(inode->i_mode))
 		return -ENODATA;
 #endif
-	fullname = kasprintf(GFP_KERNEL, "%s%s\n", handler->prefix, name);
+	fullname = kasprintf(GFP_KERNEL, "%s%s", handler->prefix, name);
 	if (!fullname)
 		return -ENOMEM;
+
 	rc = ll_xattr_list(inode, fullname, handler->flags, buffer, size,
 			   OBD_MD_FLXATTR);
 	kfree(fullname);
@@ -540,9 +566,10 @@ ssize_t ll_listxattr(struct dentry *dentry, char *buffer, size_t size)
 			   OBD_MD_FLXATTRLS);
 	if (rc < 0)
 		return rc;
+
 	/*
 	 * If we're being called to get the size of the xattr list
-	 * (buf_size == 0) then just assume that a lustre.lov xattr
+	 * (size == 0) then just assume that a lustre.lov xattr
 	 * exists.
 	 */
 	if (!size)
@@ -555,14 +582,14 @@ ssize_t ll_listxattr(struct dentry *dentry, char *buffer, size_t size)
 		len = strnlen(xattr_name, rem - 1) + 1;
 		rem -= len;
 		if (!xattr_type_filter(sbi, get_xattr_type(xattr_name))) {
-			/* Skip OK xattr type leave it in buffer */
+			/* Skip OK xattr type, leave it in buffer. */
 			xattr_name += len;
 			continue;
 		}
 
 		/*
 		 * Move up remaining xattrs in buffer
-		 * removing the xattr that is not OK
+		 * removing the xattr that is not OK.
 		 */
 		memmove(xattr_name, xattr_name + len, rem);
 		rc -= len;
@@ -605,14 +632,14 @@ static const struct xattr_handler ll_security_xattr_handler = {
 };
 
 static const struct xattr_handler ll_acl_access_xattr_handler = {
-	.prefix = XATTR_NAME_POSIX_ACL_ACCESS,
+	.name = XATTR_NAME_POSIX_ACL_ACCESS,
 	.flags = XATTR_ACL_ACCESS_T,
 	.get = ll_xattr_get_common,
 	.set = ll_xattr_set_common,
 };
 
 static const struct xattr_handler ll_acl_default_xattr_handler = {
-	.prefix = XATTR_NAME_POSIX_ACL_DEFAULT,
+	.name = XATTR_NAME_POSIX_ACL_DEFAULT,
 	.flags = XATTR_ACL_DEFAULT_T,
 	.get = ll_xattr_get_common,
 	.set = ll_xattr_set_common,
diff --git a/drivers/staging/lustre/lustre/llite/xattr_cache.c b/drivers/staging/lustre/lustre/llite/xattr_cache.c
index 4dc799d60a9f..5da69ba088c4 100644
--- a/drivers/staging/lustre/lustre/llite/xattr_cache.c
+++ b/drivers/staging/lustre/lustre/llite/xattr_cache.c
@@ -121,10 +121,12 @@ static int ll_xattr_cache_add(struct list_head *cache,
 
 	xattr->xe_name = kstrdup(xattr_name, GFP_NOFS);
 	if (!xattr->xe_name) {
-		CDEBUG(D_CACHE, "failed to alloc xattr name %u\n",
-		       xattr->xe_namelen);
+		CDEBUG(D_CACHE, "failed to alloc xattr name %s\n",
+		       xattr_name);
 		goto err_name;
 	}
+	xattr->xe_namelen = strlen(xattr_name) + 1;
+
 	xattr->xe_value = kmemdup(xattr_val, xattr_val_len, GFP_NOFS);
 	if (!xattr->xe_value)
 		goto err_value;
@@ -270,12 +272,6 @@ static int ll_xattr_find_get_lock(struct inode *inode,
 	struct lustre_handle lockh = { 0 };
 	struct md_op_data *op_data;
 	struct ll_inode_info *lli = ll_i2info(inode);
-	struct ldlm_enqueue_info einfo = {
-		.ei_type = LDLM_IBITS,
-		.ei_mode = it_to_lock_mode(oit),
-		.ei_cb_bl = &ll_md_blocking_ast,
-		.ei_cb_cp = &ldlm_completion_ast,
-	};
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
 	struct obd_export *exp = sbi->ll_md_exp;
 	int rc;
@@ -306,8 +302,9 @@ static int ll_xattr_find_get_lock(struct inode *inode,
 
 	op_data->op_valid = OBD_MD_FLXATTR | OBD_MD_FLXATTRLS;
 
-	rc = md_enqueue(exp, &einfo, NULL, oit, op_data, &lockh, 0);
+	rc = md_intent_lock(exp, op_data, oit, req, &ll_md_blocking_ast, 0);
 	ll_finish_md_op_data(op_data);
+	*req = oit->it_request;
 
 	if (rc < 0) {
 		CDEBUG(D_CACHE,
@@ -317,7 +314,6 @@ static int ll_xattr_find_get_lock(struct inode *inode,
 		return rc;
 	}
 
-	*req = oit->it_request;
 out:
 	down_write(&lli->lli_xattrs_list_rwsem);
 	mutex_unlock(&lli->lli_xattrs_enq_lock);
@@ -328,16 +324,15 @@ static int ll_xattr_find_get_lock(struct inode *inode,
 /**
  * Refill the xattr cache.
  *
- * Fetch and cache the whole of xattrs for @inode, acquiring
- * a read or a write xattr lock depending on operation in @oit.
- * Intent is dropped on exit unless the operation is setxattr.
+ * Fetch and cache the whole of xattrs for @inode, acquiring a read lock.
  *
  * \retval 0       no error occurred
  * \retval -EPROTO network protocol error
  * \retval -ENOMEM not enough memory for the cache
  */
-static int ll_xattr_cache_refill(struct inode *inode, struct lookup_intent *oit)
+static int ll_xattr_cache_refill(struct inode *inode)
 {
+	struct lookup_intent oit = { .it_op = IT_GETXATTR };
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
 	struct ptlrpc_request *req = NULL;
 	const char *xdata, *xval, *xtail, *xvtail;
@@ -346,40 +341,31 @@ static int ll_xattr_cache_refill(struct inode *inode, struct lookup_intent *oit)
 	__u32 *xsizes;
 	int rc, i;
 
-	rc = ll_xattr_find_get_lock(inode, oit, &req);
+	rc = ll_xattr_find_get_lock(inode, &oit, &req);
 	if (rc)
-		goto out_no_unlock;
+		goto err_req;
 
 	/* Do we have the data at this point? */
 	if (ll_xattr_cache_valid(lli)) {
 		ll_stats_ops_tally(sbi, LPROC_LL_GETXATTR_HITS, 1);
+		ll_intent_drop_lock(&oit);
 		rc = 0;
-		goto out_maybe_drop;
+		goto err_req;
 	}
 
 	/* Matched but no cache? Cancelled on error by a parallel refill. */
 	if (unlikely(!req)) {
 		CDEBUG(D_CACHE, "cancelled by a parallel getxattr\n");
-		rc = -EIO;
-		goto out_maybe_drop;
-	}
-
-	if (oit->it_status < 0) {
-		CDEBUG(D_CACHE,
-		       "getxattr intent returned %d for fid " DFID "\n",
-		       oit->it_status, PFID(ll_inode2fid(inode)));
-		rc = oit->it_status;
-		/* xattr data is so large that we don't want to cache it */
-		if (rc == -ERANGE)
-			rc = -EAGAIN;
-		goto out_destroy;
+		ll_intent_drop_lock(&oit);
+		rc = -EAGAIN;
+		goto err_unlock;
 	}
 
 	body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
 	if (!body) {
 		CERROR("no MDT BODY in the refill xattr reply\n");
 		rc = -EPROTO;
-		goto out_destroy;
+		goto err_cancel;
 	}
 	/* do not need swab xattr data */
 	xdata = req_capsule_server_sized_get(&req->rq_pill, &RMF_EADATA,
@@ -391,7 +377,7 @@ static int ll_xattr_cache_refill(struct inode *inode, struct lookup_intent *oit)
 	if (!xdata || !xval || !xsizes) {
 		CERROR("wrong setxattr reply\n");
 		rc = -EPROTO;
-		goto out_destroy;
+		goto err_cancel;
 	}
 
 	xtail = xdata + body->mbo_eadatasize;
@@ -427,7 +413,7 @@ static int ll_xattr_cache_refill(struct inode *inode, struct lookup_intent *oit)
 		}
 		if (rc < 0) {
 			ll_xattr_cache_destroy_locked(lli);
-			goto out_destroy;
+			goto err_cancel;
 		}
 		xdata += strlen(xdata) + 1;
 		xval  += *xsizes;
@@ -437,28 +423,24 @@ static int ll_xattr_cache_refill(struct inode *inode, struct lookup_intent *oit)
 	if (xdata != xtail || xval != xvtail)
 		CERROR("a hole in xattr data\n");
 
-	ll_set_lock_data(sbi->ll_md_exp, inode, oit, NULL);
-
-	goto out_maybe_drop;
-out_maybe_drop:
+	ll_set_lock_data(sbi->ll_md_exp, inode, &oit, NULL);
+	ll_intent_drop_lock(&oit);
 
-		ll_intent_drop_lock(oit);
-
-	if (rc != 0)
-		up_write(&lli->lli_xattrs_list_rwsem);
-out_no_unlock:
 	ptlrpc_req_finished(req);
-
 	return rc;
 
-out_destroy:
-	up_write(&lli->lli_xattrs_list_rwsem);
-
+err_cancel:
 	ldlm_lock_decref_and_cancel((struct lustre_handle *)
-					&oit->it_lock_handle,
-					oit->it_lock_mode);
+				    &oit.it_lock_handle,
+				    oit.it_lock_mode);
+err_unlock:
+	up_write(&lli->lli_xattrs_list_rwsem);
+err_req:
+	if (rc == -ERANGE)
+		rc = -EAGAIN;
 
-	goto out_no_unlock;
+	ptlrpc_req_finished(req);
+	return rc;
 }
 
 /**
@@ -478,7 +460,6 @@ static int ll_xattr_cache_refill(struct inode *inode, struct lookup_intent *oit)
 int ll_xattr_cache_get(struct inode *inode, const char *name, char *buffer,
 		       size_t size, __u64 valid)
 {
-	struct lookup_intent oit = { .it_op = IT_GETXATTR };
 	struct ll_inode_info *lli = ll_i2info(inode);
 	int rc = 0;
 
@@ -487,7 +468,7 @@ int ll_xattr_cache_get(struct inode *inode, const char *name, char *buffer,
 	down_read(&lli->lli_xattrs_list_rwsem);
 	if (!ll_xattr_cache_valid(lli)) {
 		up_read(&lli->lli_xattrs_list_rwsem);
-		rc = ll_xattr_cache_refill(inode, &oit);
+		rc = ll_xattr_cache_refill(inode);
 		if (rc)
 			return rc;
 		downgrade_write(&lli->lli_xattrs_list_rwsem);
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_intent.c b/drivers/staging/lustre/lustre/lmv/lmv_intent.c
index 1793c9f79b24..1e850fdbc623 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_intent.c
+++ b/drivers/staging/lustre/lustre/lmv/lmv_intent.c
@@ -447,6 +447,9 @@ static int lmv_intent_lookup(struct obd_export *exp,
 		}
 	}
 
+	if (!it_has_reply_body(it))
+		return 0;
+
 	/*
 	 * MDS has returned success. Probably name has been resolved in
 	 * remote inode. Let's check this.
@@ -483,7 +486,7 @@ int lmv_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
 	       (int)op_data->op_namelen, op_data->op_name,
 	       PFID(&op_data->op_fid1));
 
-	if (it->it_op & (IT_LOOKUP | IT_GETATTR | IT_LAYOUT))
+	if (it->it_op & (IT_LOOKUP | IT_GETATTR | IT_LAYOUT | IT_GETXATTR))
 		rc = lmv_intent_lookup(exp, op_data, it, reqp, cb_blocking,
 				       extra_lock_flags);
 	else if (it->it_op & IT_OPEN)
@@ -497,7 +500,8 @@ int lmv_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
 
 		if (it->it_lock_mode) {
 			lock_handle.cookie = it->it_lock_handle;
-			ldlm_lock_decref(&lock_handle, it->it_lock_mode);
+			ldlm_lock_decref_and_cancel(&lock_handle,
+						    it->it_lock_mode);
 		}
 
 		it->it_lock_handle = 0;
@@ -505,8 +509,8 @@ int lmv_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
 
 		if (it->it_remote_lock_mode) {
 			lock_handle.cookie = it->it_remote_lock_handle;
-			ldlm_lock_decref(&lock_handle,
-					 it->it_remote_lock_mode);
+			ldlm_lock_decref_and_cancel(&lock_handle,
+						    it->it_remote_lock_mode);
 		}
 
 		it->it_remote_lock_handle = 0;
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_internal.h b/drivers/staging/lustre/lustre/lmv/lmv_internal.h
index c27c3c32188d..68a99170c424 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_internal.h
+++ b/drivers/staging/lustre/lustre/lmv/lmv_internal.h
@@ -60,7 +60,7 @@ int lmv_revalidate_slaves(struct obd_export *exp,
 
 static inline struct obd_device *lmv2obd_dev(struct lmv_obd *lmv)
 {
-	return container_of0(lmv, struct obd_device, u.lmv);
+	return container_of_safe(lmv, struct obd_device, u.lmv);
 }
 
 static inline struct lmv_tgt_desc *
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_obd.c b/drivers/staging/lustre/lustre/lmv/lmv_obd.c
index e8a9b9902c37..60cdba99e4a4 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_obd.c
+++ b/drivers/staging/lustre/lustre/lmv/lmv_obd.c
@@ -37,11 +37,13 @@
 #include <linux/init.h>
 #include <linux/pagemap.h>
 #include <linux/mm.h>
+#include <linux/file.h>
 #include <asm/div64.h>
 #include <linux/seq_file.h>
 #include <linux/namei.h>
 #include <linux/uaccess.h>
 
+#include <linux/libcfs/libcfs.h>
 #include <obd_support.h>
 #include <lustre_net.h>
 #include <obd_class.h>
@@ -876,7 +878,7 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp,
 			return -EFAULT;
 
 		rc = obd_statfs(NULL, tgt->ltd_exp, &stat_buf,
-				cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+				get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 				0);
 		if (rc)
 			return rc;
@@ -1598,7 +1600,7 @@ lmv_locate_mds(struct lmv_obd *lmv, struct md_op_data *op_data,
 
 static int lmv_create(struct obd_export *exp, struct md_op_data *op_data,
 		      const void *data, size_t datalen, umode_t mode,
-		      uid_t uid, gid_t gid, cfs_cap_t cap_effective,
+		      uid_t uid, gid_t gid, kernel_cap_t cap_effective,
 		      __u64 rdev, struct ptlrpc_request **request)
 {
 	struct obd_device       *obd = exp->exp_obd;
@@ -1652,26 +1654,24 @@ static int lmv_create(struct obd_export *exp, struct md_op_data *op_data,
 
 static int
 lmv_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
-	    const union ldlm_policy_data *policy,
-	    struct lookup_intent *it, struct md_op_data *op_data,
+	    const union ldlm_policy_data *policy, struct md_op_data *op_data,
 	    struct lustre_handle *lockh, __u64 extra_lock_flags)
 {
 	struct obd_device	*obd = exp->exp_obd;
 	struct lmv_obd	   *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc      *tgt;
 
-	CDEBUG(D_INODE, "ENQUEUE '%s' on " DFID "\n",
-	       LL_IT2STR(it), PFID(&op_data->op_fid1));
+	CDEBUG(D_INODE, "ENQUEUE on " DFID "\n", PFID(&op_data->op_fid1));
 
 	tgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid1);
 	if (IS_ERR(tgt))
 		return PTR_ERR(tgt);
 
-	CDEBUG(D_INODE, "ENQUEUE '%s' on " DFID " -> mds #%u\n",
-	       LL_IT2STR(it), PFID(&op_data->op_fid1), tgt->ltd_idx);
+	CDEBUG(D_INODE, "ENQUEUE on " DFID " -> mds #%u\n",
+	       PFID(&op_data->op_fid1), tgt->ltd_idx);
 
-	return md_enqueue(tgt->ltd_exp, einfo, policy, it, op_data, lockh,
-			extra_lock_flags);
+	return md_enqueue(tgt->ltd_exp, einfo, policy, op_data, lockh,
+			  extra_lock_flags);
 }
 
 static int
@@ -1785,7 +1785,7 @@ static int lmv_link(struct obd_export *exp, struct md_op_data *op_data,
 
 	op_data->op_fsuid = from_kuid(&init_user_ns, current_fsuid());
 	op_data->op_fsgid = from_kgid(&init_user_ns, current_fsgid());
-	op_data->op_cap = cfs_curproc_cap_pack();
+	op_data->op_cap = current_cap();
 	if (op_data->op_mea2) {
 		struct lmv_stripe_md *lsm = op_data->op_mea2;
 		const struct lmv_oinfo *oinfo;
@@ -1837,7 +1837,7 @@ static int lmv_rename(struct obd_export *exp, struct md_op_data *op_data,
 
 	op_data->op_fsuid = from_kuid(&init_user_ns, current_fsuid());
 	op_data->op_fsgid = from_kgid(&init_user_ns, current_fsgid());
-	op_data->op_cap = cfs_curproc_cap_pack();
+	op_data->op_cap = current_cap();
 
 	if (op_data->op_cli_flags & CLI_MIGRATE) {
 		LASSERTF(fid_is_sane(&op_data->op_fid3), "invalid FID " DFID "\n",
@@ -2415,7 +2415,7 @@ static int lmv_unlink(struct obd_export *exp, struct md_op_data *op_data,
 
 	op_data->op_fsuid = from_kuid(&init_user_ns, current_fsuid());
 	op_data->op_fsgid = from_kgid(&init_user_ns, current_fsgid());
-	op_data->op_cap = cfs_curproc_cap_pack();
+	op_data->op_cap = current_cap();
 
 	/*
 	 * If child's fid is given, cancel unused locks for it if it is from
@@ -3110,9 +3110,14 @@ static struct md_ops lmv_md_ops = {
 static int __init lmv_init(void)
 {
 	struct lprocfs_static_vars lvars;
+	int rc;
 
 	lprocfs_lmv_init_vars(&lvars);
 
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	return class_register_type(&lmv_obd_ops, &lmv_md_ops,
 				 LUSTRE_LMV_NAME, NULL);
 }
diff --git a/drivers/staging/lustre/lustre/lov/lov_cl_internal.h b/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
index 1185eceaf497..e4f762137a4a 100644
--- a/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
+++ b/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
@@ -43,8 +43,6 @@
 #ifndef LOV_CL_INTERNAL_H
 #define LOV_CL_INTERNAL_H
 
-#include <linux/libcfs/libcfs.h>
-
 #include <obd.h>
 #include <cl_object.h>
 #include "lov_internal.h"
@@ -496,7 +494,7 @@ static inline struct lu_device *lov2lu_dev(struct lov_device *lov)
 static inline struct lov_device *lu2lov_dev(const struct lu_device *d)
 {
 	LINVRNT(d->ld_type == &lov_device_type);
-	return container_of0(d, struct lov_device, ld_cl.cd_lu_dev);
+	return container_of(d, struct lov_device, ld_cl.cd_lu_dev);
 }
 
 static inline struct cl_device *lovsub2cl_dev(struct lovsub_device *lovsub)
@@ -512,13 +510,13 @@ static inline struct lu_device *lovsub2lu_dev(struct lovsub_device *lovsub)
 static inline struct lovsub_device *lu2lovsub_dev(const struct lu_device *d)
 {
 	LINVRNT(d->ld_type == &lovsub_device_type);
-	return container_of0(d, struct lovsub_device, acid_cl.cd_lu_dev);
+	return container_of(d, struct lovsub_device, acid_cl.cd_lu_dev);
 }
 
 static inline struct lovsub_device *cl2lovsub_dev(const struct cl_device *d)
 {
 	LINVRNT(d->cd_lu_dev.ld_type == &lovsub_device_type);
-	return container_of0(d, struct lovsub_device, acid_cl);
+	return container_of(d, struct lovsub_device, acid_cl);
 }
 
 static inline struct lu_object *lov2lu(struct lov_object *lov)
@@ -534,13 +532,13 @@ static inline struct cl_object *lov2cl(struct lov_object *lov)
 static inline struct lov_object *lu2lov(const struct lu_object *obj)
 {
 	LINVRNT(lov_is_object(obj));
-	return container_of0(obj, struct lov_object, lo_cl.co_lu);
+	return container_of(obj, struct lov_object, lo_cl.co_lu);
 }
 
 static inline struct lov_object *cl2lov(const struct cl_object *obj)
 {
 	LINVRNT(lov_is_object(&obj->co_lu));
-	return container_of0(obj, struct lov_object, lo_cl);
+	return container_of(obj, struct lov_object, lo_cl);
 }
 
 static inline struct lu_object *lovsub2lu(struct lovsub_object *los)
@@ -556,13 +554,13 @@ static inline struct cl_object *lovsub2cl(struct lovsub_object *los)
 static inline struct lovsub_object *cl2lovsub(const struct cl_object *obj)
 {
 	LINVRNT(lovsub_is_object(&obj->co_lu));
-	return container_of0(obj, struct lovsub_object, lso_cl);
+	return container_of(obj, struct lovsub_object, lso_cl);
 }
 
 static inline struct lovsub_object *lu2lovsub(const struct lu_object *obj)
 {
 	LINVRNT(lovsub_is_object(obj));
-	return container_of0(obj, struct lovsub_object, lso_cl.co_lu);
+	return container_of(obj, struct lovsub_object, lso_cl.co_lu);
 }
 
 static inline struct lovsub_lock *
@@ -590,14 +588,14 @@ static inline struct lov_lock *cl2lov_lock(const struct cl_lock_slice *slice)
 static inline struct lov_page *cl2lov_page(const struct cl_page_slice *slice)
 {
 	LINVRNT(lov_is_object(&slice->cpl_obj->co_lu));
-	return container_of0(slice, struct lov_page, lps_cl);
+	return container_of(slice, struct lov_page, lps_cl);
 }
 
 static inline struct lovsub_page *
 cl2lovsub_page(const struct cl_page_slice *slice)
 {
 	LINVRNT(lovsub_is_object(&slice->cpl_obj->co_lu));
-	return container_of0(slice, struct lovsub_page, lsb_cl);
+	return container_of(slice, struct lovsub_page, lsb_cl);
 }
 
 static inline struct lov_io *cl2lov_io(const struct lu_env *env,
diff --git a/drivers/staging/lustre/lustre/lov/lov_ea.c b/drivers/staging/lustre/lustre/lov/lov_ea.c
index c56a971745e8..c80320ab0858 100644
--- a/drivers/staging/lustre/lustre/lov/lov_ea.c
+++ b/drivers/staging/lustre/lustre/lov/lov_ea.c
@@ -38,7 +38,6 @@
 #define DEBUG_SUBSYSTEM S_LOV
 
 #include <asm/div64.h>
-#include <linux/libcfs/libcfs.h>
 
 #include <obd_class.h>
 #include <uapi/linux/lustre/lustre_idl.h>
diff --git a/drivers/staging/lustre/lustre/lov/lov_internal.h b/drivers/staging/lustre/lustre/lov/lov_internal.h
index a56d71c2dda2..47042f27ca90 100644
--- a/drivers/staging/lustre/lustre/lov/lov_internal.h
+++ b/drivers/staging/lustre/lustre/lov/lov_internal.h
@@ -149,11 +149,16 @@ struct pool_desc {
 	char			 pool_name[LOV_MAXPOOLNAME + 1];
 	struct ost_pool		 pool_obds;
 	atomic_t		 pool_refcount;
-	struct hlist_node	 pool_hash;		/* access by poolname */
-	struct list_head	 pool_list;		/* serial access */
+	struct rhash_head	 pool_hash;		/* access by poolname */
+	union {
+		struct list_head	pool_list;	/* serial access */
+		struct rcu_head		rcu;		/* delayed free */
+	};
 	struct dentry		*pool_debugfs_entry;	/* file in debugfs */
 	struct obd_device	*pool_lobd;		/* owner */
 };
+int lov_pool_hash_init(struct rhashtable *tbl);
+void lov_pool_hash_destroy(struct rhashtable *tbl);
 
 struct lov_request {
 	struct obd_info	  rq_oi;
@@ -241,8 +246,6 @@ void lprocfs_lov_init_vars(struct lprocfs_static_vars *lvars);
 /* lov_cl.c */
 extern struct lu_device_type lov_device_type;
 
-/* pools */
-extern struct cfs_hash_ops pool_hash_operations;
 /* ost_pool methods */
 int lov_ost_pool_init(struct ost_pool *op, unsigned int count);
 int lov_ost_pool_extend(struct ost_pool *op, unsigned int min_count);
@@ -277,7 +280,7 @@ static inline bool lov_oinfo_is_dummy(const struct lov_oinfo *loi)
 
 static inline struct obd_device *lov2obd(const struct lov_obd *lov)
 {
-	return container_of0(lov, struct obd_device, u.lov);
+	return container_of_safe(lov, struct obd_device, u.lov);
 }
 
 #endif
diff --git a/drivers/staging/lustre/lustre/lov/lov_merge.c b/drivers/staging/lustre/lustre/lov/lov_merge.c
index 3796bbb25305..006717cf7a41 100644
--- a/drivers/staging/lustre/lustre/lov/lov_merge.c
+++ b/drivers/staging/lustre/lustre/lov/lov_merge.c
@@ -33,8 +33,6 @@
 
 #define DEBUG_SUBSYSTEM S_LOV
 
-#include <linux/libcfs/libcfs.h>
-
 #include <obd_class.h>
 #include "lov_internal.h"
 
@@ -56,7 +54,7 @@ int lov_merge_lvb_kms(struct lov_stripe_md *lsm,
 	int rc = 0;
 
 	assert_spin_locked(&lsm->lsm_lock);
-	LASSERT(lsm->lsm_lock_owner == current_pid());
+	LASSERT(lsm->lsm_lock_owner == current->pid);
 
 	CDEBUG(D_INODE, "MDT ID " DOSTID " initial value: s=%llu m=%llu a=%llu c=%llu b=%llu\n",
 	       POSTID(&lsm->lsm_oi), lvb->lvb_size, lvb->lvb_mtime,
diff --git a/drivers/staging/lustre/lustre/lov/lov_obd.c b/drivers/staging/lustre/lustre/lov/lov_obd.c
index ec70c12e5b40..7123972d4d5b 100644
--- a/drivers/staging/lustre/lustre/lov/lov_obd.c
+++ b/drivers/staging/lustre/lustre/lov/lov_obd.c
@@ -39,7 +39,6 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LOV
-#include <linux/libcfs/libcfs.h>
 
 #include <uapi/linux/lustre/lustre_idl.h>
 #include <uapi/linux/lustre/lustre_ioctl.h>
@@ -795,15 +794,11 @@ int lov_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
 
 	init_rwsem(&lov->lov_notify_lock);
 
-	lov->lov_pools_hash_body = cfs_hash_create("POOLS", HASH_POOLS_CUR_BITS,
-						   HASH_POOLS_MAX_BITS,
-						   HASH_POOLS_BKT_BITS, 0,
-						   CFS_HASH_MIN_THETA,
-						   CFS_HASH_MAX_THETA,
-						   &pool_hash_operations,
-						   CFS_HASH_DEFAULT);
 	INIT_LIST_HEAD(&lov->lov_pool_list);
 	lov->lov_pool_count = 0;
+	rc = lov_pool_hash_init(&lov->lov_pools_hash_body);
+	if (rc)
+		goto out;
 	rc = lov_ost_pool_init(&lov->lov_packed, 0);
 	if (rc)
 		goto out;
@@ -839,7 +834,7 @@ static int lov_cleanup(struct obd_device *obd)
 		/* coverity[overrun-buffer-val] */
 		lov_pool_del(obd, pool->pool_name);
 	}
-	cfs_hash_putref(lov->lov_pools_hash_body);
+	lov_pool_hash_destroy(&lov->lov_pools_hash_body);
 	lov_ost_pool_free(&lov->lov_packed);
 
 	lprocfs_obd_cleanup(obd);
@@ -1063,7 +1058,7 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 
 		/* got statfs data */
 		rc = obd_statfs(NULL, lov->lov_tgts[index]->ltd_exp, &stat_buf,
-				cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+				get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 				flags);
 		if (rc)
 			return rc;
@@ -1301,16 +1296,16 @@ static int lov_set_info_async(const struct lu_env *env, struct obd_export *exp,
 void lov_stripe_lock(struct lov_stripe_md *md)
 		__acquires(&md->lsm_lock)
 {
-	LASSERT(md->lsm_lock_owner != current_pid());
+	LASSERT(md->lsm_lock_owner != current->pid);
 	spin_lock(&md->lsm_lock);
 	LASSERT(md->lsm_lock_owner == 0);
-	md->lsm_lock_owner = current_pid();
+	md->lsm_lock_owner = current->pid;
 }
 
 void lov_stripe_unlock(struct lov_stripe_md *md)
 		__releases(&md->lsm_lock)
 {
-	LASSERT(md->lsm_lock_owner == current_pid());
+	LASSERT(md->lsm_lock_owner == current->pid);
 	md->lsm_lock_owner = 0;
 	spin_unlock(&md->lsm_lock);
 }
@@ -1407,6 +1402,10 @@ static int __init lov_init(void)
 	 */
 	CDEBUG(D_INFO, "Lustre LOV module (%p).\n", &lov_caches);
 
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	rc = lu_kmem_init(lov_caches);
 	if (rc)
 		return rc;
diff --git a/drivers/staging/lustre/lustre/lov/lov_object.c b/drivers/staging/lustre/lustre/lov/lov_object.c
index f7c69680cb7d..adc90f310fd7 100644
--- a/drivers/staging/lustre/lustre/lov/lov_object.c
+++ b/drivers/staging/lustre/lustre/lov/lov_object.c
@@ -370,7 +370,7 @@ static void lov_subobject_kill(const struct lu_env *env, struct lov_object *lov,
 	struct cl_object	*sub;
 	struct lov_layout_raid0 *r0;
 	struct lu_site	  *site;
-	struct lu_site_bkt_data *bkt;
+	wait_queue_head_t *wq;
 	wait_queue_entry_t	  *waiter;
 
 	r0  = &lov->u.raid0;
@@ -378,7 +378,7 @@ static void lov_subobject_kill(const struct lu_env *env, struct lov_object *lov,
 
 	sub  = lovsub2cl(los);
 	site = sub->co_lu.lo_dev->ld_site;
-	bkt  = lu_site_bkt_from_fid(site, &sub->co_lu.lo_header->loh_fid);
+	wq   = lu_site_wq_from_fid(site, &sub->co_lu.lo_header->loh_fid);
 
 	cl_object_kill(env, sub);
 	/* release a reference to the sub-object and ... */
@@ -391,7 +391,7 @@ static void lov_subobject_kill(const struct lu_env *env, struct lov_object *lov,
 	if (r0->lo_sub[idx] == los) {
 		waiter = &lov_env_info(env)->lti_waiter;
 		init_waitqueue_entry(waiter, current);
-		add_wait_queue(&bkt->lsb_marche_funebre, waiter);
+		add_wait_queue(wq, waiter);
 		set_current_state(TASK_UNINTERRUPTIBLE);
 		while (1) {
 			/* this wait-queue is signaled at the end of
@@ -408,7 +408,7 @@ static void lov_subobject_kill(const struct lu_env *env, struct lov_object *lov,
 				break;
 			}
 		}
-		remove_wait_queue(&bkt->lsb_marche_funebre, waiter);
+		remove_wait_queue(wq, waiter);
 	}
 	LASSERT(!r0->lo_sub[idx]);
 }
diff --git a/drivers/staging/lustre/lustre/lov/lov_offset.c b/drivers/staging/lustre/lustre/lov/lov_offset.c
index 3e16e647b334..a5f00f6ec347 100644
--- a/drivers/staging/lustre/lustre/lov/lov_offset.c
+++ b/drivers/staging/lustre/lustre/lov/lov_offset.c
@@ -33,8 +33,6 @@
 
 #define DEBUG_SUBSYSTEM S_LOV
 
-#include <linux/libcfs/libcfs.h>
-
 #include <obd_class.h>
 
 #include "lov_internal.h"
diff --git a/drivers/staging/lustre/lustre/lov/lov_pool.c b/drivers/staging/lustre/lustre/lov/lov_pool.c
index ecd9329cd073..6d2dec9266b6 100644
--- a/drivers/staging/lustre/lustre/lov/lov_pool.c
+++ b/drivers/staging/lustre/lustre/lov/lov_pool.c
@@ -41,14 +41,34 @@
 
 #define DEBUG_SUBSYSTEM S_LOV
 
-#include <linux/libcfs/libcfs.h>
-
 #include <obd.h>
 #include "lov_internal.h"
 
 #define pool_tgt(_p, _i) \
 		_p->pool_lobd->u.lov.lov_tgts[_p->pool_obds.op_array[_i]]
 
+static u32 pool_hashfh(const void *data, u32 len, u32 seed)
+{
+	const char *pool_name = data;
+	return hashlen_hash(hashlen_string((void*)(unsigned long)seed, pool_name));
+}
+
+static int pool_cmpfn(struct rhashtable_compare_arg *arg, const void *obj)
+{
+	const struct pool_desc *pool = obj;
+	const char *pool_name = arg->key;
+	return strcmp(pool_name, pool->pool_name);
+}
+
+static const struct rhashtable_params pools_hash_params = {
+	.key_len	= 1, /* actually variable */
+	.key_offset	= offsetof(struct pool_desc, pool_name),
+	.head_offset	= offsetof(struct pool_desc, pool_hash),
+	.hashfn		= pool_hashfh,
+	.obj_cmpfn	= pool_cmpfn,
+	.automatic_shrinking = true,
+};
+
 static void lov_pool_getref(struct pool_desc *pool)
 {
 	CDEBUG(D_INFO, "pool %p\n", pool);
@@ -59,96 +79,13 @@ void lov_pool_putref(struct pool_desc *pool)
 {
 	CDEBUG(D_INFO, "pool %p\n", pool);
 	if (atomic_dec_and_test(&pool->pool_refcount)) {
-		LASSERT(hlist_unhashed(&pool->pool_hash));
 		LASSERT(list_empty(&pool->pool_list));
 		LASSERT(!pool->pool_debugfs_entry);
 		lov_ost_pool_free(&pool->pool_obds);
-		kfree(pool);
+		kfree_rcu(pool, rcu);
 	}
 }
 
-static void lov_pool_putref_locked(struct pool_desc *pool)
-{
-	CDEBUG(D_INFO, "pool %p\n", pool);
-	LASSERT(atomic_read(&pool->pool_refcount) > 1);
-
-	atomic_dec(&pool->pool_refcount);
-}
-
-/*
- * hash function using a Rotating Hash algorithm
- * Knuth, D. The Art of Computer Programming,
- * Volume 3: Sorting and Searching,
- * Chapter 6.4.
- * Addison Wesley, 1973
- */
-static __u32 pool_hashfn(struct cfs_hash *hash_body, const void *key,
-			 unsigned int mask)
-{
-	int i;
-	__u32 result;
-	char *poolname;
-
-	result = 0;
-	poolname = (char *)key;
-	for (i = 0; i < LOV_MAXPOOLNAME; i++) {
-		if (poolname[i] == '\0')
-			break;
-		result = (result << 4) ^ (result >> 28) ^  poolname[i];
-	}
-	return (result % mask);
-}
-
-static void *pool_key(struct hlist_node *hnode)
-{
-	struct pool_desc *pool;
-
-	pool = hlist_entry(hnode, struct pool_desc, pool_hash);
-	return pool->pool_name;
-}
-
-static int pool_hashkey_keycmp(const void *key, struct hlist_node *compared_hnode)
-{
-	char *pool_name;
-	struct pool_desc *pool;
-
-	pool_name = (char *)key;
-	pool = hlist_entry(compared_hnode, struct pool_desc, pool_hash);
-	return !strncmp(pool_name, pool->pool_name, LOV_MAXPOOLNAME);
-}
-
-static void *pool_hashobject(struct hlist_node *hnode)
-{
-	return hlist_entry(hnode, struct pool_desc, pool_hash);
-}
-
-static void pool_hashrefcount_get(struct cfs_hash *hs, struct hlist_node *hnode)
-{
-	struct pool_desc *pool;
-
-	pool = hlist_entry(hnode, struct pool_desc, pool_hash);
-	lov_pool_getref(pool);
-}
-
-static void pool_hashrefcount_put_locked(struct cfs_hash *hs,
-					 struct hlist_node *hnode)
-{
-	struct pool_desc *pool;
-
-	pool = hlist_entry(hnode, struct pool_desc, pool_hash);
-	lov_pool_putref_locked(pool);
-}
-
-struct cfs_hash_ops pool_hash_operations = {
-	.hs_hash	= pool_hashfn,
-	.hs_key		= pool_key,
-	.hs_keycmp      = pool_hashkey_keycmp,
-	.hs_object      = pool_hashobject,
-	.hs_get		= pool_hashrefcount_get,
-	.hs_put_locked  = pool_hashrefcount_put_locked,
-
-};
-
 /*
  * pool debugfs seq_file methods
  */
@@ -396,6 +333,23 @@ int lov_ost_pool_free(struct ost_pool *op)
 	return 0;
 }
 
+static void
+pools_hash_exit(void *vpool, void *data)
+{
+	struct pool_desc *pool = vpool;
+	lov_pool_putref(pool);
+}
+
+int lov_pool_hash_init(struct rhashtable *tbl)
+{
+	return rhashtable_init(tbl, &pools_hash_params);
+}
+
+void lov_pool_hash_destroy(struct rhashtable *tbl)
+{
+	rhashtable_free_and_destroy(tbl, pools_hash_exit, NULL);
+}
+
 int lov_pool_new(struct obd_device *obd, char *poolname)
 {
 	struct lov_obd *lov;
@@ -421,8 +375,6 @@ int lov_pool_new(struct obd_device *obd, char *poolname)
 	if (rc)
 		goto out_err;
 
-	INIT_HLIST_NODE(&new_pool->pool_hash);
-
 	/* get ref for debugfs file */
 	lov_pool_getref(new_pool);
 	new_pool->pool_debugfs_entry = ldebugfs_add_simple(
@@ -443,11 +395,16 @@ int lov_pool_new(struct obd_device *obd, char *poolname)
 	lov->lov_pool_count++;
 	spin_unlock(&obd->obd_dev_lock);
 
-	/* add to find only when it fully ready  */
-	rc = cfs_hash_add_unique(lov->lov_pools_hash_body, poolname,
-				 &new_pool->pool_hash);
+	/* Add to hash table only when it is fully ready.  */
+	rc = rhashtable_lookup_insert_fast(&lov->lov_pools_hash_body,
+					   &new_pool->pool_hash, pools_hash_params);
 	if (rc) {
-		rc = -EEXIST;
+		if (rc != -EEXIST)
+			/*
+			 * Hide -E2BIG and -EBUSY which
+			 * are not helpful.
+			 */
+			rc = -ENOMEM;
 		goto out_err;
 	}
 
@@ -476,7 +433,13 @@ int lov_pool_del(struct obd_device *obd, char *poolname)
 	lov = &obd->u.lov;
 
 	/* lookup and kill hash reference */
-	pool = cfs_hash_del_key(lov->lov_pools_hash_body, poolname);
+	rcu_read_lock();
+	pool = rhashtable_lookup(&lov->lov_pools_hash_body, poolname, pools_hash_params);
+	if (pool)
+		if (rhashtable_remove_fast(&lov->lov_pools_hash_body,
+					   &pool->pool_hash, pools_hash_params) != 0)
+			pool = NULL;
+	rcu_read_unlock();
 	if (!pool)
 		return -ENOENT;
 
@@ -507,7 +470,11 @@ int lov_pool_add(struct obd_device *obd, char *poolname, char *ostname)
 
 	lov = &obd->u.lov;
 
-	pool = cfs_hash_lookup(lov->lov_pools_hash_body, poolname);
+	rcu_read_lock();
+	pool = rhashtable_lookup(&lov->lov_pools_hash_body, poolname, pools_hash_params);
+	if (pool && !atomic_inc_not_zero(&pool->pool_refcount))
+		pool = NULL;
+	rcu_read_unlock();
 	if (!pool)
 		return -ENOENT;
 
@@ -551,7 +518,11 @@ int lov_pool_remove(struct obd_device *obd, char *poolname, char *ostname)
 
 	lov = &obd->u.lov;
 
-	pool = cfs_hash_lookup(lov->lov_pools_hash_body, poolname);
+	rcu_read_lock();
+	pool = rhashtable_lookup(&lov->lov_pools_hash_body, poolname, pools_hash_params);
+	if (pool && !atomic_inc_not_zero(&pool->pool_refcount))
+		pool = NULL;
+	rcu_read_unlock();
 	if (!pool)
 		return -ENOENT;
 
diff --git a/drivers/staging/lustre/lustre/lov/lov_request.c b/drivers/staging/lustre/lustre/lov/lov_request.c
index 051450d67524..cb8567f20ea7 100644
--- a/drivers/staging/lustre/lustre/lov/lov_request.c
+++ b/drivers/staging/lustre/lustre/lov/lov_request.c
@@ -33,8 +33,6 @@
 
 #define DEBUG_SUBSYSTEM S_LOV
 
-#include <linux/libcfs/libcfs.h>
-
 #include <obd_class.h>
 #include <uapi/linux/lustre/lustre_idl.h>
 #include "lov_internal.h"
@@ -160,7 +158,7 @@ static int lov_fini_statfs(struct obd_device *obd, struct obd_statfs *osfs,
 
 		spin_lock(&obd->obd_osfs_lock);
 		memcpy(&obd->obd_osfs, osfs, sizeof(*osfs));
-		obd->obd_osfs_age = cfs_time_current_64();
+		obd->obd_osfs_age = get_jiffies_64();
 		spin_unlock(&obd->obd_osfs_lock);
 		return 0;
 	}
@@ -277,7 +275,7 @@ static int cb_statfs_update(void *cookie, int rc)
 	spin_lock(&tgtobd->obd_osfs_lock);
 	memcpy(&tgtobd->obd_osfs, lov_sfs, sizeof(*lov_sfs));
 	if ((oinfo->oi_flags & OBD_STATFS_FROM_CACHE) == 0)
-		tgtobd->obd_osfs_age = cfs_time_current_64();
+		tgtobd->obd_osfs_age = get_jiffies_64();
 	spin_unlock(&tgtobd->obd_osfs_lock);
 
 out_update:
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_internal.h b/drivers/staging/lustre/lustre/mdc/mdc_internal.h
index e0300c34ca3a..28924e927b50 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_internal.h
+++ b/drivers/staging/lustre/lustre/mdc/mdc_internal.h
@@ -50,7 +50,7 @@ void mdc_setattr_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 		      void *ea, size_t ealen);
 void mdc_create_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 		     const void *data, size_t datalen, umode_t mode, uid_t uid,
-		     gid_t gid, cfs_cap_t capability, __u64 rdev);
+		     gid_t gid, kernel_cap_t capability, __u64 rdev);
 void mdc_open_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 		   umode_t mode, __u64 rdev, __u64 flags, const void *data,
 		   size_t datalen);
@@ -77,8 +77,8 @@ int mdc_intent_lock(struct obd_export *exp,
 
 int mdc_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
 		const union ldlm_policy_data *policy,
-		struct lookup_intent *it, struct md_op_data *op_data,
-		struct lustre_handle *lockh, __u64 extra_lock_flags);
+		struct md_op_data *op_data,
+		struct lustre_handle *lockh, u64 extra_lock_flags);
 
 int mdc_resource_get_unused(struct obd_export *exp, const struct lu_fid *fid,
 			    struct list_head *cancels, enum ldlm_mode  mode,
@@ -97,7 +97,7 @@ void mdc_replay_open(struct ptlrpc_request *req);
 
 int mdc_create(struct obd_export *exp, struct md_op_data *op_data,
 	       const void *data, size_t datalen, umode_t mode, uid_t uid,
-	       gid_t gid, cfs_cap_t capability, __u64 rdev,
+	       gid_t gid, kernel_cap_t capability, __u64 rdev,
 	       struct ptlrpc_request **request);
 int mdc_link(struct obd_export *exp, struct md_op_data *op_data,
 	     struct ptlrpc_request **request);
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_lib.c b/drivers/staging/lustre/lustre/mdc/mdc_lib.c
index 46eefdc09e3a..d582968987ff 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_lib.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_lib.c
@@ -49,7 +49,7 @@ static void __mdc_pack_body(struct mdt_body *b, __u32 suppgid)
 	b->mbo_gid = from_kgid(&init_user_ns, current_gid());
 	b->mbo_fsuid = from_kuid(&init_user_ns, current_fsuid());
 	b->mbo_fsgid = from_kgid(&init_user_ns, current_fsgid());
-	b->mbo_capability = cfs_curproc_cap_pack();
+	b->mbo_capability = current_cap().cap[0];
 }
 
 void mdc_swap_layouts_pack(struct ptlrpc_request *req,
@@ -126,7 +126,8 @@ void mdc_readdir_pack(struct ptlrpc_request *req, __u64 pgoff, size_t size,
 /* packing of MDS records */
 void mdc_create_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 		     const void *data, size_t datalen, umode_t mode,
-		     uid_t uid, gid_t gid, cfs_cap_t cap_effective, __u64 rdev)
+		     uid_t uid, gid_t gid, kernel_cap_t cap_effective,
+		     __u64 rdev)
 {
 	struct mdt_rec_create	*rec;
 	char			*tmp;
@@ -138,7 +139,7 @@ void mdc_create_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 	rec->cr_opcode   = REINT_CREATE;
 	rec->cr_fsuid    = uid;
 	rec->cr_fsgid    = gid;
-	rec->cr_cap      = cap_effective;
+	rec->cr_cap      = cap_effective.cap[0];
 	rec->cr_fid1     = op_data->op_fid1;
 	rec->cr_fid2     = op_data->op_fid2;
 	rec->cr_mode     = mode;
@@ -203,7 +204,7 @@ void mdc_open_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 	rec->cr_opcode   = REINT_OPEN;
 	rec->cr_fsuid    = from_kuid(&init_user_ns, current_fsuid());
 	rec->cr_fsgid    = from_kgid(&init_user_ns, current_fsgid());
-	rec->cr_cap      = cfs_curproc_cap_pack();
+	rec->cr_cap      = current_cap().cap[0];
 	rec->cr_fid1 = op_data->op_fid1;
 	rec->cr_fid2 = op_data->op_fid2;
 
@@ -281,7 +282,7 @@ static void mdc_setattr_pack_rec(struct mdt_rec_setattr *rec,
 	rec->sa_opcode  = REINT_SETATTR;
 	rec->sa_fsuid   = from_kuid(&init_user_ns, current_fsuid());
 	rec->sa_fsgid   = from_kgid(&init_user_ns, current_fsgid());
-	rec->sa_cap     = cfs_curproc_cap_pack();
+	rec->sa_cap     = current_cap().cap[0];
 	rec->sa_suppgid = -1;
 
 	rec->sa_fid    = op_data->op_fid1;
@@ -350,7 +351,7 @@ void mdc_unlink_pack(struct ptlrpc_request *req, struct md_op_data *op_data)
 					REINT_RMENTRY : REINT_UNLINK;
 	rec->ul_fsuid    = op_data->op_fsuid;
 	rec->ul_fsgid    = op_data->op_fsgid;
-	rec->ul_cap      = op_data->op_cap;
+	rec->ul_cap      = op_data->op_cap.cap[0];
 	rec->ul_mode     = op_data->op_mode;
 	rec->ul_suppgid1 = op_data->op_suppgids[0];
 	rec->ul_suppgid2 = -1;
@@ -372,7 +373,7 @@ void mdc_link_pack(struct ptlrpc_request *req, struct md_op_data *op_data)
 	rec->lk_opcode   = REINT_LINK;
 	rec->lk_fsuid    = op_data->op_fsuid; /* current->fsuid; */
 	rec->lk_fsgid    = op_data->op_fsgid; /* current->fsgid; */
-	rec->lk_cap      = op_data->op_cap;   /* current->cap_effective; */
+	rec->lk_cap      = op_data->op_cap.cap[0]; /* current->cap_effective; */
 	rec->lk_suppgid1 = op_data->op_suppgids[0];
 	rec->lk_suppgid2 = op_data->op_suppgids[1];
 	rec->lk_fid1     = op_data->op_fid1;
@@ -423,7 +424,7 @@ void mdc_rename_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 	rec->rn_opcode   = REINT_RENAME;
 	rec->rn_fsuid    = op_data->op_fsuid;
 	rec->rn_fsgid    = op_data->op_fsgid;
-	rec->rn_cap      = op_data->op_cap;
+	rec->rn_cap      = op_data->op_cap.cap[0];
 	rec->rn_suppgid1 = op_data->op_suppgids[0];
 	rec->rn_suppgid2 = op_data->op_suppgids[1];
 	rec->rn_fid1     = op_data->op_fid1;
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_locks.c b/drivers/staging/lustre/lustre/mdc/mdc_locks.c
index 695ef44532cf..253a54550ba8 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_locks.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_locks.c
@@ -568,7 +568,7 @@ static int mdc_finish_enqueue(struct obd_export *exp,
 		  it->it_op, it->it_disposition, it->it_status);
 
 	/* We know what to expect, so we do any byte flipping required here */
-	if (it->it_op & (IT_OPEN | IT_UNLINK | IT_LOOKUP | IT_GETATTR)) {
+	if (it_has_reply_body(it)) {
 		struct mdt_body *body;
 
 		body = req_capsule_server_get(pill, &RMF_MDT_BODY);
@@ -688,10 +688,10 @@ static int mdc_finish_enqueue(struct obd_export *exp,
 /* We always reserve enough space in the reply packet for a stripe MD, because
  * we don't know in advance the file type.
  */
-int mdc_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
-		const union ldlm_policy_data *policy,
-		struct lookup_intent *it, struct md_op_data *op_data,
-		struct lustre_handle *lockh, u64 extra_lock_flags)
+int mdc_enqueue_base(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
+		     const union ldlm_policy_data *policy,
+		     struct lookup_intent *it, struct md_op_data *op_data,
+		     struct lustre_handle *lockh, u64 extra_lock_flags)
 {
 	static const union ldlm_policy_data lookup_policy = {
 		.l_inodebits = { MDS_INODELOCK_LOOKUP }
@@ -859,6 +859,15 @@ int mdc_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
 	return rc;
 }
 
+int mdc_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
+		const union ldlm_policy_data *policy,
+		struct md_op_data *op_data,
+		struct lustre_handle *lockh, u64 extra_lock_flags)
+{
+	return mdc_enqueue_base(exp, einfo, policy, NULL,
+				op_data, lockh, extra_lock_flags);
+}
+
 static int mdc_finish_intent_lock(struct obd_export *exp,
 				  struct ptlrpc_request *request,
 				  struct md_op_data *op_data,
@@ -866,9 +875,8 @@ static int mdc_finish_intent_lock(struct obd_export *exp,
 				  struct lustre_handle *lockh)
 {
 	struct lustre_handle old_lock;
-	struct mdt_body *mdt_body;
 	struct ldlm_lock *lock;
-	int rc;
+	int rc = 0;
 
 	LASSERT(request != LP_POISON);
 	LASSERT(request->rq_repmsg != LP_POISON);
@@ -876,23 +884,30 @@ static int mdc_finish_intent_lock(struct obd_export *exp,
 	if (it->it_op & IT_READDIR)
 		return 0;
 
+	if (it->it_op & (IT_GETXATTR | IT_LAYOUT)) {
+		if (it->it_status != 0) {
+			rc = it->it_status;
+			goto out;
+		}
+		goto matching_lock;
+	}
+
 	if (!it_disposition(it, DISP_IT_EXECD)) {
 		/* The server failed before it even started executing the
 		 * intent, i.e. because it couldn't unpack the request.
 		 */
 		LASSERT(it->it_status != 0);
-		return it->it_status;
+		rc = it->it_status;
+		goto out;
 	}
+
 	rc = it_open_error(DISP_IT_EXECD, it);
 	if (rc)
-		return rc;
-
-	mdt_body = req_capsule_server_get(&request->rq_pill, &RMF_MDT_BODY);
-	LASSERT(mdt_body);      /* mdc_enqueue checked */
+		goto out;
 
 	rc = it_open_error(DISP_LOOKUP_EXECD, it);
 	if (rc)
-		return rc;
+		goto out;
 
 	/* keep requests around for the multiple phases of the call
 	 * this shows the DISP_XX must guarantee we make it into the call
@@ -918,8 +933,9 @@ static int mdc_finish_intent_lock(struct obd_export *exp,
 	else if (it->it_op == IT_OPEN)
 		LASSERT(!it_disposition(it, DISP_OPEN_CREATE));
 	else
-		LASSERT(it->it_op & (IT_GETATTR | IT_LOOKUP | IT_LAYOUT));
+		LASSERT(it->it_op & (IT_GETATTR | IT_LOOKUP));
 
+matching_lock:
 	/* If we already have a matching lock, then cancel the new
 	 * one.  We have to set the data here instead of in
 	 * mdc_enqueue, because we need to use the child's inode as
@@ -932,10 +948,20 @@ static int mdc_finish_intent_lock(struct obd_export *exp,
 
 		LDLM_DEBUG(lock, "matching against this");
 
-		LASSERTF(fid_res_name_eq(&mdt_body->mbo_fid1,
-					 &lock->l_resource->lr_name),
-			 "Lock res_id: " DLDLMRES ", fid: " DFID "\n",
-			 PLDLMRES(lock->l_resource), PFID(&mdt_body->mbo_fid1));
+		if (it_has_reply_body(it)) {
+			struct mdt_body *body;
+
+			body = req_capsule_server_get(&request->rq_pill,
+						      &RMF_MDT_BODY);
+
+			/* mdc_enqueue checked */
+			LASSERT(body);
+			LASSERTF(fid_res_name_eq(&body->mbo_fid1,
+						 &lock->l_resource->lr_name),
+				 "Lock res_id: " DLDLMRES ", fid: " DFID "\n",
+				 PLDLMRES(lock->l_resource),
+				 PFID(&body->mbo_fid1));
+		}
 		LDLM_LOCK_PUT(lock);
 
 		memcpy(&old_lock, lockh, sizeof(*lockh));
@@ -948,6 +974,7 @@ static int mdc_finish_intent_lock(struct obd_export *exp,
 			it->it_lock_handle = lockh->cookie;
 		}
 	}
+out:
 	CDEBUG(D_DENTRY,
 	       "D_IT dentry %.*s intent: %s status %d disp %x rc %d\n",
 	       (int)op_data->op_namelen, op_data->op_name,
@@ -1094,8 +1121,9 @@ int mdc_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
 			return rc;
 		}
 	}
-	rc = mdc_enqueue(exp, &einfo, NULL, it, op_data, &lockh,
-			 extra_lock_flags);
+
+	rc = mdc_enqueue_base(exp, &einfo, NULL, it, op_data, &lockh,
+			      extra_lock_flags);
 	if (rc < 0)
 		return rc;
 
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_reint.c b/drivers/staging/lustre/lustre/mdc/mdc_reint.c
index 488b98007558..94ab43bcbe4f 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_reint.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_reint.c
@@ -148,7 +148,7 @@ int mdc_setattr(struct obd_export *exp, struct md_op_data *op_data,
 
 int mdc_create(struct obd_export *exp, struct md_op_data *op_data,
 	       const void *data, size_t datalen, umode_t mode,
-	       uid_t uid, gid_t gid, cfs_cap_t cap_effective,
+	       uid_t uid, gid_t gid, kernel_cap_t cap_effective,
 	       __u64 rdev, struct ptlrpc_request **request)
 {
 	struct ptlrpc_request *req;
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_request.c b/drivers/staging/lustre/lustre/mdc/mdc_request.c
index 8ee7b4d273b2..50709ffe6073 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_request.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_request.c
@@ -38,6 +38,9 @@
 # include <linux/miscdevice.h>
 # include <linux/init.h>
 # include <linux/utsname.h>
+# include <linux/file.h>
+# include <linux/kthread.h>
+#include <linux/prefetch.h>
 
 #include <lustre_errno.h>
 #include <cl_object.h>
@@ -303,7 +306,7 @@ static int mdc_xattr_common(struct obd_export *exp,
 		rec->sx_opcode = REINT_SETXATTR;
 		rec->sx_fsuid  = from_kuid(&init_user_ns, current_fsuid());
 		rec->sx_fsgid  = from_kgid(&init_user_ns, current_fsgid());
-		rec->sx_cap    = cfs_curproc_cap_pack();
+		rec->sx_cap    = current_cap().cap[0];
 		rec->sx_suppgid1 = suppgid;
 		rec->sx_suppgid2 = -1;
 		rec->sx_fid    = *fid;
@@ -2104,7 +2107,7 @@ static int mdc_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 		}
 
 		rc = mdc_statfs(NULL, obd->obd_self_export, &stat_buf,
-				cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+				get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 				0);
 		if (rc != 0)
 			goto out;
@@ -2733,6 +2736,11 @@ static struct md_ops mdc_md_ops = {
 static int __init mdc_init(void)
 {
 	struct lprocfs_static_vars lvars = { NULL };
+	int rc;
+
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
 
 	lprocfs_mdc_init_vars(&lvars);
 
diff --git a/drivers/staging/lustre/lustre/mgc/mgc_internal.h b/drivers/staging/lustre/lustre/mgc/mgc_internal.h
index 2c571c180578..9541892b67c7 100644
--- a/drivers/staging/lustre/lustre/mgc/mgc_internal.h
+++ b/drivers/staging/lustre/lustre/mgc/mgc_internal.h
@@ -34,7 +34,6 @@
 #ifndef _MGC_INTERNAL_H
 #define _MGC_INTERNAL_H
 
-#include <linux/libcfs/libcfs.h>
 #include <lustre_lib.h>
 #include <lustre_dlm.h>
 #include <lustre_log.h>
diff --git a/drivers/staging/lustre/lustre/mgc/mgc_request.c b/drivers/staging/lustre/lustre/mgc/mgc_request.c
index c61cd23a96df..32df804614d3 100644
--- a/drivers/staging/lustre/lustre/mgc/mgc_request.c
+++ b/drivers/staging/lustre/lustre/mgc/mgc_request.c
@@ -39,6 +39,7 @@
 #define D_MGC D_CONFIG /*|D_WARNING*/
 
 #include <linux/module.h>
+#include <linux/random.h>
 
 #include <lprocfs_status.h>
 #include <lustre_dlm.h>
@@ -1826,6 +1827,12 @@ static struct obd_ops mgc_obd_ops = {
 
 static int __init mgc_init(void)
 {
+	int rc;
+
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	return class_register_type(&mgc_obd_ops, NULL,
 				   LUSTRE_MGC_NAME, NULL);
 }
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_io.c b/drivers/staging/lustre/lustre/obdclass/cl_io.c
index ab84e011b560..fcdae6029258 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_io.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_io.c
@@ -599,7 +599,6 @@ int cl_io_read_ahead(const struct lu_env *env, struct cl_io *io,
 	int		       result = 0;
 
 	LINVRNT(io->ci_type == CIT_READ || io->ci_type == CIT_FAULT);
-	LINVRNT(io->ci_state == CIS_IO_GOING || io->ci_state == CIS_LOCKED);
 	LINVRNT(cl_io_invariant(io));
 
 	cl_io_for_each(scan, io) {
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_object.c b/drivers/staging/lustre/lustre/obdclass/cl_object.c
index 7809f6ae1809..42cce2dc5a45 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_object.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_object.c
@@ -47,13 +47,11 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include <linux/libcfs/libcfs.h>
 /* class_put_type() */
 #include <obd_class.h>
 #include <obd_support.h>
 #include <lustre_fid.h>
 #include <linux/list.h>
-#include <linux/libcfs/libcfs_hash.h>	/* for cfs_hash stuff */
 #include <cl_object.h>
 #include <lu_object.h>
 #include "cl_internal.h"
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_page.c b/drivers/staging/lustre/lustre/obdclass/cl_page.c
index d3b25667bc3a..916cf81c5997 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_page.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_page.c
@@ -38,7 +38,6 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include <linux/libcfs/libcfs.h>
 #include <obd_class.h>
 #include <obd_support.h>
 #include <linux/list.h>
diff --git a/drivers/staging/lustre/lustre/obdclass/class_obd.c b/drivers/staging/lustre/lustre/obdclass/class_obd.c
index 3e24b76f6301..d6c46858941b 100644
--- a/drivers/staging/lustre/lustre/obdclass/class_obd.c
+++ b/drivers/staging/lustre/lustre/obdclass/class_obd.c
@@ -32,7 +32,10 @@
  */
 
 #define DEBUG_SUBSYSTEM S_CLASS
-# include <linux/atomic.h>
+
+#include <linux/atomic.h>
+#include <linux/miscdevice.h>
+#include <linux/libcfs/libcfs.h>
 
 #include <obd_support.h>
 #include <obd_class.h>
@@ -42,6 +45,7 @@
 #include <linux/list.h>
 #include <cl_object.h>
 #include <uapi/linux/lustre/lustre_ioctl.h>
+#include <uapi/linux/lnet/libcfs_ioctl.h>
 #include "llog_internal.h"
 
 struct obd_device *obd_devs[MAX_OBD_DEVICES];
@@ -98,7 +102,7 @@ int lustre_get_jobid(char *jobid)
 	/* Use process name + fsuid as jobid */
 	if (strcmp(obd_jobid_var, JOBSTATS_PROCNAME_UID) == 0) {
 		snprintf(jobid, LUSTRE_JOBID_SIZE, "%s.%u",
-			 current_comm(),
+			 current->comm,
 			 from_kuid(&init_user_ns, current_fsuid()));
 		return 0;
 	}
@@ -447,6 +451,11 @@ static int __init obdclass_init(void)
 	LCONSOLE_INFO("Lustre: Build Version: " LUSTRE_VERSION_STRING "\n");
 
 	spin_lock_init(&obd_types_lock);
+
+	err = libcfs_setup();
+	if (err)
+		return err;
+
 	obd_zombie_impexp_init();
 
 	err = obd_init_checks();
@@ -462,7 +471,7 @@ static int __init obdclass_init(void)
 
 	err = misc_register(&obd_psdev);
 	if (err) {
-		CERROR("cannot register %d err %d\n", OBD_DEV_MINOR, err);
+		CERROR("cannot register OBD miscdevices: err %d\n", err);
 		return err;
 	}
 
diff --git a/drivers/staging/lustre/lustre/obdclass/genops.c b/drivers/staging/lustre/lustre/obdclass/genops.c
index 63ccbabb4c5a..af233b868742 100644
--- a/drivers/staging/lustre/lustre/obdclass/genops.c
+++ b/drivers/staging/lustre/lustre/obdclass/genops.c
@@ -713,7 +713,6 @@ struct obd_export *class_new_export(struct obd_device *obd,
 				    struct obd_uuid *cluuid)
 {
 	struct obd_export *export;
-	struct cfs_hash *hash = NULL;
 	int rc = 0;
 
 	export = kzalloc(sizeof(*export), GFP_NOFS);
@@ -721,8 +720,6 @@ struct obd_export *class_new_export(struct obd_device *obd,
 		return ERR_PTR(-ENOMEM);
 
 	export->exp_conn_cnt = 0;
-	export->exp_lock_hash = NULL;
-	export->exp_flock_hash = NULL;
 	atomic_set(&export->exp_refcount, 2);
 	atomic_set(&export->exp_rpc_count, 0);
 	atomic_set(&export->exp_cb_count, 0);
@@ -742,7 +739,6 @@ struct obd_export *class_new_export(struct obd_device *obd,
 	class_handle_hash(&export->exp_handle, &export_handle_ops);
 	spin_lock_init(&export->exp_lock);
 	spin_lock_init(&export->exp_rpc_lock);
-	INIT_HLIST_NODE(&export->exp_uuid_hash);
 	spin_lock_init(&export->exp_bl_list_lock);
 	INIT_LIST_HEAD(&export->exp_bl_list);
 	INIT_WORK(&export->exp_zombie_work, obd_zombie_exp_cull);
@@ -759,44 +755,24 @@ struct obd_export *class_new_export(struct obd_device *obd,
 		goto exit_unlock;
 	}
 
-	hash = cfs_hash_getref(obd->obd_uuid_hash);
-	if (!hash) {
-		rc = -ENODEV;
-		goto exit_unlock;
-	}
-	spin_unlock(&obd->obd_dev_lock);
-
 	if (!obd_uuid_equals(cluuid, &obd->obd_uuid)) {
-		rc = cfs_hash_add_unique(hash, cluuid, &export->exp_uuid_hash);
-		if (rc != 0) {
+		rc = obd_uuid_add(obd, export);
+		if (rc) {
 			LCONSOLE_WARN("%s: denying duplicate export for %s, %d\n",
 				      obd->obd_name, cluuid->uuid, rc);
-			rc = -EALREADY;
-			goto exit_err;
+			goto exit_unlock;
 		}
 	}
 
-	spin_lock(&obd->obd_dev_lock);
-	if (obd->obd_stopping) {
-		cfs_hash_del(hash, cluuid, &export->exp_uuid_hash);
-		rc = -ENODEV;
-		goto exit_unlock;
-	}
-
 	class_incref(obd, "export", export);
 	list_add(&export->exp_obd_chain, &export->exp_obd->obd_exports);
 	export->exp_obd->obd_num_exports++;
 	spin_unlock(&obd->obd_dev_lock);
-	cfs_hash_putref(hash);
 	return export;
 
 exit_unlock:
 	spin_unlock(&obd->obd_dev_lock);
-exit_err:
-	if (hash)
-		cfs_hash_putref(hash);
 	class_handle_unhash(&export->exp_handle);
-	LASSERT(hlist_unhashed(&export->exp_uuid_hash));
 	obd_destroy_export(export);
 	kfree(export);
 	return ERR_PTR(rc);
@@ -809,10 +785,8 @@ void class_unlink_export(struct obd_export *exp)
 
 	spin_lock(&exp->exp_obd->obd_dev_lock);
 	/* delete an uuid-export hashitem from hashtables */
-	if (!hlist_unhashed(&exp->exp_uuid_hash))
-		cfs_hash_del(exp->exp_obd->obd_uuid_hash,
-			     &exp->exp_client_uuid,
-			     &exp->exp_uuid_hash);
+	if (exp != exp->exp_obd->obd_self_export)
+		obd_uuid_del(exp->exp_obd, exp);
 
 	list_move(&exp->exp_obd_chain, &exp->exp_obd->obd_unlinked_exports);
 	exp->exp_obd->obd_num_exports--;
diff --git a/drivers/staging/lustre/lustre/obdclass/kernelcomm.c b/drivers/staging/lustre/lustre/obdclass/kernelcomm.c
index b9bf81607bbf..63067a7f1e19 100644
--- a/drivers/staging/lustre/lustre/obdclass/kernelcomm.c
+++ b/drivers/staging/lustre/lustre/obdclass/kernelcomm.c
@@ -39,6 +39,8 @@
 #define DEBUG_SUBSYSTEM S_CLASS
 #define D_KUC D_OTHER
 
+#include <linux/file.h>
+#include <linux/libcfs/libcfs.h>
 #include <obd_support.h>
 #include <lustre_kernelcomm.h>
 
diff --git a/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c b/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
index 7bceee7f121e..d3196439985b 100644
--- a/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
+++ b/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
@@ -42,7 +42,6 @@
 #include <linux/module.h>
 #include <linux/errno.h>
 #include <linux/kernel.h>
-#include <linux/major.h>
 #include <linux/sched.h>
 #include <linux/lp.h>
 #include <linux/slab.h>
@@ -61,7 +60,6 @@
 #include <linux/seq_file.h>
 #include <linux/kobject.h>
 
-#include <linux/libcfs/libcfs.h>
 #include <uapi/linux/lnet/lnetctl.h>
 #include <obd_support.h>
 #include <obd_class.h>
@@ -271,7 +269,7 @@ static const struct file_operations obd_psdev_fops = {
 
 /* modules setup */
 struct miscdevice obd_psdev = {
-	.minor = OBD_DEV_MINOR,
+	.minor = MISC_DYNAMIC_MINOR,
 	.name  = OBD_DEV_NAME,
 	.fops  = &obd_psdev_fops,
 };
diff --git a/drivers/staging/lustre/lustre/obdclass/llog.c b/drivers/staging/lustre/lustre/obdclass/llog.c
index 693e1129f1f9..bba84eae1e19 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog.c
@@ -44,6 +44,7 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
+#include <linux/kthread.h>
 #include <llog_swab.h>
 #include <lustre_log.h>
 #include <obd_class.h>
@@ -385,7 +386,7 @@ static int llog_process_thread(void *arg)
 	if (cd)
 		cd->lpcd_last_idx = last_called_index;
 
-	kfree(buf);
+	kvfree(buf);
 	lpi->lpi_rc = rc;
 	return 0;
 }
diff --git a/drivers/staging/lustre/lustre/obdclass/lprocfs_counters.c b/drivers/staging/lustre/lustre/obdclass/lprocfs_counters.c
index c83b7d7f8e72..85f09aff6e83 100644
--- a/drivers/staging/lustre/lustre/obdclass/lprocfs_counters.c
+++ b/drivers/staging/lustre/lustre/obdclass/lprocfs_counters.c
@@ -41,6 +41,7 @@
 #include <linux/module.h>
 #include <lprocfs_status.h>
 #include <obd_support.h>
+#include <linux/libcfs/libcfs.h>
 
 void lprocfs_counter_add(struct lprocfs_stats *stats, int idx, long amount)
 {
diff --git a/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c b/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
index 2ed350527398..eb6396add78d 100644
--- a/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
+++ b/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
@@ -407,7 +407,7 @@ static ssize_t blocksize_show(struct kobject *kobj, struct attribute *attr,
 					      obd_kobj);
 	struct obd_statfs  osfs;
 	int rc = obd_statfs(NULL, obd->obd_self_export, &osfs,
-			    cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+			    get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 			    OBD_STATFS_NODELAY);
 	if (!rc)
 		return sprintf(buf, "%u\n", osfs.os_bsize);
@@ -423,7 +423,7 @@ static ssize_t kbytestotal_show(struct kobject *kobj, struct attribute *attr,
 					      obd_kobj);
 	struct obd_statfs  osfs;
 	int rc = obd_statfs(NULL, obd->obd_self_export, &osfs,
-			    cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+			    get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 			    OBD_STATFS_NODELAY);
 	if (!rc) {
 		__u32 blk_size = osfs.os_bsize >> 10;
@@ -446,7 +446,7 @@ static ssize_t kbytesfree_show(struct kobject *kobj, struct attribute *attr,
 					      obd_kobj);
 	struct obd_statfs  osfs;
 	int rc = obd_statfs(NULL, obd->obd_self_export, &osfs,
-			    cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+			    get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 			    OBD_STATFS_NODELAY);
 	if (!rc) {
 		__u32 blk_size = osfs.os_bsize >> 10;
@@ -469,7 +469,7 @@ static ssize_t kbytesavail_show(struct kobject *kobj, struct attribute *attr,
 					      obd_kobj);
 	struct obd_statfs  osfs;
 	int rc = obd_statfs(NULL, obd->obd_self_export, &osfs,
-			    cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+			    get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 			    OBD_STATFS_NODELAY);
 	if (!rc) {
 		__u32 blk_size = osfs.os_bsize >> 10;
@@ -492,7 +492,7 @@ static ssize_t filestotal_show(struct kobject *kobj, struct attribute *attr,
 					      obd_kobj);
 	struct obd_statfs  osfs;
 	int rc = obd_statfs(NULL, obd->obd_self_export, &osfs,
-			    cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+			    get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 			    OBD_STATFS_NODELAY);
 	if (!rc)
 		return sprintf(buf, "%llu\n", osfs.os_files);
@@ -508,7 +508,7 @@ static ssize_t filesfree_show(struct kobject *kobj, struct attribute *attr,
 					      obd_kobj);
 	struct obd_statfs  osfs;
 	int rc = obd_statfs(NULL, obd->obd_self_export, &osfs,
-			    cfs_time_shift_64(-OBD_STATFS_CACHE_SECONDS),
+			    get_jiffies_64() - OBD_STATFS_CACHE_SECONDS * HZ,
 			    OBD_STATFS_NODELAY);
 	if (!rc)
 		return sprintf(buf, "%llu\n", osfs.os_ffree);
diff --git a/drivers/staging/lustre/lustre/obdclass/lu_object.c b/drivers/staging/lustre/lustre/obdclass/lu_object.c
index 3ae16e8501c2..aa9d74e087f4 100644
--- a/drivers/staging/lustre/lustre/obdclass/lu_object.c
+++ b/drivers/staging/lustre/lustre/obdclass/lu_object.c
@@ -41,8 +41,6 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include <linux/libcfs/libcfs.h>
-
 #include <linux/module.h>
 
 /* hash_long() */
@@ -56,6 +54,27 @@
 #include <lu_ref.h>
 #include <linux/list.h>
 
+struct lu_site_bkt_data {
+	/**
+	 * LRU list, updated on each access to object. Protected by
+	 * bucket lock of lu_site::ls_obj_hash.
+	 *
+	 * "Cold" end of LRU is lu_site::ls_lru.next. Accessed object are
+	 * moved to the lu_site::ls_lru.prev (this is due to the non-existence
+	 * of list_for_each_entry_safe_reverse()).
+	 */
+	struct list_head		lsb_lru;
+	/**
+	 * Wait-queue signaled when an object in this site is ultimately
+	 * destroyed (lu_object_free()). It is used by lu_object_find() to
+	 * wait before re-trying when object in the process of destruction is
+	 * found in the hash table.
+	 *
+	 * \see htable_lookup().
+	 */
+	wait_queue_head_t	       lsb_marche_funebre;
+};
+
 enum {
 	LU_CACHE_PERCENT_MAX	 = 50,
 	LU_CACHE_PERCENT_DEFAULT = 20
@@ -88,6 +107,18 @@ MODULE_PARM_DESC(lu_cache_nr, "Maximum number of objects in lu_object cache");
 static void lu_object_free(const struct lu_env *env, struct lu_object *o);
 static __u32 ls_stats_read(struct lprocfs_stats *stats, int idx);
 
+wait_queue_head_t *
+lu_site_wq_from_fid(struct lu_site *site, struct lu_fid *fid)
+{
+	struct cfs_hash_bd bd;
+	struct lu_site_bkt_data *bkt;
+
+	cfs_hash_bd_get(site->ls_obj_hash, fid, &bd);
+	bkt = cfs_hash_bd_extra_get(site->ls_obj_hash, &bd);
+	return &bkt->lsb_marche_funebre;
+}
+EXPORT_SYMBOL(lu_site_wq_from_fid);
+
 /**
  * Decrease reference counter on object. If last reference is freed, return
  * object to the cache, unless lu_object_is_dying(o) holds. In the latter
@@ -151,10 +182,9 @@ void lu_object_put(const struct lu_env *env, struct lu_object *o)
 	if (!lu_object_is_dying(top)) {
 		LASSERT(list_empty(&top->loh_lru));
 		list_add_tail(&top->loh_lru, &bkt->lsb_lru);
-		bkt->lsb_lru_len++;
 		percpu_counter_inc(&site->ls_lru_len_counter);
-		CDEBUG(D_INODE, "Add %p to site lru. hash: %p, bkt: %p, lru_len: %ld\n",
-		       o, site->ls_obj_hash, bkt, bkt->lsb_lru_len);
+		CDEBUG(D_INODE, "Add %p to site lru. hash: %p, bkt: %p\n",
+		       o, site->ls_obj_hash, bkt);
 		cfs_hash_bd_unlock(site->ls_obj_hash, &bd, 1);
 		return;
 	}
@@ -202,7 +232,6 @@ void lu_object_unhash(const struct lu_env *env, struct lu_object *o)
 
 			list_del_init(&top->loh_lru);
 			bkt = cfs_hash_bd_extra_get(obj_hash, &bd);
-			bkt->lsb_lru_len--;
 			percpu_counter_dec(&site->ls_lru_len_counter);
 		}
 		cfs_hash_bd_del_locked(obj_hash, &bd, &top->loh_hash);
@@ -288,7 +317,7 @@ static struct lu_object *lu_object_alloc(const struct lu_env *env,
  */
 static void lu_object_free(const struct lu_env *env, struct lu_object *o)
 {
-	struct lu_site_bkt_data *bkt;
+	wait_queue_head_t *wq;
 	struct lu_site	  *site;
 	struct lu_object	*scan;
 	struct list_head	      *layers;
@@ -296,7 +325,7 @@ static void lu_object_free(const struct lu_env *env, struct lu_object *o)
 
 	site   = o->lo_dev->ld_site;
 	layers = &o->lo_header->loh_layers;
-	bkt    = lu_site_bkt_from_fid(site, &o->lo_header->loh_fid);
+	wq     = lu_site_wq_from_fid(site, &o->lo_header->loh_fid);
 	/*
 	 * First call ->loo_object_delete() method to release all resources.
 	 */
@@ -319,13 +348,13 @@ static void lu_object_free(const struct lu_env *env, struct lu_object *o)
 		 * lives as long as possible and ->loo_object_free() methods
 		 * can look at its contents.
 		 */
-		o = container_of0(splice.prev, struct lu_object, lo_linkage);
+		o = container_of(splice.prev, struct lu_object, lo_linkage);
 		list_del_init(&o->lo_linkage);
 		o->lo_ops->loo_object_free(env, o);
 	}
 
-	if (waitqueue_active(&bkt->lsb_marche_funebre))
-		wake_up_all(&bkt->lsb_marche_funebre);
+	if (waitqueue_active(wq))
+		wake_up_all(wq);
 }
 
 /**
@@ -386,7 +415,6 @@ int lu_site_purge_objects(const struct lu_env *env, struct lu_site *s,
 			cfs_hash_bd_del_locked(s->ls_obj_hash,
 					       &bd2, &h->loh_hash);
 			list_move(&h->loh_lru, &dispose);
-			bkt->lsb_lru_len--;
 			percpu_counter_dec(&s->ls_lru_len_counter);
 			if (did_sth == 0)
 				did_sth = 1;
@@ -404,8 +432,8 @@ int lu_site_purge_objects(const struct lu_env *env, struct lu_site *s,
 		 * races due to the reasons described in lu_object_put().
 		 */
 		while (!list_empty(&dispose)) {
-			h = container_of0(dispose.next,
-					  struct lu_object_header, loh_lru);
+			h = container_of(dispose.next,
+					 struct lu_object_header, loh_lru);
 			list_del_init(&h->loh_lru);
 			lu_object_free(env, lu_object_top(h));
 			lprocfs_counter_incr(s->ls_stats, LU_SS_LRU_PURGED);
@@ -554,16 +582,19 @@ void lu_object_print(const struct lu_env *env, void *cookie,
 }
 EXPORT_SYMBOL(lu_object_print);
 
+/*
+ * NOTE: htable_lookup() is called with the relevant
+ * hash bucket locked, but might drop and re-acquire the lock.
+ */
 static struct lu_object *htable_lookup(struct lu_site *s,
 				       struct cfs_hash_bd *bd,
 				       const struct lu_fid *f,
-				       wait_queue_entry_t *waiter,
 				       __u64 *version)
 {
 	struct lu_site_bkt_data *bkt;
 	struct lu_object_header *h;
 	struct hlist_node	*hnode;
-	__u64  ver = cfs_hash_bd_version_get(bd);
+	u64 ver = cfs_hash_bd_version_get(bd);
 
 	if (*version == ver)
 		return ERR_PTR(-ENOENT);
@@ -579,29 +610,14 @@ static struct lu_object *htable_lookup(struct lu_site *s,
 		return ERR_PTR(-ENOENT);
 	}
 
-	h = container_of0(hnode, struct lu_object_header, loh_hash);
-	if (likely(!lu_object_is_dying(h))) {
-		cfs_hash_get(s->ls_obj_hash, hnode);
-		lprocfs_counter_incr(s->ls_stats, LU_SS_CACHE_HIT);
-		if (!list_empty(&h->loh_lru)) {
-			list_del_init(&h->loh_lru);
-			bkt->lsb_lru_len--;
-			percpu_counter_dec(&s->ls_lru_len_counter);
-		}
-		return lu_object_top(h);
+	h = container_of(hnode, struct lu_object_header, loh_hash);
+	cfs_hash_get(s->ls_obj_hash, hnode);
+	lprocfs_counter_incr(s->ls_stats, LU_SS_CACHE_HIT);
+	if (!list_empty(&h->loh_lru)) {
+		list_del_init(&h->loh_lru);
+		percpu_counter_dec(&s->ls_lru_len_counter);
 	}
-
-	/*
-	 * Lookup found an object being destroyed this object cannot be
-	 * returned (to assure that references to dying objects are eventually
-	 * drained), and moreover, lookup has to wait until object is freed.
-	 */
-
-	init_waitqueue_entry(waiter, current);
-	add_wait_queue(&bkt->lsb_marche_funebre, waiter);
-	set_current_state(TASK_UNINTERRUPTIBLE);
-	lprocfs_counter_incr(s->ls_stats, LU_SS_CACHE_DEATH_RACE);
-	return ERR_PTR(-EAGAIN);
+	return lu_object_top(h);
 }
 
 /**
@@ -641,37 +657,17 @@ static void lu_object_limit(const struct lu_env *env, struct lu_device *dev)
 			      false);
 }
 
-static struct lu_object *lu_object_new(const struct lu_env *env,
-				       struct lu_device *dev,
-				       const struct lu_fid *f,
-				       const struct lu_object_conf *conf)
-{
-	struct lu_object	*o;
-	struct cfs_hash	      *hs;
-	struct cfs_hash_bd	    bd;
-
-	o = lu_object_alloc(env, dev, f, conf);
-	if (IS_ERR(o))
-		return o;
-
-	hs = dev->ld_site->ls_obj_hash;
-	cfs_hash_bd_get_and_lock(hs, (void *)f, &bd, 1);
-	cfs_hash_bd_add_locked(hs, &bd, &o->lo_header->loh_hash);
-	cfs_hash_bd_unlock(hs, &bd, 1);
-
-	lu_object_limit(env, dev);
-
-	return o;
-}
-
 /**
  * Core logic of lu_object_find*() functions.
+ *
+ * Much like lu_object_find(), but top level device of object is specifically
+ * \a dev rather than top level device of the site. This interface allows
+ * objects of different "stacking" to be created within the same site.
  */
-static struct lu_object *lu_object_find_try(const struct lu_env *env,
-					    struct lu_device *dev,
-					    const struct lu_fid *f,
-					    const struct lu_object_conf *conf,
-					    wait_queue_entry_t *waiter)
+struct lu_object *lu_object_find_at(const struct lu_env *env,
+				    struct lu_device *dev,
+				    const struct lu_fid *f,
+				    const struct lu_object_conf *conf)
 {
 	struct lu_object      *o;
 	struct lu_object      *shadow;
@@ -697,20 +693,19 @@ static struct lu_object *lu_object_find_try(const struct lu_env *env,
 	 * It is unnecessary to perform lookup-alloc-lookup-insert, instead,
 	 * just alloc and insert directly.
 	 *
-	 * If dying object is found during index search, add @waiter to the
-	 * site wait-queue and return ERR_PTR(-EAGAIN).
 	 */
-	if (conf && conf->loc_flags & LOC_F_NEW)
-		return lu_object_new(env, dev, f, conf);
-
 	s  = dev->ld_site;
 	hs = s->ls_obj_hash;
-	cfs_hash_bd_get_and_lock(hs, (void *)f, &bd, 1);
-	o = htable_lookup(s, &bd, f, waiter, &version);
-	cfs_hash_bd_unlock(hs, &bd, 1);
-	if (!IS_ERR(o) || PTR_ERR(o) != -ENOENT)
-		return o;
 
+	cfs_hash_bd_get(hs, f, &bd);
+	if (!(conf && conf->loc_flags & LOC_F_NEW)) {
+		cfs_hash_bd_lock(hs, &bd, 1);
+		o = htable_lookup(s, &bd, f, &version);
+		cfs_hash_bd_unlock(hs, &bd, 1);
+
+		if (!IS_ERR(o) || PTR_ERR(o) != -ENOENT)
+			return o;
+	}
 	/*
 	 * Allocate new object. This may result in rather complicated
 	 * operations, including fld queries, inode loading, etc.
@@ -723,7 +718,10 @@ static struct lu_object *lu_object_find_try(const struct lu_env *env,
 
 	cfs_hash_bd_lock(hs, &bd, 1);
 
-	shadow = htable_lookup(s, &bd, f, waiter, &version);
+	if (conf && conf->loc_flags & LOC_F_NEW)
+		shadow = ERR_PTR(-ENOENT);
+	else
+		shadow = htable_lookup(s, &bd, f, &version);
 	if (likely(PTR_ERR(shadow) == -ENOENT)) {
 		cfs_hash_bd_add_locked(hs, &bd, &o->lo_header->loh_hash);
 		cfs_hash_bd_unlock(hs, &bd, 1);
@@ -738,34 +736,6 @@ static struct lu_object *lu_object_find_try(const struct lu_env *env,
 	lu_object_free(env, o);
 	return shadow;
 }
-
-/**
- * Much like lu_object_find(), but top level device of object is specifically
- * \a dev rather than top level device of the site. This interface allows
- * objects of different "stacking" to be created within the same site.
- */
-struct lu_object *lu_object_find_at(const struct lu_env *env,
-				    struct lu_device *dev,
-				    const struct lu_fid *f,
-				    const struct lu_object_conf *conf)
-{
-	struct lu_site_bkt_data *bkt;
-	struct lu_object	*obj;
-	wait_queue_entry_t	   wait;
-
-	while (1) {
-		obj = lu_object_find_try(env, dev, f, conf, &wait);
-		if (obj != ERR_PTR(-EAGAIN))
-			return obj;
-		/*
-		 * lu_object_find_try() already added waiter into the
-		 * wait queue.
-		 */
-		schedule();
-		bkt = lu_site_bkt_from_fid(dev->ld_site, (void *)f);
-		remove_wait_queue(&bkt->lsb_marche_funebre, &wait);
-	}
-}
 EXPORT_SYMBOL(lu_object_find_at);
 
 /**
@@ -1327,7 +1297,7 @@ enum {
 
 static struct lu_context_key *lu_keys[LU_CONTEXT_KEY_NR] = { NULL, };
 
-static DEFINE_SPINLOCK(lu_keys_guard);
+static DEFINE_RWLOCK(lu_keys_guard);
 static atomic_t lu_key_initing_cnt = ATOMIC_INIT(0);
 
 /**
@@ -1351,7 +1321,7 @@ int lu_context_key_register(struct lu_context_key *key)
 	LASSERT(key->lct_tags != 0);
 
 	result = -ENFILE;
-	spin_lock(&lu_keys_guard);
+	write_lock(&lu_keys_guard);
 	for (i = 0; i < ARRAY_SIZE(lu_keys); ++i) {
 		if (!lu_keys[i]) {
 			key->lct_index = i;
@@ -1363,7 +1333,7 @@ int lu_context_key_register(struct lu_context_key *key)
 			break;
 		}
 	}
-	spin_unlock(&lu_keys_guard);
+	write_unlock(&lu_keys_guard);
 	return result;
 }
 EXPORT_SYMBOL(lu_context_key_register);
@@ -1396,8 +1366,8 @@ void lu_context_key_degister(struct lu_context_key *key)
 
 	lu_context_key_quiesce(key);
 
+	write_lock(&lu_keys_guard);
 	++key_set_version;
-	spin_lock(&lu_keys_guard);
 	key_fini(&lu_shrink_env.le_ctx, key->lct_index);
 
 	/**
@@ -1405,18 +1375,18 @@ void lu_context_key_degister(struct lu_context_key *key)
 	 * run lu_context_key::lct_fini() method.
 	 */
 	while (atomic_read(&key->lct_used) > 1) {
-		spin_unlock(&lu_keys_guard);
+		write_unlock(&lu_keys_guard);
 		CDEBUG(D_INFO, "%s: \"%s\" %p, %d\n",
 		       __func__, module_name(key->lct_owner),
 		       key, atomic_read(&key->lct_used));
 		schedule();
-		spin_lock(&lu_keys_guard);
+		write_lock(&lu_keys_guard);
 	}
 	if (lu_keys[key->lct_index]) {
 		lu_keys[key->lct_index] = NULL;
 		lu_ref_fini(&key->lct_reference);
 	}
-	spin_unlock(&lu_keys_guard);
+	write_unlock(&lu_keys_guard);
 
 	LASSERTF(atomic_read(&key->lct_used) == 1,
 		 "key has instances: %d\n",
@@ -1536,7 +1506,7 @@ void lu_context_key_quiesce(struct lu_context_key *key)
 		/*
 		 * XXX memory barrier has to go here.
 		 */
-		spin_lock(&lu_keys_guard);
+		write_lock(&lu_keys_guard);
 		key->lct_tags |= LCT_QUIESCENT;
 
 		/**
@@ -1544,27 +1514,30 @@ void lu_context_key_quiesce(struct lu_context_key *key)
 		 * have completed.
 		 */
 		while (atomic_read(&lu_key_initing_cnt) > 0) {
-			spin_unlock(&lu_keys_guard);
+			write_unlock(&lu_keys_guard);
 			CDEBUG(D_INFO, "%s: \"%s\" %p, %d (%d)\n",
 			       __func__,
 			       module_name(key->lct_owner),
 			       key, atomic_read(&key->lct_used),
 			atomic_read(&lu_key_initing_cnt));
 			schedule();
-			spin_lock(&lu_keys_guard);
+			write_lock(&lu_keys_guard);
 		}
 
 		list_for_each_entry(ctx, &lu_context_remembered, lc_remember)
 			key_fini(ctx, key->lct_index);
-		spin_unlock(&lu_keys_guard);
+
 		++key_set_version;
+		write_unlock(&lu_keys_guard);
 	}
 }
 
 void lu_context_key_revive(struct lu_context_key *key)
 {
+	write_lock(&lu_keys_guard);
 	key->lct_tags &= ~LCT_QUIESCENT;
 	++key_set_version;
+	write_unlock(&lu_keys_guard);
 }
 
 static void keys_fini(struct lu_context *ctx)
@@ -1583,6 +1556,7 @@ static void keys_fini(struct lu_context *ctx)
 
 static int keys_fill(struct lu_context *ctx)
 {
+	unsigned int pre_version;
 	unsigned int i;
 
 	/*
@@ -1594,10 +1568,12 @@ static int keys_fill(struct lu_context *ctx)
 	 * An atomic_t variable is still used, in order not to reacquire the
 	 * lock when decrementing the counter.
 	 */
-	spin_lock(&lu_keys_guard);
+	read_lock(&lu_keys_guard);
 	atomic_inc(&lu_key_initing_cnt);
-	spin_unlock(&lu_keys_guard);
+	pre_version = key_set_version;
+	read_unlock(&lu_keys_guard);
 
+refill:
 	LINVRNT(ctx->lc_value);
 	for (i = 0; i < ARRAY_SIZE(lu_keys); ++i) {
 		struct lu_context_key *key;
@@ -1638,9 +1614,17 @@ static int keys_fill(struct lu_context *ctx)
 			if (key->lct_exit)
 				ctx->lc_tags |= LCT_HAS_EXIT;
 		}
-		ctx->lc_version = key_set_version;
 	}
+
+	read_lock(&lu_keys_guard);
+	if (pre_version != key_set_version) {
+		pre_version = key_set_version;
+		read_unlock(&lu_keys_guard);
+		goto refill;
+	}
+	ctx->lc_version = key_set_version;
 	atomic_dec(&lu_key_initing_cnt);
+	read_unlock(&lu_keys_guard);
 	return 0;
 }
 
@@ -1665,9 +1649,9 @@ int lu_context_init(struct lu_context *ctx, __u32 tags)
 	ctx->lc_state = LCS_INITIALIZED;
 	ctx->lc_tags = tags;
 	if (tags & LCT_REMEMBER) {
-		spin_lock(&lu_keys_guard);
+		write_lock(&lu_keys_guard);
 		list_add(&ctx->lc_remember, &lu_context_remembered);
-		spin_unlock(&lu_keys_guard);
+		write_unlock(&lu_keys_guard);
 	} else {
 		INIT_LIST_HEAD(&ctx->lc_remember);
 	}
@@ -1693,10 +1677,10 @@ void lu_context_fini(struct lu_context *ctx)
 		keys_fini(ctx);
 
 	} else { /* could race with key degister */
-		spin_lock(&lu_keys_guard);
+		write_lock(&lu_keys_guard);
 		keys_fini(ctx);
 		list_del_init(&ctx->lc_remember);
-		spin_unlock(&lu_keys_guard);
+		write_unlock(&lu_keys_guard);
 	}
 }
 EXPORT_SYMBOL(lu_context_fini);
@@ -1721,10 +1705,11 @@ void lu_context_exit(struct lu_context *ctx)
 	LINVRNT(ctx->lc_state == LCS_ENTERED);
 	ctx->lc_state = LCS_LEFT;
 	if (ctx->lc_tags & LCT_HAS_EXIT && ctx->lc_value) {
+		/* could race with key quiescency */
+		if (ctx->lc_tags & LCT_REMEMBER)
+			read_lock(&lu_keys_guard);
+
 		for (i = 0; i < ARRAY_SIZE(lu_keys); ++i) {
-			/* could race with key quiescency */
-			if (ctx->lc_tags & LCT_REMEMBER)
-				spin_lock(&lu_keys_guard);
 			if (ctx->lc_value[i]) {
 				struct lu_context_key *key;
 
@@ -1733,9 +1718,10 @@ void lu_context_exit(struct lu_context *ctx)
 					key->lct_exit(ctx,
 						      key, ctx->lc_value[i]);
 			}
-			if (ctx->lc_tags & LCT_REMEMBER)
-				spin_unlock(&lu_keys_guard);
 		}
+
+		if (ctx->lc_tags & LCT_REMEMBER)
+			read_unlock(&lu_keys_guard);
 	}
 }
 EXPORT_SYMBOL(lu_context_exit);
@@ -1747,7 +1733,14 @@ EXPORT_SYMBOL(lu_context_exit);
  */
 int lu_context_refill(struct lu_context *ctx)
 {
-	return likely(ctx->lc_version == key_set_version) ? 0 : keys_fill(ctx);
+	read_lock(&lu_keys_guard);
+	if (likely(ctx->lc_version == key_set_version)) {
+		read_unlock(&lu_keys_guard);
+		return 0;
+	}
+
+	read_unlock(&lu_keys_guard);
+	return keys_fill(ctx);
 }
 
 /**
@@ -1798,19 +1791,24 @@ struct lu_site_stats {
 	unsigned int	lss_busy;
 };
 
-static void lu_site_stats_get(struct cfs_hash *hs,
+static void lu_site_stats_get(const struct lu_site *s,
 			      struct lu_site_stats *stats, int populated)
 {
+	struct cfs_hash *hs = s->ls_obj_hash;
 	struct cfs_hash_bd bd;
 	unsigned int i;
+	/*
+	 * percpu_counter_sum_positive() won't accept a const pointer
+	 * as it does modify the struct by taking a spinlock
+	 */
+	struct lu_site *s2 = (struct lu_site *)s;
 
+	stats->lss_busy += cfs_hash_size_get(hs) -
+		percpu_counter_sum_positive(&s2->ls_lru_len_counter);
 	cfs_hash_for_each_bucket(hs, &bd, i) {
-		struct lu_site_bkt_data *bkt = cfs_hash_bd_extra_get(hs, &bd);
 		struct hlist_head	*hhead;
 
 		cfs_hash_bd_lock(hs, &bd, 1);
-		stats->lss_busy  +=
-			cfs_hash_bd_count_get(&bd) - bkt->lsb_lru_len;
 		stats->lss_total += cfs_hash_bd_count_get(&bd);
 		stats->lss_max_search = max((int)stats->lss_max_search,
 					    cfs_hash_bd_depmax_get(&bd));
@@ -2003,7 +2001,7 @@ int lu_site_stats_print(const struct lu_site *s, struct seq_file *m)
 	struct lu_site_stats stats;
 
 	memset(&stats, 0, sizeof(stats));
-	lu_site_stats_get(s->ls_obj_hash, &stats, 1);
+	lu_site_stats_get(s, &stats, 1);
 
 	seq_printf(m, "%d/%d %d/%ld %d %d %d %d %d %d %d\n",
 		   stats.lss_busy,
diff --git a/drivers/staging/lustre/lustre/obdclass/lu_ref.c b/drivers/staging/lustre/lustre/obdclass/lu_ref.c
index 54fc88206534..f67cb89ea0ba 100644
--- a/drivers/staging/lustre/lustre/obdclass/lu_ref.c
+++ b/drivers/staging/lustre/lustre/obdclass/lu_ref.c
@@ -39,8 +39,6 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include <linux/libcfs/libcfs.h>
-
 #include <obd.h>
 #include <obd_class.h>
 #include <obd_support.h>
diff --git a/drivers/staging/lustre/lustre/obdclass/lustre_handles.c b/drivers/staging/lustre/lustre/obdclass/lustre_handles.c
index f53b1a3c342e..cdc8dc10690d 100644
--- a/drivers/staging/lustre/lustre/obdclass/lustre_handles.c
+++ b/drivers/staging/lustre/lustre/obdclass/lustre_handles.c
@@ -37,6 +37,7 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
+#include <linux/random.h>
 #include <obd_support.h>
 #include <lustre_handles.h>
 #include <lustre_lib.h>
diff --git a/drivers/staging/lustre/lustre/obdclass/obd_config.c b/drivers/staging/lustre/lustre/obdclass/obd_config.c
index 277576b586db..ffc1814398a5 100644
--- a/drivers/staging/lustre/lustre/obdclass/obd_config.c
+++ b/drivers/staging/lustre/lustre/obdclass/obd_config.c
@@ -48,7 +48,69 @@
 
 #include "llog_internal.h"
 
-static struct cfs_hash_ops uuid_hash_ops;
+/*
+ * uuid<->export lustre hash operations
+ */
+/*
+ * NOTE: It is impossible to find an export that is in failed
+ *       state with this function
+ */
+static int
+uuid_keycmp(struct rhashtable_compare_arg *arg, const void *obj)
+{
+	const struct obd_uuid *uuid = arg->key;
+	const struct obd_export *exp = obj;
+
+	if (obd_uuid_equals(uuid, &exp->exp_client_uuid) &&
+	    !exp->exp_failed)
+		return 0;
+	return -ESRCH;
+}
+
+static void
+uuid_export_exit(void *vexport, void *data)
+{
+	struct obd_export *exp = vexport;
+
+	class_export_put(exp);
+}
+
+static const struct rhashtable_params uuid_hash_params = {
+	.key_len	= sizeof(struct obd_uuid),
+	.key_offset	= offsetof(struct obd_export, exp_client_uuid),
+	.head_offset	= offsetof(struct obd_export, exp_uuid_hash),
+	.obj_cmpfn	= uuid_keycmp,
+	.automatic_shrinking = true,
+};
+
+int obd_uuid_add(struct obd_device *obd, struct obd_export *export)
+{
+	int rc;
+
+	rc = rhashtable_lookup_insert_fast(&obd->obd_uuid_hash,
+					   &export->exp_uuid_hash,
+					   uuid_hash_params);
+	if (rc == 0)
+		class_export_get(export);
+	else if (rc == -EEXIST)
+		rc = -EALREADY;
+	else
+		/* map obscure error codes to -ENOMEM */
+		rc = -ENOMEM;
+	return rc;
+}
+
+void obd_uuid_del(struct obd_device *obd, struct obd_export *export)
+{
+	int rc;
+
+	rc = rhashtable_remove_fast(&obd->obd_uuid_hash,
+				    &export->exp_uuid_hash,
+				    uuid_hash_params);
+
+	if (rc == 0)
+		class_export_put(export);
+}
 
 /*********** string parsing utils *********/
 
@@ -269,7 +331,7 @@ static int class_attach(struct lustre_cfg *lcfg)
 	/* obd->obd_osfs_age must be set to a value in the distant
 	 * past to guarantee a fresh statfs is fetched on mount.
 	 */
-	obd->obd_osfs_age = cfs_time_shift_64(-1000);
+	obd->obd_osfs_age = get_jiffies_64() - 1000 * HZ;
 
 	/* XXX belongs in setup not attach  */
 	init_rwsem(&obd->obd_observer_link_sem);
@@ -347,26 +409,18 @@ static int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
 	 * other fns check that status, and we're not actually set up yet.
 	 */
 	obd->obd_starting = 1;
-	obd->obd_uuid_hash = NULL;
 	spin_unlock(&obd->obd_dev_lock);
 
 	/* create an uuid-export lustre hash */
-	obd->obd_uuid_hash = cfs_hash_create("UUID_HASH",
-					     HASH_UUID_CUR_BITS,
-					     HASH_UUID_MAX_BITS,
-					     HASH_UUID_BKT_BITS, 0,
-					     CFS_HASH_MIN_THETA,
-					     CFS_HASH_MAX_THETA,
-					     &uuid_hash_ops, CFS_HASH_DEFAULT);
-	if (!obd->obd_uuid_hash) {
-		err = -ENOMEM;
+	err = rhashtable_init(&obd->obd_uuid_hash, &uuid_hash_params);
+
+	if (err)
 		goto err_hash;
-	}
 
 	exp = class_new_export(obd, &obd->obd_uuid);
 	if (IS_ERR(exp)) {
 		err = PTR_ERR(exp);
-		goto err_hash;
+		goto err_new;
 	}
 
 	obd->obd_self_export = exp;
@@ -392,11 +446,9 @@ static int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
 		class_unlink_export(obd->obd_self_export);
 		obd->obd_self_export = NULL;
 	}
+err_new:
+	rhashtable_destroy(&obd->obd_uuid_hash);
 err_hash:
-	if (obd->obd_uuid_hash) {
-		cfs_hash_putref(obd->obd_uuid_hash);
-		obd->obd_uuid_hash = NULL;
-	}
 	obd->obd_starting = 0;
 	CERROR("setup %s failed (%d)\n", obd->obd_name, err);
 	return err;
@@ -490,10 +542,7 @@ static int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg)
 		       obd->obd_name, err);
 
 	/* destroy an uuid-export hash body */
-	if (obd->obd_uuid_hash) {
-		cfs_hash_putref(obd->obd_uuid_hash);
-		obd->obd_uuid_hash = NULL;
-	}
+	rhashtable_free_and_destroy(&obd->obd_uuid_hash, uuid_export_exit, NULL);
 
 	class_decref(obd, "setup", obd);
 	obd->obd_set_up = 0;
@@ -1487,73 +1536,3 @@ int class_manual_cleanup(struct obd_device *obd)
 	return rc;
 }
 EXPORT_SYMBOL(class_manual_cleanup);
-
-/*
- * uuid<->export lustre hash operations
- */
-
-static unsigned int
-uuid_hash(struct cfs_hash *hs, const void *key, unsigned int mask)
-{
-	return cfs_hash_djb2_hash(((struct obd_uuid *)key)->uuid,
-				  sizeof(((struct obd_uuid *)key)->uuid), mask);
-}
-
-static void *
-uuid_key(struct hlist_node *hnode)
-{
-	struct obd_export *exp;
-
-	exp = hlist_entry(hnode, struct obd_export, exp_uuid_hash);
-
-	return &exp->exp_client_uuid;
-}
-
-/*
- * NOTE: It is impossible to find an export that is in failed
- *       state with this function
- */
-static int
-uuid_keycmp(const void *key, struct hlist_node *hnode)
-{
-	struct obd_export *exp;
-
-	LASSERT(key);
-	exp = hlist_entry(hnode, struct obd_export, exp_uuid_hash);
-
-	return obd_uuid_equals(key, &exp->exp_client_uuid) &&
-	       !exp->exp_failed;
-}
-
-static void *
-uuid_export_object(struct hlist_node *hnode)
-{
-	return hlist_entry(hnode, struct obd_export, exp_uuid_hash);
-}
-
-static void
-uuid_export_get(struct cfs_hash *hs, struct hlist_node *hnode)
-{
-	struct obd_export *exp;
-
-	exp = hlist_entry(hnode, struct obd_export, exp_uuid_hash);
-	class_export_get(exp);
-}
-
-static void
-uuid_export_put_locked(struct cfs_hash *hs, struct hlist_node *hnode)
-{
-	struct obd_export *exp;
-
-	exp = hlist_entry(hnode, struct obd_export, exp_uuid_hash);
-	class_export_put(exp);
-}
-
-static struct cfs_hash_ops uuid_hash_ops = {
-	.hs_hash	= uuid_hash,
-	.hs_key		= uuid_key,
-	.hs_keycmp      = uuid_keycmp,
-	.hs_object      = uuid_export_object,
-	.hs_get		= uuid_export_get,
-	.hs_put_locked  = uuid_export_put_locked,
-};
diff --git a/drivers/staging/lustre/lustre/obdclass/obd_mount.c b/drivers/staging/lustre/lustre/obdclass/obd_mount.c
index f5e8214ac37b..06c38fdef7ba 100644
--- a/drivers/staging/lustre/lustre/obdclass/obd_mount.c
+++ b/drivers/staging/lustre/lustre/obdclass/obd_mount.c
@@ -41,6 +41,7 @@
 #define D_MOUNT (D_SUPER | D_CONFIG/*|D_WARNING */)
 #define PRINT_CMD CDEBUG
 
+#include <linux/random.h>
 #include <obd.h>
 #include <lustre_compat.h>
 #include <obd_class.h>
diff --git a/drivers/staging/lustre/lustre/obdclass/uuid.c b/drivers/staging/lustre/lustre/obdclass/uuid.c
index 6cf7a03f048f..ec8c6dc5c9a7 100644
--- a/drivers/staging/lustre/lustre/obdclass/uuid.c
+++ b/drivers/staging/lustre/lustre/obdclass/uuid.c
@@ -35,8 +35,6 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include <linux/libcfs/libcfs.h>
-
 #include <obd_support.h>
 #include <obd_class.h>
 
diff --git a/drivers/staging/lustre/lustre/obdecho/echo_client.c b/drivers/staging/lustre/lustre/obdecho/echo_client.c
index 99a76db51ae0..b692e76e7108 100644
--- a/drivers/staging/lustre/lustre/obdecho/echo_client.c
+++ b/drivers/staging/lustre/lustre/obdecho/echo_client.c
@@ -32,7 +32,6 @@
  */
 
 #define DEBUG_SUBSYSTEM S_ECHO
-#include <linux/libcfs/libcfs.h>
 
 #include <obd.h>
 #include <obd_support.h>
@@ -99,7 +98,7 @@ static int echo_client_cleanup(struct obd_device *obddev);
  */
 static inline struct echo_device *cl2echo_dev(const struct cl_device *dev)
 {
-	return container_of0(dev, struct echo_device, ed_cl);
+	return container_of_safe(dev, struct echo_device, ed_cl);
 }
 
 static inline struct cl_device *echo_dev2cl(struct echo_device *d)
@@ -1701,10 +1700,16 @@ static void echo_client_exit(void)
 
 static int __init obdecho_init(void)
 {
+	int rc;
+
 	LCONSOLE_INFO("Echo OBD driver; http://www.lustre.org/\n");
 
 	LASSERT(PAGE_SIZE % OBD_ECHO_BLOCK_SIZE == 0);
 
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	return echo_client_init();
 }
 
diff --git a/drivers/staging/lustre/lustre/osc/osc_cache.c b/drivers/staging/lustre/lustre/osc/osc_cache.c
index 459503727ce3..f26983004843 100644
--- a/drivers/staging/lustre/lustre/osc/osc_cache.c
+++ b/drivers/staging/lustre/lustre/osc/osc_cache.c
@@ -1290,7 +1290,7 @@ static int osc_make_ready(const struct lu_env *env, struct osc_async_page *oap,
 
 	result = cl_page_make_ready(env, page, CRT_WRITE);
 	if (result == 0)
-		opg->ops_submit_time = cfs_time_current();
+		opg->ops_submit_time = jiffies;
 	return result;
 }
 
@@ -2622,7 +2622,7 @@ int osc_flush_async_page(const struct lu_env *env, struct cl_io *io,
 	oap->oap_async_flags |= ASYNC_READY | ASYNC_URGENT;
 	spin_unlock(&oap->oap_lock);
 
-	if (memory_pressure_get())
+	if (current->flags & PF_MEMALLOC)
 		ext->oe_memalloc = 1;
 
 	ext->oe_urgent = 1;
diff --git a/drivers/staging/lustre/lustre/osc/osc_cl_internal.h b/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
index 1449013722f6..2d3cba16ef34 100644
--- a/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
+++ b/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
@@ -43,8 +43,6 @@
 #ifndef OSC_CL_INTERNAL_H
 #define OSC_CL_INTERNAL_H
 
-#include <linux/libcfs/libcfs.h>
-
 #include <obd.h>
 /* osc_build_res_name() */
 #include <cl_object.h>
@@ -460,7 +458,7 @@ static inline int osc_is_object(const struct lu_object *obj)
 static inline struct osc_device *lu2osc_dev(const struct lu_device *d)
 {
 	LINVRNT(d->ld_type == &osc_device_type);
-	return container_of0(d, struct osc_device, od_cl.cd_lu_dev);
+	return container_of(d, struct osc_device, od_cl.cd_lu_dev);
 }
 
 static inline struct obd_export *osc_export(const struct osc_object *obj)
@@ -476,7 +474,7 @@ static inline struct client_obd *osc_cli(const struct osc_object *obj)
 static inline struct osc_object *cl2osc(const struct cl_object *obj)
 {
 	LINVRNT(osc_is_object(&obj->co_lu));
-	return container_of0(obj, struct osc_object, oo_cl);
+	return container_of(obj, struct osc_object, oo_cl);
 }
 
 static inline struct cl_object *osc2cl(const struct osc_object *obj)
@@ -509,12 +507,12 @@ static inline enum cl_lock_mode osc_ldlm2cl_lock(enum ldlm_mode mode)
 static inline struct osc_page *cl2osc_page(const struct cl_page_slice *slice)
 {
 	LINVRNT(osc_is_object(&slice->cpl_obj->co_lu));
-	return container_of0(slice, struct osc_page, ops_cl);
+	return container_of(slice, struct osc_page, ops_cl);
 }
 
 static inline struct osc_page *oap2osc(struct osc_async_page *oap)
 {
-	return container_of0(oap, struct osc_page, ops_oap);
+	return container_of_safe(oap, struct osc_page, ops_oap);
 }
 
 static inline pgoff_t osc_index(struct osc_page *opg)
@@ -545,7 +543,7 @@ osc_cl_page_osc(struct cl_page *page, struct osc_object *osc)
 static inline struct osc_lock *cl2osc_lock(const struct cl_lock_slice *slice)
 {
 	LINVRNT(osc_is_object(&slice->cls_obj->co_lu));
-	return container_of0(slice, struct osc_lock, ols_cl);
+	return container_of(slice, struct osc_lock, ols_cl);
 }
 
 static inline struct osc_lock *osc_lock_at(const struct cl_lock *lock)
diff --git a/drivers/staging/lustre/lustre/osc/osc_internal.h b/drivers/staging/lustre/lustre/osc/osc_internal.h
index 32db150fd42e..fca020568c19 100644
--- a/drivers/staging/lustre/lustre/osc/osc_internal.h
+++ b/drivers/staging/lustre/lustre/osc/osc_internal.h
@@ -180,7 +180,7 @@ struct osc_device {
 
 static inline struct osc_device *obd2osc_dev(const struct obd_device *d)
 {
-	return container_of0(d->obd_lu_dev, struct osc_device, od_cl.cd_lu_dev);
+	return container_of_safe(d->obd_lu_dev, struct osc_device, od_cl.cd_lu_dev);
 }
 
 extern struct lu_kmem_descr osc_caches[];
@@ -188,8 +188,9 @@ extern struct lu_kmem_descr osc_caches[];
 extern struct kmem_cache *osc_quota_kmem;
 struct osc_quota_info {
 	/** linkage for quota hash table */
-	struct hlist_node oqi_hash;
-	u32	  oqi_id;
+	struct rhash_head oqi_hash;
+	u32		  oqi_id;
+	struct rcu_head	  rcu;
 };
 
 int osc_quota_setup(struct obd_device *obd);
diff --git a/drivers/staging/lustre/lustre/osc/osc_io.c b/drivers/staging/lustre/lustre/osc/osc_io.c
index 76743faf3e6d..67734a8ed331 100644
--- a/drivers/staging/lustre/lustre/osc/osc_io.c
+++ b/drivers/staging/lustre/lustre/osc/osc_io.c
@@ -55,7 +55,7 @@
 static struct osc_io *cl2osc_io(const struct lu_env *env,
 				const struct cl_io_slice *slice)
 {
-	struct osc_io *oio = container_of0(slice, struct osc_io, oi_cl);
+	struct osc_io *oio = container_of_safe(slice, struct osc_io, oi_cl);
 
 	LINVRNT(oio == osc_env_io(env));
 	return oio;
diff --git a/drivers/staging/lustre/lustre/osc/osc_lock.c b/drivers/staging/lustre/lustre/osc/osc_lock.c
index fe8ed0d0497a..d93d33dc8dc4 100644
--- a/drivers/staging/lustre/lustre/osc/osc_lock.c
+++ b/drivers/staging/lustre/lustre/osc/osc_lock.c
@@ -38,7 +38,6 @@
 
 #define DEBUG_SUBSYSTEM S_OSC
 
-#include <linux/libcfs/libcfs.h>
 /* fid_build_reg_res_name() */
 #include <lustre_fid.h>
 
diff --git a/drivers/staging/lustre/lustre/osc/osc_object.c b/drivers/staging/lustre/lustre/osc/osc_object.c
index 6baa8e2e00c9..84240181c7ea 100644
--- a/drivers/staging/lustre/lustre/osc/osc_object.c
+++ b/drivers/staging/lustre/lustre/osc/osc_object.c
@@ -58,7 +58,7 @@ static struct lu_object *osc2lu(struct osc_object *osc)
 static struct osc_object *lu2osc(const struct lu_object *obj)
 {
 	LINVRNT(osc_is_object(obj));
-	return container_of0(obj, struct osc_object, oo_cl.co_lu);
+	return container_of(obj, struct osc_object, oo_cl.co_lu);
 }
 
 /*****************************************************************************
@@ -300,7 +300,7 @@ static int osc_object_fiemap(const struct lu_env *env, struct cl_object *obj,
 
 void osc_object_set_contended(struct osc_object *obj)
 {
-	obj->oo_contention_time = cfs_time_current();
+	obj->oo_contention_time = jiffies;
 	/* mb(); */
 	obj->oo_contended = 1;
 }
@@ -314,7 +314,7 @@ int osc_object_is_contended(struct osc_object *obj)
 {
 	struct osc_device *dev = lu2osc_dev(obj->oo_cl.co_lu.lo_dev);
 	int osc_contention_time = dev->od_contention_time;
-	unsigned long cur_time = cfs_time_current();
+	unsigned long cur_time = jiffies;
 	unsigned long retry_time;
 
 	if (OBD_FAIL_CHECK(OBD_FAIL_OSC_OBJECT_CONTENTION))
@@ -327,9 +327,8 @@ int osc_object_is_contended(struct osc_object *obj)
 	 * I like copy-paste. the code is copied from
 	 * ll_file_is_contended.
 	 */
-	retry_time = cfs_time_add(obj->oo_contention_time,
-				  osc_contention_time * HZ);
-	if (cfs_time_after(cur_time, retry_time)) {
+	retry_time = obj->oo_contention_time + osc_contention_time * HZ;
+	if (time_after(cur_time, retry_time)) {
 		osc_object_clear_contended(obj);
 		return 0;
 	}
diff --git a/drivers/staging/lustre/lustre/osc/osc_page.c b/drivers/staging/lustre/lustre/osc/osc_page.c
index 01a930dbbf64..20c553ef3a5e 100644
--- a/drivers/staging/lustre/lustre/osc/osc_page.c
+++ b/drivers/staging/lustre/lustre/osc/osc_page.c
@@ -125,7 +125,7 @@ static inline unsigned long osc_submit_duration(struct osc_page *opg)
 	if (opg->ops_submit_time == 0)
 		return 0;
 
-	return (cfs_time_current() - opg->ops_submit_time);
+	return (jiffies - opg->ops_submit_time);
 }
 
 static int osc_page_print(const struct lu_env *env,
@@ -312,7 +312,7 @@ void osc_page_submit(const struct lu_env *env, struct osc_page *opg,
 		oap->oap_cmd |= OBD_BRW_NOQUOTA;
 	}
 
-	opg->ops_submit_time = cfs_time_current();
+	opg->ops_submit_time = jiffies;
 	osc_page_transfer_get(opg, "transfer\0imm");
 	osc_page_transfer_add(env, opg, crt);
 }
diff --git a/drivers/staging/lustre/lustre/osc/osc_quota.c b/drivers/staging/lustre/lustre/osc/osc_quota.c
index ce1731dc604f..723ec2fb18bf 100644
--- a/drivers/staging/lustre/lustre/osc/osc_quota.c
+++ b/drivers/staging/lustre/lustre/osc/osc_quota.c
@@ -27,6 +27,13 @@
 #include <obd_class.h>
 #include "osc_internal.h"
 
+static const struct rhashtable_params quota_hash_params = {
+	.key_len	= sizeof(u32),
+	.key_offset	= offsetof(struct osc_quota_info, oqi_id),
+	.head_offset	= offsetof(struct osc_quota_info, oqi_hash),
+	.automatic_shrinking = true,
+};
+
 static inline struct osc_quota_info *osc_oqi_alloc(u32 id)
 {
 	struct osc_quota_info *oqi;
@@ -45,9 +52,10 @@ int osc_quota_chkdq(struct client_obd *cli, const unsigned int qid[])
 	for (type = 0; type < MAXQUOTAS; type++) {
 		struct osc_quota_info *oqi;
 
-		oqi = cfs_hash_lookup(cli->cl_quota_hash[type], &qid[type]);
+		oqi = rhashtable_lookup_fast(&cli->cl_quota_hash[type], &qid[type],
+					     quota_hash_params);
 		if (oqi) {
-			/* do not try to access oqi here, it could have been
+			/* Must not access oqi here, it could have been
 			 * freed by osc_quota_setdq()
 			 */
 
@@ -63,6 +71,14 @@ int osc_quota_chkdq(struct client_obd *cli, const unsigned int qid[])
 	return QUOTA_OK;
 }
 
+static void osc_quota_free(struct rcu_head *head)
+{
+	struct osc_quota_info *oqi = container_of(head, struct osc_quota_info, rcu);
+
+	kmem_cache_free(osc_quota_kmem, oqi);
+}
+
+
 #define MD_QUOTA_FLAG(type) ((type == USRQUOTA) ? OBD_MD_FLUSRQUOTA \
 						: OBD_MD_FLGRPQUOTA)
 #define FL_QUOTA_FLAG(type) ((type == USRQUOTA) ? OBD_FL_NO_USRQUOTA \
@@ -84,11 +100,14 @@ int osc_quota_setdq(struct client_obd *cli, const unsigned int qid[],
 			continue;
 
 		/* lookup the ID in the per-type hash table */
-		oqi = cfs_hash_lookup(cli->cl_quota_hash[type], &qid[type]);
+		rcu_read_lock();
+		oqi = rhashtable_lookup_fast(&cli->cl_quota_hash[type], &qid[type],
+					     quota_hash_params);
 		if ((flags & FL_QUOTA_FLAG(type)) != 0) {
 			/* This ID is getting close to its quota limit, let's
 			 * switch to sync I/O
 			 */
+			rcu_read_unlock();
 			if (oqi)
 				continue;
 
@@ -98,12 +117,16 @@ int osc_quota_setdq(struct client_obd *cli, const unsigned int qid[],
 				break;
 			}
 
-			rc = cfs_hash_add_unique(cli->cl_quota_hash[type],
-						 &qid[type], &oqi->oqi_hash);
+			rc = rhashtable_lookup_insert_fast(&cli->cl_quota_hash[type],
+							   &oqi->oqi_hash, quota_hash_params);
 			/* race with others? */
-			if (rc == -EALREADY) {
-				rc = 0;
+			if (rc) {
 				kmem_cache_free(osc_quota_kmem, oqi);
+				if (rc != -EEXIST) {
+					rc = -ENOMEM;
+					break;
+				}
+				rc = 0;
 			}
 
 			CDEBUG(D_QUOTA, "%s: setdq to insert for %s %d (%d)\n",
@@ -114,14 +137,14 @@ int osc_quota_setdq(struct client_obd *cli, const unsigned int qid[],
 			/* This ID is now off the hook, let's remove it from
 			 * the hash table
 			 */
-			if (!oqi)
+			if (!oqi) {
+				rcu_read_unlock();
 				continue;
-
-			oqi = cfs_hash_del_key(cli->cl_quota_hash[type],
-					       &qid[type]);
-			if (oqi)
-				kmem_cache_free(osc_quota_kmem, oqi);
-
+			}
+			if (rhashtable_remove_fast(&cli->cl_quota_hash[type],
+						   &oqi->oqi_hash, quota_hash_params) == 0)
+				call_rcu(&oqi->rcu, osc_quota_free);
+			rcu_read_unlock();
 			CDEBUG(D_QUOTA, "%s: setdq to remove for %s %d (%p)\n",
 			       cli_name(cli),
 			       type == USRQUOTA ? "user" : "group",
@@ -132,93 +155,21 @@ int osc_quota_setdq(struct client_obd *cli, const unsigned int qid[],
 	return rc;
 }
 
-/*
- * Hash operations for uid/gid <-> osc_quota_info
- */
-static unsigned int
-oqi_hashfn(struct cfs_hash *hs, const void *key, unsigned int mask)
-{
-	return cfs_hash_u32_hash(*((__u32 *)key), mask);
-}
-
-static int
-oqi_keycmp(const void *key, struct hlist_node *hnode)
-{
-	struct osc_quota_info *oqi;
-	u32 uid;
-
-	LASSERT(key);
-	uid = *((u32 *)key);
-	oqi = hlist_entry(hnode, struct osc_quota_info, oqi_hash);
-
-	return uid == oqi->oqi_id;
-}
-
-static void *
-oqi_key(struct hlist_node *hnode)
-{
-	struct osc_quota_info *oqi;
-
-	oqi = hlist_entry(hnode, struct osc_quota_info, oqi_hash);
-	return &oqi->oqi_id;
-}
-
-static void *
-oqi_object(struct hlist_node *hnode)
-{
-	return hlist_entry(hnode, struct osc_quota_info, oqi_hash);
-}
-
-static void
-oqi_get(struct cfs_hash *hs, struct hlist_node *hnode)
-{
-}
-
-static void
-oqi_put_locked(struct cfs_hash *hs, struct hlist_node *hnode)
-{
-}
-
 static void
-oqi_exit(struct cfs_hash *hs, struct hlist_node *hnode)
+oqi_exit(void *vquota, void *data)
 {
-	struct osc_quota_info *oqi;
-
-	oqi = hlist_entry(hnode, struct osc_quota_info, oqi_hash);
+	struct osc_quota_info *oqi = vquota;
 
-	kmem_cache_free(osc_quota_kmem, oqi);
+	osc_quota_free(&oqi->rcu);
 }
 
-#define HASH_QUOTA_BKT_BITS 5
-#define HASH_QUOTA_CUR_BITS 5
-#define HASH_QUOTA_MAX_BITS 15
-
-static struct cfs_hash_ops quota_hash_ops = {
-	.hs_hash	= oqi_hashfn,
-	.hs_keycmp	= oqi_keycmp,
-	.hs_key		= oqi_key,
-	.hs_object	= oqi_object,
-	.hs_get		= oqi_get,
-	.hs_put_locked	= oqi_put_locked,
-	.hs_exit	= oqi_exit,
-};
-
 int osc_quota_setup(struct obd_device *obd)
 {
 	struct client_obd *cli = &obd->u.cli;
 	int i, type;
 
 	for (type = 0; type < MAXQUOTAS; type++) {
-		cli->cl_quota_hash[type] = cfs_hash_create("QUOTA_HASH",
-							   HASH_QUOTA_CUR_BITS,
-							   HASH_QUOTA_MAX_BITS,
-							   HASH_QUOTA_BKT_BITS,
-							   0,
-							   CFS_HASH_MIN_THETA,
-							   CFS_HASH_MAX_THETA,
-							   &quota_hash_ops,
-							   CFS_HASH_DEFAULT);
-		if (!cli->cl_quota_hash[type])
+		if (rhashtable_init(&cli->cl_quota_hash[type], &quota_hash_params) != 0)
 			break;
 	}
 
@@ -226,7 +177,7 @@ int osc_quota_setup(struct obd_device *obd)
 		return 0;
 
 	for (i = 0; i < type; i++)
-		cfs_hash_putref(cli->cl_quota_hash[i]);
+		rhashtable_destroy(&cli->cl_quota_hash[i]);
 
 	return -ENOMEM;
 }
@@ -237,7 +188,8 @@ int osc_quota_cleanup(struct obd_device *obd)
 	int type;
 
 	for (type = 0; type < MAXQUOTAS; type++)
-		cfs_hash_putref(cli->cl_quota_hash[type]);
+		rhashtable_free_and_destroy(&cli->cl_quota_hash[type],
+					    oqi_exit, NULL);
 
 	return 0;
 }
diff --git a/drivers/staging/lustre/lustre/osc/osc_request.c b/drivers/staging/lustre/lustre/osc/osc_request.c
index 1c2bbbf5d864..61ef6c8d7a12 100644
--- a/drivers/staging/lustre/lustre/osc/osc_request.c
+++ b/drivers/staging/lustre/lustre/osc/osc_request.c
@@ -33,7 +33,8 @@
 
 #define DEBUG_SUBSYSTEM S_OSC
 
-#include <linux/libcfs/libcfs.h>
+#include <linux/libcfs/libcfs_hash.h>
+#include <linux/sched/mm.h>
 
 #include <lustre_dlm.h>
 #include <lustre_net.h>
@@ -617,7 +618,7 @@ static void osc_announce_cached(struct client_obd *cli, struct obdo *oa,
 void osc_update_next_shrink(struct client_obd *cli)
 {
 	cli->cl_next_shrink_grant =
-		cfs_time_shift(cli->cl_grant_shrink_interval);
+		jiffies + cli->cl_grant_shrink_interval * HZ;
 	CDEBUG(D_CACHE, "next time %ld to shrink grant\n",
 	       cli->cl_next_shrink_grant);
 }
@@ -741,14 +742,14 @@ int osc_shrink_grant_to_target(struct client_obd *cli, __u64 target_bytes)
 
 static int osc_should_shrink_grant(struct client_obd *client)
 {
-	unsigned long time = cfs_time_current();
+	unsigned long time = jiffies;
 	unsigned long next_shrink = client->cl_next_shrink_grant;
 
 	if ((client->cl_import->imp_connect_data.ocd_connect_flags &
 	     OBD_CONNECT_GRANT_SHRINK) == 0)
 		return 0;
 
-	if (cfs_time_aftereq(time, next_shrink - 5 * CFS_TICK)) {
+	if (time_after_eq(time, next_shrink - 5)) {
 		/* Get the current RPC size directly, instead of going via:
 		 * cli_brw_size(obd->u.cli.cl_import->imp_obd->obd_self_export)
 		 * Keep comment here so that it can be found by searching.
@@ -1654,7 +1655,7 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	struct cl_req_attr *crattr = NULL;
 	u64 starting_offset = OBD_OBJECT_EOF;
 	u64 ending_offset = 0;
-	int mpflag = 0;
+	unsigned int mpflag = 0;
 	int mem_tight = 0;
 	int page_count = 0;
 	bool soft_sync = false;
@@ -1677,7 +1678,7 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 
 	soft_sync = osc_over_unstable_soft_limit(cli);
 	if (mem_tight)
-		mpflag = cfs_memory_pressure_get_and_set();
+		mpflag = memalloc_noreclaim_save();
 
 	pga = kcalloc(page_count, sizeof(*pga), GFP_NOFS);
 	if (!pga) {
@@ -1791,7 +1792,7 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 
 out:
 	if (mem_tight != 0)
-		cfs_memory_pressure_restore(mpflag);
+		memalloc_noreclaim_restore(mpflag);
 
 	if (rc != 0) {
 		LASSERT(!req);
@@ -2296,7 +2297,7 @@ static int osc_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 		goto out;
 	default:
 		CDEBUG(D_INODE, "unrecognised ioctl %#x by %s\n",
-		       cmd, current_comm());
+		       cmd, current->comm);
 		err = -ENOTTY;
 		goto out;
 	}
@@ -2831,25 +2832,24 @@ static int __init osc_init(void)
 	 */
 	CDEBUG(D_INFO, "Lustre OSC module (%p).\n", &osc_caches);
 
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	rc = lu_kmem_init(osc_caches);
 	if (rc)
 		return rc;
 
 	lprocfs_osc_init_vars(&lvars);
 
-	rc = class_register_type(&osc_obd_ops, NULL,
-				 LUSTRE_OSC_NAME, &osc_device_type);
-	if (rc)
-		goto out_kmem;
-
 	rc = register_shrinker(&osc_cache_shrinker);
 	if (rc)
-		goto out_type;
+		goto err;
 
 	/* This is obviously too much memory, only prevent overflow here */
 	if (osc_reqpool_mem_max >= 1 << 12 || osc_reqpool_mem_max == 0) {
 		rc = -EINVAL;
-		goto out_type;
+		goto err;
 	}
 
 	reqpool_size = osc_reqpool_mem_max << 20;
@@ -2870,14 +2870,22 @@ static int __init osc_init(void)
 	osc_rq_pool = ptlrpc_init_rq_pool(0, OST_MAXREQSIZE,
 					  ptlrpc_add_rqs_to_pool);
 
-	if (osc_rq_pool)
-		return 0;
-
 	rc = -ENOMEM;
 
-out_type:
-	class_unregister_type(LUSTRE_OSC_NAME);
-out_kmem:
+	if (!osc_rq_pool)
+		goto err;
+
+	rc = class_register_type(&osc_obd_ops, NULL,
+				 LUSTRE_OSC_NAME, &osc_device_type);
+	if (rc)
+		goto err;
+
+	return rc;
+
+err:
+	if (osc_rq_pool)
+		ptlrpc_free_rq_pool(osc_rq_pool);
+	unregister_shrinker(&osc_cache_shrinker);
 	lu_kmem_fini(osc_caches);
 	return rc;
 }
diff --git a/drivers/staging/lustre/lustre/ptlrpc/client.c b/drivers/staging/lustre/lustre/ptlrpc/client.c
index ca096fadb9c0..c1b82bf20f08 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/client.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/client.c
@@ -35,6 +35,8 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
+#include <linux/libcfs/libcfs_cpu.h>
+#include <linux/random.h>
 #include <obd_support.h>
 #include <obd_class.h>
 #include <lustre_lib.h>
@@ -938,9 +940,9 @@ struct ptlrpc_request_set *ptlrpc_prep_set(void)
 	struct ptlrpc_request_set *set;
 	int cpt;
 
-	cpt = cfs_cpt_current(cfs_cpt_table, 0);
+	cpt = cfs_cpt_current(cfs_cpt_tab, 0);
 	set = kzalloc_node(sizeof(*set), GFP_NOFS,
-			   cfs_cpt_spread_node(cfs_cpt_table, cpt));
+			   cfs_cpt_spread_node(cfs_cpt_tab, cpt));
 	if (!set)
 		return NULL;
 	atomic_set(&set->set_refcount, 1);
@@ -1050,7 +1052,7 @@ void ptlrpc_set_add_req(struct ptlrpc_request_set *set,
 	list_add_tail(&req->rq_set_chain, &set->set_requests);
 	req->rq_set = set;
 	atomic_inc(&set->set_remaining);
-	req->rq_queued_time = cfs_time_current();
+	req->rq_queued_time = jiffies;
 
 	if (req->rq_reqmsg)
 		lustre_msg_set_jobid(req->rq_reqmsg, NULL);
@@ -1081,7 +1083,7 @@ void ptlrpc_set_add_new_req(struct ptlrpcd_ctl *pc,
 	spin_lock(&set->set_new_req_lock);
 	/* The set takes over the caller's request reference.  */
 	req->rq_set = set;
-	req->rq_queued_time = cfs_time_current();
+	req->rq_queued_time = jiffies;
 	list_add_tail(&req->rq_set_chain, &set->set_new_requests);
 	count = atomic_inc_return(&set->set_new_count);
 	spin_unlock(&set->set_new_req_lock);
@@ -1552,7 +1554,7 @@ static int ptlrpc_send_new_req(struct ptlrpc_request *req)
 
 	lustre_msg_set_last_xid(req->rq_reqmsg, min_xid);
 
-	lustre_msg_set_status(req->rq_reqmsg, current_pid());
+	lustre_msg_set_status(req->rq_reqmsg, current->pid);
 
 	rc = sptlrpc_req_refresh_ctx(req, -1);
 	if (rc) {
@@ -1567,7 +1569,7 @@ static int ptlrpc_send_new_req(struct ptlrpc_request *req)
 	}
 
 	CDEBUG(D_RPCTRACE, "Sending RPC pname:cluuid:pid:xid:nid:opc %s:%s:%d:%llu:%s:%d\n",
-	       current_comm(),
+	       current->comm,
 	       imp->imp_obd->obd_uuid.uuid,
 	       lustre_msg_get_status(req->rq_reqmsg), req->rq_xid,
 	       libcfs_nid2str(imp->imp_connection->c_peer.nid),
@@ -1978,7 +1980,7 @@ int ptlrpc_check_set(const struct lu_env *env, struct ptlrpc_request_set *set)
 
 		CDEBUG(req->rq_reqmsg ? D_RPCTRACE : 0,
 		       "Completed RPC pname:cluuid:pid:xid:nid:opc %s:%s:%d:%llu:%s:%d\n",
-		       current_comm(), imp->imp_obd->obd_uuid.uuid,
+		       current->comm, imp->imp_obd->obd_uuid.uuid,
 		       lustre_msg_get_status(req->rq_reqmsg), req->rq_xid,
 		       libcfs_nid2str(imp->imp_connection->c_peer.nid),
 		       lustre_msg_get_opc(req->rq_reqmsg));
@@ -2514,7 +2516,7 @@ static int ptlrpc_unregister_reply(struct ptlrpc_request *request, int async)
 		}
 
 		DEBUG_REQ(D_WARNING, request,
-			  "Unexpectedly long timeout receiving_reply=%d req_ulinked=%d reply_unlinked=%d",
+			  "Unexpectedly long timeout receiving_reply=%d req_unlinked=%d reply_unlinked=%d",
 			  request->rq_receiving_reply,
 			  request->rq_req_unlinked,
 			  request->rq_reply_unlinked);
@@ -2760,7 +2762,7 @@ int ptlrpc_queue_wait(struct ptlrpc_request *req)
 	}
 
 	/* for distributed debugging */
-	lustre_msg_set_status(req->rq_reqmsg, current_pid());
+	lustre_msg_set_status(req->rq_reqmsg, current->pid);
 
 	/* add a ref for the set (see comment in ptlrpc_set_add_req) */
 	ptlrpc_request_addref(req);
diff --git a/drivers/staging/lustre/lustre/ptlrpc/connection.c b/drivers/staging/lustre/lustre/ptlrpc/connection.c
index dfdb4587d49d..fb35a89ca6c6 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/connection.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/connection.c
@@ -38,8 +38,41 @@
 
 #include "ptlrpc_internal.h"
 
-static struct cfs_hash *conn_hash;
-static struct cfs_hash_ops conn_hash_ops;
+static struct rhashtable conn_hash;
+
+/*
+ * struct lnet_process_id may contain unassigned bytes which might not
+ * be zero, so we cannot just hash and compare bytes.
+ */
+
+static u32 lnet_process_id_hash(const void *data, u32 len, u32 seed)
+{
+	const struct lnet_process_id *lpi = data;
+
+	seed = hash_32(seed ^ lpi->pid, 32);
+	seed ^= hash_64(lpi->nid, 32);
+	return seed;
+}
+
+static int lnet_process_id_cmp(struct rhashtable_compare_arg *arg,
+			       const void *obj)
+{
+	const struct lnet_process_id *lpi = arg->key;
+	const struct ptlrpc_connection *con = obj;
+
+	if (lpi->nid == con->c_peer.nid &&
+	    lpi->pid == con->c_peer.pid)
+		return 0;
+	return -ESRCH;
+}
+
+static const struct rhashtable_params conn_hash_params = {
+	.key_len	= 1, /* actually variable-length */
+	.key_offset	= offsetof(struct ptlrpc_connection, c_peer),
+	.head_offset	= offsetof(struct ptlrpc_connection, c_hash),
+	.hashfn		= lnet_process_id_hash,
+	.obj_cmpfn	= lnet_process_id_cmp,
+};
 
 struct ptlrpc_connection *
 ptlrpc_connection_get(struct lnet_process_id peer, lnet_nid_t self,
@@ -47,9 +80,11 @@ ptlrpc_connection_get(struct lnet_process_id peer, lnet_nid_t self,
 {
 	struct ptlrpc_connection *conn, *conn2;
 
-	conn = cfs_hash_lookup(conn_hash, &peer);
-	if (conn)
+	conn = rhashtable_lookup_fast(&conn_hash, &peer, conn_hash_params);
+	if (conn) {
+		ptlrpc_connection_addref(conn);
 		goto out;
+	}
 
 	conn = kzalloc(sizeof(*conn), GFP_NOFS);
 	if (!conn)
@@ -57,7 +92,6 @@ ptlrpc_connection_get(struct lnet_process_id peer, lnet_nid_t self,
 
 	conn->c_peer = peer;
 	conn->c_self = self;
-	INIT_HLIST_NODE(&conn->c_hash);
 	atomic_set(&conn->c_refcount, 1);
 	if (uuid)
 		obd_str2uuid(&conn->c_remote_uuid, uuid->uuid);
@@ -65,17 +99,18 @@ ptlrpc_connection_get(struct lnet_process_id peer, lnet_nid_t self,
 	/*
 	 * Add the newly created conn to the hash, on key collision we
 	 * lost a racing addition and must destroy our newly allocated
-	 * connection.  The object which exists in the has will be
-	 * returned and may be compared against out object.
-	 */
-	/* In the function below, .hs_keycmp resolves to
-	 * conn_keycmp()
+	 * connection.  The object which exists in the hash will be
+	 * returned, otherwise NULL is returned on success.
 	 */
-	/* coverity[overrun-buffer-val] */
-	conn2 = cfs_hash_findadd_unique(conn_hash, &peer, &conn->c_hash);
-	if (conn != conn2) {
+	conn2 = rhashtable_lookup_get_insert_fast(&conn_hash, &conn->c_hash,
+						  conn_hash_params);
+	if (conn2 != NULL) {
+		/* insertion failed */
 		kfree(conn);
+		if (IS_ERR(conn2))
+			return NULL;
 		conn = conn2;
+		ptlrpc_connection_addref(conn);
 	}
 out:
 	CDEBUG(D_INFO, "conn=%p refcount %d to %s\n",
@@ -91,7 +126,7 @@ int ptlrpc_connection_put(struct ptlrpc_connection *conn)
 	if (!conn)
 		return rc;
 
-	LASSERT(atomic_read(&conn->c_refcount) > 1);
+	LASSERT(atomic_read(&conn->c_refcount) > 0);
 
 	/*
 	 * We do not remove connection from hashtable and
@@ -109,7 +144,7 @@ int ptlrpc_connection_put(struct ptlrpc_connection *conn)
 	 * when ptlrpc_connection_fini()->lh_exit->conn_exit()
 	 * path is called.
 	 */
-	if (atomic_dec_return(&conn->c_refcount) == 1)
+	if (atomic_dec_return(&conn->c_refcount) == 0)
 		rc = 1;
 
 	CDEBUG(D_INFO, "PUT conn=%p refcount %d to %s\n",
@@ -130,88 +165,11 @@ ptlrpc_connection_addref(struct ptlrpc_connection *conn)
 	return conn;
 }
 
-int ptlrpc_connection_init(void)
-{
-	conn_hash = cfs_hash_create("CONN_HASH",
-				    HASH_CONN_CUR_BITS,
-				    HASH_CONN_MAX_BITS,
-				    HASH_CONN_BKT_BITS, 0,
-				    CFS_HASH_MIN_THETA,
-				    CFS_HASH_MAX_THETA,
-				    &conn_hash_ops, CFS_HASH_DEFAULT);
-	if (!conn_hash)
-		return -ENOMEM;
-
-	return 0;
-}
-
-void ptlrpc_connection_fini(void)
-{
-	cfs_hash_putref(conn_hash);
-}
-
-/*
- * Hash operations for net_peer<->connection
- */
-static unsigned int
-conn_hashfn(struct cfs_hash *hs, const void *key, unsigned int mask)
-{
-	return cfs_hash_djb2_hash(key, sizeof(struct lnet_process_id), mask);
-}
-
-static int
-conn_keycmp(const void *key, struct hlist_node *hnode)
-{
-	struct ptlrpc_connection *conn;
-	const struct lnet_process_id *conn_key;
-
-	LASSERT(key);
-	conn_key = key;
-	conn = hlist_entry(hnode, struct ptlrpc_connection, c_hash);
-
-	return conn_key->nid == conn->c_peer.nid &&
-	       conn_key->pid == conn->c_peer.pid;
-}
-
-static void *
-conn_key(struct hlist_node *hnode)
-{
-	struct ptlrpc_connection *conn;
-
-	conn = hlist_entry(hnode, struct ptlrpc_connection, c_hash);
-	return &conn->c_peer;
-}
-
-static void *
-conn_object(struct hlist_node *hnode)
-{
-	return hlist_entry(hnode, struct ptlrpc_connection, c_hash);
-}
-
 static void
-conn_get(struct cfs_hash *hs, struct hlist_node *hnode)
+conn_exit(void *vconn, void *data)
 {
-	struct ptlrpc_connection *conn;
+	struct ptlrpc_connection *conn = vconn;
 
-	conn = hlist_entry(hnode, struct ptlrpc_connection, c_hash);
-	atomic_inc(&conn->c_refcount);
-}
-
-static void
-conn_put_locked(struct cfs_hash *hs, struct hlist_node *hnode)
-{
-	struct ptlrpc_connection *conn;
-
-	conn = hlist_entry(hnode, struct ptlrpc_connection, c_hash);
-	atomic_dec(&conn->c_refcount);
-}
-
-static void
-conn_exit(struct cfs_hash *hs, struct hlist_node *hnode)
-{
-	struct ptlrpc_connection *conn;
-
-	conn = hlist_entry(hnode, struct ptlrpc_connection, c_hash);
 	/*
 	 * Nothing should be left. Connection user put it and
 	 * connection also was deleted from table by this time
@@ -223,12 +181,12 @@ conn_exit(struct cfs_hash *hs, struct hlist_node *hnode)
 	kfree(conn);
 }
 
-static struct cfs_hash_ops conn_hash_ops = {
-	.hs_hash	= conn_hashfn,
-	.hs_keycmp      = conn_keycmp,
-	.hs_key		= conn_key,
-	.hs_object      = conn_object,
-	.hs_get		= conn_get,
-	.hs_put_locked  = conn_put_locked,
-	.hs_exit	= conn_exit,
-};
+int ptlrpc_connection_init(void)
+{
+	return rhashtable_init(&conn_hash, &conn_hash_params);
+}
+
+void ptlrpc_connection_fini(void)
+{
+	rhashtable_free_and_destroy(&conn_hash, conn_exit, NULL);
+}
diff --git a/drivers/staging/lustre/lustre/ptlrpc/errno.c b/drivers/staging/lustre/lustre/ptlrpc/errno.c
index 54f0c36dc2bd..b904524fc1c6 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/errno.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/errno.c
@@ -26,7 +26,9 @@
  * Copyright (c) 2013, Intel Corporation.
  */
 
-#include <linux/libcfs/libcfs.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
 #include <lustre_errno.h>
 
 /*
diff --git a/drivers/staging/lustre/lustre/ptlrpc/import.c b/drivers/staging/lustre/lustre/ptlrpc/import.c
index a2c4fc3488b1..1a0f35dfab97 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/import.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/import.c
@@ -37,6 +37,7 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
+#include <linux/kthread.h>
 #include <obd_support.h>
 #include <lustre_ha.h>
 #include <lustre_net.h>
@@ -463,7 +464,7 @@ static int import_select_connection(struct obd_import *imp)
 		 * the last successful attempt, go with this one
 		 */
 		if ((conn->oic_last_attempt == 0) ||
-		    cfs_time_beforeq_64(conn->oic_last_attempt,
+		    time_before_eq64(conn->oic_last_attempt,
 					imp->imp_last_success_conn)) {
 			imp_conn = conn;
 			tried_all = 0;
@@ -476,7 +477,7 @@ static int import_select_connection(struct obd_import *imp)
 		 */
 		if (!imp_conn)
 			imp_conn = conn;
-		else if (cfs_time_before_64(conn->oic_last_attempt,
+		else if (time_before64(conn->oic_last_attempt,
 					    imp_conn->oic_last_attempt))
 			imp_conn = conn;
 	}
@@ -508,7 +509,7 @@ static int import_select_connection(struct obd_import *imp)
 		       imp->imp_obd->obd_name, at_get(at));
 	}
 
-	imp_conn->oic_last_attempt = cfs_time_current_64();
+	imp_conn->oic_last_attempt = get_jiffies_64();
 
 	/* switch connection, don't mind if it's same as the current one */
 	ptlrpc_connection_put(imp->imp_connection);
@@ -1486,7 +1487,7 @@ int ptlrpc_disconnect_import(struct obd_import *imp, int noclose)
 	}
 
 	if (ptlrpc_import_in_recovery(imp)) {
-		long timeout;
+		unsigned long timeout;
 
 		if (AT_OFF) {
 			if (imp->imp_server_timeout)
@@ -1501,7 +1502,7 @@ int ptlrpc_disconnect_import(struct obd_import *imp, int noclose)
 
 		if (wait_event_idle_timeout(imp->imp_recovery_waitq,
 					    !ptlrpc_import_in_recovery(imp),
-					    cfs_timeout_cap(timeout)) == 0)
+					    max(timeout, 1UL)) == 0)
 			l_wait_event_abortable(
 				imp->imp_recovery_waitq,
 				!ptlrpc_import_in_recovery(imp));
diff --git a/drivers/staging/lustre/lustre/ptlrpc/llog_client.c b/drivers/staging/lustre/lustre/ptlrpc/llog_client.c
index 254488be7093..946d538121de 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/llog_client.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/llog_client.c
@@ -39,8 +39,6 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
-#include <linux/libcfs/libcfs.h>
-
 #include <obd_class.h>
 #include <lustre_log.h>
 #include <lustre_net.h>
diff --git a/drivers/staging/lustre/lustre/ptlrpc/llog_net.c b/drivers/staging/lustre/lustre/ptlrpc/llog_net.c
index bc4398b9bd1d..b871d9e40a9e 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/llog_net.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/llog_net.c
@@ -43,8 +43,6 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
-#include <linux/libcfs/libcfs.h>
-
 #include <obd_class.h>
 #include <lustre_log.h>
 #include <linux/list.h>
diff --git a/drivers/staging/lustre/lustre/ptlrpc/niobuf.c b/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
index 86883abaad2c..2897afb8806c 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
@@ -32,6 +32,7 @@
  */
 
 #define DEBUG_SUBSYSTEM S_RPC
+#include <linux/sched/mm.h>
 #include <obd_support.h>
 #include <lustre_net.h>
 #include <lustre_lib.h>
@@ -472,7 +473,7 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 {
 	int rc;
 	int rc2;
-	int mpflag = 0;
+	unsigned int mpflag = 0;
 	struct ptlrpc_connection *connection;
 	struct lnet_handle_me reply_me_h;
 	struct lnet_md reply_md;
@@ -558,7 +559,7 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 		lustre_msg_add_flags(request->rq_reqmsg, MSG_RESENT);
 
 	if (request->rq_memalloc)
-		mpflag = cfs_memory_pressure_get_and_set();
+		mpflag = memalloc_noreclaim_save();
 
 	rc = sptlrpc_cli_wrap_request(request);
 	if (rc) {
@@ -710,7 +711,7 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 	ptlrpc_unregister_bulk(request, 0);
  out:
 	if (request->rq_memalloc)
-		cfs_memory_pressure_restore(mpflag);
+		memalloc_noreclaim_restore(mpflag);
 	return rc;
 }
 EXPORT_SYMBOL(ptl_send_rpc);
diff --git a/drivers/staging/lustre/lustre/ptlrpc/nrs.c b/drivers/staging/lustre/lustre/ptlrpc/nrs.c
index 4847f9a90cc9..e09b86529c5d 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/nrs.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/nrs.c
@@ -37,11 +37,12 @@
  */
 
 #define DEBUG_SUBSYSTEM S_RPC
+#include <linux/libcfs/libcfs.h>
+#include <linux/libcfs/libcfs_cpu.h>
 #include <obd_support.h>
 #include <obd_class.h>
 #include <lustre_net.h>
 #include <lprocfs_status.h>
-#include <linux/libcfs/libcfs.h>
 #include "ptlrpc_internal.h"
 
 /**
diff --git a/drivers/staging/lustre/lustre/ptlrpc/nrs_fifo.c b/drivers/staging/lustre/lustre/ptlrpc/nrs_fifo.c
index 8251cbf2ad68..ff630d94dd26 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/nrs_fifo.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/nrs_fifo.c
@@ -44,9 +44,10 @@
  */
 
 #define DEBUG_SUBSYSTEM S_RPC
+#include <linux/libcfs/libcfs.h>
+#include <linux/libcfs/libcfs_cpu.h>
 #include <obd_support.h>
 #include <obd_class.h>
-#include <linux/libcfs/libcfs.h>
 #include "ptlrpc_internal.h"
 
 /**
diff --git a/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c b/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
index f73463ac401f..6ac9bb570663 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
@@ -41,8 +41,6 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
-#include <linux/libcfs/libcfs.h>
-
 #include <uapi/linux/lustre/lustre_fiemap.h>
 
 #include <llog_swab.h>
diff --git a/drivers/staging/lustre/lustre/ptlrpc/pinger.c b/drivers/staging/lustre/lustre/ptlrpc/pinger.c
index 0775b7a048bb..b3297b5ce395 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/pinger.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/pinger.c
@@ -108,7 +108,7 @@ static void ptlrpc_update_next_ping(struct obd_import *imp, int soon)
 				  at_get(&imp->imp_at.iat_net_latency));
 		time = min(time, dtime);
 	}
-	imp->imp_next_ping = cfs_time_shift(time);
+	imp->imp_next_ping = jiffies + time * HZ;
 }
 
 static inline int imp_is_deactive(struct obd_import *imp)
@@ -120,9 +120,9 @@ static inline int imp_is_deactive(struct obd_import *imp)
 static inline int ptlrpc_next_reconnect(struct obd_import *imp)
 {
 	if (imp->imp_server_timeout)
-		return cfs_time_shift(obd_timeout / 2);
+		return jiffies + obd_timeout / 2 * HZ;
 	else
-		return cfs_time_shift(obd_timeout);
+		return jiffies + obd_timeout * HZ;
 }
 
 static long pinger_check_timeout(unsigned long time)
@@ -141,8 +141,7 @@ static long pinger_check_timeout(unsigned long time)
 	}
 	mutex_unlock(&pinger_mutex);
 
-	return cfs_time_sub(cfs_time_add(time, timeout * HZ),
-					 cfs_time_current());
+	return time + timeout * HZ - jiffies;
 }
 
 static bool ir_up;
@@ -181,7 +180,7 @@ static void ptlrpc_pinger_process_import(struct obd_import *imp,
 
 	imp->imp_force_verify = 0;
 
-	if (cfs_time_aftereq(imp->imp_next_ping - 5 * CFS_TICK, this_ping) &&
+	if (time_after_eq(imp->imp_next_ping - 5, this_ping) &&
 	    !force) {
 		spin_unlock(&imp->imp_lock);
 		return;
@@ -223,7 +222,7 @@ static DECLARE_DELAYED_WORK(ping_work, ptlrpc_pinger_main);
 
 static void ptlrpc_pinger_main(struct work_struct *ws)
 {
-	unsigned long this_ping = cfs_time_current();
+	unsigned long this_ping = jiffies;
 	long time_to_next_wake;
 	struct timeout_item *item;
 	struct obd_import *imp;
@@ -237,9 +236,8 @@ static void ptlrpc_pinger_main(struct work_struct *ws)
 			ptlrpc_pinger_process_import(imp, this_ping);
 			/* obd_timeout might have changed */
 			if (imp->imp_pingable && imp->imp_next_ping &&
-			    cfs_time_after(imp->imp_next_ping,
-					   cfs_time_add(this_ping,
-							PING_INTERVAL * HZ)))
+			    time_after(imp->imp_next_ping,
+				       this_ping + PING_INTERVAL * HZ))
 				ptlrpc_update_next_ping(imp, 0);
 		}
 		mutex_unlock(&pinger_mutex);
@@ -253,10 +251,9 @@ static void ptlrpc_pinger_main(struct work_struct *ws)
 		 * we will SKIP the next ping at next_ping, and the
 		 * ping will get sent 2 timeouts from now!  Beware.
 		 */
-		CDEBUG(D_INFO, "next wakeup in " CFS_DURATION_T " (%ld)\n",
+		CDEBUG(D_INFO, "next wakeup in %ld (%ld)\n",
 		       time_to_next_wake,
-		       cfs_time_add(this_ping,
-				    PING_INTERVAL * HZ));
+		       this_ping + PING_INTERVAL * HZ);
 	} while (time_to_next_wake <= 0);
 
 	queue_delayed_work(pinger_wq, &ping_work,
diff --git a/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_module.c b/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_module.c
index 38923418669f..5c32b657b3b5 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_module.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_module.c
@@ -94,6 +94,10 @@ static int __init ptlrpc_init(void)
 	mutex_init(&ptlrpcd_mutex);
 	ptlrpc_init_xid();
 
+	rc = libcfs_setup();
+	if (rc)
+		return rc;
+
 	rc = req_layout_init();
 	if (rc)
 		return rc;
diff --git a/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c b/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c
index c0fa13942bd8..531005411edf 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c
@@ -52,8 +52,10 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
+#include <linux/kthread.h>
 #include <linux/libcfs/libcfs.h>
-
+#include <linux/libcfs/libcfs_cpu.h>
+#include <linux/libcfs/libcfs_string.h>
 #include <lustre_net.h>
 #include <lustre_lib.h>
 #include <lustre_ha.h>
@@ -175,7 +177,7 @@ ptlrpcd_select_pc(struct ptlrpc_request *req)
 	if (req && req->rq_send_state != LUSTRE_IMP_FULL)
 		return &ptlrpcd_rcv;
 
-	cpt = cfs_cpt_current(cfs_cpt_table, 1);
+	cpt = cfs_cpt_current(cfs_cpt_tab, 1);
 	if (!ptlrpcds_cpt_idx)
 		idx = cpt;
 	else
@@ -387,7 +389,7 @@ static int ptlrpcd(void *arg)
 	int exit = 0;
 
 	unshare_fs_struct();
-	if (cfs_cpt_bind(cfs_cpt_table, pc->pc_cpt) != 0)
+	if (cfs_cpt_bind(cfs_cpt_tab, pc->pc_cpt) != 0)
 		CWARN("Failed to bind %s on CPT %d\n", pc->pc_name, pc->pc_cpt);
 
 	/*
@@ -529,7 +531,7 @@ static int ptlrpcd_partners(struct ptlrpcd *pd, int index)
 
 	size = sizeof(struct ptlrpcd_ctl *) * pc->pc_npartners;
 	pc->pc_partners = kzalloc_node(size, GFP_NOFS,
-				       cfs_cpt_spread_node(cfs_cpt_table,
+				       cfs_cpt_spread_node(cfs_cpt_tab,
 							   pc->pc_cpt));
 	if (!pc->pc_partners) {
 		pc->pc_npartners = 0;
@@ -675,7 +677,7 @@ static int ptlrpcd_init(void)
 	/*
 	 * Determine the CPTs that ptlrpcd threads will run on.
 	 */
-	cptable = cfs_cpt_table;
+	cptable = cfs_cpt_tab;
 	ncpts = cfs_cpt_number(cptable);
 	if (ptlrpcd_cpts) {
 		struct cfs_expr_list *el;
@@ -829,7 +831,7 @@ static int ptlrpcd_init(void)
 
 		size = offsetof(struct ptlrpcd, pd_threads[nthreads]);
 		pd = kzalloc_node(size, GFP_NOFS,
-				  cfs_cpt_spread_node(cfs_cpt_table, cpt));
+				  cfs_cpt_spread_node(cfs_cpt_tab, cpt));
 		if (!pd) {
 			rc = -ENOMEM;
 			goto out;
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec.c b/drivers/staging/lustre/lustre/ptlrpc/sec.c
index 3cb1e075f077..256421465bcd 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec.c
@@ -2236,7 +2236,7 @@ int sptlrpc_pack_user_desc(struct lustre_msg *msg, int offset)
 	pud->pud_gid = from_kgid(&init_user_ns, current_gid());
 	pud->pud_fsuid = from_kuid(&init_user_ns, current_fsuid());
 	pud->pud_fsgid = from_kgid(&init_user_ns, current_fsgid());
-	pud->pud_cap = cfs_curproc_cap_pack();
+	pud->pud_cap = current_cap().cap[0];
 	pud->pud_ngroups = (msg->lm_buflens[offset] - sizeof(*pud)) / 4;
 
 	task_lock(current);
diff --git a/drivers/staging/lustre/lustre/ptlrpc/service.c b/drivers/staging/lustre/lustre/ptlrpc/service.c
index f37364e00dfe..3fd8c746f460 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/service.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/service.c
@@ -33,12 +33,15 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
+#include <linux/kthread.h>
 #include <obd_support.h>
 #include <obd_class.h>
 #include <lustre_net.h>
 #include <lu_object.h>
 #include <uapi/linux/lnet/lnet-types.h>
 #include "ptlrpc_internal.h"
+#include <linux/libcfs/libcfs_cpu.h>
+#include <linux/libcfs/libcfs_string.h>
 
 /* The following are visible and mutable through /sys/module/ptlrpc */
 int test_req_buffer_pressure;
@@ -205,7 +208,7 @@ struct ptlrpc_hr_partition {
 #define HRT_STOPPING 1
 
 struct ptlrpc_hr_service {
-	/* CPU partition table, it's just cfs_cpt_table for now */
+	/* CPU partition table, it's just cfs_cpt_tab for now */
 	struct cfs_cpt_table		*hr_cpt_table;
 	/** controller sleep waitq */
 	wait_queue_head_t			hr_waitq;
@@ -336,7 +339,7 @@ static void ptlrpc_at_timer(struct timer_list *t)
 	svcpt = from_timer(svcpt, t, scp_at_timer);
 
 	svcpt->scp_at_check = 1;
-	svcpt->scp_at_checktime = cfs_time_current();
+	svcpt->scp_at_checktime = jiffies;
 	wake_up(&svcpt->scp_waitq);
 }
 
@@ -562,7 +565,7 @@ ptlrpc_register_service(struct ptlrpc_service_conf *conf,
 
 	cptable = cconf->cc_cptable;
 	if (!cptable)
-		cptable = cfs_cpt_table;
+		cptable = cfs_cpt_tab;
 
 	if (!conf->psc_thr.tc_cpu_affinity) {
 		ncpts = 1;
@@ -922,7 +925,7 @@ static void ptlrpc_at_set_timer(struct ptlrpc_service_part *svcpt)
 	if (next <= 0) {
 		ptlrpc_at_timer(&svcpt->scp_at_timer);
 	} else {
-		mod_timer(&svcpt->scp_at_timer, cfs_time_shift(next));
+		mod_timer(&svcpt->scp_at_timer, jiffies + next * HZ);
 		CDEBUG(D_INFO, "armed %s at %+ds\n",
 		       svcpt->scp_service->srv_name, next);
 	}
@@ -1153,7 +1156,7 @@ static void ptlrpc_at_check_timed(struct ptlrpc_service_part *svcpt)
 		spin_unlock(&svcpt->scp_at_lock);
 		return;
 	}
-	delay = cfs_time_sub(cfs_time_current(), svcpt->scp_at_checktime);
+	delay = jiffies - svcpt->scp_at_checktime;
 	svcpt->scp_at_check = 0;
 
 	if (array->paa_count == 0) {
@@ -1679,7 +1682,7 @@ ptlrpc_server_handle_request(struct ptlrpc_service_part *svcpt,
 	}
 
 	CDEBUG(D_RPCTRACE, "Handling RPC pname:cluuid+ref:pid:xid:nid:opc %s:%s+%d:%d:x%llu:%s:%d\n",
-	       current_comm(),
+	       current->comm,
 	       (request->rq_export ?
 		(char *)request->rq_export->exp_client_uuid.uuid : "0"),
 	       (request->rq_export ?
@@ -1723,7 +1726,7 @@ ptlrpc_server_handle_request(struct ptlrpc_service_part *svcpt,
 	arrived_usecs = arrived.tv_sec * USEC_PER_SEC +
 			 arrived.tv_nsec / NSEC_PER_USEC;
 	CDEBUG(D_RPCTRACE, "Handled RPC pname:cluuid+ref:pid:xid:nid:opc %s:%s+%d:%d:x%llu:%s:%d Request processed in %ldus (%ldus total) trans %llu rc %d/%d\n",
-	       current_comm(),
+	       current->comm,
 	       (request->rq_export ?
 		(char *)request->rq_export->exp_client_uuid.uuid : "0"),
 	       (request->rq_export ?
@@ -2018,7 +2021,7 @@ static int ptlrpc_main(void *arg)
 	struct lu_env *env;
 	int counter = 0, rc = 0;
 
-	thread->t_pid = current_pid();
+	thread->t_pid = current->pid;
 	unshare_fs_struct();
 
 	/* NB: we will call cfs_cpt_bind() for all threads, because we
@@ -2516,7 +2519,7 @@ int ptlrpc_hr_init(void)
 	int weight;
 
 	memset(&ptlrpc_hr, 0, sizeof(ptlrpc_hr));
-	ptlrpc_hr.hr_cpt_table = cfs_cpt_table;
+	ptlrpc_hr.hr_cpt_table = cfs_cpt_tab;
 
 	ptlrpc_hr.hr_partitions = cfs_percpt_alloc(ptlrpc_hr.hr_cpt_table,
 						   sizeof(*hrp));
diff --git a/drivers/staging/media/Kconfig b/drivers/staging/media/Kconfig
index 4c495a10025c..db5cf67047ad 100644
--- a/drivers/staging/media/Kconfig
+++ b/drivers/staging/media/Kconfig
@@ -19,8 +19,6 @@ menuconfig STAGING_MEDIA
 if STAGING_MEDIA && MEDIA_SUPPORT
 
 # Please keep them in alphabetic order
-source "drivers/staging/media/atomisp/Kconfig"
-
 source "drivers/staging/media/bcm2048/Kconfig"
 
 source "drivers/staging/media/davinci_vpfe/Kconfig"
@@ -35,4 +33,6 @@ source "drivers/staging/media/omap4iss/Kconfig"
 
 source "drivers/staging/media/tegra-vde/Kconfig"
 
+source "drivers/staging/media/zoran/Kconfig"
+
 endif
diff --git a/drivers/staging/media/Makefile b/drivers/staging/media/Makefile
index 61a5765cb98f..503fbe47fa58 100644
--- a/drivers/staging/media/Makefile
+++ b/drivers/staging/media/Makefile
@@ -5,5 +5,5 @@ obj-$(CONFIG_SOC_CAMERA_IMX074)	+= imx074/
 obj-$(CONFIG_SOC_CAMERA_MT9T031)	+= mt9t031/
 obj-$(CONFIG_VIDEO_DM365_VPFE)	+= davinci_vpfe/
 obj-$(CONFIG_VIDEO_OMAP4)	+= omap4iss/
-obj-$(CONFIG_INTEL_ATOMISP)     += atomisp/
 obj-$(CONFIG_TEGRA_VDE)		+= tegra-vde/
+obj-$(CONFIG_VIDEO_ZORAN)	+= zoran/
diff --git a/drivers/staging/media/atomisp/Kconfig b/drivers/staging/media/atomisp/Kconfig
deleted file mode 100644
index 27f078749148..000000000000
--- a/drivers/staging/media/atomisp/Kconfig
+++ /dev/null
@@ -1,12 +0,0 @@
-menuconfig INTEL_ATOMISP
-	bool "Enable support to Intel MIPI camera drivers"
-	depends on X86 && EFI && MEDIA_CONTROLLER && PCI && ACPI
-	select COMMON_CLK
-	help
-	  Enable support for the Intel ISP2 camera interfaces and MIPI
-	  sensor drivers.
-
-if INTEL_ATOMISP
-source "drivers/staging/media/atomisp/pci/Kconfig"
-source "drivers/staging/media/atomisp/i2c/Kconfig"
-endif
diff --git a/drivers/staging/media/atomisp/Makefile b/drivers/staging/media/atomisp/Makefile
deleted file mode 100644
index 403fe5edff6d..000000000000
--- a/drivers/staging/media/atomisp/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-#
-# Makefile for camera drivers.
-#
-obj-$(CONFIG_INTEL_ATOMISP) += pci/
-obj-$(CONFIG_INTEL_ATOMISP) += i2c/
-obj-$(CONFIG_INTEL_ATOMISP) += platform/
diff --git a/drivers/staging/media/atomisp/TODO b/drivers/staging/media/atomisp/TODO
deleted file mode 100644
index 255ce3630c2a..000000000000
--- a/drivers/staging/media/atomisp/TODO
+++ /dev/null
@@ -1,74 +0,0 @@
-1. A single AtomISP driver needs to be implemented to support both BYT and
-   CHT platforms. The current driver is a mechanical and hand combined merge
-   of the two using an ifdef ISP2401 to select the CHT version, which at the
-   moment is not enabled. Eventually this should become a runtime if check,
-   but there are some quite tricky things that need sorting out before that
-   will be possible.
-
-2. The file structure needs to get tidied up to resemble a normal Linux
-   driver.
-
-3. Lots of the midlayer glue. unused code and abstraction needs removing.
-
-3. The sensor drivers read MIPI settings from EFI variables or default to the
-   settings hard-coded in the platform data file for different platforms.
-   This isn't ideal but may be hard to improve as this is how existing
-   platforms work.
-
-4. The sensor drivers use the regulator framework API. In the ideal world it
-   would be using ACPI but that's not how the existing devices work.
-
-5. The AtomISP driver includes some special IOCTLS (ATOMISP_IOC_XXXX_XXXX)
-   that may need some cleaning up.
-
-6. Correct Coding Style. Please don't send coding style patches for this
-   driver until the other work is done.
-
-7. The ISP code depends on the exact FW version. The version defined in
-   BYT: 
-   drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.c
-   static const char *release_version = STR(irci_stable_candrpv_0415_20150521_0458);
-   CHT:
-   drivers/staging/media/atomisp/pci/atomisp2/css/sh_css_firmware.c
-   static const char *release_version = STR(irci_ecr-master_20150911_0724);
-
-   At some point we may need to round up a few driver versions and see if
-   there are any specific things that can be done to fold in support for
-   multiple firmware versions.
-
-8. Switch to V4L2 async API to set up sensor, lens and flash devices.
-   Control those devices using V4L2 sub-device API without custom
-   extensions.
-
-9. Switch to standard V4L2 sub-device API for sensor and lens. In
-   particular, the user space API needs to support V4L2 controls as
-   defined in the V4L2 spec and references to atomisp must be removed from
-   these drivers.
-
-10. Use LED flash API for flash LED drivers such as LM3554 (which already
-    has a LED class driver).
-
-11. Switch from videobuf1 to videobuf2. Videobuf1 is being removed!
-
-Limitations:
-
-1. To test the patches, you also need the ISP firmware
-
-   for BYT:/lib/firmware/shisp_2400b0_v21.bin
-   for CHT:/lib/firmware/shisp_2401a0_v21.bin
-
-   The firmware files will usually be found in /etc/firmware on an Android
-   device but can also be extracted from the upgrade kit if you've managed
-   to lose them somehow.
-
-2. Without a 3A libary the capture behaviour is not very good. To take a good
-   picture, you need tune ISP parameters by IOCTL functions or use a 3A libary
-   such as libxcam.
-
-3. The driver is intended to drive the PCI exposed versions of the device.
-   It will not detect those devices enumerated via ACPI as a field of the
-   i915 GPU driver.
-
-4. The driver supports only v2 of the IPU/Camera. It will not work with the
-   versions of the hardware in other SoCs.
-
diff --git a/drivers/staging/media/atomisp/i2c/Kconfig b/drivers/staging/media/atomisp/i2c/Kconfig
deleted file mode 100644
index f7f7177b9b37..000000000000
--- a/drivers/staging/media/atomisp/i2c/Kconfig
+++ /dev/null
@@ -1,86 +0,0 @@
-#
-# Kconfig for sensor drivers
-#
-
-source "drivers/staging/media/atomisp/i2c/ov5693/Kconfig"
-
-config VIDEO_ATOMISP_OV2722
-       tristate "OVT ov2722 sensor support"
-	depends on ACPI
-       depends on I2C && VIDEO_V4L2
-       ---help---
-	 This is a Video4Linux2 sensor-level driver for the OVT
-	 OV2722 raw camera.
-
-	 OVT is a 2M raw sensor.
-
-	 It currently only works with the atomisp driver.
-
-config VIDEO_ATOMISP_GC2235
-       tristate "Galaxy gc2235 sensor support"
-	depends on ACPI
-       depends on I2C && VIDEO_V4L2
-       ---help---
-	 This is a Video4Linux2 sensor-level driver for the OVT
-	 GC2235 raw camera.
-
-	 GC2235 is a 2M raw sensor.
-
-	 It currently only works with the atomisp driver.
-
-config VIDEO_ATOMISP_MSRLIST_HELPER
-       tristate "Helper library to load, parse and apply large register lists."
-       depends on I2C
-       ---help---
-	 This is a helper library to be used from a sensor driver to load, parse
-	 and apply large register lists.
-
-	 To compile this driver as a module, choose M here: the
-	 module will be called libmsrlisthelper.
-
-config VIDEO_ATOMISP_MT9M114
-       tristate "Aptina mt9m114 sensor support"
-	depends on ACPI
-       depends on I2C && VIDEO_V4L2
-       ---help---
-	 This is a Video4Linux2 sensor-level driver for the Micron
-	 mt9m114 1.3 Mpixel camera.
-
-	 mt9m114 is video camera sensor.
-
-	 It currently only works with the atomisp driver.
-
-config VIDEO_ATOMISP_GC0310
-	tristate "GC0310 sensor support"
-	depends on ACPI
-	depends on I2C && VIDEO_V4L2
-	---help---
-	  This is a Video4Linux2 sensor-level driver for the Galaxycore
-	  GC0310 0.3MP sensor.
-	 
-config VIDEO_ATOMISP_OV2680
-       tristate "Omnivision OV2680 sensor support"
-	depends on ACPI
-       depends on I2C && VIDEO_V4L2
-       ---help---
-	 This is a Video4Linux2 sensor-level driver for the Omnivision
-	 OV2680 raw camera.
-
-	 ov2680 is a 2M raw sensor.
-
-	 It currently only works with the atomisp driver.
-
-#
-# Kconfig for flash drivers
-#
-
-config VIDEO_ATOMISP_LM3554
-       tristate "LM3554 flash light driver"
-	depends on ACPI
-       depends on VIDEO_V4L2 && I2C
-       ---help---
-	 This is a Video4Linux2 sub-dev driver for the LM3554
-	 flash light driver.
-
-	 To compile this driver as a module, choose M here: the
-	 module will be called lm3554
diff --git a/drivers/staging/media/atomisp/i2c/Makefile b/drivers/staging/media/atomisp/i2c/Makefile
deleted file mode 100644
index 99ea35c043fd..000000000000
--- a/drivers/staging/media/atomisp/i2c/Makefile
+++ /dev/null
@@ -1,25 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# Makefile for sensor drivers
-#
-
-obj-$(CONFIG_VIDEO_ATOMISP_OV5693)     += ov5693/
-obj-$(CONFIG_VIDEO_ATOMISP_MT9M114)    += atomisp-mt9m114.o
-obj-$(CONFIG_VIDEO_ATOMISP_GC2235)     += atomisp-gc2235.o
-obj-$(CONFIG_VIDEO_ATOMISP_OV2722)     += atomisp-ov2722.o
-obj-$(CONFIG_VIDEO_ATOMISP_OV2680)     += atomisp-ov2680.o
-obj-$(CONFIG_VIDEO_ATOMISP_GC0310)     += atomisp-gc0310.o
-
-obj-$(CONFIG_VIDEO_ATOMISP_MSRLIST_HELPER) += atomisp-libmsrlisthelper.o
-
-# Makefile for flash drivers
-#
-
-obj-$(CONFIG_VIDEO_ATOMISP_LM3554) += atomisp-lm3554.o
-
-# HACK! While this driver is in bad shape, don't enable several warnings
-#       that would be otherwise enabled with W=1
-ccflags-y += $(call cc-disable-warning, unused-but-set-variable)
-ccflags-y += $(call cc-disable-warning, unused-const-variable)
-ccflags-y += $(call cc-disable-warning, missing-prototypes)
-ccflags-y += $(call cc-disable-warning, missing-declarations)
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
deleted file mode 100644
index 512fa87fa11b..000000000000
--- a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
+++ /dev/null
@@ -1,1392 +0,0 @@
-/*
- * Support for GalaxyCore GC0310 VGA camera sensor.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- */
-
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/string.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/kmod.h>
-#include <linux/device.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <linux/i2c.h>
-#include <linux/moduleparam.h>
-#include <media/v4l2-device.h>
-#include <linux/io.h>
-#include "../include/linux/atomisp_gmin_platform.h"
-
-#include "gc0310.h"
-
-/* i2c read/write stuff */
-static int gc0310_read_reg(struct i2c_client *client,
-			   u16 data_length, u8 reg, u8 *val)
-{
-	int err;
-	struct i2c_msg msg[2];
-	unsigned char data[1];
-
-	if (!client->adapter) {
-		dev_err(&client->dev, "%s error, no client->adapter\n",
-			__func__);
-		return -ENODEV;
-	}
-
-	if (data_length != GC0310_8BIT) {
-		dev_err(&client->dev, "%s error, invalid data length\n",
-			__func__);
-		return -EINVAL;
-	}
-
-	memset(msg, 0, sizeof(msg));
-
-	msg[0].addr = client->addr;
-	msg[0].flags = 0;
-	msg[0].len = I2C_MSG_LENGTH;
-	msg[0].buf = data;
-
-	/* high byte goes out first */
-	data[0] = (u8)(reg & 0xff);
-
-	msg[1].addr = client->addr;
-	msg[1].len = data_length;
-	msg[1].flags = I2C_M_RD;
-	msg[1].buf = data;
-
-	err = i2c_transfer(client->adapter, msg, 2);
-	if (err != 2) {
-		if (err >= 0)
-			err = -EIO;
-		dev_err(&client->dev,
-			"read from offset 0x%x error %d", reg, err);
-		return err;
-	}
-
-	*val = 0;
-	/* high byte comes first */
-	if (data_length == GC0310_8BIT)
-		*val = (u8)data[0];
-
-	return 0;
-}
-
-static int gc0310_i2c_write(struct i2c_client *client, u16 len, u8 *data)
-{
-	struct i2c_msg msg;
-	const int num_msg = 1;
-	int ret;
-
-	msg.addr = client->addr;
-	msg.flags = 0;
-	msg.len = len;
-	msg.buf = data;
-	ret = i2c_transfer(client->adapter, &msg, 1);
-
-	return ret == num_msg ? 0 : -EIO;
-}
-
-static int gc0310_write_reg(struct i2c_client *client, u16 data_length,
-							u8 reg, u8 val)
-{
-	int ret;
-	unsigned char data[2] = {0};
-	u8 *wreg = (u8 *)data;
-	const u16 len = data_length + sizeof(u8); /* 8-bit address + data */
-
-	if (data_length != GC0310_8BIT) {
-		dev_err(&client->dev,
-			"%s error, invalid data_length\n", __func__);
-		return -EINVAL;
-	}
-
-	/* high byte goes out first */
-	*wreg = (u8)(reg & 0xff);
-
-	if (data_length == GC0310_8BIT)
-		data[1] = (u8)(val);
-
-	ret = gc0310_i2c_write(client, len, data);
-	if (ret)
-		dev_err(&client->dev,
-			"write error: wrote 0x%x to offset 0x%x error %d",
-			val, reg, ret);
-
-	return ret;
-}
-
-/*
- * gc0310_write_reg_array - Initializes a list of GC0310 registers
- * @client: i2c driver client structure
- * @reglist: list of registers to be written
- *
- * This function initializes a list of registers. When consecutive addresses
- * are found in a row on the list, this function creates a buffer and sends
- * consecutive data in a single i2c_transfer().
- *
- * __gc0310_flush_reg_array, __gc0310_buf_reg_array() and
- * __gc0310_write_reg_is_consecutive() are internal functions to
- * gc0310_write_reg_array_fast() and should be not used anywhere else.
- *
- */
-
-static int __gc0310_flush_reg_array(struct i2c_client *client,
-				    struct gc0310_write_ctrl *ctrl)
-{
-	u16 size;
-
-	if (ctrl->index == 0)
-		return 0;
-
-	size = sizeof(u8) + ctrl->index; /* 8-bit address + data */
-	ctrl->buffer.addr = (u8)(ctrl->buffer.addr);
-	ctrl->index = 0;
-
-	return gc0310_i2c_write(client, size, (u8 *)&ctrl->buffer);
-}
-
-static int __gc0310_buf_reg_array(struct i2c_client *client,
-				  struct gc0310_write_ctrl *ctrl,
-				  const struct gc0310_reg *next)
-{
-	int size;
-
-	switch (next->type) {
-	case GC0310_8BIT:
-		size = 1;
-		ctrl->buffer.data[ctrl->index] = (u8)next->val;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* When first item is added, we need to store its starting address */
-	if (ctrl->index == 0)
-		ctrl->buffer.addr = next->reg;
-
-	ctrl->index += size;
-
-	/*
-	 * Buffer cannot guarantee free space for u32? Better flush it to avoid
-	 * possible lack of memory for next item.
-	 */
-	if (ctrl->index + sizeof(u8) >= GC0310_MAX_WRITE_BUF_SIZE)
-		return __gc0310_flush_reg_array(client, ctrl);
-
-	return 0;
-}
-
-static int __gc0310_write_reg_is_consecutive(struct i2c_client *client,
-					     struct gc0310_write_ctrl *ctrl,
-					     const struct gc0310_reg *next)
-{
-	if (ctrl->index == 0)
-		return 1;
-
-	return ctrl->buffer.addr + ctrl->index == next->reg;
-}
-
-static int gc0310_write_reg_array(struct i2c_client *client,
-				  const struct gc0310_reg *reglist)
-{
-	const struct gc0310_reg *next = reglist;
-	struct gc0310_write_ctrl ctrl;
-	int err;
-
-	ctrl.index = 0;
-	for (; next->type != GC0310_TOK_TERM; next++) {
-		switch (next->type & GC0310_TOK_MASK) {
-		case GC0310_TOK_DELAY:
-			err = __gc0310_flush_reg_array(client, &ctrl);
-			if (err)
-				return err;
-			msleep(next->val);
-			break;
-		default:
-			/*
-			 * If next address is not consecutive, data needs to be
-			 * flushed before proceed.
-			 */
-			if (!__gc0310_write_reg_is_consecutive(client, &ctrl,
-								next)) {
-				err = __gc0310_flush_reg_array(client, &ctrl);
-				if (err)
-					return err;
-			}
-			err = __gc0310_buf_reg_array(client, &ctrl, next);
-			if (err) {
-				dev_err(&client->dev, "%s: write error, aborted\n",
-					 __func__);
-				return err;
-			}
-			break;
-		}
-	}
-
-	return __gc0310_flush_reg_array(client, &ctrl);
-}
-static int gc0310_g_focal(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = (GC0310_FOCAL_LENGTH_NUM << 16) | GC0310_FOCAL_LENGTH_DEM;
-	return 0;
-}
-
-static int gc0310_g_fnumber(struct v4l2_subdev *sd, s32 *val)
-{
-	/*const f number for imx*/
-	*val = (GC0310_F_NUMBER_DEFAULT_NUM << 16) | GC0310_F_NUMBER_DEM;
-	return 0;
-}
-
-static int gc0310_g_fnumber_range(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = (GC0310_F_NUMBER_DEFAULT_NUM << 24) |
-		(GC0310_F_NUMBER_DEM << 16) |
-		(GC0310_F_NUMBER_DEFAULT_NUM << 8) | GC0310_F_NUMBER_DEM;
-	return 0;
-}
-
-static int gc0310_g_bin_factor_x(struct v4l2_subdev *sd, s32 *val)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
-	*val = gc0310_res[dev->fmt_idx].bin_factor_x;
-
-	return 0;
-}
-
-static int gc0310_g_bin_factor_y(struct v4l2_subdev *sd, s32 *val)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
-	*val = gc0310_res[dev->fmt_idx].bin_factor_y;
-
-	return 0;
-}
-
-static int gc0310_get_intg_factor(struct i2c_client *client,
-				struct camera_mipi_info *info,
-				const struct gc0310_resolution *res)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct atomisp_sensor_mode_data *buf = &info->data;
-	u16 val;
-	u8 reg_val;
-	int ret;
-	unsigned int hori_blanking;
-	unsigned int vert_blanking;
-	unsigned int sh_delay;
-
-	if (!info)
-		return -EINVAL;
-
-	/* pixel clock calculattion */
-	dev->vt_pix_clk_freq_mhz = 14400000; // 16.8MHz
-	buf->vt_pix_clk_freq_mhz = dev->vt_pix_clk_freq_mhz;
-	pr_info("vt_pix_clk_freq_mhz=%d\n", buf->vt_pix_clk_freq_mhz);
-
-	/* get integration time */
-	buf->coarse_integration_time_min = GC0310_COARSE_INTG_TIME_MIN;
-	buf->coarse_integration_time_max_margin =
-					GC0310_COARSE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_min = GC0310_FINE_INTG_TIME_MIN;
-	buf->fine_integration_time_max_margin =
-					GC0310_FINE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_def = GC0310_FINE_INTG_TIME_MIN;
-	buf->read_mode = res->bin_mode;
-
-	/* get the cropping and output resolution to ISP for this mode. */
-	/* Getting crop_horizontal_start */
-	ret =  gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_H_CROP_START_H, &reg_val);
-	if (ret)
-		return ret;
-	val = (reg_val & 0xFF) << 8;
-	ret =  gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_H_CROP_START_L, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_horizontal_start = val | (reg_val & 0xFF);
-	pr_info("crop_horizontal_start=%d\n", buf->crop_horizontal_start);
-
-	/* Getting crop_vertical_start */
-	ret =  gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_V_CROP_START_H, &reg_val);
-	if (ret)
-		return ret;
-	val = (reg_val & 0xFF) << 8;
-	ret =  gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_V_CROP_START_L, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_vertical_start = val | (reg_val & 0xFF);
-	pr_info("crop_vertical_start=%d\n", buf->crop_vertical_start);
-
-	/* Getting output_width */
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_H_OUTSIZE_H, &reg_val);
-	if (ret)
-		return ret;
-	val = (reg_val & 0xFF) << 8;
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_H_OUTSIZE_L, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_width = val | (reg_val & 0xFF);
-	pr_info("output_width=%d\n", buf->output_width);
-
-	/* Getting output_height */
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_V_OUTSIZE_H, &reg_val);
-	if (ret)
-		return ret;
-	val = (reg_val & 0xFF) << 8;
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_V_OUTSIZE_L, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_height = val | (reg_val & 0xFF);
-	pr_info("output_height=%d\n", buf->output_height);
-
-	buf->crop_horizontal_end = buf->crop_horizontal_start + buf->output_width - 1;
-	buf->crop_vertical_end = buf->crop_vertical_start + buf->output_height - 1;
-	pr_info("crop_horizontal_end=%d\n", buf->crop_horizontal_end);
-	pr_info("crop_vertical_end=%d\n", buf->crop_vertical_end);
-
-	/* Getting line_length_pck */
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_H_BLANKING_H, &reg_val);
-	if (ret)
-		return ret;
-	val = (reg_val & 0xFF) << 8;
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_H_BLANKING_L, &reg_val);
-	if (ret)
-		return ret;
-	hori_blanking = val | (reg_val & 0xFF);
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_SH_DELAY, &reg_val);
-	if (ret)
-		return ret;
-	sh_delay = reg_val;
-	buf->line_length_pck = buf->output_width + hori_blanking + sh_delay + 4;
-	pr_info("hori_blanking=%d sh_delay=%d line_length_pck=%d\n", hori_blanking, sh_delay, buf->line_length_pck);
-
-	/* Getting frame_length_lines */
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_V_BLANKING_H, &reg_val);
-	if (ret)
-		return ret;
-	val = (reg_val & 0xFF) << 8;
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_V_BLANKING_L, &reg_val);
-	if (ret)
-		return ret;
-	vert_blanking = val | (reg_val & 0xFF);
-	buf->frame_length_lines = buf->output_height + vert_blanking;
-	pr_info("vert_blanking=%d frame_length_lines=%d\n", vert_blanking, buf->frame_length_lines);
-
-	buf->binning_factor_x = res->bin_factor_x ?
-					res->bin_factor_x : 1;
-	buf->binning_factor_y = res->bin_factor_y ?
-					res->bin_factor_y : 1;
-	return 0;
-}
-
-static int gc0310_set_gain(struct v4l2_subdev *sd, int gain)
-
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-	u8 again, dgain;
-
-	if (gain < 0x20)
-		gain = 0x20;
-	if (gain > 0x80)
-		gain = 0x80;
-
-	if (gain >= 0x20 && gain < 0x40) {
-		again = 0x0; /* sqrt(2) */
-		dgain = gain;
-	} else {
-		again = 0x2; /* 2 * sqrt(2) */
-		dgain = gain / 2;
-	}
-
-	pr_info("gain=0x%x again=0x%x dgain=0x%x\n", gain, again, dgain);
-
-	/* set analog gain */
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-					GC0310_AGC_ADJ, again);
-	if (ret)
-		return ret;
-
-	/* set digital gain */
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-					GC0310_DGC_ADJ, dgain);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int __gc0310_set_exposure(struct v4l2_subdev *sd, int coarse_itg,
-				 int gain, int digitgain)
-
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	pr_info("coarse_itg=%d gain=%d digitgain=%d\n", coarse_itg, gain, digitgain);
-
-	/* set exposure */
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-					GC0310_AEC_PK_EXPO_L,
-					coarse_itg & 0xff);
-	if (ret)
-		return ret;
-
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-					GC0310_AEC_PK_EXPO_H,
-					(coarse_itg >> 8) & 0x0f);
-	if (ret)
-		return ret;
-
-	ret = gc0310_set_gain(sd, gain);
-	if (ret)
-		return ret;
-
-	return ret;
-}
-
-static int gc0310_set_exposure(struct v4l2_subdev *sd, int exposure,
-	int gain, int digitgain)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	int ret;
-
-	mutex_lock(&dev->input_lock);
-	ret = __gc0310_set_exposure(sd, exposure, gain, digitgain);
-	mutex_unlock(&dev->input_lock);
-
-	return ret;
-}
-
-static long gc0310_s_exposure(struct v4l2_subdev *sd,
-			       struct atomisp_exposure *exposure)
-{
-	int exp = exposure->integration_time[0];
-	int gain = exposure->gain[0];
-	int digitgain = exposure->gain[1];
-
-	/* we should not accept the invalid value below. */
-	if (gain == 0) {
-		struct i2c_client *client = v4l2_get_subdevdata(sd);
-		v4l2_err(client, "%s: invalid value\n", __func__);
-		return -EINVAL;
-	}
-
-	return gc0310_set_exposure(sd, exp, gain, digitgain);
-}
-
-/* TO DO */
-static int gc0310_v_flip(struct v4l2_subdev *sd, s32 value)
-{
-	return 0;
-}
-
-/* TO DO */
-static int gc0310_h_flip(struct v4l2_subdev *sd, s32 value)
-{
-	return 0;
-}
-
-static long gc0310_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
-{
-
-	switch (cmd) {
-	case ATOMISP_IOC_S_EXPOSURE:
-		return gc0310_s_exposure(sd, arg);
-	default:
-		return -EINVAL;
-	}
-	return 0;
-}
-
-/* This returns the exposure time being used. This should only be used
- * for filling in EXIF data, not for actual image processing.
- */
-static int gc0310_q_exposure(struct v4l2_subdev *sd, s32 *value)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u8 reg_v;
-	int ret;
-
-	/* get exposure */
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_AEC_PK_EXPO_L,
-					&reg_v);
-	if (ret)
-		goto err;
-
-	*value = reg_v;
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_AEC_PK_EXPO_H,
-					&reg_v);
-	if (ret)
-		goto err;
-
-	*value = *value + (reg_v << 8);
-err:
-	return ret;
-}
-
-static int gc0310_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct gc0310_device *dev =
-	    container_of(ctrl->handler, struct gc0310_device, ctrl_handler);
-	struct i2c_client *client = v4l2_get_subdevdata(&dev->sd);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_VFLIP:
-		dev_dbg(&client->dev, "%s: CID_VFLIP:%d.\n",
-			__func__, ctrl->val);
-		ret = gc0310_v_flip(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_HFLIP:
-		dev_dbg(&client->dev, "%s: CID_HFLIP:%d.\n",
-			__func__, ctrl->val);
-		ret = gc0310_h_flip(&dev->sd, ctrl->val);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-	return ret;
-}
-
-static int gc0310_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct gc0310_device *dev =
-	    container_of(ctrl->handler, struct gc0310_device, ctrl_handler);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_EXPOSURE_ABSOLUTE:
-		ret = gc0310_q_exposure(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FOCAL_ABSOLUTE:
-		ret = gc0310_g_focal(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_ABSOLUTE:
-		ret = gc0310_g_fnumber(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_RANGE:
-		ret = gc0310_g_fnumber_range(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_BIN_FACTOR_HORZ:
-		ret = gc0310_g_bin_factor_x(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_BIN_FACTOR_VERT:
-		ret = gc0310_g_bin_factor_y(&dev->sd, &ctrl->val);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-static const struct v4l2_ctrl_ops ctrl_ops = {
-	.s_ctrl = gc0310_s_ctrl,
-	.g_volatile_ctrl = gc0310_g_volatile_ctrl
-};
-
-static const struct v4l2_ctrl_config gc0310_controls[] = {
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_EXPOSURE_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "exposure",
-	 .min = 0x0,
-	 .max = 0xffff,
-	 .step = 0x01,
-	 .def = 0x00,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_VFLIP,
-	 .type = V4L2_CTRL_TYPE_BOOLEAN,
-	 .name = "Flip",
-	 .min = 0,
-	 .max = 1,
-	 .step = 1,
-	 .def = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_HFLIP,
-	 .type = V4L2_CTRL_TYPE_BOOLEAN,
-	 .name = "Mirror",
-	 .min = 0,
-	 .max = 1,
-	 .step = 1,
-	 .def = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FOCAL_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "focal length",
-	 .min = GC0310_FOCAL_LENGTH_DEFAULT,
-	 .max = GC0310_FOCAL_LENGTH_DEFAULT,
-	 .step = 0x01,
-	 .def = GC0310_FOCAL_LENGTH_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "f-number",
-	 .min = GC0310_F_NUMBER_DEFAULT,
-	 .max = GC0310_F_NUMBER_DEFAULT,
-	 .step = 0x01,
-	 .def = GC0310_F_NUMBER_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_RANGE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "f-number range",
-	 .min = GC0310_F_NUMBER_RANGE,
-	 .max = GC0310_F_NUMBER_RANGE,
-	 .step = 0x01,
-	 .def = GC0310_F_NUMBER_RANGE,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_BIN_FACTOR_HORZ,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "horizontal binning factor",
-	 .min = 0,
-	 .max = GC0310_BIN_FACTOR_MAX,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_BIN_FACTOR_VERT,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "vertical binning factor",
-	 .min = 0,
-	 .max = GC0310_BIN_FACTOR_MAX,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-};
-
-static int gc0310_init(struct v4l2_subdev *sd)
-{
-	int ret;
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
-	pr_info("%s S\n", __func__);
-	mutex_lock(&dev->input_lock);
-
-	/* set inital registers */
-	ret  = gc0310_write_reg_array(client, gc0310_reset_register);
-
-	/* restore settings */
-	gc0310_res = gc0310_res_preview;
-	N_RES = N_RES_PREVIEW;
-
-	mutex_unlock(&dev->input_lock);
-
-	pr_info("%s E\n", __func__);
-	return 0;
-}
-
-static int power_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	int ret = 0;
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	if (flag) {
-		/* The upstream module driver (written to Crystal
-		 * Cove) had this logic to pulse the rails low first.
-		 * This appears to break things on the MRD7 with the
-		 * X-Powers PMIC...
-		 *
-		 *     ret = dev->platform_data->v1p8_ctrl(sd, 0);
-		 *     ret |= dev->platform_data->v2p8_ctrl(sd, 0);
-		 *     mdelay(50);
-		 */
-		ret |= dev->platform_data->v1p8_ctrl(sd, 1);
-		ret |= dev->platform_data->v2p8_ctrl(sd, 1);
-		usleep_range(10000, 15000);
-	}
-
-	if (!flag || ret) {
-		ret |= dev->platform_data->v1p8_ctrl(sd, 0);
-		ret |= dev->platform_data->v2p8_ctrl(sd, 0);
-	}
-	return ret;
-}
-
-static int gpio_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	int ret;
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	/* GPIO0 == "reset" (active low), GPIO1 == "power down" */
-	if (flag) {
-		/* Pulse reset, then release power down */
-		ret = dev->platform_data->gpio0_ctrl(sd, 0);
-		usleep_range(5000, 10000);
-		ret |= dev->platform_data->gpio0_ctrl(sd, 1);
-		usleep_range(10000, 15000);
-		ret |= dev->platform_data->gpio1_ctrl(sd, 0);
-		usleep_range(10000, 15000);
-	} else {
-		ret = dev->platform_data->gpio1_ctrl(sd, 1);
-		ret |= dev->platform_data->gpio0_ctrl(sd, 0);
-	}
-	return ret;
-}
-
-
-static int power_down(struct v4l2_subdev *sd);
-
-static int power_up(struct v4l2_subdev *sd)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	pr_info("%s S\n", __func__);
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	/* power control */
-	ret = power_ctrl(sd, 1);
-	if (ret)
-		goto fail_power;
-
-	/* flis clock control */
-	ret = dev->platform_data->flisclk_ctrl(sd, 1);
-	if (ret)
-		goto fail_clk;
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 1);
-	if (ret) {
-		ret = gpio_ctrl(sd, 1);
-		if (ret)
-			goto fail_gpio;
-	}
-
-	msleep(100);
-
-	pr_info("%s E\n", __func__);
-	return 0;
-
-fail_gpio:
-	dev->platform_data->flisclk_ctrl(sd, 0);
-fail_clk:
-	power_ctrl(sd, 0);
-fail_power:
-	dev_err(&client->dev, "sensor power-up failed\n");
-
-	return ret;
-}
-
-static int power_down(struct v4l2_subdev *sd)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 0);
-	if (ret) {
-		ret = gpio_ctrl(sd, 0);
-		if (ret)
-			dev_err(&client->dev, "gpio failed 2\n");
-	}
-
-	ret = dev->platform_data->flisclk_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "flisclk failed\n");
-
-	/* power control */
-	ret = power_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "vprog failed.\n");
-
-	return ret;
-}
-
-static int gc0310_s_power(struct v4l2_subdev *sd, int on)
-{
-	int ret;
-	if (on == 0)
-		return power_down(sd);
-	else {
-		ret = power_up(sd);
-		if (!ret)
-			return gc0310_init(sd);
-	}
-	return ret;
-}
-
-/*
- * distance - calculate the distance
- * @res: resolution
- * @w: width
- * @h: height
- *
- * Get the gap between resolution and w/h.
- * res->width/height smaller than w/h wouldn't be considered.
- * Returns the value of gap or -1 if fail.
- */
-#define LARGEST_ALLOWED_RATIO_MISMATCH 800
-static int distance(struct gc0310_resolution *res, u32 w, u32 h)
-{
-	unsigned int w_ratio = (res->width << 13) / w;
-	unsigned int h_ratio;
-	int match;
-
-	if (h == 0)
-		return -1;
-	h_ratio = (res->height << 13) / h;
-	if (h_ratio == 0)
-		return -1;
-	match   = abs(((w_ratio << 13) / h_ratio) - ((int)8192));
-
-	if ((w_ratio < (int)8192) || (h_ratio < (int)8192)  ||
-		(match > LARGEST_ALLOWED_RATIO_MISMATCH))
-		return -1;
-
-	return w_ratio + h_ratio;
-}
-
-/* Return the nearest higher resolution index */
-static int nearest_resolution_index(int w, int h)
-{
-	int i;
-	int idx = -1;
-	int dist;
-	int min_dist = INT_MAX;
-	struct gc0310_resolution *tmp_res = NULL;
-
-	for (i = 0; i < N_RES; i++) {
-		tmp_res = &gc0310_res[i];
-		dist = distance(tmp_res, w, h);
-		if (dist == -1)
-			continue;
-		if (dist < min_dist) {
-			min_dist = dist;
-			idx = i;
-		}
-	}
-
-	return idx;
-}
-
-static int get_resolution_index(int w, int h)
-{
-	int i;
-
-	for (i = 0; i < N_RES; i++) {
-		if (w != gc0310_res[i].width)
-			continue;
-		if (h != gc0310_res[i].height)
-			continue;
-
-		return i;
-	}
-
-	return -1;
-}
-
-
-/* TODO: remove it. */
-static int startup(struct v4l2_subdev *sd)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	pr_info("%s S\n", __func__);
-
-	ret = gc0310_write_reg_array(client, gc0310_res[dev->fmt_idx].regs);
-	if (ret) {
-		dev_err(&client->dev, "gc0310 write register err.\n");
-		return ret;
-	}
-
-	pr_info("%s E\n", __func__);
-	return ret;
-}
-
-static int gc0310_set_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_pad_config *cfg,
-			  struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct camera_mipi_info *gc0310_info = NULL;
-	int ret = 0;
-	int idx = 0;
-	pr_info("%s S\n", __func__);
-
-	if (format->pad)
-		return -EINVAL;
-
-	if (!fmt)
-		return -EINVAL;
-
-	gc0310_info = v4l2_get_subdev_hostdata(sd);
-	if (!gc0310_info)
-		return -EINVAL;
-
-	mutex_lock(&dev->input_lock);
-
-	idx = nearest_resolution_index(fmt->width, fmt->height);
-	if (idx == -1) {
-		/* return the largest resolution */
-		fmt->width = gc0310_res[N_RES - 1].width;
-		fmt->height = gc0310_res[N_RES - 1].height;
-	} else {
-		fmt->width = gc0310_res[idx].width;
-		fmt->height = gc0310_res[idx].height;
-	}
-	fmt->code = MEDIA_BUS_FMT_SGRBG8_1X8;
-
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
-		cfg->try_fmt = *fmt;
-		mutex_unlock(&dev->input_lock);
-		return 0;
-	}
-
-	dev->fmt_idx = get_resolution_index(fmt->width, fmt->height);
-	if (dev->fmt_idx == -1) {
-		dev_err(&client->dev, "get resolution fail\n");
-		mutex_unlock(&dev->input_lock);
-		return -EINVAL;
-	}
-
-	printk("%s: before gc0310_write_reg_array %s\n", __FUNCTION__,
-	       gc0310_res[dev->fmt_idx].desc);
-	ret = startup(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc0310 startup err\n");
-		goto err;
-	}
-
-	ret = gc0310_get_intg_factor(client, gc0310_info,
-				     &gc0310_res[dev->fmt_idx]);
-	if (ret) {
-		dev_err(&client->dev, "failed to get integration_factor\n");
-		goto err;
-	}
-
-	pr_info("%s E\n", __func__);
-err:
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-
-static int gc0310_get_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_pad_config *cfg,
-			  struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
-	if (format->pad)
-		return -EINVAL;
-
-	if (!fmt)
-		return -EINVAL;
-
-	fmt->width = gc0310_res[dev->fmt_idx].width;
-	fmt->height = gc0310_res[dev->fmt_idx].height;
-	fmt->code = MEDIA_BUS_FMT_SGRBG8_1X8;
-
-	return 0;
-}
-
-static int gc0310_detect(struct i2c_client *client)
-{
-	struct i2c_adapter *adapter = client->adapter;
-	u8 high, low;
-	int ret;
-	u16 id;
-
-	pr_info("%s S\n", __func__);
-	if (!i2c_check_functionality(adapter, I2C_FUNC_I2C))
-		return -ENODEV;
-
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_SC_CMMN_CHIP_ID_H, &high);
-	if (ret) {
-		dev_err(&client->dev, "read sensor_id_high failed\n");
-		return -ENODEV;
-	}
-	ret = gc0310_read_reg(client, GC0310_8BIT,
-					GC0310_SC_CMMN_CHIP_ID_L, &low);
-	if (ret) {
-		dev_err(&client->dev, "read sensor_id_low failed\n");
-		return -ENODEV;
-	}
-	id = ((((u16) high) << 8) | (u16) low);
-	pr_info("sensor ID = 0x%x\n", id);
-
-	if (id != GC0310_ID) {
-		dev_err(&client->dev, "sensor ID error, read id = 0x%x, target id = 0x%x\n", id, GC0310_ID);
-		return -ENODEV;
-	}
-
-	dev_dbg(&client->dev, "detect gc0310 success\n");
-
-	pr_info("%s E\n", __func__);
-
-	return 0;
-}
-
-static int gc0310_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	pr_info("%s S enable=%d\n", __func__, enable);
-	mutex_lock(&dev->input_lock);
-
-	if (enable) {
-		/* enable per frame MIPI and sensor ctrl reset  */
-		ret = gc0310_write_reg(client, GC0310_8BIT,
-						0xFE, 0x30);
-		if (ret) {
-			mutex_unlock(&dev->input_lock);
-			return ret;
-		}
-	}
-
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-				GC0310_RESET_RELATED, GC0310_REGISTER_PAGE_3);
-	if (ret) {
-		mutex_unlock(&dev->input_lock);
-		return ret;
-	}
-
-	ret = gc0310_write_reg(client, GC0310_8BIT, GC0310_SW_STREAM,
-				enable ? GC0310_START_STREAMING :
-				GC0310_STOP_STREAMING);
-	if (ret) {
-		mutex_unlock(&dev->input_lock);
-		return ret;
-	}
-
-	ret = gc0310_write_reg(client, GC0310_8BIT,
-				GC0310_RESET_RELATED, GC0310_REGISTER_PAGE_0);
-	if (ret) {
-		mutex_unlock(&dev->input_lock);
-		return ret;
-	}
-
-	mutex_unlock(&dev->input_lock);
-	pr_info("%s E\n", __func__);
-	return ret;
-}
-
-
-static int gc0310_s_config(struct v4l2_subdev *sd,
-			   int irq, void *platform_data)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	pr_info("%s S\n", __func__);
-	if (!platform_data)
-		return -ENODEV;
-
-	dev->platform_data =
-		(struct camera_sensor_platform_data *)platform_data;
-
-	mutex_lock(&dev->input_lock);
-	/* power off the module, then power on it in future
-	 * as first power on by board may not fulfill the
-	 * power on sequqence needed by the module
-	 */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc0310 power-off err.\n");
-		goto fail_power_off;
-	}
-
-	ret = power_up(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc0310 power-up err.\n");
-		goto fail_power_on;
-	}
-
-	ret = dev->platform_data->csi_cfg(sd, 1);
-	if (ret)
-		goto fail_csi_cfg;
-
-	/* config & detect sensor */
-	ret = gc0310_detect(client);
-	if (ret) {
-		dev_err(&client->dev, "gc0310_detect err s_config.\n");
-		goto fail_csi_cfg;
-	}
-
-	/* turn off sensor, after probed */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc0310 power-off err.\n");
-		goto fail_csi_cfg;
-	}
-	mutex_unlock(&dev->input_lock);
-
-	pr_info("%s E\n", __func__);
-	return 0;
-
-fail_csi_cfg:
-	dev->platform_data->csi_cfg(sd, 0);
-fail_power_on:
-	power_down(sd);
-	dev_err(&client->dev, "sensor power-gating failed\n");
-fail_power_off:
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-
-static int gc0310_g_frame_interval(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_frame_interval *interval)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
-	interval->interval.numerator = 1;
-	interval->interval.denominator = gc0310_res[dev->fmt_idx].fps;
-
-	return 0;
-}
-
-static int gc0310_enum_mbus_code(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_pad_config *cfg,
-				 struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->index >= MAX_FMTS)
-		return -EINVAL;
-
-	code->code = MEDIA_BUS_FMT_SGRBG8_1X8;
-	return 0;
-}
-
-static int gc0310_enum_frame_size(struct v4l2_subdev *sd,
-				  struct v4l2_subdev_pad_config *cfg,
-				  struct v4l2_subdev_frame_size_enum *fse)
-{
-	int index = fse->index;
-
-	if (index >= N_RES)
-		return -EINVAL;
-
-	fse->min_width = gc0310_res[index].width;
-	fse->min_height = gc0310_res[index].height;
-	fse->max_width = gc0310_res[index].width;
-	fse->max_height = gc0310_res[index].height;
-
-	return 0;
-
-}
-
-
-static int gc0310_g_skip_frames(struct v4l2_subdev *sd, u32 *frames)
-{
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-
-	mutex_lock(&dev->input_lock);
-	*frames = gc0310_res[dev->fmt_idx].skip_frames;
-	mutex_unlock(&dev->input_lock);
-
-	return 0;
-}
-
-static const struct v4l2_subdev_sensor_ops gc0310_sensor_ops = {
-	.g_skip_frames	= gc0310_g_skip_frames,
-};
-
-static const struct v4l2_subdev_video_ops gc0310_video_ops = {
-	.s_stream = gc0310_s_stream,
-	.g_frame_interval = gc0310_g_frame_interval,
-};
-
-static const struct v4l2_subdev_core_ops gc0310_core_ops = {
-	.s_power = gc0310_s_power,
-	.ioctl = gc0310_ioctl,
-};
-
-static const struct v4l2_subdev_pad_ops gc0310_pad_ops = {
-	.enum_mbus_code = gc0310_enum_mbus_code,
-	.enum_frame_size = gc0310_enum_frame_size,
-	.get_fmt = gc0310_get_fmt,
-	.set_fmt = gc0310_set_fmt,
-};
-
-static const struct v4l2_subdev_ops gc0310_ops = {
-	.core = &gc0310_core_ops,
-	.video = &gc0310_video_ops,
-	.pad = &gc0310_pad_ops,
-	.sensor = &gc0310_sensor_ops,
-};
-
-static int gc0310_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct gc0310_device *dev = to_gc0310_sensor(sd);
-	dev_dbg(&client->dev, "gc0310_remove...\n");
-
-	dev->platform_data->csi_cfg(sd, 0);
-
-	v4l2_device_unregister_subdev(sd);
-	media_entity_cleanup(&dev->sd.entity);
-	v4l2_ctrl_handler_free(&dev->ctrl_handler);
-	kfree(dev);
-
-	return 0;
-}
-
-static int gc0310_probe(struct i2c_client *client)
-{
-	struct gc0310_device *dev;
-	int ret;
-	void *pdata;
-	unsigned int i;
-
-	pr_info("%s S\n", __func__);
-	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
-
-	mutex_init(&dev->input_lock);
-
-	dev->fmt_idx = 0;
-	v4l2_i2c_subdev_init(&(dev->sd), client, &gc0310_ops);
-
-	pdata = gmin_camera_platform_data(&dev->sd,
-					  ATOMISP_INPUT_FORMAT_RAW_8,
-					  atomisp_bayer_order_grbg);
-	if (!pdata) {
-		ret = -EINVAL;
-		goto out_free;
-	}
-
-	ret = gc0310_s_config(&dev->sd, client->irq, pdata);
-	if (ret)
-		goto out_free;
-
-	ret = atomisp_register_i2c_module(&dev->sd, pdata, RAW_CAMERA);
-	if (ret)
-		goto out_free;
-
-	dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	dev->pad.flags = MEDIA_PAD_FL_SOURCE;
-	dev->format.code = MEDIA_BUS_FMT_SGRBG8_1X8;
-	dev->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
-	ret =
-	    v4l2_ctrl_handler_init(&dev->ctrl_handler,
-				   ARRAY_SIZE(gc0310_controls));
-	if (ret) {
-		gc0310_remove(client);
-		return ret;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(gc0310_controls); i++)
-		v4l2_ctrl_new_custom(&dev->ctrl_handler, &gc0310_controls[i],
-				     NULL);
-
-	if (dev->ctrl_handler.error) {
-		gc0310_remove(client);
-		return dev->ctrl_handler.error;
-	}
-
-	/* Use same lock for controls as for everything else. */
-	dev->ctrl_handler.lock = &dev->input_lock;
-	dev->sd.ctrl_handler = &dev->ctrl_handler;
-
-	ret = media_entity_pads_init(&dev->sd.entity, 1, &dev->pad);
-	if (ret)
-		gc0310_remove(client);
-
-	pr_info("%s E\n", __func__);
-	return ret;
-out_free:
-	v4l2_device_unregister_subdev(&dev->sd);
-	kfree(dev);
-	return ret;
-}
-
-static const struct acpi_device_id gc0310_acpi_match[] = {
-	{"XXGC0310"},
-	{"INT0310"},
-	{},
-};
-MODULE_DEVICE_TABLE(acpi, gc0310_acpi_match);
-
-static struct i2c_driver gc0310_driver = {
-	.driver = {
-		.name = "gc0310",
-		.acpi_match_table = gc0310_acpi_match,
-	},
-	.probe_new = gc0310_probe,
-	.remove = gc0310_remove,
-};
-module_i2c_driver(gc0310_driver);
-
-MODULE_AUTHOR("Lai, Angie <angie.lai@intel.com>");
-MODULE_DESCRIPTION("A low-level driver for GalaxyCore GC0310 sensors");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
deleted file mode 100644
index 93f9c618f3d8..000000000000
--- a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
+++ /dev/null
@@ -1,1122 +0,0 @@
-/*
- * Support for GalaxyCore GC2235 2M camera sensor.
- *
- * Copyright (c) 2014 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- */
-
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/string.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/kmod.h>
-#include <linux/device.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <linux/i2c.h>
-#include <linux/moduleparam.h>
-#include <media/v4l2-device.h>
-#include "../include/linux/atomisp_gmin_platform.h"
-#include <linux/acpi.h>
-#include <linux/io.h>
-
-#include "gc2235.h"
-
-/* i2c read/write stuff */
-static int gc2235_read_reg(struct i2c_client *client,
-			   u16 data_length, u16 reg, u16 *val)
-{
-	int err;
-	struct i2c_msg msg[2];
-	unsigned char data[6];
-
-	if (!client->adapter) {
-		dev_err(&client->dev, "%s error, no client->adapter\n",
-			__func__);
-		return -ENODEV;
-	}
-
-	if (data_length != GC2235_8BIT) {
-		dev_err(&client->dev, "%s error, invalid data length\n",
-			__func__);
-		return -EINVAL;
-	}
-
-	memset(msg, 0, sizeof(msg));
-
-	msg[0].addr = client->addr;
-	msg[0].flags = 0;
-	msg[0].len = 1;
-	msg[0].buf = data;
-
-	/* high byte goes out first */
-	data[0] = (u8)(reg & 0xff);
-
-	msg[1].addr = client->addr;
-	msg[1].len = data_length;
-	msg[1].flags = I2C_M_RD;
-	msg[1].buf = data;
-
-	err = i2c_transfer(client->adapter, msg, 2);
-	if (err != 2) {
-		if (err >= 0)
-			err = -EIO;
-		dev_err(&client->dev,
-			"read from offset 0x%x error %d", reg, err);
-		return err;
-	}
-
-	*val = 0;
-	/* high byte comes first */
-	if (data_length == GC2235_8BIT)
-		*val = (u8)data[0];
-
-	return 0;
-}
-
-static int gc2235_i2c_write(struct i2c_client *client, u16 len, u8 *data)
-{
-	struct i2c_msg msg;
-	const int num_msg = 1;
-	int ret;
-
-	msg.addr = client->addr;
-	msg.flags = 0;
-	msg.len = len;
-	msg.buf = data;
-	ret = i2c_transfer(client->adapter, &msg, 1);
-
-	return ret == num_msg ? 0 : -EIO;
-}
-
-static int gc2235_write_reg(struct i2c_client *client, u16 data_length,
-							u8 reg, u8 val)
-{
-	int ret;
-	unsigned char data[4] = {0};
-	const u16 len = data_length + sizeof(u8); /* 16-bit address + data */
-
-	if (data_length != GC2235_8BIT) {
-		dev_err(&client->dev,
-			"%s error, invalid data_length\n", __func__);
-		return -EINVAL;
-	}
-
-	/* high byte goes out first */
-	data[0] = reg;
-	data[1] = val;
-
-	ret = gc2235_i2c_write(client, len, data);
-	if (ret)
-		dev_err(&client->dev,
-			"write error: wrote 0x%x to offset 0x%x error %d",
-			val, reg, ret);
-
-	return ret;
-}
-
-static int __gc2235_flush_reg_array(struct i2c_client *client,
-				    struct gc2235_write_ctrl *ctrl)
-{
-	u16 size;
-
-	if (ctrl->index == 0)
-		return 0;
-
-	size = sizeof(u8) + ctrl->index; /* 8-bit address + data */
-	ctrl->index = 0;
-
-	return gc2235_i2c_write(client, size, (u8 *)&ctrl->buffer);
-}
-
-static int __gc2235_buf_reg_array(struct i2c_client *client,
-				  struct gc2235_write_ctrl *ctrl,
-				  const struct gc2235_reg *next)
-{
-	int size;
-
-	if (next->type != GC2235_8BIT)
-		return -EINVAL;
-
-	size = 1;
-	ctrl->buffer.data[ctrl->index] = (u8)next->val;
-
-	/* When first item is added, we need to store its starting address */
-	if (ctrl->index == 0)
-		ctrl->buffer.addr = next->reg;
-
-	ctrl->index += size;
-
-	/*
-	 * Buffer cannot guarantee free space for u32? Better flush it to avoid
-	 * possible lack of memory for next item.
-	 */
-	if (ctrl->index + sizeof(u8) >= GC2235_MAX_WRITE_BUF_SIZE)
-		return __gc2235_flush_reg_array(client, ctrl);
-
-	return 0;
-}
-static int __gc2235_write_reg_is_consecutive(struct i2c_client *client,
-					     struct gc2235_write_ctrl *ctrl,
-					     const struct gc2235_reg *next)
-{
-	if (ctrl->index == 0)
-		return 1;
-
-	return ctrl->buffer.addr + ctrl->index == next->reg;
-}
-static int gc2235_write_reg_array(struct i2c_client *client,
-				  const struct gc2235_reg *reglist)
-{
-	const struct gc2235_reg *next = reglist;
-	struct gc2235_write_ctrl ctrl;
-	int err;
-
-	ctrl.index = 0;
-	for (; next->type != GC2235_TOK_TERM; next++) {
-		switch (next->type & GC2235_TOK_MASK) {
-		case GC2235_TOK_DELAY:
-			err = __gc2235_flush_reg_array(client, &ctrl);
-			if (err)
-				return err;
-			msleep(next->val);
-			break;
-		default:
-			/*
-			 * If next address is not consecutive, data needs to be
-			 * flushed before proceed.
-			 */
-			if (!__gc2235_write_reg_is_consecutive(client, &ctrl,
-								next)) {
-				err = __gc2235_flush_reg_array(client, &ctrl);
-				if (err)
-					return err;
-			}
-			err = __gc2235_buf_reg_array(client, &ctrl, next);
-			if (err) {
-				dev_err(&client->dev, "%s: write error, aborted\n",
-					 __func__);
-				return err;
-			}
-			break;
-		}
-	}
-
-	return __gc2235_flush_reg_array(client, &ctrl);
-}
-
-static int gc2235_g_focal(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = (GC2235_FOCAL_LENGTH_NUM << 16) | GC2235_FOCAL_LENGTH_DEM;
-	return 0;
-}
-
-static int gc2235_g_fnumber(struct v4l2_subdev *sd, s32 *val)
-{
-	/*const f number for imx*/
-	*val = (GC2235_F_NUMBER_DEFAULT_NUM << 16) | GC2235_F_NUMBER_DEM;
-	return 0;
-}
-
-static int gc2235_g_fnumber_range(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = (GC2235_F_NUMBER_DEFAULT_NUM << 24) |
-		(GC2235_F_NUMBER_DEM << 16) |
-		(GC2235_F_NUMBER_DEFAULT_NUM << 8) | GC2235_F_NUMBER_DEM;
-	return 0;
-}
-
-
-static int gc2235_get_intg_factor(struct i2c_client *client,
-				struct camera_mipi_info *info,
-				const struct gc2235_resolution *res)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-	struct atomisp_sensor_mode_data *buf = &info->data;
-	u16 reg_val, reg_val_h, dummy;
-	int ret;
-
-	if (!info)
-		return -EINVAL;
-
-	/* pixel clock calculattion */
-	buf->vt_pix_clk_freq_mhz = dev->vt_pix_clk_freq_mhz = 30000000;
-
-	/* get integration time */
-	buf->coarse_integration_time_min = GC2235_COARSE_INTG_TIME_MIN;
-	buf->coarse_integration_time_max_margin =
-					GC2235_COARSE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_min = GC2235_FINE_INTG_TIME_MIN;
-	buf->fine_integration_time_max_margin =
-					GC2235_FINE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_def = GC2235_FINE_INTG_TIME_MIN;
-	buf->frame_length_lines = res->lines_per_frame;
-	buf->line_length_pck = res->pixels_per_line;
-	buf->read_mode = res->bin_mode;
-
-	/* get the cropping and output resolution to ISP for this mode. */
-	ret =  gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_H_CROP_START_H, &reg_val_h);
-	ret =  gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_H_CROP_START_L, &reg_val);
-	if (ret)
-		return ret;
-
-	buf->crop_horizontal_start = (reg_val_h << 8) | reg_val;
-
-	ret =  gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_V_CROP_START_H, &reg_val_h);
-	ret =  gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_V_CROP_START_L, &reg_val);
-	if (ret)
-		return ret;
-
-	buf->crop_vertical_start = (reg_val_h << 8) | reg_val;
-
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_H_OUTSIZE_H, &reg_val_h);
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_H_OUTSIZE_L, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_width = (reg_val_h << 8) | reg_val;
-
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_V_OUTSIZE_H, &reg_val_h);
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_V_OUTSIZE_L, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_height = (reg_val_h << 8) | reg_val;
-
-	buf->crop_horizontal_end = buf->crop_horizontal_start +
-						buf->output_width - 1;
-	buf->crop_vertical_end = buf->crop_vertical_start +
-						buf->output_height - 1;
-
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_HB_H, &reg_val_h);
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_HB_L, &reg_val);
-	if (ret)
-		return ret;
-
-	dummy = (reg_val_h << 8) | reg_val;
-
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_SH_DELAY_H, &reg_val_h);
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_SH_DELAY_L, &reg_val);
-
-#if 0
-	buf->line_length_pck = buf->output_width + 16 + dummy +
-				(((u16)reg_val_h << 8) | (u16)reg_val) + 4;
-#endif
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_VB_H, &reg_val_h);
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_VB_L, &reg_val);
-	if (ret)
-		return ret;
-
-#if 0
-	buf->frame_length_lines = buf->output_height + 32 +
-				(((u16)reg_val_h << 8) | (u16)reg_val);
-#endif
-	buf->binning_factor_x = res->bin_factor_x ?
-					res->bin_factor_x : 1;
-	buf->binning_factor_y = res->bin_factor_y ?
-					res->bin_factor_y : 1;
-	return 0;
-}
-
-static long __gc2235_set_exposure(struct v4l2_subdev *sd, int coarse_itg,
-				 int gain, int digitgain)
-
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u16 coarse_integration = (u16)coarse_itg;
-	int ret = 0;
-	u16 expo_coarse_h, expo_coarse_l, gain_val = 0xF0, gain_val2 = 0xF0;
-	expo_coarse_h = coarse_integration >> 8;
-	expo_coarse_l = coarse_integration & 0xff;
-
-	ret = gc2235_write_reg(client, GC2235_8BIT,
-					GC2235_EXPOSURE_H, expo_coarse_h);
-	ret = gc2235_write_reg(client, GC2235_8BIT,
-					GC2235_EXPOSURE_L, expo_coarse_l);
-
-	if (gain <= 0x58) {
-		gain_val = 0x40;
-		gain_val2 = 0x58;
-	} else if (gain < 256) {
-		gain_val = 0x40;
-		gain_val2 = gain;
-	} else {
-		gain_val2 = 64 * gain / 256;
-		gain_val = 0xff;
-	}
-
-	ret = gc2235_write_reg(client, GC2235_8BIT,
-					GC2235_GLOBAL_GAIN, (u8)gain_val);
-	ret = gc2235_write_reg(client, GC2235_8BIT,
-					GC2235_PRE_GAIN, (u8)gain_val2);
-
-	return ret;
-}
-
-
-static int gc2235_set_exposure(struct v4l2_subdev *sd, int exposure,
-	int gain, int digitgain)
-{
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-	int ret;
-
-	mutex_lock(&dev->input_lock);
-	ret = __gc2235_set_exposure(sd, exposure, gain, digitgain);
-	mutex_unlock(&dev->input_lock);
-
-	return ret;
-}
-
-static long gc2235_s_exposure(struct v4l2_subdev *sd,
-			       struct atomisp_exposure *exposure)
-{
-	int exp = exposure->integration_time[0];
-	int gain = exposure->gain[0];
-	int digitgain = exposure->gain[1];
-
-	/* we should not accept the invalid value below. */
-	if (gain == 0) {
-		struct i2c_client *client = v4l2_get_subdevdata(sd);
-		v4l2_err(client, "%s: invalid value\n", __func__);
-		return -EINVAL;
-	}
-
-	return gc2235_set_exposure(sd, exp, gain, digitgain);
-}
-static long gc2235_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
-{
-	switch (cmd) {
-	case ATOMISP_IOC_S_EXPOSURE:
-		return gc2235_s_exposure(sd, arg);
-	default:
-		return -EINVAL;
-	}
-	return 0;
-}
-/* This returns the exposure time being used. This should only be used
- * for filling in EXIF data, not for actual image processing.
- */
-static int gc2235_q_exposure(struct v4l2_subdev *sd, s32 *value)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u16 reg_v, reg_v2;
-	int ret;
-
-	/* get exposure */
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_EXPOSURE_L,
-					&reg_v);
-	if (ret)
-		goto err;
-
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_EXPOSURE_H,
-					&reg_v2);
-	if (ret)
-		goto err;
-
-	reg_v += reg_v2 << 8;
-
-	*value = reg_v;
-err:
-	return ret;
-}
-
-static int gc2235_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct gc2235_device *dev =
-	    container_of(ctrl->handler, struct gc2235_device, ctrl_handler);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_EXPOSURE_ABSOLUTE:
-		ret = gc2235_q_exposure(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FOCAL_ABSOLUTE:
-		ret = gc2235_g_focal(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_ABSOLUTE:
-		ret = gc2235_g_fnumber(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_RANGE:
-		ret = gc2235_g_fnumber_range(&dev->sd, &ctrl->val);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-static const struct v4l2_ctrl_ops ctrl_ops = {
-	.g_volatile_ctrl = gc2235_g_volatile_ctrl
-};
-
-static struct v4l2_ctrl_config gc2235_controls[] = {
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_EXPOSURE_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "exposure",
-	 .min = 0x0,
-	 .max = 0xffff,
-	 .step = 0x01,
-	 .def = 0x00,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FOCAL_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "focal length",
-	 .min = GC2235_FOCAL_LENGTH_DEFAULT,
-	 .max = GC2235_FOCAL_LENGTH_DEFAULT,
-	 .step = 0x01,
-	 .def = GC2235_FOCAL_LENGTH_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "f-number",
-	 .min = GC2235_F_NUMBER_DEFAULT,
-	 .max = GC2235_F_NUMBER_DEFAULT,
-	 .step = 0x01,
-	 .def = GC2235_F_NUMBER_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_RANGE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "f-number range",
-	 .min = GC2235_F_NUMBER_RANGE,
-	 .max = GC2235_F_NUMBER_RANGE,
-	 .step = 0x01,
-	 .def = GC2235_F_NUMBER_RANGE,
-	 .flags = 0,
-	 },
-};
-
-static int __gc2235_init(struct v4l2_subdev *sd)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-
-	/* restore settings */
-	gc2235_res = gc2235_res_preview;
-	N_RES = N_RES_PREVIEW;
-
-	return gc2235_write_reg_array(client, gc2235_init_settings);
-}
-
-static int is_init;
-
-static int power_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	int ret = -1;
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	if (flag) {
-		ret = dev->platform_data->v1p8_ctrl(sd, 1);
-		usleep_range(60, 90);
-		if (ret == 0)
-			ret |= dev->platform_data->v2p8_ctrl(sd, 1);
-	} else {
-		ret = dev->platform_data->v1p8_ctrl(sd, 0);
-		ret |= dev->platform_data->v2p8_ctrl(sd, 0);
-	}
-	return ret;
-}
-
-static int gpio_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-	int ret = -1;
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	ret |= dev->platform_data->gpio1_ctrl(sd, !flag);
-	usleep_range(60, 90);
-	return dev->platform_data->gpio0_ctrl(sd, flag);
-}
-
-static int power_up(struct v4l2_subdev *sd)
-{
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-	/* power control */
-	ret = power_ctrl(sd, 1);
-	if (ret)
-		goto fail_power;
-
-	/* according to DS, at least 5ms is needed between DOVDD and PWDN */
-	usleep_range(5000, 6000);
-
-	ret = dev->platform_data->flisclk_ctrl(sd, 1);
-	if (ret)
-		goto fail_clk;
-	usleep_range(5000, 6000);
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 1);
-	if (ret) {
-		ret = gpio_ctrl(sd, 1);
-		if (ret)
-			goto fail_power;
-	}
-
-	msleep(5);
-	return 0;
-
-fail_clk:
-	gpio_ctrl(sd, 0);
-fail_power:
-	power_ctrl(sd, 0);
-	dev_err(&client->dev, "sensor power-up failed\n");
-
-	return ret;
-}
-
-static int power_down(struct v4l2_subdev *sd)
-{
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 0);
-	if (ret) {
-		ret = gpio_ctrl(sd, 0);
-		if (ret)
-			dev_err(&client->dev, "gpio failed 2\n");
-	}
-
-	ret = dev->platform_data->flisclk_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "flisclk failed\n");
-
-	/* power control */
-	ret = power_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "vprog failed.\n");
-
-	return ret;
-}
-
-static int gc2235_s_power(struct v4l2_subdev *sd, int on)
-{
-	int ret;
-
-	if (on == 0)
-		ret = power_down(sd);
-	else {
-		ret = power_up(sd);
-		if (!ret)
-			ret = __gc2235_init(sd);
-		is_init = 1;
-	}
-	return ret;
-}
-
-/*
- * distance - calculate the distance
- * @res: resolution
- * @w: width
- * @h: height
- *
- * Get the gap between resolution and w/h.
- * res->width/height smaller than w/h wouldn't be considered.
- * Returns the value of gap or -1 if fail.
- */
-#define LARGEST_ALLOWED_RATIO_MISMATCH 800
-static int distance(struct gc2235_resolution *res, u32 w, u32 h)
-{
-	unsigned int w_ratio = (res->width << 13) / w;
-	unsigned int h_ratio;
-	int match;
-
-	if (h == 0)
-		return -1;
-	h_ratio = (res->height << 13) / h;
-	if (h_ratio == 0)
-		return -1;
-	match   = abs(((w_ratio << 13) / h_ratio) - 8192);
-
-	if ((w_ratio < 8192) || (h_ratio < 8192) ||
-	    (match > LARGEST_ALLOWED_RATIO_MISMATCH))
-		return -1;
-
-	return w_ratio + h_ratio;
-}
-
-/* Return the nearest higher resolution index */
-static int nearest_resolution_index(int w, int h)
-{
-	int i;
-	int idx = -1;
-	int dist;
-	int min_dist = INT_MAX;
-	struct gc2235_resolution *tmp_res = NULL;
-
-	for (i = 0; i < N_RES; i++) {
-		tmp_res = &gc2235_res[i];
-		dist = distance(tmp_res, w, h);
-		if (dist == -1)
-			continue;
-		if (dist < min_dist) {
-			min_dist = dist;
-			idx = i;
-		}
-	}
-
-	return idx;
-}
-
-static int get_resolution_index(int w, int h)
-{
-	int i;
-
-	for (i = 0; i < N_RES; i++) {
-		if (w != gc2235_res[i].width)
-			continue;
-		if (h != gc2235_res[i].height)
-			continue;
-
-		return i;
-	}
-
-	return -1;
-}
-
-static int startup(struct v4l2_subdev *sd)
-{
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-	if (is_init == 0) {
-		/* force gc2235 to do a reset in res change, otherwise it
-		* can not output normal after switching res. and it is not
-		* necessary for first time run up after power on, for the sack
-		* of performance
-		*/
-		power_down(sd);
-		power_up(sd);
-		gc2235_write_reg_array(client, gc2235_init_settings);
-	}
-
-	ret = gc2235_write_reg_array(client, gc2235_res[dev->fmt_idx].regs);
-	if (ret) {
-		dev_err(&client->dev, "gc2235 write register err.\n");
-		return ret;
-	}
-	is_init = 0;
-
-	return ret;
-}
-
-static int gc2235_set_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_pad_config *cfg,
-			  struct v4l2_subdev_format *format)
-{
-
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct camera_mipi_info *gc2235_info = NULL;
-	int ret = 0;
-	int idx;
-
-	gc2235_info = v4l2_get_subdev_hostdata(sd);
-	if (!gc2235_info)
-		return -EINVAL;
-	if (format->pad)
-		return -EINVAL;
-	if (!fmt)
-		return -EINVAL;
-	mutex_lock(&dev->input_lock);
-	idx = nearest_resolution_index(fmt->width, fmt->height);
-	if (idx == -1) {
-		/* return the largest resolution */
-		fmt->width = gc2235_res[N_RES - 1].width;
-		fmt->height = gc2235_res[N_RES - 1].height;
-	} else {
-		fmt->width = gc2235_res[idx].width;
-		fmt->height = gc2235_res[idx].height;
-	}
-	fmt->code = MEDIA_BUS_FMT_SGRBG10_1X10;
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
-		cfg->try_fmt = *fmt;
-		mutex_unlock(&dev->input_lock);
-		return 0;
-	}
-
-	dev->fmt_idx = get_resolution_index(fmt->width, fmt->height);
-	if (dev->fmt_idx == -1) {
-		dev_err(&client->dev, "get resolution fail\n");
-		mutex_unlock(&dev->input_lock);
-		return -EINVAL;
-	}
-
-	ret = startup(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc2235 startup err\n");
-		goto err;
-	}
-
-	ret = gc2235_get_intg_factor(client, gc2235_info,
-				     &gc2235_res[dev->fmt_idx]);
-	if (ret)
-		dev_err(&client->dev, "failed to get integration_factor\n");
-
-err:
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-
-static int gc2235_get_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_pad_config *cfg,
-			  struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-
-	if (format->pad)
-		return -EINVAL;
-
-	if (!fmt)
-		return -EINVAL;
-
-	fmt->width = gc2235_res[dev->fmt_idx].width;
-	fmt->height = gc2235_res[dev->fmt_idx].height;
-	fmt->code = MEDIA_BUS_FMT_SGRBG10_1X10;
-
-	return 0;
-}
-
-static int gc2235_detect(struct i2c_client *client)
-{
-	struct i2c_adapter *adapter = client->adapter;
-	u16 high, low;
-	int ret;
-	u16 id;
-
-	if (!i2c_check_functionality(adapter, I2C_FUNC_I2C))
-		return -ENODEV;
-
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_SENSOR_ID_H, &high);
-	if (ret) {
-		dev_err(&client->dev, "sensor_id_high = 0x%x\n", high);
-		return -ENODEV;
-	}
-	ret = gc2235_read_reg(client, GC2235_8BIT,
-					GC2235_SENSOR_ID_L, &low);
-	id = ((high << 8) | low);
-
-	if (id != GC2235_ID) {
-		dev_err(&client->dev, "sensor ID error, 0x%x\n", id);
-		return -ENODEV;
-	}
-
-	dev_info(&client->dev, "detect gc2235 success\n");
-	return 0;
-}
-
-static int gc2235_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-	mutex_lock(&dev->input_lock);
-
-	if (enable)
-		ret = gc2235_write_reg_array(client, gc2235_stream_on);
-	else
-		ret = gc2235_write_reg_array(client, gc2235_stream_off);
-
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-
-
-static int gc2235_s_config(struct v4l2_subdev *sd,
-			   int irq, void *platform_data)
-{
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	if (!platform_data)
-		return -ENODEV;
-
-	dev->platform_data =
-		(struct camera_sensor_platform_data *)platform_data;
-
-	mutex_lock(&dev->input_lock);
-	/* power off the module, then power on it in future
-	 * as first power on by board may not fulfill the
-	 * power on sequqence needed by the module
-	 */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc2235 power-off err.\n");
-		goto fail_power_off;
-	}
-
-	ret = power_up(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc2235 power-up err.\n");
-		goto fail_power_on;
-	}
-
-	ret = dev->platform_data->csi_cfg(sd, 1);
-	if (ret)
-		goto fail_csi_cfg;
-
-	/* config & detect sensor */
-	ret = gc2235_detect(client);
-	if (ret) {
-		dev_err(&client->dev, "gc2235_detect err s_config.\n");
-		goto fail_csi_cfg;
-	}
-
-	/* turn off sensor, after probed */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "gc2235 power-off err.\n");
-		goto fail_csi_cfg;
-	}
-	mutex_unlock(&dev->input_lock);
-
-	return 0;
-
-fail_csi_cfg:
-	dev->platform_data->csi_cfg(sd, 0);
-fail_power_on:
-	power_down(sd);
-	dev_err(&client->dev, "sensor power-gating failed\n");
-fail_power_off:
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-
-static int gc2235_g_frame_interval(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_frame_interval *interval)
-{
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-
-	interval->interval.numerator = 1;
-	interval->interval.denominator = gc2235_res[dev->fmt_idx].fps;
-
-	return 0;
-}
-
-static int gc2235_enum_mbus_code(struct v4l2_subdev *sd,
-				struct v4l2_subdev_pad_config *cfg,
-				struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->index >= MAX_FMTS)
-		return -EINVAL;
-
-	code->code = MEDIA_BUS_FMT_SBGGR10_1X10;
-	return 0;
-}
-
-static int gc2235_enum_frame_size(struct v4l2_subdev *sd,
-				  struct v4l2_subdev_pad_config *cfg,
-				  struct v4l2_subdev_frame_size_enum *fse)
-{
-	int index = fse->index;
-
-	if (index >= N_RES)
-		return -EINVAL;
-
-	fse->min_width = gc2235_res[index].width;
-	fse->min_height = gc2235_res[index].height;
-	fse->max_width = gc2235_res[index].width;
-	fse->max_height = gc2235_res[index].height;
-
-	return 0;
-
-}
-
-static int gc2235_g_skip_frames(struct v4l2_subdev *sd, u32 *frames)
-{
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-
-	mutex_lock(&dev->input_lock);
-	*frames = gc2235_res[dev->fmt_idx].skip_frames;
-	mutex_unlock(&dev->input_lock);
-
-	return 0;
-}
-
-static const struct v4l2_subdev_sensor_ops gc2235_sensor_ops = {
-	.g_skip_frames	= gc2235_g_skip_frames,
-};
-
-static const struct v4l2_subdev_video_ops gc2235_video_ops = {
-	.s_stream = gc2235_s_stream,
-	.g_frame_interval = gc2235_g_frame_interval,
-};
-
-static const struct v4l2_subdev_core_ops gc2235_core_ops = {
-	.s_power = gc2235_s_power,
-	.ioctl = gc2235_ioctl,
-};
-
-static const struct v4l2_subdev_pad_ops gc2235_pad_ops = {
-	.enum_mbus_code = gc2235_enum_mbus_code,
-	.enum_frame_size = gc2235_enum_frame_size,
-	.get_fmt = gc2235_get_fmt,
-	.set_fmt = gc2235_set_fmt,
-};
-
-static const struct v4l2_subdev_ops gc2235_ops = {
-	.core = &gc2235_core_ops,
-	.video = &gc2235_video_ops,
-	.pad = &gc2235_pad_ops,
-	.sensor = &gc2235_sensor_ops,
-};
-
-static int gc2235_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct gc2235_device *dev = to_gc2235_sensor(sd);
-	dev_dbg(&client->dev, "gc2235_remove...\n");
-
-	dev->platform_data->csi_cfg(sd, 0);
-
-	v4l2_device_unregister_subdev(sd);
-	media_entity_cleanup(&dev->sd.entity);
-	v4l2_ctrl_handler_free(&dev->ctrl_handler);
-	kfree(dev);
-
-	return 0;
-}
-
-static int gc2235_probe(struct i2c_client *client)
-{
-	struct gc2235_device *dev;
-	void *gcpdev;
-	int ret;
-	unsigned int i;
-
-	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
-
-	mutex_init(&dev->input_lock);
-
-	dev->fmt_idx = 0;
-	v4l2_i2c_subdev_init(&(dev->sd), client, &gc2235_ops);
-
-	gcpdev = gmin_camera_platform_data(&dev->sd,
-				   ATOMISP_INPUT_FORMAT_RAW_10,
-				   atomisp_bayer_order_grbg);
-
-	ret = gc2235_s_config(&dev->sd, client->irq, gcpdev);
-	if (ret)
-		goto out_free;
-
-	dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	dev->pad.flags = MEDIA_PAD_FL_SOURCE;
-	dev->format.code = MEDIA_BUS_FMT_SBGGR10_1X10;
-	dev->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
-	ret =
-	    v4l2_ctrl_handler_init(&dev->ctrl_handler,
-				   ARRAY_SIZE(gc2235_controls));
-	if (ret) {
-		gc2235_remove(client);
-		return ret;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(gc2235_controls); i++)
-		v4l2_ctrl_new_custom(&dev->ctrl_handler, &gc2235_controls[i],
-				     NULL);
-
-	if (dev->ctrl_handler.error) {
-		gc2235_remove(client);
-		return dev->ctrl_handler.error;
-	}
-
-	/* Use same lock for controls as for everything else. */
-	dev->ctrl_handler.lock = &dev->input_lock;
-	dev->sd.ctrl_handler = &dev->ctrl_handler;
-
-	ret = media_entity_pads_init(&dev->sd.entity, 1, &dev->pad);
-	if (ret)
-		gc2235_remove(client);
-
-	return atomisp_register_i2c_module(&dev->sd, gcpdev, RAW_CAMERA);
-
-out_free:
-	v4l2_device_unregister_subdev(&dev->sd);
-	kfree(dev);
-
-	return ret;
-}
-
-static const struct acpi_device_id gc2235_acpi_match[] = {
-	{ "INT33F8" },
-	{},
-};
-MODULE_DEVICE_TABLE(acpi, gc2235_acpi_match);
-
-static struct i2c_driver gc2235_driver = {
-	.driver = {
-		.name = "gc2235",
-		.acpi_match_table = gc2235_acpi_match,
-	},
-	.probe_new = gc2235_probe,
-	.remove = gc2235_remove,
-};
-module_i2c_driver(gc2235_driver);
-
-MODULE_AUTHOR("Shuguang Gong <Shuguang.Gong@intel.com>");
-MODULE_DESCRIPTION("A low-level driver for GC2235 sensors");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c b/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c
deleted file mode 100644
index 81e5ec0c2b64..000000000000
--- a/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#include <linux/i2c.h>
-#include <linux/firmware.h>
-#include <linux/device.h>
-#include <linux/export.h>
-#include "../include/linux/libmsrlisthelper.h"
-#include <linux/module.h>
-#include <linux/slab.h>
-
-/* Tagged binary data container structure definitions. */
-struct tbd_header {
-	uint32_t tag;          /*!< Tag identifier, also checks endianness */
-	uint32_t size;         /*!< Container size including this header */
-	uint32_t version;      /*!< Version, format 0xYYMMDDVV */
-	uint32_t revision;     /*!< Revision, format 0xYYMMDDVV */
-	uint32_t config_bits;  /*!< Configuration flag bits set */
-	uint32_t checksum;     /*!< Global checksum, header included */
-} __packed;
-
-struct tbd_record_header {
-	uint32_t size;        /*!< Size of record including header */
-	uint8_t format_id;    /*!< tbd_format_t enumeration values used */
-	uint8_t packing_key;  /*!< Packing method; 0 = no packing */
-	uint16_t class_id;    /*!< tbd_class_t enumeration values used */
-} __packed;
-
-struct tbd_data_record_header {
-	uint16_t next_offset;
-	uint16_t flags;
-	uint16_t data_offset;
-	uint16_t data_size;
-} __packed;
-
-#define TBD_CLASS_DRV_ID 2
-
-static int set_msr_configuration(struct i2c_client *client, uint8_t *bufptr,
-		unsigned int size)
-{
-	/* The configuration data contains any number of sequences where
-	 * the first byte (that is, uint8_t) that marks the number of bytes
-	 * in the sequence to follow, is indeed followed by the indicated
-	 * number of bytes of actual data to be written to sensor.
-	 * By convention, the first two bytes of actual data should be
-	 * understood as an address in the sensor address space (hibyte
-	 * followed by lobyte) where the remaining data in the sequence
-	 * will be written. */
-
-	uint8_t *ptr = bufptr;
-	while (ptr < bufptr + size) {
-		struct i2c_msg msg = {
-			.addr = client->addr,
-			.flags = 0,
-		};
-		int ret;
-
-		/* How many bytes */
-		msg.len = *ptr++;
-		/* Where the bytes are located */
-		msg.buf = ptr;
-		ptr += msg.len;
-
-		if (ptr > bufptr + size)
-			/* Accessing data beyond bounds is not tolerated */
-			return -EINVAL;
-
-		ret = i2c_transfer(client->adapter, &msg, 1);
-		if (ret < 0) {
-			dev_err(&client->dev, "i2c write error: %d", ret);
-			return ret;
-		}
-	}
-	return 0;
-}
-
-static int parse_and_apply(struct i2c_client *client, uint8_t *buffer,
-		unsigned int size)
-{
-	uint8_t *endptr8 = buffer + size;
-	struct tbd_data_record_header *header =
-		(struct tbd_data_record_header *)buffer;
-
-	/* There may be any number of datasets present */
-	unsigned int dataset = 0;
-
-	do {
-		/* In below, four variables are read from buffer */
-		if ((uint8_t *)header + sizeof(*header) > endptr8)
-			return -EINVAL;
-
-		/* All data should be located within given buffer */
-		if ((uint8_t *)header + header->data_offset +
-				header->data_size > endptr8)
-			return -EINVAL;
-
-		/* We have a new valid dataset */
-		dataset++;
-		/* See whether there is MSR data */
-		/* If yes, update the reg info */
-		if (header->data_size && (header->flags & 1)) {
-			int ret;
-
-			dev_info(&client->dev,
-				"New MSR data for sensor driver (dataset %02d) size:%d\n",
-				dataset, header->data_size);
-			ret = set_msr_configuration(client,
-						buffer + header->data_offset,
-						header->data_size);
-			if (ret)
-				return ret;
-		}
-		header = (struct tbd_data_record_header *)(buffer +
-			header->next_offset);
-	} while (header->next_offset);
-
-	return 0;
-}
-
-int apply_msr_data(struct i2c_client *client, const struct firmware *fw)
-{
-	struct tbd_header *header;
-	struct tbd_record_header *record;
-
-	if (!fw) {
-		dev_warn(&client->dev, "Drv data is not loaded.\n");
-		return -EINVAL;
-	}
-
-	if (sizeof(*header) > fw->size)
-		return -EINVAL;
-
-	header = (struct tbd_header *)fw->data;
-	/* Check that we have drvb block. */
-	if (memcmp(&header->tag, "DRVB", 4))
-		return -EINVAL;
-
-	/* Check the size */
-	if (header->size != fw->size)
-		return -EINVAL;
-
-	if (sizeof(*header) + sizeof(*record) > fw->size)
-		return -EINVAL;
-
-	record = (struct tbd_record_header *)(header + 1);
-	/* Check that class id mathes tbd's drv id. */
-	if (record->class_id != TBD_CLASS_DRV_ID)
-		return -EINVAL;
-
-	/* Size 0 shall not be treated as an error */
-	if (!record->size)
-		return 0;
-
-	return parse_and_apply(client, (uint8_t *)(record + 1), record->size);
-}
-EXPORT_SYMBOL_GPL(apply_msr_data);
-
-int load_msr_list(struct i2c_client *client, char *name,
-		const struct firmware **fw)
-{
-	int ret = request_firmware(fw, name, &client->dev);
-	if (ret) {
-		dev_err(&client->dev,
-			"Error %d while requesting firmware %s\n",
-			ret, name);
-		return ret;
-	}
-	dev_info(&client->dev, "Received %lu bytes drv data\n",
-			(unsigned long)(*fw)->size);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(load_msr_list);
-
-void release_msr_list(struct i2c_client *client, const struct firmware *fw)
-{
-	release_firmware(fw);
-}
-EXPORT_SYMBOL_GPL(release_msr_list);
-
-static int init_msrlisthelper(void)
-{
-	return 0;
-}
-
-static void exit_msrlisthelper(void)
-{
-}
-
-module_init(init_msrlisthelper);
-module_exit(exit_msrlisthelper);
-
-MODULE_AUTHOR("Jukka Kaartinen <jukka.o.kaartinen@intel.com>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
deleted file mode 100644
index 7098bf317f16..000000000000
--- a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
+++ /dev/null
@@ -1,968 +0,0 @@
-/*
- * LED flash driver for LM3554
- *
- * Copyright (c) 2010-2012 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#include <linux/module.h>
-#include <linux/i2c.h>
-#include <linux/mutex.h>
-#include <linux/delay.h>
-#include <linux/gpio.h>
-#include <linux/slab.h>
-
-#include "../include/media/lm3554.h"
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <linux/acpi.h>
-#include <linux/gpio/consumer.h>
-#include "../include/linux/atomisp_gmin_platform.h"
-#include "../include/linux/atomisp.h"
-
-/* Registers */
-
-#define LM3554_TORCH_BRIGHTNESS_REG	0xA0
-#define LM3554_TORCH_MODE_SHIFT		0
-#define LM3554_TORCH_CURRENT_SHIFT	3
-#define LM3554_INDICATOR_CURRENT_SHIFT	6
-
-#define LM3554_FLASH_BRIGHTNESS_REG	0xB0
-#define LM3554_FLASH_MODE_SHIFT		0
-#define LM3554_FLASH_CURRENT_SHIFT	3
-#define LM3554_STROBE_SENSITIVITY_SHIFT	7
-
-#define LM3554_FLASH_DURATION_REG	0xC0
-#define LM3554_FLASH_TIMEOUT_SHIFT	0
-#define LM3554_CURRENT_LIMIT_SHIFT	5
-
-#define LM3554_FLAGS_REG		0xD0
-#define LM3554_FLAG_TIMEOUT		(1 << 0)
-#define LM3554_FLAG_THERMAL_SHUTDOWN	(1 << 1)
-#define LM3554_FLAG_LED_FAULT		(1 << 2)
-#define LM3554_FLAG_TX1_INTERRUPT	(1 << 3)
-#define LM3554_FLAG_TX2_INTERRUPT	(1 << 4)
-#define LM3554_FLAG_LED_THERMAL_FAULT	(1 << 5)
-#define LM3554_FLAG_UNUSED		(1 << 6)
-#define LM3554_FLAG_INPUT_VOLTAGE_LOW	(1 << 7)
-
-#define LM3554_CONFIG_REG_1		0xE0
-#define LM3554_ENVM_TX2_SHIFT		5
-#define LM3554_TX2_POLARITY_SHIFT	6
-
-struct lm3554 {
-	struct v4l2_subdev sd;
-
-	struct mutex power_lock;
-	struct v4l2_ctrl_handler ctrl_handler;
-	int power_count;
-
-	unsigned int mode;
-	int timeout;
-	u8 torch_current;
-	u8 indicator_current;
-	u8 flash_current;
-
-	struct timer_list flash_off_delay;
-	struct lm3554_platform_data *pdata;
-};
-
-#define to_lm3554(p_sd)	container_of(p_sd, struct lm3554, sd)
-
-/* Return negative errno else zero on success */
-static int lm3554_write(struct lm3554 *flash, u8 addr, u8 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&flash->sd);
-	int ret;
-
-	ret = i2c_smbus_write_byte_data(client, addr, val);
-
-	dev_dbg(&client->dev, "Write Addr:%02X Val:%02X %s\n", addr, val,
-		ret < 0 ? "fail" : "ok");
-
-	return ret;
-}
-
-/* Return negative errno else a data byte received from the device. */
-static int lm3554_read(struct lm3554 *flash, u8 addr)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&flash->sd);
-	int ret;
-
-	ret = i2c_smbus_read_byte_data(client, addr);
-
-	dev_dbg(&client->dev, "Read Addr:%02X Val:%02X %s\n", addr, ret,
-		ret < 0 ? "fail" : "ok");
-
-	return ret;
-}
-
-/* -----------------------------------------------------------------------------
- * Hardware configuration
- */
-
-static int lm3554_set_mode(struct lm3554 *flash, unsigned int mode)
-{
-	u8 val;
-	int ret;
-
-	val = (mode << LM3554_FLASH_MODE_SHIFT) |
-	      (flash->flash_current << LM3554_FLASH_CURRENT_SHIFT);
-
-	ret = lm3554_write(flash, LM3554_FLASH_BRIGHTNESS_REG, val);
-	if (ret == 0)
-		flash->mode = mode;
-	return ret;
-}
-
-static int lm3554_set_torch(struct lm3554 *flash)
-{
-	u8 val;
-
-	val = (flash->mode << LM3554_TORCH_MODE_SHIFT) |
-	      (flash->torch_current << LM3554_TORCH_CURRENT_SHIFT) |
-	      (flash->indicator_current << LM3554_INDICATOR_CURRENT_SHIFT);
-
-	return lm3554_write(flash, LM3554_TORCH_BRIGHTNESS_REG, val);
-}
-
-static int lm3554_set_flash(struct lm3554 *flash)
-{
-	u8 val;
-
-	val = (flash->mode << LM3554_FLASH_MODE_SHIFT) |
-	      (flash->flash_current << LM3554_FLASH_CURRENT_SHIFT);
-
-	return lm3554_write(flash, LM3554_FLASH_BRIGHTNESS_REG, val);
-}
-
-static int lm3554_set_duration(struct lm3554 *flash)
-{
-	u8 val;
-
-	val = (flash->timeout << LM3554_FLASH_TIMEOUT_SHIFT) |
-	      (flash->pdata->current_limit << LM3554_CURRENT_LIMIT_SHIFT);
-
-	return lm3554_write(flash, LM3554_FLASH_DURATION_REG, val);
-}
-
-static int lm3554_set_config1(struct lm3554 *flash)
-{
-	u8 val;
-
-	val = (flash->pdata->envm_tx2 << LM3554_ENVM_TX2_SHIFT) |
-	      (flash->pdata->tx2_polarity << LM3554_TX2_POLARITY_SHIFT);
-	return lm3554_write(flash, LM3554_CONFIG_REG_1, val);
-}
-
-/* -----------------------------------------------------------------------------
- * Hardware trigger
- */
-static void lm3554_flash_off_delay(struct timer_list *t)
-{
-	struct lm3554 *flash = from_timer(flash, t, flash_off_delay);
-	struct lm3554_platform_data *pdata = flash->pdata;
-
-	gpio_set_value(pdata->gpio_strobe, 0);
-}
-
-static int lm3554_hw_strobe(struct i2c_client *client, bool strobe)
-{
-	int ret, timer_pending;
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct lm3554 *flash = to_lm3554(sd);
-	struct lm3554_platform_data *pdata = flash->pdata;
-
-	/*
-	 * An abnormal high flash current is observed when strobe off the
-	 * flash. Workaround here is firstly set flash current to lower level,
-	 * wait a short moment, and then strobe off the flash.
-	 */
-
-	timer_pending = del_timer_sync(&flash->flash_off_delay);
-
-	/* Flash off */
-	if (!strobe) {
-		/* set current to 70mA and wait a while */
-		ret = lm3554_write(flash, LM3554_FLASH_BRIGHTNESS_REG, 0);
-		if (ret < 0)
-			goto err;
-		mod_timer(&flash->flash_off_delay,
-			  jiffies + msecs_to_jiffies(LM3554_TIMER_DELAY));
-		return 0;
-	}
-
-	/* Flash on */
-
-	/*
-	 * If timer is killed before run, flash is not strobe off,
-	 * so must strobe off here
-	 */
-	if (timer_pending)
-		gpio_set_value(pdata->gpio_strobe, 0);
-
-	/* Restore flash current settings */
-	ret = lm3554_set_flash(flash);
-	if (ret < 0)
-		goto err;
-
-	/* Strobe on Flash */
-	gpio_set_value(pdata->gpio_strobe, 1);
-
-	return 0;
-err:
-	dev_err(&client->dev, "failed to %s flash strobe (%d)\n",
-		strobe ? "on" : "off", ret);
-	return ret;
-}
-
-/* -----------------------------------------------------------------------------
- * V4L2 controls
- */
-
-static int lm3554_read_status(struct lm3554 *flash)
-{
-	int ret;
-	struct i2c_client *client = v4l2_get_subdevdata(&flash->sd);
-
-	/* NOTE: reading register clear fault status */
-	ret = lm3554_read(flash, LM3554_FLAGS_REG);
-	if (ret < 0)
-		return ret;
-
-	/*
-	 * Accordingly to datasheet we read back '1' in bit 6.
-	 * Clear it first.
-	 */
-	ret &= ~LM3554_FLAG_UNUSED;
-
-	/*
-	 * Do not take TX1/TX2 signal as an error
-	 * because MSIC will not turn off flash, but turn to
-	 * torch mode according to gsm modem signal by hardware.
-	 */
-	ret &= ~(LM3554_FLAG_TX1_INTERRUPT | LM3554_FLAG_TX2_INTERRUPT);
-
-	if (ret > 0)
-		dev_dbg(&client->dev, "LM3554 flag status: %02x\n", ret);
-
-	return ret;
-}
-
-static int lm3554_s_flash_timeout(struct v4l2_subdev *sd, u32 val)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-
-	val = clamp(val, LM3554_MIN_TIMEOUT, LM3554_MAX_TIMEOUT);
-	val = val / LM3554_TIMEOUT_STEPSIZE - 1;
-
-	flash->timeout = val;
-
-	return lm3554_set_duration(flash);
-}
-
-static int lm3554_g_flash_timeout(struct v4l2_subdev *sd, s32 *val)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-
-	*val = (u32)(flash->timeout + 1) * LM3554_TIMEOUT_STEPSIZE;
-
-	return 0;
-}
-
-static int lm3554_s_flash_intensity(struct v4l2_subdev *sd, u32 intensity)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-
-	intensity = LM3554_CLAMP_PERCENTAGE(intensity);
-	intensity = LM3554_PERCENT_TO_VALUE(intensity, LM3554_FLASH_STEP);
-
-	flash->flash_current = intensity;
-
-	return lm3554_set_flash(flash);
-}
-
-static int lm3554_g_flash_intensity(struct v4l2_subdev *sd, s32 *val)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-
-	*val = LM3554_VALUE_TO_PERCENT((u32)flash->flash_current,
-			LM3554_FLASH_STEP);
-
-	return 0;
-}
-
-static int lm3554_s_torch_intensity(struct v4l2_subdev *sd, u32 intensity)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-
-	intensity = LM3554_CLAMP_PERCENTAGE(intensity);
-	intensity = LM3554_PERCENT_TO_VALUE(intensity, LM3554_TORCH_STEP);
-
-	flash->torch_current = intensity;
-
-	return lm3554_set_torch(flash);
-}
-
-static int lm3554_g_torch_intensity(struct v4l2_subdev *sd, s32 *val)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-
-	*val = LM3554_VALUE_TO_PERCENT((u32)flash->torch_current,
-			LM3554_TORCH_STEP);
-
-	return 0;
-}
-
-static int lm3554_s_indicator_intensity(struct v4l2_subdev *sd, u32 intensity)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-
-	intensity = LM3554_CLAMP_PERCENTAGE(intensity);
-	intensity = LM3554_PERCENT_TO_VALUE(intensity, LM3554_INDICATOR_STEP);
-
-	flash->indicator_current = intensity;
-
-	return lm3554_set_torch(flash);
-}
-
-static int lm3554_g_indicator_intensity(struct v4l2_subdev *sd, s32 *val)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-
-	*val = LM3554_VALUE_TO_PERCENT((u32)flash->indicator_current,
-			LM3554_INDICATOR_STEP);
-
-	return 0;
-}
-
-static int lm3554_s_flash_strobe(struct v4l2_subdev *sd, u32 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-
-	return lm3554_hw_strobe(client, val);
-}
-
-static int lm3554_s_flash_mode(struct v4l2_subdev *sd, u32 new_mode)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-	unsigned int mode;
-
-	switch (new_mode) {
-	case ATOMISP_FLASH_MODE_OFF:
-		mode = LM3554_MODE_SHUTDOWN;
-		break;
-	case ATOMISP_FLASH_MODE_FLASH:
-		mode = LM3554_MODE_FLASH;
-		break;
-	case ATOMISP_FLASH_MODE_INDICATOR:
-		mode = LM3554_MODE_INDICATOR;
-		break;
-	case ATOMISP_FLASH_MODE_TORCH:
-		mode = LM3554_MODE_TORCH;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return lm3554_set_mode(flash, mode);
-}
-
-static int lm3554_g_flash_mode(struct v4l2_subdev *sd, s32 *val)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-	*val = flash->mode;
-	return 0;
-}
-
-static int lm3554_g_flash_status(struct v4l2_subdev *sd, s32 *val)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-	int value;
-
-	value = lm3554_read_status(flash);
-	if (value < 0)
-		return value;
-
-	if (value & LM3554_FLAG_TIMEOUT)
-		*val = ATOMISP_FLASH_STATUS_TIMEOUT;
-	else if (value > 0)
-		*val = ATOMISP_FLASH_STATUS_HW_ERROR;
-	else
-		*val = ATOMISP_FLASH_STATUS_OK;
-
-	return 0;
-}
-
-#ifndef CSS15
-static int lm3554_g_flash_status_register(struct v4l2_subdev *sd, s32 *val)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-	int ret;
-
-	ret = lm3554_read(flash, LM3554_FLAGS_REG);
-
-	if (ret < 0)
-		return ret;
-
-	*val = ret;
-	return 0;
-}
-#endif
-
-static int lm3554_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct lm3554 *dev =
-	    container_of(ctrl->handler, struct lm3554, ctrl_handler);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_FLASH_TIMEOUT:
-		ret = lm3554_s_flash_timeout(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_FLASH_INTENSITY:
-		ret = lm3554_s_flash_intensity(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_FLASH_TORCH_INTENSITY:
-		ret = lm3554_s_torch_intensity(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_FLASH_INDICATOR_INTENSITY:
-		ret = lm3554_s_indicator_intensity(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_FLASH_STROBE:
-		ret = lm3554_s_flash_strobe(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_FLASH_MODE:
-		ret = lm3554_s_flash_mode(&dev->sd, ctrl->val);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-	return ret;
-}
-
-static int lm3554_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct lm3554 *dev =
-	    container_of(ctrl->handler, struct lm3554, ctrl_handler);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_FLASH_TIMEOUT:
-		ret = lm3554_g_flash_timeout(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FLASH_INTENSITY:
-		ret = lm3554_g_flash_intensity(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FLASH_TORCH_INTENSITY:
-		ret = lm3554_g_torch_intensity(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FLASH_INDICATOR_INTENSITY:
-		ret = lm3554_g_indicator_intensity(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FLASH_MODE:
-		ret = lm3554_g_flash_mode(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FLASH_STATUS:
-		ret = lm3554_g_flash_status(&dev->sd, &ctrl->val);
-		break;
-#ifndef CSS15
-	case V4L2_CID_FLASH_STATUS_REGISTER:
-		ret = lm3554_g_flash_status_register(&dev->sd, &ctrl->val);
-		break;
-#endif
-	default:
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-static const struct v4l2_ctrl_ops ctrl_ops = {
-	.s_ctrl = lm3554_s_ctrl,
-	.g_volatile_ctrl = lm3554_g_volatile_ctrl
-};
-
-static const struct v4l2_ctrl_config lm3554_controls[] = {
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FLASH_TIMEOUT,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "Flash Timeout",
-	 .min = 0x0,
-	 .max = LM3554_MAX_TIMEOUT,
-	 .step = 0x01,
-	 .def = LM3554_DEFAULT_TIMEOUT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FLASH_INTENSITY,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "Flash Intensity",
-	 .min = LM3554_MIN_PERCENT,
-	 .max = LM3554_MAX_PERCENT,
-	 .step = 0x01,
-	 .def = LM3554_FLASH_DEFAULT_BRIGHTNESS,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FLASH_TORCH_INTENSITY,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "Torch Intensity",
-	 .min = LM3554_MIN_PERCENT,
-	 .max = LM3554_MAX_PERCENT,
-	 .step = 0x01,
-	 .def = LM3554_TORCH_DEFAULT_BRIGHTNESS,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FLASH_INDICATOR_INTENSITY,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "Indicator Intensity",
-	 .min = LM3554_MIN_PERCENT,
-	 .max = LM3554_MAX_PERCENT,
-	 .step = 0x01,
-	 .def = LM3554_INDICATOR_DEFAULT_BRIGHTNESS,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FLASH_STROBE,
-	 .type = V4L2_CTRL_TYPE_BOOLEAN,
-	 .name = "Flash Strobe",
-	 .min = 0,
-	 .max = 1,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FLASH_MODE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "Flash Mode",
-	 .min = 0,
-	 .max = 100,
-	 .step = 1,
-	 .def = ATOMISP_FLASH_MODE_OFF,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FLASH_STATUS,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "Flash Status",
-	 .min = ATOMISP_FLASH_STATUS_OK,
-	 .max = ATOMISP_FLASH_STATUS_TIMEOUT,
-	 .step = 1,
-	 .def = ATOMISP_FLASH_STATUS_OK,
-	 .flags = 0,
-	 },
-#ifndef CSS15
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FLASH_STATUS_REGISTER,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "Flash Status Register",
-	 .min = 0,
-	 .max = 255,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-#endif
-};
-
-/* -----------------------------------------------------------------------------
- * V4L2 subdev core operations
- */
-
-/* Put device into known state. */
-static int lm3554_setup(struct lm3554 *flash)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&flash->sd);
-	int ret;
-
-	/* clear the flags register */
-	ret = lm3554_read(flash, LM3554_FLAGS_REG);
-	if (ret < 0)
-		return ret;
-
-	dev_dbg(&client->dev, "Fault info: %02x\n", ret);
-
-	ret = lm3554_set_config1(flash);
-	if (ret < 0)
-		return ret;
-
-	ret = lm3554_set_duration(flash);
-	if (ret < 0)
-		return ret;
-
-	ret = lm3554_set_torch(flash);
-	if (ret < 0)
-		return ret;
-
-	ret = lm3554_set_flash(flash);
-	if (ret < 0)
-		return ret;
-
-	/* read status */
-	ret = lm3554_read_status(flash);
-	if (ret < 0)
-		return ret;
-
-	return ret ? -EIO : 0;
-}
-
-static int __lm3554_s_power(struct lm3554 *flash, int power)
-{
-	struct lm3554_platform_data *pdata = flash->pdata;
-	int ret;
-
-	/*initialize flash driver*/
-	gpio_set_value(pdata->gpio_reset, power);
-	usleep_range(100, 100 + 1);
-
-	if (power) {
-		/* Setup default values. This makes sure that the chip
-		 * is in a known state.
-		 */
-		ret = lm3554_setup(flash);
-		if (ret < 0) {
-			__lm3554_s_power(flash, 0);
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-static int lm3554_s_power(struct v4l2_subdev *sd, int power)
-{
-	struct lm3554 *flash = to_lm3554(sd);
-	int ret = 0;
-
-	mutex_lock(&flash->power_lock);
-
-	if (flash->power_count == !power) {
-		ret = __lm3554_s_power(flash, !!power);
-		if (ret < 0)
-			goto done;
-	}
-
-	flash->power_count += power ? 1 : -1;
-	WARN_ON(flash->power_count < 0);
-
-done:
-	mutex_unlock(&flash->power_lock);
-	return ret;
-}
-
-static const struct v4l2_subdev_core_ops lm3554_core_ops = {
-	.s_power = lm3554_s_power,
-};
-
-static const struct v4l2_subdev_ops lm3554_ops = {
-	.core = &lm3554_core_ops,
-};
-
-static int lm3554_detect(struct v4l2_subdev *sd)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct i2c_adapter *adapter = client->adapter;
-	struct lm3554 *flash = to_lm3554(sd);
-	int ret;
-
-	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA)) {
-		dev_err(&client->dev, "lm3554_detect i2c error\n");
-		return -ENODEV;
-	}
-
-	/* Power up the flash driver and reset it */
-	ret = lm3554_s_power(&flash->sd, 1);
-	if (ret < 0) {
-		dev_err(&client->dev, "Failed to power on lm3554 LED flash\n");
-	} else {
-		dev_dbg(&client->dev, "Successfully detected lm3554 LED flash\n");
-		lm3554_s_power(&flash->sd, 0);
-	}
-
-	return ret;
-}
-
-static int lm3554_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
-{
-	return lm3554_s_power(sd, 1);
-}
-
-static int lm3554_close(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
-{
-	return lm3554_s_power(sd, 0);
-}
-
-static const struct v4l2_subdev_internal_ops lm3554_internal_ops = {
-	.registered = lm3554_detect,
-	.open = lm3554_open,
-	.close = lm3554_close,
-};
-
-/* -----------------------------------------------------------------------------
- *  I2C driver
- */
-#ifdef CONFIG_PM
-
-static int lm3554_suspend(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
-	struct lm3554 *flash = to_lm3554(subdev);
-	int rval;
-
-	if (flash->power_count == 0)
-		return 0;
-
-	rval = __lm3554_s_power(flash, 0);
-
-	dev_dbg(&client->dev, "Suspend %s\n", rval < 0 ? "failed" : "ok");
-
-	return rval;
-}
-
-static int lm3554_resume(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
-	struct lm3554 *flash = to_lm3554(subdev);
-	int rval;
-
-	if (flash->power_count == 0)
-		return 0;
-
-	rval = __lm3554_s_power(flash, 1);
-
-	dev_dbg(&client->dev, "Resume %s\n", rval < 0 ? "fail" : "ok");
-
-	return rval;
-}
-
-#else
-
-#define lm3554_suspend NULL
-#define lm3554_resume  NULL
-
-#endif /* CONFIG_PM */
-
-static int lm3554_gpio_init(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct lm3554 *flash = to_lm3554(sd);
-	struct lm3554_platform_data *pdata = flash->pdata;
-	int ret;
-
-	if (!gpio_is_valid(pdata->gpio_reset))
-		return -EINVAL;
-
-	ret = gpio_direction_output(pdata->gpio_reset, 0);
-	if (ret < 0)
-		goto err_gpio_reset;
-	dev_info(&client->dev, "flash led reset successfully\n");
-
-	if (!gpio_is_valid(pdata->gpio_strobe)) {
-		ret = -EINVAL;
-		goto err_gpio_dir_reset;
-	}
-
-	ret = gpio_direction_output(pdata->gpio_strobe, 0);
-	if (ret < 0)
-		goto err_gpio_strobe;
-
-	return 0;
-
-err_gpio_strobe:
-	gpio_free(pdata->gpio_strobe);
-err_gpio_dir_reset:
-	gpio_direction_output(pdata->gpio_reset, 0);
-err_gpio_reset:
-	gpio_free(pdata->gpio_reset);
-
-	return ret;
-}
-
-static int lm3554_gpio_uninit(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct lm3554 *flash = to_lm3554(sd);
-	struct lm3554_platform_data *pdata = flash->pdata;
-	int ret;
-
-	ret = gpio_direction_output(pdata->gpio_strobe, 0);
-	if (ret < 0)
-		return ret;
-
-	ret = gpio_direction_output(pdata->gpio_reset, 0);
-	if (ret < 0)
-		return ret;
-
-	gpio_free(pdata->gpio_strobe);
-	gpio_free(pdata->gpio_reset);
-	return 0;
-}
-
-static void *lm3554_platform_data_func(struct i2c_client *client)
-{
-	static struct lm3554_platform_data platform_data;
-
-	platform_data.gpio_reset =
-		    desc_to_gpio(gpiod_get_index(&client->dev,
-						 NULL, 2, GPIOD_OUT_LOW));
-	platform_data.gpio_strobe =
-		    desc_to_gpio(gpiod_get_index(&client->dev,
-						 NULL, 0, GPIOD_OUT_LOW));
-	platform_data.gpio_torch =
-		    desc_to_gpio(gpiod_get_index(&client->dev,
-						 NULL, 1, GPIOD_OUT_LOW));
-	dev_info(&client->dev, "camera pdata: lm3554: reset: %d strobe %d torch %d\n",
-		platform_data.gpio_reset, platform_data.gpio_strobe,
-		platform_data.gpio_torch);
-
-	/* Set to TX2 mode, then ENVM/TX2 pin is a power amplifier sync input:
-	 * ENVM/TX pin asserted, flash forced into torch;
-	 * ENVM/TX pin desserted, flash set back;
-	 */
-	platform_data.envm_tx2 = 1;
-	platform_data.tx2_polarity = 0;
-
-	/* set peak current limit to be 1000mA */
-	platform_data.current_limit = 0;
-
-	return &platform_data;
-}
-
-static int lm3554_probe(struct i2c_client *client)
-{
-	int err = 0;
-	struct lm3554 *flash;
-	unsigned int i;
-	int ret;
-
-	flash = kzalloc(sizeof(*flash), GFP_KERNEL);
-	if (!flash)
-		return -ENOMEM;
-
-	flash->pdata = lm3554_platform_data_func(client);
-
-	v4l2_i2c_subdev_init(&flash->sd, client, &lm3554_ops);
-	flash->sd.internal_ops = &lm3554_internal_ops;
-	flash->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	flash->mode = ATOMISP_FLASH_MODE_OFF;
-	flash->timeout = LM3554_MAX_TIMEOUT / LM3554_TIMEOUT_STEPSIZE - 1;
-	ret =
-	    v4l2_ctrl_handler_init(&flash->ctrl_handler,
-				   ARRAY_SIZE(lm3554_controls));
-	if (ret) {
-		dev_err(&client->dev, "error initialize a ctrl_handler.\n");
-		goto fail2;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(lm3554_controls); i++)
-		v4l2_ctrl_new_custom(&flash->ctrl_handler, &lm3554_controls[i],
-				     NULL);
-
-	if (flash->ctrl_handler.error) {
-
-		dev_err(&client->dev, "ctrl_handler error.\n");
-		goto fail2;
-	}
-
-	flash->sd.ctrl_handler = &flash->ctrl_handler;
-	err = media_entity_pads_init(&flash->sd.entity, 0, NULL);
-	if (err) {
-		dev_err(&client->dev, "error initialize a media entity.\n");
-		goto fail1;
-	}
-
-	flash->sd.entity.function = MEDIA_ENT_F_FLASH;
-
-	mutex_init(&flash->power_lock);
-
-	timer_setup(&flash->flash_off_delay, lm3554_flash_off_delay, 0);
-
-	err = lm3554_gpio_init(client);
-	if (err) {
-		dev_err(&client->dev, "gpio request/direction_output fail");
-		goto fail2;
-	}
-	return atomisp_register_i2c_module(&flash->sd, NULL, LED_FLASH);
-fail2:
-	media_entity_cleanup(&flash->sd.entity);
-	v4l2_ctrl_handler_free(&flash->ctrl_handler);
-fail1:
-	v4l2_device_unregister_subdev(&flash->sd);
-	kfree(flash);
-
-	return err;
-}
-
-static int lm3554_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct lm3554 *flash = to_lm3554(sd);
-	int ret;
-
-	media_entity_cleanup(&flash->sd.entity);
-	v4l2_ctrl_handler_free(&flash->ctrl_handler);
-	v4l2_device_unregister_subdev(sd);
-
-	atomisp_gmin_remove_subdev(sd);
-
-	del_timer_sync(&flash->flash_off_delay);
-
-	ret = lm3554_gpio_uninit(client);
-	if (ret < 0)
-		goto fail;
-
-	kfree(flash);
-
-	return 0;
-fail:
-	dev_err(&client->dev, "gpio request/direction_output fail");
-	return ret;
-}
-
-static const struct dev_pm_ops lm3554_pm_ops = {
-	.suspend = lm3554_suspend,
-	.resume = lm3554_resume,
-};
-
-static const struct acpi_device_id lm3554_acpi_match[] = {
-	{ "INTCF1C" },
-	{},
-};
-MODULE_DEVICE_TABLE(acpi, lm3554_acpi_match);
-
-static struct i2c_driver lm3554_driver = {
-	.driver = {
-		.name = "lm3554",
-		.pm   = &lm3554_pm_ops,
-		.acpi_match_table = lm3554_acpi_match,
-	},
-	.probe_new = lm3554_probe,
-	.remove = lm3554_remove,
-};
-module_i2c_driver(lm3554_driver);
-
-MODULE_AUTHOR("Jing Tao <jing.tao@intel.com>");
-MODULE_DESCRIPTION("LED flash driver for LM3554");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c b/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c
deleted file mode 100644
index 44db9f9f1fc5..000000000000
--- a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c
+++ /dev/null
@@ -1,1917 +0,0 @@
-/*
- * Support for mt9m114 Camera Sensor.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/string.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/kmod.h>
-#include <linux/device.h>
-#include <linux/fs.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-#include <linux/i2c.h>
-#include <linux/acpi.h>
-#include "../include/linux/atomisp_gmin_platform.h"
-#include <media/v4l2-device.h>
-
-#include "mt9m114.h"
-
-#define to_mt9m114_sensor(sd) container_of(sd, struct mt9m114_device, sd)
-
-/*
- * TODO: use debug parameter to actually define when debug messages should
- * be printed.
- */
-static int debug;
-static int aaalock;
-module_param(debug, int, 0644);
-MODULE_PARM_DESC(debug, "Debug level (0-1)");
-
-static int mt9m114_t_vflip(struct v4l2_subdev *sd, int value);
-static int mt9m114_t_hflip(struct v4l2_subdev *sd, int value);
-static int mt9m114_wait_state(struct i2c_client *client, int timeout);
-
-static int
-mt9m114_read_reg(struct i2c_client *client, u16 data_length, u32 reg, u32 *val)
-{
-	int err;
-	struct i2c_msg msg[2];
-	unsigned char data[4];
-
-	if (!client->adapter) {
-		v4l2_err(client, "%s error, no client->adapter\n", __func__);
-		return -ENODEV;
-	}
-
-	if (data_length != MISENSOR_8BIT && data_length != MISENSOR_16BIT
-					 && data_length != MISENSOR_32BIT) {
-		v4l2_err(client, "%s error, invalid data length\n", __func__);
-		return -EINVAL;
-	}
-
-	msg[0].addr = client->addr;
-	msg[0].flags = 0;
-	msg[0].len = MSG_LEN_OFFSET;
-	msg[0].buf = data;
-
-	/* high byte goes out first */
-	data[0] = (u16) (reg >> 8);
-	data[1] = (u16) (reg & 0xff);
-
-	msg[1].addr = client->addr;
-	msg[1].len = data_length;
-	msg[1].flags = I2C_M_RD;
-	msg[1].buf = data;
-
-	err = i2c_transfer(client->adapter, msg, 2);
-
-	if (err >= 0) {
-		*val = 0;
-		/* high byte comes first */
-		if (data_length == MISENSOR_8BIT)
-			*val = data[0];
-		else if (data_length == MISENSOR_16BIT)
-			*val = data[1] + (data[0] << 8);
-		else
-			*val = data[3] + (data[2] << 8) +
-			    (data[1] << 16) + (data[0] << 24);
-
-		return 0;
-	}
-
-	dev_err(&client->dev, "read from offset 0x%x error %d", reg, err);
-	return err;
-}
-
-static int
-mt9m114_write_reg(struct i2c_client *client, u16 data_length, u16 reg, u32 val)
-{
-	int num_msg;
-	struct i2c_msg msg;
-	unsigned char data[6] = {0};
-	__be16 *wreg;
-	int retry = 0;
-
-	if (!client->adapter) {
-		v4l2_err(client, "%s error, no client->adapter\n", __func__);
-		return -ENODEV;
-	}
-
-	if (data_length != MISENSOR_8BIT && data_length != MISENSOR_16BIT
-					 && data_length != MISENSOR_32BIT) {
-		v4l2_err(client, "%s error, invalid data_length\n", __func__);
-		return -EINVAL;
-	}
-
-	memset(&msg, 0, sizeof(msg));
-
-again:
-	msg.addr = client->addr;
-	msg.flags = 0;
-	msg.len = 2 + data_length;
-	msg.buf = data;
-
-	/* high byte goes out first */
-	wreg = (void *)data;
-	*wreg = cpu_to_be16(reg);
-
-	if (data_length == MISENSOR_8BIT) {
-		data[2] = (u8)(val);
-	} else if (data_length == MISENSOR_16BIT) {
-		u16 *wdata = (void *)&data[2];
-
-		*wdata = be16_to_cpu(*(__be16 *)&data[2]);
-	} else {
-		/* MISENSOR_32BIT */
-		u32 *wdata = (void *)&data[2];
-
-		*wdata = be32_to_cpu(*(__be32 *)&data[2]);
-	}
-
-	num_msg = i2c_transfer(client->adapter, &msg, 1);
-
-	/*
-	 * HACK: Need some delay here for Rev 2 sensors otherwise some
-	 * registers do not seem to load correctly.
-	 */
-	mdelay(1);
-
-	if (num_msg >= 0)
-		return 0;
-
-	dev_err(&client->dev, "write error: wrote 0x%x to offset 0x%x error %d",
-		val, reg, num_msg);
-	if (retry <= I2C_RETRY_COUNT) {
-		dev_dbg(&client->dev, "retrying... %d", retry);
-		retry++;
-		msleep(20);
-		goto again;
-	}
-
-	return num_msg;
-}
-
-/**
- * misensor_rmw_reg - Read/Modify/Write a value to a register in the sensor
- * device
- * @client: i2c driver client structure
- * @data_length: 8/16/32-bits length
- * @reg: register address
- * @mask: masked out bits
- * @set: bits set
- *
- * Read/modify/write a value to a register in the  sensor device.
- * Returns zero if successful, or non-zero otherwise.
- */
-static int
-misensor_rmw_reg(struct i2c_client *client, u16 data_length, u16 reg,
-		     u32 mask, u32 set)
-{
-	int err;
-	u32 val;
-
-	/* Exit when no mask */
-	if (mask == 0)
-		return 0;
-
-	/* @mask must not exceed data length */
-	switch (data_length) {
-	case MISENSOR_8BIT:
-		if (mask & ~0xff)
-			return -EINVAL;
-		break;
-	case MISENSOR_16BIT:
-		if (mask & ~0xffff)
-			return -EINVAL;
-		break;
-	case MISENSOR_32BIT:
-		break;
-	default:
-		/* Wrong @data_length */
-		return -EINVAL;
-	}
-
-	err = mt9m114_read_reg(client, data_length, reg, &val);
-	if (err) {
-		v4l2_err(client, "misensor_rmw_reg error exit, read failed\n");
-		return -EINVAL;
-	}
-
-	val &= ~mask;
-
-	/*
-	 * Perform the OR function if the @set exists.
-	 * Shift @set value to target bit location. @set should set only
-	 * bits included in @mask.
-	 *
-	 * REVISIT: This function expects @set to be non-shifted. Its shift
-	 * value is then defined to be equal to mask's LSB position.
-	 * How about to inform values in their right offset position and avoid
-	 * this unneeded shift operation?
-	 */
-	set <<= ffs(mask) - 1;
-	val |= set & mask;
-
-	err = mt9m114_write_reg(client, data_length, reg, val);
-	if (err) {
-		v4l2_err(client, "misensor_rmw_reg error exit, write failed\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-
-static int __mt9m114_flush_reg_array(struct i2c_client *client,
-				     struct mt9m114_write_ctrl *ctrl)
-{
-	struct i2c_msg msg;
-	const int num_msg = 1;
-	int ret;
-	int retry = 0;
-	__be16 *data16 = (void *)&ctrl->buffer.addr;
-
-	if (ctrl->index == 0)
-		return 0;
-
-again:
-	msg.addr = client->addr;
-	msg.flags = 0;
-	msg.len = 2 + ctrl->index;
-	*data16 = cpu_to_be16(ctrl->buffer.addr);
-	msg.buf = (u8 *)&ctrl->buffer;
-
-	ret = i2c_transfer(client->adapter, &msg, num_msg);
-	if (ret != num_msg) {
-		if (++retry <= I2C_RETRY_COUNT) {
-			dev_dbg(&client->dev, "retrying... %d\n", retry);
-			msleep(20);
-			goto again;
-		}
-		dev_err(&client->dev, "%s: i2c transfer error\n", __func__);
-		return -EIO;
-	}
-
-	ctrl->index = 0;
-
-	/*
-	 * REVISIT: Previously we had a delay after writing data to sensor.
-	 * But it was removed as our tests have shown it is not necessary
-	 * anymore.
-	 */
-
-	return 0;
-}
-
-static int __mt9m114_buf_reg_array(struct i2c_client *client,
-				   struct mt9m114_write_ctrl *ctrl,
-				   const struct misensor_reg *next)
-{
-	__be16 *data16;
-	__be32 *data32;
-	int err;
-
-	/* Insufficient buffer? Let's flush and get more free space. */
-	if (ctrl->index + next->length >= MT9M114_MAX_WRITE_BUF_SIZE) {
-		err = __mt9m114_flush_reg_array(client, ctrl);
-		if (err)
-			return err;
-	}
-
-	switch (next->length) {
-	case MISENSOR_8BIT:
-		ctrl->buffer.data[ctrl->index] = (u8)next->val;
-		break;
-	case MISENSOR_16BIT:
-		data16 = (__be16 *)&ctrl->buffer.data[ctrl->index];
-		*data16 = cpu_to_be16((u16)next->val);
-		break;
-	case MISENSOR_32BIT:
-		data32 = (__be32 *)&ctrl->buffer.data[ctrl->index];
-		*data32 = cpu_to_be32(next->val);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* When first item is added, we need to store its starting address */
-	if (ctrl->index == 0)
-		ctrl->buffer.addr = next->reg;
-
-	ctrl->index += next->length;
-
-	return 0;
-}
-
-static int
-__mt9m114_write_reg_is_consecutive(struct i2c_client *client,
-				   struct mt9m114_write_ctrl *ctrl,
-				   const struct misensor_reg *next)
-{
-	if (ctrl->index == 0)
-		return 1;
-
-	return ctrl->buffer.addr + ctrl->index == next->reg;
-}
-
-/*
- * mt9m114_write_reg_array - Initializes a list of mt9m114 registers
- * @client: i2c driver client structure
- * @reglist: list of registers to be written
- * @poll: completion polling requirement
- * This function initializes a list of registers. When consecutive addresses
- * are found in a row on the list, this function creates a buffer and sends
- * consecutive data in a single i2c_transfer().
- *
- * __mt9m114_flush_reg_array, __mt9m114_buf_reg_array() and
- * __mt9m114_write_reg_is_consecutive() are internal functions to
- * mt9m114_write_reg_array() and should be not used anywhere else.
- *
- */
-static int mt9m114_write_reg_array(struct i2c_client *client,
-				const struct misensor_reg *reglist,
-				int poll)
-{
-	const struct misensor_reg *next = reglist;
-	struct mt9m114_write_ctrl ctrl;
-	int err;
-
-	if (poll == PRE_POLLING) {
-		err = mt9m114_wait_state(client, MT9M114_WAIT_STAT_TIMEOUT);
-		if (err)
-			return err;
-	}
-
-	ctrl.index = 0;
-	for (; next->length != MISENSOR_TOK_TERM; next++) {
-		switch (next->length & MISENSOR_TOK_MASK) {
-		case MISENSOR_TOK_DELAY:
-			err = __mt9m114_flush_reg_array(client, &ctrl);
-			if (err)
-				return err;
-			msleep(next->val);
-			break;
-		case MISENSOR_TOK_RMW:
-			err = __mt9m114_flush_reg_array(client, &ctrl);
-			err |= misensor_rmw_reg(client,
-						next->length &
-							~MISENSOR_TOK_RMW,
-						next->reg, next->val,
-						next->val2);
-			if (err) {
-				dev_err(&client->dev, "%s read err. aborted\n",
-					__func__);
-				return -EINVAL;
-			}
-			break;
-		default:
-			/*
-			 * If next address is not consecutive, data needs to be
-			 * flushed before proceed.
-			 */
-			if (!__mt9m114_write_reg_is_consecutive(client, &ctrl,
-								next)) {
-				err = __mt9m114_flush_reg_array(client, &ctrl);
-				if (err)
-					return err;
-			}
-			err = __mt9m114_buf_reg_array(client, &ctrl, next);
-			if (err) {
-				v4l2_err(client, "%s: write error, aborted\n",
-					 __func__);
-				return err;
-			}
-			break;
-		}
-	}
-
-	err = __mt9m114_flush_reg_array(client, &ctrl);
-	if (err)
-		return err;
-
-	if (poll == POST_POLLING)
-		return mt9m114_wait_state(client, MT9M114_WAIT_STAT_TIMEOUT);
-
-	return 0;
-}
-
-static int mt9m114_wait_state(struct i2c_client *client, int timeout)
-{
-	int ret;
-	unsigned int val;
-
-	while (timeout-- > 0) {
-		ret = mt9m114_read_reg(client, MISENSOR_16BIT, 0x0080, &val);
-		if (ret)
-			return ret;
-		if ((val & 0x2) == 0)
-			return 0;
-		msleep(20);
-	}
-
-	return -EINVAL;
-
-}
-
-static int mt9m114_set_suspend(struct v4l2_subdev *sd)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	return mt9m114_write_reg_array(client,
-			mt9m114_standby_reg, POST_POLLING);
-}
-
-static int mt9m114_init_common(struct v4l2_subdev *sd)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-
-	return mt9m114_write_reg_array(client, mt9m114_common, PRE_POLLING);
-}
-
-static int power_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	int ret;
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	if (flag) {
-		ret = dev->platform_data->v2p8_ctrl(sd, 1);
-		if (ret == 0) {
-			ret = dev->platform_data->v1p8_ctrl(sd, 1);
-			if (ret)
-				ret = dev->platform_data->v2p8_ctrl(sd, 0);
-		}
-	} else {
-		ret = dev->platform_data->v2p8_ctrl(sd, 0);
-		ret = dev->platform_data->v1p8_ctrl(sd, 0);
-	}
-	return ret;
-}
-
-static int gpio_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	int ret;
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	/* Note: current modules wire only one GPIO signal (RESET#),
-	 * but the schematic wires up two to the connector.  BIOS
-	 * versions have been unfortunately inconsistent with which
-	 * ACPI index RESET# is on, so hit both */
-
-	if (flag) {
-		ret = dev->platform_data->gpio0_ctrl(sd, 0);
-		ret = dev->platform_data->gpio1_ctrl(sd, 0);
-		msleep(60);
-		ret |= dev->platform_data->gpio0_ctrl(sd, 1);
-		ret |= dev->platform_data->gpio1_ctrl(sd, 1);
-	} else {
-		ret = dev->platform_data->gpio0_ctrl(sd, 0);
-		ret = dev->platform_data->gpio1_ctrl(sd, 0);
-	}
-	return ret;
-}
-
-static int power_up(struct v4l2_subdev *sd)
-{
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	if (NULL == dev->platform_data) {
-		dev_err(&client->dev, "no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	/* power control */
-	ret = power_ctrl(sd, 1);
-	if (ret)
-		goto fail_power;
-
-	/* flis clock control */
-	ret = dev->platform_data->flisclk_ctrl(sd, 1);
-	if (ret)
-		goto fail_clk;
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 1);
-	if (ret)
-		dev_err(&client->dev, "gpio failed 1\n");
-	/*
-	 * according to DS, 44ms is needed between power up and first i2c
-	 * commend
-	 */
-	msleep(50);
-
-	return 0;
-
-fail_clk:
-	dev->platform_data->flisclk_ctrl(sd, 0);
-fail_power:
-	power_ctrl(sd, 0);
-	dev_err(&client->dev, "sensor power-up failed\n");
-
-	return ret;
-}
-
-static int power_down(struct v4l2_subdev *sd)
-{
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	if (NULL == dev->platform_data) {
-		dev_err(&client->dev, "no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	ret = dev->platform_data->flisclk_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "flisclk failed\n");
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "gpio failed 1\n");
-
-	/* power control */
-	ret = power_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "vprog failed.\n");
-
-	/*according to DS, 20ms is needed after power down*/
-	msleep(20);
-
-	return ret;
-}
-
-static int mt9m114_s_power(struct v4l2_subdev *sd, int power)
-{
-	if (power == 0)
-		return power_down(sd);
-	else {
-		if (power_up(sd))
-			return -EINVAL;
-
-		return mt9m114_init_common(sd);
-	}
-}
-
-/*
- * distance - calculate the distance
- * @res: resolution
- * @w: width
- * @h: height
- *
- * Get the gap between resolution and w/h.
- * res->width/height smaller than w/h wouldn't be considered.
- * Returns the value of gap or -1 if fail.
- */
-#define LARGEST_ALLOWED_RATIO_MISMATCH 600
-static int distance(struct mt9m114_res_struct const *res, u32 w, u32 h)
-{
-	unsigned int w_ratio;
-	unsigned int h_ratio;
-	int match;
-
-	if (w == 0)
-		return -1;
-	w_ratio = (res->width << 13) / w;
-	if (h == 0)
-		return -1;
-	h_ratio = (res->height << 13) / h;
-	if (h_ratio == 0)
-		return -1;
-	match   = abs(((w_ratio << 13) / h_ratio) - 8192);
-
-	if ((w_ratio < 8192) || (h_ratio < 8192) ||
-	    (match > LARGEST_ALLOWED_RATIO_MISMATCH))
-		return -1;
-
-	return w_ratio + h_ratio;
-}
-
-/* Return the nearest higher resolution index */
-static int nearest_resolution_index(int w, int h)
-{
-	int i;
-	int idx = -1;
-	int dist;
-	int min_dist = INT_MAX;
-	const struct mt9m114_res_struct *tmp_res = NULL;
-
-	for (i = 0; i < ARRAY_SIZE(mt9m114_res); i++) {
-		tmp_res = &mt9m114_res[i];
-		dist = distance(tmp_res, w, h);
-		if (dist == -1)
-			continue;
-		if (dist < min_dist) {
-			min_dist = dist;
-			idx = i;
-		}
-	}
-
-	return idx;
-}
-
-static int mt9m114_try_res(u32 *w, u32 *h)
-{
-	int idx = 0;
-
-	if ((*w > MT9M114_RES_960P_SIZE_H)
-		|| (*h > MT9M114_RES_960P_SIZE_V)) {
-		*w = MT9M114_RES_960P_SIZE_H;
-		*h = MT9M114_RES_960P_SIZE_V;
-	} else {
-		idx = nearest_resolution_index(*w, *h);
-
-		/*
-		 * nearest_resolution_index() doesn't return smaller
-		 *  resolutions. If it fails, it means the requested
-		 *  resolution is higher than wecan support. Fallback
-		 *  to highest possible resolution in this case.
-		 */
-		if (idx == -1)
-			idx = ARRAY_SIZE(mt9m114_res) - 1;
-
-		*w = mt9m114_res[idx].width;
-		*h = mt9m114_res[idx].height;
-	}
-
-	return 0;
-}
-
-static struct mt9m114_res_struct *mt9m114_to_res(u32 w, u32 h)
-{
-	int  index;
-
-	for (index = 0; index < N_RES; index++) {
-		if ((mt9m114_res[index].width == w) &&
-		    (mt9m114_res[index].height == h))
-			break;
-	}
-
-	/* No mode found */
-	if (index >= N_RES)
-		return NULL;
-
-	return &mt9m114_res[index];
-}
-
-static int mt9m114_res2size(struct v4l2_subdev *sd, int *h_size, int *v_size)
-{
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-	unsigned short hsize;
-	unsigned short vsize;
-
-	switch (dev->res) {
-	case MT9M114_RES_736P:
-		hsize = MT9M114_RES_736P_SIZE_H;
-		vsize = MT9M114_RES_736P_SIZE_V;
-		break;
-	case MT9M114_RES_864P:
-		hsize = MT9M114_RES_864P_SIZE_H;
-		vsize = MT9M114_RES_864P_SIZE_V;
-		break;
-	case MT9M114_RES_960P:
-		hsize = MT9M114_RES_960P_SIZE_H;
-		vsize = MT9M114_RES_960P_SIZE_V;
-		break;
-	default:
-		v4l2_err(sd, "%s: Resolution 0x%08x unknown\n", __func__,
-			 dev->res);
-		return -EINVAL;
-	}
-
-	if (h_size != NULL)
-		*h_size = hsize;
-	if (v_size != NULL)
-		*v_size = vsize;
-
-	return 0;
-}
-
-static int mt9m114_get_intg_factor(struct i2c_client *client,
-				struct camera_mipi_info *info,
-				const struct mt9m114_res_struct *res)
-{
-	struct atomisp_sensor_mode_data *buf = &info->data;
-	u32 reg_val;
-	int ret;
-
-	if (info == NULL)
-		return -EINVAL;
-
-	ret =  mt9m114_read_reg(client, MISENSOR_32BIT,
-					REG_PIXEL_CLK, &reg_val);
-	if (ret)
-		return ret;
-	buf->vt_pix_clk_freq_mhz = reg_val;
-
-	/* get integration time */
-	buf->coarse_integration_time_min = MT9M114_COARSE_INTG_TIME_MIN;
-	buf->coarse_integration_time_max_margin =
-					MT9M114_COARSE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_min = MT9M114_FINE_INTG_TIME_MIN;
-	buf->fine_integration_time_max_margin =
-					MT9M114_FINE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_def = MT9M114_FINE_INTG_TIME_MIN;
-
-	buf->frame_length_lines = res->lines_per_frame;
-	buf->line_length_pck = res->pixels_per_line;
-	buf->read_mode = res->bin_mode;
-
-	/* get the cropping and output resolution to ISP for this mode. */
-	ret =  mt9m114_read_reg(client, MISENSOR_16BIT,
-					REG_H_START, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_horizontal_start = reg_val;
-
-	ret =  mt9m114_read_reg(client, MISENSOR_16BIT,
-					REG_V_START, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_vertical_start = reg_val;
-
-	ret = mt9m114_read_reg(client, MISENSOR_16BIT,
-					REG_H_END, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_horizontal_end = reg_val;
-
-	ret = mt9m114_read_reg(client, MISENSOR_16BIT,
-					REG_V_END, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_vertical_end = reg_val;
-
-	ret = mt9m114_read_reg(client, MISENSOR_16BIT,
-					REG_WIDTH, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_width = reg_val;
-
-	ret = mt9m114_read_reg(client, MISENSOR_16BIT,
-					REG_HEIGHT, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_height = reg_val;
-
-	ret = mt9m114_read_reg(client, MISENSOR_16BIT,
-					REG_TIMING_HTS, &reg_val);
-	if (ret)
-		return ret;
-	buf->line_length_pck = reg_val;
-
-	ret = mt9m114_read_reg(client, MISENSOR_16BIT,
-					REG_TIMING_VTS, &reg_val);
-	if (ret)
-		return ret;
-	buf->frame_length_lines = reg_val;
-
-	buf->binning_factor_x = res->bin_factor_x ?
-					res->bin_factor_x : 1;
-	buf->binning_factor_y = res->bin_factor_y ?
-					res->bin_factor_y : 1;
-	return 0;
-}
-
-static int mt9m114_get_fmt(struct v4l2_subdev *sd,
-				struct v4l2_subdev_pad_config *cfg,
-				struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	int width, height;
-	int ret;
-	if (format->pad)
-		return -EINVAL;
-	fmt->code = MEDIA_BUS_FMT_SGRBG10_1X10;
-
-	ret = mt9m114_res2size(sd, &width, &height);
-	if (ret)
-		return ret;
-	fmt->width = width;
-	fmt->height = height;
-
-	return 0;
-}
-
-static int mt9m114_set_fmt(struct v4l2_subdev *sd,
-			   struct v4l2_subdev_pad_config *cfg,
-			   struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct i2c_client *c = v4l2_get_subdevdata(sd);
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-	struct mt9m114_res_struct *res_index;
-	u32 width = fmt->width;
-	u32 height = fmt->height;
-	struct camera_mipi_info *mt9m114_info = NULL;
-
-	int ret;
-	if (format->pad)
-		return -EINVAL;
-	dev->streamon = 0;
-	dev->first_exp = MT9M114_DEFAULT_FIRST_EXP;
-
-	mt9m114_info = v4l2_get_subdev_hostdata(sd);
-	if (mt9m114_info == NULL)
-		return -EINVAL;
-
-	mt9m114_try_res(&width, &height);
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
-		cfg->try_fmt = *fmt;
-		return 0;
-	}
-	res_index = mt9m114_to_res(width, height);
-
-	/* Sanity check */
-	if (unlikely(!res_index)) {
-		WARN_ON(1);
-		return -EINVAL;
-	}
-
-	switch (res_index->res) {
-	case MT9M114_RES_736P:
-		ret = mt9m114_write_reg_array(c, mt9m114_736P_init, NO_POLLING);
-		ret += misensor_rmw_reg(c, MISENSOR_16BIT, MISENSOR_READ_MODE,
-				MISENSOR_R_MODE_MASK, MISENSOR_NORMAL_SET);
-		break;
-	case MT9M114_RES_864P:
-		ret = mt9m114_write_reg_array(c, mt9m114_864P_init, NO_POLLING);
-		ret += misensor_rmw_reg(c, MISENSOR_16BIT, MISENSOR_READ_MODE,
-				MISENSOR_R_MODE_MASK, MISENSOR_NORMAL_SET);
-		break;
-	case MT9M114_RES_960P:
-		ret = mt9m114_write_reg_array(c, mt9m114_976P_init, NO_POLLING);
-		/* set sensor read_mode to Normal */
-		ret += misensor_rmw_reg(c, MISENSOR_16BIT, MISENSOR_READ_MODE,
-				MISENSOR_R_MODE_MASK, MISENSOR_NORMAL_SET);
-		break;
-	default:
-		v4l2_err(sd, "set resolution: %d failed!\n", res_index->res);
-		return -EINVAL;
-	}
-
-	if (ret)
-		return -EINVAL;
-
-	ret = mt9m114_write_reg_array(c, mt9m114_chgstat_reg, POST_POLLING);
-	if (ret < 0)
-		return ret;
-
-	if (mt9m114_set_suspend(sd))
-		return -EINVAL;
-
-	if (dev->res != res_index->res) {
-		int index;
-
-		/* Switch to different size */
-		if (width <= 640) {
-			dev->nctx = 0x00; /* Set for context A */
-		} else {
-			/*
-			 * Context B is used for resolutions larger than 640x480
-			 * Using YUV for Context B.
-			 */
-			dev->nctx = 0x01; /* set for context B */
-		}
-
-		/*
-		 * Marked current sensor res as being "used"
-		 *
-		 * REVISIT: We don't need to use an "used" field on each mode
-		 * list entry to know which mode is selected. If this
-		 * information is really necessary, how about to use a single
-		 * variable on sensor dev struct?
-		 */
-		for (index = 0; index < N_RES; index++) {
-			if ((width == mt9m114_res[index].width) &&
-			    (height == mt9m114_res[index].height)) {
-				mt9m114_res[index].used = true;
-				continue;
-			}
-			mt9m114_res[index].used = false;
-		}
-	}
-	ret = mt9m114_get_intg_factor(c, mt9m114_info,
-					&mt9m114_res[res_index->res]);
-	if (ret) {
-		dev_err(&c->dev, "failed to get integration_factor\n");
-		return -EINVAL;
-	}
-	/*
-	 * mt9m114 - we don't poll for context switch
-	 * because it does not happen with streaming disabled.
-	 */
-	dev->res = res_index->res;
-
-	fmt->width = width;
-	fmt->height = height;
-	fmt->code = MEDIA_BUS_FMT_SGRBG10_1X10;
-	return 0;
-}
-
-/* TODO: Update to SOC functions, remove exposure and gain */
-static int mt9m114_g_focal(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = (MT9M114_FOCAL_LENGTH_NUM << 16) | MT9M114_FOCAL_LENGTH_DEM;
-	return 0;
-}
-
-static int mt9m114_g_fnumber(struct v4l2_subdev *sd, s32 *val)
-{
-	/*const f number for mt9m114*/
-	*val = (MT9M114_F_NUMBER_DEFAULT_NUM << 16) | MT9M114_F_NUMBER_DEM;
-	return 0;
-}
-
-static int mt9m114_g_fnumber_range(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = (MT9M114_F_NUMBER_DEFAULT_NUM << 24) |
-		(MT9M114_F_NUMBER_DEM << 16) |
-		(MT9M114_F_NUMBER_DEFAULT_NUM << 8) | MT9M114_F_NUMBER_DEM;
-	return 0;
-}
-
-/* Horizontal flip the image. */
-static int mt9m114_g_hflip(struct v4l2_subdev *sd, s32 *val)
-{
-	struct i2c_client *c = v4l2_get_subdevdata(sd);
-	int ret;
-	u32 data;
-	ret = mt9m114_read_reg(c, MISENSOR_16BIT,
-			(u32)MISENSOR_READ_MODE, &data);
-	if (ret)
-		return ret;
-	*val = !!(data & MISENSOR_HFLIP_MASK);
-
-	return 0;
-}
-
-static int mt9m114_g_vflip(struct v4l2_subdev *sd, s32 *val)
-{
-	struct i2c_client *c = v4l2_get_subdevdata(sd);
-	int ret;
-	u32 data;
-
-	ret = mt9m114_read_reg(c, MISENSOR_16BIT,
-			(u32)MISENSOR_READ_MODE, &data);
-	if (ret)
-		return ret;
-	*val = !!(data & MISENSOR_VFLIP_MASK);
-
-	return 0;
-}
-
-static long mt9m114_s_exposure(struct v4l2_subdev *sd,
-			       struct atomisp_exposure *exposure)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-	int ret = 0;
-	unsigned int coarse_integration = 0;
-	unsigned int fine_integration = 0;
-	unsigned int FLines = 0;
-	unsigned int FrameLengthLines = 0; /* ExposureTime.FrameLengthLines; */
-	unsigned int AnalogGain, DigitalGain;
-	u32 AnalogGainToWrite = 0;
-	u16 exposure_local[3];
-
-	dev_dbg(&client->dev, "%s(0x%X 0x%X 0x%X)\n", __func__,
-		    exposure->integration_time[0], exposure->gain[0],
-		    exposure->gain[1]);
-
-	coarse_integration = exposure->integration_time[0];
-	/* fine_integration = ExposureTime.FineIntegrationTime; */
-	/* FrameLengthLines = ExposureTime.FrameLengthLines; */
-	FLines = mt9m114_res[dev->res].lines_per_frame;
-	AnalogGain = exposure->gain[0];
-	DigitalGain = exposure->gain[1];
-	if (!dev->streamon) {
-		/*Save the first exposure values while stream is off*/
-		dev->first_exp = coarse_integration;
-		dev->first_gain = AnalogGain;
-		dev->first_diggain = DigitalGain;
-	}
-	/* DigitalGain = 0x400 * (((u16) DigitalGain) >> 8) +
-	((unsigned int)(0x400 * (((u16) DigitalGain) & 0xFF)) >>8); */
-
-	/* set frame length */
-	if (FLines < coarse_integration + 6)
-		FLines = coarse_integration + 6;
-	if (FLines < FrameLengthLines)
-		FLines = FrameLengthLines;
-	ret = mt9m114_write_reg(client, MISENSOR_16BIT, 0x300A, FLines);
-	if (ret) {
-		v4l2_err(client, "%s: fail to set FLines\n", __func__);
-		return -EINVAL;
-	}
-
-	/* set coarse/fine integration */
-	exposure_local[0] = REG_EXPO_COARSE;
-	exposure_local[1] = (u16)coarse_integration;
-	exposure_local[2] = (u16)fine_integration;
-	/* 3A provide real exposure time.
-		should not translate to any value here. */
-	ret = mt9m114_write_reg(client, MISENSOR_16BIT,
-			REG_EXPO_COARSE, (u16)(coarse_integration));
-	if (ret) {
-		v4l2_err(client, "%s: fail to set exposure time\n", __func__);
-		return -EINVAL;
-	}
-
-	/*
-	// set analog/digital gain
-	switch(AnalogGain)
-	{
-	case 0:
-	  AnalogGainToWrite = 0x0;
-	  break;
-	case 1:
-	  AnalogGainToWrite = 0x20;
-	  break;
-	case 2:
-	  AnalogGainToWrite = 0x60;
-	  break;
-	case 4:
-	  AnalogGainToWrite = 0xA0;
-	  break;
-	case 8:
-	  AnalogGainToWrite = 0xE0;
-	  break;
-	default:
-	  AnalogGainToWrite = 0x20;
-	  break;
-	}
-	*/
-	if (DigitalGain >= 16 || DigitalGain <= 1)
-		DigitalGain = 1;
-	/* AnalogGainToWrite =
-		(u16)((DigitalGain << 12) | AnalogGainToWrite); */
-	AnalogGainToWrite = (u16)((DigitalGain << 12) | (u16)AnalogGain);
-	ret = mt9m114_write_reg(client, MISENSOR_16BIT,
-					REG_GAIN, AnalogGainToWrite);
-	if (ret) {
-		v4l2_err(client, "%s: fail to set AnalogGainToWrite\n",
-			__func__);
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-static long mt9m114_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
-{
-
-	switch (cmd) {
-	case ATOMISP_IOC_S_EXPOSURE:
-		return mt9m114_s_exposure(sd, arg);
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/* This returns the exposure time being used. This should only be used
-   for filling in EXIF data, not for actual image processing. */
-static int mt9m114_g_exposure(struct v4l2_subdev *sd, s32 *value)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u32 coarse;
-	int ret;
-
-	/* the fine integration time is currently not calculated */
-	ret = mt9m114_read_reg(client, MISENSOR_16BIT,
-			       REG_EXPO_COARSE, &coarse);
-	if (ret)
-		return ret;
-
-	*value = coarse;
-	return 0;
-}
-#ifndef CSS15
-/*
- * This function will return the sensor supported max exposure zone number.
- * the sensor which supports max exposure zone number is 1.
- */
-static int mt9m114_g_exposure_zone_num(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = 1;
-
-	return 0;
-}
-
-/*
- * set exposure metering, average/center_weighted/spot/matrix.
- */
-static int mt9m114_s_exposure_metering(struct v4l2_subdev *sd, s32 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	switch (val) {
-	case V4L2_EXPOSURE_METERING_SPOT:
-		ret = mt9m114_write_reg_array(client, mt9m114_exp_average,
-						NO_POLLING);
-		if (ret) {
-			dev_err(&client->dev, "write exp_average reg err.\n");
-			return ret;
-		}
-		break;
-	case V4L2_EXPOSURE_METERING_CENTER_WEIGHTED:
-	default:
-		ret = mt9m114_write_reg_array(client, mt9m114_exp_center,
-						NO_POLLING);
-		if (ret) {
-			dev_err(&client->dev, "write exp_default reg err");
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-/*
- * This function is for touch exposure feature.
- */
-static int mt9m114_s_exposure_selection(struct v4l2_subdev *sd,
-					struct v4l2_subdev_pad_config *cfg,
-					struct v4l2_subdev_selection *sel)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct misensor_reg exp_reg;
-	int width, height;
-	int grid_width, grid_height;
-	int grid_left, grid_top, grid_right, grid_bottom;
-	int win_left, win_top, win_right, win_bottom;
-	int i, j;
-	int ret;
-
-	if (sel->which != V4L2_SUBDEV_FORMAT_TRY &&
-	    sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
-		return -EINVAL;
-
-	grid_left = sel->r.left;
-	grid_top = sel->r.top;
-	grid_right = sel->r.left + sel->r.width - 1;
-	grid_bottom = sel->r.top + sel->r.height - 1;
-
-	ret = mt9m114_res2size(sd, &width, &height);
-	if (ret)
-		return ret;
-
-	grid_width = width / 5;
-	grid_height = height / 5;
-
-	if (grid_width && grid_height) {
-		win_left = grid_left / grid_width;
-		win_top = grid_top / grid_height;
-		win_right = grid_right / grid_width;
-		win_bottom = grid_bottom / grid_height;
-	} else {
-		dev_err(&client->dev, "Incorrect exp grid.\n");
-		return -EINVAL;
-	}
-
-	win_left   = clamp_t(int, win_left, 0, 4);
-	win_top    = clamp_t(int, win_top, 0, 4);
-	win_right  = clamp_t(int, win_right, 0, 4);
-	win_bottom = clamp_t(int, win_bottom, 0, 4);
-
-	ret = mt9m114_write_reg_array(client, mt9m114_exp_average, NO_POLLING);
-	if (ret) {
-		dev_err(&client->dev, "write exp_average reg err.\n");
-		return ret;
-	}
-
-	for (i = win_top; i <= win_bottom; i++) {
-		for (j = win_left; j <= win_right; j++) {
-			exp_reg = mt9m114_exp_win[i][j];
-
-			ret = mt9m114_write_reg(client, exp_reg.length,
-						exp_reg.reg, exp_reg.val);
-			if (ret) {
-				dev_err(&client->dev, "write exp_reg err.\n");
-				return ret;
-			}
-		}
-	}
-
-	return 0;
-}
-#endif
-
-static int mt9m114_g_bin_factor_x(struct v4l2_subdev *sd, s32 *val)
-{
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-
-	*val = mt9m114_res[dev->res].bin_factor_x;
-
-	return 0;
-}
-
-static int mt9m114_g_bin_factor_y(struct v4l2_subdev *sd, s32 *val)
-{
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-
-	*val = mt9m114_res[dev->res].bin_factor_y;
-
-	return 0;
-}
-
-static int mt9m114_s_ev(struct v4l2_subdev *sd, s32 val)
-{
-	struct i2c_client *c = v4l2_get_subdevdata(sd);
-	s32 luma = 0x37;
-	int err;
-
-	/* EV value only support -2 to 2
-	 * 0: 0x37, 1:0x47, 2:0x57, -1:0x27, -2:0x17
-	 */
-	if (val < -2 || val > 2)
-		return -EINVAL;
-	luma += 0x10 * val;
-	dev_dbg(&c->dev, "%s val:%d luma:0x%x\n", __func__, val, luma);
-	err = mt9m114_write_reg(c, MISENSOR_16BIT, 0x098E, 0xC87A);
-	if (err) {
-		dev_err(&c->dev, "%s logic addr access error\n", __func__);
-		return err;
-	}
-	err = mt9m114_write_reg(c, MISENSOR_8BIT, 0xC87A, (u32)luma);
-	if (err) {
-		dev_err(&c->dev, "%s write target_average_luma failed\n",
-			__func__);
-		return err;
-	}
-	udelay(10);
-
-	return 0;
-}
-
-static int mt9m114_g_ev(struct v4l2_subdev *sd, s32 *val)
-{
-	struct i2c_client *c = v4l2_get_subdevdata(sd);
-	int err;
-	u32 luma;
-
-	err = mt9m114_write_reg(c, MISENSOR_16BIT, 0x098E, 0xC87A);
-	if (err) {
-		dev_err(&c->dev, "%s logic addr access error\n", __func__);
-		return err;
-	}
-	err = mt9m114_read_reg(c, MISENSOR_8BIT, 0xC87A, &luma);
-	if (err) {
-		dev_err(&c->dev, "%s read target_average_luma failed\n",
-			__func__);
-		return err;
-	}
-	luma -= 0x17;
-	luma /= 0x10;
-	*val = (s32)luma - 2;
-	dev_dbg(&c->dev, "%s val:%d\n", __func__, *val);
-
-	return 0;
-}
-
-/* Fake interface
- * mt9m114 now can not support 3a_lock
-*/
-static int mt9m114_s_3a_lock(struct v4l2_subdev *sd, s32 val)
-{
-	aaalock = val;
-	return 0;
-}
-
-static int mt9m114_g_3a_lock(struct v4l2_subdev *sd, s32 *val)
-{
-	if (aaalock)
-		return V4L2_LOCK_EXPOSURE | V4L2_LOCK_WHITE_BALANCE
-			| V4L2_LOCK_FOCUS;
-	return 0;
-}
-
-static int mt9m114_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct mt9m114_device *dev =
-	    container_of(ctrl->handler, struct mt9m114_device, ctrl_handler);
-	struct i2c_client *client = v4l2_get_subdevdata(&dev->sd);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_VFLIP:
-		dev_dbg(&client->dev, "%s: CID_VFLIP:%d.\n",
-			__func__, ctrl->val);
-		ret = mt9m114_t_vflip(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_HFLIP:
-		dev_dbg(&client->dev, "%s: CID_HFLIP:%d.\n",
-			__func__, ctrl->val);
-		ret = mt9m114_t_hflip(&dev->sd, ctrl->val);
-		break;
-#ifndef CSS15
-	case V4L2_CID_EXPOSURE_METERING:
-		ret = mt9m114_s_exposure_metering(&dev->sd, ctrl->val);
-		break;
-#endif
-	case V4L2_CID_EXPOSURE:
-		ret = mt9m114_s_ev(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_3A_LOCK:
-		ret = mt9m114_s_3a_lock(&dev->sd, ctrl->val);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-	return ret;
-}
-
-static int mt9m114_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct mt9m114_device *dev =
-	    container_of(ctrl->handler, struct mt9m114_device, ctrl_handler);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_VFLIP:
-		ret = mt9m114_g_vflip(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_HFLIP:
-		ret = mt9m114_g_hflip(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FOCAL_ABSOLUTE:
-		ret = mt9m114_g_focal(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_ABSOLUTE:
-		ret = mt9m114_g_fnumber(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_RANGE:
-		ret = mt9m114_g_fnumber_range(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_EXPOSURE_ABSOLUTE:
-		ret = mt9m114_g_exposure(&dev->sd, &ctrl->val);
-		break;
-#ifndef CSS15
-	case V4L2_CID_EXPOSURE_ZONE_NUM:
-		ret = mt9m114_g_exposure_zone_num(&dev->sd, &ctrl->val);
-		break;
-#endif
-	case V4L2_CID_BIN_FACTOR_HORZ:
-		ret = mt9m114_g_bin_factor_x(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_BIN_FACTOR_VERT:
-		ret = mt9m114_g_bin_factor_y(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_EXPOSURE:
-		ret = mt9m114_g_ev(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_3A_LOCK:
-		ret = mt9m114_g_3a_lock(&dev->sd, &ctrl->val);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-static const struct v4l2_ctrl_ops ctrl_ops = {
-	.s_ctrl = mt9m114_s_ctrl,
-	.g_volatile_ctrl = mt9m114_g_volatile_ctrl
-};
-
-static struct v4l2_ctrl_config mt9m114_controls[] = {
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_VFLIP,
-	 .name = "Image v-Flip",
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .min = 0,
-	 .max = 1,
-	 .step = 1,
-	 .def = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_HFLIP,
-	 .name = "Image h-Flip",
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .min = 0,
-	 .max = 1,
-	 .step = 1,
-	 .def = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FOCAL_ABSOLUTE,
-	 .name = "focal length",
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .min = MT9M114_FOCAL_LENGTH_DEFAULT,
-	 .max = MT9M114_FOCAL_LENGTH_DEFAULT,
-	 .step = 1,
-	 .def = MT9M114_FOCAL_LENGTH_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_ABSOLUTE,
-	 .name = "f-number",
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .min = MT9M114_F_NUMBER_DEFAULT,
-	 .max = MT9M114_F_NUMBER_DEFAULT,
-	 .step = 1,
-	 .def = MT9M114_F_NUMBER_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_RANGE,
-	 .name = "f-number range",
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .min = MT9M114_F_NUMBER_RANGE,
-	 .max = MT9M114_F_NUMBER_RANGE,
-	 .step = 1,
-	 .def = MT9M114_F_NUMBER_RANGE,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_EXPOSURE_ABSOLUTE,
-	 .name = "exposure",
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .min = 0,
-	 .max = 0xffff,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-#ifndef CSS15
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_EXPOSURE_ZONE_NUM,
-	 .name = "one-time exposure zone number",
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .min = 0,
-	 .max = 0xffff,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_EXPOSURE_METERING,
-	 .name = "metering",
-	 .type = V4L2_CTRL_TYPE_MENU,
-	 .min = 0,
-	 .max = 3,
-	 .step = 0,
-	 .def = 1,
-	 .flags = 0,
-	 },
-#endif
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_BIN_FACTOR_HORZ,
-	 .name = "horizontal binning factor",
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .min = 0,
-	 .max = MT9M114_BIN_FACTOR_MAX,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_BIN_FACTOR_VERT,
-	 .name = "vertical binning factor",
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .min = 0,
-	 .max = MT9M114_BIN_FACTOR_MAX,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_EXPOSURE,
-	 .name = "exposure biasx",
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .min = -2,
-	 .max = 2,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_3A_LOCK,
-	 .name = "3a lock",
-	 .type = V4L2_CTRL_TYPE_BITMASK,
-	 .min = 0,
-	 .max = V4L2_LOCK_EXPOSURE | V4L2_LOCK_WHITE_BALANCE | V4L2_LOCK_FOCUS,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-};
-
-static int mt9m114_detect(struct mt9m114_device *dev, struct i2c_client *client)
-{
-	struct i2c_adapter *adapter = client->adapter;
-	u32 retvalue;
-
-	if (!i2c_check_functionality(adapter, I2C_FUNC_I2C)) {
-		dev_err(&client->dev, "%s: i2c error", __func__);
-		return -ENODEV;
-	}
-	mt9m114_read_reg(client, MISENSOR_16BIT, (u32)MT9M114_PID, &retvalue);
-	dev->real_model_id = retvalue;
-
-	if (retvalue != MT9M114_MOD_ID) {
-		dev_err(&client->dev, "%s: failed: client->addr = %x\n",
-			__func__, client->addr);
-		return -ENODEV;
-	}
-
-	return 0;
-}
-
-static int
-mt9m114_s_config(struct v4l2_subdev *sd, int irq, void *platform_data)
-{
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	if (NULL == platform_data)
-		return -ENODEV;
-
-	dev->platform_data =
-	    (struct camera_sensor_platform_data *)platform_data;
-
-	ret = power_up(sd);
-	if (ret) {
-		v4l2_err(client, "mt9m114 power-up err");
-		return ret;
-	}
-
-	/* config & detect sensor */
-	ret = mt9m114_detect(dev, client);
-	if (ret) {
-		v4l2_err(client, "mt9m114_detect err s_config.\n");
-		goto fail_detect;
-	}
-
-	ret = dev->platform_data->csi_cfg(sd, 1);
-	if (ret)
-		goto fail_csi_cfg;
-
-	ret = mt9m114_set_suspend(sd);
-	if (ret) {
-		v4l2_err(client, "mt9m114 suspend err");
-		return ret;
-	}
-
-	ret = power_down(sd);
-	if (ret) {
-		v4l2_err(client, "mt9m114 power down err");
-		return ret;
-	}
-
-	return ret;
-
-fail_csi_cfg:
-	dev->platform_data->csi_cfg(sd, 0);
-fail_detect:
-	power_down(sd);
-	dev_err(&client->dev, "sensor power-gating failed\n");
-	return ret;
-}
-
-/* Horizontal flip the image. */
-static int mt9m114_t_hflip(struct v4l2_subdev *sd, int value)
-{
-	struct i2c_client *c = v4l2_get_subdevdata(sd);
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-	int err;
-	/* set for direct mode */
-	err = mt9m114_write_reg(c, MISENSOR_16BIT, 0x098E, 0xC850);
-	if (value) {
-		/* enable H flip ctx A */
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC850, 0x01, 0x01);
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC851, 0x01, 0x01);
-		/* ctx B */
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC888, 0x01, 0x01);
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC889, 0x01, 0x01);
-
-		err += misensor_rmw_reg(c, MISENSOR_16BIT, MISENSOR_READ_MODE,
-					MISENSOR_HFLIP_MASK, MISENSOR_FLIP_EN);
-
-		dev->bpat = MT9M114_BPAT_GRGRBGBG;
-	} else {
-		/* disable H flip ctx A */
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC850, 0x01, 0x00);
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC851, 0x01, 0x00);
-		/* ctx B */
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC888, 0x01, 0x00);
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC889, 0x01, 0x00);
-
-		err += misensor_rmw_reg(c, MISENSOR_16BIT, MISENSOR_READ_MODE,
-					MISENSOR_HFLIP_MASK, MISENSOR_FLIP_DIS);
-
-		dev->bpat = MT9M114_BPAT_BGBGGRGR;
-	}
-
-	err += mt9m114_write_reg(c, MISENSOR_8BIT, 0x8404, 0x06);
-	udelay(10);
-
-	return !!err;
-}
-
-/* Vertically flip the image */
-static int mt9m114_t_vflip(struct v4l2_subdev *sd, int value)
-{
-	struct i2c_client *c = v4l2_get_subdevdata(sd);
-	int err;
-	/* set for direct mode */
-	err = mt9m114_write_reg(c, MISENSOR_16BIT, 0x098E, 0xC850);
-	if (value >= 1) {
-		/* enable H flip - ctx A */
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC850, 0x02, 0x01);
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC851, 0x02, 0x01);
-		/* ctx B */
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC888, 0x02, 0x01);
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC889, 0x02, 0x01);
-
-		err += misensor_rmw_reg(c, MISENSOR_16BIT, MISENSOR_READ_MODE,
-					MISENSOR_VFLIP_MASK, MISENSOR_FLIP_EN);
-	} else {
-		/* disable H flip - ctx A */
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC850, 0x02, 0x00);
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC851, 0x02, 0x00);
-		/* ctx B */
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC888, 0x02, 0x00);
-		err += misensor_rmw_reg(c, MISENSOR_8BIT, 0xC889, 0x02, 0x00);
-
-		err += misensor_rmw_reg(c, MISENSOR_16BIT, MISENSOR_READ_MODE,
-					MISENSOR_VFLIP_MASK, MISENSOR_FLIP_DIS);
-	}
-
-	err += mt9m114_write_reg(c, MISENSOR_8BIT, 0x8404, 0x06);
-	udelay(10);
-
-	return !!err;
-}
-
-static int mt9m114_g_frame_interval(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_frame_interval *interval)
-{
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-
-	interval->interval.numerator = 1;
-	interval->interval.denominator = mt9m114_res[dev->res].fps;
-
-	return 0;
-}
-
-static int mt9m114_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	int ret;
-	struct i2c_client *c = v4l2_get_subdevdata(sd);
-	struct mt9m114_device *dev = to_mt9m114_sensor(sd);
-	struct atomisp_exposure exposure;
-
-	if (enable) {
-		ret = mt9m114_write_reg_array(c, mt9m114_chgstat_reg,
-					POST_POLLING);
-		if (ret < 0)
-			return ret;
-
-		if (dev->first_exp > MT9M114_MAX_FIRST_EXP) {
-			exposure.integration_time[0] = dev->first_exp;
-			exposure.gain[0] = dev->first_gain;
-			exposure.gain[1] = dev->first_diggain;
-			mt9m114_s_exposure(sd, &exposure);
-		}
-		dev->streamon = 1;
-
-	} else {
-		dev->streamon = 0;
-		ret = mt9m114_set_suspend(sd);
-	}
-
-	return ret;
-}
-
-static int mt9m114_enum_mbus_code(struct v4l2_subdev *sd,
-				  struct v4l2_subdev_pad_config *cfg,
-				  struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->index)
-		return -EINVAL;
-	code->code = MEDIA_BUS_FMT_SGRBG10_1X10;
-
-	return 0;
-}
-
-static int mt9m114_enum_frame_size(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_pad_config *cfg,
-				   struct v4l2_subdev_frame_size_enum *fse)
-{
-
-	unsigned int index = fse->index;
-
-	if (index >= N_RES)
-		return -EINVAL;
-
-	fse->min_width = mt9m114_res[index].width;
-	fse->min_height = mt9m114_res[index].height;
-	fse->max_width = mt9m114_res[index].width;
-	fse->max_height = mt9m114_res[index].height;
-
-	return 0;
-}
-
-static int mt9m114_g_skip_frames(struct v4l2_subdev *sd, u32 *frames)
-{
-	int index;
-	struct mt9m114_device *snr = to_mt9m114_sensor(sd);
-
-	if (frames == NULL)
-		return -EINVAL;
-
-	for (index = 0; index < N_RES; index++) {
-		if (mt9m114_res[index].res == snr->res)
-			break;
-	}
-
-	if (index >= N_RES)
-		return -EINVAL;
-
-	*frames = mt9m114_res[index].skip_frames;
-
-	return 0;
-}
-
-static const struct v4l2_subdev_video_ops mt9m114_video_ops = {
-	.s_stream = mt9m114_s_stream,
-	.g_frame_interval = mt9m114_g_frame_interval,
-};
-
-static const struct v4l2_subdev_sensor_ops mt9m114_sensor_ops = {
-	.g_skip_frames	= mt9m114_g_skip_frames,
-};
-
-static const struct v4l2_subdev_core_ops mt9m114_core_ops = {
-	.s_power = mt9m114_s_power,
-	.ioctl = mt9m114_ioctl,
-};
-
-/* REVISIT: Do we need pad operations? */
-static const struct v4l2_subdev_pad_ops mt9m114_pad_ops = {
-	.enum_mbus_code = mt9m114_enum_mbus_code,
-	.enum_frame_size = mt9m114_enum_frame_size,
-	.get_fmt = mt9m114_get_fmt,
-	.set_fmt = mt9m114_set_fmt,
-#ifndef CSS15
-	.set_selection = mt9m114_s_exposure_selection,
-#endif
-};
-
-static const struct v4l2_subdev_ops mt9m114_ops = {
-	.core = &mt9m114_core_ops,
-	.video = &mt9m114_video_ops,
-	.pad = &mt9m114_pad_ops,
-	.sensor = &mt9m114_sensor_ops,
-};
-
-static const struct media_entity_operations mt9m114_entity_ops = {
-	.link_setup = NULL,
-};
-
-static int mt9m114_remove(struct i2c_client *client)
-{
-	struct mt9m114_device *dev;
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-
-	dev = container_of(sd, struct mt9m114_device, sd);
-	dev->platform_data->csi_cfg(sd, 0);
-	v4l2_device_unregister_subdev(sd);
-	media_entity_cleanup(&dev->sd.entity);
-	v4l2_ctrl_handler_free(&dev->ctrl_handler);
-	kfree(dev);
-	return 0;
-}
-
-static int mt9m114_probe(struct i2c_client *client)
-{
-	struct mt9m114_device *dev;
-	int ret = 0;
-	unsigned int i;
-	void *pdata;
-
-	/* Setup sensor configuration structure */
-	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
-
-	v4l2_i2c_subdev_init(&dev->sd, client, &mt9m114_ops);
-	pdata = gmin_camera_platform_data(&dev->sd,
-					  ATOMISP_INPUT_FORMAT_RAW_10,
-					  atomisp_bayer_order_grbg);
-	if (pdata)
-		ret = mt9m114_s_config(&dev->sd, client->irq, pdata);
-	if (!pdata || ret) {
-		v4l2_device_unregister_subdev(&dev->sd);
-		kfree(dev);
-		return ret;
-	}
-
-	ret = atomisp_register_i2c_module(&dev->sd, pdata, RAW_CAMERA);
-	if (ret) {
-		v4l2_device_unregister_subdev(&dev->sd);
-		kfree(dev);
-		/* Coverity CID 298095 - return on error */
-		return ret;
-	}
-
-	/*TODO add format code here*/
-	dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	dev->pad.flags = MEDIA_PAD_FL_SOURCE;
-	dev->format.code = MEDIA_BUS_FMT_SGRBG10_1X10;
-	dev->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
-
-	ret =
-	    v4l2_ctrl_handler_init(&dev->ctrl_handler,
-				   ARRAY_SIZE(mt9m114_controls));
-	if (ret) {
-		mt9m114_remove(client);
-		return ret;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(mt9m114_controls); i++)
-		v4l2_ctrl_new_custom(&dev->ctrl_handler, &mt9m114_controls[i],
-				     NULL);
-
-	if (dev->ctrl_handler.error) {
-		mt9m114_remove(client);
-		return dev->ctrl_handler.error;
-	}
-
-	/* Use same lock for controls as for everything else. */
-	dev->ctrl_handler.lock = &dev->input_lock;
-	dev->sd.ctrl_handler = &dev->ctrl_handler;
-
-	/* REVISIT: Do we need media controller? */
-	ret = media_entity_pads_init(&dev->sd.entity, 1, &dev->pad);
-	if (ret) {
-		mt9m114_remove(client);
-		return ret;
-	}
-	return 0;
-}
-
-static const struct acpi_device_id mt9m114_acpi_match[] = {
-	{ "INT33F0" },
-	{ "CRMT1040" },
-	{},
-};
-MODULE_DEVICE_TABLE(acpi, mt9m114_acpi_match);
-
-static struct i2c_driver mt9m114_driver = {
-	.driver = {
-		.name = "mt9m114",
-		.acpi_match_table = mt9m114_acpi_match,
-	},
-	.probe_new = mt9m114_probe,
-	.remove = mt9m114_remove,
-};
-module_i2c_driver(mt9m114_driver);
-
-MODULE_AUTHOR("Shuguang Gong <Shuguang.gong@intel.com>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
deleted file mode 100644
index c0849299d592..000000000000
--- a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
+++ /dev/null
@@ -1,1470 +0,0 @@
-/*
- * Support for OmniVision OV2680 1080p HD camera sensor.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- */
-
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/string.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/kmod.h>
-#include <linux/device.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <linux/i2c.h>
-#include <linux/moduleparam.h>
-#include <media/v4l2-device.h>
-#include <linux/io.h>
-#include <linux/acpi.h>
-#include "../include/linux/atomisp_gmin_platform.h"
-
-#include "ov2680.h"
-
-static int h_flag = 0;
-static int v_flag = 0;
-static enum atomisp_bayer_order ov2680_bayer_order_mapping[] = {
-	atomisp_bayer_order_bggr,
-	atomisp_bayer_order_grbg,
-	atomisp_bayer_order_gbrg,
-	atomisp_bayer_order_rggb,
-};
-
-/* i2c read/write stuff */
-static int ov2680_read_reg(struct i2c_client *client,
-			   u16 data_length, u16 reg, u16 *val)
-{
-	int err;
-	struct i2c_msg msg[2];
-	unsigned char data[6];
-
-	if (!client->adapter) {
-		dev_err(&client->dev, "%s error, no client->adapter\n",
-			__func__);
-		return -ENODEV;
-	}
-
-	if (data_length != OV2680_8BIT && data_length != OV2680_16BIT
-					&& data_length != OV2680_32BIT) {
-		dev_err(&client->dev, "%s error, invalid data length\n",
-			__func__);
-		return -EINVAL;
-	}
-
-	memset(msg, 0 , sizeof(msg));
-
-	msg[0].addr = client->addr;
-	msg[0].flags = 0;
-	msg[0].len = I2C_MSG_LENGTH;
-	msg[0].buf = data;
-
-	/* high byte goes out first */
-	data[0] = (u8)(reg >> 8);
-	data[1] = (u8)(reg & 0xff);
-
-	msg[1].addr = client->addr;
-	msg[1].len = data_length;
-	msg[1].flags = I2C_M_RD;
-	msg[1].buf = data;
-
-	err = i2c_transfer(client->adapter, msg, 2);
-	if (err != 2) {
-		if (err >= 0)
-			err = -EIO;
-		dev_err(&client->dev,
-			"read from offset 0x%x error %d", reg, err);
-		return err;
-	}
-
-	*val = 0;
-	/* high byte comes first */
-	if (data_length == OV2680_8BIT)
-		*val = (u8)data[0];
-	else if (data_length == OV2680_16BIT)
-		*val = be16_to_cpu(*(__be16 *)&data[0]);
-	else
-		*val = be32_to_cpu(*(__be32 *)&data[0]);
-	//dev_dbg(&client->dev,  "++++i2c read adr%x = %x\n", reg,*val);
-	return 0;
-}
-
-static int ov2680_i2c_write(struct i2c_client *client, u16 len, u8 *data)
-{
-	struct i2c_msg msg;
-	const int num_msg = 1;
-	int ret;
-
-	msg.addr = client->addr;
-	msg.flags = 0;
-	msg.len = len;
-	msg.buf = data;
-	ret = i2c_transfer(client->adapter, &msg, 1);
-	//dev_dbg(&client->dev,  "+++i2c write reg=%x->%x\n", data[0]*256 +data[1],data[2]);
-	return ret == num_msg ? 0 : -EIO;
-}
-
-static int ov2680_write_reg(struct i2c_client *client, u16 data_length,
-							u16 reg, u16 val)
-{
-	int ret;
-	unsigned char data[4] = {0};
-	__be16 *wreg = (void *)data;
-	const u16 len = data_length + sizeof(u16); /* 16-bit address + data */
-
-	if (data_length != OV2680_8BIT && data_length != OV2680_16BIT) {
-		dev_err(&client->dev,
-			"%s error, invalid data_length\n", __func__);
-		return -EINVAL;
-	}
-
-	/* high byte goes out first */
-	*wreg = cpu_to_be16(reg);
-
-	if (data_length == OV2680_8BIT) {
-		data[2] = (u8)(val);
-	} else {
-		/* OV2680_16BIT */
-		__be16 *wdata = (void *)&data[2];
-
-		*wdata = cpu_to_be16(val);
-	}
-
-	ret = ov2680_i2c_write(client, len, data);
-	if (ret)
-		dev_err(&client->dev,
-			"write error: wrote 0x%x to offset 0x%x error %d",
-			val, reg, ret);
-
-	return ret;
-}
-
-/*
- * ov2680_write_reg_array - Initializes a list of OV2680 registers
- * @client: i2c driver client structure
- * @reglist: list of registers to be written
- *
- * This function initializes a list of registers. When consecutive addresses
- * are found in a row on the list, this function creates a buffer and sends
- * consecutive data in a single i2c_transfer().
- *
- * __ov2680_flush_reg_array, __ov2680_buf_reg_array() and
- * __ov2680_write_reg_is_consecutive() are internal functions to
- * ov2680_write_reg_array_fast() and should be not used anywhere else.
- *
- */
-
-static int __ov2680_flush_reg_array(struct i2c_client *client,
-				    struct ov2680_write_ctrl *ctrl)
-{
-	u16 size;
-	__be16 *data16 = (void *)&ctrl->buffer.addr;
-
-	if (ctrl->index == 0)
-		return 0;
-
-	size = sizeof(u16) + ctrl->index; /* 16-bit address + data */
-	*data16 = cpu_to_be16(ctrl->buffer.addr);
-	ctrl->index = 0;
-
-	return ov2680_i2c_write(client, size, (u8 *)&ctrl->buffer);
-}
-
-static int __ov2680_buf_reg_array(struct i2c_client *client,
-				  struct ov2680_write_ctrl *ctrl,
-				  const struct ov2680_reg *next)
-{
-	int size;
-	__be16 *data16;
-
-	switch (next->type) {
-	case OV2680_8BIT:
-		size = 1;
-		ctrl->buffer.data[ctrl->index] = (u8)next->val;
-		break;
-	case OV2680_16BIT:
-		size = 2;
-		data16 = (void *)&ctrl->buffer.data[ctrl->index];
-		*data16 = cpu_to_be16((u16)next->val);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* When first item is added, we need to store its starting address */
-	if (ctrl->index == 0)
-		ctrl->buffer.addr = next->reg;
-
-	ctrl->index += size;
-
-	/*
-	 * Buffer cannot guarantee free space for u32? Better flush it to avoid
-	 * possible lack of memory for next item.
-	 */
-	if (ctrl->index + sizeof(u16) >= OV2680_MAX_WRITE_BUF_SIZE)
-		return __ov2680_flush_reg_array(client, ctrl);
-
-	return 0;
-}
-
-static int __ov2680_write_reg_is_consecutive(struct i2c_client *client,
-					     struct ov2680_write_ctrl *ctrl,
-					     const struct ov2680_reg *next)
-{
-	if (ctrl->index == 0)
-		return 1;
-
-	return ctrl->buffer.addr + ctrl->index == next->reg;
-}
-
-static int ov2680_write_reg_array(struct i2c_client *client,
-				  const struct ov2680_reg *reglist)
-{
-	const struct ov2680_reg *next = reglist;
-	struct ov2680_write_ctrl ctrl;
-	int err;
-	dev_dbg(&client->dev,  "++++write reg array\n");
-	ctrl.index = 0;
-	for (; next->type != OV2680_TOK_TERM; next++) {
-		switch (next->type & OV2680_TOK_MASK) {
-		case OV2680_TOK_DELAY:
-			err = __ov2680_flush_reg_array(client, &ctrl);
-			if (err)
-				return err;
-			msleep(next->val);
-			break;
-		default:
-			/*
-			 * If next address is not consecutive, data needs to be
-			 * flushed before proceed.
-			 */
-			 dev_dbg(&client->dev,  "+++ov2680_write_reg_array reg=%x->%x\n", next->reg,next->val);
-			if (!__ov2680_write_reg_is_consecutive(client, &ctrl,
-								next)) {
-				err = __ov2680_flush_reg_array(client, &ctrl);
-				if (err)
-					return err;
-			}
-			err = __ov2680_buf_reg_array(client, &ctrl, next);
-			if (err) {
-				dev_err(&client->dev, "%s: write error, aborted\n",
-					 __func__);
-				return err;
-			}
-			break;
-		}
-	}
-
-	return __ov2680_flush_reg_array(client, &ctrl);
-}
-static int ov2680_g_focal(struct v4l2_subdev *sd, s32 *val)
-{
-
-	*val = (OV2680_FOCAL_LENGTH_NUM << 16) | OV2680_FOCAL_LENGTH_DEM;
-	return 0;
-}
-
-static int ov2680_g_fnumber(struct v4l2_subdev *sd, s32 *val)
-{
-	/*const f number for ov2680*/
-
-	*val = (OV2680_F_NUMBER_DEFAULT_NUM << 16) | OV2680_F_NUMBER_DEM;
-	return 0;
-}
-
-static int ov2680_g_fnumber_range(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = (OV2680_F_NUMBER_DEFAULT_NUM << 24) |
-		(OV2680_F_NUMBER_DEM << 16) |
-		(OV2680_F_NUMBER_DEFAULT_NUM << 8) | OV2680_F_NUMBER_DEM;
-	return 0;
-}
-
-static int ov2680_g_bin_factor_x(struct v4l2_subdev *sd, s32 *val)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	dev_dbg(&client->dev,  "++++ov2680_g_bin_factor_x\n");
-	*val = ov2680_res[dev->fmt_idx].bin_factor_x;
-
-	return 0;
-}
-
-static int ov2680_g_bin_factor_y(struct v4l2_subdev *sd, s32 *val)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-
-	*val = ov2680_res[dev->fmt_idx].bin_factor_y;
-	dev_dbg(&client->dev,  "++++ov2680_g_bin_factor_y\n");
-	return 0;
-}
-
-
-static int ov2680_get_intg_factor(struct i2c_client *client,
-				struct camera_mipi_info *info,
-				const struct ov2680_resolution *res)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	struct atomisp_sensor_mode_data *buf = &info->data;
-	unsigned int pix_clk_freq_hz;
-	u16 reg_val;
-	int ret;
-	dev_dbg(&client->dev,  "++++ov2680_get_intg_factor\n");
-	if (!info)
-		return -EINVAL;
-
-	/* pixel clock */
-	pix_clk_freq_hz = res->pix_clk_freq * 1000000;
-
-	dev->vt_pix_clk_freq_mhz = pix_clk_freq_hz;
-	buf->vt_pix_clk_freq_mhz = pix_clk_freq_hz;
-
-	/* get integration time */
-	buf->coarse_integration_time_min = OV2680_COARSE_INTG_TIME_MIN;
-	buf->coarse_integration_time_max_margin =
-					OV2680_COARSE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_min = OV2680_FINE_INTG_TIME_MIN;
-	buf->fine_integration_time_max_margin =
-					OV2680_FINE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_def = OV2680_FINE_INTG_TIME_MIN;
-	buf->frame_length_lines = res->lines_per_frame;
-	buf->line_length_pck = res->pixels_per_line;
-	buf->read_mode = res->bin_mode;
-
-	/* get the cropping and output resolution to ISP for this mode. */
-	ret =  ov2680_read_reg(client, OV2680_16BIT,
-					OV2680_HORIZONTAL_START_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_horizontal_start = reg_val;
-
-	ret =  ov2680_read_reg(client, OV2680_16BIT,
-					OV2680_VERTICAL_START_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_vertical_start = reg_val;
-
-	ret = ov2680_read_reg(client, OV2680_16BIT,
-					OV2680_HORIZONTAL_END_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_horizontal_end = reg_val;
-
-	ret = ov2680_read_reg(client, OV2680_16BIT,
-					OV2680_VERTICAL_END_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_vertical_end = reg_val;
-
-	ret = ov2680_read_reg(client, OV2680_16BIT,
-					OV2680_HORIZONTAL_OUTPUT_SIZE_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_width = reg_val;
-
-	ret = ov2680_read_reg(client, OV2680_16BIT,
-					OV2680_VERTICAL_OUTPUT_SIZE_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_height = reg_val;
-
-	buf->binning_factor_x = res->bin_factor_x ?
-					(res->bin_factor_x * 2) : 1;
-	buf->binning_factor_y = res->bin_factor_y ?
-					(res->bin_factor_y * 2) : 1;
-	return 0;
-}
-
-static long __ov2680_set_exposure(struct v4l2_subdev *sd, int coarse_itg,
-				 int gain, int digitgain)
-
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	u16 vts,hts;
-	int ret,exp_val;
-
-	dev_dbg(&client->dev,
-		"+++++++__ov2680_set_exposure coarse_itg %d, gain %d, digitgain %d++\n",
-		coarse_itg, gain, digitgain);
-
-	hts = ov2680_res[dev->fmt_idx].pixels_per_line;
-	vts = ov2680_res[dev->fmt_idx].lines_per_frame;
-
-	/* group hold */
-	ret = ov2680_write_reg(client, OV2680_8BIT,
-                                       OV2680_GROUP_ACCESS, 0x00);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV2680_GROUP_ACCESS);
-		return ret;
-	}
-
-	/* Increase the VTS to match exposure + MARGIN */
-	if (coarse_itg > vts - OV2680_INTEGRATION_TIME_MARGIN)
-		vts = (u16) coarse_itg + OV2680_INTEGRATION_TIME_MARGIN;
-
-	ret = ov2680_write_reg(client, OV2680_16BIT, OV2680_TIMING_VTS_H, vts);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV2680_TIMING_VTS_H);
-		return ret;
-	}
-
-	/* set exposure */
-
-	/* Lower four bit should be 0*/
-	exp_val = coarse_itg << 4;
-	ret = ov2680_write_reg(client, OV2680_8BIT,
-			       OV2680_EXPOSURE_L, exp_val & 0xFF);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV2680_EXPOSURE_L);
-		return ret;
-	}
-
-	ret = ov2680_write_reg(client, OV2680_8BIT,
-			       OV2680_EXPOSURE_M, (exp_val >> 8) & 0xFF);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV2680_EXPOSURE_M);
-		return ret;
-	}
-
-	ret = ov2680_write_reg(client, OV2680_8BIT,
-			       OV2680_EXPOSURE_H, (exp_val >> 16) & 0x0F);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV2680_EXPOSURE_H);
-		return ret;
-	}
-
-	/* Analog gain */
-	ret = ov2680_write_reg(client, OV2680_16BIT, OV2680_AGC_H, gain);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV2680_AGC_H);
-		return ret;
-	}
-	/* Digital gain */
-	if (digitgain) {
-		ret = ov2680_write_reg(client, OV2680_16BIT,
-				OV2680_MWB_RED_GAIN_H, digitgain);
-		if (ret) {
-			dev_err(&client->dev, "%s: write %x error, aborted\n",
-				__func__, OV2680_MWB_RED_GAIN_H);
-			return ret;
-		}
-
-		ret = ov2680_write_reg(client, OV2680_16BIT,
-				OV2680_MWB_GREEN_GAIN_H, digitgain);
-		if (ret) {
-			dev_err(&client->dev, "%s: write %x error, aborted\n",
-				__func__, OV2680_MWB_RED_GAIN_H);
-			return ret;
-		}
-
-		ret = ov2680_write_reg(client, OV2680_16BIT,
-				OV2680_MWB_BLUE_GAIN_H, digitgain);
-		if (ret) {
-			dev_err(&client->dev, "%s: write %x error, aborted\n",
-				__func__, OV2680_MWB_RED_GAIN_H);
-			return ret;
-		}
-	}
-
-	/* End group */
-	ret = ov2680_write_reg(client, OV2680_8BIT,
-			       OV2680_GROUP_ACCESS, 0x10);
-	if (ret)
-		return ret;
-
-	/* Delay launch group */
-	ret = ov2680_write_reg(client, OV2680_8BIT,
-					   OV2680_GROUP_ACCESS, 0xa0);
-	if (ret)
-		return ret;
-	return ret;
-}
-
-static int ov2680_set_exposure(struct v4l2_subdev *sd, int exposure,
-	int gain, int digitgain)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	int ret;
-
-	mutex_lock(&dev->input_lock);
-	ret = __ov2680_set_exposure(sd, exposure, gain, digitgain);
-	mutex_unlock(&dev->input_lock);
-
-	return ret;
-}
-
-static long ov2680_s_exposure(struct v4l2_subdev *sd,
-			       struct atomisp_exposure *exposure)
-{
-	u16 coarse_itg = exposure->integration_time[0];
-	u16 analog_gain = exposure->gain[0];
-	u16 digital_gain = exposure->gain[1];
-
-	/* we should not accept the invalid value below */
-	if (analog_gain == 0) {
-		struct i2c_client *client = v4l2_get_subdevdata(sd);
-		v4l2_err(client, "%s: invalid value\n", __func__);
-		return -EINVAL;
-	}
-
-	// EXPOSURE CONTROL DISABLED FOR INITIAL CHECKIN, TUNING DOESN'T WORK
-	return ov2680_set_exposure(sd, coarse_itg, analog_gain, digital_gain);
-}
-
-
-
-
-
-static long ov2680_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
-{
-
-	switch (cmd) {
-	case ATOMISP_IOC_S_EXPOSURE:
-		return ov2680_s_exposure(sd, arg);
-
-	default:
-		return -EINVAL;
-	}
-	return 0;
-}
-
-/* This returns the exposure time being used. This should only be used
- * for filling in EXIF data, not for actual image processing.
- */
-static int ov2680_q_exposure(struct v4l2_subdev *sd, s32 *value)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u16 reg_v, reg_v2;
-	int ret;
-
-	/* get exposure */
-	ret = ov2680_read_reg(client, OV2680_8BIT,
-					OV2680_EXPOSURE_L,
-					&reg_v);
-	if (ret)
-		goto err;
-
-	ret = ov2680_read_reg(client, OV2680_8BIT,
-					OV2680_EXPOSURE_M,
-					&reg_v2);
-	if (ret)
-		goto err;
-
-	reg_v += reg_v2 << 8;
-	ret = ov2680_read_reg(client, OV2680_8BIT,
-					OV2680_EXPOSURE_H,
-					&reg_v2);
-	if (ret)
-		goto err;
-
-	*value = reg_v + (((u32)reg_v2 << 16));
-err:
-	return ret;
-}
-
-static u32 ov2680_translate_bayer_order(enum atomisp_bayer_order code)
-{
-	switch (code) {
-	case atomisp_bayer_order_rggb:
-		return MEDIA_BUS_FMT_SRGGB10_1X10;
-	case atomisp_bayer_order_grbg:
-		return MEDIA_BUS_FMT_SGRBG10_1X10;
-	case atomisp_bayer_order_bggr:
-		return MEDIA_BUS_FMT_SBGGR10_1X10;
-	case atomisp_bayer_order_gbrg:
-		return MEDIA_BUS_FMT_SGBRG10_1X10;
-	}
-	return 0;
-}
-
-static int ov2680_v_flip(struct v4l2_subdev *sd, s32 value)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	struct camera_mipi_info *ov2680_info = NULL;
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-	u16 val;
-	u8 index;
-	dev_dbg(&client->dev, "@%s: value:%d\n", __func__, value);
-	ret = ov2680_read_reg(client, OV2680_8BIT, OV2680_FLIP_REG, &val);
-	if (ret)
-		return ret;
-	if (value) {
-		val |= OV2680_FLIP_MIRROR_BIT_ENABLE;
-	} else {
-		val &= ~OV2680_FLIP_MIRROR_BIT_ENABLE;
-	}
-	ret = ov2680_write_reg(client, OV2680_8BIT,
-			OV2680_FLIP_REG, val);
-	if (ret)
-		return ret;
-	index = (v_flag>0?OV2680_FLIP_BIT:0) | (h_flag>0?OV2680_MIRROR_BIT:0);
-	ov2680_info = v4l2_get_subdev_hostdata(sd);
-	if (ov2680_info) {
-		ov2680_info->raw_bayer_order = ov2680_bayer_order_mapping[index];
-		dev->format.code = ov2680_translate_bayer_order(
-			ov2680_info->raw_bayer_order);
-	}
-	return ret;
-}
-
-static int ov2680_h_flip(struct v4l2_subdev *sd, s32 value)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	struct camera_mipi_info *ov2680_info = NULL;
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-	u16 val;
-	u8 index;
-	dev_dbg(&client->dev, "@%s: value:%d\n", __func__, value);
-
-	ret = ov2680_read_reg(client, OV2680_8BIT, OV2680_MIRROR_REG, &val);
-	if (ret)
-		return ret;
-	if (value) {
-		val |= OV2680_FLIP_MIRROR_BIT_ENABLE;
-	} else {
-		val &= ~OV2680_FLIP_MIRROR_BIT_ENABLE;
-	}
-	ret = ov2680_write_reg(client, OV2680_8BIT,
-			OV2680_MIRROR_REG, val);
-	if (ret)
-		return ret;
-	index = (v_flag>0?OV2680_FLIP_BIT:0) | (h_flag>0?OV2680_MIRROR_BIT:0);
-	ov2680_info = v4l2_get_subdev_hostdata(sd);
-	if (ov2680_info) {
-		ov2680_info->raw_bayer_order = ov2680_bayer_order_mapping[index];
-		dev->format.code = ov2680_translate_bayer_order(
-			ov2680_info->raw_bayer_order);
-	}
-	return ret;
-}
-
-static int ov2680_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct ov2680_device *dev =
-	    container_of(ctrl->handler, struct ov2680_device, ctrl_handler);
-	struct i2c_client *client = v4l2_get_subdevdata(&dev->sd);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_VFLIP:
-		dev_dbg(&client->dev, "%s: CID_VFLIP:%d.\n",
-			__func__, ctrl->val);
-		ret = ov2680_v_flip(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_HFLIP:
-		dev_dbg(&client->dev, "%s: CID_HFLIP:%d.\n",
-			__func__, ctrl->val);
-		ret = ov2680_h_flip(&dev->sd, ctrl->val);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-	return ret;
-}
-
-static int ov2680_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct ov2680_device *dev =
-	    container_of(ctrl->handler, struct ov2680_device, ctrl_handler);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_EXPOSURE_ABSOLUTE:
-		ret = ov2680_q_exposure(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FOCAL_ABSOLUTE:
-		ret = ov2680_g_focal(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_ABSOLUTE:
-		ret = ov2680_g_fnumber(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_RANGE:
-		ret = ov2680_g_fnumber_range(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_BIN_FACTOR_HORZ:
-		ret = ov2680_g_bin_factor_x(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_BIN_FACTOR_VERT:
-		ret = ov2680_g_bin_factor_y(&dev->sd, &ctrl->val);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-static const struct v4l2_ctrl_ops ctrl_ops = {
-	.s_ctrl = ov2680_s_ctrl,
-	.g_volatile_ctrl = ov2680_g_volatile_ctrl
-};
-
-static const struct v4l2_ctrl_config ov2680_controls[] = {
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_EXPOSURE_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "exposure",
-	 .min = 0x0,
-	 .max = 0xffff,
-	 .step = 0x01,
-	 .def = 0x00,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FOCAL_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "focal length",
-	 .min = OV2680_FOCAL_LENGTH_DEFAULT,
-	 .max = OV2680_FOCAL_LENGTH_DEFAULT,
-	 .step = 0x01,
-	 .def = OV2680_FOCAL_LENGTH_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "f-number",
-	 .min = OV2680_F_NUMBER_DEFAULT,
-	 .max = OV2680_F_NUMBER_DEFAULT,
-	 .step = 0x01,
-	 .def = OV2680_F_NUMBER_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_RANGE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "f-number range",
-	 .min = OV2680_F_NUMBER_RANGE,
-	 .max = OV2680_F_NUMBER_RANGE,
-	 .step = 0x01,
-	 .def = OV2680_F_NUMBER_RANGE,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_BIN_FACTOR_HORZ,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "horizontal binning factor",
-	 .min = 0,
-	 .max = OV2680_BIN_FACTOR_MAX,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_BIN_FACTOR_VERT,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "vertical binning factor",
-	 .min = 0,
-	 .max = OV2680_BIN_FACTOR_MAX,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_VFLIP,
-	 .type = V4L2_CTRL_TYPE_BOOLEAN,
-	 .name = "Flip",
-	 .min = 0,
-	 .max = 1,
-	 .step = 1,
-	 .def = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_HFLIP,
-	 .type = V4L2_CTRL_TYPE_BOOLEAN,
-	 .name = "Mirror",
-	 .min = 0,
-	 .max = 1,
-	 .step = 1,
-	 .def = 0,
-	 },
-};
-
-static int ov2680_init_registers(struct v4l2_subdev *sd)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	ret = ov2680_write_reg(client, OV2680_8BIT, OV2680_SW_RESET, 0x01);
-	ret |= ov2680_write_reg_array(client, ov2680_global_setting);
-
-	return ret;
-}
-
-static int ov2680_init(struct v4l2_subdev *sd)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-
-	int ret;
-
-	mutex_lock(&dev->input_lock);
-
-	/* restore settings */
-	ov2680_res = ov2680_res_preview;
-	N_RES = N_RES_PREVIEW;
-
-	ret = ov2680_init_registers(sd);
-
-	mutex_unlock(&dev->input_lock);
-
-	return ret;
-}
-
-static int power_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	int ret = 0;
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	if (flag) {
-		ret |= dev->platform_data->v1p8_ctrl(sd, 1);
-		ret |= dev->platform_data->v2p8_ctrl(sd, 1);
-		usleep_range(10000, 15000);
-	}
-
-	if (!flag || ret) {
-		ret |= dev->platform_data->v1p8_ctrl(sd, 0);
-		ret |= dev->platform_data->v2p8_ctrl(sd, 0);
-	}
-	return ret;
-}
-
-static int gpio_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	int ret;
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	/* The OV2680 documents only one GPIO input (#XSHUTDN), but
-	 * existing integrations often wire two (reset/power_down)
-	 * because that is the way other sensors work.  There is no
-	 * way to tell how it is wired internally, so existing
-	 * firmwares expose both and we drive them symmetrically. */
-	if (flag) {
-		ret = dev->platform_data->gpio0_ctrl(sd, 1);
-		usleep_range(10000, 15000);
-		/* Ignore return from second gpio, it may not be there */
-		dev->platform_data->gpio1_ctrl(sd, 1);
-		usleep_range(10000, 15000);
-	} else {
-		dev->platform_data->gpio1_ctrl(sd, 0);
-		ret = dev->platform_data->gpio0_ctrl(sd, 0);
-	}
-	return ret;
-}
-
-static int power_up(struct v4l2_subdev *sd)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	/* power control */
-	ret = power_ctrl(sd, 1);
-	if (ret)
-		goto fail_power;
-
-	/* according to DS, at least 5ms is needed between DOVDD and PWDN */
-	usleep_range(5000, 6000);
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 1);
-	if (ret) {
-		ret = gpio_ctrl(sd, 1);
-		if (ret)
-			goto fail_power;
-	}
-
-	/* flis clock control */
-	ret = dev->platform_data->flisclk_ctrl(sd, 1);
-	if (ret)
-		goto fail_clk;
-
-	/* according to DS, 20ms is needed between PWDN and i2c access */
-	msleep(20);
-
-	return 0;
-
-fail_clk:
-	gpio_ctrl(sd, 0);
-fail_power:
-	power_ctrl(sd, 0);
-	dev_err(&client->dev, "sensor power-up failed\n");
-
-	return ret;
-}
-
-static int power_down(struct v4l2_subdev *sd)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	h_flag = 0;
-	v_flag = 0;
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	ret = dev->platform_data->flisclk_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "flisclk failed\n");
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 0);
-	if (ret) {
-		ret = gpio_ctrl(sd, 0);
-		if (ret)
-			dev_err(&client->dev, "gpio failed 2\n");
-	}
-
-	/* power control */
-	ret = power_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "vprog failed.\n");
-
-	return ret;
-}
-
-static int ov2680_s_power(struct v4l2_subdev *sd, int on)
-{
-	int ret;
-
-	if (on == 0){
-		ret = power_down(sd);
-	} else {
-		ret = power_up(sd);
-		if (!ret)
-			return ov2680_init(sd);
-	}
-	return ret;
-}
-
-/*
- * distance - calculate the distance
- * @res: resolution
- * @w: width
- * @h: height
- *
- * Get the gap between resolution and w/h.
- * res->width/height smaller than w/h wouldn't be considered.
- * Returns the value of gap or -1 if fail.
- */
-#define LARGEST_ALLOWED_RATIO_MISMATCH 600
-static int distance(struct ov2680_resolution *res, u32 w, u32 h)
-{
-	unsigned int w_ratio = (res->width << 13) / w;
-	unsigned int h_ratio;
-	int match;
-
-	if (h == 0)
-		return -1;
-	h_ratio = (res->height << 13) / h;
-	if (h_ratio == 0)
-		return -1;
-	match   = abs(((w_ratio << 13) / h_ratio) - ((int)8192));
-
-
-	if ((w_ratio < (int)8192) || (h_ratio < (int)8192)  ||
-		(match > LARGEST_ALLOWED_RATIO_MISMATCH))
-		return -1;
-
-	return w_ratio + h_ratio;
-}
-
-/* Return the nearest higher resolution index */
-static int nearest_resolution_index(int w, int h)
-{
-	int i;
-	int idx = -1;
-	int dist;
-	int min_dist = INT_MAX;
-	struct ov2680_resolution *tmp_res = NULL;
-
-	for (i = 0; i < N_RES; i++) {
-		tmp_res = &ov2680_res[i];
-		dist = distance(tmp_res, w, h);
-		if (dist == -1)
-			continue;
-		if (dist < min_dist) {
-			min_dist = dist;
-			idx = i;
-		}
-	}
-
-	return idx;
-}
-
-static int get_resolution_index(int w, int h)
-{
-	int i;
-
-	for (i = 0; i < N_RES; i++) {
-		if (w != ov2680_res[i].width)
-			continue;
-		if (h != ov2680_res[i].height)
-			continue;
-
-		return i;
-	}
-
-	return -1;
-}
-
-static int ov2680_set_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_pad_config *cfg,
-			  struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct camera_mipi_info *ov2680_info = NULL;
-	int ret = 0;
-	int idx = 0;
-	dev_dbg(&client->dev, "+++++ov2680_s_mbus_fmt+++++l\n");
-	if (format->pad)
-		return -EINVAL;
-
-	if (!fmt)
-		return -EINVAL;
-
-	ov2680_info = v4l2_get_subdev_hostdata(sd);
-	if (!ov2680_info)
-		return -EINVAL;
-
-	mutex_lock(&dev->input_lock);
-	idx = nearest_resolution_index(fmt->width, fmt->height);
-	if (idx == -1) {
-		/* return the largest resolution */
-		fmt->width = ov2680_res[N_RES - 1].width;
-		fmt->height = ov2680_res[N_RES - 1].height;
-	} else {
-		fmt->width = ov2680_res[idx].width;
-		fmt->height = ov2680_res[idx].height;
-	}
-	fmt->code = MEDIA_BUS_FMT_SBGGR10_1X10;
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
-		cfg->try_fmt = *fmt;
-		mutex_unlock(&dev->input_lock);
-		return 0;
-		}
-	dev->fmt_idx = get_resolution_index(fmt->width, fmt->height);
-	dev_dbg(&client->dev, "+++++get_resolution_index=%d+++++l\n",
-		     dev->fmt_idx);
-	if (dev->fmt_idx == -1) {
-		dev_err(&client->dev, "get resolution fail\n");
-		mutex_unlock(&dev->input_lock);
-		return -EINVAL;
-	}
-	v4l2_info(client, "__s_mbus_fmt i=%d, w=%d, h=%d\n", dev->fmt_idx,
-		  fmt->width, fmt->height);
-	dev_dbg(&client->dev, "__s_mbus_fmt i=%d, w=%d, h=%d\n",
-		     dev->fmt_idx, fmt->width, fmt->height);
-
-	ret = ov2680_write_reg_array(client, ov2680_res[dev->fmt_idx].regs);
-	if (ret)
-		dev_err(&client->dev, "ov2680 write resolution register err\n");
-
-	ret = ov2680_get_intg_factor(client, ov2680_info,
-				     &ov2680_res[dev->fmt_idx]);
-	if (ret) {
-		dev_err(&client->dev, "failed to get integration_factor\n");
-		goto err;
-	}
-
-	/*recall flip functions to avoid flip registers
-	 * were overridden by default setting
-	 */
-	if (h_flag)
-		ov2680_h_flip(sd, h_flag);
-	if (v_flag)
-		ov2680_v_flip(sd, v_flag);
-
-	v4l2_info(client, "\n%s idx %d \n", __func__, dev->fmt_idx);
-
-	/*ret = startup(sd);
-	 * if (ret)
-	 * dev_err(&client->dev, "ov2680 startup err\n");
-	 */
-err:
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-
-static int ov2680_get_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_pad_config *cfg,
-			  struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-
-	if (format->pad)
-		return -EINVAL;
-
-	if (!fmt)
-		return -EINVAL;
-
-	fmt->width = ov2680_res[dev->fmt_idx].width;
-	fmt->height = ov2680_res[dev->fmt_idx].height;
-	fmt->code = MEDIA_BUS_FMT_SBGGR10_1X10;
-
-	return 0;
-}
-
-static int ov2680_detect(struct i2c_client *client)
-{
-	struct i2c_adapter *adapter = client->adapter;
-	u16 high, low;
-	int ret;
-	u16 id;
-	u8 revision;
-
-	if (!i2c_check_functionality(adapter, I2C_FUNC_I2C))
-		return -ENODEV;
-
-	ret = ov2680_read_reg(client, OV2680_8BIT,
-					OV2680_SC_CMMN_CHIP_ID_H, &high);
-	if (ret) {
-		dev_err(&client->dev, "sensor_id_high = 0x%x\n", high);
-		return -ENODEV;
-	}
-	ret = ov2680_read_reg(client, OV2680_8BIT,
-					OV2680_SC_CMMN_CHIP_ID_L, &low);
-	id = ((((u16) high) << 8) | (u16) low);
-
-	if (id != OV2680_ID) {
-		dev_err(&client->dev, "sensor ID error 0x%x\n", id);
-		return -ENODEV;
-	}
-
-	ret = ov2680_read_reg(client, OV2680_8BIT,
-					OV2680_SC_CMMN_SUB_ID, &high);
-	revision = (u8) high & 0x0f;
-
-	dev_info(&client->dev, "sensor_revision id = 0x%x\n", id);
-
-	return 0;
-}
-
-static int ov2680_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	mutex_lock(&dev->input_lock);
-	if(enable )
-		dev_dbg(&client->dev, "ov2680_s_stream one \n");
-	else
-		dev_dbg(&client->dev, "ov2680_s_stream off \n");
-
-	ret = ov2680_write_reg(client, OV2680_8BIT, OV2680_SW_STREAM,
-				enable ? OV2680_START_STREAMING :
-				OV2680_STOP_STREAMING);
-#if 0
-	/* restore settings */
-	ov2680_res = ov2680_res_preview;
-	N_RES = N_RES_PREVIEW;
-#endif
-
-	//otp valid at stream on state
-	//if(!dev->otp_data)
-	//	dev->otp_data = ov2680_otp_read(sd);
-
-	mutex_unlock(&dev->input_lock);
-
-	return ret;
-}
-
-
-static int ov2680_s_config(struct v4l2_subdev *sd,
-			   int irq, void *platform_data)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	if (!platform_data)
-		return -ENODEV;
-
-	dev->platform_data =
-		(struct camera_sensor_platform_data *)platform_data;
-
-	mutex_lock(&dev->input_lock);
-	/* power off the module, then power on it in future
-	 * as first power on by board may not fulfill the
-	 * power on sequqence needed by the module
-	 */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "ov2680 power-off err.\n");
-		goto fail_power_off;
-	}
-
-	ret = power_up(sd);
-	if (ret) {
-		dev_err(&client->dev, "ov2680 power-up err.\n");
-		goto fail_power_on;
-	}
-
-	ret = dev->platform_data->csi_cfg(sd, 1);
-	if (ret)
-		goto fail_csi_cfg;
-
-	/* config & detect sensor */
-	ret = ov2680_detect(client);
-	if (ret) {
-		dev_err(&client->dev, "ov2680_detect err s_config.\n");
-		goto fail_csi_cfg;
-	}
-
-	/* turn off sensor, after probed */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "ov2680 power-off err.\n");
-		goto fail_csi_cfg;
-	}
-	mutex_unlock(&dev->input_lock);
-
-	return 0;
-
-fail_csi_cfg:
-	dev->platform_data->csi_cfg(sd, 0);
-fail_power_on:
-	power_down(sd);
-	dev_err(&client->dev, "sensor power-gating failed\n");
-fail_power_off:
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-
-static int ov2680_g_frame_interval(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_frame_interval *interval)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-
-	interval->interval.numerator = 1;
-	interval->interval.denominator = ov2680_res[dev->fmt_idx].fps;
-
-	return 0;
-}
-
-static int ov2680_enum_mbus_code(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_pad_config *cfg,
-				 struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->index >= MAX_FMTS)
-		return -EINVAL;
-
-	code->code = MEDIA_BUS_FMT_SBGGR10_1X10;
-	return 0;
-}
-
-static int ov2680_enum_frame_size(struct v4l2_subdev *sd,
-				  struct v4l2_subdev_pad_config *cfg,
-				  struct v4l2_subdev_frame_size_enum *fse)
-{
-	int index = fse->index;
-
-	if (index >= N_RES)
-		return -EINVAL;
-
-	fse->min_width = ov2680_res[index].width;
-	fse->min_height = ov2680_res[index].height;
-	fse->max_width = ov2680_res[index].width;
-	fse->max_height = ov2680_res[index].height;
-
-	return 0;
-
-}
-
-static int ov2680_g_skip_frames(struct v4l2_subdev *sd, u32 *frames)
-{
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-
-	mutex_lock(&dev->input_lock);
-	*frames = ov2680_res[dev->fmt_idx].skip_frames;
-	mutex_unlock(&dev->input_lock);
-
-	return 0;
-}
-
-static const struct v4l2_subdev_video_ops ov2680_video_ops = {
-	.s_stream = ov2680_s_stream,
-	.g_frame_interval = ov2680_g_frame_interval,
-};
-
-static const struct v4l2_subdev_sensor_ops ov2680_sensor_ops = {
-		.g_skip_frames	= ov2680_g_skip_frames,
-};
-
-static const struct v4l2_subdev_core_ops ov2680_core_ops = {
-	.s_power = ov2680_s_power,
-	.ioctl = ov2680_ioctl,
-};
-
-static const struct v4l2_subdev_pad_ops ov2680_pad_ops = {
-	.enum_mbus_code = ov2680_enum_mbus_code,
-	.enum_frame_size = ov2680_enum_frame_size,
-	.get_fmt = ov2680_get_fmt,
-	.set_fmt = ov2680_set_fmt,
-};
-
-static const struct v4l2_subdev_ops ov2680_ops = {
-	.core = &ov2680_core_ops,
-	.video = &ov2680_video_ops,
-	.pad = &ov2680_pad_ops,
-	.sensor = &ov2680_sensor_ops,
-};
-
-static int ov2680_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	dev_dbg(&client->dev, "ov2680_remove...\n");
-
-	dev->platform_data->csi_cfg(sd, 0);
-
-	v4l2_device_unregister_subdev(sd);
-	media_entity_cleanup(&dev->sd.entity);
-	v4l2_ctrl_handler_free(&dev->ctrl_handler);
-	kfree(dev);
-
-	return 0;
-}
-
-static int ov2680_probe(struct i2c_client *client)
-{
-	struct ov2680_device *dev;
-	int ret;
-	void *pdata;
-	unsigned int i;
-
-	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
-
-	mutex_init(&dev->input_lock);
-
-	dev->fmt_idx = 0;
-	v4l2_i2c_subdev_init(&(dev->sd), client, &ov2680_ops);
-
-	pdata = gmin_camera_platform_data(&dev->sd,
-					  ATOMISP_INPUT_FORMAT_RAW_10,
-					  atomisp_bayer_order_bggr);
-	if (!pdata) {
-		ret = -EINVAL;
-		goto out_free;
-        }
-
-	ret = ov2680_s_config(&dev->sd, client->irq, pdata);
-	if (ret)
-		goto out_free;
-
-	ret = atomisp_register_i2c_module(&dev->sd, pdata, RAW_CAMERA);
-	if (ret)
-		goto out_free;
-
-	dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	dev->pad.flags = MEDIA_PAD_FL_SOURCE;
-	dev->format.code = MEDIA_BUS_FMT_SBGGR10_1X10;
-	dev->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
-	ret =
-	    v4l2_ctrl_handler_init(&dev->ctrl_handler,
-				   ARRAY_SIZE(ov2680_controls));
-	if (ret) {
-		ov2680_remove(client);
-		return ret;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(ov2680_controls); i++)
-		v4l2_ctrl_new_custom(&dev->ctrl_handler, &ov2680_controls[i],
-				     NULL);
-
-	if (dev->ctrl_handler.error) {
-		ov2680_remove(client);
-		return dev->ctrl_handler.error;
-	}
-
-	/* Use same lock for controls as for everything else. */
-	dev->ctrl_handler.lock = &dev->input_lock;
-	dev->sd.ctrl_handler = &dev->ctrl_handler;
-
-	ret = media_entity_pads_init(&dev->sd.entity, 1, &dev->pad);
-	if (ret)
-	{
-		ov2680_remove(client);
-		dev_dbg(&client->dev, "+++ remove ov2680 \n");
-	}
-	return ret;
-out_free:
-	dev_dbg(&client->dev, "+++ out free \n");
-	v4l2_device_unregister_subdev(&dev->sd);
-	kfree(dev);
-	return ret;
-}
-
-static const struct acpi_device_id ov2680_acpi_match[] = {
-	{"XXOV2680"},
-	{"OVTI2680"},
-	{},
-};
-MODULE_DEVICE_TABLE(acpi, ov2680_acpi_match);
-
-static struct i2c_driver ov2680_driver = {
-	.driver = {
-		.name = "ov2680",
-		.acpi_match_table = ov2680_acpi_match,
-	},
-	.probe_new = ov2680_probe,
-	.remove = ov2680_remove,
-};
-module_i2c_driver(ov2680_driver);
-
-MODULE_AUTHOR("Jacky Wang <Jacky_wang@ovt.com>");
-MODULE_DESCRIPTION("A low-level driver for OmniVision 2680 sensors");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
deleted file mode 100644
index a362eebd882f..000000000000
--- a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
+++ /dev/null
@@ -1,1271 +0,0 @@
-/*
- * Support for OmniVision OV2722 1080p HD camera sensor.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- */
-
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/string.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/kmod.h>
-#include <linux/device.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <linux/i2c.h>
-#include <linux/moduleparam.h>
-#include <media/v4l2-device.h>
-#include "../include/linux/atomisp_gmin_platform.h"
-#include <linux/acpi.h>
-#include <linux/io.h>
-
-#include "ov2722.h"
-
-/* i2c read/write stuff */
-static int ov2722_read_reg(struct i2c_client *client,
-			   u16 data_length, u16 reg, u16 *val)
-{
-	int err;
-	struct i2c_msg msg[2];
-	unsigned char data[6];
-
-	if (!client->adapter) {
-		dev_err(&client->dev, "%s error, no client->adapter\n",
-			__func__);
-		return -ENODEV;
-	}
-
-	if (data_length != OV2722_8BIT && data_length != OV2722_16BIT
-					&& data_length != OV2722_32BIT) {
-		dev_err(&client->dev, "%s error, invalid data length\n",
-			__func__);
-		return -EINVAL;
-	}
-
-	memset(msg, 0 , sizeof(msg));
-
-	msg[0].addr = client->addr;
-	msg[0].flags = 0;
-	msg[0].len = I2C_MSG_LENGTH;
-	msg[0].buf = data;
-
-	/* high byte goes out first */
-	data[0] = (u8)(reg >> 8);
-	data[1] = (u8)(reg & 0xff);
-
-	msg[1].addr = client->addr;
-	msg[1].len = data_length;
-	msg[1].flags = I2C_M_RD;
-	msg[1].buf = data;
-
-	err = i2c_transfer(client->adapter, msg, 2);
-	if (err != 2) {
-		if (err >= 0)
-			err = -EIO;
-		dev_err(&client->dev,
-			"read from offset 0x%x error %d", reg, err);
-		return err;
-	}
-
-	*val = 0;
-	/* high byte comes first */
-	if (data_length == OV2722_8BIT)
-		*val = (u8)data[0];
-	else if (data_length == OV2722_16BIT)
-		*val = be16_to_cpu(*(__be16 *)&data[0]);
-	else
-		*val = be32_to_cpu(*(__be32 *)&data[0]);
-
-	return 0;
-}
-
-static int ov2722_i2c_write(struct i2c_client *client, u16 len, u8 *data)
-{
-	struct i2c_msg msg;
-	const int num_msg = 1;
-	int ret;
-
-	msg.addr = client->addr;
-	msg.flags = 0;
-	msg.len = len;
-	msg.buf = data;
-	ret = i2c_transfer(client->adapter, &msg, 1);
-
-	return ret == num_msg ? 0 : -EIO;
-}
-
-static int ov2722_write_reg(struct i2c_client *client, u16 data_length,
-							u16 reg, u16 val)
-{
-	int ret;
-	unsigned char data[4] = {0};
-	__be16 *wreg = (__be16 *)data;
-	const u16 len = data_length + sizeof(u16); /* 16-bit address + data */
-
-	if (data_length != OV2722_8BIT && data_length != OV2722_16BIT) {
-		dev_err(&client->dev,
-			"%s error, invalid data_length\n", __func__);
-		return -EINVAL;
-	}
-
-	/* high byte goes out first */
-	*wreg = cpu_to_be16(reg);
-
-	if (data_length == OV2722_8BIT) {
-		data[2] = (u8)(val);
-	} else {
-		/* OV2722_16BIT */
-		__be16 *wdata = (__be16 *)&data[2];
-
-		*wdata = cpu_to_be16(val);
-	}
-
-	ret = ov2722_i2c_write(client, len, data);
-	if (ret)
-		dev_err(&client->dev,
-			"write error: wrote 0x%x to offset 0x%x error %d",
-			val, reg, ret);
-
-	return ret;
-}
-
-/*
- * ov2722_write_reg_array - Initializes a list of OV2722 registers
- * @client: i2c driver client structure
- * @reglist: list of registers to be written
- *
- * This function initializes a list of registers. When consecutive addresses
- * are found in a row on the list, this function creates a buffer and sends
- * consecutive data in a single i2c_transfer().
- *
- * __ov2722_flush_reg_array, __ov2722_buf_reg_array() and
- * __ov2722_write_reg_is_consecutive() are internal functions to
- * ov2722_write_reg_array_fast() and should be not used anywhere else.
- *
- */
-
-static int __ov2722_flush_reg_array(struct i2c_client *client,
-				    struct ov2722_write_ctrl *ctrl)
-{
-	u16 size;
-	__be16 *data16 = (void *)&ctrl->buffer.addr;
-
-	if (ctrl->index == 0)
-		return 0;
-
-	size = sizeof(u16) + ctrl->index; /* 16-bit address + data */
-	*data16 = cpu_to_be16(ctrl->buffer.addr);
-	ctrl->index = 0;
-
-	return ov2722_i2c_write(client, size, (u8 *)&ctrl->buffer);
-}
-
-static int __ov2722_buf_reg_array(struct i2c_client *client,
-				  struct ov2722_write_ctrl *ctrl,
-				  const struct ov2722_reg *next)
-{
-	int size;
-	__be16 *data16;
-
-	switch (next->type) {
-	case OV2722_8BIT:
-		size = 1;
-		ctrl->buffer.data[ctrl->index] = (u8)next->val;
-		break;
-	case OV2722_16BIT:
-		size = 2;
-		data16 = (void *)&ctrl->buffer.data[ctrl->index];
-		*data16 = cpu_to_be16((u16)next->val);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* When first item is added, we need to store its starting address */
-	if (ctrl->index == 0)
-		ctrl->buffer.addr = next->reg;
-
-	ctrl->index += size;
-
-	/*
-	 * Buffer cannot guarantee free space for u32? Better flush it to avoid
-	 * possible lack of memory for next item.
-	 */
-	if (ctrl->index + sizeof(u16) >= OV2722_MAX_WRITE_BUF_SIZE)
-		return __ov2722_flush_reg_array(client, ctrl);
-
-	return 0;
-}
-
-static int __ov2722_write_reg_is_consecutive(struct i2c_client *client,
-					     struct ov2722_write_ctrl *ctrl,
-					     const struct ov2722_reg *next)
-{
-	if (ctrl->index == 0)
-		return 1;
-
-	return ctrl->buffer.addr + ctrl->index == next->reg;
-}
-
-static int ov2722_write_reg_array(struct i2c_client *client,
-				  const struct ov2722_reg *reglist)
-{
-	const struct ov2722_reg *next = reglist;
-	struct ov2722_write_ctrl ctrl;
-	int err;
-
-	ctrl.index = 0;
-	for (; next->type != OV2722_TOK_TERM; next++) {
-		switch (next->type & OV2722_TOK_MASK) {
-		case OV2722_TOK_DELAY:
-			err = __ov2722_flush_reg_array(client, &ctrl);
-			if (err)
-				return err;
-			msleep(next->val);
-			break;
-		default:
-			/*
-			 * If next address is not consecutive, data needs to be
-			 * flushed before proceed.
-			 */
-			if (!__ov2722_write_reg_is_consecutive(client, &ctrl,
-								next)) {
-				err = __ov2722_flush_reg_array(client, &ctrl);
-				if (err)
-					return err;
-			}
-			err = __ov2722_buf_reg_array(client, &ctrl, next);
-			if (err) {
-				dev_err(&client->dev, "%s: write error, aborted\n",
-					 __func__);
-				return err;
-			}
-			break;
-		}
-	}
-
-	return __ov2722_flush_reg_array(client, &ctrl);
-}
-static int ov2722_g_focal(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = (OV2722_FOCAL_LENGTH_NUM << 16) | OV2722_FOCAL_LENGTH_DEM;
-	return 0;
-}
-
-static int ov2722_g_fnumber(struct v4l2_subdev *sd, s32 *val)
-{
-	/*const f number for imx*/
-	*val = (OV2722_F_NUMBER_DEFAULT_NUM << 16) | OV2722_F_NUMBER_DEM;
-	return 0;
-}
-
-static int ov2722_g_fnumber_range(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = (OV2722_F_NUMBER_DEFAULT_NUM << 24) |
-		(OV2722_F_NUMBER_DEM << 16) |
-		(OV2722_F_NUMBER_DEFAULT_NUM << 8) | OV2722_F_NUMBER_DEM;
-	return 0;
-}
-
-static int ov2722_get_intg_factor(struct i2c_client *client,
-				struct camera_mipi_info *info,
-				const struct ov2722_resolution *res)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct ov2722_device *dev = NULL;
-	struct atomisp_sensor_mode_data *buf = &info->data;
-	const unsigned int ext_clk_freq_hz = 19200000;
-	const unsigned int pll_invariant_div = 10;
-	unsigned int pix_clk_freq_hz;
-	u16 pre_pll_clk_div;
-	u16 pll_multiplier;
-	u16 op_pix_clk_div;
-	u16 reg_val;
-	int ret;
-
-	if (!info)
-		return -EINVAL;
-
-	dev = to_ov2722_sensor(sd);
-
-	/* pixel clock calculattion */
-	ret =  ov2722_read_reg(client, OV2722_8BIT,
-				OV2722_SC_CMMN_PLL_CTRL3, &pre_pll_clk_div);
-	if (ret)
-		return ret;
-
-	ret =  ov2722_read_reg(client, OV2722_8BIT,
-				OV2722_SC_CMMN_PLL_MULTIPLIER, &pll_multiplier);
-	if (ret)
-		return ret;
-
-	ret =  ov2722_read_reg(client, OV2722_8BIT,
-				OV2722_SC_CMMN_PLL_DEBUG_OPT, &op_pix_clk_div);
-	if (ret)
-		return ret;
-
-	pre_pll_clk_div = (pre_pll_clk_div & 0x70) >> 4;
-	if (0 == pre_pll_clk_div)
-		return -EINVAL;
-
-	pll_multiplier = pll_multiplier & 0x7f;
-	op_pix_clk_div = op_pix_clk_div & 0x03;
-	pix_clk_freq_hz = ext_clk_freq_hz / pre_pll_clk_div * pll_multiplier
-				* op_pix_clk_div / pll_invariant_div;
-
-	dev->vt_pix_clk_freq_mhz = pix_clk_freq_hz;
-	buf->vt_pix_clk_freq_mhz = pix_clk_freq_hz;
-
-	/* get integration time */
-	buf->coarse_integration_time_min = OV2722_COARSE_INTG_TIME_MIN;
-	buf->coarse_integration_time_max_margin =
-					OV2722_COARSE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_min = OV2722_FINE_INTG_TIME_MIN;
-	buf->fine_integration_time_max_margin =
-					OV2722_FINE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_def = OV2722_FINE_INTG_TIME_MIN;
-	buf->frame_length_lines = res->lines_per_frame;
-	buf->line_length_pck = res->pixels_per_line;
-	buf->read_mode = res->bin_mode;
-
-	/* get the cropping and output resolution to ISP for this mode. */
-	ret =  ov2722_read_reg(client, OV2722_16BIT,
-					OV2722_H_CROP_START_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_horizontal_start = reg_val;
-
-	ret =  ov2722_read_reg(client, OV2722_16BIT,
-					OV2722_V_CROP_START_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_vertical_start = reg_val;
-
-	ret = ov2722_read_reg(client, OV2722_16BIT,
-					OV2722_H_CROP_END_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_horizontal_end = reg_val;
-
-	ret = ov2722_read_reg(client, OV2722_16BIT,
-					OV2722_V_CROP_END_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_vertical_end = reg_val;
-
-	ret = ov2722_read_reg(client, OV2722_16BIT,
-					OV2722_H_OUTSIZE_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_width = reg_val;
-
-	ret = ov2722_read_reg(client, OV2722_16BIT,
-					OV2722_V_OUTSIZE_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_height = reg_val;
-
-	buf->binning_factor_x = res->bin_factor_x ?
-					res->bin_factor_x : 1;
-	buf->binning_factor_y = res->bin_factor_y ?
-					res->bin_factor_y : 1;
-	return 0;
-}
-
-static long __ov2722_set_exposure(struct v4l2_subdev *sd, int coarse_itg,
-				 int gain, int digitgain)
-
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-	u16 hts, vts;
-	int ret;
-
-	dev_dbg(&client->dev, "set_exposure without group hold\n");
-
-	/* clear VTS_DIFF on manual mode */
-	ret = ov2722_write_reg(client, OV2722_16BIT, OV2722_VTS_DIFF_H, 0);
-	if (ret)
-		return ret;
-
-	hts = dev->pixels_per_line;
-	vts = dev->lines_per_frame;
-
-	if ((coarse_itg + OV2722_COARSE_INTG_TIME_MAX_MARGIN) > vts)
-		vts = coarse_itg + OV2722_COARSE_INTG_TIME_MAX_MARGIN;
-
-	coarse_itg <<= 4;
-	digitgain <<= 2;
-
-	ret = ov2722_write_reg(client, OV2722_16BIT,
-				OV2722_VTS_H, vts);
-	if (ret)
-		return ret;
-
-	ret = ov2722_write_reg(client, OV2722_16BIT,
-				OV2722_HTS_H, hts);
-	if (ret)
-		return ret;
-
-	/* set exposure */
-	ret = ov2722_write_reg(client, OV2722_8BIT,
-					OV2722_AEC_PK_EXPO_L,
-					coarse_itg & 0xff);
-	if (ret)
-		return ret;
-
-	ret = ov2722_write_reg(client, OV2722_16BIT,
-					OV2722_AEC_PK_EXPO_H,
-					(coarse_itg >> 8) & 0xfff);
-	if (ret)
-		return ret;
-
-	/* set analog gain */
-	ret = ov2722_write_reg(client, OV2722_16BIT,
-					OV2722_AGC_ADJ_H, gain);
-	if (ret)
-		return ret;
-
-	/* set digital gain */
-	ret = ov2722_write_reg(client, OV2722_16BIT,
-				OV2722_MWB_GAIN_R_H, digitgain);
-	if (ret)
-		return ret;
-
-	ret = ov2722_write_reg(client, OV2722_16BIT,
-				OV2722_MWB_GAIN_G_H, digitgain);
-	if (ret)
-		return ret;
-
-	ret = ov2722_write_reg(client, OV2722_16BIT,
-				OV2722_MWB_GAIN_B_H, digitgain);
-
-	return ret;
-}
-
-static int ov2722_set_exposure(struct v4l2_subdev *sd, int exposure,
-	int gain, int digitgain)
-{
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-	int ret;
-
-	mutex_lock(&dev->input_lock);
-	ret = __ov2722_set_exposure(sd, exposure, gain, digitgain);
-	mutex_unlock(&dev->input_lock);
-
-	return ret;
-}
-
-static long ov2722_s_exposure(struct v4l2_subdev *sd,
-			       struct atomisp_exposure *exposure)
-{
-	int exp = exposure->integration_time[0];
-	int gain = exposure->gain[0];
-	int digitgain = exposure->gain[1];
-
-	/* we should not accept the invalid value below. */
-	if (gain == 0) {
-		struct i2c_client *client = v4l2_get_subdevdata(sd);
-		v4l2_err(client, "%s: invalid value\n", __func__);
-		return -EINVAL;
-	}
-
-	return ov2722_set_exposure(sd, exp, gain, digitgain);
-}
-
-static long ov2722_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
-{
-
-	switch (cmd) {
-	case ATOMISP_IOC_S_EXPOSURE:
-		return ov2722_s_exposure(sd, arg);
-	default:
-		return -EINVAL;
-	}
-	return 0;
-}
-
-/* This returns the exposure time being used. This should only be used
- * for filling in EXIF data, not for actual image processing.
- */
-static int ov2722_q_exposure(struct v4l2_subdev *sd, s32 *value)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u16 reg_v, reg_v2;
-	int ret;
-
-	/* get exposure */
-	ret = ov2722_read_reg(client, OV2722_8BIT,
-					OV2722_AEC_PK_EXPO_L,
-					&reg_v);
-	if (ret)
-		goto err;
-
-	ret = ov2722_read_reg(client, OV2722_8BIT,
-					OV2722_AEC_PK_EXPO_M,
-					&reg_v2);
-	if (ret)
-		goto err;
-
-	reg_v += reg_v2 << 8;
-	ret = ov2722_read_reg(client, OV2722_8BIT,
-					OV2722_AEC_PK_EXPO_H,
-					&reg_v2);
-	if (ret)
-		goto err;
-
-	*value = reg_v + (((u32)reg_v2 << 16));
-err:
-	return ret;
-}
-
-static int ov2722_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct ov2722_device *dev =
-	    container_of(ctrl->handler, struct ov2722_device, ctrl_handler);
-	int ret = 0;
-	unsigned int val;
-	switch (ctrl->id) {
-	case V4L2_CID_EXPOSURE_ABSOLUTE:
-		ret = ov2722_q_exposure(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FOCAL_ABSOLUTE:
-		ret = ov2722_g_focal(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_ABSOLUTE:
-		ret = ov2722_g_fnumber(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_RANGE:
-		ret = ov2722_g_fnumber_range(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_LINK_FREQ:
-		val = ov2722_res[dev->fmt_idx].mipi_freq;
-		if (val == 0)
-			return -EINVAL;
-
-		ctrl->val = val * 1000;	/* To Hz */
-		break;
-	default:
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-static const struct v4l2_ctrl_ops ctrl_ops = {
-	.g_volatile_ctrl = ov2722_g_volatile_ctrl
-};
-
-static const struct v4l2_ctrl_config ov2722_controls[] = {
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_EXPOSURE_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "exposure",
-	 .min = 0x0,
-	 .max = 0xffff,
-	 .step = 0x01,
-	 .def = 0x00,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FOCAL_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "focal length",
-	 .min = OV2722_FOCAL_LENGTH_DEFAULT,
-	 .max = OV2722_FOCAL_LENGTH_DEFAULT,
-	 .step = 0x01,
-	 .def = OV2722_FOCAL_LENGTH_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "f-number",
-	 .min = OV2722_F_NUMBER_DEFAULT,
-	 .max = OV2722_F_NUMBER_DEFAULT,
-	 .step = 0x01,
-	 .def = OV2722_F_NUMBER_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_RANGE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "f-number range",
-	 .min = OV2722_F_NUMBER_RANGE,
-	 .max = OV2722_F_NUMBER_RANGE,
-	 .step = 0x01,
-	 .def = OV2722_F_NUMBER_RANGE,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_LINK_FREQ,
-	 .name = "Link Frequency",
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .min = 1,
-	 .max = 1500000 * 1000,
-	 .step = 1,
-	 .def = 1,
-	 .flags = V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_READ_ONLY,
-	 },
-};
-
-static int ov2722_init(struct v4l2_subdev *sd)
-{
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-
-	mutex_lock(&dev->input_lock);
-
-	/* restore settings */
-	ov2722_res = ov2722_res_preview;
-	N_RES = N_RES_PREVIEW;
-
-	mutex_unlock(&dev->input_lock);
-
-	return 0;
-}
-
-static int power_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	int ret = -1;
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	if (flag) {
-		ret = dev->platform_data->v1p8_ctrl(sd, 1);
-		if (ret == 0) {
-			ret = dev->platform_data->v2p8_ctrl(sd, 1);
-			if (ret)
-				dev->platform_data->v1p8_ctrl(sd, 0);
-		}
-	} else {
-		ret = dev->platform_data->v1p8_ctrl(sd, 0);
-		ret |= dev->platform_data->v2p8_ctrl(sd, 0);
-	}
-
-	return ret;
-}
-
-static int gpio_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-	int ret = -1;
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	/* Note: the GPIO order is asymmetric: always RESET#
-	 * before PWDN# when turning it on or off.
-	 */
-	ret = dev->platform_data->gpio0_ctrl(sd, flag);
-	/*
-	 *ov2722 PWDN# active high when pull down,opposite to the convention
-	 */
-	ret |= dev->platform_data->gpio1_ctrl(sd, !flag);
-	return ret;
-}
-
-static int power_up(struct v4l2_subdev *sd)
-{
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	/* power control */
-	ret = power_ctrl(sd, 1);
-	if (ret)
-		goto fail_power;
-
-	/* according to DS, at least 5ms is needed between DOVDD and PWDN */
-	usleep_range(5000, 6000);
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 1);
-	if (ret) {
-		ret = gpio_ctrl(sd, 0);
-		if (ret)
-			goto fail_power;
-	}
-
-	/* flis clock control */
-	ret = dev->platform_data->flisclk_ctrl(sd, 1);
-	if (ret)
-		goto fail_clk;
-
-	/* according to DS, 20ms is needed between PWDN and i2c access */
-	msleep(20);
-
-	return 0;
-
-fail_clk:
-	gpio_ctrl(sd, 0);
-fail_power:
-	power_ctrl(sd, 0);
-	dev_err(&client->dev, "sensor power-up failed\n");
-
-	return ret;
-}
-
-static int power_down(struct v4l2_subdev *sd)
-{
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	ret = dev->platform_data->flisclk_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "flisclk failed\n");
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 0);
-	if (ret) {
-		ret = gpio_ctrl(sd, 0);
-		if (ret)
-			dev_err(&client->dev, "gpio failed 2\n");
-	}
-
-	/* power control */
-	ret = power_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "vprog failed.\n");
-
-	return ret;
-}
-
-static int ov2722_s_power(struct v4l2_subdev *sd, int on)
-{
-	int ret;
-	if (on == 0)
-		return power_down(sd);
-	else {
-		ret = power_up(sd);
-		if (!ret)
-			return ov2722_init(sd);
-	}
-	return ret;
-}
-
-/*
- * distance - calculate the distance
- * @res: resolution
- * @w: width
- * @h: height
- *
- * Get the gap between resolution and w/h.
- * res->width/height smaller than w/h wouldn't be considered.
- * Returns the value of gap or -1 if fail.
- */
-#define LARGEST_ALLOWED_RATIO_MISMATCH 800
-static int distance(struct ov2722_resolution *res, u32 w, u32 h)
-{
-	unsigned int w_ratio = (res->width << 13) / w;
-	unsigned int h_ratio;
-	int match;
-
-	if (h == 0)
-		return -1;
-	h_ratio = (res->height << 13) / h;
-	if (h_ratio == 0)
-		return -1;
-	match   = abs(((w_ratio << 13) / h_ratio) - 8192);
-
-	if ((w_ratio < 8192) || (h_ratio < 8192) ||
-	    (match > LARGEST_ALLOWED_RATIO_MISMATCH))
-		return -1;
-
-	return w_ratio + h_ratio;
-}
-
-/* Return the nearest higher resolution index */
-static int nearest_resolution_index(int w, int h)
-{
-	int i;
-	int idx = -1;
-	int dist;
-	int min_dist = INT_MAX;
-	struct ov2722_resolution *tmp_res = NULL;
-
-	for (i = 0; i < N_RES; i++) {
-		tmp_res = &ov2722_res[i];
-		dist = distance(tmp_res, w, h);
-		if (dist == -1)
-			continue;
-		if (dist < min_dist) {
-			min_dist = dist;
-			idx = i;
-		}
-	}
-
-	return idx;
-}
-
-static int get_resolution_index(int w, int h)
-{
-	int i;
-
-	for (i = 0; i < N_RES; i++) {
-		if (w != ov2722_res[i].width)
-			continue;
-		if (h != ov2722_res[i].height)
-			continue;
-
-		return i;
-	}
-
-	return -1;
-}
-
-/* TODO: remove it. */
-static int startup(struct v4l2_subdev *sd)
-{
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	ret = ov2722_write_reg(client, OV2722_8BIT,
-					OV2722_SW_RESET, 0x01);
-	if (ret) {
-		dev_err(&client->dev, "ov2722 reset err.\n");
-		return ret;
-	}
-
-	ret = ov2722_write_reg_array(client, ov2722_res[dev->fmt_idx].regs);
-	if (ret) {
-		dev_err(&client->dev, "ov2722 write register err.\n");
-		return ret;
-	}
-
-	return ret;
-}
-
-static int ov2722_set_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_pad_config *cfg,
-			  struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct camera_mipi_info *ov2722_info = NULL;
-	int ret = 0;
-	int idx;
-	if (format->pad)
-		return -EINVAL;
-	if (!fmt)
-		return -EINVAL;
-	ov2722_info = v4l2_get_subdev_hostdata(sd);
-	if (!ov2722_info)
-		return -EINVAL;
-
-	mutex_lock(&dev->input_lock);
-	idx = nearest_resolution_index(fmt->width, fmt->height);
-	if (idx == -1) {
-		/* return the largest resolution */
-		fmt->width = ov2722_res[N_RES - 1].width;
-		fmt->height = ov2722_res[N_RES - 1].height;
-	} else {
-		fmt->width = ov2722_res[idx].width;
-		fmt->height = ov2722_res[idx].height;
-	}
-	fmt->code = MEDIA_BUS_FMT_SGRBG10_1X10;
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
-		cfg->try_fmt = *fmt;
-		mutex_unlock(&dev->input_lock);
-		return 0;
-	}
-
-	dev->fmt_idx = get_resolution_index(fmt->width, fmt->height);
-	if (dev->fmt_idx == -1) {
-		dev_err(&client->dev, "get resolution fail\n");
-		mutex_unlock(&dev->input_lock);
-		return -EINVAL;
-	}
-
-	dev->pixels_per_line = ov2722_res[dev->fmt_idx].pixels_per_line;
-	dev->lines_per_frame = ov2722_res[dev->fmt_idx].lines_per_frame;
-
-	ret = startup(sd);
-	if (ret) {
-		int i = 0;
-		dev_err(&client->dev, "ov2722 startup err, retry to power up\n");
-		for (i = 0; i < OV2722_POWER_UP_RETRY_NUM; i++) {
-			dev_err(&client->dev,
-				"ov2722 retry to power up %d/%d times, result: ",
-				i + 1, OV2722_POWER_UP_RETRY_NUM);
-			power_down(sd);
-			ret = power_up(sd);
-			if (ret) {
-				dev_err(&client->dev, "power up failed, continue\n");
-				continue;
-			}
-			ret = startup(sd);
-			if (ret) {
-				dev_err(&client->dev, " startup FAILED!\n");
-			} else {
-				dev_err(&client->dev, " startup SUCCESS!\n");
-				break;
-			}
-		}
-		if (ret) {
-			dev_err(&client->dev, "ov2722 startup err\n");
-			goto err;
-		}
-	}
-
-	ret = ov2722_get_intg_factor(client, ov2722_info,
-					&ov2722_res[dev->fmt_idx]);
-	if (ret)
-		dev_err(&client->dev, "failed to get integration_factor\n");
-
-err:
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-static int ov2722_get_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_pad_config *cfg,
-			  struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-
-	if (format->pad)
-		return -EINVAL;
-	if (!fmt)
-		return -EINVAL;
-
-	fmt->width = ov2722_res[dev->fmt_idx].width;
-	fmt->height = ov2722_res[dev->fmt_idx].height;
-	fmt->code = MEDIA_BUS_FMT_SBGGR10_1X10;
-
-	return 0;
-}
-
-static int ov2722_detect(struct i2c_client *client)
-{
-	struct i2c_adapter *adapter = client->adapter;
-	u16 high, low;
-	int ret;
-	u16 id;
-	u8 revision;
-
-	if (!i2c_check_functionality(adapter, I2C_FUNC_I2C))
-		return -ENODEV;
-
-	ret = ov2722_read_reg(client, OV2722_8BIT,
-					OV2722_SC_CMMN_CHIP_ID_H, &high);
-	if (ret) {
-		dev_err(&client->dev, "sensor_id_high = 0x%x\n", high);
-		return -ENODEV;
-	}
-	ret = ov2722_read_reg(client, OV2722_8BIT,
-					OV2722_SC_CMMN_CHIP_ID_L, &low);
-	id = (high << 8) | low;
-
-	if ((id != OV2722_ID) && (id != OV2720_ID)) {
-		dev_err(&client->dev, "sensor ID error\n");
-		return -ENODEV;
-	}
-
-	ret = ov2722_read_reg(client, OV2722_8BIT,
-					OV2722_SC_CMMN_SUB_ID, &high);
-	revision = (u8) high & 0x0f;
-
-	dev_dbg(&client->dev, "sensor_revision = 0x%x\n", revision);
-	dev_dbg(&client->dev, "detect ov2722 success\n");
-	return 0;
-}
-
-static int ov2722_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	mutex_lock(&dev->input_lock);
-
-	ret = ov2722_write_reg(client, OV2722_8BIT, OV2722_SW_STREAM,
-				enable ? OV2722_START_STREAMING :
-				OV2722_STOP_STREAMING);
-
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-
-static int ov2722_s_config(struct v4l2_subdev *sd,
-			   int irq, void *platform_data)
-{
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	if (!platform_data)
-		return -ENODEV;
-
-	dev->platform_data =
-		(struct camera_sensor_platform_data *)platform_data;
-
-	mutex_lock(&dev->input_lock);
-
-	/* power off the module, then power on it in future
-	 * as first power on by board may not fulfill the
-	 * power on sequqence needed by the module
-	 */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "ov2722 power-off err.\n");
-		goto fail_power_off;
-	}
-
-	ret = power_up(sd);
-	if (ret) {
-		dev_err(&client->dev, "ov2722 power-up err.\n");
-		goto fail_power_on;
-	}
-
-	ret = dev->platform_data->csi_cfg(sd, 1);
-	if (ret)
-		goto fail_csi_cfg;
-
-	/* config & detect sensor */
-	ret = ov2722_detect(client);
-	if (ret) {
-		dev_err(&client->dev, "ov2722_detect err s_config.\n");
-		goto fail_csi_cfg;
-	}
-
-	/* turn off sensor, after probed */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "ov2722 power-off err.\n");
-		goto fail_csi_cfg;
-	}
-	mutex_unlock(&dev->input_lock);
-
-	return 0;
-
-fail_csi_cfg:
-	dev->platform_data->csi_cfg(sd, 0);
-fail_power_on:
-	power_down(sd);
-	dev_err(&client->dev, "sensor power-gating failed\n");
-fail_power_off:
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-
-static int ov2722_g_frame_interval(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_frame_interval *interval)
-{
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-
-	interval->interval.numerator = 1;
-	interval->interval.denominator = ov2722_res[dev->fmt_idx].fps;
-
-	return 0;
-}
-
-static int ov2722_enum_mbus_code(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_pad_config *cfg,
-				 struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->index >= MAX_FMTS)
-		return -EINVAL;
-
-	code->code = MEDIA_BUS_FMT_SBGGR10_1X10;
-	return 0;
-}
-
-static int ov2722_enum_frame_size(struct v4l2_subdev *sd,
-				  struct v4l2_subdev_pad_config *cfg,
-				  struct v4l2_subdev_frame_size_enum *fse)
-{
-	int index = fse->index;
-
-	if (index >= N_RES)
-		return -EINVAL;
-
-	fse->min_width = ov2722_res[index].width;
-	fse->min_height = ov2722_res[index].height;
-	fse->max_width = ov2722_res[index].width;
-	fse->max_height = ov2722_res[index].height;
-
-	return 0;
-
-}
-
-
-static int ov2722_g_skip_frames(struct v4l2_subdev *sd, u32 *frames)
-{
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-
-	mutex_lock(&dev->input_lock);
-	*frames = ov2722_res[dev->fmt_idx].skip_frames;
-	mutex_unlock(&dev->input_lock);
-
-	return 0;
-}
-
-static const struct v4l2_subdev_sensor_ops ov2722_sensor_ops = {
-	.g_skip_frames	= ov2722_g_skip_frames,
-};
-
-static const struct v4l2_subdev_video_ops ov2722_video_ops = {
-	.s_stream = ov2722_s_stream,
-	.g_frame_interval = ov2722_g_frame_interval,
-};
-
-static const struct v4l2_subdev_core_ops ov2722_core_ops = {
-	.s_power = ov2722_s_power,
-	.ioctl = ov2722_ioctl,
-};
-
-static const struct v4l2_subdev_pad_ops ov2722_pad_ops = {
-	.enum_mbus_code = ov2722_enum_mbus_code,
-	.enum_frame_size = ov2722_enum_frame_size,
-	.get_fmt = ov2722_get_fmt,
-	.set_fmt = ov2722_set_fmt,
-};
-
-static const struct v4l2_subdev_ops ov2722_ops = {
-	.core = &ov2722_core_ops,
-	.video = &ov2722_video_ops,
-	.pad = &ov2722_pad_ops,
-	.sensor = &ov2722_sensor_ops,
-};
-
-static int ov2722_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct ov2722_device *dev = to_ov2722_sensor(sd);
-	dev_dbg(&client->dev, "ov2722_remove...\n");
-
-	dev->platform_data->csi_cfg(sd, 0);
-	v4l2_ctrl_handler_free(&dev->ctrl_handler);
-	v4l2_device_unregister_subdev(sd);
-
-	atomisp_gmin_remove_subdev(sd);
-
-	media_entity_cleanup(&dev->sd.entity);
-	kfree(dev);
-
-	return 0;
-}
-
-static int __ov2722_init_ctrl_handler(struct ov2722_device *dev)
-{
-	struct v4l2_ctrl_handler *hdl;
-	unsigned int i;
-	hdl = &dev->ctrl_handler;
-	v4l2_ctrl_handler_init(&dev->ctrl_handler, ARRAY_SIZE(ov2722_controls));
-	for (i = 0; i < ARRAY_SIZE(ov2722_controls); i++)
-		v4l2_ctrl_new_custom(&dev->ctrl_handler, &ov2722_controls[i],
-				     NULL);
-
-	dev->link_freq = v4l2_ctrl_find(&dev->ctrl_handler, V4L2_CID_LINK_FREQ);
-
-	if (dev->ctrl_handler.error || !dev->link_freq)
-		return dev->ctrl_handler.error;
-
-	dev->sd.ctrl_handler = hdl;
-
-	return 0;
-}
-
-static int ov2722_probe(struct i2c_client *client)
-{
-	struct ov2722_device *dev;
-	void *ovpdev;
-	int ret;
-
-	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
-
-	mutex_init(&dev->input_lock);
-
-	dev->fmt_idx = 0;
-	v4l2_i2c_subdev_init(&(dev->sd), client, &ov2722_ops);
-
-	ovpdev = gmin_camera_platform_data(&dev->sd,
-					   ATOMISP_INPUT_FORMAT_RAW_10,
-					   atomisp_bayer_order_grbg);
-
-	ret = ov2722_s_config(&dev->sd, client->irq, ovpdev);
-	if (ret)
-		goto out_free;
-
-	ret = __ov2722_init_ctrl_handler(dev);
-	if (ret)
-		goto out_ctrl_handler_free;
-
-	dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	dev->pad.flags = MEDIA_PAD_FL_SOURCE;
-	dev->format.code = MEDIA_BUS_FMT_SBGGR10_1X10;
-	dev->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
-
-	ret = media_entity_pads_init(&dev->sd.entity, 1, &dev->pad);
-	if (ret)
-		ov2722_remove(client);
-
-	return atomisp_register_i2c_module(&dev->sd, ovpdev, RAW_CAMERA);
-
-out_ctrl_handler_free:
-	v4l2_ctrl_handler_free(&dev->ctrl_handler);
-
-out_free:
-	v4l2_device_unregister_subdev(&dev->sd);
-	kfree(dev);
-	return ret;
-}
-
-static const struct acpi_device_id ov2722_acpi_match[] = {
-	{ "INT33FB" },
-	{},
-};
-MODULE_DEVICE_TABLE(acpi, ov2722_acpi_match);
-
-static struct i2c_driver ov2722_driver = {
-	.driver = {
-		.name = "ov2722",
-		.acpi_match_table = ov2722_acpi_match,
-	},
-	.probe_new = ov2722_probe,
-	.remove = ov2722_remove,
-};
-module_i2c_driver(ov2722_driver);
-
-MODULE_AUTHOR("Wei Liu <wei.liu@intel.com>");
-MODULE_DESCRIPTION("A low-level driver for OmniVision 2722 sensors");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/atomisp/i2c/gc0310.h b/drivers/staging/media/atomisp/i2c/gc0310.h
deleted file mode 100644
index 70c252c5163c..000000000000
--- a/drivers/staging/media/atomisp/i2c/gc0310.h
+++ /dev/null
@@ -1,404 +0,0 @@
-/*
- * Support for GalaxyCore GC0310 VGA camera sensor.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __GC0310_H__
-#define __GC0310_H__
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/i2c.h>
-#include <linux/acpi.h>
-#include <linux/delay.h>
-#include <linux/videodev2.h>
-#include <linux/spinlock.h>
-#include <media/v4l2-subdev.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-ctrls.h>
-#include <linux/v4l2-mediabus.h>
-#include <media/media-entity.h>
-
-#include "../include/linux/atomisp_platform.h"
-
-/* Defines for register writes and register array processing */
-#define I2C_MSG_LENGTH		1
-#define I2C_RETRY_COUNT		5
-
-#define GC0310_FOCAL_LENGTH_NUM	278	/*2.78mm*/
-#define GC0310_FOCAL_LENGTH_DEM	100
-#define GC0310_F_NUMBER_DEFAULT_NUM	26
-#define GC0310_F_NUMBER_DEM	10
-
-#define MAX_FMTS		1
-
-/*
- * focal length bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define GC0310_FOCAL_LENGTH_DEFAULT 0x1160064
-
-/*
- * current f-number bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define GC0310_F_NUMBER_DEFAULT 0x1a000a
-
-/*
- * f-number range bits definition:
- * bits 31-24: max f-number numerator
- * bits 23-16: max f-number denominator
- * bits 15-8: min f-number numerator
- * bits 7-0: min f-number denominator
- */
-#define GC0310_F_NUMBER_RANGE 0x1a0a1a0a
-#define GC0310_ID	0xa310
-
-#define GC0310_RESET_RELATED		0xFE
-#define GC0310_REGISTER_PAGE_0		0x0
-#define GC0310_REGISTER_PAGE_3		0x3
-
-#define GC0310_FINE_INTG_TIME_MIN 0
-#define GC0310_FINE_INTG_TIME_MAX_MARGIN 0
-#define GC0310_COARSE_INTG_TIME_MIN 1
-#define GC0310_COARSE_INTG_TIME_MAX_MARGIN 6
-
-/*
- * GC0310 System control registers
- */
-#define GC0310_SW_STREAM			0x10
-
-#define GC0310_SC_CMMN_CHIP_ID_H		0xf0
-#define GC0310_SC_CMMN_CHIP_ID_L		0xf1
-
-#define GC0310_AEC_PK_EXPO_H			0x03
-#define GC0310_AEC_PK_EXPO_L			0x04
-#define GC0310_AGC_ADJ			0x48
-#define GC0310_DGC_ADJ			0x71
-#if 0
-#define GC0310_GROUP_ACCESS			0x3208
-#endif
-
-#define GC0310_H_CROP_START_H			0x09
-#define GC0310_H_CROP_START_L			0x0A
-#define GC0310_V_CROP_START_H			0x0B
-#define GC0310_V_CROP_START_L			0x0C
-#define GC0310_H_OUTSIZE_H			0x0F
-#define GC0310_H_OUTSIZE_L			0x10
-#define GC0310_V_OUTSIZE_H			0x0D
-#define GC0310_V_OUTSIZE_L			0x0E
-#define GC0310_H_BLANKING_H			0x05
-#define GC0310_H_BLANKING_L			0x06
-#define GC0310_V_BLANKING_H			0x07
-#define GC0310_V_BLANKING_L			0x08
-#define GC0310_SH_DELAY			0x11
-
-#define GC0310_START_STREAMING			0x94 /* 8-bit enable */
-#define GC0310_STOP_STREAMING			0x0 /* 8-bit disable */
-
-#define GC0310_BIN_FACTOR_MAX			3
-
-struct regval_list {
-	u16 reg_num;
-	u8 value;
-};
-
-struct gc0310_resolution {
-	u8 *desc;
-	const struct gc0310_reg *regs;
-	int res;
-	int width;
-	int height;
-	int fps;
-	int pix_clk_freq;
-	u32 skip_frames;
-	u16 pixels_per_line;
-	u16 lines_per_frame;
-	u8 bin_factor_x;
-	u8 bin_factor_y;
-	u8 bin_mode;
-	bool used;
-};
-
-struct gc0310_format {
-	u8 *desc;
-	u32 pixelformat;
-	struct gc0310_reg *regs;
-};
-
-/*
- * gc0310 device structure.
- */
-struct gc0310_device {
-	struct v4l2_subdev sd;
-	struct media_pad pad;
-	struct v4l2_mbus_framefmt format;
-	struct mutex input_lock;
-	struct v4l2_ctrl_handler ctrl_handler;
-
-	struct camera_sensor_platform_data *platform_data;
-	int vt_pix_clk_freq_mhz;
-	int fmt_idx;
-	u8 res;
-	u8 type;
-};
-
-enum gc0310_tok_type {
-	GC0310_8BIT  = 0x0001,
-	GC0310_TOK_TERM   = 0xf000,	/* terminating token for reg list */
-	GC0310_TOK_DELAY  = 0xfe00,	/* delay token for reg list */
-	GC0310_TOK_MASK = 0xfff0
-};
-
-/**
- * struct gc0310_reg - MI sensor  register format
- * @type: type of the register
- * @reg: 16-bit offset to register
- * @val: 8/16/32-bit register value
- *
- * Define a structure for sensor register initialization values
- */
-struct gc0310_reg {
-	enum gc0310_tok_type type;
-	u8 reg;
-	u8 val;	/* @set value for read/mod/write, @mask */
-};
-
-#define to_gc0310_sensor(x) container_of(x, struct gc0310_device, sd)
-
-#define GC0310_MAX_WRITE_BUF_SIZE	30
-
-struct gc0310_write_buffer {
-	u8 addr;
-	u8 data[GC0310_MAX_WRITE_BUF_SIZE];
-};
-
-struct gc0310_write_ctrl {
-	int index;
-	struct gc0310_write_buffer buffer;
-};
-
-/*
- * Register settings for various resolution
- */
-static const struct gc0310_reg gc0310_reset_register[] = {
-/////////////////////////////////////////////////
-/////////////////	system reg	/////////////////
-/////////////////////////////////////////////////
-	{GC0310_8BIT, 0xfe, 0xf0},
-	{GC0310_8BIT, 0xfe, 0xf0},
-	{GC0310_8BIT, 0xfe, 0x00},
-
-	{GC0310_8BIT, 0xfc, 0x0e}, //4e
-	{GC0310_8BIT, 0xfc, 0x0e}, //16//4e // [0]apwd [6]regf_clk_gate
-	{GC0310_8BIT, 0xf2, 0x80}, //sync output
-	{GC0310_8BIT, 0xf3, 0x00}, //1f//01 data output
-	{GC0310_8BIT, 0xf7, 0x33}, //f9
-	{GC0310_8BIT, 0xf8, 0x05}, //00
-	{GC0310_8BIT, 0xf9, 0x0e}, // 0x8e //0f
-	{GC0310_8BIT, 0xfa, 0x11},
-
-/////////////////////////////////////////////////
-///////////////////   MIPI	 ////////////////////
-/////////////////////////////////////////////////
-	{GC0310_8BIT, 0xfe, 0x03},
-	{GC0310_8BIT, 0x01, 0x03}, ///mipi 1lane
-	{GC0310_8BIT, 0x02, 0x22}, // 0x33
-	{GC0310_8BIT, 0x03, 0x94},
-	{GC0310_8BIT, 0x04, 0x01}, // fifo_prog
-	{GC0310_8BIT, 0x05, 0x00}, //fifo_prog
-	{GC0310_8BIT, 0x06, 0x80}, //b0  //YUV ISP data
-	{GC0310_8BIT, 0x11, 0x2a},//1e //LDI set YUV422
-	{GC0310_8BIT, 0x12, 0x90},//00 //04 //00 //04//00 //LWC[7:0]  //
-	{GC0310_8BIT, 0x13, 0x02},//05 //05 //LWC[15:8]
-	{GC0310_8BIT, 0x15, 0x12}, // 0x10 //DPHYY_MODE read_ready
-	{GC0310_8BIT, 0x17, 0x01},
-	{GC0310_8BIT, 0x40, 0x08},
-	{GC0310_8BIT, 0x41, 0x00},
-	{GC0310_8BIT, 0x42, 0x00},
-	{GC0310_8BIT, 0x43, 0x00},
-	{GC0310_8BIT, 0x21, 0x02}, // 0x01
-	{GC0310_8BIT, 0x22, 0x02}, // 0x01
-	{GC0310_8BIT, 0x23, 0x01}, // 0x05 //Nor:0x05 DOU:0x06
-	{GC0310_8BIT, 0x29, 0x00},
-	{GC0310_8BIT, 0x2A, 0x25}, // 0x05 //data zero 0x7a de
-	{GC0310_8BIT, 0x2B, 0x02},
-
-	{GC0310_8BIT, 0xfe, 0x00},
-
-/////////////////////////////////////////////////
-/////////////////	CISCTL reg	/////////////////
-/////////////////////////////////////////////////
-	{GC0310_8BIT, 0x00, 0x2f}, //2f//0f//02//01
-	{GC0310_8BIT, 0x01, 0x0f}, //06
-	{GC0310_8BIT, 0x02, 0x04},
-	{GC0310_8BIT, 0x4f, 0x00}, //AEC 0FF
-	{GC0310_8BIT, 0x03, 0x01}, // 0x03 //04
-	{GC0310_8BIT, 0x04, 0xc0}, // 0xe8 //58
-	{GC0310_8BIT, 0x05, 0x00},
-	{GC0310_8BIT, 0x06, 0xb2}, // 0x0a //HB
-	{GC0310_8BIT, 0x07, 0x00},
-	{GC0310_8BIT, 0x08, 0x0c}, // 0x89 //VB
-	{GC0310_8BIT, 0x09, 0x00}, //row start
-	{GC0310_8BIT, 0x0a, 0x00}, //
-	{GC0310_8BIT, 0x0b, 0x00}, //col start
-	{GC0310_8BIT, 0x0c, 0x00},
-	{GC0310_8BIT, 0x0d, 0x01}, //height
-	{GC0310_8BIT, 0x0e, 0xf2}, // 0xf7 //height
-	{GC0310_8BIT, 0x0f, 0x02}, //width
-	{GC0310_8BIT, 0x10, 0x94}, // 0xa0 //height
-	{GC0310_8BIT, 0x17, 0x14},
-	{GC0310_8BIT, 0x18, 0x1a}, //0a//[4]double reset
-	{GC0310_8BIT, 0x19, 0x14}, //AD pipeline
-	{GC0310_8BIT, 0x1b, 0x48},
-	{GC0310_8BIT, 0x1e, 0x6b}, //3b//col bias
-	{GC0310_8BIT, 0x1f, 0x28}, //20//00//08//txlow
-	{GC0310_8BIT, 0x20, 0x89}, //88//0c//[3:2]DA15
-	{GC0310_8BIT, 0x21, 0x49}, //48//[3] txhigh
-	{GC0310_8BIT, 0x22, 0xb0},
-	{GC0310_8BIT, 0x23, 0x04}, //[1:0]vcm_r
-	{GC0310_8BIT, 0x24, 0x16}, //15
-	{GC0310_8BIT, 0x34, 0x20}, //[6:4] rsg high//range
-
-/////////////////////////////////////////////////
-////////////////////   BLK	 ////////////////////
-/////////////////////////////////////////////////
-	{GC0310_8BIT, 0x26, 0x23}, //[1]dark_current_en [0]offset_en
-	{GC0310_8BIT, 0x28, 0xff}, //BLK_limie_value
-	{GC0310_8BIT, 0x29, 0x00}, //global offset
-	{GC0310_8BIT, 0x33, 0x18}, //offset_ratio
-	{GC0310_8BIT, 0x37, 0x20}, //dark_current_ratio
-	{GC0310_8BIT, 0x2a, 0x00},
-	{GC0310_8BIT, 0x2b, 0x00},
-	{GC0310_8BIT, 0x2c, 0x00},
-	{GC0310_8BIT, 0x2d, 0x00},
-	{GC0310_8BIT, 0x2e, 0x00},
-	{GC0310_8BIT, 0x2f, 0x00},
-	{GC0310_8BIT, 0x30, 0x00},
-	{GC0310_8BIT, 0x31, 0x00},
-	{GC0310_8BIT, 0x47, 0x80}, //a7
-	{GC0310_8BIT, 0x4e, 0x66}, //select_row
-	{GC0310_8BIT, 0xa8, 0x02}, //win_width_dark, same with crop_win_width
-	{GC0310_8BIT, 0xa9, 0x80},
-
-/////////////////////////////////////////////////
-//////////////////	 ISP reg  ///////////////////
-/////////////////////////////////////////////////
-	{GC0310_8BIT, 0x40, 0x06}, // 0xff //ff //48
-	{GC0310_8BIT, 0x41, 0x00}, // 0x21 //00//[0]curve_en
-	{GC0310_8BIT, 0x42, 0x04}, // 0xcf //0a//[1]awn_en
-	{GC0310_8BIT, 0x44, 0x18}, // 0x18 //02
-	{GC0310_8BIT, 0x46, 0x02}, // 0x03 //sync
-	{GC0310_8BIT, 0x49, 0x03},
-	{GC0310_8BIT, 0x4c, 0x20}, //00[5]pretect exp
-	{GC0310_8BIT, 0x50, 0x01}, //crop enable
-	{GC0310_8BIT, 0x51, 0x00},
-	{GC0310_8BIT, 0x52, 0x00},
-	{GC0310_8BIT, 0x53, 0x00},
-	{GC0310_8BIT, 0x54, 0x01},
-	{GC0310_8BIT, 0x55, 0x01}, //crop window height
-	{GC0310_8BIT, 0x56, 0xf0},
-	{GC0310_8BIT, 0x57, 0x02}, //crop window width
-	{GC0310_8BIT, 0x58, 0x90},
-
-/////////////////////////////////////////////////
-///////////////////   GAIN	 ////////////////////
-/////////////////////////////////////////////////
-	{GC0310_8BIT, 0x70, 0x70}, //70 //80//global gain
-	{GC0310_8BIT, 0x71, 0x20}, // pregain gain
-	{GC0310_8BIT, 0x72, 0x40}, // post gain
-	{GC0310_8BIT, 0x5a, 0x84}, //84//analog gain 0
-	{GC0310_8BIT, 0x5b, 0xc9}, //c9
-	{GC0310_8BIT, 0x5c, 0xed}, //ed//not use pga gain highest level
-	{GC0310_8BIT, 0x77, 0x40}, // R gain 0x74 //awb gain
-	{GC0310_8BIT, 0x78, 0x40}, // G gain
-	{GC0310_8BIT, 0x79, 0x40}, // B gain 0x5f
-
-	{GC0310_8BIT, 0x48, 0x00},
-	{GC0310_8BIT, 0xfe, 0x01},
-	{GC0310_8BIT, 0x0a, 0x45}, //[7]col gain mode
-
-	{GC0310_8BIT, 0x3e, 0x40},
-	{GC0310_8BIT, 0x3f, 0x5c},
-	{GC0310_8BIT, 0x40, 0x7b},
-	{GC0310_8BIT, 0x41, 0xbd},
-	{GC0310_8BIT, 0x42, 0xf6},
-	{GC0310_8BIT, 0x43, 0x63},
-	{GC0310_8BIT, 0x03, 0x60},
-	{GC0310_8BIT, 0x44, 0x03},
-
-/////////////////////////////////////////////////
-/////////////////	dark sun   //////////////////
-/////////////////////////////////////////////////
-	{GC0310_8BIT, 0xfe, 0x01},
-	{GC0310_8BIT, 0x45, 0xa4}, // 0xf7
-	{GC0310_8BIT, 0x46, 0xf0}, // 0xff //f0//sun vaule th
-	{GC0310_8BIT, 0x48, 0x03}, //sun mode
-	{GC0310_8BIT, 0x4f, 0x60}, //sun_clamp
-	{GC0310_8BIT, 0xfe, 0x00},
-
-	{GC0310_TOK_TERM, 0, 0},
-};
-
-static struct gc0310_reg const gc0310_VGA_30fps[] = {
-	{GC0310_8BIT, 0xfe, 0x00},
-	{GC0310_8BIT, 0x0d, 0x01}, //height
-	{GC0310_8BIT, 0x0e, 0xf2}, // 0xf7 //height
-	{GC0310_8BIT, 0x0f, 0x02}, //width
-	{GC0310_8BIT, 0x10, 0x94}, // 0xa0 //height
-
-	{GC0310_8BIT, 0x50, 0x01}, //crop enable
-	{GC0310_8BIT, 0x51, 0x00},
-	{GC0310_8BIT, 0x52, 0x00},
-	{GC0310_8BIT, 0x53, 0x00},
-	{GC0310_8BIT, 0x54, 0x01},
-	{GC0310_8BIT, 0x55, 0x01}, //crop window height
-	{GC0310_8BIT, 0x56, 0xf0},
-	{GC0310_8BIT, 0x57, 0x02}, //crop window width
-	{GC0310_8BIT, 0x58, 0x90},
-
-	{GC0310_8BIT, 0xfe, 0x03},
-	{GC0310_8BIT, 0x12, 0x90},//00 //04 //00 //04//00 //LWC[7:0]  //
-	{GC0310_8BIT, 0x13, 0x02},//05 //05 //LWC[15:8]
-
-	{GC0310_8BIT, 0xfe, 0x00},
-
-	{GC0310_TOK_TERM, 0, 0},
-};
-
-static struct gc0310_resolution gc0310_res_preview[] = {
-	{
-		.desc = "gc0310_VGA_30fps",
-		.width = 656, // 648,
-		.height = 496, // 488,
-		.fps = 30,
-		//.pix_clk_freq = 73,
-		.used = 0,
-#if 0
-		.pixels_per_line = 0x0314,
-		.lines_per_frame = 0x0213,
-#endif
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.skip_frames = 2,
-		.regs = gc0310_VGA_30fps,
-	},
-};
-#define N_RES_PREVIEW (ARRAY_SIZE(gc0310_res_preview))
-
-static struct gc0310_resolution *gc0310_res = gc0310_res_preview;
-static unsigned long N_RES = N_RES_PREVIEW;
-#endif
-
diff --git a/drivers/staging/media/atomisp/i2c/gc2235.h b/drivers/staging/media/atomisp/i2c/gc2235.h
deleted file mode 100644
index 0e805bcfa4d8..000000000000
--- a/drivers/staging/media/atomisp/i2c/gc2235.h
+++ /dev/null
@@ -1,670 +0,0 @@
-/*
- * Support for GalaxyCore GC2235 2M camera sensor.
- *
- * Copyright (c) 2014 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.
- *
- */
-
-#ifndef __GC2235_H__
-#define __GC2235_H__
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/i2c.h>
-#include <linux/delay.h>
-#include <linux/videodev2.h>
-#include <linux/spinlock.h>
-#include <media/v4l2-subdev.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-ctrls.h>
-#include <linux/v4l2-mediabus.h>
-#include <media/media-entity.h>
-
-#include "../include/linux/atomisp_platform.h"
-
-/* Defines for register writes and register array processing */
-#define I2C_MSG_LENGTH		0x2
-#define I2C_RETRY_COUNT		5
-
-#define GC2235_FOCAL_LENGTH_NUM	278	/*2.78mm*/
-#define GC2235_FOCAL_LENGTH_DEM	100
-#define GC2235_F_NUMBER_DEFAULT_NUM	26
-#define GC2235_F_NUMBER_DEM	10
-
-#define MAX_FMTS		1
-
-/*
- * focal length bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define GC2235_FOCAL_LENGTH_DEFAULT 0x1160064
-
-/*
- * current f-number bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define GC2235_F_NUMBER_DEFAULT 0x1a000a
-
-/*
- * f-number range bits definition:
- * bits 31-24: max f-number numerator
- * bits 23-16: max f-number denominator
- * bits 15-8: min f-number numerator
- * bits 7-0: min f-number denominator
- */
-#define GC2235_F_NUMBER_RANGE 0x1a0a1a0a
-#define GC2235_ID	0x2235
-
-#define GC2235_FINE_INTG_TIME_MIN 0
-#define GC2235_FINE_INTG_TIME_MAX_MARGIN 0
-#define GC2235_COARSE_INTG_TIME_MIN 1
-#define GC2235_COARSE_INTG_TIME_MAX_MARGIN 6
-
-/*
- * GC2235 System control registers
- */
-/*
- * GC2235 System control registers
- */
-#define GC2235_SENSOR_ID_H		0xF0
-#define GC2235_SENSOR_ID_L		0xF1
-#define GC2235_RESET_RELATED		0xFE
-#define GC2235_SW_RESET			0x8
-#define GC2235_MIPI_RESET		0x3
-#define GC2235_RESET_BIT		0x4
-#define GC2235_REGISTER_PAGE_0		0x0
-#define GC2235_REGISTER_PAGE_3		0x3
-
-#define GC2235_V_CROP_START_H		0x91
-#define GC2235_V_CROP_START_L		0x92
-#define GC2235_H_CROP_START_H		0x93
-#define GC2235_H_CROP_START_L		0x94
-#define GC2235_V_OUTSIZE_H		0x95
-#define GC2235_V_OUTSIZE_L		0x96
-#define GC2235_H_OUTSIZE_H		0x97
-#define GC2235_H_OUTSIZE_L		0x98
-
-#define GC2235_HB_H			0x5
-#define GC2235_HB_L			0x6
-#define GC2235_VB_H			0x7
-#define GC2235_VB_L			0x8
-#define GC2235_SH_DELAY_H		0x11
-#define GC2235_SH_DELAY_L		0x12
-
-#define GC2235_CSI2_MODE		0x10
-
-#define GC2235_EXPOSURE_H		0x3
-#define GC2235_EXPOSURE_L		0x4
-#define GC2235_GLOBAL_GAIN		0xB0
-#define GC2235_PRE_GAIN			0xB1
-#define GC2235_AWB_R_GAIN		0xB3
-#define GC2235_AWB_G_GAIN		0xB4
-#define GC2235_AWB_B_GAIN		0xB5
-
-#define GC2235_START_STREAMING		0x91
-#define GC2235_STOP_STREAMING		0x0
-
-struct regval_list {
-	u16 reg_num;
-	u8 value;
-};
-
-struct gc2235_resolution {
-	u8 *desc;
-	const struct gc2235_reg *regs;
-	int res;
-	int width;
-	int height;
-	int fps;
-	int pix_clk_freq;
-	u32 skip_frames;
-	u16 pixels_per_line;
-	u16 lines_per_frame;
-	u8 bin_factor_x;
-	u8 bin_factor_y;
-	u8 bin_mode;
-	bool used;
-};
-
-struct gc2235_format {
-	u8 *desc;
-	u32 pixelformat;
-	struct gc2235_reg *regs;
-};
-
-/*
- * gc2235 device structure.
- */
-struct gc2235_device {
-	struct v4l2_subdev sd;
-	struct media_pad pad;
-	struct v4l2_mbus_framefmt format;
-	struct mutex input_lock;
-	struct v4l2_ctrl_handler ctrl_handler;
-
-	struct camera_sensor_platform_data *platform_data;
-	int vt_pix_clk_freq_mhz;
-	int fmt_idx;
-	u8 res;
-	u8 type;
-};
-
-enum gc2235_tok_type {
-	GC2235_8BIT  = 0x0001,
-	GC2235_16BIT = 0x0002,
-	GC2235_32BIT = 0x0004,
-	GC2235_TOK_TERM   = 0xf000,	/* terminating token for reg list */
-	GC2235_TOK_DELAY  = 0xfe00,	/* delay token for reg list */
-	GC2235_TOK_MASK = 0xfff0
-};
-
-/**
- * struct gc2235_reg - MI sensor  register format
- * @type: type of the register
- * @reg: 8-bit offset to register
- * @val: 8/16/32-bit register value
- *
- * Define a structure for sensor register initialization values
- */
-struct gc2235_reg {
-	enum gc2235_tok_type type;
-	u8 reg;
-	u32 val;	/* @set value for read/mod/write, @mask */
-};
-
-#define to_gc2235_sensor(x) container_of(x, struct gc2235_device, sd)
-
-#define GC2235_MAX_WRITE_BUF_SIZE	30
-
-struct gc2235_write_buffer {
-	u8 addr;
-	u8 data[GC2235_MAX_WRITE_BUF_SIZE];
-};
-
-struct gc2235_write_ctrl {
-	int index;
-	struct gc2235_write_buffer buffer;
-};
-
-static struct gc2235_reg const gc2235_stream_on[] = {
-	{ GC2235_8BIT, 0xfe, 0x03}, /* switch to P3 */
-	{ GC2235_8BIT, 0x10, 0x91}, /* start mipi */
-	{ GC2235_8BIT, 0xfe, 0x00}, /* switch to P0 */
-	{ GC2235_TOK_TERM, 0, 0 }
-};
-
-static struct gc2235_reg const gc2235_stream_off[] = {
-	{ GC2235_8BIT, 0xfe, 0x03}, /* switch to P3 */
-	{ GC2235_8BIT, 0x10, 0x01}, /* stop mipi */
-	{ GC2235_8BIT, 0xfe, 0x00}, /* switch to P0 */
-	{ GC2235_TOK_TERM, 0, 0 }
-};
-
-static struct gc2235_reg const gc2235_init_settings[] = {
-	/* Sysytem */
-	{ GC2235_8BIT, 0xfe, 0x80 },
-	{ GC2235_8BIT, 0xfe, 0x80 },
-	{ GC2235_8BIT, 0xfe, 0x80 },
-	{ GC2235_8BIT, 0xf2, 0x00 },
-	{ GC2235_8BIT, 0xf6, 0x00 },
-	{ GC2235_8BIT, 0xfc, 0x06 },
-	{ GC2235_8BIT, 0xf7, 0x15 },
-	{ GC2235_8BIT, 0xf8, 0x84 },
-	{ GC2235_8BIT, 0xf9, 0xfe },
-	{ GC2235_8BIT, 0xfa, 0x00 },
-	{ GC2235_8BIT, 0xfe, 0x00 },
-	/* Analog & cisctl */
-	{ GC2235_8BIT, 0x03, 0x04 },
-	{ GC2235_8BIT, 0x04, 0x9E },
-	{ GC2235_8BIT, 0x05, 0x00 },
-	{ GC2235_8BIT, 0x06, 0xfd },
-	{ GC2235_8BIT, 0x07, 0x00 },
-	{ GC2235_8BIT, 0x08, 0x14 },
-	{ GC2235_8BIT, 0x0a, 0x02 }, /* row start */
-	{ GC2235_8BIT, 0x0c, 0x00 }, /* col start */
-	{ GC2235_8BIT, 0x0d, 0x04 }, /* win height 1232 */
-	{ GC2235_8BIT, 0x0e, 0xd0 },
-	{ GC2235_8BIT, 0x0f, 0x06 }, /* win width: 1616 */
-	{ GC2235_8BIT, 0x10, 0x60 },
-	{ GC2235_8BIT, 0x17, 0x15 }, /* mirror flip */
-	{ GC2235_8BIT, 0x18, 0x1a },
-	{ GC2235_8BIT, 0x19, 0x06 },
-	{ GC2235_8BIT, 0x1a, 0x01 },
-	{ GC2235_8BIT, 0x1b, 0x4d },
-	{ GC2235_8BIT, 0x1e, 0x88 },
-	{ GC2235_8BIT, 0x1f, 0x48 },
-	{ GC2235_8BIT, 0x20, 0x03 },
-	{ GC2235_8BIT, 0x21, 0x7f },
-	{ GC2235_8BIT, 0x22, 0x83 },
-	{ GC2235_8BIT, 0x23, 0x42 },
-	{ GC2235_8BIT, 0x24, 0x16 },
-	{ GC2235_8BIT, 0x26, 0x01 }, /*analog gain*/
-	{ GC2235_8BIT, 0x27, 0x30 },
-	{ GC2235_8BIT, 0x3f, 0x00 }, /* PRC */
-	/* blk */
-	{ GC2235_8BIT, 0x40, 0xa3 },
-	{ GC2235_8BIT, 0x41, 0x82 },
-	{ GC2235_8BIT, 0x43, 0x20 },
-	{ GC2235_8BIT, 0x5e, 0x18 },
-	{ GC2235_8BIT, 0x5f, 0x18 },
-	{ GC2235_8BIT, 0x60, 0x18 },
-	{ GC2235_8BIT, 0x61, 0x18 },
-	{ GC2235_8BIT, 0x62, 0x18 },
-	{ GC2235_8BIT, 0x63, 0x18 },
-	{ GC2235_8BIT, 0x64, 0x18 },
-	{ GC2235_8BIT, 0x65, 0x18 },
-	{ GC2235_8BIT, 0x66, 0x20 },
-	{ GC2235_8BIT, 0x67, 0x20 },
-	{ GC2235_8BIT, 0x68, 0x20 },
-	{ GC2235_8BIT, 0x69, 0x20 },
-	/* Gain */
-	{ GC2235_8BIT, 0xb2, 0x00 },
-	{ GC2235_8BIT, 0xb3, 0x40 },
-	{ GC2235_8BIT, 0xb4, 0x40 },
-	{ GC2235_8BIT, 0xb5, 0x40 },
-	/* Dark sun */
-	{ GC2235_8BIT, 0xbc, 0x00 },
-
-	{ GC2235_8BIT, 0xfe, 0x03 },
-	{ GC2235_8BIT, 0x10, 0x01 }, /* disable mipi */
-	{ GC2235_8BIT, 0xfe, 0x00 }, /* switch to P0 */
-	{ GC2235_TOK_TERM, 0, 0 }
-};
-/*
- * Register settings for various resolution
- */
-static struct gc2235_reg const gc2235_1296_736_30fps[] = {
-	{ GC2235_8BIT, 0x8b, 0xa0 },
-	{ GC2235_8BIT, 0x8c, 0x02 },
-
-	{ GC2235_8BIT, 0x07, 0x01 }, /* VBI */
-	{ GC2235_8BIT, 0x08, 0x44 },
-	{ GC2235_8BIT, 0x09, 0x00 }, /* row start */
-	{ GC2235_8BIT, 0x0a, 0xf0 },
-	{ GC2235_8BIT, 0x0b, 0x00 }, /* col start */
-	{ GC2235_8BIT, 0x0c, 0xa0 },
-	{ GC2235_8BIT, 0x0d, 0x02 }, /* win height 736 */
-	{ GC2235_8BIT, 0x0e, 0xf0 },
-	{ GC2235_8BIT, 0x0f, 0x05 }, /* win width: 1296 */
-	{ GC2235_8BIT, 0x10, 0x20 },
-
-	{ GC2235_8BIT, 0x90, 0x01 },
-	{ GC2235_8BIT, 0x92, 0x08 },
-	{ GC2235_8BIT, 0x94, 0x08 },
-	{ GC2235_8BIT, 0x95, 0x02 }, /* crop win height 736 */
-	{ GC2235_8BIT, 0x96, 0xe0 },
-	{ GC2235_8BIT, 0x97, 0x05 }, /* crop win width 1296 */
-	{ GC2235_8BIT, 0x98, 0x10 },
-	/* mimi init */
-	{ GC2235_8BIT, 0xfe, 0x03 }, /* switch to P3 */
-	{ GC2235_8BIT, 0x01, 0x07 },
-	{ GC2235_8BIT, 0x02, 0x11 },
-	{ GC2235_8BIT, 0x03, 0x11 },
-	{ GC2235_8BIT, 0x06, 0x80 },
-	{ GC2235_8BIT, 0x11, 0x2b },
-	/* set mipi buffer */
-	{ GC2235_8BIT, 0x12, 0x54 }, /* val_low = (width * 10 / 8) & 0xFF */
-	{ GC2235_8BIT, 0x13, 0x06 }, /* val_high = (width * 10 / 8) >> 8 */
-
-	{ GC2235_8BIT, 0x15, 0x12 }, /* DPHY mode*/
-	{ GC2235_8BIT, 0x04, 0x10 },
-	{ GC2235_8BIT, 0x05, 0x00 },
-	{ GC2235_8BIT, 0x17, 0x01 },
-
-	{ GC2235_8BIT, 0x22, 0x01 },
-	{ GC2235_8BIT, 0x23, 0x05 },
-	{ GC2235_8BIT, 0x24, 0x10 },
-	{ GC2235_8BIT, 0x25, 0x10 },
-	{ GC2235_8BIT, 0x26, 0x02 },
-	{ GC2235_8BIT, 0x21, 0x10 },
-	{ GC2235_8BIT, 0x29, 0x01 },
-	{ GC2235_8BIT, 0x2a, 0x02 },
-	{ GC2235_8BIT, 0x2b, 0x02 },
-
-	{ GC2235_8BIT, 0x10, 0x01 }, /* disable mipi */
-	{ GC2235_8BIT, 0xfe, 0x00 }, /* switch to P0 */
-	{ GC2235_TOK_TERM, 0, 0 }
-};
-
-static struct gc2235_reg const gc2235_960_640_30fps[] = {
-	{ GC2235_8BIT, 0x8b, 0xa0 },
-	{ GC2235_8BIT, 0x8c, 0x02 },
-
-	{ GC2235_8BIT, 0x07, 0x02 }, /* VBI */
-	{ GC2235_8BIT, 0x08, 0xA4 },
-	{ GC2235_8BIT, 0x09, 0x01 }, /* row start */
-	{ GC2235_8BIT, 0x0a, 0x18 },
-	{ GC2235_8BIT, 0x0b, 0x01 }, /* col start */
-	{ GC2235_8BIT, 0x0c, 0x40 },
-	{ GC2235_8BIT, 0x0d, 0x02 }, /* win height 656 */
-	{ GC2235_8BIT, 0x0e, 0x90 },
-	{ GC2235_8BIT, 0x0f, 0x03 }, /* win width: 976 */
-	{ GC2235_8BIT, 0x10, 0xd0 },
-
-	{ GC2235_8BIT, 0x90, 0x01 },
-	{ GC2235_8BIT, 0x92, 0x02 },
-	{ GC2235_8BIT, 0x94, 0x06 },
-	{ GC2235_8BIT, 0x95, 0x02 }, /* crop win height 640 */
-	{ GC2235_8BIT, 0x96, 0x80 },
-	{ GC2235_8BIT, 0x97, 0x03 }, /* crop win width 960 */
-	{ GC2235_8BIT, 0x98, 0xc0 },
-	/* mimp init */
-	{ GC2235_8BIT, 0xfe, 0x03 }, /* switch to P3 */
-	{ GC2235_8BIT, 0x01, 0x07 },
-	{ GC2235_8BIT, 0x02, 0x11 },
-	{ GC2235_8BIT, 0x03, 0x11 },
-	{ GC2235_8BIT, 0x06, 0x80 },
-	{ GC2235_8BIT, 0x11, 0x2b },
-	/* set mipi buffer */
-	{ GC2235_8BIT, 0x12, 0xb0 }, /* val_low = (width * 10 / 8) & 0xFF */
-	{ GC2235_8BIT, 0x13, 0x04 }, /* val_high = (width * 10 / 8) >> 8 */
-
-	{ GC2235_8BIT, 0x15, 0x12 }, /* DPHY mode*/
-	{ GC2235_8BIT, 0x04, 0x10 },
-	{ GC2235_8BIT, 0x05, 0x00 },
-	{ GC2235_8BIT, 0x17, 0x01 },
-	{ GC2235_8BIT, 0x22, 0x01 },
-	{ GC2235_8BIT, 0x23, 0x05 },
-	{ GC2235_8BIT, 0x24, 0x10 },
-	{ GC2235_8BIT, 0x25, 0x10 },
-	{ GC2235_8BIT, 0x26, 0x02 },
-	{ GC2235_8BIT, 0x21, 0x10 },
-	{ GC2235_8BIT, 0x29, 0x01 },
-	{ GC2235_8BIT, 0x2a, 0x02 },
-	{ GC2235_8BIT, 0x2b, 0x02 },
-	{ GC2235_8BIT, 0x10, 0x01 }, /* disable mipi */
-	{ GC2235_8BIT, 0xfe, 0x00 }, /* switch to P0 */
-	{ GC2235_TOK_TERM, 0, 0 }
-};
-
-static struct gc2235_reg const gc2235_1600_900_30fps[] = {
-	{ GC2235_8BIT, 0x8b, 0xa0 },
-	{ GC2235_8BIT, 0x8c, 0x02 },
-
-	{ GC2235_8BIT, 0x0d, 0x03 }, /* win height 932 */
-	{ GC2235_8BIT, 0x0e, 0xa4 },
-	{ GC2235_8BIT, 0x0f, 0x06 }, /* win width: 1632 */
-	{ GC2235_8BIT, 0x10, 0x50 },
-
-	{ GC2235_8BIT, 0x90, 0x01 },
-	{ GC2235_8BIT, 0x92, 0x02 },
-	{ GC2235_8BIT, 0x94, 0x06 },
-	{ GC2235_8BIT, 0x95, 0x03 }, /* crop win height 900 */
-	{ GC2235_8BIT, 0x96, 0x84 },
-	{ GC2235_8BIT, 0x97, 0x06 }, /* crop win width 1600 */
-	{ GC2235_8BIT, 0x98, 0x40 },
-	/* mimi init */
-	{ GC2235_8BIT, 0xfe, 0x03 }, /* switch to P3 */
-	{ GC2235_8BIT, 0x01, 0x07 },
-	{ GC2235_8BIT, 0x02, 0x11 },
-	{ GC2235_8BIT, 0x03, 0x11 },
-	{ GC2235_8BIT, 0x06, 0x80 },
-	{ GC2235_8BIT, 0x11, 0x2b },
-	/* set mipi buffer */
-	{ GC2235_8BIT, 0x12, 0xd0 }, /* val_low = (width * 10 / 8) & 0xFF */
-	{ GC2235_8BIT, 0x13, 0x07 }, /* val_high = (width * 10 / 8) >> 8 */
-
-	{ GC2235_8BIT, 0x15, 0x12 }, /* DPHY mode*/
-	{ GC2235_8BIT, 0x04, 0x10 },
-	{ GC2235_8BIT, 0x05, 0x00 },
-	{ GC2235_8BIT, 0x17, 0x01 },
-	{ GC2235_8BIT, 0x22, 0x01 },
-	{ GC2235_8BIT, 0x23, 0x05 },
-	{ GC2235_8BIT, 0x24, 0x10 },
-	{ GC2235_8BIT, 0x25, 0x10 },
-	{ GC2235_8BIT, 0x26, 0x02 },
-	{ GC2235_8BIT, 0x21, 0x10 },
-	{ GC2235_8BIT, 0x29, 0x01 },
-	{ GC2235_8BIT, 0x2a, 0x02 },
-	{ GC2235_8BIT, 0x2b, 0x02 },
-	{ GC2235_8BIT, 0x10, 0x01 }, /* disable mipi */
-	{ GC2235_8BIT, 0xfe, 0x00 }, /* switch to P0 */
-	{ GC2235_TOK_TERM, 0, 0 }
-};
-
-static struct gc2235_reg const gc2235_1616_1082_30fps[] = {
-	{ GC2235_8BIT, 0x8b, 0xa0 },
-	{ GC2235_8BIT, 0x8c, 0x02 },
-
-	{ GC2235_8BIT, 0x0d, 0x04 }, /* win height 1232 */
-	{ GC2235_8BIT, 0x0e, 0xd0 },
-	{ GC2235_8BIT, 0x0f, 0x06 }, /* win width: 1616 */
-	{ GC2235_8BIT, 0x10, 0x50 },
-
-	{ GC2235_8BIT, 0x90, 0x01 },
-	{ GC2235_8BIT, 0x92, 0x4a },
-	{ GC2235_8BIT, 0x94, 0x00 },
-	{ GC2235_8BIT, 0x95, 0x04 }, /* crop win height 1082 */
-	{ GC2235_8BIT, 0x96, 0x3a },
-	{ GC2235_8BIT, 0x97, 0x06 }, /* crop win width 1616 */
-	{ GC2235_8BIT, 0x98, 0x50 },
-	/* mimp init */
-	{ GC2235_8BIT, 0xfe, 0x03 }, /* switch to P3 */
-	{ GC2235_8BIT, 0x01, 0x07 },
-	{ GC2235_8BIT, 0x02, 0x11 },
-	{ GC2235_8BIT, 0x03, 0x11 },
-	{ GC2235_8BIT, 0x06, 0x80 },
-	{ GC2235_8BIT, 0x11, 0x2b },
-	/* set mipi buffer */
-	{ GC2235_8BIT, 0x12, 0xe4 }, /* val_low = (width * 10 / 8) & 0xFF */
-	{ GC2235_8BIT, 0x13, 0x07 }, /* val_high = (width * 10 / 8) >> 8 */
-
-	{ GC2235_8BIT, 0x15, 0x12 }, /* DPHY mode*/
-	{ GC2235_8BIT, 0x04, 0x10 },
-	{ GC2235_8BIT, 0x05, 0x00 },
-	{ GC2235_8BIT, 0x17, 0x01 },
-	{ GC2235_8BIT, 0x22, 0x01 },
-	{ GC2235_8BIT, 0x23, 0x05 },
-	{ GC2235_8BIT, 0x24, 0x10 },
-	{ GC2235_8BIT, 0x25, 0x10 },
-	{ GC2235_8BIT, 0x26, 0x02 },
-	{ GC2235_8BIT, 0x21, 0x10 },
-	{ GC2235_8BIT, 0x29, 0x01 },
-	{ GC2235_8BIT, 0x2a, 0x02 },
-	{ GC2235_8BIT, 0x2b, 0x02 },
-	{ GC2235_8BIT, 0x10, 0x01 }, /* disable mipi */
-	{ GC2235_8BIT, 0xfe, 0x00 }, /* switch to P0 */
-	{ GC2235_TOK_TERM, 0, 0 }
-};
-
-static struct gc2235_reg const gc2235_1616_1216_30fps[] = {
-	{ GC2235_8BIT, 0x8b, 0xa0 },
-	{ GC2235_8BIT, 0x8c, 0x02 },
-
-	{ GC2235_8BIT, 0x0d, 0x04 }, /* win height 1232 */
-	{ GC2235_8BIT, 0x0e, 0xd0 },
-	{ GC2235_8BIT, 0x0f, 0x06 }, /* win width: 1616 */
-	{ GC2235_8BIT, 0x10, 0x50 },
-
-	{ GC2235_8BIT, 0x90, 0x01 },
-	{ GC2235_8BIT, 0x92, 0x02 },
-	{ GC2235_8BIT, 0x94, 0x00 },
-	{ GC2235_8BIT, 0x95, 0x04 }, /* crop win height 1216 */
-	{ GC2235_8BIT, 0x96, 0xc0 },
-	{ GC2235_8BIT, 0x97, 0x06 }, /* crop win width 1616 */
-	{ GC2235_8BIT, 0x98, 0x50 },
-	/* mimi init */
-	{ GC2235_8BIT, 0xfe, 0x03 }, /* switch to P3 */
-	{ GC2235_8BIT, 0x01, 0x07 },
-	{ GC2235_8BIT, 0x02, 0x11 },
-	{ GC2235_8BIT, 0x03, 0x11 },
-	{ GC2235_8BIT, 0x06, 0x80 },
-	{ GC2235_8BIT, 0x11, 0x2b },
-	/* set mipi buffer */
-	{ GC2235_8BIT, 0x12, 0xe4 }, /* val_low = (width * 10 / 8) & 0xFF */
-	{ GC2235_8BIT, 0x13, 0x07 }, /* val_high = (width * 10 / 8) >> 8 */
-	{ GC2235_8BIT, 0x15, 0x12 }, /* DPHY mode*/
-	{ GC2235_8BIT, 0x04, 0x10 },
-	{ GC2235_8BIT, 0x05, 0x00 },
-	{ GC2235_8BIT, 0x17, 0x01 },
-	{ GC2235_8BIT, 0x22, 0x01 },
-	{ GC2235_8BIT, 0x23, 0x05 },
-	{ GC2235_8BIT, 0x24, 0x10 },
-	{ GC2235_8BIT, 0x25, 0x10 },
-	{ GC2235_8BIT, 0x26, 0x02 },
-	{ GC2235_8BIT, 0x21, 0x10 },
-	{ GC2235_8BIT, 0x29, 0x01 },
-	{ GC2235_8BIT, 0x2a, 0x02 },
-	{ GC2235_8BIT, 0x2b, 0x02 },
-	{ GC2235_8BIT, 0x10, 0x01 }, /* disable mipi */
-	{ GC2235_8BIT, 0xfe, 0x00 }, /* switch to P0 */
-	{ GC2235_TOK_TERM, 0, 0 }
-};
-
-static struct gc2235_resolution gc2235_res_preview[] = {
-
-	{
-		.desc = "gc2235_1600_900_30fps",
-		.width = 1600,
-		.height = 900,
-		.pix_clk_freq = 30,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2132,
-		.lines_per_frame = 1068,
-		.bin_factor_x = 0,
-		.bin_factor_y = 0,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = gc2235_1600_900_30fps,
-	},
-
-	{
-		.desc = "gc2235_1600_1066_30fps",
-		.width = 1616,
-		.height = 1082,
-		.pix_clk_freq = 30,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2132,
-		.lines_per_frame = 1368,
-		.bin_factor_x = 0,
-		.bin_factor_y = 0,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = gc2235_1616_1082_30fps,
-	},
-	{
-		.desc = "gc2235_1600_1200_30fps",
-		.width = 1616,
-		.height = 1216,
-		.pix_clk_freq = 30,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2132,
-		.lines_per_frame = 1368,
-		.bin_factor_x = 0,
-		.bin_factor_y = 0,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = gc2235_1616_1216_30fps,
-	},
-
-};
-#define N_RES_PREVIEW (ARRAY_SIZE(gc2235_res_preview))
-
-/*
- * Disable non-preview configurations until the configuration selection is
- * improved.
- */
-#if 0
-static struct gc2235_resolution gc2235_res_still[] = {
-	{
-		.desc = "gc2235_1600_900_30fps",
-		.width = 1600,
-		.height = 900,
-		.pix_clk_freq = 30,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2132,
-		.lines_per_frame = 1068,
-		.bin_factor_x = 0,
-		.bin_factor_y = 0,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = gc2235_1600_900_30fps,
-	},
-	{
-		.desc = "gc2235_1600_1066_30fps",
-		.width = 1616,
-		.height = 1082,
-		.pix_clk_freq = 30,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2132,
-		.lines_per_frame = 1368,
-		.bin_factor_x = 0,
-		.bin_factor_y = 0,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = gc2235_1616_1082_30fps,
-	},
-	{
-		.desc = "gc2235_1600_1200_30fps",
-		.width = 1616,
-		.height = 1216,
-		.pix_clk_freq = 30,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2132,
-		.lines_per_frame = 1368,
-		.bin_factor_x = 0,
-		.bin_factor_y = 0,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = gc2235_1616_1216_30fps,
-	},
-
-};
-#define N_RES_STILL (ARRAY_SIZE(gc2235_res_still))
-
-static struct gc2235_resolution gc2235_res_video[] = {
-	{
-		.desc = "gc2235_1296_736_30fps",
-		.width = 1296,
-		.height = 736,
-		.pix_clk_freq = 30,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 1828,
-		.lines_per_frame = 888,
-		.bin_factor_x = 0,
-		.bin_factor_y = 0,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = gc2235_1296_736_30fps,
-	},
-	{
-		.desc = "gc2235_960_640_30fps",
-		.width = 960,
-		.height = 640,
-		.pix_clk_freq = 30,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 1492,
-		.lines_per_frame = 792,
-		.bin_factor_x = 0,
-		.bin_factor_y = 0,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = gc2235_960_640_30fps,
-	},
-
-};
-#define N_RES_VIDEO (ARRAY_SIZE(gc2235_res_video))
-#endif
-
-static struct gc2235_resolution *gc2235_res = gc2235_res_preview;
-static unsigned long N_RES = N_RES_PREVIEW;
-#endif
diff --git a/drivers/staging/media/atomisp/i2c/mt9m114.h b/drivers/staging/media/atomisp/i2c/mt9m114.h
deleted file mode 100644
index 0af79d77a404..000000000000
--- a/drivers/staging/media/atomisp/i2c/mt9m114.h
+++ /dev/null
@@ -1,1777 +0,0 @@
-/*
- * Support for mt9m114 Camera Sensor.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __A1040_H__
-#define __A1040_H__
-
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/i2c.h>
-#include <linux/delay.h>
-#include <linux/videodev2.h>
-#include <linux/spinlock.h>
-#include <media/v4l2-subdev.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-ctrls.h>
-#include <linux/v4l2-mediabus.h>
-#include <media/media-entity.h>
-#include "../include/linux/atomisp_platform.h"
-#include "../include/linux/atomisp.h"
-
-#define V4L2_IDENT_MT9M114 8245
-
-#define MT9P111_REV3
-#define FULLINISUPPORT
-
-/* #defines for register writes and register array processing */
-#define MISENSOR_8BIT		1
-#define MISENSOR_16BIT		2
-#define MISENSOR_32BIT		4
-
-#define MISENSOR_FWBURST0	0x80
-#define MISENSOR_FWBURST1	0x81
-#define MISENSOR_FWBURST4	0x84
-#define MISENSOR_FWBURST	0x88
-
-#define MISENSOR_TOK_TERM	0xf000	/* terminating token for reg list */
-#define MISENSOR_TOK_DELAY	0xfe00	/* delay token for reg list */
-#define MISENSOR_TOK_FWLOAD	0xfd00	/* token indicating load FW */
-#define MISENSOR_TOK_POLL	0xfc00	/* token indicating poll instruction */
-#define MISENSOR_TOK_RMW	0x0010  /* RMW operation */
-#define MISENSOR_TOK_MASK	0xfff0
-#define MISENSOR_AWB_STEADY	(1<<0)	/* awb steady */
-#define MISENSOR_AE_READY	(1<<3)	/* ae status ready */
-
-/* mask to set sensor read_mode via misensor_rmw_reg */
-#define MISENSOR_R_MODE_MASK	0x0330
-/* mask to set sensor vert_flip and horz_mirror */
-#define MISENSOR_VFLIP_MASK	0x0002
-#define MISENSOR_HFLIP_MASK	0x0001
-#define MISENSOR_FLIP_EN	1
-#define MISENSOR_FLIP_DIS	0
-
-/* bits set to set sensor read_mode via misensor_rmw_reg */
-#define MISENSOR_SKIPPING_SET	0x0011
-#define MISENSOR_SUMMING_SET	0x0033
-#define MISENSOR_NORMAL_SET	0x0000
-
-/* sensor register that control sensor read-mode and mirror */
-#define MISENSOR_READ_MODE	0xC834
-/* sensor ae-track status register */
-#define MISENSOR_AE_TRACK_STATUS	0xA800
-/* sensor awb status register */
-#define MISENSOR_AWB_STATUS	0xAC00
-/* sensor coarse integration time register */
-#define MISENSOR_COARSE_INTEGRATION_TIME 0xC83C
-
-/* registers */
-#define REG_SW_RESET                    0x301A
-#define REG_SW_STREAM                   0xDC00
-#define REG_SCCB_CTRL                   0x3100
-#define REG_SC_CMMN_CHIP_ID             0x0000
-#define REG_V_START                     0xc800 /* 16bits */
-#define REG_H_START                     0xc802 /* 16bits */
-#define REG_V_END                       0xc804 /* 16bits */
-#define REG_H_END                       0xc806 /* 16bits */
-#define REG_PIXEL_CLK                   0xc808 /* 32bits */
-#define REG_TIMING_VTS                  0xc812 /* 16bits */
-#define REG_TIMING_HTS                  0xc814 /* 16bits */
-#define REG_WIDTH                       0xC868 /* 16bits */
-#define REG_HEIGHT                      0xC86A /* 16bits */
-#define REG_EXPO_COARSE                 0x3012 /* 16bits */
-#define REG_EXPO_FINE                   0x3014 /* 16bits */
-#define REG_GAIN                        0x305E
-#define REG_ANALOGGAIN                  0x305F
-#define REG_ADDR_ACESSS                 0x098E /* logical_address_access */
-#define REG_COMM_Register               0x0080 /* command_register */
-
-#define SENSOR_DETECTED		1
-#define SENSOR_NOT_DETECTED	0
-
-#define I2C_RETRY_COUNT		5
-#define MSG_LEN_OFFSET		2
-
-#ifndef MIPI_CONTROL
-#define MIPI_CONTROL		0x3400	/* MIPI_Control */
-#endif
-
-/* GPIO pin on Moorestown */
-#define GPIO_SCLK_25		44
-#define GPIO_STB_PIN		47
-
-#define GPIO_STDBY_PIN		49   /* ab:new */
-#define GPIO_RESET_PIN		50
-
-/* System control register for Aptina A-1040SOC*/
-#define MT9M114_PID		0x0
-
-/* MT9P111_DEVICE_ID */
-#define MT9M114_MOD_ID		0x2481
-
-#define MT9M114_FINE_INTG_TIME_MIN 0
-#define MT9M114_FINE_INTG_TIME_MAX_MARGIN 0
-#define MT9M114_COARSE_INTG_TIME_MIN 1
-#define MT9M114_COARSE_INTG_TIME_MAX_MARGIN 6
-
-
-/* ulBPat; */
-
-#define MT9M114_BPAT_RGRGGBGB	(1 << 0)
-#define MT9M114_BPAT_GRGRBGBG	(1 << 1)
-#define MT9M114_BPAT_GBGBRGRG	(1 << 2)
-#define MT9M114_BPAT_BGBGGRGR	(1 << 3)
-
-#define MT9M114_FOCAL_LENGTH_NUM	208	/*2.08mm*/
-#define MT9M114_FOCAL_LENGTH_DEM	100
-#define MT9M114_F_NUMBER_DEFAULT_NUM	24
-#define MT9M114_F_NUMBER_DEM	10
-#define MT9M114_WAIT_STAT_TIMEOUT	100
-#define MT9M114_FLICKER_MODE_50HZ	1
-#define MT9M114_FLICKER_MODE_60HZ	2
-/*
- * focal length bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define MT9M114_FOCAL_LENGTH_DEFAULT 0xD00064
-
-/*
- * current f-number bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define MT9M114_F_NUMBER_DEFAULT 0x18000a
-
-/*
- * f-number range bits definition:
- * bits 31-24: max f-number numerator
- * bits 23-16: max f-number denominator
- * bits 15-8: min f-number numerator
- * bits 7-0: min f-number denominator
- */
-#define MT9M114_F_NUMBER_RANGE 0x180a180a
-
-/* Supported resolutions */
-enum {
-	MT9M114_RES_736P,
-	MT9M114_RES_864P,
-	MT9M114_RES_960P,
-};
-#define MT9M114_RES_960P_SIZE_H		1296
-#define MT9M114_RES_960P_SIZE_V		976
-#define MT9M114_RES_720P_SIZE_H		1280
-#define MT9M114_RES_720P_SIZE_V		720
-#define MT9M114_RES_576P_SIZE_H		1024
-#define MT9M114_RES_576P_SIZE_V		576
-#define MT9M114_RES_480P_SIZE_H		768
-#define MT9M114_RES_480P_SIZE_V		480
-#define MT9M114_RES_VGA_SIZE_H		640
-#define MT9M114_RES_VGA_SIZE_V		480
-#define MT9M114_RES_QVGA_SIZE_H		320
-#define MT9M114_RES_QVGA_SIZE_V		240
-#define MT9M114_RES_QCIF_SIZE_H		176
-#define MT9M114_RES_QCIF_SIZE_V		144
-
-#define MT9M114_RES_720_480p_768_SIZE_H 736
-#define MT9M114_RES_720_480p_768_SIZE_V 496
-#define MT9M114_RES_736P_SIZE_H 1296
-#define MT9M114_RES_736P_SIZE_V 736
-#define MT9M114_RES_864P_SIZE_H 1296
-#define MT9M114_RES_864P_SIZE_V 864
-#define MT9M114_RES_976P_SIZE_H 1296
-#define MT9M114_RES_976P_SIZE_V 976
-
-#define MT9M114_BIN_FACTOR_MAX			3
-
-#define MT9M114_DEFAULT_FIRST_EXP 0x10
-#define MT9M114_MAX_FIRST_EXP 0x302
-
-/* completion status polling requirements, usage based on Aptina .INI Rev2 */
-enum poll_reg {
-	NO_POLLING,
-	PRE_POLLING,
-	POST_POLLING,
-};
-/*
- * struct misensor_reg - MI sensor  register format
- * @length: length of the register
- * @reg: 16-bit offset to register
- * @val: 8/16/32-bit register value
- * Define a structure for sensor register initialization values
- */
-struct misensor_reg {
-	u32 length;
-	u32 reg;
-	u32 val;	/* value or for read/mod/write, AND mask */
-	u32 val2;	/* optional; for rmw, OR mask */
-};
-
-/*
- * struct misensor_fwreg - Firmware burst command
- * @type: FW burst or 8/16 bit register
- * @addr: 16-bit offset to register or other values depending on type
- * @valx: data value for burst (or other commands)
- *
- * Define a structure for sensor register initialization values
- */
-struct misensor_fwreg {
-	u32	type;	/* type of value, register or FW burst string */
-	u32	addr;	/* target address */
-	u32	val0;
-	u32	val1;
-	u32	val2;
-	u32	val3;
-	u32	val4;
-	u32	val5;
-	u32	val6;
-	u32	val7;
-};
-
-struct regval_list {
-	u16 reg_num;
-	u8 value;
-};
-
-struct mt9m114_device {
-	struct v4l2_subdev sd;
-	struct media_pad pad;
-	struct v4l2_mbus_framefmt format;
-
-	struct camera_sensor_platform_data *platform_data;
-	struct mutex input_lock;	/* serialize sensor's ioctl */
-	struct v4l2_ctrl_handler ctrl_handler;
-	int real_model_id;
-	int nctx;
-	int power;
-
-	unsigned int bus_width;
-	unsigned int mode;
-	unsigned int field_inv;
-	unsigned int field_sel;
-	unsigned int ycseq;
-	unsigned int conv422;
-	unsigned int bpat;
-	unsigned int hpol;
-	unsigned int vpol;
-	unsigned int edge;
-	unsigned int bls;
-	unsigned int gamma;
-	unsigned int cconv;
-	unsigned int res;
-	unsigned int dwn_sz;
-	unsigned int blc;
-	unsigned int agc;
-	unsigned int awb;
-	unsigned int aec;
-	/* extention SENSOR version 2 */
-	unsigned int cie_profile;
-
-	/* extention SENSOR version 3 */
-	unsigned int flicker_freq;
-
-	/* extension SENSOR version 4 */
-	unsigned int smia_mode;
-	unsigned int mipi_mode;
-
-	/* Add name here to load shared library */
-	unsigned int type;
-
-	/*Number of MIPI lanes*/
-	unsigned int mipi_lanes;
-	/*WA for low light AE*/
-	unsigned int first_exp;
-	unsigned int first_gain;
-	unsigned int first_diggain;
-	char name[32];
-
-	u8 lightfreq;
-	u8 streamon;
-};
-
-struct mt9m114_format_struct {
-	u8 *desc;
-	u32 pixelformat;
-	struct regval_list *regs;
-};
-
-struct mt9m114_res_struct {
-	u8 *desc;
-	int res;
-	int width;
-	int height;
-	int fps;
-	int skip_frames;
-	bool used;
-	struct regval_list *regs;
-	u16 pixels_per_line;
-	u16 lines_per_frame;
-	u8 bin_factor_x;
-	u8 bin_factor_y;
-	u8 bin_mode;
-};
-
-/* 2 bytes used for address: 256 bytes total */
-#define MT9M114_MAX_WRITE_BUF_SIZE	254
-struct mt9m114_write_buffer {
-	u16 addr;
-	u8 data[MT9M114_MAX_WRITE_BUF_SIZE];
-};
-
-struct mt9m114_write_ctrl {
-	int index;
-	struct mt9m114_write_buffer buffer;
-};
-
-/*
- * Modes supported by the mt9m114 driver.
- * Please, keep them in ascending order.
- */
-static struct mt9m114_res_struct mt9m114_res[] = {
-	{
-	.desc	= "720P",
-	.res	= MT9M114_RES_736P,
-	.width	= 1296,
-	.height = 736,
-	.fps	= 30,
-	.used	= false,
-	.regs	= NULL,
-	.skip_frames = 1,
-
-	.pixels_per_line = 0x0640,
-	.lines_per_frame = 0x0307,
-	.bin_factor_x = 1,
-	.bin_factor_y = 1,
-	.bin_mode = 0,
-	},
-	{
-	.desc	= "848P",
-	.res	= MT9M114_RES_864P,
-	.width	= 1296,
-	.height = 864,
-	.fps	= 30,
-	.used	= false,
-	.regs	= NULL,
-	.skip_frames = 1,
-
-	.pixels_per_line = 0x0640,
-	.lines_per_frame = 0x03E8,
-	.bin_factor_x = 1,
-	.bin_factor_y = 1,
-	.bin_mode = 0,
-	},
-	{
-	.desc	= "960P",
-	.res	= MT9M114_RES_960P,
-	.width	= 1296,
-	.height	= 976,
-	.fps	= 30,
-	.used	= false,
-	.regs	= NULL,
-	.skip_frames = 1,
-
-	.pixels_per_line = 0x0644, /* consistent with regs arrays */
-	.lines_per_frame = 0x03E5, /* consistent with regs arrays */
-	.bin_factor_x = 1,
-	.bin_factor_y = 1,
-	.bin_mode = 0,
-	},
-};
-#define N_RES (ARRAY_SIZE(mt9m114_res))
-
-static struct misensor_reg const mt9m114_exitstandby[] = {
-	{MISENSOR_16BIT,  0x098E, 0xDC00},
-	/* exit-standby */
-	{MISENSOR_8BIT,  0xDC00, 0x54},
-	{MISENSOR_16BIT,  0x0080, 0x8002},
-	{MISENSOR_TOK_TERM, 0, 0}
-};
-
-static struct misensor_reg const mt9m114_exp_win[5][5] = {
-	{
-		{MISENSOR_8BIT,  0xA407, 0x64},
-		{MISENSOR_8BIT,  0xA408, 0x64},
-		{MISENSOR_8BIT,  0xA409, 0x64},
-		{MISENSOR_8BIT,  0xA40A, 0x64},
-		{MISENSOR_8BIT,  0xA40B, 0x64},
-	},
-	{
-		{MISENSOR_8BIT,  0xA40C, 0x64},
-		{MISENSOR_8BIT,  0xA40D, 0x64},
-		{MISENSOR_8BIT,  0xA40E, 0x64},
-		{MISENSOR_8BIT,  0xA40F, 0x64},
-		{MISENSOR_8BIT,  0xA410, 0x64},
-	},
-	{
-		{MISENSOR_8BIT,  0xA411, 0x64},
-		{MISENSOR_8BIT,  0xA412, 0x64},
-		{MISENSOR_8BIT,  0xA413, 0x64},
-		{MISENSOR_8BIT,  0xA414, 0x64},
-		{MISENSOR_8BIT,  0xA415, 0x64},
-	},
-	{
-		{MISENSOR_8BIT,  0xA416, 0x64},
-		{MISENSOR_8BIT,  0xA417, 0x64},
-		{MISENSOR_8BIT,  0xA418, 0x64},
-		{MISENSOR_8BIT,  0xA419, 0x64},
-		{MISENSOR_8BIT,  0xA41A, 0x64},
-	},
-	{
-		{MISENSOR_8BIT,  0xA41B, 0x64},
-		{MISENSOR_8BIT,  0xA41C, 0x64},
-		{MISENSOR_8BIT,  0xA41D, 0x64},
-		{MISENSOR_8BIT,  0xA41E, 0x64},
-		{MISENSOR_8BIT,  0xA41F, 0x64},
-	},
-};
-
-static struct misensor_reg const mt9m114_exp_average[] = {
-	{MISENSOR_8BIT,  0xA407, 0x00},
-	{MISENSOR_8BIT,  0xA408, 0x00},
-	{MISENSOR_8BIT,  0xA409, 0x00},
-	{MISENSOR_8BIT,  0xA40A, 0x00},
-	{MISENSOR_8BIT,  0xA40B, 0x00},
-	{MISENSOR_8BIT,  0xA40C, 0x00},
-	{MISENSOR_8BIT,  0xA40D, 0x00},
-	{MISENSOR_8BIT,  0xA40E, 0x00},
-	{MISENSOR_8BIT,  0xA40F, 0x00},
-	{MISENSOR_8BIT,  0xA410, 0x00},
-	{MISENSOR_8BIT,  0xA411, 0x00},
-	{MISENSOR_8BIT,  0xA412, 0x00},
-	{MISENSOR_8BIT,  0xA413, 0x00},
-	{MISENSOR_8BIT,  0xA414, 0x00},
-	{MISENSOR_8BIT,  0xA415, 0x00},
-	{MISENSOR_8BIT,  0xA416, 0x00},
-	{MISENSOR_8BIT,  0xA417, 0x00},
-	{MISENSOR_8BIT,  0xA418, 0x00},
-	{MISENSOR_8BIT,  0xA419, 0x00},
-	{MISENSOR_8BIT,  0xA41A, 0x00},
-	{MISENSOR_8BIT,  0xA41B, 0x00},
-	{MISENSOR_8BIT,  0xA41C, 0x00},
-	{MISENSOR_8BIT,  0xA41D, 0x00},
-	{MISENSOR_8BIT,  0xA41E, 0x00},
-	{MISENSOR_8BIT,  0xA41F, 0x00},
-	{MISENSOR_TOK_TERM, 0, 0}
-};
-
-static struct misensor_reg const mt9m114_exp_center[] = {
-	{MISENSOR_8BIT,  0xA407, 0x19},
-	{MISENSOR_8BIT,  0xA408, 0x19},
-	{MISENSOR_8BIT,  0xA409, 0x19},
-	{MISENSOR_8BIT,  0xA40A, 0x19},
-	{MISENSOR_8BIT,  0xA40B, 0x19},
-	{MISENSOR_8BIT,  0xA40C, 0x19},
-	{MISENSOR_8BIT,  0xA40D, 0x4B},
-	{MISENSOR_8BIT,  0xA40E, 0x4B},
-	{MISENSOR_8BIT,  0xA40F, 0x4B},
-	{MISENSOR_8BIT,  0xA410, 0x19},
-	{MISENSOR_8BIT,  0xA411, 0x19},
-	{MISENSOR_8BIT,  0xA412, 0x4B},
-	{MISENSOR_8BIT,  0xA413, 0x64},
-	{MISENSOR_8BIT,  0xA414, 0x4B},
-	{MISENSOR_8BIT,  0xA415, 0x19},
-	{MISENSOR_8BIT,  0xA416, 0x19},
-	{MISENSOR_8BIT,  0xA417, 0x4B},
-	{MISENSOR_8BIT,  0xA418, 0x4B},
-	{MISENSOR_8BIT,  0xA419, 0x4B},
-	{MISENSOR_8BIT,  0xA41A, 0x19},
-	{MISENSOR_8BIT,  0xA41B, 0x19},
-	{MISENSOR_8BIT,  0xA41C, 0x19},
-	{MISENSOR_8BIT,  0xA41D, 0x19},
-	{MISENSOR_8BIT,  0xA41E, 0x19},
-	{MISENSOR_8BIT,  0xA41F, 0x19},
-	{MISENSOR_TOK_TERM, 0, 0}
-};
-
-static struct misensor_reg const mt9m114_suspend[] = {
-	 {MISENSOR_16BIT,  0x098E, 0xDC00},
-	 {MISENSOR_8BIT,  0xDC00, 0x40},
-	 {MISENSOR_16BIT,  0x0080, 0x8002},
-	 {MISENSOR_TOK_TERM, 0, 0}
-};
-
-static struct misensor_reg const mt9m114_streaming[] = {
-	 {MISENSOR_16BIT,  0x098E, 0xDC00},
-	 {MISENSOR_8BIT,  0xDC00, 0x34},
-	 {MISENSOR_16BIT,  0x0080, 0x8002},
-	 {MISENSOR_TOK_TERM, 0, 0}
-};
-
-static struct misensor_reg const mt9m114_standby_reg[] = {
-	 {MISENSOR_16BIT,  0x098E, 0xDC00},
-	 {MISENSOR_8BIT,  0xDC00, 0x50},
-	 {MISENSOR_16BIT,  0x0080, 0x8002},
-	 {MISENSOR_TOK_TERM, 0, 0}
-};
-
-static struct misensor_reg const mt9m114_wakeup_reg[] = {
-	 {MISENSOR_16BIT,  0x098E, 0xDC00},
-	 {MISENSOR_8BIT,  0xDC00, 0x54},
-	 {MISENSOR_16BIT,  0x0080, 0x8002},
-	 {MISENSOR_TOK_TERM, 0, 0}
-};
-
-static struct misensor_reg const mt9m114_chgstat_reg[] = {
-	{MISENSOR_16BIT,  0x098E, 0xDC00},
-	{MISENSOR_8BIT,  0xDC00, 0x28},
-	{MISENSOR_16BIT,  0x0080, 0x8002},
-	{MISENSOR_TOK_TERM, 0, 0}
-};
-
-/* [1296x976_30fps] - Intel */
-static struct misensor_reg const mt9m114_960P_init[] = {
-	{MISENSOR_16BIT, 0x098E, 0x1000},
-	{MISENSOR_8BIT, 0xC97E, 0x01},	  /* cam_sysctl_pll_enable = 1 */
-	{MISENSOR_16BIT, 0xC980, 0x0128}, /* cam_sysctl_pll_divider_m_n = 276 */
-	{MISENSOR_16BIT, 0xC982, 0x0700}, /* cam_sysctl_pll_divider_p = 1792 */
-	{MISENSOR_16BIT, 0xC800, 0x0000}, /* cam_sensor_cfg_y_addr_start = 0 */
-	{MISENSOR_16BIT, 0xC802, 0x0000}, /* cam_sensor_cfg_x_addr_start = 0 */
-	{MISENSOR_16BIT, 0xC804, 0x03CF}, /* cam_sensor_cfg_y_addr_end = 971 */
-	{MISENSOR_16BIT, 0xC806, 0x050F}, /* cam_sensor_cfg_x_addr_end = 1291 */
-	{MISENSOR_16BIT, 0xC808, 0x02DC}, /* cam_sensor_cfg_pixclk = 48000000 */
-	{MISENSOR_16BIT, 0xC80A, 0x6C00},
-	{MISENSOR_16BIT, 0xC80C, 0x0001}, /* cam_sensor_cfg_row_speed = 1 */
-	/* cam_sensor_cfg_fine_integ_time_min = 219 */
-	{MISENSOR_16BIT, 0xC80E, 0x00DB},
-	/* cam_sensor_cfg_fine_integ_time_max = 1459 */
-	{MISENSOR_16BIT, 0xC810, 0x05B3},
-	/* cam_sensor_cfg_frame_length_lines = 1006 */
-	{MISENSOR_16BIT, 0xC812, 0x03F6},
-	/* cam_sensor_cfg_line_length_pck = 1590 */
-	{MISENSOR_16BIT, 0xC814, 0x063E},
-	/* cam_sensor_cfg_fine_correction = 96 */
-	{MISENSOR_16BIT, 0xC816, 0x0060},
-	/* cam_sensor_cfg_cpipe_last_row = 963 */
-	{MISENSOR_16BIT, 0xC818, 0x03C3},
-	{MISENSOR_16BIT, 0xC826, 0x0020}, /* cam_sensor_cfg_reg_0_data = 32 */
-	{MISENSOR_16BIT, 0xC834, 0x0000}, /* cam_sensor_control_read_mode = 0 */
-	{MISENSOR_16BIT, 0xC854, 0x0000}, /* cam_crop_window_xoffset = 0 */
-	{MISENSOR_16BIT, 0xC856, 0x0000}, /* cam_crop_window_yoffset = 0 */
-	{MISENSOR_16BIT, 0xC858, 0x0508}, /* cam_crop_window_width = 1280 */
-	{MISENSOR_16BIT, 0xC85A, 0x03C8}, /* cam_crop_window_height = 960 */
-	{MISENSOR_8BIT,  0xC85C, 0x03},   /* cam_crop_cropmode = 3 */
-	{MISENSOR_16BIT, 0xC868, 0x0508}, /* cam_output_width = 1280 */
-	{MISENSOR_16BIT, 0xC86A, 0x03C8}, /* cam_output_height = 960 */
-	{MISENSOR_TOK_TERM, 0, 0},
-};
-
-/* [1296x976_30fps_768Mbps] */
-static struct misensor_reg const mt9m114_976P_init[] = {
-	{MISENSOR_16BIT, 0x98E, 0x1000},
-	{MISENSOR_8BIT, 0xC97E, 0x01},	  /* cam_sysctl_pll_enable = 1 */
-	{MISENSOR_16BIT, 0xC980, 0x0128}, /* cam_sysctl_pll_divider_m_n = 276 */
-	{MISENSOR_16BIT, 0xC982, 0x0700}, /* cam_sysctl_pll_divider_p = 1792 */
-	{MISENSOR_16BIT, 0xC800, 0x0000}, /* cam_sensor_cfg_y_addr_start = 0 */
-	{MISENSOR_16BIT, 0xC802, 0x0000}, /* cam_sensor_cfg_x_addr_start = 0 */
-	{MISENSOR_16BIT, 0xC804, 0x03CF}, /* cam_sensor_cfg_y_addr_end = 975 */
-	{MISENSOR_16BIT, 0xC806, 0x050F}, /* cam_sensor_cfg_x_addr_end = 1295 */
-	{MISENSOR_32BIT, 0xC808, 0x2DC6C00},/* cam_sensor_cfg_pixclk = 480000*/
-	{MISENSOR_16BIT, 0xC80C, 0x0001}, /* cam_sensor_cfg_row_speed = 1 */
-	/* cam_sensor_cfg_fine_integ_time_min = 219 */
-	{MISENSOR_16BIT, 0xC80E, 0x00DB},
-	 /* 0x062E //cam_sensor_cfg_fine_integ_time_max = 1459 */
-	{MISENSOR_16BIT, 0xC810, 0x05B3},
-	/* 0x074C //cam_sensor_cfg_frame_length_lines = 1006 */
-	{MISENSOR_16BIT, 0xC812, 0x03E5},
-	/* 0x06B1 /cam_sensor_cfg_line_length_pck = 1590 */
-	{MISENSOR_16BIT, 0xC814, 0x0644},
-	/* cam_sensor_cfg_fine_correction = 96 */
-	{MISENSOR_16BIT, 0xC816, 0x0060},
-	/* cam_sensor_cfg_cpipe_last_row = 963 */
-	{MISENSOR_16BIT, 0xC818, 0x03C3},
-	{MISENSOR_16BIT, 0xC826, 0x0020}, /* cam_sensor_cfg_reg_0_data = 32 */
-	{MISENSOR_16BIT, 0xC834, 0x0000}, /* cam_sensor_control_read_mode = 0 */
-	{MISENSOR_16BIT, 0xC854, 0x0000}, /* cam_crop_window_xoffset = 0 */
-	{MISENSOR_16BIT, 0xC856, 0x0000}, /* cam_crop_window_yoffset = 0 */
-	{MISENSOR_16BIT, 0xC858, 0x0508}, /* cam_crop_window_width = 1288 */
-	{MISENSOR_16BIT, 0xC85A, 0x03C8}, /* cam_crop_window_height = 968 */
-	{MISENSOR_8BIT, 0xC85C, 0x03}, /* cam_crop_cropmode = 3 */
-	{MISENSOR_16BIT, 0xC868, 0x0508}, /* cam_output_width = 1288 */
-	{MISENSOR_16BIT, 0xC86A, 0x03C8}, /* cam_output_height = 968 */
-	{MISENSOR_8BIT, 0xC878, 0x00}, /* 0x0E //cam_aet_aemode = 0 */
-	{MISENSOR_TOK_TERM, 0, 0}
-};
-
-/* [1296x864_30fps] */
-static struct misensor_reg const mt9m114_864P_init[] = {
-	{MISENSOR_16BIT, 0x98E, 0x1000},
-	{MISENSOR_8BIT, 0xC97E, 0x01},	  /* cam_sysctl_pll_enable = 1 */
-	{MISENSOR_16BIT, 0xC980, 0x0128}, /* cam_sysctl_pll_divider_m_n = 276 */
-	{MISENSOR_16BIT, 0xC982, 0x0700}, /* cam_sysctl_pll_divider_p = 1792 */
-	{MISENSOR_16BIT, 0xC800, 0x0038}, /* cam_sensor_cfg_y_addr_start = 56 */
-	{MISENSOR_16BIT, 0xC802, 0x0000}, /* cam_sensor_cfg_x_addr_start = 0 */
-	{MISENSOR_16BIT, 0xC804, 0x0397}, /* cam_sensor_cfg_y_addr_end = 919 */
-	{MISENSOR_16BIT, 0xC806, 0x050F}, /* cam_sensor_cfg_x_addr_end = 1295 */
-	/* cam_sensor_cfg_pixclk = 48000000 */
-	{MISENSOR_32BIT, 0xC808, 0x2DC6C00},
-	{MISENSOR_16BIT, 0xC80C, 0x0001}, /* cam_sensor_cfg_row_speed = 1 */
-	/* cam_sensor_cfg_fine_integ_time_min = 219 */
-	{MISENSOR_16BIT, 0xC80E, 0x00DB},
-	/* cam_sensor_cfg_fine_integ_time_max = 1469 */
-	{MISENSOR_16BIT, 0xC810, 0x05BD},
-	/* cam_sensor_cfg_frame_length_lines = 1000 */
-	{MISENSOR_16BIT, 0xC812, 0x03E8},
-	/* cam_sensor_cfg_line_length_pck = 1600 */
-	{MISENSOR_16BIT, 0xC814, 0x0640},
-	/* cam_sensor_cfg_fine_correction = 96 */
-	{MISENSOR_16BIT, 0xC816, 0x0060},
-	/* cam_sensor_cfg_cpipe_last_row = 859 */
-	{MISENSOR_16BIT, 0xC818, 0x035B},
-	{MISENSOR_16BIT, 0xC826, 0x0020}, /* cam_sensor_cfg_reg_0_data = 32 */
-	{MISENSOR_16BIT, 0xC834, 0x0000}, /* cam_sensor_control_read_mode = 0 */
-	{MISENSOR_16BIT, 0xC854, 0x0000}, /* cam_crop_window_xoffset = 0 */
-	{MISENSOR_16BIT, 0xC856, 0x0000}, /* cam_crop_window_yoffset = 0 */
-	{MISENSOR_16BIT, 0xC858, 0x0508}, /* cam_crop_window_width = 1288 */
-	{MISENSOR_16BIT, 0xC85A, 0x0358}, /* cam_crop_window_height = 856 */
-	{MISENSOR_8BIT, 0xC85C, 0x03}, /* cam_crop_cropmode = 3 */
-	{MISENSOR_16BIT, 0xC868, 0x0508}, /* cam_output_width = 1288 */
-	{MISENSOR_16BIT, 0xC86A, 0x0358}, /* cam_output_height = 856 */
-	{MISENSOR_8BIT, 0xC878, 0x00}, /* 0x0E //cam_aet_aemode = 0 */
-	{MISENSOR_TOK_TERM, 0, 0}
-};
-
-/* [1296x736_30fps] */
-static struct misensor_reg const mt9m114_736P_init[] = {
-	{MISENSOR_16BIT, 0x98E, 0x1000},
-	{MISENSOR_8BIT, 0xC97E, 0x01},	  /* cam_sysctl_pll_enable = 1 */
-	{MISENSOR_16BIT, 0xC980, 0x011F}, /* cam_sysctl_pll_divider_m_n = 287 */
-	{MISENSOR_16BIT, 0xC982, 0x0700}, /* cam_sysctl_pll_divider_p = 1792 */
-	{MISENSOR_16BIT, 0xC800, 0x0078}, /* cam_sensor_cfg_y_addr_start = 120*/
-	{MISENSOR_16BIT, 0xC802, 0x0000}, /* cam_sensor_cfg_x_addr_start = 0 */
-	{MISENSOR_16BIT, 0xC804, 0x0357}, /* cam_sensor_cfg_y_addr_end = 855 */
-	{MISENSOR_16BIT, 0xC806, 0x050F}, /* cam_sensor_cfg_x_addr_end = 1295 */
-	{MISENSOR_32BIT, 0xC808, 0x237A07F}, /* cam_sensor_cfg_pixclk=37199999*/
-	{MISENSOR_16BIT, 0xC80C, 0x0001}, /* cam_sensor_cfg_row_speed = 1 */
-	/* cam_sensor_cfg_fine_integ_time_min = 219 */
-	{MISENSOR_16BIT, 0xC80E, 0x00DB},
-	/* 0x062E //cam_sensor_cfg_fine_integ_time_max = 1469 */
-	{MISENSOR_16BIT, 0xC810, 0x05BD},
-	/* 0x074C //cam_sensor_cfg_frame_length_lines = 775 */
-	{MISENSOR_16BIT, 0xC812, 0x0307},
-	/* 0x06B1 /cam_sensor_cfg_line_length_pck = 1600 */
-	{MISENSOR_16BIT, 0xC814, 0x0640},
-	/* cam_sensor_cfg_fine_correction = 96 */
-	{MISENSOR_16BIT, 0xC816, 0x0060},
-	/* cam_sensor_cfg_cpipe_last_row = 731 */
-	{MISENSOR_16BIT, 0xC818, 0x02DB},
-	{MISENSOR_16BIT, 0xC826, 0x0020}, /* cam_sensor_cfg_reg_0_data = 32 */
-	{MISENSOR_16BIT, 0xC834, 0x0000}, /* cam_sensor_control_read_mode = 0 */
-	{MISENSOR_16BIT, 0xC854, 0x0000}, /* cam_crop_window_xoffset = 0 */
-	{MISENSOR_16BIT, 0xC856, 0x0000}, /* cam_crop_window_yoffset = 0 */
-	{MISENSOR_16BIT, 0xC858, 0x0508}, /* cam_crop_window_width = 1288 */
-	{MISENSOR_16BIT, 0xC85A, 0x02D8}, /* cam_crop_window_height = 728 */
-	{MISENSOR_8BIT, 0xC85C, 0x03}, /* cam_crop_cropmode = 3 */
-	{MISENSOR_16BIT, 0xC868, 0x0508}, /* cam_output_width = 1288 */
-	{MISENSOR_16BIT, 0xC86A, 0x02D8}, /* cam_output_height = 728 */
-	{MISENSOR_8BIT, 0xC878, 0x00}, /* 0x0E //cam_aet_aemode = 0 */
-	{MISENSOR_TOK_TERM, 0, 0}
-};
-
-/* [736x496_30fps_768Mbps] */
-static struct misensor_reg const mt9m114_720_480P_init[] = {
-	{MISENSOR_16BIT, 0x98E, 0x1000},
-	{MISENSOR_8BIT, 0xC97E, 0x01},	  /* cam_sysctl_pll_enable = 1 */
-	{MISENSOR_16BIT, 0xC980, 0x0128}, /* cam_sysctl_pll_divider_m_n = 276 */
-	{MISENSOR_16BIT, 0xC982, 0x0700}, /* cam_sysctl_pll_divider_p = 1792 */
-	{MISENSOR_16BIT, 0xC800, 0x00F0}, /* cam_sensor_cfg_y_addr_start = 240*/
-	{MISENSOR_16BIT, 0xC802, 0x0118}, /* cam_sensor_cfg_x_addr_start = 280*/
-	{MISENSOR_16BIT, 0xC804, 0x02DF}, /* cam_sensor_cfg_y_addr_end = 735 */
-	{MISENSOR_16BIT, 0xC806, 0x03F7}, /* cam_sensor_cfg_x_addr_end = 1015 */
-	/* cam_sensor_cfg_pixclk = 48000000 */
-	{MISENSOR_32BIT, 0xC808, 0x2DC6C00},
-	{MISENSOR_16BIT, 0xC80C, 0x0001}, /* cam_sensor_cfg_row_speed = 1 */
-	/* cam_sensor_cfg_fine_integ_time_min = 219 */
-	{MISENSOR_16BIT, 0xC80E, 0x00DB},
-	/* 0x062E //cam_sensor_cfg_fine_integ_time_max = 1459 */
-	{MISENSOR_16BIT, 0xC810, 0x05B3},
-	/* 0x074C //cam_sensor_cfg_frame_length_lines = 997 */
-	{MISENSOR_16BIT, 0xC812, 0x03E5},
-	/* 0x06B1 /cam_sensor_cfg_line_length_pck = 1604 */
-	{MISENSOR_16BIT, 0xC814, 0x0644},
-	/* cam_sensor_cfg_fine_correction = 96 */
-	{MISENSOR_16BIT, 0xC816, 0x0060},
-	{MISENSOR_16BIT, 0xC818, 0x03C3}, /* cam_sensor_cfg_cpipe_last_row=963*/
-	{MISENSOR_16BIT, 0xC826, 0x0020}, /* cam_sensor_cfg_reg_0_data = 32 */
-	{MISENSOR_16BIT, 0xC834, 0x0000}, /* cam_sensor_control_read_mode = 0*/
-	{MISENSOR_16BIT, 0xC854, 0x0000}, /* cam_crop_window_xoffset = 0 */
-	{MISENSOR_16BIT, 0xC856, 0x0000}, /* cam_crop_window_yoffset = 0 */
-	{MISENSOR_16BIT, 0xC858, 0x02D8}, /* cam_crop_window_width = 728 */
-	{MISENSOR_16BIT, 0xC85A, 0x01E8}, /* cam_crop_window_height = 488 */
-	{MISENSOR_8BIT, 0xC85C, 0x03}, /* cam_crop_cropmode = 3 */
-	{MISENSOR_16BIT, 0xC868, 0x02D8}, /* cam_output_width = 728 */
-	{MISENSOR_16BIT, 0xC86A, 0x01E8}, /* cam_output_height = 488 */
-	{MISENSOR_8BIT, 0xC878, 0x00}, /* 0x0E //cam_aet_aemode = 0 */
-	{MISENSOR_TOK_TERM, 0, 0}
-};
-
-static struct misensor_reg const mt9m114_common[] = {
-	/* reset */
-	{MISENSOR_16BIT,  0x301A, 0x0234},
-	/* LOAD = Step2-PLL_Timing      //PLL and Timing */
-	{MISENSOR_16BIT, 0x098E, 0x1000}, /* LOGICAL_ADDRESS_ACCESS */
-	{MISENSOR_8BIT, 0xC97E, 0x01},    /* cam_sysctl_pll_enable = 1 */
-	{MISENSOR_16BIT, 0xC980, 0x0128}, /* cam_sysctl_pll_divider_m_n = 276 */
-	{MISENSOR_16BIT, 0xC982, 0x0700}, /* cam_sysctl_pll_divider_p = 1792 */
-	{MISENSOR_16BIT, 0xC800, 0x0000}, /* cam_sensor_cfg_y_addr_start = 216*/
-	{MISENSOR_16BIT, 0xC802, 0x0000}, /* cam_sensor_cfg_x_addr_start = 168*/
-	{MISENSOR_16BIT, 0xC804, 0x03CD}, /* cam_sensor_cfg_y_addr_end = 761 */
-	{MISENSOR_16BIT, 0xC806, 0x050D}, /* cam_sensor_cfg_x_addr_end = 1127 */
-	{MISENSOR_16BIT, 0xC808, 0x02DC}, /* cam_sensor_cfg_pixclk = 24000000 */
-	{MISENSOR_16BIT, 0xC80A, 0x6C00},
-	{MISENSOR_16BIT, 0xC80C, 0x0001}, /* cam_sensor_cfg_row_speed = 1 */
-	/* cam_sensor_cfg_fine_integ_time_min = 219 */
-	{MISENSOR_16BIT, 0xC80E, 0x01C3},
-	/* cam_sensor_cfg_fine_integ_time_max = 1149 */
-	{MISENSOR_16BIT, 0xC810, 0x03F7},
-	/* cam_sensor_cfg_frame_length_lines = 625 */
-	{MISENSOR_16BIT, 0xC812, 0x0500},
-	/* cam_sensor_cfg_line_length_pck = 1280 */
-	{MISENSOR_16BIT, 0xC814, 0x04E2},
-	/* cam_sensor_cfg_fine_correction = 96 */
-	{MISENSOR_16BIT, 0xC816, 0x00E0},
-	/* cam_sensor_cfg_cpipe_last_row = 541 */
-	{MISENSOR_16BIT, 0xC818, 0x01E3},
-	{MISENSOR_16BIT, 0xC826, 0x0020}, /* cam_sensor_cfg_reg_0_data = 32 */
-	{MISENSOR_16BIT, 0xC834, 0x0330}, /* cam_sensor_control_read_mode = 0 */
-	{MISENSOR_16BIT, 0xC854, 0x0000}, /* cam_crop_window_xoffset = 0 */
-	{MISENSOR_16BIT, 0xC856, 0x0000}, /* cam_crop_window_yoffset = 0 */
-	{MISENSOR_16BIT, 0xC858, 0x0280}, /* cam_crop_window_width = 952 */
-	{MISENSOR_16BIT, 0xC85A, 0x01E0}, /* cam_crop_window_height = 538 */
-	{MISENSOR_8BIT, 0xC85C, 0x03},    /* cam_crop_cropmode = 3 */
-	{MISENSOR_16BIT, 0xC868, 0x0280}, /* cam_output_width = 952 */
-	{MISENSOR_16BIT, 0xC86A, 0x01E0}, /* cam_output_height = 538 */
-	/* LOAD = Step3-Recommended
-	 * Patch,Errata and Sensor optimization Setting */
-	{MISENSOR_16BIT, 0x316A, 0x8270}, /* DAC_TXLO_ROW */
-	{MISENSOR_16BIT, 0x316C, 0x8270}, /* DAC_TXLO */
-	{MISENSOR_16BIT, 0x3ED0, 0x2305}, /* DAC_LD_4_5 */
-	{MISENSOR_16BIT, 0x3ED2, 0x77CF}, /* DAC_LD_6_7 */
-	{MISENSOR_16BIT, 0x316E, 0x8202}, /* DAC_ECL */
-	{MISENSOR_16BIT, 0x3180, 0x87FF}, /* DELTA_DK_CONTROL */
-	{MISENSOR_16BIT, 0x30D4, 0x6080}, /* COLUMN_CORRECTION */
-	{MISENSOR_16BIT, 0xA802, 0x0008}, /* AE_TRACK_MODE */
-	{MISENSOR_16BIT, 0x3E14, 0xFF39}, /* SAMP_COL_PUP2 */
-	{MISENSOR_16BIT, 0x31E0, 0x0003}, /* PIX_DEF_ID */
-	/* LOAD = Step8-Features	//Ports, special features, etc. */
-	{MISENSOR_16BIT, 0x098E, 0x0000}, /* LOGICAL_ADDRESS_ACCESS */
-	{MISENSOR_16BIT, 0x001E, 0x0777}, /* PAD_SLEW */
-	{MISENSOR_16BIT, 0x098E, 0x0000}, /* LOGICAL_ADDRESS_ACCESS */
-	{MISENSOR_16BIT, 0xC984, 0x8001}, /* CAM_PORT_OUTPUT_CONTROL */
-	{MISENSOR_16BIT, 0xC988, 0x0F00}, /* CAM_PORT_MIPI_TIMING_T_HS_ZERO */
-	/* CAM_PORT_MIPI_TIMING_T_HS_EXIT_HS_TRAIL */
-	{MISENSOR_16BIT, 0xC98A, 0x0B07},
-	/* CAM_PORT_MIPI_TIMING_T_CLK_POST_CLK_PRE */
-	{MISENSOR_16BIT, 0xC98C, 0x0D01},
-	/* CAM_PORT_MIPI_TIMING_T_CLK_TRAIL_CLK_ZERO */
-	{MISENSOR_16BIT, 0xC98E, 0x071D},
-	{MISENSOR_16BIT, 0xC990, 0x0006}, /* CAM_PORT_MIPI_TIMING_T_LPX */
-	{MISENSOR_16BIT, 0xC992, 0x0A0C}, /* CAM_PORT_MIPI_TIMING_INIT_TIMING */
-	{MISENSOR_16BIT, 0x3C5A, 0x0009}, /* MIPI_DELAY_TRIM */
-	{MISENSOR_16BIT, 0xC86C, 0x0210}, /* CAM_OUTPUT_FORMAT */
-	{MISENSOR_16BIT, 0xA804, 0x0000}, /* AE_TRACK_ALGO */
-	/* default exposure */
-	{MISENSOR_16BIT, 0x3012, 0x0110}, /* COMMAND_REGISTER */
-	{MISENSOR_TOK_TERM, 0, 0},
-
-};
-
-static struct misensor_reg const mt9m114_antiflicker_50hz[] = {
-	 {MISENSOR_16BIT,  0x098E, 0xC88B},
-	 {MISENSOR_8BIT,  0xC88B, 0x32},
-	 {MISENSOR_8BIT,  0xDC00, 0x28},
-	 {MISENSOR_16BIT,  0x0080, 0x8002},
-	 {MISENSOR_TOK_TERM, 0, 0}
-};
-
-static struct misensor_reg const mt9m114_antiflicker_60hz[] = {
-	 {MISENSOR_16BIT,  0x098E, 0xC88B},
-	 {MISENSOR_8BIT,  0xC88B, 0x3C},
-	 {MISENSOR_8BIT,  0xDC00, 0x28},
-	 {MISENSOR_16BIT,  0x0080, 0x8002},
-	 {MISENSOR_TOK_TERM, 0, 0}
-};
-
-static struct misensor_reg const mt9m114_iq[] = {
-	/* [Step3-Recommended] [Sensor optimization] */
-	{MISENSOR_16BIT,	0x316A, 0x8270},
-	{MISENSOR_16BIT,	0x316C, 0x8270},
-	{MISENSOR_16BIT,	0x3ED0, 0x2305},
-	{MISENSOR_16BIT,	0x3ED2, 0x77CF},
-	{MISENSOR_16BIT,	0x316E, 0x8202},
-	{MISENSOR_16BIT,	0x3180, 0x87FF},
-	{MISENSOR_16BIT,	0x30D4, 0x6080},
-	{MISENSOR_16BIT,	0xA802, 0x0008},
-
-	/* This register is from vender to avoid low light color noise */
-	{MISENSOR_16BIT,	0x31E0, 0x0001},
-
-	/* LOAD=Errata item 1 */
-	{MISENSOR_16BIT,	0x3E14, 0xFF39},
-
-	/* LOAD=Errata item 2 */
-	{MISENSOR_16BIT,	0x301A, 0x8234},
-
-	/*
-	 * LOAD=Errata item 3
-	 * LOAD=Patch 0202;
-	 * Feature Recommended; Black level correction fix
-	 */
-	{MISENSOR_16BIT,	0x0982, 0x0001},
-	{MISENSOR_16BIT,	0x098A, 0x5000},
-	{MISENSOR_16BIT,	0xD000, 0x70CF},
-	{MISENSOR_16BIT,	0xD002, 0xFFFF},
-	{MISENSOR_16BIT,	0xD004, 0xC5D4},
-	{MISENSOR_16BIT,	0xD006, 0x903A},
-	{MISENSOR_16BIT,	0xD008, 0x2144},
-	{MISENSOR_16BIT,	0xD00A, 0x0C00},
-	{MISENSOR_16BIT,	0xD00C, 0x2186},
-	{MISENSOR_16BIT,	0xD00E, 0x0FF3},
-	{MISENSOR_16BIT,	0xD010, 0xB844},
-	{MISENSOR_16BIT,	0xD012, 0xB948},
-	{MISENSOR_16BIT,	0xD014, 0xE082},
-	{MISENSOR_16BIT,	0xD016, 0x20CC},
-	{MISENSOR_16BIT,	0xD018, 0x80E2},
-	{MISENSOR_16BIT,	0xD01A, 0x21CC},
-	{MISENSOR_16BIT,	0xD01C, 0x80A2},
-	{MISENSOR_16BIT,	0xD01E, 0x21CC},
-	{MISENSOR_16BIT,	0xD020, 0x80E2},
-	{MISENSOR_16BIT,	0xD022, 0xF404},
-	{MISENSOR_16BIT,	0xD024, 0xD801},
-	{MISENSOR_16BIT,	0xD026, 0xF003},
-	{MISENSOR_16BIT,	0xD028, 0xD800},
-	{MISENSOR_16BIT,	0xD02A, 0x7EE0},
-	{MISENSOR_16BIT,	0xD02C, 0xC0F1},
-	{MISENSOR_16BIT,	0xD02E, 0x08BA},
-
-	{MISENSOR_16BIT,	0xD030, 0x0600},
-	{MISENSOR_16BIT,	0xD032, 0xC1A1},
-	{MISENSOR_16BIT,	0xD034, 0x76CF},
-	{MISENSOR_16BIT,	0xD036, 0xFFFF},
-	{MISENSOR_16BIT,	0xD038, 0xC130},
-	{MISENSOR_16BIT,	0xD03A, 0x6E04},
-	{MISENSOR_16BIT,	0xD03C, 0xC040},
-	{MISENSOR_16BIT,	0xD03E, 0x71CF},
-	{MISENSOR_16BIT,	0xD040, 0xFFFF},
-	{MISENSOR_16BIT,	0xD042, 0xC790},
-	{MISENSOR_16BIT,	0xD044, 0x8103},
-	{MISENSOR_16BIT,	0xD046, 0x77CF},
-	{MISENSOR_16BIT,	0xD048, 0xFFFF},
-	{MISENSOR_16BIT,	0xD04A, 0xC7C0},
-	{MISENSOR_16BIT,	0xD04C, 0xE001},
-	{MISENSOR_16BIT,	0xD04E, 0xA103},
-	{MISENSOR_16BIT,	0xD050, 0xD800},
-	{MISENSOR_16BIT,	0xD052, 0x0C6A},
-	{MISENSOR_16BIT,	0xD054, 0x04E0},
-	{MISENSOR_16BIT,	0xD056, 0xB89E},
-	{MISENSOR_16BIT,	0xD058, 0x7508},
-	{MISENSOR_16BIT,	0xD05A, 0x8E1C},
-	{MISENSOR_16BIT,	0xD05C, 0x0809},
-	{MISENSOR_16BIT,	0xD05E, 0x0191},
-
-	{MISENSOR_16BIT,	0xD060, 0xD801},
-	{MISENSOR_16BIT,	0xD062, 0xAE1D},
-	{MISENSOR_16BIT,	0xD064, 0xE580},
-	{MISENSOR_16BIT,	0xD066, 0x20CA},
-	{MISENSOR_16BIT,	0xD068, 0x0022},
-	{MISENSOR_16BIT,	0xD06A, 0x20CF},
-	{MISENSOR_16BIT,	0xD06C, 0x0522},
-	{MISENSOR_16BIT,	0xD06E, 0x0C5C},
-	{MISENSOR_16BIT,	0xD070, 0x04E2},
-	{MISENSOR_16BIT,	0xD072, 0x21CA},
-	{MISENSOR_16BIT,	0xD074, 0x0062},
-	{MISENSOR_16BIT,	0xD076, 0xE580},
-	{MISENSOR_16BIT,	0xD078, 0xD901},
-	{MISENSOR_16BIT,	0xD07A, 0x79C0},
-	{MISENSOR_16BIT,	0xD07C, 0xD800},
-	{MISENSOR_16BIT,	0xD07E, 0x0BE6},
-	{MISENSOR_16BIT,	0xD080, 0x04E0},
-	{MISENSOR_16BIT,	0xD082, 0xB89E},
-	{MISENSOR_16BIT,	0xD084, 0x70CF},
-	{MISENSOR_16BIT,	0xD086, 0xFFFF},
-	{MISENSOR_16BIT,	0xD088, 0xC8D4},
-	{MISENSOR_16BIT,	0xD08A, 0x9002},
-	{MISENSOR_16BIT,	0xD08C, 0x0857},
-	{MISENSOR_16BIT,	0xD08E, 0x025E},
-
-	{MISENSOR_16BIT,	0xD090, 0xFFDC},
-	{MISENSOR_16BIT,	0xD092, 0xE080},
-	{MISENSOR_16BIT,	0xD094, 0x25CC},
-	{MISENSOR_16BIT,	0xD096, 0x9022},
-	{MISENSOR_16BIT,	0xD098, 0xF225},
-	{MISENSOR_16BIT,	0xD09A, 0x1700},
-	{MISENSOR_16BIT,	0xD09C, 0x108A},
-	{MISENSOR_16BIT,	0xD09E, 0x73CF},
-	{MISENSOR_16BIT,	0xD0A0, 0xFF00},
-	{MISENSOR_16BIT,	0xD0A2, 0x3174},
-	{MISENSOR_16BIT,	0xD0A4, 0x9307},
-	{MISENSOR_16BIT,	0xD0A6, 0x2A04},
-	{MISENSOR_16BIT,	0xD0A8, 0x103E},
-	{MISENSOR_16BIT,	0xD0AA, 0x9328},
-	{MISENSOR_16BIT,	0xD0AC, 0x2942},
-	{MISENSOR_16BIT,	0xD0AE, 0x7140},
-	{MISENSOR_16BIT,	0xD0B0, 0x2A04},
-	{MISENSOR_16BIT,	0xD0B2, 0x107E},
-	{MISENSOR_16BIT,	0xD0B4, 0x9349},
-	{MISENSOR_16BIT,	0xD0B6, 0x2942},
-	{MISENSOR_16BIT,	0xD0B8, 0x7141},
-	{MISENSOR_16BIT,	0xD0BA, 0x2A04},
-	{MISENSOR_16BIT,	0xD0BC, 0x10BE},
-	{MISENSOR_16BIT,	0xD0BE, 0x934A},
-
-	{MISENSOR_16BIT,	0xD0C0, 0x2942},
-	{MISENSOR_16BIT,	0xD0C2, 0x714B},
-	{MISENSOR_16BIT,	0xD0C4, 0x2A04},
-	{MISENSOR_16BIT,	0xD0C6, 0x10BE},
-	{MISENSOR_16BIT,	0xD0C8, 0x130C},
-	{MISENSOR_16BIT,	0xD0CA, 0x010A},
-	{MISENSOR_16BIT,	0xD0CC, 0x2942},
-	{MISENSOR_16BIT,	0xD0CE, 0x7142},
-	{MISENSOR_16BIT,	0xD0D0, 0x2250},
-	{MISENSOR_16BIT,	0xD0D2, 0x13CA},
-	{MISENSOR_16BIT,	0xD0D4, 0x1B0C},
-	{MISENSOR_16BIT,	0xD0D6, 0x0284},
-	{MISENSOR_16BIT,	0xD0D8, 0xB307},
-	{MISENSOR_16BIT,	0xD0DA, 0xB328},
-	{MISENSOR_16BIT,	0xD0DC, 0x1B12},
-	{MISENSOR_16BIT,	0xD0DE, 0x02C4},
-	{MISENSOR_16BIT,	0xD0E0, 0xB34A},
-	{MISENSOR_16BIT,	0xD0E2, 0xED88},
-	{MISENSOR_16BIT,	0xD0E4, 0x71CF},
-	{MISENSOR_16BIT,	0xD0E6, 0xFF00},
-	{MISENSOR_16BIT,	0xD0E8, 0x3174},
-	{MISENSOR_16BIT,	0xD0EA, 0x9106},
-	{MISENSOR_16BIT,	0xD0EC, 0xB88F},
-	{MISENSOR_16BIT,	0xD0EE, 0xB106},
-
-	{MISENSOR_16BIT,	0xD0F0, 0x210A},
-	{MISENSOR_16BIT,	0xD0F2, 0x8340},
-	{MISENSOR_16BIT,	0xD0F4, 0xC000},
-	{MISENSOR_16BIT,	0xD0F6, 0x21CA},
-	{MISENSOR_16BIT,	0xD0F8, 0x0062},
-	{MISENSOR_16BIT,	0xD0FA, 0x20F0},
-	{MISENSOR_16BIT,	0xD0FC, 0x0040},
-	{MISENSOR_16BIT,	0xD0FE, 0x0B02},
-	{MISENSOR_16BIT,	0xD100, 0x0320},
-	{MISENSOR_16BIT,	0xD102, 0xD901},
-	{MISENSOR_16BIT,	0xD104, 0x07F1},
-	{MISENSOR_16BIT,	0xD106, 0x05E0},
-	{MISENSOR_16BIT,	0xD108, 0xC0A1},
-	{MISENSOR_16BIT,	0xD10A, 0x78E0},
-	{MISENSOR_16BIT,	0xD10C, 0xC0F1},
-	{MISENSOR_16BIT,	0xD10E, 0x71CF},
-	{MISENSOR_16BIT,	0xD110, 0xFFFF},
-	{MISENSOR_16BIT,	0xD112, 0xC7C0},
-	{MISENSOR_16BIT,	0xD114, 0xD840},
-	{MISENSOR_16BIT,	0xD116, 0xA900},
-	{MISENSOR_16BIT,	0xD118, 0x71CF},
-	{MISENSOR_16BIT,	0xD11A, 0xFFFF},
-	{MISENSOR_16BIT,	0xD11C, 0xD02C},
-	{MISENSOR_16BIT,	0xD11E, 0xD81E},
-
-	{MISENSOR_16BIT,	0xD120, 0x0A5A},
-	{MISENSOR_16BIT,	0xD122, 0x04E0},
-	{MISENSOR_16BIT,	0xD124, 0xDA00},
-	{MISENSOR_16BIT,	0xD126, 0xD800},
-	{MISENSOR_16BIT,	0xD128, 0xC0D1},
-	{MISENSOR_16BIT,	0xD12A, 0x7EE0},
-
-	{MISENSOR_16BIT,	0x098E, 0x0000},
-	{MISENSOR_16BIT,	0xE000, 0x010C},
-	{MISENSOR_16BIT,	0xE002, 0x0202},
-	{MISENSOR_16BIT,	0xE004, 0x4103},
-	{MISENSOR_16BIT,	0xE006, 0x0202},
-	{MISENSOR_16BIT,	0x0080, 0xFFF0},
-	{MISENSOR_16BIT,	0x0080, 0xFFF1},
-
-	/* LOAD=Patch 0302; Feature Recommended; Adaptive Sensitivity */
-	{MISENSOR_16BIT,	0x0982, 0x0001},
-	{MISENSOR_16BIT,	0x098A, 0x512C},
-	{MISENSOR_16BIT,	0xD12C, 0x70CF},
-	{MISENSOR_16BIT,	0xD12E, 0xFFFF},
-	{MISENSOR_16BIT,	0xD130, 0xC5D4},
-	{MISENSOR_16BIT,	0xD132, 0x903A},
-	{MISENSOR_16BIT,	0xD134, 0x2144},
-	{MISENSOR_16BIT,	0xD136, 0x0C00},
-	{MISENSOR_16BIT,	0xD138, 0x2186},
-	{MISENSOR_16BIT,	0xD13A, 0x0FF3},
-	{MISENSOR_16BIT,	0xD13C, 0xB844},
-	{MISENSOR_16BIT,	0xD13E, 0x262F},
-	{MISENSOR_16BIT,	0xD140, 0xF008},
-	{MISENSOR_16BIT,	0xD142, 0xB948},
-	{MISENSOR_16BIT,	0xD144, 0x21CC},
-	{MISENSOR_16BIT,	0xD146, 0x8021},
-	{MISENSOR_16BIT,	0xD148, 0xD801},
-	{MISENSOR_16BIT,	0xD14A, 0xF203},
-	{MISENSOR_16BIT,	0xD14C, 0xD800},
-	{MISENSOR_16BIT,	0xD14E, 0x7EE0},
-	{MISENSOR_16BIT,	0xD150, 0xC0F1},
-	{MISENSOR_16BIT,	0xD152, 0x71CF},
-	{MISENSOR_16BIT,	0xD154, 0xFFFF},
-	{MISENSOR_16BIT,	0xD156, 0xC610},
-	{MISENSOR_16BIT,	0xD158, 0x910E},
-	{MISENSOR_16BIT,	0xD15A, 0x208C},
-	{MISENSOR_16BIT,	0xD15C, 0x8014},
-	{MISENSOR_16BIT,	0xD15E, 0xF418},
-	{MISENSOR_16BIT,	0xD160, 0x910F},
-	{MISENSOR_16BIT,	0xD162, 0x208C},
-	{MISENSOR_16BIT,	0xD164, 0x800F},
-	{MISENSOR_16BIT,	0xD166, 0xF414},
-	{MISENSOR_16BIT,	0xD168, 0x9116},
-	{MISENSOR_16BIT,	0xD16A, 0x208C},
-	{MISENSOR_16BIT,	0xD16C, 0x800A},
-	{MISENSOR_16BIT,	0xD16E, 0xF410},
-	{MISENSOR_16BIT,	0xD170, 0x9117},
-	{MISENSOR_16BIT,	0xD172, 0x208C},
-	{MISENSOR_16BIT,	0xD174, 0x8807},
-	{MISENSOR_16BIT,	0xD176, 0xF40C},
-	{MISENSOR_16BIT,	0xD178, 0x9118},
-	{MISENSOR_16BIT,	0xD17A, 0x2086},
-	{MISENSOR_16BIT,	0xD17C, 0x0FF3},
-	{MISENSOR_16BIT,	0xD17E, 0xB848},
-	{MISENSOR_16BIT,	0xD180, 0x080D},
-	{MISENSOR_16BIT,	0xD182, 0x0090},
-	{MISENSOR_16BIT,	0xD184, 0xFFEA},
-	{MISENSOR_16BIT,	0xD186, 0xE081},
-	{MISENSOR_16BIT,	0xD188, 0xD801},
-	{MISENSOR_16BIT,	0xD18A, 0xF203},
-	{MISENSOR_16BIT,	0xD18C, 0xD800},
-	{MISENSOR_16BIT,	0xD18E, 0xC0D1},
-	{MISENSOR_16BIT,	0xD190, 0x7EE0},
-	{MISENSOR_16BIT,	0xD192, 0x78E0},
-	{MISENSOR_16BIT,	0xD194, 0xC0F1},
-	{MISENSOR_16BIT,	0xD196, 0x71CF},
-	{MISENSOR_16BIT,	0xD198, 0xFFFF},
-	{MISENSOR_16BIT,	0xD19A, 0xC610},
-	{MISENSOR_16BIT,	0xD19C, 0x910E},
-	{MISENSOR_16BIT,	0xD19E, 0x208C},
-	{MISENSOR_16BIT,	0xD1A0, 0x800A},
-	{MISENSOR_16BIT,	0xD1A2, 0xF418},
-	{MISENSOR_16BIT,	0xD1A4, 0x910F},
-	{MISENSOR_16BIT,	0xD1A6, 0x208C},
-	{MISENSOR_16BIT,	0xD1A8, 0x8807},
-	{MISENSOR_16BIT,	0xD1AA, 0xF414},
-	{MISENSOR_16BIT,	0xD1AC, 0x9116},
-	{MISENSOR_16BIT,	0xD1AE, 0x208C},
-	{MISENSOR_16BIT,	0xD1B0, 0x800A},
-	{MISENSOR_16BIT,	0xD1B2, 0xF410},
-	{MISENSOR_16BIT,	0xD1B4, 0x9117},
-	{MISENSOR_16BIT,	0xD1B6, 0x208C},
-	{MISENSOR_16BIT,	0xD1B8, 0x8807},
-	{MISENSOR_16BIT,	0xD1BA, 0xF40C},
-	{MISENSOR_16BIT,	0xD1BC, 0x9118},
-	{MISENSOR_16BIT,	0xD1BE, 0x2086},
-	{MISENSOR_16BIT,	0xD1C0, 0x0FF3},
-	{MISENSOR_16BIT,	0xD1C2, 0xB848},
-	{MISENSOR_16BIT,	0xD1C4, 0x080D},
-	{MISENSOR_16BIT,	0xD1C6, 0x0090},
-	{MISENSOR_16BIT,	0xD1C8, 0xFFD9},
-	{MISENSOR_16BIT,	0xD1CA, 0xE080},
-	{MISENSOR_16BIT,	0xD1CC, 0xD801},
-	{MISENSOR_16BIT,	0xD1CE, 0xF203},
-	{MISENSOR_16BIT,	0xD1D0, 0xD800},
-	{MISENSOR_16BIT,	0xD1D2, 0xF1DF},
-	{MISENSOR_16BIT,	0xD1D4, 0x9040},
-	{MISENSOR_16BIT,	0xD1D6, 0x71CF},
-	{MISENSOR_16BIT,	0xD1D8, 0xFFFF},
-	{MISENSOR_16BIT,	0xD1DA, 0xC5D4},
-	{MISENSOR_16BIT,	0xD1DC, 0xB15A},
-	{MISENSOR_16BIT,	0xD1DE, 0x9041},
-	{MISENSOR_16BIT,	0xD1E0, 0x73CF},
-	{MISENSOR_16BIT,	0xD1E2, 0xFFFF},
-	{MISENSOR_16BIT,	0xD1E4, 0xC7D0},
-	{MISENSOR_16BIT,	0xD1E6, 0xB140},
-	{MISENSOR_16BIT,	0xD1E8, 0x9042},
-	{MISENSOR_16BIT,	0xD1EA, 0xB141},
-	{MISENSOR_16BIT,	0xD1EC, 0x9043},
-	{MISENSOR_16BIT,	0xD1EE, 0xB142},
-	{MISENSOR_16BIT,	0xD1F0, 0x9044},
-	{MISENSOR_16BIT,	0xD1F2, 0xB143},
-	{MISENSOR_16BIT,	0xD1F4, 0x9045},
-	{MISENSOR_16BIT,	0xD1F6, 0xB147},
-	{MISENSOR_16BIT,	0xD1F8, 0x9046},
-	{MISENSOR_16BIT,	0xD1FA, 0xB148},
-	{MISENSOR_16BIT,	0xD1FC, 0x9047},
-	{MISENSOR_16BIT,	0xD1FE, 0xB14B},
-	{MISENSOR_16BIT,	0xD200, 0x9048},
-	{MISENSOR_16BIT,	0xD202, 0xB14C},
-	{MISENSOR_16BIT,	0xD204, 0x9049},
-	{MISENSOR_16BIT,	0xD206, 0x1958},
-	{MISENSOR_16BIT,	0xD208, 0x0084},
-	{MISENSOR_16BIT,	0xD20A, 0x904A},
-	{MISENSOR_16BIT,	0xD20C, 0x195A},
-	{MISENSOR_16BIT,	0xD20E, 0x0084},
-	{MISENSOR_16BIT,	0xD210, 0x8856},
-	{MISENSOR_16BIT,	0xD212, 0x1B36},
-	{MISENSOR_16BIT,	0xD214, 0x8082},
-	{MISENSOR_16BIT,	0xD216, 0x8857},
-	{MISENSOR_16BIT,	0xD218, 0x1B37},
-	{MISENSOR_16BIT,	0xD21A, 0x8082},
-	{MISENSOR_16BIT,	0xD21C, 0x904C},
-	{MISENSOR_16BIT,	0xD21E, 0x19A7},
-	{MISENSOR_16BIT,	0xD220, 0x009C},
-	{MISENSOR_16BIT,	0xD222, 0x881A},
-	{MISENSOR_16BIT,	0xD224, 0x7FE0},
-	{MISENSOR_16BIT,	0xD226, 0x1B54},
-	{MISENSOR_16BIT,	0xD228, 0x8002},
-	{MISENSOR_16BIT,	0xD22A, 0x78E0},
-	{MISENSOR_16BIT,	0xD22C, 0x71CF},
-	{MISENSOR_16BIT,	0xD22E, 0xFFFF},
-	{MISENSOR_16BIT,	0xD230, 0xC350},
-	{MISENSOR_16BIT,	0xD232, 0xD828},
-	{MISENSOR_16BIT,	0xD234, 0xA90B},
-	{MISENSOR_16BIT,	0xD236, 0x8100},
-	{MISENSOR_16BIT,	0xD238, 0x01C5},
-	{MISENSOR_16BIT,	0xD23A, 0x0320},
-	{MISENSOR_16BIT,	0xD23C, 0xD900},
-	{MISENSOR_16BIT,	0xD23E, 0x78E0},
-	{MISENSOR_16BIT,	0xD240, 0x220A},
-	{MISENSOR_16BIT,	0xD242, 0x1F80},
-	{MISENSOR_16BIT,	0xD244, 0xFFFF},
-	{MISENSOR_16BIT,	0xD246, 0xD4E0},
-	{MISENSOR_16BIT,	0xD248, 0xC0F1},
-	{MISENSOR_16BIT,	0xD24A, 0x0811},
-	{MISENSOR_16BIT,	0xD24C, 0x0051},
-	{MISENSOR_16BIT,	0xD24E, 0x2240},
-	{MISENSOR_16BIT,	0xD250, 0x1200},
-	{MISENSOR_16BIT,	0xD252, 0xFFE1},
-	{MISENSOR_16BIT,	0xD254, 0xD801},
-	{MISENSOR_16BIT,	0xD256, 0xF006},
-	{MISENSOR_16BIT,	0xD258, 0x2240},
-	{MISENSOR_16BIT,	0xD25A, 0x1900},
-	{MISENSOR_16BIT,	0xD25C, 0xFFDE},
-	{MISENSOR_16BIT,	0xD25E, 0xD802},
-	{MISENSOR_16BIT,	0xD260, 0x1A05},
-	{MISENSOR_16BIT,	0xD262, 0x1002},
-	{MISENSOR_16BIT,	0xD264, 0xFFF2},
-	{MISENSOR_16BIT,	0xD266, 0xF195},
-	{MISENSOR_16BIT,	0xD268, 0xC0F1},
-	{MISENSOR_16BIT,	0xD26A, 0x0E7E},
-	{MISENSOR_16BIT,	0xD26C, 0x05C0},
-	{MISENSOR_16BIT,	0xD26E, 0x75CF},
-	{MISENSOR_16BIT,	0xD270, 0xFFFF},
-	{MISENSOR_16BIT,	0xD272, 0xC84C},
-	{MISENSOR_16BIT,	0xD274, 0x9502},
-	{MISENSOR_16BIT,	0xD276, 0x77CF},
-	{MISENSOR_16BIT,	0xD278, 0xFFFF},
-	{MISENSOR_16BIT,	0xD27A, 0xC344},
-	{MISENSOR_16BIT,	0xD27C, 0x2044},
-	{MISENSOR_16BIT,	0xD27E, 0x008E},
-	{MISENSOR_16BIT,	0xD280, 0xB8A1},
-	{MISENSOR_16BIT,	0xD282, 0x0926},
-	{MISENSOR_16BIT,	0xD284, 0x03E0},
-	{MISENSOR_16BIT,	0xD286, 0xB502},
-	{MISENSOR_16BIT,	0xD288, 0x9502},
-	{MISENSOR_16BIT,	0xD28A, 0x952E},
-	{MISENSOR_16BIT,	0xD28C, 0x7E05},
-	{MISENSOR_16BIT,	0xD28E, 0xB5C2},
-	{MISENSOR_16BIT,	0xD290, 0x70CF},
-	{MISENSOR_16BIT,	0xD292, 0xFFFF},
-	{MISENSOR_16BIT,	0xD294, 0xC610},
-	{MISENSOR_16BIT,	0xD296, 0x099A},
-	{MISENSOR_16BIT,	0xD298, 0x04A0},
-	{MISENSOR_16BIT,	0xD29A, 0xB026},
-	{MISENSOR_16BIT,	0xD29C, 0x0E02},
-	{MISENSOR_16BIT,	0xD29E, 0x0560},
-	{MISENSOR_16BIT,	0xD2A0, 0xDE00},
-	{MISENSOR_16BIT,	0xD2A2, 0x0A12},
-	{MISENSOR_16BIT,	0xD2A4, 0x0320},
-	{MISENSOR_16BIT,	0xD2A6, 0xB7C4},
-	{MISENSOR_16BIT,	0xD2A8, 0x0B36},
-	{MISENSOR_16BIT,	0xD2AA, 0x03A0},
-	{MISENSOR_16BIT,	0xD2AC, 0x70C9},
-	{MISENSOR_16BIT,	0xD2AE, 0x9502},
-	{MISENSOR_16BIT,	0xD2B0, 0x7608},
-	{MISENSOR_16BIT,	0xD2B2, 0xB8A8},
-	{MISENSOR_16BIT,	0xD2B4, 0xB502},
-	{MISENSOR_16BIT,	0xD2B6, 0x70CF},
-	{MISENSOR_16BIT,	0xD2B8, 0x0000},
-	{MISENSOR_16BIT,	0xD2BA, 0x5536},
-	{MISENSOR_16BIT,	0xD2BC, 0x7860},
-	{MISENSOR_16BIT,	0xD2BE, 0x2686},
-	{MISENSOR_16BIT,	0xD2C0, 0x1FFB},
-	{MISENSOR_16BIT,	0xD2C2, 0x9502},
-	{MISENSOR_16BIT,	0xD2C4, 0x78C5},
-	{MISENSOR_16BIT,	0xD2C6, 0x0631},
-	{MISENSOR_16BIT,	0xD2C8, 0x05E0},
-	{MISENSOR_16BIT,	0xD2CA, 0xB502},
-	{MISENSOR_16BIT,	0xD2CC, 0x72CF},
-	{MISENSOR_16BIT,	0xD2CE, 0xFFFF},
-	{MISENSOR_16BIT,	0xD2D0, 0xC5D4},
-	{MISENSOR_16BIT,	0xD2D2, 0x923A},
-	{MISENSOR_16BIT,	0xD2D4, 0x73CF},
-	{MISENSOR_16BIT,	0xD2D6, 0xFFFF},
-	{MISENSOR_16BIT,	0xD2D8, 0xC7D0},
-	{MISENSOR_16BIT,	0xD2DA, 0xB020},
-	{MISENSOR_16BIT,	0xD2DC, 0x9220},
-	{MISENSOR_16BIT,	0xD2DE, 0xB021},
-	{MISENSOR_16BIT,	0xD2E0, 0x9221},
-	{MISENSOR_16BIT,	0xD2E2, 0xB022},
-	{MISENSOR_16BIT,	0xD2E4, 0x9222},
-	{MISENSOR_16BIT,	0xD2E6, 0xB023},
-	{MISENSOR_16BIT,	0xD2E8, 0x9223},
-	{MISENSOR_16BIT,	0xD2EA, 0xB024},
-	{MISENSOR_16BIT,	0xD2EC, 0x9227},
-	{MISENSOR_16BIT,	0xD2EE, 0xB025},
-	{MISENSOR_16BIT,	0xD2F0, 0x9228},
-	{MISENSOR_16BIT,	0xD2F2, 0xB026},
-	{MISENSOR_16BIT,	0xD2F4, 0x922B},
-	{MISENSOR_16BIT,	0xD2F6, 0xB027},
-	{MISENSOR_16BIT,	0xD2F8, 0x922C},
-	{MISENSOR_16BIT,	0xD2FA, 0xB028},
-	{MISENSOR_16BIT,	0xD2FC, 0x1258},
-	{MISENSOR_16BIT,	0xD2FE, 0x0101},
-	{MISENSOR_16BIT,	0xD300, 0xB029},
-	{MISENSOR_16BIT,	0xD302, 0x125A},
-	{MISENSOR_16BIT,	0xD304, 0x0101},
-	{MISENSOR_16BIT,	0xD306, 0xB02A},
-	{MISENSOR_16BIT,	0xD308, 0x1336},
-	{MISENSOR_16BIT,	0xD30A, 0x8081},
-	{MISENSOR_16BIT,	0xD30C, 0xA836},
-	{MISENSOR_16BIT,	0xD30E, 0x1337},
-	{MISENSOR_16BIT,	0xD310, 0x8081},
-	{MISENSOR_16BIT,	0xD312, 0xA837},
-	{MISENSOR_16BIT,	0xD314, 0x12A7},
-	{MISENSOR_16BIT,	0xD316, 0x0701},
-	{MISENSOR_16BIT,	0xD318, 0xB02C},
-	{MISENSOR_16BIT,	0xD31A, 0x1354},
-	{MISENSOR_16BIT,	0xD31C, 0x8081},
-	{MISENSOR_16BIT,	0xD31E, 0x7FE0},
-	{MISENSOR_16BIT,	0xD320, 0xA83A},
-	{MISENSOR_16BIT,	0xD322, 0x78E0},
-	{MISENSOR_16BIT,	0xD324, 0xC0F1},
-	{MISENSOR_16BIT,	0xD326, 0x0DC2},
-	{MISENSOR_16BIT,	0xD328, 0x05C0},
-	{MISENSOR_16BIT,	0xD32A, 0x7608},
-	{MISENSOR_16BIT,	0xD32C, 0x09BB},
-	{MISENSOR_16BIT,	0xD32E, 0x0010},
-	{MISENSOR_16BIT,	0xD330, 0x75CF},
-	{MISENSOR_16BIT,	0xD332, 0xFFFF},
-	{MISENSOR_16BIT,	0xD334, 0xD4E0},
-	{MISENSOR_16BIT,	0xD336, 0x8D21},
-	{MISENSOR_16BIT,	0xD338, 0x8D00},
-	{MISENSOR_16BIT,	0xD33A, 0x2153},
-	{MISENSOR_16BIT,	0xD33C, 0x0003},
-	{MISENSOR_16BIT,	0xD33E, 0xB8C0},
-	{MISENSOR_16BIT,	0xD340, 0x8D45},
-	{MISENSOR_16BIT,	0xD342, 0x0B23},
-	{MISENSOR_16BIT,	0xD344, 0x0000},
-	{MISENSOR_16BIT,	0xD346, 0xEA8F},
-	{MISENSOR_16BIT,	0xD348, 0x0915},
-	{MISENSOR_16BIT,	0xD34A, 0x001E},
-	{MISENSOR_16BIT,	0xD34C, 0xFF81},
-	{MISENSOR_16BIT,	0xD34E, 0xE808},
-	{MISENSOR_16BIT,	0xD350, 0x2540},
-	{MISENSOR_16BIT,	0xD352, 0x1900},
-	{MISENSOR_16BIT,	0xD354, 0xFFDE},
-	{MISENSOR_16BIT,	0xD356, 0x8D00},
-	{MISENSOR_16BIT,	0xD358, 0xB880},
-	{MISENSOR_16BIT,	0xD35A, 0xF004},
-	{MISENSOR_16BIT,	0xD35C, 0x8D00},
-	{MISENSOR_16BIT,	0xD35E, 0xB8A0},
-	{MISENSOR_16BIT,	0xD360, 0xAD00},
-	{MISENSOR_16BIT,	0xD362, 0x8D05},
-	{MISENSOR_16BIT,	0xD364, 0xE081},
-	{MISENSOR_16BIT,	0xD366, 0x20CC},
-	{MISENSOR_16BIT,	0xD368, 0x80A2},
-	{MISENSOR_16BIT,	0xD36A, 0xDF00},
-	{MISENSOR_16BIT,	0xD36C, 0xF40A},
-	{MISENSOR_16BIT,	0xD36E, 0x71CF},
-	{MISENSOR_16BIT,	0xD370, 0xFFFF},
-	{MISENSOR_16BIT,	0xD372, 0xC84C},
-	{MISENSOR_16BIT,	0xD374, 0x9102},
-	{MISENSOR_16BIT,	0xD376, 0x7708},
-	{MISENSOR_16BIT,	0xD378, 0xB8A6},
-	{MISENSOR_16BIT,	0xD37A, 0x2786},
-	{MISENSOR_16BIT,	0xD37C, 0x1FFE},
-	{MISENSOR_16BIT,	0xD37E, 0xB102},
-	{MISENSOR_16BIT,	0xD380, 0x0B42},
-	{MISENSOR_16BIT,	0xD382, 0x0180},
-	{MISENSOR_16BIT,	0xD384, 0x0E3E},
-	{MISENSOR_16BIT,	0xD386, 0x0180},
-	{MISENSOR_16BIT,	0xD388, 0x0F4A},
-	{MISENSOR_16BIT,	0xD38A, 0x0160},
-	{MISENSOR_16BIT,	0xD38C, 0x70C9},
-	{MISENSOR_16BIT,	0xD38E, 0x8D05},
-	{MISENSOR_16BIT,	0xD390, 0xE081},
-	{MISENSOR_16BIT,	0xD392, 0x20CC},
-	{MISENSOR_16BIT,	0xD394, 0x80A2},
-	{MISENSOR_16BIT,	0xD396, 0xF429},
-	{MISENSOR_16BIT,	0xD398, 0x76CF},
-	{MISENSOR_16BIT,	0xD39A, 0xFFFF},
-	{MISENSOR_16BIT,	0xD39C, 0xC84C},
-	{MISENSOR_16BIT,	0xD39E, 0x082D},
-	{MISENSOR_16BIT,	0xD3A0, 0x0051},
-	{MISENSOR_16BIT,	0xD3A2, 0x70CF},
-	{MISENSOR_16BIT,	0xD3A4, 0xFFFF},
-	{MISENSOR_16BIT,	0xD3A6, 0xC90C},
-	{MISENSOR_16BIT,	0xD3A8, 0x8805},
-	{MISENSOR_16BIT,	0xD3AA, 0x09B6},
-	{MISENSOR_16BIT,	0xD3AC, 0x0360},
-	{MISENSOR_16BIT,	0xD3AE, 0xD908},
-	{MISENSOR_16BIT,	0xD3B0, 0x2099},
-	{MISENSOR_16BIT,	0xD3B2, 0x0802},
-	{MISENSOR_16BIT,	0xD3B4, 0x9634},
-	{MISENSOR_16BIT,	0xD3B6, 0xB503},
-	{MISENSOR_16BIT,	0xD3B8, 0x7902},
-	{MISENSOR_16BIT,	0xD3BA, 0x1523},
-	{MISENSOR_16BIT,	0xD3BC, 0x1080},
-	{MISENSOR_16BIT,	0xD3BE, 0xB634},
-	{MISENSOR_16BIT,	0xD3C0, 0xE001},
-	{MISENSOR_16BIT,	0xD3C2, 0x1D23},
-	{MISENSOR_16BIT,	0xD3C4, 0x1002},
-	{MISENSOR_16BIT,	0xD3C6, 0xF00B},
-	{MISENSOR_16BIT,	0xD3C8, 0x9634},
-	{MISENSOR_16BIT,	0xD3CA, 0x9503},
-	{MISENSOR_16BIT,	0xD3CC, 0x6038},
-	{MISENSOR_16BIT,	0xD3CE, 0xB614},
-	{MISENSOR_16BIT,	0xD3D0, 0x153F},
-	{MISENSOR_16BIT,	0xD3D2, 0x1080},
-	{MISENSOR_16BIT,	0xD3D4, 0xE001},
-	{MISENSOR_16BIT,	0xD3D6, 0x1D3F},
-	{MISENSOR_16BIT,	0xD3D8, 0x1002},
-	{MISENSOR_16BIT,	0xD3DA, 0xFFA4},
-	{MISENSOR_16BIT,	0xD3DC, 0x9602},
-	{MISENSOR_16BIT,	0xD3DE, 0x7F05},
-	{MISENSOR_16BIT,	0xD3E0, 0xD800},
-	{MISENSOR_16BIT,	0xD3E2, 0xB6E2},
-	{MISENSOR_16BIT,	0xD3E4, 0xAD05},
-	{MISENSOR_16BIT,	0xD3E6, 0x0511},
-	{MISENSOR_16BIT,	0xD3E8, 0x05E0},
-	{MISENSOR_16BIT,	0xD3EA, 0xD800},
-	{MISENSOR_16BIT,	0xD3EC, 0xC0F1},
-	{MISENSOR_16BIT,	0xD3EE, 0x0CFE},
-	{MISENSOR_16BIT,	0xD3F0, 0x05C0},
-	{MISENSOR_16BIT,	0xD3F2, 0x0A96},
-	{MISENSOR_16BIT,	0xD3F4, 0x05A0},
-	{MISENSOR_16BIT,	0xD3F6, 0x7608},
-	{MISENSOR_16BIT,	0xD3F8, 0x0C22},
-	{MISENSOR_16BIT,	0xD3FA, 0x0240},
-	{MISENSOR_16BIT,	0xD3FC, 0xE080},
-	{MISENSOR_16BIT,	0xD3FE, 0x20CA},
-	{MISENSOR_16BIT,	0xD400, 0x0F82},
-	{MISENSOR_16BIT,	0xD402, 0x0000},
-	{MISENSOR_16BIT,	0xD404, 0x190B},
-	{MISENSOR_16BIT,	0xD406, 0x0C60},
-	{MISENSOR_16BIT,	0xD408, 0x05A2},
-	{MISENSOR_16BIT,	0xD40A, 0x21CA},
-	{MISENSOR_16BIT,	0xD40C, 0x0022},
-	{MISENSOR_16BIT,	0xD40E, 0x0C56},
-	{MISENSOR_16BIT,	0xD410, 0x0240},
-	{MISENSOR_16BIT,	0xD412, 0xE806},
-	{MISENSOR_16BIT,	0xD414, 0x0E0E},
-	{MISENSOR_16BIT,	0xD416, 0x0220},
-	{MISENSOR_16BIT,	0xD418, 0x70C9},
-	{MISENSOR_16BIT,	0xD41A, 0xF048},
-	{MISENSOR_16BIT,	0xD41C, 0x0896},
-	{MISENSOR_16BIT,	0xD41E, 0x0440},
-	{MISENSOR_16BIT,	0xD420, 0x0E96},
-	{MISENSOR_16BIT,	0xD422, 0x0400},
-	{MISENSOR_16BIT,	0xD424, 0x0966},
-	{MISENSOR_16BIT,	0xD426, 0x0380},
-	{MISENSOR_16BIT,	0xD428, 0x75CF},
-	{MISENSOR_16BIT,	0xD42A, 0xFFFF},
-	{MISENSOR_16BIT,	0xD42C, 0xD4E0},
-	{MISENSOR_16BIT,	0xD42E, 0x8D00},
-	{MISENSOR_16BIT,	0xD430, 0x084D},
-	{MISENSOR_16BIT,	0xD432, 0x001E},
-	{MISENSOR_16BIT,	0xD434, 0xFF47},
-	{MISENSOR_16BIT,	0xD436, 0x080D},
-	{MISENSOR_16BIT,	0xD438, 0x0050},
-	{MISENSOR_16BIT,	0xD43A, 0xFF57},
-	{MISENSOR_16BIT,	0xD43C, 0x0841},
-	{MISENSOR_16BIT,	0xD43E, 0x0051},
-	{MISENSOR_16BIT,	0xD440, 0x8D04},
-	{MISENSOR_16BIT,	0xD442, 0x9521},
-	{MISENSOR_16BIT,	0xD444, 0xE064},
-	{MISENSOR_16BIT,	0xD446, 0x790C},
-	{MISENSOR_16BIT,	0xD448, 0x702F},
-	{MISENSOR_16BIT,	0xD44A, 0x0CE2},
-	{MISENSOR_16BIT,	0xD44C, 0x05E0},
-	{MISENSOR_16BIT,	0xD44E, 0xD964},
-	{MISENSOR_16BIT,	0xD450, 0x72CF},
-	{MISENSOR_16BIT,	0xD452, 0xFFFF},
-	{MISENSOR_16BIT,	0xD454, 0xC700},
-	{MISENSOR_16BIT,	0xD456, 0x9235},
-	{MISENSOR_16BIT,	0xD458, 0x0811},
-	{MISENSOR_16BIT,	0xD45A, 0x0043},
-	{MISENSOR_16BIT,	0xD45C, 0xFF3D},
-	{MISENSOR_16BIT,	0xD45E, 0x080D},
-	{MISENSOR_16BIT,	0xD460, 0x0051},
-	{MISENSOR_16BIT,	0xD462, 0xD801},
-	{MISENSOR_16BIT,	0xD464, 0xFF77},
-	{MISENSOR_16BIT,	0xD466, 0xF025},
-	{MISENSOR_16BIT,	0xD468, 0x9501},
-	{MISENSOR_16BIT,	0xD46A, 0x9235},
-	{MISENSOR_16BIT,	0xD46C, 0x0911},
-	{MISENSOR_16BIT,	0xD46E, 0x0003},
-	{MISENSOR_16BIT,	0xD470, 0xFF49},
-	{MISENSOR_16BIT,	0xD472, 0x080D},
-	{MISENSOR_16BIT,	0xD474, 0x0051},
-	{MISENSOR_16BIT,	0xD476, 0xD800},
-	{MISENSOR_16BIT,	0xD478, 0xFF72},
-	{MISENSOR_16BIT,	0xD47A, 0xF01B},
-	{MISENSOR_16BIT,	0xD47C, 0x0886},
-	{MISENSOR_16BIT,	0xD47E, 0x03E0},
-	{MISENSOR_16BIT,	0xD480, 0xD801},
-	{MISENSOR_16BIT,	0xD482, 0x0EF6},
-	{MISENSOR_16BIT,	0xD484, 0x03C0},
-	{MISENSOR_16BIT,	0xD486, 0x0F52},
-	{MISENSOR_16BIT,	0xD488, 0x0340},
-	{MISENSOR_16BIT,	0xD48A, 0x0DBA},
-	{MISENSOR_16BIT,	0xD48C, 0x0200},
-	{MISENSOR_16BIT,	0xD48E, 0x0AF6},
-	{MISENSOR_16BIT,	0xD490, 0x0440},
-	{MISENSOR_16BIT,	0xD492, 0x0C22},
-	{MISENSOR_16BIT,	0xD494, 0x0400},
-	{MISENSOR_16BIT,	0xD496, 0x0D72},
-	{MISENSOR_16BIT,	0xD498, 0x0440},
-	{MISENSOR_16BIT,	0xD49A, 0x0DC2},
-	{MISENSOR_16BIT,	0xD49C, 0x0200},
-	{MISENSOR_16BIT,	0xD49E, 0x0972},
-	{MISENSOR_16BIT,	0xD4A0, 0x0440},
-	{MISENSOR_16BIT,	0xD4A2, 0x0D3A},
-	{MISENSOR_16BIT,	0xD4A4, 0x0220},
-	{MISENSOR_16BIT,	0xD4A6, 0xD820},
-	{MISENSOR_16BIT,	0xD4A8, 0x0BFA},
-	{MISENSOR_16BIT,	0xD4AA, 0x0260},
-	{MISENSOR_16BIT,	0xD4AC, 0x70C9},
-	{MISENSOR_16BIT,	0xD4AE, 0x0451},
-	{MISENSOR_16BIT,	0xD4B0, 0x05C0},
-	{MISENSOR_16BIT,	0xD4B2, 0x78E0},
-	{MISENSOR_16BIT,	0xD4B4, 0xD900},
-	{MISENSOR_16BIT,	0xD4B6, 0xF00A},
-	{MISENSOR_16BIT,	0xD4B8, 0x70CF},
-	{MISENSOR_16BIT,	0xD4BA, 0xFFFF},
-	{MISENSOR_16BIT,	0xD4BC, 0xD520},
-	{MISENSOR_16BIT,	0xD4BE, 0x7835},
-	{MISENSOR_16BIT,	0xD4C0, 0x8041},
-	{MISENSOR_16BIT,	0xD4C2, 0x8000},
-	{MISENSOR_16BIT,	0xD4C4, 0xE102},
-	{MISENSOR_16BIT,	0xD4C6, 0xA040},
-	{MISENSOR_16BIT,	0xD4C8, 0x09F1},
-	{MISENSOR_16BIT,	0xD4CA, 0x8114},
-	{MISENSOR_16BIT,	0xD4CC, 0x71CF},
-	{MISENSOR_16BIT,	0xD4CE, 0xFFFF},
-	{MISENSOR_16BIT,	0xD4D0, 0xD4E0},
-	{MISENSOR_16BIT,	0xD4D2, 0x70CF},
-	{MISENSOR_16BIT,	0xD4D4, 0xFFFF},
-	{MISENSOR_16BIT,	0xD4D6, 0xC594},
-	{MISENSOR_16BIT,	0xD4D8, 0xB03A},
-	{MISENSOR_16BIT,	0xD4DA, 0x7FE0},
-	{MISENSOR_16BIT,	0xD4DC, 0xD800},
-	{MISENSOR_16BIT,	0xD4DE, 0x0000},
-	{MISENSOR_16BIT,	0xD4E0, 0x0000},
-	{MISENSOR_16BIT,	0xD4E2, 0x0500},
-	{MISENSOR_16BIT,	0xD4E4, 0x0500},
-	{MISENSOR_16BIT,	0xD4E6, 0x0200},
-	{MISENSOR_16BIT,	0xD4E8, 0x0330},
-	{MISENSOR_16BIT,	0xD4EA, 0x0000},
-	{MISENSOR_16BIT,	0xD4EC, 0x0000},
-	{MISENSOR_16BIT,	0xD4EE, 0x03CD},
-	{MISENSOR_16BIT,	0xD4F0, 0x050D},
-	{MISENSOR_16BIT,	0xD4F2, 0x01C5},
-	{MISENSOR_16BIT,	0xD4F4, 0x03B3},
-	{MISENSOR_16BIT,	0xD4F6, 0x00E0},
-	{MISENSOR_16BIT,	0xD4F8, 0x01E3},
-	{MISENSOR_16BIT,	0xD4FA, 0x0280},
-	{MISENSOR_16BIT,	0xD4FC, 0x01E0},
-	{MISENSOR_16BIT,	0xD4FE, 0x0109},
-	{MISENSOR_16BIT,	0xD500, 0x0080},
-	{MISENSOR_16BIT,	0xD502, 0x0500},
-	{MISENSOR_16BIT,	0xD504, 0x0000},
-	{MISENSOR_16BIT,	0xD506, 0x0000},
-	{MISENSOR_16BIT,	0xD508, 0x0000},
-	{MISENSOR_16BIT,	0xD50A, 0x0000},
-	{MISENSOR_16BIT,	0xD50C, 0x0000},
-	{MISENSOR_16BIT,	0xD50E, 0x0000},
-	{MISENSOR_16BIT,	0xD510, 0x0000},
-	{MISENSOR_16BIT,	0xD512, 0x0000},
-	{MISENSOR_16BIT,	0xD514, 0x0000},
-	{MISENSOR_16BIT,	0xD516, 0x0000},
-	{MISENSOR_16BIT,	0xD518, 0x0000},
-	{MISENSOR_16BIT,	0xD51A, 0x0000},
-	{MISENSOR_16BIT,	0xD51C, 0x0000},
-	{MISENSOR_16BIT,	0xD51E, 0x0000},
-	{MISENSOR_16BIT,	0xD520, 0xFFFF},
-	{MISENSOR_16BIT,	0xD522, 0xC9B4},
-	{MISENSOR_16BIT,	0xD524, 0xFFFF},
-	{MISENSOR_16BIT,	0xD526, 0xD324},
-	{MISENSOR_16BIT,	0xD528, 0xFFFF},
-	{MISENSOR_16BIT,	0xD52A, 0xCA34},
-	{MISENSOR_16BIT,	0xD52C, 0xFFFF},
-	{MISENSOR_16BIT,	0xD52E, 0xD3EC},
-	{MISENSOR_16BIT,	0x098E, 0x0000},
-	{MISENSOR_16BIT,	0xE000, 0x04B4},
-	{MISENSOR_16BIT,	0xE002, 0x0302},
-	{MISENSOR_16BIT,	0xE004, 0x4103},
-	{MISENSOR_16BIT,	0xE006, 0x0202},
-	{MISENSOR_16BIT,	0x0080, 0xFFF0},
-	{MISENSOR_16BIT,	0x0080, 0xFFF1},
-
-	/* PGA parameter and APGA
-	 * [Step4-APGA] [TP101_MT9M114_APGA]
-	 */
-	{MISENSOR_16BIT,	0x098E, 0x495E},
-	{MISENSOR_16BIT,	0xC95E, 0x0000},
-	{MISENSOR_16BIT,	0x3640, 0x02B0},
-	{MISENSOR_16BIT,	0x3642, 0x8063},
-	{MISENSOR_16BIT,	0x3644, 0x78D0},
-	{MISENSOR_16BIT,	0x3646, 0x50CC},
-	{MISENSOR_16BIT,	0x3648, 0x3511},
-	{MISENSOR_16BIT,	0x364A, 0x0110},
-	{MISENSOR_16BIT,	0x364C, 0xBD8A},
-	{MISENSOR_16BIT,	0x364E, 0x0CD1},
-	{MISENSOR_16BIT,	0x3650, 0x24ED},
-	{MISENSOR_16BIT,	0x3652, 0x7C11},
-	{MISENSOR_16BIT,	0x3654, 0x0150},
-	{MISENSOR_16BIT,	0x3656, 0x124C},
-	{MISENSOR_16BIT,	0x3658, 0x3130},
-	{MISENSOR_16BIT,	0x365A, 0x508C},
-	{MISENSOR_16BIT,	0x365C, 0x21F1},
-	{MISENSOR_16BIT,	0x365E, 0x0090},
-	{MISENSOR_16BIT,	0x3660, 0xBFCA},
-	{MISENSOR_16BIT,	0x3662, 0x0A11},
-	{MISENSOR_16BIT,	0x3664, 0x4F4B},
-	{MISENSOR_16BIT,	0x3666, 0x28B1},
-	{MISENSOR_16BIT,	0x3680, 0x50A9},
-	{MISENSOR_16BIT,	0x3682, 0xA04B},
-	{MISENSOR_16BIT,	0x3684, 0x0E2D},
-	{MISENSOR_16BIT,	0x3686, 0x73EC},
-	{MISENSOR_16BIT,	0x3688, 0x164F},
-	{MISENSOR_16BIT,	0x368A, 0xF829},
-	{MISENSOR_16BIT,	0x368C, 0xC1A8},
-	{MISENSOR_16BIT,	0x368E, 0xB0EC},
-	{MISENSOR_16BIT,	0x3690, 0xE76A},
-	{MISENSOR_16BIT,	0x3692, 0x69AF},
-	{MISENSOR_16BIT,	0x3694, 0x378C},
-	{MISENSOR_16BIT,	0x3696, 0xA70D},
-	{MISENSOR_16BIT,	0x3698, 0x884F},
-	{MISENSOR_16BIT,	0x369A, 0xEE8B},
-	{MISENSOR_16BIT,	0x369C, 0x5DEF},
-	{MISENSOR_16BIT,	0x369E, 0x27CC},
-	{MISENSOR_16BIT,	0x36A0, 0xCAAC},
-	{MISENSOR_16BIT,	0x36A2, 0x840E},
-	{MISENSOR_16BIT,	0x36A4, 0xDAA9},
-	{MISENSOR_16BIT,	0x36A6, 0xF00C},
-	{MISENSOR_16BIT,	0x36C0, 0x1371},
-	{MISENSOR_16BIT,	0x36C2, 0x272F},
-	{MISENSOR_16BIT,	0x36C4, 0x2293},
-	{MISENSOR_16BIT,	0x36C6, 0xE6D0},
-	{MISENSOR_16BIT,	0x36C8, 0xEC32},
-	{MISENSOR_16BIT,	0x36CA, 0x11B1},
-	{MISENSOR_16BIT,	0x36CC, 0x7BAF},
-	{MISENSOR_16BIT,	0x36CE, 0x5813},
-	{MISENSOR_16BIT,	0x36D0, 0xB871},
-	{MISENSOR_16BIT,	0x36D2, 0x8913},
-	{MISENSOR_16BIT,	0x36D4, 0x4610},
-	{MISENSOR_16BIT,	0x36D6, 0x7EEE},
-	{MISENSOR_16BIT,	0x36D8, 0x0DF3},
-	{MISENSOR_16BIT,	0x36DA, 0xB84F},
-	{MISENSOR_16BIT,	0x36DC, 0xB532},
-	{MISENSOR_16BIT,	0x36DE, 0x1171},
-	{MISENSOR_16BIT,	0x36E0, 0x13CF},
-	{MISENSOR_16BIT,	0x36E2, 0x22F3},
-	{MISENSOR_16BIT,	0x36E4, 0xE090},
-	{MISENSOR_16BIT,	0x36E6, 0x8133},
-	{MISENSOR_16BIT,	0x3700, 0x88AE},
-	{MISENSOR_16BIT,	0x3702, 0x00EA},
-	{MISENSOR_16BIT,	0x3704, 0x344F},
-	{MISENSOR_16BIT,	0x3706, 0xEC88},
-	{MISENSOR_16BIT,	0x3708, 0x3E91},
-	{MISENSOR_16BIT,	0x370A, 0xF12D},
-	{MISENSOR_16BIT,	0x370C, 0xB0EF},
-	{MISENSOR_16BIT,	0x370E, 0x77CD},
-	{MISENSOR_16BIT,	0x3710, 0x7930},
-	{MISENSOR_16BIT,	0x3712, 0x5C12},
-	{MISENSOR_16BIT,	0x3714, 0x500C},
-	{MISENSOR_16BIT,	0x3716, 0x22CE},
-	{MISENSOR_16BIT,	0x3718, 0x2370},
-	{MISENSOR_16BIT,	0x371A, 0x258F},
-	{MISENSOR_16BIT,	0x371C, 0x3D30},
-	{MISENSOR_16BIT,	0x371E, 0x370C},
-	{MISENSOR_16BIT,	0x3720, 0x03ED},
-	{MISENSOR_16BIT,	0x3722, 0x9AD0},
-	{MISENSOR_16BIT,	0x3724, 0x7ECF},
-	{MISENSOR_16BIT,	0x3726, 0x1093},
-	{MISENSOR_16BIT,	0x3740, 0x2391},
-	{MISENSOR_16BIT,	0x3742, 0xAAD0},
-	{MISENSOR_16BIT,	0x3744, 0x28F2},
-	{MISENSOR_16BIT,	0x3746, 0xBA4F},
-	{MISENSOR_16BIT,	0x3748, 0xC536},
-	{MISENSOR_16BIT,	0x374A, 0x1472},
-	{MISENSOR_16BIT,	0x374C, 0xD110},
-	{MISENSOR_16BIT,	0x374E, 0x2933},
-	{MISENSOR_16BIT,	0x3750, 0xD0D1},
-	{MISENSOR_16BIT,	0x3752, 0x9F37},
-	{MISENSOR_16BIT,	0x3754, 0x34D1},
-	{MISENSOR_16BIT,	0x3756, 0x1C6C},
-	{MISENSOR_16BIT,	0x3758, 0x3FD2},
-	{MISENSOR_16BIT,	0x375A, 0xCB72},
-	{MISENSOR_16BIT,	0x375C, 0xBA96},
-	{MISENSOR_16BIT,	0x375E, 0x1551},
-	{MISENSOR_16BIT,	0x3760, 0xB74F},
-	{MISENSOR_16BIT,	0x3762, 0x1672},
-	{MISENSOR_16BIT,	0x3764, 0x84F1},
-	{MISENSOR_16BIT,	0x3766, 0xC2D6},
-	{MISENSOR_16BIT,	0x3782, 0x01E0},
-	{MISENSOR_16BIT,	0x3784, 0x0280},
-	{MISENSOR_16BIT,	0x37C0, 0xA6EA},
-	{MISENSOR_16BIT,	0x37C2, 0x874B},
-	{MISENSOR_16BIT,	0x37C4, 0x85CB},
-	{MISENSOR_16BIT,	0x37C6, 0x968A},
-	{MISENSOR_16BIT,	0x098E, 0x0000},
-	{MISENSOR_16BIT,	0xC960, 0x0AF0},
-	{MISENSOR_16BIT,	0xC962, 0x79E2},
-	{MISENSOR_16BIT,	0xC964, 0x5EC8},
-	{MISENSOR_16BIT,	0xC966, 0x791F},
-	{MISENSOR_16BIT,	0xC968, 0x76EE},
-	{MISENSOR_16BIT,	0xC96A, 0x0FA0},
-	{MISENSOR_16BIT,	0xC96C, 0x7DFA},
-	{MISENSOR_16BIT,	0xC96E, 0x7DAF},
-	{MISENSOR_16BIT,	0xC970, 0x7E02},
-	{MISENSOR_16BIT,	0xC972, 0x7E0A},
-	{MISENSOR_16BIT,	0xC974, 0x1964},
-	{MISENSOR_16BIT,	0xC976, 0x7CDC},
-	{MISENSOR_16BIT,	0xC978, 0x7838},
-	{MISENSOR_16BIT,	0xC97A, 0x7C2F},
-	{MISENSOR_16BIT,	0xC97C, 0x7792},
-	{MISENSOR_16BIT,	0xC95E, 0x0003},
-
-	/* [Step4-APGA] */
-	{MISENSOR_16BIT,	0x098E, 0x0000},
-	{MISENSOR_16BIT,	0xC95E, 0x0003},
-
-	/* [Step5-AWB_CCM]1: LOAD=CCM */
-	{MISENSOR_16BIT,	0xC892, 0x0267},
-	{MISENSOR_16BIT,	0xC894, 0xFF1A},
-	{MISENSOR_16BIT,	0xC896, 0xFFB3},
-	{MISENSOR_16BIT,	0xC898, 0xFF80},
-	{MISENSOR_16BIT,	0xC89A, 0x0166},
-	{MISENSOR_16BIT,	0xC89C, 0x0003},
-	{MISENSOR_16BIT,	0xC89E, 0xFF9A},
-	{MISENSOR_16BIT,	0xC8A0, 0xFEB4},
-	{MISENSOR_16BIT,	0xC8A2, 0x024D},
-	{MISENSOR_16BIT,	0xC8A4, 0x01BF},
-	{MISENSOR_16BIT,	0xC8A6, 0xFF01},
-	{MISENSOR_16BIT,	0xC8A8, 0xFFF3},
-	{MISENSOR_16BIT,	0xC8AA, 0xFF75},
-	{MISENSOR_16BIT,	0xC8AC, 0x0198},
-	{MISENSOR_16BIT,	0xC8AE, 0xFFFD},
-	{MISENSOR_16BIT,	0xC8B0, 0xFF9A},
-	{MISENSOR_16BIT,	0xC8B2, 0xFEE7},
-	{MISENSOR_16BIT,	0xC8B4, 0x02A8},
-	{MISENSOR_16BIT,	0xC8B6, 0x01D9},
-	{MISENSOR_16BIT,	0xC8B8, 0xFF26},
-	{MISENSOR_16BIT,	0xC8BA, 0xFFF3},
-	{MISENSOR_16BIT,	0xC8BC, 0xFFB3},
-	{MISENSOR_16BIT,	0xC8BE, 0x0132},
-	{MISENSOR_16BIT,	0xC8C0, 0xFFE8},
-	{MISENSOR_16BIT,	0xC8C2, 0xFFDA},
-	{MISENSOR_16BIT,	0xC8C4, 0xFECD},
-	{MISENSOR_16BIT,	0xC8C6, 0x02C2},
-	{MISENSOR_16BIT,	0xC8C8, 0x0075},
-	{MISENSOR_16BIT,	0xC8CA, 0x011C},
-	{MISENSOR_16BIT,	0xC8CC, 0x009A},
-	{MISENSOR_16BIT,	0xC8CE, 0x0105},
-	{MISENSOR_16BIT,	0xC8D0, 0x00A4},
-	{MISENSOR_16BIT,	0xC8D2, 0x00AC},
-	{MISENSOR_16BIT,	0xC8D4, 0x0A8C},
-	{MISENSOR_16BIT,	0xC8D6, 0x0F0A},
-	{MISENSOR_16BIT,	0xC8D8, 0x1964},
-
-	/* LOAD=AWB */
-	{MISENSOR_16BIT,	0xC914, 0x0000},
-	{MISENSOR_16BIT,	0xC916, 0x0000},
-	{MISENSOR_16BIT,	0xC918, 0x04FF},
-	{MISENSOR_16BIT,	0xC91A, 0x02CF},
-	{MISENSOR_16BIT,	0xC904, 0x0033},
-	{MISENSOR_16BIT,	0xC906, 0x0040},
-	{MISENSOR_8BIT,   0xC8F2, 0x03},
-	{MISENSOR_8BIT,   0xC8F3, 0x02},
-	{MISENSOR_16BIT,	0xC906, 0x003C},
-	{MISENSOR_16BIT,	0xC8F4, 0x0000},
-	{MISENSOR_16BIT,	0xC8F6, 0x0000},
-	{MISENSOR_16BIT,	0xC8F8, 0x0000},
-	{MISENSOR_16BIT,	0xC8FA, 0xE724},
-	{MISENSOR_16BIT,	0xC8FC, 0x1583},
-	{MISENSOR_16BIT,	0xC8FE, 0x2045},
-	{MISENSOR_16BIT,	0xC900, 0x05DC},
-	{MISENSOR_16BIT,	0xC902, 0x007C},
-	{MISENSOR_8BIT,   0xC90C, 0x80},
-	{MISENSOR_8BIT,   0xC90D, 0x80},
-	{MISENSOR_8BIT,   0xC90E, 0x80},
-	{MISENSOR_8BIT,   0xC90F, 0x88},
-	{MISENSOR_8BIT,   0xC910, 0x80},
-	{MISENSOR_8BIT,   0xC911, 0x80},
-
-	/* LOAD=Step7-CPIPE_Preference */
-	{MISENSOR_16BIT,	0xC926, 0x0020},
-	{MISENSOR_16BIT,	0xC928, 0x009A},
-	{MISENSOR_16BIT,	0xC946, 0x0070},
-	{MISENSOR_16BIT,	0xC948, 0x00F3},
-	{MISENSOR_16BIT,	0xC952, 0x0020},
-	{MISENSOR_16BIT,	0xC954, 0x009A},
-	{MISENSOR_8BIT,   0xC92A, 0x80},
-	{MISENSOR_8BIT,   0xC92B, 0x4B},
-	{MISENSOR_8BIT,   0xC92C, 0x00},
-	{MISENSOR_8BIT,   0xC92D, 0xFF},
-	{MISENSOR_8BIT,   0xC92E, 0x3C},
-	{MISENSOR_8BIT,   0xC92F, 0x02},
-	{MISENSOR_8BIT,   0xC930, 0x06},
-	{MISENSOR_8BIT,   0xC931, 0x64},
-	{MISENSOR_8BIT,   0xC932, 0x01},
-	{MISENSOR_8BIT,   0xC933, 0x0C},
-	{MISENSOR_8BIT,   0xC934, 0x3C},
-	{MISENSOR_8BIT,   0xC935, 0x3C},
-	{MISENSOR_8BIT,   0xC936, 0x3C},
-	{MISENSOR_8BIT,   0xC937, 0x0F},
-	{MISENSOR_8BIT,   0xC938, 0x64},
-	{MISENSOR_8BIT,   0xC939, 0x64},
-	{MISENSOR_8BIT,   0xC93A, 0x64},
-	{MISENSOR_8BIT,   0xC93B, 0x32},
-	{MISENSOR_16BIT,	0xC93C, 0x0020},
-	{MISENSOR_16BIT,	0xC93E, 0x009A},
-	{MISENSOR_16BIT,	0xC940, 0x00DC},
-	{MISENSOR_8BIT,   0xC942, 0x38},
-	{MISENSOR_8BIT,   0xC943, 0x30},
-	{MISENSOR_8BIT,   0xC944, 0x50},
-	{MISENSOR_8BIT,   0xC945, 0x19},
-	{MISENSOR_16BIT,	0xC94A, 0x0230},
-	{MISENSOR_16BIT,	0xC94C, 0x0010},
-	{MISENSOR_16BIT,	0xC94E, 0x01CD},
-	{MISENSOR_8BIT,   0xC950, 0x05},
-	{MISENSOR_8BIT,   0xC951, 0x40},
-	{MISENSOR_8BIT,   0xC87B, 0x1B},
-	{MISENSOR_8BIT,   0xC878, 0x0E},
-	{MISENSOR_16BIT,	0xC890, 0x0080},
-	{MISENSOR_16BIT,	0xC886, 0x0100},
-	{MISENSOR_16BIT,	0xC87C, 0x005A},
-	{MISENSOR_8BIT,   0xB42A, 0x05},
-	{MISENSOR_8BIT,   0xA80A, 0x20},
-
-	/* Speed up AE/AWB */
-	{MISENSOR_16BIT,	0x098E, 0x2802},
-	{MISENSOR_16BIT,	0xA802, 0x0008},
-	{MISENSOR_8BIT,   0xC908, 0x01},
-	{MISENSOR_8BIT,   0xC879, 0x01},
-	{MISENSOR_8BIT,   0xC909, 0x02},
-	{MISENSOR_8BIT,   0xA80A, 0x18},
-	{MISENSOR_8BIT,   0xA80B, 0x18},
-	{MISENSOR_8BIT,   0xAC16, 0x18},
-	{MISENSOR_8BIT,   0xC878, 0x0E},
-
-	{MISENSOR_TOK_TERM, 0, 0}
-};
-
-#endif
diff --git a/drivers/staging/media/atomisp/i2c/ov2680.h b/drivers/staging/media/atomisp/i2c/ov2680.h
deleted file mode 100644
index cb38e6e79409..000000000000
--- a/drivers/staging/media/atomisp/i2c/ov2680.h
+++ /dev/null
@@ -1,856 +0,0 @@
-/*
- * Support for OmniVision OV2680 5M camera sensor.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __OV2680_H__
-#define __OV2680_H__
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/i2c.h>
-#include <linux/delay.h>
-#include <linux/videodev2.h>
-#include <linux/spinlock.h>
-#include <media/v4l2-subdev.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-ctrls.h>
-#include <linux/v4l2-mediabus.h>
-#include <media/media-entity.h>
-
-#include "../include/linux/atomisp_platform.h"
-
-/* Defines for register writes and register array processing */
-#define I2C_MSG_LENGTH		0x2
-#define I2C_RETRY_COUNT		5
-
-#define OV2680_FOCAL_LENGTH_NUM	334	/*3.34mm*/
-#define OV2680_FOCAL_LENGTH_DEM	100
-#define OV2680_F_NUMBER_DEFAULT_NUM	24
-#define OV2680_F_NUMBER_DEM	10
-
-#define OV2680_BIN_FACTOR_MAX 4
-
-#define MAX_FMTS		1
-
-/* sensor_mode_data read_mode adaptation */
-#define OV2680_READ_MODE_BINNING_ON	0x0400
-#define OV2680_READ_MODE_BINNING_OFF	0x00
-#define OV2680_INTEGRATION_TIME_MARGIN	8
-
-#define OV2680_MAX_EXPOSURE_VALUE	0xFFF1
-#define OV2680_MAX_GAIN_VALUE		0xFF
-
-/*
- * focal length bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define OV2680_FOCAL_LENGTH_DEFAULT 0x1B70064
-
-/*
- * current f-number bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define OV2680_F_NUMBER_DEFAULT 0x18000a
-
-/*
- * f-number range bits definition:
- * bits 31-24: max f-number numerator
- * bits 23-16: max f-number denominator
- * bits 15-8: min f-number numerator
- * bits 7-0: min f-number denominator
- */
-#define OV2680_F_NUMBER_RANGE 0x180a180a
-#define OV2680_ID	0x2680
-
-#define OV2680_FINE_INTG_TIME_MIN 0
-#define OV2680_FINE_INTG_TIME_MAX_MARGIN 0
-#define OV2680_COARSE_INTG_TIME_MIN 1
-#define OV2680_COARSE_INTG_TIME_MAX_MARGIN 6
-
-/*
- * OV2680 System control registers
- */
-#define OV2680_SW_SLEEP				0x0100
-#define OV2680_SW_RESET				0x0103
-#define OV2680_SW_STREAM			0x0100
-
-#define OV2680_SC_CMMN_CHIP_ID_H		0x300A
-#define OV2680_SC_CMMN_CHIP_ID_L		0x300B
-#define OV2680_SC_CMMN_SCCB_ID			0x302B /* 0x300C*/
-#define OV2680_SC_CMMN_SUB_ID			0x302A /* process, version*/
-
-#define OV2680_GROUP_ACCESS							0x3208 /*Bit[7:4] Group control, Bit[3:0] Group ID*/
-
-#define OV2680_EXPOSURE_H							0x3500 /*Bit[3:0] Bit[19:16] of exposure, remaining 16 bits lies in Reg0x3501&Reg0x3502*/
-#define OV2680_EXPOSURE_M							0x3501
-#define OV2680_EXPOSURE_L							0x3502
-#define OV2680_AGC_H								0x350A /*Bit[1:0] means Bit[9:8] of gain*/
-#define OV2680_AGC_L								0x350B /*Bit[7:0] of gain*/
-
-#define OV2680_HORIZONTAL_START_H					0x3800 /*Bit[11:8]*/
-#define OV2680_HORIZONTAL_START_L					0x3801 /*Bit[7:0]*/
-#define OV2680_VERTICAL_START_H						0x3802 /*Bit[11:8]*/
-#define OV2680_VERTICAL_START_L						0x3803 /*Bit[7:0]*/
-#define OV2680_HORIZONTAL_END_H						0x3804 /*Bit[11:8]*/
-#define OV2680_HORIZONTAL_END_L						0x3805 /*Bit[7:0]*/
-#define OV2680_VERTICAL_END_H						0x3806 /*Bit[11:8]*/
-#define OV2680_VERTICAL_END_L						0x3807 /*Bit[7:0]*/
-#define OV2680_HORIZONTAL_OUTPUT_SIZE_H				0x3808 /*Bit[3:0]*/
-#define OV2680_HORIZONTAL_OUTPUT_SIZE_L				0x3809 /*Bit[7:0]*/
-#define OV2680_VERTICAL_OUTPUT_SIZE_H				0x380a /*Bit[3:0]*/
-#define OV2680_VERTICAL_OUTPUT_SIZE_L				0x380b /*Bit[7:0]*/
-#define OV2680_TIMING_HTS_H							0x380C  /*High 8-bit, and low 8-bit HTS address is 0x380d*/
-#define OV2680_TIMING_HTS_L							0x380D  /*High 8-bit, and low 8-bit HTS address is 0x380d*/
-#define OV2680_TIMING_VTS_H							0x380e  /*High 8-bit, and low 8-bit HTS address is 0x380f*/
-#define OV2680_TIMING_VTS_L							0x380f  /*High 8-bit, and low 8-bit HTS address is 0x380f*/
-#define OV2680_FRAME_OFF_NUM						0x4202
-
-/*Flip/Mirror*/
-#define OV2680_FLIP_REG				0x3820
-#define OV2680_MIRROR_REG			0x3821
-#define OV2680_FLIP_BIT				1
-#define OV2680_MIRROR_BIT			2
-#define OV2680_FLIP_MIRROR_BIT_ENABLE		4
-
-#define OV2680_MWB_RED_GAIN_H			0x5004/*0x3400*/
-#define OV2680_MWB_GREEN_GAIN_H			0x5006/*0x3402*/
-#define OV2680_MWB_BLUE_GAIN_H			0x5008/*0x3404*/
-#define OV2680_MWB_GAIN_MAX				0x0fff
-
-#define OV2680_START_STREAMING			0x01
-#define OV2680_STOP_STREAMING			0x00
-
-
-#define OV2680_INVALID_CONFIG	0xffffffff
-
-
-struct regval_list {
-	u16 reg_num;
-	u8 value;
-};
-
-struct ov2680_resolution {
-	u8 *desc;
-	const struct ov2680_reg *regs;
-	int res;
-	int width;
-	int height;
-	int fps;
-	int pix_clk_freq;
-	u32 skip_frames;
-	u16 pixels_per_line;
-	u16 lines_per_frame;
-	u8 bin_factor_x;
-	u8 bin_factor_y;
-	u8 bin_mode;
-	bool used;
-};
-
-struct ov2680_format {
-	u8 *desc;
-	u32 pixelformat;
-	struct ov2680_reg *regs;
-};
-
-	/*
-	 * ov2680 device structure.
-	 */
-	struct ov2680_device {
-		struct v4l2_subdev sd;
-		struct media_pad pad;
-		struct v4l2_mbus_framefmt format;
-		struct mutex input_lock;
-	struct v4l2_ctrl_handler ctrl_handler;
-		struct camera_sensor_platform_data *platform_data;
-		int vt_pix_clk_freq_mhz;
-		int fmt_idx;
-		int run_mode;
-		u8 res;
-		u8 type;
-	};
-
-	enum ov2680_tok_type {
-		OV2680_8BIT  = 0x0001,
-		OV2680_16BIT = 0x0002,
-		OV2680_32BIT = 0x0004,
-		OV2680_TOK_TERM   = 0xf000,	/* terminating token for reg list */
-		OV2680_TOK_DELAY  = 0xfe00,	/* delay token for reg list */
-		OV2680_TOK_MASK = 0xfff0
-	};
-
-	/**
-	 * struct ov2680_reg - MI sensor  register format
-	 * @type: type of the register
-	 * @reg: 16-bit offset to register
-	 * @val: 8/16/32-bit register value
-	 *
-	 * Define a structure for sensor register initialization values
-	 */
-	struct ov2680_reg {
-		enum ov2680_tok_type type;
-		u16 reg;
-		u32 val;	/* @set value for read/mod/write, @mask */
-	};
-
-	#define to_ov2680_sensor(x) container_of(x, struct ov2680_device, sd)
-
-	#define OV2680_MAX_WRITE_BUF_SIZE	30
-
-	struct ov2680_write_buffer {
-		u16 addr;
-		u8 data[OV2680_MAX_WRITE_BUF_SIZE];
-	};
-
-	struct ov2680_write_ctrl {
-		int index;
-		struct ov2680_write_buffer buffer;
-	};
-
-	static struct ov2680_reg const ov2680_global_setting[] = {
-	    {OV2680_8BIT, 0x0103, 0x01},
-	    {OV2680_8BIT, 0x3002, 0x00},
-	    {OV2680_8BIT, 0x3016, 0x1c},
-	    {OV2680_8BIT, 0x3018, 0x44},
-	    {OV2680_8BIT, 0x3020, 0x00},
-	    {OV2680_8BIT, 0x3080, 0x02},
-	    {OV2680_8BIT, 0x3082, 0x45},
-	    {OV2680_8BIT, 0x3084, 0x09},
-	    {OV2680_8BIT, 0x3085, 0x04},
-	    {OV2680_8BIT, 0x3503, 0x03},
-	    {OV2680_8BIT, 0x350b, 0x36},
-	    {OV2680_8BIT, 0x3600, 0xb4},
-	    {OV2680_8BIT, 0x3603, 0x39},
-	    {OV2680_8BIT, 0x3604, 0x24},
-	    {OV2680_8BIT, 0x3605, 0x00},
-	    {OV2680_8BIT, 0x3620, 0x26},
-	    {OV2680_8BIT, 0x3621, 0x37},
-	    {OV2680_8BIT, 0x3622, 0x04},
-	    {OV2680_8BIT, 0x3628, 0x00},
-	    {OV2680_8BIT, 0x3705, 0x3c},
-	    {OV2680_8BIT, 0x370c, 0x50},
-	    {OV2680_8BIT, 0x370d, 0xc0},
-	    {OV2680_8BIT, 0x3718, 0x88},
-	    {OV2680_8BIT, 0x3720, 0x00},
-	    {OV2680_8BIT, 0x3721, 0x00},
-	    {OV2680_8BIT, 0x3722, 0x00},
-	    {OV2680_8BIT, 0x3723, 0x00},
-	    {OV2680_8BIT, 0x3738, 0x00},
-	    {OV2680_8BIT, 0x3717, 0x58},
-	    {OV2680_8BIT, 0x3781, 0x80},
-	    {OV2680_8BIT, 0x3789, 0x60},
-	    {OV2680_8BIT, 0x3800, 0x00},
-	    {OV2680_8BIT, 0x3819, 0x04},
-	    {OV2680_8BIT, 0x4000, 0x81},
-	    {OV2680_8BIT, 0x4001, 0x40},
-	    {OV2680_8BIT, 0x4602, 0x02},
-	    {OV2680_8BIT, 0x481f, 0x36},
-	    {OV2680_8BIT, 0x4825, 0x36},
-	    {OV2680_8BIT, 0x4837, 0x18},
-	    {OV2680_8BIT, 0x5002, 0x30},
-	    {OV2680_8BIT, 0x5004, 0x04},//manual awb 1x
-	    {OV2680_8BIT, 0x5005, 0x00},
-	    {OV2680_8BIT, 0x5006, 0x04},
-	    {OV2680_8BIT, 0x5007, 0x00},
-	    {OV2680_8BIT, 0x5008, 0x04},
-	    {OV2680_8BIT, 0x5009, 0x00},
-	    {OV2680_8BIT, 0x5080, 0x00},
-	    {OV2680_8BIT, 0x3701, 0x64},  //add on 14/05/13
-	    {OV2680_8BIT, 0x3784, 0x0c},  //based OV2680_R1A_AM10.ovt add on 14/06/13
-	    {OV2680_8BIT, 0x5780, 0x3e},  //based OV2680_R1A_AM10.ovt,Adjust DPC setting (57xx) on 14/06/13
-	    {OV2680_8BIT, 0x5781, 0x0f},
-	    {OV2680_8BIT, 0x5782, 0x04},
-	    {OV2680_8BIT, 0x5783, 0x02},
-	    {OV2680_8BIT, 0x5784, 0x01},
-	    {OV2680_8BIT, 0x5785, 0x01},
-	    {OV2680_8BIT, 0x5786, 0x00},
-	    {OV2680_8BIT, 0x5787, 0x04},
-	    {OV2680_8BIT, 0x5788, 0x02},
-	    {OV2680_8BIT, 0x5789, 0x00},
-	    {OV2680_8BIT, 0x578a, 0x01},
-	    {OV2680_8BIT, 0x578b, 0x02},
-	    {OV2680_8BIT, 0x578c, 0x03},
-	    {OV2680_8BIT, 0x578d, 0x03},
-	    {OV2680_8BIT, 0x578e, 0x08},
-	    {OV2680_8BIT, 0x578f, 0x0c},
-	    {OV2680_8BIT, 0x5790, 0x08},
-	    {OV2680_8BIT, 0x5791, 0x04},
-	    {OV2680_8BIT, 0x5792, 0x00},
-	    {OV2680_8BIT, 0x5793, 0x00},
-	    {OV2680_8BIT, 0x5794, 0x03}, //based OV2680_R1A_AM10.ovt,Adjust DPC setting (57xx) on 14/06/13
-		{OV2680_8BIT, 0x0100, 0x00},	//stream off
-
-		{OV2680_TOK_TERM, 0, 0}
-	};
-
-
-	/*
-	 * 176x144 30fps  VBlanking 1lane 10Bit (binning)
-	 */
-	static struct ov2680_reg const ov2680_QCIF_30fps[] = {
-	{OV2680_8BIT, 0x3086, 0x01},
-	{OV2680_8BIT, 0x3501, 0x24},
-	{OV2680_8BIT, 0x3502, 0x40},
-	{OV2680_8BIT, 0x370a, 0x23},
-	{OV2680_8BIT, 0x3801, 0xa0},
-	{OV2680_8BIT, 0x3802, 0x00},
-	{OV2680_8BIT, 0x3803, 0x78},
-	{OV2680_8BIT, 0x3804, 0x05},
-	{OV2680_8BIT, 0x3805, 0xaf},
-	{OV2680_8BIT, 0x3806, 0x04},
-	{OV2680_8BIT, 0x3807, 0x47},
-	{OV2680_8BIT, 0x3808, 0x00},
-	{OV2680_8BIT, 0x3809, 0xC0},
-	{OV2680_8BIT, 0x380a, 0x00},
-	{OV2680_8BIT, 0x380b, 0xa0},
-	{OV2680_8BIT, 0x380c, 0x06},
-	{OV2680_8BIT, 0x380d, 0xb0},
-	{OV2680_8BIT, 0x380e, 0x02},
-	{OV2680_8BIT, 0x380f, 0x84},
-	{OV2680_8BIT, 0x3810, 0x00},
-	{OV2680_8BIT, 0x3811, 0x04},
-	{OV2680_8BIT, 0x3812, 0x00},
-	{OV2680_8BIT, 0x3813, 0x04},
-	{OV2680_8BIT, 0x3814, 0x31},
-	{OV2680_8BIT, 0x3815, 0x31},
-	{OV2680_8BIT, 0x4000, 0x81},
-	{OV2680_8BIT, 0x4001, 0x40},
-	{OV2680_8BIT, 0x4008, 0x00},
-	{OV2680_8BIT, 0x4009, 0x03},
-	{OV2680_8BIT, 0x5081, 0x41},
-	{OV2680_8BIT, 0x5708, 0x00}, //add for full size flip off and mirror off 2014/09/11
-	{OV2680_8BIT, 0x5704, 0x10},
-	{OV2680_8BIT, 0x5705, 0xa0},
-	{OV2680_8BIT, 0x5706, 0x0c},
-	{OV2680_8BIT, 0x5707, 0x78},
-	{OV2680_8BIT, 0x3820, 0xc2},
-	{OV2680_8BIT, 0x3821, 0x01},
-    // {OV2680_8BIT, 0x5090, 0x0c},
-    {OV2680_TOK_TERM, 0, 0}
-	};
-
-	/*
-	 * 352x288 30fps  VBlanking 1lane 10Bit (binning)
-	 */
-	static struct ov2680_reg const ov2680_CIF_30fps[] = {
-	{OV2680_8BIT, 0x3086, 0x01},
-	{OV2680_8BIT, 0x3501, 0x24},
-	{OV2680_8BIT, 0x3502, 0x40},
-	{OV2680_8BIT, 0x370a, 0x23},
-	{OV2680_8BIT, 0x3801, 0xa0},
-	{OV2680_8BIT, 0x3802, 0x00},
-	{OV2680_8BIT, 0x3803, 0x78},
-	{OV2680_8BIT, 0x3804, 0x03},
-	{OV2680_8BIT, 0x3805, 0x8f},
-	{OV2680_8BIT, 0x3806, 0x02},
-	{OV2680_8BIT, 0x3807, 0xe7},
-	{OV2680_8BIT, 0x3808, 0x01},
-	{OV2680_8BIT, 0x3809, 0x70},
-	{OV2680_8BIT, 0x380a, 0x01},
-	{OV2680_8BIT, 0x380b, 0x30},
-	{OV2680_8BIT, 0x380c, 0x06},
-	{OV2680_8BIT, 0x380d, 0xb0},
-	{OV2680_8BIT, 0x380e, 0x02},
-	{OV2680_8BIT, 0x380f, 0x84},
-	{OV2680_8BIT, 0x3810, 0x00},
-	{OV2680_8BIT, 0x3811, 0x04},
-	{OV2680_8BIT, 0x3812, 0x00},
-	{OV2680_8BIT, 0x3813, 0x04},
-	{OV2680_8BIT, 0x3814, 0x31},
-	{OV2680_8BIT, 0x3815, 0x31},
-	{OV2680_8BIT, 0x4008, 0x00},
-	{OV2680_8BIT, 0x4009, 0x03},
-	{OV2680_8BIT, 0x5081, 0x41},
-	{OV2680_8BIT, 0x5708, 0x00}, //add for full size flip off and mirror off 2014/09/11
-	{OV2680_8BIT, 0x5704, 0x10},
-	{OV2680_8BIT, 0x5705, 0xa0},
-	{OV2680_8BIT, 0x5706, 0x0c},
-	{OV2680_8BIT, 0x5707, 0x78},
-	{OV2680_8BIT, 0x3820, 0xc2},
-	{OV2680_8BIT, 0x3821, 0x01},
-    // {OV2680_8BIT, 0x5090, 0x0c},
-    {OV2680_TOK_TERM, 0, 0}
-	};
-
-	/*
-	 * 336x256 30fps  VBlanking 1lane 10Bit (binning)
-	 */
-	static struct ov2680_reg const ov2680_QVGA_30fps[] = {
-	{OV2680_8BIT, 0x3086, 0x01},
-	{OV2680_8BIT, 0x3501, 0x24},
-	{OV2680_8BIT, 0x3502, 0x40},
-	{OV2680_8BIT, 0x370a, 0x23},
-	{OV2680_8BIT, 0x3801, 0xa0},
-	{OV2680_8BIT, 0x3802, 0x00},
-	{OV2680_8BIT, 0x3803, 0x78},
-	{OV2680_8BIT, 0x3804, 0x03},
-	{OV2680_8BIT, 0x3805, 0x4f},
-	{OV2680_8BIT, 0x3806, 0x02},
-	{OV2680_8BIT, 0x3807, 0x87},
-	{OV2680_8BIT, 0x3808, 0x01},
-	{OV2680_8BIT, 0x3809, 0x50},
-	{OV2680_8BIT, 0x380a, 0x01},
-	{OV2680_8BIT, 0x380b, 0x00},
-	{OV2680_8BIT, 0x380c, 0x06},
-	{OV2680_8BIT, 0x380d, 0xb0},
-	{OV2680_8BIT, 0x380e, 0x02},
-	{OV2680_8BIT, 0x380f, 0x84},
-	{OV2680_8BIT, 0x3810, 0x00},
-	{OV2680_8BIT, 0x3811, 0x04},
-	{OV2680_8BIT, 0x3812, 0x00},
-	{OV2680_8BIT, 0x3813, 0x04},
-	{OV2680_8BIT, 0x3814, 0x31},
-	{OV2680_8BIT, 0x3815, 0x31},
-	{OV2680_8BIT, 0x4008, 0x00},
-	{OV2680_8BIT, 0x4009, 0x03},
-	{OV2680_8BIT, 0x5081, 0x41},
-	{OV2680_8BIT, 0x5708, 0x00}, //add for full size flip off and mirror off 2014/09/11
-	{OV2680_8BIT, 0x5704, 0x10},
-	{OV2680_8BIT, 0x5705, 0xa0},
-	{OV2680_8BIT, 0x5706, 0x0c},
-	{OV2680_8BIT, 0x5707, 0x78},
-	{OV2680_8BIT, 0x3820, 0xc2},
-	{OV2680_8BIT, 0x3821, 0x01},
-    // {OV2680_8BIT, 0x5090, 0x0c},
-    {OV2680_TOK_TERM, 0, 0}
-	};
-
-
-	/*
-	 * 656x496 30fps  VBlanking 1lane 10Bit (binning)
-	 */
-	static struct ov2680_reg const ov2680_656x496_30fps[] = {
-	{OV2680_8BIT, 0x3086, 0x01},
-	{OV2680_8BIT, 0x3501, 0x24},
-	{OV2680_8BIT, 0x3502, 0x40},
-	{OV2680_8BIT, 0x370a, 0x23},
-	{OV2680_8BIT, 0x3801, 0xa0},
-	{OV2680_8BIT, 0x3802, 0x00},
-	{OV2680_8BIT, 0x3803, 0x78},
-	{OV2680_8BIT, 0x3804, 0x05},
-	{OV2680_8BIT, 0x3805, 0xcf},
-	{OV2680_8BIT, 0x3806, 0x04},
-	{OV2680_8BIT, 0x3807, 0x67},
-	{OV2680_8BIT, 0x3808, 0x02},
-	{OV2680_8BIT, 0x3809, 0x90},
-	{OV2680_8BIT, 0x380a, 0x01},
-	{OV2680_8BIT, 0x380b, 0xf0},
-	{OV2680_8BIT, 0x380c, 0x06},
-	{OV2680_8BIT, 0x380d, 0xb0},
-	{OV2680_8BIT, 0x380e, 0x02},
-	{OV2680_8BIT, 0x380f, 0x84},
-	{OV2680_8BIT, 0x3810, 0x00},
-	{OV2680_8BIT, 0x3811, 0x04},
-	{OV2680_8BIT, 0x3812, 0x00},
-	{OV2680_8BIT, 0x3813, 0x04},
-	{OV2680_8BIT, 0x3814, 0x31},
-	{OV2680_8BIT, 0x3815, 0x31},
-	{OV2680_8BIT, 0x4008, 0x00},
-	{OV2680_8BIT, 0x4009, 0x03},
-	{OV2680_8BIT, 0x5081, 0x41},
-	{OV2680_8BIT, 0x5708, 0x00}, //add for full size flip off and mirror off 2014/09/11
-	{OV2680_8BIT, 0x5704, 0x10},
-	{OV2680_8BIT, 0x5705, 0xa0},
-	{OV2680_8BIT, 0x5706, 0x0c},
-	{OV2680_8BIT, 0x5707, 0x78},
-	{OV2680_8BIT, 0x3820, 0xc2},
-	{OV2680_8BIT, 0x3821, 0x01},
-    // {OV2680_8BIT, 0x5090, 0x0c},
-    {OV2680_TOK_TERM, 0, 0}
-	};
-	/*
-	* 800x600 30fps  VBlanking 1lane 10Bit (binning)
-	*/
-	static struct ov2680_reg const ov2680_720x592_30fps[] = {
-	{OV2680_8BIT, 0x3086, 0x01},
-	{OV2680_8BIT, 0x3501, 0x26},
-	{OV2680_8BIT, 0x3502, 0x40},
-	{OV2680_8BIT, 0x370a, 0x23},
-	{OV2680_8BIT, 0x3801, 0x00}, // X_ADDR_START;
-	{OV2680_8BIT, 0x3802, 0x00},
-	{OV2680_8BIT, 0x3803, 0x00}, // Y_ADDR_START;
-	{OV2680_8BIT, 0x3804, 0x05},
-	{OV2680_8BIT, 0x3805, 0xaf}, // X_ADDR_END;
-	{OV2680_8BIT, 0x3806, 0x04},
-	{OV2680_8BIT, 0x3807, 0xaf}, // Y_ADDR_END;
-	{OV2680_8BIT, 0x3808, 0x02},
-	{OV2680_8BIT, 0x3809, 0xd0}, // X_OUTPUT_SIZE;
-	{OV2680_8BIT, 0x380a, 0x02},
-	{OV2680_8BIT, 0x380b, 0x50}, // Y_OUTPUT_SIZE;
-	{OV2680_8BIT, 0x380c, 0x06},
-	{OV2680_8BIT, 0x380d, 0xac}, // HTS;
-	{OV2680_8BIT, 0x380e, 0x02},
-	{OV2680_8BIT, 0x380f, 0x84}, // VTS;
-	{OV2680_8BIT, 0x3810, 0x00},
-	{OV2680_8BIT, 0x3811, 0x00},
-	{OV2680_8BIT, 0x3812, 0x00},
-	{OV2680_8BIT, 0x3813, 0x00},
-	{OV2680_8BIT, 0x3814, 0x31},
-	{OV2680_8BIT, 0x3815, 0x31},
-	{OV2680_8BIT, 0x4008, 0x00},
-	{OV2680_8BIT, 0x4009, 0x03},
-	{OV2680_8BIT, 0x5708, 0x00},
-	{OV2680_8BIT, 0x5704, 0x02},
-	{OV2680_8BIT, 0x5705, 0xd0}, // X_WIN;
-	{OV2680_8BIT, 0x5706, 0x02},
-	{OV2680_8BIT, 0x5707, 0x50}, // Y_WIN;
-	{OV2680_8BIT, 0x3820, 0xc2}, // FLIP_FORMAT;
-	{OV2680_8BIT, 0x3821, 0x01}, // MIRROR_FORMAT;
-    {OV2680_8BIT, 0x5090, 0x00}, // PRE ISP CTRL16, default value is 0x0C;
-                                 // BIT[3]: Mirror order, BG or GB;
-                                 // BIT[2]: Flip order, BR or RB;
-	{OV2680_8BIT, 0x5081, 0x41},
-    {OV2680_TOK_TERM, 0, 0}
-	};
-
-	/*
-	* 800x600 30fps  VBlanking 1lane 10Bit (binning)
-	*/
-	static struct ov2680_reg const ov2680_800x600_30fps[] = {
-	{OV2680_8BIT, 0x3086, 0x01},
-	{OV2680_8BIT, 0x3501, 0x26},
-	{OV2680_8BIT, 0x3502, 0x40},
-	{OV2680_8BIT, 0x370a, 0x23},
-	{OV2680_8BIT, 0x3801, 0x00},
-	{OV2680_8BIT, 0x3802, 0x00},
-	{OV2680_8BIT, 0x3803, 0x00},
-	{OV2680_8BIT, 0x3804, 0x06},
-	{OV2680_8BIT, 0x3805, 0x4f},
-	{OV2680_8BIT, 0x3806, 0x04},
-	{OV2680_8BIT, 0x3807, 0xbf},
-	{OV2680_8BIT, 0x3808, 0x03},
-	{OV2680_8BIT, 0x3809, 0x20},
-	{OV2680_8BIT, 0x380a, 0x02},
-	{OV2680_8BIT, 0x380b, 0x58},
-	{OV2680_8BIT, 0x380c, 0x06},
-	{OV2680_8BIT, 0x380d, 0xac},
-	{OV2680_8BIT, 0x380e, 0x02},
-	{OV2680_8BIT, 0x380f, 0x84},
-	{OV2680_8BIT, 0x3810, 0x00},
-	{OV2680_8BIT, 0x3811, 0x00},
-	{OV2680_8BIT, 0x3812, 0x00},
-	{OV2680_8BIT, 0x3813, 0x00},
-	{OV2680_8BIT, 0x3814, 0x31},
-	{OV2680_8BIT, 0x3815, 0x31},
-	{OV2680_8BIT, 0x5708, 0x00},
-	{OV2680_8BIT, 0x5704, 0x03},
-	{OV2680_8BIT, 0x5705, 0x20},
-	{OV2680_8BIT, 0x5706, 0x02},
-	{OV2680_8BIT, 0x5707, 0x58},
-	{OV2680_8BIT, 0x3820, 0xc2},
-	{OV2680_8BIT, 0x3821, 0x01},
-    {OV2680_8BIT, 0x5090, 0x00},
-	{OV2680_8BIT, 0x4008, 0x00},
-	{OV2680_8BIT, 0x4009, 0x03},
-	{OV2680_8BIT, 0x5081, 0x41},
-    {OV2680_TOK_TERM, 0, 0}
-	};
-
-	/*
-	 * 720p=1280*720 30fps  VBlanking 1lane 10Bit (no-Scaling)
-	 */
-	static struct ov2680_reg const ov2680_720p_30fps[] = {
-	{OV2680_8BIT, 0x3086, 0x00},
-	{OV2680_8BIT, 0x3501, 0x48},
-	{OV2680_8BIT, 0x3502, 0xe0},
-	{OV2680_8BIT, 0x370a, 0x21},
-	{OV2680_8BIT, 0x3801, 0xa0},
-	{OV2680_8BIT, 0x3802, 0x00},
-	{OV2680_8BIT, 0x3803, 0xf2},
-	{OV2680_8BIT, 0x3804, 0x05},
-	{OV2680_8BIT, 0x3805, 0xbf},
-	{OV2680_8BIT, 0x3806, 0x03},
-	{OV2680_8BIT, 0x3807, 0xdd},
-	{OV2680_8BIT, 0x3808, 0x05},
-	{OV2680_8BIT, 0x3809, 0x10},
-	{OV2680_8BIT, 0x380a, 0x02},
-	{OV2680_8BIT, 0x380b, 0xe0},
-	{OV2680_8BIT, 0x380c, 0x06},
-	{OV2680_8BIT, 0x380d, 0xa8},
-	{OV2680_8BIT, 0x380e, 0x05},
-	{OV2680_8BIT, 0x380f, 0x0e},
-	{OV2680_8BIT, 0x3810, 0x00},
-	{OV2680_8BIT, 0x3811, 0x08},
-	{OV2680_8BIT, 0x3812, 0x00},
-	{OV2680_8BIT, 0x3813, 0x06},
-	{OV2680_8BIT, 0x3814, 0x11},
-	{OV2680_8BIT, 0x3815, 0x11},
-	{OV2680_8BIT, 0x4008, 0x02},
-	{OV2680_8BIT, 0x4009, 0x09},
-	{OV2680_8BIT, 0x5081, 0x41},
-	{OV2680_8BIT, 0x5708, 0x00}, //add for full size flip off and mirror off 2014/09/11
-	{OV2680_8BIT, 0x5704, 0x10},
-	{OV2680_8BIT, 0x5705, 0xa0},
-	{OV2680_8BIT, 0x5706, 0x0c},
-	{OV2680_8BIT, 0x5707, 0x78},
-	{OV2680_8BIT, 0x3820, 0xc0},
-	{OV2680_8BIT, 0x3821, 0x00},
-    // {OV2680_8BIT, 0x5090, 0x0c},
-    {OV2680_TOK_TERM, 0, 0}
-	};
-
-	/*
-	 * 1296x976 30fps  VBlanking 1lane 10Bit(no-scaling)
-	 */
-	static struct ov2680_reg const ov2680_1296x976_30fps[] = {
-	{OV2680_8BIT, 0x3086, 0x00},
-	{OV2680_8BIT, 0x3501, 0x48},
-	{OV2680_8BIT, 0x3502, 0xe0},
-	{OV2680_8BIT, 0x370a, 0x21},
-	{OV2680_8BIT, 0x3801, 0xa0},
-	{OV2680_8BIT, 0x3802, 0x00},
-	{OV2680_8BIT, 0x3803, 0x78},
-	{OV2680_8BIT, 0x3804, 0x05},
-	{OV2680_8BIT, 0x3805, 0xbf},
-	{OV2680_8BIT, 0x3806, 0x04},
-	{OV2680_8BIT, 0x3807, 0x57},
-	{OV2680_8BIT, 0x3808, 0x05},
-	{OV2680_8BIT, 0x3809, 0x10},
-	{OV2680_8BIT, 0x380a, 0x03},
-	{OV2680_8BIT, 0x380b, 0xd0},
-	{OV2680_8BIT, 0x380c, 0x06},
-	{OV2680_8BIT, 0x380d, 0xa8},
-	{OV2680_8BIT, 0x380e, 0x05},
-	{OV2680_8BIT, 0x380f, 0x0e},
-	{OV2680_8BIT, 0x3810, 0x00},
-	{OV2680_8BIT, 0x3811, 0x08},
-	{OV2680_8BIT, 0x3812, 0x00},
-	{OV2680_8BIT, 0x3813, 0x08},
-	{OV2680_8BIT, 0x3814, 0x11},
-	{OV2680_8BIT, 0x3815, 0x11},
-	{OV2680_8BIT, 0x4008, 0x02},
-	{OV2680_8BIT, 0x4009, 0x09},
-	{OV2680_8BIT, 0x5081, 0x41},
-	{OV2680_8BIT, 0x5708, 0x00}, //add for full size flip off and mirror off 2014/09/11
-	{OV2680_8BIT, 0x5704, 0x10},
-	{OV2680_8BIT, 0x5705, 0xa0},
-	{OV2680_8BIT, 0x5706, 0x0c},
-	{OV2680_8BIT, 0x5707, 0x78},
-	{OV2680_8BIT, 0x3820, 0xc0},
-	{OV2680_8BIT, 0x3821, 0x00}, //miror/flip
-    // {OV2680_8BIT, 0x5090, 0x0c},
-    {OV2680_TOK_TERM, 0, 0}
- 	};
-
-	/*
-	 *   1456*1096 30fps  VBlanking 1lane 10bit(no-scaling)
-	*/
-	static struct ov2680_reg const ov2680_1456x1096_30fps[]= {
-	{OV2680_8BIT, 0x3086, 0x00},
-	{OV2680_8BIT, 0x3501, 0x48},
-	{OV2680_8BIT, 0x3502, 0xe0},
-	{OV2680_8BIT, 0x370a, 0x21},
-	{OV2680_8BIT, 0x3801, 0x90},
-	{OV2680_8BIT, 0x3802, 0x00},
-	{OV2680_8BIT, 0x3803, 0x78},
-	{OV2680_8BIT, 0x3804, 0x06},
-	{OV2680_8BIT, 0x3805, 0x4f},
-	{OV2680_8BIT, 0x3806, 0x04},
-	{OV2680_8BIT, 0x3807, 0xC0},
-	{OV2680_8BIT, 0x3808, 0x05},
-	{OV2680_8BIT, 0x3809, 0xb0},
-	{OV2680_8BIT, 0x380a, 0x04},
-	{OV2680_8BIT, 0x380b, 0x48},
-	{OV2680_8BIT, 0x380c, 0x06},
-	{OV2680_8BIT, 0x380d, 0xa8},
-	{OV2680_8BIT, 0x380e, 0x05},
-	{OV2680_8BIT, 0x380f, 0x0e},
-	{OV2680_8BIT, 0x3810, 0x00},
-	{OV2680_8BIT, 0x3811, 0x08},
-	{OV2680_8BIT, 0x3812, 0x00},
-	{OV2680_8BIT, 0x3813, 0x00},
-	{OV2680_8BIT, 0x3814, 0x11},
-	{OV2680_8BIT, 0x3815, 0x11},
-	{OV2680_8BIT, 0x4008, 0x02},
-	{OV2680_8BIT, 0x4009, 0x09},
-	{OV2680_8BIT, 0x5081, 0x41},
-	{OV2680_8BIT, 0x5708, 0x00}, //add for full size flip off and mirror off 2014/09/11
-	{OV2680_8BIT, 0x5704, 0x10},
-	{OV2680_8BIT, 0x5705, 0xa0},
-	{OV2680_8BIT, 0x5706, 0x0c},
-	{OV2680_8BIT, 0x5707, 0x78},
-	{OV2680_8BIT, 0x3820, 0xc0},
-	{OV2680_8BIT, 0x3821, 0x00},
-    // {OV2680_8BIT, 0x5090, 0x0c},
-	{OV2680_TOK_TERM, 0, 0}
-	};
-
-	/*
-	 *1616x916  30fps  VBlanking 1lane 10bit
-	 */
-
-	static struct ov2680_reg const ov2680_1616x916_30fps[] = {
-
-	{OV2680_8BIT, 0x3086, 0x00},
-	{OV2680_8BIT, 0x3501, 0x48},
-	{OV2680_8BIT, 0x3502, 0xe0},
-	{OV2680_8BIT, 0x370a, 0x21},
-	{OV2680_8BIT, 0x3801, 0x00},
-	{OV2680_8BIT, 0x3802, 0x00},
-	{OV2680_8BIT, 0x3803, 0x96},
-	{OV2680_8BIT, 0x3804, 0x06},
-	{OV2680_8BIT, 0x3805, 0x4f},
-	{OV2680_8BIT, 0x3806, 0x04},
-	{OV2680_8BIT, 0x3807, 0x39},
-	{OV2680_8BIT, 0x3808, 0x06},
-	{OV2680_8BIT, 0x3809, 0x50},
-	{OV2680_8BIT, 0x380a, 0x03},
-	{OV2680_8BIT, 0x380b, 0x94},
-	{OV2680_8BIT, 0x380c, 0x06},
-	{OV2680_8BIT, 0x380d, 0xa8},
-	{OV2680_8BIT, 0x380e, 0x05},
-	{OV2680_8BIT, 0x380f, 0x0e},
-	{OV2680_8BIT, 0x3810, 0x00},
-	{OV2680_8BIT, 0x3811, 0x00},
-	{OV2680_8BIT, 0x3812, 0x00},
-	{OV2680_8BIT, 0x3813, 0x08},
-	{OV2680_8BIT, 0x3814, 0x11},
-	{OV2680_8BIT, 0x3815, 0x11},
-	{OV2680_8BIT, 0x4008, 0x02},
-	{OV2680_8BIT, 0x4009, 0x09},
-	{OV2680_8BIT, 0x5081, 0x41},
-	{OV2680_8BIT, 0x5708, 0x01}, //add for full size flip off and mirror off 2014/09/11
-	{OV2680_8BIT, 0x5704, 0x06},
-	{OV2680_8BIT, 0x5705, 0x50},
-	{OV2680_8BIT, 0x5706, 0x03},
-	{OV2680_8BIT, 0x5707, 0x94},
-	{OV2680_8BIT, 0x3820, 0xc0},
-	{OV2680_8BIT, 0x3821, 0x00},
-    // {OV2680_8BIT, 0x5090, 0x0C},
-    {OV2680_TOK_TERM, 0, 0}
-	};
-
-	/*
-	 * 1612x1212 30fps VBlanking 1lane 10Bit
-	 */
-	static struct ov2680_reg const ov2680_1616x1082_30fps[] = {
-       {OV2680_8BIT, 0x3086, 0x00},
-       {OV2680_8BIT, 0x3501, 0x48},
-       {OV2680_8BIT, 0x3502, 0xe0},
-       {OV2680_8BIT, 0x370a, 0x21},
-       {OV2680_8BIT, 0x3801, 0x00},
-       {OV2680_8BIT, 0x3802, 0x00},
-       {OV2680_8BIT, 0x3803, 0x86},
-       {OV2680_8BIT, 0x3804, 0x06},
-       {OV2680_8BIT, 0x3805, 0x4f},
-       {OV2680_8BIT, 0x3806, 0x04},
-       {OV2680_8BIT, 0x3807, 0xbf},
-       {OV2680_8BIT, 0x3808, 0x06},
-       {OV2680_8BIT, 0x3809, 0x50},
-       {OV2680_8BIT, 0x380a, 0x04},
-       {OV2680_8BIT, 0x380b, 0x3a},
-       {OV2680_8BIT, 0x380c, 0x06},
-       {OV2680_8BIT, 0x380d, 0xa8},
-       {OV2680_8BIT, 0x380e, 0x05},
-       {OV2680_8BIT, 0x380f, 0x0e},
-       {OV2680_8BIT, 0x3810, 0x00},
-       {OV2680_8BIT, 0x3811, 0x00},
-       {OV2680_8BIT, 0x3812, 0x00},
-       {OV2680_8BIT, 0x3813, 0x00},
-       {OV2680_8BIT, 0x3814, 0x11},
-       {OV2680_8BIT, 0x3815, 0x11},
-       {OV2680_8BIT, 0x5708, 0x01}, //add for full size flip off and mirror off 2014/09/11
-       {OV2680_8BIT, 0x5704, 0x06},
-       {OV2680_8BIT, 0x5705, 0x50},
-       {OV2680_8BIT, 0x5706, 0x04},
-       {OV2680_8BIT, 0x5707, 0x3a},
-       {OV2680_8BIT, 0x3820, 0xc0},
-       {OV2680_8BIT, 0x3821, 0x00},
-       // {OV2680_8BIT, 0x5090, 0x0C},
-       {OV2680_8BIT, 0x4008, 0x02},
-       {OV2680_8BIT, 0x4009, 0x09},
-       {OV2680_8BIT, 0x5081, 0x41},
-		{OV2680_TOK_TERM, 0, 0}
-        };
-	/*
-	 * 1616x1216 30fps VBlanking 1lane 10Bit
-	 */
-	static struct ov2680_reg const ov2680_1616x1216_30fps[] = {
-	{OV2680_8BIT, 0x3086, 0x00},
-	{OV2680_8BIT, 0x3501, 0x48},
-	{OV2680_8BIT, 0x3502, 0xe0},
-	{OV2680_8BIT, 0x370a, 0x21},
-	{OV2680_8BIT, 0x3801, 0x00},
-	{OV2680_8BIT, 0x3802, 0x00},
-	{OV2680_8BIT, 0x3803, 0x00},
-	{OV2680_8BIT, 0x3804, 0x06},
-	{OV2680_8BIT, 0x3805, 0x4f},
-	{OV2680_8BIT, 0x3806, 0x04},
-	{OV2680_8BIT, 0x3807, 0xbf},
-	{OV2680_8BIT, 0x3808, 0x06},
-	{OV2680_8BIT, 0x3809, 0x50},//50},//4line for mirror and flip
-	{OV2680_8BIT, 0x380a, 0x04},
-	{OV2680_8BIT, 0x380b, 0xc0},//c0},
-	{OV2680_8BIT, 0x380c, 0x06},
-	{OV2680_8BIT, 0x380d, 0xa8},
-	{OV2680_8BIT, 0x380e, 0x05},
-	{OV2680_8BIT, 0x380f, 0x0e},
-	{OV2680_8BIT, 0x3810, 0x00},
-	{OV2680_8BIT, 0x3811, 0x00},
-	{OV2680_8BIT, 0x3812, 0x00},
-	{OV2680_8BIT, 0x3813, 0x00},
-	{OV2680_8BIT, 0x3814, 0x11},
-	{OV2680_8BIT, 0x3815, 0x11},
-	{OV2680_8BIT, 0x4008, 0x00},
-	{OV2680_8BIT, 0x4009, 0x0b},
-	{OV2680_8BIT, 0x5081, 0x01},
-	{OV2680_8BIT, 0x5708, 0x01}, //add for full size flip off and mirror off 2014/09/11
-	{OV2680_8BIT, 0x5704, 0x06},
-	{OV2680_8BIT, 0x5705, 0x50},
-	{OV2680_8BIT, 0x5706, 0x04},
-	{OV2680_8BIT, 0x5707, 0xcc},
-	{OV2680_8BIT, 0x3820, 0xc0},
-	{OV2680_8BIT, 0x3821, 0x00},
-    // {OV2680_8BIT, 0x5090, 0x0C},
-    {OV2680_TOK_TERM, 0, 0}
-	};
-
-	static struct ov2680_resolution ov2680_res_preview[] = {
-	{
-		.desc = "ov2680_1616x1216_30fps",
- 	  	.width = 1616,
-		.height = 1216,
-		.pix_clk_freq = 66,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 1698,//1704,
-		.lines_per_frame = 1294,
-		.bin_factor_x = 0,
-		.bin_factor_y = 0,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = ov2680_1616x1216_30fps,
-	},
-   	{
-		.desc = "ov2680_1616x916_30fps",
-		.width = 1616,
-		.height = 916,
-		.fps = 30,
-		.pix_clk_freq = 66,
-		.used = 0,
-		.pixels_per_line = 1698,//1704,
-		.lines_per_frame = 1294,
-		.bin_factor_x = 0,
-		.bin_factor_y = 0,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = ov2680_1616x916_30fps,
-	},
-};
-#define N_RES_PREVIEW (ARRAY_SIZE(ov2680_res_preview))
-
-static struct ov2680_resolution *ov2680_res = ov2680_res_preview;
-static unsigned long N_RES = N_RES_PREVIEW;
-
-#endif
diff --git a/drivers/staging/media/atomisp/i2c/ov2722.h b/drivers/staging/media/atomisp/i2c/ov2722.h
deleted file mode 100644
index 757b37613ccc..000000000000
--- a/drivers/staging/media/atomisp/i2c/ov2722.h
+++ /dev/null
@@ -1,1262 +0,0 @@
-/*
- * Support for OmniVision OV2722 1080p HD camera sensor.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __OV2722_H__
-#define __OV2722_H__
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/i2c.h>
-#include <linux/delay.h>
-#include <linux/videodev2.h>
-#include <linux/spinlock.h>
-#include <media/v4l2-subdev.h>
-#include <media/v4l2-device.h>
-#include <linux/v4l2-mediabus.h>
-#include <media/media-entity.h>
-#include <media/v4l2-ctrls.h>
-
-#include "../include/linux/atomisp_platform.h"
-
-#define OV2722_POWER_UP_RETRY_NUM 5
-
-/* Defines for register writes and register array processing */
-#define I2C_MSG_LENGTH		0x2
-#define I2C_RETRY_COUNT		5
-
-#define OV2722_FOCAL_LENGTH_NUM	278	/*2.78mm*/
-#define OV2722_FOCAL_LENGTH_DEM	100
-#define OV2722_F_NUMBER_DEFAULT_NUM	26
-#define OV2722_F_NUMBER_DEM	10
-
-#define MAX_FMTS		1
-
-/*
- * focal length bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define OV2722_FOCAL_LENGTH_DEFAULT 0x1160064
-
-/*
- * current f-number bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define OV2722_F_NUMBER_DEFAULT 0x1a000a
-
-/*
- * f-number range bits definition:
- * bits 31-24: max f-number numerator
- * bits 23-16: max f-number denominator
- * bits 15-8: min f-number numerator
- * bits 7-0: min f-number denominator
- */
-#define OV2722_F_NUMBER_RANGE 0x1a0a1a0a
-#define OV2720_ID	0x2720
-#define OV2722_ID	0x2722
-
-#define OV2722_FINE_INTG_TIME_MIN 0
-#define OV2722_FINE_INTG_TIME_MAX_MARGIN 0
-#define OV2722_COARSE_INTG_TIME_MIN 1
-#define OV2722_COARSE_INTG_TIME_MAX_MARGIN 4
-
-/*
- * OV2722 System control registers
- */
-#define OV2722_SW_SLEEP				0x0100
-#define OV2722_SW_RESET				0x0103
-#define OV2722_SW_STREAM			0x0100
-
-#define OV2722_SC_CMMN_CHIP_ID_H		0x300A
-#define OV2722_SC_CMMN_CHIP_ID_L		0x300B
-#define OV2722_SC_CMMN_SCCB_ID			0x300C
-#define OV2722_SC_CMMN_SUB_ID			0x302A /* process, version*/
-
-#define OV2722_SC_CMMN_PAD_OEN0			0x3000
-#define OV2722_SC_CMMN_PAD_OEN1			0x3001
-#define OV2722_SC_CMMN_PAD_OEN2			0x3002
-#define OV2722_SC_CMMN_PAD_OUT0			0x3008
-#define OV2722_SC_CMMN_PAD_OUT1			0x3009
-#define OV2722_SC_CMMN_PAD_OUT2			0x300D
-#define OV2722_SC_CMMN_PAD_SEL0			0x300E
-#define OV2722_SC_CMMN_PAD_SEL1			0x300F
-#define OV2722_SC_CMMN_PAD_SEL2			0x3010
-
-#define OV2722_SC_CMMN_PAD_PK			0x3011
-#define OV2722_SC_CMMN_A_PWC_PK_O_13		0x3013
-#define OV2722_SC_CMMN_A_PWC_PK_O_14		0x3014
-
-#define OV2722_SC_CMMN_CLKRST0			0x301A
-#define OV2722_SC_CMMN_CLKRST1			0x301B
-#define OV2722_SC_CMMN_CLKRST2			0x301C
-#define OV2722_SC_CMMN_CLKRST3			0x301D
-#define OV2722_SC_CMMN_CLKRST4			0x301E
-#define OV2722_SC_CMMN_CLKRST5			0x3005
-#define OV2722_SC_CMMN_PCLK_DIV_CTRL		0x3007
-#define OV2722_SC_CMMN_CLOCK_SEL		0x3020
-#define OV2722_SC_SOC_CLKRST5			0x3040
-
-#define OV2722_SC_CMMN_PLL_CTRL0		0x3034
-#define OV2722_SC_CMMN_PLL_CTRL1		0x3035
-#define OV2722_SC_CMMN_PLL_CTRL2		0x3039
-#define OV2722_SC_CMMN_PLL_CTRL3		0x3037
-#define OV2722_SC_CMMN_PLL_MULTIPLIER		0x3036
-#define OV2722_SC_CMMN_PLL_DEBUG_OPT		0x3038
-#define OV2722_SC_CMMN_PLLS_CTRL0		0x303A
-#define OV2722_SC_CMMN_PLLS_CTRL1		0x303B
-#define OV2722_SC_CMMN_PLLS_CTRL2		0x303C
-#define OV2722_SC_CMMN_PLLS_CTRL3		0x303D
-
-#define OV2722_SC_CMMN_MIPI_PHY_16		0x3016
-#define OV2722_SC_CMMN_MIPI_PHY_17		0x3017
-#define OV2722_SC_CMMN_MIPI_SC_CTRL_18		0x3018
-#define OV2722_SC_CMMN_MIPI_SC_CTRL_19		0x3019
-#define OV2722_SC_CMMN_MIPI_SC_CTRL_21		0x3021
-#define OV2722_SC_CMMN_MIPI_SC_CTRL_22		0x3022
-
-#define OV2722_AEC_PK_EXPO_H			0x3500
-#define OV2722_AEC_PK_EXPO_M			0x3501
-#define OV2722_AEC_PK_EXPO_L			0x3502
-#define OV2722_AEC_MANUAL_CTRL			0x3503
-#define OV2722_AGC_ADJ_H			0x3508
-#define OV2722_AGC_ADJ_L			0x3509
-#define OV2722_VTS_DIFF_H			0x350c
-#define OV2722_VTS_DIFF_L			0x350d
-#define OV2722_GROUP_ACCESS			0x3208
-#define OV2722_HTS_H				0x380c
-#define OV2722_HTS_L				0x380d
-#define OV2722_VTS_H				0x380e
-#define OV2722_VTS_L				0x380f
-
-#define OV2722_MWB_GAIN_R_H			0x5186
-#define OV2722_MWB_GAIN_R_L			0x5187
-#define OV2722_MWB_GAIN_G_H			0x5188
-#define OV2722_MWB_GAIN_G_L			0x5189
-#define OV2722_MWB_GAIN_B_H			0x518a
-#define OV2722_MWB_GAIN_B_L			0x518b
-
-#define OV2722_H_CROP_START_H			0x3800
-#define OV2722_H_CROP_START_L			0x3801
-#define OV2722_V_CROP_START_H			0x3802
-#define OV2722_V_CROP_START_L			0x3803
-#define OV2722_H_CROP_END_H			0x3804
-#define OV2722_H_CROP_END_L			0x3805
-#define OV2722_V_CROP_END_H			0x3806
-#define OV2722_V_CROP_END_L			0x3807
-#define OV2722_H_OUTSIZE_H			0x3808
-#define OV2722_H_OUTSIZE_L			0x3809
-#define OV2722_V_OUTSIZE_H			0x380a
-#define OV2722_V_OUTSIZE_L			0x380b
-
-#define OV2722_START_STREAMING			0x01
-#define OV2722_STOP_STREAMING			0x00
-
-struct regval_list {
-	u16 reg_num;
-	u8 value;
-};
-
-struct ov2722_resolution {
-	u8 *desc;
-	const struct ov2722_reg *regs;
-	int res;
-	int width;
-	int height;
-	int fps;
-	int pix_clk_freq;
-	u32 skip_frames;
-	u16 pixels_per_line;
-	u16 lines_per_frame;
-	u8 bin_factor_x;
-	u8 bin_factor_y;
-	u8 bin_mode;
-	bool used;
-	int mipi_freq;
-};
-
-struct ov2722_format {
-	u8 *desc;
-	u32 pixelformat;
-	struct ov2722_reg *regs;
-};
-
-/*
- * ov2722 device structure.
- */
-struct ov2722_device {
-	struct v4l2_subdev sd;
-	struct media_pad pad;
-	struct v4l2_mbus_framefmt format;
-	struct mutex input_lock;
-
-	struct camera_sensor_platform_data *platform_data;
-	int vt_pix_clk_freq_mhz;
-	int fmt_idx;
-	int run_mode;
-	u16 pixels_per_line;
-	u16 lines_per_frame;
-	u8 res;
-	u8 type;
-
-	struct v4l2_ctrl_handler ctrl_handler;
-	struct v4l2_ctrl *link_freq;
-};
-
-enum ov2722_tok_type {
-	OV2722_8BIT  = 0x0001,
-	OV2722_16BIT = 0x0002,
-	OV2722_32BIT = 0x0004,
-	OV2722_TOK_TERM   = 0xf000,	/* terminating token for reg list */
-	OV2722_TOK_DELAY  = 0xfe00,	/* delay token for reg list */
-	OV2722_TOK_MASK = 0xfff0
-};
-
-/**
- * struct ov2722_reg - MI sensor  register format
- * @type: type of the register
- * @reg: 16-bit offset to register
- * @val: 8/16/32-bit register value
- *
- * Define a structure for sensor register initialization values
- */
-struct ov2722_reg {
-	enum ov2722_tok_type type;
-	u16 reg;
-	u32 val;	/* @set value for read/mod/write, @mask */
-};
-
-#define to_ov2722_sensor(x) container_of(x, struct ov2722_device, sd)
-
-#define OV2722_MAX_WRITE_BUF_SIZE	30
-
-struct ov2722_write_buffer {
-	u16 addr;
-	u8 data[OV2722_MAX_WRITE_BUF_SIZE];
-};
-
-struct ov2722_write_ctrl {
-	int index;
-	struct ov2722_write_buffer buffer;
-};
-
-/*
- * Register settings for various resolution
- */
-static struct ov2722_reg const ov2722_QVGA_30fps[] = {
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x0c},
-	{OV2722_8BIT, 0x373a, 0x1c},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x0c},
-	{OV2722_8BIT, 0x3705, 0x06},
-	{OV2722_8BIT, 0x3730, 0x0e},
-	{OV2722_8BIT, 0x3704, 0x1c},
-	{OV2722_8BIT, 0x3f06, 0x00},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0x46},
-	{OV2722_8BIT, 0x371e, 0x00},
-	{OV2722_8BIT, 0x371f, 0x63},
-	{OV2722_8BIT, 0x3708, 0x61},
-	{OV2722_8BIT, 0x3709, 0x12},
-	{OV2722_8BIT, 0x3800, 0x01},
-	{OV2722_8BIT, 0x3801, 0x42}, /* H crop start: 322 */
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0x20}, /* V crop start: 32 */
-	{OV2722_8BIT, 0x3804, 0x06},
-	{OV2722_8BIT, 0x3805, 0x95}, /* H crop end:  1685 */
-	{OV2722_8BIT, 0x3806, 0x04},
-	{OV2722_8BIT, 0x3807, 0x27}, /* V crop end:  1063 */
-	{OV2722_8BIT, 0x3808, 0x01},
-	{OV2722_8BIT, 0x3809, 0x50}, /* H output size: 336 */
-	{OV2722_8BIT, 0x380a, 0x01},
-	{OV2722_8BIT, 0x380b, 0x00}, /* V output size: 256 */
-
-	/* H blank timing */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0x00}, /* H total size: 2048 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xa0}, /* V total size: 1184 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x04}, /* H window offset: 5 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x01}, /* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0xc0},
-	{OV2722_8BIT, 0x3821, 0x06}, /* flip isp*/
-	{OV2722_8BIT, 0x3814, 0x71},
-	{OV2722_8BIT, 0x3815, 0x71},
-	{OV2722_8BIT, 0x3612, 0x49},
-	{OV2722_8BIT, 0x3618, 0x00},
-	{OV2722_8BIT, 0x3a08, 0x01},
-	{OV2722_8BIT, 0x3a09, 0xc3},
-	{OV2722_8BIT, 0x3a0a, 0x01},
-	{OV2722_8BIT, 0x3a0b, 0x77},
-	{OV2722_8BIT, 0x3a0d, 0x00},
-	{OV2722_8BIT, 0x3a0e, 0x00},
-	{OV2722_8BIT, 0x4520, 0x09},
-	{OV2722_8BIT, 0x4837, 0x1b},
-	{OV2722_8BIT, 0x3000, 0xff},
-	{OV2722_8BIT, 0x3001, 0xff},
-	{OV2722_8BIT, 0x3002, 0xf0},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0x53}, /* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x63},
-	{OV2722_8BIT, 0x3634, 0x24},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1}, /* v_en, h_en, blc_en */
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xff},
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0}, /* AWB red */
-	{OV2722_8BIT, 0x5184, 0xb0}, /* AWB green */
-	{OV2722_8BIT, 0x5185, 0xb0}, /* AWB blue */
-	{OV2722_8BIT, 0x5180, 0x03}, /* AWB manual mode */
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x4800, 0x24}, /* clk lane gate enable */
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x26},
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-
-	/* Added for power optimization */
-	{OV2722_8BIT, 0x3000, 0x00},
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3011, 0x22},
-	{OV2722_8BIT, 0x3a00, 0x58},
-	{OV2722_8BIT, 0x3503, 0x17},
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x46},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x10},
-	{OV2722_TOK_TERM, 0, 0},
-
-};
-
-static struct ov2722_reg const ov2722_480P_30fps[] = {
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x18},
-	{OV2722_8BIT, 0x373a, 0x3c},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x1d},
-	{OV2722_8BIT, 0x3705, 0x12},
-	{OV2722_8BIT, 0x3730, 0x1f},
-	{OV2722_8BIT, 0x3704, 0x3f},
-	{OV2722_8BIT, 0x3f06, 0x1d},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0x83},
-	{OV2722_8BIT, 0x371e, 0x00},
-	{OV2722_8BIT, 0x371f, 0xbd},
-	{OV2722_8BIT, 0x3708, 0x63},
-	{OV2722_8BIT, 0x3709, 0x52},
-	{OV2722_8BIT, 0x3800, 0x00},
-	{OV2722_8BIT, 0x3801, 0xf2}, /* H crop start: 322 - 80 = 242*/
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0x20}, /* V crop start:  32*/
-	{OV2722_8BIT, 0x3804, 0x06},
-	{OV2722_8BIT, 0x3805, 0xBB}, /* H crop end:   1643 + 80 = 1723*/
-	{OV2722_8BIT, 0x3806, 0x04},
-	{OV2722_8BIT, 0x3807, 0x03}, /* V crop end:   1027*/
-	{OV2722_8BIT, 0x3808, 0x02},
-	{OV2722_8BIT, 0x3809, 0xE0}, /* H output size: 656 +80 = 736*/
-	{OV2722_8BIT, 0x380a, 0x01},
-	{OV2722_8BIT, 0x380b, 0xF0}, /* V output size: 496 */
-
-	/* H blank timing */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0x00}, /* H total size: 2048 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xa0}, /* V total size: 1184 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x04}, /* H window offset: 5 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x01}, /* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0x80},
-	{OV2722_8BIT, 0x3821, 0x06}, /* flip isp*/
-	{OV2722_8BIT, 0x3814, 0x31},
-	{OV2722_8BIT, 0x3815, 0x31},
-	{OV2722_8BIT, 0x3612, 0x4b},
-	{OV2722_8BIT, 0x3618, 0x04},
-	{OV2722_8BIT, 0x3a08, 0x02},
-	{OV2722_8BIT, 0x3a09, 0x67},
-	{OV2722_8BIT, 0x3a0a, 0x02},
-	{OV2722_8BIT, 0x3a0b, 0x00},
-	{OV2722_8BIT, 0x3a0d, 0x00},
-	{OV2722_8BIT, 0x3a0e, 0x00},
-	{OV2722_8BIT, 0x4520, 0x0a},
-	{OV2722_8BIT, 0x4837, 0x1b},
-	{OV2722_8BIT, 0x3000, 0xff},
-	{OV2722_8BIT, 0x3001, 0xff},
-	{OV2722_8BIT, 0x3002, 0xf0},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0x53}, /* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x63},
-	{OV2722_8BIT, 0x3634, 0x24},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1}, /* v_en, h_en, blc_en */
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xff},
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0}, /* AWB red */
-	{OV2722_8BIT, 0x5184, 0xb0}, /* AWB green */
-	{OV2722_8BIT, 0x5185, 0xb0}, /* AWB blue */
-	{OV2722_8BIT, 0x5180, 0x03}, /* AWB manual mode */
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x4800, 0x24}, /* clk lane gate enable */
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x26},
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-
-	/* Added for power optimization */
-	{OV2722_8BIT, 0x3000, 0x00},
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3011, 0x22},
-	{OV2722_8BIT, 0x3a00, 0x58},
-	{OV2722_8BIT, 0x3503, 0x17},
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x46},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x10},
-	{OV2722_TOK_TERM, 0, 0},
-};
-
-static struct ov2722_reg const ov2722_VGA_30fps[] = {
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x18},
-	{OV2722_8BIT, 0x373a, 0x3c},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x1d},
-	{OV2722_8BIT, 0x3705, 0x12},
-	{OV2722_8BIT, 0x3730, 0x1f},
-	{OV2722_8BIT, 0x3704, 0x3f},
-	{OV2722_8BIT, 0x3f06, 0x1d},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0x83},
-	{OV2722_8BIT, 0x371e, 0x00},
-	{OV2722_8BIT, 0x371f, 0xbd},
-	{OV2722_8BIT, 0x3708, 0x63},
-	{OV2722_8BIT, 0x3709, 0x52},
-	{OV2722_8BIT, 0x3800, 0x01},
-	{OV2722_8BIT, 0x3801, 0x42}, /* H crop start: 322 */
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0x20}, /* V crop start:  32*/
-	{OV2722_8BIT, 0x3804, 0x06},
-	{OV2722_8BIT, 0x3805, 0x6B}, /* H crop end:   1643*/
-	{OV2722_8BIT, 0x3806, 0x04},
-	{OV2722_8BIT, 0x3807, 0x03}, /* V crop end:   1027*/
-	{OV2722_8BIT, 0x3808, 0x02},
-	{OV2722_8BIT, 0x3809, 0x90}, /* H output size: 656 */
-	{OV2722_8BIT, 0x380a, 0x01},
-	{OV2722_8BIT, 0x380b, 0xF0}, /* V output size: 496 */
-
-	/* H blank timing */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0x00}, /* H total size: 2048 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xa0}, /* V total size: 1184 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x04}, /* H window offset: 5 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x01}, /* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0x80},
-	{OV2722_8BIT, 0x3821, 0x06}, /* flip isp*/
-	{OV2722_8BIT, 0x3814, 0x31},
-	{OV2722_8BIT, 0x3815, 0x31},
-	{OV2722_8BIT, 0x3612, 0x4b},
-	{OV2722_8BIT, 0x3618, 0x04},
-	{OV2722_8BIT, 0x3a08, 0x02},
-	{OV2722_8BIT, 0x3a09, 0x67},
-	{OV2722_8BIT, 0x3a0a, 0x02},
-	{OV2722_8BIT, 0x3a0b, 0x00},
-	{OV2722_8BIT, 0x3a0d, 0x00},
-	{OV2722_8BIT, 0x3a0e, 0x00},
-	{OV2722_8BIT, 0x4520, 0x0a},
-	{OV2722_8BIT, 0x4837, 0x29},
-	{OV2722_8BIT, 0x3000, 0xff},
-	{OV2722_8BIT, 0x3001, 0xff},
-	{OV2722_8BIT, 0x3002, 0xf0},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0x53}, /* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x63},
-	{OV2722_8BIT, 0x3634, 0x24},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1}, /* v_en, h_en, blc_en */
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xff},
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0}, /* AWB red */
-	{OV2722_8BIT, 0x5184, 0xb0}, /* AWB green */
-	{OV2722_8BIT, 0x5185, 0xb0}, /* AWB blue */
-	{OV2722_8BIT, 0x5180, 0x03}, /* AWB manual mode */
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x4800, 0x24}, /* clk lane gate enable */
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x26},
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-
-	/* Added for power optimization */
-	{OV2722_8BIT, 0x3000, 0x00},
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3011, 0x22},
-	{OV2722_8BIT, 0x3a00, 0x58},
-	{OV2722_8BIT, 0x3503, 0x17},
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x46},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x10},
-	{OV2722_TOK_TERM, 0, 0},
-};
-
-static struct ov2722_reg const ov2722_1632_1092_30fps[] = {
-	{OV2722_8BIT, 0x3021, 0x03}, /* For stand wait for
-				a whole frame complete.(vblank) */
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x24},
-	{OV2722_8BIT, 0x373a, 0x60},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x2e},
-	{OV2722_8BIT, 0x3705, 0x10},
-	{OV2722_8BIT, 0x3730, 0x30},
-	{OV2722_8BIT, 0x3704, 0x62},
-	{OV2722_8BIT, 0x3f06, 0x3a},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0xc4},
-	{OV2722_8BIT, 0x371e, 0x01},
-	{OV2722_8BIT, 0x371f, 0x0d},
-	{OV2722_8BIT, 0x3708, 0x61},
-	{OV2722_8BIT, 0x3709, 0x12},
-	{OV2722_8BIT, 0x3800, 0x00},
-	{OV2722_8BIT, 0x3801, 0x9E}, /* H crop start: 158 */
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0x01}, /* V crop start: 1 */
-	{OV2722_8BIT, 0x3804, 0x07},
-	{OV2722_8BIT, 0x3805, 0x05}, /* H crop end: 1797 */
-	{OV2722_8BIT, 0x3806, 0x04},
-	{OV2722_8BIT, 0x3807, 0x45}, /* V crop end: 1093 */
-
-	{OV2722_8BIT, 0x3808, 0x06},
-	{OV2722_8BIT, 0x3809, 0x60}, /* H output size: 1632 */
-	{OV2722_8BIT, 0x380a, 0x04},
-	{OV2722_8BIT, 0x380b, 0x44}, /* V output size: 1092 */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0xd4}, /* H timing: 2260 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xdc}, /* V timing: 1244 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x03}, /* H window offset: 3 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x02}, /* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0x80},
-	{OV2722_8BIT, 0x3821, 0x06}, /*  mirror */
-	{OV2722_8BIT, 0x3814, 0x11},
-	{OV2722_8BIT, 0x3815, 0x11},
-	{OV2722_8BIT, 0x3612, 0x0b},
-	{OV2722_8BIT, 0x3618, 0x04},
-	{OV2722_8BIT, 0x3a08, 0x01},
-	{OV2722_8BIT, 0x3a09, 0x50},
-	{OV2722_8BIT, 0x3a0a, 0x01},
-	{OV2722_8BIT, 0x3a0b, 0x18},
-	{OV2722_8BIT, 0x3a0d, 0x03},
-	{OV2722_8BIT, 0x3a0e, 0x03},
-	{OV2722_8BIT, 0x4520, 0x00},
-	{OV2722_8BIT, 0x4837, 0x1b},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0xd2}, /* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x23},
-	{OV2722_8BIT, 0x3634, 0x54},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1},
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xcf}, /* manual 3a */
-	{OV2722_8BIT, 0x301d, 0xf0}, /* enable group hold */
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0},
-	{OV2722_8BIT, 0x5184, 0xb0},
-	{OV2722_8BIT, 0x5185, 0xb0},
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x2c}, /* 422.4 MHz */
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-	{OV2722_8BIT, 0x3000, 0x00}, /* added for power optimization */
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3503, 0x17}, /* manual 3a */
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x3F},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x00},
-	{OV2722_TOK_TERM, 0, 0}
-};
-
-static struct ov2722_reg const ov2722_1452_1092_30fps[] = {
-	{OV2722_8BIT, 0x3021, 0x03}, /* For stand wait for
-				a whole frame complete.(vblank) */
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x24},
-	{OV2722_8BIT, 0x373a, 0x60},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x2e},
-	{OV2722_8BIT, 0x3705, 0x10},
-	{OV2722_8BIT, 0x3730, 0x30},
-	{OV2722_8BIT, 0x3704, 0x62},
-	{OV2722_8BIT, 0x3f06, 0x3a},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0xc4},
-	{OV2722_8BIT, 0x371e, 0x01},
-	{OV2722_8BIT, 0x371f, 0x0d},
-	{OV2722_8BIT, 0x3708, 0x61},
-	{OV2722_8BIT, 0x3709, 0x12},
-	{OV2722_8BIT, 0x3800, 0x00},
-	{OV2722_8BIT, 0x3801, 0xF8}, /* H crop start: 248 */
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0x01}, /* V crop start: 1 */
-	{OV2722_8BIT, 0x3804, 0x06},
-	{OV2722_8BIT, 0x3805, 0xab}, /* H crop end: 1707 */
-	{OV2722_8BIT, 0x3806, 0x04},
-	{OV2722_8BIT, 0x3807, 0x45}, /* V crop end: 1093 */
-	{OV2722_8BIT, 0x3808, 0x05},
-	{OV2722_8BIT, 0x3809, 0xac}, /* H output size: 1452 */
-	{OV2722_8BIT, 0x380a, 0x04},
-	{OV2722_8BIT, 0x380b, 0x44}, /* V output size: 1092 */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0xd4}, /* H timing: 2260 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xdc}, /* V timing: 1244 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x03}, /* H window offset: 3 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x02}, /* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0x80},
-	{OV2722_8BIT, 0x3821, 0x06}, /*  mirror */
-	{OV2722_8BIT, 0x3814, 0x11},
-	{OV2722_8BIT, 0x3815, 0x11},
-	{OV2722_8BIT, 0x3612, 0x0b},
-	{OV2722_8BIT, 0x3618, 0x04},
-	{OV2722_8BIT, 0x3a08, 0x01},
-	{OV2722_8BIT, 0x3a09, 0x50},
-	{OV2722_8BIT, 0x3a0a, 0x01},
-	{OV2722_8BIT, 0x3a0b, 0x18},
-	{OV2722_8BIT, 0x3a0d, 0x03},
-	{OV2722_8BIT, 0x3a0e, 0x03},
-	{OV2722_8BIT, 0x4520, 0x00},
-	{OV2722_8BIT, 0x4837, 0x1b},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0xd2}, /* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x23},
-	{OV2722_8BIT, 0x3634, 0x54},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1},
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xcf}, /* manual 3a */
-	{OV2722_8BIT, 0x301d, 0xf0}, /* enable group hold */
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0},
-	{OV2722_8BIT, 0x5184, 0xb0},
-	{OV2722_8BIT, 0x5185, 0xb0},
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x2c}, /* 422.4 MHz */
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-	{OV2722_8BIT, 0x3000, 0x00}, /* added for power optimization */
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3503, 0x17}, /* manual 3a */
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x3F},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x00},
-	{OV2722_TOK_TERM, 0, 0}
-};
-static struct ov2722_reg const ov2722_1M3_30fps[] = {
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x24},
-	{OV2722_8BIT, 0x373a, 0x60},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x2e},
-	{OV2722_8BIT, 0x3705, 0x10},
-	{OV2722_8BIT, 0x3730, 0x30},
-	{OV2722_8BIT, 0x3704, 0x62},
-	{OV2722_8BIT, 0x3f06, 0x3a},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0xc4},
-	{OV2722_8BIT, 0x371e, 0x01},
-	{OV2722_8BIT, 0x371f, 0x0d},
-	{OV2722_8BIT, 0x3708, 0x61},
-	{OV2722_8BIT, 0x3709, 0x12},
-	{OV2722_8BIT, 0x3800, 0x01},
-	{OV2722_8BIT, 0x3801, 0x4a},	/* H crop start: 330 */
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0x03},	/* V crop start: 3 */
-	{OV2722_8BIT, 0x3804, 0x06},
-	{OV2722_8BIT, 0x3805, 0xe1},	/* H crop end:  1761 */
-	{OV2722_8BIT, 0x3806, 0x04},
-	{OV2722_8BIT, 0x3807, 0x47},	/* V crop end:  1095 */
-	{OV2722_8BIT, 0x3808, 0x05},
-	{OV2722_8BIT, 0x3809, 0x88},	/* H output size: 1416 */
-	{OV2722_8BIT, 0x380a, 0x04},
-	{OV2722_8BIT, 0x380b, 0x0a},	/* V output size: 1034 */
-
-	/* H blank timing */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0x00},	/* H total size: 2048 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xa0},	/* V total size: 1184 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x05},	/* H window offset: 5 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x02},	/* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0x80},
-	{OV2722_8BIT, 0x3821, 0x06},	/* flip isp */
-	{OV2722_8BIT, 0x3814, 0x11},
-	{OV2722_8BIT, 0x3815, 0x11},
-	{OV2722_8BIT, 0x3612, 0x0b},
-	{OV2722_8BIT, 0x3618, 0x04},
-	{OV2722_8BIT, 0x3a08, 0x01},
-	{OV2722_8BIT, 0x3a09, 0x50},
-	{OV2722_8BIT, 0x3a0a, 0x01},
-	{OV2722_8BIT, 0x3a0b, 0x18},
-	{OV2722_8BIT, 0x3a0d, 0x03},
-	{OV2722_8BIT, 0x3a0e, 0x03},
-	{OV2722_8BIT, 0x4520, 0x00},
-	{OV2722_8BIT, 0x4837, 0x1b},
-	{OV2722_8BIT, 0x3000, 0xff},
-	{OV2722_8BIT, 0x3001, 0xff},
-	{OV2722_8BIT, 0x3002, 0xf0},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0xd2},	/* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x23},
-	{OV2722_8BIT, 0x3634, 0x54},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1},	/* v_en, h_en, blc_en */
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xcf},
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0},	/* AWB red */
-	{OV2722_8BIT, 0x5184, 0xb0},	/* AWB green */
-	{OV2722_8BIT, 0x5185, 0xb0},	/* AWB blue */
-	{OV2722_8BIT, 0x5180, 0x03},	/* AWB manual mode */
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x4800, 0x24},	/* clk lane gate enable */
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x26},
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-
-	/* Added for power optimization */
-	{OV2722_8BIT, 0x3000, 0x00},
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3503, 0x17},
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x46},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x10},
-	{OV2722_TOK_TERM, 0, 0},
-};
-
-static struct ov2722_reg const ov2722_1080p_30fps[] = {
-	{OV2722_8BIT, 0x3021, 0x03}, /* For stand wait for a whole
-					frame complete.(vblank) */
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x24},
-	{OV2722_8BIT, 0x373a, 0x60},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x2e},
-	{OV2722_8BIT, 0x3705, 0x2b},
-	{OV2722_8BIT, 0x3730, 0x30},
-	{OV2722_8BIT, 0x3704, 0x62},
-	{OV2722_8BIT, 0x3f06, 0x3a},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0xc4},
-	{OV2722_8BIT, 0x371e, 0x01},
-	{OV2722_8BIT, 0x371f, 0x28},
-	{OV2722_8BIT, 0x3708, 0x61},
-	{OV2722_8BIT, 0x3709, 0x12},
-	{OV2722_8BIT, 0x3800, 0x00},
-	{OV2722_8BIT, 0x3801, 0x08}, /* H crop start: 8 */
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0x01}, /* V crop start: 1 */
-	{OV2722_8BIT, 0x3804, 0x07},
-	{OV2722_8BIT, 0x3805, 0x9b}, /* H crop end: 1947 */
-	{OV2722_8BIT, 0x3806, 0x04},
-	{OV2722_8BIT, 0x3807, 0x45}, /* V crop end: 1093 */
-	{OV2722_8BIT, 0x3808, 0x07},
-	{OV2722_8BIT, 0x3809, 0x8c}, /* H output size: 1932 */
-	{OV2722_8BIT, 0x380a, 0x04},
-	{OV2722_8BIT, 0x380b, 0x44}, /* V output size: 1092 */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0x14}, /* H timing: 2068 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0x5a}, /* V timing: 1114 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x03}, /* H window offset: 3 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x02}, /* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0x80},
-	{OV2722_8BIT, 0x3821, 0x06}, /*  mirror */
-	{OV2722_8BIT, 0x3814, 0x11},
-	{OV2722_8BIT, 0x3815, 0x11},
-	{OV2722_8BIT, 0x3612, 0x4b},
-	{OV2722_8BIT, 0x3618, 0x04},
-	{OV2722_8BIT, 0x3a08, 0x01},
-	{OV2722_8BIT, 0x3a09, 0x50},
-	{OV2722_8BIT, 0x3a0a, 0x01},
-	{OV2722_8BIT, 0x3a0b, 0x18},
-	{OV2722_8BIT, 0x3a0d, 0x03},
-	{OV2722_8BIT, 0x3a0e, 0x03},
-	{OV2722_8BIT, 0x4520, 0x00},
-	{OV2722_8BIT, 0x4837, 0x1b},
-	{OV2722_8BIT, 0x3000, 0xff},
-	{OV2722_8BIT, 0x3001, 0xff},
-	{OV2722_8BIT, 0x3002, 0xf0},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0x53}, /* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x63},
-	{OV2722_8BIT, 0x3634, 0x24},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1},
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xcd}, /* manual 3a */
-	{OV2722_8BIT, 0x301d, 0xf0}, /* enable group hold */
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x3503, 0x17},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0},
-	{OV2722_8BIT, 0x5184, 0xb0},
-	{OV2722_8BIT, 0x5185, 0xb0},
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x24}, /* 345.6 MHz */
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-	{OV2722_8BIT, 0x3000, 0x00}, /* added for power optimization */
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3011, 0x22},
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x3F},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x00},
-	{OV2722_TOK_TERM, 0, 0}
-};
-
-static struct ov2722_reg const ov2722_720p_30fps[] = {
-	{OV2722_8BIT, 0x3021, 0x03},
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x24},
-	{OV2722_8BIT, 0x373a, 0x60},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x2e},
-	{OV2722_8BIT, 0x3705, 0x10},
-	{OV2722_8BIT, 0x3730, 0x30},
-	{OV2722_8BIT, 0x3704, 0x62},
-	{OV2722_8BIT, 0x3f06, 0x3a},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0xc4},
-	{OV2722_8BIT, 0x371e, 0x01},
-	{OV2722_8BIT, 0x371f, 0x0d},
-	{OV2722_8BIT, 0x3708, 0x61},
-	{OV2722_8BIT, 0x3709, 0x12},
-	{OV2722_8BIT, 0x3800, 0x01},
-	{OV2722_8BIT, 0x3801, 0x40}, /* H crop start: 320 */
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0xb1}, /* V crop start: 177 */
-	{OV2722_8BIT, 0x3804, 0x06},
-	{OV2722_8BIT, 0x3805, 0x55}, /* H crop end: 1621 */
-	{OV2722_8BIT, 0x3806, 0x03},
-	{OV2722_8BIT, 0x3807, 0x95}, /* V crop end: 918 */
-	{OV2722_8BIT, 0x3808, 0x05},
-	{OV2722_8BIT, 0x3809, 0x10}, /* H output size: 0x0788==1928 */
-	{OV2722_8BIT, 0x380a, 0x02},
-	{OV2722_8BIT, 0x380b, 0xe0}, /* output size: 0x02DE==734 */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0x00}, /* H timing: 2048 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xa3}, /* V timing: 1187 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x03}, /* H window offset: 3 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x02}, /* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0x80},
-	{OV2722_8BIT, 0x3821, 0x06}, /* mirror */
-	{OV2722_8BIT, 0x3814, 0x11},
-	{OV2722_8BIT, 0x3815, 0x11},
-	{OV2722_8BIT, 0x3612, 0x0b},
-	{OV2722_8BIT, 0x3618, 0x04},
-	{OV2722_8BIT, 0x3a08, 0x01},
-	{OV2722_8BIT, 0x3a09, 0x50},
-	{OV2722_8BIT, 0x3a0a, 0x01},
-	{OV2722_8BIT, 0x3a0b, 0x18},
-	{OV2722_8BIT, 0x3a0d, 0x03},
-	{OV2722_8BIT, 0x3a0e, 0x03},
-	{OV2722_8BIT, 0x4520, 0x00},
-	{OV2722_8BIT, 0x4837, 0x1b},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0xd2}, /* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x23},
-	{OV2722_8BIT, 0x3634, 0x54},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1},
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xcf}, /* manual 3a */
-	{OV2722_8BIT, 0x301d, 0xf0}, /* enable group hold */
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0},
-	{OV2722_8BIT, 0x5184, 0xb0},
-	{OV2722_8BIT, 0x5185, 0xb0},
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x26}, /* {0x3036, 0x2c}, //422.4 MHz */
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-	{OV2722_8BIT, 0x3000, 0x00}, /* added for power optimization */
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3503, 0x17}, /* manual 3a */
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x3F},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x00},
-	{OV2722_TOK_TERM, 0, 0},
-};
-
-static struct ov2722_resolution ov2722_res_preview[] = {
-	{
-		.desc = "ov2722_1632_1092_30fps",
-		.width = 1632,
-		.height = 1092,
-		.fps = 30,
-		.pix_clk_freq = 85,
-		.used = 0,
-		.pixels_per_line = 2260,
-		.lines_per_frame = 1244,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = ov2722_1632_1092_30fps,
-		.mipi_freq = 422400,
-	},
-	{
-		.desc = "ov2722_1452_1092_30fps",
-		.width = 1452,
-		.height = 1092,
-		.fps = 30,
-		.pix_clk_freq = 85,
-		.used = 0,
-		.pixels_per_line = 2260,
-		.lines_per_frame = 1244,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = ov2722_1452_1092_30fps,
-		.mipi_freq = 422400,
-	},
-	{
-		.desc = "ov2722_1080P_30fps",
-		.width = 1932,
-		.height = 1092,
-		.pix_clk_freq = 69,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2068,
-		.lines_per_frame = 1114,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = ov2722_1080p_30fps,
-		.mipi_freq = 345600,
-	},
-};
-#define N_RES_PREVIEW (ARRAY_SIZE(ov2722_res_preview))
-
-/*
- * Disable non-preview configurations until the configuration selection is
- * improved.
- */
-#if 0
-struct ov2722_resolution ov2722_res_still[] = {
-	{
-		.desc = "ov2722_480P_30fps",
-		.width = 1632,
-		.height = 1092,
-		.fps = 30,
-		.pix_clk_freq = 85,
-		.used = 0,
-		.pixels_per_line = 2260,
-		.lines_per_frame = 1244,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = ov2722_1632_1092_30fps,
-		.mipi_freq = 422400,
-	},
-	{
-		.desc = "ov2722_1452_1092_30fps",
-		.width = 1452,
-		.height = 1092,
-		.fps = 30,
-		.pix_clk_freq = 85,
-		.used = 0,
-		.pixels_per_line = 2260,
-		.lines_per_frame = 1244,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = ov2722_1452_1092_30fps,
-		.mipi_freq = 422400,
-	},
-	{
-		.desc = "ov2722_1080P_30fps",
-		.width = 1932,
-		.height = 1092,
-		.pix_clk_freq = 69,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2068,
-		.lines_per_frame = 1114,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = ov2722_1080p_30fps,
-		.mipi_freq = 345600,
-	},
-};
-#define N_RES_STILL (ARRAY_SIZE(ov2722_res_still))
-
-struct ov2722_resolution ov2722_res_video[] = {
-	{
-		.desc = "ov2722_QVGA_30fps",
-		.width = 336,
-		.height = 256,
-		.fps = 30,
-		.pix_clk_freq = 73,
-		.used = 0,
-		.pixels_per_line = 2048,
-		.lines_per_frame = 1184,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = ov2722_QVGA_30fps,
-		.mipi_freq = 364800,
-	},
-	{
-		.desc = "ov2722_480P_30fps",
-		.width = 736,
-		.height = 496,
-		.fps = 30,
-		.pix_clk_freq = 73,
-		.used = 0,
-		.pixels_per_line = 2048,
-		.lines_per_frame = 1184,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = ov2722_480P_30fps,
-	},
-	{
-		.desc = "ov2722_1080P_30fps",
-		.width = 1932,
-		.height = 1092,
-		.pix_clk_freq = 69,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2068,
-		.lines_per_frame = 1114,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.skip_frames = 3,
-		.regs = ov2722_1080p_30fps,
-		.mipi_freq = 345600,
-	},
-};
-#define N_RES_VIDEO (ARRAY_SIZE(ov2722_res_video))
-#endif
-
-static struct ov2722_resolution *ov2722_res = ov2722_res_preview;
-static unsigned long N_RES = N_RES_PREVIEW;
-#endif
diff --git a/drivers/staging/media/atomisp/i2c/ov5693/Kconfig b/drivers/staging/media/atomisp/i2c/ov5693/Kconfig
deleted file mode 100644
index 3f527f2047a7..000000000000
--- a/drivers/staging/media/atomisp/i2c/ov5693/Kconfig
+++ /dev/null
@@ -1,11 +0,0 @@
-config VIDEO_ATOMISP_OV5693
-       tristate "Omnivision ov5693 sensor support"
-	depends on ACPI
-       depends on I2C && VIDEO_V4L2
-       ---help---
-	 This is a Video4Linux2 sensor-level driver for the Micron
-	 ov5693 5 Mpixel camera.
-
-	 ov5693 is video camera sensor.
-
-	 It currently only works with the atomisp driver.
diff --git a/drivers/staging/media/atomisp/i2c/ov5693/Makefile b/drivers/staging/media/atomisp/i2c/ov5693/Makefile
deleted file mode 100644
index aa6be85c5a60..000000000000
--- a/drivers/staging/media/atomisp/i2c/ov5693/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_VIDEO_ATOMISP_OV5693) += atomisp-ov5693.o
-
-# HACK! While this driver is in bad shape, don't enable several warnings
-#       that would be otherwise enabled with W=1
-ccflags-y += $(call cc-disable-warning, unused-but-set-variable)
-ccflags-y += $(call cc-disable-warning, unused-const-variable)
-ccflags-y += $(call cc-disable-warning, missing-prototypes)
-ccflags-y += $(call cc-disable-warning, missing-declarations)
diff --git a/drivers/staging/media/atomisp/i2c/ov5693/ad5823.h b/drivers/staging/media/atomisp/i2c/ov5693/ad5823.h
deleted file mode 100644
index 4de44569fe54..000000000000
--- a/drivers/staging/media/atomisp/i2c/ov5693/ad5823.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Support for AD5823 VCM.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __AD5823_H__
-#define __AD5823_H__
-
-#include <linux/types.h>
-
-
-#define AD5823_VCM_ADDR	0x0c
-
-#define AD5823_REG_RESET		0x01
-#define AD5823_REG_MODE			0x02
-#define AD5823_REG_VCM_MOVE_TIME	0x03
-#define AD5823_REG_VCM_CODE_MSB		0x04
-#define AD5823_REG_VCM_CODE_LSB		0x05
-#define AD5823_REG_VCM_THRESHOLD_MSB	0x06
-#define AD5823_REG_VCM_THRESHOLD_LSB	0x07
-
-#define AD5823_REG_LENGTH		0x1
-
-#define AD5823_RING_CTRL_ENABLE		0x04
-#define AD5823_RING_CTRL_DISABLE	0x00
-
-#define AD5823_RESONANCE_PERIOD		100000
-#define AD5823_RESONANCE_COEF		512
-#define AD5823_HIGH_FREQ_RANGE		0x80
-
-#define VCM_CODE_MSB_MASK		0xfc
-#define AD5823_INIT_FOCUS_POS           350
-
-enum ad5823_tok_type {
-	AD5823_8BIT  = 0x1,
-	AD5823_16BIT = 0x2,
-};
-
-enum ad5823_vcm_mode {
-	AD5823_ARC_RES0 = 0x0,	/* Actuator response control RES1 */
-	AD5823_ARC_RES1 = 0x1,	/* Actuator response control RES0.5 */
-	AD5823_ARC_RES2 = 0x2,	/* Actuator response control RES2 */
-	AD5823_ESRC = 0x3,	/* Enhanced slew rate control */
-	AD5823_DIRECT = 0x4,	/* Direct control */
-};
-
-#define AD5823_INVALID_CONFIG	0xffffffff
-#define AD5823_MAX_FOCUS_POS	1023
-#define DELAY_PER_STEP_NS	1000000
-#define DELAY_MAX_PER_STEP_NS	(1000000 * 1023)
-#endif
diff --git a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c b/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c
deleted file mode 100644
index 714297c36b3e..000000000000
--- a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c
+++ /dev/null
@@ -1,1993 +0,0 @@
-/*
- * Support for OmniVision OV5693 1080p HD camera sensor.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/string.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/kmod.h>
-#include <linux/device.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <linux/i2c.h>
-#include <linux/moduleparam.h>
-#include <media/v4l2-device.h>
-#include <linux/io.h>
-#include <linux/acpi.h>
-#include "../../include/linux/atomisp_gmin_platform.h"
-
-#include "ov5693.h"
-#include "ad5823.h"
-
-#define __cci_delay(t) \
-	do { \
-		if ((t) < 10) { \
-			usleep_range((t) * 1000, ((t) + 1) * 1000); \
-		} else { \
-			msleep((t)); \
-		} \
-	} while (0)
-
-/* Value 30ms reached through experimentation on byt ecs.
- * The DS specifies a much lower value but when using a smaller value
- * the I2C bus sometimes locks up permanently when starting the camera.
- * This issue could not be reproduced on cht, so we can reduce the
- * delay value to a lower value when insmod.
- */
-static uint up_delay = 30;
-module_param(up_delay, uint, 0644);
-MODULE_PARM_DESC(up_delay, "Delay prior to the first CCI transaction for ov5693");
-
-static int vcm_ad_i2c_wr8(struct i2c_client *client, u8 reg, u8 val)
-{
-	int err;
-	struct i2c_msg msg;
-	u8 buf[2];
-
-	buf[0] = reg;
-	buf[1] = val;
-
-	msg.addr = VCM_ADDR;
-	msg.flags = 0;
-	msg.len = 2;
-	msg.buf = &buf[0];
-
-	err = i2c_transfer(client->adapter, &msg, 1);
-	if (err != 1) {
-		dev_err(&client->dev, "%s: vcm i2c fail, err code = %d\n",
-			__func__, err);
-		return -EIO;
-	}
-	return 0;
-}
-
-static int ad5823_i2c_write(struct i2c_client *client, u8 reg, u8 val)
-{
-	struct i2c_msg msg;
-	u8 buf[2];
-
-	buf[0] = reg;
-	buf[1] = val;
-	msg.addr = AD5823_VCM_ADDR;
-	msg.flags = 0;
-	msg.len = 0x02;
-	msg.buf = &buf[0];
-
-	if (i2c_transfer(client->adapter, &msg, 1) != 1)
-		return -EIO;
-	return 0;
-}
-
-static int ad5823_i2c_read(struct i2c_client *client, u8 reg, u8 *val)
-{
-	struct i2c_msg msg[2];
-	u8 buf[2];
-
-	buf[0] = reg;
-	buf[1] = 0;
-
-	msg[0].addr = AD5823_VCM_ADDR;
-	msg[0].flags = 0;
-	msg[0].len = 0x01;
-	msg[0].buf = &buf[0];
-
-	msg[1].addr = 0x0c;
-	msg[1].flags = I2C_M_RD;
-	msg[1].len = 0x01;
-	msg[1].buf = &buf[1];
-	*val = 0;
-	if (i2c_transfer(client->adapter, msg, 2) != 2)
-		return -EIO;
-	*val = buf[1];
-	return 0;
-}
-
-
-static const uint32_t ov5693_embedded_effective_size = 28;
-
-/* i2c read/write stuff */
-static int ov5693_read_reg(struct i2c_client *client,
-			   u16 data_length, u16 reg, u16 *val)
-{
-	int err;
-	struct i2c_msg msg[2];
-	unsigned char data[6];
-
-	if (!client->adapter) {
-		dev_err(&client->dev, "%s error, no client->adapter\n",
-			__func__);
-		return -ENODEV;
-	}
-
-	if (data_length != OV5693_8BIT && data_length != OV5693_16BIT
-					&& data_length != OV5693_32BIT) {
-		dev_err(&client->dev, "%s error, invalid data length\n",
-			__func__);
-		return -EINVAL;
-	}
-
-	memset(msg, 0, sizeof(msg));
-
-	msg[0].addr = client->addr;
-	msg[0].flags = 0;
-	msg[0].len = I2C_MSG_LENGTH;
-	msg[0].buf = data;
-
-	/* high byte goes out first */
-	data[0] = (u8)(reg >> 8);
-	data[1] = (u8)(reg & 0xff);
-
-	msg[1].addr = client->addr;
-	msg[1].len = data_length;
-	msg[1].flags = I2C_M_RD;
-	msg[1].buf = data;
-
-	err = i2c_transfer(client->adapter, msg, 2);
-	if (err != 2) {
-		if (err >= 0)
-			err = -EIO;
-		dev_err(&client->dev,
-			"read from offset 0x%x error %d", reg, err);
-		return err;
-	}
-
-	*val = 0;
-	/* high byte comes first */
-	if (data_length == OV5693_8BIT)
-		*val = (u8)data[0];
-	else if (data_length == OV5693_16BIT)
-		*val = be16_to_cpu(*(__be16 *)&data[0]);
-	else
-		*val = be32_to_cpu(*(__be32 *)&data[0]);
-
-	return 0;
-}
-
-static int ov5693_i2c_write(struct i2c_client *client, u16 len, u8 *data)
-{
-	struct i2c_msg msg;
-	const int num_msg = 1;
-	int ret;
-
-	msg.addr = client->addr;
-	msg.flags = 0;
-	msg.len = len;
-	msg.buf = data;
-	ret = i2c_transfer(client->adapter, &msg, 1);
-
-	return ret == num_msg ? 0 : -EIO;
-}
-
-static int vcm_dw_i2c_write(struct i2c_client *client, u16 data)
-{
-	struct i2c_msg msg;
-	const int num_msg = 1;
-	int ret;
-	__be16 val;
-
-	val = cpu_to_be16(data);
-	msg.addr = VCM_ADDR;
-	msg.flags = 0;
-	msg.len = OV5693_16BIT;
-	msg.buf = (void *)&val;
-
-	ret = i2c_transfer(client->adapter, &msg, 1);
-
-	return ret == num_msg ? 0 : -EIO;
-}
-
-/*
- * Theory: per datasheet, the two VCMs both allow for a 2-byte read.
- * The DW9714 doesn't actually specify what this does (it has a
- * two-byte write-only protocol, but specifies the read sequence as
- * legal), but it returns the same data (zeroes) always, after an
- * undocumented initial NAK.  The AD5823 has a one-byte address
- * register to which all writes go, and subsequent reads will cycle
- * through the 8 bytes of registers.  Notably, the default values (the
- * device is always power-cycled affirmatively, so we can rely on
- * these) in AD5823 are not pairwise repetitions of the same 16 bit
- * word.  So all we have to do is sequentially read two bytes at a
- * time and see if we detect a difference in any of the first four
- * pairs.
- */
-static int vcm_detect(struct i2c_client *client)
-{
-	int i, ret;
-	struct i2c_msg msg;
-	u16 data0 = 0, data;
-
-	for (i = 0; i < 4; i++) {
-		msg.addr = VCM_ADDR;
-		msg.flags = I2C_M_RD;
-		msg.len = sizeof(data);
-		msg.buf = (u8 *)&data;
-		ret = i2c_transfer(client->adapter, &msg, 1);
-
-		/*
-		 * DW9714 always fails the first read and returns
-		 * zeroes for subsequent ones
-		 */
-		if (i == 0 && ret == -EREMOTEIO) {
-			data0 = 0;
-			continue;
-		}
-
-		if (i == 0)
-			data0 = data;
-
-		if (data != data0)
-			return VCM_AD5823;
-	}
-	return ret == 1 ? VCM_DW9714 : ret;
-}
-
-static int ov5693_write_reg(struct i2c_client *client, u16 data_length,
-							u16 reg, u16 val)
-{
-	int ret;
-	unsigned char data[4] = {0};
-	__be16 *wreg = (void *)data;
-	const u16 len = data_length + sizeof(u16); /* 16-bit address + data */
-
-	if (data_length != OV5693_8BIT && data_length != OV5693_16BIT) {
-		dev_err(&client->dev,
-			"%s error, invalid data_length\n", __func__);
-		return -EINVAL;
-	}
-
-	/* high byte goes out first */
-	*wreg = cpu_to_be16(reg);
-
-	if (data_length == OV5693_8BIT) {
-		data[2] = (u8)(val);
-	} else {
-		/* OV5693_16BIT */
-		__be16 *wdata = (void *)&data[2];
-
-		*wdata = cpu_to_be16(val);
-	}
-
-	ret = ov5693_i2c_write(client, len, data);
-	if (ret)
-		dev_err(&client->dev,
-			"write error: wrote 0x%x to offset 0x%x error %d",
-			val, reg, ret);
-
-	return ret;
-}
-
-/*
- * ov5693_write_reg_array - Initializes a list of OV5693 registers
- * @client: i2c driver client structure
- * @reglist: list of registers to be written
- *
- * This function initializes a list of registers. When consecutive addresses
- * are found in a row on the list, this function creates a buffer and sends
- * consecutive data in a single i2c_transfer().
- *
- * __ov5693_flush_reg_array, __ov5693_buf_reg_array() and
- * __ov5693_write_reg_is_consecutive() are internal functions to
- * ov5693_write_reg_array_fast() and should be not used anywhere else.
- *
- */
-
-static int __ov5693_flush_reg_array(struct i2c_client *client,
-				    struct ov5693_write_ctrl *ctrl)
-{
-	u16 size;
-	__be16 *reg = (void *)&ctrl->buffer.addr;
-
-	if (ctrl->index == 0)
-		return 0;
-
-	size = sizeof(u16) + ctrl->index; /* 16-bit address + data */
-
-	*reg = cpu_to_be16(ctrl->buffer.addr);
-	ctrl->index = 0;
-
-	return ov5693_i2c_write(client, size, (u8 *)reg);
-}
-
-static int __ov5693_buf_reg_array(struct i2c_client *client,
-				  struct ov5693_write_ctrl *ctrl,
-				  const struct ov5693_reg *next)
-{
-	int size;
-	__be16 *data16;
-
-	switch (next->type) {
-	case OV5693_8BIT:
-		size = 1;
-		ctrl->buffer.data[ctrl->index] = (u8)next->val;
-		break;
-	case OV5693_16BIT:
-		size = 2;
-
-		data16 = (void *)&ctrl->buffer.data[ctrl->index];
-		*data16 = cpu_to_be16((u16)next->val);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* When first item is added, we need to store its starting address */
-	if (ctrl->index == 0)
-		ctrl->buffer.addr = next->reg;
-
-	ctrl->index += size;
-
-	/*
-	 * Buffer cannot guarantee free space for u32? Better flush it to avoid
-	 * possible lack of memory for next item.
-	 */
-	if (ctrl->index + sizeof(u16) >= OV5693_MAX_WRITE_BUF_SIZE)
-		return __ov5693_flush_reg_array(client, ctrl);
-
-	return 0;
-}
-
-static int __ov5693_write_reg_is_consecutive(struct i2c_client *client,
-					     struct ov5693_write_ctrl *ctrl,
-					     const struct ov5693_reg *next)
-{
-	if (ctrl->index == 0)
-		return 1;
-
-	return ctrl->buffer.addr + ctrl->index == next->reg;
-}
-
-static int ov5693_write_reg_array(struct i2c_client *client,
-				  const struct ov5693_reg *reglist)
-{
-	const struct ov5693_reg *next = reglist;
-	struct ov5693_write_ctrl ctrl;
-	int err;
-
-	ctrl.index = 0;
-	for (; next->type != OV5693_TOK_TERM; next++) {
-		switch (next->type & OV5693_TOK_MASK) {
-		case OV5693_TOK_DELAY:
-			err = __ov5693_flush_reg_array(client, &ctrl);
-			if (err)
-				return err;
-			msleep(next->val);
-			break;
-		default:
-			/*
-			 * If next address is not consecutive, data needs to be
-			 * flushed before proceed.
-			 */
-			if (!__ov5693_write_reg_is_consecutive(client, &ctrl,
-								next)) {
-				err = __ov5693_flush_reg_array(client, &ctrl);
-				if (err)
-					return err;
-			}
-			err = __ov5693_buf_reg_array(client, &ctrl, next);
-			if (err) {
-				dev_err(&client->dev,
-					"%s: write error, aborted\n",
-					__func__);
-				return err;
-			}
-			break;
-		}
-	}
-
-	return __ov5693_flush_reg_array(client, &ctrl);
-}
-static int ov5693_g_focal(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = (OV5693_FOCAL_LENGTH_NUM << 16) | OV5693_FOCAL_LENGTH_DEM;
-	return 0;
-}
-
-static int ov5693_g_fnumber(struct v4l2_subdev *sd, s32 *val)
-{
-	/*const f number for imx*/
-	*val = (OV5693_F_NUMBER_DEFAULT_NUM << 16) | OV5693_F_NUMBER_DEM;
-	return 0;
-}
-
-static int ov5693_g_fnumber_range(struct v4l2_subdev *sd, s32 *val)
-{
-	*val = (OV5693_F_NUMBER_DEFAULT_NUM << 24) |
-		(OV5693_F_NUMBER_DEM << 16) |
-		(OV5693_F_NUMBER_DEFAULT_NUM << 8) | OV5693_F_NUMBER_DEM;
-	return 0;
-}
-
-static int ov5693_g_bin_factor_x(struct v4l2_subdev *sd, s32 *val)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-
-	*val = ov5693_res[dev->fmt_idx].bin_factor_x;
-
-	return 0;
-}
-
-static int ov5693_g_bin_factor_y(struct v4l2_subdev *sd, s32 *val)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-
-	*val = ov5693_res[dev->fmt_idx].bin_factor_y;
-
-	return 0;
-}
-
-static int ov5693_get_intg_factor(struct i2c_client *client,
-				struct camera_mipi_info *info,
-				const struct ov5693_resolution *res)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	struct atomisp_sensor_mode_data *buf = &info->data;
-	unsigned int pix_clk_freq_hz;
-	u16 reg_val;
-	int ret;
-
-	if (info == NULL)
-		return -EINVAL;
-
-	/* pixel clock */
-	pix_clk_freq_hz = res->pix_clk_freq * 1000000;
-
-	dev->vt_pix_clk_freq_mhz = pix_clk_freq_hz;
-	buf->vt_pix_clk_freq_mhz = pix_clk_freq_hz;
-
-	/* get integration time */
-	buf->coarse_integration_time_min = OV5693_COARSE_INTG_TIME_MIN;
-	buf->coarse_integration_time_max_margin =
-					OV5693_COARSE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_min = OV5693_FINE_INTG_TIME_MIN;
-	buf->fine_integration_time_max_margin =
-					OV5693_FINE_INTG_TIME_MAX_MARGIN;
-
-	buf->fine_integration_time_def = OV5693_FINE_INTG_TIME_MIN;
-	buf->frame_length_lines = res->lines_per_frame;
-	buf->line_length_pck = res->pixels_per_line;
-	buf->read_mode = res->bin_mode;
-
-	/* get the cropping and output resolution to ISP for this mode. */
-	ret =  ov5693_read_reg(client, OV5693_16BIT,
-					OV5693_HORIZONTAL_START_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_horizontal_start = reg_val;
-
-	ret =  ov5693_read_reg(client, OV5693_16BIT,
-					OV5693_VERTICAL_START_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_vertical_start = reg_val;
-
-	ret = ov5693_read_reg(client, OV5693_16BIT,
-					OV5693_HORIZONTAL_END_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_horizontal_end = reg_val;
-
-	ret = ov5693_read_reg(client, OV5693_16BIT,
-					OV5693_VERTICAL_END_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->crop_vertical_end = reg_val;
-
-	ret = ov5693_read_reg(client, OV5693_16BIT,
-				OV5693_HORIZONTAL_OUTPUT_SIZE_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_width = reg_val;
-
-	ret = ov5693_read_reg(client, OV5693_16BIT,
-				OV5693_VERTICAL_OUTPUT_SIZE_H, &reg_val);
-	if (ret)
-		return ret;
-	buf->output_height = reg_val;
-
-	buf->binning_factor_x = res->bin_factor_x ?
-					res->bin_factor_x : 1;
-	buf->binning_factor_y = res->bin_factor_y ?
-					res->bin_factor_y : 1;
-	return 0;
-}
-
-static long __ov5693_set_exposure(struct v4l2_subdev *sd, int coarse_itg,
-				 int gain, int digitgain)
-
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	u16 vts, hts;
-	int ret, exp_val;
-
-	hts = ov5693_res[dev->fmt_idx].pixels_per_line;
-	vts = ov5693_res[dev->fmt_idx].lines_per_frame;
-	/*
-	 * If coarse_itg is larger than 1<<15, can not write to reg directly.
-	 * The way is to write coarse_itg/2 to the reg, meanwhile write 2*hts
-	 * to the reg.
-	 */
-	if (coarse_itg > (1 << 15)) {
-		hts = hts * 2;
-		coarse_itg = (int)coarse_itg / 2;
-	}
-	/* group hold */
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-				OV5693_GROUP_ACCESS, 0x00);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV5693_GROUP_ACCESS);
-		return ret;
-	}
-
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-				OV5693_TIMING_HTS_H, (hts >> 8) & 0xFF);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV5693_TIMING_HTS_H);
-		return ret;
-	}
-
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-				OV5693_TIMING_HTS_L, hts & 0xFF);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV5693_TIMING_HTS_L);
-		return ret;
-	}
-	/* Increase the VTS to match exposure + MARGIN */
-	if (coarse_itg > vts - OV5693_INTEGRATION_TIME_MARGIN)
-		vts = (u16) coarse_itg + OV5693_INTEGRATION_TIME_MARGIN;
-
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-				OV5693_TIMING_VTS_H, (vts >> 8) & 0xFF);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV5693_TIMING_VTS_H);
-		return ret;
-	}
-
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-					OV5693_TIMING_VTS_L, vts & 0xFF);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV5693_TIMING_VTS_L);
-		return ret;
-	}
-
-	/* set exposure */
-
-	/* Lower four bit should be 0*/
-	exp_val = coarse_itg << 4;
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-			       OV5693_EXPOSURE_L, exp_val & 0xFF);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV5693_EXPOSURE_L);
-		return ret;
-	}
-
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-			       OV5693_EXPOSURE_M, (exp_val >> 8) & 0xFF);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV5693_EXPOSURE_M);
-		return ret;
-	}
-
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-			       OV5693_EXPOSURE_H, (exp_val >> 16) & 0x0F);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV5693_EXPOSURE_H);
-		return ret;
-	}
-
-	/* Analog gain */
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-				OV5693_AGC_L, gain & 0xff);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV5693_AGC_L);
-		return ret;
-	}
-
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-				OV5693_AGC_H, (gain >> 8) & 0xff);
-	if (ret) {
-		dev_err(&client->dev, "%s: write %x error, aborted\n",
-			__func__, OV5693_AGC_H);
-		return ret;
-	}
-
-	/* Digital gain */
-	if (digitgain) {
-		ret = ov5693_write_reg(client, OV5693_16BIT,
-				OV5693_MWB_RED_GAIN_H, digitgain);
-		if (ret) {
-			dev_err(&client->dev, "%s: write %x error, aborted\n",
-				__func__, OV5693_MWB_RED_GAIN_H);
-			return ret;
-		}
-
-		ret = ov5693_write_reg(client, OV5693_16BIT,
-				OV5693_MWB_GREEN_GAIN_H, digitgain);
-		if (ret) {
-			dev_err(&client->dev, "%s: write %x error, aborted\n",
-				__func__, OV5693_MWB_RED_GAIN_H);
-			return ret;
-		}
-
-		ret = ov5693_write_reg(client, OV5693_16BIT,
-				OV5693_MWB_BLUE_GAIN_H, digitgain);
-		if (ret) {
-			dev_err(&client->dev, "%s: write %x error, aborted\n",
-				__func__, OV5693_MWB_RED_GAIN_H);
-			return ret;
-		}
-	}
-
-	/* End group */
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-				OV5693_GROUP_ACCESS, 0x10);
-	if (ret)
-		return ret;
-
-	/* Delay launch group */
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-				OV5693_GROUP_ACCESS, 0xa0);
-	if (ret)
-		return ret;
-	return ret;
-}
-
-static int ov5693_set_exposure(struct v4l2_subdev *sd, int exposure,
-	int gain, int digitgain)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	int ret;
-
-	mutex_lock(&dev->input_lock);
-	ret = __ov5693_set_exposure(sd, exposure, gain, digitgain);
-	mutex_unlock(&dev->input_lock);
-
-	return ret;
-}
-
-static long ov5693_s_exposure(struct v4l2_subdev *sd,
-			       struct atomisp_exposure *exposure)
-{
-	u16 coarse_itg = exposure->integration_time[0];
-	u16 analog_gain = exposure->gain[0];
-	u16 digital_gain = exposure->gain[1];
-
-	/* we should not accept the invalid value below */
-	if (analog_gain == 0) {
-		struct i2c_client *client = v4l2_get_subdevdata(sd);
-
-		v4l2_err(client, "%s: invalid value\n", __func__);
-		return -EINVAL;
-	}
-	return ov5693_set_exposure(sd, coarse_itg, analog_gain, digital_gain);
-}
-
-static int ov5693_read_otp_reg_array(struct i2c_client *client, u16 size,
-				     u16 addr, u8 *buf)
-{
-	u16 index;
-	int ret;
-	u16 *pVal = NULL;
-
-	for (index = 0; index <= size; index++) {
-		pVal = (u16 *) (buf + index);
-		ret =
-			ov5693_read_reg(client, OV5693_8BIT, addr + index,
-				    pVal);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int __ov5693_otp_read(struct v4l2_subdev *sd, u8 *buf)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	int ret;
-	int i;
-	u8 *b = buf;
-
-	dev->otp_size = 0;
-	for (i = 1; i < OV5693_OTP_BANK_MAX; i++) {
-		/*set bank NO and OTP read mode. */
-		ret = ov5693_write_reg(client, OV5693_8BIT, OV5693_OTP_BANK_REG, (i | 0xc0));	//[7:6] 2'b11 [5:0] bank no
-		if (ret) {
-			dev_err(&client->dev, "failed to prepare OTP page\n");
-			return ret;
-		}
-		//pr_debug("write 0x%x->0x%x\n",OV5693_OTP_BANK_REG,(i|0xc0));
-
-		/*enable read */
-		ret = ov5693_write_reg(client, OV5693_8BIT, OV5693_OTP_READ_REG, OV5693_OTP_MODE_READ);	// enable :1
-		if (ret) {
-			dev_err(&client->dev,
-				"failed to set OTP reading mode page");
-			return ret;
-		}
-		//pr_debug("write 0x%x->0x%x\n",OV5693_OTP_READ_REG,OV5693_OTP_MODE_READ);
-
-		/* Reading the OTP data array */
-		ret = ov5693_read_otp_reg_array(client, OV5693_OTP_BANK_SIZE,
-						OV5693_OTP_START_ADDR,
-						b);
-		if (ret) {
-			dev_err(&client->dev, "failed to read OTP data\n");
-			return ret;
-		}
-
-		//pr_debug("BANK[%2d] %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x\n", i, *b, *(b+1), *(b+2), *(b+3), *(b+4), *(b+5), *(b+6), *(b+7), *(b+8), *(b+9), *(b+10), *(b+11), *(b+12), *(b+13), *(b+14), *(b+15));
-
-		//Intel OTP map, try to read 320byts first.
-		if (i == 21) {
-			if ((*b) == 0) {
-				dev->otp_size = 320;
-				break;
-			} else {
-				b = buf;
-				continue;
-			}
-		} else if (i == 24) {		//if the first 320bytes data doesn't not exist, try to read the next 32bytes data.
-			if ((*b) == 0) {
-				dev->otp_size = 32;
-				break;
-			} else {
-				b = buf;
-				continue;
-			}
-		} else if (i == 27) {		//if the prvious 32bytes data doesn't exist, try to read the next 32bytes data again.
-			if ((*b) == 0) {
-				dev->otp_size = 32;
-				break;
-			} else {
-				dev->otp_size = 0;	// no OTP data.
-				break;
-			}
-		}
-
-		b = b + OV5693_OTP_BANK_SIZE;
-	}
-	return 0;
-}
-
-/*
- * Read otp data and store it into a kmalloced buffer.
- * The caller must kfree the buffer when no more needed.
- * @size: set to the size of the returned otp data.
- */
-static void *ov5693_otp_read(struct v4l2_subdev *sd)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u8 *buf;
-	int ret;
-
-	buf = devm_kzalloc(&client->dev, (OV5693_OTP_DATA_SIZE + 16), GFP_KERNEL);
-	if (!buf)
-		return ERR_PTR(-ENOMEM);
-
-	//otp valid after mipi on and sw stream on
-	ret = ov5693_write_reg(client, OV5693_8BIT, OV5693_FRAME_OFF_NUM, 0x00);
-
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-			       OV5693_SW_STREAM, OV5693_START_STREAMING);
-
-	ret = __ov5693_otp_read(sd, buf);
-
-	//mipi off and sw stream off after otp read
-	ret = ov5693_write_reg(client, OV5693_8BIT, OV5693_FRAME_OFF_NUM, 0x0f);
-
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-			       OV5693_SW_STREAM, OV5693_STOP_STREAMING);
-
-	/* Driver has failed to find valid data */
-	if (ret) {
-		dev_err(&client->dev, "sensor found no valid OTP data\n");
-		return ERR_PTR(ret);
-	}
-
-	return buf;
-}
-
-static int ov5693_g_priv_int_data(struct v4l2_subdev *sd,
-				  struct v4l2_private_int_data *priv)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	u8 __user *to = priv->data;
-	u32 read_size = priv->size;
-	int ret;
-
-	/* No need to copy data if size is 0 */
-	if (!read_size)
-		goto out;
-
-	if (IS_ERR(dev->otp_data)) {
-		dev_err(&client->dev, "OTP data not available");
-		return PTR_ERR(dev->otp_data);
-	}
-
-	/* Correct read_size value only if bigger than maximum */
-	if (read_size > OV5693_OTP_DATA_SIZE)
-		read_size = OV5693_OTP_DATA_SIZE;
-
-	ret = copy_to_user(to, dev->otp_data, read_size);
-	if (ret) {
-		dev_err(&client->dev, "%s: failed to copy OTP data to user\n",
-			__func__);
-		return -EFAULT;
-	}
-
-	pr_debug("%s read_size:%d\n", __func__, read_size);
-
-out:
-	/* Return correct size */
-	priv->size = dev->otp_size;
-
-	return 0;
-
-}
-
-static long ov5693_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
-{
-
-	switch (cmd) {
-	case ATOMISP_IOC_S_EXPOSURE:
-		return ov5693_s_exposure(sd, arg);
-	case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA:
-		return ov5693_g_priv_int_data(sd, arg);
-	default:
-		return -EINVAL;
-	}
-	return 0;
-}
-
-/*
- * This returns the exposure time being used. This should only be used
- * for filling in EXIF data, not for actual image processing.
- */
-static int ov5693_q_exposure(struct v4l2_subdev *sd, s32 *value)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	u16 reg_v, reg_v2;
-	int ret;
-
-	/* get exposure */
-	ret = ov5693_read_reg(client, OV5693_8BIT,
-					OV5693_EXPOSURE_L,
-					&reg_v);
-	if (ret)
-		goto err;
-
-	ret = ov5693_read_reg(client, OV5693_8BIT,
-					OV5693_EXPOSURE_M,
-					&reg_v2);
-	if (ret)
-		goto err;
-
-	reg_v += reg_v2 << 8;
-	ret = ov5693_read_reg(client, OV5693_8BIT,
-					OV5693_EXPOSURE_H,
-					&reg_v2);
-	if (ret)
-		goto err;
-
-	*value = reg_v + (((u32)reg_v2 << 16));
-err:
-	return ret;
-}
-
-static int ad5823_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = -EINVAL;
-	u8 vcm_code;
-
-	ret = ad5823_i2c_read(client, AD5823_REG_VCM_CODE_MSB, &vcm_code);
-	if (ret)
-		return ret;
-
-	/* set reg VCM_CODE_MSB Bit[1:0] */
-	vcm_code = (vcm_code & VCM_CODE_MSB_MASK) |
-		((val >> 8) & ~VCM_CODE_MSB_MASK);
-	ret = ad5823_i2c_write(client, AD5823_REG_VCM_CODE_MSB, vcm_code);
-	if (ret)
-		return ret;
-
-	/* set reg VCM_CODE_LSB Bit[7:0] */
-	ret = ad5823_i2c_write(client, AD5823_REG_VCM_CODE_LSB, (val & 0xff));
-	if (ret)
-		return ret;
-
-	/* set required vcm move time */
-	vcm_code = AD5823_RESONANCE_PERIOD / AD5823_RESONANCE_COEF
-		- AD5823_HIGH_FREQ_RANGE;
-	ret = ad5823_i2c_write(client, AD5823_REG_VCM_MOVE_TIME, vcm_code);
-
-	return ret;
-}
-
-static int ad5823_t_focus_abs(struct v4l2_subdev *sd, s32 value)
-{
-	value = min(value, AD5823_MAX_FOCUS_POS);
-	return ad5823_t_focus_vcm(sd, value);
-}
-
-static int ov5693_t_focus_abs(struct v4l2_subdev *sd, s32 value)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	dev_dbg(&client->dev, "%s: FOCUS_POS: 0x%x\n", __func__, value);
-	value = clamp(value, 0, OV5693_VCM_MAX_FOCUS_POS);
-	if (dev->vcm == VCM_DW9714) {
-		if (dev->vcm_update) {
-			ret = vcm_dw_i2c_write(client, VCM_PROTECTION_OFF);
-			if (ret)
-				return ret;
-			ret = vcm_dw_i2c_write(client, DIRECT_VCM);
-			if (ret)
-				return ret;
-			ret = vcm_dw_i2c_write(client, VCM_PROTECTION_ON);
-			if (ret)
-				return ret;
-			dev->vcm_update = false;
-		}
-		ret = vcm_dw_i2c_write(client,
-				       vcm_val(value, VCM_DEFAULT_S));
-	} else if (dev->vcm == VCM_AD5823) {
-		ad5823_t_focus_abs(sd, value);
-	}
-	if (ret == 0) {
-		dev->number_of_steps = value - dev->focus;
-		dev->focus = value;
-		dev->timestamp_t_focus_abs = ktime_get();
-	} else
-		dev_err(&client->dev,
-			"%s: i2c failed. ret %d\n", __func__, ret);
-
-	return ret;
-}
-
-static int ov5693_t_focus_rel(struct v4l2_subdev *sd, s32 value)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-
-	return ov5693_t_focus_abs(sd, dev->focus + value);
-}
-
-#define DELAY_PER_STEP_NS	1000000
-#define DELAY_MAX_PER_STEP_NS	(1000000 * 1023)
-static int ov5693_q_focus_status(struct v4l2_subdev *sd, s32 *value)
-{
-	u32 status = 0;
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	ktime_t temptime;
-	ktime_t timedelay = ns_to_ktime(min_t(u32,
-			abs(dev->number_of_steps) * DELAY_PER_STEP_NS,
-			DELAY_MAX_PER_STEP_NS));
-
-	temptime = ktime_sub(ktime_get(), (dev->timestamp_t_focus_abs));
-	if (ktime_compare(temptime, timedelay) <= 0) {
-		status |= ATOMISP_FOCUS_STATUS_MOVING;
-		status |= ATOMISP_FOCUS_HP_IN_PROGRESS;
-	} else {
-		status |= ATOMISP_FOCUS_STATUS_ACCEPTS_NEW_MOVE;
-		status |= ATOMISP_FOCUS_HP_COMPLETE;
-	}
-
-	*value = status;
-
-	return 0;
-}
-
-static int ov5693_q_focus_abs(struct v4l2_subdev *sd, s32 *value)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	s32 val;
-
-	ov5693_q_focus_status(sd, &val);
-
-	if (val & ATOMISP_FOCUS_STATUS_MOVING)
-		*value  = dev->focus - dev->number_of_steps;
-	else
-		*value  = dev->focus;
-
-	return 0;
-}
-
-static int ov5693_t_vcm_slew(struct v4l2_subdev *sd, s32 value)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-
-	dev->number_of_steps = value;
-	dev->vcm_update = true;
-	return 0;
-}
-
-static int ov5693_t_vcm_timing(struct v4l2_subdev *sd, s32 value)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-
-	dev->number_of_steps = value;
-	dev->vcm_update = true;
-	return 0;
-}
-
-static int ov5693_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct ov5693_device *dev =
-	    container_of(ctrl->handler, struct ov5693_device, ctrl_handler);
-	struct i2c_client *client = v4l2_get_subdevdata(&dev->sd);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_FOCUS_ABSOLUTE:
-		dev_dbg(&client->dev, "%s: CID_FOCUS_ABSOLUTE:%d.\n",
-			__func__, ctrl->val);
-		ret = ov5693_t_focus_abs(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_FOCUS_RELATIVE:
-		dev_dbg(&client->dev, "%s: CID_FOCUS_RELATIVE:%d.\n",
-			__func__, ctrl->val);
-		ret = ov5693_t_focus_rel(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_VCM_SLEW:
-		ret = ov5693_t_vcm_slew(&dev->sd, ctrl->val);
-		break;
-	case V4L2_CID_VCM_TIMEING:
-		ret = ov5693_t_vcm_timing(&dev->sd, ctrl->val);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-	return ret;
-}
-
-static int ov5693_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct ov5693_device *dev =
-	    container_of(ctrl->handler, struct ov5693_device, ctrl_handler);
-	int ret = 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_EXPOSURE_ABSOLUTE:
-		ret = ov5693_q_exposure(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FOCAL_ABSOLUTE:
-		ret = ov5693_g_focal(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_ABSOLUTE:
-		ret = ov5693_g_fnumber(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FNUMBER_RANGE:
-		ret = ov5693_g_fnumber_range(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FOCUS_ABSOLUTE:
-		ret = ov5693_q_focus_abs(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_FOCUS_STATUS:
-		ret = ov5693_q_focus_status(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_BIN_FACTOR_HORZ:
-		ret = ov5693_g_bin_factor_x(&dev->sd, &ctrl->val);
-		break;
-	case V4L2_CID_BIN_FACTOR_VERT:
-		ret = ov5693_g_bin_factor_y(&dev->sd, &ctrl->val);
-		break;
-	default:
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-static const struct v4l2_ctrl_ops ctrl_ops = {
-	.s_ctrl = ov5693_s_ctrl,
-	.g_volatile_ctrl = ov5693_g_volatile_ctrl
-};
-
-static const struct v4l2_ctrl_config ov5693_controls[] = {
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_EXPOSURE_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "exposure",
-	 .min = 0x0,
-	 .max = 0xffff,
-	 .step = 0x01,
-	 .def = 0x00,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FOCAL_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "focal length",
-	 .min = OV5693_FOCAL_LENGTH_DEFAULT,
-	 .max = OV5693_FOCAL_LENGTH_DEFAULT,
-	 .step = 0x01,
-	 .def = OV5693_FOCAL_LENGTH_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "f-number",
-	 .min = OV5693_F_NUMBER_DEFAULT,
-	 .max = OV5693_F_NUMBER_DEFAULT,
-	 .step = 0x01,
-	 .def = OV5693_F_NUMBER_DEFAULT,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FNUMBER_RANGE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "f-number range",
-	 .min = OV5693_F_NUMBER_RANGE,
-	 .max = OV5693_F_NUMBER_RANGE,
-	 .step = 0x01,
-	 .def = OV5693_F_NUMBER_RANGE,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FOCUS_ABSOLUTE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "focus move absolute",
-	 .min = 0,
-	 .max = OV5693_VCM_MAX_FOCUS_POS,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FOCUS_RELATIVE,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "focus move relative",
-	 .min = OV5693_VCM_MAX_FOCUS_NEG,
-	 .max = OV5693_VCM_MAX_FOCUS_POS,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_FOCUS_STATUS,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "focus status",
-	 .min = 0,
-	 .max = 100,		/* allow enum to grow in the future */
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_VCM_SLEW,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "vcm slew",
-	 .min = 0,
-	 .max = OV5693_VCM_SLEW_STEP_MAX,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_VCM_TIMEING,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "vcm step time",
-	 .min = 0,
-	 .max = OV5693_VCM_SLEW_TIME_MAX,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_BIN_FACTOR_HORZ,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "horizontal binning factor",
-	 .min = 0,
-	 .max = OV5693_BIN_FACTOR_MAX,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-	{
-	 .ops = &ctrl_ops,
-	 .id = V4L2_CID_BIN_FACTOR_VERT,
-	 .type = V4L2_CTRL_TYPE_INTEGER,
-	 .name = "vertical binning factor",
-	 .min = 0,
-	 .max = OV5693_BIN_FACTOR_MAX,
-	 .step = 1,
-	 .def = 0,
-	 .flags = 0,
-	 },
-};
-
-static int ov5693_init(struct v4l2_subdev *sd)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	pr_info("%s\n", __func__);
-	mutex_lock(&dev->input_lock);
-	dev->vcm_update = false;
-
-	if (dev->vcm == VCM_AD5823) {
-		ret = vcm_ad_i2c_wr8(client, 0x01, 0x01); /* vcm init test */
-		if (ret)
-			dev_err(&client->dev,
-				"vcm reset failed\n");
-		/*change the mode*/
-		ret = ad5823_i2c_write(client, AD5823_REG_VCM_CODE_MSB,
-				       AD5823_RING_CTRL_ENABLE);
-		if (ret)
-			dev_err(&client->dev,
-				"vcm enable ringing failed\n");
-		ret = ad5823_i2c_write(client, AD5823_REG_MODE,
-					AD5823_ARC_RES1);
-		if (ret)
-			dev_err(&client->dev,
-				"vcm change mode failed\n");
-	}
-
-	/*change initial focus value for ad5823*/
-	if (dev->vcm == VCM_AD5823) {
-		dev->focus = AD5823_INIT_FOCUS_POS;
-		ov5693_t_focus_abs(sd, AD5823_INIT_FOCUS_POS);
-	} else {
-		dev->focus = 0;
-		ov5693_t_focus_abs(sd, 0);
-	}
-
-	mutex_unlock(&dev->input_lock);
-
-	return 0;
-}
-
-static int power_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	int ret;
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	/*
-	 * This driver assumes "internal DVDD, PWDNB tied to DOVDD".
-	 * In this set up only gpio0 (XSHUTDN) should be available
-	 * but in some products (for example ECS) gpio1 (PWDNB) is
-	 * also available. If gpio1 is available we emulate it being
-	 * tied to DOVDD here.
-	 */
-	if (flag) {
-		ret = dev->platform_data->v2p8_ctrl(sd, 1);
-		dev->platform_data->gpio1_ctrl(sd, 1);
-		if (ret == 0) {
-			ret = dev->platform_data->v1p8_ctrl(sd, 1);
-			if (ret) {
-				dev->platform_data->gpio1_ctrl(sd, 0);
-				ret = dev->platform_data->v2p8_ctrl(sd, 0);
-			}
-		}
-	} else {
-		dev->platform_data->gpio1_ctrl(sd, 0);
-		ret = dev->platform_data->v1p8_ctrl(sd, 0);
-		ret |= dev->platform_data->v2p8_ctrl(sd, 0);
-	}
-
-	return ret;
-}
-
-static int gpio_ctrl(struct v4l2_subdev *sd, bool flag)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-
-	if (!dev || !dev->platform_data)
-		return -ENODEV;
-
-	return dev->platform_data->gpio0_ctrl(sd, flag);
-}
-
-static int __power_up(struct v4l2_subdev *sd)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	/* power control */
-	ret = power_ctrl(sd, 1);
-	if (ret)
-		goto fail_power;
-
-	/* according to DS, at least 5ms is needed between DOVDD and PWDN */
-	/* add this delay time to 10~11ms*/
-	usleep_range(10000, 11000);
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 1);
-	if (ret) {
-		ret = gpio_ctrl(sd, 1);
-		if (ret)
-			goto fail_power;
-	}
-
-	/* flis clock control */
-	ret = dev->platform_data->flisclk_ctrl(sd, 1);
-	if (ret)
-		goto fail_clk;
-
-	__cci_delay(up_delay);
-
-	return 0;
-
-fail_clk:
-	gpio_ctrl(sd, 0);
-fail_power:
-	power_ctrl(sd, 0);
-	dev_err(&client->dev, "sensor power-up failed\n");
-
-	return ret;
-}
-
-static int power_down(struct v4l2_subdev *sd)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	dev->focus = OV5693_INVALID_CONFIG;
-	if (!dev->platform_data) {
-		dev_err(&client->dev,
-			"no camera_sensor_platform_data");
-		return -ENODEV;
-	}
-
-	ret = dev->platform_data->flisclk_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "flisclk failed\n");
-
-	/* gpio ctrl */
-	ret = gpio_ctrl(sd, 0);
-	if (ret) {
-		ret = gpio_ctrl(sd, 0);
-		if (ret)
-			dev_err(&client->dev, "gpio failed 2\n");
-	}
-
-	/* power control */
-	ret = power_ctrl(sd, 0);
-	if (ret)
-		dev_err(&client->dev, "vprog failed.\n");
-
-	return ret;
-}
-
-static int power_up(struct v4l2_subdev *sd)
-{
-	static const int retry_count = 4;
-	int i, ret;
-
-	for (i = 0; i < retry_count; i++) {
-		ret = __power_up(sd);
-		if (!ret)
-			return 0;
-
-		power_down(sd);
-	}
-	return ret;
-}
-
-static int ov5693_s_power(struct v4l2_subdev *sd, int on)
-{
-	int ret;
-
-	pr_info("%s: on %d\n", __func__, on);
-	if (on == 0)
-		return power_down(sd);
-	else {
-		ret = power_up(sd);
-		if (!ret) {
-			ret = ov5693_init(sd);
-			/* restore settings */
-			ov5693_res = ov5693_res_preview;
-			N_RES = N_RES_PREVIEW;
-		}
-	}
-	return ret;
-}
-
-/*
- * distance - calculate the distance
- * @res: resolution
- * @w: width
- * @h: height
- *
- * Get the gap between res_w/res_h and w/h.
- * distance = (res_w/res_h - w/h) / (w/h) * 8192
- * res->width/height smaller than w/h wouldn't be considered.
- * The gap of ratio larger than 1/8 wouldn't be considered.
- * Returns the value of gap or -1 if fail.
- */
-#define LARGEST_ALLOWED_RATIO_MISMATCH 1024
-static int distance(struct ov5693_resolution *res, u32 w, u32 h)
-{
-	int ratio;
-	int distance;
-
-	if (w == 0 || h == 0 ||
-	    res->width < w || res->height < h)
-		return -1;
-
-	ratio = res->width << 13;
-	ratio /= w;
-	ratio *= h;
-	ratio /= res->height;
-
-	distance = abs(ratio - 8192);
-
-	if (distance > LARGEST_ALLOWED_RATIO_MISMATCH)
-		return -1;
-
-	return distance;
-}
-
-/* Return the nearest higher resolution index
- * Firstly try to find the approximate aspect ratio resolution
- * If we find multiple same AR resolutions, choose the
- * minimal size.
- */
-static int nearest_resolution_index(int w, int h)
-{
-	int i;
-	int idx = -1;
-	int dist;
-	int min_dist = INT_MAX;
-	int min_res_w = INT_MAX;
-	struct ov5693_resolution *tmp_res = NULL;
-
-	for (i = 0; i < N_RES; i++) {
-		tmp_res = &ov5693_res[i];
-		dist = distance(tmp_res, w, h);
-		if (dist == -1)
-			continue;
-		if (dist < min_dist) {
-			min_dist = dist;
-			idx = i;
-			min_res_w = ov5693_res[i].width;
-			continue;
-		}
-		if (dist == min_dist && ov5693_res[i].width < min_res_w)
-			idx = i;
-	}
-
-	return idx;
-}
-
-static int get_resolution_index(int w, int h)
-{
-	int i;
-
-	for (i = 0; i < N_RES; i++) {
-		if (w != ov5693_res[i].width)
-			continue;
-		if (h != ov5693_res[i].height)
-			continue;
-
-		return i;
-	}
-
-	return -1;
-}
-
-/* TODO: remove it. */
-static int startup(struct v4l2_subdev *sd)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	ret = ov5693_write_reg(client, OV5693_8BIT,
-					OV5693_SW_RESET, 0x01);
-	if (ret) {
-		dev_err(&client->dev, "ov5693 reset err.\n");
-		return ret;
-	}
-
-	ret = ov5693_write_reg_array(client, ov5693_global_setting);
-	if (ret) {
-		dev_err(&client->dev, "ov5693 write register err.\n");
-		return ret;
-	}
-
-	ret = ov5693_write_reg_array(client, ov5693_res[dev->fmt_idx].regs);
-	if (ret) {
-		dev_err(&client->dev, "ov5693 write register err.\n");
-		return ret;
-	}
-
-	return ret;
-}
-
-static int ov5693_set_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_pad_config *cfg,
-			  struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct camera_mipi_info *ov5693_info = NULL;
-	int ret = 0;
-	int idx;
-
-	if (format->pad)
-		return -EINVAL;
-	if (!fmt)
-		return -EINVAL;
-	ov5693_info = v4l2_get_subdev_hostdata(sd);
-	if (ov5693_info == NULL)
-		return -EINVAL;
-
-	mutex_lock(&dev->input_lock);
-	idx = nearest_resolution_index(fmt->width, fmt->height);
-	if (idx == -1) {
-		/* return the largest resolution */
-		fmt->width = ov5693_res[N_RES - 1].width;
-		fmt->height = ov5693_res[N_RES - 1].height;
-	} else {
-		fmt->width = ov5693_res[idx].width;
-		fmt->height = ov5693_res[idx].height;
-	}
-
-	fmt->code = MEDIA_BUS_FMT_SBGGR10_1X10;
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
-		cfg->try_fmt = *fmt;
-		mutex_unlock(&dev->input_lock);
-		return 0;
-	}
-
-	dev->fmt_idx = get_resolution_index(fmt->width, fmt->height);
-	if (dev->fmt_idx == -1) {
-		dev_err(&client->dev, "get resolution fail\n");
-		mutex_unlock(&dev->input_lock);
-		return -EINVAL;
-	}
-
-	ret = startup(sd);
-	if (ret) {
-		int i = 0;
-
-		dev_err(&client->dev, "ov5693 startup err, retry to power up\n");
-		for (i = 0; i < OV5693_POWER_UP_RETRY_NUM; i++) {
-			dev_err(&client->dev,
-				"ov5693 retry to power up %d/%d times, result: ",
-				i+1, OV5693_POWER_UP_RETRY_NUM);
-			power_down(sd);
-			ret = power_up(sd);
-			if (!ret) {
-				mutex_unlock(&dev->input_lock);
-				ov5693_init(sd);
-				mutex_lock(&dev->input_lock);
-			} else {
-				dev_err(&client->dev, "power up failed, continue\n");
-				continue;
-			}
-			ret = startup(sd);
-			if (ret) {
-				dev_err(&client->dev, " startup FAILED!\n");
-			} else {
-				dev_err(&client->dev, " startup SUCCESS!\n");
-				break;
-			}
-		}
-	}
-
-	/*
-	 * After sensor settings are set to HW, sometimes stream is started.
-	 * This would cause ISP timeout because ISP is not ready to receive
-	 * data yet. So add stop streaming here.
-	 */
-	ret = ov5693_write_reg(client, OV5693_8BIT, OV5693_SW_STREAM,
-				OV5693_STOP_STREAMING);
-	if (ret)
-		dev_warn(&client->dev, "ov5693 stream off err\n");
-
-	ret = ov5693_get_intg_factor(client, ov5693_info,
-					&ov5693_res[dev->fmt_idx]);
-	if (ret) {
-		dev_err(&client->dev, "failed to get integration_factor\n");
-		goto err;
-	}
-
-	ov5693_info->metadata_width = fmt->width * 10 / 8;
-	ov5693_info->metadata_height = 1;
-	ov5693_info->metadata_effective_width = &ov5693_embedded_effective_size;
-
-err:
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-static int ov5693_get_fmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_pad_config *cfg,
-			  struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-
-	if (format->pad)
-		return -EINVAL;
-
-	if (!fmt)
-		return -EINVAL;
-
-	fmt->width = ov5693_res[dev->fmt_idx].width;
-	fmt->height = ov5693_res[dev->fmt_idx].height;
-	fmt->code = MEDIA_BUS_FMT_SBGGR10_1X10;
-
-	return 0;
-}
-
-static int ov5693_detect(struct i2c_client *client)
-{
-	struct i2c_adapter *adapter = client->adapter;
-	u16 high, low;
-	int ret;
-	u16 id;
-	u8 revision;
-
-	if (!i2c_check_functionality(adapter, I2C_FUNC_I2C))
-		return -ENODEV;
-
-	ret = ov5693_read_reg(client, OV5693_8BIT,
-					OV5693_SC_CMMN_CHIP_ID_H, &high);
-	if (ret) {
-		dev_err(&client->dev, "sensor_id_high = 0x%x\n", high);
-		return -ENODEV;
-	}
-	ret = ov5693_read_reg(client, OV5693_8BIT,
-					OV5693_SC_CMMN_CHIP_ID_L, &low);
-	id = ((((u16) high) << 8) | (u16) low);
-
-	if (id != OV5693_ID) {
-		dev_err(&client->dev, "sensor ID error 0x%x\n", id);
-		return -ENODEV;
-	}
-
-	ret = ov5693_read_reg(client, OV5693_8BIT,
-					OV5693_SC_CMMN_SUB_ID, &high);
-	revision = (u8) high & 0x0f;
-
-	dev_dbg(&client->dev, "sensor_revision = 0x%x\n", revision);
-	dev_dbg(&client->dev, "detect ov5693 success\n");
-	return 0;
-}
-
-static int ov5693_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret;
-
-	mutex_lock(&dev->input_lock);
-
-	ret = ov5693_write_reg(client, OV5693_8BIT, OV5693_SW_STREAM,
-				enable ? OV5693_START_STREAMING :
-				OV5693_STOP_STREAMING);
-
-	mutex_unlock(&dev->input_lock);
-
-	return ret;
-}
-
-
-static int ov5693_s_config(struct v4l2_subdev *sd,
-			   int irq, void *platform_data)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int ret = 0;
-
-	if (platform_data == NULL)
-		return -ENODEV;
-
-	dev->platform_data =
-		(struct camera_sensor_platform_data *)platform_data;
-
-	mutex_lock(&dev->input_lock);
-	/* power off the module, then power on it in future
-	 * as first power on by board may not fulfill the
-	 * power on sequqence needed by the module
-	 */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "ov5693 power-off err.\n");
-		goto fail_power_off;
-	}
-
-	ret = power_up(sd);
-	if (ret) {
-		dev_err(&client->dev, "ov5693 power-up err.\n");
-		goto fail_power_on;
-	}
-
-	if (!dev->vcm)
-		dev->vcm = vcm_detect(client);
-
-	ret = dev->platform_data->csi_cfg(sd, 1);
-	if (ret)
-		goto fail_csi_cfg;
-
-	/* config & detect sensor */
-	ret = ov5693_detect(client);
-	if (ret) {
-		dev_err(&client->dev, "ov5693_detect err s_config.\n");
-		goto fail_csi_cfg;
-	}
-
-	dev->otp_data = ov5693_otp_read(sd);
-
-	/* turn off sensor, after probed */
-	ret = power_down(sd);
-	if (ret) {
-		dev_err(&client->dev, "ov5693 power-off err.\n");
-		goto fail_csi_cfg;
-	}
-	mutex_unlock(&dev->input_lock);
-
-	return ret;
-
-fail_csi_cfg:
-	dev->platform_data->csi_cfg(sd, 0);
-fail_power_on:
-	power_down(sd);
-	dev_err(&client->dev, "sensor power-gating failed\n");
-fail_power_off:
-	mutex_unlock(&dev->input_lock);
-	return ret;
-}
-
-static int ov5693_g_frame_interval(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_frame_interval *interval)
-{
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-
-	interval->interval.numerator = 1;
-	interval->interval.denominator = ov5693_res[dev->fmt_idx].fps;
-
-	return 0;
-}
-
-static int ov5693_enum_mbus_code(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_pad_config *cfg,
-				 struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->index >= MAX_FMTS)
-		return -EINVAL;
-
-	code->code = MEDIA_BUS_FMT_SBGGR10_1X10;
-	return 0;
-}
-
-static int ov5693_enum_frame_size(struct v4l2_subdev *sd,
-				  struct v4l2_subdev_pad_config *cfg,
-				  struct v4l2_subdev_frame_size_enum *fse)
-{
-	int index = fse->index;
-
-	if (index >= N_RES)
-		return -EINVAL;
-
-	fse->min_width = ov5693_res[index].width;
-	fse->min_height = ov5693_res[index].height;
-	fse->max_width = ov5693_res[index].width;
-	fse->max_height = ov5693_res[index].height;
-
-	return 0;
-
-}
-
-static const struct v4l2_subdev_video_ops ov5693_video_ops = {
-	.s_stream = ov5693_s_stream,
-	.g_frame_interval = ov5693_g_frame_interval,
-};
-
-static const struct v4l2_subdev_core_ops ov5693_core_ops = {
-	.s_power = ov5693_s_power,
-	.ioctl = ov5693_ioctl,
-};
-
-static const struct v4l2_subdev_pad_ops ov5693_pad_ops = {
-	.enum_mbus_code = ov5693_enum_mbus_code,
-	.enum_frame_size = ov5693_enum_frame_size,
-	.get_fmt = ov5693_get_fmt,
-	.set_fmt = ov5693_set_fmt,
-};
-
-static const struct v4l2_subdev_ops ov5693_ops = {
-	.core = &ov5693_core_ops,
-	.video = &ov5693_video_ops,
-	.pad = &ov5693_pad_ops,
-};
-
-static int ov5693_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct ov5693_device *dev = to_ov5693_sensor(sd);
-
-	dev_dbg(&client->dev, "ov5693_remove...\n");
-
-	dev->platform_data->csi_cfg(sd, 0);
-
-	v4l2_device_unregister_subdev(sd);
-
-	atomisp_gmin_remove_subdev(sd);
-
-	media_entity_cleanup(&dev->sd.entity);
-	v4l2_ctrl_handler_free(&dev->ctrl_handler);
-	kfree(dev);
-
-	return 0;
-}
-
-static int ov5693_probe(struct i2c_client *client)
-{
-	struct ov5693_device *dev;
-	int i2c;
-	int ret = 0;
-	void *pdata;
-	unsigned int i;
-
-	/*
-	 * Firmware workaround: Some modules use a "secondary default"
-	 * address of 0x10 which doesn't appear on schematics, and
-	 * some BIOS versions haven't gotten the memo.  Work around
-	 * via config.
-	 */
-	i2c = gmin_get_var_int(&client->dev, "I2CAddr", -1);
-	if (i2c != -1) {
-		dev_info(&client->dev,
-		"Overriding firmware-provided I2C address (0x%x) with 0x%x\n",
-			 client->addr, i2c);
-		client->addr = i2c;
-	}
-
-	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
-
-	mutex_init(&dev->input_lock);
-
-	dev->fmt_idx = 0;
-	v4l2_i2c_subdev_init(&(dev->sd), client, &ov5693_ops);
-
-	pdata = gmin_camera_platform_data(&dev->sd,
-					  ATOMISP_INPUT_FORMAT_RAW_10,
-					  atomisp_bayer_order_bggr);
-	if (!pdata)
-		goto out_free;
-
-	ret = ov5693_s_config(&dev->sd, client->irq, pdata);
-	if (ret)
-		goto out_free;
-
-	ret = atomisp_register_i2c_module(&dev->sd, pdata, RAW_CAMERA);
-	if (ret)
-		goto out_free;
-
-	dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	dev->pad.flags = MEDIA_PAD_FL_SOURCE;
-	dev->format.code = MEDIA_BUS_FMT_SBGGR10_1X10;
-	dev->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
-	ret =
-	    v4l2_ctrl_handler_init(&dev->ctrl_handler,
-				   ARRAY_SIZE(ov5693_controls));
-	if (ret) {
-		ov5693_remove(client);
-		return ret;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(ov5693_controls); i++)
-		v4l2_ctrl_new_custom(&dev->ctrl_handler, &ov5693_controls[i],
-				     NULL);
-
-	if (dev->ctrl_handler.error) {
-		ov5693_remove(client);
-		return dev->ctrl_handler.error;
-	}
-
-	/* Use same lock for controls as for everything else. */
-	dev->ctrl_handler.lock = &dev->input_lock;
-	dev->sd.ctrl_handler = &dev->ctrl_handler;
-
-	ret = media_entity_pads_init(&dev->sd.entity, 1, &dev->pad);
-	if (ret)
-		ov5693_remove(client);
-
-	return ret;
-out_free:
-	v4l2_device_unregister_subdev(&dev->sd);
-	kfree(dev);
-	return ret;
-}
-
-static const struct acpi_device_id ov5693_acpi_match[] = {
-	{"INT33BE"},
-	{},
-};
-MODULE_DEVICE_TABLE(acpi, ov5693_acpi_match);
-
-static struct i2c_driver ov5693_driver = {
-	.driver = {
-		.name = "ov5693",
-		.acpi_match_table = ov5693_acpi_match,
-	},
-	.probe_new = ov5693_probe,
-	.remove = ov5693_remove,
-};
-module_i2c_driver(ov5693_driver);
-
-MODULE_DESCRIPTION("A low-level driver for OmniVision 5693 sensors");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/atomisp/i2c/ov5693/ov5693.h b/drivers/staging/media/atomisp/i2c/ov5693/ov5693.h
deleted file mode 100644
index 9058a82455a6..000000000000
--- a/drivers/staging/media/atomisp/i2c/ov5693/ov5693.h
+++ /dev/null
@@ -1,1376 +0,0 @@
-/*
- * Support for OmniVision OV5693 5M camera sensor.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __OV5693_H__
-#define __OV5693_H__
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/i2c.h>
-#include <linux/delay.h>
-#include <linux/videodev2.h>
-#include <linux/spinlock.h>
-#include <media/v4l2-subdev.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-ctrls.h>
-#include <linux/v4l2-mediabus.h>
-#include <media/media-entity.h>
-
-#include "../../include/linux/atomisp_platform.h"
-
-#define OV5693_POWER_UP_RETRY_NUM 5
-
-/* Defines for register writes and register array processing */
-#define I2C_MSG_LENGTH		0x2
-#define I2C_RETRY_COUNT		5
-
-#define OV5693_FOCAL_LENGTH_NUM	334	/*3.34mm*/
-#define OV5693_FOCAL_LENGTH_DEM	100
-#define OV5693_F_NUMBER_DEFAULT_NUM	24
-#define OV5693_F_NUMBER_DEM	10
-
-#define MAX_FMTS		1
-
-/* sensor_mode_data read_mode adaptation */
-#define OV5693_READ_MODE_BINNING_ON	0x0400
-#define OV5693_READ_MODE_BINNING_OFF	0x00
-#define OV5693_INTEGRATION_TIME_MARGIN	8
-
-#define OV5693_MAX_EXPOSURE_VALUE	0xFFF1
-#define OV5693_MAX_GAIN_VALUE		0xFF
-
-/*
- * focal length bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define OV5693_FOCAL_LENGTH_DEFAULT 0x1B70064
-
-/*
- * current f-number bits definition:
- * bits 31-16: numerator, bits 15-0: denominator
- */
-#define OV5693_F_NUMBER_DEFAULT 0x18000a
-
-/*
- * f-number range bits definition:
- * bits 31-24: max f-number numerator
- * bits 23-16: max f-number denominator
- * bits 15-8: min f-number numerator
- * bits 7-0: min f-number denominator
- */
-#define OV5693_F_NUMBER_RANGE 0x180a180a
-#define OV5693_ID	0x5690
-
-#define OV5693_FINE_INTG_TIME_MIN 0
-#define OV5693_FINE_INTG_TIME_MAX_MARGIN 0
-#define OV5693_COARSE_INTG_TIME_MIN 1
-#define OV5693_COARSE_INTG_TIME_MAX_MARGIN 6
-
-#define OV5693_BIN_FACTOR_MAX 4
-/*
- * OV5693 System control registers
- */
-#define OV5693_SW_SLEEP				0x0100
-#define OV5693_SW_RESET				0x0103
-#define OV5693_SW_STREAM			0x0100
-
-#define OV5693_SC_CMMN_CHIP_ID_H		0x300A
-#define OV5693_SC_CMMN_CHIP_ID_L		0x300B
-#define OV5693_SC_CMMN_SCCB_ID			0x300C
-#define OV5693_SC_CMMN_SUB_ID			0x302A /* process, version*/
-/*Bit[7:4] Group control, Bit[3:0] Group ID*/
-#define OV5693_GROUP_ACCESS			0x3208
-/*
-*Bit[3:0] Bit[19:16] of exposure,
-*remaining 16 bits lies in Reg0x3501&Reg0x3502
-*/
-#define OV5693_EXPOSURE_H			0x3500
-#define OV5693_EXPOSURE_M			0x3501
-#define OV5693_EXPOSURE_L			0x3502
-/*Bit[1:0] means Bit[9:8] of gain*/
-#define OV5693_AGC_H				0x350A
-#define OV5693_AGC_L				0x350B /*Bit[7:0] of gain*/
-
-#define OV5693_HORIZONTAL_START_H		0x3800 /*Bit[11:8]*/
-#define OV5693_HORIZONTAL_START_L		0x3801 /*Bit[7:0]*/
-#define OV5693_VERTICAL_START_H			0x3802 /*Bit[11:8]*/
-#define OV5693_VERTICAL_START_L			0x3803 /*Bit[7:0]*/
-#define OV5693_HORIZONTAL_END_H			0x3804 /*Bit[11:8]*/
-#define OV5693_HORIZONTAL_END_L			0x3805 /*Bit[7:0]*/
-#define OV5693_VERTICAL_END_H			0x3806 /*Bit[11:8]*/
-#define OV5693_VERTICAL_END_L			0x3807 /*Bit[7:0]*/
-#define OV5693_HORIZONTAL_OUTPUT_SIZE_H		0x3808 /*Bit[3:0]*/
-#define OV5693_HORIZONTAL_OUTPUT_SIZE_L		0x3809 /*Bit[7:0]*/
-#define OV5693_VERTICAL_OUTPUT_SIZE_H		0x380a /*Bit[3:0]*/
-#define OV5693_VERTICAL_OUTPUT_SIZE_L		0x380b /*Bit[7:0]*/
-/*High 8-bit, and low 8-bit HTS address is 0x380d*/
-#define OV5693_TIMING_HTS_H			0x380C
-/*High 8-bit, and low 8-bit HTS address is 0x380d*/
-#define OV5693_TIMING_HTS_L			0x380D
-/*High 8-bit, and low 8-bit HTS address is 0x380f*/
-#define OV5693_TIMING_VTS_H			0x380e
-/*High 8-bit, and low 8-bit HTS address is 0x380f*/
-#define OV5693_TIMING_VTS_L			0x380f
-
-#define OV5693_MWB_RED_GAIN_H			0x3400
-#define OV5693_MWB_GREEN_GAIN_H			0x3402
-#define OV5693_MWB_BLUE_GAIN_H			0x3404
-#define OV5693_MWB_GAIN_MAX			0x0fff
-
-#define OV5693_START_STREAMING			0x01
-#define OV5693_STOP_STREAMING			0x00
-
-#define VCM_ADDR           0x0c
-#define VCM_CODE_MSB       0x04
-
-#define OV5693_INVALID_CONFIG	0xffffffff
-
-#define OV5693_VCM_SLEW_STEP			0x30F0
-#define OV5693_VCM_SLEW_STEP_MAX		0x7
-#define OV5693_VCM_SLEW_STEP_MASK		0x7
-#define OV5693_VCM_CODE				0x30F2
-#define OV5693_VCM_SLEW_TIME			0x30F4
-#define OV5693_VCM_SLEW_TIME_MAX		0xffff
-#define OV5693_VCM_ENABLE			0x8000
-
-#define OV5693_VCM_MAX_FOCUS_NEG       -1023
-#define OV5693_VCM_MAX_FOCUS_POS       1023
-
-#define DLC_ENABLE 1
-#define DLC_DISABLE 0
-#define VCM_PROTECTION_OFF     0xeca3
-#define VCM_PROTECTION_ON      0xdc51
-#define VCM_DEFAULT_S 0x0
-#define vcm_step_s(a) (u8)(a & 0xf)
-#define vcm_step_mclk(a) (u8)((a >> 4) & 0x3)
-#define vcm_dlc_mclk(dlc, mclk) (u16)((dlc << 3) | mclk | 0xa104)
-#define vcm_tsrc(tsrc) (u16)(tsrc << 3 | 0xf200)
-#define vcm_val(data, s) (u16)(data << 4 | s)
-#define DIRECT_VCM vcm_dlc_mclk(0, 0)
-
-/* Defines for OTP Data Registers */
-#define OV5693_FRAME_OFF_NUM		0x4202
-#define OV5693_OTP_BYTE_MAX		32	//change to 32 as needed by otpdata
-#define OV5693_OTP_SHORT_MAX		16
-#define OV5693_OTP_START_ADDR		0x3D00
-#define OV5693_OTP_END_ADDR		0x3D0F
-#define OV5693_OTP_DATA_SIZE		320
-#define OV5693_OTP_PROGRAM_REG      	0x3D80
-#define OV5693_OTP_READ_REG		0x3D81	// 1:Enable 0:disable
-#define OV5693_OTP_BANK_REG		0x3D84	//otp bank and mode
-#define OV5693_OTP_READY_REG_DONE	1
-#define OV5693_OTP_BANK_MAX		28
-#define OV5693_OTP_BANK_SIZE		16	//16 bytes per bank
-#define OV5693_OTP_READ_ONETIME		16
-#define OV5693_OTP_MODE_READ		1
-
-struct regval_list {
-	u16 reg_num;
-	u8 value;
-};
-
-struct ov5693_resolution {
-	u8 *desc;
-	const struct ov5693_reg *regs;
-	int res;
-	int width;
-	int height;
-	int fps;
-	int pix_clk_freq;
-	u16 pixels_per_line;
-	u16 lines_per_frame;
-	u8 bin_factor_x;
-	u8 bin_factor_y;
-	u8 bin_mode;
-	bool used;
-};
-
-struct ov5693_format {
-	u8 *desc;
-	u32 pixelformat;
-	struct ov5693_reg *regs;
-};
-
-enum vcm_type {
-	VCM_UNKNOWN,
-	VCM_AD5823,
-	VCM_DW9714,
-};
-
-/*
- * ov5693 device structure.
- */
-struct ov5693_device {
-	struct v4l2_subdev sd;
-	struct media_pad pad;
-	struct v4l2_mbus_framefmt format;
-	struct mutex input_lock;
-	struct v4l2_ctrl_handler ctrl_handler;
-
-	struct camera_sensor_platform_data *platform_data;
-	ktime_t timestamp_t_focus_abs;
-	int vt_pix_clk_freq_mhz;
-	int fmt_idx;
-	int run_mode;
-	int otp_size;
-	u8 *otp_data;
-	u32 focus;
-	s16 number_of_steps;
-	u8 res;
-	u8 type;
-	bool vcm_update;
-	enum vcm_type vcm;
-};
-
-enum ov5693_tok_type {
-	OV5693_8BIT  = 0x0001,
-	OV5693_16BIT = 0x0002,
-	OV5693_32BIT = 0x0004,
-	OV5693_TOK_TERM   = 0xf000,	/* terminating token for reg list */
-	OV5693_TOK_DELAY  = 0xfe00,	/* delay token for reg list */
-	OV5693_TOK_MASK = 0xfff0
-};
-
-/**
- * struct ov5693_reg - MI sensor  register format
- * @type: type of the register
- * @reg: 16-bit offset to register
- * @val: 8/16/32-bit register value
- *
- * Define a structure for sensor register initialization values
- */
-struct ov5693_reg {
-	enum ov5693_tok_type type;
-	u16 reg;
-	u32 val;	/* @set value for read/mod/write, @mask */
-};
-
-#define to_ov5693_sensor(x) container_of(x, struct ov5693_device, sd)
-
-#define OV5693_MAX_WRITE_BUF_SIZE	30
-
-struct ov5693_write_buffer {
-	u16 addr;
-	u8 data[OV5693_MAX_WRITE_BUF_SIZE];
-};
-
-struct ov5693_write_ctrl {
-	int index;
-	struct ov5693_write_buffer buffer;
-};
-
-static struct ov5693_reg const ov5693_global_setting[] = {
-	{OV5693_8BIT, 0x0103, 0x01},
-	{OV5693_8BIT, 0x3001, 0x0a},
-	{OV5693_8BIT, 0x3002, 0x80},
-	{OV5693_8BIT, 0x3006, 0x00},
-	{OV5693_8BIT, 0x3011, 0x21},
-	{OV5693_8BIT, 0x3012, 0x09},
-	{OV5693_8BIT, 0x3013, 0x10},
-	{OV5693_8BIT, 0x3014, 0x00},
-	{OV5693_8BIT, 0x3015, 0x08},
-	{OV5693_8BIT, 0x3016, 0xf0},
-	{OV5693_8BIT, 0x3017, 0xf0},
-	{OV5693_8BIT, 0x3018, 0xf0},
-	{OV5693_8BIT, 0x301b, 0xb4},
-	{OV5693_8BIT, 0x301d, 0x02},
-	{OV5693_8BIT, 0x3021, 0x00},
-	{OV5693_8BIT, 0x3022, 0x01},
-	{OV5693_8BIT, 0x3028, 0x44},
-	{OV5693_8BIT, 0x3098, 0x02},
-	{OV5693_8BIT, 0x3099, 0x19},
-	{OV5693_8BIT, 0x309a, 0x02},
-	{OV5693_8BIT, 0x309b, 0x01},
-	{OV5693_8BIT, 0x309c, 0x00},
-	{OV5693_8BIT, 0x30a0, 0xd2},
-	{OV5693_8BIT, 0x30a2, 0x01},
-	{OV5693_8BIT, 0x30b2, 0x00},
-	{OV5693_8BIT, 0x30b3, 0x7d},
-	{OV5693_8BIT, 0x30b4, 0x03},
-	{OV5693_8BIT, 0x30b5, 0x04},
-	{OV5693_8BIT, 0x30b6, 0x01},
-	{OV5693_8BIT, 0x3104, 0x21},
-	{OV5693_8BIT, 0x3106, 0x00},
-	{OV5693_8BIT, 0x3400, 0x04},
-	{OV5693_8BIT, 0x3401, 0x00},
-	{OV5693_8BIT, 0x3402, 0x04},
-	{OV5693_8BIT, 0x3403, 0x00},
-	{OV5693_8BIT, 0x3404, 0x04},
-	{OV5693_8BIT, 0x3405, 0x00},
-	{OV5693_8BIT, 0x3406, 0x01},
-	{OV5693_8BIT, 0x3500, 0x00},
-	{OV5693_8BIT, 0x3503, 0x07},
-	{OV5693_8BIT, 0x3504, 0x00},
-	{OV5693_8BIT, 0x3505, 0x00},
-	{OV5693_8BIT, 0x3506, 0x00},
-	{OV5693_8BIT, 0x3507, 0x02},
-	{OV5693_8BIT, 0x3508, 0x00},
-	{OV5693_8BIT, 0x3509, 0x10},
-	{OV5693_8BIT, 0x350a, 0x00},
-	{OV5693_8BIT, 0x350b, 0x40},
-	{OV5693_8BIT, 0x3601, 0x0a},
-	{OV5693_8BIT, 0x3602, 0x38},
-	{OV5693_8BIT, 0x3612, 0x80},
-	{OV5693_8BIT, 0x3620, 0x54},
-	{OV5693_8BIT, 0x3621, 0xc7},
-	{OV5693_8BIT, 0x3622, 0x0f},
-	{OV5693_8BIT, 0x3625, 0x10},
-	{OV5693_8BIT, 0x3630, 0x55},
-	{OV5693_8BIT, 0x3631, 0xf4},
-	{OV5693_8BIT, 0x3632, 0x00},
-	{OV5693_8BIT, 0x3633, 0x34},
-	{OV5693_8BIT, 0x3634, 0x02},
-	{OV5693_8BIT, 0x364d, 0x0d},
-	{OV5693_8BIT, 0x364f, 0xdd},
-	{OV5693_8BIT, 0x3660, 0x04},
-	{OV5693_8BIT, 0x3662, 0x10},
-	{OV5693_8BIT, 0x3663, 0xf1},
-	{OV5693_8BIT, 0x3665, 0x00},
-	{OV5693_8BIT, 0x3666, 0x20},
-	{OV5693_8BIT, 0x3667, 0x00},
-	{OV5693_8BIT, 0x366a, 0x80},
-	{OV5693_8BIT, 0x3680, 0xe0},
-	{OV5693_8BIT, 0x3681, 0x00},
-	{OV5693_8BIT, 0x3700, 0x42},
-	{OV5693_8BIT, 0x3701, 0x14},
-	{OV5693_8BIT, 0x3702, 0xa0},
-	{OV5693_8BIT, 0x3703, 0xd8},
-	{OV5693_8BIT, 0x3704, 0x78},
-	{OV5693_8BIT, 0x3705, 0x02},
-	{OV5693_8BIT, 0x370a, 0x00},
-	{OV5693_8BIT, 0x370b, 0x20},
-	{OV5693_8BIT, 0x370c, 0x0c},
-	{OV5693_8BIT, 0x370d, 0x11},
-	{OV5693_8BIT, 0x370e, 0x00},
-	{OV5693_8BIT, 0x370f, 0x40},
-	{OV5693_8BIT, 0x3710, 0x00},
-	{OV5693_8BIT, 0x371a, 0x1c},
-	{OV5693_8BIT, 0x371b, 0x05},
-	{OV5693_8BIT, 0x371c, 0x01},
-	{OV5693_8BIT, 0x371e, 0xa1},
-	{OV5693_8BIT, 0x371f, 0x0c},
-	{OV5693_8BIT, 0x3721, 0x00},
-	{OV5693_8BIT, 0x3724, 0x10},
-	{OV5693_8BIT, 0x3726, 0x00},
-	{OV5693_8BIT, 0x372a, 0x01},
-	{OV5693_8BIT, 0x3730, 0x10},
-	{OV5693_8BIT, 0x3738, 0x22},
-	{OV5693_8BIT, 0x3739, 0xe5},
-	{OV5693_8BIT, 0x373a, 0x50},
-	{OV5693_8BIT, 0x373b, 0x02},
-	{OV5693_8BIT, 0x373c, 0x41},
-	{OV5693_8BIT, 0x373f, 0x02},
-	{OV5693_8BIT, 0x3740, 0x42},
-	{OV5693_8BIT, 0x3741, 0x02},
-	{OV5693_8BIT, 0x3742, 0x18},
-	{OV5693_8BIT, 0x3743, 0x01},
-	{OV5693_8BIT, 0x3744, 0x02},
-	{OV5693_8BIT, 0x3747, 0x10},
-	{OV5693_8BIT, 0x374c, 0x04},
-	{OV5693_8BIT, 0x3751, 0xf0},
-	{OV5693_8BIT, 0x3752, 0x00},
-	{OV5693_8BIT, 0x3753, 0x00},
-	{OV5693_8BIT, 0x3754, 0xc0},
-	{OV5693_8BIT, 0x3755, 0x00},
-	{OV5693_8BIT, 0x3756, 0x1a},
-	{OV5693_8BIT, 0x3758, 0x00},
-	{OV5693_8BIT, 0x3759, 0x0f},
-	{OV5693_8BIT, 0x376b, 0x44},
-	{OV5693_8BIT, 0x375c, 0x04},
-	{OV5693_8BIT, 0x3774, 0x10},
-	{OV5693_8BIT, 0x3776, 0x00},
-	{OV5693_8BIT, 0x377f, 0x08},
-	{OV5693_8BIT, 0x3780, 0x22},
-	{OV5693_8BIT, 0x3781, 0x0c},
-	{OV5693_8BIT, 0x3784, 0x2c},
-	{OV5693_8BIT, 0x3785, 0x1e},
-	{OV5693_8BIT, 0x378f, 0xf5},
-	{OV5693_8BIT, 0x3791, 0xb0},
-	{OV5693_8BIT, 0x3795, 0x00},
-	{OV5693_8BIT, 0x3796, 0x64},
-	{OV5693_8BIT, 0x3797, 0x11},
-	{OV5693_8BIT, 0x3798, 0x30},
-	{OV5693_8BIT, 0x3799, 0x41},
-	{OV5693_8BIT, 0x379a, 0x07},
-	{OV5693_8BIT, 0x379b, 0xb0},
-	{OV5693_8BIT, 0x379c, 0x0c},
-	{OV5693_8BIT, 0x37c5, 0x00},
-	{OV5693_8BIT, 0x37c6, 0x00},
-	{OV5693_8BIT, 0x37c7, 0x00},
-	{OV5693_8BIT, 0x37c9, 0x00},
-	{OV5693_8BIT, 0x37ca, 0x00},
-	{OV5693_8BIT, 0x37cb, 0x00},
-	{OV5693_8BIT, 0x37de, 0x00},
-	{OV5693_8BIT, 0x37df, 0x00},
-	{OV5693_8BIT, 0x3800, 0x00},
-	{OV5693_8BIT, 0x3801, 0x00},
-	{OV5693_8BIT, 0x3802, 0x00},
-	{OV5693_8BIT, 0x3804, 0x0a},
-	{OV5693_8BIT, 0x3805, 0x3f},
-	{OV5693_8BIT, 0x3810, 0x00},
-	{OV5693_8BIT, 0x3812, 0x00},
-	{OV5693_8BIT, 0x3823, 0x00},
-	{OV5693_8BIT, 0x3824, 0x00},
-	{OV5693_8BIT, 0x3825, 0x00},
-	{OV5693_8BIT, 0x3826, 0x00},
-	{OV5693_8BIT, 0x3827, 0x00},
-	{OV5693_8BIT, 0x382a, 0x04},
-	{OV5693_8BIT, 0x3a04, 0x06},
-	{OV5693_8BIT, 0x3a05, 0x14},
-	{OV5693_8BIT, 0x3a06, 0x00},
-	{OV5693_8BIT, 0x3a07, 0xfe},
-	{OV5693_8BIT, 0x3b00, 0x00},
-	{OV5693_8BIT, 0x3b02, 0x00},
-	{OV5693_8BIT, 0x3b03, 0x00},
-	{OV5693_8BIT, 0x3b04, 0x00},
-	{OV5693_8BIT, 0x3b05, 0x00},
-	{OV5693_8BIT, 0x3e07, 0x20},
-	{OV5693_8BIT, 0x4000, 0x08},
-	{OV5693_8BIT, 0x4001, 0x04},
-	{OV5693_8BIT, 0x4002, 0x45},
-	{OV5693_8BIT, 0x4004, 0x08},
-	{OV5693_8BIT, 0x4005, 0x18},
-	{OV5693_8BIT, 0x4006, 0x20},
-	{OV5693_8BIT, 0x4008, 0x24},
-	{OV5693_8BIT, 0x4009, 0x10},
-	{OV5693_8BIT, 0x400c, 0x00},
-	{OV5693_8BIT, 0x400d, 0x00},
-	{OV5693_8BIT, 0x4058, 0x00},
-	{OV5693_8BIT, 0x404e, 0x37},
-	{OV5693_8BIT, 0x404f, 0x8f},
-	{OV5693_8BIT, 0x4058, 0x00},
-	{OV5693_8BIT, 0x4101, 0xb2},
-	{OV5693_8BIT, 0x4303, 0x00},
-	{OV5693_8BIT, 0x4304, 0x08},
-	{OV5693_8BIT, 0x4307, 0x31},
-	{OV5693_8BIT, 0x4311, 0x04},
-	{OV5693_8BIT, 0x4315, 0x01},
-	{OV5693_8BIT, 0x4511, 0x05},
-	{OV5693_8BIT, 0x4512, 0x01},
-	{OV5693_8BIT, 0x4806, 0x00},
-	{OV5693_8BIT, 0x4816, 0x52},
-	{OV5693_8BIT, 0x481f, 0x30},
-	{OV5693_8BIT, 0x4826, 0x2c},
-	{OV5693_8BIT, 0x4831, 0x64},
-	{OV5693_8BIT, 0x4d00, 0x04},
-	{OV5693_8BIT, 0x4d01, 0x71},
-	{OV5693_8BIT, 0x4d02, 0xfd},
-	{OV5693_8BIT, 0x4d03, 0xf5},
-	{OV5693_8BIT, 0x4d04, 0x0c},
-	{OV5693_8BIT, 0x4d05, 0xcc},
-	{OV5693_8BIT, 0x4837, 0x0a},
-	{OV5693_8BIT, 0x5000, 0x06},
-	{OV5693_8BIT, 0x5001, 0x01},
-	{OV5693_8BIT, 0x5003, 0x20},
-	{OV5693_8BIT, 0x5046, 0x0a},
-	{OV5693_8BIT, 0x5013, 0x00},
-	{OV5693_8BIT, 0x5046, 0x0a},
-	{OV5693_8BIT, 0x5780, 0x1c},
-	{OV5693_8BIT, 0x5786, 0x20},
-	{OV5693_8BIT, 0x5787, 0x10},
-	{OV5693_8BIT, 0x5788, 0x18},
-	{OV5693_8BIT, 0x578a, 0x04},
-	{OV5693_8BIT, 0x578b, 0x02},
-	{OV5693_8BIT, 0x578c, 0x02},
-	{OV5693_8BIT, 0x578e, 0x06},
-	{OV5693_8BIT, 0x578f, 0x02},
-	{OV5693_8BIT, 0x5790, 0x02},
-	{OV5693_8BIT, 0x5791, 0xff},
-	{OV5693_8BIT, 0x5842, 0x01},
-	{OV5693_8BIT, 0x5843, 0x2b},
-	{OV5693_8BIT, 0x5844, 0x01},
-	{OV5693_8BIT, 0x5845, 0x92},
-	{OV5693_8BIT, 0x5846, 0x01},
-	{OV5693_8BIT, 0x5847, 0x8f},
-	{OV5693_8BIT, 0x5848, 0x01},
-	{OV5693_8BIT, 0x5849, 0x0c},
-	{OV5693_8BIT, 0x5e00, 0x00},
-	{OV5693_8BIT, 0x5e10, 0x0c},
-	{OV5693_8BIT, 0x0100, 0x00},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-/*
- * 654x496 30fps 17ms VBlanking 2lane 10Bit (Scaling)
- */
-static struct ov5693_reg const ov5693_654x496[] = {
-	{OV5693_8BIT, 0x3501, 0x3d},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe6},
-	{OV5693_8BIT, 0x3709, 0xc7},
-	{OV5693_8BIT, 0x3803, 0x00},
-	{OV5693_8BIT, 0x3806, 0x07},
-	{OV5693_8BIT, 0x3807, 0xa3},
-	{OV5693_8BIT, 0x3808, 0x02},
-	{OV5693_8BIT, 0x3809, 0x90},
-	{OV5693_8BIT, 0x380a, 0x01},
-	{OV5693_8BIT, 0x380b, 0xf0},
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x08},
-	{OV5693_8BIT, 0x3813, 0x02},
-	{OV5693_8BIT, 0x3814, 0x31},
-	{OV5693_8BIT, 0x3815, 0x31},
-	{OV5693_8BIT, 0x3820, 0x04},
-	{OV5693_8BIT, 0x3821, 0x1f},
-	{OV5693_8BIT, 0x5002, 0x80},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-/*
- * 1296x976 30fps 17ms VBlanking 2lane 10Bit (Scaling)
-*DS from 2592x1952
-*/
-static struct ov5693_reg const ov5693_1296x976[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-
-	{OV5693_8BIT, 0x3800, 0x00},
-	{OV5693_8BIT, 0x3801, 0x00},
-	{OV5693_8BIT, 0x3802, 0x00},
-	{OV5693_8BIT, 0x3803, 0x00},
-
-	{OV5693_8BIT, 0x3804, 0x0a},
-	{OV5693_8BIT, 0x3805, 0x3f},
-	{OV5693_8BIT, 0x3806, 0x07},
-	{OV5693_8BIT, 0x3807, 0xA3},
-
-	{OV5693_8BIT, 0x3808, 0x05},
-	{OV5693_8BIT, 0x3809, 0x10},
-	{OV5693_8BIT, 0x380a, 0x03},
-	{OV5693_8BIT, 0x380b, 0xD0},
-
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-
-	{OV5693_8BIT, 0x3810, 0x00},
-	{OV5693_8BIT, 0x3811, 0x10},
-	{OV5693_8BIT, 0x3812, 0x00},
-	{OV5693_8BIT, 0x3813, 0x02},
-
-	{OV5693_8BIT, 0x3814, 0x11},	/*X subsample control*/
-	{OV5693_8BIT, 0x3815, 0x11},	/*Y subsample control*/
-	{OV5693_8BIT, 0x3820, 0x00},
-	{OV5693_8BIT, 0x3821, 0x1e},
-	{OV5693_8BIT, 0x5002, 0x00},
-	{OV5693_8BIT, 0x5041, 0x84}, /* scale is auto enabled */
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-
-};
-
-
-/*
- * 336x256 30fps 17ms VBlanking 2lane 10Bit (Scaling)
- DS from 2564x1956
- */
-static struct ov5693_reg const ov5693_336x256[] = {
-	{OV5693_8BIT, 0x3501, 0x3d},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe6},
-	{OV5693_8BIT, 0x3709, 0xc7},
-	{OV5693_8BIT, 0x3806, 0x07},
-	{OV5693_8BIT, 0x3807, 0xa3},
-	{OV5693_8BIT, 0x3808, 0x01},
-	{OV5693_8BIT, 0x3809, 0x50},
-	{OV5693_8BIT, 0x380a, 0x01},
-	{OV5693_8BIT, 0x380b, 0x00},
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x1E},
-	{OV5693_8BIT, 0x3814, 0x31},
-	{OV5693_8BIT, 0x3815, 0x31},
-	{OV5693_8BIT, 0x3820, 0x04},
-	{OV5693_8BIT, 0x3821, 0x1f},
-	{OV5693_8BIT, 0x5002, 0x80},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-/*
- * 336x256 30fps 17ms VBlanking 2lane 10Bit (Scaling)
- DS from 2368x1956
- */
-static struct ov5693_reg const ov5693_368x304[] = {
-	{OV5693_8BIT, 0x3501, 0x3d},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe6},
-	{OV5693_8BIT, 0x3709, 0xc7},
-	{OV5693_8BIT, 0x3808, 0x01},
-	{OV5693_8BIT, 0x3809, 0x70},
-	{OV5693_8BIT, 0x380a, 0x01},
-	{OV5693_8BIT, 0x380b, 0x30},
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x80},
-	{OV5693_8BIT, 0x3814, 0x31},
-	{OV5693_8BIT, 0x3815, 0x31},
-	{OV5693_8BIT, 0x3820, 0x04},
-	{OV5693_8BIT, 0x3821, 0x1f},
-	{OV5693_8BIT, 0x5002, 0x80},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-/*
- * ov5693_192x160 30fps 17ms VBlanking 2lane 10Bit (Scaling)
- DS from 2460x1956
- */
-static struct ov5693_reg const ov5693_192x160[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x80},
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3804, 0x0a},
-	{OV5693_8BIT, 0x3805, 0x3f},
-	{OV5693_8BIT, 0x3806, 0x07},
-	{OV5693_8BIT, 0x3807, 0xA3},
-	{OV5693_8BIT, 0x3808, 0x00},
-	{OV5693_8BIT, 0x3809, 0xC0},
-	{OV5693_8BIT, 0x380a, 0x00},
-	{OV5693_8BIT, 0x380b, 0xA0},
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x40},
-	{OV5693_8BIT, 0x3813, 0x00},
-	{OV5693_8BIT, 0x3814, 0x31},
-	{OV5693_8BIT, 0x3815, 0x31},
-	{OV5693_8BIT, 0x3820, 0x04},
-	{OV5693_8BIT, 0x3821, 0x1f},
-	{OV5693_8BIT, 0x5002, 0x80},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-
-static struct ov5693_reg const ov5693_736x496[] = {
-	{OV5693_8BIT, 0x3501, 0x3d},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe6},
-	{OV5693_8BIT, 0x3709, 0xc7},
-	{OV5693_8BIT, 0x3803, 0x68},
-	{OV5693_8BIT, 0x3806, 0x07},
-	{OV5693_8BIT, 0x3807, 0x3b},
-	{OV5693_8BIT, 0x3808, 0x02},
-	{OV5693_8BIT, 0x3809, 0xe0},
-	{OV5693_8BIT, 0x380a, 0x01},
-	{OV5693_8BIT, 0x380b, 0xf0},
-	{OV5693_8BIT, 0x380c, 0x0a}, /*hts*/
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07}, /*vts*/
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x08},
-	{OV5693_8BIT, 0x3813, 0x02},
-	{OV5693_8BIT, 0x3814, 0x31},
-	{OV5693_8BIT, 0x3815, 0x31},
-	{OV5693_8BIT, 0x3820, 0x04},
-	{OV5693_8BIT, 0x3821, 0x1f},
-	{OV5693_8BIT, 0x5002, 0x80},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-/*
-static struct ov5693_reg const ov5693_736x496[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe6},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3803, 0x00},
-	{OV5693_8BIT, 0x3806, 0x07},
-	{OV5693_8BIT, 0x3807, 0xa3},
-	{OV5693_8BIT, 0x3808, 0x02},
-	{OV5693_8BIT, 0x3809, 0xe0},
-	{OV5693_8BIT, 0x380a, 0x01},
-	{OV5693_8BIT, 0x380b, 0xf0},
-	{OV5693_8BIT, 0x380c, 0x0d},
-	{OV5693_8BIT, 0x380d, 0xb0},
-	{OV5693_8BIT, 0x380e, 0x05},
-	{OV5693_8BIT, 0x380f, 0xf2},
-	{OV5693_8BIT, 0x3811, 0x08},
-	{OV5693_8BIT, 0x3813, 0x02},
-	{OV5693_8BIT, 0x3814, 0x31},
-	{OV5693_8BIT, 0x3815, 0x31},
-	{OV5693_8BIT, 0x3820, 0x01},
-	{OV5693_8BIT, 0x3821, 0x1f},
-	{OV5693_8BIT, 0x5002, 0x00},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-*/
-/*
- * 976x556 30fps 8.8ms VBlanking 2lane 10Bit (Scaling)
- */
-static struct ov5693_reg const ov5693_976x556[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3803, 0xf0},
-	{OV5693_8BIT, 0x3806, 0x06},
-	{OV5693_8BIT, 0x3807, 0xa7},
-	{OV5693_8BIT, 0x3808, 0x03},
-	{OV5693_8BIT, 0x3809, 0xd0},
-	{OV5693_8BIT, 0x380a, 0x02},
-	{OV5693_8BIT, 0x380b, 0x2C},
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x10},
-	{OV5693_8BIT, 0x3813, 0x02},
-	{OV5693_8BIT, 0x3814, 0x11},
-	{OV5693_8BIT, 0x3815, 0x11},
-	{OV5693_8BIT, 0x3820, 0x00},
-	{OV5693_8BIT, 0x3821, 0x1e},
-	{OV5693_8BIT, 0x5002, 0x80},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-/*DS from 2624x1492*/
-static struct ov5693_reg const ov5693_1296x736[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-
-	{OV5693_8BIT, 0x3800, 0x00},
-	{OV5693_8BIT, 0x3801, 0x00},
-	{OV5693_8BIT, 0x3802, 0x00},
-	{OV5693_8BIT, 0x3803, 0x00},
-
-	{OV5693_8BIT, 0x3804, 0x0a},
-	{OV5693_8BIT, 0x3805, 0x3f},
-	{OV5693_8BIT, 0x3806, 0x07},
-	{OV5693_8BIT, 0x3807, 0xA3},
-
-	{OV5693_8BIT, 0x3808, 0x05},
-	{OV5693_8BIT, 0x3809, 0x10},
-	{OV5693_8BIT, 0x380a, 0x02},
-	{OV5693_8BIT, 0x380b, 0xe0},
-
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-
-	{OV5693_8BIT, 0x3813, 0xE8},
-
-	{OV5693_8BIT, 0x3814, 0x11},	/*X subsample control*/
-	{OV5693_8BIT, 0x3815, 0x11},	/*Y subsample control*/
-	{OV5693_8BIT, 0x3820, 0x00},
-	{OV5693_8BIT, 0x3821, 0x1e},
-	{OV5693_8BIT, 0x5002, 0x00},
-	{OV5693_8BIT, 0x5041, 0x84}, /* scale is auto enabled */
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-static struct ov5693_reg const ov5693_1636p_30fps[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3803, 0xf0},
-	{OV5693_8BIT, 0x3806, 0x06},
-	{OV5693_8BIT, 0x3807, 0xa7},
-	{OV5693_8BIT, 0x3808, 0x06},
-	{OV5693_8BIT, 0x3809, 0x64},
-	{OV5693_8BIT, 0x380a, 0x04},
-	{OV5693_8BIT, 0x380b, 0x48},
-	{OV5693_8BIT, 0x380c, 0x0a}, /*hts*/
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07}, /*vts*/
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x02},
-	{OV5693_8BIT, 0x3813, 0x02},
-	{OV5693_8BIT, 0x3814, 0x11},
-	{OV5693_8BIT, 0x3815, 0x11},
-	{OV5693_8BIT, 0x3820, 0x00},
-	{OV5693_8BIT, 0x3821, 0x1e},
-	{OV5693_8BIT, 0x5002, 0x80},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-static struct ov5693_reg const ov5693_1616x1216_30fps[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x80},
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3800, 0x00},	/*{3800,3801} Array X start*/
-	{OV5693_8BIT, 0x3801, 0x08},	/* 04 //{3800,3801} Array X start*/
-	{OV5693_8BIT, 0x3802, 0x00},	/*{3802,3803} Array Y start*/
-	{OV5693_8BIT, 0x3803, 0x04},	/* 00  //{3802,3803} Array Y start*/
-	{OV5693_8BIT, 0x3804, 0x0a},	/*{3804,3805} Array X end*/
-	{OV5693_8BIT, 0x3805, 0x37},	/* 3b  //{3804,3805} Array X end*/
-	{OV5693_8BIT, 0x3806, 0x07},	/*{3806,3807} Array Y end*/
-	{OV5693_8BIT, 0x3807, 0x9f},	/* a3  //{3806,3807} Array Y end*/
-	{OV5693_8BIT, 0x3808, 0x06},	/*{3808,3809} Final output H size*/
-	{OV5693_8BIT, 0x3809, 0x50},	/*{3808,3809} Final output H size*/
-	{OV5693_8BIT, 0x380a, 0x04},	/*{380a,380b} Final output V size*/
-	{OV5693_8BIT, 0x380b, 0xc0},	/*{380a,380b} Final output V size*/
-	{OV5693_8BIT, 0x380c, 0x0a},	/*{380c,380d} HTS*/
-	{OV5693_8BIT, 0x380d, 0x80},	/*{380c,380d} HTS*/
-	{OV5693_8BIT, 0x380e, 0x07},	/*{380e,380f} VTS*/
-	{OV5693_8BIT, 0x380f, 0xc0},	/* bc	//{380e,380f} VTS*/
-	{OV5693_8BIT, 0x3810, 0x00},	/*{3810,3811} windowing X offset*/
-	{OV5693_8BIT, 0x3811, 0x10},	/*{3810,3811} windowing X offset*/
-	{OV5693_8BIT, 0x3812, 0x00},	/*{3812,3813} windowing Y offset*/
-	{OV5693_8BIT, 0x3813, 0x06},	/*{3812,3813} windowing Y offset*/
-	{OV5693_8BIT, 0x3814, 0x11},	/*X subsample control*/
-	{OV5693_8BIT, 0x3815, 0x11},	/*Y subsample control*/
-	{OV5693_8BIT, 0x3820, 0x00},	/*FLIP/Binnning control*/
-	{OV5693_8BIT, 0x3821, 0x1e},	/*MIRROR control*/
-	{OV5693_8BIT, 0x5002, 0x00},
-	{OV5693_8BIT, 0x5041, 0x84},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-
-/*
- * 1940x1096 30fps 8.8ms VBlanking 2lane 10bit (Scaling)
- */
-static struct ov5693_reg const ov5693_1940x1096[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3803, 0xf0},
-	{OV5693_8BIT, 0x3806, 0x06},
-	{OV5693_8BIT, 0x3807, 0xa7},
-	{OV5693_8BIT, 0x3808, 0x07},
-	{OV5693_8BIT, 0x3809, 0x94},
-	{OV5693_8BIT, 0x380a, 0x04},
-	{OV5693_8BIT, 0x380b, 0x48},
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x02},
-	{OV5693_8BIT, 0x3813, 0x02},
-	{OV5693_8BIT, 0x3814, 0x11},
-	{OV5693_8BIT, 0x3815, 0x11},
-	{OV5693_8BIT, 0x3820, 0x00},
-	{OV5693_8BIT, 0x3821, 0x1e},
-	{OV5693_8BIT, 0x5002, 0x80},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-static struct ov5693_reg const ov5693_2592x1456_30fps[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3800, 0x00},
-	{OV5693_8BIT, 0x3801, 0x00},
-	{OV5693_8BIT, 0x3802, 0x00},
-	{OV5693_8BIT, 0x3803, 0xf0},
-	{OV5693_8BIT, 0x3804, 0x0a},
-	{OV5693_8BIT, 0x3805, 0x3f},
-	{OV5693_8BIT, 0x3806, 0x06},
-	{OV5693_8BIT, 0x3807, 0xa4},
-	{OV5693_8BIT, 0x3808, 0x0a},
-	{OV5693_8BIT, 0x3809, 0x20},
-	{OV5693_8BIT, 0x380a, 0x05},
-	{OV5693_8BIT, 0x380b, 0xb0},
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x10},
-	{OV5693_8BIT, 0x3813, 0x00},
-	{OV5693_8BIT, 0x3814, 0x11},
-	{OV5693_8BIT, 0x3815, 0x11},
-	{OV5693_8BIT, 0x3820, 0x00},
-	{OV5693_8BIT, 0x3821, 0x1e},
-	{OV5693_8BIT, 0x5002, 0x00},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-static struct ov5693_reg const ov5693_2576x1456_30fps[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3800, 0x00},
-	{OV5693_8BIT, 0x3801, 0x00},
-	{OV5693_8BIT, 0x3802, 0x00},
-	{OV5693_8BIT, 0x3803, 0xf0},
-	{OV5693_8BIT, 0x3804, 0x0a},
-	{OV5693_8BIT, 0x3805, 0x3f},
-	{OV5693_8BIT, 0x3806, 0x06},
-	{OV5693_8BIT, 0x3807, 0xa4},
-	{OV5693_8BIT, 0x3808, 0x0a},
-	{OV5693_8BIT, 0x3809, 0x10},
-	{OV5693_8BIT, 0x380a, 0x05},
-	{OV5693_8BIT, 0x380b, 0xb0},
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x18},
-	{OV5693_8BIT, 0x3813, 0x00},
-	{OV5693_8BIT, 0x3814, 0x11},
-	{OV5693_8BIT, 0x3815, 0x11},
-	{OV5693_8BIT, 0x3820, 0x00},
-	{OV5693_8BIT, 0x3821, 0x1e},
-	{OV5693_8BIT, 0x5002, 0x00},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-/*
- * 2592x1944 30fps 0.6ms VBlanking 2lane 10Bit
- */
-static struct ov5693_reg const ov5693_2592x1944_30fps[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3803, 0x00},
-	{OV5693_8BIT, 0x3806, 0x07},
-	{OV5693_8BIT, 0x3807, 0xa3},
-	{OV5693_8BIT, 0x3808, 0x0a},
-	{OV5693_8BIT, 0x3809, 0x20},
-	{OV5693_8BIT, 0x380a, 0x07},
-	{OV5693_8BIT, 0x380b, 0x98},
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x10},
-	{OV5693_8BIT, 0x3813, 0x00},
-	{OV5693_8BIT, 0x3814, 0x11},
-	{OV5693_8BIT, 0x3815, 0x11},
-	{OV5693_8BIT, 0x3820, 0x00},
-	{OV5693_8BIT, 0x3821, 0x1e},
-	{OV5693_8BIT, 0x5002, 0x00},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-/*
- * 11:9 Full FOV Output, expected FOV Res: 2346x1920
- * ISP Effect Res: 1408x1152
- * Sensor out: 1424x1168, DS From: 2380x1952
- *
- * WA: Left Offset: 8, Hor scal: 64
- */
-static struct ov5693_reg const ov5693_1424x1168_30fps[] = {
-	{OV5693_8BIT, 0x3501, 0x3b}, /* long exposure[15:8] */
-	{OV5693_8BIT, 0x3502, 0x80}, /* long exposure[7:0] */
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3800, 0x00}, /* TIMING_X_ADDR_START */
-	{OV5693_8BIT, 0x3801, 0x50}, /* 80 */
-	{OV5693_8BIT, 0x3802, 0x00}, /* TIMING_Y_ADDR_START */
-	{OV5693_8BIT, 0x3803, 0x02}, /* 2 */
-	{OV5693_8BIT, 0x3804, 0x09}, /* TIMING_X_ADDR_END */
-	{OV5693_8BIT, 0x3805, 0xdd}, /* 2525 */
-	{OV5693_8BIT, 0x3806, 0x07}, /* TIMING_Y_ADDR_END */
-	{OV5693_8BIT, 0x3807, 0xa1}, /* 1953 */
-	{OV5693_8BIT, 0x3808, 0x05}, /* TIMING_X_OUTPUT_SIZE */
-	{OV5693_8BIT, 0x3809, 0x90}, /* 1424 */
-	{OV5693_8BIT, 0x380a, 0x04}, /* TIMING_Y_OUTPUT_SIZE */
-	{OV5693_8BIT, 0x380b, 0x90}, /* 1168 */
-	{OV5693_8BIT, 0x380c, 0x0a}, /* TIMING_HTS */
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07}, /* TIMING_VTS */
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3810, 0x00}, /* TIMING_ISP_X_WIN */
-	{OV5693_8BIT, 0x3811, 0x02}, /* 2 */
-	{OV5693_8BIT, 0x3812, 0x00}, /* TIMING_ISP_Y_WIN */
-	{OV5693_8BIT, 0x3813, 0x00}, /* 0 */
-	{OV5693_8BIT, 0x3814, 0x11}, /* TIME_X_INC */
-	{OV5693_8BIT, 0x3815, 0x11}, /* TIME_Y_INC */
-	{OV5693_8BIT, 0x3820, 0x00},
-	{OV5693_8BIT, 0x3821, 0x1e},
-	{OV5693_8BIT, 0x5002, 0x00},
-	{OV5693_8BIT, 0x5041, 0x84}, /* scale is auto enabled */
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-/*
- * 3:2 Full FOV Output, expected FOV Res: 2560x1706
- * ISP Effect Res: 720x480
- * Sensor out: 736x496, DS From 2616x1764
- */
-static struct ov5693_reg const ov5693_736x496_30fps[] = {
-	{OV5693_8BIT, 0x3501, 0x3b}, /* long exposure[15:8] */
-	{OV5693_8BIT, 0x3502, 0x80}, /* long exposure[7:0] */
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3800, 0x00}, /* TIMING_X_ADDR_START */
-	{OV5693_8BIT, 0x3801, 0x02}, /* 2 */
-	{OV5693_8BIT, 0x3802, 0x00}, /* TIMING_Y_ADDR_START */
-	{OV5693_8BIT, 0x3803, 0x62}, /* 98 */
-	{OV5693_8BIT, 0x3804, 0x0a}, /* TIMING_X_ADDR_END */
-	{OV5693_8BIT, 0x3805, 0x3b}, /* 2619 */
-	{OV5693_8BIT, 0x3806, 0x07}, /* TIMING_Y_ADDR_END */
-	{OV5693_8BIT, 0x3807, 0x43}, /* 1859 */
-	{OV5693_8BIT, 0x3808, 0x02}, /* TIMING_X_OUTPUT_SIZE */
-	{OV5693_8BIT, 0x3809, 0xe0}, /* 736 */
-	{OV5693_8BIT, 0x380a, 0x01}, /* TIMING_Y_OUTPUT_SIZE */
-	{OV5693_8BIT, 0x380b, 0xf0}, /* 496 */
-	{OV5693_8BIT, 0x380c, 0x0a}, /* TIMING_HTS */
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07}, /* TIMING_VTS */
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3810, 0x00}, /* TIMING_ISP_X_WIN */
-	{OV5693_8BIT, 0x3811, 0x02}, /* 2 */
-	{OV5693_8BIT, 0x3812, 0x00}, /* TIMING_ISP_Y_WIN */
-	{OV5693_8BIT, 0x3813, 0x00}, /* 0 */
-	{OV5693_8BIT, 0x3814, 0x11}, /* TIME_X_INC */
-	{OV5693_8BIT, 0x3815, 0x11}, /* TIME_Y_INC */
-	{OV5693_8BIT, 0x3820, 0x00},
-	{OV5693_8BIT, 0x3821, 0x1e},
-	{OV5693_8BIT, 0x5002, 0x00},
-	{OV5693_8BIT, 0x5041, 0x84}, /* scale is auto enabled */
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-static struct ov5693_reg const ov5693_2576x1936_30fps[] = {
-	{OV5693_8BIT, 0x3501, 0x7b},
-	{OV5693_8BIT, 0x3502, 0x00},
-	{OV5693_8BIT, 0x3708, 0xe2},
-	{OV5693_8BIT, 0x3709, 0xc3},
-	{OV5693_8BIT, 0x3803, 0x00},
-	{OV5693_8BIT, 0x3806, 0x07},
-	{OV5693_8BIT, 0x3807, 0xa3},
-	{OV5693_8BIT, 0x3808, 0x0a},
-	{OV5693_8BIT, 0x3809, 0x10},
-	{OV5693_8BIT, 0x380a, 0x07},
-	{OV5693_8BIT, 0x380b, 0x90},
-	{OV5693_8BIT, 0x380c, 0x0a},
-	{OV5693_8BIT, 0x380d, 0x80},
-	{OV5693_8BIT, 0x380e, 0x07},
-	{OV5693_8BIT, 0x380f, 0xc0},
-	{OV5693_8BIT, 0x3811, 0x18},
-	{OV5693_8BIT, 0x3813, 0x00},
-	{OV5693_8BIT, 0x3814, 0x11},
-	{OV5693_8BIT, 0x3815, 0x11},
-	{OV5693_8BIT, 0x3820, 0x00},
-	{OV5693_8BIT, 0x3821, 0x1e},
-	{OV5693_8BIT, 0x5002, 0x00},
-	{OV5693_8BIT, 0x0100, 0x01},
-	{OV5693_TOK_TERM, 0, 0}
-};
-
-static struct ov5693_resolution ov5693_res_preview[] = {
-	{
-		.desc = "ov5693_736x496_30fps",
-		.width = 736,
-		.height = 496,
-		.pix_clk_freq = 160,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_736x496_30fps,
-	},
-	{
-		.desc = "ov5693_1616x1216_30fps",
-		.width = 1616,
-		.height = 1216,
-		.pix_clk_freq = 160,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_1616x1216_30fps,
-	},
-	{
-		.desc = "ov5693_5M_30fps",
-		.width = 2576,
-		.height = 1456,
-		.pix_clk_freq = 160,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_2576x1456_30fps,
-	},
-	{
-		.desc = "ov5693_5M_30fps",
-		.width = 2576,
-		.height = 1936,
-		.pix_clk_freq = 160,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_2576x1936_30fps,
-	},
-};
-#define N_RES_PREVIEW (ARRAY_SIZE(ov5693_res_preview))
-
-/*
- * Disable non-preview configurations until the configuration selection is
- * improved.
- */
-#if 0
-struct ov5693_resolution ov5693_res_still[] = {
-	{
-		.desc = "ov5693_736x496_30fps",
-		.width = 736,
-		.height = 496,
-		.pix_clk_freq = 160,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_736x496_30fps,
-	},
-	{
-		.desc = "ov5693_1424x1168_30fps",
-		.width = 1424,
-		.height = 1168,
-		.pix_clk_freq = 160,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_1424x1168_30fps,
-	},
-	{
-		.desc = "ov5693_1616x1216_30fps",
-		.width = 1616,
-		.height = 1216,
-		.pix_clk_freq = 160,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_1616x1216_30fps,
-	},
-	{
-		.desc = "ov5693_5M_30fps",
-		.width = 2592,
-		.height = 1456,
-		.pix_clk_freq = 160,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_2592x1456_30fps,
-	},
-	{
-		.desc = "ov5693_5M_30fps",
-		.width = 2592,
-		.height = 1944,
-		.pix_clk_freq = 160,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_2592x1944_30fps,
-	},
-};
-#define N_RES_STILL (ARRAY_SIZE(ov5693_res_still))
-
-struct ov5693_resolution ov5693_res_video[] = {
-	{
-		.desc = "ov5693_736x496_30fps",
-		.width = 736,
-		.height = 496,
-		.fps = 30,
-		.pix_clk_freq = 160,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 2,
-		.bin_factor_y = 2,
-		.bin_mode = 1,
-		.regs = ov5693_736x496,
-	},
-	{
-		.desc = "ov5693_336x256_30fps",
-		.width = 336,
-		.height = 256,
-		.fps = 30,
-		.pix_clk_freq = 160,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 2,
-		.bin_factor_y = 2,
-		.bin_mode = 1,
-		.regs = ov5693_336x256,
-	},
-	{
-		.desc = "ov5693_368x304_30fps",
-		.width = 368,
-		.height = 304,
-		.fps = 30,
-		.pix_clk_freq = 160,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 2,
-		.bin_factor_y = 2,
-		.bin_mode = 1,
-		.regs = ov5693_368x304,
-	},
-	{
-		.desc = "ov5693_192x160_30fps",
-		.width = 192,
-		.height = 160,
-		.fps = 30,
-		.pix_clk_freq = 160,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 2,
-		.bin_factor_y = 2,
-		.bin_mode = 1,
-		.regs = ov5693_192x160,
-	},
-	{
-		.desc = "ov5693_1296x736_30fps",
-		.width = 1296,
-		.height = 736,
-		.fps = 30,
-		.pix_clk_freq = 160,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 2,
-		.bin_factor_y = 2,
-		.bin_mode = 0,
-		.regs = ov5693_1296x736,
-	},
-	{
-		.desc = "ov5693_1296x976_30fps",
-		.width = 1296,
-		.height = 976,
-		.fps = 30,
-		.pix_clk_freq = 160,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 2,
-		.bin_factor_y = 2,
-		.bin_mode = 0,
-		.regs = ov5693_1296x976,
-	},
-	{
-		.desc = "ov5693_1636P_30fps",
-		.width = 1636,
-		.height = 1096,
-		.fps = 30,
-		.pix_clk_freq = 160,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_1636p_30fps,
-	},
-	{
-		.desc = "ov5693_1080P_30fps",
-		.width = 1940,
-		.height = 1096,
-		.fps = 30,
-		.pix_clk_freq = 160,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_1940x1096,
-	},
-	{
-		.desc = "ov5693_5M_30fps",
-		.width = 2592,
-		.height = 1456,
-		.pix_clk_freq = 160,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_2592x1456_30fps,
-	},
-	{
-		.desc = "ov5693_5M_30fps",
-		.width = 2592,
-		.height = 1944,
-		.pix_clk_freq = 160,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2688,
-		.lines_per_frame = 1984,
-		.bin_factor_x = 1,
-		.bin_factor_y = 1,
-		.bin_mode = 0,
-		.regs = ov5693_2592x1944_30fps,
-	},
-};
-#define N_RES_VIDEO (ARRAY_SIZE(ov5693_res_video))
-#endif
-
-static struct ov5693_resolution *ov5693_res = ov5693_res_preview;
-static unsigned long N_RES = N_RES_PREVIEW;
-#endif
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp.h b/drivers/staging/media/atomisp/include/linux/atomisp.h
deleted file mode 100644
index ebe193ba3871..000000000000
--- a/drivers/staging/media/atomisp/include/linux/atomisp.h
+++ /dev/null
@@ -1,1359 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifdef CSS15
-#include <linux/atomisp_css15.h>
-#else
-
-#ifndef _ATOM_ISP_H
-#define _ATOM_ISP_H
-
-#include <linux/types.h>
-#include <linux/version.h>
-
-/* struct media_device_info.hw_revision */
-#define ATOMISP_HW_REVISION_MASK	0x0000ff00
-#define ATOMISP_HW_REVISION_SHIFT	8
-#define ATOMISP_HW_REVISION_ISP2300	0x00
-#define ATOMISP_HW_REVISION_ISP2400	0x10
-#define ATOMISP_HW_REVISION_ISP2401_LEGACY 0x11
-#define ATOMISP_HW_REVISION_ISP2401	0x20
-
-#define ATOMISP_HW_STEPPING_MASK	0x000000ff
-#define ATOMISP_HW_STEPPING_A0		0x00
-#define ATOMISP_HW_STEPPING_B0		0x10
-
-/*ISP binary running mode*/
-#define CI_MODE_PREVIEW		0x8000
-#define CI_MODE_VIDEO		0x4000
-#define CI_MODE_STILL_CAPTURE	0x2000
-#define CI_MODE_CONTINUOUS	0x1000
-#define CI_MODE_NONE		0x0000
-
-#define OUTPUT_MODE_FILE 0x0100
-#define OUTPUT_MODE_TEXT 0x0200
-
-/*
- * Camera HAL sets this flag in v4l2_buffer reserved2 to indicate this
- * buffer has a per-frame parameter.
- */
-#define ATOMISP_BUFFER_HAS_PER_FRAME_SETTING	0x80000000
-
-/* Custom format for RAW capture from M10MO 0x3130314d */
-#define V4L2_PIX_FMT_CUSTOM_M10MO_RAW	v4l2_fourcc('M', '1', '0', '1')
-
-/* Custom media bus formats being used in atomisp */
-#define V4L2_MBUS_FMT_CUSTOM_YUV420	0x8001
-#define V4L2_MBUS_FMT_CUSTOM_YVU420	0x8002
-#define V4L2_MBUS_FMT_CUSTOM_YUV422P	0x8003
-#define V4L2_MBUS_FMT_CUSTOM_YUV444	0x8004
-#define V4L2_MBUS_FMT_CUSTOM_NV12	0x8005
-#define V4L2_MBUS_FMT_CUSTOM_NV21	0x8006
-#define V4L2_MBUS_FMT_CUSTOM_NV16	0x8007
-#define V4L2_MBUS_FMT_CUSTOM_YUYV	0x8008
-#define V4L2_MBUS_FMT_CUSTOM_SBGGR16	0x8009
-#define V4L2_MBUS_FMT_CUSTOM_RGB32	0x800a
-
-/* Custom media bus format for M10MO RAW capture */
-#if 0
-#define V4L2_MBUS_FMT_CUSTOM_M10MO_RAW	0x800b
-#endif
-
-/* Configuration used by Bayer noise reduction and YCC noise reduction */
-struct atomisp_nr_config {
-	/* [gain] Strength of noise reduction for Bayer NR (Used by Bayer NR) */
-	unsigned int bnr_gain;
-	/* [gain] Strength of noise reduction for YCC NR (Used by YCC NR) */
-	unsigned int ynr_gain;
-	/* [intensity] Sensitivity of Edge (Used by Bayer NR) */
-	unsigned int direction;
-	/* [intensity] coring threshold for Cb (Used by YCC NR) */
-	unsigned int threshold_cb;
-	/* [intensity] coring threshold for Cr (Used by YCC NR) */
-	unsigned int threshold_cr;
-};
-
-/* Temporal noise reduction configuration */
-struct atomisp_tnr_config {
-	unsigned int gain;	 /* [gain] Strength of NR */
-	unsigned int threshold_y;/* [intensity] Motion sensitivity for Y */
-	unsigned int threshold_uv;/* [intensity] Motion sensitivity for U/V */
-};
-
-/* Histogram. This contains num_elements values of type unsigned int.
- * The data pointer is a DDR pointer (virtual address).
- */
-struct atomisp_histogram {
-	unsigned int num_elements;
-	void __user *data;
-};
-
-enum atomisp_ob_mode {
-	atomisp_ob_mode_none,
-	atomisp_ob_mode_fixed,
-	atomisp_ob_mode_raster
-};
-
-/* Optical black level configuration */
-struct atomisp_ob_config {
-	/* Obtical black level mode (Fixed / Raster) */
-	enum atomisp_ob_mode mode;
-	/* [intensity] optical black level for GR (relevant for fixed mode) */
-	unsigned int level_gr;
-	/* [intensity] optical black level for R (relevant for fixed mode) */
-	unsigned int level_r;
-	/* [intensity] optical black level for B (relevant for fixed mode) */
-	unsigned int level_b;
-	/* [intensity] optical black level for GB (relevant for fixed mode) */
-	unsigned int level_gb;
-	/* [BQ] 0..63 start position of OB area (relevant for raster mode) */
-	unsigned short start_position;
-	/* [BQ] start..63 end position of OB area (relevant for raster mode) */
-	unsigned short end_position;
-};
-
-/* Edge enhancement (sharpen) configuration */
-struct atomisp_ee_config {
-	/* [gain] The strength of sharpness. u5_11 */
-	unsigned int gain;
-	/* [intensity] The threshold that divides noises from edge. u8_8 */
-	unsigned int threshold;
-	/* [gain] The strength of sharpness in pell-mell area. u5_11 */
-	unsigned int detail_gain;
-};
-
-struct atomisp_3a_output {
-	int ae_y;
-	int awb_cnt;
-	int awb_gr;
-	int awb_r;
-	int awb_b;
-	int awb_gb;
-	int af_hpf1;
-	int af_hpf2;
-};
-
-enum atomisp_calibration_type {
-	calibration_type1,
-	calibration_type2,
-	calibration_type3
-};
-
-struct atomisp_calibration_group {
-	unsigned int size;
-	unsigned int type;
-	unsigned short *calb_grp_values;
-};
-
-struct atomisp_gc_config {
-	__u16 gain_k1;
-	__u16 gain_k2;
-};
-
-struct atomisp_3a_config {
-	unsigned int ae_y_coef_r;	/* [gain] Weight of R for Y */
-	unsigned int ae_y_coef_g;	/* [gain] Weight of G for Y */
-	unsigned int ae_y_coef_b;	/* [gain] Weight of B for Y */
-	unsigned int awb_lg_high_raw;	/* [intensity]
-					   AWB level gate high for raw */
-	unsigned int awb_lg_low;	/* [intensity] AWB level gate low */
-	unsigned int awb_lg_high;	/* [intensity] AWB level gate high */
-	int af_fir1_coef[7];	/* [factor] AF FIR coefficients of fir1 */
-	int af_fir2_coef[7];	/* [factor] AF FIR coefficients of fir2 */
-};
-
-struct atomisp_dvs_grid_info {
-	uint32_t enable;
-	uint32_t width;
-	uint32_t aligned_width;
-	uint32_t height;
-	uint32_t aligned_height;
-	uint32_t bqs_per_grid_cell;
-	uint32_t num_hor_coefs;
-	uint32_t num_ver_coefs;
-};
-
-struct atomisp_dvs_envelop {
-	unsigned int width;
-	unsigned int height;
-};
-
-struct atomisp_grid_info {
-	uint32_t enable;
-	uint32_t use_dmem;
-	uint32_t has_histogram;
-	uint32_t s3a_width;
-	uint32_t s3a_height;
-	uint32_t aligned_width;
-	uint32_t aligned_height;
-	uint32_t s3a_bqs_per_grid_cell;
-	uint32_t deci_factor_log2;
-	uint32_t elem_bit_depth;
-};
-
-struct atomisp_dis_vector {
-	int x;
-	int y;
-};
-
-
-/* DVS 2.0 Coefficient types. This structure contains 4 pointers to
- *  arrays that contain the coeffients for each type.
- */
-struct atomisp_dvs2_coef_types {
-	short __user *odd_real; /** real part of the odd coefficients*/
-	short __user *odd_imag; /** imaginary part of the odd coefficients*/
-	short __user *even_real;/** real part of the even coefficients*/
-	short __user *even_imag;/** imaginary part of the even coefficients*/
-};
-
-/*
- * DVS 2.0 Statistic types. This structure contains 4 pointers to
- * arrays that contain the statistics for each type.
- */
-struct atomisp_dvs2_stat_types {
-	int __user *odd_real; /** real part of the odd statistics*/
-	int __user *odd_imag; /** imaginary part of the odd statistics*/
-	int __user *even_real;/** real part of the even statistics*/
-	int __user *even_imag;/** imaginary part of the even statistics*/
-};
-
-struct atomisp_dis_coefficients {
-	struct atomisp_dvs_grid_info grid_info;
-	struct atomisp_dvs2_coef_types hor_coefs;
-	struct atomisp_dvs2_coef_types ver_coefs;
-};
-
-struct atomisp_dvs2_statistics {
-	struct atomisp_dvs_grid_info grid_info;
-	struct atomisp_dvs2_stat_types hor_prod;
-	struct atomisp_dvs2_stat_types ver_prod;
-};
-
-struct atomisp_dis_statistics {
-	struct atomisp_dvs2_statistics dvs2_stat;
-	uint32_t exp_id;
-};
-
-struct atomisp_3a_rgby_output {
-	uint32_t r;
-	uint32_t g;
-	uint32_t b;
-	uint32_t y;
-};
-
-/*
- * Because we have 2 pipes at max to output metadata, therefore driver will use
- * ATOMISP_MAIN_METADATA to specify the metadata from the pipe which keeps
- * streaming always and use ATOMISP_SEC_METADATA to specify the metadata from
- * the pipe which is streaming by request like capture pipe of ZSL or SDV mode
- * as secondary metadata. And for the use case which has only one pipe
- * streaming like online capture, ATOMISP_MAIN_METADATA will be used.
- */
-enum atomisp_metadata_type {
-	ATOMISP_MAIN_METADATA = 0,
-	ATOMISP_SEC_METADATA,
-	ATOMISP_METADATA_TYPE_NUM,
-};
-
-struct atomisp_metadata_with_type {
-	/* to specify which type of metadata to get */
-	enum atomisp_metadata_type type;
-	void __user *data;
-	uint32_t width;
-	uint32_t height;
-	uint32_t stride; /* in bytes */
-	uint32_t exp_id; /* exposure ID */
-	uint32_t *effective_width; /* mipi packets valid data size */
-};
-
-struct atomisp_metadata {
-	void __user *data;
-	uint32_t width;
-	uint32_t height;
-	uint32_t stride; /* in bytes */
-	uint32_t exp_id; /* exposure ID */
-	uint32_t *effective_width; /* mipi packets valid data size */
-};
-
-struct atomisp_ext_isp_ctrl {
-	uint32_t id;
-	uint32_t data;
-};
-
-struct atomisp_3a_statistics {
-	struct atomisp_grid_info  grid_info;
-	struct atomisp_3a_output __user *data;
-	struct atomisp_3a_rgby_output __user *rgby_data;
-	uint32_t exp_id; /* exposure ID */
-	uint32_t isp_config_id; /* isp config ID */
-};
-
-/**
- * struct atomisp_cont_capture_conf - continuous capture parameters
- * @num_captures: number of still images to capture
- * @skip_frames: number of frames to skip between 2 captures
- * @offset: offset in ring buffer to start capture
- *
- * For example, to capture 1 frame from past, current, and 1 from future
- * and skip one frame between each capture, parameters would be:
- * num_captures:3
- * skip_frames:1
- * offset:-2
- */
-
-struct atomisp_cont_capture_conf {
-	int num_captures;
-	unsigned int skip_frames;
-	int offset;
-	__u32 reserved[5];
-};
-
-struct atomisp_ae_window {
-	int x_left;
-	int x_right;
-	int y_top;
-	int y_bottom;
-	int weight;
-};
-
-/* White Balance (Gain Adjust) */
-struct atomisp_wb_config {
-	unsigned int integer_bits;
-	unsigned int gr;	/* unsigned <integer_bits>.<16-integer_bits> */
-	unsigned int r;		/* unsigned <integer_bits>.<16-integer_bits> */
-	unsigned int b;		/* unsigned <integer_bits>.<16-integer_bits> */
-	unsigned int gb;	/* unsigned <integer_bits>.<16-integer_bits> */
-};
-
-/* Color Space Conversion settings */
-struct atomisp_cc_config {
-	unsigned int fraction_bits;
-	int matrix[3 * 3];	/* RGB2YUV Color matrix, signed
-				   <13-fraction_bits>.<fraction_bits> */
-};
-
-/* De pixel noise configuration */
-struct atomisp_de_config {
-	unsigned int pixelnoise;
-	unsigned int c1_coring_threshold;
-	unsigned int c2_coring_threshold;
-};
-
-/* Chroma enhancement */
-struct atomisp_ce_config {
-	unsigned char uv_level_min;
-	unsigned char uv_level_max;
-};
-
-/* Defect pixel correction configuration */
-struct atomisp_dp_config {
-	/* [intensity] The threshold of defect Pixel Correction, representing
-	 * the permissible difference of intensity between one pixel and its
-	 * surrounding pixels. Smaller values result in more frequent pixel
-	 * corrections. u0_16
-	 */
-	unsigned int threshold;
-	/* [gain] The sensitivity of mis-correction. ISP will miss a lot of
-	 * defects if the value is set too large. u8_8
-	 */
-	unsigned int gain;
-	unsigned int gr;
-	unsigned int r;
-	unsigned int b;
-	unsigned int gb;
-};
-
-/* XNR threshold */
-struct atomisp_xnr_config {
-	__u16 threshold;
-};
-
-/* metadata config */
-struct atomisp_metadata_config {
-	uint32_t metadata_height;
-	uint32_t metadata_stride;
-};
-
-/*
- * Generic resolution structure.
- */
-struct atomisp_resolution {
-	uint32_t width;  /** Width */
-	uint32_t height; /** Height */
-};
-
-/*
- * This specifies the coordinates (x,y)
- */
-struct atomisp_zoom_point {
-	int32_t x; /** x coordinate */
-	int32_t y; /** y coordinate */
-};
-
-/*
- * This specifies the region
- */
-struct atomisp_zoom_region {
-	struct atomisp_zoom_point origin; /* Starting point coordinates for the region */
-	struct atomisp_resolution resolution; /* Region resolution */
-};
-
-struct atomisp_dz_config {
-	uint32_t dx; /** Horizontal zoom factor */
-	uint32_t dy; /** Vertical zoom factor */
-	struct atomisp_zoom_region zoom_region; /** region for zoom */
-};
-
-struct atomisp_parm {
-	struct atomisp_grid_info info;
-	struct atomisp_dvs_grid_info dvs_grid;
-	struct atomisp_dvs_envelop dvs_envelop;
-	struct atomisp_wb_config wb_config;
-	struct atomisp_cc_config cc_config;
-	struct atomisp_ob_config ob_config;
-	struct atomisp_de_config de_config;
-	struct atomisp_dz_config dz_config;
-	struct atomisp_ce_config ce_config;
-	struct atomisp_dp_config dp_config;
-	struct atomisp_nr_config nr_config;
-	struct atomisp_ee_config ee_config;
-	struct atomisp_tnr_config tnr_config;
-	struct atomisp_metadata_config metadata_config;
-};
-
-struct dvs2_bq_resolution {
-	int width_bq;         /* width [BQ] */
-	int height_bq;        /* height [BQ] */
-};
-
-struct atomisp_dvs2_bq_resolutions {
-	/* GDC source image size [BQ] */
-	struct dvs2_bq_resolution source_bq;
-	/* GDC output image size [BQ] */
-	struct dvs2_bq_resolution output_bq;
-	/* GDC effective envelope size [BQ] */
-	struct dvs2_bq_resolution envelope_bq;
-	/* isp pipe filter size [BQ] */
-	struct dvs2_bq_resolution ispfilter_bq;
-	/* GDC shit size [BQ] */
-	struct dvs2_bq_resolution gdc_shift_bq;
-};
-
-struct atomisp_dvs_6axis_config {
-	uint32_t exp_id;
-	uint32_t width_y;
-	uint32_t height_y;
-	uint32_t width_uv;
-	uint32_t height_uv;
-	uint32_t *xcoords_y;
-	uint32_t *ycoords_y;
-	uint32_t *xcoords_uv;
-	uint32_t *ycoords_uv;
-};
-
-struct atomisp_formats_config {
-	uint32_t video_full_range_flag;
-};
-
-struct atomisp_parameters {
-	struct atomisp_wb_config   *wb_config;  /* White Balance config */
-	struct atomisp_cc_config   *cc_config;  /* Color Correction config */
-	struct atomisp_tnr_config  *tnr_config; /* Temporal Noise Reduction */
-	struct atomisp_ecd_config  *ecd_config; /* Eigen Color Demosaicing */
-	struct atomisp_ynr_config  *ynr_config; /* Y(Luma) Noise Reduction */
-	struct atomisp_fc_config   *fc_config;  /* Fringe Control */
-	struct atomisp_formats_config *formats_config; /* Formats Control */
-	struct atomisp_cnr_config  *cnr_config; /* Chroma Noise Reduction */
-	struct atomisp_macc_config *macc_config;  /* MACC */
-	struct atomisp_ctc_config  *ctc_config; /* Chroma Tone Control */
-	struct atomisp_aa_config   *aa_config;  /* Anti-Aliasing */
-	struct atomisp_aa_config   *baa_config;  /* Anti-Aliasing */
-	struct atomisp_ce_config   *ce_config;
-	struct atomisp_dvs_6axis_config *dvs_6axis_config;
-	struct atomisp_ob_config   *ob_config;  /* Objective Black config */
-	struct atomisp_dp_config   *dp_config;  /* Dead Pixel config */
-	struct atomisp_nr_config   *nr_config;  /* Noise Reduction config */
-	struct atomisp_ee_config   *ee_config;  /* Edge Enhancement config */
-	struct atomisp_de_config   *de_config;  /* Demosaic config */
-	struct atomisp_gc_config   *gc_config;  /* Gamma Correction config */
-	struct atomisp_anr_config  *anr_config; /* Advanced Noise Reduction */
-	struct atomisp_3a_config   *a3a_config; /* 3A Statistics config */
-	struct atomisp_xnr_config  *xnr_config; /* eXtra Noise Reduction */
-	struct atomisp_dz_config   *dz_config;  /* Digital Zoom */
-	struct atomisp_cc_config *yuv2rgb_cc_config; /* Color
-							Correction config */
-	struct atomisp_cc_config *rgb2yuv_cc_config; /* Color
-							Correction config */
-	struct atomisp_macc_table  *macc_table;
-	struct atomisp_gamma_table *gamma_table;
-	struct atomisp_ctc_table   *ctc_table;
-	struct atomisp_xnr_table   *xnr_table;
-	struct atomisp_rgb_gamma_table *r_gamma_table;
-	struct atomisp_rgb_gamma_table *g_gamma_table;
-	struct atomisp_rgb_gamma_table *b_gamma_table;
-	struct atomisp_vector      *motion_vector; /* For 2-axis DVS */
-	struct atomisp_shading_table *shading_table;
-	struct atomisp_morph_table   *morph_table;
-	struct atomisp_dvs_coefficients *dvs_coefs; /* DVS 1.0 coefficients */
-	struct atomisp_dvs2_coefficients *dvs2_coefs; /* DVS 2.0 coefficients */
-	struct atomisp_capture_config   *capture_config;
-	struct atomisp_anr_thres   *anr_thres;
-
-	void	*lin_2500_config;       /* Skylake: Linearization config */
-	void	*obgrid_2500_config;    /* Skylake: OBGRID config */
-	void	*bnr_2500_config;       /* Skylake: bayer denoise config */
-	void	*shd_2500_config;       /* Skylake: shading config */
-	void	*dm_2500_config;        /* Skylake: demosaic config */
-	void	*rgbpp_2500_config;     /* Skylake: RGBPP config */
-	void	*dvs_stat_2500_config;  /* Skylake: DVS STAT config */
-	void	*lace_stat_2500_config; /* Skylake: LACE STAT config */
-	void	*yuvp1_2500_config;     /* Skylake: yuvp1 config */
-	void	*yuvp2_2500_config;     /* Skylake: yuvp2 config */
-	void	*tnr_2500_config;       /* Skylake: TNR config */
-	void	*dpc_2500_config;       /* Skylake: DPC config */
-	void	*awb_2500_config;       /* Skylake: auto white balance config */
-	void	*awb_fr_2500_config;    /* Skylake: auto white balance filter response config */
-	void	*anr_2500_config;       /* Skylake: ANR config */
-	void	*af_2500_config;        /* Skylake: auto focus config */
-	void	*ae_2500_config;        /* Skylake: auto exposure config */
-	void	*bds_2500_config;       /* Skylake: bayer downscaler config */
-	void	*dvs_2500_config;       /* Skylake: digital video stabilization config */
-	void	*res_mgr_2500_config;
-
-	/*
-	 * Output frame pointer the config is to be applied to (optional),
-	 * set to NULL to make this config is applied as global.
-	 */
-	void	*output_frame;
-	/*
-	 * Unique ID to track which config was actually applied to a particular
-	 * frame, driver will send this id back with output frame together.
-	 */
-	uint32_t	isp_config_id;
-
-	/*
-	 * Switch to control per_frame setting:
-	 * 0: this is a global setting
-	 * 1: this is a per_frame setting
-	 * PLEASE KEEP THIS AT THE END OF THE STRUCTURE!!
-	 */
-	uint32_t	per_frame_setting;
-};
-
-#define ATOMISP_GAMMA_TABLE_SIZE        1024
-struct atomisp_gamma_table {
-	unsigned short data[ATOMISP_GAMMA_TABLE_SIZE];
-};
-
-/* Morphing table for advanced ISP.
- * Each line of width elements takes up COORD_TABLE_EXT_WIDTH elements
- * in memory.
- */
-#define ATOMISP_MORPH_TABLE_NUM_PLANES  6
-struct atomisp_morph_table {
-	unsigned int enabled;
-
-	unsigned int height;
-	unsigned int width;	/* number of valid elements per line */
-	unsigned short __user *coordinates_x[ATOMISP_MORPH_TABLE_NUM_PLANES];
-	unsigned short __user *coordinates_y[ATOMISP_MORPH_TABLE_NUM_PLANES];
-};
-
-#define ATOMISP_NUM_SC_COLORS	4
-#define ATOMISP_SC_FLAG_QUERY	(1 << 0)
-
-struct atomisp_shading_table {
-	__u32 enable;
-
-	__u32 sensor_width;
-	__u32 sensor_height;
-	__u32 width;
-	__u32 height;
-	__u32 fraction_bits;
-
-	__u16 *data[ATOMISP_NUM_SC_COLORS];
-};
-
-struct atomisp_makernote_info {
-	/* bits 31-16: numerator, bits 15-0: denominator */
-	unsigned int focal_length;
-	/* bits 31-16: numerator, bits 15-0: denominator*/
-	unsigned int f_number_curr;
-	/*
-	* bits 31-24: max f-number numerator
-	* bits 23-16: max f-number denominator
-	* bits 15-8: min f-number numerator
-	* bits 7-0: min f-number denominator
-	*/
-	unsigned int f_number_range;
-};
-
-/* parameter for MACC */
-#define ATOMISP_NUM_MACC_AXES           16
-struct atomisp_macc_table {
-	short data[4 * ATOMISP_NUM_MACC_AXES];
-};
-
-struct atomisp_macc_config {
-	int color_effect;
-	struct atomisp_macc_table table;
-};
-
-/* Parameter for ctc parameter control */
-#define ATOMISP_CTC_TABLE_SIZE          1024
-struct atomisp_ctc_table {
-	unsigned short data[ATOMISP_CTC_TABLE_SIZE];
-};
-
-/* Parameter for overlay image loading */
-struct atomisp_overlay {
-	/* the frame containing the overlay data The overlay frame width should
-	 * be the multiples of 2*ISP_VEC_NELEMS. The overlay frame height
-	 * should be the multiples of 2.
-	 */
-	struct v4l2_framebuffer *frame;
-	/* Y value of overlay background */
-	unsigned char bg_y;
-	/* U value of overlay background */
-	char bg_u;
-	/* V value of overlay background */
-	char bg_v;
-	/* the blending percent of input data for Y subpixels */
-	unsigned char blend_input_perc_y;
-	/* the blending percent of input data for U subpixels */
-	unsigned char blend_input_perc_u;
-	/* the blending percent of input data for V subpixels */
-	unsigned char blend_input_perc_v;
-	/* the blending percent of overlay data for Y subpixels */
-	unsigned char blend_overlay_perc_y;
-	/* the blending percent of overlay data for U subpixels */
-	unsigned char blend_overlay_perc_u;
-	/* the blending percent of overlay data for V subpixels */
-	unsigned char blend_overlay_perc_v;
-	/* the overlay start x pixel position on output frame It should be the
-	   multiples of 2*ISP_VEC_NELEMS. */
-	unsigned int overlay_start_x;
-	/* the overlay start y pixel position on output frame It should be the
-	   multiples of 2. */
-	unsigned int overlay_start_y;
-};
-
-/* Sensor resolution specific data for AE calculation.*/
-struct atomisp_sensor_mode_data {
-	unsigned int coarse_integration_time_min;
-	unsigned int coarse_integration_time_max_margin;
-	unsigned int fine_integration_time_min;
-	unsigned int fine_integration_time_max_margin;
-	unsigned int fine_integration_time_def;
-	unsigned int frame_length_lines;
-	unsigned int line_length_pck;
-	unsigned int read_mode;
-	unsigned int vt_pix_clk_freq_mhz;
-	unsigned int crop_horizontal_start; /* Sensor crop start cord. (x0,y0)*/
-	unsigned int crop_vertical_start;
-	unsigned int crop_horizontal_end; /* Sensor crop end cord. (x1,y1)*/
-	unsigned int crop_vertical_end;
-	unsigned int output_width; /* input size to ISP after binning/scaling */
-	unsigned int output_height;
-	uint8_t binning_factor_x; /* horizontal binning factor used */
-	uint8_t binning_factor_y; /* vertical binning factor used */
-	uint16_t hts;
-};
-
-struct atomisp_exposure {
-	unsigned int integration_time[8];
-	unsigned int shutter_speed[8];
-	unsigned int gain[4];
-	unsigned int aperture;
-};
-
-/* For texture streaming. */
-struct atomisp_bc_video_package {
-	int ioctl_cmd;
-	int device_id;
-	int inputparam;
-	int outputparam;
-};
-
-enum atomisp_focus_hp {
-	ATOMISP_FOCUS_HP_IN_PROGRESS = (1U << 2),
-	ATOMISP_FOCUS_HP_COMPLETE    = (2U << 2),
-	ATOMISP_FOCUS_HP_FAILED      = (3U << 2)
-};
-
-/* Masks */
-#define ATOMISP_FOCUS_STATUS_MOVING           (1U << 0)
-#define ATOMISP_FOCUS_STATUS_ACCEPTS_NEW_MOVE (1U << 1)
-#define ATOMISP_FOCUS_STATUS_HOME_POSITION    (3U << 2)
-
-enum atomisp_camera_port {
-	ATOMISP_CAMERA_PORT_SECONDARY,
-	ATOMISP_CAMERA_PORT_PRIMARY,
-	ATOMISP_CAMERA_PORT_TERTIARY,
-	ATOMISP_CAMERA_NR_PORTS
-};
-
-/* Flash modes. Default is off.
- * Setting a flash to TORCH or INDICATOR mode will automatically
- * turn it on. Setting it to FLASH mode will not turn on the flash
- * until the FLASH_STROBE command is sent. */
-enum atomisp_flash_mode {
-	ATOMISP_FLASH_MODE_OFF,
-	ATOMISP_FLASH_MODE_FLASH,
-	ATOMISP_FLASH_MODE_TORCH,
-	ATOMISP_FLASH_MODE_INDICATOR,
-};
-
-/* Flash statuses, used by atomisp driver to check before starting
- * flash and after having started flash. */
-enum atomisp_flash_status {
-	ATOMISP_FLASH_STATUS_OK,
-	ATOMISP_FLASH_STATUS_HW_ERROR,
-	ATOMISP_FLASH_STATUS_INTERRUPTED,
-	ATOMISP_FLASH_STATUS_TIMEOUT,
-};
-
-/* Frame status. This is used to detect corrupted frames and flash
- * exposed frames. Usually, the first 2 frames coming out of the sensor
- * are corrupted. When using flash, the frame before and the frame after
- * the flash exposed frame may be partially exposed by flash. The ISP
- * statistics for these frames should not be used by the 3A library.
- * The frame status value can be found in the "reserved" field in the
- * v4l2_buffer struct. */
-enum atomisp_frame_status {
-	ATOMISP_FRAME_STATUS_OK,
-	ATOMISP_FRAME_STATUS_CORRUPTED,
-	ATOMISP_FRAME_STATUS_FLASH_EXPOSED,
-	ATOMISP_FRAME_STATUS_FLASH_PARTIAL,
-	ATOMISP_FRAME_STATUS_FLASH_FAILED,
-};
-
-enum atomisp_acc_type {
-	ATOMISP_ACC_STANDALONE,	/* Stand-alone acceleration */
-	ATOMISP_ACC_OUTPUT,	/* Accelerator stage on output frame */
-	ATOMISP_ACC_VIEWFINDER	/* Accelerator stage on viewfinder frame */
-};
-
-enum atomisp_acc_arg_type {
-	ATOMISP_ACC_ARG_SCALAR_IN,    /* Scalar input argument */
-	ATOMISP_ACC_ARG_SCALAR_OUT,   /* Scalar output argument */
-	ATOMISP_ACC_ARG_SCALAR_IO,    /* Scalar in/output argument */
-	ATOMISP_ACC_ARG_PTR_IN,	     /* Pointer input argument */
-	ATOMISP_ACC_ARG_PTR_OUT,	     /* Pointer output argument */
-	ATOMISP_ACC_ARG_PTR_IO,	     /* Pointer in/output argument */
-	ATOMISP_ARG_PTR_NOFLUSH,  /* Pointer argument will not be flushed */
-	ATOMISP_ARG_PTR_STABLE,   /* Pointer input argument that is stable */
-	ATOMISP_ACC_ARG_FRAME	     /* Frame argument */
-};
-
-/* ISP memories, isp2400 */
-enum atomisp_acc_memory {
-	ATOMISP_ACC_MEMORY_PMEM0 = 0,
-	ATOMISP_ACC_MEMORY_DMEM0,
-	/* for backward compatibility */
-	ATOMISP_ACC_MEMORY_DMEM = ATOMISP_ACC_MEMORY_DMEM0,
-	ATOMISP_ACC_MEMORY_VMEM0,
-	ATOMISP_ACC_MEMORY_VAMEM0,
-	ATOMISP_ACC_MEMORY_VAMEM1,
-	ATOMISP_ACC_MEMORY_VAMEM2,
-	ATOMISP_ACC_MEMORY_HMEM0,
-	ATOMISP_ACC_NR_MEMORY
-};
-
-enum atomisp_ext_isp_id {
-	EXT_ISP_CID_ISO = 0,
-	EXT_ISP_CID_CAPTURE_HDR,
-	EXT_ISP_CID_CAPTURE_LLS,
-	EXT_ISP_CID_FOCUS_MODE,
-	EXT_ISP_CID_FOCUS_EXECUTION,
-	EXT_ISP_CID_TOUCH_POSX,
-	EXT_ISP_CID_TOUCH_POSY,
-	EXT_ISP_CID_CAF_STATUS,
-	EXT_ISP_CID_AF_STATUS,
-	EXT_ISP_CID_GET_AF_MODE,
-	EXT_ISP_CID_CAPTURE_BURST,
-	EXT_ISP_CID_FLASH_MODE,
-	EXT_ISP_CID_ZOOM,
-	EXT_ISP_CID_SHOT_MODE
-};
-
-#define EXT_ISP_FOCUS_MODE_NORMAL	0
-#define EXT_ISP_FOCUS_MODE_MACRO	1
-#define EXT_ISP_FOCUS_MODE_TOUCH_AF	2
-#define EXT_ISP_FOCUS_MODE_PREVIEW_CAF	3
-#define EXT_ISP_FOCUS_MODE_MOVIE_CAF	4
-#define EXT_ISP_FOCUS_MODE_FACE_CAF	5
-#define EXT_ISP_FOCUS_MODE_TOUCH_MACRO	6
-#define EXT_ISP_FOCUS_MODE_TOUCH_CAF	7
-
-#define EXT_ISP_FOCUS_STOP		0
-#define EXT_ISP_FOCUS_SEARCH		1
-#define EXT_ISP_PAN_FOCUSING		2
-
-#define EXT_ISP_CAF_RESTART_CHECK	1
-#define EXT_ISP_CAF_STATUS_FOCUSING	2
-#define EXT_ISP_CAF_STATUS_SUCCESS	3
-#define EXT_ISP_CAF_STATUS_FAIL         4
-
-#define EXT_ISP_AF_STATUS_INVALID	1
-#define EXT_ISP_AF_STATUS_FOCUSING	2
-#define EXT_ISP_AF_STATUS_SUCCESS	3
-#define EXT_ISP_AF_STATUS_FAIL		4
-
-enum atomisp_burst_capture_options {
-	EXT_ISP_BURST_CAPTURE_CTRL_START = 0,
-	EXT_ISP_BURST_CAPTURE_CTRL_STOP
-};
-
-#define EXT_ISP_FLASH_MODE_OFF		0
-#define EXT_ISP_FLASH_MODE_ON		1
-#define EXT_ISP_FLASH_MODE_AUTO		2
-#define EXT_ISP_LED_TORCH_OFF		3
-#define EXT_ISP_LED_TORCH_ON		4
-
-#define EXT_ISP_SHOT_MODE_AUTO		0
-#define EXT_ISP_SHOT_MODE_BEAUTY_FACE	1
-#define EXT_ISP_SHOT_MODE_BEST_PHOTO	2
-#define EXT_ISP_SHOT_MODE_DRAMA		3
-#define EXT_ISP_SHOT_MODE_BEST_FACE	4
-#define EXT_ISP_SHOT_MODE_ERASER	5
-#define EXT_ISP_SHOT_MODE_PANORAMA	6
-#define EXT_ISP_SHOT_MODE_RICH_TONE_HDR	7
-#define EXT_ISP_SHOT_MODE_NIGHT		8
-#define EXT_ISP_SHOT_MODE_SOUND_SHOT	9
-#define EXT_ISP_SHOT_MODE_ANIMATED_PHOTO	10
-#define EXT_ISP_SHOT_MODE_SPORTS	11
-
-struct atomisp_sp_arg {
-	enum atomisp_acc_arg_type type;	/* Type  of SP argument */
-	void                    *value;	/* Value of SP argument */
-	unsigned int             size;	/* Size  of SP argument */
-};
-
-/* Acceleration API */
-
-/* For CSS 1.0 only */
-struct atomisp_acc_fw_arg {
-	unsigned int fw_handle;
-	unsigned int index;
-	void __user *value;
-	size_t size;
-};
-
-/*
- * Set arguments after first mapping with ATOMISP_IOC_ACC_S_MAPPED_ARG.
- */
-struct atomisp_acc_s_mapped_arg {
-	unsigned int fw_handle;
-	__u32 memory;			/* one of enum atomisp_acc_memory */
-	size_t length;
-	unsigned long css_ptr;
-};
-
-struct atomisp_acc_fw_abort {
-	unsigned int fw_handle;
-	/* Timeout in us */
-	unsigned int timeout;
-};
-
-struct atomisp_acc_fw_load {
-	unsigned int size;
-	unsigned int fw_handle;
-	void __user *data;
-};
-
-/*
- * Load firmware to specified pipeline.
- */
-struct atomisp_acc_fw_load_to_pipe {
-	__u32 flags;			/* Flags, see below for valid values */
-	unsigned int fw_handle;		/* Handle, filled by kernel. */
-	__u32 size;			/* Firmware binary size */
-	void __user *data;		/* Pointer to firmware */
-	__u32 type;			/* Binary type */
-	__u32 reserved[3];		/* Set to zero */
-};
-/*
- * Set Senor run mode
- */
-struct atomisp_s_runmode {
-	__u32 mode;
-};
-
-#define ATOMISP_ACC_FW_LOAD_FL_PREVIEW		(1 << 0)
-#define ATOMISP_ACC_FW_LOAD_FL_COPY		(1 << 1)
-#define ATOMISP_ACC_FW_LOAD_FL_VIDEO		(1 << 2)
-#define ATOMISP_ACC_FW_LOAD_FL_CAPTURE		(1 << 3)
-#define ATOMISP_ACC_FW_LOAD_FL_ACC		(1 << 4)
-#define ATOMISP_ACC_FW_LOAD_FL_ENABLE		(1 << 16)
-
-#define ATOMISP_ACC_FW_LOAD_TYPE_NONE		0 /* Normal binary: don't use */
-#define ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT		1 /* Stage on output */
-#define ATOMISP_ACC_FW_LOAD_TYPE_VIEWFINDER	2 /* Stage on viewfinder */
-#define ATOMISP_ACC_FW_LOAD_TYPE_STANDALONE	3 /* Stand-alone acceleration */
-
-struct atomisp_acc_map {
-	__u32 flags;			/* Flags, see list below */
-	__u32 length;			/* Length of data in bytes */
-	void __user *user_ptr;		/* Pointer into user space */
-	unsigned long css_ptr;		/* Pointer into CSS address space */
-	__u32 reserved[4];		/* Set to zero */
-};
-
-#define ATOMISP_MAP_FLAG_NOFLUSH	0x0001	/* Do not flush cache */
-#define ATOMISP_MAP_FLAG_CACHED		0x0002	/* Enable cache */
-
-struct atomisp_acc_state {
-	__u32 flags;			/* Flags, see list below */
-#define ATOMISP_STATE_FLAG_ENABLE	ATOMISP_ACC_FW_LOAD_FL_ENABLE
-	unsigned int fw_handle;
-};
-
-struct atomisp_update_exposure {
-	unsigned int gain;
-	unsigned int digi_gain;
-	unsigned int update_gain;
-	unsigned int update_digi_gain;
-};
-
-/*
- * V4L2 private internal data interface.
- * -----------------------------------------------------------------------------
- * struct v4l2_private_int_data - request private data stored in video device
- * internal memory.
- * @size: sanity check to ensure userspace's buffer fits whole private data.
- *	  If not, kernel will make partial copy (or nothing if @size == 0).
- *	  @size is always corrected for the minimum necessary if IOCTL returns
- *	  no error.
- * @data: pointer to userspace buffer.
- */
-struct v4l2_private_int_data {
-	__u32 size;
-	void __user *data;
-	__u32 reserved[2];
-};
-
-enum atomisp_sensor_ae_bracketing_mode {
-	SENSOR_AE_BRACKETING_MODE_OFF = 0,
-	SENSOR_AE_BRACKETING_MODE_SINGLE, /* back to SW standby after bracketing */
-	SENSOR_AE_BRACKETING_MODE_SINGLE_TO_STREAMING, /* back to normal streaming after bracketing */
-	SENSOR_AE_BRACKETING_MODE_LOOP, /* continue AE bracketing in loop mode */
-};
-
-struct atomisp_sensor_ae_bracketing_info {
-	unsigned int modes; /* bit mask to indicate supported modes  */
-	unsigned int lut_depth;
-};
-
-struct atomisp_sensor_ae_bracketing_lut_entry {
-	__u16 coarse_integration_time;
-	__u16 analog_gain;
-	__u16 digital_gain;
-};
-
-struct atomisp_sensor_ae_bracketing_lut {
-	struct atomisp_sensor_ae_bracketing_lut_entry *lut;
-	unsigned int lut_size;
-};
-
-/*Private IOCTLs for ISP */
-#define ATOMISP_IOC_G_XNR \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 0, int)
-#define ATOMISP_IOC_S_XNR \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 0, int)
-#define ATOMISP_IOC_G_NR \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 1, struct atomisp_nr_config)
-#define ATOMISP_IOC_S_NR \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 1, struct atomisp_nr_config)
-#define ATOMISP_IOC_G_TNR \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 2, struct atomisp_tnr_config)
-#define ATOMISP_IOC_S_TNR \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 2, struct atomisp_tnr_config)
-#define ATOMISP_IOC_G_HISTOGRAM \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 3, struct atomisp_histogram)
-#define ATOMISP_IOC_S_HISTOGRAM \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 3, struct atomisp_histogram)
-#define ATOMISP_IOC_G_BLACK_LEVEL_COMP \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 4, struct atomisp_ob_config)
-#define ATOMISP_IOC_S_BLACK_LEVEL_COMP \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 4, struct atomisp_ob_config)
-#define ATOMISP_IOC_G_EE \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 5, struct atomisp_ee_config)
-#define ATOMISP_IOC_S_EE \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 5, struct atomisp_ee_config)
-/* Digital Image Stabilization:
- * 1. get dis statistics: reads DIS statistics from ISP (every frame)
- * 2. set dis coefficients: set DIS filter coefficients (one time)
- * 3. set dis motion vecotr: set motion vector (result of DIS, every frame)
- */
-#define ATOMISP_IOC_G_DIS_STAT \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 6, struct atomisp_dis_statistics)
-
-#define ATOMISP_IOC_G_DVS2_BQ_RESOLUTIONS \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 6, struct atomisp_dvs2_bq_resolutions)
-
-#define ATOMISP_IOC_S_DIS_COEFS \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 6, struct atomisp_dis_coefficients)
-
-#define ATOMISP_IOC_S_DIS_VECTOR \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 6, struct atomisp_dvs_6axis_config)
-
-#define ATOMISP_IOC_G_3A_STAT \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 7, struct atomisp_3a_statistics)
-#define ATOMISP_IOC_G_ISP_PARM \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 8, struct atomisp_parm)
-#define ATOMISP_IOC_S_ISP_PARM \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 8, struct atomisp_parm)
-#define ATOMISP_IOC_G_ISP_GAMMA \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 9, struct atomisp_gamma_table)
-#define ATOMISP_IOC_S_ISP_GAMMA \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 9, struct atomisp_gamma_table)
-#define ATOMISP_IOC_G_ISP_GDC_TAB \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 10, struct atomisp_morph_table)
-#define ATOMISP_IOC_S_ISP_GDC_TAB \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 10, struct atomisp_morph_table)
-#define ATOMISP_IOC_ISP_MAKERNOTE \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 11, struct atomisp_makernote_info)
-
-/* macc parameter control*/
-#define ATOMISP_IOC_G_ISP_MACC \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 12, struct atomisp_macc_config)
-#define ATOMISP_IOC_S_ISP_MACC \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 12, struct atomisp_macc_config)
-
-/* Defect pixel detection & Correction */
-#define ATOMISP_IOC_G_ISP_BAD_PIXEL_DETECTION \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 13, struct atomisp_dp_config)
-#define ATOMISP_IOC_S_ISP_BAD_PIXEL_DETECTION \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 13, struct atomisp_dp_config)
-
-/* False Color Correction */
-#define ATOMISP_IOC_G_ISP_FALSE_COLOR_CORRECTION \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 14, struct atomisp_de_config)
-#define ATOMISP_IOC_S_ISP_FALSE_COLOR_CORRECTION \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 14, struct atomisp_de_config)
-
-/* ctc parameter control */
-#define ATOMISP_IOC_G_ISP_CTC \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 15, struct atomisp_ctc_table)
-#define ATOMISP_IOC_S_ISP_CTC \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 15, struct atomisp_ctc_table)
-
-/* white balance Correction */
-#define ATOMISP_IOC_G_ISP_WHITE_BALANCE \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 16, struct atomisp_wb_config)
-#define ATOMISP_IOC_S_ISP_WHITE_BALANCE \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 16, struct atomisp_wb_config)
-
-/* fpn table loading */
-#define ATOMISP_IOC_S_ISP_FPN_TABLE \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 17, struct v4l2_framebuffer)
-
-/* overlay image loading */
-#define ATOMISP_IOC_G_ISP_OVERLAY \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 18, struct atomisp_overlay)
-#define ATOMISP_IOC_S_ISP_OVERLAY \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 18, struct atomisp_overlay)
-
-/* bcd driver bridge */
-#define ATOMISP_IOC_CAMERA_BRIDGE \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 19, struct atomisp_bc_video_package)
-
-/* Sensor resolution specific info for AE */
-#define ATOMISP_IOC_G_SENSOR_MODE_DATA \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 20, struct atomisp_sensor_mode_data)
-
-#define ATOMISP_IOC_S_EXPOSURE \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 21, struct atomisp_exposure)
-
-/* sensor calibration registers group */
-#define ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 22, struct atomisp_calibration_group)
-
-/* white balance Correction */
-#define ATOMISP_IOC_G_3A_CONFIG \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 23, struct atomisp_3a_config)
-#define ATOMISP_IOC_S_3A_CONFIG \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 23, struct atomisp_3a_config)
-
-/* Accelerate ioctls */
-#define ATOMISP_IOC_ACC_LOAD \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 24, struct atomisp_acc_fw_load)
-
-#define ATOMISP_IOC_ACC_UNLOAD \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 24, unsigned int)
-
-/* For CSS 1.0 only */
-#define ATOMISP_IOC_ACC_S_ARG \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 24, struct atomisp_acc_fw_arg)
-
-#define ATOMISP_IOC_ACC_START \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 24, unsigned int)
-
-#define ATOMISP_IOC_ACC_WAIT \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 25, unsigned int)
-
-#define ATOMISP_IOC_ACC_ABORT \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 25, struct atomisp_acc_fw_abort)
-
-#define ATOMISP_IOC_ACC_DESTAB \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 25, struct atomisp_acc_fw_arg)
-
-/* sensor OTP memory read */
-#define ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 26, struct v4l2_private_int_data)
-
-/* LCS (shading) table write */
-#define ATOMISP_IOC_S_ISP_SHD_TAB \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 27, struct atomisp_shading_table)
-
-/* Gamma Correction */
-#define ATOMISP_IOC_G_ISP_GAMMA_CORRECTION \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 28, struct atomisp_gc_config)
-
-#define ATOMISP_IOC_S_ISP_GAMMA_CORRECTION \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 28, struct atomisp_gc_config)
-
-/* motor internal memory read */
-#define ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 29, struct v4l2_private_int_data)
-
-/*
- * Ioctls to map and unmap user buffers to CSS address space for acceleration.
- * User fills fields length and user_ptr and sets other fields to zero,
- * kernel may modify the flags and sets css_ptr.
- */
-#define ATOMISP_IOC_ACC_MAP \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 30, struct atomisp_acc_map)
-
-/* User fills fields length, user_ptr, and css_ptr and zeroes other fields. */
-#define ATOMISP_IOC_ACC_UNMAP \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 30, struct atomisp_acc_map)
-
-#define ATOMISP_IOC_ACC_S_MAPPED_ARG \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 30, struct atomisp_acc_s_mapped_arg)
-
-#define ATOMISP_IOC_ACC_LOAD_TO_PIPE \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 31, struct atomisp_acc_fw_load_to_pipe)
-
-#define ATOMISP_IOC_S_PARAMETERS \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 32, struct atomisp_parameters)
-
-#define ATOMISP_IOC_S_CONT_CAPTURE_CONFIG \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 33, struct atomisp_cont_capture_conf)
-
-#define ATOMISP_IOC_G_METADATA \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 34, struct atomisp_metadata)
-
-#define ATOMISP_IOC_G_METADATA_BY_TYPE \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 34, struct atomisp_metadata_with_type)
-
-#define ATOMISP_IOC_EXT_ISP_CTRL \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 35, struct atomisp_ext_isp_ctrl)
-
-#define ATOMISP_IOC_EXP_ID_UNLOCK \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 36, int)
-
-#define ATOMISP_IOC_EXP_ID_CAPTURE \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 37, int)
-
-#define ATOMISP_IOC_S_ENABLE_DZ_CAPT_PIPE \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 38, unsigned int)
-
-#define ATOMISP_IOC_G_FORMATS_CONFIG \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 39, struct atomisp_formats_config)
-
-#define ATOMISP_IOC_S_FORMATS_CONFIG \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 39, struct atomisp_formats_config)
-
-#define ATOMISP_IOC_S_EXPOSURE_WINDOW \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 40, struct atomisp_ae_window)
-
-#define ATOMISP_IOC_S_ACC_STATE \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 41, struct atomisp_acc_state)
-
-#define ATOMISP_IOC_G_ACC_STATE \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 41, struct atomisp_acc_state)
-
-#define ATOMISP_IOC_INJECT_A_FAKE_EVENT \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 42, int)
-
-#define ATOMISP_IOC_G_SENSOR_AE_BRACKETING_INFO \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 43, struct atomisp_sensor_ae_bracketing_info)
-
-#define ATOMISP_IOC_S_SENSOR_AE_BRACKETING_MODE \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 43, unsigned int)
-
-#define ATOMISP_IOC_G_SENSOR_AE_BRACKETING_MODE \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 43, unsigned int)
-
-#define ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 43, struct atomisp_sensor_ae_bracketing_lut)
-
-#define ATOMISP_IOC_G_INVALID_FRAME_NUM \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 44, unsigned int)
-
-#define ATOMISP_IOC_S_ARRAY_RESOLUTION \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 45, struct atomisp_resolution)
-
-/* for depth mode sensor frame sync compensation */
-#define ATOMISP_IOC_G_DEPTH_SYNC_COMP \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 46, unsigned int)
-
-#define ATOMISP_IOC_S_SENSOR_EE_CONFIG \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 47, unsigned int)
-
-#define ATOMISP_IOC_S_SENSOR_RUNMODE \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 48, struct atomisp_s_runmode)
-
-#define ATOMISP_IOC_G_UPDATE_EXPOSURE \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 49, struct atomisp_update_exposure)
-
-/*
- * Reserved ioctls. We have customer implementing it internally.
- * We can't use both numbers to not cause ABI conflict.
- * Anyway, those ioctls are hacks and not implemented by us:
- *
- * #define ATOMISP_IOC_G_SENSOR_REG \
- *	_IOW('v', BASE_VIDIOC_PRIVATE + 55, struct atomisp_sensor_regs)
- * #define ATOMISP_IOC_S_SENSOR_REG \
- *	_IOW('v', BASE_VIDIOC_PRIVATE + 56, struct atomisp_sensor_regs)
- */
-
-/*  ISP Private control IDs */
-#define V4L2_CID_ATOMISP_BAD_PIXEL_DETECTION \
-	(V4L2_CID_PRIVATE_BASE + 0)
-#define V4L2_CID_ATOMISP_POSTPROCESS_GDC_CAC \
-	(V4L2_CID_PRIVATE_BASE + 1)
-#define V4L2_CID_ATOMISP_VIDEO_STABLIZATION \
-	(V4L2_CID_PRIVATE_BASE + 2)
-#define V4L2_CID_ATOMISP_FIXED_PATTERN_NR \
-	(V4L2_CID_PRIVATE_BASE + 3)
-#define V4L2_CID_ATOMISP_FALSE_COLOR_CORRECTION \
-	(V4L2_CID_PRIVATE_BASE + 4)
-#define V4L2_CID_ATOMISP_LOW_LIGHT \
-	(V4L2_CID_PRIVATE_BASE + 5)
-
-/* Camera class:
- * Exposure, Flash and privacy (indicator) light controls, to be upstreamed */
-#define V4L2_CID_CAMERA_LASTP1             (V4L2_CID_CAMERA_CLASS_BASE + 1024)
-
-#define V4L2_CID_FOCAL_ABSOLUTE            (V4L2_CID_CAMERA_LASTP1 + 0)
-#define V4L2_CID_FNUMBER_ABSOLUTE          (V4L2_CID_CAMERA_LASTP1 + 1)
-#define V4L2_CID_FNUMBER_RANGE             (V4L2_CID_CAMERA_LASTP1 + 2)
-
-/* Flash related CIDs, see also:
- * http://linuxtv.org/downloads/v4l-dvb-apis/extended-controls.html\
- * #flash-controls */
-
-/* Request a number of flash-exposed frames. The frame status can be
- * found in the reserved field in the v4l2_buffer struct. */
-#define V4L2_CID_REQUEST_FLASH             (V4L2_CID_CAMERA_LASTP1 + 3)
-/* Query flash driver status. See enum atomisp_flash_status above. */
-#define V4L2_CID_FLASH_STATUS              (V4L2_CID_CAMERA_LASTP1 + 5)
-/* Set the flash mode (see enum atomisp_flash_mode) */
-#define V4L2_CID_FLASH_MODE                (V4L2_CID_CAMERA_LASTP1 + 10)
-
-/* VCM slew control */
-#define V4L2_CID_VCM_SLEW                  (V4L2_CID_CAMERA_LASTP1 + 11)
-/* VCM step time */
-#define V4L2_CID_VCM_TIMEING               (V4L2_CID_CAMERA_LASTP1 + 12)
-
-/* Query Focus Status */
-#define V4L2_CID_FOCUS_STATUS              (V4L2_CID_CAMERA_LASTP1 + 14)
-
-/* Query sensor's binning factor */
-#define V4L2_CID_BIN_FACTOR_HORZ	   (V4L2_CID_CAMERA_LASTP1 + 15)
-#define V4L2_CID_BIN_FACTOR_VERT	   (V4L2_CID_CAMERA_LASTP1 + 16)
-
-/* number of frames to skip at stream start */
-#define V4L2_CID_G_SKIP_FRAMES		   (V4L2_CID_CAMERA_LASTP1 + 17)
-
-/* Query sensor's 2A status */
-#define V4L2_CID_2A_STATUS                 (V4L2_CID_CAMERA_LASTP1 + 18)
-#define V4L2_2A_STATUS_AE_READY            (1 << 0)
-#define V4L2_2A_STATUS_AWB_READY           (1 << 1)
-
-#define V4L2_CID_FMT_AUTO			(V4L2_CID_CAMERA_LASTP1 + 19)
-
-#define V4L2_CID_RUN_MODE			(V4L2_CID_CAMERA_LASTP1 + 20)
-#define ATOMISP_RUN_MODE_VIDEO			1
-#define ATOMISP_RUN_MODE_STILL_CAPTURE		2
-#define ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE	3
-#define ATOMISP_RUN_MODE_PREVIEW		4
-#define ATOMISP_RUN_MODE_SDV			5
-
-#define V4L2_CID_ENABLE_VFPP			(V4L2_CID_CAMERA_LASTP1 + 21)
-#define V4L2_CID_ATOMISP_CONTINUOUS_MODE	(V4L2_CID_CAMERA_LASTP1 + 22)
-#define V4L2_CID_ATOMISP_CONTINUOUS_RAW_BUFFER_SIZE \
-						(V4L2_CID_CAMERA_LASTP1 + 23)
-#define V4L2_CID_ATOMISP_CONTINUOUS_VIEWFINDER \
-						(V4L2_CID_CAMERA_LASTP1 + 24)
-
-#define V4L2_CID_VFPP				(V4L2_CID_CAMERA_LASTP1 + 25)
-#define ATOMISP_VFPP_ENABLE			0
-#define ATOMISP_VFPP_DISABLE_SCALER		1
-#define ATOMISP_VFPP_DISABLE_LOWLAT		2
-
-/* Query real flash status register value */
-#define V4L2_CID_FLASH_STATUS_REGISTER  (V4L2_CID_CAMERA_LASTP1 + 26)
-
-#define V4L2_CID_START_ZSL_CAPTURE	(V4L2_CID_CAMERA_LASTP1 + 28)
-/* Lock and unlock raw buffer */
-#define V4L2_CID_ENABLE_RAW_BUFFER_LOCK (V4L2_CID_CAMERA_LASTP1 + 29)
-
-#define V4L2_CID_DEPTH_MODE		(V4L2_CID_CAMERA_LASTP1 + 30)
-
-#define V4L2_CID_EXPOSURE_ZONE_NUM	(V4L2_CID_CAMERA_LASTP1 + 31)
-/* Disable digital zoom */
-#define V4L2_CID_DISABLE_DZ		(V4L2_CID_CAMERA_LASTP1 + 32)
-
-#define V4L2_CID_TEST_PATTERN_COLOR_R	(V4L2_CID_CAMERA_LASTP1 + 33)
-#define V4L2_CID_TEST_PATTERN_COLOR_GR	(V4L2_CID_CAMERA_LASTP1 + 34)
-#define V4L2_CID_TEST_PATTERN_COLOR_GB	(V4L2_CID_CAMERA_LASTP1 + 35)
-#define V4L2_CID_TEST_PATTERN_COLOR_B	(V4L2_CID_CAMERA_LASTP1 + 36)
-
-#define V4L2_CID_ATOMISP_SELECT_ISP_VERSION	(V4L2_CID_CAMERA_LASTP1 + 38)
-
-#define V4L2_BUF_FLAG_BUFFER_INVALID       0x0400
-#define V4L2_BUF_FLAG_BUFFER_VALID         0x0800
-
-#define V4L2_BUF_TYPE_VIDEO_CAPTURE_ION  (V4L2_BUF_TYPE_PRIVATE + 1024)
-
-#define V4L2_EVENT_ATOMISP_3A_STATS_READY   (V4L2_EVENT_PRIVATE_START + 1)
-#define V4L2_EVENT_ATOMISP_METADATA_READY   (V4L2_EVENT_PRIVATE_START + 2)
-#define V4L2_EVENT_ATOMISP_RAW_BUFFERS_ALLOC_DONE   (V4L2_EVENT_PRIVATE_START + 3)
-#define V4L2_EVENT_ATOMISP_ACC_COMPLETE     (V4L2_EVENT_PRIVATE_START + 4)
-#define V4L2_EVENT_ATOMISP_PAUSE_BUFFER	    (V4L2_EVENT_PRIVATE_START + 5)
-#define V4L2_EVENT_ATOMISP_CSS_RESET	    (V4L2_EVENT_PRIVATE_START + 6)
-/* Nonstandard color effects for V4L2_CID_COLORFX */
-enum {
-	V4L2_COLORFX_SKIN_WHITEN_LOW = 1001,
-	V4L2_COLORFX_SKIN_WHITEN_HIGH = 1002,
-	V4L2_COLORFX_WARM = 1003,
-	V4L2_COLORFX_COLD = 1004,
-	V4L2_COLORFX_WASHED = 1005,
-	V4L2_COLORFX_RED = 1006,
-	V4L2_COLORFX_GREEN = 1007,
-	V4L2_COLORFX_BLUE = 1008,
-	V4L2_COLORFX_PINK = 1009,
-	V4L2_COLORFX_YELLOW = 1010,
-	V4L2_COLORFX_PURPLE = 1011,
-};
-
-#endif /* _ATOM_ISP_H */
-#endif /* CSS15*/
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h
deleted file mode 100644
index c52c56a17e17..000000000000
--- a/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel MID SoC Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2014 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-#ifndef ATOMISP_GMIN_PLATFORM_H_
-#define ATOMISP_GMIN_PLATFORM_H_
-
-#include "atomisp_platform.h"
-
-int atomisp_register_i2c_module(struct v4l2_subdev *subdev,
-                                struct camera_sensor_platform_data *plat_data,
-                                enum intel_v4l2_subdev_type type);
-struct v4l2_subdev *atomisp_gmin_find_subdev(struct i2c_adapter *adapter,
-					     struct i2c_board_info *board_info);
-int atomisp_gmin_remove_subdev(struct v4l2_subdev *sd);
-int gmin_get_var_int(struct device *dev, const char *var, int def);
-int camera_sensor_csi(struct v4l2_subdev *sd, u32 port,
-                      u32 lanes, u32 format, u32 bayer_order, int flag);
-struct camera_sensor_platform_data *gmin_camera_platform_data(
-		struct v4l2_subdev *subdev,
-		enum atomisp_input_format csi_format,
-		enum atomisp_bayer_order csi_bayer);
-
-int atomisp_gmin_register_vcm_control(struct camera_vcm_control *);
-
-#endif
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h
deleted file mode 100644
index aa5e294e7b7d..000000000000
--- a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef ATOMISP_PLATFORM_H_
-#define ATOMISP_PLATFORM_H_
-
-#include <linux/i2c.h>
-#include <linux/sfi.h>
-#include <media/v4l2-subdev.h>
-#include "atomisp.h"
-
-#define MAX_SENSORS_PER_PORT 4
-#define MAX_STREAMS_PER_CHANNEL 2
-
-#define CAMERA_MODULE_ID_LEN 64
-
-enum atomisp_bayer_order {
-	atomisp_bayer_order_grbg,
-	atomisp_bayer_order_rggb,
-	atomisp_bayer_order_bggr,
-	atomisp_bayer_order_gbrg
-};
-
-enum atomisp_input_stream_id {
-	ATOMISP_INPUT_STREAM_GENERAL = 0,
-	ATOMISP_INPUT_STREAM_CAPTURE = 0,
-	ATOMISP_INPUT_STREAM_POSTVIEW,
-	ATOMISP_INPUT_STREAM_PREVIEW,
-	ATOMISP_INPUT_STREAM_VIDEO,
-	ATOMISP_INPUT_STREAM_NUM
-};
-
-enum atomisp_input_format {
-	ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY,/* 8 bits per subpixel (legacy) */
-	ATOMISP_INPUT_FORMAT_YUV420_8, /* 8 bits per subpixel */
-	ATOMISP_INPUT_FORMAT_YUV420_10,/* 10 bits per subpixel */
-	ATOMISP_INPUT_FORMAT_YUV420_16,/* 16 bits per subpixel */
-	ATOMISP_INPUT_FORMAT_YUV422_8, /* UYVY..UVYV, 8 bits per subpixel */
-	ATOMISP_INPUT_FORMAT_YUV422_10,/* UYVY..UVYV, 10 bits per subpixel */
-	ATOMISP_INPUT_FORMAT_YUV422_16,/* UYVY..UVYV, 16 bits per subpixel */
-	ATOMISP_INPUT_FORMAT_RGB_444,  /* BGR..BGR, 4 bits per subpixel */
-	ATOMISP_INPUT_FORMAT_RGB_555,  /* BGR..BGR, 5 bits per subpixel */
-	ATOMISP_INPUT_FORMAT_RGB_565,  /* BGR..BGR, 5 bits B and R, 6 bits G */
-	ATOMISP_INPUT_FORMAT_RGB_666,  /* BGR..BGR, 6 bits per subpixel */
-	ATOMISP_INPUT_FORMAT_RGB_888,  /* BGR..BGR, 8 bits per subpixel */
-	ATOMISP_INPUT_FORMAT_RAW_6,    /* RAW data, 6 bits per pixel */
-	ATOMISP_INPUT_FORMAT_RAW_7,    /* RAW data, 7 bits per pixel */
-	ATOMISP_INPUT_FORMAT_RAW_8,    /* RAW data, 8 bits per pixel */
-	ATOMISP_INPUT_FORMAT_RAW_10,   /* RAW data, 10 bits per pixel */
-	ATOMISP_INPUT_FORMAT_RAW_12,   /* RAW data, 12 bits per pixel */
-	ATOMISP_INPUT_FORMAT_RAW_14,   /* RAW data, 14 bits per pixel */
-	ATOMISP_INPUT_FORMAT_RAW_16,   /* RAW data, 16 bits per pixel */
-	ATOMISP_INPUT_FORMAT_BINARY_8, /* Binary byte stream. */
-
-	/* CSI2-MIPI specific format: Generic short packet data. It is used to
-	 * keep the timing information for the opening/closing of shutters,
-	 * triggering of flashes and etc.
-	 */
-	ATOMISP_INPUT_FORMAT_GENERIC_SHORT1,  /* Generic Short Packet Code 1 */
-	ATOMISP_INPUT_FORMAT_GENERIC_SHORT2,  /* Generic Short Packet Code 2 */
-	ATOMISP_INPUT_FORMAT_GENERIC_SHORT3,  /* Generic Short Packet Code 3 */
-	ATOMISP_INPUT_FORMAT_GENERIC_SHORT4,  /* Generic Short Packet Code 4 */
-	ATOMISP_INPUT_FORMAT_GENERIC_SHORT5,  /* Generic Short Packet Code 5 */
-	ATOMISP_INPUT_FORMAT_GENERIC_SHORT6,  /* Generic Short Packet Code 6 */
-	ATOMISP_INPUT_FORMAT_GENERIC_SHORT7,  /* Generic Short Packet Code 7 */
-	ATOMISP_INPUT_FORMAT_GENERIC_SHORT8,  /* Generic Short Packet Code 8 */
-
-	/* CSI2-MIPI specific format: YUV data.
-	 */
-	ATOMISP_INPUT_FORMAT_YUV420_8_SHIFT,  /* YUV420 8-bit (Chroma Shifted
-						 Pixel Sampling) */
-	ATOMISP_INPUT_FORMAT_YUV420_10_SHIFT, /* YUV420 8-bit (Chroma Shifted
-						 Pixel Sampling) */
-
-	/* CSI2-MIPI specific format: Generic long packet data
-	 */
-	ATOMISP_INPUT_FORMAT_EMBEDDED, /* Embedded 8-bit non Image Data */
-
-	/* CSI2-MIPI specific format: User defined byte-based data. For example,
-	 * the data transmitter (e.g. the SoC sensor) can keep the JPEG data as
-	 * the User Defined Data Type 4 and the MPEG data as the
-	 * User Defined Data Type 7.
-	 */
-	ATOMISP_INPUT_FORMAT_USER_DEF1,  /* User defined 8-bit data type 1 */
-	ATOMISP_INPUT_FORMAT_USER_DEF2,  /* User defined 8-bit data type 2 */
-	ATOMISP_INPUT_FORMAT_USER_DEF3,  /* User defined 8-bit data type 3 */
-	ATOMISP_INPUT_FORMAT_USER_DEF4,  /* User defined 8-bit data type 4 */
-	ATOMISP_INPUT_FORMAT_USER_DEF5,  /* User defined 8-bit data type 5 */
-	ATOMISP_INPUT_FORMAT_USER_DEF6,  /* User defined 8-bit data type 6 */
-	ATOMISP_INPUT_FORMAT_USER_DEF7,  /* User defined 8-bit data type 7 */
-	ATOMISP_INPUT_FORMAT_USER_DEF8,  /* User defined 8-bit data type 8 */
-};
-
-#define N_ATOMISP_INPUT_FORMAT (ATOMISP_INPUT_FORMAT_USER_DEF8 + 1)
-
-
-
-enum intel_v4l2_subdev_type {
-	RAW_CAMERA = 1,
-	SOC_CAMERA = 2,
-	CAMERA_MOTOR = 3,
-	LED_FLASH = 4,
-	XENON_FLASH = 5,
-	FILE_INPUT = 6,
-	TEST_PATTERN = 7,
-};
-
-struct intel_v4l2_subdev_id {
-	char name[17];
-	enum intel_v4l2_subdev_type type;
-	enum atomisp_camera_port    port;
-};
-
-struct intel_v4l2_subdev_i2c_board_info {
-	struct i2c_board_info board_info;
-	int i2c_adapter_id;
-};
-
-struct intel_v4l2_subdev_table {
-	struct intel_v4l2_subdev_i2c_board_info v4l2_subdev;
-	enum intel_v4l2_subdev_type type;
-	enum atomisp_camera_port port;
-	struct v4l2_subdev *subdev;
-};
-
-struct atomisp_platform_data {
-	struct intel_v4l2_subdev_table *subdevs;
-};
-
-/* Describe the capacities of one single sensor. */
-struct atomisp_sensor_caps {
-	/* The number of streams this sensor can output. */
-	int stream_num;
-	bool is_slave;
-};
-
-/* Describe the capacities of sensors connected to one camera port. */
-struct atomisp_camera_caps {
-	/* The number of sensors connected to this camera port. */
-	int sensor_num;
-	/* The capacities of each sensor. */
-	struct atomisp_sensor_caps sensor[MAX_SENSORS_PER_PORT];
-	/* Define whether stream control is required for multiple streams. */
-	bool multi_stream_ctrl;
-};
-
-/*
- *  Sensor of external ISP can send multiple steams with different mipi data
- * type in the same virtual channel. This information needs to come from the
- * sensor or external ISP
- */
-struct atomisp_isys_config_info {
-	u8 input_format;
-	u16 width;
-	u16 height;
-};
-
-struct atomisp_input_stream_info {
-	enum atomisp_input_stream_id stream;
-	u8 enable;
-	/* Sensor driver fills ch_id with the id
-	   of the virtual channel. */
-	u8 ch_id;
-	/* Tells how many streams in this virtual channel. If 0 ignore rest
-	 * and the input format will be from mipi_info */
-	u8 isys_configs;
-	/*
-	 * if more isys_configs is more than 0, sensor needs to configure the
-	 * input format differently. width and height can be 0. If width and
-	 * height is not zero, then the corresponsing data needs to be set
-	 */
-	struct atomisp_isys_config_info isys_info[MAX_STREAMS_PER_CHANNEL];
-};
-
-struct camera_vcm_control;
-struct camera_vcm_ops {
-	int (*power_up)(struct v4l2_subdev *sd, struct camera_vcm_control *vcm);
-	int (*power_down)(struct v4l2_subdev *sd,
-			struct camera_vcm_control *vcm);
-	int (*queryctrl)(struct v4l2_subdev *sd, struct v4l2_queryctrl *qc,
-			struct camera_vcm_control *vcm);
-	int (*g_ctrl)(struct v4l2_subdev *sd, struct v4l2_control *ctrl,
-			struct camera_vcm_control *vcm);
-	int (*s_ctrl)(struct v4l2_subdev *sd, struct v4l2_control *ctrl,
-			struct camera_vcm_control *vcm);
-};
-
-struct camera_vcm_control {
-	char camera_module[CAMERA_MODULE_ID_LEN];
-	struct camera_vcm_ops *ops;
-	struct list_head list;
-};
-
-struct camera_sensor_platform_data {
-	int (*flisclk_ctrl)(struct v4l2_subdev *subdev, int flag);
-	int (*csi_cfg)(struct v4l2_subdev *subdev, int flag);
-
-	/*
-	 * New G-Min power and GPIO interface to control individual
-	 * lines as implemented on all known camera modules.
-	 */
-	int (*gpio0_ctrl)(struct v4l2_subdev *subdev, int on);
-	int (*gpio1_ctrl)(struct v4l2_subdev *subdev, int on);
-	int (*v1p8_ctrl)(struct v4l2_subdev *subdev, int on);
-	int (*v2p8_ctrl)(struct v4l2_subdev *subdev, int on);
-	int (*v1p2_ctrl)(struct v4l2_subdev *subdev, int on);
-	struct camera_vcm_control * (*get_vcm_ctrl)(struct v4l2_subdev *subdev,
-						    char *module_id);
-};
-
-struct camera_mipi_info {
-	enum atomisp_camera_port        port;
-	unsigned int                    num_lanes;
-	enum atomisp_input_format       input_format;
-	enum atomisp_bayer_order        raw_bayer_order;
-	struct atomisp_sensor_mode_data data;
-	enum atomisp_input_format       metadata_format;
-	uint32_t                        metadata_width;
-	uint32_t                        metadata_height;
-	const uint32_t                  *metadata_effective_width;
-};
-
-extern const struct atomisp_platform_data *atomisp_get_platform_data(void);
-extern const struct atomisp_camera_caps *atomisp_get_default_camera_caps(void);
-
-/* API from old platform_camera.h, new CPUID implementation */
-#define __IS_SOC(x) (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && \
-		     boot_cpu_data.x86 == 6 &&                       \
-		     boot_cpu_data.x86_model == x)
-
-#define IS_MFLD	__IS_SOC(0x27)
-#define IS_BYT	__IS_SOC(0x37)
-#define IS_CHT	__IS_SOC(0x4C)
-#define IS_MOFD	__IS_SOC(0x5A)
-
-#endif /* ATOMISP_PLATFORM_H_ */
diff --git a/drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h b/drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h
deleted file mode 100644
index 8988b37943b3..000000000000
--- a/drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef __LIBMSRLISTHELPER_H__
-#define __LIBMSRLISTHELPER_H__
-
-struct i2c_client;
-struct firmware;
-
-extern int load_msr_list(struct i2c_client *client, char *path,
-		const struct firmware **fw);
-extern int apply_msr_data(struct i2c_client *client, const struct firmware *fw);
-extern void release_msr_list(struct i2c_client *client,
-		const struct firmware *fw);
-
-
-#endif /* ifndef __LIBMSRLISTHELPER_H__ */
diff --git a/drivers/staging/media/atomisp/include/media/lm3554.h b/drivers/staging/media/atomisp/include/media/lm3554.h
deleted file mode 100644
index 9276ce44d907..000000000000
--- a/drivers/staging/media/atomisp/include/media/lm3554.h
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * include/media/lm3554.h
- *
- * Copyright (c) 2010-2012 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef _LM3554_H_
-#define _LM3554_H_
-
-#include <linux/videodev2.h>
-#include <media/v4l2-subdev.h>
-
-#define LM3554_ID      3554
-
-#define	v4l2_queryctrl_entry_integer(_id, _name,\
-		_minimum, _maximum, _step, \
-		_default_value, _flags)	\
-	{\
-		.id = (_id), \
-		.type = V4L2_CTRL_TYPE_INTEGER, \
-		.name = _name, \
-		.minimum = (_minimum), \
-		.maximum = (_maximum), \
-		.step = (_step), \
-		.default_value = (_default_value),\
-		.flags = (_flags),\
-	}
-#define	v4l2_queryctrl_entry_boolean(_id, _name,\
-		_default_value, _flags)	\
-	{\
-		.id = (_id), \
-		.type = V4L2_CTRL_TYPE_BOOLEAN, \
-		.name = _name, \
-		.minimum = 0, \
-		.maximum = 1, \
-		.step = 1, \
-		.default_value = (_default_value),\
-		.flags = (_flags),\
-	}
-
-#define	s_ctrl_id_entry_integer(_id, _name, \
-		_minimum, _maximum, _step, \
-		_default_value, _flags, \
-		_s_ctrl, _g_ctrl)	\
-	{\
-		.qc = v4l2_queryctrl_entry_integer(_id, _name,\
-				_minimum, _maximum, _step,\
-				_default_value, _flags), \
-		.s_ctrl = _s_ctrl, \
-		.g_ctrl = _g_ctrl, \
-	}
-
-#define	s_ctrl_id_entry_boolean(_id, _name, \
-		_default_value, _flags, \
-		_s_ctrl, _g_ctrl)	\
-	{\
-		.qc = v4l2_queryctrl_entry_boolean(_id, _name,\
-				_default_value, _flags), \
-		.s_ctrl = _s_ctrl, \
-		.g_ctrl = _g_ctrl, \
-	}
-
-/* Value settings for Flash Time-out Duration*/
-#define LM3554_DEFAULT_TIMEOUT          512U
-#define LM3554_MIN_TIMEOUT              32U
-#define LM3554_MAX_TIMEOUT              1024U
-#define LM3554_TIMEOUT_STEPSIZE         32U
-
-/* Flash modes */
-#define LM3554_MODE_SHUTDOWN            0
-#define LM3554_MODE_INDICATOR           1
-#define LM3554_MODE_TORCH               2
-#define LM3554_MODE_FLASH               3
-
-/* timer delay time */
-#define LM3554_TIMER_DELAY		5
-
-/* Percentage <-> value macros */
-#define LM3554_MIN_PERCENT                   0U
-#define LM3554_MAX_PERCENT                   100U
-#define LM3554_CLAMP_PERCENTAGE(val) \
-	clamp(val, LM3554_MIN_PERCENT, LM3554_MAX_PERCENT)
-
-#define LM3554_VALUE_TO_PERCENT(v, step)     (((((unsigned long)(v))*(step))+50)/100)
-#define LM3554_PERCENT_TO_VALUE(p, step)     (((((unsigned long)(p))*100)+(step>>1))/(step))
-
-/* Product specific limits
- * TODO: get these from platform data */
-#define LM3554_FLASH_MAX_LVL   0x0F /* 1191mA */
-
-/* Flash brightness, input is percentage, output is [0..15] */
-#define LM3554_FLASH_STEP	\
-	((100ul*(LM3554_MAX_PERCENT)+((LM3554_FLASH_MAX_LVL)>>1))/((LM3554_FLASH_MAX_LVL)))
-#define LM3554_FLASH_DEFAULT_BRIGHTNESS \
-	LM3554_VALUE_TO_PERCENT(13, LM3554_FLASH_STEP)
-
-/* Torch brightness, input is percentage, output is [0..7] */
-#define LM3554_TORCH_STEP                    1250
-#define LM3554_TORCH_DEFAULT_BRIGHTNESS \
-	LM3554_VALUE_TO_PERCENT(2, LM3554_TORCH_STEP)
-
-/* Indicator brightness, input is percentage, output is [0..3] */
-#define LM3554_INDICATOR_STEP                2500
-#define LM3554_INDICATOR_DEFAULT_BRIGHTNESS \
-	LM3554_VALUE_TO_PERCENT(1, LM3554_INDICATOR_STEP)
-
-/*
- * lm3554_platform_data - Flash controller platform data
- */
-struct lm3554_platform_data {
-	int gpio_torch;
-	int gpio_strobe;
-	int gpio_reset;
-
-	unsigned int current_limit;
-	unsigned int envm_tx2;
-	unsigned int tx2_polarity;
-};
-
-#endif /* _LM3554_H_ */
-
diff --git a/drivers/staging/media/atomisp/pci/Kconfig b/drivers/staging/media/atomisp/pci/Kconfig
deleted file mode 100644
index 41f116d52060..000000000000
--- a/drivers/staging/media/atomisp/pci/Kconfig
+++ /dev/null
@@ -1,14 +0,0 @@
-#
-# Kconfig for ISP driver
-#
-
-config VIDEO_ATOMISP
-	tristate "Intel Atom Image Signal Processor Driver"
-	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
-	select IOSF_MBI
-	select VIDEOBUF_VMALLOC
-	---help---
-	  Say Y here if your platform supports Intel Atom SoC
-	  camera imaging subsystem.
-	  To compile this driver as a module, choose M here: the
-	  module will be called atomisp
diff --git a/drivers/staging/media/atomisp/pci/Makefile b/drivers/staging/media/atomisp/pci/Makefile
deleted file mode 100644
index 61ad1fbb1ee6..000000000000
--- a/drivers/staging/media/atomisp/pci/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-#
-# Makefile for ISP driver
-#
-
-obj-$(CONFIG_VIDEO_ATOMISP) += atomisp2/
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/Makefile b/drivers/staging/media/atomisp/pci/atomisp2/Makefile
deleted file mode 100644
index 7fead5fc9a7d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/Makefile
+++ /dev/null
@@ -1,349 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-atomisp-objs += \
-	atomisp_drvfs.o \
-	atomisp_file.o \
-	css2400/sh_css_mipi.o \
-	css2400/runtime/pipeline/src/pipeline.o \
-	css2400/runtime/spctrl/src/spctrl.o \
-	css2400/runtime/rmgr/src/rmgr.o \
-	css2400/runtime/rmgr/src/rmgr_vbuf.o \
-	css2400/runtime/isp_param/src/isp_param.o \
-	css2400/runtime/inputfifo/src/inputfifo.o \
-	css2400/runtime/queue/src/queue_access.o \
-	css2400/runtime/queue/src/queue.o \
-	css2400/runtime/frame/src/frame.o \
-	css2400/runtime/eventq/src/eventq.o \
-	css2400/runtime/binary/src/binary.o \
-	css2400/runtime/timer/src/timer.o \
-	css2400/runtime/isys/src/csi_rx_rmgr.o \
-	css2400/runtime/isys/src/isys_stream2mmio_rmgr.o \
-	css2400/runtime/isys/src/virtual_isys.o \
-	css2400/runtime/isys/src/rx.o \
-	css2400/runtime/isys/src/isys_dma_rmgr.o \
-	css2400/runtime/isys/src/ibuf_ctrl_rmgr.o \
-	css2400/runtime/isys/src/isys_init.o \
-	css2400/runtime/bufq/src/bufq.o \
-	css2400/runtime/ifmtr/src/ifmtr.o \
-	css2400/runtime/debug/src/ia_css_debug.o \
-	css2400/runtime/event/src/event.o \
-	css2400/sh_css_sp.o \
-	css2400/css_2400_system/spmem_dump.o \
-	css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_states.o \
-	css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_configs.o \
-	css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_params.o \
-	css2400/sh_css_stream_format.o \
-	css2400/sh_css_hrt.o \
-	css2400/sh_css_properties.o \
-	css2400/memory_realloc.o \
-	css2400/hive_isp_css_shared/host/tag.o \
-	css2400/sh_css_params.o \
-	css2400/sh_css.o \
-	css2400/isp/kernels/hdr/ia_css_hdr.host.o \
-	css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.o \
-	css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.o \
-	css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.o \
-	css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.o \
-	css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.o \
-	css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.o \
-	css2400/isp/kernels/raw/raw_1.0/ia_css_raw.host.o \
-	css2400/isp/kernels/ref/ref_1.0/ia_css_ref.host.o \
-	css2400/isp/kernels/qplane/qplane_2/ia_css_qplane.host.o \
-	css2400/isp/kernels/norm/norm_1.0/ia_css_norm.host.o \
-	css2400/isp/kernels/output/output_1.0/ia_css_output.host.o \
-	css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.o \
-	css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.o \
-	css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.o \
-	css2400/isp/kernels/macc/macc_1.0/ia_css_macc.host.o \
-	css2400/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.o \
-	css2400/isp/kernels/csc/csc_1.0/ia_css_csc.host.o \
-	css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.o \
-	css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.o \
-	css2400/isp/kernels/dpc2/ia_css_dpc2.host.o \
-	css2400/isp/kernels/fc/fc_1.0/ia_css_formats.host.o \
-	css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.o \
-	css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.o \
-	css2400/isp/kernels/ctc/ctc2/ia_css_ctc2.host.o \
-	css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.o \
-	css2400/isp/kernels/bh/bh_2/ia_css_bh.host.o \
-	css2400/isp/kernels/bnlm/ia_css_bnlm.host.o \
-	css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.o \
-	css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.o \
-	css2400/isp/kernels/anr/anr_1.0/ia_css_anr.host.o \
-	css2400/isp/kernels/anr/anr_2/ia_css_anr2_table.host.o \
-	css2400/isp/kernels/anr/anr_2/ia_css_anr2.host.o \
-	css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.o \
-	css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.o \
-	css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.o \
-	css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.o \
-	css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.o \
-	css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.o \
-	css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.o \
-	css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.o \
-	css2400/isp/kernels/de/de_1.0/ia_css_de.host.o \
-	css2400/isp/kernels/de/de_2/ia_css_de2.host.o \
-	css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.o \
-	css2400/isp/kernels/gc/gc_2/ia_css_gc2_table.host.o \
-	css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.o \
-	css2400/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.o \
-	css2400/isp/kernels/crop/crop_1.0/ia_css_crop.host.o \
-	css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.o \
-	css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.o \
-	css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.o \
-	css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.o \
-	css2400/isp/kernels/ob/ob2/ia_css_ob2.host.o \
-	css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.o \
-	css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.o \
-	css2400/isp/kernels/eed1_8/ia_css_eed1_8.host.o \
-	css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.o \
-	css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.o \
-	css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.o \
-	css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.o \
-	css2400/sh_css_pipe.o \
-	css2400/ia_css_device_access.o \
-	css2400/sh_css_host_data.o \
-	css2400/sh_css_mmu.o \
-	css2400/sh_css_metadata.o \
-	css2400/base/refcount/src/refcount.o \
-	css2400/base/circbuf/src/circbuf.o \
-	css2400/camera/pipe/src/pipe_binarydesc.o \
-	css2400/camera/pipe/src/pipe_util.o \
-	css2400/camera/pipe/src/pipe_stagedesc.o \
-	css2400/camera/util/src/util.o \
-	css2400/sh_css_metrics.o \
-	css2400/sh_css_version.o \
-	css2400/ia_css_memory_access.o \
-	css2400/sh_css_param_shading.o \
-	css2400/sh_css_morph.o \
-	css2400/sh_css_firmware.o \
-	css2400/hive_isp_css_common/host/isp.o \
-	css2400/hive_isp_css_common/host/gdc.o \
-	css2400/hive_isp_css_common/host/sp.o \
-	css2400/hive_isp_css_common/host/vmem.o \
-	css2400/hive_isp_css_common/host/dma.o \
-	css2400/hive_isp_css_common/host/input_formatter.o \
-	css2400/hive_isp_css_common/host/debug.o \
-	css2400/hive_isp_css_common/host/hmem.o \
-	css2400/hive_isp_css_common/host/gp_device.o \
-	css2400/hive_isp_css_common/host/fifo_monitor.o \
-	css2400/hive_isp_css_common/host/gp_timer.o \
-	css2400/hive_isp_css_common/host/irq.o \
-	css2400/hive_isp_css_common/host/input_system.o \
-	css2400/hive_isp_css_common/host/timed_ctrl.o \
-	css2400/hive_isp_css_common/host/mmu.o \
-	css2400/hive_isp_css_common/host/event_fifo.o \
-	css2400/sh_css_param_dvs.o \
-	css2400/sh_css_shading.o \
-	css2400/sh_css_stream.o \
-	mmu/sh_mmu_mrfld.o \
-	mmu/isp_mmu.o \
-	atomisp_acc.o \
-	atomisp_compat_css20.o \
-	atomisp_fops.o \
-	atomisp_subdev.o \
-	atomisp_ioctl.o \
-	atomisp_compat_ioctl32.o \
-	atomisp_csi2.o \
-	atomisp_cmd.o \
-	atomisp_tpg.o \
-	hmm/hmm_vm.o \
-	hmm/hmm.o \
-	hmm/hmm_bo.o \
-	hmm/hmm_reserved_pool.o \
-	hmm/hmm_dynamic_pool.o \
-	hrt/hive_isp_css_mm_hrt.o \
-	atomisp_v4l2.o
-	
-# These will be needed when clean merge CHT support nicely into the driver
-# Keep them here handy for when we get to that point
-#
-
-obj-cht= \
-	css2400/css_2401_system/spmem_dump.o \
-	css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_states.o \
-	css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_configs.o \
-	css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_params.o \
-	css2400/css_2401_csi2p_system/spmem_dump.o \
-	css2400/css_2401_csi2p_system/host/isys_stream2mmio.o \
-	css2400/css_2401_csi2p_system/host/ibuf_ctrl.o \
-	css2400/css_2401_csi2p_system/host/isys_irq.o \
-	css2400/css_2401_csi2p_system/host/isys_dma.o \
-	css2400/css_2401_csi2p_system/host/csi_rx.o \
-	css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_states.o \
-	css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_configs.o \
-	css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_params.o \
-
-#	-I$(atomisp)/css2400/css_2401_csi2p_system/ \
-#	-I$(atomisp)/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ \
-#	-I$(atomisp)/css2400/css_2401_csi2p_system/host/ \
-#	-I$(atomisp)/css2400/css_2401_csi2p_system/hrt/ \
-#	-I$(atomisp)/css2400/css_2401_system/hive_isp_css_2401_system_generated/ \
-#	-I$(atomisp)/css2400/css_2401_system/hrt/ \
-
-
-
-obj-$(CONFIG_VIDEO_ATOMISP) += atomisp.o
-
-atomisp = $(srctree)/drivers/staging/media/atomisp/pci/atomisp2
-
-INCLUDES += \
-	-I$(atomisp)/ \
-	-I$(atomisp)/css2400/ \
-	-I$(atomisp)/hrt/ \
-	-I$(atomisp)/include/ \
-	-I$(atomisp)/include/hmm/ \
-	-I$(atomisp)/include/mmu/ \
-	-I$(atomisp)/css2400/base/circbuf/interface/ \
-	-I$(atomisp)/css2400/base/refcount/interface/ \
-	-I$(atomisp)/css2400/camera/pipe/interface/ \
-	-I$(atomisp)/css2400/camera/util/interface/ \
-	-I$(atomisp)/css2400/css_2400_system/ \
-	-I$(atomisp)/css2400/css_2400_system/hive_isp_css_2400_system_generated/ \
-	-I$(atomisp)/css2400/css_2400_system/hrt/ \
-	-I$(atomisp)/css2400/hive_isp_css_common/ \
-	-I$(atomisp)/css2400/hive_isp_css_common/host/ \
-	-I$(atomisp)/css2400/hive_isp_css_include/ \
-	-I$(atomisp)/css2400/hive_isp_css_include/device_access/ \
-	-I$(atomisp)/css2400/hive_isp_css_include/host/ \
-	-I$(atomisp)/css2400/hive_isp_css_include/memory_access/ \
-	-I$(atomisp)/css2400/hive_isp_css_shared/ \
-	-I$(atomisp)/css2400/hive_isp_css_shared/host/ \
-	-I$(atomisp)/css2400/isp/kernels/ \
-	-I$(atomisp)/css2400/isp/kernels/aa/aa_2/ \
-	-I$(atomisp)/css2400/isp/kernels/anr/anr_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/anr/anr_2/ \
-	-I$(atomisp)/css2400/isp/kernels/bh/bh_2/ \
-	-I$(atomisp)/css2400/isp/kernels/bnlm/ \
-	-I$(atomisp)/css2400/isp/kernels/bnr/ \
-	-I$(atomisp)/css2400/isp/kernels/bnr/bnr_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/bnr/bnr2_2/ \
-	-I$(atomisp)/css2400/isp/kernels/cnr/ \
-	-I$(atomisp)/css2400/isp/kernels/cnr/cnr_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/cnr/cnr_2/ \
-	-I$(atomisp)/css2400/isp/kernels/conversion/ \
-	-I$(atomisp)/css2400/isp/kernels/conversion/conversion_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/copy_output/ \
-	-I$(atomisp)/css2400/isp/kernels/copy_output/copy_output_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/crop/ \
-	-I$(atomisp)/css2400/isp/kernels/crop/crop_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/csc/ \
-	-I$(atomisp)/css2400/isp/kernels/csc/csc_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/ctc/ \
-	-I$(atomisp)/css2400/isp/kernels/ctc/ctc_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/ctc/ctc1_5/ \
-	-I$(atomisp)/css2400/isp/kernels/ctc/ctc2/ \
-	-I$(atomisp)/css2400/isp/kernels/de/ \
-	-I$(atomisp)/css2400/isp/kernels/de/de_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/de/de_2/ \
-	-I$(atomisp)/css2400/isp/kernels/dpc2/ \
-	-I$(atomisp)/css2400/isp/kernels/dp/ \
-	-I$(atomisp)/css2400/isp/kernels/dp/dp_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/dvs/ \
-	-I$(atomisp)/css2400/isp/kernels/dvs/dvs_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/eed1_8/ \
-	-I$(atomisp)/css2400/isp/kernels/fc/ \
-	-I$(atomisp)/css2400/isp/kernels/fc/fc_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/fixedbds/ \
-	-I$(atomisp)/css2400/isp/kernels/fixedbds/fixedbds_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/fpn/ \
-	-I$(atomisp)/css2400/isp/kernels/fpn/fpn_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/gc/ \
-	-I$(atomisp)/css2400/isp/kernels/gc/gc_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/gc/gc_2/ \
-	-I$(atomisp)/css2400/isp/kernels/hdr/ \
-	-I$(atomisp)/css2400/isp/kernels/io_ls/ \
-	-I$(atomisp)/css2400/isp/kernels/io_ls/bayer_io_ls/ \
-	-I$(atomisp)/css2400/isp/kernels/io_ls/common/ \
-	-I$(atomisp)/css2400/isp/kernels/io_ls/yuv444_io_ls/ \
-	-I$(atomisp)/css2400/isp/kernels/ipu2_io_ls/ \
-	-I$(atomisp)/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ \
-	-I$(atomisp)/css2400/isp/kernels/ipu2_io_ls/common/ \
-	-I$(atomisp)/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ \
-	-I$(atomisp)/css2400/isp/kernels/iterator/ \
-	-I$(atomisp)/css2400/isp/kernels/iterator/iterator_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/macc/ \
-	-I$(atomisp)/css2400/isp/kernels/macc/macc_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/macc/macc1_5/ \
-	-I$(atomisp)/css2400/isp/kernels/norm/ \
-	-I$(atomisp)/css2400/isp/kernels/norm/norm_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/ob/ \
-	-I$(atomisp)/css2400/isp/kernels/ob/ob_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/ob/ob2/ \
-	-I$(atomisp)/css2400/isp/kernels/output/ \
-	-I$(atomisp)/css2400/isp/kernels/output/output_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/qplane/ \
-	-I$(atomisp)/css2400/isp/kernels/qplane/qplane_2/ \
-	-I$(atomisp)/css2400/isp/kernels/raw_aa_binning/ \
-	-I$(atomisp)/css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/raw/ \
-	-I$(atomisp)/css2400/isp/kernels/raw/raw_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/ref/ \
-	-I$(atomisp)/css2400/isp/kernels/ref/ref_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/s3a/ \
-	-I$(atomisp)/css2400/isp/kernels/s3a/s3a_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/sc/ \
-	-I$(atomisp)/css2400/isp/kernels/sc/sc_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/sdis/ \
-	-I$(atomisp)/css2400/isp/kernels/sdis/common/ \
-	-I$(atomisp)/css2400/isp/kernels/sdis/sdis_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/sdis/sdis_2/ \
-	-I$(atomisp)/css2400/isp/kernels/tdf/ \
-	-I$(atomisp)/css2400/isp/kernels/tdf/tdf_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/tnr/ \
-	-I$(atomisp)/css2400/isp/kernels/tnr/tnr_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/tnr/tnr3/ \
-	-I$(atomisp)/css2400/isp/kernels/uds/ \
-	-I$(atomisp)/css2400/isp/kernels/uds/uds_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/vf/ \
-	-I$(atomisp)/css2400/isp/kernels/vf/vf_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/wb/ \
-	-I$(atomisp)/css2400/isp/kernels/wb/wb_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/xnr/ \
-	-I$(atomisp)/css2400/isp/kernels/xnr/xnr_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/xnr/xnr_3.0/ \
-	-I$(atomisp)/css2400/isp/kernels/ynr/ \
-	-I$(atomisp)/css2400/isp/kernels/ynr/ynr_1.0/ \
-	-I$(atomisp)/css2400/isp/kernels/ynr/ynr_2/ \
-	-I$(atomisp)/css2400/isp/modes/interface/ \
-	-I$(atomisp)/css2400/runtime/binary/interface/ \
-	-I$(atomisp)/css2400/runtime/bufq/interface/ \
-	-I$(atomisp)/css2400/runtime/debug/interface/ \
-	-I$(atomisp)/css2400/runtime/event/interface/ \
-	-I$(atomisp)/css2400/runtime/eventq/interface/ \
-	-I$(atomisp)/css2400/runtime/frame/interface/ \
-	-I$(atomisp)/css2400/runtime/ifmtr/interface/ \
-	-I$(atomisp)/css2400/runtime/inputfifo/interface/ \
-	-I$(atomisp)/css2400/runtime/isp_param/interface/ \
-	-I$(atomisp)/css2400/runtime/isys/interface/ \
-	-I$(atomisp)/css2400/runtime/isys/src/ \
-	-I$(atomisp)/css2400/runtime/pipeline/interface/ \
-	-I$(atomisp)/css2400/runtime/queue/interface/ \
-	-I$(atomisp)/css2400/runtime/queue/src/ \
-	-I$(atomisp)/css2400/runtime/rmgr/interface/ \
-	-I$(atomisp)/css2400/runtime/spctrl/interface/ \
-	-I$(atomisp)/css2400/runtime/tagger/interface/
-
-ifeq ($(CONFIG_ION),y)
-INCLUDES += -I$(srctree)/drivers/staging/android/ion
-endif
-
-DEFINES := -DHRT_HW -DHRT_ISP_CSS_CUSTOM_HOST -DHRT_USE_VIR_ADDRS -D__HOST__
-#DEFINES += -DUSE_DYNAMIC_BIN
-#DEFINES += -DISP_POWER_GATING
-#DEFINES += -DUSE_INTERRUPTS
-#DEFINES += -DUSE_SSSE3
-#DEFINES += -DPUNIT_CAMERA_BUSY
-#DEFINES += -DUSE_KMEM_CACHE
-
-DEFINES += -DATOMISP_POSTFIX=\"css2400b0_v21\" -DISP2400B0
-DEFINES += -DSYSTEM_hive_isp_css_2400_system -DISP2400
-
-ccflags-y += $(INCLUDES) $(DEFINES) -fno-common
-
-# HACK! While this driver is in bad shape, don't enable several warnings
-#       that would be otherwise enabled with W=1
-ccflags-y += $(call cc-disable-warning, implicit-fallthrough)
-ccflags-y += $(call cc-disable-warning, missing-prototypes)
-ccflags-y += $(call cc-disable-warning, missing-declarations)
-ccflags-y += $(call cc-disable-warning, suggest-attribute=format)
-ccflags-y += $(call cc-disable-warning, unused-const-variable)
-ccflags-y += $(call cc-disable-warning, unused-but-set-variable)
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp-regs.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp-regs.h
deleted file mode 100644
index 5d102a4f8aff..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp-regs.h
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2012 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef ATOMISP_REGS_H
-#define ATOMISP_REGS_H
-
-/* common register definitions */
-#define PUNIT_PORT		0x04
-#define CCK_PORT		0x14
-
-#define PCICMDSTS		0x01
-#define INTR			0x0f
-#define MSI_CAPID		0x24
-#define MSI_ADDRESS		0x25
-#define MSI_DATA		0x26
-#define INTR_CTL		0x27
-
-#define PCI_MSI_CAPID		0x90
-#define PCI_MSI_ADDR		0x94
-#define PCI_MSI_DATA		0x98
-#define PCI_INTERRUPT_CTRL	0x9C
-#define PCI_I_CONTROL		0xfc
-
-/* MRFLD specific register definitions */
-#define MRFLD_CSI_AFE		0x39
-#define MRFLD_CSI_CONTROL	0x3a
-#define MRFLD_CSI_RCOMP		0x3d
-
-#define MRFLD_PCI_PMCS		0x84
-#define MRFLD_PCI_CSI_ACCESS_CTRL_VIOL	0xd4
-#define MRFLD_PCI_CSI_AFE_HS_CONTROL	0xdc
-#define MRFLD_PCI_CSI_AFE_RCOMP_CONTROL	0xe0
-#define MRFLD_PCI_CSI_CONTROL		0xe8
-#define MRFLD_PCI_CSI_AFE_TRIM_CONTROL	0xe4
-#define MRFLD_PCI_CSI_DEADLINE_CONTROL	0xec
-#define MRFLD_PCI_CSI_RCOMP_CONTROL	0xf4
-
-/* Select Arasan (legacy)/Intel input system */
-#define MRFLD_PCI_CSI_CONTROL_PARPATHEN	BIT(24)
-/* Enable CSI interface (ANN B0/K0) */
-#define MRFLD_PCI_CSI_CONTROL_CSI_READY	BIT(25)
-
-/*
- * Enables the combining of adjacent 32-byte read requests to the same
- * cache line. When cleared, each 32-byte read request is sent as a
- * separate request on the IB interface.
- */
-#define MRFLD_PCI_I_CONTROL_ENABLE_READ_COMBINING	0x1
-
-/*
- * Register: MRFLD_PCI_CSI_RCOMP_CONTROL
- * If cleared, the high speed clock going to the digital logic is gated when
- * RCOMP update is happening. The clock is gated for a minimum of 100 nsec.
- * If this bit is set, then the high speed clock is not gated during the
- * update cycle.
- */
-#define MRFLD_PCI_CSI_HS_OVR_CLK_GATE_ON_UPDATE		0x800000
-
-/*
- * Enables the combining of adjacent 32-byte write requests to the same
- * cache line. When cleared, each 32-byte write request is sent as a
- * separate request on the IB interface.
- */
-#define MRFLD_PCI_I_CONTROL_ENABLE_WRITE_COMBINING	0x2
-
-#define MRFLD_PCI_I_CONTROL_SRSE_RESET_MASK		0xc
-
-#define MRFLD_PCI_CSI1_HSRXCLKTRIM		0x2
-#define MRFLD_PCI_CSI1_HSRXCLKTRIM_SHIFT	16
-#define MRFLD_PCI_CSI2_HSRXCLKTRIM		0x3
-#define MRFLD_PCI_CSI2_HSRXCLKTRIM_SHIFT	24
-#define MRFLD_PCI_CSI3_HSRXCLKTRIM		0x2
-#define MRFLD_PCI_CSI3_HSRXCLKTRIM_SHIFT	28
-#define MRFLD_PCI_CSI_HSRXCLKTRIM_MASK		0xf
-
-/*
- * This register is IUINT MMIO register, it is used to select the CSI
- * receiver backend.
- * 1: SH CSI backend
- * 0: Arasan CSI backend
- */
-#define MRFLD_CSI_RECEIVER_SELECTION_REG       0x8081c
-
-#define MRFLD_INTR_CLEAR_REG		       0x50c
-#define MRFLD_INTR_STATUS_REG		       0x508
-#define MRFLD_INTR_ENABLE_REG		       0x510
-
-#define MRFLD_MAX_ZOOM_FACTOR	1024
-
-/* MRFLD ISP POWER related */
-#define MRFLD_ISPSSPM0         0x39
-#define MRFLD_ISPSSPM0_ISPSSC_OFFSET   0
-#define MRFLD_ISPSSPM0_ISPSSS_OFFSET   24
-#define MRFLD_ISPSSPM0_ISPSSC_MASK     0x3
-#define MRFLD_ISPSSPM0_IUNIT_POWER_ON  0
-#define MRFLD_ISPSSPM0_IUNIT_POWER_OFF 0x3
-#define MRFLD_ISPSSDVFS			0x13F
-#define MRFLD_BIT0			0x0001
-#define MRFLD_BIT1			0x0002
-
-/* MRFLD CSI lane configuration related */
-#define MRFLD_PORT_CONFIG_NUM  8
-#define MRFLD_PORT_NUM         3
-#define MRFLD_PORT1_ENABLE_SHIFT       0
-#define MRFLD_PORT2_ENABLE_SHIFT       1
-#define MRFLD_PORT3_ENABLE_SHIFT       2
-#define MRFLD_PORT1_LANES_SHIFT        3
-#define MRFLD_PORT2_LANES_SHIFT        7
-#define MRFLD_PORT3_LANES_SHIFT        8
-#define MRFLD_PORT_CONFIG_MASK 0x000f03ff
-#define MRFLD_PORT_CONFIGCODE_SHIFT    16
-#define MRFLD_ALL_CSI_PORTS_OFF_MASK   0x7
-
-#define CHV_PORT3_LANES_SHIFT		9
-#define CHV_PORT_CONFIG_MASK		0x1f07ff
-
-#define ISPSSPM1				0x3a
-#define ISP_FREQ_STAT_MASK			(0x1f << ISP_FREQ_STAT_OFFSET)
-#define ISP_REQ_FREQ_MASK			0x1f
-#define ISP_FREQ_VALID_MASK			(0x1 << ISP_FREQ_VALID_OFFSET)
-#define ISP_FREQ_STAT_OFFSET			0x18
-#define ISP_REQ_GUAR_FREQ_OFFSET		0x8
-#define ISP_REQ_FREQ_OFFSET			0x0
-#define ISP_FREQ_VALID_OFFSET			0x7
-#define ISP_FREQ_RULE_ANY			0x0
-
-#define ISP_FREQ_457MHZ				0x1C9
-#define ISP_FREQ_400MHZ				0x190
-#define ISP_FREQ_356MHZ				0x164
-#define ISP_FREQ_320MHZ				0x140
-#define ISP_FREQ_266MHZ				0x10a
-#define ISP_FREQ_200MHZ				0xc8
-#define ISP_FREQ_100MHZ				0x64
-
-#define HPLL_FREQ_800MHZ			0x320
-#define HPLL_FREQ_1600MHZ			0x640
-#define HPLL_FREQ_2000MHZ			0x7D0
-
-#define CCK_FUSE_REG_0			0x08
-#define CCK_FUSE_HPLL_FREQ_MASK		0x03
-
-#if defined(ISP2401)
-#define ISP_FREQ_MAX	ISP_FREQ_320MHZ
-#else
-#define ISP_FREQ_MAX	ISP_FREQ_400MHZ
-#endif
-
-/* ISP2401 CSI2+ receiver delay settings */
-#define CSI2_PORT_A_BASE					0xC0000
-#define CSI2_PORT_B_BASE					0xC2000
-#define CSI2_PORT_C_BASE					0xC4000
-
-#define CSI2_LANE_CL_BASE					0x418
-#define CSI2_LANE_D0_BASE					0x420
-#define CSI2_LANE_D1_BASE					0x428
-#define CSI2_LANE_D2_BASE					0x430
-#define CSI2_LANE_D3_BASE					0x438
-
-#define CSI2_REG_RX_CSI_DLY_CNT_TERMEN				0
-#define CSI2_REG_RX_CSI_DLY_CNT_SETTLE				0x4
-
-#define CSI2_PORT_A_RX_CSI_DLY_CNT_TERMEN_CLANE			0xC0418
-#define CSI2_PORT_A_RX_CSI_DLY_CNT_SETTLE_CLANE			0xC041C
-#define CSI2_PORT_A_RX_CSI_DLY_CNT_TERMEN_DLANE0		0xC0420
-#define CSI2_PORT_A_RX_CSI_DLY_CNT_SETTLE_DLANE0		0xC0424
-#define CSI2_PORT_A_RX_CSI_DLY_CNT_TERMEN_DLANE1		0xC0428
-#define CSI2_PORT_A_RX_CSI_DLY_CNT_SETTLE_DLANE1		0xC042C
-#define CSI2_PORT_A_RX_CSI_DLY_CNT_TERMEN_DLANE2		0xC0430
-#define CSI2_PORT_A_RX_CSI_DLY_CNT_SETTLE_DLANE2		0xC0434
-#define CSI2_PORT_A_RX_CSI_DLY_CNT_TERMEN_DLANE3		0xC0438
-#define CSI2_PORT_A_RX_CSI_DLY_CNT_SETTLE_DLANE3		0xC043C
-
-#define CSI2_PORT_B_RX_CSI_DLY_CNT_TERMEN_CLANE			0xC2418
-#define CSI2_PORT_B_RX_CSI_DLY_CNT_SETTLE_CLANE			0xC241C
-#define CSI2_PORT_B_RX_CSI_DLY_CNT_TERMEN_DLANE0		0xC2420
-#define CSI2_PORT_B_RX_CSI_DLY_CNT_SETTLE_DLANE0		0xC2424
-#define CSI2_PORT_B_RX_CSI_DLY_CNT_TERMEN_DLANE1		0xC2428
-#define CSI2_PORT_B_RX_CSI_DLY_CNT_SETTLE_DLANE1		0xC242C
-
-#define CSI2_PORT_C_RX_CSI_DLY_CNT_TERMEN_CLANE			0xC4418
-#define CSI2_PORT_C_RX_CSI_DLY_CNT_SETTLE_CLANE			0xC441C
-#define CSI2_PORT_C_RX_CSI_DLY_CNT_TERMEN_DLANE0		0xC4420
-#define CSI2_PORT_C_RX_CSI_DLY_CNT_SETTLE_DLANE0		0xC4424
-#define CSI2_PORT_C_RX_CSI_DLY_CNT_TERMEN_DLANE1		0xC4428
-#define CSI2_PORT_C_RX_CSI_DLY_CNT_SETTLE_DLANE1		0xC442C
-
-#define DMA_BURST_SIZE_REG					0xCD408
-
-#define ISP_DFS_TRY_TIMES	2
-
-#endif /* ATOMISP_REGS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_acc.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_acc.c
deleted file mode 100644
index a6638edee360..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_acc.c
+++ /dev/null
@@ -1,604 +0,0 @@
-/*
- * Support for Clovertrail PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2012 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-/*
- * This file implements loadable acceleration firmware API,
- * including ioctls to map and unmap acceleration parameters and buffers.
- */
-
-#include <linux/init.h>
-#include <media/v4l2-event.h>
-
-#include "atomisp_acc.h"
-#include "atomisp_internal.h"
-#include "atomisp_compat.h"
-#include "atomisp_cmd.h"
-
-#include "hrt/hive_isp_css_mm_hrt.h"
-#include "memory_access/memory_access.h"
-#include "ia_css.h"
-
-static const struct {
-	unsigned int flag;
-	enum atomisp_css_pipe_id pipe_id;
-} acc_flag_to_pipe[] = {
-	{ ATOMISP_ACC_FW_LOAD_FL_PREVIEW, CSS_PIPE_ID_PREVIEW },
-	{ ATOMISP_ACC_FW_LOAD_FL_COPY, CSS_PIPE_ID_COPY },
-	{ ATOMISP_ACC_FW_LOAD_FL_VIDEO, CSS_PIPE_ID_VIDEO },
-	{ ATOMISP_ACC_FW_LOAD_FL_CAPTURE, CSS_PIPE_ID_CAPTURE },
-	{ ATOMISP_ACC_FW_LOAD_FL_ACC, CSS_PIPE_ID_ACC }
-};
-
-/*
- * Allocate struct atomisp_acc_fw along with space for firmware.
- * The returned struct atomisp_acc_fw is cleared (firmware region is not).
- */
-static struct atomisp_acc_fw *acc_alloc_fw(unsigned int fw_size)
-{
-	struct atomisp_acc_fw *acc_fw;
-
-	acc_fw = kzalloc(sizeof(*acc_fw), GFP_KERNEL);
-	if (!acc_fw)
-		return NULL;
-
-	acc_fw->fw = vmalloc(fw_size);
-	if (!acc_fw->fw) {
-		kfree(acc_fw);
-		return NULL;
-	}
-
-	return acc_fw;
-}
-
-static void acc_free_fw(struct atomisp_acc_fw *acc_fw)
-{
-	vfree(acc_fw->fw);
-	kfree(acc_fw);
-}
-
-static struct atomisp_acc_fw *
-acc_get_fw(struct atomisp_sub_device *asd, unsigned int handle)
-{
-	struct atomisp_acc_fw *acc_fw;
-
-	list_for_each_entry(acc_fw, &asd->acc.fw, list)
-		if (acc_fw->handle == handle)
-			return acc_fw;
-
-	return NULL;
-}
-
-static struct atomisp_map *acc_get_map(struct atomisp_sub_device *asd,
-				       unsigned long css_ptr, size_t length)
-{
-	struct atomisp_map *atomisp_map;
-
-	list_for_each_entry(atomisp_map, &asd->acc.memory_maps, list) {
-		if (atomisp_map->ptr == css_ptr &&
-		    atomisp_map->length == length)
-			return atomisp_map;
-	}
-	return NULL;
-}
-
-static int acc_stop_acceleration(struct atomisp_sub_device *asd)
-{
-	int ret;
-
-	ret = atomisp_css_stop_acc_pipe(asd);
-	atomisp_css_destroy_acc_pipe(asd);
-
-	return ret;
-}
-
-void atomisp_acc_cleanup(struct atomisp_device *isp)
-{
-	int i;
-
-	for (i = 0; i < isp->num_of_streams; i++)
-		ida_destroy(&isp->asd[i].acc.ida);
-}
-
-void atomisp_acc_release(struct atomisp_sub_device *asd)
-{
-	struct atomisp_acc_fw *acc_fw, *ta;
-	struct atomisp_map *atomisp_map, *tm;
-
-	/* Stop acceleration if already running */
-	if (asd->acc.pipeline)
-		acc_stop_acceleration(asd);
-
-	/* Unload all loaded acceleration binaries */
-	list_for_each_entry_safe(acc_fw, ta, &asd->acc.fw, list) {
-		list_del(&acc_fw->list);
-		ida_remove(&asd->acc.ida, acc_fw->handle);
-		acc_free_fw(acc_fw);
-	}
-
-	/* Free all mapped memory blocks */
-	list_for_each_entry_safe(atomisp_map, tm, &asd->acc.memory_maps, list) {
-		list_del(&atomisp_map->list);
-		hmm_free(atomisp_map->ptr);
-		kfree(atomisp_map);
-	}
-}
-
-int atomisp_acc_load_to_pipe(struct atomisp_sub_device *asd,
-			     struct atomisp_acc_fw_load_to_pipe *user_fw)
-{
-	static const unsigned int pipeline_flags =
-		ATOMISP_ACC_FW_LOAD_FL_PREVIEW | ATOMISP_ACC_FW_LOAD_FL_COPY |
-		ATOMISP_ACC_FW_LOAD_FL_VIDEO |
-		ATOMISP_ACC_FW_LOAD_FL_CAPTURE | ATOMISP_ACC_FW_LOAD_FL_ACC;
-
-	struct atomisp_acc_fw *acc_fw;
-	int handle;
-
-	if (!user_fw->data || user_fw->size < sizeof(*acc_fw->fw))
-		return -EINVAL;
-
-	/* Binary has to be enabled at least for one pipeline */
-	if (!(user_fw->flags & pipeline_flags))
-		return -EINVAL;
-
-	/* We do not support other flags yet */
-	if (user_fw->flags & ~pipeline_flags)
-		return -EINVAL;
-
-	if (user_fw->type < ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT ||
-	    user_fw->type > ATOMISP_ACC_FW_LOAD_TYPE_STANDALONE)
-		return -EINVAL;
-
-	if (asd->acc.pipeline || asd->acc.extension_mode)
-		return -EBUSY;
-
-	acc_fw = acc_alloc_fw(user_fw->size);
-	if (!acc_fw)
-		return -ENOMEM;
-
-	if (copy_from_user(acc_fw->fw, user_fw->data, user_fw->size)) {
-		acc_free_fw(acc_fw);
-		return -EFAULT;
-	}
-
-	if (!ida_pre_get(&asd->acc.ida, GFP_KERNEL) ||
-	    ida_get_new_above(&asd->acc.ida, 1, &handle)) {
-		acc_free_fw(acc_fw);
-		return -ENOSPC;
-	}
-
-	user_fw->fw_handle = handle;
-	acc_fw->handle = handle;
-	acc_fw->flags = user_fw->flags;
-	acc_fw->type = user_fw->type;
-	acc_fw->fw->handle = handle;
-
-	/*
-	 * correct isp firmware type in order ISP firmware can be appended
-	 * to correct pipe properly
-	 */
-	if (acc_fw->fw->type == ia_css_isp_firmware) {
-		static const int type_to_css[] = {
-			[ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT] =
-				IA_CSS_ACC_OUTPUT,
-			[ATOMISP_ACC_FW_LOAD_TYPE_VIEWFINDER] =
-				IA_CSS_ACC_VIEWFINDER,
-			[ATOMISP_ACC_FW_LOAD_TYPE_STANDALONE] =
-				IA_CSS_ACC_STANDALONE,
-		};
-		acc_fw->fw->info.isp.type = type_to_css[acc_fw->type];
-	}
-
-	list_add_tail(&acc_fw->list, &asd->acc.fw);
-	return 0;
-}
-
-int atomisp_acc_load(struct atomisp_sub_device *asd,
-		     struct atomisp_acc_fw_load *user_fw)
-{
-	struct atomisp_acc_fw_load_to_pipe ltp = {0};
-	int r;
-
-	ltp.flags = ATOMISP_ACC_FW_LOAD_FL_ACC;
-	ltp.type = ATOMISP_ACC_FW_LOAD_TYPE_STANDALONE;
-	ltp.size = user_fw->size;
-	ltp.data = user_fw->data;
-	r = atomisp_acc_load_to_pipe(asd, &ltp);
-	user_fw->fw_handle = ltp.fw_handle;
-	return r;
-}
-
-int atomisp_acc_unload(struct atomisp_sub_device *asd, unsigned int *handle)
-{
-	struct atomisp_acc_fw *acc_fw;
-
-	if (asd->acc.pipeline || asd->acc.extension_mode)
-		return -EBUSY;
-
-	acc_fw = acc_get_fw(asd, *handle);
-	if (!acc_fw)
-		return -EINVAL;
-
-	list_del(&acc_fw->list);
-	ida_remove(&asd->acc.ida, acc_fw->handle);
-	acc_free_fw(acc_fw);
-
-	return 0;
-}
-
-int atomisp_acc_start(struct atomisp_sub_device *asd, unsigned int *handle)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_acc_fw *acc_fw;
-	int ret;
-	unsigned int nbin;
-
-	if (asd->acc.pipeline || asd->acc.extension_mode)
-		return -EBUSY;
-
-	/* Invalidate caches. FIXME: should flush only necessary buffers */
-	wbinvd();
-
-	ret = atomisp_css_create_acc_pipe(asd);
-	if (ret)
-		return ret;
-
-	nbin = 0;
-	list_for_each_entry(acc_fw, &asd->acc.fw, list) {
-		if (*handle != 0 && *handle != acc_fw->handle)
-			continue;
-
-		if (acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_STANDALONE)
-			continue;
-
-		/* Add the binary into the pipeline */
-		ret = atomisp_css_load_acc_binary(asd, acc_fw->fw, nbin);
-		if (ret < 0) {
-			dev_err(isp->dev, "acc_load_binary failed\n");
-			goto err_stage;
-		}
-
-		ret = atomisp_css_set_acc_parameters(acc_fw);
-		if (ret < 0) {
-			dev_err(isp->dev, "acc_set_parameters failed\n");
-			goto err_stage;
-		}
-		nbin++;
-	}
-	if (nbin < 1) {
-		/* Refuse creating pipelines with no binaries */
-		dev_err(isp->dev, "%s: no acc binary available\n", __func__);
-		ret = -EINVAL;
-		goto err_stage;
-	}
-
-	ret = atomisp_css_start_acc_pipe(asd);
-	if (ret) {
-		dev_err(isp->dev, "%s: atomisp_acc_start_acc_pipe failed\n",
-			__func__);
-		goto err_stage;
-	}
-
-	return 0;
-
-err_stage:
-	atomisp_css_destroy_acc_pipe(asd);
-	return ret;
-}
-
-int atomisp_acc_wait(struct atomisp_sub_device *asd, unsigned int *handle)
-{
-	struct atomisp_device *isp = asd->isp;
-	int ret;
-
-	if (!asd->acc.pipeline)
-		return -ENOENT;
-
-	if (*handle && !acc_get_fw(asd, *handle))
-		return -EINVAL;
-
-	ret = atomisp_css_wait_acc_finish(asd);
-	if (acc_stop_acceleration(asd) == -EIO) {
-		atomisp_reset(isp);
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-void atomisp_acc_done(struct atomisp_sub_device *asd, unsigned int handle)
-{
-	struct v4l2_event event = { 0 };
-
-	event.type = V4L2_EVENT_ATOMISP_ACC_COMPLETE;
-	event.u.frame_sync.frame_sequence = atomic_read(&asd->sequence);
-	event.id = handle;
-
-	v4l2_event_queue(asd->subdev.devnode, &event);
-}
-
-int atomisp_acc_map(struct atomisp_sub_device *asd, struct atomisp_acc_map *map)
-{
-	struct atomisp_map *atomisp_map;
-	ia_css_ptr cssptr;
-	int pgnr;
-
-	if (map->css_ptr)
-		return -EINVAL;
-
-	if (asd->acc.pipeline)
-		return -EBUSY;
-
-	if (map->user_ptr) {
-		/* Buffer to map must be page-aligned */
-		if ((unsigned long)map->user_ptr & ~PAGE_MASK) {
-			dev_err(asd->isp->dev,
-				"%s: mapped buffer address %p is not page aligned\n",
-				__func__, map->user_ptr);
-			return -EINVAL;
-		}
-
-		pgnr = DIV_ROUND_UP(map->length, PAGE_SIZE);
-		cssptr = hrt_isp_css_mm_alloc_user_ptr(
-				map->length, map->user_ptr,
-				pgnr, HRT_USR_PTR,
-				(map->flags & ATOMISP_MAP_FLAG_CACHED));
-	} else {
-		/* Allocate private buffer. */
-		if (map->flags & ATOMISP_MAP_FLAG_CACHED)
-			cssptr = hrt_isp_css_mm_calloc_cached(map->length);
-		else
-			cssptr = hrt_isp_css_mm_calloc(map->length);
-	}
-
-	if (!cssptr)
-		return -ENOMEM;
-
-	atomisp_map = kmalloc(sizeof(*atomisp_map), GFP_KERNEL);
-	if (!atomisp_map) {
-		hmm_free(cssptr);
-		return -ENOMEM;
-	}
-	atomisp_map->ptr = cssptr;
-	atomisp_map->length = map->length;
-	list_add(&atomisp_map->list, &asd->acc.memory_maps);
-
-	dev_dbg(asd->isp->dev, "%s: userptr %p, css_address 0x%x, size %d\n",
-		__func__, map->user_ptr, cssptr, map->length);
-	map->css_ptr = cssptr;
-	return 0;
-}
-
-int atomisp_acc_unmap(struct atomisp_sub_device *asd, struct atomisp_acc_map *map)
-{
-	struct atomisp_map *atomisp_map;
-
-	if (asd->acc.pipeline)
-		return -EBUSY;
-
-	atomisp_map = acc_get_map(asd, map->css_ptr, map->length);
-	if (!atomisp_map)
-		return -EINVAL;
-
-	list_del(&atomisp_map->list);
-	hmm_free(atomisp_map->ptr);
-	kfree(atomisp_map);
-	return 0;
-}
-
-int atomisp_acc_s_mapped_arg(struct atomisp_sub_device *asd,
-			     struct atomisp_acc_s_mapped_arg *arg)
-{
-	struct atomisp_acc_fw *acc_fw;
-
-	if (arg->memory >= ATOMISP_ACC_NR_MEMORY)
-		return -EINVAL;
-
-	if (asd->acc.pipeline)
-		return -EBUSY;
-
-	acc_fw = acc_get_fw(asd, arg->fw_handle);
-	if (!acc_fw)
-		return -EINVAL;
-
-	if (arg->css_ptr != 0 || arg->length != 0) {
-		/* Unless the parameter is cleared, check that it exists */
-		if (!acc_get_map(asd, arg->css_ptr, arg->length))
-			return -EINVAL;
-	}
-
-	acc_fw->args[arg->memory].length = arg->length;
-	acc_fw->args[arg->memory].css_ptr = arg->css_ptr;
-
-	dev_dbg(asd->isp->dev, "%s: mem %d, address %p, size %ld\n",
-		__func__, arg->memory, (void *)arg->css_ptr,
-		(unsigned long)arg->length);
-	return 0;
-}
-
-/*
- * Appends the loaded acceleration binary extensions to the
- * current ISP mode. Must be called just before sh_css_start().
- */
-int atomisp_acc_load_extensions(struct atomisp_sub_device *asd)
-{
-	struct atomisp_acc_fw *acc_fw;
-	bool ext_loaded = false;
-	bool continuous = asd->continuous_mode->val &&
-			  asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW;
-	int ret = 0, i = -1;
-	struct atomisp_device *isp = asd->isp;
-
-	if (asd->acc.pipeline || asd->acc.extension_mode)
-		return -EBUSY;
-
-	/* Invalidate caches. FIXME: should flush only necessary buffers */
-	wbinvd();
-
-	list_for_each_entry(acc_fw, &asd->acc.fw, list) {
-		if (acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT &&
-		    acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_VIEWFINDER)
-			continue;
-
-		for (i = 0; i < ARRAY_SIZE(acc_flag_to_pipe); i++) {
-			/* QoS (ACC pipe) acceleration stages are currently
-			 * allowed only in continuous mode. Skip them for
-			 * all other modes. */
-			if (!continuous &&
-			    acc_flag_to_pipe[i].flag ==
-			    ATOMISP_ACC_FW_LOAD_FL_ACC)
-				continue;
-
-			if (acc_fw->flags & acc_flag_to_pipe[i].flag) {
-				ret = atomisp_css_load_acc_extension(asd,
-					acc_fw->fw,
-					acc_flag_to_pipe[i].pipe_id,
-					acc_fw->type);
-				if (ret)
-					goto error;
-
-				ext_loaded = true;
-			}
-		}
-
-		ret = atomisp_css_set_acc_parameters(acc_fw);
-		if (ret < 0)
-			goto error;
-	}
-
-	if (!ext_loaded)
-		return ret;
-
-	ret = atomisp_css_update_stream(asd);
-	if (ret) {
-		dev_err(isp->dev, "%s: update stream failed.\n", __func__);
-		goto error;
-	}
-
-	asd->acc.extension_mode = true;
-	return 0;
-
-error:
-	while (--i >= 0) {
-		if (acc_fw->flags & acc_flag_to_pipe[i].flag) {
-			atomisp_css_unload_acc_extension(asd, acc_fw->fw,
-					acc_flag_to_pipe[i].pipe_id);
-		}
-	}
-
-	list_for_each_entry_continue_reverse(acc_fw, &asd->acc.fw, list) {
-		if (acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT &&
-		    acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_VIEWFINDER)
-			continue;
-
-		for (i = ARRAY_SIZE(acc_flag_to_pipe) - 1; i >= 0; i--) {
-			if (!continuous &&
-			    acc_flag_to_pipe[i].flag ==
-			    ATOMISP_ACC_FW_LOAD_FL_ACC)
-				continue;
-			if (acc_fw->flags & acc_flag_to_pipe[i].flag) {
-				atomisp_css_unload_acc_extension(asd,
-					acc_fw->fw,
-					acc_flag_to_pipe[i].pipe_id);
-			}
-		}
-	}
-	return ret;
-}
-
-void atomisp_acc_unload_extensions(struct atomisp_sub_device *asd)
-{
-	struct atomisp_acc_fw *acc_fw;
-	int i;
-
-	if (!asd->acc.extension_mode)
-		return;
-
-	list_for_each_entry_reverse(acc_fw, &asd->acc.fw, list) {
-		if (acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_OUTPUT &&
-		    acc_fw->type != ATOMISP_ACC_FW_LOAD_TYPE_VIEWFINDER)
-			continue;
-
-		for (i = ARRAY_SIZE(acc_flag_to_pipe) - 1; i >= 0; i--) {
-			if (acc_fw->flags & acc_flag_to_pipe[i].flag) {
-				atomisp_css_unload_acc_extension(asd,
-					acc_fw->fw,
-					acc_flag_to_pipe[i].pipe_id);
-			}
-		}
-	}
-
-	asd->acc.extension_mode = false;
-}
-
-int atomisp_acc_set_state(struct atomisp_sub_device *asd,
-			  struct atomisp_acc_state *arg)
-{
-	struct atomisp_acc_fw *acc_fw;
-	bool enable = (arg->flags & ATOMISP_STATE_FLAG_ENABLE) != 0;
-	struct ia_css_pipe *pipe;
-	enum ia_css_err r;
-	int i;
-
-	if (!asd->acc.extension_mode)
-		return -EBUSY;
-
-	if (arg->flags & ~ATOMISP_STATE_FLAG_ENABLE)
-		return -EINVAL;
-
-	acc_fw = acc_get_fw(asd, arg->fw_handle);
-	if (!acc_fw)
-		return -EINVAL;
-
-	if (enable)
-		wbinvd();
-
-	for (i = 0; i < ARRAY_SIZE(acc_flag_to_pipe); i++) {
-		if (acc_fw->flags & acc_flag_to_pipe[i].flag) {
-			pipe = asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
-				pipes[acc_flag_to_pipe[i].pipe_id];
-			r = ia_css_pipe_set_qos_ext_state(pipe, acc_fw->handle,
-							  enable);
-			if (r != IA_CSS_SUCCESS)
-				return -EBADRQC;
-		}
-	}
-
-	if (enable)
-		acc_fw->flags |= ATOMISP_ACC_FW_LOAD_FL_ENABLE;
-	else
-		acc_fw->flags &= ~ATOMISP_ACC_FW_LOAD_FL_ENABLE;
-
-	return 0;
-}
-
-int atomisp_acc_get_state(struct atomisp_sub_device *asd,
-			  struct atomisp_acc_state *arg)
-{
-	struct atomisp_acc_fw *acc_fw;
-
-	if (!asd->acc.extension_mode)
-		return -EBUSY;
-
-	acc_fw = acc_get_fw(asd, arg->fw_handle);
-	if (!acc_fw)
-		return -EINVAL;
-
-	arg->flags = acc_fw->flags;
-
-	return 0;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_acc.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_acc.h
deleted file mode 100644
index 56386154643b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_acc.h
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Support for Clovertrail PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2012 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __ATOMISP_ACC_H__
-#define __ATOMISP_ACC_H__
-
-#include "../../include/linux/atomisp.h"
-#include "atomisp_internal.h"
-
-#include "ia_css_types.h"
-
-/*
- * Interface functions for AtomISP driver acceleration API implementation.
- */
-
-struct atomisp_sub_device;
-
-void atomisp_acc_cleanup(struct atomisp_device *isp);
-
-/*
- * Free up any allocated resources.
- * Must be called each time when the device is closed.
- * Note that there isn't corresponding open() call;
- * this function may be called sequentially multiple times.
- * Must be called to free up resources before driver is unloaded.
- */
-void atomisp_acc_release(struct atomisp_sub_device *asd);
-
-/* Load acceleration binary. DEPRECATED. */
-int atomisp_acc_load(struct atomisp_sub_device *asd,
-		     struct atomisp_acc_fw_load *fw);
-
-/* Load acceleration binary with specified properties */
-int atomisp_acc_load_to_pipe(struct atomisp_sub_device *asd,
-			     struct atomisp_acc_fw_load_to_pipe *fw);
-
-/* Unload specified acceleration binary */
-int atomisp_acc_unload(struct atomisp_sub_device *asd,
-		       unsigned int *handle);
-
-/*
- * Map a memory region into ISP memory space.
- */
-int atomisp_acc_map(struct atomisp_sub_device *asd,
-		    struct atomisp_acc_map *map);
-
-/*
- * Unmap a mapped memory region.
- */
-int atomisp_acc_unmap(struct atomisp_sub_device *asd,
-		      struct atomisp_acc_map *map);
-
-/*
- * Set acceleration binary argument to a previously mapped memory region.
- */
-int atomisp_acc_s_mapped_arg(struct atomisp_sub_device *asd,
-			     struct atomisp_acc_s_mapped_arg *arg);
-
-
-/*
- * Start acceleration.
- * Return immediately, acceleration is left running in background.
- * Specify either acceleration binary or pipeline which to start.
- */
-int atomisp_acc_start(struct atomisp_sub_device *asd,
-		      unsigned int *handle);
-
-/*
- * Wait until acceleration finishes.
- * This MUST be called after each acceleration has been started.
- * Specify either acceleration binary or pipeline handle.
- */
-int atomisp_acc_wait(struct atomisp_sub_device *asd,
-		     unsigned int *handle);
-
-/*
- * Used by ISR to notify ACC stage finished.
- * This is internally used and does not export as IOCTL.
- */
-void atomisp_acc_done(struct atomisp_sub_device *asd, unsigned int handle);
-
-/*
- * Appends the loaded acceleration binary extensions to the
- * current ISP mode. Must be called just before atomisp_css_start().
- */
-int atomisp_acc_load_extensions(struct atomisp_sub_device *asd);
-
-/*
- * Must be called after streaming is stopped:
- * unloads any loaded acceleration extensions.
- */
-void atomisp_acc_unload_extensions(struct atomisp_sub_device *asd);
-
-/*
- * Set acceleration firmware flags.
- */
-int atomisp_acc_set_state(struct atomisp_sub_device *asd,
-			  struct atomisp_acc_state *arg);
-
-/*
- * Get acceleration firmware flags.
- */
-int atomisp_acc_get_state(struct atomisp_sub_device *asd,
-			  struct atomisp_acc_state *arg);
-
-#endif /* __ATOMISP_ACC_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.c
deleted file mode 100644
index fa6ea506f8b1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.c
+++ /dev/null
@@ -1,6696 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#include <linux/firmware.h>
-#include <linux/pci.h>
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/kfifo.h>
-#include <linux/pm_runtime.h>
-#include <linux/timer.h>
-
-#include <asm/iosf_mbi.h>
-
-#include <media/v4l2-event.h>
-#include <media/videobuf-vmalloc.h>
-
-#define CREATE_TRACE_POINTS
-#include "atomisp_trace_event.h"
-
-#include "atomisp_cmd.h"
-#include "atomisp_common.h"
-#include "atomisp_fops.h"
-#include "atomisp_internal.h"
-#include "atomisp_ioctl.h"
-#include "atomisp-regs.h"
-#include "atomisp_tables.h"
-#include "atomisp_acc.h"
-#include "atomisp_compat.h"
-#include "atomisp_subdev.h"
-#include "atomisp_dfs_tables.h"
-
-#include "hrt/hive_isp_css_mm_hrt.h"
-
-#include "sh_css_hrt.h"
-#include "sh_css_defs.h"
-#include "system_global.h"
-#include "sh_css_internal.h"
-#include "sh_css_sp.h"
-#include "gp_device.h"
-#include "device_access.h"
-#include "irq.h"
-
-#include "ia_css_types.h"
-#include "ia_css_stream.h"
-#include "error_support.h"
-#include "hrt/bits.h"
-
-
-/* We should never need to run the flash for more than 2 frames.
- * At 15fps this means 133ms. We set the timeout a bit longer.
- * Each flash driver is supposed to set its own timeout, but
- * just in case someone else changed the timeout, we set it
- * here to make sure we don't damage the flash hardware. */
-#define FLASH_TIMEOUT 800 /* ms */
-
-union host {
-	struct {
-		void *kernel_ptr;
-		void __user *user_ptr;
-		int size;
-	} scalar;
-	struct {
-		void *hmm_ptr;
-	} ptr;
-};
-
-/*
- * get sensor:dis71430/ov2720 related info from v4l2_subdev->priv data field.
- * subdev->priv is set in mrst.c
- */
-struct camera_mipi_info *atomisp_to_sensor_mipi_info(struct v4l2_subdev *sd)
-{
-	return (struct camera_mipi_info *)v4l2_get_subdev_hostdata(sd);
-}
-
-/*
- * get struct atomisp_video_pipe from v4l2 video_device
- */
-struct atomisp_video_pipe *atomisp_to_video_pipe(struct video_device *dev)
-{
-	return (struct atomisp_video_pipe *)
-	    container_of(dev, struct atomisp_video_pipe, vdev);
-}
-
-/*
- * get struct atomisp_acc_pipe from v4l2 video_device
- */
-struct atomisp_acc_pipe *atomisp_to_acc_pipe(struct video_device *dev)
-{
-	return (struct atomisp_acc_pipe *)
-	    container_of(dev, struct atomisp_acc_pipe, vdev);
-}
-
-static unsigned short atomisp_get_sensor_fps(struct atomisp_sub_device *asd)
-{
-	struct v4l2_subdev_frame_interval fi;
-	struct atomisp_device *isp = asd->isp;
-
-	unsigned short fps = 0;
-	int ret;
-
-	ret = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-			       video, g_frame_interval, &fi);
-
-	if (!ret && fi.interval.numerator)
-		fps = fi.interval.denominator / fi.interval.numerator;
-
-	return fps;
-}
-
-/*
- * DFS progress is shown as follows:
- * 1. Target frequency is calculated according to FPS/Resolution/ISP running
- *    mode.
- * 2. Ratio is calculated using formula: 2 * HPLL / target frequency - 1
- *    with proper rounding.
- * 3. Set ratio to ISPFREQ40, 1 to FREQVALID and ISPFREQGUAR40
- *    to 200MHz in ISPSSPM1.
- * 4. Wait for FREQVALID to be cleared by P-Unit.
- * 5. Wait for field ISPFREQSTAT40 in ISPSSPM1 turn to ratio set in 3.
- */
-static int write_target_freq_to_hw(struct atomisp_device *isp,
-				   unsigned int new_freq)
-{
-	unsigned int ratio, timeout, guar_ratio;
-	u32 isp_sspm1 = 0;
-	int i;
-
-	if (!isp->hpll_freq) {
-		dev_err(isp->dev, "failed to get hpll_freq. no change to freq\n");
-		return -EINVAL;
-	}
-
-	iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, ISPSSPM1, &isp_sspm1);
-	if (isp_sspm1 & ISP_FREQ_VALID_MASK) {
-		dev_dbg(isp->dev, "clearing ISPSSPM1 valid bit.\n");
-		iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_REG_WRITE, ISPSSPM1,
-				    isp_sspm1 & ~(1 << ISP_FREQ_VALID_OFFSET));
-	}
-
-	ratio = (2 * isp->hpll_freq + new_freq / 2) / new_freq - 1;
-	guar_ratio = (2 * isp->hpll_freq + 200 / 2) / 200 - 1;
-
-	iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, ISPSSPM1, &isp_sspm1);
-	isp_sspm1 &= ~(0x1F << ISP_REQ_FREQ_OFFSET);
-
-	for (i = 0; i < ISP_DFS_TRY_TIMES; i++) {
-		iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_REG_WRITE, ISPSSPM1,
-				   isp_sspm1
-				   | ratio << ISP_REQ_FREQ_OFFSET
-				   | 1 << ISP_FREQ_VALID_OFFSET
-				   | guar_ratio << ISP_REQ_GUAR_FREQ_OFFSET);
-
-		iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, ISPSSPM1, &isp_sspm1);
-		timeout = 20;
-		while ((isp_sspm1 & ISP_FREQ_VALID_MASK) && timeout) {
-			iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, ISPSSPM1, &isp_sspm1);
-			dev_dbg(isp->dev, "waiting for ISPSSPM1 valid bit to be 0.\n");
-			udelay(100);
-			timeout--;
-		}
-
-		if (timeout != 0)
-			break;
-	}
-
-	if (timeout == 0) {
-		dev_err(isp->dev, "DFS failed due to HW error.\n");
-		return -EINVAL;
-	}
-
-	iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, ISPSSPM1, &isp_sspm1);
-	timeout = 10;
-	while (((isp_sspm1 >> ISP_FREQ_STAT_OFFSET) != ratio) && timeout) {
-		iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, ISPSSPM1, &isp_sspm1);
-		dev_dbg(isp->dev, "waiting for ISPSSPM1 status bit to be 0x%x.\n",
-			new_freq);
-		udelay(100);
-		timeout--;
-	}
-	if (timeout == 0) {
-		dev_err(isp->dev, "DFS target freq is rejected by HW.\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-int atomisp_freq_scaling(struct atomisp_device *isp,
-			 enum atomisp_dfs_mode mode,
-			 bool force)
-{
-	/* FIXME! Only use subdev[0] status yet */
-	struct atomisp_sub_device *asd = &isp->asd[0];
-	const struct atomisp_dfs_config *dfs;
-	unsigned int new_freq;
-	struct atomisp_freq_scaling_rule curr_rules;
-	int i, ret;
-	unsigned short fps = 0;
-
-	if (isp->sw_contex.power_state != ATOM_ISP_POWER_UP) {
-		dev_err(isp->dev, "DFS cannot proceed due to no power.\n");
-		return -EINVAL;
-	}
-
-	if ((isp->pdev->device & ATOMISP_PCI_DEVICE_SOC_MASK) ==
-		ATOMISP_PCI_DEVICE_SOC_CHT && ATOMISP_USE_YUVPP(asd))
-		isp->dfs = &dfs_config_cht_soc;
-
-	dfs = isp->dfs;
-
-	if (dfs->lowest_freq == 0 || dfs->max_freq_at_vmin == 0 ||
-	    dfs->highest_freq == 0 || dfs->dfs_table_size == 0 ||
-	    !dfs->dfs_table) {
-		dev_err(isp->dev, "DFS configuration is invalid.\n");
-		return -EINVAL;
-	}
-
-	if (mode == ATOMISP_DFS_MODE_LOW) {
-		new_freq = dfs->lowest_freq;
-		goto done;
-	}
-
-	if (mode == ATOMISP_DFS_MODE_MAX) {
-		new_freq = dfs->highest_freq;
-		goto done;
-	}
-
-	fps = atomisp_get_sensor_fps(asd);
-	if (fps == 0)
-		return -EINVAL;
-
-	curr_rules.width = asd->fmt[asd->capture_pad].fmt.width;
-	curr_rules.height = asd->fmt[asd->capture_pad].fmt.height;
-	curr_rules.fps = fps;
-	curr_rules.run_mode = asd->run_mode->val;
-	/*
-	 * For continuous mode, we need to make the capture setting applied
-	 * since preview mode, because there is no chance to do this when
-	 * starting image capture.
-	 */
-	if (asd->continuous_mode->val) {
-		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-			curr_rules.run_mode = ATOMISP_RUN_MODE_SDV;
-		else
-			curr_rules.run_mode =
-				ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE;
-	}
-
-	/* search for the target frequency by looping freq rules*/
-	for (i = 0; i < dfs->dfs_table_size; i++) {
-		if (curr_rules.width != dfs->dfs_table[i].width &&
-		    dfs->dfs_table[i].width != ISP_FREQ_RULE_ANY)
-			continue;
-		if (curr_rules.height != dfs->dfs_table[i].height &&
-		    dfs->dfs_table[i].height != ISP_FREQ_RULE_ANY)
-			continue;
-		if (curr_rules.fps != dfs->dfs_table[i].fps &&
-		    dfs->dfs_table[i].fps != ISP_FREQ_RULE_ANY)
-			continue;
-		if (curr_rules.run_mode != dfs->dfs_table[i].run_mode &&
-		    dfs->dfs_table[i].run_mode != ISP_FREQ_RULE_ANY)
-			continue;
-		break;
-	}
-
-	if (i == dfs->dfs_table_size)
-		new_freq = dfs->max_freq_at_vmin;
-	else
-		new_freq = dfs->dfs_table[i].isp_freq;
-
-done:
-	dev_dbg(isp->dev, "DFS target frequency=%d.\n", new_freq);
-
-	if ((new_freq == isp->sw_contex.running_freq) && !force)
-		return 0;
-
-	dev_dbg(isp->dev, "Programming DFS frequency to %d\n", new_freq);
-
-	ret = write_target_freq_to_hw(isp, new_freq);
-	if (!ret) {
-		isp->sw_contex.running_freq = new_freq;
-		trace_ipu_pstate(new_freq, -1);
-	}
-	return ret;
-}
-
-/*
- * reset and restore ISP
- */
-int atomisp_reset(struct atomisp_device *isp)
-{
-	/* Reset ISP by power-cycling it */
-	int ret = 0;
-
-	dev_dbg(isp->dev, "%s\n", __func__);
-	atomisp_css_suspend(isp);
-	ret = atomisp_runtime_suspend(isp->dev);
-	if (ret < 0)
-		dev_err(isp->dev, "atomisp_runtime_suspend failed, %d\n", ret);
-	ret = atomisp_mrfld_power_down(isp);
-	if (ret < 0) {
-		dev_err(isp->dev, "can not disable ISP power\n");
-	} else {
-		ret = atomisp_mrfld_power_up(isp);
-		if (ret < 0)
-			dev_err(isp->dev, "can not enable ISP power\n");
-		ret = atomisp_runtime_resume(isp->dev);
-		if (ret < 0)
-			dev_err(isp->dev, "atomisp_runtime_resume failed, %d\n", ret);
-	}
-	ret = atomisp_css_resume(isp);
-	if (ret)
-		isp->isp_fatal_error = true;
-
-	return ret;
-}
-
-/*
- * interrupt disable functions
- */
-static void disable_isp_irq(enum hrt_isp_css_irq irq)
-{
-	irq_disable_channel(IRQ0_ID, irq);
-
-	if (irq != hrt_isp_css_irq_sp)
-		return;
-
-	cnd_sp_irq_enable(SP0_ID, false);
-}
-
-/*
- * interrupt clean function
- */
-static void clear_isp_irq(enum hrt_isp_css_irq irq)
-{
-	irq_clear_all(IRQ0_ID);
-}
-
-void atomisp_msi_irq_init(struct atomisp_device *isp, struct pci_dev *dev)
-{
-	u32 msg32;
-	u16 msg16;
-
-	pci_read_config_dword(dev, PCI_MSI_CAPID, &msg32);
-	msg32 |= 1 << MSI_ENABLE_BIT;
-	pci_write_config_dword(dev, PCI_MSI_CAPID, msg32);
-
-	msg32 = (1 << INTR_IER) | (1 << INTR_IIR);
-	pci_write_config_dword(dev, PCI_INTERRUPT_CTRL, msg32);
-
-	pci_read_config_word(dev, PCI_COMMAND, &msg16);
-	msg16 |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
-		  PCI_COMMAND_INTX_DISABLE);
-	pci_write_config_word(dev, PCI_COMMAND, msg16);
-}
-
-void atomisp_msi_irq_uninit(struct atomisp_device *isp, struct pci_dev *dev)
-{
-	u32 msg32;
-	u16 msg16;
-
-	pci_read_config_dword(dev, PCI_MSI_CAPID, &msg32);
-	msg32 &=  ~(1 << MSI_ENABLE_BIT);
-	pci_write_config_dword(dev, PCI_MSI_CAPID, msg32);
-
-	msg32 = 0x0;
-	pci_write_config_dword(dev, PCI_INTERRUPT_CTRL, msg32);
-
-	pci_read_config_word(dev, PCI_COMMAND, &msg16);
-	msg16 &= ~(PCI_COMMAND_MASTER);
-	pci_write_config_word(dev, PCI_COMMAND, msg16);
-}
-
-static void atomisp_sof_event(struct atomisp_sub_device *asd)
-{
-	struct v4l2_event event = {0};
-
-	event.type = V4L2_EVENT_FRAME_SYNC;
-	event.u.frame_sync.frame_sequence = atomic_read(&asd->sof_count);
-
-	v4l2_event_queue(asd->subdev.devnode, &event);
-}
-
-void atomisp_eof_event(struct atomisp_sub_device *asd, uint8_t exp_id)
-{
-	struct v4l2_event event = {0};
-
-	event.type = V4L2_EVENT_FRAME_END;
-	event.u.frame_sync.frame_sequence = exp_id;
-
-	v4l2_event_queue(asd->subdev.devnode, &event);
-}
-
-static void atomisp_3a_stats_ready_event(struct atomisp_sub_device *asd, uint8_t exp_id)
-{
-	struct v4l2_event event = {0};
-
-	event.type = V4L2_EVENT_ATOMISP_3A_STATS_READY;
-	event.u.frame_sync.frame_sequence = exp_id;
-
-	v4l2_event_queue(asd->subdev.devnode, &event);
-}
-
-static void atomisp_metadata_ready_event(struct atomisp_sub_device *asd,
-					 enum atomisp_metadata_type md_type)
-{
-	struct v4l2_event event = {0};
-
-	event.type = V4L2_EVENT_ATOMISP_METADATA_READY;
-	event.u.data[0] = md_type;
-
-	v4l2_event_queue(asd->subdev.devnode, &event);
-}
-
-static void atomisp_reset_event(struct atomisp_sub_device *asd)
-{
-	struct v4l2_event event = {0};
-
-	event.type = V4L2_EVENT_ATOMISP_CSS_RESET;
-
-	v4l2_event_queue(asd->subdev.devnode, &event);
-}
-
-
-static void print_csi_rx_errors(enum mipi_port_id port,
-				struct atomisp_device *isp)
-{
-	u32 infos = 0;
-
-	atomisp_css_rx_get_irq_info(port, &infos);
-
-	dev_err(isp->dev, "CSI Receiver port %d errors:\n", port);
-	if (infos & CSS_RX_IRQ_INFO_BUFFER_OVERRUN)
-		dev_err(isp->dev, "  buffer overrun");
-	if (infos & CSS_RX_IRQ_INFO_ERR_SOT)
-		dev_err(isp->dev, "  start-of-transmission error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_SOT_SYNC)
-		dev_err(isp->dev, "  start-of-transmission sync error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_CONTROL)
-		dev_err(isp->dev, "  control error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE)
-		dev_err(isp->dev, "  2 or more ECC errors");
-	if (infos & CSS_RX_IRQ_INFO_ERR_CRC)
-		dev_err(isp->dev, "  CRC mismatch");
-	if (infos & CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID)
-		dev_err(isp->dev, "  unknown error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_FRAME_SYNC)
-		dev_err(isp->dev, "  frame sync error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_FRAME_DATA)
-		dev_err(isp->dev, "  frame data error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT)
-		dev_err(isp->dev, "  data timeout");
-	if (infos & CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC)
-		dev_err(isp->dev, "  unknown escape command entry");
-	if (infos & CSS_RX_IRQ_INFO_ERR_LINE_SYNC)
-		dev_err(isp->dev, "  line sync error");
-}
-
-/* Clear irq reg */
-static void clear_irq_reg(struct atomisp_device *isp)
-{
-	u32 msg_ret;
-	pci_read_config_dword(isp->pdev, PCI_INTERRUPT_CTRL, &msg_ret);
-	msg_ret |= 1 << INTR_IIR;
-	pci_write_config_dword(isp->pdev, PCI_INTERRUPT_CTRL, msg_ret);
-}
-
-static struct atomisp_sub_device *
-__get_asd_from_port(struct atomisp_device *isp, enum mipi_port_id port)
-{
-	int i;
-
-	/* Check which isp subdev to send eof */
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-		struct camera_mipi_info *mipi_info;
-
-		mipi_info = atomisp_to_sensor_mipi_info(
-				isp->inputs[asd->input_curr].camera);
-
-		if (asd->streaming == ATOMISP_DEVICE_STREAMING_ENABLED &&
-		    __get_mipi_port(isp, mipi_info->port) == port) {
-			return asd;
-		}
-	}
-
-	return NULL;
-}
-
-/* interrupt handling function*/
-irqreturn_t atomisp_isr(int irq, void *dev)
-{
-	struct atomisp_device *isp = (struct atomisp_device *)dev;
-	struct atomisp_sub_device *asd;
-	struct atomisp_css_event eof_event;
-	unsigned int irq_infos = 0;
-	unsigned long flags;
-	unsigned int i;
-	int err;
-
-	spin_lock_irqsave(&isp->lock, flags);
-	if (isp->sw_contex.power_state != ATOM_ISP_POWER_UP ||
-	    !isp->css_initialized) {
-		spin_unlock_irqrestore(&isp->lock, flags);
-		return IRQ_HANDLED;
-	}
-	err = atomisp_css_irq_translate(isp, &irq_infos);
-	if (err) {
-		spin_unlock_irqrestore(&isp->lock, flags);
-		return IRQ_NONE;
-	}
-
-	dev_dbg(isp->dev, "irq:0x%x\n", irq_infos);
-
-	clear_irq_reg(isp);
-
-	if (!atomisp_streaming_count(isp) && !atomisp_is_acc_enabled(isp))
-		goto out_nowake;
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-
-		if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
-			continue;
-		/*
-		 * Current SOF only support one stream, so the SOF only valid
-		 * either solely one stream is running
-		 */
-		if (irq_infos & CSS_IRQ_INFO_CSS_RECEIVER_SOF) {
-			atomic_inc(&asd->sof_count);
-			atomisp_sof_event(asd);
-
-			/* If sequence_temp and sequence are the same
-			 * there where no frames lost so we can increase
-			 * sequence_temp.
-			 * If not then processing of frame is still in progress
-			 * and driver needs to keep old sequence_temp value.
-			 * NOTE: There is assumption here that ISP will not
-			 * start processing next frame from sensor before old
-			 * one is completely done. */
-			if (atomic_read(&asd->sequence) == atomic_read(
-						&asd->sequence_temp))
-				atomic_set(&asd->sequence_temp,
-						atomic_read(&asd->sof_count));
-		}
-		if (irq_infos & CSS_IRQ_INFO_EVENTS_READY)
-			atomic_set(&asd->sequence,
-				   atomic_read(&asd->sequence_temp));
-	}
-
-	if (irq_infos & CSS_IRQ_INFO_CSS_RECEIVER_SOF)
-		irq_infos &= ~CSS_IRQ_INFO_CSS_RECEIVER_SOF;
-
-	if ((irq_infos & CSS_IRQ_INFO_INPUT_SYSTEM_ERROR) ||
-	    (irq_infos & CSS_IRQ_INFO_IF_ERROR)) {
-		/* handle mipi receiver error */
-		u32 rx_infos;
-		enum mipi_port_id port;
-
-		for (port = MIPI_PORT0_ID; port <= MIPI_PORT2_ID;
-		     port++) {
-			print_csi_rx_errors(port, isp);
-			atomisp_css_rx_get_irq_info(port, &rx_infos);
-			atomisp_css_rx_clear_irq_info(port, rx_infos);
-		}
-	}
-
-	if (irq_infos & IA_CSS_IRQ_INFO_ISYS_EVENTS_READY) {
-		while (ia_css_dequeue_isys_event(&(eof_event.event)) ==
-		       IA_CSS_SUCCESS) {
-			/* EOF Event does not have the css_pipe returned */
-			asd = __get_asd_from_port(isp, eof_event.event.port);
-			if (!asd) {
-				dev_err(isp->dev, "%s:no subdev.event:%d",  __func__,
-					eof_event.event.type);
-				continue;
-			}
-
-			atomisp_eof_event(asd, eof_event.event.exp_id);
-			dev_dbg(isp->dev, "%s EOF exp_id %d, asd %d\n",
-				__func__, eof_event.event.exp_id, asd->index);
-		}
-
-		irq_infos &= ~IA_CSS_IRQ_INFO_ISYS_EVENTS_READY;
-		if (irq_infos == 0)
-			goto out_nowake;
-	}
-
-	spin_unlock_irqrestore(&isp->lock, flags);
-
-	return IRQ_WAKE_THREAD;
-
-out_nowake:
-	spin_unlock_irqrestore(&isp->lock, flags);
-
-	return IRQ_HANDLED;
-}
-
-void atomisp_clear_css_buffer_counters(struct atomisp_sub_device *asd)
-{
-	int i;
-	memset(asd->s3a_bufs_in_css, 0, sizeof(asd->s3a_bufs_in_css));
-	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++)
-		memset(asd->metadata_bufs_in_css[i], 0,
-		       sizeof(asd->metadata_bufs_in_css[i]));
-	asd->dis_bufs_in_css = 0;
-	asd->video_out_capture.buffers_in_css = 0;
-	asd->video_out_vf.buffers_in_css = 0;
-	asd->video_out_preview.buffers_in_css = 0;
-	asd->video_out_video_capture.buffers_in_css = 0;
-}
-
-#ifndef ISP2401
-bool atomisp_buffers_queued(struct atomisp_sub_device *asd)
-#else
-bool atomisp_buffers_queued_pipe(struct atomisp_video_pipe *pipe)
-#endif
-{
-#ifndef ISP2401
-	return asd->video_out_capture.buffers_in_css ||
-		asd->video_out_vf.buffers_in_css ||
-		asd->video_out_preview.buffers_in_css ||
-		asd->video_out_video_capture.buffers_in_css ?
-		    true : false;
-#else
-	return pipe->buffers_in_css ? true : false;
-#endif
-}
-
-/* 0x100000 is the start of dmem inside SP */
-#define SP_DMEM_BASE	0x100000
-
-void dump_sp_dmem(struct atomisp_device *isp, unsigned int addr,
-		  unsigned int size)
-{
-	unsigned int data = 0;
-	unsigned int size32 = DIV_ROUND_UP(size, sizeof(u32));
-
-	dev_dbg(isp->dev, "atomisp_io_base:%p\n", atomisp_io_base);
-	dev_dbg(isp->dev, "%s, addr:0x%x, size: %d, size32: %d\n", __func__,
-			addr, size, size32);
-	if (size32 * 4 + addr > 0x4000) {
-		dev_err(isp->dev, "illegal size (%d) or addr (0x%x)\n",
-				size32, addr);
-		return;
-	}
-	addr += SP_DMEM_BASE;
-	do {
-		data = _hrt_master_port_uload_32(addr);
-
-		dev_dbg(isp->dev, "%s, \t [0x%x]:0x%x\n", __func__, addr, data);
-		addr += sizeof(unsigned int);
-		size32 -= 1;
-	} while (size32 > 0);
-}
-
-static struct videobuf_buffer *atomisp_css_frame_to_vbuf(
-	struct atomisp_video_pipe *pipe, struct atomisp_css_frame *frame)
-{
-	struct videobuf_vmalloc_memory *vm_mem;
-	struct atomisp_css_frame *handle;
-	int i;
-
-	for (i = 0; pipe->capq.bufs[i]; i++) {
-		vm_mem = pipe->capq.bufs[i]->priv;
-		handle = vm_mem->vaddr;
-		if (handle && handle->data == frame->data)
-			return pipe->capq.bufs[i];
-	}
-
-	return NULL;
-}
-
-static void get_buf_timestamp(struct timeval *tv)
-{
-	struct timespec ts;
-	ktime_get_ts(&ts);
-	tv->tv_sec = ts.tv_sec;
-	tv->tv_usec = ts.tv_nsec / NSEC_PER_USEC;
-}
-
-static void atomisp_flush_video_pipe(struct atomisp_sub_device *asd,
-				     struct atomisp_video_pipe *pipe)
-{
-	unsigned long irqflags;
-	int i;
-
-	if (!pipe->users)
-		return;
-
-	for (i = 0; pipe->capq.bufs[i]; i++) {
-		spin_lock_irqsave(&pipe->irq_lock, irqflags);
-		if (pipe->capq.bufs[i]->state == VIDEOBUF_ACTIVE ||
-		    pipe->capq.bufs[i]->state == VIDEOBUF_QUEUED) {
-			get_buf_timestamp(&pipe->capq.bufs[i]->ts);
-			pipe->capq.bufs[i]->field_count =
-				atomic_read(&asd->sequence) << 1;
-			dev_dbg(asd->isp->dev, "release buffers on device %s\n",
-				pipe->vdev.name);
-			if (pipe->capq.bufs[i]->state == VIDEOBUF_QUEUED)
-				list_del_init(&pipe->capq.bufs[i]->queue);
-			pipe->capq.bufs[i]->state = VIDEOBUF_ERROR;
-			wake_up(&pipe->capq.bufs[i]->done);
-		}
-		spin_unlock_irqrestore(&pipe->irq_lock, irqflags);
-	}
-}
-
-/* Returns queued buffers back to video-core */
-void atomisp_flush_bufs_and_wakeup(struct atomisp_sub_device *asd)
-{
-	atomisp_flush_video_pipe(asd, &asd->video_out_capture);
-	atomisp_flush_video_pipe(asd, &asd->video_out_vf);
-	atomisp_flush_video_pipe(asd, &asd->video_out_preview);
-	atomisp_flush_video_pipe(asd, &asd->video_out_video_capture);
-}
-
-/* clean out the parameters that did not apply */
-void atomisp_flush_params_queue(struct atomisp_video_pipe *pipe)
-{
-	struct atomisp_css_params_with_list *param;
-
-	while (!list_empty(&pipe->per_frame_params)) {
-		param = list_entry(pipe->per_frame_params.next,
-				   struct atomisp_css_params_with_list, list);
-		list_del(&param->list);
-		atomisp_free_css_parameters(&param->params);
-		kvfree(param);
-	}
-}
-
-/* Re-queue per-frame parameters */
-static void atomisp_recover_params_queue(struct atomisp_video_pipe *pipe)
-{
-	struct atomisp_css_params_with_list *param;
-	int i;
-
-	for (i = 0; i < VIDEO_MAX_FRAME; i++) {
-		param = pipe->frame_params[i];
-		if (param)
-			list_add_tail(&param->list, &pipe->per_frame_params);
-		pipe->frame_params[i] = NULL;
-	}
-	atomisp_handle_parameter_and_buffer(pipe);
-}
-
-/* find atomisp_video_pipe with css pipe id, buffer type and atomisp run_mode */
-static struct atomisp_video_pipe *__atomisp_get_pipe(
-		struct atomisp_sub_device *asd,
-		enum atomisp_input_stream_id stream_id,
-		enum atomisp_css_pipe_id css_pipe_id,
-		enum atomisp_css_buffer_type buf_type)
-{
-	struct atomisp_device *isp = asd->isp;
-
-	if (css_pipe_id == CSS_PIPE_ID_COPY &&
-	    isp->inputs[asd->input_curr].camera_caps->
-		sensor[asd->sensor_curr].stream_num > 1) {
-		switch (stream_id) {
-		case ATOMISP_INPUT_STREAM_PREVIEW:
-			return &asd->video_out_preview;
-		case ATOMISP_INPUT_STREAM_POSTVIEW:
-			return &asd->video_out_vf;
-		case ATOMISP_INPUT_STREAM_VIDEO:
-			return &asd->video_out_video_capture;
-		case ATOMISP_INPUT_STREAM_CAPTURE:
-		default:
-			return &asd->video_out_capture;
-		}
-	}
-
-	/* video is same in online as in continuouscapture mode */
-	if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) {
-		/*
-		 * Disable vf_pp and run CSS in still capture mode. In this
-		 * mode, CSS does not cause extra latency with buffering, but
-		 * scaling is not available.
-		 */
-		return &asd->video_out_capture;
-	} else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) {
-		/*
-		 * Disable vf_pp and run CSS in video mode. This allows using
-		 * ISP scaling but it has one frame delay due to CSS internal
-		 * buffering.
-		 */
-		return &asd->video_out_video_capture;
-	} else if (css_pipe_id == CSS_PIPE_ID_YUVPP) {
-		/*
-		 * to SOC camera, yuvpp pipe is run for capture/video/SDV/ZSL.
-		 */
-		if (asd->continuous_mode->val) {
-			if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
-				/* SDV case */
-				switch (buf_type) {
-				case CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
-					return &asd->video_out_video_capture;
-				case CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME:
-					return &asd->video_out_preview;
-				case CSS_BUFFER_TYPE_OUTPUT_FRAME:
-					return &asd->video_out_capture;
-				default:
-					return &asd->video_out_vf;
-				}
-			} else if (asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) {
-				/* ZSL case */
-				switch (buf_type) {
-				case CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
-					return &asd->video_out_preview;
-				case CSS_BUFFER_TYPE_OUTPUT_FRAME:
-					return &asd->video_out_capture;
-				default:
-					return &asd->video_out_vf;
-				}
-			}
-		} else if (buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME) {
-			switch (asd->run_mode->val) {
-			case ATOMISP_RUN_MODE_VIDEO:
-				return &asd->video_out_video_capture;
-			case ATOMISP_RUN_MODE_PREVIEW:
-				return &asd->video_out_preview;
-			default:
-				return &asd->video_out_capture;
-			}
-		} else if (buf_type == CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) {
-			if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-				return &asd->video_out_preview;
-			else
-				return &asd->video_out_vf;
-		}
-	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
-		/* For online video or SDV video pipe. */
-		if (css_pipe_id == CSS_PIPE_ID_VIDEO ||
-		    css_pipe_id == CSS_PIPE_ID_COPY) {
-			if (buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME)
-				return &asd->video_out_video_capture;
-			return &asd->video_out_preview;
-		}
-	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) {
-		/* For online preview or ZSL preview pipe. */
-		if (css_pipe_id == CSS_PIPE_ID_PREVIEW ||
-		    css_pipe_id == CSS_PIPE_ID_COPY)
-			return &asd->video_out_preview;
-	}
-	/* For capture pipe. */
-	if (buf_type == CSS_BUFFER_TYPE_VF_OUTPUT_FRAME)
-		return &asd->video_out_vf;
-	return &asd->video_out_capture;
-}
-
-enum atomisp_metadata_type
-atomisp_get_metadata_type(struct atomisp_sub_device *asd,
-			  enum ia_css_pipe_id pipe_id)
-{
-	if (!asd->continuous_mode->val)
-		return ATOMISP_MAIN_METADATA;
-
-	if (pipe_id == IA_CSS_PIPE_ID_CAPTURE) /* online capture pipe */
-		return ATOMISP_SEC_METADATA;
-	else
-		return ATOMISP_MAIN_METADATA;
-}
-
-void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
-		      enum atomisp_css_buffer_type buf_type,
-		      enum atomisp_css_pipe_id css_pipe_id,
-		      bool q_buffers, enum atomisp_input_stream_id stream_id)
-{
-	struct videobuf_buffer *vb = NULL;
-	struct atomisp_video_pipe *pipe = NULL;
-	struct atomisp_css_buffer buffer;
-	bool requeue = false;
-	int err;
-	unsigned long irqflags;
-	struct atomisp_css_frame *frame = NULL;
-	struct atomisp_s3a_buf *s3a_buf = NULL, *_s3a_buf_tmp;
-	struct atomisp_dis_buf *dis_buf = NULL, *_dis_buf_tmp;
-	struct atomisp_metadata_buf *md_buf = NULL, *_md_buf_tmp;
-	enum atomisp_metadata_type md_type;
-	struct atomisp_device *isp = asd->isp;
-	struct v4l2_control ctrl;
-#ifdef ISP2401
-	bool reset_wdt_timer = false;
-#endif
-
-	if (
-	    buf_type != CSS_BUFFER_TYPE_METADATA &&
-	    buf_type != CSS_BUFFER_TYPE_3A_STATISTICS &&
-	    buf_type != CSS_BUFFER_TYPE_DIS_STATISTICS &&
-	    buf_type != CSS_BUFFER_TYPE_OUTPUT_FRAME &&
-	    buf_type != CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME &&
-	    buf_type != CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME &&
-	    buf_type != CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME &&
-	    buf_type != CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) {
-		dev_err(isp->dev, "%s, unsupported buffer type: %d\n",
-			__func__, buf_type);
-		return;
-	}
-
-	memset(&buffer, 0, sizeof(struct atomisp_css_buffer));
-	buffer.css_buffer.type = buf_type;
-	err = atomisp_css_dequeue_buffer(asd, stream_id, css_pipe_id,
-					 buf_type, &buffer);
-	if (err) {
-		dev_err(isp->dev,
-			"atomisp_css_dequeue_buffer failed: 0x%x\n", err);
-		return;
-	}
-
-	/* need to know the atomisp pipe for frame buffers */
-	pipe = __atomisp_get_pipe(asd, stream_id, css_pipe_id, buf_type);
-	if (pipe == NULL) {
-		dev_err(isp->dev, "error getting atomisp pipe\n");
-		return;
-	}
-
-	switch (buf_type) {
-	case CSS_BUFFER_TYPE_3A_STATISTICS:
-		list_for_each_entry_safe(s3a_buf, _s3a_buf_tmp,
-					 &asd->s3a_stats_in_css, list) {
-			if (s3a_buf->s3a_data ==
-				buffer.css_buffer.data.stats_3a) {
-				list_del_init(&s3a_buf->list);
-				list_add_tail(&s3a_buf->list,
-					      &asd->s3a_stats_ready);
-				break;
-			}
-		}
-
-		asd->s3a_bufs_in_css[css_pipe_id]--;
-		atomisp_3a_stats_ready_event(asd, buffer.css_buffer.exp_id);
-		dev_dbg(isp->dev, "%s: s3a stat with exp_id %d is ready\n",
-			__func__, s3a_buf->s3a_data->exp_id);
-		break;
-	case CSS_BUFFER_TYPE_METADATA:
-		if (error)
-			break;
-
-		md_type = atomisp_get_metadata_type(asd, css_pipe_id);
-		list_for_each_entry_safe(md_buf, _md_buf_tmp,
-					 &asd->metadata_in_css[md_type], list) {
-			if (md_buf->metadata ==
-				buffer.css_buffer.data.metadata) {
-				list_del_init(&md_buf->list);
-				list_add_tail(&md_buf->list,
-					      &asd->metadata_ready[md_type]);
-				break;
-			}
-		}
-		asd->metadata_bufs_in_css[stream_id][css_pipe_id]--;
-		atomisp_metadata_ready_event(asd, md_type);
-		dev_dbg(isp->dev, "%s: metadata with exp_id %d is ready\n",
-			__func__, md_buf->metadata->exp_id);
-		break;
-	case CSS_BUFFER_TYPE_DIS_STATISTICS:
-		list_for_each_entry_safe(dis_buf, _dis_buf_tmp,
-						&asd->dis_stats_in_css, list) {
-			if (dis_buf->dis_data ==
-				buffer.css_buffer.data.stats_dvs) {
-				spin_lock_irqsave(&asd->dis_stats_lock,
-						  irqflags);
-				list_del_init(&dis_buf->list);
-				list_add(&dis_buf->list, &asd->dis_stats);
-				asd->params.dis_proj_data_valid = true;
-				spin_unlock_irqrestore(&asd->dis_stats_lock,
-						       irqflags);
-				break;
-			}
-		}
-		asd->dis_bufs_in_css--;
-		dev_dbg(isp->dev, "%s: dis stat with exp_id %d is ready\n",
-			__func__, dis_buf->dis_data->exp_id);
-		break;
-	case CSS_BUFFER_TYPE_VF_OUTPUT_FRAME:
-	case CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME:
-#ifdef ISP2401
-		reset_wdt_timer = true;
-#endif
-		pipe->buffers_in_css--;
-		frame = buffer.css_buffer.data.frame;
-		if (!frame) {
-			WARN_ON(1);
-			break;
-		}
-		if (!frame->valid)
-			error = true;
-
-		/* FIXME:
-		 * YUVPP doesn't set postview exp_id correctlly in SDV mode.
-		 * This is a WORKAROUND to set exp_id. see HSDES-1503911606.
-		 */
-		if (IS_BYT && buf_type == CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME &&
-		    asd->continuous_mode->val && ATOMISP_USE_YUVPP(asd))
-			frame->exp_id = (asd->postview_exp_id++) %
-						(ATOMISP_MAX_EXP_ID + 1);
-
-		dev_dbg(isp->dev, "%s: vf frame with exp_id %d is ready\n",
-			__func__, frame->exp_id);
-		if (asd->params.flash_state == ATOMISP_FLASH_ONGOING) {
-			if (frame->flash_state
-			    == CSS_FRAME_FLASH_STATE_PARTIAL)
-				dev_dbg(isp->dev, "%s thumb partially flashed\n",
-					__func__);
-			else if (frame->flash_state
-				 == CSS_FRAME_FLASH_STATE_FULL)
-				dev_dbg(isp->dev, "%s thumb completely flashed\n",
-					__func__);
-			else
-				dev_dbg(isp->dev, "%s thumb no flash in this frame\n",
-					__func__);
-		}
-		vb = atomisp_css_frame_to_vbuf(pipe, frame);
-		WARN_ON(!vb);
-		if (vb)
-			pipe->frame_config_id[vb->i] = frame->isp_config_id;
-		if (css_pipe_id == IA_CSS_PIPE_ID_CAPTURE &&
-		    asd->pending_capture_request > 0) {
-			err = atomisp_css_offline_capture_configure(asd,
-					asd->params.offline_parm.num_captures,
-					asd->params.offline_parm.skip_frames,
-					asd->params.offline_parm.offset);
-#ifndef ISP2401
-			asd->pending_capture_request--;
-			dev_dbg(isp->dev, "Trigger capture again for new buffer. err=%d\n",
-				err);
-#else
-				asd->pending_capture_request--;
-				asd->re_trigger_capture = false;
-				dev_dbg(isp->dev, "Trigger capture again for new buffer. err=%d\n",
-						err);
-			} else {
-				asd->re_trigger_capture = true;
-			}
-#endif
-		}
-		break;
-	case CSS_BUFFER_TYPE_OUTPUT_FRAME:
-	case CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
-#ifdef ISP2401
-		reset_wdt_timer = true;
-#endif
-		pipe->buffers_in_css--;
-		frame = buffer.css_buffer.data.frame;
-		if (!frame) {
-			WARN_ON(1);
-			break;
-		}
-
-		if (!frame->valid)
-			error = true;
-
-		/* FIXME:
-		 * YUVPP doesn't set preview exp_id correctlly in ZSL mode.
-		 * This is a WORKAROUND to set exp_id. see HSDES-1503911606.
-		 */
-		if (IS_BYT && buf_type == CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME &&
-		    asd->continuous_mode->val && ATOMISP_USE_YUVPP(asd))
-			frame->exp_id = (asd->preview_exp_id++) %
-						(ATOMISP_MAX_EXP_ID + 1);
-
-		dev_dbg(isp->dev, "%s: main frame with exp_id %d is ready\n",
-			__func__, frame->exp_id);
-		vb = atomisp_css_frame_to_vbuf(pipe, frame);
-		if (!vb) {
-			WARN_ON(1);
-			break;
-		}
-
-		/* free the parameters */
-		if (pipe->frame_params[vb->i]) {
-			if (asd->params.dvs_6axis ==
-			    pipe->frame_params[vb->i]->params.dvs_6axis)
-				asd->params.dvs_6axis = NULL;
-			atomisp_free_css_parameters(
-				&pipe->frame_params[vb->i]->params);
-			kvfree(pipe->frame_params[vb->i]);
-			pipe->frame_params[vb->i] = NULL;
-		}
-
-		pipe->frame_config_id[vb->i] = frame->isp_config_id;
-		ctrl.id = V4L2_CID_FLASH_MODE;
-		if (asd->params.flash_state == ATOMISP_FLASH_ONGOING) {
-			if (frame->flash_state
-			    == CSS_FRAME_FLASH_STATE_PARTIAL) {
-				asd->frame_status[vb->i] =
-					ATOMISP_FRAME_STATUS_FLASH_PARTIAL;
-				dev_dbg(isp->dev, "%s partially flashed\n",
-					 __func__);
-			} else if (frame->flash_state
-				   == CSS_FRAME_FLASH_STATE_FULL) {
-				asd->frame_status[vb->i] =
-					ATOMISP_FRAME_STATUS_FLASH_EXPOSED;
-				asd->params.num_flash_frames--;
-				dev_dbg(isp->dev, "%s completely flashed\n",
-					 __func__);
-			} else {
-				asd->frame_status[vb->i] =
-					ATOMISP_FRAME_STATUS_OK;
-				dev_dbg(isp->dev,
-					 "%s no flash in this frame\n",
-					 __func__);
-			}
-
-			/* Check if flashing sequence is done */
-			if (asd->frame_status[vb->i] ==
-				ATOMISP_FRAME_STATUS_FLASH_EXPOSED)
-				asd->params.flash_state = ATOMISP_FLASH_DONE;
-		} else if (isp->flash) {
-			if (v4l2_g_ctrl(isp->flash->ctrl_handler, &ctrl) ==
-			    0 && ctrl.value == ATOMISP_FLASH_MODE_TORCH) {
-				ctrl.id = V4L2_CID_FLASH_TORCH_INTENSITY;
-				if (v4l2_g_ctrl(isp->flash->ctrl_handler, &ctrl)
-				    == 0 && ctrl.value > 0) {
-					asd->frame_status[vb->i] =
-					    ATOMISP_FRAME_STATUS_FLASH_EXPOSED;
-				} else {
-					asd->frame_status[vb->i] =
-					    ATOMISP_FRAME_STATUS_OK;
-				}
-			} else
-				asd->frame_status[vb->i] =
-				    ATOMISP_FRAME_STATUS_OK;
-		} else {
-			asd->frame_status[vb->i] = ATOMISP_FRAME_STATUS_OK;
-		}
-
-		asd->params.last_frame_status = asd->frame_status[vb->i];
-
-		if (asd->continuous_mode->val) {
-			if (css_pipe_id == CSS_PIPE_ID_PREVIEW ||
-			    css_pipe_id == CSS_PIPE_ID_VIDEO) {
-				asd->latest_preview_exp_id = frame->exp_id;
-			} else if (css_pipe_id ==
-					CSS_PIPE_ID_CAPTURE) {
-				if (asd->run_mode->val ==
-					ATOMISP_RUN_MODE_VIDEO)
-					dev_dbg(isp->dev, "SDV capture raw buffer id: %u\n",
-					    frame->exp_id);
-				else
-					dev_dbg(isp->dev, "ZSL capture raw buffer id: %u\n",
-					    frame->exp_id);
-			}
-		}
-		/*
-		 * Only after enabled the raw buffer lock
-		 * and in continuous mode.
-		 * in preview/video pipe, each buffer will
-		 * be locked automatically, so record it here.
-		 */
-		if (((css_pipe_id == CSS_PIPE_ID_PREVIEW) ||
-		    (css_pipe_id == CSS_PIPE_ID_VIDEO)) &&
-		    asd->enable_raw_buffer_lock->val &&
-		    asd->continuous_mode->val) {
-			atomisp_set_raw_buffer_bitmap(asd, frame->exp_id);
-			WARN_ON(frame->exp_id > ATOMISP_MAX_EXP_ID);
-		}
-
-		if (asd->params.css_update_params_needed) {
-			atomisp_apply_css_parameters(asd,
-						     &asd->params.css_param);
-			if (asd->params.css_param.update_flag.dz_config)
-				atomisp_css_set_dz_config(asd,
-					&asd->params.css_param.dz_config);
-			/* New global dvs 6axis config should be blocked
-			 * here if there's a buffer with per-frame parameters
-			 * pending in CSS frame buffer queue.
-			 * This is to aviod zooming vibration since global
-			 * parameters take effect immediately while
-			 * per-frame parameters are taken after previous
-			 * buffers in CSS got processed.
-			 */
-			if (asd->params.dvs_6axis)
-				atomisp_css_set_dvs_6axis(asd,
-					asd->params.dvs_6axis);
-			else
-				asd->params.css_update_params_needed = false;
-			/* The update flag should not be cleaned here
-			 * since it is still going to be used to make up
-			 * following per-frame parameters.
-			 * This will introduce more copy work since each
-			 * time when updating global parameters, the whole
-			 * parameter set are applied.
-			 * FIXME: A new set of parameter copy functions can
-			 * be added to make up per-frame parameters based on
-			 * solid structures stored in asd->params.css_param
-			 * instead of using shadow pointers in update flag.
-			 */
-			atomisp_css_update_isp_params(asd);
-		}
-		break;
-	default:
-		break;
-	}
-	if (vb) {
-		get_buf_timestamp(&vb->ts);
-		vb->field_count = atomic_read(&asd->sequence) << 1;
-		/*mark videobuffer done for dequeue*/
-		spin_lock_irqsave(&pipe->irq_lock, irqflags);
-		vb->state = !error ? VIDEOBUF_DONE : VIDEOBUF_ERROR;
-		spin_unlock_irqrestore(&pipe->irq_lock, irqflags);
-
-		/*
-		 * Frame capture done, wake up any process block on
-		 * current active buffer
-		 * possibly hold by videobuf_dqbuf()
-		 */
-		wake_up(&vb->done);
-	}
-#ifdef ISP2401
-	atomic_set(&pipe->wdt_count, 0);
-#endif
-	/*
-	 * Requeue should only be done for 3a and dis buffers.
-	 * Queue/dequeue order will change if driver recycles image buffers.
-	 */
-	if (requeue) {
-		err = atomisp_css_queue_buffer(asd,
-					       stream_id, css_pipe_id,
-					       buf_type, &buffer);
-		if (err)
-			dev_err(isp->dev, "%s, q to css fails: %d\n",
-					__func__, err);
-		return;
-	}
-	if (!error && q_buffers)
-		atomisp_qbuffers_to_css(asd);
-#ifdef ISP2401
-
-	/* If there are no buffers queued then
-	 * delete wdt timer. */
-	if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
-		return;
-	if (!atomisp_buffers_queued_pipe(pipe))
-		atomisp_wdt_stop_pipe(pipe, false);
-	else if (reset_wdt_timer)
-		/* SOF irq should not reset wdt timer. */
-		atomisp_wdt_refresh_pipe(pipe,
-					 ATOMISP_WDT_KEEP_CURRENT_DELAY);
-#endif
-}
-
-void atomisp_delayed_init_work(struct work_struct *work)
-{
-	struct atomisp_sub_device *asd = container_of(work,
-			struct atomisp_sub_device,
-			delayed_init_work);
-	/*
-	 * to SOC camera, use yuvpp pipe and no support continuous mode.
-	 */
-	if (!ATOMISP_USE_YUVPP(asd)) {
-		struct v4l2_event event = {0};
-
-		atomisp_css_allocate_continuous_frames(false, asd);
-		atomisp_css_update_continuous_frames(asd);
-
-		event.type = V4L2_EVENT_ATOMISP_RAW_BUFFERS_ALLOC_DONE;
-		v4l2_event_queue(asd->subdev.devnode, &event);
-	}
-
-	/* signal streamon after delayed init is done */
-	asd->delayed_init = ATOMISP_DELAYED_INIT_DONE;
-	complete(&asd->init_done);
-}
-
-static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout)
-{
-	enum atomisp_css_pipe_id css_pipe_id;
-	bool stream_restart[MAX_STREAM_NUM] = {0};
-	bool depth_mode = false;
-	int i, ret, depth_cnt = 0;
-
-	if (!isp->sw_contex.file_input)
-		atomisp_css_irq_enable(isp,
-				       CSS_IRQ_INFO_CSS_RECEIVER_SOF, false);
-
-	BUG_ON(isp->num_of_streams > MAX_STREAM_NUM);
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-		struct ia_css_pipeline *acc_pipeline;
-		struct ia_css_pipe *acc_pipe = NULL;
-
-		if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED &&
-		    !asd->stream_prepared)
-			continue;
-
-		/*
-		* AtomISP::waitStageUpdate is blocked when WDT happens.
-		* By calling acc_done() for all loaded fw_handles,
-		* HAL will be unblocked.
-		*/
-		acc_pipe = asd->stream_env[i].pipes[CSS_PIPE_ID_ACC];
-		if (acc_pipe != NULL) {
-			acc_pipeline = ia_css_pipe_get_pipeline(acc_pipe);
-			if (acc_pipeline) {
-				struct ia_css_pipeline_stage *stage;
-				for (stage = acc_pipeline->stages; stage;
-					stage = stage->next) {
-					const struct ia_css_fw_info *fw;
-					fw = stage->firmware;
-					atomisp_acc_done(asd, fw->handle);
-				}
-			}
-		}
-
-		depth_cnt++;
-
-		if (asd->delayed_init == ATOMISP_DELAYED_INIT_QUEUED)
-			cancel_work_sync(&asd->delayed_init_work);
-
-		complete(&asd->init_done);
-		asd->delayed_init = ATOMISP_DELAYED_INIT_NOT_QUEUED;
-
-		stream_restart[asd->index] = true;
-
-		asd->streaming = ATOMISP_DEVICE_STREAMING_STOPPING;
-
-		/* stream off sensor */
-		ret = v4l2_subdev_call(
-				isp->inputs[asd->input_curr].
-				camera, video, s_stream, 0);
-		if (ret)
-			dev_warn(isp->dev,
-					"can't stop streaming on sensor!\n");
-
-		atomisp_acc_unload_extensions(asd);
-
-		atomisp_clear_css_buffer_counters(asd);
-
-		css_pipe_id = atomisp_get_css_pipe_id(asd);
-		atomisp_css_stop(asd, css_pipe_id, true);
-
-		asd->streaming = ATOMISP_DEVICE_STREAMING_DISABLED;
-
-		asd->preview_exp_id = 1;
-		asd->postview_exp_id = 1;
-		/* notify HAL the CSS reset */
-		dev_dbg(isp->dev,
-			"send reset event to %s\n", asd->subdev.devnode->name);
-		atomisp_reset_event(asd);
-	}
-
-	/* clear irq */
-	disable_isp_irq(hrt_isp_css_irq_sp);
-	clear_isp_irq(hrt_isp_css_irq_sp);
-
-	/* Set the SRSE to 3 before resetting */
-	pci_write_config_dword(isp->pdev, PCI_I_CONTROL, isp->saved_regs.i_control |
-			       MRFLD_PCI_I_CONTROL_SRSE_RESET_MASK);
-
-	/* reset ISP and restore its state */
-	isp->isp_timeout = true;
-	atomisp_reset(isp);
-	isp->isp_timeout = false;
-
-	if (!isp_timeout) {
-		for (i = 0; i < isp->num_of_streams; i++) {
-			if (isp->asd[i].depth_mode->val)
-				return;
-		}
-	}
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-
-		if (!stream_restart[i])
-			continue;
-
-		if (isp->inputs[asd->input_curr].type != FILE_INPUT)
-			atomisp_css_input_set_mode(asd,
-					CSS_INPUT_MODE_SENSOR);
-
-		css_pipe_id = atomisp_get_css_pipe_id(asd);
-		if (atomisp_css_start(asd, css_pipe_id, true))
-			dev_warn(isp->dev,
-				"start SP failed, so do not set streaming to be enable!\n");
-		else
-			asd->streaming = ATOMISP_DEVICE_STREAMING_ENABLED;
-
-		atomisp_csi2_configure(asd);
-	}
-
-	if (!isp->sw_contex.file_input) {
-		atomisp_css_irq_enable(isp, CSS_IRQ_INFO_CSS_RECEIVER_SOF,
-				atomisp_css_valid_sof(isp));
-
-		if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_AUTO, true) < 0)
-			dev_dbg(isp->dev, "dfs failed!\n");
-	} else {
-		if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, true) < 0)
-			dev_dbg(isp->dev, "dfs failed!\n");
-	}
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd;
-
-		asd = &isp->asd[i];
-
-		if (!stream_restart[i])
-			continue;
-
-		if (asd->continuous_mode->val &&
-		    asd->delayed_init == ATOMISP_DELAYED_INIT_NOT_QUEUED) {
-			reinit_completion(&asd->init_done);
-			asd->delayed_init = ATOMISP_DELAYED_INIT_QUEUED;
-			queue_work(asd->delayed_init_workq,
-					&asd->delayed_init_work);
-		}
-		/*
-		 * dequeueing buffers is not needed. CSS will recycle
-		 * buffers that it has.
-		 */
-		atomisp_flush_bufs_and_wakeup(asd);
-
-		/* Requeue unprocessed per-frame parameters. */
-		atomisp_recover_params_queue(&asd->video_out_capture);
-		atomisp_recover_params_queue(&asd->video_out_preview);
-		atomisp_recover_params_queue(&asd->video_out_video_capture);
-
-		if ((asd->depth_mode->val) &&
-		    (depth_cnt == ATOMISP_DEPTH_SENSOR_STREAMON_COUNT)) {
-			depth_mode = true;
-			continue;
-		}
-
-		ret = v4l2_subdev_call(
-				isp->inputs[asd->input_curr].camera, video,
-				s_stream, 1);
-		if (ret)
-			dev_warn(isp->dev,
-				 "can't start streaming on sensor!\n");
-
-	}
-
-	if (depth_mode) {
-		if (atomisp_stream_on_master_slave_sensor(isp, true))
-			dev_warn(isp->dev,
-				 "master slave sensor stream on failed!\n");
-	}
-}
-
-void atomisp_wdt_work(struct work_struct *work)
-{
-	struct atomisp_device *isp = container_of(work, struct atomisp_device,
-						  wdt_work);
-	int i;
-#ifdef ISP2401
-	unsigned int pipe_wdt_cnt[MAX_STREAM_NUM][4] = { {0} };
-	bool css_recover = true;
-#endif
-
-	rt_mutex_lock(&isp->mutex);
-	if (!atomisp_streaming_count(isp)) {
-		atomic_set(&isp->wdt_work_queued, 0);
-		rt_mutex_unlock(&isp->mutex);
-		return;
-	}
-
-#ifndef ISP2401
-	dev_err(isp->dev, "timeout %d of %d\n",
-		atomic_read(&isp->wdt_count) + 1,
-		ATOMISP_ISP_MAX_TIMEOUT_COUNT);
-#else
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-		pipe_wdt_cnt[i][0] +=
-			atomic_read(&asd->video_out_capture.wdt_count);
-		pipe_wdt_cnt[i][1] +=
-			atomic_read(&asd->video_out_vf.wdt_count);
-		pipe_wdt_cnt[i][2] +=
-			atomic_read(&asd->video_out_preview.wdt_count);
-		pipe_wdt_cnt[i][3] +=
-			atomic_read(&asd->video_out_video_capture.wdt_count);
-		css_recover =
-			(pipe_wdt_cnt[i][0] <= ATOMISP_ISP_MAX_TIMEOUT_COUNT &&
-			 pipe_wdt_cnt[i][1] <= ATOMISP_ISP_MAX_TIMEOUT_COUNT &&
-			 pipe_wdt_cnt[i][2] <= ATOMISP_ISP_MAX_TIMEOUT_COUNT &&
-			 pipe_wdt_cnt[i][3] <= ATOMISP_ISP_MAX_TIMEOUT_COUNT)
-			 ? true : false;
-		dev_err(isp->dev, "pipe on asd%d timeout cnt: (%d, %d, %d, %d) of %d, recover = %d\n",
-			asd->index, pipe_wdt_cnt[i][0], pipe_wdt_cnt[i][1],
-			pipe_wdt_cnt[i][2], pipe_wdt_cnt[i][3],
-			ATOMISP_ISP_MAX_TIMEOUT_COUNT, css_recover);
-	}
-#endif
-
-#ifndef ISP2401
-	if (atomic_inc_return(&isp->wdt_count) <
-	    ATOMISP_ISP_MAX_TIMEOUT_COUNT) {
-#else
-	if (css_recover) {
-#endif
-		unsigned int old_dbglevel = dbg_level;
-		atomisp_css_debug_dump_sp_sw_debug_info();
-		atomisp_css_debug_dump_debug_info(__func__);
-		dbg_level = old_dbglevel;
-		for (i = 0; i < isp->num_of_streams; i++) {
-			struct atomisp_sub_device *asd = &isp->asd[i];
-
-			if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
-				continue;
-			dev_err(isp->dev, "%s, vdev %s buffers in css: %d\n",
-				__func__,
-				asd->video_out_capture.vdev.name,
-				asd->video_out_capture.
-				buffers_in_css);
-			dev_err(isp->dev,
-				"%s, vdev %s buffers in css: %d\n",
-				__func__,
-				asd->video_out_vf.vdev.name,
-				asd->video_out_vf.
-				buffers_in_css);
-			dev_err(isp->dev,
-				"%s, vdev %s buffers in css: %d\n",
-				__func__,
-				asd->video_out_preview.vdev.name,
-				asd->video_out_preview.
-				buffers_in_css);
-			dev_err(isp->dev,
-				"%s, vdev %s buffers in css: %d\n",
-				__func__,
-				asd->video_out_video_capture.vdev.name,
-				asd->video_out_video_capture.
-				buffers_in_css);
-			dev_err(isp->dev,
-				"%s, s3a buffers in css preview pipe:%d\n",
-				__func__,
-				asd->s3a_bufs_in_css[CSS_PIPE_ID_PREVIEW]);
-			dev_err(isp->dev,
-				"%s, s3a buffers in css capture pipe:%d\n",
-				__func__,
-				asd->s3a_bufs_in_css[CSS_PIPE_ID_CAPTURE]);
-			dev_err(isp->dev,
-				"%s, s3a buffers in css video pipe:%d\n",
-				__func__,
-				asd->s3a_bufs_in_css[CSS_PIPE_ID_VIDEO]);
-			dev_err(isp->dev,
-				"%s, dis buffers in css: %d\n",
-				__func__, asd->dis_bufs_in_css);
-			dev_err(isp->dev,
-				"%s, metadata buffers in css preview pipe:%d\n",
-				__func__,
-				asd->metadata_bufs_in_css
-				[ATOMISP_INPUT_STREAM_GENERAL]
-				[CSS_PIPE_ID_PREVIEW]);
-			dev_err(isp->dev,
-				"%s, metadata buffers in css capture pipe:%d\n",
-				__func__,
-				asd->metadata_bufs_in_css
-				[ATOMISP_INPUT_STREAM_GENERAL]
-				[CSS_PIPE_ID_CAPTURE]);
-			dev_err(isp->dev,
-				"%s, metadata buffers in css video pipe:%d\n",
-				__func__,
-				asd->metadata_bufs_in_css
-				[ATOMISP_INPUT_STREAM_GENERAL]
-				[CSS_PIPE_ID_VIDEO]);
-			if (asd->enable_raw_buffer_lock->val) {
-				unsigned int j;
-
-				dev_err(isp->dev, "%s, raw_buffer_locked_count %d\n",
-					__func__, asd->raw_buffer_locked_count);
-				for (j = 0; j <= ATOMISP_MAX_EXP_ID/32; j++)
-					dev_err(isp->dev, "%s, raw_buffer_bitmap[%d]: 0x%x\n",
-						__func__, j,
-						asd->raw_buffer_bitmap[j]);
-			}
-		}
-
-		/*sh_css_dump_sp_state();*/
-		/*sh_css_dump_isp_state();*/
-	} else {
-		for (i = 0; i < isp->num_of_streams; i++) {
-			struct atomisp_sub_device *asd = &isp->asd[i];
-			if (asd->streaming ==
-			    ATOMISP_DEVICE_STREAMING_ENABLED) {
-				atomisp_clear_css_buffer_counters(asd);
-				atomisp_flush_bufs_and_wakeup(asd);
-				complete(&asd->init_done);
-			}
-#ifdef ISP2401
-			atomisp_wdt_stop(asd, false);
-#endif
-		}
-
-#ifndef ISP2401
-		atomic_set(&isp->wdt_count, 0);
-#endif
-		isp->isp_fatal_error = true;
-		atomic_set(&isp->wdt_work_queued, 0);
-
-		rt_mutex_unlock(&isp->mutex);
-		return;
-	}
-
-	__atomisp_css_recover(isp, true);
-#ifdef ISP2401
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-		if (asd->streaming ==
-			ATOMISP_DEVICE_STREAMING_ENABLED) {
-			atomisp_wdt_refresh(asd,
-				isp->sw_contex.file_input ?
-				ATOMISP_ISP_FILE_TIMEOUT_DURATION :
-				ATOMISP_ISP_TIMEOUT_DURATION);
-		}
-	}
-#endif
-	dev_err(isp->dev, "timeout recovery handling done\n");
-	atomic_set(&isp->wdt_work_queued, 0);
-
-	rt_mutex_unlock(&isp->mutex);
-}
-
-void atomisp_css_flush(struct atomisp_device *isp)
-{
-	int i;
-
-	if (!atomisp_streaming_count(isp))
-		return;
-
-	/* Disable wdt */
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-		atomisp_wdt_stop(asd, true);
-	}
-
-	/* Start recover */
-	__atomisp_css_recover(isp, false);
-	/* Restore wdt */
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-
-		if (asd->streaming !=
-				ATOMISP_DEVICE_STREAMING_ENABLED)
-			continue;
-
-		atomisp_wdt_refresh(asd,
-				    isp->sw_contex.file_input ?
-				    ATOMISP_ISP_FILE_TIMEOUT_DURATION :
-				    ATOMISP_ISP_TIMEOUT_DURATION);
-	}
-	dev_dbg(isp->dev, "atomisp css flush done\n");
-}
-
-void atomisp_wdt(struct timer_list *t)
-{
-#ifndef ISP2401
-	struct atomisp_sub_device *asd = from_timer(asd, t, wdt);
-#else
-	struct atomisp_video_pipe *pipe = from_timer(pipe, t, wdt);
-	struct atomisp_sub_device *asd = pipe->asd;
-#endif
-	struct atomisp_device *isp = asd->isp;
-
-#ifdef ISP2401
-	atomic_inc(&pipe->wdt_count);
-	dev_warn(isp->dev,
-		"[WARNING]asd %d pipe %s ISP timeout %d!\n",
-			asd->index, pipe->vdev.name,
-			atomic_read(&pipe->wdt_count));
-#endif
-	if (atomic_read(&isp->wdt_work_queued)) {
-		dev_dbg(isp->dev, "ISP watchdog was put into workqueue\n");
-		return;
-	}
-	atomic_set(&isp->wdt_work_queued, 1);
-	queue_work(isp->wdt_work_queue, &isp->wdt_work);
-}
-
-#ifndef ISP2401
-void atomisp_wdt_refresh(struct atomisp_sub_device *asd, unsigned int delay)
-#else
-void atomisp_wdt_refresh_pipe(struct atomisp_video_pipe *pipe,
-				unsigned int delay)
-#endif
-{
-	unsigned long next;
-
-	if (delay != ATOMISP_WDT_KEEP_CURRENT_DELAY)
-#ifndef ISP2401
-		asd->wdt_duration = delay;
-#else
-		pipe->wdt_duration = delay;
-#endif
-
-#ifndef ISP2401
-	next = jiffies + asd->wdt_duration;
-#else
-	next = jiffies + pipe->wdt_duration;
-#endif
-
-	/* Override next if it has been pushed beyon the "next" time */
-#ifndef ISP2401
-	if (atomisp_is_wdt_running(asd) && time_after(asd->wdt_expires, next))
-		next = asd->wdt_expires;
-#else
-	if (atomisp_is_wdt_running(pipe) && time_after(pipe->wdt_expires, next))
-		next = pipe->wdt_expires;
-#endif
-
-#ifndef ISP2401
-	asd->wdt_expires = next;
-#else
-	pipe->wdt_expires = next;
-#endif
-
-#ifndef ISP2401
-	if (atomisp_is_wdt_running(asd))
-		dev_dbg(asd->isp->dev, "WDT will hit after %d ms\n",
-			((int)(next - jiffies) * 1000 / HZ));
-#else
-	if (atomisp_is_wdt_running(pipe))
-		dev_dbg(pipe->asd->isp->dev, "WDT will hit after %d ms (%s)\n",
-			((int)(next - jiffies) * 1000 / HZ), pipe->vdev.name);
-#endif
-	else
-#ifndef ISP2401
-		dev_dbg(asd->isp->dev, "WDT starts with %d ms period\n",
-			((int)(next - jiffies) * 1000 / HZ));
-#else
-		dev_dbg(pipe->asd->isp->dev, "WDT starts with %d ms period (%s)\n",
-			((int)(next - jiffies) * 1000 / HZ), pipe->vdev.name);
-#endif
-
-#ifndef ISP2401
-	mod_timer(&asd->wdt, next);
-	atomic_set(&asd->isp->wdt_count, 0);
-#else
-	mod_timer(&pipe->wdt, next);
-#endif
-}
-
-#ifndef ISP2401
-void atomisp_wdt_stop(struct atomisp_sub_device *asd, bool sync)
-#else
-void atomisp_wdt_refresh(struct atomisp_sub_device *asd, unsigned int delay)
-{
-	dev_dbg(asd->isp->dev, "WDT refresh all:\n");
-	if (atomisp_is_wdt_running(&asd->video_out_capture))
-		atomisp_wdt_refresh_pipe(&asd->video_out_capture, delay);
-	if (atomisp_is_wdt_running(&asd->video_out_preview))
-		atomisp_wdt_refresh_pipe(&asd->video_out_preview, delay);
-	if (atomisp_is_wdt_running(&asd->video_out_vf))
-		atomisp_wdt_refresh_pipe(&asd->video_out_vf, delay);
-	if (atomisp_is_wdt_running(&asd->video_out_video_capture))
-		atomisp_wdt_refresh_pipe(&asd->video_out_video_capture, delay);
-}
-
-
-void atomisp_wdt_stop_pipe(struct atomisp_video_pipe *pipe, bool sync)
-#endif
-{
-#ifndef ISP2401
-	dev_dbg(asd->isp->dev, "WDT stop\n");
-#else
-	if (!atomisp_is_wdt_running(pipe))
-		return;
-
-	dev_dbg(pipe->asd->isp->dev,
-		"WDT stop asd %d (%s)\n", pipe->asd->index, pipe->vdev.name);
-
-#endif
-	if (sync) {
-#ifndef ISP2401
-		del_timer_sync(&asd->wdt);
-		cancel_work_sync(&asd->isp->wdt_work);
-#else
-		del_timer_sync(&pipe->wdt);
-		cancel_work_sync(&pipe->asd->isp->wdt_work);
-#endif
-	} else {
-#ifndef ISP2401
-		del_timer(&asd->wdt);
-#else
-		del_timer(&pipe->wdt);
-#endif
-	}
-}
-
-#ifndef ISP2401
-void atomisp_wdt_start(struct atomisp_sub_device *asd)
-#else
-void atomisp_wdt_stop(struct atomisp_sub_device *asd, bool sync)
-{
-	dev_dbg(asd->isp->dev, "WDT stop all:\n");
-	atomisp_wdt_stop_pipe(&asd->video_out_capture, sync);
-	atomisp_wdt_stop_pipe(&asd->video_out_preview, sync);
-	atomisp_wdt_stop_pipe(&asd->video_out_vf, sync);
-	atomisp_wdt_stop_pipe(&asd->video_out_video_capture, sync);
-}
-
-void atomisp_wdt_start(struct atomisp_video_pipe *pipe)
-#endif
-{
-#ifndef ISP2401
-	atomisp_wdt_refresh(asd, ATOMISP_ISP_TIMEOUT_DURATION);
-#else
-	atomisp_wdt_refresh_pipe(pipe, ATOMISP_ISP_TIMEOUT_DURATION);
-#endif
-}
-
-void atomisp_setup_flash(struct atomisp_sub_device *asd)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct v4l2_control ctrl;
-
-	if (isp->flash == NULL)
-		return;
-
-	if (asd->params.flash_state != ATOMISP_FLASH_REQUESTED &&
-	    asd->params.flash_state != ATOMISP_FLASH_DONE)
-		return;
-
-	if (asd->params.num_flash_frames) {
-		/* make sure the timeout is set before setting flash mode */
-		ctrl.id = V4L2_CID_FLASH_TIMEOUT;
-		ctrl.value = FLASH_TIMEOUT;
-
-		if (v4l2_s_ctrl(NULL, isp->flash->ctrl_handler, &ctrl)) {
-			dev_err(isp->dev, "flash timeout configure failed\n");
-			return;
-		}
-
-		atomisp_css_request_flash(asd);
-		asd->params.flash_state = ATOMISP_FLASH_ONGOING;
-	} else {
-		asd->params.flash_state = ATOMISP_FLASH_IDLE;
-	}
-}
-
-irqreturn_t atomisp_isr_thread(int irq, void *isp_ptr)
-{
-	struct atomisp_device *isp = isp_ptr;
-	unsigned long flags;
-	bool frame_done_found[MAX_STREAM_NUM] = {0};
-	bool css_pipe_done[MAX_STREAM_NUM] = {0};
-	unsigned int i;
-	struct atomisp_sub_device *asd;
-
-	dev_dbg(isp->dev, ">%s\n", __func__);
-
-	spin_lock_irqsave(&isp->lock, flags);
-
-	if (!atomisp_streaming_count(isp) && !atomisp_is_acc_enabled(isp)) {
-		spin_unlock_irqrestore(&isp->lock, flags);
-		return IRQ_HANDLED;
-	}
-
-	spin_unlock_irqrestore(&isp->lock, flags);
-
-	/*
-	 * The standard CSS2.0 API tells the following calling sequence of
-	 * dequeue ready buffers:
-	 * while (ia_css_dequeue_event(...)) {
-	 *	switch (event.type) {
-	 *	...
-	 *	ia_css_pipe_dequeue_buffer()
-	 *	}
-	 * }
-	 * That is, dequeue event and buffer are one after another.
-	 *
-	 * But the following implementation is to first deuque all the event
-	 * to a FIFO, then process the event in the FIFO.
-	 * This will not have issue in single stream mode, but it do have some
-	 * issue in multiple stream case. The issue is that
-	 * ia_css_pipe_dequeue_buffer() will not return the corrent buffer in
-	 * a specific pipe.
-	 *
-	 * This is due to ia_css_pipe_dequeue_buffer() does not take the
-	 * ia_css_pipe parameter.
-	 *
-	 * So:
-	 * For CSS2.0: we change the way to not dequeue all the event at one
-	 * time, instead, dequue one and process one, then another
-	 */
-	rt_mutex_lock(&isp->mutex);
-	if (atomisp_css_isr_thread(isp, frame_done_found, css_pipe_done))
-		goto out;
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-		if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
-			continue;
-		atomisp_setup_flash(asd);
-
-	}
-out:
-	rt_mutex_unlock(&isp->mutex);
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-		if (asd->streaming == ATOMISP_DEVICE_STREAMING_ENABLED
-		    && css_pipe_done[asd->index]
-		    && isp->sw_contex.file_input)
-			v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-					 video, s_stream, 1);
-		/* FIXME! FIX ACC implementation */
-		if (asd->acc.pipeline && css_pipe_done[asd->index])
-			atomisp_css_acc_done(asd);
-	}
-	dev_dbg(isp->dev, "<%s\n", __func__);
-
-	return IRQ_HANDLED;
-}
-
-/*
- * utils for buffer allocation/free
- */
-
-int atomisp_get_frame_pgnr(struct atomisp_device *isp,
-			   const struct atomisp_css_frame *frame, u32 *p_pgnr)
-{
-	if (!frame) {
-		dev_err(isp->dev, "%s: NULL frame pointer ERROR.\n", __func__);
-		return -EINVAL;
-	}
-
-	*p_pgnr = DIV_ROUND_UP(frame->data_bytes, PAGE_SIZE);
-	return 0;
-}
-
-/*
- * Get internal fmt according to V4L2 fmt
- */
-static enum atomisp_css_frame_format
-v4l2_fmt_to_sh_fmt(u32 fmt)
-{
-	switch (fmt) {
-	case V4L2_PIX_FMT_YUV420:
-		return CSS_FRAME_FORMAT_YUV420;
-	case V4L2_PIX_FMT_YVU420:
-		return CSS_FRAME_FORMAT_YV12;
-	case V4L2_PIX_FMT_YUV422P:
-		return CSS_FRAME_FORMAT_YUV422;
-	case V4L2_PIX_FMT_YUV444:
-		return CSS_FRAME_FORMAT_YUV444;
-	case V4L2_PIX_FMT_NV12:
-		return CSS_FRAME_FORMAT_NV12;
-	case V4L2_PIX_FMT_NV21:
-		return CSS_FRAME_FORMAT_NV21;
-	case V4L2_PIX_FMT_NV16:
-		return CSS_FRAME_FORMAT_NV16;
-	case V4L2_PIX_FMT_NV61:
-		return CSS_FRAME_FORMAT_NV61;
-	case V4L2_PIX_FMT_UYVY:
-		return CSS_FRAME_FORMAT_UYVY;
-	case V4L2_PIX_FMT_YUYV:
-		return CSS_FRAME_FORMAT_YUYV;
-	case V4L2_PIX_FMT_RGB24:
-		return CSS_FRAME_FORMAT_PLANAR_RGB888;
-	case V4L2_PIX_FMT_RGB32:
-		return CSS_FRAME_FORMAT_RGBA888;
-	case V4L2_PIX_FMT_RGB565:
-		return CSS_FRAME_FORMAT_RGB565;
-	case V4L2_PIX_FMT_JPEG:
-	case V4L2_PIX_FMT_CUSTOM_M10MO_RAW:
-		return CSS_FRAME_FORMAT_BINARY_8;
-	case V4L2_PIX_FMT_SBGGR16:
-	case V4L2_PIX_FMT_SBGGR10:
-	case V4L2_PIX_FMT_SGBRG10:
-	case V4L2_PIX_FMT_SGRBG10:
-	case V4L2_PIX_FMT_SRGGB10:
-	case V4L2_PIX_FMT_SBGGR12:
-	case V4L2_PIX_FMT_SGBRG12:
-	case V4L2_PIX_FMT_SGRBG12:
-	case V4L2_PIX_FMT_SRGGB12:
-	case V4L2_PIX_FMT_SBGGR8:
-	case V4L2_PIX_FMT_SGBRG8:
-	case V4L2_PIX_FMT_SGRBG8:
-	case V4L2_PIX_FMT_SRGGB8:
-		return CSS_FRAME_FORMAT_RAW;
-	default:
-		return -EINVAL;
-	}
-}
-/*
- * raw format match between SH format and V4L2 format
- */
-static int raw_output_format_match_input(u32 input, u32 output)
-{
-	if ((input == CSS_FORMAT_RAW_12) &&
-	    ((output == V4L2_PIX_FMT_SRGGB12) ||
-	     (output == V4L2_PIX_FMT_SGRBG12) ||
-	     (output == V4L2_PIX_FMT_SBGGR12) ||
-	     (output == V4L2_PIX_FMT_SGBRG12)))
-		return 0;
-
-	if ((input == CSS_FORMAT_RAW_10) &&
-	    ((output == V4L2_PIX_FMT_SRGGB10) ||
-	     (output == V4L2_PIX_FMT_SGRBG10) ||
-	     (output == V4L2_PIX_FMT_SBGGR10) ||
-	     (output == V4L2_PIX_FMT_SGBRG10)))
-		return 0;
-
-	if ((input == CSS_FORMAT_RAW_8) &&
-	    ((output == V4L2_PIX_FMT_SRGGB8) ||
-	     (output == V4L2_PIX_FMT_SGRBG8) ||
-	     (output == V4L2_PIX_FMT_SBGGR8) ||
-	     (output == V4L2_PIX_FMT_SGBRG8)))
-		return 0;
-
-	if ((input == CSS_FORMAT_RAW_16) && (output == V4L2_PIX_FMT_SBGGR16))
-		return 0;
-
-	return -EINVAL;
-}
-
-static u32 get_pixel_depth(u32 pixelformat)
-{
-	switch (pixelformat) {
-	case V4L2_PIX_FMT_YUV420:
-	case V4L2_PIX_FMT_NV12:
-	case V4L2_PIX_FMT_NV21:
-	case V4L2_PIX_FMT_YVU420:
-		return 12;
-	case V4L2_PIX_FMT_YUV422P:
-	case V4L2_PIX_FMT_YUYV:
-	case V4L2_PIX_FMT_UYVY:
-	case V4L2_PIX_FMT_NV16:
-	case V4L2_PIX_FMT_NV61:
-	case V4L2_PIX_FMT_RGB565:
-	case V4L2_PIX_FMT_SBGGR16:
-	case V4L2_PIX_FMT_SBGGR12:
-	case V4L2_PIX_FMT_SGBRG12:
-	case V4L2_PIX_FMT_SGRBG12:
-	case V4L2_PIX_FMT_SRGGB12:
-	case V4L2_PIX_FMT_SBGGR10:
-	case V4L2_PIX_FMT_SGBRG10:
-	case V4L2_PIX_FMT_SGRBG10:
-	case V4L2_PIX_FMT_SRGGB10:
-		return 16;
-	case V4L2_PIX_FMT_RGB24:
-	case V4L2_PIX_FMT_YUV444:
-		return 24;
-	case V4L2_PIX_FMT_RGB32:
-		return 32;
-	case V4L2_PIX_FMT_JPEG:
-	case V4L2_PIX_FMT_CUSTOM_M10MO_RAW:
-	case V4L2_PIX_FMT_SBGGR8:
-	case V4L2_PIX_FMT_SGBRG8:
-	case V4L2_PIX_FMT_SGRBG8:
-	case V4L2_PIX_FMT_SRGGB8:
-		return 8;
-	default:
-		return 8 * 2;	/* raw type now */
-	}
-}
-
-bool atomisp_is_mbuscode_raw(uint32_t code)
-{
-	return code >= 0x3000 && code < 0x4000;
-}
-
-/*
- * ISP features control function
- */
-
-/*
- * Set ISP capture mode based on current settings
- */
-static void atomisp_update_capture_mode(struct atomisp_sub_device *asd)
-{
-	if (asd->params.gdc_cac_en)
-		atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_ADVANCED);
-	else if (asd->params.low_light)
-		atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_LOW_LIGHT);
-	else if (asd->video_out_capture.sh_fmt == CSS_FRAME_FORMAT_RAW)
-		atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_RAW);
-	else
-		atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_PRIMARY);
-}
-
-#ifdef ISP2401
-int atomisp_set_sensor_runmode(struct atomisp_sub_device *asd,
-		struct atomisp_s_runmode *runmode)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct v4l2_ctrl *c;
-	struct v4l2_streamparm p = {0};
-	int ret = 0;
-	int modes[] = { CI_MODE_NONE,
-			CI_MODE_VIDEO,
-			CI_MODE_STILL_CAPTURE,
-			CI_MODE_CONTINUOUS,
-			CI_MODE_PREVIEW };
-
-	if (!(runmode && (runmode->mode & RUNMODE_MASK)))
-		return -EINVAL;
-
-	mutex_lock(asd->ctrl_handler.lock);
-	c = v4l2_ctrl_find(isp->inputs[asd->input_curr].camera->ctrl_handler,
-			   V4L2_CID_RUN_MODE);
-
-	if (c)
-		ret = v4l2_ctrl_s_ctrl(c, runmode->mode);
-
-	mutex_unlock(asd->ctrl_handler.lock);
-	return ret;
-}
-
-#endif
-/*
- * Function to enable/disable lens geometry distortion correction (GDC) and
- * chromatic aberration correction (CAC)
- */
-int atomisp_gdc_cac(struct atomisp_sub_device *asd, int flag,
-		    __s32 *value)
-{
-	if (flag == 0) {
-		*value = asd->params.gdc_cac_en;
-		return 0;
-	}
-
-	asd->params.gdc_cac_en = !!*value;
-	if (asd->params.gdc_cac_en) {
-		atomisp_css_set_morph_table(asd,
-					    asd->params.css_param.morph_table);
-	} else {
-		atomisp_css_set_morph_table(asd, NULL);
-	}
-	asd->params.css_update_params_needed = true;
-	atomisp_update_capture_mode(asd);
-	return 0;
-}
-
-/*
- * Function to enable/disable low light mode including ANR
- */
-int atomisp_low_light(struct atomisp_sub_device *asd, int flag,
-		      __s32 *value)
-{
-	if (flag == 0) {
-		*value = asd->params.low_light;
-		return 0;
-	}
-
-	asd->params.low_light = (*value != 0);
-	atomisp_update_capture_mode(asd);
-	return 0;
-}
-
-/*
- * Function to enable/disable extra noise reduction (XNR) in low light
- * condition
- */
-int atomisp_xnr(struct atomisp_sub_device *asd, int flag,
-		int *xnr_enable)
-{
-	if (flag == 0) {
-		*xnr_enable = asd->params.xnr_en;
-		return 0;
-	}
-
-	atomisp_css_capture_enable_xnr(asd, !!*xnr_enable);
-
-	return 0;
-}
-
-/*
- * Function to configure bayer noise reduction
- */
-int atomisp_nr(struct atomisp_sub_device *asd, int flag,
-	       struct atomisp_nr_config *arg)
-{
-	if (flag == 0) {
-		/* Get nr config from current setup */
-		if (atomisp_css_get_nr_config(asd, arg))
-			return -EINVAL;
-	} else {
-		/* Set nr config to isp parameters */
-		memcpy(&asd->params.css_param.nr_config, arg,
-		       sizeof(struct atomisp_css_nr_config));
-		atomisp_css_set_nr_config(asd, &asd->params.css_param.nr_config);
-		asd->params.css_update_params_needed = true;
-	}
-	return 0;
-}
-
-/*
- * Function to configure temporal noise reduction (TNR)
- */
-int atomisp_tnr(struct atomisp_sub_device *asd, int flag,
-		struct atomisp_tnr_config *config)
-{
-	/* Get tnr config from current setup */
-	if (flag == 0) {
-		/* Get tnr config from current setup */
-		if (atomisp_css_get_tnr_config(asd, config))
-			return -EINVAL;
-	} else {
-		/* Set tnr config to isp parameters */
-		memcpy(&asd->params.css_param.tnr_config, config,
-			sizeof(struct atomisp_css_tnr_config));
-		atomisp_css_set_tnr_config(asd, &asd->params.css_param.tnr_config);
-		asd->params.css_update_params_needed = true;
-	}
-
-	return 0;
-}
-
-/*
- * Function to configure black level compensation
- */
-int atomisp_black_level(struct atomisp_sub_device *asd, int flag,
-			struct atomisp_ob_config *config)
-{
-	if (flag == 0) {
-		/* Get ob config from current setup */
-		if (atomisp_css_get_ob_config(asd, config))
-			return -EINVAL;
-	} else {
-		/* Set ob config to isp parameters */
-		memcpy(&asd->params.css_param.ob_config, config,
-		       sizeof(struct atomisp_css_ob_config));
-		atomisp_css_set_ob_config(asd, &asd->params.css_param.ob_config);
-		asd->params.css_update_params_needed = true;
-	}
-
-	return 0;
-}
-
-/*
- * Function to configure edge enhancement
- */
-int atomisp_ee(struct atomisp_sub_device *asd, int flag,
-	       struct atomisp_ee_config *config)
-{
-	if (flag == 0) {
-		/* Get ee config from current setup */
-		if (atomisp_css_get_ee_config(asd, config))
-			return -EINVAL;
-	} else {
-		/* Set ee config to isp parameters */
-		memcpy(&asd->params.css_param.ee_config, config,
-		       sizeof(asd->params.css_param.ee_config));
-		atomisp_css_set_ee_config(asd, &asd->params.css_param.ee_config);
-		asd->params.css_update_params_needed = true;
-	}
-
-	return 0;
-}
-
-/*
- * Function to update Gamma table for gamma, brightness and contrast config
- */
-int atomisp_gamma(struct atomisp_sub_device *asd, int flag,
-		  struct atomisp_gamma_table *config)
-{
-	if (flag == 0) {
-		/* Get gamma table from current setup */
-		if (atomisp_css_get_gamma_table(asd, config))
-			return -EINVAL;
-	} else {
-		/* Set gamma table to isp parameters */
-		memcpy(&asd->params.css_param.gamma_table, config,
-		       sizeof(asd->params.css_param.gamma_table));
-		atomisp_css_set_gamma_table(asd, &asd->params.css_param.gamma_table);
-	}
-
-	return 0;
-}
-
-/*
- * Function to update Ctc table for Chroma Enhancement
- */
-int atomisp_ctc(struct atomisp_sub_device *asd, int flag,
-		struct atomisp_ctc_table *config)
-{
-	if (flag == 0) {
-		/* Get ctc table from current setup */
-		if (atomisp_css_get_ctc_table(asd, config))
-			return -EINVAL;
-	} else {
-		/* Set ctc table to isp parameters */
-		memcpy(&asd->params.css_param.ctc_table, config,
-		       sizeof(asd->params.css_param.ctc_table));
-		atomisp_css_set_ctc_table(asd, &asd->params.css_param.ctc_table);
-	}
-
-	return 0;
-}
-
-/*
- * Function to update gamma correction parameters
- */
-int atomisp_gamma_correction(struct atomisp_sub_device *asd, int flag,
-	struct atomisp_gc_config *config)
-{
-	if (flag == 0) {
-		/* Get gamma correction params from current setup */
-		if (atomisp_css_get_gc_config(asd, config))
-			return -EINVAL;
-	} else {
-		/* Set gamma correction params to isp parameters */
-		memcpy(&asd->params.css_param.gc_config, config,
-		       sizeof(asd->params.css_param.gc_config));
-		atomisp_css_set_gc_config(asd, &asd->params.css_param.gc_config);
-		asd->params.css_update_params_needed = true;
-	}
-
-	return 0;
-}
-
-/*
- * Function to update narrow gamma flag
- */
-int atomisp_formats(struct atomisp_sub_device *asd, int flag,
-		    struct atomisp_formats_config *config)
-{
-	if (flag == 0) {
-		/* Get narrow gamma flag from current setup */
-		if (atomisp_css_get_formats_config(asd, config))
-			return -EINVAL;
-	} else {
-		/* Set narrow gamma flag to isp parameters */
-		memcpy(&asd->params.css_param.formats_config, config,
-		       sizeof(asd->params.css_param.formats_config));
-		atomisp_css_set_formats_config(asd, &asd->params.css_param.formats_config);
-	}
-
-	return 0;
-}
-
-void atomisp_free_internal_buffers(struct atomisp_sub_device *asd)
-{
-	atomisp_free_css_parameters(&asd->params.css_param);
-
-	if (asd->raw_output_frame) {
-		atomisp_css_frame_free(asd->raw_output_frame);
-		asd->raw_output_frame = NULL;
-	}
-}
-
-static void atomisp_update_grid_info(struct atomisp_sub_device *asd,
-				     enum atomisp_css_pipe_id pipe_id,
-				     int source_pad)
-{
-	struct atomisp_device *isp = asd->isp;
-	int err;
-	uint16_t stream_id = atomisp_source_pad_to_stream_id(asd, source_pad);
-
-	if (atomisp_css_get_grid_info(asd, pipe_id, source_pad))
-		return;
-
-	/* We must free all buffers because they no longer match
-	   the grid size. */
-	atomisp_css_free_stat_buffers(asd);
-
-	err = atomisp_alloc_css_stat_bufs(asd, stream_id);
-	if (err) {
-		dev_err(isp->dev, "stat_buf allocate error\n");
-		goto err;
-	}
-
-	if (atomisp_alloc_3a_output_buf(asd)) {
-		/* Failure for 3A buffers does not influence DIS buffers */
-		if (asd->params.s3a_output_bytes != 0) {
-			/* For SOC sensor happens s3a_output_bytes == 0,
-			 * using if condition to exclude false error log */
-			dev_err(isp->dev, "Failed to allocate memory for 3A statistics\n");
-		}
-		goto err;
-	}
-
-	if (atomisp_alloc_dis_coef_buf(asd)) {
-		dev_err(isp->dev,
-			"Failed to allocate memory for DIS statistics\n");
-		goto err;
-	}
-
-	if (atomisp_alloc_metadata_output_buf(asd)) {
-		dev_err(isp->dev, "Failed to allocate memory for metadata\n");
-		goto err;
-	}
-
-	return;
-
-err:
-	atomisp_css_free_stat_buffers(asd);
-	return;
-}
-
-static void atomisp_curr_user_grid_info(struct atomisp_sub_device *asd,
-					struct atomisp_grid_info *info)
-{
-	memcpy(info, &asd->params.curr_grid_info.s3a_grid,
-	       sizeof(struct atomisp_css_3a_grid_info));
-}
-
-int atomisp_compare_grid(struct atomisp_sub_device *asd,
-				struct atomisp_grid_info *atomgrid)
-{
-	struct atomisp_grid_info tmp = {0};
-
-	atomisp_curr_user_grid_info(asd, &tmp);
-	return memcmp(atomgrid, &tmp, sizeof(tmp));
-}
-
-/*
- * Function to update Gdc table for gdc
- */
-int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag,
-			  struct atomisp_morph_table *config)
-{
-	int ret;
-	int i;
-	struct atomisp_device *isp = asd->isp;
-
-	if (flag == 0) {
-		/* Get gdc table from current setup */
-		struct atomisp_css_morph_table tab = {0};
-		atomisp_css_get_morph_table(asd, &tab);
-
-		config->width = tab.width;
-		config->height = tab.height;
-
-		for (i = 0; i < CSS_MORPH_TABLE_NUM_PLANES; i++) {
-			ret = copy_to_user(config->coordinates_x[i],
-				tab.coordinates_x[i], tab.height *
-				tab.width * sizeof(*tab.coordinates_x[i]));
-			if (ret) {
-				dev_err(isp->dev,
-					"Failed to copy to User for x\n");
-				return -EFAULT;
-			}
-			ret = copy_to_user(config->coordinates_y[i],
-				tab.coordinates_y[i], tab.height *
-				tab.width * sizeof(*tab.coordinates_y[i]));
-			if (ret) {
-				dev_err(isp->dev,
-					"Failed to copy to User for y\n");
-				return -EFAULT;
-			}
-		}
-	} else {
-		struct atomisp_css_morph_table *tab =
-			asd->params.css_param.morph_table;
-
-		/* free first if we have one */
-		if (tab) {
-			atomisp_css_morph_table_free(tab);
-			asd->params.css_param.morph_table = NULL;
-		}
-
-		/* allocate new one */
-		tab = atomisp_css_morph_table_allocate(config->width,
-						       config->height);
-
-		if (!tab) {
-			dev_err(isp->dev, "out of memory\n");
-			return -EINVAL;
-		}
-
-		for (i = 0; i < CSS_MORPH_TABLE_NUM_PLANES; i++) {
-			ret = copy_from_user(tab->coordinates_x[i],
-				config->coordinates_x[i],
-				config->height * config->width *
-				sizeof(*config->coordinates_x[i]));
-			if (ret) {
-				dev_err(isp->dev,
-				"Failed to copy from User for x, ret %d\n",
-				ret);
-				atomisp_css_morph_table_free(tab);
-				return -EFAULT;
-			}
-			ret = copy_from_user(tab->coordinates_y[i],
-				config->coordinates_y[i],
-				config->height * config->width *
-				sizeof(*config->coordinates_y[i]));
-			if (ret) {
-				dev_err(isp->dev,
-				"Failed to copy from User for y, ret is %d\n",
-				ret);
-				atomisp_css_morph_table_free(tab);
-				return -EFAULT;
-			}
-		}
-		asd->params.css_param.morph_table = tab;
-		if (asd->params.gdc_cac_en)
-			atomisp_css_set_morph_table(asd, tab);
-	}
-
-	return 0;
-}
-
-int atomisp_macc_table(struct atomisp_sub_device *asd, int flag,
-		       struct atomisp_macc_config *config)
-{
-	struct atomisp_css_macc_table *macc_table;
-
-	switch (config->color_effect) {
-	case V4L2_COLORFX_NONE:
-		macc_table = &asd->params.css_param.macc_table;
-		break;
-	case V4L2_COLORFX_SKY_BLUE:
-		macc_table = &blue_macc_table;
-		break;
-	case V4L2_COLORFX_GRASS_GREEN:
-		macc_table = &green_macc_table;
-		break;
-	case V4L2_COLORFX_SKIN_WHITEN_LOW:
-		macc_table = &skin_low_macc_table;
-		break;
-	case V4L2_COLORFX_SKIN_WHITEN:
-		macc_table = &skin_medium_macc_table;
-		break;
-	case V4L2_COLORFX_SKIN_WHITEN_HIGH:
-		macc_table = &skin_high_macc_table;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	if (flag == 0) {
-		/* Get macc table from current setup */
-		memcpy(&config->table, macc_table,
-		       sizeof(struct atomisp_css_macc_table));
-	} else {
-		memcpy(macc_table, &config->table,
-		       sizeof(struct atomisp_css_macc_table));
-		if (config->color_effect == asd->params.color_effect)
-			atomisp_css_set_macc_table(asd, macc_table);
-	}
-
-	return 0;
-}
-
-int atomisp_set_dis_vector(struct atomisp_sub_device *asd,
-			   struct atomisp_dis_vector *vector)
-{
-	atomisp_css_video_set_dis_vector(asd, vector);
-
-	asd->params.dis_proj_data_valid = false;
-	asd->params.css_update_params_needed = true;
-	return 0;
-}
-
-/*
- * Function to set/get image stablization statistics
- */
-int atomisp_get_dis_stat(struct atomisp_sub_device *asd,
-			 struct atomisp_dis_statistics *stats)
-{
-	return atomisp_css_get_dis_stat(asd, stats);
-}
-
-/*
- * Function  set camrea_prefiles.xml current sensor pixel array size
- */
-int atomisp_set_array_res(struct atomisp_sub_device *asd,
-			 struct atomisp_resolution  *config)
-{
-	dev_dbg(asd->isp->dev, ">%s start\n", __func__);
-	if (!config) {
-		dev_err(asd->isp->dev, "Set sensor array size is not valid\n");
-		return -EINVAL;
-	}
-
-	asd->sensor_array_res.width = config->width;
-	asd->sensor_array_res.height = config->height;
-	return 0;
-}
-
-/*
- * Function to get DVS2 BQ resolution settings
- */
-int atomisp_get_dvs2_bq_resolutions(struct atomisp_sub_device *asd,
-			 struct atomisp_dvs2_bq_resolutions *bq_res)
-{
-	struct ia_css_pipe_config *pipe_cfg = NULL;
-	struct ia_css_stream_config *stream_cfg = NULL;
-	struct ia_css_stream_input_config *input_config = NULL;
-
-	struct ia_css_stream *stream =
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream;
-	if (!stream) {
-		dev_warn(asd->isp->dev, "stream is not created");
-		return -EAGAIN;
-	}
-
-	pipe_cfg = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.pipe_configs[CSS_PIPE_ID_VIDEO];
-	stream_cfg = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.stream_config;
-	input_config = &stream_cfg->input_config;
-
-	if (!bq_res)
-		return -EINVAL;
-
-	/* the GDC output resolution */
-	bq_res->output_bq.width_bq = pipe_cfg->output_info[0].res.width / 2;
-	bq_res->output_bq.height_bq = pipe_cfg->output_info[0].res.height / 2;
-
-	bq_res->envelope_bq.width_bq = 0;
-	bq_res->envelope_bq.height_bq = 0;
-	/* the GDC input resolution */
-	if (!asd->continuous_mode->val) {
-		bq_res->source_bq.width_bq = bq_res->output_bq.width_bq +
-				pipe_cfg->dvs_envelope.width / 2;
-		bq_res->source_bq.height_bq = bq_res->output_bq.height_bq +
-				pipe_cfg->dvs_envelope.height / 2;
-		/*
-		 * Bad pixels caused by spatial filter processing
-		 * ISP filter resolution should be given by CSS/FW, but for now
-		 * there is not such API to query, and it is fixed value, so
-		 * hardcoded here.
-		 */
-		bq_res->ispfilter_bq.width_bq = 12 / 2;
-		bq_res->ispfilter_bq.height_bq = 12 / 2;
-		/* spatial filter shift, always 4 pixels */
-		bq_res->gdc_shift_bq.width_bq = 4 / 2;
-		bq_res->gdc_shift_bq.height_bq = 4 / 2;
-
-		if (asd->params.video_dis_en) {
-			bq_res->envelope_bq.width_bq = pipe_cfg->dvs_envelope.width
-					/ 2 - bq_res->ispfilter_bq.width_bq;
-			bq_res->envelope_bq.height_bq = pipe_cfg->dvs_envelope.height
-					/ 2 - bq_res->ispfilter_bq.height_bq;
-		}
-	} else {
-		unsigned int w_padding;
-		unsigned int gdc_effective_input = 0;
-
-		/* For GDC:
-		 * gdc_effective_input = effective_input + envelope
-		 *
-		 * From the comment and formula in BZ1786,
-		 * we see the source_bq should be:
-		 * effective_input / bayer_ds_ratio
-		 */
-		bq_res->source_bq.width_bq =
-			(input_config->effective_res.width *
-			 pipe_cfg->bayer_ds_out_res.width /
-			 input_config->effective_res.width + 1) / 2;
-		bq_res->source_bq.height_bq =
-			(input_config->effective_res.height *
-			 pipe_cfg->bayer_ds_out_res.height /
-			 input_config->effective_res.height + 1) / 2;
-
-
-		if (!asd->params.video_dis_en) {
-			/*
-			 * We adjust the ispfilter_bq to:
-			 * ispfilter_bq = 128/BDS
-			 * we still need firmware team to provide an offical
-			 * formula for SDV.
-			 */
-			bq_res->ispfilter_bq.width_bq = 128 *
-				pipe_cfg->bayer_ds_out_res.width /
-				input_config->effective_res.width / 2;
-			bq_res->ispfilter_bq.height_bq = 128 *
-				pipe_cfg->bayer_ds_out_res.width /
-				input_config->effective_res.width / 2;
-
-			if (IS_HWREVISION(asd->isp, ATOMISP_HW_REVISION_ISP2401)) {
-				/* No additional left padding for ISYS2401 */
-				bq_res->gdc_shift_bq.width_bq = 4 / 2;
-				bq_res->gdc_shift_bq.height_bq = 4 / 2;
-			} else {
-				/*
-				 * For the w_padding and gdc_shift_bq cacluation
-				 * Please see the BZ 1786 and 4358 for more info.
-				 * Just test that this formula can work now,
-				 * but we still have no offical formula.
-				 *
-				 * w_padding = ceiling(gdc_effective_input
-				 *             /128, 1) * 128 - effective_width
-				 * gdc_shift_bq = w_padding/BDS/2 + ispfilter_bq/2
-				 */
-				gdc_effective_input =
-					input_config->effective_res.width +
-					pipe_cfg->dvs_envelope.width;
-				w_padding = roundup(gdc_effective_input, 128) -
-					input_config->effective_res.width;
-				w_padding = w_padding *
-					pipe_cfg->bayer_ds_out_res.width /
-					input_config->effective_res.width + 1;
-				w_padding = roundup(w_padding/2, 1);
-
-				bq_res->gdc_shift_bq.width_bq = bq_res->ispfilter_bq.width_bq / 2
-					+ w_padding;
-				bq_res->gdc_shift_bq.height_bq = 4 / 2;
-			}
-		} else {
-			unsigned int dvs_w, dvs_h, dvs_w_max, dvs_h_max;
-
-			bq_res->ispfilter_bq.width_bq = 8 / 2;
-			bq_res->ispfilter_bq.height_bq = 8 / 2;
-
-			if (IS_HWREVISION(asd->isp, ATOMISP_HW_REVISION_ISP2401)) {
-				/* No additional left padding for ISYS2401 */
-				bq_res->gdc_shift_bq.width_bq = 4 / 2;
-				bq_res->gdc_shift_bq.height_bq = 4 / 2;
-			} else {
-				w_padding =
-				    roundup(input_config->effective_res.width, 128) -
-				    input_config->effective_res.width;
-				if (w_padding < 12)
-					w_padding = 12;
-				bq_res->gdc_shift_bq.width_bq = 4 / 2 +
-				    ((w_padding - 12) *
-				     pipe_cfg->bayer_ds_out_res.width /
-				input_config->effective_res.width + 1) / 2;
-				bq_res->gdc_shift_bq.height_bq = 4 / 2;
-			}
-
-			dvs_w = pipe_cfg->bayer_ds_out_res.width -
-				pipe_cfg->output_info[0].res.width;
-			dvs_h = pipe_cfg->bayer_ds_out_res.height -
-				pipe_cfg->output_info[0].res.height;
-			dvs_w_max = rounddown(
-					pipe_cfg->output_info[0].res.width / 5,
-					ATOM_ISP_STEP_WIDTH);
-			dvs_h_max = rounddown(
-					pipe_cfg->output_info[0].res.height / 5,
-					ATOM_ISP_STEP_HEIGHT);
-			bq_res->envelope_bq.width_bq =
-				min((dvs_w / 2), (dvs_w_max / 2)) -
-				bq_res->ispfilter_bq.width_bq;
-			bq_res->envelope_bq.height_bq =
-				min((dvs_h / 2), (dvs_h_max / 2)) -
-				bq_res->ispfilter_bq.height_bq;
-		}
-	}
-
-	dev_dbg(asd->isp->dev, "source_bq.width_bq %d, source_bq.height_bq %d,\nispfilter_bq.width_bq %d, ispfilter_bq.height_bq %d,\ngdc_shift_bq.width_bq %d, gdc_shift_bq.height_bq %d,\nenvelope_bq.width_bq %d, envelope_bq.height_bq %d,\noutput_bq.width_bq %d, output_bq.height_bq %d\n",
-	      bq_res->source_bq.width_bq, bq_res->source_bq.height_bq,
-	      bq_res->ispfilter_bq.width_bq, bq_res->ispfilter_bq.height_bq,
-	      bq_res->gdc_shift_bq.width_bq, bq_res->gdc_shift_bq.height_bq,
-	      bq_res->envelope_bq.width_bq, bq_res->envelope_bq.height_bq,
-	      bq_res->output_bq.width_bq, bq_res->output_bq.height_bq);
-
-	return 0;
-}
-
-int atomisp_set_dis_coefs(struct atomisp_sub_device *asd,
-			  struct atomisp_dis_coefficients *coefs)
-{
-	return atomisp_css_set_dis_coefs(asd, coefs);
-}
-
-/*
- * Function to set/get 3A stat from isp
- */
-int atomisp_3a_stat(struct atomisp_sub_device *asd, int flag,
-		    struct atomisp_3a_statistics *config)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_s3a_buf *s3a_buf;
-	unsigned long ret;
-
-	if (flag != 0)
-		return -EINVAL;
-
-	/* sanity check to avoid writing into unallocated memory. */
-	if (asd->params.s3a_output_bytes == 0)
-		return -EINVAL;
-
-	if (atomisp_compare_grid(asd, &config->grid_info) != 0) {
-		/* If the grid info in the argument differs from the current
-		   grid info, we tell the caller to reset the grid size and
-		   try again. */
-		return -EAGAIN;
-	}
-
-	if (list_empty(&asd->s3a_stats_ready)) {
-		dev_err(isp->dev, "3a statistics is not valid.\n");
-		return -EAGAIN;
-	}
-
-	s3a_buf = list_entry(asd->s3a_stats_ready.next,
-			struct atomisp_s3a_buf, list);
-	if (s3a_buf->s3a_map)
-		ia_css_translate_3a_statistics(
-			asd->params.s3a_user_stat, s3a_buf->s3a_map);
-	else
-		ia_css_get_3a_statistics(asd->params.s3a_user_stat,
-			s3a_buf->s3a_data);
-
-	config->exp_id = s3a_buf->s3a_data->exp_id;
-	config->isp_config_id = s3a_buf->s3a_data->isp_config_id;
-
-	ret = copy_to_user(config->data, asd->params.s3a_user_stat->data,
-			   asd->params.s3a_output_bytes);
-	if (ret) {
-		dev_err(isp->dev, "copy to user failed: copied %lu bytes\n",
-				ret);
-		return -EFAULT;
-	}
-
-	/* Move to free buffer list */
-	list_del_init(&s3a_buf->list);
-	list_add_tail(&s3a_buf->list, &asd->s3a_stats);
-	dev_dbg(isp->dev, "%s: finish getting exp_id %d 3a stat, isp_config_id %d\n", __func__,
-		config->exp_id, config->isp_config_id);
-	return 0;
-}
-
-int atomisp_get_metadata(struct atomisp_sub_device *asd, int flag,
-			 struct atomisp_metadata *md)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct ia_css_stream_config *stream_config;
-	struct ia_css_stream_info *stream_info;
-	struct camera_mipi_info *mipi_info;
-	struct atomisp_metadata_buf *md_buf;
-	enum atomisp_metadata_type md_type = ATOMISP_MAIN_METADATA;
-	int ret, i;
-
-	if (flag != 0)
-		return -EINVAL;
-
-	stream_config = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
-		stream_config;
-	stream_info = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
-		stream_info;
-
-	/* We always return the resolution and stride even if there is
-	 * no valid metadata. This allows the caller to get the information
-	 * needed to allocate user-space buffers. */
-	md->width  = stream_info->metadata_info.resolution.width;
-	md->height = stream_info->metadata_info.resolution.height;
-	md->stride = stream_info->metadata_info.stride;
-
-	/* sanity check to avoid writing into unallocated memory.
-	 * This does not return an error because it is a valid way
-	 * for applications to detect that metadata is not enabled. */
-	if (md->width == 0 || md->height == 0 || !md->data)
-		return 0;
-
-	/* This is done in the atomisp_buf_done() */
-	if (list_empty(&asd->metadata_ready[md_type])) {
-		dev_warn(isp->dev, "Metadata queue is empty now!\n");
-		return -EAGAIN;
-	}
-
-	mipi_info = atomisp_to_sensor_mipi_info(
-		isp->inputs[asd->input_curr].camera);
-	if (mipi_info == NULL)
-		return -EINVAL;
-
-	if (mipi_info->metadata_effective_width != NULL) {
-		for (i = 0; i < md->height; i++)
-			md->effective_width[i] =
-				mipi_info->metadata_effective_width[i];
-	}
-
-	md_buf = list_entry(asd->metadata_ready[md_type].next,
-			    struct atomisp_metadata_buf, list);
-	md->exp_id = md_buf->metadata->exp_id;
-	if (md_buf->md_vptr) {
-		ret = copy_to_user(md->data,
-				   md_buf->md_vptr,
-				   stream_info->metadata_info.size);
-	} else {
-		hmm_load(md_buf->metadata->address,
-				    asd->params.metadata_user[md_type],
-				    stream_info->metadata_info.size);
-
-		ret = copy_to_user(md->data,
-				   asd->params.metadata_user[md_type],
-				   stream_info->metadata_info.size);
-	}
-	if (ret) {
-		dev_err(isp->dev, "copy to user failed: copied %d bytes\n",
-			ret);
-		return -EFAULT;
-	}
-
-	list_del_init(&md_buf->list);
-	list_add_tail(&md_buf->list, &asd->metadata[md_type]);
-
-	dev_dbg(isp->dev, "%s: HAL de-queued metadata type %d with exp_id %d\n",
-		__func__, md_type, md->exp_id);
-	return 0;
-}
-
-int atomisp_get_metadata_by_type(struct atomisp_sub_device *asd, int flag,
-				 struct atomisp_metadata_with_type *md)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct ia_css_stream_config *stream_config;
-	struct ia_css_stream_info *stream_info;
-	struct camera_mipi_info *mipi_info;
-	struct atomisp_metadata_buf *md_buf;
-	enum atomisp_metadata_type md_type;
-	int ret, i;
-
-	if (flag != 0)
-		return -EINVAL;
-
-	stream_config = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
-		stream_config;
-	stream_info = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
-		stream_info;
-
-	/* We always return the resolution and stride even if there is
-	 * no valid metadata. This allows the caller to get the information
-	 * needed to allocate user-space buffers. */
-	md->width  = stream_info->metadata_info.resolution.width;
-	md->height = stream_info->metadata_info.resolution.height;
-	md->stride = stream_info->metadata_info.stride;
-
-	/* sanity check to avoid writing into unallocated memory.
-	 * This does not return an error because it is a valid way
-	 * for applications to detect that metadata is not enabled. */
-	if (md->width == 0 || md->height == 0 || !md->data)
-		return 0;
-
-	md_type = md->type;
-	if (md_type < 0 || md_type >= ATOMISP_METADATA_TYPE_NUM)
-		return -EINVAL;
-
-	/* This is done in the atomisp_buf_done() */
-	if (list_empty(&asd->metadata_ready[md_type])) {
-		dev_warn(isp->dev, "Metadata queue is empty now!\n");
-		return -EAGAIN;
-	}
-
-	mipi_info = atomisp_to_sensor_mipi_info(
-		isp->inputs[asd->input_curr].camera);
-	if (mipi_info == NULL)
-		return -EINVAL;
-
-	if (mipi_info->metadata_effective_width != NULL) {
-		for (i = 0; i < md->height; i++)
-			md->effective_width[i] =
-				mipi_info->metadata_effective_width[i];
-	}
-
-	md_buf = list_entry(asd->metadata_ready[md_type].next,
-			    struct atomisp_metadata_buf, list);
-	md->exp_id = md_buf->metadata->exp_id;
-	if (md_buf->md_vptr) {
-		ret = copy_to_user(md->data,
-				   md_buf->md_vptr,
-				   stream_info->metadata_info.size);
-	} else {
-		hmm_load(md_buf->metadata->address,
-				    asd->params.metadata_user[md_type],
-				    stream_info->metadata_info.size);
-
-		ret = copy_to_user(md->data,
-				   asd->params.metadata_user[md_type],
-				   stream_info->metadata_info.size);
-	}
-	if (ret) {
-		dev_err(isp->dev, "copy to user failed: copied %d bytes\n",
-			ret);
-		return -EFAULT;
-	} else {
-		list_del_init(&md_buf->list);
-		list_add_tail(&md_buf->list, &asd->metadata[md_type]);
-	}
-	dev_dbg(isp->dev, "%s: HAL de-queued metadata type %d with exp_id %d\n",
-		__func__, md_type, md->exp_id);
-	return 0;
-}
-
-/*
- * Function to calculate real zoom region for every pipe
- */
-int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd,
-				       struct ia_css_dz_config   *dz_config,
-				       enum atomisp_css_pipe_id css_pipe_id)
-
-{
-	struct atomisp_stream_env *stream_env =
-			&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	struct atomisp_resolution  eff_res, out_res;
-#ifdef ISP2401
-	int w_offset, h_offset;
-#endif
-
-	memset(&eff_res, 0, sizeof(eff_res));
-	memset(&out_res, 0, sizeof(out_res));
-
-	if (dz_config->dx || dz_config->dy)
-		return 0;
-
-	if (css_pipe_id != IA_CSS_PIPE_ID_PREVIEW
-		&& css_pipe_id != IA_CSS_PIPE_ID_CAPTURE) {
-		dev_err(asd->isp->dev, "%s the set pipe no support crop region"
-			, __func__);
-		return -EINVAL;
-	}
-
-	eff_res.width =
-		stream_env->stream_config.input_config.effective_res.width;
-	eff_res.height =
-		stream_env->stream_config.input_config.effective_res.height;
-	if (eff_res.width == 0 || eff_res.height == 0) {
-		dev_err(asd->isp->dev, "%s err effective resolution"
-				, __func__);
-		return -EINVAL;
-	}
-
-	if (dz_config->zoom_region.resolution.width
-		== asd->sensor_array_res.width
-		|| dz_config->zoom_region.resolution.height
-		== asd->sensor_array_res.height) {
-		/*no need crop region*/
-		dz_config->zoom_region.origin.x = 0;
-		dz_config->zoom_region.origin.y = 0;
-		dz_config->zoom_region.resolution.width = eff_res.width;
-		dz_config->zoom_region.resolution.height = eff_res.height;
-		return 0;
-	}
-
-	/* FIXME:
-	 * This is not the correct implementation with Google's definition, due
-	 * to firmware limitation.
-	 * map real crop region base on above calculating base max crop region.
-	 */
-#ifdef ISP2401
-	out_res.width =
-		stream_env->pipe_configs[css_pipe_id].output_info[0].res.width;
-	out_res.height =
-		stream_env->pipe_configs[css_pipe_id].output_info[0].res.height;
-	if (out_res.width == 0 || out_res.height == 0) {
-		dev_err(asd->isp->dev, "%s err current pipe output resolution"
-				, __func__);
-		return -EINVAL;
-	}
-
-	if (asd->sensor_array_res.width * out_res.height
-			< out_res.width * asd->sensor_array_res.height) {
-		h_offset = asd->sensor_array_res.height -
-				asd->sensor_array_res.width
-				* out_res.height / out_res.width;
-		h_offset = h_offset / 2;
-		if (dz_config->zoom_region.origin.y < h_offset)
-			dz_config->zoom_region.origin.y = 0;
-		else
-			dz_config->zoom_region.origin.y =
-				dz_config->zoom_region.origin.y - h_offset;
-		w_offset = 0;
-	} else {
-		w_offset = asd->sensor_array_res.width -
-				asd->sensor_array_res.height
-				* out_res.width / out_res.height;
-		w_offset = w_offset / 2;
-		if (dz_config->zoom_region.origin.x < w_offset)
-			dz_config->zoom_region.origin.x = 0;
-		else
-			dz_config->zoom_region.origin.x =
-				dz_config->zoom_region.origin.x - w_offset;
-		h_offset = 0;
-	}
-#endif
-	dz_config->zoom_region.origin.x =
-			dz_config->zoom_region.origin.x
-			* eff_res.width
-#ifndef ISP2401
-			/ asd->sensor_array_res.width;
-#else
-			/ (asd->sensor_array_res.width -
-			2 * w_offset);
-#endif
-	dz_config->zoom_region.origin.y =
-			dz_config->zoom_region.origin.y
-			* eff_res.height
-#ifndef ISP2401
-			/ asd->sensor_array_res.height;
-#else
-			/ (asd->sensor_array_res.height -
-			2 * h_offset);
-#endif
-	dz_config->zoom_region.resolution.width =
-			dz_config->zoom_region.resolution.width
-			* eff_res.width
-#ifndef ISP2401
-			/ asd->sensor_array_res.width;
-#else
-			/ (asd->sensor_array_res.width -
-			2 * w_offset);
-#endif
-	dz_config->zoom_region.resolution.height =
-			dz_config->zoom_region.resolution.height
-			* eff_res.height
-#ifndef ISP2401
-			/ asd->sensor_array_res.height;
-#else
-			/ (asd->sensor_array_res.height -
-			2 * h_offset);
-#endif
-
-	/*
-	  * Set same ratio of crop region resolution and current pipe output
-	  * resolution
-	  */
-#ifndef ISP2401
-	out_res.width =
-		stream_env->pipe_configs[css_pipe_id].output_info[0].res.width;
-	out_res.height =
-		stream_env->pipe_configs[css_pipe_id].output_info[0].res.height;
-	if (out_res.width == 0 || out_res.height == 0) {
-		dev_err(asd->isp->dev, "%s err current pipe output resolution"
-				, __func__);
-		return -EINVAL;
-	}
-
-#endif
-	if (out_res.width * dz_config->zoom_region.resolution.height
-		> dz_config->zoom_region.resolution.width * out_res.height) {
-		dz_config->zoom_region.resolution.height =
-				dz_config->zoom_region.resolution.width
-				* out_res.height / out_res.width;
-	} else {
-		dz_config->zoom_region.resolution.width =
-				dz_config->zoom_region.resolution.height
-				* out_res.width / out_res.height;
-	}
-	dev_dbg(asd->isp->dev, "%s crop region:(%d,%d),(%d,%d) eff_res(%d, %d) array_size(%d,%d) out_res(%d, %d)\n",
-			__func__, dz_config->zoom_region.origin.x,
-			dz_config->zoom_region.origin.y,
-			dz_config->zoom_region.resolution.width,
-			dz_config->zoom_region.resolution.height,
-			eff_res.width, eff_res.height,
-			asd->sensor_array_res.width,
-			asd->sensor_array_res.height,
-			out_res.width, out_res.height);
-
-
-	if ((dz_config->zoom_region.origin.x +
-		dz_config->zoom_region.resolution.width
-		> eff_res.width) ||
-		(dz_config->zoom_region.origin.y +
-		dz_config->zoom_region.resolution.height
-		> eff_res.height))
-		return -EINVAL;
-
-	return 0;
-}
-
-
-/*
- * Function to check the zoom region whether is effective
- */
-static bool atomisp_check_zoom_region(
-			struct atomisp_sub_device *asd,
-			struct ia_css_dz_config *dz_config)
-{
-	struct atomisp_resolution  config;
-	bool flag = false;
-	unsigned int w , h;
-
-	memset(&config, 0, sizeof(struct atomisp_resolution));
-
-	if (dz_config->dx && dz_config->dy)
-		return true;
-
-	config.width = asd->sensor_array_res.width;
-	config.height = asd->sensor_array_res.height;
-	w = dz_config->zoom_region.origin.x +
-		dz_config->zoom_region.resolution.width;
-	h = dz_config->zoom_region.origin.y +
-		dz_config->zoom_region.resolution.height;
-
-	if ((w <= config.width) && (h <= config.height) && w > 0 && h > 0)
-		flag = true;
-	else
-		/* setting error zoom region */
-		dev_err(asd->isp->dev, "%s zoom region ERROR:dz_config:(%d,%d),(%d,%d)array_res(%d, %d)\n",
-			__func__, dz_config->zoom_region.origin.x,
-			dz_config->zoom_region.origin.y,
-			dz_config->zoom_region.resolution.width,
-			dz_config->zoom_region.resolution.height,
-			config.width, config.height);
-
-	return flag;
-}
-
-void atomisp_apply_css_parameters(
-				struct atomisp_sub_device *asd,
-				struct atomisp_css_params *css_param)
-{
-	if (css_param->update_flag.wb_config)
-		atomisp_css_set_wb_config(asd, &css_param->wb_config);
-
-	if (css_param->update_flag.ob_config)
-		atomisp_css_set_ob_config(asd, &css_param->ob_config);
-
-	if (css_param->update_flag.dp_config)
-		atomisp_css_set_dp_config(asd, &css_param->dp_config);
-
-	if (css_param->update_flag.nr_config)
-		atomisp_css_set_nr_config(asd, &css_param->nr_config);
-
-	if (css_param->update_flag.ee_config)
-		atomisp_css_set_ee_config(asd, &css_param->ee_config);
-
-	if (css_param->update_flag.tnr_config)
-		atomisp_css_set_tnr_config(asd, &css_param->tnr_config);
-
-	if (css_param->update_flag.a3a_config)
-		atomisp_css_set_3a_config(asd, &css_param->s3a_config);
-
-	if (css_param->update_flag.ctc_config)
-		atomisp_css_set_ctc_config(asd, &css_param->ctc_config);
-
-	if (css_param->update_flag.cnr_config)
-		atomisp_css_set_cnr_config(asd, &css_param->cnr_config);
-
-	if (css_param->update_flag.ecd_config)
-		atomisp_css_set_ecd_config(asd, &css_param->ecd_config);
-
-	if (css_param->update_flag.ynr_config)
-		atomisp_css_set_ynr_config(asd, &css_param->ynr_config);
-
-	if (css_param->update_flag.fc_config)
-		atomisp_css_set_fc_config(asd, &css_param->fc_config);
-
-	if (css_param->update_flag.macc_config)
-		atomisp_css_set_macc_config(asd, &css_param->macc_config);
-
-	if (css_param->update_flag.aa_config)
-		atomisp_css_set_aa_config(asd, &css_param->aa_config);
-
-	if (css_param->update_flag.anr_config)
-		atomisp_css_set_anr_config(asd, &css_param->anr_config);
-
-	if (css_param->update_flag.xnr_config)
-		atomisp_css_set_xnr_config(asd, &css_param->xnr_config);
-
-	if (css_param->update_flag.yuv2rgb_cc_config)
-		atomisp_css_set_yuv2rgb_cc_config(asd,
-					&css_param->yuv2rgb_cc_config);
-
-	if (css_param->update_flag.rgb2yuv_cc_config)
-		atomisp_css_set_rgb2yuv_cc_config(asd,
-					&css_param->rgb2yuv_cc_config);
-
-	if (css_param->update_flag.macc_table)
-		atomisp_css_set_macc_table(asd, &css_param->macc_table);
-
-	if (css_param->update_flag.xnr_table)
-		atomisp_css_set_xnr_table(asd, &css_param->xnr_table);
-
-	if (css_param->update_flag.r_gamma_table)
-		atomisp_css_set_r_gamma_table(asd, &css_param->r_gamma_table);
-
-	if (css_param->update_flag.g_gamma_table)
-		atomisp_css_set_g_gamma_table(asd, &css_param->g_gamma_table);
-
-	if (css_param->update_flag.b_gamma_table)
-		atomisp_css_set_b_gamma_table(asd, &css_param->b_gamma_table);
-
-	if (css_param->update_flag.anr_thres)
-		atomisp_css_set_anr_thres(asd, &css_param->anr_thres);
-
-	if (css_param->update_flag.shading_table)
-		atomisp_css_set_shading_table(asd, css_param->shading_table);
-
-	if (css_param->update_flag.morph_table && asd->params.gdc_cac_en)
-		atomisp_css_set_morph_table(asd, css_param->morph_table);
-
-	if (css_param->update_flag.dvs2_coefs) {
-		struct atomisp_css_dvs_grid_info *dvs_grid_info =
-			atomisp_css_get_dvs_grid_info(
-				&asd->params.curr_grid_info);
-
-		if (dvs_grid_info && dvs_grid_info->enable)
-			atomisp_css_set_dvs2_coefs(asd, css_param->dvs2_coeff);
-	}
-
-	if (css_param->update_flag.dvs_6axis_config)
-		atomisp_css_set_dvs_6axis(asd, css_param->dvs_6axis);
-
-	atomisp_css_set_isp_config_id(asd, css_param->isp_config_id);
-	/*
-	 * These configurations are on used by ISP1.x, not for ISP2.x,
-	 * so do not handle them. see comments of ia_css_isp_config.
-	 * 1 cc_config
-	 * 2 ce_config
-	 * 3 de_config
-	 * 4 gc_config
-	 * 5 gamma_table
-	 * 6 ctc_table
-	 * 7 dvs_coefs
-	 */
-}
-
-static unsigned int long copy_from_compatible(void *to, const void *from,
-					      unsigned long n, bool from_user)
-{
-	if (from_user)
-		return copy_from_user(to, from, n);
-	else
-		memcpy(to, from, n);
-	return 0;
-}
-
-int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
-				      struct atomisp_parameters *arg,
-				      struct atomisp_css_params *css_param,
-				      bool from_user)
-{
-	struct atomisp_parameters *cur_config = &css_param->update_flag;
-
-	if (!arg || !asd || !css_param)
-		return -EINVAL;
-
-	if (arg->wb_config && (from_user || !cur_config->wb_config)) {
-		if (copy_from_compatible(&css_param->wb_config, arg->wb_config,
-				sizeof(struct atomisp_css_wb_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.wb_config =
-			(struct atomisp_wb_config *) &css_param->wb_config;
-	}
-
-	if (arg->ob_config && (from_user || !cur_config->ob_config)) {
-		if (copy_from_compatible(&css_param->ob_config, arg->ob_config,
-				sizeof(struct atomisp_css_ob_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.ob_config =
-			(struct atomisp_ob_config *) &css_param->ob_config;
-	}
-
-	if (arg->dp_config && (from_user || !cur_config->dp_config)) {
-		if (copy_from_compatible(&css_param->dp_config, arg->dp_config,
-				sizeof(struct atomisp_css_dp_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.dp_config =
-			(struct atomisp_dp_config *) &css_param->dp_config;
-	}
-
-	if (asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO) {
-		if (arg->dz_config && (from_user || !cur_config->dz_config)) {
-			if (copy_from_compatible(&css_param->dz_config,
-				arg->dz_config,
-				sizeof(struct atomisp_css_dz_config),
-				from_user))
-				return -EFAULT;
-			if (!atomisp_check_zoom_region(asd,
-						&css_param->dz_config)) {
-				dev_err(asd->isp->dev, "crop region error!");
-				return -EINVAL;
-			}
-			css_param->update_flag.dz_config =
-				(struct atomisp_dz_config *)
-				&css_param->dz_config;
-		}
-	}
-
-	if (arg->nr_config && (from_user || !cur_config->nr_config)) {
-		if (copy_from_compatible(&css_param->nr_config, arg->nr_config,
-				sizeof(struct atomisp_css_nr_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.nr_config =
-			(struct atomisp_nr_config *) &css_param->nr_config;
-	}
-
-	if (arg->ee_config && (from_user || !cur_config->ee_config)) {
-		if (copy_from_compatible(&css_param->ee_config, arg->ee_config,
-				sizeof(struct atomisp_css_ee_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.ee_config =
-			(struct atomisp_ee_config *) &css_param->ee_config;
-	}
-
-	if (arg->tnr_config && (from_user || !cur_config->tnr_config)) {
-		if (copy_from_compatible(&css_param->tnr_config,
-				arg->tnr_config,
-				sizeof(struct atomisp_css_tnr_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.tnr_config =
-			(struct atomisp_tnr_config *)
-			&css_param->tnr_config;
-	}
-
-	if (arg->a3a_config && (from_user || !cur_config->a3a_config)) {
-		if (copy_from_compatible(&css_param->s3a_config,
-				arg->a3a_config,
-				sizeof(struct atomisp_css_3a_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.a3a_config =
-			(struct atomisp_3a_config *) &css_param->s3a_config;
-	}
-
-	if (arg->ctc_config && (from_user || !cur_config->ctc_config)) {
-		if (copy_from_compatible(&css_param->ctc_config,
-				arg->ctc_config,
-				sizeof(struct atomisp_css_ctc_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.ctc_config =
-			(struct atomisp_ctc_config *)
-			&css_param->ctc_config;
-	}
-
-	if (arg->cnr_config && (from_user || !cur_config->cnr_config)) {
-		if (copy_from_compatible(&css_param->cnr_config,
-				arg->cnr_config,
-				sizeof(struct atomisp_css_cnr_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.cnr_config =
-			(struct atomisp_cnr_config *)
-			&css_param->cnr_config;
-	}
-
-	if (arg->ecd_config && (from_user || !cur_config->ecd_config)) {
-		if (copy_from_compatible(&css_param->ecd_config,
-				arg->ecd_config,
-				sizeof(struct atomisp_css_ecd_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.ecd_config =
-			(struct atomisp_ecd_config *)
-			&css_param->ecd_config;
-	}
-
-	if (arg->ynr_config && (from_user || !cur_config->ynr_config)) {
-		if (copy_from_compatible(&css_param->ynr_config,
-				arg->ynr_config,
-				sizeof(struct atomisp_css_ynr_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.ynr_config =
-			(struct atomisp_ynr_config *)
-			&css_param->ynr_config;
-	}
-
-	if (arg->fc_config && (from_user || !cur_config->fc_config)) {
-		if (copy_from_compatible(&css_param->fc_config,
-				arg->fc_config,
-				sizeof(struct atomisp_css_fc_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.fc_config =
-			(struct atomisp_fc_config *) &css_param->fc_config;
-	}
-
-	if (arg->macc_config && (from_user || !cur_config->macc_config)) {
-		if (copy_from_compatible(&css_param->macc_config,
-				arg->macc_config,
-				sizeof(struct atomisp_css_macc_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.macc_config =
-			(struct atomisp_macc_config *)
-			&css_param->macc_config;
-	}
-
-	if (arg->aa_config && (from_user || !cur_config->aa_config)) {
-		if (copy_from_compatible(&css_param->aa_config, arg->aa_config,
-				sizeof(struct atomisp_css_aa_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.aa_config =
-			(struct atomisp_aa_config *) &css_param->aa_config;
-	}
-
-	if (arg->anr_config && (from_user || !cur_config->anr_config)) {
-		if (copy_from_compatible(&css_param->anr_config,
-				arg->anr_config,
-				sizeof(struct atomisp_css_anr_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.anr_config =
-			(struct atomisp_anr_config *)
-			&css_param->anr_config;
-	}
-
-	if (arg->xnr_config && (from_user || !cur_config->xnr_config)) {
-		if (copy_from_compatible(&css_param->xnr_config,
-				arg->xnr_config,
-				sizeof(struct atomisp_css_xnr_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.xnr_config =
-			(struct atomisp_xnr_config *)
-			&css_param->xnr_config;
-	}
-
-	if (arg->yuv2rgb_cc_config &&
-	   (from_user || !cur_config->yuv2rgb_cc_config)) {
-		if (copy_from_compatible(&css_param->yuv2rgb_cc_config,
-				arg->yuv2rgb_cc_config,
-				sizeof(struct atomisp_css_cc_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.yuv2rgb_cc_config =
-			(struct atomisp_cc_config *)
-			&css_param->yuv2rgb_cc_config;
-	}
-
-	if (arg->rgb2yuv_cc_config &&
-	   (from_user || !cur_config->rgb2yuv_cc_config)) {
-		if (copy_from_compatible(&css_param->rgb2yuv_cc_config,
-				arg->rgb2yuv_cc_config,
-				sizeof(struct atomisp_css_cc_config),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.rgb2yuv_cc_config =
-			(struct atomisp_cc_config *)
-			&css_param->rgb2yuv_cc_config;
-	}
-
-	if (arg->macc_table && (from_user || !cur_config->macc_table)) {
-		if (copy_from_compatible(&css_param->macc_table,
-				arg->macc_table,
-				sizeof(struct atomisp_css_macc_table),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.macc_table =
-			(struct atomisp_macc_table *)
-			&css_param->macc_table;
-	}
-
-	if (arg->xnr_table && (from_user || !cur_config->xnr_table)) {
-		if (copy_from_compatible(&css_param->xnr_table,
-				arg->xnr_table,
-				sizeof(struct atomisp_css_xnr_table),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.xnr_table =
-			(struct atomisp_xnr_table *) &css_param->xnr_table;
-	}
-
-	if (arg->r_gamma_table && (from_user || !cur_config->r_gamma_table)) {
-		if (copy_from_compatible(&css_param->r_gamma_table,
-				arg->r_gamma_table,
-				sizeof(struct atomisp_css_rgb_gamma_table),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.r_gamma_table =
-			(struct atomisp_rgb_gamma_table *)
-			&css_param->r_gamma_table;
-	}
-
-	if (arg->g_gamma_table && (from_user || !cur_config->g_gamma_table)) {
-		if (copy_from_compatible(&css_param->g_gamma_table,
-				arg->g_gamma_table,
-				sizeof(struct atomisp_css_rgb_gamma_table),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.g_gamma_table =
-			(struct atomisp_rgb_gamma_table *)
-			&css_param->g_gamma_table;
-	}
-
-	if (arg->b_gamma_table && (from_user || !cur_config->b_gamma_table)) {
-		if (copy_from_compatible(&css_param->b_gamma_table,
-				arg->b_gamma_table,
-				sizeof(struct atomisp_css_rgb_gamma_table),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.b_gamma_table =
-			(struct atomisp_rgb_gamma_table *)
-			&css_param->b_gamma_table;
-	}
-
-	if (arg->anr_thres && (from_user || !cur_config->anr_thres)) {
-		if (copy_from_compatible(&css_param->anr_thres, arg->anr_thres,
-				sizeof(struct atomisp_css_anr_thres),
-				from_user))
-			return -EFAULT;
-		css_param->update_flag.anr_thres =
-			(struct atomisp_anr_thres *) &css_param->anr_thres;
-	}
-
-	if (from_user)
-		css_param->isp_config_id = arg->isp_config_id;
-	/*
-	 * These configurations are on used by ISP1.x, not for ISP2.x,
-	 * so do not handle them. see comments of ia_css_isp_config.
-	 * 1 cc_config
-	 * 2 ce_config
-	 * 3 de_config
-	 * 4 gc_config
-	 * 5 gamma_table
-	 * 6 ctc_table
-	 * 7 dvs_coefs
-	 */
-	return 0;
-}
-
-int atomisp_cp_lsc_table(struct atomisp_sub_device *asd,
-			 struct atomisp_shading_table *source_st,
-			 struct atomisp_css_params *css_param,
-			 bool from_user)
-{
-	unsigned int i;
-	unsigned int len_table;
-	struct atomisp_css_shading_table *shading_table;
-	struct atomisp_css_shading_table *old_table;
-#ifdef ISP2401
-	struct atomisp_shading_table st;
-#endif
-
-	if (!source_st)
-		return 0;
-
-	if (!css_param)
-		return -EINVAL;
-
-	if (!from_user && css_param->update_flag.shading_table)
-		return 0;
-
-#ifdef ISP2401
-	if (copy_from_compatible(&st, source_st,
-				 sizeof(struct atomisp_shading_table),
-				 from_user)) {
-		dev_err(asd->isp->dev, "copy shading table failed!");
-		return -EFAULT;
-	}
-
-#endif
-	old_table = css_param->shading_table;
-
-#ifdef ISP2401
-
-#endif
-	/* user config is to disable the shading table. */
-#ifndef ISP2401
-	if (!source_st->enable) {
-#else
-	if (!st.enable) {
-#endif
-		/* Generate a minimum table with enable = 0. */
-		shading_table = atomisp_css_shading_table_alloc(1, 1);
-		if (!shading_table)
-			return -ENOMEM;
-		shading_table->enable = 0;
-		goto set_lsc;
-	}
-
-	/* Setting a new table. Validate first - all tables must be set */
-	for (i = 0; i < ATOMISP_NUM_SC_COLORS; i++) {
-#ifndef ISP2401
-		if (!source_st->data[i])
-#else
-		if (!st.data[i]) {
-			dev_err(asd->isp->dev, "shading table validate failed");
-#endif
-			return -EINVAL;
-#ifdef ISP2401
-		}
-#endif
-	}
-
-	/* Shading table size per color */
-#ifndef ISP2401
-	if (source_st->width > SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR ||
-		source_st->height > SH_CSS_MAX_SCTBL_HEIGHT_PER_COLOR)
-#else
-	if (st.width > SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR ||
-	    st.height > SH_CSS_MAX_SCTBL_HEIGHT_PER_COLOR) {
-		dev_err(asd->isp->dev, "shading table w/h validate failed!");
-#endif
-		return -EINVAL;
-#ifdef ISP2401
-	}
-#endif
-
-#ifndef ISP2401
-	shading_table = atomisp_css_shading_table_alloc(source_st->width,
-							source_st->height);
-	if (!shading_table)
-			return -ENOMEM;
-#else
-	shading_table = atomisp_css_shading_table_alloc(st.width,
-							st.height);
-	if (!shading_table) {
-		dev_err(asd->isp->dev, "shading table alloc failed!");
-		return -ENOMEM;
-	}
-#endif
-
-#ifndef ISP2401
-	len_table = source_st->width * source_st->height * ATOMISP_SC_TYPE_SIZE;
-#else
-	len_table = st.width * st.height * ATOMISP_SC_TYPE_SIZE;
-#endif
-	for (i = 0; i < ATOMISP_NUM_SC_COLORS; i++) {
-		if (copy_from_compatible(shading_table->data[i],
-#ifndef ISP2401
-			source_st->data[i], len_table, from_user)) {
-#else
-			st.data[i], len_table, from_user)) {
-#endif
-			atomisp_css_shading_table_free(shading_table);
-			return -EFAULT;
-		}
-
-	}
-#ifndef ISP2401
-	shading_table->sensor_width = source_st->sensor_width;
-	shading_table->sensor_height = source_st->sensor_height;
-	shading_table->fraction_bits = source_st->fraction_bits;
-	shading_table->enable = source_st->enable;
-#else
-	shading_table->sensor_width = st.sensor_width;
-	shading_table->sensor_height = st.sensor_height;
-	shading_table->fraction_bits = st.fraction_bits;
-	shading_table->enable = st.enable;
-#endif
-
-	/* No need to update shading table if it is the same */
-	if (old_table != NULL &&
-		old_table->sensor_width == shading_table->sensor_width &&
-		old_table->sensor_height == shading_table->sensor_height &&
-		old_table->width == shading_table->width &&
-		old_table->height == shading_table->height &&
-		old_table->fraction_bits == shading_table->fraction_bits &&
-		old_table->enable == shading_table->enable) {
-		bool data_is_same = true;
-
-		for (i = 0; i < ATOMISP_NUM_SC_COLORS; i++) {
-			if (memcmp(shading_table->data[i], old_table->data[i],
-				   len_table) != 0) {
-				data_is_same = false;
-				break;
-			}
-		}
-
-		if (data_is_same) {
-			atomisp_css_shading_table_free(shading_table);
-			return 0;
-		}
-	}
-
-set_lsc:
-	/* set LSC to CSS */
-	css_param->shading_table = shading_table;
-	css_param->update_flag.shading_table =
-		(struct atomisp_shading_table *) shading_table;
-	asd->params.sc_en = shading_table != NULL;
-
-	if (old_table)
-		atomisp_css_shading_table_free(old_table);
-
-	return 0;
-}
-
-int atomisp_css_cp_dvs2_coefs(struct atomisp_sub_device *asd,
-			      struct ia_css_dvs2_coefficients *coefs,
-			      struct atomisp_css_params *css_param,
-			      bool from_user)
-{
-	struct atomisp_css_dvs_grid_info *cur =
-		atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
-	int dvs_hor_coef_bytes, dvs_ver_coef_bytes;
-#ifdef ISP2401
-	struct ia_css_dvs2_coefficients dvs2_coefs;
-#endif
-
-	if (!coefs || !cur)
-		return 0;
-
-	if (!from_user && css_param->update_flag.dvs2_coefs)
-		return 0;
-
-#ifndef ISP2401
-	if (sizeof(*cur) != sizeof(coefs->grid) ||
-	    memcmp(&coefs->grid, cur, sizeof(coefs->grid))) {
-#else
-	if (copy_from_compatible(&dvs2_coefs, coefs,
-				 sizeof(struct ia_css_dvs2_coefficients),
-				 from_user)) {
-		dev_err(asd->isp->dev, "copy dvs2 coef failed");
-		return -EFAULT;
-	}
-
-	if (sizeof(*cur) != sizeof(dvs2_coefs.grid) ||
-	    memcmp(&dvs2_coefs.grid, cur, sizeof(dvs2_coefs.grid))) {
-#endif
-		dev_err(asd->isp->dev, "dvs grid mis-match!\n");
-		/* If the grid info in the argument differs from the current
-		   grid info, we tell the caller to reset the grid size and
-		   try again. */
-		return -EAGAIN;
-	}
-
-#ifndef ISP2401
-	if (coefs->hor_coefs.odd_real == NULL ||
-	    coefs->hor_coefs.odd_imag == NULL ||
-	    coefs->hor_coefs.even_real == NULL ||
-	    coefs->hor_coefs.even_imag == NULL ||
-	    coefs->ver_coefs.odd_real == NULL ||
-	    coefs->ver_coefs.odd_imag == NULL ||
-	    coefs->ver_coefs.even_real == NULL ||
-	    coefs->ver_coefs.even_imag == NULL)
-#else
-	if (dvs2_coefs.hor_coefs.odd_real == NULL ||
-	    dvs2_coefs.hor_coefs.odd_imag == NULL ||
-	    dvs2_coefs.hor_coefs.even_real == NULL ||
-	    dvs2_coefs.hor_coefs.even_imag == NULL ||
-	    dvs2_coefs.ver_coefs.odd_real == NULL ||
-	    dvs2_coefs.ver_coefs.odd_imag == NULL ||
-	    dvs2_coefs.ver_coefs.even_real == NULL ||
-	    dvs2_coefs.ver_coefs.even_imag == NULL)
-#endif
-		return -EINVAL;
-
-	if (!css_param->dvs2_coeff) {
-		/* DIS coefficients. */
-		css_param->dvs2_coeff = ia_css_dvs2_coefficients_allocate(cur);
-		if (!css_param->dvs2_coeff)
-			return -ENOMEM;
-	}
-
-	dvs_hor_coef_bytes = asd->params.dvs_hor_coef_bytes;
-	dvs_ver_coef_bytes = asd->params.dvs_ver_coef_bytes;
-	if (copy_from_compatible(css_param->dvs2_coeff->hor_coefs.odd_real,
-#ifndef ISP2401
-	    coefs->hor_coefs.odd_real, dvs_hor_coef_bytes, from_user) ||
-#else
-	    dvs2_coefs.hor_coefs.odd_real, dvs_hor_coef_bytes, from_user) ||
-#endif
-	    copy_from_compatible(css_param->dvs2_coeff->hor_coefs.odd_imag,
-#ifndef ISP2401
-	    coefs->hor_coefs.odd_imag, dvs_hor_coef_bytes, from_user) ||
-#else
-	    dvs2_coefs.hor_coefs.odd_imag, dvs_hor_coef_bytes, from_user) ||
-#endif
-	    copy_from_compatible(css_param->dvs2_coeff->hor_coefs.even_real,
-#ifndef ISP2401
-	    coefs->hor_coefs.even_real, dvs_hor_coef_bytes, from_user) ||
-#else
-	    dvs2_coefs.hor_coefs.even_real, dvs_hor_coef_bytes, from_user) ||
-#endif
-	    copy_from_compatible(css_param->dvs2_coeff->hor_coefs.even_imag,
-#ifndef ISP2401
-	    coefs->hor_coefs.even_imag, dvs_hor_coef_bytes, from_user) ||
-#else
-	    dvs2_coefs.hor_coefs.even_imag, dvs_hor_coef_bytes, from_user) ||
-#endif
-	    copy_from_compatible(css_param->dvs2_coeff->ver_coefs.odd_real,
-#ifndef ISP2401
-	    coefs->ver_coefs.odd_real, dvs_ver_coef_bytes, from_user) ||
-#else
-	    dvs2_coefs.ver_coefs.odd_real, dvs_ver_coef_bytes, from_user) ||
-#endif
-	    copy_from_compatible(css_param->dvs2_coeff->ver_coefs.odd_imag,
-#ifndef ISP2401
-	    coefs->ver_coefs.odd_imag, dvs_ver_coef_bytes, from_user) ||
-#else
-	    dvs2_coefs.ver_coefs.odd_imag, dvs_ver_coef_bytes, from_user) ||
-#endif
-	    copy_from_compatible(css_param->dvs2_coeff->ver_coefs.even_real,
-#ifndef ISP2401
-	    coefs->ver_coefs.even_real, dvs_ver_coef_bytes, from_user) ||
-#else
-	    dvs2_coefs.ver_coefs.even_real, dvs_ver_coef_bytes, from_user) ||
-#endif
-	    copy_from_compatible(css_param->dvs2_coeff->ver_coefs.even_imag,
-#ifndef ISP2401
-	    coefs->ver_coefs.even_imag, dvs_ver_coef_bytes, from_user)) {
-#else
-	    dvs2_coefs.ver_coefs.even_imag, dvs_ver_coef_bytes, from_user)) {
-#endif
-		ia_css_dvs2_coefficients_free(css_param->dvs2_coeff);
-		css_param->dvs2_coeff = NULL;
-		return -EFAULT;
-	}
-
-	css_param->update_flag.dvs2_coefs =
-	    (struct atomisp_dvs2_coefficients *)css_param->dvs2_coeff;
-	return 0;
-}
-
-int atomisp_cp_dvs_6axis_config(struct atomisp_sub_device *asd,
-			struct atomisp_dvs_6axis_config *source_6axis_config,
-			struct atomisp_css_params *css_param,
-			bool from_user)
-{
-	struct atomisp_css_dvs_6axis_config *dvs_6axis_config;
-	struct atomisp_css_dvs_6axis_config *old_6axis_config;
-#ifdef ISP2401
-	struct atomisp_css_dvs_6axis_config t_6axis_config;
-#endif
-	struct ia_css_stream *stream =
-			asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream;
-	struct atomisp_css_dvs_grid_info *dvs_grid_info =
-		atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
-	int ret = -EFAULT;
-
-	if (stream == NULL) {
-		dev_err(asd->isp->dev, "%s: internal error!", __func__);
-		return -EINVAL;
-	}
-
-	if (!source_6axis_config || !dvs_grid_info)
-		return 0;
-
-	if (!dvs_grid_info->enable)
-		return 0;
-
-	if (!from_user && css_param->update_flag.dvs_6axis_config)
-		return 0;
-
-	/* check whether need to reallocate for 6 axis config */
-	old_6axis_config = css_param->dvs_6axis;
-	dvs_6axis_config = old_6axis_config;
-#ifdef ISP2401
-
-	if (copy_from_compatible(&t_6axis_config, source_6axis_config,
-			   sizeof(struct atomisp_dvs_6axis_config),
-			   from_user)) {
-		dev_err(asd->isp->dev, "copy morph table failed!");
-		return -EFAULT;
-	}
-
-#endif
-	if (old_6axis_config &&
-#ifndef ISP2401
-	    (old_6axis_config->width_y != source_6axis_config->width_y ||
-	     old_6axis_config->height_y != source_6axis_config->height_y ||
-	     old_6axis_config->width_uv != source_6axis_config->width_uv ||
-	     old_6axis_config->height_uv != source_6axis_config->height_uv)) {
-#else
-	    (old_6axis_config->width_y != t_6axis_config.width_y ||
-	     old_6axis_config->height_y != t_6axis_config.height_y ||
-	     old_6axis_config->width_uv != t_6axis_config.width_uv ||
-	     old_6axis_config->height_uv != t_6axis_config.height_uv)) {
-#endif
-		ia_css_dvs2_6axis_config_free(css_param->dvs_6axis);
-		css_param->dvs_6axis = NULL;
-
-		dvs_6axis_config = ia_css_dvs2_6axis_config_allocate(stream);
-		if (!dvs_6axis_config)
-			return -ENOMEM;
-	} else if (!dvs_6axis_config) {
-		dvs_6axis_config = ia_css_dvs2_6axis_config_allocate(stream);
-		if (!dvs_6axis_config)
-			return -ENOMEM;
-	}
-
-#ifndef ISP2401
-	dvs_6axis_config->exp_id = source_6axis_config->exp_id;
-#else
-	dvs_6axis_config->exp_id = t_6axis_config.exp_id;
-#endif
-
-	if (copy_from_compatible(dvs_6axis_config->xcoords_y,
-#ifndef ISP2401
-			   source_6axis_config->xcoords_y,
-			   source_6axis_config->width_y *
-			   source_6axis_config->height_y *
-			   sizeof(*source_6axis_config->xcoords_y),
-#else
-			   t_6axis_config.xcoords_y,
-			   t_6axis_config.width_y *
-			   t_6axis_config.height_y *
-			   sizeof(*dvs_6axis_config->xcoords_y),
-#endif
-			   from_user))
-		goto error;
-	if (copy_from_compatible(dvs_6axis_config->ycoords_y,
-#ifndef ISP2401
-			   source_6axis_config->ycoords_y,
-			   source_6axis_config->width_y *
-			   source_6axis_config->height_y *
-			   sizeof(*source_6axis_config->ycoords_y),
-#else
-			   t_6axis_config.ycoords_y,
-			   t_6axis_config.width_y *
-			   t_6axis_config.height_y *
-			   sizeof(*dvs_6axis_config->ycoords_y),
-#endif
-			   from_user))
-		goto error;
-	if (copy_from_compatible(dvs_6axis_config->xcoords_uv,
-#ifndef ISP2401
-			   source_6axis_config->xcoords_uv,
-			   source_6axis_config->width_uv *
-			   source_6axis_config->height_uv *
-			   sizeof(*source_6axis_config->xcoords_uv),
-#else
-			   t_6axis_config.xcoords_uv,
-			   t_6axis_config.width_uv *
-			   t_6axis_config.height_uv *
-			   sizeof(*dvs_6axis_config->xcoords_uv),
-#endif
-			   from_user))
-		goto error;
-	if (copy_from_compatible(dvs_6axis_config->ycoords_uv,
-#ifndef ISP2401
-			   source_6axis_config->ycoords_uv,
-			   source_6axis_config->width_uv *
-			   source_6axis_config->height_uv *
-			   sizeof(*source_6axis_config->ycoords_uv),
-#else
-			   t_6axis_config.ycoords_uv,
-			   t_6axis_config.width_uv *
-			   t_6axis_config.height_uv *
-			   sizeof(*dvs_6axis_config->ycoords_uv),
-#endif
-			   from_user))
-		goto error;
-
-	css_param->dvs_6axis = dvs_6axis_config;
-	css_param->update_flag.dvs_6axis_config =
-		(struct atomisp_dvs_6axis_config *) dvs_6axis_config;
-	return 0;
-
-error:
-	if (dvs_6axis_config)
-		ia_css_dvs2_6axis_config_free(dvs_6axis_config);
-	return ret;
-}
-
-int atomisp_cp_morph_table(struct atomisp_sub_device *asd,
-				struct atomisp_morph_table *source_morph_table,
-				struct atomisp_css_params *css_param,
-				bool from_user)
-{
-	int ret = -EFAULT;
-	unsigned int i;
-	struct atomisp_css_morph_table *morph_table;
-#ifdef ISP2401
-	struct atomisp_css_morph_table mtbl;
-#endif
-	struct atomisp_css_morph_table *old_morph_table;
-
-	if (!source_morph_table)
-		return 0;
-
-	if (!from_user && css_param->update_flag.morph_table)
-		return 0;
-
-	old_morph_table = css_param->morph_table;
-
-#ifdef ISP2401
-	if (copy_from_compatible(&mtbl, source_morph_table,
-				 sizeof(struct atomisp_morph_table),
-				 from_user)) {
-		dev_err(asd->isp->dev, "copy morph table failed!");
-		return -EFAULT;
-	}
-
-#endif
-	morph_table = atomisp_css_morph_table_allocate(
-#ifndef ISP2401
-		source_morph_table->width,
-		source_morph_table->height);
-#else
-		mtbl.width,
-		mtbl.height);
-#endif
-	if (!morph_table)
-		return -ENOMEM;
-
-	for (i = 0; i < CSS_MORPH_TABLE_NUM_PLANES; i++) {
-		if (copy_from_compatible(morph_table->coordinates_x[i],
-			source_morph_table->coordinates_x[i],
-#ifndef ISP2401
-			source_morph_table->height * source_morph_table->width *
-			sizeof(*source_morph_table->coordinates_x[i]),
-#else
-			mtbl.height * mtbl.width *
-			sizeof(*morph_table->coordinates_x[i]),
-#endif
-			from_user))
-			goto error;
-
-		if (copy_from_compatible(morph_table->coordinates_y[i],
-			source_morph_table->coordinates_y[i],
-#ifndef ISP2401
-			source_morph_table->height * source_morph_table->width *
-			sizeof(*source_morph_table->coordinates_y[i]),
-#else
-			mtbl.height * mtbl.width *
-			sizeof(*morph_table->coordinates_y[i]),
-#endif
-			from_user))
-			goto error;
-	}
-
-	css_param->morph_table = morph_table;
-	if (old_morph_table)
-		atomisp_css_morph_table_free(old_morph_table);
-	css_param->update_flag.morph_table =
-		(struct atomisp_morph_table *) morph_table;
-	return 0;
-
-error:
-	if (morph_table)
-		atomisp_css_morph_table_free(morph_table);
-	return ret;
-}
-
-int atomisp_makeup_css_parameters(struct atomisp_sub_device *asd,
-				  struct atomisp_parameters *arg,
-				  struct atomisp_css_params *css_param)
-{
-	int ret;
-
-	ret = atomisp_cp_general_isp_parameters(asd, arg, css_param, false);
-	if (ret)
-		return ret;
-	ret = atomisp_cp_lsc_table(asd, arg->shading_table, css_param, false);
-	if (ret)
-		return ret;
-	ret = atomisp_cp_morph_table(asd, arg->morph_table, css_param, false);
-	if (ret)
-		return ret;
-	ret = atomisp_css_cp_dvs2_coefs(asd,
-		(struct ia_css_dvs2_coefficients *) arg->dvs2_coefs,
-		css_param, false);
-	if (ret)
-		return ret;
-	ret = atomisp_cp_dvs_6axis_config(asd, arg->dvs_6axis_config,
-					  css_param, false);
-	return ret;
-}
-
-void atomisp_free_css_parameters(struct atomisp_css_params *css_param)
-{
-	if (css_param->dvs_6axis) {
-		ia_css_dvs2_6axis_config_free(css_param->dvs_6axis);
-		css_param->dvs_6axis = NULL;
-	}
-	if (css_param->dvs2_coeff) {
-		ia_css_dvs2_coefficients_free(css_param->dvs2_coeff);
-		css_param->dvs2_coeff = NULL;
-	}
-	if (css_param->shading_table) {
-		ia_css_shading_table_free(css_param->shading_table);
-		css_param->shading_table = NULL;
-	}
-	if (css_param->morph_table) {
-		ia_css_morph_table_free(css_param->morph_table);
-		css_param->morph_table = NULL;
-	}
-}
-
-/*
- * Check parameter queue list and buffer queue list to find out if matched items
- * and then set parameter to CSS and enqueue buffer to CSS.
- * Of course, if the buffer in buffer waiting list is not bound to a per-frame
- * parameter, it will be enqueued into CSS as long as the per-frame setting
- * buffers before it get enqueued.
- */
-void atomisp_handle_parameter_and_buffer(struct atomisp_video_pipe *pipe)
-{
-	struct atomisp_sub_device *asd = pipe->asd;
-	struct videobuf_buffer *vb = NULL, *vb_tmp;
-	struct atomisp_css_params_with_list *param = NULL, *param_tmp;
-	struct videobuf_vmalloc_memory *vm_mem = NULL;
-	unsigned long irqflags;
-	bool need_to_enqueue_buffer = false;
-
-	if (atomisp_is_vf_pipe(pipe))
-		return;
-
-	/*
-	 * CSS/FW requires set parameter and enqueue buffer happen after ISP
-	 * is streamon.
-	 */
-	if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
-		return;
-
-	if (list_empty(&pipe->per_frame_params) ||
-	    list_empty(&pipe->buffers_waiting_for_param))
-		return;
-
-	list_for_each_entry_safe(vb, vb_tmp,
-			&pipe->buffers_waiting_for_param, queue) {
-		if (pipe->frame_request_config_id[vb->i]) {
-			list_for_each_entry_safe(param, param_tmp,
-				&pipe->per_frame_params, list) {
-				if (pipe->frame_request_config_id[vb->i] !=
-				    param->params.isp_config_id)
-					continue;
-
-				list_del(&param->list);
-				list_del(&vb->queue);
-				/*
-				 * clear the request config id as the buffer
-				 * will be handled and enqueued into CSS soon
-				 */
-				pipe->frame_request_config_id[vb->i] = 0;
-				pipe->frame_params[vb->i] = param;
-				vm_mem = vb->priv;
-				BUG_ON(!vm_mem);
-				break;
-			}
-
-			if (vm_mem) {
-				spin_lock_irqsave(&pipe->irq_lock, irqflags);
-				list_add_tail(&vb->queue, &pipe->activeq);
-				spin_unlock_irqrestore(&pipe->irq_lock, irqflags);
-				vm_mem = NULL;
-				need_to_enqueue_buffer = true;
-			} else {
-				/* The is the end, stop further loop */
-				break;
-			}
-		} else {
-			list_del(&vb->queue);
-			pipe->frame_params[vb->i] = NULL;
-			spin_lock_irqsave(&pipe->irq_lock, irqflags);
-			list_add_tail(&vb->queue, &pipe->activeq);
-			spin_unlock_irqrestore(&pipe->irq_lock, irqflags);
-			need_to_enqueue_buffer = true;
-		}
-	}
-
-	if (need_to_enqueue_buffer) {
-		atomisp_qbuffers_to_css(asd);
-#ifndef ISP2401
-		if (!atomisp_is_wdt_running(asd) && atomisp_buffers_queued(asd))
-			atomisp_wdt_start(asd);
-#else
-		if (atomisp_buffers_queued_pipe(pipe)) {
-			if (!atomisp_is_wdt_running(pipe))
-				atomisp_wdt_start(pipe);
-			else
-				atomisp_wdt_refresh_pipe(pipe,
-					ATOMISP_WDT_KEEP_CURRENT_DELAY);
-		}
-#endif
-	}
-}
-
-/*
-* Function to configure ISP parameters
-*/
-int atomisp_set_parameters(struct video_device *vdev,
-			struct atomisp_parameters *arg)
-{
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-	struct atomisp_sub_device *asd = pipe->asd;
-	struct atomisp_css_params_with_list *param = NULL;
-	struct atomisp_css_params *css_param = &asd->params.css_param;
-	int ret;
-
-	if (asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream == NULL) {
-		dev_err(asd->isp->dev, "%s: internal error!\n", __func__);
-		return -EINVAL;
-	}
-
-	dev_dbg(asd->isp->dev, "%s: set parameter(per_frame_setting %d) for asd%d with isp_config_id %d of %s\n",
-		__func__, arg->per_frame_setting, asd->index,
-		arg->isp_config_id, vdev->name);
-#ifdef ISP2401
-
-	if (atomisp_is_vf_pipe(pipe) && arg->per_frame_setting) {
-		dev_err(asd->isp->dev, "%s: vf pipe not support per_frame_setting",
-			__func__);
-		return -EINVAL;
-	}
-
-#endif
-	if (arg->per_frame_setting && !atomisp_is_vf_pipe(pipe)) {
-		/*
-		 * Per-frame setting enabled, we allocate a new paramter
-		 * buffer to cache the parameters and only when frame buffers
-		 * are ready, the parameters will be set to CSS.
-		 * per-frame setting only works for the main output frame.
-		 */
-		param = kvzalloc(sizeof(*param), GFP_KERNEL);
-		if (!param) {
-			dev_err(asd->isp->dev, "%s: failed to alloc params buffer\n",
-				__func__);
-			return -ENOMEM;
-		}
-		css_param = &param->params;
-	}
-
-	ret = atomisp_cp_general_isp_parameters(asd, arg, css_param, true);
-	if (ret)
-		goto apply_parameter_failed;
-
-	ret = atomisp_cp_lsc_table(asd, arg->shading_table, css_param, true);
-	if (ret)
-		goto apply_parameter_failed;
-
-	ret = atomisp_cp_morph_table(asd, arg->morph_table, css_param, true);
-	if (ret)
-		goto apply_parameter_failed;
-
-	ret = atomisp_css_cp_dvs2_coefs(asd,
-		(struct ia_css_dvs2_coefficients *) arg->dvs2_coefs,
-		css_param, true);
-	if (ret)
-		goto apply_parameter_failed;
-
-	ret = atomisp_cp_dvs_6axis_config(asd, arg->dvs_6axis_config,
-					  css_param, true);
-	if (ret)
-		goto apply_parameter_failed;
-
-	if (!(arg->per_frame_setting && !atomisp_is_vf_pipe(pipe))) {
-		/* indicate to CSS that we have parameters to be updated */
-		asd->params.css_update_params_needed = true;
-	} else {
-		list_add_tail(&param->list, &pipe->per_frame_params);
-		atomisp_handle_parameter_and_buffer(pipe);
-	}
-
-	return 0;
-
-apply_parameter_failed:
-	if (css_param)
-		atomisp_free_css_parameters(css_param);
-	if (param)
-		kvfree(param);
-
-	return ret;
-}
-
-/*
- * Function to set/get isp parameters to isp
- */
-int atomisp_param(struct atomisp_sub_device *asd, int flag,
-		  struct atomisp_parm *config)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct ia_css_pipe_config *vp_cfg =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
-		pipe_configs[IA_CSS_PIPE_ID_VIDEO];
-
-	/* Read parameter for 3A binary info */
-	if (flag == 0) {
-		struct atomisp_css_dvs_grid_info *dvs_grid_info =
-			atomisp_css_get_dvs_grid_info(
-				&asd->params.curr_grid_info);
-
-		if (&config->info == NULL) {
-			dev_err(isp->dev, "ERROR: NULL pointer in grid_info\n");
-			return -EINVAL;
-		}
-		atomisp_curr_user_grid_info(asd, &config->info);
-
-		/* We always return the resolution and stride even if there is
-		 * no valid metadata. This allows the caller to get the
-		 * information needed to allocate user-space buffers. */
-		config->metadata_config.metadata_height = asd->
-			stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream_info.
-			metadata_info.resolution.height;
-		config->metadata_config.metadata_stride = asd->
-			stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream_info.
-			metadata_info.stride;
-
-		/* update dvs grid info */
-		if (dvs_grid_info)
-			memcpy(&config->dvs_grid,
-				dvs_grid_info,
-				sizeof(struct atomisp_css_dvs_grid_info));
-
-		if (asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO) {
-			config->dvs_envelop.width = 0;
-			config->dvs_envelop.height = 0;
-			return 0;
-		}
-
-		/* update dvs envelop info */
-		if (!asd->continuous_mode->val) {
-			config->dvs_envelop.width = vp_cfg->dvs_envelope.width;
-			config->dvs_envelop.height =
-					vp_cfg->dvs_envelope.height;
-		} else {
-			unsigned int dvs_w, dvs_h, dvs_w_max, dvs_h_max;
-
-			dvs_w = vp_cfg->bayer_ds_out_res.width -
-				vp_cfg->output_info[0].res.width;
-			dvs_h = vp_cfg->bayer_ds_out_res.height -
-				vp_cfg->output_info[0].res.height;
-			dvs_w_max = rounddown(
-					vp_cfg->output_info[0].res.width / 5,
-					ATOM_ISP_STEP_WIDTH);
-			dvs_h_max = rounddown(
-					vp_cfg->output_info[0].res.height / 5,
-					ATOM_ISP_STEP_HEIGHT);
-
-			config->dvs_envelop.width = min(dvs_w, dvs_w_max);
-			config->dvs_envelop.height = min(dvs_h, dvs_h_max);
-		}
-
-		return 0;
-	}
-
-	memcpy(&asd->params.css_param.wb_config, &config->wb_config,
-	       sizeof(struct atomisp_css_wb_config));
-	memcpy(&asd->params.css_param.ob_config, &config->ob_config,
-	       sizeof(struct atomisp_css_ob_config));
-	memcpy(&asd->params.css_param.dp_config, &config->dp_config,
-	       sizeof(struct atomisp_css_dp_config));
-	memcpy(&asd->params.css_param.de_config, &config->de_config,
-	       sizeof(struct atomisp_css_de_config));
-	memcpy(&asd->params.css_param.dz_config, &config->dz_config,
-	       sizeof(struct atomisp_css_dz_config));
-	memcpy(&asd->params.css_param.ce_config, &config->ce_config,
-	       sizeof(struct atomisp_css_ce_config));
-	memcpy(&asd->params.css_param.nr_config, &config->nr_config,
-	       sizeof(struct atomisp_css_nr_config));
-	memcpy(&asd->params.css_param.ee_config, &config->ee_config,
-	       sizeof(struct atomisp_css_ee_config));
-	memcpy(&asd->params.css_param.tnr_config, &config->tnr_config,
-	       sizeof(struct atomisp_css_tnr_config));
-
-	if (asd->params.color_effect == V4L2_COLORFX_NEGATIVE) {
-		asd->params.css_param.cc_config.matrix[3] = -config->cc_config.matrix[3];
-		asd->params.css_param.cc_config.matrix[4] = -config->cc_config.matrix[4];
-		asd->params.css_param.cc_config.matrix[5] = -config->cc_config.matrix[5];
-		asd->params.css_param.cc_config.matrix[6] = -config->cc_config.matrix[6];
-		asd->params.css_param.cc_config.matrix[7] = -config->cc_config.matrix[7];
-		asd->params.css_param.cc_config.matrix[8] = -config->cc_config.matrix[8];
-	}
-
-	if (asd->params.color_effect != V4L2_COLORFX_SEPIA &&
-	    asd->params.color_effect != V4L2_COLORFX_BW) {
-		memcpy(&asd->params.css_param.cc_config, &config->cc_config,
-		       sizeof(struct atomisp_css_cc_config));
-		atomisp_css_set_cc_config(asd, &asd->params.css_param.cc_config);
-	}
-
-	atomisp_css_set_wb_config(asd, &asd->params.css_param.wb_config);
-	atomisp_css_set_ob_config(asd, &asd->params.css_param.ob_config);
-	atomisp_css_set_de_config(asd, &asd->params.css_param.de_config);
-	atomisp_css_set_dz_config(asd, &asd->params.css_param.dz_config);
-	atomisp_css_set_ce_config(asd, &asd->params.css_param.ce_config);
-	atomisp_css_set_dp_config(asd, &asd->params.css_param.dp_config);
-	atomisp_css_set_nr_config(asd, &asd->params.css_param.nr_config);
-	atomisp_css_set_ee_config(asd, &asd->params.css_param.ee_config);
-	atomisp_css_set_tnr_config(asd, &asd->params.css_param.tnr_config);
-	asd->params.css_update_params_needed = true;
-
-	return 0;
-}
-
-/*
- * Function to configure color effect of the image
- */
-int atomisp_color_effect(struct atomisp_sub_device *asd, int flag,
-			 __s32 *effect)
-{
-	struct atomisp_css_cc_config *cc_config = NULL;
-	struct atomisp_css_macc_table *macc_table = NULL;
-	struct atomisp_css_ctc_table *ctc_table = NULL;
-	int ret = 0;
-	struct v4l2_control control;
-	struct atomisp_device *isp = asd->isp;
-
-	if (flag == 0) {
-		*effect = asd->params.color_effect;
-		return 0;
-	}
-
-
-	control.id = V4L2_CID_COLORFX;
-	control.value = *effect;
-	ret =
-	    v4l2_s_ctrl(NULL, isp->inputs[asd->input_curr].camera->ctrl_handler,
-			&control);
-	/*
-	 * if set color effect to sensor successfully, return
-	 * 0 directly.
-	 */
-	if (!ret) {
-		asd->params.color_effect = (u32)*effect;
-		return 0;
-	}
-
-	if (*effect == asd->params.color_effect)
-		return 0;
-
-	/*
-	 * isp_subdev->params.macc_en should be set to false.
-	 */
-	asd->params.macc_en = false;
-
-	switch (*effect) {
-	case V4L2_COLORFX_NONE:
-		macc_table = &asd->params.css_param.macc_table;
-		asd->params.macc_en = true;
-		break;
-	case V4L2_COLORFX_SEPIA:
-		cc_config = &sepia_cc_config;
-		break;
-	case V4L2_COLORFX_NEGATIVE:
-		cc_config = &nega_cc_config;
-		break;
-	case V4L2_COLORFX_BW:
-		cc_config = &mono_cc_config;
-		break;
-	case V4L2_COLORFX_SKY_BLUE:
-		macc_table = &blue_macc_table;
-		asd->params.macc_en = true;
-		break;
-	case V4L2_COLORFX_GRASS_GREEN:
-		macc_table = &green_macc_table;
-		asd->params.macc_en = true;
-		break;
-	case V4L2_COLORFX_SKIN_WHITEN_LOW:
-		macc_table = &skin_low_macc_table;
-		asd->params.macc_en = true;
-		break;
-	case V4L2_COLORFX_SKIN_WHITEN:
-		macc_table = &skin_medium_macc_table;
-		asd->params.macc_en = true;
-		break;
-	case V4L2_COLORFX_SKIN_WHITEN_HIGH:
-		macc_table = &skin_high_macc_table;
-		asd->params.macc_en = true;
-		break;
-	case V4L2_COLORFX_VIVID:
-		ctc_table = &vivid_ctc_table;
-		break;
-	default:
-		return -EINVAL;
-	}
-	atomisp_update_capture_mode(asd);
-
-	if (cc_config)
-		atomisp_css_set_cc_config(asd, cc_config);
-	if (macc_table)
-		atomisp_css_set_macc_table(asd, macc_table);
-	if (ctc_table)
-		atomisp_css_set_ctc_table(asd, ctc_table);
-	asd->params.color_effect = (u32)*effect;
-	asd->params.css_update_params_needed = true;
-	return 0;
-}
-
-/*
- * Function to configure bad pixel correction
- */
-int atomisp_bad_pixel(struct atomisp_sub_device *asd, int flag,
-		      __s32 *value)
-{
-
-	if (flag == 0) {
-		*value = asd->params.bad_pixel_en;
-		return 0;
-	}
-	asd->params.bad_pixel_en = !!*value;
-
-	return 0;
-}
-
-/*
- * Function to configure bad pixel correction params
- */
-int atomisp_bad_pixel_param(struct atomisp_sub_device *asd, int flag,
-			    struct atomisp_dp_config *config)
-{
-	if (flag == 0) {
-		/* Get bad pixel from current setup */
-		if (atomisp_css_get_dp_config(asd, config))
-			return -EINVAL;
-	} else {
-		/* Set bad pixel to isp parameters */
-		memcpy(&asd->params.css_param.dp_config, config,
-			sizeof(asd->params.css_param.dp_config));
-		atomisp_css_set_dp_config(asd, &asd->params.css_param.dp_config);
-		asd->params.css_update_params_needed = true;
-	}
-
-	return 0;
-}
-
-/*
- * Function to enable/disable video image stablization
- */
-int atomisp_video_stable(struct atomisp_sub_device *asd, int flag,
-			 __s32 *value)
-{
-	if (flag == 0)
-		*value = asd->params.video_dis_en;
-	else
-		asd->params.video_dis_en = !!*value;
-
-	return 0;
-}
-
-/*
- * Function to configure fixed pattern noise
- */
-int atomisp_fixed_pattern(struct atomisp_sub_device *asd, int flag,
-			  __s32 *value)
-{
-
-	if (flag == 0) {
-		*value = asd->params.fpn_en;
-		return 0;
-	}
-
-	if (*value == 0) {
-		asd->params.fpn_en = false;
-		return 0;
-	}
-
-	/* Add function to get black from from sensor with shutter off */
-	return 0;
-}
-
-static unsigned int
-atomisp_bytesperline_to_padded_width(unsigned int bytesperline,
-				     enum atomisp_css_frame_format format)
-{
-	switch (format) {
-	case CSS_FRAME_FORMAT_UYVY:
-	case CSS_FRAME_FORMAT_YUYV:
-	case CSS_FRAME_FORMAT_RAW:
-	case CSS_FRAME_FORMAT_RGB565:
-		return bytesperline/2;
-	case CSS_FRAME_FORMAT_RGBA888:
-		return bytesperline/4;
-	/* The following cases could be removed, but we leave them
-	   in to document the formats that are included. */
-	case CSS_FRAME_FORMAT_NV11:
-	case CSS_FRAME_FORMAT_NV12:
-	case CSS_FRAME_FORMAT_NV16:
-	case CSS_FRAME_FORMAT_NV21:
-	case CSS_FRAME_FORMAT_NV61:
-	case CSS_FRAME_FORMAT_YV12:
-	case CSS_FRAME_FORMAT_YV16:
-	case CSS_FRAME_FORMAT_YUV420:
-	case CSS_FRAME_FORMAT_YUV420_16:
-	case CSS_FRAME_FORMAT_YUV422:
-	case CSS_FRAME_FORMAT_YUV422_16:
-	case CSS_FRAME_FORMAT_YUV444:
-	case CSS_FRAME_FORMAT_YUV_LINE:
-	case CSS_FRAME_FORMAT_PLANAR_RGB888:
-	case CSS_FRAME_FORMAT_QPLANE6:
-	case CSS_FRAME_FORMAT_BINARY_8:
-	default:
-		return bytesperline;
-	}
-}
-
-static int
-atomisp_v4l2_framebuffer_to_css_frame(const struct v4l2_framebuffer *arg,
-					 struct atomisp_css_frame **result)
-{
-	struct atomisp_css_frame *res = NULL;
-	unsigned int padded_width;
-	enum atomisp_css_frame_format sh_format;
-	char *tmp_buf = NULL;
-	int ret = 0;
-
-	sh_format = v4l2_fmt_to_sh_fmt(arg->fmt.pixelformat);
-	padded_width = atomisp_bytesperline_to_padded_width(
-					arg->fmt.bytesperline, sh_format);
-
-	/* Note: the padded width on an atomisp_css_frame is in elements, not in
-	   bytes. The RAW frame we use here should always be a 16bit RAW
-	   frame. This is why we bytesperline/2 is equal to the padded with */
-	if (atomisp_css_frame_allocate(&res, arg->fmt.width, arg->fmt.height,
-				  sh_format, padded_width, 0)) {
-		ret = -ENOMEM;
-		goto err;
-	}
-
-	tmp_buf = vmalloc(arg->fmt.sizeimage);
-	if (!tmp_buf) {
-		ret = -ENOMEM;
-		goto err;
-	}
-	if (copy_from_user(tmp_buf, (void __user __force *)arg->base,
-			   arg->fmt.sizeimage)) {
-		ret = -EFAULT;
-		goto err;
-	}
-
-	if (hmm_store(res->data, tmp_buf, arg->fmt.sizeimage)) {
-		ret = -EINVAL;
-		goto err;
-	}
-
-err:
-	if (ret && res)
-		atomisp_css_frame_free(res);
-	if (tmp_buf)
-		vfree(tmp_buf);
-	if (ret == 0)
-		*result = res;
-	return ret;
-}
-
-/*
- * Function to configure fixed pattern noise table
- */
-int atomisp_fixed_pattern_table(struct atomisp_sub_device *asd,
-				struct v4l2_framebuffer *arg)
-{
-	struct atomisp_css_frame *raw_black_frame = NULL;
-	int ret;
-
-	if (arg == NULL)
-		return -EINVAL;
-
-	ret = atomisp_v4l2_framebuffer_to_css_frame(arg, &raw_black_frame);
-	if (ret)
-		return ret;
-	if (atomisp_css_set_black_frame(asd, raw_black_frame))
-		ret = -ENOMEM;
-
-	atomisp_css_frame_free(raw_black_frame);
-	return ret;
-}
-
-/*
- * Function to configure false color correction
- */
-int atomisp_false_color(struct atomisp_sub_device *asd, int flag,
-			__s32 *value)
-{
-	/* Get nr config from current setup */
-	if (flag == 0) {
-		*value = asd->params.false_color;
-		return 0;
-	}
-
-	/* Set nr config to isp parameters */
-	if (*value) {
-		atomisp_css_set_default_de_config(asd);
-	} else {
-		asd->params.css_param.de_config.pixelnoise = 0;
-		atomisp_css_set_de_config(asd, &asd->params.css_param.de_config);
-	}
-	asd->params.css_update_params_needed = true;
-	asd->params.false_color = *value;
-	return 0;
-}
-
-/*
- * Function to configure bad pixel correction params
- */
-int atomisp_false_color_param(struct atomisp_sub_device *asd, int flag,
-			      struct atomisp_de_config *config)
-{
-	if (flag == 0) {
-		/* Get false color from current setup */
-		if (atomisp_css_get_de_config(asd, config))
-			return -EINVAL;
-	} else {
-		/* Set false color to isp parameters */
-		memcpy(&asd->params.css_param.de_config, config,
-		       sizeof(asd->params.css_param.de_config));
-		atomisp_css_set_de_config(asd, &asd->params.css_param.de_config);
-		asd->params.css_update_params_needed = true;
-	}
-
-	return 0;
-}
-
-/*
- * Function to configure white balance params
- */
-int atomisp_white_balance_param(struct atomisp_sub_device *asd, int flag,
-	struct atomisp_wb_config *config)
-{
-	if (flag == 0) {
-		/* Get white balance from current setup */
-		if (atomisp_css_get_wb_config(asd, config))
-			return -EINVAL;
-	} else {
-		/* Set white balance to isp parameters */
-		memcpy(&asd->params.css_param.wb_config, config,
-		       sizeof(asd->params.css_param.wb_config));
-		atomisp_css_set_wb_config(asd, &asd->params.css_param.wb_config);
-		asd->params.css_update_params_needed = true;
-	}
-
-	return 0;
-}
-
-int atomisp_3a_config_param(struct atomisp_sub_device *asd, int flag,
-			    struct atomisp_3a_config *config)
-{
-	struct atomisp_device *isp = asd->isp;
-
-	dev_dbg(isp->dev, ">%s %d\n", __func__, flag);
-
-	if (flag == 0) {
-		/* Get white balance from current setup */
-		if (atomisp_css_get_3a_config(asd, config))
-			return -EINVAL;
-	} else {
-		/* Set white balance to isp parameters */
-		memcpy(&asd->params.css_param.s3a_config, config,
-		       sizeof(asd->params.css_param.s3a_config));
-		atomisp_css_set_3a_config(asd, &asd->params.css_param.s3a_config);
-		asd->params.css_update_params_needed = true;
-	}
-
-	dev_dbg(isp->dev, "<%s %d\n", __func__, flag);
-	return 0;
-}
-
-/*
- * Function to setup digital zoom
- */
-int atomisp_digital_zoom(struct atomisp_sub_device *asd, int flag,
-			 __s32 *value)
-{
-	u32 zoom;
-	struct atomisp_device *isp = asd->isp;
-
-	unsigned int max_zoom = MRFLD_MAX_ZOOM_FACTOR;
-
-	if (flag == 0) {
-		atomisp_css_get_zoom_factor(asd, &zoom);
-		*value = max_zoom - zoom;
-	} else {
-		if (*value < 0)
-			return -EINVAL;
-
-		zoom = max_zoom - min_t(u32, max_zoom - 1, *value);
-		atomisp_css_set_zoom_factor(asd, zoom);
-
-		dev_dbg(isp->dev, "%s, zoom: %d\n", __func__, zoom);
-		asd->params.css_update_params_needed = true;
-	}
-
-	return 0;
-}
-
-/*
- * Function to get sensor specific info for current resolution,
- * which will be used for auto exposure conversion.
- */
-int atomisp_get_sensor_mode_data(struct atomisp_sub_device *asd,
-				 struct atomisp_sensor_mode_data *config)
-{
-	struct camera_mipi_info *mipi_info;
-	struct atomisp_device *isp = asd->isp;
-
-	mipi_info = atomisp_to_sensor_mipi_info(
-		isp->inputs[asd->input_curr].camera);
-	if (mipi_info == NULL)
-		return -EINVAL;
-
-	memcpy(config, &mipi_info->data, sizeof(*config));
-	return 0;
-}
-
-int atomisp_get_fmt(struct video_device *vdev, struct v4l2_format *f)
-{
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-
-	f->fmt.pix = pipe->pix;
-
-	return 0;
-}
-
-static void __atomisp_update_stream_env(struct atomisp_sub_device *asd,
-	uint16_t stream_index, struct atomisp_input_stream_info *stream_info)
-{
-	int i;
-
-#if defined(ISP2401_NEW_INPUT_SYSTEM)
-	/* assign virtual channel id return from sensor driver query */
-	asd->stream_env[stream_index].ch_id = stream_info->ch_id;
-#endif
-	asd->stream_env[stream_index].isys_configs = stream_info->isys_configs;
-	for (i = 0; i < stream_info->isys_configs; i++) {
-		asd->stream_env[stream_index].isys_info[i].input_format =
-			stream_info->isys_info[i].input_format;
-		asd->stream_env[stream_index].isys_info[i].width =
-			stream_info->isys_info[i].width;
-		asd->stream_env[stream_index].isys_info[i].height =
-			stream_info->isys_info[i].height;
-	}
-}
-
-static void __atomisp_init_stream_info(uint16_t stream_index,
-		struct atomisp_input_stream_info *stream_info)
-{
-	int i;
-
-	stream_info->enable = 1;
-	stream_info->stream = stream_index;
-	stream_info->ch_id = 0;
-	stream_info->isys_configs = 0;
-	for (i = 0; i < MAX_STREAMS_PER_CHANNEL; i++) {
-		stream_info->isys_info[i].input_format = 0;
-		stream_info->isys_info[i].width = 0;
-		stream_info->isys_info[i].height = 0;
-	}
-}
-
-/* This function looks up the closest available resolution. */
-int atomisp_try_fmt(struct video_device *vdev, struct v4l2_format *f,
-						bool *res_overflow)
-{
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	struct v4l2_subdev_pad_config pad_cfg;
-	struct v4l2_subdev_format format = {
-		.which = V4L2_SUBDEV_FORMAT_TRY,
-	};
-
-	struct v4l2_mbus_framefmt *snr_mbus_fmt = &format.format;
-	const struct atomisp_format_bridge *fmt;
-	struct atomisp_input_stream_info *stream_info =
-	    (struct atomisp_input_stream_info *)snr_mbus_fmt->reserved;
-	uint16_t stream_index;
-	int source_pad = atomisp_subdev_source_pad(vdev);
-	int ret;
-
-	if (isp->inputs[asd->input_curr].camera == NULL)
-		return -EINVAL;
-
-	stream_index = atomisp_source_pad_to_stream_id(asd, source_pad);
-	fmt = atomisp_get_format_bridge(f->fmt.pix.pixelformat);
-	if (fmt == NULL) {
-		dev_err(isp->dev, "unsupported pixelformat!\n");
-		fmt = atomisp_output_fmts;
-	}
-
-#ifdef ISP2401
-	if (f->fmt.pix.width <= 0 || f->fmt.pix.height <= 0)
-		return -EINVAL;
-
-#endif
-	snr_mbus_fmt->code = fmt->mbus_code;
-	snr_mbus_fmt->width = f->fmt.pix.width;
-	snr_mbus_fmt->height = f->fmt.pix.height;
-
-	__atomisp_init_stream_info(stream_index, stream_info);
-
-	dev_dbg(isp->dev, "try_mbus_fmt: asking for %ux%u\n",
-		snr_mbus_fmt->width, snr_mbus_fmt->height);
-
-	ret = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-			       pad, set_fmt, &pad_cfg, &format);
-	if (ret)
-		return ret;
-
-	dev_dbg(isp->dev, "try_mbus_fmt: got %ux%u\n",
-		snr_mbus_fmt->width, snr_mbus_fmt->height);
-
-	fmt = atomisp_get_format_bridge_from_mbus(snr_mbus_fmt->code);
-	if (fmt == NULL) {
-		dev_err(isp->dev, "unknown sensor format 0x%8.8x\n",
-			snr_mbus_fmt->code);
-		return -EINVAL;
-	}
-
-	f->fmt.pix.pixelformat = fmt->pixelformat;
-
-	/*
-	 * If the format is jpeg or custom RAW, then the width and height will
-	 * not satisfy the normal atomisp requirements and no need to check
-	 * the below conditions. So just assign to what is being returned from
-	 * the sensor driver.
-	 */
-	if (f->fmt.pix.pixelformat == V4L2_PIX_FMT_JPEG ||
-	    f->fmt.pix.pixelformat == V4L2_PIX_FMT_CUSTOM_M10MO_RAW) {
-		f->fmt.pix.width = snr_mbus_fmt->width;
-		f->fmt.pix.height = snr_mbus_fmt->height;
-		return 0;
-	}
-
-	if (snr_mbus_fmt->width < f->fmt.pix.width
-	    && snr_mbus_fmt->height < f->fmt.pix.height) {
-		f->fmt.pix.width = snr_mbus_fmt->width;
-		f->fmt.pix.height = snr_mbus_fmt->height;
-		/* Set the flag when resolution requested is
-		 * beyond the max value supported by sensor
-		 */
-		if (res_overflow != NULL)
-			*res_overflow = true;
-	}
-
-	/* app vs isp */
-	f->fmt.pix.width = rounddown(
-		clamp_t(u32, f->fmt.pix.width, ATOM_ISP_MIN_WIDTH,
-			ATOM_ISP_MAX_WIDTH), ATOM_ISP_STEP_WIDTH);
-	f->fmt.pix.height = rounddown(
-		clamp_t(u32, f->fmt.pix.height, ATOM_ISP_MIN_HEIGHT,
-			ATOM_ISP_MAX_HEIGHT), ATOM_ISP_STEP_HEIGHT);
-
-	return 0;
-}
-
-static int
-atomisp_try_fmt_file(struct atomisp_device *isp, struct v4l2_format *f)
-{
-	u32 width = f->fmt.pix.width;
-	u32 height = f->fmt.pix.height;
-	u32 pixelformat = f->fmt.pix.pixelformat;
-	enum v4l2_field field = f->fmt.pix.field;
-	u32 depth;
-
-	if (!atomisp_get_format_bridge(pixelformat)) {
-		dev_err(isp->dev, "Wrong output pixelformat\n");
-		return -EINVAL;
-	}
-
-	depth = get_pixel_depth(pixelformat);
-
-	if (field == V4L2_FIELD_ANY)
-		field = V4L2_FIELD_NONE;
-	else if (field != V4L2_FIELD_NONE) {
-		dev_err(isp->dev, "Wrong output field\n");
-		return -EINVAL;
-	}
-
-	f->fmt.pix.field = field;
-	f->fmt.pix.width = clamp_t(u32,
-				   rounddown(width, (u32)ATOM_ISP_STEP_WIDTH),
-				   ATOM_ISP_MIN_WIDTH, ATOM_ISP_MAX_WIDTH);
-	f->fmt.pix.height = clamp_t(u32, rounddown(height,
-				    (u32)ATOM_ISP_STEP_HEIGHT),
-				    ATOM_ISP_MIN_HEIGHT, ATOM_ISP_MAX_HEIGHT);
-	f->fmt.pix.bytesperline = (width * depth) >> 3;
-
-	return 0;
-}
-
-enum mipi_port_id __get_mipi_port(struct atomisp_device *isp,
-				enum atomisp_camera_port port)
-{
-	switch (port) {
-	case ATOMISP_CAMERA_PORT_PRIMARY:
-		return MIPI_PORT0_ID;
-	case ATOMISP_CAMERA_PORT_SECONDARY:
-		return MIPI_PORT1_ID;
-	case ATOMISP_CAMERA_PORT_TERTIARY:
-		if (MIPI_PORT1_ID + 1 != N_MIPI_PORT_ID)
-			return MIPI_PORT1_ID + 1;
-		/* go through down for else case */
-	default:
-		dev_err(isp->dev, "unsupported port: %d\n", port);
-		return MIPI_PORT0_ID;
-	}
-}
-
-static inline int atomisp_set_sensor_mipi_to_isp(
-				struct atomisp_sub_device *asd,
-				enum atomisp_input_stream_id stream_id,
-				struct camera_mipi_info *mipi_info)
-{
-	struct v4l2_control ctrl;
-	struct atomisp_device *isp = asd->isp;
-	const struct atomisp_in_fmt_conv *fc;
-	int mipi_freq = 0;
-	unsigned int input_format, bayer_order;
-
-	ctrl.id = V4L2_CID_LINK_FREQ;
-	if (v4l2_g_ctrl
-	    (isp->inputs[asd->input_curr].camera->ctrl_handler, &ctrl) == 0)
-		mipi_freq = ctrl.value;
-
-	if (asd->stream_env[stream_id].isys_configs == 1) {
-		input_format =
-			asd->stream_env[stream_id].isys_info[0].input_format;
-		atomisp_css_isys_set_format(asd, stream_id,
-				input_format, IA_CSS_STREAM_DEFAULT_ISYS_STREAM_IDX);
-	} else if (asd->stream_env[stream_id].isys_configs == 2) {
-		atomisp_css_isys_two_stream_cfg_update_stream1(
-				asd, stream_id,
-				asd->stream_env[stream_id].isys_info[0].input_format,
-				asd->stream_env[stream_id].isys_info[0].width,
-				asd->stream_env[stream_id].isys_info[0].height);
-
-		atomisp_css_isys_two_stream_cfg_update_stream2(
-				asd, stream_id,
-				asd->stream_env[stream_id].isys_info[1].input_format,
-				asd->stream_env[stream_id].isys_info[1].width,
-				asd->stream_env[stream_id].isys_info[1].height);
-	}
-
-	/* Compatibility for sensors which provide no media bus code
-	 * in s_mbus_framefmt() nor support pad formats. */
-	if (mipi_info->input_format != -1) {
-		bayer_order = mipi_info->raw_bayer_order;
-
-		/* Input stream config is still needs configured */
-		/* TODO: Check if this is necessary */
-		fc = atomisp_find_in_fmt_conv_by_atomisp_in_fmt(
-						mipi_info->input_format);
-		if (!fc)
-			return -EINVAL;
-		input_format = fc->css_stream_fmt;
-	} else {
-		struct v4l2_mbus_framefmt *sink;
-		sink = atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-					       V4L2_SUBDEV_FORMAT_ACTIVE,
-					       ATOMISP_SUBDEV_PAD_SINK);
-		fc = atomisp_find_in_fmt_conv(sink->code);
-		if (!fc)
-			return -EINVAL;
-		input_format = fc->css_stream_fmt;
-		bayer_order = fc->bayer_order;
-	}
-
-	atomisp_css_input_set_format(asd, stream_id, input_format);
-	atomisp_css_input_set_bayer_order(asd, stream_id, bayer_order);
-
-	fc = atomisp_find_in_fmt_conv_by_atomisp_in_fmt(
-					mipi_info->metadata_format);
-	if (!fc)
-		return -EINVAL;
-	input_format = fc->css_stream_fmt;
-	atomisp_css_input_configure_port(asd,
-				__get_mipi_port(asd->isp, mipi_info->port),
-				mipi_info->num_lanes,
-				0xffff4, mipi_freq,
-				input_format,
-				mipi_info->metadata_width,
-				mipi_info->metadata_height);
-	return 0;
-}
-
-static int __enable_continuous_mode(struct atomisp_sub_device *asd,
-				    bool enable)
-{
-	struct atomisp_device *isp = asd->isp;
-
-	dev_dbg(isp->dev,
-		"continuous mode %d, raw buffers %d, stop preview %d\n",
-		enable, asd->continuous_raw_buffer_size->val,
-		!asd->continuous_viewfinder->val);
-#ifndef ISP2401
-	atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_PRIMARY);
-#else
-	atomisp_update_capture_mode(asd);
-#endif
-	/* in case of ANR, force capture pipe to offline mode */
-	atomisp_css_capture_enable_online(asd, ATOMISP_INPUT_STREAM_GENERAL,
-			asd->params.low_light ? false : !enable);
-	atomisp_css_preview_enable_online(asd, ATOMISP_INPUT_STREAM_GENERAL,
-			!enable);
-	atomisp_css_enable_continuous(asd, enable);
-	atomisp_css_enable_cvf(asd, asd->continuous_viewfinder->val);
-
-	if (atomisp_css_continuous_set_num_raw_frames(asd,
-			asd->continuous_raw_buffer_size->val)) {
-		dev_err(isp->dev, "css_continuous_set_num_raw_frames failed\n");
-		return -EINVAL;
-	}
-
-	if (!enable) {
-		atomisp_css_enable_raw_binning(asd, false);
-		atomisp_css_input_set_two_pixels_per_clock(asd, false);
-	}
-
-	if (isp->inputs[asd->input_curr].type != FILE_INPUT)
-		atomisp_css_input_set_mode(asd, CSS_INPUT_MODE_SENSOR);
-
-	return atomisp_update_run_mode(asd);
-}
-
-static int configure_pp_input_nop(struct atomisp_sub_device *asd,
-				  unsigned int width, unsigned int height)
-{
-	return 0;
-}
-
-static int configure_output_nop(struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format sh_fmt)
-{
-	return 0;
-}
-
-static int get_frame_info_nop(struct atomisp_sub_device *asd,
-			      struct atomisp_css_frame_info *finfo)
-{
-	return 0;
-}
-
-/*
- * Resets CSS parameters that depend on input resolution.
- *
- * Update params like CSS RAW binning, 2ppc mode and pp_input
- * which depend on input size, but are not automatically
- * handled in CSS when the input resolution is changed.
- */
-static int css_input_resolution_changed(struct atomisp_sub_device *asd,
-		struct v4l2_mbus_framefmt *ffmt)
-{
-	struct atomisp_metadata_buf *md_buf = NULL, *_md_buf;
-	unsigned int i;
-
-	dev_dbg(asd->isp->dev, "css_input_resolution_changed to %ux%u\n",
-		ffmt->width, ffmt->height);
-
-#if defined(ISP2401_NEW_INPUT_SYSTEM)
-	atomisp_css_input_set_two_pixels_per_clock(asd, false);
-#else
-	atomisp_css_input_set_two_pixels_per_clock(asd, true);
-#endif
-	if (asd->continuous_mode->val) {
-		/* Note for all checks: ffmt includes pad_w+pad_h */
-		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO ||
-		    (ffmt->width >= 2048 || ffmt->height >= 1536)) {
-			/*
-			 * For preview pipe, enable only if resolution
-			 * is >= 3M for ISP2400.
-			 */
-			atomisp_css_enable_raw_binning(asd, true);
-		}
-	}
-	/*
-	 * If sensor input changed, which means metadata resolution changed
-	 * together. Release all metadata buffers here to let it re-allocated
-	 * next time in reqbufs.
-	 */
-	for (i = 0; i < ATOMISP_METADATA_TYPE_NUM; i++) {
-		list_for_each_entry_safe(md_buf, _md_buf, &asd->metadata[i],
-					 list) {
-			atomisp_css_free_metadata_buffer(md_buf);
-			list_del(&md_buf->list);
-			kfree(md_buf);
-		}
-	}
-	return 0;
-
-	/*
-	 * TODO: atomisp_css_preview_configure_pp_input() not
-	 *       reset due to CSS bug tracked as PSI BZ 115124
-	 */
-}
-
-static int atomisp_set_fmt_to_isp(struct video_device *vdev,
-			struct atomisp_css_frame_info *output_info,
-			struct atomisp_css_frame_info *raw_output_info,
-			struct v4l2_pix_format *pix,
-			unsigned int source_pad)
-{
-	struct camera_mipi_info *mipi_info;
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	const struct atomisp_format_bridge *format;
-	struct v4l2_rect *isp_sink_crop;
-	enum atomisp_css_pipe_id pipe_id;
-	struct v4l2_subdev_fh fh;
-	int (*configure_output)(struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format sh_fmt) =
-							configure_output_nop;
-	int (*get_frame_info)(struct atomisp_sub_device *asd,
-			      struct atomisp_css_frame_info *finfo) =
-							get_frame_info_nop;
-	int (*configure_pp_input)(struct atomisp_sub_device *asd,
-				  unsigned int width, unsigned int height) =
-							configure_pp_input_nop;
-	uint16_t stream_index = atomisp_source_pad_to_stream_id(asd, source_pad);
-	const struct atomisp_in_fmt_conv *fc;
-	int ret;
-
-	v4l2_fh_init(&fh.vfh, vdev);
-
-	isp_sink_crop = atomisp_subdev_get_rect(
-		&asd->subdev, NULL, V4L2_SUBDEV_FORMAT_ACTIVE,
-		ATOMISP_SUBDEV_PAD_SINK, V4L2_SEL_TGT_CROP);
-
-	format = atomisp_get_format_bridge(pix->pixelformat);
-	if (format == NULL)
-		return -EINVAL;
-
-	if (isp->inputs[asd->input_curr].type != TEST_PATTERN &&
-		isp->inputs[asd->input_curr].type != FILE_INPUT) {
-		mipi_info = atomisp_to_sensor_mipi_info(
-			isp->inputs[asd->input_curr].camera);
-		if (!mipi_info) {
-			dev_err(isp->dev, "mipi_info is NULL\n");
-			return -EINVAL;
-		}
-		if (atomisp_set_sensor_mipi_to_isp(asd, stream_index,
-					mipi_info))
-			return -EINVAL;
-		fc = atomisp_find_in_fmt_conv_by_atomisp_in_fmt(
-				mipi_info->input_format);
-		if (!fc)
-			fc = atomisp_find_in_fmt_conv(
-					atomisp_subdev_get_ffmt(&asd->subdev,
-					NULL, V4L2_SUBDEV_FORMAT_ACTIVE,
-					ATOMISP_SUBDEV_PAD_SINK)->code);
-		if (!fc)
-			return -EINVAL;
-		if (format->sh_fmt == CSS_FRAME_FORMAT_RAW &&
-		     raw_output_format_match_input(fc->css_stream_fmt,
-			pix->pixelformat))
-			return -EINVAL;
-	}
-
-	/*
-	 * Configure viewfinder also when vfpp is disabled: the
-	 * CSS still requires viewfinder configuration.
-	 */
-	if (asd->fmt_auto->val ||
-	    asd->vfpp->val != ATOMISP_VFPP_ENABLE) {
-		struct v4l2_rect vf_size = {0};
-		struct v4l2_mbus_framefmt vf_ffmt = {0};
-
-		if (pix->width < 640 || pix->height < 480) {
-			vf_size.width = pix->width;
-			vf_size.height = pix->height;
-		} else {
-			vf_size.width = 640;
-			vf_size.height = 480;
-		}
-
-		/* FIXME: proper format name for this one. See
-		   atomisp_output_fmts[] in atomisp_v4l2.c */
-		vf_ffmt.code = V4L2_MBUS_FMT_CUSTOM_YUV420;
-
-		atomisp_subdev_set_selection(&asd->subdev, fh.pad,
-					     V4L2_SUBDEV_FORMAT_ACTIVE,
-					     ATOMISP_SUBDEV_PAD_SOURCE_VF,
-					     V4L2_SEL_TGT_COMPOSE, 0, &vf_size);
-		atomisp_subdev_set_ffmt(&asd->subdev, fh.pad,
-					V4L2_SUBDEV_FORMAT_ACTIVE,
-					ATOMISP_SUBDEV_PAD_SOURCE_VF, &vf_ffmt);
-		asd->video_out_vf.sh_fmt = CSS_FRAME_FORMAT_NV12;
-
-		if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) {
-			atomisp_css_video_configure_viewfinder(asd,
-				vf_size.width, vf_size.height, 0,
-				asd->video_out_vf.sh_fmt);
-		} else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
-			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW ||
-			    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO)
-				atomisp_css_video_configure_viewfinder(asd,
-					vf_size.width, vf_size.height, 0,
-					asd->video_out_vf.sh_fmt);
-			else
-				atomisp_css_capture_configure_viewfinder(asd,
-					vf_size.width, vf_size.height, 0,
-					asd->video_out_vf.sh_fmt);
-		} else if (source_pad != ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW ||
-			 asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) {
-			atomisp_css_capture_configure_viewfinder(asd,
-				vf_size.width, vf_size.height, 0,
-				asd->video_out_vf.sh_fmt);
-		}
-	}
-
-	if (asd->continuous_mode->val) {
-		ret = __enable_continuous_mode(asd, true);
-		if (ret)
-			return -EINVAL;
-	}
-
-	atomisp_css_input_set_mode(asd, CSS_INPUT_MODE_SENSOR);
-	atomisp_css_disable_vf_pp(asd,
-			asd->vfpp->val != ATOMISP_VFPP_ENABLE);
-
-	/* ISP2401 new input system need to use copy pipe */
-	if (asd->copy_mode) {
-		pipe_id = CSS_PIPE_ID_COPY;
-		atomisp_css_capture_enable_online(asd, stream_index, false);
-	} else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) {
-		/* video same in continuouscapture and online modes */
-		configure_output = atomisp_css_video_configure_output;
-		get_frame_info = atomisp_css_video_get_output_frame_info;
-		pipe_id = CSS_PIPE_ID_VIDEO;
-	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
-		if (!asd->continuous_mode->val) {
-			configure_output = atomisp_css_video_configure_output;
-			get_frame_info =
-				atomisp_css_video_get_output_frame_info;
-			pipe_id = CSS_PIPE_ID_VIDEO;
-		} else {
-			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW ||
-			    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO) {
-				configure_output =
-					atomisp_css_video_configure_output;
-				get_frame_info =
-					atomisp_css_video_get_output_frame_info;
-				configure_pp_input =
-					atomisp_css_video_configure_pp_input;
-				pipe_id = CSS_PIPE_ID_VIDEO;
-			} else {
-				configure_output =
-					atomisp_css_capture_configure_output;
-				get_frame_info =
-					atomisp_css_capture_get_output_frame_info;
-				configure_pp_input =
-					atomisp_css_capture_configure_pp_input;
-				pipe_id = CSS_PIPE_ID_CAPTURE;
-
-				atomisp_update_capture_mode(asd);
-				atomisp_css_capture_enable_online(asd, stream_index, false);
-			}
-		}
-	} else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW) {
-		configure_output = atomisp_css_preview_configure_output;
-		get_frame_info = atomisp_css_preview_get_output_frame_info;
-		configure_pp_input = atomisp_css_preview_configure_pp_input;
-		pipe_id = CSS_PIPE_ID_PREVIEW;
-	} else {
-		/* CSS doesn't support low light mode on SOC cameras, so disable
-		 * it. FIXME: if this is done elsewhere, it gives corrupted
-		 * colors into thumbnail image.
-		 */
-		if (isp->inputs[asd->input_curr].type == SOC_CAMERA)
-			asd->params.low_light = false;
-
-		if (format->sh_fmt == CSS_FRAME_FORMAT_RAW) {
-			atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_RAW);
-			atomisp_css_enable_dz(asd, false);
-		} else {
-			atomisp_update_capture_mode(asd);
-		}
-
-		if (!asd->continuous_mode->val)
-			/* in case of ANR, force capture pipe to offline mode */
-			atomisp_css_capture_enable_online(asd, stream_index,
-					asd->params.low_light ?
-					false : asd->params.online_process);
-
-		configure_output = atomisp_css_capture_configure_output;
-		get_frame_info = atomisp_css_capture_get_output_frame_info;
-		configure_pp_input = atomisp_css_capture_configure_pp_input;
-		pipe_id = CSS_PIPE_ID_CAPTURE;
-
-		if (!asd->params.online_process &&
-		    !asd->continuous_mode->val) {
-			ret = atomisp_css_capture_get_output_raw_frame_info(asd,
-							raw_output_info);
-			if (ret)
-				return ret;
-		}
-		if (!asd->continuous_mode->val && asd->run_mode->val
-		    != ATOMISP_RUN_MODE_STILL_CAPTURE) {
-			dev_err(isp->dev,
-				    "Need to set the running mode first\n");
-			asd->run_mode->val = ATOMISP_RUN_MODE_STILL_CAPTURE;
-		}
-	}
-
-	/*
-	 * to SOC camera, use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		pipe_id = CSS_PIPE_ID_YUVPP;
-
-	if (asd->copy_mode)
-		ret = atomisp_css_copy_configure_output(asd, stream_index,
-				pix->width, pix->height,
-				format->planar ? pix->bytesperline :
-					pix->bytesperline * 8 / format->depth,
-				format->sh_fmt);
-	else
-		ret = configure_output(asd, pix->width, pix->height,
-				       format->planar ? pix->bytesperline :
-				       pix->bytesperline * 8 / format->depth,
-				       format->sh_fmt);
-	if (ret) {
-		dev_err(isp->dev, "configure_output %ux%u, format %8.8x\n",
-			pix->width, pix->height, format->sh_fmt);
-		return -EINVAL;
-	}
-
-	if (asd->continuous_mode->val &&
-	    (configure_pp_input == atomisp_css_preview_configure_pp_input ||
-	     configure_pp_input == atomisp_css_video_configure_pp_input)) {
-		/* for isp 2.2, configure pp input is available for continuous
-		 * mode */
-		ret = configure_pp_input(asd, isp_sink_crop->width,
-					 isp_sink_crop->height);
-		if (ret) {
-			dev_err(isp->dev, "configure_pp_input %ux%u\n",
-				isp_sink_crop->width,
-				isp_sink_crop->height);
-			return -EINVAL;
-		}
-	} else {
-		ret = configure_pp_input(asd, isp_sink_crop->width,
-					 isp_sink_crop->height);
-		if (ret) {
-			dev_err(isp->dev, "configure_pp_input %ux%u\n",
-				isp_sink_crop->width, isp_sink_crop->height);
-			return -EINVAL;
-		}
-	}
-	if (asd->copy_mode)
-		ret = atomisp_css_copy_get_output_frame_info(asd, stream_index,
-				output_info);
-	else
-		ret = get_frame_info(asd, output_info);
-	if (ret) {
-		dev_err(isp->dev, "get_frame_info %ux%u (padded to %u)\n",
-			pix->width, pix->height, pix->bytesperline);
-		return -EINVAL;
-	}
-
-	atomisp_update_grid_info(asd, pipe_id, source_pad);
-
-	/* Free the raw_dump buffer first */
-	atomisp_css_frame_free(asd->raw_output_frame);
-	asd->raw_output_frame = NULL;
-
-	if (!asd->continuous_mode->val &&
-	    !asd->params.online_process && !isp->sw_contex.file_input &&
-		atomisp_css_frame_allocate_from_info(&asd->raw_output_frame,
-							raw_output_info))
-		return -ENOMEM;
-
-	return 0;
-}
-
-static void atomisp_get_dis_envelop(struct atomisp_sub_device *asd,
-			    unsigned int width, unsigned int height,
-			    unsigned int *dvs_env_w, unsigned int *dvs_env_h)
-{
-	struct atomisp_device *isp = asd->isp;
-
-	/* if subdev type is SOC camera,we do not need to set DVS */
-	if (isp->inputs[asd->input_curr].type == SOC_CAMERA)
-		asd->params.video_dis_en = false;
-
-	if (asd->params.video_dis_en &&
-	    asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
-		/* envelope is 20% of the output resolution */
-		/*
-		 * dvs envelope cannot be round up.
-		 * it would cause ISP timeout and color switch issue
-		 */
-		*dvs_env_w = rounddown(width / 5, ATOM_ISP_STEP_WIDTH);
-		*dvs_env_h = rounddown(height / 5, ATOM_ISP_STEP_HEIGHT);
-	}
-
-	asd->params.dis_proj_data_valid = false;
-	asd->params.css_update_params_needed = true;
-}
-
-static void atomisp_check_copy_mode(struct atomisp_sub_device *asd,
-		int source_pad, struct v4l2_format *f)
-{
-#if defined(ISP2401_NEW_INPUT_SYSTEM)
-	struct v4l2_mbus_framefmt *sink, *src;
-
-	sink = atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-		V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SINK);
-	src = atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-		V4L2_SUBDEV_FORMAT_ACTIVE, source_pad);
-
-	if ((sink->code == src->code &&
-	    sink->width == f->fmt.pix.width &&
-	    sink->height == f->fmt.pix.height) ||
-	    ((asd->isp->inputs[asd->input_curr].type == SOC_CAMERA) &&
-	    (asd->isp->inputs[asd->input_curr].camera_caps->
-	    sensor[asd->sensor_curr].stream_num > 1)))
-		asd->copy_mode = true;
-	else
-#endif
-		/* Only used for the new input system */
-		asd->copy_mode = false;
-
-	dev_dbg(asd->isp->dev, "copy_mode: %d\n", asd->copy_mode);
-
-}
-
-static int atomisp_set_fmt_to_snr(struct video_device *vdev,
-		struct v4l2_format *f, unsigned int pixelformat,
-		unsigned int padding_w, unsigned int padding_h,
-		unsigned int dvs_env_w, unsigned int dvs_env_h)
-{
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	const struct atomisp_format_bridge *format;
-	struct v4l2_subdev_pad_config pad_cfg;
-	struct v4l2_subdev_format vformat = {
-		.which = V4L2_SUBDEV_FORMAT_TRY,
-	};
-	struct v4l2_mbus_framefmt *ffmt = &vformat.format;
-	struct v4l2_mbus_framefmt *req_ffmt;
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_input_stream_info *stream_info =
-	    (struct atomisp_input_stream_info *)ffmt->reserved;
-	uint16_t stream_index = ATOMISP_INPUT_STREAM_GENERAL;
-	int source_pad = atomisp_subdev_source_pad(vdev);
-	struct v4l2_subdev_fh fh;
-	int ret;
-
-	v4l2_fh_init(&fh.vfh, vdev);
-
-	stream_index = atomisp_source_pad_to_stream_id(asd, source_pad);
-
-	format = atomisp_get_format_bridge(pixelformat);
-	if (format == NULL)
-		return -EINVAL;
-
-	v4l2_fill_mbus_format(ffmt, &f->fmt.pix, format->mbus_code);
-	ffmt->height += padding_h + dvs_env_h;
-	ffmt->width += padding_w + dvs_env_w;
-
-	dev_dbg(isp->dev, "s_mbus_fmt: ask %ux%u (padding %ux%u, dvs %ux%u)\n",
-		ffmt->width, ffmt->height, padding_w, padding_h,
-		dvs_env_w, dvs_env_h);
-
-	__atomisp_init_stream_info(stream_index, stream_info);
-
-	req_ffmt = ffmt;
-
-	/* Disable dvs if resolution can't be supported by sensor */
-	if (asd->params.video_dis_en &&
-	    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO) {
-		vformat.which = V4L2_SUBDEV_FORMAT_TRY;
-		ret = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-			pad, set_fmt, &pad_cfg, &vformat);
-		if (ret)
-			return ret;
-		if (ffmt->width < req_ffmt->width ||
-		    ffmt->height < req_ffmt->height) {
-			req_ffmt->height -= dvs_env_h;
-			req_ffmt->width -= dvs_env_w;
-			ffmt = req_ffmt;
-			dev_warn(isp->dev,
-			  "can not enable video dis due to sensor limitation.");
-			asd->params.video_dis_en = false;
-		}
-	}
-	dev_dbg(isp->dev, "sensor width: %d, height: %d\n",
-		ffmt->width, ffmt->height);
-	vformat.which = V4L2_SUBDEV_FORMAT_ACTIVE;
-	ret = v4l2_subdev_call(isp->inputs[asd->input_curr].camera, pad,
-			       set_fmt, NULL, &vformat);
-	if (ret)
-		return ret;
-
-	__atomisp_update_stream_env(asd, stream_index, stream_info);
-
-	dev_dbg(isp->dev, "sensor width: %d, height: %d\n",
-		ffmt->width, ffmt->height);
-
-	if (ffmt->width < ATOM_ISP_STEP_WIDTH ||
-	    ffmt->height < ATOM_ISP_STEP_HEIGHT)
-			return -EINVAL;
-
-	if (asd->params.video_dis_en &&
-	    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO &&
-	    (ffmt->width < req_ffmt->width || ffmt->height < req_ffmt->height)) {
-		dev_warn(isp->dev,
-			 "can not enable video dis due to sensor limitation.");
-		asd->params.video_dis_en = false;
-	}
-
-	atomisp_subdev_set_ffmt(&asd->subdev, fh.pad,
-				V4L2_SUBDEV_FORMAT_ACTIVE,
-				ATOMISP_SUBDEV_PAD_SINK, ffmt);
-
-	return css_input_resolution_changed(asd, ffmt);
-}
-
-int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
-{
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-	struct atomisp_sub_device *asd = pipe->asd;
-	const struct atomisp_format_bridge *format_bridge;
-	const struct atomisp_format_bridge *snr_format_bridge;
-	struct atomisp_css_frame_info output_info, raw_output_info;
-	struct v4l2_format snr_fmt = *f;
-	struct v4l2_format backup_fmt = *f, s_fmt = *f;
-	unsigned int dvs_env_w = 0, dvs_env_h = 0;
-	unsigned int padding_w = pad_w, padding_h = pad_h;
-	bool res_overflow = false, crop_needs_override = false;
-	struct v4l2_mbus_framefmt isp_sink_fmt;
-	struct v4l2_mbus_framefmt isp_source_fmt = {0};
-	struct v4l2_rect isp_sink_crop;
-	uint16_t source_pad = atomisp_subdev_source_pad(vdev);
-	struct v4l2_subdev_fh fh;
-	int ret;
-
-	dev_dbg(isp->dev,
-		"setting resolution %ux%u on pad %u for asd%d, bytesperline %u\n",
-		f->fmt.pix.width, f->fmt.pix.height, source_pad,
-		asd->index, f->fmt.pix.bytesperline);
-
-	if (source_pad >= ATOMISP_SUBDEV_PADS_NUM)
-		return -EINVAL;
-
-	if (asd->streaming == ATOMISP_DEVICE_STREAMING_ENABLED) {
-		dev_warn(isp->dev, "ISP does not support set format while at streaming!\n");
-		return -EBUSY;
-	}
-
-	v4l2_fh_init(&fh.vfh, vdev);
-
-	format_bridge = atomisp_get_format_bridge(f->fmt.pix.pixelformat);
-	if (format_bridge == NULL)
-		return -EINVAL;
-
-	pipe->sh_fmt = format_bridge->sh_fmt;
-	pipe->pix.pixelformat = f->fmt.pix.pixelformat;
-
-	if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VF ||
-	    (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW
-		&& asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)) {
-		if (asd->fmt_auto->val) {
-			struct v4l2_rect *capture_comp;
-			struct v4l2_rect r = {0};
-
-			r.width = f->fmt.pix.width;
-			r.height = f->fmt.pix.height;
-
-			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW)
-				capture_comp = atomisp_subdev_get_rect(
-					&asd->subdev, NULL,
-					V4L2_SUBDEV_FORMAT_ACTIVE,
-					ATOMISP_SUBDEV_PAD_SOURCE_VIDEO,
-					V4L2_SEL_TGT_COMPOSE);
-			else
-				capture_comp = atomisp_subdev_get_rect(
-					&asd->subdev, NULL,
-					V4L2_SUBDEV_FORMAT_ACTIVE,
-					ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE,
-					V4L2_SEL_TGT_COMPOSE);
-
-			if (capture_comp->width < r.width
-			    || capture_comp->height < r.height) {
-				r.width = capture_comp->width;
-				r.height = capture_comp->height;
-			}
-
-			atomisp_subdev_set_selection(
-				&asd->subdev, fh.pad,
-				V4L2_SUBDEV_FORMAT_ACTIVE, source_pad,
-				V4L2_SEL_TGT_COMPOSE, 0, &r);
-
-			f->fmt.pix.width = r.width;
-			f->fmt.pix.height = r.height;
-		}
-
-		if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW &&
-		    (asd->isp->inputs[asd->input_curr].type == SOC_CAMERA) &&
-		    (asd->isp->inputs[asd->input_curr].camera_caps->
-		    sensor[asd->sensor_curr].stream_num > 1)) {
-			/* For M10MO outputing YUV preview images. */
-			uint16_t video_index =
-				atomisp_source_pad_to_stream_id(asd,
-					ATOMISP_SUBDEV_PAD_SOURCE_VIDEO);
-
-			ret = atomisp_css_copy_get_output_frame_info(asd,
-				video_index, &output_info);
-			if (ret) {
-				dev_err(isp->dev,
-				      "copy_get_output_frame_info ret %i", ret);
-				return -EINVAL;
-			}
-			if (!asd->yuvpp_mode) {
-				/*
-				 * If viewfinder was configured into copy_mode,
-				 * we switch to using yuvpp pipe instead.
-				 */
-				asd->yuvpp_mode = true;
-				ret = atomisp_css_copy_configure_output(
-					asd, video_index, 0, 0, 0, 0);
-				if (ret) {
-					dev_err(isp->dev,
-						"failed to disable copy pipe");
-					return -EINVAL;
-				}
-				ret = atomisp_css_yuvpp_configure_output(
-					asd, video_index,
-					output_info.res.width,
-					output_info.res.height,
-					output_info.padded_width,
-					output_info.format);
-				if (ret) {
-					dev_err(isp->dev,
-						"failed to set up yuvpp pipe\n");
-					return -EINVAL;
-				}
-				atomisp_css_video_enable_online(asd, false);
-				atomisp_css_preview_enable_online(asd,
-					ATOMISP_INPUT_STREAM_GENERAL, false);
-			}
-			atomisp_css_yuvpp_configure_viewfinder(asd, video_index,
-				f->fmt.pix.width, f->fmt.pix.height,
-				format_bridge->planar ? f->fmt.pix.bytesperline
-				: f->fmt.pix.bytesperline * 8
-				/ format_bridge->depth, format_bridge->sh_fmt);
-			atomisp_css_yuvpp_get_viewfinder_frame_info(
-				asd, video_index, &output_info);
-		} else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW) {
-			atomisp_css_video_configure_viewfinder(asd,
-				f->fmt.pix.width, f->fmt.pix.height,
-				format_bridge->planar ? f->fmt.pix.bytesperline
-				: f->fmt.pix.bytesperline * 8
-				/ format_bridge->depth,	format_bridge->sh_fmt);
-			atomisp_css_video_get_viewfinder_frame_info(asd,
-								&output_info);
-			asd->copy_mode = false;
-		} else {
-			atomisp_css_capture_configure_viewfinder(asd,
-				f->fmt.pix.width, f->fmt.pix.height,
-				format_bridge->planar ? f->fmt.pix.bytesperline
-				: f->fmt.pix.bytesperline * 8
-				/ format_bridge->depth,	format_bridge->sh_fmt);
-			atomisp_css_capture_get_viewfinder_frame_info(asd,
-								&output_info);
-			asd->copy_mode = false;
-		}
-
-		goto done;
-	}
-	/*
-	 * Check whether main resolution configured smaller
-	 * than snapshot resolution. If so, force main resolution
-	 * to be the same as snapshot resolution
-	 */
-	if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE) {
-		struct v4l2_rect *r;
-
-		r = atomisp_subdev_get_rect(
-			&asd->subdev, NULL,
-			V4L2_SUBDEV_FORMAT_ACTIVE,
-			ATOMISP_SUBDEV_PAD_SOURCE_VF, V4L2_SEL_TGT_COMPOSE);
-
-		if (r->width && r->height
-		    && (r->width > f->fmt.pix.width
-			|| r->height > f->fmt.pix.height))
-			dev_warn(isp->dev,
-				 "Main Resolution config smaller then Vf Resolution. Force to be equal with Vf Resolution.");
-	}
-
-	/* Pipeline configuration done through subdevs. Bail out now. */
-	if (!asd->fmt_auto->val)
-		goto set_fmt_to_isp;
-
-	/* get sensor resolution and format */
-	ret = atomisp_try_fmt(vdev, &snr_fmt, &res_overflow);
-	if (ret)
-		return ret;
-	f->fmt.pix.width = snr_fmt.fmt.pix.width;
-	f->fmt.pix.height = snr_fmt.fmt.pix.height;
-
-	snr_format_bridge =
-		atomisp_get_format_bridge(snr_fmt.fmt.pix.pixelformat);
-	if (!snr_format_bridge)
-		return -EINVAL;
-
-	atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-				V4L2_SUBDEV_FORMAT_ACTIVE,
-				ATOMISP_SUBDEV_PAD_SINK)->code =
-		snr_format_bridge->mbus_code;
-
-	isp_sink_fmt = *atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-					    V4L2_SUBDEV_FORMAT_ACTIVE,
-					    ATOMISP_SUBDEV_PAD_SINK);
-
-	isp_source_fmt.code = format_bridge->mbus_code;
-	atomisp_subdev_set_ffmt(&asd->subdev, fh.pad,
-				V4L2_SUBDEV_FORMAT_ACTIVE,
-				source_pad, &isp_source_fmt);
-
-	if (!atomisp_subdev_format_conversion(asd, source_pad)) {
-		padding_w = 0;
-		padding_h = 0;
-	} else if (IS_BYT) {
-		padding_w = 12;
-		padding_h = 12;
-	}
-
-	/* construct resolution supported by isp */
-	if (res_overflow && !asd->continuous_mode->val) {
-		f->fmt.pix.width = rounddown(
-			clamp_t(u32, f->fmt.pix.width - padding_w,
-				ATOM_ISP_MIN_WIDTH,
-				ATOM_ISP_MAX_WIDTH), ATOM_ISP_STEP_WIDTH);
-		f->fmt.pix.height = rounddown(
-			clamp_t(u32, f->fmt.pix.height - padding_h,
-				ATOM_ISP_MIN_HEIGHT,
-				ATOM_ISP_MAX_HEIGHT), ATOM_ISP_STEP_HEIGHT);
-	}
-
-	atomisp_get_dis_envelop(asd, f->fmt.pix.width, f->fmt.pix.height,
-				&dvs_env_w, &dvs_env_h);
-
-	if (asd->continuous_mode->val) {
-		struct v4l2_rect *r;
-
-		r = atomisp_subdev_get_rect(
-			&asd->subdev, NULL,
-			V4L2_SUBDEV_FORMAT_ACTIVE,
-			ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE,
-			V4L2_SEL_TGT_COMPOSE);
-		/*
-		 * The ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE should get resolutions
-		 * properly set otherwise, it should not be the capture_pad.
-		 */
-		if (r->width && r->height)
-			asd->capture_pad = ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE;
-		else
-			asd->capture_pad = source_pad;
-	} else {
-		asd->capture_pad = source_pad;
-	}
-	/*
-	 * set format info to sensor
-	 * In continuous mode, resolution is set only if it is higher than
-	 * existing value. This because preview pipe will be configured after
-	 * capture pipe and usually has lower resolution than capture pipe.
-	 */
-	if (!asd->continuous_mode->val ||
-	    isp_sink_fmt.width < (f->fmt.pix.width + padding_w + dvs_env_w) ||
-	    isp_sink_fmt.height < (f->fmt.pix.height + padding_h +
-				    dvs_env_h)) {
-		/*
-		 * For jpeg or custom raw format the sensor will return constant
-		 * width and height. Because we already had quried try_mbus_fmt,
-		 * f->fmt.pix.width and f->fmt.pix.height has been changed to
-		 * this fixed width and height. So we cannot select the correct
-		 * resolution with that information. So use the original width
-		 * and height while set_mbus_fmt() so actual resolutions are
-		 * being used in while set media bus format.
-		 */
-		s_fmt = *f;
-		if (f->fmt.pix.pixelformat == V4L2_PIX_FMT_JPEG ||
-		    f->fmt.pix.pixelformat == V4L2_PIX_FMT_CUSTOM_M10MO_RAW) {
-			s_fmt.fmt.pix.width = backup_fmt.fmt.pix.width;
-			s_fmt.fmt.pix.height = backup_fmt.fmt.pix.height;
-		}
-		ret = atomisp_set_fmt_to_snr(vdev, &s_fmt,
-					f->fmt.pix.pixelformat, padding_w,
-					padding_h, dvs_env_w, dvs_env_h);
-		if (ret)
-			return -EINVAL;
-
-		atomisp_csi_lane_config(isp);
-		crop_needs_override = true;
-	}
-
-	atomisp_check_copy_mode(asd, source_pad, &backup_fmt);
-	asd->yuvpp_mode = false;			/* Reset variable */
-
-	isp_sink_crop = *atomisp_subdev_get_rect(&asd->subdev, NULL,
-						 V4L2_SUBDEV_FORMAT_ACTIVE,
-						 ATOMISP_SUBDEV_PAD_SINK,
-						 V4L2_SEL_TGT_CROP);
-
-	/* Try to enable YUV downscaling if ISP input is 10 % (either
-	 * width or height) bigger than the desired result. */
-	if (isp_sink_crop.width * 9 / 10 < f->fmt.pix.width ||
-	    isp_sink_crop.height * 9 / 10 < f->fmt.pix.height ||
-	    (atomisp_subdev_format_conversion(asd, source_pad) &&
-	    ((asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
-	       !asd->continuous_mode->val) ||
-	      asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER))) {
-		/* for continuous mode, preview size might be smaller than
-		 * still capture size. if preview size still needs crop,
-		 * pick the larger one between crop size of preview and
-		 * still capture.
-		 */
-		if (asd->continuous_mode->val
-		    && source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW
-		    && !crop_needs_override) {
-			isp_sink_crop.width =
-				max_t(unsigned int, f->fmt.pix.width,
-				      isp_sink_crop.width);
-			isp_sink_crop.height =
-				max_t(unsigned int, f->fmt.pix.height,
-				      isp_sink_crop.height);
-		} else {
-			isp_sink_crop.width = f->fmt.pix.width;
-			isp_sink_crop.height = f->fmt.pix.height;
-		}
-
-		atomisp_subdev_set_selection(&asd->subdev, fh.pad,
-					     V4L2_SUBDEV_FORMAT_ACTIVE,
-					     ATOMISP_SUBDEV_PAD_SINK,
-					     V4L2_SEL_TGT_CROP,
-					     V4L2_SEL_FLAG_KEEP_CONFIG,
-					     &isp_sink_crop);
-		atomisp_subdev_set_selection(&asd->subdev, fh.pad,
-					     V4L2_SUBDEV_FORMAT_ACTIVE,
-					     source_pad, V4L2_SEL_TGT_COMPOSE,
-					     0, &isp_sink_crop);
-	} else if (IS_MOFD) {
-		struct v4l2_rect main_compose = {0};
-
-		main_compose.width = isp_sink_crop.width;
-		main_compose.height =
-			DIV_ROUND_UP(main_compose.width * f->fmt.pix.height,
-				     f->fmt.pix.width);
-		if (main_compose.height > isp_sink_crop.height) {
-			main_compose.height = isp_sink_crop.height;
-			main_compose.width =
-				DIV_ROUND_UP(main_compose.height *
-					     f->fmt.pix.width,
-					     f->fmt.pix.height);
-		}
-
-		atomisp_subdev_set_selection(&asd->subdev, fh.pad,
-				V4L2_SUBDEV_FORMAT_ACTIVE,
-				source_pad,
-				V4L2_SEL_TGT_COMPOSE, 0,
-				&main_compose);
-	} else {
-		struct v4l2_rect sink_crop = {0};
-		struct v4l2_rect main_compose = {0};
-
-		main_compose.width = f->fmt.pix.width;
-		main_compose.height = f->fmt.pix.height;
-
-#ifndef ISP2401
-		/* WORKAROUND: this override is universally enabled in
-		 * GMIN to work around a CTS failures (GMINL-539)
-		 * which appears to be related by a hardware
-		 * performance limitation.  It's unclear why this
-		 * particular code triggers the issue. */
-		if (1 ||
-		    crop_needs_override) {
-#else
-		if (crop_needs_override) {
-#endif
-			if (isp_sink_crop.width * main_compose.height >
-			    isp_sink_crop.height * main_compose.width) {
-				sink_crop.height = isp_sink_crop.height;
-				sink_crop.width = DIV_NEAREST_STEP(
-						sink_crop.height *
-						f->fmt.pix.width,
-						f->fmt.pix.height,
-						ATOM_ISP_STEP_WIDTH);
-			} else {
-				sink_crop.width = isp_sink_crop.width;
-				sink_crop.height = DIV_NEAREST_STEP(
-						sink_crop.width *
-						f->fmt.pix.height,
-						f->fmt.pix.width,
-						ATOM_ISP_STEP_HEIGHT);
-			}
-			atomisp_subdev_set_selection(&asd->subdev, fh.pad,
-				V4L2_SUBDEV_FORMAT_ACTIVE,
-				ATOMISP_SUBDEV_PAD_SINK,
-				V4L2_SEL_TGT_CROP,
-				V4L2_SEL_FLAG_KEEP_CONFIG,
-				&sink_crop);
-		}
-		atomisp_subdev_set_selection(&asd->subdev, fh.pad,
-				V4L2_SUBDEV_FORMAT_ACTIVE,
-				source_pad,
-				V4L2_SEL_TGT_COMPOSE, 0,
-				&main_compose);
-	}
-
-set_fmt_to_isp:
-	ret = atomisp_set_fmt_to_isp(vdev, &output_info, &raw_output_info,
-				     &f->fmt.pix, source_pad);
-	if (ret)
-		return -EINVAL;
-done:
-	pipe->pix.width = f->fmt.pix.width;
-	pipe->pix.height = f->fmt.pix.height;
-	pipe->pix.pixelformat = f->fmt.pix.pixelformat;
-	if (format_bridge->planar) {
-		pipe->pix.bytesperline = output_info.padded_width;
-		pipe->pix.sizeimage = PAGE_ALIGN(f->fmt.pix.height *
-			DIV_ROUND_UP(format_bridge->depth *
-				     output_info.padded_width, 8));
-	} else {
-		pipe->pix.bytesperline =
-			DIV_ROUND_UP(format_bridge->depth *
-				     output_info.padded_width, 8);
-		pipe->pix.sizeimage =
-			PAGE_ALIGN(f->fmt.pix.height * pipe->pix.bytesperline);
-
-	}
-	if (f->fmt.pix.field == V4L2_FIELD_ANY)
-		f->fmt.pix.field = V4L2_FIELD_NONE;
-	pipe->pix.field = f->fmt.pix.field;
-
-	f->fmt.pix = pipe->pix;
-	f->fmt.pix.priv = PAGE_ALIGN(pipe->pix.width *
-				     pipe->pix.height * 2);
-
-	pipe->capq.field = f->fmt.pix.field;
-
-	/*
-	 * If in video 480P case, no GFX throttle
-	 */
-	if (asd->run_mode->val == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO &&
-	    f->fmt.pix.width == 720 && f->fmt.pix.height == 480)
-		isp->need_gfx_throttle = false;
-	else
-		isp->need_gfx_throttle = true;
-
-	return 0;
-}
-
-int atomisp_set_fmt_file(struct video_device *vdev, struct v4l2_format *f)
-{
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-	struct atomisp_sub_device *asd = pipe->asd;
-	struct v4l2_mbus_framefmt ffmt = {0};
-	const struct atomisp_format_bridge *format_bridge;
-	struct v4l2_subdev_fh fh;
-	int ret;
-
-	v4l2_fh_init(&fh.vfh, vdev);
-
-	dev_dbg(isp->dev, "setting fmt %ux%u 0x%x for file inject\n",
-		f->fmt.pix.width, f->fmt.pix.height, f->fmt.pix.pixelformat);
-	ret = atomisp_try_fmt_file(isp, f);
-	if (ret) {
-		dev_err(isp->dev, "atomisp_try_fmt_file err: %d\n", ret);
-		return ret;
-	}
-
-	format_bridge = atomisp_get_format_bridge(f->fmt.pix.pixelformat);
-	if (format_bridge == NULL) {
-		dev_dbg(isp->dev, "atomisp_get_format_bridge err! fmt:0x%x\n",
-				f->fmt.pix.pixelformat);
-		return -EINVAL;
-	}
-
-	pipe->pix = f->fmt.pix;
-	atomisp_css_input_set_mode(asd, CSS_INPUT_MODE_FIFO);
-	atomisp_css_input_configure_port(asd,
-		__get_mipi_port(isp, ATOMISP_CAMERA_PORT_PRIMARY), 2, 0xffff4,
-		0, 0, 0, 0);
-	ffmt.width = f->fmt.pix.width;
-	ffmt.height = f->fmt.pix.height;
-	ffmt.code = format_bridge->mbus_code;
-
-	atomisp_subdev_set_ffmt(&asd->subdev, fh.pad, V4L2_SUBDEV_FORMAT_ACTIVE,
-				ATOMISP_SUBDEV_PAD_SINK, &ffmt);
-
-	return 0;
-}
-
-int atomisp_set_shading_table(struct atomisp_sub_device *asd,
-		struct atomisp_shading_table *user_shading_table)
-{
-	struct atomisp_css_shading_table *shading_table;
-	struct atomisp_css_shading_table *free_table;
-	unsigned int len_table;
-	int i;
-	int ret = 0;
-
-	if (!user_shading_table)
-		return -EINVAL;
-
-	if (!user_shading_table->enable) {
-		atomisp_css_set_shading_table(asd, NULL);
-		asd->params.sc_en = false;
-		return 0;
-	}
-
-	/* If enabling, all tables must be set */
-	for (i = 0; i < ATOMISP_NUM_SC_COLORS; i++) {
-		if (!user_shading_table->data[i])
-			return -EINVAL;
-	}
-
-	/* Shading table size per color */
-	if (user_shading_table->width > SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR ||
-	    user_shading_table->height > SH_CSS_MAX_SCTBL_HEIGHT_PER_COLOR)
-		return -EINVAL;
-
-	shading_table = atomisp_css_shading_table_alloc(
-			user_shading_table->width, user_shading_table->height);
-	if (!shading_table)
-		return -ENOMEM;
-
-	len_table = user_shading_table->width * user_shading_table->height *
-		    ATOMISP_SC_TYPE_SIZE;
-	for (i = 0; i < ATOMISP_NUM_SC_COLORS; i++) {
-		ret = copy_from_user(shading_table->data[i],
-				     user_shading_table->data[i], len_table);
-		if (ret) {
-			free_table = shading_table;
-			ret = -EFAULT;
-			goto out;
-		}
-	}
-	shading_table->sensor_width = user_shading_table->sensor_width;
-	shading_table->sensor_height = user_shading_table->sensor_height;
-	shading_table->fraction_bits = user_shading_table->fraction_bits;
-
-	free_table = asd->params.css_param.shading_table;
-	asd->params.css_param.shading_table = shading_table;
-	atomisp_css_set_shading_table(asd, shading_table);
-	asd->params.sc_en = true;
-
-out:
-	if (free_table != NULL)
-		atomisp_css_shading_table_free(free_table);
-
-	return ret;
-}
-
-/*Turn off ISP dphy */
-int atomisp_ospm_dphy_down(struct atomisp_device *isp)
-{
-	unsigned long flags;
-	u32 reg;
-
-	dev_dbg(isp->dev, "%s\n", __func__);
-
-	/* if ISP timeout, we can force powerdown */
-	if (isp->isp_timeout)
-		goto done;
-
-	if (!atomisp_dev_users(isp))
-		goto done;
-
-	spin_lock_irqsave(&isp->lock, flags);
-	isp->sw_contex.power_state = ATOM_ISP_POWER_DOWN;
-	spin_unlock_irqrestore(&isp->lock, flags);
-done:
-	/*
-	 * MRFLD IUNIT DPHY is located in an always-power-on island
-	 * MRFLD HW design need all CSI ports are disabled before
-	 * powering down the IUNIT.
-	 */
-	pci_read_config_dword(isp->pdev, MRFLD_PCI_CSI_CONTROL, &reg);
-	reg |= MRFLD_ALL_CSI_PORTS_OFF_MASK;
-	pci_write_config_dword(isp->pdev, MRFLD_PCI_CSI_CONTROL, reg);
-	return 0;
-}
-
-/*Turn on ISP dphy */
-int atomisp_ospm_dphy_up(struct atomisp_device *isp)
-{
-	unsigned long flags;
-	dev_dbg(isp->dev, "%s\n", __func__);
-
-	spin_lock_irqsave(&isp->lock, flags);
-	isp->sw_contex.power_state = ATOM_ISP_POWER_UP;
-	spin_unlock_irqrestore(&isp->lock, flags);
-
-	return 0;
-}
-
-
-int atomisp_exif_makernote(struct atomisp_sub_device *asd,
-			   struct atomisp_makernote_info *config)
-{
-	struct v4l2_control ctrl;
-	struct atomisp_device *isp = asd->isp;
-
-	ctrl.id = V4L2_CID_FOCAL_ABSOLUTE;
-	if (v4l2_g_ctrl
-	    (isp->inputs[asd->input_curr].camera->ctrl_handler, &ctrl)) {
-		dev_warn(isp->dev, "failed to g_ctrl for focal length\n");
-		return -EINVAL;
-	} else {
-		config->focal_length = ctrl.value;
-	}
-
-	ctrl.id = V4L2_CID_FNUMBER_ABSOLUTE;
-	if (v4l2_g_ctrl
-	    (isp->inputs[asd->input_curr].camera->ctrl_handler, &ctrl)) {
-		dev_warn(isp->dev, "failed to g_ctrl for f-number\n");
-		return -EINVAL;
-	} else {
-		config->f_number_curr = ctrl.value;
-	}
-
-	ctrl.id = V4L2_CID_FNUMBER_RANGE;
-	if (v4l2_g_ctrl
-	    (isp->inputs[asd->input_curr].camera->ctrl_handler, &ctrl)) {
-		dev_warn(isp->dev, "failed to g_ctrl for f number range\n");
-		return -EINVAL;
-	} else {
-		config->f_number_range = ctrl.value;
-	}
-
-	return 0;
-}
-
-int atomisp_offline_capture_configure(struct atomisp_sub_device *asd,
-			      struct atomisp_cont_capture_conf *cvf_config)
-{
-	struct v4l2_ctrl *c;
-
-	/*
-	* In case of M10MO ZSL capture case, we need to issue a separate
-	* capture request to M10MO which will output captured jpeg image
-	*/
-	c = v4l2_ctrl_find(
-		asd->isp->inputs[asd->input_curr].camera->ctrl_handler,
-		V4L2_CID_START_ZSL_CAPTURE);
-	if (c) {
-		int ret;
-		dev_dbg(asd->isp->dev, "%s trigger ZSL capture request\n",
-			__func__);
-		/* TODO: use the cvf_config */
-		ret = v4l2_ctrl_s_ctrl(c, 1);
-		if (ret)
-			return ret;
-
-		return v4l2_ctrl_s_ctrl(c, 0);
-	}
-
-	asd->params.offline_parm = *cvf_config;
-
-	if (asd->params.offline_parm.num_captures) {
-		if (asd->streaming == ATOMISP_DEVICE_STREAMING_DISABLED) {
-			unsigned int init_raw_num;
-
-			if (asd->enable_raw_buffer_lock->val) {
-				init_raw_num =
-					ATOMISP_CSS2_NUM_OFFLINE_INIT_CONTINUOUS_FRAMES_LOCK_EN;
-				if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
-				    asd->params.video_dis_en)
-					init_raw_num +=
-					    ATOMISP_CSS2_NUM_DVS_FRAME_DELAY;
-			} else {
-				init_raw_num =
-					ATOMISP_CSS2_NUM_OFFLINE_INIT_CONTINUOUS_FRAMES;
-			}
-
-			/* TODO: this can be removed once user-space
-			 *       has been updated to use control API */
-			asd->continuous_raw_buffer_size->val =
-				max_t(int,
-				      asd->continuous_raw_buffer_size->val,
-				      asd->params.offline_parm.
-				      num_captures + init_raw_num);
-			asd->continuous_raw_buffer_size->val =
-				min_t(int, ATOMISP_CONT_RAW_FRAMES,
-				      asd->continuous_raw_buffer_size->val);
-		}
-		asd->continuous_mode->val = true;
-	} else {
-		asd->continuous_mode->val = false;
-		__enable_continuous_mode(asd, false);
-	}
-
-	return 0;
-}
-
-/*
- * set auto exposure metering window to camera sensor
- */
-int atomisp_s_ae_window(struct atomisp_sub_device *asd,
-			struct atomisp_ae_window *arg)
-{
-	struct atomisp_device *isp = asd->isp;
-	/* Coverity CID 298071 - initialzize struct */
-	struct v4l2_subdev_selection sel = { 0 };
-
-	sel.r.left = arg->x_left;
-	sel.r.top = arg->y_top;
-	sel.r.width = arg->x_right - arg->x_left + 1;
-	sel.r.height = arg->y_bottom - arg->y_top + 1;
-
-	if (v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-			     pad, set_selection, NULL, &sel)) {
-		dev_err(isp->dev, "failed to call sensor set_selection.\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-int atomisp_flash_enable(struct atomisp_sub_device *asd, int num_frames)
-{
-	struct atomisp_device *isp = asd->isp;
-
-	if (num_frames < 0) {
-		dev_dbg(isp->dev, "%s ERROR: num_frames: %d\n", __func__,
-				num_frames);
-		return -EINVAL;
-	}
-	/* a requested flash is still in progress. */
-	if (num_frames && asd->params.flash_state != ATOMISP_FLASH_IDLE) {
-		dev_dbg(isp->dev, "%s flash busy: %d frames left: %d\n",
-				__func__, asd->params.flash_state,
-				asd->params.num_flash_frames);
-		return -EBUSY;
-	}
-
-	asd->params.num_flash_frames = num_frames;
-	asd->params.flash_state = ATOMISP_FLASH_REQUESTED;
-	return 0;
-}
-
-int atomisp_source_pad_to_stream_id(struct atomisp_sub_device *asd,
-					   uint16_t source_pad)
-{
-	int stream_id;
-	struct atomisp_device *isp = asd->isp;
-
-	if (isp->inputs[asd->input_curr].camera_caps->
-			sensor[asd->sensor_curr].stream_num == 1)
-		return ATOMISP_INPUT_STREAM_GENERAL;
-
-	switch (source_pad) {
-	case ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE:
-		stream_id = ATOMISP_INPUT_STREAM_CAPTURE;
-		break;
-	case ATOMISP_SUBDEV_PAD_SOURCE_VF:
-		stream_id = ATOMISP_INPUT_STREAM_POSTVIEW;
-		break;
-	case ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW:
-		stream_id = ATOMISP_INPUT_STREAM_PREVIEW;
-		break;
-	case ATOMISP_SUBDEV_PAD_SOURCE_VIDEO:
-		stream_id = ATOMISP_INPUT_STREAM_VIDEO;
-		break;
-	default:
-		stream_id = ATOMISP_INPUT_STREAM_GENERAL;
-	}
-
-	return stream_id;
-}
-
-bool atomisp_is_vf_pipe(struct atomisp_video_pipe *pipe)
-{
-	struct atomisp_sub_device *asd = pipe->asd;
-
-	if (pipe == &asd->video_out_vf)
-		return true;
-
-	if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
-	    pipe == &asd->video_out_preview)
-		return true;
-
-	return false;
-}
-
-static int __checking_exp_id(struct atomisp_sub_device *asd, int exp_id)
-{
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->enable_raw_buffer_lock->val) {
-		dev_warn(isp->dev, "%s Raw Buffer Lock is disable.\n", __func__);
-		return -EINVAL;
-	}
-	if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED) {
-		dev_err(isp->dev, "%s streaming %d invalid exp_id %d.\n",
-			__func__, exp_id, asd->streaming);
-		return -EINVAL;
-	}
-	if ((exp_id > ATOMISP_MAX_EXP_ID) || (exp_id <= 0)) {
-		dev_err(isp->dev, "%s exp_id %d invalid.\n", __func__, exp_id);
-		return -EINVAL;
-	}
-	return 0;
-}
-
-void atomisp_init_raw_buffer_bitmap(struct atomisp_sub_device *asd)
-{
-	unsigned long flags;
-	spin_lock_irqsave(&asd->raw_buffer_bitmap_lock, flags);
-	memset(asd->raw_buffer_bitmap, 0, sizeof(asd->raw_buffer_bitmap));
-	asd->raw_buffer_locked_count = 0;
-	spin_unlock_irqrestore(&asd->raw_buffer_bitmap_lock, flags);
-}
-
-int atomisp_set_raw_buffer_bitmap(struct atomisp_sub_device *asd, int exp_id)
-{
-	int *bitmap, bit;
-	unsigned long flags;
-
-	if (__checking_exp_id(asd, exp_id))
-		return -EINVAL;
-
-	bitmap = asd->raw_buffer_bitmap + exp_id / 32;
-	bit = exp_id % 32;
-	spin_lock_irqsave(&asd->raw_buffer_bitmap_lock, flags);
-	(*bitmap) |= (1 << bit);
-	asd->raw_buffer_locked_count++;
-	spin_unlock_irqrestore(&asd->raw_buffer_bitmap_lock, flags);
-
-	dev_dbg(asd->isp->dev, "%s: exp_id %d,  raw_buffer_locked_count %d\n",
-		__func__, exp_id, asd->raw_buffer_locked_count);
-
-	/* Check if the raw buffer after next is still locked!!! */
-	exp_id += 2;
-	if (exp_id > ATOMISP_MAX_EXP_ID)
-		exp_id -= ATOMISP_MAX_EXP_ID;
-	bitmap = asd->raw_buffer_bitmap + exp_id / 32;
-	bit = exp_id % 32;
-	if ((*bitmap) & (1 << bit)) {
-		int ret;
-
-		/* WORKAROUND unlock the raw buffer compulsively */
-		ret = atomisp_css_exp_id_unlock(asd, exp_id);
-		if (ret) {
-			dev_err(asd->isp->dev, "%s exp_id is wrapping back to %d but force unlock failed,, err %d.\n",
-				__func__, exp_id, ret);
-			return ret;
-		}
-
-		spin_lock_irqsave(&asd->raw_buffer_bitmap_lock, flags);
-		(*bitmap) &= ~(1 << bit);
-		asd->raw_buffer_locked_count--;
-		spin_unlock_irqrestore(&asd->raw_buffer_bitmap_lock, flags);
-		dev_warn(asd->isp->dev, "%s exp_id is wrapping back to %d but it is still locked so force unlock it, raw_buffer_locked_count %d\n",
-			__func__, exp_id, asd->raw_buffer_locked_count);
-	}
-	return 0;
-}
-
-static int __is_raw_buffer_locked(struct atomisp_sub_device *asd, int exp_id)
-{
-	int *bitmap, bit;
-	unsigned long flags;
-	int ret;
-
-	if (__checking_exp_id(asd, exp_id))
-		return -EINVAL;
-
-	bitmap = asd->raw_buffer_bitmap + exp_id / 32;
-	bit = exp_id % 32;
-	spin_lock_irqsave(&asd->raw_buffer_bitmap_lock, flags);
-	ret = ((*bitmap) & (1 << bit));
-	spin_unlock_irqrestore(&asd->raw_buffer_bitmap_lock, flags);
-	return !ret;
-}
-
-static int __clear_raw_buffer_bitmap(struct atomisp_sub_device *asd, int exp_id)
-{
-	int *bitmap, bit;
-	unsigned long flags;
-
-	if (__is_raw_buffer_locked(asd, exp_id))
-		return -EINVAL;
-
-	bitmap = asd->raw_buffer_bitmap + exp_id / 32;
-	bit = exp_id % 32;
-	spin_lock_irqsave(&asd->raw_buffer_bitmap_lock, flags);
-	(*bitmap) &= ~(1 << bit);
-	asd->raw_buffer_locked_count--;
-	spin_unlock_irqrestore(&asd->raw_buffer_bitmap_lock, flags);
-
-	dev_dbg(asd->isp->dev, "%s: exp_id %d,  raw_buffer_locked_count %d\n",
-		__func__, exp_id, asd->raw_buffer_locked_count);
-	return 0;
-}
-
-int atomisp_exp_id_capture(struct atomisp_sub_device *asd, int *exp_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	int value = *exp_id;
-	int ret;
-
-	ret = __is_raw_buffer_locked(asd, value);
-	if (ret) {
-		dev_err(isp->dev, "%s exp_id %d invalid %d.\n", __func__, value, ret);
-		return -EINVAL;
-	}
-
-	dev_dbg(isp->dev, "%s exp_id %d\n", __func__, value);
-	ret = atomisp_css_exp_id_capture(asd, value);
-	if (ret) {
-		dev_err(isp->dev, "%s exp_id %d failed.\n", __func__, value);
-		return -EIO;
-	}
-	return 0;
-}
-
-int atomisp_exp_id_unlock(struct atomisp_sub_device *asd, int *exp_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	int value = *exp_id;
-	int ret;
-
-	ret = __clear_raw_buffer_bitmap(asd, value);
-	if (ret) {
-		dev_err(isp->dev, "%s exp_id %d invalid %d.\n", __func__, value, ret);
-		return -EINVAL;
-	}
-
-	dev_dbg(isp->dev, "%s exp_id %d\n", __func__, value);
-	ret = atomisp_css_exp_id_unlock(asd, value);
-	if (ret)
-		dev_err(isp->dev, "%s exp_id %d failed, err %d.\n",
-			__func__, value, ret);
-
-	return ret;
-}
-
-int atomisp_enable_dz_capt_pipe(struct atomisp_sub_device *asd,
-					   unsigned int *enable)
-{
-	bool value;
-
-	if (enable == NULL)
-		return -EINVAL;
-
-	value = *enable > 0 ? true : false;
-
-	atomisp_en_dz_capt_pipe(asd, value);
-
-	return 0;
-}
-
-int atomisp_inject_a_fake_event(struct atomisp_sub_device *asd, int *event)
-{
-	if (!event || asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
-		return -EINVAL;
-
-	dev_dbg(asd->isp->dev, "%s: trying to inject a fake event 0x%x\n",
-		__func__, *event);
-
-	switch (*event) {
-	case V4L2_EVENT_FRAME_SYNC:
-		atomisp_sof_event(asd);
-		break;
-	case V4L2_EVENT_FRAME_END:
-		atomisp_eof_event(asd, 0);
-		break;
-	case V4L2_EVENT_ATOMISP_3A_STATS_READY:
-		atomisp_3a_stats_ready_event(asd, 0);
-		break;
-	case V4L2_EVENT_ATOMISP_METADATA_READY:
-		atomisp_metadata_ready_event(asd, 0);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int atomisp_get_pipe_id(struct atomisp_video_pipe *pipe)
-{
-	struct atomisp_sub_device *asd = pipe->asd;
-
-	if (ATOMISP_USE_YUVPP(asd))
-		return CSS_PIPE_ID_YUVPP;
-	else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER)
-		return CSS_PIPE_ID_VIDEO;
-	else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT)
-		return CSS_PIPE_ID_CAPTURE;
-	else if (pipe == &asd->video_out_video_capture)
-		return CSS_PIPE_ID_VIDEO;
-	else if (pipe == &asd->video_out_vf)
-		return CSS_PIPE_ID_CAPTURE;
-	else if (pipe == &asd->video_out_preview) {
-		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-			return CSS_PIPE_ID_VIDEO;
-		else
-			return CSS_PIPE_ID_PREVIEW;
-	} else if (pipe == &asd->video_out_capture) {
-		if (asd->copy_mode)
-			return IA_CSS_PIPE_ID_COPY;
-		else
-			return CSS_PIPE_ID_CAPTURE;
-	}
-
-	/* fail through */
-	dev_warn(asd->isp->dev, "%s failed to find proper pipe\n",
-		__func__);
-	return CSS_PIPE_ID_CAPTURE;
-}
-
-int atomisp_get_invalid_frame_num(struct video_device *vdev,
-					int *invalid_frame_num)
-{
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-	struct atomisp_sub_device *asd = pipe->asd;
-	enum atomisp_css_pipe_id pipe_id;
-	struct ia_css_pipe_info p_info;
-	int ret;
-
-	if (asd->isp->inputs[asd->input_curr].camera_caps->
-		sensor[asd->sensor_curr].stream_num > 1) {
-		/* External ISP */
-		*invalid_frame_num = 0;
-		return 0;
-	}
-
-	pipe_id = atomisp_get_pipe_id(pipe);
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].pipes[pipe_id]) {
-		dev_warn(asd->isp->dev, "%s pipe %d has not been created yet, do SET_FMT first!\n",
-			__func__, pipe_id);
-		return -EINVAL;
-	}
-
-	ret = ia_css_pipe_get_info(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.pipes[pipe_id], &p_info);
-	if (ret == IA_CSS_SUCCESS) {
-		*invalid_frame_num = p_info.num_invalid_frames;
-		return 0;
-	} else {
-		dev_warn(asd->isp->dev, "%s get pipe infor failed %d\n",
-			 __func__, ret);
-		return -EINVAL;
-	}
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.h
deleted file mode 100644
index 79d493dba403..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.h
+++ /dev/null
@@ -1,446 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef	__ATOMISP_CMD_H__
-#define	__ATOMISP_CMD_H__
-
-#include "../../include/linux/atomisp.h"
-#include <linux/interrupt.h>
-#include <linux/videodev2.h>
-
-#include <media/v4l2-subdev.h>
-
-#include "atomisp_internal.h"
-
-#include "ia_css_types.h"
-#include "ia_css.h"
-
-struct atomisp_device;
-struct atomisp_css_frame;
-
-#define MSI_ENABLE_BIT		16
-#define INTR_DISABLE_BIT	10
-#define BUS_MASTER_ENABLE	2
-#define MEMORY_SPACE_ENABLE	1
-#define INTR_IER		24
-#define INTR_IIR		16
-#ifdef ISP2401
-#define RUNMODE_MASK (ATOMISP_RUN_MODE_VIDEO | ATOMISP_RUN_MODE_STILL_CAPTURE \
-			| ATOMISP_RUN_MODE_PREVIEW)
-
-/* FIXME: check if can go */
-extern int atomisp_punit_hpll_freq;
-#endif
-
-/*
- * Helper function
- */
-void dump_sp_dmem(struct atomisp_device *isp, unsigned int addr,
-		  unsigned int size);
-struct camera_mipi_info *atomisp_to_sensor_mipi_info(struct v4l2_subdev *sd);
-struct atomisp_video_pipe *atomisp_to_video_pipe(struct video_device *dev);
-struct atomisp_acc_pipe *atomisp_to_acc_pipe(struct video_device *dev);
-int atomisp_reset(struct atomisp_device *isp);
-void atomisp_flush_bufs_and_wakeup(struct atomisp_sub_device *asd);
-void atomisp_clear_css_buffer_counters(struct atomisp_sub_device *asd);
-#ifndef ISP2401
-bool atomisp_buffers_queued(struct atomisp_sub_device *asd);
-#else
-bool atomisp_buffers_queued_pipe(struct atomisp_video_pipe *pipe);
-#endif
-
-/* TODO:should be here instead of atomisp_helper.h
-extern void __iomem *atomisp_io_base;
-
-static inline void __iomem *atomisp_get_io_virt_addr(unsigned int address)
-{
-	void __iomem *ret = atomisp_io_base + (address & 0x003FFFFF);
-	return ret;
-}
-*/
-
-/*
- * Interrupt functions
- */
-void atomisp_msi_irq_init(struct atomisp_device *isp, struct pci_dev *dev);
-void atomisp_msi_irq_uninit(struct atomisp_device *isp, struct pci_dev *dev);
-void atomisp_wdt_work(struct work_struct *work);
-void atomisp_wdt(struct timer_list *t);
-void atomisp_setup_flash(struct atomisp_sub_device *asd);
-irqreturn_t atomisp_isr(int irq, void *dev);
-irqreturn_t atomisp_isr_thread(int irq, void *isp_ptr);
-const struct atomisp_format_bridge *get_atomisp_format_bridge_from_mbus(
-	u32 mbus_code);
-bool atomisp_is_mbuscode_raw(uint32_t code);
-int atomisp_get_frame_pgnr(struct atomisp_device *isp,
-			   const struct atomisp_css_frame *frame, u32 *p_pgnr);
-void atomisp_delayed_init_work(struct work_struct *work);
-
-/*
- * Get internal fmt according to V4L2 fmt
- */
-
-bool atomisp_is_viewfinder_support(struct atomisp_device *isp);
-
-/*
- * ISP features control function
- */
-
-/*
-#ifdef ISP2401
- * Function to set sensor runmode by user when
- * ATOMISP_IOC_S_SENSOR_RUNMODE ioctl was called
- */
-int atomisp_set_sensor_runmode(struct atomisp_sub_device *asd,
-		struct atomisp_s_runmode *runmode);
-/*
-#endif
- * Function to enable/disable lens geometry distortion correction (GDC) and
- * chromatic aberration correction (CAC)
- */
-int atomisp_gdc_cac(struct atomisp_sub_device *asd, int flag,
-		    __s32 *value);
-
-/*
- * Function to enable/disable low light mode (including ANR)
- */
-int atomisp_low_light(struct atomisp_sub_device *asd, int flag,
-		      __s32 *value);
-
-/*
- * Function to enable/disable extra noise reduction (XNR) in low light
- * condition
- */
-int atomisp_xnr(struct atomisp_sub_device *asd, int flag, int *arg);
-
-int atomisp_formats(struct atomisp_sub_device *asd, int flag,
-		struct atomisp_formats_config *config);
-
-/*
- * Function to configure noise reduction
- */
-int atomisp_nr(struct atomisp_sub_device *asd, int flag,
-	       struct atomisp_nr_config *config);
-
-/*
- * Function to configure temporal noise reduction (TNR)
- */
-int atomisp_tnr(struct atomisp_sub_device *asd, int flag,
-		struct atomisp_tnr_config *config);
-
-/*
- * Function to configure black level compensation
- */
-int atomisp_black_level(struct atomisp_sub_device *asd, int flag,
-			struct atomisp_ob_config *config);
-
-/*
- * Function to configure edge enhancement
- */
-int atomisp_ee(struct atomisp_sub_device *asd, int flag,
-	       struct atomisp_ee_config *config);
-
-/*
- * Function to update Gamma table for gamma, brightness and contrast config
- */
-int atomisp_gamma(struct atomisp_sub_device *asd, int flag,
-		  struct atomisp_gamma_table *config);
-/*
- * Function to update Ctc table for Chroma Enhancement
- */
-int atomisp_ctc(struct atomisp_sub_device *asd, int flag,
-		struct atomisp_ctc_table *config);
-
-/*
- * Function to update gamma correction parameters
- */
-int atomisp_gamma_correction(struct atomisp_sub_device *asd, int flag,
-	struct atomisp_gc_config *config);
-
-/*
- * Function to update Gdc table for gdc
- */
-int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag,
-			  struct atomisp_morph_table *config);
-
-/*
- * Function to update table for macc
- */
-int atomisp_macc_table(struct atomisp_sub_device *asd, int flag,
-		       struct atomisp_macc_config *config);
-/*
- * Function to get DIS statistics.
- */
-int atomisp_get_dis_stat(struct atomisp_sub_device *asd,
-			 struct atomisp_dis_statistics *stats);
-
-/*
- * Function to get DVS2 BQ resolution settings
- */
-int atomisp_get_dvs2_bq_resolutions(struct atomisp_sub_device *asd,
-			 struct atomisp_dvs2_bq_resolutions *bq_res);
-
-/*
- * Function to set the DIS coefficients.
- */
-int atomisp_set_dis_coefs(struct atomisp_sub_device *asd,
-			  struct atomisp_dis_coefficients *coefs);
-
-/*
- * Function to set the DIS motion vector.
- */
-int atomisp_set_dis_vector(struct atomisp_sub_device *asd,
-			   struct atomisp_dis_vector *vector);
-
-/*
- * Function to set/get 3A stat from isp
- */
-int atomisp_3a_stat(struct atomisp_sub_device *asd, int flag,
-		    struct atomisp_3a_statistics *config);
-
-/*
- * Function to get metadata from isp
- */
-int atomisp_get_metadata(struct atomisp_sub_device *asd, int flag,
-			 struct atomisp_metadata *config);
-
-int atomisp_get_metadata_by_type(struct atomisp_sub_device *asd, int flag,
-			 struct atomisp_metadata_with_type *config);
-
-int atomisp_set_parameters(struct video_device *vdev,
-			struct atomisp_parameters *arg);
-/*
- * Function to set/get isp parameters to isp
- */
-int atomisp_param(struct atomisp_sub_device *asd, int flag,
-		  struct atomisp_parm *config);
-
-/*
- * Function to configure color effect of the image
- */
-int atomisp_color_effect(struct atomisp_sub_device *asd, int flag,
-			 __s32 *effect);
-
-/*
- * Function to configure bad pixel correction
- */
-int atomisp_bad_pixel(struct atomisp_sub_device *asd, int flag,
-		      __s32 *value);
-
-/*
- * Function to configure bad pixel correction params
- */
-int atomisp_bad_pixel_param(struct atomisp_sub_device *asd, int flag,
-			    struct atomisp_dp_config *config);
-
-/*
- * Function to enable/disable video image stablization
- */
-int atomisp_video_stable(struct atomisp_sub_device *asd, int flag,
-			 __s32 *value);
-
-/*
- * Function to configure fixed pattern noise
- */
-int atomisp_fixed_pattern(struct atomisp_sub_device *asd, int flag,
-			  __s32 *value);
-
-/*
- * Function to configure fixed pattern noise table
- */
-int atomisp_fixed_pattern_table(struct atomisp_sub_device *asd,
-				struct v4l2_framebuffer *config);
-
-/*
- * Function to configure false color correction
- */
-int atomisp_false_color(struct atomisp_sub_device *asd, int flag,
-			__s32 *value);
-
-/*
- * Function to configure false color correction params
- */
-int atomisp_false_color_param(struct atomisp_sub_device *asd, int flag,
-			      struct atomisp_de_config *config);
-
-/*
- * Function to configure white balance params
- */
-int atomisp_white_balance_param(struct atomisp_sub_device *asd, int flag,
-				struct atomisp_wb_config *config);
-
-int atomisp_3a_config_param(struct atomisp_sub_device *asd, int flag,
-			    struct atomisp_3a_config *config);
-
-/*
- * Function to setup digital zoom
- */
-int atomisp_digital_zoom(struct atomisp_sub_device *asd, int flag,
-			 __s32 *value);
-
-/*
- * Function  set camera_prefiles.xml current sensor pixel array size
- */
-int atomisp_set_array_res(struct atomisp_sub_device *asd,
-			struct atomisp_resolution  *config);
-
-/*
- * Function to calculate real zoom region for every pipe
- */
-int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd,
-			struct atomisp_css_dz_config   *dz_config,
-			enum atomisp_css_pipe_id css_pipe_id);
-
-int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
-				      struct atomisp_parameters *arg,
-				      struct atomisp_css_params *css_param,
-				      bool from_user);
-
-int atomisp_cp_lsc_table(struct atomisp_sub_device *asd,
-			 struct atomisp_shading_table *source_st,
-			 struct atomisp_css_params *css_param,
-			 bool from_user);
-
-int atomisp_css_cp_dvs2_coefs(struct atomisp_sub_device *asd,
-			      struct ia_css_dvs2_coefficients *coefs,
-			      struct atomisp_css_params *css_param,
-			      bool from_user);
-
-int atomisp_cp_morph_table(struct atomisp_sub_device *asd,
-			   struct atomisp_morph_table *source_morph_table,
-			   struct atomisp_css_params *css_param,
-			   bool from_user);
-
-int atomisp_cp_dvs_6axis_config(struct atomisp_sub_device *asd,
-			struct atomisp_dvs_6axis_config *user_6axis_config,
-			struct atomisp_css_params *css_param,
-			bool from_user);
-
-int atomisp_makeup_css_parameters(struct atomisp_sub_device *asd,
-				  struct atomisp_parameters *arg,
-				  struct atomisp_css_params *css_param);
-
-int atomisp_compare_grid(struct atomisp_sub_device *asd,
-				struct atomisp_grid_info *atomgrid);
-
-int atomisp_get_sensor_mode_data(struct atomisp_sub_device *asd,
-				 struct atomisp_sensor_mode_data *config);
-
-int atomisp_get_fmt(struct video_device *vdev, struct v4l2_format *f);
-
-
-/* This function looks up the closest available resolution. */
-int atomisp_try_fmt(struct video_device *vdev, struct v4l2_format *f,
-						bool *res_overflow);
-
-int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f);
-int atomisp_set_fmt_file(struct video_device *vdev, struct v4l2_format *f);
-
-int atomisp_set_shading_table(struct atomisp_sub_device *asd,
-			      struct atomisp_shading_table *shading_table);
-
-int atomisp_offline_capture_configure(struct atomisp_sub_device *asd,
-				struct atomisp_cont_capture_conf *cvf_config);
-
-int atomisp_ospm_dphy_down(struct atomisp_device *isp);
-int atomisp_ospm_dphy_up(struct atomisp_device *isp);
-int atomisp_exif_makernote(struct atomisp_sub_device *asd,
-			   struct atomisp_makernote_info *config);
-
-void atomisp_free_internal_buffers(struct atomisp_sub_device *asd);
-
-int atomisp_s_ae_window(struct atomisp_sub_device *asd,
-			struct atomisp_ae_window *arg);
-
-int  atomisp_flash_enable(struct atomisp_sub_device *asd,
-			  int num_frames);
-
-int atomisp_freq_scaling(struct atomisp_device *vdev,
-			 enum atomisp_dfs_mode mode,
-			 bool force);
-
-void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
-		      enum atomisp_css_buffer_type buf_type,
-		      enum atomisp_css_pipe_id css_pipe_id,
-		      bool q_buffers, enum atomisp_input_stream_id stream_id);
-
-void atomisp_css_flush(struct atomisp_device *isp);
-int atomisp_source_pad_to_stream_id(struct atomisp_sub_device *asd,
-					   uint16_t source_pad);
-
-/*
- * Events. Only one event has to be exported for now.
- */
-void atomisp_eof_event(struct atomisp_sub_device *asd, uint8_t exp_id);
-
-enum mipi_port_id __get_mipi_port(struct atomisp_device *isp,
-				enum atomisp_camera_port port);
-
-bool atomisp_is_vf_pipe(struct atomisp_video_pipe *pipe);
-
-void atomisp_apply_css_parameters(
-				struct atomisp_sub_device *asd,
-				struct atomisp_css_params *css_param);
-void atomisp_free_css_parameters(struct atomisp_css_params *css_param);
-
-void atomisp_handle_parameter_and_buffer(struct atomisp_video_pipe *pipe);
-
-void atomisp_flush_params_queue(struct atomisp_video_pipe *asd);
-/*
- * Function to do Raw Buffer related operation, after enable Lock Unlock Raw Buffer
- */
-int atomisp_exp_id_unlock(struct atomisp_sub_device *asd, int *exp_id);
-int atomisp_exp_id_capture(struct atomisp_sub_device *asd, int *exp_id);
-
-/*
- * Function to update Raw Buffer bitmap
- */
-int atomisp_set_raw_buffer_bitmap(struct atomisp_sub_device *asd, int exp_id);
-void atomisp_init_raw_buffer_bitmap(struct atomisp_sub_device *asd);
-
-/*
- * Function to enable/disable zoom for capture pipe
- */
-int atomisp_enable_dz_capt_pipe(struct atomisp_sub_device *asd,
-					   unsigned int *enable);
-
-/*
- * Function to get metadata type bu pipe id
- */
-enum atomisp_metadata_type
-atomisp_get_metadata_type(struct atomisp_sub_device *asd,
-			  enum ia_css_pipe_id pipe_id);
-
-/*
- * Function for HAL to inject a fake event to wake up poll thread
- */
-int atomisp_inject_a_fake_event(struct atomisp_sub_device *asd, int *event);
-
-/*
- * Function for HAL to query how many invalid frames at the beginning of ISP
- * pipeline output
- */
-int atomisp_get_invalid_frame_num(struct video_device *vdev,
-			int *invalid_frame_num);
-
-int atomisp_mrfld_power_up(struct atomisp_device *isp);
-int atomisp_mrfld_power_down(struct atomisp_device *isp);
-int atomisp_runtime_suspend(struct device *dev);
-int atomisp_runtime_resume(struct device *dev);
-#endif /* __ATOMISP_CMD_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_common.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_common.h
deleted file mode 100644
index 2558193045a6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_common.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef	__ATOMISP_COMMON_H__
-#define	__ATOMISP_COMMON_H__
-
-#include "../../include/linux/atomisp.h"
-
-#include <linux/v4l2-mediabus.h>
-
-#include <media/videobuf-core.h>
-
-#include "atomisp_compat.h"
-
-#include "ia_css.h"
-
-extern int dbg_level;
-extern int dbg_func;
-extern int mipicsi_flag;
-extern int pad_w;
-extern int pad_h;
-
-#define CSS_DTRACE_VERBOSITY_LEVEL	5	/* Controls trace verbosity */
-#define CSS_DTRACE_VERBOSITY_TIMEOUT	9	/* Verbosity on ISP timeout */
-#define MRFLD_MAX_ZOOM_FACTOR	1024
-#ifdef ISP2401
-#define ATOMISP_CSS_ISP_PIPE_VERSION_2_2    0
-#define ATOMISP_CSS_ISP_PIPE_VERSION_2_7    1
-#endif
-
-#define IS_ISP2401(isp)							\
-	(((isp)->media_dev.hw_revision & ATOMISP_HW_REVISION_MASK)	\
-	 >= (ATOMISP_HW_REVISION_ISP2401_LEGACY << ATOMISP_HW_REVISION_SHIFT))
-
-struct atomisp_format_bridge {
-	unsigned int pixelformat;
-	unsigned int depth;
-	u32 mbus_code;
-	enum atomisp_css_frame_format sh_fmt;
-	unsigned char description[32];	/* the same as struct v4l2_fmtdesc */
-	bool planar;
-};
-
-struct atomisp_fmt {
-	u32 pixelformat;
-	u32 depth;
-	u32 bytesperline;
-	u32 framesize;
-	u32 imagesize;
-	u32 width;
-	u32 height;
-	u32 bayer_order;
-};
-
-struct atomisp_buffer {
-	struct videobuf_buffer	vb;
-};
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat.h
deleted file mode 100644
index 6c829d0a1e4c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat.h
+++ /dev/null
@@ -1,662 +0,0 @@
-/*
- * Support for Clovertrail PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2012 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __ATOMISP_COMPAT_H__
-#define __ATOMISP_COMPAT_H__
-
-#include "atomisp_compat_css20.h"
-
-#include "../../include/linux/atomisp.h"
-#include <media/videobuf-vmalloc.h>
-
-#define CSS_RX_IRQ_INFO_BUFFER_OVERRUN \
-	CSS_ID(CSS_RX_IRQ_INFO_BUFFER_OVERRUN)
-#define CSS_RX_IRQ_INFO_ENTER_SLEEP_MODE \
-	CSS_ID(CSS_RX_IRQ_INFO_ENTER_SLEEP_MODE)
-#define CSS_RX_IRQ_INFO_EXIT_SLEEP_MODE \
-	CSS_ID(CSS_RX_IRQ_INFO_EXIT_SLEEP_MODE)
-#define CSS_RX_IRQ_INFO_ECC_CORRECTED \
-	CSS_ID(CSS_RX_IRQ_INFO_ECC_CORRECTED)
-#define CSS_RX_IRQ_INFO_ERR_SOT \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_SOT)
-#define CSS_RX_IRQ_INFO_ERR_SOT_SYNC \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_SOT_SYNC)
-#define CSS_RX_IRQ_INFO_ERR_CONTROL \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_CONTROL)
-#define CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE)
-#define CSS_RX_IRQ_INFO_ERR_CRC \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_CRC)
-#define CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID)
-#define CSS_RX_IRQ_INFO_ERR_FRAME_SYNC \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_FRAME_SYNC)
-#define CSS_RX_IRQ_INFO_ERR_FRAME_DATA \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_FRAME_DATA)
-#define CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT)
-#define CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC)
-#define CSS_RX_IRQ_INFO_ERR_LINE_SYNC \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_LINE_SYNC)
-#define CSS_RX_IRQ_INFO_INIT_TIMEOUT \
-	CSS_ID(CSS_RX_IRQ_INFO_INIT_TIMEOUT)
-
-#define CSS_IRQ_INFO_CSS_RECEIVER_SOF	CSS_ID(CSS_IRQ_INFO_CSS_RECEIVER_SOF)
-#define CSS_IRQ_INFO_CSS_RECEIVER_EOF	CSS_ID(CSS_IRQ_INFO_CSS_RECEIVER_EOF)
-#define CSS_IRQ_INFO_CSS_RECEIVER_FIFO_OVERFLOW \
-	CSS_ID(CSS_IRQ_INFO_CSS_RECEIVER_FIFO_OVERFLOW)
-#define CSS_EVENT_OUTPUT_FRAME_DONE	CSS_EVENT(OUTPUT_FRAME_DONE)
-#define CSS_EVENT_SEC_OUTPUT_FRAME_DONE	CSS_EVENT(SECOND_OUTPUT_FRAME_DONE)
-#define CSS_EVENT_VF_OUTPUT_FRAME_DONE	CSS_EVENT(VF_OUTPUT_FRAME_DONE)
-#define CSS_EVENT_SEC_VF_OUTPUT_FRAME_DONE	CSS_EVENT(SECOND_VF_OUTPUT_FRAME_DONE)
-#define CSS_EVENT_3A_STATISTICS_DONE	CSS_EVENT(3A_STATISTICS_DONE)
-#define CSS_EVENT_DIS_STATISTICS_DONE	CSS_EVENT(DIS_STATISTICS_DONE)
-#define CSS_EVENT_PIPELINE_DONE		CSS_EVENT(PIPELINE_DONE)
-#define CSS_EVENT_METADATA_DONE		CSS_EVENT(METADATA_DONE)
-#define CSS_EVENT_ACC_STAGE_COMPLETE	CSS_EVENT(ACC_STAGE_COMPLETE)
-#define CSS_EVENT_TIMER			CSS_EVENT(TIMER)
-
-#define CSS_BUFFER_TYPE_METADATA	CSS_ID(CSS_BUFFER_TYPE_METADATA)
-#define CSS_BUFFER_TYPE_3A_STATISTICS	CSS_ID(CSS_BUFFER_TYPE_3A_STATISTICS)
-#define CSS_BUFFER_TYPE_DIS_STATISTICS	CSS_ID(CSS_BUFFER_TYPE_DIS_STATISTICS)
-#define CSS_BUFFER_TYPE_INPUT_FRAME	CSS_ID(CSS_BUFFER_TYPE_INPUT_FRAME)
-#define CSS_BUFFER_TYPE_OUTPUT_FRAME	CSS_ID(CSS_BUFFER_TYPE_OUTPUT_FRAME)
-#define CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME	CSS_ID(CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME)
-#define CSS_BUFFER_TYPE_VF_OUTPUT_FRAME	CSS_ID(CSS_BUFFER_TYPE_VF_OUTPUT_FRAME)
-#define CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME	CSS_ID(CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME)
-#define CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME \
-	CSS_ID(CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME)
-
-#define CSS_FORMAT_RAW_8	CSS_FORMAT(RAW_8)
-#define CSS_FORMAT_RAW_10	CSS_FORMAT(RAW_10)
-#define CSS_FORMAT_RAW_12	CSS_FORMAT(RAW_12)
-#define CSS_FORMAT_RAW_16	CSS_FORMAT(RAW_16)
-
-#define CSS_CAPTURE_MODE_RAW		CSS_ID(CSS_CAPTURE_MODE_RAW)
-#define CSS_CAPTURE_MODE_BAYER		CSS_ID(CSS_CAPTURE_MODE_BAYER)
-#define CSS_CAPTURE_MODE_PRIMARY	CSS_ID(CSS_CAPTURE_MODE_PRIMARY)
-#define CSS_CAPTURE_MODE_ADVANCED	CSS_ID(CSS_CAPTURE_MODE_ADVANCED)
-#define CSS_CAPTURE_MODE_LOW_LIGHT	CSS_ID(CSS_CAPTURE_MODE_LOW_LIGHT)
-
-#define CSS_MORPH_TABLE_NUM_PLANES	CSS_ID(CSS_MORPH_TABLE_NUM_PLANES)
-
-#define CSS_FRAME_FORMAT_NV11		CSS_ID(CSS_FRAME_FORMAT_NV11)
-#define CSS_FRAME_FORMAT_NV12		CSS_ID(CSS_FRAME_FORMAT_NV12)
-#define CSS_FRAME_FORMAT_NV16		CSS_ID(CSS_FRAME_FORMAT_NV16)
-#define CSS_FRAME_FORMAT_NV21		CSS_ID(CSS_FRAME_FORMAT_NV21)
-#define CSS_FRAME_FORMAT_NV61		CSS_ID(CSS_FRAME_FORMAT_NV61)
-#define CSS_FRAME_FORMAT_YV12		CSS_ID(CSS_FRAME_FORMAT_YV12)
-#define CSS_FRAME_FORMAT_YV16		CSS_ID(CSS_FRAME_FORMAT_YV16)
-#define CSS_FRAME_FORMAT_YUV420		CSS_ID(CSS_FRAME_FORMAT_YUV420)
-#define CSS_FRAME_FORMAT_YUV420_16	CSS_ID(CSS_FRAME_FORMAT_YUV420_16)
-#define CSS_FRAME_FORMAT_YUV422		CSS_ID(CSS_FRAME_FORMAT_YUV422)
-#define CSS_FRAME_FORMAT_YUV422_16	CSS_ID(CSS_FRAME_FORMAT_YUV422_16)
-#define CSS_FRAME_FORMAT_UYVY		CSS_ID(CSS_FRAME_FORMAT_UYVY)
-#define CSS_FRAME_FORMAT_YUYV		CSS_ID(CSS_FRAME_FORMAT_YUYV)
-#define CSS_FRAME_FORMAT_YUV444		CSS_ID(CSS_FRAME_FORMAT_YUV444)
-#define CSS_FRAME_FORMAT_YUV_LINE	CSS_ID(CSS_FRAME_FORMAT_YUV_LINE)
-#define CSS_FRAME_FORMAT_RAW		CSS_ID(CSS_FRAME_FORMAT_RAW)
-#define CSS_FRAME_FORMAT_RGB565		CSS_ID(CSS_FRAME_FORMAT_RGB565)
-#define CSS_FRAME_FORMAT_PLANAR_RGB888	CSS_ID(CSS_FRAME_FORMAT_PLANAR_RGB888)
-#define CSS_FRAME_FORMAT_RGBA888	CSS_ID(CSS_FRAME_FORMAT_RGBA888)
-#define CSS_FRAME_FORMAT_QPLANE6	CSS_ID(CSS_FRAME_FORMAT_QPLANE6)
-#define CSS_FRAME_FORMAT_BINARY_8	CSS_ID(CSS_FRAME_FORMAT_BINARY_8)
-
-struct atomisp_device;
-struct atomisp_sub_device;
-struct video_device;
-enum atomisp_input_stream_id;
-
-struct atomisp_metadata_buf {
-	struct ia_css_metadata *metadata;
-	void *md_vptr;
-	struct list_head list;
-};
-
-void atomisp_css_debug_dump_sp_sw_debug_info(void);
-void atomisp_css_debug_dump_debug_info(const char *context);
-void atomisp_css_debug_set_dtrace_level(const unsigned int trace_level);
-
-void atomisp_store_uint32(hrt_address addr, uint32_t data);
-void atomisp_load_uint32(hrt_address addr, uint32_t *data);
-
-int atomisp_css_init(struct atomisp_device *isp);
-
-void atomisp_css_uninit(struct atomisp_device *isp);
-
-void atomisp_css_suspend(struct atomisp_device *isp);
-
-int atomisp_css_resume(struct atomisp_device *isp);
-
-void atomisp_css_init_struct(struct atomisp_sub_device *asd);
-
-int atomisp_css_irq_translate(struct atomisp_device *isp,
-			      unsigned int *infos);
-
-void atomisp_css_rx_get_irq_info(enum mipi_port_id port,
-					unsigned int *infos);
-
-void atomisp_css_rx_clear_irq_info(enum mipi_port_id port,
-					unsigned int infos);
-
-int atomisp_css_irq_enable(struct atomisp_device *isp,
-			   enum atomisp_css_irq_info info, bool enable);
-
-int atomisp_q_video_buffer_to_css(struct atomisp_sub_device *asd,
-			struct videobuf_vmalloc_memory *vm_mem,
-			enum atomisp_input_stream_id stream_id,
-			enum atomisp_css_buffer_type css_buf_type,
-			enum atomisp_css_pipe_id css_pipe_id);
-
-int atomisp_q_s3a_buffer_to_css(struct atomisp_sub_device *asd,
-			struct atomisp_s3a_buf *s3a_buf,
-			enum atomisp_input_stream_id stream_id,
-			enum atomisp_css_pipe_id css_pipe_id);
-
-int atomisp_q_metadata_buffer_to_css(struct atomisp_sub_device *asd,
-			struct atomisp_metadata_buf *metadata_buf,
-			enum atomisp_input_stream_id stream_id,
-			enum atomisp_css_pipe_id css_pipe_id);
-
-int atomisp_q_dis_buffer_to_css(struct atomisp_sub_device *asd,
-			struct atomisp_dis_buf *dis_buf,
-			enum atomisp_input_stream_id stream_id,
-			enum atomisp_css_pipe_id css_pipe_id);
-
-void atomisp_css_mmu_invalidate_cache(void);
-
-void atomisp_css_mmu_invalidate_tlb(void);
-
-int atomisp_css_start(struct atomisp_sub_device *asd,
-		      enum atomisp_css_pipe_id pipe_id, bool in_reset);
-
-void atomisp_css_update_isp_params(struct atomisp_sub_device *asd);
-void atomisp_css_update_isp_params_on_pipe(struct atomisp_sub_device *asd,
-					struct ia_css_pipe *pipe);
-
-int atomisp_css_queue_buffer(struct atomisp_sub_device *asd,
-			     enum atomisp_input_stream_id stream_id,
-			     enum atomisp_css_pipe_id pipe_id,
-			     enum atomisp_css_buffer_type buf_type,
-			     struct atomisp_css_buffer *isp_css_buffer);
-
-int atomisp_css_dequeue_buffer(struct atomisp_sub_device *asd,
-				enum atomisp_input_stream_id stream_id,
-				enum atomisp_css_pipe_id pipe_id,
-				enum atomisp_css_buffer_type buf_type,
-				struct atomisp_css_buffer *isp_css_buffer);
-
-int atomisp_css_allocate_stat_buffers(struct atomisp_sub_device *asd,
-				      uint16_t stream_id,
-				      struct atomisp_s3a_buf *s3a_buf,
-				      struct atomisp_dis_buf *dis_buf,
-				      struct atomisp_metadata_buf *md_buf);
-
-void atomisp_css_free_stat_buffers(struct atomisp_sub_device *asd);
-
-void atomisp_css_free_3a_buffer(struct atomisp_s3a_buf *s3a_buf);
-
-void atomisp_css_free_dis_buffer(struct atomisp_dis_buf *dis_buf);
-
-void atomisp_css_free_metadata_buffer(struct atomisp_metadata_buf *metadata_buf);
-
-int atomisp_css_get_grid_info(struct atomisp_sub_device *asd,
-				enum atomisp_css_pipe_id pipe_id,
-				int source_pad);
-
-int atomisp_alloc_3a_output_buf(struct atomisp_sub_device *asd);
-
-int atomisp_alloc_dis_coef_buf(struct atomisp_sub_device *asd);
-
-int atomisp_alloc_metadata_output_buf(struct atomisp_sub_device *asd);
-
-void atomisp_free_metadata_output_buf(struct atomisp_sub_device *asd);
-
-void atomisp_css_get_dis_statistics(struct atomisp_sub_device *asd,
-				    struct atomisp_css_buffer *isp_css_buffer,
-				    struct ia_css_isp_dvs_statistics_map *dvs_map);
-
-int atomisp_css_dequeue_event(struct atomisp_css_event *current_event);
-
-void atomisp_css_temp_pipe_to_pipe_id(struct atomisp_sub_device *asd,
-				      struct atomisp_css_event *current_event);
-
-int atomisp_css_isys_set_resolution(struct atomisp_sub_device *asd,
-				    enum atomisp_input_stream_id stream_id,
-				    struct v4l2_mbus_framefmt *ffmt,
-				    int isys_stream);
-
-void atomisp_css_isys_set_link(struct atomisp_sub_device *asd,
-			       enum atomisp_input_stream_id stream_id,
-			       int link,
-			       int isys_stream);
-
-void atomisp_css_isys_set_valid(struct atomisp_sub_device *asd,
-				enum atomisp_input_stream_id stream_id,
-				bool valid,
-				int isys_stream);
-
-void atomisp_css_isys_set_format(struct atomisp_sub_device *asd,
-				 enum atomisp_input_stream_id stream_id,
-				 enum atomisp_input_format format,
-				 int isys_stream);
-
-int atomisp_css_set_default_isys_config(struct atomisp_sub_device *asd,
-					enum atomisp_input_stream_id stream_id,
-					struct v4l2_mbus_framefmt *ffmt);
-
-int atomisp_css_isys_two_stream_cfg(struct atomisp_sub_device *asd,
-				    enum atomisp_input_stream_id stream_id,
-				    enum atomisp_input_format input_format);
-
-void atomisp_css_isys_two_stream_cfg_update_stream1(
-				    struct atomisp_sub_device *asd,
-				    enum atomisp_input_stream_id stream_id,
-				    enum atomisp_input_format input_format,
-				    unsigned int width, unsigned int height);
-
-void atomisp_css_isys_two_stream_cfg_update_stream2(
-				    struct atomisp_sub_device *asd,
-				    enum atomisp_input_stream_id stream_id,
-				    enum atomisp_input_format input_format,
-				    unsigned int width, unsigned int height);
-
-int atomisp_css_input_set_resolution(struct atomisp_sub_device *asd,
-					enum atomisp_input_stream_id stream_id,
-					struct v4l2_mbus_framefmt *ffmt);
-
-void atomisp_css_input_set_binning_factor(struct atomisp_sub_device *asd,
-					enum atomisp_input_stream_id stream_id,
-					unsigned int bin_factor);
-
-void atomisp_css_input_set_bayer_order(struct atomisp_sub_device *asd,
-				enum atomisp_input_stream_id stream_id,
-				enum atomisp_css_bayer_order bayer_order);
-
-void atomisp_css_input_set_format(struct atomisp_sub_device *asd,
-				enum atomisp_input_stream_id stream_id,
-				enum atomisp_input_format format);
-
-int atomisp_css_input_set_effective_resolution(
-					struct atomisp_sub_device *asd,
-					enum atomisp_input_stream_id stream_id,
-					unsigned int width,
-					unsigned int height);
-
-void atomisp_css_video_set_dis_envelope(struct atomisp_sub_device *asd,
-					unsigned int dvs_w, unsigned int dvs_h);
-
-void atomisp_css_input_set_two_pixels_per_clock(
-					struct atomisp_sub_device *asd,
-					bool two_ppc);
-
-void atomisp_css_enable_raw_binning(struct atomisp_sub_device *asd,
-					bool enable);
-
-void atomisp_css_enable_dz(struct atomisp_sub_device *asd, bool enable);
-
-void atomisp_css_capture_set_mode(struct atomisp_sub_device *asd,
-				enum atomisp_css_capture_mode mode);
-
-void atomisp_css_input_set_mode(struct atomisp_sub_device *asd,
-				enum atomisp_css_input_mode mode);
-
-void atomisp_css_capture_enable_online(struct atomisp_sub_device *asd,
-				unsigned short stream_index, bool enable);
-
-void atomisp_css_preview_enable_online(struct atomisp_sub_device *asd,
-				unsigned short stream_index, bool enable);
-
-void atomisp_css_video_enable_online(struct atomisp_sub_device *asd,
-							bool enable);
-
-void atomisp_css_enable_continuous(struct atomisp_sub_device *asd,
-							bool enable);
-
-void atomisp_css_enable_cvf(struct atomisp_sub_device *asd,
-							bool enable);
-
-int atomisp_css_input_configure_port(struct atomisp_sub_device *asd,
-				enum mipi_port_id port,
-				unsigned int num_lanes,
-				unsigned int timeout,
-				unsigned int mipi_freq,
-				enum atomisp_input_format metadata_format,
-				unsigned int metadata_width,
-				unsigned int metadata_height);
-
-int atomisp_css_frame_allocate(struct atomisp_css_frame **frame,
-				unsigned int width, unsigned int height,
-				enum atomisp_css_frame_format format,
-				unsigned int padded_width,
-				unsigned int raw_bit_depth);
-
-int atomisp_css_frame_allocate_from_info(struct atomisp_css_frame **frame,
-				const struct atomisp_css_frame_info *info);
-
-void atomisp_css_frame_free(struct atomisp_css_frame *frame);
-
-int atomisp_css_frame_map(struct atomisp_css_frame **frame,
-				const struct atomisp_css_frame_info *info,
-				const void *data, uint16_t attribute,
-				void *context);
-
-int atomisp_css_set_black_frame(struct atomisp_sub_device *asd,
-			const struct atomisp_css_frame *raw_black_frame);
-
-int atomisp_css_allocate_continuous_frames(bool init_time,
-			struct atomisp_sub_device *asd);
-
-void atomisp_css_update_continuous_frames(struct atomisp_sub_device *asd);
-
-void atomisp_create_pipes_stream(struct atomisp_sub_device *asd);
-void atomisp_destroy_pipes_stream_force(struct atomisp_sub_device *asd);
-
-int atomisp_css_stop(struct atomisp_sub_device *asd,
-			enum atomisp_css_pipe_id pipe_id, bool in_reset);
-
-int atomisp_css_continuous_set_num_raw_frames(
-					struct atomisp_sub_device *asd,
-					int num_frames);
-
-void atomisp_css_disable_vf_pp(struct atomisp_sub_device *asd,
-			       bool disable);
-
-int atomisp_css_copy_configure_output(struct atomisp_sub_device *asd,
-				unsigned int stream_index,
-				unsigned int width, unsigned int height,
-				unsigned int padded_width,
-				enum atomisp_css_frame_format format);
-
-int atomisp_css_yuvpp_configure_output(struct atomisp_sub_device *asd,
-				unsigned int stream_index,
-				unsigned int width, unsigned int height,
-				unsigned int padded_width,
-				enum atomisp_css_frame_format format);
-
-int atomisp_css_yuvpp_configure_viewfinder(
-				struct atomisp_sub_device *asd,
-				unsigned int stream_index,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format format);
-
-int atomisp_css_yuvpp_get_output_frame_info(
-					struct atomisp_sub_device *asd,
-					unsigned int stream_index,
-					struct atomisp_css_frame_info *info);
-
-int atomisp_css_yuvpp_get_viewfinder_frame_info(
-					struct atomisp_sub_device *asd,
-					unsigned int stream_index,
-					struct atomisp_css_frame_info *info);
-
-int atomisp_css_preview_configure_output(struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format format);
-
-int atomisp_css_capture_configure_output(struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format format);
-
-int atomisp_css_video_configure_output(struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format format);
-
-int atomisp_get_css_frame_info(struct atomisp_sub_device *asd,
-				uint16_t source_pad,
-				struct atomisp_css_frame_info *frame_info);
-
-int atomisp_css_video_configure_viewfinder(struct atomisp_sub_device *asd,
-					unsigned int width, unsigned int height,
-					unsigned int min_width,
-					enum atomisp_css_frame_format format);
-
-int atomisp_css_capture_configure_viewfinder(
-					struct atomisp_sub_device *asd,
-					unsigned int width, unsigned int height,
-					unsigned int min_width,
-					enum atomisp_css_frame_format format);
-
-int atomisp_css_video_get_viewfinder_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info);
-
-int atomisp_css_capture_get_viewfinder_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info);
-
-int atomisp_css_copy_get_output_frame_info(
-					struct atomisp_sub_device *asd,
-					unsigned int stream_index,
-					struct atomisp_css_frame_info *info);
-
-int atomisp_css_capture_get_output_raw_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info);
-
-int atomisp_css_preview_get_output_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info);
-
-int atomisp_css_capture_get_output_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info);
-
-int atomisp_css_video_get_output_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info);
-
-int atomisp_css_preview_configure_pp_input(
-				struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height);
-
-int atomisp_css_capture_configure_pp_input(
-				struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height);
-
-int atomisp_css_video_configure_pp_input(
-				struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height);
-
-int atomisp_css_offline_capture_configure(struct atomisp_sub_device *asd,
-			int num_captures, unsigned int skip, int offset);
-int atomisp_css_exp_id_capture(struct atomisp_sub_device *asd, int exp_id);
-int atomisp_css_exp_id_unlock(struct atomisp_sub_device *asd, int exp_id);
-
-int atomisp_css_capture_enable_xnr(struct atomisp_sub_device *asd,
-				   bool enable);
-
-void atomisp_css_send_input_frame(struct atomisp_sub_device *asd,
-				  unsigned short *data, unsigned int width,
-				  unsigned int height);
-
-bool atomisp_css_isp_has_started(void);
-
-void atomisp_css_request_flash(struct atomisp_sub_device *asd);
-
-void atomisp_css_set_wb_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_wb_config *wb_config);
-
-void atomisp_css_set_ob_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ob_config *ob_config);
-
-void atomisp_css_set_dp_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_dp_config *dp_config);
-
-void atomisp_css_set_de_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_de_config *de_config);
-
-void atomisp_css_set_dz_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_dz_config *dz_config);
-
-void atomisp_css_set_default_de_config(struct atomisp_sub_device *asd);
-
-void atomisp_css_set_ce_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ce_config *ce_config);
-
-void atomisp_css_set_nr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_nr_config *nr_config);
-
-void atomisp_css_set_ee_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ee_config *ee_config);
-
-void atomisp_css_set_tnr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_tnr_config *tnr_config);
-
-void atomisp_css_set_cc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_cc_config *cc_config);
-
-void atomisp_css_set_macc_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_macc_table *macc_table);
-
-void atomisp_css_set_gamma_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_gamma_table *gamma_table);
-
-void atomisp_css_set_ctc_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_ctc_table *ctc_table);
-
-void atomisp_css_set_gc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_gc_config *gc_config);
-
-void atomisp_css_set_3a_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_3a_config *s3a_config);
-
-void atomisp_css_video_set_dis_vector(struct atomisp_sub_device *asd,
-				struct atomisp_dis_vector *vector);
-
-void atomisp_css_set_dvs2_coefs(struct atomisp_sub_device *asd,
-				struct ia_css_dvs2_coefficients *coefs);
-
-int atomisp_css_set_dis_coefs(struct atomisp_sub_device *asd,
-			  struct atomisp_dis_coefficients *coefs);
-
-void atomisp_css_set_zoom_factor(struct atomisp_sub_device *asd,
-					unsigned int zoom);
-
-int atomisp_css_get_wb_config(struct atomisp_sub_device *asd,
-			struct atomisp_wb_config *config);
-
-int atomisp_css_get_ob_config(struct atomisp_sub_device *asd,
-			struct atomisp_ob_config *config);
-
-int atomisp_css_get_dp_config(struct atomisp_sub_device *asd,
-			struct atomisp_dp_config *config);
-
-int atomisp_css_get_de_config(struct atomisp_sub_device *asd,
-			struct atomisp_de_config *config);
-
-int atomisp_css_get_nr_config(struct atomisp_sub_device *asd,
-			struct atomisp_nr_config *config);
-
-int atomisp_css_get_ee_config(struct atomisp_sub_device *asd,
-			struct atomisp_ee_config *config);
-
-int atomisp_css_get_tnr_config(struct atomisp_sub_device *asd,
-			struct atomisp_tnr_config *config);
-
-int atomisp_css_get_ctc_table(struct atomisp_sub_device *asd,
-			struct atomisp_ctc_table *config);
-
-int atomisp_css_get_gamma_table(struct atomisp_sub_device *asd,
-			struct atomisp_gamma_table *config);
-
-int atomisp_css_get_gc_config(struct atomisp_sub_device *asd,
-			struct atomisp_gc_config *config);
-
-int atomisp_css_get_3a_config(struct atomisp_sub_device *asd,
-			struct atomisp_3a_config *config);
-
-int atomisp_css_get_formats_config(struct atomisp_sub_device *asd,
-			struct atomisp_formats_config *formats_config);
-
-void atomisp_css_set_formats_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_formats_config *formats_config);
-
-int atomisp_css_get_zoom_factor(struct atomisp_sub_device *asd,
-					unsigned int *zoom);
-
-struct atomisp_css_shading_table *atomisp_css_shading_table_alloc(
-				unsigned int width, unsigned int height);
-
-void atomisp_css_set_shading_table(struct atomisp_sub_device *asd,
-				struct atomisp_css_shading_table *table);
-
-void atomisp_css_shading_table_free(struct atomisp_css_shading_table *table);
-
-struct atomisp_css_morph_table *atomisp_css_morph_table_allocate(
-				unsigned int width, unsigned int height);
-
-void atomisp_css_set_morph_table(struct atomisp_sub_device *asd,
-				struct atomisp_css_morph_table *table);
-
-void atomisp_css_get_morph_table(struct atomisp_sub_device *asd,
-				struct atomisp_css_morph_table *table);
-
-void atomisp_css_morph_table_free(struct atomisp_css_morph_table *table);
-
-void atomisp_css_set_cont_prev_start_time(struct atomisp_device *isp,
-					unsigned int overlap);
-
-int atomisp_css_get_dis_stat(struct atomisp_sub_device *asd,
-			 struct atomisp_dis_statistics *stats);
-
-int atomisp_css_update_stream(struct atomisp_sub_device *asd);
-
-int atomisp_css_create_acc_pipe(struct atomisp_sub_device *asd);
-
-int atomisp_css_start_acc_pipe(struct atomisp_sub_device *asd);
-
-int atomisp_css_stop_acc_pipe(struct atomisp_sub_device *asd);
-
-void atomisp_css_destroy_acc_pipe(struct atomisp_sub_device *asd);
-
-int atomisp_css_load_acc_extension(struct atomisp_sub_device *asd,
-					struct atomisp_css_fw_info *fw,
-					enum atomisp_css_pipe_id pipe_id,
-					unsigned int type);
-
-void atomisp_css_unload_acc_extension(struct atomisp_sub_device *asd,
-					struct atomisp_css_fw_info *fw,
-					enum atomisp_css_pipe_id pipe_id);
-
-int atomisp_css_wait_acc_finish(struct atomisp_sub_device *asd);
-
-void atomisp_css_acc_done(struct atomisp_sub_device *asd);
-
-int atomisp_css_load_acc_binary(struct atomisp_sub_device *asd,
-					struct atomisp_css_fw_info *fw,
-					unsigned int index);
-
-void atomisp_css_unload_acc_binary(struct atomisp_sub_device *asd);
-
-struct atomisp_acc_fw;
-int atomisp_css_set_acc_parameters(struct atomisp_acc_fw *acc_fw);
-
-int atomisp_css_isr_thread(struct atomisp_device *isp,
-			   bool *frame_done_found,
-			   bool *css_pipe_done);
-
-bool atomisp_css_valid_sof(struct atomisp_device *isp);
-
-void atomisp_en_dz_capt_pipe(struct atomisp_sub_device *asd, bool enable);
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.c
deleted file mode 100644
index f668c68dc33a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.c
+++ /dev/null
@@ -1,4704 +0,0 @@
-/*
- * Support for Clovertrail PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#include <media/videobuf-vmalloc.h>
-#include <media/v4l2-dev.h>
-#include <media/v4l2-event.h>
-
-#include "mmu/isp_mmu.h"
-#include "mmu/sh_mmu_mrfld.h"
-#include "hmm/hmm_bo.h"
-#include "hmm/hmm.h"
-
-#include "atomisp_compat.h"
-#include "atomisp_internal.h"
-#include "atomisp_cmd.h"
-#include "atomisp-regs.h"
-#include "atomisp_fops.h"
-#include "atomisp_ioctl.h"
-#include "atomisp_acc.h"
-
-#include "hrt/hive_isp_css_mm_hrt.h"
-
-#include <asm/intel-mid.h>
-
-#include "ia_css_debug.h"
-#include "ia_css_isp_param.h"
-#include "sh_css_hrt.h"
-#include "ia_css_isys.h"
-
-#include <linux/pm_runtime.h>
-
-/* Assume max number of ACC stages */
-#define MAX_ACC_STAGES	20
-
-/* Ideally, this should come from CSS headers */
-#define NO_LINK -1
-
-/*
- * to serialize MMIO access , this is due to ISP2400 silicon issue Sighting
- * #4684168, if concurrency access happened, system may hard hang.
- */
-static DEFINE_SPINLOCK(mmio_lock);
-
-enum frame_info_type {
-	ATOMISP_CSS_VF_FRAME,
-	ATOMISP_CSS_SECOND_VF_FRAME,
-	ATOMISP_CSS_OUTPUT_FRAME,
-	ATOMISP_CSS_SECOND_OUTPUT_FRAME,
-	ATOMISP_CSS_RAW_FRAME,
-};
-
-struct bayer_ds_factor {
-	unsigned int numerator;
-	unsigned int denominator;
-};
-
-void atomisp_css_debug_dump_sp_sw_debug_info(void)
-{
-	ia_css_debug_dump_sp_sw_debug_info();
-}
-
-void atomisp_css_debug_dump_debug_info(const char *context)
-{
-	ia_css_debug_dump_debug_info(context);
-}
-
-void atomisp_css_debug_set_dtrace_level(const unsigned int trace_level)
-{
-	ia_css_debug_set_dtrace_level(trace_level);
-}
-
-unsigned int atomisp_css_debug_get_dtrace_level(void)
-{
-	return ia_css_debug_trace_level;
-}
-
-static void atomisp_css2_hw_store_8(hrt_address addr, uint8_t data)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mmio_lock, flags);
-	_hrt_master_port_store_8(addr, data);
-	spin_unlock_irqrestore(&mmio_lock, flags);
-}
-
-static void atomisp_css2_hw_store_16(hrt_address addr, uint16_t data)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mmio_lock, flags);
-	_hrt_master_port_store_16(addr, data);
-	spin_unlock_irqrestore(&mmio_lock, flags);
-}
-
-static void atomisp_css2_hw_store_32(hrt_address addr, uint32_t data)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mmio_lock, flags);
-	_hrt_master_port_store_32(addr, data);
-	spin_unlock_irqrestore(&mmio_lock, flags);
-}
-
-static uint8_t atomisp_css2_hw_load_8(hrt_address addr)
-{
-	unsigned long flags;
-	uint8_t ret;
-
-	spin_lock_irqsave(&mmio_lock, flags);
-	ret = _hrt_master_port_load_8(addr);
-	spin_unlock_irqrestore(&mmio_lock, flags);
-	return ret;
-}
-
-static uint16_t atomisp_css2_hw_load_16(hrt_address addr)
-{
-	unsigned long flags;
-	uint16_t ret;
-
-	spin_lock_irqsave(&mmio_lock, flags);
-	ret = _hrt_master_port_load_16(addr);
-	spin_unlock_irqrestore(&mmio_lock, flags);
-	return ret;
-}
-
-static uint32_t atomisp_css2_hw_load_32(hrt_address addr)
-{
-	unsigned long flags;
-	uint32_t ret;
-
-	spin_lock_irqsave(&mmio_lock, flags);
-	ret = _hrt_master_port_load_32(addr);
-	spin_unlock_irqrestore(&mmio_lock, flags);
-	return ret;
-}
-
-static void atomisp_css2_hw_store(hrt_address addr,
-				  const void *from, uint32_t n)
-{
-	unsigned long flags;
-	unsigned int i;
-	unsigned int _to = (unsigned int)addr;
-	const char *_from = (const char *)from;
-
-	spin_lock_irqsave(&mmio_lock, flags);
-	for (i = 0; i < n; i++, _to++, _from++)
-		_hrt_master_port_store_8(_to , *_from);
-	spin_unlock_irqrestore(&mmio_lock, flags);
-}
-
-static void atomisp_css2_hw_load(hrt_address addr, void *to, uint32_t n)
-{
-	unsigned long flags;
-	unsigned int i;
-	char *_to = (char *)to;
-	unsigned int _from = (unsigned int)addr;
-
-	spin_lock_irqsave(&mmio_lock, flags);
-	for (i = 0; i < n; i++, _to++, _from++)
-		*_to = _hrt_master_port_load_8(_from);
-	spin_unlock_irqrestore(&mmio_lock, flags);
-}
-
-static int atomisp_css2_dbg_print(const char *fmt, va_list args)
-{
-	vprintk(fmt, args);
-	return 0;
-}
-
-static int atomisp_css2_dbg_ftrace_print(const char *fmt, va_list args)
-{
-	ftrace_vprintk(fmt, args);
-	return 0;
-}
-
-static int atomisp_css2_err_print(const char *fmt, va_list args)
-{
-	vprintk(fmt, args);
-	return 0;
-}
-
-void atomisp_store_uint32(hrt_address addr, uint32_t data)
-{
-	atomisp_css2_hw_store_32(addr, data);
-}
-
-void atomisp_load_uint32(hrt_address addr, uint32_t *data)
-{
-	*data = atomisp_css2_hw_load_32(addr);
-}
-static int hmm_get_mmu_base_addr(unsigned int *mmu_base_addr)
-{
-	if (sh_mmu_mrfld.get_pd_base == NULL) {
-		dev_err(atomisp_dev, "get mmu base address failed.\n");
-		return -EINVAL;
-	}
-
-	*mmu_base_addr = sh_mmu_mrfld.get_pd_base(&bo_device.mmu,
-					bo_device.mmu.base_address);
-	return 0;
-}
-
-static void atomisp_isp_parameters_clean_up(
-				struct atomisp_css_isp_config *config)
-{
-	/*
-	 * Set NULL to configs pointer to avoid they are set into isp again when
-	 * some configs are changed and need to be updated later.
-	 */
-	memset(config, 0, sizeof(*config));
-}
-
-static void __dump_pipe_config(struct atomisp_sub_device *asd,
-			       struct atomisp_stream_env *stream_env,
-			       unsigned int pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-
-	if (stream_env->pipes[pipe_id]) {
-		struct ia_css_pipe_config *p_config;
-		struct ia_css_pipe_extra_config *pe_config;
-
-		p_config = &stream_env->pipe_configs[pipe_id];
-		pe_config = &stream_env->pipe_extra_configs[pipe_id];
-		dev_dbg(isp->dev, "dumping pipe[%d] config:\n", pipe_id);
-		dev_dbg(isp->dev,
-			 "pipe_config.pipe_mode:%d.\n", p_config->mode);
-		dev_dbg(isp->dev,
-			 "pipe_config.output_info[0] w=%d, h=%d.\n",
-			 p_config->output_info[0].res.width,
-			 p_config->output_info[0].res.height);
-		dev_dbg(isp->dev,
-			 "pipe_config.vf_pp_in_res w=%d, h=%d.\n",
-			 p_config->vf_pp_in_res.width,
-			 p_config->vf_pp_in_res.height);
-		dev_dbg(isp->dev,
-			 "pipe_config.capt_pp_in_res w=%d, h=%d.\n",
-			 p_config->capt_pp_in_res.width,
-			 p_config->capt_pp_in_res.height);
-		dev_dbg(isp->dev,
-			 "pipe_config.output.padded w=%d.\n",
-			 p_config->output_info[0].padded_width);
-		dev_dbg(isp->dev,
-			 "pipe_config.vf_output_info[0] w=%d, h=%d.\n",
-			 p_config->vf_output_info[0].res.width,
-			 p_config->vf_output_info[0].res.height);
-		dev_dbg(isp->dev,
-			 "pipe_config.bayer_ds_out_res w=%d, h=%d.\n",
-			 p_config->bayer_ds_out_res.width,
-			 p_config->bayer_ds_out_res.height);
-		dev_dbg(isp->dev,
-			 "pipe_config.envelope w=%d, h=%d.\n",
-			 p_config->dvs_envelope.width,
-			 p_config->dvs_envelope.height);
-		dev_dbg(isp->dev,
-			 "pipe_config.dvs_frame_delay=%d.\n",
-			 p_config->dvs_frame_delay);
-		dev_dbg(isp->dev,
-			 "pipe_config.isp_pipe_version:%d.\n",
-			p_config->isp_pipe_version);
-		dev_dbg(isp->dev,
-			 "pipe_config.acc_extension=%p.\n",
-			 p_config->acc_extension);
-		dev_dbg(isp->dev,
-			 "pipe_config.acc_stages=%p.\n",
-			 p_config->acc_stages);
-		dev_dbg(isp->dev,
-			 "pipe_config.num_acc_stages=%d.\n",
-			 p_config->num_acc_stages);
-		dev_dbg(isp->dev,
-			 "pipe_config.acc_num_execs=%d.\n",
-			 p_config->acc_num_execs);
-		dev_dbg(isp->dev,
-			 "pipe_config.default_capture_config.capture_mode=%d.\n",
-			 p_config->default_capture_config.mode);
-		dev_dbg(isp->dev,
-			 "pipe_config.enable_dz=%d.\n",
-			 p_config->enable_dz);
-		dev_dbg(isp->dev,
-			 "pipe_config.default_capture_config.enable_xnr=%d.\n",
-			 p_config->default_capture_config.enable_xnr);
-		dev_dbg(isp->dev,
-			 "dumping pipe[%d] extra config:\n", pipe_id);
-		dev_dbg(isp->dev,
-			 "pipe_extra_config.enable_raw_binning:%d.\n",
-			 pe_config->enable_raw_binning);
-		dev_dbg(isp->dev,
-			 "pipe_extra_config.enable_yuv_ds:%d.\n",
-			 pe_config->enable_yuv_ds);
-		dev_dbg(isp->dev,
-			 "pipe_extra_config.enable_high_speed:%d.\n",
-			 pe_config->enable_high_speed);
-		dev_dbg(isp->dev,
-			 "pipe_extra_config.enable_dvs_6axis:%d.\n",
-			 pe_config->enable_dvs_6axis);
-		dev_dbg(isp->dev,
-			 "pipe_extra_config.enable_reduced_pipe:%d.\n",
-			 pe_config->enable_reduced_pipe);
-		dev_dbg(isp->dev,
-			 "pipe_(extra_)config.enable_dz:%d.\n",
-			 p_config->enable_dz);
-		dev_dbg(isp->dev,
-			 "pipe_extra_config.disable_vf_pp:%d.\n",
-			 pe_config->disable_vf_pp);
-	}
-}
-
-static void __dump_stream_config(struct atomisp_sub_device *asd,
-				struct atomisp_stream_env *stream_env)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct ia_css_stream_config *s_config;
-	int j;
-	bool valid_stream = false;
-
-	for (j = 0; j < IA_CSS_PIPE_ID_NUM; j++) {
-		if (stream_env->pipes[j]) {
-			__dump_pipe_config(asd, stream_env, j);
-			valid_stream = true;
-		}
-	}
-	if (!valid_stream)
-		return;
-	s_config = &stream_env->stream_config;
-	dev_dbg(isp->dev, "stream_config.mode=%d.\n", s_config->mode);
-
-	if (s_config->mode == IA_CSS_INPUT_MODE_SENSOR ||
-	    s_config->mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) {
-		dev_dbg(isp->dev, "stream_config.source.port.port=%d.\n",
-				s_config->source.port.port);
-		dev_dbg(isp->dev, "stream_config.source.port.num_lanes=%d.\n",
-				s_config->source.port.num_lanes);
-		dev_dbg(isp->dev, "stream_config.source.port.timeout=%d.\n",
-				s_config->source.port.timeout);
-		dev_dbg(isp->dev, "stream_config.source.port.rxcount=0x%x.\n",
-				s_config->source.port.rxcount);
-		dev_dbg(isp->dev, "stream_config.source.port.compression.type=%d.\n",
-				s_config->source.port.compression.type);
-		dev_dbg(isp->dev, "stream_config.source.port.compression.compressed_bits_per_pixel=%d.\n",
-				s_config->source.port.compression.
-				compressed_bits_per_pixel);
-		dev_dbg(isp->dev, "stream_config.source.port.compression.uncompressed_bits_per_pixel=%d.\n",
-				s_config->source.port.compression.
-				uncompressed_bits_per_pixel);
-	} else if (s_config->mode == IA_CSS_INPUT_MODE_TPG) {
-		dev_dbg(isp->dev, "stream_config.source.tpg.id=%d.\n",
-				s_config->source.tpg.id);
-		dev_dbg(isp->dev, "stream_config.source.tpg.mode=%d.\n",
-				s_config->source.tpg.mode);
-		dev_dbg(isp->dev, "stream_config.source.tpg.x_mask=%d.\n",
-				s_config->source.tpg.x_mask);
-		dev_dbg(isp->dev, "stream_config.source.tpg.x_delta=%d.\n",
-				s_config->source.tpg.x_delta);
-		dev_dbg(isp->dev, "stream_config.source.tpg.y_mask=%d.\n",
-				s_config->source.tpg.y_mask);
-		dev_dbg(isp->dev, "stream_config.source.tpg.y_delta=%d.\n",
-				s_config->source.tpg.y_delta);
-		dev_dbg(isp->dev, "stream_config.source.tpg.xy_mask=%d.\n",
-				s_config->source.tpg.xy_mask);
-	} else if (s_config->mode == IA_CSS_INPUT_MODE_PRBS) {
-		dev_dbg(isp->dev, "stream_config.source.prbs.id=%d.\n",
-				s_config->source.prbs.id);
-		dev_dbg(isp->dev, "stream_config.source.prbs.h_blank=%d.\n",
-				s_config->source.prbs.h_blank);
-		dev_dbg(isp->dev, "stream_config.source.prbs.v_blank=%d.\n",
-				s_config->source.prbs.v_blank);
-		dev_dbg(isp->dev, "stream_config.source.prbs.seed=%d.\n",
-				s_config->source.prbs.seed);
-		dev_dbg(isp->dev, "stream_config.source.prbs.seed1=%d.\n",
-				s_config->source.prbs.seed1);
-	}
-
-	for (j = 0; j < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH; j++) {
-		dev_dbg(isp->dev, "stream_configisys_config[%d].input_res w=%d, h=%d.\n",
-			j,
-			s_config->isys_config[j].input_res.width,
-			s_config->isys_config[j].input_res.height);
-
-		dev_dbg(isp->dev, "stream_configisys_config[%d].linked_isys_stream_id=%d\n",
-			j,
-			s_config->isys_config[j].linked_isys_stream_id);
-
-		dev_dbg(isp->dev, "stream_configisys_config[%d].format=%d\n",
-			j,
-			s_config->isys_config[j].format);
-
-		dev_dbg(isp->dev, "stream_configisys_config[%d].valid=%d.\n",
-			j,
-			s_config->isys_config[j].valid);
-	}
-
-	dev_dbg(isp->dev, "stream_config.input_config.input_res w=%d, h=%d.\n",
-		s_config->input_config.input_res.width,
-		s_config->input_config.input_res.height);
-
-	dev_dbg(isp->dev, "stream_config.input_config.effective_res w=%d, h=%d.\n",
-		s_config->input_config.effective_res.width,
-		s_config->input_config.effective_res.height);
-
-	dev_dbg(isp->dev, "stream_config.input_config.format=%d\n",
-		s_config->input_config.format);
-
-	dev_dbg(isp->dev, "stream_config.input_config.bayer_order=%d.\n",
-		s_config->input_config.bayer_order);
-
-	dev_dbg(isp->dev, "stream_config.pixels_per_clock=%d.\n",
-			s_config->pixels_per_clock);
-	dev_dbg(isp->dev, "stream_config.online=%d.\n", s_config->online);
-	dev_dbg(isp->dev, "stream_config.continuous=%d.\n",
-			s_config->continuous);
-	dev_dbg(isp->dev, "stream_config.disable_cont_viewfinder=%d.\n",
-			s_config->disable_cont_viewfinder);
-	dev_dbg(isp->dev, "stream_config.channel_id=%d.\n",
-			s_config->channel_id);
-	dev_dbg(isp->dev, "stream_config.init_num_cont_raw_buf=%d.\n",
-			s_config->init_num_cont_raw_buf);
-	dev_dbg(isp->dev, "stream_config.target_num_cont_raw_buf=%d.\n",
-			s_config->target_num_cont_raw_buf);
-	dev_dbg(isp->dev, "stream_config.left_padding=%d.\n",
-			s_config->left_padding);
-	dev_dbg(isp->dev, "stream_config.sensor_binning_factor=%d.\n",
-			s_config->sensor_binning_factor);
-	dev_dbg(isp->dev, "stream_config.pixels_per_clock=%d.\n",
-			s_config->pixels_per_clock);
-	dev_dbg(isp->dev, "stream_config.pack_raw_pixels=%d.\n",
-			s_config->pack_raw_pixels);
-	dev_dbg(isp->dev, "stream_config.flash_gpio_pin=%d.\n",
-			s_config->flash_gpio_pin);
-	dev_dbg(isp->dev, "stream_config.mipi_buffer_config.size_mem_words=%d.\n",
-			s_config->mipi_buffer_config.size_mem_words);
-	dev_dbg(isp->dev, "stream_config.mipi_buffer_config.contiguous=%d.\n",
-			s_config->mipi_buffer_config.contiguous);
-	dev_dbg(isp->dev, "stream_config.metadata_config.data_type=%d.\n",
-			s_config->metadata_config.data_type);
-	dev_dbg(isp->dev, "stream_config.metadata_config.resolution w=%d, h=%d.\n",
-			s_config->metadata_config.resolution.width,
-			s_config->metadata_config.resolution.height);
-}
-
-static int __destroy_stream(struct atomisp_sub_device *asd,
-			struct atomisp_stream_env *stream_env, bool force)
-{
-	struct atomisp_device *isp = asd->isp;
-	int i;
-	unsigned long timeout;
-
-	if (!stream_env->stream)
-		return 0;
-
-	if (!force) {
-		for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
-			if (stream_env->update_pipe[i])
-				break;
-
-		if (i == IA_CSS_PIPE_ID_NUM)
-			return 0;
-	}
-
-	if (stream_env->stream_state == CSS_STREAM_STARTED
-	    && ia_css_stream_stop(stream_env->stream) != IA_CSS_SUCCESS) {
-		dev_err(isp->dev, "stop stream failed.\n");
-		return -EINVAL;
-	}
-
-	if (stream_env->stream_state == CSS_STREAM_STARTED) {
-		timeout = jiffies + msecs_to_jiffies(40);
-		while (1) {
-			if (ia_css_stream_has_stopped(stream_env->stream))
-				break;
-
-			if (time_after(jiffies, timeout)) {
-				dev_warn(isp->dev, "stop stream timeout.\n");
-				break;
-			}
-
-			usleep_range(100, 200);
-		}
-	}
-
-	stream_env->stream_state = CSS_STREAM_STOPPED;
-
-	if (ia_css_stream_destroy(stream_env->stream) != IA_CSS_SUCCESS) {
-		dev_err(isp->dev, "destroy stream failed.\n");
-		return -EINVAL;
-	}
-	stream_env->stream_state = CSS_STREAM_UNINIT;
-	stream_env->stream = NULL;
-
-	return 0;
-}
-
-static int __destroy_streams(struct atomisp_sub_device *asd, bool force)
-{
-	int ret, i;
-
-	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
-		ret = __destroy_stream(asd, &asd->stream_env[i], force);
-		if (ret)
-			return ret;
-	}
-	asd->stream_prepared = false;
-	return 0;
-}
-static int __create_stream(struct atomisp_sub_device *asd,
-			   struct atomisp_stream_env *stream_env)
-{
-	int pipe_index = 0, i;
-	struct ia_css_pipe *multi_pipes[IA_CSS_PIPE_ID_NUM];
-
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) {
-		if (stream_env->pipes[i])
-			multi_pipes[pipe_index++] = stream_env->pipes[i];
-	}
-	if (pipe_index == 0)
-		return 0;
-
-	stream_env->stream_config.target_num_cont_raw_buf =
-		asd->continuous_raw_buffer_size->val;
-	stream_env->stream_config.channel_id = stream_env->ch_id;
-	stream_env->stream_config.ia_css_enable_raw_buffer_locking =
-		asd->enable_raw_buffer_lock->val;
-
-	__dump_stream_config(asd, stream_env);
-	if (ia_css_stream_create(&stream_env->stream_config,
-	    pipe_index, multi_pipes, &stream_env->stream) != IA_CSS_SUCCESS)
-		return -EINVAL;
-	if (ia_css_stream_get_info(stream_env->stream,
-				&stream_env->stream_info) != IA_CSS_SUCCESS) {
-		ia_css_stream_destroy(stream_env->stream);
-		stream_env->stream = NULL;
-		return -EINVAL;
-	}
-
-	stream_env->stream_state = CSS_STREAM_CREATED;
-	return 0;
-}
-
-static int __create_streams(struct atomisp_sub_device *asd)
-{
-	int ret, i;
-
-	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
-		ret = __create_stream(asd, &asd->stream_env[i]);
-		if (ret)
-			goto rollback;
-	}
-	asd->stream_prepared = true;
-	return 0;
-rollback:
-	for (i--; i >= 0; i--)
-		__destroy_stream(asd, &asd->stream_env[i], true);
-	return ret;
-}
-
-static int __destroy_stream_pipes(struct atomisp_sub_device *asd,
-				  struct atomisp_stream_env *stream_env,
-				  bool force)
-{
-	struct atomisp_device *isp = asd->isp;
-	int ret = 0;
-	int i;
-
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) {
-		if (!stream_env->pipes[i] ||
-		    !(force || stream_env->update_pipe[i]))
-			continue;
-		if (ia_css_pipe_destroy(stream_env->pipes[i])
-		    != IA_CSS_SUCCESS) {
-			dev_err(isp->dev,
-				"destroy pipe[%d]failed.cannot recover.\n", i);
-			ret = -EINVAL;
-		}
-		stream_env->pipes[i] = NULL;
-		stream_env->update_pipe[i] = false;
-	}
-	return ret;
-}
-
-static int __destroy_pipes(struct atomisp_sub_device *asd, bool force)
-{
-	struct atomisp_device *isp = asd->isp;
-	int i;
-	int ret = 0;
-
-	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
-		if (asd->stream_env[i].stream) {
-
-			dev_err(isp->dev,
-				"cannot destroy css pipes for stream[%d].\n",
-				i);
-			continue;
-		}
-
-		ret = __destroy_stream_pipes(asd, &asd->stream_env[i], force);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-void atomisp_destroy_pipes_stream_force(struct atomisp_sub_device *asd)
-{
-	__destroy_streams(asd, true);
-	__destroy_pipes(asd, true);
-}
-
-static void __apply_additional_pipe_config(
-				struct atomisp_sub_device *asd,
-				struct atomisp_stream_env *stream_env,
-				enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-
-	if (pipe_id < 0 || pipe_id >= IA_CSS_PIPE_ID_NUM) {
-		dev_err(isp->dev,
-			 "wrong pipe_id for additional pipe config.\n");
-		return;
-	}
-
-	/* apply default pipe config */
-	stream_env->pipe_configs[pipe_id].isp_pipe_version = 2;
-	stream_env->pipe_configs[pipe_id].enable_dz =
-				asd->disable_dz->val ? false : true;
-	/* apply isp 2.2 specific config for baytrail*/
-	switch (pipe_id) {
-	case IA_CSS_PIPE_ID_CAPTURE:
-		/* enable capture pp/dz manually or digital zoom would
-		 * fail*/
-		if (stream_env->pipe_configs[pipe_id].
-			default_capture_config.mode == CSS_CAPTURE_MODE_RAW)
-			stream_env->pipe_configs[pipe_id].enable_dz = false;
-#ifdef ISP2401
-
-		/* the isp default to use ISP2.2 and the camera hal will
-		 * control whether use isp2.7 */
-		if (asd->select_isp_version->val ==
-			ATOMISP_CSS_ISP_PIPE_VERSION_2_7)
-			stream_env->pipe_configs[pipe_id].isp_pipe_version =
-				SH_CSS_ISP_PIPE_VERSION_2_7;
-		else
-			stream_env->pipe_configs[pipe_id].isp_pipe_version =
-				SH_CSS_ISP_PIPE_VERSION_2_2;
-#endif
-		break;
-	case IA_CSS_PIPE_ID_VIDEO:
-		/* enable reduced pipe to have binary
-		 * video_dz_2_min selected*/
-		stream_env->pipe_extra_configs[pipe_id]
-		    .enable_reduced_pipe = true;
-		stream_env->pipe_configs[pipe_id]
-		    .enable_dz = false;
-		if (ATOMISP_SOC_CAMERA(asd))
-			stream_env->pipe_configs[pipe_id].enable_dz = true;
-
-		if (asd->params.video_dis_en) {
-			stream_env->pipe_extra_configs[pipe_id]
-				.enable_dvs_6axis = true;
-			stream_env->pipe_configs[pipe_id]
-				.dvs_frame_delay =
-					ATOMISP_CSS2_NUM_DVS_FRAME_DELAY;
-		}
-		break;
-	case IA_CSS_PIPE_ID_PREVIEW:
-		break;
-	case IA_CSS_PIPE_ID_YUVPP:
-	case IA_CSS_PIPE_ID_COPY:
-		if (ATOMISP_SOC_CAMERA(asd))
-			stream_env->pipe_configs[pipe_id].enable_dz = true;
-		else
-			stream_env->pipe_configs[pipe_id].enable_dz = false;
-		break;
-	case IA_CSS_PIPE_ID_ACC:
-		stream_env->pipe_configs[pipe_id].mode = IA_CSS_PIPE_MODE_ACC;
-		stream_env->pipe_configs[pipe_id].enable_dz = false;
-		break;
-	default:
-		break;
-	}
-}
-
-static bool is_pipe_valid_to_current_run_mode(struct atomisp_sub_device *asd,
-					enum ia_css_pipe_id pipe_id)
-{
-	if (!asd)
-		return false;
-
-	if (pipe_id == CSS_PIPE_ID_ACC || pipe_id == CSS_PIPE_ID_YUVPP)
-		return true;
-
-	if (asd->vfpp) {
-		if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) {
-			if (pipe_id == IA_CSS_PIPE_ID_VIDEO)
-				return true;
-			else
-				return false;
-		} else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) {
-			if (pipe_id == IA_CSS_PIPE_ID_CAPTURE)
-				return true;
-			else
-				return false;
-		}
-	}
-
-	if (!asd->run_mode)
-		return false;
-
-	if (asd->copy_mode && pipe_id == IA_CSS_PIPE_ID_COPY)
-		return true;
-
-	switch (asd->run_mode->val) {
-	case ATOMISP_RUN_MODE_STILL_CAPTURE:
-		if (pipe_id == IA_CSS_PIPE_ID_CAPTURE)
-			return true;
-		else
-			return false;
-	case ATOMISP_RUN_MODE_PREVIEW:
-		if (!asd->continuous_mode->val) {
-			if (pipe_id == IA_CSS_PIPE_ID_PREVIEW)
-				return true;
-			else
-				return false;
-		}
-		/* fall through to ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE */
-	case ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE:
-		if (pipe_id == IA_CSS_PIPE_ID_CAPTURE ||
-		    pipe_id == IA_CSS_PIPE_ID_PREVIEW)
-			return true;
-		else
-			return false;
-	case ATOMISP_RUN_MODE_VIDEO:
-		if (!asd->continuous_mode->val) {
-			if (pipe_id == IA_CSS_PIPE_ID_VIDEO ||
-			    pipe_id == IA_CSS_PIPE_ID_YUVPP)
-				return true;
-			else
-				return false;
-		}
-		/* fall through to ATOMISP_RUN_MODE_SDV */
-	case ATOMISP_RUN_MODE_SDV:
-		if (pipe_id == IA_CSS_PIPE_ID_CAPTURE ||
-		    pipe_id == IA_CSS_PIPE_ID_VIDEO)
-			return true;
-		else
-			return false;
-	}
-
-	return false;
-}
-
-static int __create_pipe(struct atomisp_sub_device *asd,
-			 struct atomisp_stream_env *stream_env,
-			 enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct ia_css_pipe_extra_config extra_config;
-	enum ia_css_err ret;
-
-	if (pipe_id >= IA_CSS_PIPE_ID_NUM)
-		return -EINVAL;
-
-	if (pipe_id != CSS_PIPE_ID_ACC &&
-	    !stream_env->pipe_configs[pipe_id].output_info[0].res.width)
-		return 0;
-
-	if (pipe_id == CSS_PIPE_ID_ACC &&
-	    !stream_env->pipe_configs[pipe_id].acc_extension)
-		return 0;
-
-	if (!is_pipe_valid_to_current_run_mode(asd, pipe_id))
-		return 0;
-
-	ia_css_pipe_extra_config_defaults(&extra_config);
-
-	__apply_additional_pipe_config(asd, stream_env, pipe_id);
-	if (!memcmp(&extra_config,
-		    &stream_env->pipe_extra_configs[pipe_id],
-		    sizeof(extra_config)))
-		ret = ia_css_pipe_create(
-			&stream_env->pipe_configs[pipe_id],
-			&stream_env->pipes[pipe_id]);
-	else
-		ret = ia_css_pipe_create_extra(
-			&stream_env->pipe_configs[pipe_id],
-			&stream_env->pipe_extra_configs[pipe_id],
-			&stream_env->pipes[pipe_id]);
-	if (ret != IA_CSS_SUCCESS)
-		dev_err(isp->dev, "create pipe[%d] error.\n", pipe_id);
-	return ret;
-}
-
-static int __create_pipes(struct atomisp_sub_device *asd)
-{
-	enum ia_css_err ret;
-	int i, j;
-
-	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
-		for (j = 0; j < IA_CSS_PIPE_ID_NUM; j++) {
-			ret = __create_pipe(asd, &asd->stream_env[i], j);
-			if (ret != IA_CSS_SUCCESS)
-				break;
-		}
-		if (j < IA_CSS_PIPE_ID_NUM)
-			goto pipe_err;
-	}
-	return 0;
-pipe_err:
-	for (; i >= 0; i--) {
-		for (j--; j >= 0; j--) {
-			if (asd->stream_env[i].pipes[j]) {
-				ia_css_pipe_destroy(asd->stream_env[i].pipes[j]);
-				asd->stream_env[i].pipes[j] = NULL;
-			}
-		}
-		j = IA_CSS_PIPE_ID_NUM;
-	}
-	return -EINVAL;
-}
-
-void atomisp_create_pipes_stream(struct atomisp_sub_device *asd)
-{
-	__create_pipes(asd);
-	__create_streams(asd);
-}
-
-int atomisp_css_update_stream(struct atomisp_sub_device *asd)
-{
-	int ret;
-	struct atomisp_device *isp = asd->isp;
-
-	if (__destroy_streams(asd, true) != IA_CSS_SUCCESS)
-		dev_warn(isp->dev, "destroy stream failed.\n");
-
-	if (__destroy_pipes(asd, true) != IA_CSS_SUCCESS)
-		dev_warn(isp->dev, "destroy pipe failed.\n");
-
-	ret = __create_pipes(asd);
-	if (ret != IA_CSS_SUCCESS) {
-		dev_err(isp->dev, "create pipe failed %d.\n", ret);
-		return -EIO;
-	}
-
-	ret = __create_streams(asd);
-	if (ret != IA_CSS_SUCCESS) {
-		dev_warn(isp->dev, "create stream failed %d.\n", ret);
-		__destroy_pipes(asd, true);
-		return -EIO;
-	}
-
-	return 0;
-}
-
-int atomisp_css_init(struct atomisp_device *isp)
-{
-	unsigned int mmu_base_addr;
-	int ret;
-	enum ia_css_err err;
-
-	ret = hmm_get_mmu_base_addr(&mmu_base_addr);
-	if (ret)
-		return ret;
-
-	/* Init ISP */
-	err = ia_css_init(&isp->css_env.isp_css_env, NULL,
-			  (uint32_t)mmu_base_addr, IA_CSS_IRQ_TYPE_PULSE);
-	if (err != IA_CSS_SUCCESS) {
-		dev_err(isp->dev, "css init failed --- bad firmware?\n");
-		return -EINVAL;
-	}
-	ia_css_enable_isys_event_queue(true);
-
-	isp->css_initialized = true;
-	dev_dbg(isp->dev, "sh_css_init success\n");
-
-	return 0;
-}
-
-static inline int __set_css_print_env(struct atomisp_device *isp, int opt)
-{
-	int ret = 0;
-
-	if (opt == 0)
-		isp->css_env.isp_css_env.print_env.debug_print = NULL;
-	else if (opt == 1)
-		isp->css_env.isp_css_env.print_env.debug_print =
-			atomisp_css2_dbg_ftrace_print;
-	else if (opt == 2)
-		isp->css_env.isp_css_env.print_env.debug_print =
-			atomisp_css2_dbg_print;
-	else
-		ret = -EINVAL;
-
-	return ret;
-}
-
-int atomisp_css_check_firmware_version(struct atomisp_device *isp)
-{
-	if (!sh_css_check_firmware_version((void *)isp->firmware->data)) {
-		dev_err(isp->dev, "Fw version check failed.\n");
-		return -EINVAL;
-	}
-	return 0;
-}
-
-int atomisp_css_load_firmware(struct atomisp_device *isp)
-{
-	enum ia_css_err err;
-
-	/* set css env */
-	isp->css_env.isp_css_fw.data = (void *)isp->firmware->data;
-	isp->css_env.isp_css_fw.bytes = isp->firmware->size;
-
-	isp->css_env.isp_css_env.hw_access_env.store_8 =
-							atomisp_css2_hw_store_8;
-	isp->css_env.isp_css_env.hw_access_env.store_16 =
-						atomisp_css2_hw_store_16;
-	isp->css_env.isp_css_env.hw_access_env.store_32 =
-						atomisp_css2_hw_store_32;
-
-	isp->css_env.isp_css_env.hw_access_env.load_8 = atomisp_css2_hw_load_8;
-	isp->css_env.isp_css_env.hw_access_env.load_16 =
-							atomisp_css2_hw_load_16;
-	isp->css_env.isp_css_env.hw_access_env.load_32 =
-							atomisp_css2_hw_load_32;
-
-	isp->css_env.isp_css_env.hw_access_env.load = atomisp_css2_hw_load;
-	isp->css_env.isp_css_env.hw_access_env.store = atomisp_css2_hw_store;
-
-	__set_css_print_env(isp, dbg_func);
-
-	isp->css_env.isp_css_env.print_env.error_print = atomisp_css2_err_print;
-
-	/* load isp fw into ISP memory */
-	err = ia_css_load_firmware(&isp->css_env.isp_css_env,
-				   &isp->css_env.isp_css_fw);
-	if (err != IA_CSS_SUCCESS) {
-		dev_err(isp->dev, "css load fw failed.\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-void atomisp_css_unload_firmware(struct atomisp_device *isp)
-{
-	ia_css_unload_firmware();
-}
-
-void atomisp_css_uninit(struct atomisp_device *isp)
-{
-	struct atomisp_sub_device *asd;
-	unsigned int i;
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-		atomisp_isp_parameters_clean_up(&asd->params.config);
-		asd->params.css_update_params_needed = false;
-	}
-
-	isp->css_initialized = false;
-	ia_css_uninit();
-}
-
-void atomisp_css_suspend(struct atomisp_device *isp)
-{
-	isp->css_initialized = false;
-	ia_css_uninit();
-}
-
-int atomisp_css_resume(struct atomisp_device *isp)
-{
-	unsigned int mmu_base_addr;
-	int ret;
-
-	ret = hmm_get_mmu_base_addr(&mmu_base_addr);
-	if (ret) {
-		dev_err(isp->dev, "get base address error.\n");
-		return -EINVAL;
-	}
-
-	ret = ia_css_init(&isp->css_env.isp_css_env, NULL,
-			  mmu_base_addr, IA_CSS_IRQ_TYPE_PULSE);
-	if (ret) {
-		dev_err(isp->dev, "re-init css failed.\n");
-		return -EINVAL;
-	}
-	ia_css_enable_isys_event_queue(true);
-
-	isp->css_initialized = true;
-	return 0;
-}
-
-int atomisp_css_irq_translate(struct atomisp_device *isp,
-			      unsigned int *infos)
-{
-	int err;
-
-	err = ia_css_irq_translate(infos);
-	if (err != IA_CSS_SUCCESS) {
-		dev_warn(isp->dev,
-			  "%s:failed to translate irq (err = %d,infos = %d)\n",
-			  __func__, err, *infos);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-void atomisp_css_rx_get_irq_info(enum mipi_port_id port,
-					unsigned int *infos)
-{
-#ifndef ISP2401_NEW_INPUT_SYSTEM
-	ia_css_isys_rx_get_irq_info(port, infos);
-#else
-	*infos = 0;
-#endif
-}
-
-void atomisp_css_rx_clear_irq_info(enum mipi_port_id port,
-					unsigned int infos)
-{
-#ifndef ISP2401_NEW_INPUT_SYSTEM
-	ia_css_isys_rx_clear_irq_info(port, infos);
-#endif
-}
-
-int atomisp_css_irq_enable(struct atomisp_device *isp,
-			    enum atomisp_css_irq_info info, bool enable)
-{
-	if (ia_css_irq_enable(info, enable) != IA_CSS_SUCCESS) {
-		dev_warn(isp->dev, "%s:Invalid irq info.\n", __func__);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-void atomisp_css_init_struct(struct atomisp_sub_device *asd)
-{
-	int i, j;
-
-	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
-		asd->stream_env[i].stream = NULL;
-		for (j = 0; j < IA_CSS_PIPE_MODE_NUM; j++) {
-			asd->stream_env[i].pipes[j] = NULL;
-			asd->stream_env[i].update_pipe[j] = false;
-			ia_css_pipe_config_defaults(
-				&asd->stream_env[i].pipe_configs[j]);
-			ia_css_pipe_extra_config_defaults(
-				&asd->stream_env[i].pipe_extra_configs[j]);
-		}
-		ia_css_stream_config_defaults(&asd->stream_env[i].stream_config);
-	}
-}
-
-int atomisp_q_video_buffer_to_css(struct atomisp_sub_device *asd,
-			struct videobuf_vmalloc_memory *vm_mem,
-			enum atomisp_input_stream_id stream_id,
-			enum atomisp_css_buffer_type css_buf_type,
-			enum atomisp_css_pipe_id css_pipe_id)
-{
-	struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id];
-	struct ia_css_buffer css_buf = {0};
-	enum ia_css_err err;
-
-	css_buf.type = css_buf_type;
-	css_buf.data.frame = vm_mem->vaddr;
-
-	err = ia_css_pipe_enqueue_buffer(
-				stream_env->pipes[css_pipe_id], &css_buf);
-	if (err != IA_CSS_SUCCESS)
-		return -EINVAL;
-
-	return 0;
-}
-
-int atomisp_q_metadata_buffer_to_css(struct atomisp_sub_device *asd,
-			struct atomisp_metadata_buf *metadata_buf,
-			enum atomisp_input_stream_id stream_id,
-			enum atomisp_css_pipe_id css_pipe_id)
-{
-	struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id];
-	struct ia_css_buffer buffer = {0};
-	struct atomisp_device *isp = asd->isp;
-
-	buffer.type = IA_CSS_BUFFER_TYPE_METADATA;
-	buffer.data.metadata = metadata_buf->metadata;
-	if (ia_css_pipe_enqueue_buffer(stream_env->pipes[css_pipe_id],
-				&buffer)) {
-		dev_err(isp->dev, "failed to q meta data buffer\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-int atomisp_q_s3a_buffer_to_css(struct atomisp_sub_device *asd,
-			struct atomisp_s3a_buf *s3a_buf,
-			enum atomisp_input_stream_id stream_id,
-			enum atomisp_css_pipe_id css_pipe_id)
-{
-	struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id];
-	struct ia_css_buffer buffer = {0};
-	struct atomisp_device *isp = asd->isp;
-
-	buffer.type = IA_CSS_BUFFER_TYPE_3A_STATISTICS;
-	buffer.data.stats_3a = s3a_buf->s3a_data;
-	if (ia_css_pipe_enqueue_buffer(
-				stream_env->pipes[css_pipe_id],
-				&buffer)) {
-		dev_dbg(isp->dev, "failed to q s3a stat buffer\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-int atomisp_q_dis_buffer_to_css(struct atomisp_sub_device *asd,
-			struct atomisp_dis_buf *dis_buf,
-			enum atomisp_input_stream_id stream_id,
-			enum atomisp_css_pipe_id css_pipe_id)
-{
-	struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id];
-	struct ia_css_buffer buffer = {0};
-	struct atomisp_device *isp = asd->isp;
-
-	buffer.type = IA_CSS_BUFFER_TYPE_DIS_STATISTICS;
-	buffer.data.stats_dvs = dis_buf->dis_data;
-	if (ia_css_pipe_enqueue_buffer(
-				stream_env->pipes[css_pipe_id],
-				&buffer)) {
-		dev_dbg(isp->dev, "failed to q dvs stat buffer\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-void atomisp_css_mmu_invalidate_cache(void)
-{
-	ia_css_mmu_invalidate_cache();
-}
-
-void atomisp_css_mmu_invalidate_tlb(void)
-{
-	ia_css_mmu_invalidate_cache();
-}
-
-int atomisp_css_start(struct atomisp_sub_device *asd,
-			enum atomisp_css_pipe_id pipe_id, bool in_reset)
-{
-	struct atomisp_device *isp = asd->isp;
-	bool sp_is_started = false;
-	int ret = 0, i = 0;
-
-	if (in_reset) {
-		if (__destroy_streams(asd, true))
-			dev_warn(isp->dev, "destroy stream failed.\n");
-
-		if (__destroy_pipes(asd, true))
-			dev_warn(isp->dev, "destroy pipe failed.\n");
-
-		if (__create_pipes(asd)) {
-			dev_err(isp->dev, "create pipe error.\n");
-			return -EINVAL;
-		}
-		if (__create_streams(asd)) {
-			dev_err(isp->dev, "create stream error.\n");
-			ret = -EINVAL;
-			goto stream_err;
-		}
-		/* in_reset == true, extension firmwares are reloaded after the recovery */
-		atomisp_acc_load_extensions(asd);
-	}
-
-	/*
-	 * For dual steam case, it is possible that:
-	 * 1: for this stream, it is at the stage that:
-	 * - after set_fmt is called
-	 * - before stream on is called
-	 * 2: for the other stream, the stream off is called which css reset
-	 * has been done.
-	 *
-	 * Thus the stream created in set_fmt get destroyed and need to be
-	 * recreated in the next stream on.
-	 */
-	if (asd->stream_prepared == false) {
-		if (__create_pipes(asd)) {
-			dev_err(isp->dev, "create pipe error.\n");
-			return -EINVAL;
-		}
-		if (__create_streams(asd)) {
-			dev_err(isp->dev, "create stream error.\n");
-			ret = -EINVAL;
-			goto stream_err;
-		}
-	}
-	/*
-	 * SP can only be started one time
-	 * if atomisp_subdev_streaming_count() tell there already has some
-	 * subdev at streamming, then SP should already be started previously,
-	 * so need to skip start sp procedure
-	 */
-	if (atomisp_streaming_count(isp)) {
-		dev_dbg(isp->dev, "skip start sp\n");
-	} else {
-		if (!sh_css_hrt_system_is_idle())
-			dev_err(isp->dev, "CSS HW not idle before starting SP\n");
-		if (ia_css_start_sp() != IA_CSS_SUCCESS) {
-			dev_err(isp->dev, "start sp error.\n");
-			ret = -EINVAL;
-			goto start_err;
-		} else {
-			sp_is_started = true;
-		}
-	}
-
-	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
-		if (asd->stream_env[i].stream) {
-			if (ia_css_stream_start(asd->stream_env[i]
-						.stream) != IA_CSS_SUCCESS) {
-				dev_err(isp->dev, "stream[%d] start error.\n", i);
-				ret = -EINVAL;
-				goto start_err;
-			} else {
-				asd->stream_env[i].stream_state = CSS_STREAM_STARTED;
-				dev_dbg(isp->dev, "stream[%d] started.\n", i);
-			}
-		}
-	}
-
-	return 0;
-
-start_err:
-	__destroy_streams(asd, true);
-stream_err:
-	__destroy_pipes(asd, true);
-
-	/* css 2.0 API limitation: ia_css_stop_sp() could be only called after
-	 * destroy all pipes
-	 */
-	/*
-	 * SP can not be stop if other streams are in use
-	 */
-	if ((atomisp_streaming_count(isp) == 0) && sp_is_started)
-		ia_css_stop_sp();
-
-	return ret;
-}
-
-void atomisp_css_update_isp_params(struct atomisp_sub_device *asd)
-{
-	/*
-	 * FIXME!
-	 * for ISP2401 new input system, this api is under development.
-	 * Calling it would cause kernel panic.
-	 *
-	 * VIED BZ: 1458
-	 *
-	 * Check if it is Cherry Trail and also new input system
-	 */
-	if (asd->copy_mode) {
-		dev_warn(asd->isp->dev,
-			 "%s: ia_css_stream_set_isp_config() not supported in copy mode!.\n",
-				__func__);
-		return;
-	}
-
-	ia_css_stream_set_isp_config(
-			asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-			&asd->params.config);
-	atomisp_isp_parameters_clean_up(&asd->params.config);
-}
-
-
-void atomisp_css_update_isp_params_on_pipe(struct atomisp_sub_device *asd,
-					struct ia_css_pipe *pipe)
-{
-	enum ia_css_err ret;
-
-	if (!pipe) {
-		atomisp_css_update_isp_params(asd);
-		return;
-	}
-
-	dev_dbg(asd->isp->dev, "%s: apply parameter for ia_css_frame %p with isp_config_id %d on pipe %p.\n",
-		__func__, asd->params.config.output_frame,
-		asd->params.config.isp_config_id, pipe);
-
-	ret = ia_css_stream_set_isp_config_on_pipe(
-			asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-			&asd->params.config, pipe);
-	if (ret != IA_CSS_SUCCESS)
-		dev_warn(asd->isp->dev, "%s: ia_css_stream_set_isp_config_on_pipe failed %d\n",
-			__func__, ret);
-	atomisp_isp_parameters_clean_up(&asd->params.config);
-}
-
-int atomisp_css_queue_buffer(struct atomisp_sub_device *asd,
-			     enum atomisp_input_stream_id stream_id,
-			     enum atomisp_css_pipe_id pipe_id,
-			     enum atomisp_css_buffer_type buf_type,
-			     struct atomisp_css_buffer *isp_css_buffer)
-{
-	if (ia_css_pipe_enqueue_buffer(
-		asd->stream_env[stream_id].pipes[pipe_id],
-					&isp_css_buffer->css_buffer)
-					!= IA_CSS_SUCCESS)
-		return -EINVAL;
-
-	return 0;
-}
-
-int atomisp_css_dequeue_buffer(struct atomisp_sub_device *asd,
-				enum atomisp_input_stream_id stream_id,
-				enum atomisp_css_pipe_id pipe_id,
-				enum atomisp_css_buffer_type buf_type,
-				struct atomisp_css_buffer *isp_css_buffer)
-{
-	struct atomisp_device *isp = asd->isp;
-	enum ia_css_err err;
-
-	err = ia_css_pipe_dequeue_buffer(
-		asd->stream_env[stream_id].pipes[pipe_id],
-					&isp_css_buffer->css_buffer);
-	if (err != IA_CSS_SUCCESS) {
-		dev_err(isp->dev,
-			"ia_css_pipe_dequeue_buffer failed: 0x%x\n", err);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-int atomisp_css_allocate_stat_buffers(struct atomisp_sub_device   *asd,
-				      uint16_t stream_id,
-				      struct atomisp_s3a_buf      *s3a_buf,
-				      struct atomisp_dis_buf      *dis_buf,
-				      struct atomisp_metadata_buf *md_buf)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_css_dvs_grid_info *dvs_grid_info =
-		atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
-
-	if (s3a_buf && asd->params.curr_grid_info.s3a_grid.enable) {
-		void *s3a_ptr;
-
-		s3a_buf->s3a_data = ia_css_isp_3a_statistics_allocate(
-				&asd->params.curr_grid_info.s3a_grid);
-		if (!s3a_buf->s3a_data) {
-			dev_err(isp->dev, "3a buf allocation failed.\n");
-			return -EINVAL;
-		}
-
-		s3a_ptr = hmm_vmap(s3a_buf->s3a_data->data_ptr, true);
-		s3a_buf->s3a_map = ia_css_isp_3a_statistics_map_allocate(
-						s3a_buf->s3a_data, s3a_ptr);
-	}
-
-	if (dis_buf && dvs_grid_info && dvs_grid_info->enable) {
-		void *dvs_ptr;
-
-		dis_buf->dis_data = ia_css_isp_dvs2_statistics_allocate(
-					dvs_grid_info);
-		if (!dis_buf->dis_data) {
-			dev_err(isp->dev, "dvs buf allocation failed.\n");
-			if (s3a_buf)
-				ia_css_isp_3a_statistics_free(s3a_buf->s3a_data);
-			return -EINVAL;
-		}
-
-		dvs_ptr = hmm_vmap(dis_buf->dis_data->data_ptr, true);
-		dis_buf->dvs_map = ia_css_isp_dvs_statistics_map_allocate(
-						dis_buf->dis_data, dvs_ptr);
-	}
-
-	if (asd->stream_env[stream_id].stream_info.
-			metadata_info.size && md_buf) {
-		md_buf->metadata = ia_css_metadata_allocate(
-			&asd->stream_env[stream_id].stream_info.metadata_info);
-		if (!md_buf->metadata) {
-			if (s3a_buf)
-				ia_css_isp_3a_statistics_free(s3a_buf->s3a_data);
-			if (dis_buf)
-				ia_css_isp_dvs2_statistics_free(dis_buf->dis_data);
-			dev_err(isp->dev, "metadata buf allocation failed.\n");
-			return -EINVAL;
-		}
-		md_buf->md_vptr = hmm_vmap(md_buf->metadata->address, false);
-	}
-
-	return 0;
-}
-
-void atomisp_css_free_3a_buffer(struct atomisp_s3a_buf *s3a_buf)
-{
-	if (s3a_buf->s3a_data)
-		hmm_vunmap(s3a_buf->s3a_data->data_ptr);
-
-	ia_css_isp_3a_statistics_map_free(s3a_buf->s3a_map);
-	s3a_buf->s3a_map = NULL;
-	ia_css_isp_3a_statistics_free(s3a_buf->s3a_data);
-}
-
-void atomisp_css_free_dis_buffer(struct atomisp_dis_buf *dis_buf)
-{
-	if (dis_buf->dis_data)
-		hmm_vunmap(dis_buf->dis_data->data_ptr);
-
-	ia_css_isp_dvs_statistics_map_free(dis_buf->dvs_map);
-	dis_buf->dvs_map = NULL;
-	ia_css_isp_dvs2_statistics_free(dis_buf->dis_data);
-}
-
-void atomisp_css_free_metadata_buffer(struct atomisp_metadata_buf *metadata_buf)
-{
-	if (metadata_buf->md_vptr) {
-		hmm_vunmap(metadata_buf->metadata->address);
-		metadata_buf->md_vptr = NULL;
-	}
-	ia_css_metadata_free(metadata_buf->metadata);
-}
-
-void atomisp_css_free_stat_buffers(struct atomisp_sub_device *asd)
-{
-	struct atomisp_s3a_buf *s3a_buf, *_s3a_buf;
-	struct atomisp_dis_buf *dis_buf, *_dis_buf;
-	struct atomisp_metadata_buf *md_buf, *_md_buf;
-	struct atomisp_css_dvs_grid_info *dvs_grid_info =
-		atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
-	unsigned int i;
-
-	/* 3A statistics use vmalloc, DIS use kmalloc */
-	if (dvs_grid_info && dvs_grid_info->enable) {
-		ia_css_dvs2_coefficients_free(asd->params.css_param.dvs2_coeff);
-		ia_css_dvs2_statistics_free(asd->params.dvs_stat);
-		asd->params.css_param.dvs2_coeff = NULL;
-		asd->params.dvs_stat = NULL;
-		asd->params.dvs_hor_proj_bytes = 0;
-		asd->params.dvs_ver_proj_bytes = 0;
-		asd->params.dvs_hor_coef_bytes = 0;
-		asd->params.dvs_ver_coef_bytes = 0;
-		asd->params.dis_proj_data_valid = false;
-		list_for_each_entry_safe(dis_buf, _dis_buf,
-						&asd->dis_stats, list) {
-			atomisp_css_free_dis_buffer(dis_buf);
-			list_del(&dis_buf->list);
-			kfree(dis_buf);
-		}
-		list_for_each_entry_safe(dis_buf, _dis_buf,
-						&asd->dis_stats_in_css, list) {
-			atomisp_css_free_dis_buffer(dis_buf);
-			list_del(&dis_buf->list);
-			kfree(dis_buf);
-		}
-	}
-	if (asd->params.curr_grid_info.s3a_grid.enable) {
-		ia_css_3a_statistics_free(asd->params.s3a_user_stat);
-		asd->params.s3a_user_stat = NULL;
-		asd->params.s3a_output_bytes = 0;
-		list_for_each_entry_safe(s3a_buf, _s3a_buf,
-						&asd->s3a_stats, list) {
-			atomisp_css_free_3a_buffer(s3a_buf);
-			list_del(&s3a_buf->list);
-			kfree(s3a_buf);
-		}
-		list_for_each_entry_safe(s3a_buf, _s3a_buf,
-						&asd->s3a_stats_in_css, list) {
-			atomisp_css_free_3a_buffer(s3a_buf);
-			list_del(&s3a_buf->list);
-			kfree(s3a_buf);
-		}
-		list_for_each_entry_safe(s3a_buf, _s3a_buf,
-						&asd->s3a_stats_ready, list) {
-			atomisp_css_free_3a_buffer(s3a_buf);
-			list_del(&s3a_buf->list);
-			kfree(s3a_buf);
-		}
-	}
-
-	if (asd->params.css_param.dvs_6axis) {
-		ia_css_dvs2_6axis_config_free(asd->params.css_param.dvs_6axis);
-		asd->params.css_param.dvs_6axis = NULL;
-	}
-
-	for (i = 0; i < ATOMISP_METADATA_TYPE_NUM; i++) {
-		list_for_each_entry_safe(md_buf, _md_buf,
-					&asd->metadata[i], list) {
-			atomisp_css_free_metadata_buffer(md_buf);
-			list_del(&md_buf->list);
-			kfree(md_buf);
-		}
-		list_for_each_entry_safe(md_buf, _md_buf,
-					&asd->metadata_in_css[i], list) {
-			atomisp_css_free_metadata_buffer(md_buf);
-			list_del(&md_buf->list);
-			kfree(md_buf);
-		}
-		list_for_each_entry_safe(md_buf, _md_buf,
-					&asd->metadata_ready[i], list) {
-			atomisp_css_free_metadata_buffer(md_buf);
-			list_del(&md_buf->list);
-			kfree(md_buf);
-		}
-	}
-	asd->params.metadata_width_size = 0;
-	atomisp_free_metadata_output_buf(asd);
-}
-
-int atomisp_css_get_grid_info(struct atomisp_sub_device *asd,
-				enum atomisp_css_pipe_id pipe_id,
-				int source_pad)
-{
-	struct ia_css_pipe_info p_info;
-	struct ia_css_grid_info old_info;
-	struct atomisp_device *isp = asd->isp;
-	int stream_index = atomisp_source_pad_to_stream_id(asd, source_pad);
-	int md_width = asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
-		stream_config.metadata_config.resolution.width;
-
-	memset(&p_info, 0, sizeof(struct ia_css_pipe_info));
-	memset(&old_info, 0, sizeof(struct ia_css_grid_info));
-
-	if (ia_css_pipe_get_info(
-		asd->stream_env[stream_index].pipes[pipe_id],
-		&p_info) != IA_CSS_SUCCESS) {
-		dev_err(isp->dev, "ia_css_pipe_get_info failed\n");
-		return -EINVAL;
-	}
-
-	memcpy(&old_info, &asd->params.curr_grid_info,
-					sizeof(struct ia_css_grid_info));
-	memcpy(&asd->params.curr_grid_info, &p_info.grid_info,
-					sizeof(struct ia_css_grid_info));
-	/*
-	 * Record which css pipe enables s3a_grid.
-	 * Currently would have one css pipe that need it
-	 */
-	if (asd->params.curr_grid_info.s3a_grid.enable) {
-		if (asd->params.s3a_enabled_pipe != CSS_PIPE_ID_NUM)
-			dev_dbg(isp->dev, "css pipe %d enabled s3a grid replaced by: %d.\n",
-					asd->params.s3a_enabled_pipe, pipe_id);
-		asd->params.s3a_enabled_pipe = pipe_id;
-	}
-
-	/* If the grid info has not changed and the buffers for 3A and
-	 * DIS statistics buffers are allocated or buffer size would be zero
-	 * then no need to do anything. */
-	if (((!memcmp(&old_info, &asd->params.curr_grid_info, sizeof(old_info))
-	    && asd->params.s3a_user_stat && asd->params.dvs_stat)
-	    || asd->params.curr_grid_info.s3a_grid.width == 0
-	    || asd->params.curr_grid_info.s3a_grid.height == 0)
-	    && asd->params.metadata_width_size == md_width) {
-		dev_dbg(isp->dev,
-			"grid info change escape. memcmp=%d, s3a_user_stat=%d,"
-			"dvs_stat=%d, s3a.width=%d, s3a.height=%d, metadata width =%d\n",
-			!memcmp(&old_info, &asd->params.curr_grid_info,
-				 sizeof(old_info)),
-			 !!asd->params.s3a_user_stat, !!asd->params.dvs_stat,
-			 asd->params.curr_grid_info.s3a_grid.width,
-			 asd->params.curr_grid_info.s3a_grid.height,
-			 asd->params.metadata_width_size);
-		return -EINVAL;
-	}
-	asd->params.metadata_width_size = md_width;
-
-	return 0;
-}
-
-int atomisp_alloc_3a_output_buf(struct atomisp_sub_device *asd)
-{
-	if (!asd->params.curr_grid_info.s3a_grid.width ||
-			!asd->params.curr_grid_info.s3a_grid.height)
-		return 0;
-
-	asd->params.s3a_user_stat = ia_css_3a_statistics_allocate(
-				&asd->params.curr_grid_info.s3a_grid);
-	if (!asd->params.s3a_user_stat)
-		return -ENOMEM;
-	/* 3A statistics. These can be big, so we use vmalloc. */
-	asd->params.s3a_output_bytes =
-	    asd->params.curr_grid_info.s3a_grid.width *
-	    asd->params.curr_grid_info.s3a_grid.height *
-	    sizeof(*asd->params.s3a_user_stat->data);
-
-	return 0;
-}
-
-int atomisp_alloc_dis_coef_buf(struct atomisp_sub_device *asd)
-{
-	struct atomisp_css_dvs_grid_info *dvs_grid =
-		atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
-
-	if (!dvs_grid)
-		return 0;
-
-	if (!dvs_grid->enable) {
-		dev_dbg(asd->isp->dev, "%s: dvs_grid not enabled.\n", __func__);
-		return 0;
-	}
-
-	/* DIS coefficients. */
-	asd->params.css_param.dvs2_coeff = ia_css_dvs2_coefficients_allocate(
-			dvs_grid);
-	if (!asd->params.css_param.dvs2_coeff)
-		return -ENOMEM;
-
-	asd->params.dvs_hor_coef_bytes = dvs_grid->num_hor_coefs *
-		sizeof(*asd->params.css_param.dvs2_coeff->hor_coefs.odd_real);
-
-	asd->params.dvs_ver_coef_bytes = dvs_grid->num_ver_coefs *
-		sizeof(*asd->params.css_param.dvs2_coeff->ver_coefs.odd_real);
-
-	/* DIS projections. */
-	asd->params.dis_proj_data_valid = false;
-	asd->params.dvs_stat = ia_css_dvs2_statistics_allocate(dvs_grid);
-	if (!asd->params.dvs_stat)
-		return -ENOMEM;
-
-	asd->params.dvs_hor_proj_bytes =
-		dvs_grid->aligned_height * dvs_grid->aligned_width *
-		sizeof(*asd->params.dvs_stat->hor_prod.odd_real);
-
-	asd->params.dvs_ver_proj_bytes =
-		dvs_grid->aligned_height * dvs_grid->aligned_width *
-		sizeof(*asd->params.dvs_stat->ver_prod.odd_real);
-
-	return 0;
-}
-
-int atomisp_alloc_metadata_output_buf(struct atomisp_sub_device *asd)
-{
-	int i;
-
-	/* We allocate the cpu-side buffer used for communication with user
-	 * space */
-	for (i = 0; i < ATOMISP_METADATA_TYPE_NUM; i++) {
-		asd->params.metadata_user[i] = kvmalloc(
-				asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
-				stream_info.metadata_info.size, GFP_KERNEL);
-		if (!asd->params.metadata_user[i]) {
-			while (--i >= 0) {
-				kvfree(asd->params.metadata_user[i]);
-				asd->params.metadata_user[i] = NULL;
-			}
-			return -ENOMEM;
-		}
-	}
-
-	return 0;
-}
-
-void atomisp_free_metadata_output_buf(struct atomisp_sub_device *asd)
-{
-	unsigned int i;
-
-	for (i = 0; i < ATOMISP_METADATA_TYPE_NUM; i++) {
-		if (asd->params.metadata_user[i]) {
-			kvfree(asd->params.metadata_user[i]);
-			asd->params.metadata_user[i] = NULL;
-		}
-	}
-}
-
-void atomisp_css_get_dis_statistics(struct atomisp_sub_device *asd,
-				    struct atomisp_css_buffer *isp_css_buffer,
-				    struct ia_css_isp_dvs_statistics_map *dvs_map)
-{
-	if (asd->params.dvs_stat) {
-		if (dvs_map)
-			ia_css_translate_dvs2_statistics(
-				asd->params.dvs_stat, dvs_map);
-		else
-			ia_css_get_dvs2_statistics(asd->params.dvs_stat,
-				isp_css_buffer->css_buffer.data.stats_dvs);
-
-	}
-}
-
-int atomisp_css_dequeue_event(struct atomisp_css_event *current_event)
-{
-	if (ia_css_dequeue_event(&current_event->event) != IA_CSS_SUCCESS)
-		return -EINVAL;
-
-	return 0;
-}
-
-void atomisp_css_temp_pipe_to_pipe_id(struct atomisp_sub_device *asd,
-		struct atomisp_css_event *current_event)
-{
-	/*
-	 * FIXME!
-	 * Pipe ID reported in CSS event is not correct for new system's
-	 * copy pipe.
-	 * VIED BZ: 1463
-	 */
-	ia_css_temp_pipe_to_pipe_id(current_event->event.pipe,
-				    &current_event->pipe);
-	if (asd && asd->copy_mode &&
-	    current_event->pipe == IA_CSS_PIPE_ID_CAPTURE)
-		current_event->pipe = IA_CSS_PIPE_ID_COPY;
-}
-
-int atomisp_css_isys_set_resolution(struct atomisp_sub_device *asd,
-				    enum atomisp_input_stream_id stream_id,
-				    struct v4l2_mbus_framefmt *ffmt,
-				    int isys_stream)
-{
-	struct ia_css_stream_config *s_config =
-			&asd->stream_env[stream_id].stream_config;
-
-	if (isys_stream >= IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH)
-		return -EINVAL;
-
-	s_config->isys_config[isys_stream].input_res.width = ffmt->width;
-	s_config->isys_config[isys_stream].input_res.height = ffmt->height;
-	return 0;
-}
-
-int atomisp_css_input_set_resolution(struct atomisp_sub_device *asd,
-				enum atomisp_input_stream_id stream_id,
-				struct v4l2_mbus_framefmt *ffmt)
-{
-	struct ia_css_stream_config *s_config =
-			&asd->stream_env[stream_id].stream_config;
-
-	s_config->input_config.input_res.width = ffmt->width;
-	s_config->input_config.input_res.height = ffmt->height;
-	return 0;
-}
-
-void atomisp_css_input_set_binning_factor(struct atomisp_sub_device *asd,
-					enum atomisp_input_stream_id stream_id,
-					unsigned int bin_factor)
-{
-	asd->stream_env[stream_id]
-	    .stream_config.sensor_binning_factor = bin_factor;
-}
-
-void atomisp_css_input_set_bayer_order(struct atomisp_sub_device *asd,
-				enum atomisp_input_stream_id stream_id,
-				enum atomisp_css_bayer_order bayer_order)
-{
-	struct ia_css_stream_config *s_config =
-			&asd->stream_env[stream_id].stream_config;
-	s_config->input_config.bayer_order = bayer_order;
-}
-
-void atomisp_css_isys_set_link(struct atomisp_sub_device *asd,
-			       enum atomisp_input_stream_id stream_id,
-			       int link,
-			       int isys_stream)
-{
-	struct ia_css_stream_config *s_config =
-		&asd->stream_env[stream_id].stream_config;
-
-	s_config->isys_config[isys_stream].linked_isys_stream_id = link;
-}
-
-void atomisp_css_isys_set_valid(struct atomisp_sub_device *asd,
-				enum atomisp_input_stream_id stream_id,
-				bool valid,
-				int isys_stream)
-{
-	struct ia_css_stream_config *s_config =
-		&asd->stream_env[stream_id].stream_config;
-
-	s_config->isys_config[isys_stream].valid = valid;
-}
-
-void atomisp_css_isys_set_format(struct atomisp_sub_device *asd,
-				 enum atomisp_input_stream_id stream_id,
-				 enum atomisp_input_format format,
-				 int isys_stream)
-{
-
-	struct ia_css_stream_config *s_config =
-			&asd->stream_env[stream_id].stream_config;
-
-	s_config->isys_config[isys_stream].format = format;
-}
-
-void atomisp_css_input_set_format(struct atomisp_sub_device *asd,
-					enum atomisp_input_stream_id stream_id,
-					enum atomisp_input_format format)
-{
-
-	struct ia_css_stream_config *s_config =
-			&asd->stream_env[stream_id].stream_config;
-
-	s_config->input_config.format = format;
-}
-
-int atomisp_css_set_default_isys_config(struct atomisp_sub_device *asd,
-					enum atomisp_input_stream_id stream_id,
-					struct v4l2_mbus_framefmt *ffmt)
-{
-	int i;
-	struct ia_css_stream_config *s_config =
-			&asd->stream_env[stream_id].stream_config;
-	/*
-	 * Set all isys configs to not valid.
-	 * Currently we support only one stream per channel
-	 */
-	for (i = IA_CSS_STREAM_ISYS_STREAM_0;
-	     i < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH; i++)
-		s_config->isys_config[i].valid = false;
-
-	atomisp_css_isys_set_resolution(asd, stream_id, ffmt,
-					IA_CSS_STREAM_DEFAULT_ISYS_STREAM_IDX);
-	atomisp_css_isys_set_format(asd, stream_id,
-				    s_config->input_config.format,
-				    IA_CSS_STREAM_DEFAULT_ISYS_STREAM_IDX);
-	atomisp_css_isys_set_link(asd, stream_id, NO_LINK,
-				  IA_CSS_STREAM_DEFAULT_ISYS_STREAM_IDX);
-	atomisp_css_isys_set_valid(asd, stream_id, true,
-				   IA_CSS_STREAM_DEFAULT_ISYS_STREAM_IDX);
-
-	return 0;
-}
-
-int atomisp_css_isys_two_stream_cfg(struct atomisp_sub_device *asd,
-				    enum atomisp_input_stream_id stream_id,
-				    enum atomisp_input_format input_format)
-{
-	struct ia_css_stream_config *s_config =
-		&asd->stream_env[stream_id].stream_config;
-
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_1].input_res.width =
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_0].input_res.width;
-
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_1].input_res.height =
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_0].input_res.height / 2;
-
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_1].linked_isys_stream_id
-		= IA_CSS_STREAM_ISYS_STREAM_0;
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_0].format =
-		ATOMISP_INPUT_FORMAT_USER_DEF1;
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_1].format =
-		ATOMISP_INPUT_FORMAT_USER_DEF2;
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_1].valid = true;
-	return 0;
-}
-
-void atomisp_css_isys_two_stream_cfg_update_stream1(
-				    struct atomisp_sub_device *asd,
-				    enum atomisp_input_stream_id stream_id,
-				    enum atomisp_input_format input_format,
-				    unsigned int width, unsigned int height)
-{
-	struct ia_css_stream_config *s_config =
-		&asd->stream_env[stream_id].stream_config;
-
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_0].input_res.width =
-		width;
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_0].input_res.height =
-		height;
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_0].format =
-		input_format;
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_0].valid = true;
-}
-
-void atomisp_css_isys_two_stream_cfg_update_stream2(
-				    struct atomisp_sub_device *asd,
-				    enum atomisp_input_stream_id stream_id,
-				    enum atomisp_input_format input_format,
-				    unsigned int width, unsigned int height)
-{
-	struct ia_css_stream_config *s_config =
-		&asd->stream_env[stream_id].stream_config;
-
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_1].input_res.width =
-		width;
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_1].input_res.height =
-	height;
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_1].linked_isys_stream_id
-		= IA_CSS_STREAM_ISYS_STREAM_0;
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_1].format =
-		input_format;
-	s_config->isys_config[IA_CSS_STREAM_ISYS_STREAM_1].valid = true;
-}
-
-int atomisp_css_input_set_effective_resolution(
-					struct atomisp_sub_device *asd,
-					enum atomisp_input_stream_id stream_id,
-					unsigned int width, unsigned int height)
-{
-	struct ia_css_stream_config *s_config =
-			&asd->stream_env[stream_id].stream_config;
-	s_config->input_config.effective_res.width = width;
-	s_config->input_config.effective_res.height = height;
-	return 0;
-}
-
-void atomisp_css_video_set_dis_envelope(struct atomisp_sub_device *asd,
-					unsigned int dvs_w, unsigned int dvs_h)
-{
-	asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.pipe_configs[IA_CSS_PIPE_ID_VIDEO].dvs_envelope.width = dvs_w;
-	asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.pipe_configs[IA_CSS_PIPE_ID_VIDEO].dvs_envelope.height = dvs_h;
-}
-
-void atomisp_css_input_set_two_pixels_per_clock(
-					struct atomisp_sub_device *asd,
-					bool two_ppc)
-{
-	int i;
-
-	if (asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.stream_config.pixels_per_clock == (two_ppc ? 2 : 1))
-		return;
-
-	asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.stream_config.pixels_per_clock = (two_ppc ? 2 : 1);
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.update_pipe[i] = true;
-}
-
-void atomisp_css_enable_raw_binning(struct atomisp_sub_device *asd,
-					bool enable)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	unsigned int pipe;
-
-	if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-		pipe = IA_CSS_PIPE_ID_VIDEO;
-	else
-		pipe = IA_CSS_PIPE_ID_PREVIEW;
-
-	stream_env->pipe_extra_configs[pipe].enable_raw_binning = enable;
-	stream_env->update_pipe[pipe] = true;
-	if (enable)
-		stream_env->pipe_configs[pipe].output_info[0].padded_width =
-			stream_env->stream_config.input_config.effective_res.width;
-}
-
-void atomisp_css_enable_dz(struct atomisp_sub_device *asd, bool enable)
-{
-	int i;
-
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-			.pipe_configs[i].enable_dz = enable;
-}
-
-void atomisp_css_capture_set_mode(struct atomisp_sub_device *asd,
-				enum atomisp_css_capture_mode mode)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-
-	if (stream_env->pipe_configs[IA_CSS_PIPE_ID_CAPTURE]
-		.default_capture_config.mode == mode)
-		return;
-
-	stream_env->pipe_configs[IA_CSS_PIPE_ID_CAPTURE].
-					default_capture_config.mode = mode;
-	stream_env->update_pipe[IA_CSS_PIPE_ID_CAPTURE] = true;
-}
-
-void atomisp_css_input_set_mode(struct atomisp_sub_device *asd,
-				enum atomisp_css_input_mode mode)
-{
-	int i;
-	struct atomisp_device *isp = asd->isp;
-	unsigned int size_mem_words;
-
-	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++)
-		asd->stream_env[i].stream_config.mode = mode;
-
-	if (isp->inputs[asd->input_curr].type == TEST_PATTERN) {
-		struct ia_css_stream_config *s_config =
-		    &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream_config;
-		s_config->mode = IA_CSS_INPUT_MODE_TPG;
-		s_config->source.tpg.mode = IA_CSS_TPG_MODE_CHECKERBOARD;
-		s_config->source.tpg.x_mask = (1 << 4) - 1;
-		s_config->source.tpg.x_delta = -2;
-		s_config->source.tpg.y_mask = (1 << 4) - 1;
-		s_config->source.tpg.y_delta = 3;
-		s_config->source.tpg.xy_mask = (1 << 8) - 1;
-		return;
-	}
-
-	if (mode != IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
-		return;
-
-	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
-		/*
-		 * TODO: sensor needs to export the embedded_data_size_words
-		 * information to atomisp for each setting.
-		 * Here using a large safe value.
-		 */
-		struct ia_css_stream_config *s_config =
-			&asd->stream_env[i].stream_config;
-
-		if (s_config->input_config.input_res.width == 0)
-			continue;
-
-		if (ia_css_mipi_frame_calculate_size(
-					s_config->input_config.input_res.width,
-					s_config->input_config.input_res.height,
-					s_config->input_config.format,
-					true,
-					0x13000,
-					&size_mem_words) != IA_CSS_SUCCESS) {
-			if (intel_mid_identify_cpu() ==
-				INTEL_MID_CPU_CHIP_TANGIER)
-				size_mem_words = CSS_MIPI_FRAME_BUFFER_SIZE_2;
-			else
-				size_mem_words = CSS_MIPI_FRAME_BUFFER_SIZE_1;
-			dev_warn(asd->isp->dev,
-				"ia_css_mipi_frame_calculate_size failed,"
-				"applying pre-defined MIPI buffer size %u.\n",
-				size_mem_words);
-		}
-		s_config->mipi_buffer_config.size_mem_words = size_mem_words;
-		s_config->mipi_buffer_config.nof_mipi_buffers = 2;
-	}
-}
-
-void atomisp_css_capture_enable_online(struct atomisp_sub_device *asd,
-				unsigned short stream_index, bool enable)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[stream_index];
-
-	if (stream_env->stream_config.online == !!enable)
-		return;
-
-	stream_env->stream_config.online = !!enable;
-	stream_env->update_pipe[IA_CSS_PIPE_ID_CAPTURE] = true;
-}
-
-void atomisp_css_preview_enable_online(struct atomisp_sub_device *asd,
-				unsigned short stream_index, bool enable)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[stream_index];
-	int i;
-
-	if (stream_env->stream_config.online != !!enable) {
-		stream_env->stream_config.online = !!enable;
-		for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
-			stream_env->update_pipe[i] = true;
-	}
-}
-
-void atomisp_css_video_enable_online(struct atomisp_sub_device *asd,
-							bool enable)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_VIDEO];
-	int i;
-
-	if (stream_env->stream_config.online != enable) {
-		stream_env->stream_config.online = enable;
-		for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
-			stream_env->update_pipe[i] = true;
-	}
-}
-
-void atomisp_css_enable_continuous(struct atomisp_sub_device *asd,
-							bool enable)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	int i;
-
-	/*
-	 * To SOC camera, there is only one YUVPP pipe in any case
-	 * including ZSL/SDV/continuous viewfinder, so always set
-	 * stream_config.continuous to 0.
-	 */
-	if (ATOMISP_USE_YUVPP(asd)) {
-		stream_env->stream_config.continuous = 0;
-		stream_env->stream_config.online = 1;
-		return;
-	}
-
-	if (stream_env->stream_config.continuous != !!enable) {
-		stream_env->stream_config.continuous = !!enable;
-		stream_env->stream_config.pack_raw_pixels = true;
-		for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
-			stream_env->update_pipe[i] = true;
-	}
-}
-
-void atomisp_css_enable_cvf(struct atomisp_sub_device *asd,
-				bool enable)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	int i;
-
-	if (stream_env->stream_config.disable_cont_viewfinder != !enable) {
-		stream_env->stream_config.disable_cont_viewfinder = !enable;
-		for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
-			stream_env->update_pipe[i] = true;
-	}
-}
-
-int atomisp_css_input_configure_port(
-		struct atomisp_sub_device *asd,
-		enum mipi_port_id port,
-		unsigned int num_lanes,
-		unsigned int timeout,
-		unsigned int mipi_freq,
-		enum atomisp_input_format metadata_format,
-		unsigned int metadata_width,
-		unsigned int metadata_height)
-{
-	int i;
-	struct atomisp_stream_env *stream_env;
-	/*
-	 * Calculate rx_count as follows:
-	 * Input: mipi_freq                 : CSI-2 bus frequency in Hz
-	 * UI = 1 / (2 * mipi_freq)         : period of one bit on the bus
-	 * min = 85e-9 + 6 * UI             : Limits for rx_count in seconds
-	 * max = 145e-9 + 10 * UI
-	 * rxcount0 = min / (4 / mipi_freq) : convert seconds to byte clocks
-	 * rxcount = rxcount0 - 2           : adjust for better results
-	 * The formula below is simplified version of the above with
-	 * 10-bit fixed points for improved accuracy.
-	 */
-	const unsigned int rxcount =
-		min(((mipi_freq / 46000) - 1280) >> 10, 0xffU) * 0x01010101U;
-
-	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
-		stream_env = &asd->stream_env[i];
-		stream_env->stream_config.source.port.port = port;
-		stream_env->stream_config.source.port.num_lanes = num_lanes;
-		stream_env->stream_config.source.port.timeout = timeout;
-		if (mipi_freq)
-			stream_env->stream_config.source.port.rxcount = rxcount;
-		stream_env->stream_config.
-			metadata_config.data_type = metadata_format;
-		stream_env->stream_config.
-			metadata_config.resolution.width = metadata_width;
-		stream_env->stream_config.
-			metadata_config.resolution.height = metadata_height;
-	}
-
-	return 0;
-}
-
-int atomisp_css_frame_allocate(struct atomisp_css_frame **frame,
-				unsigned int width, unsigned int height,
-				enum atomisp_css_frame_format format,
-				unsigned int padded_width,
-				unsigned int raw_bit_depth)
-{
-	if (ia_css_frame_allocate(frame, width, height, format,
-			padded_width, raw_bit_depth) != IA_CSS_SUCCESS)
-		return -ENOMEM;
-
-	return 0;
-}
-
-int atomisp_css_frame_allocate_from_info(struct atomisp_css_frame **frame,
-				const struct atomisp_css_frame_info *info)
-{
-	if (ia_css_frame_allocate_from_info(frame, info) != IA_CSS_SUCCESS)
-		return -ENOMEM;
-
-	return 0;
-}
-
-void atomisp_css_frame_free(struct atomisp_css_frame *frame)
-{
-	ia_css_frame_free(frame);
-}
-
-int atomisp_css_frame_map(struct atomisp_css_frame **frame,
-				const struct atomisp_css_frame_info *info,
-				const void *data, uint16_t attribute,
-				void *context)
-{
-	if (ia_css_frame_map(frame, info, data, attribute, context)
-	    != IA_CSS_SUCCESS)
-		return -ENOMEM;
-
-	return 0;
-}
-
-int atomisp_css_set_black_frame(struct atomisp_sub_device *asd,
-				const struct atomisp_css_frame *raw_black_frame)
-{
-	if (sh_css_set_black_frame(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		raw_black_frame) != IA_CSS_SUCCESS)
-		return -ENOMEM;
-
-	return 0;
-}
-
-int atomisp_css_allocate_continuous_frames(bool init_time,
-				struct atomisp_sub_device *asd)
-{
-	if (ia_css_alloc_continuous_frame_remain(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream)
-			!= IA_CSS_SUCCESS)
-		return -EINVAL;
-	return 0;
-}
-
-void atomisp_css_update_continuous_frames(struct atomisp_sub_device *asd)
-{
-	ia_css_update_continuous_frames(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream);
-}
-
-int atomisp_css_stop(struct atomisp_sub_device *asd,
-			enum atomisp_css_pipe_id pipe_id, bool in_reset)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_s3a_buf *s3a_buf;
-	struct atomisp_dis_buf *dis_buf;
-	struct atomisp_metadata_buf *md_buf;
-	unsigned long irqflags;
-	unsigned int i;
-
-	/* if is called in atomisp_reset(), force destroy stream */
-	if (__destroy_streams(asd, true))
-		dev_err(isp->dev, "destroy stream failed.\n");
-
-	/* if is called in atomisp_reset(), force destroy all pipes */
-	if (__destroy_pipes(asd, true))
-		dev_err(isp->dev, "destroy pipes failed.\n");
-
-	atomisp_init_raw_buffer_bitmap(asd);
-
-	/*
-	 * SP can not be stop if other streams are in use
-	 */
-	if (atomisp_streaming_count(isp) == 0)
-		ia_css_stop_sp();
-
-	if (!in_reset) {
-		struct atomisp_stream_env *stream_env;
-		int i, j;
-
-		for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
-			stream_env = &asd->stream_env[i];
-			for (j = 0; j < IA_CSS_PIPE_ID_NUM; j++) {
-				ia_css_pipe_config_defaults(
-					&stream_env->pipe_configs[j]);
-				ia_css_pipe_extra_config_defaults(
-					&stream_env->pipe_extra_configs[j]);
-			}
-			ia_css_stream_config_defaults(
-				&stream_env->stream_config);
-		}
-		atomisp_isp_parameters_clean_up(&asd->params.config);
-		asd->params.css_update_params_needed = false;
-	}
-
-	/* move stats buffers to free queue list */
-	while (!list_empty(&asd->s3a_stats_in_css)) {
-		s3a_buf = list_entry(asd->s3a_stats_in_css.next,
-				struct atomisp_s3a_buf, list);
-		list_del(&s3a_buf->list);
-		list_add_tail(&s3a_buf->list, &asd->s3a_stats);
-	}
-	while (!list_empty(&asd->s3a_stats_ready)) {
-		s3a_buf = list_entry(asd->s3a_stats_ready.next,
-				struct atomisp_s3a_buf, list);
-		list_del(&s3a_buf->list);
-		list_add_tail(&s3a_buf->list, &asd->s3a_stats);
-	}
-
-	spin_lock_irqsave(&asd->dis_stats_lock, irqflags);
-	while (!list_empty(&asd->dis_stats_in_css)) {
-		dis_buf = list_entry(asd->dis_stats_in_css.next,
-				struct atomisp_dis_buf, list);
-		list_del(&dis_buf->list);
-		list_add_tail(&dis_buf->list, &asd->dis_stats);
-	}
-	asd->params.dis_proj_data_valid = false;
-	spin_unlock_irqrestore(&asd->dis_stats_lock, irqflags);
-
-	for (i = 0; i < ATOMISP_METADATA_TYPE_NUM; i++) {
-		while (!list_empty(&asd->metadata_in_css[i])) {
-			md_buf = list_entry(asd->metadata_in_css[i].next,
-					struct atomisp_metadata_buf, list);
-			list_del(&md_buf->list);
-			list_add_tail(&md_buf->list, &asd->metadata[i]);
-		}
-		while (!list_empty(&asd->metadata_ready[i])) {
-			md_buf = list_entry(asd->metadata_ready[i].next,
-					struct atomisp_metadata_buf, list);
-			list_del(&md_buf->list);
-			list_add_tail(&md_buf->list, &asd->metadata[i]);
-		}
-	}
-
-	atomisp_flush_params_queue(&asd->video_out_capture);
-	atomisp_flush_params_queue(&asd->video_out_vf);
-	atomisp_flush_params_queue(&asd->video_out_preview);
-	atomisp_flush_params_queue(&asd->video_out_video_capture);
-	atomisp_free_css_parameters(&asd->params.css_param);
-	memset(&asd->params.css_param, 0, sizeof(asd->params.css_param));
-	return 0;
-}
-
-int atomisp_css_continuous_set_num_raw_frames(
-					struct atomisp_sub_device *asd,
-					int num_frames)
-{
-	if (asd->enable_raw_buffer_lock->val) {
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.stream_config.init_num_cont_raw_buf =
-			ATOMISP_CSS2_NUM_OFFLINE_INIT_CONTINUOUS_FRAMES_LOCK_EN;
-		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
-		    asd->params.video_dis_en)
-			asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-			.stream_config.init_num_cont_raw_buf +=
-				ATOMISP_CSS2_NUM_DVS_FRAME_DELAY;
-	} else {
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.stream_config.init_num_cont_raw_buf =
-			ATOMISP_CSS2_NUM_OFFLINE_INIT_CONTINUOUS_FRAMES;
-	}
-
-	if (asd->params.video_dis_en)
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-			.stream_config.init_num_cont_raw_buf +=
-				ATOMISP_CSS2_NUM_DVS_FRAME_DELAY;
-
-	asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.stream_config.target_num_cont_raw_buf = num_frames;
-	return 0;
-}
-
-void atomisp_css_disable_vf_pp(struct atomisp_sub_device *asd,
-			       bool disable)
-{
-	int i;
-
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.pipe_extra_configs[i].disable_vf_pp = !!disable;
-}
-
-static enum ia_css_pipe_mode __pipe_id_to_pipe_mode(
-					struct atomisp_sub_device *asd,
-					enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct camera_mipi_info *mipi_info = atomisp_to_sensor_mipi_info(
-			isp->inputs[asd->input_curr].camera);
-
-	switch (pipe_id) {
-	case IA_CSS_PIPE_ID_COPY:
-		/* Currently only YUVPP mode supports YUV420_Legacy format.
-		 * Revert this when other pipe modes can support
-		 * YUV420_Legacy format.
-		 */
-		if (mipi_info && mipi_info->input_format ==
-			ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY)
-			return IA_CSS_PIPE_MODE_YUVPP;
-		return IA_CSS_PIPE_MODE_COPY;
-	case IA_CSS_PIPE_ID_PREVIEW:
-		return IA_CSS_PIPE_MODE_PREVIEW;
-	case IA_CSS_PIPE_ID_CAPTURE:
-		return IA_CSS_PIPE_MODE_CAPTURE;
-	case IA_CSS_PIPE_ID_VIDEO:
-		return IA_CSS_PIPE_MODE_VIDEO;
-	case IA_CSS_PIPE_ID_ACC:
-		return IA_CSS_PIPE_MODE_ACC;
-	case IA_CSS_PIPE_ID_YUVPP:
-		return IA_CSS_PIPE_MODE_YUVPP;
-	default:
-		WARN_ON(1);
-		return IA_CSS_PIPE_MODE_PREVIEW;
-	}
-
-}
-
-static void __configure_output(struct atomisp_sub_device *asd,
-			       unsigned int stream_index,
-			       unsigned int width, unsigned int height,
-			       unsigned int min_width,
-			       enum ia_css_frame_format format,
-			       enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[stream_index];
-	struct ia_css_stream_config *s_config = &stream_env->stream_config;
-
-	stream_env->pipe_configs[pipe_id].mode =
-		__pipe_id_to_pipe_mode(asd, pipe_id);
-	stream_env->update_pipe[pipe_id] = true;
-
-	stream_env->pipe_configs[pipe_id].output_info[0].res.width = width;
-	stream_env->pipe_configs[pipe_id].output_info[0].res.height = height;
-	stream_env->pipe_configs[pipe_id].output_info[0].format = format;
-	stream_env->pipe_configs[pipe_id].output_info[0].padded_width = min_width;
-
-	/* isp binary 2.2 specific setting*/
-	if (width > s_config->input_config.effective_res.width ||
-	    height > s_config->input_config.effective_res.height) {
-		s_config->input_config.effective_res.width = width;
-		s_config->input_config.effective_res.height = height;
-	}
-
-	dev_dbg(isp->dev, "configuring pipe[%d] output info w=%d.h=%d.f=%d.\n",
-		pipe_id, width, height, format);
-}
-
-static void __configure_video_preview_output(struct atomisp_sub_device *asd,
-			       unsigned int stream_index,
-			       unsigned int width, unsigned int height,
-			       unsigned int min_width,
-			       enum ia_css_frame_format format,
-			       enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[stream_index];
-	struct ia_css_frame_info *css_output_info;
-	struct ia_css_stream_config *stream_config = &stream_env->stream_config;
-
-	stream_env->pipe_configs[pipe_id].mode =
-		__pipe_id_to_pipe_mode(asd, pipe_id);
-	stream_env->update_pipe[pipe_id] = true;
-
-	/*
-	 * second_output will be as video main output in SDV mode
-	 * with SOC camera. output will be as video main output in
-	 * normal video mode.
-	 */
-	if (asd->continuous_mode->val)
-		css_output_info = &stream_env->pipe_configs[pipe_id].
-			output_info[ATOMISP_CSS_OUTPUT_SECOND_INDEX];
-	else
-		css_output_info = &stream_env->pipe_configs[pipe_id].
-			output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
-
-	css_output_info->res.width = width;
-	css_output_info->res.height = height;
-	css_output_info->format = format;
-	css_output_info->padded_width = min_width;
-
-	/* isp binary 2.2 specific setting*/
-	if (width > stream_config->input_config.effective_res.width ||
-	    height > stream_config->input_config.effective_res.height) {
-		stream_config->input_config.effective_res.width = width;
-		stream_config->input_config.effective_res.height = height;
-	}
-
-	dev_dbg(isp->dev, "configuring pipe[%d] output info w=%d.h=%d.f=%d.\n",
-		pipe_id, width, height, format);
-}
-
-/*
- * For CSS2.1, capture pipe uses capture_pp_in_res to configure yuv
- * downscaling input resolution.
- */
-static void __configure_capture_pp_input(struct atomisp_sub_device *asd,
-				 unsigned int width, unsigned int height,
-				 enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	struct ia_css_stream_config *stream_config = &stream_env->stream_config;
-	struct ia_css_pipe_config *pipe_configs =
-		&stream_env->pipe_configs[pipe_id];
-	struct ia_css_pipe_extra_config *pipe_extra_configs =
-		&stream_env->pipe_extra_configs[pipe_id];
-	unsigned int hor_ds_factor = 0, ver_ds_factor = 0;
-
-	if (width == 0 && height == 0)
-		return;
-
-	if (width * 9 / 10 < pipe_configs->output_info[0].res.width ||
-	    height * 9 / 10 < pipe_configs->output_info[0].res.height)
-		return;
-	/* here just copy the calculation in css */
-	hor_ds_factor = CEIL_DIV(width >> 1,
-			pipe_configs->output_info[0].res.width);
-	ver_ds_factor = CEIL_DIV(height >> 1,
-			pipe_configs->output_info[0].res.height);
-
-	if ((asd->isp->media_dev.hw_revision <
-	    (ATOMISP_HW_REVISION_ISP2401 << ATOMISP_HW_REVISION_SHIFT) ||
-	    IS_CHT) && hor_ds_factor != ver_ds_factor) {
-		dev_warn(asd->isp->dev,
-				"Cropping for capture due to FW limitation");
-		return;
-	}
-
-	pipe_configs->mode = __pipe_id_to_pipe_mode(asd, pipe_id);
-	stream_env->update_pipe[pipe_id] = true;
-
-	pipe_extra_configs->enable_yuv_ds = true;
-
-	pipe_configs->capt_pp_in_res.width =
-		stream_config->input_config.effective_res.width;
-	pipe_configs->capt_pp_in_res.height =
-		stream_config->input_config.effective_res.height;
-
-	dev_dbg(isp->dev, "configuring pipe[%d]capture pp input w=%d.h=%d.\n",
-		pipe_id, width, height);
-}
-
-/*
- * For CSS2.1, preview pipe could support bayer downscaling, yuv decimation and
- * yuv downscaling, which needs addtional configurations.
- */
-static void __configure_preview_pp_input(struct atomisp_sub_device *asd,
-				 unsigned int width, unsigned int height,
-				 enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	int out_width, out_height, yuv_ds_in_width, yuv_ds_in_height;
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	struct ia_css_stream_config *stream_config = &stream_env->stream_config;
-	struct ia_css_pipe_config *pipe_configs =
-		&stream_env->pipe_configs[pipe_id];
-	struct ia_css_pipe_extra_config *pipe_extra_configs =
-		&stream_env->pipe_extra_configs[pipe_id];
-	struct ia_css_resolution *bayer_ds_out_res =
-		&pipe_configs->bayer_ds_out_res;
-	struct ia_css_resolution *vf_pp_in_res =
-		&pipe_configs->vf_pp_in_res;
-	struct ia_css_resolution  *effective_res =
-		&stream_config->input_config.effective_res;
-
-	const struct bayer_ds_factor bds_fct[] = {{2, 1}, {3, 2}, {5, 4} };
-	/*
-	 * BZ201033: YUV decimation factor of 4 causes couple of rightmost
-	 * columns to be shaded. Remove this factor to work around the CSS bug.
-	 * const unsigned int yuv_dec_fct[] = {4, 2};
-	 */
-	const unsigned int yuv_dec_fct[] = { 2 };
-	unsigned int i;
-
-	if (width == 0 && height == 0)
-		return;
-
-	pipe_configs->mode = __pipe_id_to_pipe_mode(asd, pipe_id);
-	stream_env->update_pipe[pipe_id] = true;
-
-	out_width = pipe_configs->output_info[0].res.width;
-	out_height = pipe_configs->output_info[0].res.height;
-
-	/*
-	 * The ISP could do bayer downscaling, yuv decimation and yuv
-	 * downscaling:
-	 * 1: Bayer Downscaling: between effective resolution and
-	 * bayer_ds_res_out;
-	 * 2: YUV Decimation: between bayer_ds_res_out and vf_pp_in_res;
-	 * 3: YUV Downscaling: between vf_pp_in_res and final vf output
-	 *
-	 * Rule for Bayer Downscaling: support factor 2, 1.5 and 1.25
-	 * Rule for YUV Decimation: support factor 2, 4
-	 * Rule for YUV Downscaling: arbitary value below 2
-	 *
-	 * General rule of factor distribution among these stages:
-	 * 1: try to do Bayer downscaling first if not in online mode.
-	 * 2: try to do maximum of 2 for YUV downscaling
-	 * 3: the remainling for YUV decimation
-	 *
-	 * Note:
-	 * Do not configure bayer_ds_out_res if:
-	 * online == 1 or continuous == 0 or raw_binning = 0
-	 */
-	if (stream_config->online || !stream_config->continuous ||
-			!pipe_extra_configs->enable_raw_binning) {
-		bayer_ds_out_res->width = 0;
-		bayer_ds_out_res->height = 0;
-	} else {
-		bayer_ds_out_res->width = effective_res->width;
-		bayer_ds_out_res->height = effective_res->height;
-
-		for (i = 0; i < ARRAY_SIZE(bds_fct); i++) {
-			if (effective_res->width >= out_width *
-			    bds_fct[i].numerator / bds_fct[i].denominator &&
-			    effective_res->height >= out_height *
-			    bds_fct[i].numerator / bds_fct[i].denominator) {
-				bayer_ds_out_res->width =
-				    effective_res->width *
-				    bds_fct[i].denominator /
-				    bds_fct[i].numerator;
-				bayer_ds_out_res->height =
-				    effective_res->height *
-				    bds_fct[i].denominator /
-				    bds_fct[i].numerator;
-				break;
-			}
-		}
-	}
-	/*
-	 * calculate YUV Decimation, YUV downscaling facor:
-	 * YUV Downscaling factor must not exceed 2.
-	 * YUV Decimation factor could be 2, 4.
-	 */
-	/* first decide the yuv_ds input resolution */
-	if (bayer_ds_out_res->width == 0) {
-		yuv_ds_in_width = effective_res->width;
-		yuv_ds_in_height = effective_res->height;
-	} else {
-		yuv_ds_in_width = bayer_ds_out_res->width;
-		yuv_ds_in_height = bayer_ds_out_res->height;
-	}
-
-	vf_pp_in_res->width = yuv_ds_in_width;
-	vf_pp_in_res->height = yuv_ds_in_height;
-
-	/* find out the yuv decimation factor */
-	for (i = 0; i < ARRAY_SIZE(yuv_dec_fct); i++) {
-		if (yuv_ds_in_width >= out_width * yuv_dec_fct[i] &&
-		    yuv_ds_in_height >= out_height * yuv_dec_fct[i]) {
-			vf_pp_in_res->width = yuv_ds_in_width / yuv_dec_fct[i];
-			vf_pp_in_res->height = yuv_ds_in_height / yuv_dec_fct[i];
-			break;
-		}
-	}
-
-	if (vf_pp_in_res->width == out_width &&
-		vf_pp_in_res->height == out_height) {
-		pipe_extra_configs->enable_yuv_ds = false;
-		vf_pp_in_res->width = 0;
-		vf_pp_in_res->height = 0;
-	} else {
-		pipe_extra_configs->enable_yuv_ds = true;
-	}
-
-	dev_dbg(isp->dev, "configuring pipe[%d]preview pp input w=%d.h=%d.\n",
-		pipe_id, width, height);
-}
-
-/*
- * For CSS2.1, offline video pipe could support bayer decimation, and
- * yuv downscaling, which needs addtional configurations.
- */
-static void __configure_video_pp_input(struct atomisp_sub_device *asd,
-				 unsigned int width, unsigned int height,
-				 enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	int out_width, out_height;
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	struct ia_css_stream_config *stream_config = &stream_env->stream_config;
-	struct ia_css_pipe_config *pipe_configs =
-		&stream_env->pipe_configs[pipe_id];
-	struct ia_css_pipe_extra_config *pipe_extra_configs =
-		&stream_env->pipe_extra_configs[pipe_id];
-	struct ia_css_resolution *bayer_ds_out_res =
-		&pipe_configs->bayer_ds_out_res;
-	struct ia_css_resolution  *effective_res =
-		&stream_config->input_config.effective_res;
-
-	const struct bayer_ds_factor bds_factors[] = {
-		{8, 1}, {6, 1}, {4, 1}, {3, 1}, {2, 1}, {3, 2} };
-	unsigned int i;
-
-	if (width == 0 && height == 0)
-		return;
-
-	pipe_configs->mode = __pipe_id_to_pipe_mode(asd, pipe_id);
-	stream_env->update_pipe[pipe_id] = true;
-
-	pipe_extra_configs->enable_yuv_ds = false;
-
-	/*
-	 * If DVS is enabled,  video binary will take care the dvs envelope
-	 * and usually the bayer_ds_out_res should be larger than 120% of
-	 * destination resolution, the extra 20% will be cropped as DVS
-	 * envelope. But,  if the bayer_ds_out_res is less than 120% of the
-	 * destination. The ISP can still work,  but DVS quality is not good.
-	 */
-	/* taking at least 10% as envelope */
-	if (asd->params.video_dis_en) {
-		out_width = pipe_configs->output_info[0].res.width * 110 / 100;
-		out_height = pipe_configs->output_info[0].res.height * 110 / 100;
-	} else {
-		out_width = pipe_configs->output_info[0].res.width;
-		out_height = pipe_configs->output_info[0].res.height;
-	}
-
-	/*
-	 * calculate bayer decimate factor:
-	 * 1: only 1.5, 2, 4 and 8 get supported
-	 * 2: Do not configure bayer_ds_out_res if:
-	 *    online == 1 or continuous == 0 or raw_binning = 0
-	 */
-	if (stream_config->online || !stream_config->continuous) {
-		bayer_ds_out_res->width = 0;
-		bayer_ds_out_res->height = 0;
-		goto done;
-	}
-
-	pipe_extra_configs->enable_raw_binning = true;
-	bayer_ds_out_res->width = effective_res->width;
-	bayer_ds_out_res->height = effective_res->height;
-
-	for (i = 0; i < sizeof(bds_factors) / sizeof(struct bayer_ds_factor);
-	     i++) {
-		if (effective_res->width >= out_width *
-		    bds_factors[i].numerator / bds_factors[i].denominator &&
-		    effective_res->height >= out_height *
-		    bds_factors[i].numerator / bds_factors[i].denominator) {
-			bayer_ds_out_res->width = effective_res->width *
-			    bds_factors[i].denominator /
-			    bds_factors[i].numerator;
-			bayer_ds_out_res->height = effective_res->height *
-			    bds_factors[i].denominator /
-			    bds_factors[i].numerator;
-			break;
-		}
-	}
-
-	/*
-	 * DVS is cropped from BDS output, so we do not really need to set the
-	 * envelope to 20% of output resolution here. always set it to 12x12
-	 * per firmware requirement.
-	 */
-	pipe_configs->dvs_envelope.width = 12;
-	pipe_configs->dvs_envelope.height = 12;
-
-done:
-	if (pipe_id == IA_CSS_PIPE_ID_YUVPP)
-		stream_config->left_padding = -1;
-	else
-		stream_config->left_padding = 12;
-	dev_dbg(isp->dev, "configuring pipe[%d]video pp input w=%d.h=%d.\n",
-		pipe_id, width, height);
-}
-
-static void __configure_vf_output(struct atomisp_sub_device *asd,
-				  unsigned int width, unsigned int height,
-				  unsigned int min_width,
-				  enum atomisp_css_frame_format format,
-				  enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	stream_env->pipe_configs[pipe_id].mode =
-		__pipe_id_to_pipe_mode(asd, pipe_id);
-	stream_env->update_pipe[pipe_id] = true;
-
-	stream_env->pipe_configs[pipe_id].vf_output_info[0].res.width = width;
-	stream_env->pipe_configs[pipe_id].vf_output_info[0].res.height = height;
-	stream_env->pipe_configs[pipe_id].vf_output_info[0].format = format;
-	stream_env->pipe_configs[pipe_id].vf_output_info[0].padded_width =
-		min_width;
-	dev_dbg(isp->dev,
-		"configuring pipe[%d] vf output info w=%d.h=%d.f=%d.\n",
-		 pipe_id, width, height, format);
-}
-
-static void __configure_video_vf_output(struct atomisp_sub_device *asd,
-				  unsigned int width, unsigned int height,
-				  unsigned int min_width,
-				  enum atomisp_css_frame_format format,
-				  enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	struct ia_css_frame_info *css_output_info;
-
-	stream_env->pipe_configs[pipe_id].mode =
-					__pipe_id_to_pipe_mode(asd, pipe_id);
-	stream_env->update_pipe[pipe_id] = true;
-
-	/*
-	 * second_vf_output will be as video viewfinder in SDV mode
-	 * with SOC camera. vf_output will be as video viewfinder in
-	 * normal video mode.
-	 */
-	if (asd->continuous_mode->val)
-		css_output_info = &stream_env->pipe_configs[pipe_id].
-			vf_output_info[ATOMISP_CSS_OUTPUT_SECOND_INDEX];
-	else
-		css_output_info = &stream_env->pipe_configs[pipe_id].
-			vf_output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
-
-	css_output_info->res.width = width;
-	css_output_info->res.height = height;
-	css_output_info->format = format;
-	css_output_info->padded_width = min_width;
-	dev_dbg(isp->dev,
-		"configuring pipe[%d] vf output info w=%d.h=%d.f=%d.\n",
-		 pipe_id, width, height, format);
-}
-
-static int __get_frame_info(struct atomisp_sub_device *asd,
-				unsigned int stream_index,
-				struct atomisp_css_frame_info *info,
-				enum frame_info_type type,
-				enum ia_css_pipe_id pipe_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	enum ia_css_err ret;
-	struct ia_css_pipe_info p_info;
-
-	/* FIXME! No need to destroy/recreate all streams */
-	if (__destroy_streams(asd, true))
-		dev_warn(isp->dev, "destroy stream failed.\n");
-
-	if (__destroy_pipes(asd, true))
-		dev_warn(isp->dev, "destroy pipe failed.\n");
-
-	if (__create_pipes(asd))
-		return -EINVAL;
-
-	if (__create_streams(asd))
-		goto stream_err;
-
-	ret = ia_css_pipe_get_info(
-		asd->stream_env[stream_index]
-		.pipes[pipe_id], &p_info);
-	if (ret == IA_CSS_SUCCESS) {
-		switch (type) {
-		case ATOMISP_CSS_VF_FRAME:
-			*info = p_info.vf_output_info[0];
-			dev_dbg(isp->dev, "getting vf frame info.\n");
-			break;
-		case ATOMISP_CSS_SECOND_VF_FRAME:
-			*info = p_info.vf_output_info[1];
-			dev_dbg(isp->dev, "getting second vf frame info.\n");
-			break;
-		case ATOMISP_CSS_OUTPUT_FRAME:
-			*info = p_info.output_info[0];
-			dev_dbg(isp->dev, "getting main frame info.\n");
-			break;
-		case ATOMISP_CSS_SECOND_OUTPUT_FRAME:
-			*info = p_info.output_info[1];
-			dev_dbg(isp->dev, "getting second main frame info.\n");
-			break;
-		case ATOMISP_CSS_RAW_FRAME:
-			*info = p_info.raw_output_info;
-			dev_dbg(isp->dev, "getting raw frame info.\n");
-		}
-		dev_dbg(isp->dev, "get frame info: w=%d, h=%d, num_invalid_frames %d.\n",
-			info->res.width, info->res.height, p_info.num_invalid_frames);
-		return 0;
-	}
-
-stream_err:
-	__destroy_pipes(asd, true);
-	return -EINVAL;
-}
-
-static unsigned int atomisp_get_pipe_index(struct atomisp_sub_device *asd,
-					   uint16_t source_pad)
-{
-	struct atomisp_device *isp = asd->isp;
-	/*
-	 * to SOC camera, use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		return IA_CSS_PIPE_ID_YUVPP;
-
-	switch (source_pad) {
-	case ATOMISP_SUBDEV_PAD_SOURCE_VIDEO:
-		if (asd->yuvpp_mode)
-			return IA_CSS_PIPE_ID_YUVPP;
-		if (asd->copy_mode)
-			return IA_CSS_PIPE_ID_COPY;
-		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO
-		    || asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER)
-			return IA_CSS_PIPE_ID_VIDEO;
-		else
-			return IA_CSS_PIPE_ID_CAPTURE;
-	case ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE:
-		if (asd->copy_mode)
-			return IA_CSS_PIPE_ID_COPY;
-		return IA_CSS_PIPE_ID_CAPTURE;
-	case ATOMISP_SUBDEV_PAD_SOURCE_VF:
-		if (!atomisp_is_mbuscode_raw(
-		    asd->fmt[asd->capture_pad].fmt.code))
-			return IA_CSS_PIPE_ID_CAPTURE;
-	case ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW:
-		if (asd->yuvpp_mode)
-			return IA_CSS_PIPE_ID_YUVPP;
-		if (asd->copy_mode)
-			return IA_CSS_PIPE_ID_COPY;
-		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-			return IA_CSS_PIPE_ID_VIDEO;
-		else
-			return IA_CSS_PIPE_ID_PREVIEW;
-	}
-	dev_warn(isp->dev,
-		 "invalid source pad:%d, return default preview pipe index.\n",
-		 source_pad);
-	return IA_CSS_PIPE_ID_PREVIEW;
-}
-
-int atomisp_get_css_frame_info(struct atomisp_sub_device *asd,
-				uint16_t source_pad,
-				struct atomisp_css_frame_info *frame_info)
-{
-	struct ia_css_pipe_info info;
-	int pipe_index = atomisp_get_pipe_index(asd, source_pad);
-	int stream_index;
-	struct atomisp_device *isp = asd->isp;
-
-	if (ATOMISP_SOC_CAMERA(asd))
-		stream_index = atomisp_source_pad_to_stream_id(asd, source_pad);
-	else {
-		stream_index = (pipe_index == IA_CSS_PIPE_ID_YUVPP) ?
-			   ATOMISP_INPUT_STREAM_VIDEO :
-			   atomisp_source_pad_to_stream_id(asd, source_pad);
-	}
-
-	if (IA_CSS_SUCCESS != ia_css_pipe_get_info(asd->stream_env[stream_index]
-				 .pipes[pipe_index], &info)) {
-		dev_err(isp->dev, "ia_css_pipe_get_info FAILED");
-		return -EINVAL;
-	}
-
-	switch (source_pad) {
-	case ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE:
-		*frame_info = info.output_info[0];
-		break;
-	case ATOMISP_SUBDEV_PAD_SOURCE_VIDEO:
-		if (ATOMISP_USE_YUVPP(asd) && asd->continuous_mode->val)
-			*frame_info = info.
-				output_info[ATOMISP_CSS_OUTPUT_SECOND_INDEX];
-		else
-			*frame_info = info.
-				output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
-		break;
-	case ATOMISP_SUBDEV_PAD_SOURCE_VF:
-		if (stream_index == ATOMISP_INPUT_STREAM_POSTVIEW)
-			*frame_info = info.output_info[0];
-		else
-			*frame_info = info.vf_output_info[0];
-		break;
-	case ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW:
-		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
-		    (pipe_index == IA_CSS_PIPE_ID_VIDEO ||
-		     pipe_index == IA_CSS_PIPE_ID_YUVPP))
-			if (ATOMISP_USE_YUVPP(asd) && asd->continuous_mode->val)
-				*frame_info = info.
-					vf_output_info[ATOMISP_CSS_OUTPUT_SECOND_INDEX];
-			else
-				*frame_info = info.
-					vf_output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
-		else if (ATOMISP_USE_YUVPP(asd) && asd->continuous_mode->val)
-			*frame_info =
-				info.output_info[ATOMISP_CSS_OUTPUT_SECOND_INDEX];
-		else
-			*frame_info =
-				info.output_info[ATOMISP_CSS_OUTPUT_DEFAULT_INDEX];
-
-		break;
-	default:
-		frame_info = NULL;
-		break;
-	}
-	return frame_info ? 0 : -EINVAL;
-}
-
-int atomisp_css_copy_configure_output(struct atomisp_sub_device *asd,
-				unsigned int stream_index,
-				unsigned int width, unsigned int height,
-				unsigned int padded_width,
-				enum atomisp_css_frame_format format)
-{
-	asd->stream_env[stream_index].pipe_configs[IA_CSS_PIPE_ID_COPY].
-					default_capture_config.mode =
-					CSS_CAPTURE_MODE_RAW;
-
-	__configure_output(asd, stream_index, width, height, padded_width,
-			   format, IA_CSS_PIPE_ID_COPY);
-	return 0;
-}
-
-int atomisp_css_yuvpp_configure_output(struct atomisp_sub_device *asd,
-				unsigned int stream_index,
-				unsigned int width, unsigned int height,
-				unsigned int padded_width,
-				enum atomisp_css_frame_format format)
-{
-	asd->stream_env[stream_index].pipe_configs[IA_CSS_PIPE_ID_YUVPP].
-					default_capture_config.mode =
-					CSS_CAPTURE_MODE_RAW;
-
-	__configure_output(asd, stream_index, width, height, padded_width,
-			   format, IA_CSS_PIPE_ID_YUVPP);
-	return 0;
-}
-
-int atomisp_css_yuvpp_configure_viewfinder(
-				struct atomisp_sub_device *asd,
-				unsigned int stream_index,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format format)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[stream_index];
-	enum ia_css_pipe_id pipe_id = IA_CSS_PIPE_ID_YUVPP;
-
-	stream_env->pipe_configs[pipe_id].mode =
-		__pipe_id_to_pipe_mode(asd, pipe_id);
-	stream_env->update_pipe[pipe_id] = true;
-
-	stream_env->pipe_configs[pipe_id].vf_output_info[0].res.width = width;
-	stream_env->pipe_configs[pipe_id].vf_output_info[0].res.height = height;
-	stream_env->pipe_configs[pipe_id].vf_output_info[0].format = format;
-	stream_env->pipe_configs[pipe_id].vf_output_info[0].padded_width =
-		min_width;
-	return 0;
-}
-
-int atomisp_css_yuvpp_get_output_frame_info(
-					struct atomisp_sub_device *asd,
-					unsigned int stream_index,
-					struct atomisp_css_frame_info *info)
-{
-	return __get_frame_info(asd, stream_index, info,
-			ATOMISP_CSS_OUTPUT_FRAME, IA_CSS_PIPE_ID_YUVPP);
-}
-
-int atomisp_css_yuvpp_get_viewfinder_frame_info(
-					struct atomisp_sub_device *asd,
-					unsigned int stream_index,
-					struct atomisp_css_frame_info *info)
-{
-	return __get_frame_info(asd, stream_index, info,
-			ATOMISP_CSS_VF_FRAME, IA_CSS_PIPE_ID_YUVPP);
-}
-
-int atomisp_css_preview_configure_output(struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format format)
-{
-	/*
-	 * to SOC camera, use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		__configure_video_preview_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width, height,
-						min_width, format, IA_CSS_PIPE_ID_YUVPP);
-	else
-		__configure_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width, height,
-					min_width, format, IA_CSS_PIPE_ID_PREVIEW);
-	return 0;
-}
-
-int atomisp_css_capture_configure_output(struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format format)
-{
-	enum ia_css_pipe_id pipe_id;
-
-	/*
-	 * to SOC camera, use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-	else
-		pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-
-	__configure_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width, height,
-						min_width, format, pipe_id);
-	return 0;
-}
-
-int atomisp_css_video_configure_output(struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format format)
-{
-	/*
-	 * to SOC camera, use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		__configure_video_preview_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width, height,
-					min_width, format, IA_CSS_PIPE_ID_YUVPP);
-	else
-		__configure_output(asd, ATOMISP_INPUT_STREAM_GENERAL, width, height,
-					min_width, format, IA_CSS_PIPE_ID_VIDEO);
-	return 0;
-}
-
-int atomisp_css_video_configure_viewfinder(
-				struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format format)
-{
-	/*
-	 * to SOC camera, video will use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		__configure_video_vf_output(asd, width, height, min_width, format,
-							IA_CSS_PIPE_ID_YUVPP);
-	else
-		__configure_vf_output(asd, width, height, min_width, format,
-							IA_CSS_PIPE_ID_VIDEO);
-	return 0;
-}
-
-int atomisp_css_capture_configure_viewfinder(
-				struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height,
-				unsigned int min_width,
-				enum atomisp_css_frame_format format)
-{
-	enum ia_css_pipe_id pipe_id;
-
-	/*
-	 * to SOC camera, video will use yuvpp pipe.
-	 */
-	if (ATOMISP_USE_YUVPP(asd))
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-	else
-		pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-
-	__configure_vf_output(asd, width, height, min_width, format,
-							pipe_id);
-	return 0;
-}
-
-int atomisp_css_video_get_viewfinder_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info)
-{
-	enum ia_css_pipe_id pipe_id;
-	enum frame_info_type frame_type = ATOMISP_CSS_VF_FRAME;
-
-	if (ATOMISP_USE_YUVPP(asd)) {
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-		if (asd->continuous_mode->val)
-			frame_type = ATOMISP_CSS_SECOND_VF_FRAME;
-	} else {
-		pipe_id = IA_CSS_PIPE_ID_VIDEO;
-	}
-
-	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-						frame_type, pipe_id);
-}
-
-int atomisp_css_capture_get_viewfinder_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info)
-{
-	enum ia_css_pipe_id pipe_id;
-
-	if (ATOMISP_USE_YUVPP(asd))
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-	else
-		pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-
-	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-						ATOMISP_CSS_VF_FRAME, pipe_id);
-}
-
-int atomisp_css_capture_get_output_raw_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info)
-{
-	if (ATOMISP_USE_YUVPP(asd))
-		return 0;
-
-	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-			ATOMISP_CSS_RAW_FRAME, IA_CSS_PIPE_ID_CAPTURE);
-}
-
-int atomisp_css_copy_get_output_frame_info(
-					struct atomisp_sub_device *asd,
-					unsigned int stream_index,
-					struct atomisp_css_frame_info *info)
-{
-	return __get_frame_info(asd, stream_index, info,
-			ATOMISP_CSS_OUTPUT_FRAME, IA_CSS_PIPE_ID_COPY);
-}
-
-int atomisp_css_preview_get_output_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info)
-{
-	enum ia_css_pipe_id pipe_id;
-	enum frame_info_type frame_type = ATOMISP_CSS_OUTPUT_FRAME;
-
-	if (ATOMISP_USE_YUVPP(asd)) {
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-		if (asd->continuous_mode->val)
-			frame_type = ATOMISP_CSS_SECOND_OUTPUT_FRAME;
-	} else {
-		pipe_id = IA_CSS_PIPE_ID_PREVIEW;
-	}
-
-	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-					frame_type, pipe_id);
-}
-
-int atomisp_css_capture_get_output_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info)
-{
-	enum ia_css_pipe_id pipe_id;
-
-	if (ATOMISP_USE_YUVPP(asd))
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-	else
-		pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-
-	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-					ATOMISP_CSS_OUTPUT_FRAME, pipe_id);
-}
-
-int atomisp_css_video_get_output_frame_info(
-					struct atomisp_sub_device *asd,
-					struct atomisp_css_frame_info *info)
-{
-	enum ia_css_pipe_id pipe_id;
-	enum frame_info_type frame_type = ATOMISP_CSS_OUTPUT_FRAME;
-
-	if (ATOMISP_USE_YUVPP(asd)) {
-		pipe_id = IA_CSS_PIPE_ID_YUVPP;
-		if (asd->continuous_mode->val)
-			frame_type = ATOMISP_CSS_SECOND_OUTPUT_FRAME;
-	} else {
-		pipe_id = IA_CSS_PIPE_ID_VIDEO;
-	}
-
-	return __get_frame_info(asd, ATOMISP_INPUT_STREAM_GENERAL, info,
-					frame_type, pipe_id);
-}
-
-int atomisp_css_preview_configure_pp_input(
-				struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	__configure_preview_pp_input(asd, width, height,
-		ATOMISP_USE_YUVPP(asd) ?
-		IA_CSS_PIPE_ID_YUVPP : IA_CSS_PIPE_ID_PREVIEW);
-
-	if (width > stream_env->pipe_configs[IA_CSS_PIPE_ID_CAPTURE].
-					capt_pp_in_res.width)
-		__configure_capture_pp_input(asd, width, height,
-			ATOMISP_USE_YUVPP(asd) ?
-		IA_CSS_PIPE_ID_YUVPP : IA_CSS_PIPE_ID_CAPTURE);
-	return 0;
-}
-
-int atomisp_css_capture_configure_pp_input(
-				struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height)
-{
-	__configure_capture_pp_input(asd, width, height,
-		ATOMISP_USE_YUVPP(asd) ?
-		IA_CSS_PIPE_ID_YUVPP : IA_CSS_PIPE_ID_CAPTURE);
-	return 0;
-}
-
-int atomisp_css_video_configure_pp_input(
-				struct atomisp_sub_device *asd,
-				unsigned int width, unsigned int height)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-
-	__configure_video_pp_input(asd, width, height,
-		ATOMISP_USE_YUVPP(asd) ?
-		IA_CSS_PIPE_ID_YUVPP : IA_CSS_PIPE_ID_VIDEO);
-
-	if (width > stream_env->pipe_configs[IA_CSS_PIPE_ID_CAPTURE].
-					capt_pp_in_res.width)
-		__configure_capture_pp_input(asd, width, height,
-			ATOMISP_USE_YUVPP(asd) ?
-			IA_CSS_PIPE_ID_YUVPP : IA_CSS_PIPE_ID_CAPTURE);
-	return 0;
-}
-
-int atomisp_css_offline_capture_configure(struct atomisp_sub_device *asd,
-			int num_captures, unsigned int skip, int offset)
-{
-	enum ia_css_err ret;
-
-#ifdef ISP2401
-	dev_dbg(asd->isp->dev, "%s num_capture:%d skip:%d offset:%d\n",
-			__func__, num_captures, skip, offset);
-#endif
-	ret = ia_css_stream_capture(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		num_captures, skip, offset);
-	if (ret != IA_CSS_SUCCESS)
-		return -EINVAL;
-
-	return 0;
-}
-
-int atomisp_css_exp_id_capture(struct atomisp_sub_device *asd, int exp_id)
-{
-	enum ia_css_err ret;
-
-	ret = ia_css_stream_capture_frame(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		exp_id);
-	if (ret == IA_CSS_ERR_QUEUE_IS_FULL) {
-		/* capture cmd queue is full */
-		return -EBUSY;
-	} else if (ret != IA_CSS_SUCCESS) {
-		return -EIO;
-	}
-
-	return 0;
-}
-
-int atomisp_css_exp_id_unlock(struct atomisp_sub_device *asd, int exp_id)
-{
-	enum ia_css_err ret;
-
-	ret = ia_css_unlock_raw_frame(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		exp_id);
-	if (ret == IA_CSS_ERR_QUEUE_IS_FULL)
-		return -EAGAIN;
-	else if (ret != IA_CSS_SUCCESS)
-		return -EIO;
-
-	return 0;
-}
-
-int atomisp_css_capture_enable_xnr(struct atomisp_sub_device *asd,
-				   bool enable)
-{
-	asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.pipe_configs[IA_CSS_PIPE_ID_CAPTURE]
-		.default_capture_config.enable_xnr = enable;
-	asd->params.capture_config.enable_xnr = enable;
-	asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.update_pipe[IA_CSS_PIPE_ID_CAPTURE] = true;
-
-	return 0;
-}
-
-void atomisp_css_send_input_frame(struct atomisp_sub_device *asd,
-				  unsigned short *data, unsigned int width,
-				  unsigned int height)
-{
-	ia_css_stream_send_input_frame(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		data, width, height);
-}
-
-bool atomisp_css_isp_has_started(void)
-{
-	return ia_css_isp_has_started();
-}
-
-void atomisp_css_request_flash(struct atomisp_sub_device *asd)
-{
-	ia_css_stream_request_flash(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream);
-}
-
-void atomisp_css_set_wb_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_wb_config *wb_config)
-{
-	asd->params.config.wb_config = wb_config;
-}
-
-void atomisp_css_set_ob_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ob_config *ob_config)
-{
-	asd->params.config.ob_config = ob_config;
-}
-
-void atomisp_css_set_dp_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_dp_config *dp_config)
-{
-	asd->params.config.dp_config = dp_config;
-}
-
-void atomisp_css_set_de_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_de_config *de_config)
-{
-	asd->params.config.de_config = de_config;
-}
-
-void atomisp_css_set_dz_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_dz_config *dz_config)
-{
-	asd->params.config.dz_config = dz_config;
-}
-
-void atomisp_css_set_default_de_config(struct atomisp_sub_device *asd)
-{
-	asd->params.config.de_config = NULL;
-}
-
-void atomisp_css_set_ce_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ce_config *ce_config)
-{
-	asd->params.config.ce_config = ce_config;
-}
-
-void atomisp_css_set_nr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_nr_config *nr_config)
-{
-	asd->params.config.nr_config = nr_config;
-}
-
-void atomisp_css_set_ee_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ee_config *ee_config)
-{
-	asd->params.config.ee_config = ee_config;
-}
-
-void atomisp_css_set_tnr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_tnr_config *tnr_config)
-{
-	asd->params.config.tnr_config = tnr_config;
-}
-
-void atomisp_css_set_cc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_cc_config *cc_config)
-{
-	asd->params.config.cc_config = cc_config;
-}
-
-void atomisp_css_set_macc_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_macc_table *macc_table)
-{
-	asd->params.config.macc_table = macc_table;
-}
-
-void atomisp_css_set_macc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_macc_config *macc_config)
-{
-	asd->params.config.macc_config = macc_config;
-}
-
-void atomisp_css_set_ecd_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ecd_config *ecd_config)
-{
-	asd->params.config.ecd_config = ecd_config;
-}
-
-void atomisp_css_set_ynr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ynr_config *ynr_config)
-{
-	asd->params.config.ynr_config = ynr_config;
-}
-
-void atomisp_css_set_fc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_fc_config *fc_config)
-{
-	asd->params.config.fc_config = fc_config;
-}
-
-void atomisp_css_set_ctc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ctc_config *ctc_config)
-{
-	asd->params.config.ctc_config = ctc_config;
-}
-
-void atomisp_css_set_cnr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_cnr_config *cnr_config)
-{
-	asd->params.config.cnr_config = cnr_config;
-}
-
-void atomisp_css_set_aa_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_aa_config *aa_config)
-{
-	asd->params.config.aa_config = aa_config;
-}
-
-void atomisp_css_set_baa_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_baa_config *baa_config)
-{
-	asd->params.config.baa_config = baa_config;
-}
-
-void atomisp_css_set_anr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_anr_config *anr_config)
-{
-	asd->params.config.anr_config = anr_config;
-}
-
-void atomisp_css_set_xnr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_xnr_config *xnr_config)
-{
-	asd->params.config.xnr_config = xnr_config;
-}
-
-void atomisp_css_set_yuv2rgb_cc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_cc_config *yuv2rgb_cc_config)
-{
-	asd->params.config.yuv2rgb_cc_config = yuv2rgb_cc_config;
-}
-
-void atomisp_css_set_rgb2yuv_cc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_cc_config *rgb2yuv_cc_config)
-{
-	asd->params.config.rgb2yuv_cc_config = rgb2yuv_cc_config;
-}
-
-void atomisp_css_set_xnr_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_xnr_table *xnr_table)
-{
-	asd->params.config.xnr_table = xnr_table;
-}
-
-void atomisp_css_set_r_gamma_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_rgb_gamma_table *r_gamma_table)
-{
-	asd->params.config.r_gamma_table = r_gamma_table;
-}
-
-void atomisp_css_set_g_gamma_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_rgb_gamma_table *g_gamma_table)
-{
-	asd->params.config.g_gamma_table = g_gamma_table;
-}
-
-void atomisp_css_set_b_gamma_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_rgb_gamma_table *b_gamma_table)
-{
-	asd->params.config.b_gamma_table = b_gamma_table;
-}
-
-void atomisp_css_set_gamma_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_gamma_table *gamma_table)
-{
-	asd->params.config.gamma_table = gamma_table;
-}
-
-void atomisp_css_set_ctc_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_ctc_table *ctc_table)
-{
-	int i;
-	uint16_t *vamem_ptr = ctc_table->data.vamem_1;
-	int data_size = IA_CSS_VAMEM_1_CTC_TABLE_SIZE;
-	bool valid = false;
-
-	/* workaround: if ctc_table is all 0, do not apply it */
-	if (ctc_table->vamem_type == IA_CSS_VAMEM_TYPE_2) {
-		vamem_ptr = ctc_table->data.vamem_2;
-		data_size = IA_CSS_VAMEM_2_CTC_TABLE_SIZE;
-	}
-
-	for (i = 0; i < data_size; i++) {
-		if (*(vamem_ptr + i)) {
-			valid = true;
-			break;
-		}
-	}
-
-	if (valid)
-		asd->params.config.ctc_table = ctc_table;
-	else
-		dev_warn(asd->isp->dev, "Bypass the invalid ctc_table.\n");
-}
-
-void atomisp_css_set_anr_thres(struct atomisp_sub_device *asd,
-			struct atomisp_css_anr_thres *anr_thres)
-{
-	asd->params.config.anr_thres = anr_thres;
-}
-
-void atomisp_css_set_dvs_6axis(struct atomisp_sub_device *asd,
-			struct atomisp_css_dvs_6axis *dvs_6axis)
-{
-	asd->params.config.dvs_6axis_config = dvs_6axis;
-}
-
-void atomisp_css_set_gc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_gc_config *gc_config)
-{
-	asd->params.config.gc_config = gc_config;
-}
-
-void atomisp_css_set_3a_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_3a_config *s3a_config)
-{
-	asd->params.config.s3a_config = s3a_config;
-}
-
-void atomisp_css_video_set_dis_vector(struct atomisp_sub_device *asd,
-				struct atomisp_dis_vector *vector)
-{
-	if (!asd->params.config.motion_vector)
-		asd->params.config.motion_vector = &asd->params.css_param.motion_vector;
-
-	memset(asd->params.config.motion_vector,
-			0, sizeof(struct ia_css_vector));
-	asd->params.css_param.motion_vector.x = vector->x;
-	asd->params.css_param.motion_vector.y = vector->y;
-}
-
-static int atomisp_compare_dvs_grid(struct atomisp_sub_device *asd,
-				struct atomisp_dvs_grid_info *atomgrid)
-{
-	struct atomisp_css_dvs_grid_info *cur =
-		atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
-
-	if (!cur) {
-		dev_err(asd->isp->dev, "dvs grid not available!\n");
-		return -EINVAL;
-	}
-
-	if (sizeof(*cur) != sizeof(*atomgrid)) {
-		dev_err(asd->isp->dev, "dvs grid mis-match!\n");
-		return -EINVAL;
-	}
-
-	if (!cur->enable) {
-		dev_err(asd->isp->dev, "dvs not enabled!\n");
-		return -EINVAL;
-	}
-
-	return memcmp(atomgrid, cur, sizeof(*cur));
-}
-
-void  atomisp_css_set_dvs2_coefs(struct atomisp_sub_device *asd,
-			       struct ia_css_dvs2_coefficients *coefs)
-{
-	asd->params.config.dvs2_coefs = coefs;
-}
-
-int atomisp_css_set_dis_coefs(struct atomisp_sub_device *asd,
-			  struct atomisp_dis_coefficients *coefs)
-{
-	if (atomisp_compare_dvs_grid(asd, &coefs->grid_info) != 0)
-		/* If the grid info in the argument differs from the current
-		   grid info, we tell the caller to reset the grid size and
-		   try again. */
-		return -EAGAIN;
-
-	if (coefs->hor_coefs.odd_real == NULL ||
-	    coefs->hor_coefs.odd_imag == NULL ||
-	    coefs->hor_coefs.even_real == NULL ||
-	    coefs->hor_coefs.even_imag == NULL ||
-	    coefs->ver_coefs.odd_real == NULL ||
-	    coefs->ver_coefs.odd_imag == NULL ||
-	    coefs->ver_coefs.even_real == NULL ||
-	    coefs->ver_coefs.even_imag == NULL ||
-	    asd->params.css_param.dvs2_coeff->hor_coefs.odd_real == NULL ||
-	    asd->params.css_param.dvs2_coeff->hor_coefs.odd_imag == NULL ||
-	    asd->params.css_param.dvs2_coeff->hor_coefs.even_real == NULL ||
-	    asd->params.css_param.dvs2_coeff->hor_coefs.even_imag == NULL ||
-	    asd->params.css_param.dvs2_coeff->ver_coefs.odd_real == NULL ||
-	    asd->params.css_param.dvs2_coeff->ver_coefs.odd_imag == NULL ||
-	    asd->params.css_param.dvs2_coeff->ver_coefs.even_real == NULL ||
-	    asd->params.css_param.dvs2_coeff->ver_coefs.even_imag == NULL)
-		return -EINVAL;
-
-	if (copy_from_user(asd->params.css_param.dvs2_coeff->hor_coefs.odd_real,
-	    coefs->hor_coefs.odd_real, asd->params.dvs_hor_coef_bytes))
-		return -EFAULT;
-	if (copy_from_user(asd->params.css_param.dvs2_coeff->hor_coefs.odd_imag,
-	    coefs->hor_coefs.odd_imag, asd->params.dvs_hor_coef_bytes))
-		return -EFAULT;
-	if (copy_from_user(asd->params.css_param.dvs2_coeff->hor_coefs.even_real,
-	    coefs->hor_coefs.even_real, asd->params.dvs_hor_coef_bytes))
-		return -EFAULT;
-	if (copy_from_user(asd->params.css_param.dvs2_coeff->hor_coefs.even_imag,
-	    coefs->hor_coefs.even_imag, asd->params.dvs_hor_coef_bytes))
-		return -EFAULT;
-
-	if (copy_from_user(asd->params.css_param.dvs2_coeff->ver_coefs.odd_real,
-	    coefs->ver_coefs.odd_real, asd->params.dvs_ver_coef_bytes))
-		return -EFAULT;
-	if (copy_from_user(asd->params.css_param.dvs2_coeff->ver_coefs.odd_imag,
-	    coefs->ver_coefs.odd_imag, asd->params.dvs_ver_coef_bytes))
-		return -EFAULT;
-	if (copy_from_user(asd->params.css_param.dvs2_coeff->ver_coefs.even_real,
-	    coefs->ver_coefs.even_real, asd->params.dvs_ver_coef_bytes))
-		return -EFAULT;
-	if (copy_from_user(asd->params.css_param.dvs2_coeff->ver_coefs.even_imag,
-	    coefs->ver_coefs.even_imag, asd->params.dvs_ver_coef_bytes))
-		return -EFAULT;
-
-	asd->params.css_param.update_flag.dvs2_coefs =
-		(struct atomisp_dvs2_coefficients *)
-		asd->params.css_param.dvs2_coeff;
-	/* FIXME! */
-/*	asd->params.dis_proj_data_valid = false; */
-	asd->params.css_update_params_needed = true;
-
-	return 0;
-}
-
-void atomisp_css_set_zoom_factor(struct atomisp_sub_device *asd,
-					unsigned int zoom)
-{
-	struct atomisp_device *isp = asd->isp;
-
-	if (zoom == asd->params.css_param.dz_config.dx &&
-		 zoom == asd->params.css_param.dz_config.dy) {
-		dev_dbg(isp->dev, "same zoom scale. skipped.\n");
-		return;
-	}
-
-	memset(&asd->params.css_param.dz_config, 0,
-		sizeof(struct ia_css_dz_config));
-	asd->params.css_param.dz_config.dx = zoom;
-	asd->params.css_param.dz_config.dy = zoom;
-
-	asd->params.css_param.update_flag.dz_config =
-		(struct atomisp_dz_config *) &asd->params.css_param.dz_config;
-	asd->params.css_update_params_needed = true;
-}
-
-void atomisp_css_set_formats_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_formats_config *formats_config)
-{
-	asd->params.config.formats_config = formats_config;
-}
-
-int atomisp_css_get_wb_config(struct atomisp_sub_device *asd,
-			struct atomisp_wb_config *config)
-{
-	struct atomisp_css_wb_config wb_config;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-	memset(&wb_config, 0, sizeof(struct atomisp_css_wb_config));
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.wb_config = &wb_config;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	memcpy(config, &wb_config, sizeof(*config));
-
-	return 0;
-}
-
-int atomisp_css_get_ob_config(struct atomisp_sub_device *asd,
-			struct atomisp_ob_config *config)
-{
-	struct atomisp_css_ob_config ob_config;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-	memset(&ob_config, 0, sizeof(struct atomisp_css_ob_config));
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.ob_config = &ob_config;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	memcpy(config, &ob_config, sizeof(*config));
-
-	return 0;
-}
-
-int atomisp_css_get_dp_config(struct atomisp_sub_device *asd,
-			struct atomisp_dp_config *config)
-{
-	struct atomisp_css_dp_config dp_config;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-	memset(&dp_config, 0, sizeof(struct atomisp_css_dp_config));
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.dp_config = &dp_config;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	memcpy(config, &dp_config, sizeof(*config));
-
-	return 0;
-}
-
-int atomisp_css_get_de_config(struct atomisp_sub_device *asd,
-			struct atomisp_de_config *config)
-{
-	struct atomisp_css_de_config de_config;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-	memset(&de_config, 0, sizeof(struct atomisp_css_de_config));
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.de_config = &de_config;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	memcpy(config, &de_config, sizeof(*config));
-
-	return 0;
-}
-
-int atomisp_css_get_nr_config(struct atomisp_sub_device *asd,
-			struct atomisp_nr_config *config)
-{
-	struct atomisp_css_nr_config nr_config;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-	memset(&nr_config, 0, sizeof(struct atomisp_css_nr_config));
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-
-	isp_config.nr_config = &nr_config;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	memcpy(config, &nr_config, sizeof(*config));
-
-	return 0;
-}
-
-int atomisp_css_get_ee_config(struct atomisp_sub_device *asd,
-			struct atomisp_ee_config *config)
-{
-	struct atomisp_css_ee_config ee_config;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			 __func__);
-		return -EINVAL;
-	}
-	memset(&ee_config, 0, sizeof(struct atomisp_css_ee_config));
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.ee_config = &ee_config;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	memcpy(config, &ee_config, sizeof(*config));
-
-	return 0;
-}
-
-int atomisp_css_get_tnr_config(struct atomisp_sub_device *asd,
-			struct atomisp_tnr_config *config)
-{
-	struct atomisp_css_tnr_config tnr_config;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-	memset(&tnr_config, 0, sizeof(struct atomisp_css_tnr_config));
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.tnr_config = &tnr_config;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	memcpy(config, &tnr_config, sizeof(*config));
-
-	return 0;
-}
-
-int atomisp_css_get_ctc_table(struct atomisp_sub_device *asd,
-			struct atomisp_ctc_table *config)
-{
-	struct atomisp_css_ctc_table *tab;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-
-	tab = vzalloc(sizeof(struct atomisp_css_ctc_table));
-	if (!tab)
-		return -ENOMEM;
-
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.ctc_table = tab;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	memcpy(config, tab, sizeof(*tab));
-	vfree(tab);
-
-	return 0;
-}
-
-int atomisp_css_get_gamma_table(struct atomisp_sub_device *asd,
-			struct atomisp_gamma_table *config)
-{
-	struct atomisp_css_gamma_table *tab;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-
-	tab = vzalloc(sizeof(struct atomisp_css_gamma_table));
-	if (!tab)
-		return -ENOMEM;
-
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.gamma_table = tab;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	memcpy(config, tab, sizeof(*tab));
-	vfree(tab);
-
-	return 0;
-}
-
-int atomisp_css_get_gc_config(struct atomisp_sub_device *asd,
-			struct atomisp_gc_config *config)
-{
-	struct atomisp_css_gc_config gc_config;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-	memset(&gc_config, 0, sizeof(struct atomisp_css_gc_config));
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.gc_config = &gc_config;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	/* Get gamma correction params from current setup */
-	memcpy(config, &gc_config, sizeof(*config));
-
-	return 0;
-}
-
-int atomisp_css_get_3a_config(struct atomisp_sub_device *asd,
-			struct atomisp_3a_config *config)
-{
-	struct atomisp_css_3a_config s3a_config;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-	memset(&s3a_config, 0, sizeof(struct atomisp_css_3a_config));
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.s3a_config = &s3a_config;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	/* Get white balance from current setup */
-	memcpy(config, &s3a_config, sizeof(*config));
-
-	return 0;
-}
-
-int atomisp_css_get_formats_config(struct atomisp_sub_device *asd,
-			struct atomisp_formats_config *config)
-{
-	struct atomisp_css_formats_config formats_config;
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-	memset(&formats_config, 0, sizeof(formats_config));
-	memset(&isp_config, 0, sizeof(isp_config));
-	isp_config.formats_config = &formats_config;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	/* Get narrow gamma from current setup */
-	memcpy(config, &formats_config, sizeof(*config));
-
-	return 0;
-}
-
-int atomisp_css_get_zoom_factor(struct atomisp_sub_device *asd,
-					unsigned int *zoom)
-{
-	struct ia_css_dz_config dz_config;  /** Digital Zoom */
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev, "%s called after streamoff, skipping.\n",
-			__func__);
-		return -EINVAL;
-	}
-	memset(&dz_config, 0, sizeof(struct ia_css_dz_config));
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.dz_config = &dz_config;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-	*zoom = dz_config.dx;
-
-	return 0;
-}
-
-
-/*
- * Function to set/get image stablization statistics
- */
-int atomisp_css_get_dis_stat(struct atomisp_sub_device *asd,
-			 struct atomisp_dis_statistics *stats)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_dis_buf *dis_buf;
-	unsigned long flags;
-
-	if (asd->params.dvs_stat->hor_prod.odd_real == NULL ||
-	    asd->params.dvs_stat->hor_prod.odd_imag == NULL ||
-	    asd->params.dvs_stat->hor_prod.even_real == NULL ||
-	    asd->params.dvs_stat->hor_prod.even_imag == NULL ||
-	    asd->params.dvs_stat->ver_prod.odd_real == NULL ||
-	    asd->params.dvs_stat->ver_prod.odd_imag == NULL ||
-	    asd->params.dvs_stat->ver_prod.even_real == NULL ||
-	    asd->params.dvs_stat->ver_prod.even_imag == NULL)
-		return -EINVAL;
-
-	/* isp needs to be streaming to get DIS statistics */
-	spin_lock_irqsave(&isp->lock, flags);
-	if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED) {
-		spin_unlock_irqrestore(&isp->lock, flags);
-		return -EINVAL;
-	}
-	spin_unlock_irqrestore(&isp->lock, flags);
-
-	if (atomisp_compare_dvs_grid(asd, &stats->dvs2_stat.grid_info) != 0)
-		/* If the grid info in the argument differs from the current
-		   grid info, we tell the caller to reset the grid size and
-		   try again. */
-		return -EAGAIN;
-
-	spin_lock_irqsave(&asd->dis_stats_lock, flags);
-	if (!asd->params.dis_proj_data_valid || list_empty(&asd->dis_stats)) {
-		spin_unlock_irqrestore(&asd->dis_stats_lock, flags);
-		dev_err(isp->dev, "dis statistics is not valid.\n");
-		return -EAGAIN;
-	}
-
-	dis_buf = list_entry(asd->dis_stats.next,
-			struct atomisp_dis_buf, list);
-	list_del_init(&dis_buf->list);
-	spin_unlock_irqrestore(&asd->dis_stats_lock, flags);
-
-	if (dis_buf->dvs_map)
-		ia_css_translate_dvs2_statistics(
-			asd->params.dvs_stat, dis_buf->dvs_map);
-	else
-		ia_css_get_dvs2_statistics(asd->params.dvs_stat,
-			dis_buf->dis_data);
-	stats->exp_id = dis_buf->dis_data->exp_id;
-
-	spin_lock_irqsave(&asd->dis_stats_lock, flags);
-	list_add_tail(&dis_buf->list, &asd->dis_stats);
-	spin_unlock_irqrestore(&asd->dis_stats_lock, flags);
-
-	if (copy_to_user(stats->dvs2_stat.ver_prod.odd_real,
-			 asd->params.dvs_stat->ver_prod.odd_real,
-			 asd->params.dvs_ver_proj_bytes))
-		return -EFAULT;
-	if (copy_to_user(stats->dvs2_stat.ver_prod.odd_imag,
-			 asd->params.dvs_stat->ver_prod.odd_imag,
-			 asd->params.dvs_ver_proj_bytes))
-		return -EFAULT;
-	if (copy_to_user(stats->dvs2_stat.ver_prod.even_real,
-			 asd->params.dvs_stat->ver_prod.even_real,
-			 asd->params.dvs_ver_proj_bytes))
-		return -EFAULT;
-	if (copy_to_user(stats->dvs2_stat.ver_prod.even_imag,
-			 asd->params.dvs_stat->ver_prod.even_imag,
-			 asd->params.dvs_ver_proj_bytes))
-		return -EFAULT;
-	if (copy_to_user(stats->dvs2_stat.hor_prod.odd_real,
-			 asd->params.dvs_stat->hor_prod.odd_real,
-			 asd->params.dvs_hor_proj_bytes))
-		return -EFAULT;
-	if (copy_to_user(stats->dvs2_stat.hor_prod.odd_imag,
-			 asd->params.dvs_stat->hor_prod.odd_imag,
-			 asd->params.dvs_hor_proj_bytes))
-		return -EFAULT;
-	if (copy_to_user(stats->dvs2_stat.hor_prod.even_real,
-			 asd->params.dvs_stat->hor_prod.even_real,
-			 asd->params.dvs_hor_proj_bytes))
-		return -EFAULT;
-	if (copy_to_user(stats->dvs2_stat.hor_prod.even_imag,
-			 asd->params.dvs_stat->hor_prod.even_imag,
-			 asd->params.dvs_hor_proj_bytes))
-		return -EFAULT;
-
-	return 0;
-}
-
-struct atomisp_css_shading_table *atomisp_css_shading_table_alloc(
-				unsigned int width, unsigned int height)
-{
-	return ia_css_shading_table_alloc(width, height);
-}
-
-void atomisp_css_set_shading_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_shading_table *table)
-{
-	asd->params.config.shading_table = table;
-}
-
-void atomisp_css_shading_table_free(struct atomisp_css_shading_table *table)
-{
-	ia_css_shading_table_free(table);
-}
-
-struct atomisp_css_morph_table *atomisp_css_morph_table_allocate(
-				unsigned int width, unsigned int height)
-{
-	return ia_css_morph_table_allocate(width, height);
-}
-
-void atomisp_css_set_morph_table(struct atomisp_sub_device *asd,
-					struct atomisp_css_morph_table *table)
-{
-	asd->params.config.morph_table = table;
-}
-
-void atomisp_css_get_morph_table(struct atomisp_sub_device *asd,
-				struct atomisp_css_morph_table *table)
-{
-	struct ia_css_isp_config isp_config;
-	struct atomisp_device *isp = asd->isp;
-
-	if (!asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) {
-		dev_err(isp->dev,
-			"%s called after streamoff, skipping.\n", __func__);
-		return;
-	}
-	memset(table, 0, sizeof(struct atomisp_css_morph_table));
-	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
-	isp_config.morph_table = table;
-	ia_css_stream_get_isp_config(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		&isp_config);
-}
-
-void atomisp_css_morph_table_free(struct atomisp_css_morph_table *table)
-{
-	ia_css_morph_table_free(table);
-}
-
-void atomisp_css_set_cont_prev_start_time(struct atomisp_device *isp,
-					unsigned int overlap)
-{
-	/* CSS 2.0 doesn't support this API. */
-	dev_dbg(isp->dev, "set cont prev start time is not supported.\n");
-	return;
-}
-
-void atomisp_css_acc_done(struct atomisp_sub_device *asd)
-{
-	complete(&asd->acc.acc_done);
-}
-
-int atomisp_css_wait_acc_finish(struct atomisp_sub_device *asd)
-{
-	int ret = 0;
-	struct atomisp_device *isp = asd->isp;
-
-	/* Unlock the isp mutex taken in IOCTL handler before sleeping! */
-	rt_mutex_unlock(&isp->mutex);
-	if (wait_for_completion_interruptible_timeout(&asd->acc.acc_done,
-					ATOMISP_ISP_TIMEOUT_DURATION) == 0) {
-		dev_err(isp->dev, "<%s: completion timeout\n", __func__);
-		atomisp_css_debug_dump_sp_sw_debug_info();
-		atomisp_css_debug_dump_debug_info(__func__);
-		ret = -EIO;
-	}
-	rt_mutex_lock(&isp->mutex);
-
-	return ret;
-}
-
-/* Set the ACC binary arguments */
-int atomisp_css_set_acc_parameters(struct atomisp_acc_fw *acc_fw)
-{
-	unsigned int mem;
-
-	for (mem = 0; mem < ATOMISP_ACC_NR_MEMORY; mem++) {
-		if (acc_fw->args[mem].length == 0)
-			continue;
-
-		ia_css_isp_param_set_css_mem_init(&acc_fw->fw->mem_initializers,
-						IA_CSS_PARAM_CLASS_PARAM, mem,
-						acc_fw->args[mem].css_ptr,
-						acc_fw->args[mem].length);
-	}
-
-	return 0;
-}
-
-/* Load acc binary extension */
-int atomisp_css_load_acc_extension(struct atomisp_sub_device *asd,
-				   struct atomisp_css_fw_info *fw,
-				   enum atomisp_css_pipe_id pipe_id,
-				   unsigned int type)
-{
-	struct atomisp_css_fw_info **hd;
-
-	fw->next = NULL;
-	hd = &(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-			.pipe_configs[pipe_id].acc_extension);
-	while (*hd)
-		hd = &(*hd)->next;
-	*hd = fw;
-
-	asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.update_pipe[pipe_id] = true;
-	return 0;
-}
-
-/* Unload acc binary extension */
-void atomisp_css_unload_acc_extension(struct atomisp_sub_device *asd,
-					struct atomisp_css_fw_info *fw,
-					enum atomisp_css_pipe_id pipe_id)
-{
-	struct atomisp_css_fw_info **hd;
-
-	hd = &(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-			.pipe_configs[pipe_id].acc_extension);
-	while (*hd && *hd != fw)
-		hd = &(*hd)->next;
-	if (!*hd) {
-		dev_err(asd->isp->dev, "did not find acc fw for removal\n");
-		return;
-	}
-	*hd = fw->next;
-	fw->next = NULL;
-
-	asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.update_pipe[pipe_id] = true;
-}
-
-int atomisp_css_create_acc_pipe(struct atomisp_sub_device *asd)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct ia_css_pipe_config *pipe_config;
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-
-	if (stream_env->acc_stream) {
-		if (stream_env->acc_stream_state == CSS_STREAM_STARTED) {
-			if (ia_css_stream_stop(stream_env->acc_stream)
-				!= IA_CSS_SUCCESS) {
-				dev_err(isp->dev, "stop acc_stream failed.\n");
-				return -EBUSY;
-			}
-		}
-
-		if (ia_css_stream_destroy(stream_env->acc_stream)
-			!= IA_CSS_SUCCESS) {
-			dev_err(isp->dev, "destroy acc_stream failed.\n");
-			return -EBUSY;
-		}
-		stream_env->acc_stream = NULL;
-	}
-
-	pipe_config = &stream_env->pipe_configs[CSS_PIPE_ID_ACC];
-	ia_css_pipe_config_defaults(pipe_config);
-	asd->acc.acc_stages = kzalloc(MAX_ACC_STAGES *
-				sizeof(void *), GFP_KERNEL);
-	if (!asd->acc.acc_stages)
-		return -ENOMEM;
-	pipe_config->acc_stages = asd->acc.acc_stages;
-	pipe_config->mode = IA_CSS_PIPE_MODE_ACC;
-	pipe_config->num_acc_stages = 0;
-
-	/*
-	 * We delay the ACC pipeline creation to atomisp_css_start_acc_pipe,
-	 * because pipe configuration will soon be changed by
-	 * atomisp_css_load_acc_binary()
-	 */
-	return 0;
-}
-
-int atomisp_css_start_acc_pipe(struct atomisp_sub_device *asd)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	struct ia_css_pipe_config *pipe_config =
-			&stream_env->pipe_configs[IA_CSS_PIPE_ID_ACC];
-
-	if (ia_css_pipe_create(pipe_config,
-		&stream_env->pipes[IA_CSS_PIPE_ID_ACC]) != IA_CSS_SUCCESS) {
-		dev_err(isp->dev, "%s: ia_css_pipe_create failed\n",
-				__func__);
-		return -EBADE;
-	}
-
-	memset(&stream_env->acc_stream_config, 0,
-		sizeof(struct ia_css_stream_config));
-	if (ia_css_stream_create(&stream_env->acc_stream_config, 1,
-				&stream_env->pipes[IA_CSS_PIPE_ID_ACC],
-				&stream_env->acc_stream) != IA_CSS_SUCCESS) {
-		dev_err(isp->dev, "%s: create acc_stream error.\n", __func__);
-		return -EINVAL;
-	}
-	stream_env->acc_stream_state = CSS_STREAM_CREATED;
-
-	init_completion(&asd->acc.acc_done);
-	asd->acc.pipeline = stream_env->pipes[IA_CSS_PIPE_ID_ACC];
-
-	atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, false);
-
-	if (ia_css_start_sp() != IA_CSS_SUCCESS) {
-		dev_err(isp->dev, "start sp error.\n");
-		return -EIO;
-	}
-
-	if (ia_css_stream_start(stream_env->acc_stream)
-		!= IA_CSS_SUCCESS) {
-		dev_err(isp->dev, "acc_stream start error.\n");
-		return -EIO;
-	}
-
-	stream_env->acc_stream_state = CSS_STREAM_STARTED;
-	return 0;
-}
-
-int atomisp_css_stop_acc_pipe(struct atomisp_sub_device *asd)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	if (stream_env->acc_stream_state == CSS_STREAM_STARTED) {
-		ia_css_stream_stop(stream_env->acc_stream);
-		stream_env->acc_stream_state = CSS_STREAM_STOPPED;
-	}
-	return 0;
-}
-
-void atomisp_css_destroy_acc_pipe(struct atomisp_sub_device *asd)
-{
-	struct atomisp_stream_env *stream_env =
-		&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
-	if (stream_env->acc_stream) {
-		if (ia_css_stream_destroy(stream_env->acc_stream)
-		    != IA_CSS_SUCCESS)
-			dev_warn(asd->isp->dev,
-				"destroy acc_stream failed.\n");
-		stream_env->acc_stream = NULL;
-	}
-
-	if (stream_env->pipes[IA_CSS_PIPE_ID_ACC]) {
-		if (ia_css_pipe_destroy(stream_env->pipes[IA_CSS_PIPE_ID_ACC])
-			!= IA_CSS_SUCCESS)
-			dev_warn(asd->isp->dev,
-				"destroy ACC pipe failed.\n");
-		stream_env->pipes[IA_CSS_PIPE_ID_ACC] = NULL;
-		stream_env->update_pipe[IA_CSS_PIPE_ID_ACC] = false;
-		ia_css_pipe_config_defaults(
-			&stream_env->pipe_configs[IA_CSS_PIPE_ID_ACC]);
-		ia_css_pipe_extra_config_defaults(
-			&stream_env->pipe_extra_configs[IA_CSS_PIPE_ID_ACC]);
-	}
-	asd->acc.pipeline = NULL;
-
-	/* css 2.0 API limitation: ia_css_stop_sp() could be only called after
-	 * destroy all pipes
-	 */
-	ia_css_stop_sp();
-
-	kfree(asd->acc.acc_stages);
-	asd->acc.acc_stages = NULL;
-
-	atomisp_freq_scaling(asd->isp, ATOMISP_DFS_MODE_LOW, false);
-}
-
-int atomisp_css_load_acc_binary(struct atomisp_sub_device *asd,
-					struct atomisp_css_fw_info *fw,
-					unsigned int index)
-{
-	struct ia_css_pipe_config *pipe_config =
-			&asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-			.pipe_configs[IA_CSS_PIPE_ID_ACC];
-
-	if (index >= MAX_ACC_STAGES) {
-		dev_dbg(asd->isp->dev, "%s: index(%d) out of range\n",
-				__func__, index);
-		return -ENOMEM;
-	}
-
-	pipe_config->acc_stages[index] = fw;
-	pipe_config->num_acc_stages = index + 1;
-	pipe_config->acc_num_execs = 1;
-
-	return 0;
-}
-
-static struct atomisp_sub_device *__get_atomisp_subdev(
-					struct ia_css_pipe *css_pipe,
-					struct atomisp_device *isp,
-					enum atomisp_input_stream_id *stream_id)
-{
-	int i, j, k;
-	struct atomisp_sub_device *asd;
-	struct atomisp_stream_env *stream_env;
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-		if (asd->streaming == ATOMISP_DEVICE_STREAMING_DISABLED &&
-		    !asd->acc.pipeline)
-			continue;
-		for (j = 0; j < ATOMISP_INPUT_STREAM_NUM; j++) {
-			stream_env = &asd->stream_env[j];
-			for (k = 0; k < IA_CSS_PIPE_ID_NUM; k++) {
-				if (stream_env->pipes[k] &&
-					stream_env->pipes[k] == css_pipe) {
-						*stream_id = j;
-						return asd;
-					}
-				}
-		}
-	}
-
-	return NULL;
-}
-
-int atomisp_css_isr_thread(struct atomisp_device *isp,
-			   bool *frame_done_found,
-			   bool *css_pipe_done)
-{
-	enum atomisp_input_stream_id stream_id = 0;
-	struct atomisp_css_event current_event;
-	struct atomisp_sub_device *asd;
-#ifndef ISP2401
-	bool reset_wdt_timer[MAX_STREAM_NUM] = {false};
-#endif
-	int i;
-
-	while (!atomisp_css_dequeue_event(&current_event)) {
-		if (current_event.event.type ==
-			IA_CSS_EVENT_TYPE_FW_ASSERT) {
-			/*
-			 * Received FW assertion signal,
-			 * trigger WDT to recover
-			 */
-			dev_err(isp->dev, "%s: ISP reports FW_ASSERT event! fw_assert_module_id %d fw_assert_line_no %d\n",
-				__func__,
-				current_event.event.fw_assert_module_id,
-				current_event.event.fw_assert_line_no);
-			for (i = 0; i < isp->num_of_streams; i++)
-				atomisp_wdt_stop(&isp->asd[i], 0);
-#ifndef ISP2401
-			atomisp_wdt(&isp->asd[0].wdt);
-#else
-			queue_work(isp->wdt_work_queue, &isp->wdt_work);
-#endif
-			return -EINVAL;
-		} else if (current_event.event.type == IA_CSS_EVENT_TYPE_FW_WARNING) {
-			dev_warn(isp->dev, "%s: ISP reports warning, code is %d, exp_id %d\n",
-				__func__, current_event.event.fw_warning,
-				current_event.event.exp_id);
-			continue;
-		}
-
-		asd = __get_atomisp_subdev(current_event.event.pipe,
-					isp, &stream_id);
-		if (!asd) {
-			if (current_event.event.type == CSS_EVENT_TIMER)
-				dev_dbg(isp->dev,
-					"event: Timer event.");
-			else
-				dev_warn(isp->dev, "%s:no subdev.event:%d",
-						__func__,
-						current_event.event.type);
-			continue;
-		}
-
-		atomisp_css_temp_pipe_to_pipe_id(asd, &current_event);
-		switch (current_event.event.type) {
-		case CSS_EVENT_OUTPUT_FRAME_DONE:
-			frame_done_found[asd->index] = true;
-			atomisp_buf_done(asd, 0, CSS_BUFFER_TYPE_OUTPUT_FRAME,
-					 current_event.pipe, true, stream_id);
-#ifndef ISP2401
-			reset_wdt_timer[asd->index] = true; /* ISP running */
-#endif
-			break;
-		case CSS_EVENT_SEC_OUTPUT_FRAME_DONE:
-			frame_done_found[asd->index] = true;
-			atomisp_buf_done(asd, 0, CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME,
-					 current_event.pipe, true, stream_id);
-#ifndef ISP2401
-			reset_wdt_timer[asd->index] = true; /* ISP running */
-#endif
-			break;
-		case CSS_EVENT_3A_STATISTICS_DONE:
-			atomisp_buf_done(asd, 0,
-					 CSS_BUFFER_TYPE_3A_STATISTICS,
-					 current_event.pipe,
-					 false, stream_id);
-			break;
-		case CSS_EVENT_METADATA_DONE:
-			atomisp_buf_done(asd, 0,
-					 CSS_BUFFER_TYPE_METADATA,
-					 current_event.pipe,
-					 false, stream_id);
-			break;
-		case CSS_EVENT_VF_OUTPUT_FRAME_DONE:
-			atomisp_buf_done(asd, 0,
-					 CSS_BUFFER_TYPE_VF_OUTPUT_FRAME,
-					 current_event.pipe, true, stream_id);
-#ifndef ISP2401
-			reset_wdt_timer[asd->index] = true; /* ISP running */
-#endif
-			break;
-		case CSS_EVENT_SEC_VF_OUTPUT_FRAME_DONE:
-			atomisp_buf_done(asd, 0,
-					 CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME,
-					 current_event.pipe, true, stream_id);
-#ifndef ISP2401
-			reset_wdt_timer[asd->index] = true; /* ISP running */
-#endif
-			break;
-		case CSS_EVENT_DIS_STATISTICS_DONE:
-			atomisp_buf_done(asd, 0,
-					 CSS_BUFFER_TYPE_DIS_STATISTICS,
-					 current_event.pipe,
-					 false, stream_id);
-			break;
-		case CSS_EVENT_PIPELINE_DONE:
-			css_pipe_done[asd->index] = true;
-			break;
-		case CSS_EVENT_ACC_STAGE_COMPLETE:
-			atomisp_acc_done(asd, current_event.event.fw_handle);
-			break;
-		default:
-			dev_dbg(isp->dev, "unhandled css stored event: 0x%x\n",
-					current_event.event.type);
-			break;
-		}
-	}
-#ifndef ISP2401
-	/* If there are no buffers queued then
-	 * delete wdt timer. */
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-		if (!asd)
-			continue;
-		if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
-			continue;
-		if (!atomisp_buffers_queued(asd))
-			atomisp_wdt_stop(asd, false);
-		else if (reset_wdt_timer[i])
-		/* SOF irq should not reset wdt timer. */
-			atomisp_wdt_refresh(asd,
-					ATOMISP_WDT_KEEP_CURRENT_DELAY);
-	}
-#endif
-
-	return 0;
-}
-
-bool atomisp_css_valid_sof(struct atomisp_device *isp)
-{
-	unsigned int i, j;
-
-	/* Loop for each css stream */
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-		/* Loop for each css vc stream */
-		for (j = 0; j < ATOMISP_INPUT_STREAM_NUM; j++) {
-			if (asd->stream_env[j].stream &&
-				asd->stream_env[j].stream_config.mode ==
-				IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
-				return false;
-		}
-	}
-
-	return true;
-}
-
-int atomisp_css_debug_dump_isp_binary(void)
-{
-	ia_css_debug_dump_isp_binary();
-	return 0;
-}
-
-int atomisp_css_dump_sp_raw_copy_linecount(bool reduced)
-{
-	sh_css_dump_sp_raw_copy_linecount(reduced);
-	return 0;
-}
-
-int atomisp_css_dump_blob_infor(void)
-{
-	struct ia_css_blob_descr *bd = sh_css_blob_info;
-	unsigned int i, nm = sh_css_num_binaries;
-
-	if (nm == 0)
-		return -EPERM;
-	if (bd == NULL)
-		return -EPERM;
-
-	for (i = 1; i < sh_css_num_binaries; i++)
-		dev_dbg(atomisp_dev, "Num%d binary id is %d, name is %s\n", i,
-			bd[i-1].header.info.isp.sp.id, bd[i-1].name);
-
-	return 0;
-}
-
-void atomisp_css_set_isp_config_id(struct atomisp_sub_device *asd,
-			uint32_t isp_config_id)
-{
-	asd->params.config.isp_config_id = isp_config_id;
-}
-
-void atomisp_css_set_isp_config_applied_frame(struct atomisp_sub_device *asd,
-			struct atomisp_css_frame *output_frame)
-{
-	asd->params.config.output_frame = output_frame;
-}
-
-int atomisp_get_css_dbgfunc(void)
-{
-	return dbg_func;
-}
-
-int atomisp_set_css_dbgfunc(struct atomisp_device *isp, int opt)
-{
-	int ret;
-
-	ret = __set_css_print_env(isp, opt);
-	if (ret == 0)
-		dbg_func = opt;
-
-	return ret;
-}
-void atomisp_en_dz_capt_pipe(struct atomisp_sub_device *asd, bool enable)
-{
-	ia_css_en_dz_capt_pipe(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		enable);
-}
-
-struct atomisp_css_dvs_grid_info *atomisp_css_get_dvs_grid_info(
-	struct atomisp_css_grid_info *grid_info)
-{
-	if (!grid_info)
-		return NULL;
-
-#ifdef IA_CSS_DVS_STAT_GRID_INFO_SUPPORTED
-	return &grid_info->dvs_grid.dvs_grid_info;
-#else
-	return &grid_info->dvs_grid;
-#endif
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.h
deleted file mode 100644
index a06c5b6e8027..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.h
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Support for Clovertrail PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __ATOMISP_COMPAT_CSS20_H__
-#define __ATOMISP_COMPAT_CSS20_H__
-
-#include <media/v4l2-mediabus.h>
-
-#include "ia_css.h"
-#include "ia_css_types.h"
-#include "ia_css_acc_types.h"
-#include "sh_css_legacy.h"
-
-#define ATOMISP_CSS2_PIPE_MAX	2
-#define ATOMISP_CSS2_NUM_OFFLINE_INIT_CONTINUOUS_FRAMES     3
-#define ATOMISP_CSS2_NUM_OFFLINE_INIT_CONTINUOUS_FRAMES_LOCK_EN     4
-#define ATOMISP_CSS2_NUM_DVS_FRAME_DELAY     2
-
-#define atomisp_css_pipe_id ia_css_pipe_id
-#define atomisp_css_pipeline	ia_css_pipe
-#define atomisp_css_buffer_type ia_css_buffer_type
-#define atomisp_css_dis_data ia_css_isp_dvs_statistics
-#define atomisp_css_irq_info  ia_css_irq_info
-#define atomisp_css_isp_config ia_css_isp_config
-#define atomisp_css_bayer_order ia_css_bayer_order
-#define atomisp_css_capture_mode ia_css_capture_mode
-#define atomisp_css_input_mode ia_css_input_mode
-#define atomisp_css_frame ia_css_frame
-#define atomisp_css_frame_format ia_css_frame_format
-#define atomisp_css_frame_info ia_css_frame_info
-#define atomisp_css_dp_config	ia_css_dp_config
-#define atomisp_css_wb_config	ia_css_wb_config
-#define atomisp_css_cc_config	ia_css_cc_config
-#define atomisp_css_nr_config	ia_css_nr_config
-#define atomisp_css_ee_config	ia_css_ee_config
-#define atomisp_css_ob_config	ia_css_ob_config
-#define atomisp_css_de_config	ia_css_de_config
-#define atomisp_css_dz_config	ia_css_dz_config
-#define atomisp_css_ce_config	ia_css_ce_config
-#define atomisp_css_gc_config	ia_css_gc_config
-#define atomisp_css_tnr_config	ia_css_tnr_config
-#define atomisp_css_cnr_config	ia_css_cnr_config
-#define atomisp_css_ctc_config	ia_css_ctc_config
-#define atomisp_css_3a_config	ia_css_3a_config
-#define atomisp_css_ecd_config	ia_css_ecd_config
-#define atomisp_css_ynr_config	ia_css_ynr_config
-#define atomisp_css_fc_config	ia_css_fc_config
-#define atomisp_css_aa_config	ia_css_aa_config
-#define atomisp_css_baa_config	ia_css_aa_config
-#define atomisp_css_anr_config	ia_css_anr_config
-#define atomisp_css_xnr_config	ia_css_xnr_config
-#define atomisp_css_macc_config	ia_css_macc_config
-#define atomisp_css_gamma_table	ia_css_gamma_table
-#define atomisp_css_ctc_table	ia_css_ctc_table
-#define atomisp_css_macc_table	ia_css_macc_table
-#define atomisp_css_xnr_table	ia_css_xnr_table
-#define atomisp_css_rgb_gamma_table	ia_css_rgb_gamma_table
-#define atomisp_css_anr_thres	ia_css_anr_thres
-#define atomisp_css_dvs_6axis	ia_css_dvs_6axis_config
-#define atomisp_css_grid_info	ia_css_grid_info
-#define atomisp_css_3a_grid_info	ia_css_3a_grid_info
-#define atomisp_css_dvs_grid_info	ia_css_dvs_grid_info
-#define atomisp_css_shading_table	ia_css_shading_table
-#define atomisp_css_morph_table	ia_css_morph_table
-#define atomisp_css_dvs_6axis_config	ia_css_dvs_6axis_config
-#define atomisp_css_fw_info	ia_css_fw_info
-#define atomisp_css_formats_config	ia_css_formats_config
-
-#define CSS_PIPE_ID_PREVIEW	IA_CSS_PIPE_ID_PREVIEW
-#define CSS_PIPE_ID_COPY	IA_CSS_PIPE_ID_COPY
-#define CSS_PIPE_ID_VIDEO	IA_CSS_PIPE_ID_VIDEO
-#define CSS_PIPE_ID_CAPTURE	IA_CSS_PIPE_ID_CAPTURE
-#define CSS_PIPE_ID_ACC		IA_CSS_PIPE_ID_ACC
-#define CSS_PIPE_ID_YUVPP	IA_CSS_PIPE_ID_YUVPP
-#define CSS_PIPE_ID_NUM		IA_CSS_PIPE_ID_NUM
-
-#define CSS_INPUT_MODE_SENSOR	IA_CSS_INPUT_MODE_BUFFERED_SENSOR
-#define CSS_INPUT_MODE_FIFO	IA_CSS_INPUT_MODE_FIFO
-#define CSS_INPUT_MODE_TPG	IA_CSS_INPUT_MODE_TPG
-#define CSS_INPUT_MODE_PRBS	IA_CSS_INPUT_MODE_PRBS
-#define CSS_INPUT_MODE_MEMORY	IA_CSS_INPUT_MODE_MEMORY
-
-#define CSS_IRQ_INFO_CSS_RECEIVER_ERROR	IA_CSS_IRQ_INFO_CSS_RECEIVER_ERROR
-#define CSS_IRQ_INFO_EVENTS_READY	IA_CSS_IRQ_INFO_EVENTS_READY
-#define CSS_IRQ_INFO_INPUT_SYSTEM_ERROR \
-	IA_CSS_IRQ_INFO_INPUT_SYSTEM_ERROR
-#define CSS_IRQ_INFO_IF_ERROR	IA_CSS_IRQ_INFO_IF_ERROR
-
-#define CSS_BUFFER_TYPE_NUM	IA_CSS_BUFFER_TYPE_NUM
-
-#define CSS_FRAME_FLASH_STATE_NONE	IA_CSS_FRAME_FLASH_STATE_NONE
-#define CSS_FRAME_FLASH_STATE_PARTIAL	IA_CSS_FRAME_FLASH_STATE_PARTIAL
-#define CSS_FRAME_FLASH_STATE_FULL	IA_CSS_FRAME_FLASH_STATE_FULL
-
-#define CSS_BAYER_ORDER_GRBG	IA_CSS_BAYER_ORDER_GRBG
-#define CSS_BAYER_ORDER_RGGB	IA_CSS_BAYER_ORDER_RGGB
-#define CSS_BAYER_ORDER_BGGR	IA_CSS_BAYER_ORDER_BGGR
-#define CSS_BAYER_ORDER_GBRG	IA_CSS_BAYER_ORDER_GBRG
-
-/*
- * Hide IA_ naming difference in otherwise common CSS macros.
- */
-#define CSS_ID(val)	(IA_ ## val)
-#define CSS_EVENT(val)	(IA_CSS_EVENT_TYPE_ ## val)
-#define CSS_FORMAT(val)	(ATOMISP_INPUT_FORMAT_ ## val)
-
-#define CSS_EVENT_PORT_EOF	CSS_EVENT(PORT_EOF)
-#define CSS_EVENT_FRAME_TAGGED	CSS_EVENT(FRAME_TAGGED)
-
-#define CSS_MIPI_FRAME_BUFFER_SIZE_1	0x60000
-#define CSS_MIPI_FRAME_BUFFER_SIZE_2	0x80000
-
-struct atomisp_device;
-struct atomisp_sub_device;
-
-#define MAX_STREAMS_PER_CHANNEL	2
-
-/*
- * These are used to indicate the css stream state, corresponding
- * stream handling can be done via judging the different state.
- */
-enum atomisp_css_stream_state {
-	CSS_STREAM_UNINIT,
-	CSS_STREAM_CREATED,
-	CSS_STREAM_STARTED,
-	CSS_STREAM_STOPPED,
-};
-
-/*
- *  Sensor of external ISP can send multiple steams with different mipi data
- * type in the same virtual channel. This information needs to come from the
- * sensor or external ISP
- */
-struct atomisp_css_isys_config_info {
-	unsigned int input_format;
-	unsigned int width;
-	unsigned int height;
-};
-
-struct atomisp_stream_env {
-	struct ia_css_stream *stream;
-	struct ia_css_stream_config stream_config;
-	struct ia_css_stream_info stream_info;
-	struct ia_css_pipe *pipes[IA_CSS_PIPE_ID_NUM];
-	struct ia_css_pipe *multi_pipes[IA_CSS_PIPE_ID_NUM];
-	struct ia_css_pipe_config pipe_configs[IA_CSS_PIPE_ID_NUM];
-	struct ia_css_pipe_extra_config pipe_extra_configs[IA_CSS_PIPE_ID_NUM];
-	bool update_pipe[IA_CSS_PIPE_ID_NUM];
-	enum atomisp_css_stream_state stream_state;
-	struct ia_css_stream *acc_stream;
-	enum atomisp_css_stream_state acc_stream_state;
-	struct ia_css_stream_config acc_stream_config;
-	unsigned int ch_id; /* virtual channel ID */
-	unsigned int isys_configs;
-	struct atomisp_css_isys_config_info isys_info[MAX_STREAMS_PER_CHANNEL];
-};
-
-struct atomisp_css_env {
-	struct ia_css_env isp_css_env;
-	struct ia_css_fw isp_css_fw;
-};
-
-struct atomisp_s3a_buf {
-	struct ia_css_isp_3a_statistics *s3a_data;
-	struct ia_css_isp_3a_statistics_map *s3a_map;
-	struct list_head list;
-};
-
-struct atomisp_dis_buf {
-	struct atomisp_css_dis_data *dis_data;
-	struct ia_css_isp_dvs_statistics_map *dvs_map;
-	struct list_head list;
-};
-
-struct atomisp_css_buffer {
-	struct ia_css_buffer css_buffer;
-};
-
-struct atomisp_css_event {
-	enum atomisp_css_pipe_id pipe;
-	struct ia_css_event event;
-};
-
-void atomisp_css_set_macc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_macc_config *macc_config);
-
-void atomisp_css_set_ecd_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ecd_config *ecd_config);
-
-void atomisp_css_set_ynr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ynr_config *ynr_config);
-
-void atomisp_css_set_fc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_fc_config *fc_config);
-
-void atomisp_css_set_aa_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_aa_config *aa_config);
-
-void atomisp_css_set_baa_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_baa_config *baa_config);
-
-void atomisp_css_set_anr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_anr_config *anr_config);
-
-void atomisp_css_set_xnr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_xnr_config *xnr_config);
-
-void atomisp_css_set_cnr_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_cnr_config *cnr_config);
-
-void atomisp_css_set_ctc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_ctc_config *ctc_config);
-
-void atomisp_css_set_yuv2rgb_cc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_cc_config *yuv2rgb_cc_config);
-
-void atomisp_css_set_rgb2yuv_cc_config(struct atomisp_sub_device *asd,
-			struct atomisp_css_cc_config *rgb2yuv_cc_config);
-
-void atomisp_css_set_xnr_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_xnr_table *xnr_table);
-
-void atomisp_css_set_r_gamma_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_rgb_gamma_table *r_gamma_table);
-
-void atomisp_css_set_g_gamma_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_rgb_gamma_table *g_gamma_table);
-
-void atomisp_css_set_b_gamma_table(struct atomisp_sub_device *asd,
-			struct atomisp_css_rgb_gamma_table *b_gamma_table);
-
-void atomisp_css_set_anr_thres(struct atomisp_sub_device *asd,
-			struct atomisp_css_anr_thres *anr_thres);
-
-int atomisp_css_check_firmware_version(struct atomisp_device *isp);
-
-int atomisp_css_load_firmware(struct atomisp_device *isp);
-
-void atomisp_css_unload_firmware(struct atomisp_device *isp);
-
-void atomisp_css_set_dvs_6axis(struct atomisp_sub_device *asd,
-			struct atomisp_css_dvs_6axis *dvs_6axis);
-
-unsigned int atomisp_css_debug_get_dtrace_level(void);
-
-int atomisp_css_debug_dump_isp_binary(void);
-
-int atomisp_css_dump_sp_raw_copy_linecount(bool reduced);
-
-int atomisp_css_dump_blob_infor(void);
-
-void atomisp_css_set_isp_config_id(struct atomisp_sub_device *asd,
-			uint32_t isp_config_id);
-
-void atomisp_css_set_isp_config_applied_frame(struct atomisp_sub_device *asd,
-			struct atomisp_css_frame *output_frame);
-
-int atomisp_get_css_dbgfunc(void);
-
-int atomisp_set_css_dbgfunc(struct atomisp_device *isp, int opt);
-struct atomisp_css_dvs_grid_info *atomisp_css_get_dvs_grid_info(
-	struct atomisp_css_grid_info *grid_info);
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_ioctl32.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_ioctl32.c
deleted file mode 100644
index 44c21813a06e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_ioctl32.c
+++ /dev/null
@@ -1,1259 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifdef CONFIG_COMPAT
-#include <linux/compat.h>
-
-#include <linux/videodev2.h>
-
-#include "atomisp_internal.h"
-#include "atomisp_compat.h"
-#include "atomisp_compat_ioctl32.h"
-
-static int get_atomisp_histogram32(struct atomisp_histogram *kp,
-					struct atomisp_histogram32 __user *up)
-{
-	compat_uptr_t tmp;
-
-	if (!access_ok(VERIFY_READ, up, sizeof(struct atomisp_histogram32)) ||
-		get_user(kp->num_elements, &up->num_elements) ||
-		get_user(tmp, &up->data))
-			return -EFAULT;
-
-	kp->data = compat_ptr(tmp);
-	return 0;
-}
-
-static int put_atomisp_histogram32(struct atomisp_histogram *kp,
-					struct atomisp_histogram32 __user *up)
-{
-	compat_uptr_t tmp = (compat_uptr_t)((uintptr_t)kp->data);
-
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct atomisp_histogram32)) ||
-		put_user(kp->num_elements, &up->num_elements) ||
-		put_user(tmp, &up->data))
-			return -EFAULT;
-
-	return 0;
-}
-
-static inline int get_v4l2_pix_format(struct v4l2_pix_format *kp,
-					struct v4l2_pix_format __user *up)
-{
-	if (copy_from_user(kp, up, sizeof(struct v4l2_pix_format)))
-		return -EFAULT;
-	return 0;
-}
-
-static inline int put_v4l2_pix_format(struct v4l2_pix_format *kp,
-					struct v4l2_pix_format __user *up)
-{
-	if (copy_to_user(up, kp, sizeof(struct v4l2_pix_format)))
-		return -EFAULT;
-	return 0;
-}
-
-static int get_v4l2_framebuffer32(struct v4l2_framebuffer *kp,
-					struct v4l2_framebuffer32 __user *up)
-{
-	compat_uptr_t tmp;
-
-	if (!access_ok(VERIFY_READ, up, sizeof(struct v4l2_framebuffer32)) ||
-		get_user(tmp, &up->base) ||
-		get_user(kp->capability, &up->capability) ||
-		get_user(kp->flags, &up->flags))
-			return -EFAULT;
-
-	kp->base = compat_ptr(tmp);
-	get_v4l2_pix_format((struct v4l2_pix_format *)&kp->fmt, &up->fmt);
-	return 0;
-}
-
-static int get_atomisp_dis_statistics32(struct atomisp_dis_statistics *kp,
-				struct atomisp_dis_statistics32 __user *up)
-{
-	compat_uptr_t hor_prod_odd_real;
-	compat_uptr_t hor_prod_odd_imag;
-	compat_uptr_t hor_prod_even_real;
-	compat_uptr_t hor_prod_even_imag;
-	compat_uptr_t ver_prod_odd_real;
-	compat_uptr_t ver_prod_odd_imag;
-	compat_uptr_t ver_prod_even_real;
-	compat_uptr_t ver_prod_even_imag;
-
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_dis_statistics32)) ||
-		copy_from_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) ||
-		get_user(hor_prod_odd_real,
-				&up->dvs2_stat.hor_prod.odd_real) ||
-		get_user(hor_prod_odd_imag,
-				&up->dvs2_stat.hor_prod.odd_imag) ||
-		get_user(hor_prod_even_real,
-				&up->dvs2_stat.hor_prod.even_real) ||
-		get_user(hor_prod_even_imag,
-				&up->dvs2_stat.hor_prod.even_imag) ||
-		get_user(ver_prod_odd_real,
-				&up->dvs2_stat.ver_prod.odd_real) ||
-		get_user(ver_prod_odd_imag,
-				&up->dvs2_stat.ver_prod.odd_imag) ||
-		get_user(ver_prod_even_real,
-				&up->dvs2_stat.ver_prod.even_real) ||
-		get_user(ver_prod_even_imag,
-				&up->dvs2_stat.ver_prod.even_imag) ||
-		get_user(kp->exp_id, &up->exp_id))
-			return -EFAULT;
-
-	kp->dvs2_stat.hor_prod.odd_real = compat_ptr(hor_prod_odd_real);
-	kp->dvs2_stat.hor_prod.odd_imag = compat_ptr(hor_prod_odd_imag);
-	kp->dvs2_stat.hor_prod.even_real = compat_ptr(hor_prod_even_real);
-	kp->dvs2_stat.hor_prod.even_imag = compat_ptr(hor_prod_even_imag);
-	kp->dvs2_stat.ver_prod.odd_real = compat_ptr(ver_prod_odd_real);
-	kp->dvs2_stat.ver_prod.odd_imag = compat_ptr(ver_prod_odd_imag);
-	kp->dvs2_stat.ver_prod.even_real = compat_ptr(ver_prod_even_real);
-	kp->dvs2_stat.ver_prod.even_imag = compat_ptr(ver_prod_even_imag);
-	return 0;
-}
-
-static int put_atomisp_dis_statistics32(struct atomisp_dis_statistics *kp,
-				struct atomisp_dis_statistics32 __user *up)
-{
-	compat_uptr_t hor_prod_odd_real =
-		(compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.odd_real);
-	compat_uptr_t hor_prod_odd_imag =
-		(compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.odd_imag);
-	compat_uptr_t hor_prod_even_real =
-		(compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.even_real);
-	compat_uptr_t hor_prod_even_imag =
-		(compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.even_imag);
-	compat_uptr_t ver_prod_odd_real =
-		(compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.odd_real);
-	compat_uptr_t ver_prod_odd_imag =
-		(compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.odd_imag);
-	compat_uptr_t ver_prod_even_real =
-		(compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.even_real);
-	compat_uptr_t ver_prod_even_imag =
-		(compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.even_imag);
-
-	if (!access_ok(VERIFY_WRITE, up,
-			sizeof(struct atomisp_dis_statistics32)) ||
-		copy_to_user(up, kp, sizeof(struct atomisp_dvs_grid_info)) ||
-		put_user(hor_prod_odd_real,
-				&up->dvs2_stat.hor_prod.odd_real) ||
-		put_user(hor_prod_odd_imag,
-				&up->dvs2_stat.hor_prod.odd_imag) ||
-		put_user(hor_prod_even_real,
-				&up->dvs2_stat.hor_prod.even_real) ||
-		put_user(hor_prod_even_imag,
-				&up->dvs2_stat.hor_prod.even_imag) ||
-		put_user(ver_prod_odd_real,
-				&up->dvs2_stat.ver_prod.odd_real) ||
-		put_user(ver_prod_odd_imag,
-				&up->dvs2_stat.ver_prod.odd_imag) ||
-		put_user(ver_prod_even_real,
-				&up->dvs2_stat.ver_prod.even_real) ||
-		put_user(ver_prod_even_imag,
-				&up->dvs2_stat.ver_prod.even_imag) ||
-		put_user(kp->exp_id, &up->exp_id))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int get_atomisp_dis_coefficients32(struct atomisp_dis_coefficients *kp,
-				struct atomisp_dis_coefficients32 __user *up)
-{
-	compat_uptr_t hor_coefs_odd_real;
-	compat_uptr_t hor_coefs_odd_imag;
-	compat_uptr_t hor_coefs_even_real;
-	compat_uptr_t hor_coefs_even_imag;
-	compat_uptr_t ver_coefs_odd_real;
-	compat_uptr_t ver_coefs_odd_imag;
-	compat_uptr_t ver_coefs_even_real;
-	compat_uptr_t ver_coefs_even_imag;
-
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_dis_coefficients32)) ||
-		copy_from_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) ||
-		get_user(hor_coefs_odd_real, &up->hor_coefs.odd_real) ||
-		get_user(hor_coefs_odd_imag, &up->hor_coefs.odd_imag) ||
-		get_user(hor_coefs_even_real, &up->hor_coefs.even_real) ||
-		get_user(hor_coefs_even_imag, &up->hor_coefs.even_imag) ||
-		get_user(ver_coefs_odd_real, &up->ver_coefs.odd_real) ||
-		get_user(ver_coefs_odd_imag, &up->ver_coefs.odd_imag) ||
-		get_user(ver_coefs_even_real, &up->ver_coefs.even_real) ||
-		get_user(ver_coefs_even_imag, &up->ver_coefs.even_imag))
-			return -EFAULT;
-
-	kp->hor_coefs.odd_real = compat_ptr(hor_coefs_odd_real);
-	kp->hor_coefs.odd_imag = compat_ptr(hor_coefs_odd_imag);
-	kp->hor_coefs.even_real = compat_ptr(hor_coefs_even_real);
-	kp->hor_coefs.even_imag = compat_ptr(hor_coefs_even_imag);
-	kp->ver_coefs.odd_real = compat_ptr(ver_coefs_odd_real);
-	kp->ver_coefs.odd_imag = compat_ptr(ver_coefs_odd_imag);
-	kp->ver_coefs.even_real = compat_ptr(ver_coefs_even_real);
-	kp->ver_coefs.even_imag = compat_ptr(ver_coefs_even_imag);
-	return 0;
-}
-
-static int get_atomisp_dvs_6axis_config32(struct atomisp_dvs_6axis_config *kp,
-				struct atomisp_dvs_6axis_config32 __user *up)
-{	compat_uptr_t xcoords_y;
-	compat_uptr_t ycoords_y;
-	compat_uptr_t xcoords_uv;
-	compat_uptr_t ycoords_uv;
-
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_dvs_6axis_config32)) ||
-		get_user(kp->exp_id, &up->exp_id) ||
-		get_user(kp->width_y, &up->width_y) ||
-		get_user(kp->height_y, &up->height_y) ||
-		get_user(kp->width_uv, &up->width_uv) ||
-		get_user(kp->height_uv, &up->height_uv) ||
-		get_user(xcoords_y, &up->xcoords_y) ||
-		get_user(ycoords_y, &up->ycoords_y) ||
-		get_user(xcoords_uv, &up->xcoords_uv) ||
-		get_user(ycoords_uv, &up->ycoords_uv))
-			return -EFAULT;
-
-	kp->xcoords_y = compat_ptr(xcoords_y);
-	kp->ycoords_y = compat_ptr(ycoords_y);
-	kp->xcoords_uv = compat_ptr(xcoords_uv);
-	kp->ycoords_uv = compat_ptr(ycoords_uv);
-	return 0;
-}
-
-static int get_atomisp_3a_statistics32(struct atomisp_3a_statistics *kp,
-				struct atomisp_3a_statistics32 __user *up)
-{
-	compat_uptr_t data;
-	compat_uptr_t rgby_data;
-
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_3a_statistics32)) ||
-		copy_from_user(kp, up, sizeof(struct atomisp_grid_info)) ||
-		get_user(rgby_data, &up->rgby_data) ||
-		get_user(data, &up->data) ||
-		get_user(kp->exp_id, &up->exp_id) ||
-		get_user(kp->isp_config_id, &up->isp_config_id))
-			return -EFAULT;
-
-	kp->data = compat_ptr(data);
-	kp->rgby_data = compat_ptr(rgby_data);
-
-	return 0;
-}
-
-static int put_atomisp_3a_statistics32(struct atomisp_3a_statistics *kp,
-				struct atomisp_3a_statistics32 __user *up)
-{
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
-	compat_uptr_t rgby_data = (compat_uptr_t)((uintptr_t)kp->rgby_data);
-
-	if (!access_ok(VERIFY_WRITE, up,
-			sizeof(struct atomisp_3a_statistics32)) ||
-		copy_to_user(up, kp, sizeof(struct atomisp_grid_info)) ||
-		put_user(rgby_data, &up->rgby_data) ||
-		put_user(data, &up->data) ||
-		put_user(kp->exp_id, &up->exp_id) ||
-		put_user(kp->isp_config_id, &up->isp_config_id))
-			return -EFAULT;
-
-	return 0;
-}
-
-
-static int get_atomisp_metadata_stat32(struct atomisp_metadata *kp,
-				struct atomisp_metadata32 __user *up)
-{
-	compat_uptr_t data;
-	compat_uptr_t effective_width;
-
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_metadata32)) ||
-		get_user(data, &up->data) ||
-		get_user(kp->width, &up->width) ||
-		get_user(kp->height, &up->height) ||
-		get_user(kp->stride, &up->stride) ||
-		get_user(kp->exp_id, &up->exp_id) ||
-		get_user(effective_width, &up->effective_width))
-			return -EFAULT;
-
-	kp->data = compat_ptr(data);
-	kp->effective_width = compat_ptr(effective_width);
-	return 0;
-}
-
-
-static int put_atomisp_metadata_stat32(struct atomisp_metadata *kp,
-				struct atomisp_metadata32 __user *up)
-{
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
-	compat_uptr_t effective_width =
-		(compat_uptr_t)((uintptr_t)kp->effective_width);
-	if (!access_ok(VERIFY_WRITE, up,
-			sizeof(struct atomisp_metadata32)) ||
-		put_user(data, &up->data) ||
-		put_user(kp->width, &up->width) ||
-		put_user(kp->height, &up->height) ||
-		put_user(kp->stride, &up->stride) ||
-		put_user(kp->exp_id, &up->exp_id) ||
-		put_user(effective_width, &up->effective_width))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int put_atomisp_metadata_by_type_stat32(
-				struct atomisp_metadata_with_type *kp,
-				struct atomisp_metadata_with_type32 __user *up)
-{
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
-	compat_uptr_t effective_width =
-		(compat_uptr_t)((uintptr_t)kp->effective_width);
-	if (!access_ok(VERIFY_WRITE, up,
-			sizeof(struct atomisp_metadata_with_type32)) ||
-		put_user(data, &up->data) ||
-		put_user(kp->width, &up->width) ||
-		put_user(kp->height, &up->height) ||
-		put_user(kp->stride, &up->stride) ||
-		put_user(kp->exp_id, &up->exp_id) ||
-		put_user(effective_width, &up->effective_width) ||
-		put_user(kp->type, &up->type))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int get_atomisp_metadata_by_type_stat32(
-				struct atomisp_metadata_with_type *kp,
-				struct atomisp_metadata_with_type32 __user *up)
-{
-	compat_uptr_t data;
-	compat_uptr_t effective_width;
-
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_metadata_with_type32)) ||
-		get_user(data, &up->data) ||
-		get_user(kp->width, &up->width) ||
-		get_user(kp->height, &up->height) ||
-		get_user(kp->stride, &up->stride) ||
-		get_user(kp->exp_id, &up->exp_id) ||
-		get_user(effective_width, &up->effective_width) ||
-		get_user(kp->type, &up->type))
-			return -EFAULT;
-
-	kp->data = compat_ptr(data);
-	kp->effective_width = compat_ptr(effective_width);
-	return 0;
-}
-
-static int get_atomisp_morph_table32(struct atomisp_morph_table *kp,
-				struct atomisp_morph_table32 __user *up)
-{
-	unsigned int n = ATOMISP_MORPH_TABLE_NUM_PLANES;
-
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_morph_table32)) ||
-		get_user(kp->enabled, &up->enabled) ||
-		get_user(kp->width, &up->width) ||
-		get_user(kp->height, &up->height))
-			return -EFAULT;
-
-	while (n-- > 0) {
-		uintptr_t *coord_kp = (uintptr_t *)&kp->coordinates_x[n];
-
-		if (get_user((*coord_kp), &up->coordinates_x[n]))
-			return -EFAULT;
-
-		coord_kp = (uintptr_t *)&kp->coordinates_y[n];
-		if (get_user((*coord_kp), &up->coordinates_y[n]))
-			return -EFAULT;
-	}
-	return 0;
-}
-
-static int put_atomisp_morph_table32(struct atomisp_morph_table *kp,
-				struct atomisp_morph_table32 __user *up)
-{
-	unsigned int n = ATOMISP_MORPH_TABLE_NUM_PLANES;
-
-	if (!access_ok(VERIFY_WRITE, up,
-			sizeof(struct atomisp_morph_table32)) ||
-		put_user(kp->enabled, &up->enabled) ||
-		put_user(kp->width, &up->width) ||
-		put_user(kp->height, &up->height))
-			return -EFAULT;
-
-	while (n-- > 0) {
-		uintptr_t *coord_kp = (uintptr_t *)&kp->coordinates_x[n];
-
-		if (put_user((*coord_kp), &up->coordinates_x[n]))
-			return -EFAULT;
-
-		coord_kp = (uintptr_t *)&kp->coordinates_y[n];
-		if (put_user((*coord_kp), &up->coordinates_y[n]))
-			return -EFAULT;
-	}
-	return 0;
-}
-
-static int get_atomisp_overlay32(struct atomisp_overlay *kp,
-					struct atomisp_overlay32 __user *up)
-{
-	compat_uptr_t frame;
-	if (!access_ok(VERIFY_READ, up, sizeof(struct atomisp_overlay32)) ||
-		get_user(frame, &up->frame) ||
-		get_user(kp->bg_y, &up->bg_y) ||
-		get_user(kp->bg_u, &up->bg_u) ||
-		get_user(kp->bg_v, &up->bg_v) ||
-		get_user(kp->blend_input_perc_y, &up->blend_input_perc_y) ||
-		get_user(kp->blend_input_perc_u, &up->blend_input_perc_u) ||
-		get_user(kp->blend_input_perc_v, &up->blend_input_perc_v) ||
-		get_user(kp->blend_overlay_perc_y,
-				&up->blend_overlay_perc_y) ||
-		get_user(kp->blend_overlay_perc_u,
-				&up->blend_overlay_perc_u) ||
-		get_user(kp->blend_overlay_perc_v,
-				&up->blend_overlay_perc_v) ||
-		get_user(kp->blend_overlay_perc_u,
-				&up->blend_overlay_perc_u) ||
-		get_user(kp->overlay_start_x, &up->overlay_start_y))
-			return -EFAULT;
-
-	kp->frame = compat_ptr(frame);
-	return 0;
-}
-
-static int put_atomisp_overlay32(struct atomisp_overlay *kp,
-					struct atomisp_overlay32 __user *up)
-{
-	compat_uptr_t frame = (compat_uptr_t)((uintptr_t)kp->frame);
-
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct atomisp_overlay32)) ||
-		put_user(frame, &up->frame) ||
-		put_user(kp->bg_y, &up->bg_y) ||
-		put_user(kp->bg_u, &up->bg_u) ||
-		put_user(kp->bg_v, &up->bg_v) ||
-		put_user(kp->blend_input_perc_y, &up->blend_input_perc_y) ||
-		put_user(kp->blend_input_perc_u, &up->blend_input_perc_u) ||
-		put_user(kp->blend_input_perc_v, &up->blend_input_perc_v) ||
-		put_user(kp->blend_overlay_perc_y,
-				&up->blend_overlay_perc_y) ||
-		put_user(kp->blend_overlay_perc_u,
-				&up->blend_overlay_perc_u) ||
-		put_user(kp->blend_overlay_perc_v,
-				&up->blend_overlay_perc_v) ||
-		put_user(kp->blend_overlay_perc_u,
-				&up->blend_overlay_perc_u) ||
-		put_user(kp->overlay_start_x, &up->overlay_start_y))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int get_atomisp_calibration_group32(
-				struct atomisp_calibration_group *kp,
-				struct atomisp_calibration_group32 __user *up)
-{
-	compat_uptr_t calb_grp_values;
-
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_calibration_group32)) ||
-		get_user(kp->size, &up->size) ||
-		get_user(kp->type, &up->type) ||
-		get_user(calb_grp_values, &up->calb_grp_values))
-			return -EFAULT;
-
-	kp->calb_grp_values = compat_ptr(calb_grp_values);
-	return 0;
-}
-
-static int put_atomisp_calibration_group32(
-				struct atomisp_calibration_group *kp,
-				struct atomisp_calibration_group32 __user *up)
-{
-	compat_uptr_t calb_grp_values =
-			(compat_uptr_t)((uintptr_t)kp->calb_grp_values);
-
-	if (!access_ok(VERIFY_WRITE, up,
-			sizeof(struct atomisp_calibration_group32)) ||
-		put_user(kp->size, &up->size) ||
-		put_user(kp->type, &up->type) ||
-		put_user(calb_grp_values, &up->calb_grp_values))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int get_atomisp_acc_fw_load32(struct atomisp_acc_fw_load *kp,
-				struct atomisp_acc_fw_load32 __user *up)
-{
-	compat_uptr_t data;
-
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_acc_fw_load32)) ||
-		get_user(kp->size, &up->size) ||
-		get_user(kp->fw_handle, &up->fw_handle) ||
-		get_user(data, &up->data))
-			return -EFAULT;
-
-	kp->data = compat_ptr(data);
-	return 0;
-}
-
-static int put_atomisp_acc_fw_load32(struct atomisp_acc_fw_load *kp,
-				struct atomisp_acc_fw_load32 __user *up)
-{
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
-
-	if (!access_ok(VERIFY_WRITE, up,
-			sizeof(struct atomisp_acc_fw_load32)) ||
-		put_user(kp->size, &up->size) ||
-		put_user(kp->fw_handle, &up->fw_handle) ||
-		put_user(data, &up->data))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int get_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg *kp,
-					struct atomisp_acc_fw_arg32 __user *up)
-{
-	compat_uptr_t value;
-
-	if (!access_ok(VERIFY_READ, up, sizeof(struct atomisp_acc_fw_arg32)) ||
-		get_user(kp->fw_handle, &up->fw_handle) ||
-		get_user(kp->index, &up->index) ||
-		get_user(value, &up->value) ||
-		get_user(kp->size, &up->size))
-			return -EFAULT;
-
-	kp->value = compat_ptr(value);
-	return 0;
-}
-
-static int put_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg *kp,
-					struct atomisp_acc_fw_arg32 __user *up)
-{
-	compat_uptr_t value = (compat_uptr_t)((uintptr_t)kp->value);
-
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct atomisp_acc_fw_arg32)) ||
-		put_user(kp->fw_handle, &up->fw_handle) ||
-		put_user(kp->index, &up->index) ||
-		put_user(value, &up->value) ||
-		put_user(kp->size, &up->size))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int get_v4l2_private_int_data32(struct v4l2_private_int_data *kp,
-					struct v4l2_private_int_data32 __user *up)
-{
-	compat_uptr_t data;
-
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct v4l2_private_int_data32)) ||
-		get_user(kp->size, &up->size) ||
-		get_user(data, &up->data) ||
-		get_user(kp->reserved[0], &up->reserved[0]) ||
-		get_user(kp->reserved[1], &up->reserved[1]))
-			return -EFAULT;
-
-	kp->data = compat_ptr(data);
-	return 0;
-}
-
-static int put_v4l2_private_int_data32(struct v4l2_private_int_data *kp,
-				struct v4l2_private_int_data32 __user *up)
-{
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
-
-	if (!access_ok(VERIFY_WRITE, up,
-			sizeof(struct v4l2_private_int_data32)) ||
-		put_user(kp->size, &up->size) ||
-		put_user(data, &up->data) ||
-		put_user(kp->reserved[0], &up->reserved[0]) ||
-		put_user(kp->reserved[1], &up->reserved[1]))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int get_atomisp_shading_table32(struct atomisp_shading_table *kp,
-				struct atomisp_shading_table32 __user *up)
-{
-	unsigned int n = ATOMISP_NUM_SC_COLORS;
-
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_shading_table32)) ||
-		get_user(kp->enable, &up->enable) ||
-		get_user(kp->sensor_width, &up->sensor_width) ||
-		get_user(kp->sensor_height, &up->sensor_height) ||
-		get_user(kp->width, &up->width) ||
-		get_user(kp->height, &up->height) ||
-		get_user(kp->fraction_bits, &up->fraction_bits))
-			return -EFAULT;
-
-	while (n-- > 0) {
-		uintptr_t *data_p = (uintptr_t *)&kp->data[n];
-
-		if (get_user((*data_p), &up->data[n]))
-			return -EFAULT;
-	}
-	return 0;
-}
-
-static int get_atomisp_acc_map32(struct atomisp_acc_map *kp,
-					struct atomisp_acc_map32 __user *up)
-{
-	compat_uptr_t user_ptr;
-
-	if (!access_ok(VERIFY_READ, up, sizeof(struct atomisp_acc_map32)) ||
-		get_user(kp->flags, &up->flags) ||
-		get_user(kp->length, &up->length) ||
-		get_user(user_ptr, &up->user_ptr) ||
-		get_user(kp->css_ptr, &up->css_ptr) ||
-		get_user(kp->reserved[0], &up->reserved[0]) ||
-		get_user(kp->reserved[1], &up->reserved[1]) ||
-		get_user(kp->reserved[2], &up->reserved[2]) ||
-		get_user(kp->reserved[3], &up->reserved[3]))
-			return -EFAULT;
-
-	kp->user_ptr = compat_ptr(user_ptr);
-	return 0;
-}
-
-static int put_atomisp_acc_map32(struct atomisp_acc_map *kp,
-					struct atomisp_acc_map32 __user *up)
-{
-	compat_uptr_t user_ptr = (compat_uptr_t)((uintptr_t)kp->user_ptr);
-
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct atomisp_acc_map32)) ||
-		put_user(kp->flags, &up->flags) ||
-		put_user(kp->length, &up->length) ||
-		put_user(user_ptr, &up->user_ptr) ||
-		put_user(kp->css_ptr, &up->css_ptr) ||
-		put_user(kp->reserved[0], &up->reserved[0]) ||
-		put_user(kp->reserved[1], &up->reserved[1]) ||
-		put_user(kp->reserved[2], &up->reserved[2]) ||
-		put_user(kp->reserved[3], &up->reserved[3]))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int get_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg *kp,
-				struct atomisp_acc_s_mapped_arg32 __user *up)
-{
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_acc_s_mapped_arg32)) ||
-		get_user(kp->fw_handle, &up->fw_handle) ||
-		get_user(kp->memory, &up->memory) ||
-		get_user(kp->length, &up->length) ||
-		get_user(kp->css_ptr, &up->css_ptr))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int put_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg *kp,
-				struct atomisp_acc_s_mapped_arg32 __user *up)
-{
-	if (!access_ok(VERIFY_WRITE, up,
-			sizeof(struct atomisp_acc_s_mapped_arg32)) ||
-		put_user(kp->fw_handle, &up->fw_handle) ||
-		put_user(kp->memory, &up->memory) ||
-		put_user(kp->length, &up->length) ||
-		put_user(kp->css_ptr, &up->css_ptr))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int get_atomisp_parameters32(struct atomisp_parameters *kp,
-					struct atomisp_parameters32 __user *up)
-{
-	int n = offsetof(struct atomisp_parameters32, output_frame) /
-				sizeof(compat_uptr_t);
-	unsigned int size, offset = 0;
-	void  __user *user_ptr;
-#ifdef ISP2401
-	unsigned int stp, mtp, dcp, dscp = 0;
-
-#endif
-	if (!access_ok(VERIFY_READ, up, sizeof(struct atomisp_parameters32)))
-			return -EFAULT;
-
-	while (n >= 0) {
-		compat_uptr_t *src = (compat_uptr_t *)up + n;
-		uintptr_t *dst = (uintptr_t *)kp + n;
-
-		if (get_user((*dst), src))
-			return -EFAULT;
-		n--;
-	}
-	if (get_user(kp->isp_config_id, &up->isp_config_id) ||
-#ifndef ISP2401
-	    get_user(kp->per_frame_setting, &up->per_frame_setting))
-#else
-	    get_user(kp->per_frame_setting, &up->per_frame_setting) ||
-	    get_user(stp, &up->shading_table) ||
-	    get_user(mtp, &up->morph_table) ||
-	    get_user(dcp, &up->dvs2_coefs) ||
-	    get_user(dscp, &up->dvs_6axis_config))
-#endif
-		return -EFAULT;
-
-	{
-		union {
-			struct atomisp_shading_table shading_table;
-			struct atomisp_morph_table   morph_table;
-			struct atomisp_dis_coefficients dvs2_coefs;
-			struct atomisp_dvs_6axis_config dvs_6axis_config;
-		} karg;
-
-		size = sizeof(struct atomisp_shading_table) +
-				sizeof(struct atomisp_morph_table) +
-				sizeof(struct atomisp_dis_coefficients) +
-				sizeof(struct atomisp_dvs_6axis_config);
-		user_ptr = compat_alloc_user_space(size);
-
-		/* handle shading table */
-#ifndef ISP2401
-		if (up->shading_table != 0) {
-#else
-		if (stp != 0) {
-#endif
-			if (get_atomisp_shading_table32(&karg.shading_table,
-				(struct atomisp_shading_table32 __user *)
-#ifndef ISP2401
-						(uintptr_t)up->shading_table))
-#else
-						(uintptr_t)stp))
-#endif
-				return -EFAULT;
-
-			kp->shading_table = user_ptr + offset;
-			offset = sizeof(struct atomisp_shading_table);
-			if (!kp->shading_table)
-				return -EFAULT;
-
-			if (copy_to_user(kp->shading_table,
-					 &karg.shading_table,
-					 sizeof(struct atomisp_shading_table)))
-				return -EFAULT;
-		}
-
-		/* handle morph table */
-#ifndef ISP2401
-		if (up->morph_table != 0) {
-#else
-		if (mtp != 0) {
-#endif
-			if (get_atomisp_morph_table32(&karg.morph_table,
-					(struct atomisp_morph_table32 __user *)
-#ifndef ISP2401
-						(uintptr_t)up->morph_table))
-#else
-						(uintptr_t)mtp))
-#endif
-				return -EFAULT;
-
-			kp->morph_table = user_ptr + offset;
-			offset += sizeof(struct atomisp_morph_table);
-			if (!kp->morph_table)
-				return -EFAULT;
-
-			if (copy_to_user(kp->morph_table, &karg.morph_table,
-					   sizeof(struct atomisp_morph_table)))
-				return -EFAULT;
-		}
-
-		/* handle dvs2 coefficients */
-#ifndef ISP2401
-		if (up->dvs2_coefs != 0) {
-#else
-		if (dcp != 0) {
-#endif
-			if (get_atomisp_dis_coefficients32(&karg.dvs2_coefs,
-				(struct atomisp_dis_coefficients32 __user *)
-#ifndef ISP2401
-						(uintptr_t)up->dvs2_coefs))
-#else
-						(uintptr_t)dcp))
-#endif
-				return -EFAULT;
-
-			kp->dvs2_coefs = user_ptr + offset;
-			offset += sizeof(struct atomisp_dis_coefficients);
-			if (!kp->dvs2_coefs)
-				return -EFAULT;
-
-			if (copy_to_user(kp->dvs2_coefs, &karg.dvs2_coefs,
-				sizeof(struct atomisp_dis_coefficients)))
-				return -EFAULT;
-		}
-		/* handle dvs 6axis configuration */
-#ifndef ISP2401
-		if (up->dvs_6axis_config != 0) {
-#else
-		if (dscp != 0) {
-#endif
-			if (get_atomisp_dvs_6axis_config32(&karg.dvs_6axis_config,
-				(struct atomisp_dvs_6axis_config32 __user *)
-#ifndef ISP2401
-						(uintptr_t)up->dvs_6axis_config))
-#else
-						(uintptr_t)dscp))
-#endif
-				return -EFAULT;
-
-			kp->dvs_6axis_config = user_ptr + offset;
-			offset += sizeof(struct atomisp_dvs_6axis_config);
-			if (!kp->dvs_6axis_config)
-				return -EFAULT;
-
-			if (copy_to_user(kp->dvs_6axis_config, &karg.dvs_6axis_config,
-				sizeof(struct atomisp_dvs_6axis_config)))
-				return -EFAULT;
-		}
-	}
-	return 0;
-}
-
-static int get_atomisp_acc_fw_load_to_pipe32(
-			struct atomisp_acc_fw_load_to_pipe *kp,
-			struct atomisp_acc_fw_load_to_pipe32 __user *up)
-{
-	compat_uptr_t data;
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_acc_fw_load_to_pipe32)) ||
-		get_user(kp->flags, &up->flags) ||
-		get_user(kp->fw_handle, &up->fw_handle) ||
-		get_user(kp->size, &up->size) ||
-		get_user(kp->type, &up->type) ||
-		get_user(kp->reserved[0], &up->reserved[0]) ||
-		get_user(kp->reserved[1], &up->reserved[1]) ||
-		get_user(kp->reserved[2], &up->reserved[2]) ||
-		get_user(data, &up->data))
-			return -EFAULT;
-
-	kp->data = compat_ptr(data);
-	return 0;
-}
-
-static int put_atomisp_acc_fw_load_to_pipe32(
-			struct atomisp_acc_fw_load_to_pipe *kp,
-			struct atomisp_acc_fw_load_to_pipe32 __user *up)
-{
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
-	if (!access_ok(VERIFY_WRITE, up,
-			sizeof(struct atomisp_acc_fw_load_to_pipe32)) ||
-		put_user(kp->flags, &up->flags) ||
-		put_user(kp->fw_handle, &up->fw_handle) ||
-		put_user(kp->size, &up->size) ||
-		put_user(kp->type, &up->type) ||
-		put_user(kp->reserved[0], &up->reserved[0]) ||
-		put_user(kp->reserved[1], &up->reserved[1]) ||
-		put_user(kp->reserved[2], &up->reserved[2]) ||
-		put_user(data, &up->data))
-			return -EFAULT;
-
-	return 0;
-}
-
-static int get_atomisp_sensor_ae_bracketing_lut(
-			struct atomisp_sensor_ae_bracketing_lut *kp,
-			struct atomisp_sensor_ae_bracketing_lut32 __user *up)
-{
-	compat_uptr_t lut;
-	if (!access_ok(VERIFY_READ, up,
-			sizeof(struct atomisp_sensor_ae_bracketing_lut32)) ||
-		get_user(kp->lut_size, &up->lut_size) ||
-		get_user(lut, &up->lut))
-			return -EFAULT;
-
-	kp->lut = compat_ptr(lut);
-	return 0;
-}
-
-static long native_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
-{
-	long ret = -ENOIOCTLCMD;
-
-	if (file->f_op->unlocked_ioctl)
-		ret = file->f_op->unlocked_ioctl(file, cmd, arg);
-
-	return ret;
-}
-
-long atomisp_do_compat_ioctl(struct file *file,
-			    unsigned int cmd, unsigned long arg)
-{
-	union {
-		struct atomisp_histogram his;
-		struct atomisp_dis_statistics dis_s;
-		struct atomisp_dis_coefficients dis_c;
-		struct atomisp_dvs_6axis_config dvs_c;
-		struct atomisp_3a_statistics s3a_s;
-		struct atomisp_morph_table mor_t;
-		struct v4l2_framebuffer v4l2_buf;
-		struct atomisp_overlay overlay;
-		struct atomisp_calibration_group cal_grp;
-		struct atomisp_acc_fw_load acc_fw_load;
-		struct atomisp_acc_fw_arg acc_fw_arg;
-		struct v4l2_private_int_data v4l2_pri_data;
-		struct atomisp_shading_table shd_tbl;
-		struct atomisp_acc_map acc_map;
-		struct atomisp_acc_s_mapped_arg acc_map_arg;
-		struct atomisp_parameters param;
-		struct atomisp_acc_fw_load_to_pipe acc_fw_to_pipe;
-		struct atomisp_metadata md;
-		struct atomisp_metadata_with_type md_with_type;
-		struct atomisp_sensor_ae_bracketing_lut lut;
-	} karg;
-	mm_segment_t old_fs;
-	void __user *up = compat_ptr(arg);
-	long err = -ENOIOCTLCMD;
-
-	/* First, convert the command. */
-	switch (cmd) {
-	case ATOMISP_IOC_G_HISTOGRAM32:
-		cmd = ATOMISP_IOC_G_HISTOGRAM;
-		break;
-	case ATOMISP_IOC_S_HISTOGRAM32:
-		cmd = ATOMISP_IOC_S_HISTOGRAM;
-		break;
-	case ATOMISP_IOC_G_DIS_STAT32:
-		cmd = ATOMISP_IOC_G_DIS_STAT;
-		break;
-	case ATOMISP_IOC_S_DIS_COEFS32:
-		cmd = ATOMISP_IOC_S_DIS_COEFS;
-		break;
-	case ATOMISP_IOC_S_DIS_VECTOR32:
-		cmd = ATOMISP_IOC_S_DIS_VECTOR;
-		break;
-	case ATOMISP_IOC_G_3A_STAT32:
-		cmd = ATOMISP_IOC_G_3A_STAT;
-		break;
-	case ATOMISP_IOC_G_ISP_GDC_TAB32:
-		cmd = ATOMISP_IOC_G_ISP_GDC_TAB;
-		break;
-	case ATOMISP_IOC_S_ISP_GDC_TAB32:
-		cmd = ATOMISP_IOC_S_ISP_GDC_TAB;
-		break;
-	case ATOMISP_IOC_S_ISP_FPN_TABLE32:
-		cmd = ATOMISP_IOC_S_ISP_FPN_TABLE;
-		break;
-	case ATOMISP_IOC_G_ISP_OVERLAY32:
-		cmd = ATOMISP_IOC_G_ISP_OVERLAY;
-		break;
-	case ATOMISP_IOC_S_ISP_OVERLAY32:
-		cmd = ATOMISP_IOC_S_ISP_OVERLAY;
-		break;
-	case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP32:
-		cmd = ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP;
-		break;
-	case ATOMISP_IOC_ACC_LOAD32:
-		cmd = ATOMISP_IOC_ACC_LOAD;
-		break;
-	case ATOMISP_IOC_ACC_S_ARG32:
-		cmd = ATOMISP_IOC_ACC_S_ARG;
-		break;
-	case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA32:
-		cmd = ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA;
-		break;
-	case ATOMISP_IOC_S_ISP_SHD_TAB32:
-		cmd = ATOMISP_IOC_S_ISP_SHD_TAB;
-		break;
-	case ATOMISP_IOC_ACC_DESTAB32:
-		cmd = ATOMISP_IOC_ACC_DESTAB;
-		break;
-	case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA32:
-		cmd = ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA;
-		break;
-	case ATOMISP_IOC_ACC_MAP32:
-		cmd = ATOMISP_IOC_ACC_MAP;
-		break;
-	case ATOMISP_IOC_ACC_UNMAP32:
-		cmd = ATOMISP_IOC_ACC_UNMAP;
-		break;
-	case ATOMISP_IOC_ACC_S_MAPPED_ARG32:
-		cmd = ATOMISP_IOC_ACC_S_MAPPED_ARG;
-		break;
-	case ATOMISP_IOC_S_PARAMETERS32:
-		cmd = ATOMISP_IOC_S_PARAMETERS;
-		break;
-	case ATOMISP_IOC_ACC_LOAD_TO_PIPE32:
-		cmd = ATOMISP_IOC_ACC_LOAD_TO_PIPE;
-		break;
-	case ATOMISP_IOC_G_METADATA32:
-		cmd = ATOMISP_IOC_G_METADATA;
-		break;
-	case ATOMISP_IOC_G_METADATA_BY_TYPE32:
-		cmd = ATOMISP_IOC_G_METADATA_BY_TYPE;
-		break;
-	case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT32:
-		cmd = ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT;
-		break;
-	}
-
-	switch (cmd) {
-	case ATOMISP_IOC_G_HISTOGRAM:
-	case ATOMISP_IOC_S_HISTOGRAM:
-		err = get_atomisp_histogram32(&karg.his, up);
-		break;
-	case ATOMISP_IOC_G_DIS_STAT:
-		err = get_atomisp_dis_statistics32(&karg.dis_s, up);
-		break;
-	case ATOMISP_IOC_S_DIS_COEFS:
-		err = get_atomisp_dis_coefficients32(&karg.dis_c, up);
-		break;
-	case ATOMISP_IOC_S_DIS_VECTOR:
-		err = get_atomisp_dvs_6axis_config32(&karg.dvs_c, up);
-		break;
-	case ATOMISP_IOC_G_3A_STAT:
-		err = get_atomisp_3a_statistics32(&karg.s3a_s, up);
-		break;
-	case ATOMISP_IOC_G_ISP_GDC_TAB:
-	case ATOMISP_IOC_S_ISP_GDC_TAB:
-		err = get_atomisp_morph_table32(&karg.mor_t, up);
-		break;
-	case ATOMISP_IOC_S_ISP_FPN_TABLE:
-		err = get_v4l2_framebuffer32(&karg.v4l2_buf, up);
-		break;
-	case ATOMISP_IOC_G_ISP_OVERLAY:
-	case ATOMISP_IOC_S_ISP_OVERLAY:
-		err = get_atomisp_overlay32(&karg.overlay, up);
-		break;
-	case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP:
-		err = get_atomisp_calibration_group32(&karg.cal_grp, up);
-		break;
-	case ATOMISP_IOC_ACC_LOAD:
-		err = get_atomisp_acc_fw_load32(&karg.acc_fw_load, up);
-		break;
-	case ATOMISP_IOC_ACC_S_ARG:
-	case ATOMISP_IOC_ACC_DESTAB:
-		err = get_atomisp_acc_fw_arg32(&karg.acc_fw_arg, up);
-		break;
-	case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA:
-	case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA:
-		err = get_v4l2_private_int_data32(&karg.v4l2_pri_data, up);
-		break;
-	case ATOMISP_IOC_S_ISP_SHD_TAB:
-		err = get_atomisp_shading_table32(&karg.shd_tbl, up);
-		break;
-	case ATOMISP_IOC_ACC_MAP:
-	case ATOMISP_IOC_ACC_UNMAP:
-		err = get_atomisp_acc_map32(&karg.acc_map, up);
-		break;
-	case ATOMISP_IOC_ACC_S_MAPPED_ARG:
-		err = get_atomisp_acc_s_mapped_arg32(&karg.acc_map_arg, up);
-		break;
-	case ATOMISP_IOC_S_PARAMETERS:
-		err = get_atomisp_parameters32(&karg.param, up);
-		break;
-	case ATOMISP_IOC_ACC_LOAD_TO_PIPE:
-		err = get_atomisp_acc_fw_load_to_pipe32(&karg.acc_fw_to_pipe,
-							up);
-		break;
-	case ATOMISP_IOC_G_METADATA:
-		err = get_atomisp_metadata_stat32(&karg.md, up);
-		break;
-	case ATOMISP_IOC_G_METADATA_BY_TYPE:
-		err = get_atomisp_metadata_by_type_stat32(&karg.md_with_type,
-							up);
-		break;
-	case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT:
-		err = get_atomisp_sensor_ae_bracketing_lut(&karg.lut, up);
-		break;
-	}
-	if (err)
-		return err;
-
-	old_fs = get_fs();
-	set_fs(KERNEL_DS);
-	err = native_ioctl(file, cmd, (unsigned long)&karg);
-	set_fs(old_fs);
-	if (err)
-		return err;
-
-	switch (cmd) {
-	case ATOMISP_IOC_G_HISTOGRAM:
-		err = put_atomisp_histogram32(&karg.his, up);
-		break;
-	case ATOMISP_IOC_G_DIS_STAT:
-		err = put_atomisp_dis_statistics32(&karg.dis_s, up);
-		break;
-	case ATOMISP_IOC_G_3A_STAT:
-		err = put_atomisp_3a_statistics32(&karg.s3a_s, up);
-		break;
-	case ATOMISP_IOC_G_ISP_GDC_TAB:
-		err = put_atomisp_morph_table32(&karg.mor_t, up);
-		break;
-	case ATOMISP_IOC_G_ISP_OVERLAY:
-		err = put_atomisp_overlay32(&karg.overlay, up);
-		break;
-	case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP:
-		err = put_atomisp_calibration_group32(&karg.cal_grp, up);
-		break;
-	case ATOMISP_IOC_ACC_LOAD:
-		err = put_atomisp_acc_fw_load32(&karg.acc_fw_load, up);
-		break;
-	case ATOMISP_IOC_ACC_S_ARG:
-	case ATOMISP_IOC_ACC_DESTAB:
-		err = put_atomisp_acc_fw_arg32(&karg.acc_fw_arg, up);
-		break;
-	case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA:
-	case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA:
-		err = put_v4l2_private_int_data32(&karg.v4l2_pri_data, up);
-		break;
-	case ATOMISP_IOC_ACC_MAP:
-	case ATOMISP_IOC_ACC_UNMAP:
-		err = put_atomisp_acc_map32(&karg.acc_map, up);
-		break;
-	case ATOMISP_IOC_ACC_S_MAPPED_ARG:
-		err = put_atomisp_acc_s_mapped_arg32(&karg.acc_map_arg, up);
-		break;
-	case ATOMISP_IOC_ACC_LOAD_TO_PIPE:
-		err = put_atomisp_acc_fw_load_to_pipe32(&karg.acc_fw_to_pipe,
-							up);
-		break;
-	case ATOMISP_IOC_G_METADATA:
-		err = put_atomisp_metadata_stat32(&karg.md, up);
-		break;
-	case ATOMISP_IOC_G_METADATA_BY_TYPE:
-		err = put_atomisp_metadata_by_type_stat32(&karg.md_with_type,
-							up);
-		break;
-	}
-
-	return err;
-}
-
-long atomisp_compat_ioctl32(struct file *file,
-			    unsigned int cmd, unsigned long arg)
-{
-
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	long ret = -ENOIOCTLCMD;
-
-	if (!file->f_op->unlocked_ioctl)
-		return ret;
-
-	switch (cmd) {
-	case ATOMISP_IOC_G_XNR:
-	case ATOMISP_IOC_S_XNR:
-	case ATOMISP_IOC_G_NR:
-	case ATOMISP_IOC_S_NR:
-	case ATOMISP_IOC_G_TNR:
-	case ATOMISP_IOC_S_TNR:
-	case ATOMISP_IOC_G_BLACK_LEVEL_COMP:
-	case ATOMISP_IOC_S_BLACK_LEVEL_COMP:
-	case ATOMISP_IOC_G_EE:
-	case ATOMISP_IOC_S_EE:
-	case ATOMISP_IOC_S_DIS_VECTOR:
-	case ATOMISP_IOC_G_ISP_PARM:
-	case ATOMISP_IOC_S_ISP_PARM:
-	case ATOMISP_IOC_G_ISP_GAMMA:
-	case ATOMISP_IOC_S_ISP_GAMMA:
-	case ATOMISP_IOC_ISP_MAKERNOTE:
-	case ATOMISP_IOC_G_ISP_MACC:
-	case ATOMISP_IOC_S_ISP_MACC:
-	case ATOMISP_IOC_G_ISP_BAD_PIXEL_DETECTION:
-	case ATOMISP_IOC_S_ISP_BAD_PIXEL_DETECTION:
-	case ATOMISP_IOC_G_ISP_FALSE_COLOR_CORRECTION:
-	case ATOMISP_IOC_S_ISP_FALSE_COLOR_CORRECTION:
-	case ATOMISP_IOC_G_ISP_CTC:
-	case ATOMISP_IOC_S_ISP_CTC:
-	case ATOMISP_IOC_G_ISP_WHITE_BALANCE:
-	case ATOMISP_IOC_S_ISP_WHITE_BALANCE:
-	case ATOMISP_IOC_CAMERA_BRIDGE:
-	case ATOMISP_IOC_G_SENSOR_MODE_DATA:
-	case ATOMISP_IOC_S_EXPOSURE:
-	case ATOMISP_IOC_G_3A_CONFIG:
-	case ATOMISP_IOC_S_3A_CONFIG:
-	case ATOMISP_IOC_ACC_UNLOAD:
-	case ATOMISP_IOC_ACC_START:
-	case ATOMISP_IOC_ACC_WAIT:
-	case ATOMISP_IOC_ACC_ABORT:
-	case ATOMISP_IOC_G_ISP_GAMMA_CORRECTION:
-	case ATOMISP_IOC_S_ISP_GAMMA_CORRECTION:
-	case ATOMISP_IOC_S_CONT_CAPTURE_CONFIG:
-	case ATOMISP_IOC_G_DVS2_BQ_RESOLUTIONS:
-	case ATOMISP_IOC_EXT_ISP_CTRL:
-	case ATOMISP_IOC_EXP_ID_UNLOCK:
-	case ATOMISP_IOC_EXP_ID_CAPTURE:
-	case ATOMISP_IOC_S_ENABLE_DZ_CAPT_PIPE:
-	case ATOMISP_IOC_G_FORMATS_CONFIG:
-	case ATOMISP_IOC_S_FORMATS_CONFIG:
-	case ATOMISP_IOC_S_EXPOSURE_WINDOW:
-	case ATOMISP_IOC_S_ACC_STATE:
-	case ATOMISP_IOC_G_ACC_STATE:
-	case ATOMISP_IOC_INJECT_A_FAKE_EVENT:
-	case ATOMISP_IOC_G_SENSOR_AE_BRACKETING_INFO:
-	case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_MODE:
-	case ATOMISP_IOC_G_SENSOR_AE_BRACKETING_MODE:
-	case ATOMISP_IOC_G_INVALID_FRAME_NUM:
-	case ATOMISP_IOC_S_ARRAY_RESOLUTION:
-#ifdef ISP2401
-	case ATOMISP_IOC_S_SENSOR_RUNMODE:
-	case ATOMISP_IOC_G_UPDATE_EXPOSURE:
-#endif
-		ret = native_ioctl(file, cmd, arg);
-		break;
-
-	case ATOMISP_IOC_G_HISTOGRAM32:
-	case ATOMISP_IOC_S_HISTOGRAM32:
-	case ATOMISP_IOC_G_DIS_STAT32:
-	case ATOMISP_IOC_S_DIS_COEFS32:
-	case ATOMISP_IOC_S_DIS_VECTOR32:
-	case ATOMISP_IOC_G_3A_STAT32:
-	case ATOMISP_IOC_G_ISP_GDC_TAB32:
-	case ATOMISP_IOC_S_ISP_GDC_TAB32:
-	case ATOMISP_IOC_S_ISP_FPN_TABLE32:
-	case ATOMISP_IOC_G_ISP_OVERLAY32:
-	case ATOMISP_IOC_S_ISP_OVERLAY32:
-	case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP32:
-	case ATOMISP_IOC_ACC_LOAD32:
-	case ATOMISP_IOC_ACC_S_ARG32:
-	case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA32:
-	case ATOMISP_IOC_S_ISP_SHD_TAB32:
-	case ATOMISP_IOC_ACC_DESTAB32:
-	case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA32:
-	case ATOMISP_IOC_ACC_MAP32:
-	case ATOMISP_IOC_ACC_UNMAP32:
-	case ATOMISP_IOC_ACC_S_MAPPED_ARG32:
-	case ATOMISP_IOC_S_PARAMETERS32:
-	case ATOMISP_IOC_ACC_LOAD_TO_PIPE32:
-	case ATOMISP_IOC_G_METADATA32:
-	case ATOMISP_IOC_G_METADATA_BY_TYPE32:
-	case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT32:
-		ret = atomisp_do_compat_ioctl(file, cmd, arg);
-		break;
-
-	default:
-		dev_warn(isp->dev,
-			"%s: unknown ioctl '%c', dir=%d, #%d (0x%08x)\n",
-			__func__, _IOC_TYPE(cmd), _IOC_DIR(cmd), _IOC_NR(cmd),
-			cmd);
-		break;
-	}
-	return ret;
-}
-#endif /* CONFIG_COMPAT */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_ioctl32.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_ioctl32.h
deleted file mode 100644
index 95669eedaad1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_ioctl32.h
+++ /dev/null
@@ -1,365 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef __ATOMISP_COMPAT_IOCTL32_H__
-#define __ATOMISP_COMPAT_IOCTL32_H__
-
-#include <linux/compat.h>
-#include <linux/videodev2.h>
-
-#include "atomisp_compat.h"
-
-struct atomisp_histogram32 {
-	unsigned int num_elements;
-	compat_uptr_t data;
-};
-
-struct atomisp_dvs2_stat_types32 {
-	compat_uptr_t odd_real; /** real part of the odd statistics*/
-	compat_uptr_t odd_imag; /** imaginary part of the odd statistics*/
-	compat_uptr_t even_real;/** real part of the even statistics*/
-	compat_uptr_t even_imag;/** imaginary part of the even statistics*/
-};
-
-struct atomisp_dvs2_coef_types32 {
-	compat_uptr_t odd_real; /** real part of the odd coefficients*/
-	compat_uptr_t odd_imag; /** imaginary part of the odd coefficients*/
-	compat_uptr_t even_real;/** real part of the even coefficients*/
-	compat_uptr_t even_imag;/** imaginary part of the even coefficients*/
-};
-
-struct atomisp_dvs2_statistics32 {
-	struct atomisp_dvs_grid_info grid_info;
-	struct atomisp_dvs2_stat_types32 hor_prod;
-	struct atomisp_dvs2_stat_types32 ver_prod;
-};
-
-struct atomisp_dis_statistics32 {
-	struct atomisp_dvs2_statistics32 dvs2_stat;
-	uint32_t exp_id;
-};
-
-struct atomisp_dis_coefficients32 {
-	struct atomisp_dvs_grid_info grid_info;
-	struct atomisp_dvs2_coef_types32 hor_coefs;
-	struct atomisp_dvs2_coef_types32 ver_coefs;
-};
-
-struct atomisp_3a_statistics32 {
-	struct atomisp_grid_info  grid_info;
-	compat_uptr_t data;
-	compat_uptr_t rgby_data;
-	uint32_t exp_id;
-	uint32_t isp_config_id;
-};
-
-struct atomisp_metadata_with_type32 {
-	/* to specify which type of metadata to get */
-	enum atomisp_metadata_type type;
-	compat_uptr_t data;
-	uint32_t width;
-	uint32_t height;
-	uint32_t stride; /* in bytes */
-	uint32_t exp_id; /* exposure ID */
-	compat_uptr_t effective_width;
-};
-
-struct atomisp_metadata32 {
-	compat_uptr_t data;
-	uint32_t width;
-	uint32_t height;
-	uint32_t stride;
-	uint32_t exp_id;
-	compat_uptr_t effective_width;
-};
-
-struct atomisp_morph_table32 {
-	unsigned int enabled;
-	unsigned int height;
-	unsigned int width;	/* number of valid elements per line */
-	compat_uptr_t coordinates_x[ATOMISP_MORPH_TABLE_NUM_PLANES];
-	compat_uptr_t coordinates_y[ATOMISP_MORPH_TABLE_NUM_PLANES];
-};
-
-struct v4l2_framebuffer32 {
-	__u32			capability;
-	__u32			flags;
-	compat_uptr_t		base;
-	struct v4l2_pix_format	fmt;
-};
-
-struct atomisp_overlay32 {
-	/* the frame containing the overlay data The overlay frame width should
-	 * be the multiples of 2*ISP_VEC_NELEMS. The overlay frame height
-	 * should be the multiples of 2.
-	 */
-	compat_uptr_t frame;
-	/* Y value of overlay background */
-	unsigned char bg_y;
-	/* U value of overlay background */
-	char bg_u;
-	/* V value of overlay background */
-	char bg_v;
-	/* the blending percent of input data for Y subpixels */
-	unsigned char blend_input_perc_y;
-	/* the blending percent of input data for U subpixels */
-	unsigned char blend_input_perc_u;
-	/* the blending percent of input data for V subpixels */
-	unsigned char blend_input_perc_v;
-	/* the blending percent of overlay data for Y subpixels */
-	unsigned char blend_overlay_perc_y;
-	/* the blending percent of overlay data for U subpixels */
-	unsigned char blend_overlay_perc_u;
-	/* the blending percent of overlay data for V subpixels */
-	unsigned char blend_overlay_perc_v;
-	/* the overlay start x pixel position on output frame It should be the
-	   multiples of 2*ISP_VEC_NELEMS. */
-	unsigned int overlay_start_x;
-	/* the overlay start y pixel position on output frame It should be the
-	   multiples of 2. */
-	unsigned int overlay_start_y;
-};
-
-struct atomisp_calibration_group32 {
-	unsigned int size;
-	unsigned int type;
-	compat_uptr_t calb_grp_values;
-};
-
-struct atomisp_acc_fw_load32 {
-	unsigned int size;
-	unsigned int fw_handle;
-	compat_uptr_t data;
-};
-
-struct atomisp_acc_fw_arg32 {
-	unsigned int fw_handle;
-	unsigned int index;
-	compat_uptr_t value;
-	compat_size_t size;
-};
-
-struct v4l2_private_int_data32 {
-	__u32 size;
-	compat_uptr_t data;
-	__u32 reserved[2];
-};
-
-struct atomisp_shading_table32 {
-	__u32 enable;
-	__u32 sensor_width;
-	__u32 sensor_height;
-	__u32 width;
-	__u32 height;
-	__u32 fraction_bits;
-
-	compat_uptr_t data[ATOMISP_NUM_SC_COLORS];
-};
-
-struct atomisp_acc_map32 {
-	__u32 flags;			/* Flags, see list below */
-	__u32 length;			/* Length of data in bytes */
-	compat_uptr_t user_ptr;		/* Pointer into user space */
-	compat_ulong_t css_ptr;		/* Pointer into CSS address space */
-	__u32 reserved[4];		/* Set to zero */
-};
-
-struct atomisp_acc_s_mapped_arg32 {
-	unsigned int fw_handle;
-	__u32 memory;			/* one of enum atomisp_acc_memory */
-	compat_size_t length;
-	compat_ulong_t css_ptr;
-};
-
-struct atomisp_parameters32 {
-	compat_uptr_t wb_config;  /* White Balance config */
-	compat_uptr_t cc_config;  /* Color Correction config */
-	compat_uptr_t tnr_config; /* Temporal Noise Reduction */
-	compat_uptr_t ecd_config; /* Eigen Color Demosaicing */
-	compat_uptr_t ynr_config; /* Y(Luma) Noise Reduction */
-	compat_uptr_t fc_config;  /* Fringe Control */
-	compat_uptr_t formats_config;  /* Formats Control */
-	compat_uptr_t cnr_config; /* Chroma Noise Reduction */
-	compat_uptr_t macc_config;  /* MACC */
-	compat_uptr_t ctc_config; /* Chroma Tone Control */
-	compat_uptr_t aa_config;  /* Anti-Aliasing */
-	compat_uptr_t baa_config;  /* Anti-Aliasing */
-	compat_uptr_t ce_config;
-	compat_uptr_t dvs_6axis_config;
-	compat_uptr_t ob_config;  /* Objective Black config */
-	compat_uptr_t dp_config;  /* Dead Pixel config */
-	compat_uptr_t nr_config;  /* Noise Reduction config */
-	compat_uptr_t ee_config;  /* Edge Enhancement config */
-	compat_uptr_t de_config;  /* Demosaic config */
-	compat_uptr_t gc_config;  /* Gamma Correction config */
-	compat_uptr_t anr_config; /* Advanced Noise Reduction */
-	compat_uptr_t a3a_config; /* 3A Statistics config */
-	compat_uptr_t xnr_config; /* eXtra Noise Reduction */
-	compat_uptr_t dz_config;  /* Digital Zoom */
-	compat_uptr_t yuv2rgb_cc_config; /* Color
-							Correction config */
-	compat_uptr_t rgb2yuv_cc_config; /* Color
-							Correction config */
-	compat_uptr_t macc_table;
-	compat_uptr_t gamma_table;
-	compat_uptr_t ctc_table;
-	compat_uptr_t xnr_table;
-	compat_uptr_t r_gamma_table;
-	compat_uptr_t g_gamma_table;
-	compat_uptr_t b_gamma_table;
-	compat_uptr_t motion_vector; /* For 2-axis DVS */
-	compat_uptr_t shading_table;
-	compat_uptr_t morph_table;
-	compat_uptr_t dvs_coefs; /* DVS 1.0 coefficients */
-	compat_uptr_t dvs2_coefs; /* DVS 2.0 coefficients */
-	compat_uptr_t capture_config;
-	compat_uptr_t anr_thres;
-
-	compat_uptr_t	lin_2500_config;       /* Skylake: Linearization config */
-	compat_uptr_t	obgrid_2500_config;    /* Skylake: OBGRID config */
-	compat_uptr_t	bnr_2500_config;       /* Skylake: bayer denoise config */
-	compat_uptr_t	shd_2500_config;       /* Skylake: shading config */
-	compat_uptr_t	dm_2500_config;        /* Skylake: demosaic config */
-	compat_uptr_t	rgbpp_2500_config;     /* Skylake: RGBPP config */
-	compat_uptr_t	dvs_stat_2500_config;  /* Skylake: DVS STAT config */
-	compat_uptr_t	lace_stat_2500_config; /* Skylake: LACE STAT config */
-	compat_uptr_t	yuvp1_2500_config;     /* Skylake: yuvp1 config */
-	compat_uptr_t	yuvp2_2500_config;     /* Skylake: yuvp2 config */
-	compat_uptr_t	tnr_2500_config;       /* Skylake: TNR config */
-	compat_uptr_t	dpc_2500_config;       /* Skylake: DPC config */
-	compat_uptr_t	awb_2500_config;       /* Skylake: auto white balance config */
-	compat_uptr_t	awb_fr_2500_config;    /* Skylake: auto white balance filter response config */
-	compat_uptr_t	anr_2500_config;       /* Skylake: ANR config */
-	compat_uptr_t	af_2500_config;        /* Skylake: auto focus config */
-	compat_uptr_t	ae_2500_config;        /* Skylake: auto exposure config */
-	compat_uptr_t	bds_2500_config;       /* Skylake: bayer downscaler config */
-	compat_uptr_t	dvs_2500_config;       /* Skylake: digital video stabilization config */
-	compat_uptr_t	res_mgr_2500_config;
-
-	/*
-	 * Output frame pointer the config is to be applied to (optional),
-	 * set to NULL to make this config is applied as global.
-	 */
-	compat_uptr_t	output_frame;
-	/*
-	 * Unique ID to track which config was actually applied to a particular
-	 * frame, driver will send this id back with output frame together.
-	 */
-	uint32_t	isp_config_id;
-	uint32_t	per_frame_setting;
-};
-
-struct atomisp_acc_fw_load_to_pipe32 {
-	__u32 flags;			/* Flags, see below for valid values */
-	unsigned int fw_handle;		/* Handle, filled by kernel. */
-	__u32 size;			/* Firmware binary size */
-	compat_uptr_t data;		/* Pointer to firmware */
-	__u32 type;			/* Binary type */
-	__u32 reserved[3];		/* Set to zero */
-};
-
-struct atomisp_dvs_6axis_config32 {
-	uint32_t exp_id;
-	uint32_t width_y;
-	uint32_t height_y;
-	uint32_t width_uv;
-	uint32_t height_uv;
-	compat_uptr_t xcoords_y;
-	compat_uptr_t ycoords_y;
-	compat_uptr_t xcoords_uv;
-	compat_uptr_t ycoords_uv;
-};
-
-struct atomisp_sensor_ae_bracketing_lut32 {
-	compat_uptr_t lut;
-	unsigned int lut_size;
-};
-
-#define ATOMISP_IOC_G_HISTOGRAM32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 3, struct atomisp_histogram32)
-#define ATOMISP_IOC_S_HISTOGRAM32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 3, struct atomisp_histogram32)
-
-#define ATOMISP_IOC_G_DIS_STAT32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 6, struct atomisp_dis_statistics32)
-#define ATOMISP_IOC_S_DIS_COEFS32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 6, struct atomisp_dis_coefficients32)
-
-#define ATOMISP_IOC_S_DIS_VECTOR32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 6, struct atomisp_dvs_6axis_config32)
-
-#define ATOMISP_IOC_G_3A_STAT32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 7, struct atomisp_3a_statistics32)
-
-#define ATOMISP_IOC_G_ISP_GDC_TAB32 \
-	_IOR('v', BASE_VIDIOC_PRIVATE + 10, struct atomisp_morph_table32)
-#define ATOMISP_IOC_S_ISP_GDC_TAB32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 10, struct atomisp_morph_table32)
-
-#define ATOMISP_IOC_S_ISP_FPN_TABLE32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 17, struct v4l2_framebuffer32)
-
-#define ATOMISP_IOC_G_ISP_OVERLAY32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 18, struct atomisp_overlay32)
-#define ATOMISP_IOC_S_ISP_OVERLAY32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 18, struct atomisp_overlay32)
-
-#define ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 22, struct atomisp_calibration_group32)
-
-#define ATOMISP_IOC_ACC_LOAD32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 24, struct atomisp_acc_fw_load32)
-
-#define ATOMISP_IOC_ACC_S_ARG32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 24, struct atomisp_acc_fw_arg32)
-
-#define ATOMISP_IOC_ACC_DESTAB32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 25, struct atomisp_acc_fw_arg32)
-
-#define ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 26, struct v4l2_private_int_data32)
-
-#define ATOMISP_IOC_S_ISP_SHD_TAB32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 27, struct atomisp_shading_table32)
-
-#define ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 29, struct v4l2_private_int_data32)
-
-#define ATOMISP_IOC_ACC_MAP32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 30, struct atomisp_acc_map32)
-
-#define ATOMISP_IOC_ACC_UNMAP32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 30, struct atomisp_acc_map32)
-
-#define ATOMISP_IOC_ACC_S_MAPPED_ARG32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 30, struct atomisp_acc_s_mapped_arg32)
-
-#define ATOMISP_IOC_ACC_LOAD_TO_PIPE32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 31, struct atomisp_acc_fw_load_to_pipe32)
-
-#define ATOMISP_IOC_S_PARAMETERS32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 32, struct atomisp_parameters32)
-
-#define ATOMISP_IOC_G_METADATA32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 34, struct atomisp_metadata32)
-
-#define ATOMISP_IOC_G_METADATA_BY_TYPE32 \
-	_IOWR('v', BASE_VIDIOC_PRIVATE + 34, struct atomisp_metadata_with_type32)
-
-#define ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT32 \
-	_IOW('v', BASE_VIDIOC_PRIVATE + 43, struct atomisp_sensor_ae_bracketing_lut32)
-
-#endif /* __ATOMISP_COMPAT_IOCTL32_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_csi2.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_csi2.c
deleted file mode 100644
index fa03b78c3580..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_csi2.c
+++ /dev/null
@@ -1,442 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#include <media/v4l2-event.h>
-#include <media/v4l2-mediabus.h>
-#include "atomisp_cmd.h"
-#include "atomisp_internal.h"
-#include "atomisp-regs.h"
-
-static struct v4l2_mbus_framefmt *__csi2_get_format(struct
-						    atomisp_mipi_csi2_device
-						    *csi2,
-						    struct
-						    v4l2_subdev_pad_config *cfg,
-						    enum
-						    v4l2_subdev_format_whence
-						    which, unsigned int pad)
-{
-	if (which == V4L2_SUBDEV_FORMAT_TRY)
-		return v4l2_subdev_get_try_format(&csi2->subdev, cfg, pad);
-	else
-		return &csi2->formats[pad];
-}
-
-/*
- * csi2_enum_mbus_code - Handle pixel format enumeration
- * @sd     : pointer to v4l2 subdev structure
- * @fh     : V4L2 subdev file handle
- * @code   : pointer to v4l2_subdev_pad_mbus_code_enum structure
- * return -EINVAL or zero on success
-*/
-static int csi2_enum_mbus_code(struct v4l2_subdev *sd,
-			       struct v4l2_subdev_pad_config *cfg,
-			       struct v4l2_subdev_mbus_code_enum *code)
-{
-	const struct atomisp_in_fmt_conv *ic = atomisp_in_fmt_conv;
-	unsigned int i = 0;
-
-	while (ic->code) {
-		if (i == code->index) {
-			code->code = ic->code;
-			return 0;
-		}
-		i++, ic++;
-	}
-
-	return -EINVAL;
-}
-
-/*
- * csi2_get_format - Handle get format by pads subdev method
- * @sd : pointer to v4l2 subdev structure
- * @fh : V4L2 subdev file handle
- * @pad: pad num
- * @fmt: pointer to v4l2 format structure
- * return -EINVAL or zero on sucess
-*/
-static int csi2_get_format(struct v4l2_subdev *sd,
-			   struct v4l2_subdev_pad_config *cfg,
-			   struct v4l2_subdev_format *fmt)
-{
-	struct atomisp_mipi_csi2_device *csi2 = v4l2_get_subdevdata(sd);
-	struct v4l2_mbus_framefmt *format;
-
-	format = __csi2_get_format(csi2, cfg, fmt->which, fmt->pad);
-
-	fmt->format = *format;
-
-	return 0;
-}
-
-int atomisp_csi2_set_ffmt(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_pad_config *cfg,
-			  unsigned int which, uint16_t pad,
-			  struct v4l2_mbus_framefmt *ffmt)
-{
-	struct atomisp_mipi_csi2_device *csi2 = v4l2_get_subdevdata(sd);
-	struct v4l2_mbus_framefmt *actual_ffmt =
-#ifndef ISP2401
-		__csi2_get_format(csi2, cfg, which, pad);
-#else
-	    __csi2_get_format(csi2, cfg, which, pad);
-#endif
-
-	if (pad == CSI2_PAD_SINK) {
-		const struct atomisp_in_fmt_conv *ic;
-		struct v4l2_mbus_framefmt tmp_ffmt;
-
-		ic = atomisp_find_in_fmt_conv(ffmt->code);
-		if (ic)
-			actual_ffmt->code = ic->code;
-		else
-			actual_ffmt->code = atomisp_in_fmt_conv[0].code;
-
-		actual_ffmt->width = clamp_t(
-			u32, ffmt->width, ATOM_ISP_MIN_WIDTH,
-			ATOM_ISP_MAX_WIDTH);
-		actual_ffmt->height = clamp_t(
-			u32, ffmt->height, ATOM_ISP_MIN_HEIGHT,
-			ATOM_ISP_MAX_HEIGHT);
-
-		tmp_ffmt = *ffmt = *actual_ffmt;
-
-		return atomisp_csi2_set_ffmt(sd, cfg, which, CSI2_PAD_SOURCE,
-					     &tmp_ffmt);
-	}
-
-	/* FIXME: DPCM decompression */
-	*actual_ffmt = *ffmt =
-#ifndef ISP2401
-		*__csi2_get_format(csi2, cfg, which, CSI2_PAD_SINK);
-#else
-	    *__csi2_get_format(csi2, cfg, which, CSI2_PAD_SINK);
-#endif
-
-	return 0;
-}
-
-/*
- * csi2_set_format - Handle set format by pads subdev method
- * @sd : pointer to v4l2 subdev structure
- * @fh : V4L2 subdev file handle
- * @pad: pad num
- * @fmt: pointer to v4l2 format structure
- * return -EINVAL or zero on success
-*/
-static int csi2_set_format(struct v4l2_subdev *sd,
-			   struct v4l2_subdev_pad_config *cfg,
-			   struct v4l2_subdev_format *fmt)
-{
-	return atomisp_csi2_set_ffmt(sd, cfg, fmt->which, fmt->pad,
-				     &fmt->format);
-}
-
-/*
- * csi2_set_stream - Enable/Disable streaming on the CSI2 module
- * @sd: ISP CSI2 V4L2 subdevice
- * @enable: Enable/disable stream (1/0)
- *
- * Return 0 on success or a negative error code otherwise.
-*/
-static int csi2_set_stream(struct v4l2_subdev *sd, int enable)
-{
-	 return 0;
-}
-
-/* subdev core operations */
-static const struct v4l2_subdev_core_ops csi2_core_ops = {
-};
-
-/* subdev video operations */
-static const struct v4l2_subdev_video_ops csi2_video_ops = {
-	.s_stream = csi2_set_stream,
-};
-
-/* subdev pad operations */
-static const struct v4l2_subdev_pad_ops csi2_pad_ops = {
-	.enum_mbus_code = csi2_enum_mbus_code,
-	.get_fmt = csi2_get_format,
-	.set_fmt = csi2_set_format,
-	.link_validate = v4l2_subdev_link_validate_default,
-};
-
-/* subdev operations */
-static const struct v4l2_subdev_ops csi2_ops = {
-	.core = &csi2_core_ops,
-	.video = &csi2_video_ops,
-	.pad = &csi2_pad_ops,
-};
-
-#ifndef ISP2401
-
-#endif
-/*
- * csi2_link_setup - Setup CSI2 connections.
- * @entity : Pointer to media entity structure
- * @local  : Pointer to local pad array
- * @remote : Pointer to remote pad array
- * @flags  : Link flags
- * return -EINVAL or zero on success
-*/
-static int csi2_link_setup(struct media_entity *entity,
-	    const struct media_pad *local,
-	    const struct media_pad *remote, u32 flags)
-{
-	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(entity);
-	struct atomisp_mipi_csi2_device *csi2 = v4l2_get_subdevdata(sd);
-	u32 result = local->index | is_media_entity_v4l2_subdev(remote->entity);
-
-	switch (result) {
-	case CSI2_PAD_SOURCE | MEDIA_ENT_F_OLD_BASE:
-		/* not supported yet */
-		return -EINVAL;
-
-	case CSI2_PAD_SOURCE | MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN:
-		if (flags & MEDIA_LNK_FL_ENABLED) {
-			if (csi2->output & ~CSI2_OUTPUT_ISP_SUBDEV)
-				return -EBUSY;
-			csi2->output |= CSI2_OUTPUT_ISP_SUBDEV;
-		} else {
-			csi2->output &= ~CSI2_OUTPUT_ISP_SUBDEV;
-		}
-		break;
-
-	default:
-		/* Link from camera to CSI2 is fixed... */
-		return -EINVAL;
-	}
-	return 0;
-}
-
-/* media operations */
-static const struct media_entity_operations csi2_media_ops = {
-	.link_setup = csi2_link_setup,
-	.link_validate = v4l2_subdev_link_validate,
-};
-
-/*
-* ispcsi2_init_entities - Initialize subdev and media entity.
-* @csi2: Pointer to ispcsi2 structure.
-* return -ENOMEM or zero on success
-*/
-static int mipi_csi2_init_entities(struct atomisp_mipi_csi2_device *csi2,
-					int port)
-{
-	struct v4l2_subdev *sd = &csi2->subdev;
-	struct media_pad *pads = csi2->pads;
-	struct media_entity *me = &sd->entity;
-	int ret;
-
-	v4l2_subdev_init(sd, &csi2_ops);
-	snprintf(sd->name, sizeof(sd->name), "ATOM ISP CSI2-port%d", port);
-
-	v4l2_set_subdevdata(sd, csi2);
-	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-
-	pads[CSI2_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
-	pads[CSI2_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
-
-	me->ops = &csi2_media_ops;
-	me->function = MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN;
-	ret = media_entity_pads_init(me, CSI2_PADS_NUM, pads);
-	if (ret < 0)
-		return ret;
-
-	csi2->formats[CSI2_PAD_SINK].code =
-		csi2->formats[CSI2_PAD_SOURCE].code =
-		atomisp_in_fmt_conv[0].code;
-
-	return 0;
-}
-
-void
-atomisp_mipi_csi2_unregister_entities(struct atomisp_mipi_csi2_device *csi2)
-{
-	media_entity_cleanup(&csi2->subdev.entity);
-	v4l2_device_unregister_subdev(&csi2->subdev);
-}
-
-int atomisp_mipi_csi2_register_entities(struct atomisp_mipi_csi2_device *csi2,
-			struct v4l2_device *vdev)
-{
-	int ret;
-
-	/* Register the subdev and video nodes. */
-	ret = v4l2_device_register_subdev(vdev, &csi2->subdev);
-	if (ret < 0)
-		goto error;
-
-	return 0;
-
-error:
-	atomisp_mipi_csi2_unregister_entities(csi2);
-	return ret;
-}
-
-static const int LIMIT_SHIFT = 6;	/* Limit numeric range into 31 bits */
-
-static int
-atomisp_csi2_configure_calc(const short int coeffs[2], int mipi_freq, int def)
-{
-	/* Delay counter accuracy, 1/0.0625 for ANN/CHT, 1/0.125 for BXT */
-	static const int accinv = 16;		/* 1 / COUNT_ACC */
-	int r;
-
-	if (mipi_freq >> LIMIT_SHIFT <= 0)
-		return def;
-
-	r = accinv * coeffs[1] * (500000000 >> LIMIT_SHIFT);
-	r /= mipi_freq >> LIMIT_SHIFT;
-	r += accinv * coeffs[0];
-
-	return r;
-}
-
-static void atomisp_csi2_configure_isp2401(struct atomisp_sub_device *asd)
-{
-	/*
-	 * The ISP2401 new input system CSI2+ receiver has several
-	 * parameters affecting the receiver timings. These depend
-	 * on the MIPI bus frequency F in Hz (sensor transmitter rate)
-	 * as follows:
-	 *	register value = (A/1e9 + B * UI) / COUNT_ACC
-	 * where
-	 *	UI = 1 / (2 * F) in seconds
-	 *	COUNT_ACC = counter accuracy in seconds
-	 *	For ANN and CHV, COUNT_ACC = 0.0625 ns
-	 *	For BXT,  COUNT_ACC = 0.125 ns
-	 * A and B are coefficients from the table below,
-	 * depending whether the register minimum or maximum value is
-	 * calculated.
-	 *				       Minimum     Maximum
-	 * Clock lane			       A     B     A     B
-	 * reg_rx_csi_dly_cnt_termen_clane     0     0    38     0
-	 * reg_rx_csi_dly_cnt_settle_clane    95    -8   300   -16
-	 * Data lanes
-	 * reg_rx_csi_dly_cnt_termen_dlane0    0     0    35     4
-	 * reg_rx_csi_dly_cnt_settle_dlane0   85    -2   145    -6
-	 * reg_rx_csi_dly_cnt_termen_dlane1    0     0    35     4
-	 * reg_rx_csi_dly_cnt_settle_dlane1   85    -2   145    -6
-	 * reg_rx_csi_dly_cnt_termen_dlane2    0     0    35     4
-	 * reg_rx_csi_dly_cnt_settle_dlane2   85    -2   145    -6
-	 * reg_rx_csi_dly_cnt_termen_dlane3    0     0    35     4
-	 * reg_rx_csi_dly_cnt_settle_dlane3   85    -2   145    -6
-	 *
-	 * We use the minimum values in the calculations below.
-	 */
-	static const short int coeff_clk_termen[] = { 0, 0 };
-	static const short int coeff_clk_settle[] = { 95, -8 };
-	static const short int coeff_dat_termen[] = { 0, 0 };
-	static const short int coeff_dat_settle[] = { 85, -2 };
-	static const int TERMEN_DEFAULT		  = 0 * 0;
-	static const int SETTLE_DEFAULT		  = 0x480;
-	static const hrt_address csi2_port_base[] = {
-		[ATOMISP_CAMERA_PORT_PRIMARY]     = CSI2_PORT_A_BASE,
-		[ATOMISP_CAMERA_PORT_SECONDARY]   = CSI2_PORT_B_BASE,
-		[ATOMISP_CAMERA_PORT_TERTIARY]    = CSI2_PORT_C_BASE,
-	};
-	/* Number of lanes on each port, excluding clock lane */
-	static const unsigned char csi2_port_lanes[] = {
-		[ATOMISP_CAMERA_PORT_PRIMARY]     = 4,
-		[ATOMISP_CAMERA_PORT_SECONDARY]   = 2,
-		[ATOMISP_CAMERA_PORT_TERTIARY]    = 2,
-	};
-	static const hrt_address csi2_lane_base[] = {
-		CSI2_LANE_CL_BASE,
-		CSI2_LANE_D0_BASE,
-		CSI2_LANE_D1_BASE,
-		CSI2_LANE_D2_BASE,
-		CSI2_LANE_D3_BASE,
-	};
-
-	int clk_termen;
-	int clk_settle;
-	int dat_termen;
-	int dat_settle;
-
-	struct v4l2_control ctrl;
-	struct atomisp_device *isp = asd->isp;
-	struct camera_mipi_info *mipi_info;
-	int mipi_freq = 0;
-	enum atomisp_camera_port port;
-
-	int n;
-
-	mipi_info = atomisp_to_sensor_mipi_info(
-					isp->inputs[asd->input_curr].camera);
-	port = mipi_info->port;
-
-	ctrl.id = V4L2_CID_LINK_FREQ;
-	if (v4l2_g_ctrl
-	    (isp->inputs[asd->input_curr].camera->ctrl_handler, &ctrl) == 0)
-		mipi_freq = ctrl.value;
-
-	clk_termen = atomisp_csi2_configure_calc(coeff_clk_termen,
-						 mipi_freq, TERMEN_DEFAULT);
-	clk_settle = atomisp_csi2_configure_calc(coeff_clk_settle,
-						 mipi_freq, SETTLE_DEFAULT);
-	dat_termen = atomisp_csi2_configure_calc(coeff_dat_termen,
-						 mipi_freq, TERMEN_DEFAULT);
-	dat_settle = atomisp_csi2_configure_calc(coeff_dat_settle,
-						 mipi_freq, SETTLE_DEFAULT);
-	for (n = 0; n < csi2_port_lanes[port] + 1; n++) {
-		hrt_address base = csi2_port_base[port] + csi2_lane_base[n];
-		atomisp_store_uint32(base + CSI2_REG_RX_CSI_DLY_CNT_TERMEN,
-				     n == 0 ? clk_termen : dat_termen);
-		atomisp_store_uint32(base + CSI2_REG_RX_CSI_DLY_CNT_SETTLE,
-				     n == 0 ? clk_settle : dat_settle);
-	}
-}
-
-void atomisp_csi2_configure(struct atomisp_sub_device *asd)
-{
-	if (IS_HWREVISION(asd->isp, ATOMISP_HW_REVISION_ISP2401))
-		atomisp_csi2_configure_isp2401(asd);
-}
-
-/*
- * atomisp_mipi_csi2_cleanup - Routine for module driver cleanup
-*/
-void atomisp_mipi_csi2_cleanup(struct atomisp_device *isp)
-{
-}
-
-#ifndef ISP2401
-
-#endif
-int atomisp_mipi_csi2_init(struct atomisp_device *isp)
-{
-	struct atomisp_mipi_csi2_device *csi2_port;
-	unsigned int i;
-	int ret;
-
-	for (i = 0; i < ATOMISP_CAMERA_NR_PORTS; i++) {
-		csi2_port = &isp->csi2_port[i];
-		csi2_port->isp = isp;
-		ret = mipi_csi2_init_entities(csi2_port, i);
-		if (ret < 0)
-			goto fail;
-	}
-
-	return 0;
-
-fail:
-	atomisp_mipi_csi2_cleanup(isp);
-	return ret;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_csi2.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_csi2.h
deleted file mode 100644
index 0191d28a55bc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_csi2.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef __ATOMISP_CSI2_H__
-#define __ATOMISP_CSI2_H__
-
-#include <media/v4l2-subdev.h>
-#include <media/v4l2-ctrls.h>
-
-#define CSI2_PAD_SINK		0
-#define CSI2_PAD_SOURCE		1
-#define CSI2_PADS_NUM		2
-
-#define CSI2_OUTPUT_ISP_SUBDEV	(1 << 0)
-#define CSI2_OUTPUT_MEMORY	(1 << 1)
-
-struct atomisp_device;
-struct v4l2_device;
-struct atomisp_sub_device;
-
-struct atomisp_mipi_csi2_device {
-	struct v4l2_subdev subdev;
-	struct media_pad pads[CSI2_PADS_NUM];
-	struct v4l2_mbus_framefmt formats[CSI2_PADS_NUM];
-
-	struct v4l2_ctrl_handler ctrls;
-	struct atomisp_device *isp;
-
-	u32 output; /* output direction */
-};
-
-int atomisp_csi2_set_ffmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg,
-			  unsigned int which, uint16_t pad,
-			  struct v4l2_mbus_framefmt *ffmt);
-int atomisp_mipi_csi2_init(struct atomisp_device *isp);
-void atomisp_mipi_csi2_cleanup(struct atomisp_device *isp);
-void atomisp_mipi_csi2_unregister_entities(
-					struct atomisp_mipi_csi2_device *csi2);
-int atomisp_mipi_csi2_register_entities(struct atomisp_mipi_csi2_device *csi2,
-			struct v4l2_device *vdev);
-
-void atomisp_csi2_configure(struct atomisp_sub_device *asd);
-
-#endif /* __ATOMISP_CSI2_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_dfs_tables.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_dfs_tables.h
deleted file mode 100644
index 54e28605b5de..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_dfs_tables.h
+++ /dev/null
@@ -1,408 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef	__ATOMISP_DFS_TABLES_H__
-#define	__ATOMISP_DFS_TABLES_H__
-
-#include <linux/kernel.h>
-
-struct atomisp_freq_scaling_rule {
-	unsigned int width;
-	unsigned int height;
-	unsigned short fps;
-	unsigned int isp_freq;
-	unsigned int run_mode;
-};
-
-
-struct atomisp_dfs_config {
-	unsigned int lowest_freq;
-	unsigned int max_freq_at_vmin;
-	unsigned int highest_freq;
-	const struct atomisp_freq_scaling_rule *dfs_table;
-	unsigned int dfs_table_size;
-};
-
-static const struct atomisp_freq_scaling_rule dfs_rules_merr[] = {
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_STILL_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_PREVIEW,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_457MHZ,
-		.run_mode = ATOMISP_RUN_MODE_SDV,
-	},
-};
-
-/* Merrifield and Moorefield DFS rules */
-static const struct atomisp_dfs_config dfs_config_merr = {
-	.lowest_freq = ISP_FREQ_200MHZ,
-	.max_freq_at_vmin = ISP_FREQ_400MHZ,
-	.highest_freq = ISP_FREQ_457MHZ,
-	.dfs_table = dfs_rules_merr,
-	.dfs_table_size = ARRAY_SIZE(dfs_rules_merr),
-};
-
-static const struct atomisp_freq_scaling_rule dfs_rules_merr_1179[] = {
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_STILL_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_PREVIEW,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_SDV,
-	},
-};
-
-static const struct atomisp_dfs_config dfs_config_merr_1179 = {
-	.lowest_freq = ISP_FREQ_200MHZ,
-	.max_freq_at_vmin = ISP_FREQ_400MHZ,
-	.highest_freq = ISP_FREQ_400MHZ,
-	.dfs_table = dfs_rules_merr_1179,
-	.dfs_table_size = ARRAY_SIZE(dfs_rules_merr_1179),
-};
-
-static const struct atomisp_freq_scaling_rule dfs_rules_merr_117a[] = {
-	{
-		.width = 1920,
-		.height = 1080,
-		.fps = 30,
-		.isp_freq = ISP_FREQ_266MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = 1080,
-		.height = 1920,
-		.fps = 30,
-#ifndef ISP2401
-		.isp_freq = ISP_FREQ_266MHZ,
-#else
-		.isp_freq = ISP_FREQ_400MHZ,
-#endif
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = 1920,
-		.height = 1080,
-		.fps = 45,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = 1080,
-		.height = 1920,
-		.fps = 45,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = 60,
-		.isp_freq = ISP_FREQ_356MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_200MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_STILL_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_200MHZ,
-		.run_mode = ATOMISP_RUN_MODE_PREVIEW,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_SDV,
-	},
-};
-
-static const struct atomisp_dfs_config dfs_config_merr_117a = {
-	.lowest_freq = ISP_FREQ_200MHZ,
-	.max_freq_at_vmin = ISP_FREQ_200MHZ,
-	.highest_freq = ISP_FREQ_400MHZ,
-	.dfs_table = dfs_rules_merr_117a,
-	.dfs_table_size = ARRAY_SIZE(dfs_rules_merr_117a),
-};
-
-static const struct atomisp_freq_scaling_rule dfs_rules_byt[] = {
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_STILL_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_PREVIEW,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_400MHZ,
-		.run_mode = ATOMISP_RUN_MODE_SDV,
-	},
-};
-
-static const struct atomisp_dfs_config dfs_config_byt = {
-	.lowest_freq = ISP_FREQ_200MHZ,
-	.max_freq_at_vmin = ISP_FREQ_400MHZ,
-	.highest_freq = ISP_FREQ_400MHZ,
-	.dfs_table = dfs_rules_byt,
-	.dfs_table_size = ARRAY_SIZE(dfs_rules_byt),
-};
-
-static const struct atomisp_freq_scaling_rule dfs_rules_byt_cr[] = {
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_STILL_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_PREVIEW,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_SDV,
-	},
-};
-
-static const struct atomisp_dfs_config dfs_config_byt_cr = {
-	.lowest_freq = ISP_FREQ_200MHZ,
-	.max_freq_at_vmin = ISP_FREQ_320MHZ,
-	.highest_freq = ISP_FREQ_320MHZ,
-	.dfs_table = dfs_rules_byt_cr,
-	.dfs_table_size = ARRAY_SIZE(dfs_rules_byt_cr),
-};
-
-static const struct atomisp_freq_scaling_rule dfs_rules_cht[] = {
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_356MHZ,
-		.run_mode = ATOMISP_RUN_MODE_STILL_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_PREVIEW,
-	},
-	{
-		.width = 1280,
-		.height = 720,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_SDV,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_356MHZ,
-		.run_mode = ATOMISP_RUN_MODE_SDV,
-	},
-};
-
-static const struct atomisp_freq_scaling_rule dfs_rules_cht_soc[] = {
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_356MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_356MHZ,
-		.run_mode = ATOMISP_RUN_MODE_STILL_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_PREVIEW,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_356MHZ,
-		.run_mode = ATOMISP_RUN_MODE_SDV,
-	},
-};
-
-static const struct atomisp_dfs_config dfs_config_cht = {
-	.lowest_freq = ISP_FREQ_100MHZ,
-	.max_freq_at_vmin = ISP_FREQ_356MHZ,
-	.highest_freq = ISP_FREQ_356MHZ,
-	.dfs_table = dfs_rules_cht,
-	.dfs_table_size = ARRAY_SIZE(dfs_rules_cht),
-};
-
-static const struct atomisp_dfs_config dfs_config_cht_soc = {
-	.lowest_freq = ISP_FREQ_100MHZ,
-	.max_freq_at_vmin = ISP_FREQ_356MHZ,
-	.highest_freq = ISP_FREQ_356MHZ,
-	.dfs_table = dfs_rules_cht_soc,
-	.dfs_table_size = ARRAY_SIZE(dfs_rules_cht_soc),
-};
-
-#endif /* __ATOMISP_DFS_TABLES_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_drvfs.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_drvfs.c
deleted file mode 100644
index a815c768bda9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_drvfs.c
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Support for atomisp driver sysfs interface
- *
- * Copyright (c) 2014 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#include <linux/device.h>
-#include <linux/err.h>
-#include <linux/kernel.h>
-
-#include "atomisp_compat.h"
-#include "atomisp_internal.h"
-#include "atomisp_ioctl.h"
-#include "atomisp_drvfs.h"
-#include "hmm/hmm.h"
-
-/*
- * _iunit_debug:
- * dbglvl: iunit css driver trace level
- * dbgopt: iunit debug option:
- *        bit 0: binary list
- *        bit 1: running binary
- *        bit 2: memory statistic
-*/
-struct _iunit_debug {
-	struct device_driver	*drv;
-	struct atomisp_device	*isp;
-	unsigned int		dbglvl;
-	unsigned int		dbgfun;
-	unsigned int		dbgopt;
-};
-
-#define OPTION_BIN_LIST			(1<<0)
-#define OPTION_BIN_RUN			(1<<1)
-#define OPTION_MEM_STAT			(1<<2)
-#define OPTION_VALID			(OPTION_BIN_LIST \
-					| OPTION_BIN_RUN \
-					| OPTION_MEM_STAT)
-
-static struct _iunit_debug iunit_debug = {
-	.dbglvl = 0,
-	.dbgopt = OPTION_BIN_LIST,
-};
-
-static inline int iunit_dump_dbgopt(struct atomisp_device *isp,
-				unsigned int opt)
-{
-	int ret = 0;
-
-	if (opt & OPTION_VALID) {
-		if (opt & OPTION_BIN_LIST) {
-			ret = atomisp_css_dump_blob_infor();
-			if (ret) {
-				dev_err(atomisp_dev, "%s dump blob infor err[ret:%d]\n",
-					__func__, ret);
-				goto opt_err;
-			}
-		}
-
-		if (opt & OPTION_BIN_RUN) {
-			if (atomisp_streaming_count(isp)) {
-				atomisp_css_dump_sp_raw_copy_linecount(true);
-				atomisp_css_debug_dump_isp_binary();
-			} else {
-				ret = -EPERM;
-				dev_err(atomisp_dev, "%s dump running bin err[ret:%d]\n",
-					__func__, ret);
-				goto opt_err;
-			}
-		}
-
-		if (opt & OPTION_MEM_STAT)
-			hmm_show_mem_stat(__func__, __LINE__);
-	} else {
-		ret = -EINVAL;
-		dev_err(atomisp_dev, "%s dump nothing[ret=%d]\n", __func__,
-			ret);
-	}
-
-opt_err:
-	return ret;
-}
-
-static ssize_t iunit_dbglvl_show(struct device_driver *drv, char *buf)
-{
-	iunit_debug.dbglvl = atomisp_css_debug_get_dtrace_level();
-	return sprintf(buf, "dtrace level:%u\n", iunit_debug.dbglvl);
-}
-
-static ssize_t iunit_dbglvl_store(struct device_driver *drv, const char *buf,
-				size_t size)
-{
-	if (kstrtouint(buf, 10, &iunit_debug.dbglvl)
-		|| iunit_debug.dbglvl < 1
-		|| iunit_debug.dbglvl > 9) {
-		return -ERANGE;
-	}
-	atomisp_css_debug_set_dtrace_level(iunit_debug.dbglvl);
-
-	return size;
-}
-
-static ssize_t iunit_dbgfun_show(struct device_driver *drv, char *buf)
-{
-	iunit_debug.dbgfun = atomisp_get_css_dbgfunc();
-	return sprintf(buf, "dbgfun opt:%u\n", iunit_debug.dbgfun);
-}
-
-static ssize_t iunit_dbgfun_store(struct device_driver *drv, const char *buf,
-				size_t size)
-{
-	unsigned int opt;
-	int ret;
-
-	ret = kstrtouint(buf, 10, &opt);
-	if (ret)
-		return ret;
-
-	ret = atomisp_set_css_dbgfunc(iunit_debug.isp, opt);
-	if (ret)
-		return ret;
-
-	iunit_debug.dbgfun = opt;
-
-	return size;
-}
-
-static ssize_t iunit_dbgopt_show(struct device_driver *drv, char *buf)
-{
-	return sprintf(buf, "option:0x%x\n", iunit_debug.dbgopt);
-}
-
-static ssize_t iunit_dbgopt_store(struct device_driver *drv, const char *buf,
-				size_t size)
-{
-	unsigned int opt;
-	int ret;
-
-	ret = kstrtouint(buf, 10, &opt);
-	if (ret)
-		return ret;
-
-	iunit_debug.dbgopt = opt;
-	ret = iunit_dump_dbgopt(iunit_debug.isp, iunit_debug.dbgopt);
-	if (ret)
-		return ret;
-
-	return size;
-}
-
-static const struct driver_attribute iunit_drvfs_attrs[] = {
-	__ATTR(dbglvl, 0644, iunit_dbglvl_show, iunit_dbglvl_store),
-	__ATTR(dbgfun, 0644, iunit_dbgfun_show, iunit_dbgfun_store),
-	__ATTR(dbgopt, 0644, iunit_dbgopt_show, iunit_dbgopt_store),
-};
-
-static int iunit_drvfs_create_files(struct device_driver *drv)
-{
-	int i, ret = 0;
-
-	for (i = 0; i < ARRAY_SIZE(iunit_drvfs_attrs); i++)
-		ret |= driver_create_file(drv, &iunit_drvfs_attrs[i]);
-
-	return ret;
-}
-
-static void iunit_drvfs_remove_files(struct device_driver *drv)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(iunit_drvfs_attrs); i++)
-		driver_remove_file(drv, &iunit_drvfs_attrs[i]);
-}
-
-int atomisp_drvfs_init(struct device_driver *drv, struct atomisp_device *isp)
-{
-	int ret;
-
-	iunit_debug.isp = isp;
-	iunit_debug.drv = drv;
-
-	ret = iunit_drvfs_create_files(iunit_debug.drv);
-	if (ret) {
-		dev_err(atomisp_dev, "drvfs_create_files error: %d\n", ret);
-		iunit_drvfs_remove_files(iunit_debug.drv);
-	}
-
-	return ret;
-}
-
-void atomisp_drvfs_exit(void)
-{
-	iunit_drvfs_remove_files(iunit_debug.drv);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_drvfs.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_drvfs.h
deleted file mode 100644
index 7c99240d107a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_drvfs.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Support for atomisp driver sysfs interface.
- *
- * Copyright (c) 2014 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef	__ATOMISP_DRVFS_H__
-#define	__ATOMISP_DRVFS_H__
-
-int atomisp_drvfs_init(struct device_driver *drv, struct atomisp_device *isp);
-void atomisp_drvfs_exit(void);
-
-#endif /* __ATOMISP_DRVFS_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_file.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_file.c
deleted file mode 100644
index c6d96987561d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_file.c
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#include <media/v4l2-event.h>
-#include <media/v4l2-mediabus.h>
-
-#include <media/videobuf-vmalloc.h>
-#include <linux/delay.h>
-
-#include "ia_css.h"
-
-#include "atomisp_cmd.h"
-#include "atomisp_common.h"
-#include "atomisp_file.h"
-#include "atomisp_internal.h"
-#include "atomisp_ioctl.h"
-
-static void file_work(struct work_struct *work)
-{
-	struct atomisp_file_device *file_dev =
-			container_of(work, struct atomisp_file_device, work);
-	struct atomisp_device *isp = file_dev->isp;
-	/* only support file injection on subdev0 */
-	struct atomisp_sub_device *asd = &isp->asd[0];
-	struct atomisp_video_pipe *out_pipe = &asd->video_in;
-	unsigned short *buf = videobuf_to_vmalloc(out_pipe->outq.bufs[0]);
-	struct v4l2_mbus_framefmt isp_sink_fmt;
-
-	if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
-		return;
-
-	dev_dbg(isp->dev, ">%s: ready to start streaming\n", __func__);
-	isp_sink_fmt = *atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-						V4L2_SUBDEV_FORMAT_ACTIVE,
-						ATOMISP_SUBDEV_PAD_SINK);
-
-	while (!atomisp_css_isp_has_started())
-		usleep_range(1000, 1500);
-
-	atomisp_css_send_input_frame(asd, buf, isp_sink_fmt.width,
-				     isp_sink_fmt.height);
-	dev_dbg(isp->dev, "<%s: streaming done\n", __func__);
-}
-
-static int file_input_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	struct atomisp_file_device *file_dev = v4l2_get_subdevdata(sd);
-	struct atomisp_device *isp = file_dev->isp;
-	/* only support file injection on subdev0 */
-	struct atomisp_sub_device *asd = &isp->asd[0];
-
-	dev_dbg(isp->dev, "%s: enable %d\n", __func__, enable);
-	if (enable) {
-		if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
-			return 0;
-
-		queue_work(file_dev->work_queue, &file_dev->work);
-		return 0;
-	}
-	cancel_work_sync(&file_dev->work);
-	return 0;
-}
-
-static int file_input_get_fmt(struct v4l2_subdev *sd,
-			      struct v4l2_subdev_pad_config *cfg,
-			      struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct atomisp_file_device *file_dev = v4l2_get_subdevdata(sd);
-	struct atomisp_device *isp = file_dev->isp;
-	/* only support file injection on subdev0 */
-	struct atomisp_sub_device *asd = &isp->asd[0];
-	struct v4l2_mbus_framefmt *isp_sink_fmt;
-	if (format->pad)
-		return -EINVAL;
-	isp_sink_fmt = atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-					       V4L2_SUBDEV_FORMAT_ACTIVE,
-					       ATOMISP_SUBDEV_PAD_SINK);
-
-	fmt->width = isp_sink_fmt->width;
-	fmt->height = isp_sink_fmt->height;
-	fmt->code = isp_sink_fmt->code;
-
-	return 0;
-}
-
-static int file_input_set_fmt(struct v4l2_subdev *sd,
-			      struct v4l2_subdev_pad_config *cfg,
-			      struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	if (format->pad)
-		return -EINVAL;
-	file_input_get_fmt(sd, cfg, format);
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY)
-		cfg->try_fmt = *fmt;
-	return 0;
-}
-
-static int file_input_log_status(struct v4l2_subdev *sd)
-{
-	/*to fake*/
-	return 0;
-}
-
-static int file_input_s_power(struct v4l2_subdev *sd, int on)
-{
-	/* to fake */
-	return 0;
-}
-
-static int file_input_enum_mbus_code(struct v4l2_subdev *sd,
-				     struct v4l2_subdev_pad_config *cfg,
-				     struct v4l2_subdev_mbus_code_enum *code)
-{
-	/*to fake*/
-	return 0;
-}
-
-static int file_input_enum_frame_size(struct v4l2_subdev *sd,
-				      struct v4l2_subdev_pad_config *cfg,
-				      struct v4l2_subdev_frame_size_enum *fse)
-{
-	/*to fake*/
-	return 0;
-}
-
-static int file_input_enum_frame_ival(struct v4l2_subdev *sd,
-				      struct v4l2_subdev_pad_config *cfg,
-				      struct v4l2_subdev_frame_interval_enum
-				      *fie)
-{
-	/*to fake*/
-	return 0;
-}
-
-static const struct v4l2_subdev_video_ops file_input_video_ops = {
-	.s_stream = file_input_s_stream,
-};
-
-static const struct v4l2_subdev_core_ops file_input_core_ops = {
-	.log_status = file_input_log_status,
-	.s_power = file_input_s_power,
-};
-
-static const struct v4l2_subdev_pad_ops file_input_pad_ops = {
-	.enum_mbus_code = file_input_enum_mbus_code,
-	.enum_frame_size = file_input_enum_frame_size,
-	.enum_frame_interval = file_input_enum_frame_ival,
-	.get_fmt = file_input_get_fmt,
-	.set_fmt = file_input_set_fmt,
-};
-
-static const struct v4l2_subdev_ops file_input_ops = {
-	.core = &file_input_core_ops,
-	.video = &file_input_video_ops,
-	.pad = &file_input_pad_ops,
-};
-
-void
-atomisp_file_input_unregister_entities(struct atomisp_file_device *file_dev)
-{
-	media_entity_cleanup(&file_dev->sd.entity);
-	v4l2_device_unregister_subdev(&file_dev->sd);
-}
-
-int atomisp_file_input_register_entities(struct atomisp_file_device *file_dev,
-			struct v4l2_device *vdev)
-{
-	/* Register the subdev and video nodes. */
-	return  v4l2_device_register_subdev(vdev, &file_dev->sd);
-}
-
-void atomisp_file_input_cleanup(struct atomisp_device *isp)
-{
-	struct atomisp_file_device *file_dev = &isp->file_dev;
-
-	if (file_dev->work_queue) {
-		destroy_workqueue(file_dev->work_queue);
-		file_dev->work_queue = NULL;
-	}
-}
-
-int atomisp_file_input_init(struct atomisp_device *isp)
-{
-	struct atomisp_file_device *file_dev = &isp->file_dev;
-	struct v4l2_subdev *sd = &file_dev->sd;
-	struct media_pad *pads = file_dev->pads;
-	struct media_entity *me = &sd->entity;
-
-	file_dev->isp = isp;
-	file_dev->work_queue = alloc_workqueue(isp->v4l2_dev.name, 0, 1);
-	if (file_dev->work_queue == NULL) {
-		dev_err(isp->dev, "Failed to initialize file inject workq\n");
-		return -ENOMEM;
-	}
-
-	INIT_WORK(&file_dev->work, file_work);
-
-	v4l2_subdev_init(sd, &file_input_ops);
-	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	strcpy(sd->name, "file_input_subdev");
-	v4l2_set_subdevdata(sd, file_dev);
-
-	pads[0].flags = MEDIA_PAD_FL_SINK;
-	me->function = MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN;
-
-	return media_entity_pads_init(me, 1, pads);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_file.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_file.h
deleted file mode 100644
index 61fdeb5ee60a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_file.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __ATOMISP_FILE_H__
-#define __ATOMISP_FILE_H__
-
-#include <media/media-entity.h>
-#include <media/v4l2-subdev.h>
-
-struct atomisp_device;
-
-struct atomisp_file_device {
-	struct v4l2_subdev sd;
-	struct atomisp_device *isp;
-	struct media_pad pads[1];
-
-	struct workqueue_struct *work_queue;
-	struct work_struct work;
-};
-
-void atomisp_file_input_cleanup(struct atomisp_device *isp);
-int atomisp_file_input_init(struct atomisp_device *isp);
-void atomisp_file_input_unregister_entities(
-				struct atomisp_file_device *file_dev);
-int atomisp_file_input_register_entities(struct atomisp_file_device *file_dev,
-			struct v4l2_device *vdev);
-#endif /* __ATOMISP_FILE_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.c
deleted file mode 100644
index 709137f25700..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.c
+++ /dev/null
@@ -1,1304 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#include <linux/module.h>
-#include <linux/pm_runtime.h>
-
-#include <media/v4l2-ioctl.h>
-#include <media/videobuf-vmalloc.h>
-
-#include "atomisp_cmd.h"
-#include "atomisp_common.h"
-#include "atomisp_fops.h"
-#include "atomisp_internal.h"
-#include "atomisp_ioctl.h"
-#include "atomisp_compat.h"
-#include "atomisp_subdev.h"
-#include "atomisp_v4l2.h"
-#include "atomisp-regs.h"
-#include "hmm/hmm.h"
-
-#include "hrt/hive_isp_css_mm_hrt.h"
-
-#include "type_support.h"
-#include "device_access/device_access.h"
-#include "memory_access/memory_access.h"
-
-#include "atomisp_acc.h"
-
-#define ISP_LEFT_PAD			128	/* equal to 2*NWAY */
-
-/*
- * input image data, and current frame resolution for test
- */
-#define	ISP_PARAM_MMAP_OFFSET	0xfffff000
-
-#define MAGIC_CHECK(is, should)	\
-	do { \
-		if (unlikely((is) != (should))) { \
-			pr_err("magic mismatch: %x (expected %x)\n", \
-				is, should); \
-			BUG(); \
-		} \
-	} while (0)
-
-/*
- * Videobuf ops
- */
-static int atomisp_buf_setup(struct videobuf_queue *vq, unsigned int *count,
-			     unsigned int *size)
-{
-	struct atomisp_video_pipe *pipe = vq->priv_data;
-
-	*size = pipe->pix.sizeimage;
-
-	return 0;
-}
-
-static int atomisp_buf_prepare(struct videobuf_queue *vq,
-			       struct videobuf_buffer *vb,
-			       enum v4l2_field field)
-{
-	struct atomisp_video_pipe *pipe = vq->priv_data;
-
-	vb->size = pipe->pix.sizeimage;
-	vb->width = pipe->pix.width;
-	vb->height = pipe->pix.height;
-	vb->field = field;
-	vb->state = VIDEOBUF_PREPARED;
-
-	return 0;
-}
-
-static int atomisp_q_one_metadata_buffer(struct atomisp_sub_device *asd,
-		enum atomisp_input_stream_id stream_id,
-		enum atomisp_css_pipe_id css_pipe_id)
-{
-	struct atomisp_metadata_buf *metadata_buf;
-	enum atomisp_metadata_type md_type =
-			atomisp_get_metadata_type(asd, css_pipe_id);
-	struct list_head *metadata_list;
-
-	if (asd->metadata_bufs_in_css[stream_id][css_pipe_id] >=
-		ATOMISP_CSS_Q_DEPTH)
-		return 0; /* we have reached CSS queue depth */
-
-	if (!list_empty(&asd->metadata[md_type])) {
-		metadata_list = &asd->metadata[md_type];
-	} else if (!list_empty(&asd->metadata_ready[md_type])) {
-		metadata_list = &asd->metadata_ready[md_type];
-	} else {
-		dev_warn(asd->isp->dev, "%s: No metadata buffers available for type %d!\n",
-			__func__, md_type);
-		return -EINVAL;
-	}
-
-	metadata_buf = list_entry(metadata_list->next,
-				  struct atomisp_metadata_buf, list);
-	list_del_init(&metadata_buf->list);
-
-	if (atomisp_q_metadata_buffer_to_css(asd, metadata_buf,
-				stream_id, css_pipe_id)) {
-		list_add(&metadata_buf->list, metadata_list);
-		return -EINVAL;
-	} else {
-		list_add_tail(&metadata_buf->list,
-				&asd->metadata_in_css[md_type]);
-	}
-	asd->metadata_bufs_in_css[stream_id][css_pipe_id]++;
-
-	return 0;
-}
-
-static int atomisp_q_one_s3a_buffer(struct atomisp_sub_device *asd,
-				    enum atomisp_input_stream_id stream_id,
-				    enum atomisp_css_pipe_id css_pipe_id)
-{
-	struct atomisp_s3a_buf *s3a_buf;
-	struct list_head *s3a_list;
-	unsigned int exp_id;
-
-	if (asd->s3a_bufs_in_css[css_pipe_id] >= ATOMISP_CSS_Q_DEPTH)
-		return 0; /* we have reached CSS queue depth */
-
-	if (!list_empty(&asd->s3a_stats)) {
-		s3a_list = &asd->s3a_stats;
-	} else if (!list_empty(&asd->s3a_stats_ready)) {
-		s3a_list = &asd->s3a_stats_ready;
-	} else {
-		dev_warn(asd->isp->dev, "%s: No s3a buffers available!\n",
-			__func__);
-		return -EINVAL;
-	}
-
-	s3a_buf = list_entry(s3a_list->next, struct atomisp_s3a_buf, list);
-	list_del_init(&s3a_buf->list);
-	exp_id = s3a_buf->s3a_data->exp_id;
-
-	hmm_flush_vmap(s3a_buf->s3a_data->data_ptr);
-	if (atomisp_q_s3a_buffer_to_css(asd, s3a_buf,
-					stream_id, css_pipe_id)) {
-		/* got from head, so return back to the head */
-		list_add(&s3a_buf->list, s3a_list);
-		return -EINVAL;
-	} else {
-		list_add_tail(&s3a_buf->list, &asd->s3a_stats_in_css);
-		if (s3a_list == &asd->s3a_stats_ready)
-			dev_warn(asd->isp->dev, "%s: drop one s3a stat which has exp_id %d!\n",
-				__func__, exp_id);
-	}
-
-	asd->s3a_bufs_in_css[css_pipe_id]++;
-	return 0;
-}
-
-static int atomisp_q_one_dis_buffer(struct atomisp_sub_device *asd,
-				    enum atomisp_input_stream_id stream_id,
-				    enum atomisp_css_pipe_id css_pipe_id)
-{
-	struct atomisp_dis_buf *dis_buf;
-	unsigned long irqflags;
-
-	if (asd->dis_bufs_in_css >=  ATOMISP_CSS_Q_DEPTH)
-		return 0; /* we have reached CSS queue depth */
-
-	spin_lock_irqsave(&asd->dis_stats_lock, irqflags);
-	if (list_empty(&asd->dis_stats)) {
-		spin_unlock_irqrestore(&asd->dis_stats_lock, irqflags);
-		dev_warn(asd->isp->dev, "%s: No dis buffers available!\n",
-			__func__);
-		return -EINVAL;
-	}
-
-	dis_buf = list_entry(asd->dis_stats.prev,
-			struct atomisp_dis_buf, list);
-	list_del_init(&dis_buf->list);
-	spin_unlock_irqrestore(&asd->dis_stats_lock, irqflags);
-
-	hmm_flush_vmap(dis_buf->dis_data->data_ptr);
-	if (atomisp_q_dis_buffer_to_css(asd, dis_buf,
-					stream_id, css_pipe_id)) {
-		spin_lock_irqsave(&asd->dis_stats_lock, irqflags);
-		/* got from tail, so return back to the tail */
-		list_add_tail(&dis_buf->list, &asd->dis_stats);
-		spin_unlock_irqrestore(&asd->dis_stats_lock, irqflags);
-		return -EINVAL;
-	} else {
-		spin_lock_irqsave(&asd->dis_stats_lock, irqflags);
-		list_add_tail(&dis_buf->list, &asd->dis_stats_in_css);
-		spin_unlock_irqrestore(&asd->dis_stats_lock, irqflags);
-	}
-
-	asd->dis_bufs_in_css++;
-
-	return 0;
-}
-
-int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd,
-			     struct atomisp_video_pipe *pipe,
-			     enum atomisp_input_stream_id stream_id,
-			     enum atomisp_css_buffer_type css_buf_type,
-			     enum atomisp_css_pipe_id css_pipe_id)
-{
-	struct videobuf_vmalloc_memory *vm_mem;
-	struct atomisp_css_params_with_list *param;
-	struct atomisp_css_dvs_grid_info *dvs_grid =
-		 atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
-	unsigned long irqflags;
-	int err = 0;
-
-	while (pipe->buffers_in_css < ATOMISP_CSS_Q_DEPTH) {
-		struct videobuf_buffer *vb;
-
-		spin_lock_irqsave(&pipe->irq_lock, irqflags);
-		if (list_empty(&pipe->activeq)) {
-			spin_unlock_irqrestore(&pipe->irq_lock, irqflags);
-			return -EINVAL;
-		}
-		vb = list_entry(pipe->activeq.next,
-				struct videobuf_buffer, queue);
-		list_del_init(&vb->queue);
-		vb->state = VIDEOBUF_ACTIVE;
-		spin_unlock_irqrestore(&pipe->irq_lock, irqflags);
-
-		/*
-		 * If there is a per_frame setting to apply on the buffer,
-		 * do it before buffer en-queueing.
-		 */
-		vm_mem = vb->priv;
-
-		param = pipe->frame_params[vb->i];
-		if (param) {
-			atomisp_makeup_css_parameters(asd,
-					&asd->params.css_param.update_flag,
-					&param->params);
-			atomisp_apply_css_parameters(asd, &param->params);
-
-			if (param->params.update_flag.dz_config &&
-				asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO) {
-				err = atomisp_calculate_real_zoom_region(asd,
-					&param->params.dz_config, css_pipe_id);
-				if (!err)
-					atomisp_css_set_dz_config(asd,
-						&param->params.dz_config);
-			}
-			atomisp_css_set_isp_config_applied_frame(asd,
-						vm_mem->vaddr);
-			atomisp_css_update_isp_params_on_pipe(asd,
-				asd->stream_env[stream_id].pipes[css_pipe_id]);
-			asd->params.dvs_6axis = (struct atomisp_css_dvs_6axis *)
-				param->params.dvs_6axis;
-
-			/*
-			 * WORKAROUND:
-			 * Because the camera halv3 can't ensure to set zoom
-			 * region to per_frame setting and global setting at
-			 * same time and only set zoom region to pre_frame
-			 * setting now.so when the pre_frame setting inculde
-			 * zoom region,I will set it to global setting.
-			 */
-			if (param->params.update_flag.dz_config &&
-				asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO
-				&& !err) {
-				memcpy(&asd->params.css_param.dz_config,
-					&param->params.dz_config,
-					sizeof(struct ia_css_dz_config));
-				asd->params.css_param.update_flag.dz_config =
-					(struct atomisp_dz_config *)
-					&asd->params.css_param.dz_config;
-				asd->params.css_update_params_needed = true;
-			}
-		}
-		/* Enqueue buffer */
-		err = atomisp_q_video_buffer_to_css(asd, vm_mem, stream_id,
-						css_buf_type, css_pipe_id);
-		if (err) {
-			spin_lock_irqsave(&pipe->irq_lock, irqflags);
-			list_add_tail(&vb->queue, &pipe->activeq);
-			vb->state = VIDEOBUF_QUEUED;
-			spin_unlock_irqrestore(&pipe->irq_lock, irqflags);
-			dev_err(asd->isp->dev, "%s, css q fails: %d\n",
-					__func__, err);
-			return -EINVAL;
-		}
-		pipe->buffers_in_css++;
-
-		/* enqueue 3A/DIS/metadata buffers */
-		if (asd->params.curr_grid_info.s3a_grid.enable &&
-			css_pipe_id == asd->params.s3a_enabled_pipe &&
-			css_buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME)
-			atomisp_q_one_s3a_buffer(asd, stream_id,
-						css_pipe_id);
-
-		if (asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream_info.
-				metadata_info.size &&
-			css_buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME)
-			atomisp_q_one_metadata_buffer(asd, stream_id,
-						css_pipe_id);
-
-		if (dvs_grid && dvs_grid->enable &&
-			css_pipe_id == CSS_PIPE_ID_VIDEO &&
-			css_buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME)
-			atomisp_q_one_dis_buffer(asd, stream_id,
-						css_pipe_id);
-	}
-
-	return 0;
-}
-
-static int atomisp_get_css_buf_type(struct atomisp_sub_device *asd,
-				    enum atomisp_css_pipe_id pipe_id,
-				    uint16_t source_pad)
-{
-	if (ATOMISP_USE_YUVPP(asd)) {
-		/* when run ZSL case */
-		if (asd->continuous_mode->val &&
-			asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) {
-			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE)
-				return CSS_BUFFER_TYPE_OUTPUT_FRAME;
-			else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW)
-				return CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME;
-			else
-				return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
-		}
-
-		/*when run SDV case*/
-		if (asd->continuous_mode->val &&
-			asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
-			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE)
-				return CSS_BUFFER_TYPE_OUTPUT_FRAME;
-			else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW)
-				return CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME;
-			else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO)
-				return CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME;
-			else
-				return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
-		}
-
-		/*other case: default setting*/
-		if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE ||
-		    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO ||
-		    (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW &&
-		     asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO))
-			return CSS_BUFFER_TYPE_OUTPUT_FRAME;
-		else
-			return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
-	}
-
-	if (pipe_id == CSS_PIPE_ID_COPY ||
-	    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE ||
-	    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO ||
-	    (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW &&
-	     asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO))
-		return CSS_BUFFER_TYPE_OUTPUT_FRAME;
-	else
-		return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
-}
-
-static int atomisp_qbuffers_to_css_for_all_pipes(struct atomisp_sub_device *asd)
-{
-	enum atomisp_css_buffer_type buf_type;
-	enum atomisp_css_pipe_id css_capture_pipe_id = CSS_PIPE_ID_COPY;
-	enum atomisp_css_pipe_id css_preview_pipe_id = CSS_PIPE_ID_COPY;
-	enum atomisp_css_pipe_id css_video_pipe_id = CSS_PIPE_ID_COPY;
-	enum atomisp_input_stream_id input_stream_id;
-	struct atomisp_video_pipe *capture_pipe;
-	struct atomisp_video_pipe *preview_pipe;
-	struct atomisp_video_pipe *video_pipe;
-
-	capture_pipe = &asd->video_out_capture;
-	preview_pipe = &asd->video_out_preview;
-	video_pipe = &asd->video_out_video_capture;
-
-	buf_type = atomisp_get_css_buf_type(
-			asd, css_preview_pipe_id,
-			atomisp_subdev_source_pad(&preview_pipe->vdev));
-	input_stream_id = ATOMISP_INPUT_STREAM_PREVIEW;
-	atomisp_q_video_buffers_to_css(asd, preview_pipe,
-				       input_stream_id,
-				       buf_type, css_preview_pipe_id);
-
-	buf_type = atomisp_get_css_buf_type(asd, css_capture_pipe_id,
-			atomisp_subdev_source_pad(&capture_pipe->vdev));
-	input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
-	atomisp_q_video_buffers_to_css(asd, capture_pipe,
-					       input_stream_id,
-					       buf_type, css_capture_pipe_id);
-
-	buf_type = atomisp_get_css_buf_type(asd, css_video_pipe_id,
-			atomisp_subdev_source_pad(&video_pipe->vdev));
-	input_stream_id = ATOMISP_INPUT_STREAM_VIDEO;
-	atomisp_q_video_buffers_to_css(asd, video_pipe,
-					       input_stream_id,
-					       buf_type, css_video_pipe_id);
-	return 0;
-}
-
-
-/* queue all available buffers to css */
-int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
-{
-	enum atomisp_css_buffer_type buf_type;
-	enum atomisp_css_pipe_id css_capture_pipe_id = CSS_PIPE_ID_NUM;
-	enum atomisp_css_pipe_id css_preview_pipe_id = CSS_PIPE_ID_NUM;
-	enum atomisp_css_pipe_id css_video_pipe_id = CSS_PIPE_ID_NUM;
-	enum atomisp_input_stream_id input_stream_id;
-	struct atomisp_video_pipe *capture_pipe = NULL;
-	struct atomisp_video_pipe *vf_pipe = NULL;
-	struct atomisp_video_pipe *preview_pipe = NULL;
-	struct atomisp_video_pipe *video_pipe = NULL;
-	bool raw_mode = atomisp_is_mbuscode_raw(
-			    asd->fmt[asd->capture_pad].fmt.code);
-
-	if (asd->isp->inputs[asd->input_curr].camera_caps->
-	    sensor[asd->sensor_curr].stream_num == 2 &&
-	    !asd->yuvpp_mode)
-		return atomisp_qbuffers_to_css_for_all_pipes(asd);
-
-	if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) {
-		video_pipe = &asd->video_out_video_capture;
-		css_video_pipe_id = CSS_PIPE_ID_VIDEO;
-	} else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) {
-		preview_pipe = &asd->video_out_capture;
-		css_preview_pipe_id = CSS_PIPE_ID_CAPTURE;
-	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
-		if (asd->continuous_mode->val) {
-			capture_pipe = &asd->video_out_capture;
-			vf_pipe = &asd->video_out_vf;
-			css_capture_pipe_id = CSS_PIPE_ID_CAPTURE;
-		}
-		video_pipe = &asd->video_out_video_capture;
-		preview_pipe = &asd->video_out_preview;
-		css_video_pipe_id = CSS_PIPE_ID_VIDEO;
-		css_preview_pipe_id = CSS_PIPE_ID_VIDEO;
-	} else if (asd->continuous_mode->val) {
-		capture_pipe = &asd->video_out_capture;
-		vf_pipe = &asd->video_out_vf;
-		preview_pipe = &asd->video_out_preview;
-
-		css_preview_pipe_id = CSS_PIPE_ID_PREVIEW;
-		css_capture_pipe_id = CSS_PIPE_ID_CAPTURE;
-	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) {
-		preview_pipe = &asd->video_out_preview;
-		css_preview_pipe_id = CSS_PIPE_ID_PREVIEW;
-	} else {
-		/* ATOMISP_RUN_MODE_STILL_CAPTURE */
-		capture_pipe = &asd->video_out_capture;
-		if (!raw_mode)
-			vf_pipe = &asd->video_out_vf;
-		css_capture_pipe_id = CSS_PIPE_ID_CAPTURE;
-	}
-
-#ifdef ISP2401_NEW_INPUT_SYSTEM
-	if (asd->copy_mode) {
-		css_capture_pipe_id = CSS_PIPE_ID_COPY;
-		css_preview_pipe_id = CSS_PIPE_ID_COPY;
-		css_video_pipe_id = CSS_PIPE_ID_COPY;
-	}
-#endif
-
-	if (asd->yuvpp_mode) {
-		capture_pipe = &asd->video_out_capture;
-		video_pipe   = &asd->video_out_video_capture;
-		preview_pipe = &asd->video_out_preview;
-		css_capture_pipe_id = CSS_PIPE_ID_COPY;
-		css_video_pipe_id   = CSS_PIPE_ID_YUVPP;
-		css_preview_pipe_id = CSS_PIPE_ID_YUVPP;
-	}
-
-	if (capture_pipe) {
-		buf_type = atomisp_get_css_buf_type(
-			asd, css_capture_pipe_id,
-			atomisp_subdev_source_pad(&capture_pipe->vdev));
-		input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
-
-		/*
-		 * use yuvpp pipe for SOC camera.
-		 */
-		if (ATOMISP_USE_YUVPP(asd))
-			css_capture_pipe_id = CSS_PIPE_ID_YUVPP;
-
-		atomisp_q_video_buffers_to_css(asd, capture_pipe,
-					       input_stream_id,
-					       buf_type, css_capture_pipe_id);
-	}
-
-	if (vf_pipe) {
-		buf_type = atomisp_get_css_buf_type(
-			asd, css_capture_pipe_id,
-			atomisp_subdev_source_pad(&vf_pipe->vdev));
-		if (asd->stream_env[ATOMISP_INPUT_STREAM_POSTVIEW].stream)
-			input_stream_id = ATOMISP_INPUT_STREAM_POSTVIEW;
-		else
-			input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
-
-		/*
-		 * use yuvpp pipe for SOC camera.
-		 */
-		if (ATOMISP_USE_YUVPP(asd))
-			css_capture_pipe_id = CSS_PIPE_ID_YUVPP;
-		atomisp_q_video_buffers_to_css(asd, vf_pipe,
-					       input_stream_id,
-					       buf_type, css_capture_pipe_id);
-	}
-
-	if (preview_pipe) {
-		buf_type = atomisp_get_css_buf_type(
-			asd, css_preview_pipe_id,
-			atomisp_subdev_source_pad(&preview_pipe->vdev));
-		if (ATOMISP_SOC_CAMERA(asd) && css_preview_pipe_id == CSS_PIPE_ID_YUVPP)
-			input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
-		 /* else for ext isp use case */
-		else if (css_preview_pipe_id == CSS_PIPE_ID_YUVPP)
-			input_stream_id = ATOMISP_INPUT_STREAM_VIDEO;
-		else if (asd->stream_env[ATOMISP_INPUT_STREAM_PREVIEW].stream)
-			input_stream_id = ATOMISP_INPUT_STREAM_PREVIEW;
-		else
-			input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
-
-		/*
-		 * use yuvpp pipe for SOC camera.
-		 */
-		if (ATOMISP_USE_YUVPP(asd))
-			css_preview_pipe_id = CSS_PIPE_ID_YUVPP;
-
-		atomisp_q_video_buffers_to_css(asd, preview_pipe,
-					       input_stream_id,
-					       buf_type, css_preview_pipe_id);
-	}
-
-	if (video_pipe) {
-		buf_type = atomisp_get_css_buf_type(
-			asd, css_video_pipe_id,
-			atomisp_subdev_source_pad(&video_pipe->vdev));
-		if (asd->stream_env[ATOMISP_INPUT_STREAM_VIDEO].stream)
-			input_stream_id = ATOMISP_INPUT_STREAM_VIDEO;
-		else
-			input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
-
-		/*
-		 * use yuvpp pipe for SOC camera.
-		 */
-		if (ATOMISP_USE_YUVPP(asd))
-			css_video_pipe_id = CSS_PIPE_ID_YUVPP;
-
-		atomisp_q_video_buffers_to_css(asd, video_pipe,
-					       input_stream_id,
-					       buf_type, css_video_pipe_id);
-	}
-
-	return 0;
-}
-
-static void atomisp_buf_queue(struct videobuf_queue *vq,
-			      struct videobuf_buffer *vb)
-{
-	struct atomisp_video_pipe *pipe = vq->priv_data;
-
-	/*
-	 * when a frame buffer meets following conditions, it should be put into
-	 * the waiting list:
-	 * 1.  It is not a main output frame, and it has a per-frame parameter
-	 *     to go with it.
-	 * 2.  It is not a main output frame, and the waiting buffer list is not
-	 *     empty, to keep the FIFO sequence of frame buffer processing, it
-	 *     is put to waiting list until previous per-frame parameter buffers
-	 *     get enqueued.
-	 */
-	if (!atomisp_is_vf_pipe(pipe) &&
-	    (pipe->frame_request_config_id[vb->i] ||
-	     !list_empty(&pipe->buffers_waiting_for_param)))
-		list_add_tail(&vb->queue, &pipe->buffers_waiting_for_param);
-	else
-		list_add_tail(&vb->queue, &pipe->activeq);
-
-	vb->state = VIDEOBUF_QUEUED;
-}
-
-static void atomisp_buf_release(struct videobuf_queue *vq,
-				struct videobuf_buffer *vb)
-{
-	vb->state = VIDEOBUF_NEEDS_INIT;
-	atomisp_videobuf_free_buf(vb);
-}
-
-static int atomisp_buf_setup_output(struct videobuf_queue *vq,
-				    unsigned int *count, unsigned int *size)
-{
-	struct atomisp_video_pipe *pipe = vq->priv_data;
-
-	*size = pipe->pix.sizeimage;
-
-	return 0;
-}
-
-static int atomisp_buf_prepare_output(struct videobuf_queue *vq,
-				      struct videobuf_buffer *vb,
-				      enum v4l2_field field)
-{
-	struct atomisp_video_pipe *pipe = vq->priv_data;
-
-	vb->size = pipe->pix.sizeimage;
-	vb->width = pipe->pix.width;
-	vb->height = pipe->pix.height;
-	vb->field = field;
-	vb->state = VIDEOBUF_PREPARED;
-
-	return 0;
-}
-
-static void atomisp_buf_queue_output(struct videobuf_queue *vq,
-				     struct videobuf_buffer *vb)
-{
-	struct atomisp_video_pipe *pipe = vq->priv_data;
-
-	list_add_tail(&vb->queue, &pipe->activeq_out);
-	vb->state = VIDEOBUF_QUEUED;
-}
-
-static void atomisp_buf_release_output(struct videobuf_queue *vq,
-				       struct videobuf_buffer *vb)
-{
-	videobuf_vmalloc_free(vb);
-	vb->state = VIDEOBUF_NEEDS_INIT;
-}
-
-static const struct videobuf_queue_ops videobuf_qops = {
-	.buf_setup	= atomisp_buf_setup,
-	.buf_prepare	= atomisp_buf_prepare,
-	.buf_queue	= atomisp_buf_queue,
-	.buf_release	= atomisp_buf_release,
-};
-
-static const struct videobuf_queue_ops videobuf_qops_output = {
-	.buf_setup	= atomisp_buf_setup_output,
-	.buf_prepare	= atomisp_buf_prepare_output,
-	.buf_queue	= atomisp_buf_queue_output,
-	.buf_release	= atomisp_buf_release_output,
-};
-
-static int atomisp_init_pipe(struct atomisp_video_pipe *pipe)
-{
-	/* init locks */
-	spin_lock_init(&pipe->irq_lock);
-
-	videobuf_queue_vmalloc_init(&pipe->capq, &videobuf_qops, NULL,
-				    &pipe->irq_lock,
-				    V4L2_BUF_TYPE_VIDEO_CAPTURE,
-				    V4L2_FIELD_NONE,
-				    sizeof(struct atomisp_buffer), pipe,
-				    NULL);	/* ext_lock: NULL */
-
-	videobuf_queue_vmalloc_init(&pipe->outq, &videobuf_qops_output, NULL,
-				    &pipe->irq_lock,
-				    V4L2_BUF_TYPE_VIDEO_OUTPUT,
-				    V4L2_FIELD_NONE,
-				    sizeof(struct atomisp_buffer), pipe,
-				    NULL);	/* ext_lock: NULL */
-
-	INIT_LIST_HEAD(&pipe->activeq);
-	INIT_LIST_HEAD(&pipe->activeq_out);
-	INIT_LIST_HEAD(&pipe->buffers_waiting_for_param);
-	INIT_LIST_HEAD(&pipe->per_frame_params);
-	memset(pipe->frame_request_config_id, 0,
-		VIDEO_MAX_FRAME * sizeof(unsigned int));
-	memset(pipe->frame_params, 0,
-		VIDEO_MAX_FRAME *
-		sizeof(struct atomisp_css_params_with_list *));
-
-	return 0;
-}
-
-static void atomisp_dev_init_struct(struct atomisp_device *isp)
-{
-	unsigned int i;
-
-	isp->sw_contex.file_input = false;
-	isp->need_gfx_throttle = true;
-	isp->isp_fatal_error = false;
-	isp->mipi_frame_size = 0;
-
-	for (i = 0; i < isp->input_cnt; i++)
-		isp->inputs[i].asd = NULL;
-	/*
-	 * For Merrifield, frequency is scalable.
-	 * After boot-up, the default frequency is 200MHz.
-	 */
-	isp->sw_contex.running_freq = ISP_FREQ_200MHZ;
-}
-
-static void atomisp_subdev_init_struct(struct atomisp_sub_device *asd)
-{
-	v4l2_ctrl_s_ctrl(asd->run_mode, ATOMISP_RUN_MODE_STILL_CAPTURE);
-	memset(&asd->params.css_param, 0, sizeof(asd->params.css_param));
-	asd->params.color_effect = V4L2_COLORFX_NONE;
-	asd->params.bad_pixel_en = true;
-	asd->params.gdc_cac_en = false;
-	asd->params.video_dis_en = false;
-	asd->params.sc_en = false;
-	asd->params.fpn_en = false;
-	asd->params.xnr_en = false;
-	asd->params.false_color = 0;
-	asd->params.online_process = 1;
-	asd->params.yuv_ds_en = 0;
-	/* s3a grid not enabled for any pipe */
-	asd->params.s3a_enabled_pipe = CSS_PIPE_ID_NUM;
-
-	asd->params.offline_parm.num_captures = 1;
-	asd->params.offline_parm.skip_frames = 0;
-	asd->params.offline_parm.offset = 0;
-	asd->delayed_init = ATOMISP_DELAYED_INIT_NOT_QUEUED;
-	/* Add for channel */
-	asd->input_curr = 0;
-
-	asd->mipi_frame_size = 0;
-	asd->copy_mode = false;
-	asd->yuvpp_mode = false;
-
-	asd->stream_prepared = false;
-	asd->high_speed_mode = false;
-	asd->sensor_array_res.height = 0;
-	asd->sensor_array_res.width = 0;
-	atomisp_css_init_struct(asd);
-}
-/*
- * file operation functions
- */
-static unsigned int atomisp_subdev_users(struct atomisp_sub_device *asd)
-{
-	return asd->video_out_preview.users +
-	       asd->video_out_vf.users +
-	       asd->video_out_capture.users +
-	       asd->video_out_video_capture.users +
-	       asd->video_acc.users +
-	       asd->video_in.users;
-}
-
-unsigned int atomisp_dev_users(struct atomisp_device *isp)
-{
-	unsigned int i, sum;
-	for (i = 0, sum = 0; i < isp->num_of_streams; i++)
-		sum += atomisp_subdev_users(&isp->asd[i]);
-
-	return sum;
-}
-
-static int atomisp_open(struct file *file)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_video_pipe *pipe = NULL;
-	struct atomisp_acc_pipe *acc_pipe = NULL;
-	struct atomisp_sub_device *asd;
-	bool acc_node = false;
-	int ret;
-
-	dev_dbg(isp->dev, "open device %s\n", vdev->name);
-
-	rt_mutex_lock(&isp->mutex);
-
-	acc_node = !strncmp(vdev->name, "ATOMISP ISP ACC",
-			sizeof(vdev->name));
-	if (acc_node) {
-		acc_pipe = atomisp_to_acc_pipe(vdev);
-		asd = acc_pipe->asd;
-	} else {
-		pipe = atomisp_to_video_pipe(vdev);
-		asd = pipe->asd;
-	}
-	asd->subdev.devnode = vdev;
-	/* Deferred firmware loading case. */
-	if (isp->css_env.isp_css_fw.bytes == 0) {
-		isp->firmware = atomisp_load_firmware(isp);
-		if (!isp->firmware) {
-			dev_err(isp->dev, "Failed to load ISP firmware.\n");
-			ret = -ENOENT;
-			goto error;
-		}
-		ret = atomisp_css_load_firmware(isp);
-		if (ret) {
-			dev_err(isp->dev, "Failed to init css.\n");
-			goto error;
-		}
-		/* No need to keep FW in memory anymore. */
-		release_firmware(isp->firmware);
-		isp->firmware = NULL;
-		isp->css_env.isp_css_fw.data = NULL;
-	}
-
-	if (acc_node && acc_pipe->users) {
-		dev_dbg(isp->dev, "acc node already opened\n");
-		rt_mutex_unlock(&isp->mutex);
-		return -EBUSY;
-	} else if (acc_node) {
-		goto dev_init;
-	}
-
-	if (!isp->input_cnt) {
-		dev_err(isp->dev, "no camera attached\n");
-		ret = -EINVAL;
-		goto error;
-	}
-
-	/*
-	 * atomisp does not allow multiple open
-	 */
-	if (pipe->users) {
-		dev_dbg(isp->dev, "video node already opened\n");
-		rt_mutex_unlock(&isp->mutex);
-		return -EBUSY;
-	}
-
-	ret = atomisp_init_pipe(pipe);
-	if (ret)
-		goto error;
-
-dev_init:
-	if (atomisp_dev_users(isp)) {
-		dev_dbg(isp->dev, "skip init isp in open\n");
-		goto init_subdev;
-	}
-
-	/* runtime power management, turn on ISP */
-	ret = pm_runtime_get_sync(vdev->v4l2_dev->dev);
-	if (ret < 0) {
-		dev_err(isp->dev, "Failed to power on device\n");
-		goto error;
-	}
-
-	if (dypool_enable) {
-		ret = hmm_pool_register(dypool_pgnr, HMM_POOL_TYPE_DYNAMIC);
-		if (ret)
-			dev_err(isp->dev, "Failed to register dynamic memory pool.\n");
-	}
-
-	/* Init ISP */
-	if (atomisp_css_init(isp)) {
-		ret = -EINVAL;
-		/* Need to clean up CSS init if it fails. */
-		goto css_error;
-	}
-
-	atomisp_dev_init_struct(isp);
-
-	ret = v4l2_subdev_call(isp->flash, core, s_power, 1);
-	if (ret < 0 && ret != -ENODEV && ret != -ENOIOCTLCMD) {
-		dev_err(isp->dev, "Failed to power-on flash\n");
-		goto css_error;
-	}
-
-init_subdev:
-	if (atomisp_subdev_users(asd))
-		goto done;
-
-	atomisp_subdev_init_struct(asd);
-
-done:
-
-	if (acc_node)
-		acc_pipe->users++;
-	else
-		pipe->users++;
-	rt_mutex_unlock(&isp->mutex);
-	return 0;
-
-css_error:
-	atomisp_css_uninit(isp);
-error:
-	hmm_pool_unregister(HMM_POOL_TYPE_DYNAMIC);
-	pm_runtime_put(vdev->v4l2_dev->dev);
-	rt_mutex_unlock(&isp->mutex);
-	return ret;
-}
-
-static int atomisp_release(struct file *file)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_video_pipe *pipe;
-	struct atomisp_acc_pipe *acc_pipe;
-	struct atomisp_sub_device *asd;
-	bool acc_node;
-	struct v4l2_requestbuffers req;
-	struct v4l2_subdev_fh fh;
-	struct v4l2_rect clear_compose = {0};
-	int ret = 0;
-
-	v4l2_fh_init(&fh.vfh, vdev);
-
-	req.count = 0;
-	if (isp == NULL)
-		return -EBADF;
-
-	mutex_lock(&isp->streamoff_mutex);
-	rt_mutex_lock(&isp->mutex);
-
-	dev_dbg(isp->dev, "release device %s\n", vdev->name);
-	acc_node = !strncmp(vdev->name, "ATOMISP ISP ACC",
-			sizeof(vdev->name));
-	if (acc_node) {
-		acc_pipe = atomisp_to_acc_pipe(vdev);
-		asd = acc_pipe->asd;
-	} else {
-		pipe = atomisp_to_video_pipe(vdev);
-		asd = pipe->asd;
-	}
-	asd->subdev.devnode = vdev;
-	if (acc_node) {
-		acc_pipe->users--;
-		goto subdev_uninit;
-	}
-	pipe->users--;
-
-	if (pipe->capq.streaming)
-		dev_warn(isp->dev,
-				"%s: ISP still streaming while closing!",
-				__func__);
-
-	if (pipe->capq.streaming &&
-	    __atomisp_streamoff(file, NULL, V4L2_BUF_TYPE_VIDEO_CAPTURE)) {
-		dev_err(isp->dev,
-			"atomisp_streamoff failed on release, driver bug");
-		goto done;
-	}
-
-	if (pipe->users)
-		goto done;
-
-	if (__atomisp_reqbufs(file, NULL, &req)) {
-		dev_err(isp->dev,
-			"atomisp_reqbufs failed on release, driver bug");
-		goto done;
-	}
-
-	if (pipe->outq.bufs[0]) {
-		mutex_lock(&pipe->outq.vb_lock);
-		videobuf_queue_cancel(&pipe->outq);
-		mutex_unlock(&pipe->outq.vb_lock);
-	}
-
-	/*
-	 * A little trick here:
-	 * file injection input resolution is recorded in the sink pad,
-	 * therefore can not be cleared when releaseing one device node.
-	 * The sink pad setting can only be cleared when all device nodes
-	 * get released.
-	 */
-	if (!isp->sw_contex.file_input && asd->fmt_auto->val) {
-		struct v4l2_mbus_framefmt isp_sink_fmt = { 0 };
-		atomisp_subdev_set_ffmt(&asd->subdev, fh.pad,
-					V4L2_SUBDEV_FORMAT_ACTIVE,
-					ATOMISP_SUBDEV_PAD_SINK, &isp_sink_fmt);
-	}
-subdev_uninit:
-	if (atomisp_subdev_users(asd))
-		goto done;
-
-	/* clear the sink pad for file input */
-	if (isp->sw_contex.file_input && asd->fmt_auto->val) {
-		struct v4l2_mbus_framefmt isp_sink_fmt = { 0 };
-		atomisp_subdev_set_ffmt(&asd->subdev, fh.pad,
-					V4L2_SUBDEV_FORMAT_ACTIVE,
-					ATOMISP_SUBDEV_PAD_SINK, &isp_sink_fmt);
-	}
-
-	atomisp_css_free_stat_buffers(asd);
-	atomisp_free_internal_buffers(asd);
-	ret = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-				       core, s_power, 0);
-	if (ret)
-		dev_warn(isp->dev, "Failed to power-off sensor\n");
-
-	/* clear the asd field to show this camera is not used */
-	isp->inputs[asd->input_curr].asd = NULL;
-	asd->streaming = ATOMISP_DEVICE_STREAMING_DISABLED;
-
-	if (atomisp_dev_users(isp))
-		goto done;
-
-	atomisp_acc_release(asd);
-
-	atomisp_destroy_pipes_stream_force(asd);
-	atomisp_css_uninit(isp);
-
-	if (defer_fw_load) {
-		atomisp_css_unload_firmware(isp);
-		isp->css_env.isp_css_fw.data = NULL;
-		isp->css_env.isp_css_fw.bytes = 0;
-	}
-
-	hmm_pool_unregister(HMM_POOL_TYPE_DYNAMIC);
-
-	ret = v4l2_subdev_call(isp->flash, core, s_power, 0);
-	if (ret < 0 && ret != -ENODEV && ret != -ENOIOCTLCMD)
-		dev_warn(isp->dev, "Failed to power-off flash\n");
-
-	if (pm_runtime_put_sync(vdev->v4l2_dev->dev) < 0)
-		dev_err(isp->dev, "Failed to power off device\n");
-
-done:
-	if (!acc_node) {
-		atomisp_subdev_set_selection(&asd->subdev, fh.pad,
-				V4L2_SUBDEV_FORMAT_ACTIVE,
-				atomisp_subdev_source_pad(vdev),
-				V4L2_SEL_TGT_COMPOSE, 0,
-				&clear_compose);
-	}
-	rt_mutex_unlock(&isp->mutex);
-	mutex_unlock(&isp->streamoff_mutex);
-
-	return 0;
-}
-
-/*
- * Memory help functions for image frame and private parameters
- */
-static int do_isp_mm_remap(struct atomisp_device *isp,
-			   struct vm_area_struct *vma,
-			   ia_css_ptr isp_virt, u32 host_virt, u32 pgnr)
-{
-	u32 pfn;
-
-	while (pgnr) {
-		pfn = hmm_virt_to_phys(isp_virt) >> PAGE_SHIFT;
-		if (remap_pfn_range(vma, host_virt, pfn,
-				    PAGE_SIZE, PAGE_SHARED)) {
-			dev_err(isp->dev, "remap_pfn_range err.\n");
-			return -EAGAIN;
-		}
-
-		isp_virt += PAGE_SIZE;
-		host_virt += PAGE_SIZE;
-		pgnr--;
-	}
-
-	return 0;
-}
-
-static int frame_mmap(struct atomisp_device *isp,
-	const struct atomisp_css_frame *frame, struct vm_area_struct *vma)
-{
-	ia_css_ptr isp_virt;
-	u32 host_virt;
-	u32 pgnr;
-
-	if (!frame) {
-		dev_err(isp->dev, "%s: NULL frame pointer.\n", __func__);
-		return -EINVAL;
-	}
-
-	host_virt = vma->vm_start;
-	isp_virt = frame->data;
-	atomisp_get_frame_pgnr(isp, frame, &pgnr);
-
-	if (do_isp_mm_remap(isp, vma, isp_virt, host_virt, pgnr))
-		return -EAGAIN;
-
-	return 0;
-}
-
-int atomisp_videobuf_mmap_mapper(struct videobuf_queue *q,
-	struct vm_area_struct *vma)
-{
-	u32 offset = vma->vm_pgoff << PAGE_SHIFT;
-	int ret = -EINVAL, i;
-	struct atomisp_device *isp =
-		((struct atomisp_video_pipe *)(q->priv_data))->isp;
-	struct videobuf_vmalloc_memory *vm_mem;
-	struct videobuf_mapping *map;
-
-	MAGIC_CHECK(q->int_ops->magic, MAGIC_QTYPE_OPS);
-	if (!(vma->vm_flags & VM_WRITE) || !(vma->vm_flags & VM_SHARED)) {
-		dev_err(isp->dev, "map appl bug: PROT_WRITE and MAP_SHARED are required\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&q->vb_lock);
-	for (i = 0; i < VIDEO_MAX_FRAME; i++) {
-		struct videobuf_buffer *buf = q->bufs[i];
-		if (buf == NULL)
-			continue;
-
-		map = kzalloc(sizeof(struct videobuf_mapping), GFP_KERNEL);
-		if (!map) {
-			mutex_unlock(&q->vb_lock);
-			return -ENOMEM;
-		}
-
-		buf->map = map;
-		map->q = q;
-
-		buf->baddr = vma->vm_start;
-
-		if (buf && buf->memory == V4L2_MEMORY_MMAP &&
-		    buf->boff == offset) {
-			vm_mem = buf->priv;
-			ret = frame_mmap(isp, vm_mem->vaddr, vma);
-			vma->vm_flags |= VM_IO|VM_DONTEXPAND|VM_DONTDUMP;
-			break;
-		}
-	}
-	mutex_unlock(&q->vb_lock);
-
-	return ret;
-}
-
-/* The input frame contains left and right padding that need to be removed.
- * There is always ISP_LEFT_PAD padding on the left side.
- * There is also padding on the right (padded_width - width).
- */
-static int remove_pad_from_frame(struct atomisp_device *isp,
-		struct atomisp_css_frame *in_frame, __u32 width, __u32 height)
-{
-	unsigned int i;
-	unsigned short *buffer;
-	int ret = 0;
-	ia_css_ptr load = in_frame->data;
-	ia_css_ptr store = load;
-
-	buffer = kmalloc(width*sizeof(load), GFP_KERNEL);
-	if (!buffer)
-		return -ENOMEM;
-
-	load += ISP_LEFT_PAD;
-	for (i = 0; i < height; i++) {
-		ret = hmm_load(load, buffer, width*sizeof(load));
-		if (ret < 0)
-			goto remove_pad_error;
-
-		ret = hmm_store(store, buffer, width*sizeof(store));
-		if (ret < 0)
-			goto remove_pad_error;
-
-		load  += in_frame->info.padded_width;
-		store += width;
-	}
-
-remove_pad_error:
-	kfree(buffer);
-	return ret;
-}
-
-static int atomisp_mmap(struct file *file, struct vm_area_struct *vma)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-	struct atomisp_sub_device *asd = pipe->asd;
-	struct atomisp_css_frame *raw_virt_addr;
-	u32 start = vma->vm_start;
-	u32 end = vma->vm_end;
-	u32 size = end - start;
-	u32 origin_size, new_size;
-	int ret;
-
-	if (!(vma->vm_flags & (VM_WRITE | VM_READ)))
-		return -EACCES;
-
-	rt_mutex_lock(&isp->mutex);
-
-	if (!(vma->vm_flags & VM_SHARED)) {
-		/* Map private buffer.
-		 * Set VM_SHARED to the flags since we need
-		 * to map the buffer page by page.
-		 * Without VM_SHARED, remap_pfn_range() treats
-		 * this kind of mapping as invalid.
-		 */
-		vma->vm_flags |= VM_SHARED;
-		ret = hmm_mmap(vma, vma->vm_pgoff << PAGE_SHIFT);
-		rt_mutex_unlock(&isp->mutex);
-		return ret;
-	}
-
-	/* mmap for ISP offline raw data */
-	if (atomisp_subdev_source_pad(vdev)
-	    == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE &&
-	    vma->vm_pgoff == (ISP_PARAM_MMAP_OFFSET >> PAGE_SHIFT)) {
-		new_size = pipe->pix.width * pipe->pix.height * 2;
-		if (asd->params.online_process != 0) {
-			ret = -EINVAL;
-			goto error;
-		}
-		raw_virt_addr = asd->raw_output_frame;
-		if (raw_virt_addr == NULL) {
-			dev_err(isp->dev, "Failed to request RAW frame\n");
-			ret = -EINVAL;
-			goto error;
-		}
-
-		ret = remove_pad_from_frame(isp, raw_virt_addr,
-				      pipe->pix.width, pipe->pix.height);
-		if (ret < 0) {
-			dev_err(isp->dev, "remove pad failed.\n");
-			goto error;
-		}
-		origin_size = raw_virt_addr->data_bytes;
-		raw_virt_addr->data_bytes = new_size;
-
-		if (size != PAGE_ALIGN(new_size)) {
-			dev_err(isp->dev, "incorrect size for mmap ISP  Raw Frame\n");
-			ret = -EINVAL;
-			goto error;
-		}
-
-		if (frame_mmap(isp, raw_virt_addr, vma)) {
-			dev_err(isp->dev, "frame_mmap failed.\n");
-			raw_virt_addr->data_bytes = origin_size;
-			ret = -EAGAIN;
-			goto error;
-		}
-		raw_virt_addr->data_bytes = origin_size;
-		vma->vm_flags |= VM_IO|VM_DONTEXPAND|VM_DONTDUMP;
-		rt_mutex_unlock(&isp->mutex);
-		return 0;
-	}
-
-	/*
-	 * mmap for normal frames
-	 */
-	if (size != pipe->pix.sizeimage) {
-		dev_err(isp->dev, "incorrect size for mmap ISP frames\n");
-		ret = -EINVAL;
-		goto error;
-	}
-	rt_mutex_unlock(&isp->mutex);
-
-	return atomisp_videobuf_mmap_mapper(&pipe->capq, vma);
-
-error:
-	rt_mutex_unlock(&isp->mutex);
-
-	return ret;
-}
-
-static int atomisp_file_mmap(struct file *file, struct vm_area_struct *vma)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-
-	return videobuf_mmap_mapper(&pipe->outq, vma);
-}
-
-static __poll_t atomisp_poll(struct file *file,
-				 struct poll_table_struct *pt)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-
-	rt_mutex_lock(&isp->mutex);
-	if (pipe->capq.streaming != 1) {
-		rt_mutex_unlock(&isp->mutex);
-		return EPOLLERR;
-	}
-	rt_mutex_unlock(&isp->mutex);
-
-	return videobuf_poll_stream(file, &pipe->capq, pt);
-}
-
-const struct v4l2_file_operations atomisp_fops = {
-	.owner = THIS_MODULE,
-	.open = atomisp_open,
-	.release = atomisp_release,
-	.mmap = atomisp_mmap,
-	.unlocked_ioctl = video_ioctl2,
-#ifdef CONFIG_COMPAT
-	/*
-	 * There are problems with this code. Disable this for now.
-	.compat_ioctl32 = atomisp_compat_ioctl32,
-	 */
-#endif
-	.poll = atomisp_poll,
-};
-
-const struct v4l2_file_operations atomisp_file_fops = {
-	.owner = THIS_MODULE,
-	.open = atomisp_open,
-	.release = atomisp_release,
-	.mmap = atomisp_file_mmap,
-	.unlocked_ioctl = video_ioctl2,
-#ifdef CONFIG_COMPAT
-	/*
-	 * There are problems with this code. Disable this for now.
-	.compat_ioctl32 = atomisp_compat_ioctl32,
-	 */
-#endif
-	.poll = atomisp_poll,
-};
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.h
deleted file mode 100644
index 2faab3429d43..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef	__ATOMISP_FOPS_H__
-#define	__ATOMISP_FOPS_H__
-#include "atomisp_subdev.h"
-
-int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd,
-			     struct atomisp_video_pipe *pipe,
-			     enum atomisp_input_stream_id stream_id,
-			     enum atomisp_css_buffer_type css_buf_type,
-			     enum atomisp_css_pipe_id css_pipe_id);
-
-unsigned int atomisp_dev_users(struct atomisp_device *isp);
-unsigned int atomisp_sub_dev_users(struct atomisp_sub_device *asd);
-
-/*
- * Memory help functions for image frame and private parameters
- */
-
-int atomisp_videobuf_mmap_mapper(struct videobuf_queue *q,
-				     struct vm_area_struct *vma);
-
-int atomisp_qbuf_to_css(struct atomisp_device *isp,
-			struct atomisp_video_pipe *pipe,
-			struct videobuf_buffer *vb);
-
-int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd);
-
-extern const struct v4l2_file_operations atomisp_fops;
-
-extern bool defer_fw_load;
-
-#endif /* __ATOMISP_FOPS_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_helper.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_helper.h
deleted file mode 100644
index 55ba185b43a0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_helper.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef _atomisp_helper_h_
-#define _atomisp_helper_h_
-extern void __iomem *atomisp_io_base;
-
-static inline void __iomem *atomisp_get_io_virt_addr(unsigned int address)
-{
-	void __iomem *ret = atomisp_io_base + (address & 0x003FFFFF);
-	return ret;
-}
-#endif
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_internal.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_internal.h
deleted file mode 100644
index dc476a3dd271..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_internal.h
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef __ATOMISP_INTERNAL_H__
-#define __ATOMISP_INTERNAL_H__
-
-#include "../../include/linux/atomisp_platform.h"
-#include <linux/firmware.h>
-#include <linux/kernel.h>
-#include <linux/pm_qos.h>
-#include <linux/idr.h>
-
-#include <media/media-device.h>
-#include <media/v4l2-subdev.h>
-
-#ifndef ISP2401
-#include "ia_css_types.h"
-#include "sh_css_legacy.h"
-#else
-/*#include "ia_css_types.h"*/
-/*#include "sh_css_legacy.h"*/
-#endif
-
-#include "atomisp_csi2.h"
-#include "atomisp_file.h"
-#include "atomisp_subdev.h"
-#include "atomisp_tpg.h"
-#include "atomisp_compat.h"
-
-#include "gp_device.h"
-#include "irq.h"
-#include <linux/vmalloc.h>
-
-#define V4L2_EVENT_FRAME_END          5
-
-#define IS_HWREVISION(isp, rev) \
-	(((isp)->media_dev.hw_revision & ATOMISP_HW_REVISION_MASK) == \
-	 ((rev) << ATOMISP_HW_REVISION_SHIFT))
-
-#define MAX_STREAM_NUM	2
-
-#define ATOMISP_PCI_DEVICE_SOC_MASK	0xfff8
-/* MRFLD with 0x1178: ISP freq can burst to 457MHz */
-#define ATOMISP_PCI_DEVICE_SOC_MRFLD	0x1178
-/* MRFLD with 0x1179: max ISP freq limited to 400MHz */
-#define ATOMISP_PCI_DEVICE_SOC_MRFLD_1179	0x1179
-/* MRFLD with 0x117a: max ISP freq is 400MHz and max freq at Vmin is 200MHz */
-#define ATOMISP_PCI_DEVICE_SOC_MRFLD_117A	0x117a
-#define ATOMISP_PCI_DEVICE_SOC_BYT	0x0f38
-#define ATOMISP_PCI_DEVICE_SOC_ANN	0x1478
-#define ATOMISP_PCI_DEVICE_SOC_CHT	0x22b8
-
-#define ATOMISP_PCI_REV_MRFLD_A0_MAX	0
-#define ATOMISP_PCI_REV_BYT_A0_MAX	4
-
-#define ATOM_ISP_STEP_WIDTH	2
-#define ATOM_ISP_STEP_HEIGHT	2
-
-#define ATOM_ISP_MIN_WIDTH	4
-#define ATOM_ISP_MIN_HEIGHT	4
-#define ATOM_ISP_MAX_WIDTH	UINT_MAX
-#define ATOM_ISP_MAX_HEIGHT	UINT_MAX
-
-/* sub-QCIF resolution */
-#define ATOM_RESOLUTION_SUBQCIF_WIDTH	128
-#define ATOM_RESOLUTION_SUBQCIF_HEIGHT	96
-
-#define ATOM_ISP_MAX_WIDTH_TMP	1280
-#define ATOM_ISP_MAX_HEIGHT_TMP	720
-
-#define ATOM_ISP_I2C_BUS_1	4
-#define ATOM_ISP_I2C_BUS_2	5
-
-#define ATOM_ISP_POWER_DOWN	0
-#define ATOM_ISP_POWER_UP	1
-
-#define ATOM_ISP_MAX_INPUTS	4
-
-#define ATOMISP_SC_TYPE_SIZE	2
-
-#define ATOMISP_ISP_TIMEOUT_DURATION		(2 * HZ)
-#define ATOMISP_EXT_ISP_TIMEOUT_DURATION        (6 * HZ)
-#define ATOMISP_ISP_FILE_TIMEOUT_DURATION	(60 * HZ)
-#define ATOMISP_WDT_KEEP_CURRENT_DELAY          0
-#define ATOMISP_ISP_MAX_TIMEOUT_COUNT	2
-#define ATOMISP_CSS_STOP_TIMEOUT_US	200000
-
-#define ATOMISP_CSS_Q_DEPTH	3
-#define ATOMISP_CSS_EVENTS_MAX  16
-#define ATOMISP_CONT_RAW_FRAMES 15
-#define ATOMISP_METADATA_QUEUE_DEPTH_FOR_HAL	8
-#define ATOMISP_S3A_BUF_QUEUE_DEPTH_FOR_HAL	8
-
-#define ATOMISP_DELAYED_INIT_NOT_QUEUED	0
-#define ATOMISP_DELAYED_INIT_QUEUED	1
-#define ATOMISP_DELAYED_INIT_DONE	2
-
-#define ATOMISP_CALC_CSS_PREV_OVERLAP(lines) \
-	((lines) * 38 / 100 & 0xfffffe)
-
-/*
- * Define how fast CPU should be able to serve ISP interrupts.
- * The bigger the value, the higher risk that the ISP is not
- * triggered sufficiently fast for it to process image during
- * vertical blanking time, increasing risk of dropped frames.
- * 1000 us is a reasonable value considering that the processing
- * time is typically ~2000 us.
- */
-#define ATOMISP_MAX_ISR_LATENCY	1000
-
-/* Add new YUVPP pipe for SOC sensor. */
-#define ATOMISP_CSS_SUPPORT_YUVPP     1
-
-#define ATOMISP_CSS_OUTPUT_SECOND_INDEX     1
-#define ATOMISP_CSS_OUTPUT_DEFAULT_INDEX    0
-
-/*
- * ATOMISP_SOC_CAMERA
- * This is to differentiate between ext-isp and soc camera in
- * Moorefield/Baytrail platform.
- */
-#define ATOMISP_SOC_CAMERA(asd)  \
-	(asd->isp->inputs[asd->input_curr].type == SOC_CAMERA \
-	&& asd->isp->inputs[asd->input_curr].camera_caps-> \
-	   sensor[asd->sensor_curr].stream_num == 1)
-
-#define ATOMISP_USE_YUVPP(asd)  \
-	(ATOMISP_SOC_CAMERA(asd) && ATOMISP_CSS_SUPPORT_YUVPP && \
-	!asd->copy_mode)
-
-#define ATOMISP_DEPTH_SENSOR_STREAMON_COUNT 2
-
-#define ATOMISP_DEPTH_DEFAULT_MASTER_SENSOR 0
-#define ATOMISP_DEPTH_DEFAULT_SLAVE_SENSOR 1
-
-#ifdef ISP2401
-#define ATOMISP_ION_DEVICE_FD_OFFSET   16
-#define ATOMISP_ION_SHARED_FD_MASK     (0xFFFF)
-#define ATOMISP_ION_DEVICE_FD_MASK     (~ATOMISP_ION_SHARED_FD_MASK)
-#define ION_FD_UNSET (-1)
-
-#endif
-#define DIV_NEAREST_STEP(n, d, step) \
-	round_down((2 * (n) + (d) * (step))/(2 * (d)), (step))
-
-struct atomisp_input_subdev {
-	unsigned int type;
-	enum atomisp_camera_port port;
-	struct v4l2_subdev *camera;
-	struct v4l2_subdev *motor;
-	struct v4l2_frmsizeenum frame_size;
-
-	/*
-	 * To show this resource is used by
-	 * which stream, in ISP multiple stream mode
-	 */
-	struct atomisp_sub_device *asd;
-
-	const struct atomisp_camera_caps *camera_caps;
-	int sensor_index;
-};
-
-enum atomisp_dfs_mode {
-	ATOMISP_DFS_MODE_AUTO = 0,
-	ATOMISP_DFS_MODE_LOW,
-	ATOMISP_DFS_MODE_MAX,
-};
-
-struct atomisp_regs {
-	/* PCI config space info */
-	u16 pcicmdsts;
-	u32 ispmmadr;
-	u32 msicap;
-	u32 msi_addr;
-	u16 msi_data;
-	u8 intr;
-	u32 interrupt_control;
-	u32 pmcs;
-	u32 cg_dis;
-	u32 i_control;
-
-	/* I-Unit PHY related info */
-	u32 csi_rcomp_config;
-	u32 csi_afe_dly;
-	u32 csi_control;
-
-	/* New for MRFLD */
-	u32 csi_afe_rcomp_config;
-	u32 csi_afe_hs_control;
-	u32 csi_deadline_control;
-	u32 csi_access_viol;
-};
-
-struct atomisp_sw_contex {
-	bool file_input;
-	int power_state;
-	int running_freq;
-};
-
-
-#define ATOMISP_DEVICE_STREAMING_DISABLED	0
-#define ATOMISP_DEVICE_STREAMING_ENABLED	1
-#define ATOMISP_DEVICE_STREAMING_STOPPING	2
-
-/*
- * ci device struct
- */
-struct atomisp_device {
-	struct pci_dev *pdev;
-	struct device *dev;
-	struct v4l2_device v4l2_dev;
-	struct media_device media_dev;
-	struct atomisp_platform_data *pdata;
-	void *mmu_l1_base;
-	const struct firmware *firmware;
-
-	struct pm_qos_request pm_qos;
-	s32 max_isr_latency;
-
-	/*
-	 * ISP modules
-	 * Multiple streams are represents by multiple
-	 * atomisp_sub_device instances
-	 */
-	struct atomisp_sub_device *asd;
-	/*
-	 * this will be assiged dyanamically.
-	 * For Merr/BTY(ISP2400), 2 streams are supported.
-	 */
-	unsigned int num_of_streams;
-
-	struct atomisp_mipi_csi2_device csi2_port[ATOMISP_CAMERA_NR_PORTS];
-	struct atomisp_tpg_device tpg;
-	struct atomisp_file_device file_dev;
-
-	/* Purpose of mutex is to protect and serialize use of isp data
-	 * structures and css API calls. */
-	struct rt_mutex mutex;
-	/*
-	 * Serialise streamoff: mutex is dropped during streamoff to
-	 * cancel the watchdog queue. MUST be acquired BEFORE
-	 * "mutex".
-	 */
-	struct mutex streamoff_mutex;
-
-	unsigned int input_cnt;
-	struct atomisp_input_subdev inputs[ATOM_ISP_MAX_INPUTS];
-	struct v4l2_subdev *flash;
-	struct v4l2_subdev *motor;
-
-	struct atomisp_regs saved_regs;
-	struct atomisp_sw_contex sw_contex;
-	struct atomisp_css_env css_env;
-
-	/* isp timeout status flag */
-	bool isp_timeout;
-	bool isp_fatal_error;
-	struct workqueue_struct *wdt_work_queue;
-	struct work_struct wdt_work;
-#ifndef ISP2401
-	atomic_t wdt_count;
-#endif
-	atomic_t wdt_work_queued;
-
-	spinlock_t lock; /* Just for streaming below */
-
-	bool need_gfx_throttle;
-
-	unsigned int mipi_frame_size;
-	const struct atomisp_dfs_config *dfs;
-	unsigned int hpll_freq;
-
-	bool css_initialized;
-};
-
-#define v4l2_dev_to_atomisp_device(dev) \
-	container_of(dev, struct atomisp_device, v4l2_dev)
-
-extern struct device *atomisp_dev;
-
-#define atomisp_is_wdt_running(a) timer_pending(&(a)->wdt)
-#ifdef ISP2401
-extern void atomisp_wdt_refresh_pipe(struct atomisp_video_pipe *pipe,
-					unsigned int delay);
-#endif
-extern void atomisp_wdt_refresh(struct atomisp_sub_device *asd, unsigned int delay);
-#ifndef ISP2401
-extern void atomisp_wdt_start(struct atomisp_sub_device *asd);
-#else
-extern void atomisp_wdt_start(struct atomisp_video_pipe *pipe);
-extern void atomisp_wdt_stop_pipe(struct atomisp_video_pipe *pipe, bool sync);
-#endif
-extern void atomisp_wdt_stop(struct atomisp_sub_device *asd, bool sync);
-
-#endif /* __ATOMISP_INTERNAL_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.c
deleted file mode 100644
index 61bd550dafb9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.c
+++ /dev/null
@@ -1,3124 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#include <linux/delay.h>
-#include <linux/pci.h>
-
-
-#include <media/v4l2-ioctl.h>
-#include <media/v4l2-event.h>
-#include <media/videobuf-vmalloc.h>
-
-#include "atomisp_acc.h"
-#include "atomisp_cmd.h"
-#include "atomisp_common.h"
-#include "atomisp_fops.h"
-#include "atomisp_internal.h"
-#include "atomisp_ioctl.h"
-#include "atomisp-regs.h"
-#include "atomisp_compat.h"
-
-#include "sh_css_hrt.h"
-
-#include "gp_device.h"
-#include "device_access.h"
-#include "irq.h"
-
-#include "hrt/hive_isp_css_mm_hrt.h"
-
-/* for v4l2_capability */
-static const char *DRIVER = "atomisp";	/* max size 15 */
-static const char *CARD = "ATOM ISP";	/* max size 31 */
-static const char *BUS_INFO = "PCI-3";	/* max size 31 */
-
-/*
- * FIXME: ISP should not know beforehand all CIDs supported by sensor.
- * Instead, it needs to propagate to sensor unkonwn CIDs.
- */
-static struct v4l2_queryctrl ci_v4l2_controls[] = {
-	{
-		.id = V4L2_CID_AUTO_WHITE_BALANCE,
-		.type = V4L2_CTRL_TYPE_BOOLEAN,
-		.name = "Automatic White Balance",
-		.minimum = 0,
-		.maximum = 1,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_RED_BALANCE,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Red Balance",
-		.minimum = 0x00,
-		.maximum = 0xff,
-		.step = 1,
-		.default_value = 0x00,
-	},
-	{
-		.id = V4L2_CID_BLUE_BALANCE,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Blue Balance",
-		.minimum = 0x00,
-		.maximum = 0xff,
-		.step = 1,
-		.default_value = 0x00,
-	},
-	{
-		.id = V4L2_CID_GAMMA,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Gamma",
-		.minimum = 0x00,
-		.maximum = 0xff,
-		.step = 1,
-		.default_value = 0x00,
-	},
-	{
-		.id = V4L2_CID_POWER_LINE_FREQUENCY,
-		.type = V4L2_CTRL_TYPE_MENU,
-		.name = "Light frequency filter",
-		.minimum = 1,
-		.maximum = 2,
-		.step = 1,
-		.default_value = 1,
-	},
-	{
-		.id = V4L2_CID_COLORFX,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Image Color Effect",
-		.minimum = 0,
-		.maximum = 9,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_COLORFX_CBCR,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Image Color Effect CbCr",
-		.minimum = 0,
-		.maximum = 0xffff,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_ATOMISP_BAD_PIXEL_DETECTION,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Bad Pixel Correction",
-		.minimum = 0,
-		.maximum = 1,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_ATOMISP_POSTPROCESS_GDC_CAC,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "GDC/CAC",
-		.minimum = 0,
-		.maximum = 1,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_ATOMISP_VIDEO_STABLIZATION,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Video Stablization",
-		.minimum = 0,
-		.maximum = 1,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_ATOMISP_FIXED_PATTERN_NR,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Fixed Pattern Noise Reduction",
-		.minimum = 0,
-		.maximum = 1,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_ATOMISP_FALSE_COLOR_CORRECTION,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "False Color Correction",
-		.minimum = 0,
-		.maximum = 1,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_REQUEST_FLASH,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Request flash frames",
-		.minimum = 0,
-		.maximum = 10,
-		.step = 1,
-		.default_value = 1,
-	},
-	{
-		.id = V4L2_CID_ATOMISP_LOW_LIGHT,
-		.type = V4L2_CTRL_TYPE_BOOLEAN,
-		.name = "Low light mode",
-		.minimum = 0,
-		.maximum = 1,
-		.step = 1,
-		.default_value = 1,
-	},
-	{
-		.id = V4L2_CID_BIN_FACTOR_HORZ,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Horizontal binning factor",
-		.minimum = 0,
-		.maximum = 10,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_BIN_FACTOR_VERT,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Vertical binning factor",
-		.minimum = 0,
-		.maximum = 10,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_2A_STATUS,
-		.type = V4L2_CTRL_TYPE_BITMASK,
-		.name = "AE and AWB status",
-		.minimum = 0,
-		.maximum = V4L2_2A_STATUS_AE_READY | V4L2_2A_STATUS_AWB_READY,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_EXPOSURE,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "exposure",
-		.minimum = -4,
-		.maximum = 4,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_EXPOSURE_ZONE_NUM,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "one-time exposure zone number",
-		.minimum = 0x0,
-		.maximum = 0xffff,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_EXPOSURE_AUTO_PRIORITY,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Exposure auto priority",
-		.minimum = V4L2_EXPOSURE_AUTO,
-		.maximum = V4L2_EXPOSURE_APERTURE_PRIORITY,
-		.step = 1,
-		.default_value = V4L2_EXPOSURE_AUTO,
-	},
-	{
-		.id = V4L2_CID_SCENE_MODE,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "scene mode",
-		.minimum = 0,
-		.maximum = 13,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_ISO_SENSITIVITY,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "iso",
-		.minimum = -4,
-		.maximum = 4,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_ISO_SENSITIVITY_AUTO,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "iso mode",
-		.minimum = V4L2_ISO_SENSITIVITY_MANUAL,
-		.maximum = V4L2_ISO_SENSITIVITY_AUTO,
-		.step = 1,
-		.default_value = V4L2_ISO_SENSITIVITY_AUTO,
-	},
-	{
-		.id = V4L2_CID_AUTO_N_PRESET_WHITE_BALANCE,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "white balance",
-		.minimum = 0,
-		.maximum = 9,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_EXPOSURE_METERING,
-		.type = V4L2_CTRL_TYPE_MENU,
-		.name = "metering",
-		.minimum = 0,
-		.maximum = 3,
-		.step = 1,
-		.default_value = 1,
-	},
-	{
-		.id = V4L2_CID_3A_LOCK,
-		.type = V4L2_CTRL_TYPE_BITMASK,
-		.name = "3a lock",
-		.minimum = 0,
-		.maximum = V4L2_LOCK_EXPOSURE | V4L2_LOCK_WHITE_BALANCE
-			 | V4L2_LOCK_FOCUS,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_TEST_PATTERN,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Test Pattern",
-		.minimum = 0,
-		.maximum = 0xffff,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_TEST_PATTERN_COLOR_R,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Test Pattern Solid Color R",
-		.minimum = INT_MIN,
-		.maximum = INT_MAX,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_TEST_PATTERN_COLOR_GR,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Test Pattern Solid Color GR",
-		.minimum = INT_MIN,
-		.maximum = INT_MAX,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_TEST_PATTERN_COLOR_GB,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Test Pattern Solid Color GB",
-		.minimum = INT_MIN,
-		.maximum = INT_MAX,
-		.step = 1,
-		.default_value = 0,
-	},
-	{
-		.id = V4L2_CID_TEST_PATTERN_COLOR_B,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Test Pattern Solid Color B",
-		.minimum = INT_MIN,
-		.maximum = INT_MAX,
-		.step = 1,
-		.default_value = 0,
-	},
-};
-static const u32 ctrls_num = ARRAY_SIZE(ci_v4l2_controls);
-
-/*
- * supported V4L2 fmts and resolutions
- */
-const struct atomisp_format_bridge atomisp_output_fmts[] = {
-	{
-		.pixelformat = V4L2_PIX_FMT_YUV420,
-		.depth = 12,
-		.mbus_code = V4L2_MBUS_FMT_CUSTOM_YUV420,
-		.sh_fmt = CSS_FRAME_FORMAT_YUV420,
-		.description = "YUV420, planar",
-		.planar = true
-	}, {
-		.pixelformat = V4L2_PIX_FMT_YVU420,
-		.depth = 12,
-		.mbus_code = V4L2_MBUS_FMT_CUSTOM_YVU420,
-		.sh_fmt = CSS_FRAME_FORMAT_YV12,
-		.description = "YVU420, planar",
-		.planar = true
-	}, {
-		.pixelformat = V4L2_PIX_FMT_YUV422P,
-		.depth = 16,
-		.mbus_code = V4L2_MBUS_FMT_CUSTOM_YUV422P,
-		.sh_fmt = CSS_FRAME_FORMAT_YUV422,
-		.description = "YUV422, planar",
-		.planar = true
-	}, {
-		.pixelformat = V4L2_PIX_FMT_YUV444,
-		.depth = 24,
-		.mbus_code = V4L2_MBUS_FMT_CUSTOM_YUV444,
-		.sh_fmt = CSS_FRAME_FORMAT_YUV444,
-		.description = "YUV444"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_NV12,
-		.depth = 12,
-		.mbus_code = V4L2_MBUS_FMT_CUSTOM_NV12,
-		.sh_fmt = CSS_FRAME_FORMAT_NV12,
-		.description = "NV12, Y-plane, CbCr interleaved",
-		.planar = true
-	}, {
-		.pixelformat = V4L2_PIX_FMT_NV21,
-		.depth = 12,
-		.mbus_code = V4L2_MBUS_FMT_CUSTOM_NV21,
-		.sh_fmt = CSS_FRAME_FORMAT_NV21,
-		.description = "NV21, Y-plane, CbCr interleaved",
-		.planar = true
-	}, {
-		.pixelformat = V4L2_PIX_FMT_NV16,
-		.depth = 16,
-		.mbus_code = V4L2_MBUS_FMT_CUSTOM_NV16,
-		.sh_fmt = CSS_FRAME_FORMAT_NV16,
-		.description = "NV16, Y-plane, CbCr interleaved",
-		.planar = true
-	}, {
-		.pixelformat = V4L2_PIX_FMT_YUYV,
-		.depth = 16,
-		.mbus_code = V4L2_MBUS_FMT_CUSTOM_YUYV,
-		.sh_fmt = CSS_FRAME_FORMAT_YUYV,
-		.description = "YUYV, interleaved"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_UYVY,
-		.depth = 16,
-		.mbus_code = MEDIA_BUS_FMT_UYVY8_1X16,
-		.sh_fmt = CSS_FRAME_FORMAT_UYVY,
-		.description = "UYVY, interleaved"
-	}, { /* This one is for parallel sensors! DO NOT USE! */
-		.pixelformat = V4L2_PIX_FMT_UYVY,
-		.depth = 16,
-		.mbus_code = MEDIA_BUS_FMT_UYVY8_2X8,
-		.sh_fmt = CSS_FRAME_FORMAT_UYVY,
-		.description = "UYVY, interleaved"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SBGGR16,
-		.depth = 16,
-		.mbus_code = V4L2_MBUS_FMT_CUSTOM_SBGGR16,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 16"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SBGGR8,
-		.depth = 8,
-		.mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 8"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SGBRG8,
-		.depth = 8,
-		.mbus_code = MEDIA_BUS_FMT_SGBRG8_1X8,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 8"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SGRBG8,
-		.depth = 8,
-		.mbus_code = MEDIA_BUS_FMT_SGRBG8_1X8,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 8"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SRGGB8,
-		.depth = 8,
-		.mbus_code = MEDIA_BUS_FMT_SRGGB8_1X8,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 8"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SBGGR10,
-		.depth = 16,
-		.mbus_code = MEDIA_BUS_FMT_SBGGR10_1X10,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 10"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SGBRG10,
-		.depth = 16,
-		.mbus_code = MEDIA_BUS_FMT_SGBRG10_1X10,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 10"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SGRBG10,
-		.depth = 16,
-		.mbus_code = MEDIA_BUS_FMT_SGRBG10_1X10,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 10"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SRGGB10,
-		.depth = 16,
-		.mbus_code = MEDIA_BUS_FMT_SRGGB10_1X10,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 10"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SBGGR12,
-		.depth = 16,
-		.mbus_code = MEDIA_BUS_FMT_SBGGR12_1X12,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 12"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SGBRG12,
-		.depth = 16,
-		.mbus_code = MEDIA_BUS_FMT_SGBRG12_1X12,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 12"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SGRBG12,
-		.depth = 16,
-		.mbus_code = MEDIA_BUS_FMT_SGRBG12_1X12,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 12"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_SRGGB12,
-		.depth = 16,
-		.mbus_code = MEDIA_BUS_FMT_SRGGB12_1X12,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
-		.description = "Bayer 12"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_RGB32,
-		.depth = 32,
-		.mbus_code = V4L2_MBUS_FMT_CUSTOM_RGB32,
-		.sh_fmt = CSS_FRAME_FORMAT_RGBA888,
-		.description = "32 RGB 8-8-8-8"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_RGB565,
-		.depth = 16,
-		.mbus_code = MEDIA_BUS_FMT_BGR565_2X8_LE,
-		.sh_fmt = CSS_FRAME_FORMAT_RGB565,
-		.description = "16 RGB 5-6-5"
-	}, {
-		.pixelformat = V4L2_PIX_FMT_JPEG,
-		.depth = 8,
-		.mbus_code = MEDIA_BUS_FMT_JPEG_1X8,
-		.sh_fmt = CSS_FRAME_FORMAT_BINARY_8,
-		.description = "JPEG"
-	},
-#if 0
-	{
-	/* This is a custom format being used by M10MO to send the RAW data */
-		.pixelformat = V4L2_PIX_FMT_CUSTOM_M10MO_RAW,
-		.depth = 8,
-		.mbus_code = V4L2_MBUS_FMT_CUSTOM_M10MO_RAW,
-		.sh_fmt = CSS_FRAME_FORMAT_BINARY_8,
-		.description = "Custom RAW for M10MO"
-	},
-#endif
-};
-
-const struct atomisp_format_bridge *atomisp_get_format_bridge(
-	unsigned int pixelformat)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(atomisp_output_fmts); i++) {
-		if (atomisp_output_fmts[i].pixelformat == pixelformat)
-			return &atomisp_output_fmts[i];
-	}
-
-	return NULL;
-}
-
-const struct atomisp_format_bridge *atomisp_get_format_bridge_from_mbus(
-	u32 mbus_code)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(atomisp_output_fmts); i++) {
-		if (mbus_code == atomisp_output_fmts[i].mbus_code)
-			return &atomisp_output_fmts[i];
-	}
-
-	return NULL;
-}
-
-/*
- * v4l2 ioctls
- * return ISP capabilities
- *
- * FIXME: capabilities should be different for video0/video2/video3
- */
-static int atomisp_querycap(struct file *file, void *fh,
-			    struct v4l2_capability *cap)
-{
-	memset(cap, 0, sizeof(struct v4l2_capability));
-
-	WARN_ON(sizeof(DRIVER) > sizeof(cap->driver) ||
-		sizeof(CARD) > sizeof(cap->card) ||
-		sizeof(BUS_INFO) > sizeof(cap->bus_info));
-
-	strncpy(cap->driver, DRIVER, sizeof(cap->driver) - 1);
-	strncpy(cap->card, CARD, sizeof(cap->card) - 1);
-	strncpy(cap->bus_info, BUS_INFO, sizeof(cap->card) - 1);
-
-	cap->device_caps = V4L2_CAP_VIDEO_CAPTURE |
-	    V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_OUTPUT;
-	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
-	return 0;
-}
-
-/*
- * enum input are used to check primary/secondary camera
- */
-static int atomisp_enum_input(struct file *file, void *fh,
-	struct v4l2_input *input)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	int index = input->index;
-
-	if (index >= isp->input_cnt)
-		return -EINVAL;
-
-	if (!isp->inputs[index].camera)
-		return -EINVAL;
-
-	memset(input, 0, sizeof(struct v4l2_input));
-	strncpy(input->name, isp->inputs[index].camera->name,
-		sizeof(input->name) - 1);
-
-	/*
-	 * HACK: append actuator's name to sensor's
-	 * As currently userspace can't talk directly to subdev nodes, this
-	 * ioctl is the only way to enum inputs + possible external actuators
-	 * for 3A tuning purpose.
-	 */
-#ifndef ISP2401
-	if (isp->inputs[index].motor &&
-	    strlen(isp->inputs[index].motor->name) > 0) {
-#else
-	if (isp->motor &&
-	    strlen(isp->motor->name) > 0) {
-#endif
-		const int cur_len = strlen(input->name);
-		const int max_size = sizeof(input->name) - cur_len - 1;
-
-		if (max_size > 1) {
-			input->name[cur_len] = '+';
-			strncpy(&input->name[cur_len + 1],
-#ifndef ISP2401
-				isp->inputs[index].motor->name, max_size - 1);
-#else
-				isp->motor->name, max_size - 1);
-#endif
-		}
-	}
-
-	input->type = V4L2_INPUT_TYPE_CAMERA;
-	input->index = index;
-	input->reserved[0] = isp->inputs[index].type;
-	input->reserved[1] = isp->inputs[index].port;
-
-	return 0;
-}
-
-static unsigned int atomisp_subdev_streaming_count(
-					struct atomisp_sub_device *asd)
-{
-	return asd->video_out_preview.capq.streaming
-		+ asd->video_out_capture.capq.streaming
-		+ asd->video_out_video_capture.capq.streaming
-		+ asd->video_out_vf.capq.streaming
-		+ asd->video_in.capq.streaming;
-}
-
-unsigned int atomisp_streaming_count(struct atomisp_device *isp)
-{
-	unsigned int i, sum;
-
-	for (i = 0, sum = 0; i < isp->num_of_streams; i++)
-		sum += isp->asd[i].streaming ==
-		    ATOMISP_DEVICE_STREAMING_ENABLED;
-
-	return sum;
-}
-
-unsigned int atomisp_is_acc_enabled(struct atomisp_device *isp)
-{
-	unsigned int i;
-
-	for (i = 0; i < isp->num_of_streams; i++)
-		if (isp->asd[i].acc.pipeline)
-			return 1;
-
-	return 0;
-}
-/*
- * get input are used to get current primary/secondary camera
- */
-static int atomisp_g_input(struct file *file, void *fh, unsigned int *input)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-
-	rt_mutex_lock(&isp->mutex);
-	*input = asd->input_curr;
-	rt_mutex_unlock(&isp->mutex);
-
-	return 0;
-}
-/*
- * set input are used to set current primary/secondary camera
- */
-static int atomisp_s_input(struct file *file, void *fh, unsigned int input)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	struct v4l2_subdev *camera = NULL;
-	int ret;
-
-	rt_mutex_lock(&isp->mutex);
-	if (input >= ATOM_ISP_MAX_INPUTS || input >= isp->input_cnt) {
-		dev_dbg(isp->dev, "input_cnt: %d\n", isp->input_cnt);
-		ret = -EINVAL;
-		goto error;
-	}
-
-	/*
-	 * check whether the request camera:
-	 * 1: already in use
-	 * 2: if in use, whether it is used by other streams
-	 */
-	if (isp->inputs[input].asd != NULL && isp->inputs[input].asd != asd) {
-		dev_err(isp->dev,
-			 "%s, camera is already used by stream: %d\n", __func__,
-			 isp->inputs[input].asd->index);
-		ret = -EBUSY;
-		goto error;
-	}
-
-	camera = isp->inputs[input].camera;
-	if (!camera) {
-		dev_err(isp->dev, "%s, no camera\n", __func__);
-		ret = -EINVAL;
-		goto error;
-	}
-
-	if (atomisp_subdev_streaming_count(asd)) {
-		dev_err(isp->dev,
-			 "ISP is still streaming, stop first\n");
-		ret = -EINVAL;
-		goto error;
-	}
-
-	/* power off the current owned sensor, as it is not used this time */
-	if (isp->inputs[asd->input_curr].asd == asd &&
-	    asd->input_curr != input) {
-		ret = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-				       core, s_power, 0);
-		if (ret)
-			dev_warn(isp->dev,
-				    "Failed to power-off sensor\n");
-		/* clear the asd field to show this camera is not used */
-		isp->inputs[asd->input_curr].asd = NULL;
-	}
-
-	/* powe on the new sensor */
-	ret = v4l2_subdev_call(isp->inputs[input].camera, core, s_power, 1);
-	if (ret) {
-		dev_err(isp->dev, "Failed to power-on sensor\n");
-		goto error;
-	}
-	/*
-	 * Some sensor driver resets the run mode during power-on, thus force
-	 * update the run mode to sensor after power-on.
-	 */
-	atomisp_update_run_mode(asd);
-
-	/* select operating sensor */
-	ret = v4l2_subdev_call(isp->inputs[input].camera, video, s_routing,
-		0, isp->inputs[input].sensor_index, 0);
-	if (ret && (ret != -ENOIOCTLCMD)) {
-		dev_err(isp->dev, "Failed to select sensor\n");
-		goto error;
-	}
-
-#ifndef ISP2401
-	if (!isp->sw_contex.file_input && isp->inputs[input].motor)
-		ret = v4l2_subdev_call(isp->inputs[input].motor, core,
-				       init, 1);
-#else
-	if (isp->motor)
-		ret = v4l2_subdev_call(isp->motor, core, s_power, 1);
-
-	if (!isp->sw_contex.file_input && isp->motor)
-		ret = v4l2_subdev_call(isp->motor, core, init, 1);
-#endif
-
-	asd->input_curr = input;
-	/* mark this camera is used by the current stream */
-	isp->inputs[input].asd = asd;
-	rt_mutex_unlock(&isp->mutex);
-
-	return 0;
-
-error:
-	rt_mutex_unlock(&isp->mutex);
-
-	return ret;
-}
-
-static int atomisp_enum_fmt_cap(struct file *file, void *fh,
-	struct v4l2_fmtdesc *f)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	struct v4l2_subdev_mbus_code_enum code = { 0 };
-	unsigned int i, fi = 0;
-	int rval;
-
-	rt_mutex_lock(&isp->mutex);
-	rval = v4l2_subdev_call(isp->inputs[asd->input_curr].camera, pad,
-				enum_mbus_code, NULL, &code);
-	if (rval == -ENOIOCTLCMD) {
-		dev_warn(isp->dev, "enum_mbus_code pad op not supported. Please fix your sensor driver!\n");
-	//	rval = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-	//				video, enum_mbus_fmt, 0, &code.code);
-	}
-	rt_mutex_unlock(&isp->mutex);
-
-	if (rval)
-		return rval;
-
-	for (i = 0; i < ARRAY_SIZE(atomisp_output_fmts); i++) {
-		const struct atomisp_format_bridge *format =
-			&atomisp_output_fmts[i];
-
-		/*
-		 * Is the atomisp-supported format is valid for the
-		 * sensor (configuration)? If not, skip it.
-		 */
-		if (format->sh_fmt == CSS_FRAME_FORMAT_RAW
-		    && format->mbus_code != code.code)
-			continue;
-
-		/* Found a match. Now let's pick f->index'th one. */
-		if (fi < f->index) {
-			fi++;
-			continue;
-		}
-
-		strlcpy(f->description, format->description,
-			sizeof(f->description));
-		f->pixelformat = format->pixelformat;
-		return 0;
-	}
-
-	return -EINVAL;
-}
-
-static int atomisp_g_fmt_cap(struct file *file, void *fh,
-	struct v4l2_format *f)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-
-	int ret;
-
-	rt_mutex_lock(&isp->mutex);
-	ret = atomisp_get_fmt(vdev, f);
-	rt_mutex_unlock(&isp->mutex);
-	return ret;
-}
-
-static int atomisp_g_fmt_file(struct file *file, void *fh,
-		struct v4l2_format *f)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-
-	rt_mutex_lock(&isp->mutex);
-	f->fmt.pix = pipe->pix;
-	rt_mutex_unlock(&isp->mutex);
-
-	return 0;
-}
-
-/* This function looks up the closest available resolution. */
-static int atomisp_try_fmt_cap(struct file *file, void *fh,
-	struct v4l2_format *f)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	int ret;
-
-	rt_mutex_lock(&isp->mutex);
-	ret = atomisp_try_fmt(vdev, f, NULL);
-	rt_mutex_unlock(&isp->mutex);
-	return ret;
-}
-
-static int atomisp_s_fmt_cap(struct file *file, void *fh,
-	struct v4l2_format *f)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	int ret;
-
-	rt_mutex_lock(&isp->mutex);
-	if (isp->isp_fatal_error) {
-		ret = -EIO;
-		rt_mutex_unlock(&isp->mutex);
-		return ret;
-	}
-	ret = atomisp_set_fmt(vdev, f);
-	rt_mutex_unlock(&isp->mutex);
-	return ret;
-}
-
-static int atomisp_s_fmt_file(struct file *file, void *fh,
-				struct v4l2_format *f)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	int ret;
-
-	rt_mutex_lock(&isp->mutex);
-	ret = atomisp_set_fmt_file(vdev, f);
-	rt_mutex_unlock(&isp->mutex);
-	return ret;
-}
-
-/*
- * Free videobuffer buffer priv data
- */
-void atomisp_videobuf_free_buf(struct videobuf_buffer *vb)
-{
-	struct videobuf_vmalloc_memory *vm_mem;
-
-	if (vb == NULL)
-		return;
-
-	vm_mem = vb->priv;
-	if (vm_mem && vm_mem->vaddr) {
-		atomisp_css_frame_free(vm_mem->vaddr);
-		vm_mem->vaddr = NULL;
-	}
-}
-
-/*
- * this function is used to free video buffer queue
- */
-static void atomisp_videobuf_free_queue(struct videobuf_queue *q)
-{
-	int i;
-
-	for (i = 0; i < VIDEO_MAX_FRAME; i++) {
-		atomisp_videobuf_free_buf(q->bufs[i]);
-		kfree(q->bufs[i]);
-		q->bufs[i] = NULL;
-	}
-}
-
-int atomisp_alloc_css_stat_bufs(struct atomisp_sub_device *asd,
-	uint16_t stream_id)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct atomisp_s3a_buf *s3a_buf = NULL, *_s3a_buf;
-	struct atomisp_dis_buf *dis_buf = NULL, *_dis_buf;
-	struct atomisp_metadata_buf *md_buf = NULL, *_md_buf;
-	int count;
-	struct atomisp_css_dvs_grid_info *dvs_grid_info =
-		atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
-	unsigned int i;
-
-	if (list_empty(&asd->s3a_stats) &&
-		asd->params.curr_grid_info.s3a_grid.enable) {
-		count = ATOMISP_CSS_Q_DEPTH +
-			ATOMISP_S3A_BUF_QUEUE_DEPTH_FOR_HAL;
-		dev_dbg(isp->dev, "allocating %d 3a buffers\n", count);
-		while (count--) {
-			s3a_buf = kzalloc(sizeof(struct atomisp_s3a_buf), GFP_KERNEL);
-			if (!s3a_buf)
-				goto error;
-
-			if (atomisp_css_allocate_stat_buffers(
-					asd, stream_id, s3a_buf, NULL, NULL)) {
-				kfree(s3a_buf);
-				goto error;
-			}
-
-			list_add_tail(&s3a_buf->list, &asd->s3a_stats);
-		}
-	}
-
-	if (list_empty(&asd->dis_stats) && dvs_grid_info &&
-		dvs_grid_info->enable) {
-		count = ATOMISP_CSS_Q_DEPTH + 1;
-		dev_dbg(isp->dev, "allocating %d dis buffers\n", count);
-		while (count--) {
-			dis_buf = kzalloc(sizeof(struct atomisp_dis_buf), GFP_KERNEL);
-			if (!dis_buf) {
-				kfree(s3a_buf);
-				goto error;
-			}
-			if (atomisp_css_allocate_stat_buffers(
-					asd, stream_id, NULL, dis_buf, NULL)) {
-				kfree(dis_buf);
-				goto error;
-			}
-
-			list_add_tail(&dis_buf->list, &asd->dis_stats);
-		}
-	}
-
-	for (i = 0; i < ATOMISP_METADATA_TYPE_NUM; i++) {
-		if (list_empty(&asd->metadata[i]) &&
-		    list_empty(&asd->metadata_ready[i]) &&
-		    list_empty(&asd->metadata_in_css[i])) {
-			count = ATOMISP_CSS_Q_DEPTH +
-				ATOMISP_METADATA_QUEUE_DEPTH_FOR_HAL;
-			dev_dbg(isp->dev, "allocating %d metadata buffers for type %d\n",
-				count, i);
-			while (count--) {
-				md_buf = kzalloc(sizeof(struct atomisp_metadata_buf),
-						 GFP_KERNEL);
-				if (!md_buf)
-					goto error;
-
-				if (atomisp_css_allocate_stat_buffers(
-						asd, stream_id, NULL, NULL, md_buf)) {
-					kfree(md_buf);
-					goto error;
-				}
-				list_add_tail(&md_buf->list, &asd->metadata[i]);
-			}
-		}
-	}
-	return 0;
-
-error:
-	dev_err(isp->dev, "failed to allocate statistics buffers\n");
-
-	list_for_each_entry_safe(dis_buf, _dis_buf, &asd->dis_stats, list) {
-		atomisp_css_free_dis_buffer(dis_buf);
-		list_del(&dis_buf->list);
-		kfree(dis_buf);
-	}
-
-	list_for_each_entry_safe(s3a_buf, _s3a_buf, &asd->s3a_stats, list) {
-		atomisp_css_free_3a_buffer(s3a_buf);
-		list_del(&s3a_buf->list);
-		kfree(s3a_buf);
-	}
-
-	for (i = 0; i < ATOMISP_METADATA_TYPE_NUM; i++) {
-		list_for_each_entry_safe(md_buf, _md_buf, &asd->metadata[i],
-					list) {
-			atomisp_css_free_metadata_buffer(md_buf);
-			list_del(&md_buf->list);
-			kfree(md_buf);
-		}
-	}
-	return -ENOMEM;
-}
-
-/*
- * Initiate Memory Mapping or User Pointer I/O
- */
-int __atomisp_reqbufs(struct file *file, void *fh,
-	struct v4l2_requestbuffers *req)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-	struct atomisp_sub_device *asd = pipe->asd;
-	struct atomisp_css_frame_info frame_info;
-	struct atomisp_css_frame *frame;
-	struct videobuf_vmalloc_memory *vm_mem;
-	uint16_t source_pad = atomisp_subdev_source_pad(vdev);
-	uint16_t stream_id = atomisp_source_pad_to_stream_id(asd, source_pad);
-	int ret = 0, i = 0;
-
-	if (req->count == 0) {
-		mutex_lock(&pipe->capq.vb_lock);
-		if (!list_empty(&pipe->capq.stream))
-			videobuf_queue_cancel(&pipe->capq);
-
-		atomisp_videobuf_free_queue(&pipe->capq);
-		mutex_unlock(&pipe->capq.vb_lock);
-		/* clear request config id */
-		memset(pipe->frame_request_config_id, 0,
-			VIDEO_MAX_FRAME * sizeof(unsigned int));
-		memset(pipe->frame_params, 0,
-			VIDEO_MAX_FRAME *
-			sizeof(struct atomisp_css_params_with_list *));
-		return 0;
-	}
-
-	ret = videobuf_reqbufs(&pipe->capq, req);
-	if (ret)
-		return ret;
-
-	atomisp_alloc_css_stat_bufs(asd, stream_id);
-
-	/*
-	 * for user pointer type, buffers are not really allcated here,
-	 * buffers are setup in QBUF operation through v4l2_buffer structure
-	 */
-	if (req->memory == V4L2_MEMORY_USERPTR)
-		return 0;
-
-	ret = atomisp_get_css_frame_info(asd, source_pad, &frame_info);
-	if (ret)
-		return ret;
-
-	/*
-	 * Allocate the real frame here for selected node using our
-	 * memory management function
-	 */
-	for (i = 0; i < req->count; i++) {
-		if (atomisp_css_frame_allocate_from_info(&frame, &frame_info))
-			goto error;
-		vm_mem = pipe->capq.bufs[i]->priv;
-		vm_mem->vaddr = frame;
-	}
-
-	return ret;
-
-error:
-	while (i--) {
-		vm_mem = pipe->capq.bufs[i]->priv;
-		atomisp_css_frame_free(vm_mem->vaddr);
-	}
-
-	if (asd->vf_frame)
-		atomisp_css_frame_free(asd->vf_frame);
-
-	return -ENOMEM;
-}
-
-int atomisp_reqbufs(struct file *file, void *fh,
-	struct v4l2_requestbuffers *req)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	int ret;
-
-	rt_mutex_lock(&isp->mutex);
-	ret = __atomisp_reqbufs(file, fh, req);
-	rt_mutex_unlock(&isp->mutex);
-
-	return ret;
-}
-
-static int atomisp_reqbufs_file(struct file *file, void *fh,
-		struct v4l2_requestbuffers *req)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-
-	if (req->count == 0) {
-		mutex_lock(&pipe->outq.vb_lock);
-		atomisp_videobuf_free_queue(&pipe->outq);
-		mutex_unlock(&pipe->outq.vb_lock);
-		return 0;
-	}
-
-	return videobuf_reqbufs(&pipe->outq, req);
-}
-
-/* application query the status of a buffer */
-static int atomisp_querybuf(struct file *file, void *fh,
-	struct v4l2_buffer *buf)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-
-	return videobuf_querybuf(&pipe->capq, buf);
-}
-
-static int atomisp_querybuf_file(struct file *file, void *fh,
-				struct v4l2_buffer *buf)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-
-	return videobuf_querybuf(&pipe->outq, buf);
-}
-
-/*
- * Applications call the VIDIOC_QBUF ioctl to enqueue an empty (capturing) or
- * filled (output) buffer in the drivers incoming queue.
- */
-static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
-{
-	static const int NOFLUSH_FLAGS = V4L2_BUF_FLAG_NO_CACHE_INVALIDATE |
-					 V4L2_BUF_FLAG_NO_CACHE_CLEAN;
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-	struct atomisp_sub_device *asd = pipe->asd;
-	struct videobuf_buffer *vb;
-	struct videobuf_vmalloc_memory *vm_mem;
-	struct atomisp_css_frame_info frame_info;
-	struct atomisp_css_frame *handle = NULL;
-	u32 length;
-	u32 pgnr;
-	int ret = 0;
-
-	rt_mutex_lock(&isp->mutex);
-	if (isp->isp_fatal_error) {
-		ret = -EIO;
-		goto error;
-	}
-
-	if (asd->streaming == ATOMISP_DEVICE_STREAMING_STOPPING) {
-		dev_err(isp->dev, "%s: reject, as ISP at stopping.\n",
-				__func__);
-		ret = -EIO;
-		goto error;
-	}
-
-	if (!buf || buf->index >= VIDEO_MAX_FRAME ||
-		!pipe->capq.bufs[buf->index]) {
-		dev_err(isp->dev, "Invalid index for qbuf.\n");
-		ret = -EINVAL;
-		goto error;
-	}
-
-	/*
-	 * For userptr type frame, we convert user space address to physic
-	 * address and reprograme out page table properly
-	 */
-	if (buf->memory == V4L2_MEMORY_USERPTR) {
-		struct hrt_userbuffer_attr attributes;
-		vb = pipe->capq.bufs[buf->index];
-		vm_mem = vb->priv;
-		if (!vm_mem) {
-			ret = -EINVAL;
-			goto error;
-		}
-
-		length = vb->bsize;
-		pgnr = (length + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
-
-		if (vb->baddr == buf->m.userptr && vm_mem->vaddr)
-			goto done;
-
-		if (atomisp_get_css_frame_info(asd,
-				atomisp_subdev_source_pad(vdev), &frame_info)) {
-			ret = -EIO;
-			goto error;
-		}
-
-		attributes.pgnr = pgnr;
-#ifdef CONFIG_ION
-#ifndef ISP2401
-		attributes.type = buf->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_ION
-					? HRT_USR_ION : HRT_USR_PTR;
-#else
-		if (buf->reserved & ATOMISP_BUFFER_TYPE_IS_ION) {
-			attributes.type = HRT_USR_ION;
-			if (asd->ion_dev_fd->val !=  ION_FD_UNSET) {
-				dev_dbg(isp->dev, "ION buffer queued, share_fd=%lddev_fd=%d.\n",
-				buf->m.userptr, asd->ion_dev_fd->val);
-				/*
-				 * Make sure the shared fd we just got
-				 * from user space isn't larger than
-				 * the space we have for it.
-				 */
-				if ((buf->m.userptr &
-				(ATOMISP_ION_DEVICE_FD_MASK)) != 0) {
-					dev_err(isp->dev,
-							"Error: v4l2 buffer fd:0X%0lX > 0XFFFF.\n",
-							buf->m.userptr);
-					ret = -EINVAL;
-					goto error;
-				}
-				buf->m.userptr |= asd->ion_dev_fd->val <<
-					ATOMISP_ION_DEVICE_FD_OFFSET;
-			} else {
-				dev_err(isp->dev, "v4l2 buffer type is ION, \
-						but no dev fd set from userspace.\n");
-				ret = -EINVAL;
-				goto error;
-			}
-		} else {
-			attributes.type = HRT_USR_PTR;
-		}
-#endif
-#else
-		attributes.type = HRT_USR_PTR;
-#endif
-		ret = atomisp_css_frame_map(&handle, &frame_info,
-				       (void *)buf->m.userptr,
-				       0, &attributes);
-		if (ret) {
-			dev_err(isp->dev, "Failed to map user buffer\n");
-			goto error;
-		}
-
-		if (vm_mem->vaddr) {
-			mutex_lock(&pipe->capq.vb_lock);
-			atomisp_css_frame_free(vm_mem->vaddr);
-			vm_mem->vaddr = NULL;
-			vb->state = VIDEOBUF_NEEDS_INIT;
-			mutex_unlock(&pipe->capq.vb_lock);
-		}
-
-		vm_mem->vaddr = handle;
-
-		buf->flags &= ~V4L2_BUF_FLAG_MAPPED;
-		buf->flags |= V4L2_BUF_FLAG_QUEUED;
-		buf->flags &= ~V4L2_BUF_FLAG_DONE;
-	} else if (buf->memory == V4L2_MEMORY_MMAP) {
-		buf->flags |= V4L2_BUF_FLAG_MAPPED;
-		buf->flags |= V4L2_BUF_FLAG_QUEUED;
-		buf->flags &= ~V4L2_BUF_FLAG_DONE;
-	}
-
-done:
-	if (!((buf->flags & NOFLUSH_FLAGS) == NOFLUSH_FLAGS))
-		wbinvd();
-
-	if (!atomisp_is_vf_pipe(pipe) &&
-	    (buf->reserved2 & ATOMISP_BUFFER_HAS_PER_FRAME_SETTING)) {
-		/* this buffer will have a per-frame parameter */
-		pipe->frame_request_config_id[buf->index] = buf->reserved2 &
-					~ATOMISP_BUFFER_HAS_PER_FRAME_SETTING;
-		dev_dbg(isp->dev, "This buffer requires per_frame setting which has isp_config_id %d\n",
-			pipe->frame_request_config_id[buf->index]);
-	} else {
-		pipe->frame_request_config_id[buf->index] = 0;
-	}
-
-	pipe->frame_params[buf->index] = NULL;
-
-	rt_mutex_unlock(&isp->mutex);
-
-	ret = videobuf_qbuf(&pipe->capq, buf);
-	rt_mutex_lock(&isp->mutex);
-	if (ret)
-		goto error;
-
-	/* TODO: do this better, not best way to queue to css */
-	if (asd->streaming == ATOMISP_DEVICE_STREAMING_ENABLED) {
-		if (!list_empty(&pipe->buffers_waiting_for_param)) {
-			atomisp_handle_parameter_and_buffer(pipe);
-		} else {
-			atomisp_qbuffers_to_css(asd);
-
-#ifndef ISP2401
-			if (!atomisp_is_wdt_running(asd) && atomisp_buffers_queued(asd))
-				atomisp_wdt_start(asd);
-#else
-			if (!atomisp_is_wdt_running(pipe) &&
-				atomisp_buffers_queued_pipe(pipe))
-				atomisp_wdt_start(pipe);
-#endif
-		}
-	}
-
-	/* Workaround: Due to the design of HALv3,
-	 * sometimes in ZSL or SDV mode HAL needs to
-	 * capture multiple images within one streaming cycle.
-	 * But the capture number cannot be determined by HAL.
-	 * So HAL only sets the capture number to be 1 and queue multiple
-	 * buffers. Atomisp driver needs to check this case and re-trigger
-	 * CSS to do capture when new buffer is queued. */
-	if (asd->continuous_mode->val &&
-	    atomisp_subdev_source_pad(vdev)
-	    == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE &&
-	    pipe->capq.streaming &&
-	    !asd->enable_raw_buffer_lock->val &&
-	    asd->params.offline_parm.num_captures == 1) {
-#ifndef ISP2401
-		asd->pending_capture_request++;
-		dev_dbg(isp->dev, "Add one pending capture request.\n");
-#else
-	    if (asd->re_trigger_capture) {
-			ret = atomisp_css_offline_capture_configure(asd,
-				asd->params.offline_parm.num_captures,
-				asd->params.offline_parm.skip_frames,
-				asd->params.offline_parm.offset);
-			asd->re_trigger_capture = false;
-			dev_dbg(isp->dev, "%s Trigger capture again ret=%d\n",
-				__func__, ret);
-
-	    } else {
-			asd->pending_capture_request++;
-			asd->re_trigger_capture = false;
-			dev_dbg(isp->dev, "Add one pending capture request.\n");
-	    }
-#endif
-	}
-	rt_mutex_unlock(&isp->mutex);
-
-	dev_dbg(isp->dev, "qbuf buffer %d (%s) for asd%d\n", buf->index,
-		vdev->name, asd->index);
-
-	return ret;
-
-error:
-	rt_mutex_unlock(&isp->mutex);
-	return ret;
-}
-
-static int atomisp_qbuf_file(struct file *file, void *fh,
-					struct v4l2_buffer *buf)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-	int ret;
-
-	rt_mutex_lock(&isp->mutex);
-	if (isp->isp_fatal_error) {
-		ret = -EIO;
-		goto error;
-	}
-
-	if (!buf || buf->index >= VIDEO_MAX_FRAME ||
-		!pipe->outq.bufs[buf->index]) {
-		dev_err(isp->dev, "Invalid index for qbuf.\n");
-		ret = -EINVAL;
-		goto error;
-	}
-
-	if (buf->memory != V4L2_MEMORY_MMAP) {
-		dev_err(isp->dev, "Unsupported memory method\n");
-		ret = -EINVAL;
-		goto error;
-	}
-
-	if (buf->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) {
-		dev_err(isp->dev, "Unsupported buffer type\n");
-		ret = -EINVAL;
-		goto error;
-	}
-	rt_mutex_unlock(&isp->mutex);
-
-	return videobuf_qbuf(&pipe->outq, buf);
-
-error:
-	rt_mutex_unlock(&isp->mutex);
-
-	return ret;
-}
-
-static int __get_frame_exp_id(struct atomisp_video_pipe *pipe,
-		struct v4l2_buffer *buf)
-{
-	struct videobuf_vmalloc_memory *vm_mem;
-	struct atomisp_css_frame *handle;
-	int i;
-
-	for (i = 0; pipe->capq.bufs[i]; i++) {
-		vm_mem = pipe->capq.bufs[i]->priv;
-		handle = vm_mem->vaddr;
-		if (buf->index == pipe->capq.bufs[i]->i && handle)
-			return handle->exp_id;
-	}
-	return -EINVAL;
-}
-
-/*
- * Applications call the VIDIOC_DQBUF ioctl to dequeue a filled (capturing) or
- * displayed (output buffer)from the driver's outgoing queue
- */
-static int atomisp_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-	struct atomisp_sub_device *asd = pipe->asd;
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	int ret = 0;
-
-	rt_mutex_lock(&isp->mutex);
-
-	if (isp->isp_fatal_error) {
-		rt_mutex_unlock(&isp->mutex);
-		return -EIO;
-	}
-
-	if (asd->streaming == ATOMISP_DEVICE_STREAMING_STOPPING) {
-		rt_mutex_unlock(&isp->mutex);
-		dev_err(isp->dev, "%s: reject, as ISP at stopping.\n",
-				__func__);
-		return -EIO;
-	}
-
-	rt_mutex_unlock(&isp->mutex);
-
-	ret = videobuf_dqbuf(&pipe->capq, buf, file->f_flags & O_NONBLOCK);
-	if (ret) {
-		dev_dbg(isp->dev, "<%s: %d\n", __func__, ret);
-		return ret;
-	}
-	rt_mutex_lock(&isp->mutex);
-	buf->bytesused = pipe->pix.sizeimage;
-	buf->reserved = asd->frame_status[buf->index];
-
-	/*
-	 * Hack:
-	 * Currently frame_status in the enum type which takes no more lower
-	 * 8 bit.
-	 * use bit[31:16] for exp_id as it is only in the range of 1~255
-	 */
-	buf->reserved &= 0x0000ffff;
-	if (!(buf->flags & V4L2_BUF_FLAG_ERROR))
-		buf->reserved |= __get_frame_exp_id(pipe, buf) << 16;
-	buf->reserved2 = pipe->frame_config_id[buf->index];
-	rt_mutex_unlock(&isp->mutex);
-
-	dev_dbg(isp->dev, "dqbuf buffer %d (%s) for asd%d with exp_id %d, isp_config_id %d\n",
-		buf->index, vdev->name, asd->index, buf->reserved >> 16,
-		buf->reserved2);
-	return 0;
-}
-
-enum atomisp_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device *asd)
-{
-	if (ATOMISP_USE_YUVPP(asd))
-		return CSS_PIPE_ID_YUVPP;
-
-	if (asd->continuous_mode->val) {
-		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-			return CSS_PIPE_ID_VIDEO;
-		else
-			return CSS_PIPE_ID_PREVIEW;
-	}
-
-	/*
-	 * Disable vf_pp and run CSS in video mode. This allows using ISP
-	 * scaling but it has one frame delay due to CSS internal buffering.
-	 */
-	if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER)
-		return CSS_PIPE_ID_VIDEO;
-
-	/*
-	 * Disable vf_pp and run CSS in still capture mode. In this mode
-	 * CSS does not cause extra latency with buffering, but scaling
-	 * is not available.
-	 */
-	if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT)
-		return CSS_PIPE_ID_CAPTURE;
-
-	switch (asd->run_mode->val) {
-	case ATOMISP_RUN_MODE_PREVIEW:
-		return CSS_PIPE_ID_PREVIEW;
-	case ATOMISP_RUN_MODE_VIDEO:
-		return CSS_PIPE_ID_VIDEO;
-	case ATOMISP_RUN_MODE_STILL_CAPTURE:
-		/* fall through */
-	default:
-		return CSS_PIPE_ID_CAPTURE;
-	}
-}
-
-static unsigned int atomisp_sensor_start_stream(struct atomisp_sub_device *asd)
-{
-	struct atomisp_device *isp = asd->isp;
-
-	if (isp->inputs[asd->input_curr].camera_caps->
-	    sensor[asd->sensor_curr].stream_num > 1) {
-		if (asd->high_speed_mode)
-			return 1;
-		else
-			return 2;
-	}
-
-	if (asd->vfpp->val != ATOMISP_VFPP_ENABLE ||
-	    asd->copy_mode)
-		return 1;
-
-	if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO ||
-	    (asd->run_mode->val == ATOMISP_RUN_MODE_STILL_CAPTURE &&
-	     !atomisp_is_mbuscode_raw(
-		     asd->fmt[
-			     asd->capture_pad].fmt.code) &&
-	     !asd->continuous_mode->val))
-		return 2;
-	else
-		return 1;
-}
-
-int atomisp_stream_on_master_slave_sensor(struct atomisp_device *isp,
-	bool isp_timeout)
-{
-	unsigned int master = -1, slave = -1, delay_slave = 0;
-	int i, ret;
-
-	/*
-	 * ISP only support 2 streams now so ignore multiple master/slave
-	 * case to reduce the delay between 2 stream_on calls.
-	 */
-	for (i = 0; i < isp->num_of_streams; i++) {
-		int sensor_index = isp->asd[i].input_curr;
-		if (isp->inputs[sensor_index].camera_caps->
-				sensor[isp->asd[i].sensor_curr].is_slave)
-			slave = sensor_index;
-		else
-			master = sensor_index;
-	}
-
-	if (master == -1 || slave == -1) {
-		master = ATOMISP_DEPTH_DEFAULT_MASTER_SENSOR;
-		slave = ATOMISP_DEPTH_DEFAULT_SLAVE_SENSOR;
-		dev_warn(isp->dev,
-			 "depth mode use default master=%s.slave=%s.\n",
-			 isp->inputs[master].camera->name,
-			 isp->inputs[slave].camera->name);
-	}
-
-	ret = v4l2_subdev_call(isp->inputs[master].camera, core,
-			       ioctl, ATOMISP_IOC_G_DEPTH_SYNC_COMP,
-			       &delay_slave);
-	if (ret)
-		dev_warn(isp->dev,
-			 "get depth sensor %s compensation delay failed.\n",
-			 isp->inputs[master].camera->name);
-
-	ret = v4l2_subdev_call(isp->inputs[master].camera,
-			       video, s_stream, 1);
-	if (ret) {
-		dev_err(isp->dev, "depth mode master sensor %s stream-on failed.\n",
-			isp->inputs[master].camera->name);
-		return -EINVAL;
-	}
-
-	if (delay_slave != 0)
-		udelay(delay_slave);
-
-	ret = v4l2_subdev_call(isp->inputs[slave].camera,
-			       video, s_stream, 1);
-	if (ret) {
-		dev_err(isp->dev, "depth mode slave sensor %s stream-on failed.\n",
-			isp->inputs[slave].camera->name);
-		v4l2_subdev_call(isp->inputs[master].camera, video, s_stream, 0);
-
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/* FIXME! */
-#ifndef ISP2401
-static void __wdt_on_master_slave_sensor(struct atomisp_device *isp,
-					 unsigned int wdt_duration)
-#else
-static void __wdt_on_master_slave_sensor(struct atomisp_video_pipe *pipe,
-					 unsigned int wdt_duration,
-					 bool enable)
-#endif
-{
-#ifndef ISP2401
-	if (atomisp_buffers_queued(&isp->asd[0]))
-		atomisp_wdt_refresh(&isp->asd[0], wdt_duration);
-	if (atomisp_buffers_queued(&isp->asd[1]))
-		atomisp_wdt_refresh(&isp->asd[1], wdt_duration);
-#else
-	static struct atomisp_video_pipe *pipe0;
-
-	if (enable) {
-		if (atomisp_buffers_queued_pipe(pipe0))
-			atomisp_wdt_refresh_pipe(pipe0, wdt_duration);
-		if (atomisp_buffers_queued_pipe(pipe))
-			atomisp_wdt_refresh_pipe(pipe, wdt_duration);
-	} else {
-		pipe0 = pipe;
-	}
-#endif
-}
-
-static void atomisp_pause_buffer_event(struct atomisp_device *isp)
-{
-	struct v4l2_event event = {0};
-	int i;
-
-	event.type = V4L2_EVENT_ATOMISP_PAUSE_BUFFER;
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		int sensor_index = isp->asd[i].input_curr;
-		if (isp->inputs[sensor_index].camera_caps->
-				sensor[isp->asd[i].sensor_curr].is_slave) {
-			v4l2_event_queue(isp->asd[i].subdev.devnode, &event);
-			break;
-		}
-	}
-}
-
-/* Input system HW workaround */
-/* Input system address translation corrupts burst during */
-/* invalidate. SW workaround for this is to set burst length */
-/* manually to 128 in case of 13MPx snapshot and to 1 otherwise. */
-static void atomisp_dma_burst_len_cfg(struct atomisp_sub_device *asd)
-{
-
-	struct v4l2_mbus_framefmt *sink;
-	sink = atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-				       V4L2_SUBDEV_FORMAT_ACTIVE,
-				       ATOMISP_SUBDEV_PAD_SINK);
-
-	if (sink->width * sink->height >= 4096*3072)
-		atomisp_store_uint32(DMA_BURST_SIZE_REG, 0x7F);
-	else
-		atomisp_store_uint32(DMA_BURST_SIZE_REG, 0x00);
-}
-
-/*
- * This ioctl start the capture during streaming I/O.
- */
-static int atomisp_streamon(struct file *file, void *fh,
-	enum v4l2_buf_type type)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-	struct atomisp_sub_device *asd = pipe->asd;
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	enum atomisp_css_pipe_id css_pipe_id;
-	unsigned int sensor_start_stream;
-	unsigned int wdt_duration = ATOMISP_ISP_TIMEOUT_DURATION;
-	int ret = 0;
-	unsigned long irqflags;
-
-	dev_dbg(isp->dev, "Start stream on pad %d for asd%d\n",
-		atomisp_subdev_source_pad(vdev), asd->index);
-
-	if (type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
-		dev_dbg(isp->dev, "unsupported v4l2 buf type\n");
-		return -EINVAL;
-	}
-
-	rt_mutex_lock(&isp->mutex);
-	if (isp->isp_fatal_error) {
-		ret = -EIO;
-		goto out;
-	}
-
-	if (asd->streaming == ATOMISP_DEVICE_STREAMING_STOPPING) {
-		ret = -EBUSY;
-		goto out;
-	}
-
-	if (pipe->capq.streaming)
-		goto out;
-
-	/* Input system HW workaround */
-	atomisp_dma_burst_len_cfg(asd);
-
-	/*
-	 * The number of streaming video nodes is based on which
-	 * binary is going to be run.
-	 */
-	sensor_start_stream = atomisp_sensor_start_stream(asd);
-
-	spin_lock_irqsave(&pipe->irq_lock, irqflags);
-	if (list_empty(&(pipe->capq.stream))) {
-		spin_unlock_irqrestore(&pipe->irq_lock, irqflags);
-		dev_dbg(isp->dev, "no buffer in the queue\n");
-		ret = -EINVAL;
-		goto out;
-	}
-	spin_unlock_irqrestore(&pipe->irq_lock, irqflags);
-
-	ret = videobuf_streamon(&pipe->capq);
-	if (ret)
-		goto out;
-
-	/* Reset pending capture request count. */
-	asd->pending_capture_request = 0;
-#ifdef ISP2401
-	asd->re_trigger_capture = false;
-#endif
-
-	if ((atomisp_subdev_streaming_count(asd) > sensor_start_stream) &&
-	    (!isp->inputs[asd->input_curr].camera_caps->multi_stream_ctrl)) {
-		/* trigger still capture */
-		if (asd->continuous_mode->val &&
-		    atomisp_subdev_source_pad(vdev)
-		    == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE) {
-			if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-				dev_dbg(isp->dev, "SDV last video raw buffer id: %u\n",
-					asd->latest_preview_exp_id);
-			else
-				dev_dbg(isp->dev, "ZSL last preview raw buffer id: %u\n",
-					asd->latest_preview_exp_id);
-
-			if (asd->delayed_init == ATOMISP_DELAYED_INIT_QUEUED) {
-				flush_work(&asd->delayed_init_work);
-				rt_mutex_unlock(&isp->mutex);
-				if (wait_for_completion_interruptible(
-						&asd->init_done) != 0)
-					return -ERESTARTSYS;
-				rt_mutex_lock(&isp->mutex);
-			}
-
-			/* handle per_frame_setting parameter and buffers */
-			atomisp_handle_parameter_and_buffer(pipe);
-
-			/*
-			 * only ZSL/SDV capture request will be here, raise
-			 * the ISP freq to the highest possible to minimize
-			 * the S2S latency.
-			 */
-			atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, false);
-			/*
-			 * When asd->enable_raw_buffer_lock->val is true,
-			 * An extra IOCTL is needed to call
-			 * atomisp_css_exp_id_capture and trigger real capture
-			 */
-			if (!asd->enable_raw_buffer_lock->val) {
-				ret = atomisp_css_offline_capture_configure(asd,
-					asd->params.offline_parm.num_captures,
-					asd->params.offline_parm.skip_frames,
-					asd->params.offline_parm.offset);
-				if (ret) {
-					ret = -EINVAL;
-					goto out;
-				}
-				if (asd->depth_mode->val)
-					atomisp_pause_buffer_event(isp);
-			}
-		}
-		atomisp_qbuffers_to_css(asd);
-		goto out;
-	}
-
-	if (asd->streaming == ATOMISP_DEVICE_STREAMING_ENABLED) {
-		atomisp_qbuffers_to_css(asd);
-		goto start_sensor;
-	}
-
-	css_pipe_id = atomisp_get_css_pipe_id(asd);
-
-	ret = atomisp_acc_load_extensions(asd);
-	if (ret < 0) {
-		dev_err(isp->dev, "acc extension failed to load\n");
-		goto out;
-	}
-
-	if (asd->params.css_update_params_needed) {
-		atomisp_apply_css_parameters(asd, &asd->params.css_param);
-		if (asd->params.css_param.update_flag.dz_config)
-			atomisp_css_set_dz_config(asd,
-				&asd->params.css_param.dz_config);
-		atomisp_css_update_isp_params(asd);
-		asd->params.css_update_params_needed = false;
-		memset(&asd->params.css_param.update_flag, 0,
-		       sizeof(struct atomisp_parameters));
-	}
-	asd->params.dvs_6axis = NULL;
-
-	ret = atomisp_css_start(asd, css_pipe_id, false);
-	if (ret)
-		goto out;
-
-	asd->streaming = ATOMISP_DEVICE_STREAMING_ENABLED;
-	atomic_set(&asd->sof_count, -1);
-	atomic_set(&asd->sequence, -1);
-	atomic_set(&asd->sequence_temp, -1);
-	if (isp->sw_contex.file_input)
-		wdt_duration = ATOMISP_ISP_FILE_TIMEOUT_DURATION;
-
-	asd->params.dis_proj_data_valid = false;
-	asd->latest_preview_exp_id = 0;
-	asd->postview_exp_id = 1;
-	asd->preview_exp_id = 1;
-
-	/* handle per_frame_setting parameter and buffers */
-	atomisp_handle_parameter_and_buffer(pipe);
-
-	atomisp_qbuffers_to_css(asd);
-
-	/* Only start sensor when the last streaming instance started */
-	if (atomisp_subdev_streaming_count(asd) < sensor_start_stream)
-		goto out;
-
-start_sensor:
-	if (isp->flash) {
-		asd->params.num_flash_frames = 0;
-		asd->params.flash_state = ATOMISP_FLASH_IDLE;
-		atomisp_setup_flash(asd);
-	}
-
-	if (!isp->sw_contex.file_input) {
-		atomisp_css_irq_enable(isp, CSS_IRQ_INFO_CSS_RECEIVER_SOF,
-				atomisp_css_valid_sof(isp));
-		atomisp_csi2_configure(asd);
-		/*
-		 * set freq to max when streaming count > 1 which indicate
-		 * dual camera would run
-		*/
-		if (atomisp_streaming_count(isp) > 1) {
-			if (atomisp_freq_scaling(isp,
-				ATOMISP_DFS_MODE_MAX, false) < 0)
-				dev_dbg(isp->dev, "dfs failed!\n");
-		} else {
-			if (atomisp_freq_scaling(isp,
-				ATOMISP_DFS_MODE_AUTO, false) < 0)
-				dev_dbg(isp->dev, "dfs failed!\n");
-		}
-	} else {
-		if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, false) < 0)
-			dev_dbg(isp->dev, "dfs failed!\n");
-	}
-
-	if (asd->depth_mode->val && atomisp_streaming_count(isp) ==
-			ATOMISP_DEPTH_SENSOR_STREAMON_COUNT) {
-		ret = atomisp_stream_on_master_slave_sensor(isp, false);
-		if (ret) {
-			dev_err(isp->dev, "master slave sensor stream on failed!\n");
-			goto out;
-		}
-#ifndef ISP2401
-		__wdt_on_master_slave_sensor(isp, wdt_duration);
-#else
-		__wdt_on_master_slave_sensor(pipe, wdt_duration, true);
-#endif
-		goto start_delay_wq;
-	} else if (asd->depth_mode->val && (atomisp_streaming_count(isp) <
-		   ATOMISP_DEPTH_SENSOR_STREAMON_COUNT)) {
-#ifdef ISP2401
-		__wdt_on_master_slave_sensor(pipe, wdt_duration, false);
-#endif
-		goto start_delay_wq;
-	}
-
-	/* Enable the CSI interface on ANN B0/K0 */
-	if (isp->media_dev.hw_revision >= ((ATOMISP_HW_REVISION_ISP2401 <<
-	    ATOMISP_HW_REVISION_SHIFT) | ATOMISP_HW_STEPPING_B0)) {
-		pci_write_config_word(isp->pdev, MRFLD_PCI_CSI_CONTROL,
-				      isp->saved_regs.csi_control |
-				      MRFLD_PCI_CSI_CONTROL_CSI_READY);
-	}
-
-	/* stream on the sensor */
-	ret = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-			       video, s_stream, 1);
-	if (ret) {
-		asd->streaming = ATOMISP_DEVICE_STREAMING_DISABLED;
-		ret = -EINVAL;
-		goto out;
-	}
-
-#ifndef ISP2401
-	if (atomisp_buffers_queued(asd))
-		atomisp_wdt_refresh(asd, wdt_duration);
-#else
-	if (atomisp_buffers_queued_pipe(pipe))
-		atomisp_wdt_refresh_pipe(pipe, wdt_duration);
-#endif
-
-start_delay_wq:
-	if (asd->continuous_mode->val) {
-		struct v4l2_mbus_framefmt *sink;
-
-		sink = atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-				       V4L2_SUBDEV_FORMAT_ACTIVE,
-				       ATOMISP_SUBDEV_PAD_SINK);
-
-		reinit_completion(&asd->init_done);
-		asd->delayed_init = ATOMISP_DELAYED_INIT_QUEUED;
-		queue_work(asd->delayed_init_workq, &asd->delayed_init_work);
-		atomisp_css_set_cont_prev_start_time(isp,
-				ATOMISP_CALC_CSS_PREV_OVERLAP(sink->height));
-	} else {
-		asd->delayed_init = ATOMISP_DELAYED_INIT_NOT_QUEUED;
-	}
-out:
-	rt_mutex_unlock(&isp->mutex);
-	return ret;
-}
-
-int __atomisp_streamoff(struct file *file, void *fh, enum v4l2_buf_type type)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
-	struct atomisp_sub_device *asd = pipe->asd;
-	struct atomisp_video_pipe *capture_pipe = NULL;
-	struct atomisp_video_pipe *vf_pipe = NULL;
-	struct atomisp_video_pipe *preview_pipe = NULL;
-	struct atomisp_video_pipe *video_pipe = NULL;
-	struct videobuf_buffer *vb, *_vb;
-	enum atomisp_css_pipe_id css_pipe_id;
-	int ret;
-	unsigned long flags;
-	bool first_streamoff = false;
-
-	dev_dbg(isp->dev, "Stop stream on pad %d for asd%d\n",
-		atomisp_subdev_source_pad(vdev), asd->index);
-
-	BUG_ON(!rt_mutex_is_locked(&isp->mutex));
-	BUG_ON(!mutex_is_locked(&isp->streamoff_mutex));
-
-	if (type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
-		dev_dbg(isp->dev, "unsupported v4l2 buf type\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * do only videobuf_streamoff for capture & vf pipes in
-	 * case of continuous capture
-	 */
-	if ((asd->continuous_mode->val ||
-	    isp->inputs[asd->input_curr].camera_caps->multi_stream_ctrl) &&
-	    atomisp_subdev_source_pad(vdev) !=
-		ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW &&
-	    atomisp_subdev_source_pad(vdev) !=
-		ATOMISP_SUBDEV_PAD_SOURCE_VIDEO) {
-
-		if (isp->inputs[asd->input_curr].camera_caps->multi_stream_ctrl) {
-			v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-				video, s_stream, 0);
-		} else if (atomisp_subdev_source_pad(vdev)
-		    == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE) {
-			/* stop continuous still capture if needed */
-			if (asd->params.offline_parm.num_captures == -1)
-				atomisp_css_offline_capture_configure(asd,
-						0, 0, 0);
-			atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_AUTO, false);
-		}
-		/*
-		 * Currently there is no way to flush buffers queued to css.
-		 * When doing videobuf_streamoff, active buffers will be
-		 * marked as VIDEOBUF_NEEDS_INIT. HAL will be able to use
-		 * these buffers again, and these buffers might be queued to
-		 * css more than once! Warn here, if HAL has not dequeued all
-		 * buffers back before calling streamoff.
-		 */
-		if (pipe->buffers_in_css != 0) {
-			WARN(1, "%s: buffers of vdev %s still in CSS!\n",
-			     __func__, pipe->vdev.name);
-
-			/*
-			 * Buffers remained in css maybe dequeued out in the
-			 * next stream on, while this will causes serious
-			 * issues as buffers already get invalid after
-			 * previous stream off.
-			 *
-			 * No way to flush buffers but to reset the whole css
-			 */
-			dev_warn(isp->dev, "Reset CSS to clean up css buffers.\n");
-			atomisp_css_flush(isp);
-		}
-
-		return videobuf_streamoff(&pipe->capq);
-	}
-
-	if (!pipe->capq.streaming)
-		return 0;
-
-	spin_lock_irqsave(&isp->lock, flags);
-	if (asd->streaming == ATOMISP_DEVICE_STREAMING_ENABLED) {
-		asd->streaming = ATOMISP_DEVICE_STREAMING_STOPPING;
-		first_streamoff = true;
-	}
-	spin_unlock_irqrestore(&isp->lock, flags);
-
-	if (first_streamoff) {
-		/* if other streams are running, should not disable watch dog */
-		rt_mutex_unlock(&isp->mutex);
-		atomisp_wdt_stop(asd, true);
-
-		/*
-		 * must stop sending pixels into GP_FIFO before stop
-		 * the pipeline.
-		 */
-		if (isp->sw_contex.file_input)
-			v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-					video, s_stream, 0);
-
-		rt_mutex_lock(&isp->mutex);
-		atomisp_acc_unload_extensions(asd);
-	}
-
-	spin_lock_irqsave(&isp->lock, flags);
-	if (atomisp_subdev_streaming_count(asd) == 1)
-		asd->streaming = ATOMISP_DEVICE_STREAMING_DISABLED;
-	spin_unlock_irqrestore(&isp->lock, flags);
-
-	if (!first_streamoff) {
-		ret = videobuf_streamoff(&pipe->capq);
-		if (ret)
-			return ret;
-		goto stopsensor;
-	}
-
-	atomisp_clear_css_buffer_counters(asd);
-
-	if (!isp->sw_contex.file_input)
-		atomisp_css_irq_enable(isp, CSS_IRQ_INFO_CSS_RECEIVER_SOF,
-					false);
-
-	if (asd->delayed_init == ATOMISP_DELAYED_INIT_QUEUED) {
-		cancel_work_sync(&asd->delayed_init_work);
-		asd->delayed_init = ATOMISP_DELAYED_INIT_NOT_QUEUED;
-	}
-	if (first_streamoff) {
-		css_pipe_id = atomisp_get_css_pipe_id(asd);
-		ret = atomisp_css_stop(asd, css_pipe_id, false);
-	}
-	/* cancel work queue*/
-	if (asd->video_out_capture.users) {
-		capture_pipe = &asd->video_out_capture;
-		wake_up_interruptible(&capture_pipe->capq.wait);
-	}
-	if (asd->video_out_vf.users) {
-		vf_pipe = &asd->video_out_vf;
-		wake_up_interruptible(&vf_pipe->capq.wait);
-	}
-	if (asd->video_out_preview.users) {
-		preview_pipe = &asd->video_out_preview;
-		wake_up_interruptible(&preview_pipe->capq.wait);
-	}
-	if (asd->video_out_video_capture.users) {
-		video_pipe = &asd->video_out_video_capture;
-		wake_up_interruptible(&video_pipe->capq.wait);
-	}
-	ret = videobuf_streamoff(&pipe->capq);
-	if (ret)
-		return ret;
-
-	/* cleanup css here */
-	/* no need for this, as ISP will be reset anyway */
-	/*atomisp_flush_bufs_in_css(isp);*/
-
-	spin_lock_irqsave(&pipe->irq_lock, flags);
-	list_for_each_entry_safe(vb, _vb, &pipe->activeq, queue) {
-		vb->state = VIDEOBUF_PREPARED;
-		list_del(&vb->queue);
-	}
-	list_for_each_entry_safe(vb, _vb, &pipe->buffers_waiting_for_param, queue) {
-		vb->state = VIDEOBUF_PREPARED;
-		list_del(&vb->queue);
-		pipe->frame_request_config_id[vb->i] = 0;
-	}
-	spin_unlock_irqrestore(&pipe->irq_lock, flags);
-
-	atomisp_subdev_cleanup_pending_events(asd);
-stopsensor:
-	if (atomisp_subdev_streaming_count(asd) + 1
-	    != atomisp_sensor_start_stream(asd))
-		return 0;
-
-	if (!isp->sw_contex.file_input)
-		ret = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-				       video, s_stream, 0);
-
-	if (isp->flash) {
-		asd->params.num_flash_frames = 0;
-		asd->params.flash_state = ATOMISP_FLASH_IDLE;
-	}
-
-	/* if other streams are running, isp should not be powered off */
-	if (atomisp_streaming_count(isp)) {
-		atomisp_css_flush(isp);
-		return 0;
-	}
-
-	/* Disable the CSI interface on ANN B0/K0 */
-	if (isp->media_dev.hw_revision >= ((ATOMISP_HW_REVISION_ISP2401 <<
-	    ATOMISP_HW_REVISION_SHIFT) | ATOMISP_HW_STEPPING_B0)) {
-		pci_write_config_word(isp->pdev, MRFLD_PCI_CSI_CONTROL,
-				      isp->saved_regs.csi_control &
-				      ~MRFLD_PCI_CSI_CONTROL_CSI_READY);
-	}
-
-	if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_LOW, false))
-		dev_warn(isp->dev, "DFS failed.\n");
-	/*
-	 * ISP work around, need to reset isp
-	 * Is it correct time to reset ISP when first node does streamoff?
-	 */
-	if (isp->sw_contex.power_state == ATOM_ISP_POWER_UP) {
-		unsigned int i;
-		bool recreate_streams[MAX_STREAM_NUM] = {0};
-		if (isp->isp_timeout)
-			dev_err(isp->dev, "%s: Resetting with WA activated",
-				__func__);
-		/*
-		 * It is possible that the other asd stream is in the stage
-		 * that v4l2_setfmt is just get called on it, which will
-		 * create css stream on that stream. But at this point, there
-		 * is no way to destroy the css stream created on that stream.
-		 *
-		 * So force stream destroy here.
-		 */
-		for (i = 0; i < isp->num_of_streams; i++) {
-			if (isp->asd[i].stream_prepared) {
-				atomisp_destroy_pipes_stream_force(&isp->
-						asd[i]);
-				recreate_streams[i] = true;
-			}
-		}
-
-		/* disable  PUNIT/ISP acknowlede/handshake - SRSE=3 */
-		pci_write_config_dword(isp->pdev, PCI_I_CONTROL, isp->saved_regs.i_control |
-				MRFLD_PCI_I_CONTROL_SRSE_RESET_MASK);
-		dev_err(isp->dev, "atomisp_reset");
-		atomisp_reset(isp);
-		for (i = 0; i < isp->num_of_streams; i++) {
-			if (recreate_streams[i])
-				atomisp_create_pipes_stream(&isp->asd[i]);
-		}
-		isp->isp_timeout = false;
-	}
-	return ret;
-}
-
-static int atomisp_streamoff(struct file *file, void *fh,
-			     enum v4l2_buf_type type)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	int rval;
-
-	mutex_lock(&isp->streamoff_mutex);
-	rt_mutex_lock(&isp->mutex);
-	rval = __atomisp_streamoff(file, fh, type);
-	rt_mutex_unlock(&isp->mutex);
-	mutex_unlock(&isp->streamoff_mutex);
-
-	return rval;
-}
-
-/*
- * To get the current value of a control.
- * applications initialize the id field of a struct v4l2_control and
- * call this ioctl with a pointer to this structure
- */
-static int atomisp_g_ctrl(struct file *file, void *fh,
-	struct v4l2_control *control)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	int i, ret = -EINVAL;
-
-	for (i = 0; i < ctrls_num; i++) {
-		if (ci_v4l2_controls[i].id == control->id) {
-			ret = 0;
-			break;
-		}
-	}
-
-	if (ret)
-		return ret;
-
-	rt_mutex_lock(&isp->mutex);
-
-	switch (control->id) {
-	case V4L2_CID_IRIS_ABSOLUTE:
-	case V4L2_CID_EXPOSURE_ABSOLUTE:
-	case V4L2_CID_FNUMBER_ABSOLUTE:
-	case V4L2_CID_2A_STATUS:
-	case V4L2_CID_AUTO_N_PRESET_WHITE_BALANCE:
-	case V4L2_CID_EXPOSURE:
-	case V4L2_CID_EXPOSURE_AUTO:
-	case V4L2_CID_SCENE_MODE:
-	case V4L2_CID_ISO_SENSITIVITY:
-	case V4L2_CID_ISO_SENSITIVITY_AUTO:
-	case V4L2_CID_CONTRAST:
-	case V4L2_CID_SATURATION:
-	case V4L2_CID_SHARPNESS:
-	case V4L2_CID_3A_LOCK:
-	case V4L2_CID_EXPOSURE_ZONE_NUM:
-	case V4L2_CID_TEST_PATTERN:
-	case V4L2_CID_TEST_PATTERN_COLOR_R:
-	case V4L2_CID_TEST_PATTERN_COLOR_GR:
-	case V4L2_CID_TEST_PATTERN_COLOR_GB:
-	case V4L2_CID_TEST_PATTERN_COLOR_B:
-		rt_mutex_unlock(&isp->mutex);
-		return v4l2_g_ctrl(isp->inputs[asd->input_curr].camera->
-				   ctrl_handler, control);
-	case V4L2_CID_COLORFX:
-		ret = atomisp_color_effect(asd, 0, &control->value);
-		break;
-	case V4L2_CID_ATOMISP_BAD_PIXEL_DETECTION:
-		ret = atomisp_bad_pixel(asd, 0, &control->value);
-		break;
-	case V4L2_CID_ATOMISP_POSTPROCESS_GDC_CAC:
-		ret = atomisp_gdc_cac(asd, 0, &control->value);
-		break;
-	case V4L2_CID_ATOMISP_VIDEO_STABLIZATION:
-		ret = atomisp_video_stable(asd, 0, &control->value);
-		break;
-	case V4L2_CID_ATOMISP_FIXED_PATTERN_NR:
-		ret = atomisp_fixed_pattern(asd, 0, &control->value);
-		break;
-	case V4L2_CID_ATOMISP_FALSE_COLOR_CORRECTION:
-		ret = atomisp_false_color(asd, 0, &control->value);
-		break;
-	case V4L2_CID_ATOMISP_LOW_LIGHT:
-		ret = atomisp_low_light(asd, 0, &control->value);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-
-	rt_mutex_unlock(&isp->mutex);
-	return ret;
-}
-
-/*
- * To change the value of a control.
- * applications initialize the id and value fields of a struct v4l2_control
- * and call this ioctl.
- */
-static int atomisp_s_ctrl(struct file *file, void *fh,
-			  struct v4l2_control *control)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	int i, ret = -EINVAL;
-
-	for (i = 0; i < ctrls_num; i++) {
-		if (ci_v4l2_controls[i].id == control->id) {
-			ret = 0;
-			break;
-		}
-	}
-
-	if (ret)
-		return ret;
-
-	rt_mutex_lock(&isp->mutex);
-	switch (control->id) {
-	case V4L2_CID_AUTO_N_PRESET_WHITE_BALANCE:
-	case V4L2_CID_EXPOSURE:
-	case V4L2_CID_EXPOSURE_AUTO:
-	case V4L2_CID_EXPOSURE_AUTO_PRIORITY:
-	case V4L2_CID_SCENE_MODE:
-	case V4L2_CID_ISO_SENSITIVITY:
-	case V4L2_CID_ISO_SENSITIVITY_AUTO:
-	case V4L2_CID_POWER_LINE_FREQUENCY:
-	case V4L2_CID_EXPOSURE_METERING:
-	case V4L2_CID_CONTRAST:
-	case V4L2_CID_SATURATION:
-	case V4L2_CID_SHARPNESS:
-	case V4L2_CID_3A_LOCK:
-	case V4L2_CID_COLORFX_CBCR:
-	case V4L2_CID_TEST_PATTERN:
-	case V4L2_CID_TEST_PATTERN_COLOR_R:
-	case V4L2_CID_TEST_PATTERN_COLOR_GR:
-	case V4L2_CID_TEST_PATTERN_COLOR_GB:
-	case V4L2_CID_TEST_PATTERN_COLOR_B:
-		rt_mutex_unlock(&isp->mutex);
-		return v4l2_s_ctrl(NULL,
-				   isp->inputs[asd->input_curr].camera->
-				   ctrl_handler, control);
-	case V4L2_CID_COLORFX:
-		ret = atomisp_color_effect(asd, 1, &control->value);
-		break;
-	case V4L2_CID_ATOMISP_BAD_PIXEL_DETECTION:
-		ret = atomisp_bad_pixel(asd, 1, &control->value);
-		break;
-	case V4L2_CID_ATOMISP_POSTPROCESS_GDC_CAC:
-		ret = atomisp_gdc_cac(asd, 1, &control->value);
-		break;
-	case V4L2_CID_ATOMISP_VIDEO_STABLIZATION:
-		ret = atomisp_video_stable(asd, 1, &control->value);
-		break;
-	case V4L2_CID_ATOMISP_FIXED_PATTERN_NR:
-		ret = atomisp_fixed_pattern(asd, 1, &control->value);
-		break;
-	case V4L2_CID_ATOMISP_FALSE_COLOR_CORRECTION:
-		ret = atomisp_false_color(asd, 1, &control->value);
-		break;
-	case V4L2_CID_REQUEST_FLASH:
-		ret = atomisp_flash_enable(asd, control->value);
-		break;
-	case V4L2_CID_ATOMISP_LOW_LIGHT:
-		ret = atomisp_low_light(asd, 1, &control->value);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-	rt_mutex_unlock(&isp->mutex);
-	return ret;
-}
-/*
- * To query the attributes of a control.
- * applications set the id field of a struct v4l2_queryctrl and call the
- * this ioctl with a pointer to this structure. The driver fills
- * the rest of the structure.
- */
-static int atomisp_queryctl(struct file *file, void *fh,
-			    struct v4l2_queryctrl *qc)
-{
-	int i, ret = -EINVAL;
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-
-	switch (qc->id) {
-	case V4L2_CID_FOCUS_ABSOLUTE:
-	case V4L2_CID_FOCUS_RELATIVE:
-	case V4L2_CID_FOCUS_STATUS:
-#ifndef ISP2401
-		return v4l2_queryctrl(isp->inputs[asd->input_curr].camera->
-				      ctrl_handler, qc);
-#else
-		if (isp->motor)
-			return v4l2_queryctrl(isp->motor->ctrl_handler, qc);
-		else
-			return v4l2_queryctrl(isp->inputs[asd->input_curr].
-					      camera->ctrl_handler, qc);
-#endif
-	}
-
-	if (qc->id & V4L2_CTRL_FLAG_NEXT_CTRL)
-		return ret;
-
-	for (i = 0; i < ctrls_num; i++) {
-		if (ci_v4l2_controls[i].id == qc->id) {
-			memcpy(qc, &ci_v4l2_controls[i],
-			       sizeof(struct v4l2_queryctrl));
-			qc->reserved[0] = 0;
-			ret = 0;
-			break;
-		}
-	}
-	if (ret != 0)
-		qc->flags = V4L2_CTRL_FLAG_DISABLED;
-
-	return ret;
-}
-
-static int atomisp_camera_g_ext_ctrls(struct file *file, void *fh,
-	struct v4l2_ext_controls *c)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct v4l2_control ctrl;
-	int i;
-	int ret = 0;
-
-	for (i = 0; i < c->count; i++) {
-		ctrl.id = c->controls[i].id;
-		ctrl.value = c->controls[i].value;
-		switch (ctrl.id) {
-		case V4L2_CID_EXPOSURE_ABSOLUTE:
-		case V4L2_CID_EXPOSURE_AUTO:
-		case V4L2_CID_IRIS_ABSOLUTE:
-		case V4L2_CID_FNUMBER_ABSOLUTE:
-		case V4L2_CID_BIN_FACTOR_HORZ:
-		case V4L2_CID_BIN_FACTOR_VERT:
-		case V4L2_CID_3A_LOCK:
-		case V4L2_CID_TEST_PATTERN:
-		case V4L2_CID_TEST_PATTERN_COLOR_R:
-		case V4L2_CID_TEST_PATTERN_COLOR_GR:
-		case V4L2_CID_TEST_PATTERN_COLOR_GB:
-		case V4L2_CID_TEST_PATTERN_COLOR_B:
-			/*
-			 * Exposure related control will be handled by sensor
-			 * driver
-			 */
-			ret =
-			    v4l2_g_ctrl(isp->inputs[asd->input_curr].camera->
-					ctrl_handler, &ctrl);
-			break;
-		case V4L2_CID_FOCUS_ABSOLUTE:
-		case V4L2_CID_FOCUS_RELATIVE:
-		case V4L2_CID_FOCUS_STATUS:
-		case V4L2_CID_FOCUS_AUTO:
-#ifndef ISP2401
-			if (isp->inputs[asd->input_curr].motor)
-#else
-			if (isp->motor)
-#endif
-				ret =
-#ifndef ISP2401
-				    v4l2_g_ctrl(isp->inputs[asd->input_curr].
-						motor->ctrl_handler, &ctrl);
-#else
-				    v4l2_g_ctrl(isp->motor->ctrl_handler,
-						&ctrl);
-#endif
-			else
-				ret =
-				    v4l2_g_ctrl(isp->inputs[asd->input_curr].
-						camera->ctrl_handler, &ctrl);
-			break;
-		case V4L2_CID_FLASH_STATUS:
-		case V4L2_CID_FLASH_INTENSITY:
-		case V4L2_CID_FLASH_TORCH_INTENSITY:
-		case V4L2_CID_FLASH_INDICATOR_INTENSITY:
-		case V4L2_CID_FLASH_TIMEOUT:
-		case V4L2_CID_FLASH_STROBE:
-		case V4L2_CID_FLASH_MODE:
-		case V4L2_CID_FLASH_STATUS_REGISTER:
-			if (isp->flash)
-				ret =
-				    v4l2_g_ctrl(isp->flash->ctrl_handler,
-						&ctrl);
-			break;
-		case V4L2_CID_ZOOM_ABSOLUTE:
-			rt_mutex_lock(&isp->mutex);
-			ret = atomisp_digital_zoom(asd, 0, &ctrl.value);
-			rt_mutex_unlock(&isp->mutex);
-			break;
-		case V4L2_CID_G_SKIP_FRAMES:
-			ret = v4l2_subdev_call(
-				isp->inputs[asd->input_curr].camera,
-				sensor, g_skip_frames, (u32 *)&ctrl.value);
-			break;
-		default:
-			ret = -EINVAL;
-		}
-
-		if (ret) {
-			c->error_idx = i;
-			break;
-		}
-		c->controls[i].value = ctrl.value;
-	}
-	return ret;
-}
-
-/* This ioctl allows the application to get multiple controls by class */
-static int atomisp_g_ext_ctrls(struct file *file, void *fh,
-	struct v4l2_ext_controls *c)
-{
-	struct v4l2_control ctrl;
-	int i, ret = 0;
-
-	/* input_lock is not need for the Camera releated IOCTLs
-	 * The input_lock downgrade the FPS of 3A*/
-	ret = atomisp_camera_g_ext_ctrls(file, fh, c);
-	if (ret != -EINVAL)
-		return ret;
-
-	for (i = 0; i < c->count; i++) {
-		ctrl.id = c->controls[i].id;
-		ctrl.value = c->controls[i].value;
-		ret = atomisp_g_ctrl(file, fh, &ctrl);
-		c->controls[i].value = ctrl.value;
-		if (ret) {
-			c->error_idx = i;
-			break;
-		}
-	}
-	return ret;
-}
-
-static int atomisp_camera_s_ext_ctrls(struct file *file, void *fh,
-	struct v4l2_ext_controls *c)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct v4l2_control ctrl;
-	int i;
-	int ret = 0;
-
-	for (i = 0; i < c->count; i++) {
-		struct v4l2_ctrl *ctr;
-
-		ctrl.id = c->controls[i].id;
-		ctrl.value = c->controls[i].value;
-		switch (ctrl.id) {
-		case V4L2_CID_EXPOSURE_ABSOLUTE:
-		case V4L2_CID_EXPOSURE_AUTO:
-		case V4L2_CID_EXPOSURE_METERING:
-		case V4L2_CID_IRIS_ABSOLUTE:
-		case V4L2_CID_FNUMBER_ABSOLUTE:
-		case V4L2_CID_VCM_TIMEING:
-		case V4L2_CID_VCM_SLEW:
-		case V4L2_CID_3A_LOCK:
-		case V4L2_CID_TEST_PATTERN:
-		case V4L2_CID_TEST_PATTERN_COLOR_R:
-		case V4L2_CID_TEST_PATTERN_COLOR_GR:
-		case V4L2_CID_TEST_PATTERN_COLOR_GB:
-		case V4L2_CID_TEST_PATTERN_COLOR_B:
-			ret = v4l2_s_ctrl(NULL,
-					  isp->inputs[asd->input_curr].camera->
-					  ctrl_handler, &ctrl);
-			break;
-		case V4L2_CID_FOCUS_ABSOLUTE:
-		case V4L2_CID_FOCUS_RELATIVE:
-		case V4L2_CID_FOCUS_STATUS:
-		case V4L2_CID_FOCUS_AUTO:
-#ifndef ISP2401
-			if (isp->inputs[asd->input_curr].motor)
-#else
-			if (isp->motor)
-#endif
-				ret = v4l2_s_ctrl(NULL,
-#ifndef ISP2401
-						  isp->inputs[asd->input_curr].
-						  motor->ctrl_handler, &ctrl);
-#else
-						  isp->motor->ctrl_handler,
-						  &ctrl);
-#endif
-			else
-				ret = v4l2_s_ctrl(NULL,
-						  isp->inputs[asd->input_curr].
-						  camera->ctrl_handler, &ctrl);
-			break;
-		case V4L2_CID_FLASH_STATUS:
-		case V4L2_CID_FLASH_INTENSITY:
-		case V4L2_CID_FLASH_TORCH_INTENSITY:
-		case V4L2_CID_FLASH_INDICATOR_INTENSITY:
-		case V4L2_CID_FLASH_TIMEOUT:
-		case V4L2_CID_FLASH_STROBE:
-		case V4L2_CID_FLASH_MODE:
-		case V4L2_CID_FLASH_STATUS_REGISTER:
-			rt_mutex_lock(&isp->mutex);
-			if (isp->flash) {
-				ret =
-				    v4l2_s_ctrl(NULL, isp->flash->ctrl_handler,
-						&ctrl);
-				/* When flash mode is changed we need to reset
-				 * flash state */
-				if (ctrl.id == V4L2_CID_FLASH_MODE) {
-					asd->params.flash_state =
-						ATOMISP_FLASH_IDLE;
-					asd->params.num_flash_frames = 0;
-				}
-			}
-			rt_mutex_unlock(&isp->mutex);
-			break;
-		case V4L2_CID_ZOOM_ABSOLUTE:
-			rt_mutex_lock(&isp->mutex);
-			ret = atomisp_digital_zoom(asd, 1, &ctrl.value);
-			rt_mutex_unlock(&isp->mutex);
-			break;
-		default:
-			ctr = v4l2_ctrl_find(&asd->ctrl_handler, ctrl.id);
-			if (ctr)
-				ret = v4l2_ctrl_s_ctrl(ctr, ctrl.value);
-			else
-				ret = -EINVAL;
-		}
-
-		if (ret) {
-			c->error_idx = i;
-			break;
-		}
-		c->controls[i].value = ctrl.value;
-	}
-	return ret;
-}
-
-/* This ioctl allows the application to set multiple controls by class */
-static int atomisp_s_ext_ctrls(struct file *file, void *fh,
-	struct v4l2_ext_controls *c)
-{
-	struct v4l2_control ctrl;
-	int i, ret = 0;
-
-	/* input_lock is not need for the Camera releated IOCTLs
-	 * The input_lock downgrade the FPS of 3A*/
-	ret = atomisp_camera_s_ext_ctrls(file, fh, c);
-	if (ret != -EINVAL)
-		return ret;
-
-	for (i = 0; i < c->count; i++) {
-		ctrl.id = c->controls[i].id;
-		ctrl.value = c->controls[i].value;
-		ret = atomisp_s_ctrl(file, fh, &ctrl);
-		c->controls[i].value = ctrl.value;
-		if (ret) {
-			c->error_idx = i;
-			break;
-		}
-	}
-	return ret;
-}
-
-/*
- * vidioc_g/s_param are used to switch isp running mode
- */
-static int atomisp_g_parm(struct file *file, void *fh,
-	struct v4l2_streamparm *parm)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-
-	if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
-		dev_err(isp->dev, "unsupport v4l2 buf type\n");
-		return -EINVAL;
-	}
-
-	rt_mutex_lock(&isp->mutex);
-	parm->parm.capture.capturemode = asd->run_mode->val;
-	rt_mutex_unlock(&isp->mutex);
-
-	return 0;
-}
-
-static int atomisp_s_parm(struct file *file, void *fh,
-	struct v4l2_streamparm *parm)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
-	int mode;
-	int rval;
-	int fps;
-
-	if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
-		dev_err(isp->dev, "unsupport v4l2 buf type\n");
-		return -EINVAL;
-	}
-
-	rt_mutex_lock(&isp->mutex);
-
-	asd->high_speed_mode = false;
-	switch (parm->parm.capture.capturemode) {
-	case CI_MODE_NONE: {
-		struct v4l2_subdev_frame_interval fi = {0};
-
-		fi.interval = parm->parm.capture.timeperframe;
-
-		rval = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-					video, s_frame_interval, &fi);
-		if (!rval)
-			parm->parm.capture.timeperframe = fi.interval;
-
-		if (fi.interval.numerator != 0) {
-			fps = fi.interval.denominator / fi.interval.numerator;
-			if (fps > 30)
-				asd->high_speed_mode = true;
-		}
-
-		goto out;
-	}
-	case CI_MODE_VIDEO:
-		mode = ATOMISP_RUN_MODE_VIDEO;
-		break;
-	case CI_MODE_STILL_CAPTURE:
-		mode = ATOMISP_RUN_MODE_STILL_CAPTURE;
-		break;
-	case CI_MODE_CONTINUOUS:
-		mode = ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE;
-		break;
-	case CI_MODE_PREVIEW:
-		mode = ATOMISP_RUN_MODE_PREVIEW;
-		break;
-	default:
-		rval = -EINVAL;
-		goto out;
-	}
-
-	rval = v4l2_ctrl_s_ctrl(asd->run_mode, mode);
-
-out:
-	rt_mutex_unlock(&isp->mutex);
-
-	return rval == -ENOIOCTLCMD ? 0 : rval;
-}
-
-static int atomisp_s_parm_file(struct file *file, void *fh,
-				struct v4l2_streamparm *parm)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-
-	if (parm->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) {
-		dev_err(isp->dev, "unsupport v4l2 buf type for output\n");
-		return -EINVAL;
-	}
-
-	rt_mutex_lock(&isp->mutex);
-	isp->sw_contex.file_input = true;
-	rt_mutex_unlock(&isp->mutex);
-
-	return 0;
-}
-
-static long atomisp_vidioc_default(struct file *file, void *fh,
-	bool valid_prio, unsigned int cmd, void *arg)
-{
-	struct video_device *vdev = video_devdata(file);
-	struct atomisp_device *isp = video_get_drvdata(vdev);
-	struct atomisp_sub_device *asd;
-	bool acc_node;
-	int err;
-
-	acc_node = !strncmp(vdev->name, "ATOMISP ISP ACC",
-			sizeof(vdev->name));
-	if (acc_node)
-		asd = atomisp_to_acc_pipe(vdev)->asd;
-	else
-		asd = atomisp_to_video_pipe(vdev)->asd;
-
-	switch (cmd) {
-	case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA:
-	case ATOMISP_IOC_S_EXPOSURE:
-	case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP:
-	case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA:
-	case ATOMISP_IOC_EXT_ISP_CTRL:
-	case ATOMISP_IOC_G_SENSOR_AE_BRACKETING_INFO:
-	case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_MODE:
-	case ATOMISP_IOC_G_SENSOR_AE_BRACKETING_MODE:
-	case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT:
-	case ATOMISP_IOC_S_SENSOR_EE_CONFIG:
-#ifdef ISP2401
-	case ATOMISP_IOC_G_UPDATE_EXPOSURE:
-#endif
-		/* we do not need take isp->mutex for these IOCTLs */
-		break;
-	default:
-		rt_mutex_lock(&isp->mutex);
-		break;
-	}
-	switch (cmd) {
-#ifdef ISP2401
-	case ATOMISP_IOC_S_SENSOR_RUNMODE:
-		err = atomisp_set_sensor_runmode(asd, arg);
-		break;
-
-#endif
-	case ATOMISP_IOC_G_XNR:
-		err = atomisp_xnr(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_XNR:
-		err = atomisp_xnr(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_NR:
-		err = atomisp_nr(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_NR:
-		err = atomisp_nr(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_TNR:
-		err = atomisp_tnr(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_TNR:
-		err = atomisp_tnr(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_BLACK_LEVEL_COMP:
-		err = atomisp_black_level(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_BLACK_LEVEL_COMP:
-		err = atomisp_black_level(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_EE:
-		err = atomisp_ee(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_EE:
-		err = atomisp_ee(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_DIS_STAT:
-		err = atomisp_get_dis_stat(asd, arg);
-		break;
-
-	case ATOMISP_IOC_G_DVS2_BQ_RESOLUTIONS:
-		err = atomisp_get_dvs2_bq_resolutions(asd, arg);
-		break;
-
-	case ATOMISP_IOC_S_DIS_COEFS:
-		err = atomisp_css_cp_dvs2_coefs(asd, arg,
-				&asd->params.css_param, true);
-		if (!err && arg)
-			asd->params.css_update_params_needed = true;
-		break;
-
-	case ATOMISP_IOC_S_DIS_VECTOR:
-		err = atomisp_cp_dvs_6axis_config(asd, arg,
-				&asd->params.css_param, true);
-		if (!err && arg)
-			asd->params.css_update_params_needed = true;
-		break;
-
-	case ATOMISP_IOC_G_ISP_PARM:
-		err = atomisp_param(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_ISP_PARM:
-		err = atomisp_param(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_3A_STAT:
-		err = atomisp_3a_stat(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_G_ISP_GAMMA:
-		err = atomisp_gamma(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_ISP_GAMMA:
-		err = atomisp_gamma(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_ISP_GDC_TAB:
-		err = atomisp_gdc_cac_table(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_ISP_GDC_TAB:
-		err = atomisp_gdc_cac_table(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_ISP_MACC:
-		err = atomisp_macc_table(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_ISP_MACC:
-		err = atomisp_macc_table(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_ISP_BAD_PIXEL_DETECTION:
-		err = atomisp_bad_pixel_param(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_ISP_BAD_PIXEL_DETECTION:
-		err = atomisp_bad_pixel_param(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_ISP_FALSE_COLOR_CORRECTION:
-		err = atomisp_false_color_param(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_ISP_FALSE_COLOR_CORRECTION:
-		err = atomisp_false_color_param(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_ISP_CTC:
-		err = atomisp_ctc(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_ISP_CTC:
-		err = atomisp_ctc(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_ISP_WHITE_BALANCE:
-		err = atomisp_white_balance_param(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_ISP_WHITE_BALANCE:
-		err = atomisp_white_balance_param(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_G_3A_CONFIG:
-		err = atomisp_3a_config_param(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_3A_CONFIG:
-		err = atomisp_3a_config_param(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_S_ISP_FPN_TABLE:
-		err = atomisp_fixed_pattern_table(asd, arg);
-		break;
-
-	case ATOMISP_IOC_ISP_MAKERNOTE:
-		err = atomisp_exif_makernote(asd, arg);
-		break;
-
-	case ATOMISP_IOC_G_SENSOR_MODE_DATA:
-		err = atomisp_get_sensor_mode_data(asd, arg);
-		break;
-
-	case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA:
-#ifndef ISP2401
-		if (isp->inputs[asd->input_curr].motor)
-#else
-		if (isp->motor)
-#endif
-#ifndef ISP2401
-			err = v4l2_subdev_call(
-					isp->inputs[asd->input_curr].motor,
-					core, ioctl, cmd, arg);
-#else
-			err = v4l2_subdev_call(
-					isp->motor,
-					core, ioctl, cmd, arg);
-#endif
-		else
-			err = v4l2_subdev_call(
-					isp->inputs[asd->input_curr].camera,
-					core, ioctl, cmd, arg);
-		break;
-
-	case ATOMISP_IOC_S_EXPOSURE:
-	case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP:
-	case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA:
-	case ATOMISP_IOC_G_SENSOR_AE_BRACKETING_INFO:
-	case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_MODE:
-	case ATOMISP_IOC_G_SENSOR_AE_BRACKETING_MODE:
-	case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT:
-#ifdef ISP2401
-	case ATOMISP_IOC_G_UPDATE_EXPOSURE:
-#endif
-		err = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-					core, ioctl, cmd, arg);
-		break;
-
-	case ATOMISP_IOC_ACC_LOAD:
-		err = atomisp_acc_load(asd, arg);
-		break;
-
-	case ATOMISP_IOC_ACC_LOAD_TO_PIPE:
-		err = atomisp_acc_load_to_pipe(asd, arg);
-		break;
-
-	case ATOMISP_IOC_ACC_UNLOAD:
-		err = atomisp_acc_unload(asd, arg);
-		break;
-
-	case ATOMISP_IOC_ACC_START:
-		err = atomisp_acc_start(asd, arg);
-		break;
-
-	case ATOMISP_IOC_ACC_WAIT:
-		err = atomisp_acc_wait(asd, arg);
-		break;
-
-	case ATOMISP_IOC_ACC_MAP:
-		err = atomisp_acc_map(asd, arg);
-		break;
-
-	case ATOMISP_IOC_ACC_UNMAP:
-		err = atomisp_acc_unmap(asd, arg);
-		break;
-
-	case ATOMISP_IOC_ACC_S_MAPPED_ARG:
-		err = atomisp_acc_s_mapped_arg(asd, arg);
-		break;
-
-	case ATOMISP_IOC_S_ISP_SHD_TAB:
-		err = atomisp_set_shading_table(asd, arg);
-		break;
-
-	case ATOMISP_IOC_G_ISP_GAMMA_CORRECTION:
-		err = atomisp_gamma_correction(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_ISP_GAMMA_CORRECTION:
-		err = atomisp_gamma_correction(asd, 1, arg);
-		break;
-
-	case ATOMISP_IOC_S_PARAMETERS:
-		err = atomisp_set_parameters(vdev, arg);
-		break;
-
-	case ATOMISP_IOC_S_CONT_CAPTURE_CONFIG:
-		err = atomisp_offline_capture_configure(asd, arg);
-		break;
-	case ATOMISP_IOC_G_METADATA:
-		err = atomisp_get_metadata(asd, 0, arg);
-		break;
-	case ATOMISP_IOC_G_METADATA_BY_TYPE:
-		err = atomisp_get_metadata_by_type(asd, 0, arg);
-		break;
-	case ATOMISP_IOC_EXT_ISP_CTRL:
-		err = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
-					core, ioctl, cmd, arg);
-		break;
-	case ATOMISP_IOC_EXP_ID_UNLOCK:
-		err = atomisp_exp_id_unlock(asd, arg);
-		break;
-	case ATOMISP_IOC_EXP_ID_CAPTURE:
-		err = atomisp_exp_id_capture(asd, arg);
-		break;
-	case ATOMISP_IOC_S_ENABLE_DZ_CAPT_PIPE:
-		err = atomisp_enable_dz_capt_pipe(asd, arg);
-		break;
-	case ATOMISP_IOC_G_FORMATS_CONFIG:
-		err = atomisp_formats(asd, 0, arg);
-		break;
-
-	case ATOMISP_IOC_S_FORMATS_CONFIG:
-		err = atomisp_formats(asd, 1, arg);
-		break;
-	case ATOMISP_IOC_S_EXPOSURE_WINDOW:
-		err = atomisp_s_ae_window(asd, arg);
-		break;
-	case ATOMISP_IOC_S_ACC_STATE:
-		err = atomisp_acc_set_state(asd, arg);
-		break;
-	case ATOMISP_IOC_G_ACC_STATE:
-		err = atomisp_acc_get_state(asd, arg);
-		break;
-	case ATOMISP_IOC_INJECT_A_FAKE_EVENT:
-		err = atomisp_inject_a_fake_event(asd, arg);
-		break;
-	case ATOMISP_IOC_G_INVALID_FRAME_NUM:
-		err = atomisp_get_invalid_frame_num(vdev, arg);
-		break;
-	case ATOMISP_IOC_S_ARRAY_RESOLUTION:
-		err = atomisp_set_array_res(asd, arg);
-		break;
-	default:
-		err = -EINVAL;
-		break;
-	}
-
-	switch (cmd) {
-	case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA:
-	case ATOMISP_IOC_S_EXPOSURE:
-	case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP:
-	case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA:
-	case ATOMISP_IOC_EXT_ISP_CTRL:
-	case ATOMISP_IOC_G_SENSOR_AE_BRACKETING_INFO:
-	case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_MODE:
-	case ATOMISP_IOC_G_SENSOR_AE_BRACKETING_MODE:
-	case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT:
-#ifdef ISP2401
-	case ATOMISP_IOC_G_UPDATE_EXPOSURE:
-#endif
-		break;
-	default:
-		rt_mutex_unlock(&isp->mutex);
-		break;
-	}
-	return err;
-}
-
-const struct v4l2_ioctl_ops atomisp_ioctl_ops = {
-	.vidioc_querycap = atomisp_querycap,
-	.vidioc_enum_input = atomisp_enum_input,
-	.vidioc_g_input = atomisp_g_input,
-	.vidioc_s_input = atomisp_s_input,
-	.vidioc_queryctrl = atomisp_queryctl,
-	.vidioc_s_ctrl = atomisp_s_ctrl,
-	.vidioc_g_ctrl = atomisp_g_ctrl,
-	.vidioc_s_ext_ctrls = atomisp_s_ext_ctrls,
-	.vidioc_g_ext_ctrls = atomisp_g_ext_ctrls,
-	.vidioc_enum_fmt_vid_cap = atomisp_enum_fmt_cap,
-	.vidioc_try_fmt_vid_cap = atomisp_try_fmt_cap,
-	.vidioc_g_fmt_vid_cap = atomisp_g_fmt_cap,
-	.vidioc_s_fmt_vid_cap = atomisp_s_fmt_cap,
-	.vidioc_reqbufs = atomisp_reqbufs,
-	.vidioc_querybuf = atomisp_querybuf,
-	.vidioc_qbuf = atomisp_qbuf,
-	.vidioc_dqbuf = atomisp_dqbuf,
-	.vidioc_streamon = atomisp_streamon,
-	.vidioc_streamoff = atomisp_streamoff,
-	.vidioc_default = atomisp_vidioc_default,
-	.vidioc_s_parm = atomisp_s_parm,
-	.vidioc_g_parm = atomisp_g_parm,
-};
-
-const struct v4l2_ioctl_ops atomisp_file_ioctl_ops = {
-	.vidioc_querycap = atomisp_querycap,
-	.vidioc_g_fmt_vid_out = atomisp_g_fmt_file,
-	.vidioc_s_fmt_vid_out = atomisp_s_fmt_file,
-	.vidioc_s_parm = atomisp_s_parm_file,
-	.vidioc_reqbufs = atomisp_reqbufs_file,
-	.vidioc_querybuf = atomisp_querybuf_file,
-	.vidioc_qbuf = atomisp_qbuf_file,
-};
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.h
deleted file mode 100644
index 0d2785b9ef99..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef	__ATOMISP_IOCTL_H__
-#define	__ATOMISP_IOCTL_H__
-
-#include "ia_css.h"
-
-struct atomisp_device;
-struct atomisp_video_pipe;
-
-extern const struct atomisp_format_bridge atomisp_output_fmts[];
-
-const struct atomisp_format_bridge *atomisp_get_format_bridge(
-	unsigned int pixelformat);
-#ifndef ISP2401
-const struct atomisp_format_bridge *atomisp_get_format_bridge_from_mbus(
-	u32 mbus_code);
-#else
-const struct atomisp_format_bridge *atomisp_get_format_bridge_from_mbus(u32
-									mbus_code);
-#endif
-
-int atomisp_alloc_css_stat_bufs(struct atomisp_sub_device *asd,
-	uint16_t stream_id);
-
-int __atomisp_streamoff(struct file *file, void *fh, enum v4l2_buf_type type);
-int __atomisp_reqbufs(struct file *file, void *fh,
-		struct v4l2_requestbuffers *req);
-
-int atomisp_reqbufs(struct file *file, void *fh,
-			struct v4l2_requestbuffers *req);
-
-enum atomisp_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device
-						 *asd);
-
-void atomisp_videobuf_free_buf(struct videobuf_buffer *vb);
-
-extern const struct v4l2_file_operations atomisp_file_fops;
-
-extern const struct v4l2_ioctl_ops atomisp_ioctl_ops;
-
-extern const struct v4l2_ioctl_ops atomisp_file_ioctl_ops;
-
-unsigned int atomisp_streaming_count(struct atomisp_device *isp);
-
-unsigned int atomisp_is_acc_enabled(struct atomisp_device *isp);
-/* compat_ioctl for 32bit userland app and 64bit kernel */
-long atomisp_compat_ioctl32(struct file *file,
-			    unsigned int cmd, unsigned long arg);
-
-int atomisp_stream_on_master_slave_sensor(struct atomisp_device *isp, bool isp_timeout);
-#endif /* __ATOMISP_IOCTL_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.c
deleted file mode 100644
index 49a9973b4289..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.c
+++ /dev/null
@@ -1,1422 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#include <linux/module.h>
-#include <linux/uaccess.h>
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/mm.h>
-#include <linux/sched.h>
-#include <linux/slab.h>
-
-#include <media/v4l2-event.h>
-#include <media/v4l2-mediabus.h>
-#include "atomisp_cmd.h"
-#include "atomisp_common.h"
-#include "atomisp_compat.h"
-#include "atomisp_internal.h"
-
-const struct atomisp_in_fmt_conv atomisp_in_fmt_conv[] = {
-	{ MEDIA_BUS_FMT_SBGGR8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_BGGR, CSS_FORMAT_RAW_8 },
-	{ MEDIA_BUS_FMT_SGBRG8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_GBRG, CSS_FORMAT_RAW_8 },
-	{ MEDIA_BUS_FMT_SGRBG8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_GRBG, CSS_FORMAT_RAW_8 },
-	{ MEDIA_BUS_FMT_SRGGB8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_RGGB, CSS_FORMAT_RAW_8 },
-	{ MEDIA_BUS_FMT_SBGGR10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_BGGR, CSS_FORMAT_RAW_10 },
-	{ MEDIA_BUS_FMT_SGBRG10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_GBRG, CSS_FORMAT_RAW_10 },
-	{ MEDIA_BUS_FMT_SGRBG10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_GRBG, CSS_FORMAT_RAW_10 },
-	{ MEDIA_BUS_FMT_SRGGB10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_RGGB, CSS_FORMAT_RAW_10 },
-	{ MEDIA_BUS_FMT_SBGGR12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_BGGR, CSS_FORMAT_RAW_12 },
-	{ MEDIA_BUS_FMT_SGBRG12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_GBRG, CSS_FORMAT_RAW_12 },
-	{ MEDIA_BUS_FMT_SGRBG12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_GRBG, CSS_FORMAT_RAW_12 },
-	{ MEDIA_BUS_FMT_SRGGB12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_RGGB, CSS_FORMAT_RAW_12 },
-	{ MEDIA_BUS_FMT_UYVY8_1X16, 8, 8, ATOMISP_INPUT_FORMAT_YUV422_8, 0, ATOMISP_INPUT_FORMAT_YUV422_8 },
-	{ MEDIA_BUS_FMT_YUYV8_1X16, 8, 8, ATOMISP_INPUT_FORMAT_YUV422_8, 0, ATOMISP_INPUT_FORMAT_YUV422_8 },
-	{ MEDIA_BUS_FMT_JPEG_1X8, 8, 8, CSS_FRAME_FORMAT_BINARY_8, 0, ATOMISP_INPUT_FORMAT_BINARY_8 },
-	{ V4L2_MBUS_FMT_CUSTOM_NV12, 12, 12, CSS_FRAME_FORMAT_NV12, 0, CSS_FRAME_FORMAT_NV12 },
-	{ V4L2_MBUS_FMT_CUSTOM_NV21, 12, 12, CSS_FRAME_FORMAT_NV21, 0, CSS_FRAME_FORMAT_NV21 },
-	{ V4L2_MBUS_FMT_CUSTOM_YUV420, 12, 12, ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY, 0, ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY },
-#if 0
-	{ V4L2_MBUS_FMT_CUSTOM_M10MO_RAW, 8, 8, CSS_FRAME_FORMAT_BINARY_8, 0, ATOMISP_INPUT_FORMAT_BINARY_8 },
-#endif
-	/* no valid V4L2 MBUS code for metadata format, so leave it 0. */
-	{ 0, 0, 0, ATOMISP_INPUT_FORMAT_EMBEDDED, 0, ATOMISP_INPUT_FORMAT_EMBEDDED },
-	{}
-};
-
-static const struct {
-	u32 code;
-	u32 compressed;
-} compressed_codes[] = {
-	{ MEDIA_BUS_FMT_SBGGR10_1X10, MEDIA_BUS_FMT_SBGGR10_DPCM8_1X8 },
-	{ MEDIA_BUS_FMT_SGBRG10_1X10, MEDIA_BUS_FMT_SGBRG10_DPCM8_1X8 },
-	{ MEDIA_BUS_FMT_SGRBG10_1X10, MEDIA_BUS_FMT_SGRBG10_DPCM8_1X8 },
-	{ MEDIA_BUS_FMT_SRGGB10_1X10, MEDIA_BUS_FMT_SRGGB10_DPCM8_1X8 },
-};
-
-u32 atomisp_subdev_uncompressed_code(u32 code)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(compressed_codes); i++)
-		if (code == compressed_codes[i].compressed)
-			return compressed_codes[i].code;
-
-	return code;
-}
-
-bool atomisp_subdev_is_compressed(u32 code)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(atomisp_in_fmt_conv) - 1; i++)
-		if (code == atomisp_in_fmt_conv[i].code)
-			return atomisp_in_fmt_conv[i].bpp !=
-			       atomisp_in_fmt_conv[i].depth;
-
-	return false;
-}
-
-const struct atomisp_in_fmt_conv *atomisp_find_in_fmt_conv(u32 code)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(atomisp_in_fmt_conv) - 1; i++)
-		if (code == atomisp_in_fmt_conv[i].code)
-			return atomisp_in_fmt_conv + i;
-
-	return NULL;
-}
-
-const struct atomisp_in_fmt_conv *atomisp_find_in_fmt_conv_by_atomisp_in_fmt(
-	enum atomisp_input_format atomisp_in_fmt)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(atomisp_in_fmt_conv) - 1; i++)
-		if (atomisp_in_fmt_conv[i].atomisp_in_fmt == atomisp_in_fmt)
-			return atomisp_in_fmt_conv + i;
-
-	return NULL;
-}
-
-bool atomisp_subdev_format_conversion(struct atomisp_sub_device *asd,
-				      unsigned int source_pad)
-{
-	struct v4l2_mbus_framefmt *sink, *src;
-
-	sink = atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-				       V4L2_SUBDEV_FORMAT_ACTIVE,
-				       ATOMISP_SUBDEV_PAD_SINK);
-	src = atomisp_subdev_get_ffmt(&asd->subdev, NULL,
-				      V4L2_SUBDEV_FORMAT_ACTIVE, source_pad);
-
-	return atomisp_is_mbuscode_raw(sink->code)
-		&& !atomisp_is_mbuscode_raw(src->code);
-}
-
-uint16_t atomisp_subdev_source_pad(struct video_device * vdev)
-{
-	struct media_link *link;
-	uint16_t ret = 0;
-	list_for_each_entry(link, &vdev->entity.links, list) {
-		if (link->source) {
-			ret = link->source->index;
-			break;
-		}
-	}
-	return ret;
-}
-
-/*
- * V4L2 subdev operations
- */
-
-/*
- * isp_subdev_ioctl - CCDC module private ioctl's
- * @sd: ISP V4L2 subdevice
- * @cmd: ioctl command
- * @arg: ioctl argument
- *
- * Return 0 on success or a negative error code otherwise.
- */
-static long isp_subdev_ioctl(struct v4l2_subdev *sd,
-	unsigned int cmd, void *arg)
-{
-	return 0;
-}
-
-/*
- * isp_subdev_set_power - Power on/off the CCDC module
- * @sd: ISP V4L2 subdevice
- * @on: power on/off
- *
- * Return 0 on success or a negative error code otherwise.
- */
-static int isp_subdev_set_power(struct v4l2_subdev *sd, int on)
-{
-	return 0;
-}
-
-static int isp_subdev_subscribe_event(struct v4l2_subdev *sd,
-				      struct v4l2_fh *fh,
-				      struct v4l2_event_subscription *sub)
-{
-	struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd);
-	struct atomisp_device *isp = isp_sd->isp;
-
-	if (sub->type != V4L2_EVENT_FRAME_SYNC &&
-	    sub->type != V4L2_EVENT_FRAME_END &&
-	    sub->type != V4L2_EVENT_ATOMISP_3A_STATS_READY &&
-	    sub->type != V4L2_EVENT_ATOMISP_METADATA_READY &&
-	    sub->type != V4L2_EVENT_ATOMISP_PAUSE_BUFFER &&
-	    sub->type != V4L2_EVENT_ATOMISP_CSS_RESET &&
-	    sub->type != V4L2_EVENT_ATOMISP_RAW_BUFFERS_ALLOC_DONE &&
-	    sub->type != V4L2_EVENT_ATOMISP_ACC_COMPLETE)
-		return -EINVAL;
-
-	if (sub->type == V4L2_EVENT_FRAME_SYNC &&
-			!atomisp_css_valid_sof(isp))
-		return -EINVAL;
-
-	return v4l2_event_subscribe(fh, sub, 16, NULL);
-}
-
-static int isp_subdev_unsubscribe_event(struct v4l2_subdev *sd,
-					struct v4l2_fh *fh,
-					struct v4l2_event_subscription *sub)
-{
-	return v4l2_event_unsubscribe(fh, sub);
-}
-
-/*
- * isp_subdev_enum_mbus_code - Handle pixel format enumeration
- * @sd: pointer to v4l2 subdev structure
- * @fh : V4L2 subdev file handle
- * @code: pointer to v4l2_subdev_pad_mbus_code_enum structure
- * return -EINVAL or zero on success
- */
-static int isp_subdev_enum_mbus_code(struct v4l2_subdev *sd,
-				     struct v4l2_subdev_pad_config *cfg,
-				     struct v4l2_subdev_mbus_code_enum *code)
-{
-	if (code->index >= ARRAY_SIZE(atomisp_in_fmt_conv) - 1)
-		return -EINVAL;
-
-	code->code = atomisp_in_fmt_conv[code->index].code;
-
-	return 0;
-}
-
-static int isp_subdev_validate_rect(struct v4l2_subdev *sd, uint32_t pad,
-				    uint32_t target)
-{
-	switch (pad) {
-	case ATOMISP_SUBDEV_PAD_SINK:
-		switch (target) {
-		case V4L2_SEL_TGT_CROP:
-			return 0;
-		}
-		break;
-	default:
-		switch (target) {
-		case V4L2_SEL_TGT_COMPOSE:
-			return 0;
-		}
-		break;
-	}
-
-	return -EINVAL;
-}
-
-struct v4l2_rect *atomisp_subdev_get_rect(struct v4l2_subdev *sd,
-					  struct v4l2_subdev_pad_config *cfg,
-					  uint32_t which, uint32_t pad,
-					  uint32_t target)
-{
-	struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd);
-
-	if (which == V4L2_SUBDEV_FORMAT_TRY) {
-		switch (target) {
-		case V4L2_SEL_TGT_CROP:
-			return v4l2_subdev_get_try_crop(sd, cfg, pad);
-		case V4L2_SEL_TGT_COMPOSE:
-			return v4l2_subdev_get_try_compose(sd, cfg, pad);
-		}
-	}
-
-	switch (target) {
-	case V4L2_SEL_TGT_CROP:
-		return &isp_sd->fmt[pad].crop;
-	case V4L2_SEL_TGT_COMPOSE:
-		return &isp_sd->fmt[pad].compose;
-	}
-
-	return NULL;
-}
-
-struct v4l2_mbus_framefmt
-*atomisp_subdev_get_ffmt(struct v4l2_subdev *sd,
-			 struct v4l2_subdev_pad_config *cfg, uint32_t which,
-			 uint32_t pad)
-{
-	struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd);
-
-	if (which == V4L2_SUBDEV_FORMAT_TRY)
-		return v4l2_subdev_get_try_format(sd, cfg, pad);
-
-	return &isp_sd->fmt[pad].fmt;
-}
-
-static void isp_get_fmt_rect(struct v4l2_subdev *sd,
-			     struct v4l2_subdev_pad_config *cfg, uint32_t which,
-			     struct v4l2_mbus_framefmt **ffmt,
-			     struct v4l2_rect *crop[ATOMISP_SUBDEV_PADS_NUM],
-			     struct v4l2_rect *comp[ATOMISP_SUBDEV_PADS_NUM])
-{
-	unsigned int i;
-
-	for (i = 0; i < ATOMISP_SUBDEV_PADS_NUM; i++) {
-		ffmt[i] = atomisp_subdev_get_ffmt(sd, cfg, which, i);
-		crop[i] = atomisp_subdev_get_rect(sd, cfg, which, i,
-						  V4L2_SEL_TGT_CROP);
-		comp[i] = atomisp_subdev_get_rect(sd, cfg, which, i,
-						  V4L2_SEL_TGT_COMPOSE);
-	}
-}
-
-static void isp_subdev_propagate(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_pad_config *cfg,
-				 uint32_t which, uint32_t pad, uint32_t target,
-				 uint32_t flags)
-{
-	struct v4l2_mbus_framefmt *ffmt[ATOMISP_SUBDEV_PADS_NUM];
-	struct v4l2_rect *crop[ATOMISP_SUBDEV_PADS_NUM],
-		*comp[ATOMISP_SUBDEV_PADS_NUM];
-
-	if (flags & V4L2_SEL_FLAG_KEEP_CONFIG)
-		return;
-
-	isp_get_fmt_rect(sd, cfg, which, ffmt, crop, comp);
-
-	switch (pad) {
-	case ATOMISP_SUBDEV_PAD_SINK: {
-		struct v4l2_rect r = {0};
-
-		/* Only crop target supported on sink pad. */
-		r.width = ffmt[pad]->width;
-		r.height = ffmt[pad]->height;
-
-		atomisp_subdev_set_selection(sd, cfg, which, pad,
-					     target, flags, &r);
-		break;
-	}
-	}
-}
-
-static int isp_subdev_get_selection(struct v4l2_subdev *sd,
-				    struct v4l2_subdev_pad_config *cfg,
-				    struct v4l2_subdev_selection *sel)
-{
-	struct v4l2_rect *rec;
-	int rval = isp_subdev_validate_rect(sd, sel->pad, sel->target);
-
-	if (rval)
-		return rval;
-
-	rec = atomisp_subdev_get_rect(sd, cfg, sel->which, sel->pad,
-					sel->target);
-	if (!rec)
-		return -EINVAL;
-
-	sel->r = *rec;
-	return 0;
-}
-
-static char *atomisp_pad_str[] = { "ATOMISP_SUBDEV_PAD_SINK",
-				   "ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE",
-				   "ATOMISP_SUBDEV_PAD_SOURCE_VF",
-				   "ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW",
-				   "ATOMISP_SUBDEV_PAD_SOURCE_VIDEO"};
-
-int atomisp_subdev_set_selection(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_pad_config *cfg,
-				 uint32_t which, uint32_t pad, uint32_t target,
-				 uint32_t flags, struct v4l2_rect *r)
-{
-	struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd);
-	struct atomisp_device *isp = isp_sd->isp;
-	struct v4l2_mbus_framefmt *ffmt[ATOMISP_SUBDEV_PADS_NUM];
-	uint16_t vdev_pad = atomisp_subdev_source_pad(sd->devnode);
-	struct v4l2_rect *crop[ATOMISP_SUBDEV_PADS_NUM],
-		*comp[ATOMISP_SUBDEV_PADS_NUM];
-	enum atomisp_input_stream_id stream_id;
-	unsigned int i;
-	unsigned int padding_w = pad_w;
-	unsigned int padding_h = pad_h;
-
-	stream_id = atomisp_source_pad_to_stream_id(isp_sd, vdev_pad);
-
-	isp_get_fmt_rect(sd, cfg, which, ffmt, crop, comp);
-
-	dev_dbg(isp->dev,
-		"sel: pad %s tgt %s l %d t %d w %d h %d which %s f 0x%8.8x\n",
-		atomisp_pad_str[pad], target == V4L2_SEL_TGT_CROP
-		? "V4L2_SEL_TGT_CROP" : "V4L2_SEL_TGT_COMPOSE",
-		r->left, r->top, r->width, r->height,
-		which == V4L2_SUBDEV_FORMAT_TRY ? "V4L2_SUBDEV_FORMAT_TRY"
-		: "V4L2_SUBDEV_FORMAT_ACTIVE", flags);
-
-	r->width = rounddown(r->width, ATOM_ISP_STEP_WIDTH);
-	r->height = rounddown(r->height, ATOM_ISP_STEP_HEIGHT);
-
-	switch (pad) {
-	case ATOMISP_SUBDEV_PAD_SINK: {
-		/* Only crop target supported on sink pad. */
-		unsigned int dvs_w, dvs_h;
-
-		crop[pad]->width = ffmt[pad]->width;
-		crop[pad]->height = ffmt[pad]->height;
-
-		/* Workaround for BYT 1080p perfectshot since the maxinum resolution of
-		 * front camera ov2722 is 1932x1092 and cannot use pad_w > 12*/
-		if (!strncmp(isp->inputs[isp_sd->input_curr].camera->name,
-				"ov2722", 6) && crop[pad]->height == 1092) {
-			padding_w = 12;
-			padding_h = 12;
-		}
-
-		if (isp->inputs[isp_sd->input_curr].type == SOC_CAMERA) {
-			padding_w = 0;
-			padding_h = 0;
-		}
-
-		if (atomisp_subdev_format_conversion(isp_sd,
-						     isp_sd->capture_pad)
-		    && crop[pad]->width && crop[pad]->height)
-			crop[pad]->width -= padding_w, crop[pad]->height -= padding_h;
-
-		/* if subdev type is SOC camera,we do not need to set DVS */
-		if (isp->inputs[isp_sd->input_curr].type == SOC_CAMERA)
-			isp_sd->params.video_dis_en = 0;
-
-		if (isp_sd->params.video_dis_en &&
-		    isp_sd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
-		    !isp_sd->continuous_mode->val) {
-			/* This resolution contains 20 % of DVS slack
-			 * (of the desired captured image before
-			 * scaling, or 1 / 6 of what we get from the
-			 * sensor) in both width and height. Remove
-			 * it. */
-			crop[pad]->width = roundup(crop[pad]->width * 5 / 6,
-						   ATOM_ISP_STEP_WIDTH);
-			crop[pad]->height = roundup(crop[pad]->height * 5 / 6,
-						    ATOM_ISP_STEP_HEIGHT);
-		}
-
-		crop[pad]->width = min(crop[pad]->width, r->width);
-		crop[pad]->height = min(crop[pad]->height, r->height);
-
-		if (!(flags & V4L2_SEL_FLAG_KEEP_CONFIG)) {
-			for (i = ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE;
-			     i < ATOMISP_SUBDEV_PADS_NUM; i++) {
-				struct v4l2_rect tmp = *crop[pad];
-
-				atomisp_subdev_set_selection(
-					sd, cfg, which, i, V4L2_SEL_TGT_COMPOSE,
-					flags, &tmp);
-			}
-		}
-
-		if (which == V4L2_SUBDEV_FORMAT_TRY)
-			break;
-
-		if (isp_sd->params.video_dis_en &&
-		    isp_sd->run_mode->val == ATOMISP_RUN_MODE_VIDEO &&
-		    !isp_sd->continuous_mode->val) {
-			dvs_w = rounddown(crop[pad]->width / 5,
-					  ATOM_ISP_STEP_WIDTH);
-			dvs_h = rounddown(crop[pad]->height / 5,
-					  ATOM_ISP_STEP_HEIGHT);
-		} else if (!isp_sd->params.video_dis_en &&
-			   isp_sd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
-			/*
-			 * For CSS2.0, digital zoom needs to set dvs envelope to 12
-			 * when dvs is disabled.
-			 */
-			dvs_w = dvs_h = 12;
-		} else
-			dvs_w = dvs_h = 0;
-
-		atomisp_css_video_set_dis_envelope(isp_sd, dvs_w, dvs_h);
-		atomisp_css_input_set_effective_resolution(isp_sd, stream_id,
-					crop[pad]->width, crop[pad]->height);
-
-		break;
-	}
-	case ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE:
-	case ATOMISP_SUBDEV_PAD_SOURCE_VIDEO: {
-		/* Only compose target is supported on source pads. */
-
-		if (isp_sd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) {
-			/* Scaling is disabled in this mode */
-			r->width = crop[ATOMISP_SUBDEV_PAD_SINK]->width;
-			r->height = crop[ATOMISP_SUBDEV_PAD_SINK]->height;
-		}
-
-		if (crop[ATOMISP_SUBDEV_PAD_SINK]->width == r->width
-		    && crop[ATOMISP_SUBDEV_PAD_SINK]->height == r->height)
-			isp_sd->params.yuv_ds_en = false;
-		else
-			isp_sd->params.yuv_ds_en = true;
-
-		comp[pad]->width = r->width;
-		comp[pad]->height = r->height;
-
-		if (r->width == 0 || r->height == 0 ||
-			crop[ATOMISP_SUBDEV_PAD_SINK]->width == 0 ||
-			crop[ATOMISP_SUBDEV_PAD_SINK]->height == 0)
-			break;
-		/*
-		 * do cropping on sensor input if ratio of required resolution
-		 * is different with sensor output resolution ratio:
-		 *
-		 * ratio = width / height
-		 *
-		 * if ratio_output < ratio_sensor:
-		 *	effect_width = sensor_height * out_width / out_height;
-		 *	effect_height = sensor_height;
-		 * else
-		 *	effect_width = sensor_width;
-		 *	effect_height = sensor_width * out_height / out_width;
-		 *
-		 */
-		if (r->width * crop[ATOMISP_SUBDEV_PAD_SINK]->height <
-			crop[ATOMISP_SUBDEV_PAD_SINK]->width * r->height)
-			atomisp_css_input_set_effective_resolution(isp_sd,
-				stream_id,
-				rounddown(crop[ATOMISP_SUBDEV_PAD_SINK]->
-					height * r->width / r->height,
-					ATOM_ISP_STEP_WIDTH),
-				crop[ATOMISP_SUBDEV_PAD_SINK]->height);
-		else
-			atomisp_css_input_set_effective_resolution(isp_sd,
-				stream_id,
-				crop[ATOMISP_SUBDEV_PAD_SINK]->width,
-				rounddown(crop[ATOMISP_SUBDEV_PAD_SINK]->
-					width * r->height / r->width,
-					ATOM_ISP_STEP_WIDTH));
-
-		break;
-	}
-	case ATOMISP_SUBDEV_PAD_SOURCE_VF:
-	case ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW:
-		comp[pad]->width = r->width;
-		comp[pad]->height = r->height;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* Set format dimensions on non-sink pads as well. */
-	if (pad != ATOMISP_SUBDEV_PAD_SINK) {
-		ffmt[pad]->width = comp[pad]->width;
-		ffmt[pad]->height = comp[pad]->height;
-	}
-
-	if (!atomisp_subdev_get_rect(sd, cfg, which, pad, target))
-		return -EINVAL;
-	*r = *atomisp_subdev_get_rect(sd, cfg, which, pad, target);
-
-	dev_dbg(isp->dev, "sel actual: l %d t %d w %d h %d\n",
-		r->left, r->top, r->width, r->height);
-
-	return 0;
-}
-
-static int isp_subdev_set_selection(struct v4l2_subdev *sd,
-				    struct v4l2_subdev_pad_config *cfg,
-				    struct v4l2_subdev_selection *sel)
-{
-	int rval = isp_subdev_validate_rect(sd, sel->pad, sel->target);
-	if (rval)
-		return rval;
-
-	return atomisp_subdev_set_selection(sd, cfg, sel->which, sel->pad,
-					    sel->target, sel->flags, &sel->r);
-}
-
-static int atomisp_get_sensor_bin_factor(struct atomisp_sub_device *asd)
-{
-	struct v4l2_control ctrl = {0};
-	struct atomisp_device *isp = asd->isp;
-	int hbin, vbin;
-	int ret;
-
-	if (isp->inputs[asd->input_curr].type == FILE_INPUT ||
-		isp->inputs[asd->input_curr].type == TEST_PATTERN)
-		return 0;
-
-	ctrl.id = V4L2_CID_BIN_FACTOR_HORZ;
-	ret =
-	    v4l2_g_ctrl(isp->inputs[asd->input_curr].camera->ctrl_handler,
-			&ctrl);
-	hbin = ctrl.value;
-	ctrl.id = V4L2_CID_BIN_FACTOR_VERT;
-	ret |=
-	    v4l2_g_ctrl(isp->inputs[asd->input_curr].camera->ctrl_handler,
-			&ctrl);
-	vbin = ctrl.value;
-
-	/*
-	 * ISP needs to know binning factor from sensor.
-	 * In case horizontal and vertical sensor's binning factors
-	 * are different or sensor does not support binning factor CID,
-	 * ISP will apply default 0 value.
-	 */
-	if (ret || hbin != vbin)
-		hbin = 0;
-
-	return hbin;
-}
-
-void atomisp_subdev_set_ffmt(struct v4l2_subdev *sd,
-			     struct v4l2_subdev_pad_config *cfg, uint32_t which,
-			     uint32_t pad, struct v4l2_mbus_framefmt *ffmt)
-{
-	struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd);
-	struct atomisp_device *isp = isp_sd->isp;
-	struct v4l2_mbus_framefmt *__ffmt =
-		atomisp_subdev_get_ffmt(sd, cfg, which, pad);
-	uint16_t vdev_pad = atomisp_subdev_source_pad(sd->devnode);
-	enum atomisp_input_stream_id stream_id;
-
-	dev_dbg(isp->dev, "ffmt: pad %s w %d h %d code 0x%8.8x which %s\n",
-		atomisp_pad_str[pad], ffmt->width, ffmt->height, ffmt->code,
-		which == V4L2_SUBDEV_FORMAT_TRY ? "V4L2_SUBDEV_FORMAT_TRY"
-		: "V4L2_SUBDEV_FORMAT_ACTIVE");
-
-	stream_id = atomisp_source_pad_to_stream_id(isp_sd, vdev_pad);
-
-	switch (pad) {
-	case ATOMISP_SUBDEV_PAD_SINK: {
-		const struct atomisp_in_fmt_conv *fc =
-			atomisp_find_in_fmt_conv(ffmt->code);
-
-		if (!fc) {
-			fc = atomisp_in_fmt_conv;
-			ffmt->code = fc->code;
-			dev_dbg(isp->dev, "using 0x%8.8x instead\n",
-				ffmt->code);
-		}
-
-		*__ffmt = *ffmt;
-
-		isp_subdev_propagate(sd, cfg, which, pad,
-				     V4L2_SEL_TGT_CROP, 0);
-
-		if (which == V4L2_SUBDEV_FORMAT_ACTIVE) {
-			atomisp_css_input_set_resolution(isp_sd,
-				stream_id, ffmt);
-			atomisp_css_input_set_binning_factor(isp_sd,
-				stream_id,
-				atomisp_get_sensor_bin_factor(isp_sd));
-			atomisp_css_input_set_bayer_order(isp_sd, stream_id,
-							  fc->bayer_order);
-			atomisp_css_input_set_format(isp_sd, stream_id,
-						fc->css_stream_fmt);
-			atomisp_css_set_default_isys_config(isp_sd, stream_id,
-							    ffmt);
-		}
-
-		break;
-	}
-	case ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE:
-	case ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW:
-	case ATOMISP_SUBDEV_PAD_SOURCE_VF:
-	case ATOMISP_SUBDEV_PAD_SOURCE_VIDEO:
-		__ffmt->code = ffmt->code;
-		break;
-	}
-}
-
-/*
- * isp_subdev_get_format - Retrieve the video format on a pad
- * @sd : ISP V4L2 subdevice
- * @fh : V4L2 subdev file handle
- * @pad: Pad number
- * @fmt: Format
- *
- * Return 0 on success or -EINVAL if the pad is invalid or doesn't correspond
- * to the format type.
- */
-static int isp_subdev_get_format(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_pad_config *cfg,
-				 struct v4l2_subdev_format *fmt)
-{
-	fmt->format = *atomisp_subdev_get_ffmt(sd, cfg, fmt->which, fmt->pad);
-
-	return 0;
-}
-
-/*
- * isp_subdev_set_format - Set the video format on a pad
- * @sd : ISP subdev V4L2 subdevice
- * @fh : V4L2 subdev file handle
- * @pad: Pad number
- * @fmt: Format
- *
- * Return 0 on success or -EINVAL if the pad is invalid or doesn't correspond
- * to the format type.
- */
-static int isp_subdev_set_format(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_pad_config *cfg,
-				 struct v4l2_subdev_format *fmt)
-{
-	atomisp_subdev_set_ffmt(sd, cfg, fmt->which, fmt->pad, &fmt->format);
-
-	return 0;
-}
-
-/* V4L2 subdev core operations */
-static const struct v4l2_subdev_core_ops isp_subdev_v4l2_core_ops = {
-	 .ioctl = isp_subdev_ioctl, .s_power = isp_subdev_set_power,
-	 .subscribe_event = isp_subdev_subscribe_event,
-	 .unsubscribe_event = isp_subdev_unsubscribe_event,
-};
-
-/* V4L2 subdev pad operations */
-static const struct v4l2_subdev_pad_ops isp_subdev_v4l2_pad_ops = {
-	.enum_mbus_code = isp_subdev_enum_mbus_code,
-	.get_fmt = isp_subdev_get_format,
-	.set_fmt = isp_subdev_set_format,
-	.get_selection = isp_subdev_get_selection,
-	.set_selection = isp_subdev_set_selection,
-	.link_validate = v4l2_subdev_link_validate_default,
-};
-
-/* V4L2 subdev operations */
-static const struct v4l2_subdev_ops isp_subdev_v4l2_ops = {
-	.core = &isp_subdev_v4l2_core_ops,
-	.pad = &isp_subdev_v4l2_pad_ops,
-};
-
-static void isp_subdev_init_params(struct atomisp_sub_device *asd)
-{
-	unsigned int i;
-
-	/* parameters initialization */
-	INIT_LIST_HEAD(&asd->s3a_stats);
-	INIT_LIST_HEAD(&asd->s3a_stats_in_css);
-	INIT_LIST_HEAD(&asd->s3a_stats_ready);
-	INIT_LIST_HEAD(&asd->dis_stats);
-	INIT_LIST_HEAD(&asd->dis_stats_in_css);
-	spin_lock_init(&asd->dis_stats_lock);
-	for (i = 0; i < ATOMISP_METADATA_TYPE_NUM; i++) {
-		INIT_LIST_HEAD(&asd->metadata[i]);
-		INIT_LIST_HEAD(&asd->metadata_in_css[i]);
-		INIT_LIST_HEAD(&asd->metadata_ready[i]);
-	}
-}
-
-/*
-* isp_subdev_link_setup - Setup isp subdev connections
-* @entity: ispsubdev media entity
-* @local: Pad at the local end of the link
-* @remote: Pad at the remote end of the link
-* @flags: Link flags
-*
-* return -EINVAL or zero on success
-*/
-static int isp_subdev_link_setup(struct media_entity *entity,
-	const struct media_pad *local,
-	const struct media_pad *remote, u32 flags)
-{
-	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(entity);
-	struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd);
-	struct atomisp_device *isp = isp_sd->isp;
-	unsigned int i;
-
-	switch (local->index | is_media_entity_v4l2_subdev(remote->entity)) {
-	case ATOMISP_SUBDEV_PAD_SINK | MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN:
-		/* Read from the sensor CSI2-ports. */
-		if (!(flags & MEDIA_LNK_FL_ENABLED)) {
-			isp_sd->input = ATOMISP_SUBDEV_INPUT_NONE;
-			break;
-		}
-
-		if (isp_sd->input != ATOMISP_SUBDEV_INPUT_NONE)
-			return -EBUSY;
-
-		for (i = 0; i < ATOMISP_CAMERA_NR_PORTS; i++) {
-			if (remote->entity != &isp->csi2_port[i].subdev.entity)
-				continue;
-
-			isp_sd->input = ATOMISP_SUBDEV_INPUT_CSI2_PORT1 + i;
-			return 0;
-		}
-
-		return -EINVAL;
-
-	case ATOMISP_SUBDEV_PAD_SINK | MEDIA_ENT_F_OLD_BASE:
-		/* read from memory */
-		if (flags & MEDIA_LNK_FL_ENABLED) {
-			if (isp_sd->input >= ATOMISP_SUBDEV_INPUT_CSI2_PORT1 &&
-				isp_sd->input < (ATOMISP_SUBDEV_INPUT_CSI2_PORT1
-						+ ATOMISP_CAMERA_NR_PORTS))
-				return -EBUSY;
-			isp_sd->input = ATOMISP_SUBDEV_INPUT_MEMORY;
-		} else {
-			if (isp_sd->input == ATOMISP_SUBDEV_INPUT_MEMORY)
-				isp_sd->input = ATOMISP_SUBDEV_INPUT_NONE;
-		}
-		break;
-
-	case ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW | MEDIA_ENT_F_OLD_BASE:
-		/* always write to memory */
-		break;
-
-	case ATOMISP_SUBDEV_PAD_SOURCE_VF | MEDIA_ENT_F_OLD_BASE:
-		/* always write to memory */
-		break;
-
-	case ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE | MEDIA_ENT_F_OLD_BASE:
-		/* always write to memory */
-		break;
-
-	case ATOMISP_SUBDEV_PAD_SOURCE_VIDEO | MEDIA_ENT_F_OLD_BASE:
-		/* always write to memory */
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/* media operations */
-static const struct media_entity_operations isp_subdev_media_ops = {
-	 .link_setup = isp_subdev_link_setup,
-	 .link_validate = v4l2_subdev_link_validate,
-/*	 .set_power = v4l2_subdev_set_power,	*/
-};
-
-static int __atomisp_update_run_mode(struct atomisp_sub_device *asd)
-{
-	struct atomisp_device *isp = asd->isp;
-	struct v4l2_ctrl *ctrl = asd->run_mode;
-	struct v4l2_ctrl *c;
-	s32 mode;
-
-	if (ctrl->val != ATOMISP_RUN_MODE_VIDEO &&
-	    asd->continuous_mode->val)
-		mode = ATOMISP_RUN_MODE_PREVIEW;
-	else
-		mode = ctrl->val;
-
-	c = v4l2_ctrl_find(
-		isp->inputs[asd->input_curr].camera->ctrl_handler,
-		V4L2_CID_RUN_MODE);
-
-	if (c)
-		return v4l2_ctrl_s_ctrl(c, mode);
-
-	return 0;
-}
-
-int atomisp_update_run_mode(struct atomisp_sub_device *asd)
-{
-	int rval;
-
-	mutex_lock(asd->ctrl_handler.lock);
-	rval = __atomisp_update_run_mode(asd);
-	mutex_unlock(asd->ctrl_handler.lock);
-
-	return rval;
-}
-
-static int s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct atomisp_sub_device *asd = container_of(
-		ctrl->handler, struct atomisp_sub_device, ctrl_handler);
-
-	switch (ctrl->id) {
-	case V4L2_CID_RUN_MODE:
-		return __atomisp_update_run_mode(asd);
-	case V4L2_CID_DEPTH_MODE:
-		if (asd->streaming != ATOMISP_DEVICE_STREAMING_DISABLED) {
-			dev_err(asd->isp->dev, "ISP is streaming, it is not supported to change the depth mode\n");
-			return -EINVAL;
-		}
-		break;
-	}
-
-	return 0;
-}
-
-static const struct v4l2_ctrl_ops ctrl_ops = {
-	.s_ctrl = &s_ctrl,
-};
-
-static const struct v4l2_ctrl_config ctrl_fmt_auto = {
-	.ops = &ctrl_ops,
-	.id = V4L2_CID_FMT_AUTO,
-	.name = "Automatic format guessing",
-	.type = V4L2_CTRL_TYPE_BOOLEAN,
-	.min = 0,
-	.max = 1,
-	.step = 1,
-	.def = 1,
-};
-
-static const char * const ctrl_run_mode_menu[] = {
-	NULL,
-	"Video",
-	"Still capture",
-	"Continuous capture",
-	"Preview",
-};
-
-static const struct v4l2_ctrl_config ctrl_run_mode = {
-	.ops = &ctrl_ops,
-	.id = V4L2_CID_RUN_MODE,
-	.name = "Atomisp run mode",
-	.type = V4L2_CTRL_TYPE_MENU,
-	.min = 1,
-	.def = 1,
-	.max = 4,
-	.qmenu = ctrl_run_mode_menu,
-};
-
-static const char * const ctrl_vfpp_mode_menu[] = {
-	"Enable",			/* vfpp always enabled */
-	"Disable to scaler mode",	/* CSS into video mode and disable */
-	"Disable to low latency mode",	/* CSS into still mode and disable */
-};
-
-static const struct v4l2_ctrl_config ctrl_vfpp = {
-	.id = V4L2_CID_VFPP,
-	.name = "Atomisp vf postprocess",
-	.type = V4L2_CTRL_TYPE_MENU,
-	.min = 0,
-	.def = 0,
-	.max = 2,
-	.qmenu = ctrl_vfpp_mode_menu,
-};
-
-/*
- * Control for ISP continuous mode
- *
- * When enabled, capture processing is possible without
- * stopping the preview pipeline. When disabled, ISP needs
- * to be restarted between preview and capture.
- */
-static const struct v4l2_ctrl_config ctrl_continuous_mode = {
-	.ops = &ctrl_ops,
-	.id = V4L2_CID_ATOMISP_CONTINUOUS_MODE,
-	.type = V4L2_CTRL_TYPE_BOOLEAN,
-	.name = "Continuous mode",
-	.min = 0,
-	.max = 1,
-	.step = 1,
-	.def = 0,
-};
-
-/*
- * Control for continuous mode raw buffer size
- *
- * The size of the RAW ringbuffer sets limit on how much
- * back in time application can go when requesting capture
- * frames to be rendered, and how many frames can be rendered
- * in a burst at full sensor rate.
- *
- * Note: this setting has a big impact on memory consumption of
- * the CSS subsystem.
- */
-static const struct v4l2_ctrl_config ctrl_continuous_raw_buffer_size = {
-	.ops = &ctrl_ops,
-	.id = V4L2_CID_ATOMISP_CONTINUOUS_RAW_BUFFER_SIZE,
-	.type = V4L2_CTRL_TYPE_INTEGER,
-	.name = "Continuous raw ringbuffer size",
-	.min = 1,
-	.max = 100, /* depends on CSS version, runtime checked */
-	.step = 1,
-	.def = 3,
-};
-
-/*
- * Control for enabling continuous viewfinder
- *
- * When enabled, and ISP is in continuous mode (see ctrl_continuous_mode ),
- * preview pipeline continues concurrently with capture
- * processing. When disabled, and continuous mode is used,
- * preview is paused while captures are processed, but
- * full pipeline restart is not needed.
- *
- * By setting this to disabled, capture processing is
- * essentially given priority over preview, and the effective
- * capture output rate may be higher than with continuous
- * viewfinder enabled.
- */
-static const struct v4l2_ctrl_config ctrl_continuous_viewfinder = {
-	.id = V4L2_CID_ATOMISP_CONTINUOUS_VIEWFINDER,
-	.type = V4L2_CTRL_TYPE_BOOLEAN,
-	.name = "Continuous viewfinder",
-	.min = 0,
-	.max = 1,
-	.step = 1,
-	.def = 0,
-};
-
-/*
- * Control for enabling Lock&Unlock Raw Buffer mechanism
- *
- * When enabled, Raw Buffer can be locked and unlocked.
- * Application can hold the exp_id of Raw Buffer
- * and unlock it when no longer needed.
- * Note: Make sure set this configuration before creating stream.
- */
-static const struct v4l2_ctrl_config ctrl_enable_raw_buffer_lock = {
-	.id = V4L2_CID_ENABLE_RAW_BUFFER_LOCK,
-	.type = V4L2_CTRL_TYPE_BOOLEAN,
-	.name = "Lock Unlock Raw Buffer",
-	.min = 0,
-	.max = 1,
-	.step = 1,
-	.def = 0,
-};
-
-/*
- * Control to disable digital zoom of the whole stream
- *
- * When it is true, pipe configuation enable_dz will be set to false.
- * This can help get a better performance by disabling pp binary.
- *
- * Note: Make sure set this configuration before creating stream.
- */
-static const struct v4l2_ctrl_config ctrl_disable_dz = {
-	.id = V4L2_CID_DISABLE_DZ,
-	.type = V4L2_CTRL_TYPE_BOOLEAN,
-	.name = "Disable digital zoom",
-	.min = 0,
-	.max = 1,
-	.step = 1,
-	.def = 0,
-};
-
-/*
- * Control for ISP depth mode
- *
- * When enabled, that means ISP will deal with dual streams and sensors will be
- * in slave/master mode.
- * slave sensor will have no output until master sensor is streamed on.
- */
-static const struct v4l2_ctrl_config ctrl_depth_mode = {
-	.ops = &ctrl_ops,
-	.id = V4L2_CID_DEPTH_MODE,
-	.type = V4L2_CTRL_TYPE_BOOLEAN,
-	.name = "Depth mode",
-	.min = 0,
-	.max = 1,
-	.step = 1,
-	.def = 0,
-};
-
-#ifdef ISP2401
-/*
- * Control for selectting ISP version
- *
- * When enabled, that means ISP version will be used ISP2.7. when disable, the
- * isp will default to use ISP2.2.
- * Note: Make sure set this configuration before creating stream.
- */
-static const struct v4l2_ctrl_config ctrl_select_isp_version = {
-	.ops = &ctrl_ops,
-	.id = V4L2_CID_ATOMISP_SELECT_ISP_VERSION,
-	.type = V4L2_CTRL_TYPE_BOOLEAN,
-	.name = "Select Isp version",
-	.min = 0,
-	.max = 1,
-	.step = 1,
-	.def = 0,
-};
-
-#ifdef CONFIG_ION
-/*
- * Control for ISP ion device fd
- *
- * userspace will open ion device and pass the fd to kernel.
- * this fd will be used to map shared fd to buffer.
- */
-static const struct v4l2_ctrl_config ctrl_ion_dev_fd = {
-		.ops = &ctrl_ops,
-		.id = V4L2_CID_ATOMISP_ION_DEVICE_FD,
-		.type = V4L2_CTRL_TYPE_INTEGER,
-		.name = "Ion Device Fd",
-		.min = -1,
-		.max = 1024,
-		.step = 1,
-		.def = ION_FD_UNSET
-};
-#endif
-
-#endif
-static void atomisp_init_subdev_pipe(struct atomisp_sub_device *asd,
-		struct atomisp_video_pipe *pipe, enum v4l2_buf_type buf_type)
-{
-	pipe->type = buf_type;
-	pipe->asd = asd;
-	pipe->isp = asd->isp;
-	spin_lock_init(&pipe->irq_lock);
-	INIT_LIST_HEAD(&pipe->activeq);
-	INIT_LIST_HEAD(&pipe->activeq_out);
-	INIT_LIST_HEAD(&pipe->buffers_waiting_for_param);
-	INIT_LIST_HEAD(&pipe->per_frame_params);
-	memset(pipe->frame_request_config_id,
-	       0, VIDEO_MAX_FRAME * sizeof(unsigned int));
-	memset(pipe->frame_params,
-	       0, VIDEO_MAX_FRAME *
-		sizeof(struct atomisp_css_params_with_list *));
-}
-
-static void atomisp_init_acc_pipe(struct atomisp_sub_device *asd,
-		struct atomisp_acc_pipe *pipe)
-{
-	pipe->asd = asd;
-	pipe->isp = asd->isp;
-	INIT_LIST_HEAD(&asd->acc.fw);
-	INIT_LIST_HEAD(&asd->acc.memory_maps);
-	ida_init(&asd->acc.ida);
-}
-
-/*
- * isp_subdev_init_entities - Initialize V4L2 subdev and media entity
- * @asd: ISP CCDC module
- *
- * Return 0 on success and a negative error code on failure.
- */
-static int isp_subdev_init_entities(struct atomisp_sub_device *asd)
-{
-	struct v4l2_subdev *sd = &asd->subdev;
-	struct media_pad *pads = asd->pads;
-	struct media_entity *me = &sd->entity;
-	int ret;
-
-	asd->input = ATOMISP_SUBDEV_INPUT_NONE;
-
-	v4l2_subdev_init(sd, &isp_subdev_v4l2_ops);
-	sprintf(sd->name, "ATOMISP_SUBDEV_%d", asd->index);
-	v4l2_set_subdevdata(sd, asd);
-	sd->flags |= V4L2_SUBDEV_FL_HAS_EVENTS | V4L2_SUBDEV_FL_HAS_DEVNODE;
-
-	pads[ATOMISP_SUBDEV_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
-	pads[ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW].flags = MEDIA_PAD_FL_SOURCE;
-	pads[ATOMISP_SUBDEV_PAD_SOURCE_VF].flags = MEDIA_PAD_FL_SOURCE;
-	pads[ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE].flags = MEDIA_PAD_FL_SOURCE;
-	pads[ATOMISP_SUBDEV_PAD_SOURCE_VIDEO].flags = MEDIA_PAD_FL_SOURCE;
-
-	asd->fmt[ATOMISP_SUBDEV_PAD_SINK].fmt.code =
-		MEDIA_BUS_FMT_SBGGR10_1X10;
-	asd->fmt[ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW].fmt.code =
-		MEDIA_BUS_FMT_SBGGR10_1X10;
-	asd->fmt[ATOMISP_SUBDEV_PAD_SOURCE_VF].fmt.code =
-		MEDIA_BUS_FMT_SBGGR10_1X10;
-	asd->fmt[ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE].fmt.code =
-		MEDIA_BUS_FMT_SBGGR10_1X10;
-	asd->fmt[ATOMISP_SUBDEV_PAD_SOURCE_VIDEO].fmt.code =
-		MEDIA_BUS_FMT_SBGGR10_1X10;
-
-	me->ops = &isp_subdev_media_ops;
-	me->function = MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN;
-	ret = media_entity_pads_init(me, ATOMISP_SUBDEV_PADS_NUM, pads);
-	if (ret < 0)
-		return ret;
-
-	atomisp_init_subdev_pipe(asd, &asd->video_in,
-				 V4L2_BUF_TYPE_VIDEO_OUTPUT);
-
-	atomisp_init_subdev_pipe(asd, &asd->video_out_preview,
-				 V4L2_BUF_TYPE_VIDEO_CAPTURE);
-
-	atomisp_init_subdev_pipe(asd, &asd->video_out_vf,
-				 V4L2_BUF_TYPE_VIDEO_CAPTURE);
-
-	atomisp_init_subdev_pipe(asd, &asd->video_out_capture,
-				 V4L2_BUF_TYPE_VIDEO_CAPTURE);
-
-	atomisp_init_subdev_pipe(asd, &asd->video_out_video_capture,
-				 V4L2_BUF_TYPE_VIDEO_CAPTURE);
-
-	atomisp_init_acc_pipe(asd, &asd->video_acc);
-
-	ret = atomisp_video_init(&asd->video_in, "MEMORY");
-	if (ret < 0)
-		return ret;
-
-	ret = atomisp_video_init(&asd->video_out_capture, "CAPTURE");
-	if (ret < 0)
-		return ret;
-
-	ret = atomisp_video_init(&asd->video_out_vf, "VIEWFINDER");
-	if (ret < 0)
-		return ret;
-
-	ret = atomisp_video_init(&asd->video_out_preview, "PREVIEW");
-	if (ret < 0)
-		return ret;
-
-	ret = atomisp_video_init(&asd->video_out_video_capture, "VIDEO");
-	if (ret < 0)
-		return ret;
-
-	atomisp_acc_init(&asd->video_acc, "ACC");
-
-	ret = v4l2_ctrl_handler_init(&asd->ctrl_handler, 1);
-	if (ret)
-		return ret;
-
-	asd->fmt_auto = v4l2_ctrl_new_custom(&asd->ctrl_handler,
-						    &ctrl_fmt_auto, NULL);
-	asd->run_mode = v4l2_ctrl_new_custom(&asd->ctrl_handler,
-						    &ctrl_run_mode, NULL);
-	asd->vfpp = v4l2_ctrl_new_custom(&asd->ctrl_handler,
-						&ctrl_vfpp, NULL);
-	asd->continuous_mode = v4l2_ctrl_new_custom(&asd->ctrl_handler,
-					     &ctrl_continuous_mode, NULL);
-	asd->continuous_viewfinder = v4l2_ctrl_new_custom(&asd->ctrl_handler,
-					     &ctrl_continuous_viewfinder,
-					     NULL);
-	asd->continuous_raw_buffer_size =
-			v4l2_ctrl_new_custom(&asd->ctrl_handler,
-					     &ctrl_continuous_raw_buffer_size,
-					     NULL);
-
-	asd->enable_raw_buffer_lock =
-			v4l2_ctrl_new_custom(&asd->ctrl_handler,
-					     &ctrl_enable_raw_buffer_lock,
-					     NULL);
-	asd->depth_mode =
-			v4l2_ctrl_new_custom(&asd->ctrl_handler,
-					     &ctrl_depth_mode,
-					     NULL);
-	asd->disable_dz =
-			v4l2_ctrl_new_custom(&asd->ctrl_handler,
-					     &ctrl_disable_dz,
-					     NULL);
-#ifdef ISP2401
-	asd->select_isp_version =
-			v4l2_ctrl_new_custom(&asd->ctrl_handler,
-					     &ctrl_select_isp_version,
-					     NULL);
-
-#ifdef CONFIG_ION
-	asd->ion_dev_fd =
-			v4l2_ctrl_new_custom(&asd->ctrl_handler,
-						&ctrl_ion_dev_fd,
-						 NULL);
-#endif
-#endif
-
-	/* Make controls visible on subdev as well. */
-	asd->subdev.ctrl_handler = &asd->ctrl_handler;
-	spin_lock_init(&asd->raw_buffer_bitmap_lock);
-	return asd->ctrl_handler.error;
-}
-
-int atomisp_create_pads_links(struct atomisp_device *isp)
-{
-	struct atomisp_sub_device *asd;
-	int i, j, ret = 0;
-	isp->num_of_streams = 2;
-	for (i = 0; i < ATOMISP_CAMERA_NR_PORTS; i++) {
-		for (j = 0; j < isp->num_of_streams; j++) {
-			ret =
-			    media_create_pad_link(&isp->csi2_port[i].subdev.
-						  entity, CSI2_PAD_SOURCE,
-						  &isp->asd[j].subdev.entity,
-						  ATOMISP_SUBDEV_PAD_SINK, 0);
-			if (ret < 0)
-				return ret;
-		}
-	}
-	for (i = 0; i < isp->input_cnt - 2; i++) {
-		ret = media_create_pad_link(&isp->inputs[i].camera->entity, 0,
-					    &isp->csi2_port[isp->inputs[i].
-							    port].subdev.entity,
-					    CSI2_PAD_SINK,
-					    MEDIA_LNK_FL_ENABLED |
-					    MEDIA_LNK_FL_IMMUTABLE);
-		if (ret < 0)
-			return ret;
-	}
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-		ret = media_create_pad_link(&asd->subdev.entity,
-					    ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW,
-					    &asd->video_out_preview.vdev.entity,
-					    0, 0);
-		if (ret < 0)
-			return ret;
-		ret = media_create_pad_link(&asd->subdev.entity,
-					    ATOMISP_SUBDEV_PAD_SOURCE_VF,
-					    &asd->video_out_vf.vdev.entity, 0,
-					    0);
-		if (ret < 0)
-			return ret;
-		ret = media_create_pad_link(&asd->subdev.entity,
-					    ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE,
-					    &asd->video_out_capture.vdev.entity,
-					    0, 0);
-		if (ret < 0)
-			return ret;
-		ret = media_create_pad_link(&asd->subdev.entity,
-					    ATOMISP_SUBDEV_PAD_SOURCE_VIDEO,
-					    &asd->video_out_video_capture.vdev.
-					    entity, 0, 0);
-		if (ret < 0)
-			return ret;
-		/*
-		 * file input only supported on subdev0
-		 * so do not create pad link for subdevs other then subdev0
-		 */
-		if (asd->index)
-			return 0;
-		ret = media_create_pad_link(&asd->video_in.vdev.entity,
-					    0, &asd->subdev.entity,
-					    ATOMISP_SUBDEV_PAD_SINK, 0);
-		if (ret < 0)
-			return ret;
-	}
-	return 0;
-}
-
-static void atomisp_subdev_cleanup_entities(struct atomisp_sub_device *asd)
-{
-	v4l2_ctrl_handler_free(&asd->ctrl_handler);
-
-	media_entity_cleanup(&asd->subdev.entity);
-}
-
-void atomisp_subdev_cleanup_pending_events(struct atomisp_sub_device *asd)
-{
-	struct v4l2_fh *fh, *fh_tmp;
-	struct v4l2_event event;
-	unsigned int i, pending_event;
-
-	list_for_each_entry_safe(fh, fh_tmp,
-		&asd->subdev.devnode->fh_list, list) {
-		pending_event = v4l2_event_pending(fh);
-		for (i = 0; i < pending_event; i++)
-			v4l2_event_dequeue(fh, &event, 1);
-	}
-}
-
-void atomisp_subdev_unregister_entities(struct atomisp_sub_device *asd)
-{
-	atomisp_subdev_cleanup_entities(asd);
-	v4l2_device_unregister_subdev(&asd->subdev);
-	atomisp_video_unregister(&asd->video_in);
-	atomisp_video_unregister(&asd->video_out_preview);
-	atomisp_video_unregister(&asd->video_out_vf);
-	atomisp_video_unregister(&asd->video_out_capture);
-	atomisp_video_unregister(&asd->video_out_video_capture);
-	atomisp_acc_unregister(&asd->video_acc);
-}
-
-int atomisp_subdev_register_entities(struct atomisp_sub_device *asd,
-	struct v4l2_device *vdev)
-{
-	int ret;
-
-	/* Register the subdev and video node. */
-	ret = v4l2_device_register_subdev(vdev, &asd->subdev);
-	if (ret < 0)
-		goto error;
-
-	ret = atomisp_video_register(&asd->video_out_capture, vdev);
-	if (ret < 0)
-		goto error;
-
-	ret = atomisp_video_register(&asd->video_out_vf, vdev);
-	if (ret < 0)
-		goto error;
-
-	ret = atomisp_video_register(&asd->video_out_preview, vdev);
-	if (ret < 0)
-		goto error;
-
-	ret = atomisp_video_register(&asd->video_out_video_capture, vdev);
-	if (ret < 0)
-		goto error;
-
-	ret = atomisp_acc_register(&asd->video_acc, vdev);
-	if (ret < 0)
-		goto error;
-
-	/*
-	 * file input only supported on subdev0
-	 * so do not create video node for subdevs other then subdev0
-	 */
-	if (asd->index)
-		return 0;
-	ret = atomisp_video_register(&asd->video_in, vdev);
-	if (ret < 0)
-		goto error;
-
-	return 0;
-
-error:
-	atomisp_subdev_unregister_entities(asd);
-	return ret;
-}
-
-/*
- * atomisp_subdev_init - ISP Subdevice  initialization.
- * @dev: Device pointer specific to the ATOM ISP.
- *
- * TODO: Get the initialisation values from platform data.
- *
- * Return 0 on success or a negative error code otherwise.
- */
-int atomisp_subdev_init(struct atomisp_device *isp)
-{
-	struct atomisp_sub_device *asd;
-	int i, ret = 0;
-
-	/*
-	 * CSS2.0 running ISP2400 support
-	 * multiple streams
-	 */
-	isp->num_of_streams = 2;
-	isp->asd = devm_kzalloc(isp->dev, sizeof(struct atomisp_sub_device) *
-			       isp->num_of_streams, GFP_KERNEL);
-	if (!isp->asd)
-		return -ENOMEM;
-	for (i = 0; i < isp->num_of_streams; i++) {
-		asd = &isp->asd[i];
-		spin_lock_init(&asd->lock);
-		asd->isp = isp;
-		isp_subdev_init_params(asd);
-		asd->index = i;
-		ret = isp_subdev_init_entities(asd);
-		if (ret < 0) {
-			atomisp_subdev_cleanup_entities(asd);
-			break;
-		}
-	}
-
-	return ret;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.h
deleted file mode 100644
index 59ff8723c182..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.h
+++ /dev/null
@@ -1,467 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef __ATOMISP_SUBDEV_H__
-#define __ATOMISP_SUBDEV_H__
-
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-subdev.h>
-#include <media/videobuf-core.h>
-
-#include "atomisp_common.h"
-#include "atomisp_compat.h"
-#include "atomisp_v4l2.h"
-
-#include "ia_css.h"
-
-/* EXP_ID's ranger is 1 ~ 250 */
-#define ATOMISP_MAX_EXP_ID     (250)
-enum atomisp_subdev_input_entity {
-	ATOMISP_SUBDEV_INPUT_NONE,
-	ATOMISP_SUBDEV_INPUT_MEMORY,
-	ATOMISP_SUBDEV_INPUT_CSI2,
-	/*
-	 * The following enum for CSI2 port must go together in one row.
-	 * Otherwise it breaks the code logic.
-	 */
-	ATOMISP_SUBDEV_INPUT_CSI2_PORT1,
-	ATOMISP_SUBDEV_INPUT_CSI2_PORT2,
-	ATOMISP_SUBDEV_INPUT_CSI2_PORT3,
-};
-
-#define ATOMISP_SUBDEV_PAD_SINK			0
-/* capture output for still frames */
-#define ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE	1
-/* viewfinder output for downscaled capture output */
-#define ATOMISP_SUBDEV_PAD_SOURCE_VF		2
-/* preview output for display */
-#define ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW	3
-/* main output for video pipeline */
-#define ATOMISP_SUBDEV_PAD_SOURCE_VIDEO	4
-#define ATOMISP_SUBDEV_PADS_NUM			5
-
-struct atomisp_in_fmt_conv {
-	u32     code;
-	uint8_t bpp; /* bits per pixel */
-	uint8_t depth; /* uncompressed */
-	enum atomisp_input_format atomisp_in_fmt;
-	enum atomisp_css_bayer_order bayer_order;
-	enum atomisp_input_format css_stream_fmt;
-};
-
-struct atomisp_sub_device;
-
-struct atomisp_video_pipe {
-	struct video_device vdev;
-	enum v4l2_buf_type type;
-	struct media_pad pad;
-	struct videobuf_queue capq;
-	struct videobuf_queue outq;
-	struct list_head activeq;
-	struct list_head activeq_out;
-	/*
-	 * the buffers waiting for per-frame parameters, this is only valid
-	 * in per-frame setting mode.
-	 */
-	struct list_head buffers_waiting_for_param;
-	/* the link list to store per_frame parameters */
-	struct list_head per_frame_params;
-
-	unsigned int buffers_in_css;
-
-	/* irq_lock is used to protect video buffer state change operations and
-	 * also to make activeq, activeq_out, capq and outq list
-	 * operations atomic. */
-	spinlock_t irq_lock;
-	unsigned int users;
-
-	struct atomisp_device *isp;
-	struct v4l2_pix_format pix;
-	uint32_t sh_fmt;
-
-	struct atomisp_sub_device *asd;
-
-	/*
-	 * This frame_config_id is got from CSS when dequueues buffers from CSS,
-	 * it is used to indicate which parameter it has applied.
-	 */
-	unsigned int frame_config_id[VIDEO_MAX_FRAME];
-	/*
-	 * This config id is set when camera HAL enqueues buffer, it has a
-	 * non-zero value to indicate which parameter it needs to applu
-	 */
-	unsigned int frame_request_config_id[VIDEO_MAX_FRAME];
-	struct atomisp_css_params_with_list *frame_params[VIDEO_MAX_FRAME];
-#ifdef ISP2401
-
-	/*
-	* move wdt from asd struct to create wdt for each pipe
-	*/
-	struct timer_list wdt;
-	unsigned int wdt_duration;	/* in jiffies */
-	unsigned long wdt_expires;
-	atomic_t wdt_count;
-#endif
-};
-
-struct atomisp_acc_pipe {
-	struct video_device vdev;
-	unsigned int users;
-	bool running;
-	struct atomisp_sub_device *asd;
-	struct atomisp_device *isp;
-};
-
-struct atomisp_pad_format {
-	struct v4l2_mbus_framefmt fmt;
-	struct v4l2_rect crop;
-	struct v4l2_rect compose;
-};
-
-/* Internal states for flash process */
-enum atomisp_flash_state {
-	ATOMISP_FLASH_IDLE,
-	ATOMISP_FLASH_REQUESTED,
-	ATOMISP_FLASH_ONGOING,
-	ATOMISP_FLASH_DONE
-};
-
-/*
- * This structure is used to cache the CSS parameters, it aligns to
- * struct ia_css_isp_config but without un-supported and deprecated parts.
- */
-struct atomisp_css_params {
-	struct ia_css_wb_config   wb_config;
-	struct ia_css_cc_config   cc_config;
-	struct ia_css_tnr_config  tnr_config;
-	struct ia_css_ecd_config  ecd_config;
-	struct ia_css_ynr_config  ynr_config;
-	struct ia_css_fc_config   fc_config;
-	struct ia_css_formats_config formats_config;
-	struct ia_css_cnr_config  cnr_config;
-	struct ia_css_macc_config macc_config;
-	struct ia_css_ctc_config  ctc_config;
-	struct ia_css_aa_config   aa_config;
-	struct ia_css_aa_config   baa_config;
-	struct ia_css_ce_config   ce_config;
-	struct ia_css_ob_config   ob_config;
-	struct ia_css_dp_config   dp_config;
-	struct ia_css_de_config   de_config;
-	struct ia_css_gc_config   gc_config;
-	struct ia_css_nr_config   nr_config;
-	struct ia_css_ee_config   ee_config;
-	struct ia_css_anr_config  anr_config;
-	struct ia_css_3a_config   s3a_config;
-	struct ia_css_xnr_config  xnr_config;
-	struct ia_css_dz_config   dz_config;
-	struct ia_css_cc_config yuv2rgb_cc_config;
-	struct ia_css_cc_config rgb2yuv_cc_config;
-	struct ia_css_macc_table  macc_table;
-	struct ia_css_gamma_table gamma_table;
-	struct ia_css_ctc_table   ctc_table;
-
-	struct ia_css_xnr_table   xnr_table;
-	struct ia_css_rgb_gamma_table r_gamma_table;
-	struct ia_css_rgb_gamma_table g_gamma_table;
-	struct ia_css_rgb_gamma_table b_gamma_table;
-
-	struct ia_css_vector      motion_vector;
-	struct ia_css_anr_thres   anr_thres;
-
-	struct ia_css_dvs_6axis_config *dvs_6axis;
-	struct ia_css_dvs2_coefficients *dvs2_coeff;
-	struct ia_css_shading_table *shading_table;
-	struct ia_css_morph_table   *morph_table;
-
-	/*
-	 * Used to store the user pointer address of the frame. driver needs to
-	 * translate to ia_css_frame * and then set to CSS.
-	 */
-	void		*output_frame;
-	uint32_t	isp_config_id;
-
-	/* Indicates which parameters need to be updated. */
-	struct atomisp_parameters update_flag;
-};
-
-struct atomisp_subdev_params {
-	/* FIXME: Determines whether raw capture buffer are being passed to
-	 * user space. Unimplemented for now. */
-	int online_process;
-	int yuv_ds_en;
-	unsigned int color_effect;
-	bool gdc_cac_en;
-	bool macc_en;
-	bool bad_pixel_en;
-	bool video_dis_en;
-	bool sc_en;
-	bool fpn_en;
-	bool xnr_en;
-	bool low_light;
-	int false_color;
-	unsigned int histogram_elenum;
-
-	/* Current grid info */
-	struct atomisp_css_grid_info curr_grid_info;
-	enum atomisp_css_pipe_id s3a_enabled_pipe;
-
-	int s3a_output_bytes;
-
-	bool dis_proj_data_valid;
-
-	struct ia_css_dz_config   dz_config;  /** Digital Zoom */
-	struct ia_css_capture_config   capture_config;
-
-	struct atomisp_css_isp_config config;
-
-	/* current configurations */
-	struct atomisp_css_params css_param;
-
-	/*
-	 * Intermediate buffers used to communicate data between
-	 * CSS and user space.
-	 */
-	struct ia_css_3a_statistics *s3a_user_stat;
-
-	void *metadata_user[ATOMISP_METADATA_TYPE_NUM];
-	uint32_t metadata_width_size;
-
-	struct ia_css_dvs2_statistics *dvs_stat;
-	struct atomisp_css_dvs_6axis *dvs_6axis;
-	uint32_t exp_id;
-	int  dvs_hor_coef_bytes;
-	int  dvs_ver_coef_bytes;
-	int  dvs_ver_proj_bytes;
-	int  dvs_hor_proj_bytes;
-
-	/* Flash */
-	int num_flash_frames;
-	enum atomisp_flash_state flash_state;
-	enum atomisp_frame_status last_frame_status;
-
-	/* continuous capture */
-	struct atomisp_cont_capture_conf offline_parm;
-	/* Flag to check if driver needs to update params to css */
-	bool css_update_params_needed;
-};
-
-struct atomisp_css_params_with_list {
-	/* parameters for CSS */
-	struct atomisp_css_params params;
-	struct list_head list;
-};
-
-struct atomisp_acc_fw {
-	struct atomisp_css_fw_info *fw;
-	unsigned int handle;
-	unsigned int flags;
-	unsigned int type;
-	struct {
-		size_t length;
-		unsigned long css_ptr;
-	} args[ATOMISP_ACC_NR_MEMORY];
-	struct list_head list;
-};
-
-struct atomisp_map {
-	ia_css_ptr ptr;
-	size_t length;
-	struct list_head list;
-	/* FIXME: should keep book which maps are currently used
-	 * by binaries and not allow releasing those
-	 * which are in use. Implement by reference counting.
-	 */
-};
-
-struct atomisp_sub_device {
-	struct v4l2_subdev subdev;
-	struct media_pad pads[ATOMISP_SUBDEV_PADS_NUM];
-	struct atomisp_pad_format fmt[ATOMISP_SUBDEV_PADS_NUM];
-	uint16_t capture_pad; /* main capture pad; defines much of isp config */
-
-	enum atomisp_subdev_input_entity input;
-	unsigned int output;
-	struct atomisp_video_pipe video_in;
-	struct atomisp_video_pipe video_out_capture; /* capture output */
-	struct atomisp_video_pipe video_out_vf;      /* viewfinder output */
-	struct atomisp_video_pipe video_out_preview; /* preview output */
-	struct atomisp_acc_pipe video_acc;
-	/* video pipe main output */
-	struct atomisp_video_pipe video_out_video_capture;
-	/* struct isp_subdev_params params; */
-	spinlock_t lock;
-	struct atomisp_device *isp;
-	struct v4l2_ctrl_handler ctrl_handler;
-	struct v4l2_ctrl *fmt_auto;
-	struct v4l2_ctrl *run_mode;
-	struct v4l2_ctrl *depth_mode;
-	struct v4l2_ctrl *vfpp;
-	struct v4l2_ctrl *continuous_mode;
-	struct v4l2_ctrl *continuous_raw_buffer_size;
-	struct v4l2_ctrl *continuous_viewfinder;
-	struct v4l2_ctrl *enable_raw_buffer_lock;
-#ifdef ISP2401
-	struct v4l2_ctrl *ion_dev_fd;
-#endif
-	struct v4l2_ctrl *disable_dz;
-#ifdef ISP2401
-	struct v4l2_ctrl *select_isp_version;
-#endif
-
-	struct {
-		struct list_head fw;
-		struct list_head memory_maps;
-		struct atomisp_css_pipeline *pipeline;
-		bool extension_mode;
-		struct ida ida;
-		struct completion acc_done;
-		void *acc_stages;
-	} acc;
-
-	struct atomisp_subdev_params params;
-
-	struct atomisp_stream_env stream_env[ATOMISP_INPUT_STREAM_NUM];
-
-	struct v4l2_pix_format dvs_envelop;
-	unsigned int s3a_bufs_in_css[CSS_PIPE_ID_NUM];
-	unsigned int dis_bufs_in_css;
-
-	unsigned int metadata_bufs_in_css
-		[ATOMISP_INPUT_STREAM_NUM][CSS_PIPE_ID_NUM];
-	/* The list of free and available metadata buffers for CSS */
-	struct list_head metadata[ATOMISP_METADATA_TYPE_NUM];
-	/* The list of metadata buffers which have been en-queued to CSS */
-	struct list_head metadata_in_css[ATOMISP_METADATA_TYPE_NUM];
-	/* The list of metadata buffers which are ready for userspace to get */
-	struct list_head metadata_ready[ATOMISP_METADATA_TYPE_NUM];
-
-	/* The list of free and available s3a stat buffers for CSS */
-	struct list_head s3a_stats;
-	/* The list of s3a stat buffers which have been en-queued to CSS */
-	struct list_head s3a_stats_in_css;
-	/* The list of s3a stat buffers which are ready for userspace to get */
-	struct list_head s3a_stats_ready;
-
-	struct list_head dis_stats;
-	struct list_head dis_stats_in_css;
-	spinlock_t dis_stats_lock;
-
-	struct atomisp_css_frame *vf_frame; /* TODO: needed? */
-	struct atomisp_css_frame *raw_output_frame;
-	enum atomisp_frame_status frame_status[VIDEO_MAX_FRAME];
-
-	/* This field specifies which camera (v4l2 input) is selected. */
-	int input_curr;
-	/* This field specifies which sensor is being selected when there
-	   are multiple sensors connected to the same MIPI port. */
-	int sensor_curr;
-
-	atomic_t sof_count;
-	atomic_t sequence;      /* Sequence value that is assigned to buffer. */
-	atomic_t sequence_temp;
-
-	unsigned int streaming; /* Hold both mutex and lock to change this */
-	bool stream_prepared; /* whether css stream is created */
-
-	/* subdev index: will be used to show which subdev is holding the
-	 * resource, like which camera is used by which subdev
-	 */
-	unsigned int index;
-
-	/* delayed memory allocation for css */
-	struct completion init_done;
-	struct workqueue_struct *delayed_init_workq;
-	unsigned int delayed_init;
-	struct work_struct delayed_init_work;
-
-	unsigned int latest_preview_exp_id; /* CSS ZSL/SDV raw buffer id */
-
-	unsigned int mipi_frame_size;
-
-	bool copy_mode; /* CSI2+ use copy mode */
-	bool yuvpp_mode;	/* CSI2+ yuvpp pipe */
-
-	int raw_buffer_bitmap[ATOMISP_MAX_EXP_ID/32 + 1]; /* Record each Raw Buffer lock status */
-	int raw_buffer_locked_count;
-	spinlock_t raw_buffer_bitmap_lock;
-
-#ifndef ISP2401
-	struct timer_list wdt;
-	unsigned int wdt_duration;	/* in jiffies */
-	unsigned long wdt_expires;
-
-#endif
-	struct atomisp_resolution sensor_array_res;
-	bool high_speed_mode; /* Indicate whether now is a high speed mode */
-	int pending_capture_request; /* Indicates the number of pending capture requests. */
-#ifndef ISP2401
-
-#else
-	bool re_trigger_capture;
-#endif
-	unsigned int preview_exp_id;
-	unsigned int postview_exp_id;
-};
-
-extern const struct atomisp_in_fmt_conv atomisp_in_fmt_conv[];
-
-u32 atomisp_subdev_uncompressed_code(u32 code);
-bool atomisp_subdev_is_compressed(u32 code);
-const struct atomisp_in_fmt_conv *atomisp_find_in_fmt_conv(u32 code);
-#ifndef ISP2401
-const struct atomisp_in_fmt_conv *atomisp_find_in_fmt_conv_by_atomisp_in_fmt(
-	enum atomisp_input_format atomisp_in_fmt);
-#else
-const struct atomisp_in_fmt_conv
-    *atomisp_find_in_fmt_conv_by_atomisp_in_fmt(enum atomisp_input_format
-						atomisp_in_fmt);
-#endif
-const struct atomisp_in_fmt_conv *atomisp_find_in_fmt_conv_compressed(u32 code);
-bool atomisp_subdev_format_conversion(struct atomisp_sub_device *asd,
-				      unsigned int source_pad);
-uint16_t atomisp_subdev_source_pad(struct video_device *vdev);
-
-/* Get pointer to appropriate format */
-struct v4l2_mbus_framefmt
-*atomisp_subdev_get_ffmt(struct v4l2_subdev *sd,
-			 struct v4l2_subdev_pad_config *cfg, uint32_t which,
-			 uint32_t pad);
-struct v4l2_rect *atomisp_subdev_get_rect(struct v4l2_subdev *sd,
-					  struct v4l2_subdev_pad_config *cfg,
-					  uint32_t which, uint32_t pad,
-					  uint32_t target);
-int atomisp_subdev_set_selection(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_pad_config *cfg,
-				 uint32_t which, uint32_t pad, uint32_t target,
-				 uint32_t flags, struct v4l2_rect *r);
-/* Actually set the format */
-void atomisp_subdev_set_ffmt(struct v4l2_subdev *sd,
-			     struct v4l2_subdev_pad_config *cfg, uint32_t which,
-			     uint32_t pad, struct v4l2_mbus_framefmt *ffmt);
-
-int atomisp_update_run_mode(struct atomisp_sub_device *asd);
-
-void atomisp_subdev_cleanup_pending_events(struct atomisp_sub_device *asd);
-
-void atomisp_subdev_unregister_entities(struct atomisp_sub_device *asd);
-int atomisp_subdev_register_entities(struct atomisp_sub_device *asd,
-	struct v4l2_device *vdev);
-int atomisp_subdev_init(struct atomisp_device *isp);
-void atomisp_subdev_cleanup(struct atomisp_device *isp);
-int atomisp_create_pads_links(struct atomisp_device *isp);
-
-#endif /* __ATOMISP_SUBDEV_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_tables.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_tables.h
deleted file mode 100644
index 319ded6a96da..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_tables.h
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef	__ATOMISP_TABLES_H__
-#define	__ATOMISP_TABLES_H__
-
-#include "sh_css_params.h"
-
-/*Sepia image effect table*/
-static struct atomisp_css_cc_config sepia_cc_config = {
-	.fraction_bits  = 8,
-	.matrix	 = {141, 18, 68, -40, -5, -19, 35, 4, 16},
-};
-
-/*Negative image effect table*/
-static struct atomisp_css_cc_config nega_cc_config = {
-	.fraction_bits  = 8,
-	.matrix	 = {255, 29, 120, 0, 374, 342, 0, 672, -301},
-};
-
-/*Mono image effect table*/
-static struct atomisp_css_cc_config mono_cc_config = {
-	.fraction_bits  = 8,
-	.matrix	 = {255, 29, 120, 0, 0, 0, 0, 0, 0},
-};
-
-/*Skin whiten image effect table*/
-static struct atomisp_css_macc_table skin_low_macc_table = {
-	.data = {
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	7168, 0, 2048, 8192,
-	5120, -1024, 2048, 8192,
-	8192, 2048, -1024, 5120,
-	8192, 2048, 0, 7168,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192
-	}
-};
-
-static struct atomisp_css_macc_table skin_medium_macc_table = {
-	.data = {
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	5120, 0, 6144, 8192,
-	3072, -1024, 2048, 6144,
-	6144, 2048, -1024, 3072,
-	8192, 6144, 0, 5120,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192
-	}
-};
-
-static struct atomisp_css_macc_table skin_high_macc_table = {
-	.data = {
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	4096, 0, 8192, 8192,
-	0, -2048, 4096, 6144,
-	6144, 4096, -2048, 0,
-	8192, 8192, 0, 4096,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192
-	}
-};
-
-/*Blue enhencement image effect table*/
-static struct atomisp_css_macc_table blue_macc_table = {
-	.data = {
-	9728, -3072, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	9728, 0, -3072, 8192,
-	12800, 1536, -3072, 8192,
-	11264, 0, 0, 11264,
-	9728, -3072, 0, 11264
-	}
-};
-
-/*Green enhencement image effect table*/
-static struct atomisp_css_macc_table green_macc_table = {
-	.data = {
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	10240, 4096, 0, 8192,
-	10240, 4096, 0, 12288,
-	12288, 0, 0, 12288,
-	14336, -2048, 4096, 8192,
-	10240, 0, 4096, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192,
-	8192, 0, 0, 8192
-	}
-};
-
-static struct atomisp_css_ctc_table vivid_ctc_table = {
-	.data.vamem_2 = {
-	0,  384,  837,  957, 1011, 1062, 1083, 1080,
-	1078, 1077, 1053, 1039, 1012,  992,  969,  951,
-	929,  906,  886,  866,  845,  823,  809,  790,
-	772,  758,  741,  726,  711,  701,  688,  675,
-	666,  656,  648,  639,  633,  626,  618,  612,
-	603,  594,  582,  572,  557,  545,  529,  516,
-	504,  491,  480,  467,  459,  447,  438,  429,
-	419,  412,  404,  397,  389,  382,  376,  368,
-	363,  357,  351,  345,  340,  336,  330,  326,
-	321,  318,  312,  308,  304,  300,  297,  294,
-	291,  286,  284,  281,  278,  275,  271,  268,
-	261,  257,  251,  245,  240,  235,  232,  225,
-	223,  218,  213,  209,  206,  204,  199,  197,
-	193,  189,  186,  185,  183,  179,  177,  175,
-	172,  170,  169,  167,  164,  164,  162,  160,
-	158,  157,  156,  154,  154,  152,  151,  150,
-	149,  148,  146,  147,  146,  144,  143,  143,
-	142,  141,  140,  141,  139,  138,  138,  138,
-	137,  136,  136,  135,  134,  134,  134,  133,
-	132,  132,  131,  130,  131,  130,  129,  128,
-	129,  127,  127,  127,  127,  125,  125,  125,
-	123,  123,  122,  120,  118,  115,  114,  111,
-	110,  108,  106,  105,  103,  102,  100,   99,
-	97,   97,   96,   95,   94,   93,   93,   91,
-	91,   91,   90,   90,   89,   89,   88,   88,
-	89,   88,   88,   87,   87,   87,   87,   86,
-	87,   87,   86,   87,   86,   86,   84,   84,
-	82,   80,   78,   76,   74,   72,   70,   68,
-	67,   65,   62,   60,   58,   56,   55,   54,
-	53,   51,   49,   49,   47,   45,   45,   45,
-	41,   40,   39,   39,   34,   33,   34,   32,
-	25,   23,   24,   20,   13,    9,   12,    0,
-	0
-	}
-};
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_tpg.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_tpg.c
deleted file mode 100644
index adc900272f6f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_tpg.c
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#include <media/v4l2-event.h>
-#include <media/v4l2-mediabus.h>
-#include "atomisp_internal.h"
-#include "atomisp_tpg.h"
-
-static int tpg_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	return 0;
-}
-
-static int tpg_get_fmt(struct v4l2_subdev *sd,
-		       struct v4l2_subdev_pad_config *cfg,
-		       struct v4l2_subdev_format *format)
-{
-	/*to fake*/
-	return 0;
-}
-
-static int tpg_set_fmt(struct v4l2_subdev *sd,
-		       struct v4l2_subdev_pad_config *cfg,
-		       struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-
-	if (format->pad)
-		return -EINVAL;
-	/* only raw8 grbg is supported by TPG */
-	fmt->code = MEDIA_BUS_FMT_SGRBG8_1X8;
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
-		cfg->try_fmt = *fmt;
-		return 0;
-	}
-	return 0;
-}
-
-static int tpg_log_status(struct v4l2_subdev *sd)
-{
-	/*to fake*/
-	return 0;
-}
-
-static int tpg_s_power(struct v4l2_subdev *sd, int on)
-{
-	return 0;
-}
-
-static int tpg_enum_mbus_code(struct v4l2_subdev *sd,
-			      struct v4l2_subdev_pad_config *cfg,
-			      struct v4l2_subdev_mbus_code_enum *code)
-{
-	/*to fake*/
-	return 0;
-}
-
-static int tpg_enum_frame_size(struct v4l2_subdev *sd,
-			       struct v4l2_subdev_pad_config *cfg,
-			       struct v4l2_subdev_frame_size_enum *fse)
-{
-	/*to fake*/
-	return 0;
-}
-
-static int tpg_enum_frame_ival(struct v4l2_subdev *sd,
-			       struct v4l2_subdev_pad_config *cfg,
-			       struct v4l2_subdev_frame_interval_enum *fie)
-{
-	/*to fake*/
-	return 0;
-}
-
-static const struct v4l2_subdev_video_ops tpg_video_ops = {
-	.s_stream = tpg_s_stream,
-};
-
-static const struct v4l2_subdev_core_ops tpg_core_ops = {
-	.log_status = tpg_log_status,
-	.s_power = tpg_s_power,
-};
-
-static const struct v4l2_subdev_pad_ops tpg_pad_ops = {
-	.enum_mbus_code = tpg_enum_mbus_code,
-	.enum_frame_size = tpg_enum_frame_size,
-	.enum_frame_interval = tpg_enum_frame_ival,
-	.get_fmt = tpg_get_fmt,
-	.set_fmt = tpg_set_fmt,
-};
-
-static const struct v4l2_subdev_ops tpg_ops = {
-	.core = &tpg_core_ops,
-	.video = &tpg_video_ops,
-	.pad = &tpg_pad_ops,
-};
-
-void atomisp_tpg_unregister_entities(struct atomisp_tpg_device *tpg)
-{
-	media_entity_cleanup(&tpg->sd.entity);
-	v4l2_device_unregister_subdev(&tpg->sd);
-}
-
-int atomisp_tpg_register_entities(struct atomisp_tpg_device *tpg,
-			struct v4l2_device *vdev)
-{
-	int ret;
-	/* Register the subdev and video nodes. */
-	ret = v4l2_device_register_subdev(vdev, &tpg->sd);
-	if (ret < 0)
-		goto error;
-
-	return 0;
-
-error:
-	atomisp_tpg_unregister_entities(tpg);
-	return ret;
-}
-
-void atomisp_tpg_cleanup(struct atomisp_device *isp)
-{
-
-}
-
-int atomisp_tpg_init(struct atomisp_device *isp)
-{
-	struct atomisp_tpg_device *tpg = &isp->tpg;
-	struct v4l2_subdev *sd = &tpg->sd;
-	struct media_pad *pads = tpg->pads;
-	struct media_entity *me = &sd->entity;
-	int ret;
-
-	tpg->isp = isp;
-	v4l2_subdev_init(sd, &tpg_ops);
-	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	strcpy(sd->name, "tpg_subdev");
-	v4l2_set_subdevdata(sd, tpg);
-
-	pads[0].flags = MEDIA_PAD_FL_SINK;
-	me->function = MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN;
-
-	ret = media_entity_pads_init(me, 1, pads);
-	if (ret < 0)
-		goto fail;
-	return 0;
-fail:
-	atomisp_tpg_cleanup(isp);
-	return ret;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_tpg.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_tpg.h
deleted file mode 100644
index af354c4bfd3e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_tpg.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __ATOMISP_TPG_H__
-#define __ATOMISP_TPG_H__
-
-#include <media/media-entity.h>
-#include <media/v4l2-subdev.h>
-
-struct atomisp_tpg_device {
-	struct v4l2_subdev sd;
-	struct atomisp_device *isp;
-	struct media_pad pads[1];
-};
-
-void atomisp_tpg_cleanup(struct atomisp_device *isp);
-int atomisp_tpg_init(struct atomisp_device *isp);
-void atomisp_tpg_unregister_entities(struct atomisp_tpg_device *tpg);
-int atomisp_tpg_register_entities(struct atomisp_tpg_device *tpg,
-			struct v4l2_device *vdev);
-
-#endif /* __ATOMISP_TPG_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_trace_event.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_trace_event.h
deleted file mode 100644
index 462b296554c7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_trace_event.h
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Support Camera Imaging tracer core.
- *
- * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#undef TRACE_SYSTEM
-#define TRACE_SYSTEM atomisp
-
-#if !defined(ATOMISP_TRACE_EVENT_H) || defined(TRACE_HEADER_MULTI_READ)
-#define ATOMISP_TRACE_EVENT_H
-
-#include <linux/tracepoint.h>
-#include <linux/string.h>
-TRACE_EVENT(camera_meminfo,
-
-	TP_PROTO(const char *name, int uptr_size, int counter, int sys_size,
-		int sys_res_size, int cam_sys_use, int cam_dyc_use,
-		int cam_res_use),
-
-	TP_ARGS(name, uptr_size, counter, sys_size, sys_res_size, cam_sys_use,
-		cam_dyc_use, cam_res_use),
-
-	TP_STRUCT__entry(
-		__array(char, name, 24)
-		__field(int, uptr_size)
-		__field(int, counter)
-		__field(int, sys_size)
-		__field(int, sys_res_size)
-		__field(int, cam_res_use)
-		__field(int, cam_dyc_use)
-		__field(int, cam_sys_use)
-	),
-
-	TP_fast_assign(
-		strlcpy(__entry->name, name, 24);
-		__entry->uptr_size = uptr_size;
-		__entry->counter = counter;
-		__entry->sys_size = sys_size;
-		__entry->sys_res_size = sys_res_size;
-		__entry->cam_res_use = cam_res_use;
-		__entry->cam_dyc_use = cam_dyc_use;
-		__entry->cam_sys_use = cam_sys_use;
-	),
-
-	TP_printk(
-		"<%s> User ptr memory:%d pages,\tISP private memory used:%d"
-		" pages:\tsysFP system size:%d,\treserved size:%d"
-		"\tcamFP sysUse:%d,\tdycUse:%d,\tresUse:%d.\n",
-		__entry->name, __entry->uptr_size, __entry->counter,
-		__entry->sys_size, __entry->sys_res_size, __entry->cam_sys_use,
-		__entry->cam_dyc_use, __entry->cam_res_use)
-);
-
-TRACE_EVENT(camera_debug,
-
-	TP_PROTO(const char *name, char *info, const int line),
-
-	TP_ARGS(name, info, line),
-
-	TP_STRUCT__entry(
-		__array(char, name, 24)
-		__array(char, info, 24)
-		__field(int, line)
-	),
-
-	TP_fast_assign(
-		strlcpy(__entry->name, name, 24);
-		strlcpy(__entry->info, info, 24);
-		__entry->line = line;
-	),
-
-	TP_printk("<%s>-<%d> %s\n", __entry->name, __entry->line,
-		__entry->info)
-);
-
-TRACE_EVENT(ipu_cstate,
-
-		TP_PROTO(int cstate),
-
-		TP_ARGS(cstate),
-
-		TP_STRUCT__entry(
-			__field(int, cstate)
-		),
-
-		TP_fast_assign(
-			__entry->cstate = cstate;
-		),
-
-		TP_printk("cstate=%d", __entry->cstate)
-);
-
-TRACE_EVENT(ipu_pstate,
-
-		TP_PROTO(int freq, int util),
-
-		TP_ARGS(freq, util),
-
-		TP_STRUCT__entry(
-			__field(int, freq)
-			__field(int, util)
-		),
-
-		TP_fast_assign(
-			__entry->freq = freq;
-			__entry->util = util;
-		),
-
-		TP_printk("freq=%d util=%d", __entry->freq, __entry->util)
-);
-#endif
-
-#undef TRACE_INCLUDE_PATH
-#undef TRACE_INCLUDE_FILE
-#define TRACE_INCLUDE_PATH .
-#define TRACE_INCLUDE_FILE   atomisp_trace_event
-/* This part must be outside protection */
-#include <trace/define_trace.h>
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.c
deleted file mode 100644
index ba20344ec560..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.c
+++ /dev/null
@@ -1,1573 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010-2017 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <linux/pm_runtime.h>
-#include <linux/pm_qos.h>
-#include <linux/timer.h>
-#include <linux/delay.h>
-#include <linux/interrupt.h>
-
-#include <asm/iosf_mbi.h>
-
-#include "../../include/linux/atomisp_gmin_platform.h"
-
-#include "atomisp_cmd.h"
-#include "atomisp_common.h"
-#include "atomisp_fops.h"
-#include "atomisp_file.h"
-#include "atomisp_ioctl.h"
-#include "atomisp_internal.h"
-#include "atomisp_acc.h"
-#include "atomisp-regs.h"
-#include "atomisp_dfs_tables.h"
-#include "atomisp_drvfs.h"
-#include "hmm/hmm.h"
-#include "atomisp_trace_event.h"
-
-#include "hrt/hive_isp_css_mm_hrt.h"
-
-#include "device_access.h"
-
-/* G-Min addition: pull this in from intel_mid_pm.h */
-#define CSTATE_EXIT_LATENCY_C1  1
-
-static uint skip_fwload;
-module_param(skip_fwload, uint, 0644);
-MODULE_PARM_DESC(skip_fwload, "Skip atomisp firmware load");
-
-/* set reserved memory pool size in page */
-static unsigned int repool_pgnr;
-module_param(repool_pgnr, uint, 0644);
-MODULE_PARM_DESC(repool_pgnr,
-		"Set the reserved memory pool size in page (default:0)");
-
-/* set dynamic memory pool size in page */
-unsigned int dypool_pgnr = UINT_MAX;
-module_param(dypool_pgnr, uint, 0644);
-MODULE_PARM_DESC(dypool_pgnr,
-		"Set the dynamic memory pool size in page (default:0)");
-
-bool dypool_enable;
-module_param(dypool_enable, bool, 0644);
-MODULE_PARM_DESC(dypool_enable,
-		"dynamic memory pool enable/disable (default:disable)");
-
-/* memory optimization: deferred firmware loading */
-bool defer_fw_load;
-module_param(defer_fw_load, bool, 0644);
-MODULE_PARM_DESC(defer_fw_load,
-		"Defer FW loading until device is opened (default:disable)");
-
-/* cross componnet debug message flag */
-int dbg_level;
-module_param(dbg_level, int, 0644);
-MODULE_PARM_DESC(dbg_level, "debug message on/off (default:off)");
-
-/* log function switch */
-int dbg_func = 2;
-module_param(dbg_func, int, 0644);
-MODULE_PARM_DESC(dbg_func,
-		"log function switch non/trace_printk/printk (default:printk)");
-
-int mipicsi_flag;
-module_param(mipicsi_flag, int, 0644);
-MODULE_PARM_DESC(mipicsi_flag, "mipi csi compression predictor algorithm");
-
-/*set to 16x16 since this is the amount of lines and pixels the sensor
-exports extra. If these are kept at the 10x8 that they were on, in yuv
-downscaling modes incorrect resolutions where requested to the sensor
-driver with strange outcomes as a result. The proper way tot do this
-would be to have a list of tables the specify the sensor res, mipi rec,
-output res, and isp output res. however since we do not have this yet,
-the chosen solution is the next best thing. */
-int pad_w = 16;
-module_param(pad_w, int, 0644);
-MODULE_PARM_DESC(pad_w, "extra data for ISP processing");
-
-int pad_h = 16;
-module_param(pad_h, int, 0644);
-MODULE_PARM_DESC(pad_h, "extra data for ISP processing");
-
-struct device *atomisp_dev;
-
-void __iomem *atomisp_io_base;
-
-int atomisp_video_init(struct atomisp_video_pipe *video, const char *name)
-{
-	int ret;
-	const char *direction;
-
-	switch (video->type) {
-	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
-		direction = "output";
-		video->pad.flags = MEDIA_PAD_FL_SINK;
-		video->vdev.fops = &atomisp_fops;
-		video->vdev.ioctl_ops = &atomisp_ioctl_ops;
-		break;
-	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
-		direction = "input";
-		video->pad.flags = MEDIA_PAD_FL_SOURCE;
-		video->vdev.fops = &atomisp_file_fops;
-		video->vdev.ioctl_ops = &atomisp_file_ioctl_ops;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	ret = media_entity_pads_init(&video->vdev.entity, 1, &video->pad);
-	if (ret < 0)
-		return ret;
-
-	/* Initialize the video device. */
-	snprintf(video->vdev.name, sizeof(video->vdev.name),
-		 "ATOMISP ISP %s %s", name, direction);
-	video->vdev.release = video_device_release_empty;
-	video_set_drvdata(&video->vdev, video->isp);
-
-	return 0;
-}
-
-void atomisp_acc_init(struct atomisp_acc_pipe *video, const char *name)
-{
-	video->vdev.fops = &atomisp_fops;
-	video->vdev.ioctl_ops = &atomisp_ioctl_ops;
-
-	/* Initialize the video device. */
-	snprintf(video->vdev.name, sizeof(video->vdev.name),
-		 "ATOMISP ISP %s", name);
-	video->vdev.release = video_device_release_empty;
-	video_set_drvdata(&video->vdev, video->isp);
-}
-
-int atomisp_video_register(struct atomisp_video_pipe *video,
-	struct v4l2_device *vdev)
-{
-	int ret;
-
-	video->vdev.v4l2_dev = vdev;
-
-	ret = video_register_device(&video->vdev, VFL_TYPE_GRABBER, -1);
-	if (ret < 0)
-		dev_err(vdev->dev, "%s: could not register video device (%d)\n",
-			__func__, ret);
-
-	return ret;
-}
-
-int atomisp_acc_register(struct atomisp_acc_pipe *video,
-		struct v4l2_device *vdev)
-{
-	int ret;
-
-	video->vdev.v4l2_dev = vdev;
-
-	ret = video_register_device(&video->vdev, VFL_TYPE_GRABBER, -1);
-	if (ret < 0)
-		dev_err(vdev->dev, "%s: could not register video device (%d)\n",
-			__func__, ret);
-
-	return ret;
-}
-
-void atomisp_video_unregister(struct atomisp_video_pipe *video)
-{
-	if (video_is_registered(&video->vdev)) {
-		media_entity_cleanup(&video->vdev.entity);
-		video_unregister_device(&video->vdev);
-	}
-}
-
-void atomisp_acc_unregister(struct atomisp_acc_pipe *video)
-{
-	if (video_is_registered(&video->vdev))
-		video_unregister_device(&video->vdev);
-}
-
-static int atomisp_save_iunit_reg(struct atomisp_device *isp)
-{
-	struct pci_dev *dev = isp->pdev;
-
-	dev_dbg(isp->dev, "%s\n", __func__);
-
-	pci_read_config_word(dev, PCI_COMMAND, &isp->saved_regs.pcicmdsts);
-	/* isp->saved_regs.ispmmadr is set from the atomisp_pci_probe() */
-	pci_read_config_dword(dev, PCI_MSI_CAPID, &isp->saved_regs.msicap);
-	pci_read_config_dword(dev, PCI_MSI_ADDR, &isp->saved_regs.msi_addr);
-	pci_read_config_word(dev, PCI_MSI_DATA,  &isp->saved_regs.msi_data);
-	pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &isp->saved_regs.intr);
-	pci_read_config_dword(dev, PCI_INTERRUPT_CTRL,
-			      &isp->saved_regs.interrupt_control);
-
-	pci_read_config_dword(dev, MRFLD_PCI_PMCS,
-			      &isp->saved_regs.pmcs);
-	/* Ensure read/write combining is enabled. */
-	pci_read_config_dword(dev, PCI_I_CONTROL,
-			&isp->saved_regs.i_control);
-	isp->saved_regs.i_control |=
-			MRFLD_PCI_I_CONTROL_ENABLE_READ_COMBINING |
-			MRFLD_PCI_I_CONTROL_ENABLE_WRITE_COMBINING;
-	pci_read_config_dword(dev, MRFLD_PCI_CSI_ACCESS_CTRL_VIOL,
-			      &isp->saved_regs.csi_access_viol);
-	pci_read_config_dword(dev, MRFLD_PCI_CSI_RCOMP_CONTROL,
-			      &isp->saved_regs.csi_rcomp_config);
-	/*
-	 * Hardware bugs require setting CSI_HS_OVR_CLK_GATE_ON_UPDATE.
-	 * ANN/CHV: RCOMP updates do not happen when using CSI2+ path
-	 * and sensor sending "continuous clock".
-	 * TNG/ANN/CHV: MIPI packets are lost if the HS entry sequence
-	 * is missed, and IUNIT can hang.
-	 * For both issues, setting this bit is a workaround.
-	 */
-	isp->saved_regs.csi_rcomp_config |=
-		MRFLD_PCI_CSI_HS_OVR_CLK_GATE_ON_UPDATE;
-	pci_read_config_dword(dev, MRFLD_PCI_CSI_AFE_TRIM_CONTROL,
-			      &isp->saved_regs.csi_afe_dly);
-	pci_read_config_dword(dev, MRFLD_PCI_CSI_CONTROL,
-			      &isp->saved_regs.csi_control);
-	if (isp->media_dev.hw_revision >=
-	    (ATOMISP_HW_REVISION_ISP2401 << ATOMISP_HW_REVISION_SHIFT))
-		isp->saved_regs.csi_control |=
-			MRFLD_PCI_CSI_CONTROL_PARPATHEN;
-	/*
-	 * On CHT CSI_READY bit should be enabled before stream on
-	 */
-	if (IS_CHT && (isp->media_dev.hw_revision >= ((ATOMISP_HW_REVISION_ISP2401 <<
-	    ATOMISP_HW_REVISION_SHIFT) | ATOMISP_HW_STEPPING_B0)))
-		isp->saved_regs.csi_control |=
-			MRFLD_PCI_CSI_CONTROL_CSI_READY;
-	pci_read_config_dword(dev, MRFLD_PCI_CSI_AFE_RCOMP_CONTROL,
-			      &isp->saved_regs.csi_afe_rcomp_config);
-	pci_read_config_dword(dev, MRFLD_PCI_CSI_AFE_HS_CONTROL,
-			      &isp->saved_regs.csi_afe_hs_control);
-	pci_read_config_dword(dev, MRFLD_PCI_CSI_DEADLINE_CONTROL,
-			      &isp->saved_regs.csi_deadline_control);
-	return 0;
-}
-
-static int __maybe_unused atomisp_restore_iunit_reg(struct atomisp_device *isp)
-{
-	struct pci_dev *dev = isp->pdev;
-
-	dev_dbg(isp->dev, "%s\n", __func__);
-
-	pci_write_config_word(dev, PCI_COMMAND, isp->saved_regs.pcicmdsts);
-	pci_write_config_dword(dev, PCI_BASE_ADDRESS_0,
-			       isp->saved_regs.ispmmadr);
-	pci_write_config_dword(dev, PCI_MSI_CAPID, isp->saved_regs.msicap);
-	pci_write_config_dword(dev, PCI_MSI_ADDR, isp->saved_regs.msi_addr);
-	pci_write_config_word(dev, PCI_MSI_DATA, isp->saved_regs.msi_data);
-	pci_write_config_byte(dev, PCI_INTERRUPT_LINE, isp->saved_regs.intr);
-	pci_write_config_dword(dev, PCI_INTERRUPT_CTRL,
-			       isp->saved_regs.interrupt_control);
-	pci_write_config_dword(dev, PCI_I_CONTROL,
-					isp->saved_regs.i_control);
-
-	pci_write_config_dword(dev, MRFLD_PCI_PMCS,
-					isp->saved_regs.pmcs);
-	pci_write_config_dword(dev, MRFLD_PCI_CSI_ACCESS_CTRL_VIOL,
-			      isp->saved_regs.csi_access_viol);
-	pci_write_config_dword(dev, MRFLD_PCI_CSI_RCOMP_CONTROL,
-			      isp->saved_regs.csi_rcomp_config);
-	pci_write_config_dword(dev, MRFLD_PCI_CSI_AFE_TRIM_CONTROL,
-			      isp->saved_regs.csi_afe_dly);
-	pci_write_config_dword(dev, MRFLD_PCI_CSI_CONTROL,
-			      isp->saved_regs.csi_control);
-	pci_write_config_dword(dev, MRFLD_PCI_CSI_AFE_RCOMP_CONTROL,
-			      isp->saved_regs.csi_afe_rcomp_config);
-	pci_write_config_dword(dev, MRFLD_PCI_CSI_AFE_HS_CONTROL,
-			      isp->saved_regs.csi_afe_hs_control);
-	pci_write_config_dword(dev, MRFLD_PCI_CSI_DEADLINE_CONTROL,
-			      isp->saved_regs.csi_deadline_control);
-
-	/*
-	 * for MRFLD, Software/firmware needs to write a 1 to bit0
-	 * of the register at CSI_RECEIVER_SELECTION_REG to enable
-	 * SH CSI backend write 0 will enable Arasan CSI backend,
-	 * which has bugs(like sighting:4567697 and 4567699) and
-	 * will be removed in B0
-	 */
-	atomisp_store_uint32(MRFLD_CSI_RECEIVER_SELECTION_REG, 1);
-	return 0;
-}
-
-static int atomisp_mrfld_pre_power_down(struct atomisp_device *isp)
-{
-	struct pci_dev *dev = isp->pdev;
-	u32 irq;
-	unsigned long flags;
-
-	spin_lock_irqsave(&isp->lock, flags);
-	if (isp->sw_contex.power_state == ATOM_ISP_POWER_DOWN) {
-		spin_unlock_irqrestore(&isp->lock, flags);
-		dev_dbg(isp->dev, "<%s %d.\n", __func__, __LINE__);
-		return 0;
-	}
-	/*
-	 * MRFLD HAS requirement: cannot power off i-unit if
-	 * ISP has IRQ not serviced.
-	 * So, here we need to check if there is any pending
-	 * IRQ, if so, waiting for it to be served
-	 */
-	pci_read_config_dword(dev, PCI_INTERRUPT_CTRL, &irq);
-	irq = irq & 1 << INTR_IIR;
-	pci_write_config_dword(dev, PCI_INTERRUPT_CTRL, irq);
-
-	pci_read_config_dword(dev, PCI_INTERRUPT_CTRL, &irq);
-	if (!(irq & (1 << INTR_IIR)))
-		goto done;
-
-	atomisp_store_uint32(MRFLD_INTR_CLEAR_REG, 0xFFFFFFFF);
-	atomisp_load_uint32(MRFLD_INTR_STATUS_REG, &irq);
-	if (irq != 0) {
-		dev_err(isp->dev,
-			 "%s: fail to clear isp interrupt status reg=0x%x\n",
-			 __func__, irq);
-		spin_unlock_irqrestore(&isp->lock, flags);
-		return -EAGAIN;
-	} else {
-		pci_read_config_dword(dev, PCI_INTERRUPT_CTRL, &irq);
-		irq = irq & 1 << INTR_IIR;
-		pci_write_config_dword(dev, PCI_INTERRUPT_CTRL, irq);
-
-		pci_read_config_dword(dev, PCI_INTERRUPT_CTRL, &irq);
-		if (!(irq & (1 << INTR_IIR))) {
-			atomisp_store_uint32(MRFLD_INTR_ENABLE_REG, 0x0);
-			goto done;
-		}
-		dev_err(isp->dev,
-			 "%s: error in iunit interrupt. status reg=0x%x\n",
-			 __func__, irq);
-		spin_unlock_irqrestore(&isp->lock, flags);
-		return -EAGAIN;
-	}
-done:
-	/*
-	* MRFLD WORKAROUND:
-	* before powering off IUNIT, clear the pending interrupts
-	* and disable the interrupt. driver should avoid writing 0
-	* to IIR. It could block subsequent interrupt messages.
-	* HW sighting:4568410.
-	*/
-	pci_read_config_dword(dev, PCI_INTERRUPT_CTRL, &irq);
-	irq &= ~(1 << INTR_IER);
-	pci_write_config_dword(dev, PCI_INTERRUPT_CTRL, irq);
-
-	atomisp_msi_irq_uninit(isp, dev);
-	atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_LOW, true);
-	spin_unlock_irqrestore(&isp->lock, flags);
-
-	return 0;
-}
-
-
- /*
- * WA for DDR DVFS enable/disable
- * By default, ISP will force DDR DVFS 1600MHz before disable DVFS
- */
-static void punit_ddr_dvfs_enable(bool enable)
-{
-	int door_bell = 1 << 8;
-	int max_wait = 30;
-	int reg;
-
-	iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, MRFLD_ISPSSDVFS, &reg);
-	if (enable) {
-		reg &= ~(MRFLD_BIT0 | MRFLD_BIT1);
-	} else {
-		reg |= (MRFLD_BIT1 | door_bell);
-		reg &= ~(MRFLD_BIT0);
-	}
-	iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_REG_WRITE, MRFLD_ISPSSDVFS, reg);
-
-	/* Check Req_ACK to see freq status, wait until door_bell is cleared */
-	while ((reg & door_bell) && max_wait--) {
-		iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, MRFLD_ISPSSDVFS, &reg);
-		usleep_range(100, 500);
-	}
-
-	if (max_wait == -1)
-		pr_info("DDR DVFS, door bell is not cleared within 3ms\n");
-}
-
-/* Workaround for pmu_nc_set_power_state not ready in MRFLD */
-int atomisp_mrfld_power_down(struct atomisp_device *isp)
-{
-	unsigned long timeout;
-	u32 reg_value;
-
-	/* writing 0x3 to ISPSSPM0 bit[1:0] to power off the IUNIT */
-	iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, MRFLD_ISPSSPM0, &reg_value);
-	reg_value &= ~MRFLD_ISPSSPM0_ISPSSC_MASK;
-	reg_value |= MRFLD_ISPSSPM0_IUNIT_POWER_OFF;
-	iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_REG_WRITE, MRFLD_ISPSSPM0, reg_value);
-
-	/*WA:Enable DVFS*/
-	if (IS_CHT)
-		punit_ddr_dvfs_enable(true);
-
-	/*
-	 * There should be no iunit access while power-down is
-	 * in progress HW sighting: 4567865
-	 * FIXME: msecs_to_jiffies(50)- experienced value
-	 */
-	timeout = jiffies + msecs_to_jiffies(50);
-	while (1) {
-		iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, MRFLD_ISPSSPM0, &reg_value);
-		dev_dbg(isp->dev, "power-off in progress, ISPSSPM0: 0x%x\n",
-				reg_value);
-		/* wait until ISPSSPM0 bit[25:24] shows 0x3 */
-		if ((reg_value >> MRFLD_ISPSSPM0_ISPSSS_OFFSET) ==
-			MRFLD_ISPSSPM0_IUNIT_POWER_OFF) {
-			trace_ipu_cstate(0);
-			return 0;
-		}
-
-		if (time_after(jiffies, timeout)) {
-			dev_err(isp->dev, "power-off iunit timeout.\n");
-			return -EBUSY;
-		}
-		/* FIXME: experienced value for delay */
-		usleep_range(100, 150);
-	}
-}
-
-
-/* Workaround for pmu_nc_set_power_state not ready in MRFLD */
-int atomisp_mrfld_power_up(struct atomisp_device *isp)
-{
-	unsigned long timeout;
-	u32 reg_value;
-
-	/*WA for PUNIT, if DVFS enabled, ISP timeout observed*/
-	if (IS_CHT)
-		punit_ddr_dvfs_enable(false);
-
-	/*
-	 * FIXME:WA for ECS28A, with this sleep, CTS
-	 * android.hardware.camera2.cts.CameraDeviceTest#testCameraDeviceAbort
-	 * PASS, no impact on other platforms
-	*/
-	if (IS_BYT)
-		msleep(10);
-
-	/* writing 0x0 to ISPSSPM0 bit[1:0] to power off the IUNIT */
-	iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, MRFLD_ISPSSPM0, &reg_value);
-	reg_value &= ~MRFLD_ISPSSPM0_ISPSSC_MASK;
-	iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_REG_WRITE, MRFLD_ISPSSPM0, reg_value);
-
-	/* FIXME: experienced value for delay */
-	timeout = jiffies + msecs_to_jiffies(50);
-	while (1) {
-		iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, MRFLD_ISPSSPM0, &reg_value);
-		dev_dbg(isp->dev, "power-on in progress, ISPSSPM0: 0x%x\n",
-				reg_value);
-		/* wait until ISPSSPM0 bit[25:24] shows 0x0 */
-		if ((reg_value >> MRFLD_ISPSSPM0_ISPSSS_OFFSET) ==
-			MRFLD_ISPSSPM0_IUNIT_POWER_ON) {
-			trace_ipu_cstate(1);
-			return 0;
-		}
-
-		if (time_after(jiffies, timeout)) {
-			dev_err(isp->dev, "power-on iunit timeout.\n");
-			return -EBUSY;
-		}
-		/* FIXME: experienced value for delay */
-		usleep_range(100, 150);
-	}
-}
-
-int atomisp_runtime_suspend(struct device *dev)
-{
-	struct atomisp_device *isp = (struct atomisp_device *)
-		dev_get_drvdata(dev);
-	int ret;
-
-	ret = atomisp_mrfld_pre_power_down(isp);
-	if (ret)
-		return ret;
-
-	/*Turn off the ISP d-phy*/
-	ret = atomisp_ospm_dphy_down(isp);
-	if (ret)
-		return ret;
-	pm_qos_update_request(&isp->pm_qos, PM_QOS_DEFAULT_VALUE);
-	return atomisp_mrfld_power_down(isp);
-}
-
-int atomisp_runtime_resume(struct device *dev)
-{
-	struct atomisp_device *isp = (struct atomisp_device *)
-		dev_get_drvdata(dev);
-	int ret;
-
-	ret = atomisp_mrfld_power_up(isp);
-	if (ret)
-			return ret;
-
-	pm_qos_update_request(&isp->pm_qos, isp->max_isr_latency);
-	if (isp->sw_contex.power_state == ATOM_ISP_POWER_DOWN) {
-		/*Turn on ISP d-phy */
-		ret = atomisp_ospm_dphy_up(isp);
-		if (ret) {
-			dev_err(isp->dev, "Failed to power up ISP!.\n");
-			return -EINVAL;
-		}
-	}
-
-	/*restore register values for iUnit and iUnitPHY registers*/
-	if (isp->saved_regs.pcicmdsts)
-		atomisp_restore_iunit_reg(isp);
-
-	atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_LOW, true);
-	return 0;
-}
-
-static int __maybe_unused atomisp_suspend(struct device *dev)
-{
-	struct atomisp_device *isp = (struct atomisp_device *)
-		dev_get_drvdata(dev);
-	/* FIXME: only has one isp_subdev at present */
-	struct atomisp_sub_device *asd = &isp->asd[0];
-	unsigned long flags;
-	int ret;
-
-	/*
-	 * FIXME: Suspend is not supported by sensors. Abort if any video
-	 * node was opened.
-	 */
-	if (atomisp_dev_users(isp))
-		return -EBUSY;
-
-	spin_lock_irqsave(&isp->lock, flags);
-	if (asd->streaming != ATOMISP_DEVICE_STREAMING_DISABLED) {
-		spin_unlock_irqrestore(&isp->lock, flags);
-		dev_err(isp->dev, "atomisp cannot suspend at this time.\n");
-		return -EINVAL;
-	}
-	spin_unlock_irqrestore(&isp->lock, flags);
-
-	ret = atomisp_mrfld_pre_power_down(isp);
-	if (ret)
-		return ret;
-
-	/*Turn off the ISP d-phy */
-	ret = atomisp_ospm_dphy_down(isp);
-	if (ret) {
-		dev_err(isp->dev, "fail to power off ISP\n");
-		return ret;
-	}
-	pm_qos_update_request(&isp->pm_qos, PM_QOS_DEFAULT_VALUE);
-	return atomisp_mrfld_power_down(isp);
-}
-
-static int __maybe_unused atomisp_resume(struct device *dev)
-{
-	struct atomisp_device *isp = (struct atomisp_device *)
-		dev_get_drvdata(dev);
-	int ret;
-
-	ret = atomisp_mrfld_power_up(isp);
-	if (ret)
-		return ret;
-
-	pm_qos_update_request(&isp->pm_qos, isp->max_isr_latency);
-
-	/*Turn on ISP d-phy */
-	ret = atomisp_ospm_dphy_up(isp);
-	if (ret) {
-		dev_err(isp->dev, "Failed to power up ISP!.\n");
-		return -EINVAL;
-	}
-
-	/*restore register values for iUnit and iUnitPHY registers*/
-	if (isp->saved_regs.pcicmdsts)
-		atomisp_restore_iunit_reg(isp);
-
-	atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_LOW, true);
-	return 0;
-}
-
-int atomisp_csi_lane_config(struct atomisp_device *isp)
-{
-	static const struct {
-		u8 code;
-		u8 lanes[MRFLD_PORT_NUM];
-	} portconfigs[] = {
-		/* Tangier/Merrifield available lane configurations */
-		{ 0x00, { 4, 1, 0 } },		/* 00000 */
-		{ 0x01, { 3, 1, 0 } },		/* 00001 */
-		{ 0x02, { 2, 1, 0 } },		/* 00010 */
-		{ 0x03, { 1, 1, 0 } },		/* 00011 */
-		{ 0x04, { 2, 1, 2 } },		/* 00100 */
-		{ 0x08, { 3, 1, 1 } },		/* 01000 */
-		{ 0x09, { 2, 1, 1 } },		/* 01001 */
-		{ 0x0a, { 1, 1, 1 } },		/* 01010 */
-
-		/* Anniedale/Moorefield only configurations */
-		{ 0x10, { 4, 2, 0 } },		/* 10000 */
-		{ 0x11, { 3, 2, 0 } },		/* 10001 */
-		{ 0x12, { 2, 2, 0 } },		/* 10010 */
-		{ 0x13, { 1, 2, 0 } },		/* 10011 */
-		{ 0x14, { 2, 2, 2 } },		/* 10100 */
-		{ 0x18, { 3, 2, 1 } },		/* 11000 */
-		{ 0x19, { 2, 2, 1 } },		/* 11001 */
-		{ 0x1a, { 1, 2, 1 } },		/* 11010 */
-	};
-
-	unsigned int i, j;
-	u8 sensor_lanes[MRFLD_PORT_NUM] = { 0 };
-	u32 csi_control;
-	int nportconfigs;
-	u32 port_config_mask;
-	int port3_lanes_shift;
-
-	if (isp->media_dev.hw_revision <
-		ATOMISP_HW_REVISION_ISP2401_LEGACY <<
-		ATOMISP_HW_REVISION_SHIFT) {
-		/* Merrifield */
-		port_config_mask = MRFLD_PORT_CONFIG_MASK;
-		port3_lanes_shift = MRFLD_PORT3_LANES_SHIFT;
-	} else {
-		/* Moorefield / Cherryview */
-		port_config_mask = CHV_PORT_CONFIG_MASK;
-		port3_lanes_shift = CHV_PORT3_LANES_SHIFT;
-	}
-
-	if (isp->media_dev.hw_revision <
-		ATOMISP_HW_REVISION_ISP2401 <<
-		ATOMISP_HW_REVISION_SHIFT) {
-		/* Merrifield / Moorefield legacy input system */
-		nportconfigs = MRFLD_PORT_CONFIG_NUM;
-	} else {
-		/* Moorefield / Cherryview new input system */
-		nportconfigs = ARRAY_SIZE(portconfigs);
-	}
-
-	for (i = 0; i < isp->input_cnt; i++) {
-		struct camera_mipi_info *mipi_info;
-
-		if (isp->inputs[i].type != RAW_CAMERA &&
-		    isp->inputs[i].type != SOC_CAMERA)
-			continue;
-
-		mipi_info = atomisp_to_sensor_mipi_info(isp->inputs[i].camera);
-		if (!mipi_info)
-			continue;
-
-		switch (mipi_info->port) {
-		case ATOMISP_CAMERA_PORT_PRIMARY:
-			sensor_lanes[0] = mipi_info->num_lanes;
-			break;
-		case ATOMISP_CAMERA_PORT_SECONDARY:
-			sensor_lanes[1] = mipi_info->num_lanes;
-			break;
-		case ATOMISP_CAMERA_PORT_TERTIARY:
-			sensor_lanes[2] = mipi_info->num_lanes;
-			break;
-		default:
-			dev_err(isp->dev,
-				"%s: invalid port: %d for the %dth sensor\n",
-				__func__, mipi_info->port, i);
-			return -EINVAL;
-		}
-	}
-
-	for (i = 0; i < nportconfigs; i++) {
-		for (j = 0; j < MRFLD_PORT_NUM; j++)
-			if (sensor_lanes[j] &&
-			    sensor_lanes[j] != portconfigs[i].lanes[j])
-				break;
-
-		if (j == MRFLD_PORT_NUM)
-			break;			/* Found matching setting */
-	}
-
-	if (i >= nportconfigs) {
-		dev_err(isp->dev,
-			"%s: could not find the CSI port setting for %d-%d-%d\n",
-			__func__,
-			sensor_lanes[0], sensor_lanes[1], sensor_lanes[2]);
-		return -EINVAL;
-	}
-
-	pci_read_config_dword(isp->pdev, MRFLD_PCI_CSI_CONTROL, &csi_control);
-	csi_control &= ~port_config_mask;
-	csi_control |= (portconfigs[i].code << MRFLD_PORT_CONFIGCODE_SHIFT)
-		| (portconfigs[i].lanes[0] ? 0 : (1 << MRFLD_PORT1_ENABLE_SHIFT))
-		| (portconfigs[i].lanes[1] ? 0 : (1 << MRFLD_PORT2_ENABLE_SHIFT))
-		| (portconfigs[i].lanes[2] ? 0 : (1 << MRFLD_PORT3_ENABLE_SHIFT))
-		| (((1 << portconfigs[i].lanes[0]) - 1) << MRFLD_PORT1_LANES_SHIFT)
-		| (((1 << portconfigs[i].lanes[1]) - 1) << MRFLD_PORT2_LANES_SHIFT)
-		| (((1 << portconfigs[i].lanes[2]) - 1) << port3_lanes_shift);
-
-	pci_write_config_dword(isp->pdev, MRFLD_PCI_CSI_CONTROL, csi_control);
-
-	dev_dbg(isp->dev,
-		"%s: the portconfig is %d-%d-%d, CSI_CONTROL is 0x%08X\n",
-		__func__, portconfigs[i].lanes[0], portconfigs[i].lanes[1],
-		portconfigs[i].lanes[2], csi_control);
-
-	return 0;
-}
-
-static int atomisp_subdev_probe(struct atomisp_device *isp)
-{
-	const struct atomisp_platform_data *pdata;
-	struct intel_v4l2_subdev_table *subdevs;
-	int ret, raw_index = -1;
-
-	pdata = atomisp_get_platform_data();
-	if (pdata == NULL) {
-		dev_err(isp->dev, "no platform data available\n");
-		return 0;
-	}
-
-	for (subdevs = pdata->subdevs; subdevs->type; ++subdevs) {
-		struct v4l2_subdev *subdev;
-		struct i2c_board_info *board_info =
-			&subdevs->v4l2_subdev.board_info;
-		struct i2c_adapter *adapter =
-			i2c_get_adapter(subdevs->v4l2_subdev.i2c_adapter_id);
-		int sensor_num, i;
-
-		if (adapter == NULL) {
-			dev_err(isp->dev,
-				"Failed to find i2c adapter for subdev %s\n",
-				board_info->type);
-			break;
-		}
-
-		/* In G-Min, the sensor devices will already be probed
-		 * (via ACPI) and registered, do not create new
-		 * ones */
-		subdev = atomisp_gmin_find_subdev(adapter, board_info);
-		ret = v4l2_device_register_subdev(&isp->v4l2_dev, subdev);
-		if (ret) {
-			dev_warn(isp->dev, "Subdev %s detection fail\n",
-				 board_info->type);
-			continue;
-		}
-
-		if (subdev == NULL) {
-			dev_warn(isp->dev, "Subdev %s detection fail\n",
-				 board_info->type);
-			continue;
-		}
-
-		dev_info(isp->dev, "Subdev %s successfully register\n",
-			 board_info->type);
-
-		switch (subdevs->type) {
-		case RAW_CAMERA:
-			raw_index = isp->input_cnt;
-			dev_dbg(isp->dev, "raw_index: %d\n", raw_index);
-		case SOC_CAMERA:
-			dev_dbg(isp->dev, "SOC_INDEX: %d\n", isp->input_cnt);
-			if (isp->input_cnt >= ATOM_ISP_MAX_INPUTS) {
-				dev_warn(isp->dev,
-					 "too many atomisp inputs, ignored\n");
-				break;
-			}
-
-			isp->inputs[isp->input_cnt].type = subdevs->type;
-			isp->inputs[isp->input_cnt].port = subdevs->port;
-			isp->inputs[isp->input_cnt].camera = subdev;
-			isp->inputs[isp->input_cnt].sensor_index = 0;
-			/*
-			 * initialize the subdev frame size, then next we can
-			 * judge whether frame_size store effective value via
-			 * pixel_format.
-			 */
-			isp->inputs[isp->input_cnt].frame_size.pixel_format = 0;
-			isp->inputs[isp->input_cnt].camera_caps =
-					atomisp_get_default_camera_caps();
-			sensor_num = isp->inputs[isp->input_cnt]
-				.camera_caps->sensor_num;
-			isp->input_cnt++;
-			for (i = 1; i < sensor_num; i++) {
-				if (isp->input_cnt >= ATOM_ISP_MAX_INPUTS) {
-					dev_warn(isp->dev,
-						"atomisp inputs out of range\n");
-					break;
-				}
-				isp->inputs[isp->input_cnt] =
-					isp->inputs[isp->input_cnt - 1];
-				isp->inputs[isp->input_cnt].sensor_index = i;
-				isp->input_cnt++;
-			}
-			break;
-		case CAMERA_MOTOR:
-			isp->motor = subdev;
-			break;
-		case LED_FLASH:
-		case XENON_FLASH:
-			isp->flash = subdev;
-			break;
-		default:
-			dev_dbg(isp->dev, "unknown subdev probed\n");
-			break;
-		}
-
-	}
-
-	/*
-	 * HACK: Currently VCM belongs to primary sensor only, but correct
-	 * approach must be to acquire from platform code which sensor
-	 * owns it.
-	 */
-	if (isp->motor && raw_index >= 0)
-		isp->inputs[raw_index].motor = isp->motor;
-
-	/* Proceed even if no modules detected. For COS mode and no modules. */
-	if (!isp->inputs[0].camera)
-		dev_warn(isp->dev, "no camera attached or fail to detect\n");
-
-	return atomisp_csi_lane_config(isp);
-}
-
-static void atomisp_unregister_entities(struct atomisp_device *isp)
-{
-	unsigned int i;
-	struct v4l2_subdev *sd, *next;
-
-	for (i = 0; i < isp->num_of_streams; i++)
-		atomisp_subdev_unregister_entities(&isp->asd[i]);
-	atomisp_tpg_unregister_entities(&isp->tpg);
-	atomisp_file_input_unregister_entities(&isp->file_dev);
-	for (i = 0; i < ATOMISP_CAMERA_NR_PORTS; i++)
-		atomisp_mipi_csi2_unregister_entities(&isp->csi2_port[i]);
-
-	list_for_each_entry_safe(sd, next, &isp->v4l2_dev.subdevs, list)
-		v4l2_device_unregister_subdev(sd);
-
-	v4l2_device_unregister(&isp->v4l2_dev);
-	media_device_unregister(&isp->media_dev);
-}
-
-static int atomisp_register_entities(struct atomisp_device *isp)
-{
-	int ret = 0;
-	unsigned int i;
-
-	isp->media_dev.dev = isp->dev;
-
-	strlcpy(isp->media_dev.model, "Intel Atom ISP",
-		sizeof(isp->media_dev.model));
-
-	media_device_init(&isp->media_dev);
-	isp->v4l2_dev.mdev = &isp->media_dev;
-	ret = v4l2_device_register(isp->dev, &isp->v4l2_dev);
-	if (ret < 0) {
-		dev_err(isp->dev, "%s: V4L2 device registration failed (%d)\n",
-			__func__, ret);
-		goto v4l2_device_failed;
-	}
-
-	ret = atomisp_subdev_probe(isp);
-	if (ret < 0)
-		goto csi_and_subdev_probe_failed;
-
-	/* Register internal entities */
-	for (i = 0; i < ATOMISP_CAMERA_NR_PORTS; i++) {
-		ret = atomisp_mipi_csi2_register_entities(&isp->csi2_port[i],
-								&isp->v4l2_dev);
-		if (ret == 0)
-			continue;
-
-		/* error case */
-		dev_err(isp->dev, "failed to register the CSI port: %d\n", i);
-		/* deregister all registered CSI ports */
-		while (i--)
-			atomisp_mipi_csi2_unregister_entities(
-							&isp->csi2_port[i]);
-
-		goto csi_and_subdev_probe_failed;
-	}
-
-	ret =
-	atomisp_file_input_register_entities(&isp->file_dev, &isp->v4l2_dev);
-	if (ret < 0) {
-		dev_err(isp->dev, "atomisp_file_input_register_entities\n");
-		goto file_input_register_failed;
-	}
-
-	ret = atomisp_tpg_register_entities(&isp->tpg, &isp->v4l2_dev);
-	if (ret < 0) {
-		dev_err(isp->dev, "atomisp_tpg_register_entities\n");
-		goto tpg_register_failed;
-	}
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-
-		ret = atomisp_subdev_register_entities(asd, &isp->v4l2_dev);
-		if (ret < 0) {
-			dev_err(isp->dev,
-				"atomisp_subdev_register_entities fail\n");
-			for (; i > 0; i--)
-				atomisp_subdev_unregister_entities(
-						&isp->asd[i - 1]);
-			goto subdev_register_failed;
-		}
-	}
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-
-		init_completion(&asd->init_done);
-
-		asd->delayed_init_workq =
-			alloc_workqueue(isp->v4l2_dev.name, WQ_CPU_INTENSIVE,
-					1);
-		if (asd->delayed_init_workq == NULL) {
-			dev_err(isp->dev,
-					"Failed to initialize delayed init workq\n");
-			ret = -ENOMEM;
-
-			for (; i > 0; i--)
-				destroy_workqueue(isp->asd[i - 1].
-						delayed_init_workq);
-			goto wq_alloc_failed;
-		}
-		INIT_WORK(&asd->delayed_init_work, atomisp_delayed_init_work);
-	}
-
-	for (i = 0; i < isp->input_cnt; i++) {
-		if (isp->inputs[i].port >= ATOMISP_CAMERA_NR_PORTS) {
-			dev_err(isp->dev, "isp->inputs port %d not supported\n",
-					isp->inputs[i].port);
-			ret = -EINVAL;
-			goto link_failed;
-		}
-	}
-
-	dev_dbg(isp->dev,
-		"FILE_INPUT enable, camera_cnt: %d\n", isp->input_cnt);
-	isp->inputs[isp->input_cnt].type = FILE_INPUT;
-	isp->inputs[isp->input_cnt].port = -1;
-	isp->inputs[isp->input_cnt].camera_caps =
-		    atomisp_get_default_camera_caps();
-	isp->inputs[isp->input_cnt++].camera = &isp->file_dev.sd;
-
-	if (isp->input_cnt < ATOM_ISP_MAX_INPUTS) {
-		dev_dbg(isp->dev,
-			"TPG detected, camera_cnt: %d\n", isp->input_cnt);
-		isp->inputs[isp->input_cnt].type = TEST_PATTERN;
-		isp->inputs[isp->input_cnt].port = -1;
-		isp->inputs[isp->input_cnt].camera_caps =
-		    atomisp_get_default_camera_caps();
-		isp->inputs[isp->input_cnt++].camera = &isp->tpg.sd;
-	} else {
-		dev_warn(isp->dev, "too many atomisp inputs, TPG ignored.\n");
-	}
-
-	ret = v4l2_device_register_subdev_nodes(&isp->v4l2_dev);
-	if (ret < 0)
-		goto link_failed;
-
-	return media_device_register(&isp->media_dev);
-
-link_failed:
-	for (i = 0; i < isp->num_of_streams; i++)
-		destroy_workqueue(isp->asd[i].
-				delayed_init_workq);
-wq_alloc_failed:
-	for (i = 0; i < isp->num_of_streams; i++)
-		atomisp_subdev_unregister_entities(
-					&isp->asd[i]);
-subdev_register_failed:
-	atomisp_tpg_unregister_entities(&isp->tpg);
-tpg_register_failed:
-	atomisp_file_input_unregister_entities(&isp->file_dev);
-file_input_register_failed:
-	for (i = 0; i < ATOMISP_CAMERA_NR_PORTS; i++)
-		atomisp_mipi_csi2_unregister_entities(&isp->csi2_port[i]);
-csi_and_subdev_probe_failed:
-	v4l2_device_unregister(&isp->v4l2_dev);
-v4l2_device_failed:
-	media_device_unregister(&isp->media_dev);
-	media_device_cleanup(&isp->media_dev);
-	return ret;
-}
-
-static int atomisp_initialize_modules(struct atomisp_device *isp)
-{
-	int ret;
-
-	ret = atomisp_mipi_csi2_init(isp);
-	if (ret < 0) {
-		dev_err(isp->dev, "mipi csi2 initialization failed\n");
-		goto error_mipi_csi2;
-	}
-
-	ret = atomisp_file_input_init(isp);
-	if (ret < 0) {
-		dev_err(isp->dev,
-			"file input device initialization failed\n");
-		goto error_file_input;
-	}
-
-	ret = atomisp_tpg_init(isp);
-	if (ret < 0) {
-		dev_err(isp->dev, "tpg initialization failed\n");
-		goto error_tpg;
-	}
-
-	ret = atomisp_subdev_init(isp);
-	if (ret < 0) {
-		dev_err(isp->dev, "ISP subdev initialization failed\n");
-		goto error_isp_subdev;
-	}
-
-
-	return 0;
-
-error_isp_subdev:
-error_tpg:
-	atomisp_tpg_cleanup(isp);
-error_file_input:
-	atomisp_file_input_cleanup(isp);
-error_mipi_csi2:
-	atomisp_mipi_csi2_cleanup(isp);
-	return ret;
-}
-
-static void atomisp_uninitialize_modules(struct atomisp_device *isp)
-{
-	atomisp_tpg_cleanup(isp);
-	atomisp_file_input_cleanup(isp);
-	atomisp_mipi_csi2_cleanup(isp);
-}
-
-const struct firmware *
-atomisp_load_firmware(struct atomisp_device *isp)
-{
-	const struct firmware *fw;
-	int rc;
-	char *fw_path = NULL;
-
-	if (skip_fwload)
-		return NULL;
-
-	if (isp->media_dev.hw_revision ==
-	    ((ATOMISP_HW_REVISION_ISP2401 << ATOMISP_HW_REVISION_SHIFT)
-	     | ATOMISP_HW_STEPPING_A0))
-		fw_path = "shisp_2401a0_v21.bin";
-
-	if (isp->media_dev.hw_revision ==
-	    ((ATOMISP_HW_REVISION_ISP2401_LEGACY << ATOMISP_HW_REVISION_SHIFT)
-	     | ATOMISP_HW_STEPPING_A0))
-		fw_path = "shisp_2401a0_legacy_v21.bin";
-
-	if (isp->media_dev.hw_revision ==
-	    ((ATOMISP_HW_REVISION_ISP2400 << ATOMISP_HW_REVISION_SHIFT)
-	     | ATOMISP_HW_STEPPING_B0))
-		fw_path = "shisp_2400b0_v21.bin";
-
-	if (!fw_path) {
-		dev_err(isp->dev, "Unsupported hw_revision 0x%x\n",
-			isp->media_dev.hw_revision);
-		return NULL;
-	}
-
-	rc = request_firmware(&fw, fw_path, isp->dev);
-	if (rc) {
-		dev_err(isp->dev,
-			"atomisp: Error %d while requesting firmware %s\n",
-			rc, fw_path);
-		return NULL;
-	}
-
-	return fw;
-}
-
-/*
- * Check for flags the driver was compiled with against the PCI
- * device. Always returns true on other than ISP 2400.
- */
-static bool is_valid_device(struct pci_dev *dev,
-			    const struct pci_device_id *id)
-{
-	unsigned int a0_max_id;
-
-	switch (id->device & ATOMISP_PCI_DEVICE_SOC_MASK) {
-	case ATOMISP_PCI_DEVICE_SOC_MRFLD:
-		a0_max_id = ATOMISP_PCI_REV_MRFLD_A0_MAX;
-		break;
-	case ATOMISP_PCI_DEVICE_SOC_BYT:
-		a0_max_id = ATOMISP_PCI_REV_BYT_A0_MAX;
-		break;
-	default:
-		return true;
-	}
-
-	return dev->revision > a0_max_id;
-}
-
-static int init_atomisp_wdts(struct atomisp_device *isp)
-{
-	int i, err;
-
-	atomic_set(&isp->wdt_work_queued, 0);
-	isp->wdt_work_queue = alloc_workqueue(isp->v4l2_dev.name, 0, 1);
-	if (isp->wdt_work_queue == NULL) {
-		dev_err(isp->dev, "Failed to initialize wdt work queue\n");
-		err = -ENOMEM;
-		goto alloc_fail;
-	}
-	INIT_WORK(&isp->wdt_work, atomisp_wdt_work);
-
-	for (i = 0; i < isp->num_of_streams; i++) {
-		struct atomisp_sub_device *asd = &isp->asd[i];
-#ifndef ISP2401
-		timer_setup(&asd->wdt, atomisp_wdt, 0);
-#else
-		timer_setup(&asd->video_out_capture.wdt, atomisp_wdt, 0);
-		timer_setup(&asd->video_out_preview.wdt, atomisp_wdt, 0);
-		timer_setup(&asd->video_out_vf.wdt, atomisp_wdt, 0);
-		timer_setup(&asd->video_out_video_capture.wdt, atomisp_wdt, 0);
-#endif
-	}
-	return 0;
-alloc_fail:
-	return err;
-}
-
-#define ATOM_ISP_PCI_BAR	0
-
-static int atomisp_pci_probe(struct pci_dev *dev,
-				       const struct pci_device_id *id)
-{
-	const struct atomisp_platform_data *pdata;
-	struct atomisp_device *isp;
-	unsigned int start;
-	void __iomem *base;
-	int err, val;
-	u32 irq;
-
-	if (!dev) {
-		dev_err(&dev->dev, "atomisp: error device ptr\n");
-		return -EINVAL;
-	}
-
-	if (!is_valid_device(dev, id))
-		return -ENODEV;
-	/* Pointer to struct device. */
-	atomisp_dev = &dev->dev;
-
-	pdata = atomisp_get_platform_data();
-	if (pdata == NULL)
-		dev_warn(&dev->dev, "no platform data available\n");
-
-	err = pcim_enable_device(dev);
-	if (err) {
-		dev_err(&dev->dev, "Failed to enable CI ISP device (%d)\n",
-			err);
-		return err;
-	}
-
-	start = pci_resource_start(dev, ATOM_ISP_PCI_BAR);
-	dev_dbg(&dev->dev, "start: 0x%x\n", start);
-
-	err = pcim_iomap_regions(dev, 1 << ATOM_ISP_PCI_BAR, pci_name(dev));
-	if (err) {
-		dev_err(&dev->dev, "Failed to I/O memory remapping (%d)\n",
-			err);
-		return err;
-	}
-
-	base = pcim_iomap_table(dev)[ATOM_ISP_PCI_BAR];
-	dev_dbg(&dev->dev, "base: %p\n", base);
-
-	atomisp_io_base = base;
-
-	dev_dbg(&dev->dev, "atomisp_io_base: %p\n", atomisp_io_base);
-
-	isp = devm_kzalloc(&dev->dev, sizeof(struct atomisp_device), GFP_KERNEL);
-	if (!isp) {
-		dev_err(&dev->dev, "Failed to alloc CI ISP structure\n");
-		return -ENOMEM;
-	}
-	isp->pdev = dev;
-	isp->dev = &dev->dev;
-	isp->sw_contex.power_state = ATOM_ISP_POWER_UP;
-	isp->saved_regs.ispmmadr = start;
-
-	rt_mutex_init(&isp->mutex);
-	mutex_init(&isp->streamoff_mutex);
-	spin_lock_init(&isp->lock);
-
-	/* This is not a true PCI device on SoC, so the delay is not needed. */
-	isp->pdev->d3_delay = 0;
-
-	switch (id->device & ATOMISP_PCI_DEVICE_SOC_MASK) {
-	case ATOMISP_PCI_DEVICE_SOC_MRFLD:
-		isp->media_dev.hw_revision =
-			(ATOMISP_HW_REVISION_ISP2400
-			 << ATOMISP_HW_REVISION_SHIFT) |
-			ATOMISP_HW_STEPPING_B0;
-
-		switch (id->device) {
-		case ATOMISP_PCI_DEVICE_SOC_MRFLD_1179:
-			isp->dfs = &dfs_config_merr_1179;
-			break;
-		case ATOMISP_PCI_DEVICE_SOC_MRFLD_117A:
-			isp->dfs = &dfs_config_merr_117a;
-			break;
-		default:
-			isp->dfs = &dfs_config_merr;
-			break;
-		}
-		isp->hpll_freq = HPLL_FREQ_1600MHZ;
-		break;
-	case ATOMISP_PCI_DEVICE_SOC_BYT:
-		isp->media_dev.hw_revision =
-			(ATOMISP_HW_REVISION_ISP2400
-			 << ATOMISP_HW_REVISION_SHIFT) |
-			ATOMISP_HW_STEPPING_B0;
-#ifdef FIXME			
-		if (INTEL_MID_BOARD(3, TABLET, BYT, BLK, PRO, CRV2) ||
-			INTEL_MID_BOARD(3, TABLET, BYT, BLK, ENG, CRV2)) {
-			isp->dfs = &dfs_config_byt_cr;
-			isp->hpll_freq = HPLL_FREQ_2000MHZ;
-		} else
-#endif		
-		{
-			isp->dfs = &dfs_config_byt;
-			isp->hpll_freq = HPLL_FREQ_1600MHZ;
-		}
-		/* HPLL frequency is known to be device-specific, but we don't
-		 * have specs yet for exactly how it varies.  Default to
-		 * BYT-CR but let provisioning set it via EFI variable */
-		isp->hpll_freq = gmin_get_var_int(&dev->dev, "HpllFreq",
-					HPLL_FREQ_2000MHZ);
-
-		/*
-		 * for BYT/CHT we are put isp into D3cold to avoid pci registers access
-		 * in power off. Set d3cold_delay to 0 since default 100ms is not
-		 * necessary.
-		 */
-		isp->pdev->d3cold_delay = 0;
-		break;
-	case ATOMISP_PCI_DEVICE_SOC_ANN:
-		isp->media_dev.hw_revision = (
-#ifdef ISP2401_NEW_INPUT_SYSTEM
-			 ATOMISP_HW_REVISION_ISP2401
-#else
-			 ATOMISP_HW_REVISION_ISP2401_LEGACY
-#endif
-			 << ATOMISP_HW_REVISION_SHIFT);
-		isp->media_dev.hw_revision |= isp->pdev->revision < 2 ?
-			ATOMISP_HW_STEPPING_A0 : ATOMISP_HW_STEPPING_B0;
-		isp->dfs = &dfs_config_merr;
-		isp->hpll_freq = HPLL_FREQ_1600MHZ;
-		break;
-	case ATOMISP_PCI_DEVICE_SOC_CHT:
-		isp->media_dev.hw_revision = (
-#ifdef ISP2401_NEW_INPUT_SYSTEM
-			 ATOMISP_HW_REVISION_ISP2401
-#else
-			 ATOMISP_HW_REVISION_ISP2401_LEGACY
-#endif
-			<< ATOMISP_HW_REVISION_SHIFT);
-		isp->media_dev.hw_revision |= isp->pdev->revision < 2 ?
-			 ATOMISP_HW_STEPPING_A0 : ATOMISP_HW_STEPPING_B0;
-
-		isp->dfs = &dfs_config_cht;
-		isp->pdev->d3cold_delay = 0;
-
-		iosf_mbi_read(CCK_PORT, MBI_REG_READ, CCK_FUSE_REG_0, &val);
-		switch (val & CCK_FUSE_HPLL_FREQ_MASK) {
-		case 0x00:
-			isp->hpll_freq = HPLL_FREQ_800MHZ;
-			break;
-		case 0x01:
-			isp->hpll_freq = HPLL_FREQ_1600MHZ;
-			break;
-		case 0x02:
-			isp->hpll_freq = HPLL_FREQ_2000MHZ;
-			break;
-		default:
-			isp->hpll_freq = HPLL_FREQ_1600MHZ;
-			dev_warn(isp->dev,
-				 "read HPLL from cck failed.default 1600MHz.\n");
-		}
-		break;
-	default:
-		dev_err(&dev->dev, "un-supported IUNIT device\n");
-		return -ENODEV;
-	}
-
-	dev_info(&dev->dev, "ISP HPLL frequency base = %d MHz\n",
-		 isp->hpll_freq);
-
-	isp->max_isr_latency = ATOMISP_MAX_ISR_LATENCY;
-
-	/* Load isp firmware from user space */
-	if (!defer_fw_load) {
-		isp->firmware = atomisp_load_firmware(isp);
-		if (!isp->firmware) {
-			err = -ENOENT;
-			goto load_fw_fail;
-		}
-
-		err = atomisp_css_check_firmware_version(isp);
-		if (err) {
-			dev_dbg(&dev->dev, "Firmware version check failed\n");
-			goto fw_validation_fail;
-		}
-	}
-
-	pci_set_master(dev);
-	pci_set_drvdata(dev, isp);
-
-	err = pci_enable_msi(dev);
-	if (err) {
-		dev_err(&dev->dev, "Failed to enable msi (%d)\n", err);
-		goto enable_msi_fail;
-	}
-
-	atomisp_msi_irq_init(isp, dev);
-
-	pm_qos_add_request(&isp->pm_qos, PM_QOS_CPU_DMA_LATENCY,
-			   PM_QOS_DEFAULT_VALUE);
-
-	/*
-	 * for MRFLD, Software/firmware needs to write a 1 to bit 0 of
-	 * the register at CSI_RECEIVER_SELECTION_REG to enable SH CSI
-	 * backend write 0 will enable Arasan CSI backend, which has
-	 * bugs(like sighting:4567697 and 4567699) and will be removed
-	 * in B0
-	 */
-	atomisp_store_uint32(MRFLD_CSI_RECEIVER_SELECTION_REG, 1);
-
-	if ((id->device & ATOMISP_PCI_DEVICE_SOC_MASK) ==
-			ATOMISP_PCI_DEVICE_SOC_MRFLD) {
-		u32 csi_afe_trim;
-
-		/*
-		 * Workaround for imbalance data eye issue which is observed
-		 * on TNG B0.
-		 */
-		pci_read_config_dword(dev, MRFLD_PCI_CSI_AFE_TRIM_CONTROL,
-				      &csi_afe_trim);
-		csi_afe_trim &= ~((MRFLD_PCI_CSI_HSRXCLKTRIM_MASK <<
-					MRFLD_PCI_CSI1_HSRXCLKTRIM_SHIFT) |
-				  (MRFLD_PCI_CSI_HSRXCLKTRIM_MASK <<
-					MRFLD_PCI_CSI2_HSRXCLKTRIM_SHIFT) |
-				  (MRFLD_PCI_CSI_HSRXCLKTRIM_MASK <<
-					MRFLD_PCI_CSI3_HSRXCLKTRIM_SHIFT));
-		csi_afe_trim |= (MRFLD_PCI_CSI1_HSRXCLKTRIM <<
-					MRFLD_PCI_CSI1_HSRXCLKTRIM_SHIFT) |
-				(MRFLD_PCI_CSI2_HSRXCLKTRIM <<
-					MRFLD_PCI_CSI2_HSRXCLKTRIM_SHIFT) |
-				(MRFLD_PCI_CSI3_HSRXCLKTRIM <<
-					MRFLD_PCI_CSI3_HSRXCLKTRIM_SHIFT);
-		pci_write_config_dword(dev, MRFLD_PCI_CSI_AFE_TRIM_CONTROL,
-				      csi_afe_trim);
-	}
-
-	err = atomisp_initialize_modules(isp);
-	if (err < 0) {
-		dev_err(&dev->dev, "atomisp_initialize_modules (%d)\n", err);
-		goto initialize_modules_fail;
-	}
-
-	err = atomisp_register_entities(isp);
-	if (err < 0) {
-		dev_err(&dev->dev, "atomisp_register_entities failed (%d)\n",
-			err);
-		goto register_entities_fail;
-	}
-	err = atomisp_create_pads_links(isp);
-	if (err < 0)
-		goto register_entities_fail;
-	/* init atomisp wdts */
-	if (init_atomisp_wdts(isp) != 0)
-		goto wdt_work_queue_fail;
-
-	/* save the iunit context only once after all the values are init'ed. */
-	atomisp_save_iunit_reg(isp);
-
-	pm_runtime_put_noidle(&dev->dev);
-	pm_runtime_allow(&dev->dev);
-
-	hmm_init_mem_stat(repool_pgnr, dypool_enable, dypool_pgnr);
-	err = hmm_pool_register(repool_pgnr, HMM_POOL_TYPE_RESERVED);
-	if (err) {
-		dev_err(&dev->dev, "Failed to register reserved memory pool.\n");
-		goto hmm_pool_fail;
-	}
-
-	/* Init ISP memory management */
-	hmm_init();
-
-	err = devm_request_threaded_irq(&dev->dev, dev->irq,
-					atomisp_isr, atomisp_isr_thread,
-					IRQF_SHARED, "isp_irq", isp);
-	if (err) {
-		dev_err(&dev->dev, "Failed to request irq (%d)\n", err);
-		goto request_irq_fail;
-	}
-
-	/* Load firmware into ISP memory */
-	if (!defer_fw_load) {
-		err = atomisp_css_load_firmware(isp);
-		if (err) {
-			dev_err(&dev->dev, "Failed to init css.\n");
-			goto css_init_fail;
-		}
-	} else {
-		dev_dbg(&dev->dev, "Skip css init.\n");
-	}
-	/* Clear FW image from memory */
-	release_firmware(isp->firmware);
-	isp->firmware = NULL;
-	isp->css_env.isp_css_fw.data = NULL;
-
-	atomisp_drvfs_init(&dev->driver->driver, isp);
-
-	return 0;
-
-css_init_fail:
-	devm_free_irq(&dev->dev, dev->irq, isp);
-request_irq_fail:
-	hmm_cleanup();
-	hmm_pool_unregister(HMM_POOL_TYPE_RESERVED);
-hmm_pool_fail:
-	destroy_workqueue(isp->wdt_work_queue);
-wdt_work_queue_fail:
-	atomisp_acc_cleanup(isp);
-	atomisp_unregister_entities(isp);
-register_entities_fail:
-	atomisp_uninitialize_modules(isp);
-initialize_modules_fail:
-	pm_qos_remove_request(&isp->pm_qos);
-	atomisp_msi_irq_uninit(isp, dev);
-enable_msi_fail:
-fw_validation_fail:
-	release_firmware(isp->firmware);
-load_fw_fail:
-	/*
-	 * Switch off ISP, as keeping it powered on would prevent
-	 * reaching S0ix states.
-	 *
-	 * The following lines have been copied from atomisp suspend path
-	 */
-
-	pci_read_config_dword(dev, PCI_INTERRUPT_CTRL, &irq);
-	irq = irq & 1 << INTR_IIR;
-	pci_write_config_dword(dev, PCI_INTERRUPT_CTRL, irq);
-
-	pci_read_config_dword(dev, PCI_INTERRUPT_CTRL, &irq);
-	irq &= ~(1 << INTR_IER);
-	pci_write_config_dword(dev, PCI_INTERRUPT_CTRL, irq);
-
-	atomisp_msi_irq_uninit(isp, dev);
-
-	atomisp_ospm_dphy_down(isp);
-
-	/* Address later when we worry about the ...field chips */
-	if (IS_ENABLED(CONFIG_PM) && atomisp_mrfld_power_down(isp))
-		dev_err(&dev->dev, "Failed to switch off ISP\n");
-	return err;
-}
-
-static void atomisp_pci_remove(struct pci_dev *dev)
-{
-	struct atomisp_device *isp = (struct atomisp_device *)
-		pci_get_drvdata(dev);
-
-	atomisp_drvfs_exit();
-
-	atomisp_acc_cleanup(isp);
-
-	atomisp_css_unload_firmware(isp);
-	hmm_cleanup();
-
-	pm_runtime_forbid(&dev->dev);
-	pm_runtime_get_noresume(&dev->dev);
-	pm_qos_remove_request(&isp->pm_qos);
-
-	atomisp_msi_irq_uninit(isp, dev);
-	atomisp_unregister_entities(isp);
-
-	destroy_workqueue(isp->wdt_work_queue);
-	atomisp_file_input_cleanup(isp);
-
-	release_firmware(isp->firmware);
-
-	hmm_pool_unregister(HMM_POOL_TYPE_RESERVED);
-}
-
-static const struct pci_device_id atomisp_pci_tbl[] = {
-#if defined(ISP2400) || defined(ISP2400B0)
-	/* Merrifield */
-	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x1178)},
-	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x1179)},
-	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x117a)},
-	/* Baytrail */
-	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0f38)},
-#elif defined(ISP2401)
-	/* Anniedale (Merrifield+ / Moorefield) */
-	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x1478)},
-	/* Cherrytrail */
-	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x22b8)},
-#endif
-	{0,}
-};
-
-MODULE_DEVICE_TABLE(pci, atomisp_pci_tbl);
-
-static const struct dev_pm_ops atomisp_pm_ops = {
-	.runtime_suspend = atomisp_runtime_suspend,
-	.runtime_resume = atomisp_runtime_resume,
-	.suspend = atomisp_suspend,
-	.resume = atomisp_resume,
-};
-
-static struct pci_driver atomisp_pci_driver = {
-	.driver = {
-		.pm = &atomisp_pm_ops,
-	},
-	.name = "atomisp-isp2",
-	.id_table = atomisp_pci_tbl,
-	.probe = atomisp_pci_probe,
-	.remove = atomisp_pci_remove,
-};
-
-static int __init atomisp_init(void)
-{
-	return pci_register_driver(&atomisp_pci_driver);
-}
-
-static void __exit atomisp_exit(void)
-{
-	pci_unregister_driver(&atomisp_pci_driver);
-}
-
-module_init(atomisp_init);
-module_exit(atomisp_exit);
-
-MODULE_AUTHOR("Wen Wang <wen.w.wang@intel.com>");
-MODULE_AUTHOR("Xiaolin Zhang <xiaolin.zhang@intel.com>");
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Intel ATOM Platform ISP Driver");
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.h
deleted file mode 100644
index 944a6cf40a2f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef __ATOMISP_V4L2_H__
-#define __ATOMISP_V4L2_H__
-
-struct atomisp_video_pipe;
-struct atomisp_acc_pipe;
-struct v4l2_device;
-struct atomisp_device;
-struct firmware;
-
-int atomisp_video_init(struct atomisp_video_pipe *video, const char *name);
-void atomisp_acc_init(struct atomisp_acc_pipe *video, const char *name);
-void atomisp_video_unregister(struct atomisp_video_pipe *video);
-int atomisp_video_register(struct atomisp_video_pipe *video,
-	struct v4l2_device *vdev);
-void atomisp_acc_unregister(struct atomisp_acc_pipe *video);
-int atomisp_acc_register(struct atomisp_acc_pipe *video,
-	struct v4l2_device *vdev);
-const struct firmware *atomisp_load_firmware(struct atomisp_device *isp);
-int atomisp_csi_lane_config(struct atomisp_device *isp);
-
-#endif /* __ATOMISP_V4L2_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/Makefile b/drivers/staging/media/atomisp/pci/atomisp2/css2400/Makefile
deleted file mode 100644
index ee5631b0e635..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/Makefile
+++ /dev/null
@@ -1,2 +0,0 @@
-ccflags-y += -DISP2400B0
-ISP2400B0 := y
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf.h
deleted file mode 100644
index 914aa7f98700..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf.h
+++ /dev/null
@@ -1,376 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_CIRCBUF_H
-#define _IA_CSS_CIRCBUF_H
-
-#include <sp.h>
-#include <type_support.h>
-#include <math_support.h>
-#include <assert_support.h>
-#include <platform_support.h>
-#include "ia_css_circbuf_comm.h"
-#include "ia_css_circbuf_desc.h"
-
-/****************************************************************
- *
- * Data structures.
- *
- ****************************************************************/
-/**
- * @brief Data structure for the circular buffer.
- */
-typedef struct ia_css_circbuf_s ia_css_circbuf_t;
-struct ia_css_circbuf_s {
-	ia_css_circbuf_desc_t *desc;    /* Pointer to the descriptor of the circbuf */
-	ia_css_circbuf_elem_t *elems;	/* an array of elements    */
-};
-
-/**
- * @brief Create the circular buffer.
- *
- * @param cb	The pointer to the circular buffer.
- * @param elems	An array of elements.
- * @param desc	The descriptor set to the size using ia_css_circbuf_desc_init().
- */
-extern void ia_css_circbuf_create(
-	ia_css_circbuf_t *cb,
-	ia_css_circbuf_elem_t *elems,
-	ia_css_circbuf_desc_t *desc);
-
-/**
- * @brief Destroy the circular buffer.
- *
- * @param cb The pointer to the circular buffer.
- */
-extern void ia_css_circbuf_destroy(
-		ia_css_circbuf_t *cb);
-
-/**
- * @brief Pop a value out of the circular buffer.
- * Get a value at the head of the circular buffer.
- * The user should call "ia_css_circbuf_is_empty()"
- * to avoid accessing to an empty buffer.
- *
- * @param cb	The pointer to the circular buffer.
- *
- * @return the pop-out value.
- */
-extern uint32_t ia_css_circbuf_pop(
-		ia_css_circbuf_t *cb);
-
-/**
- * @brief Extract a value out of the circular buffer.
- * Get a value at an arbitrary poistion in the circular
- * buffer. The user should call "ia_css_circbuf_is_empty()"
- * to avoid accessing to an empty buffer.
- *
- * @param cb	 The pointer to the circular buffer.
- * @param offset The offset from "start" to the target position.
- *
- * @return the extracted value.
- */
-extern uint32_t ia_css_circbuf_extract(
-	ia_css_circbuf_t *cb,
-	int offset);
-
-/****************************************************************
- *
- * Inline functions.
- *
- ****************************************************************/
-/**
- * @brief Set the "val" field in the element.
- *
- * @param elem The pointer to the element.
- * @param val  The value to be set.
- */
-static inline void ia_css_circbuf_elem_set_val(
-	ia_css_circbuf_elem_t *elem,
-	uint32_t val)
-{
-	OP___assert(elem != NULL);
-
-	elem->val = val;
-}
-
-/**
- * @brief Initialize the element.
- *
- * @param elem The pointer to the element.
- */
-static inline void ia_css_circbuf_elem_init(
-		ia_css_circbuf_elem_t *elem)
-{
-	OP___assert(elem != NULL);
-	ia_css_circbuf_elem_set_val(elem, 0);
-}
-
-/**
- * @brief Copy an element.
- *
- * @param src  The element as the copy source.
- * @param dest The element as the copy destination.
- */
-static inline void ia_css_circbuf_elem_cpy(
-	ia_css_circbuf_elem_t *src,
-	ia_css_circbuf_elem_t *dest)
-{
-	OP___assert(src != NULL);
-	OP___assert(dest != NULL);
-
-	ia_css_circbuf_elem_set_val(dest, src->val);
-}
-
-/**
- * @brief Get position in the circular buffer.
- *
- * @param cb		The pointer to the circular buffer.
- * @param base		The base position.
- * @param offset	The offset.
- *
- * @return the position at offset.
- */
-static inline uint8_t ia_css_circbuf_get_pos_at_offset(
-	ia_css_circbuf_t *cb,
-	uint32_t base,
-	int offset)
-{
-	uint8_t dest;
-
-	OP___assert(cb != NULL);
-	OP___assert(cb->desc != NULL);
-	OP___assert(cb->desc->size > 0);
-
-	/* step 1: adjudst the offset  */
-	while (offset < 0) {
-		offset += cb->desc->size;
-	}
-
-	/* step 2: shift and round by the upper limit */
-	dest = OP_std_modadd(base, offset, cb->desc->size);
-
-	return dest;
-}
-
-/**
- * @brief Get the offset between two positions in the circular buffer.
- * Get the offset from the source position to the terminal position,
- * along the direction in which the new elements come in.
- *
- * @param cb		The pointer to the circular buffer.
- * @param src_pos	The source position.
- * @param dest_pos	The terminal position.
- *
- * @return the offset.
- */
-static inline int ia_css_circbuf_get_offset(
-	ia_css_circbuf_t *cb,
-	uint32_t src_pos,
-	uint32_t dest_pos)
-{
-	int offset;
-
-	OP___assert(cb != NULL);
-	OP___assert(cb->desc != NULL);
-
-	offset = (int)(dest_pos - src_pos);
-	offset += (offset < 0) ? cb->desc->size : 0;
-
-	return offset;
-}
-
-/**
- * @brief Get the maximum number of elements.
- *
- * @param cb The pointer to the circular buffer.
- *
- * @return the maximum number of elements.
- *
- * TODO: Test this API.
- */
-static inline uint32_t ia_css_circbuf_get_size(
-		ia_css_circbuf_t *cb)
-{
-	OP___assert(cb != NULL);
-	OP___assert(cb->desc != NULL);
-
-	return cb->desc->size;
-}
-
-/**
- * @brief Get the number of available elements.
- *
- * @param cb The pointer to the circular buffer.
- *
- * @return the number of available elements.
- */
-static inline uint32_t ia_css_circbuf_get_num_elems(
-		ia_css_circbuf_t *cb)
-{
-	int num;
-
-	OP___assert(cb != NULL);
-	OP___assert(cb->desc != NULL);
-
-	num = ia_css_circbuf_get_offset(cb, cb->desc->start, cb->desc->end);
-
-	return (uint32_t)num;
-}
-
-/**
- * @brief Test if the circular buffer is empty.
- *
- * @param cb	The pointer to the circular buffer.
- *
- * @return
- *	- true when it is empty.
- *	- false when it is not empty.
- */
-static inline bool ia_css_circbuf_is_empty(
-		ia_css_circbuf_t *cb)
-{
-	OP___assert(cb != NULL);
-	OP___assert(cb->desc != NULL);
-
-	return ia_css_circbuf_desc_is_empty(cb->desc);
-}
-
-/**
- * @brief Test if the circular buffer is full.
- *
- * @param cb	The pointer to the circular buffer.
- *
- * @return
- *	- true when it is full.
- *	- false when it is not full.
- */
-static inline bool ia_css_circbuf_is_full(ia_css_circbuf_t *cb)
-{
-	OP___assert(cb != NULL);
-	OP___assert(cb->desc != NULL);
-
-	return ia_css_circbuf_desc_is_full(cb->desc);
-}
-
-/**
- * @brief Write a new element into the circular buffer.
- * Write a new element WITHOUT checking whether the
- * circular buffer is full or not. So it also overwrites
- * the oldest element when the buffer is full.
- *
- * @param cb	The pointer to the circular buffer.
- * @param elem	The new element.
- */
-static inline void ia_css_circbuf_write(
-	ia_css_circbuf_t *cb,
-	ia_css_circbuf_elem_t elem)
-{
-	OP___assert(cb != NULL);
-	OP___assert(cb->desc != NULL);
-
-	/* Cannot continue as the queue is full*/
-	assert(!ia_css_circbuf_is_full(cb));
-
-	ia_css_circbuf_elem_cpy(&elem, &cb->elems[cb->desc->end]);
-
-	cb->desc->end = ia_css_circbuf_get_pos_at_offset(cb, cb->desc->end, 1);
-}
-
-/**
- * @brief Push a value in the circular buffer.
- * Put a new value at the tail of the circular buffer.
- * The user should call "ia_css_circbuf_is_full()"
- * to avoid accessing to a full buffer.
- *
- * @param cb	The pointer to the circular buffer.
- * @param val	The value to be pushed in.
- */
-static inline void ia_css_circbuf_push(
-	ia_css_circbuf_t *cb,
-	uint32_t val)
-{
-	ia_css_circbuf_elem_t elem;
-
-	OP___assert(cb != NULL);
-
-	/* set up an element */
-	ia_css_circbuf_elem_init(&elem);
-	ia_css_circbuf_elem_set_val(&elem, val);
-
-	/* write the element into the buffer */
-	ia_css_circbuf_write(cb, elem);
-}
-
-/**
- * @brief Get the number of free elements.
- *
- * @param cb The pointer to the circular buffer.
- *
- * @return: The number of free elements.
- */
-static inline uint32_t ia_css_circbuf_get_free_elems(
-		ia_css_circbuf_t *cb)
-{
-	OP___assert(cb != NULL);
-	OP___assert(cb->desc != NULL);
-
-	return ia_css_circbuf_desc_get_free_elems(cb->desc);
-}
-
-/**
- * @brief Peek an element in Circular Buffer.
- *
- * @param cb	 The pointer to the circular buffer.
- * @param offset Offset to the element.
- *
- * @return the elements value.
- */
-extern uint32_t ia_css_circbuf_peek(
-	ia_css_circbuf_t *cb,
-	int offset);
-
-/**
- * @brief Get an element in Circular Buffer.
- *
- * @param cb	 The pointer to the circular buffer.
- * @param offset Offset to the element.
- *
- * @return the elements value.
- */
-extern uint32_t ia_css_circbuf_peek_from_start(
-	ia_css_circbuf_t *cb,
-	int offset);
-
-/**
- * @brief Increase Size of a Circular Buffer.
- * Use 'CAUTION' before using this function, This was added to
- * support / fix issue with increasing size for tagger only
- *
- * @param cb The pointer to the circular buffer.
- * @param sz_delta delta increase for new size
- * @param elems (optional) pointers to new additional elements
- *		cb element array size will not be increased dynamically,
- * 		but new elements should be added at the end to existing
- * 		cb element array which if of max_size >= new size
- *
- * @return	true on succesfully increasing the size
- * 			false on failure
- */
-extern bool ia_css_circbuf_increase_size(
-		ia_css_circbuf_t *cb,
-		unsigned int sz_delta,
-		ia_css_circbuf_elem_t *elems);
-
-#endif /*_IA_CSS_CIRCBUF_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf_comm.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf_comm.h
deleted file mode 100644
index 3fc0330b9526..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf_comm.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_CIRCBUF_COMM_H
-#define _IA_CSS_CIRCBUF_COMM_H
-
-#include <type_support.h>  /* uint8_t, uint32_t */
-
-#define IA_CSS_CIRCBUF_PADDING 1 /* The circular buffer is implemented in lock-less manner, wherein
-				   * the head and tail can advance independently without any locks.
-				   * But to achieve this, an extra buffer element is required to detect
-				   * queue full & empty conditions, wherein the tail trails the head for
-				   * full and is equal to head for empty condition. This causes 1 buffer
-				   * not being available for use.
-				   */
-
-/****************************************************************
- *
- * Portable Data structures
- *
- ****************************************************************/
-/**
- * @brief Data structure for the circular descriptor.
- */
-typedef struct ia_css_circbuf_desc_s ia_css_circbuf_desc_t;
-struct ia_css_circbuf_desc_s {
-	uint8_t size;	/* the maximum number of elements*/
-	uint8_t step;   /* number of bytes per element */
-	uint8_t start;	/* index of the oldest element */
-	uint8_t end;	/* index at which to write the new element */
-};
-#define SIZE_OF_IA_CSS_CIRCBUF_DESC_S_STRUCT				\
-	(4 * sizeof(uint8_t))
-
-/**
- * @brief Data structure for the circular buffer element.
- */
-typedef struct ia_css_circbuf_elem_s ia_css_circbuf_elem_t;
-struct ia_css_circbuf_elem_s {
-	uint32_t val;	/* the value stored in the element */
-};
-#define SIZE_OF_IA_CSS_CIRCBUF_ELEM_S_STRUCT				\
-	(sizeof(uint32_t))
-
-#endif /*_IA_CSS_CIRCBUF_COMM_H*/
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf_desc.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf_desc.h
deleted file mode 100644
index 8dd7cd6cd3d8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/interface/ia_css_circbuf_desc.h
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_CIRCBUF_DESC_H_
-#define _IA_CSS_CIRCBUF_DESC_H_
-
-#include <type_support.h>
-#include <math_support.h>
-#include <platform_support.h>
-#include <sp.h>
-#include "ia_css_circbuf_comm.h"
-/****************************************************************
- *
- * Inline functions.
- *
- ****************************************************************/
-/**
- * @brief Test if the circular buffer is empty.
- *
- * @param cb_desc The pointer to the circular buffer descriptor.
- *
- * @return
- *	- true when it is empty.
- *	- false when it is not empty.
- */
-static inline bool ia_css_circbuf_desc_is_empty(
-		ia_css_circbuf_desc_t *cb_desc)
-{
-	OP___assert(cb_desc != NULL);
-	return (cb_desc->end == cb_desc->start);
-}
-
-/**
- * @brief Test if the circular buffer descriptor is full.
- *
- * @param cb_desc	The pointer to the circular buffer
- *			descriptor.
- *
- * @return
- *	- true when it is full.
- *	- false when it is not full.
- */
-static inline bool ia_css_circbuf_desc_is_full(
-		ia_css_circbuf_desc_t *cb_desc)
-{
-	OP___assert(cb_desc != NULL);
-	return (OP_std_modadd(cb_desc->end, 1, cb_desc->size) == cb_desc->start);
-}
-
-/**
- * @brief Initialize the circular buffer descriptor
- *
- * @param cb_desc	The pointer circular buffer descriptor
- * @param size		The size of the circular buffer
- */
-static inline void ia_css_circbuf_desc_init(
-	ia_css_circbuf_desc_t *cb_desc,
-	int8_t size)
-{
-	OP___assert(cb_desc != NULL);
-	cb_desc->size = size;
-}
-
-/**
- * @brief Get a position in the circular buffer descriptor.
- *
- * @param cb     The pointer to the circular buffer descriptor.
- * @param base   The base position.
- * @param offset The offset.
- *
- * @return the position in the circular buffer descriptor.
- */
-static inline uint8_t ia_css_circbuf_desc_get_pos_at_offset(
-	ia_css_circbuf_desc_t *cb_desc,
-	uint32_t base,
-	int offset)
-{
-	uint8_t dest;
-	OP___assert(cb_desc != NULL);
-	OP___assert(cb_desc->size > 0);
-
-	/* step 1: adjust the offset  */
-	while (offset < 0) {
-		offset += cb_desc->size;
-	}
-
-	/* step 2: shift and round by the upper limit */
-	dest = OP_std_modadd(base, offset, cb_desc->size);
-
-	return dest;
-}
-
-/**
- * @brief Get the offset between two positions in the circular buffer
- * descriptor.
- * Get the offset from the source position to the terminal position,
- * along the direction in which the new elements come in.
- *
- * @param cb_desc	The pointer to the circular buffer descriptor.
- * @param src_pos	The source position.
- * @param dest_pos	The terminal position.
- *
- * @return the offset.
- */
-static inline int ia_css_circbuf_desc_get_offset(
-	ia_css_circbuf_desc_t *cb_desc,
-	uint32_t src_pos,
-	uint32_t dest_pos)
-{
-	int offset;
-	OP___assert(cb_desc != NULL);
-
-	offset = (int)(dest_pos - src_pos);
-	offset += (offset < 0) ? cb_desc->size : 0;
-
-	return offset;
-}
-
-/**
- * @brief Get the number of available elements.
- *
- * @param cb_desc The pointer to the circular buffer.
- *
- * @return The number of available elements.
- */
-static inline uint32_t ia_css_circbuf_desc_get_num_elems(
-		ia_css_circbuf_desc_t *cb_desc)
-{
-	int num;
-	OP___assert(cb_desc != NULL);
-
-	num = ia_css_circbuf_desc_get_offset(cb_desc,
-				cb_desc->start,
-				cb_desc->end);
-
-	return (uint32_t)num;
-}
-
-/**
- * @brief Get the number of free elements.
- *
- * @param cb_desc The pointer to the circular buffer descriptor.
- *
- * @return: The number of free elements.
- */
-static inline uint32_t ia_css_circbuf_desc_get_free_elems(
-		ia_css_circbuf_desc_t *cb_desc)
-{
-	uint32_t num;
-	OP___assert(cb_desc != NULL);
-
-	num = ia_css_circbuf_desc_get_offset(cb_desc,
-				cb_desc->start,
-				cb_desc->end);
-
-	return (cb_desc->size - num);
-}
-#endif /*_IA_CSS_CIRCBUF_DESC_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/src/circbuf.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/src/circbuf.c
deleted file mode 100644
index 050d60f0894f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/circbuf/src/circbuf.c
+++ /dev/null
@@ -1,321 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_circbuf.h"
-
-#include <assert_support.h>
-
-/**********************************************************************
- *
- * Forward declarations.
- *
- **********************************************************************/
-/*
- * @brief Read the oldest element from the circular buffer.
- * Read the oldest element WITHOUT checking whehter the
- * circular buffer is empty or not. The oldest element is
- * also removed out from the circular buffer.
- *
- * @param cb The pointer to the circular buffer.
- *
- * @return the oldest element.
- */
-static inline ia_css_circbuf_elem_t
-ia_css_circbuf_read(ia_css_circbuf_t *cb);
-
-/*
- * @brief Shift a chunk of elements in the circular buffer.
- * A chunk of elements (i.e. the ones from the "start" position
- * to the "chunk_src" position) are shifted in the circular buffer,
- * along the direction of new elements coming.
- *
- * @param cb	     The pointer to the circular buffer.
- * @param chunk_src  The position at which the first element in the chunk is.
- * @param chunk_dest The position to which the first element in the chunk would be shift.
- */
-static inline void ia_css_circbuf_shift_chunk(ia_css_circbuf_t *cb,
-						   uint32_t chunk_src,
-						   uint32_t chunk_dest);
-
-/*
- * @brief Get the "val" field in the element.
- *
- * @param elem The pointer to the element.
- *
- * @return the "val" field.
- */
-static inline uint32_t
-ia_css_circbuf_elem_get_val(ia_css_circbuf_elem_t *elem);
-
-/**********************************************************************
- *
- * Non-inline functions.
- *
- **********************************************************************/
-/*
- * @brief Create the circular buffer.
- * Refer to "ia_css_circbuf.h" for details.
- */
-void
-ia_css_circbuf_create(ia_css_circbuf_t *cb,
-			   ia_css_circbuf_elem_t *elems,
-			   ia_css_circbuf_desc_t *desc)
-{
-	uint32_t i;
-
-	OP___assert(desc);
-
-	cb->desc = desc;
-	/* Initialize to defaults */
-	cb->desc->start = 0;
-	cb->desc->end = 0;
-	cb->desc->step = 0;
-
-	for (i = 0; i < cb->desc->size; i++)
-		ia_css_circbuf_elem_init(&elems[i]);
-
-	cb->elems = elems;
-}
-
-/*
- * @brief Destroy the circular buffer.
- * Refer to "ia_css_circbuf.h" for details.
- */
-void ia_css_circbuf_destroy(ia_css_circbuf_t *cb)
-{
-	cb->desc = NULL;
-
-	cb->elems = NULL;
-}
-
-/*
- * @brief Pop a value out of the circular buffer.
- * Refer to "ia_css_circbuf.h" for details.
- */
-uint32_t ia_css_circbuf_pop(ia_css_circbuf_t *cb)
-{
-	uint32_t ret;
-	ia_css_circbuf_elem_t elem;
-
-	assert(!ia_css_circbuf_is_empty(cb));
-
-	/* read an element from the buffer */
-	elem = ia_css_circbuf_read(cb);
-	ret = ia_css_circbuf_elem_get_val(&elem);
-	return ret;
-}
-
-/*
- * @brief Extract a value out of the circular buffer.
- * Refer to "ia_css_circbuf.h" for details.
- */
-uint32_t ia_css_circbuf_extract(ia_css_circbuf_t *cb, int offset)
-{
-	int max_offset;
-	uint32_t val;
-	uint32_t pos;
-	uint32_t src_pos;
-	uint32_t dest_pos;
-
-	/* get the maximum offest */
-	max_offset = ia_css_circbuf_get_offset(cb, cb->desc->start, cb->desc->end);
-	max_offset--;
-
-	/*
-	 * Step 1: When the target element is at the "start" position.
-	 */
-	if (offset == 0) {
-		val = ia_css_circbuf_pop(cb);
-		return val;
-	}
-
-	/*
-	 * Step 2: When the target element is out of the range.
-	 */
-	if (offset > max_offset) {
-		val = 0;
-		return val;
-	}
-
-	/*
-	 * Step 3: When the target element is between the "start" and
-	 * "end" position.
-	 */
-	/* get the position of the target element */
-	pos = ia_css_circbuf_get_pos_at_offset(cb, cb->desc->start, offset);
-
-	/* get the value from the target element */
-	val = ia_css_circbuf_elem_get_val(&cb->elems[pos]);
-
-	/* shift the elements */
-	src_pos = ia_css_circbuf_get_pos_at_offset(cb, pos, -1);
-	dest_pos = pos;
-	ia_css_circbuf_shift_chunk(cb, src_pos, dest_pos);
-
-	return val;
-}
-
-/*
- * @brief Peek an element from the circular buffer.
- * Refer to "ia_css_circbuf.h" for details.
- */
-uint32_t ia_css_circbuf_peek(ia_css_circbuf_t *cb, int offset)
-{
-	int pos;
-
-	pos = ia_css_circbuf_get_pos_at_offset(cb, cb->desc->end, offset);
-
-	/* get the value at the position */
-	return cb->elems[pos].val;
-}
-
-/*
- * @brief Get the value of an element from the circular buffer.
- * Refer to "ia_css_circbuf.h" for details.
- */
-uint32_t ia_css_circbuf_peek_from_start(ia_css_circbuf_t *cb, int offset)
-{
-	int pos;
-
-	pos = ia_css_circbuf_get_pos_at_offset(cb, cb->desc->start, offset);
-
-	/* get the value at the position */
-	return cb->elems[pos].val;
-}
-
-/* @brief increase size of a circular buffer.
- * Use 'CAUTION' before using this function. This was added to
- * support / fix issue with increasing size for tagger only
- * Please refer to "ia_css_circbuf.h" for details.
- */
-bool ia_css_circbuf_increase_size(
-				ia_css_circbuf_t *cb,
-				unsigned int sz_delta,
-				ia_css_circbuf_elem_t *elems)
-{
-	uint8_t curr_size;
-	uint8_t curr_end;
-	unsigned int i = 0;
-
-	if (!cb || sz_delta == 0)
-		return false;
-
-	curr_size = cb->desc->size;
-	curr_end = cb->desc->end;
-	/* We assume cb was pre defined as global to allow
-	 * increase in size */
-	/* FM: are we sure this cannot cause size to become too big? */
-	if (((uint8_t)(cb->desc->size + (uint8_t)sz_delta) > cb->desc->size) && ((uint8_t)sz_delta == sz_delta))
-		cb->desc->size += (uint8_t)sz_delta;
-	else
-		return false; /* overflow in size */
-
-	/* If elems are passed update them else we assume its been taken
-	 * care before calling this function */
-	if (elems) {
-		/* cb element array size will not be increased dynamically,
-		 * but pointers to new elements can be added at the end
-		 * of existing pre defined cb element array of
-		 * size >= new size if not already added */
-		for (i = curr_size; i <  cb->desc->size; i++)
-			cb->elems[i] = elems[i - curr_size];
-	}
-	/* Fix Start / End */
-	if (curr_end < cb->desc->start) {
-		if (curr_end == 0) {
-			/* Easily fix End */
-			cb->desc->end = curr_size;
-		} else {
-			/* Move elements and fix Start*/
-			ia_css_circbuf_shift_chunk(cb,
-						curr_size - 1,
-						curr_size + sz_delta - 1);
-		}
-	}
-
-	return true;
-}
-
-/****************************************************************
- *
- * Inline functions.
- *
- ****************************************************************/
-/*
- * @brief Get the "val" field in the element.
- * Refer to "Forward declarations" for details.
- */
-static inline uint32_t
-ia_css_circbuf_elem_get_val(ia_css_circbuf_elem_t *elem)
-{
-	return elem->val;
-}
-
-/*
- * @brief Read the oldest element from the circular buffer.
- * Refer to "Forward declarations" for details.
- */
-static inline ia_css_circbuf_elem_t
-ia_css_circbuf_read(ia_css_circbuf_t *cb)
-{
-	ia_css_circbuf_elem_t elem;
-
-	/* get the element from the target position */
-	elem = cb->elems[cb->desc->start];
-
-	/* clear the target position */
-	ia_css_circbuf_elem_init(&cb->elems[cb->desc->start]);
-
-	/* adjust the "start" position */
-	cb->desc->start = ia_css_circbuf_get_pos_at_offset(cb, cb->desc->start, 1);
-	return elem;
-}
-
-/*
- * @brief Shift a chunk of elements in the circular buffer.
- * Refer to "Forward declarations" for details.
- */
-static inline void
-ia_css_circbuf_shift_chunk(ia_css_circbuf_t *cb,
-				uint32_t chunk_src, uint32_t chunk_dest)
-{
-	int chunk_offset;
-	int chunk_sz;
-	int i;
-
-	/* get the chunk offset and size */
-	chunk_offset = ia_css_circbuf_get_offset(cb,
-						      chunk_src, chunk_dest);
-	chunk_sz = ia_css_circbuf_get_offset(cb, cb->desc->start, chunk_src) + 1;
-
-	/* shift each element to its terminal position */
-	for (i = 0; i < chunk_sz; i++) {
-
-		/* copy the element from the source to the destination */
-		ia_css_circbuf_elem_cpy(&cb->elems[chunk_src],
-					     &cb->elems[chunk_dest]);
-
-		/* clear the source position */
-		ia_css_circbuf_elem_init(&cb->elems[chunk_src]);
-
-		/* adjust the source/terminal positions */
-		chunk_src = ia_css_circbuf_get_pos_at_offset(cb, chunk_src, -1);
-		chunk_dest = ia_css_circbuf_get_pos_at_offset(cb, chunk_dest, -1);
-
-	}
-
-	/* adjust the index "start" */
-	cb->desc->start = ia_css_circbuf_get_pos_at_offset(cb, cb->desc->start, chunk_offset);
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/refcount/interface/ia_css_refcount.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/refcount/interface/ia_css_refcount.h
deleted file mode 100644
index 20db4de6beeb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/refcount/interface/ia_css_refcount.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_REFCOUNT_H_
-#define _IA_CSS_REFCOUNT_H_
-
-#include <type_support.h>
-#include <system_types.h>
-#include <ia_css_err.h>
-
-typedef void (*clear_func)(hrt_vaddress ptr);
-
-/*! \brief Function for initializing refcount list
- *
- * \param[in]	size		Size of the refcount list.
- * \return				ia_css_err
- */
-extern enum ia_css_err ia_css_refcount_init(uint32_t size);
-
-/*! \brief Function for de-initializing refcount list
- *
- * \return				None
- */
-extern void ia_css_refcount_uninit(void);
-
-/*! \brief Function for increasing reference by 1.
- *
- * \param[in]	id		ID of the object.
- * \param[in]	ptr		Data of the object (ptr).
- * \return				hrt_vaddress (saved address)
- */
-extern hrt_vaddress ia_css_refcount_increment(int32_t id, hrt_vaddress ptr);
-
-/*! \brief Function for decrease reference by 1.
- *
- * \param[in]	id		ID of the object.
- * \param[in]	ptr		Data of the object (ptr).
- *
- *	- true, if it is successful.
- *	- false, otherwise.
- */
-extern bool ia_css_refcount_decrement(int32_t id, hrt_vaddress ptr);
-
-/*! \brief Function to check if reference count is 1.
- *
- * \param[in]	ptr		Data of the object (ptr).
- *
- *	- true, if it is successful.
- *	- false, otherwise.
- */
-extern bool ia_css_refcount_is_single(hrt_vaddress ptr);
-
-/*! \brief Function to clear reference list objects.
- *
- * \param[in]	id			ID of the object.
- * \param[in] clear_func	function to be run to free reference objects.
- *
- *  return				None
- */
-extern void ia_css_refcount_clear(int32_t id,
-				  clear_func clear_func_ptr);
-
-/*! \brief Function to verify if object is valid
- *
- * \param[in] ptr       Data of the object (ptr)
- *
- *      - true, if valid
- *      - false, if invalid
- */
-extern bool ia_css_refcount_is_valid(hrt_vaddress ptr);
-
-#endif /* _IA_CSS_REFCOUNT_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/refcount/src/refcount.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/refcount/src/refcount.c
deleted file mode 100644
index 6e3bd773ee4c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/base/refcount/src/refcount.c
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_refcount.h"
-#include "memory_access/memory_access.h"
-#include "sh_css_defs.h"
-
-#include "platform_support.h"
-
-#include "assert_support.h"
-
-#include "ia_css_debug.h"
-
-/* TODO: enable for other memory aswell
-	 now only for hrt_vaddress */
-struct ia_css_refcount_entry {
-	uint32_t count;
-	hrt_vaddress data;
-	int32_t id;
-};
-
-struct ia_css_refcount_list {
-	uint32_t size;
-	struct ia_css_refcount_entry *items;
-};
-
-static struct ia_css_refcount_list myrefcount;
-
-static struct ia_css_refcount_entry *refcount_find_entry(hrt_vaddress ptr,
-							 bool firstfree)
-{
-	uint32_t i;
-
-	if (ptr == 0)
-		return NULL;
-	if (myrefcount.items == NULL) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-				    "refcount_find_entry(): Ref count not initiliazed!\n");
-		return NULL;
-	}
-
-	for (i = 0; i < myrefcount.size; i++) {
-
-		if ((&myrefcount.items[i])->data == 0) {
-			if (firstfree) {
-				/* for new entry */
-				return &myrefcount.items[i];
-			}
-		}
-		if ((&myrefcount.items[i])->data == ptr) {
-			/* found entry */
-			return &myrefcount.items[i];
-		}
-	}
-	return NULL;
-}
-
-enum ia_css_err ia_css_refcount_init(uint32_t size)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	if (size == 0) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				    "ia_css_refcount_init(): Size of 0 for Ref count init!\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	if (myrefcount.items != NULL) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				    "ia_css_refcount_init(): Ref count is already initialized\n");
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-	myrefcount.items =
-	    sh_css_malloc(sizeof(struct ia_css_refcount_entry) * size);
-	if (!myrefcount.items)
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	if (err == IA_CSS_SUCCESS) {
-		memset(myrefcount.items, 0,
-		       sizeof(struct ia_css_refcount_entry) * size);
-		myrefcount.size = size;
-	}
-	return err;
-}
-
-void ia_css_refcount_uninit(void)
-{
-	struct ia_css_refcount_entry *entry;
-	uint32_t i;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			    "ia_css_refcount_uninit() entry\n");
-	for (i = 0; i < myrefcount.size; i++) {
-		/* driver verifier tool has issues with &arr[i]
-		   and prefers arr + i; as these are actually equivalent
-		   the line below uses + i
-		*/
-		entry = myrefcount.items + i;
-		if (entry->data != mmgr_NULL) {
-			/*	ia_css_debug_dtrace(IA_CSS_DBG_TRACE,
-				"ia_css_refcount_uninit: freeing (%x)\n",
-				entry->data);*/
-			hmm_free(entry->data);
-			entry->data = mmgr_NULL;
-			entry->count = 0;
-			entry->id = 0;
-		}
-	}
-	sh_css_free(myrefcount.items);
-	myrefcount.items = NULL;
-	myrefcount.size = 0;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			    "ia_css_refcount_uninit() leave\n");
-}
-
-hrt_vaddress ia_css_refcount_increment(int32_t id, hrt_vaddress ptr)
-{
-	struct ia_css_refcount_entry *entry;
-
-	if (ptr == mmgr_NULL)
-		return ptr;
-
-	entry = refcount_find_entry(ptr, false);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			    "ia_css_refcount_increment(%x) 0x%x\n", id, ptr);
-
-	if (!entry) {
-		entry = refcount_find_entry(ptr, true);
-		assert(entry != NULL);
-		if (entry == NULL)
-			return mmgr_NULL;
-		entry->id = id;
-	}
-
-	if (entry->id != id) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-			    "ia_css_refcount_increment(): Ref count IDS do not match!\n");
-		return mmgr_NULL;
-	}
-
-	if (entry->data == ptr)
-		entry->count += 1;
-	else if (entry->data == mmgr_NULL) {
-		entry->data = ptr;
-		entry->count = 1;
-	} else
-		return mmgr_NULL;
-
-	return ptr;
-}
-
-bool ia_css_refcount_decrement(int32_t id, hrt_vaddress ptr)
-{
-	struct ia_css_refcount_entry *entry;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			    "ia_css_refcount_decrement(%x) 0x%x\n", id, ptr);
-
-	if (ptr == mmgr_NULL)
-		return false;
-
-	entry = refcount_find_entry(ptr, false);
-
-	if (entry) {
-		if (entry->id != id) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-					    "ia_css_refcount_decrement(): Ref count IDS do not match!\n");
-			return false;
-		}
-		if (entry->count > 0) {
-			entry->count -= 1;
-			if (entry->count == 0) {
-				/* ia_css_debug_dtrace(IA_CSS_DBEUG_TRACE,
-				   "ia_css_refcount_decrement: freeing\n");*/
-				hmm_free(ptr);
-				entry->data = mmgr_NULL;
-				entry->id = 0;
-			}
-			return true;
-		}
-	}
-
-	/* SHOULD NOT HAPPEN: ptr not managed by refcount, or not
-	   valid anymore */
-	if (entry)
-		IA_CSS_ERROR("id %x, ptr 0x%x entry %p entry->id %x entry->count %d\n",
-			id, ptr, entry, entry->id, entry->count);
-	else
-		IA_CSS_ERROR("entry NULL\n");
-#ifdef ISP2401
-	assert(false);
-#endif
-
-	return false;
-}
-
-bool ia_css_refcount_is_single(hrt_vaddress ptr)
-{
-	struct ia_css_refcount_entry *entry;
-
-	if (ptr == mmgr_NULL)
-		return false;
-
-	entry = refcount_find_entry(ptr, false);
-
-	if (entry)
-		return (entry->count == 1);
-
-	return true;
-}
-
-void ia_css_refcount_clear(int32_t id, clear_func clear_func_ptr)
-{
-	struct ia_css_refcount_entry *entry;
-	uint32_t i;
-	uint32_t count = 0;
-
-	assert(clear_func_ptr != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_refcount_clear(%x)\n",
-			    id);
-
-	for (i = 0; i < myrefcount.size; i++) {
-		/* driver verifier tool has issues with &arr[i]
-		   and prefers arr + i; as these are actually equivalent
-		   the line below uses + i
-		*/
-		entry = myrefcount.items + i;
-		if ((entry->data != mmgr_NULL) && (entry->id == id)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-					    "ia_css_refcount_clear:"
-					    " %x: 0x%x\n", id, entry->data);
-			if (clear_func_ptr) {
-				/* clear using provided function */
-				clear_func_ptr(entry->data);
-			} else {
-				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-						    "ia_css_refcount_clear: "
-						    "using hmm_free: "
-						    "no clear_func\n");
-				hmm_free(entry->data);
-			}
-#ifndef ISP2401
-
-#else
-			assert(entry->count == 0);
-#endif
-			if (entry->count != 0) {
-				IA_CSS_WARNING("Ref count for entry %x is not zero!", entry->id);
-			}
-			entry->data = mmgr_NULL;
-			entry->count = 0;
-			entry->id = 0;
-			count++;
-		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			    "ia_css_refcount_clear(%x): cleared %d\n", id,
-			    count);
-}
-
-bool ia_css_refcount_is_valid(hrt_vaddress ptr)
-{
-	struct ia_css_refcount_entry *entry;
-
-	if (ptr == mmgr_NULL)
-		return false;
-
-	entry = refcount_find_entry(ptr, false);
-
-	return entry != NULL;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_binarydesc.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_binarydesc.h
deleted file mode 100644
index a6d650a9a1f4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_binarydesc.h
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_PIPE_BINARYDESC_H__
-#define __IA_CSS_PIPE_BINARYDESC_H__
-
-#include <ia_css_types.h>		/* ia_css_pipe */
-#include <ia_css_frame_public.h>	/* ia_css_frame_info */
-#include <ia_css_binary.h>		/* ia_css_binary_descr */
-
-/* @brief Get a binary descriptor for copy.
- *
- * @param[in] pipe
- * @param[out] copy_desc
- * @param[in/out] in_info
- * @param[in/out] out_info
- * @param[in/out] vf_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_copy_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *copy_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info);
-
-/* @brief Get a binary descriptor for vfpp.
- *
- * @param[in] pipe
- * @param[out] vfpp_descr
- * @param[in/out] in_info
- * @param[in/out] out_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_vfpp_binarydesc(
-		struct ia_css_pipe const * const pipe,
-		struct ia_css_binary_descr *vf_pp_descr,
-		struct ia_css_frame_info *in_info,
-		struct ia_css_frame_info *out_info);
-
-/* @brief Get numerator and denominator of bayer downscaling factor.
- *
- * @param[in] bds_factor: The bayer downscaling factor.
- *		(= The bds_factor member in the sh_css_bds_factor structure.)
- * @param[out] bds_factor_numerator: The numerator of the bayer downscaling factor.
- *		(= The numerator member in the sh_css_bds_factor structure.)
- * @param[out] bds_factor_denominator: The denominator of the bayer downscaling factor.
- *		(= The denominator member in the sh_css_bds_factor structure.)
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
- */
-extern enum ia_css_err sh_css_bds_factor_get_numerator_denominator(
-	unsigned int bds_factor,
-	unsigned int *bds_factor_numerator,
-	unsigned int *bds_factor_denominator);
-
-/* @brief Get a binary descriptor for preview stage.
- *
- * @param[in] pipe
- * @param[out] preview_descr
- * @param[in/out] in_info
- * @param[in/out] bds_out_info
- * @param[in/out] out_info
- * @param[in/out] vf_info
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
- */
-extern enum ia_css_err ia_css_pipe_get_preview_binarydesc(
-	struct ia_css_pipe * const pipe,
-	struct ia_css_binary_descr *preview_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *bds_out_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info);
-
-/* @brief Get a binary descriptor for video stage.
- *
- * @param[in/out] pipe
- * @param[out] video_descr
- * @param[in/out] in_info
- * @param[in/out] bds_out_info
- * @param[in/out] vf_info
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
- */
-extern enum ia_css_err ia_css_pipe_get_video_binarydesc(
-	struct ia_css_pipe * const pipe,
-	struct ia_css_binary_descr *video_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *bds_out_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info,
-	int stream_config_left_padding);
-
-/* @brief Get a binary descriptor for yuv scaler stage.
- *
- * @param[in/out] pipe
- * @param[out] yuv_scaler_descr
- * @param[in/out] in_info
- * @param[in/out] out_info
- * @param[in/out] internal_out_info
- * @param[in/out] vf_info
- * @return    None
- *
- */
-void ia_css_pipe_get_yuvscaler_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *yuv_scaler_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *internal_out_info,
-	struct ia_css_frame_info *vf_info);
-
-/* @brief Get a binary descriptor for capture pp stage.
- *
- * @param[in/out] pipe
- * @param[out] capture_pp_descr
- * @param[in/out] in_info
- * @param[in/out] vf_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_capturepp_binarydesc(
-	struct ia_css_pipe * const pipe,
-	struct ia_css_binary_descr *capture_pp_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info);
-
-/* @brief Get a binary descriptor for primary capture.
- *
- * @param[in] pipe
- * @param[out] prim_descr
- * @param[in/out] in_info
- * @param[in/out] out_info
- * @param[in/out] vf_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_primary_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *prim_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info,
-	unsigned int stage_idx);
-
-/* @brief Get a binary descriptor for pre gdc stage.
- *
- * @param[in] pipe
- * @param[out] pre_gdc_descr
- * @param[in/out] in_info
- * @param[in/out] out_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_pre_gdc_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *gdc_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info);
-
-/* @brief Get a binary descriptor for gdc stage.
- *
- * @param[in] pipe
- * @param[out] gdc_descr
- * @param[in/out] in_info
- * @param[in/out] out_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_gdc_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *gdc_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info);
-
-/* @brief Get a binary descriptor for post gdc.
- *
- * @param[in] pipe
- * @param[out] post_gdc_descr
- * @param[in/out] in_info
- * @param[in/out] out_info
- * @param[in/out] vf_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_post_gdc_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *post_gdc_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info);
-
-/* @brief Get a binary descriptor for de.
- *
- * @param[in] pipe
- * @param[out] pre_de_descr
- * @param[in/out] in_info
- * @param[in/out] out_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_pre_de_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *pre_de_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info);
-
-/* @brief Get a binary descriptor for pre anr stage.
- *
- * @param[in] pipe
- * @param[out] pre_anr_descr
- * @param[in/out] in_info
- * @param[in/out] out_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_pre_anr_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *pre_anr_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info);
-
-/* @brief Get a binary descriptor for ANR stage.
- *
- * @param[in] pipe
- * @param[out] anr_descr
- * @param[in/out] in_info
- * @param[in/out] out_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_anr_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *anr_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info);
-
-/* @brief Get a binary descriptor for post anr stage.
- *
- * @param[in] pipe
- * @param[out] post_anr_descr
- * @param[in/out] in_info
- * @param[in/out] out_info
- * @param[in/out] vf_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_post_anr_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *post_anr_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info);
-
-/* @brief Get a binary descriptor for ldc stage.
- *
- * @param[in/out] pipe
- * @param[out] capture_pp_descr
- * @param[in/out] in_info
- * @param[in/out] vf_info
- * @return    None
- *
- */
-extern void ia_css_pipe_get_ldc_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *ldc_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info);
-
-/* @brief Calculates the required BDS factor
- *
- * @param[in] input_res
- * @param[in] output_res
- * @param[in/out] bds_factor
- * @return	IA_CSS_SUCCESS or error code upon error.
- */
-enum ia_css_err binarydesc_calculate_bds_factor(
-	struct ia_css_resolution input_res,
-	struct ia_css_resolution output_res,
-	unsigned int *bds_factor);
-
-#endif /* __IA_CSS_PIPE_BINARYDESC_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_stagedesc.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_stagedesc.h
deleted file mode 100644
index 38690ea093c2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_stagedesc.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_PIPE_STAGEDESC_H__
-#define __IA_CSS_PIPE_STAGEDESC_H__
-
-#include <ia_css_acc_types.h> /* ia_css_fw_info */
-#include <ia_css_frame_public.h>
-#include <ia_css_binary.h>
-#include "ia_css_pipeline.h"
-#include "ia_css_pipeline_common.h"
-
-extern void ia_css_pipe_get_generic_stage_desc(
-	struct ia_css_pipeline_stage_desc *stage_desc,
-	struct ia_css_binary *binary,
-	struct ia_css_frame *out_frame[],
-	struct ia_css_frame *in_frame,
-	struct ia_css_frame *vf_frame);
-
-extern void ia_css_pipe_get_firmwares_stage_desc(
-	struct ia_css_pipeline_stage_desc *stage_desc,
-	struct ia_css_binary *binary,
-	struct ia_css_frame *out_frame[],
-	struct ia_css_frame *in_frame,
-	struct ia_css_frame *vf_frame,
-	const struct ia_css_fw_info *fw,
-	unsigned int mode);
-
-extern void ia_css_pipe_get_acc_stage_desc(
-	struct ia_css_pipeline_stage_desc *stage_desc,
-	struct ia_css_binary *binary,
-	struct ia_css_fw_info *fw);
-
-extern void ia_css_pipe_get_sp_func_stage_desc(
-	struct ia_css_pipeline_stage_desc *stage_desc,
-	struct ia_css_frame *out_frame,
-	enum ia_css_pipeline_stage_sp_func sp_func,
-	unsigned max_input_width);
-
-#endif /*__IA_CSS_PIPE_STAGEDESC__H__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_util.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_util.h
deleted file mode 100644
index 155b6fb4722b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/interface/ia_css_pipe_util.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_PIPE_UTIL_H__
-#define __IA_CSS_PIPE_UTIL_H__
-
-#include <ia_css_types.h>
-#include <ia_css_frame_public.h>
-
-/* @brief Get Input format bits per pixel based on stream configuration of this
- * pipe.
- *
- * @param[in] pipe
- * @return   bits per pixel for the underlying stream
- *
- */
-extern unsigned int ia_css_pipe_util_pipe_input_format_bpp(
-	const struct ia_css_pipe * const pipe);
-
-extern void ia_css_pipe_util_create_output_frames(
-	struct ia_css_frame *frames[]);
-
-extern void ia_css_pipe_util_set_output_frames(
-	struct ia_css_frame *frames[],
-	unsigned int idx,
-	struct ia_css_frame *frame);
-
-#endif /* __IA_CSS_PIPE_UTIL_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_binarydesc.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_binarydesc.c
deleted file mode 100644
index 98a2a3e9b3e6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_binarydesc.c
+++ /dev/null
@@ -1,880 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_pipe_binarydesc.h"
-#include "ia_css_frame_format.h"
-#include "ia_css_pipe.h"
-#include "ia_css_pipe_util.h"
-#include "ia_css_util.h"
-#include "ia_css_debug.h"
-#include "sh_css_params.h"
-#include <assert_support.h>
-/* HRT_GDC_N */
-#include "gdc_device.h"
-#include <linux/kernel.h>
-
-/* This module provides a binary descriptions to used to find a binary. Since,
- * every stage is associated with a binary, it implicity helps stage
- * description. Apart from providing a binary description, this module also
- * populates the frame info's when required.*/
-
-/* Generic descriptor for offline binaries. Internal function. */
-static void pipe_binarydesc_get_offline(
-	struct ia_css_pipe const * const pipe,
-	const int mode,
-	struct ia_css_binary_descr *descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info[],
-	struct ia_css_frame_info *vf_info)
-{
-	unsigned int i;
-	/* in_info, out_info, vf_info can be NULL */
-	assert(pipe != NULL);
-	assert(descr != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-			    "pipe_binarydesc_get_offline() enter:\n");
-
-	descr->mode = mode;
-	descr->online = false;
-	descr->continuous = pipe->stream->config.continuous;
-	descr->striped = false;
-	descr->two_ppc = false;
-	descr->enable_yuv_ds = false;
-	descr->enable_high_speed = false;
-	descr->enable_dvs_6axis = false;
-	descr->enable_reduced_pipe = false;
-	descr->enable_dz = true;
-	descr->enable_xnr = false;
-	descr->enable_dpc = false;
-#ifdef ISP2401
-	descr->enable_luma_only = false;
-	descr->enable_tnr = false;
-#endif
-	descr->enable_capture_pp_bli = false;
-	descr->enable_fractional_ds = false;
-	descr->dvs_env.width = 0;
-	descr->dvs_env.height = 0;
-	descr->stream_format = pipe->stream->config.input_config.format;
-	descr->in_info = in_info;
-	descr->bds_out_info = NULL;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		descr->out_info[i] = out_info[i];
-	descr->vf_info = vf_info;
-	descr->isp_pipe_version = pipe->config.isp_pipe_version;
-	descr->required_bds_factor = SH_CSS_BDS_FACTOR_1_00;
-	descr->stream_config_left_padding = -1;
-}
-
-void ia_css_pipe_get_copy_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *copy_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info)
-{
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	unsigned int i;
-	/* out_info can be NULL */
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-	*in_info = *out_info;
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-	pipe_binarydesc_get_offline(pipe, IA_CSS_BINARY_MODE_COPY,
-				    copy_descr, in_info, out_infos, vf_info);
-	copy_descr->online = true;
-	copy_descr->continuous = false;
-	copy_descr->two_ppc = (pipe->stream->config.pixels_per_clock == 2);
-	copy_descr->enable_dz = false;
-	copy_descr->isp_pipe_version = IA_CSS_PIPE_VERSION_1;
-	IA_CSS_LEAVE_PRIVATE("");
-}
-void ia_css_pipe_get_vfpp_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *vf_pp_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info)
-{
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	unsigned int i;
-	/* out_info can be NULL ??? */
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-	in_info->raw_bit_depth = 0;
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe, IA_CSS_BINARY_MODE_VF_PP,
-			       vf_pp_descr, in_info, out_infos, NULL);
-	vf_pp_descr->enable_fractional_ds = true;
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-static struct sh_css_bds_factor bds_factors_list[] = {
-	{1, 1, SH_CSS_BDS_FACTOR_1_00},
-	{5, 4, SH_CSS_BDS_FACTOR_1_25},
-	{3, 2, SH_CSS_BDS_FACTOR_1_50},
-	{2, 1, SH_CSS_BDS_FACTOR_2_00},
-	{9, 4, SH_CSS_BDS_FACTOR_2_25},
-	{5, 2, SH_CSS_BDS_FACTOR_2_50},
-	{3, 1, SH_CSS_BDS_FACTOR_3_00},
-	{4, 1, SH_CSS_BDS_FACTOR_4_00},
-	{9, 2, SH_CSS_BDS_FACTOR_4_50},
-	{5, 1, SH_CSS_BDS_FACTOR_5_00},
-	{6, 1, SH_CSS_BDS_FACTOR_6_00},
-	{8, 1, SH_CSS_BDS_FACTOR_8_00}
-};
-
-enum ia_css_err sh_css_bds_factor_get_numerator_denominator(
-	unsigned int bds_factor,
-	unsigned int *bds_factor_numerator,
-	unsigned int *bds_factor_denominator)
-{
-	unsigned int i;
-
-	/* Loop over all bds factors until a match is found */
-	for (i = 0; i < ARRAY_SIZE(bds_factors_list); i++) {
-		if (bds_factors_list[i].bds_factor == bds_factor) {
-			*bds_factor_numerator = bds_factors_list[i].numerator;
-			*bds_factor_denominator = bds_factors_list[i].denominator;
-			return IA_CSS_SUCCESS;
-		}
-	}
-
-	/* Throw an error since bds_factor cannot be found
-	in bds_factors_list */
-	return IA_CSS_ERR_INVALID_ARGUMENTS;
-}
-
-enum ia_css_err binarydesc_calculate_bds_factor(
-	struct ia_css_resolution input_res,
-	struct ia_css_resolution output_res,
-	unsigned int *bds_factor)
-{
-	unsigned int i;
-	unsigned int in_w = input_res.width,
-	    in_h = input_res.height,
-	    out_w = output_res.width, out_h = output_res.height;
-
-	unsigned int max_bds_factor = 8;
-	unsigned int max_rounding_margin = 2;
-	/* delta in pixels to account for rounding margin in the calculation */
-	unsigned int delta = max_bds_factor * max_rounding_margin;
-
-	/* Assert if the resolutions are not set */
-	assert(in_w != 0 && in_h != 0);
-	assert(out_w != 0 && out_h != 0);
-
-	/* Loop over all bds factors until a match is found */
-	for (i = 0; i < ARRAY_SIZE(bds_factors_list); i++) {
-		unsigned num = bds_factors_list[i].numerator;
-		unsigned den = bds_factors_list[i].denominator;
-
-		/* See width-wise and height-wise if this bds_factor
-		 * satisfies the condition */
-		bool cond = (out_w * num / den + delta > in_w) &&
-		    (out_w * num / den <= in_w) &&
-		    (out_h * num / den + delta > in_h) &&
-		    (out_h * num / den <= in_h);
-
-		if (cond) {
-			*bds_factor = bds_factors_list[i].bds_factor;
-			return IA_CSS_SUCCESS;
-		}
-	}
-
-	/* Throw an error since a suitable bds_factor cannot be found */
-	return IA_CSS_ERR_INVALID_ARGUMENTS;
-}
-
-enum ia_css_err ia_css_pipe_get_preview_binarydesc(
-	struct ia_css_pipe * const pipe,
-	struct ia_css_binary_descr *preview_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *bds_out_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info)
-{
-	enum ia_css_err err;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	int mode = IA_CSS_BINARY_MODE_PREVIEW;
-	unsigned int i;
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	assert(out_info != NULL);
-	assert(vf_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-	/*
-	 * Set up the info of the input frame with
-	 * the ISP required resolution
-	 */
-	in_info->res = pipe->config.input_effective_res;
-	in_info->padded_width = in_info->res.width;
-	in_info->raw_bit_depth = ia_css_pipe_util_pipe_input_format_bpp(pipe);
-
-	if (ia_css_util_is_input_format_yuv(pipe->stream->config.input_config.format))
-		mode = IA_CSS_BINARY_MODE_COPY;
-	else
-		in_info->format = IA_CSS_FRAME_FORMAT_RAW;
-
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe, mode,
-			       preview_descr, in_info, out_infos, vf_info);
-	if (pipe->stream->config.online) {
-		preview_descr->online = pipe->stream->config.online;
-		preview_descr->two_ppc =
-		    (pipe->stream->config.pixels_per_clock == 2);
-	}
-	preview_descr->stream_format = pipe->stream->config.input_config.format;
-
-	/* TODO: Remove this when bds_out_info is available! */
-	*bds_out_info = *in_info;
-
-	if (pipe->extra_config.enable_raw_binning) {
-		if (pipe->config.bayer_ds_out_res.width != 0 &&
-		    pipe->config.bayer_ds_out_res.height != 0) {
-			bds_out_info->res.width =
-			    pipe->config.bayer_ds_out_res.width;
-			bds_out_info->res.height =
-			    pipe->config.bayer_ds_out_res.height;
-			bds_out_info->padded_width =
-			    pipe->config.bayer_ds_out_res.width;
-			err =
-			    binarydesc_calculate_bds_factor(in_info->res,
-				    bds_out_info->res,
-				    &preview_descr->required_bds_factor);
-			if (err != IA_CSS_SUCCESS)
-				return err;
-		} else {
-			bds_out_info->res.width = in_info->res.width / 2;
-			bds_out_info->res.height = in_info->res.height / 2;
-			bds_out_info->padded_width = in_info->padded_width / 2;
-			preview_descr->required_bds_factor =
-			    SH_CSS_BDS_FACTOR_2_00;
-		}
-	} else {
-		/* TODO: Remove this when bds_out_info->is available! */
-		bds_out_info->res.width = in_info->res.width;
-		bds_out_info->res.height = in_info->res.height;
-		bds_out_info->padded_width = in_info->padded_width;
-		preview_descr->required_bds_factor = SH_CSS_BDS_FACTOR_1_00;
-	}
-	pipe->required_bds_factor = preview_descr->required_bds_factor;
-
-	/* bayer ds and fractional ds cannot be enabled at the same time,
-	so we disable bds_out_info when fractional ds is used */
-	if (!pipe->extra_config.enable_fractional_ds)
-		preview_descr->bds_out_info = bds_out_info;
-	else
-		preview_descr->bds_out_info = NULL;
-	/*
-	   ----Preview binary-----
-	   --in-->|--out->|vf_veceven|--|--->vf
-	   -----------------------
-	   * Preview binary normally doesn't have a vf_port but
-	   * instead it has an output port. However, the output is
-	   * generated by vf_veceven module in which we might have
-	   * a downscaling (by 1x, 2x, or 4x). Because the resolution
-	   * might change, we need two different info, namely out_info
-	   * & vf_info. In fill_binary_info we use out&vf info to
-	   * calculate vf decimation factor.
-	 */
-	*out_info = *vf_info;
-
-	/* In case of preview_ds binary, we can do any fractional amount
-	 * of downscale, so there is no DS needed in vf_veceven. Therefore,
-	 * out and vf infos will be the same. Otherwise, we set out resolution
-	 * equal to in resolution. */
-	if (!pipe->extra_config.enable_fractional_ds) {
-		/* TODO: Change this when bds_out_info is available! */
-		out_info->res.width = bds_out_info->res.width;
-		out_info->res.height = bds_out_info->res.height;
-		out_info->padded_width = bds_out_info->padded_width;
-	}
-	preview_descr->enable_fractional_ds =
-	    pipe->extra_config.enable_fractional_ds;
-
-	preview_descr->enable_dpc = pipe->config.enable_dpc;
-
-	preview_descr->isp_pipe_version = pipe->config.isp_pipe_version;
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err ia_css_pipe_get_video_binarydesc(
-	struct ia_css_pipe * const pipe,
-	struct ia_css_binary_descr *video_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *bds_out_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info,
-	int stream_config_left_padding)
-{
-	int mode = IA_CSS_BINARY_MODE_VIDEO;
-	unsigned int i;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	bool stream_dz_config = false;
-
-	/* vf_info can be NULL */
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	/* assert(vf_info != NULL); */
-	IA_CSS_ENTER_PRIVATE("");
-
-	/* The solution below is not optimal; we should move to using ia_css_pipe_get_copy_binarydesc()
-	 * But for now this fixes things; this code used to be there but was removed
-	 * with gerrit 8908 as this was wrong for Skycam; however 240x still needs this
-	 */
-	if (ia_css_util_is_input_format_yuv(pipe->stream->config.input_config.format))
-		mode = IA_CSS_BINARY_MODE_COPY;
-
-	in_info->res = pipe->config.input_effective_res;
-	in_info->padded_width = in_info->res.width;
-	in_info->format = IA_CSS_FRAME_FORMAT_RAW;
-	in_info->raw_bit_depth = ia_css_pipe_util_pipe_input_format_bpp(pipe);
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe, mode,
-	       video_descr, in_info, out_infos, vf_info);
-
-	if (pipe->stream->config.online) {
-		video_descr->online = pipe->stream->config.online;
-		video_descr->two_ppc =
-		    (pipe->stream->config.pixels_per_clock == 2);
-	}
-
-	if (mode == IA_CSS_BINARY_MODE_VIDEO) {
-		stream_dz_config =
-		    ((pipe->stream->isp_params_configs->dz_config.dx !=
-		      HRT_GDC_N)
-		     || (pipe->stream->isp_params_configs->dz_config.dy !=
-			 HRT_GDC_N));
-
-		video_descr->enable_dz = pipe->config.enable_dz
-		    || stream_dz_config;
-		video_descr->dvs_env = pipe->config.dvs_envelope;
-		video_descr->enable_yuv_ds = pipe->extra_config.enable_yuv_ds;
-		video_descr->enable_high_speed =
-		    pipe->extra_config.enable_high_speed;
-		video_descr->enable_dvs_6axis =
-		    pipe->extra_config.enable_dvs_6axis;
-		video_descr->enable_reduced_pipe =
-		    pipe->extra_config.enable_reduced_pipe;
-		video_descr->isp_pipe_version = pipe->config.isp_pipe_version;
-		video_descr->enable_fractional_ds =
-		    pipe->extra_config.enable_fractional_ds;
-		video_descr->enable_dpc =
-			pipe->config.enable_dpc;
-#ifdef ISP2401
-		video_descr->enable_luma_only =
-			pipe->config.enable_luma_only;
-		video_descr->enable_tnr =
-			pipe->config.enable_tnr;
-#endif
-
-		if (pipe->extra_config.enable_raw_binning) {
-			if (pipe->config.bayer_ds_out_res.width != 0 &&
-			    pipe->config.bayer_ds_out_res.height != 0) {
-				bds_out_info->res.width =
-				    pipe->config.bayer_ds_out_res.width;
-				bds_out_info->res.height =
-				    pipe->config.bayer_ds_out_res.height;
-				bds_out_info->padded_width =
-				    pipe->config.bayer_ds_out_res.width;
-				err =
-				binarydesc_calculate_bds_factor(
-					in_info->res, bds_out_info->res,
-					&video_descr->required_bds_factor);
-				if (err != IA_CSS_SUCCESS)
-					return err;
-			} else {
-				bds_out_info->res.width =
-				    in_info->res.width / 2;
-				bds_out_info->res.height =
-				    in_info->res.height / 2;
-				bds_out_info->padded_width =
-				    in_info->padded_width / 2;
-				video_descr->required_bds_factor =
-				    SH_CSS_BDS_FACTOR_2_00;
-			}
-		} else {
-			bds_out_info->res.width = in_info->res.width;
-			bds_out_info->res.height = in_info->res.height;
-			bds_out_info->padded_width = in_info->padded_width;
-			video_descr->required_bds_factor =
-			    SH_CSS_BDS_FACTOR_1_00;
-		}
-
-		pipe->required_bds_factor = video_descr->required_bds_factor;
-
-		/* bayer ds and fractional ds cannot be enabled
-		at the same time, so we disable bds_out_info when
-		fractional ds is used */
-		if (!pipe->extra_config.enable_fractional_ds)
-			video_descr->bds_out_info = bds_out_info;
-		else
-			video_descr->bds_out_info = NULL;
-
-		video_descr->enable_fractional_ds =
-		    pipe->extra_config.enable_fractional_ds;
-		video_descr->stream_config_left_padding = stream_config_left_padding;
-	}
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-void ia_css_pipe_get_yuvscaler_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *yuv_scaler_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *internal_out_info,
-	struct ia_css_frame_info *vf_info)
-{
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	struct ia_css_frame_info *this_vf_info = NULL;
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	/* Note: if the following assert fails, the number of ports has been
-	 * changed; in that case an additional initializer must be added
-	 * a few lines below after which this assert can be updated.
-	 */
-	assert(IA_CSS_BINARY_MAX_OUTPUT_PORTS == 2);
-	IA_CSS_ENTER_PRIVATE("");
-
-	in_info->padded_width = in_info->res.width;
-	in_info->raw_bit_depth = 0;
-	ia_css_frame_info_set_width(in_info, in_info->res.width, 0);
-	out_infos[0] = out_info;
-	out_infos[1] = internal_out_info;
-	/* add initializers here if
-	 * assert(IA_CSS_BINARY_MAX_OUTPUT_PORTS == ...);
-	 * fails
-	 */
-
-	if (vf_info) {
-		this_vf_info = (vf_info->res.width == 0 &&
-			vf_info->res.height == 0) ? NULL : vf_info;
-	}
-
-	pipe_binarydesc_get_offline(pipe,
-			       IA_CSS_BINARY_MODE_CAPTURE_PP,
-			       yuv_scaler_descr,
-			       in_info, out_infos, this_vf_info);
-
-	yuv_scaler_descr->enable_fractional_ds = true;
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void ia_css_pipe_get_capturepp_binarydesc(
-	struct ia_css_pipe * const pipe,
-	struct ia_css_binary_descr *capture_pp_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info)
-{
-	unsigned int i;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	assert(vf_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-
-	/* the in_info is only used for resolution to enable
-	   bayer down scaling. */
-	if (pipe->out_yuv_ds_input_info.res.width)
-		*in_info = pipe->out_yuv_ds_input_info;
-	else
-		*in_info = *out_info;
-	in_info->format = IA_CSS_FRAME_FORMAT_YUV420;
-	in_info->raw_bit_depth = 0;
-	ia_css_frame_info_set_width(in_info, in_info->res.width, 0);
-
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe,
-			       IA_CSS_BINARY_MODE_CAPTURE_PP,
-			       capture_pp_descr,
-			       in_info, out_infos, vf_info);
-
-	capture_pp_descr->enable_capture_pp_bli =
-		pipe->config.default_capture_config.enable_capture_pp_bli;
-	capture_pp_descr->enable_fractional_ds = true;
-	capture_pp_descr->enable_xnr =
-		pipe->config.default_capture_config.enable_xnr != 0;
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-/* lookup table for high quality primary binaries */
-static unsigned int primary_hq_binary_modes[NUM_PRIMARY_HQ_STAGES] =
-{
-	IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE0,
-	IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE1,
-	IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE2,
-	IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE3,
-	IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE4,
-	IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE5
-};
-
-void ia_css_pipe_get_primary_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *prim_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info,
-	unsigned int stage_idx)
-{
-	enum ia_css_pipe_version pipe_version = pipe->config.isp_pipe_version;
-	int mode;
-	unsigned int i;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	assert(out_info != NULL);
-	assert(stage_idx < NUM_PRIMARY_HQ_STAGES);
-	/* vf_info can be NULL - example video_binarydescr */
-	/*assert(vf_info != NULL);*/
-	IA_CSS_ENTER_PRIVATE("");
-
-	if (pipe_version == IA_CSS_PIPE_VERSION_2_6_1)
-		mode = primary_hq_binary_modes[stage_idx];
-	else
-		mode = IA_CSS_BINARY_MODE_PRIMARY;
-
-	if (ia_css_util_is_input_format_yuv(pipe->stream->config.input_config.format))
-		mode = IA_CSS_BINARY_MODE_COPY;
-
-	in_info->res = pipe->config.input_effective_res;
-	in_info->padded_width = in_info->res.width;
-
-#if !defined(HAS_NO_PACKED_RAW_PIXELS)
-	if (pipe->stream->config.pack_raw_pixels)
-		in_info->format = IA_CSS_FRAME_FORMAT_RAW_PACKED;
-	else
-#endif
-		in_info->format = IA_CSS_FRAME_FORMAT_RAW;
-
-	in_info->raw_bit_depth = ia_css_pipe_util_pipe_input_format_bpp(pipe);
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe, mode,
-			       prim_descr, in_info, out_infos, vf_info);
-
-	if (pipe->stream->config.online &&
-	    pipe->stream->config.mode != IA_CSS_INPUT_MODE_MEMORY) {
-		prim_descr->online = true;
-		prim_descr->two_ppc =
-		    (pipe->stream->config.pixels_per_clock == 2);
-		prim_descr->stream_format = pipe->stream->config.input_config.format;
-	}
-	if (mode == IA_CSS_BINARY_MODE_PRIMARY) {
-		prim_descr->isp_pipe_version = pipe->config.isp_pipe_version;
-		prim_descr->enable_fractional_ds =
-		    pipe->extra_config.enable_fractional_ds;
-#ifdef ISP2401
-		prim_descr->enable_luma_only =
-			pipe->config.enable_luma_only;
-#endif
-		/* We have both striped and non-striped primary binaries,
-		 * if continuous viewfinder is required, then we must select
-		 * a striped one. Otherwise we prefer to use a non-striped
-		 * since it has better performance. */
-		if (pipe_version == IA_CSS_PIPE_VERSION_2_6_1)
-			prim_descr->striped = false;
-		else
-#ifndef ISP2401
-			prim_descr->striped = prim_descr->continuous && (!pipe->stream->stop_copy_preview || !pipe->stream->disable_cont_vf);
-#else
-			prim_descr->striped = prim_descr->continuous && !pipe->stream->disable_cont_vf;
-
-		if ((pipe->config.default_capture_config.enable_xnr != 0) &&
-			(pipe->extra_config.enable_dvs_6axis == true))
-				prim_descr->enable_xnr = true;
-#endif
-	}
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void ia_css_pipe_get_pre_gdc_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *pre_gdc_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info)
-{
-	unsigned int i;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	assert(out_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-	*in_info = *out_info;
-	in_info->format = IA_CSS_FRAME_FORMAT_RAW;
-	in_info->raw_bit_depth = ia_css_pipe_util_pipe_input_format_bpp(pipe);
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe, IA_CSS_BINARY_MODE_PRE_ISP,
-			       pre_gdc_descr, in_info, out_infos, NULL);
-	pre_gdc_descr->isp_pipe_version = pipe->config.isp_pipe_version;
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void ia_css_pipe_get_gdc_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *gdc_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info)
-{
-	unsigned int i;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	assert(out_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-	*in_info = *out_info;
-	in_info->format = IA_CSS_FRAME_FORMAT_QPLANE6;
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe, IA_CSS_BINARY_MODE_GDC,
-			       gdc_descr, in_info, out_infos, NULL);
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void ia_css_pipe_get_post_gdc_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *post_gdc_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info)
-{
-	unsigned int i;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	assert(out_info != NULL);
-	assert(vf_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-	*in_info = *out_info;
-	in_info->format = IA_CSS_FRAME_FORMAT_YUV420_16;
-	in_info->raw_bit_depth = 16;
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe, IA_CSS_BINARY_MODE_POST_ISP,
-			       post_gdc_descr, in_info, out_infos, vf_info);
-
-	post_gdc_descr->isp_pipe_version = pipe->config.isp_pipe_version;
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void ia_css_pipe_get_pre_de_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *pre_de_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info)
-{
-	unsigned int i;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	assert(out_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-	*in_info = *out_info;
-	in_info->format = IA_CSS_FRAME_FORMAT_RAW;
-	in_info->raw_bit_depth = ia_css_pipe_util_pipe_input_format_bpp(pipe);
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	if (pipe->config.isp_pipe_version == IA_CSS_PIPE_VERSION_1)
-		pipe_binarydesc_get_offline(pipe, IA_CSS_BINARY_MODE_PRE_ISP,
-				       pre_de_descr, in_info, out_infos, NULL);
-	else if (pipe->config.isp_pipe_version == IA_CSS_PIPE_VERSION_2_2) {
-		pipe_binarydesc_get_offline(pipe, IA_CSS_BINARY_MODE_PRE_DE,
-				       pre_de_descr, in_info, out_infos, NULL);
-	}
-
-	if (pipe->stream->config.online) {
-		pre_de_descr->online = true;
-		pre_de_descr->two_ppc =
-		    (pipe->stream->config.pixels_per_clock == 2);
-		pre_de_descr->stream_format = pipe->stream->config.input_config.format;
-	}
-	pre_de_descr->isp_pipe_version = pipe->config.isp_pipe_version;
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void ia_css_pipe_get_pre_anr_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *pre_anr_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info)
-{
-	unsigned int i;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	assert(out_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-	*in_info = *out_info;
-	in_info->format = IA_CSS_FRAME_FORMAT_RAW;
-	in_info->raw_bit_depth = ia_css_pipe_util_pipe_input_format_bpp(pipe);
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe, IA_CSS_BINARY_MODE_PRE_ISP,
-			       pre_anr_descr, in_info, out_infos, NULL);
-
-	if (pipe->stream->config.online) {
-		pre_anr_descr->online = true;
-		pre_anr_descr->two_ppc =
-		    (pipe->stream->config.pixels_per_clock == 2);
-		pre_anr_descr->stream_format = pipe->stream->config.input_config.format;
-	}
-	pre_anr_descr->isp_pipe_version = pipe->config.isp_pipe_version;
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void ia_css_pipe_get_anr_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *anr_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info)
-{
-	unsigned int i;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	assert(out_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-	*in_info = *out_info;
-	in_info->format = IA_CSS_FRAME_FORMAT_RAW;
-	in_info->raw_bit_depth = ANR_ELEMENT_BITS;
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe, IA_CSS_BINARY_MODE_ANR,
-			       anr_descr, in_info, out_infos, NULL);
-
-	anr_descr->isp_pipe_version = pipe->config.isp_pipe_version;
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-
-void ia_css_pipe_get_post_anr_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *post_anr_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info)
-{
-	unsigned int i;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	assert(out_info != NULL);
-	assert(vf_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-	*in_info = *out_info;
-	in_info->format = IA_CSS_FRAME_FORMAT_RAW;
-	in_info->raw_bit_depth = ANR_ELEMENT_BITS;
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe, IA_CSS_BINARY_MODE_POST_ISP,
-			       post_anr_descr, in_info, out_infos, vf_info);
-
-	post_anr_descr->isp_pipe_version = pipe->config.isp_pipe_version;
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void ia_css_pipe_get_ldc_binarydesc(
-	struct ia_css_pipe const * const pipe,
-	struct ia_css_binary_descr *ldc_descr,
-	struct ia_css_frame_info *in_info,
-	struct ia_css_frame_info *out_info)
-{
-	unsigned int i;
-	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-
-	assert(pipe != NULL);
-	assert(in_info != NULL);
-	assert(out_info != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-#ifndef ISP2401
-	*in_info = *out_info;
-#else
-	if (pipe->out_yuv_ds_input_info.res.width)
-		*in_info = pipe->out_yuv_ds_input_info;
-	else
-		*in_info = *out_info;
-#endif
-	in_info->format = IA_CSS_FRAME_FORMAT_YUV420;
-	in_info->raw_bit_depth = 0;
-	ia_css_frame_info_set_width(in_info, in_info->res.width, 0);
-
-	out_infos[0] = out_info;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		out_infos[i] = NULL;
-
-	pipe_binarydesc_get_offline(pipe, IA_CSS_BINARY_MODE_CAPTURE_PP,
-			       ldc_descr, in_info, out_infos, NULL);
-	ldc_descr->enable_dvs_6axis =
-		    pipe->extra_config.enable_dvs_6axis;
-	IA_CSS_LEAVE_PRIVATE("");
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_stagedesc.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_stagedesc.c
deleted file mode 100644
index 40af8daf5ad9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_stagedesc.c
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_pipe_stagedesc.h"
-#include "assert_support.h"
-#include "ia_css_debug.h"
-
-void ia_css_pipe_get_generic_stage_desc(
-	struct ia_css_pipeline_stage_desc *stage_desc,
-	struct ia_css_binary *binary,
-	struct ia_css_frame *out_frame[],
-	struct ia_css_frame *in_frame,
-	struct ia_css_frame *vf_frame)
-{
-	unsigned int i;
-	IA_CSS_ENTER_PRIVATE("stage_desc = %p, binary = %p, out_frame = %p, in_frame = %p, vf_frame = %p",
-			     stage_desc, binary, out_frame, in_frame, vf_frame);
-
-	assert(stage_desc != NULL && binary != NULL && binary->info != NULL);
-	if (stage_desc == NULL || binary == NULL || binary->info == NULL) {
-		IA_CSS_ERROR("invalid arguments");
-		goto ERR;
-	}
-
-	stage_desc->binary = binary;
-	stage_desc->firmware = NULL;
-	stage_desc->sp_func = IA_CSS_PIPELINE_NO_FUNC;
-	stage_desc->max_input_width = 0;
-	stage_desc->mode = binary->info->sp.pipeline.mode;
-	stage_desc->in_frame = in_frame;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		stage_desc->out_frame[i] = out_frame[i];
-	}
-	stage_desc->vf_frame = vf_frame;
-ERR:
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void ia_css_pipe_get_firmwares_stage_desc(
-	struct ia_css_pipeline_stage_desc *stage_desc,
-	struct ia_css_binary *binary,
-	struct ia_css_frame *out_frame[],
-	struct ia_css_frame *in_frame,
-	struct ia_css_frame *vf_frame,
-	const struct ia_css_fw_info *fw,
-	unsigned int mode)
-{
-	unsigned int i;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_pipe_get_firmwares_stage_desc() enter:\n");
-	stage_desc->binary = binary;
-	stage_desc->firmware = fw;
-	stage_desc->sp_func = IA_CSS_PIPELINE_NO_FUNC;
-	stage_desc->max_input_width = 0;
-	stage_desc->mode = mode;
-	stage_desc->in_frame = in_frame;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		stage_desc->out_frame[i] = out_frame[i];
-	}
-	stage_desc->vf_frame = vf_frame;
-}
-
-void ia_css_pipe_get_acc_stage_desc(
-	struct ia_css_pipeline_stage_desc *stage_desc,
-	struct ia_css_binary *binary,
-	struct ia_css_fw_info *fw)
-{
-	unsigned int i;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_pipe_get_acc_stage_desc() enter:\n");
-	stage_desc->binary = binary;
-	stage_desc->firmware = fw;
-	stage_desc->sp_func = IA_CSS_PIPELINE_NO_FUNC;
-	stage_desc->max_input_width = 0;
-	stage_desc->mode = IA_CSS_BINARY_MODE_VF_PP;
-	stage_desc->in_frame = NULL;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		stage_desc->out_frame[i] = NULL;
-	}
-	stage_desc->vf_frame = NULL;
-}
-
-void ia_css_pipe_get_sp_func_stage_desc(
-	struct ia_css_pipeline_stage_desc *stage_desc,
-	struct ia_css_frame *out_frame,
-	enum ia_css_pipeline_stage_sp_func sp_func,
-	unsigned max_input_width)
-{
-	unsigned int i;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_pipe_get_sp_func_stage_desc() enter:\n");
-	stage_desc->binary = NULL;
-	stage_desc->firmware = NULL;
-	stage_desc->sp_func = sp_func;
-	stage_desc->max_input_width = max_input_width;
-	stage_desc->mode = (unsigned int)-1;
-	stage_desc->in_frame = NULL;
-	stage_desc->out_frame[0] = out_frame;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		stage_desc->out_frame[i] = NULL;
-	}
-	stage_desc->vf_frame = NULL;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_util.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_util.c
deleted file mode 100644
index 5fc1718cb2bd..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/pipe/src/pipe_util.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_pipe_util.h"
-#include "ia_css_frame_public.h"
-#include "ia_css_pipe.h"
-#include "ia_css_util.h"
-#include "assert_support.h"
-
-unsigned int ia_css_pipe_util_pipe_input_format_bpp(
-	const struct ia_css_pipe * const pipe)
-{
-	assert(pipe != NULL);
-	assert(pipe->stream != NULL);
-
-	return ia_css_util_input_format_bpp(pipe->stream->config.input_config.format,
-			  pipe->stream->config.pixels_per_clock == 2);
-}
-
-void ia_css_pipe_util_create_output_frames(
-	struct ia_css_frame *frames[])
-{
-	unsigned int i;
-
-	assert(frames != NULL);
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		frames[i] = NULL;
-	}
-}
-
-void ia_css_pipe_util_set_output_frames(
-	struct ia_css_frame *frames[],
-	unsigned int idx,
-	struct ia_css_frame *frame)
-{
-	assert(idx < IA_CSS_BINARY_MAX_OUTPUT_PORTS);
-
-	frames[idx] = frame;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/util/interface/ia_css_util.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/util/interface/ia_css_util.h
deleted file mode 100644
index 5ab48f346790..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/util/interface/ia_css_util.h
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_UTIL_H__
-#define __IA_CSS_UTIL_H__
-
-#include <ia_css_err.h>
-#include <error_support.h>
-#include <type_support.h>
-#include <ia_css_frame_public.h>
-#include <ia_css_stream_public.h>
-#include <ia_css_stream_format.h>
-
-/* @brief convert "errno" error code to "ia_css_err" error code
- *
- * @param[in]	"errno" error code
- * @return	"ia_css_err" error code
- *
- */
-enum ia_css_err ia_css_convert_errno(
-	int in_err);
-
-/* @brief check vf frame info.
- *
- * @param[in] info
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
- */
-extern enum ia_css_err ia_css_util_check_vf_info(
-	const struct ia_css_frame_info * const info);
-
-/* @brief check input configuration.
- *
- * @param[in] stream_config
- * @param[in] must_be_raw
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
- */
-extern enum ia_css_err ia_css_util_check_input(
-	const struct ia_css_stream_config * const stream_config,
-	bool must_be_raw,
-	bool must_be_yuv);
-
-/* @brief check vf and out frame info.
- *
- * @param[in] out_info
- * @param[in] vf_info
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
- */
-extern enum ia_css_err ia_css_util_check_vf_out_info(
-	const struct ia_css_frame_info * const out_info,
-	const struct ia_css_frame_info * const vf_info);
-
-/* @brief check width and height
- *
- * @param[in] width
- * @param[in] height
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
- */
-extern enum ia_css_err ia_css_util_check_res(
-	unsigned int width,
-	unsigned int height);
-
-#ifdef ISP2401
-/* @brief compare resolutions (less or equal)
- *
- * @param[in] a resolution
- * @param[in] b resolution
- * @return    true if both dimensions of a are less or
- *            equal than those of b, false otherwise
- *
- */
-extern bool ia_css_util_res_leq(
-	struct ia_css_resolution a,
-	struct ia_css_resolution b);
-
-/**
- * @brief Check if resolution is zero
- *
- * @param[in] resolution The resolution to check
- *
- * @returns true if resolution is zero
- */
-extern bool ia_css_util_resolution_is_zero(
-		const struct ia_css_resolution resolution);
-
-/**
- * @brief Check if resolution is even
- *
- * @param[in] resolution The resolution to check
- *
- * @returns true if resolution is even
- */
-extern bool ia_css_util_resolution_is_even(
-		const struct ia_css_resolution resolution);
-
-#endif
-/* @brief check width and height
- *
- * @param[in] stream_format
- * @param[in] two_ppc
- * @return bits per pixel based on given parameters.
- *
- */
-extern unsigned int ia_css_util_input_format_bpp(
-	enum atomisp_input_format stream_format,
-	bool two_ppc);
-
-/* @brief check if input format it raw
- *
- * @param[in] stream_format
- * @return true if the input format is raw or false otherwise
- *
- */
-extern bool ia_css_util_is_input_format_raw(
-	enum atomisp_input_format stream_format);
-
-/* @brief check if input format it yuv
- *
- * @param[in] stream_format
- * @return true if the input format is yuv or false otherwise
- *
- */
-extern bool ia_css_util_is_input_format_yuv(
-	enum atomisp_input_format stream_format);
-
-#endif /* __IA_CSS_UTIL_H__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/util/src/util.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/util/src/util.c
deleted file mode 100644
index 91e586112332..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/camera/util/src/util.c
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_util.h"
-#include <ia_css_frame.h>
-#include <assert_support.h>
-#include <math_support.h>
-
-/* for ia_css_binary_max_vf_width() */
-#include "ia_css_binary.h"
-
-
-enum ia_css_err ia_css_convert_errno(
-				int in_err)
-{
-	enum ia_css_err out_err;
-
-	switch (in_err) {
-		case 0:
-			out_err = IA_CSS_SUCCESS;
-			break;
-		case EINVAL:
-			out_err = IA_CSS_ERR_INVALID_ARGUMENTS;
-			break;
-		case ENODATA:
-			out_err = IA_CSS_ERR_QUEUE_IS_EMPTY;
-			break;
-		case ENOSYS:
-		case ENOTSUP:
-			out_err = IA_CSS_ERR_INTERNAL_ERROR;
-			break;
-		case ENOBUFS:
-			out_err = IA_CSS_ERR_QUEUE_IS_FULL;
-			break;
-		default:
-			out_err = IA_CSS_ERR_INTERNAL_ERROR;
-			break;
-	}
-	return out_err;
-}
-
-/* MW: Table look-up ??? */
-unsigned int ia_css_util_input_format_bpp(
-	enum atomisp_input_format format,
-	bool two_ppc)
-{
-	unsigned int rval = 0;
-	switch (format) {
-	case ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY:
-	case ATOMISP_INPUT_FORMAT_YUV420_8:
-	case ATOMISP_INPUT_FORMAT_YUV422_8:
-	case ATOMISP_INPUT_FORMAT_RGB_888:
-	case ATOMISP_INPUT_FORMAT_RAW_8:
-	case ATOMISP_INPUT_FORMAT_BINARY_8:
-	case ATOMISP_INPUT_FORMAT_EMBEDDED:
-		rval = 8;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_10:
-	case ATOMISP_INPUT_FORMAT_YUV422_10:
-	case ATOMISP_INPUT_FORMAT_RAW_10:
-		rval = 10;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_16:
-	case ATOMISP_INPUT_FORMAT_YUV422_16:
-		rval = 16;
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_444:
-		rval = 4;
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_555:
-		rval = 5;
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_565:
-		rval = 65;
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_666:
-	case ATOMISP_INPUT_FORMAT_RAW_6:
-		rval = 6;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_7:
-		rval = 7;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_12:
-		rval = 12;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_14:
-		if (two_ppc)
-			rval = 14;
-		else
-			rval = 12;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_16:
-		if (two_ppc)
-			rval = 16;
-		else
-			rval = 12;
-		break;
-	default:
-		rval = 0;
-		break;
-
-	}
-	return rval;
-}
-
-enum ia_css_err ia_css_util_check_vf_info(
-	const struct ia_css_frame_info * const info)
-{
-	enum ia_css_err err;
-	unsigned int max_vf_width;
-	assert(info != NULL);
-	err = ia_css_frame_check_info(info);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	max_vf_width = ia_css_binary_max_vf_width();
-	if (max_vf_width != 0 && info->res.width > max_vf_width*2)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err ia_css_util_check_vf_out_info(
-	const struct ia_css_frame_info * const out_info,
-	const struct ia_css_frame_info * const vf_info)
-{
-	enum ia_css_err err;
-
-	assert(out_info != NULL);
-	assert(vf_info != NULL);
-
-	err = ia_css_frame_check_info(out_info);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	err = ia_css_util_check_vf_info(vf_info);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err ia_css_util_check_res(unsigned int width, unsigned int height)
-{
-	/* height can be odd number for jpeg/embedded data from ISYS2401 */
-	if (((width  == 0)   ||
-	     (height == 0)   ||
-	     IS_ODD(width))) {
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	return IA_CSS_SUCCESS;
-}
-
-#ifdef ISP2401
-bool ia_css_util_res_leq(struct ia_css_resolution a, struct ia_css_resolution b)
-{
-	return a.width <= b.width && a.height <= b.height;
-}
-
-bool ia_css_util_resolution_is_zero(const struct ia_css_resolution resolution)
-{
-	return (resolution.width == 0) || (resolution.height == 0);
-}
-
-bool ia_css_util_resolution_is_even(const struct ia_css_resolution resolution)
-{
-	return IS_EVEN(resolution.height) && IS_EVEN(resolution.width);
-}
-
-#endif
-bool ia_css_util_is_input_format_raw(enum atomisp_input_format format)
-{
-	return ((format == ATOMISP_INPUT_FORMAT_RAW_6) ||
-		(format == ATOMISP_INPUT_FORMAT_RAW_7) ||
-		(format == ATOMISP_INPUT_FORMAT_RAW_8) ||
-		(format == ATOMISP_INPUT_FORMAT_RAW_10) ||
-		(format == ATOMISP_INPUT_FORMAT_RAW_12));
-	/* raw_14 and raw_16 are not supported as input formats to the ISP.
-	 * They can only be copied to a frame in memory using the
-	 * copy binary.
-	 */
-}
-
-bool ia_css_util_is_input_format_yuv(enum atomisp_input_format format)
-{
-	return format == ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY ||
-	    format == ATOMISP_INPUT_FORMAT_YUV420_8  ||
-	    format == ATOMISP_INPUT_FORMAT_YUV420_10 ||
-	    format == ATOMISP_INPUT_FORMAT_YUV420_16 ||
-	    format == ATOMISP_INPUT_FORMAT_YUV422_8  ||
-	    format == ATOMISP_INPUT_FORMAT_YUV422_10 ||
-	    format == ATOMISP_INPUT_FORMAT_YUV422_16;
-}
-
-enum ia_css_err ia_css_util_check_input(
-	const struct ia_css_stream_config * const stream_config,
-	bool must_be_raw,
-	bool must_be_yuv)
-{
-	assert(stream_config != NULL);
-
-	if (stream_config == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-#ifdef IS_ISP_2400_SYSTEM
-	if (stream_config->input_config.effective_res.width == 0 ||
-	    stream_config->input_config.effective_res.height == 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-#endif
-	if (must_be_raw &&
-	    !ia_css_util_is_input_format_raw(stream_config->input_config.format))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	if (must_be_yuv &&
-	    !ia_css_util_is_input_format_yuv(stream_config->input_config.format))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	return IA_CSS_SUCCESS;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_configs.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_configs.c
deleted file mode 100644
index 325b821f276c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_configs.c
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Generated code: do not edit or commmit. */
-
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_configs.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_iterator(
-	const struct ia_css_binary *binary,
-	const struct ia_css_iterator_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_iterator() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.iterator.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.iterator.offset;
-		}
-		if (size) {
-			ia_css_iterator_config((struct sh_css_isp_iterator_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_iterator() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_copy_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_copy_output_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_copy_output() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.copy_output.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.copy_output.offset;
-		}
-		if (size) {
-			ia_css_copy_output_config((struct sh_css_isp_copy_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_copy_output() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_crop(
-	const struct ia_css_binary *binary,
-	const struct ia_css_crop_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_crop() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.crop.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.crop.offset;
-		}
-		if (size) {
-			ia_css_crop_config((struct sh_css_isp_crop_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_crop() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_fpn(
-	const struct ia_css_binary *binary,
-	const struct ia_css_fpn_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_fpn() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.fpn.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.fpn.offset;
-		}
-		if (size) {
-			ia_css_fpn_config((struct sh_css_isp_fpn_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_fpn() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_dvs(
-	const struct ia_css_binary *binary,
-	const struct ia_css_dvs_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_dvs() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.dvs.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.dvs.offset;
-		}
-		if (size) {
-			ia_css_dvs_config((struct sh_css_isp_dvs_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_dvs() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_qplane(
-	const struct ia_css_binary *binary,
-	const struct ia_css_qplane_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_qplane() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.qplane.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.qplane.offset;
-		}
-		if (size) {
-			ia_css_qplane_config((struct sh_css_isp_qplane_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_qplane() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output0(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output0_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output0() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.output0.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.output0.offset;
-		}
-		if (size) {
-			ia_css_output0_config((struct sh_css_isp_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output0() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output1(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output1_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output1() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.output1.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.output1.offset;
-		}
-		if (size) {
-			ia_css_output1_config((struct sh_css_isp_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output1() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.output.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.output.offset;
-		}
-		if (size) {
-			ia_css_output_config((struct sh_css_isp_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-#ifdef ISP2401
-
-void
-ia_css_configure_sc(
-	const struct ia_css_binary *binary,
-	const struct ia_css_sc_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_sc() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.sc.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.sc.offset;
-		}
-		if (size) {
-			ia_css_sc_config((struct sh_css_isp_sc_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_sc() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-#endif
-
-void
-ia_css_configure_raw(
-	const struct ia_css_binary *binary,
-	const struct ia_css_raw_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_raw() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.raw.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.raw.offset;
-		}
-		if (size) {
-			ia_css_raw_config((struct sh_css_isp_raw_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_raw() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_tnr(
-	const struct ia_css_binary *binary,
-	const struct ia_css_tnr_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_tnr() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.tnr.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.tnr.offset;
-		}
-		if (size) {
-			ia_css_tnr_config((struct sh_css_isp_tnr_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_tnr() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_ref(
-	const struct ia_css_binary *binary,
-	const struct ia_css_ref_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_ref() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.ref.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.ref.offset;
-		}
-		if (size) {
-			ia_css_ref_config((struct sh_css_isp_ref_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_ref() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_vf(
-	const struct ia_css_binary *binary,
-	const struct ia_css_vf_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_vf() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.vf.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.vf.offset;
-		}
-		if (size) {
-			ia_css_vf_config((struct sh_css_isp_vf_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_vf() leave:\n");
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_configs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_configs.h
deleted file mode 100644
index 8aacd3dbc05a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_configs.h
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifdef IA_CSS_INCLUDE_CONFIGURATIONS
-#include "isp/kernels/crop/crop_1.0/ia_css_crop.host.h"
-#include "isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h"
-#include "isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h"
-#include "isp/kernels/ob/ob_1.0/ia_css_ob.host.h"
-#include "isp/kernels/output/output_1.0/ia_css_output.host.h"
-#include "isp/kernels/qplane/qplane_2/ia_css_qplane.host.h"
-#include "isp/kernels/raw/raw_1.0/ia_css_raw.host.h"
-#include "isp/kernels/ref/ref_1.0/ia_css_ref.host.h"
-#include "isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h"
-#ifdef ISP2401
-#include "isp/kernels/sc/sc_1.0/ia_css_sc.host.h"
-#endif
-#include "isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h"
-#include "isp/kernels/vf/vf_1.0/ia_css_vf.host.h"
-#include "isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h"
-#include "isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h"
-#endif /* IA_CSS_INCLUDE_CONFIGURATIONS */
-/* Generated code: do not edit or commmit. */
-
-#ifndef _IA_CSS_ISP_CONFIG_H
-#define _IA_CSS_ISP_CONFIG_H
-
-/* Code generated by genparam/gencode.c:gen_param_enum() */
-
-enum ia_css_configuration_ids {
-	IA_CSS_ITERATOR_CONFIG_ID,
-	IA_CSS_COPY_OUTPUT_CONFIG_ID,
-	IA_CSS_CROP_CONFIG_ID,
-	IA_CSS_FPN_CONFIG_ID,
-	IA_CSS_DVS_CONFIG_ID,
-	IA_CSS_QPLANE_CONFIG_ID,
-	IA_CSS_OUTPUT0_CONFIG_ID,
-	IA_CSS_OUTPUT1_CONFIG_ID,
-	IA_CSS_OUTPUT_CONFIG_ID,
-#ifdef ISP2401
-	IA_CSS_SC_CONFIG_ID,
-#endif
-	IA_CSS_RAW_CONFIG_ID,
-	IA_CSS_TNR_CONFIG_ID,
-	IA_CSS_REF_CONFIG_ID,
-	IA_CSS_VF_CONFIG_ID,
-	IA_CSS_NUM_CONFIGURATION_IDS
-};
-
-/* Code generated by genparam/gencode.c:gen_param_offsets() */
-
-struct ia_css_config_memory_offsets {
-	struct {
-		struct ia_css_isp_parameter iterator;
-		struct ia_css_isp_parameter copy_output;
-		struct ia_css_isp_parameter crop;
-		struct ia_css_isp_parameter fpn;
-		struct ia_css_isp_parameter dvs;
-		struct ia_css_isp_parameter qplane;
-		struct ia_css_isp_parameter output0;
-		struct ia_css_isp_parameter output1;
-		struct ia_css_isp_parameter output;
-#ifdef ISP2401
-		struct ia_css_isp_parameter sc;
-#endif
-		struct ia_css_isp_parameter raw;
-		struct ia_css_isp_parameter tnr;
-		struct ia_css_isp_parameter ref;
-		struct ia_css_isp_parameter vf;
-	} dmem;
-};
-
-#if defined(IA_CSS_INCLUDE_CONFIGURATIONS)
-
-#include "ia_css_stream.h"   /* struct ia_css_stream */
-#include "ia_css_binary.h"   /* struct ia_css_binary */
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_iterator(
-	const struct ia_css_binary *binary,
-	const struct ia_css_iterator_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_copy_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_copy_output_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_crop(
-	const struct ia_css_binary *binary,
-	const struct ia_css_crop_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_fpn(
-	const struct ia_css_binary *binary,
-	const struct ia_css_fpn_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_dvs(
-	const struct ia_css_binary *binary,
-	const struct ia_css_dvs_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_qplane(
-	const struct ia_css_binary *binary,
-	const struct ia_css_qplane_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output0(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output0_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output1(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output1_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-#ifdef ISP2401
-void
-ia_css_configure_sc(
-	const struct ia_css_binary *binary,
-	const struct ia_css_sc_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-#endif
-void
-ia_css_configure_raw(
-	const struct ia_css_binary *binary,
-	const struct ia_css_raw_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_tnr(
-	const struct ia_css_binary *binary,
-	const struct ia_css_tnr_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_ref(
-	const struct ia_css_binary *binary,
-	const struct ia_css_ref_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_vf(
-	const struct ia_css_binary *binary,
-	const struct ia_css_vf_configuration *config_dmem);
-
-#endif /* IA_CSS_INCLUDE_CONFIGURATION */
-
-#endif /* _IA_CSS_ISP_CONFIG_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_params.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_params.c
deleted file mode 100644
index d418e763b755..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_params.c
+++ /dev/null
@@ -1,3221 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#define IA_CSS_INCLUDE_PARAMETERS
-#include "sh_css_params.h"
-#include "isp/kernels/aa/aa_2/ia_css_aa2.host.h"
-#include "isp/kernels/anr/anr_1.0/ia_css_anr.host.h"
-#include "isp/kernels/anr/anr_2/ia_css_anr2.host.h"
-#include "isp/kernels/bh/bh_2/ia_css_bh.host.h"
-#include "isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h"
-#include "isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h"
-#include "isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h"
-#include "isp/kernels/crop/crop_1.0/ia_css_crop.host.h"
-#include "isp/kernels/csc/csc_1.0/ia_css_csc.host.h"
-#include "isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h"
-#include "isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h"
-#include "isp/kernels/ctc/ctc2/ia_css_ctc2.host.h"
-#include "isp/kernels/de/de_1.0/ia_css_de.host.h"
-#include "isp/kernels/de/de_2/ia_css_de2.host.h"
-#include "isp/kernels/dp/dp_1.0/ia_css_dp.host.h"
-#include "isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h"
-#include "isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h"
-#include "isp/kernels/gc/gc_1.0/ia_css_gc.host.h"
-#include "isp/kernels/gc/gc_2/ia_css_gc2.host.h"
-#include "isp/kernels/macc/macc_1.0/ia_css_macc.host.h"
-#include "isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h"
-#include "isp/kernels/ob/ob_1.0/ia_css_ob.host.h"
-#include "isp/kernels/ob/ob2/ia_css_ob2.host.h"
-#include "isp/kernels/output/output_1.0/ia_css_output.host.h"
-#include "isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h"
-#include "isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h"
-#include "isp/kernels/sc/sc_1.0/ia_css_sc.host.h"
-#include "isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h"
-#include "isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h"
-#include "isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h"
-#include "isp/kernels/uds/uds_1.0/ia_css_uds_param.h"
-#include "isp/kernels/wb/wb_1.0/ia_css_wb.host.h"
-#include "isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h"
-#include "isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h"
-#include "isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h"
-#include "isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h"
-#include "isp/kernels/fc/fc_1.0/ia_css_formats.host.h"
-#include "isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h"
-#include "isp/kernels/dpc2/ia_css_dpc2.host.h"
-#include "isp/kernels/eed1_8/ia_css_eed1_8.host.h"
-#include "isp/kernels/bnlm/ia_css_bnlm.host.h"
-#include "isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h"
-/* Generated code: do not edit or commmit. */
-
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_params.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_aa(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.aa.size;
-	unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.aa.offset;
-
-	if (size) {
-		struct sh_css_isp_aa_params *t =  (struct sh_css_isp_aa_params *)
-				&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-		t->strength = params->aa_config.strength;
-	}
-	params->isp_params_changed = true;
-	params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_anr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.anr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.anr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr() enter:\n");
-
-			ia_css_anr_encode((struct sh_css_isp_anr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->anr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_anr2(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.anr2.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.anr2.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr2() enter:\n");
-
-			ia_css_anr2_vmem_encode((struct ia_css_isp_anr2_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->anr_thres,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr2() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_bh(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.bh.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.bh.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() enter:\n");
-
-			ia_css_bh_encode((struct sh_css_isp_bh_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->s3a_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->hmem0.bh.size;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() enter:\n");
-
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_HMEM0] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_cnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.cnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.cnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_cnr() enter:\n");
-
-			ia_css_cnr_encode((struct sh_css_isp_cnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->cnr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_cnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_crop(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.crop.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.crop.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_crop() enter:\n");
-
-			ia_css_crop_encode((struct sh_css_isp_crop_isp_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->crop_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_crop() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_csc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.csc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.csc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_csc() enter:\n");
-
-			ia_css_csc_encode((struct sh_css_isp_csc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->cc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_csc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_dp(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.dp.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.dp.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_dp() enter:\n");
-
-			ia_css_dp_encode((struct sh_css_isp_dp_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dp_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_dp() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_bnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.bnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.bnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bnr() enter:\n");
-
-			ia_css_bnr_encode((struct sh_css_isp_bnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->nr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_de(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.de.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.de.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_de() enter:\n");
-
-			ia_css_de_encode((struct sh_css_isp_de_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->de_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_de() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ecd(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ecd.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ecd.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ecd() enter:\n");
-
-			ia_css_ecd_encode((struct sh_css_isp_ecd_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ecd_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ecd() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_formats(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.formats.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.formats.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_formats() enter:\n");
-
-			ia_css_formats_encode((struct sh_css_isp_formats_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->formats_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_formats() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_fpn(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.fpn.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.fpn.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fpn() enter:\n");
-
-			ia_css_fpn_encode((struct sh_css_isp_fpn_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->fpn_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fpn() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_gc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.gc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.gc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() enter:\n");
-
-			ia_css_gc_encode((struct sh_css_isp_gc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->gc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem1.gc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem1.gc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() enter:\n");
-
-			ia_css_gc_vamem_encode((struct sh_css_isp_gc_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM1].address[offset],
-					&params->gc_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM1] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ce(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ce.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ce.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ce() enter:\n");
-
-			ia_css_ce_encode((struct sh_css_isp_ce_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ce_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ce() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_yuv2rgb(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.yuv2rgb.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.yuv2rgb.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yuv2rgb() enter:\n");
-
-			ia_css_yuv2rgb_encode((struct sh_css_isp_csc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->yuv2rgb_cc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yuv2rgb() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_rgb2yuv(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.rgb2yuv.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.rgb2yuv.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_rgb2yuv() enter:\n");
-
-			ia_css_rgb2yuv_encode((struct sh_css_isp_csc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->rgb2yuv_cc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_rgb2yuv() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_r_gamma(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem0.r_gamma.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem0.r_gamma.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_r_gamma() enter:\n");
-
-			ia_css_r_gamma_vamem_encode((struct sh_css_isp_rgb_gamma_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM0].address[offset],
-					&params->r_gamma_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM0] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_r_gamma() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_g_gamma(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem1.g_gamma.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem1.g_gamma.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_g_gamma() enter:\n");
-
-			ia_css_g_gamma_vamem_encode((struct sh_css_isp_rgb_gamma_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM1].address[offset],
-					&params->g_gamma_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM1] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_g_gamma() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_b_gamma(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem2.b_gamma.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem2.b_gamma.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_b_gamma() enter:\n");
-
-			ia_css_b_gamma_vamem_encode((struct sh_css_isp_rgb_gamma_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM2].address[offset],
-					&params->b_gamma_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM2] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_b_gamma() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_uds(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.uds.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.uds.offset;
-
-		if (size) {
-			struct sh_css_sp_uds_params *p;
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_uds() enter:\n");
-
-			p = (struct sh_css_sp_uds_params *)
-				&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-			p->crop_pos = params->uds_config.crop_pos;
-			p->uds = params->uds_config.uds;
-
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_uds() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_raa(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.raa.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.raa.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_raa() enter:\n");
-
-			ia_css_raa_encode((struct sh_css_isp_aa_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->raa_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_raa() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_s3a(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.s3a.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.s3a.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_s3a() enter:\n");
-
-			ia_css_s3a_encode((struct sh_css_isp_s3a_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->s3a_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_s3a() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ob(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ob.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ob.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() enter:\n");
-
-			ia_css_ob_encode((struct sh_css_isp_ob_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ob_config,
-&params->stream_configs.ob, size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.ob.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.ob.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() enter:\n");
-
-			ia_css_ob_vmem_encode((struct sh_css_isp_ob_vmem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->ob_config,
-&params->stream_configs.ob, size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_output(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.output.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.output.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_output() enter:\n");
-
-			ia_css_output_encode((struct sh_css_isp_output_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->output_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_output() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sc() enter:\n");
-
-			ia_css_sc_encode((struct sh_css_isp_sc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->sc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_bds(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.bds.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.bds.offset;
-
-		if (size) {
-			struct sh_css_isp_bds_params *p;
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bds() enter:\n");
-
-			p = (struct sh_css_isp_bds_params *)
-				&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-			p->baf_strength = params->bds_config.strength;
-
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bds() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_tnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.tnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.tnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_tnr() enter:\n");
-
-			ia_css_tnr_encode((struct sh_css_isp_tnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->tnr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_tnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_macc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.macc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.macc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_macc() enter:\n");
-
-			ia_css_macc_encode((struct sh_css_isp_macc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->macc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_macc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_horicoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_horicoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_horicoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horicoef() enter:\n");
-
-			ia_css_sdis_horicoef_vmem_encode((struct sh_css_isp_sdis_hori_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horicoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_vertcoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_vertcoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_vertcoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertcoef() enter:\n");
-
-			ia_css_sdis_vertcoef_vmem_encode((struct sh_css_isp_sdis_vert_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertcoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_horiproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_horiproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_horiproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horiproj() enter:\n");
-
-			ia_css_sdis_horiproj_encode((struct sh_css_isp_sdis_hori_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horiproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_vertproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_vertproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_vertproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertproj() enter:\n");
-
-			ia_css_sdis_vertproj_encode((struct sh_css_isp_sdis_vert_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_horicoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_horicoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_horicoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horicoef() enter:\n");
-
-			ia_css_sdis2_horicoef_vmem_encode((struct sh_css_isp_sdis_hori_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horicoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_vertcoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_vertcoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_vertcoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertcoef() enter:\n");
-
-			ia_css_sdis2_vertcoef_vmem_encode((struct sh_css_isp_sdis_vert_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertcoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_horiproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_horiproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_horiproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horiproj() enter:\n");
-
-			ia_css_sdis2_horiproj_encode((struct sh_css_isp_sdis_hori_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horiproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_vertproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_vertproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_vertproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertproj() enter:\n");
-
-			ia_css_sdis2_vertproj_encode((struct sh_css_isp_sdis_vert_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_wb(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.wb.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.wb.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_wb() enter:\n");
-
-			ia_css_wb_encode((struct sh_css_isp_wb_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->wb_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_wb() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_nr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.nr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.nr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_nr() enter:\n");
-
-			ia_css_nr_encode((struct sh_css_isp_ynr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->nr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_nr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_yee(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.yee.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.yee.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yee() enter:\n");
-
-			ia_css_yee_encode((struct sh_css_isp_yee_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->yee_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yee() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ynr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ynr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ynr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ynr() enter:\n");
-
-			ia_css_ynr_encode((struct sh_css_isp_yee2_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ynr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ynr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_fc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.fc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.fc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fc() enter:\n");
-
-			ia_css_fc_encode((struct sh_css_isp_fc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->fc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ctc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ctc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ctc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() enter:\n");
-
-			ia_css_ctc_encode((struct sh_css_isp_ctc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ctc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem0.ctc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem0.ctc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() enter:\n");
-
-			ia_css_ctc_vamem_encode((struct sh_css_isp_ctc_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM0].address[offset],
-					&params->ctc_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM0] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_xnr_table(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem1.xnr_table.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem1.xnr_table.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr_table() enter:\n");
-
-			ia_css_xnr_table_vamem_encode((struct sh_css_isp_xnr_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM1].address[offset],
-					&params->xnr_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM1] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr_table() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_xnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.xnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.xnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr() enter:\n");
-
-			ia_css_xnr_encode((struct sh_css_isp_xnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->xnr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_xnr3(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.xnr3.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.xnr3.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() enter:\n");
-
-			ia_css_xnr3_encode((struct sh_css_isp_xnr3_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->xnr3_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() leave:\n");
-		}
-
-	}
-#ifdef ISP2401
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.xnr3.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.xnr3.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() enter:\n");
-
-			ia_css_xnr3_vmem_encode((struct sh_css_isp_xnr3_vmem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->xnr3_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() leave:\n");
-		}
-
-	}
-#endif
-}
-
-/* Code generated by genparam/gencode.c:gen_param_process_table() */
-
-void (* ia_css_kernel_process_param[IA_CSS_NUM_PARAMETER_IDS])(
-			unsigned pipe_id,
-			const struct ia_css_pipeline_stage *stage,
-			struct ia_css_isp_parameters *params) = {
-	ia_css_process_aa,
-	ia_css_process_anr,
-	ia_css_process_anr2,
-	ia_css_process_bh,
-	ia_css_process_cnr,
-	ia_css_process_crop,
-	ia_css_process_csc,
-	ia_css_process_dp,
-	ia_css_process_bnr,
-	ia_css_process_de,
-	ia_css_process_ecd,
-	ia_css_process_formats,
-	ia_css_process_fpn,
-	ia_css_process_gc,
-	ia_css_process_ce,
-	ia_css_process_yuv2rgb,
-	ia_css_process_rgb2yuv,
-	ia_css_process_r_gamma,
-	ia_css_process_g_gamma,
-	ia_css_process_b_gamma,
-	ia_css_process_uds,
-	ia_css_process_raa,
-	ia_css_process_s3a,
-	ia_css_process_ob,
-	ia_css_process_output,
-	ia_css_process_sc,
-	ia_css_process_bds,
-	ia_css_process_tnr,
-	ia_css_process_macc,
-	ia_css_process_sdis_horicoef,
-	ia_css_process_sdis_vertcoef,
-	ia_css_process_sdis_horiproj,
-	ia_css_process_sdis_vertproj,
-	ia_css_process_sdis2_horicoef,
-	ia_css_process_sdis2_vertcoef,
-	ia_css_process_sdis2_horiproj,
-	ia_css_process_sdis2_vertproj,
-	ia_css_process_wb,
-	ia_css_process_nr,
-	ia_css_process_yee,
-	ia_css_process_ynr,
-	ia_css_process_fc,
-	ia_css_process_ctc,
-	ia_css_process_xnr_table,
-	ia_css_process_xnr,
-	ia_css_process_xnr3,
-};
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_dp_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dp_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_dp_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dp_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_dp_config() leave\n");
-	ia_css_dp_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_dp_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dp_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_dp_config() enter:\n");
-	ia_css_dp_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dp_config = *config;
-	params->config_changed[IA_CSS_DP_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_DP_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_dp_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_wb_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_wb_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_wb_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->wb_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_wb_config() leave\n");
-	ia_css_wb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_wb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_wb_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_wb_config() enter:\n");
-	ia_css_wb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->wb_config = *config;
-	params->config_changed[IA_CSS_WB_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_WB_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_wb_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_tnr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_tnr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_tnr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->tnr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_tnr_config() leave\n");
-	ia_css_tnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_tnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_tnr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_tnr_config() enter:\n");
-	ia_css_tnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->tnr_config = *config;
-	params->config_changed[IA_CSS_TNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_TNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_tnr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ob_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ob_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ob_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ob_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ob_config() leave\n");
-	ia_css_ob_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ob_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ob_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ob_config() enter:\n");
-	ia_css_ob_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ob_config = *config;
-	params->config_changed[IA_CSS_OB_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_OB_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ob_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_de_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_de_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_de_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->de_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_de_config() leave\n");
-	ia_css_de_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_de_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_de_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_de_config() enter:\n");
-	ia_css_de_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->de_config = *config;
-	params->config_changed[IA_CSS_DE_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_DE_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_de_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_anr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_anr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->anr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr_config() leave\n");
-	ia_css_anr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_anr_config() enter:\n");
-	ia_css_anr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->anr_config = *config;
-	params->config_changed[IA_CSS_ANR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_ANR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_anr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_anr2_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_anr_thres *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr2_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->anr_thres;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr2_config() leave\n");
-	ia_css_anr2_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr2_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_thres *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_anr2_config() enter:\n");
-	ia_css_anr2_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->anr_thres = *config;
-	params->config_changed[IA_CSS_ANR2_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_ANR2_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_anr2_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ce_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ce_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ce_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ce_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ce_config() leave\n");
-	ia_css_ce_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ce_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ce_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ce_config() enter:\n");
-	ia_css_ce_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ce_config = *config;
-	params->config_changed[IA_CSS_CE_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CE_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ce_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ecd_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ecd_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ecd_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ecd_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ecd_config() leave\n");
-	ia_css_ecd_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ecd_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ecd_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ecd_config() enter:\n");
-	ia_css_ecd_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ecd_config = *config;
-	params->config_changed[IA_CSS_ECD_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_ECD_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ecd_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ynr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ynr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ynr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ynr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ynr_config() leave\n");
-	ia_css_ynr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ynr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ynr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ynr_config() enter:\n");
-	ia_css_ynr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ynr_config = *config;
-	params->config_changed[IA_CSS_YNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_YNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ynr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_fc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_fc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_fc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->fc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_fc_config() leave\n");
-	ia_css_fc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_fc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_fc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_fc_config() enter:\n");
-	ia_css_fc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->fc_config = *config;
-	params->config_changed[IA_CSS_FC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_FC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_fc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_cnr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cnr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_cnr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->cnr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_cnr_config() leave\n");
-	ia_css_cnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_cnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cnr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_cnr_config() enter:\n");
-	ia_css_cnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->cnr_config = *config;
-	params->config_changed[IA_CSS_CNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_cnr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_macc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_macc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_macc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->macc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_macc_config() leave\n");
-	ia_css_macc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_macc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_macc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_macc_config() enter:\n");
-	ia_css_macc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->macc_config = *config;
-	params->config_changed[IA_CSS_MACC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_MACC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_macc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ctc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ctc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ctc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ctc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ctc_config() leave\n");
-	ia_css_ctc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ctc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ctc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ctc_config() enter:\n");
-	ia_css_ctc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ctc_config = *config;
-	params->config_changed[IA_CSS_CTC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CTC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ctc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_aa_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_aa_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_aa_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->aa_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_aa_config() leave\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_aa_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_aa_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_aa_config() enter:\n");
-	params->aa_config = *config;
-	params->config_changed[IA_CSS_AA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_AA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_aa_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_yuv2rgb_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_yuv2rgb_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->yuv2rgb_cc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_yuv2rgb_config() leave\n");
-	ia_css_yuv2rgb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_yuv2rgb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_yuv2rgb_config() enter:\n");
-	ia_css_yuv2rgb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->yuv2rgb_cc_config = *config;
-	params->config_changed[IA_CSS_YUV2RGB_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_YUV2RGB_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_yuv2rgb_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_rgb2yuv_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_rgb2yuv_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->rgb2yuv_cc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_rgb2yuv_config() leave\n");
-	ia_css_rgb2yuv_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_rgb2yuv_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_rgb2yuv_config() enter:\n");
-	ia_css_rgb2yuv_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->rgb2yuv_cc_config = *config;
-	params->config_changed[IA_CSS_RGB2YUV_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_RGB2YUV_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_rgb2yuv_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_csc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_csc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->cc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_csc_config() leave\n");
-	ia_css_csc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_csc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_csc_config() enter:\n");
-	ia_css_csc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->cc_config = *config;
-	params->config_changed[IA_CSS_CSC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CSC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_csc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_nr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_nr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_nr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->nr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_nr_config() leave\n");
-	ia_css_nr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_nr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_nr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_nr_config() enter:\n");
-	ia_css_nr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->nr_config = *config;
-	params->config_changed[IA_CSS_BNR_ID] = true;
-	params->config_changed[IA_CSS_NR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_NR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_nr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_gc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_gc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_gc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->gc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_gc_config() leave\n");
-	ia_css_gc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_gc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_gc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_gc_config() enter:\n");
-	ia_css_gc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->gc_config = *config;
-	params->config_changed[IA_CSS_GC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_GC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_gc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_horicoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horicoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horicoef_config() leave\n");
-	ia_css_sdis_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_horicoef_config() enter:\n");
-	ia_css_sdis_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_horicoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_vertcoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertcoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertcoef_config() leave\n");
-	ia_css_sdis_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_vertcoef_config() enter:\n");
-	ia_css_sdis_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_vertcoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_horiproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horiproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horiproj_config() leave\n");
-	ia_css_sdis_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_horiproj_config() enter:\n");
-	ia_css_sdis_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_horiproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_vertproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertproj_config() leave\n");
-	ia_css_sdis_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_vertproj_config() enter:\n");
-	ia_css_sdis_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_vertproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_horicoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horicoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horicoef_config() leave\n");
-	ia_css_sdis2_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_horicoef_config() enter:\n");
-	ia_css_sdis2_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_horicoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_vertcoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertcoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertcoef_config() leave\n");
-	ia_css_sdis2_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_vertcoef_config() enter:\n");
-	ia_css_sdis2_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_vertcoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_horiproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horiproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horiproj_config() leave\n");
-	ia_css_sdis2_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_horiproj_config() enter:\n");
-	ia_css_sdis2_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_horiproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_vertproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertproj_config() leave\n");
-	ia_css_sdis2_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_vertproj_config() enter:\n");
-	ia_css_sdis2_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_vertproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_r_gamma_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_rgb_gamma_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_r_gamma_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->r_gamma_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_r_gamma_config() leave\n");
-	ia_css_r_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_r_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_r_gamma_config() enter:\n");
-	ia_css_r_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->r_gamma_table = *config;
-	params->config_changed[IA_CSS_R_GAMMA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_R_GAMMA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_r_gamma_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_g_gamma_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_rgb_gamma_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_g_gamma_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->g_gamma_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_g_gamma_config() leave\n");
-	ia_css_g_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_g_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_g_gamma_config() enter:\n");
-	ia_css_g_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->g_gamma_table = *config;
-	params->config_changed[IA_CSS_G_GAMMA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_G_GAMMA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_g_gamma_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_b_gamma_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_rgb_gamma_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_b_gamma_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->b_gamma_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_b_gamma_config() leave\n");
-	ia_css_b_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_b_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_b_gamma_config() enter:\n");
-	ia_css_b_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->b_gamma_table = *config;
-	params->config_changed[IA_CSS_B_GAMMA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_B_GAMMA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_b_gamma_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_xnr_table_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_xnr_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_table_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->xnr_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_table_config() leave\n");
-	ia_css_xnr_table_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_table_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_xnr_table_config() enter:\n");
-	ia_css_xnr_table_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->xnr_table = *config;
-	params->config_changed[IA_CSS_XNR_TABLE_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_XNR_TABLE_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_xnr_table_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_formats_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_formats_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_formats_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->formats_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_formats_config() leave\n");
-	ia_css_formats_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_formats_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_formats_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_formats_config() enter:\n");
-	ia_css_formats_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->formats_config = *config;
-	params->config_changed[IA_CSS_FORMATS_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_FORMATS_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_formats_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_xnr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_xnr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->xnr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_config() leave\n");
-	ia_css_xnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_xnr_config() enter:\n");
-	ia_css_xnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->xnr_config = *config;
-	params->config_changed[IA_CSS_XNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_XNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_xnr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_xnr3_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_xnr3_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr3_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->xnr3_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr3_config() leave\n");
-	ia_css_xnr3_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr3_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr3_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_xnr3_config() enter:\n");
-	ia_css_xnr3_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->xnr3_config = *config;
-	params->config_changed[IA_CSS_XNR3_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_XNR3_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_xnr3_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_s3a_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_3a_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_s3a_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->s3a_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_s3a_config() leave\n");
-	ia_css_s3a_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_s3a_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_3a_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_s3a_config() enter:\n");
-	ia_css_s3a_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->s3a_config = *config;
-	params->config_changed[IA_CSS_BH_ID] = true;
-	params->config_changed[IA_CSS_S3A_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_S3A_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_s3a_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_output_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_output_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_output_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->output_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_output_config() leave\n");
-	ia_css_output_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_output_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_output_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_output_config() enter:\n");
-	ia_css_output_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->output_config = *config;
-	params->config_changed[IA_CSS_OUTPUT_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_OUTPUT_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_output_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_get_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-{
-	ia_css_get_dp_config(params, config->dp_config);
-	ia_css_get_wb_config(params, config->wb_config);
-	ia_css_get_tnr_config(params, config->tnr_config);
-	ia_css_get_ob_config(params, config->ob_config);
-	ia_css_get_de_config(params, config->de_config);
-	ia_css_get_anr_config(params, config->anr_config);
-	ia_css_get_anr2_config(params, config->anr_thres);
-	ia_css_get_ce_config(params, config->ce_config);
-	ia_css_get_ecd_config(params, config->ecd_config);
-	ia_css_get_ynr_config(params, config->ynr_config);
-	ia_css_get_fc_config(params, config->fc_config);
-	ia_css_get_cnr_config(params, config->cnr_config);
-	ia_css_get_macc_config(params, config->macc_config);
-	ia_css_get_ctc_config(params, config->ctc_config);
-	ia_css_get_aa_config(params, config->aa_config);
-	ia_css_get_yuv2rgb_config(params, config->yuv2rgb_cc_config);
-	ia_css_get_rgb2yuv_config(params, config->rgb2yuv_cc_config);
-	ia_css_get_csc_config(params, config->cc_config);
-	ia_css_get_nr_config(params, config->nr_config);
-	ia_css_get_gc_config(params, config->gc_config);
-	ia_css_get_sdis_horicoef_config(params, config->dvs_coefs);
-	ia_css_get_sdis_vertcoef_config(params, config->dvs_coefs);
-	ia_css_get_sdis_horiproj_config(params, config->dvs_coefs);
-	ia_css_get_sdis_vertproj_config(params, config->dvs_coefs);
-	ia_css_get_sdis2_horicoef_config(params, config->dvs2_coefs);
-	ia_css_get_sdis2_vertcoef_config(params, config->dvs2_coefs);
-	ia_css_get_sdis2_horiproj_config(params, config->dvs2_coefs);
-	ia_css_get_sdis2_vertproj_config(params, config->dvs2_coefs);
-	ia_css_get_r_gamma_config(params, config->r_gamma_table);
-	ia_css_get_g_gamma_config(params, config->g_gamma_table);
-	ia_css_get_b_gamma_config(params, config->b_gamma_table);
-	ia_css_get_xnr_table_config(params, config->xnr_table);
-	ia_css_get_formats_config(params, config->formats_config);
-	ia_css_get_xnr_config(params, config->xnr_config);
-	ia_css_get_xnr3_config(params, config->xnr3_config);
-	ia_css_get_s3a_config(params, config->s3a_config);
-	ia_css_get_output_config(params, config->output_config);
-}
-
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_set_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-{
-	ia_css_set_dp_config(params, config->dp_config);
-	ia_css_set_wb_config(params, config->wb_config);
-	ia_css_set_tnr_config(params, config->tnr_config);
-	ia_css_set_ob_config(params, config->ob_config);
-	ia_css_set_de_config(params, config->de_config);
-	ia_css_set_anr_config(params, config->anr_config);
-	ia_css_set_anr2_config(params, config->anr_thres);
-	ia_css_set_ce_config(params, config->ce_config);
-	ia_css_set_ecd_config(params, config->ecd_config);
-	ia_css_set_ynr_config(params, config->ynr_config);
-	ia_css_set_fc_config(params, config->fc_config);
-	ia_css_set_cnr_config(params, config->cnr_config);
-	ia_css_set_macc_config(params, config->macc_config);
-	ia_css_set_ctc_config(params, config->ctc_config);
-	ia_css_set_aa_config(params, config->aa_config);
-	ia_css_set_yuv2rgb_config(params, config->yuv2rgb_cc_config);
-	ia_css_set_rgb2yuv_config(params, config->rgb2yuv_cc_config);
-	ia_css_set_csc_config(params, config->cc_config);
-	ia_css_set_nr_config(params, config->nr_config);
-	ia_css_set_gc_config(params, config->gc_config);
-	ia_css_set_sdis_horicoef_config(params, config->dvs_coefs);
-	ia_css_set_sdis_vertcoef_config(params, config->dvs_coefs);
-	ia_css_set_sdis_horiproj_config(params, config->dvs_coefs);
-	ia_css_set_sdis_vertproj_config(params, config->dvs_coefs);
-	ia_css_set_sdis2_horicoef_config(params, config->dvs2_coefs);
-	ia_css_set_sdis2_vertcoef_config(params, config->dvs2_coefs);
-	ia_css_set_sdis2_horiproj_config(params, config->dvs2_coefs);
-	ia_css_set_sdis2_vertproj_config(params, config->dvs2_coefs);
-	ia_css_set_r_gamma_config(params, config->r_gamma_table);
-	ia_css_set_g_gamma_config(params, config->g_gamma_table);
-	ia_css_set_b_gamma_config(params, config->b_gamma_table);
-	ia_css_set_xnr_table_config(params, config->xnr_table);
-	ia_css_set_formats_config(params, config->formats_config);
-	ia_css_set_xnr_config(params, config->xnr_config);
-	ia_css_set_xnr3_config(params, config->xnr3_config);
-	ia_css_set_s3a_config(params, config->s3a_config);
-	ia_css_set_output_config(params, config->output_config);
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_params.h
deleted file mode 100644
index 5b3deb7f74ae..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_params.h
+++ /dev/null
@@ -1,399 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Generated code: do not edit or commmit. */
-
-#ifndef _IA_CSS_ISP_PARAM_H
-#define _IA_CSS_ISP_PARAM_H
-
-/* Code generated by genparam/gencode.c:gen_param_enum() */
-
-enum ia_css_parameter_ids {
-	IA_CSS_AA_ID,
-	IA_CSS_ANR_ID,
-	IA_CSS_ANR2_ID,
-	IA_CSS_BH_ID,
-	IA_CSS_CNR_ID,
-	IA_CSS_CROP_ID,
-	IA_CSS_CSC_ID,
-	IA_CSS_DP_ID,
-	IA_CSS_BNR_ID,
-	IA_CSS_DE_ID,
-	IA_CSS_ECD_ID,
-	IA_CSS_FORMATS_ID,
-	IA_CSS_FPN_ID,
-	IA_CSS_GC_ID,
-	IA_CSS_CE_ID,
-	IA_CSS_YUV2RGB_ID,
-	IA_CSS_RGB2YUV_ID,
-	IA_CSS_R_GAMMA_ID,
-	IA_CSS_G_GAMMA_ID,
-	IA_CSS_B_GAMMA_ID,
-	IA_CSS_UDS_ID,
-	IA_CSS_RAA_ID,
-	IA_CSS_S3A_ID,
-	IA_CSS_OB_ID,
-	IA_CSS_OUTPUT_ID,
-	IA_CSS_SC_ID,
-	IA_CSS_BDS_ID,
-	IA_CSS_TNR_ID,
-	IA_CSS_MACC_ID,
-	IA_CSS_SDIS_HORICOEF_ID,
-	IA_CSS_SDIS_VERTCOEF_ID,
-	IA_CSS_SDIS_HORIPROJ_ID,
-	IA_CSS_SDIS_VERTPROJ_ID,
-	IA_CSS_SDIS2_HORICOEF_ID,
-	IA_CSS_SDIS2_VERTCOEF_ID,
-	IA_CSS_SDIS2_HORIPROJ_ID,
-	IA_CSS_SDIS2_VERTPROJ_ID,
-	IA_CSS_WB_ID,
-	IA_CSS_NR_ID,
-	IA_CSS_YEE_ID,
-	IA_CSS_YNR_ID,
-	IA_CSS_FC_ID,
-	IA_CSS_CTC_ID,
-	IA_CSS_XNR_TABLE_ID,
-	IA_CSS_XNR_ID,
-	IA_CSS_XNR3_ID,
-	IA_CSS_NUM_PARAMETER_IDS
-};
-
-/* Code generated by genparam/gencode.c:gen_param_offsets() */
-
-struct ia_css_memory_offsets {
-	struct {
-		struct ia_css_isp_parameter aa;
-		struct ia_css_isp_parameter anr;
-		struct ia_css_isp_parameter bh;
-		struct ia_css_isp_parameter cnr;
-		struct ia_css_isp_parameter crop;
-		struct ia_css_isp_parameter csc;
-		struct ia_css_isp_parameter dp;
-		struct ia_css_isp_parameter bnr;
-		struct ia_css_isp_parameter de;
-		struct ia_css_isp_parameter ecd;
-		struct ia_css_isp_parameter formats;
-		struct ia_css_isp_parameter fpn;
-		struct ia_css_isp_parameter gc;
-		struct ia_css_isp_parameter ce;
-		struct ia_css_isp_parameter yuv2rgb;
-		struct ia_css_isp_parameter rgb2yuv;
-		struct ia_css_isp_parameter uds;
-		struct ia_css_isp_parameter raa;
-		struct ia_css_isp_parameter s3a;
-		struct ia_css_isp_parameter ob;
-		struct ia_css_isp_parameter output;
-		struct ia_css_isp_parameter sc;
-		struct ia_css_isp_parameter bds;
-		struct ia_css_isp_parameter tnr;
-		struct ia_css_isp_parameter macc;
-		struct ia_css_isp_parameter sdis_horiproj;
-		struct ia_css_isp_parameter sdis_vertproj;
-		struct ia_css_isp_parameter sdis2_horiproj;
-		struct ia_css_isp_parameter sdis2_vertproj;
-		struct ia_css_isp_parameter wb;
-		struct ia_css_isp_parameter nr;
-		struct ia_css_isp_parameter yee;
-		struct ia_css_isp_parameter ynr;
-		struct ia_css_isp_parameter fc;
-		struct ia_css_isp_parameter ctc;
-		struct ia_css_isp_parameter xnr;
-		struct ia_css_isp_parameter xnr3;
-		struct ia_css_isp_parameter get;
-		struct ia_css_isp_parameter put;
-	} dmem;
-	struct {
-		struct ia_css_isp_parameter anr2;
-		struct ia_css_isp_parameter ob;
-		struct ia_css_isp_parameter sdis_horicoef;
-		struct ia_css_isp_parameter sdis_vertcoef;
-		struct ia_css_isp_parameter sdis2_horicoef;
-		struct ia_css_isp_parameter sdis2_vertcoef;
-#ifdef ISP2401
-		struct ia_css_isp_parameter xnr3;
-#endif
-	} vmem;
-	struct {
-		struct ia_css_isp_parameter bh;
-	} hmem0;
-	struct {
-		struct ia_css_isp_parameter gc;
-		struct ia_css_isp_parameter g_gamma;
-		struct ia_css_isp_parameter xnr_table;
-	} vamem1;
-	struct {
-		struct ia_css_isp_parameter r_gamma;
-		struct ia_css_isp_parameter ctc;
-	} vamem0;
-	struct {
-		struct ia_css_isp_parameter b_gamma;
-	} vamem2;
-};
-
-#if defined(IA_CSS_INCLUDE_PARAMETERS)
-
-#include "ia_css_stream.h"   /* struct ia_css_stream */
-#include "ia_css_binary.h"   /* struct ia_css_binary */
-/* Code generated by genparam/gencode.c:gen_param_process_table() */
-
-struct ia_css_pipeline_stage; /* forward declaration */
-
-extern void (* ia_css_kernel_process_param[IA_CSS_NUM_PARAMETER_IDS])(
-			unsigned pipe_id,
-			const struct ia_css_pipeline_stage *stage,
-			struct ia_css_isp_parameters *params);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_dp_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dp_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_wb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_wb_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_tnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_tnr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ob_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ob_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_de_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_de_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr2_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_thres *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ce_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ce_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ecd_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ecd_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ynr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ynr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_fc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_fc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_cnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cnr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_macc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_macc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ctc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ctc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_aa_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_aa_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_yuv2rgb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_rgb2yuv_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_csc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_nr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_nr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_gc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_gc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_r_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_g_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_b_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_table_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_formats_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_formats_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr3_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr3_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_s3a_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_3a_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_output_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_output_config *config);
-
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_get_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-;
-#ifdef ISP2401
-
-#endif
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_set_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-;
-#ifdef ISP2401
-
-#endif
-#endif /* IA_CSS_INCLUDE_PARAMETER */
-
-#endif /* _IA_CSS_ISP_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_states.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_states.c
deleted file mode 100644
index fb3ba08f69c1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_states.c
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-/* Generated code: do not edit or commmit. */
-
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_states.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_aa_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_aa_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.aa.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.aa.offset;
-
-		if (size)
-			memset(&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset], 0, size);
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_aa_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_cnr_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.cnr.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.cnr.offset;
-
-		if (size) {
-			ia_css_init_cnr_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_cnr2_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr2_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.cnr2.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.cnr2.offset;
-
-		if (size) {
-			ia_css_init_cnr2_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr2_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_dp_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_dp_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.dp.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.dp.offset;
-
-		if (size) {
-			ia_css_init_dp_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_dp_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_de_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_de_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.de.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.de.offset;
-
-		if (size) {
-			ia_css_init_de_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_de_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_tnr_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_tnr_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->dmem.tnr.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->dmem.tnr.offset;
-
-		if (size) {
-			ia_css_init_tnr_state((struct sh_css_isp_tnr_dmem_state *)
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_DMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_tnr_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_ref_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ref_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->dmem.ref.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->dmem.ref.offset;
-
-		if (size) {
-			ia_css_init_ref_state((struct sh_css_isp_ref_dmem_state *)
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_DMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ref_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_ynr_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ynr_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.ynr.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.ynr.offset;
-
-		if (size) {
-			ia_css_init_ynr_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ynr_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_state_init_table() */
-
-void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(const struct ia_css_binary *binary) = {
-	ia_css_initialize_aa_state,
-	ia_css_initialize_cnr_state,
-	ia_css_initialize_cnr2_state,
-	ia_css_initialize_dp_state,
-	ia_css_initialize_de_state,
-	ia_css_initialize_tnr_state,
-	ia_css_initialize_ref_state,
-	ia_css_initialize_ynr_state,
-};
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_states.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_states.h
deleted file mode 100644
index 732adafb0a63..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hive_isp_css_2400_system_generated/ia_css_isp_states.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#define IA_CSS_INCLUDE_STATES
-#include "isp/kernels/aa/aa_2/ia_css_aa2.host.h"
-#include "isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h"
-#include "isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h"
-#include "isp/kernels/de/de_1.0/ia_css_de.host.h"
-#include "isp/kernels/dp/dp_1.0/ia_css_dp.host.h"
-#include "isp/kernels/ref/ref_1.0/ia_css_ref.host.h"
-#include "isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h"
-#include "isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h"
-#include "isp/kernels/dpc2/ia_css_dpc2.host.h"
-#include "isp/kernels/eed1_8/ia_css_eed1_8.host.h"
-/* Generated code: do not edit or commmit. */
-
-#ifndef _IA_CSS_ISP_STATE_H
-#define _IA_CSS_ISP_STATE_H
-
-/* Code generated by genparam/gencode.c:gen_param_enum() */
-
-enum ia_css_state_ids {
-	IA_CSS_AA_STATE_ID,
-	IA_CSS_CNR_STATE_ID,
-	IA_CSS_CNR2_STATE_ID,
-	IA_CSS_DP_STATE_ID,
-	IA_CSS_DE_STATE_ID,
-	IA_CSS_TNR_STATE_ID,
-	IA_CSS_REF_STATE_ID,
-	IA_CSS_YNR_STATE_ID,
-	IA_CSS_NUM_STATE_IDS
-};
-
-/* Code generated by genparam/gencode.c:gen_param_offsets() */
-
-struct ia_css_state_memory_offsets {
-	struct {
-		struct ia_css_isp_parameter aa;
-		struct ia_css_isp_parameter cnr;
-		struct ia_css_isp_parameter cnr2;
-		struct ia_css_isp_parameter dp;
-		struct ia_css_isp_parameter de;
-		struct ia_css_isp_parameter ynr;
-	} vmem;
-	struct {
-		struct ia_css_isp_parameter tnr;
-		struct ia_css_isp_parameter ref;
-	} dmem;
-};
-
-#if defined(IA_CSS_INCLUDE_STATES)
-
-#include "ia_css_stream.h"   /* struct ia_css_stream */
-#include "ia_css_binary.h"   /* struct ia_css_binary */
-/* Code generated by genparam/genstate.c:gen_state_init_table() */
-
-extern void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(const struct ia_css_binary *binary);
-
-#endif /* IA_CSS_INCLUDE_STATE */
-
-#endif /* _IA_CSS_ISP_STATE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/bits.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/bits.h
deleted file mode 100644
index e71e33d9d143..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/bits.h
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_BITS_H
-#define _HRT_BITS_H
-
-#include "defs.h"
-
-#define _hrt_ones(n) HRTCAT(_hrt_ones_, n)
-#define _hrt_ones_0x0  0x00000000U
-#define _hrt_ones_0x1  0x00000001U
-#define _hrt_ones_0x2  0x00000003U
-#define _hrt_ones_0x3  0x00000007U
-#define _hrt_ones_0x4  0x0000000FU
-#define _hrt_ones_0x5  0x0000001FU
-#define _hrt_ones_0x6  0x0000003FU
-#define _hrt_ones_0x7  0x0000007FU
-#define _hrt_ones_0x8  0x000000FFU
-#define _hrt_ones_0x9  0x000001FFU
-#define _hrt_ones_0xA  0x000003FFU
-#define _hrt_ones_0xB  0x000007FFU
-#define _hrt_ones_0xC  0x00000FFFU
-#define _hrt_ones_0xD  0x00001FFFU
-#define _hrt_ones_0xE  0x00003FFFU
-#define _hrt_ones_0xF  0x00007FFFU
-#define _hrt_ones_0x10 0x0000FFFFU
-#define _hrt_ones_0x11 0x0001FFFFU
-#define _hrt_ones_0x12 0x0003FFFFU
-#define _hrt_ones_0x13 0x0007FFFFU
-#define _hrt_ones_0x14 0x000FFFFFU
-#define _hrt_ones_0x15 0x001FFFFFU
-#define _hrt_ones_0x16 0x003FFFFFU
-#define _hrt_ones_0x17 0x007FFFFFU
-#define _hrt_ones_0x18 0x00FFFFFFU
-#define _hrt_ones_0x19 0x01FFFFFFU
-#define _hrt_ones_0x1A 0x03FFFFFFU
-#define _hrt_ones_0x1B 0x07FFFFFFU
-#define _hrt_ones_0x1C 0x0FFFFFFFU
-#define _hrt_ones_0x1D 0x1FFFFFFFU
-#define _hrt_ones_0x1E 0x3FFFFFFFU
-#define _hrt_ones_0x1F 0x7FFFFFFFU
-#define _hrt_ones_0x20 0xFFFFFFFFU
-
-#define _hrt_ones_0  _hrt_ones_0x0
-#define _hrt_ones_1  _hrt_ones_0x1
-#define _hrt_ones_2  _hrt_ones_0x2
-#define _hrt_ones_3  _hrt_ones_0x3
-#define _hrt_ones_4  _hrt_ones_0x4
-#define _hrt_ones_5  _hrt_ones_0x5
-#define _hrt_ones_6  _hrt_ones_0x6
-#define _hrt_ones_7  _hrt_ones_0x7
-#define _hrt_ones_8  _hrt_ones_0x8
-#define _hrt_ones_9  _hrt_ones_0x9
-#define _hrt_ones_10 _hrt_ones_0xA
-#define _hrt_ones_11 _hrt_ones_0xB
-#define _hrt_ones_12 _hrt_ones_0xC
-#define _hrt_ones_13 _hrt_ones_0xD
-#define _hrt_ones_14 _hrt_ones_0xE
-#define _hrt_ones_15 _hrt_ones_0xF
-#define _hrt_ones_16 _hrt_ones_0x10
-#define _hrt_ones_17 _hrt_ones_0x11
-#define _hrt_ones_18 _hrt_ones_0x12
-#define _hrt_ones_19 _hrt_ones_0x13
-#define _hrt_ones_20 _hrt_ones_0x14
-#define _hrt_ones_21 _hrt_ones_0x15
-#define _hrt_ones_22 _hrt_ones_0x16
-#define _hrt_ones_23 _hrt_ones_0x17
-#define _hrt_ones_24 _hrt_ones_0x18
-#define _hrt_ones_25 _hrt_ones_0x19
-#define _hrt_ones_26 _hrt_ones_0x1A
-#define _hrt_ones_27 _hrt_ones_0x1B
-#define _hrt_ones_28 _hrt_ones_0x1C
-#define _hrt_ones_29 _hrt_ones_0x1D
-#define _hrt_ones_30 _hrt_ones_0x1E
-#define _hrt_ones_31 _hrt_ones_0x1F
-#define _hrt_ones_32 _hrt_ones_0x20
-
-#define _hrt_mask(b, n) \
-  (_hrt_ones(n) << (b))
-#define _hrt_get_bits(w, b, n) \
-  (((w) >> (b)) & _hrt_ones(n))
-#define _hrt_set_bits(w, b, n, v) \
-  (((w) & ~_hrt_mask(b, n)) | (((v) & _hrt_ones(n)) << (b)))
-#define _hrt_get_bit(w, b) \
-  (((w) >> (b)) & 1)
-#define _hrt_set_bit(w, b, v) \
-  (((w) & (~(1 << (b)))) | (((v)&1) << (b)))
-#define _hrt_set_lower_half(w, v) \
-  _hrt_set_bits(w, 0, 16, v)
-#define _hrt_set_upper_half(w, v) \
-  _hrt_set_bits(w, 16, 16, v)
-
-#endif /* _HRT_BITS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/cell_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/cell_params.h
deleted file mode 100644
index b5756bfe8eb6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/cell_params.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _cell_params_h
-#define _cell_params_h
-
-#define SP_PMEM_LOG_WIDTH_BITS           6  /*Width of PC, 64 bits, 8 bytes*/
-#define SP_ICACHE_TAG_BITS               4  /*size of tag*/
-#define SP_ICACHE_SET_BITS               8  /* 256 sets*/
-#define SP_ICACHE_BLOCKS_PER_SET_BITS    1  /* 2 way associative*/
-#define SP_ICACHE_BLOCK_ADDRESS_BITS     11 /* 2048 lines capacity*/
-
-#define SP_ICACHE_ADDRESS_BITS \
-	                    (SP_ICACHE_TAG_BITS+SP_ICACHE_BLOCK_ADDRESS_BITS)
-
-#define SP_PMEM_DEPTH        (1<<SP_ICACHE_ADDRESS_BITS)
-
-#define SP_FIFO_0_DEPTH      0
-#define SP_FIFO_1_DEPTH      0
-#define SP_FIFO_2_DEPTH      0
-#define SP_FIFO_3_DEPTH      0
-#define SP_FIFO_4_DEPTH      0
-#define SP_FIFO_5_DEPTH      0
-#define SP_FIFO_6_DEPTH      0
-#define SP_FIFO_7_DEPTH      0
-
-
-#define SP_SLV_BUS_MAXBURSTSIZE        1
-
-#endif /* _cell_params_h */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/css_receiver_2400_common_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/css_receiver_2400_common_defs.h
deleted file mode 100644
index f3054fe04d03..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/css_receiver_2400_common_defs.h
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _css_receiver_2400_common_defs_h_
-#define _css_receiver_2400_common_defs_h_
-#ifndef _mipi_backend_common_defs_h_
-#define _mipi_backend_common_defs_h_
-
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_DATA_WIDTH     16
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_CH_ID_WIDTH     2
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_FMT_TYPE_WIDTH  3
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_REAL_WIDTH (_HRT_CSS_RECEIVER_2400_GEN_SHORT_DATA_WIDTH + _HRT_CSS_RECEIVER_2400_GEN_SHORT_CH_ID_WIDTH + _HRT_CSS_RECEIVER_2400_GEN_SHORT_FMT_TYPE_WIDTH)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_WIDTH      32 /* use 32 to be compatibel with streaming monitor !, MSB's of interface are tied to '0' */ 
-
-/* Definition of data format ID at the interface CSS_receiver capture/acquisition units */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8          24   /* 01 1000 YUV420 8-bit                                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_10         25   /* 01 1001  YUV420 10-bit                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8L         26   /* 01 1010   YUV420 8-bit legacy                               */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV422_8          30   /* 01 1110   YUV422 8-bit                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV422_10         31   /* 01 1111   YUV422 10-bit                                     */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB444            32   /* 10 0000   RGB444                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB555            33   /* 10 0001   RGB555                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB565            34   /* 10 0010   RGB565                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB666            35   /* 10 0011   RGB666                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB888            36   /* 10 0100   RGB888                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW6              40   /* 10 1000   RAW6                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW7              41   /* 10 1001   RAW7                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW8              42   /* 10 1010   RAW8                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW10             43   /* 10 1011   RAW10                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW12             44   /* 10 1100   RAW12                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW14             45   /* 10 1101   RAW14                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_1         48   /* 11 0000    JPEG [User Defined 8-bit Data Type 1]            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_2         49   /* 11 0001    User Defined 8-bit Data Type 2                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_3         50   /* 11 0010    User Defined 8-bit Data Type 3                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_4         51   /* 11 0011    User Defined 8-bit Data Type 4                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_5         52   /* 11 0100    User Defined 8-bit Data Type 5                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_6         53   /* 11 0101    User Defined 8-bit Data Type 6                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_7         54   /* 11 0110    User Defined 8-bit Data Type 7                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_8         55   /* 11 0111    User Defined 8-bit Data Type 8                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_Emb               18   /* 01 0010    embedded eight bit non image data                */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_SOF                0   /* 00 0000    frame start                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_EOF                1   /* 00 0001    frame end                                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_SOL                2   /* 00 0010    line start                                       */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_EOL                3   /* 00 0011    line end                                         */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH1            8   /* 00 1000  Generic Short Packet Code 1                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH2            9   /* 00 1001    Generic Short Packet Code 2                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH3           10   /* 00 1010    Generic Short Packet Code 3                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH4           11   /* 00 1011    Generic Short Packet Code 4                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH5           12   /* 00 1100    Generic Short Packet Code 5                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH6           13   /* 00 1101    Generic Short Packet Code 6                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH7           14   /* 00 1110    Generic Short Packet Code 7                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH8           15   /* 00 1111    Generic Short Packet Code 8                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8_CSPS     28   /* 01 1100   YUV420 8-bit (Chroma Shifted Pixel Sampling)      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_10_CSPS    29   /* 01 1101   YUV420 10-bit (Chroma Shifted Pixel Sampling)     */
-/* used reseved mipi positions for these */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW16             46 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18             47 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18_2           37 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18_3           38 
-
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_WIDTH              6
-
-/* Definition of format_types at the interface CSS --> input_selector*/
-/* !! Changes here should be copied to systems/isp/isp_css/bin/conv_transmitter_cmd.tcl !! */
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB888           0  // 36 'h24
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB555           1  // 33 'h
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB444           2  // 32
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB565           3  // 34
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB666           4  // 35
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW8             5  // 42 
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW10            6  // 43
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW6             7  // 40
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW7             8  // 41
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW12            9  // 43
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW14           10  // 45
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8        11  // 30
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_10       12  // 25
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV422_8        13  // 30
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV422_10       14  // 31
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_1       15  // 48
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8L       16  // 26
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_Emb             17  // 18
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_2       18  // 49
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_3       19  // 50
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_4       20  // 51
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_5       21  // 52
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_6       22  // 53
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_7       23  // 54
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_8       24  // 55
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8_CSPS   25  // 28
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_10_CSPS  26  // 29
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW16           27  // ?
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18           28  // ?
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18_2         29  // ? Option 2 for depacketiser
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18_3         30  // ? Option 3 for depacketiser
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_CUSTOM          31  // to signal custom decoding 
-
-/* definition for state machine of data FIFO for decode different type of data */
-#define _HRT_CSS_RECEIVER_2400_YUV420_8_REPEAT_PTN                 1  
-#define _HRT_CSS_RECEIVER_2400_YUV420_10_REPEAT_PTN                5
-#define _HRT_CSS_RECEIVER_2400_YUV420_8L_REPEAT_PTN                1
-#define _HRT_CSS_RECEIVER_2400_YUV422_8_REPEAT_PTN                 1
-#define _HRT_CSS_RECEIVER_2400_YUV422_10_REPEAT_PTN                5
-#define _HRT_CSS_RECEIVER_2400_RGB444_REPEAT_PTN                   2 
-#define _HRT_CSS_RECEIVER_2400_RGB555_REPEAT_PTN                   2
-#define _HRT_CSS_RECEIVER_2400_RGB565_REPEAT_PTN                   2
-#define _HRT_CSS_RECEIVER_2400_RGB666_REPEAT_PTN                   9                       
-#define _HRT_CSS_RECEIVER_2400_RGB888_REPEAT_PTN                   3
-#define _HRT_CSS_RECEIVER_2400_RAW6_REPEAT_PTN                     3
-#define _HRT_CSS_RECEIVER_2400_RAW7_REPEAT_PTN                     7
-#define _HRT_CSS_RECEIVER_2400_RAW8_REPEAT_PTN                     1
-#define _HRT_CSS_RECEIVER_2400_RAW10_REPEAT_PTN                    5
-#define _HRT_CSS_RECEIVER_2400_RAW12_REPEAT_PTN                    3        
-#define _HRT_CSS_RECEIVER_2400_RAW14_REPEAT_PTN                    7
-
-#define _HRT_CSS_RECEIVER_2400_MAX_REPEAT_PTN                      _HRT_CSS_RECEIVER_2400_RGB666_REPEAT_PTN
-
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FMT_IDX                     0
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FMT_WIDTH                   3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_PRED_IDX                    3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_PRED_WIDTH                  1
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_USD_BITS                    4  /* bits per USD type */
-
-#define _HRT_CSS_RECEIVER_2400_BE_RAW16_DATAID_IDX                 0
-#define _HRT_CSS_RECEIVER_2400_BE_RAW16_EN_IDX                     6
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_DATAID_IDX                 0
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_OPTION_IDX                 6
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_EN_IDX                     8
-
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_NO_COMP                     0
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_6_10                     1
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_7_10                     2
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_8_10                     3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_6_12                     4
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_7_12                     5
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_8_12                     6
-
-
-/* packet bit definition */
-#define _HRT_CSS_RECEIVER_2400_PKT_SOP_IDX                        32
-#define _HRT_CSS_RECEIVER_2400_PKT_SOP_BITS                        1
-#define _HRT_CSS_RECEIVER_2400_PKT_CH_ID_IDX                      22
-#define _HRT_CSS_RECEIVER_2400_PKT_CH_ID_BITS                      2
-#define _HRT_CSS_RECEIVER_2400_PKT_FMT_ID_IDX                     16
-#define _HRT_CSS_RECEIVER_2400_PKT_FMT_ID_BITS                     6
-#define _HRT_CSS_RECEIVER_2400_PH_DATA_FIELD_IDX                   0
-#define _HRT_CSS_RECEIVER_2400_PH_DATA_FIELD_BITS                 16
-#define _HRT_CSS_RECEIVER_2400_PKT_PAYLOAD_IDX                     0
-#define _HRT_CSS_RECEIVER_2400_PKT_PAYLOAD_BITS                   32
-
-
-/*************************************************************************************************/
-/* Custom Decoding                                                                               */
-/* These Custom Defs are defined based on design-time config in "csi_be_pixel_formatter.chdl" !! */
-/*************************************************************************************************/
-#define BE_CUST_EN_IDX                     0     /* 2bits */
-#define BE_CUST_EN_DATAID_IDX              2     /* 6bits MIPI DATA ID */ 
-#define BE_CUST_EN_WIDTH                   8     
-#define BE_CUST_MODE_ALL                   1     /* Enable Custom Decoding for all DATA IDs */
-#define BE_CUST_MODE_ONE                   3     /* Enable Custom Decoding for ONE DATA ID, programmed in CUST_EN_DATA_ID */
-
-/* Data State config = {get_bits(6bits), valid(1bit)}  */
-#define BE_CUST_DATA_STATE_S0_IDX          0     /* 7bits */ 
-#define BE_CUST_DATA_STATE_S1_IDX          7     /* 7bits */ 
-#define BE_CUST_DATA_STATE_S2_IDX          14    /* 7bits */
-#define BE_CUST_DATA_STATE_WIDTH           21    
-#define BE_CUST_DATA_STATE_VALID_IDX       0     /* 1bits */
-#define BE_CUST_DATA_STATE_GETBITS_IDX     1     /* 6bits */
-
-/* Pixel Extractor config */
-#define BE_CUST_PIX_EXT_DATA_ALIGN_IDX     0     /* 5bits */
-#define BE_CUST_PIX_EXT_PIX_ALIGN_IDX      5     /* 5bits */
-#define BE_CUST_PIX_EXT_PIX_MASK_IDX       10    /* 18bits */
-#define BE_CUST_PIX_EXT_PIX_EN_IDX         28    /* 1bits */
-#define BE_CUST_PIX_EXT_WIDTH              29    
-
-/* Pixel Valid & EoP config = {[eop,valid](especial), [eop,valid](normal)} */
-#define BE_CUST_PIX_VALID_EOP_P0_IDX        0    /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_P1_IDX        4    /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_P2_IDX        8    /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_P3_IDX        12   /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_WIDTH         16 
-#define BE_CUST_PIX_VALID_EOP_NOR_VALID_IDX 0    /* Normal (NO less get_bits case) Valid - 1bits */
-#define BE_CUST_PIX_VALID_EOP_NOR_EOP_IDX   1    /* Normal (NO less get_bits case) EoP - 1bits */
-#define BE_CUST_PIX_VALID_EOP_ESP_VALID_IDX 2    /* Especial (less get_bits case) Valid - 1bits */
-#define BE_CUST_PIX_VALID_EOP_ESP_EOP_IDX   3    /* Especial (less get_bits case) EoP - 1bits */
-
-#endif /* _mipi_backend_common_defs_h_ */
-#endif /* _css_receiver_2400_common_defs_h_ */ 
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/css_receiver_2400_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/css_receiver_2400_defs.h
deleted file mode 100644
index 6f5b7d3d3715..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/css_receiver_2400_defs.h
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _css_receiver_2400_defs_h_
-#define _css_receiver_2400_defs_h_
-
-#include "css_receiver_2400_common_defs.h"
-
-#define CSS_RECEIVER_DATA_WIDTH                8
-#define CSS_RECEIVER_RX_TRIG                   4
-#define CSS_RECEIVER_RF_WORD                  32
-#define CSS_RECEIVER_IMG_PROC_RF_ADDR         10
-#define CSS_RECEIVER_CSI_RF_ADDR               4
-#define CSS_RECEIVER_DATA_OUT                 12
-#define CSS_RECEIVER_CHN_NO                    2
-#define CSS_RECEIVER_DWORD_CNT                11
-#define CSS_RECEIVER_FORMAT_TYP                5
-#define CSS_RECEIVER_HRESPONSE                 2
-#define CSS_RECEIVER_STATE_WIDTH               3
-#define CSS_RECEIVER_FIFO_DAT                 32
-#define CSS_RECEIVER_CNT_VAL                   2
-#define CSS_RECEIVER_PRED10_VAL               10
-#define CSS_RECEIVER_PRED12_VAL               12
-#define CSS_RECEIVER_CNT_WIDTH                 8
-#define CSS_RECEIVER_WORD_CNT                 16
-#define CSS_RECEIVER_PIXEL_LEN                 6
-#define CSS_RECEIVER_PIXEL_CNT                 5
-#define CSS_RECEIVER_COMP_8_BIT                8
-#define CSS_RECEIVER_COMP_7_BIT                7
-#define CSS_RECEIVER_COMP_6_BIT                6
-
-#define CSI_CONFIG_WIDTH                       4
-
-/* division of gen_short data, ch_id and fmt_type over streaming data interface */
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_DATA_BIT_LSB     0
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_LSB (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_DATA_BIT_LSB     + _HRT_CSS_RECEIVER_2400_GEN_SHORT_DATA_WIDTH)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_CH_ID_BIT_LSB    (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_LSB + _HRT_CSS_RECEIVER_2400_GEN_SHORT_FMT_TYPE_WIDTH)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_DATA_BIT_MSB     (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_LSB - 1)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_MSB (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_CH_ID_BIT_LSB    - 1)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_CH_ID_BIT_MSB    (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_REAL_WIDTH       - 1)
-
-#define _HRT_CSS_RECEIVER_2400_REG_ALIGN 4
-#define _HRT_CSS_RECEIVER_2400_BYTES_PER_PKT             4
-
-#define hrt_css_receiver_2400_4_lane_port_offset  0x100
-#define hrt_css_receiver_2400_1_lane_port_offset  0x200
-#define hrt_css_receiver_2400_2_lane_port_offset  0x300
-#define hrt_css_receiver_2400_backend_port_offset 0x100
-
-#define _HRT_CSS_RECEIVER_2400_DEVICE_READY_REG_IDX      0
-#define _HRT_CSS_RECEIVER_2400_IRQ_STATUS_REG_IDX        1
-#define _HRT_CSS_RECEIVER_2400_IRQ_ENABLE_REG_IDX        2
-#define _HRT_CSS_RECEIVER_2400_CSI2_FUNC_PROG_REG_IDX    3
-#define _HRT_CSS_RECEIVER_2400_INIT_COUNT_REG_IDX        4
-#define _HRT_CSS_RECEIVER_2400_FS_TO_LS_DELAY_REG_IDX    7
-#define _HRT_CSS_RECEIVER_2400_LS_TO_DATA_DELAY_REG_IDX  8
-#define _HRT_CSS_RECEIVER_2400_DATA_TO_LE_DELAY_REG_IDX  9
-#define _HRT_CSS_RECEIVER_2400_LE_TO_FE_DELAY_REG_IDX   10
-#define _HRT_CSS_RECEIVER_2400_FE_TO_FS_DELAY_REG_IDX   11
-#define _HRT_CSS_RECEIVER_2400_LE_TO_LS_DELAY_REG_IDX   12
-#define _HRT_CSS_RECEIVER_2400_TWO_PIXEL_EN_REG_IDX     13
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_REG_IDX  14
-#define _HRT_CSS_RECEIVER_2400_SYNC_COUNT_REG_IDX       15
-#define _HRT_CSS_RECEIVER_2400_RX_COUNT_REG_IDX         16
-#define _HRT_CSS_RECEIVER_2400_BACKEND_RST_REG_IDX      17
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG0_IDX 18
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG1_IDX 19
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG0_IDX 20
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG1_IDX 21
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG0_IDX 22
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG1_IDX 23
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG0_IDX 24
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG1_IDX 25
-#define _HRT_CSS_RECEIVER_2400_RAW18_REG_IDX            26
-#define _HRT_CSS_RECEIVER_2400_FORCE_RAW8_REG_IDX       27
-#define _HRT_CSS_RECEIVER_2400_RAW16_REG_IDX            28
-
-/* Interrupt bits for IRQ_STATUS and IRQ_ENABLE registers */
-#define _HRT_CSS_RECEIVER_2400_IRQ_OVERRUN_BIT                0
-#define _HRT_CSS_RECEIVER_2400_IRQ_RESERVED_BIT               1
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_ENTRY_BIT       2
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_EXIT_BIT        3
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_HS_BIT             4
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_SYNC_HS_BIT        5
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CONTROL_BIT            6
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_DOUBLE_BIT         7
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_CORRECTED_BIT      8
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_NO_CORRECTION_BIT  9
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CRC_BIT               10
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ID_BIT                11
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_SYNC_BIT        12
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_DATA_BIT        13
-#define _HRT_CSS_RECEIVER_2400_IRQ_DATA_TIMEOUT_BIT          14
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ESCAPE_BIT            15
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_LINE_SYNC_BIT         16
-
-#define _HRT_CSS_RECEIVER_2400_IRQ_OVERRUN_CAUSE_                  "Fifo Overrun"
-#define _HRT_CSS_RECEIVER_2400_IRQ_RESERVED_CAUSE_                 "Reserved"
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_ENTRY_CAUSE_         "Sleep mode entry"
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_EXIT_CAUSE_          "Sleep mode exit"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_HS_CAUSE_               "Error high speed SOT"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_SYNC_HS_CAUSE_          "Error high speed sync SOT"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CONTROL_CAUSE_              "Error control"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_DOUBLE_CAUSE_           "Error correction double bit"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_CORRECTED_CAUSE_        "Error correction single bit"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_NO_CORRECTION_CAUSE_    "No error"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CRC_CAUSE_                  "Error cyclic redundancy check"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ID_CAUSE_                   "Error id"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_SYNC_CAUSE_           "Error frame sync"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_DATA_CAUSE_           "Error frame data"
-#define _HRT_CSS_RECEIVER_2400_IRQ_DATA_TIMEOUT_CAUSE_             "Data time-out"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ESCAPE_CAUSE_               "Error escape"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_LINE_SYNC_CAUSE_            "Error line sync"
-
-/* Bits for CSI2_DEVICE_READY register */
-#define _HRT_CSS_RECEIVER_2400_CSI2_DEVICE_READY_IDX                          0
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_INIT_TIME_OUT_ERR_IDX                2
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_OVER_RUN_ERR_IDX                     3
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_SOT_SYNC_ERR_IDX                     4
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_RECEIVE_DATA_TIME_OUT_ERR_IDX        5
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_ECC_TWO_BIT_ERR_IDX                  6
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_DATA_ID_ERR_IDX                      7
-
-                                  
-/* Bits for CSI2_FUNC_PROG register */
-#define _HRT_CSS_RECEIVER_2400_CSI2_DATA_TIMEOUT_IDX    0
-#define _HRT_CSS_RECEIVER_2400_CSI2_DATA_TIMEOUT_BITS   19
-
-/* Bits for INIT_COUNT register */
-#define _HRT_CSS_RECEIVER_2400_INIT_TIMER_IDX  0
-#define _HRT_CSS_RECEIVER_2400_INIT_TIMER_BITS 16
-
-/* Bits for COUNT registers */
-#define _HRT_CSS_RECEIVER_2400_SYNC_COUNT_IDX     0
-#define _HRT_CSS_RECEIVER_2400_SYNC_COUNT_BITS    8
-#define _HRT_CSS_RECEIVER_2400_RX_COUNT_IDX       0
-#define _HRT_CSS_RECEIVER_2400_RX_COUNT_BITS      8
-
-/* Bits for RAW116_18_DATAID register */
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW16_BITS_IDX   0
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW16_BITS_BITS  6
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW18_BITS_IDX   8
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW18_BITS_BITS  6
-
-/* Bits for COMP_FORMAT register, this selects the compression data format */
-#define _HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_IDX  0
-#define _HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_BITS 8
-#define _HRT_CSS_RECEIVER_2400_COMP_NUM_BITS_IDX  (_HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_IDX + _HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_BITS)
-#define _HRT_CSS_RECEIVER_2400_COMP_NUM_BITS_BITS 8
-
-/* Bits for COMP_PREDICT register, this selects the predictor algorithm */
-#define _HRT_CSS_RECEIVER_2400_PREDICT_NO_COMP 0
-#define _HRT_CSS_RECEIVER_2400_PREDICT_1       1
-#define _HRT_CSS_RECEIVER_2400_PREDICT_2       2
-
-/* Number of bits used for the delay registers */
-#define _HRT_CSS_RECEIVER_2400_DELAY_BITS 8
-
-/* Bits for COMP_SCHEME register, this  selects the compression scheme for a VC */
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD1_BITS_IDX  0
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD2_BITS_IDX  5
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD3_BITS_IDX  10
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD4_BITS_IDX  15
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD5_BITS_IDX  20
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD6_BITS_IDX  25
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD7_BITS_IDX  0
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD8_BITS_IDX  5
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_BITS_BITS  5
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_FMT_BITS_IDX   0
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_FMT_BITS_BITS  3
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_PRED_BITS_IDX  3
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_PRED_BITS_BITS 2
-
-
-/* BITS for backend RAW16 and RAW 18 registers */
-
-#define _HRT_CSS_RECEIVER_2400_RAW18_DATAID_IDX    0
-#define _HRT_CSS_RECEIVER_2400_RAW18_DATAID_BITS   6
-#define _HRT_CSS_RECEIVER_2400_RAW18_OPTION_IDX    6
-#define _HRT_CSS_RECEIVER_2400_RAW18_OPTION_BITS   2
-#define _HRT_CSS_RECEIVER_2400_RAW18_EN_IDX        8
-#define _HRT_CSS_RECEIVER_2400_RAW18_EN_BITS       1
-
-#define _HRT_CSS_RECEIVER_2400_RAW16_DATAID_IDX    0
-#define _HRT_CSS_RECEIVER_2400_RAW16_DATAID_BITS   6
-#define _HRT_CSS_RECEIVER_2400_RAW16_OPTION_IDX    6
-#define _HRT_CSS_RECEIVER_2400_RAW16_OPTION_BITS   2
-#define _HRT_CSS_RECEIVER_2400_RAW16_EN_IDX        8
-#define _HRT_CSS_RECEIVER_2400_RAW16_EN_BITS       1
-
-/* These hsync and vsync values are for HSS simulation only */
-#define _HRT_CSS_RECEIVER_2400_HSYNC_VAL (1<<16)
-#define _HRT_CSS_RECEIVER_2400_VSYNC_VAL (1<<17)
-
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_WIDTH                 28
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_LSB              0
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_MSB             (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_LSB + CSS_RECEIVER_DATA_OUT - 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_VAL_BIT         (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_MSB + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_LSB             (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_VAL_BIT + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_MSB             (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_LSB + CSS_RECEIVER_DATA_OUT - 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_VAL_BIT         (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_MSB + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_SOP_BIT               (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_VAL_BIT + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_EOP_BIT               (_HRT_CSS_RECEIVER_2400_BE_STREAMING_SOP_BIT + 1)
-
-// SH Backend Register IDs
-#define _HRT_CSS_RECEIVER_2400_BE_GSP_ACC_OVL_REG_IDX              0
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_REG_IDX                     1
-#define _HRT_CSS_RECEIVER_2400_BE_TWO_PPC_REG_IDX                  2
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG0_IDX             3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG1_IDX             4
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG2_IDX             5
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG3_IDX             6
-#define _HRT_CSS_RECEIVER_2400_BE_SEL_REG_IDX                      7
-#define _HRT_CSS_RECEIVER_2400_BE_RAW16_CONFIG_REG_IDX             8
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_CONFIG_REG_IDX             9
-#define _HRT_CSS_RECEIVER_2400_BE_FORCE_RAW8_REG_IDX              10
-#define _HRT_CSS_RECEIVER_2400_BE_IRQ_STATUS_REG_IDX              11
-#define _HRT_CSS_RECEIVER_2400_BE_IRQ_CLEAR_REG_IDX               12
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_EN_REG_IDX                 13
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_DATA_STATE_REG_IDX         14    /* Data State 0,1,2 config */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P0_REG_IDX       15    /* Pixel Extractor config for Data State 0 & Pix 0 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P1_REG_IDX       16    /* Pixel Extractor config for Data State 0 & Pix 1 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P2_REG_IDX       17    /* Pixel Extractor config for Data State 0 & Pix 2 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P3_REG_IDX       18    /* Pixel Extractor config for Data State 0 & Pix 3 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P0_REG_IDX       19    /* Pixel Extractor config for Data State 1 & Pix 0 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P1_REG_IDX       20    /* Pixel Extractor config for Data State 1 & Pix 1 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P2_REG_IDX       21    /* Pixel Extractor config for Data State 1 & Pix 2 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P3_REG_IDX       22    /* Pixel Extractor config for Data State 1 & Pix 3 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P0_REG_IDX       23    /* Pixel Extractor config for Data State 2 & Pix 0 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P1_REG_IDX       24    /* Pixel Extractor config for Data State 2 & Pix 1 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P2_REG_IDX       25    /* Pixel Extractor config for Data State 2 & Pix 2 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P3_REG_IDX       26    /* Pixel Extractor config for Data State 2 & Pix 3 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_VALID_EOP_REG_IDX      27    /* Pixel Valid & EoP config for Pix 0,1,2,3 */
-
-#define _HRT_CSS_RECEIVER_2400_BE_NOF_REGISTERS                   28
-
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_HE                          0
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_RCF                         1
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_PF                          2
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_SM                          3
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_PD                          4
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_SD                          5
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_OT                          6
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_BC                          7
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_WIDTH                       8
-
-#endif /* _css_receiver_2400_defs_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/defs.h
deleted file mode 100644
index 47505f41790c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/defs.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_DEFS_H_
-#define _HRT_DEFS_H_
-
-#ifndef HRTCAT
-#define _HRTCAT(m, n)     m##n
-#define HRTCAT(m, n)      _HRTCAT(m, n)
-#endif
-
-#ifndef HRTSTR
-#define _HRTSTR(x)   #x
-#define HRTSTR(x)    _HRTSTR(x)
-#endif
-
-#ifndef HRTMIN
-#define HRTMIN(a, b) (((a) < (b)) ? (a) : (b))
-#endif
-
-#ifndef HRTMAX
-#define HRTMAX(a, b) (((a) > (b)) ? (a) : (b))
-#endif
-
-#endif /* _HRT_DEFS_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/dma_v2_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/dma_v2_defs.h
deleted file mode 100644
index d184a8b313c9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/dma_v2_defs.h
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _dma_v2_defs_h
-#define _dma_v2_defs_h
-
-#define _DMA_V2_NUM_CHANNELS_ID               MaxNumChannels
-#define _DMA_V2_CONNECTIONS_ID                Connections
-#define _DMA_V2_DEV_ELEM_WIDTHS_ID            DevElemWidths
-#define _DMA_V2_DEV_FIFO_DEPTH_ID             DevFifoDepth
-#define _DMA_V2_DEV_FIFO_RD_LAT_ID            DevFifoRdLat
-#define _DMA_V2_DEV_FIFO_LAT_BYPASS_ID        DevFifoRdLatBypass
-#define _DMA_V2_DEV_NO_BURST_ID               DevNoBurst
-#define _DMA_V2_DEV_RD_ACCEPT_ID              DevRdAccept
-#define _DMA_V2_DEV_SRMD_ID                   DevSRMD
-#define _DMA_V2_DEV_HAS_CRUN_ID               CRunMasters
-#define _DMA_V2_CTRL_ACK_FIFO_DEPTH_ID        CtrlAckFifoDepth
-#define _DMA_V2_CMD_FIFO_DEPTH_ID             CommandFifoDepth
-#define _DMA_V2_CMD_FIFO_RD_LAT_ID            CommandFifoRdLat
-#define _DMA_V2_CMD_FIFO_LAT_BYPASS_ID        CommandFifoRdLatBypass
-#define _DMA_V2_NO_PACK_ID                    has_no_pack
-
-#define _DMA_V2_REG_ALIGN                4
-#define _DMA_V2_REG_ADDR_BITS            2
-
-/* Command word */
-#define _DMA_V2_CMD_IDX            0
-#define _DMA_V2_CMD_BITS           6
-#define _DMA_V2_CHANNEL_IDX        (_DMA_V2_CMD_IDX + _DMA_V2_CMD_BITS)
-#define _DMA_V2_CHANNEL_BITS       5
-
-/* The command to set a parameter contains the PARAM field next */
-#define _DMA_V2_PARAM_IDX          (_DMA_V2_CHANNEL_IDX + _DMA_V2_CHANNEL_BITS)
-#define _DMA_V2_PARAM_BITS         4
-
-/* Commands to read, write or init specific blocks contain these
-   three values */
-#define _DMA_V2_SPEC_DEV_A_XB_IDX  (_DMA_V2_CHANNEL_IDX + _DMA_V2_CHANNEL_BITS)
-#define _DMA_V2_SPEC_DEV_A_XB_BITS 8
-#define _DMA_V2_SPEC_DEV_B_XB_IDX  (_DMA_V2_SPEC_DEV_A_XB_IDX + _DMA_V2_SPEC_DEV_A_XB_BITS)
-#define _DMA_V2_SPEC_DEV_B_XB_BITS 8
-#define _DMA_V2_SPEC_YB_IDX        (_DMA_V2_SPEC_DEV_B_XB_IDX + _DMA_V2_SPEC_DEV_B_XB_BITS)
-#define _DMA_V2_SPEC_YB_BITS       (32-_DMA_V2_SPEC_DEV_B_XB_BITS-_DMA_V2_SPEC_DEV_A_XB_BITS-_DMA_V2_CMD_BITS-_DMA_V2_CHANNEL_BITS)
-
-/* */
-#define _DMA_V2_CMD_CTRL_IDX       4
-#define _DMA_V2_CMD_CTRL_BITS      4
-
-/* Packing setup word */
-#define _DMA_V2_CONNECTION_IDX     0
-#define _DMA_V2_CONNECTION_BITS    4
-#define _DMA_V2_EXTENSION_IDX      (_DMA_V2_CONNECTION_IDX + _DMA_V2_CONNECTION_BITS)
-#define _DMA_V2_EXTENSION_BITS     1
-
-/* Elements packing word */
-#define _DMA_V2_ELEMENTS_IDX        0
-#define _DMA_V2_ELEMENTS_BITS       8
-#define _DMA_V2_LEFT_CROPPING_IDX  (_DMA_V2_ELEMENTS_IDX + _DMA_V2_ELEMENTS_BITS)
-#define _DMA_V2_LEFT_CROPPING_BITS  8
-
-#define _DMA_V2_WIDTH_IDX           0
-#define _DMA_V2_WIDTH_BITS         16
-
-#define _DMA_V2_HEIGHT_IDX          0
-#define _DMA_V2_HEIGHT_BITS        16
-
-#define _DMA_V2_STRIDE_IDX          0
-#define _DMA_V2_STRIDE_BITS        32
-
-/* Command IDs */
-#define _DMA_V2_MOVE_B2A_COMMAND                             0      
-#define _DMA_V2_MOVE_B2A_BLOCK_COMMAND                       1      
-#define _DMA_V2_MOVE_B2A_NO_SYNC_CHK_COMMAND                 2      
-#define _DMA_V2_MOVE_B2A_BLOCK_NO_SYNC_CHK_COMMAND           3      
-#define _DMA_V2_MOVE_A2B_COMMAND                             4      
-#define _DMA_V2_MOVE_A2B_BLOCK_COMMAND                       5      
-#define _DMA_V2_MOVE_A2B_NO_SYNC_CHK_COMMAND                 6      
-#define _DMA_V2_MOVE_A2B_BLOCK_NO_SYNC_CHK_COMMAND           7      
-#define _DMA_V2_INIT_A_COMMAND                               8      
-#define _DMA_V2_INIT_A_BLOCK_COMMAND                         9      
-#define _DMA_V2_INIT_A_NO_SYNC_CHK_COMMAND                  10      
-#define _DMA_V2_INIT_A_BLOCK_NO_SYNC_CHK_COMMAND            11      
-#define _DMA_V2_INIT_B_COMMAND                              12      
-#define _DMA_V2_INIT_B_BLOCK_COMMAND                        13      
-#define _DMA_V2_INIT_B_NO_SYNC_CHK_COMMAND                  14      
-#define _DMA_V2_INIT_B_BLOCK_NO_SYNC_CHK_COMMAND            15      
-#define _DMA_V2_NO_ACK_MOVE_B2A_NO_SYNC_CHK_COMMAND         (_DMA_V2_MOVE_B2A_NO_SYNC_CHK_COMMAND       + 16) 
-#define _DMA_V2_NO_ACK_MOVE_B2A_BLOCK_NO_SYNC_CHK_COMMAND   (_DMA_V2_MOVE_B2A_BLOCK_NO_SYNC_CHK_COMMAND + 16) 
-#define _DMA_V2_NO_ACK_MOVE_A2B_NO_SYNC_CHK_COMMAND         (_DMA_V2_MOVE_A2B_NO_SYNC_CHK_COMMAND       + 16) 
-#define _DMA_V2_NO_ACK_MOVE_A2B_BLOCK_NO_SYNC_CHK_COMMAND   (_DMA_V2_MOVE_A2B_BLOCK_NO_SYNC_CHK_COMMAND + 16) 
-#define _DMA_V2_NO_ACK_INIT_A_NO_SYNC_CHK_COMMAND           (_DMA_V2_INIT_A_NO_SYNC_CHK_COMMAND         + 16) 
-#define _DMA_V2_NO_ACK_INIT_A_BLOCK_NO_SYNC_CHK_COMMAND     (_DMA_V2_INIT_A_BLOCK_NO_SYNC_CHK_COMMAND   + 16) 
-#define _DMA_V2_NO_ACK_INIT_B_NO_SYNC_CHK_COMMAND           (_DMA_V2_INIT_B_NO_SYNC_CHK_COMMAND         + 16) 
-#define _DMA_V2_NO_ACK_INIT_B_BLOCK_NO_SYNC_CHK_COMMAND     (_DMA_V2_INIT_B_BLOCK_NO_SYNC_CHK_COMMAND   + 16) 
-#define _DMA_V2_CONFIG_CHANNEL_COMMAND                      32   
-#define _DMA_V2_SET_CHANNEL_PARAM_COMMAND                   33   
-#define _DMA_V2_SET_CRUN_COMMAND                            62   
-
-/* Channel Parameter IDs */
-#define _DMA_V2_PACKING_SETUP_PARAM                     0  
-#define _DMA_V2_STRIDE_A_PARAM                          1  
-#define _DMA_V2_ELEM_CROPPING_A_PARAM                   2  
-#define _DMA_V2_WIDTH_A_PARAM                           3  
-#define _DMA_V2_STRIDE_B_PARAM                          4  
-#define _DMA_V2_ELEM_CROPPING_B_PARAM                   5  
-#define _DMA_V2_WIDTH_B_PARAM                           6  
-#define _DMA_V2_HEIGHT_PARAM                            7  
-#define _DMA_V2_QUEUED_CMDS                             8  
-
-/* Parameter Constants */
-#define _DMA_V2_ZERO_EXTEND                             0
-#define _DMA_V2_SIGN_EXTEND                             1
-
-  /* SLAVE address map */
-#define _DMA_V2_SEL_FSM_CMD                             0
-#define _DMA_V2_SEL_CH_REG                              1
-#define _DMA_V2_SEL_CONN_GROUP                          2
-#define _DMA_V2_SEL_DEV_INTERF                          3
-
-#define _DMA_V2_ADDR_SEL_COMP_IDX                      12
-#define _DMA_V2_ADDR_SEL_COMP_BITS                      4
-#define _DMA_V2_ADDR_SEL_CH_REG_IDX                     2
-#define _DMA_V2_ADDR_SEL_CH_REG_BITS                    6
-#define _DMA_V2_ADDR_SEL_PARAM_IDX                      (_DMA_V2_ADDR_SEL_CH_REG_BITS+_DMA_V2_ADDR_SEL_CH_REG_IDX)
-#define _DMA_V2_ADDR_SEL_PARAM_BITS                     4
-
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_IDX                 2
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_BITS                6
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_INFO_IDX            (_DMA_V2_ADDR_SEL_GROUP_COMP_BITS + _DMA_V2_ADDR_SEL_GROUP_COMP_IDX)
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_INFO_BITS           4
-
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_IDX_IDX             2
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_IDX_BITS            6
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_INFO_IDX            (_DMA_V2_ADDR_SEL_DEV_INTERF_IDX_IDX+_DMA_V2_ADDR_SEL_DEV_INTERF_IDX_BITS)
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_INFO_BITS           4
-
-#define _DMA_V2_FSM_GROUP_CMD_IDX                       0
-#define _DMA_V2_FSM_GROUP_ADDR_SRC_IDX                  1
-#define _DMA_V2_FSM_GROUP_ADDR_DEST_IDX                 2
-#define _DMA_V2_FSM_GROUP_CMD_CTRL_IDX                  3
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_IDX                  4
-#define _DMA_V2_FSM_GROUP_FSM_PACK_IDX                  5
-#define _DMA_V2_FSM_GROUP_FSM_REQ_IDX                   6
-#define _DMA_V2_FSM_GROUP_FSM_WR_IDX                    7
-  
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_STATE_IDX            0
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_DEV_IDX          1
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_ADDR_IDX         2
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_STRIDE_IDX       3
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_XB_IDX           4
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_YB_IDX           5
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_REQ_DEV_IDX     6
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_DEV_IDX      7
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_WR_ADDR_IDX          8
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_WR_STRIDE_IDX        9
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_REQ_XB_IDX     10
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_YB_IDX      11
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_XB_IDX      12
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_ELEM_REQ_IDX   13
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_ELEM_WR_IDX    14
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_S_Z_IDX        15
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_CMD_CTRL_IDX        15
-
-#define _DMA_V2_FSM_GROUP_FSM_PACK_STATE_IDX            0
-#define _DMA_V2_FSM_GROUP_FSM_PACK_CNT_YB_IDX           1
-#define _DMA_V2_FSM_GROUP_FSM_PACK_CNT_XB_REQ_IDX       2
-#define _DMA_V2_FSM_GROUP_FSM_PACK_CNT_XB_WR_IDX        3
-
-#define _DMA_V2_FSM_GROUP_FSM_REQ_STATE_IDX             0
-#define _DMA_V2_FSM_GROUP_FSM_REQ_CNT_YB_IDX            1
-#define _DMA_V2_FSM_GROUP_FSM_REQ_CNT_XB_IDX            2
-#define _DMA_V2_FSM_GROUP_FSM_REQ_XB_REMAINING_IDX      3
-#define _DMA_V2_FSM_GROUP_FSM_REQ_CNT_BURST_IDX         4
-
-#define _DMA_V2_FSM_GROUP_FSM_WR_STATE_IDX              0
-#define _DMA_V2_FSM_GROUP_FSM_WR_CNT_YB_IDX             1
-#define _DMA_V2_FSM_GROUP_FSM_WR_CNT_XB_IDX             2
-#define _DMA_V2_FSM_GROUP_FSM_WR_XB_REMAINING_IDX       3
-#define _DMA_V2_FSM_GROUP_FSM_WR_CNT_BURST_IDX          4
-
-#define _DMA_V2_DEV_INTERF_REQ_SIDE_STATUS_IDX          0
-#define _DMA_V2_DEV_INTERF_SEND_SIDE_STATUS_IDX         1
-#define _DMA_V2_DEV_INTERF_FIFO_STATUS_IDX              2
-#define _DMA_V2_DEV_INTERF_REQ_ONLY_COMPLETE_BURST_IDX  3
-#define _DMA_V2_DEV_INTERF_MAX_BURST_IDX                4
-#define _DMA_V2_DEV_INTERF_CHK_ADDR_ALIGN               5
-
-#endif /* _dma_v2_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gdc_v2_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gdc_v2_defs.h
deleted file mode 100644
index 77722d205701..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gdc_v2_defs.h
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef HRT_GDC_v2_defs_h_
-#define HRT_GDC_v2_defs_h_
-
-#define HRT_GDC_IS_V2
-
-#define HRT_GDC_N                     1024 /* Top-level design constant, equal to the number of entries in the LUT      */
-#define HRT_GDC_FRAC_BITS               10 /* Number of fractional bits in the GDC block, driven by the size of the LUT */
-
-#define HRT_GDC_BLI_FRAC_BITS            4 /* Number of fractional bits for the bi-linear interpolation type            */
-#define HRT_GDC_BLI_COEF_ONE             (1 << HRT_GDC_BLI_FRAC_BITS)
-
-#define HRT_GDC_BCI_COEF_BITS           14 /* 14 bits per coefficient                                                   */
-#define HRT_GDC_BCI_COEF_ONE             (1 << (HRT_GDC_BCI_COEF_BITS-2))  /* We represent signed 10 bit coefficients.  */
-                                                                        /* The supported range is [-256, .., +256]      */
-                                                                        /* in 14-bit signed notation,                   */
-                                                                        /* We need all ten bits (MSB must be zero).     */
-                                                                        /* -s is inserted to solve this issue, and      */
-                                                                        /* therefore "1" is equal to +256.              */
-#define HRT_GDC_BCI_COEF_MASK            ((1 << HRT_GDC_BCI_COEF_BITS) - 1) 
-
-#define HRT_GDC_LUT_BYTES                (HRT_GDC_N*4*2)                /* 1024 addresses, 4 coefficients per address,  */
-                                                                        /* 2 bytes per coefficient                      */
-
-#define _HRT_GDC_REG_ALIGN               4                              
-
-  //     31  30  29    25 24                     0
-  //  |-----|---|--------|------------------------|
-  //  | CMD | C | Reg_ID |        Value           |
-
-
-  // There are just two commands possible for the GDC block:
-  // 1 - Configure reg 
-  // 0 - Data token    
-  
-  // C      - Reserved bit
-  //          Used in protocol to indicate whether it is C-run or other type of runs
-  //          In case of C-run, this bit has a value of 1, for all the other runs, it is 0.
-
-  // Reg_ID - Address of the register to be configured
-  
-  // Value  - Value to store to the addressed register, maximum of 24 bits
-
-  // Configure reg command is not followed by any other token. 
-  // The address of the register and the data to be filled in is contained in the same token 
-  
-  // When the first data token is received, it must be:
-  //   1. FRX and FRY (device configured in one of the  scaling modes) ***DEFAULT MODE***, or,
-  //   2. P0'X        (device configured in one of the tetragon modes)
-  // After the first data token is received, pre-defined number of tokens with the following meaning follow:
-  //   1. two  tokens: SRC address ; DST address
-  //   2. nine tokens: P0'Y, .., P3'Y ; SRC address ; DST address
-  
-#define HRT_GDC_CONFIG_CMD             1
-#define HRT_GDC_DATA_CMD               0
-
-
-#define HRT_GDC_CMD_POS               31
-#define HRT_GDC_CMD_BITS               1
-#define HRT_GDC_CRUN_POS              30
-#define HRT_GDC_REG_ID_POS            25
-#define HRT_GDC_REG_ID_BITS            5
-#define HRT_GDC_DATA_POS               0
-#define HRT_GDC_DATA_BITS             25
-
-#define HRT_GDC_FRYIPXFRX_BITS        26
-#define HRT_GDC_P0X_BITS              23
-
-
-#define HRT_GDC_MAX_OXDIM           (8192-64)
-#define HRT_GDC_MAX_OYDIM           4095
-#define HRT_GDC_MAX_IXDIM           (8192-64)
-#define HRT_GDC_MAX_IYDIM           4095
-#define HRT_GDC_MAX_DS_FAC            16
-#define HRT_GDC_MAX_DX                 (HRT_GDC_MAX_DS_FAC*HRT_GDC_N - 1)
-#define HRT_GDC_MAX_DY                 HRT_GDC_MAX_DX
-
-
-/* GDC lookup tables entries are 10 bits values, but they're
-   stored 2 by 2 as 32 bit values, yielding 16 bits per entry.
-   A GDC lookup table contains 64 * 4 elements */
-
-#define HRT_GDC_PERF_1_1_pix          0
-#define HRT_GDC_PERF_2_1_pix          1
-#define HRT_GDC_PERF_1_2_pix          2
-#define HRT_GDC_PERF_2_2_pix          3
-
-#define HRT_GDC_NND_MODE              0
-#define HRT_GDC_BLI_MODE              1
-#define HRT_GDC_BCI_MODE              2
-#define HRT_GDC_LUT_MODE              3
-
-#define HRT_GDC_SCAN_STB              0
-#define HRT_GDC_SCAN_STR              1
-
-#define HRT_GDC_MODE_SCALING          0
-#define HRT_GDC_MODE_TETRAGON         1
-
-#define HRT_GDC_LUT_COEFF_OFFSET     16 
-#define HRT_GDC_FRY_BIT_OFFSET       16 
-// FRYIPXFRX is the only register where we store two values in one field, 
-// to save one token in the scaling protocol. 
-// Like this, we have three tokens in the scaling protocol, 
-// Otherwise, we would have had four.
-// The register bit-map is:
-//   31  26 25      16 15  10 9        0
-//  |------|----------|------|----------|
-//  | XXXX |   FRY    |  IPX |   FRX    |
-
-
-#define HRT_GDC_CE_FSM0_POS           0
-#define HRT_GDC_CE_FSM0_LEN           2
-#define HRT_GDC_CE_OPY_POS            2
-#define HRT_GDC_CE_OPY_LEN           14
-#define HRT_GDC_CE_OPX_POS           16
-#define HRT_GDC_CE_OPX_LEN           16
-// CHK_ENGINE register bit-map:
-//   31            16 15        2 1  0
-//  |----------------|-----------|----|
-//  |      OPX       |    OPY    |FSM0|
-// However, for the time being at least, 
-// this implementation is meaningless in hss model,
-// So, we just return 0
-
-
-#define HRT_GDC_CHK_ENGINE_IDX        0
-#define HRT_GDC_WOIX_IDX              1
-#define HRT_GDC_WOIY_IDX              2
-#define HRT_GDC_BPP_IDX               3
-#define HRT_GDC_FRYIPXFRX_IDX         4
-#define HRT_GDC_OXDIM_IDX             5
-#define HRT_GDC_OYDIM_IDX             6
-#define HRT_GDC_SRC_ADDR_IDX          7
-#define HRT_GDC_SRC_END_ADDR_IDX      8
-#define HRT_GDC_SRC_WRAP_ADDR_IDX     9
-#define HRT_GDC_SRC_STRIDE_IDX       10
-#define HRT_GDC_DST_ADDR_IDX         11
-#define HRT_GDC_DST_STRIDE_IDX       12
-#define HRT_GDC_DX_IDX               13
-#define HRT_GDC_DY_IDX               14
-#define HRT_GDC_P0X_IDX              15
-#define HRT_GDC_P0Y_IDX              16
-#define HRT_GDC_P1X_IDX              17
-#define HRT_GDC_P1Y_IDX              18
-#define HRT_GDC_P2X_IDX              19
-#define HRT_GDC_P2Y_IDX              20
-#define HRT_GDC_P3X_IDX              21
-#define HRT_GDC_P3Y_IDX              22
-#define HRT_GDC_PERF_POINT_IDX       23  // 1x1 ; 1x2 ; 2x1 ; 2x2 pixels per cc
-#define HRT_GDC_INTERP_TYPE_IDX      24  // NND ; BLI ; BCI ; LUT
-#define HRT_GDC_SCAN_IDX             25  // 0 = STB (Slide To Bottom) ; 1 = STR (Slide To Right)
-#define HRT_GDC_PROC_MODE_IDX        26  // 0 = Scaling ; 1 = Tetragon
-
-#define HRT_GDC_LUT_IDX              32
-
-
-#endif /* HRT_GDC_v2_defs_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gp_timer_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gp_timer_defs.h
deleted file mode 100644
index 3082e2f5e014..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gp_timer_defs.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _gp_timer_defs_h
-#define _gp_timer_defs_h
-
-#define _HRT_GP_TIMER_REG_ALIGN 4
-
-#define HIVE_GP_TIMER_RESET_REG_IDX                              0
-#define HIVE_GP_TIMER_OVERALL_ENABLE_REG_IDX                     1
-#define HIVE_GP_TIMER_ENABLE_REG_IDX(timer)                     (HIVE_GP_TIMER_OVERALL_ENABLE_REG_IDX + 1 + timer)
-#define HIVE_GP_TIMER_VALUE_REG_IDX(timer,timers)               (HIVE_GP_TIMER_ENABLE_REG_IDX(timers) + timer)
-#define HIVE_GP_TIMER_COUNT_TYPE_REG_IDX(timer,timers)          (HIVE_GP_TIMER_VALUE_REG_IDX(timers, timers) + timer)
-#define HIVE_GP_TIMER_SIGNAL_SELECT_REG_IDX(timer,timers)       (HIVE_GP_TIMER_COUNT_TYPE_REG_IDX(timers, timers) + timer)
-#define HIVE_GP_TIMER_IRQ_TRIGGER_VALUE_REG_IDX(irq,timers)     (HIVE_GP_TIMER_SIGNAL_SELECT_REG_IDX(timers, timers) + irq)
-#define HIVE_GP_TIMER_IRQ_TIMER_SELECT_REG_IDX(irq,timers,irqs) (HIVE_GP_TIMER_IRQ_TRIGGER_VALUE_REG_IDX(irqs, timers) + irq)
-#define HIVE_GP_TIMER_IRQ_ENABLE_REG_IDX(irq,timers,irqs)       (HIVE_GP_TIMER_IRQ_TIMER_SELECT_REG_IDX(irqs, timers, irqs) + irq)
-
-#define HIVE_GP_TIMER_COUNT_TYPE_HIGH                            0
-#define HIVE_GP_TIMER_COUNT_TYPE_LOW                             1
-#define HIVE_GP_TIMER_COUNT_TYPE_POSEDGE                         2
-#define HIVE_GP_TIMER_COUNT_TYPE_NEGEDGE                         3
-#define HIVE_GP_TIMER_COUNT_TYPES                                4
-
-#endif /* _gp_timer_defs_h */   
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gpio_block_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gpio_block_defs.h
deleted file mode 100644
index a807d4c99041..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/gpio_block_defs.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _gpio_block_defs_h_
-#define _gpio_block_defs_h_
-
-#define _HRT_GPIO_BLOCK_REG_ALIGN 4
-
-/* R/W registers */
-#define _gpio_block_reg_do_e			         0
-#define _gpio_block_reg_do_select		       1
-#define _gpio_block_reg_do_0			         2
-#define _gpio_block_reg_do_1			         3
-#define _gpio_block_reg_do_pwm_cnt_0	     4
-#define _gpio_block_reg_do_pwm_cnt_1	     5
-#define _gpio_block_reg_do_pwm_cnt_2	     6
-#define _gpio_block_reg_do_pwm_cnt_3	     7
-#define _gpio_block_reg_do_pwm_main_cnt    8
-#define _gpio_block_reg_do_pwm_enable      9
-#define _gpio_block_reg_di_debounce_sel	  10
-#define _gpio_block_reg_di_debounce_cnt_0	11
-#define _gpio_block_reg_di_debounce_cnt_1	12
-#define _gpio_block_reg_di_debounce_cnt_2	13
-#define _gpio_block_reg_di_debounce_cnt_3	14
-#define _gpio_block_reg_di_active_level	  15
-
-
-/* read-only registers */
-#define _gpio_block_reg_di			          16
-
-#endif /* _gpio_block_defs_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_defs.h
deleted file mode 100644
index 39584996092e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_defs.h
+++ /dev/null
@@ -1,416 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _hive_isp_css_defs_h__
-#define _hive_isp_css_defs_h__
-
-#define HIVE_ISP_CSS_IS_2400B0_SYSTEM
-
-#define HIVE_ISP_CTRL_DATA_WIDTH     32
-#define HIVE_ISP_CTRL_ADDRESS_WIDTH  32
-#define HIVE_ISP_CTRL_MAX_BURST_SIZE  1
-#define HIVE_ISP_DDR_ADDRESS_WIDTH   36
-
-#define HIVE_ISP_HOST_MAX_BURST_SIZE  8 /* host supports bursts in order to prevent repeating DDRAM accesses */
-#define HIVE_ISP_NUM_GPIO_PINS       12
-
-/* This list of vector num_elems/elem_bits pairs is valid both in C as initializer
-   and in the DMA parameter list */
-#define HIVE_ISP_DDR_DMA_SPECS {{32,  8}, {16, 16}, {18, 14}, {25, 10}, {21, 12}}
-#define HIVE_ISP_DDR_WORD_BITS 256
-#define HIVE_ISP_DDR_WORD_BYTES  (HIVE_ISP_DDR_WORD_BITS/8)
-#define HIVE_ISP_DDR_BYTES       (512 * 1024 * 1024) /* hss only */
-#define HIVE_ISP_DDR_BYTES_RTL   (127 * 1024 * 1024) /* RTL only */
-#define HIVE_ISP_DDR_SMALL_BYTES (128 * 256 / 8)
-#define HIVE_ISP_PAGE_SHIFT    12
-#define HIVE_ISP_PAGE_SIZE     (1<<HIVE_ISP_PAGE_SHIFT)
-
-#define CSS_DDR_WORD_BITS        HIVE_ISP_DDR_WORD_BITS
-#define CSS_DDR_WORD_BYTES       HIVE_ISP_DDR_WORD_BYTES
-
-/* If HIVE_ISP_DDR_BASE_OFFSET is set to a non-zero value, the wide bus just before the DDRAM gets an extra dummy port where         */
-/* address range 0 .. HIVE_ISP_DDR_BASE_OFFSET-1 maps onto. This effectively creates an offset for the DDRAM from system perspective */
-#define HIVE_ISP_DDR_BASE_OFFSET 0x120000000 /* 0x200000 */
-
-#define HIVE_DMA_ISP_BUS_CONN 0
-#define HIVE_DMA_ISP_DDR_CONN 1
-#define HIVE_DMA_BUS_DDR_CONN 2
-#define HIVE_DMA_ISP_MASTER master_port0
-#define HIVE_DMA_BUS_MASTER master_port1
-#define HIVE_DMA_DDR_MASTER master_port2
-
-#define HIVE_DMA_NUM_CHANNELS       32 /* old value was  8 */
-#define HIVE_DMA_CMD_FIFO_DEPTH     24 /* old value was 12 */
-
-#define HIVE_IF_PIXEL_WIDTH 12
-
-#define HIVE_MMU_TLB_SETS           8
-#define HIVE_MMU_TLB_SET_BLOCKS     8
-#define HIVE_MMU_TLB_BLOCK_ELEMENTS 8
-#define HIVE_MMU_PAGE_TABLE_LEVELS  2
-#define HIVE_MMU_PAGE_BYTES         HIVE_ISP_PAGE_SIZE
-
-#define HIVE_ISP_CH_ID_BITS    2
-#define HIVE_ISP_FMT_TYPE_BITS 5
-#define HIVE_ISP_ISEL_SEL_BITS 2
-
-#define HIVE_GP_REGS_SDRAM_WAKEUP_IDX                           0
-#define HIVE_GP_REGS_IDLE_IDX                                   1
-#define HIVE_GP_REGS_IRQ_0_IDX                                  2
-#define HIVE_GP_REGS_IRQ_1_IDX                                  3
-#define HIVE_GP_REGS_SP_STREAM_STAT_IDX                         4
-#define HIVE_GP_REGS_SP_STREAM_STAT_B_IDX                       5
-#define HIVE_GP_REGS_ISP_STREAM_STAT_IDX                        6
-#define HIVE_GP_REGS_MOD_STREAM_STAT_IDX                        7
-#define HIVE_GP_REGS_SP_STREAM_STAT_IRQ_COND_IDX                8
-#define HIVE_GP_REGS_SP_STREAM_STAT_B_IRQ_COND_IDX              9
-#define HIVE_GP_REGS_ISP_STREAM_STAT_IRQ_COND_IDX              10
-#define HIVE_GP_REGS_MOD_STREAM_STAT_IRQ_COND_IDX              11
-#define HIVE_GP_REGS_SP_STREAM_STAT_IRQ_ENABLE_IDX             12
-#define HIVE_GP_REGS_SP_STREAM_STAT_B_IRQ_ENABLE_IDX           13
-#define HIVE_GP_REGS_ISP_STREAM_STAT_IRQ_ENABLE_IDX            14
-#define HIVE_GP_REGS_MOD_STREAM_STAT_IRQ_ENABLE_IDX            15
-#define HIVE_GP_REGS_SWITCH_PRIM_IF_IDX                        16
-#define HIVE_GP_REGS_SWITCH_GDC1_IDX                           17
-#define HIVE_GP_REGS_SWITCH_GDC2_IDX                           18
-#define HIVE_GP_REGS_SRST_IDX                                  19
-#define HIVE_GP_REGS_SLV_REG_SRST_IDX                          20
-#define HIVE_GP_REGS_VISA_REG_IDX                              21
-
-/* Bit numbers of the soft reset register */
-#define HIVE_GP_REGS_SRST_ISYS_CBUS                             0
-#define HIVE_GP_REGS_SRST_ISEL_CBUS                             1
-#define HIVE_GP_REGS_SRST_IFMT_CBUS                             2
-#define HIVE_GP_REGS_SRST_GPDEV_CBUS                            3
-#define HIVE_GP_REGS_SRST_GPIO                                  4
-#define HIVE_GP_REGS_SRST_TC                                    5
-#define HIVE_GP_REGS_SRST_GPTIMER                               6
-#define HIVE_GP_REGS_SRST_FACELLFIFOS                           7
-#define HIVE_GP_REGS_SRST_D_OSYS                                8
-#define HIVE_GP_REGS_SRST_IFT_SEC_PIPE                          9
-#define HIVE_GP_REGS_SRST_GDC1                                 10
-#define HIVE_GP_REGS_SRST_GDC2                                 11
-#define HIVE_GP_REGS_SRST_VEC_BUS                              12
-#define HIVE_GP_REGS_SRST_ISP                                  13
-#define HIVE_GP_REGS_SRST_SLV_GRP_BUS                          14
-#define HIVE_GP_REGS_SRST_DMA                                  15
-#define HIVE_GP_REGS_SRST_SF_ISP_SP                            16
-#define HIVE_GP_REGS_SRST_SF_PIF_CELLS                         17
-#define HIVE_GP_REGS_SRST_SF_SIF_SP                            18
-#define HIVE_GP_REGS_SRST_SF_MC_SP                             19
-#define HIVE_GP_REGS_SRST_SF_ISYS_SP                           20
-#define HIVE_GP_REGS_SRST_SF_DMA_CELLS                         21
-#define HIVE_GP_REGS_SRST_SF_GDC1_CELLS                        22
-#define HIVE_GP_REGS_SRST_SF_GDC2_CELLS                        23
-#define HIVE_GP_REGS_SRST_SP                                   24
-#define HIVE_GP_REGS_SRST_OCP2CIO                              25
-#define HIVE_GP_REGS_SRST_NBUS                                 26
-#define HIVE_GP_REGS_SRST_HOST12BUS                            27
-#define HIVE_GP_REGS_SRST_WBUS                                 28
-#define HIVE_GP_REGS_SRST_IC_OSYS                              29
-#define HIVE_GP_REGS_SRST_WBUS_IC                              30
-
-/* Bit numbers of the slave register soft reset register */
-#define HIVE_GP_REGS_SLV_REG_SRST_DMA                           0
-#define HIVE_GP_REGS_SLV_REG_SRST_GDC1                          1
-#define HIVE_GP_REGS_SLV_REG_SRST_GDC2                          2
-
-/* order of the input bits for the irq controller */
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_0_BIT_ID                       0
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_1_BIT_ID                       1
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_2_BIT_ID                       2
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_3_BIT_ID                       3
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_4_BIT_ID                       4
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_5_BIT_ID                       5
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_6_BIT_ID                       6
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_7_BIT_ID                       7
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_8_BIT_ID                       8
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_9_BIT_ID                       9
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_10_BIT_ID                     10
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_11_BIT_ID                     11
-#define HIVE_GP_DEV_IRQ_SP_BIT_ID                              12
-#define HIVE_GP_DEV_IRQ_ISP_BIT_ID                             13
-#define HIVE_GP_DEV_IRQ_ISYS_BIT_ID                            14
-#define HIVE_GP_DEV_IRQ_ISEL_BIT_ID                            15
-#define HIVE_GP_DEV_IRQ_IFMT_BIT_ID                            16
-#define HIVE_GP_DEV_IRQ_SP_STREAM_MON_BIT_ID                   17
-#define HIVE_GP_DEV_IRQ_ISP_STREAM_MON_BIT_ID                  18
-#define HIVE_GP_DEV_IRQ_MOD_STREAM_MON_BIT_ID                  19
-#define HIVE_GP_DEV_IRQ_ISP_PMEM_ERROR_BIT_ID                  20
-#define HIVE_GP_DEV_IRQ_ISP_BAMEM_ERROR_BIT_ID                 21
-#define HIVE_GP_DEV_IRQ_ISP_DMEM_ERROR_BIT_ID                  22
-#define HIVE_GP_DEV_IRQ_SP_ICACHE_MEM_ERROR_BIT_ID             23
-#define HIVE_GP_DEV_IRQ_SP_DMEM_ERROR_BIT_ID                   24
-#define HIVE_GP_DEV_IRQ_MMU_CACHE_MEM_ERROR_BIT_ID             25
-#define HIVE_GP_DEV_IRQ_GP_TIMER_0_BIT_ID                      26
-#define HIVE_GP_DEV_IRQ_GP_TIMER_1_BIT_ID                      27
-#define HIVE_GP_DEV_IRQ_SW_PIN_0_BIT_ID                        28
-#define HIVE_GP_DEV_IRQ_SW_PIN_1_BIT_ID                        29
-#define HIVE_GP_DEV_IRQ_DMA_BIT_ID                             30
-#define HIVE_GP_DEV_IRQ_SP_STREAM_MON_B_BIT_ID                 31
-
-#define HIVE_GP_REGS_NUM_SW_IRQ_REGS                            2
-
-/* order of the input bits for the timed controller */
-#define HIVE_GP_DEV_TC_GPIO_PIN_0_BIT_ID                       0
-#define HIVE_GP_DEV_TC_GPIO_PIN_1_BIT_ID                       1
-#define HIVE_GP_DEV_TC_GPIO_PIN_2_BIT_ID                       2
-#define HIVE_GP_DEV_TC_GPIO_PIN_3_BIT_ID                       3
-#define HIVE_GP_DEV_TC_GPIO_PIN_4_BIT_ID                       4
-#define HIVE_GP_DEV_TC_GPIO_PIN_5_BIT_ID                       5
-#define HIVE_GP_DEV_TC_GPIO_PIN_6_BIT_ID                       6
-#define HIVE_GP_DEV_TC_GPIO_PIN_7_BIT_ID                       7
-#define HIVE_GP_DEV_TC_GPIO_PIN_8_BIT_ID                       8
-#define HIVE_GP_DEV_TC_GPIO_PIN_9_BIT_ID                       9
-#define HIVE_GP_DEV_TC_GPIO_PIN_10_BIT_ID                     10
-#define HIVE_GP_DEV_TC_GPIO_PIN_11_BIT_ID                     11
-#define HIVE_GP_DEV_TC_SP_BIT_ID                              12
-#define HIVE_GP_DEV_TC_ISP_BIT_ID                             13
-#define HIVE_GP_DEV_TC_ISYS_BIT_ID                            14
-#define HIVE_GP_DEV_TC_ISEL_BIT_ID                            15
-#define HIVE_GP_DEV_TC_IFMT_BIT_ID                            16
-#define HIVE_GP_DEV_TC_GP_TIMER_0_BIT_ID                      17
-#define HIVE_GP_DEV_TC_GP_TIMER_1_BIT_ID                      18
-#define HIVE_GP_DEV_TC_MIPI_SOL_BIT_ID                        19
-#define HIVE_GP_DEV_TC_MIPI_EOL_BIT_ID                        20
-#define HIVE_GP_DEV_TC_MIPI_SOF_BIT_ID                        21
-#define HIVE_GP_DEV_TC_MIPI_EOF_BIT_ID                        22
-#define HIVE_GP_DEV_TC_INPSYS_SM                              23
-
-/* definitions for the gp_timer block */
-#define HIVE_GP_TIMER_0                                         0
-#define HIVE_GP_TIMER_1                                         1
-#define HIVE_GP_TIMER_2                                         2
-#define HIVE_GP_TIMER_3                                         3
-#define HIVE_GP_TIMER_4                                         4
-#define HIVE_GP_TIMER_5                                         5
-#define HIVE_GP_TIMER_6                                         6
-#define HIVE_GP_TIMER_7                                         7
-#define HIVE_GP_TIMER_NUM_COUNTERS                              8
-
-#define HIVE_GP_TIMER_IRQ_0                                     0
-#define HIVE_GP_TIMER_IRQ_1                                     1
-#define HIVE_GP_TIMER_NUM_IRQS                                  2
-
-#define HIVE_GP_TIMER_GPIO_0_BIT_ID                             0
-#define HIVE_GP_TIMER_GPIO_1_BIT_ID                             1
-#define HIVE_GP_TIMER_GPIO_2_BIT_ID                             2
-#define HIVE_GP_TIMER_GPIO_3_BIT_ID                             3
-#define HIVE_GP_TIMER_GPIO_4_BIT_ID                             4
-#define HIVE_GP_TIMER_GPIO_5_BIT_ID                             5
-#define HIVE_GP_TIMER_GPIO_6_BIT_ID                             6
-#define HIVE_GP_TIMER_GPIO_7_BIT_ID                             7
-#define HIVE_GP_TIMER_GPIO_8_BIT_ID                             8
-#define HIVE_GP_TIMER_GPIO_9_BIT_ID                             9
-#define HIVE_GP_TIMER_GPIO_10_BIT_ID                           10
-#define HIVE_GP_TIMER_GPIO_11_BIT_ID                           11
-#define HIVE_GP_TIMER_INP_SYS_IRQ                              12
-#define HIVE_GP_TIMER_ISEL_IRQ                                 13
-#define HIVE_GP_TIMER_IFMT_IRQ                                 14
-#define HIVE_GP_TIMER_SP_STRMON_IRQ                            15
-#define HIVE_GP_TIMER_SP_B_STRMON_IRQ                          16
-#define HIVE_GP_TIMER_ISP_STRMON_IRQ                           17
-#define HIVE_GP_TIMER_MOD_STRMON_IRQ                           18
-#define HIVE_GP_TIMER_ISP_PMEM_ERROR_IRQ                       19
-#define HIVE_GP_TIMER_ISP_BAMEM_ERROR_IRQ                      20
-#define HIVE_GP_TIMER_ISP_DMEM_ERROR_IRQ                       21
-#define HIVE_GP_TIMER_SP_ICACHE_MEM_ERROR_IRQ                  22
-#define HIVE_GP_TIMER_SP_DMEM_ERROR_IRQ                        23
-#define HIVE_GP_TIMER_SP_OUT_RUN_DP                            24
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I0         25
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I1         26
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I2         27
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I3         28
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I4         29
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I5         30
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I6         31
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I7         32
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I8         33
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I9         34
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I10        35
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I1_I0         36
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I2_I0         37
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I3_I0         38
-#define HIVE_GP_TIMER_ISP_OUT_RUN_DP                           39
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I0_I0        40
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I0_I1        41
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I1_I0        42
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I0        43
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I1        44
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I2        45
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I3        46
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I4        47
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I5        48
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I6        49
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I3_I0        50
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I4_I0        51
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I5_I0        52
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I6_I0        53
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I7_I0        54                                                         
-#define HIVE_GP_TIMER_MIPI_SOL_BIT_ID                          55
-#define HIVE_GP_TIMER_MIPI_EOL_BIT_ID                          56
-#define HIVE_GP_TIMER_MIPI_SOF_BIT_ID                          57
-#define HIVE_GP_TIMER_MIPI_EOF_BIT_ID                          58
-#define HIVE_GP_TIMER_INPSYS_SM                                59
-
-/* port definitions for the streaming monitors */
-/* port definititions SP streaming monitor, monitors the status of streaming ports at the SP side of the streaming FIFO's */
-#define SP_STR_MON_PORT_SP2SIF            0
-#define SP_STR_MON_PORT_SIF2SP            1
-#define SP_STR_MON_PORT_SP2MC             2 
-#define SP_STR_MON_PORT_MC2SP             3
-#define SP_STR_MON_PORT_SP2DMA            4 
-#define SP_STR_MON_PORT_DMA2SP            5
-#define SP_STR_MON_PORT_SP2ISP            6 
-#define SP_STR_MON_PORT_ISP2SP            7
-#define SP_STR_MON_PORT_SP2GPD            8
-#define SP_STR_MON_PORT_FA2SP             9
-#define SP_STR_MON_PORT_SP2ISYS          10 
-#define SP_STR_MON_PORT_ISYS2SP          11
-#define SP_STR_MON_PORT_SP2PIFA          12
-#define SP_STR_MON_PORT_PIFA2SP          13
-#define SP_STR_MON_PORT_SP2PIFB          14
-#define SP_STR_MON_PORT_PIFB2SP          15
-
-#define SP_STR_MON_PORT_B_SP2GDC1         0
-#define SP_STR_MON_PORT_B_GDC12SP         1
-#define SP_STR_MON_PORT_B_SP2GDC2         2
-#define SP_STR_MON_PORT_B_GDC22SP         3
-
-/* previously used SP streaming monitor port identifiers, kept for backward compatibility */
-#define SP_STR_MON_PORT_SND_SIF           SP_STR_MON_PORT_SP2SIF
-#define SP_STR_MON_PORT_RCV_SIF           SP_STR_MON_PORT_SIF2SP
-#define SP_STR_MON_PORT_SND_MC            SP_STR_MON_PORT_SP2MC
-#define SP_STR_MON_PORT_RCV_MC            SP_STR_MON_PORT_MC2SP
-#define SP_STR_MON_PORT_SND_DMA           SP_STR_MON_PORT_SP2DMA
-#define SP_STR_MON_PORT_RCV_DMA           SP_STR_MON_PORT_DMA2SP
-#define SP_STR_MON_PORT_SND_ISP           SP_STR_MON_PORT_SP2ISP
-#define SP_STR_MON_PORT_RCV_ISP           SP_STR_MON_PORT_ISP2SP
-#define SP_STR_MON_PORT_SND_GPD           SP_STR_MON_PORT_SP2GPD
-#define SP_STR_MON_PORT_RCV_GPD           SP_STR_MON_PORT_FA2SP
-/* Deprecated */
-#define SP_STR_MON_PORT_SND_PIF           SP_STR_MON_PORT_SP2PIFA
-#define SP_STR_MON_PORT_RCV_PIF           SP_STR_MON_PORT_PIFA2SP
-#define SP_STR_MON_PORT_SND_PIFB          SP_STR_MON_PORT_SP2PIFB
-#define SP_STR_MON_PORT_RCV_PIFB          SP_STR_MON_PORT_PIFB2SP
-
-#define SP_STR_MON_PORT_SND_PIF_A         SP_STR_MON_PORT_SP2PIFA
-#define SP_STR_MON_PORT_RCV_PIF_A         SP_STR_MON_PORT_PIFA2SP
-#define SP_STR_MON_PORT_SND_PIF_B         SP_STR_MON_PORT_SP2PIFB
-#define SP_STR_MON_PORT_RCV_PIF_B         SP_STR_MON_PORT_PIFB2SP
-
-/* port definititions ISP streaming monitor, monitors the status of streaming ports at the ISP side of the streaming FIFO's */
-#define ISP_STR_MON_PORT_ISP2PIFA         0
-#define ISP_STR_MON_PORT_PIFA2ISP         1
-#define ISP_STR_MON_PORT_ISP2PIFB         2 
-#define ISP_STR_MON_PORT_PIFB2ISP         3
-#define ISP_STR_MON_PORT_ISP2DMA          4 
-#define ISP_STR_MON_PORT_DMA2ISP          5
-#define ISP_STR_MON_PORT_ISP2GDC1         6 
-#define ISP_STR_MON_PORT_GDC12ISP         7
-#define ISP_STR_MON_PORT_ISP2GDC2         8 
-#define ISP_STR_MON_PORT_GDC22ISP         9
-#define ISP_STR_MON_PORT_ISP2GPD         10 
-#define ISP_STR_MON_PORT_FA2ISP          11
-#define ISP_STR_MON_PORT_ISP2SP          12 
-#define ISP_STR_MON_PORT_SP2ISP          13
-
-/* previously used ISP streaming monitor port identifiers, kept for backward compatibility */
-#define ISP_STR_MON_PORT_SND_PIF_A       ISP_STR_MON_PORT_ISP2PIFA
-#define ISP_STR_MON_PORT_RCV_PIF_A       ISP_STR_MON_PORT_PIFA2ISP
-#define ISP_STR_MON_PORT_SND_PIF_B       ISP_STR_MON_PORT_ISP2PIFB 
-#define ISP_STR_MON_PORT_RCV_PIF_B       ISP_STR_MON_PORT_PIFB2ISP
-#define ISP_STR_MON_PORT_SND_DMA         ISP_STR_MON_PORT_ISP2DMA  
-#define ISP_STR_MON_PORT_RCV_DMA         ISP_STR_MON_PORT_DMA2ISP 
-#define ISP_STR_MON_PORT_SND_GDC         ISP_STR_MON_PORT_ISP2GDC1 
-#define ISP_STR_MON_PORT_RCV_GDC         ISP_STR_MON_PORT_GDC12ISP
-#define ISP_STR_MON_PORT_SND_GPD         ISP_STR_MON_PORT_ISP2GPD 
-#define ISP_STR_MON_PORT_RCV_GPD         ISP_STR_MON_PORT_FA2ISP
-#define ISP_STR_MON_PORT_SND_SP          ISP_STR_MON_PORT_ISP2SP
-#define ISP_STR_MON_PORT_RCV_SP          ISP_STR_MON_PORT_SP2ISP
-                                           
-/* port definititions MOD streaming monitor, monitors the status of streaming ports at the module side of the streaming FIFO's */
-
-#define MOD_STR_MON_PORT_PIFA2CELLS       0
-#define MOD_STR_MON_PORT_CELLS2PIFA       1
-#define MOD_STR_MON_PORT_PIFB2CELLS       2
-#define MOD_STR_MON_PORT_CELLS2PIFB       3
-#define MOD_STR_MON_PORT_SIF2SP           4
-#define MOD_STR_MON_PORT_SP2SIF           5
-#define MOD_STR_MON_PORT_MC2SP            6
-#define MOD_STR_MON_PORT_SP2MC            7
-#define MOD_STR_MON_PORT_DMA2ISP          8
-#define MOD_STR_MON_PORT_ISP2DMA          9
-#define MOD_STR_MON_PORT_DMA2SP          10
-#define MOD_STR_MON_PORT_SP2DMA          11
-#define MOD_STR_MON_PORT_GDC12CELLS      12
-#define MOD_STR_MON_PORT_CELLS2GDC1      13
-#define MOD_STR_MON_PORT_GDC22CELLS      14
-#define MOD_STR_MON_PORT_CELLS2GDC2      15
-
-#define MOD_STR_MON_PORT_SND_PIF_A        0
-#define MOD_STR_MON_PORT_RCV_PIF_A        1
-#define MOD_STR_MON_PORT_SND_PIF_B        2
-#define MOD_STR_MON_PORT_RCV_PIF_B        3
-#define MOD_STR_MON_PORT_SND_SIF          4
-#define MOD_STR_MON_PORT_RCV_SIF          5
-#define MOD_STR_MON_PORT_SND_MC           6
-#define MOD_STR_MON_PORT_RCV_MC           7
-#define MOD_STR_MON_PORT_SND_DMA2ISP      8
-#define MOD_STR_MON_PORT_RCV_DMA_FR_ISP   9
-#define MOD_STR_MON_PORT_SND_DMA2SP      10
-#define MOD_STR_MON_PORT_RCV_DMA_FR_SP   11
-#define MOD_STR_MON_PORT_SND_GDC         12
-#define MOD_STR_MON_PORT_RCV_GDC         13
-
-
-/* testbench signals:       */
-
-/* testbench GP adapter register ids  */
-#define HIVE_TESTBENCH_GPIO_DATA_OUT_REG_IDX                    0
-#define HIVE_TESTBENCH_GPIO_DIR_OUT_REG_IDX                     1
-#define HIVE_TESTBENCH_IRQ_REG_IDX                              2
-#define HIVE_TESTBENCH_SDRAM_WAKEUP_REG_IDX                     3
-#define HIVE_TESTBENCH_IDLE_REG_IDX                             4
-#define HIVE_TESTBENCH_GPIO_DATA_IN_REG_IDX                     5
-#define HIVE_TESTBENCH_MIPI_BFM_EN_REG_IDX                      6
-#define HIVE_TESTBENCH_CSI_CONFIG_REG_IDX                       7 
-#define HIVE_TESTBENCH_DDR_STALL_EN_REG_IDX                     8
-
-#define HIVE_TESTBENCH_ISP_PMEM_ERROR_IRQ_REG_IDX               9
-#define HIVE_TESTBENCH_ISP_BAMEM_ERROR_IRQ_REG_IDX             10
-#define HIVE_TESTBENCH_ISP_DMEM_ERROR_IRQ_REG_IDX              11
-#define HIVE_TESTBENCH_SP_ICACHE_MEM_ERROR_IRQ_REG_IDX         12
-#define HIVE_TESTBENCH_SP_DMEM_ERROR_IRQ_REG_IDX               13
-
-/* Signal monitor input bit ids */
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_O_BIT_ID                0
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_1_BIT_ID                1
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_2_BIT_ID                2
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_3_BIT_ID                3
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_4_BIT_ID                4
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_5_BIT_ID                5
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_6_BIT_ID                6
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_7_BIT_ID                7
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_8_BIT_ID                8
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_9_BIT_ID                9
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_10_BIT_ID              10
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_11_BIT_ID              11
-#define HIVE_TESTBENCH_SIG_MON_IRQ_PIN_BIT_ID                  12
-#define HIVE_TESTBENCH_SIG_MON_SDRAM_WAKEUP_PIN_BIT_ID         13
-#define HIVE_TESTBENCH_SIG_MON_IDLE_PIN_BIT_ID                 14
-
-#define ISP2400_DEBUG_NETWORK    1
-
-#endif /* _hive_isp_css_defs_h__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_host_ids_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_host_ids_hrt.h
deleted file mode 100644
index f4d033e221cc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_host_ids_hrt.h
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _hive_isp_css_host_ids_hrt_h_
-#define _hive_isp_css_host_ids_hrt_h_
-
-/* ISP_CSS identifiers */
-#define INP_SYS       testbench_isp_inp_sys
-#define ISYS_GP_REGS  testbench_isp_inp_sys_gpreg
-#define ISYS_IRQ_CTRL testbench_isp_inp_sys_irq_ctrl
-#define ISYS_CAP_A    testbench_isp_inp_sys_capt_unit_a
-#define ISYS_CAP_B    testbench_isp_inp_sys_capt_unit_b
-#define ISYS_CAP_C    testbench_isp_inp_sys_capt_unit_c
-#define ISYS_INP_BUF  testbench_isp_inp_sys_input_buffer
-#define ISYS_INP_CTRL testbench_isp_inp_sys_inp_ctrl
-#define ISYS_ACQ      testbench_isp_inp_sys_acq_unit
-
-#define ISP           testbench_isp_isp
-#define SP            testbench_isp_scp
-
-#define IF_PRIM       testbench_isp_ifmt_ift_prim  
-#define IF_PRIM_B     testbench_isp_ifmt_ift_prim_b
-#define IF_SEC        testbench_isp_ifmt_ift_sec
-#define IF_SEC_MASTER testbench_isp_ifmt_ift_sec_mt_out
-#define STR_TO_MEM    testbench_isp_ifmt_mem_cpy
-#define IFMT_GP_REGS  testbench_isp_ifmt_gp_reg
-#define IFMT_IRQ_CTRL testbench_isp_ifmt_irq_ctrl
-
-#define CSS_RECEIVER  testbench_isp_inp_sys_csi_receiver
-
-#define TC            testbench_isp_gpd_tc
-#define GPTIMER       testbench_isp_gpd_gptimer
-#define DMA           testbench_isp_isp_dma
-#define GDC           testbench_isp_gdc1
-#define GDC2          testbench_isp_gdc2
-#define IRQ_CTRL      testbench_isp_gpd_irq_ctrl
-#define GPIO          testbench_isp_gpd_c_gpio
-#define GP_REGS       testbench_isp_gpd_gp_reg
-#define ISEL_GP_REGS  testbench_isp_isel_gpr
-#define ISEL_IRQ_CTRL testbench_isp_isel_irq_ctrl
-#define DATA_MMU      testbench_isp_data_out_sys_c_mmu
-#define ICACHE_MMU    testbench_isp_icache_out_sys_c_mmu
-
-/* next is actually not FIFO but FIFO adapter, or slave to streaming adapter */
-#define ISP_SP_FIFO   testbench_isp_fa_sp_isp
-#define ISEL_FIFO     testbench_isp_isel_sf_fa_in
-
-#define FIFO_GPF_SP   testbench_isp_sf_fa2sp_in
-#define FIFO_GPF_ISP  testbench_isp_sf_fa2isp_in
-#define FIFO_SP_GPF   testbench_isp_sf_sp2fa_in
-#define FIFO_ISP_GPF  testbench_isp_sf_isp2fa_in
-
-#define DATA_OCP_MASTER    testbench_isp_data_out_sys_cio2ocp_wide_data_out_mt
-#define ICACHE_OCP_MASTER  testbench_isp_icache_out_sys_cio2ocp_wide_data_out_mt
-
-#define SP_IN_FIFO    testbench_isp_sf_fa2sp_in
-#define SP_OUT_FIFO   testbench_isp_sf_sp2fa_out
-#define ISP_IN_FIFO   testbench_isp_sf_fa2isp_in
-#define ISP_OUT_FIFO  testbench_isp_sf_isp2fa_out
-#define GEN_SHORT_PACK_PORT testbench_isp_inp_sys_csi_str_mon_fa_gensh_out
-#define ISYS_GP_REGS  testbench_isp_inp_sys_gpreg
-
-/* Testbench identifiers */
-#define DDR             testbench_ddram
-#define DDR_SMALL       testbench_ddram_small
-#define XMEM            DDR
-#define GPIO_ADAPTER    testbench_gp_adapter
-#define SIG_MONITOR     testbench_sig_mon
-#define DDR_SLAVE       testbench_ddram_ip0
-#define DDR_SMALL_SLAVE testbench_ddram_small_ip0
-#define HOST_MASTER     host_op0
-
-#endif /* _hive_isp_css_host_ids_hrt_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_irq_types_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_irq_types_hrt.h
deleted file mode 100644
index 04c237083835..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_irq_types_hrt.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HIVE_ISP_CSS_IRQ_TYPES_HRT_H_
-#define _HIVE_ISP_CSS_IRQ_TYPES_HRT_H_
-
-/*
- * These are the indices of each interrupt in the interrupt
- * controller's registers. these can be used as the irq_id
- * argument to the hrt functions irq_controller.h.
- *
- * The definitions are taken from <system>_defs.h
- */
-typedef enum hrt_isp_css_irq {
-  hrt_isp_css_irq_gpio_pin_0           = HIVE_GP_DEV_IRQ_GPIO_PIN_0_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_1           = HIVE_GP_DEV_IRQ_GPIO_PIN_1_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_2           = HIVE_GP_DEV_IRQ_GPIO_PIN_2_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_3           = HIVE_GP_DEV_IRQ_GPIO_PIN_3_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_4           = HIVE_GP_DEV_IRQ_GPIO_PIN_4_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_5           = HIVE_GP_DEV_IRQ_GPIO_PIN_5_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_6           = HIVE_GP_DEV_IRQ_GPIO_PIN_6_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_7           = HIVE_GP_DEV_IRQ_GPIO_PIN_7_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_8           = HIVE_GP_DEV_IRQ_GPIO_PIN_8_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_9           = HIVE_GP_DEV_IRQ_GPIO_PIN_9_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_10          = HIVE_GP_DEV_IRQ_GPIO_PIN_10_BIT_ID         ,              
-  hrt_isp_css_irq_gpio_pin_11          = HIVE_GP_DEV_IRQ_GPIO_PIN_11_BIT_ID         ,              
-  hrt_isp_css_irq_sp                   = HIVE_GP_DEV_IRQ_SP_BIT_ID                  ,                       
-  hrt_isp_css_irq_isp                  = HIVE_GP_DEV_IRQ_ISP_BIT_ID                 ,                      
-  hrt_isp_css_irq_isys                 = HIVE_GP_DEV_IRQ_ISYS_BIT_ID                ,                     
-  hrt_isp_css_irq_isel                 = HIVE_GP_DEV_IRQ_ISEL_BIT_ID                ,                     
-  hrt_isp_css_irq_ifmt                 = HIVE_GP_DEV_IRQ_IFMT_BIT_ID                ,                     
-  hrt_isp_css_irq_sp_stream_mon        = HIVE_GP_DEV_IRQ_SP_STREAM_MON_BIT_ID       ,            
-  hrt_isp_css_irq_isp_stream_mon       = HIVE_GP_DEV_IRQ_ISP_STREAM_MON_BIT_ID      ,           
-  hrt_isp_css_irq_mod_stream_mon       = HIVE_GP_DEV_IRQ_MOD_STREAM_MON_BIT_ID      ,
-#ifdef _HIVE_ISP_CSS_2401_SYSTEM
-  hrt_isp_css_irq_is2401               = HIVE_GP_DEV_IRQ_IS2401_BIT_ID              ,           
-#else
-  hrt_isp_css_irq_isp_pmem_error       = HIVE_GP_DEV_IRQ_ISP_PMEM_ERROR_BIT_ID      ,           
-#endif
-  hrt_isp_css_irq_isp_bamem_error      = HIVE_GP_DEV_IRQ_ISP_BAMEM_ERROR_BIT_ID     ,          
-  hrt_isp_css_irq_isp_dmem_error       = HIVE_GP_DEV_IRQ_ISP_DMEM_ERROR_BIT_ID      ,           
-  hrt_isp_css_irq_sp_icache_mem_error  = HIVE_GP_DEV_IRQ_SP_ICACHE_MEM_ERROR_BIT_ID ,      
-  hrt_isp_css_irq_sp_dmem_error        = HIVE_GP_DEV_IRQ_SP_DMEM_ERROR_BIT_ID       ,            
-  hrt_isp_css_irq_mmu_cache_mem_error  = HIVE_GP_DEV_IRQ_MMU_CACHE_MEM_ERROR_BIT_ID ,      
-  hrt_isp_css_irq_gp_timer_0           = HIVE_GP_DEV_IRQ_GP_TIMER_0_BIT_ID          ,               
-  hrt_isp_css_irq_gp_timer_1           = HIVE_GP_DEV_IRQ_GP_TIMER_1_BIT_ID          ,               
-  hrt_isp_css_irq_sw_pin_0             = HIVE_GP_DEV_IRQ_SW_PIN_0_BIT_ID            ,                 
-  hrt_isp_css_irq_sw_pin_1             = HIVE_GP_DEV_IRQ_SW_PIN_1_BIT_ID            ,                 
-  hrt_isp_css_irq_dma                  = HIVE_GP_DEV_IRQ_DMA_BIT_ID                 ,
-  hrt_isp_css_irq_sp_stream_mon_b      = HIVE_GP_DEV_IRQ_SP_STREAM_MON_B_BIT_ID     ,
-  /* this must (obviously) be the last on in the enum */
-  hrt_isp_css_irq_num_irqs
-} hrt_isp_css_irq_t;
-
-typedef enum hrt_isp_css_irq_status {
-  hrt_isp_css_irq_status_error,
-  hrt_isp_css_irq_status_more_irqs,
-  hrt_isp_css_irq_status_success
-} hrt_isp_css_irq_status_t;
-
-#endif /* _HIVE_ISP_CSS_IRQ_TYPES_HRT_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
deleted file mode 100644
index b4211a0c631a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _hive_isp_css_streaming_to_mipi_types_hrt_h_
-#define _hive_isp_css_streaming_to_mipi_types_hrt_h_
-
-#include <streaming_to_mipi_defs.h>
-
-#define _HIVE_ISP_CH_ID_MASK    ((1U << HIVE_ISP_CH_ID_BITS)-1)
-#define _HIVE_ISP_FMT_TYPE_MASK ((1U << HIVE_ISP_FMT_TYPE_BITS)-1)
-
-#define _HIVE_STR_TO_MIPI_FMT_TYPE_LSB (HIVE_STR_TO_MIPI_CH_ID_LSB + HIVE_ISP_CH_ID_BITS)
-#define _HIVE_STR_TO_MIPI_DATA_B_LSB   (HIVE_STR_TO_MIPI_DATA_A_LSB + HIVE_IF_PIXEL_WIDTH)
- 
-#endif /* _hive_isp_css_streaming_to_mipi_types_hrt_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_types.h
deleted file mode 100644
index 58b0e6effbd0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/hive_types.h
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_HIVE_TYPES_H 
-#define _HRT_HIVE_TYPES_H 
-
-#include "version.h"
-#include "defs.h"
-
-#ifndef HRTCAT3
-#define _HRTCAT3(m,n,o)     m##n##o
-#define HRTCAT3(m,n,o)      _HRTCAT3(m,n,o)
-#endif
-
-#ifndef HRTCAT4
-#define _HRTCAT4(m,n,o,p)     m##n##o##p
-#define HRTCAT4(m,n,o,p)      _HRTCAT4(m,n,o,p)
-#endif
-
-#ifndef HRTMIN
-#define HRTMIN(a,b) (((a)<(b))?(a):(b))
-#endif
-                                 
-#ifndef HRTMAX
-#define HRTMAX(a,b) (((a)>(b))?(a):(b))
-#endif
-
-/* boolean data type */
-typedef unsigned int hive_bool;
-#define hive_false 0
-#define hive_true  1
-
-typedef char                 hive_int8;
-typedef short                hive_int16;
-typedef int                  hive_int32;
-typedef long long            hive_int64;
-
-typedef unsigned char        hive_uint8;
-typedef unsigned short       hive_uint16;
-typedef unsigned int         hive_uint32;
-typedef unsigned long long   hive_uint64;
-
-/* by default assume 32 bit master port (both data and address) */
-#ifndef HRT_DATA_WIDTH
-#define HRT_DATA_WIDTH 32
-#endif
-#ifndef HRT_ADDRESS_WIDTH
-#define HRT_ADDRESS_WIDTH 32
-#endif
-
-#define HRT_DATA_BYTES    (HRT_DATA_WIDTH/8)
-#define HRT_ADDRESS_BYTES (HRT_ADDRESS_WIDTH/8)
-
-#if HRT_DATA_WIDTH == 64
-typedef hive_uint64 hrt_data;
-#elif HRT_DATA_WIDTH == 32
-typedef hive_uint32 hrt_data;
-#else
-#error data width not supported
-#endif
-
-#if HRT_ADDRESS_WIDTH == 64
-typedef hive_uint64 hrt_address; 
-#elif HRT_ADDRESS_WIDTH == 32
-typedef hive_uint32 hrt_address;
-#else
-#error adddres width not supported
-#endif
-
-/* The SP side representation of an HMM virtual address */
-typedef hive_uint32 hrt_vaddress;
-
-/* use 64 bit addresses in simulation, where possible */
-typedef hive_uint64  hive_sim_address;
-
-/* below is for csim, not for hrt, rename and move this elsewhere */
-
-typedef unsigned int hive_uint;
-typedef hive_uint32  hive_address;
-typedef hive_address hive_slave_address;
-typedef hive_address hive_mem_address;
-
-/* MMIO devices */
-typedef hive_uint    hive_mmio_id;
-typedef hive_mmio_id hive_slave_id;
-typedef hive_mmio_id hive_port_id;
-typedef hive_mmio_id hive_master_id; 
-typedef hive_mmio_id hive_mem_id;
-typedef hive_mmio_id hive_dev_id;
-typedef hive_mmio_id hive_fifo_id;
-
-typedef hive_uint      hive_hier_id;
-typedef hive_hier_id   hive_device_id;
-typedef hive_device_id hive_proc_id;
-typedef hive_device_id hive_cell_id;
-typedef hive_device_id hive_host_id;
-typedef hive_device_id hive_bus_id;
-typedef hive_device_id hive_bridge_id;
-typedef hive_device_id hive_fifo_adapter_id;
-typedef hive_device_id hive_custom_device_id;
-
-typedef hive_uint hive_slot_id;
-typedef hive_uint hive_fu_id;
-typedef hive_uint hive_reg_file_id;
-typedef hive_uint hive_reg_id;
-
-/* Streaming devices */
-typedef hive_uint hive_outport_id;
-typedef hive_uint hive_inport_id;
-
-typedef hive_uint hive_msink_id;
-
-/* HRT specific */
-typedef char* hive_program;
-typedef char* hive_function;
-
-#endif /* _HRT_HIVE_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/if_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/if_defs.h
deleted file mode 100644
index 7d39e45796ae..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/if_defs.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IF_DEFS_H
-#define _IF_DEFS_H
-
-#define HIVE_IF_FRAME_REQUEST        0xA000
-#define HIVE_IF_LINES_REQUEST        0xB000
-#define HIVE_IF_VECTORS_REQUEST      0xC000
-
-#endif /* _IF_DEFS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_formatter_subsystem_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_formatter_subsystem_defs.h
deleted file mode 100644
index 7766f78cd123..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_formatter_subsystem_defs.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _if_subsystem_defs_h__
-#define _if_subsystem_defs_h__
-
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_0            0
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_1            1
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_2            2
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_3            3
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_4            4
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_5            5
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_6            6
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_7            7 
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_FSYNC_LUT_REG        8
-#define HIVE_IFMT_GP_REGS_SRST_IDX                          9
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IDX                 10
-
-#define HIVE_IFMT_GP_REGS_CH_ID_FMT_TYPE_IDX               11
-
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_BASE         HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_0
-
-/* order of the input bits for the ifmt irq controller */
-#define HIVE_IFMT_IRQ_IFT_PRIM_BIT_ID                       0
-#define HIVE_IFMT_IRQ_IFT_PRIM_B_BIT_ID                     1
-#define HIVE_IFMT_IRQ_IFT_SEC_BIT_ID                        2
-#define HIVE_IFMT_IRQ_MEM_CPY_BIT_ID                        3
-#define HIVE_IFMT_IRQ_SIDEBAND_CHANGED_BIT_ID               4
-
-/* order of the input bits for the ifmt Soft reset register */
-#define HIVE_IFMT_GP_REGS_SRST_IFT_PRIM_BIT_IDX             0
-#define HIVE_IFMT_GP_REGS_SRST_IFT_PRIM_B_BIT_IDX           1
-#define HIVE_IFMT_GP_REGS_SRST_IFT_SEC_BIT_IDX              2
-#define HIVE_IFMT_GP_REGS_SRST_MEM_CPY_BIT_IDX              3
-
-/* order of the input bits for the ifmt Soft reset register */
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IFT_PRIM_BIT_IDX     0
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IFT_PRIM_B_BIT_IDX   1
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IFT_SEC_BIT_IDX      2
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_MEM_CPY_BIT_IDX      3
-
-#endif /* _if_subsystem_defs_h__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_selector_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_selector_defs.h
deleted file mode 100644
index 87fbf82edb5b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_selector_defs.h
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_selector_defs_h
-#define _input_selector_defs_h
-
-#ifndef HIVE_ISP_ISEL_SEL_BITS
-#define HIVE_ISP_ISEL_SEL_BITS                                  2
-#endif
-
-#ifndef HIVE_ISP_CH_ID_BITS
-#define HIVE_ISP_CH_ID_BITS                                     2
-#endif
-
-#ifndef HIVE_ISP_FMT_TYPE_BITS
-#define HIVE_ISP_FMT_TYPE_BITS                                  5
-#endif
-
-/* gp_register register id's -- Outputs */
-#define HIVE_ISEL_GP_REGS_SYNCGEN_ENABLE_IDX                    0
-#define HIVE_ISEL_GP_REGS_SYNCGEN_FREE_RUNNING_IDX              1
-#define HIVE_ISEL_GP_REGS_SYNCGEN_PAUSE_IDX                     2
-#define HIVE_ISEL_GP_REGS_SYNCGEN_NR_FRAMES_IDX                 3 
-#define HIVE_ISEL_GP_REGS_SYNCGEN_NR_PIX_IDX                    4      
-#define HIVE_ISEL_GP_REGS_SYNCGEN_NR_LINES_IDX                  5      
-#define HIVE_ISEL_GP_REGS_SYNCGEN_HBLANK_CYCLES_IDX             6      
-#define HIVE_ISEL_GP_REGS_SYNCGEN_VBLANK_CYCLES_IDX             7      
-
-#define HIVE_ISEL_GP_REGS_SOF_IDX                               8 
-#define HIVE_ISEL_GP_REGS_EOF_IDX                               9 
-#define HIVE_ISEL_GP_REGS_SOL_IDX                              10 
-#define HIVE_ISEL_GP_REGS_EOL_IDX                              11 
-
-#define HIVE_ISEL_GP_REGS_PRBS_ENABLE                          12      
-#define HIVE_ISEL_GP_REGS_PRBS_ENABLE_PORT_B                   13      
-#define HIVE_ISEL_GP_REGS_PRBS_LFSR_RESET_VALUE                14      
-
-#define HIVE_ISEL_GP_REGS_TPG_ENABLE                           15      
-#define HIVE_ISEL_GP_REGS_TPG_ENABLE_PORT_B                    16      
-#define HIVE_ISEL_GP_REGS_TPG_HOR_CNT_MASK_IDX                 17      
-#define HIVE_ISEL_GP_REGS_TPG_VER_CNT_MASK_IDX                 18      
-#define HIVE_ISEL_GP_REGS_TPG_XY_CNT_MASK_IDX                  19      
-#define HIVE_ISEL_GP_REGS_TPG_HOR_CNT_DELTA_IDX                20      
-#define HIVE_ISEL_GP_REGS_TPG_VER_CNT_DELTA_IDX                21      
-#define HIVE_ISEL_GP_REGS_TPG_MODE_IDX                         22     
-#define HIVE_ISEL_GP_REGS_TPG_R1_IDX                           23 
-#define HIVE_ISEL_GP_REGS_TPG_G1_IDX                           24
-#define HIVE_ISEL_GP_REGS_TPG_B1_IDX                           25
-#define HIVE_ISEL_GP_REGS_TPG_R2_IDX                           26
-#define HIVE_ISEL_GP_REGS_TPG_G2_IDX                           27
-#define HIVE_ISEL_GP_REGS_TPG_B2_IDX                           28
-
-
-#define HIVE_ISEL_GP_REGS_CH_ID_IDX                            29
-#define HIVE_ISEL_GP_REGS_FMT_TYPE_IDX                         30
-#define HIVE_ISEL_GP_REGS_DATA_SEL_IDX                         31
-#define HIVE_ISEL_GP_REGS_SBAND_SEL_IDX                        32
-#define HIVE_ISEL_GP_REGS_SYNC_SEL_IDX                         33
-#define HIVE_ISEL_GP_REGS_SRST_IDX                             37
-
-#define HIVE_ISEL_GP_REGS_SRST_SYNCGEN_BIT                      0
-#define HIVE_ISEL_GP_REGS_SRST_PRBS_BIT                         1
-#define HIVE_ISEL_GP_REGS_SRST_TPG_BIT                          2
-#define HIVE_ISEL_GP_REGS_SRST_FIFO_BIT                         3
-
-/* gp_register register id's -- Inputs   */
-#define HIVE_ISEL_GP_REGS_SYNCGEN_HOR_CNT_IDX                  34
-#define HIVE_ISEL_GP_REGS_SYNCGEN_VER_CNT_IDX                  35
-#define HIVE_ISEL_GP_REGS_SYNCGEN_FRAMES_CNT_IDX               36
-
-/* irq sources isel irq controller */
-#define HIVE_ISEL_IRQ_SYNC_GEN_SOF_BIT_ID                       0
-#define HIVE_ISEL_IRQ_SYNC_GEN_EOF_BIT_ID                       1
-#define HIVE_ISEL_IRQ_SYNC_GEN_SOL_BIT_ID                       2
-#define HIVE_ISEL_IRQ_SYNC_GEN_EOL_BIT_ID                       3
-#define HIVE_ISEL_IRQ_NUM_IRQS                                  4
-
-#endif /* _input_selector_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_switch_2400_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_switch_2400_defs.h
deleted file mode 100644
index 20a13c4cdb56..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_switch_2400_defs.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_switch_2400_defs_h
-#define _input_switch_2400_defs_h
-
-#define _HIVE_INPUT_SWITCH_GET_LUT_REG_ID(ch_id, fmt_type) (((ch_id)*2) + ((fmt_type)>=16))
-#define _HIVE_INPUT_SWITCH_GET_LUT_REG_LSB(fmt_type)        (((fmt_type)%16) * 2)
-
-#define HIVE_INPUT_SWITCH_SELECT_NO_OUTPUT   0
-#define HIVE_INPUT_SWITCH_SELECT_IF_PRIM     1
-#define HIVE_INPUT_SWITCH_SELECT_IF_SEC      2
-#define HIVE_INPUT_SWITCH_SELECT_STR_TO_MEM  3
-#define HIVE_INPUT_SWITCH_VSELECT_NO_OUTPUT  0
-#define HIVE_INPUT_SWITCH_VSELECT_IF_PRIM    1
-#define HIVE_INPUT_SWITCH_VSELECT_IF_SEC     2
-#define HIVE_INPUT_SWITCH_VSELECT_STR_TO_MEM 4
-
-#endif /* _input_switch_2400_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_system_ctrl_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_system_ctrl_defs.h
deleted file mode 100644
index a7f0ca80bc9b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_system_ctrl_defs.h
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_system_ctrl_defs_h
-#define _input_system_ctrl_defs_h
-
-#define _INPUT_SYSTEM_CTRL_REG_ALIGN                    4  /* assuming 32 bit control bus width */
-
-/* --------------------------------------------------*/
-
-/* --------------------------------------------------*/
-/* REGISTER INFO */
-/* --------------------------------------------------*/
-
-// Number of registers
-#define ISYS_CTRL_NOF_REGS                              23
-
-// Register id's of MMIO slave accesible registers
-#define ISYS_CTRL_CAPT_START_ADDR_A_REG_ID              0
-#define ISYS_CTRL_CAPT_START_ADDR_B_REG_ID              1
-#define ISYS_CTRL_CAPT_START_ADDR_C_REG_ID              2
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_A_REG_ID         3
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_B_REG_ID         4
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_C_REG_ID         5
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_A_REG_ID         6
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_B_REG_ID         7
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_C_REG_ID         8
-#define ISYS_CTRL_ACQ_START_ADDR_REG_ID                 9
-#define ISYS_CTRL_ACQ_MEM_REGION_SIZE_REG_ID            10
-#define ISYS_CTRL_ACQ_NUM_MEM_REGIONS_REG_ID            11
-#define ISYS_CTRL_INIT_REG_ID                           12
-#define ISYS_CTRL_LAST_COMMAND_REG_ID                   13
-#define ISYS_CTRL_NEXT_COMMAND_REG_ID                   14
-#define ISYS_CTRL_LAST_ACKNOWLEDGE_REG_ID               15
-#define ISYS_CTRL_NEXT_ACKNOWLEDGE_REG_ID               16
-#define ISYS_CTRL_FSM_STATE_INFO_REG_ID                 17
-#define ISYS_CTRL_CAPT_A_FSM_STATE_INFO_REG_ID          18
-#define ISYS_CTRL_CAPT_B_FSM_STATE_INFO_REG_ID          19
-#define ISYS_CTRL_CAPT_C_FSM_STATE_INFO_REG_ID          20
-#define ISYS_CTRL_ACQ_FSM_STATE_INFO_REG_ID             21
-#define ISYS_CTRL_CAPT_RESERVE_ONE_MEM_REGION_REG_ID    22
- 
-
-/* register reset value */
-#define ISYS_CTRL_CAPT_START_ADDR_A_REG_RSTVAL           0
-#define ISYS_CTRL_CAPT_START_ADDR_B_REG_RSTVAL           0
-#define ISYS_CTRL_CAPT_START_ADDR_C_REG_RSTVAL           0
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_A_REG_RSTVAL      128
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_B_REG_RSTVAL      128
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_C_REG_RSTVAL      128
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_A_REG_RSTVAL      3 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_B_REG_RSTVAL      3 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_C_REG_RSTVAL      3 
-#define ISYS_CTRL_ACQ_START_ADDR_REG_RSTVAL              0
-#define ISYS_CTRL_ACQ_MEM_REGION_SIZE_REG_RSTVAL         128 
-#define ISYS_CTRL_ACQ_NUM_MEM_REGIONS_REG_RSTVAL         3 
-#define ISYS_CTRL_INIT_REG_RSTVAL                        0
-#define ISYS_CTRL_LAST_COMMAND_REG_RSTVAL                15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)  
-#define ISYS_CTRL_NEXT_COMMAND_REG_RSTVAL                15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)
-#define ISYS_CTRL_LAST_ACKNOWLEDGE_REG_RSTVAL            15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)
-#define ISYS_CTRL_NEXT_ACKNOWLEDGE_REG_RSTVAL            15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)
-#define ISYS_CTRL_FSM_STATE_INFO_REG_RSTVAL              0
-#define ISYS_CTRL_CAPT_A_FSM_STATE_INFO_REG_RSTVAL       0 
-#define ISYS_CTRL_CAPT_B_FSM_STATE_INFO_REG_RSTVAL       0
-#define ISYS_CTRL_CAPT_C_FSM_STATE_INFO_REG_RSTVAL       0
-#define ISYS_CTRL_ACQ_FSM_STATE_INFO_REG_RSTVAL          0
-#define ISYS_CTRL_CAPT_RESERVE_ONE_MEM_REGION_REG_RSTVAL 0
-
-/* register width value */
-#define ISYS_CTRL_CAPT_START_ADDR_A_REG_WIDTH            9 
-#define ISYS_CTRL_CAPT_START_ADDR_B_REG_WIDTH            9 
-#define ISYS_CTRL_CAPT_START_ADDR_C_REG_WIDTH            9 
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_A_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_B_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_C_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_A_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_B_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_C_REG_WIDTH       9 
-#define ISYS_CTRL_ACQ_START_ADDR_REG_WIDTH               9 
-#define ISYS_CTRL_ACQ_MEM_REGION_SIZE_REG_WIDTH          9 
-#define ISYS_CTRL_ACQ_NUM_MEM_REGIONS_REG_WIDTH          9 
-#define ISYS_CTRL_INIT_REG_WIDTH                         3 
-#define ISYS_CTRL_LAST_COMMAND_REG_WIDTH                 32    /* slave data width */
-#define ISYS_CTRL_NEXT_COMMAND_REG_WIDTH                 32
-#define ISYS_CTRL_LAST_ACKNOWLEDGE_REG_WIDTH             32
-#define ISYS_CTRL_NEXT_ACKNOWLEDGE_REG_WIDTH             32
-#define ISYS_CTRL_FSM_STATE_INFO_REG_WIDTH               32
-#define ISYS_CTRL_CAPT_A_FSM_STATE_INFO_REG_WIDTH        32
-#define ISYS_CTRL_CAPT_B_FSM_STATE_INFO_REG_WIDTH        32
-#define ISYS_CTRL_CAPT_C_FSM_STATE_INFO_REG_WIDTH        32
-#define ISYS_CTRL_ACQ_FSM_STATE_INFO_REG_WIDTH           32
-#define ISYS_CTRL_CAPT_RESERVE_ONE_MEM_REGION_REG_WIDTH  1
-
-/* bit definitions */
-
-/* --------------------------------------------------*/
-/* TOKEN INFO */
-/* --------------------------------------------------*/
-
-/*
-InpSysCaptFramesAcq  1/0  [3:0] - 'b0000
-[7:4] - CaptPortId,
-           CaptA-'b0000
-           CaptB-'b0001
-           CaptC-'b0010
-[31:16] - NOF_frames
-InpSysCaptFrameExt  2/0  [3:0] - 'b0001'
-[7:4] - CaptPortId,
-           'b0000 - CaptA 
-           'b0001 - CaptB
-           'b0010 - CaptC
-
-  2/1  [31:0] - external capture address
-InpSysAcqFrame  2/0  [3:0] - 'b0010, 
-[31:4] - NOF_ext_mem_words
-  2/1  [31:0] - external memory read start address
-InpSysOverruleON  1/0  [3:0] - 'b0011, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysOverruleOFF  1/0  [3:0] - 'b0100, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysOverruleCmd  2/0  [3:0] - 'b0101, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-  2/1  [31:0] - command token value for port opid
-
-
-acknowledge tokens:
-
-InpSysAckCFA  1/0   [3:0] - 'b0000
- [7:4] - CaptPortId,
-           CaptA-'b0000
-           CaptB- 'b0001
-           CaptC-'b0010
- [31:16] - NOF_frames
-InpSysAckCFE  1/0  [3:0] - 'b0001'
-[7:4] - CaptPortId,
-           'b0000 - CaptA 
-           'b0001 - CaptB
-           'b0010 - CaptC
-
-InpSysAckAF  1/0  [3:0] - 'b0010
-InpSysAckOverruleON  1/0  [3:0] - 'b0011, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysAckOverruleOFF  1/0  [3:0] - 'b0100, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysAckOverrule  2/0  [3:0] - 'b0101, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-  2/1  [31:0] - acknowledge token value from port opid
-
-
-
-*/
-
-
-/* Command and acknowledge tokens IDs */
-#define ISYS_CTRL_CAPT_FRAMES_ACQ_TOKEN_ID        0 /* 0000b */
-#define ISYS_CTRL_CAPT_FRAME_EXT_TOKEN_ID         1 /* 0001b */
-#define ISYS_CTRL_ACQ_FRAME_TOKEN_ID              2 /* 0010b */
-#define ISYS_CTRL_OVERRULE_ON_TOKEN_ID            3 /* 0011b */
-#define ISYS_CTRL_OVERRULE_OFF_TOKEN_ID           4 /* 0100b */
-#define ISYS_CTRL_OVERRULE_TOKEN_ID               5 /* 0101b */
-
-#define ISYS_CTRL_ACK_CFA_TOKEN_ID                0
-#define ISYS_CTRL_ACK_CFE_TOKEN_ID                1
-#define ISYS_CTRL_ACK_AF_TOKEN_ID                 2
-#define ISYS_CTRL_ACK_OVERRULE_ON_TOKEN_ID        3
-#define ISYS_CTRL_ACK_OVERRULE_OFF_TOKEN_ID       4
-#define ISYS_CTRL_ACK_OVERRULE_TOKEN_ID           5
-#define ISYS_CTRL_ACK_DEVICE_ERROR_TOKEN_ID       6
-
-#define ISYS_CTRL_TOKEN_ID_MSB                    3
-#define ISYS_CTRL_TOKEN_ID_LSB                    0
-#define ISYS_CTRL_PORT_ID_TOKEN_MSB               7
-#define ISYS_CTRL_PORT_ID_TOKEN_LSB               4
-#define ISYS_CTRL_NOF_CAPT_TOKEN_MSB              31
-#define ISYS_CTRL_NOF_CAPT_TOKEN_LSB              16
-#define ISYS_CTRL_NOF_EXT_TOKEN_MSB               31
-#define ISYS_CTRL_NOF_EXT_TOKEN_LSB               8
-
-#define ISYS_CTRL_TOKEN_ID_IDX                    0
-#define ISYS_CTRL_TOKEN_ID_BITS                   (ISYS_CTRL_TOKEN_ID_MSB - ISYS_CTRL_TOKEN_ID_LSB + 1)
-#define ISYS_CTRL_PORT_ID_IDX                     (ISYS_CTRL_TOKEN_ID_IDX + ISYS_CTRL_TOKEN_ID_BITS)
-#define ISYS_CTRL_PORT_ID_BITS                    (ISYS_CTRL_PORT_ID_TOKEN_MSB - ISYS_CTRL_PORT_ID_TOKEN_LSB +1)
-#define ISYS_CTRL_NOF_CAPT_IDX                    ISYS_CTRL_NOF_CAPT_TOKEN_LSB    
-#define ISYS_CTRL_NOF_CAPT_BITS                   (ISYS_CTRL_NOF_CAPT_TOKEN_MSB - ISYS_CTRL_NOF_CAPT_TOKEN_LSB + 1)
-#define ISYS_CTRL_NOF_EXT_IDX                     ISYS_CTRL_NOF_EXT_TOKEN_LSB    
-#define ISYS_CTRL_NOF_EXT_BITS                    (ISYS_CTRL_NOF_EXT_TOKEN_MSB - ISYS_CTRL_NOF_EXT_TOKEN_LSB + 1)
-
-#define ISYS_CTRL_PORT_ID_CAPT_A                  0 /* device ID for capture unit A      */
-#define ISYS_CTRL_PORT_ID_CAPT_B                  1 /* device ID for capture unit B      */
-#define ISYS_CTRL_PORT_ID_CAPT_C                  2 /* device ID for capture unit C      */
-#define ISYS_CTRL_PORT_ID_ACQUISITION             3 /* device ID for acquistion unit     */
-#define ISYS_CTRL_PORT_ID_DMA_CAPT_A              4 /* device ID for dma unit            */
-#define ISYS_CTRL_PORT_ID_DMA_CAPT_B              5 /* device ID for dma unit            */
-#define ISYS_CTRL_PORT_ID_DMA_CAPT_C              6 /* device ID for dma unit            */
-#define ISYS_CTRL_PORT_ID_DMA_ACQ                 7 /* device ID for dma unit            */
-
-#define ISYS_CTRL_NO_ACQ_ACK                      16 /* no ack from acquisition unit */
-#define ISYS_CTRL_NO_DMA_ACK                      0 
-#define ISYS_CTRL_NO_CAPT_ACK                     16
-
-#endif /* _input_system_ctrl_defs_h */ 
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_system_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_system_defs.h
deleted file mode 100644
index ae62163034a6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/input_system_defs.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_system_defs_h
-#define _input_system_defs_h
-
-/* csi controller modes */
-#define HIVE_CSI_CONFIG_MAIN                   0
-#define HIVE_CSI_CONFIG_STEREO1                4
-#define HIVE_CSI_CONFIG_STEREO2                8
-
-/* general purpose register IDs */
-
-/* Stream Multicast select modes */
-#define HIVE_ISYS_GPREG_MULTICAST_A_IDX           0
-#define HIVE_ISYS_GPREG_MULTICAST_B_IDX           1
-#define HIVE_ISYS_GPREG_MULTICAST_C_IDX           2
-
-/* Stream Mux select modes */
-#define HIVE_ISYS_GPREG_MUX_IDX                   3
-
-/* streaming monitor status and control */
-#define HIVE_ISYS_GPREG_STRMON_STAT_IDX           4
-#define HIVE_ISYS_GPREG_STRMON_COND_IDX           5
-#define HIVE_ISYS_GPREG_STRMON_IRQ_EN_IDX         6
-#define HIVE_ISYS_GPREG_SRST_IDX                  7
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_IDX          8
-#define HIVE_ISYS_GPREG_REG_PORT_A_IDX            9
-#define HIVE_ISYS_GPREG_REG_PORT_B_IDX            10
-
-/* Bit numbers of the soft reset register */
-#define HIVE_ISYS_GPREG_SRST_CAPT_FIFO_A_BIT      0
-#define HIVE_ISYS_GPREG_SRST_CAPT_FIFO_B_BIT      1
-#define HIVE_ISYS_GPREG_SRST_CAPT_FIFO_C_BIT      2
-#define HIVE_ISYS_GPREG_SRST_MULTICAST_A_BIT      3
-#define HIVE_ISYS_GPREG_SRST_MULTICAST_B_BIT      4
-#define HIVE_ISYS_GPREG_SRST_MULTICAST_C_BIT      5
-#define HIVE_ISYS_GPREG_SRST_CAPT_A_BIT           6
-#define HIVE_ISYS_GPREG_SRST_CAPT_B_BIT           7
-#define HIVE_ISYS_GPREG_SRST_CAPT_C_BIT           8
-#define HIVE_ISYS_GPREG_SRST_ACQ_BIT              9
-/* For ISYS_CTRL 5bits are defined to allow soft-reset per sub-controller and top-ctrl */
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_BIT        10  /*LSB for 5bit vector */
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_CAPT_A_BIT 10
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_CAPT_B_BIT 11
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_CAPT_C_BIT 12
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_ACQ_BIT    13
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_TOP_BIT    14
-/* -- */
-#define HIVE_ISYS_GPREG_SRST_STR_MUX_BIT          15
-#define HIVE_ISYS_GPREG_SRST_CIO2AHB_BIT          16
-#define HIVE_ISYS_GPREG_SRST_GEN_SHORT_FIFO_BIT   17
-#define HIVE_ISYS_GPREG_SRST_WIDE_BUS_BIT         18 // includes CIO conv
-#define HIVE_ISYS_GPREG_SRST_DMA_BIT              19
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_CAPT_A_BIT   20
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_CAPT_B_BIT   21
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_CAPT_C_BIT   22
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_ACQ_BIT      23
-#define HIVE_ISYS_GPREG_SRST_CSI_BE_OUT_BIT       24
-
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_CAPT_A_BIT    0
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_CAPT_B_BIT    1
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_CAPT_C_BIT    2
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_ACQ_BIT       3
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_DMA_BIT        4
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_ISYS_CTRL_BIT  5
-
-/* streaming monitor port id's */
-#define HIVE_ISYS_STR_MON_PORT_CAPA            0
-#define HIVE_ISYS_STR_MON_PORT_CAPB            1
-#define HIVE_ISYS_STR_MON_PORT_CAPC            2
-#define HIVE_ISYS_STR_MON_PORT_ACQ             3
-#define HIVE_ISYS_STR_MON_PORT_CSS_GENSH       4
-#define HIVE_ISYS_STR_MON_PORT_SF_GENSH        5
-#define HIVE_ISYS_STR_MON_PORT_SP2ISYS         6
-#define HIVE_ISYS_STR_MON_PORT_ISYS2SP         7
-#define HIVE_ISYS_STR_MON_PORT_PIXA            8
-#define HIVE_ISYS_STR_MON_PORT_PIXB            9
-
-/* interrupt bit ID's        */
-#define HIVE_ISYS_IRQ_CSI_SOF_BIT_ID           0
-#define HIVE_ISYS_IRQ_CSI_EOF_BIT_ID           1
-#define HIVE_ISYS_IRQ_CSI_SOL_BIT_ID           2
-#define HIVE_ISYS_IRQ_CSI_EOL_BIT_ID           3
-#define HIVE_ISYS_IRQ_CSI_RECEIVER_BIT_ID      4
-#define HIVE_ISYS_IRQ_CSI_RECEIVER_BE_BIT_ID   5
-#define HIVE_ISYS_IRQ_CAP_UNIT_A_NO_SOP        6
-#define HIVE_ISYS_IRQ_CAP_UNIT_A_LATE_SOP      7
-/*#define HIVE_ISYS_IRQ_CAP_UNIT_A_UNDEF_PH      7*/
-#define HIVE_ISYS_IRQ_CAP_UNIT_B_NO_SOP        8
-#define HIVE_ISYS_IRQ_CAP_UNIT_B_LATE_SOP      9
-/*#define HIVE_ISYS_IRQ_CAP_UNIT_B_UNDEF_PH     10*/
-#define HIVE_ISYS_IRQ_CAP_UNIT_C_NO_SOP       10
-#define HIVE_ISYS_IRQ_CAP_UNIT_C_LATE_SOP     11
-/*#define HIVE_ISYS_IRQ_CAP_UNIT_C_UNDEF_PH     13*/
-#define HIVE_ISYS_IRQ_ACQ_UNIT_SOP_MISMATCH   12
-/*#define HIVE_ISYS_IRQ_ACQ_UNIT_UNDEF_PH       15*/
-#define HIVE_ISYS_IRQ_INP_CTRL_CAPA           13
-#define HIVE_ISYS_IRQ_INP_CTRL_CAPB           14
-#define HIVE_ISYS_IRQ_INP_CTRL_CAPC           15
-#define HIVE_ISYS_IRQ_CIO2AHB                 16
-#define HIVE_ISYS_IRQ_DMA_BIT_ID              17
-#define HIVE_ISYS_IRQ_STREAM_MON_BIT_ID       18
-#define HIVE_ISYS_IRQ_NUM_BITS                19
-
-/* DMA */
-#define HIVE_ISYS_DMA_CHANNEL                  0
-#define HIVE_ISYS_DMA_IBUF_DDR_CONN            0
-#define HIVE_ISYS_DMA_HEIGHT                   1
-#define HIVE_ISYS_DMA_ELEMS                    1 /* both master buses of same width */
-#define HIVE_ISYS_DMA_STRIDE                   0 /* no stride required as height is fixed to 1 */
-#define HIVE_ISYS_DMA_CROP                     0 /* no cropping */
-#define HIVE_ISYS_DMA_EXTENSION                0 /* no extension as elem width is same on both side */
-
-#endif /* _input_system_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/irq_controller_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/irq_controller_defs.h
deleted file mode 100644
index ec6dd4487158..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/irq_controller_defs.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _irq_controller_defs_h
-#define _irq_controller_defs_h
-
-#define _HRT_IRQ_CONTROLLER_EDGE_REG_IDX           0
-#define _HRT_IRQ_CONTROLLER_MASK_REG_IDX           1
-#define _HRT_IRQ_CONTROLLER_STATUS_REG_IDX         2
-#define _HRT_IRQ_CONTROLLER_CLEAR_REG_IDX          3
-#define _HRT_IRQ_CONTROLLER_ENABLE_REG_IDX         4
-#define _HRT_IRQ_CONTROLLER_EDGE_NOT_PULSE_REG_IDX 5
-#define _HRT_IRQ_CONTROLLER_STR_OUT_ENABLE_REG_IDX 6
-
-#define _HRT_IRQ_CONTROLLER_REG_ALIGN 4
-
-#endif /* _irq_controller_defs_h */   
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp2400_mamoiada_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp2400_mamoiada_params.h
deleted file mode 100644
index 669060d17c4f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp2400_mamoiada_params.h
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Version */
-#define RTL_VERSION
-
-/* Cell name  */
-#define ISP_CELL_TYPE                          isp2400_mamoiada
-#define ISP_VMEM                               simd_vmem
-#define _HRT_ISP_VMEM                          isp2400_mamoiada_simd_vmem
-
-/* instruction pipeline depth */
-#define ISP_BRANCHDELAY                        5
-
-/* bus */
-#define ISP_BUS_WIDTH                          32
-#define ISP_BUS_ADDR_WIDTH                     32
-#define ISP_BUS_BURST_SIZE                     1
-
-/* data-path */
-#define ISP_SCALAR_WIDTH                       32
-#define ISP_SLICE_NELEMS                       4
-#define ISP_VEC_NELEMS                         64
-#define ISP_VEC_ELEMBITS                       14
-#define ISP_VEC_ELEM8BITS                      16
-#define ISP_CLONE_DATAPATH_IS_16               1
-
-/* memories */
-#define ISP_DMEM_DEPTH                         4096
-#define ISP_DMEM_BSEL_DOWNSAMPLE               8
-#define ISP_VMEM_DEPTH                         3072
-#define ISP_VMEM_BSEL_DOWNSAMPLE               8
-#define ISP_VMEM_ELEMBITS                      14
-#define ISP_VMEM_ELEM_PRECISION                14
-#define ISP_VMEM_IS_BAMEM                      1
-#if ISP_VMEM_IS_BAMEM
-  #define ISP_VMEM_BAMEM_MAX_BOI_HEIGHT        8
-  #define ISP_VMEM_BAMEM_LATENCY               5
-  #define ISP_VMEM_BAMEM_BANK_NARROWING_FACTOR 2
-  #define ISP_VMEM_BAMEM_NR_DATA_PLANES        8
-  #define ISP_VMEM_BAMEM_NR_CFG_REGISTERS      16
-  #define ISP_VMEM_BAMEM_LININT                0
-  #define ISP_VMEM_BAMEM_DAP_BITS              3
-  #define ISP_VMEM_BAMEM_LININT_FRAC_BITS      0
-  #define ISP_VMEM_BAMEM_PID_BITS              3
-  #define ISP_VMEM_BAMEM_OFFSET_BITS           19
-  #define ISP_VMEM_BAMEM_ADDRESS_BITS          25
-  #define ISP_VMEM_BAMEM_RID_BITS              4
-  #define ISP_VMEM_BAMEM_TRANSPOSITION         1
-  #define ISP_VMEM_BAMEM_VEC_PLUS_SLICE        1
-  #define ISP_VMEM_BAMEM_ARB_SERVICE_CYCLE_BITS 1
-  #define ISP_VMEM_BAMEM_LUT_ELEMS             16
-  #define ISP_VMEM_BAMEM_LUT_ADDR_WIDTH        14
-  #define ISP_VMEM_BAMEM_HALF_BLOCK_WRITE      1
-  #define ISP_VMEM_BAMEM_SMART_FETCH           1
-  #define ISP_VMEM_BAMEM_BIG_ENDIANNESS        0
-#endif /* ISP_VMEM_IS_BAMEM */
-#define ISP_PMEM_DEPTH                         2048
-#define ISP_PMEM_WIDTH                         640
-#define ISP_VAMEM_ADDRESS_BITS                 12
-#define ISP_VAMEM_ELEMBITS                     12
-#define ISP_VAMEM_DEPTH                        2048
-#define ISP_VAMEM_ALIGNMENT                    2
-#define ISP_VA_ADDRESS_WIDTH                   896
-#define ISP_VEC_VALSU_LATENCY                  ISP_VEC_NELEMS
-#define ISP_HIST_ADDRESS_BITS                  12
-#define ISP_HIST_ALIGNMENT                     4
-#define ISP_HIST_COMP_IN_PREC                  12
-#define ISP_HIST_DEPTH                         1024
-#define ISP_HIST_WIDTH                         24
-#define ISP_HIST_COMPONENTS                    4
-
-/* program counter */
-#define ISP_PC_WIDTH                           13
-
-/* Template switches */
-#define ISP_SHIELD_INPUT_DMEM                  0
-#define ISP_SHIELD_OUTPUT_DMEM                 1
-#define ISP_SHIELD_INPUT_VMEM                  0
-#define ISP_SHIELD_OUTPUT_VMEM                 0
-#define ISP_SHIELD_INPUT_PMEM                  1
-#define ISP_SHIELD_OUTPUT_PMEM                 1
-#define ISP_SHIELD_INPUT_HIST                  1
-#define ISP_SHIELD_OUTPUT_HIST                 1
-/* When LUT is select the shielding is always on */
-#define ISP_SHIELD_INPUT_VAMEM                 1
-#define ISP_SHIELD_OUTPUT_VAMEM                1
-
-#define ISP_HAS_IRQ                            1
-#define ISP_HAS_SOFT_RESET                     1
-#define ISP_HAS_VEC_DIV                        0
-#define ISP_HAS_VFU_W_2O                       1
-#define ISP_HAS_DEINT3                         1
-#define ISP_HAS_LUT                            1
-#define ISP_HAS_HIST                           1
-#define ISP_HAS_VALSU                          1
-#define ISP_HAS_3rdVALSU                       1
-#define ISP_VRF1_HAS_2P                        1
-
-#define ISP_SRU_GUARDING                       1
-#define ISP_VLSU_GUARDING                      1
-
-#define ISP_VRF_RAM     	                     1
-#define ISP_SRF_RAM     	                     1
-
-#define ISP_SPLIT_VMUL_VADD_IS                 0
-#define ISP_RFSPLIT_FPGA                       0
-
-/* RSN or Bus pipelining */
-#define ISP_RSN_PIPE                           1
-#define ISP_VSF_BUS_PIPE                       0
-
-/* extra slave port to vmem */
-#define ISP_IF_VMEM                            0
-#define ISP_GDC_VMEM                           0
-
-/* Streaming ports */
-#define ISP_IF                                 1
-#define ISP_IF_B                               1
-#define ISP_GDC                                1
-#define ISP_SCL                                1
-#define ISP_GPFIFO                             1
-#define ISP_SP                                 1
-
-/* Removing Issue Slot(s) */
-#define ISP_HAS_NOT_SIMD_IS2                   0
-#define ISP_HAS_NOT_SIMD_IS3                   0
-#define ISP_HAS_NOT_SIMD_IS4                   0
-#define ISP_HAS_NOT_SIMD_IS4_VADD              0
-#define ISP_HAS_NOT_SIMD_IS5                   0
-#define ISP_HAS_NOT_SIMD_IS6                   0
-#define ISP_HAS_NOT_SIMD_IS7                   0
-#define ISP_HAS_NOT_SIMD_IS8                   0
-
-/* ICache  */
-#define ISP_ICACHE                             1
-#define ISP_ICACHE_ONLY                        0
-#define ISP_ICACHE_PREFETCH                    1
-#define ISP_ICACHE_INDEX_BITS                  8
-#define ISP_ICACHE_SET_BITS                    5
-#define ISP_ICACHE_BLOCKS_PER_SET_BITS         1
-
-/* Experimental Flags */
-#define ISP_EXP_1                              0
-#define ISP_EXP_2                              0
-#define ISP_EXP_3                              0
-#define ISP_EXP_4                              0
-#define ISP_EXP_5                              0
-#define ISP_EXP_6                              0
-
-/* Derived values */
-#define ISP_LOG2_PMEM_WIDTH                    10
-#define ISP_VEC_WIDTH                          896
-#define ISP_SLICE_WIDTH                        56
-#define ISP_VMEM_WIDTH                         896
-#define ISP_VMEM_ALIGN                         128
-#if ISP_VMEM_IS_BAMEM
-  #define ISP_VMEM_ALIGN_ELEM                  2
-#endif /* ISP_VMEM_IS_BAMEM */
-#define ISP_SIMDLSU                            1
-#define ISP_LSU_IMM_BITS                       12
-
-/* convenient shortcuts for software*/
-#define ISP_NWAY                               ISP_VEC_NELEMS
-#define NBITS                                  ISP_VEC_ELEMBITS
-
-#define _isp_ceil_div(a,b)                     (((a)+(b)-1)/(b))
-
-#define ISP_VEC_ALIGN                          ISP_VMEM_ALIGN
-
-/* HRT specific vector support */
-#define isp2400_mamoiada_vector_alignment         ISP_VEC_ALIGN
-#define isp2400_mamoiada_vector_elem_bits         ISP_VMEM_ELEMBITS
-#define isp2400_mamoiada_vector_elem_precision    ISP_VMEM_ELEM_PRECISION
-#define isp2400_mamoiada_vector_num_elems         ISP_VEC_NELEMS
-
-/* register file sizes */
-#define ISP_RF0_SIZE        64
-#define ISP_RF1_SIZE        16
-#define ISP_RF2_SIZE        64
-#define ISP_RF3_SIZE        4
-#define ISP_RF4_SIZE        64
-#define ISP_RF5_SIZE        16
-#define ISP_RF6_SIZE        16
-#define ISP_RF7_SIZE        16
-#define ISP_RF8_SIZE        16
-#define ISP_RF9_SIZE        16
-#define ISP_RF10_SIZE       16
-#define ISP_RF11_SIZE       16
-#define ISP_VRF1_SIZE       24
-#define ISP_VRF2_SIZE       24
-#define ISP_VRF3_SIZE       24
-#define ISP_VRF4_SIZE       24
-#define ISP_VRF5_SIZE       24
-#define ISP_VRF6_SIZE       24
-#define ISP_VRF7_SIZE       24
-#define ISP_VRF8_SIZE       24
-#define ISP_SRF1_SIZE       4
-#define ISP_SRF2_SIZE       64
-#define ISP_SRF3_SIZE       64
-#define ISP_SRF4_SIZE       32
-#define ISP_SRF5_SIZE       64
-#define ISP_FRF0_SIZE       16
-#define ISP_FRF1_SIZE       4
-#define ISP_FRF2_SIZE       16
-#define ISP_FRF3_SIZE       4
-#define ISP_FRF4_SIZE       4
-#define ISP_FRF5_SIZE       8
-#define ISP_FRF6_SIZE       4
-/* register file read latency */
-#define ISP_VRF1_READ_LAT       1
-#define ISP_VRF2_READ_LAT       1
-#define ISP_VRF3_READ_LAT       1
-#define ISP_VRF4_READ_LAT       1
-#define ISP_VRF5_READ_LAT       1
-#define ISP_VRF6_READ_LAT       1
-#define ISP_VRF7_READ_LAT       1
-#define ISP_VRF8_READ_LAT       1
-#define ISP_SRF1_READ_LAT       1
-#define ISP_SRF2_READ_LAT       1
-#define ISP_SRF3_READ_LAT       1
-#define ISP_SRF4_READ_LAT       1
-#define ISP_SRF5_READ_LAT       1
-#define ISP_SRF5_READ_LAT       1
-/* immediate sizes */
-#define ISP_IS1_IMM_BITS        14
-#define ISP_IS2_IMM_BITS        13
-#define ISP_IS3_IMM_BITS        14
-#define ISP_IS4_IMM_BITS        14
-#define ISP_IS5_IMM_BITS        9
-#define ISP_IS6_IMM_BITS        16
-#define ISP_IS7_IMM_BITS        9
-#define ISP_IS8_IMM_BITS        16
-#define ISP_IS9_IMM_BITS        11
-/* fifo depths */
-#define ISP_IF_FIFO_DEPTH         0
-#define ISP_IF_B_FIFO_DEPTH       0
-#define ISP_DMA_FIFO_DEPTH        0
-#define ISP_OF_FIFO_DEPTH         0
-#define ISP_GDC_FIFO_DEPTH        0
-#define ISP_SCL_FIFO_DEPTH        0
-#define ISP_GPFIFO_FIFO_DEPTH     0
-#define ISP_SP_FIFO_DEPTH         0
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp2400_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp2400_support.h
deleted file mode 100644
index e00bc841d0f0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp2400_support.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _isp2400_support_h
-#define _isp2400_support_h
-
-#ifndef ISP2400_VECTOR_TYPES
-/* This typedef is to be able to include hive header files
-   in the host code which is useful in crun */
-typedef char *tmemvectors, *tmemvectoru, *tvector;
-#endif
-
-#define hrt_isp_vamem1_store_16(cell, addr, val) hrt_mem_store_16(cell, HRT_PROC_TYPE_PROP(cell, _simd_vamem1), addr, val)
-#define hrt_isp_vamem2_store_16(cell, addr, val) hrt_mem_store_16(cell, HRT_PROC_TYPE_PROP(cell, _simd_vamem2), addr, val)
-
-#define hrt_isp_dmem(cell) HRT_PROC_TYPE_PROP(cell, _base_dmem)
-#define hrt_isp_vmem(cell) HRT_PROC_TYPE_PROP(cell, _simd_vmem)
-
-#define hrt_isp_dmem_master_port_address(cell) hrt_mem_master_port_address(cell, hrt_isp_dmem(cell))
-#define hrt_isp_vmem_master_port_address(cell) hrt_mem_master_port_address(cell, hrt_isp_vmem(cell))
-
-#if ISP_HAS_HIST
-  #define hrt_isp_hist(cell) HRT_PROC_TYPE_PROP(cell, _simd_histogram)
-  #define hrt_isp_hist_master_port_address(cell) hrt_mem_master_port_address(cell, hrt_isp_hist(cell))
-#endif
-
-#endif /* _isp2400_support_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp_acquisition_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp_acquisition_defs.h
deleted file mode 100644
index 593620721627..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp_acquisition_defs.h
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _isp_acquisition_defs_h
-#define _isp_acquisition_defs_h
-
-#define _ISP_ACQUISITION_REG_ALIGN                4  /* assuming 32 bit control bus width */
-#define _ISP_ACQUISITION_BYTES_PER_ELEM           4		
-
-/* --------------------------------------------------*/
-
-#define NOF_ACQ_IRQS                              1
-
-/* --------------------------------------------------*/
-/* FSM */
-/* --------------------------------------------------*/
-#define MEM2STREAM_FSM_STATE_BITS                 2
-#define ACQ_SYNCHRONIZER_FSM_STATE_BITS           2
-
-/* --------------------------------------------------*/
-/* REGISTER INFO */
-/* --------------------------------------------------*/
-
-#define NOF_ACQ_REGS                              12      
-
-// Register id's of MMIO slave accesible registers
-#define ACQ_START_ADDR_REG_ID                     0              
-#define ACQ_MEM_REGION_SIZE_REG_ID                1
-#define ACQ_NUM_MEM_REGIONS_REG_ID                2
-#define ACQ_INIT_REG_ID                           3 
-#define ACQ_RECEIVED_SHORT_PACKETS_REG_ID         4
-#define ACQ_RECEIVED_LONG_PACKETS_REG_ID          5
-#define ACQ_LAST_COMMAND_REG_ID                   6
-#define ACQ_NEXT_COMMAND_REG_ID                   7
-#define ACQ_LAST_ACKNOWLEDGE_REG_ID               8
-#define ACQ_NEXT_ACKNOWLEDGE_REG_ID               9
-#define ACQ_FSM_STATE_INFO_REG_ID                 10
-#define ACQ_INT_CNTR_INFO_REG_ID                  11
- 
-// Register width
-#define ACQ_START_ADDR_REG_WIDTH                  9               
-#define ACQ_MEM_REGION_SIZE_REG_WIDTH             9  
-#define ACQ_NUM_MEM_REGIONS_REG_WIDTH             9  
-#define ACQ_INIT_REG_WIDTH                        3  
-#define ACQ_RECEIVED_SHORT_PACKETS_REG_WIDTH      32 
-#define ACQ_RECEIVED_LONG_PACKETS_REG_WIDTH       32  
-#define ACQ_LAST_COMMAND_REG_WIDTH                32  
-#define ACQ_NEXT_COMMAND_REG_WIDTH                32  
-#define ACQ_LAST_ACKNOWLEDGE_REG_WIDTH            32  
-#define ACQ_NEXT_ACKNOWLEDGE_REG_WIDTH            32  
-#define ACQ_FSM_STATE_INFO_REG_WIDTH              ((MEM2STREAM_FSM_STATE_BITS * 3) + (ACQ_SYNCHRONIZER_FSM_STATE_BITS *3))
-#define ACQ_INT_CNTR_INFO_REG_WIDTH               32
-
-/* register reset value */
-#define ACQ_START_ADDR_REG_RSTVAL                 0              
-#define ACQ_MEM_REGION_SIZE_REG_RSTVAL            128
-#define ACQ_NUM_MEM_REGIONS_REG_RSTVAL            3
-#define ACQ_INIT_REG_RSTVAL                       0                           
-#define ACQ_RECEIVED_SHORT_PACKETS_REG_RSTVAL     0
-#define ACQ_RECEIVED_LONG_PACKETS_REG_RSTVAL      0
-#define ACQ_LAST_COMMAND_REG_RSTVAL               0
-#define ACQ_NEXT_COMMAND_REG_RSTVAL               0
-#define ACQ_LAST_ACKNOWLEDGE_REG_RSTVAL           0
-#define ACQ_NEXT_ACKNOWLEDGE_REG_RSTVAL           0 
-#define ACQ_FSM_STATE_INFO_REG_RSTVAL             0
-#define ACQ_INT_CNTR_INFO_REG_RSTVAL              0 
-
-/* bit definitions */
-#define ACQ_INIT_RST_REG_BIT                      0
-#define ACQ_INIT_RESYNC_BIT                       2
-#define ACQ_INIT_RST_IDX                          ACQ_INIT_RST_REG_BIT
-#define ACQ_INIT_RST_BITS                         1
-#define ACQ_INIT_RESYNC_IDX                       ACQ_INIT_RESYNC_BIT
-#define ACQ_INIT_RESYNC_BITS                      1
-
-/* --------------------------------------------------*/
-/* TOKEN INFO */
-/* --------------------------------------------------*/
-#define ACQ_TOKEN_ID_LSB                          0
-#define ACQ_TOKEN_ID_MSB                          3            
-#define ACQ_TOKEN_WIDTH                           (ACQ_TOKEN_ID_MSB - ACQ_TOKEN_ID_LSB  + 1) // 4
-#define ACQ_TOKEN_ID_IDX                          0
-#define ACQ_TOKEN_ID_BITS                         ACQ_TOKEN_WIDTH
-#define ACQ_INIT_CMD_INIT_IDX                     4
-#define ACQ_INIT_CMD_INIT_BITS                    3
-#define ACQ_CMD_START_ADDR_IDX                    4
-#define ACQ_CMD_START_ADDR_BITS                   9
-#define ACQ_CMD_NOFWORDS_IDX                      13
-#define ACQ_CMD_NOFWORDS_BITS                     9  
-#define ACQ_MEM_REGION_ID_IDX                     22
-#define ACQ_MEM_REGION_ID_BITS                    9 
-#define ACQ_PACKET_LENGTH_TOKEN_MSB               21
-#define ACQ_PACKET_LENGTH_TOKEN_LSB               13
-#define ACQ_PACKET_DATA_FORMAT_ID_TOKEN_MSB       9
-#define ACQ_PACKET_DATA_FORMAT_ID_TOKEN_LSB       4
-#define ACQ_PACKET_CH_ID_TOKEN_MSB                11
-#define ACQ_PACKET_CH_ID_TOKEN_LSB                10
-#define ACQ_PACKET_MEM_REGION_ID_TOKEN_MSB        12		/* only for capt_end_of_packet_written */
-#define ACQ_PACKET_MEM_REGION_ID_TOKEN_LSB        4		/* only for capt_end_of_packet_written */
-
-
-/* Command tokens IDs */
-#define ACQ_READ_REGION_AUTO_INCR_TOKEN_ID        0 //0000b
-#define ACQ_READ_REGION_TOKEN_ID                  1 //0001b
-#define ACQ_READ_REGION_SOP_TOKEN_ID              2 //0010b  
-#define ACQ_INIT_TOKEN_ID                         8 //1000b
-
-/* Acknowledge token IDs */
-#define ACQ_READ_REGION_ACK_TOKEN_ID              0 //0000b
-#define ACQ_END_OF_PACKET_TOKEN_ID                4 //0100b
-#define ACQ_END_OF_REGION_TOKEN_ID                5 //0101b
-#define ACQ_SOP_MISMATCH_TOKEN_ID                 6 //0110b
-#define ACQ_UNDEF_PH_TOKEN_ID                     7 //0111b
-
-#define ACQ_TOKEN_MEMREGIONID_MSB                 30
-#define ACQ_TOKEN_MEMREGIONID_LSB                 22
-#define ACQ_TOKEN_NOFWORDS_MSB                    21
-#define ACQ_TOKEN_NOFWORDS_LSB                    13
-#define ACQ_TOKEN_STARTADDR_MSB                   12
-#define ACQ_TOKEN_STARTADDR_LSB                   4  
-
-
-/* --------------------------------------------------*/
-/* MIPI */
-/* --------------------------------------------------*/
-
-#define WORD_COUNT_WIDTH                          16
-#define PKT_CODE_WIDTH                            6            
-#define CHN_NO_WIDTH                              2  
-#define ERROR_INFO_WIDTH                          8
-  
-#define LONG_PKTCODE_MAX                          63
-#define LONG_PKTCODE_MIN                          16
-#define SHORT_PKTCODE_MAX                         15
-
-#define EOF_CODE                                  1
-
-/* --------------------------------------------------*/
-/* Packet Info */
-/* --------------------------------------------------*/
-#define ACQ_START_OF_FRAME                        0
-#define ACQ_END_OF_FRAME                          1
-#define ACQ_START_OF_LINE                         2
-#define ACQ_END_OF_LINE                           3
-#define ACQ_LINE_PAYLOAD                          4
-#define ACQ_GEN_SH_PKT                            5
-
-
-/* bit definition */
-#define ACQ_PKT_TYPE_IDX                          16
-#define ACQ_PKT_TYPE_BITS                         6
-#define ACQ_PKT_SOP_IDX                           32
-#define ACQ_WORD_CNT_IDX                          0
-#define ACQ_WORD_CNT_BITS                         16
-#define ACQ_PKT_INFO_IDX                          16
-#define ACQ_PKT_INFO_BITS                         8
-#define ACQ_HEADER_DATA_IDX                       0
-#define ACQ_HEADER_DATA_BITS                      16
-#define ACQ_ACK_TOKEN_ID_IDX                      ACQ_TOKEN_ID_IDX
-#define ACQ_ACK_TOKEN_ID_BITS                     ACQ_TOKEN_ID_BITS
-#define ACQ_ACK_NOFWORDS_IDX                      13
-#define ACQ_ACK_NOFWORDS_BITS                     9
-#define ACQ_ACK_PKT_LEN_IDX                       4
-#define ACQ_ACK_PKT_LEN_BITS                      16
-
-
-/* --------------------------------------------------*/
-/* Packet Data Type */
-/* --------------------------------------------------*/
-
-
-#define ACQ_YUV420_8_DATA                       24   /* 01 1000 YUV420 8-bit                                        */
-#define ACQ_YUV420_10_DATA                      25   /* 01 1001  YUV420 10-bit                                      */
-#define ACQ_YUV420_8L_DATA                      26   /* 01 1010   YUV420 8-bit legacy                               */
-#define ACQ_YUV422_8_DATA                       30   /* 01 1110   YUV422 8-bit                                      */
-#define ACQ_YUV422_10_DATA                      31   /* 01 1111   YUV422 10-bit                                     */
-#define ACQ_RGB444_DATA                         32   /* 10 0000   RGB444                                            */
-#define ACQ_RGB555_DATA              					 33   /* 10 0001   RGB555                                            */
-#define ACQ_RGB565_DATA              					 34   /* 10 0010   RGB565                                            */
-#define ACQ_RGB666_DATA              					 35   /* 10 0011   RGB666                                            */
-#define ACQ_RGB888_DATA              					 36   /* 10 0100   RGB888                                            */
-#define ACQ_RAW6_DATA                					 40   /* 10 1000   RAW6                                              */
-#define ACQ_RAW7_DATA                					 41   /* 10 1001   RAW7                                              */
-#define ACQ_RAW8_DATA                					 42   /* 10 1010   RAW8                                              */
-#define ACQ_RAW10_DATA               					 43   /* 10 1011   RAW10                                             */
-#define ACQ_RAW12_DATA               					 44   /* 10 1100   RAW12                                             */
-#define ACQ_RAW14_DATA               					 45   /* 10 1101   RAW14                                             */
-#define ACQ_USR_DEF_1_DATA           					 48   /* 11 0000    JPEG [User Defined 8-bit Data Type 1]            */
-#define ACQ_USR_DEF_2_DATA           					 49   /* 11 0001    User Defined 8-bit Data Type 2                   */
-#define ACQ_USR_DEF_3_DATA           					 50   /* 11 0010    User Defined 8-bit Data Type 3                   */
-#define ACQ_USR_DEF_4_DATA           					 51   /* 11 0011    User Defined 8-bit Data Type 4                   */
-#define ACQ_USR_DEF_5_DATA           					 52   /* 11 0100    User Defined 8-bit Data Type 5                   */
-#define ACQ_USR_DEF_6_DATA           					 53   /* 11 0101    User Defined 8-bit Data Type 6                   */
-#define ACQ_USR_DEF_7_DATA           					 54   /* 11 0110    User Defined 8-bit Data Type 7                   */
-#define ACQ_USR_DEF_8_DATA           					 55   /* 11 0111    User Defined 8-bit Data Type 8                   */
-#define ACQ_Emb_DATA                 					 18   /* 01 0010    embedded eight bit non image data                */
-#define ACQ_SOF_DATA                 					 0   /* 00 0000    frame start                                      */
-#define ACQ_EOF_DATA                 					 1   /* 00 0001    frame end                                        */
-#define ACQ_SOL_DATA                 					 2   /* 00 0010    line start                                       */
-#define ACQ_EOL_DATA                 					 3   /* 00 0011    line end                                         */
-#define ACQ_GEN_SH1_DATA             					 8   /* 00 1000  Generic Short Packet Code 1                        */
-#define ACQ_GEN_SH2_DATA             					 9   /* 00 1001    Generic Short Packet Code 2                      */
-#define ACQ_GEN_SH3_DATA             					 10   /* 00 1010    Generic Short Packet Code 3                      */
-#define ACQ_GEN_SH4_DATA             					 11   /* 00 1011    Generic Short Packet Code 4                      */
-#define ACQ_GEN_SH5_DATA             					 12   /* 00 1100    Generic Short Packet Code 5                      */
-#define ACQ_GEN_SH6_DATA             					 13   /* 00 1101    Generic Short Packet Code 6                      */
-#define ACQ_GEN_SH7_DATA             					 14   /* 00 1110    Generic Short Packet Code 7                      */
-#define ACQ_GEN_SH8_DATA             					 15   /* 00 1111    Generic Short Packet Code 8                      */
-#define ACQ_YUV420_8_CSPS_DATA       					 28   /* 01 1100   YUV420 8-bit (Chroma Shifted Pixel Sampling)      */
-#define ACQ_YUV420_10_CSPS_DATA      					 29   /* 01 1101   YUV420 10-bit (Chroma Shifted Pixel Sampling)     */
-#define ACQ_RESERVED_DATA_TYPE_MIN              56
-#define ACQ_RESERVED_DATA_TYPE_MAX              63
-#define ACQ_GEN_LONG_RESERVED_DATA_TYPE_MIN     19
-#define ACQ_GEN_LONG_RESERVED_DATA_TYPE_MAX     23
-#define ACQ_YUV_RESERVED_DATA_TYPE              27
-#define ACQ_RGB_RESERVED_DATA_TYPE_MIN          37
-#define ACQ_RGB_RESERVED_DATA_TYPE_MAX          39
-#define ACQ_RAW_RESERVED_DATA_TYPE_MIN          46
-#define ACQ_RAW_RESERVED_DATA_TYPE_MAX          47
-
-/* --------------------------------------------------*/
-
-#endif /* _isp_acquisition_defs_h */ 
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp_capture_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp_capture_defs.h
deleted file mode 100644
index 0a249ce3e589..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/isp_capture_defs.h
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _isp_capture_defs_h
-#define _isp_capture_defs_h
-
-#define _ISP_CAPTURE_REG_ALIGN                    4  /* assuming 32 bit control bus width */
-#define _ISP_CAPTURE_BITS_PER_ELEM                32  /* only for data, not SOP */						           
-#define _ISP_CAPTURE_BYTES_PER_ELEM               (_ISP_CAPTURE_BITS_PER_ELEM/8	)				           
-#define _ISP_CAPTURE_BYTES_PER_WORD               32		/* 256/8 */	
-#define _ISP_CAPTURE_ELEM_PER_WORD                _ISP_CAPTURE_BYTES_PER_WORD / _ISP_CAPTURE_BYTES_PER_ELEM		           
-
-//#define CAPT_RCV_ACK                              1
-//#define CAPT_WRT_ACK                              2               
-//#define CAPT_IRQ_ACK                              3                        
-
-/* --------------------------------------------------*/
-
-#define NOF_IRQS                                  2
-
-/* --------------------------------------------------*/
-/* REGISTER INFO */
-/* --------------------------------------------------*/
-
-// Number of registers
-#define CAPT_NOF_REGS                             16
-
-// Register id's of MMIO slave accesible registers
-#define CAPT_START_MODE_REG_ID                    0
-#define CAPT_START_ADDR_REG_ID                    1 
-#define CAPT_MEM_REGION_SIZE_REG_ID               2 
-#define CAPT_NUM_MEM_REGIONS_REG_ID               3 
-#define CAPT_INIT_REG_ID                          4 
-#define CAPT_START_REG_ID                         5
-#define CAPT_STOP_REG_ID                          6  
-
-#define CAPT_PACKET_LENGTH_REG_ID                 7
-#define CAPT_RECEIVED_LENGTH_REG_ID               8 
-#define CAPT_RECEIVED_SHORT_PACKETS_REG_ID        9 
-#define CAPT_RECEIVED_LONG_PACKETS_REG_ID         10 
-#define CAPT_LAST_COMMAND_REG_ID                  11        
-#define CAPT_NEXT_COMMAND_REG_ID                  12
-#define CAPT_LAST_ACKNOWLEDGE_REG_ID              13
-#define CAPT_NEXT_ACKNOWLEDGE_REG_ID              14
-#define CAPT_FSM_STATE_INFO_REG_ID                15
-
-// Register width
-#define CAPT_START_MODE_REG_WIDTH                 1 
-#define CAPT_START_ADDR_REG_WIDTH                 9
-#define CAPT_MEM_REGION_SIZE_REG_WIDTH            9
-#define CAPT_NUM_MEM_REGIONS_REG_WIDTH            9
-#define CAPT_INIT_REG_WIDTH                       (18 + 4)
-
-#define CAPT_START_REG_WIDTH                      1
-#define CAPT_STOP_REG_WIDTH                       1
-
-/* --------------------------------------------------*/
-/* FSM */
-/* --------------------------------------------------*/
-#define CAPT_WRITE2MEM_FSM_STATE_BITS             2
-#define CAPT_SYNCHRONIZER_FSM_STATE_BITS          3
-
-
-#define CAPT_PACKET_LENGTH_REG_WIDTH              17
-#define CAPT_RECEIVED_LENGTH_REG_WIDTH            17   
-#define CAPT_RECEIVED_SHORT_PACKETS_REG_WIDTH     32
-#define CAPT_RECEIVED_LONG_PACKETS_REG_WIDTH      32
-#define CAPT_LAST_COMMAND_REG_WIDTH               32
-/* #define CAPT_NEXT_COMMAND_REG_WIDTH               32 */  
-#define CAPT_LAST_ACKNOWLEDGE_REG_WIDTH           32
-#define CAPT_NEXT_ACKNOWLEDGE_REG_WIDTH           32
-#define CAPT_FSM_STATE_INFO_REG_WIDTH             ((CAPT_WRITE2MEM_FSM_STATE_BITS * 3) + (CAPT_SYNCHRONIZER_FSM_STATE_BITS * 3))
-
-#define CAPT_INIT_RESTART_MEM_ADDR_WIDTH          9   
-#define CAPT_INIT_RESTART_MEM_REGION_WIDTH        9 
-
-/* register reset value */
-#define CAPT_START_MODE_REG_RSTVAL                0   
-#define CAPT_START_ADDR_REG_RSTVAL                0
-#define CAPT_MEM_REGION_SIZE_REG_RSTVAL           128
-#define CAPT_NUM_MEM_REGIONS_REG_RSTVAL           3 
-#define CAPT_INIT_REG_RSTVAL                      0
-
-#define CAPT_START_REG_RSTVAL                     0
-#define CAPT_STOP_REG_RSTVAL                      0
-
-#define CAPT_PACKET_LENGTH_REG_RSTVAL             0
-#define CAPT_RECEIVED_LENGTH_REG_RSTVAL           0
-#define CAPT_RECEIVED_SHORT_PACKETS_REG_RSTVAL    0
-#define CAPT_RECEIVED_LONG_PACKETS_REG_RSTVAL     0
-#define CAPT_LAST_COMMAND_REG_RSTVAL              0
-#define CAPT_NEXT_COMMAND_REG_RSTVAL              0
-#define CAPT_LAST_ACKNOWLEDGE_REG_RSTVAL          0
-#define CAPT_NEXT_ACKNOWLEDGE_REG_RSTVAL          0
-#define CAPT_FSM_STATE_INFO_REG_RSTVAL            0
-
-/* bit definitions */
-#define CAPT_INIT_RST_REG_BIT                     0
-#define CAPT_INIT_FLUSH_BIT                       1
-#define CAPT_INIT_RESYNC_BIT                      2
-#define CAPT_INIT_RESTART_BIT                     3
-#define CAPT_INIT_RESTART_MEM_ADDR_LSB            4
-#define CAPT_INIT_RESTART_MEM_ADDR_MSB            12
-#define CAPT_INIT_RESTART_MEM_REGION_LSB          13
-#define CAPT_INIT_RESTART_MEM_REGION_MSB          21
-
-
-#define CAPT_INIT_RST_REG_IDX                     CAPT_INIT_RST_REG_BIT
-#define CAPT_INIT_RST_REG_BITS                    1
-#define CAPT_INIT_FLUSH_IDX                       CAPT_INIT_FLUSH_BIT
-#define CAPT_INIT_FLUSH_BITS                      1
-#define CAPT_INIT_RESYNC_IDX                      CAPT_INIT_RESYNC_BIT
-#define CAPT_INIT_RESYNC_BITS                     1
-#define CAPT_INIT_RESTART_IDX                     CAPT_INIT_RESTART_BIT
-#define CAPT_INIT_RESTART_BITS					  				1
-#define CAPT_INIT_RESTART_MEM_ADDR_IDX            CAPT_INIT_RESTART_MEM_ADDR_LSB
-#define CAPT_INIT_RESTART_MEM_ADDR_BITS           (CAPT_INIT_RESTART_MEM_ADDR_MSB - CAPT_INIT_RESTART_MEM_ADDR_LSB + 1)
-#define CAPT_INIT_RESTART_MEM_REGION_IDX          CAPT_INIT_RESTART_MEM_REGION_LSB
-#define CAPT_INIT_RESTART_MEM_REGION_BITS         (CAPT_INIT_RESTART_MEM_REGION_MSB - CAPT_INIT_RESTART_MEM_REGION_LSB + 1)
-
-
-
-/* --------------------------------------------------*/
-/* TOKEN INFO */
-/* --------------------------------------------------*/
-#define CAPT_TOKEN_ID_LSB                         0
-#define CAPT_TOKEN_ID_MSB                         3            
-#define CAPT_TOKEN_WIDTH                         (CAPT_TOKEN_ID_MSB - CAPT_TOKEN_ID_LSB  + 1) /* 4 */
-
-/* Command tokens IDs */
-#define CAPT_START_TOKEN_ID                       0 /* 0000b */
-#define CAPT_STOP_TOKEN_ID                        1 /* 0001b */
-#define CAPT_FREEZE_TOKEN_ID                      2 /* 0010b */  
-#define CAPT_RESUME_TOKEN_ID                      3 /* 0011b */
-#define CAPT_INIT_TOKEN_ID                        8 /* 1000b */
-
-#define CAPT_START_TOKEN_BIT                      0      
-#define CAPT_STOP_TOKEN_BIT                       0
-#define CAPT_FREEZE_TOKEN_BIT                     0
-#define CAPT_RESUME_TOKEN_BIT                     0
-#define CAPT_INIT_TOKEN_BIT                       0
-
-/* Acknowledge token IDs */
-#define CAPT_END_OF_PACKET_RECEIVED_TOKEN_ID      0 /* 0000b */
-#define CAPT_END_OF_PACKET_WRITTEN_TOKEN_ID       1 /* 0001b */
-#define CAPT_END_OF_REGION_WRITTEN_TOKEN_ID       2 /* 0010b */
-#define CAPT_FLUSH_DONE_TOKEN_ID                  3 /* 0011b */
-#define CAPT_PREMATURE_SOP_TOKEN_ID               4 /* 0100b */
-#define CAPT_MISSING_SOP_TOKEN_ID                 5 /* 0101b */
-#define CAPT_UNDEF_PH_TOKEN_ID                    6 /* 0110b */
-#define CAPT_STOP_ACK_TOKEN_ID                    7 /* 0111b */
-
-#define CAPT_PACKET_LENGTH_TOKEN_MSB             19
-#define CAPT_PACKET_LENGTH_TOKEN_LSB              4
-#define CAPT_SUPER_PACKET_LENGTH_TOKEN_MSB       20
-#define CAPT_SUPER_PACKET_LENGTH_TOKEN_LSB        4
-#define CAPT_PACKET_DATA_FORMAT_ID_TOKEN_MSB     25
-#define CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB     20
-#define CAPT_PACKET_CH_ID_TOKEN_MSB              27
-#define CAPT_PACKET_CH_ID_TOKEN_LSB              26
-#define CAPT_PACKET_MEM_REGION_ID_TOKEN_MSB      29		
-#define CAPT_PACKET_MEM_REGION_ID_TOKEN_LSB      21		
-
-/*  bit definition */
-#define CAPT_CMD_IDX                              CAPT_TOKEN_ID_LSB
-#define	CAPT_CMD_BITS                             (CAPT_TOKEN_ID_MSB - CAPT_TOKEN_ID_LSB + 1)
-#define CAPT_SOP_IDX                              32
-#define CAPT_SOP_BITS                             1
-#define CAPT_PKT_INFO_IDX                         16
-#define CAPT_PKT_INFO_BITS                        8
-#define CAPT_PKT_TYPE_IDX                         0
-#define CAPT_PKT_TYPE_BITS                        6
-#define CAPT_HEADER_DATA_IDX                      0
-#define CAPT_HEADER_DATA_BITS                     16
-#define CAPT_PKT_DATA_IDX                         0
-#define CAPT_PKT_DATA_BITS                        32
-#define CAPT_WORD_CNT_IDX                         0
-#define CAPT_WORD_CNT_BITS                        16
-#define CAPT_ACK_TOKEN_ID_IDX                     0
-#define CAPT_ACK_TOKEN_ID_BITS                    4
-//#define CAPT_ACK_PKT_LEN_IDX                      CAPT_PACKET_LENGTH_TOKEN_LSB
-//#define CAPT_ACK_PKT_LEN_BITS                     (CAPT_PACKET_LENGTH_TOKEN_MSB - CAPT_PACKET_LENGTH_TOKEN_LSB + 1)
-//#define CAPT_ACK_PKT_INFO_IDX                     20
-//#define CAPT_ACK_PKT_INFO_BITS                    8
-//#define CAPT_ACK_MEM_REG_ID1_IDX                  20			/* for capt_end_of_packet_written */
-//#define CAPT_ACK_MEM_REG_ID2_IDX                  4       /* for capt_end_of_region_written */
-#define CAPT_ACK_PKT_LEN_IDX                      CAPT_PACKET_LENGTH_TOKEN_LSB
-#define CAPT_ACK_PKT_LEN_BITS                     (CAPT_PACKET_LENGTH_TOKEN_MSB - CAPT_PACKET_LENGTH_TOKEN_LSB + 1)
-#define CAPT_ACK_SUPER_PKT_LEN_IDX                CAPT_SUPER_PACKET_LENGTH_TOKEN_LSB
-#define CAPT_ACK_SUPER_PKT_LEN_BITS               (CAPT_SUPER_PACKET_LENGTH_TOKEN_MSB - CAPT_SUPER_PACKET_LENGTH_TOKEN_LSB + 1)
-#define CAPT_ACK_PKT_INFO_IDX                     CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB
-#define CAPT_ACK_PKT_INFO_BITS                    (CAPT_PACKET_CH_ID_TOKEN_MSB - CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB + 1)
-#define CAPT_ACK_MEM_REGION_ID_IDX                CAPT_PACKET_MEM_REGION_ID_TOKEN_LSB
-#define CAPT_ACK_MEM_REGION_ID_BITS               (CAPT_PACKET_MEM_REGION_ID_TOKEN_MSB - CAPT_PACKET_MEM_REGION_ID_TOKEN_LSB + 1)
-#define CAPT_ACK_PKT_TYPE_IDX                     CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB
-#define CAPT_ACK_PKT_TYPE_BITS                    (CAPT_PACKET_DATA_FORMAT_ID_TOKEN_MSB - CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB + 1)
-#define CAPT_INIT_TOKEN_INIT_IDX                  4
-#define CAPT_INIT_TOKEN_INIT_BITS                 22
-
-
-/* --------------------------------------------------*/
-/* MIPI */
-/* --------------------------------------------------*/
-
-#define CAPT_WORD_COUNT_WIDTH                     16      
-#define CAPT_PKT_CODE_WIDTH                       6                  
-#define CAPT_CHN_NO_WIDTH                         2        
-#define CAPT_ERROR_INFO_WIDTH                     8       
-
-#define LONG_PKTCODE_MAX                          63
-#define LONG_PKTCODE_MIN                          16
-#define SHORT_PKTCODE_MAX                         15
-
-
-/* --------------------------------------------------*/
-/* Packet Info */
-/* --------------------------------------------------*/
-#define CAPT_START_OF_FRAME                       0
-#define CAPT_END_OF_FRAME                         1
-#define CAPT_START_OF_LINE                        2
-#define CAPT_END_OF_LINE                          3
-#define CAPT_LINE_PAYLOAD                         4
-#define CAPT_GEN_SH_PKT                           5
-
-
-/* --------------------------------------------------*/
-/* Packet Data Type */
-/* --------------------------------------------------*/
-
-#define CAPT_YUV420_8_DATA                       24   /* 01 1000 YUV420 8-bit                                        */
-#define CAPT_YUV420_10_DATA                      25   /* 01 1001  YUV420 10-bit                                      */
-#define CAPT_YUV420_8L_DATA                      26   /* 01 1010   YUV420 8-bit legacy                               */
-#define CAPT_YUV422_8_DATA                       30   /* 01 1110   YUV422 8-bit                                      */
-#define CAPT_YUV422_10_DATA                      31   /* 01 1111   YUV422 10-bit                                     */
-#define CAPT_RGB444_DATA                         32   /* 10 0000   RGB444                                            */
-#define CAPT_RGB555_DATA              					 33   /* 10 0001   RGB555                                            */
-#define CAPT_RGB565_DATA              					 34   /* 10 0010   RGB565                                            */
-#define CAPT_RGB666_DATA              					 35   /* 10 0011   RGB666                                            */
-#define CAPT_RGB888_DATA              					 36   /* 10 0100   RGB888                                            */
-#define CAPT_RAW6_DATA                					 40   /* 10 1000   RAW6                                              */
-#define CAPT_RAW7_DATA                					 41   /* 10 1001   RAW7                                              */
-#define CAPT_RAW8_DATA                					 42   /* 10 1010   RAW8                                              */
-#define CAPT_RAW10_DATA               					 43   /* 10 1011   RAW10                                             */
-#define CAPT_RAW12_DATA               					 44   /* 10 1100   RAW12                                             */
-#define CAPT_RAW14_DATA               					 45   /* 10 1101   RAW14                                             */
-#define CAPT_USR_DEF_1_DATA           					 48   /* 11 0000    JPEG [User Defined 8-bit Data Type 1]            */
-#define CAPT_USR_DEF_2_DATA           					 49   /* 11 0001    User Defined 8-bit Data Type 2                   */
-#define CAPT_USR_DEF_3_DATA           					 50   /* 11 0010    User Defined 8-bit Data Type 3                   */
-#define CAPT_USR_DEF_4_DATA           					 51   /* 11 0011    User Defined 8-bit Data Type 4                   */
-#define CAPT_USR_DEF_5_DATA           					 52   /* 11 0100    User Defined 8-bit Data Type 5                   */
-#define CAPT_USR_DEF_6_DATA           					 53   /* 11 0101    User Defined 8-bit Data Type 6                   */
-#define CAPT_USR_DEF_7_DATA           					 54   /* 11 0110    User Defined 8-bit Data Type 7                   */
-#define CAPT_USR_DEF_8_DATA           					 55   /* 11 0111    User Defined 8-bit Data Type 8                   */
-#define CAPT_Emb_DATA                 					 18   /* 01 0010    embedded eight bit non image data                */
-#define CAPT_SOF_DATA                 					 0   /* 00 0000    frame start                                      */
-#define CAPT_EOF_DATA                 					 1   /* 00 0001    frame end                                        */
-#define CAPT_SOL_DATA                 					 2   /* 00 0010    line start                                       */
-#define CAPT_EOL_DATA                 					 3   /* 00 0011    line end                                         */
-#define CAPT_GEN_SH1_DATA             					 8   /* 00 1000  Generic Short Packet Code 1                        */
-#define CAPT_GEN_SH2_DATA             					 9   /* 00 1001    Generic Short Packet Code 2                      */
-#define CAPT_GEN_SH3_DATA             					 10   /* 00 1010    Generic Short Packet Code 3                      */
-#define CAPT_GEN_SH4_DATA             					 11   /* 00 1011    Generic Short Packet Code 4                      */
-#define CAPT_GEN_SH5_DATA             					 12   /* 00 1100    Generic Short Packet Code 5                      */
-#define CAPT_GEN_SH6_DATA             					 13   /* 00 1101    Generic Short Packet Code 6                      */
-#define CAPT_GEN_SH7_DATA             					 14   /* 00 1110    Generic Short Packet Code 7                      */
-#define CAPT_GEN_SH8_DATA             					 15   /* 00 1111    Generic Short Packet Code 8                      */
-#define CAPT_YUV420_8_CSPS_DATA       					 28   /* 01 1100   YUV420 8-bit (Chroma Shifted Pixel Sampling)      */
-#define CAPT_YUV420_10_CSPS_DATA      					 29   /* 01 1101   YUV420 10-bit (Chroma Shifted Pixel Sampling)     */
-#define CAPT_RESERVED_DATA_TYPE_MIN              56
-#define CAPT_RESERVED_DATA_TYPE_MAX              63
-#define CAPT_GEN_LONG_RESERVED_DATA_TYPE_MIN     19
-#define CAPT_GEN_LONG_RESERVED_DATA_TYPE_MAX     23
-#define CAPT_YUV_RESERVED_DATA_TYPE              27
-#define CAPT_RGB_RESERVED_DATA_TYPE_MIN          37
-#define CAPT_RGB_RESERVED_DATA_TYPE_MAX          39
-#define CAPT_RAW_RESERVED_DATA_TYPE_MIN          46
-#define CAPT_RAW_RESERVED_DATA_TYPE_MAX          47
-
-
-/* --------------------------------------------------*/
-/* Capture Unit State */
-/* --------------------------------------------------*/
-#define CAPT_FREE_RUN                             0
-#define CAPT_NO_SYNC                              1
-#define CAPT_SYNC_SWP                             2
-#define CAPT_SYNC_MWP                             3
-#define CAPT_SYNC_WAIT                            4
-#define CAPT_FREEZE                               5
-#define CAPT_RUN                                  6
-
-
-/* --------------------------------------------------*/
-
-#endif /* _isp_capture_defs_h */ 
-
-
-
-
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/mmu_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/mmu_defs.h
deleted file mode 100644
index c038f39ffd25..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/mmu_defs.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _mmu_defs_h
-#define _mmu_defs_h
-
-#define _HRT_MMU_INVALIDATE_TLB_REG_IDX          0
-#define _HRT_MMU_PAGE_TABLE_BASE_ADDRESS_REG_IDX 1
-
-#define _HRT_MMU_REG_ALIGN 4
-
-#endif /* _mmu_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/scalar_processor_2400_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/scalar_processor_2400_params.h
deleted file mode 100644
index 9b6c2893d950..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/scalar_processor_2400_params.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _scalar_processor_2400_params_h
-#define _scalar_processor_2400_params_h
-
-#include "cell_params.h"
-
-#endif /* _scalar_processor_2400_params_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/str2mem_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/str2mem_defs.h
deleted file mode 100644
index 1cb62444cf68..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/str2mem_defs.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _ST2MEM_DEFS_H
-#define _ST2MEM_DEFS_H
-
-#define _STR2MEM_CRUN_BIT               0x100000
-#define _STR2MEM_CMD_BITS               0x0F0000
-#define _STR2MEM_COUNT_BITS             0x00FFFF
-
-#define _STR2MEM_BLOCKS_CMD             0xA0000
-#define _STR2MEM_PACKETS_CMD            0xB0000
-#define _STR2MEM_BYTES_CMD              0xC0000
-#define _STR2MEM_BYTES_FROM_PACKET_CMD  0xD0000
-
-#define _STR2MEM_SOFT_RESET_REG_ID                   0
-#define _STR2MEM_INPUT_ENDIANNESS_REG_ID             1
-#define _STR2MEM_OUTPUT_ENDIANNESS_REG_ID            2
-#define _STR2MEM_BIT_SWAPPING_REG_ID                 3
-#define _STR2MEM_BLOCK_SYNC_LEVEL_REG_ID             4
-#define _STR2MEM_PACKET_SYNC_LEVEL_REG_ID            5
-#define _STR2MEM_READ_POST_WRITE_SYNC_ENABLE_REG_ID  6
-#define _STR2MEM_DUAL_BYTE_INPUTS_ENABLED_REG_ID     7
-#define _STR2MEM_EN_STAT_UPDATE_ID                   8
-
-#define _STR2MEM_REG_ALIGN      4
-
-#endif /* _ST2MEM_DEFS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/streaming_to_mipi_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/streaming_to_mipi_defs.h
deleted file mode 100644
index 60143b8743a2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/streaming_to_mipi_defs.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _streaming_to_mipi_defs_h
-#define _streaming_to_mipi_defs_h
-
-#define HIVE_STR_TO_MIPI_VALID_A_BIT 0
-#define HIVE_STR_TO_MIPI_VALID_B_BIT 1
-#define HIVE_STR_TO_MIPI_SOL_BIT     2
-#define HIVE_STR_TO_MIPI_EOL_BIT     3
-#define HIVE_STR_TO_MIPI_SOF_BIT     4
-#define HIVE_STR_TO_MIPI_EOF_BIT     5
-#define HIVE_STR_TO_MIPI_CH_ID_LSB   6
-
-#define HIVE_STR_TO_MIPI_DATA_A_LSB  (HIVE_STR_TO_MIPI_VALID_B_BIT + 1)
-
-#endif /* _streaming_to_mipi_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/timed_controller_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/timed_controller_defs.h
deleted file mode 100644
index d2b8972b0d9e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/timed_controller_defs.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _timed_controller_defs_h
-#define _timed_controller_defs_h
-
-#define _HRT_TIMED_CONTROLLER_CMD_REG_IDX 0
-
-#define _HRT_TIMED_CONTROLLER_REG_ALIGN 4
-
-#endif /* _timed_controller_defs_h */   
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/var.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/var.h
deleted file mode 100644
index 5bc0ad34616e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/var.h
+++ /dev/null
@@ -1,74 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_VAR_H
-#define _HRT_VAR_H
-
-#include "version.h"
-#include "system_api.h"
-#include "hive_types.h"
-
-#define hrt_int_type_of_char   char
-#define hrt_int_type_of_uchar  unsigned char
-#define hrt_int_type_of_short  short
-#define hrt_int_type_of_ushort unsigned short
-#define hrt_int_type_of_int    int
-#define hrt_int_type_of_uint   unsigned int
-#define hrt_int_type_of_long   long
-#define hrt_int_type_of_ulong  unsigned long
-#define hrt_int_type_of_ptr    unsigned int
-
-#define hrt_host_type_of_char   char
-#define hrt_host_type_of_uchar  unsigned char
-#define hrt_host_type_of_short  short
-#define hrt_host_type_of_ushort unsigned short
-#define hrt_host_type_of_int    int
-#define hrt_host_type_of_uint   unsigned int
-#define hrt_host_type_of_long   long
-#define hrt_host_type_of_ulong  unsigned long
-#define hrt_host_type_of_ptr    void*
-
-#define HRT_TYPE_BYTES(cell, type) (HRT_TYPE_BITS(cell, type)/8)
-#define HRT_HOST_TYPE(cell_type)   HRTCAT(hrt_host_type_of_, cell_type)
-#define HRT_INT_TYPE(type)         HRTCAT(hrt_int_type_of_, type)
-
-#define hrt_scalar_store(cell, type, var, data) \
-  HRTCAT(hrt_mem_store_,HRT_TYPE_BITS(cell, type))(\
-	       cell, \
-	       HRTCAT(HIVE_MEM_,var), \
-	       HRTCAT(HIVE_ADDR_,var), \
-	       (HRT_INT_TYPE(type))(data))
-
-#define hrt_scalar_load(cell, type, var) \
-  (HRT_HOST_TYPE(type))(HRTCAT4(_hrt_mem_load_,HRT_PROC_TYPE(cell),_,type) ( \
-	       cell, \
-	       HRTCAT(HIVE_MEM_,var), \
-	       HRTCAT(HIVE_ADDR_,var)))
-
-#define hrt_indexed_store(cell, type, array, index, data) \
-  HRTCAT(hrt_mem_store_,HRT_TYPE_BITS(cell, type))(\
-	       cell, \
-	       HRTCAT(HIVE_MEM_,array), \
-	       (HRTCAT(HIVE_ADDR_,array))+((index)*HRT_TYPE_BYTES(cell, type)), \
-	       (HRT_INT_TYPE(type))(data))
-
-#define hrt_indexed_load(cell, type, array, index) \
-  (HRT_HOST_TYPE(type))(HRTCAT4(_hrt_mem_load_,HRT_PROC_TYPE(cell),_,type) ( \
-         cell, \
-	       HRTCAT(HIVE_MEM_,array), \
-	       (HRTCAT(HIVE_ADDR_,array))+((index)*HRT_TYPE_BYTES(cell, type))))
-
-#endif /* _HRT_VAR_H */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/version.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/version.h
deleted file mode 100644
index bbc4948baea9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/hrt/version.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef HRT_VERSION_H
-#define HRT_VERSION_H
-#define HRT_VERSION_MAJOR 1
-#define HRT_VERSION_MINOR 4
-#define HRT_VERSION 1_4
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/spmem_dump.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/spmem_dump.c
deleted file mode 100644
index ddc7a8f05153..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2400_system/spmem_dump.c
+++ /dev/null
@@ -1,3634 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _sp_map_h_
-#define _sp_map_h_
-
-
-#ifndef _hrt_dummy_use_blob_sp
-#define _hrt_dummy_use_blob_sp()
-#endif
-
-#define _hrt_cell_load_program_sp(proc) _hrt_cell_load_program_embedded(proc, sp)
-
-#ifndef ISP2401
-/* function input_system_acquisition_stop: ADE */
-#else
-/* function input_system_acquisition_stop: AD8 */
-#endif
-
-#ifndef ISP2401
-/* function longjmp: 684E */
-#else
-/* function longjmp: 69C1 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_HIVE_IF_SRST_MASK
-#define HIVE_MEM_HIVE_IF_SRST_MASK scalar_processor_2400_dmem
-#define HIVE_ADDR_HIVE_IF_SRST_MASK 0x1C8
-#define HIVE_SIZE_HIVE_IF_SRST_MASK 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_HIVE_IF_SRST_MASK scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_HIVE_IF_SRST_MASK 0x1C8
-#define HIVE_SIZE_sp_HIVE_IF_SRST_MASK 16
-
-#ifndef ISP2401
-/* function tmpmem_init_dmem: 6580 */
-#else
-/* function tmpmem_init_dmem: 66BB */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_receive_ack: 5EC4 */
-#else
-/* function ia_css_isys_sp_token_map_receive_ack: 5FFF */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_addr_B: 332C */
-#else
-/* function ia_css_dmaproxy_sp_set_addr_B: 3520 */
-
-/* function ia_css_pipe_data_init_tagger_resources: A4F */
-#endif
-
-/* function debug_buffer_set_ddr_addr: DD */
-
-#ifndef ISP2401
-/* function receiver_port_reg_load: AC2 */
-#else
-/* function receiver_port_reg_load: ABC */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_mipi
-#define HIVE_MEM_vbuf_mipi scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_mipi 0x631C
-#else
-#define HIVE_ADDR_vbuf_mipi 0x6378
-#endif
-#define HIVE_SIZE_vbuf_mipi 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_mipi scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_mipi 0x631C
-#else
-#define HIVE_ADDR_sp_vbuf_mipi 0x6378
-#endif
-#define HIVE_SIZE_sp_vbuf_mipi 12
-
-#ifndef ISP2401
-/* function ia_css_event_sp_decode: 351D */
-#else
-/* function ia_css_event_sp_decode: 3711 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_get_size: 48A5 */
-#else
-/* function ia_css_queue_get_size: 4B2D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_load: 4EE6 */
-#else
-/* function ia_css_queue_load: 5144 */
-#endif
-
-#ifndef ISP2401
-/* function setjmp: 6857 */
-#else
-/* function setjmp: 69CA */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_sp2host_isys_event_queue
-#define HIVE_MEM_sem_for_sp2host_isys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_sp2host_isys_event_queue 0x4684
-#else
-#define HIVE_ADDR_sem_for_sp2host_isys_event_queue 0x46CC
-#endif
-#define HIVE_SIZE_sem_for_sp2host_isys_event_queue 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_sp2host_isys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_sp2host_isys_event_queue 0x4684
-#else
-#define HIVE_ADDR_sp_sem_for_sp2host_isys_event_queue 0x46CC
-#endif
-#define HIVE_SIZE_sp_sem_for_sp2host_isys_event_queue 20
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_wait_for_ack: 6E07 */
-#else
-/* function ia_css_dmaproxy_sp_wait_for_ack: 6F4B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_func: 510B */
-#else
-/* function ia_css_sp_rawcopy_func: 5369 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_pop_marked: 29F7 */
-#else
-/* function ia_css_tagger_buf_sp_pop_marked: 2B99 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_stage
-#define HIVE_MEM_isp_stage scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_stage 0x5C00
-#else
-#define HIVE_ADDR_isp_stage 0x5C60
-#endif
-#define HIVE_SIZE_isp_stage 832
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_stage scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_stage 0x5C00
-#else
-#define HIVE_ADDR_sp_isp_stage 0x5C60
-#endif
-#define HIVE_SIZE_sp_isp_stage 832
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_raw
-#define HIVE_MEM_vbuf_raw scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_raw 0x2F4
-#else
-#define HIVE_ADDR_vbuf_raw 0x30C
-#endif
-#define HIVE_SIZE_vbuf_raw 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_raw scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_raw 0x2F4
-#else
-#define HIVE_ADDR_sp_vbuf_raw 0x30C
-#endif
-#define HIVE_SIZE_sp_vbuf_raw 4
-
-#ifndef ISP2401
-/* function ia_css_sp_bin_copy_func: 5032 */
-#else
-/* function ia_css_sp_bin_copy_func: 5290 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_item_store: 4C34 */
-#else
-/* function ia_css_queue_item_store: 4E92 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_metadata_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_metadata_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_metadata_bufs 0x4AA0
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_metadata_bufs 0x4AFC
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_metadata_bufs 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_metadata_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_metadata_bufs 0x4AA0
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_metadata_bufs 0x4AFC
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_metadata_bufs 20
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_buffer_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_buffer_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_buffer_bufs 0x4AB4
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_buffer_bufs 0x4B10
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_buffer_bufs 160
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_buffer_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_buffer_bufs 0x4AB4
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_buffer_bufs 0x4B10
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_buffer_bufs 160
-
-/* function sp_start_isp: 45D */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_binary_group
-#define HIVE_MEM_sp_binary_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_binary_group 0x5FF0
-#else
-#define HIVE_ADDR_sp_binary_group 0x6050
-#endif
-#define HIVE_SIZE_sp_binary_group 32
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_binary_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_binary_group 0x5FF0
-#else
-#define HIVE_ADDR_sp_sp_binary_group 0x6050
-#endif
-#define HIVE_SIZE_sp_sp_binary_group 32
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_sw_state
-#define HIVE_MEM_sp_sw_state scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sw_state 0x62AC
-#else
-#define HIVE_ADDR_sp_sw_state 0x6308
-#endif
-#define HIVE_SIZE_sp_sw_state 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_sw_state scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_sw_state 0x62AC
-#else
-#define HIVE_ADDR_sp_sp_sw_state 0x6308
-#endif
-#define HIVE_SIZE_sp_sp_sw_state 4
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_main: D5B */
-#else
-/* function ia_css_thread_sp_main: D50 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_internal_buffers: 3723 */
-#else
-/* function ia_css_ispctrl_sp_init_internal_buffers: 3952 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp2host_psys_event_queue_handle
-#define HIVE_MEM_sp2host_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp2host_psys_event_queue_handle 0x4B54
-#else
-#define HIVE_ADDR_sp2host_psys_event_queue_handle 0x4BB0
-#endif
-#define HIVE_SIZE_sp2host_psys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp2host_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp2host_psys_event_queue_handle 0x4B54
-#else
-#define HIVE_ADDR_sp_sp2host_psys_event_queue_handle 0x4BB0
-#endif
-#define HIVE_SIZE_sp_sp2host_psys_event_queue_handle 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_sp2host_psys_event_queue
-#define HIVE_MEM_sem_for_sp2host_psys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_sp2host_psys_event_queue 0x4698
-#else
-#define HIVE_ADDR_sem_for_sp2host_psys_event_queue 0x46E0
-#endif
-#define HIVE_SIZE_sem_for_sp2host_psys_event_queue 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_sp2host_psys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_sp2host_psys_event_queue 0x4698
-#else
-#define HIVE_ADDR_sp_sem_for_sp2host_psys_event_queue 0x46E0
-#endif
-#define HIVE_SIZE_sp_sem_for_sp2host_psys_event_queue 20
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_propagate_frame: 2410 */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_stop_copy_preview
-#define HIVE_MEM_sp_stop_copy_preview scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_stop_copy_preview 0x6290
-#define HIVE_SIZE_sp_stop_copy_preview 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_stop_copy_preview scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_stop_copy_preview 0x6290
-#define HIVE_SIZE_sp_sp_stop_copy_preview 4
-#else
-/* function ia_css_tagger_sp_propagate_frame: 2460 */
-#endif
-
-#ifndef ISP2401
-/* function input_system_reg_load: B17 */
-#else
-/* function input_system_reg_load: B11 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_handles
-#define HIVE_MEM_vbuf_handles scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_handles 0x6328
-#else
-#define HIVE_ADDR_vbuf_handles 0x6384
-#endif
-#define HIVE_SIZE_vbuf_handles 960
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_handles scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_handles 0x6328
-#else
-#define HIVE_ADDR_sp_vbuf_handles 0x6384
-#endif
-#define HIVE_SIZE_sp_vbuf_handles 960
-
-#ifndef ISP2401
-/* function ia_css_queue_store: 4D9A */
-
-/* function ia_css_sp_flash_register: 2C2C */
-#else
-/* function ia_css_queue_store: 4FF8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_dummy_function: 5652 */
-#else
-/* function ia_css_sp_flash_register: 2DCE */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_create: 5B37 */
-#else
-/* function ia_css_isys_sp_backend_create: 5C72 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_init: 1833 */
-#else
-/* function ia_css_pipeline_sp_init: 186D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_configure: 2300 */
-#else
-/* function ia_css_tagger_sp_configure: 2350 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_end_binary: 3566 */
-#else
-/* function ia_css_ispctrl_sp_end_binary: 375A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs
-#define HIVE_MEM_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x4B60
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x4BBC
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x4B60
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x4BBC
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 20
-
-#ifndef ISP2401
-/* function receiver_port_reg_store: AC9 */
-#else
-/* function receiver_port_reg_store: AC3 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_event_is_pending_mask
-#define HIVE_MEM_event_is_pending_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_event_is_pending_mask 0x5C
-#define HIVE_SIZE_event_is_pending_mask 44
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_event_is_pending_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_event_is_pending_mask 0x5C
-#define HIVE_SIZE_sp_event_is_pending_mask 44
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cb_elems_frame
-#define HIVE_MEM_sp_all_cb_elems_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cb_elems_frame 0x46AC
-#else
-#define HIVE_ADDR_sp_all_cb_elems_frame 0x46F4
-#endif
-#define HIVE_SIZE_sp_all_cb_elems_frame 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cb_elems_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cb_elems_frame 0x46AC
-#else
-#define HIVE_ADDR_sp_sp_all_cb_elems_frame 0x46F4
-#endif
-#define HIVE_SIZE_sp_sp_all_cb_elems_frame 16
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp2host_isys_event_queue_handle
-#define HIVE_MEM_sp2host_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp2host_isys_event_queue_handle 0x4B74
-#else
-#define HIVE_ADDR_sp2host_isys_event_queue_handle 0x4BD0
-#endif
-#define HIVE_SIZE_sp2host_isys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp2host_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp2host_isys_event_queue_handle 0x4B74
-#else
-#define HIVE_ADDR_sp_sp2host_isys_event_queue_handle 0x4BD0
-#endif
-#define HIVE_SIZE_sp_sp2host_isys_event_queue_handle 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host_sp_com
-#define HIVE_MEM_host_sp_com scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host_sp_com 0x4114
-#else
-#define HIVE_ADDR_host_sp_com 0x4134
-#endif
-#define HIVE_SIZE_host_sp_com 220
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host_sp_com scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host_sp_com 0x4114
-#else
-#define HIVE_ADDR_sp_host_sp_com 0x4134
-#endif
-#define HIVE_SIZE_sp_host_sp_com 220
-
-#ifndef ISP2401
-/* function ia_css_queue_get_free_space: 49F9 */
-#else
-/* function ia_css_queue_get_free_space: 4C57 */
-#endif
-
-#ifndef ISP2401
-/* function exec_image_pipe: 6C4 */
-#else
-/* function exec_image_pipe: 658 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_init_dmem_data
-#define HIVE_MEM_sp_init_dmem_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_init_dmem_data 0x62B0
-#else
-#define HIVE_ADDR_sp_init_dmem_data 0x630C
-#endif
-#define HIVE_SIZE_sp_init_dmem_data 24
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_init_dmem_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_init_dmem_data 0x62B0
-#else
-#define HIVE_ADDR_sp_sp_init_dmem_data 0x630C
-#endif
-#define HIVE_SIZE_sp_sp_init_dmem_data 24
-
-#ifndef ISP2401
-/* function ia_css_sp_metadata_start: 5914 */
-#else
-/* function ia_css_sp_metadata_start: 5A4F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_init_buffer_queues: 2C9B */
-#else
-/* function ia_css_bufq_sp_init_buffer_queues: 2E3D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_stop: 1816 */
-#else
-/* function ia_css_pipeline_sp_stop: 1850 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_connect_pipes: 27EA */
-#else
-/* function ia_css_tagger_sp_connect_pipes: 283A */
-#endif
-
-#ifndef ISP2401
-/* function sp_isys_copy_wait: 70D */
-#else
-/* function sp_isys_copy_wait: 6A1 */
-#endif
-
-/* function is_isp_debug_buffer_full: 337 */
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_configure_channel_from_info: 32AF */
-#else
-/* function ia_css_dmaproxy_sp_configure_channel_from_info: 3490 */
-#endif
-
-#ifndef ISP2401
-/* function encode_and_post_timer_event: A30 */
-#else
-/* function encode_and_post_timer_event: 9C4 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_per_frame_data
-#define HIVE_MEM_sp_per_frame_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_per_frame_data 0x41F0
-#else
-#define HIVE_ADDR_sp_per_frame_data 0x4210
-#endif
-#define HIVE_SIZE_sp_per_frame_data 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_per_frame_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_per_frame_data 0x41F0
-#else
-#define HIVE_ADDR_sp_sp_per_frame_data 0x4210
-#endif
-#define HIVE_SIZE_sp_sp_per_frame_data 4
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_vbuf_dequeue: 62D4 */
-#else
-/* function ia_css_rmgr_sp_vbuf_dequeue: 640F */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_psys_event_queue_handle
-#define HIVE_MEM_host2sp_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_psys_event_queue_handle 0x4B80
-#else
-#define HIVE_ADDR_host2sp_psys_event_queue_handle 0x4BDC
-#endif
-#define HIVE_SIZE_host2sp_psys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_psys_event_queue_handle 0x4B80
-#else
-#define HIVE_ADDR_sp_host2sp_psys_event_queue_handle 0x4BDC
-#endif
-#define HIVE_SIZE_sp_host2sp_psys_event_queue_handle 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_xmem_bin_addr
-#define HIVE_MEM_xmem_bin_addr scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_xmem_bin_addr 0x41F4
-#else
-#define HIVE_ADDR_xmem_bin_addr 0x4214
-#endif
-#define HIVE_SIZE_xmem_bin_addr 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_xmem_bin_addr scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_xmem_bin_addr 0x41F4
-#else
-#define HIVE_ADDR_sp_xmem_bin_addr 0x4214
-#endif
-#define HIVE_SIZE_sp_xmem_bin_addr 4
-
-#ifndef ISP2401
-/* function tmr_clock_init: 65A0 */
-#else
-/* function tmr_clock_init: 66DB */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_run: 1403 */
-#else
-/* function ia_css_pipeline_sp_run: 1424 */
-#endif
-
-#ifndef ISP2401
-/* function memcpy: 68F7 */
-#else
-/* function memcpy: 6A6A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_GP_DEVICE_BASE
-#define HIVE_MEM_GP_DEVICE_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_GP_DEVICE_BASE 0x2FC
-#else
-#define HIVE_ADDR_GP_DEVICE_BASE 0x314
-#endif
-#define HIVE_SIZE_GP_DEVICE_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_GP_DEVICE_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_GP_DEVICE_BASE 0x2FC
-#else
-#define HIVE_ADDR_sp_GP_DEVICE_BASE 0x314
-#endif
-#define HIVE_SIZE_sp_GP_DEVICE_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_thread_sp_ready_queue
-#define HIVE_MEM_ia_css_thread_sp_ready_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_thread_sp_ready_queue 0x1E0
-#else
-#define HIVE_ADDR_ia_css_thread_sp_ready_queue 0x1E4
-#endif
-#define HIVE_SIZE_ia_css_thread_sp_ready_queue 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_thread_sp_ready_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_thread_sp_ready_queue 0x1E0
-#else
-#define HIVE_ADDR_sp_ia_css_thread_sp_ready_queue 0x1E4
-#endif
-#define HIVE_SIZE_sp_ia_css_thread_sp_ready_queue 12
-
-#ifndef ISP2401
-/* function input_system_reg_store: B1E */
-#else
-/* function input_system_reg_store: B18 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_start: 5D4D */
-#else
-/* function ia_css_isys_sp_frontend_start: 5E88 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_uds_sp_scale_params: 6600 */
-#else
-/* function ia_css_uds_sp_scale_params: 6773 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_increase_size: E40 */
-#else
-/* function ia_css_circbuf_increase_size: E35 */
-#endif
-
-#ifndef ISP2401
-/* function __divu: 6875 */
-#else
-/* function __divu: 69E8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_get_state: C83 */
-#else
-/* function ia_css_thread_sp_get_state: C78 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_cont_capt_stop
-#define HIVE_MEM_sem_for_cont_capt_stop scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_cont_capt_stop 0x46BC
-#else
-#define HIVE_ADDR_sem_for_cont_capt_stop 0x4704
-#endif
-#define HIVE_SIZE_sem_for_cont_capt_stop 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_cont_capt_stop scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_cont_capt_stop 0x46BC
-#else
-#define HIVE_ADDR_sp_sem_for_cont_capt_stop 0x4704
-#endif
-#define HIVE_SIZE_sp_sem_for_cont_capt_stop 20
-
-#ifndef ISP2401
-/* function thread_fiber_sp_main: E39 */
-#else
-/* function thread_fiber_sp_main: E2E */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_isp_pipe_thread
-#define HIVE_MEM_sp_isp_pipe_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_pipe_thread 0x4800
-#define HIVE_SIZE_sp_isp_pipe_thread 340
-#else
-#define HIVE_ADDR_sp_isp_pipe_thread 0x4848
-#define HIVE_SIZE_sp_isp_pipe_thread 360
-#endif
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_isp_pipe_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_isp_pipe_thread 0x4800
-#define HIVE_SIZE_sp_sp_isp_pipe_thread 340
-#else
-#define HIVE_ADDR_sp_sp_isp_pipe_thread 0x4848
-#define HIVE_SIZE_sp_sp_isp_pipe_thread 360
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_handle_parameter_sets: 128A */
-#else
-/* function ia_css_parambuf_sp_handle_parameter_sets: 127F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_set_state: 5943 */
-#else
-/* function ia_css_spctrl_sp_set_state: 5A7E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sem_sp_signal: 6AF7 */
-#else
-/* function ia_css_thread_sem_sp_signal: 6C6C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_IRQ_BASE
-#define HIVE_MEM_IRQ_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_IRQ_BASE 0x2C
-#define HIVE_SIZE_IRQ_BASE 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_IRQ_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_IRQ_BASE 0x2C
-#define HIVE_SIZE_sp_IRQ_BASE 16
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_TIMED_CTRL_BASE
-#define HIVE_MEM_TIMED_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_TIMED_CTRL_BASE 0x40
-#define HIVE_SIZE_TIMED_CTRL_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_TIMED_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_TIMED_CTRL_BASE 0x40
-#define HIVE_SIZE_sp_TIMED_CTRL_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_isr: 6FDC */
-
-/* function ia_css_isys_sp_generate_exp_id: 60E5 */
-#else
-/* function ia_css_isys_sp_isr: 7139 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_init: 61CF */
-#else
-/* function ia_css_isys_sp_generate_exp_id: 6220 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sem_sp_init: 6BC8 */
-#else
-/* function ia_css_rmgr_sp_init: 630A */
-#endif
-
-#ifndef ISP2401
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_is_isp_requested
-#define HIVE_MEM_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_is_isp_requested 0x308
-#define HIVE_SIZE_is_isp_requested 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_is_isp_requested 0x308
-#define HIVE_SIZE_sp_is_isp_requested 4
-#else
-/* function ia_css_thread_sem_sp_init: 6D3B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_reading_cb_frame
-#define HIVE_MEM_sem_for_reading_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_reading_cb_frame 0x46D0
-#else
-#define HIVE_ADDR_sem_for_reading_cb_frame 0x4718
-#endif
-#define HIVE_SIZE_sem_for_reading_cb_frame 40
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_reading_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_reading_cb_frame 0x46D0
-#else
-#define HIVE_ADDR_sp_sem_for_reading_cb_frame 0x4718
-#endif
-#define HIVE_SIZE_sp_sem_for_reading_cb_frame 40
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_execute: 3217 */
-#else
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_is_isp_requested
-#define HIVE_MEM_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_is_isp_requested 0x320
-#define HIVE_SIZE_is_isp_requested 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_is_isp_requested 0x320
-#define HIVE_SIZE_sp_is_isp_requested 4
-
-/* function ia_css_dmaproxy_sp_execute: 33F6 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_is_empty: 48E0 */
-#else
-/* function ia_css_queue_is_empty: 7098 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_has_stopped: 180C */
-#else
-/* function ia_css_pipeline_sp_has_stopped: 1846 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_extract: F44 */
-#else
-/* function ia_css_circbuf_extract: F39 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_is_locked_from_start: 2B0D */
-#else
-/* function ia_css_tagger_buf_sp_is_locked_from_start: 2CAF */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_current_sp_thread
-#define HIVE_MEM_current_sp_thread scalar_processor_2400_dmem
-#define HIVE_ADDR_current_sp_thread 0x1DC
-#define HIVE_SIZE_current_sp_thread 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_current_sp_thread scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_current_sp_thread 0x1DC
-#define HIVE_SIZE_sp_current_sp_thread 4
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_get_spid: 594A */
-#else
-/* function ia_css_spctrl_sp_get_spid: 5A85 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_reset_buffers: 2D22 */
-#else
-/* function ia_css_bufq_sp_reset_buffers: 2EC4 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_read_byte_addr: 6E35 */
-#else
-/* function ia_css_dmaproxy_sp_read_byte_addr: 6F79 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_uninit: 61C8 */
-#else
-/* function ia_css_rmgr_sp_uninit: 6303 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_threads_stack
-#define HIVE_MEM_sp_threads_stack scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_threads_stack 0x164
-#define HIVE_SIZE_sp_threads_stack 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_threads_stack scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_threads_stack 0x164
-#define HIVE_SIZE_sp_sp_threads_stack 28
-
-#ifndef ISP2401
-/* function ia_css_circbuf_peek: F26 */
-#else
-/* function ia_css_circbuf_peek: F1B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_wait_for_in_param: 1053 */
-#else
-/* function ia_css_parambuf_sp_wait_for_in_param: 1048 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_get_exp_id: 5FAD */
-#else
-/* function ia_css_isys_sp_token_map_get_exp_id: 60E8 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cb_elems_param
-#define HIVE_MEM_sp_all_cb_elems_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cb_elems_param 0x46F8
-#else
-#define HIVE_ADDR_sp_all_cb_elems_param 0x4740
-#endif
-#define HIVE_SIZE_sp_all_cb_elems_param 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cb_elems_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cb_elems_param 0x46F8
-#else
-#define HIVE_ADDR_sp_sp_all_cb_elems_param 0x4740
-#endif
-#define HIVE_SIZE_sp_sp_all_cb_elems_param 16
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_pipeline_sp_curr_binary_id
-#define HIVE_MEM_pipeline_sp_curr_binary_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_pipeline_sp_curr_binary_id 0x1EC
-#else
-#define HIVE_ADDR_pipeline_sp_curr_binary_id 0x1F0
-#endif
-#define HIVE_SIZE_pipeline_sp_curr_binary_id 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_pipeline_sp_curr_binary_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_pipeline_sp_curr_binary_id 0x1EC
-#else
-#define HIVE_ADDR_sp_pipeline_sp_curr_binary_id 0x1F0
-#endif
-#define HIVE_SIZE_sp_pipeline_sp_curr_binary_id 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_frame_desc
-#define HIVE_MEM_sp_all_cbs_frame_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_frame_desc 0x4708
-#else
-#define HIVE_ADDR_sp_all_cbs_frame_desc 0x4750
-#endif
-#define HIVE_SIZE_sp_all_cbs_frame_desc 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_frame_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_frame_desc 0x4708
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_frame_desc 0x4750
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_frame_desc 8
-
-#ifndef ISP2401
-/* function sp_isys_copy_func_v2: 706 */
-#else
-/* function sp_isys_copy_func_v2: 69A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_reading_cb_param
-#define HIVE_MEM_sem_for_reading_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_reading_cb_param 0x4710
-#else
-#define HIVE_ADDR_sem_for_reading_cb_param 0x4758
-#endif
-#define HIVE_SIZE_sem_for_reading_cb_param 40
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_reading_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_reading_cb_param 0x4710
-#else
-#define HIVE_ADDR_sp_sem_for_reading_cb_param 0x4758
-#endif
-#define HIVE_SIZE_sp_sem_for_reading_cb_param 40
-
-#ifndef ISP2401
-/* function ia_css_queue_get_used_space: 49AD */
-#else
-/* function ia_css_queue_get_used_space: 4C0B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_cont_capt_start
-#define HIVE_MEM_sem_for_cont_capt_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_cont_capt_start 0x4738
-#else
-#define HIVE_ADDR_sem_for_cont_capt_start 0x4780
-#endif
-#define HIVE_SIZE_sem_for_cont_capt_start 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_cont_capt_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_cont_capt_start 0x4738
-#else
-#define HIVE_ADDR_sp_sem_for_cont_capt_start 0x4780
-#endif
-#define HIVE_SIZE_sp_sem_for_cont_capt_start 20
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_tmp_heap
-#define HIVE_MEM_tmp_heap scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_tmp_heap 0x6010
-#else
-#define HIVE_ADDR_tmp_heap 0x6070
-#endif
-#define HIVE_SIZE_tmp_heap 640
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_tmp_heap scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_tmp_heap 0x6010
-#else
-#define HIVE_ADDR_sp_tmp_heap 0x6070
-#endif
-#define HIVE_SIZE_sp_tmp_heap 640
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_get_num_vbuf: 64D8 */
-#else
-/* function ia_css_rmgr_sp_get_num_vbuf: 6613 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_output_compute_dma_info: 3F49 */
-#else
-/* function ia_css_ispctrl_sp_output_compute_dma_info: 418C */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_lock_exp_id: 20CD */
-#else
-/* function ia_css_tagger_sp_lock_exp_id: 211D */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_s3a_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_s3a_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_s3a_bufs 0x4B8C
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_s3a_bufs 0x4BE8
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_s3a_bufs 60
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_s3a_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_s3a_bufs 0x4B8C
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_s3a_bufs 0x4BE8
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_s3a_bufs 60
-
-#ifndef ISP2401
-/* function ia_css_queue_is_full: 4A44 */
-#else
-/* function ia_css_queue_is_full: 4CA2 */
-#endif
-
-/* function debug_buffer_init_isp: E4 */
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_uninit: 5D07 */
-#else
-/* function ia_css_isys_sp_frontend_uninit: 5E42 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_exp_id_is_locked: 2003 */
-#else
-/* function ia_css_tagger_sp_exp_id_is_locked: 2053 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_rmgr_sp_mipi_frame_sem
-#define HIVE_MEM_ia_css_rmgr_sp_mipi_frame_sem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_rmgr_sp_mipi_frame_sem 0x66E8
-#else
-#define HIVE_ADDR_ia_css_rmgr_sp_mipi_frame_sem 0x6744
-#endif
-#define HIVE_SIZE_ia_css_rmgr_sp_mipi_frame_sem 60
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_rmgr_sp_mipi_frame_sem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_rmgr_sp_mipi_frame_sem 0x66E8
-#else
-#define HIVE_ADDR_sp_ia_css_rmgr_sp_mipi_frame_sem 0x6744
-#endif
-#define HIVE_SIZE_sp_ia_css_rmgr_sp_mipi_frame_sem 60
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_dump: 62AF */
-#else
-/* function ia_css_rmgr_sp_refcount_dump: 63EA */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_isp_parameters_id
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_isp_parameters_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x4BC8
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x4C24
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_isp_parameters_id 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x4BC8
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x4C24
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id 20
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_pipe_threads
-#define HIVE_MEM_sp_pipe_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_pipe_threads 0x150
-#define HIVE_SIZE_sp_pipe_threads 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_pipe_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_pipe_threads 0x150
-#define HIVE_SIZE_sp_sp_pipe_threads 20
-
-#ifndef ISP2401
-/* function sp_event_proxy_func: 71B */
-#else
-/* function sp_event_proxy_func: 6AF */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_isys_event_queue_handle
-#define HIVE_MEM_host2sp_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_isys_event_queue_handle 0x4BDC
-#else
-#define HIVE_ADDR_host2sp_isys_event_queue_handle 0x4C38
-#endif
-#define HIVE_SIZE_host2sp_isys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_isys_event_queue_handle 0x4BDC
-#else
-#define HIVE_ADDR_sp_host2sp_isys_event_queue_handle 0x4C38
-#endif
-#define HIVE_SIZE_sp_host2sp_isys_event_queue_handle 12
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_yield: 6A70 */
-#else
-/* function ia_css_thread_sp_yield: 6BEA */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_param_desc
-#define HIVE_MEM_sp_all_cbs_param_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_param_desc 0x474C
-#else
-#define HIVE_ADDR_sp_all_cbs_param_desc 0x4794
-#endif
-#define HIVE_SIZE_sp_all_cbs_param_desc 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_param_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_param_desc 0x474C
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_param_desc 0x4794
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_param_desc 8
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_dmaproxy_sp_invalidate_tlb
-#define HIVE_MEM_ia_css_dmaproxy_sp_invalidate_tlb scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb 0x5BF4
-#else
-#define HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb 0x5C50
-#endif
-#define HIVE_SIZE_ia_css_dmaproxy_sp_invalidate_tlb 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_dmaproxy_sp_invalidate_tlb scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_dmaproxy_sp_invalidate_tlb 0x5BF4
-#else
-#define HIVE_ADDR_sp_ia_css_dmaproxy_sp_invalidate_tlb 0x5C50
-#endif
-#define HIVE_SIZE_sp_ia_css_dmaproxy_sp_invalidate_tlb 4
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_fork: D10 */
-#else
-/* function ia_css_thread_sp_fork: D05 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_destroy: 27F4 */
-#else
-/* function ia_css_tagger_sp_destroy: 2844 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_vmem_read: 31B7 */
-#else
-/* function ia_css_dmaproxy_sp_vmem_read: 3396 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ifmtr_sp_init: 6136 */
-#else
-/* function ia_css_ifmtr_sp_init: 6271 */
-#endif
-
-#ifndef ISP2401
-/* function initialize_sp_group: 6D4 */
-#else
-/* function initialize_sp_group: 668 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_peek: 2919 */
-#else
-/* function ia_css_tagger_buf_sp_peek: 2ABB */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_init: D3C */
-#else
-/* function ia_css_thread_sp_init: D31 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_reset_exp_id: 60DD */
-#else
-/* function ia_css_isys_sp_reset_exp_id: 6218 */
-#endif
-
-#ifndef ISP2401
-/* function qos_scheduler_update_fps: 65F0 */
-#else
-/* function qos_scheduler_update_fps: 6763 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_set_stream_base_addr: 461E */
-#else
-/* function ia_css_ispctrl_sp_set_stream_base_addr: 4879 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_DMEM_BASE
-#define HIVE_MEM_ISP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_DMEM_BASE 0x10
-#define HIVE_SIZE_ISP_DMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_DMEM_BASE 0x10
-#define HIVE_SIZE_sp_ISP_DMEM_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_SP_DMEM_BASE
-#define HIVE_MEM_SP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_SP_DMEM_BASE 0x4
-#define HIVE_SIZE_SP_DMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_SP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_SP_DMEM_BASE 0x4
-#define HIVE_SIZE_sp_SP_DMEM_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_read: 322D */
-#else
-/* function __ia_css_queue_is_empty_text: 4B68 */
-
-/* function ia_css_dmaproxy_sp_read: 340C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_raw_copy_line_count
-#define HIVE_MEM_raw_copy_line_count scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_raw_copy_line_count 0x2C8
-#else
-#define HIVE_ADDR_raw_copy_line_count 0x2E0
-#endif
-#define HIVE_SIZE_raw_copy_line_count 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_raw_copy_line_count scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_raw_copy_line_count 0x2C8
-#else
-#define HIVE_ADDR_sp_raw_copy_line_count 0x2E0
-#endif
-#define HIVE_SIZE_sp_raw_copy_line_count 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_tag_cmd_queue_handle
-#define HIVE_MEM_host2sp_tag_cmd_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_tag_cmd_queue_handle 0x4BE8
-#else
-#define HIVE_ADDR_host2sp_tag_cmd_queue_handle 0x4C44
-#endif
-#define HIVE_SIZE_host2sp_tag_cmd_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_tag_cmd_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_tag_cmd_queue_handle 0x4BE8
-#else
-#define HIVE_ADDR_sp_host2sp_tag_cmd_queue_handle 0x4C44
-#endif
-#define HIVE_SIZE_sp_host2sp_tag_cmd_queue_handle 12
-
-#ifndef ISP2401
-/* function ia_css_queue_peek: 4923 */
-#else
-/* function ia_css_queue_peek: 4B81 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_frame_cnt
-#define HIVE_MEM_ia_css_flash_sp_frame_cnt scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_frame_cnt 0x4A94
-#else
-#define HIVE_ADDR_ia_css_flash_sp_frame_cnt 0x4AF0
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_frame_cnt 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_frame_cnt scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_frame_cnt 0x4A94
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_frame_cnt 0x4AF0
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_frame_cnt 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_event_can_send_token_mask
-#define HIVE_MEM_event_can_send_token_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_event_can_send_token_mask 0x88
-#define HIVE_SIZE_event_can_send_token_mask 44
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_event_can_send_token_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_event_can_send_token_mask 0x88
-#define HIVE_SIZE_sp_event_can_send_token_mask 44
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_thread
-#define HIVE_MEM_isp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_thread 0x5F40
-#else
-#define HIVE_ADDR_isp_thread 0x5FA0
-#endif
-#define HIVE_SIZE_isp_thread 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_thread 0x5F40
-#else
-#define HIVE_ADDR_sp_isp_thread 0x5FA0
-#endif
-#define HIVE_SIZE_sp_isp_thread 4
-
-#ifndef ISP2401
-/* function encode_and_post_sp_event_non_blocking: A78 */
-#else
-/* function encode_and_post_sp_event_non_blocking: A0C */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_destroy: 5DDF */
-#else
-/* function ia_css_isys_sp_frontend_destroy: 5F1A */
-#endif
-
-/* function is_ddr_debug_buffer_full: 2CC */
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_stop: 5D1F */
-#else
-/* function ia_css_isys_sp_frontend_stop: 5E5A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_init: 607B */
-#else
-/* function ia_css_isys_sp_token_map_init: 61B6 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_get_oldest_marked_offset: 2969 */
-#else
-/* function ia_css_tagger_buf_sp_get_oldest_marked_offset: 2B0B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_threads_fiber
-#define HIVE_MEM_sp_threads_fiber scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_threads_fiber 0x19C
-#define HIVE_SIZE_sp_threads_fiber 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_threads_fiber scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_threads_fiber 0x19C
-#define HIVE_SIZE_sp_sp_threads_fiber 28
-
-#ifndef ISP2401
-/* function encode_and_post_sp_event: A01 */
-#else
-/* function encode_and_post_sp_event: 995 */
-#endif
-
-/* function debug_enqueue_ddr: EE */
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_init_vbuf: 626A */
-#else
-/* function ia_css_rmgr_sp_refcount_init_vbuf: 63A5 */
-#endif
-
-#ifndef ISP2401
-/* function dmaproxy_sp_read_write: 6EE4 */
-#else
-/* function dmaproxy_sp_read_write: 7017 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_dmaproxy_isp_dma_cmd_buffer
-#define HIVE_MEM_ia_css_dmaproxy_isp_dma_cmd_buffer scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_dmaproxy_isp_dma_cmd_buffer 0x5BF8
-#else
-#define HIVE_ADDR_ia_css_dmaproxy_isp_dma_cmd_buffer 0x5C54
-#endif
-#define HIVE_SIZE_ia_css_dmaproxy_isp_dma_cmd_buffer 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_dmaproxy_isp_dma_cmd_buffer scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_dmaproxy_isp_dma_cmd_buffer 0x5BF8
-#else
-#define HIVE_ADDR_sp_ia_css_dmaproxy_isp_dma_cmd_buffer 0x5C54
-#endif
-#define HIVE_SIZE_sp_ia_css_dmaproxy_isp_dma_cmd_buffer 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_buffer_queue_handle
-#define HIVE_MEM_host2sp_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_buffer_queue_handle 0x4BF4
-#else
-#define HIVE_ADDR_host2sp_buffer_queue_handle 0x4C50
-#endif
-#define HIVE_SIZE_host2sp_buffer_queue_handle 480
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_buffer_queue_handle 0x4BF4
-#else
-#define HIVE_ADDR_sp_host2sp_buffer_queue_handle 0x4C50
-#endif
-#define HIVE_SIZE_sp_host2sp_buffer_queue_handle 480
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_in_service
-#define HIVE_MEM_ia_css_flash_sp_in_service scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_in_service 0x3178
-#else
-#define HIVE_ADDR_ia_css_flash_sp_in_service 0x3198
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_in_service 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_in_service scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_service 0x3178
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_service 0x3198
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_in_service 4
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_process: 6BF0 */
-#else
-/* function ia_css_dmaproxy_sp_process: 6D63 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_mark_from_end: 2BF1 */
-#else
-/* function ia_css_tagger_buf_sp_mark_from_end: 2D93 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_rcv_acquire_ack: 59EC */
-#else
-/* function ia_css_isys_sp_backend_rcv_acquire_ack: 5B27 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_pre_acquire_request: 5A02 */
-#else
-/* function ia_css_isys_sp_backend_pre_acquire_request: 5B3D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_cs: 3653 */
-#else
-/* function ia_css_ispctrl_sp_init_cs: 3855 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_init: 5958 */
-#else
-/* function ia_css_spctrl_sp_init: 5A93 */
-#endif
-
-#ifndef ISP2401
-/* function sp_event_proxy_init: 730 */
-#else
-/* function sp_event_proxy_init: 6C4 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_previous_clock_tick
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_previous_clock_tick scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x4DD4
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x4E30
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_previous_clock_tick 40
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x4DD4
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x4E30
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick 40
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_output
-#define HIVE_MEM_sp_output scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_output 0x41F8
-#else
-#define HIVE_ADDR_sp_output 0x4218
-#endif
-#define HIVE_SIZE_sp_output 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_output scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_output 0x41F8
-#else
-#define HIVE_ADDR_sp_sp_output 0x4218
-#endif
-#define HIVE_SIZE_sp_sp_output 16
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_sems_for_host2sp_buf_queues
-#define HIVE_MEM_ia_css_bufq_sp_sems_for_host2sp_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x4DFC
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x4E58
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_sems_for_host2sp_buf_queues 800
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x4DFC
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x4E58
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues 800
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_CTRL_BASE
-#define HIVE_MEM_ISP_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_CTRL_BASE 0x8
-#define HIVE_SIZE_ISP_CTRL_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_CTRL_BASE 0x8
-#define HIVE_SIZE_sp_ISP_CTRL_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_INPUT_FORMATTER_BASE
-#define HIVE_MEM_INPUT_FORMATTER_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_INPUT_FORMATTER_BASE 0x4C
-#define HIVE_SIZE_INPUT_FORMATTER_BASE 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_INPUT_FORMATTER_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_INPUT_FORMATTER_BASE 0x4C
-#define HIVE_SIZE_sp_INPUT_FORMATTER_BASE 16
-
-#ifndef ISP2401
-/* function sp_dma_proxy_reset_channels: 3487 */
-#else
-/* function sp_dma_proxy_reset_channels: 367B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_acquire: 5B0D */
-#else
-/* function ia_css_isys_sp_backend_acquire: 5C48 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_update_size: 28E8 */
-#else
-/* function ia_css_tagger_sp_update_size: 2A8A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_host_sp_queue
-#define HIVE_MEM_ia_css_bufq_host_sp_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_host_sp_queue 0x511C
-#else
-#define HIVE_ADDR_ia_css_bufq_host_sp_queue 0x5178
-#endif
-#define HIVE_SIZE_ia_css_bufq_host_sp_queue 2008
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_host_sp_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_host_sp_queue 0x511C
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_host_sp_queue 0x5178
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_host_sp_queue 2008
-
-#ifndef ISP2401
-/* function thread_fiber_sp_create: DA8 */
-#else
-/* function thread_fiber_sp_create: D9D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_increments: 3319 */
-#else
-/* function ia_css_dmaproxy_sp_set_increments: 350D */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_writing_cb_frame
-#define HIVE_MEM_sem_for_writing_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_writing_cb_frame 0x4754
-#else
-#define HIVE_ADDR_sem_for_writing_cb_frame 0x479C
-#endif
-#define HIVE_SIZE_sem_for_writing_cb_frame 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_writing_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_writing_cb_frame 0x4754
-#else
-#define HIVE_ADDR_sp_sem_for_writing_cb_frame 0x479C
-#endif
-#define HIVE_SIZE_sp_sem_for_writing_cb_frame 20
-
-#ifndef ISP2401
-/* function receiver_reg_store: AD7 */
-#else
-/* function receiver_reg_store: AD1 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_writing_cb_param
-#define HIVE_MEM_sem_for_writing_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_writing_cb_param 0x4768
-#else
-#define HIVE_ADDR_sem_for_writing_cb_param 0x47B0
-#endif
-#define HIVE_SIZE_sem_for_writing_cb_param 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_writing_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_writing_cb_param 0x4768
-#else
-#define HIVE_ADDR_sp_sem_for_writing_cb_param 0x47B0
-#endif
-#define HIVE_SIZE_sp_sem_for_writing_cb_param 20
-
-/* function sp_start_isp_entry: 453 */
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifdef HIVE_ADDR_sp_start_isp_entry
-#endif
-#define HIVE_ADDR_sp_start_isp_entry 0x453
-#endif
-#define HIVE_ADDR_sp_sp_start_isp_entry 0x453
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_unmark_all: 2B75 */
-#else
-/* function ia_css_tagger_buf_sp_unmark_all: 2D17 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_unmark_from_start: 2BB6 */
-#else
-/* function ia_css_tagger_buf_sp_unmark_from_start: 2D58 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_channel_acquire: 34B3 */
-#else
-/* function ia_css_dmaproxy_sp_channel_acquire: 36A7 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_add_num_vbuf: 64B4 */
-#else
-/* function ia_css_rmgr_sp_add_num_vbuf: 65EF */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_create: 60C4 */
-#else
-/* function ia_css_isys_sp_token_map_create: 61FF */
-#endif
-
-#ifndef ISP2401
-/* function __ia_css_dmaproxy_sp_wait_for_ack_text: 3183 */
-#else
-/* function __ia_css_dmaproxy_sp_wait_for_ack_text: 3362 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_acquire_buf_elem: 1FDB */
-#else
-/* function ia_css_tagger_sp_acquire_buf_elem: 202B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_is_dynamic_buffer: 306C */
-#else
-/* function ia_css_bufq_sp_is_dynamic_buffer: 320E */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_group
-#define HIVE_MEM_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_group 0x4208
-#define HIVE_SIZE_sp_group 1144
-#else
-#define HIVE_ADDR_sp_group 0x4228
-#define HIVE_SIZE_sp_group 1184
-#endif
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_group 0x4208
-#define HIVE_SIZE_sp_sp_group 1144
-#else
-#define HIVE_ADDR_sp_sp_group 0x4228
-#define HIVE_SIZE_sp_sp_group 1184
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_event_proxy_thread
-#define HIVE_MEM_sp_event_proxy_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_event_proxy_thread 0x4954
-#define HIVE_SIZE_sp_event_proxy_thread 68
-#else
-#define HIVE_ADDR_sp_event_proxy_thread 0x49B0
-#define HIVE_SIZE_sp_event_proxy_thread 72
-#endif
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_event_proxy_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_event_proxy_thread 0x4954
-#define HIVE_SIZE_sp_sp_event_proxy_thread 68
-#else
-#define HIVE_ADDR_sp_sp_event_proxy_thread 0x49B0
-#define HIVE_SIZE_sp_sp_event_proxy_thread 72
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_kill: CD6 */
-#else
-/* function ia_css_thread_sp_kill: CCB */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_create: 28A2 */
-#else
-/* function ia_css_tagger_sp_create: 2A38 */
-#endif
-
-#ifndef ISP2401
-/* function tmpmem_acquire_dmem: 6561 */
-#else
-/* function tmpmem_acquire_dmem: 669C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_MMU_BASE
-#define HIVE_MEM_MMU_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_MMU_BASE 0x24
-#define HIVE_SIZE_MMU_BASE 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_MMU_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_MMU_BASE 0x24
-#define HIVE_SIZE_sp_MMU_BASE 8
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_channel_release: 349F */
-#else
-/* function ia_css_dmaproxy_sp_channel_release: 3693 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_is_idle: 347F */
-#else
-/* function ia_css_dmaproxy_sp_is_idle: 3673 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_qos_start
-#define HIVE_MEM_sem_for_qos_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_qos_start 0x477C
-#else
-#define HIVE_ADDR_sem_for_qos_start 0x47C4
-#endif
-#define HIVE_SIZE_sem_for_qos_start 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_qos_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_qos_start 0x477C
-#else
-#define HIVE_ADDR_sp_sem_for_qos_start 0x47C4
-#endif
-#define HIVE_SIZE_sp_sem_for_qos_start 20
-
-#ifndef ISP2401
-/* function isp_hmem_load: B55 */
-#else
-/* function isp_hmem_load: B4F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_release_buf_elem: 1FB7 */
-#else
-/* function ia_css_tagger_sp_release_buf_elem: 2007 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_eventq_sp_send: 34F5 */
-#else
-/* function ia_css_eventq_sp_send: 36E9 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_isys_sp_error_cnt
-#define HIVE_MEM_ia_css_isys_sp_error_cnt scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_isys_sp_error_cnt 0x62D4
-#else
-#define HIVE_ADDR_ia_css_isys_sp_error_cnt 0x6330
-#endif
-#define HIVE_SIZE_ia_css_isys_sp_error_cnt 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_isys_sp_error_cnt scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_isys_sp_error_cnt 0x62D4
-#else
-#define HIVE_ADDR_sp_ia_css_isys_sp_error_cnt 0x6330
-#endif
-#define HIVE_SIZE_sp_ia_css_isys_sp_error_cnt 16
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_unlock_from_start: 2AA5 */
-#else
-/* function ia_css_tagger_buf_sp_unlock_from_start: 2C47 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_debug_buffer_ddr_address
-#define HIVE_MEM_debug_buffer_ddr_address scalar_processor_2400_dmem
-#define HIVE_ADDR_debug_buffer_ddr_address 0xBC
-#define HIVE_SIZE_debug_buffer_ddr_address 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_debug_buffer_ddr_address scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_debug_buffer_ddr_address 0xBC
-#define HIVE_SIZE_sp_debug_buffer_ddr_address 4
-
-#ifndef ISP2401
-/* function sp_isys_copy_request: 714 */
-#else
-/* function sp_isys_copy_request: 6A8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_retain_vbuf: 6344 */
-#else
-/* function ia_css_rmgr_sp_refcount_retain_vbuf: 647F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_set_priority: CCE */
-#else
-/* function ia_css_thread_sp_set_priority: CC3 */
-#endif
-
-#ifndef ISP2401
-/* function sizeof_hmem: BFC */
-#else
-/* function sizeof_hmem: BF6 */
-#endif
-
-#ifndef ISP2401
-/* function tmpmem_release_dmem: 6550 */
-#else
-/* function tmpmem_release_dmem: 668B */
-#endif
-
-/* function cnd_input_system_cfg: 392 */
-
-#ifndef ISP2401
-/* function __ia_css_sp_rawcopy_func_critical: 6F65 */
-#else
-/* function __ia_css_sp_rawcopy_func_critical: 70C2 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_width_exception: 3304 */
-#else
-/* function __ia_css_dmaproxy_sp_process_text: 3306 */
-#endif
-
-#ifndef ISP2401
-/* function sp_event_assert: 8B1 */
-#else
-/* function ia_css_dmaproxy_sp_set_width_exception: 34F8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_flash_sp_init_internal_params: 2C90 */
-#else
-/* function sp_event_assert: 845 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_pop_unmarked_and_unlocked: 29AB */
-#else
-/* function ia_css_flash_sp_init_internal_params: 2E32 */
-#endif
-
-#ifndef ISP2401
-/* function __modu: 68BB */
-#else
-/* function ia_css_tagger_buf_sp_pop_unmarked_and_unlocked: 2B4D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init_isp_vector: 3189 */
-#else
-/* function __modu: 6A2E */
-
-/* function ia_css_dmaproxy_sp_init_isp_vector: 3368 */
-#endif
-
-/* function isp_vamem_store: 0 */
-
-#ifdef ISP2401
-/* function ia_css_tagger_sp_set_copy_pipe: 2A2F */
-
-#endif
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_GDC_BASE
-#define HIVE_MEM_GDC_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_GDC_BASE 0x44
-#define HIVE_SIZE_GDC_BASE 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_GDC_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_GDC_BASE 0x44
-#define HIVE_SIZE_sp_GDC_BASE 8
-
-#ifndef ISP2401
-/* function ia_css_queue_local_init: 4C0E */
-#else
-/* function ia_css_queue_local_init: 4E6C */
-#endif
-
-#ifndef ISP2401
-/* function sp_event_proxy_callout_func: 6988 */
-#else
-/* function sp_event_proxy_callout_func: 6AFB */
-#endif
-
-#ifndef ISP2401
-/* function qos_scheduler_schedule_stage: 65C1 */
-#else
-/* function qos_scheduler_schedule_stage: 670F */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_thread_sp_num_ready_threads
-#define HIVE_MEM_ia_css_thread_sp_num_ready_threads scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_thread_sp_num_ready_threads 0x49E0
-#else
-#define HIVE_ADDR_ia_css_thread_sp_num_ready_threads 0x4A40
-#endif
-#define HIVE_SIZE_ia_css_thread_sp_num_ready_threads 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_thread_sp_num_ready_threads scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_thread_sp_num_ready_threads 0x49E0
-#else
-#define HIVE_ADDR_sp_ia_css_thread_sp_num_ready_threads 0x4A40
-#endif
-#define HIVE_SIZE_sp_ia_css_thread_sp_num_ready_threads 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_threads_stack_size
-#define HIVE_MEM_sp_threads_stack_size scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_threads_stack_size 0x180
-#define HIVE_SIZE_sp_threads_stack_size 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_threads_stack_size scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_threads_stack_size 0x180
-#define HIVE_SIZE_sp_sp_threads_stack_size 28
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_isp_done_row_striping: 3F2F */
-#else
-/* function ia_css_ispctrl_sp_isp_done_row_striping: 4172 */
-#endif
-
-#ifndef ISP2401
-/* function __ia_css_isys_sp_isr_text: 5E09 */
-#else
-/* function __ia_css_isys_sp_isr_text: 5F44 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_dequeue: 4A8C */
-#else
-/* function ia_css_queue_dequeue: 4CEA */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_configure_channel: 6E4C */
-#else
-/* function is_qos_standalone_mode: 66EA */
-
-/* function ia_css_dmaproxy_sp_configure_channel: 6F90 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_current_thread_fiber_sp
-#define HIVE_MEM_current_thread_fiber_sp scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_current_thread_fiber_sp 0x49E8
-#else
-#define HIVE_ADDR_current_thread_fiber_sp 0x4A44
-#endif
-#define HIVE_SIZE_current_thread_fiber_sp 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_current_thread_fiber_sp scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_current_thread_fiber_sp 0x49E8
-#else
-#define HIVE_ADDR_sp_current_thread_fiber_sp 0x4A44
-#endif
-#define HIVE_SIZE_sp_current_thread_fiber_sp 4
-
-#ifndef ISP2401
-/* function ia_css_circbuf_pop: FD8 */
-#else
-/* function ia_css_circbuf_pop: FCD */
-#endif
-
-#ifndef ISP2401
-/* function memset: 693A */
-#else
-/* function memset: 6AAD */
-#endif
-
-/* function irq_raise_set_token: B6 */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_GPIO_BASE
-#define HIVE_MEM_GPIO_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_GPIO_BASE 0x3C
-#define HIVE_SIZE_GPIO_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_GPIO_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_GPIO_BASE 0x3C
-#define HIVE_SIZE_sp_GPIO_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_pipeline_acc_stage_enable: 17D7 */
-#else
-/* function ia_css_pipeline_acc_stage_enable: 17FF */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_unlock_exp_id: 2028 */
-#else
-/* function ia_css_tagger_sp_unlock_exp_id: 2078 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_ph
-#define HIVE_MEM_isp_ph scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_ph 0x62E4
-#else
-#define HIVE_ADDR_isp_ph 0x6340
-#endif
-#define HIVE_SIZE_isp_ph 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_ph scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_ph 0x62E4
-#else
-#define HIVE_ADDR_sp_isp_ph 0x6340
-#endif
-#define HIVE_SIZE_sp_isp_ph 28
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_flush: 6009 */
-#else
-/* function ia_css_isys_sp_token_map_flush: 6144 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_ds: 37B2 */
-#else
-/* function ia_css_ispctrl_sp_init_ds: 39E1 */
-#endif
-
-#ifndef ISP2401
-/* function get_xmem_base_addr_raw: 3B5F */
-#else
-/* function get_xmem_base_addr_raw: 3D9A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_param
-#define HIVE_MEM_sp_all_cbs_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_param 0x4790
-#else
-#define HIVE_ADDR_sp_all_cbs_param 0x47D8
-#endif
-#define HIVE_SIZE_sp_all_cbs_param 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_param 0x4790
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_param 0x47D8
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_param 16
-
-#ifndef ISP2401
-/* function ia_css_circbuf_create: 1026 */
-#else
-/* function ia_css_circbuf_create: 101B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_sp_group
-#define HIVE_MEM_sem_for_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_sp_group 0x47A0
-#else
-#define HIVE_ADDR_sem_for_sp_group 0x47E8
-#endif
-#define HIVE_SIZE_sem_for_sp_group 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_sp_group 0x47A0
-#else
-#define HIVE_ADDR_sp_sem_for_sp_group 0x47E8
-#endif
-#define HIVE_SIZE_sp_sem_for_sp_group 20
-
-#ifndef ISP2401
-/* function ia_css_framebuf_sp_wait_for_in_frame: 64DF */
-#else
-/* function __ia_css_dmaproxy_sp_configure_channel_text: 34D7 */
-
-/* function ia_css_framebuf_sp_wait_for_in_frame: 661A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_tag_frame: 556F */
-#else
-/* function ia_css_sp_rawcopy_tag_frame: 57B0 */
-#endif
-
-#ifndef ISP2401
-/* function isp_hmem_clear: B25 */
-#else
-/* function isp_hmem_clear: B1F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_framebuf_sp_release_in_frame: 6522 */
-#else
-/* function ia_css_framebuf_sp_release_in_frame: 665D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_snd_acquire_request: 5A5F */
-#else
-/* function ia_css_isys_sp_backend_snd_acquire_request: 5B9A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_is_full: 5E90 */
-#else
-/* function ia_css_isys_sp_token_map_is_full: 5FCB */
-#endif
-
-#ifndef ISP2401
-/* function input_system_acquisition_run: AF9 */
-#else
-/* function input_system_acquisition_run: AF3 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_start_binary: 3631 */
-#else
-/* function ia_css_ispctrl_sp_start_binary: 3833 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_h_pipe_private_ddr_ptrs
-#define HIVE_MEM_ia_css_bufq_sp_h_pipe_private_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x58F4
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x5950
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x58F4
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x5950
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 20
-
-#ifndef ISP2401
-/* function ia_css_eventq_sp_recv: 34C7 */
-#else
-/* function ia_css_eventq_sp_recv: 36BB */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_pool
-#define HIVE_MEM_isp_pool scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_pool 0x2E8
-#else
-#define HIVE_ADDR_isp_pool 0x300
-#endif
-#define HIVE_SIZE_isp_pool 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_pool scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_pool 0x2E8
-#else
-#define HIVE_ADDR_sp_isp_pool 0x300
-#endif
-#define HIVE_SIZE_sp_isp_pool 4
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_rel_gen: 6211 */
-#else
-/* function ia_css_rmgr_sp_rel_gen: 634C */
-
-/* function ia_css_tagger_sp_unblock_clients: 2900 */
-#endif
-
-#ifndef ISP2401
-/* function css_get_frame_processing_time_end: 1FA7 */
-#else
-/* function css_get_frame_processing_time_end: 1FF7 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_event_any_pending_mask
-#define HIVE_MEM_event_any_pending_mask scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_event_any_pending_mask 0x300
-#else
-#define HIVE_ADDR_event_any_pending_mask 0x318
-#endif
-#define HIVE_SIZE_event_any_pending_mask 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_event_any_pending_mask scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_event_any_pending_mask 0x300
-#else
-#define HIVE_ADDR_sp_event_any_pending_mask 0x318
-#endif
-#define HIVE_SIZE_sp_event_any_pending_mask 8
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_push: 5A16 */
-#else
-/* function ia_css_isys_sp_backend_push: 5B51 */
-#endif
-
-/* function sh_css_decode_tag_descr: 352 */
-
-/* function debug_enqueue_isp: 27B */
-
-#ifndef ISP2401
-/* function qos_scheduler_update_stage_budget: 65AF */
-#else
-/* function qos_scheduler_update_stage_budget: 66F2 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_uninit: 5951 */
-#else
-/* function ia_css_spctrl_sp_uninit: 5A8C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_HIVE_IF_SWITCH_CODE
-#define HIVE_MEM_HIVE_IF_SWITCH_CODE scalar_processor_2400_dmem
-#define HIVE_ADDR_HIVE_IF_SWITCH_CODE 0x1D8
-#define HIVE_SIZE_HIVE_IF_SWITCH_CODE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_HIVE_IF_SWITCH_CODE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_HIVE_IF_SWITCH_CODE 0x1D8
-#define HIVE_SIZE_sp_HIVE_IF_SWITCH_CODE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_dis_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_dis_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_dis_bufs 0x5908
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_dis_bufs 0x5964
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_dis_bufs 140
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_dis_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_dis_bufs 0x5908
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_dis_bufs 0x5964
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_dis_bufs 140
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_lock_from_start: 2AD9 */
-#else
-/* function ia_css_tagger_buf_sp_lock_from_start: 2C7B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_isp_idle
-#define HIVE_MEM_sem_for_isp_idle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_isp_idle 0x47B4
-#else
-#define HIVE_ADDR_sem_for_isp_idle 0x47FC
-#endif
-#define HIVE_SIZE_sem_for_isp_idle 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_isp_idle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_isp_idle 0x47B4
-#else
-#define HIVE_ADDR_sp_sem_for_isp_idle 0x47FC
-#endif
-#define HIVE_SIZE_sp_sem_for_isp_idle 20
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_write_byte_addr: 31E6 */
-#else
-/* function ia_css_dmaproxy_sp_write_byte_addr: 33C5 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init: 315D */
-#else
-/* function ia_css_dmaproxy_sp_init: 333C */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_release_dynamic_buf_clock_tick: 2D62 */
-#else
-/* function ia_css_bufq_sp_release_dynamic_buf_clock_tick: 2F04 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_VAMEM_BASE
-#define HIVE_MEM_ISP_VAMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_VAMEM_BASE 0x14
-#define HIVE_SIZE_ISP_VAMEM_BASE 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_VAMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_VAMEM_BASE 0x14
-#define HIVE_SIZE_sp_ISP_VAMEM_BASE 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_rawcopy_sp_tagger
-#define HIVE_MEM_ia_css_rawcopy_sp_tagger scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_rawcopy_sp_tagger 0x6294
-#else
-#define HIVE_ADDR_ia_css_rawcopy_sp_tagger 0x62F0
-#endif
-#define HIVE_SIZE_ia_css_rawcopy_sp_tagger 24
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_rawcopy_sp_tagger scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_rawcopy_sp_tagger 0x6294
-#else
-#define HIVE_ADDR_sp_ia_css_rawcopy_sp_tagger 0x62F0
-#endif
-#define HIVE_SIZE_sp_ia_css_rawcopy_sp_tagger 24
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_exp_ids
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_exp_ids scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_exp_ids 0x5994
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_exp_ids 0x59F0
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_exp_ids 70
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_exp_ids scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_exp_ids 0x5994
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_exp_ids 0x59F0
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_exp_ids 70
-
-#ifndef ISP2401
-/* function ia_css_queue_item_load: 4D00 */
-#else
-/* function ia_css_queue_item_load: 4F5E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_get_state: 593C */
-#else
-/* function ia_css_spctrl_sp_get_state: 5A77 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_uninit: 6026 */
-#else
-/* function ia_css_isys_sp_token_map_uninit: 6161 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_callout_sp_thread
-#define HIVE_MEM_callout_sp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_callout_sp_thread 0x49DC
-#else
-#define HIVE_ADDR_callout_sp_thread 0x1E0
-#endif
-#define HIVE_SIZE_callout_sp_thread 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_callout_sp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_callout_sp_thread 0x49DC
-#else
-#define HIVE_ADDR_sp_callout_sp_thread 0x1E0
-#endif
-#define HIVE_SIZE_sp_callout_sp_thread 4
-
-#ifndef ISP2401
-/* function thread_fiber_sp_init: E2F */
-#else
-/* function thread_fiber_sp_init: E24 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_SP_PMEM_BASE
-#define HIVE_MEM_SP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_SP_PMEM_BASE 0x0
-#define HIVE_SIZE_SP_PMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_SP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_SP_PMEM_BASE 0x0
-#define HIVE_SIZE_sp_SP_PMEM_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_snd_acquire_req: 5F96 */
-#else
-/* function ia_css_isys_sp_token_map_snd_acquire_req: 60D1 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_isp_input_stream_format
-#define HIVE_MEM_sp_isp_input_stream_format scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_input_stream_format 0x40F8
-#else
-#define HIVE_ADDR_sp_isp_input_stream_format 0x4118
-#endif
-#define HIVE_SIZE_sp_isp_input_stream_format 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_isp_input_stream_format scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_isp_input_stream_format 0x40F8
-#else
-#define HIVE_ADDR_sp_sp_isp_input_stream_format 0x4118
-#endif
-#define HIVE_SIZE_sp_sp_isp_input_stream_format 20
-
-#ifndef ISP2401
-/* function __mod: 68A7 */
-#else
-/* function __mod: 6A1A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init_dmem_channel: 3247 */
-#else
-/* function ia_css_dmaproxy_sp_init_dmem_channel: 3426 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_join: CFF */
-#else
-/* function ia_css_thread_sp_join: CF4 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_add_command: 6F4F */
-#else
-/* function ia_css_dmaproxy_sp_add_command: 7082 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_metadata_thread_func: 57F0 */
-#else
-/* function ia_css_sp_metadata_thread_func: 594F */
-#endif
-
-#ifndef ISP2401
-/* function __sp_event_proxy_func_critical: 6975 */
-#else
-/* function __sp_event_proxy_func_critical: 6AE8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_metadata_wait: 5903 */
-#else
-/* function ia_css_sp_metadata_wait: 5A3E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_peek_from_start: F08 */
-#else
-/* function ia_css_circbuf_peek_from_start: EFD */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_event_sp_encode: 3552 */
-#else
-/* function ia_css_event_sp_encode: 3746 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_run: D72 */
-#else
-/* function ia_css_thread_sp_run: D67 */
-#endif
-
-#ifndef ISP2401
-/* function sp_isys_copy_func: 6F6 */
-#else
-/* function sp_isys_copy_func: 68A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_flush: 5A7F */
-#else
-/* function ia_css_isys_sp_backend_flush: 5BBA */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_frame_exists: 599B */
-#else
-/* function ia_css_isys_sp_backend_frame_exists: 5AD6 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_isp_param_init_isp_memories: 4789 */
-#else
-/* function ia_css_sp_isp_param_init_isp_memories: 4A11 */
-#endif
-
-#ifndef ISP2401
-/* function register_isr: 8A9 */
-#else
-/* function register_isr: 83D */
-#endif
-
-/* function irq_raise: C8 */
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_mmu_invalidate: 3124 */
-#else
-/* function ia_css_dmaproxy_sp_mmu_invalidate: 32CC */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_HIVE_IF_SRST_ADDRESS
-#define HIVE_MEM_HIVE_IF_SRST_ADDRESS scalar_processor_2400_dmem
-#define HIVE_ADDR_HIVE_IF_SRST_ADDRESS 0x1B8
-#define HIVE_SIZE_HIVE_IF_SRST_ADDRESS 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_HIVE_IF_SRST_ADDRESS scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_HIVE_IF_SRST_ADDRESS 0x1B8
-#define HIVE_SIZE_sp_HIVE_IF_SRST_ADDRESS 16
-
-#ifndef ISP2401
-/* function pipeline_sp_initialize_stage: 190B */
-#else
-/* function pipeline_sp_initialize_stage: 1945 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_isys_sp_frontend_states
-#define HIVE_MEM_ia_css_isys_sp_frontend_states scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_isys_sp_frontend_states 0x62C8
-#else
-#define HIVE_ADDR_ia_css_isys_sp_frontend_states 0x6324
-#endif
-#define HIVE_SIZE_ia_css_isys_sp_frontend_states 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_isys_sp_frontend_states scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_isys_sp_frontend_states 0x62C8
-#else
-#define HIVE_ADDR_sp_ia_css_isys_sp_frontend_states 0x6324
-#endif
-#define HIVE_SIZE_sp_ia_css_isys_sp_frontend_states 12
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_read_byte_addr_mmio: 6E1E */
-#else
-/* function ia_css_dmaproxy_sp_read_byte_addr_mmio: 6F62 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_done_ds: 3799 */
-#else
-/* function ia_css_ispctrl_sp_done_ds: 39C8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_isp_param_get_mem_inits: 4764 */
-#else
-/* function ia_css_sp_isp_param_get_mem_inits: 49EC */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_init_buffer_queues: 13D0 */
-#else
-/* function ia_css_parambuf_sp_init_buffer_queues: 13F1 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_pfp_spref
-#define HIVE_MEM_vbuf_pfp_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_pfp_spref 0x2F0
-#else
-#define HIVE_ADDR_vbuf_pfp_spref 0x308
-#endif
-#define HIVE_SIZE_vbuf_pfp_spref 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_pfp_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_pfp_spref 0x2F0
-#else
-#define HIVE_ADDR_sp_vbuf_pfp_spref 0x308
-#endif
-#define HIVE_SIZE_sp_vbuf_pfp_spref 4
-
-#ifndef ISP2401
-/* function input_system_cfg: ABB */
-#else
-/* function input_system_cfg: AB5 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_HMEM_BASE
-#define HIVE_MEM_ISP_HMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_HMEM_BASE 0x20
-#define HIVE_SIZE_ISP_HMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_HMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_HMEM_BASE 0x20
-#define HIVE_SIZE_sp_ISP_HMEM_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_frames
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_frames 0x59DC
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_frames 0x5A38
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_frames 280
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_frames 0x59DC
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_frames 0x5A38
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_frames 280
-
-#ifndef ISP2401
-/* function qos_scheduler_init_stage_budget: 65E8 */
-#else
-/* function qos_scheduler_init_stage_budget: 6750 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_release: 5AF4 */
-#else
-/* function ia_css_isys_sp_backend_release: 5C2F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_destroy: 5B1E */
-#else
-/* function ia_css_isys_sp_backend_destroy: 5C59 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp2host_buffer_queue_handle
-#define HIVE_MEM_sp2host_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp2host_buffer_queue_handle 0x5AF4
-#else
-#define HIVE_ADDR_sp2host_buffer_queue_handle 0x5B50
-#endif
-#define HIVE_SIZE_sp2host_buffer_queue_handle 96
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp2host_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp2host_buffer_queue_handle 0x5AF4
-#else
-#define HIVE_ADDR_sp_sp2host_buffer_queue_handle 0x5B50
-#endif
-#define HIVE_SIZE_sp_sp2host_buffer_queue_handle 96
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_check_mipi_frame_size: 5F5A */
-#else
-/* function ia_css_isys_sp_token_map_check_mipi_frame_size: 6095 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_isp_vars: 4483 */
-#else
-/* function ia_css_ispctrl_sp_init_isp_vars: 46DE */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_has_empty_mipi_buffer_cb: 5B70 */
-#else
-/* function ia_css_isys_sp_frontend_has_empty_mipi_buffer_cb: 5CAB */
-#endif
-
-#ifndef ISP2401
-/* function sp_warning: 8DC */
-#else
-/* function sp_warning: 870 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_vbuf_enqueue: 6304 */
-#else
-/* function ia_css_rmgr_sp_vbuf_enqueue: 643F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_tag_exp_id: 2142 */
-#else
-/* function ia_css_tagger_sp_tag_exp_id: 2192 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_write: 31FD */
-#else
-/* function ia_css_dmaproxy_sp_write: 33DC */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_release_in_param: 1250 */
-#else
-/* function ia_css_parambuf_sp_release_in_param: 1245 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_irq_sw_interrupt_token
-#define HIVE_MEM_irq_sw_interrupt_token scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_irq_sw_interrupt_token 0x40F4
-#else
-#define HIVE_ADDR_irq_sw_interrupt_token 0x4114
-#endif
-#define HIVE_SIZE_irq_sw_interrupt_token 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_irq_sw_interrupt_token scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_irq_sw_interrupt_token 0x40F4
-#else
-#define HIVE_ADDR_sp_irq_sw_interrupt_token 0x4114
-#endif
-#define HIVE_SIZE_sp_irq_sw_interrupt_token 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_isp_addresses
-#define HIVE_MEM_sp_isp_addresses scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_addresses 0x5F44
-#else
-#define HIVE_ADDR_sp_isp_addresses 0x5FA4
-#endif
-#define HIVE_SIZE_sp_isp_addresses 172
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_isp_addresses scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_isp_addresses 0x5F44
-#else
-#define HIVE_ADDR_sp_sp_isp_addresses 0x5FA4
-#endif
-#define HIVE_SIZE_sp_sp_isp_addresses 172
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_acq_gen: 6229 */
-#else
-/* function ia_css_rmgr_sp_acq_gen: 6364 */
-#endif
-
-#ifndef ISP2401
-/* function receiver_reg_load: AD0 */
-#else
-/* function receiver_reg_load: ACA */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isps
-#define HIVE_MEM_isps scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isps 0x6300
-#else
-#define HIVE_ADDR_isps 0x635C
-#endif
-#define HIVE_SIZE_isps 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isps scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isps 0x6300
-#else
-#define HIVE_ADDR_sp_isps 0x635C
-#endif
-#define HIVE_SIZE_sp_isps 28
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host_sp_queues_initialized
-#define HIVE_MEM_host_sp_queues_initialized scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host_sp_queues_initialized 0x410C
-#else
-#define HIVE_ADDR_host_sp_queues_initialized 0x412C
-#endif
-#define HIVE_SIZE_host_sp_queues_initialized 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host_sp_queues_initialized scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host_sp_queues_initialized 0x410C
-#else
-#define HIVE_ADDR_sp_host_sp_queues_initialized 0x412C
-#endif
-#define HIVE_SIZE_sp_host_sp_queues_initialized 4
-
-#ifndef ISP2401
-/* function ia_css_queue_uninit: 4BCC */
-#else
-/* function ia_css_queue_uninit: 4E2A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_ispctrl_sp_isp_started
-#define HIVE_MEM_ia_css_ispctrl_sp_isp_started scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_ispctrl_sp_isp_started 0x5BFC
-#else
-#define HIVE_ADDR_ia_css_ispctrl_sp_isp_started 0x5C58
-#endif
-#define HIVE_SIZE_ia_css_ispctrl_sp_isp_started 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_ispctrl_sp_isp_started scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_ispctrl_sp_isp_started 0x5BFC
-#else
-#define HIVE_ADDR_sp_ia_css_ispctrl_sp_isp_started 0x5C58
-#endif
-#define HIVE_SIZE_sp_ia_css_ispctrl_sp_isp_started 4
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_release_dynamic_buf: 2DCE */
-#else
-/* function ia_css_bufq_sp_release_dynamic_buf: 2F70 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_height_exception: 32F5 */
-#else
-/* function ia_css_dmaproxy_sp_set_height_exception: 34E9 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init_vmem_channel: 327A */
-#else
-/* function ia_css_dmaproxy_sp_init_vmem_channel: 345A */
-#endif
-
-#ifndef ISP2401
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_num_ready_threads
-#define HIVE_MEM_num_ready_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_num_ready_threads 0x49E4
-#define HIVE_SIZE_num_ready_threads 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_num_ready_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_num_ready_threads 0x49E4
-#define HIVE_SIZE_sp_num_ready_threads 4
-
-/* function ia_css_dmaproxy_sp_write_byte_addr_mmio: 31CF */
-#else
-/* function ia_css_dmaproxy_sp_write_byte_addr_mmio: 33AE */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_spref
-#define HIVE_MEM_vbuf_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_spref 0x2EC
-#else
-#define HIVE_ADDR_vbuf_spref 0x304
-#endif
-#define HIVE_SIZE_vbuf_spref 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_spref 0x2EC
-#else
-#define HIVE_ADDR_sp_vbuf_spref 0x304
-#endif
-#define HIVE_SIZE_sp_vbuf_spref 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_metadata_thread
-#define HIVE_MEM_sp_metadata_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_metadata_thread 0x4998
-#define HIVE_SIZE_sp_metadata_thread 68
-#else
-#define HIVE_ADDR_sp_metadata_thread 0x49F8
-#define HIVE_SIZE_sp_metadata_thread 72
-#endif
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_metadata_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_metadata_thread 0x4998
-#define HIVE_SIZE_sp_sp_metadata_thread 68
-#else
-#define HIVE_ADDR_sp_sp_metadata_thread 0x49F8
-#define HIVE_SIZE_sp_sp_metadata_thread 72
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_enqueue: 4B16 */
-#else
-/* function ia_css_queue_enqueue: 4D74 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_request
-#define HIVE_MEM_ia_css_flash_sp_request scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_request 0x4A98
-#else
-#define HIVE_ADDR_ia_css_flash_sp_request 0x4AF4
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_request 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_request scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_request 0x4A98
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_request 0x4AF4
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_request 4
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_vmem_write: 31A0 */
-#else
-/* function ia_css_dmaproxy_sp_vmem_write: 337F */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_tagger_frames
-#define HIVE_MEM_tagger_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_tagger_frames 0x49EC
-#else
-#define HIVE_ADDR_tagger_frames 0x4A48
-#endif
-#define HIVE_SIZE_tagger_frames 168
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_tagger_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_tagger_frames 0x49EC
-#else
-#define HIVE_ADDR_sp_tagger_frames 0x4A48
-#endif
-#define HIVE_SIZE_sp_tagger_frames 168
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_snd_capture_req: 5FB8 */
-#else
-/* function ia_css_isys_sp_token_map_snd_capture_req: 60F3 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_reading_if
-#define HIVE_MEM_sem_for_reading_if scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_reading_if 0x47C8
-#else
-#define HIVE_ADDR_sem_for_reading_if 0x4810
-#endif
-#define HIVE_SIZE_sem_for_reading_if 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_reading_if scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_reading_if 0x47C8
-#else
-#define HIVE_ADDR_sp_sem_for_reading_if 0x4810
-#endif
-#define HIVE_SIZE_sp_sem_for_reading_if 20
-
-#ifndef ISP2401
-/* function sp_generate_interrupts: 95B */
-#else
-/* function sp_generate_interrupts: 8EF */
-
-/* function ia_css_pipeline_sp_start: 1858 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_start: 181E */
-#else
-/* function ia_css_thread_default_callout: 6BE3 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_init: 50F3 */
-#else
-/* function ia_css_sp_rawcopy_init: 5351 */
-#endif
-
-#ifndef ISP2401
-/* function tmr_clock_read: 6596 */
-#else
-/* function tmr_clock_read: 66D1 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_BAMEM_BASE
-#define HIVE_MEM_ISP_BAMEM_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ISP_BAMEM_BASE 0x2F8
-#else
-#define HIVE_ADDR_ISP_BAMEM_BASE 0x310
-#endif
-#define HIVE_SIZE_ISP_BAMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_BAMEM_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ISP_BAMEM_BASE 0x2F8
-#else
-#define HIVE_ADDR_sp_ISP_BAMEM_BASE 0x310
-#endif
-#define HIVE_SIZE_sp_ISP_BAMEM_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_rcv_capture_ack: 5C1F */
-#else
-/* function ia_css_isys_sp_frontend_rcv_capture_ack: 5D5A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_sems_for_sp2host_buf_queues
-#define HIVE_MEM_ia_css_bufq_sp_sems_for_sp2host_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x5B54
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x5BB0
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_sems_for_sp2host_buf_queues 160
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x5B54
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x5BB0
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues 160
-
-#ifndef ISP2401
-/* function css_get_frame_processing_time_start: 1FAF */
-#else
-/* function css_get_frame_processing_time_start: 1FFF */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_frame
-#define HIVE_MEM_sp_all_cbs_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_frame 0x47DC
-#else
-#define HIVE_ADDR_sp_all_cbs_frame 0x4824
-#endif
-#define HIVE_SIZE_sp_all_cbs_frame 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_frame 0x47DC
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_frame 0x4824
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_frame 16
-
-#ifndef ISP2401
-/* function thread_sp_queue_print: D8F */
-#else
-/* function thread_sp_queue_print: D84 */
-#endif
-
-#ifndef ISP2401
-/* function sp_notify_eof: 907 */
-#else
-/* function sp_notify_eof: 89B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_str2mem
-#define HIVE_MEM_sem_for_str2mem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_str2mem 0x47EC
-#else
-#define HIVE_ADDR_sem_for_str2mem 0x4834
-#endif
-#define HIVE_SIZE_sem_for_str2mem 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_str2mem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_str2mem 0x47EC
-#else
-#define HIVE_ADDR_sp_sem_for_str2mem 0x4834
-#endif
-#define HIVE_SIZE_sp_sem_for_str2mem 20
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_is_marked_from_start: 2B41 */
-#else
-/* function ia_css_tagger_buf_sp_is_marked_from_start: 2CE3 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_acquire_dynamic_buf: 2F86 */
-#else
-/* function ia_css_bufq_sp_acquire_dynamic_buf: 3128 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_destroy: 101D */
-#else
-/* function ia_css_circbuf_destroy: 1012 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_PMEM_BASE
-#define HIVE_MEM_ISP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_PMEM_BASE 0xC
-#define HIVE_SIZE_ISP_PMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_PMEM_BASE 0xC
-#define HIVE_SIZE_sp_ISP_PMEM_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_sp_isp_param_mem_load: 46F7 */
-#else
-/* function ia_css_sp_isp_param_mem_load: 497F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_pop_from_start: 292D */
-#else
-/* function ia_css_tagger_buf_sp_pop_from_start: 2ACF */
-#endif
-
-#ifndef ISP2401
-/* function __div: 685F */
-#else
-/* function __div: 69D2 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_create: 5DF0 */
-#else
-/* function ia_css_isys_sp_frontend_create: 5F2B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_release_vbuf: 6323 */
-#else
-/* function ia_css_rmgr_sp_refcount_release_vbuf: 645E */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_in_use
-#define HIVE_MEM_ia_css_flash_sp_in_use scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_in_use 0x4A9C
-#else
-#define HIVE_ADDR_ia_css_flash_sp_in_use 0x4AF8
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_in_use 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_in_use scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_use 0x4A9C
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_use 0x4AF8
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_in_use 4
-
-#ifndef ISP2401
-/* function ia_css_thread_sem_sp_wait: 6B42 */
-#else
-/* function ia_css_thread_sem_sp_wait: 6CB7 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_sleep_mode
-#define HIVE_MEM_sp_sleep_mode scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sleep_mode 0x4110
-#else
-#define HIVE_ADDR_sp_sleep_mode 0x4130
-#endif
-#define HIVE_SIZE_sp_sleep_mode 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_sleep_mode scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_sleep_mode 0x4110
-#else
-#define HIVE_ADDR_sp_sp_sleep_mode 0x4130
-#endif
-#define HIVE_SIZE_sp_sp_sleep_mode 4
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_push: 2A3C */
-#else
-/* function ia_css_tagger_buf_sp_push: 2BDE */
-#endif
-
-/* function mmu_invalidate_cache: D3 */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_max_cb_elems
-#define HIVE_MEM_sp_max_cb_elems scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_max_cb_elems 0x148
-#define HIVE_SIZE_sp_max_cb_elems 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_max_cb_elems scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_max_cb_elems 0x148
-#define HIVE_SIZE_sp_sp_max_cb_elems 8
-
-#ifndef ISP2401
-/* function ia_css_queue_remote_init: 4BEE */
-#else
-/* function ia_css_queue_remote_init: 4E4C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_stop_req
-#define HIVE_MEM_isp_stop_req scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_stop_req 0x4680
-#else
-#define HIVE_ADDR_isp_stop_req 0x46C8
-#endif
-#define HIVE_SIZE_isp_stop_req 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_stop_req scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_stop_req 0x4680
-#else
-#define HIVE_ADDR_sp_isp_stop_req 0x46C8
-#endif
-#define HIVE_SIZE_sp_isp_stop_req 4
-
-#ifndef ISP2401
-#define HIVE_ICACHE_sp_critical_SEGMENT_START 0
-#define HIVE_ICACHE_sp_critical_NUM_SEGMENTS  1
-#endif
-
-#endif /* _sp_map_h_ */
-#ifndef ISP2401
-extern void sh_css_dump_sp_dmem(void);
-void sh_css_dump_sp_dmem(void)
-{
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/csi_rx_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/csi_rx_global.h
deleted file mode 100644
index 146a578b7c74..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/csi_rx_global.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __CSI_RX_GLOBAL_H_INCLUDED__
-#define __CSI_RX_GLOBAL_H_INCLUDED__
-
-#include <type_support.h>
-
-typedef enum {
-	CSI_MIPI_PACKET_TYPE_UNDEFINED = 0,
-	CSI_MIPI_PACKET_TYPE_LONG,
-	CSI_MIPI_PACKET_TYPE_SHORT,
-	CSI_MIPI_PACKET_TYPE_RESERVED,
-	N_CSI_MIPI_PACKET_TYPE
-} csi_mipi_packet_type_t;
-
-typedef struct csi_rx_backend_lut_entry_s	csi_rx_backend_lut_entry_t;
-struct csi_rx_backend_lut_entry_s {
-	uint32_t	long_packet_entry;
-	uint32_t	short_packet_entry;
-};
-
-typedef struct csi_rx_backend_cfg_s csi_rx_backend_cfg_t;
-struct csi_rx_backend_cfg_s {
-	/* LUT entry for the packet */
-	csi_rx_backend_lut_entry_t lut_entry;
-
-	/* can be derived from the Data Type */
-	csi_mipi_packet_type_t csi_mipi_packet_type;
-
-	struct {
-		bool     comp_enable;
-		uint32_t virtual_channel;
-		uint32_t data_type;
-		uint32_t comp_scheme;
-		uint32_t comp_predictor;
-		uint32_t comp_bit_idx;
-	} csi_mipi_cfg;
-};
-
-typedef struct csi_rx_frontend_cfg_s csi_rx_frontend_cfg_t;
-struct csi_rx_frontend_cfg_s {
-	uint32_t active_lanes;
-};
-
-extern const uint32_t N_SHORT_PACKET_LUT_ENTRIES[N_CSI_RX_BACKEND_ID];
-extern const uint32_t N_LONG_PACKET_LUT_ENTRIES[N_CSI_RX_BACKEND_ID];
-extern const uint32_t N_CSI_RX_FE_CTRL_DLANES[N_CSI_RX_FRONTEND_ID];
-/* sid_width for CSI_RX_BACKEND<N>_ID */
-extern const uint32_t N_CSI_RX_BE_SID_WIDTH[N_CSI_RX_BACKEND_ID];
-
-#endif /* __CSI_RX_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_configs.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_configs.c
deleted file mode 100644
index 325b821f276c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_configs.c
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Generated code: do not edit or commmit. */
-
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_configs.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_iterator(
-	const struct ia_css_binary *binary,
-	const struct ia_css_iterator_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_iterator() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.iterator.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.iterator.offset;
-		}
-		if (size) {
-			ia_css_iterator_config((struct sh_css_isp_iterator_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_iterator() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_copy_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_copy_output_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_copy_output() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.copy_output.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.copy_output.offset;
-		}
-		if (size) {
-			ia_css_copy_output_config((struct sh_css_isp_copy_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_copy_output() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_crop(
-	const struct ia_css_binary *binary,
-	const struct ia_css_crop_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_crop() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.crop.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.crop.offset;
-		}
-		if (size) {
-			ia_css_crop_config((struct sh_css_isp_crop_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_crop() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_fpn(
-	const struct ia_css_binary *binary,
-	const struct ia_css_fpn_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_fpn() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.fpn.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.fpn.offset;
-		}
-		if (size) {
-			ia_css_fpn_config((struct sh_css_isp_fpn_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_fpn() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_dvs(
-	const struct ia_css_binary *binary,
-	const struct ia_css_dvs_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_dvs() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.dvs.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.dvs.offset;
-		}
-		if (size) {
-			ia_css_dvs_config((struct sh_css_isp_dvs_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_dvs() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_qplane(
-	const struct ia_css_binary *binary,
-	const struct ia_css_qplane_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_qplane() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.qplane.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.qplane.offset;
-		}
-		if (size) {
-			ia_css_qplane_config((struct sh_css_isp_qplane_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_qplane() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output0(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output0_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output0() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.output0.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.output0.offset;
-		}
-		if (size) {
-			ia_css_output0_config((struct sh_css_isp_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output0() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output1(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output1_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output1() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.output1.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.output1.offset;
-		}
-		if (size) {
-			ia_css_output1_config((struct sh_css_isp_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output1() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.output.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.output.offset;
-		}
-		if (size) {
-			ia_css_output_config((struct sh_css_isp_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-#ifdef ISP2401
-
-void
-ia_css_configure_sc(
-	const struct ia_css_binary *binary,
-	const struct ia_css_sc_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_sc() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.sc.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.sc.offset;
-		}
-		if (size) {
-			ia_css_sc_config((struct sh_css_isp_sc_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_sc() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-#endif
-
-void
-ia_css_configure_raw(
-	const struct ia_css_binary *binary,
-	const struct ia_css_raw_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_raw() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.raw.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.raw.offset;
-		}
-		if (size) {
-			ia_css_raw_config((struct sh_css_isp_raw_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_raw() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_tnr(
-	const struct ia_css_binary *binary,
-	const struct ia_css_tnr_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_tnr() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.tnr.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.tnr.offset;
-		}
-		if (size) {
-			ia_css_tnr_config((struct sh_css_isp_tnr_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_tnr() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_ref(
-	const struct ia_css_binary *binary,
-	const struct ia_css_ref_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_ref() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.ref.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.ref.offset;
-		}
-		if (size) {
-			ia_css_ref_config((struct sh_css_isp_ref_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_ref() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_vf(
-	const struct ia_css_binary *binary,
-	const struct ia_css_vf_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_vf() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.vf.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.vf.offset;
-		}
-		if (size) {
-			ia_css_vf_config((struct sh_css_isp_vf_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_vf() leave:\n");
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_configs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_configs.h
deleted file mode 100644
index 8aacd3dbc05a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_configs.h
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifdef IA_CSS_INCLUDE_CONFIGURATIONS
-#include "isp/kernels/crop/crop_1.0/ia_css_crop.host.h"
-#include "isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h"
-#include "isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h"
-#include "isp/kernels/ob/ob_1.0/ia_css_ob.host.h"
-#include "isp/kernels/output/output_1.0/ia_css_output.host.h"
-#include "isp/kernels/qplane/qplane_2/ia_css_qplane.host.h"
-#include "isp/kernels/raw/raw_1.0/ia_css_raw.host.h"
-#include "isp/kernels/ref/ref_1.0/ia_css_ref.host.h"
-#include "isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h"
-#ifdef ISP2401
-#include "isp/kernels/sc/sc_1.0/ia_css_sc.host.h"
-#endif
-#include "isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h"
-#include "isp/kernels/vf/vf_1.0/ia_css_vf.host.h"
-#include "isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h"
-#include "isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h"
-#endif /* IA_CSS_INCLUDE_CONFIGURATIONS */
-/* Generated code: do not edit or commmit. */
-
-#ifndef _IA_CSS_ISP_CONFIG_H
-#define _IA_CSS_ISP_CONFIG_H
-
-/* Code generated by genparam/gencode.c:gen_param_enum() */
-
-enum ia_css_configuration_ids {
-	IA_CSS_ITERATOR_CONFIG_ID,
-	IA_CSS_COPY_OUTPUT_CONFIG_ID,
-	IA_CSS_CROP_CONFIG_ID,
-	IA_CSS_FPN_CONFIG_ID,
-	IA_CSS_DVS_CONFIG_ID,
-	IA_CSS_QPLANE_CONFIG_ID,
-	IA_CSS_OUTPUT0_CONFIG_ID,
-	IA_CSS_OUTPUT1_CONFIG_ID,
-	IA_CSS_OUTPUT_CONFIG_ID,
-#ifdef ISP2401
-	IA_CSS_SC_CONFIG_ID,
-#endif
-	IA_CSS_RAW_CONFIG_ID,
-	IA_CSS_TNR_CONFIG_ID,
-	IA_CSS_REF_CONFIG_ID,
-	IA_CSS_VF_CONFIG_ID,
-	IA_CSS_NUM_CONFIGURATION_IDS
-};
-
-/* Code generated by genparam/gencode.c:gen_param_offsets() */
-
-struct ia_css_config_memory_offsets {
-	struct {
-		struct ia_css_isp_parameter iterator;
-		struct ia_css_isp_parameter copy_output;
-		struct ia_css_isp_parameter crop;
-		struct ia_css_isp_parameter fpn;
-		struct ia_css_isp_parameter dvs;
-		struct ia_css_isp_parameter qplane;
-		struct ia_css_isp_parameter output0;
-		struct ia_css_isp_parameter output1;
-		struct ia_css_isp_parameter output;
-#ifdef ISP2401
-		struct ia_css_isp_parameter sc;
-#endif
-		struct ia_css_isp_parameter raw;
-		struct ia_css_isp_parameter tnr;
-		struct ia_css_isp_parameter ref;
-		struct ia_css_isp_parameter vf;
-	} dmem;
-};
-
-#if defined(IA_CSS_INCLUDE_CONFIGURATIONS)
-
-#include "ia_css_stream.h"   /* struct ia_css_stream */
-#include "ia_css_binary.h"   /* struct ia_css_binary */
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_iterator(
-	const struct ia_css_binary *binary,
-	const struct ia_css_iterator_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_copy_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_copy_output_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_crop(
-	const struct ia_css_binary *binary,
-	const struct ia_css_crop_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_fpn(
-	const struct ia_css_binary *binary,
-	const struct ia_css_fpn_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_dvs(
-	const struct ia_css_binary *binary,
-	const struct ia_css_dvs_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_qplane(
-	const struct ia_css_binary *binary,
-	const struct ia_css_qplane_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output0(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output0_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output1(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output1_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-#ifdef ISP2401
-void
-ia_css_configure_sc(
-	const struct ia_css_binary *binary,
-	const struct ia_css_sc_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-#endif
-void
-ia_css_configure_raw(
-	const struct ia_css_binary *binary,
-	const struct ia_css_raw_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_tnr(
-	const struct ia_css_binary *binary,
-	const struct ia_css_tnr_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_ref(
-	const struct ia_css_binary *binary,
-	const struct ia_css_ref_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_vf(
-	const struct ia_css_binary *binary,
-	const struct ia_css_vf_configuration *config_dmem);
-
-#endif /* IA_CSS_INCLUDE_CONFIGURATION */
-
-#endif /* _IA_CSS_ISP_CONFIG_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_params.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_params.c
deleted file mode 100644
index 11e4463ebb50..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_params.c
+++ /dev/null
@@ -1,3220 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#define IA_CSS_INCLUDE_PARAMETERS
-#include "sh_css_params.h"
-#include "isp/kernels/aa/aa_2/ia_css_aa2.host.h"
-#include "isp/kernels/anr/anr_1.0/ia_css_anr.host.h"
-#include "isp/kernels/anr/anr_2/ia_css_anr2.host.h"
-#include "isp/kernels/bh/bh_2/ia_css_bh.host.h"
-#include "isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h"
-#include "isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h"
-#include "isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h"
-#include "isp/kernels/crop/crop_1.0/ia_css_crop.host.h"
-#include "isp/kernels/csc/csc_1.0/ia_css_csc.host.h"
-#include "isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h"
-#include "isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h"
-#include "isp/kernels/ctc/ctc2/ia_css_ctc2.host.h"
-#include "isp/kernels/de/de_1.0/ia_css_de.host.h"
-#include "isp/kernels/de/de_2/ia_css_de2.host.h"
-#include "isp/kernels/dp/dp_1.0/ia_css_dp.host.h"
-#include "isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h"
-#include "isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h"
-#include "isp/kernels/gc/gc_1.0/ia_css_gc.host.h"
-#include "isp/kernels/gc/gc_2/ia_css_gc2.host.h"
-#include "isp/kernels/macc/macc_1.0/ia_css_macc.host.h"
-#include "isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h"
-#include "isp/kernels/ob/ob_1.0/ia_css_ob.host.h"
-#include "isp/kernels/ob/ob2/ia_css_ob2.host.h"
-#include "isp/kernels/output/output_1.0/ia_css_output.host.h"
-#include "isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h"
-#include "isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h"
-#include "isp/kernels/sc/sc_1.0/ia_css_sc.host.h"
-#include "isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h"
-#include "isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h"
-#include "isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h"
-#include "isp/kernels/uds/uds_1.0/ia_css_uds_param.h"
-#include "isp/kernels/wb/wb_1.0/ia_css_wb.host.h"
-#include "isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h"
-#include "isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h"
-#include "isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h"
-#include "isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h"
-#include "isp/kernels/fc/fc_1.0/ia_css_formats.host.h"
-#include "isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h"
-#include "isp/kernels/dpc2/ia_css_dpc2.host.h"
-#include "isp/kernels/eed1_8/ia_css_eed1_8.host.h"
-#include "isp/kernels/bnlm/ia_css_bnlm.host.h"
-#include "isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h"
-/* Generated code: do not edit or commmit. */
-
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_params.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_aa(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.aa.size;
-	unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.aa.offset;
-
-	if (size) {
-		struct sh_css_isp_aa_params *t =  (struct sh_css_isp_aa_params *)
-			&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-		t->strength = params->aa_config.strength;
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_anr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.anr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.anr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr() enter:\n");
-
-			ia_css_anr_encode((struct sh_css_isp_anr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->anr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_anr2(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.anr2.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.anr2.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr2() enter:\n");
-
-			ia_css_anr2_vmem_encode((struct ia_css_isp_anr2_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->anr_thres,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr2() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_bh(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.bh.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.bh.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() enter:\n");
-
-			ia_css_bh_encode((struct sh_css_isp_bh_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->s3a_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->hmem0.bh.size;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() enter:\n");
-
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_HMEM0] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_cnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.cnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.cnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_cnr() enter:\n");
-
-			ia_css_cnr_encode((struct sh_css_isp_cnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->cnr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_cnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_crop(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.crop.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.crop.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_crop() enter:\n");
-
-			ia_css_crop_encode((struct sh_css_isp_crop_isp_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->crop_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_crop() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_csc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.csc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.csc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_csc() enter:\n");
-
-			ia_css_csc_encode((struct sh_css_isp_csc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->cc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_csc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_dp(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.dp.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.dp.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_dp() enter:\n");
-
-			ia_css_dp_encode((struct sh_css_isp_dp_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dp_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_dp() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_bnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.bnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.bnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bnr() enter:\n");
-
-			ia_css_bnr_encode((struct sh_css_isp_bnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->nr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_de(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.de.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.de.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_de() enter:\n");
-
-			ia_css_de_encode((struct sh_css_isp_de_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->de_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_de() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ecd(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ecd.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ecd.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ecd() enter:\n");
-
-			ia_css_ecd_encode((struct sh_css_isp_ecd_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ecd_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ecd() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_formats(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.formats.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.formats.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_formats() enter:\n");
-
-			ia_css_formats_encode((struct sh_css_isp_formats_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->formats_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_formats() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_fpn(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.fpn.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.fpn.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fpn() enter:\n");
-
-			ia_css_fpn_encode((struct sh_css_isp_fpn_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->fpn_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fpn() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_gc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.gc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.gc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() enter:\n");
-
-			ia_css_gc_encode((struct sh_css_isp_gc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->gc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem1.gc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem1.gc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() enter:\n");
-
-			ia_css_gc_vamem_encode((struct sh_css_isp_gc_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM1].address[offset],
-					&params->gc_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM1] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ce(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ce.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ce.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ce() enter:\n");
-
-			ia_css_ce_encode((struct sh_css_isp_ce_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ce_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ce() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_yuv2rgb(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.yuv2rgb.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.yuv2rgb.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yuv2rgb() enter:\n");
-
-			ia_css_yuv2rgb_encode((struct sh_css_isp_csc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->yuv2rgb_cc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yuv2rgb() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_rgb2yuv(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.rgb2yuv.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.rgb2yuv.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_rgb2yuv() enter:\n");
-
-			ia_css_rgb2yuv_encode((struct sh_css_isp_csc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->rgb2yuv_cc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_rgb2yuv() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_r_gamma(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem0.r_gamma.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem0.r_gamma.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_r_gamma() enter:\n");
-
-			ia_css_r_gamma_vamem_encode((struct sh_css_isp_rgb_gamma_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM0].address[offset],
-					&params->r_gamma_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM0] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_r_gamma() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_g_gamma(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem1.g_gamma.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem1.g_gamma.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_g_gamma() enter:\n");
-
-			ia_css_g_gamma_vamem_encode((struct sh_css_isp_rgb_gamma_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM1].address[offset],
-					&params->g_gamma_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM1] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_g_gamma() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_b_gamma(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem2.b_gamma.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem2.b_gamma.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_b_gamma() enter:\n");
-
-			ia_css_b_gamma_vamem_encode((struct sh_css_isp_rgb_gamma_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM2].address[offset],
-					&params->b_gamma_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM2] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_b_gamma() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_uds(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.uds.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.uds.offset;
-
-		if (size) {
-			struct sh_css_sp_uds_params *p;
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_uds() enter:\n");
-
-			p = (struct sh_css_sp_uds_params *)
-				&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-			p->crop_pos = params->uds_config.crop_pos;
-			p->uds = params->uds_config.uds;
-
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_uds() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_raa(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.raa.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.raa.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_raa() enter:\n");
-
-			ia_css_raa_encode((struct sh_css_isp_aa_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->raa_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_raa() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_s3a(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.s3a.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.s3a.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_s3a() enter:\n");
-
-			ia_css_s3a_encode((struct sh_css_isp_s3a_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->s3a_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_s3a() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ob(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ob.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ob.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() enter:\n");
-
-			ia_css_ob_encode((struct sh_css_isp_ob_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ob_config,
-&params->stream_configs.ob, size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.ob.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.ob.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() enter:\n");
-
-			ia_css_ob_vmem_encode((struct sh_css_isp_ob_vmem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->ob_config,
-&params->stream_configs.ob, size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_output(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.output.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.output.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_output() enter:\n");
-
-			ia_css_output_encode((struct sh_css_isp_output_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->output_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_output() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sc() enter:\n");
-
-			ia_css_sc_encode((struct sh_css_isp_sc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->sc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_bds(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.bds.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.bds.offset;
-
-		if (size) {
-			struct sh_css_isp_bds_params *p;
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bds() enter:\n");
-
-			p = (struct sh_css_isp_bds_params *)
-				&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-			p->baf_strength = params->bds_config.strength;
-
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bds() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_tnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.tnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.tnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_tnr() enter:\n");
-
-			ia_css_tnr_encode((struct sh_css_isp_tnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->tnr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_tnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_macc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.macc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.macc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_macc() enter:\n");
-
-			ia_css_macc_encode((struct sh_css_isp_macc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->macc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_macc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_horicoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_horicoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_horicoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horicoef() enter:\n");
-
-			ia_css_sdis_horicoef_vmem_encode((struct sh_css_isp_sdis_hori_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horicoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_vertcoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_vertcoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_vertcoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertcoef() enter:\n");
-
-			ia_css_sdis_vertcoef_vmem_encode((struct sh_css_isp_sdis_vert_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertcoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_horiproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_horiproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_horiproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horiproj() enter:\n");
-
-			ia_css_sdis_horiproj_encode((struct sh_css_isp_sdis_hori_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horiproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_vertproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_vertproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_vertproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertproj() enter:\n");
-
-			ia_css_sdis_vertproj_encode((struct sh_css_isp_sdis_vert_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_horicoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_horicoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_horicoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horicoef() enter:\n");
-
-			ia_css_sdis2_horicoef_vmem_encode((struct sh_css_isp_sdis_hori_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horicoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_vertcoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_vertcoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_vertcoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertcoef() enter:\n");
-
-			ia_css_sdis2_vertcoef_vmem_encode((struct sh_css_isp_sdis_vert_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertcoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_horiproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_horiproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_horiproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horiproj() enter:\n");
-
-			ia_css_sdis2_horiproj_encode((struct sh_css_isp_sdis_hori_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horiproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_vertproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_vertproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_vertproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertproj() enter:\n");
-
-			ia_css_sdis2_vertproj_encode((struct sh_css_isp_sdis_vert_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_wb(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.wb.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.wb.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_wb() enter:\n");
-
-			ia_css_wb_encode((struct sh_css_isp_wb_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->wb_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_wb() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_nr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.nr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.nr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_nr() enter:\n");
-
-			ia_css_nr_encode((struct sh_css_isp_ynr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->nr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_nr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_yee(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.yee.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.yee.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yee() enter:\n");
-
-			ia_css_yee_encode((struct sh_css_isp_yee_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->yee_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yee() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ynr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ynr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ynr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ynr() enter:\n");
-
-			ia_css_ynr_encode((struct sh_css_isp_yee2_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ynr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ynr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_fc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.fc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.fc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fc() enter:\n");
-
-			ia_css_fc_encode((struct sh_css_isp_fc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->fc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ctc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ctc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ctc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() enter:\n");
-
-			ia_css_ctc_encode((struct sh_css_isp_ctc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ctc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem0.ctc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem0.ctc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() enter:\n");
-
-			ia_css_ctc_vamem_encode((struct sh_css_isp_ctc_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM0].address[offset],
-					&params->ctc_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM0] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_xnr_table(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem1.xnr_table.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem1.xnr_table.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr_table() enter:\n");
-
-			ia_css_xnr_table_vamem_encode((struct sh_css_isp_xnr_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM1].address[offset],
-					&params->xnr_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM1] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr_table() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_xnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.xnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.xnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr() enter:\n");
-
-			ia_css_xnr_encode((struct sh_css_isp_xnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->xnr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_xnr3(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.xnr3.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.xnr3.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() enter:\n");
-
-			ia_css_xnr3_encode((struct sh_css_isp_xnr3_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->xnr3_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() leave:\n");
-		}
-
-	}
-#ifdef ISP2401
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.xnr3.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.xnr3.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() enter:\n");
-
-			ia_css_xnr3_vmem_encode((struct sh_css_isp_xnr3_vmem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->xnr3_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() leave:\n");
-		}
-
-	}
-#endif
-}
-
-/* Code generated by genparam/gencode.c:gen_param_process_table() */
-
-void (* ia_css_kernel_process_param[IA_CSS_NUM_PARAMETER_IDS])(
-			unsigned pipe_id,
-			const struct ia_css_pipeline_stage *stage,
-			struct ia_css_isp_parameters *params) = {
-	ia_css_process_aa,
-	ia_css_process_anr,
-	ia_css_process_anr2,
-	ia_css_process_bh,
-	ia_css_process_cnr,
-	ia_css_process_crop,
-	ia_css_process_csc,
-	ia_css_process_dp,
-	ia_css_process_bnr,
-	ia_css_process_de,
-	ia_css_process_ecd,
-	ia_css_process_formats,
-	ia_css_process_fpn,
-	ia_css_process_gc,
-	ia_css_process_ce,
-	ia_css_process_yuv2rgb,
-	ia_css_process_rgb2yuv,
-	ia_css_process_r_gamma,
-	ia_css_process_g_gamma,
-	ia_css_process_b_gamma,
-	ia_css_process_uds,
-	ia_css_process_raa,
-	ia_css_process_s3a,
-	ia_css_process_ob,
-	ia_css_process_output,
-	ia_css_process_sc,
-	ia_css_process_bds,
-	ia_css_process_tnr,
-	ia_css_process_macc,
-	ia_css_process_sdis_horicoef,
-	ia_css_process_sdis_vertcoef,
-	ia_css_process_sdis_horiproj,
-	ia_css_process_sdis_vertproj,
-	ia_css_process_sdis2_horicoef,
-	ia_css_process_sdis2_vertcoef,
-	ia_css_process_sdis2_horiproj,
-	ia_css_process_sdis2_vertproj,
-	ia_css_process_wb,
-	ia_css_process_nr,
-	ia_css_process_yee,
-	ia_css_process_ynr,
-	ia_css_process_fc,
-	ia_css_process_ctc,
-	ia_css_process_xnr_table,
-	ia_css_process_xnr,
-	ia_css_process_xnr3,
-};
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_dp_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dp_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_dp_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dp_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_dp_config() leave\n");
-	ia_css_dp_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_dp_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dp_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_dp_config() enter:\n");
-	ia_css_dp_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dp_config = *config;
-	params->config_changed[IA_CSS_DP_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_DP_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_dp_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_wb_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_wb_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_wb_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->wb_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_wb_config() leave\n");
-	ia_css_wb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_wb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_wb_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_wb_config() enter:\n");
-	ia_css_wb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->wb_config = *config;
-	params->config_changed[IA_CSS_WB_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_WB_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_wb_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_tnr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_tnr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_tnr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->tnr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_tnr_config() leave\n");
-	ia_css_tnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_tnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_tnr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_tnr_config() enter:\n");
-	ia_css_tnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->tnr_config = *config;
-	params->config_changed[IA_CSS_TNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_TNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_tnr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ob_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ob_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ob_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ob_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ob_config() leave\n");
-	ia_css_ob_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ob_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ob_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ob_config() enter:\n");
-	ia_css_ob_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ob_config = *config;
-	params->config_changed[IA_CSS_OB_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_OB_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ob_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_de_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_de_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_de_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->de_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_de_config() leave\n");
-	ia_css_de_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_de_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_de_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_de_config() enter:\n");
-	ia_css_de_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->de_config = *config;
-	params->config_changed[IA_CSS_DE_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_DE_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_de_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_anr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_anr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->anr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr_config() leave\n");
-	ia_css_anr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_anr_config() enter:\n");
-	ia_css_anr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->anr_config = *config;
-	params->config_changed[IA_CSS_ANR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_ANR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_anr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_anr2_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_anr_thres *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr2_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->anr_thres;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr2_config() leave\n");
-	ia_css_anr2_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr2_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_thres *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_anr2_config() enter:\n");
-	ia_css_anr2_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->anr_thres = *config;
-	params->config_changed[IA_CSS_ANR2_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_ANR2_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_anr2_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ce_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ce_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ce_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ce_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ce_config() leave\n");
-	ia_css_ce_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ce_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ce_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ce_config() enter:\n");
-	ia_css_ce_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ce_config = *config;
-	params->config_changed[IA_CSS_CE_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CE_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ce_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ecd_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ecd_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ecd_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ecd_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ecd_config() leave\n");
-	ia_css_ecd_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ecd_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ecd_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ecd_config() enter:\n");
-	ia_css_ecd_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ecd_config = *config;
-	params->config_changed[IA_CSS_ECD_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_ECD_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ecd_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ynr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ynr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ynr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ynr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ynr_config() leave\n");
-	ia_css_ynr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ynr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ynr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ynr_config() enter:\n");
-	ia_css_ynr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ynr_config = *config;
-	params->config_changed[IA_CSS_YNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_YNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ynr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_fc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_fc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_fc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->fc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_fc_config() leave\n");
-	ia_css_fc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_fc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_fc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_fc_config() enter:\n");
-	ia_css_fc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->fc_config = *config;
-	params->config_changed[IA_CSS_FC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_FC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_fc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_cnr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cnr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_cnr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->cnr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_cnr_config() leave\n");
-	ia_css_cnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_cnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cnr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_cnr_config() enter:\n");
-	ia_css_cnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->cnr_config = *config;
-	params->config_changed[IA_CSS_CNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_cnr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_macc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_macc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_macc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->macc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_macc_config() leave\n");
-	ia_css_macc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_macc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_macc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_macc_config() enter:\n");
-	ia_css_macc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->macc_config = *config;
-	params->config_changed[IA_CSS_MACC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_MACC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_macc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ctc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ctc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ctc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ctc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ctc_config() leave\n");
-	ia_css_ctc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ctc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ctc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ctc_config() enter:\n");
-	ia_css_ctc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ctc_config = *config;
-	params->config_changed[IA_CSS_CTC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CTC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ctc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_aa_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_aa_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_aa_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->aa_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_aa_config() leave\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_aa_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_aa_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_aa_config() enter:\n");
-	params->aa_config = *config;
-	params->config_changed[IA_CSS_AA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_AA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_aa_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_yuv2rgb_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_yuv2rgb_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->yuv2rgb_cc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_yuv2rgb_config() leave\n");
-	ia_css_yuv2rgb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_yuv2rgb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_yuv2rgb_config() enter:\n");
-	ia_css_yuv2rgb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->yuv2rgb_cc_config = *config;
-	params->config_changed[IA_CSS_YUV2RGB_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_YUV2RGB_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_yuv2rgb_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_rgb2yuv_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_rgb2yuv_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->rgb2yuv_cc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_rgb2yuv_config() leave\n");
-	ia_css_rgb2yuv_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_rgb2yuv_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_rgb2yuv_config() enter:\n");
-	ia_css_rgb2yuv_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->rgb2yuv_cc_config = *config;
-	params->config_changed[IA_CSS_RGB2YUV_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_RGB2YUV_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_rgb2yuv_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_csc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_csc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->cc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_csc_config() leave\n");
-	ia_css_csc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_csc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_csc_config() enter:\n");
-	ia_css_csc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->cc_config = *config;
-	params->config_changed[IA_CSS_CSC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CSC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_csc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_nr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_nr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_nr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->nr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_nr_config() leave\n");
-	ia_css_nr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_nr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_nr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_nr_config() enter:\n");
-	ia_css_nr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->nr_config = *config;
-	params->config_changed[IA_CSS_BNR_ID] = true;
-	params->config_changed[IA_CSS_NR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_NR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_nr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_gc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_gc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_gc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->gc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_gc_config() leave\n");
-	ia_css_gc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_gc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_gc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_gc_config() enter:\n");
-	ia_css_gc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->gc_config = *config;
-	params->config_changed[IA_CSS_GC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_GC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_gc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_horicoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horicoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horicoef_config() leave\n");
-	ia_css_sdis_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_horicoef_config() enter:\n");
-	ia_css_sdis_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_horicoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_vertcoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertcoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertcoef_config() leave\n");
-	ia_css_sdis_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_vertcoef_config() enter:\n");
-	ia_css_sdis_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_vertcoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_horiproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horiproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horiproj_config() leave\n");
-	ia_css_sdis_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_horiproj_config() enter:\n");
-	ia_css_sdis_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_horiproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_vertproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertproj_config() leave\n");
-	ia_css_sdis_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_vertproj_config() enter:\n");
-	ia_css_sdis_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_vertproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_horicoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horicoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horicoef_config() leave\n");
-	ia_css_sdis2_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_horicoef_config() enter:\n");
-	ia_css_sdis2_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_horicoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_vertcoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertcoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertcoef_config() leave\n");
-	ia_css_sdis2_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_vertcoef_config() enter:\n");
-	ia_css_sdis2_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_vertcoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_horiproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horiproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horiproj_config() leave\n");
-	ia_css_sdis2_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_horiproj_config() enter:\n");
-	ia_css_sdis2_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_horiproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_vertproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertproj_config() leave\n");
-	ia_css_sdis2_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_vertproj_config() enter:\n");
-	ia_css_sdis2_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_vertproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_r_gamma_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_rgb_gamma_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_r_gamma_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->r_gamma_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_r_gamma_config() leave\n");
-	ia_css_r_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_r_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_r_gamma_config() enter:\n");
-	ia_css_r_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->r_gamma_table = *config;
-	params->config_changed[IA_CSS_R_GAMMA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_R_GAMMA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_r_gamma_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_g_gamma_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_rgb_gamma_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_g_gamma_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->g_gamma_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_g_gamma_config() leave\n");
-	ia_css_g_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_g_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_g_gamma_config() enter:\n");
-	ia_css_g_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->g_gamma_table = *config;
-	params->config_changed[IA_CSS_G_GAMMA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_G_GAMMA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_g_gamma_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_b_gamma_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_rgb_gamma_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_b_gamma_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->b_gamma_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_b_gamma_config() leave\n");
-	ia_css_b_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_b_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_b_gamma_config() enter:\n");
-	ia_css_b_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->b_gamma_table = *config;
-	params->config_changed[IA_CSS_B_GAMMA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_B_GAMMA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_b_gamma_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_xnr_table_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_xnr_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_table_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->xnr_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_table_config() leave\n");
-	ia_css_xnr_table_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_table_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_xnr_table_config() enter:\n");
-	ia_css_xnr_table_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->xnr_table = *config;
-	params->config_changed[IA_CSS_XNR_TABLE_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_XNR_TABLE_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_xnr_table_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_formats_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_formats_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_formats_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->formats_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_formats_config() leave\n");
-	ia_css_formats_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_formats_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_formats_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_formats_config() enter:\n");
-	ia_css_formats_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->formats_config = *config;
-	params->config_changed[IA_CSS_FORMATS_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_FORMATS_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_formats_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_xnr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_xnr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->xnr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_config() leave\n");
-	ia_css_xnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_xnr_config() enter:\n");
-	ia_css_xnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->xnr_config = *config;
-	params->config_changed[IA_CSS_XNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_XNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_xnr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_xnr3_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_xnr3_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr3_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->xnr3_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr3_config() leave\n");
-	ia_css_xnr3_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr3_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr3_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_xnr3_config() enter:\n");
-	ia_css_xnr3_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->xnr3_config = *config;
-	params->config_changed[IA_CSS_XNR3_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_XNR3_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_xnr3_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_s3a_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_3a_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_s3a_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->s3a_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_s3a_config() leave\n");
-	ia_css_s3a_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_s3a_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_3a_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_s3a_config() enter:\n");
-	ia_css_s3a_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->s3a_config = *config;
-	params->config_changed[IA_CSS_BH_ID] = true;
-	params->config_changed[IA_CSS_S3A_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_S3A_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_s3a_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_output_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_output_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_output_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->output_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_output_config() leave\n");
-	ia_css_output_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_output_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_output_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_output_config() enter:\n");
-	ia_css_output_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->output_config = *config;
-	params->config_changed[IA_CSS_OUTPUT_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_OUTPUT_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_output_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_get_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-{
-	ia_css_get_dp_config(params, config->dp_config);
-	ia_css_get_wb_config(params, config->wb_config);
-	ia_css_get_tnr_config(params, config->tnr_config);
-	ia_css_get_ob_config(params, config->ob_config);
-	ia_css_get_de_config(params, config->de_config);
-	ia_css_get_anr_config(params, config->anr_config);
-	ia_css_get_anr2_config(params, config->anr_thres);
-	ia_css_get_ce_config(params, config->ce_config);
-	ia_css_get_ecd_config(params, config->ecd_config);
-	ia_css_get_ynr_config(params, config->ynr_config);
-	ia_css_get_fc_config(params, config->fc_config);
-	ia_css_get_cnr_config(params, config->cnr_config);
-	ia_css_get_macc_config(params, config->macc_config);
-	ia_css_get_ctc_config(params, config->ctc_config);
-	ia_css_get_aa_config(params, config->aa_config);
-	ia_css_get_yuv2rgb_config(params, config->yuv2rgb_cc_config);
-	ia_css_get_rgb2yuv_config(params, config->rgb2yuv_cc_config);
-	ia_css_get_csc_config(params, config->cc_config);
-	ia_css_get_nr_config(params, config->nr_config);
-	ia_css_get_gc_config(params, config->gc_config);
-	ia_css_get_sdis_horicoef_config(params, config->dvs_coefs);
-	ia_css_get_sdis_vertcoef_config(params, config->dvs_coefs);
-	ia_css_get_sdis_horiproj_config(params, config->dvs_coefs);
-	ia_css_get_sdis_vertproj_config(params, config->dvs_coefs);
-	ia_css_get_sdis2_horicoef_config(params, config->dvs2_coefs);
-	ia_css_get_sdis2_vertcoef_config(params, config->dvs2_coefs);
-	ia_css_get_sdis2_horiproj_config(params, config->dvs2_coefs);
-	ia_css_get_sdis2_vertproj_config(params, config->dvs2_coefs);
-	ia_css_get_r_gamma_config(params, config->r_gamma_table);
-	ia_css_get_g_gamma_config(params, config->g_gamma_table);
-	ia_css_get_b_gamma_config(params, config->b_gamma_table);
-	ia_css_get_xnr_table_config(params, config->xnr_table);
-	ia_css_get_formats_config(params, config->formats_config);
-	ia_css_get_xnr_config(params, config->xnr_config);
-	ia_css_get_xnr3_config(params, config->xnr3_config);
-	ia_css_get_s3a_config(params, config->s3a_config);
-	ia_css_get_output_config(params, config->output_config);
-}
-
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_set_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-{
-	ia_css_set_dp_config(params, config->dp_config);
-	ia_css_set_wb_config(params, config->wb_config);
-	ia_css_set_tnr_config(params, config->tnr_config);
-	ia_css_set_ob_config(params, config->ob_config);
-	ia_css_set_de_config(params, config->de_config);
-	ia_css_set_anr_config(params, config->anr_config);
-	ia_css_set_anr2_config(params, config->anr_thres);
-	ia_css_set_ce_config(params, config->ce_config);
-	ia_css_set_ecd_config(params, config->ecd_config);
-	ia_css_set_ynr_config(params, config->ynr_config);
-	ia_css_set_fc_config(params, config->fc_config);
-	ia_css_set_cnr_config(params, config->cnr_config);
-	ia_css_set_macc_config(params, config->macc_config);
-	ia_css_set_ctc_config(params, config->ctc_config);
-	ia_css_set_aa_config(params, config->aa_config);
-	ia_css_set_yuv2rgb_config(params, config->yuv2rgb_cc_config);
-	ia_css_set_rgb2yuv_config(params, config->rgb2yuv_cc_config);
-	ia_css_set_csc_config(params, config->cc_config);
-	ia_css_set_nr_config(params, config->nr_config);
-	ia_css_set_gc_config(params, config->gc_config);
-	ia_css_set_sdis_horicoef_config(params, config->dvs_coefs);
-	ia_css_set_sdis_vertcoef_config(params, config->dvs_coefs);
-	ia_css_set_sdis_horiproj_config(params, config->dvs_coefs);
-	ia_css_set_sdis_vertproj_config(params, config->dvs_coefs);
-	ia_css_set_sdis2_horicoef_config(params, config->dvs2_coefs);
-	ia_css_set_sdis2_vertcoef_config(params, config->dvs2_coefs);
-	ia_css_set_sdis2_horiproj_config(params, config->dvs2_coefs);
-	ia_css_set_sdis2_vertproj_config(params, config->dvs2_coefs);
-	ia_css_set_r_gamma_config(params, config->r_gamma_table);
-	ia_css_set_g_gamma_config(params, config->g_gamma_table);
-	ia_css_set_b_gamma_config(params, config->b_gamma_table);
-	ia_css_set_xnr_table_config(params, config->xnr_table);
-	ia_css_set_formats_config(params, config->formats_config);
-	ia_css_set_xnr_config(params, config->xnr_config);
-	ia_css_set_xnr3_config(params, config->xnr3_config);
-	ia_css_set_s3a_config(params, config->s3a_config);
-	ia_css_set_output_config(params, config->output_config);
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_params.h
deleted file mode 100644
index 5b3deb7f74ae..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_params.h
+++ /dev/null
@@ -1,399 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Generated code: do not edit or commmit. */
-
-#ifndef _IA_CSS_ISP_PARAM_H
-#define _IA_CSS_ISP_PARAM_H
-
-/* Code generated by genparam/gencode.c:gen_param_enum() */
-
-enum ia_css_parameter_ids {
-	IA_CSS_AA_ID,
-	IA_CSS_ANR_ID,
-	IA_CSS_ANR2_ID,
-	IA_CSS_BH_ID,
-	IA_CSS_CNR_ID,
-	IA_CSS_CROP_ID,
-	IA_CSS_CSC_ID,
-	IA_CSS_DP_ID,
-	IA_CSS_BNR_ID,
-	IA_CSS_DE_ID,
-	IA_CSS_ECD_ID,
-	IA_CSS_FORMATS_ID,
-	IA_CSS_FPN_ID,
-	IA_CSS_GC_ID,
-	IA_CSS_CE_ID,
-	IA_CSS_YUV2RGB_ID,
-	IA_CSS_RGB2YUV_ID,
-	IA_CSS_R_GAMMA_ID,
-	IA_CSS_G_GAMMA_ID,
-	IA_CSS_B_GAMMA_ID,
-	IA_CSS_UDS_ID,
-	IA_CSS_RAA_ID,
-	IA_CSS_S3A_ID,
-	IA_CSS_OB_ID,
-	IA_CSS_OUTPUT_ID,
-	IA_CSS_SC_ID,
-	IA_CSS_BDS_ID,
-	IA_CSS_TNR_ID,
-	IA_CSS_MACC_ID,
-	IA_CSS_SDIS_HORICOEF_ID,
-	IA_CSS_SDIS_VERTCOEF_ID,
-	IA_CSS_SDIS_HORIPROJ_ID,
-	IA_CSS_SDIS_VERTPROJ_ID,
-	IA_CSS_SDIS2_HORICOEF_ID,
-	IA_CSS_SDIS2_VERTCOEF_ID,
-	IA_CSS_SDIS2_HORIPROJ_ID,
-	IA_CSS_SDIS2_VERTPROJ_ID,
-	IA_CSS_WB_ID,
-	IA_CSS_NR_ID,
-	IA_CSS_YEE_ID,
-	IA_CSS_YNR_ID,
-	IA_CSS_FC_ID,
-	IA_CSS_CTC_ID,
-	IA_CSS_XNR_TABLE_ID,
-	IA_CSS_XNR_ID,
-	IA_CSS_XNR3_ID,
-	IA_CSS_NUM_PARAMETER_IDS
-};
-
-/* Code generated by genparam/gencode.c:gen_param_offsets() */
-
-struct ia_css_memory_offsets {
-	struct {
-		struct ia_css_isp_parameter aa;
-		struct ia_css_isp_parameter anr;
-		struct ia_css_isp_parameter bh;
-		struct ia_css_isp_parameter cnr;
-		struct ia_css_isp_parameter crop;
-		struct ia_css_isp_parameter csc;
-		struct ia_css_isp_parameter dp;
-		struct ia_css_isp_parameter bnr;
-		struct ia_css_isp_parameter de;
-		struct ia_css_isp_parameter ecd;
-		struct ia_css_isp_parameter formats;
-		struct ia_css_isp_parameter fpn;
-		struct ia_css_isp_parameter gc;
-		struct ia_css_isp_parameter ce;
-		struct ia_css_isp_parameter yuv2rgb;
-		struct ia_css_isp_parameter rgb2yuv;
-		struct ia_css_isp_parameter uds;
-		struct ia_css_isp_parameter raa;
-		struct ia_css_isp_parameter s3a;
-		struct ia_css_isp_parameter ob;
-		struct ia_css_isp_parameter output;
-		struct ia_css_isp_parameter sc;
-		struct ia_css_isp_parameter bds;
-		struct ia_css_isp_parameter tnr;
-		struct ia_css_isp_parameter macc;
-		struct ia_css_isp_parameter sdis_horiproj;
-		struct ia_css_isp_parameter sdis_vertproj;
-		struct ia_css_isp_parameter sdis2_horiproj;
-		struct ia_css_isp_parameter sdis2_vertproj;
-		struct ia_css_isp_parameter wb;
-		struct ia_css_isp_parameter nr;
-		struct ia_css_isp_parameter yee;
-		struct ia_css_isp_parameter ynr;
-		struct ia_css_isp_parameter fc;
-		struct ia_css_isp_parameter ctc;
-		struct ia_css_isp_parameter xnr;
-		struct ia_css_isp_parameter xnr3;
-		struct ia_css_isp_parameter get;
-		struct ia_css_isp_parameter put;
-	} dmem;
-	struct {
-		struct ia_css_isp_parameter anr2;
-		struct ia_css_isp_parameter ob;
-		struct ia_css_isp_parameter sdis_horicoef;
-		struct ia_css_isp_parameter sdis_vertcoef;
-		struct ia_css_isp_parameter sdis2_horicoef;
-		struct ia_css_isp_parameter sdis2_vertcoef;
-#ifdef ISP2401
-		struct ia_css_isp_parameter xnr3;
-#endif
-	} vmem;
-	struct {
-		struct ia_css_isp_parameter bh;
-	} hmem0;
-	struct {
-		struct ia_css_isp_parameter gc;
-		struct ia_css_isp_parameter g_gamma;
-		struct ia_css_isp_parameter xnr_table;
-	} vamem1;
-	struct {
-		struct ia_css_isp_parameter r_gamma;
-		struct ia_css_isp_parameter ctc;
-	} vamem0;
-	struct {
-		struct ia_css_isp_parameter b_gamma;
-	} vamem2;
-};
-
-#if defined(IA_CSS_INCLUDE_PARAMETERS)
-
-#include "ia_css_stream.h"   /* struct ia_css_stream */
-#include "ia_css_binary.h"   /* struct ia_css_binary */
-/* Code generated by genparam/gencode.c:gen_param_process_table() */
-
-struct ia_css_pipeline_stage; /* forward declaration */
-
-extern void (* ia_css_kernel_process_param[IA_CSS_NUM_PARAMETER_IDS])(
-			unsigned pipe_id,
-			const struct ia_css_pipeline_stage *stage,
-			struct ia_css_isp_parameters *params);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_dp_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dp_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_wb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_wb_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_tnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_tnr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ob_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ob_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_de_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_de_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr2_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_thres *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ce_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ce_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ecd_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ecd_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ynr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ynr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_fc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_fc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_cnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cnr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_macc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_macc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ctc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ctc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_aa_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_aa_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_yuv2rgb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_rgb2yuv_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_csc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_nr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_nr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_gc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_gc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_r_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_g_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_b_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_table_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_formats_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_formats_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr3_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr3_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_s3a_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_3a_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_output_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_output_config *config);
-
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_get_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-;
-#ifdef ISP2401
-
-#endif
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_set_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-;
-#ifdef ISP2401
-
-#endif
-#endif /* IA_CSS_INCLUDE_PARAMETER */
-
-#endif /* _IA_CSS_ISP_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_states.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_states.c
deleted file mode 100644
index e87d05bc73ae..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_states.c
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Generated code: do not edit or commmit. */
-
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_states.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_aa_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_aa_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.aa.size;
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.aa.offset;
-
-		if (size)
-			memset(&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset], 0, size);
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_aa_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_cnr_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.cnr.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.cnr.offset;
-
-		if (size) {
-			ia_css_init_cnr_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_cnr2_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr2_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.cnr2.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.cnr2.offset;
-
-		if (size) {
-			ia_css_init_cnr2_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr2_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_dp_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_dp_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.dp.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.dp.offset;
-
-		if (size) {
-			ia_css_init_dp_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_dp_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_de_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_de_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.de.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.de.offset;
-
-		if (size) {
-			ia_css_init_de_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_de_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_tnr_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_tnr_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->dmem.tnr.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->dmem.tnr.offset;
-
-		if (size) {
-			ia_css_init_tnr_state((struct sh_css_isp_tnr_dmem_state *)
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_DMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_tnr_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_ref_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ref_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->dmem.ref.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->dmem.ref.offset;
-
-		if (size) {
-			ia_css_init_ref_state((struct sh_css_isp_ref_dmem_state *)
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_DMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ref_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_ynr_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ynr_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.ynr.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.ynr.offset;
-
-		if (size) {
-			ia_css_init_ynr_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ynr_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_state_init_table() */
-
-void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(const struct ia_css_binary *binary) = {
-	ia_css_initialize_aa_state,
-	ia_css_initialize_cnr_state,
-	ia_css_initialize_cnr2_state,
-	ia_css_initialize_dp_state,
-	ia_css_initialize_de_state,
-	ia_css_initialize_tnr_state,
-	ia_css_initialize_ref_state,
-	ia_css_initialize_ynr_state,
-};
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_states.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_states.h
deleted file mode 100644
index 732adafb0a63..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hive_isp_css_2401_system_csi2p_generated/ia_css_isp_states.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#define IA_CSS_INCLUDE_STATES
-#include "isp/kernels/aa/aa_2/ia_css_aa2.host.h"
-#include "isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h"
-#include "isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h"
-#include "isp/kernels/de/de_1.0/ia_css_de.host.h"
-#include "isp/kernels/dp/dp_1.0/ia_css_dp.host.h"
-#include "isp/kernels/ref/ref_1.0/ia_css_ref.host.h"
-#include "isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h"
-#include "isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h"
-#include "isp/kernels/dpc2/ia_css_dpc2.host.h"
-#include "isp/kernels/eed1_8/ia_css_eed1_8.host.h"
-/* Generated code: do not edit or commmit. */
-
-#ifndef _IA_CSS_ISP_STATE_H
-#define _IA_CSS_ISP_STATE_H
-
-/* Code generated by genparam/gencode.c:gen_param_enum() */
-
-enum ia_css_state_ids {
-	IA_CSS_AA_STATE_ID,
-	IA_CSS_CNR_STATE_ID,
-	IA_CSS_CNR2_STATE_ID,
-	IA_CSS_DP_STATE_ID,
-	IA_CSS_DE_STATE_ID,
-	IA_CSS_TNR_STATE_ID,
-	IA_CSS_REF_STATE_ID,
-	IA_CSS_YNR_STATE_ID,
-	IA_CSS_NUM_STATE_IDS
-};
-
-/* Code generated by genparam/gencode.c:gen_param_offsets() */
-
-struct ia_css_state_memory_offsets {
-	struct {
-		struct ia_css_isp_parameter aa;
-		struct ia_css_isp_parameter cnr;
-		struct ia_css_isp_parameter cnr2;
-		struct ia_css_isp_parameter dp;
-		struct ia_css_isp_parameter de;
-		struct ia_css_isp_parameter ynr;
-	} vmem;
-	struct {
-		struct ia_css_isp_parameter tnr;
-		struct ia_css_isp_parameter ref;
-	} dmem;
-};
-
-#if defined(IA_CSS_INCLUDE_STATES)
-
-#include "ia_css_stream.h"   /* struct ia_css_stream */
-#include "ia_css_binary.h"   /* struct ia_css_binary */
-/* Code generated by genparam/genstate.c:gen_state_init_table() */
-
-extern void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(const struct ia_css_binary *binary);
-
-#endif /* IA_CSS_INCLUDE_STATE */
-
-#endif /* _IA_CSS_ISP_STATE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx.c
deleted file mode 100644
index 505e2b600beb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-
-#include "system_global.h"
-
-const uint32_t N_SHORT_PACKET_LUT_ENTRIES[N_CSI_RX_BACKEND_ID] = {
-	4,	/* 4 entries at CSI_RX_BACKEND0_ID*/
-	4,	/* 4 entries at CSI_RX_BACKEND1_ID*/
-	4	/* 4 entries at CSI_RX_BACKEND2_ID*/
-};
-
-const uint32_t N_LONG_PACKET_LUT_ENTRIES[N_CSI_RX_BACKEND_ID] = {
-	8,	/* 8 entries at CSI_RX_BACKEND0_ID*/
-	4,	/* 4 entries at CSI_RX_BACKEND1_ID*/
-	4	/* 4 entries at CSI_RX_BACKEND2_ID*/
-};
-
-const uint32_t N_CSI_RX_FE_CTRL_DLANES[N_CSI_RX_FRONTEND_ID] = {
-	N_CSI_RX_DLANE_ID,	/* 4 dlanes for CSI_RX_FR0NTEND0_ID */
-	N_CSI_RX_DLANE_ID,	/* 4 dlanes for CSI_RX_FR0NTEND1_ID */
-	N_CSI_RX_DLANE_ID	/* 4 dlanes for CSI_RX_FR0NTEND2_ID */
-};
-
-/* sid_width for CSI_RX_BACKEND<N>_ID */
-const uint32_t N_CSI_RX_BE_SID_WIDTH[N_CSI_RX_BACKEND_ID] = {
-	3,
-	2,
-	2
-};
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx_local.h
deleted file mode 100644
index a2e9d54a4a37..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx_local.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __CSI_RX_LOCAL_H_INCLUDED__
-#define __CSI_RX_LOCAL_H_INCLUDED__
-
-#include "csi_rx_global.h"
-#define N_CSI_RX_BE_MIPI_COMP_FMT_REG 		4
-#define N_CSI_RX_BE_MIPI_CUSTOM_PEC		12
-#define N_CSI_RX_BE_SHORT_PKT_LUT		4
-#define N_CSI_RX_BE_LONG_PKT_LUT		8
-typedef struct csi_rx_fe_ctrl_state_s		csi_rx_fe_ctrl_state_t;
-typedef struct csi_rx_fe_ctrl_lane_s		csi_rx_fe_ctrl_lane_t;
-typedef struct csi_rx_be_ctrl_state_s		csi_rx_be_ctrl_state_t;
-/*mipi_backend_custom_mode_pixel_extraction_config*/
-typedef struct csi_rx_be_ctrl_pec_s		csi_rx_be_ctrl_pec_t;
-
-
-struct csi_rx_fe_ctrl_lane_s {
-	hrt_data	termen;
-	hrt_data	settle;
-};
-struct csi_rx_fe_ctrl_state_s {
-	hrt_data 		enable;
-	hrt_data 		nof_enable_lanes;
-	hrt_data 		error_handling;
-	hrt_data 		status;
-	hrt_data 		status_dlane_hs;
-	hrt_data 		status_dlane_lp;
-	csi_rx_fe_ctrl_lane_t 	clane;
-	csi_rx_fe_ctrl_lane_t 	dlane[N_CSI_RX_DLANE_ID];
-};
-struct csi_rx_be_ctrl_state_s {
-	hrt_data 		enable;
-	hrt_data 		status;
-	hrt_data 		comp_format_reg[N_CSI_RX_BE_MIPI_COMP_FMT_REG];
-	hrt_data 		raw16;
-	hrt_data 		raw18;
-	hrt_data 		force_raw8;
-	hrt_data 		irq_status;
-	hrt_data 		custom_mode_enable;
-	hrt_data 		custom_mode_data_state;
-	hrt_data 		pec[N_CSI_RX_BE_MIPI_CUSTOM_PEC];
-	hrt_data 		custom_mode_valid_eop_config;
-	hrt_data 		global_lut_disregard_reg;
-	hrt_data 		packet_status_stall;
-	hrt_data 		short_packet_lut_entry[N_CSI_RX_BE_SHORT_PKT_LUT];
-	hrt_data 		long_packet_lut_entry[N_CSI_RX_BE_LONG_PKT_LUT];
-};
-#endif /* __CSI_RX_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx_private.h
deleted file mode 100644
index 9c0cb4a63862..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/csi_rx_private.h
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __CSI_RX_PRIVATE_H_INCLUDED__
-#define __CSI_RX_PRIVATE_H_INCLUDED__
-
-#include "rx_csi_defs.h"
-#include "mipi_backend_defs.h"
-#include "csi_rx_public.h"
-
-#include "device_access.h"	/* ia_css_device_load_uint32 */
-
-#include "assert_support.h" /* assert */
-#include "print_support.h" /* print */
-
-
-/*****************************************************
- *
- * Native command interface (NCI).
- *
- *****************************************************/
-/**
- * @brief Get the csi rx fe state.
- * Refer to "csi_rx_public.h" for details.
- */
-static inline void csi_rx_fe_ctrl_get_state(
-		const csi_rx_frontend_ID_t ID,
-		csi_rx_fe_ctrl_state_t *state)
-{
-	uint32_t i;
-
-	state->enable =
-		csi_rx_fe_ctrl_reg_load(ID, _HRT_CSI_RX_ENABLE_REG_IDX);
-	state->nof_enable_lanes =
-		csi_rx_fe_ctrl_reg_load(ID, _HRT_CSI_RX_NOF_ENABLED_LANES_REG_IDX);
-	state->error_handling =
-		csi_rx_fe_ctrl_reg_load(ID, _HRT_CSI_RX_ERROR_HANDLING_REG_IDX);
-	state->status =
-		csi_rx_fe_ctrl_reg_load(ID, _HRT_CSI_RX_STATUS_REG_IDX);
-	state->status_dlane_hs =
-		csi_rx_fe_ctrl_reg_load(ID, _HRT_CSI_RX_STATUS_DLANE_HS_REG_IDX);
-	state->status_dlane_lp =
-		csi_rx_fe_ctrl_reg_load(ID, _HRT_CSI_RX_STATUS_DLANE_LP_REG_IDX);
-	state->clane.termen =
-		csi_rx_fe_ctrl_reg_load(ID, _HRT_CSI_RX_DLY_CNT_TERMEN_CLANE_REG_IDX);
-	state->clane.settle =
-		csi_rx_fe_ctrl_reg_load(ID, _HRT_CSI_RX_DLY_CNT_SETTLE_CLANE_REG_IDX);
-
-	/*
-	 * Get the values of the register-set per
-	 * dlane.
-	 */
-	for (i = 0; i < N_CSI_RX_FE_CTRL_DLANES[ID]; i++) {
-		csi_rx_fe_ctrl_get_dlane_state(
-				ID,
-				i,
-				&(state->dlane[i]));
-	}
-}
-
-/**
- * @brief Get the state of the csi rx fe dlane process.
- * Refer to "csi_rx_public.h" for details.
- */
-static inline void csi_rx_fe_ctrl_get_dlane_state(
-		const csi_rx_frontend_ID_t ID,
-		const uint32_t lane,
-		csi_rx_fe_ctrl_lane_t *dlane_state)
-{
-
-	dlane_state->termen =
-		csi_rx_fe_ctrl_reg_load(ID, _HRT_CSI_RX_DLY_CNT_TERMEN_DLANE_REG_IDX(lane));
-	dlane_state->settle =
-		csi_rx_fe_ctrl_reg_load(ID, _HRT_CSI_RX_DLY_CNT_SETTLE_DLANE_REG_IDX(lane));
-
-}
-/**
- * @brief dump the csi rx fe state.
- * Refer to "csi_rx_public.h" for details.
- */
-static inline void csi_rx_fe_ctrl_dump_state(
-		const csi_rx_frontend_ID_t ID,
-		csi_rx_fe_ctrl_state_t *state)
-{
-	uint32_t i;
-
-	ia_css_print("CSI RX FE STATE Controller %d Enable state 0x%x \n", ID, state->enable);
-	ia_css_print("CSI RX FE STATE Controller %d No Of enable lanes 0x%x \n", ID, state->nof_enable_lanes);
-	ia_css_print("CSI RX FE STATE Controller %d Error handling 0x%x \n", ID, state->error_handling);
-	ia_css_print("CSI RX FE STATE Controller %d Status 0x%x \n", ID, state->status);
-	ia_css_print("CSI RX FE STATE Controller %d Status Dlane HS 0x%x \n", ID, state->status_dlane_hs);
-	ia_css_print("CSI RX FE STATE Controller %d Status Dlane LP 0x%x \n", ID, state->status_dlane_lp);
-	ia_css_print("CSI RX FE STATE Controller %d Status term enable LP 0x%x \n", ID, state->clane.termen);
-	ia_css_print("CSI RX FE STATE Controller %d Status term settle LP 0x%x \n", ID, state->clane.settle);
-
-	/*
-	 * Get the values of the register-set per
-	 * dlane.
-	 */
-	for (i = 0; i < N_CSI_RX_FE_CTRL_DLANES[ID]; i++) {
-		ia_css_print("CSI RX FE STATE Controller %d DLANE ID %d termen 0x%x \n", ID, i, state->dlane[i].termen);
-		ia_css_print("CSI RX FE STATE Controller %d DLANE ID %d settle 0x%x \n", ID, i, state->dlane[i].settle);
-	}
-}
-
-/**
- * @brief Get the csi rx be state.
- * Refer to "csi_rx_public.h" for details.
- */
-static inline void csi_rx_be_ctrl_get_state(
-		const csi_rx_backend_ID_t ID,
-		csi_rx_be_ctrl_state_t *state)
-{
-	uint32_t i;
-
-	state->enable =
-		csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_ENABLE_REG_IDX);
-
-	state->status =
-		csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_STATUS_REG_IDX);
-
-	for(i = 0; i <N_CSI_RX_BE_MIPI_COMP_FMT_REG ; i++) {
-		state->comp_format_reg[i] =
-			csi_rx_be_ctrl_reg_load(ID, 
-						_HRT_MIPI_BACKEND_COMP_FORMAT_REG0_IDX+i);
-	}
-
-	state->raw16 =
-		csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_RAW16_CONFIG_REG_IDX);
-
-	state->raw18 =
-		csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_RAW18_CONFIG_REG_IDX);
-	state->force_raw8 =
-		csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_FORCE_RAW8_REG_IDX);
-	state->irq_status =
-		csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_IRQ_STATUS_REG_IDX);
-#if 0 /* device access error for these registers */
-	/* ToDo: rootcause this failure */
-	state->custom_mode_enable =
-		csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_CUST_EN_REG_IDX);
-
-	state->custom_mode_data_state =
-		csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_CUST_DATA_STATE_REG_IDX);
-	for(i = 0; i <N_CSI_RX_BE_MIPI_CUSTOM_PEC ; i++) {
-		state->pec[i] = 
-			csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_CUST_PIX_EXT_S0P0_REG_IDX + i);
-	}
-	state->custom_mode_valid_eop_config =
-		csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_CUST_PIX_VALID_EOP_REG_IDX);
-#endif
-	state->global_lut_disregard_reg =
-		csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_GLOBAL_LUT_DISREGARD_REG_IDX);
-	state->packet_status_stall =
-		csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_PKT_STALL_STATUS_REG_IDX);
-	/*
-	 * Get the values of the register-set per
-	 * lut.
-	 */
-	for (i = 0; i < N_SHORT_PACKET_LUT_ENTRIES[ID]; i++) {
-		state->short_packet_lut_entry[i] =
-			csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_SP_LUT_ENTRY_0_REG_IDX + i);
-	}
-	for (i = 0; i < N_LONG_PACKET_LUT_ENTRIES[ID]; i++) {
-		state->long_packet_lut_entry[i] =
-			csi_rx_be_ctrl_reg_load(ID, _HRT_MIPI_BACKEND_LP_LUT_ENTRY_0_REG_IDX + i);
-	}
-}
-
-/**
- * @brief Dump the csi rx be state.
- * Refer to "csi_rx_public.h" for details.
- */
-static inline void csi_rx_be_ctrl_dump_state(
-		const csi_rx_backend_ID_t ID,
-		csi_rx_be_ctrl_state_t *state)
-{
-	uint32_t i;
-
-	ia_css_print("CSI RX BE STATE Controller %d Enable 0x%x \n", ID, state->enable);
-	ia_css_print("CSI RX BE STATE Controller %d Status 0x%x \n", ID, state->status);
-
-	for(i = 0; i <N_CSI_RX_BE_MIPI_COMP_FMT_REG ; i++) {
-		ia_css_print("CSI RX BE STATE Controller %d comp format reg vc%d value 0x%x \n", ID, i, state->status);
-	}
-	ia_css_print("CSI RX BE STATE Controller %d RAW16 0x%x \n", ID, state->raw16);
-	ia_css_print("CSI RX BE STATE Controller %d RAW18 0x%x \n", ID, state->raw18);
-	ia_css_print("CSI RX BE STATE Controller %d Force RAW8 0x%x \n", ID, state->force_raw8);
-	ia_css_print("CSI RX BE STATE Controller %d IRQ state 0x%x \n", ID, state->irq_status);
-#if 0   /* ToDo:Getting device access error for this register */
-	for(i = 0; i <N_CSI_RX_BE_MIPI_CUSTOM_PEC ; i++) {
-		ia_css_print("CSI RX BE STATE Controller %d PEC ID %d custom pec 0x%x \n", ID, i, state->pec[i]);
-	}
-#endif
-	ia_css_print("CSI RX BE STATE Controller %d Global LUT diregard reg 0x%x \n", ID, state->global_lut_disregard_reg);
-	ia_css_print("CSI RX BE STATE Controller %d packet stall reg 0x%x \n", ID, state->packet_status_stall);
-	/*
-	 * Get the values of the register-set per
-	 * lut.
-	 */
-	for (i = 0; i < N_SHORT_PACKET_LUT_ENTRIES[ID]; i++) {
-		ia_css_print("CSI RX BE STATE Controller ID %d Short packat entry %d shart packet lut id 0x%x \n", ID, i, state->short_packet_lut_entry[i]);
-	}
-	for (i = 0; i < N_LONG_PACKET_LUT_ENTRIES[ID]; i++) {
-		ia_css_print("CSI RX BE STATE Controller ID %d Long packat entry %d Long packet lut id 0x%x \n", ID, i, state->long_packet_lut_entry[i]);
-	}
-}
-/* end of NCI */
-/*****************************************************
- *
- * Device level interface (DLI).
- *
- *****************************************************/
-/**
- * @brief Load the register value.
- * Refer to "csi_rx_public.h" for details.
- */
-static inline hrt_data csi_rx_fe_ctrl_reg_load(
-	const csi_rx_frontend_ID_t ID,
-	const hrt_address reg)
-{
-	assert(ID < N_CSI_RX_FRONTEND_ID);
-	assert(CSI_RX_FE_CTRL_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(CSI_RX_FE_CTRL_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-
-/**
- * @brief Store a value to the register.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-static inline void csi_rx_fe_ctrl_reg_store(
-	const csi_rx_frontend_ID_t ID,
-	const hrt_address reg,
-	const hrt_data value)
-{
-	assert(ID < N_CSI_RX_FRONTEND_ID);
-	assert(CSI_RX_FE_CTRL_BASE[ID] != (hrt_address)-1);
-
-	ia_css_device_store_uint32(CSI_RX_FE_CTRL_BASE[ID] + reg*sizeof(hrt_data), value);
-}
-/**
- * @brief Load the register value.
- * Refer to "csi_rx_public.h" for details.
- */
-static inline hrt_data csi_rx_be_ctrl_reg_load(
-	const csi_rx_backend_ID_t ID,
-	const hrt_address reg)
-{
-	assert(ID < N_CSI_RX_BACKEND_ID);
-	assert(CSI_RX_BE_CTRL_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(CSI_RX_BE_CTRL_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-
-/**
- * @brief Store a value to the register.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-static inline void csi_rx_be_ctrl_reg_store(
-	const csi_rx_backend_ID_t ID,
-	const hrt_address reg,
-	const hrt_data value)
-{
-	assert(ID < N_CSI_RX_BACKEND_ID);
-	assert(CSI_RX_BE_CTRL_BASE[ID] != (hrt_address)-1);
-
-	ia_css_device_store_uint32(CSI_RX_BE_CTRL_BASE[ID] + reg*sizeof(hrt_data), value);
-}
-/* end of DLI */
-
-#endif /* __CSI_RX_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl.c
deleted file mode 100644
index 14973d1c2756..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl.c
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <type_support.h>
-#include "system_global.h"
-
-const uint32_t N_IBUF_CTRL_PROCS[N_IBUF_CTRL_ID] = {
-	8,	/* IBUF_CTRL0_ID supports at most 8 processes */
-	4,	/* IBUF_CTRL1_ID supports at most 4 processes */
-	4	/* IBUF_CTRL2_ID supports at most 4 processes */
-};
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl_local.h
deleted file mode 100644
index ea40284623d1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl_local.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IBUF_CTRL_LOCAL_H_INCLUDED__
-#define __IBUF_CTRL_LOCAL_H_INCLUDED__
-
-#include "ibuf_ctrl_global.h"
-
-typedef struct ibuf_ctrl_proc_state_s	ibuf_ctrl_proc_state_t;
-typedef struct ibuf_ctrl_state_s		ibuf_ctrl_state_t;
-
-struct ibuf_ctrl_proc_state_s {
-	hrt_data num_items;
-	hrt_data num_stores;
-	hrt_data dma_channel;
-	hrt_data dma_command;
-	hrt_data ibuf_st_addr;
-	hrt_data ibuf_stride;
-	hrt_data ibuf_end_addr;
-	hrt_data dest_st_addr;
-	hrt_data dest_stride;
-	hrt_data dest_end_addr;
-	hrt_data sync_frame;
-	hrt_data sync_command;
-	hrt_data store_command;
-	hrt_data shift_returned_items;
-	hrt_data elems_ibuf;
-	hrt_data elems_dest;
-	hrt_data cur_stores;
-	hrt_data cur_acks;
-	hrt_data cur_s2m_ibuf_addr;
-	hrt_data cur_dma_ibuf_addr;
-	hrt_data cur_dma_dest_addr;
-	hrt_data cur_isp_dest_addr;
-	hrt_data dma_cmds_send;
-	hrt_data main_cntrl_state;
-	hrt_data dma_sync_state;
-	hrt_data isp_sync_state;
-};
-
-struct ibuf_ctrl_state_s {
-	hrt_data	recalc_words;
-	hrt_data	arbiters;
-	ibuf_ctrl_proc_state_t	proc_state[N_STREAM2MMIO_SID_ID];
-};
-
-#endif /* __IBUF_CTRL_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl_private.h
deleted file mode 100644
index 4d07c2fe1469..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/ibuf_ctrl_private.h
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IBUF_CTRL_PRIVATE_H_INCLUDED__
-#define __IBUF_CTRL_PRIVATE_H_INCLUDED__
-
-#include "ibuf_ctrl_public.h"
-
-#include "device_access.h"	/* ia_css_device_load_uint32 */
-
-#include "assert_support.h" /* assert */
-#include "print_support.h" /* print */
-
-
-/*****************************************************
- *
- * Native command interface (NCI).
- *
- *****************************************************/
-/**
- * @brief Get the ibuf-controller state.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-STORAGE_CLASS_IBUF_CTRL_C void ibuf_ctrl_get_state(
-		const ibuf_ctrl_ID_t ID,
-		ibuf_ctrl_state_t *state)
-{
-	uint32_t i;
-
-	state->recalc_words =
-		ibuf_ctrl_reg_load(ID, _IBUF_CNTRL_RECALC_WORDS_STATUS);
-	state->arbiters =
-		ibuf_ctrl_reg_load(ID, _IBUF_CNTRL_ARBITERS_STATUS);
-
-	/*
-	 * Get the values of the register-set per
-	 * ibuf-controller process.
-	 */
-	for (i = 0; i < N_IBUF_CTRL_PROCS[ID]; i++) {
-		ibuf_ctrl_get_proc_state(
-				ID,
-				i,
-				&(state->proc_state[i]));
-	}
-}
-
-/**
- * @brief Get the state of the ibuf-controller process.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-STORAGE_CLASS_IBUF_CTRL_C void ibuf_ctrl_get_proc_state(
-		const ibuf_ctrl_ID_t ID,
-		const uint32_t proc_id,
-		ibuf_ctrl_proc_state_t	*state)
-{
-	hrt_address reg_bank_offset;
-
-	reg_bank_offset =
-		_IBUF_CNTRL_PROC_REG_ALIGN * (1 + proc_id);
-
-	state->num_items =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_NUM_ITEMS_PER_STORE);
-
-	state->num_stores =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_NUM_STORES_PER_FRAME);
-
-	state->dma_channel =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DMA_CHANNEL);
-
-	state->dma_command =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DMA_CMD);
-
-	state->ibuf_st_addr =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_BUFFER_START_ADDRESS);
-
-	state->ibuf_stride =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_BUFFER_STRIDE);
-
-	state->ibuf_end_addr =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_BUFFER_END_ADDRESS);
-
-	state->dest_st_addr =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DEST_START_ADDRESS);
-
-	state->dest_stride =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DEST_STRIDE);
-
-	state->dest_end_addr =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DEST_END_ADDRESS);
-
-	state->sync_frame =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_SYNC_FRAME);
-
-	state->sync_command =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_STR2MMIO_SYNC_CMD);
-
-	state->store_command =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_STR2MMIO_STORE_CMD);
-
-	state->shift_returned_items =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_SHIFT_ITEMS);
-
-	state->elems_ibuf =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_ELEMS_P_WORD_IBUF);
-
-	state->elems_dest =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_ELEMS_P_WORD_DEST);
-
-	state->cur_stores =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_STORES);
-
-	state->cur_acks =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_ACKS);
-
-	state->cur_s2m_ibuf_addr =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_S2M_IBUF_ADDR);
-
-	state->cur_dma_ibuf_addr =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_DMA_IBUF_ADDR);
-
-	state->cur_dma_dest_addr =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_DMA_DEST_ADDR);
-
-	state->cur_isp_dest_addr =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_ISP_DEST_ADDR);
-
-	state->dma_cmds_send =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_NR_DMA_CMDS_SEND);
-
-	state->main_cntrl_state =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_MAIN_CNTRL_STATE);
-
-	state->dma_sync_state =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DMA_SYNC_STATE);
-
-	state->isp_sync_state =
-		ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_ISP_SYNC_STATE);
-}
-/**
- * @brief Dump the ibuf-controller state.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-STORAGE_CLASS_IBUF_CTRL_C void ibuf_ctrl_dump_state(
-		const ibuf_ctrl_ID_t ID,
-		ibuf_ctrl_state_t *state)
-{
-	uint32_t i;
-	ia_css_print("IBUF controller ID %d recalculate words 0x%x\n", ID, state->recalc_words);
-	ia_css_print("IBUF controller ID %d arbiters 0x%x\n", ID, state->arbiters);
-
-	/*
-	 * Dump the values of the register-set per
-	 * ibuf-controller process.
-	 */
-	for (i = 0; i < N_IBUF_CTRL_PROCS[ID]; i++) {
-		ia_css_print("IBUF controller ID %d Process ID %d num_items 0x%x\n", ID, i, state->proc_state[i].num_items);
-		ia_css_print("IBUF controller ID %d Process ID %d num_stores 0x%x\n", ID, i, state->proc_state[i].num_stores);
-		ia_css_print("IBUF controller ID %d Process ID %d dma_channel 0x%x\n", ID, i, state->proc_state[i].dma_channel);
-		ia_css_print("IBUF controller ID %d Process ID %d dma_command 0x%x\n", ID, i, state->proc_state[i].dma_command);
-		ia_css_print("IBUF controller ID %d Process ID %d ibuf_st_addr 0x%x\n", ID, i, state->proc_state[i].ibuf_st_addr);
-		ia_css_print("IBUF controller ID %d Process ID %d ibuf_stride 0x%x\n", ID, i, state->proc_state[i].ibuf_stride);
-		ia_css_print("IBUF controller ID %d Process ID %d ibuf_end_addr 0x%x\n", ID, i, state->proc_state[i].ibuf_end_addr);
-		ia_css_print("IBUF controller ID %d Process ID %d dest_st_addr 0x%x\n", ID, i, state->proc_state[i].dest_st_addr);
-		ia_css_print("IBUF controller ID %d Process ID %d dest_stride 0x%x\n", ID, i, state->proc_state[i].dest_stride);
-		ia_css_print("IBUF controller ID %d Process ID %d dest_end_addr 0x%x\n", ID, i, state->proc_state[i].dest_end_addr);
-		ia_css_print("IBUF controller ID %d Process ID %d sync_frame 0x%x\n", ID, i, state->proc_state[i].sync_frame);
-		ia_css_print("IBUF controller ID %d Process ID %d sync_command 0x%x\n", ID, i, state->proc_state[i].sync_command);
-		ia_css_print("IBUF controller ID %d Process ID %d store_command 0x%x\n", ID, i, state->proc_state[i].store_command);
-		ia_css_print("IBUF controller ID %d Process ID %d shift_returned_items 0x%x\n", ID, i, state->proc_state[i].shift_returned_items);
-		ia_css_print("IBUF controller ID %d Process ID %d elems_ibuf 0x%x\n", ID, i, state->proc_state[i].elems_ibuf);
-		ia_css_print("IBUF controller ID %d Process ID %d elems_dest 0x%x\n", ID, i, state->proc_state[i].elems_dest);
-		ia_css_print("IBUF controller ID %d Process ID %d cur_stores 0x%x\n", ID, i, state->proc_state[i].cur_stores);
-		ia_css_print("IBUF controller ID %d Process ID %d cur_acks 0x%x\n", ID, i, state->proc_state[i].cur_acks);
-		ia_css_print("IBUF controller ID %d Process ID %d cur_s2m_ibuf_addr 0x%x\n", ID, i, state->proc_state[i].cur_s2m_ibuf_addr);
-		ia_css_print("IBUF controller ID %d Process ID %d cur_dma_ibuf_addr 0x%x\n", ID, i, state->proc_state[i].cur_dma_ibuf_addr);
-		ia_css_print("IBUF controller ID %d Process ID %d cur_dma_dest_addr 0x%x\n", ID, i, state->proc_state[i].cur_dma_dest_addr);
-		ia_css_print("IBUF controller ID %d Process ID %d cur_isp_dest_addr 0x%x\n", ID, i, state->proc_state[i].cur_isp_dest_addr);
-		ia_css_print("IBUF controller ID %d Process ID %d dma_cmds_send 0x%x\n", ID, i, state->proc_state[i].dma_cmds_send);
-		ia_css_print("IBUF controller ID %d Process ID %d main_cntrl_state 0x%x\n", ID, i, state->proc_state[i].main_cntrl_state);
-		ia_css_print("IBUF controller ID %d Process ID %d dma_sync_state 0x%x\n", ID, i, state->proc_state[i].dma_sync_state);
-		ia_css_print("IBUF controller ID %d Process ID %d isp_sync_state 0x%x\n", ID, i, state->proc_state[i].isp_sync_state);
-	}
-}
-/* end of NCI */
-
-/*****************************************************
- *
- * Device level interface (DLI).
- *
- *****************************************************/
-/**
- * @brief Load the register value.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-STORAGE_CLASS_IBUF_CTRL_C hrt_data ibuf_ctrl_reg_load(
-	const ibuf_ctrl_ID_t ID,
-	const hrt_address reg)
-{
-	assert(ID < N_IBUF_CTRL_ID);
-	assert(IBUF_CTRL_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(IBUF_CTRL_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-
-/**
- * @brief Store a value to the register.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-STORAGE_CLASS_IBUF_CTRL_C void ibuf_ctrl_reg_store(
-	const ibuf_ctrl_ID_t ID,
-	const hrt_address reg,
-	const hrt_data value)
-{
-	assert(ID < N_IBUF_CTRL_ID);
-	assert(IBUF_CTRL_BASE[ID] != (hrt_address)-1);
-
-	ia_css_device_store_uint32(IBUF_CTRL_BASE[ID] + reg*sizeof(hrt_data), value);
-}
-/* end of DLI */
-
-
-#endif /* __IBUF_CTRL_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/input_system_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/input_system_local.h
deleted file mode 100644
index f199423e28da..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/input_system_local.h
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_SYSTEM_LOCAL_H_INCLUDED__
-#define __INPUT_SYSTEM_LOCAL_H_INCLUDED__
-
-#include "type_support.h"
-#include "input_system_global.h"
-
-#include "ibuf_ctrl.h"
-#include "csi_rx.h"
-#include "pixelgen.h"
-#include "isys_stream2mmio.h"
-#include "isys_irq.h"
-
-typedef input_system_err_t input_system_error_t;
-
-typedef enum {
-	MIPI_FORMAT_SHORT1 = 0x08,
-	MIPI_FORMAT_SHORT2,
-	MIPI_FORMAT_SHORT3,
-	MIPI_FORMAT_SHORT4,
-	MIPI_FORMAT_SHORT5,
-	MIPI_FORMAT_SHORT6,
-	MIPI_FORMAT_SHORT7,
-	MIPI_FORMAT_SHORT8,
-	MIPI_FORMAT_EMBEDDED = 0x12,
-	MIPI_FORMAT_YUV420_8 = 0x18,
-	MIPI_FORMAT_YUV420_10,
-	MIPI_FORMAT_YUV420_8_LEGACY,
-	MIPI_FORMAT_YUV420_8_SHIFT = 0x1C,
-	MIPI_FORMAT_YUV420_10_SHIFT,
-	MIPI_FORMAT_YUV422_8 = 0x1E,
-	MIPI_FORMAT_YUV422_10,
-	MIPI_FORMAT_RGB444 = 0x20,
-	MIPI_FORMAT_RGB555,
-	MIPI_FORMAT_RGB565,
-	MIPI_FORMAT_RGB666,
-	MIPI_FORMAT_RGB888,
-	MIPI_FORMAT_RAW6 = 0x28,
-	MIPI_FORMAT_RAW7,
-	MIPI_FORMAT_RAW8,
-	MIPI_FORMAT_RAW10,
-	MIPI_FORMAT_RAW12,
-	MIPI_FORMAT_RAW14,
-	MIPI_FORMAT_CUSTOM0 = 0x30,
-	MIPI_FORMAT_CUSTOM1,
-	MIPI_FORMAT_CUSTOM2,
-	MIPI_FORMAT_CUSTOM3,
-	MIPI_FORMAT_CUSTOM4,
-	MIPI_FORMAT_CUSTOM5,
-	MIPI_FORMAT_CUSTOM6,
-	MIPI_FORMAT_CUSTOM7,
-	//MIPI_FORMAT_RAW16, /*not supported by 2401*/
-	//MIPI_FORMAT_RAW18,
-	N_MIPI_FORMAT
-} mipi_format_t;
-
-#define N_MIPI_FORMAT_CUSTOM	8
-
-/* The number of stores for compressed format types */
-#define	N_MIPI_COMPRESSOR_CONTEXT	(N_RX_CHANNEL_ID * N_MIPI_FORMAT_CUSTOM)
-#define UNCOMPRESSED_BITS_PER_PIXEL_10	10
-#define UNCOMPRESSED_BITS_PER_PIXEL_12	12
-#define COMPRESSED_BITS_PER_PIXEL_6	6
-#define COMPRESSED_BITS_PER_PIXEL_7	7
-#define COMPRESSED_BITS_PER_PIXEL_8	8
-enum mipi_compressor {
-	MIPI_COMPRESSOR_NONE = 0,
-	MIPI_COMPRESSOR_10_6_10,
-	MIPI_COMPRESSOR_10_7_10,
-	MIPI_COMPRESSOR_10_8_10,
-	MIPI_COMPRESSOR_12_6_12,
-	MIPI_COMPRESSOR_12_7_12,
-	MIPI_COMPRESSOR_12_8_12,
-	N_MIPI_COMPRESSOR_METHODS
-};
-
-typedef enum {
-	MIPI_PREDICTOR_NONE = 0,
-	MIPI_PREDICTOR_TYPE1,
-	MIPI_PREDICTOR_TYPE2,
-	N_MIPI_PREDICTOR_TYPES
-} mipi_predictor_t;
-
-typedef struct input_system_state_s	input_system_state_t;
-struct input_system_state_s {
-	ibuf_ctrl_state_t	ibuf_ctrl_state[N_IBUF_CTRL_ID];
-	csi_rx_fe_ctrl_state_t	csi_rx_fe_ctrl_state[N_CSI_RX_FRONTEND_ID];
-	csi_rx_be_ctrl_state_t	csi_rx_be_ctrl_state[N_CSI_RX_BACKEND_ID];
-	pixelgen_ctrl_state_t	pixelgen_ctrl_state[N_PIXELGEN_ID];
-	stream2mmio_state_t	stream2mmio_state[N_STREAM2MMIO_ID];
-	isys_irqc_state_t	isys_irqc_state[N_ISYS_IRQ_ID];
-};
-#endif /* __INPUT_SYSTEM_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/input_system_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/input_system_private.h
deleted file mode 100644
index 97505e436047..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/input_system_private.h
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_SYSTEM_PRIVATE_H_INCLUDED__
-#define __INPUT_SYSTEM_PRIVATE_H_INCLUDED__
-
-#include "input_system_public.h"
-
-STORAGE_CLASS_INPUT_SYSTEM_C input_system_err_t input_system_get_state(
-	const input_system_ID_t	ID,
-	input_system_state_t *state)
-{
-	uint32_t i;
-
-	(void)(ID);
-
-	/*  get the states of all CSI RX frontend devices */
-	for (i = 0; i < N_CSI_RX_FRONTEND_ID; i++) {
-		csi_rx_fe_ctrl_get_state(
-				(csi_rx_frontend_ID_t)i,
-				&(state->csi_rx_fe_ctrl_state[i]));
-	}
-
-	/*  get the states of all CIS RX backend devices */
-	for (i = 0; i < N_CSI_RX_BACKEND_ID; i++) {
-		csi_rx_be_ctrl_get_state(
-				(csi_rx_backend_ID_t)i,
-				&(state->csi_rx_be_ctrl_state[i]));
-	}
-
-	/* get the states of all pixelgen devices */
-	for (i = 0; i < N_PIXELGEN_ID; i++) {
-		pixelgen_ctrl_get_state(
-				(pixelgen_ID_t)i,
-				&(state->pixelgen_ctrl_state[i]));
-	}
-
-	/* get the states of all stream2mmio devices */
-	for (i = 0; i < N_STREAM2MMIO_ID; i++) {
-		stream2mmio_get_state(
-				(stream2mmio_ID_t)i,
-				&(state->stream2mmio_state[i]));
-	}
-
-	/* get the states of all ibuf-controller devices */
-	for (i = 0; i < N_IBUF_CTRL_ID; i++) {
-		ibuf_ctrl_get_state(
-				(ibuf_ctrl_ID_t)i,
-				&(state->ibuf_ctrl_state[i]));
-	}
-
-	/* get the states of all isys irq controllers */
-	for (i = 0; i < N_ISYS_IRQ_ID; i++) {
-		isys_irqc_state_get((isys_irq_ID_t)i, &(state->isys_irqc_state[i]));
-	}
-
-	/* TODO: get the states of all ISYS2401 DMA devices  */
-	for (i = 0; i < N_ISYS2401_DMA_ID; i++) {
-	}
-
-	return INPUT_SYSTEM_ERR_NO_ERROR;
-}
-STORAGE_CLASS_INPUT_SYSTEM_C void input_system_dump_state(
-	const input_system_ID_t	ID,
-	input_system_state_t *state)
-{
-	uint32_t i;
-
-	(void)(ID);
-
-	/*  dump the states of all CSI RX frontend devices */
-	for (i = 0; i < N_CSI_RX_FRONTEND_ID; i++) {
-		csi_rx_fe_ctrl_dump_state(
-				(csi_rx_frontend_ID_t)i,
-				&(state->csi_rx_fe_ctrl_state[i]));
-	}
-
-	/*  dump the states of all CIS RX backend devices */
-	for (i = 0; i < N_CSI_RX_BACKEND_ID; i++) {
-		csi_rx_be_ctrl_dump_state(
-				(csi_rx_backend_ID_t)i,
-				&(state->csi_rx_be_ctrl_state[i]));
-	}
-
-	/* dump the states of all pixelgen devices */
-	for (i = 0; i < N_PIXELGEN_ID; i++) {
-		pixelgen_ctrl_dump_state(
-				(pixelgen_ID_t)i,
-				&(state->pixelgen_ctrl_state[i]));
-	}
-
-	/* dump the states of all st2mmio devices */
-	for (i = 0; i < N_STREAM2MMIO_ID; i++) {
-		stream2mmio_dump_state(
-				(stream2mmio_ID_t)i,
-				&(state->stream2mmio_state[i]));
-	}
-
-	/* dump the states of all ibuf-controller devices */
-	for (i = 0; i < N_IBUF_CTRL_ID; i++) {
-		ibuf_ctrl_dump_state(
-				(ibuf_ctrl_ID_t)i,
-				&(state->ibuf_ctrl_state[i]));
-	}
-
-	/* dump the states of all isys irq controllers */
-	for (i = 0; i < N_ISYS_IRQ_ID; i++) {
-		isys_irqc_state_dump((isys_irq_ID_t)i, &(state->isys_irqc_state[i]));
-	}
-
-	/* TODO: dump the states of all ISYS2401 DMA devices  */
-	for (i = 0; i < N_ISYS2401_DMA_ID; i++) {
-	}
-
-	return;
-}
-#endif /* __INPUT_SYSTEM_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma.c
deleted file mode 100644
index 77767228985e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma.c
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "isys_dma.h"
-#include "assert_support.h"
-
-#ifndef __INLINE_ISYS2401_DMA__
-/*
- * Include definitions for isys dma register access functions. isys_dma.h
- * includes declarations of these functions by including isys_dma_public.h.
- */
-#include "isys_dma_private.h"
-#endif
-
-const isys2401_dma_channel N_ISYS2401_DMA_CHANNEL_PROCS[N_ISYS2401_DMA_ID] = {
-	N_ISYS2401_DMA_CHANNEL
-};
-
-void isys2401_dma_set_max_burst_size(
-	const isys2401_dma_ID_t	dma_id,
-	uint32_t		max_burst_size)
-{
-	assert(dma_id < N_ISYS2401_DMA_ID);
-	assert((max_burst_size > 0x00) && (max_burst_size <= 0xFF));
-
-	isys2401_dma_reg_store(dma_id,
-		DMA_DEV_INFO_REG_IDX(_DMA_V2_DEV_INTERF_MAX_BURST_IDX, HIVE_DMA_BUS_DDR_CONN),
-		(max_burst_size - 1));
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma_local.h
deleted file mode 100644
index 5c694a26386e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma_local.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_DMA_LOCAL_H_INCLUDED__
-#define __ISYS_DMA_LOCAL_H_INCLUDED__
-
-#include "isys_dma_global.h"
-
-#endif /* __ISYS_DMA_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma_private.h
deleted file mode 100644
index 2cd1aeecf617..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_dma_private.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_DMA_PRIVATE_H_INCLUDED__
-#define __ISYS_DMA_PRIVATE_H_INCLUDED__
-
-#include "isys_dma_public.h"
-#include "device_access.h"
-#include "assert_support.h"
-#include "dma.h"
-#include "dma_v2_defs.h"
-#include "print_support.h"
-
-
-STORAGE_CLASS_ISYS2401_DMA_C void isys2401_dma_reg_store(
-	const isys2401_dma_ID_t	dma_id,
-	const unsigned int	reg,
-	const hrt_data		value)
-{
-	unsigned int reg_loc;
-
-	assert(dma_id < N_ISYS2401_DMA_ID);
-	assert(ISYS2401_DMA_BASE[dma_id] != (hrt_address)-1);
-
-	reg_loc = ISYS2401_DMA_BASE[dma_id] + (reg * sizeof(hrt_data));
-
-	ia_css_print("isys dma store at addr(0x%x) val(%u)\n", reg_loc, (unsigned int)value);
-	ia_css_device_store_uint32(reg_loc, value);
-}
-
-STORAGE_CLASS_ISYS2401_DMA_C hrt_data isys2401_dma_reg_load(
-	const isys2401_dma_ID_t	dma_id,
-	const unsigned int	reg)
-{
-	unsigned int reg_loc;
-	hrt_data value;
-
-	assert(dma_id < N_ISYS2401_DMA_ID);
-	assert(ISYS2401_DMA_BASE[dma_id] != (hrt_address)-1);
-
-	reg_loc = ISYS2401_DMA_BASE[dma_id] + (reg * sizeof(hrt_data));
-
-	value = ia_css_device_load_uint32(reg_loc);
-	ia_css_print("isys dma load from addr(0x%x) val(%u)\n", reg_loc, (unsigned int)value);
-
-	return value;
-}
-
-#endif /* __ISYS_DMA_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq.c
deleted file mode 100644
index 842ae340ae13..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <system_local.h>
-#include "device_access.h"
-#include "assert_support.h"
-#include "ia_css_debug.h"
-#include "isys_irq.h"
-
-#ifndef __INLINE_ISYS2401_IRQ__
-/*
- * Include definitions for isys irq private functions. isys_irq.h includes
- * declarations of these functions by including isys_irq_public.h.
- */
-#include "isys_irq_private.h"
-#endif
-
-/* Public interface */
-STORAGE_CLASS_ISYS2401_IRQ_C void isys_irqc_status_enable(
-	const isys_irq_ID_t	isys_irqc_id)
-{
-	assert(isys_irqc_id < N_ISYS_IRQ_ID);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "Setting irq mask for port %u\n", isys_irqc_id);
-	isys_irqc_reg_store(isys_irqc_id, ISYS_IRQ_MASK_REG_IDX, ISYS_IRQ_MASK_REG_VALUE);
-	isys_irqc_reg_store(isys_irqc_id, ISYS_IRQ_CLEAR_REG_IDX, ISYS_IRQ_CLEAR_REG_VALUE);
-	isys_irqc_reg_store(isys_irqc_id, ISYS_IRQ_ENABLE_REG_IDX, ISYS_IRQ_ENABLE_REG_VALUE);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq_local.h
deleted file mode 100644
index 0bffb5680e25..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq_local.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_IRQ_LOCAL_H__
-#define __ISYS_IRQ_LOCAL_H__
-
-#include <type_support.h>
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-
-typedef struct isys_irqc_state_s isys_irqc_state_t;
-
-struct isys_irqc_state_s {
-	hrt_data edge;
-	hrt_data mask;
-	hrt_data status;
-	hrt_data enable;
-	hrt_data level_no;
-/*hrt_data clear;	*/	/* write-only register */
-};
-
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2401) */
-
-#endif	/* __ISYS_IRQ_LOCAL_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq_private.h
deleted file mode 100644
index e69f39893bd2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_irq_private.h
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_IRQ_PRIVATE_H__
-#define __ISYS_IRQ_PRIVATE_H__
-
-#include "isys_irq_global.h"
-#include "isys_irq_local.h"
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-
-/* -------------------------------------------------------+
- |             Native command interface (NCI)             |
- + -------------------------------------------------------*/
-
-/**
-* @brief Get the isys irq status.
-* Refer to "isys_irq.h" for details.
-*/
-STORAGE_CLASS_ISYS2401_IRQ_C void isys_irqc_state_get(
-	const isys_irq_ID_t	isys_irqc_id,
-	isys_irqc_state_t *state)
-{
-	state->edge     = isys_irqc_reg_load(isys_irqc_id, ISYS_IRQ_EDGE_REG_IDX);
-	state->mask     = isys_irqc_reg_load(isys_irqc_id, ISYS_IRQ_MASK_REG_IDX);
-	state->status   = isys_irqc_reg_load(isys_irqc_id, ISYS_IRQ_STATUS_REG_IDX);
-	state->enable   = isys_irqc_reg_load(isys_irqc_id, ISYS_IRQ_ENABLE_REG_IDX);
-	state->level_no = isys_irqc_reg_load(isys_irqc_id, ISYS_IRQ_LEVEL_NO_REG_IDX);
-	/*
-	** Invalid to read/load from write-only register 'clear'
-	** state->clear = isys_irqc_reg_load(isys_irqc_id, ISYS_IRQ_CLEAR_REG_IDX);
-	*/
-}
-
-/**
-* @brief Dump the isys irq status.
-* Refer to "isys_irq.h" for details.
-*/
-STORAGE_CLASS_ISYS2401_IRQ_C void isys_irqc_state_dump(
-	const isys_irq_ID_t	isys_irqc_id,
-	const isys_irqc_state_t *state)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"isys irq controller id %d"
-		"\n\tstatus:0x%x\n\tedge:0x%x\n\tmask:0x%x"
-		"\n\tenable:0x%x\n\tlevel_not_pulse:0x%x\n",
-		isys_irqc_id,
-		state->status, state->edge, state->mask, state->enable, state->level_no);
-}
-
-/* end of NCI */
-
-/* -------------------------------------------------------+
- |              Device level interface (DLI)              |
- + -------------------------------------------------------*/
-
-/* Support functions */
-STORAGE_CLASS_ISYS2401_IRQ_C void isys_irqc_reg_store(
-	const isys_irq_ID_t	isys_irqc_id,
-	const unsigned int	reg_idx,
-	const hrt_data	value)
-{
-	unsigned int reg_addr;
-
-	assert(isys_irqc_id < N_ISYS_IRQ_ID);
-	assert(reg_idx <= ISYS_IRQ_LEVEL_NO_REG_IDX);
-
-	reg_addr = ISYS_IRQ_BASE[isys_irqc_id] + (reg_idx * sizeof(hrt_data));
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"isys irq store at addr(0x%x) val(%u)\n", reg_addr, (unsigned int)value);
-
-	ia_css_device_store_uint32(reg_addr, value);
-}
-
-STORAGE_CLASS_ISYS2401_IRQ_C hrt_data isys_irqc_reg_load(
-	const isys_irq_ID_t	isys_irqc_id,
-	const unsigned int	reg_idx)
-{
-	unsigned int reg_addr;
-	hrt_data value;
-
-	assert(isys_irqc_id < N_ISYS_IRQ_ID);
-	assert(reg_idx <= ISYS_IRQ_LEVEL_NO_REG_IDX);
-
-	reg_addr = ISYS_IRQ_BASE[isys_irqc_id] + (reg_idx * sizeof(hrt_data));
-	value = ia_css_device_load_uint32(reg_addr);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"isys irq load from addr(0x%x) val(%u)\n", reg_addr, (unsigned int)value);
-
-	return value;
-}
-
-/* end of DLI */
-
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2401) */
-
-#endif	/* __ISYS_IRQ_PRIVATE_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio.c
deleted file mode 100644
index 67570138ba24..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio.c
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "isys_stream2mmio.h"
-
-const stream2mmio_sid_ID_t N_STREAM2MMIO_SID_PROCS[N_STREAM2MMIO_ID] = {
-	N_STREAM2MMIO_SID_ID,
-	STREAM2MMIO_SID4_ID,
-	STREAM2MMIO_SID4_ID
-};
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio_local.h
deleted file mode 100644
index 801523977e1d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio_local.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_STREAM2MMIO_LOCAL_H_INCLUDED__
-#define __ISYS_STREAM2MMIO_LOCAL_H_INCLUDED__
-
-#include "isys_stream2mmio_global.h"
-
-typedef struct stream2mmio_state_s		stream2mmio_state_t;
-typedef struct stream2mmio_sid_state_s	stream2mmio_sid_state_t;
-
-struct stream2mmio_sid_state_s {
-	hrt_data rcv_ack;
-	hrt_data pix_width_id;
-	hrt_data start_addr;
-	hrt_data end_addr;
-	hrt_data strides;
-	hrt_data num_items;
-	hrt_data block_when_no_cmd;
-};
-
-struct stream2mmio_state_s {
-	stream2mmio_sid_state_t 	sid_state[N_STREAM2MMIO_SID_ID];
-};
-#endif /* __ISYS_STREAM2MMIO_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio_private.h
deleted file mode 100644
index f946105ddf43..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/isys_stream2mmio_private.h
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_STREAM2MMIO_PRIVATE_H_INCLUDED__
-#define __ISYS_STREAM2MMIO_PRIVATE_H_INCLUDED__
-
-#include "isys_stream2mmio_public.h"
-#include "device_access.h"	/* ia_css_device_load_uint32 */
-#include "assert_support.h"	/* assert */
-#include "print_support.h"	/* print */
-
-#define STREAM2MMIO_COMMAND_REG_ID             0
-#define STREAM2MMIO_ACKNOWLEDGE_REG_ID         1
-#define STREAM2MMIO_PIX_WIDTH_ID_REG_ID        2
-#define STREAM2MMIO_START_ADDR_REG_ID          3      /* master port address,NOT Byte */
-#define STREAM2MMIO_END_ADDR_REG_ID            4      /* master port address,NOT Byte */
-#define STREAM2MMIO_STRIDE_REG_ID              5      /* stride in master port words, increment is per packet for long sids, stride is not used for short sid's*/
-#define STREAM2MMIO_NUM_ITEMS_REG_ID           6      /* number of packets for store packets cmd, number of words for store_words cmd */
-#define STREAM2MMIO_BLOCK_WHEN_NO_CMD_REG_ID   7      /* if this register is 1, input will be stalled if there is no pending command for this sid */
-#define STREAM2MMIO_REGS_PER_SID               8
-
-/*****************************************************
- *
- * Native command interface (NCI).
- *
- *****************************************************/
-/**
- * @brief Get the stream2mmio-controller state.
- * Refer to "stream2mmio_public.h" for details.
- */
-STORAGE_CLASS_STREAM2MMIO_C void stream2mmio_get_state(
-		const stream2mmio_ID_t ID,
-		stream2mmio_state_t *state)
-{
-	stream2mmio_sid_ID_t i;
-
-	/*
-	 * Get the values of the register-set per
-	 * stream2mmio-controller sids.
-	 */
-	for (i = STREAM2MMIO_SID0_ID; i < N_STREAM2MMIO_SID_PROCS[ID]; i++) {
-		stream2mmio_get_sid_state(ID, i, &(state->sid_state[i]));
-	}
-}
-
-/**
- * @brief Get the state of the stream2mmio-controller sidess.
- * Refer to "stream2mmio_public.h" for details.
- */
-STORAGE_CLASS_STREAM2MMIO_C void stream2mmio_get_sid_state(
-		const stream2mmio_ID_t ID,
-		const stream2mmio_sid_ID_t sid_id,
-		stream2mmio_sid_state_t	*state)
-{
-
-	state->rcv_ack =
-		stream2mmio_reg_load(ID, sid_id, STREAM2MMIO_ACKNOWLEDGE_REG_ID);
-
-	state->pix_width_id =
-		stream2mmio_reg_load(ID, sid_id, STREAM2MMIO_PIX_WIDTH_ID_REG_ID);
-
-	state->start_addr =
-		stream2mmio_reg_load(ID, sid_id, STREAM2MMIO_START_ADDR_REG_ID);
-
-	state->end_addr =
-		stream2mmio_reg_load(ID, sid_id, STREAM2MMIO_END_ADDR_REG_ID);
-
-	state->strides =
-		stream2mmio_reg_load(ID, sid_id, STREAM2MMIO_STRIDE_REG_ID);
-
-	state->num_items =
-		stream2mmio_reg_load(ID, sid_id, STREAM2MMIO_NUM_ITEMS_REG_ID);
-
-	state->block_when_no_cmd =
-		stream2mmio_reg_load(ID, sid_id, STREAM2MMIO_BLOCK_WHEN_NO_CMD_REG_ID);
-
-}
-
-/**
- * @brief Dump the state of the stream2mmio-controller sidess.
- * Refer to "stream2mmio_public.h" for details.
- */
-STORAGE_CLASS_STREAM2MMIO_C void stream2mmio_print_sid_state(
-		stream2mmio_sid_state_t	*state)
-{
-	ia_css_print("\t \t Receive acks 0x%x\n", state->rcv_ack);
-	ia_css_print("\t \t Pixel width 0x%x\n", state->pix_width_id);
-	ia_css_print("\t \t Startaddr 0x%x\n", state->start_addr);
-	ia_css_print("\t \t Endaddr 0x%x\n", state->end_addr);
-	ia_css_print("\t \t Strides 0x%x\n", state->strides);
-	ia_css_print("\t \t Num Items 0x%x\n", state->num_items);
-	ia_css_print("\t \t block when no cmd 0x%x\n", state->block_when_no_cmd);
-
-}
-/**
- * @brief Dump the ibuf-controller state.
- * Refer to "stream2mmio_public.h" for details.
- */
-STORAGE_CLASS_STREAM2MMIO_C void stream2mmio_dump_state(
-		const stream2mmio_ID_t ID,
-		stream2mmio_state_t *state)
-{
-	stream2mmio_sid_ID_t i;
-
-	/*
-	 * Get the values of the register-set per
-	 * stream2mmio-controller sids.
-	 */
-	for (i = STREAM2MMIO_SID0_ID; i < N_STREAM2MMIO_SID_PROCS[ID]; i++) {
-		ia_css_print("StREAM2MMIO ID %d SID %d\n", ID, i);
-		stream2mmio_print_sid_state(&(state->sid_state[i]));
-	}
-}
-/* end of NCI */
-
-/*****************************************************
- *
- * Device level interface (DLI).
- *
- *****************************************************/
-/**
- * @brief Load the register value.
- * Refer to "stream2mmio_public.h" for details.
- */
-STORAGE_CLASS_STREAM2MMIO_C hrt_data stream2mmio_reg_load(
-		const stream2mmio_ID_t ID,
-		const stream2mmio_sid_ID_t sid_id,
-		const uint32_t reg_idx)
-{
-	uint32_t reg_bank_offset;
-
-	assert(ID < N_STREAM2MMIO_ID);
-
-	reg_bank_offset = STREAM2MMIO_REGS_PER_SID * sid_id;
-	return ia_css_device_load_uint32(STREAM2MMIO_CTRL_BASE[ID] +
-			(reg_bank_offset + reg_idx) * sizeof(hrt_data));
-}
-
-
-/**
- * @brief Store a value to the register.
- * Refer to "stream2mmio_public.h" for details.
- */
-STORAGE_CLASS_STREAM2MMIO_C void stream2mmio_reg_store(
-		const stream2mmio_ID_t ID,
-		const hrt_address reg,
-		const hrt_data value)
-{
-	assert(ID < N_STREAM2MMIO_ID);
-	assert(STREAM2MMIO_CTRL_BASE[ID] != (hrt_address)-1);
-
-	ia_css_device_store_uint32(STREAM2MMIO_CTRL_BASE[ID] +
-		reg * sizeof(hrt_data), value);
-}
-/* end of DLI */
-
-#endif /* __ISYS_STREAM2MMIO_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/pixelgen_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/pixelgen_local.h
deleted file mode 100644
index 24f4da9aef40..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/pixelgen_local.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __PIXELGEN_LOCAL_H_INCLUDED__
-#define __PIXELGEN_LOCAL_H_INCLUDED__
-
-#include "pixelgen_global.h"
-
-typedef struct pixelgen_ctrl_state_s	pixelgen_ctrl_state_t;
-struct pixelgen_ctrl_state_s {
-	hrt_data	com_enable;
-	hrt_data	prbs_rstval0;
-	hrt_data	prbs_rstval1;
-	hrt_data	syng_sid;
-	hrt_data	syng_free_run;
-	hrt_data	syng_pause;
-	hrt_data	syng_nof_frames;
-	hrt_data	syng_nof_pixels;
-	hrt_data	syng_nof_line;
-	hrt_data	syng_hblank_cyc;
-	hrt_data	syng_vblank_cyc;
-	hrt_data	syng_stat_hcnt;
-	hrt_data	syng_stat_vcnt;
-	hrt_data	syng_stat_fcnt;
-	hrt_data	syng_stat_done;
-	hrt_data	tpg_mode;
-	hrt_data	tpg_hcnt_mask;
-	hrt_data	tpg_vcnt_mask;
-	hrt_data	tpg_xycnt_mask;
-	hrt_data	tpg_hcnt_delta;
-	hrt_data	tpg_vcnt_delta;
-	hrt_data	tpg_r1;
-	hrt_data	tpg_g1;
-	hrt_data	tpg_b1;
-	hrt_data	tpg_r2;
-	hrt_data	tpg_g2;
-	hrt_data	tpg_b2;
-};
-#endif /* __PIXELGEN_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/pixelgen_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/pixelgen_private.h
deleted file mode 100644
index c5bf540eadf1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/pixelgen_private.h
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __PIXELGEN_PRIVATE_H_INCLUDED__
-#define __PIXELGEN_PRIVATE_H_INCLUDED__
-#include "pixelgen_public.h"
-#include "hive_isp_css_host_ids_hrt.h"
-#include "PixelGen_SysBlock_defs.h"
-#include "device_access.h"	/* ia_css_device_load_uint32 */
-#include "assert_support.h" /* assert */
-
-
-/*****************************************************
- *
- * Native command interface (NCI).
- *
- *****************************************************/
-/**
- * @brief Get the pixelgen state.
- * Refer to "pixelgen_public.h" for details.
- */
-STORAGE_CLASS_PIXELGEN_C void pixelgen_ctrl_get_state(
-		const pixelgen_ID_t ID,
-		pixelgen_ctrl_state_t *state)
-{
-
-	state->com_enable =
-		pixelgen_ctrl_reg_load(ID, _PXG_COM_ENABLE_REG_IDX);
-	state->prbs_rstval0 =
-		pixelgen_ctrl_reg_load(ID, _PXG_PRBS_RSTVAL_REG0_IDX);
-	state->prbs_rstval1 =
-		pixelgen_ctrl_reg_load(ID, _PXG_PRBS_RSTVAL_REG1_IDX);
-	state->syng_sid =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_SID_REG_IDX);
-	state->syng_free_run =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_FREE_RUN_REG_IDX);
-	state->syng_pause =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_PAUSE_REG_IDX);
-	state->syng_nof_frames =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_NOF_FRAME_REG_IDX);
-	state->syng_nof_pixels =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_NOF_PIXEL_REG_IDX);
-	state->syng_nof_line =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_NOF_LINE_REG_IDX);
-	state->syng_hblank_cyc =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_HBLANK_CYC_REG_IDX);
-	state->syng_vblank_cyc =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_VBLANK_CYC_REG_IDX);
-	state->syng_stat_hcnt =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_STAT_HCNT_REG_IDX);
-	state->syng_stat_vcnt =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_STAT_VCNT_REG_IDX);
-	state->syng_stat_fcnt =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_STAT_FCNT_REG_IDX);
-	state->syng_stat_done =
-		pixelgen_ctrl_reg_load(ID, _PXG_SYNG_STAT_DONE_REG_IDX);
-	state->tpg_mode =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_MODE_REG_IDX);
-	state->tpg_hcnt_mask =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_HCNT_MASK_REG_IDX);
-	state->tpg_vcnt_mask =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_VCNT_MASK_REG_IDX);
-	state->tpg_xycnt_mask =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_XYCNT_MASK_REG_IDX);
-	state->tpg_hcnt_delta =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_HCNT_DELTA_REG_IDX);
-	state->tpg_vcnt_delta =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_VCNT_DELTA_REG_IDX);
-	state->tpg_r1 =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_R1_REG_IDX);
-	state->tpg_g1 =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_G1_REG_IDX);
-	state->tpg_b1 =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_B1_REG_IDX);
-	state->tpg_r2 =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_R2_REG_IDX);
-	state->tpg_g2 =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_G2_REG_IDX);
-	state->tpg_b2 =
-		pixelgen_ctrl_reg_load(ID, _PXG_TPG_B2_REG_IDX);
-}
-/**
- * @brief Dump the pixelgen state.
- * Refer to "pixelgen_public.h" for details.
- */
-STORAGE_CLASS_PIXELGEN_C void pixelgen_ctrl_dump_state(
-		const pixelgen_ID_t ID,
-		pixelgen_ctrl_state_t *state)
-{
-	ia_css_print("Pixel Generator ID %d Enable  0x%x \n", ID, state->com_enable);
-	ia_css_print("Pixel Generator ID %d PRBS reset vlue 0 0x%x \n", ID, state->prbs_rstval0);
-	ia_css_print("Pixel Generator ID %d PRBS reset vlue 1 0x%x \n", ID, state->prbs_rstval1);
-	ia_css_print("Pixel Generator ID %d SYNC SID 0x%x \n", ID, state->syng_sid);
-	ia_css_print("Pixel Generator ID %d syng free run 0x%x \n", ID, state->syng_free_run);
-	ia_css_print("Pixel Generator ID %d syng pause 0x%x \n", ID, state->syng_pause);
-	ia_css_print("Pixel Generator ID %d syng no of frames 0x%x \n", ID, state->syng_nof_frames);
-	ia_css_print("Pixel Generator ID %d syng no of pixels 0x%x \n", ID, state->syng_nof_pixels);
-	ia_css_print("Pixel Generator ID %d syng no of line 0x%x \n", ID, state->syng_nof_line);
-	ia_css_print("Pixel Generator ID %d syng hblank cyc  0x%x \n", ID, state->syng_hblank_cyc);
-	ia_css_print("Pixel Generator ID %d syng vblank cyc  0x%x \n", ID, state->syng_vblank_cyc);
-	ia_css_print("Pixel Generator ID %d syng stat hcnt  0x%x \n", ID, state->syng_stat_hcnt);
-	ia_css_print("Pixel Generator ID %d syng stat vcnt  0x%x \n", ID, state->syng_stat_vcnt);
-	ia_css_print("Pixel Generator ID %d syng stat fcnt  0x%x \n", ID, state->syng_stat_fcnt);
-	ia_css_print("Pixel Generator ID %d syng stat done  0x%x \n", ID, state->syng_stat_done);
-	ia_css_print("Pixel Generator ID %d tpg modee  0x%x \n", ID, state->tpg_mode);
-	ia_css_print("Pixel Generator ID %d tpg hcnt mask  0x%x \n", ID, state->tpg_hcnt_mask);
-	ia_css_print("Pixel Generator ID %d tpg hcnt mask  0x%x \n", ID, state->tpg_hcnt_mask);
-	ia_css_print("Pixel Generator ID %d tpg xycnt mask  0x%x \n", ID, state->tpg_xycnt_mask);
-	ia_css_print("Pixel Generator ID %d tpg hcnt delta  0x%x \n", ID, state->tpg_hcnt_delta);
-	ia_css_print("Pixel Generator ID %d tpg vcnt delta  0x%x \n", ID, state->tpg_vcnt_delta);
-	ia_css_print("Pixel Generator ID %d tpg r1 0x%x \n", ID, state->tpg_r1);
-	ia_css_print("Pixel Generator ID %d tpg g1 0x%x \n", ID, state->tpg_g1);
-	ia_css_print("Pixel Generator ID %d tpg b1 0x%x \n", ID, state->tpg_b1);
-	ia_css_print("Pixel Generator ID %d tpg r2 0x%x \n", ID, state->tpg_r2);
-	ia_css_print("Pixel Generator ID %d tpg g2 0x%x \n", ID, state->tpg_g2);
-	ia_css_print("Pixel Generator ID %d tpg b2 0x%x \n", ID, state->tpg_b2);
-}
-/* end of NCI */
-/*****************************************************
- *
- * Device level interface (DLI).
- *
- *****************************************************/
-/**
- * @brief Load the register value.
- * Refer to "pixelgen_public.h" for details.
- */
-STORAGE_CLASS_PIXELGEN_C hrt_data pixelgen_ctrl_reg_load(
-	const pixelgen_ID_t ID,
-	const hrt_address reg)
-{
-	assert(ID < N_PIXELGEN_ID);
-	assert(PIXELGEN_CTRL_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(PIXELGEN_CTRL_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-
-/**
- * @brief Store a value to the register.
- * Refer to "pixelgen_ctrl_public.h" for details.
- */
-STORAGE_CLASS_PIXELGEN_C void pixelgen_ctrl_reg_store(
-	const pixelgen_ID_t ID,
-	const hrt_address reg,
-	const hrt_data value)
-{
-	assert(ID < N_PIXELGEN_ID);
-	assert(PIXELGEN_CTRL_BASE[ID] != (hrt_address)-1);
-
-	ia_css_device_store_uint32(PIXELGEN_CTRL_BASE[ID] + reg*sizeof(hrt_data), value);
-}
-/* end of DLI */
-#endif /* __PIXELGEN_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/system_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/system_local.h
deleted file mode 100644
index c16670989702..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/host/system_local.h
+++ /dev/null
@@ -1,381 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SYSTEM_LOCAL_H_INCLUDED__
-#define __SYSTEM_LOCAL_H_INCLUDED__
-
-#ifdef HRT_ISP_CSS_CUSTOM_HOST
-#ifndef HRT_USE_VIR_ADDRS
-#define HRT_USE_VIR_ADDRS
-#endif
-/* This interface is deprecated */
-/*#include "hive_isp_css_custom_host_hrt.h"*/
-#endif
-
-#include "system_global.h"
-
-#ifdef __FIST__
-#define HRT_ADDRESS_WIDTH	32		/* Surprise, this is a local property and even differs per platform */
-#else
-#define HRT_ADDRESS_WIDTH	64		/* Surprise, this is a local property */
-#endif
-
-#if !defined(__KERNEL__) || (1 == 1)
-/* This interface is deprecated */
-#include "hrt/hive_types.h"
-#else  /* __KERNEL__ */
-#include <type_support.h>
-
-#if HRT_ADDRESS_WIDTH == 64
-typedef uint64_t			hrt_address;
-#elif HRT_ADDRESS_WIDTH == 32
-typedef uint32_t			hrt_address;
-#else
-#error "system_local.h: HRT_ADDRESS_WIDTH must be one of {32,64}"
-#endif
-
-typedef uint32_t			hrt_vaddress;
-typedef uint32_t			hrt_data;
-#endif /* __KERNEL__ */
-
-/*
- * Cell specific address maps
- */
-#if HRT_ADDRESS_WIDTH == 64
-
-#define GP_FIFO_BASE   ((hrt_address)0x0000000000090104)		/* This is NOT a base address */
-
-/* DDR */
-static const hrt_address DDR_BASE[N_DDR_ID] = {
-	0x0000000120000000ULL};
-
-/* ISP */
-static const hrt_address ISP_CTRL_BASE[N_ISP_ID] = {
-	0x0000000000020000ULL};
-
-static const hrt_address ISP_DMEM_BASE[N_ISP_ID] = {
-	0x0000000000200000ULL};
-
-static const hrt_address ISP_BAMEM_BASE[N_BAMEM_ID] = {
-	0x0000000000100000ULL};
-
-static const hrt_address ISP_VAMEM_BASE[N_VAMEM_ID] = {
-	0x00000000001C0000ULL,
-	0x00000000001D0000ULL,
-	0x00000000001E0000ULL};
-
-static const hrt_address ISP_HMEM_BASE[N_HMEM_ID] = {
-	0x00000000001F0000ULL};
-
-/* SP */
-static const hrt_address SP_CTRL_BASE[N_SP_ID] = {
-	0x0000000000010000ULL};
-
-static const hrt_address SP_DMEM_BASE[N_SP_ID] = {
-	0x0000000000300000ULL};
-
-/* MMU */
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) || defined(IS_ISP_2401_MAMOIADA_SYSTEM)
-/*
- * MMU0_ID: The data MMU
- * MMU1_ID: The icache MMU
- */
-static const hrt_address MMU_BASE[N_MMU_ID] = {
-	0x0000000000070000ULL,
-	0x00000000000A0000ULL};
-#else
-#error "system_local.h: SYSTEM must be one of {2400, 2401 }"
-#endif
-
-/* DMA */
-static const hrt_address DMA_BASE[N_DMA_ID] = {
-	0x0000000000040000ULL};
-
-static const hrt_address ISYS2401_DMA_BASE[N_ISYS2401_DMA_ID] = {
-	0x00000000000CA000ULL};
-
-/* IRQ */
-static const hrt_address IRQ_BASE[N_IRQ_ID] = {
-	0x0000000000000500ULL,
-	0x0000000000030A00ULL,
-	0x000000000008C000ULL,
-	0x0000000000090200ULL};
-/*
-	0x0000000000000500ULL};
- */
-
-/* GDC */
-static const hrt_address GDC_BASE[N_GDC_ID] = {
-	0x0000000000050000ULL,
-	0x0000000000060000ULL};
-
-/* FIFO_MONITOR (not a subset of GP_DEVICE) */
-static const hrt_address FIFO_MONITOR_BASE[N_FIFO_MONITOR_ID] = {
-	0x0000000000000000ULL};
-
-/*
-static const hrt_address GP_REGS_BASE[N_GP_REGS_ID] = {
-	0x0000000000000000ULL};
-
-static const hrt_address GP_DEVICE_BASE[N_GP_DEVICE_ID] = {
-	0x0000000000090000ULL};
-*/
-
-/* GP_DEVICE (single base for all separate GP_REG instances) */
-static const hrt_address GP_DEVICE_BASE[N_GP_DEVICE_ID] = {
-	0x0000000000000000ULL};
-
-/*GP TIMER , all timer registers are inter-twined,
- * so, having multiple base addresses for
- * different timers does not help*/
-static const hrt_address GP_TIMER_BASE =
-	(hrt_address)0x0000000000000600ULL;
-
-/* GPIO */
-static const hrt_address GPIO_BASE[N_GPIO_ID] = {
-	0x0000000000000400ULL};
-
-/* TIMED_CTRL */
-static const hrt_address TIMED_CTRL_BASE[N_TIMED_CTRL_ID] = {
-	0x0000000000000100ULL};
-
-
-/* INPUT_FORMATTER */
-static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = {
-	0x0000000000030000ULL,
-	0x0000000000030200ULL,
-	0x0000000000030400ULL,
-	0x0000000000030600ULL}; /* memcpy() */
-
-/* INPUT_SYSTEM */
-static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = {
-	0x0000000000080000ULL};
-/*	0x0000000000081000ULL, */ /* capture A */
-/*	0x0000000000082000ULL, */ /* capture B */
-/*	0x0000000000083000ULL, */ /* capture C */
-/*	0x0000000000084000ULL, */ /* Acquisition */
-/*	0x0000000000085000ULL, */ /* DMA */
-/*	0x0000000000089000ULL, */ /* ctrl */
-/*	0x000000000008A000ULL, */ /* GP regs */
-/*	0x000000000008B000ULL, */ /* FIFO */
-/*	0x000000000008C000ULL, */ /* IRQ */
-
-/* RX, the MIPI lane control regs start at offset 0 */
-static const hrt_address RX_BASE[N_RX_ID] = {
-	0x0000000000080100ULL};
-
-/* IBUF_CTRL, part of the Input System 2401 */
-static const hrt_address IBUF_CTRL_BASE[N_IBUF_CTRL_ID] = {
-	0x00000000000C1800ULL,	/* ibuf controller A */
-	0x00000000000C3800ULL,	/* ibuf controller B */
-	0x00000000000C5800ULL	/* ibuf controller C */
-};
-
-/* ISYS IRQ Controllers, part of the Input System 2401 */
-static const hrt_address ISYS_IRQ_BASE[N_ISYS_IRQ_ID] = {
-	0x00000000000C1400ULL,	/* port a */
-	0x00000000000C3400ULL,	/* port b */
-	0x00000000000C5400ULL	/* port c */
-};
-
-/* CSI FE, part of the Input System 2401 */
-static const hrt_address CSI_RX_FE_CTRL_BASE[N_CSI_RX_FRONTEND_ID] = {
-	0x00000000000C0400ULL,	/* csi fe controller A */
-	0x00000000000C2400ULL,	/* csi fe controller B */
-	0x00000000000C4400ULL	/* csi fe controller C */
-};
-/* CSI BE, part of the Input System 2401 */
-static const hrt_address CSI_RX_BE_CTRL_BASE[N_CSI_RX_BACKEND_ID] = {
-	0x00000000000C0800ULL,	/* csi be controller A */
-	0x00000000000C2800ULL,	/* csi be controller B */
-	0x00000000000C4800ULL	/* csi be controller C */
-};
-/* PIXEL Generator, part of the Input System 2401 */
-static const hrt_address PIXELGEN_CTRL_BASE[N_PIXELGEN_ID] = {
-	0x00000000000C1000ULL,	/* pixel gen controller A */
-	0x00000000000C3000ULL,	/* pixel gen controller B */
-	0x00000000000C5000ULL	/* pixel gen controller C */
-};
-/* Stream2MMIO, part of the Input System 2401 */
-static const hrt_address STREAM2MMIO_CTRL_BASE[N_STREAM2MMIO_ID] = {
-	0x00000000000C0C00ULL,	/* stream2mmio controller A */
-	0x00000000000C2C00ULL,	/* stream2mmio controller B */
-	0x00000000000C4C00ULL	/* stream2mmio controller C */
-};
-#elif HRT_ADDRESS_WIDTH == 32
-
-#define GP_FIFO_BASE   ((hrt_address)0x00090104)		/* This is NOT a base address */
-
-/* DDR : Attention, this value not defined in 32-bit */
-static const hrt_address DDR_BASE[N_DDR_ID] = {
-	0x00000000UL};
-
-/* ISP */
-static const hrt_address ISP_CTRL_BASE[N_ISP_ID] = {
-	0x00020000UL};
-
-static const hrt_address ISP_DMEM_BASE[N_ISP_ID] = {
-	0xffffffffUL};
-
-static const hrt_address ISP_BAMEM_BASE[N_BAMEM_ID] = {
-	0xffffffffUL};
-
-static const hrt_address ISP_VAMEM_BASE[N_VAMEM_ID] = {
-	0xffffffffUL,
-	0xffffffffUL,
-	0xffffffffUL};
-
-static const hrt_address ISP_HMEM_BASE[N_HMEM_ID] = {
-	0xffffffffUL};
-
-/* SP */
-static const hrt_address SP_CTRL_BASE[N_SP_ID] = {
-	0x00010000UL};
-
-static const hrt_address SP_DMEM_BASE[N_SP_ID] = {
-	0x00300000UL};
-
-/* MMU */
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) || defined(IS_ISP_2401_MAMOIADA_SYSTEM)
-/*
- * MMU0_ID: The data MMU
- * MMU1_ID: The icache MMU
- */
-static const hrt_address MMU_BASE[N_MMU_ID] = {
-	0x00070000UL,
-	0x000A0000UL};
-#else
-#error "system_local.h: SYSTEM must be one of {2400, 2401 }"
-#endif
-
-/* DMA */
-static const hrt_address DMA_BASE[N_DMA_ID] = {
-	0x00040000UL};
-
-static const hrt_address ISYS2401_DMA_BASE[N_ISYS2401_DMA_ID] = {
-	0x000CA000UL};
-
-/* IRQ */
-static const hrt_address IRQ_BASE[N_IRQ_ID] = {
-	0x00000500UL,
-	0x00030A00UL,
-	0x0008C000UL,
-	0x00090200UL};
-/*
-	0x00000500UL};
- */
-
-/* GDC */
-static const hrt_address GDC_BASE[N_GDC_ID] = {
-	0x00050000UL,
-	0x00060000UL};
-
-/* FIFO_MONITOR (not a subset of GP_DEVICE) */
-static const hrt_address FIFO_MONITOR_BASE[N_FIFO_MONITOR_ID] = {
-	0x00000000UL};
-
-/*
-static const hrt_address GP_REGS_BASE[N_GP_REGS_ID] = {
-	0x00000000UL};
-
-static const hrt_address GP_DEVICE_BASE[N_GP_DEVICE_ID] = {
-	0x00090000UL};
-*/
-
-/* GP_DEVICE (single base for all separate GP_REG instances) */
-static const hrt_address GP_DEVICE_BASE[N_GP_DEVICE_ID] = {
-	0x00000000UL};
-
-/*GP TIMER , all timer registers are inter-twined,
- * so, having multiple base addresses for
- * different timers does not help*/
-static const hrt_address GP_TIMER_BASE =
-	(hrt_address)0x00000600UL;
-/* GPIO */
-static const hrt_address GPIO_BASE[N_GPIO_ID] = {
-	0x00000400UL};
-
-/* TIMED_CTRL */
-static const hrt_address TIMED_CTRL_BASE[N_TIMED_CTRL_ID] = {
-	0x00000100UL};
-
-
-/* INPUT_FORMATTER */
-static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = {
-	0x00030000UL,
-	0x00030200UL,
-	0x00030400UL};
-/*	0x00030600UL, */ /* memcpy() */
-
-/* INPUT_SYSTEM */
-static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = {
-	0x00080000UL};
-/*	0x00081000UL, */ /* capture A */
-/*	0x00082000UL, */ /* capture B */
-/*	0x00083000UL, */ /* capture C */
-/*	0x00084000UL, */ /* Acquisition */
-/*	0x00085000UL, */ /* DMA */
-/*	0x00089000UL, */ /* ctrl */
-/*	0x0008A000UL, */ /* GP regs */
-/*	0x0008B000UL, */ /* FIFO */
-/*	0x0008C000UL, */ /* IRQ */
-
-/* RX, the MIPI lane control regs start at offset 0 */
-static const hrt_address RX_BASE[N_RX_ID] = {
-	0x00080100UL};
-
-/* IBUF_CTRL, part of the Input System 2401 */
-static const hrt_address IBUF_CTRL_BASE[N_IBUF_CTRL_ID] = {
-	0x000C1800UL,	/* ibuf controller A */
-	0x000C3800UL,	/* ibuf controller B */
-	0x000C5800UL	/* ibuf controller C */
-};
-
-/* ISYS IRQ Controllers, part of the Input System 2401 */
-static const hrt_address ISYS_IRQ_BASE[N_ISYS_IRQ_ID] = {
-	0x000C1400ULL,	/* port a */
-	0x000C3400ULL,	/* port b */
-	0x000C5400ULL	/* port c */
-};
-
-/* CSI FE, part of the Input System 2401 */
-static const hrt_address CSI_RX_FE_CTRL_BASE[N_CSI_RX_FRONTEND_ID] = {
-	0x000C0400UL,	/* csi fe controller A */
-	0x000C2400UL,	/* csi fe controller B */
-	0x000C4400UL	/* csi fe controller C */
-};
-/* CSI BE, part of the Input System 2401 */
-static const hrt_address CSI_RX_FE_CTRL_BASE[N_CSI_RX_BACKEND_ID] = {
-	0x000C0800UL,	/* csi be controller A */
-	0x000C2800UL,	/* csi be controller B */
-	0x000C4800UL	/* csi be controller C */
-};
-/* PIXEL Generator, part of the Input System 2401 */
-static const hrt_address PIXELGEN_CTRL_BASE[N_PIXELGEN_ID] = {
-	0x000C1000UL,	/* pixel gen controller A */
-	0x000C3000UL,	/* pixel gen controller B */
-	0x000C5000UL	/* pixel gen controller C */
-};
-/* Stream2MMIO, part of the Input System 2401 */
-static const hrt_address STREAM2MMIO_CTRL_BASE[N_STREAM2MMIO_ID] = {
-	0x000C0C00UL,	/* stream2mmio controller A */
-	0x000C2C00UL,	/* stream2mmio controller B */
-	0x000C4C00UL	/* stream2mmio controller C */
-};
-
-#else
-#error "system_local.h: HRT_ADDRESS_WIDTH must be one of {32,64}"
-#endif
-
-#endif /* __SYSTEM_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/PixelGen_SysBlock_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/PixelGen_SysBlock_defs.h
deleted file mode 100644
index 1b3391c242a3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/PixelGen_SysBlock_defs.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _PixelGen_SysBlock_defs_h
-#define _PixelGen_SysBlock_defs_h
-
-#ifdef ISYS2401_PXG_A
-#else
-#ifdef ISYS2401_PXG_B
-#else
-#ifdef ISYS2401_PXG_C
-#else
-#include <mipi_backend/hrt/include/mipi_backend_defs.h>
-#endif
-#endif
-#endif
-
-/* Parematers and User_Parameters for HSS */
-#define _PXG_PPC                       Ppc
-#define _PXG_PIXEL_BITS                PixelWidth
-#define _PXG_MAX_NOF_SID               MaxNofSids
-#define _PXG_DATA_BITS                 DataWidth
-#define _PXG_CNT_BITS                  CntWidth
-#define _PXG_FIFODEPTH                 FifoDepth
-#define _PXG_DBG                       Dbg_device_not_included
-
-/* ID's and Address */
-#define _PXG_ADRRESS_ALIGN_REG         4
-
-#define _PXG_COM_ENABLE_REG_IDX        0
-#define _PXG_PRBS_RSTVAL_REG0_IDX      1
-#define _PXG_PRBS_RSTVAL_REG1_IDX      2
-#define _PXG_SYNG_SID_REG_IDX          3
-#define _PXG_SYNG_FREE_RUN_REG_IDX     4
-#define _PXG_SYNG_PAUSE_REG_IDX        5
-#define _PXG_SYNG_NOF_FRAME_REG_IDX    6
-#define _PXG_SYNG_NOF_PIXEL_REG_IDX    7
-#define _PXG_SYNG_NOF_LINE_REG_IDX     8
-#define _PXG_SYNG_HBLANK_CYC_REG_IDX   9
-#define _PXG_SYNG_VBLANK_CYC_REG_IDX  10
-#define _PXG_SYNG_STAT_HCNT_REG_IDX   11
-#define _PXG_SYNG_STAT_VCNT_REG_IDX   12
-#define _PXG_SYNG_STAT_FCNT_REG_IDX   13
-#define _PXG_SYNG_STAT_DONE_REG_IDX   14
-#define _PXG_TPG_MODE_REG_IDX         15
-#define _PXG_TPG_HCNT_MASK_REG_IDX    16
-#define _PXG_TPG_VCNT_MASK_REG_IDX    17
-#define _PXG_TPG_XYCNT_MASK_REG_IDX   18
-#define _PXG_TPG_HCNT_DELTA_REG_IDX   19
-#define _PXG_TPG_VCNT_DELTA_REG_IDX   20
-#define _PXG_TPG_R1_REG_IDX           21
-#define _PXG_TPG_G1_REG_IDX           22
-#define _PXG_TPG_B1_REG_IDX           23
-#define _PXG_TPG_R2_REG_IDX           24
-#define _PXG_TPG_G2_REG_IDX           25
-#define _PXG_TPG_B2_REG_IDX           26
-/* */
-#define _PXG_SYNG_PAUSE_CYCLES        0
-/* Subblock ID's */
-#define _PXG_DISBALE_IDX              0
-#define _PXG_PRBS_IDX                 0
-#define _PXG_TPG_IDX                  1
-#define _PXG_SYNG_IDX                 2
-#define _PXG_SMUX_IDX                 3
-/* Register Widths */
-#define _PXG_COM_ENABLE_REG_WIDTH     2
-#define _PXG_COM_SRST_REG_WIDTH       4
-#define _PXG_PRBS_RSTVAL_REG0_WIDTH  31
-#define _PXG_PRBS_RSTVAL_REG1_WIDTH  31
-
-#define _PXG_SYNG_SID_REG_WIDTH        3
-
-#define _PXG_SYNG_FREE_RUN_REG_WIDTH   1
-#define _PXG_SYNG_PAUSE_REG_WIDTH      1
-/*
-#define _PXG_SYNG_NOF_FRAME_REG_WIDTH  <sync_gen_cnt_width>
-#define _PXG_SYNG_NOF_PIXEL_REG_WIDTH  <sync_gen_cnt_width>
-#define _PXG_SYNG_NOF_LINE_REG_WIDTH   <sync_gen_cnt_width>
-#define _PXG_SYNG_HBLANK_CYC_REG_WIDTH <sync_gen_cnt_width>
-#define _PXG_SYNG_VBLANK_CYC_REG_WIDTH <sync_gen_cnt_width>
-#define _PXG_SYNG_STAT_HCNT_REG_WIDTH  <sync_gen_cnt_width>
-#define _PXG_SYNG_STAT_VCNT_REG_WIDTH  <sync_gen_cnt_width>
-#define _PXG_SYNG_STAT_FCNT_REG_WIDTH  <sync_gen_cnt_width>
-*/
-#define _PXG_SYNG_STAT_DONE_REG_WIDTH  1
-#define _PXG_TPG_MODE_REG_WIDTH        2
-/*
-#define _PXG_TPG_HCNT_MASK_REG_WIDTH   <sync_gen_cnt_width>
-#define _PXG_TPG_VCNT_MASK_REG_WIDTH   <sync_gen_cnt_width>
-#define _PXG_TPG_XYCNT_MASK_REG_WIDTH  <pixle_width>
-*/
-#define _PXG_TPG_HCNT_DELTA_REG_WIDTH  4
-#define _PXG_TPG_VCNT_DELTA_REG_WIDTH  4
-/*
-#define _PXG_TPG_R1_REG_WIDTH          <pixle_width>
-#define _PXG_TPG_G1_REG_WIDTH          <pixle_width>
-#define _PXG_TPG_B1_REG_WIDTH          <pixle_width>
-#define _PXG_TPG_R2_REG_WIDTH          <pixle_width>
-#define _PXG_TPG_G2_REG_WIDTH          <pixle_width>
-#define _PXG_TPG_B2_REG_WIDTH          <pixle_width>
-*/
-#define _PXG_FIFO_DEPTH                2
-/* MISC */
-#define _PXG_ENABLE_REG_VAL            1
-#define _PXG_PRBS_ENABLE_REG_VAL       1
-#define _PXG_TPG_ENABLE_REG_VAL        2
-#define _PXG_SYNG_ENABLE_REG_VAL       4
-#define _PXG_FIFO_ENABLE_REG_VAL       8
-#define _PXG_PXL_BITS                 14
-#define _PXG_INVALID_FLAG              0xDEADBEEF
-#define _PXG_CAFE_FLAG                 0xCAFEBABE
-
-
-#endif /* _PixelGen_SysBlock_defs_h */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/bits.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/bits.h
deleted file mode 100644
index e71e33d9d143..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/bits.h
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_BITS_H
-#define _HRT_BITS_H
-
-#include "defs.h"
-
-#define _hrt_ones(n) HRTCAT(_hrt_ones_, n)
-#define _hrt_ones_0x0  0x00000000U
-#define _hrt_ones_0x1  0x00000001U
-#define _hrt_ones_0x2  0x00000003U
-#define _hrt_ones_0x3  0x00000007U
-#define _hrt_ones_0x4  0x0000000FU
-#define _hrt_ones_0x5  0x0000001FU
-#define _hrt_ones_0x6  0x0000003FU
-#define _hrt_ones_0x7  0x0000007FU
-#define _hrt_ones_0x8  0x000000FFU
-#define _hrt_ones_0x9  0x000001FFU
-#define _hrt_ones_0xA  0x000003FFU
-#define _hrt_ones_0xB  0x000007FFU
-#define _hrt_ones_0xC  0x00000FFFU
-#define _hrt_ones_0xD  0x00001FFFU
-#define _hrt_ones_0xE  0x00003FFFU
-#define _hrt_ones_0xF  0x00007FFFU
-#define _hrt_ones_0x10 0x0000FFFFU
-#define _hrt_ones_0x11 0x0001FFFFU
-#define _hrt_ones_0x12 0x0003FFFFU
-#define _hrt_ones_0x13 0x0007FFFFU
-#define _hrt_ones_0x14 0x000FFFFFU
-#define _hrt_ones_0x15 0x001FFFFFU
-#define _hrt_ones_0x16 0x003FFFFFU
-#define _hrt_ones_0x17 0x007FFFFFU
-#define _hrt_ones_0x18 0x00FFFFFFU
-#define _hrt_ones_0x19 0x01FFFFFFU
-#define _hrt_ones_0x1A 0x03FFFFFFU
-#define _hrt_ones_0x1B 0x07FFFFFFU
-#define _hrt_ones_0x1C 0x0FFFFFFFU
-#define _hrt_ones_0x1D 0x1FFFFFFFU
-#define _hrt_ones_0x1E 0x3FFFFFFFU
-#define _hrt_ones_0x1F 0x7FFFFFFFU
-#define _hrt_ones_0x20 0xFFFFFFFFU
-
-#define _hrt_ones_0  _hrt_ones_0x0
-#define _hrt_ones_1  _hrt_ones_0x1
-#define _hrt_ones_2  _hrt_ones_0x2
-#define _hrt_ones_3  _hrt_ones_0x3
-#define _hrt_ones_4  _hrt_ones_0x4
-#define _hrt_ones_5  _hrt_ones_0x5
-#define _hrt_ones_6  _hrt_ones_0x6
-#define _hrt_ones_7  _hrt_ones_0x7
-#define _hrt_ones_8  _hrt_ones_0x8
-#define _hrt_ones_9  _hrt_ones_0x9
-#define _hrt_ones_10 _hrt_ones_0xA
-#define _hrt_ones_11 _hrt_ones_0xB
-#define _hrt_ones_12 _hrt_ones_0xC
-#define _hrt_ones_13 _hrt_ones_0xD
-#define _hrt_ones_14 _hrt_ones_0xE
-#define _hrt_ones_15 _hrt_ones_0xF
-#define _hrt_ones_16 _hrt_ones_0x10
-#define _hrt_ones_17 _hrt_ones_0x11
-#define _hrt_ones_18 _hrt_ones_0x12
-#define _hrt_ones_19 _hrt_ones_0x13
-#define _hrt_ones_20 _hrt_ones_0x14
-#define _hrt_ones_21 _hrt_ones_0x15
-#define _hrt_ones_22 _hrt_ones_0x16
-#define _hrt_ones_23 _hrt_ones_0x17
-#define _hrt_ones_24 _hrt_ones_0x18
-#define _hrt_ones_25 _hrt_ones_0x19
-#define _hrt_ones_26 _hrt_ones_0x1A
-#define _hrt_ones_27 _hrt_ones_0x1B
-#define _hrt_ones_28 _hrt_ones_0x1C
-#define _hrt_ones_29 _hrt_ones_0x1D
-#define _hrt_ones_30 _hrt_ones_0x1E
-#define _hrt_ones_31 _hrt_ones_0x1F
-#define _hrt_ones_32 _hrt_ones_0x20
-
-#define _hrt_mask(b, n) \
-  (_hrt_ones(n) << (b))
-#define _hrt_get_bits(w, b, n) \
-  (((w) >> (b)) & _hrt_ones(n))
-#define _hrt_set_bits(w, b, n, v) \
-  (((w) & ~_hrt_mask(b, n)) | (((v) & _hrt_ones(n)) << (b)))
-#define _hrt_get_bit(w, b) \
-  (((w) >> (b)) & 1)
-#define _hrt_set_bit(w, b, v) \
-  (((w) & (~(1 << (b)))) | (((v)&1) << (b)))
-#define _hrt_set_lower_half(w, v) \
-  _hrt_set_bits(w, 0, 16, v)
-#define _hrt_set_upper_half(w, v) \
-  _hrt_set_bits(w, 16, 16, v)
-
-#endif /* _HRT_BITS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/cell_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/cell_params.h
deleted file mode 100644
index b5756bfe8eb6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/cell_params.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _cell_params_h
-#define _cell_params_h
-
-#define SP_PMEM_LOG_WIDTH_BITS           6  /*Width of PC, 64 bits, 8 bytes*/
-#define SP_ICACHE_TAG_BITS               4  /*size of tag*/
-#define SP_ICACHE_SET_BITS               8  /* 256 sets*/
-#define SP_ICACHE_BLOCKS_PER_SET_BITS    1  /* 2 way associative*/
-#define SP_ICACHE_BLOCK_ADDRESS_BITS     11 /* 2048 lines capacity*/
-
-#define SP_ICACHE_ADDRESS_BITS \
-	                    (SP_ICACHE_TAG_BITS+SP_ICACHE_BLOCK_ADDRESS_BITS)
-
-#define SP_PMEM_DEPTH        (1<<SP_ICACHE_ADDRESS_BITS)
-
-#define SP_FIFO_0_DEPTH      0
-#define SP_FIFO_1_DEPTH      0
-#define SP_FIFO_2_DEPTH      0
-#define SP_FIFO_3_DEPTH      0
-#define SP_FIFO_4_DEPTH      0
-#define SP_FIFO_5_DEPTH      0
-#define SP_FIFO_6_DEPTH      0
-#define SP_FIFO_7_DEPTH      0
-
-
-#define SP_SLV_BUS_MAXBURSTSIZE        1
-
-#endif /* _cell_params_h */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/css_receiver_2400_common_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/css_receiver_2400_common_defs.h
deleted file mode 100644
index f3054fe04d03..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/css_receiver_2400_common_defs.h
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _css_receiver_2400_common_defs_h_
-#define _css_receiver_2400_common_defs_h_
-#ifndef _mipi_backend_common_defs_h_
-#define _mipi_backend_common_defs_h_
-
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_DATA_WIDTH     16
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_CH_ID_WIDTH     2
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_FMT_TYPE_WIDTH  3
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_REAL_WIDTH (_HRT_CSS_RECEIVER_2400_GEN_SHORT_DATA_WIDTH + _HRT_CSS_RECEIVER_2400_GEN_SHORT_CH_ID_WIDTH + _HRT_CSS_RECEIVER_2400_GEN_SHORT_FMT_TYPE_WIDTH)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_WIDTH      32 /* use 32 to be compatibel with streaming monitor !, MSB's of interface are tied to '0' */ 
-
-/* Definition of data format ID at the interface CSS_receiver capture/acquisition units */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8          24   /* 01 1000 YUV420 8-bit                                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_10         25   /* 01 1001  YUV420 10-bit                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8L         26   /* 01 1010   YUV420 8-bit legacy                               */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV422_8          30   /* 01 1110   YUV422 8-bit                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV422_10         31   /* 01 1111   YUV422 10-bit                                     */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB444            32   /* 10 0000   RGB444                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB555            33   /* 10 0001   RGB555                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB565            34   /* 10 0010   RGB565                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB666            35   /* 10 0011   RGB666                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB888            36   /* 10 0100   RGB888                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW6              40   /* 10 1000   RAW6                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW7              41   /* 10 1001   RAW7                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW8              42   /* 10 1010   RAW8                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW10             43   /* 10 1011   RAW10                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW12             44   /* 10 1100   RAW12                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW14             45   /* 10 1101   RAW14                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_1         48   /* 11 0000    JPEG [User Defined 8-bit Data Type 1]            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_2         49   /* 11 0001    User Defined 8-bit Data Type 2                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_3         50   /* 11 0010    User Defined 8-bit Data Type 3                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_4         51   /* 11 0011    User Defined 8-bit Data Type 4                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_5         52   /* 11 0100    User Defined 8-bit Data Type 5                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_6         53   /* 11 0101    User Defined 8-bit Data Type 6                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_7         54   /* 11 0110    User Defined 8-bit Data Type 7                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_8         55   /* 11 0111    User Defined 8-bit Data Type 8                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_Emb               18   /* 01 0010    embedded eight bit non image data                */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_SOF                0   /* 00 0000    frame start                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_EOF                1   /* 00 0001    frame end                                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_SOL                2   /* 00 0010    line start                                       */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_EOL                3   /* 00 0011    line end                                         */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH1            8   /* 00 1000  Generic Short Packet Code 1                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH2            9   /* 00 1001    Generic Short Packet Code 2                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH3           10   /* 00 1010    Generic Short Packet Code 3                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH4           11   /* 00 1011    Generic Short Packet Code 4                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH5           12   /* 00 1100    Generic Short Packet Code 5                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH6           13   /* 00 1101    Generic Short Packet Code 6                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH7           14   /* 00 1110    Generic Short Packet Code 7                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH8           15   /* 00 1111    Generic Short Packet Code 8                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8_CSPS     28   /* 01 1100   YUV420 8-bit (Chroma Shifted Pixel Sampling)      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_10_CSPS    29   /* 01 1101   YUV420 10-bit (Chroma Shifted Pixel Sampling)     */
-/* used reseved mipi positions for these */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW16             46 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18             47 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18_2           37 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18_3           38 
-
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_WIDTH              6
-
-/* Definition of format_types at the interface CSS --> input_selector*/
-/* !! Changes here should be copied to systems/isp/isp_css/bin/conv_transmitter_cmd.tcl !! */
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB888           0  // 36 'h24
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB555           1  // 33 'h
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB444           2  // 32
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB565           3  // 34
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB666           4  // 35
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW8             5  // 42 
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW10            6  // 43
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW6             7  // 40
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW7             8  // 41
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW12            9  // 43
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW14           10  // 45
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8        11  // 30
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_10       12  // 25
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV422_8        13  // 30
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV422_10       14  // 31
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_1       15  // 48
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8L       16  // 26
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_Emb             17  // 18
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_2       18  // 49
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_3       19  // 50
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_4       20  // 51
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_5       21  // 52
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_6       22  // 53
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_7       23  // 54
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_8       24  // 55
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8_CSPS   25  // 28
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_10_CSPS  26  // 29
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW16           27  // ?
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18           28  // ?
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18_2         29  // ? Option 2 for depacketiser
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18_3         30  // ? Option 3 for depacketiser
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_CUSTOM          31  // to signal custom decoding 
-
-/* definition for state machine of data FIFO for decode different type of data */
-#define _HRT_CSS_RECEIVER_2400_YUV420_8_REPEAT_PTN                 1  
-#define _HRT_CSS_RECEIVER_2400_YUV420_10_REPEAT_PTN                5
-#define _HRT_CSS_RECEIVER_2400_YUV420_8L_REPEAT_PTN                1
-#define _HRT_CSS_RECEIVER_2400_YUV422_8_REPEAT_PTN                 1
-#define _HRT_CSS_RECEIVER_2400_YUV422_10_REPEAT_PTN                5
-#define _HRT_CSS_RECEIVER_2400_RGB444_REPEAT_PTN                   2 
-#define _HRT_CSS_RECEIVER_2400_RGB555_REPEAT_PTN                   2
-#define _HRT_CSS_RECEIVER_2400_RGB565_REPEAT_PTN                   2
-#define _HRT_CSS_RECEIVER_2400_RGB666_REPEAT_PTN                   9                       
-#define _HRT_CSS_RECEIVER_2400_RGB888_REPEAT_PTN                   3
-#define _HRT_CSS_RECEIVER_2400_RAW6_REPEAT_PTN                     3
-#define _HRT_CSS_RECEIVER_2400_RAW7_REPEAT_PTN                     7
-#define _HRT_CSS_RECEIVER_2400_RAW8_REPEAT_PTN                     1
-#define _HRT_CSS_RECEIVER_2400_RAW10_REPEAT_PTN                    5
-#define _HRT_CSS_RECEIVER_2400_RAW12_REPEAT_PTN                    3        
-#define _HRT_CSS_RECEIVER_2400_RAW14_REPEAT_PTN                    7
-
-#define _HRT_CSS_RECEIVER_2400_MAX_REPEAT_PTN                      _HRT_CSS_RECEIVER_2400_RGB666_REPEAT_PTN
-
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FMT_IDX                     0
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FMT_WIDTH                   3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_PRED_IDX                    3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_PRED_WIDTH                  1
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_USD_BITS                    4  /* bits per USD type */
-
-#define _HRT_CSS_RECEIVER_2400_BE_RAW16_DATAID_IDX                 0
-#define _HRT_CSS_RECEIVER_2400_BE_RAW16_EN_IDX                     6
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_DATAID_IDX                 0
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_OPTION_IDX                 6
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_EN_IDX                     8
-
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_NO_COMP                     0
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_6_10                     1
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_7_10                     2
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_8_10                     3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_6_12                     4
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_7_12                     5
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_8_12                     6
-
-
-/* packet bit definition */
-#define _HRT_CSS_RECEIVER_2400_PKT_SOP_IDX                        32
-#define _HRT_CSS_RECEIVER_2400_PKT_SOP_BITS                        1
-#define _HRT_CSS_RECEIVER_2400_PKT_CH_ID_IDX                      22
-#define _HRT_CSS_RECEIVER_2400_PKT_CH_ID_BITS                      2
-#define _HRT_CSS_RECEIVER_2400_PKT_FMT_ID_IDX                     16
-#define _HRT_CSS_RECEIVER_2400_PKT_FMT_ID_BITS                     6
-#define _HRT_CSS_RECEIVER_2400_PH_DATA_FIELD_IDX                   0
-#define _HRT_CSS_RECEIVER_2400_PH_DATA_FIELD_BITS                 16
-#define _HRT_CSS_RECEIVER_2400_PKT_PAYLOAD_IDX                     0
-#define _HRT_CSS_RECEIVER_2400_PKT_PAYLOAD_BITS                   32
-
-
-/*************************************************************************************************/
-/* Custom Decoding                                                                               */
-/* These Custom Defs are defined based on design-time config in "csi_be_pixel_formatter.chdl" !! */
-/*************************************************************************************************/
-#define BE_CUST_EN_IDX                     0     /* 2bits */
-#define BE_CUST_EN_DATAID_IDX              2     /* 6bits MIPI DATA ID */ 
-#define BE_CUST_EN_WIDTH                   8     
-#define BE_CUST_MODE_ALL                   1     /* Enable Custom Decoding for all DATA IDs */
-#define BE_CUST_MODE_ONE                   3     /* Enable Custom Decoding for ONE DATA ID, programmed in CUST_EN_DATA_ID */
-
-/* Data State config = {get_bits(6bits), valid(1bit)}  */
-#define BE_CUST_DATA_STATE_S0_IDX          0     /* 7bits */ 
-#define BE_CUST_DATA_STATE_S1_IDX          7     /* 7bits */ 
-#define BE_CUST_DATA_STATE_S2_IDX          14    /* 7bits */
-#define BE_CUST_DATA_STATE_WIDTH           21    
-#define BE_CUST_DATA_STATE_VALID_IDX       0     /* 1bits */
-#define BE_CUST_DATA_STATE_GETBITS_IDX     1     /* 6bits */
-
-/* Pixel Extractor config */
-#define BE_CUST_PIX_EXT_DATA_ALIGN_IDX     0     /* 5bits */
-#define BE_CUST_PIX_EXT_PIX_ALIGN_IDX      5     /* 5bits */
-#define BE_CUST_PIX_EXT_PIX_MASK_IDX       10    /* 18bits */
-#define BE_CUST_PIX_EXT_PIX_EN_IDX         28    /* 1bits */
-#define BE_CUST_PIX_EXT_WIDTH              29    
-
-/* Pixel Valid & EoP config = {[eop,valid](especial), [eop,valid](normal)} */
-#define BE_CUST_PIX_VALID_EOP_P0_IDX        0    /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_P1_IDX        4    /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_P2_IDX        8    /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_P3_IDX        12   /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_WIDTH         16 
-#define BE_CUST_PIX_VALID_EOP_NOR_VALID_IDX 0    /* Normal (NO less get_bits case) Valid - 1bits */
-#define BE_CUST_PIX_VALID_EOP_NOR_EOP_IDX   1    /* Normal (NO less get_bits case) EoP - 1bits */
-#define BE_CUST_PIX_VALID_EOP_ESP_VALID_IDX 2    /* Especial (less get_bits case) Valid - 1bits */
-#define BE_CUST_PIX_VALID_EOP_ESP_EOP_IDX   3    /* Especial (less get_bits case) EoP - 1bits */
-
-#endif /* _mipi_backend_common_defs_h_ */
-#endif /* _css_receiver_2400_common_defs_h_ */ 
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/css_receiver_2400_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/css_receiver_2400_defs.h
deleted file mode 100644
index 6f5b7d3d3715..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/css_receiver_2400_defs.h
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _css_receiver_2400_defs_h_
-#define _css_receiver_2400_defs_h_
-
-#include "css_receiver_2400_common_defs.h"
-
-#define CSS_RECEIVER_DATA_WIDTH                8
-#define CSS_RECEIVER_RX_TRIG                   4
-#define CSS_RECEIVER_RF_WORD                  32
-#define CSS_RECEIVER_IMG_PROC_RF_ADDR         10
-#define CSS_RECEIVER_CSI_RF_ADDR               4
-#define CSS_RECEIVER_DATA_OUT                 12
-#define CSS_RECEIVER_CHN_NO                    2
-#define CSS_RECEIVER_DWORD_CNT                11
-#define CSS_RECEIVER_FORMAT_TYP                5
-#define CSS_RECEIVER_HRESPONSE                 2
-#define CSS_RECEIVER_STATE_WIDTH               3
-#define CSS_RECEIVER_FIFO_DAT                 32
-#define CSS_RECEIVER_CNT_VAL                   2
-#define CSS_RECEIVER_PRED10_VAL               10
-#define CSS_RECEIVER_PRED12_VAL               12
-#define CSS_RECEIVER_CNT_WIDTH                 8
-#define CSS_RECEIVER_WORD_CNT                 16
-#define CSS_RECEIVER_PIXEL_LEN                 6
-#define CSS_RECEIVER_PIXEL_CNT                 5
-#define CSS_RECEIVER_COMP_8_BIT                8
-#define CSS_RECEIVER_COMP_7_BIT                7
-#define CSS_RECEIVER_COMP_6_BIT                6
-
-#define CSI_CONFIG_WIDTH                       4
-
-/* division of gen_short data, ch_id and fmt_type over streaming data interface */
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_DATA_BIT_LSB     0
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_LSB (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_DATA_BIT_LSB     + _HRT_CSS_RECEIVER_2400_GEN_SHORT_DATA_WIDTH)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_CH_ID_BIT_LSB    (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_LSB + _HRT_CSS_RECEIVER_2400_GEN_SHORT_FMT_TYPE_WIDTH)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_DATA_BIT_MSB     (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_LSB - 1)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_MSB (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_CH_ID_BIT_LSB    - 1)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_CH_ID_BIT_MSB    (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_REAL_WIDTH       - 1)
-
-#define _HRT_CSS_RECEIVER_2400_REG_ALIGN 4
-#define _HRT_CSS_RECEIVER_2400_BYTES_PER_PKT             4
-
-#define hrt_css_receiver_2400_4_lane_port_offset  0x100
-#define hrt_css_receiver_2400_1_lane_port_offset  0x200
-#define hrt_css_receiver_2400_2_lane_port_offset  0x300
-#define hrt_css_receiver_2400_backend_port_offset 0x100
-
-#define _HRT_CSS_RECEIVER_2400_DEVICE_READY_REG_IDX      0
-#define _HRT_CSS_RECEIVER_2400_IRQ_STATUS_REG_IDX        1
-#define _HRT_CSS_RECEIVER_2400_IRQ_ENABLE_REG_IDX        2
-#define _HRT_CSS_RECEIVER_2400_CSI2_FUNC_PROG_REG_IDX    3
-#define _HRT_CSS_RECEIVER_2400_INIT_COUNT_REG_IDX        4
-#define _HRT_CSS_RECEIVER_2400_FS_TO_LS_DELAY_REG_IDX    7
-#define _HRT_CSS_RECEIVER_2400_LS_TO_DATA_DELAY_REG_IDX  8
-#define _HRT_CSS_RECEIVER_2400_DATA_TO_LE_DELAY_REG_IDX  9
-#define _HRT_CSS_RECEIVER_2400_LE_TO_FE_DELAY_REG_IDX   10
-#define _HRT_CSS_RECEIVER_2400_FE_TO_FS_DELAY_REG_IDX   11
-#define _HRT_CSS_RECEIVER_2400_LE_TO_LS_DELAY_REG_IDX   12
-#define _HRT_CSS_RECEIVER_2400_TWO_PIXEL_EN_REG_IDX     13
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_REG_IDX  14
-#define _HRT_CSS_RECEIVER_2400_SYNC_COUNT_REG_IDX       15
-#define _HRT_CSS_RECEIVER_2400_RX_COUNT_REG_IDX         16
-#define _HRT_CSS_RECEIVER_2400_BACKEND_RST_REG_IDX      17
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG0_IDX 18
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG1_IDX 19
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG0_IDX 20
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG1_IDX 21
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG0_IDX 22
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG1_IDX 23
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG0_IDX 24
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG1_IDX 25
-#define _HRT_CSS_RECEIVER_2400_RAW18_REG_IDX            26
-#define _HRT_CSS_RECEIVER_2400_FORCE_RAW8_REG_IDX       27
-#define _HRT_CSS_RECEIVER_2400_RAW16_REG_IDX            28
-
-/* Interrupt bits for IRQ_STATUS and IRQ_ENABLE registers */
-#define _HRT_CSS_RECEIVER_2400_IRQ_OVERRUN_BIT                0
-#define _HRT_CSS_RECEIVER_2400_IRQ_RESERVED_BIT               1
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_ENTRY_BIT       2
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_EXIT_BIT        3
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_HS_BIT             4
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_SYNC_HS_BIT        5
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CONTROL_BIT            6
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_DOUBLE_BIT         7
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_CORRECTED_BIT      8
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_NO_CORRECTION_BIT  9
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CRC_BIT               10
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ID_BIT                11
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_SYNC_BIT        12
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_DATA_BIT        13
-#define _HRT_CSS_RECEIVER_2400_IRQ_DATA_TIMEOUT_BIT          14
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ESCAPE_BIT            15
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_LINE_SYNC_BIT         16
-
-#define _HRT_CSS_RECEIVER_2400_IRQ_OVERRUN_CAUSE_                  "Fifo Overrun"
-#define _HRT_CSS_RECEIVER_2400_IRQ_RESERVED_CAUSE_                 "Reserved"
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_ENTRY_CAUSE_         "Sleep mode entry"
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_EXIT_CAUSE_          "Sleep mode exit"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_HS_CAUSE_               "Error high speed SOT"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_SYNC_HS_CAUSE_          "Error high speed sync SOT"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CONTROL_CAUSE_              "Error control"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_DOUBLE_CAUSE_           "Error correction double bit"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_CORRECTED_CAUSE_        "Error correction single bit"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_NO_CORRECTION_CAUSE_    "No error"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CRC_CAUSE_                  "Error cyclic redundancy check"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ID_CAUSE_                   "Error id"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_SYNC_CAUSE_           "Error frame sync"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_DATA_CAUSE_           "Error frame data"
-#define _HRT_CSS_RECEIVER_2400_IRQ_DATA_TIMEOUT_CAUSE_             "Data time-out"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ESCAPE_CAUSE_               "Error escape"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_LINE_SYNC_CAUSE_            "Error line sync"
-
-/* Bits for CSI2_DEVICE_READY register */
-#define _HRT_CSS_RECEIVER_2400_CSI2_DEVICE_READY_IDX                          0
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_INIT_TIME_OUT_ERR_IDX                2
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_OVER_RUN_ERR_IDX                     3
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_SOT_SYNC_ERR_IDX                     4
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_RECEIVE_DATA_TIME_OUT_ERR_IDX        5
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_ECC_TWO_BIT_ERR_IDX                  6
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_DATA_ID_ERR_IDX                      7
-
-                                  
-/* Bits for CSI2_FUNC_PROG register */
-#define _HRT_CSS_RECEIVER_2400_CSI2_DATA_TIMEOUT_IDX    0
-#define _HRT_CSS_RECEIVER_2400_CSI2_DATA_TIMEOUT_BITS   19
-
-/* Bits for INIT_COUNT register */
-#define _HRT_CSS_RECEIVER_2400_INIT_TIMER_IDX  0
-#define _HRT_CSS_RECEIVER_2400_INIT_TIMER_BITS 16
-
-/* Bits for COUNT registers */
-#define _HRT_CSS_RECEIVER_2400_SYNC_COUNT_IDX     0
-#define _HRT_CSS_RECEIVER_2400_SYNC_COUNT_BITS    8
-#define _HRT_CSS_RECEIVER_2400_RX_COUNT_IDX       0
-#define _HRT_CSS_RECEIVER_2400_RX_COUNT_BITS      8
-
-/* Bits for RAW116_18_DATAID register */
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW16_BITS_IDX   0
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW16_BITS_BITS  6
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW18_BITS_IDX   8
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW18_BITS_BITS  6
-
-/* Bits for COMP_FORMAT register, this selects the compression data format */
-#define _HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_IDX  0
-#define _HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_BITS 8
-#define _HRT_CSS_RECEIVER_2400_COMP_NUM_BITS_IDX  (_HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_IDX + _HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_BITS)
-#define _HRT_CSS_RECEIVER_2400_COMP_NUM_BITS_BITS 8
-
-/* Bits for COMP_PREDICT register, this selects the predictor algorithm */
-#define _HRT_CSS_RECEIVER_2400_PREDICT_NO_COMP 0
-#define _HRT_CSS_RECEIVER_2400_PREDICT_1       1
-#define _HRT_CSS_RECEIVER_2400_PREDICT_2       2
-
-/* Number of bits used for the delay registers */
-#define _HRT_CSS_RECEIVER_2400_DELAY_BITS 8
-
-/* Bits for COMP_SCHEME register, this  selects the compression scheme for a VC */
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD1_BITS_IDX  0
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD2_BITS_IDX  5
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD3_BITS_IDX  10
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD4_BITS_IDX  15
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD5_BITS_IDX  20
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD6_BITS_IDX  25
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD7_BITS_IDX  0
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD8_BITS_IDX  5
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_BITS_BITS  5
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_FMT_BITS_IDX   0
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_FMT_BITS_BITS  3
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_PRED_BITS_IDX  3
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_PRED_BITS_BITS 2
-
-
-/* BITS for backend RAW16 and RAW 18 registers */
-
-#define _HRT_CSS_RECEIVER_2400_RAW18_DATAID_IDX    0
-#define _HRT_CSS_RECEIVER_2400_RAW18_DATAID_BITS   6
-#define _HRT_CSS_RECEIVER_2400_RAW18_OPTION_IDX    6
-#define _HRT_CSS_RECEIVER_2400_RAW18_OPTION_BITS   2
-#define _HRT_CSS_RECEIVER_2400_RAW18_EN_IDX        8
-#define _HRT_CSS_RECEIVER_2400_RAW18_EN_BITS       1
-
-#define _HRT_CSS_RECEIVER_2400_RAW16_DATAID_IDX    0
-#define _HRT_CSS_RECEIVER_2400_RAW16_DATAID_BITS   6
-#define _HRT_CSS_RECEIVER_2400_RAW16_OPTION_IDX    6
-#define _HRT_CSS_RECEIVER_2400_RAW16_OPTION_BITS   2
-#define _HRT_CSS_RECEIVER_2400_RAW16_EN_IDX        8
-#define _HRT_CSS_RECEIVER_2400_RAW16_EN_BITS       1
-
-/* These hsync and vsync values are for HSS simulation only */
-#define _HRT_CSS_RECEIVER_2400_HSYNC_VAL (1<<16)
-#define _HRT_CSS_RECEIVER_2400_VSYNC_VAL (1<<17)
-
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_WIDTH                 28
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_LSB              0
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_MSB             (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_LSB + CSS_RECEIVER_DATA_OUT - 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_VAL_BIT         (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_MSB + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_LSB             (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_VAL_BIT + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_MSB             (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_LSB + CSS_RECEIVER_DATA_OUT - 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_VAL_BIT         (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_MSB + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_SOP_BIT               (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_VAL_BIT + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_EOP_BIT               (_HRT_CSS_RECEIVER_2400_BE_STREAMING_SOP_BIT + 1)
-
-// SH Backend Register IDs
-#define _HRT_CSS_RECEIVER_2400_BE_GSP_ACC_OVL_REG_IDX              0
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_REG_IDX                     1
-#define _HRT_CSS_RECEIVER_2400_BE_TWO_PPC_REG_IDX                  2
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG0_IDX             3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG1_IDX             4
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG2_IDX             5
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG3_IDX             6
-#define _HRT_CSS_RECEIVER_2400_BE_SEL_REG_IDX                      7
-#define _HRT_CSS_RECEIVER_2400_BE_RAW16_CONFIG_REG_IDX             8
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_CONFIG_REG_IDX             9
-#define _HRT_CSS_RECEIVER_2400_BE_FORCE_RAW8_REG_IDX              10
-#define _HRT_CSS_RECEIVER_2400_BE_IRQ_STATUS_REG_IDX              11
-#define _HRT_CSS_RECEIVER_2400_BE_IRQ_CLEAR_REG_IDX               12
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_EN_REG_IDX                 13
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_DATA_STATE_REG_IDX         14    /* Data State 0,1,2 config */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P0_REG_IDX       15    /* Pixel Extractor config for Data State 0 & Pix 0 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P1_REG_IDX       16    /* Pixel Extractor config for Data State 0 & Pix 1 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P2_REG_IDX       17    /* Pixel Extractor config for Data State 0 & Pix 2 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P3_REG_IDX       18    /* Pixel Extractor config for Data State 0 & Pix 3 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P0_REG_IDX       19    /* Pixel Extractor config for Data State 1 & Pix 0 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P1_REG_IDX       20    /* Pixel Extractor config for Data State 1 & Pix 1 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P2_REG_IDX       21    /* Pixel Extractor config for Data State 1 & Pix 2 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P3_REG_IDX       22    /* Pixel Extractor config for Data State 1 & Pix 3 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P0_REG_IDX       23    /* Pixel Extractor config for Data State 2 & Pix 0 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P1_REG_IDX       24    /* Pixel Extractor config for Data State 2 & Pix 1 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P2_REG_IDX       25    /* Pixel Extractor config for Data State 2 & Pix 2 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P3_REG_IDX       26    /* Pixel Extractor config for Data State 2 & Pix 3 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_VALID_EOP_REG_IDX      27    /* Pixel Valid & EoP config for Pix 0,1,2,3 */
-
-#define _HRT_CSS_RECEIVER_2400_BE_NOF_REGISTERS                   28
-
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_HE                          0
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_RCF                         1
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_PF                          2
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_SM                          3
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_PD                          4
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_SD                          5
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_OT                          6
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_BC                          7
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_WIDTH                       8
-
-#endif /* _css_receiver_2400_defs_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/defs.h
deleted file mode 100644
index 47505f41790c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/defs.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_DEFS_H_
-#define _HRT_DEFS_H_
-
-#ifndef HRTCAT
-#define _HRTCAT(m, n)     m##n
-#define HRTCAT(m, n)      _HRTCAT(m, n)
-#endif
-
-#ifndef HRTSTR
-#define _HRTSTR(x)   #x
-#define HRTSTR(x)    _HRTSTR(x)
-#endif
-
-#ifndef HRTMIN
-#define HRTMIN(a, b) (((a) < (b)) ? (a) : (b))
-#endif
-
-#ifndef HRTMAX
-#define HRTMAX(a, b) (((a) > (b)) ? (a) : (b))
-#endif
-
-#endif /* _HRT_DEFS_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/dma_v2_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/dma_v2_defs.h
deleted file mode 100644
index d184a8b313c9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/dma_v2_defs.h
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _dma_v2_defs_h
-#define _dma_v2_defs_h
-
-#define _DMA_V2_NUM_CHANNELS_ID               MaxNumChannels
-#define _DMA_V2_CONNECTIONS_ID                Connections
-#define _DMA_V2_DEV_ELEM_WIDTHS_ID            DevElemWidths
-#define _DMA_V2_DEV_FIFO_DEPTH_ID             DevFifoDepth
-#define _DMA_V2_DEV_FIFO_RD_LAT_ID            DevFifoRdLat
-#define _DMA_V2_DEV_FIFO_LAT_BYPASS_ID        DevFifoRdLatBypass
-#define _DMA_V2_DEV_NO_BURST_ID               DevNoBurst
-#define _DMA_V2_DEV_RD_ACCEPT_ID              DevRdAccept
-#define _DMA_V2_DEV_SRMD_ID                   DevSRMD
-#define _DMA_V2_DEV_HAS_CRUN_ID               CRunMasters
-#define _DMA_V2_CTRL_ACK_FIFO_DEPTH_ID        CtrlAckFifoDepth
-#define _DMA_V2_CMD_FIFO_DEPTH_ID             CommandFifoDepth
-#define _DMA_V2_CMD_FIFO_RD_LAT_ID            CommandFifoRdLat
-#define _DMA_V2_CMD_FIFO_LAT_BYPASS_ID        CommandFifoRdLatBypass
-#define _DMA_V2_NO_PACK_ID                    has_no_pack
-
-#define _DMA_V2_REG_ALIGN                4
-#define _DMA_V2_REG_ADDR_BITS            2
-
-/* Command word */
-#define _DMA_V2_CMD_IDX            0
-#define _DMA_V2_CMD_BITS           6
-#define _DMA_V2_CHANNEL_IDX        (_DMA_V2_CMD_IDX + _DMA_V2_CMD_BITS)
-#define _DMA_V2_CHANNEL_BITS       5
-
-/* The command to set a parameter contains the PARAM field next */
-#define _DMA_V2_PARAM_IDX          (_DMA_V2_CHANNEL_IDX + _DMA_V2_CHANNEL_BITS)
-#define _DMA_V2_PARAM_BITS         4
-
-/* Commands to read, write or init specific blocks contain these
-   three values */
-#define _DMA_V2_SPEC_DEV_A_XB_IDX  (_DMA_V2_CHANNEL_IDX + _DMA_V2_CHANNEL_BITS)
-#define _DMA_V2_SPEC_DEV_A_XB_BITS 8
-#define _DMA_V2_SPEC_DEV_B_XB_IDX  (_DMA_V2_SPEC_DEV_A_XB_IDX + _DMA_V2_SPEC_DEV_A_XB_BITS)
-#define _DMA_V2_SPEC_DEV_B_XB_BITS 8
-#define _DMA_V2_SPEC_YB_IDX        (_DMA_V2_SPEC_DEV_B_XB_IDX + _DMA_V2_SPEC_DEV_B_XB_BITS)
-#define _DMA_V2_SPEC_YB_BITS       (32-_DMA_V2_SPEC_DEV_B_XB_BITS-_DMA_V2_SPEC_DEV_A_XB_BITS-_DMA_V2_CMD_BITS-_DMA_V2_CHANNEL_BITS)
-
-/* */
-#define _DMA_V2_CMD_CTRL_IDX       4
-#define _DMA_V2_CMD_CTRL_BITS      4
-
-/* Packing setup word */
-#define _DMA_V2_CONNECTION_IDX     0
-#define _DMA_V2_CONNECTION_BITS    4
-#define _DMA_V2_EXTENSION_IDX      (_DMA_V2_CONNECTION_IDX + _DMA_V2_CONNECTION_BITS)
-#define _DMA_V2_EXTENSION_BITS     1
-
-/* Elements packing word */
-#define _DMA_V2_ELEMENTS_IDX        0
-#define _DMA_V2_ELEMENTS_BITS       8
-#define _DMA_V2_LEFT_CROPPING_IDX  (_DMA_V2_ELEMENTS_IDX + _DMA_V2_ELEMENTS_BITS)
-#define _DMA_V2_LEFT_CROPPING_BITS  8
-
-#define _DMA_V2_WIDTH_IDX           0
-#define _DMA_V2_WIDTH_BITS         16
-
-#define _DMA_V2_HEIGHT_IDX          0
-#define _DMA_V2_HEIGHT_BITS        16
-
-#define _DMA_V2_STRIDE_IDX          0
-#define _DMA_V2_STRIDE_BITS        32
-
-/* Command IDs */
-#define _DMA_V2_MOVE_B2A_COMMAND                             0      
-#define _DMA_V2_MOVE_B2A_BLOCK_COMMAND                       1      
-#define _DMA_V2_MOVE_B2A_NO_SYNC_CHK_COMMAND                 2      
-#define _DMA_V2_MOVE_B2A_BLOCK_NO_SYNC_CHK_COMMAND           3      
-#define _DMA_V2_MOVE_A2B_COMMAND                             4      
-#define _DMA_V2_MOVE_A2B_BLOCK_COMMAND                       5      
-#define _DMA_V2_MOVE_A2B_NO_SYNC_CHK_COMMAND                 6      
-#define _DMA_V2_MOVE_A2B_BLOCK_NO_SYNC_CHK_COMMAND           7      
-#define _DMA_V2_INIT_A_COMMAND                               8      
-#define _DMA_V2_INIT_A_BLOCK_COMMAND                         9      
-#define _DMA_V2_INIT_A_NO_SYNC_CHK_COMMAND                  10      
-#define _DMA_V2_INIT_A_BLOCK_NO_SYNC_CHK_COMMAND            11      
-#define _DMA_V2_INIT_B_COMMAND                              12      
-#define _DMA_V2_INIT_B_BLOCK_COMMAND                        13      
-#define _DMA_V2_INIT_B_NO_SYNC_CHK_COMMAND                  14      
-#define _DMA_V2_INIT_B_BLOCK_NO_SYNC_CHK_COMMAND            15      
-#define _DMA_V2_NO_ACK_MOVE_B2A_NO_SYNC_CHK_COMMAND         (_DMA_V2_MOVE_B2A_NO_SYNC_CHK_COMMAND       + 16) 
-#define _DMA_V2_NO_ACK_MOVE_B2A_BLOCK_NO_SYNC_CHK_COMMAND   (_DMA_V2_MOVE_B2A_BLOCK_NO_SYNC_CHK_COMMAND + 16) 
-#define _DMA_V2_NO_ACK_MOVE_A2B_NO_SYNC_CHK_COMMAND         (_DMA_V2_MOVE_A2B_NO_SYNC_CHK_COMMAND       + 16) 
-#define _DMA_V2_NO_ACK_MOVE_A2B_BLOCK_NO_SYNC_CHK_COMMAND   (_DMA_V2_MOVE_A2B_BLOCK_NO_SYNC_CHK_COMMAND + 16) 
-#define _DMA_V2_NO_ACK_INIT_A_NO_SYNC_CHK_COMMAND           (_DMA_V2_INIT_A_NO_SYNC_CHK_COMMAND         + 16) 
-#define _DMA_V2_NO_ACK_INIT_A_BLOCK_NO_SYNC_CHK_COMMAND     (_DMA_V2_INIT_A_BLOCK_NO_SYNC_CHK_COMMAND   + 16) 
-#define _DMA_V2_NO_ACK_INIT_B_NO_SYNC_CHK_COMMAND           (_DMA_V2_INIT_B_NO_SYNC_CHK_COMMAND         + 16) 
-#define _DMA_V2_NO_ACK_INIT_B_BLOCK_NO_SYNC_CHK_COMMAND     (_DMA_V2_INIT_B_BLOCK_NO_SYNC_CHK_COMMAND   + 16) 
-#define _DMA_V2_CONFIG_CHANNEL_COMMAND                      32   
-#define _DMA_V2_SET_CHANNEL_PARAM_COMMAND                   33   
-#define _DMA_V2_SET_CRUN_COMMAND                            62   
-
-/* Channel Parameter IDs */
-#define _DMA_V2_PACKING_SETUP_PARAM                     0  
-#define _DMA_V2_STRIDE_A_PARAM                          1  
-#define _DMA_V2_ELEM_CROPPING_A_PARAM                   2  
-#define _DMA_V2_WIDTH_A_PARAM                           3  
-#define _DMA_V2_STRIDE_B_PARAM                          4  
-#define _DMA_V2_ELEM_CROPPING_B_PARAM                   5  
-#define _DMA_V2_WIDTH_B_PARAM                           6  
-#define _DMA_V2_HEIGHT_PARAM                            7  
-#define _DMA_V2_QUEUED_CMDS                             8  
-
-/* Parameter Constants */
-#define _DMA_V2_ZERO_EXTEND                             0
-#define _DMA_V2_SIGN_EXTEND                             1
-
-  /* SLAVE address map */
-#define _DMA_V2_SEL_FSM_CMD                             0
-#define _DMA_V2_SEL_CH_REG                              1
-#define _DMA_V2_SEL_CONN_GROUP                          2
-#define _DMA_V2_SEL_DEV_INTERF                          3
-
-#define _DMA_V2_ADDR_SEL_COMP_IDX                      12
-#define _DMA_V2_ADDR_SEL_COMP_BITS                      4
-#define _DMA_V2_ADDR_SEL_CH_REG_IDX                     2
-#define _DMA_V2_ADDR_SEL_CH_REG_BITS                    6
-#define _DMA_V2_ADDR_SEL_PARAM_IDX                      (_DMA_V2_ADDR_SEL_CH_REG_BITS+_DMA_V2_ADDR_SEL_CH_REG_IDX)
-#define _DMA_V2_ADDR_SEL_PARAM_BITS                     4
-
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_IDX                 2
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_BITS                6
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_INFO_IDX            (_DMA_V2_ADDR_SEL_GROUP_COMP_BITS + _DMA_V2_ADDR_SEL_GROUP_COMP_IDX)
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_INFO_BITS           4
-
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_IDX_IDX             2
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_IDX_BITS            6
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_INFO_IDX            (_DMA_V2_ADDR_SEL_DEV_INTERF_IDX_IDX+_DMA_V2_ADDR_SEL_DEV_INTERF_IDX_BITS)
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_INFO_BITS           4
-
-#define _DMA_V2_FSM_GROUP_CMD_IDX                       0
-#define _DMA_V2_FSM_GROUP_ADDR_SRC_IDX                  1
-#define _DMA_V2_FSM_GROUP_ADDR_DEST_IDX                 2
-#define _DMA_V2_FSM_GROUP_CMD_CTRL_IDX                  3
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_IDX                  4
-#define _DMA_V2_FSM_GROUP_FSM_PACK_IDX                  5
-#define _DMA_V2_FSM_GROUP_FSM_REQ_IDX                   6
-#define _DMA_V2_FSM_GROUP_FSM_WR_IDX                    7
-  
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_STATE_IDX            0
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_DEV_IDX          1
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_ADDR_IDX         2
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_STRIDE_IDX       3
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_XB_IDX           4
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_YB_IDX           5
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_REQ_DEV_IDX     6
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_DEV_IDX      7
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_WR_ADDR_IDX          8
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_WR_STRIDE_IDX        9
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_REQ_XB_IDX     10
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_YB_IDX      11
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_XB_IDX      12
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_ELEM_REQ_IDX   13
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_ELEM_WR_IDX    14
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_S_Z_IDX        15
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_CMD_CTRL_IDX        15
-
-#define _DMA_V2_FSM_GROUP_FSM_PACK_STATE_IDX            0
-#define _DMA_V2_FSM_GROUP_FSM_PACK_CNT_YB_IDX           1
-#define _DMA_V2_FSM_GROUP_FSM_PACK_CNT_XB_REQ_IDX       2
-#define _DMA_V2_FSM_GROUP_FSM_PACK_CNT_XB_WR_IDX        3
-
-#define _DMA_V2_FSM_GROUP_FSM_REQ_STATE_IDX             0
-#define _DMA_V2_FSM_GROUP_FSM_REQ_CNT_YB_IDX            1
-#define _DMA_V2_FSM_GROUP_FSM_REQ_CNT_XB_IDX            2
-#define _DMA_V2_FSM_GROUP_FSM_REQ_XB_REMAINING_IDX      3
-#define _DMA_V2_FSM_GROUP_FSM_REQ_CNT_BURST_IDX         4
-
-#define _DMA_V2_FSM_GROUP_FSM_WR_STATE_IDX              0
-#define _DMA_V2_FSM_GROUP_FSM_WR_CNT_YB_IDX             1
-#define _DMA_V2_FSM_GROUP_FSM_WR_CNT_XB_IDX             2
-#define _DMA_V2_FSM_GROUP_FSM_WR_XB_REMAINING_IDX       3
-#define _DMA_V2_FSM_GROUP_FSM_WR_CNT_BURST_IDX          4
-
-#define _DMA_V2_DEV_INTERF_REQ_SIDE_STATUS_IDX          0
-#define _DMA_V2_DEV_INTERF_SEND_SIDE_STATUS_IDX         1
-#define _DMA_V2_DEV_INTERF_FIFO_STATUS_IDX              2
-#define _DMA_V2_DEV_INTERF_REQ_ONLY_COMPLETE_BURST_IDX  3
-#define _DMA_V2_DEV_INTERF_MAX_BURST_IDX                4
-#define _DMA_V2_DEV_INTERF_CHK_ADDR_ALIGN               5
-
-#endif /* _dma_v2_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gdc_v2_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gdc_v2_defs.h
deleted file mode 100644
index 77722d205701..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gdc_v2_defs.h
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef HRT_GDC_v2_defs_h_
-#define HRT_GDC_v2_defs_h_
-
-#define HRT_GDC_IS_V2
-
-#define HRT_GDC_N                     1024 /* Top-level design constant, equal to the number of entries in the LUT      */
-#define HRT_GDC_FRAC_BITS               10 /* Number of fractional bits in the GDC block, driven by the size of the LUT */
-
-#define HRT_GDC_BLI_FRAC_BITS            4 /* Number of fractional bits for the bi-linear interpolation type            */
-#define HRT_GDC_BLI_COEF_ONE             (1 << HRT_GDC_BLI_FRAC_BITS)
-
-#define HRT_GDC_BCI_COEF_BITS           14 /* 14 bits per coefficient                                                   */
-#define HRT_GDC_BCI_COEF_ONE             (1 << (HRT_GDC_BCI_COEF_BITS-2))  /* We represent signed 10 bit coefficients.  */
-                                                                        /* The supported range is [-256, .., +256]      */
-                                                                        /* in 14-bit signed notation,                   */
-                                                                        /* We need all ten bits (MSB must be zero).     */
-                                                                        /* -s is inserted to solve this issue, and      */
-                                                                        /* therefore "1" is equal to +256.              */
-#define HRT_GDC_BCI_COEF_MASK            ((1 << HRT_GDC_BCI_COEF_BITS) - 1) 
-
-#define HRT_GDC_LUT_BYTES                (HRT_GDC_N*4*2)                /* 1024 addresses, 4 coefficients per address,  */
-                                                                        /* 2 bytes per coefficient                      */
-
-#define _HRT_GDC_REG_ALIGN               4                              
-
-  //     31  30  29    25 24                     0
-  //  |-----|---|--------|------------------------|
-  //  | CMD | C | Reg_ID |        Value           |
-
-
-  // There are just two commands possible for the GDC block:
-  // 1 - Configure reg 
-  // 0 - Data token    
-  
-  // C      - Reserved bit
-  //          Used in protocol to indicate whether it is C-run or other type of runs
-  //          In case of C-run, this bit has a value of 1, for all the other runs, it is 0.
-
-  // Reg_ID - Address of the register to be configured
-  
-  // Value  - Value to store to the addressed register, maximum of 24 bits
-
-  // Configure reg command is not followed by any other token. 
-  // The address of the register and the data to be filled in is contained in the same token 
-  
-  // When the first data token is received, it must be:
-  //   1. FRX and FRY (device configured in one of the  scaling modes) ***DEFAULT MODE***, or,
-  //   2. P0'X        (device configured in one of the tetragon modes)
-  // After the first data token is received, pre-defined number of tokens with the following meaning follow:
-  //   1. two  tokens: SRC address ; DST address
-  //   2. nine tokens: P0'Y, .., P3'Y ; SRC address ; DST address
-  
-#define HRT_GDC_CONFIG_CMD             1
-#define HRT_GDC_DATA_CMD               0
-
-
-#define HRT_GDC_CMD_POS               31
-#define HRT_GDC_CMD_BITS               1
-#define HRT_GDC_CRUN_POS              30
-#define HRT_GDC_REG_ID_POS            25
-#define HRT_GDC_REG_ID_BITS            5
-#define HRT_GDC_DATA_POS               0
-#define HRT_GDC_DATA_BITS             25
-
-#define HRT_GDC_FRYIPXFRX_BITS        26
-#define HRT_GDC_P0X_BITS              23
-
-
-#define HRT_GDC_MAX_OXDIM           (8192-64)
-#define HRT_GDC_MAX_OYDIM           4095
-#define HRT_GDC_MAX_IXDIM           (8192-64)
-#define HRT_GDC_MAX_IYDIM           4095
-#define HRT_GDC_MAX_DS_FAC            16
-#define HRT_GDC_MAX_DX                 (HRT_GDC_MAX_DS_FAC*HRT_GDC_N - 1)
-#define HRT_GDC_MAX_DY                 HRT_GDC_MAX_DX
-
-
-/* GDC lookup tables entries are 10 bits values, but they're
-   stored 2 by 2 as 32 bit values, yielding 16 bits per entry.
-   A GDC lookup table contains 64 * 4 elements */
-
-#define HRT_GDC_PERF_1_1_pix          0
-#define HRT_GDC_PERF_2_1_pix          1
-#define HRT_GDC_PERF_1_2_pix          2
-#define HRT_GDC_PERF_2_2_pix          3
-
-#define HRT_GDC_NND_MODE              0
-#define HRT_GDC_BLI_MODE              1
-#define HRT_GDC_BCI_MODE              2
-#define HRT_GDC_LUT_MODE              3
-
-#define HRT_GDC_SCAN_STB              0
-#define HRT_GDC_SCAN_STR              1
-
-#define HRT_GDC_MODE_SCALING          0
-#define HRT_GDC_MODE_TETRAGON         1
-
-#define HRT_GDC_LUT_COEFF_OFFSET     16 
-#define HRT_GDC_FRY_BIT_OFFSET       16 
-// FRYIPXFRX is the only register where we store two values in one field, 
-// to save one token in the scaling protocol. 
-// Like this, we have three tokens in the scaling protocol, 
-// Otherwise, we would have had four.
-// The register bit-map is:
-//   31  26 25      16 15  10 9        0
-//  |------|----------|------|----------|
-//  | XXXX |   FRY    |  IPX |   FRX    |
-
-
-#define HRT_GDC_CE_FSM0_POS           0
-#define HRT_GDC_CE_FSM0_LEN           2
-#define HRT_GDC_CE_OPY_POS            2
-#define HRT_GDC_CE_OPY_LEN           14
-#define HRT_GDC_CE_OPX_POS           16
-#define HRT_GDC_CE_OPX_LEN           16
-// CHK_ENGINE register bit-map:
-//   31            16 15        2 1  0
-//  |----------------|-----------|----|
-//  |      OPX       |    OPY    |FSM0|
-// However, for the time being at least, 
-// this implementation is meaningless in hss model,
-// So, we just return 0
-
-
-#define HRT_GDC_CHK_ENGINE_IDX        0
-#define HRT_GDC_WOIX_IDX              1
-#define HRT_GDC_WOIY_IDX              2
-#define HRT_GDC_BPP_IDX               3
-#define HRT_GDC_FRYIPXFRX_IDX         4
-#define HRT_GDC_OXDIM_IDX             5
-#define HRT_GDC_OYDIM_IDX             6
-#define HRT_GDC_SRC_ADDR_IDX          7
-#define HRT_GDC_SRC_END_ADDR_IDX      8
-#define HRT_GDC_SRC_WRAP_ADDR_IDX     9
-#define HRT_GDC_SRC_STRIDE_IDX       10
-#define HRT_GDC_DST_ADDR_IDX         11
-#define HRT_GDC_DST_STRIDE_IDX       12
-#define HRT_GDC_DX_IDX               13
-#define HRT_GDC_DY_IDX               14
-#define HRT_GDC_P0X_IDX              15
-#define HRT_GDC_P0Y_IDX              16
-#define HRT_GDC_P1X_IDX              17
-#define HRT_GDC_P1Y_IDX              18
-#define HRT_GDC_P2X_IDX              19
-#define HRT_GDC_P2Y_IDX              20
-#define HRT_GDC_P3X_IDX              21
-#define HRT_GDC_P3Y_IDX              22
-#define HRT_GDC_PERF_POINT_IDX       23  // 1x1 ; 1x2 ; 2x1 ; 2x2 pixels per cc
-#define HRT_GDC_INTERP_TYPE_IDX      24  // NND ; BLI ; BCI ; LUT
-#define HRT_GDC_SCAN_IDX             25  // 0 = STB (Slide To Bottom) ; 1 = STR (Slide To Right)
-#define HRT_GDC_PROC_MODE_IDX        26  // 0 = Scaling ; 1 = Tetragon
-
-#define HRT_GDC_LUT_IDX              32
-
-
-#endif /* HRT_GDC_v2_defs_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gp_timer_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gp_timer_defs.h
deleted file mode 100644
index 3082e2f5e014..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gp_timer_defs.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _gp_timer_defs_h
-#define _gp_timer_defs_h
-
-#define _HRT_GP_TIMER_REG_ALIGN 4
-
-#define HIVE_GP_TIMER_RESET_REG_IDX                              0
-#define HIVE_GP_TIMER_OVERALL_ENABLE_REG_IDX                     1
-#define HIVE_GP_TIMER_ENABLE_REG_IDX(timer)                     (HIVE_GP_TIMER_OVERALL_ENABLE_REG_IDX + 1 + timer)
-#define HIVE_GP_TIMER_VALUE_REG_IDX(timer,timers)               (HIVE_GP_TIMER_ENABLE_REG_IDX(timers) + timer)
-#define HIVE_GP_TIMER_COUNT_TYPE_REG_IDX(timer,timers)          (HIVE_GP_TIMER_VALUE_REG_IDX(timers, timers) + timer)
-#define HIVE_GP_TIMER_SIGNAL_SELECT_REG_IDX(timer,timers)       (HIVE_GP_TIMER_COUNT_TYPE_REG_IDX(timers, timers) + timer)
-#define HIVE_GP_TIMER_IRQ_TRIGGER_VALUE_REG_IDX(irq,timers)     (HIVE_GP_TIMER_SIGNAL_SELECT_REG_IDX(timers, timers) + irq)
-#define HIVE_GP_TIMER_IRQ_TIMER_SELECT_REG_IDX(irq,timers,irqs) (HIVE_GP_TIMER_IRQ_TRIGGER_VALUE_REG_IDX(irqs, timers) + irq)
-#define HIVE_GP_TIMER_IRQ_ENABLE_REG_IDX(irq,timers,irqs)       (HIVE_GP_TIMER_IRQ_TIMER_SELECT_REG_IDX(irqs, timers, irqs) + irq)
-
-#define HIVE_GP_TIMER_COUNT_TYPE_HIGH                            0
-#define HIVE_GP_TIMER_COUNT_TYPE_LOW                             1
-#define HIVE_GP_TIMER_COUNT_TYPE_POSEDGE                         2
-#define HIVE_GP_TIMER_COUNT_TYPE_NEGEDGE                         3
-#define HIVE_GP_TIMER_COUNT_TYPES                                4
-
-#endif /* _gp_timer_defs_h */   
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gpio_block_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gpio_block_defs.h
deleted file mode 100644
index a807d4c99041..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/gpio_block_defs.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _gpio_block_defs_h_
-#define _gpio_block_defs_h_
-
-#define _HRT_GPIO_BLOCK_REG_ALIGN 4
-
-/* R/W registers */
-#define _gpio_block_reg_do_e			         0
-#define _gpio_block_reg_do_select		       1
-#define _gpio_block_reg_do_0			         2
-#define _gpio_block_reg_do_1			         3
-#define _gpio_block_reg_do_pwm_cnt_0	     4
-#define _gpio_block_reg_do_pwm_cnt_1	     5
-#define _gpio_block_reg_do_pwm_cnt_2	     6
-#define _gpio_block_reg_do_pwm_cnt_3	     7
-#define _gpio_block_reg_do_pwm_main_cnt    8
-#define _gpio_block_reg_do_pwm_enable      9
-#define _gpio_block_reg_di_debounce_sel	  10
-#define _gpio_block_reg_di_debounce_cnt_0	11
-#define _gpio_block_reg_di_debounce_cnt_1	12
-#define _gpio_block_reg_di_debounce_cnt_2	13
-#define _gpio_block_reg_di_debounce_cnt_3	14
-#define _gpio_block_reg_di_active_level	  15
-
-
-/* read-only registers */
-#define _gpio_block_reg_di			          16
-
-#endif /* _gpio_block_defs_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_2401_irq_types_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_2401_irq_types_hrt.h
deleted file mode 100644
index 2f7cb2dff0e9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_2401_irq_types_hrt.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_
-#define _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_
-
-/*
- * These are the indices of each interrupt in the interrupt
- * controller's registers. these can be used as the irq_id
- * argument to the hrt functions irq_controller.h.
- *
- * The definitions are taken from <system>_defs.h
- */
-typedef enum hrt_isp_css_irq {
-  hrt_isp_css_irq_gpio_pin_0           = HIVE_GP_DEV_IRQ_GPIO_PIN_0_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_1           = HIVE_GP_DEV_IRQ_GPIO_PIN_1_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_2           = HIVE_GP_DEV_IRQ_GPIO_PIN_2_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_3           = HIVE_GP_DEV_IRQ_GPIO_PIN_3_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_4           = HIVE_GP_DEV_IRQ_GPIO_PIN_4_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_5           = HIVE_GP_DEV_IRQ_GPIO_PIN_5_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_6           = HIVE_GP_DEV_IRQ_GPIO_PIN_6_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_7           = HIVE_GP_DEV_IRQ_GPIO_PIN_7_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_8           = HIVE_GP_DEV_IRQ_GPIO_PIN_8_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_9           = HIVE_GP_DEV_IRQ_GPIO_PIN_9_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_10          = HIVE_GP_DEV_IRQ_GPIO_PIN_10_BIT_ID         ,              
-  hrt_isp_css_irq_gpio_pin_11          = HIVE_GP_DEV_IRQ_GPIO_PIN_11_BIT_ID         ,              
-  hrt_isp_css_irq_sp                   = HIVE_GP_DEV_IRQ_SP_BIT_ID                  ,                       
-  hrt_isp_css_irq_isp                  = HIVE_GP_DEV_IRQ_ISP_BIT_ID                 ,                      
-  hrt_isp_css_irq_isys                 = HIVE_GP_DEV_IRQ_ISYS_BIT_ID                ,                     
-  hrt_isp_css_irq_isel                 = HIVE_GP_DEV_IRQ_ISEL_BIT_ID                ,                     
-  hrt_isp_css_irq_ifmt                 = HIVE_GP_DEV_IRQ_IFMT_BIT_ID                ,                     
-  hrt_isp_css_irq_sp_stream_mon        = HIVE_GP_DEV_IRQ_SP_STREAM_MON_BIT_ID       ,            
-  hrt_isp_css_irq_isp_stream_mon       = HIVE_GP_DEV_IRQ_ISP_STREAM_MON_BIT_ID      ,           
-  hrt_isp_css_irq_mod_stream_mon       = HIVE_GP_DEV_IRQ_MOD_STREAM_MON_BIT_ID      ,
-  hrt_isp_css_irq_is2401               = HIVE_GP_DEV_IRQ_IS2401_BIT_ID              ,           
-  hrt_isp_css_irq_isp_bamem_error      = HIVE_GP_DEV_IRQ_ISP_BAMEM_ERROR_BIT_ID     ,          
-  hrt_isp_css_irq_isp_dmem_error       = HIVE_GP_DEV_IRQ_ISP_DMEM_ERROR_BIT_ID      ,           
-  hrt_isp_css_irq_sp_icache_mem_error  = HIVE_GP_DEV_IRQ_SP_ICACHE_MEM_ERROR_BIT_ID ,      
-  hrt_isp_css_irq_sp_dmem_error        = HIVE_GP_DEV_IRQ_SP_DMEM_ERROR_BIT_ID       ,            
-  hrt_isp_css_irq_mmu_cache_mem_error  = HIVE_GP_DEV_IRQ_MMU_CACHE_MEM_ERROR_BIT_ID ,      
-  hrt_isp_css_irq_gp_timer_0           = HIVE_GP_DEV_IRQ_GP_TIMER_0_BIT_ID          ,               
-  hrt_isp_css_irq_gp_timer_1           = HIVE_GP_DEV_IRQ_GP_TIMER_1_BIT_ID          ,               
-  hrt_isp_css_irq_sw_pin_0             = HIVE_GP_DEV_IRQ_SW_PIN_0_BIT_ID            ,                 
-  hrt_isp_css_irq_sw_pin_1             = HIVE_GP_DEV_IRQ_SW_PIN_1_BIT_ID            ,                 
-  hrt_isp_css_irq_dma                  = HIVE_GP_DEV_IRQ_DMA_BIT_ID                 ,
-  hrt_isp_css_irq_sp_stream_mon_b      = HIVE_GP_DEV_IRQ_SP_STREAM_MON_B_BIT_ID     ,
-  /* this must (obviously) be the last on in the enum */
-  hrt_isp_css_irq_num_irqs
-} hrt_isp_css_irq_t;
-
-typedef enum hrt_isp_css_irq_status {
-  hrt_isp_css_irq_status_error,
-  hrt_isp_css_irq_status_more_irqs,
-  hrt_isp_css_irq_status_success
-} hrt_isp_css_irq_status_t;
-
-#endif /* _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_defs.h
deleted file mode 100644
index 5a2ce9108ae4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_defs.h
+++ /dev/null
@@ -1,435 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _hive_isp_css_defs_h__
-#define _hive_isp_css_defs_h__
-
-#define _HIVE_ISP_CSS_2401_SYSTEM     1
-#define HIVE_ISP_CTRL_DATA_WIDTH     32
-#define HIVE_ISP_CTRL_ADDRESS_WIDTH  32
-#define HIVE_ISP_CTRL_MAX_BURST_SIZE  1
-#define HIVE_ISP_DDR_ADDRESS_WIDTH   36
-
-#define HIVE_ISP_HOST_MAX_BURST_SIZE  8 /* host supports bursts in order to prevent repeating DDRAM accesses */
-#define HIVE_ISP_NUM_GPIO_PINS       12
-
-/* This list of vector num_elems/elem_bits pairs is valid both in C as initializer
-   and in the DMA parameter list */
-#define HIVE_ISP_DDR_DMA_SPECS {{32,  8}, {16, 16}, {18, 14}, {25, 10}, {21, 12}}
-#define HIVE_ISP_DDR_WORD_BITS 256
-#define HIVE_ISP_DDR_WORD_BYTES  (HIVE_ISP_DDR_WORD_BITS/8)
-#define HIVE_ISP_DDR_BYTES       (512 * 1024 * 1024)
-#define HIVE_ISP_DDR_BYTES_RTL   (127 * 1024 * 1024)
-#define HIVE_ISP_DDR_SMALL_BYTES (128 * 256 / 8)
-#define HIVE_ISP_PAGE_SHIFT    12
-#define HIVE_ISP_PAGE_SIZE     (1<<HIVE_ISP_PAGE_SHIFT)
-
-#define CSS_DDR_WORD_BITS        HIVE_ISP_DDR_WORD_BITS
-#define CSS_DDR_WORD_BYTES       HIVE_ISP_DDR_WORD_BYTES
-
-/* settings used in applications */
-#define HIVE_XMEM_WIDTH            HIVE_ISP_DDR_WORD_BITS
-#define HIVE_VMEM_VECTOR_ELEMENTS  64
-#define HIVE_VMEM_ELEMENT_BITS     14
-#define HIVE_XMEM_ELEMENT_BITS     16
-#define HIVE_VMEM_VECTOR_BYTES (HIVE_VMEM_VECTOR_ELEMENTS*HIVE_XMEM_ELEMENT_BITS/8) /* used for # addr bytes for one vector */
-#define HIVE_XMEM_PACKED_WORD_VMEM_ELEMENTS (HIVE_XMEM_WIDTH/HIVE_VMEM_ELEMENT_BITS)
-#define HIVE_XMEM_WORD_VMEM_ELEMENTS        (HIVE_XMEM_WIDTH/HIVE_XMEM_ELEMENT_BITS)
-#define XMEM_INT_SIZE              4
-
-
-
-#define HIVE_ISYS_INP_BUFFER_BYTES (64*1024)  /* 64 kByte = 2k words (of 256 bits) */
-
-/* If HIVE_ISP_DDR_BASE_OFFSET is set to a non-zero value, the wide bus just before the DDRAM gets an extra dummy port where         */
-/* address range 0 .. HIVE_ISP_DDR_BASE_OFFSET-1 maps onto. This effectively creates an offset for the DDRAM from system perspective */
-#define HIVE_ISP_DDR_BASE_OFFSET 0x120000000 /* 0x200000 */
-
-#define HIVE_DMA_ISP_BUS_CONN 0
-#define HIVE_DMA_ISP_DDR_CONN 1
-#define HIVE_DMA_BUS_DDR_CONN 2
-#define HIVE_DMA_ISP_MASTER master_port0
-#define HIVE_DMA_BUS_MASTER master_port1
-#define HIVE_DMA_DDR_MASTER master_port2
-
-#define HIVE_DMA_NUM_CHANNELS       32 /* old value was  8 */
-#define HIVE_DMA_CMD_FIFO_DEPTH     24 /* old value was 12 */
-
-#define HIVE_IF_PIXEL_WIDTH 12
-
-#define HIVE_MMU_TLB_SETS           8
-#define HIVE_MMU_TLB_SET_BLOCKS     8
-#define HIVE_MMU_TLB_BLOCK_ELEMENTS 8
-#define HIVE_MMU_PAGE_TABLE_LEVELS  2
-#define HIVE_MMU_PAGE_BYTES         HIVE_ISP_PAGE_SIZE
-
-#define HIVE_ISP_CH_ID_BITS    2
-#define HIVE_ISP_FMT_TYPE_BITS 5
-#define HIVE_ISP_ISEL_SEL_BITS 2
-
-#define HIVE_GP_REGS_SDRAM_WAKEUP_IDX                           0
-#define HIVE_GP_REGS_IDLE_IDX                                   1
-#define HIVE_GP_REGS_IRQ_0_IDX                                  2
-#define HIVE_GP_REGS_IRQ_1_IDX                                  3
-#define HIVE_GP_REGS_SP_STREAM_STAT_IDX                         4
-#define HIVE_GP_REGS_SP_STREAM_STAT_B_IDX                       5
-#define HIVE_GP_REGS_ISP_STREAM_STAT_IDX                        6
-#define HIVE_GP_REGS_MOD_STREAM_STAT_IDX                        7
-#define HIVE_GP_REGS_SP_STREAM_STAT_IRQ_COND_IDX                8
-#define HIVE_GP_REGS_SP_STREAM_STAT_B_IRQ_COND_IDX              9
-#define HIVE_GP_REGS_ISP_STREAM_STAT_IRQ_COND_IDX              10
-#define HIVE_GP_REGS_MOD_STREAM_STAT_IRQ_COND_IDX              11
-#define HIVE_GP_REGS_SP_STREAM_STAT_IRQ_ENABLE_IDX             12
-#define HIVE_GP_REGS_SP_STREAM_STAT_B_IRQ_ENABLE_IDX           13
-#define HIVE_GP_REGS_ISP_STREAM_STAT_IRQ_ENABLE_IDX            14
-#define HIVE_GP_REGS_MOD_STREAM_STAT_IRQ_ENABLE_IDX            15
-#define HIVE_GP_REGS_SWITCH_PRIM_IF_IDX                        16
-#define HIVE_GP_REGS_SWITCH_GDC1_IDX                           17
-#define HIVE_GP_REGS_SWITCH_GDC2_IDX                           18
-#define HIVE_GP_REGS_SRST_IDX                                  19
-#define HIVE_GP_REGS_SLV_REG_SRST_IDX                          20
-#define HIVE_GP_REGS_SWITCH_ISYS_IDX                           21
-
-/* Bit numbers of the soft reset register */
-#define HIVE_GP_REGS_SRST_ISYS_CBUS                             0
-#define HIVE_GP_REGS_SRST_ISEL_CBUS                             1
-#define HIVE_GP_REGS_SRST_IFMT_CBUS                             2
-#define HIVE_GP_REGS_SRST_GPDEV_CBUS                            3
-#define HIVE_GP_REGS_SRST_GPIO                                  4
-#define HIVE_GP_REGS_SRST_TC                                    5
-#define HIVE_GP_REGS_SRST_GPTIMER                               6
-#define HIVE_GP_REGS_SRST_FACELLFIFOS                           7
-#define HIVE_GP_REGS_SRST_D_OSYS                                8
-#define HIVE_GP_REGS_SRST_IFT_SEC_PIPE                          9
-#define HIVE_GP_REGS_SRST_GDC1                                 10
-#define HIVE_GP_REGS_SRST_GDC2                                 11
-#define HIVE_GP_REGS_SRST_VEC_BUS                              12
-#define HIVE_GP_REGS_SRST_ISP                                  13
-#define HIVE_GP_REGS_SRST_SLV_GRP_BUS                          14
-#define HIVE_GP_REGS_SRST_DMA                                  15
-#define HIVE_GP_REGS_SRST_SF_ISP_SP                            16
-#define HIVE_GP_REGS_SRST_SF_PIF_CELLS                         17
-#define HIVE_GP_REGS_SRST_SF_SIF_SP                            18
-#define HIVE_GP_REGS_SRST_SF_MC_SP                             19
-#define HIVE_GP_REGS_SRST_SF_ISYS_SP                           20
-#define HIVE_GP_REGS_SRST_SF_DMA_CELLS                         21
-#define HIVE_GP_REGS_SRST_SF_GDC1_CELLS                        22
-#define HIVE_GP_REGS_SRST_SF_GDC2_CELLS                        23
-#define HIVE_GP_REGS_SRST_SP                                   24
-#define HIVE_GP_REGS_SRST_OCP2CIO                              25
-#define HIVE_GP_REGS_SRST_NBUS                                 26
-#define HIVE_GP_REGS_SRST_HOST12BUS                            27
-#define HIVE_GP_REGS_SRST_WBUS                                 28
-#define HIVE_GP_REGS_SRST_IC_OSYS                              29
-#define HIVE_GP_REGS_SRST_WBUS_IC                              30
-#define HIVE_GP_REGS_SRST_ISYS_INP_BUF_BUS                     31
-
-/* Bit numbers of the slave register soft reset register */
-#define HIVE_GP_REGS_SLV_REG_SRST_DMA                           0
-#define HIVE_GP_REGS_SLV_REG_SRST_GDC1                          1
-#define HIVE_GP_REGS_SLV_REG_SRST_GDC2                          2
-
-/* order of the input bits for the irq controller */
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_0_BIT_ID                       0
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_1_BIT_ID                       1
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_2_BIT_ID                       2
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_3_BIT_ID                       3
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_4_BIT_ID                       4
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_5_BIT_ID                       5
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_6_BIT_ID                       6
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_7_BIT_ID                       7
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_8_BIT_ID                       8
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_9_BIT_ID                       9
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_10_BIT_ID                     10
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_11_BIT_ID                     11
-#define HIVE_GP_DEV_IRQ_SP_BIT_ID                              12
-#define HIVE_GP_DEV_IRQ_ISP_BIT_ID                             13
-#define HIVE_GP_DEV_IRQ_ISYS_BIT_ID                            14
-#define HIVE_GP_DEV_IRQ_ISEL_BIT_ID                            15
-#define HIVE_GP_DEV_IRQ_IFMT_BIT_ID                            16
-#define HIVE_GP_DEV_IRQ_SP_STREAM_MON_BIT_ID                   17
-#define HIVE_GP_DEV_IRQ_ISP_STREAM_MON_BIT_ID                  18
-#define HIVE_GP_DEV_IRQ_MOD_STREAM_MON_BIT_ID                  19
-#define HIVE_GP_DEV_IRQ_IS2401_BIT_ID                          20
-#define HIVE_GP_DEV_IRQ_ISP_BAMEM_ERROR_BIT_ID                 21
-#define HIVE_GP_DEV_IRQ_ISP_DMEM_ERROR_BIT_ID                  22
-#define HIVE_GP_DEV_IRQ_SP_ICACHE_MEM_ERROR_BIT_ID             23
-#define HIVE_GP_DEV_IRQ_SP_DMEM_ERROR_BIT_ID                   24
-#define HIVE_GP_DEV_IRQ_MMU_CACHE_MEM_ERROR_BIT_ID             25
-#define HIVE_GP_DEV_IRQ_GP_TIMER_0_BIT_ID                      26
-#define HIVE_GP_DEV_IRQ_GP_TIMER_1_BIT_ID                      27
-#define HIVE_GP_DEV_IRQ_SW_PIN_0_BIT_ID                        28
-#define HIVE_GP_DEV_IRQ_SW_PIN_1_BIT_ID                        29
-#define HIVE_GP_DEV_IRQ_DMA_BIT_ID                             30
-#define HIVE_GP_DEV_IRQ_SP_STREAM_MON_B_BIT_ID                 31
-
-#define HIVE_GP_REGS_NUM_SW_IRQ_REGS                            2
-
-/* order of the input bits for the timed controller */
-#define HIVE_GP_DEV_TC_GPIO_PIN_0_BIT_ID                       0
-#define HIVE_GP_DEV_TC_GPIO_PIN_1_BIT_ID                       1
-#define HIVE_GP_DEV_TC_GPIO_PIN_2_BIT_ID                       2
-#define HIVE_GP_DEV_TC_GPIO_PIN_3_BIT_ID                       3
-#define HIVE_GP_DEV_TC_GPIO_PIN_4_BIT_ID                       4
-#define HIVE_GP_DEV_TC_GPIO_PIN_5_BIT_ID                       5
-#define HIVE_GP_DEV_TC_GPIO_PIN_6_BIT_ID                       6
-#define HIVE_GP_DEV_TC_GPIO_PIN_7_BIT_ID                       7
-#define HIVE_GP_DEV_TC_GPIO_PIN_8_BIT_ID                       8
-#define HIVE_GP_DEV_TC_GPIO_PIN_9_BIT_ID                       9
-#define HIVE_GP_DEV_TC_GPIO_PIN_10_BIT_ID                     10
-#define HIVE_GP_DEV_TC_GPIO_PIN_11_BIT_ID                     11
-#define HIVE_GP_DEV_TC_SP_BIT_ID                              12
-#define HIVE_GP_DEV_TC_ISP_BIT_ID                             13
-#define HIVE_GP_DEV_TC_ISYS_BIT_ID                            14
-#define HIVE_GP_DEV_TC_ISEL_BIT_ID                            15
-#define HIVE_GP_DEV_TC_IFMT_BIT_ID                            16
-#define HIVE_GP_DEV_TC_GP_TIMER_0_BIT_ID                      17
-#define HIVE_GP_DEV_TC_GP_TIMER_1_BIT_ID                      18
-#define HIVE_GP_DEV_TC_MIPI_SOL_BIT_ID                        19
-#define HIVE_GP_DEV_TC_MIPI_EOL_BIT_ID                        20
-#define HIVE_GP_DEV_TC_MIPI_SOF_BIT_ID                        21
-#define HIVE_GP_DEV_TC_MIPI_EOF_BIT_ID                        22
-#define HIVE_GP_DEV_TC_INPSYS_SM                              23
-
-/* definitions for the gp_timer block */
-#define HIVE_GP_TIMER_0                                         0
-#define HIVE_GP_TIMER_1                                         1
-#define HIVE_GP_TIMER_2                                         2
-#define HIVE_GP_TIMER_3                                         3
-#define HIVE_GP_TIMER_4                                         4
-#define HIVE_GP_TIMER_5                                         5
-#define HIVE_GP_TIMER_6                                         6
-#define HIVE_GP_TIMER_7                                         7
-#define HIVE_GP_TIMER_NUM_COUNTERS                              8
-
-#define HIVE_GP_TIMER_IRQ_0                                     0
-#define HIVE_GP_TIMER_IRQ_1                                     1
-#define HIVE_GP_TIMER_NUM_IRQS                                  2
-
-#define HIVE_GP_TIMER_GPIO_0_BIT_ID                             0
-#define HIVE_GP_TIMER_GPIO_1_BIT_ID                             1
-#define HIVE_GP_TIMER_GPIO_2_BIT_ID                             2
-#define HIVE_GP_TIMER_GPIO_3_BIT_ID                             3
-#define HIVE_GP_TIMER_GPIO_4_BIT_ID                             4
-#define HIVE_GP_TIMER_GPIO_5_BIT_ID                             5
-#define HIVE_GP_TIMER_GPIO_6_BIT_ID                             6
-#define HIVE_GP_TIMER_GPIO_7_BIT_ID                             7
-#define HIVE_GP_TIMER_GPIO_8_BIT_ID                             8
-#define HIVE_GP_TIMER_GPIO_9_BIT_ID                             9
-#define HIVE_GP_TIMER_GPIO_10_BIT_ID                           10
-#define HIVE_GP_TIMER_GPIO_11_BIT_ID                           11
-#define HIVE_GP_TIMER_INP_SYS_IRQ                              12
-#define HIVE_GP_TIMER_ISEL_IRQ                                 13
-#define HIVE_GP_TIMER_IFMT_IRQ                                 14
-#define HIVE_GP_TIMER_SP_STRMON_IRQ                            15
-#define HIVE_GP_TIMER_SP_B_STRMON_IRQ                          16
-#define HIVE_GP_TIMER_ISP_STRMON_IRQ                           17
-#define HIVE_GP_TIMER_MOD_STRMON_IRQ                           18
-#define HIVE_GP_TIMER_IS2401_IRQ                               19
-#define HIVE_GP_TIMER_ISP_BAMEM_ERROR_IRQ                      20
-#define HIVE_GP_TIMER_ISP_DMEM_ERROR_IRQ                       21
-#define HIVE_GP_TIMER_SP_ICACHE_MEM_ERROR_IRQ                  22
-#define HIVE_GP_TIMER_SP_DMEM_ERROR_IRQ                        23
-#define HIVE_GP_TIMER_SP_OUT_RUN_DP                            24
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I0         25
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I1         26
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I2         27
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I3         28
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I4         29
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I5         30
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I6         31
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I7         32
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I8         33
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I9         34
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I10        35
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I1_I0         36
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I2_I0         37
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I3_I0         38
-#define HIVE_GP_TIMER_ISP_OUT_RUN_DP                           39
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I0_I0        40
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I0_I1        41
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I1_I0        42
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I0        43
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I1        44
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I2        45
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I3        46
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I4        47
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I5        48
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I6        49
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I3_I0        50
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I4_I0        51
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I5_I0        52
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I6_I0        53
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I7_I0        54                                                         
-#define HIVE_GP_TIMER_MIPI_SOL_BIT_ID                          55
-#define HIVE_GP_TIMER_MIPI_EOL_BIT_ID                          56
-#define HIVE_GP_TIMER_MIPI_SOF_BIT_ID                          57
-#define HIVE_GP_TIMER_MIPI_EOF_BIT_ID                          58
-#define HIVE_GP_TIMER_INPSYS_SM                                59
-#define HIVE_GP_TIMER_ISP_PMEM_ERROR_IRQ                       60
-
-/* port definitions for the streaming monitors */
-/* port definititions SP streaming monitor, monitors the status of streaming ports at the SP side of the streaming FIFO's */
-#define SP_STR_MON_PORT_SP2SIF            0
-#define SP_STR_MON_PORT_SIF2SP            1
-#define SP_STR_MON_PORT_SP2MC             2 
-#define SP_STR_MON_PORT_MC2SP             3
-#define SP_STR_MON_PORT_SP2DMA            4 
-#define SP_STR_MON_PORT_DMA2SP            5
-#define SP_STR_MON_PORT_SP2ISP            6 
-#define SP_STR_MON_PORT_ISP2SP            7
-#define SP_STR_MON_PORT_SP2GPD            8
-#define SP_STR_MON_PORT_FA2SP             9
-#define SP_STR_MON_PORT_SP2ISYS          10 
-#define SP_STR_MON_PORT_ISYS2SP          11
-#define SP_STR_MON_PORT_SP2PIFA          12
-#define SP_STR_MON_PORT_PIFA2SP          13
-#define SP_STR_MON_PORT_SP2PIFB          14
-#define SP_STR_MON_PORT_PIFB2SP          15
-
-#define SP_STR_MON_PORT_B_SP2GDC1         0
-#define SP_STR_MON_PORT_B_GDC12SP         1
-#define SP_STR_MON_PORT_B_SP2GDC2         2
-#define SP_STR_MON_PORT_B_GDC22SP         3
-
-/* previously used SP streaming monitor port identifiers, kept for backward compatibility */
-#define SP_STR_MON_PORT_SND_SIF           SP_STR_MON_PORT_SP2SIF
-#define SP_STR_MON_PORT_RCV_SIF           SP_STR_MON_PORT_SIF2SP
-#define SP_STR_MON_PORT_SND_MC            SP_STR_MON_PORT_SP2MC
-#define SP_STR_MON_PORT_RCV_MC            SP_STR_MON_PORT_MC2SP
-#define SP_STR_MON_PORT_SND_DMA           SP_STR_MON_PORT_SP2DMA
-#define SP_STR_MON_PORT_RCV_DMA           SP_STR_MON_PORT_DMA2SP
-#define SP_STR_MON_PORT_SND_ISP           SP_STR_MON_PORT_SP2ISP
-#define SP_STR_MON_PORT_RCV_ISP           SP_STR_MON_PORT_ISP2SP
-#define SP_STR_MON_PORT_SND_GPD           SP_STR_MON_PORT_SP2GPD
-#define SP_STR_MON_PORT_RCV_GPD           SP_STR_MON_PORT_FA2SP
-/* Deprecated */
-#define SP_STR_MON_PORT_SND_PIF           SP_STR_MON_PORT_SP2PIFA
-#define SP_STR_MON_PORT_RCV_PIF           SP_STR_MON_PORT_PIFA2SP
-#define SP_STR_MON_PORT_SND_PIFB          SP_STR_MON_PORT_SP2PIFB
-#define SP_STR_MON_PORT_RCV_PIFB          SP_STR_MON_PORT_PIFB2SP
-
-#define SP_STR_MON_PORT_SND_PIF_A         SP_STR_MON_PORT_SP2PIFA
-#define SP_STR_MON_PORT_RCV_PIF_A         SP_STR_MON_PORT_PIFA2SP
-#define SP_STR_MON_PORT_SND_PIF_B         SP_STR_MON_PORT_SP2PIFB
-#define SP_STR_MON_PORT_RCV_PIF_B         SP_STR_MON_PORT_PIFB2SP
-
-/* port definititions ISP streaming monitor, monitors the status of streaming ports at the ISP side of the streaming FIFO's */
-#define ISP_STR_MON_PORT_ISP2PIFA         0
-#define ISP_STR_MON_PORT_PIFA2ISP         1
-#define ISP_STR_MON_PORT_ISP2PIFB         2 
-#define ISP_STR_MON_PORT_PIFB2ISP         3
-#define ISP_STR_MON_PORT_ISP2DMA          4 
-#define ISP_STR_MON_PORT_DMA2ISP          5
-#define ISP_STR_MON_PORT_ISP2GDC1         6 
-#define ISP_STR_MON_PORT_GDC12ISP         7
-#define ISP_STR_MON_PORT_ISP2GDC2         8 
-#define ISP_STR_MON_PORT_GDC22ISP         9
-#define ISP_STR_MON_PORT_ISP2GPD         10 
-#define ISP_STR_MON_PORT_FA2ISP          11
-#define ISP_STR_MON_PORT_ISP2SP          12 
-#define ISP_STR_MON_PORT_SP2ISP          13
-
-/* previously used ISP streaming monitor port identifiers, kept for backward compatibility */
-#define ISP_STR_MON_PORT_SND_PIF_A       ISP_STR_MON_PORT_ISP2PIFA
-#define ISP_STR_MON_PORT_RCV_PIF_A       ISP_STR_MON_PORT_PIFA2ISP
-#define ISP_STR_MON_PORT_SND_PIF_B       ISP_STR_MON_PORT_ISP2PIFB 
-#define ISP_STR_MON_PORT_RCV_PIF_B       ISP_STR_MON_PORT_PIFB2ISP
-#define ISP_STR_MON_PORT_SND_DMA         ISP_STR_MON_PORT_ISP2DMA  
-#define ISP_STR_MON_PORT_RCV_DMA         ISP_STR_MON_PORT_DMA2ISP 
-#define ISP_STR_MON_PORT_SND_GDC         ISP_STR_MON_PORT_ISP2GDC1 
-#define ISP_STR_MON_PORT_RCV_GDC         ISP_STR_MON_PORT_GDC12ISP
-#define ISP_STR_MON_PORT_SND_GPD         ISP_STR_MON_PORT_ISP2GPD 
-#define ISP_STR_MON_PORT_RCV_GPD         ISP_STR_MON_PORT_FA2ISP
-#define ISP_STR_MON_PORT_SND_SP          ISP_STR_MON_PORT_ISP2SP
-#define ISP_STR_MON_PORT_RCV_SP          ISP_STR_MON_PORT_SP2ISP
-                                           
-/* port definititions MOD streaming monitor, monitors the status of streaming ports at the module side of the streaming FIFO's */
-
-#define MOD_STR_MON_PORT_PIFA2CELLS       0
-#define MOD_STR_MON_PORT_CELLS2PIFA       1
-#define MOD_STR_MON_PORT_PIFB2CELLS       2
-#define MOD_STR_MON_PORT_CELLS2PIFB       3
-#define MOD_STR_MON_PORT_SIF2SP           4
-#define MOD_STR_MON_PORT_SP2SIF           5
-#define MOD_STR_MON_PORT_MC2SP            6
-#define MOD_STR_MON_PORT_SP2MC            7
-#define MOD_STR_MON_PORT_DMA2ISP          8
-#define MOD_STR_MON_PORT_ISP2DMA          9
-#define MOD_STR_MON_PORT_DMA2SP          10
-#define MOD_STR_MON_PORT_SP2DMA          11
-#define MOD_STR_MON_PORT_GDC12CELLS      12
-#define MOD_STR_MON_PORT_CELLS2GDC1      13
-#define MOD_STR_MON_PORT_GDC22CELLS      14
-#define MOD_STR_MON_PORT_CELLS2GDC2      15
-
-#define MOD_STR_MON_PORT_SND_PIF_A        0
-#define MOD_STR_MON_PORT_RCV_PIF_A        1
-#define MOD_STR_MON_PORT_SND_PIF_B        2
-#define MOD_STR_MON_PORT_RCV_PIF_B        3
-#define MOD_STR_MON_PORT_SND_SIF          4
-#define MOD_STR_MON_PORT_RCV_SIF          5
-#define MOD_STR_MON_PORT_SND_MC           6
-#define MOD_STR_MON_PORT_RCV_MC           7
-#define MOD_STR_MON_PORT_SND_DMA2ISP      8
-#define MOD_STR_MON_PORT_RCV_DMA_FR_ISP   9
-#define MOD_STR_MON_PORT_SND_DMA2SP      10
-#define MOD_STR_MON_PORT_RCV_DMA_FR_SP   11
-#define MOD_STR_MON_PORT_SND_GDC         12
-#define MOD_STR_MON_PORT_RCV_GDC         13
-
-
-/* testbench signals:       */
-
-/* testbench GP adapter register ids  */
-#define HIVE_TESTBENCH_GPIO_DATA_OUT_REG_IDX                    0
-#define HIVE_TESTBENCH_GPIO_DIR_OUT_REG_IDX                     1
-#define HIVE_TESTBENCH_IRQ_REG_IDX                              2
-#define HIVE_TESTBENCH_SDRAM_WAKEUP_REG_IDX                     3
-#define HIVE_TESTBENCH_IDLE_REG_IDX                             4
-#define HIVE_TESTBENCH_GPIO_DATA_IN_REG_IDX                     5
-#define HIVE_TESTBENCH_MIPI_BFM_EN_REG_IDX                      6
-#define HIVE_TESTBENCH_CSI_CONFIG_REG_IDX                       7
-#define HIVE_TESTBENCH_DDR_STALL_EN_REG_IDX                     8
-
-#define HIVE_TESTBENCH_ISP_PMEM_ERROR_IRQ_REG_IDX               9
-#define HIVE_TESTBENCH_ISP_BAMEM_ERROR_IRQ_REG_IDX             10
-#define HIVE_TESTBENCH_ISP_DMEM_ERROR_IRQ_REG_IDX              11
-#define HIVE_TESTBENCH_SP_ICACHE_MEM_ERROR_IRQ_REG_IDX         12
-#define HIVE_TESTBENCH_SP_DMEM_ERROR_IRQ_REG_IDX               13
-
-#define HIVE_TESTBENCH_MIPI_PARPATHEN_REG_IDX                  14
-#define HIVE_TESTBENCH_FB_HPLL_FREQ_REG_IDX                    15
-#define HIVE_TESTBENCH_ISCLK_RATIO_REG_IDX                     16
-
-/* Signal monitor input bit ids */
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_O_BIT_ID                0
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_1_BIT_ID                1
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_2_BIT_ID                2
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_3_BIT_ID                3
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_4_BIT_ID                4
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_5_BIT_ID                5
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_6_BIT_ID                6
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_7_BIT_ID                7
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_8_BIT_ID                8
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_9_BIT_ID                9
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_10_BIT_ID              10
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_11_BIT_ID              11
-#define HIVE_TESTBENCH_SIG_MON_IRQ_PIN_BIT_ID                  12
-#define HIVE_TESTBENCH_SIG_MON_SDRAM_WAKEUP_PIN_BIT_ID         13
-#define HIVE_TESTBENCH_SIG_MON_IDLE_PIN_BIT_ID                 14
-
-#define ISP2400_DEBUG_NETWORK    1
-
-#endif /* _hive_isp_css_defs_h__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_host_ids_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_host_ids_hrt.h
deleted file mode 100644
index 8d4c9d67c0e2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_host_ids_hrt.h
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _hive_isp_css_host_ids_hrt_h_
-#define _hive_isp_css_host_ids_hrt_h_
-
-/* ISP_CSS identifiers */
-#define INP_SYS       testbench_isp_isp_css_part_is_2400_inp_sys
-#define ISYS_GP_REGS  testbench_isp_isp_css_part_is_2400_inp_sys_gpreg
-#define ISYS_IRQ_CTRL testbench_isp_isp_css_part_is_2400_inp_sys_irq_ctrl
-#define ISYS_CAP_A    testbench_isp_isp_css_part_is_2400_inp_sys_capt_unit_a
-#define ISYS_CAP_B    testbench_isp_isp_css_part_is_2400_inp_sys_capt_unit_b
-#define ISYS_CAP_C    testbench_isp_isp_css_part_is_2400_inp_sys_capt_unit_c
-#define ISYS_INP_BUF  testbench_isp_isp_css_part_input_buffer
-#define ISYS_INP_CTRL testbench_isp_isp_css_part_is_2400_inp_sys_inp_ctrl
-#define ISYS_ACQ      testbench_isp_isp_css_part_is_2400_inp_sys_acq_unit
-
-#define ISP           testbench_isp_isp_css_sec_part_isp
-#define SP            testbench_isp_isp_css_sec_part_scp
-
-#define IF_PRIM       testbench_isp_isp_css_part_is_2400_ifmt_ift_prim  
-#define IF_PRIM_B     testbench_isp_isp_css_part_is_2400_ifmt_ift_prim_b
-#define IF_SEC        testbench_isp_isp_css_part_is_2400_ifmt_ift_sec
-#define IF_SEC_MASTER testbench_isp_isp_css_part_is_2400_ifmt_ift_sec_mt_out
-#define STR_TO_MEM    testbench_isp_isp_css_part_is_2400_ifmt_mem_cpy
-#define IFMT_GP_REGS  testbench_isp_isp_css_part_is_2400_ifmt_gp_reg
-#define IFMT_IRQ_CTRL testbench_isp_isp_css_part_is_2400_ifmt_irq_ctrl
-
-#define CSS_RECEIVER  testbench_isp_isp_css_part_is_2400_inp_sys_csi_receiver
-
-#define TC            testbench_isp_isp_css_sec_part_gpd_tc
-#define GPTIMER       testbench_isp_isp_css_sec_part_gpd_gptimer
-#define DMA           testbench_isp_isp_css_sec_part_isp_dma
-#define GDC           testbench_isp_isp_css_sec_part_gdc1
-#define GDC2          testbench_isp_isp_css_sec_part_gdc2
-#define IRQ_CTRL      testbench_isp_isp_css_sec_part_gpd_irq_ctrl
-#define GPIO          testbench_isp_isp_css_sec_part_gpd_c_gpio
-#define GP_REGS       testbench_isp_isp_css_sec_part_gpd_gp_reg
-#define ISEL_GP_REGS  testbench_isp_isp_css_part_is_2400_isel_gpr
-#define ISEL_IRQ_CTRL testbench_isp_isp_css_part_is_2400_isel_irq_ctrl
-#define DATA_MMU      testbench_isp_isp_css_sec_part_data_out_sys_c_mmu
-#define ICACHE_MMU    testbench_isp_isp_css_sec_part_icache_out_sys_c_mmu
-
-/* next is actually not FIFO but FIFO adapter, or slave to streaming adapter */
-#define ISP_SP_FIFO   testbench_isp_isp_css_sec_part_fa_sp_isp
-#define ISEL_FIFO     testbench_isp_isp_css_part_is_2400_isel_sf_fa_in
-
-#define FIFO_GPF_SP   testbench_isp_isp_css_sec_part_sf_fa2sp_in
-#define FIFO_GPF_ISP  testbench_isp_isp_css_sec_part_sf_fa2isp_in
-#define FIFO_SP_GPF   testbench_isp_isp_css_sec_part_sf_sp2fa_in
-#define FIFO_ISP_GPF  testbench_isp_isp_css_sec_part_sf_isp2fa_in
-
-#define DATA_OCP_MASTER    testbench_isp_isp_css_sec_part_data_out_sys_cio2ocp_wide_data_out_mt
-#define ICACHE_OCP_MASTER  testbench_isp_isp_css_sec_part_icache_out_sys_cio2ocp_wide_data_out_mt
-
-#define SP_IN_FIFO    testbench_isp_isp_css_sec_part_sf_fa2sp_in
-#define SP_OUT_FIFO   testbench_isp_isp_css_sec_part_sf_sp2fa_out
-#define ISP_IN_FIFO   testbench_isp_isp_css_sec_part_sf_fa2isp_in
-#define ISP_OUT_FIFO  testbench_isp_isp_css_sec_part_sf_isp2fa_out
-#define GEN_SHORT_PACK_PORT testbench_isp_isp_css_part_is_2400_inp_sys_csi_str_mon_fa_gensh_out
-
-/* input_system_2401 identifiers */
-#define ISYS2401_GP_REGS    testbench_isp_isp_css_part_is_2401_gpreg
-#define ISYS2401_DMA        testbench_isp_isp_css_part_is_2401_dma
-#define ISYS2401_IRQ_CTRL   testbench_isp_isp_css_part_is_2401_isys_irq_ctrl
-
-#define ISYS2401_CSI_RX_A     testbench_isp_isp_css_part_is_2401_is_pipe_a_csi_rx
-#define ISYS2401_MIPI_BE_A    testbench_isp_isp_css_part_is_2401_is_pipe_a_mipi_be
-#define ISYS2401_S2M_A        testbench_isp_isp_css_part_is_2401_is_pipe_a_s2m
-#define ISYS2401_PXG_A        testbench_isp_isp_css_part_is_2401_is_pipe_a_pxlgen
-#define ISYS2401_IBUF_CNTRL_A testbench_isp_isp_css_part_is_2401_is_pipe_a_ibuf_ctrl
-#define ISYS2401_IRQ_CTRL_A   testbench_isp_isp_css_part_is_2401_is_pipe_a_irq_ctrl_pipe
-
-#define ISYS2401_CSI_RX_B     testbench_isp_isp_css_part_is_2401_is_pipe_b_csi_rx
-#define ISYS2401_MIPI_BE_B    testbench_isp_isp_css_part_is_2401_is_pipe_b_mipi_be
-#define ISYS2401_S2M_B        testbench_isp_isp_css_part_is_2401_is_pipe_b_s2m
-#define ISYS2401_PXG_B        testbench_isp_isp_css_part_is_2401_is_pipe_b_pxlgen
-#define ISYS2401_IBUF_CNTRL_B testbench_isp_isp_css_part_is_2401_is_pipe_b_ibuf_ctrl
-#define ISYS2401_IRQ_CTRL_B   testbench_isp_isp_css_part_is_2401_is_pipe_b_irq_ctrl_pipe
-
-#define ISYS2401_CSI_RX_C     testbench_isp_isp_css_part_is_2401_is_pipe_c_csi_rx
-#define ISYS2401_MIPI_BE_C    testbench_isp_isp_css_part_is_2401_is_pipe_c_mipi_be
-#define ISYS2401_S2M_C        testbench_isp_isp_css_part_is_2401_is_pipe_c_s2m
-#define ISYS2401_PXG_C        testbench_isp_isp_css_part_is_2401_is_pipe_c_pxlgen
-#define ISYS2401_IBUF_CNTRL_C testbench_isp_isp_css_part_is_2401_is_pipe_c_ibuf_ctrl
-#define ISYS2401_IRQ_CTRL_C   testbench_isp_isp_css_part_is_2401_is_pipe_c_irq_ctrl_pipe
-
-
-/* Testbench identifiers */
-#define DDR             testbench_ddram
-#define DDR_SMALL       testbench_ddram_small
-#define XMEM            DDR
-#define GPIO_ADAPTER    testbench_gp_adapter
-#define SIG_MONITOR     testbench_sig_mon
-#define DDR_SLAVE       testbench_ddram_ip0
-#define DDR_SMALL_SLAVE testbench_ddram_small_ip0
-#define HOST_MASTER     host_op0
-
-#define CSI_SENSOR         testbench_vied_sensor
-#define CSI_SENSOR_GP_REGS testbench_vied_sensor_gpreg
-#define CSI_STR_IN_A       testbench_vied_sensor_tx_a_csi_tx_data_in
-#define CSI_STR_IN_B       testbench_vied_sensor_tx_b_csi_tx_data_in
-#define CSI_STR_IN_C       testbench_vied_sensor_tx_c_csi_tx_data_in
-#define CSI_SENSOR_TX_A    testbench_vied_sensor_tx_a
-#define CSI_SENSOR_TX_B    testbench_vied_sensor_tx_b
-#define CSI_SENSOR_TX_C    testbench_vied_sensor_tx_c
-
-#endif /* _hive_isp_css_host_ids_hrt_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
deleted file mode 100644
index b4211a0c631a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _hive_isp_css_streaming_to_mipi_types_hrt_h_
-#define _hive_isp_css_streaming_to_mipi_types_hrt_h_
-
-#include <streaming_to_mipi_defs.h>
-
-#define _HIVE_ISP_CH_ID_MASK    ((1U << HIVE_ISP_CH_ID_BITS)-1)
-#define _HIVE_ISP_FMT_TYPE_MASK ((1U << HIVE_ISP_FMT_TYPE_BITS)-1)
-
-#define _HIVE_STR_TO_MIPI_FMT_TYPE_LSB (HIVE_STR_TO_MIPI_CH_ID_LSB + HIVE_ISP_CH_ID_BITS)
-#define _HIVE_STR_TO_MIPI_DATA_B_LSB   (HIVE_STR_TO_MIPI_DATA_A_LSB + HIVE_IF_PIXEL_WIDTH)
- 
-#endif /* _hive_isp_css_streaming_to_mipi_types_hrt_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_types.h
deleted file mode 100644
index 58b0e6effbd0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/hive_types.h
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_HIVE_TYPES_H 
-#define _HRT_HIVE_TYPES_H 
-
-#include "version.h"
-#include "defs.h"
-
-#ifndef HRTCAT3
-#define _HRTCAT3(m,n,o)     m##n##o
-#define HRTCAT3(m,n,o)      _HRTCAT3(m,n,o)
-#endif
-
-#ifndef HRTCAT4
-#define _HRTCAT4(m,n,o,p)     m##n##o##p
-#define HRTCAT4(m,n,o,p)      _HRTCAT4(m,n,o,p)
-#endif
-
-#ifndef HRTMIN
-#define HRTMIN(a,b) (((a)<(b))?(a):(b))
-#endif
-                                 
-#ifndef HRTMAX
-#define HRTMAX(a,b) (((a)>(b))?(a):(b))
-#endif
-
-/* boolean data type */
-typedef unsigned int hive_bool;
-#define hive_false 0
-#define hive_true  1
-
-typedef char                 hive_int8;
-typedef short                hive_int16;
-typedef int                  hive_int32;
-typedef long long            hive_int64;
-
-typedef unsigned char        hive_uint8;
-typedef unsigned short       hive_uint16;
-typedef unsigned int         hive_uint32;
-typedef unsigned long long   hive_uint64;
-
-/* by default assume 32 bit master port (both data and address) */
-#ifndef HRT_DATA_WIDTH
-#define HRT_DATA_WIDTH 32
-#endif
-#ifndef HRT_ADDRESS_WIDTH
-#define HRT_ADDRESS_WIDTH 32
-#endif
-
-#define HRT_DATA_BYTES    (HRT_DATA_WIDTH/8)
-#define HRT_ADDRESS_BYTES (HRT_ADDRESS_WIDTH/8)
-
-#if HRT_DATA_WIDTH == 64
-typedef hive_uint64 hrt_data;
-#elif HRT_DATA_WIDTH == 32
-typedef hive_uint32 hrt_data;
-#else
-#error data width not supported
-#endif
-
-#if HRT_ADDRESS_WIDTH == 64
-typedef hive_uint64 hrt_address; 
-#elif HRT_ADDRESS_WIDTH == 32
-typedef hive_uint32 hrt_address;
-#else
-#error adddres width not supported
-#endif
-
-/* The SP side representation of an HMM virtual address */
-typedef hive_uint32 hrt_vaddress;
-
-/* use 64 bit addresses in simulation, where possible */
-typedef hive_uint64  hive_sim_address;
-
-/* below is for csim, not for hrt, rename and move this elsewhere */
-
-typedef unsigned int hive_uint;
-typedef hive_uint32  hive_address;
-typedef hive_address hive_slave_address;
-typedef hive_address hive_mem_address;
-
-/* MMIO devices */
-typedef hive_uint    hive_mmio_id;
-typedef hive_mmio_id hive_slave_id;
-typedef hive_mmio_id hive_port_id;
-typedef hive_mmio_id hive_master_id; 
-typedef hive_mmio_id hive_mem_id;
-typedef hive_mmio_id hive_dev_id;
-typedef hive_mmio_id hive_fifo_id;
-
-typedef hive_uint      hive_hier_id;
-typedef hive_hier_id   hive_device_id;
-typedef hive_device_id hive_proc_id;
-typedef hive_device_id hive_cell_id;
-typedef hive_device_id hive_host_id;
-typedef hive_device_id hive_bus_id;
-typedef hive_device_id hive_bridge_id;
-typedef hive_device_id hive_fifo_adapter_id;
-typedef hive_device_id hive_custom_device_id;
-
-typedef hive_uint hive_slot_id;
-typedef hive_uint hive_fu_id;
-typedef hive_uint hive_reg_file_id;
-typedef hive_uint hive_reg_id;
-
-/* Streaming devices */
-typedef hive_uint hive_outport_id;
-typedef hive_uint hive_inport_id;
-
-typedef hive_uint hive_msink_id;
-
-/* HRT specific */
-typedef char* hive_program;
-typedef char* hive_function;
-
-#endif /* _HRT_HIVE_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/ibuf_cntrl_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/ibuf_cntrl_defs.h
deleted file mode 100644
index f82bb79785cf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/ibuf_cntrl_defs.h
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _ibuf_cntrl_defs_h_
-#define _ibuf_cntrl_defs_h_
-
-#include <stream2mmio_defs.h>
-#include <dma_v2_defs.h>
-
-#define _IBUF_CNTRL_REG_ALIGN 4
-  /* alignment of register banks, first bank are shared configuration and status registers: */
-#define _IBUF_CNTRL_PROC_REG_ALIGN        32
-
-  /* the actual amount of configuration registers per proc: */
-#define _IBUF_CNTRL_CONFIG_REGS_PER_PROC 18
-  /* the actual amount of shared configuration registers: */
-#define _IBUF_CNTRL_CONFIG_REGS_NO_PROC  0
-
-  /* the actual amount of status registers per proc */
-#define _IBUF_CNTRL_STATUS_REGS_PER_PROC (_IBUF_CNTRL_CONFIG_REGS_PER_PROC + 10)
-  /* the actual amount shared status registers */
-#define _IBUF_CNTRL_STATUS_REGS_NO_PROC  (_IBUF_CNTRL_CONFIG_REGS_NO_PROC + 2)
-
-  /* time out bits, maximum time out value is 2^_IBUF_CNTRL_TIME_OUT_BITS - 1 */
-#define _IBUF_CNTRL_TIME_OUT_BITS         5
-
-/* command token definition */
-#define _IBUF_CNTRL_CMD_TOKEN_LSB          0
-#define _IBUF_CNTRL_CMD_TOKEN_MSB          1
-
-/* Str2MMIO defines */
-#define _IBUF_CNTRL_STREAM2MMIO_CMD_TOKEN_MSB        _STREAM2MMIO_CMD_TOKEN_CMD_MSB
-#define _IBUF_CNTRL_STREAM2MMIO_CMD_TOKEN_LSB        _STREAM2MMIO_CMD_TOKEN_CMD_LSB
-#define _IBUF_CNTRL_STREAM2MMIO_NUM_ITEMS_BITS       _STREAM2MMIO_PACK_NUM_ITEMS_BITS
-#define _IBUF_CNTRL_STREAM2MMIO_ACK_EOF_BIT          _STREAM2MMIO_PACK_ACK_EOF_BIT
-#define _IBUF_CNTRL_STREAM2MMIO_ACK_TOKEN_VALID_BIT  _STREAM2MMIO_ACK_TOKEN_VALID_BIT
-
-/* acknowledge token definition */
-#define _IBUF_CNTRL_ACK_TOKEN_STORES_IDX    0
-#define _IBUF_CNTRL_ACK_TOKEN_STORES_BITS   15
-#define _IBUF_CNTRL_ACK_TOKEN_ITEMS_IDX     (_IBUF_CNTRL_ACK_TOKEN_STORES_BITS + _IBUF_CNTRL_ACK_TOKEN_STORES_IDX)
-#define _IBUF_CNTRL_ACK_TOKEN_ITEMS_BITS    _STREAM2MMIO_PACK_NUM_ITEMS_BITS
-#define _IBUF_CNTRL_ACK_TOKEN_LSB          _IBUF_CNTRL_ACK_TOKEN_STORES_IDX
-#define _IBUF_CNTRL_ACK_TOKEN_MSB          (_IBUF_CNTRL_ACK_TOKEN_ITEMS_BITS + _IBUF_CNTRL_ACK_TOKEN_ITEMS_IDX - 1)
-          /* bit 31 indicates a valid ack: */
-#define _IBUF_CNTRL_ACK_TOKEN_VALID_BIT    (_IBUF_CNTRL_ACK_TOKEN_ITEMS_BITS + _IBUF_CNTRL_ACK_TOKEN_ITEMS_IDX)
-
-
-/*shared registers:*/
-#define _IBUF_CNTRL_RECALC_WORDS_STATUS     0
-#define _IBUF_CNTRL_ARBITERS_STATUS         1
-
-#define _IBUF_CNTRL_SET_CRUN                2 /* NO PHYSICAL REGISTER!! Only used in HSS model */
-
-
-/*register addresses for each proc: */
-#define _IBUF_CNTRL_CMD                   0
-#define _IBUF_CNTRL_ACK                   1
-
-        /* number of items (packets or words) per frame: */
-#define _IBUF_CNTRL_NUM_ITEMS_PER_STORE   2
-
-        /* number of stores (packets or words) per store/buffer: */
-#define _IBUF_CNTRL_NUM_STORES_PER_FRAME  3
-
-        /* the channel and command in the DMA */
-#define _IBUF_CNTRL_DMA_CHANNEL           4
-#define _IBUF_CNTRL_DMA_CMD               5
-
-        /* the start address and stride of the buffers */
-#define _IBUF_CNTRL_BUFFER_START_ADDRESS  6
-#define _IBUF_CNTRL_BUFFER_STRIDE         7
-#define _IBUF_CNTRL_BUFFER_END_ADDRESS    8
-
-        /* destination start address, stride and end address; should be the same as in the DMA */
-#define _IBUF_CNTRL_DEST_START_ADDRESS    9
-#define _IBUF_CNTRL_DEST_STRIDE           10
-#define _IBUF_CNTRL_DEST_END_ADDRESS      11
-
-        /* send a frame sync or not, default 1 */
-#define _IBUF_CNTRL_SYNC_FRAME            12
-
-        /* str2mmio cmds */
-#define _IBUF_CNTRL_STR2MMIO_SYNC_CMD     13
-#define _IBUF_CNTRL_STR2MMIO_STORE_CMD    14
-
-        /* num elems p word*/
-#define _IBUF_CNTRL_SHIFT_ITEMS           15
-#define _IBUF_CNTRL_ELEMS_P_WORD_IBUF     16
-#define _IBUF_CNTRL_ELEMS_P_WORD_DEST     17
-
-
-   /* STATUS */
-        /* current frame and stores in buffer */
-#define _IBUF_CNTRL_CUR_STORES            18
-#define _IBUF_CNTRL_CUR_ACKS              19
-
-        /* current buffer and destination address for DMA cmd's */
-#define _IBUF_CNTRL_CUR_S2M_IBUF_ADDR     20
-#define _IBUF_CNTRL_CUR_DMA_IBUF_ADDR     21
-#define _IBUF_CNTRL_CUR_DMA_DEST_ADDR     22
-#define _IBUF_CNTRL_CUR_ISP_DEST_ADDR     23
-
-#define _IBUF_CNTRL_CUR_NR_DMA_CMDS_SEND  24
-
-#define _IBUF_CNTRL_MAIN_CNTRL_STATE      25
-#define _IBUF_CNTRL_DMA_SYNC_STATE        26
-#define _IBUF_CNTRL_ISP_SYNC_STATE        27
-
-
-/*Commands: */
-#define _IBUF_CNTRL_CMD_STORE_FRAME_IDX     0
-#define _IBUF_CNTRL_CMD_ONLINE_IDX          1
-
-  /* initialize, copy st_addr to cur_addr etc */
-#define _IBUF_CNTRL_CMD_INITIALIZE          0
-
-  /* store an online frame (sync with ISP, use end cfg start, stride and end address: */
-#define _IBUF_CNTRL_CMD_STORE_ONLINE_FRAME  ((1<<_IBUF_CNTRL_CMD_STORE_FRAME_IDX) | (1<<_IBUF_CNTRL_CMD_ONLINE_IDX))
-
-  /* store an offline frame (don't sync with ISP, requires start address as 2nd token, no end address: */
-#define _IBUF_CNTRL_CMD_STORE_OFFLINE_FRAME  (1<<_IBUF_CNTRL_CMD_STORE_FRAME_IDX)
-
-  /* false command token, should be different then commands. Use online bit, not store frame: */
-#define _IBUF_CNTRL_FALSE_ACK               2
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/if_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/if_defs.h
deleted file mode 100644
index 7d39e45796ae..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/if_defs.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IF_DEFS_H
-#define _IF_DEFS_H
-
-#define HIVE_IF_FRAME_REQUEST        0xA000
-#define HIVE_IF_LINES_REQUEST        0xB000
-#define HIVE_IF_VECTORS_REQUEST      0xC000
-
-#endif /* _IF_DEFS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_formatter_subsystem_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_formatter_subsystem_defs.h
deleted file mode 100644
index 7766f78cd123..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_formatter_subsystem_defs.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _if_subsystem_defs_h__
-#define _if_subsystem_defs_h__
-
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_0            0
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_1            1
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_2            2
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_3            3
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_4            4
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_5            5
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_6            6
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_7            7 
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_FSYNC_LUT_REG        8
-#define HIVE_IFMT_GP_REGS_SRST_IDX                          9
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IDX                 10
-
-#define HIVE_IFMT_GP_REGS_CH_ID_FMT_TYPE_IDX               11
-
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_BASE         HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_0
-
-/* order of the input bits for the ifmt irq controller */
-#define HIVE_IFMT_IRQ_IFT_PRIM_BIT_ID                       0
-#define HIVE_IFMT_IRQ_IFT_PRIM_B_BIT_ID                     1
-#define HIVE_IFMT_IRQ_IFT_SEC_BIT_ID                        2
-#define HIVE_IFMT_IRQ_MEM_CPY_BIT_ID                        3
-#define HIVE_IFMT_IRQ_SIDEBAND_CHANGED_BIT_ID               4
-
-/* order of the input bits for the ifmt Soft reset register */
-#define HIVE_IFMT_GP_REGS_SRST_IFT_PRIM_BIT_IDX             0
-#define HIVE_IFMT_GP_REGS_SRST_IFT_PRIM_B_BIT_IDX           1
-#define HIVE_IFMT_GP_REGS_SRST_IFT_SEC_BIT_IDX              2
-#define HIVE_IFMT_GP_REGS_SRST_MEM_CPY_BIT_IDX              3
-
-/* order of the input bits for the ifmt Soft reset register */
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IFT_PRIM_BIT_IDX     0
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IFT_PRIM_B_BIT_IDX   1
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IFT_SEC_BIT_IDX      2
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_MEM_CPY_BIT_IDX      3
-
-#endif /* _if_subsystem_defs_h__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_selector_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_selector_defs.h
deleted file mode 100644
index 87fbf82edb5b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_selector_defs.h
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_selector_defs_h
-#define _input_selector_defs_h
-
-#ifndef HIVE_ISP_ISEL_SEL_BITS
-#define HIVE_ISP_ISEL_SEL_BITS                                  2
-#endif
-
-#ifndef HIVE_ISP_CH_ID_BITS
-#define HIVE_ISP_CH_ID_BITS                                     2
-#endif
-
-#ifndef HIVE_ISP_FMT_TYPE_BITS
-#define HIVE_ISP_FMT_TYPE_BITS                                  5
-#endif
-
-/* gp_register register id's -- Outputs */
-#define HIVE_ISEL_GP_REGS_SYNCGEN_ENABLE_IDX                    0
-#define HIVE_ISEL_GP_REGS_SYNCGEN_FREE_RUNNING_IDX              1
-#define HIVE_ISEL_GP_REGS_SYNCGEN_PAUSE_IDX                     2
-#define HIVE_ISEL_GP_REGS_SYNCGEN_NR_FRAMES_IDX                 3 
-#define HIVE_ISEL_GP_REGS_SYNCGEN_NR_PIX_IDX                    4      
-#define HIVE_ISEL_GP_REGS_SYNCGEN_NR_LINES_IDX                  5      
-#define HIVE_ISEL_GP_REGS_SYNCGEN_HBLANK_CYCLES_IDX             6      
-#define HIVE_ISEL_GP_REGS_SYNCGEN_VBLANK_CYCLES_IDX             7      
-
-#define HIVE_ISEL_GP_REGS_SOF_IDX                               8 
-#define HIVE_ISEL_GP_REGS_EOF_IDX                               9 
-#define HIVE_ISEL_GP_REGS_SOL_IDX                              10 
-#define HIVE_ISEL_GP_REGS_EOL_IDX                              11 
-
-#define HIVE_ISEL_GP_REGS_PRBS_ENABLE                          12      
-#define HIVE_ISEL_GP_REGS_PRBS_ENABLE_PORT_B                   13      
-#define HIVE_ISEL_GP_REGS_PRBS_LFSR_RESET_VALUE                14      
-
-#define HIVE_ISEL_GP_REGS_TPG_ENABLE                           15      
-#define HIVE_ISEL_GP_REGS_TPG_ENABLE_PORT_B                    16      
-#define HIVE_ISEL_GP_REGS_TPG_HOR_CNT_MASK_IDX                 17      
-#define HIVE_ISEL_GP_REGS_TPG_VER_CNT_MASK_IDX                 18      
-#define HIVE_ISEL_GP_REGS_TPG_XY_CNT_MASK_IDX                  19      
-#define HIVE_ISEL_GP_REGS_TPG_HOR_CNT_DELTA_IDX                20      
-#define HIVE_ISEL_GP_REGS_TPG_VER_CNT_DELTA_IDX                21      
-#define HIVE_ISEL_GP_REGS_TPG_MODE_IDX                         22     
-#define HIVE_ISEL_GP_REGS_TPG_R1_IDX                           23 
-#define HIVE_ISEL_GP_REGS_TPG_G1_IDX                           24
-#define HIVE_ISEL_GP_REGS_TPG_B1_IDX                           25
-#define HIVE_ISEL_GP_REGS_TPG_R2_IDX                           26
-#define HIVE_ISEL_GP_REGS_TPG_G2_IDX                           27
-#define HIVE_ISEL_GP_REGS_TPG_B2_IDX                           28
-
-
-#define HIVE_ISEL_GP_REGS_CH_ID_IDX                            29
-#define HIVE_ISEL_GP_REGS_FMT_TYPE_IDX                         30
-#define HIVE_ISEL_GP_REGS_DATA_SEL_IDX                         31
-#define HIVE_ISEL_GP_REGS_SBAND_SEL_IDX                        32
-#define HIVE_ISEL_GP_REGS_SYNC_SEL_IDX                         33
-#define HIVE_ISEL_GP_REGS_SRST_IDX                             37
-
-#define HIVE_ISEL_GP_REGS_SRST_SYNCGEN_BIT                      0
-#define HIVE_ISEL_GP_REGS_SRST_PRBS_BIT                         1
-#define HIVE_ISEL_GP_REGS_SRST_TPG_BIT                          2
-#define HIVE_ISEL_GP_REGS_SRST_FIFO_BIT                         3
-
-/* gp_register register id's -- Inputs   */
-#define HIVE_ISEL_GP_REGS_SYNCGEN_HOR_CNT_IDX                  34
-#define HIVE_ISEL_GP_REGS_SYNCGEN_VER_CNT_IDX                  35
-#define HIVE_ISEL_GP_REGS_SYNCGEN_FRAMES_CNT_IDX               36
-
-/* irq sources isel irq controller */
-#define HIVE_ISEL_IRQ_SYNC_GEN_SOF_BIT_ID                       0
-#define HIVE_ISEL_IRQ_SYNC_GEN_EOF_BIT_ID                       1
-#define HIVE_ISEL_IRQ_SYNC_GEN_SOL_BIT_ID                       2
-#define HIVE_ISEL_IRQ_SYNC_GEN_EOL_BIT_ID                       3
-#define HIVE_ISEL_IRQ_NUM_IRQS                                  4
-
-#endif /* _input_selector_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_switch_2400_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_switch_2400_defs.h
deleted file mode 100644
index 20a13c4cdb56..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_switch_2400_defs.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_switch_2400_defs_h
-#define _input_switch_2400_defs_h
-
-#define _HIVE_INPUT_SWITCH_GET_LUT_REG_ID(ch_id, fmt_type) (((ch_id)*2) + ((fmt_type)>=16))
-#define _HIVE_INPUT_SWITCH_GET_LUT_REG_LSB(fmt_type)        (((fmt_type)%16) * 2)
-
-#define HIVE_INPUT_SWITCH_SELECT_NO_OUTPUT   0
-#define HIVE_INPUT_SWITCH_SELECT_IF_PRIM     1
-#define HIVE_INPUT_SWITCH_SELECT_IF_SEC      2
-#define HIVE_INPUT_SWITCH_SELECT_STR_TO_MEM  3
-#define HIVE_INPUT_SWITCH_VSELECT_NO_OUTPUT  0
-#define HIVE_INPUT_SWITCH_VSELECT_IF_PRIM    1
-#define HIVE_INPUT_SWITCH_VSELECT_IF_SEC     2
-#define HIVE_INPUT_SWITCH_VSELECT_STR_TO_MEM 4
-
-#endif /* _input_switch_2400_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_system_ctrl_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_system_ctrl_defs.h
deleted file mode 100644
index a7f0ca80bc9b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_system_ctrl_defs.h
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_system_ctrl_defs_h
-#define _input_system_ctrl_defs_h
-
-#define _INPUT_SYSTEM_CTRL_REG_ALIGN                    4  /* assuming 32 bit control bus width */
-
-/* --------------------------------------------------*/
-
-/* --------------------------------------------------*/
-/* REGISTER INFO */
-/* --------------------------------------------------*/
-
-// Number of registers
-#define ISYS_CTRL_NOF_REGS                              23
-
-// Register id's of MMIO slave accesible registers
-#define ISYS_CTRL_CAPT_START_ADDR_A_REG_ID              0
-#define ISYS_CTRL_CAPT_START_ADDR_B_REG_ID              1
-#define ISYS_CTRL_CAPT_START_ADDR_C_REG_ID              2
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_A_REG_ID         3
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_B_REG_ID         4
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_C_REG_ID         5
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_A_REG_ID         6
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_B_REG_ID         7
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_C_REG_ID         8
-#define ISYS_CTRL_ACQ_START_ADDR_REG_ID                 9
-#define ISYS_CTRL_ACQ_MEM_REGION_SIZE_REG_ID            10
-#define ISYS_CTRL_ACQ_NUM_MEM_REGIONS_REG_ID            11
-#define ISYS_CTRL_INIT_REG_ID                           12
-#define ISYS_CTRL_LAST_COMMAND_REG_ID                   13
-#define ISYS_CTRL_NEXT_COMMAND_REG_ID                   14
-#define ISYS_CTRL_LAST_ACKNOWLEDGE_REG_ID               15
-#define ISYS_CTRL_NEXT_ACKNOWLEDGE_REG_ID               16
-#define ISYS_CTRL_FSM_STATE_INFO_REG_ID                 17
-#define ISYS_CTRL_CAPT_A_FSM_STATE_INFO_REG_ID          18
-#define ISYS_CTRL_CAPT_B_FSM_STATE_INFO_REG_ID          19
-#define ISYS_CTRL_CAPT_C_FSM_STATE_INFO_REG_ID          20
-#define ISYS_CTRL_ACQ_FSM_STATE_INFO_REG_ID             21
-#define ISYS_CTRL_CAPT_RESERVE_ONE_MEM_REGION_REG_ID    22
- 
-
-/* register reset value */
-#define ISYS_CTRL_CAPT_START_ADDR_A_REG_RSTVAL           0
-#define ISYS_CTRL_CAPT_START_ADDR_B_REG_RSTVAL           0
-#define ISYS_CTRL_CAPT_START_ADDR_C_REG_RSTVAL           0
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_A_REG_RSTVAL      128
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_B_REG_RSTVAL      128
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_C_REG_RSTVAL      128
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_A_REG_RSTVAL      3 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_B_REG_RSTVAL      3 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_C_REG_RSTVAL      3 
-#define ISYS_CTRL_ACQ_START_ADDR_REG_RSTVAL              0
-#define ISYS_CTRL_ACQ_MEM_REGION_SIZE_REG_RSTVAL         128 
-#define ISYS_CTRL_ACQ_NUM_MEM_REGIONS_REG_RSTVAL         3 
-#define ISYS_CTRL_INIT_REG_RSTVAL                        0
-#define ISYS_CTRL_LAST_COMMAND_REG_RSTVAL                15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)  
-#define ISYS_CTRL_NEXT_COMMAND_REG_RSTVAL                15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)
-#define ISYS_CTRL_LAST_ACKNOWLEDGE_REG_RSTVAL            15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)
-#define ISYS_CTRL_NEXT_ACKNOWLEDGE_REG_RSTVAL            15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)
-#define ISYS_CTRL_FSM_STATE_INFO_REG_RSTVAL              0
-#define ISYS_CTRL_CAPT_A_FSM_STATE_INFO_REG_RSTVAL       0 
-#define ISYS_CTRL_CAPT_B_FSM_STATE_INFO_REG_RSTVAL       0
-#define ISYS_CTRL_CAPT_C_FSM_STATE_INFO_REG_RSTVAL       0
-#define ISYS_CTRL_ACQ_FSM_STATE_INFO_REG_RSTVAL          0
-#define ISYS_CTRL_CAPT_RESERVE_ONE_MEM_REGION_REG_RSTVAL 0
-
-/* register width value */
-#define ISYS_CTRL_CAPT_START_ADDR_A_REG_WIDTH            9 
-#define ISYS_CTRL_CAPT_START_ADDR_B_REG_WIDTH            9 
-#define ISYS_CTRL_CAPT_START_ADDR_C_REG_WIDTH            9 
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_A_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_B_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_C_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_A_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_B_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_C_REG_WIDTH       9 
-#define ISYS_CTRL_ACQ_START_ADDR_REG_WIDTH               9 
-#define ISYS_CTRL_ACQ_MEM_REGION_SIZE_REG_WIDTH          9 
-#define ISYS_CTRL_ACQ_NUM_MEM_REGIONS_REG_WIDTH          9 
-#define ISYS_CTRL_INIT_REG_WIDTH                         3 
-#define ISYS_CTRL_LAST_COMMAND_REG_WIDTH                 32    /* slave data width */
-#define ISYS_CTRL_NEXT_COMMAND_REG_WIDTH                 32
-#define ISYS_CTRL_LAST_ACKNOWLEDGE_REG_WIDTH             32
-#define ISYS_CTRL_NEXT_ACKNOWLEDGE_REG_WIDTH             32
-#define ISYS_CTRL_FSM_STATE_INFO_REG_WIDTH               32
-#define ISYS_CTRL_CAPT_A_FSM_STATE_INFO_REG_WIDTH        32
-#define ISYS_CTRL_CAPT_B_FSM_STATE_INFO_REG_WIDTH        32
-#define ISYS_CTRL_CAPT_C_FSM_STATE_INFO_REG_WIDTH        32
-#define ISYS_CTRL_ACQ_FSM_STATE_INFO_REG_WIDTH           32
-#define ISYS_CTRL_CAPT_RESERVE_ONE_MEM_REGION_REG_WIDTH  1
-
-/* bit definitions */
-
-/* --------------------------------------------------*/
-/* TOKEN INFO */
-/* --------------------------------------------------*/
-
-/*
-InpSysCaptFramesAcq  1/0  [3:0] - 'b0000
-[7:4] - CaptPortId,
-           CaptA-'b0000
-           CaptB-'b0001
-           CaptC-'b0010
-[31:16] - NOF_frames
-InpSysCaptFrameExt  2/0  [3:0] - 'b0001'
-[7:4] - CaptPortId,
-           'b0000 - CaptA 
-           'b0001 - CaptB
-           'b0010 - CaptC
-
-  2/1  [31:0] - external capture address
-InpSysAcqFrame  2/0  [3:0] - 'b0010, 
-[31:4] - NOF_ext_mem_words
-  2/1  [31:0] - external memory read start address
-InpSysOverruleON  1/0  [3:0] - 'b0011, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysOverruleOFF  1/0  [3:0] - 'b0100, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysOverruleCmd  2/0  [3:0] - 'b0101, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-  2/1  [31:0] - command token value for port opid
-
-
-acknowledge tokens:
-
-InpSysAckCFA  1/0   [3:0] - 'b0000
- [7:4] - CaptPortId,
-           CaptA-'b0000
-           CaptB- 'b0001
-           CaptC-'b0010
- [31:16] - NOF_frames
-InpSysAckCFE  1/0  [3:0] - 'b0001'
-[7:4] - CaptPortId,
-           'b0000 - CaptA 
-           'b0001 - CaptB
-           'b0010 - CaptC
-
-InpSysAckAF  1/0  [3:0] - 'b0010
-InpSysAckOverruleON  1/0  [3:0] - 'b0011, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysAckOverruleOFF  1/0  [3:0] - 'b0100, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysAckOverrule  2/0  [3:0] - 'b0101, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-  2/1  [31:0] - acknowledge token value from port opid
-
-
-
-*/
-
-
-/* Command and acknowledge tokens IDs */
-#define ISYS_CTRL_CAPT_FRAMES_ACQ_TOKEN_ID        0 /* 0000b */
-#define ISYS_CTRL_CAPT_FRAME_EXT_TOKEN_ID         1 /* 0001b */
-#define ISYS_CTRL_ACQ_FRAME_TOKEN_ID              2 /* 0010b */
-#define ISYS_CTRL_OVERRULE_ON_TOKEN_ID            3 /* 0011b */
-#define ISYS_CTRL_OVERRULE_OFF_TOKEN_ID           4 /* 0100b */
-#define ISYS_CTRL_OVERRULE_TOKEN_ID               5 /* 0101b */
-
-#define ISYS_CTRL_ACK_CFA_TOKEN_ID                0
-#define ISYS_CTRL_ACK_CFE_TOKEN_ID                1
-#define ISYS_CTRL_ACK_AF_TOKEN_ID                 2
-#define ISYS_CTRL_ACK_OVERRULE_ON_TOKEN_ID        3
-#define ISYS_CTRL_ACK_OVERRULE_OFF_TOKEN_ID       4
-#define ISYS_CTRL_ACK_OVERRULE_TOKEN_ID           5
-#define ISYS_CTRL_ACK_DEVICE_ERROR_TOKEN_ID       6
-
-#define ISYS_CTRL_TOKEN_ID_MSB                    3
-#define ISYS_CTRL_TOKEN_ID_LSB                    0
-#define ISYS_CTRL_PORT_ID_TOKEN_MSB               7
-#define ISYS_CTRL_PORT_ID_TOKEN_LSB               4
-#define ISYS_CTRL_NOF_CAPT_TOKEN_MSB              31
-#define ISYS_CTRL_NOF_CAPT_TOKEN_LSB              16
-#define ISYS_CTRL_NOF_EXT_TOKEN_MSB               31
-#define ISYS_CTRL_NOF_EXT_TOKEN_LSB               8
-
-#define ISYS_CTRL_TOKEN_ID_IDX                    0
-#define ISYS_CTRL_TOKEN_ID_BITS                   (ISYS_CTRL_TOKEN_ID_MSB - ISYS_CTRL_TOKEN_ID_LSB + 1)
-#define ISYS_CTRL_PORT_ID_IDX                     (ISYS_CTRL_TOKEN_ID_IDX + ISYS_CTRL_TOKEN_ID_BITS)
-#define ISYS_CTRL_PORT_ID_BITS                    (ISYS_CTRL_PORT_ID_TOKEN_MSB - ISYS_CTRL_PORT_ID_TOKEN_LSB +1)
-#define ISYS_CTRL_NOF_CAPT_IDX                    ISYS_CTRL_NOF_CAPT_TOKEN_LSB    
-#define ISYS_CTRL_NOF_CAPT_BITS                   (ISYS_CTRL_NOF_CAPT_TOKEN_MSB - ISYS_CTRL_NOF_CAPT_TOKEN_LSB + 1)
-#define ISYS_CTRL_NOF_EXT_IDX                     ISYS_CTRL_NOF_EXT_TOKEN_LSB    
-#define ISYS_CTRL_NOF_EXT_BITS                    (ISYS_CTRL_NOF_EXT_TOKEN_MSB - ISYS_CTRL_NOF_EXT_TOKEN_LSB + 1)
-
-#define ISYS_CTRL_PORT_ID_CAPT_A                  0 /* device ID for capture unit A      */
-#define ISYS_CTRL_PORT_ID_CAPT_B                  1 /* device ID for capture unit B      */
-#define ISYS_CTRL_PORT_ID_CAPT_C                  2 /* device ID for capture unit C      */
-#define ISYS_CTRL_PORT_ID_ACQUISITION             3 /* device ID for acquistion unit     */
-#define ISYS_CTRL_PORT_ID_DMA_CAPT_A              4 /* device ID for dma unit            */
-#define ISYS_CTRL_PORT_ID_DMA_CAPT_B              5 /* device ID for dma unit            */
-#define ISYS_CTRL_PORT_ID_DMA_CAPT_C              6 /* device ID for dma unit            */
-#define ISYS_CTRL_PORT_ID_DMA_ACQ                 7 /* device ID for dma unit            */
-
-#define ISYS_CTRL_NO_ACQ_ACK                      16 /* no ack from acquisition unit */
-#define ISYS_CTRL_NO_DMA_ACK                      0 
-#define ISYS_CTRL_NO_CAPT_ACK                     16
-
-#endif /* _input_system_ctrl_defs_h */ 
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_system_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_system_defs.h
deleted file mode 100644
index ae62163034a6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/input_system_defs.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_system_defs_h
-#define _input_system_defs_h
-
-/* csi controller modes */
-#define HIVE_CSI_CONFIG_MAIN                   0
-#define HIVE_CSI_CONFIG_STEREO1                4
-#define HIVE_CSI_CONFIG_STEREO2                8
-
-/* general purpose register IDs */
-
-/* Stream Multicast select modes */
-#define HIVE_ISYS_GPREG_MULTICAST_A_IDX           0
-#define HIVE_ISYS_GPREG_MULTICAST_B_IDX           1
-#define HIVE_ISYS_GPREG_MULTICAST_C_IDX           2
-
-/* Stream Mux select modes */
-#define HIVE_ISYS_GPREG_MUX_IDX                   3
-
-/* streaming monitor status and control */
-#define HIVE_ISYS_GPREG_STRMON_STAT_IDX           4
-#define HIVE_ISYS_GPREG_STRMON_COND_IDX           5
-#define HIVE_ISYS_GPREG_STRMON_IRQ_EN_IDX         6
-#define HIVE_ISYS_GPREG_SRST_IDX                  7
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_IDX          8
-#define HIVE_ISYS_GPREG_REG_PORT_A_IDX            9
-#define HIVE_ISYS_GPREG_REG_PORT_B_IDX            10
-
-/* Bit numbers of the soft reset register */
-#define HIVE_ISYS_GPREG_SRST_CAPT_FIFO_A_BIT      0
-#define HIVE_ISYS_GPREG_SRST_CAPT_FIFO_B_BIT      1
-#define HIVE_ISYS_GPREG_SRST_CAPT_FIFO_C_BIT      2
-#define HIVE_ISYS_GPREG_SRST_MULTICAST_A_BIT      3
-#define HIVE_ISYS_GPREG_SRST_MULTICAST_B_BIT      4
-#define HIVE_ISYS_GPREG_SRST_MULTICAST_C_BIT      5
-#define HIVE_ISYS_GPREG_SRST_CAPT_A_BIT           6
-#define HIVE_ISYS_GPREG_SRST_CAPT_B_BIT           7
-#define HIVE_ISYS_GPREG_SRST_CAPT_C_BIT           8
-#define HIVE_ISYS_GPREG_SRST_ACQ_BIT              9
-/* For ISYS_CTRL 5bits are defined to allow soft-reset per sub-controller and top-ctrl */
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_BIT        10  /*LSB for 5bit vector */
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_CAPT_A_BIT 10
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_CAPT_B_BIT 11
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_CAPT_C_BIT 12
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_ACQ_BIT    13
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_TOP_BIT    14
-/* -- */
-#define HIVE_ISYS_GPREG_SRST_STR_MUX_BIT          15
-#define HIVE_ISYS_GPREG_SRST_CIO2AHB_BIT          16
-#define HIVE_ISYS_GPREG_SRST_GEN_SHORT_FIFO_BIT   17
-#define HIVE_ISYS_GPREG_SRST_WIDE_BUS_BIT         18 // includes CIO conv
-#define HIVE_ISYS_GPREG_SRST_DMA_BIT              19
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_CAPT_A_BIT   20
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_CAPT_B_BIT   21
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_CAPT_C_BIT   22
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_ACQ_BIT      23
-#define HIVE_ISYS_GPREG_SRST_CSI_BE_OUT_BIT       24
-
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_CAPT_A_BIT    0
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_CAPT_B_BIT    1
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_CAPT_C_BIT    2
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_ACQ_BIT       3
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_DMA_BIT        4
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_ISYS_CTRL_BIT  5
-
-/* streaming monitor port id's */
-#define HIVE_ISYS_STR_MON_PORT_CAPA            0
-#define HIVE_ISYS_STR_MON_PORT_CAPB            1
-#define HIVE_ISYS_STR_MON_PORT_CAPC            2
-#define HIVE_ISYS_STR_MON_PORT_ACQ             3
-#define HIVE_ISYS_STR_MON_PORT_CSS_GENSH       4
-#define HIVE_ISYS_STR_MON_PORT_SF_GENSH        5
-#define HIVE_ISYS_STR_MON_PORT_SP2ISYS         6
-#define HIVE_ISYS_STR_MON_PORT_ISYS2SP         7
-#define HIVE_ISYS_STR_MON_PORT_PIXA            8
-#define HIVE_ISYS_STR_MON_PORT_PIXB            9
-
-/* interrupt bit ID's        */
-#define HIVE_ISYS_IRQ_CSI_SOF_BIT_ID           0
-#define HIVE_ISYS_IRQ_CSI_EOF_BIT_ID           1
-#define HIVE_ISYS_IRQ_CSI_SOL_BIT_ID           2
-#define HIVE_ISYS_IRQ_CSI_EOL_BIT_ID           3
-#define HIVE_ISYS_IRQ_CSI_RECEIVER_BIT_ID      4
-#define HIVE_ISYS_IRQ_CSI_RECEIVER_BE_BIT_ID   5
-#define HIVE_ISYS_IRQ_CAP_UNIT_A_NO_SOP        6
-#define HIVE_ISYS_IRQ_CAP_UNIT_A_LATE_SOP      7
-/*#define HIVE_ISYS_IRQ_CAP_UNIT_A_UNDEF_PH      7*/
-#define HIVE_ISYS_IRQ_CAP_UNIT_B_NO_SOP        8
-#define HIVE_ISYS_IRQ_CAP_UNIT_B_LATE_SOP      9
-/*#define HIVE_ISYS_IRQ_CAP_UNIT_B_UNDEF_PH     10*/
-#define HIVE_ISYS_IRQ_CAP_UNIT_C_NO_SOP       10
-#define HIVE_ISYS_IRQ_CAP_UNIT_C_LATE_SOP     11
-/*#define HIVE_ISYS_IRQ_CAP_UNIT_C_UNDEF_PH     13*/
-#define HIVE_ISYS_IRQ_ACQ_UNIT_SOP_MISMATCH   12
-/*#define HIVE_ISYS_IRQ_ACQ_UNIT_UNDEF_PH       15*/
-#define HIVE_ISYS_IRQ_INP_CTRL_CAPA           13
-#define HIVE_ISYS_IRQ_INP_CTRL_CAPB           14
-#define HIVE_ISYS_IRQ_INP_CTRL_CAPC           15
-#define HIVE_ISYS_IRQ_CIO2AHB                 16
-#define HIVE_ISYS_IRQ_DMA_BIT_ID              17
-#define HIVE_ISYS_IRQ_STREAM_MON_BIT_ID       18
-#define HIVE_ISYS_IRQ_NUM_BITS                19
-
-/* DMA */
-#define HIVE_ISYS_DMA_CHANNEL                  0
-#define HIVE_ISYS_DMA_IBUF_DDR_CONN            0
-#define HIVE_ISYS_DMA_HEIGHT                   1
-#define HIVE_ISYS_DMA_ELEMS                    1 /* both master buses of same width */
-#define HIVE_ISYS_DMA_STRIDE                   0 /* no stride required as height is fixed to 1 */
-#define HIVE_ISYS_DMA_CROP                     0 /* no cropping */
-#define HIVE_ISYS_DMA_EXTENSION                0 /* no extension as elem width is same on both side */
-
-#endif /* _input_system_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/irq_controller_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/irq_controller_defs.h
deleted file mode 100644
index ec6dd4487158..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/irq_controller_defs.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _irq_controller_defs_h
-#define _irq_controller_defs_h
-
-#define _HRT_IRQ_CONTROLLER_EDGE_REG_IDX           0
-#define _HRT_IRQ_CONTROLLER_MASK_REG_IDX           1
-#define _HRT_IRQ_CONTROLLER_STATUS_REG_IDX         2
-#define _HRT_IRQ_CONTROLLER_CLEAR_REG_IDX          3
-#define _HRT_IRQ_CONTROLLER_ENABLE_REG_IDX         4
-#define _HRT_IRQ_CONTROLLER_EDGE_NOT_PULSE_REG_IDX 5
-#define _HRT_IRQ_CONTROLLER_STR_OUT_ENABLE_REG_IDX 6
-
-#define _HRT_IRQ_CONTROLLER_REG_ALIGN 4
-
-#endif /* _irq_controller_defs_h */   
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp2400_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp2400_support.h
deleted file mode 100644
index e00bc841d0f0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp2400_support.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _isp2400_support_h
-#define _isp2400_support_h
-
-#ifndef ISP2400_VECTOR_TYPES
-/* This typedef is to be able to include hive header files
-   in the host code which is useful in crun */
-typedef char *tmemvectors, *tmemvectoru, *tvector;
-#endif
-
-#define hrt_isp_vamem1_store_16(cell, addr, val) hrt_mem_store_16(cell, HRT_PROC_TYPE_PROP(cell, _simd_vamem1), addr, val)
-#define hrt_isp_vamem2_store_16(cell, addr, val) hrt_mem_store_16(cell, HRT_PROC_TYPE_PROP(cell, _simd_vamem2), addr, val)
-
-#define hrt_isp_dmem(cell) HRT_PROC_TYPE_PROP(cell, _base_dmem)
-#define hrt_isp_vmem(cell) HRT_PROC_TYPE_PROP(cell, _simd_vmem)
-
-#define hrt_isp_dmem_master_port_address(cell) hrt_mem_master_port_address(cell, hrt_isp_dmem(cell))
-#define hrt_isp_vmem_master_port_address(cell) hrt_mem_master_port_address(cell, hrt_isp_vmem(cell))
-
-#if ISP_HAS_HIST
-  #define hrt_isp_hist(cell) HRT_PROC_TYPE_PROP(cell, _simd_histogram)
-  #define hrt_isp_hist_master_port_address(cell) hrt_mem_master_port_address(cell, hrt_isp_hist(cell))
-#endif
-
-#endif /* _isp2400_support_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp2401_mamoiada_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp2401_mamoiada_params.h
deleted file mode 100644
index 033e23bcf672..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp2401_mamoiada_params.h
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Version */
-#define RTL_VERSION
-
-/* Cell name  */
-#define ISP_CELL_TYPE                          isp2401_mamoiada
-#define ISP_VMEM                               simd_vmem
-#define _HRT_ISP_VMEM                          isp2401_mamoiada_simd_vmem
-
-/* instruction pipeline depth */
-#define ISP_BRANCHDELAY                        5
-
-/* bus */
-#define ISP_BUS_WIDTH                          32
-#define ISP_BUS_ADDR_WIDTH                     32
-#define ISP_BUS_BURST_SIZE                     1
-
-/* data-path */
-#define ISP_SCALAR_WIDTH                       32
-#define ISP_SLICE_NELEMS                       4
-#define ISP_VEC_NELEMS                         64
-#define ISP_VEC_ELEMBITS                       14
-#define ISP_VEC_ELEM8BITS                      16
-#define ISP_CLONE_DATAPATH_IS_16               1
-
-/* memories */
-#define ISP_DMEM_DEPTH                         4096
-#define ISP_DMEM_BSEL_DOWNSAMPLE               8
-#define ISP_VMEM_DEPTH                         3072
-#define ISP_VMEM_BSEL_DOWNSAMPLE               8
-#define ISP_VMEM_ELEMBITS                      14
-#define ISP_VMEM_ELEM_PRECISION                14
-#define ISP_VMEM_IS_BAMEM                      1
-#if ISP_VMEM_IS_BAMEM
-  #define ISP_VMEM_BAMEM_MAX_BOI_HEIGHT        8
-  #define ISP_VMEM_BAMEM_LATENCY               5
-  #define ISP_VMEM_BAMEM_BANK_NARROWING_FACTOR 2
-  #define ISP_VMEM_BAMEM_NR_DATA_PLANES        8
-  #define ISP_VMEM_BAMEM_NR_CFG_REGISTERS      16
-  #define ISP_VMEM_BAMEM_LININT                0
-  #define ISP_VMEM_BAMEM_DAP_BITS              3
-  #define ISP_VMEM_BAMEM_LININT_FRAC_BITS      0
-  #define ISP_VMEM_BAMEM_PID_BITS              3
-  #define ISP_VMEM_BAMEM_OFFSET_BITS           19
-  #define ISP_VMEM_BAMEM_ADDRESS_BITS          25
-  #define ISP_VMEM_BAMEM_RID_BITS              4
-  #define ISP_VMEM_BAMEM_TRANSPOSITION         1
-  #define ISP_VMEM_BAMEM_VEC_PLUS_SLICE        1
-  #define ISP_VMEM_BAMEM_ARB_SERVICE_CYCLE_BITS 1
-  #define ISP_VMEM_BAMEM_LUT_ELEMS             16
-  #define ISP_VMEM_BAMEM_LUT_ADDR_WIDTH        14
-  #define ISP_VMEM_BAMEM_HALF_BLOCK_WRITE      1
-  #define ISP_VMEM_BAMEM_SMART_FETCH           1
-  #define ISP_VMEM_BAMEM_BIG_ENDIANNESS        0
-#endif /* ISP_VMEM_IS_BAMEM */
-#define ISP_PMEM_DEPTH                         2048
-#define ISP_PMEM_WIDTH                         640
-#define ISP_VAMEM_ADDRESS_BITS                 12
-#define ISP_VAMEM_ELEMBITS                     12
-#define ISP_VAMEM_DEPTH                        2048
-#define ISP_VAMEM_ALIGNMENT                    2
-#define ISP_VA_ADDRESS_WIDTH                   896
-#define ISP_VEC_VALSU_LATENCY                  ISP_VEC_NELEMS
-#define ISP_HIST_ADDRESS_BITS                  12
-#define ISP_HIST_ALIGNMENT                     4
-#define ISP_HIST_COMP_IN_PREC                  12
-#define ISP_HIST_DEPTH                         1024
-#define ISP_HIST_WIDTH                         24
-#define ISP_HIST_COMPONENTS                    4
-
-/* program counter */
-#define ISP_PC_WIDTH                           13
-
-/* Template switches */
-#define ISP_SHIELD_INPUT_DMEM                  0
-#define ISP_SHIELD_OUTPUT_DMEM                 1
-#define ISP_SHIELD_INPUT_VMEM                  0
-#define ISP_SHIELD_OUTPUT_VMEM                 0
-#define ISP_SHIELD_INPUT_PMEM                  1
-#define ISP_SHIELD_OUTPUT_PMEM                 1
-#define ISP_SHIELD_INPUT_HIST                  1
-#define ISP_SHIELD_OUTPUT_HIST                 1
-/* When LUT is select the shielding is always on */
-#define ISP_SHIELD_INPUT_VAMEM                 1
-#define ISP_SHIELD_OUTPUT_VAMEM                1
-
-#define ISP_HAS_IRQ                            1
-#define ISP_HAS_SOFT_RESET                     1
-#define ISP_HAS_VEC_DIV                        0
-#define ISP_HAS_VFU_W_2O                       1
-#define ISP_HAS_DEINT3                         1
-#define ISP_HAS_LUT                            1
-#define ISP_HAS_HIST                           1
-#define ISP_HAS_VALSU                          1
-#define ISP_HAS_3rdVALSU                       1
-#define ISP_VRF1_HAS_2P                        1
-
-#define ISP_SRU_GUARDING                       1
-#define ISP_VLSU_GUARDING                      1
-
-#define ISP_VRF_RAM     	                     1
-#define ISP_SRF_RAM     	                     1
-
-#define ISP_SPLIT_VMUL_VADD_IS                 0
-#define ISP_RFSPLIT_FPGA                       0
-
-/* RSN or Bus pipelining */
-#define ISP_RSN_PIPE                           1
-#define ISP_VSF_BUS_PIPE                       0
-
-/* extra slave port to vmem */
-#define ISP_IF_VMEM                            0
-#define ISP_GDC_VMEM                           0
-
-/* Streaming ports */
-#define ISP_IF                                 1
-#define ISP_IF_B                               1
-#define ISP_GDC                                1
-#define ISP_SCL                                1
-#define ISP_GPFIFO                             1
-#define ISP_SP                                 1
-
-/* Removing Issue Slot(s) */
-#define ISP_HAS_NOT_SIMD_IS2                   0
-#define ISP_HAS_NOT_SIMD_IS3                   0
-#define ISP_HAS_NOT_SIMD_IS4                   0
-#define ISP_HAS_NOT_SIMD_IS4_VADD              0
-#define ISP_HAS_NOT_SIMD_IS5                   0
-#define ISP_HAS_NOT_SIMD_IS6                   0
-#define ISP_HAS_NOT_SIMD_IS7                   0
-#define ISP_HAS_NOT_SIMD_IS8                   0
-
-/* ICache  */
-#define ISP_ICACHE                             1
-#define ISP_ICACHE_ONLY                        0
-#define ISP_ICACHE_PREFETCH                    1
-#define ISP_ICACHE_INDEX_BITS                  8
-#define ISP_ICACHE_SET_BITS                    5
-#define ISP_ICACHE_BLOCKS_PER_SET_BITS         1
-
-/* Experimental Flags */
-#define ISP_EXP_1                              0
-#define ISP_EXP_2                              0
-#define ISP_EXP_3                              0
-#define ISP_EXP_4                              0
-#define ISP_EXP_5                              0
-#define ISP_EXP_6                              0
-
-/* Derived values */
-#define ISP_LOG2_PMEM_WIDTH                    10
-#define ISP_VEC_WIDTH                          896
-#define ISP_SLICE_WIDTH                        56
-#define ISP_VMEM_WIDTH                         896
-#define ISP_VMEM_ALIGN                         128
-#if ISP_VMEM_IS_BAMEM
-  #define ISP_VMEM_ALIGN_ELEM                  2
-#endif /* ISP_VMEM_IS_BAMEM */
-#define ISP_SIMDLSU                            1
-#define ISP_LSU_IMM_BITS                       12
-
-/* convenient shortcuts for software*/
-#define ISP_NWAY                               ISP_VEC_NELEMS
-#define NBITS                                  ISP_VEC_ELEMBITS
-
-#define _isp_ceil_div(a,b)                     (((a)+(b)-1)/(b))
-
-#ifdef C_RUN
-#define ISP_VEC_ALIGN                          (_isp_ceil_div(ISP_VEC_WIDTH, 64)*8)
-#else
-#define ISP_VEC_ALIGN                          ISP_VMEM_ALIGN
-#endif
-
-/* HRT specific vector support */
-#define isp2401_mamoiada_vector_alignment         ISP_VEC_ALIGN
-#define isp2401_mamoiada_vector_elem_bits         ISP_VMEM_ELEMBITS
-#define isp2401_mamoiada_vector_elem_precision    ISP_VMEM_ELEM_PRECISION
-#define isp2401_mamoiada_vector_num_elems         ISP_VEC_NELEMS
-
-/* register file sizes */
-#define ISP_RF0_SIZE        64
-#define ISP_RF1_SIZE        16
-#define ISP_RF2_SIZE        64
-#define ISP_RF3_SIZE        4
-#define ISP_RF4_SIZE        64
-#define ISP_RF5_SIZE        16
-#define ISP_RF6_SIZE        16
-#define ISP_RF7_SIZE        16
-#define ISP_RF8_SIZE        16
-#define ISP_RF9_SIZE        16
-#define ISP_RF10_SIZE       16
-#define ISP_RF11_SIZE       16
-#define ISP_VRF1_SIZE       32
-#define ISP_VRF2_SIZE       32
-#define ISP_VRF3_SIZE       32
-#define ISP_VRF4_SIZE       32
-#define ISP_VRF5_SIZE       32
-#define ISP_VRF6_SIZE       32
-#define ISP_VRF7_SIZE       32
-#define ISP_VRF8_SIZE       32
-#define ISP_SRF1_SIZE       4
-#define ISP_SRF2_SIZE       64
-#define ISP_SRF3_SIZE       64
-#define ISP_SRF4_SIZE       32
-#define ISP_SRF5_SIZE       64
-#define ISP_FRF0_SIZE       16
-#define ISP_FRF1_SIZE       4
-#define ISP_FRF2_SIZE       16
-#define ISP_FRF3_SIZE       4
-#define ISP_FRF4_SIZE       4
-#define ISP_FRF5_SIZE       8
-#define ISP_FRF6_SIZE       4
-/* register file read latency */
-#define ISP_VRF1_READ_LAT       1
-#define ISP_VRF2_READ_LAT       1
-#define ISP_VRF3_READ_LAT       1
-#define ISP_VRF4_READ_LAT       1
-#define ISP_VRF5_READ_LAT       1
-#define ISP_VRF6_READ_LAT       1
-#define ISP_VRF7_READ_LAT       1
-#define ISP_VRF8_READ_LAT       1
-#define ISP_SRF1_READ_LAT       1
-#define ISP_SRF2_READ_LAT       1
-#define ISP_SRF3_READ_LAT       1
-#define ISP_SRF4_READ_LAT       1
-#define ISP_SRF5_READ_LAT       1
-#define ISP_SRF5_READ_LAT       1
-/* immediate sizes */
-#define ISP_IS1_IMM_BITS        14
-#define ISP_IS2_IMM_BITS        13
-#define ISP_IS3_IMM_BITS        14
-#define ISP_IS4_IMM_BITS        14
-#define ISP_IS5_IMM_BITS        9
-#define ISP_IS6_IMM_BITS        16
-#define ISP_IS7_IMM_BITS        9
-#define ISP_IS8_IMM_BITS        16
-#define ISP_IS9_IMM_BITS        11
-/* fifo depths */
-#define ISP_IF_FIFO_DEPTH         0
-#define ISP_IF_B_FIFO_DEPTH       0
-#define ISP_DMA_FIFO_DEPTH        0
-#define ISP_OF_FIFO_DEPTH         0
-#define ISP_GDC_FIFO_DEPTH        0
-#define ISP_SCL_FIFO_DEPTH        0
-#define ISP_GPFIFO_FIFO_DEPTH     0
-#define ISP_SP_FIFO_DEPTH         0
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp_acquisition_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp_acquisition_defs.h
deleted file mode 100644
index 593620721627..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp_acquisition_defs.h
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _isp_acquisition_defs_h
-#define _isp_acquisition_defs_h
-
-#define _ISP_ACQUISITION_REG_ALIGN                4  /* assuming 32 bit control bus width */
-#define _ISP_ACQUISITION_BYTES_PER_ELEM           4		
-
-/* --------------------------------------------------*/
-
-#define NOF_ACQ_IRQS                              1
-
-/* --------------------------------------------------*/
-/* FSM */
-/* --------------------------------------------------*/
-#define MEM2STREAM_FSM_STATE_BITS                 2
-#define ACQ_SYNCHRONIZER_FSM_STATE_BITS           2
-
-/* --------------------------------------------------*/
-/* REGISTER INFO */
-/* --------------------------------------------------*/
-
-#define NOF_ACQ_REGS                              12      
-
-// Register id's of MMIO slave accesible registers
-#define ACQ_START_ADDR_REG_ID                     0              
-#define ACQ_MEM_REGION_SIZE_REG_ID                1
-#define ACQ_NUM_MEM_REGIONS_REG_ID                2
-#define ACQ_INIT_REG_ID                           3 
-#define ACQ_RECEIVED_SHORT_PACKETS_REG_ID         4
-#define ACQ_RECEIVED_LONG_PACKETS_REG_ID          5
-#define ACQ_LAST_COMMAND_REG_ID                   6
-#define ACQ_NEXT_COMMAND_REG_ID                   7
-#define ACQ_LAST_ACKNOWLEDGE_REG_ID               8
-#define ACQ_NEXT_ACKNOWLEDGE_REG_ID               9
-#define ACQ_FSM_STATE_INFO_REG_ID                 10
-#define ACQ_INT_CNTR_INFO_REG_ID                  11
- 
-// Register width
-#define ACQ_START_ADDR_REG_WIDTH                  9               
-#define ACQ_MEM_REGION_SIZE_REG_WIDTH             9  
-#define ACQ_NUM_MEM_REGIONS_REG_WIDTH             9  
-#define ACQ_INIT_REG_WIDTH                        3  
-#define ACQ_RECEIVED_SHORT_PACKETS_REG_WIDTH      32 
-#define ACQ_RECEIVED_LONG_PACKETS_REG_WIDTH       32  
-#define ACQ_LAST_COMMAND_REG_WIDTH                32  
-#define ACQ_NEXT_COMMAND_REG_WIDTH                32  
-#define ACQ_LAST_ACKNOWLEDGE_REG_WIDTH            32  
-#define ACQ_NEXT_ACKNOWLEDGE_REG_WIDTH            32  
-#define ACQ_FSM_STATE_INFO_REG_WIDTH              ((MEM2STREAM_FSM_STATE_BITS * 3) + (ACQ_SYNCHRONIZER_FSM_STATE_BITS *3))
-#define ACQ_INT_CNTR_INFO_REG_WIDTH               32
-
-/* register reset value */
-#define ACQ_START_ADDR_REG_RSTVAL                 0              
-#define ACQ_MEM_REGION_SIZE_REG_RSTVAL            128
-#define ACQ_NUM_MEM_REGIONS_REG_RSTVAL            3
-#define ACQ_INIT_REG_RSTVAL                       0                           
-#define ACQ_RECEIVED_SHORT_PACKETS_REG_RSTVAL     0
-#define ACQ_RECEIVED_LONG_PACKETS_REG_RSTVAL      0
-#define ACQ_LAST_COMMAND_REG_RSTVAL               0
-#define ACQ_NEXT_COMMAND_REG_RSTVAL               0
-#define ACQ_LAST_ACKNOWLEDGE_REG_RSTVAL           0
-#define ACQ_NEXT_ACKNOWLEDGE_REG_RSTVAL           0 
-#define ACQ_FSM_STATE_INFO_REG_RSTVAL             0
-#define ACQ_INT_CNTR_INFO_REG_RSTVAL              0 
-
-/* bit definitions */
-#define ACQ_INIT_RST_REG_BIT                      0
-#define ACQ_INIT_RESYNC_BIT                       2
-#define ACQ_INIT_RST_IDX                          ACQ_INIT_RST_REG_BIT
-#define ACQ_INIT_RST_BITS                         1
-#define ACQ_INIT_RESYNC_IDX                       ACQ_INIT_RESYNC_BIT
-#define ACQ_INIT_RESYNC_BITS                      1
-
-/* --------------------------------------------------*/
-/* TOKEN INFO */
-/* --------------------------------------------------*/
-#define ACQ_TOKEN_ID_LSB                          0
-#define ACQ_TOKEN_ID_MSB                          3            
-#define ACQ_TOKEN_WIDTH                           (ACQ_TOKEN_ID_MSB - ACQ_TOKEN_ID_LSB  + 1) // 4
-#define ACQ_TOKEN_ID_IDX                          0
-#define ACQ_TOKEN_ID_BITS                         ACQ_TOKEN_WIDTH
-#define ACQ_INIT_CMD_INIT_IDX                     4
-#define ACQ_INIT_CMD_INIT_BITS                    3
-#define ACQ_CMD_START_ADDR_IDX                    4
-#define ACQ_CMD_START_ADDR_BITS                   9
-#define ACQ_CMD_NOFWORDS_IDX                      13
-#define ACQ_CMD_NOFWORDS_BITS                     9  
-#define ACQ_MEM_REGION_ID_IDX                     22
-#define ACQ_MEM_REGION_ID_BITS                    9 
-#define ACQ_PACKET_LENGTH_TOKEN_MSB               21
-#define ACQ_PACKET_LENGTH_TOKEN_LSB               13
-#define ACQ_PACKET_DATA_FORMAT_ID_TOKEN_MSB       9
-#define ACQ_PACKET_DATA_FORMAT_ID_TOKEN_LSB       4
-#define ACQ_PACKET_CH_ID_TOKEN_MSB                11
-#define ACQ_PACKET_CH_ID_TOKEN_LSB                10
-#define ACQ_PACKET_MEM_REGION_ID_TOKEN_MSB        12		/* only for capt_end_of_packet_written */
-#define ACQ_PACKET_MEM_REGION_ID_TOKEN_LSB        4		/* only for capt_end_of_packet_written */
-
-
-/* Command tokens IDs */
-#define ACQ_READ_REGION_AUTO_INCR_TOKEN_ID        0 //0000b
-#define ACQ_READ_REGION_TOKEN_ID                  1 //0001b
-#define ACQ_READ_REGION_SOP_TOKEN_ID              2 //0010b  
-#define ACQ_INIT_TOKEN_ID                         8 //1000b
-
-/* Acknowledge token IDs */
-#define ACQ_READ_REGION_ACK_TOKEN_ID              0 //0000b
-#define ACQ_END_OF_PACKET_TOKEN_ID                4 //0100b
-#define ACQ_END_OF_REGION_TOKEN_ID                5 //0101b
-#define ACQ_SOP_MISMATCH_TOKEN_ID                 6 //0110b
-#define ACQ_UNDEF_PH_TOKEN_ID                     7 //0111b
-
-#define ACQ_TOKEN_MEMREGIONID_MSB                 30
-#define ACQ_TOKEN_MEMREGIONID_LSB                 22
-#define ACQ_TOKEN_NOFWORDS_MSB                    21
-#define ACQ_TOKEN_NOFWORDS_LSB                    13
-#define ACQ_TOKEN_STARTADDR_MSB                   12
-#define ACQ_TOKEN_STARTADDR_LSB                   4  
-
-
-/* --------------------------------------------------*/
-/* MIPI */
-/* --------------------------------------------------*/
-
-#define WORD_COUNT_WIDTH                          16
-#define PKT_CODE_WIDTH                            6            
-#define CHN_NO_WIDTH                              2  
-#define ERROR_INFO_WIDTH                          8
-  
-#define LONG_PKTCODE_MAX                          63
-#define LONG_PKTCODE_MIN                          16
-#define SHORT_PKTCODE_MAX                         15
-
-#define EOF_CODE                                  1
-
-/* --------------------------------------------------*/
-/* Packet Info */
-/* --------------------------------------------------*/
-#define ACQ_START_OF_FRAME                        0
-#define ACQ_END_OF_FRAME                          1
-#define ACQ_START_OF_LINE                         2
-#define ACQ_END_OF_LINE                           3
-#define ACQ_LINE_PAYLOAD                          4
-#define ACQ_GEN_SH_PKT                            5
-
-
-/* bit definition */
-#define ACQ_PKT_TYPE_IDX                          16
-#define ACQ_PKT_TYPE_BITS                         6
-#define ACQ_PKT_SOP_IDX                           32
-#define ACQ_WORD_CNT_IDX                          0
-#define ACQ_WORD_CNT_BITS                         16
-#define ACQ_PKT_INFO_IDX                          16
-#define ACQ_PKT_INFO_BITS                         8
-#define ACQ_HEADER_DATA_IDX                       0
-#define ACQ_HEADER_DATA_BITS                      16
-#define ACQ_ACK_TOKEN_ID_IDX                      ACQ_TOKEN_ID_IDX
-#define ACQ_ACK_TOKEN_ID_BITS                     ACQ_TOKEN_ID_BITS
-#define ACQ_ACK_NOFWORDS_IDX                      13
-#define ACQ_ACK_NOFWORDS_BITS                     9
-#define ACQ_ACK_PKT_LEN_IDX                       4
-#define ACQ_ACK_PKT_LEN_BITS                      16
-
-
-/* --------------------------------------------------*/
-/* Packet Data Type */
-/* --------------------------------------------------*/
-
-
-#define ACQ_YUV420_8_DATA                       24   /* 01 1000 YUV420 8-bit                                        */
-#define ACQ_YUV420_10_DATA                      25   /* 01 1001  YUV420 10-bit                                      */
-#define ACQ_YUV420_8L_DATA                      26   /* 01 1010   YUV420 8-bit legacy                               */
-#define ACQ_YUV422_8_DATA                       30   /* 01 1110   YUV422 8-bit                                      */
-#define ACQ_YUV422_10_DATA                      31   /* 01 1111   YUV422 10-bit                                     */
-#define ACQ_RGB444_DATA                         32   /* 10 0000   RGB444                                            */
-#define ACQ_RGB555_DATA              					 33   /* 10 0001   RGB555                                            */
-#define ACQ_RGB565_DATA              					 34   /* 10 0010   RGB565                                            */
-#define ACQ_RGB666_DATA              					 35   /* 10 0011   RGB666                                            */
-#define ACQ_RGB888_DATA              					 36   /* 10 0100   RGB888                                            */
-#define ACQ_RAW6_DATA                					 40   /* 10 1000   RAW6                                              */
-#define ACQ_RAW7_DATA                					 41   /* 10 1001   RAW7                                              */
-#define ACQ_RAW8_DATA                					 42   /* 10 1010   RAW8                                              */
-#define ACQ_RAW10_DATA               					 43   /* 10 1011   RAW10                                             */
-#define ACQ_RAW12_DATA               					 44   /* 10 1100   RAW12                                             */
-#define ACQ_RAW14_DATA               					 45   /* 10 1101   RAW14                                             */
-#define ACQ_USR_DEF_1_DATA           					 48   /* 11 0000    JPEG [User Defined 8-bit Data Type 1]            */
-#define ACQ_USR_DEF_2_DATA           					 49   /* 11 0001    User Defined 8-bit Data Type 2                   */
-#define ACQ_USR_DEF_3_DATA           					 50   /* 11 0010    User Defined 8-bit Data Type 3                   */
-#define ACQ_USR_DEF_4_DATA           					 51   /* 11 0011    User Defined 8-bit Data Type 4                   */
-#define ACQ_USR_DEF_5_DATA           					 52   /* 11 0100    User Defined 8-bit Data Type 5                   */
-#define ACQ_USR_DEF_6_DATA           					 53   /* 11 0101    User Defined 8-bit Data Type 6                   */
-#define ACQ_USR_DEF_7_DATA           					 54   /* 11 0110    User Defined 8-bit Data Type 7                   */
-#define ACQ_USR_DEF_8_DATA           					 55   /* 11 0111    User Defined 8-bit Data Type 8                   */
-#define ACQ_Emb_DATA                 					 18   /* 01 0010    embedded eight bit non image data                */
-#define ACQ_SOF_DATA                 					 0   /* 00 0000    frame start                                      */
-#define ACQ_EOF_DATA                 					 1   /* 00 0001    frame end                                        */
-#define ACQ_SOL_DATA                 					 2   /* 00 0010    line start                                       */
-#define ACQ_EOL_DATA                 					 3   /* 00 0011    line end                                         */
-#define ACQ_GEN_SH1_DATA             					 8   /* 00 1000  Generic Short Packet Code 1                        */
-#define ACQ_GEN_SH2_DATA             					 9   /* 00 1001    Generic Short Packet Code 2                      */
-#define ACQ_GEN_SH3_DATA             					 10   /* 00 1010    Generic Short Packet Code 3                      */
-#define ACQ_GEN_SH4_DATA             					 11   /* 00 1011    Generic Short Packet Code 4                      */
-#define ACQ_GEN_SH5_DATA             					 12   /* 00 1100    Generic Short Packet Code 5                      */
-#define ACQ_GEN_SH6_DATA             					 13   /* 00 1101    Generic Short Packet Code 6                      */
-#define ACQ_GEN_SH7_DATA             					 14   /* 00 1110    Generic Short Packet Code 7                      */
-#define ACQ_GEN_SH8_DATA             					 15   /* 00 1111    Generic Short Packet Code 8                      */
-#define ACQ_YUV420_8_CSPS_DATA       					 28   /* 01 1100   YUV420 8-bit (Chroma Shifted Pixel Sampling)      */
-#define ACQ_YUV420_10_CSPS_DATA      					 29   /* 01 1101   YUV420 10-bit (Chroma Shifted Pixel Sampling)     */
-#define ACQ_RESERVED_DATA_TYPE_MIN              56
-#define ACQ_RESERVED_DATA_TYPE_MAX              63
-#define ACQ_GEN_LONG_RESERVED_DATA_TYPE_MIN     19
-#define ACQ_GEN_LONG_RESERVED_DATA_TYPE_MAX     23
-#define ACQ_YUV_RESERVED_DATA_TYPE              27
-#define ACQ_RGB_RESERVED_DATA_TYPE_MIN          37
-#define ACQ_RGB_RESERVED_DATA_TYPE_MAX          39
-#define ACQ_RAW_RESERVED_DATA_TYPE_MIN          46
-#define ACQ_RAW_RESERVED_DATA_TYPE_MAX          47
-
-/* --------------------------------------------------*/
-
-#endif /* _isp_acquisition_defs_h */ 
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp_capture_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp_capture_defs.h
deleted file mode 100644
index aa413df022f2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/isp_capture_defs.h
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _isp_capture_defs_h
-#define _isp_capture_defs_h
-
-#define _ISP_CAPTURE_REG_ALIGN                    4  /* assuming 32 bit control bus width */
-#define _ISP_CAPTURE_BITS_PER_ELEM                32  /* only for data, not SOP */						           
-#define _ISP_CAPTURE_BYTES_PER_ELEM               (_ISP_CAPTURE_BITS_PER_ELEM/8	)				           
-#define _ISP_CAPTURE_BYTES_PER_WORD               32		/* 256/8 */	
-#define _ISP_CAPTURE_ELEM_PER_WORD                _ISP_CAPTURE_BYTES_PER_WORD / _ISP_CAPTURE_BYTES_PER_ELEM		           
-
-//#define CAPT_RCV_ACK                              1
-//#define CAPT_WRT_ACK                              2               
-//#define CAPT_IRQ_ACK                              3                        
-
-/* --------------------------------------------------*/
-
-#define NOF_IRQS                                  2
-
-/* --------------------------------------------------*/
-/* REGISTER INFO */
-/* --------------------------------------------------*/
-
-// Number of registers
-#define CAPT_NOF_REGS                             16
-
-// Register id's of MMIO slave accesible registers
-#define CAPT_START_MODE_REG_ID                    0
-#define CAPT_START_ADDR_REG_ID                    1 
-#define CAPT_MEM_REGION_SIZE_REG_ID               2 
-#define CAPT_NUM_MEM_REGIONS_REG_ID               3 
-#define CAPT_INIT_REG_ID                          4 
-#define CAPT_START_REG_ID                         5
-#define CAPT_STOP_REG_ID                          6  
-
-#define CAPT_PACKET_LENGTH_REG_ID                 7
-#define CAPT_RECEIVED_LENGTH_REG_ID               8 
-#define CAPT_RECEIVED_SHORT_PACKETS_REG_ID        9 
-#define CAPT_RECEIVED_LONG_PACKETS_REG_ID         10 
-#define CAPT_LAST_COMMAND_REG_ID                  11        
-#define CAPT_NEXT_COMMAND_REG_ID                  12
-#define CAPT_LAST_ACKNOWLEDGE_REG_ID              13
-#define CAPT_NEXT_ACKNOWLEDGE_REG_ID              14
-#define CAPT_FSM_STATE_INFO_REG_ID                15
-
-// Register width
-#define CAPT_START_MODE_REG_WIDTH                 1 
-//#define CAPT_START_ADDR_REG_WIDTH                 9
-//#define CAPT_MEM_REGION_SIZE_REG_WIDTH            9
-//#define CAPT_NUM_MEM_REGIONS_REG_WIDTH            9
-#define CAPT_INIT_REG_WIDTH                       (22 + 4)
-
-#define CAPT_START_REG_WIDTH                      1
-#define CAPT_STOP_REG_WIDTH                       1
-
-/* --------------------------------------------------*/
-/* FSM */
-/* --------------------------------------------------*/
-#define CAPT_WRITE2MEM_FSM_STATE_BITS             2
-#define CAPT_SYNCHRONIZER_FSM_STATE_BITS          3
-
-
-#define CAPT_PACKET_LENGTH_REG_WIDTH              17
-#define CAPT_RECEIVED_LENGTH_REG_WIDTH            17   
-#define CAPT_RECEIVED_SHORT_PACKETS_REG_WIDTH     32
-#define CAPT_RECEIVED_LONG_PACKETS_REG_WIDTH      32
-#define CAPT_LAST_COMMAND_REG_WIDTH               32
-/* #define CAPT_NEXT_COMMAND_REG_WIDTH               32 */  
-#define CAPT_LAST_ACKNOWLEDGE_REG_WIDTH           32
-#define CAPT_NEXT_ACKNOWLEDGE_REG_WIDTH           32
-#define CAPT_FSM_STATE_INFO_REG_WIDTH             ((CAPT_WRITE2MEM_FSM_STATE_BITS * 3) + (CAPT_SYNCHRONIZER_FSM_STATE_BITS * 3))
-
-//#define CAPT_INIT_RESTART_MEM_ADDR_WIDTH          9   
-//#define CAPT_INIT_RESTART_MEM_REGION_WIDTH        9 
-
-/* register reset value */
-#define CAPT_START_MODE_REG_RSTVAL                0   
-#define CAPT_START_ADDR_REG_RSTVAL                0
-#define CAPT_MEM_REGION_SIZE_REG_RSTVAL           128
-#define CAPT_NUM_MEM_REGIONS_REG_RSTVAL           3 
-#define CAPT_INIT_REG_RSTVAL                      0
-
-#define CAPT_START_REG_RSTVAL                     0
-#define CAPT_STOP_REG_RSTVAL                      0
-
-#define CAPT_PACKET_LENGTH_REG_RSTVAL             0
-#define CAPT_RECEIVED_LENGTH_REG_RSTVAL           0
-#define CAPT_RECEIVED_SHORT_PACKETS_REG_RSTVAL    0
-#define CAPT_RECEIVED_LONG_PACKETS_REG_RSTVAL     0
-#define CAPT_LAST_COMMAND_REG_RSTVAL              0
-#define CAPT_NEXT_COMMAND_REG_RSTVAL              0
-#define CAPT_LAST_ACKNOWLEDGE_REG_RSTVAL          0
-#define CAPT_NEXT_ACKNOWLEDGE_REG_RSTVAL          0
-#define CAPT_FSM_STATE_INFO_REG_RSTVAL            0
-
-/* bit definitions */
-#define CAPT_INIT_RST_REG_BIT                     0
-#define CAPT_INIT_FLUSH_BIT                       1
-#define CAPT_INIT_RESYNC_BIT                      2
-#define CAPT_INIT_RESTART_BIT                     3
-#define CAPT_INIT_RESTART_MEM_ADDR_LSB            4
-#define CAPT_INIT_RESTART_MEM_ADDR_MSB            14
-#define CAPT_INIT_RESTART_MEM_REGION_LSB          15
-#define CAPT_INIT_RESTART_MEM_REGION_MSB          25
-
-
-#define CAPT_INIT_RST_REG_IDX                     CAPT_INIT_RST_REG_BIT
-#define CAPT_INIT_RST_REG_BITS                    1
-#define CAPT_INIT_FLUSH_IDX                       CAPT_INIT_FLUSH_BIT
-#define CAPT_INIT_FLUSH_BITS                      1
-#define CAPT_INIT_RESYNC_IDX                      CAPT_INIT_RESYNC_BIT
-#define CAPT_INIT_RESYNC_BITS                     1
-#define CAPT_INIT_RESTART_IDX                     CAPT_INIT_RESTART_BIT
-#define CAPT_INIT_RESTART_BITS					  				1
-#define CAPT_INIT_RESTART_MEM_ADDR_IDX            CAPT_INIT_RESTART_MEM_ADDR_LSB
-#define CAPT_INIT_RESTART_MEM_ADDR_BITS           (CAPT_INIT_RESTART_MEM_ADDR_MSB - CAPT_INIT_RESTART_MEM_ADDR_LSB + 1)
-#define CAPT_INIT_RESTART_MEM_REGION_IDX          CAPT_INIT_RESTART_MEM_REGION_LSB
-#define CAPT_INIT_RESTART_MEM_REGION_BITS         (CAPT_INIT_RESTART_MEM_REGION_MSB - CAPT_INIT_RESTART_MEM_REGION_LSB + 1)
-
-
-
-/* --------------------------------------------------*/
-/* TOKEN INFO */
-/* --------------------------------------------------*/
-#define CAPT_TOKEN_ID_LSB                         0
-#define CAPT_TOKEN_ID_MSB                         3            
-#define CAPT_TOKEN_WIDTH                         (CAPT_TOKEN_ID_MSB - CAPT_TOKEN_ID_LSB  + 1) /* 4 */
-
-/* Command tokens IDs */
-#define CAPT_START_TOKEN_ID                       0 /* 0000b */
-#define CAPT_STOP_TOKEN_ID                        1 /* 0001b */
-#define CAPT_FREEZE_TOKEN_ID                      2 /* 0010b */  
-#define CAPT_RESUME_TOKEN_ID                      3 /* 0011b */
-#define CAPT_INIT_TOKEN_ID                        8 /* 1000b */
-
-#define CAPT_START_TOKEN_BIT                      0      
-#define CAPT_STOP_TOKEN_BIT                       0
-#define CAPT_FREEZE_TOKEN_BIT                     0
-#define CAPT_RESUME_TOKEN_BIT                     0
-#define CAPT_INIT_TOKEN_BIT                       0
-
-/* Acknowledge token IDs */
-#define CAPT_END_OF_PACKET_RECEIVED_TOKEN_ID      0 /* 0000b */
-#define CAPT_END_OF_PACKET_WRITTEN_TOKEN_ID       1 /* 0001b */
-#define CAPT_END_OF_REGION_WRITTEN_TOKEN_ID       2 /* 0010b */
-#define CAPT_FLUSH_DONE_TOKEN_ID                  3 /* 0011b */
-#define CAPT_PREMATURE_SOP_TOKEN_ID               4 /* 0100b */
-#define CAPT_MISSING_SOP_TOKEN_ID                 5 /* 0101b */
-#define CAPT_UNDEF_PH_TOKEN_ID                    6 /* 0110b */
-#define CAPT_STOP_ACK_TOKEN_ID                    7 /* 0111b */
-
-#define CAPT_PACKET_LENGTH_TOKEN_MSB             19
-#define CAPT_PACKET_LENGTH_TOKEN_LSB              4
-#define CAPT_SUPER_PACKET_LENGTH_TOKEN_MSB       20
-#define CAPT_SUPER_PACKET_LENGTH_TOKEN_LSB        4
-#define CAPT_PACKET_DATA_FORMAT_ID_TOKEN_MSB     25
-#define CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB     20
-#define CAPT_PACKET_CH_ID_TOKEN_MSB              27
-#define CAPT_PACKET_CH_ID_TOKEN_LSB              26
-#define CAPT_PACKET_MEM_REGION_ID_TOKEN_MSB      29		
-#define CAPT_PACKET_MEM_REGION_ID_TOKEN_LSB      21		
-
-/*  bit definition */
-#define CAPT_CMD_IDX                              CAPT_TOKEN_ID_LSB
-#define	CAPT_CMD_BITS                             (CAPT_TOKEN_ID_MSB - CAPT_TOKEN_ID_LSB + 1)
-#define CAPT_SOP_IDX                              32
-#define CAPT_SOP_BITS                             1
-#define CAPT_PKT_INFO_IDX                         16
-#define CAPT_PKT_INFO_BITS                        8
-#define CAPT_PKT_TYPE_IDX                         0
-#define CAPT_PKT_TYPE_BITS                        6
-#define CAPT_HEADER_DATA_IDX                      0
-#define CAPT_HEADER_DATA_BITS                     16
-#define CAPT_PKT_DATA_IDX                         0
-#define CAPT_PKT_DATA_BITS                        32
-#define CAPT_WORD_CNT_IDX                         0
-#define CAPT_WORD_CNT_BITS                        16
-#define CAPT_ACK_TOKEN_ID_IDX                     0
-#define CAPT_ACK_TOKEN_ID_BITS                    4
-//#define CAPT_ACK_PKT_LEN_IDX                      CAPT_PACKET_LENGTH_TOKEN_LSB
-//#define CAPT_ACK_PKT_LEN_BITS                     (CAPT_PACKET_LENGTH_TOKEN_MSB - CAPT_PACKET_LENGTH_TOKEN_LSB + 1)
-//#define CAPT_ACK_PKT_INFO_IDX                     20
-//#define CAPT_ACK_PKT_INFO_BITS                    8
-//#define CAPT_ACK_MEM_REG_ID1_IDX                  20			/* for capt_end_of_packet_written */
-//#define CAPT_ACK_MEM_REG_ID2_IDX                  4       /* for capt_end_of_region_written */
-#define CAPT_ACK_PKT_LEN_IDX                      CAPT_PACKET_LENGTH_TOKEN_LSB
-#define CAPT_ACK_PKT_LEN_BITS                     (CAPT_PACKET_LENGTH_TOKEN_MSB - CAPT_PACKET_LENGTH_TOKEN_LSB + 1)
-#define CAPT_ACK_SUPER_PKT_LEN_IDX                CAPT_SUPER_PACKET_LENGTH_TOKEN_LSB
-#define CAPT_ACK_SUPER_PKT_LEN_BITS               (CAPT_SUPER_PACKET_LENGTH_TOKEN_MSB - CAPT_SUPER_PACKET_LENGTH_TOKEN_LSB + 1)
-#define CAPT_ACK_PKT_INFO_IDX                     CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB
-#define CAPT_ACK_PKT_INFO_BITS                    (CAPT_PACKET_CH_ID_TOKEN_MSB - CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB + 1)
-#define CAPT_ACK_MEM_REGION_ID_IDX                CAPT_PACKET_MEM_REGION_ID_TOKEN_LSB
-#define CAPT_ACK_MEM_REGION_ID_BITS               (CAPT_PACKET_MEM_REGION_ID_TOKEN_MSB - CAPT_PACKET_MEM_REGION_ID_TOKEN_LSB + 1)
-#define CAPT_ACK_PKT_TYPE_IDX                     CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB
-#define CAPT_ACK_PKT_TYPE_BITS                    (CAPT_PACKET_DATA_FORMAT_ID_TOKEN_MSB - CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB + 1)
-#define CAPT_INIT_TOKEN_INIT_IDX                  4
-#define CAPT_INIT_TOKEN_INIT_BITS                 22
-
-
-/* --------------------------------------------------*/
-/* MIPI */
-/* --------------------------------------------------*/
-
-#define CAPT_WORD_COUNT_WIDTH                     16      
-#define CAPT_PKT_CODE_WIDTH                       6                  
-#define CAPT_CHN_NO_WIDTH                         2        
-#define CAPT_ERROR_INFO_WIDTH                     8       
-
-#define LONG_PKTCODE_MAX                          63
-#define LONG_PKTCODE_MIN                          16
-#define SHORT_PKTCODE_MAX                         15
-
-
-/* --------------------------------------------------*/
-/* Packet Info */
-/* --------------------------------------------------*/
-#define CAPT_START_OF_FRAME                       0
-#define CAPT_END_OF_FRAME                         1
-#define CAPT_START_OF_LINE                        2
-#define CAPT_END_OF_LINE                          3
-#define CAPT_LINE_PAYLOAD                         4
-#define CAPT_GEN_SH_PKT                           5
-
-
-/* --------------------------------------------------*/
-/* Packet Data Type */
-/* --------------------------------------------------*/
-
-#define CAPT_YUV420_8_DATA                       24   /* 01 1000 YUV420 8-bit                                        */
-#define CAPT_YUV420_10_DATA                      25   /* 01 1001  YUV420 10-bit                                      */
-#define CAPT_YUV420_8L_DATA                      26   /* 01 1010   YUV420 8-bit legacy                               */
-#define CAPT_YUV422_8_DATA                       30   /* 01 1110   YUV422 8-bit                                      */
-#define CAPT_YUV422_10_DATA                      31   /* 01 1111   YUV422 10-bit                                     */
-#define CAPT_RGB444_DATA                         32   /* 10 0000   RGB444                                            */
-#define CAPT_RGB555_DATA              					 33   /* 10 0001   RGB555                                            */
-#define CAPT_RGB565_DATA              					 34   /* 10 0010   RGB565                                            */
-#define CAPT_RGB666_DATA              					 35   /* 10 0011   RGB666                                            */
-#define CAPT_RGB888_DATA              					 36   /* 10 0100   RGB888                                            */
-#define CAPT_RAW6_DATA                					 40   /* 10 1000   RAW6                                              */
-#define CAPT_RAW7_DATA                					 41   /* 10 1001   RAW7                                              */
-#define CAPT_RAW8_DATA                					 42   /* 10 1010   RAW8                                              */
-#define CAPT_RAW10_DATA               					 43   /* 10 1011   RAW10                                             */
-#define CAPT_RAW12_DATA               					 44   /* 10 1100   RAW12                                             */
-#define CAPT_RAW14_DATA               					 45   /* 10 1101   RAW14                                             */
-#define CAPT_USR_DEF_1_DATA           					 48   /* 11 0000    JPEG [User Defined 8-bit Data Type 1]            */
-#define CAPT_USR_DEF_2_DATA           					 49   /* 11 0001    User Defined 8-bit Data Type 2                   */
-#define CAPT_USR_DEF_3_DATA           					 50   /* 11 0010    User Defined 8-bit Data Type 3                   */
-#define CAPT_USR_DEF_4_DATA           					 51   /* 11 0011    User Defined 8-bit Data Type 4                   */
-#define CAPT_USR_DEF_5_DATA           					 52   /* 11 0100    User Defined 8-bit Data Type 5                   */
-#define CAPT_USR_DEF_6_DATA           					 53   /* 11 0101    User Defined 8-bit Data Type 6                   */
-#define CAPT_USR_DEF_7_DATA           					 54   /* 11 0110    User Defined 8-bit Data Type 7                   */
-#define CAPT_USR_DEF_8_DATA           					 55   /* 11 0111    User Defined 8-bit Data Type 8                   */
-#define CAPT_Emb_DATA                 					 18   /* 01 0010    embedded eight bit non image data                */
-#define CAPT_SOF_DATA                 					 0   /* 00 0000    frame start                                      */
-#define CAPT_EOF_DATA                 					 1   /* 00 0001    frame end                                        */
-#define CAPT_SOL_DATA                 					 2   /* 00 0010    line start                                       */
-#define CAPT_EOL_DATA                 					 3   /* 00 0011    line end                                         */
-#define CAPT_GEN_SH1_DATA             					 8   /* 00 1000  Generic Short Packet Code 1                        */
-#define CAPT_GEN_SH2_DATA             					 9   /* 00 1001    Generic Short Packet Code 2                      */
-#define CAPT_GEN_SH3_DATA             					 10   /* 00 1010    Generic Short Packet Code 3                      */
-#define CAPT_GEN_SH4_DATA             					 11   /* 00 1011    Generic Short Packet Code 4                      */
-#define CAPT_GEN_SH5_DATA             					 12   /* 00 1100    Generic Short Packet Code 5                      */
-#define CAPT_GEN_SH6_DATA             					 13   /* 00 1101    Generic Short Packet Code 6                      */
-#define CAPT_GEN_SH7_DATA             					 14   /* 00 1110    Generic Short Packet Code 7                      */
-#define CAPT_GEN_SH8_DATA             					 15   /* 00 1111    Generic Short Packet Code 8                      */
-#define CAPT_YUV420_8_CSPS_DATA       					 28   /* 01 1100   YUV420 8-bit (Chroma Shifted Pixel Sampling)      */
-#define CAPT_YUV420_10_CSPS_DATA      					 29   /* 01 1101   YUV420 10-bit (Chroma Shifted Pixel Sampling)     */
-#define CAPT_RESERVED_DATA_TYPE_MIN              56
-#define CAPT_RESERVED_DATA_TYPE_MAX              63
-#define CAPT_GEN_LONG_RESERVED_DATA_TYPE_MIN     19
-#define CAPT_GEN_LONG_RESERVED_DATA_TYPE_MAX     23
-#define CAPT_YUV_RESERVED_DATA_TYPE              27
-#define CAPT_RGB_RESERVED_DATA_TYPE_MIN          37
-#define CAPT_RGB_RESERVED_DATA_TYPE_MAX          39
-#define CAPT_RAW_RESERVED_DATA_TYPE_MIN          46
-#define CAPT_RAW_RESERVED_DATA_TYPE_MAX          47
-
-
-/* --------------------------------------------------*/
-/* Capture Unit State */
-/* --------------------------------------------------*/
-#define CAPT_FREE_RUN                             0
-#define CAPT_NO_SYNC                              1
-#define CAPT_SYNC_SWP                             2
-#define CAPT_SYNC_MWP                             3
-#define CAPT_SYNC_WAIT                            4
-#define CAPT_FREEZE                               5
-#define CAPT_RUN                                  6
-
-
-/* --------------------------------------------------*/
-
-#endif /* _isp_capture_defs_h */ 
-
-
-
-
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mipi_backend_common_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mipi_backend_common_defs.h
deleted file mode 100644
index 76705d7a2b44..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mipi_backend_common_defs.h
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _css_receiver_2400_common_defs_h_
-#define _css_receiver_2400_common_defs_h_
-#ifndef _mipi_backend_common_defs_h_
-#define _mipi_backend_common_defs_h_
-
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_DATA_WIDTH     16
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_CH_ID_WIDTH     2
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_FMT_TYPE_WIDTH  3
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_REAL_WIDTH (_HRT_CSS_RECEIVER_2400_GEN_SHORT_DATA_WIDTH + _HRT_CSS_RECEIVER_2400_GEN_SHORT_CH_ID_WIDTH + _HRT_CSS_RECEIVER_2400_GEN_SHORT_FMT_TYPE_WIDTH)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_WIDTH      32 /* use 32 to be compatibel with streaming monitor !, MSB's of interface are tied to '0' */ 
-
-/* Definition of data format ID at the interface CSS_receiver capture/acquisition units */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8          24   /* 01 1000 YUV420 8-bit                                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_10         25   /* 01 1001  YUV420 10-bit                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8L         26   /* 01 1010   YUV420 8-bit legacy                               */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV422_8          30   /* 01 1110   YUV422 8-bit                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV422_10         31   /* 01 1111   YUV422 10-bit                                     */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB444            32   /* 10 0000   RGB444                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB555            33   /* 10 0001   RGB555                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB565            34   /* 10 0010   RGB565                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB666            35   /* 10 0011   RGB666                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB888            36   /* 10 0100   RGB888                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW6              40   /* 10 1000   RAW6                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW7              41   /* 10 1001   RAW7                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW8              42   /* 10 1010   RAW8                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW10             43   /* 10 1011   RAW10                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW12             44   /* 10 1100   RAW12                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW14             45   /* 10 1101   RAW14                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_1         48   /* 11 0000    JPEG [User Defined 8-bit Data Type 1]            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_2         49   /* 11 0001    User Defined 8-bit Data Type 2                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_3         50   /* 11 0010    User Defined 8-bit Data Type 3                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_4         51   /* 11 0011    User Defined 8-bit Data Type 4                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_5         52   /* 11 0100    User Defined 8-bit Data Type 5                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_6         53   /* 11 0101    User Defined 8-bit Data Type 6                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_7         54   /* 11 0110    User Defined 8-bit Data Type 7                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_8         55   /* 11 0111    User Defined 8-bit Data Type 8                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_Emb               18   /* 01 0010    embedded eight bit non image data                */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_SOF                0   /* 00 0000    frame start                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_EOF                1   /* 00 0001    frame end                                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_SOL                2   /* 00 0010    line start                                       */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_EOL                3   /* 00 0011    line end                                         */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH1            8   /* 00 1000  Generic Short Packet Code 1                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH2            9   /* 00 1001    Generic Short Packet Code 2                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH3           10   /* 00 1010    Generic Short Packet Code 3                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH4           11   /* 00 1011    Generic Short Packet Code 4                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH5           12   /* 00 1100    Generic Short Packet Code 5                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH6           13   /* 00 1101    Generic Short Packet Code 6                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH7           14   /* 00 1110    Generic Short Packet Code 7                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH8           15   /* 00 1111    Generic Short Packet Code 8                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8_CSPS     28   /* 01 1100   YUV420 8-bit (Chroma Shifted Pixel Sampling)      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_10_CSPS    29   /* 01 1101   YUV420 10-bit (Chroma Shifted Pixel Sampling)     */
-/* used reseved mipi positions for these */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW16             46 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18             47 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18_2           37 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18_3           38 
-
-//_HRT_CSS_RECEIVER_2400_FMT_TYPE_CUSTOM 63
-#define _HRT_MIPI_BACKEND_FMT_TYPE_CUSTOM                       63
-
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_WIDTH              6
-
-/* Definition of format_types at the interface CSS --> input_selector*/
-/* !! Changes here should be copied to systems/isp/isp_css/bin/conv_transmitter_cmd.tcl !! */
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB888           0  // 36 'h24
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB555           1  // 33 'h
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB444           2  // 32
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB565           3  // 34
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB666           4  // 35
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW8             5  // 42 
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW10            6  // 43
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW6             7  // 40
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW7             8  // 41
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW12            9  // 43
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW14           10  // 45
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8        11  // 30
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_10       12  // 25
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV422_8        13  // 30
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV422_10       14  // 31
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_1       15  // 48
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8L       16  // 26
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_Emb             17  // 18
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_2       18  // 49
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_3       19  // 50
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_4       20  // 51
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_5       21  // 52
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_6       22  // 53
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_7       23  // 54
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_8       24  // 55
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8_CSPS   25  // 28
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_10_CSPS  26  // 29
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW16           27  // ?
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18           28  // ?
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18_2         29  // ? Option 2 for depacketiser
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18_3         30  // ? Option 3 for depacketiser
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_CUSTOM          31  // to signal custom decoding 
-
-/* definition for state machine of data FIFO for decode different type of data */
-#define _HRT_CSS_RECEIVER_2400_YUV420_8_REPEAT_PTN                 1  
-#define _HRT_CSS_RECEIVER_2400_YUV420_10_REPEAT_PTN                5
-#define _HRT_CSS_RECEIVER_2400_YUV420_8L_REPEAT_PTN                1
-#define _HRT_CSS_RECEIVER_2400_YUV422_8_REPEAT_PTN                 1
-#define _HRT_CSS_RECEIVER_2400_YUV422_10_REPEAT_PTN                5
-#define _HRT_CSS_RECEIVER_2400_RGB444_REPEAT_PTN                   2 
-#define _HRT_CSS_RECEIVER_2400_RGB555_REPEAT_PTN                   2
-#define _HRT_CSS_RECEIVER_2400_RGB565_REPEAT_PTN                   2
-#define _HRT_CSS_RECEIVER_2400_RGB666_REPEAT_PTN                   9                       
-#define _HRT_CSS_RECEIVER_2400_RGB888_REPEAT_PTN                   3
-#define _HRT_CSS_RECEIVER_2400_RAW6_REPEAT_PTN                     3
-#define _HRT_CSS_RECEIVER_2400_RAW7_REPEAT_PTN                     7
-#define _HRT_CSS_RECEIVER_2400_RAW8_REPEAT_PTN                     1
-#define _HRT_CSS_RECEIVER_2400_RAW10_REPEAT_PTN                    5
-#define _HRT_CSS_RECEIVER_2400_RAW12_REPEAT_PTN                    3        
-#define _HRT_CSS_RECEIVER_2400_RAW14_REPEAT_PTN                    7
-
-#define _HRT_CSS_RECEIVER_2400_MAX_REPEAT_PTN                      _HRT_CSS_RECEIVER_2400_RGB666_REPEAT_PTN
-
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FMT_IDX                     0
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FMT_WIDTH                   3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_PRED_IDX                    3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_PRED_WIDTH                  1
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_USD_BITS                    4  /* bits per USD type */
-
-#define _HRT_CSS_RECEIVER_2400_BE_RAW16_DATAID_IDX                 0
-#define _HRT_CSS_RECEIVER_2400_BE_RAW16_EN_IDX                     6
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_DATAID_IDX                 0
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_OPTION_IDX                 6
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_EN_IDX                     8
-
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_NO_COMP                     0
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_6_10                     1
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_7_10                     2
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_8_10                     3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_6_12                     4
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_7_12                     5
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_8_12                     6
-
-
-/* packet bit definition */
-#define _HRT_CSS_RECEIVER_2400_PKT_SOP_IDX                        32
-#define _HRT_CSS_RECEIVER_2400_PKT_SOP_BITS                        1
-#define _HRT_CSS_RECEIVER_2400_PKT_CH_ID_IDX                      22
-#define _HRT_CSS_RECEIVER_2400_PKT_CH_ID_BITS                      2
-#define _HRT_CSS_RECEIVER_2400_PKT_FMT_ID_IDX                     16
-#define _HRT_CSS_RECEIVER_2400_PKT_FMT_ID_BITS                     6
-#define _HRT_CSS_RECEIVER_2400_PH_DATA_FIELD_IDX                   0
-#define _HRT_CSS_RECEIVER_2400_PH_DATA_FIELD_BITS                 16
-#define _HRT_CSS_RECEIVER_2400_PKT_PAYLOAD_IDX                     0
-#define _HRT_CSS_RECEIVER_2400_PKT_PAYLOAD_BITS                   32
-
-
-/*************************************************************************************************/
-/* Custom Decoding                                                                               */
-/* These Custom Defs are defined based on design-time config in "mipi_backend_pixel_formatter.chdl" !! */
-/*************************************************************************************************/
-/*
-#define BE_CUST_EN_IDX                     0     // 2bits 
-#define BE_CUST_EN_DATAID_IDX              2     // 6bits MIPI DATA ID 
-#define BE_CUST_EN_WIDTH                   8     
-#define BE_CUST_MODE_ALL                   1     // Enable Custom Decoding for all DATA IDs 
-#define BE_CUST_MODE_ONE                   3     // Enable Custom Decoding for ONE DATA ID, programmed in CUST_EN_DATA_ID 
-
-// Data State config = {get_bits(6bits), valid(1bit)}  //
-#define BE_CUST_DATA_STATE_S0_IDX          0     // 7bits
-#define BE_CUST_DATA_STATE_S1_IDX          8 //7      // 7bits 
-#define BE_CUST_DATA_STATE_S2_IDX          16//14    // 7bits /
-#define BE_CUST_DATA_STATE_WIDTH           24//21    
-#define BE_CUST_DATA_STATE_VALID_IDX       0     // 1bits 
-#define BE_CUST_DATA_STATE_GETBITS_IDX     1     // 6bits 
-
-
-
-
-// Pixel Extractor config 
-#define BE_CUST_PIX_EXT_DATA_ALIGN_IDX     0     // 6bits 
-#define BE_CUST_PIX_EXT_PIX_ALIGN_IDX      6//5     // 5bits 
-#define BE_CUST_PIX_EXT_PIX_MASK_IDX       11//10    // 18bits
-#define BE_CUST_PIX_EXT_PIX_EN_IDX         29 //28    // 1bits
-
-#define BE_CUST_PIX_EXT_WIDTH              30//29    
-
-// Pixel Valid & EoP config = {[eop,valid](especial), [eop,valid](normal)} 
-#define BE_CUST_PIX_VALID_EOP_P0_IDX        0    // 4bits 
-#define BE_CUST_PIX_VALID_EOP_P1_IDX        4    // 4bits 
-#define BE_CUST_PIX_VALID_EOP_P2_IDX        8    // 4bits 
-#define BE_CUST_PIX_VALID_EOP_P3_IDX        12   // 4bits 
-#define BE_CUST_PIX_VALID_EOP_WIDTH         16 
-#define BE_CUST_PIX_VALID_EOP_NOR_VALID_IDX 0    // Normal (NO less get_bits case) Valid - 1bits
-#define BE_CUST_PIX_VALID_EOP_NOR_EOP_IDX   1    // Normal (NO less get_bits case) EoP - 1bits 
-#define BE_CUST_PIX_VALID_EOP_ESP_VALID_IDX 2    // Especial (less get_bits case) Valid - 1bits 
-#define BE_CUST_PIX_VALID_EOP_ESP_EOP_IDX   3    // Especial (less get_bits case) EoP - 1bits
-
-*/
-
-#endif /* _mipi_backend_common_defs_h_ */
-#endif /* _css_receiver_2400_common_defs_h_ */ 
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mipi_backend_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mipi_backend_defs.h
deleted file mode 100644
index db5a1d2caba0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mipi_backend_defs.h
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _mipi_backend_defs_h
-#define _mipi_backend_defs_h
-
-#include "mipi_backend_common_defs.h"
-
-#define MIPI_BACKEND_REG_ALIGN                    4 // assuming 32 bit control bus width 
-
-#define _HRT_MIPI_BACKEND_NOF_IRQS                         3 // sid_lut     
-
-// SH Backend Register IDs
-#define _HRT_MIPI_BACKEND_ENABLE_REG_IDX                   0  
-#define _HRT_MIPI_BACKEND_STATUS_REG_IDX                   1  
-//#define _HRT_MIPI_BACKEND_HIGH_PREC_REG_IDX                2
-#define _HRT_MIPI_BACKEND_COMP_FORMAT_REG0_IDX             2
-#define _HRT_MIPI_BACKEND_COMP_FORMAT_REG1_IDX             3
-#define _HRT_MIPI_BACKEND_COMP_FORMAT_REG2_IDX             4
-#define _HRT_MIPI_BACKEND_COMP_FORMAT_REG3_IDX             5
-#define _HRT_MIPI_BACKEND_RAW16_CONFIG_REG_IDX             6
-#define _HRT_MIPI_BACKEND_RAW18_CONFIG_REG_IDX             7
-#define _HRT_MIPI_BACKEND_FORCE_RAW8_REG_IDX               8
-#define _HRT_MIPI_BACKEND_IRQ_STATUS_REG_IDX               9
-#define _HRT_MIPI_BACKEND_IRQ_CLEAR_REG_IDX               10
-////
-#define _HRT_MIPI_BACKEND_CUST_EN_REG_IDX                 11        
-#define _HRT_MIPI_BACKEND_CUST_DATA_STATE_REG_IDX         12
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S0P0_REG_IDX       13
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S0P1_REG_IDX       14
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S0P2_REG_IDX       15
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S0P3_REG_IDX       16
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S1P0_REG_IDX       17
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S1P1_REG_IDX       18
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S1P2_REG_IDX       19
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S1P3_REG_IDX       20
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S2P0_REG_IDX       21
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S2P1_REG_IDX       22
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S2P2_REG_IDX       23
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_S2P3_REG_IDX       24
-#define _HRT_MIPI_BACKEND_CUST_PIX_VALID_EOP_REG_IDX      25
-////
-#define _HRT_MIPI_BACKEND_GLOBAL_LUT_DISREGARD_REG_IDX    26
-#define _HRT_MIPI_BACKEND_PKT_STALL_STATUS_REG_IDX        27
-//#define _HRT_MIPI_BACKEND_SP_LUT_ENABLE_REG_IDX           28
-#define _HRT_MIPI_BACKEND_SP_LUT_ENTRY_0_REG_IDX          28 
-#define _HRT_MIPI_BACKEND_SP_LUT_ENTRY_1_REG_IDX          29 
-#define _HRT_MIPI_BACKEND_SP_LUT_ENTRY_2_REG_IDX          30  
-#define _HRT_MIPI_BACKEND_SP_LUT_ENTRY_3_REG_IDX          31 
-
-#define _HRT_MIPI_BACKEND_NOF_REGISTERS                   32 // excluding the LP LUT entries
-
-#define _HRT_MIPI_BACKEND_LP_LUT_ENTRY_0_REG_IDX          32
-
-
-/////////////////////////////////////////////////////////////////////////////////////////////////////
-#define _HRT_MIPI_BACKEND_ENABLE_REG_WIDTH                 1  
-#define _HRT_MIPI_BACKEND_STATUS_REG_WIDTH                 1  
-//#define _HRT_MIPI_BACKEND_HIGH_PREC_REG_WIDTH              1
-#define _HRT_MIPI_BACKEND_COMP_FORMAT_REG_WIDTH           32
-#define _HRT_MIPI_BACKEND_RAW16_CONFIG_REG_WIDTH           7 
-#define _HRT_MIPI_BACKEND_RAW18_CONFIG_REG_WIDTH           9
-#define _HRT_MIPI_BACKEND_FORCE_RAW8_REG_WIDTH             8
-#define _HRT_MIPI_BACKEND_IRQ_STATUS_REG_WIDTH            _HRT_MIPI_BACKEND_NOF_IRQS
-#define _HRT_MIPI_BACKEND_IRQ_CLEAR_REG_WIDTH              0 
-#define _HRT_MIPI_BACKEND_GLOBAL_LUT_DISREGARD_REG_WIDTH   1
-#define _HRT_MIPI_BACKEND_PKT_STALL_STATUS_REG_WIDTH       1+2+6
-//#define _HRT_MIPI_BACKEND_SP_LUT_ENABLE_REG_WIDTH          1
-//#define _HRT_MIPI_BACKEND_SP_LUT_ENTRY_0_REG_WIDTH         7 
-//#define _HRT_MIPI_BACKEND_SP_LUT_ENTRY_1_REG_WIDTH         7 
-//#define _HRT_MIPI_BACKEND_SP_LUT_ENTRY_2_REG_WIDTH         7 
-//#define _HRT_MIPI_BACKEND_SP_LUT_ENTRY_3_REG_WIDTH         7 
-
-/////////////////////////////////////////////////////////////////////////////////////////////////////
-
-#define _HRT_MIPI_BACKEND_NOF_SP_LUT_ENTRIES               4
-
-//#define _HRT_MIPI_BACKEND_MAX_NOF_LP_LUT_ENTRIES           16  // to satisfy hss model static array declaration
- 
-
-#define _HRT_MIPI_BACKEND_CHANNEL_ID_WIDTH                 2
-#define _HRT_MIPI_BACKEND_FORMAT_TYPE_WIDTH                6
-#define _HRT_MIPI_BACKEND_PACKET_ID_WIDTH                  _HRT_MIPI_BACKEND_CHANNEL_ID_WIDTH + _HRT_MIPI_BACKEND_FORMAT_TYPE_WIDTH
-
-#define _HRT_MIPI_BACKEND_STREAMING_PIX_A_LSB                 0
-#define _HRT_MIPI_BACKEND_STREAMING_PIX_A_MSB(pix_width)     (_HRT_MIPI_BACKEND_STREAMING_PIX_A_LSB + (pix_width) - 1)
-#define _HRT_MIPI_BACKEND_STREAMING_PIX_A_VAL_BIT(pix_width) (_HRT_MIPI_BACKEND_STREAMING_PIX_A_MSB(pix_width) + 1)
-#define _HRT_MIPI_BACKEND_STREAMING_PIX_B_LSB(pix_width)     (_HRT_MIPI_BACKEND_STREAMING_PIX_A_VAL_BIT(pix_width) + 1)
-#define _HRT_MIPI_BACKEND_STREAMING_PIX_B_MSB(pix_width)     (_HRT_MIPI_BACKEND_STREAMING_PIX_B_LSB(pix_width) + (pix_width) - 1)
-#define _HRT_MIPI_BACKEND_STREAMING_PIX_B_VAL_BIT(pix_width) (_HRT_MIPI_BACKEND_STREAMING_PIX_B_MSB(pix_width) + 1)
-#define _HRT_MIPI_BACKEND_STREAMING_SOP_BIT(pix_width)       (_HRT_MIPI_BACKEND_STREAMING_PIX_B_VAL_BIT(pix_width) + 1)
-#define _HRT_MIPI_BACKEND_STREAMING_EOP_BIT(pix_width)       (_HRT_MIPI_BACKEND_STREAMING_SOP_BIT(pix_width) + 1)
-#define _HRT_MIPI_BACKEND_STREAMING_WIDTH(pix_width)         (_HRT_MIPI_BACKEND_STREAMING_EOP_BIT(pix_width) + 1)
-
-/*************************************************************************************************/
-/* Custom Decoding                                                                               */
-/* These Custom Defs are defined based on design-time config in "mipi_backend_pixel_formatter.chdl" !! */
-/*************************************************************************************************/
-#define _HRT_MIPI_BACKEND_CUST_EN_IDX                     0     /* 2bits */
-#define _HRT_MIPI_BACKEND_CUST_EN_DATAID_IDX              2     /* 6bits MIPI DATA ID */ 
-#define _HRT_MIPI_BACKEND_CUST_EN_HIGH_PREC_IDX           8     // 1 bit
-#define _HRT_MIPI_BACKEND_CUST_EN_WIDTH                   9     
-#define _HRT_MIPI_BACKEND_CUST_MODE_ALL                   1     /* Enable Custom Decoding for all DATA IDs */
-#define _HRT_MIPI_BACKEND_CUST_MODE_ONE                   3     /* Enable Custom Decoding for ONE DATA ID, programmed in CUST_EN_DATA_ID */
-
-#define _HRT_MIPI_BACKEND_CUST_EN_OPTION_IDX              1    
-
-/* Data State config = {get_bits(6bits), valid(1bit)}  */
-#define _HRT_MIPI_BACKEND_CUST_DATA_STATE_S0_IDX          0     /* 7bits */ 
-#define _HRT_MIPI_BACKEND_CUST_DATA_STATE_S1_IDX          8     /* 7bits */ 
-#define _HRT_MIPI_BACKEND_CUST_DATA_STATE_S2_IDX          16    /* was 14 7bits */
-#define _HRT_MIPI_BACKEND_CUST_DATA_STATE_WIDTH           24    /* was 21*/
-#define _HRT_MIPI_BACKEND_CUST_DATA_STATE_VALID_IDX       0     /* 1bits */
-#define _HRT_MIPI_BACKEND_CUST_DATA_STATE_GETBITS_IDX     1     /* 6bits */
-
-/* Pixel Extractor config */
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_DATA_ALIGN_IDX     0     /* 6bits */
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_PIX_ALIGN_IDX      6     /* 5bits */
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_PIX_MASK_IDX       11    /* was 10 18bits */
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_PIX_EN_IDX         29    /* was 28 1bits */
-
-#define _HRT_MIPI_BACKEND_CUST_PIX_EXT_WIDTH              30    /* was 29 */
-
-/* Pixel Valid & EoP config = {[eop,valid](especial), [eop,valid](normal)} */
-#define _HRT_MIPI_BACKEND_CUST_PIX_VALID_EOP_P0_IDX        0    /* 4bits */
-#define _HRT_MIPI_BACKEND_CUST_PIX_VALID_EOP_P1_IDX        4    /* 4bits */
-#define _HRT_MIPI_BACKEND_CUST_PIX_VALID_EOP_P2_IDX        8    /* 4bits */
-#define _HRT_MIPI_BACKEND_CUST_PIX_VALID_EOP_P3_IDX        12   /* 4bits */
-#define _HRT_MIPI_BACKEND_CUST_PIX_VALID_EOP_WIDTH         16 
-#define _HRT_MIPI_BACKEND_CUST_PIX_VALID_EOP_NOR_VALID_IDX 0    /* Normal (NO less get_bits case) Valid - 1bits */
-#define _HRT_MIPI_BACKEND_CUST_PIX_VALID_EOP_NOR_EOP_IDX   1    /* Normal (NO less get_bits case) EoP - 1bits */
-#define _HRT_MIPI_BACKEND_CUST_PIX_VALID_EOP_ESP_VALID_IDX 2    /* Especial (less get_bits case) Valid - 1bits */
-#define _HRT_MIPI_BACKEND_CUST_PIX_VALID_EOP_ESP_EOP_IDX   3    /* Especial (less get_bits case) EoP - 1bits */
-
-/*************************************************************************************************/
-/* MIPI backend output streaming interface definition                                            */
-/* These parameters define the fields within the streaming bus. These should also be used by the */
-/* subsequent block, ie stream2mmio.                                                             */
-/*************************************************************************************************/
-/* The pipe backend - stream2mmio should be design time configurable in                          */
-/*   PixWidth - Number of bits per pixel                                                         */
-/*   PPC      - Pixel per Clocks                                                                 */
-/*   NumSids  - Max number of source Ids (ifc's)  and derived from that:                         */
-/*   SidWidth - Number of bits required for the sid parameter                                    */
-/* In order to keep this configurability, below Macro's have these as a parameter                */
-/*************************************************************************************************/
-
-#define HRT_MIPI_BACKEND_STREAM_EOP_BIT                      0
-#define HRT_MIPI_BACKEND_STREAM_SOP_BIT                      1
-#define HRT_MIPI_BACKEND_STREAM_EOF_BIT                      2
-#define HRT_MIPI_BACKEND_STREAM_SOF_BIT                      3
-#define HRT_MIPI_BACKEND_STREAM_CHID_LS_BIT                  4
-#define HRT_MIPI_BACKEND_STREAM_CHID_MS_BIT(sid_width)      (HRT_MIPI_BACKEND_STREAM_CHID_LS_BIT+(sid_width)-1)
-#define HRT_MIPI_BACKEND_STREAM_PIX_VAL_BIT(sid_width,p)    (HRT_MIPI_BACKEND_STREAM_CHID_MS_BIT(sid_width)+1+p)
-
-#define HRT_MIPI_BACKEND_STREAM_PIX_LS_BIT(sid_width,ppc,pix_width,p) (HRT_MIPI_BACKEND_STREAM_PIX_VAL_BIT(sid_width,ppc)+ ((pix_width)*p))
-#define HRT_MIPI_BACKEND_STREAM_PIX_MS_BIT(sid_width,ppc,pix_width,p) (HRT_MIPI_BACKEND_STREAM_PIX_LS_BIT(sid_width,ppc,pix_width,p) + (pix_width) - 1)
-
-#if 0
-//#define HRT_MIPI_BACKEND_STREAM_PIX_BITS                    14
-//#define HRT_MIPI_BACKEND_STREAM_CHID_BITS                    4
-//#define HRT_MIPI_BACKEND_STREAM_PPC                          4
-#endif
-
-#define HRT_MIPI_BACKEND_STREAM_BITS(sid_width,ppc,pix_width)         (HRT_MIPI_BACKEND_STREAM_PIX_MS_BIT(sid_width,ppc,pix_width,(ppc-1))+1)
-
-
-/* SP and LP LUT BIT POSITIONS */
-#define HRT_MIPI_BACKEND_LUT_PKT_DISREGARD_BIT              0                                                                                           // 0    
-#define HRT_MIPI_BACKEND_LUT_SID_LS_BIT                     HRT_MIPI_BACKEND_LUT_PKT_DISREGARD_BIT + 1                                                  // 1    
-#define HRT_MIPI_BACKEND_LUT_SID_MS_BIT(sid_width)          (HRT_MIPI_BACKEND_LUT_SID_LS_BIT+(sid_width)-1)                                             // 1 + (4) - 1 = 4  
-#define HRT_MIPI_BACKEND_LUT_MIPI_CH_ID_LS_BIT(sid_width)   HRT_MIPI_BACKEND_LUT_SID_MS_BIT(sid_width) + 1                                              // 5
-#define HRT_MIPI_BACKEND_LUT_MIPI_CH_ID_MS_BIT(sid_width)   HRT_MIPI_BACKEND_LUT_MIPI_CH_ID_LS_BIT(sid_width) + _HRT_MIPI_BACKEND_CHANNEL_ID_WIDTH - 1  // 6
-#define HRT_MIPI_BACKEND_LUT_MIPI_FMT_LS_BIT(sid_width)     HRT_MIPI_BACKEND_LUT_MIPI_CH_ID_MS_BIT(sid_width) + 1                                       // 7
-#define HRT_MIPI_BACKEND_LUT_MIPI_FMT_MS_BIT(sid_width)     HRT_MIPI_BACKEND_LUT_MIPI_FMT_LS_BIT(sid_width) + _HRT_MIPI_BACKEND_FORMAT_TYPE_WIDTH - 1   // 12    
-
-/* #define HRT_MIPI_BACKEND_SP_LUT_BITS(sid_width)             HRT_MIPI_BACKEND_LUT_MIPI_CH_ID_MS_BIT(sid_width) + 1                                       // 7          */
-
-#define HRT_MIPI_BACKEND_SP_LUT_BITS(sid_width)             HRT_MIPI_BACKEND_LUT_SID_MS_BIT(sid_width) + 1  
-#define HRT_MIPI_BACKEND_LP_LUT_BITS(sid_width)             HRT_MIPI_BACKEND_LUT_MIPI_FMT_MS_BIT(sid_width) + 1                                         // 13
-
-
-// temp solution
-//#define HRT_MIPI_BACKEND_STREAM_PIXA_VAL_BIT                HRT_MIPI_BACKEND_STREAM_CHID_MS_BIT  + 1                                    // 8                     
-//#define HRT_MIPI_BACKEND_STREAM_PIXB_VAL_BIT                HRT_MIPI_BACKEND_STREAM_PIXA_VAL_BIT + 1                                    // 9
-//#define HRT_MIPI_BACKEND_STREAM_PIXC_VAL_BIT                HRT_MIPI_BACKEND_STREAM_PIXB_VAL_BIT + 1                                    // 10
-//#define HRT_MIPI_BACKEND_STREAM_PIXD_VAL_BIT                HRT_MIPI_BACKEND_STREAM_PIXC_VAL_BIT + 1                                    // 11
-//#define HRT_MIPI_BACKEND_STREAM_PIXA_LS_BIT                 HRT_MIPI_BACKEND_STREAM_PIXD_VAL_BIT + 1                                    // 12
-//#define HRT_MIPI_BACKEND_STREAM_PIXA_MS_BIT                 HRT_MIPI_BACKEND_STREAM_PIXA_LS_BIT  + HRT_MIPI_BACKEND_STREAM_PIX_BITS - 1 // 25
-//#define HRT_MIPI_BACKEND_STREAM_PIXB_LS_BIT                 HRT_MIPI_BACKEND_STREAM_PIXA_MS_BIT + 1                                     // 26
-//#define HRT_MIPI_BACKEND_STREAM_PIXB_MS_BIT                 HRT_MIPI_BACKEND_STREAM_PIXB_LS_BIT  + HRT_MIPI_BACKEND_STREAM_PIX_BITS - 1 // 39
-//#define HRT_MIPI_BACKEND_STREAM_PIXC_LS_BIT                 HRT_MIPI_BACKEND_STREAM_PIXB_MS_BIT + 1                                     // 40
-//#define HRT_MIPI_BACKEND_STREAM_PIXC_MS_BIT                 HRT_MIPI_BACKEND_STREAM_PIXC_LS_BIT  + HRT_MIPI_BACKEND_STREAM_PIX_BITS - 1 // 53
-//#define HRT_MIPI_BACKEND_STREAM_PIXD_LS_BIT                 HRT_MIPI_BACKEND_STREAM_PIXC_MS_BIT + 1                                     // 54
-//#define HRT_MIPI_BACKEND_STREAM_PIXD_MS_BIT                 HRT_MIPI_BACKEND_STREAM_PIXD_LS_BIT  + HRT_MIPI_BACKEND_STREAM_PIX_BITS - 1 // 67
- 
-// vc hidden in pixb data (passed as raw12 the pipe)
-#define HRT_MIPI_BACKEND_STREAM_VC_LS_BIT(sid_width,ppc,pix_width)  HRT_MIPI_BACKEND_STREAM_PIX_LS_BIT(sid_width,ppc,pix_width,1) + 10  //HRT_MIPI_BACKEND_STREAM_PIXB_LS_BIT + 10 // 36 
-#define HRT_MIPI_BACKEND_STREAM_VC_MS_BIT(sid_width,ppc,pix_width)  HRT_MIPI_BACKEND_STREAM_VC_LS_BIT(sid_width,ppc,pix_width) + 1    // 37
-
-
-
-
-#endif /* _mipi_backend_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mmu_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mmu_defs.h
deleted file mode 100644
index c038f39ffd25..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/mmu_defs.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _mmu_defs_h
-#define _mmu_defs_h
-
-#define _HRT_MMU_INVALIDATE_TLB_REG_IDX          0
-#define _HRT_MMU_PAGE_TABLE_BASE_ADDRESS_REG_IDX 1
-
-#define _HRT_MMU_REG_ALIGN 4
-
-#endif /* _mmu_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/rx_csi_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/rx_csi_defs.h
deleted file mode 100644
index 0aad86e2e914..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/rx_csi_defs.h
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _csi_rx_defs_h
-#define _csi_rx_defs_h
-
-//#include "rx_csi_common_defs.h"
-
-
-
-#define MIPI_PKT_DATA_WIDTH                         32
-//#define CLK_CROSSING_FIFO_DEPTH                     16
-#define _CSI_RX_REG_ALIGN                            4
-
-//define number of IRQ (see below for definition of each IRQ bits)
-#define CSI_RX_NOF_IRQS_BYTE_DOMAIN                11
-#define CSI_RX_NOF_IRQS_ISP_DOMAIN                 15 // CSI_RX_NOF_IRQS_BYTE_DOMAIN + remaining from Dphy_rx already on ISP clock domain
-
-// REGISTER DESCRIPTION
-//#define _HRT_CSI_RX_SOFTRESET_REG_IDX                0
-#define _HRT_CSI_RX_ENABLE_REG_IDX                   0
-#define _HRT_CSI_RX_NOF_ENABLED_LANES_REG_IDX        1  
-#define _HRT_CSI_RX_ERROR_HANDLING_REG_IDX           2
-#define _HRT_CSI_RX_STATUS_REG_IDX                   3  
-#define _HRT_CSI_RX_STATUS_DLANE_HS_REG_IDX          4  
-#define _HRT_CSI_RX_STATUS_DLANE_LP_REG_IDX          5  
-//#define _HRT_CSI_RX_IRQ_CONFIG_REG_IDX               6  
-#define _HRT_CSI_RX_DLY_CNT_TERMEN_CLANE_REG_IDX     6
-#define _HRT_CSI_RX_DLY_CNT_SETTLE_CLANE_REG_IDX     7
-#define _HRT_CSI_RX_DLY_CNT_TERMEN_DLANE_REG_IDX(lane_idx)    (8+(2*lane_idx))
-#define _HRT_CSI_RX_DLY_CNT_SETTLE_DLANE_REG_IDX(lane_idx)    (8+(2*lane_idx)+1)
-
-#define _HRT_CSI_RX_NOF_REGISTERS(nof_dlanes)      (8+2*(nof_dlanes))
-
-
-//#define _HRT_CSI_RX_SOFTRESET_REG_WIDTH              1
-#define _HRT_CSI_RX_ENABLE_REG_WIDTH                 1
-#define _HRT_CSI_RX_NOF_ENABLED_LANES_REG_WIDTH      3
-#define _HRT_CSI_RX_ERROR_HANDLING_REG_WIDTH         4 
-#define _HRT_CSI_RX_STATUS_REG_WIDTH                 1   
-#define _HRT_CSI_RX_STATUS_DLANE_HS_REG_WIDTH        8  
-#define _HRT_CSI_RX_STATUS_DLANE_LP_REG_WIDTH        24
-#define _HRT_CSI_RX_IRQ_CONFIG_REG_WIDTH             (CSI_RX_NOF_IRQS_ISP_DOMAIN)
-#define _HRT_CSI_RX_DLY_CNT_REG_WIDTH                24
-//#define _HRT_CSI_RX_IRQ_STATUS_REG_WIDTH            NOF_IRQS 
-//#define _HRT_CSI_RX_IRQ_CLEAR_REG_WIDTH             0
-
-
-#define ONE_LANE_ENABLED                             0
-#define TWO_LANES_ENABLED                            1
-#define THREE_LANES_ENABLED                          2    
-#define FOUR_LANES_ENABLED                           3
-
-// Error handling reg bit positions
-#define ERR_DECISION_BIT      0
-#define DISC_RESERVED_SP_BIT  1
-#define DISC_RESERVED_LP_BIT  2
-#define DIS_INCOMP_PKT_CHK_BIT	3
-
-#define _HRT_CSI_RX_IRQ_CONFIG_REG_VAL_POSEDGE      0
-#define _HRT_CSI_RX_IRQ_CONFIG_REG_VAL_ORIGINAL     1
-
-// Interrupt bits 
-#define _HRT_RX_CSI_IRQ_SINGLE_PH_ERROR_CORRECTED   0
-#define _HRT_RX_CSI_IRQ_MULTIPLE_PH_ERROR_DETECTED  1
-#define _HRT_RX_CSI_IRQ_PAYLOAD_CHECKSUM_ERROR      2
-#define _HRT_RX_CSI_IRQ_FIFO_FULL_ERROR             3
-#define _HRT_RX_CSI_IRQ_RESERVED_SP_DETECTED        4
-#define _HRT_RX_CSI_IRQ_RESERVED_LP_DETECTED        5
-//#define _HRT_RX_CSI_IRQ_PREMATURE_SOP               6
-#define _HRT_RX_CSI_IRQ_INCOMPLETE_PACKET           6
-#define _HRT_RX_CSI_IRQ_FRAME_SYNC_ERROR            7
-#define _HRT_RX_CSI_IRQ_LINE_SYNC_ERROR             8
-#define _HRT_RX_CSI_IRQ_DLANE_HS_SOT_ERROR          9
-#define _HRT_RX_CSI_IRQ_DLANE_HS_SOT_SYNC_ERROR    10
-
-#define _HRT_RX_CSI_IRQ_DLANE_ESC_ERROR            11
-#define _HRT_RX_CSI_IRQ_DLANE_TRIGGERESC           12
-#define _HRT_RX_CSI_IRQ_DLANE_ULPSESC              13
-#define _HRT_RX_CSI_IRQ_CLANE_ULPSCLKNOT           14
-
-/* OLD ARASAN FRONTEND IRQs
-#define _HRT_RX_CSI_IRQ_OVERRUN_BIT                0
-#define _HRT_RX_CSI_IRQ_RESERVED_BIT               1
-#define _HRT_RX_CSI_IRQ_SLEEP_MODE_ENTRY_BIT       2
-#define _HRT_RX_CSI_IRQ_SLEEP_MODE_EXIT_BIT        3
-#define _HRT_RX_CSI_IRQ_ERR_SOT_HS_BIT             4
-#define _HRT_RX_CSI_IRQ_ERR_SOT_SYNC_HS_BIT        5
-#define _HRT_RX_CSI_IRQ_ERR_CONTROL_BIT            6
-#define _HRT_RX_CSI_IRQ_ERR_ECC_DOUBLE_BIT         7
-#define _HRT_RX_CSI_IRQ_ERR_ECC_CORRECTED_BIT      8
-#define _HRT_RX_CSI_IRQ_ERR_ECC_NO_CORRECTION_BIT  9
-#define _HRT_RX_CSI_IRQ_ERR_CRC_BIT               10
-#define _HRT_RX_CSI_IRQ_ERR_ID_BIT                11
-#define _HRT_RX_CSI_IRQ_ERR_FRAME_SYNC_BIT        12
-#define _HRT_RX_CSI_IRQ_ERR_FRAME_DATA_BIT        13
-#define _HRT_RX_CSI_IRQ_DATA_TIMEOUT_BIT          14
-#define _HRT_RX_CSI_IRQ_ERR_ESCAPE_BIT            15
-#define _HRT_RX_CSI_IRQ_ERR_LINE_SYNC_BIT         16
-*/
-
-
-////Bit Description for reg _HRT_CSI_RX_STATUS_DLANE_HS_REG_IDX
-#define _HRT_CSI_RX_STATUS_DLANE_HS_SOT_ERR_LANE0        0
-#define _HRT_CSI_RX_STATUS_DLANE_HS_SOT_ERR_LANE1        1
-#define _HRT_CSI_RX_STATUS_DLANE_HS_SOT_ERR_LANE2        2
-#define _HRT_CSI_RX_STATUS_DLANE_HS_SOT_ERR_LANE3        3
-#define _HRT_CSI_RX_STATUS_DLANE_HS_SOT_SYNC_ERR_LANE0   4
-#define _HRT_CSI_RX_STATUS_DLANE_HS_SOT_SYNC_ERR_LANE1   5
-#define _HRT_CSI_RX_STATUS_DLANE_HS_SOT_SYNC_ERR_LANE2   6
-#define _HRT_CSI_RX_STATUS_DLANE_HS_SOT_SYNC_ERR_LANE3   7
-
-////Bit Description for reg _HRT_CSI_RX_STATUS_DLANE_LP_REG_IDX
-#define _HRT_CSI_RX_STATUS_DLANE_LP_ESC_ERR_LANE0        0
-#define _HRT_CSI_RX_STATUS_DLANE_LP_ESC_ERR_LANE1        1
-#define _HRT_CSI_RX_STATUS_DLANE_LP_ESC_ERR_LANE2        2
-#define _HRT_CSI_RX_STATUS_DLANE_LP_ESC_ERR_LANE3        3
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC0_LANE0    4
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC1_LANE0    5
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC2_LANE0    6
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC3_LANE0    7
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC0_LANE1    8
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC1_LANE1    9
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC2_LANE1    10
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC3_LANE1    11
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC0_LANE2    12
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC1_LANE2    13
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC2_LANE2    14
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC3_LANE2    15
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC0_LANE3    16
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC1_LANE3    17
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC2_LANE3    18
-#define _HRT_CSI_RX_STATUS_DLANE_LP_TRIGGERESC3_LANE3    19
-#define _HRT_CSI_RX_STATUS_DLANE_LP_ULPSESC_LANE0        20
-#define _HRT_CSI_RX_STATUS_DLANE_LP_ULPSESC_LANE1        21
-#define _HRT_CSI_RX_STATUS_DLANE_LP_ULPSESC_LANE2        22
-#define _HRT_CSI_RX_STATUS_DLANE_LP_ULPSESC_LANE3        23
-
-/*********************************************************/
-/*** Relevant declarations from rx_csi_common_defs.h *****/
-/*********************************************************/
-/* packet bit definition */
-#define _HRT_RX_CSI_PKT_SOP_BITPOS                       32
-#define _HRT_RX_CSI_PKT_EOP_BITPOS                       33
-#define _HRT_RX_CSI_PKT_PAYLOAD_BITPOS                    0
-#define _HRT_RX_CSI_PH_CH_ID_BITPOS                      22
-#define _HRT_RX_CSI_PH_FMT_ID_BITPOS                     16
-#define _HRT_RX_CSI_PH_DATA_FIELD_BITPOS                  0
-
-#define _HRT_RX_CSI_PKT_SOP_BITS                          1
-#define _HRT_RX_CSI_PKT_EOP_BITS                          1
-#define _HRT_RX_CSI_PKT_PAYLOAD_BITS                     32
-#define _HRT_RX_CSI_PH_CH_ID_BITS                         2
-#define _HRT_RX_CSI_PH_FMT_ID_BITS                        6
-#define _HRT_RX_CSI_PH_DATA_FIELD_BITS                   16
-
-/* Definition of data format ID at the interface CSS_receiver units */
-#define _HRT_RX_CSI_DATA_FORMAT_ID_SOF                0   /* 00 0000    frame start                                      */
-#define _HRT_RX_CSI_DATA_FORMAT_ID_EOF                1   /* 00 0001    frame end                                        */
-#define _HRT_RX_CSI_DATA_FORMAT_ID_SOL                2   /* 00 0010    line start                                       */
-#define _HRT_RX_CSI_DATA_FORMAT_ID_EOL                3   /* 00 0011    line end                                         */
-
-
-#endif /* _csi_rx_defs_h */ 
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/scalar_processor_2400_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/scalar_processor_2400_params.h
deleted file mode 100644
index 9b6c2893d950..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/scalar_processor_2400_params.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _scalar_processor_2400_params_h
-#define _scalar_processor_2400_params_h
-
-#include "cell_params.h"
-
-#endif /* _scalar_processor_2400_params_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/str2mem_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/str2mem_defs.h
deleted file mode 100644
index 1cb62444cf68..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/str2mem_defs.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _ST2MEM_DEFS_H
-#define _ST2MEM_DEFS_H
-
-#define _STR2MEM_CRUN_BIT               0x100000
-#define _STR2MEM_CMD_BITS               0x0F0000
-#define _STR2MEM_COUNT_BITS             0x00FFFF
-
-#define _STR2MEM_BLOCKS_CMD             0xA0000
-#define _STR2MEM_PACKETS_CMD            0xB0000
-#define _STR2MEM_BYTES_CMD              0xC0000
-#define _STR2MEM_BYTES_FROM_PACKET_CMD  0xD0000
-
-#define _STR2MEM_SOFT_RESET_REG_ID                   0
-#define _STR2MEM_INPUT_ENDIANNESS_REG_ID             1
-#define _STR2MEM_OUTPUT_ENDIANNESS_REG_ID            2
-#define _STR2MEM_BIT_SWAPPING_REG_ID                 3
-#define _STR2MEM_BLOCK_SYNC_LEVEL_REG_ID             4
-#define _STR2MEM_PACKET_SYNC_LEVEL_REG_ID            5
-#define _STR2MEM_READ_POST_WRITE_SYNC_ENABLE_REG_ID  6
-#define _STR2MEM_DUAL_BYTE_INPUTS_ENABLED_REG_ID     7
-#define _STR2MEM_EN_STAT_UPDATE_ID                   8
-
-#define _STR2MEM_REG_ALIGN      4
-
-#endif /* _ST2MEM_DEFS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/stream2mmio_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/stream2mmio_defs.h
deleted file mode 100644
index 46b52fe5ae99..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/stream2mmio_defs.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _STREAM2MMMIO_DEFS_H
-#define _STREAM2MMMIO_DEFS_H
-
-#include <mipi_backend_defs.h>
-
-#define _STREAM2MMIO_REG_ALIGN                  4
-
-#define _STREAM2MMIO_COMMAND_REG_ID             0
-#define _STREAM2MMIO_ACKNOWLEDGE_REG_ID         1
-#define _STREAM2MMIO_PIX_WIDTH_ID_REG_ID        2
-#define _STREAM2MMIO_START_ADDR_REG_ID          3      /* master port address,NOT Byte */
-#define _STREAM2MMIO_END_ADDR_REG_ID            4      /* master port address,NOT Byte */
-#define _STREAM2MMIO_STRIDE_REG_ID              5      /* stride in master port words, increment is per packet for long sids, stride is not used for short sid's*/
-#define _STREAM2MMIO_NUM_ITEMS_REG_ID           6      /* number of packets for store packets cmd, number of words for store_words cmd */ 
-#define _STREAM2MMIO_BLOCK_WHEN_NO_CMD_REG_ID   7      /* if this register is 1, input will be stalled if there is no pending command for this sid */
-#define _STREAM2MMIO_REGS_PER_SID               8
-
-#define _STREAM2MMIO_SID_REG_OFFSET             8
-#define _STREAM2MMIO_MAX_NOF_SIDS              64      /* value used in hss model */
-
-/* command token definition     */
-#define _STREAM2MMIO_CMD_TOKEN_CMD_LSB          0      /* bits 1-0 is for the command field */
-#define _STREAM2MMIO_CMD_TOKEN_CMD_MSB          1
-
-#define _STREAM2MMIO_CMD_TOKEN_WIDTH           (_STREAM2MMIO_CMD_TOKEN_CMD_MSB+1-_STREAM2MMIO_CMD_TOKEN_CMD_LSB)
-
-#define _STREAM2MMIO_CMD_TOKEN_STORE_WORDS              0      /* command for storing a number of output words indicated by reg _STREAM2MMIO_NUM_ITEMS */
-#define _STREAM2MMIO_CMD_TOKEN_STORE_PACKETS            1      /* command for storing a number of packets indicated by reg _STREAM2MMIO_NUM_ITEMS      */
-#define _STREAM2MMIO_CMD_TOKEN_SYNC_FRAME               2      /* command for waiting for a frame start                                                */
-
-/* acknowledges from packer module */
-/* fields: eof   - indicates whether last (short) packet received was an eof packet */
-/*         eop   - indicates whether command has ended due to packet end or due to no of words requested has been received */
-/*         count - indicates number of words stored */
-#define _STREAM2MMIO_PACK_NUM_ITEMS_BITS        16
-#define _STREAM2MMIO_PACK_ACK_EOP_BIT           _STREAM2MMIO_PACK_NUM_ITEMS_BITS
-#define _STREAM2MMIO_PACK_ACK_EOF_BIT           (_STREAM2MMIO_PACK_ACK_EOP_BIT+1)
-
-/* acknowledge token definition */
-#define _STREAM2MMIO_ACK_TOKEN_NUM_ITEMS_LSB    0      /* bits 3-0 is for the command field */
-#define _STREAM2MMIO_ACK_TOKEN_NUM_ITEMS_MSB   (_STREAM2MMIO_PACK_NUM_ITEMS_BITS-1)
-#define _STREAM2MMIO_ACK_TOKEN_EOP_BIT         _STREAM2MMIO_PACK_ACK_EOP_BIT
-#define _STREAM2MMIO_ACK_TOKEN_EOF_BIT         _STREAM2MMIO_PACK_ACK_EOF_BIT
-#define _STREAM2MMIO_ACK_TOKEN_VALID_BIT       (_STREAM2MMIO_ACK_TOKEN_EOF_BIT+1)      /* this bit indicates a valid ack    */
-                                                                                       /* if there is no valid ack, a read  */
-                                                                                       /* on the ack register returns 0     */
-#define _STREAM2MMIO_ACK_TOKEN_WIDTH           (_STREAM2MMIO_ACK_TOKEN_VALID_BIT+1)
-
-/* commands for packer module */
-#define _STREAM2MMIO_PACK_CMD_STORE_WORDS        0
-#define _STREAM2MMIO_PACK_CMD_STORE_LONG_PACKET  1
-#define _STREAM2MMIO_PACK_CMD_STORE_SHORT_PACKET 2
-
-
-
-
-#endif /* _STREAM2MMIO_DEFS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/streaming_to_mipi_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/streaming_to_mipi_defs.h
deleted file mode 100644
index 60143b8743a2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/streaming_to_mipi_defs.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _streaming_to_mipi_defs_h
-#define _streaming_to_mipi_defs_h
-
-#define HIVE_STR_TO_MIPI_VALID_A_BIT 0
-#define HIVE_STR_TO_MIPI_VALID_B_BIT 1
-#define HIVE_STR_TO_MIPI_SOL_BIT     2
-#define HIVE_STR_TO_MIPI_EOL_BIT     3
-#define HIVE_STR_TO_MIPI_SOF_BIT     4
-#define HIVE_STR_TO_MIPI_EOF_BIT     5
-#define HIVE_STR_TO_MIPI_CH_ID_LSB   6
-
-#define HIVE_STR_TO_MIPI_DATA_A_LSB  (HIVE_STR_TO_MIPI_VALID_B_BIT + 1)
-
-#endif /* _streaming_to_mipi_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/timed_controller_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/timed_controller_defs.h
deleted file mode 100644
index d2b8972b0d9e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/timed_controller_defs.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _timed_controller_defs_h
-#define _timed_controller_defs_h
-
-#define _HRT_TIMED_CONTROLLER_CMD_REG_IDX 0
-
-#define _HRT_TIMED_CONTROLLER_REG_ALIGN 4
-
-#endif /* _timed_controller_defs_h */   
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/var.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/var.h
deleted file mode 100644
index 19b19ef484f9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/var.h
+++ /dev/null
@@ -1,99 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_VAR_H
-#define _HRT_VAR_H
-
-#include "version.h"
-#include "system_api.h"
-#include "hive_types.h"
-
-#define hrt_int_type_of_char   char
-#define hrt_int_type_of_uchar  unsigned char
-#define hrt_int_type_of_short  short
-#define hrt_int_type_of_ushort unsigned short
-#define hrt_int_type_of_int    int
-#define hrt_int_type_of_uint   unsigned int
-#define hrt_int_type_of_long   long
-#define hrt_int_type_of_ulong  unsigned long
-#define hrt_int_type_of_ptr    unsigned int
-
-#define hrt_host_type_of_char   char
-#define hrt_host_type_of_uchar  unsigned char
-#define hrt_host_type_of_short  short
-#define hrt_host_type_of_ushort unsigned short
-#define hrt_host_type_of_int    int
-#define hrt_host_type_of_uint   unsigned int
-#define hrt_host_type_of_long   long
-#define hrt_host_type_of_ulong  unsigned long
-#define hrt_host_type_of_ptr    void*
-
-#define HRT_TYPE_BYTES(cell, type) (HRT_TYPE_BITS(cell, type)/8)
-#define HRT_HOST_TYPE(cell_type)   HRTCAT(hrt_host_type_of_, cell_type)
-#define HRT_INT_TYPE(type)         HRTCAT(hrt_int_type_of_, type)
-
-#ifdef C_RUN
-
-#ifdef C_RUN_DYNAMIC_LINK_PROGRAMS
-extern void *csim_processor_get_crun_symbol(hive_proc_id p, const char *sym);
-#define _hrt_cell_get_crun_symbol(cell,sym)          csim_processor_get_crun_symbol(cell,HRTSTR(sym))
-#define _hrt_cell_get_crun_indexed_symbol(cell,sym)  csim_processor_get_crun_symbol(cell,HRTSTR(sym))
-#else
-#define _hrt_cell_get_crun_symbol(cell,sym)         (&sym)
-#define _hrt_cell_get_crun_indexed_symbol(cell,sym) (sym)
-#endif //  C_RUN_DYNAMIC_LINK_PROGRAMS
-
-#define hrt_scalar_store(cell, type, var, data) \
-	((*(HRT_HOST_TYPE(type)*)_hrt_cell_get_crun_symbol(cell,var)) = (data))
-#define hrt_scalar_load(cell, type, var) \
-	((*(HRT_HOST_TYPE(type)*)_hrt_cell_get_crun_symbol(cell,var)))
-
-#define hrt_indexed_store(cell, type, array, index, data) \
-	((((HRT_HOST_TYPE(type)*)_hrt_cell_get_crun_indexed_symbol(cell,array))[index]) = (data))
-#define hrt_indexed_load(cell, type, array, index) \
-	(((HRT_HOST_TYPE(type)*)_hrt_cell_get_crun_indexed_symbol(cell,array))[index])
-
-#else /* C_RUN */
-
-#define hrt_scalar_store(cell, type, var, data) \
-  HRTCAT(hrt_mem_store_,HRT_TYPE_BITS(cell, type))(\
-	       cell, \
-	       HRTCAT(HIVE_MEM_,var), \
-	       HRTCAT(HIVE_ADDR_,var), \
-	       (HRT_INT_TYPE(type))(data))
-
-#define hrt_scalar_load(cell, type, var) \
-  (HRT_HOST_TYPE(type))(HRTCAT4(_hrt_mem_load_,HRT_PROC_TYPE(cell),_,type) ( \
-	       cell, \
-	       HRTCAT(HIVE_MEM_,var), \
-	       HRTCAT(HIVE_ADDR_,var)))
-
-#define hrt_indexed_store(cell, type, array, index, data) \
-  HRTCAT(hrt_mem_store_,HRT_TYPE_BITS(cell, type))(\
-	       cell, \
-	       HRTCAT(HIVE_MEM_,array), \
-	       (HRTCAT(HIVE_ADDR_,array))+((index)*HRT_TYPE_BYTES(cell, type)), \
-	       (HRT_INT_TYPE(type))(data))
-
-#define hrt_indexed_load(cell, type, array, index) \
-  (HRT_HOST_TYPE(type))(HRTCAT4(_hrt_mem_load_,HRT_PROC_TYPE(cell),_,type) ( \
-         cell, \
-	       HRTCAT(HIVE_MEM_,array), \
-	       (HRTCAT(HIVE_ADDR_,array))+((index)*HRT_TYPE_BYTES(cell, type))))
-
-#endif /* C_RUN */
-
-#endif /* _HRT_VAR_H */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/version.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/version.h
deleted file mode 100644
index bbc4948baea9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/hrt/version.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef HRT_VERSION_H
-#define HRT_VERSION_H
-#define HRT_VERSION_MAJOR 1
-#define HRT_VERSION_MINOR 4
-#define HRT_VERSION 1_4
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/ibuf_ctrl_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/ibuf_ctrl_global.h
deleted file mode 100644
index edb23252c48e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/ibuf_ctrl_global.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IBUF_CTRL_GLOBAL_H_INCLUDED__
-#define __IBUF_CTRL_GLOBAL_H_INCLUDED__
-
-#include <type_support.h>
-
-#include <ibuf_cntrl_defs.h>	/* _IBUF_CNTRL_RECALC_WORDS_STATUS,
-				 * _IBUF_CNTRL_ARBITERS_STATUS,
-				 * _IBUF_CNTRL_PROC_REG_ALIGN,
-				 * etc.
-				 */
-
-/* Definition of contents of main controller state register is lacking
- * in ibuf_cntrl_defs.h, so define these here:
- */
-#define _IBUF_CNTRL_MAIN_CNTRL_FSM_MASK			0xf
-#define _IBUF_CNTRL_MAIN_CNTRL_FSM_NEXT_COMMAND_CHECK	0x9
-#define _IBUF_CNTRL_MAIN_CNTRL_MEM_INP_BUF_ALLOC	(1 << 8)
-#define _IBUF_CNTRL_DMA_SYNC_WAIT_FOR_SYNC		1
-#define _IBUF_CNTRL_DMA_SYNC_FSM_WAIT_FOR_ACK		(0x3 << 1)
-
-typedef struct ib_buffer_s	ib_buffer_t;
-struct	ib_buffer_s {
-	uint32_t	start_addr;	/* start address of the buffer in the
-					 * "input-buffer hardware block"
-					 */
-
-	uint32_t	stride;		/* stride per buffer line (in bytes) */
-	uint32_t	lines;		/* lines in the buffer */
-};
-
-typedef struct ibuf_ctrl_cfg_s ibuf_ctrl_cfg_t;
-struct ibuf_ctrl_cfg_s {
-
-	bool online;
-
-	struct {
-		/* DMA configuration */
-		uint32_t channel;
-		uint32_t cmd; /* must be _DMA_V2_MOVE_A2B_NO_SYNC_CHK_COMMAND */
-
-		/* DMA reconfiguration */
-		uint32_t shift_returned_items;
-		uint32_t elems_per_word_in_ibuf;
-		uint32_t elems_per_word_in_dest;
-	} dma_cfg;
-
-	ib_buffer_t ib_buffer;
-
-	struct {
-		uint32_t stride;
-		uint32_t start_addr;
-		uint32_t lines;
-	} dest_buf_cfg;
-
-	uint32_t items_per_store;
-	uint32_t stores_per_frame;
-
-	struct {
-		uint32_t sync_cmd;	/* must be _STREAM2MMIO_CMD_TOKEN_SYNC_FRAME */
-		uint32_t store_cmd;	/* must be _STREAM2MMIO_CMD_TOKEN_STORE_PACKETS */
-	} stream2mmio_cfg;
-};
-
-extern const uint32_t N_IBUF_CTRL_PROCS[N_IBUF_CTRL_ID];
-
-#endif /* __IBUF_CTRL_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/input_system_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/input_system_global.h
deleted file mode 100644
index 25e3f04f374b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/input_system_global.h
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_SYSTEM_GLOBAL_H_INCLUDED__
-#define __INPUT_SYSTEM_GLOBAL_H_INCLUDED__
-
-#define IS_INPUT_SYSTEM_VERSION_VERSION_2401
-
-/* CSI reveiver has 3 ports. */
-#define		N_CSI_PORTS (3)
-
-#include "isys_dma.h"		/*	isys2401_dma_channel,
-				 *	isys2401_dma_cfg_t
-				 */
-
-#include "ibuf_ctrl.h"		/*	ibuf_cfg_t,
-				 *	ibuf_ctrl_cfg_t
-				 */
-
-#include "isys_stream2mmio.h"	/*	stream2mmio_cfg_t */
-
-#include "csi_rx.h"		/*	csi_rx_frontend_cfg_t,
-				 *	csi_rx_backend_cfg_t,
-				 *	csi_rx_backend_lut_entry_t
-				 */
-#include "pixelgen.h"
-
-
-#define INPUT_SYSTEM_N_STREAM_ID  6	/* maximum number of simultaneous
-					virtual channels supported*/
-
-typedef enum {
-	INPUT_SYSTEM_ERR_NO_ERROR = 0,
-	INPUT_SYSTEM_ERR_CREATE_CHANNEL_FAIL,
-	INPUT_SYSTEM_ERR_CONFIGURE_CHANNEL_FAIL,
-	INPUT_SYSTEM_ERR_OPEN_CHANNEL_FAIL,
-	INPUT_SYSTEM_ERR_TRANSFER_FAIL,
-	INPUT_SYSTEM_ERR_CREATE_INPUT_PORT_FAIL,
-	INPUT_SYSTEM_ERR_CONFIGURE_INPUT_PORT_FAIL,
-	INPUT_SYSTEM_ERR_OPEN_INPUT_PORT_FAIL,
-	N_INPUT_SYSTEM_ERR
-} input_system_err_t;
-
-typedef enum {
-	INPUT_SYSTEM_SOURCE_TYPE_UNDEFINED = 0,
-	INPUT_SYSTEM_SOURCE_TYPE_SENSOR,
-	INPUT_SYSTEM_SOURCE_TYPE_TPG,
-	INPUT_SYSTEM_SOURCE_TYPE_PRBS,
-	N_INPUT_SYSTEM_SOURCE_TYPE
-} input_system_source_type_t;
-
-typedef enum {
-	INPUT_SYSTEM_POLL_ON_WAIT_FOR_FRAME,
-	INPUT_SYSTEM_POLL_ON_CAPTURE_REQUEST,
-} input_system_polling_mode_t;
-
-typedef struct input_system_channel_s input_system_channel_t;
-struct input_system_channel_s {
-	stream2mmio_ID_t	stream2mmio_id;
-	stream2mmio_sid_ID_t	stream2mmio_sid_id;
-
-	ibuf_ctrl_ID_t		ibuf_ctrl_id;
-	ib_buffer_t		ib_buffer;
-
-	isys2401_dma_ID_t	dma_id;
-	isys2401_dma_channel	dma_channel;
-};
-
-typedef struct input_system_channel_cfg_s input_system_channel_cfg_t;
-struct input_system_channel_cfg_s {
-	stream2mmio_cfg_t	stream2mmio_cfg;
-	ibuf_ctrl_cfg_t		ibuf_ctrl_cfg;
-	isys2401_dma_cfg_t	dma_cfg;
-	isys2401_dma_port_cfg_t	dma_src_port_cfg;
-	isys2401_dma_port_cfg_t	dma_dest_port_cfg;
-};
-
-typedef struct input_system_input_port_s input_system_input_port_t;
-struct input_system_input_port_s {
-	input_system_source_type_t	source_type;
-
-	struct {
-		csi_rx_frontend_ID_t		frontend_id;
-		csi_rx_backend_ID_t		backend_id;
-		csi_mipi_packet_type_t		packet_type;
-		csi_rx_backend_lut_entry_t	backend_lut_entry;
-	} csi_rx;
-
-	struct {
-		csi_mipi_packet_type_t		packet_type;
-		csi_rx_backend_lut_entry_t	backend_lut_entry;
-	} metadata;
-
-	struct {
-		pixelgen_ID_t			pixelgen_id;
-	} pixelgen;
-};
-
-typedef struct input_system_input_port_cfg_s input_system_input_port_cfg_t;
-struct input_system_input_port_cfg_s {
-	struct {
-		csi_rx_frontend_cfg_t	frontend_cfg;
-		csi_rx_backend_cfg_t	backend_cfg;
-		csi_rx_backend_cfg_t	md_backend_cfg;
-	} csi_rx_cfg;
-
-	struct {
-		pixelgen_tpg_cfg_t	tpg_cfg;
-		pixelgen_prbs_cfg_t	prbs_cfg;
-	} pixelgen_cfg;
-};
-
-typedef struct input_system_cfg_s input_system_cfg_t;
-struct input_system_cfg_s {
-	input_system_input_port_ID_t	input_port_id;
-
-	input_system_source_type_t	mode;
-#ifdef ISP2401
-	input_system_polling_mode_t	polling_mode;
-#endif
-
-	bool online;
-	bool raw_packed;
-	int8_t linked_isys_stream_id;
-
-	struct {
-		bool	comp_enable;
-		int32_t	active_lanes;
-		int32_t	fmt_type;
-		int32_t	ch_id;
-		int32_t comp_predictor;
-		int32_t comp_scheme;
-	} csi_port_attr;
-
-	pixelgen_tpg_cfg_t	tpg_port_attr;
-
-	pixelgen_prbs_cfg_t prbs_port_attr;
-
-	struct {
-		int32_t align_req_in_bytes;
-		int32_t bits_per_pixel;
-		int32_t pixels_per_line;
-		int32_t lines_per_frame;
-	} input_port_resolution;
-
-	struct {
-		int32_t left_padding;
-		int32_t max_isp_input_width;
-	} output_port_attr;
-
-	struct {
-		bool    enable;
-		int32_t fmt_type;
-		int32_t align_req_in_bytes;
-		int32_t bits_per_pixel;
-		int32_t pixels_per_line;
-		int32_t lines_per_frame;
-	} metadata;
-};
-
-typedef struct virtual_input_system_stream_s virtual_input_system_stream_t;
-struct virtual_input_system_stream_s {
-	uint32_t id;				/*Used when multiple MIPI data types and/or virtual channels are used.
-								Must be unique within one CSI RX
-								and lower than SH_CSS_MAX_ISYS_CHANNEL_NODES */
-	uint8_t enable_metadata;
-	input_system_input_port_t	input_port;
-	input_system_channel_t		channel;
-	input_system_channel_t		md_channel; /* metadata channel */
-	uint8_t online;
-	int8_t linked_isys_stream_id;
-	uint8_t valid;
-#ifdef ISP2401
-	input_system_polling_mode_t	polling_mode;
-	int32_t subscr_index;
-#endif
-};
-
-typedef struct virtual_input_system_stream_cfg_s virtual_input_system_stream_cfg_t;
-struct virtual_input_system_stream_cfg_s {
-	uint8_t enable_metadata;
-	input_system_input_port_cfg_t	input_port_cfg;
-	input_system_channel_cfg_t	channel_cfg;
-	input_system_channel_cfg_t	md_channel_cfg;
-	uint8_t valid;
-};
-
-#define ISP_INPUT_BUF_START_ADDR	0
-#define NUM_OF_INPUT_BUF		2
-#define NUM_OF_LINES_PER_BUF		2
-#define LINES_OF_ISP_INPUT_BUF		(NUM_OF_INPUT_BUF * NUM_OF_LINES_PER_BUF)
-#define ISP_INPUT_BUF_STRIDE		SH_CSS_MAX_SENSOR_WIDTH
-
-
-#endif /* __INPUT_SYSTEM_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_dma_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_dma_global.h
deleted file mode 100644
index 1be5c6956d65..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_dma_global.h
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_DMA_GLOBAL_H_INCLUDED__
-#define __ISYS_DMA_GLOBAL_H_INCLUDED__
-
-#include <type_support.h>
-
-#define HIVE_ISYS2401_DMA_IBUF_DDR_CONN	0
-#define HIVE_ISYS2401_DMA_IBUF_VMEM_CONN	1
-#define _DMA_V2_ZERO_EXTEND		0
-#define _DMA_V2_SIGN_EXTEND		1
-
-#define _DMA_ZERO_EXTEND     _DMA_V2_ZERO_EXTEND
-#define _DMA_SIGN_EXTEND     _DMA_V2_SIGN_EXTEND
-
-/********************************************************
- *
- * DMA Port.
- *
- * The DMA port definition for the input system
- * 2401 DMA is the duplication of the DMA port
- * definition for the CSS system DMA. It is duplicated
- * here just as the temporal step before the device libary
- * is available. The device libary is suppose to provide
- * the capability of reusing the control interface of the
- * same device prototypes. The refactor team will work on
- * this, right?
- *
- ********************************************************/
-typedef struct isys2401_dma_port_cfg_s isys2401_dma_port_cfg_t;
-struct isys2401_dma_port_cfg_s {
-	uint32_t stride;
-	uint32_t elements;
-	uint32_t cropping;
-	uint32_t width;
- };
-/* end of DMA Port */
-
-/************************************************
- *
- * DMA Device.
- *
- * The DMA device definition for the input system
- * 2401 DMA is the duplicattion of the DMA device
- * definition for the CSS system DMA. It is duplicated
- * here just as the temporal step before the device libary
- * is available. The device libary is suppose to provide
- * the capability of reusing the control interface of the
- * same device prototypes. The refactor team will work on
- * this, right?
- *
- ************************************************/
-typedef enum {
-	isys2401_dma_ibuf_to_ddr_connection	= HIVE_ISYS2401_DMA_IBUF_DDR_CONN,
-	isys2401_dma_ibuf_to_vmem_connection	= HIVE_ISYS2401_DMA_IBUF_VMEM_CONN
-} isys2401_dma_connection;
-
-typedef enum {
-  isys2401_dma_zero_extension = _DMA_ZERO_EXTEND,
-  isys2401_dma_sign_extension = _DMA_SIGN_EXTEND
-} isys2401_dma_extension;
-
-typedef struct isys2401_dma_cfg_s isys2401_dma_cfg_t;
-struct isys2401_dma_cfg_s {
-	isys2401_dma_channel	channel;
-	isys2401_dma_connection	connection;
-	isys2401_dma_extension	extension;
-	uint32_t		height;
-};
-/* end of DMA Device */
-
-/* isys2401_dma_channel limits per DMA ID */
-extern const isys2401_dma_channel N_ISYS2401_DMA_CHANNEL_PROCS[N_ISYS2401_DMA_ID];
-
-#endif /* __ISYS_DMA_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_irq_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_irq_global.h
deleted file mode 100644
index 41d051db3987..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_irq_global.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_IRQ_GLOBAL_H__
-#define __ISYS_IRQ_GLOBAL_H__
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-
-/* Register offset/index from base location */
-#define ISYS_IRQ_EDGE_REG_IDX		(0)
-#define ISYS_IRQ_MASK_REG_IDX		(ISYS_IRQ_EDGE_REG_IDX + 1)
-#define ISYS_IRQ_STATUS_REG_IDX		(ISYS_IRQ_EDGE_REG_IDX + 2)
-#define ISYS_IRQ_CLEAR_REG_IDX		(ISYS_IRQ_EDGE_REG_IDX + 3)
-#define ISYS_IRQ_ENABLE_REG_IDX		(ISYS_IRQ_EDGE_REG_IDX + 4)
-#define ISYS_IRQ_LEVEL_NO_REG_IDX	(ISYS_IRQ_EDGE_REG_IDX + 5)
-
-/* Register values */
-#define ISYS_IRQ_MASK_REG_VALUE		(0xFFFF)
-#define ISYS_IRQ_CLEAR_REG_VALUE	(0xFFFF)
-#define ISYS_IRQ_ENABLE_REG_VALUE	(0xFFFF)
-
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2401) */
-
-#endif	/* __ISYS_IRQ_GLOBAL_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_stream2mmio_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_stream2mmio_global.h
deleted file mode 100644
index 649f44fd2408..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/isys_stream2mmio_global.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_STREAM2MMIO_GLOBAL_H_INCLUDED__
-#define __ISYS_STREAM2MMIO_GLOBAL_H_INCLUDED__
-
-#include <type_support.h>
-
-typedef struct stream2mmio_cfg_s stream2mmio_cfg_t;
-struct stream2mmio_cfg_s {
-	uint32_t				bits_per_pixel;
-	uint32_t				enable_blocking;
-};
-
-/* Stream2MMIO limits  per ID*/
-/*
- * Stream2MMIO 0 has 8 SIDs that are indexed by
- * [STREAM2MMIO_SID0_ID...STREAM2MMIO_SID7_ID].
- *
- * Stream2MMIO 1 has 4 SIDs that are indexed by
- * [STREAM2MMIO_SID0_ID...TREAM2MMIO_SID3_ID].
- *
- * Stream2MMIO 2 has 4 SIDs that are indexed by
- * [STREAM2MMIO_SID0_ID...STREAM2MMIO_SID3_ID].
- */
-extern const stream2mmio_sid_ID_t N_STREAM2MMIO_SID_PROCS[N_STREAM2MMIO_ID];
-
-#endif /* __ISYS_STREAM2MMIO_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/pixelgen_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/pixelgen_global.h
deleted file mode 100644
index 0bf2feb8bbfb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/pixelgen_global.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __PIXELGEN_GLOBAL_H_INCLUDED__
-#define __PIXELGEN_GLOBAL_H_INCLUDED__
-
-#include <type_support.h>
-
-/**
- * Pixel-generator. ("pixelgen_global.h")
- */
-/*
- * Duplicates "sync_generator_cfg_t" in "input_system_global.h".
- */
-typedef struct sync_generator_cfg_s sync_generator_cfg_t;
-struct sync_generator_cfg_s {
-	uint32_t	hblank_cycles;
-	uint32_t	vblank_cycles;
-	uint32_t	pixels_per_clock;
-	uint32_t	nr_of_frames;
-	uint32_t	pixels_per_line;
-	uint32_t	lines_per_frame;
-};
-
-typedef enum {
-	PIXELGEN_TPG_MODE_RAMP = 0,
-	PIXELGEN_TPG_MODE_CHBO,
-	PIXELGEN_TPG_MODE_MONO,
-	N_PIXELGEN_TPG_MODE
-} pixelgen_tpg_mode_t;
-
-/*
- * "pixelgen_tpg_cfg_t" duplicates parts of
- * "tpg_cfg_t" in "input_system_global.h".
- */
-typedef struct pixelgen_tpg_cfg_s pixelgen_tpg_cfg_t;
-struct pixelgen_tpg_cfg_s {
-	pixelgen_tpg_mode_t	mode;	/* CHBO, MONO */
-
-	struct {
-		/* be used by CHBO and MON */
-		uint32_t R1;
-		uint32_t G1;
-		uint32_t B1;
-
-		/* be used by CHBO only */
-		uint32_t R2;
-		uint32_t G2;
-		uint32_t B2;
-	} color_cfg;
-
-	struct {
-		uint32_t	h_mask;		/* horizontal mask */
-		uint32_t	v_mask;		/* vertical mask */
-		uint32_t	hv_mask;	/* horizontal+vertical mask? */
-	} mask_cfg;
-
-	struct {
-		int32_t	h_delta;	/* horizontal delta? */
-		int32_t	v_delta;	/* vertical delta? */
-	} delta_cfg;
-
-	sync_generator_cfg_t	 sync_gen_cfg;
-};
-
-/*
- * "pixelgen_prbs_cfg_t" duplicates parts of
- * prbs_cfg_t" in "input_system_global.h".
- */
-typedef struct pixelgen_prbs_cfg_s pixelgen_prbs_cfg_t;
-struct pixelgen_prbs_cfg_s {
-	int32_t	seed0;
-	int32_t	seed1;
-
-	sync_generator_cfg_t	sync_gen_cfg;
-};
-
-/* end of Pixel-generator: TPG. ("pixelgen_global.h") */
-#endif /* __PIXELGEN_GLOBAL_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/spmem_dump.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/spmem_dump.c
deleted file mode 100644
index d733a3503a20..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/spmem_dump.c
+++ /dev/null
@@ -1,3686 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _sp_map_h_
-#define _sp_map_h_
-
-
-#ifndef _hrt_dummy_use_blob_sp
-#define _hrt_dummy_use_blob_sp()
-#endif
-
-#define _hrt_cell_load_program_sp(proc) _hrt_cell_load_program_embedded(proc, sp)
-
-#ifndef ISP2401
-/* function longjmp: 680D */
-#else
-/* function longjmp: 6A0B */
-#endif
-
-#ifndef ISP2401
-/* function tmpmem_init_dmem: 6558 */
-#else
-/* function tmpmem_init_dmem: 671E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_addr_B: 3C50 */
-#else
-/* function ia_css_dmaproxy_sp_set_addr_B: 3DC5 */
-
-/* function ia_css_pipe_data_init_tagger_resources: AC7 */
-#endif
-
-/* function debug_buffer_set_ddr_addr: DD */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_mipi
-#define HIVE_MEM_vbuf_mipi scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_mipi 0x7398
-#else
-#define HIVE_ADDR_vbuf_mipi 0x7444
-#endif
-#define HIVE_SIZE_vbuf_mipi 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_mipi scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_mipi 0x7398
-#else
-#define HIVE_ADDR_sp_vbuf_mipi 0x7444
-#endif
-#define HIVE_SIZE_sp_vbuf_mipi 12
-
-#ifndef ISP2401
-/* function ia_css_event_sp_decode: 3E41 */
-#else
-/* function ia_css_event_sp_decode: 3FB6 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_get_size: 51BF */
-#else
-/* function ia_css_queue_get_size: 53C8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_load: 5800 */
-#else
-/* function ia_css_queue_load: 59DF */
-#endif
-
-#ifndef ISP2401
-/* function setjmp: 6816 */
-#else
-/* function setjmp: 6A14 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_sfi_get_current_frame: 27BF */
-#else
-/* function ia_css_pipeline_sp_sfi_get_current_frame: 2790 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_sp2host_isys_event_queue
-#define HIVE_MEM_sem_for_sp2host_isys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_sp2host_isys_event_queue 0x5760
-#else
-#define HIVE_ADDR_sem_for_sp2host_isys_event_queue 0x57FC
-#endif
-#define HIVE_SIZE_sem_for_sp2host_isys_event_queue 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_sp2host_isys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_sp2host_isys_event_queue 0x5760
-#else
-#define HIVE_ADDR_sp_sem_for_sp2host_isys_event_queue 0x57FC
-#endif
-#define HIVE_SIZE_sp_sem_for_sp2host_isys_event_queue 20
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_wait_for_ack: 6DA9 */
-#else
-/* function ia_css_dmaproxy_sp_wait_for_ack: 6FF7 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_func: 596B */
-#else
-/* function ia_css_sp_rawcopy_func: 5B4A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_pop_marked: 3339 */
-#else
-/* function ia_css_tagger_buf_sp_pop_marked: 345C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_N_CSI_RX_BE_SID_WIDTH
-#define HIVE_MEM_N_CSI_RX_BE_SID_WIDTH scalar_processor_2400_dmem
-#define HIVE_ADDR_N_CSI_RX_BE_SID_WIDTH 0x1D0
-#define HIVE_SIZE_N_CSI_RX_BE_SID_WIDTH 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_N_CSI_RX_BE_SID_WIDTH scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_N_CSI_RX_BE_SID_WIDTH 0x1D0
-#define HIVE_SIZE_sp_N_CSI_RX_BE_SID_WIDTH 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_stage
-#define HIVE_MEM_isp_stage scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_stage 0x6C98
-#else
-#define HIVE_ADDR_isp_stage 0x6D48
-#endif
-#define HIVE_SIZE_isp_stage 832
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_stage scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_stage 0x6C98
-#else
-#define HIVE_ADDR_sp_isp_stage 0x6D48
-#endif
-#define HIVE_SIZE_sp_isp_stage 832
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_raw
-#define HIVE_MEM_vbuf_raw scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_raw 0x37C
-#else
-#define HIVE_ADDR_vbuf_raw 0x394
-#endif
-#define HIVE_SIZE_vbuf_raw 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_raw scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_raw 0x37C
-#else
-#define HIVE_ADDR_sp_vbuf_raw 0x394
-#endif
-#define HIVE_SIZE_sp_vbuf_raw 4
-
-#ifndef ISP2401
-/* function ia_css_sp_bin_copy_func: 594C */
-#else
-/* function ia_css_sp_bin_copy_func: 5B2B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_item_store: 554E */
-#else
-/* function ia_css_queue_item_store: 572D */
-#endif
-
-#ifndef ISP2401
-/* function input_system_reset: 1286 */
-#else
-/* function input_system_reset: 1201 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_metadata_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_metadata_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_metadata_bufs 0x5B38
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_metadata_bufs 0x5BE4
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_metadata_bufs 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_metadata_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_metadata_bufs 0x5B38
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_metadata_bufs 0x5BE4
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_metadata_bufs 20
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_buffer_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_buffer_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_buffer_bufs 0x5B4C
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_buffer_bufs 0x5BF8
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_buffer_bufs 160
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_buffer_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_buffer_bufs 0x5B4C
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_buffer_bufs 0x5BF8
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_buffer_bufs 160
-
-/* function sp_start_isp: 39C */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_binary_group
-#define HIVE_MEM_sp_binary_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_binary_group 0x7088
-#else
-#define HIVE_ADDR_sp_binary_group 0x7138
-#endif
-#define HIVE_SIZE_sp_binary_group 32
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_binary_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_binary_group 0x7088
-#else
-#define HIVE_ADDR_sp_sp_binary_group 0x7138
-#endif
-#define HIVE_SIZE_sp_sp_binary_group 32
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_sw_state
-#define HIVE_MEM_sp_sw_state scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sw_state 0x7344
-#else
-#define HIVE_ADDR_sp_sw_state 0x73F0
-#endif
-#define HIVE_SIZE_sp_sw_state 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_sw_state scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_sw_state 0x7344
-#else
-#define HIVE_ADDR_sp_sp_sw_state 0x73F0
-#endif
-#define HIVE_SIZE_sp_sp_sw_state 4
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_main: 13F7 */
-#else
-/* function ia_css_thread_sp_main: 136D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_internal_buffers: 4047 */
-#else
-/* function ia_css_ispctrl_sp_init_internal_buffers: 41F7 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp2host_psys_event_queue_handle
-#define HIVE_MEM_sp2host_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp2host_psys_event_queue_handle 0x5BEC
-#else
-#define HIVE_ADDR_sp2host_psys_event_queue_handle 0x5C98
-#endif
-#define HIVE_SIZE_sp2host_psys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp2host_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp2host_psys_event_queue_handle 0x5BEC
-#else
-#define HIVE_ADDR_sp_sp2host_psys_event_queue_handle 0x5C98
-#endif
-#define HIVE_SIZE_sp_sp2host_psys_event_queue_handle 12
-
-#ifndef ISP2401
-/* function pixelgen_unit_test: E68 */
-#else
-/* function pixelgen_unit_test: E62 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_sp2host_psys_event_queue
-#define HIVE_MEM_sem_for_sp2host_psys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_sp2host_psys_event_queue 0x5774
-#else
-#define HIVE_ADDR_sem_for_sp2host_psys_event_queue 0x5810
-#endif
-#define HIVE_SIZE_sem_for_sp2host_psys_event_queue 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_sp2host_psys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_sp2host_psys_event_queue 0x5774
-#else
-#define HIVE_ADDR_sp_sem_for_sp2host_psys_event_queue 0x5810
-#endif
-#define HIVE_SIZE_sp_sem_for_sp2host_psys_event_queue 20
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_propagate_frame: 2D52 */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_stop_copy_preview
-#define HIVE_MEM_sp_stop_copy_preview scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_stop_copy_preview 0x7328
-#define HIVE_SIZE_sp_stop_copy_preview 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_stop_copy_preview scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_stop_copy_preview 0x7328
-#define HIVE_SIZE_sp_sp_stop_copy_preview 4
-#else
-/* function ia_css_tagger_sp_propagate_frame: 2D23 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_handles
-#define HIVE_MEM_vbuf_handles scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_handles 0x73A4
-#else
-#define HIVE_ADDR_vbuf_handles 0x7450
-#endif
-#define HIVE_SIZE_vbuf_handles 960
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_handles scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_handles 0x73A4
-#else
-#define HIVE_ADDR_sp_vbuf_handles 0x7450
-#endif
-#define HIVE_SIZE_sp_vbuf_handles 960
-
-#ifndef ISP2401
-/* function ia_css_queue_store: 56B4 */
-
-/* function ia_css_sp_flash_register: 356E */
-#else
-/* function ia_css_queue_store: 5893 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_dummy_function: 5CF7 */
-#else
-/* function ia_css_sp_flash_register: 3691 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_init: 201C */
-#else
-/* function ia_css_pipeline_sp_init: 1FD7 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_configure: 2C42 */
-#else
-/* function ia_css_tagger_sp_configure: 2C13 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_end_binary: 3E8A */
-#else
-/* function ia_css_ispctrl_sp_end_binary: 3FFF */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs
-#define HIVE_MEM_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x5BF8
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x5CA4
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x5BF8
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x5CA4
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 20
-
-#ifndef ISP2401
-/* function pixelgen_tpg_run: F1E */
-#else
-/* function pixelgen_tpg_run: F18 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_event_is_pending_mask
-#define HIVE_MEM_event_is_pending_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_event_is_pending_mask 0x5C
-#define HIVE_SIZE_event_is_pending_mask 44
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_event_is_pending_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_event_is_pending_mask 0x5C
-#define HIVE_SIZE_sp_event_is_pending_mask 44
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cb_elems_frame
-#define HIVE_MEM_sp_all_cb_elems_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cb_elems_frame 0x5788
-#else
-#define HIVE_ADDR_sp_all_cb_elems_frame 0x5824
-#endif
-#define HIVE_SIZE_sp_all_cb_elems_frame 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cb_elems_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cb_elems_frame 0x5788
-#else
-#define HIVE_ADDR_sp_sp_all_cb_elems_frame 0x5824
-#endif
-#define HIVE_SIZE_sp_sp_all_cb_elems_frame 16
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp2host_isys_event_queue_handle
-#define HIVE_MEM_sp2host_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp2host_isys_event_queue_handle 0x5C0C
-#else
-#define HIVE_ADDR_sp2host_isys_event_queue_handle 0x5CB8
-#endif
-#define HIVE_SIZE_sp2host_isys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp2host_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp2host_isys_event_queue_handle 0x5C0C
-#else
-#define HIVE_ADDR_sp_sp2host_isys_event_queue_handle 0x5CB8
-#endif
-#define HIVE_SIZE_sp_sp2host_isys_event_queue_handle 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host_sp_com
-#define HIVE_MEM_host_sp_com scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host_sp_com 0x3E48
-#else
-#define HIVE_ADDR_host_sp_com 0x3E6C
-#endif
-#define HIVE_SIZE_host_sp_com 220
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host_sp_com scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host_sp_com 0x3E48
-#else
-#define HIVE_ADDR_sp_host_sp_com 0x3E6C
-#endif
-#define HIVE_SIZE_sp_host_sp_com 220
-
-#ifndef ISP2401
-/* function ia_css_queue_get_free_space: 5313 */
-#else
-/* function ia_css_queue_get_free_space: 54F2 */
-#endif
-
-#ifndef ISP2401
-/* function exec_image_pipe: 5E6 */
-#else
-/* function exec_image_pipe: 57A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_init_dmem_data
-#define HIVE_MEM_sp_init_dmem_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_init_dmem_data 0x7348
-#else
-#define HIVE_ADDR_sp_init_dmem_data 0x73F4
-#endif
-#define HIVE_SIZE_sp_init_dmem_data 24
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_init_dmem_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_init_dmem_data 0x7348
-#else
-#define HIVE_ADDR_sp_sp_init_dmem_data 0x73F4
-#endif
-#define HIVE_SIZE_sp_sp_init_dmem_data 24
-
-#ifndef ISP2401
-/* function ia_css_sp_metadata_start: 5DD1 */
-#else
-/* function ia_css_sp_metadata_start: 5EB3 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_init_buffer_queues: 35BF */
-#else
-/* function ia_css_bufq_sp_init_buffer_queues: 36E2 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_stop: 1FFF */
-#else
-/* function ia_css_pipeline_sp_stop: 1FBA */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_connect_pipes: 312C */
-#else
-/* function ia_css_tagger_sp_connect_pipes: 30FD */
-#endif
-
-#ifndef ISP2401
-/* function sp_isys_copy_wait: 644 */
-#else
-/* function sp_isys_copy_wait: 5D8 */
-#endif
-
-/* function is_isp_debug_buffer_full: 337 */
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_configure_channel_from_info: 3BD3 */
-#else
-/* function ia_css_dmaproxy_sp_configure_channel_from_info: 3D35 */
-#endif
-
-#ifndef ISP2401
-/* function encode_and_post_timer_event: AA8 */
-#else
-/* function encode_and_post_timer_event: A3C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_input_system_bz2788_active
-#define HIVE_MEM_input_system_bz2788_active scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_input_system_bz2788_active 0x250C
-#else
-#define HIVE_ADDR_input_system_bz2788_active 0x2524
-#endif
-#define HIVE_SIZE_input_system_bz2788_active 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_input_system_bz2788_active scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_input_system_bz2788_active 0x250C
-#else
-#define HIVE_ADDR_sp_input_system_bz2788_active 0x2524
-#endif
-#define HIVE_SIZE_sp_input_system_bz2788_active 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_N_IBUF_CTRL_PROCS
-#define HIVE_MEM_N_IBUF_CTRL_PROCS scalar_processor_2400_dmem
-#define HIVE_ADDR_N_IBUF_CTRL_PROCS 0x1FC
-#define HIVE_SIZE_N_IBUF_CTRL_PROCS 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_N_IBUF_CTRL_PROCS scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_N_IBUF_CTRL_PROCS 0x1FC
-#define HIVE_SIZE_sp_N_IBUF_CTRL_PROCS 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_per_frame_data
-#define HIVE_MEM_sp_per_frame_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_per_frame_data 0x3F24
-#else
-#define HIVE_ADDR_sp_per_frame_data 0x3F48
-#endif
-#define HIVE_SIZE_sp_per_frame_data 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_per_frame_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_per_frame_data 0x3F24
-#else
-#define HIVE_ADDR_sp_sp_per_frame_data 0x3F48
-#endif
-#define HIVE_SIZE_sp_sp_per_frame_data 4
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_vbuf_dequeue: 62AC */
-#else
-/* function ia_css_rmgr_sp_vbuf_dequeue: 6472 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_psys_event_queue_handle
-#define HIVE_MEM_host2sp_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_psys_event_queue_handle 0x5C18
-#else
-#define HIVE_ADDR_host2sp_psys_event_queue_handle 0x5CC4
-#endif
-#define HIVE_SIZE_host2sp_psys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_psys_event_queue_handle 0x5C18
-#else
-#define HIVE_ADDR_sp_host2sp_psys_event_queue_handle 0x5CC4
-#endif
-#define HIVE_SIZE_sp_host2sp_psys_event_queue_handle 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_xmem_bin_addr
-#define HIVE_MEM_xmem_bin_addr scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_xmem_bin_addr 0x3F28
-#else
-#define HIVE_ADDR_xmem_bin_addr 0x3F4C
-#endif
-#define HIVE_SIZE_xmem_bin_addr 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_xmem_bin_addr scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_xmem_bin_addr 0x3F28
-#else
-#define HIVE_ADDR_sp_xmem_bin_addr 0x3F4C
-#endif
-#define HIVE_SIZE_sp_xmem_bin_addr 4
-
-#ifndef ISP2401
-/* function tmr_clock_init: 16F9 */
-#else
-/* function tmr_clock_init: 166F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_run: 1ABF */
-#else
-/* function ia_css_pipeline_sp_run: 1A61 */
-#endif
-
-#ifndef ISP2401
-/* function memcpy: 68B6 */
-#else
-/* function memcpy: 6AB4 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_N_ISYS2401_DMA_CHANNEL_PROCS
-#define HIVE_MEM_N_ISYS2401_DMA_CHANNEL_PROCS scalar_processor_2400_dmem
-#define HIVE_ADDR_N_ISYS2401_DMA_CHANNEL_PROCS 0x214
-#define HIVE_SIZE_N_ISYS2401_DMA_CHANNEL_PROCS 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_N_ISYS2401_DMA_CHANNEL_PROCS scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_N_ISYS2401_DMA_CHANNEL_PROCS 0x214
-#define HIVE_SIZE_sp_N_ISYS2401_DMA_CHANNEL_PROCS 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_GP_DEVICE_BASE
-#define HIVE_MEM_GP_DEVICE_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_GP_DEVICE_BASE 0x384
-#else
-#define HIVE_ADDR_GP_DEVICE_BASE 0x39C
-#endif
-#define HIVE_SIZE_GP_DEVICE_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_GP_DEVICE_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_GP_DEVICE_BASE 0x384
-#else
-#define HIVE_ADDR_sp_GP_DEVICE_BASE 0x39C
-#endif
-#define HIVE_SIZE_sp_GP_DEVICE_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_thread_sp_ready_queue
-#define HIVE_MEM_ia_css_thread_sp_ready_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_thread_sp_ready_queue 0x278
-#else
-#define HIVE_ADDR_ia_css_thread_sp_ready_queue 0x27C
-#endif
-#define HIVE_SIZE_ia_css_thread_sp_ready_queue 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_thread_sp_ready_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_thread_sp_ready_queue 0x278
-#else
-#define HIVE_ADDR_sp_ia_css_thread_sp_ready_queue 0x27C
-#endif
-#define HIVE_SIZE_sp_ia_css_thread_sp_ready_queue 12
-
-#ifndef ISP2401
-/* function stream2mmio_send_command: E0A */
-#else
-/* function stream2mmio_send_command: E04 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_uds_sp_scale_params: 65BF */
-#else
-/* function ia_css_uds_sp_scale_params: 67BD */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_increase_size: 14DC */
-#else
-/* function ia_css_circbuf_increase_size: 1452 */
-#endif
-
-#ifndef ISP2401
-/* function __divu: 6834 */
-#else
-/* function __divu: 6A32 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_get_state: 131F */
-#else
-/* function ia_css_thread_sp_get_state: 1295 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_cont_capt_stop
-#define HIVE_MEM_sem_for_cont_capt_stop scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_cont_capt_stop 0x5798
-#else
-#define HIVE_ADDR_sem_for_cont_capt_stop 0x5834
-#endif
-#define HIVE_SIZE_sem_for_cont_capt_stop 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_cont_capt_stop scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_cont_capt_stop 0x5798
-#else
-#define HIVE_ADDR_sp_sem_for_cont_capt_stop 0x5834
-#endif
-#define HIVE_SIZE_sp_sem_for_cont_capt_stop 20
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_N_SHORT_PACKET_LUT_ENTRIES
-#define HIVE_MEM_N_SHORT_PACKET_LUT_ENTRIES scalar_processor_2400_dmem
-#define HIVE_ADDR_N_SHORT_PACKET_LUT_ENTRIES 0x1AC
-#define HIVE_SIZE_N_SHORT_PACKET_LUT_ENTRIES 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_N_SHORT_PACKET_LUT_ENTRIES scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_N_SHORT_PACKET_LUT_ENTRIES 0x1AC
-#define HIVE_SIZE_sp_N_SHORT_PACKET_LUT_ENTRIES 12
-
-#ifndef ISP2401
-/* function thread_fiber_sp_main: 14D5 */
-#else
-/* function thread_fiber_sp_main: 144B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_isp_pipe_thread
-#define HIVE_MEM_sp_isp_pipe_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_pipe_thread 0x58DC
-#define HIVE_SIZE_sp_isp_pipe_thread 340
-#else
-#define HIVE_ADDR_sp_isp_pipe_thread 0x5978
-#define HIVE_SIZE_sp_isp_pipe_thread 360
-#endif
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_isp_pipe_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_isp_pipe_thread 0x58DC
-#define HIVE_SIZE_sp_sp_isp_pipe_thread 340
-#else
-#define HIVE_ADDR_sp_sp_isp_pipe_thread 0x5978
-#define HIVE_SIZE_sp_sp_isp_pipe_thread 360
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_handle_parameter_sets: 193F */
-#else
-/* function ia_css_parambuf_sp_handle_parameter_sets: 18B5 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_set_state: 5DED */
-#else
-/* function ia_css_spctrl_sp_set_state: 5ECF */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sem_sp_signal: 6A99 */
-#else
-/* function ia_css_thread_sem_sp_signal: 6D18 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_IRQ_BASE
-#define HIVE_MEM_IRQ_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_IRQ_BASE 0x2C
-#define HIVE_SIZE_IRQ_BASE 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_IRQ_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_IRQ_BASE 0x2C
-#define HIVE_SIZE_sp_IRQ_BASE 16
-
-#ifndef ISP2401
-/* function ia_css_virtual_isys_sp_isr_init: 5E8C */
-#else
-/* function ia_css_virtual_isys_sp_isr_init: 5F70 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_TIMED_CTRL_BASE
-#define HIVE_MEM_TIMED_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_TIMED_CTRL_BASE 0x40
-#define HIVE_SIZE_TIMED_CTRL_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_TIMED_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_TIMED_CTRL_BASE 0x40
-#define HIVE_SIZE_sp_TIMED_CTRL_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_generate_exp_id: 613C */
-
-/* function ia_css_rmgr_sp_init: 61A7 */
-#else
-/* function ia_css_isys_sp_generate_exp_id: 6302 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sem_sp_init: 6B6A */
-#else
-/* function ia_css_rmgr_sp_init: 636D */
-#endif
-
-#ifndef ISP2401
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_is_isp_requested
-#define HIVE_MEM_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_is_isp_requested 0x390
-#define HIVE_SIZE_is_isp_requested 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_is_isp_requested 0x390
-#define HIVE_SIZE_sp_is_isp_requested 4
-#else
-/* function ia_css_thread_sem_sp_init: 6DE7 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_reading_cb_frame
-#define HIVE_MEM_sem_for_reading_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_reading_cb_frame 0x57AC
-#else
-#define HIVE_ADDR_sem_for_reading_cb_frame 0x5848
-#endif
-#define HIVE_SIZE_sem_for_reading_cb_frame 40
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_reading_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_reading_cb_frame 0x57AC
-#else
-#define HIVE_ADDR_sp_sem_for_reading_cb_frame 0x5848
-#endif
-#define HIVE_SIZE_sp_sem_for_reading_cb_frame 40
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_execute: 3B3B */
-#else
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_is_isp_requested
-#define HIVE_MEM_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_is_isp_requested 0x3A8
-#define HIVE_SIZE_is_isp_requested 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_is_isp_requested 0x3A8
-#define HIVE_SIZE_sp_is_isp_requested 4
-
-/* function ia_css_dmaproxy_sp_execute: 3C9B */
-#endif
-
-#ifndef ISP2401
-/* function csi_rx_backend_rst: CE6 */
-#else
-/* function csi_rx_backend_rst: CE0 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_is_empty: 51FA */
-#else
-/* function ia_css_queue_is_empty: 7144 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_has_stopped: 1FF5 */
-#else
-/* function ia_css_pipeline_sp_has_stopped: 1FB0 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_extract: 15E0 */
-#else
-/* function ia_css_circbuf_extract: 1556 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_is_locked_from_start: 344F */
-#else
-/* function ia_css_tagger_buf_sp_is_locked_from_start: 3572 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_current_sp_thread
-#define HIVE_MEM_current_sp_thread scalar_processor_2400_dmem
-#define HIVE_ADDR_current_sp_thread 0x274
-#define HIVE_SIZE_current_sp_thread 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_current_sp_thread scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_current_sp_thread 0x274
-#define HIVE_SIZE_sp_current_sp_thread 4
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_get_spid: 5DF4 */
-#else
-/* function ia_css_spctrl_sp_get_spid: 5ED6 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_reset_buffers: 3646 */
-#else
-/* function ia_css_bufq_sp_reset_buffers: 3769 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_read_byte_addr: 6DD7 */
-#else
-/* function ia_css_dmaproxy_sp_read_byte_addr: 7025 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_uninit: 61A0 */
-#else
-/* function ia_css_rmgr_sp_uninit: 6366 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_threads_stack
-#define HIVE_MEM_sp_threads_stack scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_threads_stack 0x164
-#define HIVE_SIZE_sp_threads_stack 24
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_threads_stack scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_threads_stack 0x164
-#define HIVE_SIZE_sp_sp_threads_stack 24
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_N_STREAM2MMIO_SID_PROCS
-#define HIVE_MEM_N_STREAM2MMIO_SID_PROCS scalar_processor_2400_dmem
-#define HIVE_ADDR_N_STREAM2MMIO_SID_PROCS 0x218
-#define HIVE_SIZE_N_STREAM2MMIO_SID_PROCS 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_N_STREAM2MMIO_SID_PROCS scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_N_STREAM2MMIO_SID_PROCS 0x218
-#define HIVE_SIZE_sp_N_STREAM2MMIO_SID_PROCS 12
-
-#ifndef ISP2401
-/* function ia_css_circbuf_peek: 15C2 */
-#else
-/* function ia_css_circbuf_peek: 1538 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_wait_for_in_param: 1708 */
-#else
-/* function ia_css_parambuf_sp_wait_for_in_param: 167E */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cb_elems_param
-#define HIVE_MEM_sp_all_cb_elems_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cb_elems_param 0x57D4
-#else
-#define HIVE_ADDR_sp_all_cb_elems_param 0x5870
-#endif
-#define HIVE_SIZE_sp_all_cb_elems_param 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cb_elems_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cb_elems_param 0x57D4
-#else
-#define HIVE_ADDR_sp_sp_all_cb_elems_param 0x5870
-#endif
-#define HIVE_SIZE_sp_sp_all_cb_elems_param 16
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_pipeline_sp_curr_binary_id
-#define HIVE_MEM_pipeline_sp_curr_binary_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_pipeline_sp_curr_binary_id 0x284
-#else
-#define HIVE_ADDR_pipeline_sp_curr_binary_id 0x288
-#endif
-#define HIVE_SIZE_pipeline_sp_curr_binary_id 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_pipeline_sp_curr_binary_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_pipeline_sp_curr_binary_id 0x284
-#else
-#define HIVE_ADDR_sp_pipeline_sp_curr_binary_id 0x288
-#endif
-#define HIVE_SIZE_sp_pipeline_sp_curr_binary_id 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_frame_desc
-#define HIVE_MEM_sp_all_cbs_frame_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_frame_desc 0x57E4
-#else
-#define HIVE_ADDR_sp_all_cbs_frame_desc 0x5880
-#endif
-#define HIVE_SIZE_sp_all_cbs_frame_desc 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_frame_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_frame_desc 0x57E4
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_frame_desc 0x5880
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_frame_desc 8
-
-#ifndef ISP2401
-/* function sp_isys_copy_func_v2: 629 */
-#else
-/* function sp_isys_copy_func_v2: 5BD */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_reading_cb_param
-#define HIVE_MEM_sem_for_reading_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_reading_cb_param 0x57EC
-#else
-#define HIVE_ADDR_sem_for_reading_cb_param 0x5888
-#endif
-#define HIVE_SIZE_sem_for_reading_cb_param 40
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_reading_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_reading_cb_param 0x57EC
-#else
-#define HIVE_ADDR_sp_sem_for_reading_cb_param 0x5888
-#endif
-#define HIVE_SIZE_sp_sem_for_reading_cb_param 40
-
-#ifndef ISP2401
-/* function ia_css_queue_get_used_space: 52C7 */
-#else
-/* function ia_css_queue_get_used_space: 54A6 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_cont_capt_start
-#define HIVE_MEM_sem_for_cont_capt_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_cont_capt_start 0x5814
-#else
-#define HIVE_ADDR_sem_for_cont_capt_start 0x58B0
-#endif
-#define HIVE_SIZE_sem_for_cont_capt_start 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_cont_capt_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_cont_capt_start 0x5814
-#else
-#define HIVE_ADDR_sp_sem_for_cont_capt_start 0x58B0
-#endif
-#define HIVE_SIZE_sp_sem_for_cont_capt_start 20
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_tmp_heap
-#define HIVE_MEM_tmp_heap scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_tmp_heap 0x70A8
-#else
-#define HIVE_ADDR_tmp_heap 0x7158
-#endif
-#define HIVE_SIZE_tmp_heap 640
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_tmp_heap scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_tmp_heap 0x70A8
-#else
-#define HIVE_ADDR_sp_tmp_heap 0x7158
-#endif
-#define HIVE_SIZE_sp_tmp_heap 640
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_get_num_vbuf: 64B0 */
-#else
-/* function ia_css_rmgr_sp_get_num_vbuf: 6676 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_output_compute_dma_info: 4863 */
-#else
-/* function ia_css_ispctrl_sp_output_compute_dma_info: 4A27 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_lock_exp_id: 2A0F */
-#else
-/* function ia_css_tagger_sp_lock_exp_id: 29E0 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_s3a_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_s3a_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_s3a_bufs 0x5C24
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_s3a_bufs 0x5CD0
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_s3a_bufs 60
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_s3a_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_s3a_bufs 0x5C24
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_s3a_bufs 0x5CD0
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_s3a_bufs 60
-
-#ifndef ISP2401
-/* function ia_css_queue_is_full: 535E */
-#else
-/* function ia_css_queue_is_full: 553D */
-#endif
-
-/* function debug_buffer_init_isp: E4 */
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_exp_id_is_locked: 2945 */
-#else
-/* function ia_css_tagger_sp_exp_id_is_locked: 2916 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_rmgr_sp_mipi_frame_sem
-#define HIVE_MEM_ia_css_rmgr_sp_mipi_frame_sem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_rmgr_sp_mipi_frame_sem 0x7764
-#else
-#define HIVE_ADDR_ia_css_rmgr_sp_mipi_frame_sem 0x7810
-#endif
-#define HIVE_SIZE_ia_css_rmgr_sp_mipi_frame_sem 60
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_rmgr_sp_mipi_frame_sem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_rmgr_sp_mipi_frame_sem 0x7764
-#else
-#define HIVE_ADDR_sp_ia_css_rmgr_sp_mipi_frame_sem 0x7810
-#endif
-#define HIVE_SIZE_sp_ia_css_rmgr_sp_mipi_frame_sem 60
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_dump: 6287 */
-#else
-/* function ia_css_rmgr_sp_refcount_dump: 644D */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_isp_parameters_id
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_isp_parameters_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x5C60
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x5D0C
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_isp_parameters_id 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x5C60
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x5D0C
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id 20
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_pipe_threads
-#define HIVE_MEM_sp_pipe_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_pipe_threads 0x150
-#define HIVE_SIZE_sp_pipe_threads 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_pipe_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_pipe_threads 0x150
-#define HIVE_SIZE_sp_sp_pipe_threads 20
-
-#ifndef ISP2401
-/* function sp_event_proxy_func: 78D */
-#else
-/* function sp_event_proxy_func: 721 */
-#endif
-
-#ifndef ISP2401
-/* function ibuf_ctrl_run: D7F */
-#else
-/* function ibuf_ctrl_run: D79 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_isys_event_queue_handle
-#define HIVE_MEM_host2sp_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_isys_event_queue_handle 0x5C74
-#else
-#define HIVE_ADDR_host2sp_isys_event_queue_handle 0x5D20
-#endif
-#define HIVE_SIZE_host2sp_isys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_isys_event_queue_handle 0x5C74
-#else
-#define HIVE_ADDR_sp_host2sp_isys_event_queue_handle 0x5D20
-#endif
-#define HIVE_SIZE_sp_host2sp_isys_event_queue_handle 12
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_yield: 6A12 */
-#else
-/* function ia_css_thread_sp_yield: 6C96 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_param_desc
-#define HIVE_MEM_sp_all_cbs_param_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_param_desc 0x5828
-#else
-#define HIVE_ADDR_sp_all_cbs_param_desc 0x58C4
-#endif
-#define HIVE_SIZE_sp_all_cbs_param_desc 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_param_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_param_desc 0x5828
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_param_desc 0x58C4
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_param_desc 8
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_dmaproxy_sp_invalidate_tlb
-#define HIVE_MEM_ia_css_dmaproxy_sp_invalidate_tlb scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb 0x6C8C
-#else
-#define HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb 0x6D38
-#endif
-#define HIVE_SIZE_ia_css_dmaproxy_sp_invalidate_tlb 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_dmaproxy_sp_invalidate_tlb scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_dmaproxy_sp_invalidate_tlb 0x6C8C
-#else
-#define HIVE_ADDR_sp_ia_css_dmaproxy_sp_invalidate_tlb 0x6D38
-#endif
-#define HIVE_SIZE_sp_ia_css_dmaproxy_sp_invalidate_tlb 4
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_fork: 13AC */
-#else
-/* function ia_css_thread_sp_fork: 1322 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_destroy: 3136 */
-#else
-/* function ia_css_tagger_sp_destroy: 3107 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_vmem_read: 3ADB */
-#else
-/* function ia_css_dmaproxy_sp_vmem_read: 3C3B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_N_LONG_PACKET_LUT_ENTRIES
-#define HIVE_MEM_N_LONG_PACKET_LUT_ENTRIES scalar_processor_2400_dmem
-#define HIVE_ADDR_N_LONG_PACKET_LUT_ENTRIES 0x1B8
-#define HIVE_SIZE_N_LONG_PACKET_LUT_ENTRIES 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_N_LONG_PACKET_LUT_ENTRIES scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_N_LONG_PACKET_LUT_ENTRIES 0x1B8
-#define HIVE_SIZE_sp_N_LONG_PACKET_LUT_ENTRIES 12
-
-#ifndef ISP2401
-/* function initialize_sp_group: 5F6 */
-#else
-/* function initialize_sp_group: 58A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_peek: 325B */
-#else
-/* function ia_css_tagger_buf_sp_peek: 337E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_init: 13D8 */
-#else
-/* function ia_css_thread_sp_init: 134E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_reset_exp_id: 6133 */
-#else
-/* function qos_scheduler_update_fps: 67AD */
-#endif
-
-#ifndef ISP2401
-/* function qos_scheduler_update_fps: 65AF */
-#else
-/* function ia_css_isys_sp_reset_exp_id: 62F9 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_set_stream_base_addr: 4F38 */
-#else
-/* function ia_css_ispctrl_sp_set_stream_base_addr: 5114 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_DMEM_BASE
-#define HIVE_MEM_ISP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_DMEM_BASE 0x10
-#define HIVE_SIZE_ISP_DMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_DMEM_BASE 0x10
-#define HIVE_SIZE_sp_ISP_DMEM_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_SP_DMEM_BASE
-#define HIVE_MEM_SP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_SP_DMEM_BASE 0x4
-#define HIVE_SIZE_SP_DMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_SP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_SP_DMEM_BASE 0x4
-#define HIVE_SIZE_sp_SP_DMEM_BASE 4
-
-#ifndef ISP2401
-/* function ibuf_ctrl_transfer: D67 */
-#else
-/* function ibuf_ctrl_transfer: D61 */
-
-/* function __ia_css_queue_is_empty_text: 5403 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_read: 3B51 */
-#else
-/* function ia_css_dmaproxy_sp_read: 3CB1 */
-#endif
-
-#ifndef ISP2401
-/* function virtual_isys_stream_is_capture_done: 5EB0 */
-#else
-/* function virtual_isys_stream_is_capture_done: 5F94 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_raw_copy_line_count
-#define HIVE_MEM_raw_copy_line_count scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_raw_copy_line_count 0x360
-#else
-#define HIVE_ADDR_raw_copy_line_count 0x378
-#endif
-#define HIVE_SIZE_raw_copy_line_count 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_raw_copy_line_count scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_raw_copy_line_count 0x360
-#else
-#define HIVE_ADDR_sp_raw_copy_line_count 0x378
-#endif
-#define HIVE_SIZE_sp_raw_copy_line_count 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_tag_cmd_queue_handle
-#define HIVE_MEM_host2sp_tag_cmd_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_tag_cmd_queue_handle 0x5C80
-#else
-#define HIVE_ADDR_host2sp_tag_cmd_queue_handle 0x5D2C
-#endif
-#define HIVE_SIZE_host2sp_tag_cmd_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_tag_cmd_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_tag_cmd_queue_handle 0x5C80
-#else
-#define HIVE_ADDR_sp_host2sp_tag_cmd_queue_handle 0x5D2C
-#endif
-#define HIVE_SIZE_sp_host2sp_tag_cmd_queue_handle 12
-
-#ifndef ISP2401
-/* function ia_css_queue_peek: 523D */
-#else
-/* function ia_css_queue_peek: 541C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_frame_cnt
-#define HIVE_MEM_ia_css_flash_sp_frame_cnt scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_frame_cnt 0x5B2C
-#else
-#define HIVE_ADDR_ia_css_flash_sp_frame_cnt 0x5BD8
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_frame_cnt 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_frame_cnt scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_frame_cnt 0x5B2C
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_frame_cnt 0x5BD8
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_frame_cnt 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_event_can_send_token_mask
-#define HIVE_MEM_event_can_send_token_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_event_can_send_token_mask 0x88
-#define HIVE_SIZE_event_can_send_token_mask 44
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_event_can_send_token_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_event_can_send_token_mask 0x88
-#define HIVE_SIZE_sp_event_can_send_token_mask 44
-
-#ifndef ISP2401
-/* function csi_rx_frontend_stop: C11 */
-#else
-/* function csi_rx_frontend_stop: C0B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_thread
-#define HIVE_MEM_isp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_thread 0x6FD8
-#else
-#define HIVE_ADDR_isp_thread 0x7088
-#endif
-#define HIVE_SIZE_isp_thread 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_thread 0x6FD8
-#else
-#define HIVE_ADDR_sp_isp_thread 0x7088
-#endif
-#define HIVE_SIZE_sp_isp_thread 4
-
-#ifndef ISP2401
-/* function encode_and_post_sp_event_non_blocking: AF0 */
-#else
-/* function encode_and_post_sp_event_non_blocking: A84 */
-#endif
-
-/* function is_ddr_debug_buffer_full: 2CC */
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_get_oldest_marked_offset: 32AB */
-#else
-/* function ia_css_tagger_buf_sp_get_oldest_marked_offset: 33CE */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_threads_fiber
-#define HIVE_MEM_sp_threads_fiber scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_threads_fiber 0x194
-#define HIVE_SIZE_sp_threads_fiber 24
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_threads_fiber scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_threads_fiber 0x194
-#define HIVE_SIZE_sp_sp_threads_fiber 24
-
-#ifndef ISP2401
-/* function encode_and_post_sp_event: A79 */
-#else
-/* function encode_and_post_sp_event: A0D */
-#endif
-
-/* function debug_enqueue_ddr: EE */
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_init_vbuf: 6242 */
-#else
-/* function ia_css_rmgr_sp_refcount_init_vbuf: 6408 */
-#endif
-
-#ifndef ISP2401
-/* function dmaproxy_sp_read_write: 6E86 */
-#else
-/* function dmaproxy_sp_read_write: 70C3 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_dmaproxy_isp_dma_cmd_buffer
-#define HIVE_MEM_ia_css_dmaproxy_isp_dma_cmd_buffer scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_dmaproxy_isp_dma_cmd_buffer 0x6C90
-#else
-#define HIVE_ADDR_ia_css_dmaproxy_isp_dma_cmd_buffer 0x6D3C
-#endif
-#define HIVE_SIZE_ia_css_dmaproxy_isp_dma_cmd_buffer 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_dmaproxy_isp_dma_cmd_buffer scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_dmaproxy_isp_dma_cmd_buffer 0x6C90
-#else
-#define HIVE_ADDR_sp_ia_css_dmaproxy_isp_dma_cmd_buffer 0x6D3C
-#endif
-#define HIVE_SIZE_sp_ia_css_dmaproxy_isp_dma_cmd_buffer 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_buffer_queue_handle
-#define HIVE_MEM_host2sp_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_buffer_queue_handle 0x5C8C
-#else
-#define HIVE_ADDR_host2sp_buffer_queue_handle 0x5D38
-#endif
-#define HIVE_SIZE_host2sp_buffer_queue_handle 480
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_buffer_queue_handle 0x5C8C
-#else
-#define HIVE_ADDR_sp_host2sp_buffer_queue_handle 0x5D38
-#endif
-#define HIVE_SIZE_sp_host2sp_buffer_queue_handle 480
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_in_service
-#define HIVE_MEM_ia_css_flash_sp_in_service scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_in_service 0x3054
-#else
-#define HIVE_ADDR_ia_css_flash_sp_in_service 0x3074
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_in_service 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_in_service scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_service 0x3054
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_service 0x3074
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_in_service 4
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_process: 6B92 */
-#else
-/* function ia_css_dmaproxy_sp_process: 6E0F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_mark_from_end: 3533 */
-#else
-/* function ia_css_tagger_buf_sp_mark_from_end: 3656 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_cs: 3F77 */
-#else
-/* function ia_css_ispctrl_sp_init_cs: 40FA */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_init: 5E02 */
-#else
-/* function ia_css_spctrl_sp_init: 5EE4 */
-#endif
-
-#ifndef ISP2401
-/* function sp_event_proxy_init: 7A2 */
-#else
-/* function sp_event_proxy_init: 736 */
-#endif
-
-#ifndef ISP2401
-/* function input_system_input_port_close: 109B */
-#else
-/* function input_system_input_port_close: 1095 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_previous_clock_tick
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_previous_clock_tick scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x5E6C
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x5F18
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_previous_clock_tick 40
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x5E6C
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x5F18
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick 40
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_output
-#define HIVE_MEM_sp_output scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_output 0x3F2C
-#else
-#define HIVE_ADDR_sp_output 0x3F50
-#endif
-#define HIVE_SIZE_sp_output 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_output scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_output 0x3F2C
-#else
-#define HIVE_ADDR_sp_sp_output 0x3F50
-#endif
-#define HIVE_SIZE_sp_sp_output 16
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_sems_for_host2sp_buf_queues
-#define HIVE_MEM_ia_css_bufq_sp_sems_for_host2sp_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x5E94
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x5F40
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_sems_for_host2sp_buf_queues 800
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x5E94
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x5F40
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues 800
-
-#ifndef ISP2401
-/* function pixelgen_prbs_config: E93 */
-#else
-/* function pixelgen_prbs_config: E8D */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_CTRL_BASE
-#define HIVE_MEM_ISP_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_CTRL_BASE 0x8
-#define HIVE_SIZE_ISP_CTRL_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_CTRL_BASE 0x8
-#define HIVE_SIZE_sp_ISP_CTRL_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_INPUT_FORMATTER_BASE
-#define HIVE_MEM_INPUT_FORMATTER_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_INPUT_FORMATTER_BASE 0x4C
-#define HIVE_SIZE_INPUT_FORMATTER_BASE 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_INPUT_FORMATTER_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_INPUT_FORMATTER_BASE 0x4C
-#define HIVE_SIZE_sp_INPUT_FORMATTER_BASE 16
-
-#ifndef ISP2401
-/* function sp_dma_proxy_reset_channels: 3DAB */
-#else
-/* function sp_dma_proxy_reset_channels: 3F20 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_update_size: 322A */
-#else
-/* function ia_css_tagger_sp_update_size: 334D */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_host_sp_queue
-#define HIVE_MEM_ia_css_bufq_host_sp_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_host_sp_queue 0x61B4
-#else
-#define HIVE_ADDR_ia_css_bufq_host_sp_queue 0x6260
-#endif
-#define HIVE_SIZE_ia_css_bufq_host_sp_queue 2008
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_host_sp_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_host_sp_queue 0x61B4
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_host_sp_queue 0x6260
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_host_sp_queue 2008
-
-#ifndef ISP2401
-/* function thread_fiber_sp_create: 1444 */
-#else
-/* function thread_fiber_sp_create: 13BA */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_increments: 3C3D */
-#else
-/* function ia_css_dmaproxy_sp_set_increments: 3DB2 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_writing_cb_frame
-#define HIVE_MEM_sem_for_writing_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_writing_cb_frame 0x5830
-#else
-#define HIVE_ADDR_sem_for_writing_cb_frame 0x58CC
-#endif
-#define HIVE_SIZE_sem_for_writing_cb_frame 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_writing_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_writing_cb_frame 0x5830
-#else
-#define HIVE_ADDR_sp_sem_for_writing_cb_frame 0x58CC
-#endif
-#define HIVE_SIZE_sp_sem_for_writing_cb_frame 20
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_writing_cb_param
-#define HIVE_MEM_sem_for_writing_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_writing_cb_param 0x5844
-#else
-#define HIVE_ADDR_sem_for_writing_cb_param 0x58E0
-#endif
-#define HIVE_SIZE_sem_for_writing_cb_param 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_writing_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_writing_cb_param 0x5844
-#else
-#define HIVE_ADDR_sp_sem_for_writing_cb_param 0x58E0
-#endif
-#define HIVE_SIZE_sp_sem_for_writing_cb_param 20
-
-#ifndef ISP2401
-/* function pixelgen_tpg_is_done: F0D */
-#else
-/* function pixelgen_tpg_is_done: F07 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_stream_capture_indication: 5FB6 */
-#else
-/* function ia_css_isys_stream_capture_indication: 60D7 */
-#endif
-
-/* function sp_start_isp_entry: 392 */
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifdef HIVE_ADDR_sp_start_isp_entry
-#endif
-#define HIVE_ADDR_sp_start_isp_entry 0x392
-#endif
-#define HIVE_ADDR_sp_sp_start_isp_entry 0x392
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_unmark_all: 34B7 */
-#else
-/* function ia_css_tagger_buf_sp_unmark_all: 35DA */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_unmark_from_start: 34F8 */
-#else
-/* function ia_css_tagger_buf_sp_unmark_from_start: 361B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_channel_acquire: 3DD7 */
-#else
-/* function ia_css_dmaproxy_sp_channel_acquire: 3F4C */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_add_num_vbuf: 648C */
-#else
-/* function ia_css_rmgr_sp_add_num_vbuf: 6652 */
-#endif
-
-#ifndef ISP2401
-/* function ibuf_ctrl_config: D8B */
-#else
-/* function ibuf_ctrl_config: D85 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_stream_stop: 602E */
-#else
-/* function ia_css_isys_stream_stop: 61F4 */
-#endif
-
-#ifndef ISP2401
-/* function __ia_css_dmaproxy_sp_wait_for_ack_text: 3AA7 */
-#else
-/* function __ia_css_dmaproxy_sp_wait_for_ack_text: 3C07 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_acquire_buf_elem: 291D */
-#else
-/* function ia_css_tagger_sp_acquire_buf_elem: 28EE */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_is_dynamic_buffer: 3990 */
-#else
-/* function ia_css_bufq_sp_is_dynamic_buffer: 3AB3 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_group
-#define HIVE_MEM_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_group 0x3F3C
-#define HIVE_SIZE_sp_group 6176
-#else
-#define HIVE_ADDR_sp_group 0x3F60
-#define HIVE_SIZE_sp_group 6296
-#endif
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_group 0x3F3C
-#define HIVE_SIZE_sp_sp_group 6176
-#else
-#define HIVE_ADDR_sp_sp_group 0x3F60
-#define HIVE_SIZE_sp_sp_group 6296
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_event_proxy_thread
-#define HIVE_MEM_sp_event_proxy_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_event_proxy_thread 0x5A30
-#define HIVE_SIZE_sp_event_proxy_thread 68
-#else
-#define HIVE_ADDR_sp_event_proxy_thread 0x5AE0
-#define HIVE_SIZE_sp_event_proxy_thread 72
-#endif
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_event_proxy_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_event_proxy_thread 0x5A30
-#define HIVE_SIZE_sp_sp_event_proxy_thread 68
-#else
-#define HIVE_ADDR_sp_sp_event_proxy_thread 0x5AE0
-#define HIVE_SIZE_sp_sp_event_proxy_thread 72
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_kill: 1372 */
-#else
-/* function ia_css_thread_sp_kill: 12E8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_create: 31E4 */
-#else
-/* function ia_css_tagger_sp_create: 32FB */
-#endif
-
-#ifndef ISP2401
-/* function tmpmem_acquire_dmem: 6539 */
-#else
-/* function tmpmem_acquire_dmem: 66FF */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_MMU_BASE
-#define HIVE_MEM_MMU_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_MMU_BASE 0x24
-#define HIVE_SIZE_MMU_BASE 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_MMU_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_MMU_BASE 0x24
-#define HIVE_SIZE_sp_MMU_BASE 8
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_channel_release: 3DC3 */
-#else
-/* function ia_css_dmaproxy_sp_channel_release: 3F38 */
-#endif
-
-#ifndef ISP2401
-/* function pixelgen_prbs_run: E81 */
-#else
-/* function pixelgen_prbs_run: E7B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_is_idle: 3DA3 */
-#else
-/* function ia_css_dmaproxy_sp_is_idle: 3F18 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_qos_start
-#define HIVE_MEM_sem_for_qos_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_qos_start 0x5858
-#else
-#define HIVE_ADDR_sem_for_qos_start 0x58F4
-#endif
-#define HIVE_SIZE_sem_for_qos_start 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_qos_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_qos_start 0x5858
-#else
-#define HIVE_ADDR_sp_sem_for_qos_start 0x58F4
-#endif
-#define HIVE_SIZE_sp_sem_for_qos_start 20
-
-#ifndef ISP2401
-/* function isp_hmem_load: B63 */
-#else
-/* function isp_hmem_load: B5D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_release_buf_elem: 28F9 */
-#else
-/* function ia_css_tagger_sp_release_buf_elem: 28CA */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_eventq_sp_send: 3E19 */
-#else
-/* function ia_css_eventq_sp_send: 3F8E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_unlock_from_start: 33E7 */
-#else
-/* function ia_css_tagger_buf_sp_unlock_from_start: 350A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_debug_buffer_ddr_address
-#define HIVE_MEM_debug_buffer_ddr_address scalar_processor_2400_dmem
-#define HIVE_ADDR_debug_buffer_ddr_address 0xBC
-#define HIVE_SIZE_debug_buffer_ddr_address 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_debug_buffer_ddr_address scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_debug_buffer_ddr_address 0xBC
-#define HIVE_SIZE_sp_debug_buffer_ddr_address 4
-
-#ifndef ISP2401
-/* function sp_isys_copy_request: 6ED */
-#else
-/* function sp_isys_copy_request: 681 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_retain_vbuf: 631C */
-#else
-/* function ia_css_rmgr_sp_refcount_retain_vbuf: 64E2 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_set_priority: 136A */
-#else
-/* function ia_css_thread_sp_set_priority: 12E0 */
-#endif
-
-#ifndef ISP2401
-/* function sizeof_hmem: C0A */
-#else
-/* function sizeof_hmem: C04 */
-#endif
-
-#ifndef ISP2401
-/* function input_system_channel_open: 1241 */
-#else
-/* function input_system_channel_open: 11BC */
-#endif
-
-#ifndef ISP2401
-/* function pixelgen_tpg_stop: EFB */
-#else
-/* function pixelgen_tpg_stop: EF5 */
-#endif
-
-#ifndef ISP2401
-/* function tmpmem_release_dmem: 6528 */
-#else
-/* function tmpmem_release_dmem: 66EE */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_width_exception: 3C28 */
-#else
-/* function __ia_css_dmaproxy_sp_process_text: 3BAB */
-#endif
-
-#ifndef ISP2401
-/* function sp_event_assert: 929 */
-#else
-/* function ia_css_dmaproxy_sp_set_width_exception: 3D9D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_flash_sp_init_internal_params: 35B4 */
-#else
-/* function sp_event_assert: 8BD */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_pop_unmarked_and_unlocked: 32ED */
-#else
-/* function ia_css_flash_sp_init_internal_params: 36D7 */
-#endif
-
-#ifndef ISP2401
-/* function __modu: 687A */
-#else
-/* function ia_css_tagger_buf_sp_pop_unmarked_and_unlocked: 3410 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init_isp_vector: 3AAD */
-#else
-/* function __modu: 6A78 */
-#endif
-
-#ifndef ISP2401
-/* function input_system_channel_transfer: 122A */
-#else
-/* function ia_css_dmaproxy_sp_init_isp_vector: 3C0D */
-
-/* function input_system_channel_transfer: 11A5 */
-#endif
-
-/* function isp_vamem_store: 0 */
-
-#ifdef ISP2401
-/* function ia_css_tagger_sp_set_copy_pipe: 32F2 */
-
-#endif
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_GDC_BASE
-#define HIVE_MEM_GDC_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_GDC_BASE 0x44
-#define HIVE_SIZE_GDC_BASE 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_GDC_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_GDC_BASE 0x44
-#define HIVE_SIZE_sp_GDC_BASE 8
-
-#ifndef ISP2401
-/* function ia_css_queue_local_init: 5528 */
-#else
-/* function ia_css_queue_local_init: 5707 */
-#endif
-
-#ifndef ISP2401
-/* function sp_event_proxy_callout_func: 6947 */
-#else
-/* function sp_event_proxy_callout_func: 6B45 */
-#endif
-
-#ifndef ISP2401
-/* function qos_scheduler_schedule_stage: 6580 */
-#else
-/* function qos_scheduler_schedule_stage: 6759 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_thread_sp_num_ready_threads
-#define HIVE_MEM_ia_css_thread_sp_num_ready_threads scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_thread_sp_num_ready_threads 0x5A78
-#else
-#define HIVE_ADDR_ia_css_thread_sp_num_ready_threads 0x5B28
-#endif
-#define HIVE_SIZE_ia_css_thread_sp_num_ready_threads 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_thread_sp_num_ready_threads scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_thread_sp_num_ready_threads 0x5A78
-#else
-#define HIVE_ADDR_sp_ia_css_thread_sp_num_ready_threads 0x5B28
-#endif
-#define HIVE_SIZE_sp_ia_css_thread_sp_num_ready_threads 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_threads_stack_size
-#define HIVE_MEM_sp_threads_stack_size scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_threads_stack_size 0x17C
-#define HIVE_SIZE_sp_threads_stack_size 24
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_threads_stack_size scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_threads_stack_size 0x17C
-#define HIVE_SIZE_sp_sp_threads_stack_size 24
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_isp_done_row_striping: 4849 */
-#else
-/* function ia_css_ispctrl_sp_isp_done_row_striping: 4A0D */
-#endif
-
-#ifndef ISP2401
-/* function __ia_css_virtual_isys_sp_isr_text: 5E45 */
-#else
-/* function __ia_css_virtual_isys_sp_isr_text: 5F4E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_dequeue: 53A6 */
-#else
-/* function ia_css_queue_dequeue: 5585 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_configure_channel: 6DEE */
-#else
-/* function is_qos_standalone_mode: 6734 */
-
-/* function ia_css_dmaproxy_sp_configure_channel: 703C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_current_thread_fiber_sp
-#define HIVE_MEM_current_thread_fiber_sp scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_current_thread_fiber_sp 0x5A80
-#else
-#define HIVE_ADDR_current_thread_fiber_sp 0x5B2C
-#endif
-#define HIVE_SIZE_current_thread_fiber_sp 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_current_thread_fiber_sp scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_current_thread_fiber_sp 0x5A80
-#else
-#define HIVE_ADDR_sp_current_thread_fiber_sp 0x5B2C
-#endif
-#define HIVE_SIZE_sp_current_thread_fiber_sp 4
-
-#ifndef ISP2401
-/* function ia_css_circbuf_pop: 1674 */
-#else
-/* function ia_css_circbuf_pop: 15EA */
-#endif
-
-#ifndef ISP2401
-/* function memset: 68F9 */
-#else
-/* function memset: 6AF7 */
-#endif
-
-/* function irq_raise_set_token: B6 */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_GPIO_BASE
-#define HIVE_MEM_GPIO_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_GPIO_BASE 0x3C
-#define HIVE_SIZE_GPIO_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_GPIO_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_GPIO_BASE 0x3C
-#define HIVE_SIZE_sp_GPIO_BASE 4
-
-#ifndef ISP2401
-/* function pixelgen_prbs_stop: E6F */
-#else
-/* function pixelgen_prbs_stop: E69 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_acc_stage_enable: 1FC0 */
-#else
-/* function ia_css_pipeline_acc_stage_enable: 1F69 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_unlock_exp_id: 296A */
-#else
-/* function ia_css_tagger_sp_unlock_exp_id: 293B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_ph
-#define HIVE_MEM_isp_ph scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_ph 0x7360
-#else
-#define HIVE_ADDR_isp_ph 0x740C
-#endif
-#define HIVE_SIZE_isp_ph 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_ph scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_ph 0x7360
-#else
-#define HIVE_ADDR_sp_isp_ph 0x740C
-#endif
-#define HIVE_SIZE_sp_isp_ph 28
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_ds: 40D6 */
-#else
-/* function ia_css_ispctrl_sp_init_ds: 4286 */
-#endif
-
-#ifndef ISP2401
-/* function get_xmem_base_addr_raw: 4479 */
-#else
-/* function get_xmem_base_addr_raw: 4635 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_param
-#define HIVE_MEM_sp_all_cbs_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_param 0x586C
-#else
-#define HIVE_ADDR_sp_all_cbs_param 0x5908
-#endif
-#define HIVE_SIZE_sp_all_cbs_param 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_param 0x586C
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_param 0x5908
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_param 16
-
-#ifndef ISP2401
-/* function pixelgen_tpg_config: F30 */
-#else
-/* function pixelgen_tpg_config: F2A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_create: 16C2 */
-#else
-/* function ia_css_circbuf_create: 1638 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_sp_group
-#define HIVE_MEM_sem_for_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_sp_group 0x587C
-#else
-#define HIVE_ADDR_sem_for_sp_group 0x5918
-#endif
-#define HIVE_SIZE_sem_for_sp_group 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_sp_group 0x587C
-#else
-#define HIVE_ADDR_sp_sem_for_sp_group 0x5918
-#endif
-#define HIVE_SIZE_sp_sem_for_sp_group 20
-
-#ifndef ISP2401
-/* function csi_rx_frontend_run: C22 */
-#else
-/* function csi_rx_frontend_run: C1C */
-
-/* function __ia_css_dmaproxy_sp_configure_channel_text: 3D7C */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_framebuf_sp_wait_for_in_frame: 64B7 */
-#else
-/* function ia_css_framebuf_sp_wait_for_in_frame: 667D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_stream_open: 60E3 */
-#else
-/* function ia_css_isys_stream_open: 62A9 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_tag_frame: 5C71 */
-#else
-/* function ia_css_sp_rawcopy_tag_frame: 5E35 */
-#endif
-
-#ifndef ISP2401
-/* function input_system_channel_configure: 125D */
-#else
-/* function input_system_channel_configure: 11D8 */
-#endif
-
-#ifndef ISP2401
-/* function isp_hmem_clear: B33 */
-#else
-/* function isp_hmem_clear: B2D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_framebuf_sp_release_in_frame: 64FA */
-#else
-/* function ia_css_framebuf_sp_release_in_frame: 66C0 */
-#endif
-
-#ifndef ISP2401
-/* function stream2mmio_config: E1B */
-#else
-/* function stream2mmio_config: E15 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_start_binary: 3F55 */
-#else
-/* function ia_css_ispctrl_sp_start_binary: 40D8 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_h_pipe_private_ddr_ptrs
-#define HIVE_MEM_ia_css_bufq_sp_h_pipe_private_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x698C
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x6A38
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x698C
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x6A38
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 20
-
-#ifndef ISP2401
-/* function ia_css_eventq_sp_recv: 3DEB */
-#else
-/* function ia_css_eventq_sp_recv: 3F60 */
-#endif
-
-#ifndef ISP2401
-/* function csi_rx_frontend_config: C7A */
-#else
-/* function csi_rx_frontend_config: C74 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_pool
-#define HIVE_MEM_isp_pool scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_pool 0x370
-#else
-#define HIVE_ADDR_isp_pool 0x388
-#endif
-#define HIVE_SIZE_isp_pool 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_pool scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_pool 0x370
-#else
-#define HIVE_ADDR_sp_isp_pool 0x388
-#endif
-#define HIVE_SIZE_sp_isp_pool 4
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_rel_gen: 61E9 */
-#else
-/* function ia_css_rmgr_sp_rel_gen: 63AF */
-
-/* function ia_css_tagger_sp_unblock_clients: 31C3 */
-#endif
-
-#ifndef ISP2401
-/* function css_get_frame_processing_time_end: 28E9 */
-#else
-/* function css_get_frame_processing_time_end: 28BA */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_event_any_pending_mask
-#define HIVE_MEM_event_any_pending_mask scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_event_any_pending_mask 0x388
-#else
-#define HIVE_ADDR_event_any_pending_mask 0x3A0
-#endif
-#define HIVE_SIZE_event_any_pending_mask 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_event_any_pending_mask scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_event_any_pending_mask 0x388
-#else
-#define HIVE_ADDR_sp_event_any_pending_mask 0x3A0
-#endif
-#define HIVE_SIZE_sp_event_any_pending_mask 8
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_get_pipe_io_status: 1AB8 */
-#else
-/* function ia_css_pipeline_sp_get_pipe_io_status: 1A5A */
-#endif
-
-/* function sh_css_decode_tag_descr: 352 */
-
-/* function debug_enqueue_isp: 27B */
-
-#ifndef ISP2401
-/* function qos_scheduler_update_stage_budget: 656E */
-#else
-/* function qos_scheduler_update_stage_budget: 673C */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_uninit: 5DFB */
-#else
-/* function ia_css_spctrl_sp_uninit: 5EDD */
-#endif
-
-#ifndef ISP2401
-/* function csi_rx_backend_run: C68 */
-#else
-/* function csi_rx_backend_run: C62 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_dis_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_dis_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_dis_bufs 0x69A0
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_dis_bufs 0x6A4C
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_dis_bufs 140
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_dis_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_dis_bufs 0x69A0
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_dis_bufs 0x6A4C
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_dis_bufs 140
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_lock_from_start: 341B */
-#else
-/* function ia_css_tagger_buf_sp_lock_from_start: 353E */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_isp_idle
-#define HIVE_MEM_sem_for_isp_idle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_isp_idle 0x5890
-#else
-#define HIVE_ADDR_sem_for_isp_idle 0x592C
-#endif
-#define HIVE_SIZE_sem_for_isp_idle 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_isp_idle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_isp_idle 0x5890
-#else
-#define HIVE_ADDR_sp_sem_for_isp_idle 0x592C
-#endif
-#define HIVE_SIZE_sp_sem_for_isp_idle 20
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_write_byte_addr: 3B0A */
-#else
-/* function ia_css_dmaproxy_sp_write_byte_addr: 3C6A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init: 3A81 */
-#else
-/* function ia_css_dmaproxy_sp_init: 3BE1 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_release_dynamic_buf_clock_tick: 3686 */
-#else
-/* function ia_css_bufq_sp_release_dynamic_buf_clock_tick: 37A9 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_VAMEM_BASE
-#define HIVE_MEM_ISP_VAMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_VAMEM_BASE 0x14
-#define HIVE_SIZE_ISP_VAMEM_BASE 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_VAMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_VAMEM_BASE 0x14
-#define HIVE_SIZE_sp_ISP_VAMEM_BASE 12
-
-#ifndef ISP2401
-/* function input_system_channel_sync: 11A4 */
-#else
-/* function input_system_channel_sync: 6C10 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_rawcopy_sp_tagger
-#define HIVE_MEM_ia_css_rawcopy_sp_tagger scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_rawcopy_sp_tagger 0x732C
-#else
-#define HIVE_ADDR_ia_css_rawcopy_sp_tagger 0x73D8
-#endif
-#define HIVE_SIZE_ia_css_rawcopy_sp_tagger 24
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_rawcopy_sp_tagger scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_rawcopy_sp_tagger 0x732C
-#else
-#define HIVE_ADDR_sp_ia_css_rawcopy_sp_tagger 0x73D8
-#endif
-#define HIVE_SIZE_sp_ia_css_rawcopy_sp_tagger 24
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_exp_ids
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_exp_ids scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_exp_ids 0x6A2C
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_exp_ids 0x6AD8
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_exp_ids 70
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_exp_ids scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_exp_ids 0x6A2C
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_exp_ids 0x6AD8
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_exp_ids 70
-
-#ifndef ISP2401
-/* function ia_css_queue_item_load: 561A */
-#else
-/* function ia_css_queue_item_load: 57F9 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_get_state: 5DE6 */
-#else
-/* function ia_css_spctrl_sp_get_state: 5EC8 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_callout_sp_thread
-#define HIVE_MEM_callout_sp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_callout_sp_thread 0x5A74
-#else
-#define HIVE_ADDR_callout_sp_thread 0x278
-#endif
-#define HIVE_SIZE_callout_sp_thread 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_callout_sp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_callout_sp_thread 0x5A74
-#else
-#define HIVE_ADDR_sp_callout_sp_thread 0x278
-#endif
-#define HIVE_SIZE_sp_callout_sp_thread 4
-
-#ifndef ISP2401
-/* function thread_fiber_sp_init: 14CB */
-#else
-/* function thread_fiber_sp_init: 1441 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_SP_PMEM_BASE
-#define HIVE_MEM_SP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_SP_PMEM_BASE 0x0
-#define HIVE_SIZE_SP_PMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_SP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_SP_PMEM_BASE 0x0
-#define HIVE_SIZE_sp_SP_PMEM_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_isp_input_stream_format
-#define HIVE_MEM_sp_isp_input_stream_format scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_input_stream_format 0x3E2C
-#else
-#define HIVE_ADDR_sp_isp_input_stream_format 0x3E50
-#endif
-#define HIVE_SIZE_sp_isp_input_stream_format 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_isp_input_stream_format scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_isp_input_stream_format 0x3E2C
-#else
-#define HIVE_ADDR_sp_sp_isp_input_stream_format 0x3E50
-#endif
-#define HIVE_SIZE_sp_sp_isp_input_stream_format 20
-
-#ifndef ISP2401
-/* function __mod: 6866 */
-#else
-/* function __mod: 6A64 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init_dmem_channel: 3B6B */
-#else
-/* function ia_css_dmaproxy_sp_init_dmem_channel: 3CCB */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_join: 139B */
-#else
-/* function ia_css_thread_sp_join: 1311 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_add_command: 6EF1 */
-#else
-/* function ia_css_dmaproxy_sp_add_command: 712E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_metadata_thread_func: 5DDF */
-#else
-/* function ia_css_sp_metadata_thread_func: 5EC1 */
-#endif
-
-#ifndef ISP2401
-/* function __sp_event_proxy_func_critical: 6934 */
-#else
-/* function __sp_event_proxy_func_critical: 6B32 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_wait_for_isys_stream_N: 5F53 */
-#else
-/* function ia_css_pipeline_sp_wait_for_isys_stream_N: 6074 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_metadata_wait: 5DD8 */
-#else
-/* function ia_css_sp_metadata_wait: 5EBA */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_peek_from_start: 15A4 */
-#else
-/* function ia_css_circbuf_peek_from_start: 151A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_event_sp_encode: 3E76 */
-#else
-/* function ia_css_event_sp_encode: 3FEB */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_run: 140E */
-#else
-/* function ia_css_thread_sp_run: 1384 */
-#endif
-
-#ifndef ISP2401
-/* function sp_isys_copy_func: 618 */
-#else
-/* function sp_isys_copy_func: 5AC */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_isp_param_init_isp_memories: 50A3 */
-#else
-/* function ia_css_sp_isp_param_init_isp_memories: 52AC */
-#endif
-
-#ifndef ISP2401
-/* function register_isr: 921 */
-#else
-/* function register_isr: 8B5 */
-#endif
-
-/* function irq_raise: C8 */
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_mmu_invalidate: 3A48 */
-#else
-/* function ia_css_dmaproxy_sp_mmu_invalidate: 3B71 */
-#endif
-
-#ifndef ISP2401
-/* function csi_rx_backend_disable: C34 */
-#else
-/* function csi_rx_backend_disable: C2E */
-#endif
-
-#ifndef ISP2401
-/* function pipeline_sp_initialize_stage: 2104 */
-#else
-/* function pipeline_sp_initialize_stage: 20BF */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_N_CSI_RX_FE_CTRL_DLANES
-#define HIVE_MEM_N_CSI_RX_FE_CTRL_DLANES scalar_processor_2400_dmem
-#define HIVE_ADDR_N_CSI_RX_FE_CTRL_DLANES 0x1C4
-#define HIVE_SIZE_N_CSI_RX_FE_CTRL_DLANES 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_N_CSI_RX_FE_CTRL_DLANES scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_N_CSI_RX_FE_CTRL_DLANES 0x1C4
-#define HIVE_SIZE_sp_N_CSI_RX_FE_CTRL_DLANES 12
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_read_byte_addr_mmio: 6DC0 */
-#else
-/* function ia_css_dmaproxy_sp_read_byte_addr_mmio: 700E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_done_ds: 40BD */
-#else
-/* function ia_css_ispctrl_sp_done_ds: 426D */
-#endif
-
-#ifndef ISP2401
-/* function csi_rx_backend_config: C8B */
-#else
-/* function csi_rx_backend_config: C85 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_isp_param_get_mem_inits: 507E */
-#else
-/* function ia_css_sp_isp_param_get_mem_inits: 5287 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_init_buffer_queues: 1A85 */
-#else
-/* function ia_css_parambuf_sp_init_buffer_queues: 1A27 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_pfp_spref
-#define HIVE_MEM_vbuf_pfp_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_pfp_spref 0x378
-#else
-#define HIVE_ADDR_vbuf_pfp_spref 0x390
-#endif
-#define HIVE_SIZE_vbuf_pfp_spref 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_pfp_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_pfp_spref 0x378
-#else
-#define HIVE_ADDR_sp_vbuf_pfp_spref 0x390
-#endif
-#define HIVE_SIZE_sp_vbuf_pfp_spref 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_HMEM_BASE
-#define HIVE_MEM_ISP_HMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_HMEM_BASE 0x20
-#define HIVE_SIZE_ISP_HMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_HMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_HMEM_BASE 0x20
-#define HIVE_SIZE_sp_ISP_HMEM_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_frames
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_frames 0x6A74
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_frames 0x6B20
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_frames 280
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_frames 0x6A74
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_frames 0x6B20
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_frames 280
-
-#ifndef ISP2401
-/* function qos_scheduler_init_stage_budget: 65A7 */
-#else
-/* function qos_scheduler_init_stage_budget: 679A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp2host_buffer_queue_handle
-#define HIVE_MEM_sp2host_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp2host_buffer_queue_handle 0x6B8C
-#else
-#define HIVE_ADDR_sp2host_buffer_queue_handle 0x6C38
-#endif
-#define HIVE_SIZE_sp2host_buffer_queue_handle 96
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp2host_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp2host_buffer_queue_handle 0x6B8C
-#else
-#define HIVE_ADDR_sp_sp2host_buffer_queue_handle 0x6C38
-#endif
-#define HIVE_SIZE_sp_sp2host_buffer_queue_handle 96
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_isp_vars: 4D9D */
-#else
-/* function ia_css_ispctrl_sp_init_isp_vars: 4F79 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_stream_start: 6010 */
-#else
-/* function ia_css_isys_stream_start: 6187 */
-#endif
-
-#ifndef ISP2401
-/* function sp_warning: 954 */
-#else
-/* function sp_warning: 8E8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_vbuf_enqueue: 62DC */
-#else
-/* function ia_css_rmgr_sp_vbuf_enqueue: 64A2 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_tag_exp_id: 2A84 */
-#else
-/* function ia_css_tagger_sp_tag_exp_id: 2A55 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_sfi_release_current_frame: 276B */
-#else
-/* function ia_css_pipeline_sp_sfi_release_current_frame: 273C */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_write: 3B21 */
-#else
-/* function ia_css_dmaproxy_sp_write: 3C81 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_stream_start_async: 608A */
-#else
-/* function ia_css_isys_stream_start_async: 6250 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_release_in_param: 1905 */
-#else
-/* function ia_css_parambuf_sp_release_in_param: 187B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_irq_sw_interrupt_token
-#define HIVE_MEM_irq_sw_interrupt_token scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_irq_sw_interrupt_token 0x3E28
-#else
-#define HIVE_ADDR_irq_sw_interrupt_token 0x3E4C
-#endif
-#define HIVE_SIZE_irq_sw_interrupt_token 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_irq_sw_interrupt_token scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_irq_sw_interrupt_token 0x3E28
-#else
-#define HIVE_ADDR_sp_irq_sw_interrupt_token 0x3E4C
-#endif
-#define HIVE_SIZE_sp_irq_sw_interrupt_token 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_isp_addresses
-#define HIVE_MEM_sp_isp_addresses scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_addresses 0x6FDC
-#else
-#define HIVE_ADDR_sp_isp_addresses 0x708C
-#endif
-#define HIVE_SIZE_sp_isp_addresses 172
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_isp_addresses scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_isp_addresses 0x6FDC
-#else
-#define HIVE_ADDR_sp_sp_isp_addresses 0x708C
-#endif
-#define HIVE_SIZE_sp_sp_isp_addresses 172
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_acq_gen: 6201 */
-#else
-/* function ia_css_rmgr_sp_acq_gen: 63C7 */
-#endif
-
-#ifndef ISP2401
-/* function input_system_input_port_open: 10ED */
-#else
-/* function input_system_input_port_open: 10E7 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isps
-#define HIVE_MEM_isps scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isps 0x737C
-#else
-#define HIVE_ADDR_isps 0x7428
-#endif
-#define HIVE_SIZE_isps 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isps scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isps 0x737C
-#else
-#define HIVE_ADDR_sp_isps 0x7428
-#endif
-#define HIVE_SIZE_sp_isps 28
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host_sp_queues_initialized
-#define HIVE_MEM_host_sp_queues_initialized scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host_sp_queues_initialized 0x3E40
-#else
-#define HIVE_ADDR_host_sp_queues_initialized 0x3E64
-#endif
-#define HIVE_SIZE_host_sp_queues_initialized 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host_sp_queues_initialized scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host_sp_queues_initialized 0x3E40
-#else
-#define HIVE_ADDR_sp_host_sp_queues_initialized 0x3E64
-#endif
-#define HIVE_SIZE_sp_host_sp_queues_initialized 4
-
-#ifndef ISP2401
-/* function ia_css_queue_uninit: 54E6 */
-#else
-/* function ia_css_queue_uninit: 56C5 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_ispctrl_sp_isp_started
-#define HIVE_MEM_ia_css_ispctrl_sp_isp_started scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_ispctrl_sp_isp_started 0x6C94
-#else
-#define HIVE_ADDR_ia_css_ispctrl_sp_isp_started 0x6D40
-#endif
-#define HIVE_SIZE_ia_css_ispctrl_sp_isp_started 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_ispctrl_sp_isp_started scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_ispctrl_sp_isp_started 0x6C94
-#else
-#define HIVE_ADDR_sp_ia_css_ispctrl_sp_isp_started 0x6D40
-#endif
-#define HIVE_SIZE_sp_ia_css_ispctrl_sp_isp_started 4
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_release_dynamic_buf: 36F2 */
-#else
-/* function ia_css_bufq_sp_release_dynamic_buf: 3815 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_height_exception: 3C19 */
-#else
-/* function ia_css_dmaproxy_sp_set_height_exception: 3D8E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init_vmem_channel: 3B9E */
-#else
-/* function ia_css_dmaproxy_sp_init_vmem_channel: 3CFF */
-#endif
-
-#ifndef ISP2401
-/* function csi_rx_backend_stop: C57 */
-#else
-/* function csi_rx_backend_stop: C51 */
-#endif
-
-#ifndef ISP2401
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_num_ready_threads
-#define HIVE_MEM_num_ready_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_num_ready_threads 0x5A7C
-#define HIVE_SIZE_num_ready_threads 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_num_ready_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_num_ready_threads 0x5A7C
-#define HIVE_SIZE_sp_num_ready_threads 4
-
-/* function ia_css_dmaproxy_sp_write_byte_addr_mmio: 3AF3 */
-#else
-/* function ia_css_dmaproxy_sp_write_byte_addr_mmio: 3C53 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_spref
-#define HIVE_MEM_vbuf_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_spref 0x374
-#else
-#define HIVE_ADDR_vbuf_spref 0x38C
-#endif
-#define HIVE_SIZE_vbuf_spref 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_spref 0x374
-#else
-#define HIVE_ADDR_sp_vbuf_spref 0x38C
-#endif
-#define HIVE_SIZE_sp_vbuf_spref 4
-
-#ifndef ISP2401
-/* function ia_css_queue_enqueue: 5430 */
-#else
-/* function ia_css_queue_enqueue: 560F */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_request
-#define HIVE_MEM_ia_css_flash_sp_request scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_request 0x5B30
-#else
-#define HIVE_ADDR_ia_css_flash_sp_request 0x5BDC
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_request 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_request scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_request 0x5B30
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_request 0x5BDC
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_request 4
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_vmem_write: 3AC4 */
-#else
-/* function ia_css_dmaproxy_sp_vmem_write: 3C24 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_tagger_frames
-#define HIVE_MEM_tagger_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_tagger_frames 0x5A84
-#else
-#define HIVE_ADDR_tagger_frames 0x5B30
-#endif
-#define HIVE_SIZE_tagger_frames 168
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_tagger_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_tagger_frames 0x5A84
-#else
-#define HIVE_ADDR_sp_tagger_frames 0x5B30
-#endif
-#define HIVE_SIZE_sp_tagger_frames 168
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_reading_if
-#define HIVE_MEM_sem_for_reading_if scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_reading_if 0x58A4
-#else
-#define HIVE_ADDR_sem_for_reading_if 0x5940
-#endif
-#define HIVE_SIZE_sem_for_reading_if 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_reading_if scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_reading_if 0x58A4
-#else
-#define HIVE_ADDR_sp_sem_for_reading_if 0x5940
-#endif
-#define HIVE_SIZE_sp_sem_for_reading_if 20
-
-#ifndef ISP2401
-/* function sp_generate_interrupts: 9D3 */
-#else
-/* function sp_generate_interrupts: 967 */
-
-/* function ia_css_pipeline_sp_start: 1FC2 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_start: 2007 */
-#else
-/* function ia_css_thread_default_callout: 6C8F */
-#endif
-
-#ifndef ISP2401
-/* function csi_rx_backend_enable: C45 */
-#else
-/* function csi_rx_backend_enable: C3F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_init: 5953 */
-#else
-/* function ia_css_sp_rawcopy_init: 5B32 */
-#endif
-
-#ifndef ISP2401
-/* function input_system_input_port_configure: 113F */
-#else
-/* function input_system_input_port_configure: 1139 */
-#endif
-
-#ifndef ISP2401
-/* function tmr_clock_read: 16EF */
-#else
-/* function tmr_clock_read: 1665 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_BAMEM_BASE
-#define HIVE_MEM_ISP_BAMEM_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ISP_BAMEM_BASE 0x380
-#else
-#define HIVE_ADDR_ISP_BAMEM_BASE 0x398
-#endif
-#define HIVE_SIZE_ISP_BAMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_BAMEM_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ISP_BAMEM_BASE 0x380
-#else
-#define HIVE_ADDR_sp_ISP_BAMEM_BASE 0x398
-#endif
-#define HIVE_SIZE_sp_ISP_BAMEM_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_sems_for_sp2host_buf_queues
-#define HIVE_MEM_ia_css_bufq_sp_sems_for_sp2host_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x6BEC
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x6C98
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_sems_for_sp2host_buf_queues 160
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x6BEC
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x6C98
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues 160
-
-#ifndef ISP2401
-/* function isys2401_dma_config_legacy: DE0 */
-#else
-/* function isys2401_dma_config_legacy: DDA */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ibuf_ctrl_master_ports
-#define HIVE_MEM_ibuf_ctrl_master_ports scalar_processor_2400_dmem
-#define HIVE_ADDR_ibuf_ctrl_master_ports 0x208
-#define HIVE_SIZE_ibuf_ctrl_master_ports 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ibuf_ctrl_master_ports scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ibuf_ctrl_master_ports 0x208
-#define HIVE_SIZE_sp_ibuf_ctrl_master_ports 12
-
-#ifndef ISP2401
-/* function css_get_frame_processing_time_start: 28F1 */
-#else
-/* function css_get_frame_processing_time_start: 28C2 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_frame
-#define HIVE_MEM_sp_all_cbs_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_frame 0x58B8
-#else
-#define HIVE_ADDR_sp_all_cbs_frame 0x5954
-#endif
-#define HIVE_SIZE_sp_all_cbs_frame 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_frame 0x58B8
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_frame 0x5954
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_frame 16
-
-#ifndef ISP2401
-/* function ia_css_virtual_isys_sp_isr: 6F07 */
-#else
-/* function ia_css_virtual_isys_sp_isr: 716E */
-#endif
-
-#ifndef ISP2401
-/* function thread_sp_queue_print: 142B */
-#else
-/* function thread_sp_queue_print: 13A1 */
-#endif
-
-#ifndef ISP2401
-/* function sp_notify_eof: 97F */
-#else
-/* function sp_notify_eof: 913 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_str2mem
-#define HIVE_MEM_sem_for_str2mem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_str2mem 0x58C8
-#else
-#define HIVE_ADDR_sem_for_str2mem 0x5964
-#endif
-#define HIVE_SIZE_sem_for_str2mem 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_str2mem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_str2mem 0x58C8
-#else
-#define HIVE_ADDR_sp_sem_for_str2mem 0x5964
-#endif
-#define HIVE_SIZE_sp_sem_for_str2mem 20
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_is_marked_from_start: 3483 */
-#else
-/* function ia_css_tagger_buf_sp_is_marked_from_start: 35A6 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_acquire_dynamic_buf: 38AA */
-#else
-/* function ia_css_bufq_sp_acquire_dynamic_buf: 39CD */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_sfi_mode_is_enabled: 28BF */
-#else
-/* function ia_css_pipeline_sp_sfi_mode_is_enabled: 2890 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_destroy: 16B9 */
-#else
-/* function ia_css_circbuf_destroy: 162F */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_PMEM_BASE
-#define HIVE_MEM_ISP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_PMEM_BASE 0xC
-#define HIVE_SIZE_ISP_PMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_PMEM_BASE 0xC
-#define HIVE_SIZE_sp_ISP_PMEM_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_sp_isp_param_mem_load: 5011 */
-#else
-/* function ia_css_sp_isp_param_mem_load: 521A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_pop_from_start: 326F */
-#else
-/* function ia_css_tagger_buf_sp_pop_from_start: 3392 */
-#endif
-
-#ifndef ISP2401
-/* function __div: 681E */
-#else
-/* function __div: 6A1C */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_release_vbuf: 62FB */
-#else
-/* function ia_css_rmgr_sp_refcount_release_vbuf: 64C1 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_in_use
-#define HIVE_MEM_ia_css_flash_sp_in_use scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_in_use 0x5B34
-#else
-#define HIVE_ADDR_ia_css_flash_sp_in_use 0x5BE0
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_in_use 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_in_use scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_use 0x5B34
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_use 0x5BE0
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_in_use 4
-
-#ifndef ISP2401
-/* function ia_css_thread_sem_sp_wait: 6AE4 */
-#else
-/* function ia_css_thread_sem_sp_wait: 6D63 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_sleep_mode
-#define HIVE_MEM_sp_sleep_mode scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sleep_mode 0x3E44
-#else
-#define HIVE_ADDR_sp_sleep_mode 0x3E68
-#endif
-#define HIVE_SIZE_sp_sleep_mode 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_sleep_mode scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_sleep_mode 0x3E44
-#else
-#define HIVE_ADDR_sp_sp_sleep_mode 0x3E68
-#endif
-#define HIVE_SIZE_sp_sp_sleep_mode 4
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_push: 337E */
-#else
-/* function ia_css_tagger_buf_sp_push: 34A1 */
-#endif
-
-/* function mmu_invalidate_cache: D3 */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_max_cb_elems
-#define HIVE_MEM_sp_max_cb_elems scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_max_cb_elems 0x148
-#define HIVE_SIZE_sp_max_cb_elems 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_max_cb_elems scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_max_cb_elems 0x148
-#define HIVE_SIZE_sp_sp_max_cb_elems 8
-
-#ifndef ISP2401
-/* function ia_css_queue_remote_init: 5508 */
-#else
-/* function ia_css_queue_remote_init: 56E7 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_stop_req
-#define HIVE_MEM_isp_stop_req scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_stop_req 0x575C
-#else
-#define HIVE_ADDR_isp_stop_req 0x57F8
-#endif
-#define HIVE_SIZE_isp_stop_req 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_stop_req scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_stop_req 0x575C
-#else
-#define HIVE_ADDR_sp_isp_stop_req 0x57F8
-#endif
-#define HIVE_SIZE_sp_isp_stop_req 4
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_sfi_request_next_frame: 2781 */
-#else
-/* function ia_css_pipeline_sp_sfi_request_next_frame: 2752 */
-#endif
-
-#ifndef ISP2401
-#define HIVE_ICACHE_sp_critical_SEGMENT_START 0
-#define HIVE_ICACHE_sp_critical_NUM_SEGMENTS  1
-#endif
-
-#endif /* _sp_map_h_ */
-#ifndef ISP2401
-extern void sh_css_dump_sp_dmem(void);
-void sh_css_dump_sp_dmem(void)
-{
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/system_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/system_global.h
deleted file mode 100644
index 7907f0ff6d6c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_csi2p_system/system_global.h
+++ /dev/null
@@ -1,458 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SYSTEM_GLOBAL_H_INCLUDED__
-#define __SYSTEM_GLOBAL_H_INCLUDED__
-
-#include <hive_isp_css_defs.h>
-#include <type_support.h>
-
-/*
- * The longest allowed (uninteruptible) bus transfer, does not
- * take stalling into account
- */
-#define HIVE_ISP_MAX_BURST_LENGTH	1024
-
-/*
- * Maximum allowed burst length in words for the ISP DMA
- * This value is set to 2 to prevent the ISP DMA from blocking
- * the bus for too long; as the input system can only buffer
- * 2 lines on Moorefield and Cherrytrail, the input system buffers
- * may overflow if blocked for too long (BZ 2726).
- */
-#define ISP_DMA_MAX_BURST_LENGTH	2
-
-/*
- * Create a list of HAS and IS properties that defines the system
- *
- * The configuration assumes the following
- * - The system is hetereogeneous; Multiple cells and devices classes
- * - The cell and device instances are homogeneous, each device type
- *   belongs to the same class
- * - Device instances supporting a subset of the class capabilities are
- *   allowed
- *
- * We could manage different device classes through the enumerated
- * lists (C) or the use of classes (C++), but that is presently not
- * fully supported
- *
- * N.B. the 3 input formatters are of 2 different classess
- */
-
-#define USE_INPUT_SYSTEM_VERSION_2401
-
-#define IS_ISP_2400_SYSTEM
-/*
- * Since this file is visible everywhere and the system definition
- * macros are not, detect the separate definitions for {host, SP, ISP}
- *
- * The 2401 system has the nice property that it uses a vanilla 2400 SP
- * so the SP will believe it is a 2400 system rather than 2401...
- */
-/* #if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada) || defined(__scalar_processor_2401) */
-#if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada)
-#define IS_ISP_2401_MAMOIADA_SYSTEM
-#define HAS_ISP_2401_MAMOIADA
-#define HAS_SP_2400
-/* #elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada) || defined(__scalar_processor_2400)*/
-#elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada)
-#define IS_ISP_2400_MAMOIADA_SYSTEM
-#define HAS_ISP_2400_MAMOIADA
-#define HAS_SP_2400
-#else
-#error "system_global.h: 2400_SYSTEM must be one of {2400, 2401 }"
-#endif
-
-#define HAS_MMU_VERSION_2
-#define HAS_DMA_VERSION_2
-#define HAS_GDC_VERSION_2
-#define HAS_VAMEM_VERSION_2
-#define HAS_HMEM_VERSION_1
-#define HAS_BAMEM_VERSION_2
-#define HAS_IRQ_VERSION_2
-#define HAS_IRQ_MAP_VERSION_2
-#define HAS_INPUT_FORMATTER_VERSION_2
-/* 2401: HAS_INPUT_SYSTEM_VERSION_3 */
-/* 2400: HAS_INPUT_SYSTEM_VERSION_2 */
-#define HAS_INPUT_SYSTEM_VERSION_2
-#define HAS_INPUT_SYSTEM_VERSION_2401
-#define HAS_BUFFERED_SENSOR
-#define HAS_FIFO_MONITORS_VERSION_2
-/* #define HAS_GP_REGS_VERSION_2 */
-#define HAS_GP_DEVICE_VERSION_2
-#define HAS_GPIO_VERSION_1
-#define HAS_TIMED_CTRL_VERSION_1
-#define HAS_RX_VERSION_2
-#define HAS_NO_INPUT_FORMATTER
-/*#define HAS_NO_PACKED_RAW_PIXELS*/
-/*#define HAS_NO_DVS_6AXIS_CONFIG_UPDATE*/
-
-#define DMA_DDR_TO_VAMEM_WORKAROUND
-#define DMA_DDR_TO_HMEM_WORKAROUND
-
-
-/*
- * Semi global. "HRT" is accessible from SP, but
- * the HRT types do not fully apply
- */
-#define HRT_VADDRESS_WIDTH	32
-/* Surprise, this is a local property*/
-/*#define HRT_ADDRESS_WIDTH	64 */
-#define HRT_DATA_WIDTH		32
-
-#define SIZEOF_HRT_REG		(HRT_DATA_WIDTH>>3)
-#define HIVE_ISP_CTRL_DATA_BYTES (HIVE_ISP_CTRL_DATA_WIDTH/8)
-
-/* The main bus connecting all devices */
-#define HRT_BUS_WIDTH		HIVE_ISP_CTRL_DATA_WIDTH
-#define HRT_BUS_BYTES		HIVE_ISP_CTRL_DATA_BYTES
-
-#define CSI2P_DISABLE_ISYS2401_ONLINE_MODE
-
-/* per-frame parameter handling support */
-#define SH_CSS_ENABLE_PER_FRAME_PARAMS
-
-typedef uint32_t			hrt_bus_align_t;
-
-/*
- * Enumerate the devices, device access through the API is by ID,
- * through the DLI by address. The enumerator terminators are used
- * to size the wiring arrays and as an exception value.
- */
-typedef enum {
-	DDR0_ID = 0,
-	N_DDR_ID
-} ddr_ID_t;
-
-typedef enum {
-	ISP0_ID = 0,
-	N_ISP_ID
-} isp_ID_t;
-
-typedef enum {
-	SP0_ID = 0,
-	N_SP_ID
-} sp_ID_t;
-
-#if defined(IS_ISP_2401_MAMOIADA_SYSTEM)
-typedef enum {
-	MMU0_ID = 0,
-	MMU1_ID,
-	N_MMU_ID
-} mmu_ID_t;
-#elif defined(IS_ISP_2400_MAMOIADA_SYSTEM)
-typedef enum {
-	MMU0_ID = 0,
-	MMU1_ID,
-	N_MMU_ID
-} mmu_ID_t;
-#else
-#error "system_global.h: SYSTEM must be one of {2400, 2401}"
-#endif
-
-typedef enum {
-	DMA0_ID = 0,
-	N_DMA_ID
-} dma_ID_t;
-
-typedef enum {
-	GDC0_ID = 0,
-	GDC1_ID,
-	N_GDC_ID
-} gdc_ID_t;
-
-/* this extra define is needed because we want to use it also
-   in the preprocessor, and that doesn't work with enums.
- */
-#define N_GDC_ID_CPP 2
-
-typedef enum {
-	VAMEM0_ID = 0,
-	VAMEM1_ID,
-	VAMEM2_ID,
-	N_VAMEM_ID
-} vamem_ID_t;
-
-typedef enum {
-	BAMEM0_ID = 0,
-	N_BAMEM_ID
-} bamem_ID_t;
-
-typedef enum {
-	HMEM0_ID = 0,
-	N_HMEM_ID
-} hmem_ID_t;
-
-typedef enum {
-	ISYS_IRQ0_ID = 0,	/* port a */
-	ISYS_IRQ1_ID,	/* port b */
-	ISYS_IRQ2_ID,	/* port c */
-	N_ISYS_IRQ_ID
-} isys_irq_ID_t;
-
-typedef enum {
-	IRQ0_ID = 0,	/* GP IRQ block */
-	IRQ1_ID,	/* Input formatter */
-	IRQ2_ID,	/* input system */
-	IRQ3_ID,	/* input selector */
-	N_IRQ_ID
-} irq_ID_t;
-
-typedef enum {
-	FIFO_MONITOR0_ID = 0,
-	N_FIFO_MONITOR_ID
-} fifo_monitor_ID_t;
-
-/*
- * Deprecated: Since all gp_reg instances are different
- * and put in the address maps of other devices we cannot
- * enumerate them as that assumes the instrances are the
- * same.
- *
- * We define a single GP_DEVICE containing all gp_regs
- * w.r.t. a single base address
- *
-typedef enum {
-	GP_REGS0_ID = 0,
-	N_GP_REGS_ID
-} gp_regs_ID_t;
- */
-typedef enum {
-	GP_DEVICE0_ID = 0,
-	N_GP_DEVICE_ID
-} gp_device_ID_t;
-
-typedef enum {
-	GP_TIMER0_ID = 0,
-	GP_TIMER1_ID,
-	GP_TIMER2_ID,
-	GP_TIMER3_ID,
-	GP_TIMER4_ID,
-	GP_TIMER5_ID,
-	GP_TIMER6_ID,
-	GP_TIMER7_ID,
-	N_GP_TIMER_ID
-} gp_timer_ID_t;
-
-typedef enum {
-	GPIO0_ID = 0,
-	N_GPIO_ID
-} gpio_ID_t;
-
-typedef enum {
-	TIMED_CTRL0_ID = 0,
-	N_TIMED_CTRL_ID
-} timed_ctrl_ID_t;
-
-typedef enum {
-	INPUT_FORMATTER0_ID = 0,
-	INPUT_FORMATTER1_ID,
-	INPUT_FORMATTER2_ID,
-	INPUT_FORMATTER3_ID,
-	N_INPUT_FORMATTER_ID
-} input_formatter_ID_t;
-
-/* The IF RST is outside the IF */
-#define INPUT_FORMATTER0_SRST_OFFSET	0x0824
-#define INPUT_FORMATTER1_SRST_OFFSET	0x0624
-#define INPUT_FORMATTER2_SRST_OFFSET	0x0424
-#define INPUT_FORMATTER3_SRST_OFFSET	0x0224
-
-#define INPUT_FORMATTER0_SRST_MASK		0x0001
-#define INPUT_FORMATTER1_SRST_MASK		0x0002
-#define INPUT_FORMATTER2_SRST_MASK		0x0004
-#define INPUT_FORMATTER3_SRST_MASK		0x0008
-
-typedef enum {
-	INPUT_SYSTEM0_ID = 0,
-	N_INPUT_SYSTEM_ID
-} input_system_ID_t;
-
-typedef enum {
-	RX0_ID = 0,
-	N_RX_ID
-} rx_ID_t;
-
-enum mipi_port_id {
-	MIPI_PORT0_ID = 0,
-	MIPI_PORT1_ID,
-	MIPI_PORT2_ID,
-	N_MIPI_PORT_ID
-};
-
-#define	N_RX_CHANNEL_ID		4
-
-/* Generic port enumeration with an internal port type ID */
-typedef enum {
-	CSI_PORT0_ID = 0,
-	CSI_PORT1_ID,
-	CSI_PORT2_ID,
-	TPG_PORT0_ID,
-	PRBS_PORT0_ID,
-	FIFO_PORT0_ID,
-	MEMORY_PORT0_ID,
-	N_INPUT_PORT_ID
-} input_port_ID_t;
-
-typedef enum {
-	CAPTURE_UNIT0_ID = 0,
-	CAPTURE_UNIT1_ID,
-	CAPTURE_UNIT2_ID,
-	ACQUISITION_UNIT0_ID,
-	DMA_UNIT0_ID,
-	CTRL_UNIT0_ID,
-	GPREGS_UNIT0_ID,
-	FIFO_UNIT0_ID,
-	IRQ_UNIT0_ID,
-	N_SUB_SYSTEM_ID
-} sub_system_ID_t;
-
-#define	N_CAPTURE_UNIT_ID		3
-#define	N_ACQUISITION_UNIT_ID	1
-#define	N_CTRL_UNIT_ID			1
-
-/*
- * Input-buffer Controller.
- */
-typedef enum {
-	IBUF_CTRL0_ID = 0,	/* map to ISYS2401_IBUF_CNTRL_A */
-	IBUF_CTRL1_ID,		/* map to ISYS2401_IBUF_CNTRL_B */
-	IBUF_CTRL2_ID,		/* map ISYS2401_IBUF_CNTRL_C */
-	N_IBUF_CTRL_ID
-} ibuf_ctrl_ID_t;
-/* end of Input-buffer Controller */
-
-/*
- * Stream2MMIO.
- */
-typedef enum {
-	STREAM2MMIO0_ID = 0,	/* map to ISYS2401_S2M_A */
-	STREAM2MMIO1_ID,	/* map to ISYS2401_S2M_B */
-	STREAM2MMIO2_ID,	/* map to ISYS2401_S2M_C */
-	N_STREAM2MMIO_ID
-} stream2mmio_ID_t;
-
-typedef enum {
-	/*
-	 * Stream2MMIO 0 has 8 SIDs that are indexed by
-	 * [STREAM2MMIO_SID0_ID...STREAM2MMIO_SID7_ID].
-	 *
-	 * Stream2MMIO 1 has 4 SIDs that are indexed by
-	 * [STREAM2MMIO_SID0_ID...TREAM2MMIO_SID3_ID].
-	 *
-	 * Stream2MMIO 2 has 4 SIDs that are indexed by
-	 * [STREAM2MMIO_SID0_ID...STREAM2MMIO_SID3_ID].
-	 */
-	STREAM2MMIO_SID0_ID = 0,
-	STREAM2MMIO_SID1_ID,
-	STREAM2MMIO_SID2_ID,
-	STREAM2MMIO_SID3_ID,
-	STREAM2MMIO_SID4_ID,
-	STREAM2MMIO_SID5_ID,
-	STREAM2MMIO_SID6_ID,
-	STREAM2MMIO_SID7_ID,
-	N_STREAM2MMIO_SID_ID
-} stream2mmio_sid_ID_t;
-/* end of Stream2MMIO */
-
-/**
- * Input System 2401: CSI-MIPI recevier.
- */
-typedef enum {
-	CSI_RX_BACKEND0_ID = 0,	/* map to ISYS2401_MIPI_BE_A */
-	CSI_RX_BACKEND1_ID,		/* map to ISYS2401_MIPI_BE_B */
-	CSI_RX_BACKEND2_ID,		/* map to ISYS2401_MIPI_BE_C */
-	N_CSI_RX_BACKEND_ID
-} csi_rx_backend_ID_t;
-
-typedef enum {
-	CSI_RX_FRONTEND0_ID = 0,	/* map to ISYS2401_CSI_RX_A */
-	CSI_RX_FRONTEND1_ID,		/* map to ISYS2401_CSI_RX_B */
-	CSI_RX_FRONTEND2_ID,		/* map to ISYS2401_CSI_RX_C */
-#define N_CSI_RX_FRONTEND_ID (CSI_RX_FRONTEND2_ID+1)
-} csi_rx_frontend_ID_t;
-
-typedef enum {
-	CSI_RX_DLANE0_ID = 0,		/* map to DLANE0 in CSI RX */
-	CSI_RX_DLANE1_ID,		/* map to DLANE1 in CSI RX */
-	CSI_RX_DLANE2_ID,		/* map to DLANE2 in CSI RX */
-	CSI_RX_DLANE3_ID,		/* map to DLANE3 in CSI RX */
-	N_CSI_RX_DLANE_ID
-} csi_rx_fe_dlane_ID_t;
-/* end of CSI-MIPI receiver */
-
-typedef enum {
-	ISYS2401_DMA0_ID = 0,
-	N_ISYS2401_DMA_ID
-} isys2401_dma_ID_t;
-
-/**
- * Pixel-generator. ("system_global.h")
- */
-typedef enum {
-	PIXELGEN0_ID = 0,
-	PIXELGEN1_ID,
-	PIXELGEN2_ID,
-	N_PIXELGEN_ID
-} pixelgen_ID_t;
-/* end of pixel-generator. ("system_global.h") */
-
-typedef enum {
-	INPUT_SYSTEM_CSI_PORT0_ID = 0,
-	INPUT_SYSTEM_CSI_PORT1_ID,
-	INPUT_SYSTEM_CSI_PORT2_ID,
-
-	INPUT_SYSTEM_PIXELGEN_PORT0_ID,
-	INPUT_SYSTEM_PIXELGEN_PORT1_ID,
-	INPUT_SYSTEM_PIXELGEN_PORT2_ID,
-
-	N_INPUT_SYSTEM_INPUT_PORT_ID
-} input_system_input_port_ID_t;
-
-#define N_INPUT_SYSTEM_CSI_PORT	3
-
-typedef enum {
-	ISYS2401_DMA_CHANNEL_0 = 0,
-	ISYS2401_DMA_CHANNEL_1,
-	ISYS2401_DMA_CHANNEL_2,
-	ISYS2401_DMA_CHANNEL_3,
-	ISYS2401_DMA_CHANNEL_4,
-	ISYS2401_DMA_CHANNEL_5,
-	ISYS2401_DMA_CHANNEL_6,
-	ISYS2401_DMA_CHANNEL_7,
-	ISYS2401_DMA_CHANNEL_8,
-	ISYS2401_DMA_CHANNEL_9,
-	ISYS2401_DMA_CHANNEL_10,
-	ISYS2401_DMA_CHANNEL_11,
-	N_ISYS2401_DMA_CHANNEL
-} isys2401_dma_channel;
-
-enum ia_css_isp_memories {
-	IA_CSS_ISP_PMEM0 = 0,
-	IA_CSS_ISP_DMEM0,
-	IA_CSS_ISP_VMEM0,
-	IA_CSS_ISP_VAMEM0,
-	IA_CSS_ISP_VAMEM1,
-	IA_CSS_ISP_VAMEM2,
-	IA_CSS_ISP_HMEM0,
-	IA_CSS_SP_DMEM0,
-	IA_CSS_DDR,
-	N_IA_CSS_MEMORIES
-};
-#define IA_CSS_NUM_MEMORIES 9
-/* For driver compatability */
-#define N_IA_CSS_ISP_MEMORIES   IA_CSS_NUM_MEMORIES
-#define IA_CSS_NUM_ISP_MEMORIES IA_CSS_NUM_MEMORIES
-
-#endif /* __SYSTEM_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_configs.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_configs.c
deleted file mode 100644
index 325b821f276c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_configs.c
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Generated code: do not edit or commmit. */
-
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_configs.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_iterator(
-	const struct ia_css_binary *binary,
-	const struct ia_css_iterator_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_iterator() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.iterator.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.iterator.offset;
-		}
-		if (size) {
-			ia_css_iterator_config((struct sh_css_isp_iterator_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_iterator() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_copy_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_copy_output_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_copy_output() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.copy_output.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.copy_output.offset;
-		}
-		if (size) {
-			ia_css_copy_output_config((struct sh_css_isp_copy_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_copy_output() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_crop(
-	const struct ia_css_binary *binary,
-	const struct ia_css_crop_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_crop() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.crop.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.crop.offset;
-		}
-		if (size) {
-			ia_css_crop_config((struct sh_css_isp_crop_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_crop() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_fpn(
-	const struct ia_css_binary *binary,
-	const struct ia_css_fpn_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_fpn() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.fpn.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.fpn.offset;
-		}
-		if (size) {
-			ia_css_fpn_config((struct sh_css_isp_fpn_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_fpn() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_dvs(
-	const struct ia_css_binary *binary,
-	const struct ia_css_dvs_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_dvs() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.dvs.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.dvs.offset;
-		}
-		if (size) {
-			ia_css_dvs_config((struct sh_css_isp_dvs_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_dvs() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_qplane(
-	const struct ia_css_binary *binary,
-	const struct ia_css_qplane_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_qplane() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.qplane.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.qplane.offset;
-		}
-		if (size) {
-			ia_css_qplane_config((struct sh_css_isp_qplane_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_qplane() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output0(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output0_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output0() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.output0.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.output0.offset;
-		}
-		if (size) {
-			ia_css_output0_config((struct sh_css_isp_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output0() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output1(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output1_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output1() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.output1.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.output1.offset;
-		}
-		if (size) {
-			ia_css_output1_config((struct sh_css_isp_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output1() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.output.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.output.offset;
-		}
-		if (size) {
-			ia_css_output_config((struct sh_css_isp_output_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_output() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-#ifdef ISP2401
-
-void
-ia_css_configure_sc(
-	const struct ia_css_binary *binary,
-	const struct ia_css_sc_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_sc() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.sc.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.sc.offset;
-		}
-		if (size) {
-			ia_css_sc_config((struct sh_css_isp_sc_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_sc() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-#endif
-
-void
-ia_css_configure_raw(
-	const struct ia_css_binary *binary,
-	const struct ia_css_raw_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_raw() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.raw.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.raw.offset;
-		}
-		if (size) {
-			ia_css_raw_config((struct sh_css_isp_raw_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_raw() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_tnr(
-	const struct ia_css_binary *binary,
-	const struct ia_css_tnr_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_tnr() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.tnr.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.tnr.offset;
-		}
-		if (size) {
-			ia_css_tnr_config((struct sh_css_isp_tnr_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_tnr() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_ref(
-	const struct ia_css_binary *binary,
-	const struct ia_css_ref_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_ref() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.ref.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.ref.offset;
-		}
-		if (size) {
-			ia_css_ref_config((struct sh_css_isp_ref_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_ref() leave:\n");
-}
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_vf(
-	const struct ia_css_binary *binary,
-	const struct ia_css_vf_configuration *config_dmem)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_vf() enter:\n");
-
-	{
-		unsigned offset = 0;
-		unsigned size   = 0;
-		if (binary->info->mem_offsets.offsets.config) {
-			size   = binary->info->mem_offsets.offsets.config->dmem.vf.size;
-			offset = binary->info->mem_offsets.offsets.config->dmem.vf.offset;
-		}
-		if (size) {
-			ia_css_vf_config((struct sh_css_isp_vf_isp_config *)
-					&binary->mem_params.params[IA_CSS_PARAM_CLASS_CONFIG][IA_CSS_ISP_DMEM].address[offset],
-					config_dmem, size);		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_configure_vf() leave:\n");
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_configs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_configs.h
deleted file mode 100644
index 8aacd3dbc05a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_configs.h
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifdef IA_CSS_INCLUDE_CONFIGURATIONS
-#include "isp/kernels/crop/crop_1.0/ia_css_crop.host.h"
-#include "isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h"
-#include "isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h"
-#include "isp/kernels/ob/ob_1.0/ia_css_ob.host.h"
-#include "isp/kernels/output/output_1.0/ia_css_output.host.h"
-#include "isp/kernels/qplane/qplane_2/ia_css_qplane.host.h"
-#include "isp/kernels/raw/raw_1.0/ia_css_raw.host.h"
-#include "isp/kernels/ref/ref_1.0/ia_css_ref.host.h"
-#include "isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h"
-#ifdef ISP2401
-#include "isp/kernels/sc/sc_1.0/ia_css_sc.host.h"
-#endif
-#include "isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h"
-#include "isp/kernels/vf/vf_1.0/ia_css_vf.host.h"
-#include "isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h"
-#include "isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h"
-#endif /* IA_CSS_INCLUDE_CONFIGURATIONS */
-/* Generated code: do not edit or commmit. */
-
-#ifndef _IA_CSS_ISP_CONFIG_H
-#define _IA_CSS_ISP_CONFIG_H
-
-/* Code generated by genparam/gencode.c:gen_param_enum() */
-
-enum ia_css_configuration_ids {
-	IA_CSS_ITERATOR_CONFIG_ID,
-	IA_CSS_COPY_OUTPUT_CONFIG_ID,
-	IA_CSS_CROP_CONFIG_ID,
-	IA_CSS_FPN_CONFIG_ID,
-	IA_CSS_DVS_CONFIG_ID,
-	IA_CSS_QPLANE_CONFIG_ID,
-	IA_CSS_OUTPUT0_CONFIG_ID,
-	IA_CSS_OUTPUT1_CONFIG_ID,
-	IA_CSS_OUTPUT_CONFIG_ID,
-#ifdef ISP2401
-	IA_CSS_SC_CONFIG_ID,
-#endif
-	IA_CSS_RAW_CONFIG_ID,
-	IA_CSS_TNR_CONFIG_ID,
-	IA_CSS_REF_CONFIG_ID,
-	IA_CSS_VF_CONFIG_ID,
-	IA_CSS_NUM_CONFIGURATION_IDS
-};
-
-/* Code generated by genparam/gencode.c:gen_param_offsets() */
-
-struct ia_css_config_memory_offsets {
-	struct {
-		struct ia_css_isp_parameter iterator;
-		struct ia_css_isp_parameter copy_output;
-		struct ia_css_isp_parameter crop;
-		struct ia_css_isp_parameter fpn;
-		struct ia_css_isp_parameter dvs;
-		struct ia_css_isp_parameter qplane;
-		struct ia_css_isp_parameter output0;
-		struct ia_css_isp_parameter output1;
-		struct ia_css_isp_parameter output;
-#ifdef ISP2401
-		struct ia_css_isp_parameter sc;
-#endif
-		struct ia_css_isp_parameter raw;
-		struct ia_css_isp_parameter tnr;
-		struct ia_css_isp_parameter ref;
-		struct ia_css_isp_parameter vf;
-	} dmem;
-};
-
-#if defined(IA_CSS_INCLUDE_CONFIGURATIONS)
-
-#include "ia_css_stream.h"   /* struct ia_css_stream */
-#include "ia_css_binary.h"   /* struct ia_css_binary */
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_iterator(
-	const struct ia_css_binary *binary,
-	const struct ia_css_iterator_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_copy_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_copy_output_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_crop(
-	const struct ia_css_binary *binary,
-	const struct ia_css_crop_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_fpn(
-	const struct ia_css_binary *binary,
-	const struct ia_css_fpn_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_dvs(
-	const struct ia_css_binary *binary,
-	const struct ia_css_dvs_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_qplane(
-	const struct ia_css_binary *binary,
-	const struct ia_css_qplane_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output0(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output0_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output1(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output1_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_output(
-	const struct ia_css_binary *binary,
-	const struct ia_css_output_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-#ifdef ISP2401
-void
-ia_css_configure_sc(
-	const struct ia_css_binary *binary,
-	const struct ia_css_sc_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-#endif
-void
-ia_css_configure_raw(
-	const struct ia_css_binary *binary,
-	const struct ia_css_raw_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_tnr(
-	const struct ia_css_binary *binary,
-	const struct ia_css_tnr_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_ref(
-	const struct ia_css_binary *binary,
-	const struct ia_css_ref_configuration *config_dmem);
-
-/* Code generated by genparam/genconfig.c:gen_configure_function() */
-
-void
-ia_css_configure_vf(
-	const struct ia_css_binary *binary,
-	const struct ia_css_vf_configuration *config_dmem);
-
-#endif /* IA_CSS_INCLUDE_CONFIGURATION */
-
-#endif /* _IA_CSS_ISP_CONFIG_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_params.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_params.c
deleted file mode 100644
index 11e4463ebb50..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_params.c
+++ /dev/null
@@ -1,3220 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#define IA_CSS_INCLUDE_PARAMETERS
-#include "sh_css_params.h"
-#include "isp/kernels/aa/aa_2/ia_css_aa2.host.h"
-#include "isp/kernels/anr/anr_1.0/ia_css_anr.host.h"
-#include "isp/kernels/anr/anr_2/ia_css_anr2.host.h"
-#include "isp/kernels/bh/bh_2/ia_css_bh.host.h"
-#include "isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h"
-#include "isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h"
-#include "isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h"
-#include "isp/kernels/crop/crop_1.0/ia_css_crop.host.h"
-#include "isp/kernels/csc/csc_1.0/ia_css_csc.host.h"
-#include "isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h"
-#include "isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h"
-#include "isp/kernels/ctc/ctc2/ia_css_ctc2.host.h"
-#include "isp/kernels/de/de_1.0/ia_css_de.host.h"
-#include "isp/kernels/de/de_2/ia_css_de2.host.h"
-#include "isp/kernels/dp/dp_1.0/ia_css_dp.host.h"
-#include "isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h"
-#include "isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h"
-#include "isp/kernels/gc/gc_1.0/ia_css_gc.host.h"
-#include "isp/kernels/gc/gc_2/ia_css_gc2.host.h"
-#include "isp/kernels/macc/macc_1.0/ia_css_macc.host.h"
-#include "isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h"
-#include "isp/kernels/ob/ob_1.0/ia_css_ob.host.h"
-#include "isp/kernels/ob/ob2/ia_css_ob2.host.h"
-#include "isp/kernels/output/output_1.0/ia_css_output.host.h"
-#include "isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h"
-#include "isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h"
-#include "isp/kernels/sc/sc_1.0/ia_css_sc.host.h"
-#include "isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h"
-#include "isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h"
-#include "isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h"
-#include "isp/kernels/uds/uds_1.0/ia_css_uds_param.h"
-#include "isp/kernels/wb/wb_1.0/ia_css_wb.host.h"
-#include "isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h"
-#include "isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h"
-#include "isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h"
-#include "isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h"
-#include "isp/kernels/fc/fc_1.0/ia_css_formats.host.h"
-#include "isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h"
-#include "isp/kernels/dpc2/ia_css_dpc2.host.h"
-#include "isp/kernels/eed1_8/ia_css_eed1_8.host.h"
-#include "isp/kernels/bnlm/ia_css_bnlm.host.h"
-#include "isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h"
-/* Generated code: do not edit or commmit. */
-
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_params.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_aa(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.aa.size;
-	unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.aa.offset;
-
-	if (size) {
-		struct sh_css_isp_aa_params *t =  (struct sh_css_isp_aa_params *)
-			&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-		t->strength = params->aa_config.strength;
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_anr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.anr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.anr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr() enter:\n");
-
-			ia_css_anr_encode((struct sh_css_isp_anr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->anr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_anr2(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.anr2.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.anr2.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr2() enter:\n");
-
-			ia_css_anr2_vmem_encode((struct ia_css_isp_anr2_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->anr_thres,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_anr2() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_bh(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.bh.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.bh.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() enter:\n");
-
-			ia_css_bh_encode((struct sh_css_isp_bh_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->s3a_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->hmem0.bh.size;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() enter:\n");
-
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_HMEM0] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bh() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_cnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.cnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.cnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_cnr() enter:\n");
-
-			ia_css_cnr_encode((struct sh_css_isp_cnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->cnr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_cnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_crop(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.crop.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.crop.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_crop() enter:\n");
-
-			ia_css_crop_encode((struct sh_css_isp_crop_isp_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->crop_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_crop() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_csc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.csc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.csc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_csc() enter:\n");
-
-			ia_css_csc_encode((struct sh_css_isp_csc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->cc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_csc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_dp(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.dp.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.dp.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_dp() enter:\n");
-
-			ia_css_dp_encode((struct sh_css_isp_dp_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dp_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_dp() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_bnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.bnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.bnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bnr() enter:\n");
-
-			ia_css_bnr_encode((struct sh_css_isp_bnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->nr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_de(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.de.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.de.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_de() enter:\n");
-
-			ia_css_de_encode((struct sh_css_isp_de_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->de_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_de() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ecd(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ecd.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ecd.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ecd() enter:\n");
-
-			ia_css_ecd_encode((struct sh_css_isp_ecd_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ecd_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ecd() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_formats(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.formats.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.formats.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_formats() enter:\n");
-
-			ia_css_formats_encode((struct sh_css_isp_formats_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->formats_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_formats() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_fpn(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.fpn.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.fpn.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fpn() enter:\n");
-
-			ia_css_fpn_encode((struct sh_css_isp_fpn_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->fpn_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fpn() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_gc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.gc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.gc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() enter:\n");
-
-			ia_css_gc_encode((struct sh_css_isp_gc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->gc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem1.gc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem1.gc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() enter:\n");
-
-			ia_css_gc_vamem_encode((struct sh_css_isp_gc_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM1].address[offset],
-					&params->gc_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM1] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_gc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ce(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ce.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ce.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ce() enter:\n");
-
-			ia_css_ce_encode((struct sh_css_isp_ce_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ce_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ce() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_yuv2rgb(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.yuv2rgb.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.yuv2rgb.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yuv2rgb() enter:\n");
-
-			ia_css_yuv2rgb_encode((struct sh_css_isp_csc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->yuv2rgb_cc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yuv2rgb() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_rgb2yuv(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.rgb2yuv.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.rgb2yuv.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_rgb2yuv() enter:\n");
-
-			ia_css_rgb2yuv_encode((struct sh_css_isp_csc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->rgb2yuv_cc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_rgb2yuv() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_r_gamma(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem0.r_gamma.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem0.r_gamma.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_r_gamma() enter:\n");
-
-			ia_css_r_gamma_vamem_encode((struct sh_css_isp_rgb_gamma_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM0].address[offset],
-					&params->r_gamma_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM0] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_r_gamma() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_g_gamma(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem1.g_gamma.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem1.g_gamma.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_g_gamma() enter:\n");
-
-			ia_css_g_gamma_vamem_encode((struct sh_css_isp_rgb_gamma_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM1].address[offset],
-					&params->g_gamma_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM1] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_g_gamma() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_b_gamma(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem2.b_gamma.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem2.b_gamma.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_b_gamma() enter:\n");
-
-			ia_css_b_gamma_vamem_encode((struct sh_css_isp_rgb_gamma_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM2].address[offset],
-					&params->b_gamma_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM2] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_b_gamma() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_uds(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.uds.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.uds.offset;
-
-		if (size) {
-			struct sh_css_sp_uds_params *p;
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_uds() enter:\n");
-
-			p = (struct sh_css_sp_uds_params *)
-				&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-			p->crop_pos = params->uds_config.crop_pos;
-			p->uds = params->uds_config.uds;
-
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_uds() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_raa(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.raa.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.raa.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_raa() enter:\n");
-
-			ia_css_raa_encode((struct sh_css_isp_aa_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->raa_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_raa() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_s3a(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.s3a.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.s3a.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_s3a() enter:\n");
-
-			ia_css_s3a_encode((struct sh_css_isp_s3a_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->s3a_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_s3a() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ob(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ob.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ob.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() enter:\n");
-
-			ia_css_ob_encode((struct sh_css_isp_ob_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ob_config,
-&params->stream_configs.ob, size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.ob.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.ob.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() enter:\n");
-
-			ia_css_ob_vmem_encode((struct sh_css_isp_ob_vmem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->ob_config,
-&params->stream_configs.ob, size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ob() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_output(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.output.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.output.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_output() enter:\n");
-
-			ia_css_output_encode((struct sh_css_isp_output_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->output_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_output() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sc() enter:\n");
-
-			ia_css_sc_encode((struct sh_css_isp_sc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->sc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_bds(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.bds.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.bds.offset;
-
-		if (size) {
-			struct sh_css_isp_bds_params *p;
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bds() enter:\n");
-
-			p = (struct sh_css_isp_bds_params *)
-				&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-			p->baf_strength = params->bds_config.strength;
-
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_bds() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_tnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.tnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.tnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_tnr() enter:\n");
-
-			ia_css_tnr_encode((struct sh_css_isp_tnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->tnr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_tnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_macc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.macc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.macc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_macc() enter:\n");
-
-			ia_css_macc_encode((struct sh_css_isp_macc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->macc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_macc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_horicoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_horicoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_horicoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horicoef() enter:\n");
-
-			ia_css_sdis_horicoef_vmem_encode((struct sh_css_isp_sdis_hori_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horicoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_vertcoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_vertcoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis_vertcoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertcoef() enter:\n");
-
-			ia_css_sdis_vertcoef_vmem_encode((struct sh_css_isp_sdis_vert_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertcoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_horiproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_horiproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_horiproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horiproj() enter:\n");
-
-			ia_css_sdis_horiproj_encode((struct sh_css_isp_sdis_hori_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_horiproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis_vertproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_vertproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis_vertproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertproj() enter:\n");
-
-			ia_css_sdis_vertproj_encode((struct sh_css_isp_sdis_vert_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis_vertproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_horicoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_horicoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_horicoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horicoef() enter:\n");
-
-			ia_css_sdis2_horicoef_vmem_encode((struct sh_css_isp_sdis_hori_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horicoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_vertcoef(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_vertcoef.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.sdis2_vertcoef.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertcoef() enter:\n");
-
-			ia_css_sdis2_vertcoef_vmem_encode((struct sh_css_isp_sdis_vert_coef_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertcoef() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_horiproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_horiproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_horiproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horiproj() enter:\n");
-
-			ia_css_sdis2_horiproj_encode((struct sh_css_isp_sdis_hori_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_horiproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_sdis2_vertproj(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_vertproj.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.sdis2_vertproj.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertproj() enter:\n");
-
-			ia_css_sdis2_vertproj_encode((struct sh_css_isp_sdis_vert_proj_tbl *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->dvs2_coefs,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_sdis2_vertproj() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_wb(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.wb.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.wb.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_wb() enter:\n");
-
-			ia_css_wb_encode((struct sh_css_isp_wb_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->wb_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_wb() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_nr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.nr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.nr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_nr() enter:\n");
-
-			ia_css_nr_encode((struct sh_css_isp_ynr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->nr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_nr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_yee(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.yee.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.yee.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yee() enter:\n");
-
-			ia_css_yee_encode((struct sh_css_isp_yee_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->yee_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_yee() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ynr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ynr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ynr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ynr() enter:\n");
-
-			ia_css_ynr_encode((struct sh_css_isp_yee2_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ynr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ynr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_fc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.fc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.fc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fc() enter:\n");
-
-			ia_css_fc_encode((struct sh_css_isp_fc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->fc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_fc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_ctc(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.ctc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.ctc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() enter:\n");
-
-			ia_css_ctc_encode((struct sh_css_isp_ctc_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->ctc_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() leave:\n");
-		}
-
-	}
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem0.ctc.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem0.ctc.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() enter:\n");
-
-			ia_css_ctc_vamem_encode((struct sh_css_isp_ctc_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM0].address[offset],
-					&params->ctc_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM0] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_ctc() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_xnr_table(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vamem1.xnr_table.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vamem1.xnr_table.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr_table() enter:\n");
-
-			ia_css_xnr_table_vamem_encode((struct sh_css_isp_xnr_vamem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VAMEM1].address[offset],
-					&params->xnr_table,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VAMEM1] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr_table() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_xnr(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.xnr.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.xnr.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr() enter:\n");
-
-			ia_css_xnr_encode((struct sh_css_isp_xnr_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->xnr_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr() leave:\n");
-		}
-
-	}
-}
-
-/* Code generated by genparam/gencode.c:gen_process_function() */
-
-static void
-ia_css_process_xnr3(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.xnr3.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.xnr3.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() enter:\n");
-
-			ia_css_xnr3_encode((struct sh_css_isp_xnr3_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-					&params->xnr3_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() leave:\n");
-		}
-
-	}
-#ifdef ISP2401
-	{
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->vmem.xnr3.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->vmem.xnr3.offset;
-
-		if (size) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() enter:\n");
-
-			ia_css_xnr3_vmem_encode((struct sh_css_isp_xnr3_vmem_params *)
-					&stage->binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_VMEM].address[offset],
-					&params->xnr3_config,
-size);
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_VMEM] = true;
-
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_process_xnr3() leave:\n");
-		}
-
-	}
-#endif
-}
-
-/* Code generated by genparam/gencode.c:gen_param_process_table() */
-
-void (* ia_css_kernel_process_param[IA_CSS_NUM_PARAMETER_IDS])(
-			unsigned pipe_id,
-			const struct ia_css_pipeline_stage *stage,
-			struct ia_css_isp_parameters *params) = {
-	ia_css_process_aa,
-	ia_css_process_anr,
-	ia_css_process_anr2,
-	ia_css_process_bh,
-	ia_css_process_cnr,
-	ia_css_process_crop,
-	ia_css_process_csc,
-	ia_css_process_dp,
-	ia_css_process_bnr,
-	ia_css_process_de,
-	ia_css_process_ecd,
-	ia_css_process_formats,
-	ia_css_process_fpn,
-	ia_css_process_gc,
-	ia_css_process_ce,
-	ia_css_process_yuv2rgb,
-	ia_css_process_rgb2yuv,
-	ia_css_process_r_gamma,
-	ia_css_process_g_gamma,
-	ia_css_process_b_gamma,
-	ia_css_process_uds,
-	ia_css_process_raa,
-	ia_css_process_s3a,
-	ia_css_process_ob,
-	ia_css_process_output,
-	ia_css_process_sc,
-	ia_css_process_bds,
-	ia_css_process_tnr,
-	ia_css_process_macc,
-	ia_css_process_sdis_horicoef,
-	ia_css_process_sdis_vertcoef,
-	ia_css_process_sdis_horiproj,
-	ia_css_process_sdis_vertproj,
-	ia_css_process_sdis2_horicoef,
-	ia_css_process_sdis2_vertcoef,
-	ia_css_process_sdis2_horiproj,
-	ia_css_process_sdis2_vertproj,
-	ia_css_process_wb,
-	ia_css_process_nr,
-	ia_css_process_yee,
-	ia_css_process_ynr,
-	ia_css_process_fc,
-	ia_css_process_ctc,
-	ia_css_process_xnr_table,
-	ia_css_process_xnr,
-	ia_css_process_xnr3,
-};
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_dp_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dp_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_dp_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dp_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_dp_config() leave\n");
-	ia_css_dp_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_dp_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dp_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_dp_config() enter:\n");
-	ia_css_dp_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dp_config = *config;
-	params->config_changed[IA_CSS_DP_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_DP_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_dp_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_wb_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_wb_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_wb_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->wb_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_wb_config() leave\n");
-	ia_css_wb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_wb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_wb_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_wb_config() enter:\n");
-	ia_css_wb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->wb_config = *config;
-	params->config_changed[IA_CSS_WB_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_WB_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_wb_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_tnr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_tnr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_tnr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->tnr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_tnr_config() leave\n");
-	ia_css_tnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_tnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_tnr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_tnr_config() enter:\n");
-	ia_css_tnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->tnr_config = *config;
-	params->config_changed[IA_CSS_TNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_TNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_tnr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ob_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ob_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ob_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ob_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ob_config() leave\n");
-	ia_css_ob_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ob_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ob_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ob_config() enter:\n");
-	ia_css_ob_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ob_config = *config;
-	params->config_changed[IA_CSS_OB_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_OB_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ob_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_de_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_de_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_de_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->de_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_de_config() leave\n");
-	ia_css_de_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_de_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_de_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_de_config() enter:\n");
-	ia_css_de_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->de_config = *config;
-	params->config_changed[IA_CSS_DE_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_DE_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_de_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_anr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_anr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->anr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr_config() leave\n");
-	ia_css_anr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_anr_config() enter:\n");
-	ia_css_anr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->anr_config = *config;
-	params->config_changed[IA_CSS_ANR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_ANR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_anr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_anr2_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_anr_thres *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr2_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->anr_thres;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_anr2_config() leave\n");
-	ia_css_anr2_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr2_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_thres *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_anr2_config() enter:\n");
-	ia_css_anr2_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->anr_thres = *config;
-	params->config_changed[IA_CSS_ANR2_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_ANR2_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_anr2_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ce_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ce_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ce_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ce_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ce_config() leave\n");
-	ia_css_ce_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ce_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ce_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ce_config() enter:\n");
-	ia_css_ce_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ce_config = *config;
-	params->config_changed[IA_CSS_CE_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CE_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ce_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ecd_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ecd_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ecd_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ecd_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ecd_config() leave\n");
-	ia_css_ecd_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ecd_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ecd_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ecd_config() enter:\n");
-	ia_css_ecd_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ecd_config = *config;
-	params->config_changed[IA_CSS_ECD_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_ECD_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ecd_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ynr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ynr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ynr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ynr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ynr_config() leave\n");
-	ia_css_ynr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ynr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ynr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ynr_config() enter:\n");
-	ia_css_ynr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ynr_config = *config;
-	params->config_changed[IA_CSS_YNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_YNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ynr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_fc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_fc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_fc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->fc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_fc_config() leave\n");
-	ia_css_fc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_fc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_fc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_fc_config() enter:\n");
-	ia_css_fc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->fc_config = *config;
-	params->config_changed[IA_CSS_FC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_FC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_fc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_cnr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cnr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_cnr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->cnr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_cnr_config() leave\n");
-	ia_css_cnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_cnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cnr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_cnr_config() enter:\n");
-	ia_css_cnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->cnr_config = *config;
-	params->config_changed[IA_CSS_CNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_cnr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_macc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_macc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_macc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->macc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_macc_config() leave\n");
-	ia_css_macc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_macc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_macc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_macc_config() enter:\n");
-	ia_css_macc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->macc_config = *config;
-	params->config_changed[IA_CSS_MACC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_MACC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_macc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_ctc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ctc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ctc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->ctc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_ctc_config() leave\n");
-	ia_css_ctc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ctc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ctc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_ctc_config() enter:\n");
-	ia_css_ctc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->ctc_config = *config;
-	params->config_changed[IA_CSS_CTC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CTC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_ctc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_aa_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_aa_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_aa_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->aa_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_aa_config() leave\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_aa_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_aa_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_aa_config() enter:\n");
-	params->aa_config = *config;
-	params->config_changed[IA_CSS_AA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_AA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_aa_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_yuv2rgb_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_yuv2rgb_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->yuv2rgb_cc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_yuv2rgb_config() leave\n");
-	ia_css_yuv2rgb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_yuv2rgb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_yuv2rgb_config() enter:\n");
-	ia_css_yuv2rgb_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->yuv2rgb_cc_config = *config;
-	params->config_changed[IA_CSS_YUV2RGB_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_YUV2RGB_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_yuv2rgb_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_rgb2yuv_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_rgb2yuv_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->rgb2yuv_cc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_rgb2yuv_config() leave\n");
-	ia_css_rgb2yuv_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_rgb2yuv_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_rgb2yuv_config() enter:\n");
-	ia_css_rgb2yuv_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->rgb2yuv_cc_config = *config;
-	params->config_changed[IA_CSS_RGB2YUV_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_RGB2YUV_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_rgb2yuv_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_csc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_cc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_csc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->cc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_csc_config() leave\n");
-	ia_css_csc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_csc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_csc_config() enter:\n");
-	ia_css_csc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->cc_config = *config;
-	params->config_changed[IA_CSS_CSC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_CSC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_csc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_nr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_nr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_nr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->nr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_nr_config() leave\n");
-	ia_css_nr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_nr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_nr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_nr_config() enter:\n");
-	ia_css_nr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->nr_config = *config;
-	params->config_changed[IA_CSS_BNR_ID] = true;
-	params->config_changed[IA_CSS_NR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_NR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_nr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_gc_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_gc_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_gc_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->gc_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_gc_config() leave\n");
-	ia_css_gc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_gc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_gc_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_gc_config() enter:\n");
-	ia_css_gc_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->gc_config = *config;
-	params->config_changed[IA_CSS_GC_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_GC_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_gc_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_horicoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horicoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horicoef_config() leave\n");
-	ia_css_sdis_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_horicoef_config() enter:\n");
-	ia_css_sdis_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_horicoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_vertcoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertcoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertcoef_config() leave\n");
-	ia_css_sdis_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_vertcoef_config() enter:\n");
-	ia_css_sdis_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_vertcoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_horiproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horiproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_horiproj_config() leave\n");
-	ia_css_sdis_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_horiproj_config() enter:\n");
-	ia_css_sdis_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_horiproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis_vertproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis_vertproj_config() leave\n");
-	ia_css_sdis_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis_vertproj_config() enter:\n");
-	ia_css_sdis_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs_coefs = *config;
-	params->config_changed[IA_CSS_SDIS_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS_VERTPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis_vertproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_horicoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horicoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horicoef_config() leave\n");
-	ia_css_sdis2_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_horicoef_config() enter:\n");
-	ia_css_sdis2_horicoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_horicoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_vertcoef_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertcoef_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertcoef_config() leave\n");
-	ia_css_sdis2_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_vertcoef_config() enter:\n");
-	ia_css_sdis2_vertcoef_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_vertcoef_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_horiproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horiproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_horiproj_config() leave\n");
-	ia_css_sdis2_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_horiproj_config() enter:\n");
-	ia_css_sdis2_horiproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_horiproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_sdis2_vertproj_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dvs2_coefficients *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertproj_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->dvs2_coefs;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_sdis2_vertproj_config() leave\n");
-	ia_css_sdis2_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_sdis2_vertproj_config() enter:\n");
-	ia_css_sdis2_vertproj_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->dvs2_coefs = *config;
-	params->config_changed[IA_CSS_SDIS2_HORICOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTCOEF_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_HORIPROJ_ID] = true;
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_SDIS2_VERTPROJ_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_sdis2_vertproj_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_r_gamma_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_rgb_gamma_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_r_gamma_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->r_gamma_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_r_gamma_config() leave\n");
-	ia_css_r_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_r_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_r_gamma_config() enter:\n");
-	ia_css_r_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->r_gamma_table = *config;
-	params->config_changed[IA_CSS_R_GAMMA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_R_GAMMA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_r_gamma_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_g_gamma_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_rgb_gamma_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_g_gamma_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->g_gamma_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_g_gamma_config() leave\n");
-	ia_css_g_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_g_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_g_gamma_config() enter:\n");
-	ia_css_g_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->g_gamma_table = *config;
-	params->config_changed[IA_CSS_G_GAMMA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_G_GAMMA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_g_gamma_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_b_gamma_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_rgb_gamma_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_b_gamma_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->b_gamma_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_b_gamma_config() leave\n");
-	ia_css_b_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_b_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_b_gamma_config() enter:\n");
-	ia_css_b_gamma_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->b_gamma_table = *config;
-	params->config_changed[IA_CSS_B_GAMMA_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_B_GAMMA_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_b_gamma_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_xnr_table_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_xnr_table *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_table_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->xnr_table;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_table_config() leave\n");
-	ia_css_xnr_table_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_table_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_table *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_xnr_table_config() enter:\n");
-	ia_css_xnr_table_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->xnr_table = *config;
-	params->config_changed[IA_CSS_XNR_TABLE_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_XNR_TABLE_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_xnr_table_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_formats_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_formats_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_formats_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->formats_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_formats_config() leave\n");
-	ia_css_formats_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_formats_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_formats_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_formats_config() enter:\n");
-	ia_css_formats_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->formats_config = *config;
-	params->config_changed[IA_CSS_FORMATS_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_FORMATS_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_formats_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_xnr_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_xnr_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->xnr_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr_config() leave\n");
-	ia_css_xnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_xnr_config() enter:\n");
-	ia_css_xnr_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->xnr_config = *config;
-	params->config_changed[IA_CSS_XNR_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_XNR_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_xnr_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_xnr3_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_xnr3_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr3_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->xnr3_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_xnr3_config() leave\n");
-	ia_css_xnr3_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr3_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr3_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_xnr3_config() enter:\n");
-	ia_css_xnr3_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->xnr3_config = *config;
-	params->config_changed[IA_CSS_XNR3_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_XNR3_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_xnr3_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_s3a_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_3a_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_s3a_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->s3a_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_s3a_config() leave\n");
-	ia_css_s3a_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_s3a_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_3a_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_s3a_config() enter:\n");
-	ia_css_s3a_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->s3a_config = *config;
-	params->config_changed[IA_CSS_BH_ID] = true;
-	params->config_changed[IA_CSS_S3A_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_S3A_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_s3a_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_get_function() */
-
-static void
-ia_css_get_output_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_output_config *config){
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_output_config() enter: "
-		"config=%p\n",config);
-
-	*config = params->output_config;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_get_output_config() leave\n");
-	ia_css_output_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-}
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_output_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_output_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_set_output_config() enter:\n");
-	ia_css_output_debug_dtrace(config, IA_CSS_DEBUG_TRACE);
-	params->output_config = *config;
-	params->config_changed[IA_CSS_OUTPUT_ID] = true;
-#ifndef ISP2401
-	params->config_changed[IA_CSS_OUTPUT_ID] = true;
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_set_output_config() leave: "
-		"return_void\n");
-}
-
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_get_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-{
-	ia_css_get_dp_config(params, config->dp_config);
-	ia_css_get_wb_config(params, config->wb_config);
-	ia_css_get_tnr_config(params, config->tnr_config);
-	ia_css_get_ob_config(params, config->ob_config);
-	ia_css_get_de_config(params, config->de_config);
-	ia_css_get_anr_config(params, config->anr_config);
-	ia_css_get_anr2_config(params, config->anr_thres);
-	ia_css_get_ce_config(params, config->ce_config);
-	ia_css_get_ecd_config(params, config->ecd_config);
-	ia_css_get_ynr_config(params, config->ynr_config);
-	ia_css_get_fc_config(params, config->fc_config);
-	ia_css_get_cnr_config(params, config->cnr_config);
-	ia_css_get_macc_config(params, config->macc_config);
-	ia_css_get_ctc_config(params, config->ctc_config);
-	ia_css_get_aa_config(params, config->aa_config);
-	ia_css_get_yuv2rgb_config(params, config->yuv2rgb_cc_config);
-	ia_css_get_rgb2yuv_config(params, config->rgb2yuv_cc_config);
-	ia_css_get_csc_config(params, config->cc_config);
-	ia_css_get_nr_config(params, config->nr_config);
-	ia_css_get_gc_config(params, config->gc_config);
-	ia_css_get_sdis_horicoef_config(params, config->dvs_coefs);
-	ia_css_get_sdis_vertcoef_config(params, config->dvs_coefs);
-	ia_css_get_sdis_horiproj_config(params, config->dvs_coefs);
-	ia_css_get_sdis_vertproj_config(params, config->dvs_coefs);
-	ia_css_get_sdis2_horicoef_config(params, config->dvs2_coefs);
-	ia_css_get_sdis2_vertcoef_config(params, config->dvs2_coefs);
-	ia_css_get_sdis2_horiproj_config(params, config->dvs2_coefs);
-	ia_css_get_sdis2_vertproj_config(params, config->dvs2_coefs);
-	ia_css_get_r_gamma_config(params, config->r_gamma_table);
-	ia_css_get_g_gamma_config(params, config->g_gamma_table);
-	ia_css_get_b_gamma_config(params, config->b_gamma_table);
-	ia_css_get_xnr_table_config(params, config->xnr_table);
-	ia_css_get_formats_config(params, config->formats_config);
-	ia_css_get_xnr_config(params, config->xnr_config);
-	ia_css_get_xnr3_config(params, config->xnr3_config);
-	ia_css_get_s3a_config(params, config->s3a_config);
-	ia_css_get_output_config(params, config->output_config);
-}
-
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_set_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-{
-	ia_css_set_dp_config(params, config->dp_config);
-	ia_css_set_wb_config(params, config->wb_config);
-	ia_css_set_tnr_config(params, config->tnr_config);
-	ia_css_set_ob_config(params, config->ob_config);
-	ia_css_set_de_config(params, config->de_config);
-	ia_css_set_anr_config(params, config->anr_config);
-	ia_css_set_anr2_config(params, config->anr_thres);
-	ia_css_set_ce_config(params, config->ce_config);
-	ia_css_set_ecd_config(params, config->ecd_config);
-	ia_css_set_ynr_config(params, config->ynr_config);
-	ia_css_set_fc_config(params, config->fc_config);
-	ia_css_set_cnr_config(params, config->cnr_config);
-	ia_css_set_macc_config(params, config->macc_config);
-	ia_css_set_ctc_config(params, config->ctc_config);
-	ia_css_set_aa_config(params, config->aa_config);
-	ia_css_set_yuv2rgb_config(params, config->yuv2rgb_cc_config);
-	ia_css_set_rgb2yuv_config(params, config->rgb2yuv_cc_config);
-	ia_css_set_csc_config(params, config->cc_config);
-	ia_css_set_nr_config(params, config->nr_config);
-	ia_css_set_gc_config(params, config->gc_config);
-	ia_css_set_sdis_horicoef_config(params, config->dvs_coefs);
-	ia_css_set_sdis_vertcoef_config(params, config->dvs_coefs);
-	ia_css_set_sdis_horiproj_config(params, config->dvs_coefs);
-	ia_css_set_sdis_vertproj_config(params, config->dvs_coefs);
-	ia_css_set_sdis2_horicoef_config(params, config->dvs2_coefs);
-	ia_css_set_sdis2_vertcoef_config(params, config->dvs2_coefs);
-	ia_css_set_sdis2_horiproj_config(params, config->dvs2_coefs);
-	ia_css_set_sdis2_vertproj_config(params, config->dvs2_coefs);
-	ia_css_set_r_gamma_config(params, config->r_gamma_table);
-	ia_css_set_g_gamma_config(params, config->g_gamma_table);
-	ia_css_set_b_gamma_config(params, config->b_gamma_table);
-	ia_css_set_xnr_table_config(params, config->xnr_table);
-	ia_css_set_formats_config(params, config->formats_config);
-	ia_css_set_xnr_config(params, config->xnr_config);
-	ia_css_set_xnr3_config(params, config->xnr3_config);
-	ia_css_set_s3a_config(params, config->s3a_config);
-	ia_css_set_output_config(params, config->output_config);
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_params.h
deleted file mode 100644
index 5b3deb7f74ae..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_params.h
+++ /dev/null
@@ -1,399 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Generated code: do not edit or commmit. */
-
-#ifndef _IA_CSS_ISP_PARAM_H
-#define _IA_CSS_ISP_PARAM_H
-
-/* Code generated by genparam/gencode.c:gen_param_enum() */
-
-enum ia_css_parameter_ids {
-	IA_CSS_AA_ID,
-	IA_CSS_ANR_ID,
-	IA_CSS_ANR2_ID,
-	IA_CSS_BH_ID,
-	IA_CSS_CNR_ID,
-	IA_CSS_CROP_ID,
-	IA_CSS_CSC_ID,
-	IA_CSS_DP_ID,
-	IA_CSS_BNR_ID,
-	IA_CSS_DE_ID,
-	IA_CSS_ECD_ID,
-	IA_CSS_FORMATS_ID,
-	IA_CSS_FPN_ID,
-	IA_CSS_GC_ID,
-	IA_CSS_CE_ID,
-	IA_CSS_YUV2RGB_ID,
-	IA_CSS_RGB2YUV_ID,
-	IA_CSS_R_GAMMA_ID,
-	IA_CSS_G_GAMMA_ID,
-	IA_CSS_B_GAMMA_ID,
-	IA_CSS_UDS_ID,
-	IA_CSS_RAA_ID,
-	IA_CSS_S3A_ID,
-	IA_CSS_OB_ID,
-	IA_CSS_OUTPUT_ID,
-	IA_CSS_SC_ID,
-	IA_CSS_BDS_ID,
-	IA_CSS_TNR_ID,
-	IA_CSS_MACC_ID,
-	IA_CSS_SDIS_HORICOEF_ID,
-	IA_CSS_SDIS_VERTCOEF_ID,
-	IA_CSS_SDIS_HORIPROJ_ID,
-	IA_CSS_SDIS_VERTPROJ_ID,
-	IA_CSS_SDIS2_HORICOEF_ID,
-	IA_CSS_SDIS2_VERTCOEF_ID,
-	IA_CSS_SDIS2_HORIPROJ_ID,
-	IA_CSS_SDIS2_VERTPROJ_ID,
-	IA_CSS_WB_ID,
-	IA_CSS_NR_ID,
-	IA_CSS_YEE_ID,
-	IA_CSS_YNR_ID,
-	IA_CSS_FC_ID,
-	IA_CSS_CTC_ID,
-	IA_CSS_XNR_TABLE_ID,
-	IA_CSS_XNR_ID,
-	IA_CSS_XNR3_ID,
-	IA_CSS_NUM_PARAMETER_IDS
-};
-
-/* Code generated by genparam/gencode.c:gen_param_offsets() */
-
-struct ia_css_memory_offsets {
-	struct {
-		struct ia_css_isp_parameter aa;
-		struct ia_css_isp_parameter anr;
-		struct ia_css_isp_parameter bh;
-		struct ia_css_isp_parameter cnr;
-		struct ia_css_isp_parameter crop;
-		struct ia_css_isp_parameter csc;
-		struct ia_css_isp_parameter dp;
-		struct ia_css_isp_parameter bnr;
-		struct ia_css_isp_parameter de;
-		struct ia_css_isp_parameter ecd;
-		struct ia_css_isp_parameter formats;
-		struct ia_css_isp_parameter fpn;
-		struct ia_css_isp_parameter gc;
-		struct ia_css_isp_parameter ce;
-		struct ia_css_isp_parameter yuv2rgb;
-		struct ia_css_isp_parameter rgb2yuv;
-		struct ia_css_isp_parameter uds;
-		struct ia_css_isp_parameter raa;
-		struct ia_css_isp_parameter s3a;
-		struct ia_css_isp_parameter ob;
-		struct ia_css_isp_parameter output;
-		struct ia_css_isp_parameter sc;
-		struct ia_css_isp_parameter bds;
-		struct ia_css_isp_parameter tnr;
-		struct ia_css_isp_parameter macc;
-		struct ia_css_isp_parameter sdis_horiproj;
-		struct ia_css_isp_parameter sdis_vertproj;
-		struct ia_css_isp_parameter sdis2_horiproj;
-		struct ia_css_isp_parameter sdis2_vertproj;
-		struct ia_css_isp_parameter wb;
-		struct ia_css_isp_parameter nr;
-		struct ia_css_isp_parameter yee;
-		struct ia_css_isp_parameter ynr;
-		struct ia_css_isp_parameter fc;
-		struct ia_css_isp_parameter ctc;
-		struct ia_css_isp_parameter xnr;
-		struct ia_css_isp_parameter xnr3;
-		struct ia_css_isp_parameter get;
-		struct ia_css_isp_parameter put;
-	} dmem;
-	struct {
-		struct ia_css_isp_parameter anr2;
-		struct ia_css_isp_parameter ob;
-		struct ia_css_isp_parameter sdis_horicoef;
-		struct ia_css_isp_parameter sdis_vertcoef;
-		struct ia_css_isp_parameter sdis2_horicoef;
-		struct ia_css_isp_parameter sdis2_vertcoef;
-#ifdef ISP2401
-		struct ia_css_isp_parameter xnr3;
-#endif
-	} vmem;
-	struct {
-		struct ia_css_isp_parameter bh;
-	} hmem0;
-	struct {
-		struct ia_css_isp_parameter gc;
-		struct ia_css_isp_parameter g_gamma;
-		struct ia_css_isp_parameter xnr_table;
-	} vamem1;
-	struct {
-		struct ia_css_isp_parameter r_gamma;
-		struct ia_css_isp_parameter ctc;
-	} vamem0;
-	struct {
-		struct ia_css_isp_parameter b_gamma;
-	} vamem2;
-};
-
-#if defined(IA_CSS_INCLUDE_PARAMETERS)
-
-#include "ia_css_stream.h"   /* struct ia_css_stream */
-#include "ia_css_binary.h"   /* struct ia_css_binary */
-/* Code generated by genparam/gencode.c:gen_param_process_table() */
-
-struct ia_css_pipeline_stage; /* forward declaration */
-
-extern void (* ia_css_kernel_process_param[IA_CSS_NUM_PARAMETER_IDS])(
-			unsigned pipe_id,
-			const struct ia_css_pipeline_stage *stage,
-			struct ia_css_isp_parameters *params);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_dp_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dp_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_wb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_wb_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_tnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_tnr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ob_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ob_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_de_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_de_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_anr2_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_anr_thres *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ce_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ce_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ecd_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ecd_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ynr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ynr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_fc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_fc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_cnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cnr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_macc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_macc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_ctc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ctc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_aa_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_aa_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_yuv2rgb_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_rgb2yuv_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_csc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_cc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_nr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_nr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_gc_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_gc_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horicoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertcoef_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_horiproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_sdis2_vertproj_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dvs2_coefficients *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_r_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_g_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_b_gamma_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_rgb_gamma_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_table_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_table *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_formats_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_formats_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_xnr3_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_xnr3_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_s3a_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_3a_config *config);
-
-/* Code generated by genparam/gencode.c:gen_set_function() */
-
-void
-ia_css_set_output_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_output_config *config);
-
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_get_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-;
-#ifdef ISP2401
-
-#endif
-/* Code generated by genparam/gencode.c:gen_global_access_function() */
-
-void
-ia_css_set_configs(struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config)
-;
-#ifdef ISP2401
-
-#endif
-#endif /* IA_CSS_INCLUDE_PARAMETER */
-
-#endif /* _IA_CSS_ISP_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_states.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_states.c
deleted file mode 100644
index e87d05bc73ae..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_states.c
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Generated code: do not edit or commmit. */
-
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_states.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_aa_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_aa_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.aa.size;
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.aa.offset;
-
-		if (size)
-			memset(&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset], 0, size);
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_aa_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_cnr_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.cnr.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.cnr.offset;
-
-		if (size) {
-			ia_css_init_cnr_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_cnr2_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr2_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.cnr2.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.cnr2.offset;
-
-		if (size) {
-			ia_css_init_cnr2_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_cnr2_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_dp_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_dp_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.dp.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.dp.offset;
-
-		if (size) {
-			ia_css_init_dp_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_dp_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_de_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_de_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.de.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.de.offset;
-
-		if (size) {
-			ia_css_init_de_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_de_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_tnr_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_tnr_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->dmem.tnr.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->dmem.tnr.offset;
-
-		if (size) {
-			ia_css_init_tnr_state((struct sh_css_isp_tnr_dmem_state *)
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_DMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_tnr_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_ref_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ref_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->dmem.ref.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->dmem.ref.offset;
-
-		if (size) {
-			ia_css_init_ref_state((struct sh_css_isp_ref_dmem_state *)
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_DMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ref_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_init_function() */
-
-static void
-ia_css_initialize_ynr_state(
-	const struct ia_css_binary *binary)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ynr_state() enter:\n");
-
-	{
-		unsigned size   = binary->info->mem_offsets.offsets.state->vmem.ynr.size;
-
-		unsigned offset = binary->info->mem_offsets.offsets.state->vmem.ynr.offset;
-
-		if (size) {
-			ia_css_init_ynr_state(
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_STATE][IA_CSS_ISP_VMEM].address[offset],
-				size);
-		}
-
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_initialize_ynr_state() leave:\n");
-}
-
-/* Code generated by genparam/genstate.c:gen_state_init_table() */
-
-void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(const struct ia_css_binary *binary) = {
-	ia_css_initialize_aa_state,
-	ia_css_initialize_cnr_state,
-	ia_css_initialize_cnr2_state,
-	ia_css_initialize_dp_state,
-	ia_css_initialize_de_state,
-	ia_css_initialize_tnr_state,
-	ia_css_initialize_ref_state,
-	ia_css_initialize_ynr_state,
-};
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_states.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_states.h
deleted file mode 100644
index 732adafb0a63..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hive_isp_css_2401_system_generated/ia_css_isp_states.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#define IA_CSS_INCLUDE_STATES
-#include "isp/kernels/aa/aa_2/ia_css_aa2.host.h"
-#include "isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h"
-#include "isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h"
-#include "isp/kernels/de/de_1.0/ia_css_de.host.h"
-#include "isp/kernels/dp/dp_1.0/ia_css_dp.host.h"
-#include "isp/kernels/ref/ref_1.0/ia_css_ref.host.h"
-#include "isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h"
-#include "isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h"
-#include "isp/kernels/dpc2/ia_css_dpc2.host.h"
-#include "isp/kernels/eed1_8/ia_css_eed1_8.host.h"
-/* Generated code: do not edit or commmit. */
-
-#ifndef _IA_CSS_ISP_STATE_H
-#define _IA_CSS_ISP_STATE_H
-
-/* Code generated by genparam/gencode.c:gen_param_enum() */
-
-enum ia_css_state_ids {
-	IA_CSS_AA_STATE_ID,
-	IA_CSS_CNR_STATE_ID,
-	IA_CSS_CNR2_STATE_ID,
-	IA_CSS_DP_STATE_ID,
-	IA_CSS_DE_STATE_ID,
-	IA_CSS_TNR_STATE_ID,
-	IA_CSS_REF_STATE_ID,
-	IA_CSS_YNR_STATE_ID,
-	IA_CSS_NUM_STATE_IDS
-};
-
-/* Code generated by genparam/gencode.c:gen_param_offsets() */
-
-struct ia_css_state_memory_offsets {
-	struct {
-		struct ia_css_isp_parameter aa;
-		struct ia_css_isp_parameter cnr;
-		struct ia_css_isp_parameter cnr2;
-		struct ia_css_isp_parameter dp;
-		struct ia_css_isp_parameter de;
-		struct ia_css_isp_parameter ynr;
-	} vmem;
-	struct {
-		struct ia_css_isp_parameter tnr;
-		struct ia_css_isp_parameter ref;
-	} dmem;
-};
-
-#if defined(IA_CSS_INCLUDE_STATES)
-
-#include "ia_css_stream.h"   /* struct ia_css_stream */
-#include "ia_css_binary.h"   /* struct ia_css_binary */
-/* Code generated by genparam/genstate.c:gen_state_init_table() */
-
-extern void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(const struct ia_css_binary *binary);
-
-#endif /* IA_CSS_INCLUDE_STATE */
-
-#endif /* _IA_CSS_ISP_STATE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/bits.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/bits.h
deleted file mode 100644
index e71e33d9d143..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/bits.h
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_BITS_H
-#define _HRT_BITS_H
-
-#include "defs.h"
-
-#define _hrt_ones(n) HRTCAT(_hrt_ones_, n)
-#define _hrt_ones_0x0  0x00000000U
-#define _hrt_ones_0x1  0x00000001U
-#define _hrt_ones_0x2  0x00000003U
-#define _hrt_ones_0x3  0x00000007U
-#define _hrt_ones_0x4  0x0000000FU
-#define _hrt_ones_0x5  0x0000001FU
-#define _hrt_ones_0x6  0x0000003FU
-#define _hrt_ones_0x7  0x0000007FU
-#define _hrt_ones_0x8  0x000000FFU
-#define _hrt_ones_0x9  0x000001FFU
-#define _hrt_ones_0xA  0x000003FFU
-#define _hrt_ones_0xB  0x000007FFU
-#define _hrt_ones_0xC  0x00000FFFU
-#define _hrt_ones_0xD  0x00001FFFU
-#define _hrt_ones_0xE  0x00003FFFU
-#define _hrt_ones_0xF  0x00007FFFU
-#define _hrt_ones_0x10 0x0000FFFFU
-#define _hrt_ones_0x11 0x0001FFFFU
-#define _hrt_ones_0x12 0x0003FFFFU
-#define _hrt_ones_0x13 0x0007FFFFU
-#define _hrt_ones_0x14 0x000FFFFFU
-#define _hrt_ones_0x15 0x001FFFFFU
-#define _hrt_ones_0x16 0x003FFFFFU
-#define _hrt_ones_0x17 0x007FFFFFU
-#define _hrt_ones_0x18 0x00FFFFFFU
-#define _hrt_ones_0x19 0x01FFFFFFU
-#define _hrt_ones_0x1A 0x03FFFFFFU
-#define _hrt_ones_0x1B 0x07FFFFFFU
-#define _hrt_ones_0x1C 0x0FFFFFFFU
-#define _hrt_ones_0x1D 0x1FFFFFFFU
-#define _hrt_ones_0x1E 0x3FFFFFFFU
-#define _hrt_ones_0x1F 0x7FFFFFFFU
-#define _hrt_ones_0x20 0xFFFFFFFFU
-
-#define _hrt_ones_0  _hrt_ones_0x0
-#define _hrt_ones_1  _hrt_ones_0x1
-#define _hrt_ones_2  _hrt_ones_0x2
-#define _hrt_ones_3  _hrt_ones_0x3
-#define _hrt_ones_4  _hrt_ones_0x4
-#define _hrt_ones_5  _hrt_ones_0x5
-#define _hrt_ones_6  _hrt_ones_0x6
-#define _hrt_ones_7  _hrt_ones_0x7
-#define _hrt_ones_8  _hrt_ones_0x8
-#define _hrt_ones_9  _hrt_ones_0x9
-#define _hrt_ones_10 _hrt_ones_0xA
-#define _hrt_ones_11 _hrt_ones_0xB
-#define _hrt_ones_12 _hrt_ones_0xC
-#define _hrt_ones_13 _hrt_ones_0xD
-#define _hrt_ones_14 _hrt_ones_0xE
-#define _hrt_ones_15 _hrt_ones_0xF
-#define _hrt_ones_16 _hrt_ones_0x10
-#define _hrt_ones_17 _hrt_ones_0x11
-#define _hrt_ones_18 _hrt_ones_0x12
-#define _hrt_ones_19 _hrt_ones_0x13
-#define _hrt_ones_20 _hrt_ones_0x14
-#define _hrt_ones_21 _hrt_ones_0x15
-#define _hrt_ones_22 _hrt_ones_0x16
-#define _hrt_ones_23 _hrt_ones_0x17
-#define _hrt_ones_24 _hrt_ones_0x18
-#define _hrt_ones_25 _hrt_ones_0x19
-#define _hrt_ones_26 _hrt_ones_0x1A
-#define _hrt_ones_27 _hrt_ones_0x1B
-#define _hrt_ones_28 _hrt_ones_0x1C
-#define _hrt_ones_29 _hrt_ones_0x1D
-#define _hrt_ones_30 _hrt_ones_0x1E
-#define _hrt_ones_31 _hrt_ones_0x1F
-#define _hrt_ones_32 _hrt_ones_0x20
-
-#define _hrt_mask(b, n) \
-  (_hrt_ones(n) << (b))
-#define _hrt_get_bits(w, b, n) \
-  (((w) >> (b)) & _hrt_ones(n))
-#define _hrt_set_bits(w, b, n, v) \
-  (((w) & ~_hrt_mask(b, n)) | (((v) & _hrt_ones(n)) << (b)))
-#define _hrt_get_bit(w, b) \
-  (((w) >> (b)) & 1)
-#define _hrt_set_bit(w, b, v) \
-  (((w) & (~(1 << (b)))) | (((v)&1) << (b)))
-#define _hrt_set_lower_half(w, v) \
-  _hrt_set_bits(w, 0, 16, v)
-#define _hrt_set_upper_half(w, v) \
-  _hrt_set_bits(w, 16, 16, v)
-
-#endif /* _HRT_BITS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/cell_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/cell_params.h
deleted file mode 100644
index b5756bfe8eb6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/cell_params.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _cell_params_h
-#define _cell_params_h
-
-#define SP_PMEM_LOG_WIDTH_BITS           6  /*Width of PC, 64 bits, 8 bytes*/
-#define SP_ICACHE_TAG_BITS               4  /*size of tag*/
-#define SP_ICACHE_SET_BITS               8  /* 256 sets*/
-#define SP_ICACHE_BLOCKS_PER_SET_BITS    1  /* 2 way associative*/
-#define SP_ICACHE_BLOCK_ADDRESS_BITS     11 /* 2048 lines capacity*/
-
-#define SP_ICACHE_ADDRESS_BITS \
-	                    (SP_ICACHE_TAG_BITS+SP_ICACHE_BLOCK_ADDRESS_BITS)
-
-#define SP_PMEM_DEPTH        (1<<SP_ICACHE_ADDRESS_BITS)
-
-#define SP_FIFO_0_DEPTH      0
-#define SP_FIFO_1_DEPTH      0
-#define SP_FIFO_2_DEPTH      0
-#define SP_FIFO_3_DEPTH      0
-#define SP_FIFO_4_DEPTH      0
-#define SP_FIFO_5_DEPTH      0
-#define SP_FIFO_6_DEPTH      0
-#define SP_FIFO_7_DEPTH      0
-
-
-#define SP_SLV_BUS_MAXBURSTSIZE        1
-
-#endif /* _cell_params_h */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/css_receiver_2400_common_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/css_receiver_2400_common_defs.h
deleted file mode 100644
index f3054fe04d03..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/css_receiver_2400_common_defs.h
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _css_receiver_2400_common_defs_h_
-#define _css_receiver_2400_common_defs_h_
-#ifndef _mipi_backend_common_defs_h_
-#define _mipi_backend_common_defs_h_
-
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_DATA_WIDTH     16
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_CH_ID_WIDTH     2
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_FMT_TYPE_WIDTH  3
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_REAL_WIDTH (_HRT_CSS_RECEIVER_2400_GEN_SHORT_DATA_WIDTH + _HRT_CSS_RECEIVER_2400_GEN_SHORT_CH_ID_WIDTH + _HRT_CSS_RECEIVER_2400_GEN_SHORT_FMT_TYPE_WIDTH)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_WIDTH      32 /* use 32 to be compatibel with streaming monitor !, MSB's of interface are tied to '0' */ 
-
-/* Definition of data format ID at the interface CSS_receiver capture/acquisition units */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8          24   /* 01 1000 YUV420 8-bit                                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_10         25   /* 01 1001  YUV420 10-bit                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8L         26   /* 01 1010   YUV420 8-bit legacy                               */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV422_8          30   /* 01 1110   YUV422 8-bit                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV422_10         31   /* 01 1111   YUV422 10-bit                                     */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB444            32   /* 10 0000   RGB444                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB555            33   /* 10 0001   RGB555                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB565            34   /* 10 0010   RGB565                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB666            35   /* 10 0011   RGB666                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RGB888            36   /* 10 0100   RGB888                                            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW6              40   /* 10 1000   RAW6                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW7              41   /* 10 1001   RAW7                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW8              42   /* 10 1010   RAW8                                              */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW10             43   /* 10 1011   RAW10                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW12             44   /* 10 1100   RAW12                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW14             45   /* 10 1101   RAW14                                             */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_1         48   /* 11 0000    JPEG [User Defined 8-bit Data Type 1]            */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_2         49   /* 11 0001    User Defined 8-bit Data Type 2                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_3         50   /* 11 0010    User Defined 8-bit Data Type 3                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_4         51   /* 11 0011    User Defined 8-bit Data Type 4                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_5         52   /* 11 0100    User Defined 8-bit Data Type 5                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_6         53   /* 11 0101    User Defined 8-bit Data Type 6                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_7         54   /* 11 0110    User Defined 8-bit Data Type 7                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_USR_DEF_8         55   /* 11 0111    User Defined 8-bit Data Type 8                   */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_Emb               18   /* 01 0010    embedded eight bit non image data                */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_SOF                0   /* 00 0000    frame start                                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_EOF                1   /* 00 0001    frame end                                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_SOL                2   /* 00 0010    line start                                       */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_EOL                3   /* 00 0011    line end                                         */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH1            8   /* 00 1000  Generic Short Packet Code 1                        */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH2            9   /* 00 1001    Generic Short Packet Code 2                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH3           10   /* 00 1010    Generic Short Packet Code 3                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH4           11   /* 00 1011    Generic Short Packet Code 4                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH5           12   /* 00 1100    Generic Short Packet Code 5                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH6           13   /* 00 1101    Generic Short Packet Code 6                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH7           14   /* 00 1110    Generic Short Packet Code 7                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_GEN_SH8           15   /* 00 1111    Generic Short Packet Code 8                      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_8_CSPS     28   /* 01 1100   YUV420 8-bit (Chroma Shifted Pixel Sampling)      */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_YUV420_10_CSPS    29   /* 01 1101   YUV420 10-bit (Chroma Shifted Pixel Sampling)     */
-/* used reseved mipi positions for these */
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW16             46 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18             47 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18_2           37 
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_RAW18_3           38 
-
-#define _HRT_CSS_RECEIVER_2400_DATA_FORMAT_ID_WIDTH              6
-
-/* Definition of format_types at the interface CSS --> input_selector*/
-/* !! Changes here should be copied to systems/isp/isp_css/bin/conv_transmitter_cmd.tcl !! */
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB888           0  // 36 'h24
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB555           1  // 33 'h
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB444           2  // 32
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB565           3  // 34
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RGB666           4  // 35
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW8             5  // 42 
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW10            6  // 43
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW6             7  // 40
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW7             8  // 41
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW12            9  // 43
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW14           10  // 45
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8        11  // 30
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_10       12  // 25
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV422_8        13  // 30
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV422_10       14  // 31
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_1       15  // 48
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8L       16  // 26
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_Emb             17  // 18
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_2       18  // 49
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_3       19  // 50
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_4       20  // 51
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_5       21  // 52
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_6       22  // 53
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_7       23  // 54
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_USR_DEF_8       24  // 55
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_8_CSPS   25  // 28
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_YUV420_10_CSPS  26  // 29
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW16           27  // ?
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18           28  // ?
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18_2         29  // ? Option 2 for depacketiser
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_RAW18_3         30  // ? Option 3 for depacketiser
-#define _HRT_CSS_RECEIVER_2400_FMT_TYPE_CUSTOM          31  // to signal custom decoding 
-
-/* definition for state machine of data FIFO for decode different type of data */
-#define _HRT_CSS_RECEIVER_2400_YUV420_8_REPEAT_PTN                 1  
-#define _HRT_CSS_RECEIVER_2400_YUV420_10_REPEAT_PTN                5
-#define _HRT_CSS_RECEIVER_2400_YUV420_8L_REPEAT_PTN                1
-#define _HRT_CSS_RECEIVER_2400_YUV422_8_REPEAT_PTN                 1
-#define _HRT_CSS_RECEIVER_2400_YUV422_10_REPEAT_PTN                5
-#define _HRT_CSS_RECEIVER_2400_RGB444_REPEAT_PTN                   2 
-#define _HRT_CSS_RECEIVER_2400_RGB555_REPEAT_PTN                   2
-#define _HRT_CSS_RECEIVER_2400_RGB565_REPEAT_PTN                   2
-#define _HRT_CSS_RECEIVER_2400_RGB666_REPEAT_PTN                   9                       
-#define _HRT_CSS_RECEIVER_2400_RGB888_REPEAT_PTN                   3
-#define _HRT_CSS_RECEIVER_2400_RAW6_REPEAT_PTN                     3
-#define _HRT_CSS_RECEIVER_2400_RAW7_REPEAT_PTN                     7
-#define _HRT_CSS_RECEIVER_2400_RAW8_REPEAT_PTN                     1
-#define _HRT_CSS_RECEIVER_2400_RAW10_REPEAT_PTN                    5
-#define _HRT_CSS_RECEIVER_2400_RAW12_REPEAT_PTN                    3        
-#define _HRT_CSS_RECEIVER_2400_RAW14_REPEAT_PTN                    7
-
-#define _HRT_CSS_RECEIVER_2400_MAX_REPEAT_PTN                      _HRT_CSS_RECEIVER_2400_RGB666_REPEAT_PTN
-
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FMT_IDX                     0
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FMT_WIDTH                   3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_PRED_IDX                    3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_PRED_WIDTH                  1
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_USD_BITS                    4  /* bits per USD type */
-
-#define _HRT_CSS_RECEIVER_2400_BE_RAW16_DATAID_IDX                 0
-#define _HRT_CSS_RECEIVER_2400_BE_RAW16_EN_IDX                     6
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_DATAID_IDX                 0
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_OPTION_IDX                 6
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_EN_IDX                     8
-
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_NO_COMP                     0
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_6_10                     1
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_7_10                     2
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_10_8_10                     3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_6_12                     4
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_7_12                     5
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_12_8_12                     6
-
-
-/* packet bit definition */
-#define _HRT_CSS_RECEIVER_2400_PKT_SOP_IDX                        32
-#define _HRT_CSS_RECEIVER_2400_PKT_SOP_BITS                        1
-#define _HRT_CSS_RECEIVER_2400_PKT_CH_ID_IDX                      22
-#define _HRT_CSS_RECEIVER_2400_PKT_CH_ID_BITS                      2
-#define _HRT_CSS_RECEIVER_2400_PKT_FMT_ID_IDX                     16
-#define _HRT_CSS_RECEIVER_2400_PKT_FMT_ID_BITS                     6
-#define _HRT_CSS_RECEIVER_2400_PH_DATA_FIELD_IDX                   0
-#define _HRT_CSS_RECEIVER_2400_PH_DATA_FIELD_BITS                 16
-#define _HRT_CSS_RECEIVER_2400_PKT_PAYLOAD_IDX                     0
-#define _HRT_CSS_RECEIVER_2400_PKT_PAYLOAD_BITS                   32
-
-
-/*************************************************************************************************/
-/* Custom Decoding                                                                               */
-/* These Custom Defs are defined based on design-time config in "csi_be_pixel_formatter.chdl" !! */
-/*************************************************************************************************/
-#define BE_CUST_EN_IDX                     0     /* 2bits */
-#define BE_CUST_EN_DATAID_IDX              2     /* 6bits MIPI DATA ID */ 
-#define BE_CUST_EN_WIDTH                   8     
-#define BE_CUST_MODE_ALL                   1     /* Enable Custom Decoding for all DATA IDs */
-#define BE_CUST_MODE_ONE                   3     /* Enable Custom Decoding for ONE DATA ID, programmed in CUST_EN_DATA_ID */
-
-/* Data State config = {get_bits(6bits), valid(1bit)}  */
-#define BE_CUST_DATA_STATE_S0_IDX          0     /* 7bits */ 
-#define BE_CUST_DATA_STATE_S1_IDX          7     /* 7bits */ 
-#define BE_CUST_DATA_STATE_S2_IDX          14    /* 7bits */
-#define BE_CUST_DATA_STATE_WIDTH           21    
-#define BE_CUST_DATA_STATE_VALID_IDX       0     /* 1bits */
-#define BE_CUST_DATA_STATE_GETBITS_IDX     1     /* 6bits */
-
-/* Pixel Extractor config */
-#define BE_CUST_PIX_EXT_DATA_ALIGN_IDX     0     /* 5bits */
-#define BE_CUST_PIX_EXT_PIX_ALIGN_IDX      5     /* 5bits */
-#define BE_CUST_PIX_EXT_PIX_MASK_IDX       10    /* 18bits */
-#define BE_CUST_PIX_EXT_PIX_EN_IDX         28    /* 1bits */
-#define BE_CUST_PIX_EXT_WIDTH              29    
-
-/* Pixel Valid & EoP config = {[eop,valid](especial), [eop,valid](normal)} */
-#define BE_CUST_PIX_VALID_EOP_P0_IDX        0    /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_P1_IDX        4    /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_P2_IDX        8    /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_P3_IDX        12   /* 4bits */
-#define BE_CUST_PIX_VALID_EOP_WIDTH         16 
-#define BE_CUST_PIX_VALID_EOP_NOR_VALID_IDX 0    /* Normal (NO less get_bits case) Valid - 1bits */
-#define BE_CUST_PIX_VALID_EOP_NOR_EOP_IDX   1    /* Normal (NO less get_bits case) EoP - 1bits */
-#define BE_CUST_PIX_VALID_EOP_ESP_VALID_IDX 2    /* Especial (less get_bits case) Valid - 1bits */
-#define BE_CUST_PIX_VALID_EOP_ESP_EOP_IDX   3    /* Especial (less get_bits case) EoP - 1bits */
-
-#endif /* _mipi_backend_common_defs_h_ */
-#endif /* _css_receiver_2400_common_defs_h_ */ 
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/css_receiver_2400_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/css_receiver_2400_defs.h
deleted file mode 100644
index 6f5b7d3d3715..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/css_receiver_2400_defs.h
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _css_receiver_2400_defs_h_
-#define _css_receiver_2400_defs_h_
-
-#include "css_receiver_2400_common_defs.h"
-
-#define CSS_RECEIVER_DATA_WIDTH                8
-#define CSS_RECEIVER_RX_TRIG                   4
-#define CSS_RECEIVER_RF_WORD                  32
-#define CSS_RECEIVER_IMG_PROC_RF_ADDR         10
-#define CSS_RECEIVER_CSI_RF_ADDR               4
-#define CSS_RECEIVER_DATA_OUT                 12
-#define CSS_RECEIVER_CHN_NO                    2
-#define CSS_RECEIVER_DWORD_CNT                11
-#define CSS_RECEIVER_FORMAT_TYP                5
-#define CSS_RECEIVER_HRESPONSE                 2
-#define CSS_RECEIVER_STATE_WIDTH               3
-#define CSS_RECEIVER_FIFO_DAT                 32
-#define CSS_RECEIVER_CNT_VAL                   2
-#define CSS_RECEIVER_PRED10_VAL               10
-#define CSS_RECEIVER_PRED12_VAL               12
-#define CSS_RECEIVER_CNT_WIDTH                 8
-#define CSS_RECEIVER_WORD_CNT                 16
-#define CSS_RECEIVER_PIXEL_LEN                 6
-#define CSS_RECEIVER_PIXEL_CNT                 5
-#define CSS_RECEIVER_COMP_8_BIT                8
-#define CSS_RECEIVER_COMP_7_BIT                7
-#define CSS_RECEIVER_COMP_6_BIT                6
-
-#define CSI_CONFIG_WIDTH                       4
-
-/* division of gen_short data, ch_id and fmt_type over streaming data interface */
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_DATA_BIT_LSB     0
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_LSB (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_DATA_BIT_LSB     + _HRT_CSS_RECEIVER_2400_GEN_SHORT_DATA_WIDTH)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_CH_ID_BIT_LSB    (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_LSB + _HRT_CSS_RECEIVER_2400_GEN_SHORT_FMT_TYPE_WIDTH)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_DATA_BIT_MSB     (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_LSB - 1)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_FMT_TYPE_BIT_MSB (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_CH_ID_BIT_LSB    - 1)
-#define _HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_CH_ID_BIT_MSB    (_HRT_CSS_RECEIVER_2400_GEN_SHORT_STR_REAL_WIDTH       - 1)
-
-#define _HRT_CSS_RECEIVER_2400_REG_ALIGN 4
-#define _HRT_CSS_RECEIVER_2400_BYTES_PER_PKT             4
-
-#define hrt_css_receiver_2400_4_lane_port_offset  0x100
-#define hrt_css_receiver_2400_1_lane_port_offset  0x200
-#define hrt_css_receiver_2400_2_lane_port_offset  0x300
-#define hrt_css_receiver_2400_backend_port_offset 0x100
-
-#define _HRT_CSS_RECEIVER_2400_DEVICE_READY_REG_IDX      0
-#define _HRT_CSS_RECEIVER_2400_IRQ_STATUS_REG_IDX        1
-#define _HRT_CSS_RECEIVER_2400_IRQ_ENABLE_REG_IDX        2
-#define _HRT_CSS_RECEIVER_2400_CSI2_FUNC_PROG_REG_IDX    3
-#define _HRT_CSS_RECEIVER_2400_INIT_COUNT_REG_IDX        4
-#define _HRT_CSS_RECEIVER_2400_FS_TO_LS_DELAY_REG_IDX    7
-#define _HRT_CSS_RECEIVER_2400_LS_TO_DATA_DELAY_REG_IDX  8
-#define _HRT_CSS_RECEIVER_2400_DATA_TO_LE_DELAY_REG_IDX  9
-#define _HRT_CSS_RECEIVER_2400_LE_TO_FE_DELAY_REG_IDX   10
-#define _HRT_CSS_RECEIVER_2400_FE_TO_FS_DELAY_REG_IDX   11
-#define _HRT_CSS_RECEIVER_2400_LE_TO_LS_DELAY_REG_IDX   12
-#define _HRT_CSS_RECEIVER_2400_TWO_PIXEL_EN_REG_IDX     13
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_REG_IDX  14
-#define _HRT_CSS_RECEIVER_2400_SYNC_COUNT_REG_IDX       15
-#define _HRT_CSS_RECEIVER_2400_RX_COUNT_REG_IDX         16
-#define _HRT_CSS_RECEIVER_2400_BACKEND_RST_REG_IDX      17
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG0_IDX 18
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG1_IDX 19
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG0_IDX 20
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG1_IDX 21
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG0_IDX 22
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG1_IDX 23
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG0_IDX 24
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG1_IDX 25
-#define _HRT_CSS_RECEIVER_2400_RAW18_REG_IDX            26
-#define _HRT_CSS_RECEIVER_2400_FORCE_RAW8_REG_IDX       27
-#define _HRT_CSS_RECEIVER_2400_RAW16_REG_IDX            28
-
-/* Interrupt bits for IRQ_STATUS and IRQ_ENABLE registers */
-#define _HRT_CSS_RECEIVER_2400_IRQ_OVERRUN_BIT                0
-#define _HRT_CSS_RECEIVER_2400_IRQ_RESERVED_BIT               1
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_ENTRY_BIT       2
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_EXIT_BIT        3
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_HS_BIT             4
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_SYNC_HS_BIT        5
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CONTROL_BIT            6
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_DOUBLE_BIT         7
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_CORRECTED_BIT      8
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_NO_CORRECTION_BIT  9
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CRC_BIT               10
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ID_BIT                11
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_SYNC_BIT        12
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_DATA_BIT        13
-#define _HRT_CSS_RECEIVER_2400_IRQ_DATA_TIMEOUT_BIT          14
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ESCAPE_BIT            15
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_LINE_SYNC_BIT         16
-
-#define _HRT_CSS_RECEIVER_2400_IRQ_OVERRUN_CAUSE_                  "Fifo Overrun"
-#define _HRT_CSS_RECEIVER_2400_IRQ_RESERVED_CAUSE_                 "Reserved"
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_ENTRY_CAUSE_         "Sleep mode entry"
-#define _HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_EXIT_CAUSE_          "Sleep mode exit"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_HS_CAUSE_               "Error high speed SOT"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_SYNC_HS_CAUSE_          "Error high speed sync SOT"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CONTROL_CAUSE_              "Error control"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_DOUBLE_CAUSE_           "Error correction double bit"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_CORRECTED_CAUSE_        "Error correction single bit"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_NO_CORRECTION_CAUSE_    "No error"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_CRC_CAUSE_                  "Error cyclic redundancy check"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ID_CAUSE_                   "Error id"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_SYNC_CAUSE_           "Error frame sync"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_DATA_CAUSE_           "Error frame data"
-#define _HRT_CSS_RECEIVER_2400_IRQ_DATA_TIMEOUT_CAUSE_             "Data time-out"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_ESCAPE_CAUSE_               "Error escape"
-#define _HRT_CSS_RECEIVER_2400_IRQ_ERR_LINE_SYNC_CAUSE_            "Error line sync"
-
-/* Bits for CSI2_DEVICE_READY register */
-#define _HRT_CSS_RECEIVER_2400_CSI2_DEVICE_READY_IDX                          0
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_INIT_TIME_OUT_ERR_IDX                2
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_OVER_RUN_ERR_IDX                     3
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_SOT_SYNC_ERR_IDX                     4
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_RECEIVE_DATA_TIME_OUT_ERR_IDX        5
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_ECC_TWO_BIT_ERR_IDX                  6
-#define _HRT_CSS_RECEIVER_2400_CSI2_MASK_DATA_ID_ERR_IDX                      7
-
-                                  
-/* Bits for CSI2_FUNC_PROG register */
-#define _HRT_CSS_RECEIVER_2400_CSI2_DATA_TIMEOUT_IDX    0
-#define _HRT_CSS_RECEIVER_2400_CSI2_DATA_TIMEOUT_BITS   19
-
-/* Bits for INIT_COUNT register */
-#define _HRT_CSS_RECEIVER_2400_INIT_TIMER_IDX  0
-#define _HRT_CSS_RECEIVER_2400_INIT_TIMER_BITS 16
-
-/* Bits for COUNT registers */
-#define _HRT_CSS_RECEIVER_2400_SYNC_COUNT_IDX     0
-#define _HRT_CSS_RECEIVER_2400_SYNC_COUNT_BITS    8
-#define _HRT_CSS_RECEIVER_2400_RX_COUNT_IDX       0
-#define _HRT_CSS_RECEIVER_2400_RX_COUNT_BITS      8
-
-/* Bits for RAW116_18_DATAID register */
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW16_BITS_IDX   0
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW16_BITS_BITS  6
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW18_BITS_IDX   8
-#define _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_RAW18_BITS_BITS  6
-
-/* Bits for COMP_FORMAT register, this selects the compression data format */
-#define _HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_IDX  0
-#define _HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_BITS 8
-#define _HRT_CSS_RECEIVER_2400_COMP_NUM_BITS_IDX  (_HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_IDX + _HRT_CSS_RECEIVER_2400_COMP_RAW_BITS_BITS)
-#define _HRT_CSS_RECEIVER_2400_COMP_NUM_BITS_BITS 8
-
-/* Bits for COMP_PREDICT register, this selects the predictor algorithm */
-#define _HRT_CSS_RECEIVER_2400_PREDICT_NO_COMP 0
-#define _HRT_CSS_RECEIVER_2400_PREDICT_1       1
-#define _HRT_CSS_RECEIVER_2400_PREDICT_2       2
-
-/* Number of bits used for the delay registers */
-#define _HRT_CSS_RECEIVER_2400_DELAY_BITS 8
-
-/* Bits for COMP_SCHEME register, this  selects the compression scheme for a VC */
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD1_BITS_IDX  0
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD2_BITS_IDX  5
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD3_BITS_IDX  10
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD4_BITS_IDX  15
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD5_BITS_IDX  20
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD6_BITS_IDX  25
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD7_BITS_IDX  0
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD8_BITS_IDX  5
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_BITS_BITS  5
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_FMT_BITS_IDX   0
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_FMT_BITS_BITS  3
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_PRED_BITS_IDX  3
-#define _HRT_CSS_RECEIVER_2400_COMP_SCHEME_USD_PRED_BITS_BITS 2
-
-
-/* BITS for backend RAW16 and RAW 18 registers */
-
-#define _HRT_CSS_RECEIVER_2400_RAW18_DATAID_IDX    0
-#define _HRT_CSS_RECEIVER_2400_RAW18_DATAID_BITS   6
-#define _HRT_CSS_RECEIVER_2400_RAW18_OPTION_IDX    6
-#define _HRT_CSS_RECEIVER_2400_RAW18_OPTION_BITS   2
-#define _HRT_CSS_RECEIVER_2400_RAW18_EN_IDX        8
-#define _HRT_CSS_RECEIVER_2400_RAW18_EN_BITS       1
-
-#define _HRT_CSS_RECEIVER_2400_RAW16_DATAID_IDX    0
-#define _HRT_CSS_RECEIVER_2400_RAW16_DATAID_BITS   6
-#define _HRT_CSS_RECEIVER_2400_RAW16_OPTION_IDX    6
-#define _HRT_CSS_RECEIVER_2400_RAW16_OPTION_BITS   2
-#define _HRT_CSS_RECEIVER_2400_RAW16_EN_IDX        8
-#define _HRT_CSS_RECEIVER_2400_RAW16_EN_BITS       1
-
-/* These hsync and vsync values are for HSS simulation only */
-#define _HRT_CSS_RECEIVER_2400_HSYNC_VAL (1<<16)
-#define _HRT_CSS_RECEIVER_2400_VSYNC_VAL (1<<17)
-
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_WIDTH                 28
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_LSB              0
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_MSB             (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_LSB + CSS_RECEIVER_DATA_OUT - 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_VAL_BIT         (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_MSB + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_LSB             (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_A_VAL_BIT + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_MSB             (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_LSB + CSS_RECEIVER_DATA_OUT - 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_VAL_BIT         (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_MSB + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_SOP_BIT               (_HRT_CSS_RECEIVER_2400_BE_STREAMING_PIX_B_VAL_BIT + 1)
-#define _HRT_CSS_RECEIVER_2400_BE_STREAMING_EOP_BIT               (_HRT_CSS_RECEIVER_2400_BE_STREAMING_SOP_BIT + 1)
-
-// SH Backend Register IDs
-#define _HRT_CSS_RECEIVER_2400_BE_GSP_ACC_OVL_REG_IDX              0
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_REG_IDX                     1
-#define _HRT_CSS_RECEIVER_2400_BE_TWO_PPC_REG_IDX                  2
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG0_IDX             3
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG1_IDX             4
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG2_IDX             5
-#define _HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG3_IDX             6
-#define _HRT_CSS_RECEIVER_2400_BE_SEL_REG_IDX                      7
-#define _HRT_CSS_RECEIVER_2400_BE_RAW16_CONFIG_REG_IDX             8
-#define _HRT_CSS_RECEIVER_2400_BE_RAW18_CONFIG_REG_IDX             9
-#define _HRT_CSS_RECEIVER_2400_BE_FORCE_RAW8_REG_IDX              10
-#define _HRT_CSS_RECEIVER_2400_BE_IRQ_STATUS_REG_IDX              11
-#define _HRT_CSS_RECEIVER_2400_BE_IRQ_CLEAR_REG_IDX               12
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_EN_REG_IDX                 13
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_DATA_STATE_REG_IDX         14    /* Data State 0,1,2 config */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P0_REG_IDX       15    /* Pixel Extractor config for Data State 0 & Pix 0 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P1_REG_IDX       16    /* Pixel Extractor config for Data State 0 & Pix 1 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P2_REG_IDX       17    /* Pixel Extractor config for Data State 0 & Pix 2 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S0P3_REG_IDX       18    /* Pixel Extractor config for Data State 0 & Pix 3 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P0_REG_IDX       19    /* Pixel Extractor config for Data State 1 & Pix 0 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P1_REG_IDX       20    /* Pixel Extractor config for Data State 1 & Pix 1 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P2_REG_IDX       21    /* Pixel Extractor config for Data State 1 & Pix 2 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S1P3_REG_IDX       22    /* Pixel Extractor config for Data State 1 & Pix 3 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P0_REG_IDX       23    /* Pixel Extractor config for Data State 2 & Pix 0 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P1_REG_IDX       24    /* Pixel Extractor config for Data State 2 & Pix 1 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P2_REG_IDX       25    /* Pixel Extractor config for Data State 2 & Pix 2 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_EXT_S2P3_REG_IDX       26    /* Pixel Extractor config for Data State 2 & Pix 3 */
-#define _HRT_CSS_RECEIVER_2400_BE_CUST_PIX_VALID_EOP_REG_IDX      27    /* Pixel Valid & EoP config for Pix 0,1,2,3 */
-
-#define _HRT_CSS_RECEIVER_2400_BE_NOF_REGISTERS                   28
-
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_HE                          0
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_RCF                         1
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_PF                          2
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_SM                          3
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_PD                          4
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_SD                          5
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_OT                          6
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_BC                          7
-#define _HRT_CSS_RECEIVER_2400_BE_SRST_WIDTH                       8
-
-#endif /* _css_receiver_2400_defs_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/defs.h
deleted file mode 100644
index 47505f41790c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/defs.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_DEFS_H_
-#define _HRT_DEFS_H_
-
-#ifndef HRTCAT
-#define _HRTCAT(m, n)     m##n
-#define HRTCAT(m, n)      _HRTCAT(m, n)
-#endif
-
-#ifndef HRTSTR
-#define _HRTSTR(x)   #x
-#define HRTSTR(x)    _HRTSTR(x)
-#endif
-
-#ifndef HRTMIN
-#define HRTMIN(a, b) (((a) < (b)) ? (a) : (b))
-#endif
-
-#ifndef HRTMAX
-#define HRTMAX(a, b) (((a) > (b)) ? (a) : (b))
-#endif
-
-#endif /* _HRT_DEFS_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/dma_v2_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/dma_v2_defs.h
deleted file mode 100644
index d184a8b313c9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/dma_v2_defs.h
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _dma_v2_defs_h
-#define _dma_v2_defs_h
-
-#define _DMA_V2_NUM_CHANNELS_ID               MaxNumChannels
-#define _DMA_V2_CONNECTIONS_ID                Connections
-#define _DMA_V2_DEV_ELEM_WIDTHS_ID            DevElemWidths
-#define _DMA_V2_DEV_FIFO_DEPTH_ID             DevFifoDepth
-#define _DMA_V2_DEV_FIFO_RD_LAT_ID            DevFifoRdLat
-#define _DMA_V2_DEV_FIFO_LAT_BYPASS_ID        DevFifoRdLatBypass
-#define _DMA_V2_DEV_NO_BURST_ID               DevNoBurst
-#define _DMA_V2_DEV_RD_ACCEPT_ID              DevRdAccept
-#define _DMA_V2_DEV_SRMD_ID                   DevSRMD
-#define _DMA_V2_DEV_HAS_CRUN_ID               CRunMasters
-#define _DMA_V2_CTRL_ACK_FIFO_DEPTH_ID        CtrlAckFifoDepth
-#define _DMA_V2_CMD_FIFO_DEPTH_ID             CommandFifoDepth
-#define _DMA_V2_CMD_FIFO_RD_LAT_ID            CommandFifoRdLat
-#define _DMA_V2_CMD_FIFO_LAT_BYPASS_ID        CommandFifoRdLatBypass
-#define _DMA_V2_NO_PACK_ID                    has_no_pack
-
-#define _DMA_V2_REG_ALIGN                4
-#define _DMA_V2_REG_ADDR_BITS            2
-
-/* Command word */
-#define _DMA_V2_CMD_IDX            0
-#define _DMA_V2_CMD_BITS           6
-#define _DMA_V2_CHANNEL_IDX        (_DMA_V2_CMD_IDX + _DMA_V2_CMD_BITS)
-#define _DMA_V2_CHANNEL_BITS       5
-
-/* The command to set a parameter contains the PARAM field next */
-#define _DMA_V2_PARAM_IDX          (_DMA_V2_CHANNEL_IDX + _DMA_V2_CHANNEL_BITS)
-#define _DMA_V2_PARAM_BITS         4
-
-/* Commands to read, write or init specific blocks contain these
-   three values */
-#define _DMA_V2_SPEC_DEV_A_XB_IDX  (_DMA_V2_CHANNEL_IDX + _DMA_V2_CHANNEL_BITS)
-#define _DMA_V2_SPEC_DEV_A_XB_BITS 8
-#define _DMA_V2_SPEC_DEV_B_XB_IDX  (_DMA_V2_SPEC_DEV_A_XB_IDX + _DMA_V2_SPEC_DEV_A_XB_BITS)
-#define _DMA_V2_SPEC_DEV_B_XB_BITS 8
-#define _DMA_V2_SPEC_YB_IDX        (_DMA_V2_SPEC_DEV_B_XB_IDX + _DMA_V2_SPEC_DEV_B_XB_BITS)
-#define _DMA_V2_SPEC_YB_BITS       (32-_DMA_V2_SPEC_DEV_B_XB_BITS-_DMA_V2_SPEC_DEV_A_XB_BITS-_DMA_V2_CMD_BITS-_DMA_V2_CHANNEL_BITS)
-
-/* */
-#define _DMA_V2_CMD_CTRL_IDX       4
-#define _DMA_V2_CMD_CTRL_BITS      4
-
-/* Packing setup word */
-#define _DMA_V2_CONNECTION_IDX     0
-#define _DMA_V2_CONNECTION_BITS    4
-#define _DMA_V2_EXTENSION_IDX      (_DMA_V2_CONNECTION_IDX + _DMA_V2_CONNECTION_BITS)
-#define _DMA_V2_EXTENSION_BITS     1
-
-/* Elements packing word */
-#define _DMA_V2_ELEMENTS_IDX        0
-#define _DMA_V2_ELEMENTS_BITS       8
-#define _DMA_V2_LEFT_CROPPING_IDX  (_DMA_V2_ELEMENTS_IDX + _DMA_V2_ELEMENTS_BITS)
-#define _DMA_V2_LEFT_CROPPING_BITS  8
-
-#define _DMA_V2_WIDTH_IDX           0
-#define _DMA_V2_WIDTH_BITS         16
-
-#define _DMA_V2_HEIGHT_IDX          0
-#define _DMA_V2_HEIGHT_BITS        16
-
-#define _DMA_V2_STRIDE_IDX          0
-#define _DMA_V2_STRIDE_BITS        32
-
-/* Command IDs */
-#define _DMA_V2_MOVE_B2A_COMMAND                             0      
-#define _DMA_V2_MOVE_B2A_BLOCK_COMMAND                       1      
-#define _DMA_V2_MOVE_B2A_NO_SYNC_CHK_COMMAND                 2      
-#define _DMA_V2_MOVE_B2A_BLOCK_NO_SYNC_CHK_COMMAND           3      
-#define _DMA_V2_MOVE_A2B_COMMAND                             4      
-#define _DMA_V2_MOVE_A2B_BLOCK_COMMAND                       5      
-#define _DMA_V2_MOVE_A2B_NO_SYNC_CHK_COMMAND                 6      
-#define _DMA_V2_MOVE_A2B_BLOCK_NO_SYNC_CHK_COMMAND           7      
-#define _DMA_V2_INIT_A_COMMAND                               8      
-#define _DMA_V2_INIT_A_BLOCK_COMMAND                         9      
-#define _DMA_V2_INIT_A_NO_SYNC_CHK_COMMAND                  10      
-#define _DMA_V2_INIT_A_BLOCK_NO_SYNC_CHK_COMMAND            11      
-#define _DMA_V2_INIT_B_COMMAND                              12      
-#define _DMA_V2_INIT_B_BLOCK_COMMAND                        13      
-#define _DMA_V2_INIT_B_NO_SYNC_CHK_COMMAND                  14      
-#define _DMA_V2_INIT_B_BLOCK_NO_SYNC_CHK_COMMAND            15      
-#define _DMA_V2_NO_ACK_MOVE_B2A_NO_SYNC_CHK_COMMAND         (_DMA_V2_MOVE_B2A_NO_SYNC_CHK_COMMAND       + 16) 
-#define _DMA_V2_NO_ACK_MOVE_B2A_BLOCK_NO_SYNC_CHK_COMMAND   (_DMA_V2_MOVE_B2A_BLOCK_NO_SYNC_CHK_COMMAND + 16) 
-#define _DMA_V2_NO_ACK_MOVE_A2B_NO_SYNC_CHK_COMMAND         (_DMA_V2_MOVE_A2B_NO_SYNC_CHK_COMMAND       + 16) 
-#define _DMA_V2_NO_ACK_MOVE_A2B_BLOCK_NO_SYNC_CHK_COMMAND   (_DMA_V2_MOVE_A2B_BLOCK_NO_SYNC_CHK_COMMAND + 16) 
-#define _DMA_V2_NO_ACK_INIT_A_NO_SYNC_CHK_COMMAND           (_DMA_V2_INIT_A_NO_SYNC_CHK_COMMAND         + 16) 
-#define _DMA_V2_NO_ACK_INIT_A_BLOCK_NO_SYNC_CHK_COMMAND     (_DMA_V2_INIT_A_BLOCK_NO_SYNC_CHK_COMMAND   + 16) 
-#define _DMA_V2_NO_ACK_INIT_B_NO_SYNC_CHK_COMMAND           (_DMA_V2_INIT_B_NO_SYNC_CHK_COMMAND         + 16) 
-#define _DMA_V2_NO_ACK_INIT_B_BLOCK_NO_SYNC_CHK_COMMAND     (_DMA_V2_INIT_B_BLOCK_NO_SYNC_CHK_COMMAND   + 16) 
-#define _DMA_V2_CONFIG_CHANNEL_COMMAND                      32   
-#define _DMA_V2_SET_CHANNEL_PARAM_COMMAND                   33   
-#define _DMA_V2_SET_CRUN_COMMAND                            62   
-
-/* Channel Parameter IDs */
-#define _DMA_V2_PACKING_SETUP_PARAM                     0  
-#define _DMA_V2_STRIDE_A_PARAM                          1  
-#define _DMA_V2_ELEM_CROPPING_A_PARAM                   2  
-#define _DMA_V2_WIDTH_A_PARAM                           3  
-#define _DMA_V2_STRIDE_B_PARAM                          4  
-#define _DMA_V2_ELEM_CROPPING_B_PARAM                   5  
-#define _DMA_V2_WIDTH_B_PARAM                           6  
-#define _DMA_V2_HEIGHT_PARAM                            7  
-#define _DMA_V2_QUEUED_CMDS                             8  
-
-/* Parameter Constants */
-#define _DMA_V2_ZERO_EXTEND                             0
-#define _DMA_V2_SIGN_EXTEND                             1
-
-  /* SLAVE address map */
-#define _DMA_V2_SEL_FSM_CMD                             0
-#define _DMA_V2_SEL_CH_REG                              1
-#define _DMA_V2_SEL_CONN_GROUP                          2
-#define _DMA_V2_SEL_DEV_INTERF                          3
-
-#define _DMA_V2_ADDR_SEL_COMP_IDX                      12
-#define _DMA_V2_ADDR_SEL_COMP_BITS                      4
-#define _DMA_V2_ADDR_SEL_CH_REG_IDX                     2
-#define _DMA_V2_ADDR_SEL_CH_REG_BITS                    6
-#define _DMA_V2_ADDR_SEL_PARAM_IDX                      (_DMA_V2_ADDR_SEL_CH_REG_BITS+_DMA_V2_ADDR_SEL_CH_REG_IDX)
-#define _DMA_V2_ADDR_SEL_PARAM_BITS                     4
-
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_IDX                 2
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_BITS                6
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_INFO_IDX            (_DMA_V2_ADDR_SEL_GROUP_COMP_BITS + _DMA_V2_ADDR_SEL_GROUP_COMP_IDX)
-#define _DMA_V2_ADDR_SEL_GROUP_COMP_INFO_BITS           4
-
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_IDX_IDX             2
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_IDX_BITS            6
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_INFO_IDX            (_DMA_V2_ADDR_SEL_DEV_INTERF_IDX_IDX+_DMA_V2_ADDR_SEL_DEV_INTERF_IDX_BITS)
-#define _DMA_V2_ADDR_SEL_DEV_INTERF_INFO_BITS           4
-
-#define _DMA_V2_FSM_GROUP_CMD_IDX                       0
-#define _DMA_V2_FSM_GROUP_ADDR_SRC_IDX                  1
-#define _DMA_V2_FSM_GROUP_ADDR_DEST_IDX                 2
-#define _DMA_V2_FSM_GROUP_CMD_CTRL_IDX                  3
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_IDX                  4
-#define _DMA_V2_FSM_GROUP_FSM_PACK_IDX                  5
-#define _DMA_V2_FSM_GROUP_FSM_REQ_IDX                   6
-#define _DMA_V2_FSM_GROUP_FSM_WR_IDX                    7
-  
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_STATE_IDX            0
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_DEV_IDX          1
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_ADDR_IDX         2
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_STRIDE_IDX       3
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_XB_IDX           4
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_REQ_YB_IDX           5
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_REQ_DEV_IDX     6
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_DEV_IDX      7
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_WR_ADDR_IDX          8
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_WR_STRIDE_IDX        9
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_REQ_XB_IDX     10
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_YB_IDX      11
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_XB_IDX      12
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_ELEM_REQ_IDX   13
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_ELEM_WR_IDX    14
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_PACK_S_Z_IDX        15
-#define _DMA_V2_FSM_GROUP_FSM_CTRL_CMD_CTRL_IDX        15
-
-#define _DMA_V2_FSM_GROUP_FSM_PACK_STATE_IDX            0
-#define _DMA_V2_FSM_GROUP_FSM_PACK_CNT_YB_IDX           1
-#define _DMA_V2_FSM_GROUP_FSM_PACK_CNT_XB_REQ_IDX       2
-#define _DMA_V2_FSM_GROUP_FSM_PACK_CNT_XB_WR_IDX        3
-
-#define _DMA_V2_FSM_GROUP_FSM_REQ_STATE_IDX             0
-#define _DMA_V2_FSM_GROUP_FSM_REQ_CNT_YB_IDX            1
-#define _DMA_V2_FSM_GROUP_FSM_REQ_CNT_XB_IDX            2
-#define _DMA_V2_FSM_GROUP_FSM_REQ_XB_REMAINING_IDX      3
-#define _DMA_V2_FSM_GROUP_FSM_REQ_CNT_BURST_IDX         4
-
-#define _DMA_V2_FSM_GROUP_FSM_WR_STATE_IDX              0
-#define _DMA_V2_FSM_GROUP_FSM_WR_CNT_YB_IDX             1
-#define _DMA_V2_FSM_GROUP_FSM_WR_CNT_XB_IDX             2
-#define _DMA_V2_FSM_GROUP_FSM_WR_XB_REMAINING_IDX       3
-#define _DMA_V2_FSM_GROUP_FSM_WR_CNT_BURST_IDX          4
-
-#define _DMA_V2_DEV_INTERF_REQ_SIDE_STATUS_IDX          0
-#define _DMA_V2_DEV_INTERF_SEND_SIDE_STATUS_IDX         1
-#define _DMA_V2_DEV_INTERF_FIFO_STATUS_IDX              2
-#define _DMA_V2_DEV_INTERF_REQ_ONLY_COMPLETE_BURST_IDX  3
-#define _DMA_V2_DEV_INTERF_MAX_BURST_IDX                4
-#define _DMA_V2_DEV_INTERF_CHK_ADDR_ALIGN               5
-
-#endif /* _dma_v2_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gdc_v2_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gdc_v2_defs.h
deleted file mode 100644
index 77722d205701..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gdc_v2_defs.h
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef HRT_GDC_v2_defs_h_
-#define HRT_GDC_v2_defs_h_
-
-#define HRT_GDC_IS_V2
-
-#define HRT_GDC_N                     1024 /* Top-level design constant, equal to the number of entries in the LUT      */
-#define HRT_GDC_FRAC_BITS               10 /* Number of fractional bits in the GDC block, driven by the size of the LUT */
-
-#define HRT_GDC_BLI_FRAC_BITS            4 /* Number of fractional bits for the bi-linear interpolation type            */
-#define HRT_GDC_BLI_COEF_ONE             (1 << HRT_GDC_BLI_FRAC_BITS)
-
-#define HRT_GDC_BCI_COEF_BITS           14 /* 14 bits per coefficient                                                   */
-#define HRT_GDC_BCI_COEF_ONE             (1 << (HRT_GDC_BCI_COEF_BITS-2))  /* We represent signed 10 bit coefficients.  */
-                                                                        /* The supported range is [-256, .., +256]      */
-                                                                        /* in 14-bit signed notation,                   */
-                                                                        /* We need all ten bits (MSB must be zero).     */
-                                                                        /* -s is inserted to solve this issue, and      */
-                                                                        /* therefore "1" is equal to +256.              */
-#define HRT_GDC_BCI_COEF_MASK            ((1 << HRT_GDC_BCI_COEF_BITS) - 1) 
-
-#define HRT_GDC_LUT_BYTES                (HRT_GDC_N*4*2)                /* 1024 addresses, 4 coefficients per address,  */
-                                                                        /* 2 bytes per coefficient                      */
-
-#define _HRT_GDC_REG_ALIGN               4                              
-
-  //     31  30  29    25 24                     0
-  //  |-----|---|--------|------------------------|
-  //  | CMD | C | Reg_ID |        Value           |
-
-
-  // There are just two commands possible for the GDC block:
-  // 1 - Configure reg 
-  // 0 - Data token    
-  
-  // C      - Reserved bit
-  //          Used in protocol to indicate whether it is C-run or other type of runs
-  //          In case of C-run, this bit has a value of 1, for all the other runs, it is 0.
-
-  // Reg_ID - Address of the register to be configured
-  
-  // Value  - Value to store to the addressed register, maximum of 24 bits
-
-  // Configure reg command is not followed by any other token. 
-  // The address of the register and the data to be filled in is contained in the same token 
-  
-  // When the first data token is received, it must be:
-  //   1. FRX and FRY (device configured in one of the  scaling modes) ***DEFAULT MODE***, or,
-  //   2. P0'X        (device configured in one of the tetragon modes)
-  // After the first data token is received, pre-defined number of tokens with the following meaning follow:
-  //   1. two  tokens: SRC address ; DST address
-  //   2. nine tokens: P0'Y, .., P3'Y ; SRC address ; DST address
-  
-#define HRT_GDC_CONFIG_CMD             1
-#define HRT_GDC_DATA_CMD               0
-
-
-#define HRT_GDC_CMD_POS               31
-#define HRT_GDC_CMD_BITS               1
-#define HRT_GDC_CRUN_POS              30
-#define HRT_GDC_REG_ID_POS            25
-#define HRT_GDC_REG_ID_BITS            5
-#define HRT_GDC_DATA_POS               0
-#define HRT_GDC_DATA_BITS             25
-
-#define HRT_GDC_FRYIPXFRX_BITS        26
-#define HRT_GDC_P0X_BITS              23
-
-
-#define HRT_GDC_MAX_OXDIM           (8192-64)
-#define HRT_GDC_MAX_OYDIM           4095
-#define HRT_GDC_MAX_IXDIM           (8192-64)
-#define HRT_GDC_MAX_IYDIM           4095
-#define HRT_GDC_MAX_DS_FAC            16
-#define HRT_GDC_MAX_DX                 (HRT_GDC_MAX_DS_FAC*HRT_GDC_N - 1)
-#define HRT_GDC_MAX_DY                 HRT_GDC_MAX_DX
-
-
-/* GDC lookup tables entries are 10 bits values, but they're
-   stored 2 by 2 as 32 bit values, yielding 16 bits per entry.
-   A GDC lookup table contains 64 * 4 elements */
-
-#define HRT_GDC_PERF_1_1_pix          0
-#define HRT_GDC_PERF_2_1_pix          1
-#define HRT_GDC_PERF_1_2_pix          2
-#define HRT_GDC_PERF_2_2_pix          3
-
-#define HRT_GDC_NND_MODE              0
-#define HRT_GDC_BLI_MODE              1
-#define HRT_GDC_BCI_MODE              2
-#define HRT_GDC_LUT_MODE              3
-
-#define HRT_GDC_SCAN_STB              0
-#define HRT_GDC_SCAN_STR              1
-
-#define HRT_GDC_MODE_SCALING          0
-#define HRT_GDC_MODE_TETRAGON         1
-
-#define HRT_GDC_LUT_COEFF_OFFSET     16 
-#define HRT_GDC_FRY_BIT_OFFSET       16 
-// FRYIPXFRX is the only register where we store two values in one field, 
-// to save one token in the scaling protocol. 
-// Like this, we have three tokens in the scaling protocol, 
-// Otherwise, we would have had four.
-// The register bit-map is:
-//   31  26 25      16 15  10 9        0
-//  |------|----------|------|----------|
-//  | XXXX |   FRY    |  IPX |   FRX    |
-
-
-#define HRT_GDC_CE_FSM0_POS           0
-#define HRT_GDC_CE_FSM0_LEN           2
-#define HRT_GDC_CE_OPY_POS            2
-#define HRT_GDC_CE_OPY_LEN           14
-#define HRT_GDC_CE_OPX_POS           16
-#define HRT_GDC_CE_OPX_LEN           16
-// CHK_ENGINE register bit-map:
-//   31            16 15        2 1  0
-//  |----------------|-----------|----|
-//  |      OPX       |    OPY    |FSM0|
-// However, for the time being at least, 
-// this implementation is meaningless in hss model,
-// So, we just return 0
-
-
-#define HRT_GDC_CHK_ENGINE_IDX        0
-#define HRT_GDC_WOIX_IDX              1
-#define HRT_GDC_WOIY_IDX              2
-#define HRT_GDC_BPP_IDX               3
-#define HRT_GDC_FRYIPXFRX_IDX         4
-#define HRT_GDC_OXDIM_IDX             5
-#define HRT_GDC_OYDIM_IDX             6
-#define HRT_GDC_SRC_ADDR_IDX          7
-#define HRT_GDC_SRC_END_ADDR_IDX      8
-#define HRT_GDC_SRC_WRAP_ADDR_IDX     9
-#define HRT_GDC_SRC_STRIDE_IDX       10
-#define HRT_GDC_DST_ADDR_IDX         11
-#define HRT_GDC_DST_STRIDE_IDX       12
-#define HRT_GDC_DX_IDX               13
-#define HRT_GDC_DY_IDX               14
-#define HRT_GDC_P0X_IDX              15
-#define HRT_GDC_P0Y_IDX              16
-#define HRT_GDC_P1X_IDX              17
-#define HRT_GDC_P1Y_IDX              18
-#define HRT_GDC_P2X_IDX              19
-#define HRT_GDC_P2Y_IDX              20
-#define HRT_GDC_P3X_IDX              21
-#define HRT_GDC_P3Y_IDX              22
-#define HRT_GDC_PERF_POINT_IDX       23  // 1x1 ; 1x2 ; 2x1 ; 2x2 pixels per cc
-#define HRT_GDC_INTERP_TYPE_IDX      24  // NND ; BLI ; BCI ; LUT
-#define HRT_GDC_SCAN_IDX             25  // 0 = STB (Slide To Bottom) ; 1 = STR (Slide To Right)
-#define HRT_GDC_PROC_MODE_IDX        26  // 0 = Scaling ; 1 = Tetragon
-
-#define HRT_GDC_LUT_IDX              32
-
-
-#endif /* HRT_GDC_v2_defs_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gp_timer_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gp_timer_defs.h
deleted file mode 100644
index 3082e2f5e014..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gp_timer_defs.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _gp_timer_defs_h
-#define _gp_timer_defs_h
-
-#define _HRT_GP_TIMER_REG_ALIGN 4
-
-#define HIVE_GP_TIMER_RESET_REG_IDX                              0
-#define HIVE_GP_TIMER_OVERALL_ENABLE_REG_IDX                     1
-#define HIVE_GP_TIMER_ENABLE_REG_IDX(timer)                     (HIVE_GP_TIMER_OVERALL_ENABLE_REG_IDX + 1 + timer)
-#define HIVE_GP_TIMER_VALUE_REG_IDX(timer,timers)               (HIVE_GP_TIMER_ENABLE_REG_IDX(timers) + timer)
-#define HIVE_GP_TIMER_COUNT_TYPE_REG_IDX(timer,timers)          (HIVE_GP_TIMER_VALUE_REG_IDX(timers, timers) + timer)
-#define HIVE_GP_TIMER_SIGNAL_SELECT_REG_IDX(timer,timers)       (HIVE_GP_TIMER_COUNT_TYPE_REG_IDX(timers, timers) + timer)
-#define HIVE_GP_TIMER_IRQ_TRIGGER_VALUE_REG_IDX(irq,timers)     (HIVE_GP_TIMER_SIGNAL_SELECT_REG_IDX(timers, timers) + irq)
-#define HIVE_GP_TIMER_IRQ_TIMER_SELECT_REG_IDX(irq,timers,irqs) (HIVE_GP_TIMER_IRQ_TRIGGER_VALUE_REG_IDX(irqs, timers) + irq)
-#define HIVE_GP_TIMER_IRQ_ENABLE_REG_IDX(irq,timers,irqs)       (HIVE_GP_TIMER_IRQ_TIMER_SELECT_REG_IDX(irqs, timers, irqs) + irq)
-
-#define HIVE_GP_TIMER_COUNT_TYPE_HIGH                            0
-#define HIVE_GP_TIMER_COUNT_TYPE_LOW                             1
-#define HIVE_GP_TIMER_COUNT_TYPE_POSEDGE                         2
-#define HIVE_GP_TIMER_COUNT_TYPE_NEGEDGE                         3
-#define HIVE_GP_TIMER_COUNT_TYPES                                4
-
-#endif /* _gp_timer_defs_h */   
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gpio_block_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gpio_block_defs.h
deleted file mode 100644
index a807d4c99041..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/gpio_block_defs.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _gpio_block_defs_h_
-#define _gpio_block_defs_h_
-
-#define _HRT_GPIO_BLOCK_REG_ALIGN 4
-
-/* R/W registers */
-#define _gpio_block_reg_do_e			         0
-#define _gpio_block_reg_do_select		       1
-#define _gpio_block_reg_do_0			         2
-#define _gpio_block_reg_do_1			         3
-#define _gpio_block_reg_do_pwm_cnt_0	     4
-#define _gpio_block_reg_do_pwm_cnt_1	     5
-#define _gpio_block_reg_do_pwm_cnt_2	     6
-#define _gpio_block_reg_do_pwm_cnt_3	     7
-#define _gpio_block_reg_do_pwm_main_cnt    8
-#define _gpio_block_reg_do_pwm_enable      9
-#define _gpio_block_reg_di_debounce_sel	  10
-#define _gpio_block_reg_di_debounce_cnt_0	11
-#define _gpio_block_reg_di_debounce_cnt_1	12
-#define _gpio_block_reg_di_debounce_cnt_2	13
-#define _gpio_block_reg_di_debounce_cnt_3	14
-#define _gpio_block_reg_di_active_level	  15
-
-
-/* read-only registers */
-#define _gpio_block_reg_di			          16
-
-#endif /* _gpio_block_defs_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_2401_irq_types_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_2401_irq_types_hrt.h
deleted file mode 100644
index 7a94c1d85b08..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_2401_irq_types_hrt.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
-#ifndef ISP2401
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_
-#define _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_
-
-/*
- * These are the indices of each interrupt in the interrupt
- * controller's registers. these can be used as the irq_id
- * argument to the hrt functions irq_controller.h.
- *
- * The definitions are taken from <system>_defs.h
- */
-typedef enum hrt_isp_css_irq {
-  hrt_isp_css_irq_gpio_pin_0           = HIVE_GP_DEV_IRQ_GPIO_PIN_0_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_1           = HIVE_GP_DEV_IRQ_GPIO_PIN_1_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_2           = HIVE_GP_DEV_IRQ_GPIO_PIN_2_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_3           = HIVE_GP_DEV_IRQ_GPIO_PIN_3_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_4           = HIVE_GP_DEV_IRQ_GPIO_PIN_4_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_5           = HIVE_GP_DEV_IRQ_GPIO_PIN_5_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_6           = HIVE_GP_DEV_IRQ_GPIO_PIN_6_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_7           = HIVE_GP_DEV_IRQ_GPIO_PIN_7_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_8           = HIVE_GP_DEV_IRQ_GPIO_PIN_8_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_9           = HIVE_GP_DEV_IRQ_GPIO_PIN_9_BIT_ID          ,               
-  hrt_isp_css_irq_gpio_pin_10          = HIVE_GP_DEV_IRQ_GPIO_PIN_10_BIT_ID         ,              
-  hrt_isp_css_irq_gpio_pin_11          = HIVE_GP_DEV_IRQ_GPIO_PIN_11_BIT_ID         ,              
-  hrt_isp_css_irq_sp                   = HIVE_GP_DEV_IRQ_SP_BIT_ID                  ,                       
-  hrt_isp_css_irq_isp                  = HIVE_GP_DEV_IRQ_ISP_BIT_ID                 ,                      
-  hrt_isp_css_irq_isys                 = HIVE_GP_DEV_IRQ_ISYS_BIT_ID                ,                     
-  hrt_isp_css_irq_isel                 = HIVE_GP_DEV_IRQ_ISEL_BIT_ID                ,                     
-  hrt_isp_css_irq_ifmt                 = HIVE_GP_DEV_IRQ_IFMT_BIT_ID                ,                     
-  hrt_isp_css_irq_sp_stream_mon        = HIVE_GP_DEV_IRQ_SP_STREAM_MON_BIT_ID       ,            
-  hrt_isp_css_irq_isp_stream_mon       = HIVE_GP_DEV_IRQ_ISP_STREAM_MON_BIT_ID      ,           
-  hrt_isp_css_irq_mod_stream_mon       = HIVE_GP_DEV_IRQ_MOD_STREAM_MON_BIT_ID      ,
-  hrt_isp_css_irq_is2401               = HIVE_GP_DEV_IRQ_IS2401_BIT_ID              ,           
-  hrt_isp_css_irq_isp_bamem_error      = HIVE_GP_DEV_IRQ_ISP_BAMEM_ERROR_BIT_ID     ,          
-  hrt_isp_css_irq_isp_dmem_error       = HIVE_GP_DEV_IRQ_ISP_DMEM_ERROR_BIT_ID      ,           
-  hrt_isp_css_irq_sp_icache_mem_error  = HIVE_GP_DEV_IRQ_SP_ICACHE_MEM_ERROR_BIT_ID ,      
-  hrt_isp_css_irq_sp_dmem_error        = HIVE_GP_DEV_IRQ_SP_DMEM_ERROR_BIT_ID       ,            
-  hrt_isp_css_irq_mmu_cache_mem_error  = HIVE_GP_DEV_IRQ_MMU_CACHE_MEM_ERROR_BIT_ID ,      
-  hrt_isp_css_irq_gp_timer_0           = HIVE_GP_DEV_IRQ_GP_TIMER_0_BIT_ID          ,               
-  hrt_isp_css_irq_gp_timer_1           = HIVE_GP_DEV_IRQ_GP_TIMER_1_BIT_ID          ,               
-  hrt_isp_css_irq_sw_pin_0             = HIVE_GP_DEV_IRQ_SW_PIN_0_BIT_ID            ,                 
-  hrt_isp_css_irq_sw_pin_1             = HIVE_GP_DEV_IRQ_SW_PIN_1_BIT_ID            ,                 
-  hrt_isp_css_irq_dma                  = HIVE_GP_DEV_IRQ_DMA_BIT_ID                 ,
-  hrt_isp_css_irq_sp_stream_mon_b      = HIVE_GP_DEV_IRQ_SP_STREAM_MON_B_BIT_ID     ,
-  /* this must (obviously) be the last on in the enum */
-  hrt_isp_css_irq_num_irqs
-} hrt_isp_css_irq_t;
-
-typedef enum hrt_isp_css_irq_status {
-  hrt_isp_css_irq_status_error,
-  hrt_isp_css_irq_status_more_irqs,
-  hrt_isp_css_irq_status_success
-} hrt_isp_css_irq_status_t;
-
-#endif /* _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_defs.h
deleted file mode 100644
index 5a2ce9108ae4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_defs.h
+++ /dev/null
@@ -1,435 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _hive_isp_css_defs_h__
-#define _hive_isp_css_defs_h__
-
-#define _HIVE_ISP_CSS_2401_SYSTEM     1
-#define HIVE_ISP_CTRL_DATA_WIDTH     32
-#define HIVE_ISP_CTRL_ADDRESS_WIDTH  32
-#define HIVE_ISP_CTRL_MAX_BURST_SIZE  1
-#define HIVE_ISP_DDR_ADDRESS_WIDTH   36
-
-#define HIVE_ISP_HOST_MAX_BURST_SIZE  8 /* host supports bursts in order to prevent repeating DDRAM accesses */
-#define HIVE_ISP_NUM_GPIO_PINS       12
-
-/* This list of vector num_elems/elem_bits pairs is valid both in C as initializer
-   and in the DMA parameter list */
-#define HIVE_ISP_DDR_DMA_SPECS {{32,  8}, {16, 16}, {18, 14}, {25, 10}, {21, 12}}
-#define HIVE_ISP_DDR_WORD_BITS 256
-#define HIVE_ISP_DDR_WORD_BYTES  (HIVE_ISP_DDR_WORD_BITS/8)
-#define HIVE_ISP_DDR_BYTES       (512 * 1024 * 1024)
-#define HIVE_ISP_DDR_BYTES_RTL   (127 * 1024 * 1024)
-#define HIVE_ISP_DDR_SMALL_BYTES (128 * 256 / 8)
-#define HIVE_ISP_PAGE_SHIFT    12
-#define HIVE_ISP_PAGE_SIZE     (1<<HIVE_ISP_PAGE_SHIFT)
-
-#define CSS_DDR_WORD_BITS        HIVE_ISP_DDR_WORD_BITS
-#define CSS_DDR_WORD_BYTES       HIVE_ISP_DDR_WORD_BYTES
-
-/* settings used in applications */
-#define HIVE_XMEM_WIDTH            HIVE_ISP_DDR_WORD_BITS
-#define HIVE_VMEM_VECTOR_ELEMENTS  64
-#define HIVE_VMEM_ELEMENT_BITS     14
-#define HIVE_XMEM_ELEMENT_BITS     16
-#define HIVE_VMEM_VECTOR_BYTES (HIVE_VMEM_VECTOR_ELEMENTS*HIVE_XMEM_ELEMENT_BITS/8) /* used for # addr bytes for one vector */
-#define HIVE_XMEM_PACKED_WORD_VMEM_ELEMENTS (HIVE_XMEM_WIDTH/HIVE_VMEM_ELEMENT_BITS)
-#define HIVE_XMEM_WORD_VMEM_ELEMENTS        (HIVE_XMEM_WIDTH/HIVE_XMEM_ELEMENT_BITS)
-#define XMEM_INT_SIZE              4
-
-
-
-#define HIVE_ISYS_INP_BUFFER_BYTES (64*1024)  /* 64 kByte = 2k words (of 256 bits) */
-
-/* If HIVE_ISP_DDR_BASE_OFFSET is set to a non-zero value, the wide bus just before the DDRAM gets an extra dummy port where         */
-/* address range 0 .. HIVE_ISP_DDR_BASE_OFFSET-1 maps onto. This effectively creates an offset for the DDRAM from system perspective */
-#define HIVE_ISP_DDR_BASE_OFFSET 0x120000000 /* 0x200000 */
-
-#define HIVE_DMA_ISP_BUS_CONN 0
-#define HIVE_DMA_ISP_DDR_CONN 1
-#define HIVE_DMA_BUS_DDR_CONN 2
-#define HIVE_DMA_ISP_MASTER master_port0
-#define HIVE_DMA_BUS_MASTER master_port1
-#define HIVE_DMA_DDR_MASTER master_port2
-
-#define HIVE_DMA_NUM_CHANNELS       32 /* old value was  8 */
-#define HIVE_DMA_CMD_FIFO_DEPTH     24 /* old value was 12 */
-
-#define HIVE_IF_PIXEL_WIDTH 12
-
-#define HIVE_MMU_TLB_SETS           8
-#define HIVE_MMU_TLB_SET_BLOCKS     8
-#define HIVE_MMU_TLB_BLOCK_ELEMENTS 8
-#define HIVE_MMU_PAGE_TABLE_LEVELS  2
-#define HIVE_MMU_PAGE_BYTES         HIVE_ISP_PAGE_SIZE
-
-#define HIVE_ISP_CH_ID_BITS    2
-#define HIVE_ISP_FMT_TYPE_BITS 5
-#define HIVE_ISP_ISEL_SEL_BITS 2
-
-#define HIVE_GP_REGS_SDRAM_WAKEUP_IDX                           0
-#define HIVE_GP_REGS_IDLE_IDX                                   1
-#define HIVE_GP_REGS_IRQ_0_IDX                                  2
-#define HIVE_GP_REGS_IRQ_1_IDX                                  3
-#define HIVE_GP_REGS_SP_STREAM_STAT_IDX                         4
-#define HIVE_GP_REGS_SP_STREAM_STAT_B_IDX                       5
-#define HIVE_GP_REGS_ISP_STREAM_STAT_IDX                        6
-#define HIVE_GP_REGS_MOD_STREAM_STAT_IDX                        7
-#define HIVE_GP_REGS_SP_STREAM_STAT_IRQ_COND_IDX                8
-#define HIVE_GP_REGS_SP_STREAM_STAT_B_IRQ_COND_IDX              9
-#define HIVE_GP_REGS_ISP_STREAM_STAT_IRQ_COND_IDX              10
-#define HIVE_GP_REGS_MOD_STREAM_STAT_IRQ_COND_IDX              11
-#define HIVE_GP_REGS_SP_STREAM_STAT_IRQ_ENABLE_IDX             12
-#define HIVE_GP_REGS_SP_STREAM_STAT_B_IRQ_ENABLE_IDX           13
-#define HIVE_GP_REGS_ISP_STREAM_STAT_IRQ_ENABLE_IDX            14
-#define HIVE_GP_REGS_MOD_STREAM_STAT_IRQ_ENABLE_IDX            15
-#define HIVE_GP_REGS_SWITCH_PRIM_IF_IDX                        16
-#define HIVE_GP_REGS_SWITCH_GDC1_IDX                           17
-#define HIVE_GP_REGS_SWITCH_GDC2_IDX                           18
-#define HIVE_GP_REGS_SRST_IDX                                  19
-#define HIVE_GP_REGS_SLV_REG_SRST_IDX                          20
-#define HIVE_GP_REGS_SWITCH_ISYS_IDX                           21
-
-/* Bit numbers of the soft reset register */
-#define HIVE_GP_REGS_SRST_ISYS_CBUS                             0
-#define HIVE_GP_REGS_SRST_ISEL_CBUS                             1
-#define HIVE_GP_REGS_SRST_IFMT_CBUS                             2
-#define HIVE_GP_REGS_SRST_GPDEV_CBUS                            3
-#define HIVE_GP_REGS_SRST_GPIO                                  4
-#define HIVE_GP_REGS_SRST_TC                                    5
-#define HIVE_GP_REGS_SRST_GPTIMER                               6
-#define HIVE_GP_REGS_SRST_FACELLFIFOS                           7
-#define HIVE_GP_REGS_SRST_D_OSYS                                8
-#define HIVE_GP_REGS_SRST_IFT_SEC_PIPE                          9
-#define HIVE_GP_REGS_SRST_GDC1                                 10
-#define HIVE_GP_REGS_SRST_GDC2                                 11
-#define HIVE_GP_REGS_SRST_VEC_BUS                              12
-#define HIVE_GP_REGS_SRST_ISP                                  13
-#define HIVE_GP_REGS_SRST_SLV_GRP_BUS                          14
-#define HIVE_GP_REGS_SRST_DMA                                  15
-#define HIVE_GP_REGS_SRST_SF_ISP_SP                            16
-#define HIVE_GP_REGS_SRST_SF_PIF_CELLS                         17
-#define HIVE_GP_REGS_SRST_SF_SIF_SP                            18
-#define HIVE_GP_REGS_SRST_SF_MC_SP                             19
-#define HIVE_GP_REGS_SRST_SF_ISYS_SP                           20
-#define HIVE_GP_REGS_SRST_SF_DMA_CELLS                         21
-#define HIVE_GP_REGS_SRST_SF_GDC1_CELLS                        22
-#define HIVE_GP_REGS_SRST_SF_GDC2_CELLS                        23
-#define HIVE_GP_REGS_SRST_SP                                   24
-#define HIVE_GP_REGS_SRST_OCP2CIO                              25
-#define HIVE_GP_REGS_SRST_NBUS                                 26
-#define HIVE_GP_REGS_SRST_HOST12BUS                            27
-#define HIVE_GP_REGS_SRST_WBUS                                 28
-#define HIVE_GP_REGS_SRST_IC_OSYS                              29
-#define HIVE_GP_REGS_SRST_WBUS_IC                              30
-#define HIVE_GP_REGS_SRST_ISYS_INP_BUF_BUS                     31
-
-/* Bit numbers of the slave register soft reset register */
-#define HIVE_GP_REGS_SLV_REG_SRST_DMA                           0
-#define HIVE_GP_REGS_SLV_REG_SRST_GDC1                          1
-#define HIVE_GP_REGS_SLV_REG_SRST_GDC2                          2
-
-/* order of the input bits for the irq controller */
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_0_BIT_ID                       0
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_1_BIT_ID                       1
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_2_BIT_ID                       2
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_3_BIT_ID                       3
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_4_BIT_ID                       4
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_5_BIT_ID                       5
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_6_BIT_ID                       6
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_7_BIT_ID                       7
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_8_BIT_ID                       8
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_9_BIT_ID                       9
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_10_BIT_ID                     10
-#define HIVE_GP_DEV_IRQ_GPIO_PIN_11_BIT_ID                     11
-#define HIVE_GP_DEV_IRQ_SP_BIT_ID                              12
-#define HIVE_GP_DEV_IRQ_ISP_BIT_ID                             13
-#define HIVE_GP_DEV_IRQ_ISYS_BIT_ID                            14
-#define HIVE_GP_DEV_IRQ_ISEL_BIT_ID                            15
-#define HIVE_GP_DEV_IRQ_IFMT_BIT_ID                            16
-#define HIVE_GP_DEV_IRQ_SP_STREAM_MON_BIT_ID                   17
-#define HIVE_GP_DEV_IRQ_ISP_STREAM_MON_BIT_ID                  18
-#define HIVE_GP_DEV_IRQ_MOD_STREAM_MON_BIT_ID                  19
-#define HIVE_GP_DEV_IRQ_IS2401_BIT_ID                          20
-#define HIVE_GP_DEV_IRQ_ISP_BAMEM_ERROR_BIT_ID                 21
-#define HIVE_GP_DEV_IRQ_ISP_DMEM_ERROR_BIT_ID                  22
-#define HIVE_GP_DEV_IRQ_SP_ICACHE_MEM_ERROR_BIT_ID             23
-#define HIVE_GP_DEV_IRQ_SP_DMEM_ERROR_BIT_ID                   24
-#define HIVE_GP_DEV_IRQ_MMU_CACHE_MEM_ERROR_BIT_ID             25
-#define HIVE_GP_DEV_IRQ_GP_TIMER_0_BIT_ID                      26
-#define HIVE_GP_DEV_IRQ_GP_TIMER_1_BIT_ID                      27
-#define HIVE_GP_DEV_IRQ_SW_PIN_0_BIT_ID                        28
-#define HIVE_GP_DEV_IRQ_SW_PIN_1_BIT_ID                        29
-#define HIVE_GP_DEV_IRQ_DMA_BIT_ID                             30
-#define HIVE_GP_DEV_IRQ_SP_STREAM_MON_B_BIT_ID                 31
-
-#define HIVE_GP_REGS_NUM_SW_IRQ_REGS                            2
-
-/* order of the input bits for the timed controller */
-#define HIVE_GP_DEV_TC_GPIO_PIN_0_BIT_ID                       0
-#define HIVE_GP_DEV_TC_GPIO_PIN_1_BIT_ID                       1
-#define HIVE_GP_DEV_TC_GPIO_PIN_2_BIT_ID                       2
-#define HIVE_GP_DEV_TC_GPIO_PIN_3_BIT_ID                       3
-#define HIVE_GP_DEV_TC_GPIO_PIN_4_BIT_ID                       4
-#define HIVE_GP_DEV_TC_GPIO_PIN_5_BIT_ID                       5
-#define HIVE_GP_DEV_TC_GPIO_PIN_6_BIT_ID                       6
-#define HIVE_GP_DEV_TC_GPIO_PIN_7_BIT_ID                       7
-#define HIVE_GP_DEV_TC_GPIO_PIN_8_BIT_ID                       8
-#define HIVE_GP_DEV_TC_GPIO_PIN_9_BIT_ID                       9
-#define HIVE_GP_DEV_TC_GPIO_PIN_10_BIT_ID                     10
-#define HIVE_GP_DEV_TC_GPIO_PIN_11_BIT_ID                     11
-#define HIVE_GP_DEV_TC_SP_BIT_ID                              12
-#define HIVE_GP_DEV_TC_ISP_BIT_ID                             13
-#define HIVE_GP_DEV_TC_ISYS_BIT_ID                            14
-#define HIVE_GP_DEV_TC_ISEL_BIT_ID                            15
-#define HIVE_GP_DEV_TC_IFMT_BIT_ID                            16
-#define HIVE_GP_DEV_TC_GP_TIMER_0_BIT_ID                      17
-#define HIVE_GP_DEV_TC_GP_TIMER_1_BIT_ID                      18
-#define HIVE_GP_DEV_TC_MIPI_SOL_BIT_ID                        19
-#define HIVE_GP_DEV_TC_MIPI_EOL_BIT_ID                        20
-#define HIVE_GP_DEV_TC_MIPI_SOF_BIT_ID                        21
-#define HIVE_GP_DEV_TC_MIPI_EOF_BIT_ID                        22
-#define HIVE_GP_DEV_TC_INPSYS_SM                              23
-
-/* definitions for the gp_timer block */
-#define HIVE_GP_TIMER_0                                         0
-#define HIVE_GP_TIMER_1                                         1
-#define HIVE_GP_TIMER_2                                         2
-#define HIVE_GP_TIMER_3                                         3
-#define HIVE_GP_TIMER_4                                         4
-#define HIVE_GP_TIMER_5                                         5
-#define HIVE_GP_TIMER_6                                         6
-#define HIVE_GP_TIMER_7                                         7
-#define HIVE_GP_TIMER_NUM_COUNTERS                              8
-
-#define HIVE_GP_TIMER_IRQ_0                                     0
-#define HIVE_GP_TIMER_IRQ_1                                     1
-#define HIVE_GP_TIMER_NUM_IRQS                                  2
-
-#define HIVE_GP_TIMER_GPIO_0_BIT_ID                             0
-#define HIVE_GP_TIMER_GPIO_1_BIT_ID                             1
-#define HIVE_GP_TIMER_GPIO_2_BIT_ID                             2
-#define HIVE_GP_TIMER_GPIO_3_BIT_ID                             3
-#define HIVE_GP_TIMER_GPIO_4_BIT_ID                             4
-#define HIVE_GP_TIMER_GPIO_5_BIT_ID                             5
-#define HIVE_GP_TIMER_GPIO_6_BIT_ID                             6
-#define HIVE_GP_TIMER_GPIO_7_BIT_ID                             7
-#define HIVE_GP_TIMER_GPIO_8_BIT_ID                             8
-#define HIVE_GP_TIMER_GPIO_9_BIT_ID                             9
-#define HIVE_GP_TIMER_GPIO_10_BIT_ID                           10
-#define HIVE_GP_TIMER_GPIO_11_BIT_ID                           11
-#define HIVE_GP_TIMER_INP_SYS_IRQ                              12
-#define HIVE_GP_TIMER_ISEL_IRQ                                 13
-#define HIVE_GP_TIMER_IFMT_IRQ                                 14
-#define HIVE_GP_TIMER_SP_STRMON_IRQ                            15
-#define HIVE_GP_TIMER_SP_B_STRMON_IRQ                          16
-#define HIVE_GP_TIMER_ISP_STRMON_IRQ                           17
-#define HIVE_GP_TIMER_MOD_STRMON_IRQ                           18
-#define HIVE_GP_TIMER_IS2401_IRQ                               19
-#define HIVE_GP_TIMER_ISP_BAMEM_ERROR_IRQ                      20
-#define HIVE_GP_TIMER_ISP_DMEM_ERROR_IRQ                       21
-#define HIVE_GP_TIMER_SP_ICACHE_MEM_ERROR_IRQ                  22
-#define HIVE_GP_TIMER_SP_DMEM_ERROR_IRQ                        23
-#define HIVE_GP_TIMER_SP_OUT_RUN_DP                            24
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I0         25
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I1         26
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I2         27
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I3         28
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I4         29
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I5         30
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I6         31
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I7         32
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I8         33
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I9         34
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I0_I10        35
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I1_I0         36
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I2_I0         37
-#define HIVE_GP_TIMER_SP_WIRE_DEBUG_LM_MSINK_RUN_I3_I0         38
-#define HIVE_GP_TIMER_ISP_OUT_RUN_DP                           39
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I0_I0        40
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I0_I1        41
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I1_I0        42
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I0        43
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I1        44
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I2        45
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I3        46
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I4        47
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I5        48
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I2_I6        49
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I3_I0        50
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I4_I0        51
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I5_I0        52
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I6_I0        53
-#define HIVE_GP_TIMER_ISP_WIRE_DEBUG_LM_MSINK_RUN_I7_I0        54                                                         
-#define HIVE_GP_TIMER_MIPI_SOL_BIT_ID                          55
-#define HIVE_GP_TIMER_MIPI_EOL_BIT_ID                          56
-#define HIVE_GP_TIMER_MIPI_SOF_BIT_ID                          57
-#define HIVE_GP_TIMER_MIPI_EOF_BIT_ID                          58
-#define HIVE_GP_TIMER_INPSYS_SM                                59
-#define HIVE_GP_TIMER_ISP_PMEM_ERROR_IRQ                       60
-
-/* port definitions for the streaming monitors */
-/* port definititions SP streaming monitor, monitors the status of streaming ports at the SP side of the streaming FIFO's */
-#define SP_STR_MON_PORT_SP2SIF            0
-#define SP_STR_MON_PORT_SIF2SP            1
-#define SP_STR_MON_PORT_SP2MC             2 
-#define SP_STR_MON_PORT_MC2SP             3
-#define SP_STR_MON_PORT_SP2DMA            4 
-#define SP_STR_MON_PORT_DMA2SP            5
-#define SP_STR_MON_PORT_SP2ISP            6 
-#define SP_STR_MON_PORT_ISP2SP            7
-#define SP_STR_MON_PORT_SP2GPD            8
-#define SP_STR_MON_PORT_FA2SP             9
-#define SP_STR_MON_PORT_SP2ISYS          10 
-#define SP_STR_MON_PORT_ISYS2SP          11
-#define SP_STR_MON_PORT_SP2PIFA          12
-#define SP_STR_MON_PORT_PIFA2SP          13
-#define SP_STR_MON_PORT_SP2PIFB          14
-#define SP_STR_MON_PORT_PIFB2SP          15
-
-#define SP_STR_MON_PORT_B_SP2GDC1         0
-#define SP_STR_MON_PORT_B_GDC12SP         1
-#define SP_STR_MON_PORT_B_SP2GDC2         2
-#define SP_STR_MON_PORT_B_GDC22SP         3
-
-/* previously used SP streaming monitor port identifiers, kept for backward compatibility */
-#define SP_STR_MON_PORT_SND_SIF           SP_STR_MON_PORT_SP2SIF
-#define SP_STR_MON_PORT_RCV_SIF           SP_STR_MON_PORT_SIF2SP
-#define SP_STR_MON_PORT_SND_MC            SP_STR_MON_PORT_SP2MC
-#define SP_STR_MON_PORT_RCV_MC            SP_STR_MON_PORT_MC2SP
-#define SP_STR_MON_PORT_SND_DMA           SP_STR_MON_PORT_SP2DMA
-#define SP_STR_MON_PORT_RCV_DMA           SP_STR_MON_PORT_DMA2SP
-#define SP_STR_MON_PORT_SND_ISP           SP_STR_MON_PORT_SP2ISP
-#define SP_STR_MON_PORT_RCV_ISP           SP_STR_MON_PORT_ISP2SP
-#define SP_STR_MON_PORT_SND_GPD           SP_STR_MON_PORT_SP2GPD
-#define SP_STR_MON_PORT_RCV_GPD           SP_STR_MON_PORT_FA2SP
-/* Deprecated */
-#define SP_STR_MON_PORT_SND_PIF           SP_STR_MON_PORT_SP2PIFA
-#define SP_STR_MON_PORT_RCV_PIF           SP_STR_MON_PORT_PIFA2SP
-#define SP_STR_MON_PORT_SND_PIFB          SP_STR_MON_PORT_SP2PIFB
-#define SP_STR_MON_PORT_RCV_PIFB          SP_STR_MON_PORT_PIFB2SP
-
-#define SP_STR_MON_PORT_SND_PIF_A         SP_STR_MON_PORT_SP2PIFA
-#define SP_STR_MON_PORT_RCV_PIF_A         SP_STR_MON_PORT_PIFA2SP
-#define SP_STR_MON_PORT_SND_PIF_B         SP_STR_MON_PORT_SP2PIFB
-#define SP_STR_MON_PORT_RCV_PIF_B         SP_STR_MON_PORT_PIFB2SP
-
-/* port definititions ISP streaming monitor, monitors the status of streaming ports at the ISP side of the streaming FIFO's */
-#define ISP_STR_MON_PORT_ISP2PIFA         0
-#define ISP_STR_MON_PORT_PIFA2ISP         1
-#define ISP_STR_MON_PORT_ISP2PIFB         2 
-#define ISP_STR_MON_PORT_PIFB2ISP         3
-#define ISP_STR_MON_PORT_ISP2DMA          4 
-#define ISP_STR_MON_PORT_DMA2ISP          5
-#define ISP_STR_MON_PORT_ISP2GDC1         6 
-#define ISP_STR_MON_PORT_GDC12ISP         7
-#define ISP_STR_MON_PORT_ISP2GDC2         8 
-#define ISP_STR_MON_PORT_GDC22ISP         9
-#define ISP_STR_MON_PORT_ISP2GPD         10 
-#define ISP_STR_MON_PORT_FA2ISP          11
-#define ISP_STR_MON_PORT_ISP2SP          12 
-#define ISP_STR_MON_PORT_SP2ISP          13
-
-/* previously used ISP streaming monitor port identifiers, kept for backward compatibility */
-#define ISP_STR_MON_PORT_SND_PIF_A       ISP_STR_MON_PORT_ISP2PIFA
-#define ISP_STR_MON_PORT_RCV_PIF_A       ISP_STR_MON_PORT_PIFA2ISP
-#define ISP_STR_MON_PORT_SND_PIF_B       ISP_STR_MON_PORT_ISP2PIFB 
-#define ISP_STR_MON_PORT_RCV_PIF_B       ISP_STR_MON_PORT_PIFB2ISP
-#define ISP_STR_MON_PORT_SND_DMA         ISP_STR_MON_PORT_ISP2DMA  
-#define ISP_STR_MON_PORT_RCV_DMA         ISP_STR_MON_PORT_DMA2ISP 
-#define ISP_STR_MON_PORT_SND_GDC         ISP_STR_MON_PORT_ISP2GDC1 
-#define ISP_STR_MON_PORT_RCV_GDC         ISP_STR_MON_PORT_GDC12ISP
-#define ISP_STR_MON_PORT_SND_GPD         ISP_STR_MON_PORT_ISP2GPD 
-#define ISP_STR_MON_PORT_RCV_GPD         ISP_STR_MON_PORT_FA2ISP
-#define ISP_STR_MON_PORT_SND_SP          ISP_STR_MON_PORT_ISP2SP
-#define ISP_STR_MON_PORT_RCV_SP          ISP_STR_MON_PORT_SP2ISP
-                                           
-/* port definititions MOD streaming monitor, monitors the status of streaming ports at the module side of the streaming FIFO's */
-
-#define MOD_STR_MON_PORT_PIFA2CELLS       0
-#define MOD_STR_MON_PORT_CELLS2PIFA       1
-#define MOD_STR_MON_PORT_PIFB2CELLS       2
-#define MOD_STR_MON_PORT_CELLS2PIFB       3
-#define MOD_STR_MON_PORT_SIF2SP           4
-#define MOD_STR_MON_PORT_SP2SIF           5
-#define MOD_STR_MON_PORT_MC2SP            6
-#define MOD_STR_MON_PORT_SP2MC            7
-#define MOD_STR_MON_PORT_DMA2ISP          8
-#define MOD_STR_MON_PORT_ISP2DMA          9
-#define MOD_STR_MON_PORT_DMA2SP          10
-#define MOD_STR_MON_PORT_SP2DMA          11
-#define MOD_STR_MON_PORT_GDC12CELLS      12
-#define MOD_STR_MON_PORT_CELLS2GDC1      13
-#define MOD_STR_MON_PORT_GDC22CELLS      14
-#define MOD_STR_MON_PORT_CELLS2GDC2      15
-
-#define MOD_STR_MON_PORT_SND_PIF_A        0
-#define MOD_STR_MON_PORT_RCV_PIF_A        1
-#define MOD_STR_MON_PORT_SND_PIF_B        2
-#define MOD_STR_MON_PORT_RCV_PIF_B        3
-#define MOD_STR_MON_PORT_SND_SIF          4
-#define MOD_STR_MON_PORT_RCV_SIF          5
-#define MOD_STR_MON_PORT_SND_MC           6
-#define MOD_STR_MON_PORT_RCV_MC           7
-#define MOD_STR_MON_PORT_SND_DMA2ISP      8
-#define MOD_STR_MON_PORT_RCV_DMA_FR_ISP   9
-#define MOD_STR_MON_PORT_SND_DMA2SP      10
-#define MOD_STR_MON_PORT_RCV_DMA_FR_SP   11
-#define MOD_STR_MON_PORT_SND_GDC         12
-#define MOD_STR_MON_PORT_RCV_GDC         13
-
-
-/* testbench signals:       */
-
-/* testbench GP adapter register ids  */
-#define HIVE_TESTBENCH_GPIO_DATA_OUT_REG_IDX                    0
-#define HIVE_TESTBENCH_GPIO_DIR_OUT_REG_IDX                     1
-#define HIVE_TESTBENCH_IRQ_REG_IDX                              2
-#define HIVE_TESTBENCH_SDRAM_WAKEUP_REG_IDX                     3
-#define HIVE_TESTBENCH_IDLE_REG_IDX                             4
-#define HIVE_TESTBENCH_GPIO_DATA_IN_REG_IDX                     5
-#define HIVE_TESTBENCH_MIPI_BFM_EN_REG_IDX                      6
-#define HIVE_TESTBENCH_CSI_CONFIG_REG_IDX                       7
-#define HIVE_TESTBENCH_DDR_STALL_EN_REG_IDX                     8
-
-#define HIVE_TESTBENCH_ISP_PMEM_ERROR_IRQ_REG_IDX               9
-#define HIVE_TESTBENCH_ISP_BAMEM_ERROR_IRQ_REG_IDX             10
-#define HIVE_TESTBENCH_ISP_DMEM_ERROR_IRQ_REG_IDX              11
-#define HIVE_TESTBENCH_SP_ICACHE_MEM_ERROR_IRQ_REG_IDX         12
-#define HIVE_TESTBENCH_SP_DMEM_ERROR_IRQ_REG_IDX               13
-
-#define HIVE_TESTBENCH_MIPI_PARPATHEN_REG_IDX                  14
-#define HIVE_TESTBENCH_FB_HPLL_FREQ_REG_IDX                    15
-#define HIVE_TESTBENCH_ISCLK_RATIO_REG_IDX                     16
-
-/* Signal monitor input bit ids */
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_O_BIT_ID                0
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_1_BIT_ID                1
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_2_BIT_ID                2
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_3_BIT_ID                3
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_4_BIT_ID                4
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_5_BIT_ID                5
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_6_BIT_ID                6
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_7_BIT_ID                7
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_8_BIT_ID                8
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_9_BIT_ID                9
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_10_BIT_ID              10
-#define HIVE_TESTBENCH_SIG_MON_GPIO_PIN_11_BIT_ID              11
-#define HIVE_TESTBENCH_SIG_MON_IRQ_PIN_BIT_ID                  12
-#define HIVE_TESTBENCH_SIG_MON_SDRAM_WAKEUP_PIN_BIT_ID         13
-#define HIVE_TESTBENCH_SIG_MON_IDLE_PIN_BIT_ID                 14
-
-#define ISP2400_DEBUG_NETWORK    1
-
-#endif /* _hive_isp_css_defs_h__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_host_ids_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_host_ids_hrt.h
deleted file mode 100644
index 8d4c9d67c0e2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_host_ids_hrt.h
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _hive_isp_css_host_ids_hrt_h_
-#define _hive_isp_css_host_ids_hrt_h_
-
-/* ISP_CSS identifiers */
-#define INP_SYS       testbench_isp_isp_css_part_is_2400_inp_sys
-#define ISYS_GP_REGS  testbench_isp_isp_css_part_is_2400_inp_sys_gpreg
-#define ISYS_IRQ_CTRL testbench_isp_isp_css_part_is_2400_inp_sys_irq_ctrl
-#define ISYS_CAP_A    testbench_isp_isp_css_part_is_2400_inp_sys_capt_unit_a
-#define ISYS_CAP_B    testbench_isp_isp_css_part_is_2400_inp_sys_capt_unit_b
-#define ISYS_CAP_C    testbench_isp_isp_css_part_is_2400_inp_sys_capt_unit_c
-#define ISYS_INP_BUF  testbench_isp_isp_css_part_input_buffer
-#define ISYS_INP_CTRL testbench_isp_isp_css_part_is_2400_inp_sys_inp_ctrl
-#define ISYS_ACQ      testbench_isp_isp_css_part_is_2400_inp_sys_acq_unit
-
-#define ISP           testbench_isp_isp_css_sec_part_isp
-#define SP            testbench_isp_isp_css_sec_part_scp
-
-#define IF_PRIM       testbench_isp_isp_css_part_is_2400_ifmt_ift_prim  
-#define IF_PRIM_B     testbench_isp_isp_css_part_is_2400_ifmt_ift_prim_b
-#define IF_SEC        testbench_isp_isp_css_part_is_2400_ifmt_ift_sec
-#define IF_SEC_MASTER testbench_isp_isp_css_part_is_2400_ifmt_ift_sec_mt_out
-#define STR_TO_MEM    testbench_isp_isp_css_part_is_2400_ifmt_mem_cpy
-#define IFMT_GP_REGS  testbench_isp_isp_css_part_is_2400_ifmt_gp_reg
-#define IFMT_IRQ_CTRL testbench_isp_isp_css_part_is_2400_ifmt_irq_ctrl
-
-#define CSS_RECEIVER  testbench_isp_isp_css_part_is_2400_inp_sys_csi_receiver
-
-#define TC            testbench_isp_isp_css_sec_part_gpd_tc
-#define GPTIMER       testbench_isp_isp_css_sec_part_gpd_gptimer
-#define DMA           testbench_isp_isp_css_sec_part_isp_dma
-#define GDC           testbench_isp_isp_css_sec_part_gdc1
-#define GDC2          testbench_isp_isp_css_sec_part_gdc2
-#define IRQ_CTRL      testbench_isp_isp_css_sec_part_gpd_irq_ctrl
-#define GPIO          testbench_isp_isp_css_sec_part_gpd_c_gpio
-#define GP_REGS       testbench_isp_isp_css_sec_part_gpd_gp_reg
-#define ISEL_GP_REGS  testbench_isp_isp_css_part_is_2400_isel_gpr
-#define ISEL_IRQ_CTRL testbench_isp_isp_css_part_is_2400_isel_irq_ctrl
-#define DATA_MMU      testbench_isp_isp_css_sec_part_data_out_sys_c_mmu
-#define ICACHE_MMU    testbench_isp_isp_css_sec_part_icache_out_sys_c_mmu
-
-/* next is actually not FIFO but FIFO adapter, or slave to streaming adapter */
-#define ISP_SP_FIFO   testbench_isp_isp_css_sec_part_fa_sp_isp
-#define ISEL_FIFO     testbench_isp_isp_css_part_is_2400_isel_sf_fa_in
-
-#define FIFO_GPF_SP   testbench_isp_isp_css_sec_part_sf_fa2sp_in
-#define FIFO_GPF_ISP  testbench_isp_isp_css_sec_part_sf_fa2isp_in
-#define FIFO_SP_GPF   testbench_isp_isp_css_sec_part_sf_sp2fa_in
-#define FIFO_ISP_GPF  testbench_isp_isp_css_sec_part_sf_isp2fa_in
-
-#define DATA_OCP_MASTER    testbench_isp_isp_css_sec_part_data_out_sys_cio2ocp_wide_data_out_mt
-#define ICACHE_OCP_MASTER  testbench_isp_isp_css_sec_part_icache_out_sys_cio2ocp_wide_data_out_mt
-
-#define SP_IN_FIFO    testbench_isp_isp_css_sec_part_sf_fa2sp_in
-#define SP_OUT_FIFO   testbench_isp_isp_css_sec_part_sf_sp2fa_out
-#define ISP_IN_FIFO   testbench_isp_isp_css_sec_part_sf_fa2isp_in
-#define ISP_OUT_FIFO  testbench_isp_isp_css_sec_part_sf_isp2fa_out
-#define GEN_SHORT_PACK_PORT testbench_isp_isp_css_part_is_2400_inp_sys_csi_str_mon_fa_gensh_out
-
-/* input_system_2401 identifiers */
-#define ISYS2401_GP_REGS    testbench_isp_isp_css_part_is_2401_gpreg
-#define ISYS2401_DMA        testbench_isp_isp_css_part_is_2401_dma
-#define ISYS2401_IRQ_CTRL   testbench_isp_isp_css_part_is_2401_isys_irq_ctrl
-
-#define ISYS2401_CSI_RX_A     testbench_isp_isp_css_part_is_2401_is_pipe_a_csi_rx
-#define ISYS2401_MIPI_BE_A    testbench_isp_isp_css_part_is_2401_is_pipe_a_mipi_be
-#define ISYS2401_S2M_A        testbench_isp_isp_css_part_is_2401_is_pipe_a_s2m
-#define ISYS2401_PXG_A        testbench_isp_isp_css_part_is_2401_is_pipe_a_pxlgen
-#define ISYS2401_IBUF_CNTRL_A testbench_isp_isp_css_part_is_2401_is_pipe_a_ibuf_ctrl
-#define ISYS2401_IRQ_CTRL_A   testbench_isp_isp_css_part_is_2401_is_pipe_a_irq_ctrl_pipe
-
-#define ISYS2401_CSI_RX_B     testbench_isp_isp_css_part_is_2401_is_pipe_b_csi_rx
-#define ISYS2401_MIPI_BE_B    testbench_isp_isp_css_part_is_2401_is_pipe_b_mipi_be
-#define ISYS2401_S2M_B        testbench_isp_isp_css_part_is_2401_is_pipe_b_s2m
-#define ISYS2401_PXG_B        testbench_isp_isp_css_part_is_2401_is_pipe_b_pxlgen
-#define ISYS2401_IBUF_CNTRL_B testbench_isp_isp_css_part_is_2401_is_pipe_b_ibuf_ctrl
-#define ISYS2401_IRQ_CTRL_B   testbench_isp_isp_css_part_is_2401_is_pipe_b_irq_ctrl_pipe
-
-#define ISYS2401_CSI_RX_C     testbench_isp_isp_css_part_is_2401_is_pipe_c_csi_rx
-#define ISYS2401_MIPI_BE_C    testbench_isp_isp_css_part_is_2401_is_pipe_c_mipi_be
-#define ISYS2401_S2M_C        testbench_isp_isp_css_part_is_2401_is_pipe_c_s2m
-#define ISYS2401_PXG_C        testbench_isp_isp_css_part_is_2401_is_pipe_c_pxlgen
-#define ISYS2401_IBUF_CNTRL_C testbench_isp_isp_css_part_is_2401_is_pipe_c_ibuf_ctrl
-#define ISYS2401_IRQ_CTRL_C   testbench_isp_isp_css_part_is_2401_is_pipe_c_irq_ctrl_pipe
-
-
-/* Testbench identifiers */
-#define DDR             testbench_ddram
-#define DDR_SMALL       testbench_ddram_small
-#define XMEM            DDR
-#define GPIO_ADAPTER    testbench_gp_adapter
-#define SIG_MONITOR     testbench_sig_mon
-#define DDR_SLAVE       testbench_ddram_ip0
-#define DDR_SMALL_SLAVE testbench_ddram_small_ip0
-#define HOST_MASTER     host_op0
-
-#define CSI_SENSOR         testbench_vied_sensor
-#define CSI_SENSOR_GP_REGS testbench_vied_sensor_gpreg
-#define CSI_STR_IN_A       testbench_vied_sensor_tx_a_csi_tx_data_in
-#define CSI_STR_IN_B       testbench_vied_sensor_tx_b_csi_tx_data_in
-#define CSI_STR_IN_C       testbench_vied_sensor_tx_c_csi_tx_data_in
-#define CSI_SENSOR_TX_A    testbench_vied_sensor_tx_a
-#define CSI_SENSOR_TX_B    testbench_vied_sensor_tx_b
-#define CSI_SENSOR_TX_C    testbench_vied_sensor_tx_c
-
-#endif /* _hive_isp_css_host_ids_hrt_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
deleted file mode 100644
index b4211a0c631a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_isp_css_streaming_to_mipi_types_hrt.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _hive_isp_css_streaming_to_mipi_types_hrt_h_
-#define _hive_isp_css_streaming_to_mipi_types_hrt_h_
-
-#include <streaming_to_mipi_defs.h>
-
-#define _HIVE_ISP_CH_ID_MASK    ((1U << HIVE_ISP_CH_ID_BITS)-1)
-#define _HIVE_ISP_FMT_TYPE_MASK ((1U << HIVE_ISP_FMT_TYPE_BITS)-1)
-
-#define _HIVE_STR_TO_MIPI_FMT_TYPE_LSB (HIVE_STR_TO_MIPI_CH_ID_LSB + HIVE_ISP_CH_ID_BITS)
-#define _HIVE_STR_TO_MIPI_DATA_B_LSB   (HIVE_STR_TO_MIPI_DATA_A_LSB + HIVE_IF_PIXEL_WIDTH)
- 
-#endif /* _hive_isp_css_streaming_to_mipi_types_hrt_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_types.h
deleted file mode 100644
index 58b0e6effbd0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/hive_types.h
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_HIVE_TYPES_H 
-#define _HRT_HIVE_TYPES_H 
-
-#include "version.h"
-#include "defs.h"
-
-#ifndef HRTCAT3
-#define _HRTCAT3(m,n,o)     m##n##o
-#define HRTCAT3(m,n,o)      _HRTCAT3(m,n,o)
-#endif
-
-#ifndef HRTCAT4
-#define _HRTCAT4(m,n,o,p)     m##n##o##p
-#define HRTCAT4(m,n,o,p)      _HRTCAT4(m,n,o,p)
-#endif
-
-#ifndef HRTMIN
-#define HRTMIN(a,b) (((a)<(b))?(a):(b))
-#endif
-                                 
-#ifndef HRTMAX
-#define HRTMAX(a,b) (((a)>(b))?(a):(b))
-#endif
-
-/* boolean data type */
-typedef unsigned int hive_bool;
-#define hive_false 0
-#define hive_true  1
-
-typedef char                 hive_int8;
-typedef short                hive_int16;
-typedef int                  hive_int32;
-typedef long long            hive_int64;
-
-typedef unsigned char        hive_uint8;
-typedef unsigned short       hive_uint16;
-typedef unsigned int         hive_uint32;
-typedef unsigned long long   hive_uint64;
-
-/* by default assume 32 bit master port (both data and address) */
-#ifndef HRT_DATA_WIDTH
-#define HRT_DATA_WIDTH 32
-#endif
-#ifndef HRT_ADDRESS_WIDTH
-#define HRT_ADDRESS_WIDTH 32
-#endif
-
-#define HRT_DATA_BYTES    (HRT_DATA_WIDTH/8)
-#define HRT_ADDRESS_BYTES (HRT_ADDRESS_WIDTH/8)
-
-#if HRT_DATA_WIDTH == 64
-typedef hive_uint64 hrt_data;
-#elif HRT_DATA_WIDTH == 32
-typedef hive_uint32 hrt_data;
-#else
-#error data width not supported
-#endif
-
-#if HRT_ADDRESS_WIDTH == 64
-typedef hive_uint64 hrt_address; 
-#elif HRT_ADDRESS_WIDTH == 32
-typedef hive_uint32 hrt_address;
-#else
-#error adddres width not supported
-#endif
-
-/* The SP side representation of an HMM virtual address */
-typedef hive_uint32 hrt_vaddress;
-
-/* use 64 bit addresses in simulation, where possible */
-typedef hive_uint64  hive_sim_address;
-
-/* below is for csim, not for hrt, rename and move this elsewhere */
-
-typedef unsigned int hive_uint;
-typedef hive_uint32  hive_address;
-typedef hive_address hive_slave_address;
-typedef hive_address hive_mem_address;
-
-/* MMIO devices */
-typedef hive_uint    hive_mmio_id;
-typedef hive_mmio_id hive_slave_id;
-typedef hive_mmio_id hive_port_id;
-typedef hive_mmio_id hive_master_id; 
-typedef hive_mmio_id hive_mem_id;
-typedef hive_mmio_id hive_dev_id;
-typedef hive_mmio_id hive_fifo_id;
-
-typedef hive_uint      hive_hier_id;
-typedef hive_hier_id   hive_device_id;
-typedef hive_device_id hive_proc_id;
-typedef hive_device_id hive_cell_id;
-typedef hive_device_id hive_host_id;
-typedef hive_device_id hive_bus_id;
-typedef hive_device_id hive_bridge_id;
-typedef hive_device_id hive_fifo_adapter_id;
-typedef hive_device_id hive_custom_device_id;
-
-typedef hive_uint hive_slot_id;
-typedef hive_uint hive_fu_id;
-typedef hive_uint hive_reg_file_id;
-typedef hive_uint hive_reg_id;
-
-/* Streaming devices */
-typedef hive_uint hive_outport_id;
-typedef hive_uint hive_inport_id;
-
-typedef hive_uint hive_msink_id;
-
-/* HRT specific */
-typedef char* hive_program;
-typedef char* hive_function;
-
-#endif /* _HRT_HIVE_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/if_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/if_defs.h
deleted file mode 100644
index 7d39e45796ae..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/if_defs.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IF_DEFS_H
-#define _IF_DEFS_H
-
-#define HIVE_IF_FRAME_REQUEST        0xA000
-#define HIVE_IF_LINES_REQUEST        0xB000
-#define HIVE_IF_VECTORS_REQUEST      0xC000
-
-#endif /* _IF_DEFS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_formatter_subsystem_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_formatter_subsystem_defs.h
deleted file mode 100644
index 7766f78cd123..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_formatter_subsystem_defs.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _if_subsystem_defs_h__
-#define _if_subsystem_defs_h__
-
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_0            0
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_1            1
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_2            2
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_3            3
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_4            4
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_5            5
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_6            6
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_7            7 
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_FSYNC_LUT_REG        8
-#define HIVE_IFMT_GP_REGS_SRST_IDX                          9
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IDX                 10
-
-#define HIVE_IFMT_GP_REGS_CH_ID_FMT_TYPE_IDX               11
-
-#define HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_BASE         HIVE_IFMT_GP_REGS_INPUT_SWITCH_LUT_REG_0
-
-/* order of the input bits for the ifmt irq controller */
-#define HIVE_IFMT_IRQ_IFT_PRIM_BIT_ID                       0
-#define HIVE_IFMT_IRQ_IFT_PRIM_B_BIT_ID                     1
-#define HIVE_IFMT_IRQ_IFT_SEC_BIT_ID                        2
-#define HIVE_IFMT_IRQ_MEM_CPY_BIT_ID                        3
-#define HIVE_IFMT_IRQ_SIDEBAND_CHANGED_BIT_ID               4
-
-/* order of the input bits for the ifmt Soft reset register */
-#define HIVE_IFMT_GP_REGS_SRST_IFT_PRIM_BIT_IDX             0
-#define HIVE_IFMT_GP_REGS_SRST_IFT_PRIM_B_BIT_IDX           1
-#define HIVE_IFMT_GP_REGS_SRST_IFT_SEC_BIT_IDX              2
-#define HIVE_IFMT_GP_REGS_SRST_MEM_CPY_BIT_IDX              3
-
-/* order of the input bits for the ifmt Soft reset register */
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IFT_PRIM_BIT_IDX     0
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IFT_PRIM_B_BIT_IDX   1
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_IFT_SEC_BIT_IDX      2
-#define HIVE_IFMT_GP_REGS_SLV_REG_SRST_MEM_CPY_BIT_IDX      3
-
-#endif /* _if_subsystem_defs_h__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_selector_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_selector_defs.h
deleted file mode 100644
index 87fbf82edb5b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_selector_defs.h
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_selector_defs_h
-#define _input_selector_defs_h
-
-#ifndef HIVE_ISP_ISEL_SEL_BITS
-#define HIVE_ISP_ISEL_SEL_BITS                                  2
-#endif
-
-#ifndef HIVE_ISP_CH_ID_BITS
-#define HIVE_ISP_CH_ID_BITS                                     2
-#endif
-
-#ifndef HIVE_ISP_FMT_TYPE_BITS
-#define HIVE_ISP_FMT_TYPE_BITS                                  5
-#endif
-
-/* gp_register register id's -- Outputs */
-#define HIVE_ISEL_GP_REGS_SYNCGEN_ENABLE_IDX                    0
-#define HIVE_ISEL_GP_REGS_SYNCGEN_FREE_RUNNING_IDX              1
-#define HIVE_ISEL_GP_REGS_SYNCGEN_PAUSE_IDX                     2
-#define HIVE_ISEL_GP_REGS_SYNCGEN_NR_FRAMES_IDX                 3 
-#define HIVE_ISEL_GP_REGS_SYNCGEN_NR_PIX_IDX                    4      
-#define HIVE_ISEL_GP_REGS_SYNCGEN_NR_LINES_IDX                  5      
-#define HIVE_ISEL_GP_REGS_SYNCGEN_HBLANK_CYCLES_IDX             6      
-#define HIVE_ISEL_GP_REGS_SYNCGEN_VBLANK_CYCLES_IDX             7      
-
-#define HIVE_ISEL_GP_REGS_SOF_IDX                               8 
-#define HIVE_ISEL_GP_REGS_EOF_IDX                               9 
-#define HIVE_ISEL_GP_REGS_SOL_IDX                              10 
-#define HIVE_ISEL_GP_REGS_EOL_IDX                              11 
-
-#define HIVE_ISEL_GP_REGS_PRBS_ENABLE                          12      
-#define HIVE_ISEL_GP_REGS_PRBS_ENABLE_PORT_B                   13      
-#define HIVE_ISEL_GP_REGS_PRBS_LFSR_RESET_VALUE                14      
-
-#define HIVE_ISEL_GP_REGS_TPG_ENABLE                           15      
-#define HIVE_ISEL_GP_REGS_TPG_ENABLE_PORT_B                    16      
-#define HIVE_ISEL_GP_REGS_TPG_HOR_CNT_MASK_IDX                 17      
-#define HIVE_ISEL_GP_REGS_TPG_VER_CNT_MASK_IDX                 18      
-#define HIVE_ISEL_GP_REGS_TPG_XY_CNT_MASK_IDX                  19      
-#define HIVE_ISEL_GP_REGS_TPG_HOR_CNT_DELTA_IDX                20      
-#define HIVE_ISEL_GP_REGS_TPG_VER_CNT_DELTA_IDX                21      
-#define HIVE_ISEL_GP_REGS_TPG_MODE_IDX                         22     
-#define HIVE_ISEL_GP_REGS_TPG_R1_IDX                           23 
-#define HIVE_ISEL_GP_REGS_TPG_G1_IDX                           24
-#define HIVE_ISEL_GP_REGS_TPG_B1_IDX                           25
-#define HIVE_ISEL_GP_REGS_TPG_R2_IDX                           26
-#define HIVE_ISEL_GP_REGS_TPG_G2_IDX                           27
-#define HIVE_ISEL_GP_REGS_TPG_B2_IDX                           28
-
-
-#define HIVE_ISEL_GP_REGS_CH_ID_IDX                            29
-#define HIVE_ISEL_GP_REGS_FMT_TYPE_IDX                         30
-#define HIVE_ISEL_GP_REGS_DATA_SEL_IDX                         31
-#define HIVE_ISEL_GP_REGS_SBAND_SEL_IDX                        32
-#define HIVE_ISEL_GP_REGS_SYNC_SEL_IDX                         33
-#define HIVE_ISEL_GP_REGS_SRST_IDX                             37
-
-#define HIVE_ISEL_GP_REGS_SRST_SYNCGEN_BIT                      0
-#define HIVE_ISEL_GP_REGS_SRST_PRBS_BIT                         1
-#define HIVE_ISEL_GP_REGS_SRST_TPG_BIT                          2
-#define HIVE_ISEL_GP_REGS_SRST_FIFO_BIT                         3
-
-/* gp_register register id's -- Inputs   */
-#define HIVE_ISEL_GP_REGS_SYNCGEN_HOR_CNT_IDX                  34
-#define HIVE_ISEL_GP_REGS_SYNCGEN_VER_CNT_IDX                  35
-#define HIVE_ISEL_GP_REGS_SYNCGEN_FRAMES_CNT_IDX               36
-
-/* irq sources isel irq controller */
-#define HIVE_ISEL_IRQ_SYNC_GEN_SOF_BIT_ID                       0
-#define HIVE_ISEL_IRQ_SYNC_GEN_EOF_BIT_ID                       1
-#define HIVE_ISEL_IRQ_SYNC_GEN_SOL_BIT_ID                       2
-#define HIVE_ISEL_IRQ_SYNC_GEN_EOL_BIT_ID                       3
-#define HIVE_ISEL_IRQ_NUM_IRQS                                  4
-
-#endif /* _input_selector_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_switch_2400_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_switch_2400_defs.h
deleted file mode 100644
index 20a13c4cdb56..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_switch_2400_defs.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_switch_2400_defs_h
-#define _input_switch_2400_defs_h
-
-#define _HIVE_INPUT_SWITCH_GET_LUT_REG_ID(ch_id, fmt_type) (((ch_id)*2) + ((fmt_type)>=16))
-#define _HIVE_INPUT_SWITCH_GET_LUT_REG_LSB(fmt_type)        (((fmt_type)%16) * 2)
-
-#define HIVE_INPUT_SWITCH_SELECT_NO_OUTPUT   0
-#define HIVE_INPUT_SWITCH_SELECT_IF_PRIM     1
-#define HIVE_INPUT_SWITCH_SELECT_IF_SEC      2
-#define HIVE_INPUT_SWITCH_SELECT_STR_TO_MEM  3
-#define HIVE_INPUT_SWITCH_VSELECT_NO_OUTPUT  0
-#define HIVE_INPUT_SWITCH_VSELECT_IF_PRIM    1
-#define HIVE_INPUT_SWITCH_VSELECT_IF_SEC     2
-#define HIVE_INPUT_SWITCH_VSELECT_STR_TO_MEM 4
-
-#endif /* _input_switch_2400_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_system_ctrl_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_system_ctrl_defs.h
deleted file mode 100644
index a7f0ca80bc9b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_system_ctrl_defs.h
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_system_ctrl_defs_h
-#define _input_system_ctrl_defs_h
-
-#define _INPUT_SYSTEM_CTRL_REG_ALIGN                    4  /* assuming 32 bit control bus width */
-
-/* --------------------------------------------------*/
-
-/* --------------------------------------------------*/
-/* REGISTER INFO */
-/* --------------------------------------------------*/
-
-// Number of registers
-#define ISYS_CTRL_NOF_REGS                              23
-
-// Register id's of MMIO slave accesible registers
-#define ISYS_CTRL_CAPT_START_ADDR_A_REG_ID              0
-#define ISYS_CTRL_CAPT_START_ADDR_B_REG_ID              1
-#define ISYS_CTRL_CAPT_START_ADDR_C_REG_ID              2
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_A_REG_ID         3
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_B_REG_ID         4
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_C_REG_ID         5
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_A_REG_ID         6
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_B_REG_ID         7
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_C_REG_ID         8
-#define ISYS_CTRL_ACQ_START_ADDR_REG_ID                 9
-#define ISYS_CTRL_ACQ_MEM_REGION_SIZE_REG_ID            10
-#define ISYS_CTRL_ACQ_NUM_MEM_REGIONS_REG_ID            11
-#define ISYS_CTRL_INIT_REG_ID                           12
-#define ISYS_CTRL_LAST_COMMAND_REG_ID                   13
-#define ISYS_CTRL_NEXT_COMMAND_REG_ID                   14
-#define ISYS_CTRL_LAST_ACKNOWLEDGE_REG_ID               15
-#define ISYS_CTRL_NEXT_ACKNOWLEDGE_REG_ID               16
-#define ISYS_CTRL_FSM_STATE_INFO_REG_ID                 17
-#define ISYS_CTRL_CAPT_A_FSM_STATE_INFO_REG_ID          18
-#define ISYS_CTRL_CAPT_B_FSM_STATE_INFO_REG_ID          19
-#define ISYS_CTRL_CAPT_C_FSM_STATE_INFO_REG_ID          20
-#define ISYS_CTRL_ACQ_FSM_STATE_INFO_REG_ID             21
-#define ISYS_CTRL_CAPT_RESERVE_ONE_MEM_REGION_REG_ID    22
- 
-
-/* register reset value */
-#define ISYS_CTRL_CAPT_START_ADDR_A_REG_RSTVAL           0
-#define ISYS_CTRL_CAPT_START_ADDR_B_REG_RSTVAL           0
-#define ISYS_CTRL_CAPT_START_ADDR_C_REG_RSTVAL           0
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_A_REG_RSTVAL      128
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_B_REG_RSTVAL      128
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_C_REG_RSTVAL      128
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_A_REG_RSTVAL      3 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_B_REG_RSTVAL      3 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_C_REG_RSTVAL      3 
-#define ISYS_CTRL_ACQ_START_ADDR_REG_RSTVAL              0
-#define ISYS_CTRL_ACQ_MEM_REGION_SIZE_REG_RSTVAL         128 
-#define ISYS_CTRL_ACQ_NUM_MEM_REGIONS_REG_RSTVAL         3 
-#define ISYS_CTRL_INIT_REG_RSTVAL                        0
-#define ISYS_CTRL_LAST_COMMAND_REG_RSTVAL                15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)  
-#define ISYS_CTRL_NEXT_COMMAND_REG_RSTVAL                15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)
-#define ISYS_CTRL_LAST_ACKNOWLEDGE_REG_RSTVAL            15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)
-#define ISYS_CTRL_NEXT_ACKNOWLEDGE_REG_RSTVAL            15    //0x0000_000F (to signal non-valid cmd/ack after reset/soft-reset)
-#define ISYS_CTRL_FSM_STATE_INFO_REG_RSTVAL              0
-#define ISYS_CTRL_CAPT_A_FSM_STATE_INFO_REG_RSTVAL       0 
-#define ISYS_CTRL_CAPT_B_FSM_STATE_INFO_REG_RSTVAL       0
-#define ISYS_CTRL_CAPT_C_FSM_STATE_INFO_REG_RSTVAL       0
-#define ISYS_CTRL_ACQ_FSM_STATE_INFO_REG_RSTVAL          0
-#define ISYS_CTRL_CAPT_RESERVE_ONE_MEM_REGION_REG_RSTVAL 0
-
-/* register width value */
-#define ISYS_CTRL_CAPT_START_ADDR_A_REG_WIDTH            9 
-#define ISYS_CTRL_CAPT_START_ADDR_B_REG_WIDTH            9 
-#define ISYS_CTRL_CAPT_START_ADDR_C_REG_WIDTH            9 
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_A_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_B_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_MEM_REGION_SIZE_C_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_A_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_B_REG_WIDTH       9 
-#define ISYS_CTRL_CAPT_NUM_MEM_REGIONS_C_REG_WIDTH       9 
-#define ISYS_CTRL_ACQ_START_ADDR_REG_WIDTH               9 
-#define ISYS_CTRL_ACQ_MEM_REGION_SIZE_REG_WIDTH          9 
-#define ISYS_CTRL_ACQ_NUM_MEM_REGIONS_REG_WIDTH          9 
-#define ISYS_CTRL_INIT_REG_WIDTH                         3 
-#define ISYS_CTRL_LAST_COMMAND_REG_WIDTH                 32    /* slave data width */
-#define ISYS_CTRL_NEXT_COMMAND_REG_WIDTH                 32
-#define ISYS_CTRL_LAST_ACKNOWLEDGE_REG_WIDTH             32
-#define ISYS_CTRL_NEXT_ACKNOWLEDGE_REG_WIDTH             32
-#define ISYS_CTRL_FSM_STATE_INFO_REG_WIDTH               32
-#define ISYS_CTRL_CAPT_A_FSM_STATE_INFO_REG_WIDTH        32
-#define ISYS_CTRL_CAPT_B_FSM_STATE_INFO_REG_WIDTH        32
-#define ISYS_CTRL_CAPT_C_FSM_STATE_INFO_REG_WIDTH        32
-#define ISYS_CTRL_ACQ_FSM_STATE_INFO_REG_WIDTH           32
-#define ISYS_CTRL_CAPT_RESERVE_ONE_MEM_REGION_REG_WIDTH  1
-
-/* bit definitions */
-
-/* --------------------------------------------------*/
-/* TOKEN INFO */
-/* --------------------------------------------------*/
-
-/*
-InpSysCaptFramesAcq  1/0  [3:0] - 'b0000
-[7:4] - CaptPortId,
-           CaptA-'b0000
-           CaptB-'b0001
-           CaptC-'b0010
-[31:16] - NOF_frames
-InpSysCaptFrameExt  2/0  [3:0] - 'b0001'
-[7:4] - CaptPortId,
-           'b0000 - CaptA 
-           'b0001 - CaptB
-           'b0010 - CaptC
-
-  2/1  [31:0] - external capture address
-InpSysAcqFrame  2/0  [3:0] - 'b0010, 
-[31:4] - NOF_ext_mem_words
-  2/1  [31:0] - external memory read start address
-InpSysOverruleON  1/0  [3:0] - 'b0011, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysOverruleOFF  1/0  [3:0] - 'b0100, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysOverruleCmd  2/0  [3:0] - 'b0101, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-  2/1  [31:0] - command token value for port opid
-
-
-acknowledge tokens:
-
-InpSysAckCFA  1/0   [3:0] - 'b0000
- [7:4] - CaptPortId,
-           CaptA-'b0000
-           CaptB- 'b0001
-           CaptC-'b0010
- [31:16] - NOF_frames
-InpSysAckCFE  1/0  [3:0] - 'b0001'
-[7:4] - CaptPortId,
-           'b0000 - CaptA 
-           'b0001 - CaptB
-           'b0010 - CaptC
-
-InpSysAckAF  1/0  [3:0] - 'b0010
-InpSysAckOverruleON  1/0  [3:0] - 'b0011, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysAckOverruleOFF  1/0  [3:0] - 'b0100, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-InpSysAckOverrule  2/0  [3:0] - 'b0101, 
-[7:4] - overrule port id (opid)
-           'b0000 - CaptA
-           'b0001 - CaptB
-           'b0010 - CaptC
-           'b0011 - Acq
-           'b0100 - DMA
-
-
-  2/1  [31:0] - acknowledge token value from port opid
-
-
-
-*/
-
-
-/* Command and acknowledge tokens IDs */
-#define ISYS_CTRL_CAPT_FRAMES_ACQ_TOKEN_ID        0 /* 0000b */
-#define ISYS_CTRL_CAPT_FRAME_EXT_TOKEN_ID         1 /* 0001b */
-#define ISYS_CTRL_ACQ_FRAME_TOKEN_ID              2 /* 0010b */
-#define ISYS_CTRL_OVERRULE_ON_TOKEN_ID            3 /* 0011b */
-#define ISYS_CTRL_OVERRULE_OFF_TOKEN_ID           4 /* 0100b */
-#define ISYS_CTRL_OVERRULE_TOKEN_ID               5 /* 0101b */
-
-#define ISYS_CTRL_ACK_CFA_TOKEN_ID                0
-#define ISYS_CTRL_ACK_CFE_TOKEN_ID                1
-#define ISYS_CTRL_ACK_AF_TOKEN_ID                 2
-#define ISYS_CTRL_ACK_OVERRULE_ON_TOKEN_ID        3
-#define ISYS_CTRL_ACK_OVERRULE_OFF_TOKEN_ID       4
-#define ISYS_CTRL_ACK_OVERRULE_TOKEN_ID           5
-#define ISYS_CTRL_ACK_DEVICE_ERROR_TOKEN_ID       6
-
-#define ISYS_CTRL_TOKEN_ID_MSB                    3
-#define ISYS_CTRL_TOKEN_ID_LSB                    0
-#define ISYS_CTRL_PORT_ID_TOKEN_MSB               7
-#define ISYS_CTRL_PORT_ID_TOKEN_LSB               4
-#define ISYS_CTRL_NOF_CAPT_TOKEN_MSB              31
-#define ISYS_CTRL_NOF_CAPT_TOKEN_LSB              16
-#define ISYS_CTRL_NOF_EXT_TOKEN_MSB               31
-#define ISYS_CTRL_NOF_EXT_TOKEN_LSB               8
-
-#define ISYS_CTRL_TOKEN_ID_IDX                    0
-#define ISYS_CTRL_TOKEN_ID_BITS                   (ISYS_CTRL_TOKEN_ID_MSB - ISYS_CTRL_TOKEN_ID_LSB + 1)
-#define ISYS_CTRL_PORT_ID_IDX                     (ISYS_CTRL_TOKEN_ID_IDX + ISYS_CTRL_TOKEN_ID_BITS)
-#define ISYS_CTRL_PORT_ID_BITS                    (ISYS_CTRL_PORT_ID_TOKEN_MSB - ISYS_CTRL_PORT_ID_TOKEN_LSB +1)
-#define ISYS_CTRL_NOF_CAPT_IDX                    ISYS_CTRL_NOF_CAPT_TOKEN_LSB    
-#define ISYS_CTRL_NOF_CAPT_BITS                   (ISYS_CTRL_NOF_CAPT_TOKEN_MSB - ISYS_CTRL_NOF_CAPT_TOKEN_LSB + 1)
-#define ISYS_CTRL_NOF_EXT_IDX                     ISYS_CTRL_NOF_EXT_TOKEN_LSB    
-#define ISYS_CTRL_NOF_EXT_BITS                    (ISYS_CTRL_NOF_EXT_TOKEN_MSB - ISYS_CTRL_NOF_EXT_TOKEN_LSB + 1)
-
-#define ISYS_CTRL_PORT_ID_CAPT_A                  0 /* device ID for capture unit A      */
-#define ISYS_CTRL_PORT_ID_CAPT_B                  1 /* device ID for capture unit B      */
-#define ISYS_CTRL_PORT_ID_CAPT_C                  2 /* device ID for capture unit C      */
-#define ISYS_CTRL_PORT_ID_ACQUISITION             3 /* device ID for acquistion unit     */
-#define ISYS_CTRL_PORT_ID_DMA_CAPT_A              4 /* device ID for dma unit            */
-#define ISYS_CTRL_PORT_ID_DMA_CAPT_B              5 /* device ID for dma unit            */
-#define ISYS_CTRL_PORT_ID_DMA_CAPT_C              6 /* device ID for dma unit            */
-#define ISYS_CTRL_PORT_ID_DMA_ACQ                 7 /* device ID for dma unit            */
-
-#define ISYS_CTRL_NO_ACQ_ACK                      16 /* no ack from acquisition unit */
-#define ISYS_CTRL_NO_DMA_ACK                      0 
-#define ISYS_CTRL_NO_CAPT_ACK                     16
-
-#endif /* _input_system_ctrl_defs_h */ 
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_system_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_system_defs.h
deleted file mode 100644
index ae62163034a6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/input_system_defs.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _input_system_defs_h
-#define _input_system_defs_h
-
-/* csi controller modes */
-#define HIVE_CSI_CONFIG_MAIN                   0
-#define HIVE_CSI_CONFIG_STEREO1                4
-#define HIVE_CSI_CONFIG_STEREO2                8
-
-/* general purpose register IDs */
-
-/* Stream Multicast select modes */
-#define HIVE_ISYS_GPREG_MULTICAST_A_IDX           0
-#define HIVE_ISYS_GPREG_MULTICAST_B_IDX           1
-#define HIVE_ISYS_GPREG_MULTICAST_C_IDX           2
-
-/* Stream Mux select modes */
-#define HIVE_ISYS_GPREG_MUX_IDX                   3
-
-/* streaming monitor status and control */
-#define HIVE_ISYS_GPREG_STRMON_STAT_IDX           4
-#define HIVE_ISYS_GPREG_STRMON_COND_IDX           5
-#define HIVE_ISYS_GPREG_STRMON_IRQ_EN_IDX         6
-#define HIVE_ISYS_GPREG_SRST_IDX                  7
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_IDX          8
-#define HIVE_ISYS_GPREG_REG_PORT_A_IDX            9
-#define HIVE_ISYS_GPREG_REG_PORT_B_IDX            10
-
-/* Bit numbers of the soft reset register */
-#define HIVE_ISYS_GPREG_SRST_CAPT_FIFO_A_BIT      0
-#define HIVE_ISYS_GPREG_SRST_CAPT_FIFO_B_BIT      1
-#define HIVE_ISYS_GPREG_SRST_CAPT_FIFO_C_BIT      2
-#define HIVE_ISYS_GPREG_SRST_MULTICAST_A_BIT      3
-#define HIVE_ISYS_GPREG_SRST_MULTICAST_B_BIT      4
-#define HIVE_ISYS_GPREG_SRST_MULTICAST_C_BIT      5
-#define HIVE_ISYS_GPREG_SRST_CAPT_A_BIT           6
-#define HIVE_ISYS_GPREG_SRST_CAPT_B_BIT           7
-#define HIVE_ISYS_GPREG_SRST_CAPT_C_BIT           8
-#define HIVE_ISYS_GPREG_SRST_ACQ_BIT              9
-/* For ISYS_CTRL 5bits are defined to allow soft-reset per sub-controller and top-ctrl */
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_BIT        10  /*LSB for 5bit vector */
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_CAPT_A_BIT 10
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_CAPT_B_BIT 11
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_CAPT_C_BIT 12
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_ACQ_BIT    13
-#define HIVE_ISYS_GPREG_SRST_ISYS_CTRL_TOP_BIT    14
-/* -- */
-#define HIVE_ISYS_GPREG_SRST_STR_MUX_BIT          15
-#define HIVE_ISYS_GPREG_SRST_CIO2AHB_BIT          16
-#define HIVE_ISYS_GPREG_SRST_GEN_SHORT_FIFO_BIT   17
-#define HIVE_ISYS_GPREG_SRST_WIDE_BUS_BIT         18 // includes CIO conv
-#define HIVE_ISYS_GPREG_SRST_DMA_BIT              19
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_CAPT_A_BIT   20
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_CAPT_B_BIT   21
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_CAPT_C_BIT   22
-#define HIVE_ISYS_GPREG_SRST_SF_CTRL_ACQ_BIT      23
-#define HIVE_ISYS_GPREG_SRST_CSI_BE_OUT_BIT       24
-
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_CAPT_A_BIT    0
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_CAPT_B_BIT    1
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_CAPT_C_BIT    2
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_ACQ_BIT       3
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_DMA_BIT        4
-#define HIVE_ISYS_GPREG_SLV_REG_SRST_ISYS_CTRL_BIT  5
-
-/* streaming monitor port id's */
-#define HIVE_ISYS_STR_MON_PORT_CAPA            0
-#define HIVE_ISYS_STR_MON_PORT_CAPB            1
-#define HIVE_ISYS_STR_MON_PORT_CAPC            2
-#define HIVE_ISYS_STR_MON_PORT_ACQ             3
-#define HIVE_ISYS_STR_MON_PORT_CSS_GENSH       4
-#define HIVE_ISYS_STR_MON_PORT_SF_GENSH        5
-#define HIVE_ISYS_STR_MON_PORT_SP2ISYS         6
-#define HIVE_ISYS_STR_MON_PORT_ISYS2SP         7
-#define HIVE_ISYS_STR_MON_PORT_PIXA            8
-#define HIVE_ISYS_STR_MON_PORT_PIXB            9
-
-/* interrupt bit ID's        */
-#define HIVE_ISYS_IRQ_CSI_SOF_BIT_ID           0
-#define HIVE_ISYS_IRQ_CSI_EOF_BIT_ID           1
-#define HIVE_ISYS_IRQ_CSI_SOL_BIT_ID           2
-#define HIVE_ISYS_IRQ_CSI_EOL_BIT_ID           3
-#define HIVE_ISYS_IRQ_CSI_RECEIVER_BIT_ID      4
-#define HIVE_ISYS_IRQ_CSI_RECEIVER_BE_BIT_ID   5
-#define HIVE_ISYS_IRQ_CAP_UNIT_A_NO_SOP        6
-#define HIVE_ISYS_IRQ_CAP_UNIT_A_LATE_SOP      7
-/*#define HIVE_ISYS_IRQ_CAP_UNIT_A_UNDEF_PH      7*/
-#define HIVE_ISYS_IRQ_CAP_UNIT_B_NO_SOP        8
-#define HIVE_ISYS_IRQ_CAP_UNIT_B_LATE_SOP      9
-/*#define HIVE_ISYS_IRQ_CAP_UNIT_B_UNDEF_PH     10*/
-#define HIVE_ISYS_IRQ_CAP_UNIT_C_NO_SOP       10
-#define HIVE_ISYS_IRQ_CAP_UNIT_C_LATE_SOP     11
-/*#define HIVE_ISYS_IRQ_CAP_UNIT_C_UNDEF_PH     13*/
-#define HIVE_ISYS_IRQ_ACQ_UNIT_SOP_MISMATCH   12
-/*#define HIVE_ISYS_IRQ_ACQ_UNIT_UNDEF_PH       15*/
-#define HIVE_ISYS_IRQ_INP_CTRL_CAPA           13
-#define HIVE_ISYS_IRQ_INP_CTRL_CAPB           14
-#define HIVE_ISYS_IRQ_INP_CTRL_CAPC           15
-#define HIVE_ISYS_IRQ_CIO2AHB                 16
-#define HIVE_ISYS_IRQ_DMA_BIT_ID              17
-#define HIVE_ISYS_IRQ_STREAM_MON_BIT_ID       18
-#define HIVE_ISYS_IRQ_NUM_BITS                19
-
-/* DMA */
-#define HIVE_ISYS_DMA_CHANNEL                  0
-#define HIVE_ISYS_DMA_IBUF_DDR_CONN            0
-#define HIVE_ISYS_DMA_HEIGHT                   1
-#define HIVE_ISYS_DMA_ELEMS                    1 /* both master buses of same width */
-#define HIVE_ISYS_DMA_STRIDE                   0 /* no stride required as height is fixed to 1 */
-#define HIVE_ISYS_DMA_CROP                     0 /* no cropping */
-#define HIVE_ISYS_DMA_EXTENSION                0 /* no extension as elem width is same on both side */
-
-#endif /* _input_system_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/irq_controller_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/irq_controller_defs.h
deleted file mode 100644
index ec6dd4487158..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/irq_controller_defs.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _irq_controller_defs_h
-#define _irq_controller_defs_h
-
-#define _HRT_IRQ_CONTROLLER_EDGE_REG_IDX           0
-#define _HRT_IRQ_CONTROLLER_MASK_REG_IDX           1
-#define _HRT_IRQ_CONTROLLER_STATUS_REG_IDX         2
-#define _HRT_IRQ_CONTROLLER_CLEAR_REG_IDX          3
-#define _HRT_IRQ_CONTROLLER_ENABLE_REG_IDX         4
-#define _HRT_IRQ_CONTROLLER_EDGE_NOT_PULSE_REG_IDX 5
-#define _HRT_IRQ_CONTROLLER_STR_OUT_ENABLE_REG_IDX 6
-
-#define _HRT_IRQ_CONTROLLER_REG_ALIGN 4
-
-#endif /* _irq_controller_defs_h */   
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp2400_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp2400_support.h
deleted file mode 100644
index e00bc841d0f0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp2400_support.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _isp2400_support_h
-#define _isp2400_support_h
-
-#ifndef ISP2400_VECTOR_TYPES
-/* This typedef is to be able to include hive header files
-   in the host code which is useful in crun */
-typedef char *tmemvectors, *tmemvectoru, *tvector;
-#endif
-
-#define hrt_isp_vamem1_store_16(cell, addr, val) hrt_mem_store_16(cell, HRT_PROC_TYPE_PROP(cell, _simd_vamem1), addr, val)
-#define hrt_isp_vamem2_store_16(cell, addr, val) hrt_mem_store_16(cell, HRT_PROC_TYPE_PROP(cell, _simd_vamem2), addr, val)
-
-#define hrt_isp_dmem(cell) HRT_PROC_TYPE_PROP(cell, _base_dmem)
-#define hrt_isp_vmem(cell) HRT_PROC_TYPE_PROP(cell, _simd_vmem)
-
-#define hrt_isp_dmem_master_port_address(cell) hrt_mem_master_port_address(cell, hrt_isp_dmem(cell))
-#define hrt_isp_vmem_master_port_address(cell) hrt_mem_master_port_address(cell, hrt_isp_vmem(cell))
-
-#if ISP_HAS_HIST
-  #define hrt_isp_hist(cell) HRT_PROC_TYPE_PROP(cell, _simd_histogram)
-  #define hrt_isp_hist_master_port_address(cell) hrt_mem_master_port_address(cell, hrt_isp_hist(cell))
-#endif
-
-#endif /* _isp2400_support_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp2401_mamoiada_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp2401_mamoiada_params.h
deleted file mode 100644
index 033e23bcf672..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp2401_mamoiada_params.h
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Version */
-#define RTL_VERSION
-
-/* Cell name  */
-#define ISP_CELL_TYPE                          isp2401_mamoiada
-#define ISP_VMEM                               simd_vmem
-#define _HRT_ISP_VMEM                          isp2401_mamoiada_simd_vmem
-
-/* instruction pipeline depth */
-#define ISP_BRANCHDELAY                        5
-
-/* bus */
-#define ISP_BUS_WIDTH                          32
-#define ISP_BUS_ADDR_WIDTH                     32
-#define ISP_BUS_BURST_SIZE                     1
-
-/* data-path */
-#define ISP_SCALAR_WIDTH                       32
-#define ISP_SLICE_NELEMS                       4
-#define ISP_VEC_NELEMS                         64
-#define ISP_VEC_ELEMBITS                       14
-#define ISP_VEC_ELEM8BITS                      16
-#define ISP_CLONE_DATAPATH_IS_16               1
-
-/* memories */
-#define ISP_DMEM_DEPTH                         4096
-#define ISP_DMEM_BSEL_DOWNSAMPLE               8
-#define ISP_VMEM_DEPTH                         3072
-#define ISP_VMEM_BSEL_DOWNSAMPLE               8
-#define ISP_VMEM_ELEMBITS                      14
-#define ISP_VMEM_ELEM_PRECISION                14
-#define ISP_VMEM_IS_BAMEM                      1
-#if ISP_VMEM_IS_BAMEM
-  #define ISP_VMEM_BAMEM_MAX_BOI_HEIGHT        8
-  #define ISP_VMEM_BAMEM_LATENCY               5
-  #define ISP_VMEM_BAMEM_BANK_NARROWING_FACTOR 2
-  #define ISP_VMEM_BAMEM_NR_DATA_PLANES        8
-  #define ISP_VMEM_BAMEM_NR_CFG_REGISTERS      16
-  #define ISP_VMEM_BAMEM_LININT                0
-  #define ISP_VMEM_BAMEM_DAP_BITS              3
-  #define ISP_VMEM_BAMEM_LININT_FRAC_BITS      0
-  #define ISP_VMEM_BAMEM_PID_BITS              3
-  #define ISP_VMEM_BAMEM_OFFSET_BITS           19
-  #define ISP_VMEM_BAMEM_ADDRESS_BITS          25
-  #define ISP_VMEM_BAMEM_RID_BITS              4
-  #define ISP_VMEM_BAMEM_TRANSPOSITION         1
-  #define ISP_VMEM_BAMEM_VEC_PLUS_SLICE        1
-  #define ISP_VMEM_BAMEM_ARB_SERVICE_CYCLE_BITS 1
-  #define ISP_VMEM_BAMEM_LUT_ELEMS             16
-  #define ISP_VMEM_BAMEM_LUT_ADDR_WIDTH        14
-  #define ISP_VMEM_BAMEM_HALF_BLOCK_WRITE      1
-  #define ISP_VMEM_BAMEM_SMART_FETCH           1
-  #define ISP_VMEM_BAMEM_BIG_ENDIANNESS        0
-#endif /* ISP_VMEM_IS_BAMEM */
-#define ISP_PMEM_DEPTH                         2048
-#define ISP_PMEM_WIDTH                         640
-#define ISP_VAMEM_ADDRESS_BITS                 12
-#define ISP_VAMEM_ELEMBITS                     12
-#define ISP_VAMEM_DEPTH                        2048
-#define ISP_VAMEM_ALIGNMENT                    2
-#define ISP_VA_ADDRESS_WIDTH                   896
-#define ISP_VEC_VALSU_LATENCY                  ISP_VEC_NELEMS
-#define ISP_HIST_ADDRESS_BITS                  12
-#define ISP_HIST_ALIGNMENT                     4
-#define ISP_HIST_COMP_IN_PREC                  12
-#define ISP_HIST_DEPTH                         1024
-#define ISP_HIST_WIDTH                         24
-#define ISP_HIST_COMPONENTS                    4
-
-/* program counter */
-#define ISP_PC_WIDTH                           13
-
-/* Template switches */
-#define ISP_SHIELD_INPUT_DMEM                  0
-#define ISP_SHIELD_OUTPUT_DMEM                 1
-#define ISP_SHIELD_INPUT_VMEM                  0
-#define ISP_SHIELD_OUTPUT_VMEM                 0
-#define ISP_SHIELD_INPUT_PMEM                  1
-#define ISP_SHIELD_OUTPUT_PMEM                 1
-#define ISP_SHIELD_INPUT_HIST                  1
-#define ISP_SHIELD_OUTPUT_HIST                 1
-/* When LUT is select the shielding is always on */
-#define ISP_SHIELD_INPUT_VAMEM                 1
-#define ISP_SHIELD_OUTPUT_VAMEM                1
-
-#define ISP_HAS_IRQ                            1
-#define ISP_HAS_SOFT_RESET                     1
-#define ISP_HAS_VEC_DIV                        0
-#define ISP_HAS_VFU_W_2O                       1
-#define ISP_HAS_DEINT3                         1
-#define ISP_HAS_LUT                            1
-#define ISP_HAS_HIST                           1
-#define ISP_HAS_VALSU                          1
-#define ISP_HAS_3rdVALSU                       1
-#define ISP_VRF1_HAS_2P                        1
-
-#define ISP_SRU_GUARDING                       1
-#define ISP_VLSU_GUARDING                      1
-
-#define ISP_VRF_RAM     	                     1
-#define ISP_SRF_RAM     	                     1
-
-#define ISP_SPLIT_VMUL_VADD_IS                 0
-#define ISP_RFSPLIT_FPGA                       0
-
-/* RSN or Bus pipelining */
-#define ISP_RSN_PIPE                           1
-#define ISP_VSF_BUS_PIPE                       0
-
-/* extra slave port to vmem */
-#define ISP_IF_VMEM                            0
-#define ISP_GDC_VMEM                           0
-
-/* Streaming ports */
-#define ISP_IF                                 1
-#define ISP_IF_B                               1
-#define ISP_GDC                                1
-#define ISP_SCL                                1
-#define ISP_GPFIFO                             1
-#define ISP_SP                                 1
-
-/* Removing Issue Slot(s) */
-#define ISP_HAS_NOT_SIMD_IS2                   0
-#define ISP_HAS_NOT_SIMD_IS3                   0
-#define ISP_HAS_NOT_SIMD_IS4                   0
-#define ISP_HAS_NOT_SIMD_IS4_VADD              0
-#define ISP_HAS_NOT_SIMD_IS5                   0
-#define ISP_HAS_NOT_SIMD_IS6                   0
-#define ISP_HAS_NOT_SIMD_IS7                   0
-#define ISP_HAS_NOT_SIMD_IS8                   0
-
-/* ICache  */
-#define ISP_ICACHE                             1
-#define ISP_ICACHE_ONLY                        0
-#define ISP_ICACHE_PREFETCH                    1
-#define ISP_ICACHE_INDEX_BITS                  8
-#define ISP_ICACHE_SET_BITS                    5
-#define ISP_ICACHE_BLOCKS_PER_SET_BITS         1
-
-/* Experimental Flags */
-#define ISP_EXP_1                              0
-#define ISP_EXP_2                              0
-#define ISP_EXP_3                              0
-#define ISP_EXP_4                              0
-#define ISP_EXP_5                              0
-#define ISP_EXP_6                              0
-
-/* Derived values */
-#define ISP_LOG2_PMEM_WIDTH                    10
-#define ISP_VEC_WIDTH                          896
-#define ISP_SLICE_WIDTH                        56
-#define ISP_VMEM_WIDTH                         896
-#define ISP_VMEM_ALIGN                         128
-#if ISP_VMEM_IS_BAMEM
-  #define ISP_VMEM_ALIGN_ELEM                  2
-#endif /* ISP_VMEM_IS_BAMEM */
-#define ISP_SIMDLSU                            1
-#define ISP_LSU_IMM_BITS                       12
-
-/* convenient shortcuts for software*/
-#define ISP_NWAY                               ISP_VEC_NELEMS
-#define NBITS                                  ISP_VEC_ELEMBITS
-
-#define _isp_ceil_div(a,b)                     (((a)+(b)-1)/(b))
-
-#ifdef C_RUN
-#define ISP_VEC_ALIGN                          (_isp_ceil_div(ISP_VEC_WIDTH, 64)*8)
-#else
-#define ISP_VEC_ALIGN                          ISP_VMEM_ALIGN
-#endif
-
-/* HRT specific vector support */
-#define isp2401_mamoiada_vector_alignment         ISP_VEC_ALIGN
-#define isp2401_mamoiada_vector_elem_bits         ISP_VMEM_ELEMBITS
-#define isp2401_mamoiada_vector_elem_precision    ISP_VMEM_ELEM_PRECISION
-#define isp2401_mamoiada_vector_num_elems         ISP_VEC_NELEMS
-
-/* register file sizes */
-#define ISP_RF0_SIZE        64
-#define ISP_RF1_SIZE        16
-#define ISP_RF2_SIZE        64
-#define ISP_RF3_SIZE        4
-#define ISP_RF4_SIZE        64
-#define ISP_RF5_SIZE        16
-#define ISP_RF6_SIZE        16
-#define ISP_RF7_SIZE        16
-#define ISP_RF8_SIZE        16
-#define ISP_RF9_SIZE        16
-#define ISP_RF10_SIZE       16
-#define ISP_RF11_SIZE       16
-#define ISP_VRF1_SIZE       32
-#define ISP_VRF2_SIZE       32
-#define ISP_VRF3_SIZE       32
-#define ISP_VRF4_SIZE       32
-#define ISP_VRF5_SIZE       32
-#define ISP_VRF6_SIZE       32
-#define ISP_VRF7_SIZE       32
-#define ISP_VRF8_SIZE       32
-#define ISP_SRF1_SIZE       4
-#define ISP_SRF2_SIZE       64
-#define ISP_SRF3_SIZE       64
-#define ISP_SRF4_SIZE       32
-#define ISP_SRF5_SIZE       64
-#define ISP_FRF0_SIZE       16
-#define ISP_FRF1_SIZE       4
-#define ISP_FRF2_SIZE       16
-#define ISP_FRF3_SIZE       4
-#define ISP_FRF4_SIZE       4
-#define ISP_FRF5_SIZE       8
-#define ISP_FRF6_SIZE       4
-/* register file read latency */
-#define ISP_VRF1_READ_LAT       1
-#define ISP_VRF2_READ_LAT       1
-#define ISP_VRF3_READ_LAT       1
-#define ISP_VRF4_READ_LAT       1
-#define ISP_VRF5_READ_LAT       1
-#define ISP_VRF6_READ_LAT       1
-#define ISP_VRF7_READ_LAT       1
-#define ISP_VRF8_READ_LAT       1
-#define ISP_SRF1_READ_LAT       1
-#define ISP_SRF2_READ_LAT       1
-#define ISP_SRF3_READ_LAT       1
-#define ISP_SRF4_READ_LAT       1
-#define ISP_SRF5_READ_LAT       1
-#define ISP_SRF5_READ_LAT       1
-/* immediate sizes */
-#define ISP_IS1_IMM_BITS        14
-#define ISP_IS2_IMM_BITS        13
-#define ISP_IS3_IMM_BITS        14
-#define ISP_IS4_IMM_BITS        14
-#define ISP_IS5_IMM_BITS        9
-#define ISP_IS6_IMM_BITS        16
-#define ISP_IS7_IMM_BITS        9
-#define ISP_IS8_IMM_BITS        16
-#define ISP_IS9_IMM_BITS        11
-/* fifo depths */
-#define ISP_IF_FIFO_DEPTH         0
-#define ISP_IF_B_FIFO_DEPTH       0
-#define ISP_DMA_FIFO_DEPTH        0
-#define ISP_OF_FIFO_DEPTH         0
-#define ISP_GDC_FIFO_DEPTH        0
-#define ISP_SCL_FIFO_DEPTH        0
-#define ISP_GPFIFO_FIFO_DEPTH     0
-#define ISP_SP_FIFO_DEPTH         0
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp_acquisition_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp_acquisition_defs.h
deleted file mode 100644
index 593620721627..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp_acquisition_defs.h
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _isp_acquisition_defs_h
-#define _isp_acquisition_defs_h
-
-#define _ISP_ACQUISITION_REG_ALIGN                4  /* assuming 32 bit control bus width */
-#define _ISP_ACQUISITION_BYTES_PER_ELEM           4		
-
-/* --------------------------------------------------*/
-
-#define NOF_ACQ_IRQS                              1
-
-/* --------------------------------------------------*/
-/* FSM */
-/* --------------------------------------------------*/
-#define MEM2STREAM_FSM_STATE_BITS                 2
-#define ACQ_SYNCHRONIZER_FSM_STATE_BITS           2
-
-/* --------------------------------------------------*/
-/* REGISTER INFO */
-/* --------------------------------------------------*/
-
-#define NOF_ACQ_REGS                              12      
-
-// Register id's of MMIO slave accesible registers
-#define ACQ_START_ADDR_REG_ID                     0              
-#define ACQ_MEM_REGION_SIZE_REG_ID                1
-#define ACQ_NUM_MEM_REGIONS_REG_ID                2
-#define ACQ_INIT_REG_ID                           3 
-#define ACQ_RECEIVED_SHORT_PACKETS_REG_ID         4
-#define ACQ_RECEIVED_LONG_PACKETS_REG_ID          5
-#define ACQ_LAST_COMMAND_REG_ID                   6
-#define ACQ_NEXT_COMMAND_REG_ID                   7
-#define ACQ_LAST_ACKNOWLEDGE_REG_ID               8
-#define ACQ_NEXT_ACKNOWLEDGE_REG_ID               9
-#define ACQ_FSM_STATE_INFO_REG_ID                 10
-#define ACQ_INT_CNTR_INFO_REG_ID                  11
- 
-// Register width
-#define ACQ_START_ADDR_REG_WIDTH                  9               
-#define ACQ_MEM_REGION_SIZE_REG_WIDTH             9  
-#define ACQ_NUM_MEM_REGIONS_REG_WIDTH             9  
-#define ACQ_INIT_REG_WIDTH                        3  
-#define ACQ_RECEIVED_SHORT_PACKETS_REG_WIDTH      32 
-#define ACQ_RECEIVED_LONG_PACKETS_REG_WIDTH       32  
-#define ACQ_LAST_COMMAND_REG_WIDTH                32  
-#define ACQ_NEXT_COMMAND_REG_WIDTH                32  
-#define ACQ_LAST_ACKNOWLEDGE_REG_WIDTH            32  
-#define ACQ_NEXT_ACKNOWLEDGE_REG_WIDTH            32  
-#define ACQ_FSM_STATE_INFO_REG_WIDTH              ((MEM2STREAM_FSM_STATE_BITS * 3) + (ACQ_SYNCHRONIZER_FSM_STATE_BITS *3))
-#define ACQ_INT_CNTR_INFO_REG_WIDTH               32
-
-/* register reset value */
-#define ACQ_START_ADDR_REG_RSTVAL                 0              
-#define ACQ_MEM_REGION_SIZE_REG_RSTVAL            128
-#define ACQ_NUM_MEM_REGIONS_REG_RSTVAL            3
-#define ACQ_INIT_REG_RSTVAL                       0                           
-#define ACQ_RECEIVED_SHORT_PACKETS_REG_RSTVAL     0
-#define ACQ_RECEIVED_LONG_PACKETS_REG_RSTVAL      0
-#define ACQ_LAST_COMMAND_REG_RSTVAL               0
-#define ACQ_NEXT_COMMAND_REG_RSTVAL               0
-#define ACQ_LAST_ACKNOWLEDGE_REG_RSTVAL           0
-#define ACQ_NEXT_ACKNOWLEDGE_REG_RSTVAL           0 
-#define ACQ_FSM_STATE_INFO_REG_RSTVAL             0
-#define ACQ_INT_CNTR_INFO_REG_RSTVAL              0 
-
-/* bit definitions */
-#define ACQ_INIT_RST_REG_BIT                      0
-#define ACQ_INIT_RESYNC_BIT                       2
-#define ACQ_INIT_RST_IDX                          ACQ_INIT_RST_REG_BIT
-#define ACQ_INIT_RST_BITS                         1
-#define ACQ_INIT_RESYNC_IDX                       ACQ_INIT_RESYNC_BIT
-#define ACQ_INIT_RESYNC_BITS                      1
-
-/* --------------------------------------------------*/
-/* TOKEN INFO */
-/* --------------------------------------------------*/
-#define ACQ_TOKEN_ID_LSB                          0
-#define ACQ_TOKEN_ID_MSB                          3            
-#define ACQ_TOKEN_WIDTH                           (ACQ_TOKEN_ID_MSB - ACQ_TOKEN_ID_LSB  + 1) // 4
-#define ACQ_TOKEN_ID_IDX                          0
-#define ACQ_TOKEN_ID_BITS                         ACQ_TOKEN_WIDTH
-#define ACQ_INIT_CMD_INIT_IDX                     4
-#define ACQ_INIT_CMD_INIT_BITS                    3
-#define ACQ_CMD_START_ADDR_IDX                    4
-#define ACQ_CMD_START_ADDR_BITS                   9
-#define ACQ_CMD_NOFWORDS_IDX                      13
-#define ACQ_CMD_NOFWORDS_BITS                     9  
-#define ACQ_MEM_REGION_ID_IDX                     22
-#define ACQ_MEM_REGION_ID_BITS                    9 
-#define ACQ_PACKET_LENGTH_TOKEN_MSB               21
-#define ACQ_PACKET_LENGTH_TOKEN_LSB               13
-#define ACQ_PACKET_DATA_FORMAT_ID_TOKEN_MSB       9
-#define ACQ_PACKET_DATA_FORMAT_ID_TOKEN_LSB       4
-#define ACQ_PACKET_CH_ID_TOKEN_MSB                11
-#define ACQ_PACKET_CH_ID_TOKEN_LSB                10
-#define ACQ_PACKET_MEM_REGION_ID_TOKEN_MSB        12		/* only for capt_end_of_packet_written */
-#define ACQ_PACKET_MEM_REGION_ID_TOKEN_LSB        4		/* only for capt_end_of_packet_written */
-
-
-/* Command tokens IDs */
-#define ACQ_READ_REGION_AUTO_INCR_TOKEN_ID        0 //0000b
-#define ACQ_READ_REGION_TOKEN_ID                  1 //0001b
-#define ACQ_READ_REGION_SOP_TOKEN_ID              2 //0010b  
-#define ACQ_INIT_TOKEN_ID                         8 //1000b
-
-/* Acknowledge token IDs */
-#define ACQ_READ_REGION_ACK_TOKEN_ID              0 //0000b
-#define ACQ_END_OF_PACKET_TOKEN_ID                4 //0100b
-#define ACQ_END_OF_REGION_TOKEN_ID                5 //0101b
-#define ACQ_SOP_MISMATCH_TOKEN_ID                 6 //0110b
-#define ACQ_UNDEF_PH_TOKEN_ID                     7 //0111b
-
-#define ACQ_TOKEN_MEMREGIONID_MSB                 30
-#define ACQ_TOKEN_MEMREGIONID_LSB                 22
-#define ACQ_TOKEN_NOFWORDS_MSB                    21
-#define ACQ_TOKEN_NOFWORDS_LSB                    13
-#define ACQ_TOKEN_STARTADDR_MSB                   12
-#define ACQ_TOKEN_STARTADDR_LSB                   4  
-
-
-/* --------------------------------------------------*/
-/* MIPI */
-/* --------------------------------------------------*/
-
-#define WORD_COUNT_WIDTH                          16
-#define PKT_CODE_WIDTH                            6            
-#define CHN_NO_WIDTH                              2  
-#define ERROR_INFO_WIDTH                          8
-  
-#define LONG_PKTCODE_MAX                          63
-#define LONG_PKTCODE_MIN                          16
-#define SHORT_PKTCODE_MAX                         15
-
-#define EOF_CODE                                  1
-
-/* --------------------------------------------------*/
-/* Packet Info */
-/* --------------------------------------------------*/
-#define ACQ_START_OF_FRAME                        0
-#define ACQ_END_OF_FRAME                          1
-#define ACQ_START_OF_LINE                         2
-#define ACQ_END_OF_LINE                           3
-#define ACQ_LINE_PAYLOAD                          4
-#define ACQ_GEN_SH_PKT                            5
-
-
-/* bit definition */
-#define ACQ_PKT_TYPE_IDX                          16
-#define ACQ_PKT_TYPE_BITS                         6
-#define ACQ_PKT_SOP_IDX                           32
-#define ACQ_WORD_CNT_IDX                          0
-#define ACQ_WORD_CNT_BITS                         16
-#define ACQ_PKT_INFO_IDX                          16
-#define ACQ_PKT_INFO_BITS                         8
-#define ACQ_HEADER_DATA_IDX                       0
-#define ACQ_HEADER_DATA_BITS                      16
-#define ACQ_ACK_TOKEN_ID_IDX                      ACQ_TOKEN_ID_IDX
-#define ACQ_ACK_TOKEN_ID_BITS                     ACQ_TOKEN_ID_BITS
-#define ACQ_ACK_NOFWORDS_IDX                      13
-#define ACQ_ACK_NOFWORDS_BITS                     9
-#define ACQ_ACK_PKT_LEN_IDX                       4
-#define ACQ_ACK_PKT_LEN_BITS                      16
-
-
-/* --------------------------------------------------*/
-/* Packet Data Type */
-/* --------------------------------------------------*/
-
-
-#define ACQ_YUV420_8_DATA                       24   /* 01 1000 YUV420 8-bit                                        */
-#define ACQ_YUV420_10_DATA                      25   /* 01 1001  YUV420 10-bit                                      */
-#define ACQ_YUV420_8L_DATA                      26   /* 01 1010   YUV420 8-bit legacy                               */
-#define ACQ_YUV422_8_DATA                       30   /* 01 1110   YUV422 8-bit                                      */
-#define ACQ_YUV422_10_DATA                      31   /* 01 1111   YUV422 10-bit                                     */
-#define ACQ_RGB444_DATA                         32   /* 10 0000   RGB444                                            */
-#define ACQ_RGB555_DATA              					 33   /* 10 0001   RGB555                                            */
-#define ACQ_RGB565_DATA              					 34   /* 10 0010   RGB565                                            */
-#define ACQ_RGB666_DATA              					 35   /* 10 0011   RGB666                                            */
-#define ACQ_RGB888_DATA              					 36   /* 10 0100   RGB888                                            */
-#define ACQ_RAW6_DATA                					 40   /* 10 1000   RAW6                                              */
-#define ACQ_RAW7_DATA                					 41   /* 10 1001   RAW7                                              */
-#define ACQ_RAW8_DATA                					 42   /* 10 1010   RAW8                                              */
-#define ACQ_RAW10_DATA               					 43   /* 10 1011   RAW10                                             */
-#define ACQ_RAW12_DATA               					 44   /* 10 1100   RAW12                                             */
-#define ACQ_RAW14_DATA               					 45   /* 10 1101   RAW14                                             */
-#define ACQ_USR_DEF_1_DATA           					 48   /* 11 0000    JPEG [User Defined 8-bit Data Type 1]            */
-#define ACQ_USR_DEF_2_DATA           					 49   /* 11 0001    User Defined 8-bit Data Type 2                   */
-#define ACQ_USR_DEF_3_DATA           					 50   /* 11 0010    User Defined 8-bit Data Type 3                   */
-#define ACQ_USR_DEF_4_DATA           					 51   /* 11 0011    User Defined 8-bit Data Type 4                   */
-#define ACQ_USR_DEF_5_DATA           					 52   /* 11 0100    User Defined 8-bit Data Type 5                   */
-#define ACQ_USR_DEF_6_DATA           					 53   /* 11 0101    User Defined 8-bit Data Type 6                   */
-#define ACQ_USR_DEF_7_DATA           					 54   /* 11 0110    User Defined 8-bit Data Type 7                   */
-#define ACQ_USR_DEF_8_DATA           					 55   /* 11 0111    User Defined 8-bit Data Type 8                   */
-#define ACQ_Emb_DATA                 					 18   /* 01 0010    embedded eight bit non image data                */
-#define ACQ_SOF_DATA                 					 0   /* 00 0000    frame start                                      */
-#define ACQ_EOF_DATA                 					 1   /* 00 0001    frame end                                        */
-#define ACQ_SOL_DATA                 					 2   /* 00 0010    line start                                       */
-#define ACQ_EOL_DATA                 					 3   /* 00 0011    line end                                         */
-#define ACQ_GEN_SH1_DATA             					 8   /* 00 1000  Generic Short Packet Code 1                        */
-#define ACQ_GEN_SH2_DATA             					 9   /* 00 1001    Generic Short Packet Code 2                      */
-#define ACQ_GEN_SH3_DATA             					 10   /* 00 1010    Generic Short Packet Code 3                      */
-#define ACQ_GEN_SH4_DATA             					 11   /* 00 1011    Generic Short Packet Code 4                      */
-#define ACQ_GEN_SH5_DATA             					 12   /* 00 1100    Generic Short Packet Code 5                      */
-#define ACQ_GEN_SH6_DATA             					 13   /* 00 1101    Generic Short Packet Code 6                      */
-#define ACQ_GEN_SH7_DATA             					 14   /* 00 1110    Generic Short Packet Code 7                      */
-#define ACQ_GEN_SH8_DATA             					 15   /* 00 1111    Generic Short Packet Code 8                      */
-#define ACQ_YUV420_8_CSPS_DATA       					 28   /* 01 1100   YUV420 8-bit (Chroma Shifted Pixel Sampling)      */
-#define ACQ_YUV420_10_CSPS_DATA      					 29   /* 01 1101   YUV420 10-bit (Chroma Shifted Pixel Sampling)     */
-#define ACQ_RESERVED_DATA_TYPE_MIN              56
-#define ACQ_RESERVED_DATA_TYPE_MAX              63
-#define ACQ_GEN_LONG_RESERVED_DATA_TYPE_MIN     19
-#define ACQ_GEN_LONG_RESERVED_DATA_TYPE_MAX     23
-#define ACQ_YUV_RESERVED_DATA_TYPE              27
-#define ACQ_RGB_RESERVED_DATA_TYPE_MIN          37
-#define ACQ_RGB_RESERVED_DATA_TYPE_MAX          39
-#define ACQ_RAW_RESERVED_DATA_TYPE_MIN          46
-#define ACQ_RAW_RESERVED_DATA_TYPE_MAX          47
-
-/* --------------------------------------------------*/
-
-#endif /* _isp_acquisition_defs_h */ 
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp_capture_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp_capture_defs.h
deleted file mode 100644
index aa413df022f2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/isp_capture_defs.h
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _isp_capture_defs_h
-#define _isp_capture_defs_h
-
-#define _ISP_CAPTURE_REG_ALIGN                    4  /* assuming 32 bit control bus width */
-#define _ISP_CAPTURE_BITS_PER_ELEM                32  /* only for data, not SOP */						           
-#define _ISP_CAPTURE_BYTES_PER_ELEM               (_ISP_CAPTURE_BITS_PER_ELEM/8	)				           
-#define _ISP_CAPTURE_BYTES_PER_WORD               32		/* 256/8 */	
-#define _ISP_CAPTURE_ELEM_PER_WORD                _ISP_CAPTURE_BYTES_PER_WORD / _ISP_CAPTURE_BYTES_PER_ELEM		           
-
-//#define CAPT_RCV_ACK                              1
-//#define CAPT_WRT_ACK                              2               
-//#define CAPT_IRQ_ACK                              3                        
-
-/* --------------------------------------------------*/
-
-#define NOF_IRQS                                  2
-
-/* --------------------------------------------------*/
-/* REGISTER INFO */
-/* --------------------------------------------------*/
-
-// Number of registers
-#define CAPT_NOF_REGS                             16
-
-// Register id's of MMIO slave accesible registers
-#define CAPT_START_MODE_REG_ID                    0
-#define CAPT_START_ADDR_REG_ID                    1 
-#define CAPT_MEM_REGION_SIZE_REG_ID               2 
-#define CAPT_NUM_MEM_REGIONS_REG_ID               3 
-#define CAPT_INIT_REG_ID                          4 
-#define CAPT_START_REG_ID                         5
-#define CAPT_STOP_REG_ID                          6  
-
-#define CAPT_PACKET_LENGTH_REG_ID                 7
-#define CAPT_RECEIVED_LENGTH_REG_ID               8 
-#define CAPT_RECEIVED_SHORT_PACKETS_REG_ID        9 
-#define CAPT_RECEIVED_LONG_PACKETS_REG_ID         10 
-#define CAPT_LAST_COMMAND_REG_ID                  11        
-#define CAPT_NEXT_COMMAND_REG_ID                  12
-#define CAPT_LAST_ACKNOWLEDGE_REG_ID              13
-#define CAPT_NEXT_ACKNOWLEDGE_REG_ID              14
-#define CAPT_FSM_STATE_INFO_REG_ID                15
-
-// Register width
-#define CAPT_START_MODE_REG_WIDTH                 1 
-//#define CAPT_START_ADDR_REG_WIDTH                 9
-//#define CAPT_MEM_REGION_SIZE_REG_WIDTH            9
-//#define CAPT_NUM_MEM_REGIONS_REG_WIDTH            9
-#define CAPT_INIT_REG_WIDTH                       (22 + 4)
-
-#define CAPT_START_REG_WIDTH                      1
-#define CAPT_STOP_REG_WIDTH                       1
-
-/* --------------------------------------------------*/
-/* FSM */
-/* --------------------------------------------------*/
-#define CAPT_WRITE2MEM_FSM_STATE_BITS             2
-#define CAPT_SYNCHRONIZER_FSM_STATE_BITS          3
-
-
-#define CAPT_PACKET_LENGTH_REG_WIDTH              17
-#define CAPT_RECEIVED_LENGTH_REG_WIDTH            17   
-#define CAPT_RECEIVED_SHORT_PACKETS_REG_WIDTH     32
-#define CAPT_RECEIVED_LONG_PACKETS_REG_WIDTH      32
-#define CAPT_LAST_COMMAND_REG_WIDTH               32
-/* #define CAPT_NEXT_COMMAND_REG_WIDTH               32 */  
-#define CAPT_LAST_ACKNOWLEDGE_REG_WIDTH           32
-#define CAPT_NEXT_ACKNOWLEDGE_REG_WIDTH           32
-#define CAPT_FSM_STATE_INFO_REG_WIDTH             ((CAPT_WRITE2MEM_FSM_STATE_BITS * 3) + (CAPT_SYNCHRONIZER_FSM_STATE_BITS * 3))
-
-//#define CAPT_INIT_RESTART_MEM_ADDR_WIDTH          9   
-//#define CAPT_INIT_RESTART_MEM_REGION_WIDTH        9 
-
-/* register reset value */
-#define CAPT_START_MODE_REG_RSTVAL                0   
-#define CAPT_START_ADDR_REG_RSTVAL                0
-#define CAPT_MEM_REGION_SIZE_REG_RSTVAL           128
-#define CAPT_NUM_MEM_REGIONS_REG_RSTVAL           3 
-#define CAPT_INIT_REG_RSTVAL                      0
-
-#define CAPT_START_REG_RSTVAL                     0
-#define CAPT_STOP_REG_RSTVAL                      0
-
-#define CAPT_PACKET_LENGTH_REG_RSTVAL             0
-#define CAPT_RECEIVED_LENGTH_REG_RSTVAL           0
-#define CAPT_RECEIVED_SHORT_PACKETS_REG_RSTVAL    0
-#define CAPT_RECEIVED_LONG_PACKETS_REG_RSTVAL     0
-#define CAPT_LAST_COMMAND_REG_RSTVAL              0
-#define CAPT_NEXT_COMMAND_REG_RSTVAL              0
-#define CAPT_LAST_ACKNOWLEDGE_REG_RSTVAL          0
-#define CAPT_NEXT_ACKNOWLEDGE_REG_RSTVAL          0
-#define CAPT_FSM_STATE_INFO_REG_RSTVAL            0
-
-/* bit definitions */
-#define CAPT_INIT_RST_REG_BIT                     0
-#define CAPT_INIT_FLUSH_BIT                       1
-#define CAPT_INIT_RESYNC_BIT                      2
-#define CAPT_INIT_RESTART_BIT                     3
-#define CAPT_INIT_RESTART_MEM_ADDR_LSB            4
-#define CAPT_INIT_RESTART_MEM_ADDR_MSB            14
-#define CAPT_INIT_RESTART_MEM_REGION_LSB          15
-#define CAPT_INIT_RESTART_MEM_REGION_MSB          25
-
-
-#define CAPT_INIT_RST_REG_IDX                     CAPT_INIT_RST_REG_BIT
-#define CAPT_INIT_RST_REG_BITS                    1
-#define CAPT_INIT_FLUSH_IDX                       CAPT_INIT_FLUSH_BIT
-#define CAPT_INIT_FLUSH_BITS                      1
-#define CAPT_INIT_RESYNC_IDX                      CAPT_INIT_RESYNC_BIT
-#define CAPT_INIT_RESYNC_BITS                     1
-#define CAPT_INIT_RESTART_IDX                     CAPT_INIT_RESTART_BIT
-#define CAPT_INIT_RESTART_BITS					  				1
-#define CAPT_INIT_RESTART_MEM_ADDR_IDX            CAPT_INIT_RESTART_MEM_ADDR_LSB
-#define CAPT_INIT_RESTART_MEM_ADDR_BITS           (CAPT_INIT_RESTART_MEM_ADDR_MSB - CAPT_INIT_RESTART_MEM_ADDR_LSB + 1)
-#define CAPT_INIT_RESTART_MEM_REGION_IDX          CAPT_INIT_RESTART_MEM_REGION_LSB
-#define CAPT_INIT_RESTART_MEM_REGION_BITS         (CAPT_INIT_RESTART_MEM_REGION_MSB - CAPT_INIT_RESTART_MEM_REGION_LSB + 1)
-
-
-
-/* --------------------------------------------------*/
-/* TOKEN INFO */
-/* --------------------------------------------------*/
-#define CAPT_TOKEN_ID_LSB                         0
-#define CAPT_TOKEN_ID_MSB                         3            
-#define CAPT_TOKEN_WIDTH                         (CAPT_TOKEN_ID_MSB - CAPT_TOKEN_ID_LSB  + 1) /* 4 */
-
-/* Command tokens IDs */
-#define CAPT_START_TOKEN_ID                       0 /* 0000b */
-#define CAPT_STOP_TOKEN_ID                        1 /* 0001b */
-#define CAPT_FREEZE_TOKEN_ID                      2 /* 0010b */  
-#define CAPT_RESUME_TOKEN_ID                      3 /* 0011b */
-#define CAPT_INIT_TOKEN_ID                        8 /* 1000b */
-
-#define CAPT_START_TOKEN_BIT                      0      
-#define CAPT_STOP_TOKEN_BIT                       0
-#define CAPT_FREEZE_TOKEN_BIT                     0
-#define CAPT_RESUME_TOKEN_BIT                     0
-#define CAPT_INIT_TOKEN_BIT                       0
-
-/* Acknowledge token IDs */
-#define CAPT_END_OF_PACKET_RECEIVED_TOKEN_ID      0 /* 0000b */
-#define CAPT_END_OF_PACKET_WRITTEN_TOKEN_ID       1 /* 0001b */
-#define CAPT_END_OF_REGION_WRITTEN_TOKEN_ID       2 /* 0010b */
-#define CAPT_FLUSH_DONE_TOKEN_ID                  3 /* 0011b */
-#define CAPT_PREMATURE_SOP_TOKEN_ID               4 /* 0100b */
-#define CAPT_MISSING_SOP_TOKEN_ID                 5 /* 0101b */
-#define CAPT_UNDEF_PH_TOKEN_ID                    6 /* 0110b */
-#define CAPT_STOP_ACK_TOKEN_ID                    7 /* 0111b */
-
-#define CAPT_PACKET_LENGTH_TOKEN_MSB             19
-#define CAPT_PACKET_LENGTH_TOKEN_LSB              4
-#define CAPT_SUPER_PACKET_LENGTH_TOKEN_MSB       20
-#define CAPT_SUPER_PACKET_LENGTH_TOKEN_LSB        4
-#define CAPT_PACKET_DATA_FORMAT_ID_TOKEN_MSB     25
-#define CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB     20
-#define CAPT_PACKET_CH_ID_TOKEN_MSB              27
-#define CAPT_PACKET_CH_ID_TOKEN_LSB              26
-#define CAPT_PACKET_MEM_REGION_ID_TOKEN_MSB      29		
-#define CAPT_PACKET_MEM_REGION_ID_TOKEN_LSB      21		
-
-/*  bit definition */
-#define CAPT_CMD_IDX                              CAPT_TOKEN_ID_LSB
-#define	CAPT_CMD_BITS                             (CAPT_TOKEN_ID_MSB - CAPT_TOKEN_ID_LSB + 1)
-#define CAPT_SOP_IDX                              32
-#define CAPT_SOP_BITS                             1
-#define CAPT_PKT_INFO_IDX                         16
-#define CAPT_PKT_INFO_BITS                        8
-#define CAPT_PKT_TYPE_IDX                         0
-#define CAPT_PKT_TYPE_BITS                        6
-#define CAPT_HEADER_DATA_IDX                      0
-#define CAPT_HEADER_DATA_BITS                     16
-#define CAPT_PKT_DATA_IDX                         0
-#define CAPT_PKT_DATA_BITS                        32
-#define CAPT_WORD_CNT_IDX                         0
-#define CAPT_WORD_CNT_BITS                        16
-#define CAPT_ACK_TOKEN_ID_IDX                     0
-#define CAPT_ACK_TOKEN_ID_BITS                    4
-//#define CAPT_ACK_PKT_LEN_IDX                      CAPT_PACKET_LENGTH_TOKEN_LSB
-//#define CAPT_ACK_PKT_LEN_BITS                     (CAPT_PACKET_LENGTH_TOKEN_MSB - CAPT_PACKET_LENGTH_TOKEN_LSB + 1)
-//#define CAPT_ACK_PKT_INFO_IDX                     20
-//#define CAPT_ACK_PKT_INFO_BITS                    8
-//#define CAPT_ACK_MEM_REG_ID1_IDX                  20			/* for capt_end_of_packet_written */
-//#define CAPT_ACK_MEM_REG_ID2_IDX                  4       /* for capt_end_of_region_written */
-#define CAPT_ACK_PKT_LEN_IDX                      CAPT_PACKET_LENGTH_TOKEN_LSB
-#define CAPT_ACK_PKT_LEN_BITS                     (CAPT_PACKET_LENGTH_TOKEN_MSB - CAPT_PACKET_LENGTH_TOKEN_LSB + 1)
-#define CAPT_ACK_SUPER_PKT_LEN_IDX                CAPT_SUPER_PACKET_LENGTH_TOKEN_LSB
-#define CAPT_ACK_SUPER_PKT_LEN_BITS               (CAPT_SUPER_PACKET_LENGTH_TOKEN_MSB - CAPT_SUPER_PACKET_LENGTH_TOKEN_LSB + 1)
-#define CAPT_ACK_PKT_INFO_IDX                     CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB
-#define CAPT_ACK_PKT_INFO_BITS                    (CAPT_PACKET_CH_ID_TOKEN_MSB - CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB + 1)
-#define CAPT_ACK_MEM_REGION_ID_IDX                CAPT_PACKET_MEM_REGION_ID_TOKEN_LSB
-#define CAPT_ACK_MEM_REGION_ID_BITS               (CAPT_PACKET_MEM_REGION_ID_TOKEN_MSB - CAPT_PACKET_MEM_REGION_ID_TOKEN_LSB + 1)
-#define CAPT_ACK_PKT_TYPE_IDX                     CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB
-#define CAPT_ACK_PKT_TYPE_BITS                    (CAPT_PACKET_DATA_FORMAT_ID_TOKEN_MSB - CAPT_PACKET_DATA_FORMAT_ID_TOKEN_LSB + 1)
-#define CAPT_INIT_TOKEN_INIT_IDX                  4
-#define CAPT_INIT_TOKEN_INIT_BITS                 22
-
-
-/* --------------------------------------------------*/
-/* MIPI */
-/* --------------------------------------------------*/
-
-#define CAPT_WORD_COUNT_WIDTH                     16      
-#define CAPT_PKT_CODE_WIDTH                       6                  
-#define CAPT_CHN_NO_WIDTH                         2        
-#define CAPT_ERROR_INFO_WIDTH                     8       
-
-#define LONG_PKTCODE_MAX                          63
-#define LONG_PKTCODE_MIN                          16
-#define SHORT_PKTCODE_MAX                         15
-
-
-/* --------------------------------------------------*/
-/* Packet Info */
-/* --------------------------------------------------*/
-#define CAPT_START_OF_FRAME                       0
-#define CAPT_END_OF_FRAME                         1
-#define CAPT_START_OF_LINE                        2
-#define CAPT_END_OF_LINE                          3
-#define CAPT_LINE_PAYLOAD                         4
-#define CAPT_GEN_SH_PKT                           5
-
-
-/* --------------------------------------------------*/
-/* Packet Data Type */
-/* --------------------------------------------------*/
-
-#define CAPT_YUV420_8_DATA                       24   /* 01 1000 YUV420 8-bit                                        */
-#define CAPT_YUV420_10_DATA                      25   /* 01 1001  YUV420 10-bit                                      */
-#define CAPT_YUV420_8L_DATA                      26   /* 01 1010   YUV420 8-bit legacy                               */
-#define CAPT_YUV422_8_DATA                       30   /* 01 1110   YUV422 8-bit                                      */
-#define CAPT_YUV422_10_DATA                      31   /* 01 1111   YUV422 10-bit                                     */
-#define CAPT_RGB444_DATA                         32   /* 10 0000   RGB444                                            */
-#define CAPT_RGB555_DATA              					 33   /* 10 0001   RGB555                                            */
-#define CAPT_RGB565_DATA              					 34   /* 10 0010   RGB565                                            */
-#define CAPT_RGB666_DATA              					 35   /* 10 0011   RGB666                                            */
-#define CAPT_RGB888_DATA              					 36   /* 10 0100   RGB888                                            */
-#define CAPT_RAW6_DATA                					 40   /* 10 1000   RAW6                                              */
-#define CAPT_RAW7_DATA                					 41   /* 10 1001   RAW7                                              */
-#define CAPT_RAW8_DATA                					 42   /* 10 1010   RAW8                                              */
-#define CAPT_RAW10_DATA               					 43   /* 10 1011   RAW10                                             */
-#define CAPT_RAW12_DATA               					 44   /* 10 1100   RAW12                                             */
-#define CAPT_RAW14_DATA               					 45   /* 10 1101   RAW14                                             */
-#define CAPT_USR_DEF_1_DATA           					 48   /* 11 0000    JPEG [User Defined 8-bit Data Type 1]            */
-#define CAPT_USR_DEF_2_DATA           					 49   /* 11 0001    User Defined 8-bit Data Type 2                   */
-#define CAPT_USR_DEF_3_DATA           					 50   /* 11 0010    User Defined 8-bit Data Type 3                   */
-#define CAPT_USR_DEF_4_DATA           					 51   /* 11 0011    User Defined 8-bit Data Type 4                   */
-#define CAPT_USR_DEF_5_DATA           					 52   /* 11 0100    User Defined 8-bit Data Type 5                   */
-#define CAPT_USR_DEF_6_DATA           					 53   /* 11 0101    User Defined 8-bit Data Type 6                   */
-#define CAPT_USR_DEF_7_DATA           					 54   /* 11 0110    User Defined 8-bit Data Type 7                   */
-#define CAPT_USR_DEF_8_DATA           					 55   /* 11 0111    User Defined 8-bit Data Type 8                   */
-#define CAPT_Emb_DATA                 					 18   /* 01 0010    embedded eight bit non image data                */
-#define CAPT_SOF_DATA                 					 0   /* 00 0000    frame start                                      */
-#define CAPT_EOF_DATA                 					 1   /* 00 0001    frame end                                        */
-#define CAPT_SOL_DATA                 					 2   /* 00 0010    line start                                       */
-#define CAPT_EOL_DATA                 					 3   /* 00 0011    line end                                         */
-#define CAPT_GEN_SH1_DATA             					 8   /* 00 1000  Generic Short Packet Code 1                        */
-#define CAPT_GEN_SH2_DATA             					 9   /* 00 1001    Generic Short Packet Code 2                      */
-#define CAPT_GEN_SH3_DATA             					 10   /* 00 1010    Generic Short Packet Code 3                      */
-#define CAPT_GEN_SH4_DATA             					 11   /* 00 1011    Generic Short Packet Code 4                      */
-#define CAPT_GEN_SH5_DATA             					 12   /* 00 1100    Generic Short Packet Code 5                      */
-#define CAPT_GEN_SH6_DATA             					 13   /* 00 1101    Generic Short Packet Code 6                      */
-#define CAPT_GEN_SH7_DATA             					 14   /* 00 1110    Generic Short Packet Code 7                      */
-#define CAPT_GEN_SH8_DATA             					 15   /* 00 1111    Generic Short Packet Code 8                      */
-#define CAPT_YUV420_8_CSPS_DATA       					 28   /* 01 1100   YUV420 8-bit (Chroma Shifted Pixel Sampling)      */
-#define CAPT_YUV420_10_CSPS_DATA      					 29   /* 01 1101   YUV420 10-bit (Chroma Shifted Pixel Sampling)     */
-#define CAPT_RESERVED_DATA_TYPE_MIN              56
-#define CAPT_RESERVED_DATA_TYPE_MAX              63
-#define CAPT_GEN_LONG_RESERVED_DATA_TYPE_MIN     19
-#define CAPT_GEN_LONG_RESERVED_DATA_TYPE_MAX     23
-#define CAPT_YUV_RESERVED_DATA_TYPE              27
-#define CAPT_RGB_RESERVED_DATA_TYPE_MIN          37
-#define CAPT_RGB_RESERVED_DATA_TYPE_MAX          39
-#define CAPT_RAW_RESERVED_DATA_TYPE_MIN          46
-#define CAPT_RAW_RESERVED_DATA_TYPE_MAX          47
-
-
-/* --------------------------------------------------*/
-/* Capture Unit State */
-/* --------------------------------------------------*/
-#define CAPT_FREE_RUN                             0
-#define CAPT_NO_SYNC                              1
-#define CAPT_SYNC_SWP                             2
-#define CAPT_SYNC_MWP                             3
-#define CAPT_SYNC_WAIT                            4
-#define CAPT_FREEZE                               5
-#define CAPT_RUN                                  6
-
-
-/* --------------------------------------------------*/
-
-#endif /* _isp_capture_defs_h */ 
-
-
-
-
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/mmu_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/mmu_defs.h
deleted file mode 100644
index c038f39ffd25..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/mmu_defs.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _mmu_defs_h
-#define _mmu_defs_h
-
-#define _HRT_MMU_INVALIDATE_TLB_REG_IDX          0
-#define _HRT_MMU_PAGE_TABLE_BASE_ADDRESS_REG_IDX 1
-
-#define _HRT_MMU_REG_ALIGN 4
-
-#endif /* _mmu_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/scalar_processor_2400_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/scalar_processor_2400_params.h
deleted file mode 100644
index 9b6c2893d950..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/scalar_processor_2400_params.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _scalar_processor_2400_params_h
-#define _scalar_processor_2400_params_h
-
-#include "cell_params.h"
-
-#endif /* _scalar_processor_2400_params_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/str2mem_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/str2mem_defs.h
deleted file mode 100644
index 1cb62444cf68..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/str2mem_defs.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _ST2MEM_DEFS_H
-#define _ST2MEM_DEFS_H
-
-#define _STR2MEM_CRUN_BIT               0x100000
-#define _STR2MEM_CMD_BITS               0x0F0000
-#define _STR2MEM_COUNT_BITS             0x00FFFF
-
-#define _STR2MEM_BLOCKS_CMD             0xA0000
-#define _STR2MEM_PACKETS_CMD            0xB0000
-#define _STR2MEM_BYTES_CMD              0xC0000
-#define _STR2MEM_BYTES_FROM_PACKET_CMD  0xD0000
-
-#define _STR2MEM_SOFT_RESET_REG_ID                   0
-#define _STR2MEM_INPUT_ENDIANNESS_REG_ID             1
-#define _STR2MEM_OUTPUT_ENDIANNESS_REG_ID            2
-#define _STR2MEM_BIT_SWAPPING_REG_ID                 3
-#define _STR2MEM_BLOCK_SYNC_LEVEL_REG_ID             4
-#define _STR2MEM_PACKET_SYNC_LEVEL_REG_ID            5
-#define _STR2MEM_READ_POST_WRITE_SYNC_ENABLE_REG_ID  6
-#define _STR2MEM_DUAL_BYTE_INPUTS_ENABLED_REG_ID     7
-#define _STR2MEM_EN_STAT_UPDATE_ID                   8
-
-#define _STR2MEM_REG_ALIGN      4
-
-#endif /* _ST2MEM_DEFS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/streaming_to_mipi_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/streaming_to_mipi_defs.h
deleted file mode 100644
index 60143b8743a2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/streaming_to_mipi_defs.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _streaming_to_mipi_defs_h
-#define _streaming_to_mipi_defs_h
-
-#define HIVE_STR_TO_MIPI_VALID_A_BIT 0
-#define HIVE_STR_TO_MIPI_VALID_B_BIT 1
-#define HIVE_STR_TO_MIPI_SOL_BIT     2
-#define HIVE_STR_TO_MIPI_EOL_BIT     3
-#define HIVE_STR_TO_MIPI_SOF_BIT     4
-#define HIVE_STR_TO_MIPI_EOF_BIT     5
-#define HIVE_STR_TO_MIPI_CH_ID_LSB   6
-
-#define HIVE_STR_TO_MIPI_DATA_A_LSB  (HIVE_STR_TO_MIPI_VALID_B_BIT + 1)
-
-#endif /* _streaming_to_mipi_defs_h */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/timed_controller_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/timed_controller_defs.h
deleted file mode 100644
index d2b8972b0d9e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/timed_controller_defs.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _timed_controller_defs_h
-#define _timed_controller_defs_h
-
-#define _HRT_TIMED_CONTROLLER_CMD_REG_IDX 0
-
-#define _HRT_TIMED_CONTROLLER_REG_ALIGN 4
-
-#endif /* _timed_controller_defs_h */   
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/var.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/var.h
deleted file mode 100644
index 19b19ef484f9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/var.h
+++ /dev/null
@@ -1,99 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HRT_VAR_H
-#define _HRT_VAR_H
-
-#include "version.h"
-#include "system_api.h"
-#include "hive_types.h"
-
-#define hrt_int_type_of_char   char
-#define hrt_int_type_of_uchar  unsigned char
-#define hrt_int_type_of_short  short
-#define hrt_int_type_of_ushort unsigned short
-#define hrt_int_type_of_int    int
-#define hrt_int_type_of_uint   unsigned int
-#define hrt_int_type_of_long   long
-#define hrt_int_type_of_ulong  unsigned long
-#define hrt_int_type_of_ptr    unsigned int
-
-#define hrt_host_type_of_char   char
-#define hrt_host_type_of_uchar  unsigned char
-#define hrt_host_type_of_short  short
-#define hrt_host_type_of_ushort unsigned short
-#define hrt_host_type_of_int    int
-#define hrt_host_type_of_uint   unsigned int
-#define hrt_host_type_of_long   long
-#define hrt_host_type_of_ulong  unsigned long
-#define hrt_host_type_of_ptr    void*
-
-#define HRT_TYPE_BYTES(cell, type) (HRT_TYPE_BITS(cell, type)/8)
-#define HRT_HOST_TYPE(cell_type)   HRTCAT(hrt_host_type_of_, cell_type)
-#define HRT_INT_TYPE(type)         HRTCAT(hrt_int_type_of_, type)
-
-#ifdef C_RUN
-
-#ifdef C_RUN_DYNAMIC_LINK_PROGRAMS
-extern void *csim_processor_get_crun_symbol(hive_proc_id p, const char *sym);
-#define _hrt_cell_get_crun_symbol(cell,sym)          csim_processor_get_crun_symbol(cell,HRTSTR(sym))
-#define _hrt_cell_get_crun_indexed_symbol(cell,sym)  csim_processor_get_crun_symbol(cell,HRTSTR(sym))
-#else
-#define _hrt_cell_get_crun_symbol(cell,sym)         (&sym)
-#define _hrt_cell_get_crun_indexed_symbol(cell,sym) (sym)
-#endif //  C_RUN_DYNAMIC_LINK_PROGRAMS
-
-#define hrt_scalar_store(cell, type, var, data) \
-	((*(HRT_HOST_TYPE(type)*)_hrt_cell_get_crun_symbol(cell,var)) = (data))
-#define hrt_scalar_load(cell, type, var) \
-	((*(HRT_HOST_TYPE(type)*)_hrt_cell_get_crun_symbol(cell,var)))
-
-#define hrt_indexed_store(cell, type, array, index, data) \
-	((((HRT_HOST_TYPE(type)*)_hrt_cell_get_crun_indexed_symbol(cell,array))[index]) = (data))
-#define hrt_indexed_load(cell, type, array, index) \
-	(((HRT_HOST_TYPE(type)*)_hrt_cell_get_crun_indexed_symbol(cell,array))[index])
-
-#else /* C_RUN */
-
-#define hrt_scalar_store(cell, type, var, data) \
-  HRTCAT(hrt_mem_store_,HRT_TYPE_BITS(cell, type))(\
-	       cell, \
-	       HRTCAT(HIVE_MEM_,var), \
-	       HRTCAT(HIVE_ADDR_,var), \
-	       (HRT_INT_TYPE(type))(data))
-
-#define hrt_scalar_load(cell, type, var) \
-  (HRT_HOST_TYPE(type))(HRTCAT4(_hrt_mem_load_,HRT_PROC_TYPE(cell),_,type) ( \
-	       cell, \
-	       HRTCAT(HIVE_MEM_,var), \
-	       HRTCAT(HIVE_ADDR_,var)))
-
-#define hrt_indexed_store(cell, type, array, index, data) \
-  HRTCAT(hrt_mem_store_,HRT_TYPE_BITS(cell, type))(\
-	       cell, \
-	       HRTCAT(HIVE_MEM_,array), \
-	       (HRTCAT(HIVE_ADDR_,array))+((index)*HRT_TYPE_BYTES(cell, type)), \
-	       (HRT_INT_TYPE(type))(data))
-
-#define hrt_indexed_load(cell, type, array, index) \
-  (HRT_HOST_TYPE(type))(HRTCAT4(_hrt_mem_load_,HRT_PROC_TYPE(cell),_,type) ( \
-         cell, \
-	       HRTCAT(HIVE_MEM_,array), \
-	       (HRTCAT(HIVE_ADDR_,array))+((index)*HRT_TYPE_BYTES(cell, type))))
-
-#endif /* C_RUN */
-
-#endif /* _HRT_VAR_H */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/version.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/version.h
deleted file mode 100644
index bbc4948baea9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/hrt/version.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef HRT_VERSION_H
-#define HRT_VERSION_H
-#define HRT_VERSION_MAJOR 1
-#define HRT_VERSION_MINOR 4
-#define HRT_VERSION 1_4
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/spmem_dump.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/spmem_dump.c
deleted file mode 100644
index 09f0780f0c80..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_2401_system/spmem_dump.c
+++ /dev/null
@@ -1,3634 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _sp_map_h_
-#define _sp_map_h_
-
-
-#ifndef _hrt_dummy_use_blob_sp
-#define _hrt_dummy_use_blob_sp()
-#endif
-
-#define _hrt_cell_load_program_sp(proc) _hrt_cell_load_program_embedded(proc, sp)
-
-#ifndef ISP2401
-/* function input_system_acquisition_stop: ADE */
-#else
-/* function input_system_acquisition_stop: AD8 */
-#endif
-
-#ifndef ISP2401
-/* function longjmp: 684E */
-#else
-/* function longjmp: 69C1 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_HIVE_IF_SRST_MASK
-#define HIVE_MEM_HIVE_IF_SRST_MASK scalar_processor_2400_dmem
-#define HIVE_ADDR_HIVE_IF_SRST_MASK 0x1C8
-#define HIVE_SIZE_HIVE_IF_SRST_MASK 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_HIVE_IF_SRST_MASK scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_HIVE_IF_SRST_MASK 0x1C8
-#define HIVE_SIZE_sp_HIVE_IF_SRST_MASK 16
-
-#ifndef ISP2401
-/* function tmpmem_init_dmem: 6599 */
-#else
-/* function tmpmem_init_dmem: 66D4 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_receive_ack: 5EDD */
-#else
-/* function ia_css_isys_sp_token_map_receive_ack: 6018 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_addr_B: 3345 */
-#else
-/* function ia_css_dmaproxy_sp_set_addr_B: 3539 */
-
-/* function ia_css_pipe_data_init_tagger_resources: A4F */
-#endif
-
-/* function debug_buffer_set_ddr_addr: DD */
-
-#ifndef ISP2401
-/* function receiver_port_reg_load: AC2 */
-#else
-/* function receiver_port_reg_load: ABC */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_mipi
-#define HIVE_MEM_vbuf_mipi scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_mipi 0x631C
-#else
-#define HIVE_ADDR_vbuf_mipi 0x6378
-#endif
-#define HIVE_SIZE_vbuf_mipi 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_mipi scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_mipi 0x631C
-#else
-#define HIVE_ADDR_sp_vbuf_mipi 0x6378
-#endif
-#define HIVE_SIZE_sp_vbuf_mipi 12
-
-#ifndef ISP2401
-/* function ia_css_event_sp_decode: 3536 */
-#else
-/* function ia_css_event_sp_decode: 372A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_get_size: 48BE */
-#else
-/* function ia_css_queue_get_size: 4B46 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_load: 4EFF */
-#else
-/* function ia_css_queue_load: 515D */
-#endif
-
-#ifndef ISP2401
-/* function setjmp: 6857 */
-#else
-/* function setjmp: 69CA */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_sp2host_isys_event_queue
-#define HIVE_MEM_sem_for_sp2host_isys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_sp2host_isys_event_queue 0x4684
-#else
-#define HIVE_ADDR_sem_for_sp2host_isys_event_queue 0x46CC
-#endif
-#define HIVE_SIZE_sem_for_sp2host_isys_event_queue 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_sp2host_isys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_sp2host_isys_event_queue 0x4684
-#else
-#define HIVE_ADDR_sp_sem_for_sp2host_isys_event_queue 0x46CC
-#endif
-#define HIVE_SIZE_sp_sem_for_sp2host_isys_event_queue 20
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_wait_for_ack: 6E07 */
-#else
-/* function ia_css_dmaproxy_sp_wait_for_ack: 6F4B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_func: 5124 */
-#else
-/* function ia_css_sp_rawcopy_func: 5382 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_pop_marked: 2A10 */
-#else
-/* function ia_css_tagger_buf_sp_pop_marked: 2BB2 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_stage
-#define HIVE_MEM_isp_stage scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_stage 0x5C00
-#else
-#define HIVE_ADDR_isp_stage 0x5C60
-#endif
-#define HIVE_SIZE_isp_stage 832
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_stage scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_stage 0x5C00
-#else
-#define HIVE_ADDR_sp_isp_stage 0x5C60
-#endif
-#define HIVE_SIZE_sp_isp_stage 832
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_raw
-#define HIVE_MEM_vbuf_raw scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_raw 0x2F4
-#else
-#define HIVE_ADDR_vbuf_raw 0x30C
-#endif
-#define HIVE_SIZE_vbuf_raw 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_raw scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_raw 0x2F4
-#else
-#define HIVE_ADDR_sp_vbuf_raw 0x30C
-#endif
-#define HIVE_SIZE_sp_vbuf_raw 4
-
-#ifndef ISP2401
-/* function ia_css_sp_bin_copy_func: 504B */
-#else
-/* function ia_css_sp_bin_copy_func: 52A9 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_item_store: 4C4D */
-#else
-/* function ia_css_queue_item_store: 4EAB */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_metadata_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_metadata_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_metadata_bufs 0x4AA0
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_metadata_bufs 0x4AFC
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_metadata_bufs 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_metadata_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_metadata_bufs 0x4AA0
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_metadata_bufs 0x4AFC
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_metadata_bufs 20
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_buffer_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_buffer_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_buffer_bufs 0x4AB4
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_buffer_bufs 0x4B10
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_buffer_bufs 160
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_buffer_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_buffer_bufs 0x4AB4
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_buffer_bufs 0x4B10
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_buffer_bufs 160
-
-/* function sp_start_isp: 45D */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_binary_group
-#define HIVE_MEM_sp_binary_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_binary_group 0x5FF0
-#else
-#define HIVE_ADDR_sp_binary_group 0x6050
-#endif
-#define HIVE_SIZE_sp_binary_group 32
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_binary_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_binary_group 0x5FF0
-#else
-#define HIVE_ADDR_sp_sp_binary_group 0x6050
-#endif
-#define HIVE_SIZE_sp_sp_binary_group 32
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_sw_state
-#define HIVE_MEM_sp_sw_state scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sw_state 0x62AC
-#else
-#define HIVE_ADDR_sp_sw_state 0x6308
-#endif
-#define HIVE_SIZE_sp_sw_state 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_sw_state scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_sw_state 0x62AC
-#else
-#define HIVE_ADDR_sp_sp_sw_state 0x6308
-#endif
-#define HIVE_SIZE_sp_sp_sw_state 4
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_main: D5B */
-#else
-/* function ia_css_thread_sp_main: D50 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_internal_buffers: 373C */
-#else
-/* function ia_css_ispctrl_sp_init_internal_buffers: 396B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp2host_psys_event_queue_handle
-#define HIVE_MEM_sp2host_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp2host_psys_event_queue_handle 0x4B54
-#else
-#define HIVE_ADDR_sp2host_psys_event_queue_handle 0x4BB0
-#endif
-#define HIVE_SIZE_sp2host_psys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp2host_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp2host_psys_event_queue_handle 0x4B54
-#else
-#define HIVE_ADDR_sp_sp2host_psys_event_queue_handle 0x4BB0
-#endif
-#define HIVE_SIZE_sp_sp2host_psys_event_queue_handle 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_sp2host_psys_event_queue
-#define HIVE_MEM_sem_for_sp2host_psys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_sp2host_psys_event_queue 0x4698
-#else
-#define HIVE_ADDR_sem_for_sp2host_psys_event_queue 0x46E0
-#endif
-#define HIVE_SIZE_sem_for_sp2host_psys_event_queue 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_sp2host_psys_event_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_sp2host_psys_event_queue 0x4698
-#else
-#define HIVE_ADDR_sp_sem_for_sp2host_psys_event_queue 0x46E0
-#endif
-#define HIVE_SIZE_sp_sem_for_sp2host_psys_event_queue 20
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_propagate_frame: 2429 */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_stop_copy_preview
-#define HIVE_MEM_sp_stop_copy_preview scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_stop_copy_preview 0x6290
-#define HIVE_SIZE_sp_stop_copy_preview 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_stop_copy_preview scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_stop_copy_preview 0x6290
-#define HIVE_SIZE_sp_sp_stop_copy_preview 4
-#else
-/* function ia_css_tagger_sp_propagate_frame: 2479 */
-#endif
-
-#ifndef ISP2401
-/* function input_system_reg_load: B17 */
-#else
-/* function input_system_reg_load: B11 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_handles
-#define HIVE_MEM_vbuf_handles scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_handles 0x6328
-#else
-#define HIVE_ADDR_vbuf_handles 0x6384
-#endif
-#define HIVE_SIZE_vbuf_handles 960
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_handles scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_handles 0x6328
-#else
-#define HIVE_ADDR_sp_vbuf_handles 0x6384
-#endif
-#define HIVE_SIZE_sp_vbuf_handles 960
-
-#ifndef ISP2401
-/* function ia_css_queue_store: 4DB3 */
-
-/* function ia_css_sp_flash_register: 2C45 */
-#else
-/* function ia_css_queue_store: 5011 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_dummy_function: 566B */
-#else
-/* function ia_css_sp_flash_register: 2DE7 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_create: 5B50 */
-#else
-/* function ia_css_isys_sp_backend_create: 5C8B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_init: 184C */
-#else
-/* function ia_css_pipeline_sp_init: 1886 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_configure: 2319 */
-#else
-/* function ia_css_tagger_sp_configure: 2369 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_end_binary: 357F */
-#else
-/* function ia_css_ispctrl_sp_end_binary: 3773 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs
-#define HIVE_MEM_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x4B60
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x4BBC
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x4B60
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 0x4BBC
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_h_pipe_private_per_frame_ddr_ptrs 20
-
-#ifndef ISP2401
-/* function receiver_port_reg_store: AC9 */
-#else
-/* function receiver_port_reg_store: AC3 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_event_is_pending_mask
-#define HIVE_MEM_event_is_pending_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_event_is_pending_mask 0x5C
-#define HIVE_SIZE_event_is_pending_mask 44
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_event_is_pending_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_event_is_pending_mask 0x5C
-#define HIVE_SIZE_sp_event_is_pending_mask 44
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cb_elems_frame
-#define HIVE_MEM_sp_all_cb_elems_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cb_elems_frame 0x46AC
-#else
-#define HIVE_ADDR_sp_all_cb_elems_frame 0x46F4
-#endif
-#define HIVE_SIZE_sp_all_cb_elems_frame 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cb_elems_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cb_elems_frame 0x46AC
-#else
-#define HIVE_ADDR_sp_sp_all_cb_elems_frame 0x46F4
-#endif
-#define HIVE_SIZE_sp_sp_all_cb_elems_frame 16
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp2host_isys_event_queue_handle
-#define HIVE_MEM_sp2host_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp2host_isys_event_queue_handle 0x4B74
-#else
-#define HIVE_ADDR_sp2host_isys_event_queue_handle 0x4BD0
-#endif
-#define HIVE_SIZE_sp2host_isys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp2host_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp2host_isys_event_queue_handle 0x4B74
-#else
-#define HIVE_ADDR_sp_sp2host_isys_event_queue_handle 0x4BD0
-#endif
-#define HIVE_SIZE_sp_sp2host_isys_event_queue_handle 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host_sp_com
-#define HIVE_MEM_host_sp_com scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host_sp_com 0x4114
-#else
-#define HIVE_ADDR_host_sp_com 0x4134
-#endif
-#define HIVE_SIZE_host_sp_com 220
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host_sp_com scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host_sp_com 0x4114
-#else
-#define HIVE_ADDR_sp_host_sp_com 0x4134
-#endif
-#define HIVE_SIZE_sp_host_sp_com 220
-
-#ifndef ISP2401
-/* function ia_css_queue_get_free_space: 4A12 */
-#else
-/* function ia_css_queue_get_free_space: 4C70 */
-#endif
-
-#ifndef ISP2401
-/* function exec_image_pipe: 6C4 */
-#else
-/* function exec_image_pipe: 658 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_init_dmem_data
-#define HIVE_MEM_sp_init_dmem_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_init_dmem_data 0x62B0
-#else
-#define HIVE_ADDR_sp_init_dmem_data 0x630C
-#endif
-#define HIVE_SIZE_sp_init_dmem_data 24
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_init_dmem_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_init_dmem_data 0x62B0
-#else
-#define HIVE_ADDR_sp_sp_init_dmem_data 0x630C
-#endif
-#define HIVE_SIZE_sp_sp_init_dmem_data 24
-
-#ifndef ISP2401
-/* function ia_css_sp_metadata_start: 592D */
-#else
-/* function ia_css_sp_metadata_start: 5A68 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_init_buffer_queues: 2CB4 */
-#else
-/* function ia_css_bufq_sp_init_buffer_queues: 2E56 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_stop: 182F */
-#else
-/* function ia_css_pipeline_sp_stop: 1869 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_connect_pipes: 2803 */
-#else
-/* function ia_css_tagger_sp_connect_pipes: 2853 */
-#endif
-
-#ifndef ISP2401
-/* function sp_isys_copy_wait: 70D */
-#else
-/* function sp_isys_copy_wait: 6A1 */
-#endif
-
-/* function is_isp_debug_buffer_full: 337 */
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_configure_channel_from_info: 32C8 */
-#else
-/* function ia_css_dmaproxy_sp_configure_channel_from_info: 34A9 */
-#endif
-
-#ifndef ISP2401
-/* function encode_and_post_timer_event: A30 */
-#else
-/* function encode_and_post_timer_event: 9C4 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_per_frame_data
-#define HIVE_MEM_sp_per_frame_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_per_frame_data 0x41F0
-#else
-#define HIVE_ADDR_sp_per_frame_data 0x4210
-#endif
-#define HIVE_SIZE_sp_per_frame_data 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_per_frame_data scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_per_frame_data 0x41F0
-#else
-#define HIVE_ADDR_sp_sp_per_frame_data 0x4210
-#endif
-#define HIVE_SIZE_sp_sp_per_frame_data 4
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_vbuf_dequeue: 62ED */
-#else
-/* function ia_css_rmgr_sp_vbuf_dequeue: 6428 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_psys_event_queue_handle
-#define HIVE_MEM_host2sp_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_psys_event_queue_handle 0x4B80
-#else
-#define HIVE_ADDR_host2sp_psys_event_queue_handle 0x4BDC
-#endif
-#define HIVE_SIZE_host2sp_psys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_psys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_psys_event_queue_handle 0x4B80
-#else
-#define HIVE_ADDR_sp_host2sp_psys_event_queue_handle 0x4BDC
-#endif
-#define HIVE_SIZE_sp_host2sp_psys_event_queue_handle 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_xmem_bin_addr
-#define HIVE_MEM_xmem_bin_addr scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_xmem_bin_addr 0x41F4
-#else
-#define HIVE_ADDR_xmem_bin_addr 0x4214
-#endif
-#define HIVE_SIZE_xmem_bin_addr 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_xmem_bin_addr scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_xmem_bin_addr 0x41F4
-#else
-#define HIVE_ADDR_sp_xmem_bin_addr 0x4214
-#endif
-#define HIVE_SIZE_sp_xmem_bin_addr 4
-
-#ifndef ISP2401
-/* function tmr_clock_init: 13FB */
-#else
-/* function tmr_clock_init: 141C */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_run: 141C */
-#else
-/* function ia_css_pipeline_sp_run: 143D */
-#endif
-
-#ifndef ISP2401
-/* function memcpy: 68F7 */
-#else
-/* function memcpy: 6A6A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_GP_DEVICE_BASE
-#define HIVE_MEM_GP_DEVICE_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_GP_DEVICE_BASE 0x2FC
-#else
-#define HIVE_ADDR_GP_DEVICE_BASE 0x314
-#endif
-#define HIVE_SIZE_GP_DEVICE_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_GP_DEVICE_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_GP_DEVICE_BASE 0x2FC
-#else
-#define HIVE_ADDR_sp_GP_DEVICE_BASE 0x314
-#endif
-#define HIVE_SIZE_sp_GP_DEVICE_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_thread_sp_ready_queue
-#define HIVE_MEM_ia_css_thread_sp_ready_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_thread_sp_ready_queue 0x1E0
-#else
-#define HIVE_ADDR_ia_css_thread_sp_ready_queue 0x1E4
-#endif
-#define HIVE_SIZE_ia_css_thread_sp_ready_queue 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_thread_sp_ready_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_thread_sp_ready_queue 0x1E0
-#else
-#define HIVE_ADDR_sp_ia_css_thread_sp_ready_queue 0x1E4
-#endif
-#define HIVE_SIZE_sp_ia_css_thread_sp_ready_queue 12
-
-#ifndef ISP2401
-/* function input_system_reg_store: B1E */
-#else
-/* function input_system_reg_store: B18 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_start: 5D66 */
-#else
-/* function ia_css_isys_sp_frontend_start: 5EA1 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_uds_sp_scale_params: 6600 */
-#else
-/* function ia_css_uds_sp_scale_params: 6773 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_increase_size: E40 */
-#else
-/* function ia_css_circbuf_increase_size: E35 */
-#endif
-
-#ifndef ISP2401
-/* function __divu: 6875 */
-#else
-/* function __divu: 69E8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_get_state: C83 */
-#else
-/* function ia_css_thread_sp_get_state: C78 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_cont_capt_stop
-#define HIVE_MEM_sem_for_cont_capt_stop scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_cont_capt_stop 0x46BC
-#else
-#define HIVE_ADDR_sem_for_cont_capt_stop 0x4704
-#endif
-#define HIVE_SIZE_sem_for_cont_capt_stop 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_cont_capt_stop scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_cont_capt_stop 0x46BC
-#else
-#define HIVE_ADDR_sp_sem_for_cont_capt_stop 0x4704
-#endif
-#define HIVE_SIZE_sp_sem_for_cont_capt_stop 20
-
-#ifndef ISP2401
-/* function thread_fiber_sp_main: E39 */
-#else
-/* function thread_fiber_sp_main: E2E */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_isp_pipe_thread
-#define HIVE_MEM_sp_isp_pipe_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_pipe_thread 0x4800
-#define HIVE_SIZE_sp_isp_pipe_thread 340
-#else
-#define HIVE_ADDR_sp_isp_pipe_thread 0x4848
-#define HIVE_SIZE_sp_isp_pipe_thread 360
-#endif
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_isp_pipe_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_isp_pipe_thread 0x4800
-#define HIVE_SIZE_sp_sp_isp_pipe_thread 340
-#else
-#define HIVE_ADDR_sp_sp_isp_pipe_thread 0x4848
-#define HIVE_SIZE_sp_sp_isp_pipe_thread 360
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_handle_parameter_sets: 128A */
-#else
-/* function ia_css_parambuf_sp_handle_parameter_sets: 127F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_set_state: 595C */
-#else
-/* function ia_css_spctrl_sp_set_state: 5A97 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sem_sp_signal: 6AF7 */
-#else
-/* function ia_css_thread_sem_sp_signal: 6C6C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_IRQ_BASE
-#define HIVE_MEM_IRQ_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_IRQ_BASE 0x2C
-#define HIVE_SIZE_IRQ_BASE 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_IRQ_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_IRQ_BASE 0x2C
-#define HIVE_SIZE_sp_IRQ_BASE 16
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_TIMED_CTRL_BASE
-#define HIVE_MEM_TIMED_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_TIMED_CTRL_BASE 0x40
-#define HIVE_SIZE_TIMED_CTRL_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_TIMED_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_TIMED_CTRL_BASE 0x40
-#define HIVE_SIZE_sp_TIMED_CTRL_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_isr: 6FDC */
-
-/* function ia_css_isys_sp_generate_exp_id: 60FE */
-#else
-/* function ia_css_isys_sp_isr: 7139 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_init: 61E8 */
-#else
-/* function ia_css_isys_sp_generate_exp_id: 6239 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sem_sp_init: 6BC8 */
-#else
-/* function ia_css_rmgr_sp_init: 6323 */
-#endif
-
-#ifndef ISP2401
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_is_isp_requested
-#define HIVE_MEM_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_is_isp_requested 0x308
-#define HIVE_SIZE_is_isp_requested 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_is_isp_requested 0x308
-#define HIVE_SIZE_sp_is_isp_requested 4
-#else
-/* function ia_css_thread_sem_sp_init: 6D3B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_reading_cb_frame
-#define HIVE_MEM_sem_for_reading_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_reading_cb_frame 0x46D0
-#else
-#define HIVE_ADDR_sem_for_reading_cb_frame 0x4718
-#endif
-#define HIVE_SIZE_sem_for_reading_cb_frame 40
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_reading_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_reading_cb_frame 0x46D0
-#else
-#define HIVE_ADDR_sp_sem_for_reading_cb_frame 0x4718
-#endif
-#define HIVE_SIZE_sp_sem_for_reading_cb_frame 40
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_execute: 3230 */
-#else
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_is_isp_requested
-#define HIVE_MEM_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_is_isp_requested 0x320
-#define HIVE_SIZE_is_isp_requested 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_is_isp_requested scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_is_isp_requested 0x320
-#define HIVE_SIZE_sp_is_isp_requested 4
-
-/* function ia_css_dmaproxy_sp_execute: 340F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_is_empty: 48F9 */
-#else
-/* function ia_css_queue_is_empty: 7098 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_has_stopped: 1825 */
-#else
-/* function ia_css_pipeline_sp_has_stopped: 185F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_extract: F44 */
-#else
-/* function ia_css_circbuf_extract: F39 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_is_locked_from_start: 2B26 */
-#else
-/* function ia_css_tagger_buf_sp_is_locked_from_start: 2CC8 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_current_sp_thread
-#define HIVE_MEM_current_sp_thread scalar_processor_2400_dmem
-#define HIVE_ADDR_current_sp_thread 0x1DC
-#define HIVE_SIZE_current_sp_thread 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_current_sp_thread scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_current_sp_thread 0x1DC
-#define HIVE_SIZE_sp_current_sp_thread 4
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_get_spid: 5963 */
-#else
-/* function ia_css_spctrl_sp_get_spid: 5A9E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_reset_buffers: 2D3B */
-#else
-/* function ia_css_bufq_sp_reset_buffers: 2EDD */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_read_byte_addr: 6E35 */
-#else
-/* function ia_css_dmaproxy_sp_read_byte_addr: 6F79 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_uninit: 61E1 */
-#else
-/* function ia_css_rmgr_sp_uninit: 631C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_threads_stack
-#define HIVE_MEM_sp_threads_stack scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_threads_stack 0x164
-#define HIVE_SIZE_sp_threads_stack 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_threads_stack scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_threads_stack 0x164
-#define HIVE_SIZE_sp_sp_threads_stack 28
-
-#ifndef ISP2401
-/* function ia_css_circbuf_peek: F26 */
-#else
-/* function ia_css_circbuf_peek: F1B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_wait_for_in_param: 1053 */
-#else
-/* function ia_css_parambuf_sp_wait_for_in_param: 1048 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_get_exp_id: 5FC6 */
-#else
-/* function ia_css_isys_sp_token_map_get_exp_id: 6101 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cb_elems_param
-#define HIVE_MEM_sp_all_cb_elems_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cb_elems_param 0x46F8
-#else
-#define HIVE_ADDR_sp_all_cb_elems_param 0x4740
-#endif
-#define HIVE_SIZE_sp_all_cb_elems_param 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cb_elems_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cb_elems_param 0x46F8
-#else
-#define HIVE_ADDR_sp_sp_all_cb_elems_param 0x4740
-#endif
-#define HIVE_SIZE_sp_sp_all_cb_elems_param 16
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_pipeline_sp_curr_binary_id
-#define HIVE_MEM_pipeline_sp_curr_binary_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_pipeline_sp_curr_binary_id 0x1EC
-#else
-#define HIVE_ADDR_pipeline_sp_curr_binary_id 0x1F0
-#endif
-#define HIVE_SIZE_pipeline_sp_curr_binary_id 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_pipeline_sp_curr_binary_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_pipeline_sp_curr_binary_id 0x1EC
-#else
-#define HIVE_ADDR_sp_pipeline_sp_curr_binary_id 0x1F0
-#endif
-#define HIVE_SIZE_sp_pipeline_sp_curr_binary_id 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_frame_desc
-#define HIVE_MEM_sp_all_cbs_frame_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_frame_desc 0x4708
-#else
-#define HIVE_ADDR_sp_all_cbs_frame_desc 0x4750
-#endif
-#define HIVE_SIZE_sp_all_cbs_frame_desc 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_frame_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_frame_desc 0x4708
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_frame_desc 0x4750
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_frame_desc 8
-
-#ifndef ISP2401
-/* function sp_isys_copy_func_v2: 706 */
-#else
-/* function sp_isys_copy_func_v2: 69A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_reading_cb_param
-#define HIVE_MEM_sem_for_reading_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_reading_cb_param 0x4710
-#else
-#define HIVE_ADDR_sem_for_reading_cb_param 0x4758
-#endif
-#define HIVE_SIZE_sem_for_reading_cb_param 40
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_reading_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_reading_cb_param 0x4710
-#else
-#define HIVE_ADDR_sp_sem_for_reading_cb_param 0x4758
-#endif
-#define HIVE_SIZE_sp_sem_for_reading_cb_param 40
-
-#ifndef ISP2401
-/* function ia_css_queue_get_used_space: 49C6 */
-#else
-/* function ia_css_queue_get_used_space: 4C24 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_cont_capt_start
-#define HIVE_MEM_sem_for_cont_capt_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_cont_capt_start 0x4738
-#else
-#define HIVE_ADDR_sem_for_cont_capt_start 0x4780
-#endif
-#define HIVE_SIZE_sem_for_cont_capt_start 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_cont_capt_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_cont_capt_start 0x4738
-#else
-#define HIVE_ADDR_sp_sem_for_cont_capt_start 0x4780
-#endif
-#define HIVE_SIZE_sp_sem_for_cont_capt_start 20
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_tmp_heap
-#define HIVE_MEM_tmp_heap scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_tmp_heap 0x6010
-#else
-#define HIVE_ADDR_tmp_heap 0x6070
-#endif
-#define HIVE_SIZE_tmp_heap 640
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_tmp_heap scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_tmp_heap 0x6010
-#else
-#define HIVE_ADDR_sp_tmp_heap 0x6070
-#endif
-#define HIVE_SIZE_sp_tmp_heap 640
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_get_num_vbuf: 64F1 */
-#else
-/* function ia_css_rmgr_sp_get_num_vbuf: 662C */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_output_compute_dma_info: 3F62 */
-#else
-/* function ia_css_ispctrl_sp_output_compute_dma_info: 41A5 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_lock_exp_id: 20E6 */
-#else
-/* function ia_css_tagger_sp_lock_exp_id: 2136 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_s3a_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_s3a_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_s3a_bufs 0x4B8C
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_s3a_bufs 0x4BE8
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_s3a_bufs 60
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_s3a_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_s3a_bufs 0x4B8C
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_s3a_bufs 0x4BE8
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_s3a_bufs 60
-
-#ifndef ISP2401
-/* function ia_css_queue_is_full: 4A5D */
-#else
-/* function ia_css_queue_is_full: 4CBB */
-#endif
-
-/* function debug_buffer_init_isp: E4 */
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_uninit: 5D20 */
-#else
-/* function ia_css_isys_sp_frontend_uninit: 5E5B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_exp_id_is_locked: 201C */
-#else
-/* function ia_css_tagger_sp_exp_id_is_locked: 206C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_rmgr_sp_mipi_frame_sem
-#define HIVE_MEM_ia_css_rmgr_sp_mipi_frame_sem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_rmgr_sp_mipi_frame_sem 0x66E8
-#else
-#define HIVE_ADDR_ia_css_rmgr_sp_mipi_frame_sem 0x6744
-#endif
-#define HIVE_SIZE_ia_css_rmgr_sp_mipi_frame_sem 60
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_rmgr_sp_mipi_frame_sem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_rmgr_sp_mipi_frame_sem 0x66E8
-#else
-#define HIVE_ADDR_sp_ia_css_rmgr_sp_mipi_frame_sem 0x6744
-#endif
-#define HIVE_SIZE_sp_ia_css_rmgr_sp_mipi_frame_sem 60
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_dump: 62C8 */
-#else
-/* function ia_css_rmgr_sp_refcount_dump: 6403 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_isp_parameters_id
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_isp_parameters_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x4BC8
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x4C24
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_isp_parameters_id 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x4BC8
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id 0x4C24
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_isp_parameters_id 20
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_pipe_threads
-#define HIVE_MEM_sp_pipe_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_pipe_threads 0x150
-#define HIVE_SIZE_sp_pipe_threads 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_pipe_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_pipe_threads 0x150
-#define HIVE_SIZE_sp_sp_pipe_threads 20
-
-#ifndef ISP2401
-/* function sp_event_proxy_func: 71B */
-#else
-/* function sp_event_proxy_func: 6AF */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_isys_event_queue_handle
-#define HIVE_MEM_host2sp_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_isys_event_queue_handle 0x4BDC
-#else
-#define HIVE_ADDR_host2sp_isys_event_queue_handle 0x4C38
-#endif
-#define HIVE_SIZE_host2sp_isys_event_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_isys_event_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_isys_event_queue_handle 0x4BDC
-#else
-#define HIVE_ADDR_sp_host2sp_isys_event_queue_handle 0x4C38
-#endif
-#define HIVE_SIZE_sp_host2sp_isys_event_queue_handle 12
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_yield: 6A70 */
-#else
-/* function ia_css_thread_sp_yield: 6BEA */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_param_desc
-#define HIVE_MEM_sp_all_cbs_param_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_param_desc 0x474C
-#else
-#define HIVE_ADDR_sp_all_cbs_param_desc 0x4794
-#endif
-#define HIVE_SIZE_sp_all_cbs_param_desc 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_param_desc scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_param_desc 0x474C
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_param_desc 0x4794
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_param_desc 8
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_dmaproxy_sp_invalidate_tlb
-#define HIVE_MEM_ia_css_dmaproxy_sp_invalidate_tlb scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb 0x5BF4
-#else
-#define HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb 0x5C50
-#endif
-#define HIVE_SIZE_ia_css_dmaproxy_sp_invalidate_tlb 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_dmaproxy_sp_invalidate_tlb scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_dmaproxy_sp_invalidate_tlb 0x5BF4
-#else
-#define HIVE_ADDR_sp_ia_css_dmaproxy_sp_invalidate_tlb 0x5C50
-#endif
-#define HIVE_SIZE_sp_ia_css_dmaproxy_sp_invalidate_tlb 4
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_fork: D10 */
-#else
-/* function ia_css_thread_sp_fork: D05 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_destroy: 280D */
-#else
-/* function ia_css_tagger_sp_destroy: 285D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_vmem_read: 31D0 */
-#else
-/* function ia_css_dmaproxy_sp_vmem_read: 33AF */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ifmtr_sp_init: 614F */
-#else
-/* function ia_css_ifmtr_sp_init: 628A */
-#endif
-
-#ifndef ISP2401
-/* function initialize_sp_group: 6D4 */
-#else
-/* function initialize_sp_group: 668 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_peek: 2932 */
-#else
-/* function ia_css_tagger_buf_sp_peek: 2AD4 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_init: D3C */
-#else
-/* function ia_css_thread_sp_init: D31 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_reset_exp_id: 60F6 */
-#else
-/* function ia_css_isys_sp_reset_exp_id: 6231 */
-#endif
-
-#ifndef ISP2401
-/* function qos_scheduler_update_fps: 65F0 */
-#else
-/* function qos_scheduler_update_fps: 6763 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_set_stream_base_addr: 4637 */
-#else
-/* function ia_css_ispctrl_sp_set_stream_base_addr: 4892 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_DMEM_BASE
-#define HIVE_MEM_ISP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_DMEM_BASE 0x10
-#define HIVE_SIZE_ISP_DMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_DMEM_BASE 0x10
-#define HIVE_SIZE_sp_ISP_DMEM_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_SP_DMEM_BASE
-#define HIVE_MEM_SP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_SP_DMEM_BASE 0x4
-#define HIVE_SIZE_SP_DMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_SP_DMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_SP_DMEM_BASE 0x4
-#define HIVE_SIZE_sp_SP_DMEM_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_read: 3246 */
-#else
-/* function __ia_css_queue_is_empty_text: 4B81 */
-
-/* function ia_css_dmaproxy_sp_read: 3425 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_raw_copy_line_count
-#define HIVE_MEM_raw_copy_line_count scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_raw_copy_line_count 0x2C8
-#else
-#define HIVE_ADDR_raw_copy_line_count 0x2E0
-#endif
-#define HIVE_SIZE_raw_copy_line_count 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_raw_copy_line_count scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_raw_copy_line_count 0x2C8
-#else
-#define HIVE_ADDR_sp_raw_copy_line_count 0x2E0
-#endif
-#define HIVE_SIZE_sp_raw_copy_line_count 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_tag_cmd_queue_handle
-#define HIVE_MEM_host2sp_tag_cmd_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_tag_cmd_queue_handle 0x4BE8
-#else
-#define HIVE_ADDR_host2sp_tag_cmd_queue_handle 0x4C44
-#endif
-#define HIVE_SIZE_host2sp_tag_cmd_queue_handle 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_tag_cmd_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_tag_cmd_queue_handle 0x4BE8
-#else
-#define HIVE_ADDR_sp_host2sp_tag_cmd_queue_handle 0x4C44
-#endif
-#define HIVE_SIZE_sp_host2sp_tag_cmd_queue_handle 12
-
-#ifndef ISP2401
-/* function ia_css_queue_peek: 493C */
-#else
-/* function ia_css_queue_peek: 4B9A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_frame_cnt
-#define HIVE_MEM_ia_css_flash_sp_frame_cnt scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_frame_cnt 0x4A94
-#else
-#define HIVE_ADDR_ia_css_flash_sp_frame_cnt 0x4AF0
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_frame_cnt 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_frame_cnt scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_frame_cnt 0x4A94
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_frame_cnt 0x4AF0
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_frame_cnt 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_event_can_send_token_mask
-#define HIVE_MEM_event_can_send_token_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_event_can_send_token_mask 0x88
-#define HIVE_SIZE_event_can_send_token_mask 44
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_event_can_send_token_mask scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_event_can_send_token_mask 0x88
-#define HIVE_SIZE_sp_event_can_send_token_mask 44
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_thread
-#define HIVE_MEM_isp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_thread 0x5F40
-#else
-#define HIVE_ADDR_isp_thread 0x5FA0
-#endif
-#define HIVE_SIZE_isp_thread 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_thread 0x5F40
-#else
-#define HIVE_ADDR_sp_isp_thread 0x5FA0
-#endif
-#define HIVE_SIZE_sp_isp_thread 4
-
-#ifndef ISP2401
-/* function encode_and_post_sp_event_non_blocking: A78 */
-#else
-/* function encode_and_post_sp_event_non_blocking: A0C */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_destroy: 5DF8 */
-#else
-/* function ia_css_isys_sp_frontend_destroy: 5F33 */
-#endif
-
-/* function is_ddr_debug_buffer_full: 2CC */
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_stop: 5D38 */
-#else
-/* function ia_css_isys_sp_frontend_stop: 5E73 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_init: 6094 */
-#else
-/* function ia_css_isys_sp_token_map_init: 61CF */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_get_oldest_marked_offset: 2982 */
-#else
-/* function ia_css_tagger_buf_sp_get_oldest_marked_offset: 2B24 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_threads_fiber
-#define HIVE_MEM_sp_threads_fiber scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_threads_fiber 0x19C
-#define HIVE_SIZE_sp_threads_fiber 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_threads_fiber scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_threads_fiber 0x19C
-#define HIVE_SIZE_sp_sp_threads_fiber 28
-
-#ifndef ISP2401
-/* function encode_and_post_sp_event: A01 */
-#else
-/* function encode_and_post_sp_event: 995 */
-#endif
-
-/* function debug_enqueue_ddr: EE */
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_init_vbuf: 6283 */
-#else
-/* function ia_css_rmgr_sp_refcount_init_vbuf: 63BE */
-#endif
-
-#ifndef ISP2401
-/* function dmaproxy_sp_read_write: 6EE4 */
-#else
-/* function dmaproxy_sp_read_write: 7017 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_dmaproxy_isp_dma_cmd_buffer
-#define HIVE_MEM_ia_css_dmaproxy_isp_dma_cmd_buffer scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_dmaproxy_isp_dma_cmd_buffer 0x5BF8
-#else
-#define HIVE_ADDR_ia_css_dmaproxy_isp_dma_cmd_buffer 0x5C54
-#endif
-#define HIVE_SIZE_ia_css_dmaproxy_isp_dma_cmd_buffer 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_dmaproxy_isp_dma_cmd_buffer scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_dmaproxy_isp_dma_cmd_buffer 0x5BF8
-#else
-#define HIVE_ADDR_sp_ia_css_dmaproxy_isp_dma_cmd_buffer 0x5C54
-#endif
-#define HIVE_SIZE_sp_ia_css_dmaproxy_isp_dma_cmd_buffer 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host2sp_buffer_queue_handle
-#define HIVE_MEM_host2sp_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host2sp_buffer_queue_handle 0x4BF4
-#else
-#define HIVE_ADDR_host2sp_buffer_queue_handle 0x4C50
-#endif
-#define HIVE_SIZE_host2sp_buffer_queue_handle 480
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host2sp_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host2sp_buffer_queue_handle 0x4BF4
-#else
-#define HIVE_ADDR_sp_host2sp_buffer_queue_handle 0x4C50
-#endif
-#define HIVE_SIZE_sp_host2sp_buffer_queue_handle 480
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_in_service
-#define HIVE_MEM_ia_css_flash_sp_in_service scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_in_service 0x3178
-#else
-#define HIVE_ADDR_ia_css_flash_sp_in_service 0x3198
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_in_service 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_in_service scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_service 0x3178
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_service 0x3198
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_in_service 4
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_process: 6BF0 */
-#else
-/* function ia_css_dmaproxy_sp_process: 6D63 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_mark_from_end: 2C0A */
-#else
-/* function ia_css_tagger_buf_sp_mark_from_end: 2DAC */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_rcv_acquire_ack: 5A05 */
-#else
-/* function ia_css_isys_sp_backend_rcv_acquire_ack: 5B40 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_pre_acquire_request: 5A1B */
-#else
-/* function ia_css_isys_sp_backend_pre_acquire_request: 5B56 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_cs: 366C */
-#else
-/* function ia_css_ispctrl_sp_init_cs: 386E */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_init: 5971 */
-#else
-/* function ia_css_spctrl_sp_init: 5AAC */
-#endif
-
-#ifndef ISP2401
-/* function sp_event_proxy_init: 730 */
-#else
-/* function sp_event_proxy_init: 6C4 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_previous_clock_tick
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_previous_clock_tick scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x4DD4
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x4E30
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_previous_clock_tick 40
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x4DD4
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick 0x4E30
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_previous_clock_tick 40
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_output
-#define HIVE_MEM_sp_output scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_output 0x41F8
-#else
-#define HIVE_ADDR_sp_output 0x4218
-#endif
-#define HIVE_SIZE_sp_output 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_output scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_output 0x41F8
-#else
-#define HIVE_ADDR_sp_sp_output 0x4218
-#endif
-#define HIVE_SIZE_sp_sp_output 16
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_sems_for_host2sp_buf_queues
-#define HIVE_MEM_ia_css_bufq_sp_sems_for_host2sp_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x4DFC
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x4E58
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_sems_for_host2sp_buf_queues 800
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x4DFC
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues 0x4E58
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_sems_for_host2sp_buf_queues 800
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_CTRL_BASE
-#define HIVE_MEM_ISP_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_CTRL_BASE 0x8
-#define HIVE_SIZE_ISP_CTRL_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_CTRL_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_CTRL_BASE 0x8
-#define HIVE_SIZE_sp_ISP_CTRL_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_INPUT_FORMATTER_BASE
-#define HIVE_MEM_INPUT_FORMATTER_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_INPUT_FORMATTER_BASE 0x4C
-#define HIVE_SIZE_INPUT_FORMATTER_BASE 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_INPUT_FORMATTER_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_INPUT_FORMATTER_BASE 0x4C
-#define HIVE_SIZE_sp_INPUT_FORMATTER_BASE 16
-
-#ifndef ISP2401
-/* function sp_dma_proxy_reset_channels: 34A0 */
-#else
-/* function sp_dma_proxy_reset_channels: 3694 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_acquire: 5B26 */
-#else
-/* function ia_css_isys_sp_backend_acquire: 5C61 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_update_size: 2901 */
-#else
-/* function ia_css_tagger_sp_update_size: 2AA3 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_host_sp_queue
-#define HIVE_MEM_ia_css_bufq_host_sp_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_host_sp_queue 0x511C
-#else
-#define HIVE_ADDR_ia_css_bufq_host_sp_queue 0x5178
-#endif
-#define HIVE_SIZE_ia_css_bufq_host_sp_queue 2008
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_host_sp_queue scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_host_sp_queue 0x511C
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_host_sp_queue 0x5178
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_host_sp_queue 2008
-
-#ifndef ISP2401
-/* function thread_fiber_sp_create: DA8 */
-#else
-/* function thread_fiber_sp_create: D9D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_increments: 3332 */
-#else
-/* function ia_css_dmaproxy_sp_set_increments: 3526 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_writing_cb_frame
-#define HIVE_MEM_sem_for_writing_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_writing_cb_frame 0x4754
-#else
-#define HIVE_ADDR_sem_for_writing_cb_frame 0x479C
-#endif
-#define HIVE_SIZE_sem_for_writing_cb_frame 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_writing_cb_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_writing_cb_frame 0x4754
-#else
-#define HIVE_ADDR_sp_sem_for_writing_cb_frame 0x479C
-#endif
-#define HIVE_SIZE_sp_sem_for_writing_cb_frame 20
-
-#ifndef ISP2401
-/* function receiver_reg_store: AD7 */
-#else
-/* function receiver_reg_store: AD1 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_writing_cb_param
-#define HIVE_MEM_sem_for_writing_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_writing_cb_param 0x4768
-#else
-#define HIVE_ADDR_sem_for_writing_cb_param 0x47B0
-#endif
-#define HIVE_SIZE_sem_for_writing_cb_param 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_writing_cb_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_writing_cb_param 0x4768
-#else
-#define HIVE_ADDR_sp_sem_for_writing_cb_param 0x47B0
-#endif
-#define HIVE_SIZE_sp_sem_for_writing_cb_param 20
-
-/* function sp_start_isp_entry: 453 */
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifdef HIVE_ADDR_sp_start_isp_entry
-#endif
-#define HIVE_ADDR_sp_start_isp_entry 0x453
-#endif
-#define HIVE_ADDR_sp_sp_start_isp_entry 0x453
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_unmark_all: 2B8E */
-#else
-/* function ia_css_tagger_buf_sp_unmark_all: 2D30 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_unmark_from_start: 2BCF */
-#else
-/* function ia_css_tagger_buf_sp_unmark_from_start: 2D71 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_channel_acquire: 34CC */
-#else
-/* function ia_css_dmaproxy_sp_channel_acquire: 36C0 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_add_num_vbuf: 64CD */
-#else
-/* function ia_css_rmgr_sp_add_num_vbuf: 6608 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_create: 60DD */
-#else
-/* function ia_css_isys_sp_token_map_create: 6218 */
-#endif
-
-#ifndef ISP2401
-/* function __ia_css_dmaproxy_sp_wait_for_ack_text: 319C */
-#else
-/* function __ia_css_dmaproxy_sp_wait_for_ack_text: 337B */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_acquire_buf_elem: 1FF4 */
-#else
-/* function ia_css_tagger_sp_acquire_buf_elem: 2044 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_is_dynamic_buffer: 3085 */
-#else
-/* function ia_css_bufq_sp_is_dynamic_buffer: 3227 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_group
-#define HIVE_MEM_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_group 0x4208
-#define HIVE_SIZE_sp_group 1144
-#else
-#define HIVE_ADDR_sp_group 0x4228
-#define HIVE_SIZE_sp_group 1184
-#endif
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_group 0x4208
-#define HIVE_SIZE_sp_sp_group 1144
-#else
-#define HIVE_ADDR_sp_sp_group 0x4228
-#define HIVE_SIZE_sp_sp_group 1184
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_event_proxy_thread
-#define HIVE_MEM_sp_event_proxy_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_event_proxy_thread 0x4954
-#define HIVE_SIZE_sp_event_proxy_thread 68
-#else
-#define HIVE_ADDR_sp_event_proxy_thread 0x49B0
-#define HIVE_SIZE_sp_event_proxy_thread 72
-#endif
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_event_proxy_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_event_proxy_thread 0x4954
-#define HIVE_SIZE_sp_sp_event_proxy_thread 68
-#else
-#define HIVE_ADDR_sp_sp_event_proxy_thread 0x49B0
-#define HIVE_SIZE_sp_sp_event_proxy_thread 72
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_kill: CD6 */
-#else
-/* function ia_css_thread_sp_kill: CCB */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_create: 28BB */
-#else
-/* function ia_css_tagger_sp_create: 2A51 */
-#endif
-
-#ifndef ISP2401
-/* function tmpmem_acquire_dmem: 657A */
-#else
-/* function tmpmem_acquire_dmem: 66B5 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_MMU_BASE
-#define HIVE_MEM_MMU_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_MMU_BASE 0x24
-#define HIVE_SIZE_MMU_BASE 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_MMU_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_MMU_BASE 0x24
-#define HIVE_SIZE_sp_MMU_BASE 8
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_channel_release: 34B8 */
-#else
-/* function ia_css_dmaproxy_sp_channel_release: 36AC */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_is_idle: 3498 */
-#else
-/* function ia_css_dmaproxy_sp_is_idle: 368C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_qos_start
-#define HIVE_MEM_sem_for_qos_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_qos_start 0x477C
-#else
-#define HIVE_ADDR_sem_for_qos_start 0x47C4
-#endif
-#define HIVE_SIZE_sem_for_qos_start 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_qos_start scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_qos_start 0x477C
-#else
-#define HIVE_ADDR_sp_sem_for_qos_start 0x47C4
-#endif
-#define HIVE_SIZE_sp_sem_for_qos_start 20
-
-#ifndef ISP2401
-/* function isp_hmem_load: B55 */
-#else
-/* function isp_hmem_load: B4F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_release_buf_elem: 1FD0 */
-#else
-/* function ia_css_tagger_sp_release_buf_elem: 2020 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_eventq_sp_send: 350E */
-#else
-/* function ia_css_eventq_sp_send: 3702 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_isys_sp_error_cnt
-#define HIVE_MEM_ia_css_isys_sp_error_cnt scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_isys_sp_error_cnt 0x62D4
-#else
-#define HIVE_ADDR_ia_css_isys_sp_error_cnt 0x6330
-#endif
-#define HIVE_SIZE_ia_css_isys_sp_error_cnt 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_isys_sp_error_cnt scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_isys_sp_error_cnt 0x62D4
-#else
-#define HIVE_ADDR_sp_ia_css_isys_sp_error_cnt 0x6330
-#endif
-#define HIVE_SIZE_sp_ia_css_isys_sp_error_cnt 16
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_unlock_from_start: 2ABE */
-#else
-/* function ia_css_tagger_buf_sp_unlock_from_start: 2C60 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_debug_buffer_ddr_address
-#define HIVE_MEM_debug_buffer_ddr_address scalar_processor_2400_dmem
-#define HIVE_ADDR_debug_buffer_ddr_address 0xBC
-#define HIVE_SIZE_debug_buffer_ddr_address 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_debug_buffer_ddr_address scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_debug_buffer_ddr_address 0xBC
-#define HIVE_SIZE_sp_debug_buffer_ddr_address 4
-
-#ifndef ISP2401
-/* function sp_isys_copy_request: 714 */
-#else
-/* function sp_isys_copy_request: 6A8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_retain_vbuf: 635D */
-#else
-/* function ia_css_rmgr_sp_refcount_retain_vbuf: 6498 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_set_priority: CCE */
-#else
-/* function ia_css_thread_sp_set_priority: CC3 */
-#endif
-
-#ifndef ISP2401
-/* function sizeof_hmem: BFC */
-#else
-/* function sizeof_hmem: BF6 */
-#endif
-
-#ifndef ISP2401
-/* function tmpmem_release_dmem: 6569 */
-#else
-/* function tmpmem_release_dmem: 66A4 */
-#endif
-
-/* function cnd_input_system_cfg: 392 */
-
-#ifndef ISP2401
-/* function __ia_css_sp_rawcopy_func_critical: 6F65 */
-#else
-/* function __ia_css_sp_rawcopy_func_critical: 70C2 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_width_exception: 331D */
-#else
-/* function __ia_css_dmaproxy_sp_process_text: 331F */
-#endif
-
-#ifndef ISP2401
-/* function sp_event_assert: 8B1 */
-#else
-/* function ia_css_dmaproxy_sp_set_width_exception: 3511 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_flash_sp_init_internal_params: 2CA9 */
-#else
-/* function sp_event_assert: 845 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_pop_unmarked_and_unlocked: 29C4 */
-#else
-/* function ia_css_flash_sp_init_internal_params: 2E4B */
-#endif
-
-#ifndef ISP2401
-/* function __modu: 68BB */
-#else
-/* function ia_css_tagger_buf_sp_pop_unmarked_and_unlocked: 2B66 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init_isp_vector: 31A2 */
-#else
-/* function __modu: 6A2E */
-
-/* function ia_css_dmaproxy_sp_init_isp_vector: 3381 */
-#endif
-
-/* function isp_vamem_store: 0 */
-
-#ifdef ISP2401
-/* function ia_css_tagger_sp_set_copy_pipe: 2A48 */
-
-#endif
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_GDC_BASE
-#define HIVE_MEM_GDC_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_GDC_BASE 0x44
-#define HIVE_SIZE_GDC_BASE 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_GDC_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_GDC_BASE 0x44
-#define HIVE_SIZE_sp_GDC_BASE 8
-
-#ifndef ISP2401
-/* function ia_css_queue_local_init: 4C27 */
-#else
-/* function ia_css_queue_local_init: 4E85 */
-#endif
-
-#ifndef ISP2401
-/* function sp_event_proxy_callout_func: 6988 */
-#else
-/* function sp_event_proxy_callout_func: 6AFB */
-#endif
-
-#ifndef ISP2401
-/* function qos_scheduler_schedule_stage: 65C1 */
-#else
-/* function qos_scheduler_schedule_stage: 670F */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_thread_sp_num_ready_threads
-#define HIVE_MEM_ia_css_thread_sp_num_ready_threads scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_thread_sp_num_ready_threads 0x49E0
-#else
-#define HIVE_ADDR_ia_css_thread_sp_num_ready_threads 0x4A40
-#endif
-#define HIVE_SIZE_ia_css_thread_sp_num_ready_threads 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_thread_sp_num_ready_threads scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_thread_sp_num_ready_threads 0x49E0
-#else
-#define HIVE_ADDR_sp_ia_css_thread_sp_num_ready_threads 0x4A40
-#endif
-#define HIVE_SIZE_sp_ia_css_thread_sp_num_ready_threads 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_threads_stack_size
-#define HIVE_MEM_sp_threads_stack_size scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_threads_stack_size 0x180
-#define HIVE_SIZE_sp_threads_stack_size 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_threads_stack_size scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_threads_stack_size 0x180
-#define HIVE_SIZE_sp_sp_threads_stack_size 28
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_isp_done_row_striping: 3F48 */
-#else
-/* function ia_css_ispctrl_sp_isp_done_row_striping: 418B */
-#endif
-
-#ifndef ISP2401
-/* function __ia_css_isys_sp_isr_text: 5E22 */
-#else
-/* function __ia_css_isys_sp_isr_text: 5F5D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_dequeue: 4AA5 */
-#else
-/* function ia_css_queue_dequeue: 4D03 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_configure_channel: 6E4C */
-#else
-/* function is_qos_standalone_mode: 66EA */
-
-/* function ia_css_dmaproxy_sp_configure_channel: 6F90 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_current_thread_fiber_sp
-#define HIVE_MEM_current_thread_fiber_sp scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_current_thread_fiber_sp 0x49E8
-#else
-#define HIVE_ADDR_current_thread_fiber_sp 0x4A44
-#endif
-#define HIVE_SIZE_current_thread_fiber_sp 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_current_thread_fiber_sp scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_current_thread_fiber_sp 0x49E8
-#else
-#define HIVE_ADDR_sp_current_thread_fiber_sp 0x4A44
-#endif
-#define HIVE_SIZE_sp_current_thread_fiber_sp 4
-
-#ifndef ISP2401
-/* function ia_css_circbuf_pop: FD8 */
-#else
-/* function ia_css_circbuf_pop: FCD */
-#endif
-
-#ifndef ISP2401
-/* function memset: 693A */
-#else
-/* function memset: 6AAD */
-#endif
-
-/* function irq_raise_set_token: B6 */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_GPIO_BASE
-#define HIVE_MEM_GPIO_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_GPIO_BASE 0x3C
-#define HIVE_SIZE_GPIO_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_GPIO_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_GPIO_BASE 0x3C
-#define HIVE_SIZE_sp_GPIO_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_pipeline_acc_stage_enable: 17F0 */
-#else
-/* function ia_css_pipeline_acc_stage_enable: 1818 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_unlock_exp_id: 2041 */
-#else
-/* function ia_css_tagger_sp_unlock_exp_id: 2091 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_ph
-#define HIVE_MEM_isp_ph scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_ph 0x62E4
-#else
-#define HIVE_ADDR_isp_ph 0x6340
-#endif
-#define HIVE_SIZE_isp_ph 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_ph scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_ph 0x62E4
-#else
-#define HIVE_ADDR_sp_isp_ph 0x6340
-#endif
-#define HIVE_SIZE_sp_isp_ph 28
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_flush: 6022 */
-#else
-/* function ia_css_isys_sp_token_map_flush: 615D */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_ds: 37CB */
-#else
-/* function ia_css_ispctrl_sp_init_ds: 39FA */
-#endif
-
-#ifndef ISP2401
-/* function get_xmem_base_addr_raw: 3B78 */
-#else
-/* function get_xmem_base_addr_raw: 3DB3 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_param
-#define HIVE_MEM_sp_all_cbs_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_param 0x4790
-#else
-#define HIVE_ADDR_sp_all_cbs_param 0x47D8
-#endif
-#define HIVE_SIZE_sp_all_cbs_param 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_param scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_param 0x4790
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_param 0x47D8
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_param 16
-
-#ifndef ISP2401
-/* function ia_css_circbuf_create: 1026 */
-#else
-/* function ia_css_circbuf_create: 101B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_sp_group
-#define HIVE_MEM_sem_for_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_sp_group 0x47A0
-#else
-#define HIVE_ADDR_sem_for_sp_group 0x47E8
-#endif
-#define HIVE_SIZE_sem_for_sp_group 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_sp_group scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_sp_group 0x47A0
-#else
-#define HIVE_ADDR_sp_sem_for_sp_group 0x47E8
-#endif
-#define HIVE_SIZE_sp_sem_for_sp_group 20
-
-#ifndef ISP2401
-/* function ia_css_framebuf_sp_wait_for_in_frame: 64F8 */
-#else
-/* function __ia_css_dmaproxy_sp_configure_channel_text: 34F0 */
-
-/* function ia_css_framebuf_sp_wait_for_in_frame: 6633 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_tag_frame: 5588 */
-#else
-/* function ia_css_sp_rawcopy_tag_frame: 57C9 */
-#endif
-
-#ifndef ISP2401
-/* function isp_hmem_clear: B25 */
-#else
-/* function isp_hmem_clear: B1F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_framebuf_sp_release_in_frame: 653B */
-#else
-/* function ia_css_framebuf_sp_release_in_frame: 6676 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_snd_acquire_request: 5A78 */
-#else
-/* function ia_css_isys_sp_backend_snd_acquire_request: 5BB3 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_is_full: 5EA9 */
-#else
-/* function ia_css_isys_sp_token_map_is_full: 5FE4 */
-#endif
-
-#ifndef ISP2401
-/* function input_system_acquisition_run: AF9 */
-#else
-/* function input_system_acquisition_run: AF3 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_start_binary: 364A */
-#else
-/* function ia_css_ispctrl_sp_start_binary: 384C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_h_pipe_private_ddr_ptrs
-#define HIVE_MEM_ia_css_bufq_sp_h_pipe_private_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x58F4
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x5950
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x58F4
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 0x5950
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_h_pipe_private_ddr_ptrs 20
-
-#ifndef ISP2401
-/* function ia_css_eventq_sp_recv: 34E0 */
-#else
-/* function ia_css_eventq_sp_recv: 36D4 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_pool
-#define HIVE_MEM_isp_pool scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_pool 0x2E8
-#else
-#define HIVE_ADDR_isp_pool 0x300
-#endif
-#define HIVE_SIZE_isp_pool 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_pool scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_pool 0x2E8
-#else
-#define HIVE_ADDR_sp_isp_pool 0x300
-#endif
-#define HIVE_SIZE_sp_isp_pool 4
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_rel_gen: 622A */
-#else
-/* function ia_css_rmgr_sp_rel_gen: 6365 */
-
-/* function ia_css_tagger_sp_unblock_clients: 2919 */
-#endif
-
-#ifndef ISP2401
-/* function css_get_frame_processing_time_end: 1FC0 */
-#else
-/* function css_get_frame_processing_time_end: 2010 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_event_any_pending_mask
-#define HIVE_MEM_event_any_pending_mask scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_event_any_pending_mask 0x300
-#else
-#define HIVE_ADDR_event_any_pending_mask 0x318
-#endif
-#define HIVE_SIZE_event_any_pending_mask 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_event_any_pending_mask scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_event_any_pending_mask 0x300
-#else
-#define HIVE_ADDR_sp_event_any_pending_mask 0x318
-#endif
-#define HIVE_SIZE_sp_event_any_pending_mask 8
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_push: 5A2F */
-#else
-/* function ia_css_isys_sp_backend_push: 5B6A */
-#endif
-
-/* function sh_css_decode_tag_descr: 352 */
-
-/* function debug_enqueue_isp: 27B */
-
-#ifndef ISP2401
-/* function qos_scheduler_update_stage_budget: 65AF */
-#else
-/* function qos_scheduler_update_stage_budget: 66F2 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_uninit: 596A */
-#else
-/* function ia_css_spctrl_sp_uninit: 5AA5 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_HIVE_IF_SWITCH_CODE
-#define HIVE_MEM_HIVE_IF_SWITCH_CODE scalar_processor_2400_dmem
-#define HIVE_ADDR_HIVE_IF_SWITCH_CODE 0x1D8
-#define HIVE_SIZE_HIVE_IF_SWITCH_CODE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_HIVE_IF_SWITCH_CODE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_HIVE_IF_SWITCH_CODE 0x1D8
-#define HIVE_SIZE_sp_HIVE_IF_SWITCH_CODE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_dis_bufs
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_dis_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_dis_bufs 0x5908
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_dis_bufs 0x5964
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_dis_bufs 140
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_dis_bufs scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_dis_bufs 0x5908
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_dis_bufs 0x5964
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_dis_bufs 140
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_lock_from_start: 2AF2 */
-#else
-/* function ia_css_tagger_buf_sp_lock_from_start: 2C94 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_isp_idle
-#define HIVE_MEM_sem_for_isp_idle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_isp_idle 0x47B4
-#else
-#define HIVE_ADDR_sem_for_isp_idle 0x47FC
-#endif
-#define HIVE_SIZE_sem_for_isp_idle 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_isp_idle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_isp_idle 0x47B4
-#else
-#define HIVE_ADDR_sp_sem_for_isp_idle 0x47FC
-#endif
-#define HIVE_SIZE_sp_sem_for_isp_idle 20
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_write_byte_addr: 31FF */
-#else
-/* function ia_css_dmaproxy_sp_write_byte_addr: 33DE */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init: 3176 */
-#else
-/* function ia_css_dmaproxy_sp_init: 3355 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_release_dynamic_buf_clock_tick: 2D7B */
-#else
-/* function ia_css_bufq_sp_release_dynamic_buf_clock_tick: 2F1D */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_VAMEM_BASE
-#define HIVE_MEM_ISP_VAMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_VAMEM_BASE 0x14
-#define HIVE_SIZE_ISP_VAMEM_BASE 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_VAMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_VAMEM_BASE 0x14
-#define HIVE_SIZE_sp_ISP_VAMEM_BASE 12
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_rawcopy_sp_tagger
-#define HIVE_MEM_ia_css_rawcopy_sp_tagger scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_rawcopy_sp_tagger 0x6294
-#else
-#define HIVE_ADDR_ia_css_rawcopy_sp_tagger 0x62F0
-#endif
-#define HIVE_SIZE_ia_css_rawcopy_sp_tagger 24
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_rawcopy_sp_tagger scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_rawcopy_sp_tagger 0x6294
-#else
-#define HIVE_ADDR_sp_ia_css_rawcopy_sp_tagger 0x62F0
-#endif
-#define HIVE_SIZE_sp_ia_css_rawcopy_sp_tagger 24
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_exp_ids
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_exp_ids scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_exp_ids 0x5994
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_exp_ids 0x59F0
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_exp_ids 70
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_exp_ids scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_exp_ids 0x5994
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_exp_ids 0x59F0
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_exp_ids 70
-
-#ifndef ISP2401
-/* function ia_css_queue_item_load: 4D19 */
-#else
-/* function ia_css_queue_item_load: 4F77 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_spctrl_sp_get_state: 5955 */
-#else
-/* function ia_css_spctrl_sp_get_state: 5A90 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_uninit: 603F */
-#else
-/* function ia_css_isys_sp_token_map_uninit: 617A */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_callout_sp_thread
-#define HIVE_MEM_callout_sp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_callout_sp_thread 0x49DC
-#else
-#define HIVE_ADDR_callout_sp_thread 0x1E0
-#endif
-#define HIVE_SIZE_callout_sp_thread 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_callout_sp_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_callout_sp_thread 0x49DC
-#else
-#define HIVE_ADDR_sp_callout_sp_thread 0x1E0
-#endif
-#define HIVE_SIZE_sp_callout_sp_thread 4
-
-#ifndef ISP2401
-/* function thread_fiber_sp_init: E2F */
-#else
-/* function thread_fiber_sp_init: E24 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_SP_PMEM_BASE
-#define HIVE_MEM_SP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_SP_PMEM_BASE 0x0
-#define HIVE_SIZE_SP_PMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_SP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_SP_PMEM_BASE 0x0
-#define HIVE_SIZE_sp_SP_PMEM_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_snd_acquire_req: 5FAF */
-#else
-/* function ia_css_isys_sp_token_map_snd_acquire_req: 60EA */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_isp_input_stream_format
-#define HIVE_MEM_sp_isp_input_stream_format scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_input_stream_format 0x40F8
-#else
-#define HIVE_ADDR_sp_isp_input_stream_format 0x4118
-#endif
-#define HIVE_SIZE_sp_isp_input_stream_format 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_isp_input_stream_format scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_isp_input_stream_format 0x40F8
-#else
-#define HIVE_ADDR_sp_sp_isp_input_stream_format 0x4118
-#endif
-#define HIVE_SIZE_sp_sp_isp_input_stream_format 20
-
-#ifndef ISP2401
-/* function __mod: 68A7 */
-#else
-/* function __mod: 6A1A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init_dmem_channel: 3260 */
-#else
-/* function ia_css_dmaproxy_sp_init_dmem_channel: 343F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_join: CFF */
-#else
-/* function ia_css_thread_sp_join: CF4 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_add_command: 6F4F */
-#else
-/* function ia_css_dmaproxy_sp_add_command: 7082 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_metadata_thread_func: 5809 */
-#else
-/* function ia_css_sp_metadata_thread_func: 5968 */
-#endif
-
-#ifndef ISP2401
-/* function __sp_event_proxy_func_critical: 6975 */
-#else
-/* function __sp_event_proxy_func_critical: 6AE8 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_metadata_wait: 591C */
-#else
-/* function ia_css_sp_metadata_wait: 5A57 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_peek_from_start: F08 */
-#else
-/* function ia_css_circbuf_peek_from_start: EFD */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_event_sp_encode: 356B */
-#else
-/* function ia_css_event_sp_encode: 375F */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_thread_sp_run: D72 */
-#else
-/* function ia_css_thread_sp_run: D67 */
-#endif
-
-#ifndef ISP2401
-/* function sp_isys_copy_func: 6F6 */
-#else
-/* function sp_isys_copy_func: 68A */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_flush: 5A98 */
-#else
-/* function ia_css_isys_sp_backend_flush: 5BD3 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_frame_exists: 59B4 */
-#else
-/* function ia_css_isys_sp_backend_frame_exists: 5AEF */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_isp_param_init_isp_memories: 47A2 */
-#else
-/* function ia_css_sp_isp_param_init_isp_memories: 4A2A */
-#endif
-
-#ifndef ISP2401
-/* function register_isr: 8A9 */
-#else
-/* function register_isr: 83D */
-#endif
-
-/* function irq_raise: C8 */
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_mmu_invalidate: 313D */
-#else
-/* function ia_css_dmaproxy_sp_mmu_invalidate: 32E5 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_HIVE_IF_SRST_ADDRESS
-#define HIVE_MEM_HIVE_IF_SRST_ADDRESS scalar_processor_2400_dmem
-#define HIVE_ADDR_HIVE_IF_SRST_ADDRESS 0x1B8
-#define HIVE_SIZE_HIVE_IF_SRST_ADDRESS 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_HIVE_IF_SRST_ADDRESS scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_HIVE_IF_SRST_ADDRESS 0x1B8
-#define HIVE_SIZE_sp_HIVE_IF_SRST_ADDRESS 16
-
-#ifndef ISP2401
-/* function pipeline_sp_initialize_stage: 1924 */
-#else
-/* function pipeline_sp_initialize_stage: 195E */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_isys_sp_frontend_states
-#define HIVE_MEM_ia_css_isys_sp_frontend_states scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_isys_sp_frontend_states 0x62C8
-#else
-#define HIVE_ADDR_ia_css_isys_sp_frontend_states 0x6324
-#endif
-#define HIVE_SIZE_ia_css_isys_sp_frontend_states 12
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_isys_sp_frontend_states scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_isys_sp_frontend_states 0x62C8
-#else
-#define HIVE_ADDR_sp_ia_css_isys_sp_frontend_states 0x6324
-#endif
-#define HIVE_SIZE_sp_ia_css_isys_sp_frontend_states 12
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_read_byte_addr_mmio: 6E1E */
-#else
-/* function ia_css_dmaproxy_sp_read_byte_addr_mmio: 6F62 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_done_ds: 37B2 */
-#else
-/* function ia_css_ispctrl_sp_done_ds: 39E1 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_isp_param_get_mem_inits: 477D */
-#else
-/* function ia_css_sp_isp_param_get_mem_inits: 4A05 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_init_buffer_queues: 13D0 */
-#else
-/* function ia_css_parambuf_sp_init_buffer_queues: 13F1 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_pfp_spref
-#define HIVE_MEM_vbuf_pfp_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_pfp_spref 0x2F0
-#else
-#define HIVE_ADDR_vbuf_pfp_spref 0x308
-#endif
-#define HIVE_SIZE_vbuf_pfp_spref 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_pfp_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_pfp_spref 0x2F0
-#else
-#define HIVE_ADDR_sp_vbuf_pfp_spref 0x308
-#endif
-#define HIVE_SIZE_sp_vbuf_pfp_spref 4
-
-#ifndef ISP2401
-/* function input_system_cfg: ABB */
-#else
-/* function input_system_cfg: AB5 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_HMEM_BASE
-#define HIVE_MEM_ISP_HMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_HMEM_BASE 0x20
-#define HIVE_SIZE_ISP_HMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_HMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_HMEM_BASE 0x20
-#define HIVE_SIZE_sp_ISP_HMEM_BASE 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_pipe_private_frames
-#define HIVE_MEM_ia_css_bufq_sp_pipe_private_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_frames 0x59DC
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_pipe_private_frames 0x5A38
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_pipe_private_frames 280
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_pipe_private_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_frames 0x59DC
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_pipe_private_frames 0x5A38
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_pipe_private_frames 280
-
-#ifndef ISP2401
-/* function qos_scheduler_init_stage_budget: 65E8 */
-#else
-/* function qos_scheduler_init_stage_budget: 6750 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_release: 5B0D */
-#else
-/* function ia_css_isys_sp_backend_release: 5C48 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_backend_destroy: 5B37 */
-#else
-/* function ia_css_isys_sp_backend_destroy: 5C72 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp2host_buffer_queue_handle
-#define HIVE_MEM_sp2host_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp2host_buffer_queue_handle 0x5AF4
-#else
-#define HIVE_ADDR_sp2host_buffer_queue_handle 0x5B50
-#endif
-#define HIVE_SIZE_sp2host_buffer_queue_handle 96
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp2host_buffer_queue_handle scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp2host_buffer_queue_handle 0x5AF4
-#else
-#define HIVE_ADDR_sp_sp2host_buffer_queue_handle 0x5B50
-#endif
-#define HIVE_SIZE_sp_sp2host_buffer_queue_handle 96
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_check_mipi_frame_size: 5F73 */
-#else
-/* function ia_css_isys_sp_token_map_check_mipi_frame_size: 60AE */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_ispctrl_sp_init_isp_vars: 449C */
-#else
-/* function ia_css_ispctrl_sp_init_isp_vars: 46F7 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_has_empty_mipi_buffer_cb: 5B89 */
-#else
-/* function ia_css_isys_sp_frontend_has_empty_mipi_buffer_cb: 5CC4 */
-#endif
-
-#ifndef ISP2401
-/* function sp_warning: 8DC */
-#else
-/* function sp_warning: 870 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_vbuf_enqueue: 631D */
-#else
-/* function ia_css_rmgr_sp_vbuf_enqueue: 6458 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_sp_tag_exp_id: 215B */
-#else
-/* function ia_css_tagger_sp_tag_exp_id: 21AB */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_write: 3216 */
-#else
-/* function ia_css_dmaproxy_sp_write: 33F5 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_parambuf_sp_release_in_param: 1250 */
-#else
-/* function ia_css_parambuf_sp_release_in_param: 1245 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_irq_sw_interrupt_token
-#define HIVE_MEM_irq_sw_interrupt_token scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_irq_sw_interrupt_token 0x40F4
-#else
-#define HIVE_ADDR_irq_sw_interrupt_token 0x4114
-#endif
-#define HIVE_SIZE_irq_sw_interrupt_token 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_irq_sw_interrupt_token scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_irq_sw_interrupt_token 0x40F4
-#else
-#define HIVE_ADDR_sp_irq_sw_interrupt_token 0x4114
-#endif
-#define HIVE_SIZE_sp_irq_sw_interrupt_token 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_isp_addresses
-#define HIVE_MEM_sp_isp_addresses scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_addresses 0x5F44
-#else
-#define HIVE_ADDR_sp_isp_addresses 0x5FA4
-#endif
-#define HIVE_SIZE_sp_isp_addresses 172
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_isp_addresses scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_isp_addresses 0x5F44
-#else
-#define HIVE_ADDR_sp_sp_isp_addresses 0x5FA4
-#endif
-#define HIVE_SIZE_sp_sp_isp_addresses 172
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_acq_gen: 6242 */
-#else
-/* function ia_css_rmgr_sp_acq_gen: 637D */
-#endif
-
-#ifndef ISP2401
-/* function receiver_reg_load: AD0 */
-#else
-/* function receiver_reg_load: ACA */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isps
-#define HIVE_MEM_isps scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isps 0x6300
-#else
-#define HIVE_ADDR_isps 0x635C
-#endif
-#define HIVE_SIZE_isps 28
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isps scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isps 0x6300
-#else
-#define HIVE_ADDR_sp_isps 0x635C
-#endif
-#define HIVE_SIZE_sp_isps 28
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_host_sp_queues_initialized
-#define HIVE_MEM_host_sp_queues_initialized scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_host_sp_queues_initialized 0x410C
-#else
-#define HIVE_ADDR_host_sp_queues_initialized 0x412C
-#endif
-#define HIVE_SIZE_host_sp_queues_initialized 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_host_sp_queues_initialized scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_host_sp_queues_initialized 0x410C
-#else
-#define HIVE_ADDR_sp_host_sp_queues_initialized 0x412C
-#endif
-#define HIVE_SIZE_sp_host_sp_queues_initialized 4
-
-#ifndef ISP2401
-/* function ia_css_queue_uninit: 4BE5 */
-#else
-/* function ia_css_queue_uninit: 4E43 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_ispctrl_sp_isp_started
-#define HIVE_MEM_ia_css_ispctrl_sp_isp_started scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_ispctrl_sp_isp_started 0x5BFC
-#else
-#define HIVE_ADDR_ia_css_ispctrl_sp_isp_started 0x5C58
-#endif
-#define HIVE_SIZE_ia_css_ispctrl_sp_isp_started 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_ispctrl_sp_isp_started scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_ispctrl_sp_isp_started 0x5BFC
-#else
-#define HIVE_ADDR_sp_ia_css_ispctrl_sp_isp_started 0x5C58
-#endif
-#define HIVE_SIZE_sp_ia_css_ispctrl_sp_isp_started 4
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_release_dynamic_buf: 2DE7 */
-#else
-/* function ia_css_bufq_sp_release_dynamic_buf: 2F89 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_set_height_exception: 330E */
-#else
-/* function ia_css_dmaproxy_sp_set_height_exception: 3502 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_init_vmem_channel: 3293 */
-#else
-/* function ia_css_dmaproxy_sp_init_vmem_channel: 3473 */
-#endif
-
-#ifndef ISP2401
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_num_ready_threads
-#define HIVE_MEM_num_ready_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_num_ready_threads 0x49E4
-#define HIVE_SIZE_num_ready_threads 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_num_ready_threads scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_num_ready_threads 0x49E4
-#define HIVE_SIZE_sp_num_ready_threads 4
-
-/* function ia_css_dmaproxy_sp_write_byte_addr_mmio: 31E8 */
-#else
-/* function ia_css_dmaproxy_sp_write_byte_addr_mmio: 33C7 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_vbuf_spref
-#define HIVE_MEM_vbuf_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_vbuf_spref 0x2EC
-#else
-#define HIVE_ADDR_vbuf_spref 0x304
-#endif
-#define HIVE_SIZE_vbuf_spref 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_vbuf_spref scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_vbuf_spref 0x2EC
-#else
-#define HIVE_ADDR_sp_vbuf_spref 0x304
-#endif
-#define HIVE_SIZE_sp_vbuf_spref 4
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_metadata_thread
-#define HIVE_MEM_sp_metadata_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_metadata_thread 0x4998
-#define HIVE_SIZE_sp_metadata_thread 68
-#else
-#define HIVE_ADDR_sp_metadata_thread 0x49F8
-#define HIVE_SIZE_sp_metadata_thread 72
-#endif
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_metadata_thread scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_metadata_thread 0x4998
-#define HIVE_SIZE_sp_sp_metadata_thread 68
-#else
-#define HIVE_ADDR_sp_sp_metadata_thread 0x49F8
-#define HIVE_SIZE_sp_sp_metadata_thread 72
-#endif
-
-#ifndef ISP2401
-/* function ia_css_queue_enqueue: 4B2F */
-#else
-/* function ia_css_queue_enqueue: 4D8D */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_request
-#define HIVE_MEM_ia_css_flash_sp_request scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_request 0x4A98
-#else
-#define HIVE_ADDR_ia_css_flash_sp_request 0x4AF4
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_request 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_request scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_request 0x4A98
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_request 0x4AF4
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_request 4
-
-#ifndef ISP2401
-/* function ia_css_dmaproxy_sp_vmem_write: 31B9 */
-#else
-/* function ia_css_dmaproxy_sp_vmem_write: 3398 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_tagger_frames
-#define HIVE_MEM_tagger_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_tagger_frames 0x49EC
-#else
-#define HIVE_ADDR_tagger_frames 0x4A48
-#endif
-#define HIVE_SIZE_tagger_frames 168
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_tagger_frames scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_tagger_frames 0x49EC
-#else
-#define HIVE_ADDR_sp_tagger_frames 0x4A48
-#endif
-#define HIVE_SIZE_sp_tagger_frames 168
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_token_map_snd_capture_req: 5FD1 */
-#else
-/* function ia_css_isys_sp_token_map_snd_capture_req: 610C */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_reading_if
-#define HIVE_MEM_sem_for_reading_if scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_reading_if 0x47C8
-#else
-#define HIVE_ADDR_sem_for_reading_if 0x4810
-#endif
-#define HIVE_SIZE_sem_for_reading_if 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_reading_if scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_reading_if 0x47C8
-#else
-#define HIVE_ADDR_sp_sem_for_reading_if 0x4810
-#endif
-#define HIVE_SIZE_sp_sem_for_reading_if 20
-
-#ifndef ISP2401
-/* function sp_generate_interrupts: 95B */
-#else
-/* function sp_generate_interrupts: 8EF */
-
-/* function ia_css_pipeline_sp_start: 1871 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_pipeline_sp_start: 1837 */
-#else
-/* function ia_css_thread_default_callout: 6BE3 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_sp_rawcopy_init: 510C */
-#else
-/* function ia_css_sp_rawcopy_init: 536A */
-#endif
-
-#ifndef ISP2401
-/* function tmr_clock_read: 13F1 */
-#else
-/* function tmr_clock_read: 1412 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_BAMEM_BASE
-#define HIVE_MEM_ISP_BAMEM_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ISP_BAMEM_BASE 0x2F8
-#else
-#define HIVE_ADDR_ISP_BAMEM_BASE 0x310
-#endif
-#define HIVE_SIZE_ISP_BAMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_BAMEM_BASE scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ISP_BAMEM_BASE 0x2F8
-#else
-#define HIVE_ADDR_sp_ISP_BAMEM_BASE 0x310
-#endif
-#define HIVE_SIZE_sp_ISP_BAMEM_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_rcv_capture_ack: 5C38 */
-#else
-/* function ia_css_isys_sp_frontend_rcv_capture_ack: 5D73 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_bufq_sp_sems_for_sp2host_buf_queues
-#define HIVE_MEM_ia_css_bufq_sp_sems_for_sp2host_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x5B54
-#else
-#define HIVE_ADDR_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x5BB0
-#endif
-#define HIVE_SIZE_ia_css_bufq_sp_sems_for_sp2host_buf_queues 160
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x5B54
-#else
-#define HIVE_ADDR_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues 0x5BB0
-#endif
-#define HIVE_SIZE_sp_ia_css_bufq_sp_sems_for_sp2host_buf_queues 160
-
-#ifndef ISP2401
-/* function css_get_frame_processing_time_start: 1FC8 */
-#else
-/* function css_get_frame_processing_time_start: 2018 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_all_cbs_frame
-#define HIVE_MEM_sp_all_cbs_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_all_cbs_frame 0x47DC
-#else
-#define HIVE_ADDR_sp_all_cbs_frame 0x4824
-#endif
-#define HIVE_SIZE_sp_all_cbs_frame 16
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_all_cbs_frame scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_all_cbs_frame 0x47DC
-#else
-#define HIVE_ADDR_sp_sp_all_cbs_frame 0x4824
-#endif
-#define HIVE_SIZE_sp_sp_all_cbs_frame 16
-
-#ifndef ISP2401
-/* function thread_sp_queue_print: D8F */
-#else
-/* function thread_sp_queue_print: D84 */
-#endif
-
-#ifndef ISP2401
-/* function sp_notify_eof: 907 */
-#else
-/* function sp_notify_eof: 89B */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sem_for_str2mem
-#define HIVE_MEM_sem_for_str2mem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sem_for_str2mem 0x47EC
-#else
-#define HIVE_ADDR_sem_for_str2mem 0x4834
-#endif
-#define HIVE_SIZE_sem_for_str2mem 20
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sem_for_str2mem scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sem_for_str2mem 0x47EC
-#else
-#define HIVE_ADDR_sp_sem_for_str2mem 0x4834
-#endif
-#define HIVE_SIZE_sp_sem_for_str2mem 20
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_is_marked_from_start: 2B5A */
-#else
-/* function ia_css_tagger_buf_sp_is_marked_from_start: 2CFC */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_bufq_sp_acquire_dynamic_buf: 2F9F */
-#else
-/* function ia_css_bufq_sp_acquire_dynamic_buf: 3141 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_circbuf_destroy: 101D */
-#else
-/* function ia_css_circbuf_destroy: 1012 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ISP_PMEM_BASE
-#define HIVE_MEM_ISP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_ISP_PMEM_BASE 0xC
-#define HIVE_SIZE_ISP_PMEM_BASE 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ISP_PMEM_BASE scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_ISP_PMEM_BASE 0xC
-#define HIVE_SIZE_sp_ISP_PMEM_BASE 4
-
-#ifndef ISP2401
-/* function ia_css_sp_isp_param_mem_load: 4710 */
-#else
-/* function ia_css_sp_isp_param_mem_load: 4998 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_pop_from_start: 2946 */
-#else
-/* function ia_css_tagger_buf_sp_pop_from_start: 2AE8 */
-#endif
-
-#ifndef ISP2401
-/* function __div: 685F */
-#else
-/* function __div: 69D2 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_isys_sp_frontend_create: 5E09 */
-#else
-/* function ia_css_isys_sp_frontend_create: 5F44 */
-#endif
-
-#ifndef ISP2401
-/* function ia_css_rmgr_sp_refcount_release_vbuf: 633C */
-#else
-/* function ia_css_rmgr_sp_refcount_release_vbuf: 6477 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_ia_css_flash_sp_in_use
-#define HIVE_MEM_ia_css_flash_sp_in_use scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_ia_css_flash_sp_in_use 0x4A9C
-#else
-#define HIVE_ADDR_ia_css_flash_sp_in_use 0x4AF8
-#endif
-#define HIVE_SIZE_ia_css_flash_sp_in_use 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_ia_css_flash_sp_in_use scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_use 0x4A9C
-#else
-#define HIVE_ADDR_sp_ia_css_flash_sp_in_use 0x4AF8
-#endif
-#define HIVE_SIZE_sp_ia_css_flash_sp_in_use 4
-
-#ifndef ISP2401
-/* function ia_css_thread_sem_sp_wait: 6B42 */
-#else
-/* function ia_css_thread_sem_sp_wait: 6CB7 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_sleep_mode
-#define HIVE_MEM_sp_sleep_mode scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sleep_mode 0x4110
-#else
-#define HIVE_ADDR_sp_sleep_mode 0x4130
-#endif
-#define HIVE_SIZE_sp_sleep_mode 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_sleep_mode scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_sp_sleep_mode 0x4110
-#else
-#define HIVE_ADDR_sp_sp_sleep_mode 0x4130
-#endif
-#define HIVE_SIZE_sp_sp_sleep_mode 4
-
-#ifndef ISP2401
-/* function ia_css_tagger_buf_sp_push: 2A55 */
-#else
-/* function ia_css_tagger_buf_sp_push: 2BF7 */
-#endif
-
-/* function mmu_invalidate_cache: D3 */
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_sp_max_cb_elems
-#define HIVE_MEM_sp_max_cb_elems scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_max_cb_elems 0x148
-#define HIVE_SIZE_sp_max_cb_elems 8
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_sp_max_cb_elems scalar_processor_2400_dmem
-#define HIVE_ADDR_sp_sp_max_cb_elems 0x148
-#define HIVE_SIZE_sp_sp_max_cb_elems 8
-
-#ifndef ISP2401
-/* function ia_css_queue_remote_init: 4C07 */
-#else
-/* function ia_css_queue_remote_init: 4E65 */
-#endif
-
-#ifndef HIVE_MULTIPLE_PROGRAMS
-#ifndef HIVE_MEM_isp_stop_req
-#define HIVE_MEM_isp_stop_req scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_isp_stop_req 0x4680
-#else
-#define HIVE_ADDR_isp_stop_req 0x46C8
-#endif
-#define HIVE_SIZE_isp_stop_req 4
-#else
-#endif
-#endif
-#define HIVE_MEM_sp_isp_stop_req scalar_processor_2400_dmem
-#ifndef ISP2401
-#define HIVE_ADDR_sp_isp_stop_req 0x4680
-#else
-#define HIVE_ADDR_sp_isp_stop_req 0x46C8
-#endif
-#define HIVE_SIZE_sp_isp_stop_req 4
-
-#ifndef ISP2401
-#define HIVE_ICACHE_sp_critical_SEGMENT_START 0
-#define HIVE_ICACHE_sp_critical_NUM_SEGMENTS  1
-#endif
-
-#endif /* _sp_map_h_ */
-#ifndef ISP2401
-extern void sh_css_dump_sp_dmem(void);
-void sh_css_dump_sp_dmem(void)
-{
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_trace.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_trace.h
deleted file mode 100644
index 01f7c33b5b40..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/css_trace.h
+++ /dev/null
@@ -1,388 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __CSS_TRACE_H_
-#define __CSS_TRACE_H_
-
-#include <type_support.h>
-#ifdef ISP2401
-#include "sh_css_internal.h"	/* for SH_CSS_MAX_SP_THREADS */
-#endif
-
-/*
-	structs and constants for tracing
-*/
-
-/* one tracer item: major, minor and counter. The counter value can be used for GP data */
-struct trace_item_t {
-	uint8_t   major;
-	uint8_t   minor;
-	uint16_t  counter;
-};
-
-#ifdef ISP2401
-#define MAX_SCRATCH_DATA	4
-#define MAX_CMD_DATA		2
-
-#endif
-/* trace header: holds the version and the topology of the tracer. */
-struct trace_header_t {
-#ifndef ISP2401
-	/* 1st dword */
-#else
-	/* 1st dword: descriptor */
-#endif
-	uint8_t   version;
-	uint8_t   max_threads;
-	uint16_t  max_tracer_points;
-#ifdef ISP2401
-	/* 2nd field: command + data */
-#endif
-	/* 2nd dword */
-	uint32_t  command;
-	/* 3rd & 4th dword */
-#ifndef ISP2401
-	uint32_t  data[2];
-#else
-	uint32_t  data[MAX_CMD_DATA];
-	/* 3rd field: debug pointer */
-#endif
-	/* 5th & 6th dword: debug pointer mechanism */
-	uint32_t  debug_ptr_signature;
-	uint32_t  debug_ptr_value;
-#ifdef ISP2401
-	/* Rest of the header: status & scratch data */
-	uint8_t   thr_status_byte[SH_CSS_MAX_SP_THREADS];
-	uint16_t  thr_status_word[SH_CSS_MAX_SP_THREADS];
-	uint32_t  thr_status_dword[SH_CSS_MAX_SP_THREADS];
-	uint32_t  scratch_debug[MAX_SCRATCH_DATA];
-#endif
-};
-
-#ifndef ISP2401
-#define TRACER_VER			2
-#else
-/* offsets for master_port read/write */
-#define HDR_HDR_OFFSET              0	/* offset of the header */
-#define HDR_COMMAND_OFFSET          offsetof(struct trace_header_t, command)
-#define HDR_DATA_OFFSET             offsetof(struct trace_header_t, data)
-#define HDR_DEBUG_SIGNATURE_OFFSET  offsetof(struct trace_header_t, debug_ptr_signature)
-#define HDR_DEBUG_POINTER_OFFSET    offsetof(struct trace_header_t, debug_ptr_value)
-#define HDR_STATUS_OFFSET           offsetof(struct trace_header_t, thr_status_byte)
-#define HDR_STATUS_OFFSET_BYTE      offsetof(struct trace_header_t, thr_status_byte)
-#define HDR_STATUS_OFFSET_WORD      offsetof(struct trace_header_t, thr_status_word)
-#define HDR_STATUS_OFFSET_DWORD     offsetof(struct trace_header_t, thr_status_dword)
-#define HDR_STATUS_OFFSET_SCRATCH   offsetof(struct trace_header_t, scratch_debug)
-
-/*
-Trace version history:
- 1: initial version, hdr = descr, command & ptr.
- 2: added ISP + 24-bit fields.
- 3: added thread ID.
- 4: added status in header.
-*/
-#define TRACER_VER			4
-
-#endif
-#define TRACE_BUFF_ADDR       0xA000
-#define TRACE_BUFF_SIZE       0x1000	/* 4K allocated */
-
-#define TRACE_ENABLE_SP0 0
-#define TRACE_ENABLE_SP1 0
-#define TRACE_ENABLE_ISP 0
-
-#ifndef ISP2401
-typedef enum {
-#else
-enum TRACE_CORE_ID {
-#endif
-	TRACE_SP0_ID,
-	TRACE_SP1_ID,
-	TRACE_ISP_ID
-#ifndef ISP2401
-} TRACE_CORE_ID;
-#else
-};
-#endif
-
-/* TODO: add timing format? */
-#ifndef ISP2401
-typedef enum {
-	TRACE_DUMP_FORMAT_POINT,
-	TRACE_DUMP_FORMAT_VALUE24_HEX,
-	TRACE_DUMP_FORMAT_VALUE24_DEC,
-#else
-enum TRACE_DUMP_FORMAT {
-	TRACE_DUMP_FORMAT_POINT_NO_TID,
-	TRACE_DUMP_FORMAT_VALUE24,
-#endif
-	TRACE_DUMP_FORMAT_VALUE24_TIMING,
-#ifndef ISP2401
-	TRACE_DUMP_FORMAT_VALUE24_TIMING_DELTA
-} TRACE_DUMP_FORMAT;
-#else
-	TRACE_DUMP_FORMAT_VALUE24_TIMING_DELTA,
-	TRACE_DUMP_FORMAT_POINT
-};
-#endif
-
-
-/* currently divided as follows:*/
-#if (TRACE_ENABLE_SP0 + TRACE_ENABLE_SP1 + TRACE_ENABLE_ISP == 3)
-/* can be divided as needed */
-#define TRACE_SP0_SIZE (TRACE_BUFF_SIZE/4)
-#define TRACE_SP1_SIZE (TRACE_BUFF_SIZE/4)
-#define TRACE_ISP_SIZE (TRACE_BUFF_SIZE/2)
-#elif (TRACE_ENABLE_SP0 + TRACE_ENABLE_SP1 + TRACE_ENABLE_ISP == 2)
-#if TRACE_ENABLE_SP0
-#define TRACE_SP0_SIZE (TRACE_BUFF_SIZE/2)
-#else
-#define TRACE_SP0_SIZE (0)
-#endif
-#if TRACE_ENABLE_SP1
-#define TRACE_SP1_SIZE (TRACE_BUFF_SIZE/2)
-#else
-#define TRACE_SP1_SIZE (0)
-#endif
-#if TRACE_ENABLE_ISP
-#define TRACE_ISP_SIZE (TRACE_BUFF_SIZE/2)
-#else
-#define TRACE_ISP_SIZE (0)
-#endif
-#elif (TRACE_ENABLE_SP0 + TRACE_ENABLE_SP1 + TRACE_ENABLE_ISP == 1)
-#if TRACE_ENABLE_SP0
-#define TRACE_SP0_SIZE (TRACE_BUFF_SIZE)
-#else
-#define TRACE_SP0_SIZE (0)
-#endif
-#if TRACE_ENABLE_SP1
-#define TRACE_SP1_SIZE (TRACE_BUFF_SIZE)
-#else
-#define TRACE_SP1_SIZE (0)
-#endif
-#if TRACE_ENABLE_ISP
-#define TRACE_ISP_SIZE (TRACE_BUFF_SIZE)
-#else
-#define TRACE_ISP_SIZE (0)
-#endif
-#else
-#define TRACE_SP0_SIZE (0)
-#define TRACE_SP1_SIZE (0)
-#define TRACE_ISP_SIZE (0)
-#endif
-
-#define TRACE_SP0_ADDR (TRACE_BUFF_ADDR)
-#define TRACE_SP1_ADDR (TRACE_SP0_ADDR + TRACE_SP0_SIZE)
-#define TRACE_ISP_ADDR (TRACE_SP1_ADDR + TRACE_SP1_SIZE)
-
-/* check if it's a legal division */
-#if (TRACE_BUFF_SIZE < TRACE_SP0_SIZE + TRACE_SP1_SIZE + TRACE_ISP_SIZE)
-#error trace sizes are not divided correctly and are above limit
-#endif
-
-#define TRACE_SP0_HEADER_ADDR (TRACE_SP0_ADDR)
-#define TRACE_SP0_HEADER_SIZE (sizeof(struct trace_header_t))
-#ifndef ISP2401
-#define TRACE_SP0_ITEM_SIZE (sizeof(struct trace_item_t))
-#define TRACE_SP0_DATA_ADDR (TRACE_SP0_HEADER_ADDR + TRACE_SP0_HEADER_SIZE)
-#define TRACE_SP0_DATA_SIZE (TRACE_SP0_SIZE - TRACE_SP0_HEADER_SIZE)
-#define TRACE_SP0_MAX_POINTS (TRACE_SP0_DATA_SIZE / TRACE_SP0_ITEM_SIZE)
-#else
-#define TRACE_SP0_ITEM_SIZE   (sizeof(struct trace_item_t))
-#define TRACE_SP0_DATA_ADDR   (TRACE_SP0_HEADER_ADDR + TRACE_SP0_HEADER_SIZE)
-#define TRACE_SP0_DATA_SIZE   (TRACE_SP0_SIZE - TRACE_SP0_HEADER_SIZE)
-#define TRACE_SP0_MAX_POINTS  (TRACE_SP0_DATA_SIZE / TRACE_SP0_ITEM_SIZE)
-#endif
-
-#define TRACE_SP1_HEADER_ADDR (TRACE_SP1_ADDR)
-#define TRACE_SP1_HEADER_SIZE (sizeof(struct trace_header_t))
-#ifndef ISP2401
-#define TRACE_SP1_ITEM_SIZE (sizeof(struct trace_item_t))
-#define TRACE_SP1_DATA_ADDR (TRACE_SP1_HEADER_ADDR + TRACE_SP1_HEADER_SIZE)
-#define TRACE_SP1_DATA_SIZE (TRACE_SP1_SIZE - TRACE_SP1_HEADER_SIZE)
-#define TRACE_SP1_MAX_POINTS (TRACE_SP1_DATA_SIZE / TRACE_SP1_ITEM_SIZE)
-#else
-#define TRACE_SP1_ITEM_SIZE   (sizeof(struct trace_item_t))
-#define TRACE_SP1_DATA_ADDR   (TRACE_SP1_HEADER_ADDR + TRACE_SP1_HEADER_SIZE)
-#define TRACE_SP1_DATA_SIZE   (TRACE_SP1_SIZE - TRACE_SP1_HEADER_SIZE)
-#define TRACE_SP1_MAX_POINTS  (TRACE_SP1_DATA_SIZE / TRACE_SP1_ITEM_SIZE)
-#endif
-
-#define TRACE_ISP_HEADER_ADDR (TRACE_ISP_ADDR)
-#define TRACE_ISP_HEADER_SIZE (sizeof(struct trace_header_t))
-#ifndef ISP2401
-#define TRACE_ISP_ITEM_SIZE (sizeof(struct trace_item_t))
-#define TRACE_ISP_DATA_ADDR (TRACE_ISP_HEADER_ADDR + TRACE_ISP_HEADER_SIZE)
-#define TRACE_ISP_DATA_SIZE (TRACE_ISP_SIZE - TRACE_ISP_HEADER_SIZE)
-#define TRACE_ISP_MAX_POINTS (TRACE_ISP_DATA_SIZE / TRACE_ISP_ITEM_SIZE)
-
-#else
-#define TRACE_ISP_ITEM_SIZE   (sizeof(struct trace_item_t))
-#define TRACE_ISP_DATA_ADDR   (TRACE_ISP_HEADER_ADDR + TRACE_ISP_HEADER_SIZE)
-#define TRACE_ISP_DATA_SIZE   (TRACE_ISP_SIZE - TRACE_ISP_HEADER_SIZE)
-#define TRACE_ISP_MAX_POINTS  (TRACE_ISP_DATA_SIZE / TRACE_ISP_ITEM_SIZE)
-#endif
-
-#ifndef ISP2401
-/* offsets for master_port read/write */
-#define HDR_HDR_OFFSET              0	/* offset of the header */
-#define HDR_COMMAND_OFFSET          4	/* offset of the command */
-#define HDR_DATA_OFFSET             8	/* offset of the command data */
-#define HDR_DEBUG_SIGNATURE_OFFSET  16	/* offset of the param debug signature in trace_header_t */
-#define HDR_DEBUG_POINTER_OFFSET    20	/* offset of the param debug pointer in trace_header_t */
-#endif
-
-/* common majors */
-#ifdef ISP2401
-/* SP0 */
-#endif
-#define MAJOR_MAIN              1
-#define MAJOR_ISP_STAGE_ENTRY   2
-#define MAJOR_DMA_PRXY          3
-#define MAJOR_START_ISP         4
-#ifdef ISP2401
-/* SP1 */
-#define MAJOR_OBSERVER_ISP0_EVENT          21
-#define MAJOR_OBSERVER_OUTPUT_FORM_EVENT   22
-#define MAJOR_OBSERVER_OUTPUT_SCAL_EVENT   23
-#define MAJOR_OBSERVER_IF_ACK              24
-#define MAJOR_OBSERVER_SP0_EVENT           25
-#define MAJOR_OBSERVER_SP_TERMINATE_EVENT  26
-#define MAJOR_OBSERVER_DMA_ACK             27
-#define MAJOR_OBSERVER_ACC_ACK             28
-#endif
-
-#define DEBUG_PTR_SIGNATURE     0xABCD	/* signature for the debug parameter pointer */
-
-/* command codes (1st byte) */
-typedef enum {
-	CMD_SET_ONE_MAJOR = 1,		/* mask in one major. 2nd byte in the command is the major code */
-	CMD_UNSET_ONE_MAJOR = 2,	/* mask out one major. 2nd byte in the command is the major code */
-	CMD_SET_ALL_MAJORS = 3,		/* set the major print mask. the full mask is in the data DWORD */
-	CMD_SET_VERBOSITY = 4		/* set verbosity level */
-} DBG_commands;
-
-/* command signature */
-#define CMD_SIGNATURE	0xAABBCC00
-
-/* shared macros in traces infrastructure */
-/* increment the pointer cyclicly */
-#define DBG_NEXT_ITEM(x, max_items) (((x+1) >= max_items) ? 0 : x+1)
-#define DBG_PREV_ITEM(x, max_items) ((x) ? x-1 : max_items-1)
-
-#define FIELD_MASK(width) (((1 << (width)) - 1))
-#define FIELD_PACK(value,mask,offset) (((value) & (mask)) << (offset))
-#define FIELD_UNPACK(value,mask,offset) (((value) >> (offset)) & (mask))
-
-
-#define FIELD_VALUE_OFFSET		(0)
-#define FIELD_VALUE_WIDTH		(16)
-#define FIELD_VALUE_MASK		FIELD_MASK(FIELD_VALUE_WIDTH)
-#define FIELD_VALUE_PACK(f)		FIELD_PACK(f,FIELD_VALUE_MASK,FIELD_VALUE_OFFSET)
-#ifndef ISP2401
-#define FIELD_VALUE_UNPACK(f)	FIELD_UNPACK(f,FIELD_VALUE_MASK,FIELD_VALUE_OFFSET)
-#else
-#define FIELD_VALUE_UNPACK(f)		FIELD_UNPACK(f,FIELD_VALUE_MASK,FIELD_VALUE_OFFSET)
-#endif
-
-#define FIELD_MINOR_OFFSET		(FIELD_VALUE_OFFSET + FIELD_VALUE_WIDTH)
-#define FIELD_MINOR_WIDTH		(8)
-#define FIELD_MINOR_MASK		FIELD_MASK(FIELD_MINOR_WIDTH)
-#define FIELD_MINOR_PACK(f)		FIELD_PACK(f,FIELD_MINOR_MASK,FIELD_MINOR_OFFSET)
-#ifndef ISP2401
-#define FIELD_MINOR_UNPACK(f)	FIELD_UNPACK(f,FIELD_MINOR_MASK,FIELD_MINOR_OFFSET)
-#else
-#define FIELD_MINOR_UNPACK(f)		FIELD_UNPACK(f,FIELD_MINOR_MASK,FIELD_MINOR_OFFSET)
-#endif
-
-#define FIELD_MAJOR_OFFSET		(FIELD_MINOR_OFFSET + FIELD_MINOR_WIDTH)
-#define FIELD_MAJOR_WIDTH		(5)
-#define FIELD_MAJOR_MASK		FIELD_MASK(FIELD_MAJOR_WIDTH)
-#define FIELD_MAJOR_PACK(f)		FIELD_PACK(f,FIELD_MAJOR_MASK,FIELD_MAJOR_OFFSET)
-#ifndef ISP2401
-#define FIELD_MAJOR_UNPACK(f)	FIELD_UNPACK(f,FIELD_MAJOR_MASK,FIELD_MAJOR_OFFSET)
-#else
-#define FIELD_MAJOR_UNPACK(f)		FIELD_UNPACK(f,FIELD_MAJOR_MASK,FIELD_MAJOR_OFFSET)
-#endif
-
-#ifndef ISP2401
-#define FIELD_FORMAT_OFFSET		(FIELD_MAJOR_OFFSET + FIELD_MAJOR_WIDTH)
-#define FIELD_FORMAT_WIDTH 		(3)
-#define FIELD_FORMAT_MASK 		FIELD_MASK(FIELD_FORMAT_WIDTH)
-#define FIELD_FORMAT_PACK(f)	FIELD_PACK(f,FIELD_FORMAT_MASK,FIELD_FORMAT_OFFSET)
-#define FIELD_FORMAT_UNPACK(f)	FIELD_UNPACK(f,FIELD_FORMAT_MASK,FIELD_FORMAT_OFFSET)
-#else
-/* for quick traces - only insertion, compatible with the regular point */
-#define FIELD_FULL_MAJOR_WIDTH		(8)
-#define FIELD_FULL_MAJOR_MASK		FIELD_MASK(FIELD_FULL_MAJOR_WIDTH)
-#define FIELD_FULL_MAJOR_PACK(f)	FIELD_PACK(f,FIELD_FULL_MAJOR_MASK,FIELD_MAJOR_OFFSET)
-
-/* The following 2 fields are used only when FIELD_TID value is 111b.
- * it means we don't want to use thread id, but format. In this case,
- * the last 2 MSB bits of the major field will indicates the format
- */
-#define FIELD_MAJOR_W_FMT_OFFSET	FIELD_MAJOR_OFFSET
-#define FIELD_MAJOR_W_FMT_WIDTH		(3)
-#define FIELD_MAJOR_W_FMT_MASK		FIELD_MASK(FIELD_MAJOR_W_FMT_WIDTH)
-#define FIELD_MAJOR_W_FMT_PACK(f)	FIELD_PACK(f,FIELD_MAJOR_W_FMT_MASK,FIELD_MAJOR_W_FMT_OFFSET)
-#define FIELD_MAJOR_W_FMT_UNPACK(f)	FIELD_UNPACK(f,FIELD_MAJOR_W_FMT_MASK,FIELD_MAJOR_W_FMT_OFFSET)
-
-#define FIELD_FORMAT_OFFSET		(FIELD_MAJOR_OFFSET + FIELD_MAJOR_W_FMT_WIDTH)
-#define FIELD_FORMAT_WIDTH 		(2)
-#define FIELD_FORMAT_MASK 		FIELD_MASK(FIELD_MAJOR_W_FMT_WIDTH)
-#define FIELD_FORMAT_PACK(f)		FIELD_PACK(f,FIELD_FORMAT_MASK,FIELD_FORMAT_OFFSET)
-#define FIELD_FORMAT_UNPACK(f)		FIELD_UNPACK(f,FIELD_FORMAT_MASK,FIELD_FORMAT_OFFSET)
-
-#define FIELD_TID_SEL_FORMAT_PAT	(7)
-
-#define FIELD_TID_OFFSET		(FIELD_MAJOR_OFFSET + FIELD_MAJOR_WIDTH)
-#define FIELD_TID_WIDTH			(3)
-#define FIELD_TID_MASK			FIELD_MASK(FIELD_TID_WIDTH)
-#define FIELD_TID_PACK(f)		FIELD_PACK(f,FIELD_TID_MASK,FIELD_TID_OFFSET)
-#define FIELD_TID_UNPACK(f)		FIELD_UNPACK(f,FIELD_TID_MASK,FIELD_TID_OFFSET)
-#endif
-
-#define FIELD_VALUE_24_OFFSET		(0)
-#define FIELD_VALUE_24_WIDTH		(24)
-#ifndef ISP2401
-#define FIELD_VALUE_24_MASK			FIELD_MASK(FIELD_VALUE_24_WIDTH)
-#else
-#define FIELD_VALUE_24_MASK		FIELD_MASK(FIELD_VALUE_24_WIDTH)
-#endif
-#define FIELD_VALUE_24_PACK(f)		FIELD_PACK(f,FIELD_VALUE_24_MASK,FIELD_VALUE_24_OFFSET)
-#define FIELD_VALUE_24_UNPACK(f)	FIELD_UNPACK(f,FIELD_VALUE_24_MASK,FIELD_VALUE_24_OFFSET)
-
-#ifndef ISP2401
-#define PACK_TRACEPOINT(format,major, minor, value)	\
-	(FIELD_FORMAT_PACK(format) | FIELD_MAJOR_PACK(major) | FIELD_MINOR_PACK(minor) | FIELD_VALUE_PACK(value))
-#else
-#define PACK_TRACEPOINT(tid, major, minor, value)	\
-	(FIELD_TID_PACK(tid) | FIELD_MAJOR_PACK(major) | FIELD_MINOR_PACK(minor) | FIELD_VALUE_PACK(value))
-
-#define PACK_QUICK_TRACEPOINT(major, minor)	\
-	(FIELD_FULL_MAJOR_PACK(major) | FIELD_MINOR_PACK(minor))
-
-#define PACK_FORMATTED_TRACEPOINT(format, major, minor, value)	\
-	(FIELD_TID_PACK(FIELD_TID_SEL_FORMAT_PAT) | FIELD_FORMAT_PACK(format) | FIELD_MAJOR_PACK(major) | FIELD_MINOR_PACK(minor) | FIELD_VALUE_PACK(value))
-#endif
-
-#ifndef ISP2401
-#define PACK_TRACE_VALUE24(format, major, value)	\
-	(FIELD_FORMAT_PACK(format) | FIELD_MAJOR_PACK(major) | FIELD_VALUE_24_PACK(value))
-#else
-#define PACK_TRACE_VALUE24(major, value)	\
-	(FIELD_TID_PACK(FIELD_TID_SEL_FORMAT_PAT) | FIELD_MAJOR_PACK(major) | FIELD_VALUE_24_PACK(value))
-#endif
-
-#endif /* __CSS_TRACE_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/debug_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/debug_global.h
deleted file mode 100644
index 076c4ba76175..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/debug_global.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __DEBUG_GLOBAL_H_INCLUDED__
-#define __DEBUG_GLOBAL_H_INCLUDED__
-
-#include <type_support.h>
-
-#define DEBUG_BUF_SIZE	1024
-#define DEBUG_BUF_MASK	(DEBUG_BUF_SIZE - 1)
-
-#define DEBUG_DATA_ENABLE_ADDR		0x00
-#define DEBUG_DATA_BUF_MODE_ADDR	0x04
-#define DEBUG_DATA_HEAD_ADDR		0x08
-#define DEBUG_DATA_TAIL_ADDR		0x0C
-#define DEBUG_DATA_BUF_ADDR			0x10
-
-#define DEBUG_DATA_ENABLE_DDR_ADDR		0x00
-#define DEBUG_DATA_BUF_MODE_DDR_ADDR	HIVE_ISP_DDR_WORD_BYTES
-#define DEBUG_DATA_HEAD_DDR_ADDR		(2 * HIVE_ISP_DDR_WORD_BYTES)
-#define DEBUG_DATA_TAIL_DDR_ADDR		(3 * HIVE_ISP_DDR_WORD_BYTES)
-#define DEBUG_DATA_BUF_DDR_ADDR			(4 * HIVE_ISP_DDR_WORD_BYTES)
-
-#define DEBUG_BUFFER_ISP_DMEM_ADDR       0x0
-
-/*
- * Enable HAS_WATCHDOG_SP_THREAD_DEBUG for additional SP thread and
- * pipe information on watchdog output
- * #undef HAS_WATCHDOG_SP_THREAD_DEBUG
- * #define HAS_WATCHDOG_SP_THREAD_DEBUG
- */
-
-
-/*
- * The linear buffer mode will accept data until the first
- * overflow and then stop accepting new data
- * The circular buffer mode will accept if there is place
- * and discard the data if the buffer is full
- */
-typedef enum {
-	DEBUG_BUFFER_MODE_LINEAR = 0,
-	DEBUG_BUFFER_MODE_CIRCULAR,
-	N_DEBUG_BUFFER_MODE
-} debug_buf_mode_t;
-
-struct debug_data_s {
-	uint32_t			enable;
-	uint32_t			bufmode;
-	uint32_t			head;
-	uint32_t			tail;
-	uint32_t			buf[DEBUG_BUF_SIZE];
-};
-
-/* thread.sp.c doesn't have a notion of HIVE_ISP_DDR_WORD_BYTES
-   still one point of control is needed for debug purposes */
-
-#ifdef HIVE_ISP_DDR_WORD_BYTES
-struct debug_data_ddr_s {
-	uint32_t			enable;
-	int8_t				padding1[HIVE_ISP_DDR_WORD_BYTES-sizeof(uint32_t)];
-	uint32_t			bufmode;
-	int8_t				padding2[HIVE_ISP_DDR_WORD_BYTES-sizeof(uint32_t)];
-	uint32_t			head;
-	int8_t				padding3[HIVE_ISP_DDR_WORD_BYTES-sizeof(uint32_t)];
-	uint32_t			tail;
-	int8_t				padding4[HIVE_ISP_DDR_WORD_BYTES-sizeof(uint32_t)];
-	uint32_t			buf[DEBUG_BUF_SIZE];
-};
-#endif
-
-#endif /* __DEBUG_GLOBAL_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/dma_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/dma_global.h
deleted file mode 100644
index 60d6de1332cd..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/dma_global.h
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __DMA_GLOBAL_H_INCLUDED__
-#define __DMA_GLOBAL_H_INCLUDED__
-
-#include <type_support.h>
-
-#define IS_DMA_VERSION_2
-
-#define HIVE_ISP_NUM_DMA_CONNS		3
-#define HIVE_ISP_NUM_DMA_CHANNELS	32
-
-#define N_DMA_CHANNEL_ID	HIVE_ISP_NUM_DMA_CHANNELS
-
-#include "dma_v2_defs.h"
-
-/*
- * Command token bit mappings
- *
- * transfer / config
- *    param id[4] channel id[5] cmd id[6]
- *	| b14 .. b11 | b10 ... b6 | b5 ... b0 |
- *
- *
- * fast transfer:
- *    height[5]     width[8]      width[8]  channel id[5] cmd id[6]
- *	| b31 .. b26 | b25 .. b18 | b17 .. b11 | b10 ... b6 | b5 ... b0 |
- *
- */
-
-#define _DMA_PACKING_SETUP_PARAM	_DMA_V2_PACKING_SETUP_PARAM
-#define _DMA_HEIGHT_PARAM			_DMA_V2_HEIGHT_PARAM
-#define _DMA_STRIDE_A_PARAM			_DMA_V2_STRIDE_A_PARAM
-#define _DMA_ELEM_CROPPING_A_PARAM	_DMA_V2_ELEM_CROPPING_A_PARAM
-#define _DMA_WIDTH_A_PARAM			_DMA_V2_WIDTH_A_PARAM
-#define _DMA_STRIDE_B_PARAM			_DMA_V2_STRIDE_B_PARAM
-#define _DMA_ELEM_CROPPING_B_PARAM	_DMA_V2_ELEM_CROPPING_B_PARAM
-#define _DMA_WIDTH_B_PARAM			_DMA_V2_WIDTH_B_PARAM
-
-#define _DMA_ZERO_EXTEND     _DMA_V2_ZERO_EXTEND
-#define _DMA_SIGN_EXTEND     _DMA_V2_SIGN_EXTEND
-
-
-typedef unsigned int dma_channel;
-
-typedef enum {
-  dma_isp_to_bus_connection = HIVE_DMA_ISP_BUS_CONN,
-  dma_isp_to_ddr_connection = HIVE_DMA_ISP_DDR_CONN,
-  dma_bus_to_ddr_connection = HIVE_DMA_BUS_DDR_CONN,
-} dma_connection;
-
-typedef enum {
-  dma_zero_extension = _DMA_ZERO_EXTEND,
-  dma_sign_extension = _DMA_SIGN_EXTEND
-} dma_extension;
-
-
-#define DMA_PROP_SHIFT(val, param)       ((val) << _DMA_V2_ ## param ## _IDX)
-#define DMA_PROP_MASK(param)             ((1U << _DMA_V2_ ## param ## _BITS)-1)
-#define DMA_PACK(val, param)             DMA_PROP_SHIFT((val) & DMA_PROP_MASK(param), param)
-
-#define DMA_PACK_COMMAND(cmd)            DMA_PACK(cmd, CMD)
-#define DMA_PACK_CHANNEL(ch)             DMA_PACK(ch,  CHANNEL)
-#define DMA_PACK_PARAM(par)              DMA_PACK(par, PARAM)
-#define DMA_PACK_EXTENSION(ext)          DMA_PACK(ext, EXTENSION)
-#define DMA_PACK_LEFT_CROPPING(lc)       DMA_PACK(lc,  LEFT_CROPPING)
-#define DMA_PACK_WIDTH_A(w)              DMA_PACK(w,   SPEC_DEV_A_XB)
-#define DMA_PACK_WIDTH_B(w)              DMA_PACK(w,   SPEC_DEV_B_XB)
-#define DMA_PACK_HEIGHT(h)               DMA_PACK(h,   SPEC_YB)
-
-#define DMA_PACK_CMD_CHANNEL(cmd, ch)	 (DMA_PACK_COMMAND(cmd) | DMA_PACK_CHANNEL(ch))
-#define DMA_PACK_SETUP(conn, ext)        ((conn) | DMA_PACK_EXTENSION(ext))
-#define DMA_PACK_CROP_ELEMS(elems, crop) ((elems) | DMA_PACK_LEFT_CROPPING(crop))
-
-#define hive_dma_snd(dma_id, token) OP_std_snd(dma_id, (unsigned int)(token))
-
-#define DMA_PACK_BLOCK_CMD(cmd, ch, width_a, width_b, height) \
-  (DMA_PACK_COMMAND(cmd)     | \
-   DMA_PACK_CHANNEL(ch)      | \
-   DMA_PACK_WIDTH_A(width_a) | \
-   DMA_PACK_WIDTH_B(width_b) | \
-   DMA_PACK_HEIGHT(height))
-
-#define hive_dma_move_data(dma_id, read, channel, addr_a, addr_b, to_is_var, from_is_var) \
-{ \
-  hive_dma_snd(dma_id, DMA_PACK(_DMA_V2_SET_CRUN_COMMAND, CMD)); \
-  hive_dma_snd(dma_id, DMA_PACK_CMD_CHANNEL(read?_DMA_V2_MOVE_B2A_COMMAND:_DMA_V2_MOVE_A2B_COMMAND, channel)); \
-  hive_dma_snd(dma_id, read?(unsigned)(addr_b):(unsigned)(addr_a)); \
-  hive_dma_snd(dma_id, read?(unsigned)(addr_a):(unsigned)(addr_b)); \
-  hive_dma_snd(dma_id, to_is_var); \
-  hive_dma_snd(dma_id, from_is_var); \
-}
-#define hive_dma_move_data_no_ack(dma_id, read, channel, addr_a, addr_b, to_is_var, from_is_var) \
-{ \
-  hive_dma_snd(dma_id, DMA_PACK(_DMA_V2_SET_CRUN_COMMAND, CMD)); \
-  hive_dma_snd(dma_id, DMA_PACK_CMD_CHANNEL(read?_DMA_V2_NO_ACK_MOVE_B2A_NO_SYNC_CHK_COMMAND:_DMA_V2_NO_ACK_MOVE_A2B_NO_SYNC_CHK_COMMAND, channel)); \
-  hive_dma_snd(dma_id, read?(unsigned)(addr_b):(unsigned)(addr_a)); \
-  hive_dma_snd(dma_id, read?(unsigned)(addr_a):(unsigned)(addr_b)); \
-  hive_dma_snd(dma_id, to_is_var); \
-  hive_dma_snd(dma_id, from_is_var); \
-}
-
-#define hive_dma_move_b2a_data(dma_id, channel, to_addr, from_addr, to_is_var, from_is_var) \
-{ \
-  hive_dma_move_data(dma_id, true, channel, to_addr, from_addr, to_is_var, from_is_var) \
-}
-
-#define hive_dma_move_a2b_data(dma_id, channel, from_addr, to_addr, from_is_var, to_is_var) \
-{ \
-  hive_dma_move_data(dma_id, false, channel, from_addr, to_addr, from_is_var, to_is_var) \
-}
-
-#define hive_dma_set_data(dma_id, channel, address, value, is_var) \
-{ \
-  hive_dma_snd(dma_id, DMA_PACK(_DMA_V2_SET_CRUN_COMMAND, CMD)); \
-  hive_dma_snd(dma_id, DMA_PACK_CMD_CHANNEL(_DMA_V2_INIT_A_COMMAND, channel)); \
-  hive_dma_snd(dma_id, value); \
-  hive_dma_snd(dma_id, address); \
-  hive_dma_snd(dma_id, is_var); \
-}
-
-#define hive_dma_clear_data(dma_id, channel, address, is_var) hive_dma_set_data(dma_id, channel, address, 0, is_var)
-
-#define hive_dma_configure(dma_id, channel, connection, extension, height, \
-	stride_A, elems_A, cropping_A, width_A, \
-	stride_B, elems_B, cropping_B, width_B) \
-{ \
-  hive_dma_snd(dma_id, DMA_PACK_CMD_CHANNEL(_DMA_V2_CONFIG_CHANNEL_COMMAND, channel)); \
-  hive_dma_snd(dma_id, DMA_PACK_SETUP(connection, extension)); \
-  hive_dma_snd(dma_id, stride_A); \
-  hive_dma_snd(dma_id, DMA_PACK_CROP_ELEMS(elems_A, cropping_A)); \
-  hive_dma_snd(dma_id, width_A); \
-  hive_dma_snd(dma_id, stride_B); \
-  hive_dma_snd(dma_id, DMA_PACK_CROP_ELEMS(elems_B, cropping_B)); \
-  hive_dma_snd(dma_id, width_B); \
-  hive_dma_snd(dma_id, height); \
-}
-
-#define hive_dma_execute(dma_id, channel, cmd, to_addr, from_addr_value, to_is_var, from_is_var) \
-{ \
-  hive_dma_snd(dma_id, DMA_PACK(_DMA_V2_SET_CRUN_COMMAND, CMD)); \
-  hive_dma_snd(dma_id, DMA_PACK_CMD_CHANNEL(cmd, channel)); \
-  hive_dma_snd(dma_id, to_addr); \
-  hive_dma_snd(dma_id, from_addr_value); \
-  hive_dma_snd(dma_id, to_is_var); \
-  if ((cmd & DMA_CLEAR_CMDBIT) == 0) { \
-	hive_dma_snd(dma_id, from_is_var); \
-  } \
-}
-
-#define hive_dma_configure_fast(dma_id, channel, connection, extension, elems_A, elems_B) \
-{ \
-  hive_dma_snd(dma_id, DMA_PACK_CMD_CHANNEL(_DMA_V2_CONFIG_CHANNEL_COMMAND, channel)); \
-  hive_dma_snd(dma_id, DMA_PACK_SETUP(connection, extension)); \
-  hive_dma_snd(dma_id, 0); \
-  hive_dma_snd(dma_id, DMA_PACK_CROP_ELEMS(elems_A, 0)); \
-  hive_dma_snd(dma_id, 0); \
-  hive_dma_snd(dma_id, 0); \
-  hive_dma_snd(dma_id, DMA_PACK_CROP_ELEMS(elems_B, 0)); \
-  hive_dma_snd(dma_id, 0); \
-  hive_dma_snd(dma_id, 1); \
-}
-
-#define hive_dma_set_parameter(dma_id, channel, param, value) \
-{ \
-  hive_dma_snd(dma_id, _DMA_V2_SET_CHANNEL_PARAM_COMMAND | DMA_PACK_CHANNEL(channel) | DMA_PACK_PARAM(param)); \
-  hive_dma_snd(dma_id, value); \
-}
-
-#define	DMA_SPECIFIC_CMDBIT	0x01
-#define	DMA_CHECK_CMDBIT	0x02
-#define	DMA_RW_CMDBIT		0x04
-#define	DMA_CLEAR_CMDBIT	0x08
-#define	DMA_ACK_CMDBIT		0x10
-#define	DMA_CFG_CMDBIT		0x20
-#define	DMA_PARAM_CMDBIT	0x01
-
-/* Write complete check not necessary if there's no ack */
-#define	DMA_NOACK_CMD		(DMA_ACK_CMDBIT | DMA_CHECK_CMDBIT)
-#define	DMA_CFG_CMD			(DMA_CFG_CMDBIT)
-#define	DMA_CFGPARAM_CMD	(DMA_CFG_CMDBIT | DMA_PARAM_CMDBIT)
-
-#define DMA_CMD_NEEDS_ACK(cmd) ((cmd & DMA_NOACK_CMD) == 0)
-#define DMA_CMD_IS_TRANSFER(cmd) ((cmd & DMA_CFG_CMDBIT) == 0)
-#define DMA_CMD_IS_WR(cmd) ((cmd & DMA_RW_CMDBIT) != 0)
-#define DMA_CMD_IS_RD(cmd) ((cmd & DMA_RW_CMDBIT) == 0)
-#define DMA_CMD_IS_CLR(cmd) ((cmd & DMA_CLEAR_CMDBIT) != 0)
-#define DMA_CMD_IS_CFG(cmd) ((cmd & DMA_CFG_CMDBIT) != 0)
-#define DMA_CMD_IS_PARAMCFG(cmd) ((cmd & DMA_CFGPARAM_CMD) == DMA_CFGPARAM_CMD)
-
-/* As a matter of convention */
-#define DMA_TRANSFER_READ		DMA_TRANSFER_B2A
-#define DMA_TRANSFER_WRITE		DMA_TRANSFER_A2B
-/* store/load from the PoV of the system(memory) */
-#define DMA_TRANSFER_STORE		DMA_TRANSFER_B2A
-#define DMA_TRANSFER_LOAD		DMA_TRANSFER_A2B
-#define DMA_TRANSFER_CLEAR		DMA_TRANSFER_CLEAR_A
-
-typedef enum {
-	DMA_TRANSFER_CLEAR_A = DMA_CLEAR_CMDBIT,                                       /* 8 */
-	DMA_TRANSFER_CLEAR_B = DMA_CLEAR_CMDBIT | DMA_RW_CMDBIT,                       /* 12 */
-	DMA_TRANSFER_A2B = DMA_RW_CMDBIT,                                              /* 4 */
-	DMA_TRANSFER_B2A = 0,                                                          /* 0 */
-	DMA_TRANSFER_CLEAR_A_NOACK = DMA_CLEAR_CMDBIT | DMA_NOACK_CMD,                 /* 26 */
-	DMA_TRANSFER_CLEAR_B_NOACK = DMA_CLEAR_CMDBIT | DMA_RW_CMDBIT | DMA_NOACK_CMD, /* 30 */
-	DMA_TRANSFER_A2B_NOACK = DMA_RW_CMDBIT | DMA_NOACK_CMD,                        /* 22 */
-	DMA_TRANSFER_B2A_NOACK = DMA_NOACK_CMD,                                        /* 18 */
-	DMA_FASTTRANSFER_CLEAR_A = DMA_CLEAR_CMDBIT | DMA_SPECIFIC_CMDBIT,
-	DMA_FASTTRANSFER_CLEAR_B = DMA_CLEAR_CMDBIT | DMA_RW_CMDBIT | DMA_SPECIFIC_CMDBIT,
-	DMA_FASTTRANSFER_A2B = DMA_RW_CMDBIT | DMA_SPECIFIC_CMDBIT,
-	DMA_FASTTRANSFER_B2A = DMA_SPECIFIC_CMDBIT,
-	DMA_FASTTRANSFER_CLEAR_A_NOACK = DMA_CLEAR_CMDBIT | DMA_NOACK_CMD | DMA_SPECIFIC_CMDBIT,
-	DMA_FASTTRANSFER_CLEAR_B_NOACK = DMA_CLEAR_CMDBIT | DMA_RW_CMDBIT | DMA_NOACK_CMD | DMA_SPECIFIC_CMDBIT,
-	DMA_FASTTRANSFER_A2B_NOACK = DMA_RW_CMDBIT | DMA_NOACK_CMD | DMA_SPECIFIC_CMDBIT,
-	DMA_FASTTRANSFER_B2A_NOACK = DMA_NOACK_CMD | DMA_SPECIFIC_CMDBIT,
-} dma_transfer_type_t;
-
-typedef enum {
-	DMA_CONFIG_SETUP = _DMA_V2_PACKING_SETUP_PARAM,
-	DMA_CONFIG_HEIGHT = _DMA_V2_HEIGHT_PARAM,
-	DMA_CONFIG_STRIDE_A_ = _DMA_V2_STRIDE_A_PARAM,
-	DMA_CONFIG_CROP_ELEM_A = _DMA_V2_ELEM_CROPPING_A_PARAM,
-	DMA_CONFIG_WIDTH_A = _DMA_V2_WIDTH_A_PARAM,
-	DMA_CONFIG_STRIDE_B_ = _DMA_V2_STRIDE_B_PARAM,
-	DMA_CONFIG_CROP_ELEM_B = _DMA_V2_ELEM_CROPPING_B_PARAM,
-	DMA_CONFIG_WIDTH_B = _DMA_V2_WIDTH_B_PARAM,
-} dma_config_type_t;
-
-struct dma_port_config {
-	uint8_t  crop, elems;
-	uint16_t width;
-	uint32_t stride;
-};
-
-/* Descriptor for dma configuration */
-struct dma_channel_config {
-	uint8_t  connection;
-	uint8_t  extension;
-	uint8_t  height;
-	struct dma_port_config a, b;
-};
-
-#endif /* __DMA_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/event_fifo_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/event_fifo_global.h
deleted file mode 100644
index 4df7a405cdcf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/event_fifo_global.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __EVENT_FIFO_GLOBAL_H
-#define __EVENT_FIFO_GLOBAL_H
-
-/*#error "event_global.h: No global event information permitted"*/
-
-#endif /* __EVENT_FIFO_GLOBAL_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/fifo_monitor_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/fifo_monitor_global.h
deleted file mode 100644
index f43bf0ad2468..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/fifo_monitor_global.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __FIFO_MONITOR_GLOBAL_H_INCLUDED__
-#define __FIFO_MONITOR_GLOBAL_H_INCLUDED__
-
-#define IS_FIFO_MONITOR_VERSION_2
-
-/*
-#define HIVE_ISP_CSS_STREAM_SWITCH_NONE      0
-#define HIVE_ISP_CSS_STREAM_SWITCH_SP        1
-#define HIVE_ISP_CSS_STREAM_SWITCH_ISP       2
- *
- * Actually, "HIVE_ISP_CSS_STREAM_SWITCH_SP = 1", "HIVE_ISP_CSS_STREAM_SWITCH_ISP = 0"
- * "hive_isp_css_stream_switch_hrt.h"
- */
-#define HIVE_ISP_CSS_STREAM_SWITCH_ISP       0
-#define HIVE_ISP_CSS_STREAM_SWITCH_SP        1
-#define HIVE_ISP_CSS_STREAM_SWITCH_NONE      2
-
-#endif /* __FIFO_MONITOR_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gdc_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gdc_global.h
deleted file mode 100644
index 4505775b224c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gdc_global.h
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GDC_GLOBAL_H_INCLUDED__
-#define __GDC_GLOBAL_H_INCLUDED__
-
-#define IS_GDC_VERSION_2
-
-#include <type_support.h>
-#include "gdc_v2_defs.h"
-
-/*
- * Storage addresses for packed data transfer
- */
-#define GDC_PARAM_ICX_LEFT_ROUNDED_IDX            0
-#define GDC_PARAM_OXDIM_FLOORED_IDX               1
-#define GDC_PARAM_OXDIM_LAST_IDX                  2
-#define GDC_PARAM_WOIX_LAST_IDX                   3
-#define GDC_PARAM_IY_TOPLEFT_IDX                  4
-#define GDC_PARAM_CHUNK_CNT_IDX                   5
-/*#define GDC_PARAM_ELEMENTS_PER_XMEM_ADDR_IDX    6 */		/* Derived from bpp */
-#define GDC_PARAM_BPP_IDX                         6
-#define GDC_PARAM_BLOCK_HEIGHT_IDX                7
-/*#define GDC_PARAM_DMA_CHANNEL_STRIDE_A_IDX      8*/		/* The DMA stride == the GDC buffer stride */
-#define GDC_PARAM_WOIX_IDX                        8
-#define GDC_PARAM_DMA_CHANNEL_STRIDE_B_IDX        9
-#define GDC_PARAM_DMA_CHANNEL_WIDTH_A_IDX        10
-#define GDC_PARAM_DMA_CHANNEL_WIDTH_B_IDX        11
-#define GDC_PARAM_VECTORS_PER_LINE_IN_IDX        12
-#define GDC_PARAM_VECTORS_PER_LINE_OUT_IDX       13
-#define GDC_PARAM_VMEM_IN_DIMY_IDX               14
-#define GDC_PARAM_COMMAND_IDX                    15
-#define N_GDC_PARAM                              16
-
-/* Because of the packed parameter transfer max(params) == max(fragments) */
-#define	N_GDC_FRAGMENTS		N_GDC_PARAM
-
-/* The GDC is capable of higher internal precision than the parameter data structures */
-#define HRT_GDC_COORD_SCALE_BITS	6
-#define HRT_GDC_COORD_SCALE			(1 << HRT_GDC_COORD_SCALE_BITS)
-
-typedef enum {
-	GDC_CH0_ID = 0,
-	N_GDC_CHANNEL_ID
-} gdc_channel_ID_t;
-
-typedef enum {
-	gdc_8_bpp  = 8,
-	gdc_10_bpp = 10,
-	gdc_12_bpp = 12,
-	gdc_14_bpp = 14
-} gdc_bits_per_pixel_t;
-
-typedef struct gdc_scale_param_mem_s {
-	uint16_t  params[N_GDC_PARAM];
-	uint16_t  ipx_start_array[N_GDC_PARAM];
-	uint16_t  ibuf_offset[N_GDC_PARAM];
-	uint16_t  obuf_offset[N_GDC_PARAM];
-} gdc_scale_param_mem_t;
-
-typedef struct gdc_warp_param_mem_s {
-	uint32_t      origin_x;
-	uint32_t      origin_y;
-	uint32_t      in_addr_offset;
-	uint32_t      in_block_width;
-	uint32_t      in_block_height;
-	uint32_t      p0_x;
-	uint32_t      p0_y;
-	uint32_t      p1_x;
-	uint32_t      p1_y;
-	uint32_t      p2_x;
-	uint32_t      p2_y;
-	uint32_t      p3_x;
-	uint32_t      p3_y;
-	uint32_t      padding[3];
-} gdc_warp_param_mem_t;
-
-
-#endif /* __GDC_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gp_device_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gp_device_global.h
deleted file mode 100644
index 30ad77059d93..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gp_device_global.h
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GP_DEVICE_GLOBAL_H_INCLUDED__
-#define __GP_DEVICE_GLOBAL_H_INCLUDED__
-
-#define IS_GP_DEVICE_VERSION_2
-
-#define _REG_GP_IRQ_REQ0_ADDR				0x08
-#define _REG_GP_IRQ_REQ1_ADDR				0x0C
-/* The SP sends SW interrupt info to this register */
-#define _REG_GP_IRQ_REQUEST0_ADDR			_REG_GP_IRQ_REQ0_ADDR
-#define _REG_GP_IRQ_REQUEST1_ADDR			_REG_GP_IRQ_REQ1_ADDR
-
-/* The SP configures FIFO switches in these registers */
-#define _REG_GP_SWITCH_IF_ADDR						0x40
-#define _REG_GP_SWITCH_GDC1_ADDR					0x44
-#define _REG_GP_SWITCH_GDC2_ADDR					0x48
-/* @ INPUT_FORMATTER_BASE -> GP_DEVICE_BASE */
-#define _REG_GP_IFMT_input_switch_lut_reg0			0x00030800
-#define _REG_GP_IFMT_input_switch_lut_reg1			0x00030804
-#define _REG_GP_IFMT_input_switch_lut_reg2			0x00030808
-#define _REG_GP_IFMT_input_switch_lut_reg3			0x0003080C
-#define _REG_GP_IFMT_input_switch_lut_reg4			0x00030810
-#define _REG_GP_IFMT_input_switch_lut_reg5			0x00030814
-#define _REG_GP_IFMT_input_switch_lut_reg6			0x00030818
-#define _REG_GP_IFMT_input_switch_lut_reg7			0x0003081C
-#define _REG_GP_IFMT_input_switch_fsync_lut			0x00030820
-#define _REG_GP_IFMT_srst							0x00030824
-#define _REG_GP_IFMT_slv_reg_srst					0x00030828
-#define _REG_GP_IFMT_input_switch_ch_id_fmt_type	0x0003082C
-
-/* @ GP_DEVICE_BASE */
-#define _REG_GP_SYNCGEN_ENABLE_ADDR					0x00090000
-#define _REG_GP_SYNCGEN_FREE_RUNNING_ADDR			0x00090004
-#define _REG_GP_SYNCGEN_PAUSE_ADDR					0x00090008
-#define _REG_GP_NR_FRAMES_ADDR						0x0009000C
-#define _REG_GP_SYNGEN_NR_PIX_ADDR					0x00090010
-#define _REG_GP_SYNGEN_NR_LINES_ADDR				0x00090014
-#define _REG_GP_SYNGEN_HBLANK_CYCLES_ADDR			0x00090018
-#define _REG_GP_SYNGEN_VBLANK_CYCLES_ADDR			0x0009001C
-#define _REG_GP_ISEL_SOF_ADDR						0x00090020
-#define _REG_GP_ISEL_EOF_ADDR						0x00090024
-#define _REG_GP_ISEL_SOL_ADDR						0x00090028
-#define _REG_GP_ISEL_EOL_ADDR						0x0009002C
-#define _REG_GP_ISEL_LFSR_ENABLE_ADDR				0x00090030
-#define _REG_GP_ISEL_LFSR_ENABLE_B_ADDR				0x00090034
-#define _REG_GP_ISEL_LFSR_RESET_VALUE_ADDR			0x00090038
-#define _REG_GP_ISEL_TPG_ENABLE_ADDR				0x0009003C
-#define _REG_GP_ISEL_TPG_ENABLE_B_ADDR				0x00090040
-#define _REG_GP_ISEL_HOR_CNT_MASK_ADDR				0x00090044
-#define _REG_GP_ISEL_VER_CNT_MASK_ADDR				0x00090048
-#define _REG_GP_ISEL_XY_CNT_MASK_ADDR				0x0009004C
-#define _REG_GP_ISEL_HOR_CNT_DELTA_ADDR				0x00090050
-#define _REG_GP_ISEL_VER_CNT_DELTA_ADDR				0x00090054
-#define _REG_GP_ISEL_TPG_MODE_ADDR					0x00090058
-#define _REG_GP_ISEL_TPG_RED1_ADDR					0x0009005C
-#define _REG_GP_ISEL_TPG_GREEN1_ADDR				0x00090060
-#define _REG_GP_ISEL_TPG_BLUE1_ADDR					0x00090064
-#define _REG_GP_ISEL_TPG_RED2_ADDR					0x00090068
-#define _REG_GP_ISEL_TPG_GREEN2_ADDR				0x0009006C
-#define _REG_GP_ISEL_TPG_BLUE2_ADDR					0x00090070
-#define _REG_GP_ISEL_CH_ID_ADDR						0x00090074
-#define _REG_GP_ISEL_FMT_TYPE_ADDR					0x00090078
-#define _REG_GP_ISEL_DATA_SEL_ADDR					0x0009007C
-#define _REG_GP_ISEL_SBAND_SEL_ADDR					0x00090080
-#define _REG_GP_ISEL_SYNC_SEL_ADDR					0x00090084
-#define _REG_GP_SYNCGEN_HOR_CNT_ADDR				0x00090088
-#define _REG_GP_SYNCGEN_VER_CNT_ADDR				0x0009008C
-#define _REG_GP_SYNCGEN_FRAME_CNT_ADDR				0x00090090
-#define _REG_GP_SOFT_RESET_ADDR						0x00090094
-
-
-#endif /* __GP_DEVICE_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gp_timer_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gp_timer_global.h
deleted file mode 100644
index ee636ad6c5b3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gp_timer_global.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GP_TIMER_GLOBAL_H_INCLUDED__
-#define __GP_TIMER_GLOBAL_H_INCLUDED__
-
-#include "hive_isp_css_defs.h" /*HIVE_GP_TIMER_SP_DMEM_ERROR_IRQ */
-
-/* from gp_timer_defs.h*/
-#define GP_TIMER_COUNT_TYPE_HIGH             0
-#define GP_TIMER_COUNT_TYPE_LOW              1
-#define GP_TIMER_COUNT_TYPE_POSEDGE          2
-#define GP_TIMER_COUNT_TYPE_NEGEDGE          3
-#define GP_TIMER_COUNT_TYPE_TYPES            4
-
-/* timer - 3 is selected */
-#define GP_TIMER_SEL                         3
-
-/*HIVE_GP_TIMER_SP_DMEM_ERROR_IRQ is selected*/
-#define GP_TIMER_SIGNAL_SELECT  HIVE_GP_TIMER_SP_DMEM_ERROR_IRQ
-
-#endif /* __GP_TIMER_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gpio_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gpio_global.h
deleted file mode 100644
index a82ca2a8cada..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/gpio_global.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GPIO_GLOBAL_H_INCLUDED__
-#define __GPIO_GLOBAL_H_INCLUDED__
-
-#define IS_GPIO_VERSION_1
-
-#include <gpio_block_defs.h>
-
-/* pqiao: following part only defines in hive_isp_css_defs.h in fpga system.
-	port it here
-*/
-
-/* GPIO pin defines */
-/*#define HIVE_GPIO_CAMERA_BOARD_RESET_PIN_NR                   0
-#define HIVE_GPIO_LCD_CLOCK_SELECT_PIN_NR                     7
-#define HIVE_GPIO_HDMI_CLOCK_SELECT_PIN_NR                    8
-#define HIVE_GPIO_LCD_VERT_FLIP_PIN_NR                        8
-#define HIVE_GPIO_LCD_HOR_FLIP_PIN_NR                         9
-#define HIVE_GPIO_AS3683_GPIO_P0_PIN_NR                       1
-#define HIVE_GPIO_AS3683_DATA_P1_PIN_NR                       2
-#define HIVE_GPIO_AS3683_CLK_P2_PIN_NR                        3
-#define HIVE_GPIO_AS3683_T1_F0_PIN_NR                         4
-#define HIVE_GPIO_AS3683_SFL_F1_PIN_NR                        5
-#define HIVE_GPIO_AS3683_STROBE_F2_PIN_NR                     6
-#define HIVE_GPIO_MAX1577_EN1_PIN_NR                          1
-#define HIVE_GPIO_MAX1577_EN2_PIN_NR                          2
-#define HIVE_GPIO_MAX8685A_EN_PIN_NR                          3
-#define HIVE_GPIO_MAX8685A_TRIG_PIN_NR                        4*/
-
-#define HIVE_GPIO_STROBE_TRIGGER_PIN		2
-
-#endif /* __GPIO_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/hmem_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/hmem_global.h
deleted file mode 100644
index 7e05d7d880d1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/hmem_global.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __HMEM_GLOBAL_H_INCLUDED__
-#define __HMEM_GLOBAL_H_INCLUDED__
-
-#include <type_support.h>
-
-#define IS_HMEM_VERSION_1
-
-#include "isp.h"
-
-/*
-#define ISP_HIST_ADDRESS_BITS                  12
-#define ISP_HIST_ALIGNMENT                     4
-#define ISP_HIST_COMP_IN_PREC                  12
-#define ISP_HIST_DEPTH                         1024
-#define ISP_HIST_WIDTH                         24
-#define ISP_HIST_COMPONENTS                    4
-*/
-#define ISP_HIST_ALIGNMENT_LOG2		2
-
-#define HMEM_SIZE_LOG2		(ISP_HIST_ADDRESS_BITS-ISP_HIST_ALIGNMENT_LOG2)
-#define HMEM_SIZE			ISP_HIST_DEPTH
-
-#define HMEM_UNIT_SIZE		(HMEM_SIZE/ISP_HIST_COMPONENTS)
-#define HMEM_UNIT_COUNT		ISP_HIST_COMPONENTS
-
-#define HMEM_RANGE_LOG2		ISP_HIST_WIDTH
-#define HMEM_RANGE			(1UL<<HMEM_RANGE_LOG2)
-
-typedef uint32_t			hmem_data_t;
-
-#endif /* __HMEM_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug.c
deleted file mode 100644
index dcb9a3127cfe..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2016, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "debug.h"
-
-#ifndef __INLINE_DEBUG__
-#include "debug_private.h"
-#endif /* __INLINE_DEBUG__ */
-
-#include "memory_access.h"
-
-#define __INLINE_SP__
-#include "sp.h"
-
-#include "assert_support.h"
-
-/* The address of the remote copy */
-hrt_address	debug_buffer_address = (hrt_address)-1;
-hrt_vaddress	debug_buffer_ddr_address = (hrt_vaddress)-1;
-/* The local copy */
-static debug_data_t		debug_data;
-debug_data_t		*debug_data_ptr = &debug_data;
-
-void debug_buffer_init(const hrt_address addr)
-{
-	debug_buffer_address = addr;
-
-	debug_data.head = 0;
-	debug_data.tail = 0;
-}
-
-void debug_buffer_ddr_init(const hrt_vaddress addr)
-{
-	debug_buf_mode_t mode = DEBUG_BUFFER_MODE_LINEAR;
-	uint32_t enable = 1;
-	uint32_t head = 0;
-	uint32_t tail = 0;
-	/* set the ddr queue */
-	debug_buffer_ddr_address = addr;
-	mmgr_store(addr + DEBUG_DATA_BUF_MODE_DDR_ADDR,
-				&mode, sizeof(debug_buf_mode_t));
-	mmgr_store(addr + DEBUG_DATA_HEAD_DDR_ADDR,
-				&head, sizeof(uint32_t));
-	mmgr_store(addr + DEBUG_DATA_TAIL_DDR_ADDR,
-				&tail, sizeof(uint32_t));
-	mmgr_store(addr + DEBUG_DATA_ENABLE_DDR_ADDR,
-				&enable, sizeof(uint32_t));
-
-	/* set the local copy */
-	debug_data.head = 0;
-	debug_data.tail = 0;
-}
-
-void debug_buffer_setmode(const debug_buf_mode_t mode)
-{
-	assert(debug_buffer_address != ((hrt_address)-1));
-
-	sp_dmem_store_uint32(SP0_ID,
-		debug_buffer_address + DEBUG_DATA_BUF_MODE_ADDR, mode);
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug_local.h
deleted file mode 100644
index 2b0c5f48c0e7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug_local.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __DEBUG_LOCAL_H_INCLUDED__
-#define __DEBUG_LOCAL_H_INCLUDED__
-
-#include "debug_global.h"
-
-#endif /* __DEBUG_LOCAL_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug_private.h
deleted file mode 100644
index a047aadc6619..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/debug_private.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __DEBUG_PRIVATE_H_INCLUDED__
-#define __DEBUG_PRIVATE_H_INCLUDED__
-
-#include "debug_public.h"
-
-#include "sp.h"
-
-#define __INLINE_ISP__
-#include "isp.h"
-
-#include "memory_access.h"
-
-#include "assert_support.h"
-
-STORAGE_CLASS_DEBUG_C bool is_debug_buffer_empty(void)
-{
-	return (debug_data_ptr->head == debug_data_ptr->tail);
-}
-
-STORAGE_CLASS_DEBUG_C hrt_data debug_dequeue(void)
-{
-	hrt_data value = 0;
-
-	assert(debug_buffer_address != ((hrt_address)-1));
-
-	debug_synch_queue();
-
-	if (!is_debug_buffer_empty()) {
-		value = debug_data_ptr->buf[debug_data_ptr->head];
-		debug_data_ptr->head = (debug_data_ptr->head + 1) & DEBUG_BUF_MASK;
-		sp_dmem_store_uint32(SP0_ID, debug_buffer_address + DEBUG_DATA_HEAD_ADDR, debug_data_ptr->head);
-	}
-
-	return value;
-}
-
-STORAGE_CLASS_DEBUG_C void debug_synch_queue(void)
-{
-	uint32_t remote_tail = sp_dmem_load_uint32(SP0_ID, debug_buffer_address + DEBUG_DATA_TAIL_ADDR);
-/* We could move the remote head after the upload, but we would have to limit the upload w.r.t. the local head. This is easier */
-	if (remote_tail > debug_data_ptr->tail) {
-		size_t	delta = remote_tail - debug_data_ptr->tail;
-		sp_dmem_load(SP0_ID, debug_buffer_address + DEBUG_DATA_BUF_ADDR + debug_data_ptr->tail*sizeof(uint32_t), (void *)&(debug_data_ptr->buf[debug_data_ptr->tail]), delta*sizeof(uint32_t));
-	} else if (remote_tail < debug_data_ptr->tail) {
-		size_t	delta = DEBUG_BUF_SIZE - debug_data_ptr->tail;
-		sp_dmem_load(SP0_ID, debug_buffer_address + DEBUG_DATA_BUF_ADDR + debug_data_ptr->tail*sizeof(uint32_t), (void *)&(debug_data_ptr->buf[debug_data_ptr->tail]), delta*sizeof(uint32_t));
-		sp_dmem_load(SP0_ID, debug_buffer_address + DEBUG_DATA_BUF_ADDR, (void *)&(debug_data_ptr->buf[0]), remote_tail*sizeof(uint32_t));
-	} /* else we are up to date */
-	debug_data_ptr->tail = remote_tail;
-}
-
-STORAGE_CLASS_DEBUG_C void debug_synch_queue_isp(void)
-{
-	uint32_t remote_tail = isp_dmem_load_uint32(ISP0_ID, DEBUG_BUFFER_ISP_DMEM_ADDR + DEBUG_DATA_TAIL_ADDR);
-/* We could move the remote head after the upload, but we would have to limit the upload w.r.t. the local head. This is easier */
-	if (remote_tail > debug_data_ptr->tail) {
-		size_t	delta = remote_tail - debug_data_ptr->tail;
-		isp_dmem_load(ISP0_ID, DEBUG_BUFFER_ISP_DMEM_ADDR + DEBUG_DATA_BUF_ADDR + debug_data_ptr->tail*sizeof(uint32_t), (void *)&(debug_data_ptr->buf[debug_data_ptr->tail]), delta*sizeof(uint32_t));
-	} else if (remote_tail < debug_data_ptr->tail) {
-		size_t	delta = DEBUG_BUF_SIZE - debug_data_ptr->tail;
-		isp_dmem_load(ISP0_ID, DEBUG_BUFFER_ISP_DMEM_ADDR + DEBUG_DATA_BUF_ADDR + debug_data_ptr->tail*sizeof(uint32_t), (void *)&(debug_data_ptr->buf[debug_data_ptr->tail]), delta*sizeof(uint32_t));
-		isp_dmem_load(ISP0_ID, DEBUG_BUFFER_ISP_DMEM_ADDR + DEBUG_DATA_BUF_ADDR, (void *)&(debug_data_ptr->buf[0]), remote_tail*sizeof(uint32_t));
-	} /* else we are up to date */
-	debug_data_ptr->tail = remote_tail;
-}
-
-STORAGE_CLASS_DEBUG_C void debug_synch_queue_ddr(void)
-{
-	uint32_t	remote_tail;
-
-	mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_TAIL_DDR_ADDR, &remote_tail, sizeof(uint32_t));
-/* We could move the remote head after the upload, but we would have to limit the upload w.r.t. the local head. This is easier */
-	if (remote_tail > debug_data_ptr->tail) {
-		size_t	delta = remote_tail - debug_data_ptr->tail;
-		mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR + debug_data_ptr->tail*sizeof(uint32_t), (void *)&(debug_data_ptr->buf[debug_data_ptr->tail]), delta*sizeof(uint32_t));
-	} else if (remote_tail < debug_data_ptr->tail) {
-		size_t	delta = DEBUG_BUF_SIZE - debug_data_ptr->tail;
-		mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR + debug_data_ptr->tail*sizeof(uint32_t), (void *)&(debug_data_ptr->buf[debug_data_ptr->tail]), delta*sizeof(uint32_t));
-		mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR, (void *)&(debug_data_ptr->buf[0]), remote_tail*sizeof(uint32_t));
-	} /* else we are up to date */
-	debug_data_ptr->tail = remote_tail;
-}
-
-#endif /* __DEBUG_PRIVATE_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma.c
deleted file mode 100644
index 770db7dff5d3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma.c
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2016, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <linux/kernel.h>
-
-#include "dma.h"
-
-#include "assert_support.h"
-
-#ifndef __INLINE_DMA__
-#include "dma_private.h"
-#endif /* __INLINE_DMA__ */
-
-void dma_get_state(const dma_ID_t ID, dma_state_t *state)
-{
-	int			i;
-	hrt_data	tmp;
-
-	assert(ID < N_DMA_ID);
-	assert(state != NULL);
-
-	tmp = dma_reg_load(ID, DMA_COMMAND_FSM_REG_IDX);
-	//reg  [3:0] : flags error [3], stall, run, idle [0]
-	//reg  [9:4] : command
-	//reg[14:10] : channel
-	//reg [23:15] : param
-	state->fsm_command_idle = tmp & 0x1;
-	state->fsm_command_run = tmp & 0x2;
-	state->fsm_command_stalling = tmp & 0x4;
-	state->fsm_command_error    = tmp & 0x8;
-	state->last_command_channel = (tmp>>10 & 0x1F);
-	state->last_command_param =  (tmp>>15 & 0x0F);
-	tmp = (tmp>>4) & 0x3F;
-/* state->last_command = (dma_commands_t)tmp; */
-/* if the enumerator is made non-linear */
-	/* AM: the list below does not cover all the cases*/
-	/*  and these are not correct */
-	/* therefore for just dumpinmg this command*/
-	state->last_command = tmp;
-
-/*
-	if (tmp == 0)
-		state->last_command = DMA_COMMAND_READ;
-	if (tmp == 1)
-		state->last_command = DMA_COMMAND_WRITE;
-	if (tmp == 2)
-		state->last_command = DMA_COMMAND_SET_CHANNEL;
-	if (tmp == 3)
-		state->last_command = DMA_COMMAND_SET_PARAM;
-	if (tmp == 4)
-		state->last_command = DMA_COMMAND_READ_SPECIFIC;
-	if (tmp == 5)
-		state->last_command = DMA_COMMAND_WRITE_SPECIFIC;
-	if (tmp == 8)
-		state->last_command = DMA_COMMAND_INIT;
-	if (tmp == 12)
-		state->last_command = DMA_COMMAND_INIT_SPECIFIC;
-	if (tmp == 15)
-		state->last_command = DMA_COMMAND_RST;
-*/
-
-/* No sub-fields, idx = 0 */
-	state->current_command = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(0, _DMA_FSM_GROUP_CMD_IDX));
-	state->current_addr_a = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(0, _DMA_FSM_GROUP_ADDR_A_IDX));
-	state->current_addr_b = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(0, _DMA_FSM_GROUP_ADDR_B_IDX));
-
-	tmp =  dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_STATE_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_idle = tmp & 0x1;
-	state->fsm_ctrl_run = tmp & 0x2;
-	state->fsm_ctrl_stalling = tmp & 0x4;
-	state->fsm_ctrl_error = tmp & 0x8;
-	tmp = tmp >> 4;
-/* state->fsm_ctrl_state = (dma_ctrl_states_t)tmp; */
-	if (tmp == 0)
-		state->fsm_ctrl_state = DMA_CTRL_STATE_IDLE;
-	if (tmp == 1)
-		state->fsm_ctrl_state = DMA_CTRL_STATE_REQ_RCV;
-	if (tmp == 2)
-		state->fsm_ctrl_state = DMA_CTRL_STATE_RCV;
-	if (tmp == 3)
-		state->fsm_ctrl_state = DMA_CTRL_STATE_RCV_REQ;
-	if (tmp == 4)
-		state->fsm_ctrl_state = DMA_CTRL_STATE_INIT;
-	state->fsm_ctrl_source_dev = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_REQ_DEV_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_source_addr = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_REQ_ADDR_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_source_stride = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_REQ_STRIDE_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_source_width = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_REQ_XB_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_source_height = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_REQ_YB_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_pack_source_dev = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_PACK_REQ_DEV_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_pack_dest_dev = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_PACK_WR_DEV_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_dest_addr = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_WR_ADDR_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_dest_stride = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_WR_STRIDE_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_pack_source_width = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_PACK_REQ_XB_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_pack_dest_height = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_PACK_WR_YB_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_pack_dest_width = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_PACK_WR_XB_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_pack_source_elems = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_PACK_ELEM_REQ_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_pack_dest_elems = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_PACK_ELEM_WR_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-	state->fsm_ctrl_pack_extension = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_CTRL_PACK_S_Z_IDX,
-		_DMA_FSM_GROUP_FSM_CTRL_IDX));
-
-	tmp = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_PACK_STATE_IDX,
-		_DMA_FSM_GROUP_FSM_PACK_IDX));
-	state->pack_idle     = tmp & 0x1;
-	state->pack_run      = tmp & 0x2;
-	state->pack_stalling = tmp & 0x4;
-	state->pack_error    = tmp & 0x8;
-	state->pack_cnt_height = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_PACK_CNT_YB_IDX,
-		_DMA_FSM_GROUP_FSM_PACK_IDX));
-	state->pack_src_cnt_width = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_PACK_CNT_XB_REQ_IDX,
-		_DMA_FSM_GROUP_FSM_PACK_IDX));
-	state->pack_dest_cnt_width = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_PACK_CNT_XB_WR_IDX,
-		_DMA_FSM_GROUP_FSM_PACK_IDX));
-
-	tmp = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_REQ_STATE_IDX,
-		_DMA_FSM_GROUP_FSM_REQ_IDX));
-/* state->read_state = (dma_rw_states_t)tmp; */
-	if (tmp == 0)
-		state->read_state = DMA_RW_STATE_IDLE;
-	if (tmp == 1)
-		state->read_state = DMA_RW_STATE_REQ;
-	if (tmp == 2)
-		state->read_state = DMA_RW_STATE_NEXT_LINE;
-	if (tmp == 3)
-		state->read_state = DMA_RW_STATE_UNLOCK_CHANNEL;
-	state->read_cnt_height = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_REQ_CNT_YB_IDX,
-		_DMA_FSM_GROUP_FSM_REQ_IDX));
-	state->read_cnt_width = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_REQ_CNT_XB_IDX,
-		_DMA_FSM_GROUP_FSM_REQ_IDX));
-
-	tmp = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_WR_STATE_IDX,
-		_DMA_FSM_GROUP_FSM_WR_IDX));
-/* state->write_state = (dma_rw_states_t)tmp; */
-	if (tmp == 0)
-		state->write_state = DMA_RW_STATE_IDLE;
-	if (tmp == 1)
-		state->write_state = DMA_RW_STATE_REQ;
-	if (tmp == 2)
-		state->write_state = DMA_RW_STATE_NEXT_LINE;
-	if (tmp == 3)
-		state->write_state = DMA_RW_STATE_UNLOCK_CHANNEL;
-	state->write_height = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_WR_CNT_YB_IDX,
-		_DMA_FSM_GROUP_FSM_WR_IDX));
-	state->write_width = dma_reg_load(ID,
-		DMA_CG_INFO_REG_IDX(
-		_DMA_FSM_GROUP_FSM_WR_CNT_XB_IDX,
-		_DMA_FSM_GROUP_FSM_WR_IDX));
-
-	for (i = 0; i < HIVE_ISP_NUM_DMA_CONNS; i++) {
-		dma_port_state_t *port = &(state->port_states[i]);
-
-		tmp = dma_reg_load(ID, DMA_DEV_INFO_REG_IDX(0, i));
-		port->req_cs   = ((tmp & 0x1) != 0);
-		port->req_we_n = ((tmp & 0x2) != 0);
-		port->req_run  = ((tmp & 0x4) != 0);
-		port->req_ack  = ((tmp & 0x8) != 0);
-
-		tmp = dma_reg_load(ID, DMA_DEV_INFO_REG_IDX(1, i));
-		port->send_cs   = ((tmp & 0x1) != 0);
-		port->send_we_n = ((tmp & 0x2) != 0);
-		port->send_run  = ((tmp & 0x4) != 0);
-		port->send_ack  = ((tmp & 0x8) != 0);
-
-		tmp = dma_reg_load(ID, DMA_DEV_INFO_REG_IDX(2, i));
-		if (tmp & 0x1)
-			port->fifo_state = DMA_FIFO_STATE_WILL_BE_FULL;
-		if (tmp & 0x2)
-			port->fifo_state = DMA_FIFO_STATE_FULL;
-		if (tmp & 0x4)
-			port->fifo_state = DMA_FIFO_STATE_EMPTY;
-		port->fifo_counter = tmp >> 3;
-	}
-
-	for (i = 0; i < HIVE_DMA_NUM_CHANNELS; i++) {
-		dma_channel_state_t *ch = &(state->channel_states[i]);
-
-		ch->connection = DMA_GET_CONNECTION(dma_reg_load(ID,
-			DMA_CHANNEL_PARAM_REG_IDX(i,
-			_DMA_PACKING_SETUP_PARAM)));
-		ch->sign_extend = DMA_GET_EXTENSION(dma_reg_load(ID,
-			DMA_CHANNEL_PARAM_REG_IDX(i,
-			_DMA_PACKING_SETUP_PARAM)));
-		ch->height = dma_reg_load(ID,
-			DMA_CHANNEL_PARAM_REG_IDX(i,
-			_DMA_HEIGHT_PARAM));
-		ch->stride_a = dma_reg_load(ID,
-			DMA_CHANNEL_PARAM_REG_IDX(i,
-			_DMA_STRIDE_A_PARAM));
-		ch->elems_a = DMA_GET_ELEMENTS(dma_reg_load(ID,
-			DMA_CHANNEL_PARAM_REG_IDX(i,
-			_DMA_ELEM_CROPPING_A_PARAM)));
-		ch->cropping_a = DMA_GET_CROPPING(dma_reg_load(ID,
-			DMA_CHANNEL_PARAM_REG_IDX(i,
-			_DMA_ELEM_CROPPING_A_PARAM)));
-		ch->width_a = dma_reg_load(ID,
-			DMA_CHANNEL_PARAM_REG_IDX(i,
-			_DMA_WIDTH_A_PARAM));
-		ch->stride_b = dma_reg_load(ID,
-			DMA_CHANNEL_PARAM_REG_IDX(i,
-			_DMA_STRIDE_B_PARAM));
-		ch->elems_b = DMA_GET_ELEMENTS(dma_reg_load(ID,
-			DMA_CHANNEL_PARAM_REG_IDX(i,
-			_DMA_ELEM_CROPPING_B_PARAM)));
-		ch->cropping_b = DMA_GET_CROPPING(dma_reg_load(ID,
-			DMA_CHANNEL_PARAM_REG_IDX(i,
-			_DMA_ELEM_CROPPING_B_PARAM)));
-		ch->width_b = dma_reg_load(ID,
-			DMA_CHANNEL_PARAM_REG_IDX(i,
-			_DMA_WIDTH_B_PARAM));
-	}
-}
-
-void
-dma_set_max_burst_size(const dma_ID_t ID, dma_connection conn,
-		       uint32_t max_burst_size)
-{
-	assert(ID < N_DMA_ID);
-	assert(max_burst_size > 0);
-	dma_reg_store(ID, DMA_DEV_INFO_REG_IDX(_DMA_DEV_INTERF_MAX_BURST_IDX, conn),
-		      max_burst_size - 1);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma_local.h
deleted file mode 100644
index ab631e6f64b5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma_local.h
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __DMA_LOCAL_H_INCLUDED__
-#define __DMA_LOCAL_H_INCLUDED__
-
-#include <type_support.h>
-#include "dma_global.h"
-
-#include <hrt/defs.h>				/* HRTCAT() */
-#include <hrt/bits.h>				/* _hrt_get_bits() */
-#include <hive_isp_css_defs.h>		/* HIVE_DMA_NUM_CHANNELS */
-#include <dma_v2_defs.h>
-
-#define _DMA_FSM_GROUP_CMD_IDX						_DMA_V2_FSM_GROUP_CMD_IDX
-#define _DMA_FSM_GROUP_ADDR_A_IDX					_DMA_V2_FSM_GROUP_ADDR_SRC_IDX
-#define _DMA_FSM_GROUP_ADDR_B_IDX					_DMA_V2_FSM_GROUP_ADDR_DEST_IDX
-
-#define _DMA_FSM_GROUP_CMD_CTRL_IDX					_DMA_V2_FSM_GROUP_CMD_CTRL_IDX
-
-#define _DMA_FSM_GROUP_FSM_CTRL_IDX					_DMA_V2_FSM_GROUP_FSM_CTRL_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_STATE_IDX			_DMA_V2_FSM_GROUP_FSM_CTRL_STATE_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_REQ_DEV_IDX			_DMA_V2_FSM_GROUP_FSM_CTRL_REQ_DEV_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_REQ_ADDR_IDX		_DMA_V2_FSM_GROUP_FSM_CTRL_REQ_ADDR_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_REQ_STRIDE_IDX		_DMA_V2_FSM_GROUP_FSM_CTRL_REQ_STRIDE_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_REQ_XB_IDX			_DMA_V2_FSM_GROUP_FSM_CTRL_REQ_XB_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_REQ_YB_IDX			_DMA_V2_FSM_GROUP_FSM_CTRL_REQ_YB_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_PACK_REQ_DEV_IDX	_DMA_V2_FSM_GROUP_FSM_CTRL_PACK_REQ_DEV_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_PACK_WR_DEV_IDX		_DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_DEV_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_WR_ADDR_IDX			_DMA_V2_FSM_GROUP_FSM_CTRL_WR_ADDR_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_WR_STRIDE_IDX		_DMA_V2_FSM_GROUP_FSM_CTRL_WR_STRIDE_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_PACK_REQ_XB_IDX		_DMA_V2_FSM_GROUP_FSM_CTRL_PACK_REQ_XB_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_PACK_WR_YB_IDX		_DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_YB_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_PACK_WR_XB_IDX		_DMA_V2_FSM_GROUP_FSM_CTRL_PACK_WR_XB_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_PACK_ELEM_REQ_IDX	_DMA_V2_FSM_GROUP_FSM_CTRL_PACK_ELEM_REQ_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_PACK_ELEM_WR_IDX	_DMA_V2_FSM_GROUP_FSM_CTRL_PACK_ELEM_WR_IDX
-#define _DMA_FSM_GROUP_FSM_CTRL_PACK_S_Z_IDX		_DMA_V2_FSM_GROUP_FSM_CTRL_PACK_S_Z_IDX
-
-#define _DMA_FSM_GROUP_FSM_PACK_IDX					_DMA_V2_FSM_GROUP_FSM_PACK_IDX
-#define _DMA_FSM_GROUP_FSM_PACK_STATE_IDX			_DMA_V2_FSM_GROUP_FSM_PACK_STATE_IDX
-#define _DMA_FSM_GROUP_FSM_PACK_CNT_YB_IDX			_DMA_V2_FSM_GROUP_FSM_PACK_CNT_YB_IDX
-#define _DMA_FSM_GROUP_FSM_PACK_CNT_XB_REQ_IDX		_DMA_V2_FSM_GROUP_FSM_PACK_CNT_XB_REQ_IDX
-#define _DMA_FSM_GROUP_FSM_PACK_CNT_XB_WR_IDX		_DMA_V2_FSM_GROUP_FSM_PACK_CNT_XB_WR_IDX
-
-#define _DMA_FSM_GROUP_FSM_REQ_IDX					_DMA_V2_FSM_GROUP_FSM_REQ_IDX
-#define _DMA_FSM_GROUP_FSM_REQ_STATE_IDX			_DMA_V2_FSM_GROUP_FSM_REQ_STATE_IDX
-#define _DMA_FSM_GROUP_FSM_REQ_CNT_YB_IDX			_DMA_V2_FSM_GROUP_FSM_REQ_CNT_YB_IDX
-#define _DMA_FSM_GROUP_FSM_REQ_CNT_XB_IDX			_DMA_V2_FSM_GROUP_FSM_REQ_CNT_XB_IDX
-
-#define _DMA_FSM_GROUP_FSM_WR_IDX					_DMA_V2_FSM_GROUP_FSM_WR_IDX
-#define _DMA_FSM_GROUP_FSM_WR_STATE_IDX				_DMA_V2_FSM_GROUP_FSM_WR_STATE_IDX
-#define _DMA_FSM_GROUP_FSM_WR_CNT_YB_IDX			_DMA_V2_FSM_GROUP_FSM_WR_CNT_YB_IDX
-#define _DMA_FSM_GROUP_FSM_WR_CNT_XB_IDX			_DMA_V2_FSM_GROUP_FSM_WR_CNT_XB_IDX
-
-#define _DMA_DEV_INTERF_MAX_BURST_IDX			_DMA_V2_DEV_INTERF_MAX_BURST_IDX
-
-/*
- * Macro's to compute the DMA parameter register indices
- */
-#define DMA_SEL_COMP(comp)     (((comp)  & _hrt_ones(_DMA_V2_ADDR_SEL_COMP_BITS))            << _DMA_V2_ADDR_SEL_COMP_IDX)
-#define DMA_SEL_CH(ch)         (((ch)    & _hrt_ones(_DMA_V2_ADDR_SEL_CH_REG_BITS))          << _DMA_V2_ADDR_SEL_CH_REG_IDX)
-#define DMA_SEL_PARAM(param)   (((param) & _hrt_ones(_DMA_V2_ADDR_SEL_PARAM_BITS))           << _DMA_V2_ADDR_SEL_PARAM_IDX)
-/* CG = Connection Group */
-#define DMA_SEL_CG_INFO(info)  (((info)  & _hrt_ones(_DMA_V2_ADDR_SEL_GROUP_COMP_INFO_BITS)) << _DMA_V2_ADDR_SEL_GROUP_COMP_INFO_IDX)
-#define DMA_SEL_CG_COMP(comp)  (((comp)  & _hrt_ones(_DMA_V2_ADDR_SEL_GROUP_COMP_BITS))      << _DMA_V2_ADDR_SEL_GROUP_COMP_IDX)
-#define DMA_SEL_DEV_INFO(info) (((info)  & _hrt_ones(_DMA_V2_ADDR_SEL_DEV_INTERF_INFO_BITS)) << _DMA_V2_ADDR_SEL_DEV_INTERF_INFO_IDX)
-#define DMA_SEL_DEV_ID(dev)    (((dev)   & _hrt_ones(_DMA_V2_ADDR_SEL_DEV_INTERF_IDX_BITS))  << _DMA_V2_ADDR_SEL_DEV_INTERF_IDX_IDX)
-
-#define DMA_COMMAND_FSM_REG_IDX					(DMA_SEL_COMP(_DMA_V2_SEL_FSM_CMD) >> 2)
-#define DMA_CHANNEL_PARAM_REG_IDX(ch, param)	((DMA_SEL_COMP(_DMA_V2_SEL_CH_REG) | DMA_SEL_CH(ch) | DMA_SEL_PARAM(param)) >> 2)
-#define DMA_CG_INFO_REG_IDX(info_id, comp_id)	((DMA_SEL_COMP(_DMA_V2_SEL_CONN_GROUP) | DMA_SEL_CG_INFO(info_id) | DMA_SEL_CG_COMP(comp_id)) >> 2)
-#define DMA_DEV_INFO_REG_IDX(info_id, dev_id)	((DMA_SEL_COMP(_DMA_V2_SEL_DEV_INTERF) | DMA_SEL_DEV_INFO(info_id) | DMA_SEL_DEV_ID(dev_id)) >> 2)
-#define DMA_RST_REG_IDX							(DMA_SEL_COMP(_DMA_V2_SEL_RESET) >> 2)
-
-#define DMA_GET_CONNECTION(val)    _hrt_get_bits(val, _DMA_V2_CONNECTION_IDX,    _DMA_V2_CONNECTION_BITS)
-#define DMA_GET_EXTENSION(val)     _hrt_get_bits(val, _DMA_V2_EXTENSION_IDX,     _DMA_V2_EXTENSION_BITS)
-#define DMA_GET_ELEMENTS(val)      _hrt_get_bits(val, _DMA_V2_ELEMENTS_IDX,      _DMA_V2_ELEMENTS_BITS)
-#define DMA_GET_CROPPING(val)      _hrt_get_bits(val, _DMA_V2_LEFT_CROPPING_IDX, _DMA_V2_LEFT_CROPPING_BITS)
-
-typedef enum {
-	DMA_CTRL_STATE_IDLE,
-	DMA_CTRL_STATE_REQ_RCV,
-	DMA_CTRL_STATE_RCV,
-	DMA_CTRL_STATE_RCV_REQ,
-	DMA_CTRL_STATE_INIT,
-	N_DMA_CTRL_STATES
-} dma_ctrl_states_t;
-
-typedef enum {
-	DMA_COMMAND_READ,
-	DMA_COMMAND_WRITE,
-	DMA_COMMAND_SET_CHANNEL,
-	DMA_COMMAND_SET_PARAM,
-	DMA_COMMAND_READ_SPECIFIC,
-	DMA_COMMAND_WRITE_SPECIFIC,
-	DMA_COMMAND_INIT,
-	DMA_COMMAND_INIT_SPECIFIC,
-	DMA_COMMAND_RST,
-	N_DMA_COMMANDS
-} dma_commands_t;
-
-typedef enum {
-	DMA_RW_STATE_IDLE,
-	DMA_RW_STATE_REQ,
-	DMA_RW_STATE_NEXT_LINE,
-	DMA_RW_STATE_UNLOCK_CHANNEL,
-	N_DMA_RW_STATES
-} dma_rw_states_t;
-
-typedef enum {
-	DMA_FIFO_STATE_WILL_BE_FULL,
-	DMA_FIFO_STATE_FULL,
-	DMA_FIFO_STATE_EMPTY,
-	N_DMA_FIFO_STATES
-} dma_fifo_states_t;
-
-/* typedef struct dma_state_s			dma_state_t; */
-typedef struct dma_channel_state_s	dma_channel_state_t;
-typedef struct dma_port_state_s		dma_port_state_t;
-
-struct dma_port_state_s {
-	bool                       req_cs;
-	bool                       req_we_n;
-	bool                       req_run;
-	bool                       req_ack;
-	bool                       send_cs;
-	bool                       send_we_n;
-	bool                       send_run;
-	bool                       send_ack;
-	dma_fifo_states_t          fifo_state;
-	int                        fifo_counter;
-};
-
-struct dma_channel_state_s {
-	int                        connection;
-	bool                       sign_extend;
-	int                        height;
-	int                        stride_a;
-	int                        elems_a;
-	int                        cropping_a;
-	int                        width_a;
-	int                        stride_b;
-	int                        elems_b;
-	int                        cropping_b;
-	int                        width_b;
-};
-
-struct dma_state_s {
-	bool                       fsm_command_idle;
-	bool                       fsm_command_run;
-	bool                       fsm_command_stalling;
-	bool                       fsm_command_error;
-	dma_commands_t             last_command;
-	int                        last_command_channel;
-	int                        last_command_param;
-	dma_commands_t             current_command;
-	int                        current_addr_a;
-	int                        current_addr_b;
-	bool                       fsm_ctrl_idle;
-	bool                       fsm_ctrl_run;
-	bool                       fsm_ctrl_stalling;
-	bool                       fsm_ctrl_error;
-	dma_ctrl_states_t          fsm_ctrl_state;
-	int                        fsm_ctrl_source_dev;
-	int                        fsm_ctrl_source_addr;
-	int                        fsm_ctrl_source_stride;
-	int                        fsm_ctrl_source_width;
-	int                        fsm_ctrl_source_height;
-	int                        fsm_ctrl_pack_source_dev;
-	int                        fsm_ctrl_pack_dest_dev;
-	int                        fsm_ctrl_dest_addr;
-	int                        fsm_ctrl_dest_stride;
-	int                        fsm_ctrl_pack_source_width;
-	int                        fsm_ctrl_pack_dest_height;
-	int                        fsm_ctrl_pack_dest_width;
-	int                        fsm_ctrl_pack_source_elems;
-	int                        fsm_ctrl_pack_dest_elems;
-	int                        fsm_ctrl_pack_extension;
-	int						   pack_idle;
-	int	                       pack_run;
-	int		               	   pack_stalling;
-	int		                   pack_error;
-	int                        pack_cnt_height;
-	int                        pack_src_cnt_width;
-	int                        pack_dest_cnt_width;
-	dma_rw_states_t            read_state;
-	int                        read_cnt_height;
-	int                        read_cnt_width;
-	dma_rw_states_t            write_state;
-	int                        write_height;
-	int                        write_width;
-	dma_port_state_t           port_states[HIVE_ISP_NUM_DMA_CONNS];
-	dma_channel_state_t        channel_states[HIVE_DMA_NUM_CHANNELS];
-};
-
-#endif /* __DMA_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma_private.h
deleted file mode 100644
index ba54b1f0467b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/dma_private.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __DMA_PRIVATE_H_INCLUDED__
-#define __DMA_PRIVATE_H_INCLUDED__
-
-#include "dma_public.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-STORAGE_CLASS_DMA_C void dma_reg_store(const dma_ID_t ID,
-			const unsigned int reg,
-			const hrt_data value)
-{
-	assert(ID < N_DMA_ID);
-	assert(DMA_BASE[ID] != (hrt_address)-1);
-	ia_css_device_store_uint32(DMA_BASE[ID] + reg*sizeof(hrt_data), value);
-}
-
-STORAGE_CLASS_DMA_C hrt_data dma_reg_load(const dma_ID_t ID,
-					  const unsigned int reg)
-{
-	assert(ID < N_DMA_ID);
-	assert(DMA_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(DMA_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-#endif /* __DMA_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo.c
deleted file mode 100644
index 777670948d6f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo.c
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "event_fifo.h"
-
-#ifndef __INLINE_EVENT__
-#include "event_fifo_private.h"
-#endif /* __INLINE_EVENT__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo_local.h
deleted file mode 100644
index c595692c6ea9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo_local.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _EVENT_FIFO_LOCAL_H
-#define _EVENT_FIFO_LOCAL_H
-
-/*
- * All events come from connections mapped on the system
- * bus but do not use a global IRQ
- */
-#include "event_fifo_global.h"
-
-typedef enum {
-	SP0_EVENT_ID,
-	ISP0_EVENT_ID,
-	STR2MIPI_EVENT_ID,
-	N_EVENT_ID
-} event_ID_t;
-
-#define	EVENT_QUERY_BIT		0
-
-/* Events are read from FIFO */
-static const hrt_address event_source_addr[N_EVENT_ID] = {
-	0x0000000000380000ULL,
-	0x0000000000380004ULL,
-	0xffffffffffffffffULL};
-
-/* Read from FIFO are blocking, query data availability */
-static const hrt_address event_source_query_addr[N_EVENT_ID] = {
-	0x0000000000380010ULL,
-	0x0000000000380014ULL,
-	0xffffffffffffffffULL};
-
-/* Events are written to FIFO */
-static const hrt_address event_sink_addr[N_EVENT_ID] = {
-	0x0000000000380008ULL,
-	0x000000000038000CULL,
-	0x0000000000090104ULL};
-
-/* Writes to FIFO are blocking, query data space */
-static const hrt_address event_sink_query_addr[N_EVENT_ID] = {
-	0x0000000000380018ULL,
-	0x000000000038001CULL,
-	0x000000000009010CULL};
-
-#endif /* _EVENT_FIFO_LOCAL_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo_private.h
deleted file mode 100644
index bcfb734c2ed3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/event_fifo_private.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __EVENT_FIFO_PRIVATE_H
-#define __EVENT_FIFO_PRIVATE_H
-
-#include "event_fifo_public.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-#include <hrt/bits.h>			/* _hrt_get_bits() */
-
-STORAGE_CLASS_EVENT_C void event_wait_for(const event_ID_t ID)
-{
-	assert(ID < N_EVENT_ID);
-	assert(event_source_addr[ID] != ((hrt_address)-1));
-	(void)ia_css_device_load_uint32(event_source_addr[ID]);
-	return;
-}
-
-STORAGE_CLASS_EVENT_C void cnd_event_wait_for(const event_ID_t ID,
-						const bool cnd)
-{
-	if (cnd) {
-		event_wait_for(ID);
-	}
-}
-
-STORAGE_CLASS_EVENT_C hrt_data event_receive_token(const event_ID_t ID)
-{
-	assert(ID < N_EVENT_ID);
-	assert(event_source_addr[ID] != ((hrt_address)-1));
-	return ia_css_device_load_uint32(event_source_addr[ID]);
-}
-
-STORAGE_CLASS_EVENT_C void event_send_token(const event_ID_t ID,
-					    const hrt_data token)
-{
-	assert(ID < N_EVENT_ID);
-	assert(event_sink_addr[ID] != ((hrt_address)-1));
-	ia_css_device_store_uint32(event_sink_addr[ID], token);
-}
-
-STORAGE_CLASS_EVENT_C bool is_event_pending(const event_ID_t ID)
-{
-	hrt_data	value;
-	assert(ID < N_EVENT_ID);
-	assert(event_source_query_addr[ID] != ((hrt_address)-1));
-	value = ia_css_device_load_uint32(event_source_query_addr[ID]);
-	return !_hrt_get_bit(value, EVENT_QUERY_BIT);
-}
-
-STORAGE_CLASS_EVENT_C bool can_event_send_token(const event_ID_t ID)
-{
-	hrt_data	value;
-	assert(ID < N_EVENT_ID);
-	assert(event_sink_query_addr[ID] != ((hrt_address)-1));
-	value = ia_css_device_load_uint32(event_sink_query_addr[ID]);
-	return !_hrt_get_bit(value, EVENT_QUERY_BIT);
-}
-
-#endif /* __EVENT_FIFO_PRIVATE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor.c
deleted file mode 100644
index 1bf292401adc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor.c
+++ /dev/null
@@ -1,567 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "fifo_monitor.h"
-
-#include <type_support.h>
-#include "device_access.h"
-
-#include <hrt/bits.h>
-
-#include "gp_device.h"
-
-#include "assert_support.h"
-
-#ifndef __INLINE_FIFO_MONITOR__
-#define STORAGE_CLASS_FIFO_MONITOR_DATA static const
-#else
-#define STORAGE_CLASS_FIFO_MONITOR_DATA const
-#endif /* __INLINE_FIFO_MONITOR__ */
-
-STORAGE_CLASS_FIFO_MONITOR_DATA unsigned int FIFO_SWITCH_ADDR[N_FIFO_SWITCH] = {
-	_REG_GP_SWITCH_IF_ADDR,
-	_REG_GP_SWITCH_GDC1_ADDR,
-	_REG_GP_SWITCH_GDC2_ADDR};
-
-#ifndef __INLINE_FIFO_MONITOR__
-#include "fifo_monitor_private.h"
-#endif /* __INLINE_FIFO_MONITOR__ */
-
-static inline bool fifo_monitor_status_valid (
-	const fifo_monitor_ID_t		ID,
-	const unsigned int			reg,
-	const unsigned int			port_id);
-
-static inline bool fifo_monitor_status_accept(
-	const fifo_monitor_ID_t		ID,
-	const unsigned int			reg,
-	const unsigned int			port_id);
-
-
-void fifo_channel_get_state(
-	const fifo_monitor_ID_t		ID,
-	const fifo_channel_t		channel_id,
-	fifo_channel_state_t		*state)
-{
-	assert(channel_id < N_FIFO_CHANNEL);
-	assert(state != NULL);
-
-	switch (channel_id) {
-	case FIFO_CHANNEL_ISP0_TO_SP0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_SP); /* ISP_STR_MON_PORT_ISP2SP */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_SP);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_ISP); /* ISP_STR_MON_PORT_SP2ISP */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_ISP);
-		break;
-	case FIFO_CHANNEL_SP0_TO_ISP0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_ISP); /* ISP_STR_MON_PORT_SP2ISP */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_ISP);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_SP); /* ISP_STR_MON_PORT_ISP2SP */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_SP);
-		break;
-	case FIFO_CHANNEL_ISP0_TO_IF0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_PIF_A); /* ISP_STR_MON_PORT_ISP2PIFA */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_PIF_A);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_PIF_A); /* MOD_STR_MON_PORT_CELLS2PIFA */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_PIF_A);
-		break;
-	case FIFO_CHANNEL_IF0_TO_ISP0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_PIF_A); /* MOD_STR_MON_PORT_PIFA2CELLS */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_PIF_A);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_PIF_A); /* ISP_STR_MON_PORT_PIFA2ISP */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_PIF_A);
-		break;
-	case FIFO_CHANNEL_ISP0_TO_IF1:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_PIF_B); /* ISP_STR_MON_PORT_ISP2PIFA */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_PIF_B);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_PIF_B); /* MOD_STR_MON_PORT_CELLS2PIFB */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_PIF_B);
-		break;
-	case FIFO_CHANNEL_IF1_TO_ISP0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_PIF_B); /* MOD_STR_MON_PORT_PIFB2CELLS */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_PIF_B);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_PIF_B); /* ISP_STR_MON_PORT_PIFB2ISP */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_PIF_B);
-		break;
-	case FIFO_CHANNEL_ISP0_TO_DMA0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_DMA); /* ISP_STR_MON_PORT_ISP2DMA */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_DMA);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_DMA_FR_ISP); /* MOD_STR_MON_PORT_ISP2DMA */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_DMA_FR_ISP);
-		break;
-	case FIFO_CHANNEL_DMA0_TO_ISP0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_DMA2ISP); /* MOD_STR_MON_PORT_DMA2ISP */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_DMA2ISP);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_DMA); /* ISP_STR_MON_PORT_DMA2ISP */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_DMA);
-		break;
-	case FIFO_CHANNEL_ISP0_TO_GDC0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_GDC); /* ISP_STR_MON_PORT_ISP2GDC1 */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_GDC);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_GDC); /* MOD_STR_MON_PORT_CELLS2GDC1 */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_GDC);
-		break;
-	case FIFO_CHANNEL_GDC0_TO_ISP0:
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_GDC); /* MOD_STR_MON_PORT_GDC12CELLS */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_GDC);
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_GDC); /* ISP_STR_MON_PORT_GDC12ISP */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_GDC);
-		break;
-	case FIFO_CHANNEL_ISP0_TO_GDC1:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_ISP2GDC2);
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_ISP2GDC2);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_CELLS2GDC2);
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_CELLS2GDC2);
-		break;
-	case FIFO_CHANNEL_GDC1_TO_ISP0:
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_GDC22CELLS);
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_GDC22CELLS);
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_GDC22ISP);
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_GDC22ISP);
-		break;
-	case FIFO_CHANNEL_ISP0_TO_HOST0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_GPD); /* ISP_STR_MON_PORT_ISP2GPD */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_SND_GPD);
-		{
-		hrt_data	value = ia_css_device_load_uint32(0x0000000000380014ULL);
-		state->fifo_valid  = !_hrt_get_bit(value, 0);
-		state->sink_accept = false; /* no monitor connected */
-		}
-		break;
-	case FIFO_CHANNEL_HOST0_TO_ISP0:
-		{
-		hrt_data	value = ia_css_device_load_uint32(0x000000000038001CULL);
-		state->fifo_valid  = false; /* no monitor connected */
-		state->sink_accept = !_hrt_get_bit(value, 0);
-		}
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_GPD); /* ISP_STR_MON_PORT_FA2ISP */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_ISP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_GPD);
-		break;
-	case FIFO_CHANNEL_SP0_TO_IF0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_PIF_A); /* SP_STR_MON_PORT_SP2PIFA */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_PIF_A);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_PIF_A); /* MOD_STR_MON_PORT_CELLS2PIFA */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_PIF_A);
-		break;
-	case FIFO_CHANNEL_IF0_TO_SP0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_PIF_A); /* MOD_STR_MON_PORT_PIFA2CELLS */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_PIF_A);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_PIF_A); /* SP_STR_MON_PORT_PIFA2SP */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_PIF_A);
-		break;
-	case FIFO_CHANNEL_SP0_TO_IF1:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_PIF_B); /* SP_STR_MON_PORT_SP2PIFB */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_PIF_B);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_PIF_B); /* MOD_STR_MON_PORT_CELLS2PIFB */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_PIF_B);
-		break;
-	case FIFO_CHANNEL_IF1_TO_SP0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_PIF_B); /* MOD_STR_MON_PORT_PIFB2CELLS */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_PIF_B);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_PIF_B); /* SP_STR_MON_PORT_PIFB2SP */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			ISP_STR_MON_PORT_RCV_PIF_B);
-		break;
-	case FIFO_CHANNEL_SP0_TO_IF2:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_SIF); /* SP_STR_MON_PORT_SP2SIF */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_SIF);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_SIF); /* MOD_STR_MON_PORT_SP2SIF */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_SIF);
-		break;
-	case FIFO_CHANNEL_IF2_TO_SP0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_SIF); /* MOD_STR_MON_PORT_SIF2SP */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_SIF);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_SIF); /* SP_STR_MON_PORT_SIF2SP */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_SIF);
-		break;
-	case FIFO_CHANNEL_SP0_TO_DMA0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_DMA); /* SP_STR_MON_PORT_SP2DMA */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_DMA);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_DMA_FR_SP); /* MOD_STR_MON_PORT_SP2DMA */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_DMA_FR_SP);
-		break;
-	case FIFO_CHANNEL_DMA0_TO_SP0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_DMA2SP); /* MOD_STR_MON_PORT_DMA2SP */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_DMA2SP);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_DMA); /* SP_STR_MON_PORT_DMA2SP */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_DMA);
-		break;
-	case FIFO_CHANNEL_SP0_TO_GDC0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_B_IDX,
-			SP_STR_MON_PORT_B_SP2GDC1);
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_B_IDX,
-			SP_STR_MON_PORT_B_SP2GDC1);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_CELLS2GDC1);
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_CELLS2GDC1);
-		break;
-	case FIFO_CHANNEL_GDC0_TO_SP0:
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_GDC12CELLS);
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_GDC12CELLS);
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_B_IDX,
-			SP_STR_MON_PORT_B_GDC12SP);
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_B_IDX,
-			SP_STR_MON_PORT_B_GDC12SP);
-		break;
-	case FIFO_CHANNEL_SP0_TO_GDC1:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_B_IDX,
-			SP_STR_MON_PORT_B_SP2GDC2);
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_B_IDX,
-			SP_STR_MON_PORT_B_SP2GDC2);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_CELLS2GDC2);
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_CELLS2GDC2);
-		break;
-	case FIFO_CHANNEL_GDC1_TO_SP0:
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_GDC22CELLS);
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_GDC22CELLS);
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_B_IDX,
-			SP_STR_MON_PORT_B_GDC22SP);
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_B_IDX,
-			SP_STR_MON_PORT_B_GDC22SP);
-		break;
-	case FIFO_CHANNEL_SP0_TO_HOST0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_GPD); /* SP_STR_MON_PORT_SP2GPD */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_GPD);
-		{
-		hrt_data	value = ia_css_device_load_uint32(0x0000000000380010ULL);
-		state->fifo_valid  = !_hrt_get_bit(value, 0);
-		state->sink_accept = false; /* no monitor connected */
-		}
-		break;
-	case FIFO_CHANNEL_HOST0_TO_SP0:
-		{
-		hrt_data	value = ia_css_device_load_uint32(0x0000000000380018ULL);
-		state->fifo_valid  = false; /* no monitor connected */
-		state->sink_accept = !_hrt_get_bit(value, 0);
-		}
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_GPD); /* SP_STR_MON_PORT_FA2SP */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_GPD);
-		break;
-	case FIFO_CHANNEL_SP0_TO_STREAM2MEM0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_MC); /* SP_STR_MON_PORT_SP2MC */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SND_MC);
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_MC); /* MOD_STR_MON_PORT_SP2MC */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_RCV_MC);
-		break;
-	case FIFO_CHANNEL_STREAM2MEM0_TO_SP0:
-		state->fifo_valid  = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_MC); /* SP_STR_MON_PORT_MC2SP */
-		state->sink_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_MOD_STREAM_STAT_IDX,
-			MOD_STR_MON_PORT_SND_MC);
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_MC); /* MOD_STR_MON_PORT_MC2SP */
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_RCV_MC);
-		break;
-	case FIFO_CHANNEL_SP0_TO_INPUT_SYSTEM0:
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SP2ISYS);
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_SP2ISYS);
-		state->fifo_valid  = false;
-		state->sink_accept = false;
-		break;
-	case FIFO_CHANNEL_INPUT_SYSTEM0_TO_SP0:
-		state->fifo_valid  = false;
-		state->sink_accept = false;
-		state->src_valid   = fifo_monitor_status_valid(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_ISYS2SP);
-		state->fifo_accept = fifo_monitor_status_accept(ID,
-			HIVE_GP_REGS_SP_STREAM_STAT_IDX,
-			SP_STR_MON_PORT_ISYS2SP);
-		break;
-	default:
-		assert(0);
-		break;
-	}
-
-	return;
-}
-
-void fifo_switch_get_state(
-	const fifo_monitor_ID_t		ID,
-	const fifo_switch_t			switch_id,
-	fifo_switch_state_t			*state)
-{
-	hrt_data		data = (hrt_data)-1;
-
-	assert(ID == FIFO_MONITOR0_ID);
-	assert(switch_id < N_FIFO_SWITCH);
-	assert(state != NULL);
-
-	(void)ID;
-
-	data = gp_device_reg_load(GP_DEVICE0_ID, FIFO_SWITCH_ADDR[switch_id]);
-
-	state->is_none = (data == HIVE_ISP_CSS_STREAM_SWITCH_NONE);
-	state->is_sp = (data == HIVE_ISP_CSS_STREAM_SWITCH_SP);
-	state->is_isp = (data == HIVE_ISP_CSS_STREAM_SWITCH_ISP);
-
-	return;
-}
-
-void fifo_monitor_get_state(
-	const fifo_monitor_ID_t		ID,
-	fifo_monitor_state_t		*state)
-{
-	fifo_channel_t	ch_id;
-	fifo_switch_t	sw_id;
-
-	assert(ID < N_FIFO_MONITOR_ID);
-	assert(state != NULL);
-
-	for (ch_id = 0; ch_id < N_FIFO_CHANNEL; ch_id++) {
-		fifo_channel_get_state(ID, ch_id,
-			&(state->fifo_channels[ch_id]));
-	}
-
-	for (sw_id = 0; sw_id < N_FIFO_SWITCH; sw_id++) {
-		fifo_switch_get_state(ID, sw_id,
-			&(state->fifo_switches[sw_id]));
-	}
-	return;
-}
-
-static inline bool fifo_monitor_status_valid (
-	const fifo_monitor_ID_t		ID,
-	const unsigned int			reg,
-	const unsigned int			port_id)
-{
-	hrt_data	data = fifo_monitor_reg_load(ID, reg);
-
-	return (data >> (((port_id * 2) + _hive_str_mon_valid_offset))) & 0x1;
-}
-
-static inline bool fifo_monitor_status_accept(
-	const fifo_monitor_ID_t		ID,
-	const unsigned int			reg,
-	const unsigned int			port_id)
-{
-	hrt_data	data = fifo_monitor_reg_load(ID, reg);
-
-	return (data >> (((port_id * 2) + _hive_str_mon_accept_offset))) & 0x1;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor_local.h
deleted file mode 100644
index ed2f86181788..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor_local.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __FIFO_MONITOR_LOCAL_H_INCLUDED__
-#define __FIFO_MONITOR_LOCAL_H_INCLUDED__
-
-#include <type_support.h>
-#include "fifo_monitor_global.h"
-
-#include "hive_isp_css_defs.h"	/* ISP_STR_MON_PORT_SND_SP, ... */
-
-#define _hive_str_mon_valid_offset   0
-#define _hive_str_mon_accept_offset  1
-
-#define	FIFO_CHANNEL_SP_VALID_MASK		0x55555555
-#define	FIFO_CHANNEL_SP_VALID_B_MASK	0x00000055
-#define	FIFO_CHANNEL_ISP_VALID_MASK		0x15555555
-#define	FIFO_CHANNEL_MOD_VALID_MASK		0x55555555
-
-typedef enum fifo_switch {
-	FIFO_SWITCH_IF,
-	FIFO_SWITCH_GDC0,
-	FIFO_SWITCH_GDC1,
-	N_FIFO_SWITCH
-} fifo_switch_t;
-
-typedef enum fifo_channel {
-	FIFO_CHANNEL_ISP0_TO_SP0,
-	FIFO_CHANNEL_SP0_TO_ISP0,
-	FIFO_CHANNEL_ISP0_TO_IF0,
-	FIFO_CHANNEL_IF0_TO_ISP0,
-	FIFO_CHANNEL_ISP0_TO_IF1,
-	FIFO_CHANNEL_IF1_TO_ISP0,
-	FIFO_CHANNEL_ISP0_TO_DMA0,
-	FIFO_CHANNEL_DMA0_TO_ISP0,
-	FIFO_CHANNEL_ISP0_TO_GDC0,
-	FIFO_CHANNEL_GDC0_TO_ISP0,
-	FIFO_CHANNEL_ISP0_TO_GDC1,
-	FIFO_CHANNEL_GDC1_TO_ISP0,
-	FIFO_CHANNEL_ISP0_TO_HOST0,
-	FIFO_CHANNEL_HOST0_TO_ISP0,
-	FIFO_CHANNEL_SP0_TO_IF0,
-	FIFO_CHANNEL_IF0_TO_SP0,
-	FIFO_CHANNEL_SP0_TO_IF1,
-	FIFO_CHANNEL_IF1_TO_SP0,
-	FIFO_CHANNEL_SP0_TO_IF2,
-	FIFO_CHANNEL_IF2_TO_SP0,
-	FIFO_CHANNEL_SP0_TO_DMA0,
-	FIFO_CHANNEL_DMA0_TO_SP0,
-	FIFO_CHANNEL_SP0_TO_GDC0,
-	FIFO_CHANNEL_GDC0_TO_SP0,
-	FIFO_CHANNEL_SP0_TO_GDC1,
-	FIFO_CHANNEL_GDC1_TO_SP0,
-	FIFO_CHANNEL_SP0_TO_HOST0,
-	FIFO_CHANNEL_HOST0_TO_SP0,
-	FIFO_CHANNEL_SP0_TO_STREAM2MEM0,
-	FIFO_CHANNEL_STREAM2MEM0_TO_SP0,
-	FIFO_CHANNEL_SP0_TO_INPUT_SYSTEM0,
-	FIFO_CHANNEL_INPUT_SYSTEM0_TO_SP0,
-/*
- * No clue what this is
- *
-	FIFO_CHANNEL_SP0_TO_IRQ0,
-	FIFO_CHANNEL_IRQ0_TO_SP0,
- */
-	N_FIFO_CHANNEL
-} fifo_channel_t;
-
-struct fifo_channel_state_s {
-	bool	src_valid;
-	bool	fifo_accept;
-	bool	fifo_valid;
-	bool	sink_accept;
-};
-
-/* The switch is tri-state */
-struct fifo_switch_state_s {
-	bool	is_none;
-	bool	is_isp;
-	bool	is_sp;
-};
-
-struct fifo_monitor_state_s {
-	struct fifo_channel_state_s	fifo_channels[N_FIFO_CHANNEL];
-	struct fifo_switch_state_s	fifo_switches[N_FIFO_SWITCH];
-};
-
-#endif /* __FIFO_MONITOR_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor_private.h
deleted file mode 100644
index d58cd7d1828d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/fifo_monitor_private.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __FIFO_MONITOR_PRIVATE_H_INCLUDED__
-#define __FIFO_MONITOR_PRIVATE_H_INCLUDED__
-
-#include "fifo_monitor_public.h"
-
-#define __INLINE_GP_DEVICE__
-#include "gp_device.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-#ifdef __INLINE_FIFO_MONITOR__
-extern const unsigned int FIFO_SWITCH_ADDR[N_FIFO_SWITCH];
-#endif
-
-STORAGE_CLASS_FIFO_MONITOR_C void fifo_switch_set(
-	const fifo_monitor_ID_t		ID,
-	const fifo_switch_t			switch_id,
-	const hrt_data				sel)
-{
-	assert(ID == FIFO_MONITOR0_ID);
-	assert(FIFO_MONITOR_BASE[ID] != (hrt_address)-1);
-	assert(switch_id < N_FIFO_SWITCH);
-	(void)ID;
-
-	gp_device_reg_store(GP_DEVICE0_ID, FIFO_SWITCH_ADDR[switch_id], sel);
-
-	return;
-}
-
-STORAGE_CLASS_FIFO_MONITOR_C hrt_data fifo_switch_get(
-	const fifo_monitor_ID_t		ID,
-	const fifo_switch_t			switch_id)
-{
-	assert(ID == FIFO_MONITOR0_ID);
-	assert(FIFO_MONITOR_BASE[ID] != (hrt_address)-1);
-	assert(switch_id < N_FIFO_SWITCH);
-	(void)ID;
-
-	return gp_device_reg_load(GP_DEVICE0_ID, FIFO_SWITCH_ADDR[switch_id]);
-}
-
-
-STORAGE_CLASS_FIFO_MONITOR_C void fifo_monitor_reg_store(
-	const fifo_monitor_ID_t		ID,
-	const unsigned int			reg,
-	const hrt_data				value)
-{
-	assert(ID < N_FIFO_MONITOR_ID);
-	assert(FIFO_MONITOR_BASE[ID] != (hrt_address)-1);
-	ia_css_device_store_uint32(FIFO_MONITOR_BASE[ID] + reg*sizeof(hrt_data), value);
-	return;
-}
-
-STORAGE_CLASS_FIFO_MONITOR_C hrt_data fifo_monitor_reg_load(
-	const fifo_monitor_ID_t		ID,
-	const unsigned int			reg)
-{
-	assert(ID < N_FIFO_MONITOR_ID);
-	assert(FIFO_MONITOR_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(FIFO_MONITOR_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-#endif /* __FIFO_MONITOR_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc.c
deleted file mode 100644
index 1966b147f8ab..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc.c
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* The name "gdc.h is already taken" */
-#include "gdc_device.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-/*
- * Local function declarations
- */
-static inline void gdc_reg_store(
-	const gdc_ID_t		ID,
-	const unsigned int	reg,
-	const hrt_data		value);
-
-static inline hrt_data gdc_reg_load(
-	const gdc_ID_t		ID,
-	const unsigned int	reg);
-
-
-#ifndef __INLINE_GDC__
-#include "gdc_private.h"
-#endif /* __INLINE_GDC__ */
-
-/*
- * Exported function implementations
- */
-void gdc_lut_store(
-	const gdc_ID_t		ID,
-	const int			data[4][HRT_GDC_N])
-{
-	unsigned int i, lut_offset = HRT_GDC_LUT_IDX;
-
-	assert(ID < N_GDC_ID);
-	assert(HRT_GDC_LUT_COEFF_OFFSET <= (4*sizeof(hrt_data)));
-
-	for (i = 0; i < HRT_GDC_N; i++) {
-		hrt_data	entry_0 = data[0][i] & HRT_GDC_BCI_COEF_MASK;
-		hrt_data	entry_1 = data[1][i] & HRT_GDC_BCI_COEF_MASK;
-		hrt_data	entry_2 = data[2][i] & HRT_GDC_BCI_COEF_MASK;
-		hrt_data	entry_3 = data[3][i] & HRT_GDC_BCI_COEF_MASK;
-
-		hrt_data	word_0 = entry_0 |
-			(entry_1 << HRT_GDC_LUT_COEFF_OFFSET);
-		hrt_data	word_1 = entry_2 |
-			(entry_3 << HRT_GDC_LUT_COEFF_OFFSET);
-
-		gdc_reg_store(ID, lut_offset++, word_0);
-		gdc_reg_store(ID, lut_offset++, word_1);
-	}
-	return;
-}
-
-/*
- * Input LUT format:
- * c0[0-1023], c1[0-1023], c2[0-1023] c3[0-1023]
- *
- * Output LUT format (interleaved):
- * c0[0], c1[0], c2[0], c3[0], c0[1], c1[1], c2[1], c3[1], ....
- * c0[1023], c1[1023], c2[1023], c3[1023]
- *
- * The first format needs c0[0], c1[0] (which are 1024 words apart)
- * to program gdc LUT registers. This makes it difficult to do piecemeal
- * reads in SP side gdc_lut_store
- *
- * Interleaved format allows use of contiguous bytes to store into
- * gdc LUT registers.
- *
- * See gdc_lut_store() definition in host/gdc.c vs sp/gdc_private.h
- *
- */
-void gdc_lut_convert_to_isp_format(const int in_lut[4][HRT_GDC_N],
-	int out_lut[4][HRT_GDC_N])
-{
-	unsigned int i;
-	int *out = (int *)out_lut;
-
-	for (i = 0; i < HRT_GDC_N; i++) {
-		out[0] = in_lut[0][i];
-		out[1] = in_lut[1][i];
-		out[2] = in_lut[2][i];
-		out[3] = in_lut[3][i];
-		out += 4;
-	}
-}
-
-int gdc_get_unity(
-	const gdc_ID_t		ID)
-{
-	assert(ID < N_GDC_ID);
-	(void)ID;
-	return (int)(1UL << HRT_GDC_FRAC_BITS);
-}
-
-
-/*
- * Local function implementations
- */
-static inline void gdc_reg_store(
-	const gdc_ID_t		ID,
-	const unsigned int	reg,
-	const hrt_data		value)
-{
-	ia_css_device_store_uint32(GDC_BASE[ID] + reg*sizeof(hrt_data), value);
-	return;
-}
-
-static inline hrt_data gdc_reg_load(
-	const gdc_ID_t		ID,
-	const unsigned int	reg)
-{
-	return ia_css_device_load_uint32(GDC_BASE[ID] + reg*sizeof(hrt_data));
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc_local.h
deleted file mode 100644
index 0c6de867e012..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc_local.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GDC_LOCAL_H_INCLUDED__
-#define __GDC_LOCAL_H_INCLUDED__
-
-#include "gdc_global.h"
-
-#endif /* __GDC_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc_private.h
deleted file mode 100644
index f7dec75adf78..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gdc_private.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GDC_PRIVATE_H_INCLUDED__
-#define __GDC_PRIVATE_H_INCLUDED__
-
-#include "gdc_public.h"
-
-#endif /* __GDC_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device.c
deleted file mode 100644
index da88aa3af664..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device.c
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "assert_support.h"
-#include "gp_device.h"
-
-#ifndef __INLINE_GP_DEVICE__
-#include "gp_device_private.h"
-#endif /* __INLINE_GP_DEVICE__ */
-
-void gp_device_get_state(
-	const gp_device_ID_t		ID,
-	gp_device_state_t			*state)
-{
-	assert(ID < N_GP_DEVICE_ID);
-	assert(state != NULL);
-
-	state->syncgen_enable = gp_device_reg_load(ID,
-		_REG_GP_SYNCGEN_ENABLE_ADDR);
-	state->syncgen_free_running = gp_device_reg_load(ID,
-		_REG_GP_SYNCGEN_FREE_RUNNING_ADDR);
-	state->syncgen_pause = gp_device_reg_load(ID,
-		_REG_GP_SYNCGEN_PAUSE_ADDR);
-	state->nr_frames = gp_device_reg_load(ID,
-		_REG_GP_NR_FRAMES_ADDR);
-	state->syngen_nr_pix = gp_device_reg_load(ID,
-		_REG_GP_SYNGEN_NR_PIX_ADDR);
-	state->syngen_nr_pix = gp_device_reg_load(ID,
-		_REG_GP_SYNGEN_NR_PIX_ADDR);
-	state->syngen_nr_lines = gp_device_reg_load(ID,
-		_REG_GP_SYNGEN_NR_LINES_ADDR);
-	state->syngen_hblank_cycles = gp_device_reg_load(ID,
-		_REG_GP_SYNGEN_HBLANK_CYCLES_ADDR);
-	state->syngen_vblank_cycles = gp_device_reg_load(ID,
-		_REG_GP_SYNGEN_VBLANK_CYCLES_ADDR);
-	state->isel_sof = gp_device_reg_load(ID,
-		_REG_GP_ISEL_SOF_ADDR);
-	state->isel_eof = gp_device_reg_load(ID,
-		_REG_GP_ISEL_EOF_ADDR);
-	state->isel_sol = gp_device_reg_load(ID,
-		_REG_GP_ISEL_SOL_ADDR);
-	state->isel_eol = gp_device_reg_load(ID,
-		_REG_GP_ISEL_EOL_ADDR);
-	state->isel_lfsr_enable = gp_device_reg_load(ID,
-		_REG_GP_ISEL_LFSR_ENABLE_ADDR);
-	state->isel_lfsr_enable_b = gp_device_reg_load(ID,
-		_REG_GP_ISEL_LFSR_ENABLE_B_ADDR);
-	state->isel_lfsr_reset_value = gp_device_reg_load(ID,
-		_REG_GP_ISEL_LFSR_RESET_VALUE_ADDR);
-	state->isel_tpg_enable = gp_device_reg_load(ID,
-		_REG_GP_ISEL_TPG_ENABLE_ADDR);
-	state->isel_tpg_enable_b = gp_device_reg_load(ID,
-		_REG_GP_ISEL_TPG_ENABLE_B_ADDR);
-	state->isel_hor_cnt_mask = gp_device_reg_load(ID,
-		_REG_GP_ISEL_HOR_CNT_MASK_ADDR);
-	state->isel_ver_cnt_mask = gp_device_reg_load(ID,
-		_REG_GP_ISEL_VER_CNT_MASK_ADDR);
-	state->isel_xy_cnt_mask = gp_device_reg_load(ID,
-		_REG_GP_ISEL_XY_CNT_MASK_ADDR);
-	state->isel_hor_cnt_delta = gp_device_reg_load(ID,
-		_REG_GP_ISEL_HOR_CNT_DELTA_ADDR);
-	state->isel_ver_cnt_delta = gp_device_reg_load(ID,
-		_REG_GP_ISEL_VER_CNT_DELTA_ADDR);
-	state->isel_tpg_mode = gp_device_reg_load(ID,
-		_REG_GP_ISEL_TPG_MODE_ADDR);
-	state->isel_tpg_red1 = gp_device_reg_load(ID,
-		_REG_GP_ISEL_TPG_RED1_ADDR);
-	state->isel_tpg_green1 = gp_device_reg_load(ID,
-		_REG_GP_ISEL_TPG_GREEN1_ADDR);
-	state->isel_tpg_blue1 = gp_device_reg_load(ID,
-		_REG_GP_ISEL_TPG_BLUE1_ADDR);
-	state->isel_tpg_red2 = gp_device_reg_load(ID,
-		_REG_GP_ISEL_TPG_RED2_ADDR);
-	state->isel_tpg_green2 = gp_device_reg_load(ID,
-		_REG_GP_ISEL_TPG_GREEN2_ADDR);
-	state->isel_tpg_blue2 = gp_device_reg_load(ID,
-		_REG_GP_ISEL_TPG_BLUE2_ADDR);
-	state->isel_ch_id = gp_device_reg_load(ID,
-		_REG_GP_ISEL_CH_ID_ADDR);
-	state->isel_fmt_type = gp_device_reg_load(ID,
-		_REG_GP_ISEL_FMT_TYPE_ADDR);
-	state->isel_data_sel = gp_device_reg_load(ID,
-		_REG_GP_ISEL_DATA_SEL_ADDR);
-	state->isel_sband_sel = gp_device_reg_load(ID,
-		_REG_GP_ISEL_SBAND_SEL_ADDR);
-	state->isel_sync_sel = gp_device_reg_load(ID,
-		_REG_GP_ISEL_SYNC_SEL_ADDR);
-	state->syncgen_hor_cnt = gp_device_reg_load(ID,
-		_REG_GP_SYNCGEN_HOR_CNT_ADDR);
-	state->syncgen_ver_cnt = gp_device_reg_load(ID,
-		_REG_GP_SYNCGEN_VER_CNT_ADDR);
-	state->syncgen_frame_cnt = gp_device_reg_load(ID,
-		_REG_GP_SYNCGEN_FRAME_CNT_ADDR);
-	state->soft_reset = gp_device_reg_load(ID,
-		_REG_GP_SOFT_RESET_ADDR);
-	return;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device_local.h
deleted file mode 100644
index 113d5ed32d42..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device_local.h
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GP_DEVICE_LOCAL_H_INCLUDED__
-#define __GP_DEVICE_LOCAL_H_INCLUDED__
-
-#include "gp_device_global.h"
-
-/* @ GP_REGS_BASE -> GP_DEVICE_BASE */
-#define _REG_GP_SDRAM_WAKEUP_ADDR					0x00
-#define _REG_GP_IDLE_ADDR							0x04
-/* #define _REG_GP_IRQ_REQ0_ADDR					0x08 */
-/* #define _REG_GP_IRQ_REQ1_ADDR					0x0C */
-#define _REG_GP_SP_STREAM_STAT_ADDR					0x10
-#define _REG_GP_SP_STREAM_STAT_B_ADDR				0x14
-#define _REG_GP_ISP_STREAM_STAT_ADDR				0x18
-#define _REG_GP_MOD_STREAM_STAT_ADDR				0x1C
-#define _REG_GP_SP_STREAM_STAT_IRQ_COND_ADDR		0x20
-#define _REG_GP_SP_STREAM_STAT_B_IRQ_COND_ADDR		0x24
-#define _REG_GP_ISP_STREAM_STAT_IRQ_COND_ADDR		0x28
-#define _REG_GP_MOD_STREAM_STAT_IRQ_COND_ADDR		0x2C
-#define _REG_GP_SP_STREAM_STAT_IRQ_ENABLE_ADDR		0x30
-#define _REG_GP_SP_STREAM_STAT_B_IRQ_ENABLE_ADDR	0x34
-#define _REG_GP_ISP_STREAM_STAT_IRQ_ENABLE_ADDR		0x38
-#define _REG_GP_MOD_STREAM_STAT_IRQ_ENABLE_ADDR		0x3C
-/*
-#define _REG_GP_SWITCH_IF_ADDR						0x40
-#define _REG_GP_SWITCH_GDC1_ADDR					0x44
-#define _REG_GP_SWITCH_GDC2_ADDR					0x48
-*/
-#define _REG_GP_SLV_REG_RST_ADDR					0x50
-#define _REG_GP_SWITCH_ISYS2401_ADDR				0x54
-
-/* @ INPUT_FORMATTER_BASE -> GP_DEVICE_BASE */
-/*
-#define _REG_GP_IFMT_input_switch_lut_reg0			0x00030800
-#define _REG_GP_IFMT_input_switch_lut_reg1			0x00030804
-#define _REG_GP_IFMT_input_switch_lut_reg2			0x00030808
-#define _REG_GP_IFMT_input_switch_lut_reg3			0x0003080C
-#define _REG_GP_IFMT_input_switch_lut_reg4			0x00030810
-#define _REG_GP_IFMT_input_switch_lut_reg5			0x00030814
-#define _REG_GP_IFMT_input_switch_lut_reg6			0x00030818
-#define _REG_GP_IFMT_input_switch_lut_reg7			0x0003081C
-#define _REG_GP_IFMT_input_switch_fsync_lut			0x00030820
-#define _REG_GP_IFMT_srst							0x00030824
-#define _REG_GP_IFMT_slv_reg_srst					0x00030828
-#define _REG_GP_IFMT_input_switch_ch_id_fmt_type	0x0003082C
-*/
-/* @ GP_DEVICE_BASE */
-/*
-#define _REG_GP_SYNCGEN_ENABLE_ADDR					0x00090000
-#define _REG_GP_SYNCGEN_FREE_RUNNING_ADDR			0x00090004
-#define _REG_GP_SYNCGEN_PAUSE_ADDR					0x00090008
-#define _REG_GP_NR_FRAMES_ADDR						0x0009000C
-#define _REG_GP_SYNGEN_NR_PIX_ADDR					0x00090010
-#define _REG_GP_SYNGEN_NR_LINES_ADDR				0x00090014
-#define _REG_GP_SYNGEN_HBLANK_CYCLES_ADDR			0x00090018
-#define _REG_GP_SYNGEN_VBLANK_CYCLES_ADDR			0x0009001C
-#define _REG_GP_ISEL_SOF_ADDR						0x00090020
-#define _REG_GP_ISEL_EOF_ADDR						0x00090024
-#define _REG_GP_ISEL_SOL_ADDR						0x00090028
-#define _REG_GP_ISEL_EOL_ADDR						0x0009002C
-#define _REG_GP_ISEL_LFSR_ENABLE_ADDR				0x00090030
-#define _REG_GP_ISEL_LFSR_ENABLE_B_ADDR				0x00090034
-#define _REG_GP_ISEL_LFSR_RESET_VALUE_ADDR			0x00090038
-#define _REG_GP_ISEL_TPG_ENABLE_ADDR				0x0009003C
-#define _REG_GP_ISEL_TPG_ENABLE_B_ADDR				0x00090040
-#define _REG_GP_ISEL_HOR_CNT_MASK_ADDR				0x00090044
-#define _REG_GP_ISEL_VER_CNT_MASK_ADDR				0x00090048
-#define _REG_GP_ISEL_XY_CNT_MASK_ADDR				0x0009004C
-#define _REG_GP_ISEL_HOR_CNT_DELTA_ADDR				0x00090050
-#define _REG_GP_ISEL_VER_CNT_DELTA_ADDR				0x00090054
-#define _REG_GP_ISEL_TPG_MODE_ADDR					0x00090058
-#define _REG_GP_ISEL_TPG_RED1_ADDR					0x0009005C
-#define _REG_GP_ISEL_TPG_GREEN1_ADDR				0x00090060
-#define _REG_GP_ISEL_TPG_BLUE1_ADDR					0x00090064
-#define _REG_GP_ISEL_TPG_RED2_ADDR					0x00090068
-#define _REG_GP_ISEL_TPG_GREEN2_ADDR				0x0009006C
-#define _REG_GP_ISEL_TPG_BLUE2_ADDR					0x00090070
-#define _REG_GP_ISEL_CH_ID_ADDR						0x00090074
-#define _REG_GP_ISEL_FMT_TYPE_ADDR					0x00090078
-#define _REG_GP_ISEL_DATA_SEL_ADDR					0x0009007C
-#define _REG_GP_ISEL_SBAND_SEL_ADDR					0x00090080
-#define _REG_GP_ISEL_SYNC_SEL_ADDR					0x00090084
-#define _REG_GP_SYNCGEN_HOR_CNT_ADDR				0x00090088
-#define _REG_GP_SYNCGEN_VER_CNT_ADDR				0x0009008C
-#define _REG_GP_SYNCGEN_FRAME_CNT_ADDR				0x00090090
-#define _REG_GP_SOFT_RESET_ADDR						0x00090094
-*/
-
-struct gp_device_state_s {
-	int syncgen_enable;
-	int syncgen_free_running;
-	int syncgen_pause;
-	int nr_frames;
-	int syngen_nr_pix;
-	int syngen_nr_lines;
-	int syngen_hblank_cycles;
-	int syngen_vblank_cycles;
-	int isel_sof;
-	int isel_eof;
-	int isel_sol;
-	int isel_eol;
-	int isel_lfsr_enable;
-	int isel_lfsr_enable_b;
-	int isel_lfsr_reset_value;
-	int isel_tpg_enable;
-	int isel_tpg_enable_b;
-	int isel_hor_cnt_mask;
-	int isel_ver_cnt_mask;
-	int isel_xy_cnt_mask;
-	int isel_hor_cnt_delta;
-	int isel_ver_cnt_delta;
-	int isel_tpg_mode;
-	int isel_tpg_red1;
-	int isel_tpg_green1;
-	int isel_tpg_blue1;
-	int isel_tpg_red2;
-	int isel_tpg_green2;
-	int isel_tpg_blue2;
-	int isel_ch_id;
-	int isel_fmt_type;
-	int isel_data_sel;
-	int isel_sband_sel;
-	int isel_sync_sel;
-	int syncgen_hor_cnt;
-	int syncgen_ver_cnt;
-	int syncgen_frame_cnt;
-	int soft_reset;
-};
-
-#endif /* __GP_DEVICE_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device_private.h
deleted file mode 100644
index 7c0362c29411..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_device_private.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GP_DEVICE_PRIVATE_H_INCLUDED__
-#define __GP_DEVICE_PRIVATE_H_INCLUDED__
-
-#include "gp_device_public.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-STORAGE_CLASS_GP_DEVICE_C void gp_device_reg_store(
-	const gp_device_ID_t	ID,
-	const unsigned int		reg_addr,
-	const hrt_data			value)
-{
-	assert(ID < N_GP_DEVICE_ID);
-	assert(GP_DEVICE_BASE[ID] != (hrt_address)-1);
-	assert((reg_addr % sizeof(hrt_data)) == 0);
-	ia_css_device_store_uint32(GP_DEVICE_BASE[ID] + reg_addr, value);
-	return;
-}
-
-STORAGE_CLASS_GP_DEVICE_C hrt_data gp_device_reg_load(
-	const gp_device_ID_t	ID,
-	const hrt_address	reg_addr)
-{
-	assert(ID < N_GP_DEVICE_ID);
-	assert(GP_DEVICE_BASE[ID] != (hrt_address)-1);
-	assert((reg_addr % sizeof(hrt_data)) == 0);
-	return ia_css_device_load_uint32(GP_DEVICE_BASE[ID] + reg_addr);
-}
-
-#endif /* __GP_DEVICE_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer.c
deleted file mode 100644
index b6b1344786b1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer.c
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <type_support.h> /*uint32_t */
-#include "gp_timer.h"   /*system_local.h,
-			  gp_timer_public.h*/
-
-#ifndef __INLINE_GP_TIMER__
-#include "gp_timer_private.h"  /*device_access.h*/
-#endif /* __INLINE_GP_TIMER__ */
-#include "system_local.h"
-
-/* FIXME: not sure if reg_load(), reg_store() should be API.
- */
-static uint32_t
-gp_timer_reg_load(uint32_t reg);
-
-static void
-gp_timer_reg_store(uint32_t reg, uint32_t value);
-
-static uint32_t
-gp_timer_reg_load(uint32_t reg)
-{
-	return ia_css_device_load_uint32(
-			GP_TIMER_BASE +
-			(reg * sizeof(uint32_t)));
-}
-
-static void
-gp_timer_reg_store(uint32_t reg, uint32_t value)
-{
-	ia_css_device_store_uint32((GP_TIMER_BASE +
-				    (reg * sizeof(uint32_t))),
-				    value);
-}
-
-void gp_timer_init(gp_timer_ID_t ID)
-{
-	/* set_overall_enable*/
-	gp_timer_reg_store(_REG_GP_TIMER_OVERALL_ENABLE, 1);
-
-	/*set enable*/
-	gp_timer_reg_store(_REG_GP_TIMER_ENABLE_ID(ID), 1);
-
-	/* set signal select */
-	gp_timer_reg_store(_REG_GP_TIMER_SIGNAL_SELECT_ID(ID), GP_TIMER_SIGNAL_SELECT);
-
-	/*set count type */
-	gp_timer_reg_store(_REG_GP_TIMER_COUNT_TYPE_ID(ID), GP_TIMER_COUNT_TYPE_LOW);
-
-	/*reset gp timer */
-	gp_timer_reg_store(_REG_GP_TIMER_RESET_REG, 0xFF);
-}
-
-uint32_t
-gp_timer_read(gp_timer_ID_t ID)
-{
-	return	gp_timer_reg_load(_REG_GP_TIMER_VALUE_ID(ID));
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer_local.h
deleted file mode 100644
index 19ce35d87291..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer_local.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GP_TIMER_LOCAL_H_INCLUDED__
-#define  __GP_TIMER_LOCAL_H_INCLUDED__
-
-#include "gp_timer_global.h" /*GP_TIMER_SEL
-				GP_TIMER_SIGNAL_SELECT*/
-
-#include "gp_timer_defs.h"    /*HIVE_GP_TIMER_xxx registers*/
-#include "hive_isp_css_defs.h" /*HIVE_GP_TIMER_NUM_COUNTERS
-				 HIVE_GP_TIMER_NUM_IRQS*/
-
-#define _REG_GP_TIMER_RESET_REG HIVE_GP_TIMER_RESET_REG_IDX
-#define _REG_GP_TIMER_OVERALL_ENABLE HIVE_GP_TIMER_OVERALL_ENABLE_REG_IDX
-
-/*Register offsets for timers [1,7] can be obtained
- * by adding (GP_TIMERx_ID * sizeof(uint32_t))*/
-#define _REG_GP_TIMER_ENABLE_ID(timer_id)        HIVE_GP_TIMER_ENABLE_REG_IDX(timer_id)
-#define _REG_GP_TIMER_VALUE_ID(timer_id)  	 HIVE_GP_TIMER_VALUE_REG_IDX(timer_id, HIVE_GP_TIMER_NUM_COUNTERS)
-#define _REG_GP_TIMER_COUNT_TYPE_ID(timer_id)    HIVE_GP_TIMER_COUNT_TYPE_REG_IDX(timer_id, HIVE_GP_TIMER_NUM_COUNTERS)
-#define _REG_GP_TIMER_SIGNAL_SELECT_ID(timer_id) HIVE_GP_TIMER_SIGNAL_SELECT_REG_IDX(timer_id, HIVE_GP_TIMER_NUM_COUNTERS)
-
-
-#define _REG_GP_TIMER_IRQ_TRIGGER_VALUE_ID(irq_id) HIVE_GP_TIMER_IRQ_TRIGGER_VALUE_REG_IDX(irq_id, HIVE_GP_TIMER_NUM_COUNTERS)
-
-#define _REG_GP_TIMER_IRQ_TIMER_SELECT_ID(irq_id)   \
-	HIVE_GP_TIMER_IRQ_TIMER_SELECT_REG_IDX(irq_id, HIVE_GP_TIMER_NUM_COUNTERS, HIVE_GP_TIMER_NUM_IRQS)
-
-#define _REG_GP_TIMER_IRQ_ENABLE_ID(irq_id) \
-	HIVE_GP_TIMER_IRQ_ENABLE_REG_IDX(irq_id, HIVE_GP_TIMER_NUM_COUNTERS, HIVE_GP_TIMER_NUM_IRQS)
-
-
-#endif  /*__GP_TIMER_LOCAL_H_INCLUDED__*/
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer_private.h
deleted file mode 100644
index 705be5e5cc70..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gp_timer_private.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GP_TIMER_PRIVATE_H_INCLUDED__
-#define __GP_TIMER_PRIVATE_H_INCLUDED__
-
-#include "gp_timer_public.h"
-#include "device_access.h"
-#include "assert_support.h"
-
-#endif /* __GP_TIMER_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gpio_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gpio_local.h
deleted file mode 100644
index f4652b79734d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gpio_local.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GPIO_LOCAL_H_INCLUDED__
-#define __GPIO_LOCAL_H_INCLUDED__
-
-#include "gpio_global.h"
-
-#endif /* __GPIO_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gpio_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gpio_private.h
deleted file mode 100644
index b6ebf34eaa9d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/gpio_private.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GPIO_PRIVATE_H_INCLUDED__
-#define __GPIO_PRIVATE_H_INCLUDED__
-
-#include "gpio_public.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-STORAGE_CLASS_GPIO_C void gpio_reg_store(
-	const gpio_ID_t	ID,
-	const unsigned int		reg,
-	const hrt_data			value)
-{
-OP___assert(ID < N_GPIO_ID);
-OP___assert(GPIO_BASE[ID] != (hrt_address)-1);
-	ia_css_device_store_uint32(GPIO_BASE[ID] + reg*sizeof(hrt_data), value);
-	return;
-}
-
-STORAGE_CLASS_GPIO_C hrt_data gpio_reg_load(
-	const gpio_ID_t	ID,
-	const unsigned int		reg)
-{
-OP___assert(ID < N_GPIO_ID);
-OP___assert(GPIO_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(GPIO_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-#endif /* __GPIO_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem.c
deleted file mode 100644
index e48f180c9507..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem.c
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "hmem.h"
-
-#ifndef __INLINE_HMEM__
-#include "hmem_private.h"
-#endif /* __INLINE_HMEM__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem_local.h
deleted file mode 100644
index 499f55f07253..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem_local.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __HMEM_LOCAL_H_INCLUDED__
-#define __HMEM_LOCAL_H_INCLUDED__
-
-#include "hmem_global.h"
-
-#endif /* __HMEM_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem_private.h
deleted file mode 100644
index 32a780380e11..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/hmem_private.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __HMEM_PRIVATE_H_INCLUDED__
-#define __HMEM_PRIVATE_H_INCLUDED__
-
-#include "hmem_public.h"
-
-#include "assert_support.h"
-
-STORAGE_CLASS_HMEM_C size_t sizeof_hmem(
-	const hmem_ID_t		ID)
-{
-	assert(ID < N_HMEM_ID);
-	(void)ID;
-	return HMEM_SIZE*sizeof(hmem_data_t);
-}
-
-#endif /* __HMEM_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter.c
deleted file mode 100644
index 0e1ca995fb06..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter.c
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "system_global.h"
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2
-
-#include "input_formatter.h"
-#include <type_support.h>
-#include "gp_device.h"
-
-#include "assert_support.h"
-
-#ifndef __INLINE_INPUT_FORMATTER__
-#include "input_formatter_private.h"
-#endif /* __INLINE_INPUT_FORMATTER__ */
-
-const hrt_address HIVE_IF_SRST_ADDRESS[N_INPUT_FORMATTER_ID] = {
-	INPUT_FORMATTER0_SRST_OFFSET,
-	INPUT_FORMATTER1_SRST_OFFSET,
-	INPUT_FORMATTER2_SRST_OFFSET,
-	INPUT_FORMATTER3_SRST_OFFSET};
-
-const hrt_data HIVE_IF_SRST_MASK[N_INPUT_FORMATTER_ID] = {
-	INPUT_FORMATTER0_SRST_MASK,
-	INPUT_FORMATTER1_SRST_MASK,
-	INPUT_FORMATTER2_SRST_MASK,
-	INPUT_FORMATTER3_SRST_MASK};
-
-const uint8_t HIVE_IF_SWITCH_CODE[N_INPUT_FORMATTER_ID] = {
-	HIVE_INPUT_SWITCH_SELECT_IF_PRIM,
-	HIVE_INPUT_SWITCH_SELECT_IF_PRIM,
-	HIVE_INPUT_SWITCH_SELECT_IF_SEC,
-	HIVE_INPUT_SWITCH_SELECT_STR_TO_MEM};
-
-/* MW Should be part of system_global.h, where we have the main enumeration */
-static const bool HIVE_IF_BIN_COPY[N_INPUT_FORMATTER_ID] = {
-	false, false, false, true
-};
-
-void input_formatter_rst(
-	const input_formatter_ID_t		ID)
-{
-	hrt_address	addr;
-	hrt_data	rst;
-
-	assert(ID < N_INPUT_FORMATTER_ID);
-
-	addr = HIVE_IF_SRST_ADDRESS[ID];
-	rst = HIVE_IF_SRST_MASK[ID];
-
-	/* TEMPORARY HACK: THIS RESET BREAKS THE METADATA FEATURE
-	 * WICH USES THE STREAM2MEMRY BLOCK.
-	 * MUST BE FIXED PROPERLY
-	 */
-	if (!HIVE_IF_BIN_COPY[ID]) {
-		input_formatter_reg_store(ID, addr, rst);
-	}
-
-	return;
-}
-
-unsigned int input_formatter_get_alignment(
-	const input_formatter_ID_t		ID)
-{
-	assert(ID < N_INPUT_FORMATTER_ID);
-
-	return input_formatter_alignment[ID];
-}
-
-void input_formatter_set_fifo_blocking_mode(
-	const input_formatter_ID_t		ID,
-	const bool						enable)
-{
-	assert(ID < N_INPUT_FORMATTER_ID);
-
-	/* cnd_input_formatter_reg_store() */
-	if (!HIVE_IF_BIN_COPY[ID]) {
-		input_formatter_reg_store(ID,
-			 HIVE_IF_BLOCK_FIFO_NO_REQ_ADDRESS, enable);
-	}
-	return;
-}
-
-void input_formatter_get_switch_state(
-	const input_formatter_ID_t		ID,
-	input_formatter_switch_state_t	*state)
-{
-	assert(ID < N_INPUT_FORMATTER_ID);
-	assert(state != NULL);
-
-	/* We'll change this into an intelligent function to get switch info per IF */
-	(void)ID;
-
-	state->if_input_switch_lut_reg[0] = gp_device_reg_load(GP_DEVICE0_ID, _REG_GP_IFMT_input_switch_lut_reg0);
-	state->if_input_switch_lut_reg[1] = gp_device_reg_load(GP_DEVICE0_ID, _REG_GP_IFMT_input_switch_lut_reg1);
-	state->if_input_switch_lut_reg[2] = gp_device_reg_load(GP_DEVICE0_ID, _REG_GP_IFMT_input_switch_lut_reg2);
-	state->if_input_switch_lut_reg[3] = gp_device_reg_load(GP_DEVICE0_ID, _REG_GP_IFMT_input_switch_lut_reg3);
-	state->if_input_switch_lut_reg[4] = gp_device_reg_load(GP_DEVICE0_ID, _REG_GP_IFMT_input_switch_lut_reg4);
-	state->if_input_switch_lut_reg[5] = gp_device_reg_load(GP_DEVICE0_ID, _REG_GP_IFMT_input_switch_lut_reg5);
-	state->if_input_switch_lut_reg[6] = gp_device_reg_load(GP_DEVICE0_ID, _REG_GP_IFMT_input_switch_lut_reg6);
-	state->if_input_switch_lut_reg[7] = gp_device_reg_load(GP_DEVICE0_ID, _REG_GP_IFMT_input_switch_lut_reg7);
-	state->if_input_switch_fsync_lut = gp_device_reg_load(GP_DEVICE0_ID, _REG_GP_IFMT_input_switch_fsync_lut);
-	state->if_input_switch_ch_id_fmt_type = gp_device_reg_load(GP_DEVICE0_ID, _REG_GP_IFMT_input_switch_ch_id_fmt_type);
-
-	return;
-}
-
-void input_formatter_get_state(
-	const input_formatter_ID_t		ID,
-	input_formatter_state_t			*state)
-{
-	assert(ID < N_INPUT_FORMATTER_ID);
-	assert(state != NULL);
-/*
-	state->reset = input_formatter_reg_load(ID,
-		HIVE_IF_RESET_ADDRESS);
- */
-	state->start_line = input_formatter_reg_load(ID,
-		HIVE_IF_START_LINE_ADDRESS);
-	state->start_column = input_formatter_reg_load(ID,
-		HIVE_IF_START_COLUMN_ADDRESS);
-	state->cropped_height = input_formatter_reg_load(ID,
-		HIVE_IF_CROPPED_HEIGHT_ADDRESS);
-	state->cropped_width = input_formatter_reg_load(ID,
-		HIVE_IF_CROPPED_WIDTH_ADDRESS);
-	state->ver_decimation = input_formatter_reg_load(ID,
-		HIVE_IF_VERTICAL_DECIMATION_ADDRESS);
-	state->hor_decimation = input_formatter_reg_load(ID,
-		HIVE_IF_HORIZONTAL_DECIMATION_ADDRESS);
-	state->hor_deinterleaving = input_formatter_reg_load(ID,
-		HIVE_IF_H_DEINTERLEAVING_ADDRESS);
-	state->left_padding = input_formatter_reg_load(ID,
-		HIVE_IF_LEFTPADDING_WIDTH_ADDRESS);
-	state->eol_offset = input_formatter_reg_load(ID,
-		HIVE_IF_END_OF_LINE_OFFSET_ADDRESS);
-	state->vmem_start_address = input_formatter_reg_load(ID,
-		HIVE_IF_VMEM_START_ADDRESS_ADDRESS);
-	state->vmem_end_address = input_formatter_reg_load(ID,
-		HIVE_IF_VMEM_END_ADDRESS_ADDRESS);
-	state->vmem_increment = input_formatter_reg_load(ID,
-		HIVE_IF_VMEM_INCREMENT_ADDRESS);
-	state->is_yuv420 = input_formatter_reg_load(ID,
-		HIVE_IF_YUV_420_FORMAT_ADDRESS);
-	state->vsync_active_low = input_formatter_reg_load(ID,
-		HIVE_IF_VSYNCK_ACTIVE_LOW_ADDRESS);
-	state->hsync_active_low = input_formatter_reg_load(ID,
-		HIVE_IF_HSYNCK_ACTIVE_LOW_ADDRESS);
-	state->allow_fifo_overflow = input_formatter_reg_load(ID,
-		HIVE_IF_ALLOW_FIFO_OVERFLOW_ADDRESS);
-	state->block_fifo_when_no_req = input_formatter_reg_load(ID,
-		HIVE_IF_BLOCK_FIFO_NO_REQ_ADDRESS);
-	state->ver_deinterleaving = input_formatter_reg_load(ID,
-		HIVE_IF_V_DEINTERLEAVING_ADDRESS);
-/* FSM */
-	state->fsm_sync_status = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_SYNC_STATUS);
-	state->fsm_sync_counter = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_SYNC_COUNTER);
-	state->fsm_crop_status = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_CROP_STATUS);
-	state->fsm_crop_line_counter = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_CROP_LINE_COUNTER);
-	state->fsm_crop_pixel_counter = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_CROP_PIXEL_COUNTER);
-	state->fsm_deinterleaving_index = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_DEINTERLEAVING_IDX);
-	state->fsm_dec_h_counter = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_DECIMATION_H_COUNTER);
-	state->fsm_dec_v_counter = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_DECIMATION_V_COUNTER);
-	state->fsm_dec_block_v_counter = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_DECIMATION_BLOCK_V_COUNTER);
-	state->fsm_padding_status = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_PADDING_STATUS);
-	state->fsm_padding_elem_counter = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_PADDING_ELEMENT_COUNTER);
-	state->fsm_vector_support_error = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_VECTOR_SUPPORT_ERROR);
-	state->fsm_vector_buffer_full = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_VECTOR_SUPPORT_BUFF_FULL);
-	state->vector_support = input_formatter_reg_load(ID,
-		HIVE_IF_FSM_VECTOR_SUPPORT);
-	state->sensor_data_lost = input_formatter_reg_load(ID,
-		HIVE_IF_FIFO_SENSOR_STATUS);
-
-	return;
-}
-
-void input_formatter_bin_get_state(
-	const input_formatter_ID_t		ID,
-	input_formatter_bin_state_t		*state)
-{
-	assert(ID < N_INPUT_FORMATTER_ID);
-	assert(state != NULL);
-
-	state->reset = input_formatter_reg_load(ID,
-		HIVE_STR2MEM_SOFT_RESET_REG_ADDRESS);
-	state->input_endianness = input_formatter_reg_load(ID,
-		HIVE_STR2MEM_INPUT_ENDIANNESS_REG_ADDRESS);
-	state->output_endianness = input_formatter_reg_load(ID,
-		HIVE_STR2MEM_OUTPUT_ENDIANNESS_REG_ADDRESS);
-	state->bitswap = input_formatter_reg_load(ID,
-		HIVE_STR2MEM_BIT_SWAPPING_REG_ADDRESS);
-	state->block_synch = input_formatter_reg_load(ID,
-		HIVE_STR2MEM_BLOCK_SYNC_LEVEL_REG_ADDRESS);
-	state->packet_synch = input_formatter_reg_load(ID,
-		HIVE_STR2MEM_PACKET_SYNC_LEVEL_REG_ADDRESS);
-	state->readpostwrite_synch = input_formatter_reg_load(ID,
-		HIVE_STR2MEM_READ_POST_WRITE_SYNC_ENABLE_REG_ADDRESS);
-	state->is_2ppc = input_formatter_reg_load(ID,
-		HIVE_STR2MEM_DUAL_BYTE_INPUTS_ENABLED_REG_ADDRESS);
-	state->en_status_update = input_formatter_reg_load(ID,
-		HIVE_STR2MEM_EN_STAT_UPDATE_ADDRESS);
-	return;
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter_local.h
deleted file mode 100644
index 3e00b5e6bad7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter_local.h
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_FORMATTER_LOCAL_H_INCLUDED__
-#define __INPUT_FORMATTER_LOCAL_H_INCLUDED__
-
-#include "input_formatter_global.h"
-
-#include "isp.h"		/* ISP_VEC_ALIGN */
-
-typedef struct input_formatter_switch_state_s	input_formatter_switch_state_t;
-typedef struct input_formatter_state_s			input_formatter_state_t;
-typedef struct input_formatter_bin_state_s		input_formatter_bin_state_t;
-
-#define HIVE_IF_FSM_SYNC_STATUS                 0x100
-#define HIVE_IF_FSM_SYNC_COUNTER                0x104
-#define HIVE_IF_FSM_DEINTERLEAVING_IDX          0x114
-#define HIVE_IF_FSM_DECIMATION_H_COUNTER        0x118
-#define HIVE_IF_FSM_DECIMATION_V_COUNTER        0x11C
-#define HIVE_IF_FSM_DECIMATION_BLOCK_V_COUNTER  0x120
-#define HIVE_IF_FSM_PADDING_STATUS              0x124
-#define HIVE_IF_FSM_PADDING_ELEMENT_COUNTER     0x128
-#define HIVE_IF_FSM_VECTOR_SUPPORT_ERROR        0x12C
-#define HIVE_IF_FSM_VECTOR_SUPPORT_BUFF_FULL    0x130
-#define HIVE_IF_FSM_VECTOR_SUPPORT              0x134
-#define HIVE_IF_FIFO_SENSOR_STATUS              0x138
-
-/*
- * The switch LUT's coding defines a sink for each
- * single channel ID + channel format type. Conversely
- * the sink (i.e. an input formatter) can be reached
- * from multiple channel & format type combinations
- *
- * LUT[0,1] channel=0, format type {0,1,...31}
- * LUT[2,3] channel=1, format type {0,1,...31}
- * LUT[4,5] channel=2, format type {0,1,...31}
- * LUT[6,7] channel=3, format type {0,1,...31}
- *
- * Each register hold 16 2-bit fields encoding the sink
- * {0,1,2,3}, "0" means unconnected.
- *
- * The single FSYNCH register uses four 3-bit fields of 1-hot
- * encoded sink information, "0" means unconnected.
- *
- * The encoding is redundant. The FSYNCH setting will connect
- * a channel to a sink. At that point the LUT's belonging to
- * that channel can be directed to another sink. Thus the data
- * goes to another place than the synch
- */
-struct input_formatter_switch_state_s {
-	int	if_input_switch_lut_reg[8];
-	int	if_input_switch_fsync_lut;
-	int	if_input_switch_ch_id_fmt_type;
-	bool if_input_switch_map[HIVE_SWITCH_N_CHANNELS][HIVE_SWITCH_N_FORMATTYPES];
-};
-
-struct input_formatter_state_s {
-/*	int	reset; */
-	int	start_line;
-	int	start_column;
-	int	cropped_height;
-	int	cropped_width;
-	int	ver_decimation;
-	int	hor_decimation;
-	int	ver_deinterleaving;
-	int	hor_deinterleaving;
-	int	left_padding;
-	int	eol_offset;
-	int	vmem_start_address;
-	int	vmem_end_address;
-	int	vmem_increment;
-	int	is_yuv420;
-	int	vsync_active_low;
-	int	hsync_active_low;
-	int	allow_fifo_overflow;
-	int block_fifo_when_no_req;
-	int	fsm_sync_status;
-	int	fsm_sync_counter;
-	int	fsm_crop_status;
-	int	fsm_crop_line_counter;
-	int	fsm_crop_pixel_counter;
-	int	fsm_deinterleaving_index;
-	int	fsm_dec_h_counter;
-	int	fsm_dec_v_counter;
-	int	fsm_dec_block_v_counter;
-	int	fsm_padding_status;
-	int	fsm_padding_elem_counter;
-	int	fsm_vector_support_error;
-	int	fsm_vector_buffer_full;
-	int	vector_support;
-	int	sensor_data_lost;
-};
-
-struct input_formatter_bin_state_s {
-	uint32_t	reset;
-	uint32_t	input_endianness;
-	uint32_t	output_endianness;
-	uint32_t	bitswap;
-	uint32_t	block_synch;
-	uint32_t	packet_synch;
-	uint32_t	readpostwrite_synch;
-	uint32_t	is_2ppc;
-	uint32_t	en_status_update;
-};
-
-static const unsigned int input_formatter_alignment[N_INPUT_FORMATTER_ID] = {
-	ISP_VEC_ALIGN, ISP_VEC_ALIGN, HIVE_ISP_CTRL_DATA_BYTES};
-
-#endif /* __INPUT_FORMATTER_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter_private.h
deleted file mode 100644
index 2f42a9c2771c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_formatter_private.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_FORMATTER_PRIVATE_H_INCLUDED__
-#define __INPUT_FORMATTER_PRIVATE_H_INCLUDED__
-
-#include "input_formatter_public.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-STORAGE_CLASS_INPUT_FORMATTER_C void input_formatter_reg_store(
-	const input_formatter_ID_t		ID,
-	const hrt_address			reg_addr,
-	const hrt_data				value)
-{
-	assert(ID < N_INPUT_FORMATTER_ID);
-	assert(INPUT_FORMATTER_BASE[ID] != (hrt_address)-1);
-	assert((reg_addr % sizeof(hrt_data)) == 0);
-	ia_css_device_store_uint32(INPUT_FORMATTER_BASE[ID] + reg_addr, value);
-	return;
-}
-
-STORAGE_CLASS_INPUT_FORMATTER_C hrt_data input_formatter_reg_load(
-	const input_formatter_ID_t	ID,
-	const unsigned int			reg_addr)
-{
-	assert(ID < N_INPUT_FORMATTER_ID);
-	assert(INPUT_FORMATTER_BASE[ID] != (hrt_address)-1);
-	assert((reg_addr % sizeof(hrt_data)) == 0);
-	return ia_css_device_load_uint32(INPUT_FORMATTER_BASE[ID] + reg_addr);
-}
-
-#endif /* __INPUT_FORMATTER_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system.c
deleted file mode 100644
index 2515e162828f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system.c
+++ /dev/null
@@ -1,1823 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "system_global.h"
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2
-
-#include "input_system.h"
-#include <type_support.h>
-#include "gp_device.h"
-
-#include "assert_support.h"
-
-#ifndef __INLINE_INPUT_SYSTEM__
-#include "input_system_private.h"
-#endif /* __INLINE_INPUT_SYSTEM__ */
-
-#define ZERO (0x0)
-#define ONE  (1U)
-
-static const ib_buffer_t   IB_BUFFER_NULL = {0 ,0, 0 };
-
-static input_system_error_t input_system_configure_channel(
-	const channel_cfg_t		channel);
-
-static input_system_error_t input_system_configure_channel_sensor(
-	const channel_cfg_t		channel);
-
-static input_system_error_t input_buffer_configuration(void);
-
-static input_system_error_t configuration_to_registers(void);
-
-static void receiver_rst(const rx_ID_t ID);
-static void input_system_network_rst(const input_system_ID_t ID);
-
-static void capture_unit_configure(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	const ib_buffer_t* const		cfg);
-
-static void acquisition_unit_configure(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	const ib_buffer_t* const		cfg);
-
-static void ctrl_unit_configure(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	const ctrl_unit_cfg_t* const	cfg);
-
-static void input_system_network_configure(
-	const input_system_ID_t			ID,
-	const input_system_network_cfg_t * const	cfg);
-
-// MW: CSI is previously named as "rx" short for "receiver"
-static input_system_error_t set_csi_cfg(
-	csi_cfg_t* const						lhs,
-	const csi_cfg_t* const					rhs,
-	input_system_config_flags_t* const 		flags);
-
-static input_system_error_t set_source_type(
-	input_system_source_t* const 			lhs,
-	const input_system_source_t	 			rhs,
-	input_system_config_flags_t* const		flags);
-
-static input_system_error_t input_system_multiplexer_cfg(
-	input_system_multiplex_t* const			lhs,
-	const input_system_multiplex_t			rhs,
-	input_system_config_flags_t* const		flags);
-
-
-
-static inline void capture_unit_get_state(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	capture_unit_state_t			*state);
-
-static inline void acquisition_unit_get_state(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	acquisition_unit_state_t		*state);
-
-static inline void ctrl_unit_get_state(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	ctrl_unit_state_t				*state);
-
-static inline void mipi_port_get_state(
-	const rx_ID_t					ID,
-	const enum mipi_port_id			port_ID,
-	mipi_port_state_t				*state);
-
-static inline void rx_channel_get_state(
-	const rx_ID_t					ID,
-	const unsigned int				ch_id,
-	rx_channel_state_t				*state);
-
-static void gp_device_rst(const gp_device_ID_t		ID);
-
-static void input_selector_cfg_for_sensor(const gp_device_ID_t	ID);
-
-static void input_switch_rst(const gp_device_ID_t	ID);
-
-static void input_switch_cfg(
-	const gp_device_ID_t				ID,
-	const input_switch_cfg_t * const	cfg
-);
-
-void input_system_get_state(
-	const input_system_ID_t			ID,
-	input_system_state_t			*state)
-{
-	sub_system_ID_t	sub_id;
-
-	assert(ID < N_INPUT_SYSTEM_ID);
-	assert(state != NULL);
-
-	state->str_multicastA_sel = input_system_sub_system_reg_load(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MULTICAST_A_IDX);
-	state->str_multicastB_sel = input_system_sub_system_reg_load(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MULTICAST_B_IDX);
-	state->str_multicastC_sel = input_system_sub_system_reg_load(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MULTICAST_C_IDX);
-	state->str_mux_sel = input_system_sub_system_reg_load(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MUX_IDX);
-	state->str_mon_status = input_system_sub_system_reg_load(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_STRMON_STAT_IDX);
-	state->str_mon_irq_cond = input_system_sub_system_reg_load(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_STRMON_COND_IDX);
-	state->str_mon_irq_en = input_system_sub_system_reg_load(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_STRMON_IRQ_EN_IDX);
-	state->isys_srst = input_system_sub_system_reg_load(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_SRST_IDX);
-	state->isys_slv_reg_srst = input_system_sub_system_reg_load(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_SLV_REG_SRST_IDX);
-	state->str_deint_portA_cnt = input_system_sub_system_reg_load(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_REG_PORT_A_IDX);
-	state->str_deint_portB_cnt = input_system_sub_system_reg_load(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_REG_PORT_B_IDX);
-
-	for (sub_id = CAPTURE_UNIT0_ID; sub_id < CAPTURE_UNIT0_ID + N_CAPTURE_UNIT_ID; sub_id++) {
-		capture_unit_get_state(ID, sub_id,
-			&(state->capture_unit[sub_id - CAPTURE_UNIT0_ID]));
-	}
-	for (sub_id = ACQUISITION_UNIT0_ID; sub_id < ACQUISITION_UNIT0_ID + N_ACQUISITION_UNIT_ID; sub_id++) {
-		acquisition_unit_get_state(ID, sub_id,
-			&(state->acquisition_unit[sub_id - ACQUISITION_UNIT0_ID]));
-	}
-	for (sub_id = CTRL_UNIT0_ID; sub_id < CTRL_UNIT0_ID + N_CTRL_UNIT_ID; sub_id++) {
-		ctrl_unit_get_state(ID, sub_id,
-			&(state->ctrl_unit_state[sub_id - CTRL_UNIT0_ID]));
-	}
-
-	return;
-}
-
-void receiver_get_state(
-	const rx_ID_t				ID,
-	receiver_state_t			*state)
-{
-	enum mipi_port_id	port_id;
-	unsigned int	ch_id;
-
-	assert(ID < N_RX_ID);
-	assert(state != NULL);
-
-	state->fs_to_ls_delay = (uint8_t)receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_FS_TO_LS_DELAY_REG_IDX);
-	state->ls_to_data_delay = (uint8_t)receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_LS_TO_DATA_DELAY_REG_IDX);
-	state->data_to_le_delay = (uint8_t)receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_DATA_TO_LE_DELAY_REG_IDX);
-	state->le_to_fe_delay = (uint8_t)receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_LE_TO_FE_DELAY_REG_IDX);
-	state->fe_to_fs_delay = (uint8_t)receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_FE_TO_FS_DELAY_REG_IDX);
-	state->le_to_fs_delay = (uint8_t)receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_LE_TO_LS_DELAY_REG_IDX);
-	state->is_two_ppc = (bool)receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_TWO_PIXEL_EN_REG_IDX);
-	state->backend_rst = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BACKEND_RST_REG_IDX);
-	state->raw18 = (uint16_t)receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_RAW18_REG_IDX);
-	state->force_raw8 = (bool)receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_FORCE_RAW8_REG_IDX);
-	state->raw16 = (uint16_t)receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_RAW16_REG_IDX);
-
-	for (port_id = (enum mipi_port_id)0; port_id < N_MIPI_PORT_ID; port_id++) {
-		mipi_port_get_state(ID, port_id,
-			&(state->mipi_port_state[port_id]));
-	}
-	for (ch_id = (unsigned int)0; ch_id < N_RX_CHANNEL_ID; ch_id++) {
-		rx_channel_get_state(ID, ch_id,
-			&(state->rx_channel_state[ch_id]));
-	}
-
-	state->be_gsp_acc_ovl = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_GSP_ACC_OVL_REG_IDX);
-	state->be_srst = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_SRST_REG_IDX);
-	state->be_is_two_ppc = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_TWO_PPC_REG_IDX);
-	state->be_comp_format0 = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_COMP_FORMAT_REG0_IDX);
-	state->be_comp_format1 = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_COMP_FORMAT_REG1_IDX);
-	state->be_comp_format2 = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_COMP_FORMAT_REG2_IDX);
-	state->be_comp_format3 = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_COMP_FORMAT_REG3_IDX);
-	state->be_sel = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_SEL_REG_IDX);
-	state->be_raw16_config = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_RAW16_CONFIG_REG_IDX);
-	state->be_raw18_config = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_RAW18_CONFIG_REG_IDX);
-	state->be_force_raw8 = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_FORCE_RAW8_REG_IDX);
-	state->be_irq_status = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_IRQ_STATUS_REG_IDX);
-	state->be_irq_clear = receiver_reg_load(ID,
-		_HRT_CSS_RECEIVER_BE_IRQ_CLEAR_REG_IDX);
-
-	return;
-}
-
-bool is_mipi_format_yuv420(
-	const mipi_format_t			mipi_format)
-{
-	bool	is_yuv420 = (
-		(mipi_format == MIPI_FORMAT_YUV420_8) ||
-		(mipi_format == MIPI_FORMAT_YUV420_10) ||
-		(mipi_format == MIPI_FORMAT_YUV420_8_SHIFT) ||
-		(mipi_format == MIPI_FORMAT_YUV420_10_SHIFT));
-/* MIPI_FORMAT_YUV420_8_LEGACY is not YUV420 */
-
-	return is_yuv420;
-}
-
-void receiver_set_compression(
-	const rx_ID_t			ID,
-	const unsigned int		cfg_ID,
-	const mipi_compressor_t		comp,
-	const mipi_predictor_t		pred)
-{
-	const unsigned int		field_id = cfg_ID % N_MIPI_FORMAT_CUSTOM;
-	const unsigned int		ch_id = cfg_ID / N_MIPI_FORMAT_CUSTOM;
-	hrt_data			val;
-	hrt_address			addr = 0;
-	hrt_data			reg;
-
-	assert(ID < N_RX_ID);
-	assert(cfg_ID < N_MIPI_COMPRESSOR_CONTEXT);
-	assert(field_id < N_MIPI_FORMAT_CUSTOM);
-	assert(ch_id < N_RX_CHANNEL_ID);
-	assert(comp < N_MIPI_COMPRESSOR_METHODS);
-	assert(pred < N_MIPI_PREDICTOR_TYPES);
-
-	val = (((uint8_t)pred) << 3) | comp;
-
-	switch (ch_id) {
-	case 0: addr = ((field_id<6)?_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG0_IDX:_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG1_IDX);
-		break;
-	case 1: addr = ((field_id<6)?_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG0_IDX:_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG1_IDX);
-		break;
-	case 2: addr = ((field_id<6)?_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG0_IDX:_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG1_IDX);
-		break;
-	case 3: addr = ((field_id<6)?_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG0_IDX:_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG1_IDX);
-		break;
-	default:
-		/* should not happen */
-		assert(false);
-		return;
-	}
-
-	reg = ((field_id < 6)?(val << (field_id * 5)):(val << ((field_id - 6) * 5)));
-	receiver_reg_store(ID, addr, reg);
-
-	return;
-}
-
-void receiver_port_enable(
-	const rx_ID_t			ID,
-	const enum mipi_port_id		port_ID,
-	const bool			cnd)
-{
-	hrt_data	reg = receiver_port_reg_load(ID, port_ID,
-		_HRT_CSS_RECEIVER_DEVICE_READY_REG_IDX);
-
-	if (cnd) {
-		reg |= 0x01;
-	} else {
-		reg &= ~0x01;
-	}
-
-	receiver_port_reg_store(ID, port_ID,
-		_HRT_CSS_RECEIVER_DEVICE_READY_REG_IDX, reg);
-	return;
-}
-
-bool is_receiver_port_enabled(
-	const rx_ID_t			ID,
-	const enum mipi_port_id		port_ID)
-{
-	hrt_data	reg = receiver_port_reg_load(ID, port_ID,
-		_HRT_CSS_RECEIVER_DEVICE_READY_REG_IDX);
-	return ((reg & 0x01) != 0);
-}
-
-void receiver_irq_enable(
-	const rx_ID_t			ID,
-	const enum mipi_port_id		port_ID,
-	const rx_irq_info_t		irq_info)
-{
-	receiver_port_reg_store(ID,
-		port_ID, _HRT_CSS_RECEIVER_IRQ_ENABLE_REG_IDX, irq_info);
-	return;
-}
-
-rx_irq_info_t receiver_get_irq_info(
-	const rx_ID_t			ID,
-	const enum mipi_port_id		port_ID)
-{
-	return receiver_port_reg_load(ID,
-	port_ID, _HRT_CSS_RECEIVER_IRQ_STATUS_REG_IDX);
-}
-
-void receiver_irq_clear(
-	const rx_ID_t			ID,
-	const enum mipi_port_id		port_ID,
-	const rx_irq_info_t		irq_info)
-{
-	receiver_port_reg_store(ID,
-		port_ID, _HRT_CSS_RECEIVER_IRQ_STATUS_REG_IDX, irq_info);
-	return;
-}
-
-static inline void capture_unit_get_state(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	capture_unit_state_t			*state)
-{
-	assert(/*(sub_id >= CAPTURE_UNIT0_ID) &&*/ (sub_id <= CAPTURE_UNIT2_ID));
-	assert(state != NULL);
-
-	state->StartMode = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_START_MODE_REG_ID);
-	state->Start_Addr = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_START_ADDR_REG_ID);
-	state->Mem_Region_Size = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_MEM_REGION_SIZE_REG_ID);
-	state->Num_Mem_Regions = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_NUM_MEM_REGIONS_REG_ID);
-//	AM: Illegal read from following registers.
-/*	state->Init = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_INIT_REG_ID);
-	state->Start = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_START_REG_ID);
-	state->Stop = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_STOP_REG_ID);
-*/
-	state->Packet_Length = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_PACKET_LENGTH_REG_ID);
-	state->Received_Length = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_RECEIVED_LENGTH_REG_ID);
-	state->Received_Short_Packets = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_RECEIVED_SHORT_PACKETS_REG_ID);
-	state->Received_Long_Packets = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_RECEIVED_LONG_PACKETS_REG_ID);
-	state->Last_Command = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_LAST_COMMAND_REG_ID);
-	state->Next_Command = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_NEXT_COMMAND_REG_ID);
-	state->Last_Acknowledge = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_LAST_ACKNOWLEDGE_REG_ID);
-	state->Next_Acknowledge = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_NEXT_ACKNOWLEDGE_REG_ID);
-	state->FSM_State_Info = input_system_sub_system_reg_load(ID,
-		sub_id,
-		CAPT_FSM_STATE_INFO_REG_ID);
-
-	return;
-}
-
-static inline void acquisition_unit_get_state(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	acquisition_unit_state_t		*state)
-{
-	assert(sub_id == ACQUISITION_UNIT0_ID);
-	assert(state != NULL);
-
-	state->Start_Addr = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_START_ADDR_REG_ID);
-	state->Mem_Region_Size = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_MEM_REGION_SIZE_REG_ID);
-	state->Num_Mem_Regions = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_NUM_MEM_REGIONS_REG_ID);
-//	AM: Illegal read from following registers.
-/*	state->Init = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_INIT_REG_ID);
-*/
-	state->Received_Short_Packets = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_RECEIVED_SHORT_PACKETS_REG_ID);
-	state->Received_Long_Packets = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_RECEIVED_LONG_PACKETS_REG_ID);
-	state->Last_Command = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_LAST_COMMAND_REG_ID);
-	state->Next_Command = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_NEXT_COMMAND_REG_ID);
-	state->Last_Acknowledge = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_LAST_ACKNOWLEDGE_REG_ID);
-	state->Next_Acknowledge = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_NEXT_ACKNOWLEDGE_REG_ID);
-	state->FSM_State_Info = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_FSM_STATE_INFO_REG_ID);
-	state->Int_Cntr_Info = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ACQ_INT_CNTR_INFO_REG_ID);
-
-	return;
-}
-
-static inline void ctrl_unit_get_state(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	ctrl_unit_state_t			*state)
-{
-	assert(sub_id == CTRL_UNIT0_ID);
-	assert(state != NULL);
-
-	state->captA_start_addr = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_START_ADDR_A_REG_ID);
-	state->captB_start_addr = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_START_ADDR_B_REG_ID);
-	state->captC_start_addr = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_START_ADDR_C_REG_ID);
-	state->captA_mem_region_size = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_MEM_REGION_SIZE_A_REG_ID);
-	state->captB_mem_region_size = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_MEM_REGION_SIZE_B_REG_ID);
-	state->captC_mem_region_size = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_MEM_REGION_SIZE_C_REG_ID);
-	state->captA_num_mem_regions = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_NUM_MEM_REGIONS_A_REG_ID);
-	state->captB_num_mem_regions = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_NUM_MEM_REGIONS_B_REG_ID);
-	state->captC_num_mem_regions = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_NUM_MEM_REGIONS_C_REG_ID);
-	state->acq_start_addr = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_ACQ_START_ADDR_REG_ID);
-	state->acq_mem_region_size = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_ACQ_MEM_REGION_SIZE_REG_ID);
-	state->acq_num_mem_regions = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_ACQ_NUM_MEM_REGIONS_REG_ID);
-//	AM: Illegal read from following registers.
-/*	state->ctrl_init = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_INIT_REG_ID);
-*/
-	state->last_cmd = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_LAST_COMMAND_REG_ID);
-	state->next_cmd = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_NEXT_COMMAND_REG_ID);
-	state->last_ack = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_LAST_ACKNOWLEDGE_REG_ID);
-	state->next_ack = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_NEXT_ACKNOWLEDGE_REG_ID);
-	state->top_fsm_state = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_FSM_STATE_INFO_REG_ID);
-	state->captA_fsm_state = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_A_FSM_STATE_INFO_REG_ID);
-	state->captB_fsm_state = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_B_FSM_STATE_INFO_REG_ID);
-	state->captC_fsm_state = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_C_FSM_STATE_INFO_REG_ID);
-	state->acq_fsm_state = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_ACQ_FSM_STATE_INFO_REG_ID);
-	state->capt_reserve_one_mem_region = input_system_sub_system_reg_load(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_RESERVE_ONE_MEM_REGION_REG_ID);
-
-	return;
-}
-
-static inline void mipi_port_get_state(
-	const rx_ID_t				ID,
-	const enum mipi_port_id			port_ID,
-	mipi_port_state_t			*state)
-{
-	int	i;
-
-	assert(ID < N_RX_ID);
-	assert(port_ID < N_MIPI_PORT_ID);
-	assert(state != NULL);
-
-	state->device_ready = receiver_port_reg_load(ID,
-		port_ID, _HRT_CSS_RECEIVER_DEVICE_READY_REG_IDX);
-	state->irq_status = receiver_port_reg_load(ID,
-		port_ID, _HRT_CSS_RECEIVER_IRQ_STATUS_REG_IDX);
-	state->irq_enable = receiver_port_reg_load(ID,
-		port_ID, _HRT_CSS_RECEIVER_IRQ_ENABLE_REG_IDX);
-	state->timeout_count = receiver_port_reg_load(ID,
-		port_ID, _HRT_CSS_RECEIVER_TIMEOUT_COUNT_REG_IDX);
-	state->init_count = (uint16_t)receiver_port_reg_load(ID,
-		port_ID, _HRT_CSS_RECEIVER_INIT_COUNT_REG_IDX);
-	state->raw16_18 = (uint16_t)receiver_port_reg_load(ID,
-		port_ID, _HRT_CSS_RECEIVER_RAW16_18_DATAID_REG_IDX);
-	state->sync_count = receiver_port_reg_load(ID,
-		port_ID, _HRT_CSS_RECEIVER_SYNC_COUNT_REG_IDX);
-	state->rx_count = receiver_port_reg_load(ID,
-		port_ID, _HRT_CSS_RECEIVER_RX_COUNT_REG_IDX);
-
-	for (i = 0; i < MIPI_4LANE_CFG ; i++) {
-		state->lane_sync_count[i] = (uint8_t)((state->sync_count)>>(i*8));
-		state->lane_rx_count[i] = (uint8_t)((state->rx_count)>>(i*8));
-	}
-
-	return;
-}
-
-static inline void rx_channel_get_state(
-	const rx_ID_t					ID,
-	const unsigned int				ch_id,
-	rx_channel_state_t				*state)
-{
-	int	i;
-
-	assert(ID < N_RX_ID);
-	assert(ch_id < N_RX_CHANNEL_ID);
-	assert(state != NULL);
-
-	switch (ch_id) {
-	case 0:
-		state->comp_scheme0 = receiver_reg_load(ID,
-				_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG0_IDX);
-		state->comp_scheme1 = receiver_reg_load(ID,
-				_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG1_IDX);
-		break;
-	case 1:
-		state->comp_scheme0 = receiver_reg_load(ID,
-				_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG0_IDX);
-		state->comp_scheme1 = receiver_reg_load(ID,
-				_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG1_IDX);
-		break;
-	case 2:
-		state->comp_scheme0 = receiver_reg_load(ID,
-				_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG0_IDX);
-		state->comp_scheme1 = receiver_reg_load(ID,
-				_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG1_IDX);
-		break;
-	case 3:
-		state->comp_scheme0 = receiver_reg_load(ID,
-				_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG0_IDX);
-		state->comp_scheme1 = receiver_reg_load(ID,
-				_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG1_IDX);
-		break;
-	}
-
-/* See Table 7.1.17,..., 7.1.24 */
-	for (i = 0; i < 6; i++) {
-		uint8_t	val = (uint8_t)((state->comp_scheme0)>>(i*5)) & 0x1f;
-		state->comp[i] = (mipi_compressor_t)(val & 0x07);
-		state->pred[i] = (mipi_predictor_t)((val & 0x18) >> 3);
-	}
-	for (i = 6; i < N_MIPI_FORMAT_CUSTOM; i++) {
-		uint8_t	val = (uint8_t)((state->comp_scheme0)>>((i-6)*5)) & 0x1f;
-		state->comp[i] = (mipi_compressor_t)(val & 0x07);
-		state->pred[i] = (mipi_predictor_t)((val & 0x18) >> 3);
-	}
-
-	return;
-}
-
-// MW: "2400" in the name is not good, but this is to avoid a naming conflict
-static input_system_cfg2400_t config;
-
-static void receiver_rst(
-	const rx_ID_t				ID)
-{
-	enum mipi_port_id		port_id;
-
-	assert(ID < N_RX_ID);
-
-// Disable all ports.
-	for (port_id = MIPI_PORT0_ID; port_id < N_MIPI_PORT_ID; port_id++) {
-		receiver_port_enable(ID, port_id, false);
-	}
-
-	// AM: Additional actions for stopping receiver?
-
-	return;
-}
-
-//Single function to reset all the devices mapped via GP_DEVICE.
-static void gp_device_rst(const gp_device_ID_t		ID)
-{
-	assert(ID < N_GP_DEVICE_ID);
-
-	gp_device_reg_store(ID, _REG_GP_SYNCGEN_ENABLE_ADDR, ZERO);
-	// gp_device_reg_store(ID, _REG_GP_SYNCGEN_FREE_RUNNING_ADDR, ZERO);
-	// gp_device_reg_store(ID, _REG_GP_SYNCGEN_PAUSE_ADDR, ONE);
-	// gp_device_reg_store(ID, _REG_GP_NR_FRAMES_ADDR, ZERO);
-	// gp_device_reg_store(ID, _REG_GP_SYNGEN_NR_PIX_ADDR, ZERO);
-	// gp_device_reg_store(ID, _REG_GP_SYNGEN_NR_PIX_ADDR, ZERO);
-	// gp_device_reg_store(ID, _REG_GP_SYNGEN_NR_LINES_ADDR, ZERO);
-	// gp_device_reg_store(ID, _REG_GP_SYNGEN_HBLANK_CYCLES_ADDR, ZERO);
-	// gp_device_reg_store(ID, _REG_GP_SYNGEN_VBLANK_CYCLES_ADDR, ZERO);
-// AM: Following calls cause strange warnings. Probably they should not be initialized.
-//	gp_device_reg_store(ID, _REG_GP_ISEL_SOF_ADDR, ZERO);
-//	gp_device_reg_store(ID, _REG_GP_ISEL_EOF_ADDR, ZERO);
-//	gp_device_reg_store(ID, _REG_GP_ISEL_SOL_ADDR, ZERO);
-//	gp_device_reg_store(ID, _REG_GP_ISEL_EOL_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_LFSR_ENABLE_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_LFSR_ENABLE_B_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_LFSR_RESET_VALUE_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_TPG_ENABLE_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_TPG_ENABLE_B_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_HOR_CNT_MASK_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_VER_CNT_MASK_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_XY_CNT_MASK_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_HOR_CNT_DELTA_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_VER_CNT_DELTA_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_TPG_MODE_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_TPG_RED1_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_TPG_GREEN1_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_TPG_BLUE1_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_TPG_RED2_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_TPG_GREEN2_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_TPG_BLUE2_ADDR, ZERO);
-	//gp_device_reg_store(ID, _REG_GP_ISEL_CH_ID_ADDR, ZERO);
-	//gp_device_reg_store(ID, _REG_GP_ISEL_FMT_TYPE_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_DATA_SEL_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_SBAND_SEL_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_SYNC_SEL_ADDR, ZERO);
-	//	gp_device_reg_store(ID, _REG_GP_SYNCGEN_HOR_CNT_ADDR, ZERO);
-	//	gp_device_reg_store(ID, _REG_GP_SYNCGEN_VER_CNT_ADDR, ZERO);
-	//	gp_device_reg_store(ID, _REG_GP_SYNCGEN_FRAME_CNT_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_SOFT_RESET_ADDR, ZERO); // AM: Maybe this soft reset is not safe.
-
-	return;
-}
-
-static void input_selector_cfg_for_sensor(const gp_device_ID_t ID)
-{
-	assert(ID < N_GP_DEVICE_ID);
-
-	gp_device_reg_store(ID, _REG_GP_ISEL_SOF_ADDR, ONE);
-	gp_device_reg_store(ID, _REG_GP_ISEL_EOF_ADDR, ONE);
-	gp_device_reg_store(ID, _REG_GP_ISEL_SOL_ADDR, ONE);
-	gp_device_reg_store(ID, _REG_GP_ISEL_EOL_ADDR, ONE);
-	gp_device_reg_store(ID, _REG_GP_ISEL_CH_ID_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_FMT_TYPE_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_DATA_SEL_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_SBAND_SEL_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_ISEL_SYNC_SEL_ADDR, ZERO);
-	gp_device_reg_store(ID, _REG_GP_SOFT_RESET_ADDR, ZERO);
-
-	return;
-}
-
-static void input_switch_rst(const gp_device_ID_t ID)
-{
-	int addr;
-
-	assert(ID < N_GP_DEVICE_ID);
-
-	// Initialize the data&hsync LUT.
-	for (addr = _REG_GP_IFMT_input_switch_lut_reg0;
-			 addr <= _REG_GP_IFMT_input_switch_lut_reg7; addr += SIZEOF_HRT_REG) {
-
-		gp_device_reg_store(ID, addr, ZERO);
-	}
-
-	// Initialize the vsync LUT.
-	gp_device_reg_store(ID,
-		_REG_GP_IFMT_input_switch_fsync_lut,
-		ZERO);
-
-	return;
-}
-
-static void input_switch_cfg(
-	const gp_device_ID_t			ID,
-	const input_switch_cfg_t * const	cfg)
-{
-	int addr_offset;
-
-	assert(ID < N_GP_DEVICE_ID);
-	assert(cfg != NULL);
-
-	// Initialize the data&hsync LUT.
-	for (addr_offset = 0; addr_offset < N_RX_CHANNEL_ID * 2; addr_offset++) {
-		assert(addr_offset * SIZEOF_HRT_REG + _REG_GP_IFMT_input_switch_lut_reg0 <= _REG_GP_IFMT_input_switch_lut_reg7);
-		gp_device_reg_store(ID,
-			_REG_GP_IFMT_input_switch_lut_reg0 + addr_offset * SIZEOF_HRT_REG,
-			cfg->hsync_data_reg[addr_offset]);
-	}
-
-	// Initialize the vsync LUT.
-	gp_device_reg_store(ID,
-		_REG_GP_IFMT_input_switch_fsync_lut,
-		cfg->vsync_data_reg);
-
-	return;
-}
-
-
-static void input_system_network_rst(const input_system_ID_t ID)
-{
-	unsigned int sub_id;
-
-	// Reset all 3 multicasts.
-	input_system_sub_system_reg_store(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MULTICAST_A_IDX,
-		INPUT_SYSTEM_DISCARD_ALL);
-	input_system_sub_system_reg_store(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MULTICAST_B_IDX,
-		INPUT_SYSTEM_DISCARD_ALL);
-	input_system_sub_system_reg_store(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MULTICAST_C_IDX,
-		INPUT_SYSTEM_DISCARD_ALL);
-
-	// Reset stream mux.
-	input_system_sub_system_reg_store(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MUX_IDX,
-		N_INPUT_SYSTEM_MULTIPLEX);
-
-	// Reset 3 capture units.
-	for (sub_id = CAPTURE_UNIT0_ID; sub_id < CAPTURE_UNIT0_ID + N_CAPTURE_UNIT_ID; sub_id++) {
-		input_system_sub_system_reg_store(ID,
-			sub_id,
-			CAPT_INIT_REG_ID,
-			1U << CAPT_INIT_RST_REG_BIT);
-	}
-
-	// Reset acquisition unit.
-	for (sub_id = ACQUISITION_UNIT0_ID; sub_id < ACQUISITION_UNIT0_ID + N_ACQUISITION_UNIT_ID; sub_id++) {
-		input_system_sub_system_reg_store(ID,
-			sub_id,
-			ACQ_INIT_REG_ID,
-			1U << ACQ_INIT_RST_REG_BIT);
-	}
-
-	// DMA unit reset is not needed.
-
-	// Reset controller units.
-	// NB: In future we need to keep part of ctrl_state for split capture and
-	for (sub_id = CTRL_UNIT0_ID; sub_id < CTRL_UNIT0_ID + N_CTRL_UNIT_ID; sub_id++) {
-		input_system_sub_system_reg_store(ID,
-			sub_id,
-			ISYS_CTRL_INIT_REG_ID,
-			1U); //AM: Is there any named constant?
-	}
-
-	return;
-}
-
-// Function that resets current configuration.
-input_system_error_t input_system_configuration_reset(void)
-{
-	unsigned int i;
-
-	receiver_rst(RX0_ID);
-
-	input_system_network_rst(INPUT_SYSTEM0_ID);
-
-	gp_device_rst(INPUT_SYSTEM0_ID);
-
-	input_switch_rst(INPUT_SYSTEM0_ID);
-
-	//target_rst();
-
-	// Reset IRQ_CTRLs.
-
-	// Reset configuration data structures.
-	for (i = 0; i < N_CHANNELS; i++ ) {
-		config.ch_flags[i] = INPUT_SYSTEM_CFG_FLAG_RESET;
-		config.target_isp_flags[i] = INPUT_SYSTEM_CFG_FLAG_RESET;
-		config.target_sp_flags[i] = INPUT_SYSTEM_CFG_FLAG_RESET;
-		config.target_strm2mem_flags[i] = INPUT_SYSTEM_CFG_FLAG_RESET;
-	}
-
-	for (i = 0; i < N_CSI_PORTS; i++ ) {
-		config.csi_buffer_flags[i]	 = INPUT_SYSTEM_CFG_FLAG_RESET;
-		config.multicast[i]		 = INPUT_SYSTEM_CFG_FLAG_RESET;
-	}
-
-	config.source_type_flags				 = INPUT_SYSTEM_CFG_FLAG_RESET;
-	config.acquisition_buffer_unique_flags	 = INPUT_SYSTEM_CFG_FLAG_RESET;
-	config.unallocated_ib_mem_words			 = IB_CAPACITY_IN_WORDS;
-	//config.acq_allocated_ib_mem_words		 = 0;
-
-	// Set the start of the session cofiguration.
-	config.session_flags = INPUT_SYSTEM_CFG_FLAG_REQUIRED;
-
-	return INPUT_SYSTEM_ERR_NO_ERROR;
-}
-
-// MW: Comments are good, but doxygen is required, place it at the declaration
-// Function that appends the channel to current configuration.
-static input_system_error_t input_system_configure_channel(
-	const channel_cfg_t		channel)
-{
-	input_system_error_t error = INPUT_SYSTEM_ERR_NO_ERROR;
-	// Check if channel is not already configured.
-	if (config.ch_flags[channel.ch_id] & INPUT_SYSTEM_CFG_FLAG_SET){
-		return INPUT_SYSTEM_ERR_CHANNEL_ALREADY_SET;
-	} else {
-		switch (channel.source_type){
-			case INPUT_SYSTEM_SOURCE_SENSOR :
-				error = input_system_configure_channel_sensor(channel);
-				break;
-			case INPUT_SYSTEM_SOURCE_TPG :
-				return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-				break;
-			case INPUT_SYSTEM_SOURCE_PRBS :
-				return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-				break;
-			case INPUT_SYSTEM_SOURCE_FIFO :
-				return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-				break;
-			default :
-				return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-				break;
-		}
-
-		if (error != INPUT_SYSTEM_ERR_NO_ERROR) return error;
-		// Input switch channel configurations must be combined in united config.
-		config.input_switch_cfg.hsync_data_reg[channel.source_cfg.csi_cfg.csi_port * 2] =
-				channel.target_cfg.input_switch_channel_cfg.hsync_data_reg[0];
-		config.input_switch_cfg.hsync_data_reg[channel.source_cfg.csi_cfg.csi_port * 2 + 1] =
-				channel.target_cfg.input_switch_channel_cfg.hsync_data_reg[1];
-		config.input_switch_cfg.vsync_data_reg |=
-				 (channel.target_cfg.input_switch_channel_cfg.vsync_data_reg & 0x7) << (channel.source_cfg.csi_cfg.csi_port * 3);
-
-		// Other targets are just copied and marked as set.
-		config.target_isp[channel.source_cfg.csi_cfg.csi_port] = channel.target_cfg.target_isp_cfg;
-		config.target_sp[channel.source_cfg.csi_cfg.csi_port] = channel.target_cfg.target_sp_cfg;
-		config.target_strm2mem[channel.source_cfg.csi_cfg.csi_port] = channel.target_cfg.target_strm2mem_cfg;
-		config.target_isp_flags[channel.source_cfg.csi_cfg.csi_port] |= INPUT_SYSTEM_CFG_FLAG_SET;
-		config.target_sp_flags[channel.source_cfg.csi_cfg.csi_port] |= INPUT_SYSTEM_CFG_FLAG_SET;
-		config.target_strm2mem_flags[channel.source_cfg.csi_cfg.csi_port] |= INPUT_SYSTEM_CFG_FLAG_SET;
-
-		config.ch_flags[channel.ch_id] = INPUT_SYSTEM_CFG_FLAG_SET;
-	}
-	return INPUT_SYSTEM_ERR_NO_ERROR;
-}
-
-// Function that partitions input buffer space with determining addresses.
-static input_system_error_t input_buffer_configuration(void)
-{
-	uint32_t current_address    = 0;
-	uint32_t unallocated_memory = IB_CAPACITY_IN_WORDS;
-
-	ib_buffer_t 	candidate_buffer_acq  = IB_BUFFER_NULL;
-	uint32_t size_requested;
-	input_system_config_flags_t 	acq_already_specified = INPUT_SYSTEM_CFG_FLAG_RESET;
-	input_system_csi_port_t port;
-	for (port = INPUT_SYSTEM_PORT_A; port < N_INPUT_SYSTEM_PORTS; port++) {
-
-		csi_cfg_t source = config.csi_value[port];//.csi_cfg;
-
-		if ( config.csi_flags[port] & INPUT_SYSTEM_CFG_FLAG_SET) {
-
-			// Check and set csi buffer in input buffer.
-			switch (source.buffering_mode) {
-				case INPUT_SYSTEM_FIFO_CAPTURE :
-				case INPUT_SYSTEM_XMEM_ACQUIRE :
-					config.csi_buffer_flags[port] = INPUT_SYSTEM_CFG_FLAG_BLOCKED; // Well, not used.
-					break;
-
-				case INPUT_SYSTEM_FIFO_CAPTURE_WITH_COUNTING :
-				case INPUT_SYSTEM_SRAM_BUFFERING :
-				case INPUT_SYSTEM_XMEM_BUFFERING :
-				case INPUT_SYSTEM_XMEM_CAPTURE :
-					size_requested = source.csi_buffer.mem_reg_size * source.csi_buffer.nof_mem_regs;
-					if (source.csi_buffer.mem_reg_size > 0
-						&& source.csi_buffer.nof_mem_regs >0
-						&& size_requested <= unallocated_memory
-						) {
-							config.csi_buffer[port].mem_reg_addr = current_address;
-							config.csi_buffer[port].mem_reg_size = source.csi_buffer.mem_reg_size;
-							config.csi_buffer[port].nof_mem_regs = source.csi_buffer.nof_mem_regs;
-							current_address		+= size_requested;
-							unallocated_memory 	-= size_requested;
-							config.csi_buffer_flags[port] = INPUT_SYSTEM_CFG_FLAG_SET;
-					} else {
-							config.csi_buffer_flags[port] |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-							return INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE;
-					}
-					break;
-
-				default :
-					config.csi_buffer_flags[port] |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-					return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-					break;
-			}
-
-			// Check acquisition buffer specified but set it later since it has to be unique.
-			switch (source.buffering_mode) {
-				case INPUT_SYSTEM_FIFO_CAPTURE :
-				case INPUT_SYSTEM_SRAM_BUFFERING :
-				case INPUT_SYSTEM_XMEM_CAPTURE :
-					// Nothing to do.
-					break;
-
-				case INPUT_SYSTEM_FIFO_CAPTURE_WITH_COUNTING :
-				case INPUT_SYSTEM_XMEM_BUFFERING :
-				case INPUT_SYSTEM_XMEM_ACQUIRE :
-					if (acq_already_specified == INPUT_SYSTEM_CFG_FLAG_RESET) {
-						size_requested = source.acquisition_buffer.mem_reg_size
-													* source.acquisition_buffer.nof_mem_regs;
-						if (source.acquisition_buffer.mem_reg_size > 0
-							&& source.acquisition_buffer.nof_mem_regs >0
-							&& size_requested <= unallocated_memory
-							) {
-								candidate_buffer_acq = source.acquisition_buffer;
-								acq_already_specified = INPUT_SYSTEM_CFG_FLAG_SET;
-						}
-					} else {
-						// Check if specified acquisition buffer is the same as specified before.
-						if (source.acquisition_buffer.mem_reg_size != candidate_buffer_acq.mem_reg_size
-							|| source.acquisition_buffer.nof_mem_regs !=  candidate_buffer_acq.nof_mem_regs
-						   ) {
-							config.acquisition_buffer_unique_flags |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-							return INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE;
-						}
-					}
-					break;
-
-				default :
-					return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-					break;
-			}
-		} else {
-			config.csi_buffer_flags[port] = INPUT_SYSTEM_CFG_FLAG_BLOCKED;
-		}
-	} // end of for ( port )
-
-	// Set the acquisition buffer at the end.
-	size_requested = candidate_buffer_acq.mem_reg_size * candidate_buffer_acq.nof_mem_regs;
-	if (acq_already_specified == INPUT_SYSTEM_CFG_FLAG_SET
-		&& size_requested <= unallocated_memory) {
-		config.acquisition_buffer_unique.mem_reg_addr = current_address;
-		config.acquisition_buffer_unique.mem_reg_size = candidate_buffer_acq.mem_reg_size;
-		config.acquisition_buffer_unique.nof_mem_regs = candidate_buffer_acq.nof_mem_regs;
-		current_address		+= size_requested;
-		unallocated_memory 	-= size_requested;
-		config.acquisition_buffer_unique_flags = INPUT_SYSTEM_CFG_FLAG_SET;
-
-		assert(current_address <= IB_CAPACITY_IN_WORDS);
-	}
-
-	return INPUT_SYSTEM_ERR_NO_ERROR;
-}
-
-static void capture_unit_configure(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	const ib_buffer_t* const		cfg)
-{
-	assert(ID < N_INPUT_SYSTEM_ID);
-	assert(/*(sub_id >= CAPTURE_UNIT0_ID) &&*/ (sub_id <= CAPTURE_UNIT2_ID)); // Commented part is always true.
-	assert(cfg != NULL);
-
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		CAPT_START_ADDR_REG_ID,
-		cfg->mem_reg_addr);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		CAPT_MEM_REGION_SIZE_REG_ID,
-		cfg->mem_reg_size);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		CAPT_NUM_MEM_REGIONS_REG_ID,
-		cfg->nof_mem_regs);
-
-	return;
-}
-
-
-static void acquisition_unit_configure(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	const ib_buffer_t* const		cfg)
-{
-	assert(ID < N_INPUT_SYSTEM_ID);
-	assert(sub_id == ACQUISITION_UNIT0_ID);
-	assert(cfg != NULL);
-
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ACQ_START_ADDR_REG_ID,
-		cfg->mem_reg_addr);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ACQ_NUM_MEM_REGIONS_REG_ID,
-		cfg->nof_mem_regs);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ACQ_MEM_REGION_SIZE_REG_ID,
-		cfg->mem_reg_size);
-
-	return;
-}
-
-
-static void ctrl_unit_configure(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_id,
-	const ctrl_unit_cfg_t* const		cfg)
-{
-	assert(ID < N_INPUT_SYSTEM_ID);
-	assert(sub_id == CTRL_UNIT0_ID);
-	assert(cfg != NULL);
-
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_START_ADDR_A_REG_ID,
-		cfg->buffer_mipi[CAPTURE_UNIT0_ID].mem_reg_addr);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_MEM_REGION_SIZE_A_REG_ID,
-		cfg->buffer_mipi[CAPTURE_UNIT0_ID].mem_reg_size);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_NUM_MEM_REGIONS_A_REG_ID,
-		cfg->buffer_mipi[CAPTURE_UNIT0_ID].nof_mem_regs);
-
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_START_ADDR_B_REG_ID,
-		cfg->buffer_mipi[CAPTURE_UNIT1_ID].mem_reg_addr);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_MEM_REGION_SIZE_B_REG_ID,
-		cfg->buffer_mipi[CAPTURE_UNIT1_ID].mem_reg_size);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_NUM_MEM_REGIONS_B_REG_ID,
-		cfg->buffer_mipi[CAPTURE_UNIT1_ID].nof_mem_regs);
-
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_START_ADDR_C_REG_ID,
-		cfg->buffer_mipi[CAPTURE_UNIT2_ID].mem_reg_addr);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_MEM_REGION_SIZE_C_REG_ID,
-		cfg->buffer_mipi[CAPTURE_UNIT2_ID].mem_reg_size);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_CAPT_NUM_MEM_REGIONS_C_REG_ID,
-		cfg->buffer_mipi[CAPTURE_UNIT2_ID].nof_mem_regs);
-
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_ACQ_START_ADDR_REG_ID,
-		cfg->buffer_acquire[ACQUISITION_UNIT0_ID - ACQUISITION_UNIT0_ID].mem_reg_addr);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_ACQ_MEM_REGION_SIZE_REG_ID,
-		cfg->buffer_acquire[ACQUISITION_UNIT0_ID - ACQUISITION_UNIT0_ID].mem_reg_size);
-	input_system_sub_system_reg_store(ID,
-		sub_id,
-		ISYS_CTRL_ACQ_NUM_MEM_REGIONS_REG_ID,
-		cfg->buffer_acquire[ACQUISITION_UNIT0_ID - ACQUISITION_UNIT0_ID].nof_mem_regs);
-	input_system_sub_system_reg_store(ID,
-                sub_id,
-                ISYS_CTRL_CAPT_RESERVE_ONE_MEM_REGION_REG_ID,
-		0);
-	return;
-}
-
-static void input_system_network_configure(
-	const input_system_ID_t				ID,
-	const input_system_network_cfg_t * const 	cfg)
-{
-	uint32_t sub_id;
-
-	assert(ID < N_INPUT_SYSTEM_ID);
-	assert(cfg != NULL);
-
-	// Set all 3 multicasts.
-	input_system_sub_system_reg_store(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MULTICAST_A_IDX,
-		cfg->multicast_cfg[CAPTURE_UNIT0_ID]);
-	input_system_sub_system_reg_store(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MULTICAST_B_IDX,
-		cfg->multicast_cfg[CAPTURE_UNIT1_ID]);
-	input_system_sub_system_reg_store(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MULTICAST_C_IDX,
-		cfg->multicast_cfg[CAPTURE_UNIT2_ID]);
-
-	// Set stream mux.
-	input_system_sub_system_reg_store(ID,
-		GPREGS_UNIT0_ID,
-		HIVE_ISYS_GPREG_MUX_IDX,
-		cfg->mux_cfg);
-
-	// Set capture units.
-	for (sub_id = CAPTURE_UNIT0_ID; sub_id < CAPTURE_UNIT0_ID + N_CAPTURE_UNIT_ID; sub_id++) {
-		capture_unit_configure(ID,
-			sub_id,
-			&(cfg->ctrl_unit_cfg[ID].buffer_mipi[sub_id - CAPTURE_UNIT0_ID]));
-	}
-
-	// Set acquisition units.
-	for (sub_id = ACQUISITION_UNIT0_ID; sub_id < ACQUISITION_UNIT0_ID + N_ACQUISITION_UNIT_ID; sub_id++) {
-		acquisition_unit_configure(ID,
-			sub_id,
-			&(cfg->ctrl_unit_cfg[sub_id - ACQUISITION_UNIT0_ID].buffer_acquire[sub_id - ACQUISITION_UNIT0_ID]));
-	}
-
-	// No DMA configuration needed. Ctrl_unit will fully control it.
-
-	// Set controller units.
-	for (sub_id = CTRL_UNIT0_ID; sub_id < CTRL_UNIT0_ID + N_CTRL_UNIT_ID; sub_id++) {
-		ctrl_unit_configure(ID,
-			sub_id,
-			&(cfg->ctrl_unit_cfg[sub_id - CTRL_UNIT0_ID]));
-	}
-
-	return;
-}
-
-static input_system_error_t configuration_to_registers(void)
-{
-	input_system_network_cfg_t input_system_network_cfg;
-	int i;
-
-	assert(config.source_type_flags & INPUT_SYSTEM_CFG_FLAG_SET);
-
-	switch (config.source_type) {
-		case INPUT_SYSTEM_SOURCE_SENSOR :
-
-			// Determine stream multicasts setting based on the mode of csi_cfg_t.
-			// AM: This should be moved towards earlier function call, e.g. in
-			// the commit function.
-			for (i = MIPI_PORT0_ID; i < N_MIPI_PORT_ID; i++) {
-				if (config.csi_flags[i] & INPUT_SYSTEM_CFG_FLAG_SET) {
-
-					switch (config.csi_value[i].buffering_mode) {
-
-						case INPUT_SYSTEM_FIFO_CAPTURE:
-							config.multicast[i] = INPUT_SYSTEM_CSI_BACKEND;
-							break;
-
-						case INPUT_SYSTEM_XMEM_CAPTURE:
-						case INPUT_SYSTEM_SRAM_BUFFERING:
-						case INPUT_SYSTEM_XMEM_BUFFERING:
-							config.multicast[i] = INPUT_SYSTEM_INPUT_BUFFER;
-							break;
-
-						case INPUT_SYSTEM_FIFO_CAPTURE_WITH_COUNTING:
-							config.multicast[i] = INPUT_SYSTEM_MULTICAST;
-							break;
-
-						case INPUT_SYSTEM_XMEM_ACQUIRE:
-							config.multicast[i] = INPUT_SYSTEM_DISCARD_ALL;
-							break;
-
-						default:
-							config.multicast[i] = INPUT_SYSTEM_DISCARD_ALL;
-							return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-							//break;
-					}
-				} else {
-					config.multicast[i]= INPUT_SYSTEM_DISCARD_ALL;
-				}
-
-				input_system_network_cfg.multicast_cfg[i] = config.multicast[i];
-
-			} // for
-
-			input_system_network_cfg.mux_cfg = config.multiplexer;
-
-			input_system_network_cfg.ctrl_unit_cfg[CTRL_UNIT0_ID - CTRL_UNIT0_ID].buffer_mipi[CAPTURE_UNIT0_ID] = config.csi_buffer[MIPI_PORT0_ID];
-			input_system_network_cfg.ctrl_unit_cfg[CTRL_UNIT0_ID - CTRL_UNIT0_ID].buffer_mipi[CAPTURE_UNIT1_ID] = config.csi_buffer[MIPI_PORT1_ID];
-			input_system_network_cfg.ctrl_unit_cfg[CTRL_UNIT0_ID - CTRL_UNIT0_ID].buffer_mipi[CAPTURE_UNIT2_ID] = config.csi_buffer[MIPI_PORT2_ID];
-			input_system_network_cfg.ctrl_unit_cfg[CTRL_UNIT0_ID - CTRL_UNIT0_ID].buffer_acquire[ACQUISITION_UNIT0_ID - ACQUISITION_UNIT0_ID] =
-					config.acquisition_buffer_unique;
-
-			// First set input network around CSI receiver.
-			input_system_network_configure(INPUT_SYSTEM0_ID, &input_system_network_cfg);
-
-			// Set the CSI receiver.
-			//...
-			break;
-
-		case INPUT_SYSTEM_SOURCE_TPG :
-
-			break;
-
-		case INPUT_SYSTEM_SOURCE_PRBS :
-
-			break;
-
-		case INPUT_SYSTEM_SOURCE_FIFO :
-			break;
-
-		default :
-			return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-			break;
-
-	} // end of switch (source_type)
-
-	// Set input selector.
-	input_selector_cfg_for_sensor(INPUT_SYSTEM0_ID);
-
-	// Set input switch.
-	input_switch_cfg(INPUT_SYSTEM0_ID, &config.input_switch_cfg);
-
-	// Set input formatters.
-	// AM: IF are set dynamically.
-	return INPUT_SYSTEM_ERR_NO_ERROR;
-}
-
-
-// Function that applies the whole configuration.
-input_system_error_t input_system_configuration_commit(void)
-{
-	// The last configuration step is to configure the input buffer.
-	input_system_error_t error = input_buffer_configuration();
-	if (error != INPUT_SYSTEM_ERR_NO_ERROR) {
-		return error;
-	}
-
-	// Translate the whole configuration into registers.
-	error = configuration_to_registers();
-	if (error != INPUT_SYSTEM_ERR_NO_ERROR) {
-		return error;
-	}
-
-	// Translate the whole configuration into ctrl commands etc.
-
-	return INPUT_SYSTEM_ERR_NO_ERROR;
-}
-
-
-
-// FIFO
-
-input_system_error_t	input_system_csi_fifo_channel_cfg(
-		uint32_t		ch_id,
-		input_system_csi_port_t	port,
-		backend_channel_cfg_t	backend_ch,
-		target_cfg2400_t	target
-)
-{
-	channel_cfg_t channel;
-
-	channel.ch_id 	= ch_id;
-	channel.backend_ch 	= backend_ch;
-	channel.source_type = INPUT_SYSTEM_SOURCE_SENSOR;
-	//channel.source
-	channel.source_cfg.csi_cfg.csi_port			= port;
-	channel.source_cfg.csi_cfg.buffering_mode	= INPUT_SYSTEM_FIFO_CAPTURE;
-	channel.source_cfg.csi_cfg.csi_buffer 			= IB_BUFFER_NULL;
-	channel.source_cfg.csi_cfg.acquisition_buffer	= IB_BUFFER_NULL;
-	channel.source_cfg.csi_cfg.nof_xmem_buffers	= 0;
-
-	channel.target_cfg	= target;
-	return input_system_configure_channel(channel);
-}
-
-
-input_system_error_t	input_system_csi_fifo_channel_with_counting_cfg(
-		uint32_t				ch_id,
-		uint32_t				nof_frames,
-		input_system_csi_port_t			port,
-		backend_channel_cfg_t			backend_ch,
-		uint32_t				csi_mem_reg_size,
-		uint32_t				csi_nof_mem_regs,
-		target_cfg2400_t			target
-)
-{
-	channel_cfg_t channel;
-
-	channel.ch_id 	= ch_id;
-	channel.backend_ch 	= backend_ch;
-	channel.source_type		= INPUT_SYSTEM_SOURCE_SENSOR;
-	//channel.source
-	channel.source_cfg.csi_cfg.csi_port			= port;
-	channel.source_cfg.csi_cfg.buffering_mode	= INPUT_SYSTEM_FIFO_CAPTURE_WITH_COUNTING;
-	channel.source_cfg.csi_cfg.csi_buffer.mem_reg_size		= csi_mem_reg_size;
-	channel.source_cfg.csi_cfg.csi_buffer.nof_mem_regs 		= csi_nof_mem_regs;
-	channel.source_cfg.csi_cfg.csi_buffer.mem_reg_addr		= 0;
-	channel.source_cfg.csi_cfg.acquisition_buffer			= IB_BUFFER_NULL;
-	channel.source_cfg.csi_cfg.nof_xmem_buffers	= nof_frames;
-
-	channel.target_cfg	= target;
-	return input_system_configure_channel(channel);
-}
-
-
-// SRAM
-
-input_system_error_t	input_system_csi_sram_channel_cfg(
-		uint32_t				ch_id,
-		input_system_csi_port_t			port,
-		backend_channel_cfg_t			backend_ch,
-		uint32_t				csi_mem_reg_size,
-		uint32_t				csi_nof_mem_regs,
-	//	uint32_t				acq_mem_reg_size,
-	//	uint32_t				acq_nof_mem_regs,
-		target_cfg2400_t 			target
-)
-{
-	channel_cfg_t channel;
-
-	channel.ch_id 	= ch_id;
-	channel.backend_ch 	= backend_ch;
-	channel.source_type		= INPUT_SYSTEM_SOURCE_SENSOR;
-	//channel.source
-	channel.source_cfg.csi_cfg.csi_port			= port;
-	channel.source_cfg.csi_cfg.buffering_mode	= INPUT_SYSTEM_SRAM_BUFFERING;
-	channel.source_cfg.csi_cfg.csi_buffer.mem_reg_size		= csi_mem_reg_size;
-	channel.source_cfg.csi_cfg.csi_buffer.nof_mem_regs 		= csi_nof_mem_regs;
-	channel.source_cfg.csi_cfg.csi_buffer.mem_reg_addr		= 0;
-	channel.source_cfg.csi_cfg.acquisition_buffer			= IB_BUFFER_NULL;
-	channel.source_cfg.csi_cfg.nof_xmem_buffers	= 0;
-
-	channel.target_cfg	= target;
-	return input_system_configure_channel(channel);
-}
-
-
-//XMEM
-
-// Collects all parameters and puts them in channel_cfg_t.
-input_system_error_t	input_system_csi_xmem_channel_cfg(
-		uint32_t 				ch_id,
-		input_system_csi_port_t			port,
-		backend_channel_cfg_t			backend_ch,
-		uint32_t 				csi_mem_reg_size,
-		uint32_t 				csi_nof_mem_regs,
-		uint32_t 				acq_mem_reg_size,
-		uint32_t 				acq_nof_mem_regs,
-		target_cfg2400_t 			target,
-		uint32_t 				nof_xmem_buffers
-)
-{
-	channel_cfg_t channel;
-
-	channel.ch_id 	= ch_id;
-	channel.backend_ch 	= backend_ch;
-	channel.source_type		= INPUT_SYSTEM_SOURCE_SENSOR;
-	//channel.source
-	channel.source_cfg.csi_cfg.csi_port			= port;
-	channel.source_cfg.csi_cfg.buffering_mode	= INPUT_SYSTEM_XMEM_BUFFERING;
-	channel.source_cfg.csi_cfg.csi_buffer.mem_reg_size		= csi_mem_reg_size;
-	channel.source_cfg.csi_cfg.csi_buffer.nof_mem_regs 		= csi_nof_mem_regs;
-	channel.source_cfg.csi_cfg.csi_buffer.mem_reg_addr		= 0;
-	channel.source_cfg.csi_cfg.acquisition_buffer.mem_reg_size	= acq_mem_reg_size;
-	channel.source_cfg.csi_cfg.acquisition_buffer.nof_mem_regs 	= acq_nof_mem_regs;
-	channel.source_cfg.csi_cfg.acquisition_buffer.mem_reg_addr	= 0;
-	channel.source_cfg.csi_cfg.nof_xmem_buffers	= nof_xmem_buffers;
-
-	channel.target_cfg	= target;
-	return input_system_configure_channel(channel);
-}
-
-
-
-
-input_system_error_t	input_system_csi_xmem_acquire_only_channel_cfg(
-		uint32_t 				ch_id,
-		uint32_t 				nof_frames,
-		input_system_csi_port_t			port,
-		backend_channel_cfg_t			backend_ch,
-		uint32_t 				acq_mem_reg_size,
-		uint32_t 				acq_nof_mem_regs,
-		target_cfg2400_t 			target)
-{
-	channel_cfg_t channel;
-
-	channel.ch_id 	= ch_id;
-	channel.backend_ch 	= backend_ch;
-	channel.source_type		= INPUT_SYSTEM_SOURCE_SENSOR;
-	//channel.source
-	channel.source_cfg.csi_cfg.csi_port			= port;
-	channel.source_cfg.csi_cfg.buffering_mode	= INPUT_SYSTEM_XMEM_ACQUIRE;
-	channel.source_cfg.csi_cfg.csi_buffer		= IB_BUFFER_NULL;
-	channel.source_cfg.csi_cfg.acquisition_buffer.mem_reg_size	= acq_mem_reg_size;
-	channel.source_cfg.csi_cfg.acquisition_buffer.nof_mem_regs 	= acq_nof_mem_regs;
-	channel.source_cfg.csi_cfg.acquisition_buffer.mem_reg_addr	= 0;
-	channel.source_cfg.csi_cfg.nof_xmem_buffers	= nof_frames;
-
-	channel.target_cfg	= target;
-	return input_system_configure_channel(channel);
-}
-
-
-input_system_error_t	input_system_csi_xmem_capture_only_channel_cfg(
-		uint32_t 				ch_id,
-		uint32_t 				nof_frames,
-		input_system_csi_port_t			port,
-		uint32_t 				csi_mem_reg_size,
-		uint32_t 				csi_nof_mem_regs,
-		uint32_t 				acq_mem_reg_size,
-		uint32_t 				acq_nof_mem_regs,
-		target_cfg2400_t 			target)
-{
-	channel_cfg_t channel;
-
-	channel.ch_id 	= ch_id;
-	//channel.backend_ch 	= backend_ch;
-	channel.source_type		= INPUT_SYSTEM_SOURCE_SENSOR;
-	//channel.source
-	channel.source_cfg.csi_cfg.csi_port			= port;
-	//channel.source_cfg.csi_cfg.backend_ch 		= backend_ch;
-	channel.source_cfg.csi_cfg.buffering_mode	= INPUT_SYSTEM_XMEM_CAPTURE;
-	channel.source_cfg.csi_cfg.csi_buffer.mem_reg_size		= csi_mem_reg_size;
-	channel.source_cfg.csi_cfg.csi_buffer.nof_mem_regs 		= csi_nof_mem_regs;
-	channel.source_cfg.csi_cfg.csi_buffer.mem_reg_addr		= 0;
-	channel.source_cfg.csi_cfg.acquisition_buffer.mem_reg_size	= acq_mem_reg_size;
-	channel.source_cfg.csi_cfg.acquisition_buffer.nof_mem_regs 	= acq_nof_mem_regs;
-	channel.source_cfg.csi_cfg.acquisition_buffer.mem_reg_addr	= 0;
-	channel.source_cfg.csi_cfg.nof_xmem_buffers	= nof_frames;
-
-	channel.target_cfg	= target;
-	return input_system_configure_channel(channel);
-}
-
-
-
-// Non - CSI
-
-input_system_error_t	input_system_prbs_channel_cfg(
-		uint32_t 		ch_id,
-		uint32_t		nof_frames,//not used yet
-		uint32_t		seed,
-		uint32_t		sync_gen_width,
-		uint32_t		sync_gen_height,
-		uint32_t		sync_gen_hblank_cycles,
-		uint32_t		sync_gen_vblank_cycles,
-		target_cfg2400_t	target
-)
-{
-	channel_cfg_t channel;
-
-	(void)nof_frames;
-
-	channel.ch_id 	= ch_id;
-	channel.source_type= INPUT_SYSTEM_SOURCE_PRBS;
-
-	channel.source_cfg.prbs_cfg.seed = seed;
-	channel.source_cfg.prbs_cfg.sync_gen_cfg.width         	= sync_gen_width;
-	channel.source_cfg.prbs_cfg.sync_gen_cfg.height        	= sync_gen_height;
-	channel.source_cfg.prbs_cfg.sync_gen_cfg.hblank_cycles	= sync_gen_hblank_cycles;
-	channel.source_cfg.prbs_cfg.sync_gen_cfg.vblank_cycles 	= sync_gen_vblank_cycles;
-
-	channel.target_cfg	= target;
-
-	return input_system_configure_channel(channel);
-}
-
-
-
-input_system_error_t	input_system_tpg_channel_cfg(
-		uint32_t 		ch_id,
-		uint32_t 		nof_frames,//not used yet
-		uint32_t		x_mask,
-		uint32_t		y_mask,
-		uint32_t		x_delta,
-		uint32_t		y_delta,
-		uint32_t		xy_mask,
-		uint32_t		sync_gen_width,
-		uint32_t		sync_gen_height,
-		uint32_t		sync_gen_hblank_cycles,
-		uint32_t		sync_gen_vblank_cycles,
-		target_cfg2400_t	target
-)
-{
-	channel_cfg_t channel;
-
-	(void)nof_frames;
-
-	channel.ch_id 	= ch_id;
-	channel.source_type		= INPUT_SYSTEM_SOURCE_TPG;
-
-	channel.source_cfg.tpg_cfg.x_mask	= x_mask;
-	channel.source_cfg.tpg_cfg.y_mask	= y_mask;
-	channel.source_cfg.tpg_cfg.x_delta	= x_delta;
-	channel.source_cfg.tpg_cfg.y_delta	= y_delta;
-	channel.source_cfg.tpg_cfg.xy_mask	= xy_mask;
-	channel.source_cfg.tpg_cfg.sync_gen_cfg.width         	= sync_gen_width;
-	channel.source_cfg.tpg_cfg.sync_gen_cfg.height        	= sync_gen_height;
-	channel.source_cfg.tpg_cfg.sync_gen_cfg.hblank_cycles	= sync_gen_hblank_cycles;
-	channel.source_cfg.tpg_cfg.sync_gen_cfg.vblank_cycles 	= sync_gen_vblank_cycles;
-
-	channel.target_cfg	= target;
-	return input_system_configure_channel(channel);
-}
-
-// MW: Don't use system specific names, (even in system specific files) "cfg2400" -> cfg
-input_system_error_t	input_system_gpfifo_channel_cfg(
-		uint32_t 		ch_id,
-		uint32_t 		nof_frames, //not used yet
-		target_cfg2400_t	target)
-{
-	channel_cfg_t channel;
-
-	(void)nof_frames;
-
-	channel.ch_id 	= ch_id;
-	channel.source_type	= INPUT_SYSTEM_SOURCE_FIFO;
-
-	channel.target_cfg	= target;
-	return input_system_configure_channel(channel);
-}
-
-///////////////////////////////////////////////////////////////////////////
-//
-// Private specialized functions for channel setting.
-//
-///////////////////////////////////////////////////////////////////////////
-
-// Fills the parameters to config.csi_value[port]
-static input_system_error_t input_system_configure_channel_sensor(
-	const channel_cfg_t channel)
-{
-	const uint32_t port = channel.source_cfg.csi_cfg.csi_port;
-	input_system_error_t status = INPUT_SYSTEM_ERR_NO_ERROR;
-
-	input_system_multiplex_t mux;
-
-	if (port >= N_INPUT_SYSTEM_PORTS)
-		return INPUT_SYSTEM_ERR_GENERIC;
-
-	//check if port > N_INPUT_SYSTEM_MULTIPLEX
-
-	status = set_source_type(&(config.source_type), channel.source_type, &config.source_type_flags);
-	if (status != INPUT_SYSTEM_ERR_NO_ERROR) return status;
-
-	// Check for conflicts on source (implicitly on multicast, capture unit and input buffer).
-
-	status = set_csi_cfg(&(config.csi_value[port]), &channel.source_cfg.csi_cfg, &(config.csi_flags[port]));
-	if (status != INPUT_SYSTEM_ERR_NO_ERROR) return status;
-
-
-	switch (channel.source_cfg.csi_cfg.buffering_mode){
-		case INPUT_SYSTEM_FIFO_CAPTURE:
-
-			// Check for conflicts on mux.
-			mux = INPUT_SYSTEM_MIPI_PORT0 + port;
-			status = input_system_multiplexer_cfg(&config.multiplexer, mux, &config.multiplexer_flags);
-			if (status != INPUT_SYSTEM_ERR_NO_ERROR) return status;
-			config.multicast[port] = INPUT_SYSTEM_CSI_BACKEND;
-
-			// Shared resource, so it should be blocked.
-			//config.mux_flags |= INPUT_SYSTEM_CFG_FLAG_BLOCKED;
-			//config.csi_buffer_flags[port] |= INPUT_SYSTEM_CFG_FLAG_BLOCKED;
-			//config.acquisition_buffer_unique_flags |= INPUT_SYSTEM_CFG_FLAG_BLOCKED;
-
-			break;
-		case INPUT_SYSTEM_SRAM_BUFFERING :
-
-			// Check for conflicts on mux.
-			mux = INPUT_SYSTEM_ACQUISITION_UNIT;
-			status = input_system_multiplexer_cfg(&config.multiplexer, mux, &config.multiplexer_flags);
-			if (status != INPUT_SYSTEM_ERR_NO_ERROR) return status;
-			config.multicast[port] = INPUT_SYSTEM_INPUT_BUFFER;
-
-			// Shared resource, so it should be blocked.
-			//config.mux_flags |= INPUT_SYSTEM_CFG_FLAG_BLOCKED;
-			//config.csi_buffer_flags[port] |= INPUT_SYSTEM_CFG_FLAG_BLOCKED;
-			//config.acquisition_buffer_unique_flags |= INPUT_SYSTEM_CFG_FLAG_BLOCKED;
-
-			break;
-		case INPUT_SYSTEM_XMEM_BUFFERING :
-
-			// Check for conflicts on mux.
-			mux = INPUT_SYSTEM_ACQUISITION_UNIT;
-			status = input_system_multiplexer_cfg(&config.multiplexer, mux, &config.multiplexer_flags);
-			if (status != INPUT_SYSTEM_ERR_NO_ERROR) return status;
-			config.multicast[port] = INPUT_SYSTEM_INPUT_BUFFER;
-
-			// Shared resource, so it should be blocked.
-			//config.mux_flags |= INPUT_SYSTEM_CFG_FLAG_BLOCKED;
-			//config.csi_buffer_flags[port] |= INPUT_SYSTEM_CFG_FLAG_BLOCKED;
-			//config.acquisition_buffer_unique_flags |= INPUT_SYSTEM_CFG_FLAG_BLOCKED;
-
-			break;
-		case INPUT_SYSTEM_FIFO_CAPTURE_WITH_COUNTING :
-			return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-			break;
-		case INPUT_SYSTEM_XMEM_CAPTURE :
-			return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-			break;
-		case INPUT_SYSTEM_XMEM_ACQUIRE :
-			return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-			break;
-		default :
-			return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-			break;
-	}
-	return INPUT_SYSTEM_ERR_NO_ERROR;
-}
-
-// Test flags and set structure.
-static input_system_error_t set_source_type(
-		input_system_source_t * const 			lhs,
-		const input_system_source_t 			rhs,
-		input_system_config_flags_t * const	 	flags)
-{
-	// MW: Not enough asserts
-	assert(lhs != NULL);
-	assert(flags != NULL);
-
-	if ((*flags) & INPUT_SYSTEM_CFG_FLAG_BLOCKED) {
-		*flags |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-		return INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE;
-	}
-
-	if ((*flags) & INPUT_SYSTEM_CFG_FLAG_SET) {
-		// Check for consistency with already set value.
-		if ((*lhs) == (rhs)) {
-			return INPUT_SYSTEM_ERR_NO_ERROR;
-		}
-		else {
-			*flags |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-			return INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE;
-		}
-	}
-	// Check the value (individually).
-	if (rhs >= N_INPUT_SYSTEM_SOURCE) {
-		*flags |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-		return INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE;
-	}
-	// Set the value.
-	*lhs = rhs;
-
-	*flags |= INPUT_SYSTEM_CFG_FLAG_SET;
-	return INPUT_SYSTEM_ERR_NO_ERROR;
-}
-
-
-// Test flags and set structure.
-static input_system_error_t set_csi_cfg(
-		csi_cfg_t* const 			lhs,
-		const csi_cfg_t* const 			rhs,
-		input_system_config_flags_t * const 	flags)
-{
-	uint32_t memory_required;
-	uint32_t acq_memory_required;
-
-	assert(lhs != NULL);
-	assert(flags != NULL);
-
-	if ((*flags) & INPUT_SYSTEM_CFG_FLAG_BLOCKED) {
-		*flags |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-		return INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE;
-	}
-
-	if (*flags & INPUT_SYSTEM_CFG_FLAG_SET) {
-		// check for consistency with already set value.
-		if (/*lhs->backend_ch == rhs.backend_ch
-			&&*/ lhs->buffering_mode == rhs->buffering_mode
-			&& lhs->csi_buffer.mem_reg_size == rhs->csi_buffer.mem_reg_size
-			&& lhs->csi_buffer.nof_mem_regs  == rhs->csi_buffer.nof_mem_regs
-			&& lhs->acquisition_buffer.mem_reg_size == rhs->acquisition_buffer.mem_reg_size
-			&& lhs->acquisition_buffer.nof_mem_regs  == rhs->acquisition_buffer.nof_mem_regs
-			&& lhs->nof_xmem_buffers  == rhs->nof_xmem_buffers
-			) {
-			return INPUT_SYSTEM_ERR_NO_ERROR;
-		}
-		else {
-			*flags |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-			return INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE;
-		}
-	}
-	// Check the value (individually).
-	// no check for backend_ch
-	// no check for nof_xmem_buffers
-	memory_required = rhs->csi_buffer.mem_reg_size * rhs->csi_buffer.nof_mem_regs;
-	acq_memory_required = rhs->acquisition_buffer.mem_reg_size * rhs->acquisition_buffer.nof_mem_regs;
-	if (rhs->buffering_mode >= N_INPUT_SYSTEM_BUFFERING_MODE
-		||
-	// Check if required memory is available in input buffer (SRAM).
-		(memory_required + acq_memory_required )> config.unallocated_ib_mem_words
-
-		) {
-		*flags |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-		return INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE;
-	}
-	// Set the value.
-	//lhs[port]->backend_ch 		= rhs.backend_ch;
-	lhs->buffering_mode 	= rhs->buffering_mode;
-	lhs->nof_xmem_buffers = rhs->nof_xmem_buffers;
-
-	lhs->csi_buffer.mem_reg_size = rhs->csi_buffer.mem_reg_size;
-	lhs->csi_buffer.nof_mem_regs  = rhs->csi_buffer.nof_mem_regs;
-	lhs->acquisition_buffer.mem_reg_size = rhs->acquisition_buffer.mem_reg_size;
-	lhs->acquisition_buffer.nof_mem_regs  = rhs->acquisition_buffer.nof_mem_regs;
-    // ALX: NB: Here we just set buffer parameters, but still not allocate it
-	// (no addresses determined). That will be done during commit.
-
-	//  FIXIT:	acq_memory_required is not deducted, since it can be allocated multiple times.
-	config.unallocated_ib_mem_words -= memory_required;
-//assert(config.unallocated_ib_mem_words >=0);
-	*flags |= INPUT_SYSTEM_CFG_FLAG_SET;
-	return INPUT_SYSTEM_ERR_NO_ERROR;
-}
-
-
-// Test flags and set structure.
-static input_system_error_t input_system_multiplexer_cfg(
-	input_system_multiplex_t* const		lhs,
-	const input_system_multiplex_t		rhs,
-	input_system_config_flags_t* const	flags)
-{
-	assert(lhs != NULL);
-	assert(flags != NULL);
-
-	if ((*flags) & INPUT_SYSTEM_CFG_FLAG_BLOCKED) {
-		*flags |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-		return INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE;
-	}
-
-	if ((*flags) & INPUT_SYSTEM_CFG_FLAG_SET) {
-		// Check for consistency with already set value.
-		if ((*lhs) == (rhs)) {
-			return INPUT_SYSTEM_ERR_NO_ERROR;
-		}
-		else {
-			*flags |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-			return INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE;
-		}
-	}
-	// Check the value (individually).
-	if (rhs >= N_INPUT_SYSTEM_MULTIPLEX) {
-		*flags |= INPUT_SYSTEM_CFG_FLAG_CONFLICT;
-		return INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED;
-	}
-	// Set the value.
-	*lhs = rhs;
-
-	*flags |= INPUT_SYSTEM_CFG_FLAG_SET;
-	return INPUT_SYSTEM_ERR_NO_ERROR;
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system_local.h
deleted file mode 100644
index bf9230fd08f2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system_local.h
+++ /dev/null
@@ -1,533 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_SYSTEM_LOCAL_H_INCLUDED__
-#define __INPUT_SYSTEM_LOCAL_H_INCLUDED__
-
-#include <type_support.h>
-
-#include "input_system_global.h"
-
-#include "input_system_defs.h"		/* HIVE_ISYS_GPREG_MULTICAST_A_IDX,... */
-#include "css_receiver_2400_defs.h"	/* _HRT_CSS_RECEIVER_2400_TWO_PIXEL_EN_REG_IDX, _HRT_CSS_RECEIVER_2400_CSI2_FUNC_PROG_REG_IDX,... */
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM)
-#include "isp_capture_defs.h"
-#elif defined(IS_ISP_2401_MAMOIADA_SYSTEM)
-/* Same name, but keep the distinction,it is a different device */
-#include "isp_capture_defs.h"
-#else
-#error "input_system_local.h: 2400_SYSTEM must be one of {2400, 2401 }"
-#endif
-#include "isp_acquisition_defs.h"
-#include "input_system_ctrl_defs.h"
-
-
-typedef enum {
-	INPUT_SYSTEM_ERR_NO_ERROR = 0,
-	INPUT_SYSTEM_ERR_GENERIC,
-	INPUT_SYSTEM_ERR_CHANNEL_ALREADY_SET,
-	INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE,
-	INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED,
-	N_INPUT_SYSTEM_ERR
-} input_system_error_t;
-
-typedef enum {
-	INPUT_SYSTEM_PORT_A = 0,
-	INPUT_SYSTEM_PORT_B,
-	INPUT_SYSTEM_PORT_C,
-	N_INPUT_SYSTEM_PORTS
-} input_system_csi_port_t;
-
-typedef struct ctrl_unit_cfg_s			ctrl_unit_cfg_t;
-typedef struct input_system_network_cfg_s	input_system_network_cfg_t;
-typedef struct target_cfg2400_s 		target_cfg2400_t;
-typedef struct channel_cfg_s 			channel_cfg_t;
-typedef struct backend_channel_cfg_s 		backend_channel_cfg_t;
-typedef struct input_system_cfg2400_s 		input_system_cfg2400_t;
-typedef struct mipi_port_state_s		mipi_port_state_t;
-typedef struct rx_channel_state_s		rx_channel_state_t;
-typedef struct input_switch_cfg_channel_s 	input_switch_cfg_channel_t;
-typedef struct input_switch_cfg_s 		input_switch_cfg_t;
-
-struct ctrl_unit_cfg_s {
-	ib_buffer_t		buffer_mipi[N_CAPTURE_UNIT_ID];
-	ib_buffer_t		buffer_acquire[N_ACQUISITION_UNIT_ID];
-};
-
-struct input_system_network_cfg_s {
-	input_system_connection_t	multicast_cfg[N_CAPTURE_UNIT_ID];
-	input_system_multiplex_t	mux_cfg;
-	ctrl_unit_cfg_t				ctrl_unit_cfg[N_CTRL_UNIT_ID];
-};
-
-typedef struct {
-// TBD.
-	uint32_t 	dummy_parameter;
-} target_isp_cfg_t;
-
-
-typedef struct {
-// TBD.
-	uint32_t 	dummy_parameter;
-} target_sp_cfg_t;
-
-
-typedef struct {
-// TBD.
-	uint32_t 	dummy_parameter;
-} target_strm2mem_cfg_t;
-
-struct input_switch_cfg_channel_s {
-	uint32_t hsync_data_reg[2];
-	uint32_t vsync_data_reg;
-};
-
-struct target_cfg2400_s {
-	input_switch_cfg_channel_t 		input_switch_channel_cfg;
-	target_isp_cfg_t	target_isp_cfg;
-	target_sp_cfg_t		target_sp_cfg;
-	target_strm2mem_cfg_t	target_strm2mem_cfg;
-};
-
-struct backend_channel_cfg_s {
-	uint32_t	fmt_control_word_1; // Format config.
-	uint32_t	fmt_control_word_2;
-	uint32_t	no_side_band;
-};
-
-typedef union  {
-	csi_cfg_t	csi_cfg;
-	tpg_cfg_t	tpg_cfg;
-	prbs_cfg_t	prbs_cfg;
-	gpfifo_cfg_t	gpfifo_cfg;
-} source_cfg_t;
-
-
-struct input_switch_cfg_s {
-	uint32_t hsync_data_reg[N_RX_CHANNEL_ID * 2];
-	uint32_t vsync_data_reg;
-};
-
-// Configuration of a channel.
-struct channel_cfg_s {
-	uint32_t		ch_id;
-	backend_channel_cfg_t	backend_ch;
-	input_system_source_t	source_type;
-	source_cfg_t		source_cfg;
-	target_cfg2400_t	target_cfg;
-};
-
-
-// Complete configuration for input system.
-struct input_system_cfg2400_s {
-
-	input_system_source_t source_type;				input_system_config_flags_t	source_type_flags;
-	//channel_cfg_t		channel[N_CHANNELS];
-	input_system_config_flags_t	ch_flags[N_CHANNELS];
-	//  This is the place where the buffers' settings are collected, as given.
-	csi_cfg_t			csi_value[N_CSI_PORTS];		input_system_config_flags_t	csi_flags[N_CSI_PORTS];
-
-	// Possible another struct for ib.
-	// This buffers set at the end, based on the all configurations.
-	ib_buffer_t			csi_buffer[N_CSI_PORTS];	input_system_config_flags_t	csi_buffer_flags[N_CSI_PORTS];
-	ib_buffer_t			acquisition_buffer_unique;	input_system_config_flags_t	acquisition_buffer_unique_flags;
-	uint32_t			unallocated_ib_mem_words; // Used for check.DEFAULT = IB_CAPACITY_IN_WORDS.
-	//uint32_t			acq_allocated_ib_mem_words;
-
-	input_system_connection_t		multicast[N_CSI_PORTS];
-	input_system_multiplex_t		multiplexer;   					input_system_config_flags_t		multiplexer_flags;
-
-
-	tpg_cfg_t			tpg_value;			input_system_config_flags_t	tpg_flags;
-	prbs_cfg_t			prbs_value;			input_system_config_flags_t	prbs_flags;
-	gpfifo_cfg_t		gpfifo_value;		input_system_config_flags_t	gpfifo_flags;
-
-
-	input_switch_cfg_t		input_switch_cfg;
-
-
-	target_isp_cfg_t		target_isp      [N_CHANNELS];	input_system_config_flags_t	target_isp_flags      [N_CHANNELS];
-	target_sp_cfg_t			target_sp       [N_CHANNELS];	input_system_config_flags_t	target_sp_flags       [N_CHANNELS];
-	target_strm2mem_cfg_t	target_strm2mem [N_CHANNELS];	input_system_config_flags_t	target_strm2mem_flags [N_CHANNELS];
-
-	input_system_config_flags_t		session_flags;
-
-};
-
-/*
- * For each MIPI port
- */
-#define _HRT_CSS_RECEIVER_DEVICE_READY_REG_IDX			_HRT_CSS_RECEIVER_2400_DEVICE_READY_REG_IDX
-#define _HRT_CSS_RECEIVER_IRQ_STATUS_REG_IDX			_HRT_CSS_RECEIVER_2400_IRQ_STATUS_REG_IDX
-#define _HRT_CSS_RECEIVER_IRQ_ENABLE_REG_IDX			_HRT_CSS_RECEIVER_2400_IRQ_ENABLE_REG_IDX
-#define _HRT_CSS_RECEIVER_TIMEOUT_COUNT_REG_IDX		    _HRT_CSS_RECEIVER_2400_CSI2_FUNC_PROG_REG_IDX
-#define _HRT_CSS_RECEIVER_INIT_COUNT_REG_IDX			_HRT_CSS_RECEIVER_2400_INIT_COUNT_REG_IDX
-/* new regs for each MIPI port w.r.t. 2300 */
-#define _HRT_CSS_RECEIVER_RAW16_18_DATAID_REG_IDX       _HRT_CSS_RECEIVER_2400_RAW16_18_DATAID_REG_IDX
-#define _HRT_CSS_RECEIVER_SYNC_COUNT_REG_IDX            _HRT_CSS_RECEIVER_2400_SYNC_COUNT_REG_IDX
-#define _HRT_CSS_RECEIVER_RX_COUNT_REG_IDX              _HRT_CSS_RECEIVER_2400_RX_COUNT_REG_IDX
-
-/* _HRT_CSS_RECEIVER_2400_COMP_FORMAT_REG_IDX is not defined per MIPI port but per channel */
-/* _HRT_CSS_RECEIVER_2400_COMP_PREDICT_REG_IDX is not defined per MIPI port but per channel */
-#define _HRT_CSS_RECEIVER_FS_TO_LS_DELAY_REG_IDX        _HRT_CSS_RECEIVER_2400_FS_TO_LS_DELAY_REG_IDX
-#define _HRT_CSS_RECEIVER_LS_TO_DATA_DELAY_REG_IDX      _HRT_CSS_RECEIVER_2400_LS_TO_DATA_DELAY_REG_IDX
-#define _HRT_CSS_RECEIVER_DATA_TO_LE_DELAY_REG_IDX      _HRT_CSS_RECEIVER_2400_DATA_TO_LE_DELAY_REG_IDX
-#define _HRT_CSS_RECEIVER_LE_TO_FE_DELAY_REG_IDX        _HRT_CSS_RECEIVER_2400_LE_TO_FE_DELAY_REG_IDX
-#define _HRT_CSS_RECEIVER_FE_TO_FS_DELAY_REG_IDX        _HRT_CSS_RECEIVER_2400_FE_TO_FS_DELAY_REG_IDX
-#define _HRT_CSS_RECEIVER_LE_TO_LS_DELAY_REG_IDX        _HRT_CSS_RECEIVER_2400_LE_TO_LS_DELAY_REG_IDX
-#define _HRT_CSS_RECEIVER_TWO_PIXEL_EN_REG_IDX			_HRT_CSS_RECEIVER_2400_TWO_PIXEL_EN_REG_IDX
-#define _HRT_CSS_RECEIVER_BACKEND_RST_REG_IDX           _HRT_CSS_RECEIVER_2400_BACKEND_RST_REG_IDX
-#define _HRT_CSS_RECEIVER_RAW18_REG_IDX                 _HRT_CSS_RECEIVER_2400_RAW18_REG_IDX
-#define _HRT_CSS_RECEIVER_FORCE_RAW8_REG_IDX            _HRT_CSS_RECEIVER_2400_FORCE_RAW8_REG_IDX
-#define _HRT_CSS_RECEIVER_RAW16_REG_IDX                 _HRT_CSS_RECEIVER_2400_RAW16_REG_IDX
-
-/* Previously MIPI port regs, now 2x2 logical channel regs */
-#define _HRT_CSS_RECEIVER_COMP_SCHEME_VC0_REG0_IDX		_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG0_IDX
-#define _HRT_CSS_RECEIVER_COMP_SCHEME_VC0_REG1_IDX		_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC0_REG1_IDX
-#define _HRT_CSS_RECEIVER_COMP_SCHEME_VC1_REG0_IDX		_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG0_IDX
-#define _HRT_CSS_RECEIVER_COMP_SCHEME_VC1_REG1_IDX		_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC1_REG1_IDX
-#define _HRT_CSS_RECEIVER_COMP_SCHEME_VC2_REG0_IDX		_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG0_IDX
-#define _HRT_CSS_RECEIVER_COMP_SCHEME_VC2_REG1_IDX		_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC2_REG1_IDX
-#define _HRT_CSS_RECEIVER_COMP_SCHEME_VC3_REG0_IDX		_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG0_IDX
-#define _HRT_CSS_RECEIVER_COMP_SCHEME_VC3_REG1_IDX		_HRT_CSS_RECEIVER_2400_COMP_SCHEME_VC3_REG1_IDX
-
-/* Second backend is at offset 0x0700 w.r.t. the first port at offset 0x0100 */
-#define _HRT_CSS_BE_OFFSET                              448
-#define _HRT_CSS_RECEIVER_BE_GSP_ACC_OVL_REG_IDX        (_HRT_CSS_RECEIVER_2400_BE_GSP_ACC_OVL_REG_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_SRST_REG_IDX               (_HRT_CSS_RECEIVER_2400_BE_SRST_REG_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_TWO_PPC_REG_IDX            (_HRT_CSS_RECEIVER_2400_BE_TWO_PPC_REG_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_COMP_FORMAT_REG0_IDX       (_HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG0_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_COMP_FORMAT_REG1_IDX       (_HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG1_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_COMP_FORMAT_REG2_IDX       (_HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG2_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_COMP_FORMAT_REG3_IDX       (_HRT_CSS_RECEIVER_2400_BE_COMP_FORMAT_REG3_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_SEL_REG_IDX                (_HRT_CSS_RECEIVER_2400_BE_SEL_REG_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_RAW16_CONFIG_REG_IDX       (_HRT_CSS_RECEIVER_2400_BE_RAW16_CONFIG_REG_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_RAW18_CONFIG_REG_IDX       (_HRT_CSS_RECEIVER_2400_BE_RAW18_CONFIG_REG_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_FORCE_RAW8_REG_IDX         (_HRT_CSS_RECEIVER_2400_BE_FORCE_RAW8_REG_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_IRQ_STATUS_REG_IDX         (_HRT_CSS_RECEIVER_2400_BE_IRQ_STATUS_REG_IDX + _HRT_CSS_BE_OFFSET)
-#define _HRT_CSS_RECEIVER_BE_IRQ_CLEAR_REG_IDX          (_HRT_CSS_RECEIVER_2400_BE_IRQ_CLEAR_REG_IDX + _HRT_CSS_BE_OFFSET)
-
-
-#define _HRT_CSS_RECEIVER_IRQ_OVERRUN_BIT		_HRT_CSS_RECEIVER_2400_IRQ_OVERRUN_BIT
-#define _HRT_CSS_RECEIVER_IRQ_INIT_TIMEOUT_BIT		_HRT_CSS_RECEIVER_2400_IRQ_RESERVED_BIT
-#define _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_ENTRY_BIT	_HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_ENTRY_BIT
-#define _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_EXIT_BIT	_HRT_CSS_RECEIVER_2400_IRQ_SLEEP_MODE_EXIT_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_SOT_HS_BIT		_HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_HS_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_SOT_SYNC_HS_BIT	_HRT_CSS_RECEIVER_2400_IRQ_ERR_SOT_SYNC_HS_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_CONTROL_BIT		_HRT_CSS_RECEIVER_2400_IRQ_ERR_CONTROL_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_ECC_DOUBLE_BIT	_HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_DOUBLE_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_ECC_CORRECTED_BIT	_HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_CORRECTED_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_ECC_NO_CORRECTION_BIT	_HRT_CSS_RECEIVER_2400_IRQ_ERR_ECC_NO_CORRECTION_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_CRC_BIT		_HRT_CSS_RECEIVER_2400_IRQ_ERR_CRC_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_ID_BIT		_HRT_CSS_RECEIVER_2400_IRQ_ERR_ID_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_FRAME_SYNC_BIT	_HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_SYNC_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_FRAME_DATA_BIT	_HRT_CSS_RECEIVER_2400_IRQ_ERR_FRAME_DATA_BIT
-#define _HRT_CSS_RECEIVER_IRQ_DATA_TIMEOUT_BIT		_HRT_CSS_RECEIVER_2400_IRQ_DATA_TIMEOUT_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_ESCAPE_BIT		_HRT_CSS_RECEIVER_2400_IRQ_ERR_ESCAPE_BIT
-#define _HRT_CSS_RECEIVER_IRQ_ERR_LINE_SYNC_BIT		_HRT_CSS_RECEIVER_2400_IRQ_ERR_LINE_SYNC_BIT
-
-#define _HRT_CSS_RECEIVER_FUNC_PROG_REG_IDX		_HRT_CSS_RECEIVER_2400_CSI2_FUNC_PROG_REG_IDX
-#define	_HRT_CSS_RECEIVER_DATA_TIMEOUT_IDX		_HRT_CSS_RECEIVER_2400_CSI2_DATA_TIMEOUT_IDX
-#define	_HRT_CSS_RECEIVER_DATA_TIMEOUT_BITS		_HRT_CSS_RECEIVER_2400_CSI2_DATA_TIMEOUT_BITS
-
-typedef struct capture_unit_state_s	capture_unit_state_t;
-typedef struct acquisition_unit_state_s	acquisition_unit_state_t;
-typedef struct ctrl_unit_state_s	ctrl_unit_state_t;
-
-/*
- * In 2300 ports can be configured independently and stream
- * formats need to be specified. In 2400, there are only 8
- * supported configurations but the HW is fused to support
- * only a single one.
- *
- * In 2300 the compressed format types are programmed by the
- * user. In 2400 all stream formats are encoded on the stream.
- *
- * Use the enum to check validity of a user configuration
- */
-typedef enum {
-	MONO_4L_1L_0L = 0,
-	MONO_3L_1L_0L,
-	MONO_2L_1L_0L,
-	MONO_1L_1L_0L,
-	STEREO_2L_1L_2L,
-	STEREO_3L_1L_1L,
-	STEREO_2L_1L_1L,
-	STEREO_1L_1L_1L,
-	N_RX_MODE
-} rx_mode_t;
-
-typedef enum {
-	MIPI_PREDICTOR_NONE = 0,
-	MIPI_PREDICTOR_TYPE1,
-	MIPI_PREDICTOR_TYPE2,
-	N_MIPI_PREDICTOR_TYPES
-} mipi_predictor_t;
-
-typedef enum {
-	MIPI_COMPRESSOR_NONE = 0,
-	MIPI_COMPRESSOR_10_6_10,
-	MIPI_COMPRESSOR_10_7_10,
-	MIPI_COMPRESSOR_10_8_10,
-	MIPI_COMPRESSOR_12_6_12,
-	MIPI_COMPRESSOR_12_7_12,
-	MIPI_COMPRESSOR_12_8_12,
-	N_MIPI_COMPRESSOR_METHODS
-} mipi_compressor_t;
-
-typedef enum {
-	MIPI_FORMAT_RGB888 = 0,
-	MIPI_FORMAT_RGB555,
-	MIPI_FORMAT_RGB444,
-	MIPI_FORMAT_RGB565,
-	MIPI_FORMAT_RGB666,
-	MIPI_FORMAT_RAW8,		/* 5 */
-	MIPI_FORMAT_RAW10,
-	MIPI_FORMAT_RAW6,
-	MIPI_FORMAT_RAW7,
-	MIPI_FORMAT_RAW12,
-	MIPI_FORMAT_RAW14,		/* 10 */
-	MIPI_FORMAT_YUV420_8,
-	MIPI_FORMAT_YUV420_10,
-	MIPI_FORMAT_YUV422_8,
-	MIPI_FORMAT_YUV422_10,
-	MIPI_FORMAT_CUSTOM0,	/* 15 */
-	MIPI_FORMAT_YUV420_8_LEGACY,
-	MIPI_FORMAT_EMBEDDED,
-	MIPI_FORMAT_CUSTOM1,
-	MIPI_FORMAT_CUSTOM2,
-	MIPI_FORMAT_CUSTOM3,	/* 20 */
-	MIPI_FORMAT_CUSTOM4,
-	MIPI_FORMAT_CUSTOM5,
-	MIPI_FORMAT_CUSTOM6,
-	MIPI_FORMAT_CUSTOM7,
-	MIPI_FORMAT_YUV420_8_SHIFT,	/* 25 */
-	MIPI_FORMAT_YUV420_10_SHIFT,
-	MIPI_FORMAT_RAW16,
-	MIPI_FORMAT_RAW18,
-	N_MIPI_FORMAT,
-} mipi_format_t;
-
-#define MIPI_FORMAT_JPEG		MIPI_FORMAT_CUSTOM0
-#define MIPI_FORMAT_BINARY_8	MIPI_FORMAT_CUSTOM0
-#define N_MIPI_FORMAT_CUSTOM	8
-
-/* The number of stores for compressed format types */
-#define	N_MIPI_COMPRESSOR_CONTEXT	(N_RX_CHANNEL_ID * N_MIPI_FORMAT_CUSTOM)
-
-typedef enum {
-	RX_IRQ_INFO_BUFFER_OVERRUN   = 1UL << _HRT_CSS_RECEIVER_IRQ_OVERRUN_BIT,
-	RX_IRQ_INFO_INIT_TIMEOUT     = 1UL << _HRT_CSS_RECEIVER_IRQ_INIT_TIMEOUT_BIT,
-	RX_IRQ_INFO_ENTER_SLEEP_MODE = 1UL << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_ENTRY_BIT,
-	RX_IRQ_INFO_EXIT_SLEEP_MODE  = 1UL << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_EXIT_BIT,
-	RX_IRQ_INFO_ECC_CORRECTED    = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_ECC_CORRECTED_BIT,
-	RX_IRQ_INFO_ERR_SOT          = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_SOT_HS_BIT,
-	RX_IRQ_INFO_ERR_SOT_SYNC     = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_SOT_SYNC_HS_BIT,
-	RX_IRQ_INFO_ERR_CONTROL      = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_CONTROL_BIT,
-	RX_IRQ_INFO_ERR_ECC_DOUBLE   = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_ECC_DOUBLE_BIT,
-/*	RX_IRQ_INFO_NO_ERR           = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_ECC_NO_CORRECTION_BIT, */
-	RX_IRQ_INFO_ERR_CRC          = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_CRC_BIT,
-	RX_IRQ_INFO_ERR_UNKNOWN_ID   = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_ID_BIT,
-	RX_IRQ_INFO_ERR_FRAME_SYNC   = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_FRAME_SYNC_BIT,
-	RX_IRQ_INFO_ERR_FRAME_DATA   = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_FRAME_DATA_BIT,
-	RX_IRQ_INFO_ERR_DATA_TIMEOUT = 1UL << _HRT_CSS_RECEIVER_IRQ_DATA_TIMEOUT_BIT,
-	RX_IRQ_INFO_ERR_UNKNOWN_ESC  = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_ESCAPE_BIT,
-	RX_IRQ_INFO_ERR_LINE_SYNC    = 1UL << _HRT_CSS_RECEIVER_IRQ_ERR_LINE_SYNC_BIT,
-}  rx_irq_info_t;
-
-typedef struct rx_cfg_s		rx_cfg_t;
-
-/*
- * Applied per port
- */
-struct rx_cfg_s {
-	rx_mode_t			mode;	/* The HW config */
-	enum mipi_port_id		port;	/* The port ID to apply the control on */
-	unsigned int		timeout;
-	unsigned int		initcount;
-	unsigned int		synccount;
-	unsigned int		rxcount;
-	mipi_predictor_t	comp;	/* Just for backward compatibility */
-	bool                is_two_ppc;
-};
-
-/* NOTE: The base has already an offset of 0x0100 */
-static const hrt_address MIPI_PORT_OFFSET[N_MIPI_PORT_ID] = {
-	0x00000000UL,
-	0x00000100UL,
-	0x00000200UL};
-
-static const mipi_lane_cfg_t MIPI_PORT_MAXLANES[N_MIPI_PORT_ID] = {
-	MIPI_4LANE_CFG,
-	MIPI_1LANE_CFG,
-	MIPI_2LANE_CFG};
-
-static const bool MIPI_PORT_ACTIVE[N_RX_MODE][N_MIPI_PORT_ID] = {
-	{true, true, false},
-	{true, true, false},
-	{true, true, false},
-	{true, true, false},
-	{true, true, true},
-	{true, true, true},
-	{true, true, true},
-	{true, true, true}};
-
-static const mipi_lane_cfg_t MIPI_PORT_LANES[N_RX_MODE][N_MIPI_PORT_ID] = {
-	{MIPI_4LANE_CFG, MIPI_1LANE_CFG, MIPI_0LANE_CFG},
-	{MIPI_3LANE_CFG, MIPI_1LANE_CFG, MIPI_0LANE_CFG},
-	{MIPI_2LANE_CFG, MIPI_1LANE_CFG, MIPI_0LANE_CFG},
-	{MIPI_1LANE_CFG, MIPI_1LANE_CFG, MIPI_0LANE_CFG},
-	{MIPI_2LANE_CFG, MIPI_1LANE_CFG, MIPI_2LANE_CFG},
-	{MIPI_3LANE_CFG, MIPI_1LANE_CFG, MIPI_1LANE_CFG},
-	{MIPI_2LANE_CFG, MIPI_1LANE_CFG, MIPI_1LANE_CFG},
-	{MIPI_1LANE_CFG, MIPI_1LANE_CFG, MIPI_1LANE_CFG}};
-
-static const hrt_address SUB_SYSTEM_OFFSET[N_SUB_SYSTEM_ID] = {
-	0x00001000UL,
-	0x00002000UL,
-	0x00003000UL,
-	0x00004000UL,
-	0x00005000UL,
-	0x00009000UL,
-	0x0000A000UL,
-	0x0000B000UL,
-	0x0000C000UL};
-
-struct capture_unit_state_s {
-	int	Packet_Length;
-	int	Received_Length;
-	int	Received_Short_Packets;
-	int	Received_Long_Packets;
-	int	Last_Command;
-	int	Next_Command;
-	int	Last_Acknowledge;
-	int	Next_Acknowledge;
-	int	FSM_State_Info;
-	int	StartMode;
-	int	Start_Addr;
-	int	Mem_Region_Size;
-	int	Num_Mem_Regions;
-/*	int	Init;   write-only registers
-	int	Start;
-	int	Stop;      */
-};
-
-struct acquisition_unit_state_s {
-/*	int	Init;   write-only register */
-	int	Received_Short_Packets;
-	int	Received_Long_Packets;
-	int	Last_Command;
-	int	Next_Command;
-	int	Last_Acknowledge;
-	int	Next_Acknowledge;
-	int	FSM_State_Info;
-	int	Int_Cntr_Info;
-	int	Start_Addr;
-	int	Mem_Region_Size;
-	int	Num_Mem_Regions;
-};
-
-struct ctrl_unit_state_s {
-	int	last_cmd;
-	int	next_cmd;
-	int	last_ack;
-	int	next_ack;
-	int	top_fsm_state;
-	int	captA_fsm_state;
-	int	captB_fsm_state;
-	int	captC_fsm_state;
-	int	acq_fsm_state;
-	int	captA_start_addr;
-	int	captB_start_addr;
-	int	captC_start_addr;
-	int	captA_mem_region_size;
-	int	captB_mem_region_size;
-	int	captC_mem_region_size;
-	int	captA_num_mem_regions;
-	int	captB_num_mem_regions;
-	int	captC_num_mem_regions;
-	int	acq_start_addr;
-	int	acq_mem_region_size;
-	int	acq_num_mem_regions;
-/*	int	ctrl_init;  write only register */
-	int	capt_reserve_one_mem_region;
-};
-
-struct input_system_state_s {
-	int	str_multicastA_sel;
-	int	str_multicastB_sel;
-	int	str_multicastC_sel;
-	int	str_mux_sel;
-	int	str_mon_status;
-	int	str_mon_irq_cond;
-	int	str_mon_irq_en;
-	int	isys_srst;
-	int	isys_slv_reg_srst;
-	int	str_deint_portA_cnt;
-	int	str_deint_portB_cnt;
-	struct capture_unit_state_s		capture_unit[N_CAPTURE_UNIT_ID];
-	struct acquisition_unit_state_s	acquisition_unit[N_ACQUISITION_UNIT_ID];
-	struct ctrl_unit_state_s		ctrl_unit_state[N_CTRL_UNIT_ID];
-};
-
-struct mipi_port_state_s {
-	int	device_ready;
-	int	irq_status;
-	int	irq_enable;
-	uint32_t	timeout_count;
-	uint16_t	init_count;
-	uint16_t	raw16_18;
-	uint32_t	sync_count;		/*4 x uint8_t */
-	uint32_t	rx_count;		/*4 x uint8_t */
-	uint8_t		lane_sync_count[MIPI_4LANE_CFG];
-	uint8_t		lane_rx_count[MIPI_4LANE_CFG];
-};
-
-struct rx_channel_state_s {
-	uint32_t	comp_scheme0;
-	uint32_t	comp_scheme1;
-	mipi_predictor_t		pred[N_MIPI_FORMAT_CUSTOM];
-	mipi_compressor_t		comp[N_MIPI_FORMAT_CUSTOM];
-};
-
-struct receiver_state_s {
-	uint8_t	fs_to_ls_delay;
-	uint8_t	ls_to_data_delay;
-	uint8_t	data_to_le_delay;
-	uint8_t	le_to_fe_delay;
-	uint8_t	fe_to_fs_delay;
-	uint8_t	le_to_fs_delay;
-	bool	is_two_ppc;
-	int	backend_rst;
-	uint16_t	raw18;
-	bool		force_raw8;
-	uint16_t	raw16;
-	struct mipi_port_state_s	mipi_port_state[N_MIPI_PORT_ID];
-	struct rx_channel_state_s	rx_channel_state[N_RX_CHANNEL_ID];
-	int	be_gsp_acc_ovl;
-	int	be_srst;
-	int	be_is_two_ppc;
-	int	be_comp_format0;
-	int	be_comp_format1;
-	int	be_comp_format2;
-	int	be_comp_format3;
-	int	be_sel;
-	int	be_raw16_config;
-	int	be_raw18_config;
-	int	be_force_raw8;
-	int	be_irq_status;
-	int	be_irq_clear;
-};
-
-#endif /* __INPUT_SYSTEM_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system_private.h
deleted file mode 100644
index 48876bb08b70..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/input_system_private.h
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_SYSTEM_PRIVATE_H_INCLUDED__
-#define __INPUT_SYSTEM_PRIVATE_H_INCLUDED__
-
-#include "input_system_public.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-STORAGE_CLASS_INPUT_SYSTEM_C void input_system_reg_store(
-	const input_system_ID_t			ID,
-	const hrt_address			reg,
-	const hrt_data				value)
-{
-	assert(ID < N_INPUT_SYSTEM_ID);
-	assert(INPUT_SYSTEM_BASE[ID] != (hrt_address)-1);
-	ia_css_device_store_uint32(INPUT_SYSTEM_BASE[ID] + reg*sizeof(hrt_data), value);
-	return;
-}
-
-STORAGE_CLASS_INPUT_SYSTEM_C hrt_data input_system_reg_load(
-	const input_system_ID_t			ID,
-	const hrt_address			reg)
-{
-	assert(ID < N_INPUT_SYSTEM_ID);
-	assert(INPUT_SYSTEM_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(INPUT_SYSTEM_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-STORAGE_CLASS_INPUT_SYSTEM_C void receiver_reg_store(
-	const rx_ID_t				ID,
-	const hrt_address			reg,
-	const hrt_data				value)
-{
-	assert(ID < N_RX_ID);
-	assert(RX_BASE[ID] != (hrt_address)-1);
-	ia_css_device_store_uint32(RX_BASE[ID] + reg*sizeof(hrt_data), value);
-	return;
-}
-
-STORAGE_CLASS_INPUT_SYSTEM_C hrt_data receiver_reg_load(
-	const rx_ID_t				ID,
-	const hrt_address			reg)
-{
-	assert(ID < N_RX_ID);
-	assert(RX_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(RX_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-STORAGE_CLASS_INPUT_SYSTEM_C void receiver_port_reg_store(
-	const rx_ID_t				ID,
-	const enum mipi_port_id			port_ID,
-	const hrt_address			reg,
-	const hrt_data				value)
-{
-	assert(ID < N_RX_ID);
-	assert(port_ID < N_MIPI_PORT_ID);
-	assert(RX_BASE[ID] != (hrt_address)-1);
-	assert(MIPI_PORT_OFFSET[port_ID] != (hrt_address)-1);
-	ia_css_device_store_uint32(RX_BASE[ID] + MIPI_PORT_OFFSET[port_ID] + reg*sizeof(hrt_data), value);
-	return;
-}
-
-STORAGE_CLASS_INPUT_SYSTEM_C hrt_data receiver_port_reg_load(
-	const rx_ID_t				ID,
-	const enum mipi_port_id			port_ID,
-	const hrt_address			reg)
-{
-	assert(ID < N_RX_ID);
-	assert(port_ID < N_MIPI_PORT_ID);
-	assert(RX_BASE[ID] != (hrt_address)-1);
-	assert(MIPI_PORT_OFFSET[port_ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(RX_BASE[ID] + MIPI_PORT_OFFSET[port_ID] + reg*sizeof(hrt_data));
-}
-
-STORAGE_CLASS_INPUT_SYSTEM_C void input_system_sub_system_reg_store(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_ID,
-	const hrt_address			reg,
-	const hrt_data				value)
-{
-	assert(ID < N_INPUT_SYSTEM_ID);
-	assert(sub_ID < N_SUB_SYSTEM_ID);
-	assert(INPUT_SYSTEM_BASE[ID] != (hrt_address)-1);
-	assert(SUB_SYSTEM_OFFSET[sub_ID] != (hrt_address)-1);
-	ia_css_device_store_uint32(INPUT_SYSTEM_BASE[ID] + SUB_SYSTEM_OFFSET[sub_ID] + reg*sizeof(hrt_data), value);
-	return;
-}
-
-STORAGE_CLASS_INPUT_SYSTEM_C hrt_data input_system_sub_system_reg_load(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_ID,
-	const hrt_address			reg)
-{
-	assert(ID < N_INPUT_SYSTEM_ID);
-	assert(sub_ID < N_SUB_SYSTEM_ID);
-	assert(INPUT_SYSTEM_BASE[ID] != (hrt_address)-1);
-	assert(SUB_SYSTEM_OFFSET[sub_ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(INPUT_SYSTEM_BASE[ID] + SUB_SYSTEM_OFFSET[sub_ID] + reg*sizeof(hrt_data));
-}
-
-#endif /* __INPUT_SYSTEM_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq.c
deleted file mode 100644
index 51daf76c2aea..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq.c
+++ /dev/null
@@ -1,448 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "assert_support.h"
-#include "irq.h"
-
-#ifndef __INLINE_GP_DEVICE__
-#define __INLINE_GP_DEVICE__
-#endif
-#include "gp_device.h"	/* _REG_GP_IRQ_REQUEST_ADDR */
-
-#include "platform_support.h"			/* hrt_sleep() */
-
-static inline void irq_wait_for_write_complete(
-	const irq_ID_t		ID);
-
-static inline bool any_irq_channel_enabled(
-	const irq_ID_t				ID);
-
-static inline irq_ID_t virq_get_irq_id(
-	const virq_id_t		irq_ID,
-	unsigned int		*channel_ID);
-
-#ifndef __INLINE_IRQ__
-#include "irq_private.h"
-#endif /* __INLINE_IRQ__ */
-
-static unsigned short IRQ_N_CHANNEL[N_IRQ_ID] = {
-	IRQ0_ID_N_CHANNEL,
-	IRQ1_ID_N_CHANNEL,
-	IRQ2_ID_N_CHANNEL,
-	IRQ3_ID_N_CHANNEL};
-
-static unsigned short IRQ_N_ID_OFFSET[N_IRQ_ID + 1] = {
-	IRQ0_ID_OFFSET,
-	IRQ1_ID_OFFSET,
-	IRQ2_ID_OFFSET,
-	IRQ3_ID_OFFSET,
-	IRQ_END_OFFSET};
-
-static virq_id_t IRQ_NESTING_ID[N_IRQ_ID] = {
-	N_virq_id,
-	virq_ifmt,
-	virq_isys,
-	virq_isel};
-
-void irq_clear_all(
-	const irq_ID_t				ID)
-{
-	hrt_data	mask = 0xFFFFFFFF;
-
-	assert(ID < N_IRQ_ID);
-	assert(IRQ_N_CHANNEL[ID] <= HRT_DATA_WIDTH);
-
-	if (IRQ_N_CHANNEL[ID] < HRT_DATA_WIDTH) {
-		mask = ~((~(hrt_data)0)>>IRQ_N_CHANNEL[ID]);
-	}
-
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_CLEAR_REG_IDX, mask);
-	return;
-}
-
-/*
- * Do we want the user to be able to set the signalling method ?
- */
-void irq_enable_channel(
-	const irq_ID_t				ID,
-    const unsigned int			irq_id)
-{
-	unsigned int mask = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_MASK_REG_IDX);
-	unsigned int enable = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_ENABLE_REG_IDX);
-	unsigned int edge_in = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_EDGE_REG_IDX);
-	unsigned int me = 1U << irq_id;
-
-	assert(ID < N_IRQ_ID);
-	assert(irq_id < IRQ_N_CHANNEL[ID]);
-
-	mask |= me;
-	enable |= me;
-	edge_in |= me;	/* rising edge */
-
-/* to avoid mishaps configuration must follow the following order */
-
-/* mask this interrupt */
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_MASK_REG_IDX, mask & ~me);
-/* rising edge at input */
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_EDGE_REG_IDX, edge_in);
-/* enable interrupt to output */
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_ENABLE_REG_IDX, enable);
-/* clear current irq only */
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_CLEAR_REG_IDX, me);
-/* unmask interrupt from input */
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_MASK_REG_IDX, mask);
-
-	irq_wait_for_write_complete(ID);
-
-	return;
-}
-
-void irq_enable_pulse(
-	const irq_ID_t	ID,
-	bool 			pulse)
-{
-	unsigned int edge_out = 0x0;
-
-	if (pulse) {
-		edge_out = 0xffffffff;
-	}
-	/* output is given as edge, not pulse */
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_EDGE_NOT_PULSE_REG_IDX, edge_out);
-	return;
-}
-
-void irq_disable_channel(
-	const irq_ID_t				ID,
-	const unsigned int			irq_id)
-{
-	unsigned int mask = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_MASK_REG_IDX);
-	unsigned int enable = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_ENABLE_REG_IDX);
-	unsigned int me = 1U << irq_id;
-
-	assert(ID < N_IRQ_ID);
-	assert(irq_id < IRQ_N_CHANNEL[ID]);
-
-	mask &= ~me;
-	enable &= ~me;
-
-/* enable interrupt to output */
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_ENABLE_REG_IDX, enable);
-/* unmask interrupt from input */
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_MASK_REG_IDX, mask);
-/* clear current irq only */
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_CLEAR_REG_IDX, me);
-
-	irq_wait_for_write_complete(ID);
-
-	return;
-}
-
-enum hrt_isp_css_irq_status irq_get_channel_id(
-	const irq_ID_t				ID,
-	unsigned int				*irq_id)
-{
-	unsigned int irq_status = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_STATUS_REG_IDX);
-	unsigned int idx;
-	enum hrt_isp_css_irq_status status = hrt_isp_css_irq_status_success;
-
-	assert(ID < N_IRQ_ID);
-	assert(irq_id != NULL);
-
-/* find the first irq bit */
-	for (idx = 0; idx < IRQ_N_CHANNEL[ID]; idx++) {
-		if (irq_status & (1U << idx))
-			break;
-	}
-	if (idx == IRQ_N_CHANNEL[ID])
-		return hrt_isp_css_irq_status_error;
-
-/* now check whether there are more bits set */
-	if (irq_status != (1U << idx))
-		status = hrt_isp_css_irq_status_more_irqs;
-
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_CLEAR_REG_IDX, 1U << idx);
-
-	irq_wait_for_write_complete(ID);
-
-	if (irq_id != NULL)
-		*irq_id = (unsigned int)idx;
-
-	return status;
-}
-
-static const hrt_address IRQ_REQUEST_ADDR[N_IRQ_SW_CHANNEL_ID] = {
-	_REG_GP_IRQ_REQUEST0_ADDR,
-	_REG_GP_IRQ_REQUEST1_ADDR};
-
-void irq_raise(
-	const irq_ID_t				ID,
-	const irq_sw_channel_id_t	irq_id)
-{
-	hrt_address		addr;
-
-	OP___assert(ID == IRQ0_ID);
-	OP___assert(IRQ_BASE[ID] != (hrt_address)-1);
-	OP___assert(irq_id < N_IRQ_SW_CHANNEL_ID);
-
-	(void)ID;
-
-	addr = IRQ_REQUEST_ADDR[irq_id];
-/* The SW IRQ pins are remapped to offset zero */
-	gp_device_reg_store(GP_DEVICE0_ID,
-		(unsigned int)addr, 1);
-	gp_device_reg_store(GP_DEVICE0_ID,
-		(unsigned int)addr, 0);
-	return;
-}
-
-void irq_controller_get_state(
-	const irq_ID_t				ID,
-	irq_controller_state_t		*state)
-{
-	assert(ID < N_IRQ_ID);
-	assert(state != NULL);
-
-	state->irq_edge = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_EDGE_REG_IDX);
-	state->irq_mask = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_MASK_REG_IDX);
-	state->irq_status = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_STATUS_REG_IDX);
-	state->irq_enable = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_ENABLE_REG_IDX);
-	state->irq_level_not_pulse = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_EDGE_NOT_PULSE_REG_IDX);
-	return;
-}
-
-bool any_virq_signal(void)
-{
-	unsigned int irq_status = irq_reg_load(IRQ0_ID,
-		_HRT_IRQ_CONTROLLER_STATUS_REG_IDX);
-
-	return (irq_status != 0);
-}
-
-void cnd_virq_enable_channel(
-	const virq_id_t				irq_ID,
-	const bool					en)
-{
-	irq_ID_t		i;
-	unsigned int	channel_ID;
-	irq_ID_t		ID = virq_get_irq_id(irq_ID, &channel_ID);
-	
-	assert(ID < N_IRQ_ID);
-
-	for (i=IRQ1_ID;i<N_IRQ_ID;i++) {
-	/* It is not allowed to enable the pin of a nested IRQ directly */
-		assert(irq_ID != IRQ_NESTING_ID[i]);
-	}
-
-	if (en) {
-		irq_enable_channel(ID, channel_ID);
-		if (IRQ_NESTING_ID[ID] != N_virq_id) {
-/* Single level nesting, otherwise we'd need to recurse */
-			irq_enable_channel(IRQ0_ID, IRQ_NESTING_ID[ID]);
-		}
-	} else {
-		irq_disable_channel(ID, channel_ID);
-		if ((IRQ_NESTING_ID[ID] != N_virq_id) && !any_irq_channel_enabled(ID)) {
-/* Only disable the top if the nested ones are empty */
-			irq_disable_channel(IRQ0_ID, IRQ_NESTING_ID[ID]);
-		}
-	}
-	return;
-}
-
-
-void virq_clear_all(void)
-{
-	irq_ID_t	irq_id;
-
-	for (irq_id = (irq_ID_t)0; irq_id < N_IRQ_ID; irq_id++) {
-		irq_clear_all(irq_id);
-	}
-	return;
-}
-
-enum hrt_isp_css_irq_status virq_get_channel_signals(
-	virq_info_t					*irq_info)
-{
-	enum hrt_isp_css_irq_status irq_status = hrt_isp_css_irq_status_error;
-	irq_ID_t ID;
-
-	assert(irq_info != NULL);
-
-	for (ID = (irq_ID_t)0 ; ID < N_IRQ_ID; ID++) {
-		if (any_irq_channel_enabled(ID)) {
-			hrt_data	irq_data = irq_reg_load(ID,
-				_HRT_IRQ_CONTROLLER_STATUS_REG_IDX);
-
-			if (irq_data != 0) {
-/* The error condition is an IRQ pulse received with no IRQ status written */
-				irq_status = hrt_isp_css_irq_status_success;
-			}
-
-			irq_info->irq_status_reg[ID] |= irq_data;
-
-			irq_reg_store(ID,
-				_HRT_IRQ_CONTROLLER_CLEAR_REG_IDX, irq_data);
-
-			irq_wait_for_write_complete(ID);
-		}
-	}
-
-	return irq_status;
-}
-
-void virq_clear_info(
-	virq_info_t					*irq_info)
-{
-	irq_ID_t ID;
-
-	assert(irq_info != NULL);
-
-	for (ID = (irq_ID_t)0 ; ID < N_IRQ_ID; ID++) {
-			irq_info->irq_status_reg[ID] = 0;
-	}
-	return;
-}
-
-enum hrt_isp_css_irq_status virq_get_channel_id(
-	virq_id_t					*irq_id)
-{
-	unsigned int irq_status = irq_reg_load(IRQ0_ID,
-		_HRT_IRQ_CONTROLLER_STATUS_REG_IDX);
-	unsigned int idx;
-	enum hrt_isp_css_irq_status status = hrt_isp_css_irq_status_success;
-	irq_ID_t ID;
-
-	assert(irq_id != NULL);
-
-/* find the first irq bit on device 0 */
-	for (idx = 0; idx < IRQ_N_CHANNEL[IRQ0_ID]; idx++) {
-		if (irq_status & (1U << idx))
-			break;
-	}
-
-	if (idx == IRQ_N_CHANNEL[IRQ0_ID]) {
-		return hrt_isp_css_irq_status_error;
-	}
-
-/* Check whether there are more bits set on device 0 */
-	if (irq_status != (1U << idx)) {
-		status = hrt_isp_css_irq_status_more_irqs;
-	}
-
-/* Check whether we have an IRQ on one of the nested devices */
-	for (ID = N_IRQ_ID-1 ; ID > (irq_ID_t)0; ID--) {
-		if (IRQ_NESTING_ID[ID] == (virq_id_t)idx) {
-			break;
-		}
-	}
-
-/* If we have a nested IRQ, load that state, discard the device 0 state */
-	if (ID != IRQ0_ID) {
-		irq_status = irq_reg_load(ID,
-			_HRT_IRQ_CONTROLLER_STATUS_REG_IDX);
-/* find the first irq bit on device "id" */
-		for (idx = 0; idx < IRQ_N_CHANNEL[ID]; idx++) {
-			if (irq_status & (1U << idx))
-				break;
-		}
-
-		if (idx == IRQ_N_CHANNEL[ID]) {
-			return hrt_isp_css_irq_status_error;
-		}
-
-/* Alternatively check whether there are more bits set on this device */
-		if (irq_status != (1U << idx)) {
-			status = hrt_isp_css_irq_status_more_irqs;
-		} else {
-/* If this device is empty, clear the state on device 0 */
-			irq_reg_store(IRQ0_ID,
-				_HRT_IRQ_CONTROLLER_CLEAR_REG_IDX, 1U << IRQ_NESTING_ID[ID]);
-		}
-	} /* if (ID != IRQ0_ID) */
-
-/* Here we proceed to clear the IRQ on detected device, if no nested IRQ, this is device 0 */
-	irq_reg_store(ID,
-		_HRT_IRQ_CONTROLLER_CLEAR_REG_IDX, 1U << idx);
-
-	irq_wait_for_write_complete(ID);
-
-	idx += IRQ_N_ID_OFFSET[ID];
-	if (irq_id != NULL)
-		*irq_id = (virq_id_t)idx;
-
-	return status;
-}
-
-static inline void irq_wait_for_write_complete(
-	const irq_ID_t		ID)
-{
-	assert(ID < N_IRQ_ID);
-	assert(IRQ_BASE[ID] != (hrt_address)-1);
-	(void)ia_css_device_load_uint32(IRQ_BASE[ID] +
-		_HRT_IRQ_CONTROLLER_ENABLE_REG_IDX*sizeof(hrt_data));
-}
-
-static inline bool any_irq_channel_enabled(
-	const irq_ID_t				ID)
-{
-	hrt_data	en_reg;
-
-	assert(ID < N_IRQ_ID);
-
-	en_reg = irq_reg_load(ID,
-		_HRT_IRQ_CONTROLLER_ENABLE_REG_IDX);
-
-	return (en_reg != 0);
-}
-
-static inline irq_ID_t virq_get_irq_id(
-	const virq_id_t		irq_ID,
-	unsigned int		*channel_ID)
-{
-	irq_ID_t ID;
-
-	assert(channel_ID != NULL);
-
-	for (ID = (irq_ID_t)0 ; ID < N_IRQ_ID; ID++) {
-		if (irq_ID < IRQ_N_ID_OFFSET[ID + 1]) {
-			break;
-		}
-	}
-
-	*channel_ID = (unsigned int)irq_ID - IRQ_N_ID_OFFSET[ID];
-
-	return ID;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq_local.h
deleted file mode 100644
index f522dfd1a9f1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq_local.h
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IRQ_LOCAL_H_INCLUDED__
-#define __IRQ_LOCAL_H_INCLUDED__
-
-#include "irq_global.h"
-
-#include <irq_controller_defs.h>
-
-/* IRQ0_ID */
-#include "hive_isp_css_defs.h"
-#define HIVE_GP_DEV_IRQ_NUM_IRQS	32
-/* IRQ1_ID */
-#include "input_formatter_subsystem_defs.h"
-#define HIVE_IFMT_IRQ_NUM_IRQS		5
-/* IRQ2_ID */
-#include "input_system_defs.h"
-/* IRQ3_ID */
-#include "input_selector_defs.h"
-
-
-#define	IRQ_ID_OFFSET	32
-#define	IRQ0_ID_OFFSET	0
-#define	IRQ1_ID_OFFSET	IRQ_ID_OFFSET
-#define	IRQ2_ID_OFFSET	(2*IRQ_ID_OFFSET)
-#define	IRQ3_ID_OFFSET	(3*IRQ_ID_OFFSET)
-#define	IRQ_END_OFFSET	(4*IRQ_ID_OFFSET)
-
-#define	IRQ0_ID_N_CHANNEL	HIVE_GP_DEV_IRQ_NUM_IRQS
-#define	IRQ1_ID_N_CHANNEL	HIVE_IFMT_IRQ_NUM_IRQS
-#define	IRQ2_ID_N_CHANNEL	HIVE_ISYS_IRQ_NUM_BITS
-#define	IRQ3_ID_N_CHANNEL	HIVE_ISEL_IRQ_NUM_IRQS
-
-typedef struct virq_info_s					virq_info_t;
-typedef struct irq_controller_state_s		irq_controller_state_t;
-
-
-typedef enum {
-	virq_gpio_pin_0            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_0_BIT_ID,
-	virq_gpio_pin_1            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_1_BIT_ID,
-	virq_gpio_pin_2            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_2_BIT_ID,
-	virq_gpio_pin_3            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_3_BIT_ID,
-	virq_gpio_pin_4            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_4_BIT_ID,
-	virq_gpio_pin_5            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_5_BIT_ID,
-	virq_gpio_pin_6            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_6_BIT_ID,
-	virq_gpio_pin_7            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_7_BIT_ID,
-	virq_gpio_pin_8            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_8_BIT_ID,
-	virq_gpio_pin_9            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_9_BIT_ID,
-	virq_gpio_pin_10           = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_10_BIT_ID,
-	virq_gpio_pin_11           = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_11_BIT_ID,
-	virq_sp                    = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_SP_BIT_ID,
-	virq_isp                   = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_BIT_ID,
-	virq_isys                  = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISYS_BIT_ID,
-	virq_isel                  = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISEL_BIT_ID,
-	virq_ifmt                  = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_IFMT_BIT_ID,
-	virq_sp_stream_mon         = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_SP_STREAM_MON_BIT_ID,
-	virq_isp_stream_mon        = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_STREAM_MON_BIT_ID,
-	virq_mod_stream_mon        = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_MOD_STREAM_MON_BIT_ID,
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM)
-	virq_isp_pmem_error        = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_PMEM_ERROR_BIT_ID,
-#elif defined(IS_ISP_2401_MAMOIADA_SYSTEM)
-	virq_isys_2401             = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_IS2401_BIT_ID,
-#else
-#error "irq_local.h: 2400_SYSTEM must be one of {2400, 2401 }"
-#endif
-	virq_isp_bamem_error       = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_BAMEM_ERROR_BIT_ID,
-	virq_isp_dmem_error        = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_DMEM_ERROR_BIT_ID,
-	virq_sp_icache_mem_error   = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_SP_ICACHE_MEM_ERROR_BIT_ID,
-	virq_sp_dmem_error         = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_SP_DMEM_ERROR_BIT_ID,
-	virq_mmu_cache_mem_error   = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_MMU_CACHE_MEM_ERROR_BIT_ID,
-	virq_gp_timer_0            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GP_TIMER_0_BIT_ID,
-	virq_gp_timer_1            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GP_TIMER_1_BIT_ID,              
-	virq_sw_pin_0              = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_SW_PIN_0_BIT_ID,
-	virq_sw_pin_1              = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_SW_PIN_1_BIT_ID,
-	virq_dma                   = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_DMA_BIT_ID,
-	virq_sp_stream_mon_b       = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_SP_STREAM_MON_B_BIT_ID,
-
-	virq_ifmt0_id              = IRQ1_ID_OFFSET + HIVE_IFMT_IRQ_IFT_PRIM_BIT_ID,
-	virq_ifmt1_id              = IRQ1_ID_OFFSET + HIVE_IFMT_IRQ_IFT_PRIM_B_BIT_ID,
-	virq_ifmt2_id              = IRQ1_ID_OFFSET + HIVE_IFMT_IRQ_IFT_SEC_BIT_ID,
-	virq_ifmt3_id              = IRQ1_ID_OFFSET + HIVE_IFMT_IRQ_MEM_CPY_BIT_ID,
-	virq_ifmt_sideband_changed = IRQ1_ID_OFFSET + HIVE_IFMT_IRQ_SIDEBAND_CHANGED_BIT_ID,
-
-	virq_isys_sof              = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CSI_SOF_BIT_ID,
-	virq_isys_eof              = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CSI_EOF_BIT_ID,
-	virq_isys_sol              = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CSI_SOL_BIT_ID,
-	virq_isys_eol              = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CSI_EOL_BIT_ID,
-	virq_isys_csi              = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CSI_RECEIVER_BIT_ID,
-	virq_isys_csi_be           = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CSI_RECEIVER_BE_BIT_ID,
-	virq_isys_capt0_id_no_sop  = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CAP_UNIT_A_NO_SOP,
-	virq_isys_capt0_id_late_sop= IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CAP_UNIT_A_LATE_SOP,
-	virq_isys_capt1_id_no_sop  = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CAP_UNIT_B_NO_SOP,
-	virq_isys_capt1_id_late_sop= IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CAP_UNIT_B_LATE_SOP,
-	virq_isys_capt2_id_no_sop  = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CAP_UNIT_C_NO_SOP,
-	virq_isys_capt2_id_late_sop= IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CAP_UNIT_C_LATE_SOP,
-	virq_isys_acq_sop_mismatch = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_ACQ_UNIT_SOP_MISMATCH,
-	virq_isys_ctrl_capt0       = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_INP_CTRL_CAPA,
-	virq_isys_ctrl_capt1       = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_INP_CTRL_CAPB,
-	virq_isys_ctrl_capt2       = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_INP_CTRL_CAPC,
-	virq_isys_cio_to_ahb       = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_CIO2AHB,
-	virq_isys_dma              = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_DMA_BIT_ID,
-	virq_isys_fifo_monitor     = IRQ2_ID_OFFSET + HIVE_ISYS_IRQ_STREAM_MON_BIT_ID,
-
-	virq_isel_sof              = IRQ3_ID_OFFSET + HIVE_ISEL_IRQ_SYNC_GEN_SOF_BIT_ID,
-	virq_isel_eof              = IRQ3_ID_OFFSET + HIVE_ISEL_IRQ_SYNC_GEN_EOF_BIT_ID,
-	virq_isel_sol              = IRQ3_ID_OFFSET + HIVE_ISEL_IRQ_SYNC_GEN_SOL_BIT_ID,
-	virq_isel_eol              = IRQ3_ID_OFFSET + HIVE_ISEL_IRQ_SYNC_GEN_EOL_BIT_ID,
-
-	N_virq_id                  = IRQ_END_OFFSET
-} virq_id_t;
-
-struct virq_info_s {
-	hrt_data		irq_status_reg[N_IRQ_ID];
-};
-
-struct irq_controller_state_s {
-	unsigned int	irq_edge;
-	unsigned int	irq_mask;
-	unsigned int	irq_status;
-	unsigned int	irq_enable;
-	unsigned int	irq_level_not_pulse;
-};
-
-#endif /* __IRQ_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq_private.h
deleted file mode 100644
index 23a13ac696c2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/irq_private.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IRQ_PRIVATE_H_INCLUDED__
-#define __IRQ_PRIVATE_H_INCLUDED__
-
-#include "irq_public.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-STORAGE_CLASS_IRQ_C void irq_reg_store(
-	const irq_ID_t		ID,
-	const unsigned int	reg,
-	const hrt_data		value)
-{
-	assert(ID < N_IRQ_ID);
-	assert(IRQ_BASE[ID] != (hrt_address)-1);
-	ia_css_device_store_uint32(IRQ_BASE[ID] + reg*sizeof(hrt_data), value);
-	return;
-}
-
-STORAGE_CLASS_IRQ_C hrt_data irq_reg_load(
-	const irq_ID_t		ID,
-	const unsigned int	reg)
-{
-	assert(ID < N_IRQ_ID);
-	assert(IRQ_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(IRQ_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-#endif /* __IRQ_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp.c
deleted file mode 100644
index 531c932a48f5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <system_global.h>
-#include "isp.h"
-
-#ifndef __INLINE_ISP__
-#include "isp_private.h"
-#endif /* __INLINE_ISP__ */
-
-#include "assert_support.h"
-#include "platform_support.h"			/* hrt_sleep() */
-
-void cnd_isp_irq_enable(
-	const isp_ID_t		ID,
-	const bool		cnd)
-{
-	if (cnd) {
-		isp_ctrl_setbit(ID, ISP_IRQ_READY_REG, ISP_IRQ_READY_BIT);
-/* Enabling the IRQ immediately triggers an interrupt, clear it */
-		isp_ctrl_setbit(ID, ISP_IRQ_CLEAR_REG, ISP_IRQ_CLEAR_BIT);
-	} else {
-		isp_ctrl_clearbit(ID, ISP_IRQ_READY_REG,
-			ISP_IRQ_READY_BIT);
-	}
-	return;
-}
-
-void isp_get_state(
-	const isp_ID_t		ID,
-	isp_state_t			*state,
-	isp_stall_t			*stall)
-{
-	hrt_data sc = isp_ctrl_load(ID, ISP_SC_REG);
-
-	assert(state != NULL);
-	assert(stall != NULL);
-
-#if defined(_hrt_sysmem_ident_address)
-	/* Patch to avoid compiler unused symbol warning in C_RUN build */
-	(void)__hrt_sysmem_ident_address;
-	(void)_hrt_sysmem_map_var;
-#endif
-
-	state->pc = isp_ctrl_load(ID, ISP_PC_REG);
-	state->status_register = sc;
-	state->is_broken = isp_ctrl_getbit(ID, ISP_SC_REG, ISP_BROKEN_BIT);
-	state->is_idle = isp_ctrl_getbit(ID, ISP_SC_REG, ISP_IDLE_BIT);
-	state->is_sleeping = isp_ctrl_getbit(ID, ISP_SC_REG, ISP_SLEEPING_BIT);
-	state->is_stalling = isp_ctrl_getbit(ID, ISP_SC_REG, ISP_STALLING_BIT);
-	stall->stat_ctrl =
-		!isp_ctrl_getbit(ID, ISP_CTRL_SINK_REG, ISP_CTRL_SINK_BIT);
-	stall->pmem =
-		!isp_ctrl_getbit(ID, ISP_PMEM_SINK_REG, ISP_PMEM_SINK_BIT);
-	stall->dmem =
-		!isp_ctrl_getbit(ID, ISP_DMEM_SINK_REG, ISP_DMEM_SINK_BIT);
-	stall->vmem =
-		!isp_ctrl_getbit(ID, ISP_VMEM_SINK_REG, ISP_VMEM_SINK_BIT);
-	stall->fifo0 =
-		!isp_ctrl_getbit(ID, ISP_FIFO0_SINK_REG, ISP_FIFO0_SINK_BIT);
-	stall->fifo1 =
-		!isp_ctrl_getbit(ID, ISP_FIFO1_SINK_REG, ISP_FIFO1_SINK_BIT);
-	stall->fifo2 =
-		!isp_ctrl_getbit(ID, ISP_FIFO2_SINK_REG, ISP_FIFO2_SINK_BIT);
-	stall->fifo3 =
-		!isp_ctrl_getbit(ID, ISP_FIFO3_SINK_REG, ISP_FIFO3_SINK_BIT);
-	stall->fifo4 =
-		!isp_ctrl_getbit(ID, ISP_FIFO4_SINK_REG, ISP_FIFO4_SINK_BIT);
-	stall->fifo5 =
-		!isp_ctrl_getbit(ID, ISP_FIFO5_SINK_REG, ISP_FIFO5_SINK_BIT);
-	stall->fifo6 =
-		!isp_ctrl_getbit(ID, ISP_FIFO6_SINK_REG, ISP_FIFO6_SINK_BIT);
-	stall->vamem1 =
-		!isp_ctrl_getbit(ID, ISP_VAMEM1_SINK_REG, ISP_VAMEM1_SINK_BIT);
-	stall->vamem2 =
-		!isp_ctrl_getbit(ID, ISP_VAMEM2_SINK_REG, ISP_VAMEM2_SINK_BIT);
-	stall->vamem3 =
-		!isp_ctrl_getbit(ID, ISP_VAMEM3_SINK_REG, ISP_VAMEM3_SINK_BIT);
-	stall->hmem =
-		!isp_ctrl_getbit(ID, ISP_HMEM_SINK_REG, ISP_HMEM_SINK_BIT);
-/*
-	stall->icache_master =
-		!isp_ctrl_getbit(ID, ISP_ICACHE_MT_SINK_REG,
-			ISP_ICACHE_MT_SINK_BIT);
- */
-	return;
-}
-
-/* ISP functions to control the ISP state from the host, even in crun. */
-
-/* Inspect readiness of an ISP indexed by ID */
-unsigned isp_is_ready(isp_ID_t ID)
-{
-	assert (ID < N_ISP_ID);
-	return isp_ctrl_getbit(ID, ISP_SC_REG, ISP_IDLE_BIT);
-}
-
-/* Inspect sleeping of an ISP indexed by ID */
-unsigned isp_is_sleeping(isp_ID_t ID)
-{
-	assert (ID < N_ISP_ID);
-	return isp_ctrl_getbit(ID, ISP_SC_REG, ISP_SLEEPING_BIT);
-}
-
-/* To be called by the host immediately before starting ISP ID. */
-void isp_start(isp_ID_t ID)
-{
-	assert (ID < N_ISP_ID);
-}
-
-/* Wake up ISP ID. */
-void isp_wake(isp_ID_t ID)
-{
-	assert (ID < N_ISP_ID);
-	isp_ctrl_setbit(ID, ISP_SC_REG, ISP_START_BIT);
-	hrt_sleep();
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp_local.h
deleted file mode 100644
index 5dcc52dff3dd..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp_local.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISP_LOCAL_H_INCLUDED__
-#define __ISP_LOCAL_H_INCLUDED__
-
-#include <stdbool.h>
-
-#include "isp_global.h"
-
-#include <isp2400_support.h>
-
-#define HIVE_ISP_VMEM_MASK	((1U<<ISP_VMEM_ELEMBITS)-1)
-
-typedef struct isp_state_s		isp_state_t;
-typedef struct isp_stall_s		isp_stall_t;
-
-struct isp_state_s {
-	int	pc;
-	int	status_register;
-	bool	is_broken;
-	bool	is_idle;
-	bool	is_sleeping;
-	bool	is_stalling;
-};
-
-struct isp_stall_s {
-	bool	fifo0;
-	bool	fifo1;
-	bool	fifo2;
-	bool	fifo3;
-	bool	fifo4;
-	bool	fifo5;
-	bool	fifo6;
-	bool	stat_ctrl;
-	bool	dmem;
-	bool	vmem;
-	bool	vamem1;
-	bool	vamem2;
-	bool	vamem3;
-	bool	hmem;
-	bool	pmem;
-	bool	icache_master;
-};
-
-#endif /* __ISP_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp_private.h
deleted file mode 100644
index 7f63255d3fc5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/isp_private.h
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISP_PRIVATE_H_INCLUDED__
-#define __ISP_PRIVATE_H_INCLUDED__
-
-#ifdef HRT_MEMORY_ACCESS
-#include <hrt/api.h>
-#endif
-
-#include "isp_public.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-#include "type_support.h"
-
-STORAGE_CLASS_ISP_C void isp_ctrl_store(
-	const isp_ID_t		ID,
-	const unsigned int	reg,
-	const hrt_data		value)
-{
-	assert(ID < N_ISP_ID);
-	assert(ISP_CTRL_BASE[ID] != (hrt_address)-1);
-#if !defined(HRT_MEMORY_ACCESS)
-	ia_css_device_store_uint32(ISP_CTRL_BASE[ID] + reg*sizeof(hrt_data), value);
-#else
-	hrt_master_port_store_32(ISP_CTRL_BASE[ID] + reg*sizeof(hrt_data), value);
-#endif
-	return;
-}
-
-STORAGE_CLASS_ISP_C hrt_data isp_ctrl_load(
-	const isp_ID_t		ID,
-	const unsigned int	reg)
-{
-	assert(ID < N_ISP_ID);
-	assert(ISP_CTRL_BASE[ID] != (hrt_address)-1);
-#if !defined(HRT_MEMORY_ACCESS)
-	return ia_css_device_load_uint32(ISP_CTRL_BASE[ID] + reg*sizeof(hrt_data));
-#else
-	return hrt_master_port_uload_32(ISP_CTRL_BASE[ID] + reg*sizeof(hrt_data));
-#endif
-}
-
-STORAGE_CLASS_ISP_C bool isp_ctrl_getbit(
-	const isp_ID_t		ID,
-	const unsigned int	reg,
-	const unsigned int	bit)
-{
-	hrt_data val = isp_ctrl_load(ID, reg);
-	return (val & (1UL << bit)) != 0;
-}
-
-STORAGE_CLASS_ISP_C void isp_ctrl_setbit(
-	const isp_ID_t		ID,
-	const unsigned int	reg,
-	const unsigned int	bit)
-{
-	hrt_data	data = isp_ctrl_load(ID, reg);
-	isp_ctrl_store(ID, reg, (data | (1UL << bit)));
-	return;
-}
-
-STORAGE_CLASS_ISP_C void isp_ctrl_clearbit(
-	const isp_ID_t		ID,
-	const unsigned int	reg,
-	const unsigned int	bit)
-{
-	hrt_data	data = isp_ctrl_load(ID, reg);
-	isp_ctrl_store(ID, reg, (data & ~(1UL << bit)));
-	return;
-}
-
-STORAGE_CLASS_ISP_C void isp_dmem_store(
-	const isp_ID_t		ID,
-	unsigned int		addr,
-	const void		*data,
-	const size_t		size)
-{
-	assert(ID < N_ISP_ID);
-	assert(ISP_DMEM_BASE[ID] != (hrt_address)-1);
-#if !defined(HRT_MEMORY_ACCESS)
-	ia_css_device_store(ISP_DMEM_BASE[ID] + addr, data, size);
-#else
-	hrt_master_port_store(ISP_DMEM_BASE[ID] + addr, data, size);
-#endif
-	return;
-}
-
-STORAGE_CLASS_ISP_C void isp_dmem_load(
-	const isp_ID_t		ID,
-	const unsigned int	addr,
-	void			*data,
-	const size_t		size)
-{
-	assert(ID < N_ISP_ID);
-	assert(ISP_DMEM_BASE[ID] != (hrt_address)-1);
-#if !defined(HRT_MEMORY_ACCESS)
-	ia_css_device_load(ISP_DMEM_BASE[ID] + addr, data, size);
-#else
-	hrt_master_port_load(ISP_DMEM_BASE[ID] + addr, data, size);
-#endif
-	return;
-}
-
-STORAGE_CLASS_ISP_C void isp_dmem_store_uint32(
-	const isp_ID_t		ID,
-	unsigned int		addr,
-	const uint32_t		data)
-{
-	assert(ID < N_ISP_ID);
-	assert(ISP_DMEM_BASE[ID] != (hrt_address)-1);
-	(void)ID;
-#if !defined(HRT_MEMORY_ACCESS)
-	ia_css_device_store_uint32(ISP_DMEM_BASE[ID] + addr, data);
-#else
-	hrt_master_port_store_32(ISP_DMEM_BASE[ID] + addr, data);
-#endif
-	return;
-}
-
-STORAGE_CLASS_ISP_C uint32_t isp_dmem_load_uint32(
-	const isp_ID_t		ID,
-	const unsigned int	addr)
-{
-	assert(ID < N_ISP_ID);
-	assert(ISP_DMEM_BASE[ID] != (hrt_address)-1);
-	(void)ID;
-#if !defined(HRT_MEMORY_ACCESS)
-	return ia_css_device_load_uint32(ISP_DMEM_BASE[ID] + addr);
-#else
-	return hrt_master_port_uload_32(ISP_DMEM_BASE[ID] + addr);
-#endif
-}
-
-STORAGE_CLASS_ISP_C uint32_t isp_2w_cat_1w(
-	const uint16_t		x0,
-	const uint16_t		x1)
-{
-	uint32_t out = ((uint32_t)(x1 & HIVE_ISP_VMEM_MASK) << ISP_VMEM_ELEMBITS)
-		| (x0 & HIVE_ISP_VMEM_MASK);
-	return out;
-}
-
-#endif /* __ISP_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu.c
deleted file mode 100644
index a28b67eb66ea..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu.c
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* The name "mmu.h is already taken" */
-#include "mmu_device.h"
-
-#ifndef __INLINE_MMU__
-#include "mmu_private.h"
-#endif /* __INLINE_MMU__ */
-
-void mmu_set_page_table_base_index(
-	const mmu_ID_t		ID,
-	const hrt_data		base_index)
-{
-	mmu_reg_store(ID, _HRT_MMU_PAGE_TABLE_BASE_ADDRESS_REG_IDX, base_index);
-	return;
-}
-
-hrt_data mmu_get_page_table_base_index(
-	const mmu_ID_t		ID)
-{
-	return mmu_reg_load(ID, _HRT_MMU_PAGE_TABLE_BASE_ADDRESS_REG_IDX);
-}
-
-void mmu_invalidate_cache(
-	const mmu_ID_t		ID)
-{
-	mmu_reg_store(ID, _HRT_MMU_INVALIDATE_TLB_REG_IDX, 1);
-	return;
-}
-
-void mmu_invalidate_cache_all(void)
-{
-	mmu_ID_t	mmu_id;
-	for (mmu_id = (mmu_ID_t)0;mmu_id < N_MMU_ID; mmu_id++) {
-		mmu_invalidate_cache(mmu_id);
-	}
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu_local.h
deleted file mode 100644
index 7c3ad157189f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu_local.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __MMU_LOCAL_H_INCLUDED__
-#define __MMU_LOCAL_H_INCLUDED__
-
-#include "mmu_global.h"
-
-#endif /* __MMU_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu_private.h
deleted file mode 100644
index 7377666f6eb7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/mmu_private.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __MMU_PRIVATE_H_INCLUDED__
-#define __MMU_PRIVATE_H_INCLUDED__
-
-#include "mmu_public.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-STORAGE_CLASS_MMU_H void mmu_reg_store(
-	const mmu_ID_t		ID,
-	const unsigned int	reg,
-	const hrt_data		value)
-{
-	assert(ID < N_MMU_ID);
-	assert(MMU_BASE[ID] != (hrt_address)-1);
-	ia_css_device_store_uint32(MMU_BASE[ID] + reg*sizeof(hrt_data), value);
-	return;
-}
-
-STORAGE_CLASS_MMU_H hrt_data mmu_reg_load(
-	const mmu_ID_t		ID,
-	const unsigned int	reg)
-{
-	assert(ID < N_MMU_ID);
-	assert(MMU_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(MMU_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-#endif /* __MMU_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp.c
deleted file mode 100644
index db694d3a6fbb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "sp.h"
-
-#ifndef __INLINE_SP__
-#include "sp_private.h"
-#endif /* __INLINE_SP__ */
-
-#include "assert_support.h"
-
-void cnd_sp_irq_enable(
-	const sp_ID_t		ID,
-	const bool		cnd)
-{
-	if (cnd) {
-		sp_ctrl_setbit(ID, SP_IRQ_READY_REG, SP_IRQ_READY_BIT);
-/* Enabling the IRQ immediately triggers an interrupt, clear it */
-		sp_ctrl_setbit(ID, SP_IRQ_CLEAR_REG, SP_IRQ_CLEAR_BIT);
-	} else {
-		sp_ctrl_clearbit(ID, SP_IRQ_READY_REG, SP_IRQ_READY_BIT);
-	}
-}
-
-void sp_get_state(
-	const sp_ID_t			ID,
-	sp_state_t				*state,
-	sp_stall_t				*stall)
-{
-	hrt_data sc = sp_ctrl_load(ID, SP_SC_REG);
-
-	assert(state != NULL);
-	assert(stall != NULL);
-
-	state->pc = sp_ctrl_load(ID, SP_PC_REG);
-	state->status_register = sc;
-	state->is_broken   = (sc & (1U << SP_BROKEN_BIT)) != 0;
-	state->is_idle     = (sc & (1U << SP_IDLE_BIT)) != 0;
-	state->is_sleeping = (sc & (1U << SP_SLEEPING_BIT)) != 0;
-	state->is_stalling = (sc & (1U << SP_STALLING_BIT)) != 0;
-	stall->fifo0 =
-		!sp_ctrl_getbit(ID, SP_FIFO0_SINK_REG, SP_FIFO0_SINK_BIT);
-	stall->fifo1 =
-		!sp_ctrl_getbit(ID, SP_FIFO1_SINK_REG, SP_FIFO1_SINK_BIT);
-	stall->fifo2 =
-		!sp_ctrl_getbit(ID, SP_FIFO2_SINK_REG, SP_FIFO2_SINK_BIT);
-	stall->fifo3 =
-		!sp_ctrl_getbit(ID, SP_FIFO3_SINK_REG, SP_FIFO3_SINK_BIT);
-	stall->fifo4 =
-		!sp_ctrl_getbit(ID, SP_FIFO4_SINK_REG, SP_FIFO4_SINK_BIT);
-	stall->fifo5 =
-		!sp_ctrl_getbit(ID, SP_FIFO5_SINK_REG, SP_FIFO5_SINK_BIT);
-	stall->fifo6 =
-		!sp_ctrl_getbit(ID, SP_FIFO6_SINK_REG, SP_FIFO6_SINK_BIT);
-	stall->fifo7 =
-		!sp_ctrl_getbit(ID, SP_FIFO7_SINK_REG, SP_FIFO7_SINK_BIT);
-	stall->fifo8 =
-		!sp_ctrl_getbit(ID, SP_FIFO8_SINK_REG, SP_FIFO8_SINK_BIT);
-	stall->fifo9 =
-		!sp_ctrl_getbit(ID, SP_FIFO9_SINK_REG, SP_FIFO9_SINK_BIT);
-	stall->fifoa =
-		!sp_ctrl_getbit(ID, SP_FIFOA_SINK_REG, SP_FIFOA_SINK_BIT);
-	stall->dmem =
-		!sp_ctrl_getbit(ID, SP_DMEM_SINK_REG, SP_DMEM_SINK_BIT);
-	stall->control_master =
-		!sp_ctrl_getbit(ID, SP_CTRL_MT_SINK_REG, SP_CTRL_MT_SINK_BIT);
-	stall->icache_master =
-		!sp_ctrl_getbit(ID, SP_ICACHE_MT_SINK_REG,
-			SP_ICACHE_MT_SINK_BIT);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp_local.h
deleted file mode 100644
index 3c70b8fdb532..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp_local.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SP_LOCAL_H_INCLUDED__
-#define __SP_LOCAL_H_INCLUDED__
-
-#include <type_support.h>
-#include "sp_global.h"
-
-struct sp_state_s {
-	int		pc;
-	int		status_register;
-	bool	is_broken;
-	bool	is_idle;
-	bool	is_sleeping;
-	bool	is_stalling;
-};
-
-struct sp_stall_s {
-	bool	fifo0;
-	bool	fifo1;
-	bool	fifo2;
-	bool	fifo3;
-	bool	fifo4;
-	bool	fifo5;
-	bool	fifo6;
-	bool	fifo7;
-	bool	fifo8;
-	bool	fifo9;
-	bool	fifoa;
-	bool	dmem;
-	bool	control_master;
-	bool	icache_master;
-};
-
-#define sp_address_of(var)	(HIVE_ADDR_ ## var)
-
-/*
- * deprecated
- */
-#define store_sp_int(var, value) \
-	sp_dmem_store_uint32(SP0_ID, (unsigned)sp_address_of(var), \
-		(uint32_t)(value))
-
-#define store_sp_ptr(var, value) \
-	sp_dmem_store_uint32(SP0_ID, (unsigned)sp_address_of(var), \
-		(uint32_t)(value))
-
-#define load_sp_uint(var) \
-	sp_dmem_load_uint32(SP0_ID, (unsigned)sp_address_of(var))
-
-#define load_sp_array_uint8(array_name, index) \
-	sp_dmem_load_uint8(SP0_ID, (unsigned)sp_address_of(array_name) + \
-		(index)*sizeof(uint8_t))
-
-#define load_sp_array_uint16(array_name, index) \
-	sp_dmem_load_uint16(SP0_ID, (unsigned)sp_address_of(array_name) + \
-		(index)*sizeof(uint16_t))
-
-#define load_sp_array_uint(array_name, index) \
-	sp_dmem_load_uint32(SP0_ID, (unsigned)sp_address_of(array_name) + \
-		(index)*sizeof(uint32_t))
-
-#define store_sp_var(var, data, bytes) \
-	sp_dmem_store(SP0_ID, (unsigned)sp_address_of(var), data, bytes)
-
-#define store_sp_array_uint8(array_name, index, value) \
-	sp_dmem_store_uint8(SP0_ID, (unsigned)sp_address_of(array_name) + \
-		(index)*sizeof(uint8_t), value)
-
-#define store_sp_array_uint16(array_name, index, value) \
-	sp_dmem_store_uint16(SP0_ID, (unsigned)sp_address_of(array_name) + \
-		(index)*sizeof(uint16_t), value)
-
-#define store_sp_array_uint(array_name, index, value) \
-	sp_dmem_store_uint32(SP0_ID, (unsigned)sp_address_of(array_name) + \
-		(index)*sizeof(uint32_t), value)
-
-#define store_sp_var_with_offset(var, offset, data, bytes) \
-	sp_dmem_store(SP0_ID, (unsigned)sp_address_of(var) + \
-		offset, data, bytes)
-
-#define load_sp_var(var, data, bytes) \
-	sp_dmem_load(SP0_ID, (unsigned)sp_address_of(var), data, bytes)
-
-#define load_sp_var_with_offset(var, offset, data, bytes) \
-	sp_dmem_load(SP0_ID, (unsigned)sp_address_of(var) + offset, \
-		data, bytes)
-
-#endif /* __SP_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp_private.h
deleted file mode 100644
index 5ea81c0e82d1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/sp_private.h
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SP_PRIVATE_H_INCLUDED__
-#define __SP_PRIVATE_H_INCLUDED__
-
-#include "sp_public.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-STORAGE_CLASS_SP_C void sp_ctrl_store(
-	const sp_ID_t		ID,
-	const hrt_address	reg,
-	const hrt_data		value)
-{
-	assert(ID < N_SP_ID);
-	assert(SP_CTRL_BASE[ID] != (hrt_address)-1);
-	ia_css_device_store_uint32(SP_CTRL_BASE[ID] + reg*sizeof(hrt_data), value);
-	return;
-}
-
-STORAGE_CLASS_SP_C hrt_data sp_ctrl_load(
-	const sp_ID_t		ID,
-	const hrt_address	reg)
-{
-	assert(ID < N_SP_ID);
-	assert(SP_CTRL_BASE[ID] != (hrt_address)-1);
-	return ia_css_device_load_uint32(SP_CTRL_BASE[ID] + reg*sizeof(hrt_data));
-}
-
-STORAGE_CLASS_SP_C bool sp_ctrl_getbit(
-	const sp_ID_t		ID,
-	const hrt_address	reg,
-	const unsigned int	bit)
-{
-	hrt_data val = sp_ctrl_load(ID, reg);
-	return (val & (1UL << bit)) != 0;
-}
-
-STORAGE_CLASS_SP_C void sp_ctrl_setbit(
-	const sp_ID_t		ID,
-	const hrt_address	reg,
-	const unsigned int	bit)
-{
-	hrt_data	data = sp_ctrl_load(ID, reg);
-	sp_ctrl_store(ID, reg, (data | (1UL << bit)));
-	return;
-}
-
-STORAGE_CLASS_SP_C void sp_ctrl_clearbit(
-	const sp_ID_t		ID,
-	const hrt_address	reg,
-	const unsigned int	bit)
-{
-	hrt_data	data = sp_ctrl_load(ID, reg);
-	sp_ctrl_store(ID, reg, (data & ~(1UL << bit)));
-	return;
-}
-
-STORAGE_CLASS_SP_C void sp_dmem_store(
-	const sp_ID_t		ID,
-	hrt_address		addr,
-	const void			*data,
-	const size_t		size)
-{
-	assert(ID < N_SP_ID);
-	assert(SP_DMEM_BASE[ID] != (hrt_address)-1);
-	ia_css_device_store(SP_DMEM_BASE[ID] + addr, data, size);
-	return;
-}
-
-STORAGE_CLASS_SP_C void sp_dmem_load(
-	const sp_ID_t		ID,
-	const hrt_address	addr,
-	void				*data,
-	const size_t		size)
-{
-	assert(ID < N_SP_ID);
-	assert(SP_DMEM_BASE[ID] != (hrt_address)-1);
-	ia_css_device_load(SP_DMEM_BASE[ID] + addr, data, size);
-	return;
-}
-
-STORAGE_CLASS_SP_C void sp_dmem_store_uint8(
-	const sp_ID_t		ID,
-	hrt_address		addr,
-	const uint8_t		data)
-{
-	assert(ID < N_SP_ID);
-	assert(SP_DMEM_BASE[ID] != (hrt_address)-1);
-	(void)ID;
-	ia_css_device_store_uint8(SP_DMEM_BASE[SP0_ID] + addr, data);
-	return;
-}
-
-STORAGE_CLASS_SP_C void sp_dmem_store_uint16(
-	const sp_ID_t		ID,
-	hrt_address		addr,
-	const uint16_t		data)
-{
-	assert(ID < N_SP_ID);
-	assert(SP_DMEM_BASE[ID] != (hrt_address)-1);
-	(void)ID;
-	ia_css_device_store_uint16(SP_DMEM_BASE[SP0_ID] + addr, data);
-	return;
-}
-
-STORAGE_CLASS_SP_C void sp_dmem_store_uint32(
-	const sp_ID_t		ID,
-	hrt_address		addr,
-	const uint32_t		data)
-{
-	assert(ID < N_SP_ID);
-	assert(SP_DMEM_BASE[ID] != (hrt_address)-1);
-	(void)ID;
-	ia_css_device_store_uint32(SP_DMEM_BASE[SP0_ID] + addr, data);
-	return;
-}
-
-STORAGE_CLASS_SP_C uint8_t sp_dmem_load_uint8(
-	const sp_ID_t		ID,
-	const hrt_address	addr)
-{
-	assert(ID < N_SP_ID);
-	assert(SP_DMEM_BASE[ID] != (hrt_address)-1);
-	(void)ID;
-	return ia_css_device_load_uint8(SP_DMEM_BASE[SP0_ID] + addr);
-}
-
-STORAGE_CLASS_SP_C uint16_t sp_dmem_load_uint16(
-	const sp_ID_t		ID,
-	const hrt_address	addr)
-{
-	assert(ID < N_SP_ID);
-	assert(SP_DMEM_BASE[ID] != (hrt_address)-1);
-	(void)ID;
-	return ia_css_device_load_uint16(SP_DMEM_BASE[SP0_ID] + addr);
-}
-
-STORAGE_CLASS_SP_C uint32_t sp_dmem_load_uint32(
-	const sp_ID_t		ID,
-	const hrt_address	addr)
-{
-	assert(ID < N_SP_ID);
-	assert(SP_DMEM_BASE[ID] != (hrt_address)-1);
-	(void)ID;
-	return ia_css_device_load_uint32(SP_DMEM_BASE[SP0_ID] + addr);
-}
-
-#endif /* __SP_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/system_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/system_local.h
deleted file mode 100644
index 111b346dfafb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/system_local.h
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SYSTEM_LOCAL_H_INCLUDED__
-#define __SYSTEM_LOCAL_H_INCLUDED__
-
-#ifdef HRT_ISP_CSS_CUSTOM_HOST
-#ifndef HRT_USE_VIR_ADDRS
-#define HRT_USE_VIR_ADDRS
-#endif
-/* This interface is deprecated */
-/*#include "hive_isp_css_custom_host_hrt.h"*/
-#endif
-
-#include "system_global.h"
-
-#ifdef __FIST__
-#define HRT_ADDRESS_WIDTH	32		/* Surprise, this is a local property and even differs per platform */
-#else
-/* HRT assumes 32 by default (see Linux/include/hrt/hive_types.h), overrule it in case it is different */
-#undef HRT_ADDRESS_WIDTH
-#define HRT_ADDRESS_WIDTH	64		/* Surprise, this is a local property */
-#endif
-
-#if !defined(__KERNEL__) || (1==1)
-/* This interface is deprecated */
-#include "hrt/hive_types.h"
-#else  /* __KERNEL__ */
-#include <linux/types.h>
-
-#if HRT_ADDRESS_WIDTH==64
-typedef uint64_t			hrt_address;
-#elif HRT_ADDRESS_WIDTH==32
-typedef uint32_t			hrt_address;
-#else
-#error "system_local.h: HRT_ADDRESS_WIDTH must be one of {32,64}"
-#endif
-
-typedef uint32_t			hrt_vaddress;
-typedef uint32_t			hrt_data;
-#endif /* __KERNEL__ */
-
-/*
- * Cell specific address maps
- */
-#if HRT_ADDRESS_WIDTH==64
-
-#define GP_FIFO_BASE   ((hrt_address)0x0000000000090104)		/* This is NOT a base address */
-
-/* DDR */
-static const hrt_address DDR_BASE[N_DDR_ID] = {
-	(hrt_address)0x0000000120000000ULL};
-
-/* ISP */
-static const hrt_address ISP_CTRL_BASE[N_ISP_ID] = {
-	(hrt_address)0x0000000000020000ULL};
-
-static const hrt_address ISP_DMEM_BASE[N_ISP_ID] = {
-	(hrt_address)0x0000000000200000ULL};
-
-static const hrt_address ISP_BAMEM_BASE[N_BAMEM_ID] = {
-	(hrt_address)0x0000000000100000ULL};
-
-static const hrt_address ISP_VAMEM_BASE[N_VAMEM_ID] = {
-	(hrt_address)0x00000000001C0000ULL,
-	(hrt_address)0x00000000001D0000ULL,
-	(hrt_address)0x00000000001E0000ULL};
-
-static const hrt_address ISP_HMEM_BASE[N_HMEM_ID] = {
-	(hrt_address)0x00000000001F0000ULL};
-
-/* SP */
-static const hrt_address SP_CTRL_BASE[N_SP_ID] = {
-	(hrt_address)0x0000000000010000ULL};
-
-static const hrt_address SP_DMEM_BASE[N_SP_ID] = {
-	(hrt_address)0x0000000000300000ULL};
-
-static const hrt_address SP_PMEM_BASE[N_SP_ID] = {
-	(hrt_address)0x00000000000B0000ULL};
-
-/* MMU */
-#if defined (IS_ISP_2400_MAMOIADA_SYSTEM) || defined (IS_ISP_2401_MAMOIADA_SYSTEM)
-/*
- * MMU0_ID: The data MMU
- * MMU1_ID: The icache MMU
- */
-static const hrt_address MMU_BASE[N_MMU_ID] = {
-	(hrt_address)0x0000000000070000ULL,
-	(hrt_address)0x00000000000A0000ULL};
-#else
-#error "system_local.h: SYSTEM must be one of {2400, 2401 }"
-#endif
-
-/* DMA */
-static const hrt_address DMA_BASE[N_DMA_ID] = {
-	(hrt_address)0x0000000000040000ULL};
-
-/* IRQ */
-static const hrt_address IRQ_BASE[N_IRQ_ID] = {
-	(hrt_address)0x0000000000000500ULL,
-	(hrt_address)0x0000000000030A00ULL,
-	(hrt_address)0x000000000008C000ULL,
-	(hrt_address)0x0000000000090200ULL};
-/*
-	(hrt_address)0x0000000000000500ULL};
- */
-
-/* GDC */
-static const hrt_address GDC_BASE[N_GDC_ID] = {
-	(hrt_address)0x0000000000050000ULL,
-	(hrt_address)0x0000000000060000ULL};
-
-/* FIFO_MONITOR (not a subset of GP_DEVICE) */
-static const hrt_address FIFO_MONITOR_BASE[N_FIFO_MONITOR_ID] = {
-	(hrt_address)0x0000000000000000ULL};
-
-/*
-static const hrt_address GP_REGS_BASE[N_GP_REGS_ID] = {
-	(hrt_address)0x0000000000000000ULL};
-
-static const hrt_address GP_DEVICE_BASE[N_GP_DEVICE_ID] = {
-	(hrt_address)0x0000000000090000ULL};
-*/
-
-/* GP_DEVICE (single base for all separate GP_REG instances) */
-static const hrt_address GP_DEVICE_BASE[N_GP_DEVICE_ID] = {
-	(hrt_address)0x0000000000000000ULL};
-
-/*GP TIMER , all timer registers are inter-twined,
- * so, having multiple base addresses for
- * different timers does not help*/
-static const hrt_address GP_TIMER_BASE =
-	(hrt_address)0x0000000000000600ULL;
-/* GPIO */
-static const hrt_address GPIO_BASE[N_GPIO_ID] = {
-	(hrt_address)0x0000000000000400ULL};
-
-/* TIMED_CTRL */
-static const hrt_address TIMED_CTRL_BASE[N_TIMED_CTRL_ID] = {
-	(hrt_address)0x0000000000000100ULL};
-
-
-/* INPUT_FORMATTER */
-static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = {
-	(hrt_address)0x0000000000030000ULL,
-	(hrt_address)0x0000000000030200ULL,
-	(hrt_address)0x0000000000030400ULL,
-	(hrt_address)0x0000000000030600ULL}; /* memcpy() */
-
-/* INPUT_SYSTEM */
-static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = {
-	(hrt_address)0x0000000000080000ULL};
-/*	(hrt_address)0x0000000000081000ULL, */ /* capture A */
-/*	(hrt_address)0x0000000000082000ULL, */ /* capture B */
-/*	(hrt_address)0x0000000000083000ULL, */ /* capture C */
-/*	(hrt_address)0x0000000000084000ULL, */ /* Acquisition */
-/*	(hrt_address)0x0000000000085000ULL, */ /* DMA */
-/*	(hrt_address)0x0000000000089000ULL, */ /* ctrl */
-/*	(hrt_address)0x000000000008A000ULL, */ /* GP regs */
-/*	(hrt_address)0x000000000008B000ULL, */ /* FIFO */
-/*	(hrt_address)0x000000000008C000ULL, */ /* IRQ */
-
-/* RX, the MIPI lane control regs start at offset 0 */
-static const hrt_address RX_BASE[N_RX_ID] = {
-	(hrt_address)0x0000000000080100ULL};
-
-#elif HRT_ADDRESS_WIDTH==32
-
-#define GP_FIFO_BASE   ((hrt_address)0x00090104)		/* This is NOT a base address */
-
-/* DDR : Attention, this value not defined in 32-bit */
-static const hrt_address DDR_BASE[N_DDR_ID] = {
-	(hrt_address)0x00000000UL};
-
-/* ISP */
-static const hrt_address ISP_CTRL_BASE[N_ISP_ID] = {
-	(hrt_address)0x00020000UL};
-
-static const hrt_address ISP_DMEM_BASE[N_ISP_ID] = {
-	(hrt_address)0x00200000UL};
-
-static const hrt_address ISP_BAMEM_BASE[N_BAMEM_ID] = {
-	(hrt_address)0x100000UL};
-
-static const hrt_address ISP_VAMEM_BASE[N_VAMEM_ID] = {
-	(hrt_address)0xffffffffUL,
-	(hrt_address)0xffffffffUL,
-	(hrt_address)0xffffffffUL};
-
-static const hrt_address ISP_HMEM_BASE[N_HMEM_ID] = {
-	(hrt_address)0xffffffffUL};
-
-/* SP */
-static const hrt_address SP_CTRL_BASE[N_SP_ID] = {
-	(hrt_address)0x00010000UL};
-
-static const hrt_address SP_DMEM_BASE[N_SP_ID] = {
-	(hrt_address)0x00300000UL};
-
-static const hrt_address SP_PMEM_BASE[N_SP_ID] = {
-	(hrt_address)0x000B0000UL};
-
-/* MMU */
-#if defined (IS_ISP_2400_MAMOIADA_SYSTEM) || defined (IS_ISP_2401_MAMOIADA_SYSTEM)
-/*
- * MMU0_ID: The data MMU
- * MMU1_ID: The icache MMU
- */
-static const hrt_address MMU_BASE[N_MMU_ID] = {
-	(hrt_address)0x00070000UL,
-	(hrt_address)0x000A0000UL};
-#else
-#error "system_local.h: SYSTEM must be one of {2400, 2401 }"
-#endif
-
-/* DMA */
-static const hrt_address DMA_BASE[N_DMA_ID] = {
-	(hrt_address)0x00040000UL};
-
-/* IRQ */
-static const hrt_address IRQ_BASE[N_IRQ_ID] = {
-	(hrt_address)0x00000500UL,
-	(hrt_address)0x00030A00UL,
-	(hrt_address)0x0008C000UL,
-	(hrt_address)0x00090200UL};
-/*
-	(hrt_address)0x00000500UL};
- */
-
-/* GDC */
-static const hrt_address GDC_BASE[N_GDC_ID] = {
-	(hrt_address)0x00050000UL,
-	(hrt_address)0x00060000UL};
-
-/* FIFO_MONITOR (not a subset of GP_DEVICE) */
-static const hrt_address FIFO_MONITOR_BASE[N_FIFO_MONITOR_ID] = {
-	(hrt_address)0x00000000UL};
-
-/*
-static const hrt_address GP_REGS_BASE[N_GP_REGS_ID] = {
-	(hrt_address)0x00000000UL};
-
-static const hrt_address GP_DEVICE_BASE[N_GP_DEVICE_ID] = {
-	(hrt_address)0x00090000UL};
-*/
-
-/* GP_DEVICE (single base for all separate GP_REG instances) */
-static const hrt_address GP_DEVICE_BASE[N_GP_DEVICE_ID] = {
-	(hrt_address)0x00000000UL};
-
-/*GP TIMER , all timer registers are inter-twined,
- * so, having multiple base addresses for
- * different timers does not help*/
-static const hrt_address GP_TIMER_BASE =
-	(hrt_address)0x00000600UL;
-
-/* GPIO */
-static const hrt_address GPIO_BASE[N_GPIO_ID] = {
-	(hrt_address)0x00000400UL};
-
-/* TIMED_CTRL */
-static const hrt_address TIMED_CTRL_BASE[N_TIMED_CTRL_ID] = {
-	(hrt_address)0x00000100UL};
-
-
-/* INPUT_FORMATTER */
-static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = {
-	(hrt_address)0x00030000UL,
-	(hrt_address)0x00030200UL,
-	(hrt_address)0x00030400UL};
-/*	(hrt_address)0x00030600UL, */ /* memcpy() */
-
-/* INPUT_SYSTEM */
-static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = {
-	(hrt_address)0x00080000UL};
-/*	(hrt_address)0x00081000UL, */ /* capture A */
-/*	(hrt_address)0x00082000UL, */ /* capture B */
-/*	(hrt_address)0x00083000UL, */ /* capture C */
-/*	(hrt_address)0x00084000UL, */ /* Acquisition */
-/*	(hrt_address)0x00085000UL, */ /* DMA */
-/*	(hrt_address)0x00089000UL, */ /* ctrl */
-/*	(hrt_address)0x0008A000UL, */ /* GP regs */
-/*	(hrt_address)0x0008B000UL, */ /* FIFO */
-/*	(hrt_address)0x0008C000UL, */ /* IRQ */
-
-/* RX, the MIPI lane control regs start at offset 0 */
-static const hrt_address RX_BASE[N_RX_ID] = {
-	(hrt_address)0x00080100UL};
-
-#else
-#error "system_local.h: HRT_ADDRESS_WIDTH must be one of {32,64}"
-#endif
-
-#endif /* __SYSTEM_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl.c
deleted file mode 100644
index cd12d74024f7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "timed_ctrl.h"
-
-#ifndef __INLINE_TIMED_CTRL__
-#include "timed_ctrl_private.h"
-#endif /* __INLINE_TIMED_CTRL__ */
-
-#include "assert_support.h"
-
-void timed_ctrl_snd_commnd(
-	const timed_ctrl_ID_t			ID,
-	hrt_data				mask,
-	hrt_data				condition,
-	hrt_data				counter,
-	hrt_address				addr,
-	hrt_data				value)
-{
-	OP___assert(ID == TIMED_CTRL0_ID);
-	OP___assert(TIMED_CTRL_BASE[ID] != (hrt_address)-1);
-
-	timed_ctrl_reg_store(ID, _HRT_TIMED_CONTROLLER_CMD_REG_IDX, mask);
-	timed_ctrl_reg_store(ID, _HRT_TIMED_CONTROLLER_CMD_REG_IDX, condition);
-	timed_ctrl_reg_store(ID, _HRT_TIMED_CONTROLLER_CMD_REG_IDX, counter);
-	timed_ctrl_reg_store(ID, _HRT_TIMED_CONTROLLER_CMD_REG_IDX, (hrt_data)addr);
-	timed_ctrl_reg_store(ID, _HRT_TIMED_CONTROLLER_CMD_REG_IDX, value);
-}
-
-/* pqiao TODO: make sure the following commands get
-	correct BASE address both for csim and android */
-
-void timed_ctrl_snd_sp_commnd(
-	const timed_ctrl_ID_t			ID,
-	hrt_data				mask,
-	hrt_data				condition,
-	hrt_data				counter,
-	const sp_ID_t				SP_ID,
-	hrt_address				offset,
-	hrt_data				value)
-{
-	OP___assert(SP_ID < N_SP_ID);
-	OP___assert(SP_DMEM_BASE[SP_ID] != (hrt_address)-1);
-
-	timed_ctrl_snd_commnd(ID, mask, condition, counter,
-				SP_DMEM_BASE[SP_ID]+offset, value);
-}
-
-void timed_ctrl_snd_gpio_commnd(
-	const timed_ctrl_ID_t			ID,
-	hrt_data				mask,
-	hrt_data				condition,
-	hrt_data				counter,
-	const gpio_ID_t				GPIO_ID,
-	hrt_address				offset,
-	hrt_data				value)
-{
-	OP___assert(GPIO_ID < N_GPIO_ID);
-	OP___assert(GPIO_BASE[GPIO_ID] != (hrt_address)-1);
-
-	timed_ctrl_snd_commnd(ID, mask, condition, counter,
-				GPIO_BASE[GPIO_ID]+offset, value);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl_local.h
deleted file mode 100644
index e570813af28d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl_local.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __TIMED_CTRL_LOCAL_H_INCLUDED__
-#define __TIMED_CTRL_LOCAL_H_INCLUDED__
-
-#include "timed_ctrl_global.h"
-
-#endif /* __TIMED_CTRL_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl_private.h
deleted file mode 100644
index fb0fdbb88435..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/timed_ctrl_private.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __TIMED_CTRL_PRIVATE_H_INCLUDED__
-#define __TIMED_CTRL_PRIVATE_H_INCLUDED__
-
-#include "timed_ctrl_public.h"
-
-#include "device_access.h"
-
-#include "assert_support.h"
-
-STORAGE_CLASS_TIMED_CTRL_C void timed_ctrl_reg_store(
-	const timed_ctrl_ID_t	ID,
-	const unsigned int		reg,
-	const hrt_data			value)
-{
-OP___assert(ID < N_TIMED_CTRL_ID);
-OP___assert(TIMED_CTRL_BASE[ID] != (hrt_address)-1);
-	ia_css_device_store_uint32(TIMED_CTRL_BASE[ID] + reg*sizeof(hrt_data), value);
-}
-
-#endif /* __GP_DEVICE_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vamem_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vamem_local.h
deleted file mode 100644
index c4e99afe0d29..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vamem_local.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __VAMEM_LOCAL_H_INCLUDED__
-#define __VAMEM_LOCAL_H_INCLUDED__
-
-#include "vamem_global.h"
-
-#endif /* __VAMEM_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vamem_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vamem_private.h
deleted file mode 100644
index 5e05258673d5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vamem_private.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __VAMEM_PRIVATE_H_INCLUDED__
-#define __VAMEM_PRIVATE_H_INCLUDED__
-
-#include "vamem_public.h"
-
-#include <hrt/api.h>
-
-#include "assert_support.h"
-
-
-STORAGE_CLASS_ISP_C void isp_vamem_store(
-	const vamem_ID_t	ID,
-	vamem_data_t		*addr,
-	const vamem_data_t	*data,
-	const size_t		size) /* in vamem_data_t */
-{
-	assert(ID < N_VAMEM_ID);
-	assert(ISP_VAMEM_BASE[ID] != (hrt_address)-1);
-	hrt_master_port_store(ISP_VAMEM_BASE[ID] + (unsigned)addr, data, size * sizeof(vamem_data_t));
-}
-
-
-#endif /* __VAMEM_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem.c
deleted file mode 100644
index ea22c23fc7a4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem.c
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010 - 2016, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "isp.h"
-#include "vmem.h"
-#include "vmem_local.h"
-
-#if !defined(HRT_MEMORY_ACCESS)
-#include "ia_css_device_access.h"
-#endif
-#include "assert_support.h"
-#include "platform_support.h"			/* hrt_sleep() */
-
-typedef unsigned long long hive_uedge;
-typedef hive_uedge *hive_wide;
-
-/* Copied from SDK: sim_semantics.c */
-
-/* subword bits move like this:         MSB[____xxxx____]LSB -> MSB[00000000xxxx]LSB */
-#define SUBWORD(w, start, end)     (((w) & (((1ULL << ((end)-1))-1) << 1 | 1)) >> (start))
-
-/* inverse subword bits move like this: MSB[xxxx____xxxx]LSB -> MSB[xxxx0000xxxx]LSB */
-#define INV_SUBWORD(w, start, end) ((w) & (~(((1ULL << ((end)-1))-1) << 1 | 1) | ((1ULL << (start))-1)) )
-
-#define uedge_bits (8*sizeof(hive_uedge))
-#define move_lower_bits(target, target_bit, src, src_bit) move_subword(target, target_bit, src, 0, src_bit)
-#define move_upper_bits(target, target_bit, src, src_bit) move_subword(target, target_bit, src, src_bit, uedge_bits)
-#define move_word(target, target_bit, src) move_subword(target, target_bit, src, 0, uedge_bits)
-
-static void
-move_subword (
-	hive_uedge *target,
-	unsigned target_bit,
-	hive_uedge src,
-	unsigned src_start,
-	unsigned src_end)
-{
-	unsigned int start_elem = target_bit / uedge_bits;
-	unsigned int start_bit  = target_bit % uedge_bits;
-	unsigned subword_width = src_end - src_start;
-
-	hive_uedge src_subword = SUBWORD(src, src_start, src_end);
-
-	if (subword_width + start_bit > uedge_bits) { /* overlap */
-		hive_uedge old_val1;
-		hive_uedge old_val0 = INV_SUBWORD(target[start_elem], start_bit, uedge_bits);
-		target[start_elem] = old_val0 | (src_subword << start_bit);
-		old_val1 = INV_SUBWORD(target[start_elem+1], 0, subword_width + start_bit - uedge_bits);
-		target[start_elem+1] = old_val1 | (src_subword >> ( uedge_bits - start_bit));
-	} else {
-		hive_uedge old_val = INV_SUBWORD(target[start_elem], start_bit, start_bit + subword_width);
-		target[start_elem] = old_val | (src_subword << start_bit);
-	}
-}
-
-static void
-hive_sim_wide_unpack(
-	hive_wide vector,
-	hive_wide elem,
-	hive_uint elem_bits,
-	hive_uint index)
-{
-	/* pointers into wide_type: */
-	unsigned int start_elem = (elem_bits * index) / uedge_bits;
-	unsigned int start_bit  = (elem_bits * index) % uedge_bits;
-	unsigned int end_elem   = (elem_bits * (index + 1) - 1) / uedge_bits;
-	unsigned int end_bit    = ((elem_bits * (index + 1) - 1) % uedge_bits) + 1;
-
-	if (elem_bits == uedge_bits) {
-		/* easy case for speedup: */
-		elem[0] = vector[index];
-	} else if (start_elem == end_elem) {
-		/* only one (<=64 bits) element needs to be (partly) copied: */
-		move_subword(elem, 0, vector[start_elem], start_bit, end_bit);
-	} else {
-		/* general case: handles edge spanning cases (includes >64bit elements) */
-		unsigned int bits_written = 0;
-		unsigned int i;
-		move_upper_bits(elem, bits_written, vector[start_elem], start_bit);
-		bits_written += (64 - start_bit);
-		for(i = start_elem+1; i < end_elem; i++) {
-			move_word(elem, bits_written, vector[i]);
-			bits_written += uedge_bits;
-		}
-		move_lower_bits(elem, bits_written , vector[end_elem], end_bit);
-	}
-}
-
-static void
-hive_sim_wide_pack(
-	hive_wide vector,
-	hive_wide elem,
-	hive_uint elem_bits,
-	hive_uint index)
-{
-	/* pointers into wide_type: */
-	unsigned int start_elem = (elem_bits * index) / uedge_bits;
-
-	/* easy case for speedup: */
-	if (elem_bits == uedge_bits) {
-		vector[start_elem] = elem[0];
-	} else if (elem_bits > uedge_bits) {
-		unsigned bits_to_write = elem_bits;
-		unsigned start_bit = elem_bits * index;
-		unsigned i = 0;
-		for(; bits_to_write > uedge_bits; bits_to_write -= uedge_bits, i++, start_bit += uedge_bits) {
-			move_word(vector, start_bit, elem[i]);
-		}
-		move_lower_bits(vector, start_bit, elem[i], bits_to_write);
-	} else {
-		/* only one element needs to be (partly) copied: */
-		move_lower_bits(vector, elem_bits * index, elem[0], elem_bits);
-	}
-}
-
-static void load_vector (
-	const isp_ID_t		ID,
-	t_vmem_elem		*to,
-	const t_vmem_elem	*from)
-{
-	unsigned i;
-	hive_uedge *data;
-	unsigned size = sizeof(short)*ISP_NWAY;
-	VMEM_ARRAY(v, 2*ISP_NWAY); /* Need 2 vectors to work around vmem hss bug */
-	assert(ISP_BAMEM_BASE[ID] != (hrt_address)-1);
-#if !defined(HRT_MEMORY_ACCESS)
-	ia_css_device_load(ISP_BAMEM_BASE[ID] + (unsigned long)from, &v[0][0], size);
-#else
-	hrt_master_port_load(ISP_BAMEM_BASE[ID] + (unsigned long)from, &v[0][0], size);
-#endif
-	data = (hive_uedge *)v;
-	for (i = 0; i < ISP_NWAY; i++) {
-		hive_uedge elem = 0;
-		hive_sim_wide_unpack(data, &elem, ISP_VEC_ELEMBITS, i);
-		to[i] = elem;
-	}
-	hrt_sleep(); /* Spend at least 1 cycles per vector */
-}
-
-static void store_vector (
-	const isp_ID_t		ID,
-	t_vmem_elem		*to,
-	const t_vmem_elem	*from)
-{
-	unsigned i;
-	unsigned size = sizeof(short)*ISP_NWAY;
-	VMEM_ARRAY(v, 2*ISP_NWAY); /* Need 2 vectors to work around vmem hss bug */
-	//load_vector (&v[1][0], &to[ISP_NWAY]); /* Fetch the next vector, since it will be overwritten. */
-	hive_uedge *data = (hive_uedge *)v;
-	for (i = 0; i < ISP_NWAY; i++) {
-		hive_sim_wide_pack(data, (hive_wide)&from[i], ISP_VEC_ELEMBITS, i);
-	}
-	assert(ISP_BAMEM_BASE[ID] != (hrt_address)-1);
-#if !defined(HRT_MEMORY_ACCESS)
-	ia_css_device_store(ISP_BAMEM_BASE[ID] + (unsigned long)to, &v, size);
-#else
-	//hrt_mem_store (ISP, VMEM, (unsigned)to, &v, siz); /* This will overwrite the next vector as well */
-	hrt_master_port_store(ISP_BAMEM_BASE[ID] + (unsigned long)to, &v, size);
-#endif
-	hrt_sleep(); /* Spend at least 1 cycles per vector */
-}
-
-void isp_vmem_load(
-	const isp_ID_t		ID,
-	const t_vmem_elem	*from,
-	t_vmem_elem		*to,
-	unsigned		elems) /* In t_vmem_elem */
-{
-	unsigned c;
-	const t_vmem_elem *vp = from;
-	assert(ID < N_ISP_ID);
-	assert((unsigned long)from % ISP_VEC_ALIGN == 0);
-	assert(elems % ISP_NWAY == 0);
-	for (c = 0; c < elems; c += ISP_NWAY) {
-		load_vector(ID, &to[c], vp);
-		vp = (t_vmem_elem *)((char*)vp + ISP_VEC_ALIGN);
-	}
-}
-
-void isp_vmem_store(
-	const isp_ID_t		ID,
-	t_vmem_elem		*to,
-	const t_vmem_elem	*from,
-	unsigned		elems) /* In t_vmem_elem */
-{
-	unsigned c;
-	t_vmem_elem *vp = to;
-	assert(ID < N_ISP_ID);
-	assert((unsigned long)to % ISP_VEC_ALIGN == 0);
-	assert(elems % ISP_NWAY == 0);
-	for (c = 0; c < elems; c += ISP_NWAY) {
-		store_vector (ID, vp, &from[c]);
-		vp = (t_vmem_elem *)((char*)vp + ISP_VEC_ALIGN);
-	}
-}
-
-void isp_vmem_2d_load (
-	const isp_ID_t		ID,
-	const t_vmem_elem	*from,
-	t_vmem_elem		*to,
-	unsigned height,
-	unsigned width,
-	unsigned stride_to,  /* In t_vmem_elem */
-	unsigned stride_from /* In t_vmem_elem */)
-{
-	unsigned h;
-
-	assert(ID < N_ISP_ID);
-	assert((unsigned long)from % ISP_VEC_ALIGN == 0);
-	assert(width % ISP_NWAY == 0);
-	assert(stride_from % ISP_NWAY == 0);
-	for (h = 0; h < height; h++) {
-		unsigned c;
-		const t_vmem_elem *vp = from;
-		for (c = 0; c < width; c += ISP_NWAY) {
-			load_vector(ID, &to[stride_to*h + c], vp);
-			vp = (t_vmem_elem *)((char*)vp + ISP_VEC_ALIGN);
-		}
-		from = (const t_vmem_elem *)((const char *)from + stride_from/ISP_NWAY*ISP_VEC_ALIGN);
-	}
-}
-
-void isp_vmem_2d_store (
-	const isp_ID_t		ID,
-	t_vmem_elem		*to,
-	const t_vmem_elem	*from,
-	unsigned height,
-	unsigned width,
-	unsigned stride_to,  /* In t_vmem_elem */
-	unsigned stride_from /* In t_vmem_elem */)
-{
-	unsigned h;
-
-	assert(ID < N_ISP_ID);
-	assert((unsigned long)to % ISP_VEC_ALIGN == 0);
-	assert(width % ISP_NWAY == 0);
-	assert(stride_to % ISP_NWAY == 0);
-	for (h = 0; h < height; h++) {
-		unsigned c;
-		t_vmem_elem *vp = to;
-		for (c = 0; c < width; c += ISP_NWAY) {
-			store_vector (ID, vp, &from[stride_from*h + c]);
-			vp = (t_vmem_elem *)((char*)vp + ISP_VEC_ALIGN);
-		}
-		to = (t_vmem_elem *)((char *)to + stride_to/ISP_NWAY*ISP_VEC_ALIGN);
-	}
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem_local.h
deleted file mode 100644
index de85644b885e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem_local.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __VMEM_LOCAL_H_INCLUDED__
-#define __VMEM_LOCAL_H_INCLUDED__
-
-#include "type_support.h"
-#include "vmem_global.h"
-
-typedef uint16_t t_vmem_elem;
-
-#define VMEM_ARRAY(x,s)    t_vmem_elem x[s/ISP_NWAY][ISP_NWAY]
-
-void isp_vmem_load(
-	const isp_ID_t		ID,
-	const t_vmem_elem	*from,
-	t_vmem_elem		*to,
-	unsigned		elems); /* In t_vmem_elem */
-
-void isp_vmem_store(
-	const isp_ID_t		ID,
-	t_vmem_elem		*to,
-	const t_vmem_elem	*from,
-	unsigned		elems); /* In t_vmem_elem */
-
-void isp_vmem_2d_load (
-	const isp_ID_t		ID,
-	const t_vmem_elem	*from,
-	t_vmem_elem		*to,
-	unsigned		height,
-	unsigned		width,
-	unsigned		stride_to,  /* In t_vmem_elem */
-	unsigned		stride_from /* In t_vmem_elem */);
-
-void isp_vmem_2d_store (
-	const isp_ID_t		ID,
-	t_vmem_elem		*to,
-	const t_vmem_elem	*from,
-	unsigned		height,
-	unsigned		width,
-	unsigned		stride_to,  /* In t_vmem_elem */
-	unsigned		stride_from /* In t_vmem_elem */);
-
-#endif /* __VMEM_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem_private.h
deleted file mode 100644
index f48d1281b5a7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/host/vmem_private.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __VMEM_PRIVATE_H_INCLUDED__
-#define __VMEM_PRIVATE_H_INCLUDED__
-
-#include "vmem_public.h"
-
-#endif /* __VMEM_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/input_formatter_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/input_formatter_global.h
deleted file mode 100644
index 7558f4964313..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/input_formatter_global.h
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_FORMATTER_GLOBAL_H_INCLUDED__
-#define __INPUT_FORMATTER_GLOBAL_H_INCLUDED__
-
-#define IS_INPUT_FORMATTER_VERSION2
-#define IS_INPUT_SWITCH_VERSION2
-
-#include <type_support.h>
-#include <system_types.h>
-#include "if_defs.h"
-#include "str2mem_defs.h"
-#include "input_switch_2400_defs.h"
-
-#define _HIVE_INPUT_SWITCH_GET_FSYNC_REG_LSB(ch_id)        ((ch_id) * 3)
-
-#define HIVE_SWITCH_N_CHANNELS				4
-#define HIVE_SWITCH_N_FORMATTYPES			32
-#define HIVE_SWITCH_N_SWITCH_CODE			4
-#define HIVE_SWITCH_M_CHANNELS				0x00000003
-#define HIVE_SWITCH_M_FORMATTYPES			0x0000001f
-#define HIVE_SWITCH_M_SWITCH_CODE			0x00000003
-#define HIVE_SWITCH_M_FSYNC					0x00000007
-
-#define HIVE_SWITCH_ENCODE_FSYNC(x) \
-	(1U<<(((x)-1)&HIVE_SWITCH_M_CHANNELS))
-
-#define _HIVE_INPUT_SWITCH_GET_LUT_FIELD(reg, bit_index) \
-	(((reg) >> (bit_index)) & HIVE_SWITCH_M_SWITCH_CODE)
-#define _HIVE_INPUT_SWITCH_SET_LUT_FIELD(reg, bit_index, val) \
-	(((reg) & ~(HIVE_SWITCH_M_SWITCH_CODE<<(bit_index))) | (((hrt_data)(val)&HIVE_SWITCH_M_SWITCH_CODE)<<(bit_index)))
-#define _HIVE_INPUT_SWITCH_GET_FSYNC_FIELD(reg, bit_index) \
-	(((reg) >> (bit_index)) & HIVE_SWITCH_M_FSYNC)
-#define _HIVE_INPUT_SWITCH_SET_FSYNC_FIELD(reg, bit_index, val) \
-	(((reg) & ~(HIVE_SWITCH_M_FSYNC<<(bit_index))) | (((hrt_data)(val)&HIVE_SWITCH_M_FSYNC)<<(bit_index)))
-
-typedef struct input_formatter_cfg_s	input_formatter_cfg_t;
-
-/* Hardware registers */
-/*#define HIVE_IF_RESET_ADDRESS                   0x000*/ /* deprecated */
-#define HIVE_IF_START_LINE_ADDRESS              0x004
-#define HIVE_IF_START_COLUMN_ADDRESS            0x008
-#define HIVE_IF_CROPPED_HEIGHT_ADDRESS          0x00C
-#define HIVE_IF_CROPPED_WIDTH_ADDRESS           0x010
-#define HIVE_IF_VERTICAL_DECIMATION_ADDRESS     0x014
-#define HIVE_IF_HORIZONTAL_DECIMATION_ADDRESS   0x018
-#define HIVE_IF_H_DEINTERLEAVING_ADDRESS        0x01C
-#define HIVE_IF_LEFTPADDING_WIDTH_ADDRESS       0x020
-#define HIVE_IF_END_OF_LINE_OFFSET_ADDRESS      0x024
-#define HIVE_IF_VMEM_START_ADDRESS_ADDRESS      0x028
-#define HIVE_IF_VMEM_END_ADDRESS_ADDRESS        0x02C
-#define HIVE_IF_VMEM_INCREMENT_ADDRESS          0x030
-#define HIVE_IF_YUV_420_FORMAT_ADDRESS          0x034
-#define HIVE_IF_VSYNCK_ACTIVE_LOW_ADDRESS       0x038
-#define HIVE_IF_HSYNCK_ACTIVE_LOW_ADDRESS       0x03C
-#define HIVE_IF_ALLOW_FIFO_OVERFLOW_ADDRESS     0x040
-#define HIVE_IF_BLOCK_FIFO_NO_REQ_ADDRESS       0x044
-#define HIVE_IF_V_DEINTERLEAVING_ADDRESS        0x048
-#define HIVE_IF_FSM_CROP_PIXEL_COUNTER          0x110
-#define HIVE_IF_FSM_CROP_LINE_COUNTER           0x10C
-#define HIVE_IF_FSM_CROP_STATUS                 0x108
-
-/* Registers only for simulation */
-#define HIVE_IF_CRUN_MODE_ADDRESS               0x04C
-#define HIVE_IF_DUMP_OUTPUT_ADDRESS             0x050
-
-/* Follow the DMA syntax, "cmd" last */
-#define IF_PACK(val, cmd)             ((val & 0x0fff) | (cmd /*& 0xf000*/))
-
-#define HIVE_STR2MEM_SOFT_RESET_REG_ADDRESS                   (_STR2MEM_SOFT_RESET_REG_ID * _STR2MEM_REG_ALIGN)
-#define HIVE_STR2MEM_INPUT_ENDIANNESS_REG_ADDRESS             (_STR2MEM_INPUT_ENDIANNESS_REG_ID * _STR2MEM_REG_ALIGN)
-#define HIVE_STR2MEM_OUTPUT_ENDIANNESS_REG_ADDRESS            (_STR2MEM_OUTPUT_ENDIANNESS_REG_ID * _STR2MEM_REG_ALIGN)
-#define HIVE_STR2MEM_BIT_SWAPPING_REG_ADDRESS                 (_STR2MEM_BIT_SWAPPING_REG_ID * _STR2MEM_REG_ALIGN)
-#define HIVE_STR2MEM_BLOCK_SYNC_LEVEL_REG_ADDRESS             (_STR2MEM_BLOCK_SYNC_LEVEL_REG_ID * _STR2MEM_REG_ALIGN)
-#define HIVE_STR2MEM_PACKET_SYNC_LEVEL_REG_ADDRESS            (_STR2MEM_PACKET_SYNC_LEVEL_REG_ID * _STR2MEM_REG_ALIGN)
-#define HIVE_STR2MEM_READ_POST_WRITE_SYNC_ENABLE_REG_ADDRESS  (_STR2MEM_READ_POST_WRITE_SYNC_ENABLE_REG_ID * _STR2MEM_REG_ALIGN)
-#define HIVE_STR2MEM_DUAL_BYTE_INPUTS_ENABLED_REG_ADDRESS     (_STR2MEM_DUAL_BYTE_INPUTS_ENABLED_REG_ID * _STR2MEM_REG_ALIGN)
-#define HIVE_STR2MEM_EN_STAT_UPDATE_ADDRESS                   (_STR2MEM_EN_STAT_UPDATE_ID * _STR2MEM_REG_ALIGN)
-
-/*
- * This data structure is shared between host and SP
- */
-struct input_formatter_cfg_s {
-	uint32_t	start_line;
-	uint32_t	start_column;
-	uint32_t	left_padding;
-	uint32_t	cropped_height;
-	uint32_t	cropped_width;
-	uint32_t	deinterleaving;
-	uint32_t	buf_vecs;
-	uint32_t	buf_start_index;
-	uint32_t	buf_increment;
-	uint32_t	buf_eol_offset;
-	uint32_t	is_yuv420_format;
-	uint32_t	block_no_reqs;
-};
-
-extern const hrt_address HIVE_IF_SRST_ADDRESS[N_INPUT_FORMATTER_ID];
-extern const hrt_data HIVE_IF_SRST_MASK[N_INPUT_FORMATTER_ID];
-extern const uint8_t HIVE_IF_SWITCH_CODE[N_INPUT_FORMATTER_ID];
-
-#endif /* __INPUT_FORMATTER_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/input_system_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/input_system_global.h
deleted file mode 100644
index 9ba36525e8d3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/input_system_global.h
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_SYSTEM_GLOBAL_H_INCLUDED__
-#define __INPUT_SYSTEM_GLOBAL_H_INCLUDED__
-
-#define IS_INPUT_SYSTEM_VERSION_2
-
-#include <type_support.h>
-
-//CSI reveiver has 3 ports.
-#define		N_CSI_PORTS (3)
-//AM: Use previous define for this.
-
-//MIPI allows upto 4 channels.
-#define		N_CHANNELS  (4) 
-// 12KB = 256bit x 384 words
-#define		IB_CAPACITY_IN_WORDS (384)
-
-typedef enum {
-	MIPI_0LANE_CFG = 0,
-	MIPI_1LANE_CFG = 1,
-	MIPI_2LANE_CFG = 2,
-	MIPI_3LANE_CFG = 3,
-	MIPI_4LANE_CFG = 4
-} mipi_lane_cfg_t;
-
-typedef enum {
-	INPUT_SYSTEM_SOURCE_SENSOR = 0,
-	INPUT_SYSTEM_SOURCE_FIFO,
-	INPUT_SYSTEM_SOURCE_TPG,
-	INPUT_SYSTEM_SOURCE_PRBS,
-	INPUT_SYSTEM_SOURCE_MEMORY,
-	N_INPUT_SYSTEM_SOURCE
-} input_system_source_t;
-
-/* internal routing configuration */
-typedef enum {
-	INPUT_SYSTEM_DISCARD_ALL = 0,
-	INPUT_SYSTEM_CSI_BACKEND = 1,
-	INPUT_SYSTEM_INPUT_BUFFER = 2, 
-	INPUT_SYSTEM_MULTICAST = 3,
-	N_INPUT_SYSTEM_CONNECTION
-} input_system_connection_t;
-
-typedef enum {
-	INPUT_SYSTEM_MIPI_PORT0,
-	INPUT_SYSTEM_MIPI_PORT1,
-	INPUT_SYSTEM_MIPI_PORT2,
-	INPUT_SYSTEM_ACQUISITION_UNIT,
-	N_INPUT_SYSTEM_MULTIPLEX
-} input_system_multiplex_t;
-
-typedef enum {
-	INPUT_SYSTEM_SINK_MEMORY = 0,
-	INPUT_SYSTEM_SINK_ISP,
-	INPUT_SYSTEM_SINK_SP,
-	N_INPUT_SYSTEM_SINK
-} input_system_sink_t;
-
-typedef enum {
-	INPUT_SYSTEM_FIFO_CAPTURE = 0,
-	INPUT_SYSTEM_FIFO_CAPTURE_WITH_COUNTING,
-	INPUT_SYSTEM_SRAM_BUFFERING,
-	INPUT_SYSTEM_XMEM_BUFFERING,
-	INPUT_SYSTEM_XMEM_CAPTURE,
-	INPUT_SYSTEM_XMEM_ACQUIRE,
-	N_INPUT_SYSTEM_BUFFERING_MODE
-} buffering_mode_t;
-
-typedef struct input_system_cfg_s	input_system_cfg_t;
-typedef struct sync_generator_cfg_s	sync_generator_cfg_t;
-typedef struct tpg_cfg_s			tpg_cfg_t;
-typedef struct prbs_cfg_s			prbs_cfg_t;
-
-/* MW: uint16_t should be sufficient */
-struct input_system_cfg_s {
-	uint32_t	no_side_band;
-	uint32_t	fmt_type;
-	uint32_t	ch_id;
-	uint32_t	input_mode;
-};
-
-struct sync_generator_cfg_s {
-	uint32_t	width;
-	uint32_t	height;
-	uint32_t	hblank_cycles;
-	uint32_t	vblank_cycles;
-};
-
-/* MW: tpg & prbs are exclusive */
-struct tpg_cfg_s {
-	uint32_t	x_mask;
-	uint32_t	y_mask;
-	uint32_t	x_delta;
-	uint32_t	y_delta;
-	uint32_t	xy_mask;
-	sync_generator_cfg_t sync_gen_cfg;
-};
-
-struct prbs_cfg_s {
-	uint32_t	seed;
-	sync_generator_cfg_t sync_gen_cfg;
-};
-
-struct gpfifo_cfg_s {
-// TBD.
-	sync_generator_cfg_t sync_gen_cfg;
-};
-
-typedef struct gpfifo_cfg_s		gpfifo_cfg_t;
-
-//ALX:Commented out to pass the compilation.
-//typedef struct input_system_cfg_s input_system_cfg_t;
-
-struct ib_buffer_s {
-	uint32_t	mem_reg_size;
-	uint32_t	nof_mem_regs;
-	uint32_t	mem_reg_addr;
-};
-
-typedef struct ib_buffer_s	ib_buffer_t;
-
-struct csi_cfg_s {
-	uint32_t			csi_port;
-    buffering_mode_t	buffering_mode;
-	ib_buffer_t			csi_buffer;
-	ib_buffer_t			acquisition_buffer;
-	uint32_t			nof_xmem_buffers;
-};
-
-typedef struct csi_cfg_s	 csi_cfg_t;
-
-typedef enum {
-	INPUT_SYSTEM_CFG_FLAG_RESET	= 0,
-	INPUT_SYSTEM_CFG_FLAG_SET		= 1U << 0,
-	INPUT_SYSTEM_CFG_FLAG_BLOCKED	= 1U << 1,
-	INPUT_SYSTEM_CFG_FLAG_REQUIRED	= 1U << 2,
-	INPUT_SYSTEM_CFG_FLAG_CONFLICT	= 1U << 3	// To mark a conflicting configuration.
-} input_system_cfg_flag_t;
-
-typedef uint32_t input_system_config_flags_t; 
-
-#endif /* __INPUT_SYSTEM_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/irq_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/irq_global.h
deleted file mode 100644
index 64554d80dc0b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/irq_global.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IRQ_GLOBAL_H_INCLUDED__
-#define __IRQ_GLOBAL_H_INCLUDED__
-
-#include <system_types.h>
-
-#define IS_IRQ_VERSION_2
-#define IS_IRQ_MAP_VERSION_2
-
-/* We cannot include the (hrt host ID) file defining the "CSS_RECEIVER" property without side effects */
-#ifndef HAS_NO_RX
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM)
-/*#define CSS_RECEIVER testbench_isp_inp_sys_csi_receiver*/
-#include "hive_isp_css_irq_types_hrt.h"	/* enum	hrt_isp_css_irq */
-#elif defined(IS_ISP_2401_MAMOIADA_SYSTEM)
-/*#define CSS_RECEIVER testbench_isp_is_2400_inp_sys_csi_receiver*/
-#include "hive_isp_css_2401_irq_types_hrt.h"	/* enum	hrt_isp_css_irq */
-#else
-#error "irq_global.h: 2400_SYSTEM must be one of {2400, 2401 }"
-#endif
-#endif
-
-/* The IRQ is not mapped uniformly on its related interfaces */
-#define	IRQ_SW_CHANNEL_OFFSET	hrt_isp_css_irq_sw_pin_0
-
-typedef enum {
-	IRQ_SW_CHANNEL0_ID = hrt_isp_css_irq_sw_pin_0 - IRQ_SW_CHANNEL_OFFSET,
-	IRQ_SW_CHANNEL1_ID = hrt_isp_css_irq_sw_pin_1 - IRQ_SW_CHANNEL_OFFSET,
-	N_IRQ_SW_CHANNEL_ID
-} irq_sw_channel_id_t;
-
-#endif /* __IRQ_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/isp_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/isp_global.h
deleted file mode 100644
index 14d574849a5b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/isp_global.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISP_GLOBAL_H_INCLUDED__
-#define __ISP_GLOBAL_H_INCLUDED__
-
-#include <system_types.h>
-
-#if defined (HAS_ISP_2401_MAMOIADA)
-#define IS_ISP_2401_MAMOIADA
-
-#include "isp2401_mamoiada_params.h"
-#elif defined (HAS_ISP_2400_MAMOIADA)
-#define IS_ISP_2400_MAMOIADA
-
-#include "isp2400_mamoiada_params.h"
-#else
-#error "isp_global_h: ISP_2400_MAMOIDA must be one of {2400, 2401 }"
-#endif
-
-#define ISP_PMEM_WIDTH_LOG2		ISP_LOG2_PMEM_WIDTH
-#define ISP_PMEM_SIZE			ISP_PMEM_DEPTH
-
-#define ISP_NWAY_LOG2			6
-#define ISP_VEC_NELEMS_LOG2		ISP_NWAY_LOG2
-
-#ifdef ISP2401
-#ifdef PIPE_GENERATION
-#define PIPEMEM(x) MEM(x)
-#define ISP_NWAY   (1<<ISP_NWAY_LOG2)
-#else
-#define PIPEMEM(x)
-#endif
-
-#endif
-/* The number of data bytes in a vector disregarding the reduced precision */
-#define ISP_VEC_BYTES			(ISP_VEC_NELEMS*sizeof(uint16_t))
-
-/* ISP SC Registers */
-#define ISP_SC_REG			0x00
-#define ISP_PC_REG			0x07
-#define ISP_IRQ_READY_REG		0x00
-#define ISP_IRQ_CLEAR_REG		0x00
-
-/* ISP SC Register bits */
-#define ISP_RST_BIT			0x00
-#define ISP_START_BIT			0x01
-#define ISP_BREAK_BIT			0x02
-#define ISP_RUN_BIT			0x03
-#define ISP_BROKEN_BIT			0x04
-#define ISP_IDLE_BIT			0x05     /* READY */
-#define ISP_SLEEPING_BIT		0x06
-#define ISP_STALLING_BIT		0x07
-#define ISP_IRQ_CLEAR_BIT		0x08
-#define ISP_IRQ_READY_BIT		0x0A
-#define ISP_IRQ_SLEEPING_BIT		0x0B
-
-/* ISP Register bits */
-#define ISP_CTRL_SINK_BIT		0x00
-#define ISP_PMEM_SINK_BIT		0x01
-#define ISP_DMEM_SINK_BIT		0x02
-#define ISP_FIFO0_SINK_BIT		0x03
-#define ISP_FIFO1_SINK_BIT		0x04
-#define ISP_FIFO2_SINK_BIT		0x05
-#define ISP_FIFO3_SINK_BIT		0x06
-#define ISP_FIFO4_SINK_BIT		0x07
-#define ISP_FIFO5_SINK_BIT		0x08
-#define ISP_FIFO6_SINK_BIT		0x09
-#define ISP_VMEM_SINK_BIT		0x0A
-#define ISP_VAMEM1_SINK_BIT		0x0B
-#define ISP_VAMEM2_SINK_BIT		0x0C
-#define ISP_VAMEM3_SINK_BIT		0x0D
-#define ISP_HMEM_SINK_BIT		0x0E
-
-#define ISP_CTRL_SINK_REG		0x08
-#define ISP_PMEM_SINK_REG		0x08
-#define ISP_DMEM_SINK_REG		0x08
-#define ISP_FIFO0_SINK_REG		0x08
-#define ISP_FIFO1_SINK_REG		0x08
-#define ISP_FIFO2_SINK_REG		0x08
-#define ISP_FIFO3_SINK_REG		0x08
-#define ISP_FIFO4_SINK_REG		0x08
-#define ISP_FIFO5_SINK_REG		0x08
-#define ISP_FIFO6_SINK_REG		0x08
-#define ISP_VMEM_SINK_REG		0x08
-#define ISP_VAMEM1_SINK_REG		0x08
-#define ISP_VAMEM2_SINK_REG		0x08
-#define ISP_VAMEM3_SINK_REG		0x08
-#define ISP_HMEM_SINK_REG		0x08
-
-#ifdef ISP2401
-#define ISP_BAMEM_ALIGN_ELEM ISP_VMEM_ALIGN_ELEM
-#define BAMEM VMEM
-
-#define XNR3_DOWN_BAMEM_BASE_ADDRESS    (0x16880)
-#define XNR3_UP_BAMEM_BASE_ADDRESS      (0x12880)
-
-#define bmem_ldrow(fu, pid, offset, data) bmem_ldrow_s(fu, pid, offset, data)
-#define bmem_strow(fu, pid, offset, data) bmem_strow_s(fu, pid, offset, data)
-#define bmem_ldblk(fu, pid, offset, data) bmem_ldblk_s(fu, pid, offset, data)
-#define bmem_stblk(fu, pid, offset, data) bmem_stblk_s(fu, pid, offset, data)
-
-#endif
-#endif /* __ISP_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/mmu_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/mmu_global.h
deleted file mode 100644
index 83ca418c8ff2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/mmu_global.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __MMU_GLOBAL_H_INCLUDED__
-#define __MMU_GLOBAL_H_INCLUDED__
-
-#define IS_MMU_VERSION_2
-
-#include <mmu_defs.h>
-
-#endif /* __MMU_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/sp_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/sp_global.h
deleted file mode 100644
index 6ec4e590e3b4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/sp_global.h
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SP_GLOBAL_H_INCLUDED__
-#define __SP_GLOBAL_H_INCLUDED__
-
-#include <system_types.h>
-
-#if defined(HAS_SP_2401)
-#define IS_SP_2401
-/* 2401 uses 2400 */
-#include <scalar_processor_2400_params.h>
-#elif defined(HAS_SP_2400)
-#define IS_SP_2400
-
-#include <scalar_processor_2400_params.h>
-#else
-#error "sp_global.h: SP_2400 must be one of {2400, 2401 }"
-#endif
-
-#define SP_PMEM_WIDTH_LOG2		SP_PMEM_LOG_WIDTH_BITS
-#define SP_PMEM_SIZE			SP_PMEM_DEPTH
-
-#define SP_DMEM_SIZE			0x4000
-
-/* SP Registers */
-#define SP_PC_REG				0x09
-#define SP_SC_REG				0x00
-#define SP_START_ADDR_REG		0x01
-#define SP_ICACHE_ADDR_REG		0x05
-#define SP_IRQ_READY_REG		0x00
-#define SP_IRQ_CLEAR_REG		0x00
-#define SP_ICACHE_INV_REG		0x00
-#define SP_CTRL_SINK_REG		0x0A
-
-/* SP Register bits */
-#define SP_RST_BIT			0x00
-#define SP_START_BIT			0x01
-#define SP_BREAK_BIT			0x02
-#define SP_RUN_BIT			0x03
-#define SP_BROKEN_BIT			0x04
-#define SP_IDLE_BIT			0x05     /* READY */
-#define SP_SLEEPING_BIT			0x06
-#define SP_STALLING_BIT			0x07
-#define SP_IRQ_CLEAR_BIT		0x08
-#define SP_IRQ_READY_BIT		0x0A
-#define SP_IRQ_SLEEPING_BIT		0x0B
-
-#define SP_ICACHE_INV_BIT		0x0C
-#define SP_IPREFETCH_EN_BIT		0x0D
-
-#define SP_FIFO0_SINK_BIT		0x00
-#define SP_FIFO1_SINK_BIT		0x01
-#define SP_FIFO2_SINK_BIT		0x02
-#define SP_FIFO3_SINK_BIT		0x03
-#define SP_FIFO4_SINK_BIT		0x04
-#define SP_FIFO5_SINK_BIT		0x05
-#define SP_FIFO6_SINK_BIT		0x06
-#define SP_FIFO7_SINK_BIT		0x07
-#define SP_FIFO8_SINK_BIT		0x08
-#define SP_FIFO9_SINK_BIT		0x09
-#define SP_FIFOA_SINK_BIT		0x0A
-#define SP_DMEM_SINK_BIT		0x0B
-#define SP_CTRL_MT_SINK_BIT		0x0C
-#define SP_ICACHE_MT_SINK_BIT	0x0D
-
-#define SP_FIFO0_SINK_REG		0x0A
-#define SP_FIFO1_SINK_REG		0x0A
-#define SP_FIFO2_SINK_REG		0x0A
-#define SP_FIFO3_SINK_REG		0x0A
-#define SP_FIFO4_SINK_REG		0x0A
-#define SP_FIFO5_SINK_REG		0x0A
-#define SP_FIFO6_SINK_REG		0x0A
-#define SP_FIFO7_SINK_REG		0x0A
-#define SP_FIFO8_SINK_REG		0x0A
-#define SP_FIFO9_SINK_REG		0x0A
-#define SP_FIFOA_SINK_REG		0x0A
-#define SP_DMEM_SINK_REG		0x0A
-#define SP_CTRL_MT_SINK_REG		0x0A
-#define SP_ICACHE_MT_SINK_REG	0x0A
-
-#endif /* __SP_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/system_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/system_global.h
deleted file mode 100644
index 6f63962a54e8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/system_global.h
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SYSTEM_GLOBAL_H_INCLUDED__
-#define __SYSTEM_GLOBAL_H_INCLUDED__
-
-#include <hive_isp_css_defs.h>
-#include <type_support.h>
-
-/*
- * The longest allowed (uninteruptible) bus transfer, does not
- * take stalling into account
- */
-#define HIVE_ISP_MAX_BURST_LENGTH	1024
-
-/*
- * Maximum allowed burst length in words for the ISP DMA
- */
-#define ISP_DMA_MAX_BURST_LENGTH	128
-
-/*
- * Create a list of HAS and IS properties that defines the system
- *
- * The configuration assumes the following
- * - The system is hetereogeneous; Multiple cells and devices classes
- * - The cell and device instances are homogeneous, each device type
- *   belongs to the same class
- * - Device instances supporting a subset of the class capabilities are
- *   allowed
- *
- * We could manage different device classes through the enumerated
- * lists (C) or the use of classes (C++), but that is presently not
- * fully supported
- *
- * N.B. the 3 input formatters are of 2 different classess
- */
-
-#define IS_ISP_2400_SYSTEM
-/*
- * Since this file is visible everywhere and the system definition
- * macros are not, detect the separate definitions for {host, SP, ISP}
- *
- * The 2401 system has the nice property that it uses a vanilla 2400 SP
- * so the SP will believe it is a 2400 system rather than 2401...
- */
-//#if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada) || defined(__scalar_processor_2401)
-#if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada)
-#define IS_ISP_2401_MAMOIADA_SYSTEM
-#define HAS_ISP_2401_MAMOIADA
-#define HAS_SP_2400
-//#elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada) || defined(__scalar_processor_2400)
-#elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada)
-#define IS_ISP_2400_MAMOIADA_SYSTEM
-#define HAS_ISP_2400_MAMOIADA
-#define HAS_SP_2400
-#else
-#error "system_global.h: 2400_SYSTEM must be one of {2400, 2401 }"
-#endif
-
-#define USE_INPUT_SYSTEM_VERSION_2
-
-#define HAS_MMU_VERSION_2
-#define HAS_DMA_VERSION_2
-#define HAS_GDC_VERSION_2
-#define HAS_VAMEM_VERSION_2
-#define HAS_HMEM_VERSION_1
-#define HAS_BAMEM_VERSION_2
-#define HAS_IRQ_VERSION_2
-#define HAS_IRQ_MAP_VERSION_2
-#define HAS_INPUT_FORMATTER_VERSION_2
-/* 2401: HAS_INPUT_SYSTEM_VERSION_2401 */
-#define HAS_INPUT_SYSTEM_VERSION_2
-#define HAS_BUFFERED_SENSOR
-#define HAS_FIFO_MONITORS_VERSION_2
-/* #define HAS_GP_REGS_VERSION_2 */
-#define HAS_GP_DEVICE_VERSION_2
-#define HAS_GPIO_VERSION_1
-#define HAS_TIMED_CTRL_VERSION_1
-#define HAS_RX_VERSION_2
-
-#define DMA_DDR_TO_VAMEM_WORKAROUND
-#define DMA_DDR_TO_HMEM_WORKAROUND
-
-/*
- * Semi global. "HRT" is accessible from SP, but the HRT types do not fully apply
- */
-#define HRT_VADDRESS_WIDTH	32
-//#define HRT_ADDRESS_WIDTH	64		/* Surprise, this is a local property*/
-#define HRT_DATA_WIDTH		32
-
-#define SIZEOF_HRT_REG		(HRT_DATA_WIDTH>>3)
-#define HIVE_ISP_CTRL_DATA_BYTES (HIVE_ISP_CTRL_DATA_WIDTH/8)
-
-/* The main bus connecting all devices */
-#define HRT_BUS_WIDTH		HIVE_ISP_CTRL_DATA_WIDTH
-#define HRT_BUS_BYTES		HIVE_ISP_CTRL_DATA_BYTES
-
-/* per-frame parameter handling support */
-#define SH_CSS_ENABLE_PER_FRAME_PARAMS
-
-typedef uint32_t			hrt_bus_align_t;
-
-/*
- * Enumerate the devices, device access through the API is by ID, through the DLI by address
- * The enumerator terminators are used to size the wiring arrays and as an exception value.
- */
-typedef enum {
-	DDR0_ID = 0,
-	N_DDR_ID
-} ddr_ID_t;
-
-typedef enum {
-	ISP0_ID = 0,
-	N_ISP_ID
-} isp_ID_t;
-
-typedef enum {
-	SP0_ID = 0,
-	N_SP_ID
-} sp_ID_t;
-
-#if defined (IS_ISP_2401_MAMOIADA_SYSTEM)
-typedef enum {
-	MMU0_ID = 0,
-	MMU1_ID,
-	N_MMU_ID
-} mmu_ID_t;
-#elif defined (IS_ISP_2400_MAMOIADA_SYSTEM)
-typedef enum {
-	MMU0_ID = 0,
-	MMU1_ID,
-	N_MMU_ID
-} mmu_ID_t;
-#else
-#error "system_global.h: SYSTEM must be one of {2400, 2401}"
-#endif
-
-typedef enum {
-	DMA0_ID = 0,
-	N_DMA_ID
-} dma_ID_t;
-
-typedef enum {
-	GDC0_ID = 0,
-	GDC1_ID,
-	N_GDC_ID
-} gdc_ID_t;
-
-#define N_GDC_ID_CPP 2 // this extra define is needed because we want to use it also in the preprocessor, and that doesn't work with enums.
-
-typedef enum {
-	VAMEM0_ID = 0,
-	VAMEM1_ID,
-	VAMEM2_ID,
-	N_VAMEM_ID
-} vamem_ID_t;
-
-typedef enum {
-	BAMEM0_ID = 0,
-	N_BAMEM_ID
-} bamem_ID_t;
-
-typedef enum {
-	HMEM0_ID = 0,
-	N_HMEM_ID
-} hmem_ID_t;
-
-/*
-typedef enum {
-	IRQ0_ID = 0,
-	N_IRQ_ID
-} irq_ID_t;
-*/
-
-typedef enum {
-	IRQ0_ID = 0,	// GP IRQ block
-	IRQ1_ID,		// Input formatter
-	IRQ2_ID,		// input system
-	IRQ3_ID,		// input selector
-	N_IRQ_ID
-} irq_ID_t;
-
-typedef enum {
-	FIFO_MONITOR0_ID = 0,
-	N_FIFO_MONITOR_ID
-} fifo_monitor_ID_t;
-
-/*
- * Deprecated: Since all gp_reg instances are different
- * and put in the address maps of other devices we cannot
- * enumerate them as that assumes the instrances are the
- * same.
- *
- * We define a single GP_DEVICE containing all gp_regs
- * w.r.t. a single base address
- *
-typedef enum {
-	GP_REGS0_ID = 0,
-	N_GP_REGS_ID
-} gp_regs_ID_t;
- */
-typedef enum {
-	GP_DEVICE0_ID = 0,
-	N_GP_DEVICE_ID
-} gp_device_ID_t;
-
-typedef enum {
-	GP_TIMER0_ID = 0,
-	GP_TIMER1_ID,
-	GP_TIMER2_ID,
-	GP_TIMER3_ID,
-	GP_TIMER4_ID,
-	GP_TIMER5_ID,
-	GP_TIMER6_ID,
-	GP_TIMER7_ID,
-	N_GP_TIMER_ID
-} gp_timer_ID_t;
-
-typedef enum {
-	GPIO0_ID = 0,
-	N_GPIO_ID
-} gpio_ID_t;
-
-typedef enum {
-	TIMED_CTRL0_ID = 0,
-	N_TIMED_CTRL_ID
-} timed_ctrl_ID_t;
-
-typedef enum {
-	INPUT_FORMATTER0_ID = 0,
-	INPUT_FORMATTER1_ID,
-	INPUT_FORMATTER2_ID,
-	INPUT_FORMATTER3_ID,
-	N_INPUT_FORMATTER_ID
-} input_formatter_ID_t;
-
-/* The IF RST is outside the IF */
-#define INPUT_FORMATTER0_SRST_OFFSET	0x0824
-#define INPUT_FORMATTER1_SRST_OFFSET	0x0624
-#define INPUT_FORMATTER2_SRST_OFFSET	0x0424
-#define INPUT_FORMATTER3_SRST_OFFSET	0x0224
-
-#define INPUT_FORMATTER0_SRST_MASK		0x0001
-#define INPUT_FORMATTER1_SRST_MASK		0x0002
-#define INPUT_FORMATTER2_SRST_MASK		0x0004
-#define INPUT_FORMATTER3_SRST_MASK		0x0008
-
-typedef enum {
-	INPUT_SYSTEM0_ID = 0,
-	N_INPUT_SYSTEM_ID
-} input_system_ID_t;
-
-typedef enum {
-	RX0_ID = 0,
-	N_RX_ID
-} rx_ID_t;
-
-enum mipi_port_id {
-	MIPI_PORT0_ID = 0,
-	MIPI_PORT1_ID,
-	MIPI_PORT2_ID,
-	N_MIPI_PORT_ID
-};
-
-#define	N_RX_CHANNEL_ID		4
-
-/* Generic port enumeration with an internal port type ID */
-typedef enum {
-	CSI_PORT0_ID = 0,
-	CSI_PORT1_ID,
-	CSI_PORT2_ID,
-	TPG_PORT0_ID,
-	PRBS_PORT0_ID,
-	FIFO_PORT0_ID,
-	MEMORY_PORT0_ID,
-	N_INPUT_PORT_ID
-} input_port_ID_t;
-
-typedef enum {
-	CAPTURE_UNIT0_ID = 0,
-	CAPTURE_UNIT1_ID,
-	CAPTURE_UNIT2_ID,
-	ACQUISITION_UNIT0_ID,
-	DMA_UNIT0_ID,
-	CTRL_UNIT0_ID,
-	GPREGS_UNIT0_ID,
-	FIFO_UNIT0_ID,
-	IRQ_UNIT0_ID,
-	N_SUB_SYSTEM_ID
-} sub_system_ID_t;
-
-#define	N_CAPTURE_UNIT_ID		3
-#define	N_ACQUISITION_UNIT_ID	1
-#define	N_CTRL_UNIT_ID			1
-
-enum ia_css_isp_memories {
-	IA_CSS_ISP_PMEM0 = 0,
-	IA_CSS_ISP_DMEM0,
-	IA_CSS_ISP_VMEM0,
-	IA_CSS_ISP_VAMEM0,
-	IA_CSS_ISP_VAMEM1,
-	IA_CSS_ISP_VAMEM2,
-	IA_CSS_ISP_HMEM0,
-	IA_CSS_SP_DMEM0,
-	IA_CSS_DDR,
-	N_IA_CSS_MEMORIES
-};
-#define IA_CSS_NUM_MEMORIES 9
-/* For driver compatability */
-#define N_IA_CSS_ISP_MEMORIES   IA_CSS_NUM_MEMORIES
-#define IA_CSS_NUM_ISP_MEMORIES IA_CSS_NUM_MEMORIES
-
-#if 0
-typedef enum {
-	dev_chn, /* device channels, external resource */
-	ext_mem, /* external memories */
-	int_mem, /* internal memories */
-	int_chn  /* internal channels, user defined */
-} resource_type_t;
-
-/* if this enum is extended with other memory resources, pls also extend the function resource_to_memptr() */
-typedef enum {
-	vied_nci_dev_chn_dma_ext0,
-	int_mem_vmem0,
-	int_mem_dmem0
-} resource_id_t;
-
-/* enum listing the different memories within a program group.
-   This enum is used in the mem_ptr_t type */
-typedef enum {
-	buf_mem_invalid = 0,
-	buf_mem_vmem_prog0,
-	buf_mem_dmem_prog0
-} buf_mem_t;
-
-#endif
-#endif /* __SYSTEM_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/timed_ctrl_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/timed_ctrl_global.h
deleted file mode 100644
index c3e8a0104092..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/timed_ctrl_global.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __TIMED_CTRL_GLOBAL_H_INCLUDED__
-#define __TIMED_CTRL_GLOBAL_H_INCLUDED__
-
-#define IS_TIMED_CTRL_VERSION_1
-
-#include <timed_controller_defs.h>
-
-/**
- * Order of the input bits for the timed controller taken from
- * ISP_CSS_2401 System Architecture Description valid for
- * 2400, 2401.
- *
- * Check for other systems.
- */
-#define HIVE_TIMED_CTRL_GPIO_PIN_0_BIT_ID                       0
-#define HIVE_TIMED_CTRL_GPIO_PIN_1_BIT_ID                       1
-#define HIVE_TIMED_CTRL_GPIO_PIN_2_BIT_ID                       2
-#define HIVE_TIMED_CTRL_GPIO_PIN_3_BIT_ID                       3
-#define HIVE_TIMED_CTRL_GPIO_PIN_4_BIT_ID                       4
-#define HIVE_TIMED_CTRL_GPIO_PIN_5_BIT_ID                       5
-#define HIVE_TIMED_CTRL_GPIO_PIN_6_BIT_ID                       6
-#define HIVE_TIMED_CTRL_GPIO_PIN_7_BIT_ID                       7
-#define HIVE_TIMED_CTRL_GPIO_PIN_8_BIT_ID                       8
-#define HIVE_TIMED_CTRL_GPIO_PIN_9_BIT_ID                       9
-#define HIVE_TIMED_CTRL_GPIO_PIN_10_BIT_ID                      10
-#define HIVE_TIMED_CTRL_GPIO_PIN_11_BIT_ID                      11
-#define HIVE_TIMED_CTRL_IRQ_SP_BIT_ID                           12
-#define HIVE_TIMED_CTRL_IRQ_ISP_BIT_ID                          13
-#define HIVE_TIMED_CTRL_IRQ_INPUT_SYSTEM_BIT_ID                 14
-#define HIVE_TIMED_CTRL_IRQ_INPUT_SELECTOR_BIT_ID               15
-#define HIVE_TIMED_CTRL_IRQ_IF_BLOCK_BIT_ID                     16
-#define HIVE_TIMED_CTRL_IRQ_GP_TIMER_0_BIT_ID                   17
-#define HIVE_TIMED_CTRL_IRQ_GP_TIMER_1_BIT_ID                   18
-#define HIVE_TIMED_CTRL_CSI_SOL_BIT_ID                          19
-#define HIVE_TIMED_CTRL_CSI_EOL_BIT_ID                          20
-#define HIVE_TIMED_CTRL_CSI_SOF_BIT_ID                          21
-#define HIVE_TIMED_CTRL_CSI_EOF_BIT_ID                          22
-#define HIVE_TIMED_CTRL_IRQ_IS_STREAMING_MONITOR_BIT_ID         23
-
-
-
-#endif /* __TIMED_CTRL_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/vamem_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/vamem_global.h
deleted file mode 100644
index 58713c6583b9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/vamem_global.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __VAMEM_GLOBAL_H_INCLUDED__
-#define __VAMEM_GLOBAL_H_INCLUDED__
-
-#include <type_support.h>
-
-#define IS_VAMEM_VERSION_2
-
-/* (log) stepsize of linear interpolation */
-#define VAMEM_INTERP_STEP_LOG2	4
-#define VAMEM_INTERP_STEP		(1<<VAMEM_INTERP_STEP_LOG2)
-/* (physical) size of the tables */
-#define VAMEM_TABLE_UNIT_SIZE	((1<<(ISP_VAMEM_ADDRESS_BITS-VAMEM_INTERP_STEP_LOG2)) + 1)
-/* (logical) size of the tables */
-#define VAMEM_TABLE_UNIT_STEP	((VAMEM_TABLE_UNIT_SIZE-1)<<1)
-/* Number of tables */
-#define VAMEM_TABLE_UNIT_COUNT	(ISP_VAMEM_DEPTH/VAMEM_TABLE_UNIT_STEP)
-
-typedef uint16_t				vamem_data_t;
-
-#endif /* __VAMEM_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/vmem_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/vmem_global.h
deleted file mode 100644
index 7867cd137f3f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_common/vmem_global.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __VMEM_GLOBAL_H_INCLUDED__
-#define __VMEM_GLOBAL_H_INCLUDED__
-
-#include "isp.h"
-
-#define VMEM_SIZE	ISP_VMEM_DEPTH
-#define VMEM_ELEMBITS	ISP_VMEM_ELEMBITS
-#define VMEM_ALIGN	ISP_VMEM_ALIGN
-
-#ifndef PIPE_GENERATION
-typedef tvector *pvector;
-#endif
-
-#endif /* __VMEM_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/assert_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/assert_support.h
deleted file mode 100644
index fd0d92e87c36..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/assert_support.h
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ASSERT_SUPPORT_H_INCLUDED__
-#define __ASSERT_SUPPORT_H_INCLUDED__
-
-
-/**
- * The following macro can help to test the size of a struct at compile
- * time rather than at run-time. It does not work for all compilers; see
- * below.
- *
- * Depending on the value of 'condition', the following macro is expanded to:
- * - condition==true:
- *     an expression containing an array declaration with negative size,
- *     usually resulting in a compilation error
- * - condition==false:
- *     (void) 1; // C statement with no effect
- *
- * example:
- *  COMPILATION_ERROR_IF( sizeof(struct host_sp_queues) != SIZE_OF_HOST_SP_QUEUES_STRUCT);
- *
- * verify that the macro indeed triggers a compilation error with your compiler:
- *  COMPILATION_ERROR_IF( sizeof(struct host_sp_queues) != (sizeof(struct host_sp_queues)+1) );
- *
- * Not all compilers will trigger an error with this macro; use a search engine to search for
- * BUILD_BUG_ON to find other methods.
- */
-#define COMPILATION_ERROR_IF(condition) ((void)sizeof(char[1 - 2*!!(condition)]))
-
-/* Compile time assertion */
-#ifndef CT_ASSERT
-#define CT_ASSERT(cnd) ((void)sizeof(char[(cnd)?1:-1]))
-#endif /* CT_ASSERT */
-
-#ifdef NDEBUG
-
-#define assert(cnd) ((void)0)
-
-#else
-
-#if defined(_MSC_VER)
-#ifdef _KERNEL_MODE
-/* Windows kernel mode compilation */
-#include <wdm.h>
-#define assert(cnd) ASSERT(cnd)
-#else
-/* Windows usermode compilation */
-#include <assert.h>
-#endif
-
-#elif defined(__KERNEL__)
-#include <linux/bug.h>
-
-/* TODO: it would be cleaner to use this:
- * #define assert(cnd) BUG_ON(cnd)
- * but that causes many compiler warnings (==errors) under Android
- * because it seems that the BUG_ON() macro is not seen as a check by
- * gcc like the BUG() macro is. */
-#define assert(cnd) \
-	do { \
-		if (!(cnd)) \
-			BUG(); \
-	} while (0)
-
-#elif defined(__FIST__) || defined(__GNUC__)
-
-/* enable assert for crun */
-#include "assert.h"
-
-#else /* default for unknown environments */
-#define assert(cnd) ((void)0)
-#endif
-
-#endif /* NDEBUG */
-
-#ifndef PIPE_GENERATION
-/* Deprecated OP___assert, this is still used in ~1000 places
- * in the code. This will be removed over time.
- * The implemenation for the pipe generation tool is in see support.isp.h */
-#define OP___assert(cnd) assert(cnd)
-
-static inline void compile_time_assert (unsigned cond)
-{
-	/* Call undefined function if cond is false */
-	extern void _compile_time_assert (void);
-	if (!cond) _compile_time_assert();
-}
-#endif /* PIPE_GENERATION */
-
-#endif /* __ASSERT_SUPPORT_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/bitop_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/bitop_support.h
deleted file mode 100644
index 1b271c3c6a25..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/bitop_support.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __BITOP_SUPPORT_H_INCLUDED__
-#define __BITOP_SUPPORT_H_INCLUDED__
-
-#define bitop_setbit(a, b) ((a) |= (1UL << (b)))
-
-#define bitop_getbit(a, b) (((a) & (1UL << (b))) != 0)
-
-#define bitop_clearbit(a, b) ((a) &= ~(1UL << (b)))
-
-#endif /* __BITOP_SUPPORT_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/csi_rx.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/csi_rx.h
deleted file mode 100644
index 917ee8cdb1d9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/csi_rx.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __CSI_RX_H_INCLUDED__
-#define __CSI_RX_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- * - system and cell agnostic interfaces, constants and identifiers
- * - public:  system agnostic, cell specific interfaces
- * - private: system dependent, cell specific interfaces &
- *   inline implementations
- * - global:  system specific constants and identifiers
- * - local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "csi_rx_local.h"
-
-#ifndef __INLINE_CSI_RX__
-#include "csi_rx_public.h"
-#else  /* __INLINE_CSI_RX__ */
-#include "csi_rx_private.h"
-#endif /* __INLINE_CSI_RX__ */
-
-#endif /* __CSI_RX_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/debug.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/debug.h
deleted file mode 100644
index 0aa22446e27e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/debug.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __DEBUG_H_INCLUDED__
-#define __DEBUG_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the DMA device. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- *
- */
-
-
-#include "system_local.h"
-#include "debug_local.h"
-
-#ifndef __INLINE_DEBUG__
-#define STORAGE_CLASS_DEBUG_H extern
-#define STORAGE_CLASS_DEBUG_C 
-#include "debug_public.h"
-#else  /* __INLINE_DEBUG__ */
-#define STORAGE_CLASS_DEBUG_H static inline
-#define STORAGE_CLASS_DEBUG_C static inline
-#include "debug_private.h"
-#endif /* __INLINE_DEBUG__ */
-
-#endif /* __DEBUG_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/device_access/device_access.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/device_access/device_access.h
deleted file mode 100644
index 834e7c3e0814..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/device_access/device_access.h
+++ /dev/null
@@ -1,194 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __DEVICE_ACCESS_H_INCLUDED__
-#define __DEVICE_ACCESS_H_INCLUDED__
-
-/*!
- * \brief
- * Define the public interface for physical system
- * access functions to SRAM and registers. Access
- * types are limited to those defined in <stdint.h>
- * All accesses are aligned
- *
- * The address representation is private to the system
- * and represented as/stored in "hrt_address".
- *
- * The system global address can differ by an offset;
- * The device base address. This offset must be added
- * by the implementation of the access function
- *
- * "store" is a transfer to the device
- * "load" is a transfer from the device
- */
-
-#include <type_support.h>
-
-/*
- * User provided file that defines the system address types:
- *	- hrt_address	a type that can hold the (sub)system address range
- */
-#include "system_types.h"
-/*
- * We cannot assume that the global system address size is the size of
- * a pointer because a (say) 64-bit host can be simulated in a 32-bit
- * environment. Only if the host environment is modelled as on the target
- * we could use a pointer. Even then, prototyping may need to be done
- * before the target environment is available. AS we cannot wait for that
- * we are stuck with integer addresses
- */
-
-/*typedef	char *sys_address;*/
-typedef	hrt_address		sys_address;
-
-/*! Set the (sub)system base address
-
- \param	base_addr[in]		The offset on which the (sub)system is located
-							in the global address map
-
- \return none,
- */
-extern void device_set_base_address(
-	const sys_address		base_addr);
-
-
-/*! Get the (sub)system base address
-
- \return base_address,
- */
-extern sys_address device_get_base_address(void);
-
-/*! Read an 8-bit value from a device register or memory in the device
-
- \param	addr[in]			Local address
-
- \return device[addr]
- */
-extern uint8_t ia_css_device_load_uint8(
-	const hrt_address		addr);
-
-/*! Read a 16-bit value from a device register or memory in the device
-
- \param	addr[in]			Local address
-
- \return device[addr]
- */
-extern uint16_t ia_css_device_load_uint16(
-	const hrt_address		addr);
-
-/*! Read a 32-bit value from a device register or memory in the device
-
- \param	addr[in]			Local address
-
- \return device[addr]
- */
-extern uint32_t ia_css_device_load_uint32(
-	const hrt_address		addr);
-
-/*! Read a 64-bit value from a device register or memory in the device
-
- \param	addr[in]			Local address
-
- \return device[addr]
- */
-extern uint64_t ia_css_device_load_uint64(
-	const hrt_address		addr);
-
-/*! Write an 8-bit value to a device register or memory in the device
-
- \param	addr[in]			Local address
- \param	data[in]			value
-
- \return none, device[addr] = value
- */
-extern void ia_css_device_store_uint8(
-	const hrt_address		addr,
-	const uint8_t			data);
-
-/*! Write a 16-bit value to a device register or memory in the device
-
- \param	addr[in]			Local address
- \param	data[in]			value
-
- \return none, device[addr] = value
- */
-extern void ia_css_device_store_uint16(
-	const hrt_address		addr,
-	const uint16_t			data);
-
-/*! Write a 32-bit value to a device register or memory in the device
-
- \param	addr[in]			Local address
- \param	data[in]			value
-
- \return none, device[addr] = value
- */
-extern void ia_css_device_store_uint32(
-	const hrt_address		addr,
-	const uint32_t			data);
-
-/*! Write a 64-bit value to a device register or memory in the device
-
- \param	addr[in]			Local address
- \param	data[in]			value
-
- \return none, device[addr] = value
- */
-extern void ia_css_device_store_uint64(
-	const hrt_address		addr,
-	const uint64_t			data);
-
-/*! Read an array of bytes from device registers or memory in the device
-
- \param	addr[in]			Local address
- \param	data[out]			pointer to the destination array
- \param	size[in]			number of bytes to read
-
- \return none
- */
-extern void ia_css_device_load(
-	const hrt_address		addr,
-	void					*data,
-	const size_t			size);
-
-/*! Write an array of bytes to device registers or memory in the device
-
- \param	addr[in]			Local address
- \param	data[in]			pointer to the source array
- \param	size[in]			number of bytes to write
-
- \return none
- */
-extern void ia_css_device_store(
-	const hrt_address		addr,
-	const void				*data,
-	const size_t			size);
-
-#endif /* __DEVICE_ACCESS_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/dma.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/dma.h
deleted file mode 100644
index d9dee691e3f8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/dma.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __DMA_H_INCLUDED__
-#define __DMA_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the DMA device. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- *
- */
-
-
-#include "system_local.h"
-#include "dma_local.h"
-
-#ifndef __INLINE_DMA__
-#define STORAGE_CLASS_DMA_H extern
-#define STORAGE_CLASS_DMA_C 
-#include "dma_public.h"
-#else  /* __INLINE_DMA__ */
-#define STORAGE_CLASS_DMA_H static inline
-#define STORAGE_CLASS_DMA_C static inline
-#include "dma_private.h"
-#endif /* __INLINE_DMA__ */
-
-#endif /* __DMA_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/error_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/error_support.h
deleted file mode 100644
index 6e5e5dd4107d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/error_support.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ERROR_SUPPORT_H_INCLUDED__
-#define __ERROR_SUPPORT_H_INCLUDED__
-
-#if defined(_MSC_VER)
-#include <errno.h>
-/*
- * Put here everything _MSC_VER specific not covered in
- * "errno.h"
- */
-#define EINVAL  22
-#define EBADE   52
-#define ENODATA 61
-#define ENOTCONN 107
-#define ENOTSUP 252
-#define ENOBUFS 233
-
-
-#elif defined(__KERNEL__)
-#include <linux/errno.h>
-/*
- * Put here everything __KERNEL__ specific not covered in
- * "errno.h"
- */
-#define ENOTSUP 252
-
-#elif defined(__GNUC__)
-#include <errno.h>
-/*
- * Put here everything __GNUC__ specific not covered in
- * "errno.h"
- */
-
-#else /* default is for the FIST environment */
-#include <errno.h>
-/*
- * Put here everything FIST specific not covered in
- * "errno.h"
- */
-
-#endif
-
-#define verifexit(cond,error_tag)  \
-do {                               \
-	if (!(cond)){              \
-		goto EXIT;         \
-	}                          \
-} while(0)
-
-#define verifjmpexit(cond)         \
-do {                               \
-	if (!(cond)){              \
-		goto EXIT;         \
-	}                          \
-} while(0)
-
-#endif /* __ERROR_SUPPORT_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/event_fifo.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/event_fifo.h
deleted file mode 100644
index df579e902796..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/event_fifo.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __EVENT_FIFO_H
-#define __EVENT_FIFO_H
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the IRQ device. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "event_fifo_local.h"
-
-#ifndef __INLINE_EVENT__
-#define STORAGE_CLASS_EVENT_H extern
-#define STORAGE_CLASS_EVENT_C 
-#include "event_fifo_public.h"
-#else  /* __INLINE_EVENT__ */
-#define STORAGE_CLASS_EVENT_H static inline
-#define STORAGE_CLASS_EVENT_C static inline
-#include "event_fifo_private.h"
-#endif /* __INLINE_EVENT__ */
-
-#endif /* __EVENT_FIFO_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/fifo_monitor.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/fifo_monitor.h
deleted file mode 100644
index f10c4fa2e32b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/fifo_monitor.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __FIFO_MONITOR_H_INCLUDED__
-#define __FIFO_MONITOR_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "fifo_monitor_local.h"
-
-#ifndef __INLINE_FIFO_MONITOR__
-#define STORAGE_CLASS_FIFO_MONITOR_H extern
-#define STORAGE_CLASS_FIFO_MONITOR_C 
-#include "fifo_monitor_public.h"
-#else  /* __INLINE_FIFO_MONITOR__ */
-#define STORAGE_CLASS_FIFO_MONITOR_H static inline
-#define STORAGE_CLASS_FIFO_MONITOR_C static inline
-#include "fifo_monitor_private.h"
-#endif /* __INLINE_FIFO_MONITOR__ */
-
-#endif /* __FIFO_MONITOR_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gdc_device.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gdc_device.h
deleted file mode 100644
index 75c6854c8e7b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gdc_device.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GDC_DEVICE_H_INCLUDED__
-#define __GDC_DEVICE_H_INCLUDED__
-
-/* The file gdc.h already exists */
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the GDC device. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "gdc_local.h"
-
-#ifndef __INLINE_GDC__
-#define STORAGE_CLASS_GDC_H extern
-#define STORAGE_CLASS_GDC_C 
-#include "gdc_public.h"
-#else  /* __INLINE_GDC__ */
-#define STORAGE_CLASS_GDC_H static inline
-#define STORAGE_CLASS_GDC_C static inline
-#include "gdc_private.h"
-#endif /* __INLINE_GDC__ */
-
-#endif /* __GDC_DEVICE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gp_device.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gp_device.h
deleted file mode 100644
index aba94e623043..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gp_device.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GP_DEVICE_H_INCLUDED__
-#define __GP_DEVICE_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "gp_device_local.h"
-
-#ifndef __INLINE_GP_DEVICE__
-#define STORAGE_CLASS_GP_DEVICE_H extern
-#define STORAGE_CLASS_GP_DEVICE_C 
-#include "gp_device_public.h"
-#else  /* __INLINE_GP_DEVICE__ */
-#define STORAGE_CLASS_GP_DEVICE_H static inline
-#define STORAGE_CLASS_GP_DEVICE_C static inline
-#include "gp_device_private.h"
-#endif /* __INLINE_GP_DEVICE__ */
-
-#endif /* __GP_DEVICE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gp_timer.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gp_timer.h
deleted file mode 100644
index d5d2df24e11a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gp_timer.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GP_TIMER_H_INCLUDED__
-#define __GP_TIMER_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"    /*GP_TIMER_BASE address */
-#include "gp_timer_local.h"  /*GP_TIMER register offsets */
-
-#ifndef __INLINE_GP_TIMER__
-#define STORAGE_CLASS_GP_TIMER_H extern
-#define STORAGE_CLASS_GP_TIMER_C
-#include "gp_timer_public.h"   /* functions*/
-#else  /* __INLINE_GP_TIMER__ */
-#define STORAGE_CLASS_GP_TIMER_H static inline
-#define STORAGE_CLASS_GP_TIMER_C static inline
-#include "gp_timer_private.h"  /* inline functions*/
-#endif /* __INLINE_GP_TIMER__ */
-
-#endif /* __GP_TIMER_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gpio.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gpio.h
deleted file mode 100644
index d37f7166aa4a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/gpio.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GPIO_H_INCLUDED__
-#define __GPIO_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "gpio_local.h"
-
-#ifndef __INLINE_GPIO__
-#define STORAGE_CLASS_GPIO_H extern
-#define STORAGE_CLASS_GPIO_C 
-#include "gpio_public.h"
-#else  /* __INLINE_GPIO__ */
-#define STORAGE_CLASS_GPIO_H static inline
-#define STORAGE_CLASS_GPIO_C static inline
-#include "gpio_private.h"
-#endif /* __INLINE_GPIO__ */
-
-#endif /* __GPIO_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/hmem.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/hmem.h
deleted file mode 100644
index a82fd3a21e98..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/hmem.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __HMEM_H_INCLUDED__
-#define __HMEM_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the HMEM device. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "hmem_local.h"
-
-#ifndef __INLINE_HMEM__
-#define STORAGE_CLASS_HMEM_H extern
-#define STORAGE_CLASS_HMEM_C 
-#include "hmem_public.h"
-#else  /* __INLINE_HMEM__ */
-#define STORAGE_CLASS_HMEM_H static inline
-#define STORAGE_CLASS_HMEM_C static inline
-#include "hmem_private.h"
-#endif /* __INLINE_HMEM__ */
-
-#endif /* __HMEM_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/csi_rx_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/csi_rx_public.h
deleted file mode 100644
index 426d022d3a26..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/csi_rx_public.h
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __CSI_RX_PUBLIC_H_INCLUDED__
-#define __CSI_RX_PUBLIC_H_INCLUDED__
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-/*****************************************************
- *
- * Native command interface (NCI).
- *
- *****************************************************/
-/**
- * @brief Get the csi rx frontend state.
- * Get the state of the csi rx frontend regiester-set.
- *
- * @param[in]	id	The global unique ID of the csi rx fe controller.
- * @param[out]	state	Point to the register-state.
- */
-extern void csi_rx_fe_ctrl_get_state(
-		const csi_rx_frontend_ID_t ID,
-		csi_rx_fe_ctrl_state_t *state);
-/**
- * @brief Dump the csi rx frontend state.
- * Dump the state of the csi rx frontend regiester-set.
- *
- * @param[in]	id	The global unique ID of the csi rx fe controller.
- * @param[in]	state	Point to the register-state.
- */
-extern void csi_rx_fe_ctrl_dump_state(
-		const csi_rx_frontend_ID_t ID,
-		csi_rx_fe_ctrl_state_t *state);
-/**
- * @brief Get the state of the csi rx fe dlane.
- * Get the state of the register set per dlane process.
- *
- * @param[in]	id			The global unique ID of the input-buffer controller.
- * @param[in]	lane		The lane ID.
- * @param[out]	state		Point to the dlane state.
- */
-extern void csi_rx_fe_ctrl_get_dlane_state(
-		const csi_rx_frontend_ID_t ID,
-		const uint32_t lane,
-		csi_rx_fe_ctrl_lane_t *dlane_state);
-/**
- * @brief Get the csi rx backend state.
- * Get the state of the csi rx backend regiester-set.
- *
- * @param[in]	id	The global unique ID of the csi rx be controller.
- * @param[out]	state	Point to the register-state.
- */
-extern void csi_rx_be_ctrl_get_state(
-		const csi_rx_backend_ID_t ID,
-		csi_rx_be_ctrl_state_t *state);
-/**
- * @brief Dump the csi rx backend state.
- * Dump the state of the csi rx backend regiester-set.
- *
- * @param[in]	id	The global unique ID of the csi rx be controller.
- * @param[in]	state	Point to the register-state.
- */
-extern void csi_rx_be_ctrl_dump_state(
-		const csi_rx_backend_ID_t ID,
-		csi_rx_be_ctrl_state_t *state);
-/* end of NCI */
-
-/*****************************************************
- *
- * Device level interface (DLI).
- *
- *****************************************************/
-/**
- * @brief Load the register value.
- * Load the value of the register of the csi rx fe.
- *
- * @param[in]	ID	The global unique ID for the ibuf-controller instance.
- * @param[in]	reg	The offet address of the register.
- *
- * @return the value of the register.
- */
-extern hrt_data csi_rx_fe_ctrl_reg_load(
-	const csi_rx_frontend_ID_t ID,
-	const hrt_address reg);
-/**
- * @brief Store a value to the register.
- * Store a value to the registe of the csi rx fe.
- *
- * @param[in]	ID		The global unique ID for the ibuf-controller instance.
- * @param[in]	reg		The offet address of the register.
- * @param[in]	value	The value to be stored.
- *
- */
-extern void csi_rx_fe_ctrl_reg_store(
-	const csi_rx_frontend_ID_t ID,
-	const hrt_address reg,
-	const hrt_data value);
-/**
- * @brief Load the register value.
- * Load the value of the register of the csirx be.
- *
- * @param[in]	ID	The global unique ID for the ibuf-controller instance.
- * @param[in]	reg	The offet address of the register.
- *
- * @return the value of the register.
- */
-extern hrt_data csi_rx_be_ctrl_reg_load(
-	const csi_rx_backend_ID_t ID,
-	const hrt_address reg);
-/**
- * @brief Store a value to the register.
- * Store a value to the registe of the csi rx be.
- *
- * @param[in]	ID		The global unique ID for the ibuf-controller instance.
- * @param[in]	reg		The offet address of the register.
- * @param[in]	value	The value to be stored.
- *
- */
-extern void csi_rx_be_ctrl_reg_store(
-	const csi_rx_backend_ID_t ID,
-	const hrt_address reg,
-	const hrt_data value);
-/* end of DLI */
-#endif /* USE_INPUT_SYSTEM_VERSION_2401 */
-#endif /* __CSI_RX_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/debug_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/debug_public.h
deleted file mode 100644
index 90b4ba7e023f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/debug_public.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __DEBUG_PUBLIC_H_INCLUDED__
-#define __DEBUG_PUBLIC_H_INCLUDED__
-
-#include <type_support.h>
-#include "system_types.h"
-
-/*! brief
- *
- * Simple queuing trace buffer for debug data
- * instantiatable in SP DMEM
- *
- * The buffer has a remote and and a local store
- * which contain duplicate data (when in sync).
- * The buffers are automatically synched when the
- * user dequeues, or manualy using the synch function
- *
- * An alternative (storage efficient) implementation
- * could manage the buffers to contain unique data
- *
- * The buffer empty status is computed from local
- * state which does not reflect the presence of data
- * in the remote buffer (unless the alternative
- * implementation is followed)
- */
-
-typedef struct debug_data_s		debug_data_t;
-typedef struct debug_data_ddr_s	debug_data_ddr_t;
-
-extern debug_data_t				*debug_data_ptr;
-extern hrt_address				debug_buffer_address;
-extern hrt_vaddress				debug_buffer_ddr_address;
-
-/*! Check the empty state of the local debug data buffer
- 
- \return isEmpty(buffer)
- */
-STORAGE_CLASS_DEBUG_H bool is_debug_buffer_empty(void);
-
-/*! Dequeue a token from the debug data buffer
- 
- \return isEmpty(buffer)?0:buffer[head]
- */
-STORAGE_CLASS_DEBUG_H hrt_data debug_dequeue(void);
-
-/*! Synchronise the remote buffer to the local buffer
- 
- \return none
- */
-STORAGE_CLASS_DEBUG_H void debug_synch_queue(void);
-
-/*! Synchronise the remote buffer to the local buffer
- 
- \return none
- */
-STORAGE_CLASS_DEBUG_H void debug_synch_queue_isp(void);
-
-
-/*! Synchronise the remote buffer to the local buffer
- 
- \return none
- */
-STORAGE_CLASS_DEBUG_H void debug_synch_queue_ddr(void);
-
-/*! Set the offset/address of the (remote) debug buffer
- 
- \return none
- */
-extern void debug_buffer_init(
-	const hrt_address		addr);
-
-/*! Set the offset/address of the (remote) debug buffer
- 
- \return none
- */
-extern void debug_buffer_ddr_init(
-	const hrt_vaddress		addr);
-
-/*! Set the (remote) operating mode of the debug buffer
- 
- \return none
- */
-extern void debug_buffer_setmode(
-	const debug_buf_mode_t	mode);
-
-#endif /* __DEBUG_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/dma_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/dma_public.h
deleted file mode 100644
index 1d5e38ffe938..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/dma_public.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __DMA_PUBLIC_H_INCLUDED__
-#define __DMA_PUBLIC_H_INCLUDED__
-
-#include "system_types.h"
-
-typedef struct dma_state_s		dma_state_t;
-
-/*! Read the control registers of DMA[ID]
-
- \param	ID[in]				DMA identifier
- \param	state[out]			input formatter state structure
-
- \return none, state = DMA[ID].state
- */
-extern void dma_get_state(
-	const dma_ID_t		ID,
-	dma_state_t			*state);
-
-/*! Write to a control register of DMA[ID]
-
- \param	ID[in]				DMA identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return none, DMA[ID].ctrl[reg] = value
- */
-STORAGE_CLASS_DMA_H void dma_reg_store(
-	const dma_ID_t		ID,
-	const unsigned int	reg,
-	const hrt_data		value);
-
-/*! Read from a control register of DMA[ID]
-
- \param	ID[in]				DMA identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return DMA[ID].ctrl[reg]
- */
-STORAGE_CLASS_DMA_H hrt_data dma_reg_load(
-	const dma_ID_t		ID,
-	const unsigned int	reg);
-
-
-/*! Set maximum burst size of DMA[ID]
-
- \param ID[in]				DMA identifier
- \param conn[in]			Connection to set max burst size for
- \param max_burst_size[in]		Maximum burst size in words
-
- \return none
-*/
-void
-dma_set_max_burst_size(
-	dma_ID_t		ID,
-	dma_connection		conn,
-	uint32_t		max_burst_size);
-
-#endif /* __DMA_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/event_fifo_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/event_fifo_public.h
deleted file mode 100644
index d95bc7070f4c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/event_fifo_public.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __EVENT_FIFO_PUBLIC_H
-#define __EVENT_FIFO_PUBLIC_H
-
-#include <type_support.h>
-#include "system_types.h"
-
-/*! Blocking read from an event source EVENT[ID]
- 
- \param	ID[in]				EVENT identifier
-
- \return none, dequeue(event_queue[ID])
- */
-STORAGE_CLASS_EVENT_H void event_wait_for(
-	const event_ID_t		ID);
-
-/*! Conditional blocking wait for an event source EVENT[ID]
- 
- \param	ID[in]				EVENT identifier
- \param	cnd[in]				predicate
-
- \return none, if(cnd) dequeue(event_queue[ID])
- */
-STORAGE_CLASS_EVENT_H void cnd_event_wait_for(
-	const event_ID_t		ID,
-	const bool				cnd);
-
-/*! Blocking read from an event source EVENT[ID]
- 
- \param	ID[in]				EVENT identifier
-
- \return dequeue(event_queue[ID])
- */
-STORAGE_CLASS_EVENT_H hrt_data event_receive_token(
-	const event_ID_t		ID);
-
-/*! Blocking write to an event sink EVENT[ID]
- 
- \param	ID[in]				EVENT identifier
- \param	token[in]			token to be written on the event
-
- \return none, enqueue(event_queue[ID])
- */
-STORAGE_CLASS_EVENT_H void event_send_token(
-	const event_ID_t		ID,
-	const hrt_data			token);
-
-/*! Query an event source EVENT[ID]
- 
- \param	ID[in]				EVENT identifier
-
- \return !isempty(event_queue[ID])
- */
-STORAGE_CLASS_EVENT_H bool is_event_pending(
-	const event_ID_t		ID);
-
-/*! Query an event sink EVENT[ID]
- 
- \param	ID[in]				EVENT identifier
-
- \return !isfull(event_queue[ID])
- */
-STORAGE_CLASS_EVENT_H bool can_event_send_token(
-	const event_ID_t		ID);
-
-#endif /* __EVENT_FIFO_PUBLIC_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/fifo_monitor_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/fifo_monitor_public.h
deleted file mode 100644
index 329f5d5049f2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/fifo_monitor_public.h
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __FIFO_MONITOR_PUBLIC_H_INCLUDED__
-#define __FIFO_MONITOR_PUBLIC_H_INCLUDED__
-
-#include "system_types.h"
-
-typedef struct fifo_channel_state_s		fifo_channel_state_t;
-typedef struct fifo_switch_state_s		fifo_switch_state_t;
-typedef struct fifo_monitor_state_s		fifo_monitor_state_t;
-
-/*! Set a fifo switch multiplex
- 
- \param	ID[in]				FIFO_MONITOR identifier
- \param	switch_id[in]		fifo switch identifier
- \param	sel[in]				fifo switch selector
-
- \return none, fifo_switch[switch_id].sel = sel
- */
-STORAGE_CLASS_FIFO_MONITOR_H void fifo_switch_set(
-	const fifo_monitor_ID_t		ID,
-	const fifo_switch_t			switch_id,
-	const hrt_data				sel);
-
-/*! Get a fifo switch multiplex
- 
- \param	ID[in]				FIFO_MONITOR identifier
- \param	switch_id[in]		fifo switch identifier
-
- \return fifo_switch[switch_id].sel
- */
-STORAGE_CLASS_FIFO_MONITOR_H hrt_data fifo_switch_get(
-	const fifo_monitor_ID_t		ID,
-	const fifo_switch_t			switch_id);
-
-/*! Read the state of FIFO_MONITOR[ID]
- 
- \param	ID[in]				FIFO_MONITOR identifier
- \param	state[out]			fifo monitor state structure
-
- \return none, state = FIFO_MONITOR[ID].state
- */
-extern void fifo_monitor_get_state(
-	const fifo_monitor_ID_t		ID,
-	fifo_monitor_state_t		*state);
-
-/*! Read the state of a fifo channel
- 
- \param	ID[in]				FIFO_MONITOR identifier
- \param	channel_id[in]		fifo channel identifier
- \param	state[out]			fifo channel state structure
-
- \return none, state = fifo_channel[channel_id].state
- */
-extern void fifo_channel_get_state(
-	const fifo_monitor_ID_t		ID,
-	const fifo_channel_t		channel_id,
-	fifo_channel_state_t		*state);
-
-/*! Read the state of a fifo switch
- 
- \param	ID[in]				FIFO_MONITOR identifier
- \param	switch_id[in]		fifo switch identifier
- \param	state[out]			fifo switch state structure
-
- \return none, state = fifo_switch[switch_id].state
- */
-extern void fifo_switch_get_state(
-	const fifo_monitor_ID_t		ID,
-	const fifo_switch_t			switch_id,
-	fifo_switch_state_t			*state);
-
-/*! Write to a control register of FIFO_MONITOR[ID]
- 
- \param	ID[in]				FIFO_MONITOR identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return none, FIFO_MONITOR[ID].ctrl[reg] = value
- */
-STORAGE_CLASS_FIFO_MONITOR_H void fifo_monitor_reg_store(
-	const fifo_monitor_ID_t		ID,
-	const unsigned int			reg,
-	const hrt_data				value);
-
-/*! Read from a control register of FIFO_MONITOR[ID]
- 
- \param	ID[in]				FIFO_MONITOR identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return FIFO_MONITOR[ID].ctrl[reg]
- */
-STORAGE_CLASS_FIFO_MONITOR_H hrt_data fifo_monitor_reg_load(
-	const fifo_monitor_ID_t		ID,
-	const unsigned int			reg);
-
-#endif /* __FIFO_MONITOR_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gdc_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gdc_public.h
deleted file mode 100644
index d09d1e320306..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gdc_public.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GDC_PUBLIC_H_INCLUDED__
-#define __GDC_PUBLIC_H_INCLUDED__
-
-/*! Write the bicubic interpolation table of GDC[ID]
-
- \param	ID[in]				GDC identifier
- \param data[in]			The data matrix to be written
-
- \pre
-	- data must point to a matrix[4][HRT_GDC_N]
-
- \implementation dependent
-	- The value of "HRT_GDC_N" is device specific
-	- The LUT should not be partially written
-	- The LUT format is a quadri-phase interpolation
-	  table. The layout is device specific
-	- The range of the values data[n][m] is device
-	  specific
-
- \return none, GDC[ID].lut[0...3][0...HRT_GDC_N-1] = data
- */
-extern void gdc_lut_store(
-	const gdc_ID_t		ID,
-	const int			data[4][HRT_GDC_N]);
-
-/*! Convert the bicubic interpolation table of GDC[ID] to the ISP-specific format
-
- \param	ID[in]				GDC identifier
- \param in_lut[in]			The data matrix to be converted
- \param out_lut[out]			The data matrix as the output of conversion
- */
-extern void gdc_lut_convert_to_isp_format(
-	const int in_lut[4][HRT_GDC_N],
-	int out_lut[4][HRT_GDC_N]);
-
-/*! Return the integer representation of 1.0 of GDC[ID]
- 
- \param	ID[in]				GDC identifier
-
- \return unity
- */
-extern int gdc_get_unity(
-	const gdc_ID_t		ID);
-
-#endif /* __GDC_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gp_device_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gp_device_public.h
deleted file mode 100644
index acbce0fd658f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gp_device_public.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GP_DEVICE_PUBLIC_H_INCLUDED__
-#define __GP_DEVICE_PUBLIC_H_INCLUDED__
-
-#include "system_types.h"
-
-typedef struct gp_device_state_s		gp_device_state_t;
-
-/*! Read the state of GP_DEVICE[ID]
- 
- \param	ID[in]				GP_DEVICE identifier
- \param	state[out]			gp device state structure
-
- \return none, state = GP_DEVICE[ID].state
- */
-extern void gp_device_get_state(
-	const gp_device_ID_t		ID,
-	gp_device_state_t			*state);
-
-/*! Write to a control register of GP_DEVICE[ID]
-
- \param	ID[in]				GP_DEVICE identifier
- \param	reg_addr[in]		register byte address
- \param value[in]			The data to be written
-
- \return none, GP_DEVICE[ID].ctrl[reg] = value
- */
-STORAGE_CLASS_GP_DEVICE_H void gp_device_reg_store(
-	const gp_device_ID_t	ID,
-	const unsigned int		reg_addr,
-	const hrt_data			value);
-
-/*! Read from a control register of GP_DEVICE[ID]
- 
- \param	ID[in]				GP_DEVICE identifier
- \param	reg_addr[in]		register byte address
- \param value[in]			The data to be written
-
- \return GP_DEVICE[ID].ctrl[reg]
- */
-STORAGE_CLASS_GP_DEVICE_H hrt_data gp_device_reg_load(
-	const gp_device_ID_t	ID,
-	const hrt_address	reg_addr);
-
-#endif /* __GP_DEVICE_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gp_timer_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gp_timer_public.h
deleted file mode 100644
index 276e2fa9b1e7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gp_timer_public.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GP_TIMER_PUBLIC_H_INCLUDED__
-#define __GP_TIMER_PUBLIC_H_INCLUDED__
-
-#include "system_types.h"
-
-/*! initialize mentioned timer
-param ID		timer_id
-*/
-extern void
-gp_timer_init(gp_timer_ID_t ID);
-
-
-/*! read timer value for (platform selected)selected timer.
-param ID		timer_id
- \return uint32_t	32 bit timer value
-*/
-extern uint32_t
-gp_timer_read(gp_timer_ID_t ID);
-
-#endif /* __GP_TIMER_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gpio_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gpio_public.h
deleted file mode 100644
index 82eaa0d48bee..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/gpio_public.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __GPIO_PUBLIC_H_INCLUDED__
-#define __GPIO_PUBLIC_H_INCLUDED__
-
-#include "system_types.h"
-
-/*! Write to a control register of GPIO[ID]
-
- \param	ID[in]				GPIO identifier
- \param	reg_addr[in]		register byte address
- \param value[in]			The data to be written
-
- \return none, GPIO[ID].ctrl[reg] = value
- */
-STORAGE_CLASS_GPIO_H void gpio_reg_store(
-	const gpio_ID_t	ID,
-	const unsigned int		reg_addr,
-	const hrt_data			value);
-
-/*! Read from a control register of GPIO[ID]
- 
- \param	ID[in]				GPIO identifier
- \param	reg_addr[in]		register byte address
- \param value[in]			The data to be written
-
- \return GPIO[ID].ctrl[reg]
- */
-STORAGE_CLASS_GPIO_H hrt_data gpio_reg_load(
-	const gpio_ID_t	ID,
-	const unsigned int		reg_addr);
-
-#endif /* __GPIO_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/hmem_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/hmem_public.h
deleted file mode 100644
index 8538f86ab5e6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/hmem_public.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __HMEM_PUBLIC_H_INCLUDED__
-#define __HMEM_PUBLIC_H_INCLUDED__
-
-#include <linux/types.h>		/* size_t */
-
-/*! Return the size of HMEM[ID]
-
- \param	ID[in]				HMEM identifier
-
- \Note: The size is the byte size of the area it occupies
-		in the address map. I.e. disregarding internal structure
-
- \return sizeof(HMEM[ID])
- */
-STORAGE_CLASS_HMEM_H size_t sizeof_hmem(
-	const hmem_ID_t		ID);
-
-#endif /* __HMEM_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/ibuf_ctrl_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/ibuf_ctrl_public.h
deleted file mode 100644
index 98ee9947fb8e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/ibuf_ctrl_public.h
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IBUF_CTRL_PUBLIC_H_INCLUDED__
-#define __IBUF_CTRL_PUBLIC_H_INCLUDED__
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-/*****************************************************
- *
- * Native command interface (NCI).
- *
- *****************************************************/
-/**
- * @brief Get the ibuf-controller state.
- * Get the state of the ibuf-controller regiester-set.
- *
- * @param[in]	id		The global unique ID of the input-buffer controller.
- * @param[out]	state	Point to the register-state.
- */
-STORAGE_CLASS_IBUF_CTRL_H void ibuf_ctrl_get_state(
-		const ibuf_ctrl_ID_t ID,
-		ibuf_ctrl_state_t *state);
-
-/**
- * @brief Get the state of the ibuf-controller process.
- * Get the state of the register set per buf-controller process.
- *
- * @param[in]	id			The global unique ID of the input-buffer controller.
- * @param[in]	proc_id		The process ID.
- * @param[out]	state		Point to the process state.
- */
-STORAGE_CLASS_IBUF_CTRL_H void ibuf_ctrl_get_proc_state(
-		const ibuf_ctrl_ID_t ID,
-		const uint32_t proc_id,
-		ibuf_ctrl_proc_state_t *state);
-/**
- * @brief Dump the ibuf-controller state.
- * Dump the state of the ibuf-controller regiester-set.
- *
- * @param[in]	id		The global unique ID of the input-buffer controller.
- * @param[in]	state		Pointer to the register-state.
- */
-STORAGE_CLASS_IBUF_CTRL_H void ibuf_ctrl_dump_state(
-		const ibuf_ctrl_ID_t ID,
-		ibuf_ctrl_state_t *state);
-/* end of NCI */
-
-/*****************************************************
- *
- * Device level interface (DLI).
- *
- *****************************************************/
-/**
- * @brief Load the register value.
- * Load the value of the register of the ibuf-controller.
- *
- * @param[in]	ID	The global unique ID for the ibuf-controller instance.
- * @param[in]	reg	The offet address of the register.
- *
- * @return the value of the register.
- */
-STORAGE_CLASS_IBUF_CTRL_H hrt_data ibuf_ctrl_reg_load(
-	const ibuf_ctrl_ID_t ID,
-	const hrt_address reg);
-
-/**
- * @brief Store a value to the register.
- * Store a value to the registe of the ibuf-controller.
- *
- * @param[in]	ID		The global unique ID for the ibuf-controller instance.
- * @param[in]	reg		The offet address of the register.
- * @param[in]	value	The value to be stored.
- *
- */
-STORAGE_CLASS_IBUF_CTRL_H void ibuf_ctrl_reg_store(
-	const ibuf_ctrl_ID_t ID,
-	const hrt_address reg,
-	const hrt_data value);
-/* end of DLI */
-
-#endif /* USE_INPUT_SYSTEM_VERSION_2401 */
-#endif /* __IBUF_CTRL_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/input_formatter_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/input_formatter_public.h
deleted file mode 100644
index 2db70893daf9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/input_formatter_public.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_FORMATTER_PUBLIC_H_INCLUDED__
-#define __INPUT_FORMATTER_PUBLIC_H_INCLUDED__
-
-#include <type_support.h>
-#include "system_types.h"
-
-/*! Reset INPUT_FORMATTER[ID]
- 
- \param	ID[in]				INPUT_FORMATTER identifier
-
- \return none, reset(INPUT_FORMATTER[ID])
- */
-extern void input_formatter_rst(
-	const input_formatter_ID_t		ID);
-
-/*! Set the blocking mode of INPUT_FORMATTER[ID]
- 
- \param	ID[in]				INPUT_FORMATTER identifier
- \param	enable[in]			blocking enable flag
-
- \use
-	- In HW, the capture unit will deliver an infinite stream of frames,
-	  the input formatter will synchronise on the first SOF. In simulation
-	  there are only a fixed number of frames, presented only once. By
-	  enabling blocking the inputformatter will wait on the first presented
-	  frame, thus avoiding race in the simulation setup.
-
- \return none, INPUT_FORMATTER[ID].blocking_mode = enable
- */
-extern void input_formatter_set_fifo_blocking_mode(
-	const input_formatter_ID_t		ID,
-	const bool						enable);
-
-/*! Return the data alignment of INPUT_FORMATTER[ID]
- 
- \param	ID[in]				INPUT_FORMATTER identifier
-
- \return alignment(INPUT_FORMATTER[ID].data)
- */
-extern unsigned int input_formatter_get_alignment(
-	const input_formatter_ID_t		ID);
-
-/*! Read the source switch state into INPUT_FORMATTER[ID]
- 
- \param	ID[in]				INPUT_FORMATTER identifier
- \param	state[out]			input formatter switch state structure
-
- \return none, state = INPUT_FORMATTER[ID].switch_state
- */
-extern void input_formatter_get_switch_state(
-	const input_formatter_ID_t		ID,
-	input_formatter_switch_state_t	*state);
-
-/*! Read the control registers of INPUT_FORMATTER[ID]
- 
- \param	ID[in]				INPUT_FORMATTER identifier
- \param	state[out]			input formatter state structure
-
- \return none, state = INPUT_FORMATTER[ID].state
- */
-extern void input_formatter_get_state(
-	const input_formatter_ID_t		ID,
-	input_formatter_state_t			*state);
-
-/*! Read the control registers of bin copy INPUT_FORMATTER[ID]
- 
- \param	ID[in]				INPUT_FORMATTER identifier
- \param	state[out]			input formatter state structure
-
- \return none, state = INPUT_FORMATTER[ID].state
- */
-extern void input_formatter_bin_get_state(
-	const input_formatter_ID_t		ID,
-	input_formatter_bin_state_t		*state);
-
-/*! Write to a control register of INPUT_FORMATTER[ID]
- 
- \param	ID[in]				INPUT_FORMATTER identifier
- \param	reg_addr[in]		register byte address
- \param value[in]			The data to be written
-
- \return none, INPUT_FORMATTER[ID].ctrl[reg] = value
- */
-STORAGE_CLASS_INPUT_FORMATTER_H void input_formatter_reg_store(
-	const input_formatter_ID_t	ID,
-	const hrt_address		reg_addr,
-	const hrt_data				value);
-
-/*! Read from a control register of INPUT_FORMATTER[ID]
- 
- \param	ID[in]				INPUT_FORMATTER identifier
- \param	reg_addr[in]		register byte address
- \param value[in]			The data to be written
-
- \return INPUT_FORMATTER[ID].ctrl[reg]
- */
-STORAGE_CLASS_INPUT_FORMATTER_H hrt_data input_formatter_reg_load(
-	const input_formatter_ID_t	ID,
-	const unsigned int			reg_addr);
-
-#endif /* __INPUT_FORMATTER_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/input_system_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/input_system_public.h
deleted file mode 100644
index 6e37ff0fe0f9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/input_system_public.h
+++ /dev/null
@@ -1,376 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_SYSTEM_PUBLIC_H_INCLUDED__
-#define __INPUT_SYSTEM_PUBLIC_H_INCLUDED__
-
-#include <type_support.h>
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-#include "isys_public.h"
-#else
-
-typedef struct input_system_state_s		input_system_state_t;
-typedef struct receiver_state_s			receiver_state_t;
-
-/*! Read the state of INPUT_SYSTEM[ID]
-
- \param	ID[in]				INPUT_SYSTEM identifier
- \param	state[out]			input system state structure
-
- \return none, state = INPUT_SYSTEM[ID].state
- */
-extern void input_system_get_state(
-	const input_system_ID_t		ID,
-	input_system_state_t		*state);
-
-/*! Read the state of RECEIVER[ID]
-
- \param	ID[in]				RECEIVER identifier
- \param	state[out]			receiver state structure
-
- \return none, state = RECEIVER[ID].state
- */
-extern void receiver_get_state(
-	const rx_ID_t				ID,
-	receiver_state_t			*state);
-
-/*! Flag whether a MIPI format is YUV420
-
- \param	mipi_format[in]		MIPI format
-
- \return mipi_format == YUV420
- */
-extern bool is_mipi_format_yuv420(
-	const mipi_format_t			mipi_format);
-
-/*! Set compression parameters for cfg[cfg_ID] of RECEIVER[ID]
-
- \param	ID[in]				RECEIVER identifier
- \param	cfg_ID[in]			Configuration identifier
- \param	comp[in]			Compression method
- \param	pred[in]			Predictor method
-
- \NOTE: the storage of compression configuration is
-        implementation specific. The config can be
-        carried either on MIPI ports or on MIPI channels
-
- \return none, RECEIVER[ID].cfg[cfg_ID] = {comp, pred}
- */
-extern void receiver_set_compression(
-	const rx_ID_t				ID,
-	const unsigned int			cfg_ID,
-	const mipi_compressor_t		comp,
-	const mipi_predictor_t		pred);
-
-/*! Enable PORT[port_ID] of RECEIVER[ID]
-
- \param	ID[in]				RECEIVER identifier
- \param	port_ID[in]			mipi PORT identifier
- \param	cnd[in]				irq predicate
-
- \return None, enable(RECEIVER[ID].PORT[port_ID])
- */
-extern void receiver_port_enable(
-	const rx_ID_t				ID,
-	const enum mipi_port_id		port_ID,
-	const bool					cnd);
-
-/*! Flag if PORT[port_ID] of RECEIVER[ID] is enabled
-
- \param	ID[in]				RECEIVER identifier
- \param	port_ID[in]			mipi PORT identifier
-
- \return enable(RECEIVER[ID].PORT[port_ID]) == true
- */
-extern bool is_receiver_port_enabled(
-	const rx_ID_t				ID,
-	const enum mipi_port_id		port_ID);
-
-/*! Enable the IRQ channels of PORT[port_ID] of RECEIVER[ID]
-
- \param	ID[in]				RECEIVER identifier
- \param	port_ID[in]			mipi PORT identifier
- \param	irq_info[in]		irq channels
-
- \return None, enable(RECEIVER[ID].PORT[port_ID].irq_info)
- */
-extern void receiver_irq_enable(
-	const rx_ID_t				ID,
-	const enum mipi_port_id		port_ID,
-	const rx_irq_info_t			irq_info);
-
-/*! Return the IRQ status of PORT[port_ID] of RECEIVER[ID]
-
- \param	ID[in]				RECEIVER identifier
- \param	port_ID[in]			mipi PORT identifier
-
- \return RECEIVER[ID].PORT[port_ID].irq_info
- */
-extern rx_irq_info_t receiver_get_irq_info(
-	const rx_ID_t				ID,
-	const enum mipi_port_id		port_ID);
-
-/*! Clear the IRQ status of PORT[port_ID] of RECEIVER[ID]
-
- \param	ID[in]				RECEIVER identifier
- \param	port_ID[in]			mipi PORT identifier
- \param	irq_info[in]		irq status
-
- \return None, clear(RECEIVER[ID].PORT[port_ID].irq_info)
- */
-extern void receiver_irq_clear(
-	const rx_ID_t				ID,
-	const enum mipi_port_id			port_ID,
-	const rx_irq_info_t			irq_info);
-
-/*! Write to a control register of INPUT_SYSTEM[ID]
-
- \param	ID[in]				INPUT_SYSTEM identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return none, INPUT_SYSTEM[ID].ctrl[reg] = value
- */
-STORAGE_CLASS_INPUT_SYSTEM_H void input_system_reg_store(
-	const input_system_ID_t			ID,
-	const hrt_address			reg,
-	const hrt_data				value);
-
-/*! Read from a control register of INPUT_SYSTEM[ID]
-
- \param	ID[in]				INPUT_SYSTEM identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return INPUT_SYSTEM[ID].ctrl[reg]
- */
-STORAGE_CLASS_INPUT_SYSTEM_H hrt_data input_system_reg_load(
-	const input_system_ID_t			ID,
-	const hrt_address			reg);
-
-/*! Write to a control register of RECEIVER[ID]
-
- \param	ID[in]				RECEIVER identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return none, RECEIVER[ID].ctrl[reg] = value
- */
-STORAGE_CLASS_INPUT_SYSTEM_H void receiver_reg_store(
-	const rx_ID_t				ID,
-	const hrt_address			reg,
-	const hrt_data				value);
-
-/*! Read from a control register of RECEIVER[ID]
-
- \param	ID[in]				RECEIVER identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return RECEIVER[ID].ctrl[reg]
- */
-STORAGE_CLASS_INPUT_SYSTEM_H hrt_data receiver_reg_load(
-	const rx_ID_t				ID,
-	const hrt_address			reg);
-
-/*! Write to a control register of PORT[port_ID] of RECEIVER[ID]
-
- \param	ID[in]				RECEIVER identifier
- \param	port_ID[in]			mipi PORT identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return none, RECEIVER[ID].PORT[port_ID].ctrl[reg] = value
- */
-STORAGE_CLASS_INPUT_SYSTEM_H void receiver_port_reg_store(
-	const rx_ID_t				ID,
-	const enum mipi_port_id			port_ID,
-	const hrt_address			reg,
-	const hrt_data				value);
-
-/*! Read from a control register PORT[port_ID] of of RECEIVER[ID]
-
- \param	ID[in]				RECEIVER identifier
- \param	port_ID[in]			mipi PORT identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return RECEIVER[ID].PORT[port_ID].ctrl[reg]
- */
-STORAGE_CLASS_INPUT_SYSTEM_H hrt_data receiver_port_reg_load(
-	const rx_ID_t				ID,
-	const enum mipi_port_id		port_ID,
-	const hrt_address			reg);
-
-/*! Write to a control register of SUB_SYSTEM[sub_ID] of INPUT_SYSTEM[ID]
-
- \param	ID[in]				INPUT_SYSTEM identifier
- \param	port_ID[in]			sub system identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return none, INPUT_SYSTEM[ID].SUB_SYSTEM[sub_ID].ctrl[reg] = value
- */
-STORAGE_CLASS_INPUT_SYSTEM_H void input_system_sub_system_reg_store(
-	const input_system_ID_t			ID,
-	const sub_system_ID_t			sub_ID,
-	const hrt_address			reg,
-	const hrt_data				value);
-
-/*! Read from a control register SUB_SYSTEM[sub_ID] of INPUT_SYSTEM[ID]
-
- \param	ID[in]				INPUT_SYSTEM identifier
- \param	port_ID[in]			sub system identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return INPUT_SYSTEM[ID].SUB_SYSTEM[sub_ID].ctrl[reg]
- */
-STORAGE_CLASS_INPUT_SYSTEM_H hrt_data input_system_sub_system_reg_load(
-	const input_system_ID_t		ID,
-	const sub_system_ID_t		sub_ID,
-	const hrt_address			reg);
-
-
-
-///////////////////////////////////////////////////////////////////////////
-//
-//    Functions for configuration phase on input system.
-//
-///////////////////////////////////////////////////////////////////////////
-
-// Function that resets current configuration.
-// remove the argument since it should be private.
-input_system_error_t input_system_configuration_reset(void);
-
-// Function that commits current configuration.
-// remove the argument since it should be private.
-input_system_error_t input_system_configuration_commit(void);
-
-///////////////////////////////////////////////////////////////////////////
-//
-// User functions:
-//		(encoded generic function)
-//    - no checking
-//    - decoding name and agruments into the generic (channel) configuration
-//    function.
-//
-///////////////////////////////////////////////////////////////////////////
-
-
-// FIFO channel config function user
-
-input_system_error_t	input_system_csi_fifo_channel_cfg(
-	uint32_t				ch_id,
-	input_system_csi_port_t	port,
-	backend_channel_cfg_t	backend_ch,
-	target_cfg2400_t			target
-);
-
-input_system_error_t	input_system_csi_fifo_channel_with_counting_cfg(
-	uint32_t				ch_id,
-	uint32_t				nof_frame,
-	input_system_csi_port_t	port,
-	backend_channel_cfg_t	backend_ch,
-	uint32_t				mem_region_size,
-	uint32_t				nof_mem_regions,
-	target_cfg2400_t			target
-);
-
-
-// SRAM channel config function user
-
-input_system_error_t	input_system_csi_sram_channel_cfg(
-	uint32_t				ch_id,
-	input_system_csi_port_t	port,
-	backend_channel_cfg_t	backend_ch,
-	uint32_t				csi_mem_region_size,
-	uint32_t				csi_nof_mem_regions,
-	target_cfg2400_t 			target
-);
-
-
-//XMEM channel config function user
-
-input_system_error_t	input_system_csi_xmem_channel_cfg(
-	uint32_t 				ch_id,
-	input_system_csi_port_t port,
-	backend_channel_cfg_t	backend_ch,
-	uint32_t 				mem_region_size,
-	uint32_t 				nof_mem_regions,
-	uint32_t 				acq_mem_region_size,
-	uint32_t 				acq_nof_mem_regions,
-	target_cfg2400_t 			target,
-	uint32_t 				nof_xmem_buffers
-);
-
-input_system_error_t	input_system_csi_xmem_capture_only_channel_cfg(
-	uint32_t 				ch_id,
-	uint32_t 				nof_frames,
-	input_system_csi_port_t port,
-	uint32_t 				csi_mem_region_size,
-	uint32_t 				csi_nof_mem_regions,
-	uint32_t 				acq_mem_region_size,
-	uint32_t 				acq_nof_mem_regions,
-	target_cfg2400_t 			target
-);
-
-input_system_error_t	input_system_csi_xmem_acquire_only_channel_cfg(
-	uint32_t 				ch_id,
-	uint32_t 				nof_frames,
-	input_system_csi_port_t port,
-	backend_channel_cfg_t	backend_ch,
-	uint32_t 				acq_mem_region_size,
-	uint32_t 				acq_nof_mem_regions,
-	target_cfg2400_t 			target
-);
-
-// Non - CSI channel config function user
-
-input_system_error_t	input_system_prbs_channel_cfg(
-	uint32_t 		ch_id,
-	uint32_t		nof_frames,
-	uint32_t		seed,
-	uint32_t		sync_gen_width,
-	uint32_t		sync_gen_height,
-	uint32_t		sync_gen_hblank_cycles,
-	uint32_t		sync_gen_vblank_cycles,
-	target_cfg2400_t	target
-);
-
-
-input_system_error_t	input_system_tpg_channel_cfg(
-	uint32_t 		ch_id,
-	uint32_t 		nof_frames,//not used yet
-	uint32_t		x_mask,
-	uint32_t		y_mask,
-	uint32_t		x_delta,
-	uint32_t		y_delta,
-	uint32_t		xy_mask,
-	uint32_t		sync_gen_width,
-	uint32_t		sync_gen_height,
-	uint32_t		sync_gen_hblank_cycles,
-	uint32_t		sync_gen_vblank_cycles,
-	target_cfg2400_t	target
-);
-
-
-input_system_error_t	input_system_gpfifo_channel_cfg(
-	uint32_t 		ch_id,
-	uint32_t 		nof_frames,
-	target_cfg2400_t	target
-);
-#endif /* #ifdef USE_INPUT_SYSTEM_VERSION_2401 */
-
-#endif /* __INPUT_SYSTEM_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/irq_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/irq_public.h
deleted file mode 100644
index 9aeaf8f082d2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/irq_public.h
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IRQ_PUBLIC_H_INCLUDED__
-#define __IRQ_PUBLIC_H_INCLUDED__
-
-#include <type_support.h>
-#include "system_types.h"
-
-/*! Read the control registers of IRQ[ID]
-
- \param	ID[in]				IRQ identifier
- \param	state[out]			irq controller state structure
-
- \return none, state = IRQ[ID].state
- */
-extern void irq_controller_get_state(
-	const irq_ID_t				ID,
-	irq_controller_state_t		*state);
-
-/*! Write to a control register of IRQ[ID]
-
- \param	ID[in]				IRQ identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return none, IRQ[ID].ctrl[reg] = value
- */
-STORAGE_CLASS_IRQ_H void irq_reg_store(
-	const irq_ID_t		ID,
-	const unsigned int	reg,
-	const hrt_data		value);
-
-/*! Read from a control register of IRQ[ID]
-
- \param	ID[in]				IRQ identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return IRQ[ID].ctrl[reg]
- */
-STORAGE_CLASS_IRQ_H hrt_data irq_reg_load(
-	const irq_ID_t		ID,
-	const unsigned int	reg);
-
-/*! Enable an IRQ channel of IRQ[ID] with a mode
-
- \param	ID[in]				IRQ (device) identifier
- \param	irq[in]				IRQ (channel) identifier
-
- \return none, enable(IRQ[ID].channel[irq_ID])
- */
-extern void irq_enable_channel(
-	const irq_ID_t				ID,
-	const unsigned int			irq_ID);
-
-/*! Enable pulse interrupts for IRQ[ID] with a mode
-
- \param	ID[in]				IRQ (device) identifier
- \param	enable				enable/disable pulse interrupts
-
- \return none
- */
-extern void irq_enable_pulse(
-	const irq_ID_t	ID,
-	bool 			pulse);
-
-/*! Disable an IRQ channel of IRQ[ID]
-
- \param	ID[in]				IRQ (device) identifier
- \param	irq[in]				IRQ (channel) identifier
-
- \return none, disable(IRQ[ID].channel[irq_ID])
- */
-extern void irq_disable_channel(
-	const irq_ID_t				ID,
-	const unsigned int			irq);
-
-/*! Clear the state of all IRQ channels of IRQ[ID]
-
- \param	ID[in]				IRQ (device) identifier
-
- \return none, clear(IRQ[ID].channel[])
- */
-extern void irq_clear_all(
-	const irq_ID_t				ID);
-
-/*! Return the ID of a signalling IRQ channel of IRQ[ID]
-
- \param	ID[in]				IRQ (device) identifier
- \param irq_id[out]			active IRQ (channel) identifier
-
- \Note: This function operates as strtok(), based on the return
-  state the user is informed if there are additional signalling
-  channels
-
- \return state(IRQ[ID])
- */
-extern enum hrt_isp_css_irq_status irq_get_channel_id(
-	const irq_ID_t				ID,
-	unsigned int				*irq_id);
-
-/*! Raise an interrupt on channel irq_id of device IRQ[ID]
-
- \param	ID[in]				IRQ (device) identifier
- \param	irq_id[in]			IRQ (channel) identifier
-
- \return none, signal(IRQ[ID].channel[irq_id])
- */
-extern void irq_raise(
-	const irq_ID_t				ID,
-	const irq_sw_channel_id_t	irq_id);
-
-/*! Test if any IRQ channel of the virtual super IRQ has raised a signal
-
- \return any(VIRQ.channel[irq_ID] != 0)
- */
-extern bool any_virq_signal(void);
-
-/*! Enable an IRQ channel of the virtual super IRQ
-
- \param	irq[in]				IRQ (channel) identifier
- \param	en[in]				predicate channel enable
-
- \return none, VIRQ.channel[irq_ID].enable = en
- */
-extern void cnd_virq_enable_channel(
-	const virq_id_t				irq_ID,
-	const bool					en);
-
-/*! Clear the state of all IRQ channels of the virtual super IRQ
-
- \return none, clear(VIRQ.channel[])
- */
-extern void virq_clear_all(void);
-
-/*! Clear the IRQ info state of the virtual super IRQ
-
- \param irq_info[in/out]	The IRQ (channel) state
-
- \return none
- */
-extern void virq_clear_info(
-	virq_info_t					*irq_info);
-
-/*! Return the ID of a signalling IRQ channel of the virtual super IRQ
-
- \param irq_id[out]			active IRQ (channel) identifier
-
- \Note: This function operates as strtok(), based on the return
-  state the user is informed if there are additional signalling
-  channels
-
- \return state(IRQ[...])
- */
-extern enum hrt_isp_css_irq_status virq_get_channel_id(
-	virq_id_t					*irq_id);
-
-/*! Return the IDs of all signaling IRQ channels of the virtual super IRQ
-
- \param irq_info[out]		all active IRQ (channel) identifiers
-
- \Note: Unlike "irq_get_channel_id()" this function returns all
-  channel signaling info. The new info is OR'd with the current
-  info state. N.B. this is the same as repeatedly calling the function
-  "irq_get_channel_id()" in a (non-blocked) handler routine
-
- \return (error(state(IRQ[...]))
- */
-extern enum hrt_isp_css_irq_status virq_get_channel_signals(
-	virq_info_t					*irq_info);
-
-#endif /* __IRQ_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isp_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isp_public.h
deleted file mode 100644
index 808ec050efc0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isp_public.h
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISP_PUBLIC_H_INCLUDED__
-#define __ISP_PUBLIC_H_INCLUDED__
-
-#include <type_support.h>
-#include "system_types.h"
-
-/*! Enable or disable the program complete irq signal of ISP[ID]
-
- \param	ID[in]				SP identifier
- \param	cnd[in]				predicate
-
- \return none, if(cnd) enable(ISP[ID].irq) else disable(ISP[ID].irq)
- */
-extern void cnd_isp_irq_enable(
-	const isp_ID_t		ID,
-	const bool			cnd);
-
-/*! Read the state of cell ISP[ID]
-
- \param	ID[in]				ISP identifier
- \param	state[out]			isp state structure
- \param	stall[out]			isp stall conditions
-
- \return none, state = ISP[ID].state, stall = ISP[ID].stall
- */
-extern void isp_get_state(
-	const isp_ID_t		ID,
-	isp_state_t			*state,
-	isp_stall_t			*stall);
-
-
-/*! Write to the status and control register of ISP[ID]
-
- \param	ID[in]				ISP identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return none, ISP[ID].sc[reg] = value
- */
-STORAGE_CLASS_ISP_H void isp_ctrl_store(
-	const isp_ID_t		ID,
-	const unsigned int	reg,
-	const hrt_data		value);
-
-/*! Read from the status and control register of ISP[ID]
-
- \param	ID[in]				ISP identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return ISP[ID].sc[reg]
- */
-STORAGE_CLASS_ISP_H hrt_data isp_ctrl_load(
-	const isp_ID_t		ID,
-	const unsigned int	reg);
-
-/*! Get the status of a bitfield in the control register of ISP[ID]
-
- \param	ID[in]				ISP identifier
- \param	reg[in]				register index
- \param bit[in]				The bit index to be checked
-
- \return  (ISP[ID].sc[reg] & (1<<bit)) != 0
- */
-STORAGE_CLASS_ISP_H bool isp_ctrl_getbit(
-	const isp_ID_t		ID,
-	const unsigned int	reg,
-	const unsigned int	bit);
-
-/*! Set a bitfield in the control register of ISP[ID]
-
- \param	ID[in]				ISP identifier
- \param	reg[in]				register index
- \param bit[in]				The bit index to be set
-
- \return none, ISP[ID].sc[reg] |= (1<<bit)
- */
-STORAGE_CLASS_ISP_H void isp_ctrl_setbit(
-	const isp_ID_t		ID,
-	const unsigned int	reg,
-	const unsigned int	bit);
-
-/*! Clear a bitfield in the control register of ISP[ID]
-
- \param	ID[in]				ISP identifier
- \param	reg[in]				register index
- \param bit[in]				The bit index to be set
-
- \return none, ISP[ID].sc[reg] &= ~(1<<bit)
- */
-STORAGE_CLASS_ISP_H void isp_ctrl_clearbit(
-	const isp_ID_t		ID,
-	const unsigned int	reg,
-	const unsigned int	bit);
-
-/*! Write to the DMEM of ISP[ID]
-
- \param	ID[in]				ISP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be written
- \param size[in]			The size(in bytes) of the data to be written
-
- \return none, ISP[ID].dmem[addr...addr+size-1] = data
- */
-STORAGE_CLASS_ISP_H void isp_dmem_store(
-	const isp_ID_t		ID,
-	unsigned int		addr,
-	const void			*data,
-	const size_t		size);
-
-/*! Read from the DMEM of ISP[ID]
-
- \param	ID[in]				ISP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be read
- \param size[in]			The size(in bytes) of the data to be read
-
- \return none, data = ISP[ID].dmem[addr...addr+size-1]
- */
-STORAGE_CLASS_ISP_H void isp_dmem_load(
-	const isp_ID_t		ID,
-	const unsigned int	addr,
-	void				*data,
-	const size_t		size);
-
-/*! Write a 32-bit datum to the DMEM of ISP[ID]
-
- \param	ID[in]				ISP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be written
- \param size[in]			The size(in bytes) of the data to be written
-
- \return none, ISP[ID].dmem[addr] = data
- */
-STORAGE_CLASS_ISP_H void isp_dmem_store_uint32(
-	const isp_ID_t		ID,
-	unsigned int		addr,
-	const uint32_t		data);
-
-/*! Load a 32-bit datum from the DMEM of ISP[ID]
-
- \param	ID[in]				ISP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be read
- \param size[in]			The size(in bytes) of the data to be read
-
- \return none, data = ISP[ID].dmem[addr]
- */
-STORAGE_CLASS_ISP_H uint32_t isp_dmem_load_uint32(
-	const isp_ID_t		ID,
-	const unsigned int	addr);
-
-/*! Concatenate the LSW and MSW into a double precision word
-
- \param	x0[in]				Integer containing the LSW
- \param	x1[in]				Integer containing the MSW
-
- \return x0 | (x1 << bits_per_vector_element)
- */
-STORAGE_CLASS_ISP_H uint32_t isp_2w_cat_1w(
-	const uint16_t		x0,
-	const uint16_t		x1);
-
-unsigned isp_is_ready(isp_ID_t ID);
-
-unsigned isp_is_sleeping(isp_ID_t ID);
-
-void isp_start(isp_ID_t ID);
-
-void isp_wake(isp_ID_t ID);
-
-#endif /* __ISP_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_dma_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_dma_public.h
deleted file mode 100644
index 4b1603895f06..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_dma_public.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_DMA_PUBLIC_H_INCLUDED__
-#define __ISYS_DMA_PUBLIC_H_INCLUDED__
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-
-#include "system_types.h"
-#include "type_support.h"
-
-STORAGE_CLASS_ISYS2401_DMA_H void isys2401_dma_reg_store(
-	const isys2401_dma_ID_t dma_id,
-	const unsigned int	reg,
-	const hrt_data		value);
-
-STORAGE_CLASS_ISYS2401_DMA_H hrt_data isys2401_dma_reg_load(
-	const isys2401_dma_ID_t dma_id,
-	const unsigned int	reg);
-
-extern void isys2401_dma_set_max_burst_size(
-	const isys2401_dma_ID_t dma_id,
-	uint32_t		max_burst_size);
-
-#endif /* USE_INPUT_SYSTEM_VERSION_2401 */
-
-#endif /* __ISYS_DMA_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_irq_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_irq_public.h
deleted file mode 100644
index c3e6f76f8c4d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_irq_public.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_IRQ_PUBLIC_H__
-#define __ISYS_IRQ_PUBLIC_H__
-
-#include "isys_irq_global.h"
-#include "isys_irq_local.h"
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-
-STORAGE_CLASS_ISYS2401_IRQ_H void isys_irqc_state_get(
-	const isys_irq_ID_t	isys_irqc_id,
-	isys_irqc_state_t	*state);
-
-STORAGE_CLASS_ISYS2401_IRQ_H void isys_irqc_state_dump(
-	const isys_irq_ID_t	isys_irqc_id,
-	const isys_irqc_state_t *state);
-
-STORAGE_CLASS_ISYS2401_IRQ_H void isys_irqc_reg_store(
-	const isys_irq_ID_t	isys_irqc_id,
-	const unsigned int	reg_idx,
-	const hrt_data		value);
-
-STORAGE_CLASS_ISYS2401_IRQ_H hrt_data isys_irqc_reg_load(
-	const isys_irq_ID_t	isys_irqc_id,
-	const unsigned int	reg_idx);
-
-STORAGE_CLASS_ISYS2401_IRQ_H void isys_irqc_status_enable(
-	const isys_irq_ID_t	isys_irqc_id);
-
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2401) */
-
-#endif	/* __ISYS_IRQ_PUBLIC_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_public.h
deleted file mode 100644
index 097dde852c8a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_public.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_PUBLIC_H_INCLUDED__
-#define __ISYS_PUBLIC_H_INCLUDED__
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-/*! Read the state of INPUT_SYSTEM[ID]
- \param ID[in]		INPUT_SYSTEM identifier
- \param state[out]	pointer to input system state structure
- \return none, state = INPUT_SYSTEM[ID].state
- */
-STORAGE_CLASS_INPUT_SYSTEM_H input_system_err_t input_system_get_state(
-	const input_system_ID_t	ID,
-	input_system_state_t *state);
-/*! Dump the state of INPUT_SYSTEM[ID]
- \param ID[in]		INPUT_SYSTEM identifier
- \param state[in]	pointer to input system state structure
- \return none
- \depends on host supplied print function as part of ia_css_init()
- */
-STORAGE_CLASS_INPUT_SYSTEM_H void input_system_dump_state(
-	const input_system_ID_t	ID,
-	input_system_state_t *state);
-#endif /* USE_INPUT_SYSTEM_VERSION_2401 */
-#endif /* __ISYS_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_stream2mmio_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_stream2mmio_public.h
deleted file mode 100644
index 6c53ca9df96c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/isys_stream2mmio_public.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_STREAM2MMIO_PUBLIC_H_INCLUDED__
-#define __ISYS_STREAM2MMIO_PUBLIC_H_INCLUDED__
-
-/*****************************************************
- *
- * Native command interface (NCI).
- *
- *****************************************************/
-/**
- * @brief Get the stream2mmio-controller state.
- * Get the state of the stream2mmio-controller regiester-set.
- *
- * @param[in]	id		The global unique ID of the steeam2mmio controller.
- * @param[out]	state	Point to the register-state.
- */
-STORAGE_CLASS_STREAM2MMIO_H void stream2mmio_get_state(
-		const stream2mmio_ID_t ID,
-		stream2mmio_state_t *state);
-
-/**
- * @brief Get the state of the stream2mmio-controller sidess.
- * Get the state of the register set per buf-controller sidess.
- *
- * @param[in]	id		The global unique ID of the steeam2mmio controller.
- * @param[in]	sid_id		The sid ID.
- * @param[out]	state		Point to the sid state.
- */
-STORAGE_CLASS_STREAM2MMIO_H void stream2mmio_get_sid_state(
-		const stream2mmio_ID_t ID,
-		const stream2mmio_sid_ID_t sid_id,
-		stream2mmio_sid_state_t *state);
-/* end of NCI */
-
-/*****************************************************
- *
- * Device level interface (DLI).
- *
- *****************************************************/
-/**
- * @brief Load the register value.
- * Load the value of the register of the stream2mmio-controller.
- *
- * @param[in]	ID	The global unique ID for the stream2mmio-controller instance.
- * @param[in]	sid_id	The SID in question.
- * @param[in]	reg_idx	The offet address of the register.
- *
- * @return the value of the register.
- */
-STORAGE_CLASS_STREAM2MMIO_H hrt_data stream2mmio_reg_load(
-		const stream2mmio_ID_t ID,
-		const stream2mmio_sid_ID_t sid_id,
-		const uint32_t reg_idx);
-
-/**
- * @brief Dump the SID processor state.
- * Dump the state of the sid regiester-set.
- *
- * @param[in]	state		Pointer to the register-state.
- */
-STORAGE_CLASS_STREAM2MMIO_H void stream2mmio_print_sid_state(
-		stream2mmio_sid_state_t	*state);
-/**
- * @brief Dump the stream2mmio state.
- * Dump the state of the ibuf-controller regiester-set.
- *
- * @param[in]	id		The global unique ID of the st2mmio
- * @param[in]	state		Pointer to the register-state.
- */
-STORAGE_CLASS_STREAM2MMIO_H void stream2mmio_dump_state(
-		const stream2mmio_ID_t ID,
-		stream2mmio_state_t *state);
-/**
- * @brief Store a value to the register.
- * Store a value to the registe of the stream2mmio-controller.
- *
- * @param[in]	ID		The global unique ID for the stream2mmio-controller instance.
- * @param[in]	reg		The offet address of the register.
- * @param[in]	value	The value to be stored.
- *
- */
-STORAGE_CLASS_STREAM2MMIO_H void stream2mmio_reg_store(
-		const stream2mmio_ID_t ID,
-		const hrt_address reg,
-		const hrt_data value);
-/* end of DLI */
-
-#endif /* __ISYS_STREAM2MMIO_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/mmu_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/mmu_public.h
deleted file mode 100644
index 0a13eda73607..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/mmu_public.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __MMU_PUBLIC_H_INCLUDED__
-#define __MMU_PUBLIC_H_INCLUDED__
-
-#include "system_types.h"
-
-/*! Set the page table base index of MMU[ID]
-
- \param	ID[in]				MMU identifier
- \param	base_index[in]		page table base index
-
- \return none, MMU[ID].page_table_base_index = base_index
- */
-extern void mmu_set_page_table_base_index(
-	const mmu_ID_t		ID,
-	const hrt_data		base_index);
-
-/*! Get the page table base index of MMU[ID]
-
- \param	ID[in]				MMU identifier
- \param	base_index[in]		page table base index
-
- \return MMU[ID].page_table_base_index
- */
-extern hrt_data mmu_get_page_table_base_index(
-	const mmu_ID_t		ID);
-
-/*! Invalidate the page table cache of MMU[ID]
-
- \param	ID[in]				MMU identifier
-
- \return none
- */
-extern void mmu_invalidate_cache(
-	const mmu_ID_t		ID);
-
-
-/*! Invalidate the page table cache of all MMUs
-
- \return none
- */
-extern void mmu_invalidate_cache_all(void);
-
-/*! Write to a control register of MMU[ID]
-
- \param	ID[in]				MMU identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return none, MMU[ID].ctrl[reg] = value
- */
-STORAGE_CLASS_MMU_H void mmu_reg_store(
-	const mmu_ID_t		ID,
-	const unsigned int	reg,
-	const hrt_data		value);
-
-/*! Read from a control register of MMU[ID]
-
- \param	ID[in]				MMU identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return MMU[ID].ctrl[reg]
- */
-STORAGE_CLASS_MMU_H hrt_data mmu_reg_load(
-	const mmu_ID_t		ID,
-	const unsigned int	reg);
-
-#endif /* __MMU_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/pixelgen_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/pixelgen_public.h
deleted file mode 100644
index f597e07d7c4f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/pixelgen_public.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __PIXELGEN_PUBLIC_H_INCLUDED__
-#define __PIXELGEN_PUBLIC_H_INCLUDED__
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-/*****************************************************
- *
- * Native command interface (NCI).
- *
- *****************************************************/
-/**
- * @brief Get the pixelgen state.
- * Get the state of the pixelgen regiester-set.
- *
- * @param[in]	id	The global unique ID of the pixelgen controller.
- * @param[out]	state	Point to the register-state.
- */
-STORAGE_CLASS_PIXELGEN_H void pixelgen_ctrl_get_state(
-		const pixelgen_ID_t ID,
-		pixelgen_ctrl_state_t *state);
-/**
- * @brief Dump the pixelgen state.
- * Dump the state of the pixelgen regiester-set.
- *
- * @param[in]	id	The global unique ID of the pixelgen controller.
- * @param[in]	state	Point to the register-state.
- */
-STORAGE_CLASS_PIXELGEN_H void pixelgen_ctrl_dump_state(
-		const pixelgen_ID_t ID,
-		pixelgen_ctrl_state_t *state);
-/* end of NCI */
-
-/*****************************************************
- *
- * Device level interface (DLI).
- *
- *****************************************************/
-/**
- * @brief Load the register value.
- * Load the value of the register of the pixelgen
- *
- * @param[in]	ID	The global unique ID for the pixelgen instance.
- * @param[in]	reg	The offet address of the register.
- *
- * @return the value of the register.
- */
-STORAGE_CLASS_PIXELGEN_H hrt_data pixelgen_ctrl_reg_load(
-	const pixelgen_ID_t ID,
-	const hrt_address reg);
-/**
- * @brief Store a value to the register.
- * Store a value to the registe of the pixelgen
- *
- * @param[in]	ID		The global unique ID for the pixelgen.
- * @param[in]	reg		The offet address of the register.
- * @param[in]	value	The value to be stored.
- *
- */
-STORAGE_CLASS_PIXELGEN_H void pixelgen_ctrl_reg_store(
-	const pixelgen_ID_t ID,
-	const hrt_address reg,
-	const hrt_data value);
-/* end of DLI */
-
-#endif /* USE_INPUT_SYSTEM_VERSION_2401 */
-#endif /* __PIXELGEN_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/sp_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/sp_public.h
deleted file mode 100644
index 974ce6a33b4b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/sp_public.h
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SP_PUBLIC_H_INCLUDED__
-#define __SP_PUBLIC_H_INCLUDED__
-
-#include <type_support.h>
-#include "system_types.h"
-
-typedef struct sp_state_s		sp_state_t;
-typedef struct sp_stall_s		sp_stall_t;
-
-/*! Enable or disable the program complete irq signal of SP[ID]
- 
- \param	ID[in]				SP identifier
- \param	cnd[in]				predicate
-
- \return none, if(cnd) enable(SP[ID].irq) else disable(SP[ID].irq)
- */
-extern void cnd_sp_irq_enable(
-	const sp_ID_t		ID,
-	const bool			cnd);
-
-/*! Read the state of cell SP[ID]
- 
- \param	ID[in]				SP identifier
- \param	state[out]			sp state structure
- \param	stall[out]			isp stall conditions
-
- \return none, state = SP[ID].state, stall = SP[ID].stall
- */
-extern void sp_get_state(
-	const sp_ID_t		ID,
-	sp_state_t			*state,
-	sp_stall_t			*stall);
-
-/*! Write to the status and control register of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return none, SP[ID].sc[reg] = value
- */
-STORAGE_CLASS_SP_H void sp_ctrl_store(
-	const sp_ID_t		ID,
-	const hrt_address	reg,
-	const hrt_data		value);
-
-/*! Read from the status and control register of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return SP[ID].sc[reg]
- */
-STORAGE_CLASS_SP_H hrt_data sp_ctrl_load(
-	const sp_ID_t		ID,
-	const hrt_address	reg);
-
-/*! Get the status of a bitfield in the control register of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	reg[in]				register index
- \param bit[in]				The bit index to be checked
-
- \return  (SP[ID].sc[reg] & (1<<bit)) != 0
- */
-STORAGE_CLASS_SP_H bool sp_ctrl_getbit(
-	const sp_ID_t		ID,
-	const hrt_address	reg,
-	const unsigned int	bit);
-
-/*! Set a bitfield in the control register of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	reg[in]				register index
- \param bit[in]				The bit index to be set
-
- \return none, SP[ID].sc[reg] |= (1<<bit)
- */
-STORAGE_CLASS_SP_H void sp_ctrl_setbit(
-	const sp_ID_t		ID,
-	const hrt_address	reg,
-	const unsigned int	bit);
-
-/*! Clear a bitfield in the control register of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	reg[in]				register index
- \param bit[in]				The bit index to be set
-
- \return none, SP[ID].sc[reg] &= ~(1<<bit)
- */
-STORAGE_CLASS_SP_H void sp_ctrl_clearbit(
-	const sp_ID_t		ID,
-	const hrt_address	reg,
-	const unsigned int	bit);
-
-/*! Write to the DMEM of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be written
- \param size[in]			The size(in bytes) of the data to be written
-
- \return none, SP[ID].dmem[addr...addr+size-1] = data
- */
-STORAGE_CLASS_SP_H void sp_dmem_store(
-	const sp_ID_t		ID,
-	hrt_address		addr,
-	const void			*data,
-	const size_t		size);
-
-/*! Read from the DMEM of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be read
- \param size[in]			The size(in bytes) of the data to be read
-
- \return none, data = SP[ID].dmem[addr...addr+size-1]
- */
-STORAGE_CLASS_SP_H void sp_dmem_load(
-	const sp_ID_t		ID,
-	const hrt_address	addr,
-	void			*data,
-	const size_t		size);
-
-/*! Write a 8-bit datum to the DMEM of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be written
- \param size[in]			The size(in bytes) of the data to be written
-
- \return none, SP[ID].dmem[addr...addr+size-1] = data
- */
-STORAGE_CLASS_SP_H void sp_dmem_store_uint8(
-	const sp_ID_t		ID,
-	hrt_address		addr,
-	const uint8_t		data);
-
-/*! Write a 16-bit datum to the DMEM of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be written
- \param size[in]			The size(in bytes) of the data to be written
-
- \return none, SP[ID].dmem[addr...addr+size-1] = data
- */
-STORAGE_CLASS_SP_H void sp_dmem_store_uint16(
-	const sp_ID_t		ID,
-	hrt_address		addr,
-	const uint16_t		data);
-
-/*! Write a 32-bit datum to the DMEM of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be written
- \param size[in]			The size(in bytes) of the data to be written
-
- \return none, SP[ID].dmem[addr...addr+size-1] = data
- */
-STORAGE_CLASS_SP_H void sp_dmem_store_uint32(
-	const sp_ID_t		ID,
-	hrt_address		addr,
-	const uint32_t		data);
-
-/*! Load a 8-bit datum from the DMEM of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be read
- \param size[in]			The size(in bytes) of the data to be read
-
- \return none, data = SP[ID].dmem[addr...addr+size-1]
- */
-STORAGE_CLASS_SP_H uint8_t sp_dmem_load_uint8(
-	const sp_ID_t		ID,
-	const hrt_address	addr);
-
-/*! Load a 16-bit datum from the DMEM of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be read
- \param size[in]			The size(in bytes) of the data to be read
-
- \return none, data = SP[ID].dmem[addr...addr+size-1]
- */
-STORAGE_CLASS_SP_H uint16_t sp_dmem_load_uint16(
-	const sp_ID_t		ID,
-	const hrt_address	addr);
-
-/*! Load a 32-bit datum from the DMEM of SP[ID]
-
- \param	ID[in]				SP identifier
- \param	addr[in]			the address in DMEM
- \param data[in]			The data to be read
- \param size[in]			The size(in bytes) of the data to be read
-
- \return none, data = SP[ID].dmem[addr...addr+size-1]
- */
-STORAGE_CLASS_SP_H uint32_t sp_dmem_load_uint32(
-	const sp_ID_t		ID,
-	const hrt_address	addr);
-
-#endif /* __SP_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/tag_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/tag_public.h
deleted file mode 100644
index 22ef747f3d4a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/tag_public.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __TAG_PUBLIC_H_INCLUDED__
-#define __TAG_PUBLIC_H_INCLUDED__
-
-/**
- * @brief	Creates the tag description from the given parameters.
- * @param[in]	num_captures
- * @param[in]	skip
- * @param[in]	offset
- * @param[out]	tag_descr
- */
-void
-sh_css_create_tag_descr(int num_captures,
-			unsigned int skip,
-			int offset,
-			unsigned int exp_id,
-			struct sh_css_tag_descr *tag_descr);
-
-/**
- * @brief	Encodes the members of tag description into a 32-bit value.
- * @param[in]	tag		Pointer to the tag description
- * @return	(unsigned int)	Encoded 32-bit tag-info
- */
-unsigned int
-sh_css_encode_tag_descr(struct sh_css_tag_descr *tag);
-
-#endif /* __TAG_PUBLIC_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/timed_ctrl_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/timed_ctrl_public.h
deleted file mode 100644
index b3becac16f49..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/timed_ctrl_public.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __TIMED_CTRL_PUBLIC_H_INCLUDED__
-#define __TIMED_CTRL_PUBLIC_H_INCLUDED__
-
-#include "system_types.h"
-
-/*! Write to a control register of TIMED_CTRL[ID]
-
- \param	ID[in]				TIMED_CTRL identifier
- \param	reg_addr[in]		register byte address
- \param value[in]			The data to be written
-
- \return none, TIMED_CTRL[ID].ctrl[reg] = value
- */
-STORAGE_CLASS_TIMED_CTRL_H void timed_ctrl_reg_store(
-	const timed_ctrl_ID_t	ID,
-	const unsigned int		reg_addr,
-	const hrt_data			value);
-
-extern void timed_ctrl_snd_commnd(
-	const timed_ctrl_ID_t				ID,
-	hrt_data				mask,
-	hrt_data				condition,
-	hrt_data				counter,
-	hrt_address				addr,
-	hrt_data				value);
-
-extern void timed_ctrl_snd_sp_commnd(
-	const timed_ctrl_ID_t				ID,
-	hrt_data				mask,
-	hrt_data				condition,
-	hrt_data				counter,
-	const sp_ID_t			SP_ID,
-	hrt_address				offset,
-	hrt_data				value);
-
-extern void timed_ctrl_snd_gpio_commnd(
-	const timed_ctrl_ID_t				ID,
-	hrt_data				mask,
-	hrt_data				condition,
-	hrt_data				counter,
-	const gpio_ID_t			GPIO_ID,
-	hrt_address				offset,
-	hrt_data				value);
-
-#endif /* __TIMED_CTRL_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/vamem_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/vamem_public.h
deleted file mode 100644
index cee15d0ab2d8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/vamem_public.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __VAMEM_PUBLIC_H_INCLUDED__
-#define __VAMEM_PUBLIC_H_INCLUDED__
-
-
-
-#endif /* __VAMEM_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/vmem_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/vmem_public.h
deleted file mode 100644
index e9801c0fe147..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/host/vmem_public.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __VMEM_PUBLIC_H_INCLUDED__
-#define __VMEM_PUBLIC_H_INCLUDED__
-
-#include "isp.h" /* tmemvectoru */
-
-#endif /* __VMEM_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/ibuf_ctrl.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/ibuf_ctrl.h
deleted file mode 100644
index c7d9095472b1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/ibuf_ctrl.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IBUF_CTRL_H_INCLUDED__
-#define __IBUF_CTRL_H_INCLUDED__
-
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- * - system and cell agnostic interfaces, constants and identifiers
- * - public:  system agnostic, cell specific interfaces
- * - private: system dependent, cell specific interfaces &
- *   inline implementations
- * - global:  system specific constants and identifiers
- * - local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "ibuf_ctrl_local.h"
-
-#ifndef __INLINE_IBUF_CTRL__
-#define STORAGE_CLASS_IBUF_CTRL_H extern
-#define STORAGE_CLASS_IBUF_CTRL_C
-#include "ibuf_ctrl_public.h"
-#else  /* __INLINE_IBUF_CTRL__ */
-#define STORAGE_CLASS_IBUF_CTRL_H static inline
-#define STORAGE_CLASS_IBUF_CTRL_C static inline
-#include "ibuf_ctrl_private.h"
-#endif /* __INLINE_IBUF_CTRL__ */
-
-#endif /* __IBUF_CTRL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/input_formatter.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/input_formatter.h
deleted file mode 100644
index eeaaecdd57ba..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/input_formatter.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_FORMATTER_H_INCLUDED__
-#define __INPUT_FORMATTER_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "input_formatter_local.h"
-
-#ifndef __INLINE_INPUT_FORMATTER__
-#define STORAGE_CLASS_INPUT_FORMATTER_H extern
-#define STORAGE_CLASS_INPUT_FORMATTER_C 
-#include "input_formatter_public.h"
-#else  /* __INLINE_INPUT_FORMATTER__ */
-#define STORAGE_CLASS_INPUT_FORMATTER_H static inline
-#define STORAGE_CLASS_INPUT_FORMATTER_C static inline
-#include "input_formatter_private.h"
-#endif /* __INLINE_INPUT_FORMATTER__ */
-
-#endif /* __INPUT_FORMATTER_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/input_system.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/input_system.h
deleted file mode 100644
index 3f02d9ec9588..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/input_system.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __INPUT_SYSTEM_H_INCLUDED__
-#define __INPUT_SYSTEM_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "input_system_local.h"
-
-#ifndef __INLINE_INPUT_SYSTEM__
-#define STORAGE_CLASS_INPUT_SYSTEM_H extern
-#define STORAGE_CLASS_INPUT_SYSTEM_C 
-#include "input_system_public.h"
-#else  /* __INLINE_INPUT_SYSTEM__ */
-#define STORAGE_CLASS_INPUT_SYSTEM_H static inline
-#define STORAGE_CLASS_INPUT_SYSTEM_C static inline
-#include "input_system_private.h"
-#endif /* __INLINE_INPUT_SYSTEM__ */
-
-#endif /* __INPUT_SYSTEM_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/irq.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/irq.h
deleted file mode 100644
index e1446388dee5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/irq.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IRQ_H_INCLUDED__
-#define __IRQ_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the IRQ device. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "irq_local.h"
-
-#ifndef __INLINE_IRQ__
-#define STORAGE_CLASS_IRQ_H extern
-#define STORAGE_CLASS_IRQ_C 
-#include "irq_public.h"
-#else  /* __INLINE_IRQ__ */
-#define STORAGE_CLASS_IRQ_H static inline
-#define STORAGE_CLASS_IRQ_C static inline
-#include "irq_private.h"
-#endif /* __INLINE_IRQ__ */
-
-#endif /* __IRQ_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isp.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isp.h
deleted file mode 100644
index b916953e7f47..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isp.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISP_H_INCLUDED__
-#define __ISP_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the ISP cell. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "isp_local.h"
-
-#ifndef __INLINE_ISP__
-#define STORAGE_CLASS_ISP_H extern
-#define STORAGE_CLASS_ISP_C 
-#include "isp_public.h"
-#else  /* __INLINE_iSP__ */
-#define STORAGE_CLASS_ISP_H static inline
-#define STORAGE_CLASS_ISP_C static inline
-#include "isp_private.h"
-#endif /* __INLINE_ISP__ */
-
-#endif /* __ISP_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_dma.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_dma.h
deleted file mode 100644
index 76aba114a5c1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_dma.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_DMA_H_INCLUDED__
-#define __ISYS_DMA_H_INCLUDED__
-
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- * - system and cell agnostic interfaces, constants and identifiers
- * - public:  system agnostic, cell specific interfaces
- * - private: system dependent, cell specific interfaces &
- *   inline implementations
- * - global:  system specific constants and identifiers
- * - local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "isys_dma_local.h"
-
-#ifndef __INLINE_ISYS2401_DMA__
-#define STORAGE_CLASS_ISYS2401_DMA_H extern
-#define STORAGE_CLASS_ISYS2401_DMA_C
-#include "isys_dma_public.h"
-#else  /* __INLINE_ISYS2401_DMA__ */
-#define STORAGE_CLASS_ISYS2401_DMA_H static inline
-#define STORAGE_CLASS_ISYS2401_DMA_C static inline
-#include "isys_dma_private.h"
-#endif /* __INLINE_ISYS2401_DMA__ */
-
-#endif /* __ISYS_DMA_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_irq.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_irq.h
deleted file mode 100644
index d3f64cfd0b7d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_irq.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ISYS_IRQ_H__
-#define __IA_CSS_ISYS_IRQ_H__
-
-#include <type_support.h>
-#include <system_local.h>
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-
-#ifndef __INLINE_ISYS2401_IRQ__
-
-#define STORAGE_CLASS_ISYS2401_IRQ_H extern
-#define STORAGE_CLASS_ISYS2401_IRQ_C extern
-#include "isys_irq_public.h"
-
-#else  /* __INLINE_ISYS2401_IRQ__ */
-
-#define STORAGE_CLASS_ISYS2401_IRQ_H static inline
-#define STORAGE_CLASS_ISYS2401_IRQ_C static inline
-#include "isys_irq_private.h"
-
-#endif /* __INLINE_ISYS2401_IRQ__ */
-
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2401) */
-
-#endif	/* __IA_CSS_ISYS_IRQ_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_stream2mmio.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_stream2mmio.h
deleted file mode 100644
index 16fbf9d25eba..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/isys_stream2mmio.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_STREAM2MMIO_H_INCLUDED__
-#define __ISYS_STREAM2MMIO_H_INCLUDED__
-
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- * - system and cell agnostic interfaces, constants and identifiers
- * - public:  system agnostic, cell specific interfaces
- * - private: system dependent, cell specific interfaces &
- *   inline implementations
- * - global:  system specific constants and identifiers
- * - local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "isys_stream2mmio_local.h"
-
-#ifndef __INLINE_STREAM2MMIO__
-#define STORAGE_CLASS_STREAM2MMIO_H extern
-#define STORAGE_CLASS_STREAM2MMIO_C
-#include "isys_stream2mmio_public.h"
-#else  /* __INLINE_STREAM2MMIO__ */
-#define STORAGE_CLASS_STREAM2MMIO_H static inline
-#define STORAGE_CLASS_STREAM2MMIO_C static inline
-#include "isys_stream2mmio_private.h"
-#endif /* __INLINE_STREAM2MMIO__ */
-
-#endif /* __ISYS_STREAM2MMIO_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/math_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/math_support.h
deleted file mode 100644
index 6436dae0007e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/math_support.h
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __MATH_SUPPORT_H
-#define __MATH_SUPPORT_H
-
-#if defined(__KERNEL__)
-#include <linux/kernel.h> /* Override the definition of max/min from linux kernel*/
-#endif /*__KERNEL__*/
-
-#if defined(_MSC_VER)
-#include <stdlib.h> /* Override the definition of max/min from stdlib.h*/
-#endif /* _MSC_VER */
-
-/* in case we have min/max/MIN/MAX macro's undefine them */
-#ifdef min
-#undef min
-#endif
-#ifdef max
-#undef max
-#endif
-#ifdef MIN /* also defined in include/hrt/numeric.h from SDK */
-#undef MIN
-#endif
-#ifdef MAX
-#undef MAX
-#endif
-#ifdef ABS
-#undef ABS
-#endif
-
-#define IS_ODD(a)            ((a) & 0x1)
-#define IS_EVEN(a)           (!IS_ODD(a))
-
-/* force a value to a lower even value */
-#define EVEN_FLOOR(x)        ((x) & ~1)
-
-#ifdef ISP2401
-/* If the number is odd, find the next even number */
-#define EVEN_CEIL(x)         ((IS_ODD(x)) ? ((x) + 1) : (x))
-
-#endif
-/* A => B */
-#define IMPLIES(a, b)        (!(a) || (b))
-
-#define ABS(a)               ((a) >= 0 ? (a) : -(a))
-
-/* for preprocessor and array sizing use MIN and MAX
-   otherwise use min and max */
-#define MAX(a, b)            (((a) > (b)) ? (a) : (b))
-#define MIN(a, b)            (((a) < (b)) ? (a) : (b))
-#ifdef ISP2401
-#define ROUND_DIV(a, b)      (((b) != 0) ? ((a) + ((b) >> 1)) / (b) : 0)
-#endif
-#define CEIL_DIV(a, b)       (((b) != 0) ? ((a) + (b) - 1) / (b) : 0)
-#define CEIL_MUL(a, b)       (CEIL_DIV(a, b) * (b))
-#define CEIL_MUL2(a, b)      (((a) + (b) - 1) & ~((b) - 1))
-#define CEIL_SHIFT(a, b)     (((a) + (1 << (b)) - 1)>>(b))
-#define CEIL_SHIFT_MUL(a, b) (CEIL_SHIFT(a, b) << (b))
-#ifdef ISP2401
-#define ROUND_HALF_DOWN_DIV(a, b)	(((b) != 0) ? ((a) + (b / 2) - 1) / (b) : 0)
-#define ROUND_HALF_DOWN_MUL(a, b)	(ROUND_HALF_DOWN_DIV(a, b) * (b))
-#endif
-
-
-/*To Find next power of 2 number from x */
-#define bit2(x)            ((x)      | ((x) >> 1))
-#define bit4(x)            (bit2(x)  | (bit2(x) >> 2))
-#define bit8(x)            (bit4(x)  | (bit4(x) >> 4))
-#define bit16(x)           (bit8(x)  | (bit8(x) >> 8))
-#define bit32(x)           (bit16(x) | (bit16(x) >> 16))
-#define NEXT_POWER_OF_2(x) (bit32(x-1) + 1)
-
-
-/* min and max should not be macros as they will evaluate their arguments twice.
-   if you really need a macro (e.g. for CPP or for initializing an array)
-   use MIN() and MAX(), otherwise use min() and max().
-
-
-*/
-
-#if !defined(PIPE_GENERATION)
-
-#ifndef INLINE_MATH_SUPPORT_UTILS
-/*
-This macro versions are added back as we are mixing types in usage of inline.
-This causes corner cases of calculations to be incorrect due to conversions
-between signed and unsigned variables or overflows.
-Before the addition of the inline functions, max, min and ceil_div were macros
-and therefore adding them back.
-
-Leaving out the other math utility functions as they are newly added
-*/
-
-#define max(a, b)		(MAX(a, b))
-#define min(a, b)		(MIN(a, b))
-#define ceil_div(a, b)		(CEIL_DIV(a, b))
-
-#else /* !defined(INLINE_MATH_SUPPORT_UTILS) */
-
-static inline int max(int a, int b)
-{
-	return MAX(a, b);
-}
-
-static inline int min(int a, int b)
-{
-	return MIN(a, b);
-}
-
-static inline unsigned int ceil_div(unsigned int a, unsigned int b)
-{
-	return CEIL_DIV(a, b);
-}
-#endif /* !defined(INLINE_MATH_SUPPORT_UTILS) */
-
-static inline unsigned int umax(unsigned int a, unsigned int b)
-{
-	return MAX(a, b);
-}
-
-static inline unsigned int umin(unsigned int a, unsigned int b)
-{
-	return MIN(a, b);
-}
-
-
-static inline unsigned int ceil_mul(unsigned int a, unsigned int b)
-{
-	return CEIL_MUL(a, b);
-}
-
-static inline unsigned int ceil_mul2(unsigned int a, unsigned int b)
-{
-	return CEIL_MUL2(a, b);
-}
-
-static inline unsigned int ceil_shift(unsigned int a, unsigned int b)
-{
-	return CEIL_SHIFT(a, b);
-}
-
-static inline unsigned int ceil_shift_mul(unsigned int a, unsigned int b)
-{
-	return CEIL_SHIFT_MUL(a, b);
-}
-
-#ifdef ISP2401
-static inline unsigned int round_half_down_div(unsigned int a, unsigned int b)
-{
-	return ROUND_HALF_DOWN_DIV(a, b);
-}
-
-static inline unsigned int round_half_down_mul(unsigned int a, unsigned int b)
-{
-	return ROUND_HALF_DOWN_MUL(a, b);
-}
-#endif
-
-/* @brief Next Power of Two
- *
- *  @param[in] unsigned number
- *
- *  @return next power of two
- *
- * This function rounds input to the nearest power of 2 (2^x)
- * towards infinity
- *
- * Input Range: 0 .. 2^(8*sizeof(int)-1)
- *
- * IF input is a power of 2
- *     out = in
- * OTHERWISE
- *     out = 2^(ceil(log2(in))
- *
- */
-
-static inline unsigned int ceil_pow2(unsigned int a)
-{
-	if (a == 0) {
-		return 1;
-	}
-	/* IF input is already a power of two*/
-	else if ((!((a)&((a)-1)))) {
-		return a;
-	}
-	else {
-		unsigned int v = a;
-		v |= v>>1;
-		v |= v>>2;
-		v |= v>>4;
-		v |= v>>8;
-		v |= v>>16;
-		return (v+1);
-	}
-}
-
-#endif /* !defined(PIPE_GENERATION) */
-
-#if !defined(__ISP)
-/*
- * For SP and ISP, SDK provides the definition of OP_std_modadd.
- * We need it only for host
- */
-#define OP_std_modadd(base, offset, size) ((base+offset)%(size))
-#endif /* !defined(__ISP) */
-
-#if !defined(__KERNEL__)
-#define clamp(a, min_val, max_val) MIN(MAX((a), (min_val)), (max_val))
-#endif /* !defined(__KERNEL__) */
-
-#endif /* __MATH_SUPPORT_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/memory_access/memory_access.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/memory_access/memory_access.h
deleted file mode 100644
index 195c4a5bceeb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/memory_access/memory_access.h
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015-2017, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __MEMORY_ACCESS_H_INCLUDED__
-#define __MEMORY_ACCESS_H_INCLUDED__
-
-/*!
- * \brief
- * Define the public interface for virtual memory
- * access functions. Access types are limited to
- * those defined in <stdint.h>
- *
- * The address representation is private to the system
- * and represented as "hrt_vaddress" rather than a
- * pointer, as the memory allocation cannot be accessed
- * by dereferencing but reaquires load and store access
- * functions
- *
- * The page table selection or virtual memory context;
- * The page table base index; Is implicit. This page
- * table base index must be set by the implementation
- * of the access function
- *
- * "store" is a transfer to the system
- * "load" is a transfer from the system
- *
- * Allocation properties can be specified by setting
- * attributes (see below) in case of multiple physical
- * memories the memory ID is encoded on the attribute
- *
- * Allocations in the same physical memory, but in a
- * different (set of) page tables can be shared through
- * a page table information mapping function
- */
-
-#include <type_support.h>
-#include "platform_support.h"	/* for __func__ */
-
-/*
- * User provided file that defines the (sub)system address types:
- *	- hrt_vaddress	a type that can hold the (sub)system virtual address range
- */
-#include "system_types.h"
-
-/*
- * The MMU base address is a physical address, thus the same type is used
- * as for the device base address
- */
-#include "device_access.h"
-
-#include "hmm/hmm.h"
-
-/*!
- * \brief
- * Bit masks for specialised allocation functions
- * the default is "uncached", "not contiguous",
- * "not page aligned" and "not cleared"
- *
- * Forcing alignment (usually) returns a pointer
- * at an alignment boundary that is offset from
- * the allocated pointer. Without storing this
- * pointer/offset, we cannot free it. The memory
- * manager is responsible for the bookkeeping, e.g.
- * the allocation function creates a sentinel
- * within the allocation referencable from the
- * returned pointer/address.
- */
-#define MMGR_ATTRIBUTE_MASK		0x000f
-#define MMGR_ATTRIBUTE_CACHED		0x0001
-#define MMGR_ATTRIBUTE_CONTIGUOUS	0x0002
-#define MMGR_ATTRIBUTE_PAGEALIGN	0x0004
-#define MMGR_ATTRIBUTE_CLEARED		0x0008
-#define MMGR_ATTRIBUTE_UNUSED		0xfff0
-
-/* #define MMGR_ATTRIBUTE_DEFAULT	(MMGR_ATTRIBUTE_CACHED) */
-#define MMGR_ATTRIBUTE_DEFAULT	0
-
-extern const hrt_vaddress	mmgr_NULL;
-extern const hrt_vaddress	mmgr_EXCEPTION;
-
-/*! Return the address of an allocation in memory
-
- \param	size[in]		Size in bytes of the allocation
- \param	caller_func[in]		Caller function name
- \param	caller_line[in]		Caller function line number
-
- \return vaddress
- */
-extern hrt_vaddress mmgr_malloc(const size_t size);
-
-/*! Return the address of a zero initialised allocation in memory
-
- \param	N[in]			Horizontal dimension of array
- \param	size[in]		Vertical dimension of array  Total size is N*size
-
- \return vaddress
- */
-extern hrt_vaddress mmgr_calloc(const size_t N, const size_t size);
-
-/*! Return the address of an allocation in memory
-
- \param	size[in]		Size in bytes of the allocation
- \param	attribute[in]		Bit vector specifying the properties
-				of the allocation including zero initialisation
-
- \return vaddress
- */
-
-extern hrt_vaddress mmgr_alloc_attr(const size_t size, const uint16_t attribute);
-
-/*! Return the address of a mapped existing allocation in memory
-
- \param	ptr[in]			Pointer to an allocation in a different
-				virtual memory page table, but the same
-				physical memory
- \param size[in]		Size of the memory of the pointer
- \param	attribute[in]		Bit vector specifying the properties
-				of the allocation
- \param context			Pointer of a context provided by
-				client/driver for additonal parameters
-				needed by the implementation
- \Note
-	This interface is tentative, limited to the desired function
-	the actual interface may require furhter parameters
-
- \return vaddress
- */
-extern hrt_vaddress mmgr_mmap(
-	const void *ptr,
-	const size_t size,
-	uint16_t attribute,
-	void *context);
-
-/*! Zero initialise an allocation in memory
-
- \param	vaddr[in]		Address of an allocation
- \param	size[in]		Size in bytes of the area to be cleared
-
- \return none
- */
-extern void mmgr_clear(hrt_vaddress vaddr, const size_t	size);
-
-/*! Read an array of bytes from a virtual memory address
-
- \param	vaddr[in]		Address of an allocation
- \param	data[out]		pointer to the destination array
- \param	size[in]		number of bytes to read
-
- \return none
- */
-extern void mmgr_load(const hrt_vaddress vaddr, void *data, const size_t size);
-
-/*! Write an array of bytes to device registers or memory in the device
-
- \param	vaddr[in]		Address of an allocation
- \param	data[in]		pointer to the source array
- \param	size[in]		number of bytes to write
-
- \return none
- */
-extern void mmgr_store(const hrt_vaddress vaddr, const void *data, const size_t size);
-
-#endif /* __MEMORY_ACCESS_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/memory_realloc.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/memory_realloc.h
deleted file mode 100644
index f3b7273fed1b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/memory_realloc.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#ifndef __MEMORY_REALLOC_H_INCLUDED__
-#define __MEMORY_REALLOC_H_INCLUDED__
-
-/*!
- * \brief
- * Define the internal reallocation of private css memory
- *
- */
-
-#include <type_support.h>
-/*
- * User provided file that defines the (sub)system address types:
- *	- hrt_vaddress	a type that can hold the (sub)system virtual address range
- */
-#include "system_types.h"
-#include "ia_css_err.h"
-
-bool reallocate_buffer(
-	hrt_vaddress *curr_buf,
-	size_t *curr_size,
-	size_t needed_size,
-	bool force,
-	enum ia_css_err *err);
-
-#endif /*__MEMORY_REALLOC_H_INCLUDED__*/
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/misc_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/misc_support.h
deleted file mode 100644
index 38db1ecef3c8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/misc_support.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __MISC_SUPPORT_H_INCLUDED__
-#define __MISC_SUPPORT_H_INCLUDED__
-
-/* suppress compiler warnings on unused variables */
-#ifndef NOT_USED
-#define NOT_USED(a) ((void)(a))
-#endif
-
-/* Calculate the  total bytes for pow(2) byte alignment */
-#define tot_bytes_for_pow2_align(pow2, cur_bytes)	((cur_bytes + (pow2 - 1)) & ~(pow2 - 1))
-
-#endif /* __MISC_SUPPORT_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/mmu_device.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/mmu_device.h
deleted file mode 100644
index 519e850ec390..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/mmu_device.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __MMU_DEVICE_H_INCLUDED__
-#define __MMU_DEVICE_H_INCLUDED__
-
-/* The file mmu.h already exists */
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the MMU device. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "mmu_local.h"
-
-#ifndef __INLINE_MMU__
-#define STORAGE_CLASS_MMU_H extern
-#define STORAGE_CLASS_MMU_C 
-#include "mmu_public.h"
-#else  /* __INLINE_MMU__ */
-#define STORAGE_CLASS_MMU_H static inline
-#define STORAGE_CLASS_MMU_C static inline
-#include "mmu_private.h"
-#endif /* __INLINE_MMU__ */
-
-#endif /* __MMU_DEVICE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/pixelgen.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/pixelgen.h
deleted file mode 100644
index 418d02382d76..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/pixelgen.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __PIXELGEN_H_INCLUDED__
-#define __PIXELGEN_H_INCLUDED__
-
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- * - system and cell agnostic interfaces, constants and identifiers
- * - public:  system agnostic, cell specific interfaces
- * - private: system dependent, cell specific interfaces &
- *   inline implementations
- * - global:  system specific constants and identifiers
- * - local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "pixelgen_local.h"
-
-#ifndef __INLINE_PIXELGEN__
-#define STORAGE_CLASS_PIXELGEN_H extern
-#define STORAGE_CLASS_PIXELGEN_C
-#include "pixelgen_public.h"
-#else  /* __INLINE_PIXELGEN__ */
-#define STORAGE_CLASS_PIXELGEN_H static inline
-#define STORAGE_CLASS_PIXELGEN_C static inline
-#include "pixelgen_private.h"
-#endif /* __INLINE_PIXELGEN__ */
-
-#endif /* __PIXELGEN_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/platform_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/platform_support.h
deleted file mode 100644
index 39a125ba563d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/platform_support.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __PLATFORM_SUPPORT_H_INCLUDED__
-#define __PLATFORM_SUPPORT_H_INCLUDED__
-
-/**
-* @file
-* Platform specific includes and functionality.
-*/
-
-#include <linux/delay.h>
-#include <linux/kernel.h>
-#include <linux/string.h>
-
-/* For definition of hrt_sleep() */
-#include "hive_isp_css_custom_host_hrt.h"
-
-#define UINT16_MAX USHRT_MAX
-#define UINT32_MAX UINT_MAX
-#define UCHAR_MAX  (255)
-
-#define CSS_ALIGN(d, a) d __attribute__((aligned(a)))
-
-/*
- * Put here everything __KERNEL__ specific not covered in
- * "assert_support.h", "math_support.h", etc
- */
-
-#endif /* __PLATFORM_SUPPORT_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/print_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/print_support.h
deleted file mode 100644
index ca0fbbb57788..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/print_support.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __PRINT_SUPPORT_H_INCLUDED__
-#define __PRINT_SUPPORT_H_INCLUDED__
-
-
-#include <stdarg.h>
-#if !defined(__KERNEL__)
-#include <stdio.h>
-#endif
-
-extern int (*sh_css_printf) (const char *fmt, va_list args);
-/* depends on host supplied print function in ia_css_init() */
-static inline void ia_css_print(const char *fmt, ...)
-{
-	va_list ap;
-	if (sh_css_printf) {
-		va_start(ap, fmt);
-		sh_css_printf(fmt, ap);
-		va_end(ap);
-	}
-}
-
-/* Start adding support for bxt tracing functions for poc. From
- * bxt_sandbox/support/print_support.h. */
-/* TODO: support these macros in userspace. */
-#define PWARN(format, ...) ia_css_print("warning: ", ##__VA_ARGS__)
-#define PRINT(format, ...) ia_css_print(format, ##__VA_ARGS__)
-#define PERROR(format, ...) ia_css_print("error: " format, ##__VA_ARGS__)
-#define PDEBUG(format, ...) ia_css_print("debug: " format, ##__VA_ARGS__)
-
-#endif /* __PRINT_SUPPORT_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/queue.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/queue.h
deleted file mode 100644
index aa5fadf5aadb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/queue.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __QUEUE_H_INCLUDED__
-#define __QUEUE_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and is system agnostic
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - system and cell agnostic interfaces, constants and identifiers
- *	- public:  cell specific interfaces
- *	- private: cell specific inline implementations
- *	- global:  inter cell constants and identifiers
- *	- local:   cell specific constants and identifiers
- *
- */
-
-
-#include "queue_local.h"
-
-#ifndef __INLINE_QUEUE__
-#define STORAGE_CLASS_QUEUE_H extern
-#define STORAGE_CLASS_QUEUE_C 
-/* #include "queue_public.h" */
-#include "ia_css_queue.h"
-#else  /* __INLINE_QUEUE__ */
-#define STORAGE_CLASS_QUEUE_H static inline
-#define STORAGE_CLASS_QUEUE_C static inline
-#include "queue_private.h"
-#endif /* __INLINE_QUEUE__ */
-
-#endif /* __QUEUE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/resource.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/resource.h
deleted file mode 100644
index bd9f53e6b680..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/resource.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __RESOURCE_H_INCLUDED__
-#define __RESOURCE_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses a RESOURCE manager. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- *
- */
-
-
-#include "system_local.h"
-#include "resource_local.h"
-
-#ifndef __INLINE_RESOURCE__
-#define STORAGE_CLASS_RESOURCE_H extern
-#define STORAGE_CLASS_RESOURCE_C 
-#include "resource_public.h"
-#else  /* __INLINE_RESOURCE__ */
-#define STORAGE_CLASS_RESOURCE_H static inline
-#define STORAGE_CLASS_RESOURCE_C static inline
-#include "resource_private.h"
-#endif /* __INLINE_RESOURCE__ */
-
-#endif /* __RESOURCE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/socket.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/socket.h
deleted file mode 100644
index 43cfb0cb4aa8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/socket.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SOCKET_H_INCLUDED__
-#define __SOCKET_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the DMA device. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- *
- */
-
-
-#include "system_local.h"
-#include "socket_local.h"
-
-#ifndef __INLINE_SOCKET__
-#define STORAGE_CLASS_SOCKET_H extern
-#define STORAGE_CLASS_SOCKET_C
-#include "socket_public.h"
-#else  /* __INLINE_SOCKET__ */
-#define STORAGE_CLASS_SOCKET_H static inline
-#define STORAGE_CLASS_SOCKET_C static inline
-#include "socket_private.h"
-#endif /* __INLINE_SOCKET__ */
-
-#endif /* __SOCKET_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/sp.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/sp.h
deleted file mode 100644
index 8f57f2060791..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/sp.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SP_H_INCLUDED__
-#define __SP_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the SP cell. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "sp_local.h"
-
-#ifndef __INLINE_SP__
-#define STORAGE_CLASS_SP_H extern
-#define STORAGE_CLASS_SP_C 
-#include "sp_public.h"
-#else  /* __INLINE_SP__ */
-#define STORAGE_CLASS_SP_H static inline
-#define STORAGE_CLASS_SP_C static inline
-#include "sp_private.h"
-#endif /* __INLINE_SP__ */
-
-#endif /* __SP_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/string_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/string_support.h
deleted file mode 100644
index f4d9674cdab6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/string_support.h
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __STRING_SUPPORT_H_INCLUDED__
-#define __STRING_SUPPORT_H_INCLUDED__
-#include <platform_support.h>
-#include <type_support.h>
-
-#if !defined(_MSC_VER)
-/*
- * For all non microsoft cases, we need the following functions
- */
-
-
-/* @brief Copy from src_buf to dest_buf.
- *
- * @param[out] dest_buf. Destination buffer to copy to
- * @param[in]  dest_size. The size of the destination buffer in bytes
- * @param[in]  src_buf. The source buffer
- * @param[in]  src_size. The size of the source buffer in bytes
- * @return     0 on success, error code on failure
- * @return     EINVAL on Invalid arguments
- * @return     ERANGE on Destination size too small
- */
-static inline int memcpy_s(
-	void* dest_buf,
-	size_t dest_size,
-	const void* src_buf,
-	size_t src_size)
-{
-	if ((src_buf == NULL) || (dest_buf == NULL)) {
-		/* Invalid arguments*/
-		return EINVAL;
-	}
-
-	if ((dest_size < src_size) || (src_size == 0)) {
-		/* Destination too small*/
-		return ERANGE;
-	}
-
-	memcpy(dest_buf, src_buf, src_size);
-	return 0;
-}
-
-/* @brief Get the length of the string, excluding the null terminator
- *
- * @param[in]  src_str. The source string
- * @param[in]  max_len. Look only for max_len bytes in the string
- * @return     Return the string length excluding null character
- * @return     Return max_len if no null character in the first max_len bytes
- * @return     Returns 0 if src_str is NULL
- */
-static size_t strnlen_s(
-	const char* src_str,
-	size_t max_len)
-{
-	size_t ix;
-	if (src_str == NULL) {
-		/* Invalid arguments*/
-		return 0;
-	}
-
-	for (ix = 0; ix < max_len && src_str[ix] != '\0'; ix++)
-		;
-
-	/* On Error, it will return src_size == max_len*/
-	return ix;
-}
-
-/* @brief Copy string from src_str to dest_str
- *
- * @param[out] dest_str. Destination buffer to copy to
- * @param[in]  dest_size. The size of the destination buffer in bytes
- * @param[in]  src_str. The source buffer
- * @param[in]  src_size. The size of the source buffer in bytes
- * @return     Returns 0 on success
- * @return     Returns EINVAL on invalid arguments
- * @return     Returns ERANGE on destination size too small
- */
-static inline int strncpy_s(
-	char* dest_str,
-	size_t dest_size,
-	const char* src_str,
-	size_t src_size)
-{
-	size_t len;
-	if (dest_str == NULL) {
-		/* Invalid arguments*/
-		return EINVAL;
-	}
-
-	if ((src_str == NULL) || (dest_size == 0)) {
-		/* Invalid arguments*/
-		dest_str[0] = '\0';
-		return EINVAL;
-	}
-
-	len = strnlen_s(src_str, src_size);
-
-	if (len >= dest_size) {
-		/* Destination too small*/
-		dest_str[0] = '\0';
-		return ERANGE;
-	}
-
-	/* dest_str is big enough for the len */
-	strncpy(dest_str, src_str, len);
-	dest_str[len] = '\0';
-	return 0;
-}
-
-/* @brief Copy string from src_str to dest_str
- *
- * @param[out] dest_str. Destination buffer to copy to
- * @param[in]  dest_size. The size of the destination buffer in bytes
- * @param[in]  src_str. The source buffer
- * @return     Returns 0 on success
- * @return     Returns EINVAL on invalid arguments
- * @return     Returns ERANGE on destination size too small
- */
-static inline int strcpy_s(
-	char* dest_str,
-	size_t dest_size,
-	const char* src_str)
-{
-	size_t len;
-	if (dest_str == NULL) {
-		/* Invalid arguments*/
-		return EINVAL;
-	}
-
-	if ((src_str == NULL) || (dest_size == 0)) {
-		/* Invalid arguments*/
-		dest_str[0] = '\0';
-		return EINVAL;
-	}
-
-	len = strnlen_s(src_str, dest_size);
-
-	if (len >= dest_size) {
-		/* Destination too small*/
-		dest_str[0] = '\0';
-		return ERANGE;
-	}
-
-	/* dest_str is big enough for the len */
-	strncpy(dest_str, src_str, len);
-	dest_str[len] = '\0';
-	return 0;
-}
-
-#endif /*!defined(_MSC_VER)*/
-
-#endif /* __STRING_SUPPORT_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/system_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/system_types.h
deleted file mode 100644
index a8c19cee17da..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/system_types.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#ifndef __SYSTEM_TYPES_H_INCLUDED__
-#define __SYSTEM_TYPES_H_INCLUDED__
-
-/**
-* @file
-* Platform specific types.
-*/
-
-
-#include "system_local.h"
-
-#endif /* __SYSTEM_TYPES_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/tag.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/tag.h
deleted file mode 100644
index ace695643369..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/tag.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __TAG_H_INCLUDED__
-#define __TAG_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and is system agnostic
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  cell specific interfaces
- *	- private: cell specific inline implementations
- *	- global:  inter cell constants and identifiers
- *	- local:   cell specific constants and identifiers
- *
- */
-
-
-#include "tag_local.h"
-
-#ifndef __INLINE_TAG__
-#define STORAGE_CLASS_TAG_H extern
-#define STORAGE_CLASS_TAG_C 
-#include "tag_public.h"
-#else  /* __INLINE_TAG__ */
-#define STORAGE_CLASS_TAG_H static inline
-#define STORAGE_CLASS_TAG_C static inline
-#include "tag_private.h"
-#endif /* __INLINE_TAG__ */
-
-#endif /* __TAG_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/timed_ctrl.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/timed_ctrl.h
deleted file mode 100644
index f6bc1c47553f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/timed_ctrl.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __TIMED_CTRL_H_INCLUDED__
-#define __TIMED_CTRL_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "timed_ctrl_local.h"
-
-#ifndef __INLINE_TIMED_CTRL__
-#define STORAGE_CLASS_TIMED_CTRL_H extern
-#define STORAGE_CLASS_TIMED_CTRL_C 
-#include "timed_ctrl_public.h"
-#else  /* __INLINE_TIMED_CTRL__ */
-#define STORAGE_CLASS_TIMED_CTRL_H static inline
-#define STORAGE_CLASS_TIMED_CTRL_C static inline
-#include "timed_ctrl_private.h"
-#endif /* __INLINE_TIMED_CTRL__ */
-
-#endif /* __TIMED_CTRL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/type_support.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/type_support.h
deleted file mode 100644
index bc77537fa73a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/type_support.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __TYPE_SUPPORT_H_INCLUDED__
-#define __TYPE_SUPPORT_H_INCLUDED__
-
-/**
-* @file
-* Platform specific types.
-*
-* Per the DLI spec, types are in "type_support.h" and
-* "platform_support.h" is for unclassified/to be refactored
-* platform specific definitions.
-*/
-
-#define IA_CSS_UINT8_T_BITS						8
-#define IA_CSS_UINT16_T_BITS					16
-#define IA_CSS_UINT32_T_BITS					32
-#define IA_CSS_INT32_T_BITS						32
-#define IA_CSS_UINT64_T_BITS					64
-
-#define CHAR_BIT (8)
-
-#include <linux/types.h>
-#include <linux/limits.h>
-#include <linux/errno.h>
-#define HOST_ADDRESS(x) (unsigned long)(x)
-
-#endif /* __TYPE_SUPPORT_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/vamem.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/vamem.h
deleted file mode 100644
index 82d447bf9704..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/vamem.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __VAMEM_H_INCLUDED__
-#define __VAMEM_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the VAMEM device. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "vamem_local.h"
-
-#ifndef __INLINE_VAMEM__
-#define STORAGE_CLASS_VAMEM_H extern
-#define STORAGE_CLASS_VAMEM_C 
-#include "vamem_public.h"
-#else  /* __INLINE_VAMEM__ */
-#define STORAGE_CLASS_VAMEM_H static inline
-#define STORAGE_CLASS_VAMEM_C static inline
-#include "vamem_private.h"
-#endif /* __INLINE_VAMEM__ */
-
-#endif /* __VAMEM_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/vmem.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/vmem.h
deleted file mode 100644
index d3375729c441..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_include/vmem.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __VMEM_H_INCLUDED__
-#define __VMEM_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the VMEM device. It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- *  - .        system and cell agnostic interfaces, constants and identifiers
- *	- public:  system agnostic, cell specific interfaces
- *	- private: system dependent, cell specific interfaces & inline implementations
- *	- global:  system specific constants and identifiers
- *	- local:   system and cell specific constants and identifiers
- */
-
-
-#include "system_local.h"
-#include "vmem_local.h"
-
-#ifndef __INLINE_VMEM__
-#define STORAGE_CLASS_VMEM_H extern
-#define STORAGE_CLASS_VMEM_C 
-#include "vmem_public.h"
-#else  /* __INLINE_VMEM__ */
-#define STORAGE_CLASS_VMEM_H static inline
-#define STORAGE_CLASS_VMEM_C static inline
-#include "vmem_private.h"
-#endif /* __INLINE_VMEM__ */
-
-#endif /* __VMEM_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/queue_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/queue_local.h
deleted file mode 100644
index 9f4060319b4b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/queue_local.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __QUEUE_LOCAL_H_INCLUDED__
-#define __QUEUE_LOCAL_H_INCLUDED__
-
-#include "queue_global.h"
-
-#endif /* __QUEUE_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/queue_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/queue_private.h
deleted file mode 100644
index 2b396955cdad..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/queue_private.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __QUEUE_PRIVATE_H_INCLUDED__
-#define __QUEUE_PRIVATE_H_INCLUDED__
-
-#endif /* __QUEUE_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag.c
deleted file mode 100644
index 2cf1d58941bf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag.c
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "tag.h"
-#include <platform_support.h>	/* NULL */
-#include <assert_support.h>
-#include "tag_local.h"
-
-/*
- * @brief	Creates the tag description from the given parameters.
- * @param[in]	num_captures
- * @param[in]	skip
- * @param[in]	offset
- * @param[out]	tag_descr
- */
-void
-sh_css_create_tag_descr(int num_captures,
-			unsigned int skip,
-			int offset,
-			unsigned int exp_id,
-			struct sh_css_tag_descr *tag_descr)
-{
-	assert(tag_descr != NULL);
-
-	tag_descr->num_captures = num_captures;
-	tag_descr->skip		= skip;
-	tag_descr->offset	= offset;
-	tag_descr->exp_id	= exp_id;
-}
-
-/*
- * @brief	Encodes the members of tag description into a 32-bit value.
- * @param[in]	tag		Pointer to the tag description
- * @return	(unsigned int)	Encoded 32-bit tag-info
- */
-unsigned int
-sh_css_encode_tag_descr(struct sh_css_tag_descr *tag)
-{
-	int num_captures;
-	unsigned int num_captures_sign;
-	unsigned int skip;
-	int offset;
-	unsigned int offset_sign;
-	unsigned int exp_id;
-	unsigned int encoded_tag;
-
-	assert(tag != NULL);
-
-	if (tag->num_captures < 0) {
-		num_captures = -tag->num_captures;
-		num_captures_sign = 1;
-	} else {
-		num_captures = tag->num_captures;
-		num_captures_sign = 0;
-	}
-	skip = tag->skip;
-	if (tag->offset < 0) {
-		offset = -tag->offset;
-		offset_sign = 1;
-	} else {
-		offset = tag->offset;
-		offset_sign = 0;
-	}
-	exp_id = tag->exp_id;
-
-	if (exp_id != 0)
-	{
-		/* we encode either an exp_id or capture data */
-		assert((num_captures == 0) && (skip == 0) && (offset == 0));
-
-		encoded_tag = TAG_EXP | (exp_id & 0xFF) << TAG_EXP_ID_SHIFT;
-	}
-	else
-	{
-		encoded_tag = TAG_CAP 
-				| ((num_captures_sign & 0x00000001) << TAG_NUM_CAPTURES_SIGN_SHIFT)
-				| ((offset_sign       & 0x00000001) << TAG_OFFSET_SIGN_SHIFT)
-				| ((num_captures      & 0x000000FF) << TAG_NUM_CAPTURES_SHIFT)
-				| ((skip              & 0x000000FF) << TAG_OFFSET_SHIFT)
-				| ((offset            & 0x000000FF) << TAG_SKIP_SHIFT);
-
-	}
-	return encoded_tag;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag_local.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag_local.h
deleted file mode 100644
index 01a8977c189e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag_local.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __TAG_LOCAL_H_INCLUDED__
-#define __TAG_LOCAL_H_INCLUDED__
-
-#include "tag_global.h"
-
-#define SH_CSS_MINIMUM_TAG_ID (-1)
-
-#endif /* __TAG_LOCAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag_private.h
deleted file mode 100644
index 0570a95ec5bf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/host/tag_private.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __TAG_PRIVATE_H_INCLUDED__
-#define __TAG_PRIVATE_H_INCLUDED__
-
-#endif /* __TAG_PRIVATE_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/queue_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/queue_global.h
deleted file mode 100644
index 61330daab734..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/queue_global.h
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __QUEUE_GLOBAL_H_INCLUDED__
-#define __QUEUE_GLOBAL_H_INCLUDED__
-
-#endif /* __QUEUE_GLOBAL_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/sw_event_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/sw_event_global.h
deleted file mode 100644
index c0d2efadbbe3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/sw_event_global.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SW_EVENT_GLOBAL_H_INCLUDED__
-#define __SW_EVENT_GLOBAL_H_INCLUDED__
-
-#define MAX_NR_OF_PAYLOADS_PER_SW_EVENT 4
-
-enum ia_css_psys_sw_event {
-	IA_CSS_PSYS_SW_EVENT_BUFFER_ENQUEUED, /* from host to SP */
-	IA_CSS_PSYS_SW_EVENT_BUFFER_DEQUEUED, /* from SP to host */
-	IA_CSS_PSYS_SW_EVENT_EVENT_DEQUEUED, /* from SP to host, one way only */
-	IA_CSS_PSYS_SW_EVENT_START_STREAM,
-	IA_CSS_PSYS_SW_EVENT_STOP_STREAM,
-	IA_CSS_PSYS_SW_EVENT_MIPI_BUFFERS_READY,
-	IA_CSS_PSYS_SW_EVENT_UNLOCK_RAW_BUFFER,
-	IA_CSS_PSYS_SW_EVENT_STAGE_ENABLE_DISABLE /* for extension state change enable/disable */
-};
-
-enum ia_css_isys_sw_event {
-	IA_CSS_ISYS_SW_EVENT_EVENT_DEQUEUED
-};
-
-#endif /* __SW_EVENT_GLOBAL_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/tag_global.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/tag_global.h
deleted file mode 100644
index fda457792c9c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/hive_isp_css_shared/tag_global.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __TAG_GLOBAL_H_INCLUDED__
-#define __TAG_GLOBAL_H_INCLUDED__
-
-/* offsets for encoding/decoding the tag into an uint32_t */
-
-#define TAG_CAP	1
-#define TAG_EXP	2
-
-#define TAG_NUM_CAPTURES_SIGN_SHIFT	 6
-#define TAG_OFFSET_SIGN_SHIFT 		 7
-#define TAG_NUM_CAPTURES_SHIFT 		 8
-#define TAG_OFFSET_SHIFT 		16
-#define TAG_SKIP_SHIFT 			24
-
-#define TAG_EXP_ID_SHIFT 		 8
-
-/* Data structure containing the tagging information which is used in
- * continuous mode to specify which frames should be captured.
- * num_captures		The number of RAW frames to be processed to
- *                      YUV. Setting this to -1 will make continuous
- *                      capture run until it is stopped.
- * skip			Skip N frames in between captures. This can be
- *                      used to select a slower capture frame rate than
- *                      the sensor output frame rate.
- * offset		Start the RAW-to-YUV processing at RAW buffer
- *                      with this offset. This allows the user to
- *                      process RAW frames that were captured in the
- *                      past or future.
- * exp_id		Exposure id of the RAW frame to tag.
- *
- * NOTE: Either exp_id = 0 or all other fields are 0
- *	 (so yeah, this could be a union)
- */
-
-struct sh_css_tag_descr {
-	int num_captures;
-	unsigned int skip;
-	int offset;
-	unsigned int exp_id;
-};
-
-#endif /* __TAG_GLOBAL_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css.h
deleted file mode 100644
index e44df6916d90..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/* Release Version: irci_stable_candrpv_0415_20150521_0458 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_H_
-#define _IA_CSS_H_
-
-/* @file
- * This file is the starting point of the CSS-API. It includes all CSS-API
- * header files.
- */
-
-#include "ia_css_3a.h"
-#include "ia_css_acc_types.h"
-#include "ia_css_buffer.h"
-#include "ia_css_control.h"
-#include "ia_css_device_access.h"
-#include "ia_css_dvs.h"
-#include "ia_css_env.h"
-#include "ia_css_err.h"
-#include "ia_css_event_public.h"
-#include "ia_css_firmware.h"
-#include "ia_css_frame_public.h"
-#include "ia_css_input_port.h"
-#include "ia_css_irq.h"
-#include "ia_css_metadata.h"
-#include "ia_css_mipi.h"
-#include "ia_css_pipe_public.h"
-#include "ia_css_prbs.h"
-#include "ia_css_properties.h"
-#include "ia_css_stream_format.h"
-#include "ia_css_stream_public.h"
-#include "ia_css_tpg.h"
-#include "ia_css_version.h"
-#include "ia_css_mmu.h"
-#include "ia_css_morph.h"
-#include "ia_css_shading.h"
-#include "ia_css_timer.h"
-
-/*
-   Please do not add code to this file. Public functionality is to be
-   exposed in a function/data type specific header file.
-   Please add to the appropriate header file or create a new one.
- */
-
-#endif /* _IA_CSS_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_3a.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_3a.h
deleted file mode 100644
index 080198796ad0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_3a.h
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_3A_H
-#define __IA_CSS_3A_H
-
-/* @file
- * This file contains types used for 3A statistics
- */
-
-#include <type_support.h>
-#include "ia_css_types.h"
-#include "ia_css_err.h"
-#include "system_global.h"
-
-enum ia_css_3a_tables {
-	IA_CSS_S3A_TBL_HI,
-	IA_CSS_S3A_TBL_LO,
-	IA_CSS_RGBY_TBL,
-	IA_CSS_NUM_3A_TABLES
-};
-
-/* Structure that holds 3A statistics in the ISP internal
- * format. Use ia_css_get_3a_statistics() to translate
- * this to the format used on the host (3A library).
- * */
-struct ia_css_isp_3a_statistics {
-	union {
-		struct {
-			ia_css_ptr s3a_tbl;
-		} dmem;
-		struct {
-			ia_css_ptr s3a_tbl_hi;
-			ia_css_ptr s3a_tbl_lo;
-		} vmem;
-	} data;
-	struct {
-		ia_css_ptr rgby_tbl;
-	} data_hmem;
-	uint32_t exp_id;     /** exposure id, to match statistics to a frame,
-			          see ia_css_event_public.h for more detail. */
-	uint32_t isp_config_id;/** Unique ID to track which config was actually applied to a particular frame */
-	ia_css_ptr data_ptr; /** pointer to base of all data */
-	uint32_t   size;     /** total size of all data */
-	uint32_t   dmem_size;
-	uint32_t   vmem_size; /** both lo and hi have this size */
-	uint32_t   hmem_size;
-};
-#define SIZE_OF_DMEM_STRUCT						\
-	(SIZE_OF_IA_CSS_PTR)
-
-#define SIZE_OF_VMEM_STRUCT						\
-	(2 * SIZE_OF_IA_CSS_PTR)
-
-#define SIZE_OF_DATA_UNION						\
-	(MAX(SIZE_OF_DMEM_STRUCT, SIZE_OF_VMEM_STRUCT))
-
-#define SIZE_OF_DATA_HMEM_STRUCT					\
-	(SIZE_OF_IA_CSS_PTR)
-
-#define SIZE_OF_IA_CSS_ISP_3A_STATISTICS_STRUCT				\
-	(SIZE_OF_DATA_UNION +						\
-	 SIZE_OF_DATA_HMEM_STRUCT +					\
-	 sizeof(uint32_t) +						\
-	 sizeof(uint32_t) +						\
-	 SIZE_OF_IA_CSS_PTR +						\
-	 4 * sizeof(uint32_t))
-
-/* Map with host-side pointers to ISP-format statistics.
- * These pointers can either be copies of ISP data or memory mapped
- * ISP pointers.
- * All of the data behind these pointers is allocated contiguously, the
- * allocated pointer is stored in the data_ptr field. The other fields
- * point into this one block of data.
- */
-struct ia_css_isp_3a_statistics_map {
-	void                    *data_ptr; /** Pointer to start of memory */
-	struct ia_css_3a_output *dmem_stats;
-	uint16_t                *vmem_stats_hi;
-	uint16_t                *vmem_stats_lo;
-	struct ia_css_bh_table  *hmem_stats;
-	uint32_t                 size; /** total size in bytes of data_ptr */
-	uint32_t                 data_allocated; /** indicate whether data_ptr
-						    was allocated or not. */
-};
-
-/* @brief Copy and translate 3A statistics from an ISP buffer to a host buffer
- * @param[out]	host_stats Host buffer.
- * @param[in]	isp_stats ISP buffer.
- * @return	error value if temporary memory cannot be allocated
- *
- * This copies 3a statistics from an ISP pointer to a host pointer and then
- * translates some of the statistics, details depend on which ISP binary is
- * used.
- * Always use this function, never copy the buffer directly.
- */
-enum ia_css_err
-ia_css_get_3a_statistics(struct ia_css_3a_statistics           *host_stats,
-			 const struct ia_css_isp_3a_statistics *isp_stats);
-
-/* @brief Translate 3A statistics from ISP format to host format.
- * @param[out]	host_stats host-format statistics
- * @param[in]	isp_stats  ISP-format statistics
- * @return	None
- *
- * This function translates statistics from the internal ISP-format to
- * the host-format. This function does not include an additional copy
- * step.
- * */
-void
-ia_css_translate_3a_statistics(
-		struct ia_css_3a_statistics               *host_stats,
-		const struct ia_css_isp_3a_statistics_map *isp_stats);
-
-/* Convenience functions for alloc/free of certain datatypes */
-
-/* @brief Allocate memory for the 3a statistics on the ISP
- * @param[in]	grid The grid.
- * @return		Pointer to the allocated 3a statistics buffer on the ISP
-*/
-struct ia_css_isp_3a_statistics *
-ia_css_isp_3a_statistics_allocate(const struct ia_css_3a_grid_info *grid);
-
-/* @brief Free the 3a statistics memory on the isp
- * @param[in]	me Pointer to the 3a statistics buffer on the ISP.
- * @return		None
-*/
-void
-ia_css_isp_3a_statistics_free(struct ia_css_isp_3a_statistics *me);
-
-/* @brief Allocate memory for the 3a statistics on the host
- * @param[in]	grid The grid.
- * @return		Pointer to the allocated 3a statistics buffer on the host
-*/
-struct ia_css_3a_statistics *
-ia_css_3a_statistics_allocate(const struct ia_css_3a_grid_info *grid);
-
-/* @brief Free the 3a statistics memory on the host
- * @param[in]	me Pointer to the 3a statistics buffer on the host.
- * @return		None
- */
-void
-ia_css_3a_statistics_free(struct ia_css_3a_statistics *me);
-
-/* @brief Allocate a 3a statistics map structure
- * @param[in]	isp_stats pointer to ISP 3a statistis struct
- * @param[in]	data_ptr  host-side pointer to ISP 3a statistics.
- * @return		Pointer to the allocated 3a statistics map
- *
- * This function allocates the ISP 3a statistics map structure
- * and uses the data_ptr as base pointer to set the appropriate
- * pointers to all relevant subsets of the 3a statistics (dmem,
- * vmem, hmem).
- * If the data_ptr is NULL, this function will allocate the host-side
- * memory. This information is stored in the struct and used in the
- * ia_css_isp_3a_statistics_map_free() function to determine whether
- * the memory should be freed or not.
- * Note that this function does not allocate or map any ISP
- * memory.
-*/
-struct ia_css_isp_3a_statistics_map *
-ia_css_isp_3a_statistics_map_allocate(
-	const struct ia_css_isp_3a_statistics *isp_stats,
-	void *data_ptr);
-
-/* @brief Free the 3a statistics map
- * @param[in]	me Pointer to the 3a statistics map
- * @return		None
- *
- * This function frees the map struct. If the data_ptr inside it
- * was allocated inside ia_css_isp_3a_statistics_map_allocate(), it
- * will be freed in this function. Otherwise it will not be freed.
- */
-void
-ia_css_isp_3a_statistics_map_free(struct ia_css_isp_3a_statistics_map *me);
-
-#endif /* __IA_CSS_3A_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_acc_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_acc_types.h
deleted file mode 100644
index 138bc3bb4627..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_acc_types.h
+++ /dev/null
@@ -1,468 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_ACC_TYPES_H
-#define _IA_CSS_ACC_TYPES_H
-
-/* @file
- * This file contains types used for acceleration
- */
-
-#include <system_types.h>	/* HAS_IRQ_MAP_VERSION_# */
-#include <type_support.h>
-#include <platform_support.h>
-#include <debug_global.h>
-
-#include "ia_css_types.h"
-#include "ia_css_frame_format.h"
-
-/* Should be included without the path.
-   However, that requires adding the path to numerous makefiles
-   that have nothing to do with isp parameters.
- */
-#include "runtime/isp_param/interface/ia_css_isp_param_types.h"
-
-/* Types for the acceleration API.
- * These should be moved to sh_css_internal.h once the old acceleration
- * argument handling has been completed.
- * After that, interpretation of these structures is no longer needed
- * in the kernel and HAL.
-*/
-
-/* Type of acceleration.
- */
-enum ia_css_acc_type {
-	IA_CSS_ACC_NONE,	/** Normal binary */
-	IA_CSS_ACC_OUTPUT,	/** Accelerator stage on output frame */
-	IA_CSS_ACC_VIEWFINDER,	/** Accelerator stage on viewfinder frame */
-	IA_CSS_ACC_STANDALONE,	/** Stand-alone acceleration */
-};
-
-/* Cells types
- */
-enum ia_css_cell_type {
-	IA_CSS_SP0 = 0,
-	IA_CSS_SP1,
-	IA_CSS_ISP,
-	MAX_NUM_OF_CELLS
-};
-
-/* Firmware types.
- */
-enum ia_css_fw_type {
-	ia_css_sp_firmware,		/** Firmware for the SP */
-	ia_css_isp_firmware,	/** Firmware for the ISP */
-	ia_css_bootloader_firmware, /** Firmware for the BootLoader */
-	ia_css_acc_firmware		/** Firmware for accelrations */
-};
-
-struct ia_css_blob_descr;
-
-/* Blob descriptor.
- * This structure describes an SP or ISP blob.
- * It describes the test, data and bss sections as well as position in a
- * firmware file.
- * For convenience, it contains dynamic data after loading.
- */
-struct ia_css_blob_info {
-	/** Static blob data */
-	uint32_t offset;		/** Blob offset in fw file */
-	struct ia_css_isp_param_memory_offsets memory_offsets;  /** offset wrt hdr in bytes */
-	uint32_t prog_name_offset;  /** offset wrt hdr in bytes */
-	uint32_t size;			/** Size of blob */
-	uint32_t padding_size;	/** total cummulative of bytes added due to section alignment */
-	uint32_t icache_source;	/** Position of icache in blob */
-	uint32_t icache_size;	/** Size of icache section */
-	uint32_t icache_padding;/** bytes added due to icache section alignment */
-	uint32_t text_source;	/** Position of text in blob */
-	uint32_t text_size;		/** Size of text section */
-	uint32_t text_padding;	/** bytes added due to text section alignment */
-	uint32_t data_source;	/** Position of data in blob */
-	uint32_t data_target;	/** Start of data in SP dmem */
-	uint32_t data_size;		/** Size of text section */
-	uint32_t data_padding;	/** bytes added due to data section alignment */
-	uint32_t bss_target;	/** Start position of bss in SP dmem */
-	uint32_t bss_size;		/** Size of bss section */
-	/** Dynamic data filled by loader */
-	CSS_ALIGN(const void  *code, 8);		/** Code section absolute pointer within fw, code = icache + text */
-	CSS_ALIGN(const void  *data, 8);		/** Data section absolute pointer within fw, data = data + bss */
-};
-
-struct ia_css_binary_input_info {
-	uint32_t		min_width;
-	uint32_t		min_height;
-	uint32_t		max_width;
-	uint32_t		max_height;
-	uint32_t		source; /* memory, sensor, variable */
-};
-
-struct ia_css_binary_output_info {
-	uint32_t		min_width;
-	uint32_t		min_height;
-	uint32_t		max_width;
-	uint32_t		max_height;
-	uint32_t		num_chunks;
-	uint32_t		variable_format;
-};
-
-struct ia_css_binary_internal_info {
-	uint32_t		max_width;
-	uint32_t		max_height;
-};
-
-struct ia_css_binary_bds_info {
-	uint32_t		supported_bds_factors;
-};
-
-struct ia_css_binary_dvs_info {
-	uint32_t		max_envelope_width;
-	uint32_t		max_envelope_height;
-};
-
-struct ia_css_binary_vf_dec_info {
-	uint32_t		is_variable;
-	uint32_t		max_log_downscale;
-};
-
-struct ia_css_binary_s3a_info {
-	uint32_t		s3atbl_use_dmem;
-	uint32_t		fixed_s3a_deci_log;
-};
-
-/* DPC related binary info */
-struct ia_css_binary_dpc_info {
-	uint32_t		bnr_lite; /** bnr lite enable flag */
-};
-
-struct ia_css_binary_iterator_info {
-	uint32_t		num_stripes;
-	uint32_t		row_stripes_height;
-	uint32_t		row_stripes_overlap_lines;
-};
-
-struct ia_css_binary_address_info {
-	uint32_t		isp_addresses;	/* Address in ISP dmem */
-	uint32_t		main_entry;	/* Address of entry fct */
-	uint32_t		in_frame;	/* Address in ISP dmem */
-	uint32_t		out_frame;	/* Address in ISP dmem */
-	uint32_t		in_data;	/* Address in ISP dmem */
-	uint32_t		out_data;	/* Address in ISP dmem */
-	uint32_t		sh_dma_cmd_ptr;     /* In ISP dmem */
-};
-
-struct ia_css_binary_uds_info {
-	uint16_t	bpp;
-	uint16_t	use_bci;
-	uint16_t	use_str;
-	uint16_t	woix;
-	uint16_t	woiy;
-	uint16_t	extra_out_vecs;
-	uint16_t	vectors_per_line_in;
-	uint16_t	vectors_per_line_out;
-	uint16_t	vectors_c_per_line_in;
-	uint16_t	vectors_c_per_line_out;
-	uint16_t	vmem_gdc_in_block_height_y;
-	uint16_t	vmem_gdc_in_block_height_c;
-	/* uint16_t padding; */
-};
-
-struct ia_css_binary_pipeline_info {
-	uint32_t	mode;
-	uint32_t	isp_pipe_version;
-	uint32_t	pipelining;
-	uint32_t	c_subsampling;
-	uint32_t	top_cropping;
-	uint32_t	left_cropping;
-	uint32_t	variable_resolution;
-};
-
-struct ia_css_binary_block_info {
-	uint32_t	block_width;
-	uint32_t	block_height;
-	uint32_t	output_block_height;
-};
-
-/* Structure describing an ISP binary.
- * It describes the capabilities of a binary, like the maximum resolution,
- * support features, dma channels, uds features, etc.
- * This part is to be used by the SP.
- * Future refactoring should move binary properties to ia_css_binary_xinfo,
- * thereby making the SP code more binary independent.
- */
-struct ia_css_binary_info {
-	CSS_ALIGN(uint32_t			id, 8); /* IA_CSS_BINARY_ID_* */
-	struct ia_css_binary_pipeline_info	pipeline;
-	struct ia_css_binary_input_info		input;
-	struct ia_css_binary_output_info	output;
-	struct ia_css_binary_internal_info	internal;
-	struct ia_css_binary_bds_info		bds;
-	struct ia_css_binary_dvs_info		dvs;
-	struct ia_css_binary_vf_dec_info	vf_dec;
-	struct ia_css_binary_s3a_info		s3a;
-	struct ia_css_binary_dpc_info		dpc_bnr; /** DPC related binary info */
-	struct ia_css_binary_iterator_info	iterator;
-	struct ia_css_binary_address_info	addresses;
-	struct ia_css_binary_uds_info		uds;
-	struct ia_css_binary_block_info		block;
-	struct ia_css_isp_param_isp_segments	mem_initializers;
-/* MW: Packing (related) bools in an integer ?? */
-	struct {
-#ifdef ISP2401
-		uint8_t	luma_only;
-		uint8_t	input_yuv;
-		uint8_t	input_raw;
-#endif
-		uint8_t	reduced_pipe;
-		uint8_t	vf_veceven;
-		uint8_t	dis;
-		uint8_t	dvs_envelope;
-		uint8_t	uds;
-		uint8_t	dvs_6axis;
-		uint8_t	block_output;
-		uint8_t	streaming_dma;
-		uint8_t	ds;
-		uint8_t	bayer_fir_6db;
-		uint8_t	raw_binning;
-		uint8_t	continuous;
-		uint8_t	s3a;
-		uint8_t	fpnr;
-		uint8_t	sc;
-		uint8_t	macc;
-		uint8_t	output;
-		uint8_t	ref_frame;
-		uint8_t	tnr;
-		uint8_t	xnr;
-		uint8_t	params;
-		uint8_t	ca_gdc;
-		uint8_t	isp_addresses;
-		uint8_t	in_frame;
-		uint8_t	out_frame;
-		uint8_t	high_speed;
-		uint8_t	dpc;
-		uint8_t padding[2];
-	} enable;
-	struct {
-/* DMA channel ID: [0,...,HIVE_ISP_NUM_DMA_CHANNELS> */
-		uint8_t	ref_y_channel;
-		uint8_t	ref_c_channel;
-		uint8_t	tnr_channel;
-		uint8_t	tnr_out_channel;
-		uint8_t	dvs_coords_channel;
-		uint8_t	output_channel;
-		uint8_t	c_channel;
-		uint8_t	vfout_channel;
-		uint8_t	vfout_c_channel;
-		uint8_t	vfdec_bits_per_pixel;
-		uint8_t	claimed_by_isp;
-		uint8_t padding[2];
-	} dma;
-};
-
-/* Structure describing an ISP binary.
- * It describes the capabilities of a binary, like the maximum resolution,
- * support features, dma channels, uds features, etc.
- */
-struct ia_css_binary_xinfo {
-	/* Part that is of interest to the SP. */
-	struct ia_css_binary_info    sp;
-
-	/* Rest of the binary info, only interesting to the host. */
-	enum ia_css_acc_type	     type;
-	CSS_ALIGN(int32_t	     num_output_formats, 8);
-	enum ia_css_frame_format     output_formats[IA_CSS_FRAME_FORMAT_NUM];
-	CSS_ALIGN(int32_t	     num_vf_formats, 8); /** number of supported vf formats */
-	enum ia_css_frame_format     vf_formats[IA_CSS_FRAME_FORMAT_NUM]; /** types of supported vf formats */
-	uint8_t			     num_output_pins;
-	ia_css_ptr		     xmem_addr;
-	CSS_ALIGN(const struct ia_css_blob_descr *blob, 8);
-	CSS_ALIGN(uint32_t blob_index, 8);
-	CSS_ALIGN(union ia_css_all_memory_offsets mem_offsets, 8);
-	CSS_ALIGN(struct ia_css_binary_xinfo *next, 8);
-};
-
-/* Structure describing the Bootloader (an ISP binary).
- * It contains several address, either in ddr, isp_dmem or
- * the entry function in icache.
- */
-struct ia_css_bl_info {
-	uint32_t num_dma_cmds;	/** Number of cmds sent by CSS */
-	uint32_t dma_cmd_list;	/** Dma command list sent by CSS */
-	uint32_t sw_state;	/** Polled from css */
-	/* Entry functions */
-	uint32_t bl_entry;	/** The SP entry function */
-};
-
-/* Structure describing the SP binary.
- * It contains several address, either in ddr, sp_dmem or
- * the entry function in pmem.
- */
-struct ia_css_sp_info {
-	uint32_t init_dmem_data; /** data sect config, stored to dmem */
-	uint32_t per_frame_data; /** Per frame data, stored to dmem */
-	uint32_t group;		/** Per pipeline data, loaded by dma */
-	uint32_t output;		/** SP output data, loaded by dmem */
-	uint32_t host_sp_queue;	/** Host <-> SP queues */
-	uint32_t host_sp_com;/** Host <-> SP commands */
-	uint32_t isp_started;	/** Polled from sensor thread, csim only */
-	uint32_t sw_state;	/** Polled from css */
-	uint32_t host_sp_queues_initialized; /** Polled from the SP */
-	uint32_t sleep_mode;  /** different mode to halt SP */
-	uint32_t invalidate_tlb;		/** inform SP to invalidate mmu TLB */
-#ifndef ISP2401
-	uint32_t stop_copy_preview;       /** suspend copy and preview pipe when capture */
-#endif
-	uint32_t debug_buffer_ddr_address;	/** inform SP the address
-	of DDR debug queue */
-	uint32_t perf_counter_input_system_error; /** input system perf
-	counter array */
-#ifdef HAS_WATCHDOG_SP_THREAD_DEBUG
-	uint32_t debug_wait; /** thread/pipe post mortem debug */
-	uint32_t debug_stage; /** thread/pipe post mortem debug */
-	uint32_t debug_stripe; /** thread/pipe post mortem debug */
-#endif
-	uint32_t threads_stack; /** sp thread's stack pointers */
-	uint32_t threads_stack_size; /** sp thread's stack sizes */
-	uint32_t curr_binary_id;        /** current binary id */
-	uint32_t raw_copy_line_count;   /** raw copy line counter */
-	uint32_t ddr_parameter_address; /** acc param ddrptr, sp dmem */
-	uint32_t ddr_parameter_size;    /** acc param size, sp dmem */
-	/* Entry functions */
-	uint32_t sp_entry;	/** The SP entry function */
-	uint32_t tagger_frames_addr;   /** Base address of tagger state */
-};
-
-/* The following #if is there because this header file is also included
-   by SP and ISP code but they do not need this data and HIVECC has alignment
-   issue with the firmware struct/union's.
-   More permanent solution will be to refactor this include.
-*/
-#if !defined(__ISP)
-/* Accelerator firmware information.
- */
-struct ia_css_acc_info {
-	uint32_t per_frame_data; /** Dummy for now */
-};
-
-/* Firmware information.
- */
-union ia_css_fw_union {
-	struct ia_css_binary_xinfo	isp; /** ISP info */
-	struct ia_css_sp_info		sp;  /** SP info */
-	struct ia_css_bl_info           bl;  /** Bootloader info */
-	struct ia_css_acc_info		acc; /** Accelerator info */
-};
-
-/* Firmware information.
- */
-struct ia_css_fw_info {
-	size_t			 header_size; /** size of fw header */
-	CSS_ALIGN(uint32_t type, 8);
-	union ia_css_fw_union	 info; /** Binary info */
-	struct ia_css_blob_info  blob; /** Blob info */
-	/* Dynamic part */
-	struct ia_css_fw_info   *next;
-	CSS_ALIGN(uint32_t       loaded, 8);	/** Firmware has been loaded */
-	CSS_ALIGN(const uint8_t *isp_code, 8);  /** ISP pointer to code */
-	/** Firmware handle between user space and kernel */
-	CSS_ALIGN(uint32_t	handle, 8);
-	/** Sections to copy from/to ISP */
-	struct ia_css_isp_param_css_segments mem_initializers;
-	/** Initializer for local ISP memories */
-};
-
-struct ia_css_blob_descr {
-	const unsigned char  *blob;
-	struct ia_css_fw_info header;
-	const char	     *name;
-	union ia_css_all_memory_offsets mem_offsets;
-};
-
-struct ia_css_acc_fw;
-
-/* Structure describing the SP binary of a stand-alone accelerator.
- */
-struct ia_css_acc_sp {
-	void (*init)(struct ia_css_acc_fw *);	/** init for crun */
-	uint32_t sp_prog_name_offset;		/** program name offset wrt hdr in bytes */
-	uint32_t sp_blob_offset;		/** blob offset wrt hdr in bytes */
-	void	 *entry;			/** Address of sp entry point */
-	uint32_t *css_abort;			/** SP dmem abort flag */
-	void	 *isp_code;			/** SP dmem address holding xmem
-						     address of isp code */
-	struct ia_css_fw_info fw;		/** SP fw descriptor */
-	const uint8_t *code;			/** ISP pointer of allocated SP code */
-};
-
-/* Acceleration firmware descriptor.
-  * This descriptor descibes either SP code (stand-alone), or
-  * ISP code (a separate pipeline stage).
-  */
-struct ia_css_acc_fw_hdr {
-	enum ia_css_acc_type type;	/** Type of accelerator */
-	uint32_t	isp_prog_name_offset; /** program name offset wrt
-						   header in bytes */
-	uint32_t	isp_blob_offset;      /** blob offset wrt header
-						   in bytes */
-	uint32_t	isp_size;	      /** Size of isp blob */
-	const uint8_t  *isp_code;	      /** ISP pointer to code */
-	struct ia_css_acc_sp  sp;  /** Standalone sp code */
-	/** Firmware handle between user space and kernel */
-	uint32_t	handle;
-	struct ia_css_data parameters; /** Current SP parameters */
-};
-
-/* Firmware structure.
-  * This contains the header and actual blobs.
-  * For standalone, it contains SP and ISP blob.
-  * For a pipeline stage accelerator, it contains ISP code only.
-  * Since its members are variable size, their offsets are described in the
-  * header and computed using the access macros below.
-  */
-struct ia_css_acc_fw {
-	struct ia_css_acc_fw_hdr header; /** firmware header */
-	/*
-	int8_t   isp_progname[];	  **< ISP program name
-	int8_t   sp_progname[];	  **< SP program name, stand-alone only
-	uint8_t sp_code[];  **< SP blob, stand-alone only
-	uint8_t isp_code[]; **< ISP blob
-	*/
-};
-
-/* Access macros for firmware */
-#define IA_CSS_ACC_OFFSET(t, f, n) ((t)((uint8_t *)(f)+(f->header.n)))
-#define IA_CSS_ACC_SP_PROG_NAME(f) IA_CSS_ACC_OFFSET(const char *, f, \
-						 sp.sp_prog_name_offset)
-#define IA_CSS_ACC_ISP_PROG_NAME(f) IA_CSS_ACC_OFFSET(const char *, f, \
-						 isp_prog_name_offset)
-#define IA_CSS_ACC_SP_CODE(f)      IA_CSS_ACC_OFFSET(uint8_t *, f, \
-						 sp.sp_blob_offset)
-#define IA_CSS_ACC_SP_DATA(f)      (IA_CSS_ACC_SP_CODE(f) + \
-					(f)->header.sp.fw.blob.data_source)
-#define IA_CSS_ACC_ISP_CODE(f)     IA_CSS_ACC_OFFSET(uint8_t*, f,\
-						 isp_blob_offset)
-#define IA_CSS_ACC_ISP_SIZE(f)     ((f)->header.isp_size)
-
-/* Binary name follows header immediately */
-#define IA_CSS_EXT_ISP_PROG_NAME(f)   ((const char *)(f)+(f)->blob.prog_name_offset)
-#define IA_CSS_EXT_ISP_MEM_OFFSETS(f) \
-	((const struct ia_css_memory_offsets *)((const char *)(f)+(f)->blob.mem_offsets))
-
-#endif /* !defined(__ISP) */
-
-enum ia_css_sp_sleep_mode {
-	SP_DISABLE_SLEEP_MODE = 0,
-	SP_SLEEP_AFTER_FRAME = 1 << 0,
-	SP_SLEEP_AFTER_IRQ = 1 << 1
-};
-#endif /* _IA_CSS_ACC_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_buffer.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_buffer.h
deleted file mode 100644
index a0058eac7d5a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_buffer.h
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BUFFER_H
-#define __IA_CSS_BUFFER_H
-
-/* @file
- * This file contains datastructures and types for buffers used in CSS
- */
-
-#include <type_support.h>
-#include "ia_css_types.h"
-#include "ia_css_timer.h"
-
-/* Enumeration of buffer types. Buffers can be queued and de-queued
- *  to hand them over between IA and ISP.
- */
-enum ia_css_buffer_type {
-	IA_CSS_BUFFER_TYPE_INVALID = -1,
-	IA_CSS_BUFFER_TYPE_3A_STATISTICS = 0,
-	IA_CSS_BUFFER_TYPE_DIS_STATISTICS,
-	IA_CSS_BUFFER_TYPE_LACE_STATISTICS,
-	IA_CSS_BUFFER_TYPE_INPUT_FRAME,
-	IA_CSS_BUFFER_TYPE_OUTPUT_FRAME,
-	IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME,
-	IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME,
-	IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME,
-	IA_CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME,
-	IA_CSS_BUFFER_TYPE_CUSTOM_INPUT,
-	IA_CSS_BUFFER_TYPE_CUSTOM_OUTPUT,
-	IA_CSS_BUFFER_TYPE_METADATA,
-	IA_CSS_BUFFER_TYPE_PARAMETER_SET,
-	IA_CSS_BUFFER_TYPE_PER_FRAME_PARAMETER_SET,
-	IA_CSS_NUM_DYNAMIC_BUFFER_TYPE,
-	IA_CSS_NUM_BUFFER_TYPE
-};
-
-/* Driver API is not SP/ISP visible, 64 bit types not supported on hivecc */
-#if !defined(__ISP)
-/* Buffer structure. This is a container structure that enables content
- *  independent buffer queues and access functions.
- */
-struct ia_css_buffer {
-	enum ia_css_buffer_type type; /** Buffer type. */
-	unsigned int exp_id;
-	/** exposure id for this buffer; 0 = not available
-	     see ia_css_event_public.h for more detail. */
-	union {
-		struct ia_css_isp_3a_statistics  *stats_3a;    /** 3A statistics & optionally RGBY statistics. */
-		struct ia_css_isp_dvs_statistics *stats_dvs;   /** DVS statistics. */
-		struct ia_css_isp_skc_dvs_statistics *stats_skc_dvs;  /** SKC DVS statistics. */
-		struct ia_css_frame              *frame;       /** Frame buffer. */
-		struct ia_css_acc_param          *custom_data; /** Custom buffer. */
-		struct ia_css_metadata           *metadata;    /** Sensor metadata. */
-	} data; /** Buffer data pointer. */
-	uint64_t driver_cookie; /** cookie for the driver */
-	struct ia_css_time_meas timing_data; /** timing data (readings from the timer) */
-	struct ia_css_clock_tick isys_eof_clock_tick; /** ISYS's end of frame timer tick*/
-};
-
-/* @brief Dequeue param buffers from sp2host_queue
- *
- * @return                                       None
- *
- * This function must be called at every driver interrupt handler to prevent
- * overflow of sp2host_queue.
- */
-void
-ia_css_dequeue_param_buffers(void);
-
-#endif /* !__ISP */
-
-#endif /* __IA_CSS_BUFFER_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_control.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_control.h
deleted file mode 100644
index 021a313fab85..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_control.h
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CONTROL_H
-#define __IA_CSS_CONTROL_H
-
-/* @file
- * This file contains functionality for starting and controlling CSS
- */
-
-#include <type_support.h>
-#include <ia_css_env.h>
-#include <ia_css_firmware.h>
-#include <ia_css_irq.h>
-
-/* @brief Initialize the CSS API.
- * @param[in]	env		Environment, provides functions to access the
- *				environment in which the CSS code runs. This is
- *				used for host side memory access and message
- *				printing. May not be NULL.
- * @param[in]	fw		Firmware package containing the firmware for all
- *				predefined ISP binaries.
- *				if fw is NULL the firmware must be loaded before
- *				through a call of ia_css_load_firmware
- * @param[in]	l1_base         Base index (isp2400)
- *                              of the L1 page table. This is a physical
- *                              address or index.
- * @param[in]	irq_type	The type of interrupt to be used (edge or level)
- * @return				Returns IA_CSS_ERR_INTERNAL_ERROR in case of any
- *				errors and IA_CSS_SUCCESS otherwise.
- *
- * This function initializes the API which includes allocating and initializing
- * internal data structures. This also interprets the firmware package. All
- * contents of this firmware package are copied into local data structures, so
- * the fw pointer could be freed after this function completes.
- */
-enum ia_css_err ia_css_init(
-	const struct ia_css_env *env,
-	const struct ia_css_fw  *fw,
-	uint32_t                 l1_base,
-	enum ia_css_irq_type     irq_type);
-
-/* @brief Un-initialize the CSS API.
- * @return	None
- *
- * This function deallocates all memory that has been allocated by the CSS API
- * Exception: if you explicitly loaded firmware through ia_css_load_firmware
- * you need to call ia_css_unload_firmware to deallocate the memory reserved
- * for the firmware.
- * After this function is called, no other CSS functions should be called
- * with the exception of ia_css_init which will re-initialize the CSS code,
- * ia_css_unload_firmware to unload the firmware or ia_css_load_firmware
- * to load new firmware
- */
-void
-ia_css_uninit(void);
-
-/* @brief Suspend CSS API for power down
- * @return	success or faulure code
- *
- * suspend shuts down the system by:
- *  unloading all the streams
- *  stopping SP
- *  performing uninit
- *
- *  Currently stream memory is deallocated because of rmmgr issues.
- *  Need to come up with a bypass that will leave the streams intact.
- */
-enum ia_css_err
-ia_css_suspend(void);
-
-/* @brief Resume CSS API from power down
- * @return	success or failure code
- *
- * After a power cycle, this function will bring the CSS API back into
- * a state where it can be started.
- * This will re-initialize the hardware and all the streams.
- * Call this function only after ia_css_suspend() has been called.
- */
-enum ia_css_err
-ia_css_resume(void);
-
-/* @brief Enable use of a separate queue for ISYS events.
- *
- * @param[in]	enable: enable or disable use of separate ISYS event queues.
- * @return		error if called when SP is running.
- *
- * @deprecated{This is a temporary function that allows drivers to migrate to
- * the use of the separate ISYS event queue. Once all drivers supports this, it
- * will be made the default and this function will be removed.
- * This function should only be called when the SP is not running, calling it
- * when the SP is running will result in an error value being returned. }
- */
-enum ia_css_err
-ia_css_enable_isys_event_queue(bool enable);
-
-/* @brief Test whether the ISP has started.
- *
- * @return	Boolean flag true if the ISP has started or false otherwise.
- *
- * Temporary function to poll whether the ISP has been started. Once it has,
- * the sensor can also be started. */
-bool
-ia_css_isp_has_started(void);
-
-/* @brief Test whether the SP has initialized.
- *
- * @return	Boolean flag true if the SP has initialized or false otherwise.
- *
- * Temporary function to poll whether the SP has been initialized. Once it has,
- * we can enqueue buffers. */
-bool
-ia_css_sp_has_initialized(void);
-
-/* @brief Test whether the SP has terminated.
- *
- * @return	Boolean flag true if the SP has terminated or false otherwise.
- *
- * Temporary function to poll whether the SP has been terminated. Once it has,
- * we can switch mode. */
-bool
-ia_css_sp_has_terminated(void);
-
-/* @brief start SP hardware
- *
- * @return			IA_CSS_SUCCESS or error code upon error.
- *
- * It will boot the SP hardware and start multi-threading infrastructure.
- * All threads will be started and blocked by semaphore. This function should
- * be called before any ia_css_stream_start().
- */
-enum ia_css_err
-ia_css_start_sp(void);
-
-
-/* @brief stop SP hardware
- *
- * @return			IA_CSS_SUCCESS or error code upon error.
- *
- * This function will terminate all threads and shut down SP. It should be
- * called after all ia_css_stream_stop().
- */
-enum ia_css_err
-ia_css_stop_sp(void);
-
-#endif /* __IA_CSS_CONTROL_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_device_access.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_device_access.c
deleted file mode 100644
index 21b842379acc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_device_access.c
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_device_access.h"
-#include <type_support.h>   /* for uint*, size_t */
-#include <system_types.h>   /* for hrt_address */
-#include <ia_css_env.h>     /* for ia_css_hw_access_env */
-#include <assert_support.h> /* for assert */
-
-static struct ia_css_hw_access_env my_env;
-
-void
-ia_css_device_access_init(const struct ia_css_hw_access_env *env)
-{
-	assert(env != NULL);
-
-	my_env = *env;
-}
-
-uint8_t
-ia_css_device_load_uint8(const hrt_address addr)
-{
-	return my_env.load_8(addr);
-}
-
-uint16_t
-ia_css_device_load_uint16(const hrt_address addr)
-{
-	return my_env.load_16(addr);
-}
-
-uint32_t
-ia_css_device_load_uint32(const hrt_address addr)
-{
-	return my_env.load_32(addr);
-}
-
-uint64_t
-ia_css_device_load_uint64(const hrt_address addr)
-{
-	assert(0);
-
-	(void)addr;
-	return 0;
-}
-
-void
-ia_css_device_store_uint8(const hrt_address addr, const uint8_t data)
-{
-	my_env.store_8(addr, data);
-}
-
-void
-ia_css_device_store_uint16(const hrt_address addr, const uint16_t data)
-{
-	my_env.store_16(addr, data);
-}
-
-void
-ia_css_device_store_uint32(const hrt_address addr, const uint32_t data)
-{
-	my_env.store_32(addr, data);
-}
-
-void
-ia_css_device_store_uint64(const hrt_address addr, const uint64_t data)
-{
-	assert(0);
-
-	(void)addr;
-	(void)data;
-}
-
-void
-ia_css_device_load(const hrt_address addr, void *data, const size_t size)
-{
-	my_env.load(addr, data, (uint32_t)size);
-}
-
-void
-ia_css_device_store(const hrt_address addr, const void *data, const size_t size)
-{
-	my_env.store(addr, data, (uint32_t)size);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_device_access.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_device_access.h
deleted file mode 100644
index 84a960b7abbc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_device_access.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_DEVICE_ACCESS_H
-#define _IA_CSS_DEVICE_ACCESS_H
-
-/* @file
- * File containing internal functions for the CSS-API to access the CSS device.
- */
-
-#include <type_support.h> /* for uint*, size_t */
-#include <system_types.h> /* for hrt_address */
-#include <ia_css_env.h>   /* for ia_css_hw_access_env */
-
-void
-ia_css_device_access_init(const struct ia_css_hw_access_env *env);
-
-uint8_t
-ia_css_device_load_uint8(const hrt_address addr);
-
-uint16_t
-ia_css_device_load_uint16(const hrt_address addr);
-
-uint32_t
-ia_css_device_load_uint32(const hrt_address addr);
-
-uint64_t
-ia_css_device_load_uint64(const hrt_address addr);
-
-void
-ia_css_device_store_uint8(const hrt_address addr, const uint8_t data);
-
-void
-ia_css_device_store_uint16(const hrt_address addr, const uint16_t data);
-
-void
-ia_css_device_store_uint32(const hrt_address addr, const uint32_t data);
-
-void
-ia_css_device_store_uint64(const hrt_address addr, const uint64_t data);
-
-void
-ia_css_device_load(const hrt_address addr, void *data, const size_t size);
-
-void
-ia_css_device_store(const hrt_address addr, const void *data, const size_t size);
-
-#endif /* _IA_CSS_DEVICE_ACCESS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_dvs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_dvs.h
deleted file mode 100644
index 1f01534964e3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_dvs.h
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DVS_H
-#define __IA_CSS_DVS_H
-
-/* @file
- * This file contains types for DVS statistics
- */
-
-#include <type_support.h>
-#include "ia_css_types.h"
-#include "ia_css_err.h"
-#include "ia_css_stream_public.h"
-
-enum dvs_statistics_type {
-	DVS_STATISTICS,
-	DVS2_STATISTICS,
-	SKC_DVS_STATISTICS
-};
-
-
-/* Structure that holds DVS statistics in the ISP internal
- * format. Use ia_css_get_dvs_statistics() to translate
- * this to the format used on the host (DVS engine).
- * */
-struct ia_css_isp_dvs_statistics {
-	ia_css_ptr hor_proj;
-	ia_css_ptr ver_proj;
-	uint32_t   hor_size;
-	uint32_t   ver_size;
-	uint32_t   exp_id;   /** see ia_css_event_public.h for more detail */
-	ia_css_ptr data_ptr; /* base pointer containing all memory */
-	uint32_t   size;     /* size of allocated memory in data_ptr */
-};
-
-/* Structure that holds SKC DVS statistics in the ISP internal
- * format. Use ia_css_dvs_statistics_get() to translate this to
- * the format used on the host.
- * */
-struct ia_css_isp_skc_dvs_statistics;
-
-
-#define SIZE_OF_IA_CSS_ISP_DVS_STATISTICS_STRUCT			\
-	((3 * SIZE_OF_IA_CSS_PTR) +					\
-	 (4 * sizeof(uint32_t)))
-
-/* Map with host-side pointers to ISP-format statistics.
- * These pointers can either be copies of ISP data or memory mapped
- * ISP pointers.
- * All of the data behind these pointers is allocatd contiguously, the
- * allocated pointer is stored in the data_ptr field. The other fields
- * point into this one block of data.
- */
-struct ia_css_isp_dvs_statistics_map {
-	void    *data_ptr;
-	int32_t *hor_proj;
-	int32_t *ver_proj;
-	uint32_t size;		 /* total size in bytes */
-	uint32_t data_allocated; /* indicate whether data was allocated */
-};
-
-union ia_css_dvs_statistics_isp {
-	struct ia_css_isp_dvs_statistics *p_dvs_statistics_isp;
-	struct ia_css_isp_skc_dvs_statistics *p_skc_dvs_statistics_isp;
-};
-
-union ia_css_dvs_statistics_host {
-	struct ia_css_dvs_statistics *p_dvs_statistics_host;
-	struct ia_css_dvs2_statistics *p_dvs2_statistics_host;
-	struct ia_css_skc_dvs_statistics *p_skc_dvs_statistics_host;
-};
-
-/* @brief Copy DVS statistics from an ISP buffer to a host buffer.
- * @param[in]	host_stats Host buffer
- * @param[in]	isp_stats ISP buffer
- * @return	error value if temporary memory cannot be allocated
- *
- * This may include a translation step as well depending
- * on the ISP version.
- * Always use this function, never copy the buffer directly.
- * Note that this function uses the mem_load function from the CSS
- * environment struct.
- * In certain environments this may be slow. In those cases it is
- * advised to map the ISP memory into a host-side pointer and use
- * the ia_css_translate_dvs_statistics() function instead.
- */
-enum ia_css_err
-ia_css_get_dvs_statistics(struct ia_css_dvs_statistics *host_stats,
-			  const struct ia_css_isp_dvs_statistics *isp_stats);
-
-/* @brief Translate DVS statistics from ISP format to host format
- * @param[in]	host_stats Host buffer
- * @param[in]	isp_stats ISP buffer
- * @return	None
- *
- * This function translates the dvs statistics from the ISP-internal
- * format to the format used by the DVS library on the CPU.
- * This function takes a host-side pointer as input. This can either
- * point to a copy of the data or be a memory mapped pointer to the
- * ISP memory pages.
- */
-void
-ia_css_translate_dvs_statistics(
-		struct ia_css_dvs_statistics *host_stats,
-		const struct ia_css_isp_dvs_statistics_map *isp_stats);
-
-/* @brief Copy DVS 2.0 statistics from an ISP buffer to a host buffer.
- * @param[in]	host_stats Host buffer
- * @param[in]	isp_stats ISP buffer
- * @return	error value if temporary memory cannot be allocated
- *
- * This may include a translation step as well depending
- * on the ISP version.
- * Always use this function, never copy the buffer directly.
- * Note that this function uses the mem_load function from the CSS
- * environment struct.
- * In certain environments this may be slow. In those cases it is
- * advised to map the ISP memory into a host-side pointer and use
- * the ia_css_translate_dvs2_statistics() function instead.
- */
-enum ia_css_err
-ia_css_get_dvs2_statistics(struct ia_css_dvs2_statistics *host_stats,
-			   const struct ia_css_isp_dvs_statistics *isp_stats);
-
-/* @brief Translate DVS2 statistics from ISP format to host format
- * @param[in]	host_stats Host buffer
- * @param[in]	isp_stats ISP buffer
- * @return		None
- *
- * This function translates the dvs2 statistics from the ISP-internal
- * format to the format used by the DVS2 library on the CPU.
- * This function takes a host-side pointer as input. This can either
- * point to a copy of the data or be a memory mapped pointer to the
- * ISP memory pages.
- */
-void
-ia_css_translate_dvs2_statistics(
-		struct ia_css_dvs2_statistics	   *host_stats,
-		const struct ia_css_isp_dvs_statistics_map *isp_stats);
-
-/* @brief Copy DVS statistics from an ISP buffer to a host buffer.
- * @param[in] type - DVS statistics type
- * @param[in] host_stats Host buffer
- * @param[in] isp_stats ISP buffer
- * @return None
- */
-void
-ia_css_dvs_statistics_get(enum dvs_statistics_type type,
-			  union ia_css_dvs_statistics_host  *host_stats,
-			  const union ia_css_dvs_statistics_isp *isp_stats);
-
-/* @brief Allocate the DVS statistics memory on the ISP
- * @param[in]	grid The grid.
- * @return	Pointer to the allocated DVS statistics buffer on the ISP
-*/
-struct ia_css_isp_dvs_statistics *
-ia_css_isp_dvs_statistics_allocate(const struct ia_css_dvs_grid_info *grid);
-
-/* @brief Free the DVS statistics memory on the ISP
- * @param[in]	me Pointer to the DVS statistics buffer on the ISP.
- * @return	None
-*/
-void
-ia_css_isp_dvs_statistics_free(struct ia_css_isp_dvs_statistics *me);
-
-/* @brief Allocate the DVS 2.0 statistics memory
- * @param[in]	grid The grid.
- * @return	Pointer to the allocated DVS statistics buffer on the ISP
-*/
-struct ia_css_isp_dvs_statistics *
-ia_css_isp_dvs2_statistics_allocate(const struct ia_css_dvs_grid_info *grid);
-
-/* @brief Free the DVS 2.0 statistics memory
- * @param[in]	me Pointer to the DVS statistics buffer on the ISP.
- * @return	None
-*/
-void
-ia_css_isp_dvs2_statistics_free(struct ia_css_isp_dvs_statistics *me);
-
-/* @brief Allocate the DVS statistics memory on the host
- * @param[in]	grid The grid.
- * @return	Pointer to the allocated DVS statistics buffer on the host
-*/
-struct ia_css_dvs_statistics *
-ia_css_dvs_statistics_allocate(const struct ia_css_dvs_grid_info *grid);
-
-/* @brief Free the DVS statistics memory on the host
- * @param[in]	me Pointer to the DVS statistics buffer on the host.
- * @return	None
-*/
-void
-ia_css_dvs_statistics_free(struct ia_css_dvs_statistics *me);
-
-/* @brief Allocate the DVS coefficients memory
- * @param[in]	grid The grid.
- * @return	Pointer to the allocated DVS coefficients buffer
-*/
-struct ia_css_dvs_coefficients *
-ia_css_dvs_coefficients_allocate(const struct ia_css_dvs_grid_info *grid);
-
-/* @brief Free the DVS coefficients memory
- * @param[in]	me Pointer to the DVS coefficients buffer.
- * @return	None
- */
-void
-ia_css_dvs_coefficients_free(struct ia_css_dvs_coefficients *me);
-
-/* @brief Allocate the DVS 2.0 statistics memory on the host
- * @param[in]	grid The grid.
- * @return	Pointer to the allocated DVS 2.0 statistics buffer on the host
- */
-struct ia_css_dvs2_statistics *
-ia_css_dvs2_statistics_allocate(const struct ia_css_dvs_grid_info *grid);
-
-/* @brief Free the DVS 2.0 statistics memory
- * @param[in]	me Pointer to the DVS 2.0 statistics buffer on the host.
- * @return	None
-*/
-void
-ia_css_dvs2_statistics_free(struct ia_css_dvs2_statistics *me);
-
-/* @brief Allocate the DVS 2.0 coefficients memory
- * @param[in]	grid The grid.
- * @return	Pointer to the allocated DVS 2.0 coefficients buffer
-*/
-struct ia_css_dvs2_coefficients *
-ia_css_dvs2_coefficients_allocate(const struct ia_css_dvs_grid_info *grid);
-
-/* @brief Free the DVS 2.0 coefficients memory
- * @param[in]	me Pointer to the DVS 2.0 coefficients buffer.
- * @return	None
-*/
-void
-ia_css_dvs2_coefficients_free(struct ia_css_dvs2_coefficients *me);
-
-/* @brief Allocate the DVS 2.0 6-axis config memory
- * @param[in]	stream The stream.
- * @return	Pointer to the allocated DVS 6axis configuration buffer
-*/
-struct ia_css_dvs_6axis_config *
-ia_css_dvs2_6axis_config_allocate(const struct ia_css_stream *stream);
-
-/* @brief Free the DVS 2.0 6-axis config memory
- * @param[in]	dvs_6axis_config Pointer to the DVS 6axis configuration buffer
- * @return	None
- */
-void
-ia_css_dvs2_6axis_config_free(struct ia_css_dvs_6axis_config *dvs_6axis_config);
-
-/* @brief Allocate a dvs statistics map structure
- * @param[in]	isp_stats pointer to ISP dvs statistis struct
- * @param[in]	data_ptr  host-side pointer to ISP dvs statistics.
- * @return	Pointer to the allocated dvs statistics map
- *
- * This function allocates the ISP dvs statistics map structure
- * and uses the data_ptr as base pointer to set the appropriate
- * pointers to all relevant subsets of the dvs statistics (dmem,
- * vmem, hmem).
- * If the data_ptr is NULL, this function will allocate the host-side
- * memory. This information is stored in the struct and used in the
- * ia_css_isp_dvs_statistics_map_free() function to determine whether
- * the memory should be freed or not.
- * Note that this function does not allocate or map any ISP
- * memory.
-*/
-struct ia_css_isp_dvs_statistics_map *
-ia_css_isp_dvs_statistics_map_allocate(
-	const struct ia_css_isp_dvs_statistics *isp_stats,
-	void *data_ptr);
-
-/* @brief Free the dvs statistics map
- * @param[in]	me Pointer to the dvs statistics map
- * @return	None
- *
- * This function frees the map struct. If the data_ptr inside it
- * was allocated inside ia_css_isp_dvs_statistics_map_allocate(), it
- * will be freed in this function. Otherwise it will not be freed.
- */
-void
-ia_css_isp_dvs_statistics_map_free(struct ia_css_isp_dvs_statistics_map *me);
-
-/* @brief Allocate memory for the SKC DVS statistics on the ISP
- * @return		Pointer to the allocated ACC DVS statistics buffer on the ISP
-*/
-struct ia_css_isp_skc_dvs_statistics *ia_css_skc_dvs_statistics_allocate(void);
-
-#endif /*  __IA_CSS_DVS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_env.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_env.h
deleted file mode 100644
index 8b0218ee658d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_env.h
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ENV_H
-#define __IA_CSS_ENV_H
-
-#include <type_support.h>
-#include <stdarg.h> /* va_list */
-#include "ia_css_types.h"
-#include "ia_css_acc_types.h"
-
-/* @file
- * This file contains prototypes for functions that need to be provided to the
- * CSS-API host-code by the environment in which the CSS-API code runs.
- */
-
-/* Memory allocation attributes, for use in ia_css_css_mem_env. */
-enum ia_css_mem_attr {
-	IA_CSS_MEM_ATTR_CACHED = 1 << 0,
-	IA_CSS_MEM_ATTR_ZEROED = 1 << 1,
-	IA_CSS_MEM_ATTR_PAGEALIGN = 1 << 2,
-	IA_CSS_MEM_ATTR_CONTIGUOUS = 1 << 3,
-};
-
-/* Environment with function pointers for local IA memory allocation.
- *  This provides the CSS code with environment specific functionality
- *  for memory allocation of small local buffers such as local data structures.
- *  This is never expected to allocate more than one page of memory (4K bytes).
- */
-struct ia_css_cpu_mem_env {
-	void (*flush)(struct ia_css_acc_fw *fw);
-	/** Flush function to flush the cache for given accelerator. */
-};
-
-/* Environment with function pointers to access the CSS hardware. This includes
- *  registers and local memories.
- */
-struct ia_css_hw_access_env {
-	void (*store_8)(hrt_address addr, uint8_t data);
-	/** Store an 8 bit value into an address in the CSS HW address space.
-	     The address must be an 8 bit aligned address. */
-	void (*store_16)(hrt_address addr, uint16_t data);
-	/** Store a 16 bit value into an address in the CSS HW address space.
-	     The address must be a 16 bit aligned address. */
-	void (*store_32)(hrt_address addr, uint32_t data);
-	/** Store a 32 bit value into an address in the CSS HW address space.
-	     The address must be a 32 bit aligned address. */
-	uint8_t (*load_8)(hrt_address addr);
-	/** Load an 8 bit value from an address in the CSS HW address
-	     space. The address must be an 8 bit aligned address. */
-	uint16_t (*load_16)(hrt_address addr);
-	/** Load a 16 bit value from an address in the CSS HW address
-	     space. The address must be a 16 bit aligned address. */
-	uint32_t (*load_32)(hrt_address addr);
-	/** Load a 32 bit value from an address in the CSS HW address
-	     space. The address must be a 32 bit aligned address. */
-	void (*store)(hrt_address addr, const void *data, uint32_t bytes);
-	/** Store a number of bytes into a byte-aligned address in the CSS HW address space. */
-	void (*load)(hrt_address addr, void *data, uint32_t bytes);
-	/** Load a number of bytes from a byte-aligned address in the CSS HW address space. */
-};
-
-/* Environment with function pointers to print error and debug messages.
- */
-struct ia_css_print_env {
-	int (*debug_print)(const char *fmt, va_list args);
-	/** Print a debug message. */
-	int (*error_print)(const char *fmt, va_list args);
-	/** Print an error message.*/
-};
-
-/* Environment structure. This includes function pointers to access several
- *  features provided by the environment in which the CSS API is used.
- *  This is used to run the camera IP in multiple platforms such as Linux,
- *  Windows and several simulation environments.
- */
-struct ia_css_env {
-	struct ia_css_cpu_mem_env   cpu_mem_env;   /** local flush. */
-	struct ia_css_hw_access_env hw_access_env; /** CSS HW access functions */
-	struct ia_css_print_env     print_env;     /** Message printing env. */
-};
-
-#endif /* __IA_CSS_ENV_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_err.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_err.h
deleted file mode 100644
index cf895815ea31..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_err.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ERR_H
-#define __IA_CSS_ERR_H
-
-/* @file
- * This file contains possible return values for most
- * functions in the CSS-API.
- */
-
-/* Errors, these values are used as the return value for most
- *  functions in this API.
- */
-enum ia_css_err {
-	IA_CSS_SUCCESS,
-	IA_CSS_ERR_INTERNAL_ERROR,
-	IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY,
-	IA_CSS_ERR_INVALID_ARGUMENTS,
-	IA_CSS_ERR_SYSTEM_NOT_IDLE,
-	IA_CSS_ERR_MODE_HAS_NO_VIEWFINDER,
-	IA_CSS_ERR_QUEUE_IS_FULL,
-	IA_CSS_ERR_QUEUE_IS_EMPTY,
-	IA_CSS_ERR_RESOURCE_NOT_AVAILABLE,
-	IA_CSS_ERR_RESOURCE_LIST_TO_SMALL,
-	IA_CSS_ERR_RESOURCE_ITEMS_STILL_ALLOCATED,
-	IA_CSS_ERR_RESOURCE_EXHAUSTED,
-	IA_CSS_ERR_RESOURCE_ALREADY_ALLOCATED,
-	IA_CSS_ERR_VERSION_MISMATCH,
-	IA_CSS_ERR_NOT_SUPPORTED
-};
-
-/* FW warnings. This enum contains a value for each warning that
- * the SP FW could indicate potential performance issue
- */
-enum ia_css_fw_warning {
-	IA_CSS_FW_WARNING_NONE,
-	IA_CSS_FW_WARNING_ISYS_QUEUE_FULL, /* < CSS system delayed because of insufficient space in the ISys queue.
-		This warning can be avoided by de-queing ISYS buffers more timely. */
-	IA_CSS_FW_WARNING_PSYS_QUEUE_FULL, /* < CSS system delayed because of insufficient space in the PSys queue.
-		This warning can be avoided by de-queing PSYS buffers more timely. */
-	IA_CSS_FW_WARNING_CIRCBUF_ALL_LOCKED, /* < CSS system delayed because of insufficient available buffers.
-		This warning can be avoided by unlocking locked frame-buffers more timely. */
-	IA_CSS_FW_WARNING_EXP_ID_LOCKED, /* < Exposure ID skipped because the frame associated to it was still locked.
-		This warning can be avoided by unlocking locked frame-buffers more timely. */
-	IA_CSS_FW_WARNING_TAG_EXP_ID_FAILED, /* < Exposure ID cannot be found on the circular buffer.
-		This warning can be avoided by unlocking locked frame-buffers more timely. */
-	IA_CSS_FW_WARNING_FRAME_PARAM_MISMATCH, /* < Frame and param pair mismatched in tagger.
-		This warning can be avoided by providing a param set for each frame. */
-};
-
-#endif /* __IA_CSS_ERR_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_event_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_event_public.h
deleted file mode 100644
index 036a2f03d3bd..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_event_public.h
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_EVENT_PUBLIC_H
-#define __IA_CSS_EVENT_PUBLIC_H
-
-/* @file
- * This file contains CSS-API events functionality
- */
-
-#include <type_support.h>	/* uint8_t */
-#include <ia_css_err.h>		/* ia_css_err */
-#include <ia_css_types.h>	/* ia_css_pipe */
-#include <ia_css_timer.h>	/* ia_css_timer */
-
-/* The event type, distinguishes the kind of events that
- * can are generated by the CSS system.
- *
- * !!!IMPORTANT!!! KEEP THE FOLLOWING IN SYNC:
- * 1) "enum ia_css_event_type"					(ia_css_event_public.h)
- * 2) "enum sh_css_sp_event_type"				(sh_css_internal.h)
- * 3) "enum ia_css_event_type event_id_2_event_mask"		(event_handler.sp.c)
- * 4) "enum ia_css_event_type convert_event_sp_to_host_domain"	(sh_css.c)
- */
-enum ia_css_event_type {
-	IA_CSS_EVENT_TYPE_OUTPUT_FRAME_DONE		= 1 << 0,
-	/** Output frame ready. */
-	IA_CSS_EVENT_TYPE_SECOND_OUTPUT_FRAME_DONE	= 1 << 1,
-	/** Second output frame ready. */
-	IA_CSS_EVENT_TYPE_VF_OUTPUT_FRAME_DONE		= 1 << 2,
-	/** Viewfinder Output frame ready. */
-	IA_CSS_EVENT_TYPE_SECOND_VF_OUTPUT_FRAME_DONE	= 1 << 3,
-	/** Second viewfinder Output frame ready. */
-	IA_CSS_EVENT_TYPE_3A_STATISTICS_DONE		= 1 << 4,
-	/** Indication that 3A statistics are available. */
-	IA_CSS_EVENT_TYPE_DIS_STATISTICS_DONE		= 1 << 5,
-	/** Indication that DIS statistics are available. */
-	IA_CSS_EVENT_TYPE_PIPELINE_DONE			= 1 << 6,
-	/** Pipeline Done event, sent after last pipeline stage. */
-	IA_CSS_EVENT_TYPE_FRAME_TAGGED			= 1 << 7,
-	/** Frame tagged. */
-	IA_CSS_EVENT_TYPE_INPUT_FRAME_DONE		= 1 << 8,
-	/** Input frame ready. */
-	IA_CSS_EVENT_TYPE_METADATA_DONE			= 1 << 9,
-	/** Metadata ready. */
-	IA_CSS_EVENT_TYPE_LACE_STATISTICS_DONE		= 1 << 10,
-	/** Indication that LACE statistics are available. */
-	IA_CSS_EVENT_TYPE_ACC_STAGE_COMPLETE		= 1 << 11,
-	/** Extension stage complete. */
-	IA_CSS_EVENT_TYPE_TIMER				= 1 << 12,
-	/** Timer event for measuring the SP side latencies. It contains the
-             32-bit timer value from the SP */
-	IA_CSS_EVENT_TYPE_PORT_EOF			= 1 << 13,
-	/** End Of Frame event, sent when in buffered sensor mode. */
-	IA_CSS_EVENT_TYPE_FW_WARNING			= 1 << 14,
-	/** Performance warning encounter by FW */
-	IA_CSS_EVENT_TYPE_FW_ASSERT			= 1 << 15,
-	/** Assertion hit by FW */
-};
-
-#define IA_CSS_EVENT_TYPE_NONE 0
-
-/* IA_CSS_EVENT_TYPE_ALL is a mask for all pipe related events.
- * The other events (such as PORT_EOF) cannot be enabled/disabled
- * and are hence excluded from this macro.
- */
-#define IA_CSS_EVENT_TYPE_ALL \
-	(IA_CSS_EVENT_TYPE_OUTPUT_FRAME_DONE		| \
-	 IA_CSS_EVENT_TYPE_SECOND_OUTPUT_FRAME_DONE	| \
-	 IA_CSS_EVENT_TYPE_VF_OUTPUT_FRAME_DONE		| \
-	 IA_CSS_EVENT_TYPE_SECOND_VF_OUTPUT_FRAME_DONE	| \
-	 IA_CSS_EVENT_TYPE_3A_STATISTICS_DONE		| \
-	 IA_CSS_EVENT_TYPE_DIS_STATISTICS_DONE		| \
-	 IA_CSS_EVENT_TYPE_PIPELINE_DONE		| \
-	 IA_CSS_EVENT_TYPE_FRAME_TAGGED			| \
-	 IA_CSS_EVENT_TYPE_INPUT_FRAME_DONE		| \
-	 IA_CSS_EVENT_TYPE_METADATA_DONE		| \
-	 IA_CSS_EVENT_TYPE_LACE_STATISTICS_DONE		| \
-	 IA_CSS_EVENT_TYPE_ACC_STAGE_COMPLETE)
-
-/* The event struct, container for the event type and its related values.
- * Depending on the event type, either pipe or port will be filled.
- * Pipeline related events (like buffer/frame events) will return a valid and filled pipe handle.
- * For non pipeline related events (but i.e. stream specific, like EOF event), the port will be
- * filled.
- */
-struct ia_css_event {
-	struct ia_css_pipe    *pipe;
-	/** Pipe handle on which event happened, NULL for non pipe related
-	     events. */
-	enum ia_css_event_type type;
-	/** Type of Event, always valid/filled. */
-	uint8_t                port;
-	/** Port number for EOF event (not valid for other events). */
-	uint8_t                exp_id;
-	/** Exposure id for EOF/FRAME_TAGGED/FW_WARNING event (not valid for other events)
-	     The exposure ID is unique only within a logical stream and it is
-	     only generated on systems that have an input system (such as 2400
-	     and 2401).
-	     Most outputs produced by the CSS are tagged with an exposure ID.
-	     This allows users of the CSS API to keep track of which buffer
-	     was generated from which sensor output frame. This includes:
-	     EOF event, output frames, 3A statistics, DVS statistics and
-	     sensor metadata.
-	     Exposure IDs start at IA_CSS_MIN_EXPOSURE_ID, increment by one
-	     until IA_CSS_MAX_EXPOSURE_ID is reached, after that they wrap
-	     around to IA_CSS_MIN_EXPOSURE_ID again.
-	     Note that in case frames are dropped, this will not be reflected
-	     in the exposure IDs. Therefor applications should not use this
-	     to detect frame drops. */
-	uint32_t               fw_handle;
-	/** Firmware Handle for ACC_STAGE_COMPLETE event (not valid for other
-	     events). */
-	enum ia_css_fw_warning fw_warning;
-	/** Firmware warning code, only for WARNING events. */
-	uint8_t                fw_assert_module_id;
-	/** Firmware module id, only for ASSERT events, should be logged by driver. */
-	uint16_t               fw_assert_line_no;
-	/** Firmware line number, only for ASSERT events, should be logged by driver. */
-	clock_value_t	       timer_data;
-	/** For storing the full 32-bit of the timer value. Valid only for TIMER
-	     event */
-	uint8_t                timer_code;
-	/** For storing the code of the TIMER event. Valid only for
-	     TIMER event */
-	uint8_t                timer_subcode;
-	/** For storing the subcode of the TIMER event. Valid only
-	     for TIMER event */
-};
-
-/* @brief Dequeue a PSYS event from the CSS system.
- *
- * @param[out]	event   Pointer to the event struct which will be filled by
- *                      this function if an event is available.
- * @return		IA_CSS_ERR_QUEUE_IS_EMPTY if no events are
- *			available or
- *			IA_CSS_SUCCESS otherwise.
- *
- * This function dequeues an event from the PSYS event queue. The queue is
- * between the Host CPU and the CSS system. This function can be
- * called after an interrupt has been generated that signalled that a new event
- * was available and can be used in a polling-like situation where the NO_EVENT
- * return value is used to determine whether an event was available or not.
- */
-enum ia_css_err
-ia_css_dequeue_psys_event(struct ia_css_event *event);
-
-/* @brief Dequeue an event from the CSS system.
- *
- * @param[out]	event   Pointer to the event struct which will be filled by
- *                      this function if an event is available.
- * @return		IA_CSS_ERR_QUEUE_IS_EMPTY if no events are
- *			available or
- *			IA_CSS_SUCCESS otherwise.
- *
- * deprecated{Use ia_css_dequeue_psys_event instead}.
- * Unless the isys event queue is explicitly enabled, this function will
- * dequeue both isys (EOF) and psys events (all others).
- */
-enum ia_css_err
-ia_css_dequeue_event(struct ia_css_event *event);
-
-/* @brief Dequeue an ISYS event from the CSS system.
- *
- * @param[out]	event   Pointer to the event struct which will be filled by
- *                      this function if an event is available.
- * @return		IA_CSS_ERR_QUEUE_IS_EMPTY if no events are
- *			available or
- *			IA_CSS_SUCCESS otherwise.
- *
- * This function dequeues an event from the ISYS event queue. The queue is
- * between host and the CSS system.
- * Unlike the ia_css_dequeue_event() function, this function can be called
- * directly from an interrupt service routine (ISR) and it is safe to call
- * this function in parallel with other CSS API functions (but only one
- * call to this function should be in flight at any point in time).
- *
- * The reason for having the ISYS events separate is to prevent them from
- * incurring additional latency due to locks being held by other CSS API
- * functions.
- */
-enum ia_css_err
-ia_css_dequeue_isys_event(struct ia_css_event *event);
-
-#endif /* __IA_CSS_EVENT_PUBLIC_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_firmware.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_firmware.h
deleted file mode 100644
index d7d7f0a995e5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_firmware.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_FIRMWARE_H
-#define __IA_CSS_FIRMWARE_H
-
-/* @file
- * This file contains firmware loading/unloading support functionality
- */
-
-#include "ia_css_err.h"
-#include "ia_css_env.h"
-
-/* CSS firmware package structure.
- */
-struct ia_css_fw {
-	void	    *data;  /** pointer to the firmware data */
-	unsigned int bytes; /** length in bytes of firmware data */
-};
-
-/* @brief Loads the firmware
- * @param[in]	env		Environment, provides functions to access the
- *				environment in which the CSS code runs. This is
- *				used for host side memory access and message
- *				printing.
- * @param[in]	fw		Firmware package containing the firmware for all
- *				predefined ISP binaries.
- * @return			Returns IA_CSS_ERR_INTERNAL_ERROR in case of any
- *				errors and IA_CSS_SUCCESS otherwise.
- *
- * This function interprets the firmware package. All
- * contents of this firmware package are copied into local data structures, so
- * the fw pointer could be freed after this function completes.
- *
- * Rationale for this function is that it can be called before ia_css_init, and thus
- * speeds up ia_css_init (ia_css_init is called each time a stream is created but the
- * firmware only needs to be loaded once).
- */
-enum ia_css_err
-ia_css_load_firmware(const struct ia_css_env *env,
-	    const struct ia_css_fw  *fw);
-
-/* @brief Unloads the firmware
- * @return	None
- *
- * This function unloads the firmware loaded by ia_css_load_firmware.
- * It is pointless to call this function if no firmware is loaded,
- * but it won't harm. Use this to deallocate all memory associated with the firmware.
- */
-void
-ia_css_unload_firmware(void);
-
-/* @brief Checks firmware version
- * @param[in]	fw	Firmware package containing the firmware for all
- *			predefined ISP binaries.
- * @return		Returns true when the firmware version matches with the CSS
- *			host code version and returns false otherwise.
- * This function checks if the firmware package version matches with the CSS host code version.
- */
-bool
-ia_css_check_firmware_version(const struct ia_css_fw  *fw);
-
-#endif /* __IA_CSS_FIRMWARE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frac.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frac.h
deleted file mode 100644
index e5ffc579aef1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frac.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_FRAC_H
-#define _IA_CSS_FRAC_H
-
-/* @file
- * This file contains typedefs used for fractional numbers
- */
-
-#include <type_support.h>
-
-/* Fixed point types.
- * NOTE: the 16 bit fixed point types actually occupy 32 bits
- * to save on extension operations in the ISP code.
- */
-/* Unsigned fixed point value, 0 integer bits, 16 fractional bits */
-typedef uint32_t ia_css_u0_16;
-/* Unsigned fixed point value, 5 integer bits, 11 fractional bits */
-typedef uint32_t ia_css_u5_11;
-/* Unsigned fixed point value, 8 integer bits, 8 fractional bits */
-typedef uint32_t ia_css_u8_8;
-/* Signed fixed point value, 0 integer bits, 15 fractional bits */
-typedef int32_t ia_css_s0_15;
-
-#endif /* _IA_CSS_FRAC_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frame_format.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frame_format.h
deleted file mode 100644
index 2f177edc36ac..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frame_format.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_FRAME_FORMAT_H
-#define __IA_CSS_FRAME_FORMAT_H
-
-/* @file
- * This file contains information about formats supported in the ISP
- */
-
-/* Frame formats, some of these come from fourcc.org, others are
-   better explained by video4linux2. The NV11 seems to be described only
-   on MSDN pages, but even those seem to be gone now.
-   Frames can come in many forms, the main categories are RAW, RGB and YUV
-   (or YCbCr). The YUV frames come in 4 flavors, determined by how the U and V
-   values are subsampled:
-   1. YUV420: hor = 2, ver = 2
-   2. YUV411: hor = 4, ver = 1
-   3. YUV422: hor = 2, ver = 1
-   4. YUV444: hor = 1, ver = 1
-
-  Warning: not all frame formats are supported as input or output to/from ISP.
-    Some of these formats are therefore not defined in the output table module.
-    Modifications in below frame format enum can require modifications in the
-    output table module.
-
-  Warning2: Throughout the CSS code assumptions are made on the order
-	of formats in this enumeration type, or some sort of copy is maintained.
-	The following files are identified:
-	- FileSupport.h
-	- css/isp/kernels/fc/fc_1.0/formats.isp.c
-	- css/isp/kernels/output/output_1.0/output_table.isp.c
-	- css/isp/kernels/output/sc_output_1.0/formats.hive.c
-	- css/isp/modes/interface/isp_formats.isp.h
-	- css/bxt_sandbox/psyspoc/interface/ia_css_pg_info.h
-	- css/bxt_sandbox/psysapi/data/interface/ia_css_program_group_data.h
-	- css/bxt_sandbox/isysapi/interface/ia_css_isysapi_fw_types.h
-*/
-enum ia_css_frame_format {
-	IA_CSS_FRAME_FORMAT_NV11 = 0,   /** 12 bit YUV 411, Y, UV plane */
-	IA_CSS_FRAME_FORMAT_NV12,       /** 12 bit YUV 420, Y, UV plane */
-	IA_CSS_FRAME_FORMAT_NV12_16,    /** 16 bit YUV 420, Y, UV plane */
-	IA_CSS_FRAME_FORMAT_NV12_TILEY, /** 12 bit YUV 420, Intel proprietary tiled format, TileY */
-	IA_CSS_FRAME_FORMAT_NV16,       /** 16 bit YUV 422, Y, UV plane */
-	IA_CSS_FRAME_FORMAT_NV21,       /** 12 bit YUV 420, Y, VU plane */
-	IA_CSS_FRAME_FORMAT_NV61,       /** 16 bit YUV 422, Y, VU plane */
-	IA_CSS_FRAME_FORMAT_YV12,       /** 12 bit YUV 420, Y, V, U plane */
-	IA_CSS_FRAME_FORMAT_YV16,       /** 16 bit YUV 422, Y, V, U plane */
-	IA_CSS_FRAME_FORMAT_YUV420,     /** 12 bit YUV 420, Y, U, V plane */
-	IA_CSS_FRAME_FORMAT_YUV420_16,  /** yuv420, 16 bits per subpixel */
-	IA_CSS_FRAME_FORMAT_YUV422,     /** 16 bit YUV 422, Y, U, V plane */
-	IA_CSS_FRAME_FORMAT_YUV422_16,  /** yuv422, 16 bits per subpixel */
-	IA_CSS_FRAME_FORMAT_UYVY,       /** 16 bit YUV 422, UYVY interleaved */
-	IA_CSS_FRAME_FORMAT_YUYV,       /** 16 bit YUV 422, YUYV interleaved */
-	IA_CSS_FRAME_FORMAT_YUV444,     /** 24 bit YUV 444, Y, U, V plane */
-	IA_CSS_FRAME_FORMAT_YUV_LINE,   /** Internal format, 2 y lines followed
-					     by a uvinterleaved line */
-	IA_CSS_FRAME_FORMAT_RAW,	/** RAW, 1 plane */
-	IA_CSS_FRAME_FORMAT_RGB565,     /** 16 bit RGB, 1 plane. Each 3 sub
-					     pixels are packed into one 16 bit
-					     value, 5 bits for R, 6 bits for G
-					     and 5 bits for B. */
-	IA_CSS_FRAME_FORMAT_PLANAR_RGB888, /** 24 bit RGB, 3 planes */
-	IA_CSS_FRAME_FORMAT_RGBA888,	/** 32 bit RGBA, 1 plane, A=Alpha
-					     (alpha is unused) */
-	IA_CSS_FRAME_FORMAT_QPLANE6, /** Internal, for advanced ISP */
-	IA_CSS_FRAME_FORMAT_BINARY_8,	/** byte stream, used for jpeg. For
-					     frames of this type, we set the
-					     height to 1 and the width to the
-					     number of allocated bytes. */
-	IA_CSS_FRAME_FORMAT_MIPI,	/** MIPI frame, 1 plane */
-	IA_CSS_FRAME_FORMAT_RAW_PACKED, /** RAW, 1 plane, packed */
-	IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_8,	      /** 8 bit per Y/U/V.
-							   Y odd line; UYVY
-							   interleaved even line */
-	IA_CSS_FRAME_FORMAT_CSI_MIPI_LEGACY_YUV420_8, /** Legacy YUV420. UY odd
-							   line; VY even line */
-	IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_10       /** 10 bit per Y/U/V. Y odd
-							   line; UYVY interleaved
-							   even line */
-};
-
-/* NOTE: IA_CSS_FRAME_FORMAT_NUM was purposely defined outside of enum type ia_css_frame_format, */
-/*       because of issues this would cause with the Clockwork code checking tool.               */
-#define IA_CSS_FRAME_FORMAT_NUM (IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_10 + 1)
-
-/* Number of valid output frame formats for ISP **/
-#define IA_CSS_FRAME_OUT_FORMAT_NUM	(IA_CSS_FRAME_FORMAT_RGBA888 + 1)
-
-#endif /* __IA_CSS_FRAME_FORMAT_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frame_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frame_public.h
deleted file mode 100644
index 0beb7347a4f3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_frame_public.h
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_FRAME_PUBLIC_H
-#define __IA_CSS_FRAME_PUBLIC_H
-
-/* @file
- * This file contains structs to describe various frame-formats supported by the ISP.
- */
-
-#include <type_support.h>
-#include "ia_css_err.h"
-#include "ia_css_types.h"
-#include "ia_css_frame_format.h"
-#include "ia_css_buffer.h"
-
-/* For RAW input, the bayer order needs to be specified separately. There
- *  are 4 possible orders. The name is constructed by taking the first two
- *  colors on the first line and the first two colors from the second line.
- */
-enum ia_css_bayer_order {
-	IA_CSS_BAYER_ORDER_GRBG, /** GRGRGRGRGR .. BGBGBGBGBG */
-	IA_CSS_BAYER_ORDER_RGGB, /** RGRGRGRGRG .. GBGBGBGBGB */
-	IA_CSS_BAYER_ORDER_BGGR, /** BGBGBGBGBG .. GRGRGRGRGR */
-	IA_CSS_BAYER_ORDER_GBRG, /** GBGBGBGBGB .. RGRGRGRGRG */
-};
-#define IA_CSS_BAYER_ORDER_NUM (IA_CSS_BAYER_ORDER_GBRG + 1)
-
-/* Frame plane structure. This describes one plane in an image
- *  frame buffer.
- */
-struct ia_css_frame_plane {
-	unsigned int height; /** height of a plane in lines */
-	unsigned int width;  /** width of a line, in DMA elements, note that
-				  for RGB565 the three subpixels are stored in
-				  one element. For all other formats this is
-				  the number of subpixels per line. */
-	unsigned int stride; /** stride of a line in bytes */
-	unsigned int offset; /** offset in bytes to start of frame data.
-				  offset is wrt data field in ia_css_frame */
-};
-
-/* Binary "plane". This is used to story binary streams such as jpeg
- *  images. This is not actually a real plane.
- */
-struct ia_css_frame_binary_plane {
-	unsigned int		  size; /** number of bytes in the stream */
-	struct ia_css_frame_plane data; /** plane */
-};
-
-/* Container for planar YUV frames. This contains 3 planes.
- */
-struct ia_css_frame_yuv_planes {
-	struct ia_css_frame_plane y; /** Y plane */
-	struct ia_css_frame_plane u; /** U plane */
-	struct ia_css_frame_plane v; /** V plane */
-};
-
-/* Container for semi-planar YUV frames.
-  */
-struct ia_css_frame_nv_planes {
-	struct ia_css_frame_plane y;  /** Y plane */
-	struct ia_css_frame_plane uv; /** UV plane */
-};
-
-/* Container for planar RGB frames. Each color has its own plane.
- */
-struct ia_css_frame_rgb_planes {
-	struct ia_css_frame_plane r; /** Red plane */
-	struct ia_css_frame_plane g; /** Green plane */
-	struct ia_css_frame_plane b; /** Blue plane */
-};
-
-/* Container for 6-plane frames. These frames are used internally
- *  in the advanced ISP only.
- */
-struct ia_css_frame_plane6_planes {
-	struct ia_css_frame_plane r;	  /** Red plane */
-	struct ia_css_frame_plane r_at_b; /** Red at blue plane */
-	struct ia_css_frame_plane gr;	  /** Red-green plane */
-	struct ia_css_frame_plane gb;	  /** Blue-green plane */
-	struct ia_css_frame_plane b;	  /** Blue plane */
-	struct ia_css_frame_plane b_at_r; /** Blue at red plane */
-};
-
-/* Crop info struct - stores the lines to be cropped in isp */
-struct ia_css_crop_info {
-	/* the final start column and start line
-	 * sum of lines to be cropped + bayer offset
-	 */
-	unsigned int start_column;
-	unsigned int start_line;
-};
-
-/* Frame info struct. This describes the contents of an image frame buffer.
-  */
-struct ia_css_frame_info {
-	struct ia_css_resolution res; /** Frame resolution (valid data) */
-	unsigned int padded_width; /** stride of line in memory (in pixels) */
-	enum ia_css_frame_format format; /** format of the frame data */
-	unsigned int raw_bit_depth; /** number of valid bits per pixel,
-					 only valid for RAW bayer frames */
-	enum ia_css_bayer_order raw_bayer_order; /** bayer order, only valid
-						      for RAW bayer frames */
-	/* the params below are computed based on bayer_order
-	 * we can remove the raw_bayer_order if it is redundant
-	 * keeping it for now as bxt and fpn code seem to use it
-	 */
-	struct ia_css_crop_info crop_info;
-};
-
-#define IA_CSS_BINARY_DEFAULT_FRAME_INFO \
-(struct ia_css_frame_info) { \
-	.format			= IA_CSS_FRAME_FORMAT_NUM,  \
-	.raw_bayer_order	= IA_CSS_BAYER_ORDER_NUM, \
-}
-
-/**
- *  Specifies the DVS loop delay in "frame periods"
- */
-enum ia_css_frame_delay {
-	IA_CSS_FRAME_DELAY_0, /** Frame delay = 0 */
-	IA_CSS_FRAME_DELAY_1, /** Frame delay = 1 */
-	IA_CSS_FRAME_DELAY_2  /** Frame delay = 2 */
-};
-
-enum ia_css_frame_flash_state {
-	IA_CSS_FRAME_FLASH_STATE_NONE,
-	IA_CSS_FRAME_FLASH_STATE_PARTIAL,
-	IA_CSS_FRAME_FLASH_STATE_FULL
-};
-
-/* Frame structure. This structure describes an image buffer or frame.
- *  This is the main structure used for all input and output images.
- */
-struct ia_css_frame {
-	struct ia_css_frame_info info; /** info struct describing the frame */
-	ia_css_ptr   data;	       /** pointer to start of image data */
-	unsigned int data_bytes;       /** size of image data in bytes */
-	/* LA: move this to ia_css_buffer */
-	/*
-	 * -1 if data address is static during life time of pipeline
-	 * >=0 if data address can change per pipeline/frame iteration
-	 *     index to dynamic data: ia_css_frame_in, ia_css_frame_out
-	 *                            ia_css_frame_out_vf
-	 *     index to host-sp queue id: queue_0, queue_1 etc.
-	 */
-	int dynamic_queue_id;
-	/*
-	 * if it is dynamic frame, buf_type indicates which buffer type it
-	 * should use for event generation. we have this because in vf_pp
-	 * binary, we use output port, but we expect VF_OUTPUT_DONE event
-	 */
-	enum ia_css_buffer_type buf_type;
-	enum ia_css_frame_flash_state flash_state;
-	unsigned int exp_id;
-	/** exposure id, see ia_css_event_public.h for more detail */
-	uint32_t isp_config_id; /** Unique ID to track which config was actually applied to a particular frame */
-	bool valid; /** First video output frame is not valid */
-	bool contiguous; /** memory is allocated physically contiguously */
-	union {
-		unsigned int	_initialisation_dummy;
-		struct ia_css_frame_plane raw;
-		struct ia_css_frame_plane rgb;
-		struct ia_css_frame_rgb_planes planar_rgb;
-		struct ia_css_frame_plane yuyv;
-		struct ia_css_frame_yuv_planes yuv;
-		struct ia_css_frame_nv_planes nv;
-		struct ia_css_frame_plane6_planes plane6;
-		struct ia_css_frame_binary_plane binary;
-	} planes; /** frame planes, select the right one based on
-		       info.format */
-};
-
-#define DEFAULT_FRAME \
-(struct ia_css_frame) { \
-	.info			= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
-	.dynamic_queue_id	= SH_CSS_INVALID_QUEUE_ID, \
-	.buf_type		= IA_CSS_BUFFER_TYPE_INVALID, \
-	.flash_state		= IA_CSS_FRAME_FLASH_STATE_NONE, \
-}
-
-/* @brief Fill a frame with zeros
- *
- * @param	frame		The frame.
- * @return	None
- *
- * Fill a frame with pixel values of zero
- */
-void ia_css_frame_zero(struct ia_css_frame *frame);
-
-/* @brief Allocate a CSS frame structure
- *
- * @param	frame		The allocated frame.
- * @param	width		The width (in pixels) of the frame.
- * @param	height		The height (in lines) of the frame.
- * @param	format		The frame format.
- * @param	stride		The padded stride, in pixels.
- * @param	raw_bit_depth	The raw bit depth, in bits.
- * @return			The error code.
- *
- * Allocate a CSS frame structure. The memory for the frame data will be
- * allocated in the CSS address space.
- */
-enum ia_css_err
-ia_css_frame_allocate(struct ia_css_frame **frame,
-		      unsigned int width,
-		      unsigned int height,
-		      enum ia_css_frame_format format,
-		      unsigned int stride,
-		      unsigned int raw_bit_depth);
-
-/* @brief Allocate a CSS frame structure using a frame info structure.
- *
- * @param	frame	The allocated frame.
- * @param[in]	info	The frame info structure.
- * @return		The error code.
- *
- * Allocate a frame using the resolution and format from a frame info struct.
- * This is a convenience function, implemented on top of
- * ia_css_frame_allocate().
- */
-enum ia_css_err
-ia_css_frame_allocate_from_info(struct ia_css_frame **frame,
-				const struct ia_css_frame_info *info);
-/* @brief Free a CSS frame structure.
- *
- * @param[in]	frame	Pointer to the frame.
- * @return	None
- *
- * Free a CSS frame structure. This will free both the frame structure
- * and the pixel data pointer contained within the frame structure.
- */
-void
-ia_css_frame_free(struct ia_css_frame *frame);
-
-/* @brief Allocate a contiguous CSS frame structure
- *
- * @param	frame		The allocated frame.
- * @param	width		The width (in pixels) of the frame.
- * @param	height		The height (in lines) of the frame.
- * @param	format		The frame format.
- * @param	stride		The padded stride, in pixels.
- * @param	raw_bit_depth	The raw bit depth, in bits.
- * @return			The error code.
- *
- * Contiguous frame allocation, only for FPGA display driver which needs
- * physically contiguous memory.
- * Deprecated.
- */
-enum ia_css_err
-ia_css_frame_allocate_contiguous(struct ia_css_frame **frame,
-				 unsigned int width,
-				 unsigned int height,
-				 enum ia_css_frame_format format,
-				 unsigned int stride,
-				 unsigned int raw_bit_depth);
-
-/* @brief Allocate a contiguous CSS frame from a frame info structure.
- *
- * @param	frame	The allocated frame.
- * @param[in]	info	The frame info structure.
- * @return		The error code.
- *
- * Allocate a frame using the resolution and format from a frame info struct.
- * This is a convenience function, implemented on top of
- * ia_css_frame_allocate_contiguous().
- * Only for FPGA display driver which needs physically contiguous memory.
- * Deprecated.
- */
-enum ia_css_err
-ia_css_frame_allocate_contiguous_from_info(struct ia_css_frame **frame,
-					  const struct ia_css_frame_info *info);
-
-/* @brief Allocate a CSS frame structure using a frame info structure.
- *
- * @param	frame	The allocated frame.
- * @param[in]	info	The frame info structure.
- * @return		The error code.
- *
- * Allocate an empty CSS frame with no data buffer using the parameters
- * in the frame info.
- */
-enum ia_css_err
-ia_css_frame_create_from_info(struct ia_css_frame **frame,
-	const struct ia_css_frame_info *info);
-
-/* @brief Set a mapped data buffer to a CSS frame
- *
- * @param[in]	frame       Valid CSS frame pointer
- * @param[in]	mapped_data  Mapped data buffer to be assigned to the CSS frame
- * @param[in]	data_size_bytes  Size of the mapped_data in bytes
- * @return      The error code.
- *
- * Sets a mapped data buffer to this frame. This function can be called multiple
- * times with different buffers or NULL to reset the data pointer. This API
- * would not try free the mapped_data and its the callers responsiblity to
- * free the mapped_data buffer. However if ia_css_frame_free() is called and
- * the frame had a valid data buffer, it would be freed along with the frame.
- */
-enum ia_css_err
-ia_css_frame_set_data(struct ia_css_frame *frame,
-	const ia_css_ptr   mapped_data,
-	size_t data_size_bytes);
-
-/* @brief Map an existing frame data pointer to a CSS frame.
- *
- * @param	frame		Pointer to the frame to be initialized
- * @param[in]	info		The frame info.
- * @param[in]	data		Pointer to the allocated frame data.
- * @param[in]	attribute	Attributes to be passed to mmgr_mmap.
- * @param[in]	context		Pointer to the a context to be passed to mmgr_mmap.
- * @return			The allocated frame structure.
- *
- * This function maps a pre-allocated pointer into a CSS frame. This can be
- * used when an upper software layer is responsible for allocating the frame
- * data and it wants to share that frame pointer with the CSS code.
- * This function will fill the CSS frame structure just like
- * ia_css_frame_allocate() does, but instead of allocating the memory, it will
- * map the pre-allocated memory into the CSS address space.
- */
-enum ia_css_err
-ia_css_frame_map(struct ia_css_frame **frame,
-		 const struct ia_css_frame_info *info,
-		 const void *data,
-		 uint16_t attribute,
-		 void *context);
-
-/* @brief Unmap a CSS frame structure.
- *
- * @param[in]	frame	Pointer to the CSS frame.
- * @return	None
- *
- * This function unmaps the frame data pointer within a CSS frame and
- * then frees the CSS frame structure. Use this for frame pointers created
- * using ia_css_frame_map().
- */
-void
-ia_css_frame_unmap(struct ia_css_frame *frame);
-
-#endif /* __IA_CSS_FRAME_PUBLIC_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_host_data.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_host_data.h
deleted file mode 100644
index 4557e66891df..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_host_data.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/* Release Version: irci_stable_candrpv_0415_20150521_0458 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SH_CSS_HOST_DATA_H
-#define __SH_CSS_HOST_DATA_H
-
-#include <ia_css_types.h>	/* ia_css_pipe */
-
-/**
- * @brief Allocate structure ia_css_host_data.
- *
- * @param[in]	size		Size of the requested host data
- *
- * @return
- *	- NULL, can't allocate requested size
- *	- pointer to structure, field address points to host data with size bytes
- */
-struct ia_css_host_data *
-ia_css_host_data_allocate(size_t size);
-
-/**
- * @brief Free structure ia_css_host_data.
- *
- * @param[in]	me	Pointer to structure, if a NULL is passed functions
- *			returns without error. Otherwise a valid pointer to
- *			structure must be passed and a related memory
- *			is freed.
- *
- * @return
- */
-void ia_css_host_data_free(struct ia_css_host_data *me);
-
-#endif /* __SH_CSS_HOST_DATA_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_input_port.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_input_port.h
deleted file mode 100644
index ad9ca5449369..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_input_port.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* For MIPI_PORT0_ID to MIPI_PORT2_ID */
-#include "system_global.h"
-
-#ifndef __IA_CSS_INPUT_PORT_H
-#define __IA_CSS_INPUT_PORT_H
-
-/* @file
- * This file contains information about the possible input ports for CSS
- */
-
-/* Backward compatible for CSS API 2.0 only
- *  TO BE REMOVED when all drivers move to CSS	API 2.1
- */
-#define	IA_CSS_CSI2_PORT_4LANE MIPI_PORT0_ID
-#define	IA_CSS_CSI2_PORT_1LANE MIPI_PORT1_ID
-#define	IA_CSS_CSI2_PORT_2LANE MIPI_PORT2_ID
-
-/* The CSI2 interface supports 2 types of compression or can
- *  be run without compression.
- */
-enum ia_css_csi2_compression_type {
-	IA_CSS_CSI2_COMPRESSION_TYPE_NONE, /** No compression */
-	IA_CSS_CSI2_COMPRESSION_TYPE_1,    /** Compression scheme 1 */
-	IA_CSS_CSI2_COMPRESSION_TYPE_2     /** Compression scheme 2 */
-};
-
-struct ia_css_csi2_compression {
-	enum ia_css_csi2_compression_type type;
-	/** Compression used */
-	unsigned int                      compressed_bits_per_pixel;
-	/** Compressed bits per pixel (only when compression is enabled) */
-	unsigned int                      uncompressed_bits_per_pixel;
-	/** Uncompressed bits per pixel (only when compression is enabled) */
-};
-
-/* Input port structure.
- */
-struct ia_css_input_port {
-	enum mipi_port_id port; /** Physical CSI-2 port */
-	unsigned int num_lanes; /** Number of lanes used (4-lane port only) */
-	unsigned int timeout;   /** Timeout value */
-	unsigned int rxcount;   /** Register value, should include all lanes */
-	struct ia_css_csi2_compression compression; /** Compression used */
-};
-
-#endif /* __IA_CSS_INPUT_PORT_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_irq.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_irq.h
deleted file mode 100644
index c8840138899a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_irq.h
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_IRQ_H
-#define __IA_CSS_IRQ_H
-
-/* @file
- * This file contains information for Interrupts/IRQs from CSS
- */
-
-#include "ia_css_err.h"
-#include "ia_css_pipe_public.h"
-#include "ia_css_input_port.h"
-
-/* Interrupt types, these enumerate all supported interrupt types.
- */
-enum ia_css_irq_type {
-	IA_CSS_IRQ_TYPE_EDGE,  /** Edge (level) sensitive interrupt */
-	IA_CSS_IRQ_TYPE_PULSE  /** Pulse-shaped interrupt */
-};
-
-/* Interrupt request type.
- *  When the CSS hardware generates an interrupt, a function in this API
- *  needs to be called to retrieve information about the interrupt.
- *  This interrupt type is part of this information and indicates what
- *  type of information the interrupt signals.
- *
- *  Note that one interrupt can carry multiple interrupt types. For
- *  example: the online video ISP will generate only 2 interrupts, one to
- *  signal that the statistics (3a and DIS) are ready and one to signal
- *  that all output frames are done (output and viewfinder).
- *
- * DEPRECATED, this interface is not portable it should only define user
- * (SW) interrupts
- */
-enum ia_css_irq_info {
-	IA_CSS_IRQ_INFO_CSS_RECEIVER_ERROR            = 1 << 0,
-	/** the css receiver has encountered an error */
-	IA_CSS_IRQ_INFO_CSS_RECEIVER_FIFO_OVERFLOW    = 1 << 1,
-	/** the FIFO in the csi receiver has overflown */
-	IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF              = 1 << 2,
-	/** the css receiver received the start of frame */
-	IA_CSS_IRQ_INFO_CSS_RECEIVER_EOF              = 1 << 3,
-	/** the css receiver received the end of frame */
-	IA_CSS_IRQ_INFO_CSS_RECEIVER_SOL              = 1 << 4,
-	/** the css receiver received the start of line */
-	IA_CSS_IRQ_INFO_PSYS_EVENTS_READY             = 1 << 5,
-	/** One or more events are available in the PSYS event queue */
-	IA_CSS_IRQ_INFO_EVENTS_READY = IA_CSS_IRQ_INFO_PSYS_EVENTS_READY,
-	/** deprecated{obsolete version of IA_CSS_IRQ_INFO_PSYS_EVENTS_READY,
-	 * same functionality.} */
-	IA_CSS_IRQ_INFO_CSS_RECEIVER_EOL              = 1 << 6,
-	/** the css receiver received the end of line */
-	IA_CSS_IRQ_INFO_CSS_RECEIVER_SIDEBAND_CHANGED = 1 << 7,
-	/** the css receiver received a change in side band signals */
-	IA_CSS_IRQ_INFO_CSS_RECEIVER_GEN_SHORT_0      = 1 << 8,
-	/** generic short packets (0) */
-	IA_CSS_IRQ_INFO_CSS_RECEIVER_GEN_SHORT_1      = 1 << 9,
-	/** generic short packets (1) */
-	IA_CSS_IRQ_INFO_IF_PRIM_ERROR                 = 1 << 10,
-	/** the primary input formatter (A) has encountered an error */
-	IA_CSS_IRQ_INFO_IF_PRIM_B_ERROR               = 1 << 11,
-	/** the primary input formatter (B) has encountered an error */
-	IA_CSS_IRQ_INFO_IF_SEC_ERROR                  = 1 << 12,
-	/** the secondary input formatter has encountered an error */
-	IA_CSS_IRQ_INFO_STREAM_TO_MEM_ERROR           = 1 << 13,
-	/** the stream-to-memory device has encountered an error */
-	IA_CSS_IRQ_INFO_SW_0                          = 1 << 14,
-	/** software interrupt 0 */
-	IA_CSS_IRQ_INFO_SW_1                          = 1 << 15,
-	/** software interrupt 1 */
-	IA_CSS_IRQ_INFO_SW_2                          = 1 << 16,
-	/** software interrupt 2 */
-	IA_CSS_IRQ_INFO_ISP_BINARY_STATISTICS_READY   = 1 << 17,
-	/** ISP binary statistics are ready */
-	IA_CSS_IRQ_INFO_INPUT_SYSTEM_ERROR            = 1 << 18,
-	/** the input system in in error */
-	IA_CSS_IRQ_INFO_IF_ERROR                      = 1 << 19,
-	/** the input formatter in in error */
-	IA_CSS_IRQ_INFO_DMA_ERROR                     = 1 << 20,
-	/** the dma in in error */
-	IA_CSS_IRQ_INFO_ISYS_EVENTS_READY             = 1 << 21,
-	/** end-of-frame events are ready in the isys_event queue */
-};
-
-/* CSS receiver error types. Whenever the CSS receiver has encountered
- *  an error, this enumeration is used to indicate which errors have occurred.
- *
- *  Note that multiple error flags can be enabled at once and that this is in
- *  fact common (whenever an error occurs, it usually results in multiple
- *  errors).
- *
- * DEPRECATED: This interface is not portable, different systems have
- * different receiver types, or possibly none in case of tests systems.
- */
-enum ia_css_rx_irq_info {
-	IA_CSS_RX_IRQ_INFO_BUFFER_OVERRUN   = 1U << 0, /** buffer overrun */
-	IA_CSS_RX_IRQ_INFO_ENTER_SLEEP_MODE = 1U << 1, /** entering sleep mode */
-	IA_CSS_RX_IRQ_INFO_EXIT_SLEEP_MODE  = 1U << 2, /** exited sleep mode */
-	IA_CSS_RX_IRQ_INFO_ECC_CORRECTED    = 1U << 3, /** ECC corrected */
-	IA_CSS_RX_IRQ_INFO_ERR_SOT          = 1U << 4,
-						/** Start of transmission */
-	IA_CSS_RX_IRQ_INFO_ERR_SOT_SYNC     = 1U << 5, /** SOT sync (??) */
-	IA_CSS_RX_IRQ_INFO_ERR_CONTROL      = 1U << 6, /** Control (??) */
-	IA_CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE   = 1U << 7, /** Double ECC */
-	IA_CSS_RX_IRQ_INFO_ERR_CRC          = 1U << 8, /** CRC error */
-	IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID   = 1U << 9, /** Unknown ID */
-	IA_CSS_RX_IRQ_INFO_ERR_FRAME_SYNC   = 1U << 10,/** Frame sync error */
-	IA_CSS_RX_IRQ_INFO_ERR_FRAME_DATA   = 1U << 11,/** Frame data error */
-	IA_CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT = 1U << 12,/** Timeout occurred */
-	IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC  = 1U << 13,/** Unknown escape seq. */
-	IA_CSS_RX_IRQ_INFO_ERR_LINE_SYNC    = 1U << 14,/** Line Sync error */
-	IA_CSS_RX_IRQ_INFO_INIT_TIMEOUT     = 1U << 15,
-};
-
-/* Interrupt info structure. This structure contains information about an
- *  interrupt. This needs to be used after an interrupt is received on the IA
- *  to perform the correct action.
- */
-struct ia_css_irq {
-	enum ia_css_irq_info type; /** Interrupt type. */
-	unsigned int sw_irq_0_val; /** In case of SW interrupt 0, value. */
-	unsigned int sw_irq_1_val; /** In case of SW interrupt 1, value. */
-	unsigned int sw_irq_2_val; /** In case of SW interrupt 2, value. */
-	struct ia_css_pipe *pipe;
-	/** The image pipe that generated the interrupt. */
-};
-
-/* @brief Obtain interrupt information.
- *
- * @param[out] info	Pointer to the interrupt info. The interrupt
- *			information wil be written to this info.
- * @return		If an error is encountered during the interrupt info
- *			and no interrupt could be translated successfully, this
- *			will return IA_CSS_INTERNAL_ERROR. Otherwise
- *			IA_CSS_SUCCESS.
- *
- * This function is expected to be executed after an interrupt has been sent
- * to the IA from the CSS. This function returns information about the interrupt
- * which is needed by the IA code to properly handle the interrupt. This
- * information includes the image pipe, buffer type etc.
- */
-enum ia_css_err
-ia_css_irq_translate(unsigned int *info);
-
-/* @brief Get CSI receiver error info.
- *
- * @param[out] irq_bits	Pointer to the interrupt bits. The interrupt
- *			bits will be written this info.
- *			This will be the error bits that are enabled in the CSI
- *			receiver error register.
- * @return	None
- *
- * This function should be used whenever a CSI receiver error interrupt is
- * generated. It provides the detailed information (bits) on the exact error
- * that occurred.
- *
- *@deprecated {this function is DEPRECATED since it only works on CSI port 1.
- * Use the function below instead and specify the appropriate port.}
- */
-void
-ia_css_rx_get_irq_info(unsigned int *irq_bits);
-
-/* @brief Get CSI receiver error info.
- *
- * @param[in]  port     Input port identifier.
- * @param[out] irq_bits	Pointer to the interrupt bits. The interrupt
- *			bits will be written this info.
- *			This will be the error bits that are enabled in the CSI
- *			receiver error register.
- * @return	None
- *
- * This function should be used whenever a CSI receiver error interrupt is
- * generated. It provides the detailed information (bits) on the exact error
- * that occurred.
- */
-void
-ia_css_rx_port_get_irq_info(enum mipi_port_id port, unsigned int *irq_bits);
-
-/* @brief Clear CSI receiver error info.
- *
- * @param[in] irq_bits	The bits that should be cleared from the CSI receiver
- *			interrupt bits register.
- * @return	None
- *
- * This function should be called after ia_css_rx_get_irq_info has been called
- * and the error bits have been interpreted. It is advised to use the return
- * value of that function as the argument to this function to make sure no new
- * error bits get overwritten.
- *
- * @deprecated{this function is DEPRECATED since it only works on CSI port 1.
- * Use the function below instead and specify the appropriate port.}
- */
-void
-ia_css_rx_clear_irq_info(unsigned int irq_bits);
-
-/* @brief Clear CSI receiver error info.
- *
- * @param[in] port      Input port identifier.
- * @param[in] irq_bits	The bits that should be cleared from the CSI receiver
- *			interrupt bits register.
- * @return	None
- *
- * This function should be called after ia_css_rx_get_irq_info has been called
- * and the error bits have been interpreted. It is advised to use the return
- * value of that function as the argument to this function to make sure no new
- * error bits get overwritten.
- */
-void
-ia_css_rx_port_clear_irq_info(enum mipi_port_id port, unsigned int irq_bits);
-
-/* @brief Enable or disable specific interrupts.
- *
- * @param[in] type	The interrupt type that will be enabled/disabled.
- * @param[in] enable	enable or disable.
- * @return		Returns IA_CSS_INTERNAL_ERROR if this interrupt
- *			type cannot be enabled/disabled which is true for
- *			CSS internal interrupts. Otherwise returns
- *			IA_CSS_SUCCESS.
- */
-enum ia_css_err
-ia_css_irq_enable(enum ia_css_irq_info type, bool enable);
-
-#endif /* __IA_CSS_IRQ_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_memory_access.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_memory_access.c
deleted file mode 100644
index 282075942ba6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_memory_access.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015-2017, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <type_support.h>
-#include <system_types.h>
-#include <assert_support.h>
-#include <memory_access.h>
-#include <ia_css_env.h>
-#include <hrt/hive_isp_css_mm_hrt.h>
-
-const hrt_vaddress mmgr_NULL = (hrt_vaddress)0;
-const hrt_vaddress mmgr_EXCEPTION = (hrt_vaddress)-1;
-
-hrt_vaddress
-mmgr_malloc(const size_t size)
-{
-	return mmgr_alloc_attr(size, 0);
-}
-
-hrt_vaddress mmgr_alloc_attr(const size_t size, const uint16_t attrs)
-{
-	uint16_t masked_attrs = attrs & MMGR_ATTRIBUTE_MASK;
-	WARN_ON(attrs & MMGR_ATTRIBUTE_CONTIGUOUS);
-
-	if (masked_attrs & MMGR_ATTRIBUTE_CLEARED) {
-		if (masked_attrs & MMGR_ATTRIBUTE_CACHED)
-			return (ia_css_ptr) hrt_isp_css_mm_calloc_cached(size);
-		else
-			return (ia_css_ptr) hrt_isp_css_mm_calloc(size);
-	} else {
-		if (masked_attrs & MMGR_ATTRIBUTE_CACHED)
-			return (ia_css_ptr) hrt_isp_css_mm_alloc_cached(size);
-		else
-			return (ia_css_ptr) hrt_isp_css_mm_alloc(size);
-	}
-}
-
-hrt_vaddress
-mmgr_calloc(const size_t N, const size_t size)
-{
-	return mmgr_alloc_attr(size * N, MMGR_ATTRIBUTE_CLEARED);
-}
-
-void mmgr_clear(hrt_vaddress vaddr, const size_t size)
-{
-	if (vaddr)
-		hmm_set(vaddr, 0, size);
-}
-
-void mmgr_load(const hrt_vaddress vaddr, void *data, const size_t size)
-{
-	if (vaddr && data)
-		hmm_load(vaddr, data, size);
-}
-
-void
-mmgr_store(const hrt_vaddress vaddr, const void *data, const size_t size)
-{
-	if (vaddr && data)
-		hmm_store(vaddr, data, size);
-}
-
-hrt_vaddress
-mmgr_mmap(const void *ptr, const size_t size,
-	  uint16_t attribute, void *context)
-{
-	struct hrt_userbuffer_attr *userbuffer_attr = context;
-	return hrt_isp_css_mm_alloc_user_ptr(
-			size, (void *)ptr, userbuffer_attr->pgnr,
-			userbuffer_attr->type,
-			attribute & HRT_BUF_FLAG_CACHED);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_metadata.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_metadata.h
deleted file mode 100644
index ed0b6ab371da..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_metadata.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_METADATA_H
-#define __IA_CSS_METADATA_H
-
-/* @file
- * This file contains structure for processing sensor metadata.
- */
-
-#include <type_support.h>
-#include "ia_css_types.h"
-#include "ia_css_stream_format.h"
-
-/* Metadata configuration. This data structure contains necessary info
- *  to process sensor metadata.
- */
-struct ia_css_metadata_config {
-	enum atomisp_input_format data_type; /** Data type of CSI-2 embedded
-			data. The default value is ATOMISP_INPUT_FORMAT_EMBEDDED. For
-			certain sensors, user can choose non-default data type for embedded
-			data. */
-	struct ia_css_resolution  resolution; /** Resolution */
-};
-
-struct ia_css_metadata_info {
-	struct ia_css_resolution resolution; /** Resolution */
-	uint32_t                 stride;     /** Stride in bytes */
-	uint32_t                 size;       /** Total size in bytes */
-};
-
-struct ia_css_metadata {
-	struct ia_css_metadata_info info;    /** Layout info */
-	ia_css_ptr	            address; /** CSS virtual address */
-	uint32_t	            exp_id;
-	/** Exposure ID, see ia_css_event_public.h for more detail */
-};
-#define SIZE_OF_IA_CSS_METADATA_STRUCT sizeof(struct ia_css_metadata)
-
-/* @brief Allocate a metadata buffer.
- * @param[in]   metadata_info Metadata info struct, contains details on metadata buffers.
- * @return      Pointer of metadata buffer or NULL (if error)
- *
- * This function allocates a metadata buffer according to the properties
- * specified in the metadata_info struct.
- */
-struct ia_css_metadata *
-ia_css_metadata_allocate(const struct ia_css_metadata_info *metadata_info);
-
-/* @brief Free a metadata buffer.
- *
- * @param[in]	metadata	Pointer of metadata buffer.
- * @return	None
- *
- * This function frees a metadata buffer.
- */
-void
-ia_css_metadata_free(struct ia_css_metadata *metadata);
-
-#endif /* __IA_CSS_METADATA_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mipi.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mipi.h
deleted file mode 100644
index 367b2aafa5e8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mipi.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MIPI_H
-#define __IA_CSS_MIPI_H
-
-/* @file
- * This file contains MIPI support functionality
- */
-
-#include <type_support.h>
-#include "ia_css_err.h"
-#include "ia_css_stream_format.h"
-#include "ia_css_input_port.h"
-
-/* Backward compatible for CSS API 2.0 only
- * TO BE REMOVED when all drivers move to CSS API 2.1.
- */
-/* @brief Specify a CSS MIPI frame buffer.
- *
- * @param[in]	size_mem_words	The frame size in memory words (32B).
- * @param[in]	contiguous	Allocate memory physically contiguously or not.
- * @return		The error code.
- *
- * \deprecated{Use ia_css_mipi_buffer_config instead.}
- *
- * Specifies a CSS MIPI frame buffer: size in memory words (32B).
- */
-enum ia_css_err
-ia_css_mipi_frame_specify(const unsigned int	size_mem_words,
-				const bool contiguous);
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-/* @brief Register size of a CSS MIPI frame for check during capturing.
- *
- * @param[in]	port	CSI-2 port this check is registered.
- * @param[in]	size_mem_words	The frame size in memory words (32B).
- * @return		Return the error in case of failure. E.g. MAX_NOF_ENTRIES REACHED
- *
- * Register size of a CSS MIPI frame to check during capturing. Up to
- *		IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES entries per port allowed. Entries are reset
- *		when stream is stopped.
- *
- *
- */
-enum ia_css_err
-ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port,
-				const unsigned int	size_mem_words);
-#endif
-
-/* @brief Calculate the size of a mipi frame.
- *
- * @param[in]	width		The width (in pixels) of the frame.
- * @param[in]	height		The height (in lines) of the frame.
- * @param[in]	format		The frame (MIPI) format.
- * @param[in]	hasSOLandEOL	Whether frame (MIPI) contains (optional) SOL and EOF packets.
- * @param[in]	embedded_data_size_words		Embedded data size in memory words.
- * @param		size_mem_words					The mipi frame size in memory words (32B).
- * @return		The error code.
- *
- * Calculate the size of a mipi frame, based on the resolution and format.
- */
-enum ia_css_err
-ia_css_mipi_frame_calculate_size(const unsigned int width,
-				const unsigned int height,
-				const enum atomisp_input_format format,
-				const bool hasSOLandEOL,
-				const unsigned int embedded_data_size_words,
-				unsigned int *size_mem_words);
-
-#endif /* __IA_CSS_MIPI_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mmu.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mmu.h
deleted file mode 100644
index 13c21056bfbf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mmu.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MMU_H
-#define __IA_CSS_MMU_H
-
-/* @file
- * This file contains one support function for invalidating the CSS MMU cache
- */
-
-/* @brief Invalidate the MMU internal cache.
- * @return	None
- *
- * This function triggers an invalidation of the translate-look-aside
- * buffer (TLB) that's inside the CSS MMU. This function should be called
- * every time the page tables used by the MMU change.
- */
-void
-ia_css_mmu_invalidate_cache(void);
-
-#endif /* __IA_CSS_MMU_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mmu_private.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mmu_private.h
deleted file mode 100644
index 1021e4f380a5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_mmu_private.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MMU_PRIVATE_H
-#define __IA_CSS_MMU_PRIVATE_H
-
-#include "system_local.h"
-
-/*
- * This function sets the L1 pagetable address.
- * After power-up of the ISP the L1 pagetable can be set.
- * Once being set the L1 pagetable is protected against
- * further modifications.
- */
-void
-sh_css_mmu_set_page_table_base_index(hrt_data base_index);
-
-#endif /* __IA_CSS_MMU_PRIVATE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_morph.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_morph.h
deleted file mode 100644
index de409638d009..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_morph.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MORPH_H
-#define __IA_CSS_MORPH_H
-
-/* @file
- * This file contains supporting for morphing table
- */
-
-#include <ia_css_types.h>
-
-/* @brief Morphing table
- * @param[in]	width Width of the morphing table.
- * @param[in]	height Height of the morphing table.
- * @return		Pointer to the morphing table
-*/
-struct ia_css_morph_table *
-ia_css_morph_table_allocate(unsigned int width, unsigned int height);
-
-/* @brief Free the morph table
- * @param[in]	me Pointer to the morph table.
- * @return		None
-*/
-void
-ia_css_morph_table_free(struct ia_css_morph_table *me);
-
-#endif /* __IA_CSS_MORPH_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_pipe.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_pipe.h
deleted file mode 100644
index f6870fa7a18c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_pipe.h
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_PIPE_H__
-#define __IA_CSS_PIPE_H__
-
-#include <type_support.h>
-#include "ia_css_stream.h"
-#include "ia_css_frame.h"
-#include "ia_css_pipeline.h"
-#include "ia_css_binary.h"
-#include "sh_css_legacy.h"
-
-#define PIPE_ENTRY_EMPTY_TOKEN                (~0U)
-#define PIPE_ENTRY_RESERVED_TOKEN             (0x1)
-
-struct ia_css_preview_settings {
-	struct ia_css_binary copy_binary;
-	struct ia_css_binary preview_binary;
-	struct ia_css_binary vf_pp_binary;
-
-	/* 2401 only for these two - do we in fact use them for anything real */
-	struct ia_css_frame *delay_frames[MAX_NUM_DELAY_FRAMES];
-	struct ia_css_frame *tnr_frames[NUM_TNR_FRAMES];
-
-	struct ia_css_pipe *copy_pipe;
-	struct ia_css_pipe *capture_pipe;
-	struct ia_css_pipe *acc_pipe;
-};
-
-#define IA_CSS_DEFAULT_PREVIEW_SETTINGS \
-(struct ia_css_preview_settings) { \
-	.copy_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-	.preview_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-	.vf_pp_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-}
-
-struct ia_css_capture_settings {
-	struct ia_css_binary copy_binary;
-	/* we extend primary binary to multiple stages because in ISP2.6.1
-	 * the computation load is too high to fit in one single binary. */
-	struct ia_css_binary primary_binary[MAX_NUM_PRIMARY_STAGES];
-	unsigned int num_primary_stage;
-	struct ia_css_binary pre_isp_binary;
-	struct ia_css_binary anr_gdc_binary;
-	struct ia_css_binary post_isp_binary;
-	struct ia_css_binary capture_pp_binary;
-	struct ia_css_binary vf_pp_binary;
-	struct ia_css_binary capture_ldc_binary;
-	struct ia_css_binary *yuv_scaler_binary;
-	struct ia_css_frame *delay_frames[MAX_NUM_VIDEO_DELAY_FRAMES];
-	bool *is_output_stage;
-	unsigned int num_yuv_scaler;
-};
-
-#define IA_CSS_DEFAULT_CAPTURE_SETTINGS \
-(struct ia_css_capture_settings) { \
-	.copy_binary		= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-	.primary_binary		= {IA_CSS_BINARY_DEFAULT_SETTINGS}, \
-	.pre_isp_binary		= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-	.anr_gdc_binary		= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-	.post_isp_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-	.capture_pp_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-	.vf_pp_binary		= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-	.capture_ldc_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-}
-
-struct ia_css_video_settings {
-	struct ia_css_binary copy_binary;
-	struct ia_css_binary video_binary;
-	struct ia_css_binary vf_pp_binary;
-	struct ia_css_binary *yuv_scaler_binary;
-	struct ia_css_frame *delay_frames[MAX_NUM_VIDEO_DELAY_FRAMES];
-#ifndef ISP2401
-	struct ia_css_frame *tnr_frames[NUM_VIDEO_TNR_FRAMES];
-#else
-	struct ia_css_frame *tnr_frames[NUM_TNR_FRAMES];
-#endif
-	struct ia_css_frame *vf_pp_in_frame;
-	struct ia_css_pipe *copy_pipe;
-	struct ia_css_pipe *capture_pipe;
-	bool *is_output_stage;
-	unsigned int num_yuv_scaler;
-};
-
-#define IA_CSS_DEFAULT_VIDEO_SETTINGS \
-(struct ia_css_video_settings) { \
-	.copy_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-	.video_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-	.vf_pp_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-}
-
-struct ia_css_yuvpp_settings {
-	struct ia_css_binary copy_binary;
-	struct ia_css_binary *yuv_scaler_binary;
-	struct ia_css_binary *vf_pp_binary;
-	bool *is_output_stage;
-	unsigned int num_yuv_scaler;
-	unsigned int num_vf_pp;
-	unsigned int num_output;
-};
-
-#define IA_CSS_DEFAULT_YUVPP_SETTINGS \
-(struct ia_css_yuvpp_settings) { \
-	.copy_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
-}
-
-struct osys_object;
-
-struct ia_css_pipe {
-	/* TODO: Remove stop_requested and use stop_requested in the pipeline */
-	bool                            stop_requested;
-	struct ia_css_pipe_config       config;
-	struct ia_css_pipe_extra_config extra_config;
-	struct ia_css_pipe_info         info;
-	enum ia_css_pipe_id		mode;
-	struct ia_css_shading_table	*shading_table;
-	struct ia_css_pipeline		pipeline;
-	struct ia_css_frame_info	output_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	struct ia_css_frame_info	bds_output_info;
-	struct ia_css_frame_info	vf_output_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	struct ia_css_frame_info	out_yuv_ds_input_info;
-	struct ia_css_frame_info	vf_yuv_ds_input_info;
-	struct ia_css_fw_info		*output_stage;	/* extra output stage */
-	struct ia_css_fw_info		*vf_stage;	/* extra vf_stage */
-	unsigned int			required_bds_factor;
-	unsigned int			dvs_frame_delay;
-	int				num_invalid_frames;
-	bool				enable_viewfinder[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	struct ia_css_stream		*stream;
-	struct ia_css_frame		in_frame_struct;
-	struct ia_css_frame		out_frame_struct;
-	struct ia_css_frame		vf_frame_struct;
-	struct ia_css_frame		*continuous_frames[NUM_CONTINUOUS_FRAMES];
-	struct ia_css_metadata	*cont_md_buffers[NUM_CONTINUOUS_FRAMES];
-	union {
-		struct ia_css_preview_settings preview;
-		struct ia_css_video_settings   video;
-		struct ia_css_capture_settings capture;
-		struct ia_css_yuvpp_settings yuvpp;
-	} pipe_settings;
-	hrt_vaddress scaler_pp_lut;
-	struct osys_object *osys_obj;
-
-	/* This number is unique per pipe each instance of css. This number is
-	 * reused as pipeline number also. There is a 1-1 mapping between pipe_num
-	 * and sp thread id. Current logic limits pipe_num to
-	 * SH_CSS_MAX_SP_THREADS */
-	unsigned int pipe_num;
-};
-
-#define IA_CSS_DEFAULT_PIPE \
-(struct ia_css_pipe) { \
-	.config			= DEFAULT_PIPE_CONFIG, \
-	.info			= DEFAULT_PIPE_INFO, \
-	.mode			= IA_CSS_PIPE_ID_ACC, /* (pipe_id) */ \
-	.pipeline		= DEFAULT_PIPELINE, \
-	.output_info		= {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \
-	.bds_output_info	= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
-	.vf_output_info		= {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \
-	.out_yuv_ds_input_info	= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
-	.vf_yuv_ds_input_info	= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
-	.required_bds_factor	= SH_CSS_BDS_FACTOR_1_00, \
-	.dvs_frame_delay	= 1, \
-	.enable_viewfinder	= {true}, \
-	.in_frame_struct	= DEFAULT_FRAME, \
-	.out_frame_struct	= DEFAULT_FRAME, \
-	.vf_frame_struct	= DEFAULT_FRAME, \
-	.pipe_settings		= { \
-		.preview = IA_CSS_DEFAULT_PREVIEW_SETTINGS \
-	}, \
-	.pipe_num		= PIPE_ENTRY_EMPTY_TOKEN, \
-}
-
-void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map);
-
-enum ia_css_err
-sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
-				struct ia_css_isp_parameters *params,
-				bool commit, struct ia_css_pipe *pipe);
-
-
-
-#endif /* __IA_CSS_PIPE_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_pipe_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_pipe_public.h
deleted file mode 100644
index 11225d5ac442..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_pipe_public.h
+++ /dev/null
@@ -1,579 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_PIPE_PUBLIC_H
-#define __IA_CSS_PIPE_PUBLIC_H
-
-/* @file
- * This file contains the public interface for CSS pipes.
- */
-
-#include <type_support.h>
-#include <ia_css_err.h>
-#include <ia_css_types.h>
-#include <ia_css_frame_public.h>
-#include <ia_css_buffer.h>
-#ifdef ISP2401
-#include <ia_css_acc_types.h>
-#endif
-
-enum {
-	IA_CSS_PIPE_OUTPUT_STAGE_0 = 0,
-	IA_CSS_PIPE_OUTPUT_STAGE_1,
-	IA_CSS_PIPE_MAX_OUTPUT_STAGE,
-};
-
-/* Enumeration of pipe modes. This mode can be used to create
- *  an image pipe for this mode. These pipes can be combined
- *  to configure and run streams on the ISP.
- *
- *  For example, one can create a preview and capture pipe to
- *  create a continuous capture stream.
- */
-enum ia_css_pipe_mode {
-	IA_CSS_PIPE_MODE_PREVIEW,	/** Preview pipe */
-	IA_CSS_PIPE_MODE_VIDEO,		/** Video pipe */
-	IA_CSS_PIPE_MODE_CAPTURE,	/** Still capture pipe */
-	IA_CSS_PIPE_MODE_ACC,		/** Accelerated pipe */
-	IA_CSS_PIPE_MODE_COPY,		/** Copy pipe, only used for embedded/image data copying */
-	IA_CSS_PIPE_MODE_YUVPP,		/** YUV post processing pipe, used for all use cases with YUV input,
-									for SoC sensor and external ISP */
-};
-/* Temporary define  */
-#define IA_CSS_PIPE_MODE_NUM (IA_CSS_PIPE_MODE_YUVPP + 1)
-
-/**
- * Enumeration of pipe versions.
- * the order should match with definition in sh_css_defs.h
- */
-enum ia_css_pipe_version {
-	IA_CSS_PIPE_VERSION_1 = 1,		/** ISP1.0 pipe */
-	IA_CSS_PIPE_VERSION_2_2 = 2,		/** ISP2.2 pipe */
-	IA_CSS_PIPE_VERSION_2_6_1 = 3,		/** ISP2.6.1 pipe */
-	IA_CSS_PIPE_VERSION_2_7 = 4		/** ISP2.7 pipe */
-};
-
-/**
- * Pipe configuration structure.
- * Resolution properties are filled by Driver, kernel configurations are
- * set by AIC
- */
-struct ia_css_pipe_config {
-	enum ia_css_pipe_mode mode;
-	/** mode, indicates which mode the pipe should use. */
-	enum ia_css_pipe_version isp_pipe_version;
-	/** pipe version, indicates which imaging pipeline the pipe should use. */
-	struct ia_css_resolution input_effective_res;
-	/** input effective resolution */
-	struct ia_css_resolution bayer_ds_out_res;
-	/** bayer down scaling */
-	struct ia_css_resolution capt_pp_in_res;
-#ifndef ISP2401
-	/** bayer down scaling */
-#else
-	/** capture post processing input resolution */
-#endif
-	struct ia_css_resolution vf_pp_in_res;
-#ifndef ISP2401
-	/** bayer down scaling */
-#else
-	/** view finder post processing input resolution */
-	struct ia_css_resolution output_system_in_res;
-	/** For IPU3 only: use output_system_in_res to specify what input resolution
-	     will OSYS receive, this resolution is equal to the output resolution of GDC
-	     if not determined CSS will set output_system_in_res with main osys output pin resolution
-	     All other IPUs may ignore this property */
-#endif
-	struct ia_css_resolution dvs_crop_out_res;
-	/** dvs crop, video only, not in use yet. Use dvs_envelope below. */
-	struct ia_css_frame_info output_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	/** output of YUV scaling */
-	struct ia_css_frame_info vf_output_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	/** output of VF YUV scaling */
-	struct ia_css_fw_info *acc_extension;
-	/** Pipeline extension accelerator */
-	struct ia_css_fw_info **acc_stages;
-	/** Standalone accelerator stages */
-	uint32_t num_acc_stages;
-	/** Number of standalone accelerator stages */
-	struct ia_css_capture_config default_capture_config;
-	/** Default capture config for initial capture pipe configuration. */
-	struct ia_css_resolution dvs_envelope; /** temporary */
-	enum ia_css_frame_delay dvs_frame_delay;
-	/** indicates the DVS loop delay in frame periods */
-	int acc_num_execs;
-	/** For acceleration pipes only: determine how many times the pipe
-	     should be run. Setting this to -1 means it will run until
-	     stopped. */
-	bool enable_dz;
-	/** Disabling digital zoom for a pipeline, if this is set to false,
-	     then setting a zoom factor will have no effect.
-	     In some use cases this provides better performance. */
-	bool enable_dpc;
-	/** Disabling "Defect Pixel Correction" for a pipeline, if this is set
-	     to false. In some use cases this provides better performance. */
-	bool enable_vfpp_bci;
-	/** Enabling BCI mode will cause yuv_scale binary to be picked up
-	     instead of vf_pp. This only applies to viewfinder post
-	     processing stages. */
-#ifdef ISP2401
-	bool enable_luma_only;
-	/** Enabling of monochrome mode for a pipeline. If enabled only luma processing
-	     will be done. */
-	bool enable_tnr;
-	/** Enabling of TNR (temporal noise reduction). This is only applicable to video
-	     pipes. Non video-pipes should always set this parameter to false. */
-#endif
-	struct ia_css_isp_config *p_isp_config;
-	/** Pointer to ISP configuration */
-	struct ia_css_resolution gdc_in_buffer_res;
-	/** GDC in buffer resolution. */
-	struct ia_css_point gdc_in_buffer_offset;
-	/** GDC in buffer offset - indicates the pixel coordinates of the first valid pixel inside the buffer */
-#ifdef ISP2401
-	struct ia_css_coordinate internal_frame_origin_bqs_on_sctbl;
-	/** Origin of internal frame positioned on shading table at shading correction in ISP.
-	     NOTE: Shading table is larger than or equal to internal frame.
-		   Shading table has shading gains and internal frame has bayer data.
-		   The origin of internal frame is used in shading correction in ISP
-		   to retrieve shading gains which correspond to bayer data. */
-#endif
-};
-
-
-/**
- * Default settings for newly created pipe configurations.
- */
-#define DEFAULT_PIPE_CONFIG \
-(struct ia_css_pipe_config) { \
-	.mode			= IA_CSS_PIPE_MODE_PREVIEW, \
-	.isp_pipe_version	= 1, \
-	.output_info		= {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \
-	.vf_output_info		= {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \
-	.default_capture_config	= DEFAULT_CAPTURE_CONFIG, \
-	.dvs_frame_delay	= IA_CSS_FRAME_DELAY_1, \
-	.acc_num_execs		= -1, \
-}
-
-/* Pipe info, this struct describes properties of a pipe after it's stream has
- * been created.
- * ~~~** DO NOT ADD NEW FIELD **~~~ This structure will be deprecated.
- *           - On the Behalf of CSS-API Committee.
- */
-struct ia_css_pipe_info {
-	struct ia_css_frame_info output_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	/** Info about output resolution. This contains the stride which
-	     should be used for memory allocation. */
-	struct ia_css_frame_info vf_output_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	/** Info about viewfinder output resolution (optional). This contains
-	     the stride that should be used for memory allocation. */
-	struct ia_css_frame_info raw_output_info;
-	/** Raw output resolution. This indicates the resolution of the
-	     RAW bayer output for pipes that support this. Currently, only the
-	     still capture pipes support this feature. When this resolution is
-	     smaller than the input resolution, cropping will be performed by
-	     the ISP. The first cropping that will be performed is on the upper
-	     left corner where we crop 8 lines and 8 columns to remove the
-	     pixels normally used to initialize the ISP filters.
-	     This is why the raw output resolution should normally be set to
-	     the input resolution - 8x8. */
-#ifdef ISP2401
-	struct ia_css_resolution output_system_in_res_info;
-	/** For IPU3 only. Info about output system in resolution which is considered
-	     as gdc out resolution. */
-#endif
-	struct ia_css_shading_info shading_info;
-	/** After an image pipe is created, this field will contain the info
-	     for the shading correction. */
-	struct ia_css_grid_info  grid_info;
-	/** After an image pipe is created, this field will contain the grid
-	     info for 3A and DVS. */
-	int num_invalid_frames;
-	/** The very first frames in a started stream do not contain valid data.
-	     In this field, the CSS-firmware communicates to the host-driver how
-	     many initial frames will contain invalid data; this allows the
-	     host-driver to discard those initial invalid frames and start it's
-	     output at the first valid frame. */
-};
-
-/**
- * Defaults for ia_css_pipe_info structs.
- */
-#define DEFAULT_PIPE_INFO \
-(struct ia_css_pipe_info) { \
-	.output_info		= {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \
-	.vf_output_info		= {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \
-	.raw_output_info	= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
-	.shading_info		= DEFAULT_SHADING_INFO, \
-	.grid_info		= DEFAULT_GRID_INFO, \
-}
-
-/* @brief Load default pipe configuration
- * @param[out]	pipe_config The pipe configuration.
- * @return	None
- *
- * This function will load the default pipe configuration:
-@code
-	struct ia_css_pipe_config def_config = {
-		IA_CSS_PIPE_MODE_PREVIEW,  // mode
-		1,      // isp_pipe_version
-		{0, 0}, // bayer_ds_out_res
-		{0, 0}, // capt_pp_in_res
-		{0, 0}, // vf_pp_in_res
-		{0, 0}, // dvs_crop_out_res
-		{{0, 0}, 0, 0, 0, 0}, // output_info
-		{{0, 0}, 0, 0, 0, 0}, // second_output_info
-		{{0, 0}, 0, 0, 0, 0}, // vf_output_info
-		{{0, 0}, 0, 0, 0, 0}, // second_vf_output_info
-		NULL,   // acc_extension
-		NULL,   // acc_stages
-		0,      // num_acc_stages
-		{
-			IA_CSS_CAPTURE_MODE_RAW, // mode
-			false, // enable_xnr
-			false  // enable_raw_output
-		},      // default_capture_config
-		{0, 0}, // dvs_envelope
-		1,      // dvs_frame_delay
-		-1,     // acc_num_execs
-		true,   // enable_dz
-		NULL,   // p_isp_config
-	};
-@endcode
- */
-void ia_css_pipe_config_defaults(struct ia_css_pipe_config *pipe_config);
-
-/* @brief Create a pipe
- * @param[in]	config The pipe configuration.
- * @param[out]	pipe The pipe.
- * @return	IA_CSS_SUCCESS or the error code.
- *
- * This function will create a pipe with the given
- * configuration.
- */
-enum ia_css_err
-ia_css_pipe_create(const struct ia_css_pipe_config *config,
-		   struct ia_css_pipe **pipe);
-
-/* @brief Destroy a pipe
- * @param[in]	pipe The pipe.
- * @return	IA_CSS_SUCCESS or the error code.
- *
- * This function will destroy a given pipe.
- */
-enum ia_css_err
-ia_css_pipe_destroy(struct ia_css_pipe *pipe);
-
-/* @brief Provides information about a pipe
- * @param[in]	pipe The pipe.
- * @param[out]	pipe_info The pipe information.
- * @return	IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS.
- *
- * This function will provide information about a given pipe.
- */
-enum ia_css_err
-ia_css_pipe_get_info(const struct ia_css_pipe *pipe,
-		     struct ia_css_pipe_info *pipe_info);
-
-/* @brief Configure a pipe with filter coefficients.
- * @param[in]	pipe	The pipe.
- * @param[in]	config	The pointer to ISP configuration.
- * @return		IA_CSS_SUCCESS or error code upon error.
- *
- * This function configures the filter coefficients for an image
- * pipe.
- */
-enum ia_css_err
-ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe,
-						   struct ia_css_isp_config *config);
-
-/* @brief Controls when the Event generator raises an IRQ to the Host.
- *
- * @param[in]	pipe	The pipe.
- * @param[in]	or_mask	Binary or of enum ia_css_event_irq_mask_type. Each pipe
-			related event that is part of this mask will directly
-			raise an IRQ to	the Host when the event occurs in the
-			CSS.
- * @param[in]	and_mask Binary or of enum ia_css_event_irq_mask_type. An event
-			IRQ for the Host is only raised after all pipe related
-			events have occurred at least once for all the active
-			pipes. Events are remembered and don't need to occure
-			at the same moment in time. There is no control over
-			the order of these events. Once an IRQ has been raised
-			all remembered events are reset.
- * @return		IA_CSS_SUCCESS.
- *
- Controls when the Event generator in the CSS raises an IRQ to the Host.
- The main purpose of this function is to reduce the amount of interrupts
- between the CSS and the Host. This will help saving power as it wakes up the
- Host less often. In case both or_mask and and_mask are
- IA_CSS_EVENT_TYPE_NONE for all pipes, no event IRQ's will be raised. An
- exception holds for IA_CSS_EVENT_TYPE_PORT_EOF, for this event an IRQ is always
- raised.
- Note that events are still queued and the Host can poll for them. The
- or_mask and and_mask may be active at the same time\n
- \n
- Default values, for all pipe id's, after ia_css_init:\n
- or_mask = IA_CSS_EVENT_TYPE_ALL\n
- and_mask = IA_CSS_EVENT_TYPE_NONE\n
- \n
- Examples\n
- \code
- ia_css_pipe_set_irq_mask(h_pipe,
- IA_CSS_EVENT_TYPE_3A_STATISTICS_DONE |
- IA_CSS_EVENT_TYPE_DIS_STATISTICS_DONE ,
- IA_CSS_EVENT_TYPE_NONE);
- \endcode
- The event generator will only raise an interrupt to the Host when there are
- 3A or DIS statistics available from the preview pipe. It will not generate
- an interrupt for any other event of the preview pipe e.g when there is an
- output frame available.
-
- \code
- ia_css_pipe_set_irq_mask(h_pipe_preview,
-	IA_CSS_EVENT_TYPE_NONE,
-	IA_CSS_EVENT_TYPE_OUTPUT_FRAME_DONE |
-	IA_CSS_EVENT_TYPE_3A_STATISTICS_DONE );
-
- ia_css_pipe_set_irq_mask(h_pipe_capture,
-	IA_CSS_EVENT_TYPE_NONE,
-	IA_CSS_EVENT_TYPE_OUTPUT_FRAME_DONE );
- \endcode
- The event generator will only raise an interrupt to the Host when there is
- both a frame done and 3A event available from the preview pipe AND when there
- is a frame done available from the capture pipe. Note that these events
- may occur at different moments in time. Also the order of the events is not
- relevant.
-
- \code
- ia_css_pipe_set_irq_mask(h_pipe_preview,
-	IA_CSS_EVENT_TYPE_OUTPUT_FRAME_DONE,
-	IA_CSS_EVENT_TYPE_ALL );
-
- ia_css_pipe_set_irq_mask(h_pipe_capture,
-	IA_CSS_EVENT_TYPE_OUTPUT_FRAME_DONE,
-	IA_CSS_EVENT_TYPE_ALL );
- \endcode
- The event generator will only raise an interrupt to the Host when there is an
- output frame from the preview pipe OR an output frame from the capture pipe.
- All other events (3A, VF output, pipeline done) will not raise an interrupt
- to the Host. These events are not lost but always stored in the event queue.
- */
-enum ia_css_err
-ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe,
-			 unsigned int or_mask,
-			 unsigned int and_mask);
-
-/* @brief Reads the current event IRQ mask from the CSS.
- *
- * @param[in]	pipe The pipe.
- * @param[out]	or_mask	Current or_mask. The bits in this mask are a binary or
-		of enum ia_css_event_irq_mask_type. Pointer may be NULL.
- * @param[out]	and_mask Current and_mask.The bits in this mask are a binary or
-		of enum ia_css_event_irq_mask_type. Pointer may be NULL.
- * @return	IA_CSS_SUCCESS.
- *
- Reads the current event IRQ mask from the CSS. Reading returns the actual
- values as used by the SP and not any mirrored values stored at the Host.\n
-\n
-Precondition:\n
-SP must be running.\n
-
-*/
-enum ia_css_err
-ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe,
-			  unsigned int *or_mask,
-			  unsigned int *and_mask);
-
-/* @brief Queue a buffer for an image pipe.
- *
- * @param[in] pipe	The pipe that will own the buffer.
- * @param[in] buffer	Pointer to the buffer.
- *			Note that the caller remains owner of the buffer
- *			structure. Only the data pointer within it will
- *			be passed into the internal queues.
- * @return		IA_CSS_INTERNAL_ERROR in case of unexpected errors,
- *			IA_CSS_SUCCESS otherwise.
- *
- * This function adds a buffer (which has a certain buffer type) to the queue
- * for this type. This queue is owned by the image pipe. After this function
- * completes successfully, the buffer is now owned by the image pipe and should
- * no longer be accessed by any other code until it gets dequeued. The image
- * pipe will dequeue buffers from this queue, use them and return them to the
- * host code via an interrupt. Buffers will be consumed in the same order they
- * get queued, but may be returned to the host out of order.
- */
-enum ia_css_err
-ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
-			   const struct ia_css_buffer *buffer);
-
-/* @brief Dequeue a buffer from an image pipe.
- *
- * @param[in]    pipe	 The pipeline that the buffer queue belongs to.
- * @param[in,out] buffer The buffer is used to lookup the type which determines
- *			 which internal queue to use.
- *			 The resulting buffer pointer is written into the dta
- *			 field.
- * @return		 IA_CSS_ERR_NO_BUFFER if the queue is empty or
- *			 IA_CSS_SUCCESS otherwise.
- *
- * This function dequeues a buffer from a buffer queue. The queue is indicated
- * by the buffer type argument. This function can be called after an interrupt
- * has been generated that signalled that a new buffer was available and can
- * be used in a polling-like situation where the NO_BUFFER return value is used
- * to determine whether a buffer was available or not.
- */
-enum ia_css_err
-ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
-			   struct ia_css_buffer *buffer);
-
-
-/* @brief  Set the state (Enable or Disable) of the Extension stage in the
- *          given pipe.
- * @param[in] pipe         Pipe handle.
- * @param[in] fw_handle    Extension firmware Handle (ia_css_fw_info.handle)
- * @param[in] enable       Enable Flag (1 to enable ; 0 to disable)
- *
- * @return
- * IA_CSS_SUCCESS 			: Success
- * IA_CSS_ERR_INVALID_ARGUMENTS		: Invalid Parameters
- * IA_CSS_ERR_RESOURCE_NOT_AVAILABLE	: Inactive QOS Pipe
- * 					(No active stream with this pipe)
- *
- * This function will request state change (enable or disable) for the Extension
- * stage (firmware handle) in the given pipe.
- *
- * Note:
- * 	1. Extension can be enabled/disabled only on QOS Extensions
- * 	2. Extension can be enabled/disabled only with an active QOS Pipe
- * 	3. Initial(Default) state of QOS Extensions is Disabled
- * 	4. State change cannot be guaranteed immediately OR on frame boundary
- *
- */
-enum ia_css_err
-ia_css_pipe_set_qos_ext_state (struct ia_css_pipe *pipe,
-                           uint32_t fw_handle,
-                           bool  enable);
-
-/* @brief  Get the state (Enable or Disable) of the Extension stage in the
- *          given pipe.
- * @param[in]  pipe        Pipe handle.
- * @param[in]  fw_handle   Extension firmware Handle (ia_css_fw_info.handle)
- * @param[out] *enable     Enable Flag
- *
- * @return
- * IA_CSS_SUCCESS 			: Success
- * IA_CSS_ERR_INVALID_ARGUMENTS		: Invalid Parameters
- * IA_CSS_ERR_RESOURCE_NOT_AVAILABLE	: Inactive QOS Pipe
- * 					(No active stream with this pipe)
- *
- * This function will query the state of the Extension stage (firmware handle)
- * in the given Pipe.
- *
- * Note:
- * 	1. Extension state can be queried only on QOS Extensions
- * 	2. Extension can be enabled/disabled only with an active QOS Pipe
- * 	3. Initial(Default) state of QOS Extensions is Disabled.
- *
- */
-enum ia_css_err
-ia_css_pipe_get_qos_ext_state (struct ia_css_pipe *pipe,
-                           uint32_t fw_handle,
-                           bool * enable);
-
-#ifdef ISP2401
-/* @brief  Update mapped CSS and ISP arguments for QoS pipe during SP runtime.
- * @param[in] pipe     	Pipe handle.
- * @param[in] fw_handle	Extension firmware Handle (ia_css_fw_info.handle).
- * @param[in] css_seg  	Parameter memory descriptors for CSS segments.
- * @param[in] isp_seg  	Parameter memory descriptors for ISP segments.
- *
- * @return
- * IA_CSS_SUCCESS 			: Success
- * IA_CSS_ERR_INVALID_ARGUMENTS		: Invalid Parameters
- * IA_CSS_ERR_RESOURCE_NOT_AVAILABLE	: Inactive QOS Pipe
- * 					(No active stream with this pipe)
- *
- * \deprecated{This interface is used to temporarily support a late-developed,
- * specific use-case on a specific IPU2 platform. It will not be supported or
- * maintained on IPU3 or further.}
- */
-enum ia_css_err
-ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, uint32_t fw_handle,
-			struct ia_css_isp_param_css_segments *css_seg,
-			struct ia_css_isp_param_isp_segments *isp_seg);
-
-#endif
-/* @brief Get selected configuration settings
- * @param[in]	pipe	The pipe.
- * @param[out]	config	Configuration settings.
- * @return		None
- */
-void
-ia_css_pipe_get_isp_config(struct ia_css_pipe *pipe,
-			     struct ia_css_isp_config *config);
-
-/* @brief Set the scaler lut on this pipe. A copy of lut is made in the inuit
- *         address space. So the LUT can be freed by caller.
- * @param[in]  pipe        Pipe handle.
- * @param[in]  lut         Look up tabel
- *
- * @return
- * IA_CSS_SUCCESS 			: Success
- * IA_CSS_ERR_INVALID_ARGUMENTS		: Invalid Parameters
- *
- * Note:
- * 1) Note that both GDC's are programmed with the same table.
- * 2) Current implementation ignores the pipe and overrides the
- *    global lut. This will be fixed in the future
- * 3) This function must be called before stream start
- *
- */
-enum ia_css_err
-ia_css_pipe_set_bci_scaler_lut( struct ia_css_pipe *pipe,
-				const void *lut);
-/* @brief Checking of DVS statistics ability
- * @param[in]	pipe_info	The pipe info.
- * @return		true - has DVS statistics ability
- * 			false - otherwise
- */
-bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info);
-
-#ifdef ISP2401
-/* @brief Override the frameformat set on the output pins.
- * @param[in]  pipe        Pipe handle.
- * @param[in]  output_pin  Pin index to set the format on
- *                         0 - main output pin
- *                         1 - display output pin
- * @param[in]  format      Format to set
- *
- * @return
- * IA_CSS_SUCCESS		: Success
- * IA_CSS_ERR_INVALID_ARGUMENTS	: Invalid Parameters
- * IA_CSS_ERR_INTERNAL_ERROR	: Pipe misses binary info
- *
- * Note:
- * 1) This is an optional function to override the formats set in the pipe.
- * 2) Only overriding with IA_CSS_FRAME_FORMAT_NV12_TILEY is currently allowed.
- * 3) This function is only to be used on pipes that use the output system.
- * 4) If this function is used, it MUST be called after ia_css_pipe_create.
- * 5) If this function is used, this function MUST be called before ia_css_stream_start.
- */
-enum ia_css_err
-ia_css_pipe_override_frame_format(struct ia_css_pipe *pipe,
-				int output_pin,
-				enum ia_css_frame_format format);
-
-#endif
-#endif /* __IA_CSS_PIPE_PUBLIC_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_prbs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_prbs.h
deleted file mode 100644
index 6f24656b6cb4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_prbs.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_PRBS_H
-#define __IA_CSS_PRBS_H
-
-/* @file
- * This file contains support for Pseudo Random Bit Sequence (PRBS) inputs
- */
-
-/* Enumerate the PRBS IDs.
- */
-enum ia_css_prbs_id {
-	IA_CSS_PRBS_ID0,
-	IA_CSS_PRBS_ID1,
-	IA_CSS_PRBS_ID2
-};
-
-/**
- * Maximum number of PRBS IDs.
- *
- * Make sure the value of this define gets changed to reflect the correct
- * number of ia_css_prbs_id enum if you add/delete an item in the enum.
- */
-#define N_CSS_PRBS_IDS (IA_CSS_PRBS_ID2+1)
-
-/**
- * PRBS configuration structure.
- *
- * Seed the for the Pseudo Random Bit Sequence.
- *
- * @deprecated{This interface is deprecated, it is not portable -> move to input system API}
- */
-struct ia_css_prbs_config {
-	enum ia_css_prbs_id	id;
-	unsigned int		h_blank;	/** horizontal blank */
-	unsigned int		v_blank;	/** vertical blank */
-	int			seed;	/** random seed for the 1st 2-pixel-components/clock */
-	int			seed1;	/** random seed for the 2nd 2-pixel-components/clock */
-};
-
-#endif /* __IA_CSS_PRBS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_properties.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_properties.h
deleted file mode 100644
index 9a167306611c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_properties.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_PROPERTIES_H
-#define __IA_CSS_PROPERTIES_H
-
-/* @file
- * This file contains support for retrieving properties of some hardware the CSS system
- */
-
-#include <type_support.h> /* bool */
-#include <ia_css_types.h> /* ia_css_vamem_type */
-
-struct ia_css_properties {
-	int  gdc_coord_one;
-	bool l1_base_is_index; /** Indicate whether the L1 page base
-				    is a page index or a byte address. */
-	enum ia_css_vamem_type vamem_type;
-};
-
-/* @brief Get hardware properties
- * @param[in,out]	properties The hardware properties
- * @return	None
- *
- * This function returns a number of hardware properties.
- */
-void
-ia_css_get_properties(struct ia_css_properties *properties);
-
-#endif /* __IA_CSS_PROPERTIES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_shading.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_shading.h
deleted file mode 100644
index 588f53d32b72..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_shading.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_SHADING_H
-#define __IA_CSS_SHADING_H
-
-/* @file
- * This file contains support for setting the shading table for CSS
- */
-
-#include <ia_css_types.h>
-
-/* @brief Shading table
- * @param[in]	width Width of the shading table.
- * @param[in]	height Height of the shading table.
- * @return		Pointer to the shading table
-*/
-struct ia_css_shading_table *
-ia_css_shading_table_alloc(unsigned int width,
-			   unsigned int height);
-
-/* @brief Free shading table
- * @param[in]	table Pointer to the shading table.
- * @return		None
-*/
-void
-ia_css_shading_table_free(struct ia_css_shading_table *table);
-
-#endif /* __IA_CSS_SHADING_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream.h
deleted file mode 100644
index fb6e8c2ca8bf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream.h
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_STREAM_H_
-#define _IA_CSS_STREAM_H_
-
-#include <type_support.h>
-#include <system_local.h>
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
-#include <input_system.h>
-#endif
-#include "ia_css_types.h"
-#include "ia_css_stream_public.h"
-
-/**
- * structure to hold all internal stream related information
- */
-struct ia_css_stream {
-	struct ia_css_stream_config    config;
-	struct ia_css_stream_info      info;
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
-	rx_cfg_t                       csi_rx_config;
-#endif
-	bool                           reconfigure_css_rx;
-	struct ia_css_pipe            *last_pipe;
-	int                            num_pipes;
-	struct ia_css_pipe           **pipes;
-	struct ia_css_pipe            *continuous_pipe;
-	struct ia_css_isp_parameters  *isp_params_configs;
-	struct ia_css_isp_parameters  *per_frame_isp_params_configs;
-
-	bool                           cont_capt;
-	bool                           disable_cont_vf;
-#ifndef ISP2401
-	bool                           stop_copy_preview;
-#endif
-	bool                           started;
-};
-
-/* @brief Get a binary in the stream, which binary has the shading correction.
- *
- * @param[in] stream: The stream.
- * @return	The binary which has the shading correction.
- *
- */
-struct ia_css_binary *
-ia_css_stream_get_shading_correction_binary(const struct ia_css_stream *stream);
-
-struct ia_css_binary *
-ia_css_stream_get_dvs_binary(const struct ia_css_stream *stream);
-
-struct ia_css_binary *
-ia_css_stream_get_3a_binary(const struct ia_css_stream *stream);
-
-unsigned int
-ia_css_stream_input_format_bits_per_pixel(struct ia_css_stream *stream);
-
-bool
-sh_css_params_set_binning_factor(struct ia_css_stream *stream, unsigned int sensor_binning);
-
-void
-sh_css_invalidate_params(struct ia_css_stream *stream);
-
-/* The following functions are used for testing purposes only */
-const struct ia_css_fpn_table *
-ia_css_get_fpn_table(struct ia_css_stream *stream);
-
-/* @brief Get a pointer to the shading table.
- *
- * @param[in] stream: The stream.
- * @return	The pointer to the shading table.
- *
- */
-struct ia_css_shading_table *
-ia_css_get_shading_table(struct ia_css_stream *stream);
-
-void
-ia_css_get_isp_dis_coefficients(struct ia_css_stream *stream,
-				short *horizontal_coefficients,
-				short *vertical_coefficients);
-
-void
-ia_css_get_isp_dvs2_coefficients(struct ia_css_stream *stream,
-	short *hor_coefs_odd_real,
-	short *hor_coefs_odd_imag,
-	short *hor_coefs_even_real,
-	short *hor_coefs_even_imag,
-	short *ver_coefs_odd_real,
-	short *ver_coefs_odd_imag,
-	short *ver_coefs_even_real,
-	short *ver_coefs_even_imag);
-
-enum ia_css_err
-ia_css_stream_isp_parameters_init(struct ia_css_stream *stream);
-
-void
-ia_css_stream_isp_parameters_uninit(struct ia_css_stream *stream);
-
-#endif /*_IA_CSS_STREAM_H_*/
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream_format.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream_format.h
deleted file mode 100644
index f97b9eb2b19c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream_format.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_STREAM_FORMAT_H
-#define __IA_CSS_STREAM_FORMAT_H
-
-/* @file
- * This file contains formats usable for ISP streaming input
- */
-
-#include <type_support.h> /* bool */
-#include "../../../include/linux/atomisp_platform.h"
-
-unsigned int ia_css_util_input_format_bpp(
-	enum atomisp_input_format format,
-	bool two_ppc);
-
-#endif /* __ATOMISP_INPUT_FORMAT_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream_public.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream_public.h
deleted file mode 100644
index ddefad330db7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_stream_public.h
+++ /dev/null
@@ -1,582 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_STREAM_PUBLIC_H
-#define __IA_CSS_STREAM_PUBLIC_H
-
-/* @file
- * This file contains support for configuring and controlling streams
- */
-
-#include <type_support.h>
-#include "ia_css_types.h"
-#include "ia_css_pipe_public.h"
-#include "ia_css_metadata.h"
-#include "ia_css_tpg.h"
-#include "ia_css_prbs.h"
-#include "ia_css_input_port.h"
-
-/* Input modes, these enumerate all supported input modes.
- *  Note that not all ISP modes support all input modes.
- */
-enum ia_css_input_mode {
-	IA_CSS_INPUT_MODE_SENSOR, /** data from sensor */
-	IA_CSS_INPUT_MODE_FIFO,   /** data from input-fifo */
-	IA_CSS_INPUT_MODE_TPG,    /** data from test-pattern generator */
-	IA_CSS_INPUT_MODE_PRBS,   /** data from pseudo-random bit stream */
-	IA_CSS_INPUT_MODE_MEMORY, /** data from a frame in memory */
-	IA_CSS_INPUT_MODE_BUFFERED_SENSOR /** data is sent through mipi buffer */
-};
-
-/* Structure of the MIPI buffer configuration
- */
-struct ia_css_mipi_buffer_config {
-	unsigned int size_mem_words; /** The frame size in the system memory
-					  words (32B) */
-	bool contiguous;	     /** Allocated memory physically
-					  contiguously or not. \deprecated{Will be false always.}*/
-	unsigned int nof_mipi_buffers; /** The number of MIPI buffers required for this
-					stream */
-};
-
-enum {
-	IA_CSS_STREAM_ISYS_STREAM_0 = 0,
-	IA_CSS_STREAM_DEFAULT_ISYS_STREAM_IDX = IA_CSS_STREAM_ISYS_STREAM_0,
-	IA_CSS_STREAM_ISYS_STREAM_1,
-	IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH
-};
-
-/* This is input data configuration for one MIPI data type. We can have
- *  multiple of this in one virtual channel.
- */
-struct ia_css_stream_isys_stream_config {
-	struct ia_css_resolution  input_res; /** Resolution of input data */
-	enum atomisp_input_format format; /** Format of input stream. This data
-					       format will be mapped to MIPI data
-					       type internally. */
-	int linked_isys_stream_id; /** default value is -1, other value means
-							current isys_stream shares the same buffer with
-							indicated isys_stream*/
-	bool valid; /** indicate whether other fields have valid value */
-};
-
-struct ia_css_stream_input_config {
-	struct ia_css_resolution  input_res; /** Resolution of input data */
-	struct ia_css_resolution  effective_res; /** Resolution of input data.
-							Used for CSS 2400/1 System and deprecated for other
-							systems (replaced by input_effective_res in
-							ia_css_pipe_config) */
-	enum atomisp_input_format format; /** Format of input stream. This data
-					       format will be mapped to MIPI data
-					       type internally. */
-	enum ia_css_bayer_order bayer_order; /** Bayer order for RAW streams */
-};
-
-
-/* Input stream description. This describes how input will flow into the
- *  CSS. This is used to program the CSS hardware.
- */
-struct ia_css_stream_config {
-	enum ia_css_input_mode    mode; /** Input mode */
-	union {
-		struct ia_css_input_port  port; /** Port, for sensor only. */
-		struct ia_css_tpg_config  tpg;  /** TPG configuration */
-		struct ia_css_prbs_config prbs; /** PRBS configuration */
-	} source; /** Source of input data */
-	unsigned int	      channel_id; /** Channel on which input data
-						   will arrive. Use this field
-						   to specify virtual channel id.
-						   Valid values are: 0, 1, 2, 3 */
-	struct ia_css_stream_isys_stream_config isys_config[IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH];
-	struct ia_css_stream_input_config input_config;
-
-#ifdef ISP2401
-	/* Currently, Android and Windows platforms interpret the binning_factor parameter
-	 * differently. In Android, the binning factor is expressed in the form
-	 * 2^N * 2^N, whereas in Windows platform, the binning factor is N*N
-	 * To use the Windows method of specification, the caller has to define
-	 * macro USE_WINDOWS_BINNING_FACTOR. This is for backward compatibility only
-	 * and will be deprecated. In the future,all platforms will use the N*N method
-	 */
-#endif
-	unsigned int sensor_binning_factor; /** Binning factor used by sensor
-						 to produce image data. This is
-						 used for shading correction. */
-	unsigned int pixels_per_clock; /** Number of pixels per clock, which can be
-					    1, 2 or 4. */
-	bool online; /** offline will activate RAW copy on SP, use this for
-			  continuous capture. */
-		/* ISYS2401 usage: ISP receives data directly from sensor, no copy. */
-	unsigned init_num_cont_raw_buf; /** initial number of raw buffers to
-					     allocate */
-	unsigned target_num_cont_raw_buf; /** total number of raw buffers to
-					     allocate */
-	bool pack_raw_pixels; /** Pack pixels in the raw buffers */
-	bool continuous; /** Use SP copy feature to continuously capture frames
-			      to system memory and run pipes in offline mode */
-	bool disable_cont_viewfinder; /** disable continous viewfinder for ZSL use case */
-	int32_t flash_gpio_pin; /** pin on which the flash is connected, -1 for no flash */
-	int left_padding; /** The number of input-formatter left-paddings, -1 for default from binary.*/
-	struct ia_css_mipi_buffer_config mipi_buffer_config; /** mipi buffer configuration */
-	struct ia_css_metadata_config	metadata_config;     /** Metadata configuration. */
-	bool ia_css_enable_raw_buffer_locking; /** Enable Raw Buffer Locking for HALv3 Support */
-	bool lock_all;
-	/** Lock all RAW buffers (true) or lock only buffers processed by
-	     video or preview pipe (false).
-	     This setting needs to be enabled to allow raw buffer locking
-	     without continuous viewfinder. */
-};
-
-struct ia_css_stream;
-
-/* Stream info, this struct describes properties of a stream after it has been
- *  created.
- */
-struct ia_css_stream_info {
-	struct ia_css_metadata_info metadata_info;
-	/** Info about the metadata layout, this contains the stride. */
-};
-
-/* @brief Load default stream configuration
- * @param[in,out]	stream_config The stream configuration.
- * @return	None
- *
- * This function will reset the stream configuration to the default state:
-@code
-	memset(stream_config, 0, sizeof(*stream_config));
-	stream_config->online = true;
-	stream_config->left_padding = -1;
-@endcode
- */
-void ia_css_stream_config_defaults(struct ia_css_stream_config *stream_config);
-
-/*
- * create the internal structures and fill in the configuration data and pipes
- */
-
- /* @brief Creates a stream
- * @param[in]	stream_config The stream configuration.
- * @param[in]	num_pipes The number of pipes to incorporate in the stream.
- * @param[in]	pipes The pipes.
- * @param[out]	stream The stream.
- * @return	IA_CSS_SUCCESS or the error code.
- *
- * This function will create a stream with a given configuration and given pipes.
- */
-enum ia_css_err
-ia_css_stream_create(const struct ia_css_stream_config *stream_config,
-					 int num_pipes,
-					 struct ia_css_pipe *pipes[],
-					 struct ia_css_stream **stream);
-
-/* @brief Destroys a stream
- * @param[in]	stream The stream.
- * @return	IA_CSS_SUCCESS or the error code.
- *
- * This function will destroy a given stream.
- */
-enum ia_css_err
-ia_css_stream_destroy(struct ia_css_stream *stream);
-
-/* @brief Provides information about a stream
- * @param[in]	stream The stream.
- * @param[out]	stream_info The information about the stream.
- * @return	IA_CSS_SUCCESS or the error code.
- *
- * This function will destroy a given stream.
- */
-enum ia_css_err
-ia_css_stream_get_info(const struct ia_css_stream *stream,
-		       struct ia_css_stream_info *stream_info);
-
-/* @brief load (rebuild) a stream that was unloaded.
- * @param[in]	stream The stream
- * @return		IA_CSS_SUCCESS or the error code
- *
- * Rebuild a stream, including allocating structs, setting configuration and
- * building the required pipes.
- */
-enum ia_css_err
-ia_css_stream_load(struct ia_css_stream *stream);
-
-/* @brief Starts the stream.
- * @param[in]	stream The stream.
- * @return IA_CSS_SUCCESS or the error code.
- *
- * The dynamic data in
- * the buffers are not used and need to be queued with a separate call
- * to ia_css_pipe_enqueue_buffer.
- * NOTE: this function will only send start event to corresponding
- * thread and will not start SP any more.
- */
-enum ia_css_err
-ia_css_stream_start(struct ia_css_stream *stream);
-
-/* @brief Stop the stream.
- * @param[in]	stream The stream.
- * @return	IA_CSS_SUCCESS or the error code.
- *
- * NOTE: this function will send stop event to pipes belong to this
- * stream but will not terminate threads.
- */
-enum ia_css_err
-ia_css_stream_stop(struct ia_css_stream *stream);
-
-/* @brief Check if a stream has stopped
- * @param[in]	stream The stream.
- * @return	boolean flag
- *
- * This function will check if the stream has stopped and return the correspondent boolean flag.
- */
-bool
-ia_css_stream_has_stopped(struct ia_css_stream *stream);
-
-/* @brief	destroy a stream according to the stream seed previosly saved in the seed array.
- * @param[in]	stream The stream.
- * @return	IA_CSS_SUCCESS (no other errors are generated now)
- *
- * Destroy the stream and all the pipes related to it.
- */
-enum ia_css_err
-ia_css_stream_unload(struct ia_css_stream *stream);
-
-/* @brief Returns stream format
- * @param[in]	stream The stream.
- * @return	format of the string
- *
- * This function will return the stream format.
- */
-enum atomisp_input_format
-ia_css_stream_get_format(const struct ia_css_stream *stream);
-
-/* @brief Check if the stream is configured for 2 pixels per clock
- * @param[in]	stream The stream.
- * @return	boolean flag
- *
- * This function will check if the stream is configured for 2 pixels per clock and
- * return the correspondent boolean flag.
- */
-bool
-ia_css_stream_get_two_pixels_per_clock(const struct ia_css_stream *stream);
-
-/* @brief Sets the output frame stride (at the last pipe)
- * @param[in]	stream The stream
- * @param[in]	output_padded_width - the output buffer stride.
- * @return	ia_css_err
- *
- * This function will Set the output frame stride (at the last pipe)
- */
-enum ia_css_err
-ia_css_stream_set_output_padded_width(struct ia_css_stream *stream, unsigned int output_padded_width);
-
-/* @brief Return max number of continuous RAW frames.
- * @param[in]	stream The stream.
- * @param[out]	buffer_depth The maximum number of continuous RAW frames.
- * @return	IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS
- *
- * This function will return the maximum number of continuous RAW frames
- * the system can support.
- */
-enum ia_css_err
-ia_css_stream_get_max_buffer_depth(struct ia_css_stream *stream, int *buffer_depth);
-
-/* @brief Set nr of continuous RAW frames to use.
- *
- * @param[in]	stream The stream.
- * @param[in]	buffer_depth	Number of frames to set.
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
- * Set the number of continuous frames to use during continuous modes.
- */
-enum ia_css_err
-ia_css_stream_set_buffer_depth(struct ia_css_stream *stream, int buffer_depth);
-
-/* @brief Get number of continuous RAW frames to use.
- * @param[in]	stream The stream.
- * @param[out]	buffer_depth The number of frames to use
- * @return	IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS
- *
- * Get the currently set number of continuous frames
- * to use during continuous modes.
- */
-enum ia_css_err
-ia_css_stream_get_buffer_depth(struct ia_css_stream *stream, int *buffer_depth);
-
-/* ===== CAPTURE ===== */
-
-/* @brief Configure the continuous capture
- *
- * @param[in]	stream		The stream.
- * @param[in]	num_captures	The number of RAW frames to be processed to
- *				YUV. Setting this to -1 will make continuous
- *				capture run until it is stopped.
- *				This number will also be used to allocate RAW
- *				buffers. To allow the viewfinder to also
- *				keep operating, 2 extra buffers will always be
- *				allocated.
- *				If the offset is negative and the skip setting
- *				is greater than 0, additional buffers may be
- *				needed.
- * @param[in]	skip		Skip N frames in between captures. This can be
- *				used to select a slower capture frame rate than
- *				the sensor output frame rate.
- * @param[in]	offset		Start the RAW-to-YUV processing at RAW buffer
- *				with this offset. This allows the user to
- *				process RAW frames that were captured in the
- *				past or future.
- * @return			IA_CSS_SUCCESS or error code upon error.
- *
- *  For example, to capture the current frame plus the 2 previous
- *  frames and 2 subsequent frames, you would call
- *  ia_css_stream_capture(5, 0, -2).
- */
-enum ia_css_err
-ia_css_stream_capture(struct ia_css_stream *stream,
-			int num_captures,
-			unsigned int skip,
-			int offset);
-
-/* @brief Specify which raw frame to tag based on exp_id found in frame info
- *
- * @param[in]	stream The stream.
- * @param[in]	exp_id	The exposure id of the raw frame to tag.
- *
- * @return			IA_CSS_SUCCESS or error code upon error.
- *
- * This function allows the user to tag a raw frame based on the exposure id
- * found in the viewfinder frames' frame info.
- */
-enum ia_css_err
-ia_css_stream_capture_frame(struct ia_css_stream *stream,
-			unsigned int exp_id);
-
-/* ===== VIDEO ===== */
-
-/* @brief Send streaming data into the css input FIFO
- *
- * @param[in]	stream	The stream.
- * @param[in]	data	Pointer to the pixels to be send.
- * @param[in]	width	Width of the input frame.
- * @param[in]	height	Height of the input frame.
- * @return	None
- *
- * Send streaming data into the css input FIFO. This is for testing purposes
- * only. This uses the channel ID and input format as set by the user with
- * the regular functions for this.
- * This function blocks until the entire frame has been written into the
- * input FIFO.
- *
- * Note:
- * For higher flexibility the ia_css_stream_send_input_frame is replaced by
- * three separate functions:
- * 1) ia_css_stream_start_input_frame
- * 2) ia_css_stream_send_input_line
- * 3) ia_css_stream_end_input_frame
- * In this way it is possible to stream multiple frames on different
- * channel ID's on a line basis. It will be possible to simulate
- * line-interleaved Stereo 3D muxed on 1 mipi port.
- * These 3 functions are for testing purpose only and can be used in
- * conjunction with ia_css_stream_send_input_frame
- */
-void
-ia_css_stream_send_input_frame(const struct ia_css_stream *stream,
-			       const unsigned short *data,
-			       unsigned int width,
-			       unsigned int height);
-
-/* @brief Start an input frame on the CSS input FIFO.
- *
- * @param[in]	stream The stream.
- * @return	None
- *
- * Starts the streaming to mipi frame by sending SoF for channel channel_id.
- * It will use the input_format and two_pixels_per_clock as provided by
- * the user.
- * For the "correct" use-case, input_format and two_pixels_per_clock must match
- * with the values as set by the user with the regular functions.
- * To simulate an error, the user can provide "incorrect" values for
- * input_format and/or two_pixels_per_clock.
- */
-void
-ia_css_stream_start_input_frame(const struct ia_css_stream *stream);
-
-/* @brief Send a line of input data into the CSS input FIFO.
- *
- * @param[in]	stream		The stream.
- * @param[in]	data	Array of the first line of image data.
- * @param	width	The width (in pixels) of the first line.
- * @param[in]	data2	Array of the second line of image data.
- * @param	width2	The width (in pixels) of the second line.
- * @return	None
- *
- * Sends 1 frame line. Start with SoL followed by width bytes of data, followed
- * by width2 bytes of data2 and followed by and EoL
- * It will use the input_format and two_pixels_per_clock settings as provided
- * with the ia_css_stream_start_input_frame function call.
- *
- * This function blocks until the entire line has been written into the
- * input FIFO.
- */
-void
-ia_css_stream_send_input_line(const struct ia_css_stream *stream,
-			      const unsigned short *data,
-			      unsigned int width,
-			      const unsigned short *data2,
-			      unsigned int width2);
-
-/* @brief Send a line of input embedded data into the CSS input FIFO.
- *
- * @param[in]	stream     Pointer of the stream.
- * @param[in]	format     Format of the embedded data.
- * @param[in]	data       Pointer of the embedded data line.
- * @param[in]	width      The width (in pixels) of the line.
- * @return		None
- *
- * Sends one embedded data line to input fifo. Start with SoL followed by
- * width bytes of data, and followed by and EoL.
- * It will use the two_pixels_per_clock settings as provided with the
- * ia_css_stream_start_input_frame function call.
- *
- * This function blocks until the entire line has been written into the
- * input FIFO.
- */
-void
-ia_css_stream_send_input_embedded_line(const struct ia_css_stream *stream,
-			      enum atomisp_input_format format,
-			      const unsigned short *data,
-			      unsigned int width);
-
-/* @brief End an input frame on the CSS input FIFO.
- *
- * @param[in]	stream	The stream.
- * @return	None
- *
- * Send the end-of-frame signal into the CSS input FIFO.
- */
-void
-ia_css_stream_end_input_frame(const struct ia_css_stream *stream);
-
-/* @brief send a request flash command to SP
- *
- * @param[in]	stream The stream.
- * @return	None
- *
- * Driver needs to call this function to send a flash request command
- * to SP, SP will be responsible for switching on/off the flash at proper
- * time. Due to the SP multi-threading environment, this request may have
- * one-frame delay, the driver needs to check the flashed flag in frame info
- * to determine which frame is being flashed.
- */
-void
-ia_css_stream_request_flash(struct ia_css_stream *stream);
-
-/* @brief Configure a stream with filter coefficients.
- *  	   @deprecated {Replaced by
- *  				   ia_css_pipe_set_isp_config_on_pipe()}
- *
- * @param[in]	stream The stream.
- * @param[in]	config	The set of filter coefficients.
- * @param[in]   pipe Pipe to be updated when set isp config, NULL means to
- *		   update all pipes in the stream.
- * @return		IA_CSS_SUCCESS or error code upon error.
- *
- * This function configures the filter coefficients for an image
- * stream. For image pipes that do not execute any ISP filters, this
- * function will have no effect.
- * It is safe to call this function while the image stream is running,
- * in fact this is the expected behavior most of the time. Proper
- * resource locking and double buffering is in place to allow for this.
- */
-enum ia_css_err
-ia_css_stream_set_isp_config_on_pipe(struct ia_css_stream *stream,
-			     const struct ia_css_isp_config *config,
-			     struct ia_css_pipe *pipe);
-
-/* @brief Configure a stream with filter coefficients.
- *  	   @deprecated {Replaced by
- *  				   ia_css_pipe_set_isp_config()}
- * @param[in]	stream	The stream.
- * @param[in]	config	The set of filter coefficients.
- * @return		IA_CSS_SUCCESS or error code upon error.
- *
- * This function configures the filter coefficients for an image
- * stream. For image pipes that do not execute any ISP filters, this
- * function will have no effect. All pipes of a stream will be updated.
- * See ::ia_css_stream_set_isp_config_on_pipe() for the per-pipe alternative.
- * It is safe to call this function while the image stream is running,
- * in fact this is the expected behaviour most of the time. Proper
- * resource locking and double buffering is in place to allow for this.
- */
-enum ia_css_err
-ia_css_stream_set_isp_config(
-	struct ia_css_stream *stream,
-	const struct ia_css_isp_config *config);
-
-/* @brief Get selected configuration settings
- * @param[in]	stream	The stream.
- * @param[out]	config	Configuration settings.
- * @return		None
- */
-void
-ia_css_stream_get_isp_config(const struct ia_css_stream *stream,
-			     struct ia_css_isp_config *config);
-
-/* @brief allocate continuous raw frames for continuous capture
- * @param[in]	stream The stream.
- * @return IA_CSS_SUCCESS or error code.
- *
- *  because this allocation takes a long time (around 120ms per frame),
- *  we separate the allocation part and update part to let driver call
- *  this function without locking. This function is the allocation part
- *  and next one is update part
- */
-enum ia_css_err
-ia_css_alloc_continuous_frame_remain(struct ia_css_stream *stream);
-
-/* @brief allocate continuous raw frames for continuous capture
- * @param[in]	stream The stream.
- * @return	IA_CSS_SUCCESS or error code.
- *
- *  because this allocation takes a long time (around 120ms per frame),
- *  we separate the allocation part and update part to let driver call
- *  this function without locking. This function is the update part
- */
-enum ia_css_err
-ia_css_update_continuous_frames(struct ia_css_stream *stream);
-
-/* @brief ia_css_unlock_raw_frame . unlock a raw frame (HALv3 Support)
- * @param[in]	stream The stream.
- * @param[in]   exp_id exposure id that uniquely identifies the locked Raw Frame Buffer
- * @return      ia_css_err IA_CSS_SUCCESS or error code
- *
- * As part of HALv3 Feature requirement, SP locks raw buffer until the Application
- * releases its reference to a raw buffer (which are managed by SP), this function allows
- * application to explicitly unlock that buffer in SP.
- */
-enum ia_css_err
-ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id);
-
-/* @brief ia_css_en_dz_capt_pipe . Enable/Disable digital zoom for capture pipe
- * @param[in]   stream The stream.
- * @param[in]   enable - true, disable - false
- * @return      None
- *
- * Enables or disables digital zoom for capture pipe in provided stream, if capture pipe
- * exists. This function sets enable_zoom flag in CAPTURE_PP stage of the capture pipe.
- * In process_zoom_and_motion(), decision to enable or disable zoom for every stage depends
- * on this flag.
- */
-void
-ia_css_en_dz_capt_pipe(struct ia_css_stream *stream, bool enable);
-#endif /* __IA_CSS_STREAM_PUBLIC_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_timer.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_timer.h
deleted file mode 100644
index b256d7c88716..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_timer.h
+++ /dev/null
@@ -1,84 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_TIMER_H
-#define __IA_CSS_TIMER_H
-
-/* @file
- * Timer interface definitions
- */
-#include <type_support.h>		/* for uint32_t */
-#include "ia_css_err.h"
-
-/* @brief timer reading definition */
-typedef uint32_t clock_value_t;
-
-/* @brief 32 bit clock tick,(timestamp based on timer-value of CSS-internal timer)*/
-struct ia_css_clock_tick {
-	clock_value_t ticks; /** measured time in ticks.*/
-};
-
-/* @brief TIMER event codes */
-enum ia_css_tm_event {
-	IA_CSS_TM_EVENT_AFTER_INIT,
-	/** Timer Event after Initialization */
-	IA_CSS_TM_EVENT_MAIN_END,
-	/** Timer Event after end of Main */
-	IA_CSS_TM_EVENT_THREAD_START,
-	/** Timer Event after thread start */
-	IA_CSS_TM_EVENT_FRAME_PROC_START,
-	/** Timer Event after Frame Process Start */
-	IA_CSS_TM_EVENT_FRAME_PROC_END
-	/** Timer Event after Frame Process End */
-};
-
-/* @brief code measurement common struct */
-struct ia_css_time_meas {
-	clock_value_t	start_timer_value;	/** measured time in ticks */
-	clock_value_t	end_timer_value;	/** measured time in ticks */
-};
-
-/**@brief SIZE_OF_IA_CSS_CLOCK_TICK_STRUCT checks to ensure correct alignment for struct ia_css_clock_tick. */
-#define SIZE_OF_IA_CSS_CLOCK_TICK_STRUCT sizeof(clock_value_t)
-/* @brief checks to ensure correct alignment for ia_css_time_meas. */
-#define SIZE_OF_IA_CSS_TIME_MEAS_STRUCT (sizeof(clock_value_t) \
-					+ sizeof(clock_value_t))
-
-/* @brief API to fetch timer count directly
-*
-* @param curr_ts [out] measured count value
-* @return IA_CSS_SUCCESS if success
-*
-*/
-enum ia_css_err
-ia_css_timer_get_current_tick(
-	struct ia_css_clock_tick *curr_ts);
-
-#endif  /* __IA_CSS_TIMER_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_tpg.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_tpg.h
deleted file mode 100644
index 81498bd7485b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_tpg.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_TPG_H
-#define __IA_CSS_TPG_H
-
-/* @file
- * This file contains support for the test pattern generator (TPG)
- */
-
-/* Enumerate the TPG IDs.
- */
-enum ia_css_tpg_id {
-	IA_CSS_TPG_ID0,
-	IA_CSS_TPG_ID1,
-	IA_CSS_TPG_ID2
-};
-
-/**
- * Maximum number of TPG IDs.
- *
- * Make sure the value of this define gets changed to reflect the correct
- * number of ia_css_tpg_id enum if you add/delete an item in the enum.
- */
-#define N_CSS_TPG_IDS (IA_CSS_TPG_ID2+1)
-
-/* Enumerate the TPG modes.
- */
-enum ia_css_tpg_mode {
-	IA_CSS_TPG_MODE_RAMP,
-	IA_CSS_TPG_MODE_CHECKERBOARD,
-	IA_CSS_TPG_MODE_FRAME_BASED_COLOR,
-	IA_CSS_TPG_MODE_MONO
-};
-
-/* @brief Configure the test pattern generator.
- *
- * Configure the Test Pattern Generator, the way these values are used to
- * generate the pattern can be seen in the HRT extension for the test pattern
- * generator:
- * devices/test_pat_gen/hrt/include/test_pat_gen.h: hrt_calc_tpg_data().
- *
- * This interface is deprecated, it is not portable -> move to input system API
- *
-@code
-unsigned int test_pattern_value(unsigned int x, unsigned int y)
-{
- unsigned int x_val, y_val;
- if (x_delta > 0) (x_val = (x << x_delta) & x_mask;
- else (x_val = (x >> -x_delta) & x_mask;
- if (y_delta > 0) (y_val = (y << y_delta) & y_mask;
- else (y_val = (y >> -y_delta) & x_mask;
- return (x_val + y_val) & xy_mask;
-}
-@endcode
- */
-struct ia_css_tpg_config {
-	enum ia_css_tpg_id   id;
-	enum ia_css_tpg_mode mode;
-	unsigned int         x_mask;
-	int                  x_delta;
-	unsigned int         y_mask;
-	int                  y_delta;
-	unsigned int         xy_mask;
-};
-
-#endif /* __IA_CSS_TPG_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_types.h
deleted file mode 100644
index 259ab3f074ba..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_types.h
+++ /dev/null
@@ -1,616 +0,0 @@
-/* Release Version: irci_stable_candrpv_0415_20150521_0458 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_TYPES_H
-#define _IA_CSS_TYPES_H
-
-/* @file
- * This file contains types used for the ia_css parameters.
- * These types are in a separate file because they are expected
- * to be used in software layers that do not access the CSS API
- * directly but still need to forward parameters for it.
- */
-
-#include <type_support.h>
-
-#include "ia_css_frac.h"
-
-#include "isp/kernels/aa/aa_2/ia_css_aa2_types.h"
-#include "isp/kernels/anr/anr_1.0/ia_css_anr_types.h"
-#include "isp/kernels/anr/anr_2/ia_css_anr2_types.h"
-#include "isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h"
-#include "isp/kernels/csc/csc_1.0/ia_css_csc_types.h"
-#include "isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h"
-#include "isp/kernels/dp/dp_1.0/ia_css_dp_types.h"
-#include "isp/kernels/de/de_1.0/ia_css_de_types.h"
-#include "isp/kernels/de/de_2/ia_css_de2_types.h"
-#include "isp/kernels/fc/fc_1.0/ia_css_formats_types.h"
-#include "isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h"
-#include "isp/kernels/gc/gc_1.0/ia_css_gc_types.h"
-#include "isp/kernels/gc/gc_2/ia_css_gc2_types.h"
-#include "isp/kernels/macc/macc_1.0/ia_css_macc_types.h"
-#include "isp/kernels/ob/ob_1.0/ia_css_ob_types.h"
-#include "isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h"
-#include "isp/kernels/sc/sc_1.0/ia_css_sc_types.h"
-#include "isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h"
-#include "isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h"
-#include "isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h"
-#include "isp/kernels/wb/wb_1.0/ia_css_wb_types.h"
-#include "isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h"
-#include "isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h"
-#ifdef ISP2401
-#include "isp/kernels/tnr/tnr3/ia_css_tnr3_types.h"
-#endif
-#include "isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h"
-#include "isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h"
-#include "isp/kernels/output/output_1.0/ia_css_output_types.h"
-
-#define IA_CSS_DVS_STAT_GRID_INFO_SUPPORTED
-/** Should be removed after Driver adaptation will be done */
-
-#define IA_CSS_VERSION_MAJOR    2
-#define IA_CSS_VERSION_MINOR    0
-#define IA_CSS_VERSION_REVISION 2
-
-#define IA_CSS_MORPH_TABLE_NUM_PLANES  6
-
-/* Min and max exposure IDs. These macros are here to allow
- * the drivers to get this information. Changing these macros
- * constitutes a CSS API change. */
-#define IA_CSS_ISYS_MIN_EXPOSURE_ID 1   /** Minimum exposure ID */
-#define IA_CSS_ISYS_MAX_EXPOSURE_ID 250 /** Maximum exposure ID */
-
-/* opaque types */
-struct ia_css_isp_parameters;
-struct ia_css_pipe;
-struct ia_css_memory_offsets;
-struct ia_css_config_memory_offsets;
-struct ia_css_state_memory_offsets;
-
-/* Virtual address within the CSS address space. */
-typedef uint32_t ia_css_ptr;
-
-/* Generic resolution structure.
- */
-struct ia_css_resolution {
-	uint32_t width;  /** Width */
-	uint32_t height; /** Height */
-};
-
-/* Generic coordinate structure.
- */
-struct ia_css_coordinate {
-	int32_t x;	/** Value of a coordinate on the horizontal axis */
-	int32_t y;	/** Value of a coordinate on the vertical axis */
-};
-
-/* Vector with signed values. This is used to indicate motion for
- * Digital Image Stabilization.
- */
-struct ia_css_vector {
-	int32_t x; /** horizontal motion (in pixels) */
-	int32_t y; /** vertical motion (in pixels) */
-};
-
-/* Short hands */
-#define IA_CSS_ISP_DMEM IA_CSS_ISP_DMEM0
-#define IA_CSS_ISP_VMEM IA_CSS_ISP_VMEM0
-
-/* CSS data descriptor */
-struct ia_css_data {
-	ia_css_ptr address; /** CSS virtual address */
-	uint32_t   size;    /** Disabled if 0 */
-};
-
-/* Host data descriptor */
-struct ia_css_host_data {
-	char      *address; /** Host address */
-	uint32_t   size;    /** Disabled if 0 */
-};
-
-/* ISP data descriptor */
-struct ia_css_isp_data {
-	uint32_t   address; /** ISP address */
-	uint32_t   size;    /** Disabled if 0 */
-};
-
-/* Shading Correction types. */
-enum ia_css_shading_correction_type {
-#ifndef ISP2401
-	IA_CSS_SHADING_CORRECTION_TYPE_1 /** Shading Correction 1.0 (pipe 1.0 on ISP2300, pipe 2.2 on ISP2400) */
-#else
-	IA_CSS_SHADING_CORRECTION_NONE,	 /** Shading Correction is not processed in the pipe. */
-	IA_CSS_SHADING_CORRECTION_TYPE_1 /** Shading Correction 1.0 (pipe 1.0 on ISP2300, pipe 2.2 on ISP2400/2401) */
-#endif
-
-	/** More shading correction types can be added in the future. */
-};
-
-/* Shading Correction information. */
-struct ia_css_shading_info {
-	enum ia_css_shading_correction_type type; /** Shading Correction type. */
-
-	union {	/* Shading Correction information of each Shading Correction types. */
-
-		/* Shading Correction information of IA_CSS_SHADING_CORRECTION_TYPE_1.
-		 *
-		 *  This structure contains the information necessary to generate
-		 *  the shading table required in the isp.
-		 *  This structure is filled in the css,
-		 *  and the driver needs to get it to generate the shading table.
-		 *
-		 *  Before the shading correction is applied, NxN-filter and/or scaling
-		 *  are applied in the isp, depending on the isp binaries.
-		 *  Then, these should be considered in generating the shading table.
-		 *    - Bad pixels on left/top sides generated by NxN-filter
-		 *      (Bad pixels are NOT considered currently,
-		 *      because they are subtle.)
-		 *    - Down-scaling/Up-scaling factor
-		 *
-		 *  Shading correction is applied to the area
-		 *  which has real sensor data and margin.
-		 *  Then, the shading table should cover the area including margin.
-		 *  This structure has this information.
-		 *    - Origin coordinate of bayer (real sensor data)
-		 *      on the shading table
-		 *
-		 * ------------------------ISP 2401-----------------------
-		 *
-		 *  the shading table directly required from ISP.
-		 *  This structure is filled in CSS, and the driver needs to get it to generate the shading table.
-		 *
-		 *  The shading correction is applied to the bayer area which contains sensor data and padding data.
-		 *  The shading table should cover this bayer area.
-		 *
-		 *  The shading table size directly required from ISP is expressed by these parameters.
-		 *    1. uint32_t num_hor_grids;
-		 *    2. uint32_t num_ver_grids;
-		 *    3. uint32_t bqs_per_grid_cell;
-		 *
-		 *  In some isp binaries, the bayer scaling is applied before the shading correction is applied.
-		 *  Then, this scaling factor should be considered in generating the shading table.
-		 *  The scaling factor is expressed by these parameters.
-		 *    4. uint32_t bayer_scale_hor_ratio_in;
-		 *    5. uint32_t bayer_scale_hor_ratio_out;
-		 *    6. uint32_t bayer_scale_ver_ratio_in;
-		 *    7. uint32_t bayer_scale_ver_ratio_out;
-		 *
-		 *  The sensor data size inputted to ISP is expressed by this parameter.
-		 *  This is the size BEFORE the bayer scaling is applied.
-		 *    8. struct ia_css_resolution isp_input_sensor_data_res_bqs;
-		 *
-		 *  The origin of the sensor data area positioned on the shading table at the shading correction
-		 *  is expressed by this parameter.
-		 *  The size of this area assumes the size AFTER the bayer scaling is applied
-		 *  to the isp_input_sensor_data_resolution_bqs.
-		 *    9. struct ia_css_coordinate sensor_data_origin_bqs_on_sctbl;
-		 *
-		 *  ****** Definitions of the shading table and the sensor data at the shading correction ******
-		 *
-		 * (0,0)--------------------- TW -------------------------------
-		 *   |                                            shading table |
-		 *   |      (ox,oy)---------- W --------------------------      |
-		 *   |        |                               sensor data |     |
-		 *   |        |                                           |     |
-		 *  TH        H             sensor data center            |     |
-		 *   |        |                  (cx,cy)                  |     |
-		 *   |        |                                           |     |
-		 *   |        |                                           |     |
-		 *   |        |                                           |     |
-		 *   |         -------------------------------------------      |
-		 *   |                                                          |
-		 *    ----------------------------------------------------------
-		 *
-		 *    Example of still mode for output 1080p:
-		 *
-		 *    num_hor_grids = 66
-		 *    num_ver_grids = 37
-		 *    bqs_per_grid_cell = 16
-		 *    bayer_scale_hor_ratio_in = 1
-		 *    bayer_scale_hor_ratio_out = 1
-		 *    bayer_scale_ver_ratio_in = 1
-		 *    bayer_scale_ver_ratio_out = 1
-		 *    isp_input_sensor_data_resolution_bqs = {966, 546}
-		 *    sensor_data_origin_bqs_on_sctbl = {61, 15}
-		 *
-		 *    TW, TH [bqs]: width and height of shading table
-		 *        TW = (num_hor_grids - 1) * bqs_per_grid_cell = (66 - 1) * 16 = 1040
-		 *        TH = (num_ver_grids - 1) * bqs_per_grid_cell = (37 - 1) * 16 = 576
-		 *
-		 *    W, H [bqs]: width and height of sensor data at shading correction
-		 *        W = sensor_data_res_bqs.width
-		 *          = isp_input_sensor_data_res_bqs.width
-		 *              * bayer_scale_hor_ratio_out / bayer_scale_hor_ratio_in + 0.5 = 966
-		 *        H = sensor_data_res_bqs.height
-		 *          = isp_input_sensor_data_res_bqs.height
-		 *               * bayer_scale_ver_ratio_out / bayer_scale_ver_ratio_in + 0.5 = 546
-		 *
-		 *    (ox, oy) [bqs]: origin of sensor data positioned on shading table at shading correction
-		 *        ox = sensor_data_origin_bqs_on_sctbl.x = 61
-		 *        oy = sensor_data_origin_bqs_on_sctbl.y = 15
-		 *
-		 *    (cx, cy) [bqs]: center of sensor data positioned on shading table at shading correction
-		 *        cx = ox + W/2 = 61 + 966/2 = 544
-		 *        cy = oy + H/2 = 15 + 546/2 = 288
-		 *
-		 *  ****** Relation between the shading table and the sensor data ******
-		 *
-		 *    The origin of the sensor data should be on the shading table.
-		 *        0 <= ox < TW,  0 <= oy < TH
-		 *
-		 *  ****** How to center the shading table on the sensor data ******
-		 *
-		 *    To center the shading table on the sensor data,
-		 *    CSS decides the shading table size so that a certain grid point is positioned
-		 *    on the center of the sensor data at the shading correction.
-		 *    CSS expects the shading center is set on this grid point
-		 *    when the shading table data is calculated in AIC.
-		 *
-		 *    W, H [bqs]: width and height of sensor data at shading correction
-		 *	W = sensor_data_res_bqs.width
-		 *	H = sensor_data_res_bqs.height
-		 *
-		 *    (cx, cy) [bqs]: center of sensor data positioned on shading table at shading correction
-		 *	cx = sensor_data_origin_bqs_on_sctbl.x + W/2
-		 *	cy = sensor_data_origin_bqs_on_sctbl.y + H/2
-		 *
-		 *    CSS decides the shading table size and the sensor data position
-		 *    so that the (cx, cy) satisfies this condition.
-		 *	mod(cx, bqs_per_grid_cell) = 0
-		 *	mod(cy, bqs_per_grid_cell) = 0
-		 *
-		 *  ****** How to change the sensor data size by processes in the driver and ISP ******
-		 *
-		 *    1. sensor data size: Physical sensor size
-		 *			   (The struct ia_css_shading_info does not have this information.)
-		 *    2. process:          Driver applies the sensor cropping/binning/scaling to physical sensor size.
-		 *    3. sensor data size: ISP input size (== shading_info.isp_input_sensor_data_res_bqs)
-		 *			   (ISP assumes the ISP input sensor data is centered on the physical sensor.)
-		 *    4. process:          ISP applies the bayer scaling by the factor of shading_info.bayer_scale_*.
-		 *    5. sensor data size: Scaling factor * ISP input size (== shading_info.sensor_data_res_bqs)
-		 *    6. process:          ISP applies the shading correction.
-		 *
-		 *  ISP block: SC1
-		 *  ISP1: SC1 is used.
-		 *  ISP2: SC1 is used.
-		 */
-		struct {
-#ifndef ISP2401
-			uint32_t enable;	/** Shading correction enabled.
-						     0:disabled, 1:enabled */
-			uint32_t num_hor_grids;	/** Number of data points per line
-						     per color on shading table. */
-			uint32_t num_ver_grids;	/** Number of lines of data points
-						     per color on shading table. */
-			uint32_t bqs_per_grid_cell; /** Grid cell size
-						in BQ(Bayer Quad) unit.
-						(1BQ means {Gr,R,B,Gb}(2x2 pixels).)
-						Valid values are 8,16,32,64. */
-#else
-			uint32_t num_hor_grids;	/** Number of data points per line per color on shading table. */
-			uint32_t num_ver_grids;	/** Number of lines of data points per color on shading table. */
-			uint32_t bqs_per_grid_cell; /** Grid cell size in BQ unit.
-							 NOTE: bqs = size in BQ(Bayer Quad) unit.
-							       1BQ means {Gr,R,B,Gb} (2x2 pixels).
-							       Horizontal 1 bqs corresponds to horizontal 2 pixels.
-							       Vertical 1 bqs corresponds to vertical 2 pixels. */
-#endif
-			uint32_t bayer_scale_hor_ratio_in;
-			uint32_t bayer_scale_hor_ratio_out;
-#ifndef ISP2401
-			/** Horizontal ratio of bayer scaling
-			between input width and output width, for the scaling
-			which should be done before shading correction.
-			  output_width = input_width * bayer_scale_hor_ratio_out
-						/ bayer_scale_hor_ratio_in */
-#else
-				/** Horizontal ratio of bayer scaling between input width and output width,
-				     for the scaling which should be done before shading correction.
-					output_width = input_width * bayer_scale_hor_ratio_out
-									/ bayer_scale_hor_ratio_in + 0.5 */
-#endif
-			uint32_t bayer_scale_ver_ratio_in;
-			uint32_t bayer_scale_ver_ratio_out;
-#ifndef ISP2401
-			/** Vertical ratio of bayer scaling
-			between input height and output height, for the scaling
-			which should be done before shading correction.
-			  output_height = input_height * bayer_scale_ver_ratio_out
-						/ bayer_scale_ver_ratio_in */
-			uint32_t sc_bayer_origin_x_bqs_on_shading_table;
-			/** X coordinate (in bqs) of bayer origin on shading table.
-			This indicates the left-most pixel of bayer
-			(not include margin) inputted to the shading correction.
-			This corresponds to the left-most pixel of bayer
-			inputted to isp from sensor. */
-			uint32_t sc_bayer_origin_y_bqs_on_shading_table;
-			/** Y coordinate (in bqs) of bayer origin on shading table.
-			This indicates the top pixel of bayer
-			(not include margin) inputted to the shading correction.
-			This corresponds to the top pixel of bayer
-			inputted to isp from sensor. */
-#else
-				/** Vertical ratio of bayer scaling between input height and output height,
-				     for the scaling which should be done before shading correction.
-					output_height = input_height * bayer_scale_ver_ratio_out
-									/ bayer_scale_ver_ratio_in + 0.5 */
-			struct ia_css_resolution isp_input_sensor_data_res_bqs;
-				/** Sensor data size (in bqs) inputted to ISP. This is the size BEFORE bayer scaling.
-				     NOTE: This is NOT the size of the physical sensor size.
-					   CSS requests the driver that ISP inputs sensor data
-					   by the size of isp_input_sensor_data_res_bqs.
-					   The driver sends the sensor data to ISP,
-					   after the adequate cropping/binning/scaling
-					   are applied to the physical sensor data area.
-					   ISP assumes the area of isp_input_sensor_data_res_bqs
-					   is centered on the physical sensor. */
-			struct ia_css_resolution sensor_data_res_bqs;
-				/** Sensor data size (in bqs) at shading correction.
-				     This is the size AFTER bayer scaling. */
-			struct ia_css_coordinate sensor_data_origin_bqs_on_sctbl;
-				/** Origin of sensor data area positioned on shading table at shading correction.
-				     The coordinate x,y should be positive values. */
-#endif
-		} type_1;
-
-		/** More structures can be added here when more shading correction types will be added
-		     in the future. */
-	} info;
-};
-
-/* Default Shading Correction information of Shading Correction Type 1. */
-#define DEFAULT_SHADING_INFO_TYPE_1 \
-(struct ia_css_shading_info) { \
-	.type = IA_CSS_SHADING_CORRECTION_TYPE_1, \
-	.info = { \
-		.type_1 = { \
-			.bayer_scale_hor_ratio_in	= 1, \
-			.bayer_scale_hor_ratio_out	= 1, \
-			.bayer_scale_ver_ratio_in	= 1, \
-			.bayer_scale_ver_ratio_out	= 1, \
-		} \
-	} \
-}
-
-/* Default Shading Correction information. */
-#define DEFAULT_SHADING_INFO	DEFAULT_SHADING_INFO_TYPE_1
-
-/* structure that describes the 3A and DIS grids */
-struct ia_css_grid_info {
-	/* \name ISP input size
-	  * that is visible for user
-	  * @{
-	  */
-	uint32_t isp_in_width;
-	uint32_t isp_in_height;
-	/* @}*/
-
-	struct ia_css_3a_grid_info  s3a_grid; /** 3A grid info */
-	union ia_css_dvs_grid_u dvs_grid;
-		/** All types of DVS statistics grid info union */
-
-	enum ia_css_vamem_type vamem_type;
-};
-
-/* defaults for ia_css_grid_info structs */
-#define DEFAULT_GRID_INFO \
-(struct ia_css_grid_info) { \
-	.dvs_grid	= DEFAULT_DVS_GRID_INFO, \
-	.vamem_type	= IA_CSS_VAMEM_TYPE_1 \
-}
-
-/* Morphing table, used for geometric distortion and chromatic abberration
- *  correction (GDCAC, also called GDC).
- *  This table describes the imperfections introduced by the lens, the
- *  advanced ISP can correct for these imperfections using this table.
- */
-struct ia_css_morph_table {
-	uint32_t enable; /** To disable GDC, set this field to false. The
-			  coordinates fields can be set to NULL in this case. */
-	uint32_t height; /** Table height */
-	uint32_t width;  /** Table width */
-	uint16_t *coordinates_x[IA_CSS_MORPH_TABLE_NUM_PLANES];
-	/** X coordinates that describe the sensor imperfection */
-	uint16_t *coordinates_y[IA_CSS_MORPH_TABLE_NUM_PLANES];
-	/** Y coordinates that describe the sensor imperfection */
-};
-
-struct ia_css_dvs_6axis_config {
-	unsigned int exp_id;
-	/** Exposure ID, see ia_css_event_public.h for more detail */
-	uint32_t width_y;
-	uint32_t height_y;
-	uint32_t width_uv;
-	uint32_t height_uv;
-	uint32_t *xcoords_y;
-	uint32_t *ycoords_y;
-	uint32_t *xcoords_uv;
-	uint32_t *ycoords_uv;
-};
-
-/**
- * This specifies the coordinates (x,y)
- */
-struct ia_css_point {
-	int32_t x; /** x coordinate */
-	int32_t y; /** y coordinate */
-};
-
-/**
- * This specifies the region
- */
-struct ia_css_region {
-	struct ia_css_point origin; /** Starting point coordinates for the region */
-	struct ia_css_resolution resolution; /** Region resolution */
-};
-
-/**
- * Digital zoom:
- * This feature is currently available only for video, but will become
- * available for preview and capture as well.
- * Set the digital zoom factor, this is a logarithmic scale. The actual zoom
- * factor will be 64/x.
- * Setting dx or dy to 0 disables digital zoom for that direction.
- * New API change for Digital zoom:(added struct ia_css_region zoom_region)
- * zoom_region specifies the origin of the zoom region and width and
- * height of that region.
- * origin : This is the coordinate (x,y) within the effective input resolution
- * of the stream. where, x >= 0 and y >= 0. (0,0) maps to the upper left of the
- * effective input resolution.
- * resolution : This is resolution of zoom region.
- * where, x + width <= effective input width
- * y + height <= effective input height
- */
-struct ia_css_dz_config {
-	uint32_t dx; /** Horizontal zoom factor */
-	uint32_t dy; /** Vertical zoom factor */
-	struct ia_css_region zoom_region; /** region for zoom */
-};
-
-/* The still capture mode, this can be RAW (simply copy sensor input to DDR),
- *  Primary ISP, the Advanced ISP (GDC) or the low-light ISP (ANR).
- */
-enum ia_css_capture_mode {
-	IA_CSS_CAPTURE_MODE_RAW,      /** no processing, copy data only */
-	IA_CSS_CAPTURE_MODE_BAYER,    /** bayer processing, up to demosaic */
-	IA_CSS_CAPTURE_MODE_PRIMARY,  /** primary ISP */
-	IA_CSS_CAPTURE_MODE_ADVANCED, /** advanced ISP (GDC) */
-	IA_CSS_CAPTURE_MODE_LOW_LIGHT /** low light ISP (ANR) */
-};
-
-struct ia_css_capture_config {
-	enum ia_css_capture_mode mode; /** Still capture mode */
-	uint32_t enable_xnr;	       /** Enable/disable XNR */
-	uint32_t enable_raw_output;
-	bool enable_capture_pp_bli;    /** Enable capture_pp_bli mode */
-};
-
-/* default settings for ia_css_capture_config structs */
-#define DEFAULT_CAPTURE_CONFIG \
-(struct ia_css_capture_config) { \
-	.mode	= IA_CSS_CAPTURE_MODE_PRIMARY, \
-}
-
-
-/* ISP filter configuration. This is a collection of configurations
- *  for each of the ISP filters (modules).
- *
- *  NOTE! The contents of all pointers is copied when get or set with the
- *  exception of the shading and morph tables. For these we only copy the
- *  pointer, so the caller must make sure the memory contents of these pointers
- *  remain valid as long as they are used by the CSS. This will be fixed in the
- *  future by copying the contents instead of just the pointer.
- *
- *  Comment:
- *    ["ISP block", 1&2]   : ISP block is used both for ISP1 and ISP2.
- *    ["ISP block", 1only] : ISP block is used only for ISP1.
- *    ["ISP block", 2only] : ISP block is used only for ISP2.
- */
-struct ia_css_isp_config {
-	struct ia_css_wb_config   *wb_config;	/** White Balance
-							[WB1, 1&2] */
-	struct ia_css_cc_config   *cc_config;	/** Color Correction
-							[CSC1, 1only] */
-	struct ia_css_tnr_config  *tnr_config;	/** Temporal Noise Reduction
-							[TNR1, 1&2] */
-	struct ia_css_ecd_config  *ecd_config;	/** Eigen Color Demosaicing
-							[DE2, 2only] */
-	struct ia_css_ynr_config  *ynr_config;	/** Y(Luma) Noise Reduction
-							[YNR2&YEE2, 2only] */
-	struct ia_css_fc_config   *fc_config;	/** Fringe Control
-							[FC2, 2only] */
-	struct ia_css_formats_config   *formats_config;	/** Formats Control for main output
-							[FORMATS, 1&2] */
-	struct ia_css_cnr_config  *cnr_config;	/** Chroma Noise Reduction
-							[CNR2, 2only] */
-	struct ia_css_macc_config *macc_config;	/** MACC
-							[MACC2, 2only] */
-	struct ia_css_ctc_config  *ctc_config;	/** Chroma Tone Control
-							[CTC2, 2only] */
-	struct ia_css_aa_config   *aa_config;	/** YUV Anti-Aliasing
-							[AA2, 2only]
-							(not used currently) */
-	struct ia_css_aa_config   *baa_config;	/** Bayer Anti-Aliasing
-							[BAA2, 1&2] */
-	struct ia_css_ce_config   *ce_config;	/** Chroma Enhancement
-							[CE1, 1only] */
-	struct ia_css_dvs_6axis_config *dvs_6axis_config;
-	struct ia_css_ob_config   *ob_config;  /** Objective Black
-							[OB1, 1&2] */
-	struct ia_css_dp_config   *dp_config;  /** Defect Pixel Correction
-							[DPC1/DPC2, 1&2] */
-	struct ia_css_nr_config   *nr_config;  /** Noise Reduction
-							[BNR1&YNR1&CNR1, 1&2]*/
-	struct ia_css_ee_config   *ee_config;  /** Edge Enhancement
-							[YEE1, 1&2] */
-	struct ia_css_de_config   *de_config;  /** Demosaic
-							[DE1, 1only] */
-	struct ia_css_gc_config   *gc_config;  /** Gamma Correction (for YUV)
-							[GC1, 1only] */
-	struct ia_css_anr_config  *anr_config; /** Advanced Noise Reduction */
-	struct ia_css_3a_config   *s3a_config; /** 3A Statistics config */
-	struct ia_css_xnr_config  *xnr_config; /** eXtra Noise Reduction */
-	struct ia_css_dz_config   *dz_config;  /** Digital Zoom */
-	struct ia_css_cc_config *yuv2rgb_cc_config; /** Color Correction
-							[CCM2, 2only] */
-	struct ia_css_cc_config *rgb2yuv_cc_config; /** Color Correction
-							[CSC2, 2only] */
-	struct ia_css_macc_table  *macc_table;	/** MACC
-							[MACC1/MACC2, 1&2]*/
-	struct ia_css_gamma_table *gamma_table;	/** Gamma Correction (for YUV)
-							[GC1, 1only] */
-	struct ia_css_ctc_table   *ctc_table;	/** Chroma Tone Control
-							[CTC1, 1only] */
-
-	/* \deprecated */
-	struct ia_css_xnr_table   *xnr_table;	/** eXtra Noise Reduction
-							[XNR1, 1&2] */
-	struct ia_css_rgb_gamma_table *r_gamma_table;/** sRGB Gamma Correction
-							[GC2, 2only] */
-	struct ia_css_rgb_gamma_table *g_gamma_table;/** sRGB Gamma Correction
-							[GC2, 2only] */
-	struct ia_css_rgb_gamma_table *b_gamma_table;/** sRGB Gamma Correction
-							[GC2, 2only] */
-	struct ia_css_vector      *motion_vector; /** For 2-axis DVS */
-	struct ia_css_shading_table *shading_table;
-	struct ia_css_morph_table   *morph_table;
-	struct ia_css_dvs_coefficients *dvs_coefs; /** DVS 1.0 coefficients */
-	struct ia_css_dvs2_coefficients *dvs2_coefs; /** DVS 2.0 coefficients */
-	struct ia_css_capture_config   *capture_config;
-	struct ia_css_anr_thres   *anr_thres;
-	/* @deprecated{Old shading settings, see bugzilla bz675 for details} */
-	struct ia_css_shading_settings *shading_settings;
-	struct ia_css_xnr3_config *xnr3_config; /** eXtreme Noise Reduction v3 */
-	/* comment from Lasse: Be aware how this feature will affect coordinate
-	 *  normalization in different parts of the system. (e.g. face detection,
-	 *  touch focus, 3A statistics and windows of interest, shading correction,
-	 *  DVS, GDC) from IQ tool level and application level down-to ISP FW level.
-	 *  the risk for regression is not in the individual blocks, but how they
-	 *  integrate together. */
-	struct ia_css_output_config   *output_config;	/** Main Output Mirroring, flipping */
-
-#ifdef ISP2401
-	struct ia_css_tnr3_kernel_config         *tnr3_config;           /** TNR3 config */
-#endif
-	struct ia_css_scaler_config              *scaler_config;         /** Skylake: scaler config (optional) */
-	struct ia_css_formats_config             *formats_config_display;/** Formats control for viewfinder/display output (optional)
-										[OSYS, n/a] */
-	struct ia_css_output_config              *output_config_display; /** Viewfinder/display output mirroring, flipping (optional) */
-
-	struct ia_css_frame                      *output_frame;          /** Output frame the config is to be applied to (optional) */
-	uint32_t			isp_config_id;	/** Unique ID to track which config was actually applied to a particular frame */
-};
-
-#endif /* _IA_CSS_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_version.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_version.h
deleted file mode 100644
index 1e88901e0b82..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_version.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_VERSION_H
-#define __IA_CSS_VERSION_H
-
-/* @file
- * This file contains functions to retrieve CSS-API version information
- */
-
-#include <ia_css_err.h>
-
-/* a common size for the version arrays */
-#define MAX_VERSION_SIZE	500
-
-/* @brief Retrieves the current CSS version
- * @param[out]	version		A pointer to a buffer where to put the generated
- *				version string. NULL is ignored.
- * @param[in]	max_size	Size of the version buffer. If version string
- *				would be larger than max_size, an error is
- *				returned by this function.
- *
- * This function generates and returns the version string. If FW is loaded, it
- * attaches the FW version.
- */
-enum ia_css_err
-ia_css_get_version(char *version, int max_size);
-
-#endif /* __IA_CSS_VERSION_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_version_data.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_version_data.h
deleted file mode 100644
index aad592cb86ef..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/ia_css_version_data.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-//
-// This file contains the version data for the CSS
-//
-// === Do not change - automatically generated ===
-//
-
-#ifndef __IA_CSS_VERSION_DATA_H
-#define __IA_CSS_VERSION_DATA_H
-
-
-#ifndef ISP2401
-#define CSS_VERSION_STRING "REL:20150521_21.4_0539; API:2.1.15.3; GIT:irci_candrpv_0415_20150504_35b345#35b345be52ac575f8934abb3a88fea26a94e7343; SDK:/nfs/iir/disks/iir_hivepackages_003/iir_hivepkgs_disk017/Css_Mizuchi/packages/Css_Mizuchi/int_css_mizuchi_20140829_1053; USER:viedifw; "
-#else
-#define CSS_VERSION_STRING "REL:20150911_37.5_1652; API:2.1.20.9; GIT:irci___#ebf437d53a8951bb7ff6d13fdb7270dab393a92a; SDK:; USER:viedifw; "
-#endif
-
-
-#endif
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.c
deleted file mode 100644
index f7dd256b6f7a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#ifndef IA_CSS_NO_DEBUG
-#include "ia_css_debug.h"
-#endif
-
-#include "ia_css_aa2.host.h"
-
-/* YUV Anti-Aliasing configuration. */
-const struct ia_css_aa_config default_aa_config = {
-	8191 /* default should be 0 */
-};
-
-/* Bayer Anti-Aliasing configuration. */
-const struct ia_css_aa_config default_baa_config = {
-	8191 /* default should be 0 */
-};
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.h
deleted file mode 100644
index 71587d85ff2d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2.host.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_AA_HOST_H
-#define __IA_CSS_AA_HOST_H
-
-#include "ia_css_aa2_types.h"
-#include "ia_css_aa2_param.h"
-
-/* YUV Anti-Aliasing configuration. */
-extern const struct ia_css_aa_config default_aa_config;
-
-/* Bayer Anti-Aliasing configuration. */
-extern const struct ia_css_aa_config default_baa_config;
-
-#endif /* __IA_CSS_AA_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2_param.h
deleted file mode 100644
index dbab4d6c6cd5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2_param.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_AA_PARAM_H
-#define __IA_CSS_AA_PARAM_H
-
-#include "type_support.h"
-
-struct sh_css_isp_aa_params {
-	int32_t strength;
-};
-
-#endif /* __IA_CSS_AA_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2_types.h
deleted file mode 100644
index 0b95bf9b9aaf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/aa/aa_2/ia_css_aa2_types.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_AA2_TYPES_H
-#define __IA_CSS_AA2_TYPES_H
-
-/* @file
-* CSS-API header file for Anti-Aliasing parameters.
-*/
-
-
-/* Anti-Aliasing configuration.
- *
- *  This structure is used both for YUV AA and Bayer AA.
- *
- *  1. YUV Anti-Aliasing
- *     struct ia_css_aa_config   *aa_config
- *
- *     ISP block: AA2
- *    (ISP1: AA2 is not used.)
- *     ISP2: AA2 should be used. But, AA2 is not used currently.
- *
- *  2. Bayer Anti-Aliasing
- *     struct ia_css_aa_config   *baa_config
- *
- *     ISP block: BAA2
- *     ISP1: BAA2 is used.
- *     ISP2: BAA2 is used.
- */
-struct ia_css_aa_config {
-	uint16_t strength;	/** Strength of the filter.
-					u0.13, [0,8191],
-					default/ineffective 0 */
-};
-
-#endif /* __IA_CSS_AA2_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr.host.c
deleted file mode 100644
index edc4f1ae6d5e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr.host.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-
-#include "ia_css_anr.host.h"
-
-const struct ia_css_anr_config default_anr_config = {
-	10,
-	{ 0, 3, 1, 2, 3, 6, 4, 5, 1, 4, 2, 3, 2, 5, 3, 4,
-	  0, 3, 1, 2, 3, 6, 4, 5, 1, 4, 2, 3, 2, 5, 3, 4,
-	  0, 3, 1, 2, 3, 6, 4, 5, 1, 4, 2, 3, 2, 5, 3, 4,
-	  0, 3, 1, 2, 3, 6, 4, 5, 1, 4, 2, 3, 2, 5, 3, 4},
-	{10, 20, 30}
-};
-
-void
-ia_css_anr_encode(
-	struct sh_css_isp_anr_params *to,
-	const struct ia_css_anr_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->threshold = from->threshold;
-}
-
-void
-ia_css_anr_dump(
-	const struct sh_css_isp_anr_params *anr,
-	unsigned level)
-{
-	if (!anr) return;
-	ia_css_debug_dtrace(level, "Advance Noise Reduction:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"anr_threshold", anr->threshold);
-}
-
-void
-ia_css_anr_debug_dtrace(
-	const struct ia_css_anr_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.threshold=%d\n",
-		config->threshold);
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr.host.h
deleted file mode 100644
index 29566c07653c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr.host.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ANR_HOST_H
-#define __IA_CSS_ANR_HOST_H
-
-#include "ia_css_anr_types.h"
-#include "ia_css_anr_param.h"
-
-extern const struct ia_css_anr_config default_anr_config;
-
-void
-ia_css_anr_encode(
-	struct sh_css_isp_anr_params *to,
-	const struct ia_css_anr_config *from,
-	unsigned size);
-
-void
-ia_css_anr_dump(
-	const struct sh_css_isp_anr_params *anr,
-	unsigned level);
-
-void
-ia_css_anr_debug_dtrace(
-	const struct ia_css_anr_config *config, unsigned level)
-;
-
-#endif /* __IA_CSS_ANR_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr_param.h
deleted file mode 100644
index 2621b920c3dc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr_param.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ANR_PARAM_H
-#define __IA_CSS_ANR_PARAM_H
-
-#include "type_support.h"
-
-/* ANR (Advanced Noise Reduction) */
-struct sh_css_isp_anr_params {
-	int32_t threshold;
-};
-
-#endif /* __IA_CSS_ANR_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr_types.h
deleted file mode 100644
index dc317a857369..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_1.0/ia_css_anr_types.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ANR_TYPES_H
-#define __IA_CSS_ANR_TYPES_H
-
-/* @file
-* CSS-API header file for Advanced Noise Reduction kernel v1
-*/
-
-/* Application specific DMA settings  */
-#define ANR_BPP                 10
-#define ANR_ELEMENT_BITS        ((CEIL_DIV(ANR_BPP, 8))*8)
-
-/* Advanced Noise Reduction configuration.
- *  This is also known as Low-Light.
- */
-struct ia_css_anr_config {
-	int32_t threshold; /** Threshold */
-	int32_t thresholds[4*4*4];
-	int32_t factors[3];
-};
-
-#endif /* __IA_CSS_ANR_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2.host.c
deleted file mode 100644
index b338c434453e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2.host.c
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-
-#include "ia_css_anr2.host.h"
-
-void
-ia_css_anr2_vmem_encode(
-	struct ia_css_isp_anr2_params *to,
-	const struct ia_css_anr_thres *from,
-	size_t size)
-{
-	unsigned i;
-
-	(void)size;
-	for (i = 0; i < ANR_PARAM_SIZE; i++) {
-		unsigned j;
-		for (j = 0; j < ISP_VEC_NELEMS; j++) {
-			to->data[i][j] = from->data[i*ISP_VEC_NELEMS+j];
-		}
-	}
-}
-
-void
-ia_css_anr2_debug_dtrace(
-	const struct ia_css_anr_thres *config,
-	unsigned level)
-{
-	(void)config;
-	(void)level;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2.host.h
deleted file mode 100644
index 83c37e328591..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2.host.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ANR2_HOST_H
-#define __IA_CSS_ANR2_HOST_H
-
-#include "sh_css_params.h"
-
-#include "ia_css_anr2_types.h"
-#include "ia_css_anr_param.h"
-#include "ia_css_anr2_table.host.h"
-
-void
-ia_css_anr2_vmem_encode(
-	struct ia_css_isp_anr2_params *to,
-	const struct ia_css_anr_thres *from,
-	size_t size);
-
-void
-ia_css_anr2_debug_dtrace(
-	const struct ia_css_anr_thres *config, unsigned level)
-;
-
-#endif /* __IA_CSS_ANR2_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c
deleted file mode 100644
index 2de51fe45623..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "system_global.h"
-#include "ia_css_types.h"
-#include "ia_css_anr2_table.host.h"
-
-#if 1
-const struct ia_css_anr_thres default_anr_thres = {
-{128, 384, 640, 896, 896, 640, 384, 128, 384, 1152, 1920, 2688, 2688, 1920, 1152, 384, 640, 1920, 3200, 4480, 4480, 3200, 1920, 640, 896, 2688, 4480, 6272, 6272, 4480, 2688, 896, 896, 2688, 4480, 6272, 6272, 4480, 2688, 896, 640, 1920, 3200, 4480, 4480, 3200, 1920, 640, 384, 1152, 1920, 2688, 2688, 1920, 1152, 384, 128, 384, 640, 896, 896, 640, 384, 128,
-0, 0, 30, 30, 10, 10, 20, 20, 0, 0, 30, 30, 10, 10, 20, 20, 0, 0, 30, 30, 10, 10, 20, 20, 0, 0, 30, 30, 10, 10, 20, 20, 0, 0, 30, 30, 10, 10, 20, 20, 0, 0, 30, 30, 10, 10, 20, 20, 0, 0, 30, 30, 10, 10, 20, 20, 0, 0, 30, 30, 10, 10, 20, 20,
-0, 0, 60, 60, 20, 20, 40, 40, 0, 0, 60, 60, 20, 20, 40, 40, 0, 0, 60, 60, 20, 20, 40, 40, 0, 0, 60, 60, 20, 20, 40, 40, 0, 0, 60, 60, 20, 20, 40, 40, 0, 0, 60, 60, 20, 20, 40, 40, 0, 0, 60, 60, 20, 20, 40, 40, 0, 0, 60, 60, 20, 20, 40, 40,
-0, 0, 90, 90, 30, 30, 60, 60, 0, 0, 90, 90, 30, 30, 60, 60, 0, 0, 90, 90, 30, 30, 60, 60, 0, 0, 90, 90, 30, 30, 60, 60, 0, 0, 90, 90, 30, 30, 60, 60, 0, 0, 90, 90, 30, 30, 60, 60, 0, 0, 90, 90, 30, 30, 60, 60, 0, 0, 90, 90, 30, 30, 60, 60,
-30, 30, 60, 60, 40, 40, 50, 50, 30, 30, 60, 60, 40, 40, 50, 50, 30, 30, 60, 60, 40, 40, 50, 50, 30, 30, 60, 60, 40, 40, 50, 50, 30, 30, 60, 60, 40, 40, 50, 50, 30, 30, 60, 60, 40, 40, 50, 50, 30, 30, 60, 60, 40, 40, 50, 50, 30, 30, 60, 60, 40, 40, 50, 50,
-60, 60, 120, 120, 80, 80, 100, 100, 60, 60, 120, 120, 80, 80, 100, 100, 60, 60, 120, 120, 80, 80, 100, 100, 60, 60, 120, 120, 80, 80, 100, 100, 60, 60, 120, 120, 80, 80, 100, 100, 60, 60, 120, 120, 80, 80, 100, 100, 60, 60, 120, 120, 80, 80, 100, 100, 60, 60, 120, 120, 80, 80, 100, 100,
-90, 90, 180, 180, 120, 120, 150, 150, 90, 90, 180, 180, 120, 120, 150, 150, 90, 90, 180, 180, 120, 120, 150, 150, 90, 90, 180, 180, 120, 120, 150, 150, 90, 90, 180, 180, 120, 120, 150, 150, 90, 90, 180, 180, 120, 120, 150, 150, 90, 90, 180, 180, 120, 120, 150, 150, 90, 90, 180, 180, 120, 120, 150, 150,
-10, 10, 40, 40, 20, 20, 30, 30, 10, 10, 40, 40, 20, 20, 30, 30, 10, 10, 40, 40, 20, 20, 30, 30, 10, 10, 40, 40, 20, 20, 30, 30, 10, 10, 40, 40, 20, 20, 30, 30, 10, 10, 40, 40, 20, 20, 30, 30, 10, 10, 40, 40, 20, 20, 30, 30, 10, 10, 40, 40, 20, 20, 30, 30,
-20, 20, 80, 80, 40, 40, 60, 60, 20, 20, 80, 80, 40, 40, 60, 60, 20, 20, 80, 80, 40, 40, 60, 60, 20, 20, 80, 80, 40, 40, 60, 60, 20, 20, 80, 80, 40, 40, 60, 60, 20, 20, 80, 80, 40, 40, 60, 60, 20, 20, 80, 80, 40, 40, 60, 60, 20, 20, 80, 80, 40, 40, 60, 60,
-30, 30, 120, 120, 60, 60, 90, 90, 30, 30, 120, 120, 60, 60, 90, 90, 30, 30, 120, 120, 60, 60, 90, 90, 30, 30, 120, 120, 60, 60, 90, 90, 30, 30, 120, 120, 60, 60, 90, 90, 30, 30, 120, 120, 60, 60, 90, 90, 30, 30, 120, 120, 60, 60, 90, 90, 30, 30, 120, 120, 60, 60, 90, 90,
-20, 20, 50, 50, 30, 30, 40, 40, 20, 20, 50, 50, 30, 30, 40, 40, 20, 20, 50, 50, 30, 30, 40, 40, 20, 20, 50, 50, 30, 30, 40, 40, 20, 20, 50, 50, 30, 30, 40, 40, 20, 20, 50, 50, 30, 30, 40, 40, 20, 20, 50, 50, 30, 30, 40, 40, 20, 20, 50, 50, 30, 30, 40, 40,
-40, 40, 100, 100, 60, 60, 80, 80, 40, 40, 100, 100, 60, 60, 80, 80, 40, 40, 100, 100, 60, 60, 80, 80, 40, 40, 100, 100, 60, 60, 80, 80, 40, 40, 100, 100, 60, 60, 80, 80, 40, 40, 100, 100, 60, 60, 80, 80, 40, 40, 100, 100, 60, 60, 80, 80, 40, 40, 100, 100, 60, 60, 80, 80,
-60, 60, 150, 150, 90, 90, 120, 120, 60, 60, 150, 150, 90, 90, 120, 120, 60, 60, 150, 150, 90, 90, 120, 120, 60, 60, 150, 150, 90, 90, 120, 120, 60, 60, 150, 150, 90, 90, 120, 120, 60, 60, 150, 150, 90, 90, 120, 120, 60, 60, 150, 150, 90, 90, 120, 120, 60, 60, 150, 150, 90, 90, 120, 120}
-};
-#else
-const struct ia_css_anr_thres default_anr_thres = {
-{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
-};
-#endif
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h
deleted file mode 100644
index 534119e064c1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ANR2_TABLE_HOST_H
-#define __IA_CSS_ANR2_TABLE_HOST_H
-
-#include "ia_css_anr2_types.h"
-
-extern const struct ia_css_anr_thres default_anr_thres;
-
-#endif /* __IA_CSS_ANR2_TABLE_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_types.h
deleted file mode 100644
index 9b611315392c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr2_types.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ANR2_TYPES_H
-#define __IA_CSS_ANR2_TYPES_H
-
-/* @file
-* CSS-API header file for Advanced Noise Reduction kernel v2
-*/
-
-#include "type_support.h"
-
-#define ANR_PARAM_SIZE          13
-
-/* Advanced Noise Reduction (ANR) thresholds */
-struct ia_css_anr_thres {
-	int16_t data[13*64];
-};
-
-#endif /* __IA_CSS_ANR2_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr_param.h
deleted file mode 100644
index 312141793fd2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/anr/anr_2/ia_css_anr_param.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ANR2_PARAM_H
-#define __IA_CSS_ANR2_PARAM_H
-
-#include "vmem.h"
-#include "ia_css_anr2_types.h"
-
-/* Advanced Noise Reduction (ANR) thresholds */
-
-struct ia_css_isp_anr2_params {
-	VMEM_ARRAY(data, ANR_PARAM_SIZE*ISP_VEC_NELEMS);
-};
-
-#endif /* __IA_CSS_ANR2_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh.host.c
deleted file mode 100644
index 99c80d2d8f11..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh.host.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#if !defined(HAS_NO_HMEM)
-
-#include "memory_access.h"
-#include "ia_css_types.h"
-#include "sh_css_internal.h"
-#include "assert_support.h"
-#include "sh_css_frac.h"
-
-#include "ia_css_bh.host.h"
-
-void
-ia_css_bh_hmem_decode(
-	struct ia_css_3a_rgby_output *out_ptr,
-	const struct ia_css_bh_table *hmem_buf)
-{
-	int i;
-
-	/*
-	 * No weighted histogram, hence no grid definition
-	 */
-	if(!hmem_buf)
-		return;
-	assert(sizeof_hmem(HMEM0_ID) == sizeof(*hmem_buf));
-
-	/* Deinterleave */
-	for (i = 0; i < HMEM_UNIT_SIZE; i++) {
-		out_ptr[i].r = hmem_buf->hmem[BH_COLOR_R][i];
-		out_ptr[i].g = hmem_buf->hmem[BH_COLOR_G][i];
-		out_ptr[i].b = hmem_buf->hmem[BH_COLOR_B][i];
-		out_ptr[i].y = hmem_buf->hmem[BH_COLOR_Y][i];
-		/* sh_css_print ("hmem[%d] = %d, %d, %d, %d\n",
-			i, out_ptr[i].r, out_ptr[i].g, out_ptr[i].b, out_ptr[i].y); */
-	}
-}
-
-void
-ia_css_bh_encode(
-	struct sh_css_isp_bh_params *to,
-	const struct ia_css_3a_config *from,
-	unsigned size)
-{
-	(void)size;
-	/* coefficients to calculate Y */
-	to->y_coef_r =
-	    uDIGIT_FITTING(from->ae_y_coef_r, 16, SH_CSS_AE_YCOEF_SHIFT);
-	to->y_coef_g =
-	    uDIGIT_FITTING(from->ae_y_coef_g, 16, SH_CSS_AE_YCOEF_SHIFT);
-	to->y_coef_b =
-	    uDIGIT_FITTING(from->ae_y_coef_b, 16, SH_CSS_AE_YCOEF_SHIFT);
-}
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh.host.h
deleted file mode 100644
index cbb09299cf21..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh.host.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BH_HOST_H
-#define __IA_CSS_BH_HOST_H
-
-#include "ia_css_bh_param.h"
-#include "s3a/s3a_1.0/ia_css_s3a_types.h"
-
-void
-ia_css_bh_hmem_decode(
-	struct ia_css_3a_rgby_output *out_ptr,
-	const struct ia_css_bh_table *hmem_buf);
-
-void
-ia_css_bh_encode(
-	struct sh_css_isp_bh_params *to,
-	const struct ia_css_3a_config *from,
-	unsigned size);
-
-#endif /* __IA_CSS_BH_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh_param.h
deleted file mode 100644
index b0a8ef3862e0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh_param.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_HB_PARAM_H
-#define __IA_CSS_HB_PARAM_H
-
-#include "type_support.h"
-
-#ifndef PIPE_GENERATION
-#define __INLINE_HMEM__
-#include "hmem.h"
-#endif
-
-#include "ia_css_bh_types.h"
-
-/* AE (3A Support) */
-struct sh_css_isp_bh_params {
-	/* coefficients to calculate Y */
-	int32_t y_coef_r;
-	int32_t y_coef_g;
-	int32_t y_coef_b;
-};
-
-/* This should be hmem_data_t, but that breaks the pipe generator */
-struct sh_css_isp_bh_hmem_params {
-	uint32_t bh[ISP_HIST_COMPONENTS][IA_CSS_HMEM_BH_UNIT_SIZE];
-};
-
-#endif /* __IA_CSS_HB_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh_types.h
deleted file mode 100644
index ec1688e7352d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bh/bh_2/ia_css_bh_types.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BH_TYPES_H
-#define __IA_CSS_BH_TYPES_H
-
-/* Number of elements in the BH table.
-  * Should be consistent with hmem.h
-  */
-#define IA_CSS_HMEM_BH_TABLE_SIZE	ISP_HIST_DEPTH
-#define IA_CSS_HMEM_BH_UNIT_SIZE	(ISP_HIST_DEPTH/ISP_HIST_COMPONENTS)
-
-#define BH_COLOR_R	(0)
-#define BH_COLOR_G	(1)
-#define BH_COLOR_B	(2)
-#define BH_COLOR_Y	(3)
-#define BH_COLOR_NUM	(4)
-
-/* BH table */
-struct ia_css_bh_table {
-	uint32_t hmem[ISP_HIST_COMPONENTS][IA_CSS_HMEM_BH_UNIT_SIZE];
-};
-
-#endif /* __IA_CSS_BH_TYPES_H */
-
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm.host.c
deleted file mode 100644
index 6d12e031e6fc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm.host.c
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "type_support.h"
-#include "ia_css_bnlm.host.h"
-
-#ifndef IA_CSS_NO_DEBUG
-#include "ia_css_debug.h" /* ia_css_debug_dtrace() */
-#endif
-#include <assert_support.h>
-
-#define BNLM_DIV_LUT_SIZE	(12)
-static const int32_t div_lut_nearests[BNLM_DIV_LUT_SIZE] = {
-	0, 454, 948, 1484, 2070, 2710, 3412, 4184, 5035, 5978, 7025, 8191
-};
-
-static const int32_t div_lut_slopes[BNLM_DIV_LUT_SIZE] = {
-	-7760, -6960, -6216, -5536, -4912, -4344, -3832, -3360, -2936, -2552, -2208, -2208
-};
-
-static const int32_t div_lut_intercepts[BNLM_DIV_LUT_SIZE] = {
-	8184, 7752, 7336, 6928, 6536, 6152, 5776, 5416, 5064, 4728, 4408, 4408
-};
-
-/* Encodes a look-up table from BNLM public parameters to vmem parameters.
- * Input:
- *	lut	:	bnlm_lut struct containing encoded vmem parameters look-up table
- *	lut_thr	:	array containing threshold values for lut
- *	lut_val	:	array containing output values related to lut_thr
- *	lut_size:	Size of lut_val array
- */
-static inline void
-bnlm_lut_encode(struct bnlm_lut *lut, const int32_t *lut_thr, const int32_t *lut_val, const uint32_t lut_size)
-{
-	u32 blk, i;
-	const u32 block_size = 16;
-	const u32 total_blocks = ISP_VEC_NELEMS / block_size;
-
-	/* Create VMEM LUTs from the threshold and value arrays.
-	 *
-	 * Min size of the LUT is 2 entries.
-	 *
-	 * Max size of the LUT is 16 entries, so that the LUT can fit into a
-	 * single group of 16 elements inside a vector.
-	 * Then these elements are copied into other groups inside the same
-	 * vector. If the LUT size is less than 16, then remaining elements are
-	 * set to 0.
-	 */
-	assert((lut_size >= 2) && (lut_size <= block_size));
-	/* array lut_thr has (lut_size-1) entries */
-	for (i = 0; i < lut_size-2; i++) {
-		/* Check if the lut_thr is monotonically increasing */
-		assert(lut_thr[i] <= lut_thr[i+1]);
-	}
-
-	/* Initialize */
-	for (i = 0; i < total_blocks * block_size; i++) {
-		lut->thr[0][i] = 0;
-		lut->val[0][i] = 0;
-	}
-
-	/* Copy all data */
-	for (i = 0; i < lut_size - 1; i++) {
-		lut->thr[0][i] = lut_thr[i];
-		lut->val[0][i] = lut_val[i];
-	}
-	lut->val[0][i] = lut_val[i]; /* val has one more element than thr */
-
-	/* Copy data from first block to all blocks */
-	for (blk = 1; blk < total_blocks; blk++) {
-		u32 blk_offset = blk * block_size;
-		for (i = 1; i < lut_size; i++) {
-			lut->thr[0][blk_offset + i] = lut->thr[0][i];
-			lut->val[0][blk_offset + i] = lut->val[0][i];
-		}
-	}
-}
-
-/*
- * - Encodes BNLM public parameters into VMEM parameters
- * - Generates VMEM parameters which will needed internally ISP
- */
-void
-ia_css_bnlm_vmem_encode(
-			struct bnlm_vmem_params *to,
-			const struct ia_css_bnlm_config *from,
-			size_t size)
-{
-	int i;
-	(void)size;
-
-	/* Initialize LUTs in VMEM parameters */
-	bnlm_lut_encode(&to->mu_root_lut, from->mu_root_lut_thr, from->mu_root_lut_val, 16);
-	bnlm_lut_encode(&to->sad_norm_lut, from->sad_norm_lut_thr, from->sad_norm_lut_val, 16);
-	bnlm_lut_encode(&to->sig_detail_lut, from->sig_detail_lut_thr, from->sig_detail_lut_val, 16);
-	bnlm_lut_encode(&to->sig_rad_lut, from->sig_rad_lut_thr, from->sig_rad_lut_val, 16);
-	bnlm_lut_encode(&to->rad_pow_lut, from->rad_pow_lut_thr, from->rad_pow_lut_val, 16);
-	bnlm_lut_encode(&to->nl_0_lut, from->nl_0_lut_thr, from->nl_0_lut_val, 16);
-	bnlm_lut_encode(&to->nl_1_lut, from->nl_1_lut_thr, from->nl_1_lut_val, 16);
-	bnlm_lut_encode(&to->nl_2_lut, from->nl_2_lut_thr, from->nl_2_lut_val, 16);
-	bnlm_lut_encode(&to->nl_3_lut, from->nl_3_lut_thr, from->nl_3_lut_val, 16);
-
-	/* Initialize arrays in VMEM parameters */
-	memset(to->nl_th, 0, sizeof(to->nl_th));
-	to->nl_th[0][0] = from->nl_th[0];
-	to->nl_th[0][1] = from->nl_th[1];
-	to->nl_th[0][2] = from->nl_th[2];
-
-	memset(to->match_quality_max_idx, 0, sizeof(to->match_quality_max_idx));
-	to->match_quality_max_idx[0][0] = from->match_quality_max_idx[0];
-	to->match_quality_max_idx[0][1] = from->match_quality_max_idx[1];
-	to->match_quality_max_idx[0][2] = from->match_quality_max_idx[2];
-	to->match_quality_max_idx[0][3] = from->match_quality_max_idx[3];
-
-	bnlm_lut_encode(&to->div_lut, div_lut_nearests, div_lut_slopes, BNLM_DIV_LUT_SIZE);
-	memset(to->div_lut_intercepts, 0, sizeof(to->div_lut_intercepts));
-	for(i = 0; i < BNLM_DIV_LUT_SIZE; i++) {
-		to->div_lut_intercepts[0][i] = div_lut_intercepts[i];
-	}
-
-	memset(to->power_of_2, 0, sizeof(to->power_of_2));
-	for (i = 0; i < (ISP_VEC_ELEMBITS-1); i++) {
-		to->power_of_2[0][i] = 1 << i;
-	}
-}
-
-/* - Encodes BNLM public parameters into DMEM parameters */
-void
-ia_css_bnlm_encode(
-	struct bnlm_dmem_params *to,
-	const struct ia_css_bnlm_config *from,
-	size_t size)
-{
-	(void)size;
-	to->rad_enable = from->rad_enable;
-	to->rad_x_origin = from->rad_x_origin;
-	to->rad_y_origin = from->rad_y_origin;
-	to->avg_min_th = from->avg_min_th;
-	to->max_min_th = from->max_min_th;
-
-	to->exp_coeff_a = from->exp_coeff_a;
-	to->exp_coeff_b = from->exp_coeff_b;
-	to->exp_coeff_c = from->exp_coeff_c;
-	to->exp_exponent = from->exp_exponent;
-}
-
-/* Prints debug traces for BNLM public parameters */
-void
-ia_css_bnlm_debug_trace(
-	const struct ia_css_bnlm_config *config,
-	unsigned level)
-{
-	if (!config)
-		return;
-
-#ifndef IA_CSS_NO_DEBUG
-	ia_css_debug_dtrace(level, "BNLM:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "rad_enable", config->rad_enable);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "rad_x_origin", config->rad_x_origin);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "rad_y_origin", config->rad_y_origin);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "avg_min_th", config->avg_min_th);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "max_min_th", config->max_min_th);
-
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "exp_coeff_a", config->exp_coeff_a);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "exp_coeff_b", config->exp_coeff_b);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "exp_coeff_c", config->exp_coeff_c);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "exp_exponent", config->exp_exponent);
-
-	/* ToDo: print traces for LUTs */
-#endif /* IA_CSS_NO_DEBUG */
-
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm.host.h
deleted file mode 100644
index 675f6e539b3f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm.host.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BNLM_HOST_H
-#define __IA_CSS_BNLM_HOST_H
-
-#include "ia_css_bnlm_types.h"
-#include "ia_css_bnlm_param.h"
-
-void
-ia_css_bnlm_vmem_encode(
-			struct bnlm_vmem_params *to,
-			const struct ia_css_bnlm_config *from,
-			size_t size);
-
-void
-ia_css_bnlm_encode(
-	struct bnlm_dmem_params *to,
-	const struct ia_css_bnlm_config *from,
-	size_t size);
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_bnlm_debug_trace(
-	const struct ia_css_bnlm_config *config,
-	unsigned level);
-#endif
-
-#endif /* __IA_CSS_BNLM_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm_param.h
deleted file mode 100644
index 2f4be43e594e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm_param.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BNLM_PARAM_H
-#define __IA_CSS_BNLM_PARAM_H
-
-#include "type_support.h"
-#include "vmem.h" /* needed for VMEM_ARRAY */
-
-struct bnlm_lut {
-	VMEM_ARRAY(thr, ISP_VEC_NELEMS); /* thresholds */
-	VMEM_ARRAY(val, ISP_VEC_NELEMS); /* values */
-};
-
-struct bnlm_vmem_params {
-	VMEM_ARRAY(nl_th, ISP_VEC_NELEMS);
-	VMEM_ARRAY(match_quality_max_idx, ISP_VEC_NELEMS);
-	struct bnlm_lut mu_root_lut;
-	struct bnlm_lut sad_norm_lut;
-	struct bnlm_lut sig_detail_lut;
-	struct bnlm_lut sig_rad_lut;
-	struct bnlm_lut rad_pow_lut;
-	struct bnlm_lut nl_0_lut;
-	struct bnlm_lut nl_1_lut;
-	struct bnlm_lut nl_2_lut;
-	struct bnlm_lut nl_3_lut;
-
-	/* LUTs used for division approximiation */
-	struct bnlm_lut div_lut;
-	VMEM_ARRAY(div_lut_intercepts, ISP_VEC_NELEMS);
-
-	/* 240x does not have an ISP instruction to left shift each element of a
-	 * vector by different shift value. Hence it will be simulated by multiplying
-	 * the elements by required 2^shift. */
-	VMEM_ARRAY(power_of_2, ISP_VEC_NELEMS);
-};
-
-/* BNLM ISP parameters */
-struct bnlm_dmem_params {
-	bool rad_enable;
-	int32_t rad_x_origin;
-	int32_t rad_y_origin;
-	int32_t avg_min_th;
-	int32_t max_min_th;
-
-	int32_t exp_coeff_a;
-	uint32_t exp_coeff_b;
-	int32_t exp_coeff_c;
-	uint32_t exp_exponent;
-};
-
-#endif /* __IA_CSS_BNLM_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm_types.h
deleted file mode 100644
index 87e0f19c856b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnlm/ia_css_bnlm_types.h
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BNLM_TYPES_H
-#define __IA_CSS_BNLM_TYPES_H
-
-/* @file
-* CSS-API header file for Bayer Non-Linear Mean parameters.
-*/
-
-#include "type_support.h" /* int32_t */
-
-/* Bayer Non-Linear Mean configuration
- *
- * \brief BNLM public parameters.
- * \details Struct with all parameters for the BNLM kernel that can be set
- * from the CSS API.
- *
- * ISP2.6.1: BNLM is used.
- */
-struct ia_css_bnlm_config {
-	bool		rad_enable;	/** Enable a radial dependency in a weight calculation */
-	int32_t		rad_x_origin;	/** Initial x coordinate for a radius calculation */
-	int32_t		rad_y_origin;	/** Initial x coordinate for a radius calculation */
-	/* a threshold for average of weights if this < Th, do not denoise pixel */
-	int32_t		avg_min_th;
-	/* minimum weight for denoising if max < th, do not denoise pixel */
-	int32_t		max_min_th;
-
-	/**@{*/
-	/* Coefficient for approximation, in the form of (1 + x / N)^N,
-	 * that fits the first-order exp() to default exp_lut in BNLM sheet
-	 * */
-	int32_t		exp_coeff_a;
-	uint32_t	exp_coeff_b;
-	int32_t		exp_coeff_c;
-	uint32_t	exp_exponent;
-	/**@}*/
-
-	int32_t nl_th[3];	/** Detail thresholds */
-
-	/* Index for n-th maximum candidate weight for each detail group */
-	int32_t match_quality_max_idx[4];
-
-	/**@{*/
-	/* A lookup table for 1/sqrt(1+mu) approximation */
-	int32_t mu_root_lut_thr[15];
-	int32_t mu_root_lut_val[16];
-	/**@}*/
-	/**@{*/
-	/* A lookup table for SAD normalization */
-	int32_t sad_norm_lut_thr[15];
-	int32_t sad_norm_lut_val[16];
-	/**@}*/
-	/**@{*/
-	/* A lookup table that models a weight's dependency on textures */
-	int32_t sig_detail_lut_thr[15];
-	int32_t sig_detail_lut_val[16];
-	/**@}*/
-	/**@{*/
-	/* A lookup table that models a weight's dependency on a pixel's radial distance */
-	int32_t sig_rad_lut_thr[15];
-	int32_t sig_rad_lut_val[16];
-	/**@}*/
-	/**@{*/
-	/* A lookup table to control denoise power depending on a pixel's radial distance */
-	int32_t rad_pow_lut_thr[15];
-	int32_t rad_pow_lut_val[16];
-	/**@}*/
-	/**@{*/
-	/* Non linear transfer functions to calculate the blending coefficient depending on detail group */
-	/* detail group 0 */
-	/**@{*/
-	int32_t nl_0_lut_thr[15];
-	int32_t nl_0_lut_val[16];
-	/**@}*/
-	/**@{*/
-	/* detail group 1 */
-	int32_t nl_1_lut_thr[15];
-	int32_t nl_1_lut_val[16];
-	/**@}*/
-	/**@{*/
-	/* detail group 2 */
-	int32_t nl_2_lut_thr[15];
-	int32_t nl_2_lut_val[16];
-	/**@}*/
-	/**@{*/
-	/* detail group 3 */
-	int32_t nl_3_lut_thr[15];
-	int32_t nl_3_lut_val[16];
-	/**@}*/
-	/**@}*/
-};
-
-#endif /* __IA_CSS_BNLM_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c
deleted file mode 100644
index a7de6ecb950d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "type_support.h"
-#include "ia_css_bnr2_2.host.h"
-
-#ifndef IA_CSS_NO_DEBUG
-#include "ia_css_debug.h" /* ia_css_debug_dtrace() */
-#endif
-
-/* Default kernel parameters. */
-const struct ia_css_bnr2_2_config default_bnr2_2_config = {
-	200,
-	200,
-	200,
-	0,
-	0,
-	0,
-	200,
-	200,
-	200,
-	0,
-	0,
-	0,
-	0,
-	4096,
-	8191,
-	128,
-	1,
-	0,
-	0,
-	0,
-	8191,
-	0,
-	8191
-};
-
-void
-ia_css_bnr2_2_encode(
-	struct sh_css_isp_bnr2_2_params *to,
-	const struct ia_css_bnr2_2_config *from,
-	size_t size)
-{
-	(void)size;
-	to->d_var_gain_r = from->d_var_gain_r;
-	to->d_var_gain_g = from->d_var_gain_g;
-	to->d_var_gain_b = from->d_var_gain_b;
-	to->d_var_gain_slope_r = from->d_var_gain_slope_r;
-	to->d_var_gain_slope_g = from->d_var_gain_slope_g;
-	to->d_var_gain_slope_b = from->d_var_gain_slope_b;
-
-	to->n_var_gain_r = from->n_var_gain_r;
-	to->n_var_gain_g = from->n_var_gain_g;
-	to->n_var_gain_b = from->n_var_gain_b;
-	to->n_var_gain_slope_r = from->n_var_gain_slope_r;
-	to->n_var_gain_slope_g = from->n_var_gain_slope_g;
-	to->n_var_gain_slope_b = from->n_var_gain_slope_b;
-
-	to->dir_thres = from->dir_thres;
-	to->dir_thres_w = from->dir_thres_w;
-	to->var_offset_coef = from->var_offset_coef;
-
-	to->dir_gain = from->dir_gain;
-	to->detail_gain	= from->detail_gain;
-	to->detail_gain_divisor = from->detail_gain_divisor;
-	to->detail_level_offset = from->detail_level_offset;
-
-	to->d_var_th_min = from->d_var_th_min;
-	to->d_var_th_max = from->d_var_th_max;
-	to->n_var_th_min = from->n_var_th_min;
-	to->n_var_th_max = from->n_var_th_max;
-}
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_bnr2_2_debug_dtrace(
-	const struct ia_css_bnr2_2_config *bnr,
-	unsigned level)
-{
-	if (!bnr)
-		return;
-
-	ia_css_debug_dtrace(level, "Bayer Noise Reduction 2.2:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "d_var_gain_r", bnr->d_var_gain_r);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "d_var_gain_g", bnr->d_var_gain_g);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "d_var_gain_b", bnr->d_var_gain_b);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "d_var_gain_slope_r", bnr->d_var_gain_slope_r);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "d_var_gain_slope_g", bnr->d_var_gain_slope_g);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "d_var_gain_slope_b", bnr->d_var_gain_slope_b);
-
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "n_var_gain_r", bnr->n_var_gain_r);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "n_var_gain_g", bnr->n_var_gain_g);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "n_var_gain_b", bnr->n_var_gain_b);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "n_var_gain_slope_r", bnr->n_var_gain_slope_r);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "n_var_gain_slope_g", bnr->n_var_gain_slope_g);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "n_var_gain_slope_b", bnr->n_var_gain_slope_b);
-
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "dir_thres", bnr->dir_thres);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "dir_thres_w", bnr->dir_thres_w);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "var_offset_coef", bnr->var_offset_coef);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "dir_gain", bnr->dir_gain);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "detail_gain", bnr->detail_gain);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "detail_gain_divisor", bnr->detail_gain_divisor);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "detail_level_offset", bnr->detail_level_offset);
-
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "d_var_th_min", bnr->d_var_th_min);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "d_var_th_max", bnr->d_var_th_max);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "n_var_th_min", bnr->n_var_th_min);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "n_var_th_max", bnr->n_var_th_max);
-}
-#endif /* IA_CSS_NO_DEBUG */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h
deleted file mode 100644
index c94b366b8142..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#ifndef __IA_CSS_BNR2_2_HOST_H
-#define __IA_CSS_BNR2_2_HOST_H
-
-#include "ia_css_bnr2_2_types.h"
-#include "ia_css_bnr2_2_param.h"
-
-extern const struct ia_css_bnr2_2_config default_bnr2_2_config;
-
-void
-ia_css_bnr2_2_encode(
-	struct sh_css_isp_bnr2_2_params *to,
-	const struct ia_css_bnr2_2_config *from,
-	size_t size);
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_bnr2_2_debug_dtrace(
-	const struct ia_css_bnr2_2_config *config,
-	unsigned level);
-#endif
-
-#endif /* __IA_CSS_BNR2_2_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h
deleted file mode 100644
index 6dec27a99d8f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BNR2_2_PARAM_H
-#define __IA_CSS_BNR2_2_PARAM_H
-
-#include "type_support.h"
-
-/* BNR (Bayer Noise Reduction) ISP parameters */
-struct sh_css_isp_bnr2_2_params {
-	int32_t d_var_gain_r;
-	int32_t d_var_gain_g;
-	int32_t d_var_gain_b;
-	int32_t d_var_gain_slope_r;
-	int32_t d_var_gain_slope_g;
-	int32_t d_var_gain_slope_b;
-	int32_t n_var_gain_r;
-	int32_t n_var_gain_g;
-	int32_t n_var_gain_b;
-	int32_t n_var_gain_slope_r;
-	int32_t n_var_gain_slope_g;
-	int32_t n_var_gain_slope_b;
-	int32_t dir_thres;
-	int32_t dir_thres_w;
-	int32_t var_offset_coef;
-	int32_t dir_gain;
-	int32_t detail_gain;
-	int32_t detail_gain_divisor;
-	int32_t detail_level_offset;
-	int32_t d_var_th_min;
-	int32_t d_var_th_max;
-	int32_t n_var_th_min;
-	int32_t n_var_th_max;
-};
-
-#endif /* __IA_CSS_BNR2_2_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h
deleted file mode 100644
index 551bd0ed3bac..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BNR2_2_TYPES_H
-#define __IA_CSS_BNR2_2_TYPES_H
-
-/* @file
-* CSS-API header file for Bayer Noise Reduction parameters.
-*/
-
-#include "type_support.h" /* int32_t */
-
-/* Bayer Noise Reduction 2.2 configuration
- *
- * \brief BNR2_2 public parameters.
- * \details Struct with all parameters for the BNR2.2 kernel that can be set
- * from the CSS API.
- *
- * ISP2.6.1: BNR2.2 is used.
- */
-struct ia_css_bnr2_2_config {
-	/**@{*/
-	/* Directional variance gain for R/G/B components in dark region */
-	int32_t d_var_gain_r;
-	int32_t d_var_gain_g;
-	int32_t d_var_gain_b;
-	/**@}*/
-	/**@{*/
-	/* Slope of Directional variance gain between dark and bright region */
-	int32_t d_var_gain_slope_r;
-	int32_t d_var_gain_slope_g;
-	int32_t d_var_gain_slope_b;
-	/**@}*/
-	/**@{*/
-	/* Non-Directional variance gain for R/G/B components in dark region */
-	int32_t n_var_gain_r;
-	int32_t n_var_gain_g;
-	int32_t n_var_gain_b;
-	/**@}*/
-	/**@{*/
-	/* Slope of Non-Directional variance gain between dark and bright region */
-	int32_t n_var_gain_slope_r;
-	int32_t n_var_gain_slope_g;
-	int32_t n_var_gain_slope_b;
-	/**@}*/
-
-	int32_t dir_thres;		/** Threshold for directional filtering */
-	int32_t dir_thres_w;		/** Threshold width for directional filtering */
-	int32_t var_offset_coef;	/** Variance offset coefficient */
-	int32_t dir_gain;		/** Gain for directional coefficient */
-	int32_t detail_gain;		/** Gain for low contrast texture control */
-	int32_t detail_gain_divisor;	/** Gain divisor for low contrast texture control */
-	int32_t detail_level_offset;	/** Bias value for low contrast texture control */
-	int32_t d_var_th_min;		/** Minimum clipping value for directional variance*/
-	int32_t d_var_th_max;		/** Maximum clipping value for diretional variance*/
-	int32_t n_var_th_min;		/** Minimum clipping value for non-directional variance*/
-	int32_t n_var_th_max;		/** Maximum clipping value for non-directional variance*/
-};
-
-#endif /* __IA_CSS_BNR2_2_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c
deleted file mode 100644
index d1baca54c3ad..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "sh_css_frac.h"
-
-#include "ia_css_bnr.host.h"
-
-void
-ia_css_bnr_encode(
-	struct sh_css_isp_bnr_params *to,
-	const struct ia_css_nr_config *from,
-	unsigned size)
-{
-	(void)size;
-	/* BNR (Bayer Noise Reduction) */
-	to->threshold_low =
-	    uDIGIT_FITTING(from->direction, 16, SH_CSS_BAYER_BITS);
-	to->threshold_width_log2 = uFRACTION_BITS_FITTING(8);
-	to->threshold_width =
-	    1 << to->threshold_width_log2;
-	to->gain_all =
-	    uDIGIT_FITTING(from->bnr_gain, 16, SH_CSS_BNR_GAIN_SHIFT);
-	to->gain_dir =
-	    uDIGIT_FITTING(from->bnr_gain, 16, SH_CSS_BNR_GAIN_SHIFT);
-	to->clip = uDIGIT_FITTING((unsigned)16384, 16, SH_CSS_BAYER_BITS);
-}
-
-void
-ia_css_bnr_dump(
-	const struct sh_css_isp_bnr_params *bnr,
-	unsigned level)
-{
-	if (!bnr) return;
-	ia_css_debug_dtrace(level, "Bayer Noise Reduction:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"bnr_gain_all", bnr->gain_all);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"bnr_gain_dir", bnr->gain_dir);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"bnr_threshold_low",
-			bnr->threshold_low);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"bnr_threshold_width_log2",
-			bnr->threshold_width_log2);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"bnr_threshold_width",
-			bnr->threshold_width);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"bnr_clip", bnr->clip);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h
deleted file mode 100644
index ccd2abc60537..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BNR_HOST_H
-#define __IA_CSS_BNR_HOST_H
-
-#include "sh_css_params.h"
-
-#include "ynr/ynr_1.0/ia_css_ynr_types.h"
-#include "ia_css_bnr_param.h"
-
-void
-ia_css_bnr_encode(
-	struct sh_css_isp_bnr_params *to,
-	const struct ia_css_nr_config *from,
-	unsigned size);
-
-void
-ia_css_bnr_dump(
-	const struct sh_css_isp_bnr_params *bnr,
-	unsigned level);
-
-#endif /* __IA_CSS_DP_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h
deleted file mode 100644
index 331e05885ef4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BNR_PARAM_H
-#define __IA_CSS_BNR_PARAM_H
-
-#include "type_support.h"
-
-/* BNR (Bayer Noise Reduction) */
-struct sh_css_isp_bnr_params {
-	int32_t gain_all;
-	int32_t gain_dir;
-	int32_t threshold_low;
-	int32_t threshold_width_log2;
-	int32_t threshold_width;
-	int32_t clip;
-};
-
-#endif /* __IA_CSS_BNR_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c
deleted file mode 100644
index d14fd8fc08b1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-
-#include "ia_css_cnr.host.h"
-
-/* keep the interface here, it is not enabled yet because host doesn't know the size of individual state */
-void
-ia_css_init_cnr_state(
-	void/*struct sh_css_isp_cnr_vmem_state*/ *state,
-	size_t size)
-{
-	memset(state, 0, size);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h
deleted file mode 100644
index 6f00d280b7d6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CNR_HOST_H
-#define __IA_CSS_CNR_HOST_H
-
-#include "ia_css_cnr_param.h"
-
-void
-ia_css_init_cnr_state(
-	void/*struct sh_css_isp_cnr_vmem_state*/ *state,
-	size_t size);
-
-#endif /* __IA_CSS_CNR_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h
deleted file mode 100644
index c1af207cbf9a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CNR_PARAM_H
-#define __IA_CSS_CNR_PARAM_H
-
-#include "type_support.h"
-
-/* CNR (Chroma Noise Reduction) */
-/* Reuse YNR1 param structure */
-#include  "../../ynr/ynr_1.0/ia_css_ynr_param.h"
-
-#endif /* __IA_CSS_CNR_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c
deleted file mode 100644
index 4b4b2b715407..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-
-#include "ia_css_cnr2.host.h"
-
-const struct ia_css_cnr_config default_cnr_config = {
-	0,
-	0,
-	100,
-	100,
-	100,
-	50,
-	50,
-	50
-};
-
-void
-ia_css_cnr_encode(
-	struct sh_css_isp_cnr_params *to,
-	const struct ia_css_cnr_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->coring_u = from->coring_u;
-	to->coring_v = from->coring_v;
-	to->sense_gain_vy = from->sense_gain_vy;
-	to->sense_gain_vu = from->sense_gain_vu;
-	to->sense_gain_vv = from->sense_gain_vv;
-	to->sense_gain_hy = from->sense_gain_hy;
-	to->sense_gain_hu = from->sense_gain_hu;
-	to->sense_gain_hv = from->sense_gain_hv;
-}
-
-void
-ia_css_cnr_dump(
-	const struct sh_css_isp_cnr_params *cnr,
-	unsigned level);
-
-void
-ia_css_cnr_debug_dtrace(
-	const struct ia_css_cnr_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.coring_u=%d, config.coring_v=%d, "
-		"config.sense_gain_vy=%d, config.sense_gain_hy=%d, "
-		"config.sense_gain_vu=%d, config.sense_gain_hu=%d, "
-		"config.sense_gain_vv=%d, config.sense_gain_hv=%d\n",
-		config->coring_u, config->coring_v,
-		config->sense_gain_vy, config->sense_gain_hy,
-		config->sense_gain_vu, config->sense_gain_hu,
-		config->sense_gain_vv, config->sense_gain_hv);
-}
-
-void
-ia_css_init_cnr2_state(
-	void/*struct sh_css_isp_cnr_vmem_state*/ *state,
-	size_t size)
-{
-	memset(state, 0, size);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h
deleted file mode 100644
index abcf0eba706f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CNR2_HOST_H
-#define __IA_CSS_CNR2_HOST_H
-
-#include "ia_css_cnr2_types.h"
-#include "ia_css_cnr2_param.h"
-
-extern const struct ia_css_cnr_config default_cnr_config;
-
-void
-ia_css_cnr_encode(
-	struct sh_css_isp_cnr_params *to,
-	const struct ia_css_cnr_config *from,
-	unsigned size);
-
-void
-ia_css_cnr_dump(
-	const struct sh_css_isp_cnr_params *cnr,
-	unsigned level);
-
-void
-ia_css_cnr_debug_dtrace(
-	const struct ia_css_cnr_config *config,
-	unsigned level);
-
-void
-ia_css_init_cnr2_state(
-	void/*struct sh_css_isp_cnr_vmem_state*/ *state,
-	size_t size);
-#endif /* __IA_CSS_CNR2_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h
deleted file mode 100644
index d6f490e26c94..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CNR2_PARAM_H
-#define __IA_CSS_CNR2_PARAM_H
-
-#include "type_support.h"
-
-/* CNR (Chroma Noise Reduction) */
-struct sh_css_isp_cnr_params {
-	int32_t coring_u;
-	int32_t coring_v;
-	int32_t sense_gain_vy;
-	int32_t sense_gain_vu;
-	int32_t sense_gain_vv;
-	int32_t sense_gain_hy;
-	int32_t sense_gain_hu;
-	int32_t sense_gain_hv;
-};
-
-#endif /* __IA_CSS_CNR2_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h
deleted file mode 100644
index 3ebc069d8ada..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CNR2_TYPES_H
-#define __IA_CSS_CNR2_TYPES_H
-
-/* @file
-* CSS-API header file for Chroma Noise Reduction (CNR) parameters
-*/
-
-/* Chroma Noise Reduction configuration.
- *
- *  Small sensitivity of edge means strong smoothness and NR performance.
- *  If you see blurred color on vertical edges,
- *  set higher values on sense_gain_h*.
- *  If you see blurred color on horizontal edges,
- *  set higher values on sense_gain_v*.
- *
- *  ISP block: CNR2
- * (ISP1: CNR1 is used.)
- * (ISP2: CNR1 is used for Preview/Video.)
- *  ISP2: CNR2 is used for Still.
- */
-struct ia_css_cnr_config {
-	uint16_t coring_u;	/** Coring level of U.
-				u0.13, [0,8191], default/ineffective 0 */
-	uint16_t coring_v;	/** Coring level of V.
-				u0.13, [0,8191], default/ineffective 0 */
-	uint16_t sense_gain_vy;	/** Sensitivity of horizontal edge of Y.
-				u13.0, [0,8191], default 100, ineffective 8191 */
-	uint16_t sense_gain_vu;	/** Sensitivity of horizontal edge of U.
-				u13.0, [0,8191], default 100, ineffective 8191 */
-	uint16_t sense_gain_vv;	/** Sensitivity of horizontal edge of V.
-				u13.0, [0,8191], default 100, ineffective 8191 */
-	uint16_t sense_gain_hy;	/** Sensitivity of vertical edge of Y.
-				u13.0, [0,8191], default 50, ineffective 8191 */
-	uint16_t sense_gain_hu;	/** Sensitivity of vertical edge of U.
-				u13.0, [0,8191], default 50, ineffective 8191 */
-	uint16_t sense_gain_hv;	/** Sensitivity of vertical edge of V.
-				u13.0, [0,8191], default 50, ineffective 8191 */
-};
-
-#endif /* __IA_CSS_CNR2_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr_param.h
deleted file mode 100644
index 56651ba62598..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/cnr/cnr_2/ia_css_cnr_param.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CNRX_PARAM_H
-#define __IA_CSS_CNRX_PARAM_H
-
-#include  "ia_css_cnr2_param.h"
-
-#endif /* __IA_CSS_CNRX_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c
deleted file mode 100644
index 8f25ee180cda..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "ia_css_conversion.host.h"
-
-const struct ia_css_conversion_config default_conversion_config = {
-	0,
-	0,
-	0,
-	0,
-};
-
-void
-ia_css_conversion_encode(
-	struct sh_css_isp_conversion_params *to,
-	const struct ia_css_conversion_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->en     = from->en;
-	to->dummy0 = from->dummy0;
-	to->dummy1 = from->dummy1;
-	to->dummy2 = from->dummy2;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h
deleted file mode 100644
index da7a0a034a71..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CONVERSION_HOST_H
-#define __IA_CSS_CONVERSION_HOST_H
-
-#include "ia_css_conversion_types.h"
-#include "ia_css_conversion_param.h"
-
-extern const struct ia_css_conversion_config default_conversion_config;
-
-void
-ia_css_conversion_encode(
-	struct sh_css_isp_conversion_params *to,
-	const struct ia_css_conversion_config *from,
-	unsigned size);
-
-#ifdef ISP2401
-/* workaround until code generation in isp_kernelparameters.host.c is fixed */
-#define ia_css_conversion_par_encode(to, from, size) ia_css_conversion_encode(to, from, size)
-#endif
-#endif /* __IA_CSS_CONVERSION_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h
deleted file mode 100644
index 301d506f447e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CONVERSION_PARAM_H
-#define __IA_CSS_CONVERSION_PARAM_H
-
-#include "type_support.h"
-
-/* CONVERSION */
-struct sh_css_isp_conversion_params {
-	uint32_t en;
-	uint32_t dummy0;
-	uint32_t dummy1;
-	uint32_t dummy2;
-};
-
-#endif /* __IA_CSS_CONVERSION_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h
deleted file mode 100644
index 47a38fd65950..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CONVERSION_TYPES_H
-#define __IA_CSS_CONVERSION_TYPES_H
-
-/**
- *  Conversion Kernel parameters.
- *  Deinterleave bayer quad into isys format
- *
- *  ISP block: CONVERSION
- *
- */
-struct ia_css_conversion_config {
-	uint32_t en;     /** en parameter */
-	uint32_t dummy0; /** dummy0 dummy parameter 0 */
-	uint32_t dummy1; /** dummy1 dummy parameter 1 */
-	uint32_t dummy2; /** dummy2 dummy parameter 2 */
-};
-
-#endif /* __IA_CSS_CONVERSION_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c
deleted file mode 100644
index 45e1ea8b1fb0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_copy_output.host.h"
-#include "ia_css_binary.h"
-#include "type_support.h"
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-#include "isp.h"
-
-static const struct ia_css_copy_output_configuration default_config = {
-	.enable = false,
-};
-
-void
-ia_css_copy_output_config(
-	struct sh_css_isp_copy_output_isp_config      *to,
-	const struct ia_css_copy_output_configuration *from,
-	unsigned size)
-{
-	(void)size;
-	to->enable = from->enable;
-}
-
-void
-ia_css_copy_output_configure(
-	const struct ia_css_binary     *binary,
-	bool enable)
-{
-	struct ia_css_copy_output_configuration config = default_config;
-
-	config.enable = enable;
-
-	ia_css_configure_copy_output(binary, &config);
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h
deleted file mode 100644
index 3eb77365f8d0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_COPY_OUTPUT_HOST_H
-#define __IA_CSS_COPY_OUTPUT_HOST_H
-
-#include "type_support.h"
-#include "ia_css_binary.h"
-
-#include "ia_css_copy_output_param.h"
-
-void
-ia_css_copy_output_config(
-	struct sh_css_isp_copy_output_isp_config      *to,
-	const struct ia_css_copy_output_configuration *from,
-	unsigned size);
-
-void
-ia_css_copy_output_configure(
-	const struct ia_css_binary     *binary,
-	bool enable);
-
-#endif /* __IA_CSS_COPY_OUTPUT_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h
deleted file mode 100644
index 622d9181e13f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_COPY_PARAM_H
-#define __IA_CSS_COPY_PARAM_H
-
-struct ia_css_copy_output_configuration {
-	bool enable;
-};
-
-struct sh_css_isp_copy_output_isp_config {
-	uint32_t enable;
-};
-
-#endif /* __IA_CSS_COPY_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop.host.c
deleted file mode 100644
index 92905220d862..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop.host.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <assert_support.h>
-#include <ia_css_frame_public.h>
-#include <ia_css_frame.h>
-#include <ia_css_binary.h>
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-#include "isp.h"
-#include "ia_css_crop.host.h"
-
-static const struct ia_css_crop_configuration default_config = {
-	.info = (struct ia_css_frame_info *)NULL,
-};
-
-void
-ia_css_crop_encode(
-	struct sh_css_isp_crop_isp_params *to,
-	const struct ia_css_crop_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->crop_pos = from->crop_pos;
-}
-
-void
-ia_css_crop_config(
-	struct sh_css_isp_crop_isp_config *to,
-	const struct ia_css_crop_configuration  *from,
-	unsigned size)
-{
-	unsigned elems_a = ISP_VEC_NELEMS;
-
-	(void)size;
-	ia_css_dma_configure_from_info(&to->port_b, from->info);
-	to->width_a_over_b = elems_a / to->port_b.elems;
-
-	/* Assume divisiblity here, may need to generalize to fixed point. */
-	assert (elems_a % to->port_b.elems == 0);
-}
-
-void
-ia_css_crop_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *info)
-{
-	struct ia_css_crop_configuration config = default_config;
-
-	config.info = info;
-
-	ia_css_configure_crop(binary, &config);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop.host.h
deleted file mode 100644
index 9c1a4c7cac98..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop.host.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CROP_HOST_H
-#define __IA_CSS_CROP_HOST_H
-
-#include <ia_css_frame_public.h>
-#include <ia_css_binary.h>
-
-#include "ia_css_crop_types.h"
-#include "ia_css_crop_param.h"
-
-void
-ia_css_crop_encode(
-	struct sh_css_isp_crop_isp_params *to,
-	const struct ia_css_crop_config *from,
-	unsigned size);
-
-void
-ia_css_crop_config(
-	struct sh_css_isp_crop_isp_config      *to,
-	const struct ia_css_crop_configuration *from,
-	unsigned size);
-
-void
-ia_css_crop_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *from);
-
-#endif /* __IA_CSS_CROP_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop_param.h
deleted file mode 100644
index 0f1812cdd92a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop_param.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CROP_PARAM_H
-#define __IA_CSS_CROP_PARAM_H
-
-#include <type_support.h>
-#include "dma.h"
-#include "sh_css_internal.h" /* sh_css_crop_pos */
-
-/* Crop frame */
-struct sh_css_isp_crop_isp_config {
-	uint32_t width_a_over_b;
-	struct dma_port_config port_b;
-};
-
-struct sh_css_isp_crop_isp_params {
-	struct sh_css_crop_pos crop_pos;
-};
-
-#endif /* __IA_CSS_CROP_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop_types.h
deleted file mode 100644
index b5d454225f89..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/crop/crop_1.0/ia_css_crop_types.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CROP_TYPES_H
-#define __IA_CSS_CROP_TYPES_H
-
-/* Crop frame
- *
- *  ISP block: crop frame
- */
-
-#include <ia_css_frame_public.h>
-#include "sh_css_uds.h" /* sh_css_crop_pos */
-
-struct ia_css_crop_config {
-	struct sh_css_crop_pos crop_pos;
-};
-
-struct ia_css_crop_configuration {
-	const struct ia_css_frame_info *info;
-};
-
-#endif /* __IA_CSS_CROP_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc.host.c
deleted file mode 100644
index 9f94ef1de572..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc.host.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#ifndef IA_CSS_NO_DEBUG
-/* FIXME: See BZ 4427 */
-#include "ia_css_debug.h"
-#endif
-
-#include "ia_css_csc.host.h"
-
-const struct ia_css_cc_config default_cc_config = {
-	8,
-	{255, 29, 120, 0, -374, -342, 0, -672, 301},
-};
-
-void
-ia_css_encode_cc(
-	struct sh_css_isp_csc_params *to,
-	const struct ia_css_cc_config *from,
-	unsigned size)
-{
-	(void)size;
-#ifndef IA_CSS_NO_DEBUG
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_encode_cc() enter:\n");
-#endif
-
-	to->m_shift    = (int16_t) from->fraction_bits;
-	to->m00 = (int16_t) from->matrix[0];
-	to->m01 = (int16_t) from->matrix[1];
-	to->m02 = (int16_t) from->matrix[2];
-	to->m10 = (int16_t) from->matrix[3];
-	to->m11 = (int16_t) from->matrix[4];
-	to->m12 = (int16_t) from->matrix[5];
-	to->m20 = (int16_t) from->matrix[6];
-	to->m21 = (int16_t) from->matrix[7];
-	to->m22 = (int16_t) from->matrix[8];
-
-#ifndef IA_CSS_NO_DEBUG
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_encode_cc() leave:\n");
-#endif
-}
-
-void
-ia_css_csc_encode(
-	struct sh_css_isp_csc_params *to,
-	const struct ia_css_cc_config *from,
-	unsigned size)
-{
-	ia_css_encode_cc(to, from, size);
-}
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_cc_dump(
-	const struct sh_css_isp_csc_params *csc,
-	unsigned level,
-	const char *name)
-{
-	if (!csc) return;
-	ia_css_debug_dtrace(level, "%s\n", name);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"m_shift",
-		csc->m_shift);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"m00",
-		csc->m00);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"m01",
-		csc->m01);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"m02",
-		csc->m02);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"m10",
-		csc->m10);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"m11",
-		csc->m11);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"m12",
-		csc->m12);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"m20",
-		csc->m20);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"m21",
-		csc->m21);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"m22",
-		csc->m22);
-}
-
-void
-ia_css_csc_dump(
-	const struct sh_css_isp_csc_params *csc,
-	unsigned level)
-{
-	ia_css_cc_dump(csc, level, "Color Space Conversion");
-}
-
-void
-ia_css_cc_config_debug_dtrace(
-	const struct ia_css_cc_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.m[0]=%d, "
-		"config.m[1]=%d, config.m[2]=%d, "
-		"config.m[3]=%d, config.m[4]=%d, "
-		"config.m[5]=%d, config.m[6]=%d, "
-		"config.m[7]=%d, config.m[8]=%d\n",
-		config->matrix[0],
-		config->matrix[1], config->matrix[2],
-		config->matrix[3], config->matrix[4],
-		config->matrix[5], config->matrix[6],
-		config->matrix[7], config->matrix[8]);
-}
-#endif
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc.host.h
deleted file mode 100644
index eb10d8a5709d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc.host.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CSC_HOST_H
-#define __IA_CSS_CSC_HOST_H
-
-#include "ia_css_csc_types.h"
-#include "ia_css_csc_param.h"
-
-extern const struct ia_css_cc_config default_cc_config;
-
-void
-ia_css_encode_cc(
-	struct sh_css_isp_csc_params *to,
-	const struct ia_css_cc_config *from,
-	unsigned size);
-
-void
-ia_css_csc_encode(
-	struct sh_css_isp_csc_params *to,
-	const struct ia_css_cc_config *from,
-	unsigned size);
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_cc_dump(
-	const struct sh_css_isp_csc_params *csc, unsigned level,
-	const char *name);
-
-void
-ia_css_csc_dump(
-	const struct sh_css_isp_csc_params *csc,
-	unsigned level);
-
-void
-ia_css_cc_config_debug_dtrace(
-	const struct ia_css_cc_config *config,
-	unsigned level);
-
-#define ia_css_csc_debug_dtrace ia_css_cc_config_debug_dtrace
-#endif
-
-#endif /* __IA_CSS_CSC_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc_param.h
deleted file mode 100644
index 0b054a939baf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc_param.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CSC_PARAM_H
-#define __IA_CSS_CSC_PARAM_H
-
-#include "type_support.h"
-/* CSC (Color Space Conversion) */
-struct sh_css_isp_csc_params {
-	uint16_t	m_shift;
-	int16_t		m00;
-	int16_t		m01;
-	int16_t		m02;
-	int16_t		m10;
-	int16_t		m11;
-	int16_t		m12;
-	int16_t		m20;
-	int16_t		m21;
-	int16_t		m22;
-};
-
-
-#endif /* __IA_CSS_CSC_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc_types.h
deleted file mode 100644
index 10404380c637..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/csc/csc_1.0/ia_css_csc_types.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CSC_TYPES_H
-#define __IA_CSS_CSC_TYPES_H
-
-/* @file
-* CSS-API header file for Color Space Conversion parameters.
-*/
-
-/* Color Correction configuration.
- *
- *  This structure is used for 3 cases.
- *  ("YCgCo" is the output format of Demosaic.)
- *
- *  1. Color Space Conversion (YCgCo to YUV) for ISP1.
- *     ISP block: CSC1 (Color Space Conversion)
- *     struct ia_css_cc_config   *cc_config
- *
- *  2. Color Correction Matrix (YCgCo to RGB) for ISP2.
- *     ISP block: CCM2 (Color Correction Matrix)
- *     struct ia_css_cc_config   *yuv2rgb_cc_config
- *
- *  3. Color Space Conversion (RGB to YUV) for ISP2.
- *     ISP block: CSC2 (Color Space Conversion)
- *     struct ia_css_cc_config   *rgb2yuv_cc_config
- *
- *  default/ineffective:
- *  1. YCgCo -> YUV
- *  	1	0.174		0.185
- *  	0	-0.66252	-0.66874
- *  	0	-0.83738	0.58131
- *
- *	fraction_bits = 12
- *  	4096	713	758
- *  	0	-2714	-2739
- *  	0	-3430	2381
- *
- *  2. YCgCo -> RGB
- *  	1	-1	1
- *  	1	1	0
- *  	1	-1	-1
- *
- *	fraction_bits = 12
- *  	4096	-4096	4096
- *  	4096	4096	0
- *  	4096	-4096	-4096
- *
- *  3. RGB -> YUV
- *	0.299	   0.587	0.114
- * 	-0.16874   -0.33126	0.5
- * 	0.5	   -0.41869	-0.08131
- *
- *	fraction_bits = 13
- *  	2449	4809	934
- *  	-1382	-2714	4096
- *  	4096	-3430	-666
- */
-struct ia_css_cc_config {
-	uint32_t fraction_bits;/** Fractional bits of matrix.
-					u8.0, [0,13] */
-	int32_t matrix[3 * 3]; /** Conversion matrix.
-					s[13-fraction_bits].[fraction_bits],
-					[-8192,8191] */
-};
-
-#endif /* __IA_CSS_CSC_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c
deleted file mode 100644
index e27648c46a25..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-#include "ctc/ctc_1.0/ia_css_ctc.host.h"
-#include "ia_css_ctc1_5.host.h"
-
-static void ctc_gradient(
-	int *dydx, int *shift,
-	int y1, int y0, int x1, int x0)
-{
-	int frc_bits = max(IA_CSS_CTC_COEF_SHIFT, 16);
-	int dy = y1 - y0;
-	int dx = x1 - x0;
-	int dydx_int;
-	int dydx_frc;
-	int sft;
-	/* max_dydx = the maxinum gradient = the maximum y (gain) */
-	int max_dydx = (1 << IA_CSS_CTC_COEF_SHIFT) - 1;
-
-	if (dx == 0) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ctc_gradient() error, illegal division operation\n");
-		return;
-	} else {
-		dydx_int = dy / dx;
-		dydx_frc = ((dy - dydx_int * dx) << frc_bits) / dx;
-	}
-
-	assert(y0 >= 0 && y0 <= max_dydx);
-	assert(y1 >= 0 && y1 <= max_dydx);
-	assert(x0 < x1);
-	assert(dydx != NULL);
-	assert(shift != NULL);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ctc_gradient() enter:\n");
-
-	/* search "sft" which meets this condition:
-		   (1 << (IA_CSS_CTC_COEF_SHIFT - 1))
-		<= (((float)dy / (float)dx) * (1 << sft))
-		<= ((1 << IA_CSS_CTC_COEF_SHIFT) - 1) */
-	for (sft = 0; sft <= IA_CSS_CTC_COEF_SHIFT; sft++) {
-		int tmp_dydx = (dydx_int << sft)
-			     + (dydx_frc >> (frc_bits - sft));
-		if (tmp_dydx <= max_dydx) {
-			*dydx = tmp_dydx;
-			*shift = sft;
-		}
-		if (tmp_dydx >= max_dydx)
-			break;
-	}
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ctc_gradient() leave:\n");
-}
-
-void
-ia_css_ctc_encode(
-	struct sh_css_isp_ctc_params *to,
-	const struct ia_css_ctc_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->y0 = from->y0;
-	to->y1 = from->y1;
-	to->y2 = from->y2;
-	to->y3 = from->y3;
-	to->y4 = from->y4;
-	to->y5 = from->y5;
-
-	to->ce_gain_exp = from->ce_gain_exp;
-
-	to->x1 = from->x1;
-	to->x2 = from->x2;
-	to->x3 = from->x3;
-	to->x4 = from->x4;
-
-	ctc_gradient(&(to->dydx0),
-		     &(to->dydx0_shift),
-		     from->y1, from->y0,
-		     from->x1, 0);
-
-	ctc_gradient(&(to->dydx1),
-		     &(to->dydx1_shift),
-		     from->y2, from->y1,
-		     from->x2, from->x1);
-
-	ctc_gradient(&to->dydx2,
-		     &to->dydx2_shift,
-		     from->y3, from->y2,
-		     from->x3, from->x2);
-
-	ctc_gradient(&to->dydx3,
-		     &to->dydx3_shift,
-		     from->y4, from->y3,
-		     from->x4, from->x3);
-
-	ctc_gradient(&(to->dydx4),
-		     &(to->dydx4_shift),
-		     from->y5, from->y4,
-		     SH_CSS_BAYER_MAXVAL, from->x4);
-}
-
-void
-ia_css_ctc_dump(
-	const struct sh_css_isp_ctc_params *ctc,
-	unsigned level);
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h
deleted file mode 100644
index d943aff28152..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CTC1_5_HOST_H
-#define __IA_CSS_CTC1_5_HOST_H
-
-#include "sh_css_params.h"
-
-#include "ia_css_ctc1_5_param.h"
-
-void
-ia_css_ctc_encode(
-	struct sh_css_isp_ctc_params *to,
-	const struct ia_css_ctc_config *from,
-	unsigned size);
-
-void
-ia_css_ctc_dump(
-	const struct sh_css_isp_ctc_params *ctc,
-	unsigned level);
-
-#endif /* __IA_CSS_CTC1_5_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h
deleted file mode 100644
index 8d9ac2b1832c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CTC1_5_PARAM_H
-#define __IA_CSS_CTC1_5_PARAM_H
-
-#include "type_support.h"
-#include "ctc/ctc_1.0/ia_css_ctc_param.h" /* vamem params */
-
-/* CTC (Color Tone Control) */
-struct sh_css_isp_ctc_params {
-	int32_t y0;
-	int32_t y1;
-	int32_t y2;
-	int32_t y3;
-	int32_t y4;
-	int32_t y5;
-	int32_t ce_gain_exp;
-	int32_t x1;
-	int32_t x2;
-	int32_t x3;
-	int32_t x4;
-	int32_t dydx0;
-	int32_t dydx0_shift;
-	int32_t dydx1;
-	int32_t dydx1_shift;
-	int32_t dydx2;
-	int32_t dydx2_shift;
-	int32_t dydx3;
-	int32_t dydx3_shift;
-	int32_t dydx4;
-	int32_t dydx4_shift;
-};
-
-#endif /* __IA_CSS_CTC1_5_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc_param.h
deleted file mode 100644
index dcd471f9bd66..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc1_5/ia_css_ctc_param.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CTCX_PARAM_H
-#define __IA_CSS_CTCX_PARAM_H
-
-#include "ia_css_ctc1_5_param.h"
-
-#endif /* __IA_CSS_CTCX_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c
deleted file mode 100644
index 07bd24edc7bf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "assert_support.h"
-
-#include "ia_css_ctc2.host.h"
-
-#define INEFFECTIVE_VAL 4096
-#define BASIC_VAL 819
-
-/*Default configuration of parameters for Ctc2*/
-const struct ia_css_ctc2_config default_ctc2_config = {
-	INEFFECTIVE_VAL, INEFFECTIVE_VAL, INEFFECTIVE_VAL,
-	INEFFECTIVE_VAL, INEFFECTIVE_VAL, INEFFECTIVE_VAL,
-	BASIC_VAL * 2, BASIC_VAL * 4, BASIC_VAL * 6,
-	BASIC_VAL * 8, INEFFECTIVE_VAL, INEFFECTIVE_VAL,
-	BASIC_VAL >> 1, BASIC_VAL};
-
-/* (dydx) = ctc2_slope(y1, y0, x1, x0)
- * -----------------------------------------------
- * Calculation of the Slope of a Line = ((y1 - y0) >> 8)/(x1 - x0)
- *
- * Note: y1, y0 , x1 & x0 must lie within the range 0 <-> 8191
- */
-static int ctc2_slope(int y1, int y0, int x1, int x0)
-{
-	const int shift_val = 8;
-	const int max_slope = (1 << IA_CSS_CTC_COEF_SHIFT) - 1;
-	int dy = y1 - y0;
-	int dx = x1 - x0;
-	int rounding = (dx + 1) >> 1;
-	int dy_shift = dy << shift_val;
-	int slope, dydx;
-
-	/*Protection for paramater values, & avoiding zero divisions*/
-	assert(y0 >= 0 && y0 <= max_slope);
-	assert(y1 >= 0 && y1 <= max_slope);
-	assert(x0 >= 0 && x0 <= max_slope);
-	assert(x1 > 0 && x1 <= max_slope);
-	assert(dx > 0);
-
-	if (dy < 0)
-		rounding = -rounding;
-	slope = (int) (dy_shift + rounding) / dx;
-
-	/*the slope must lie within the range
-	  (-max_slope-1) >= (dydx) >= (max_slope)
-	*/
-	if (slope <= -max_slope-1) {
-		dydx = -max_slope-1;
-	} else if (slope >= max_slope) {
-		dydx = max_slope;
-	} else {
-		dydx = slope;
-	}
-
-	return dydx;
-}
-
-/* (void) = ia_css_ctc2_vmem_encode(*to, *from)
- * -----------------------------------------------
- * VMEM Encode Function to translate Y parameters from userspace into ISP space
- */
-void ia_css_ctc2_vmem_encode(struct ia_css_isp_ctc2_vmem_params *to,
-			     const struct ia_css_ctc2_config *from,
-			     size_t size)
-{
-	unsigned i, j;
-	const unsigned shffl_blck = 4;
-	const unsigned lenght_zeros = 11;
-	short dydx0, dydx1, dydx2, dydx3, dydx4;
-
-	(void)size;
-	/*
-	*  Calculation of slopes of lines interconnecting
-	*  0.0 -> y_x1 -> y_x2 -> y _x3 -> y_x4 -> 1.0
-	*/
-	dydx0 = ctc2_slope(from->y_y1, from->y_y0,
-			    from->y_x1, 0);
-	dydx1 = ctc2_slope(from->y_y2, from->y_y1,
-			    from->y_x2, from->y_x1);
-	dydx2 = ctc2_slope(from->y_y3, from->y_y2,
-			    from->y_x3, from->y_x2);
-	dydx3 = ctc2_slope(from->y_y4, from->y_y3,
-			    from->y_x4, from->y_x3);
-	dydx4 = ctc2_slope(from->y_y5, from->y_y4,
-			    SH_CSS_BAYER_MAXVAL, from->y_x4);
-
-	/*Fill 3 arrays with:
-	 * - Luma input gain values y_y0, y_y1, y_y2, y_3, y_y4
-	 * - Luma kneepoints 0, y_x1, y_x2, y_x3, y_x4
-	 * - Calculated slopes dydx0, dyxd1, dydx2, dydx3, dydx4
-	 *
-	 * - Each 64-element array is divided in blocks of 16 elements:
-	 *   the 5 parameters + zeros in the remaining 11 positions
-	 * - All blocks of the same array will contain the same data
-	 */
-	for (i = 0; i < shffl_blck; i++) {
-		to->y_x[0][(i << shffl_blck)]     = 0;
-		to->y_x[0][(i << shffl_blck) + 1] = from->y_x1;
-		to->y_x[0][(i << shffl_blck) + 2] = from->y_x2;
-		to->y_x[0][(i << shffl_blck) + 3] = from->y_x3;
-		to->y_x[0][(i << shffl_blck) + 4] = from->y_x4;
-
-		to->y_y[0][(i << shffl_blck)]     = from->y_y0;
-		to->y_y[0][(i << shffl_blck) + 1] = from->y_y1;
-		to->y_y[0][(i << shffl_blck) + 2] = from->y_y2;
-		to->y_y[0][(i << shffl_blck) + 3] = from->y_y3;
-		to->y_y[0][(i << shffl_blck) + 4] = from->y_y4;
-
-		to->e_y_slope[0][(i << shffl_blck)]    = dydx0;
-		to->e_y_slope[0][(i << shffl_blck) + 1] = dydx1;
-		to->e_y_slope[0][(i << shffl_blck) + 2] = dydx2;
-		to->e_y_slope[0][(i << shffl_blck) + 3] = dydx3;
-		to->e_y_slope[0][(i << shffl_blck) + 4] = dydx4;
-
-		for (j = 0; j < lenght_zeros; j++) {
-			to->y_x[0][(i << shffl_blck) + 5 + j] = 0;
-			to->y_y[0][(i << shffl_blck) + 5 + j] = 0;
-			to->e_y_slope[0][(i << shffl_blck)+ 5 + j] = 0;
-		}
-	}
-}
-
-/* (void) = ia_css_ctc2_encode(*to, *from)
- * -----------------------------------------------
- * DMEM Encode Function to translate UV parameters from userspace into ISP space
- */
-void ia_css_ctc2_encode(struct ia_css_isp_ctc2_dmem_params *to,
-			struct ia_css_ctc2_config *from,
-			size_t size)
-{
-	(void)size;
-
-	to->uv_y0 = from->uv_y0;
-	to->uv_y1 = from->uv_y1;
-	to->uv_x0 = from->uv_x0;
-	to->uv_x1 = from->uv_x1;
-
-	/*Slope Calculation*/
-	to->uv_dydx = ctc2_slope(from->uv_y1, from->uv_y0,
-				  from->uv_x1, from->uv_x0);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h
deleted file mode 100644
index 3733aee24dcd..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CTC2_HOST_H
-#define __IA_CSS_CTC2_HOST_H
-
-#include "ia_css_ctc2_param.h"
-#include "ia_css_ctc2_types.h"
-
-extern const struct ia_css_ctc2_config default_ctc2_config;
-
-/*Encode Functions to translate parameters from userspace into ISP space*/
-
-void ia_css_ctc2_vmem_encode(struct ia_css_isp_ctc2_vmem_params *to,
-			     const struct ia_css_ctc2_config *from,
-			     size_t size);
-
-void ia_css_ctc2_encode(struct ia_css_isp_ctc2_dmem_params *to,
-			struct ia_css_ctc2_config *from,
-			size_t size);
-
-#endif /* __IA_CSS_CTC2_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h
deleted file mode 100644
index ad7040c9d7cb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CTC2_PARAM_H
-#define __IA_CSS_CTC2_PARAM_H
-
-#define IA_CSS_CTC_COEF_SHIFT          13
-#include "vmem.h" /* needed for VMEM_ARRAY */
-
-/* CTC (Chroma Tone Control)ISP Parameters */
-
-/*VMEM Luma params*/
-struct ia_css_isp_ctc2_vmem_params {
-	/** Gains by Y(Luma) at Y = 0.0,Y_X1, Y_X2, Y_X3, Y_X4*/
-	VMEM_ARRAY(y_x, ISP_VEC_NELEMS);
-	/* kneepoints by Y(Luma) 0.0, y_x1, y_x2, y _x3, y_x4*/
-	VMEM_ARRAY(y_y, ISP_VEC_NELEMS);
-	/* Slopes of lines interconnecting
-	 *  0.0 -> y_x1 -> y_x2 -> y _x3 -> y_x4 -> 1.0*/
-	VMEM_ARRAY(e_y_slope, ISP_VEC_NELEMS);
-};
-
-/*DMEM Chroma params*/
-struct ia_css_isp_ctc2_dmem_params {
-
-	/* Gains by UV(Chroma) under kneepoints uv_x0 and uv_x1*/
-	int32_t uv_y0;
-	int32_t uv_y1;
-
-	/* Kneepoints by UV(Chroma)- uv_x0 and uv_x1*/
-	int32_t uv_x0;
-	int32_t uv_x1;
-
-	/* Slope of line interconnecting uv_x0 -> uv_x1*/
-	int32_t uv_dydx;
-
-};
-#endif /* __IA_CSS_CTC2_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h
deleted file mode 100644
index 1222cf33e851..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CTC2_TYPES_H
-#define __IA_CSS_CTC2_TYPES_H
-
-/* Chroma Tone Control configuration.
-*
-*  ISP block: CTC2 (CTC by polygonal approximation)
-* (ISP1: CTC1 (CTC by look-up table) is used.)
-*  ISP2: CTC2 is used.
-*  ISP261: CTC2 (CTC by Fast Approximate Distance)
-*/
-struct ia_css_ctc2_config {
-
-	/** Gains by Y(Luma) at Y =0.0,Y_X1, Y_X2, Y_X3, Y_X4 and Y_X5
-	*   --default/ineffective value: 4096(0.5f)
-	*/
-	int32_t y_y0;
-	int32_t y_y1;
-	int32_t y_y2;
-	int32_t y_y3;
-	int32_t y_y4;
-	int32_t y_y5;
-	/* 1st-4th  kneepoints by Y(Luma) --default/ineffective value:n/a
-	*   requirement: 0.0 < y_x1 < y_x2 <y _x3 < y_x4 < 1.0
-	*/
-	int32_t y_x1;
-	int32_t y_x2;
-	int32_t y_x3;
-	int32_t y_x4;
-	/* Gains by UV(Chroma) under threholds uv_x0 and uv_x1
-	*   --default/ineffective value: 4096(0.5f)
-	*/
-	int32_t uv_y0;
-	int32_t uv_y1;
-	/* Minimum and Maximum Thresholds by UV(Chroma)- uv_x0 and uv_x1
-	*   --default/ineffective value: n/a
-	*/
-	int32_t uv_x0;
-	int32_t uv_x1;
-	};
-
-#endif /* __IA_CSS_CTC2_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c
deleted file mode 100644
index 7c1a367918a0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-#include "ia_css_ctc.host.h"
-
-const struct ia_css_ctc_config default_ctc_config = {
-	((1 << IA_CSS_CTC_COEF_SHIFT) + 1) / 2,		/* 0.5 */
-	((1 << IA_CSS_CTC_COEF_SHIFT) + 1) / 2,		/* 0.5 */
-	((1 << IA_CSS_CTC_COEF_SHIFT) + 1) / 2,		/* 0.5 */
-	((1 << IA_CSS_CTC_COEF_SHIFT) + 1) / 2,		/* 0.5 */
-	((1 << IA_CSS_CTC_COEF_SHIFT) + 1) / 2,		/* 0.5 */
-	((1 << IA_CSS_CTC_COEF_SHIFT) + 1) / 2,		/* 0.5 */
-	1,
-	SH_CSS_BAYER_MAXVAL / 5,	/* To be implemented */
-	SH_CSS_BAYER_MAXVAL * 2 / 5,	/* To be implemented */
-	SH_CSS_BAYER_MAXVAL * 3 / 5,	/* To be implemented */
-	SH_CSS_BAYER_MAXVAL * 4 / 5,	/* To be implemented */
-};
-
-void
-ia_css_ctc_vamem_encode(
-	struct sh_css_isp_ctc_vamem_params *to,
-	const struct ia_css_ctc_table *from,
-	unsigned size)
-{
-	(void)size;
-	memcpy (&to->ctc,  &from->data, sizeof(to->ctc));
-}
-
-void
-ia_css_ctc_debug_dtrace(
-	const struct ia_css_ctc_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.ce_gain_exp=%d, config.y0=%d, "
-		"config.x1=%d, config.y1=%d, "
-		"config.x2=%d, config.y2=%d, "
-		"config.x3=%d, config.y3=%d, "
-		"config.x4=%d, config.y4=%d\n",
-		config->ce_gain_exp, config->y0,
-		config->x1, config->y1,
-		config->x2, config->y2,
-		config->x3, config->y3,
-		config->x4, config->y4);
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h
deleted file mode 100644
index bec52a6519f9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CTC_HOST_H
-#define __IA_CSS_CTC_HOST_H
-
-#include "sh_css_params.h"
-
-#include "ia_css_ctc_param.h"
-#include "ia_css_ctc_table.host.h"
-
-extern const struct ia_css_ctc_config default_ctc_config;
-
-void
-ia_css_ctc_vamem_encode(
-	struct sh_css_isp_ctc_vamem_params *to,
-	const struct ia_css_ctc_table *from,
-	unsigned size);
-
-void
-ia_css_ctc_debug_dtrace(
-	const struct ia_css_ctc_config *config, unsigned level)
-;
-
-#endif /* __IA_CSS_CTC_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h
deleted file mode 100644
index 6e88ad3d2420..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CTC_PARAM_H
-#define __IA_CSS_CTC_PARAM_H
-
-#include "type_support.h"
-#include <system_global.h>
-
-#include "ia_css_ctc_types.h"
-
-#ifndef PIPE_GENERATION
-#if defined(HAS_VAMEM_VERSION_2)
-#define SH_CSS_ISP_CTC_TABLE_SIZE_LOG2       IA_CSS_VAMEM_2_CTC_TABLE_SIZE_LOG2
-#define SH_CSS_ISP_CTC_TABLE_SIZE            IA_CSS_VAMEM_2_CTC_TABLE_SIZE
-#elif defined(HAS_VAMEM_VERSION_1)
-#define SH_CSS_ISP_CTC_TABLE_SIZE_LOG2       IA_CSS_VAMEM_1_CTC_TABLE_SIZE_LOG2
-#define SH_CSS_ISP_CTC_TABLE_SIZE            IA_CSS_VAMEM_1_CTC_TABLE_SIZE
-#else
-#error "VAMEM should be {VERSION1, VERSION2}" 
-#endif
-
-#else
-/* For pipe generation, the size is not relevant */
-#define SH_CSS_ISP_CTC_TABLE_SIZE 0
-#endif
-
-/* This should be vamem_data_t, but that breaks the pipe generator */
-struct sh_css_isp_ctc_vamem_params {
-	uint16_t ctc[SH_CSS_ISP_CTC_TABLE_SIZE];
-};
-
-#endif /* __IA_CSS_CTC_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c
deleted file mode 100644
index edf85aba7716..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <type_support.h>
-#include <string_support.h> /* memcpy */
-#include "system_global.h"
-#include "vamem.h"
-#include "ia_css_types.h"
-#include "ia_css_ctc_table.host.h"
-
-struct ia_css_ctc_table       default_ctc_table;
-
-#if defined(HAS_VAMEM_VERSION_2)
-
-static const uint16_t
-default_ctc_table_data[IA_CSS_VAMEM_2_CTC_TABLE_SIZE] = {
-   0,  384,  837,  957, 1011, 1062, 1083, 1080,
-1078, 1077, 1053, 1039, 1012,  992,  969,  951,
- 929,  906,  886,  866,  845,  823,  809,  790,
- 772,  758,  741,  726,  711,  701,  688,  675,
- 666,  656,  648,  639,  633,  626,  618,  612,
- 603,  594,  582,  572,  557,  545,  529,  516,
- 504,  491,  480,  467,  459,  447,  438,  429,
- 419,  412,  404,  397,  389,  382,  376,  368,
- 363,  357,  351,  345,  340,  336,  330,  326,
- 321,  318,  312,  308,  304,  300,  297,  294,
- 291,  286,  284,  281,  278,  275,  271,  268,
- 261,  257,  251,  245,  240,  235,  232,  225,
- 223,  218,  213,  209,  206,  204,  199,  197,
- 193,  189,  186,  185,  183,  179,  177,  175,
- 172,  170,  169,  167,  164,  164,  162,  160,
- 158,  157,  156,  154,  154,  152,  151,  150,
- 149,  148,  146,  147,  146,  144,  143,  143,
- 142,  141,  140,  141,  139,  138,  138,  138,
- 137,  136,  136,  135,  134,  134,  134,  133,
- 132,  132,  131,  130,  131,  130,  129,  128,
- 129,  127,  127,  127,  127,  125,  125,  125,
- 123,  123,  122,  120,  118,  115,  114,  111,
- 110,  108,  106,  105,  103,  102,  100,   99,
-  97,   97,   96,   95,   94,   93,   93,   91,
-  91,   91,   90,   90,   89,   89,   88,   88,
-  89,   88,   88,   87,   87,   87,   87,   86,
-  87,   87,   86,   87,   86,   86,   84,   84,
-  82,   80,   78,   76,   74,   72,   70,   68,
-  67,   65,   62,   60,   58,   56,   55,   54,
-  53,   51,   49,   49,   47,   45,   45,   45,
-  41,   40,   39,   39,   34,   33,   34,   32,
-  25,   23,   24,   20,   13,    9,   12,    0,
-   0
-};
-
-#elif defined(HAS_VAMEM_VERSION_1)
-
-/* Default Parameters */
-static const uint16_t
-default_ctc_table_data[IA_CSS_VAMEM_1_CTC_TABLE_SIZE] = {
-		0, 0, 256, 384, 384, 497, 765, 806,
-		837, 851, 888, 901, 957, 981, 993, 1001,
-		1011, 1029, 1028, 1039, 1062, 1059, 1073, 1080,
-		1083, 1085, 1085, 1098, 1080, 1084, 1085, 1093,
-		1078, 1073, 1070, 1069, 1077, 1066, 1072, 1063,
-		1053, 1044, 1046, 1053, 1039, 1028, 1025, 1024,
-		1012, 1013, 1016, 996, 992, 990, 990, 980,
-		969, 968, 961, 955, 951, 949, 933, 930,
-		929, 925, 921, 916, 906, 901, 895, 893,
-		886, 877, 872, 869, 866, 861, 857, 849,
-		845, 838, 836, 832, 823, 821, 815, 813,
-		809, 805, 796, 793, 790, 785, 784, 778,
-		772, 768, 766, 763, 758, 752, 749, 745,
-		741, 740, 736, 730, 726, 724, 723, 718,
-		711, 709, 706, 704, 701, 698, 691, 689,
-		688, 683, 683, 678, 675, 673, 671, 669,
-		666, 663, 661, 660, 656, 656, 653, 650,
-		648, 647, 646, 643, 639, 638, 637, 635,
-		633, 632, 629, 627, 626, 625, 622, 621,
-		618, 618, 614, 614, 612, 609, 606, 606,
-		603, 600, 600, 597, 594, 591, 590, 586,
-		582, 581, 578, 575, 572, 569, 563, 560,
-		557, 554, 551, 548, 545, 539, 536, 533,
-		529, 527, 524, 519, 516, 513, 510, 507,
-		504, 501, 498, 493, 491, 488, 485, 484,
-		480, 476, 474, 471, 467, 466, 464, 460,
-		459, 455, 453, 449, 447, 446, 443, 441,
-		438, 435, 432, 432, 429, 427, 426, 422,
-		419, 418, 416, 414, 412, 410, 408, 406,
-		404, 402, 401, 398, 397, 395, 393, 390,
-		389, 388, 387, 384, 382, 380, 378, 377,
-		376, 375, 372, 370, 368, 368, 366, 364,
-		363, 361, 360, 358, 357, 355, 354, 352,
-		351, 350, 349, 346, 345, 344, 344, 342,
-		340, 339, 337, 337, 336, 335, 333, 331,
-		330, 329, 328, 326, 326, 324, 324, 322,
-		321, 320, 318, 318, 318, 317, 315, 313,
-		312, 311, 311, 310, 308, 307, 306, 306,
-		304, 304, 302, 301, 300, 300, 299, 297,
-		297, 296, 296, 294, 294, 292, 291, 291,
-		291, 290, 288, 287, 286, 286, 287, 285,
-		284, 283, 282, 282, 281, 281, 279, 278,
-		278, 278, 276, 276, 275, 274, 274, 273,
-		271, 270, 269, 268, 268, 267, 265, 262,
-		261, 260, 260, 259, 257, 254, 252, 252,
-		251, 251, 249, 246, 245, 244, 243, 242,
-		240, 239, 239, 237, 235, 235, 233, 231,
-		232, 230, 229, 226, 225, 224, 225, 224,
-		223, 220, 219, 219, 218, 217, 217, 214,
-		213, 213, 212, 211, 209, 209, 209, 208,
-		206, 205, 204, 203, 204, 203, 201, 200,
-		199, 197, 198, 198, 197, 195, 194, 194,
-		193, 192, 192, 191, 189, 190, 189, 188,
-		186, 187, 186, 185, 185, 184, 183, 181,
-		183, 182, 181, 180, 179, 178, 178, 178,
-		177, 176, 175, 176, 175, 174, 174, 173,
-		172, 173, 172, 171, 170, 170, 169, 169,
-		169, 168, 167, 166, 167, 167, 166, 165,
-		164, 164, 164, 163, 164, 163, 162, 163,
-		162, 161, 160, 161, 160, 160, 160, 159,
-		158, 157, 158, 158, 157, 157, 156, 156,
-		156, 156, 155, 155, 154, 154, 154, 154,
-		154, 153, 152, 153, 152, 152, 151, 152,
-		151, 152, 151, 150, 150, 149, 149, 150,
-		149, 149, 148, 148, 148, 149, 148, 147,
-		146, 146, 147, 146, 147, 146, 145, 146,
-		146, 145, 144, 145, 144, 145, 144, 144,
-		143, 143, 143, 144, 143, 142, 142, 142,
-		142, 142, 142, 141, 141, 141, 141, 140,
-		140, 141, 140, 140, 141, 140, 139, 139,
-		139, 140, 139, 139, 138, 138, 137, 139,
-		138, 138, 138, 137, 138, 137, 137, 137,
-		137, 136, 137, 136, 136, 136, 136, 135,
-		136, 135, 135, 135, 135, 136, 135, 135,
-		134, 134, 133, 135, 134, 134, 134, 133,
-		134, 133, 134, 133, 133, 132, 133, 133,
-		132, 133, 132, 132, 132, 132, 131, 131,
-		131, 132, 131, 131, 130, 131, 130, 132,
-		131, 130, 130, 129, 130, 129, 130, 129,
-		129, 129, 130, 129, 128, 128, 128, 128,
-		129, 128, 128, 127, 127, 128, 128, 127,
-		127, 126, 126, 127, 127, 126, 126, 126,
-		127, 126, 126, 126, 125, 125, 126, 125,
-		125, 124, 124, 124, 125, 125, 124, 124,
-		123, 124, 124, 123, 123, 122, 122, 122,
-		122, 122, 121, 120, 120, 119, 118, 118,
-		118, 117, 117, 116, 115, 115, 115, 114,
-		114, 113, 113, 112, 111, 111, 111, 110,
-		110, 109, 109, 108, 108, 108, 107, 107,
-		106, 106, 105, 105, 105, 104, 104, 103,
-		103, 102, 102, 102, 102, 101, 101, 100,
-		100, 99, 99, 99, 99, 99, 99, 98,
-		97, 98, 97, 97, 97, 96, 96, 95,
-		96, 95, 96, 95, 95, 94, 94, 95,
-		94, 94, 94, 93, 93, 92, 93, 93,
-		93, 93, 92, 92, 91, 92, 92, 92,
-		91, 91, 90, 90, 91, 91, 91, 90,
-		90, 90, 90, 91, 90, 90, 90, 89,
-		89, 89, 90, 89, 89, 89, 89, 89,
-		88, 89, 89, 88, 88, 88, 88, 87,
-		89, 88, 88, 88, 88, 88, 87, 88,
-		88, 88, 87, 87, 87, 87, 87, 88,
-		87, 87, 87, 87, 87, 87, 88, 87,
-		87, 87, 87, 86, 86, 87, 87, 87,
-		87, 86, 86, 86, 87, 87, 86, 87,
-		86, 86, 86, 87, 87, 86, 86, 86,
-		86, 86, 87, 87, 86, 85, 85, 85,
-		84, 85, 85, 84, 84, 83, 83, 82,
-		82, 82, 81, 81, 80, 79, 79, 79,
-		78, 77, 77, 76, 76, 76, 75, 74,
-		74, 74, 73, 73, 72, 71, 71, 71,
-		70, 70, 69, 69, 68, 68, 67, 67,
-		67, 66, 66, 65, 65, 64, 64, 63,
-		62, 62, 62, 61, 60, 60, 59, 59,
-		58, 58, 57, 57, 56, 56, 56, 55,
-		55, 54, 55, 55, 54, 53, 53, 52,
-		53, 53, 52, 51, 51, 50, 51, 50,
-		49, 49, 50, 49, 49, 48, 48, 47,
-		47, 48, 46, 45, 45, 45, 46, 45,
-		45, 44, 45, 45, 45, 43, 42, 42,
-		41, 43, 41, 40, 40, 39, 40, 41,
-		39, 39, 39, 39, 39, 38, 35, 35,
-		34, 37, 36, 34, 33, 33, 33, 35,
-		34, 32, 32, 31, 32, 30, 29, 26,
-		25, 25, 27, 26, 23, 23, 23, 25,
-		24, 24, 22, 21, 20, 19, 16, 14,
-		13, 13, 13, 10, 9, 7, 7, 7,
-		12, 12, 12, 7, 0, 0, 0, 0
-};
-
-#else
-#error "VAMEM version must be one of {VAMEM_VERSION_1, VAMEM_VERSION_2}"
-#endif
-
-void
-ia_css_config_ctc_table(void)
-{
-#if defined(HAS_VAMEM_VERSION_2)
-	memcpy(default_ctc_table.data.vamem_2, default_ctc_table_data,
-	       sizeof(default_ctc_table_data));
-	default_ctc_table.vamem_type     = IA_CSS_VAMEM_TYPE_2;
-#else
-	memcpy(default_ctc_table.data.vamem_1, default_ctc_table_data,
-	       sizeof(default_ctc_table_data));
-	default_ctc_table.vamem_type     = 1IA_CSS_VAMEM_TYPE_1;
-#endif
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h
deleted file mode 100644
index a350dec8b4ad..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CTC_TABLE_HOST_H
-#define __IA_CSS_CTC_TABLE_HOST_H
-
-#include "ia_css_ctc_types.h"
-
-extern struct ia_css_ctc_table default_ctc_table;
-
-void ia_css_config_ctc_table(void);
-
-#endif /* __IA_CSS_CTC_TABLE_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h
deleted file mode 100644
index 4ac47ce10566..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_CTC_TYPES_H
-#define __IA_CSS_CTC_TYPES_H
-
-/* @file
-* CSS-API header file for Chroma Tone Control parameters.
-*/
-
-/* Fractional bits for CTC gain (used only for ISP1).
- *
- *  IA_CSS_CTC_COEF_SHIFT(=13) includes not only the fractional bits
- *  of gain(=8), but also the bits(=5) to convert chroma
- *  from 13bit precision to 8bit precision.
- *
- *    Gain (struct ia_css_ctc_table) : u5.8
- *    Input(Chorma) : s0.12 (13bit precision)
- *    Output(Chorma): s0.7  (8bit precision)
- *    Output = (Input * Gain) >> IA_CSS_CTC_COEF_SHIFT
- */
-#define IA_CSS_CTC_COEF_SHIFT          13
-
-/* Number of elements in the CTC table. */
-#define IA_CSS_VAMEM_1_CTC_TABLE_SIZE_LOG2      10
-/* Number of elements in the CTC table. */
-#define IA_CSS_VAMEM_1_CTC_TABLE_SIZE           (1U<<IA_CSS_VAMEM_1_CTC_TABLE_SIZE_LOG2)
-
-/* Number of elements in the CTC table. */
-#define IA_CSS_VAMEM_2_CTC_TABLE_SIZE_LOG2      8
-/* Number of elements in the CTC table. */
-#define IA_CSS_VAMEM_2_CTC_TABLE_SIZE           ((1U<<IA_CSS_VAMEM_2_CTC_TABLE_SIZE_LOG2) + 1)
-
-enum ia_css_vamem_type {
-	IA_CSS_VAMEM_TYPE_1,
-	IA_CSS_VAMEM_TYPE_2
-};
-
-/* Chroma Tone Control configuration.
- *
- *  ISP block: CTC2 (CTC by polygonal line approximation)
- * (ISP1: CTC1 (CTC by look-up table) is used.)
- *  ISP2: CTC2 is used.
- */
-struct ia_css_ctc_config {
-	uint16_t y0;	/** 1st kneepoint gain.
-				u[ce_gain_exp].[13-ce_gain_exp], [0,8191],
-				default/ineffective 4096(0.5) */
-	uint16_t y1;	/** 2nd kneepoint gain.
-				u[ce_gain_exp].[13-ce_gain_exp], [0,8191],
-				default/ineffective 4096(0.5) */
-	uint16_t y2;	/** 3rd kneepoint gain.
-				u[ce_gain_exp].[13-ce_gain_exp], [0,8191],
-				default/ineffective 4096(0.5) */
-	uint16_t y3;	/** 4th kneepoint gain.
-				u[ce_gain_exp].[13-ce_gain_exp], [0,8191],
-				default/ineffective 4096(0.5) */
-	uint16_t y4;	/** 5th kneepoint gain.
-				u[ce_gain_exp].[13-ce_gain_exp], [0,8191],
-				default/ineffective 4096(0.5) */
-	uint16_t y5;	/** 6th kneepoint gain.
-				u[ce_gain_exp].[13-ce_gain_exp], [0,8191],
-				default/ineffective 4096(0.5) */
-	uint16_t ce_gain_exp;	/** Common exponent of y-axis gain.
-				u8.0, [0,13],
-				default/ineffective 1 */
-	uint16_t x1;	/** 2nd kneepoint luma.
-				u0.13, [0,8191], constraints: 0<x1<x2,
-				default/ineffective 1024 */
-	uint16_t x2;	/** 3rd kneepoint luma.
-				u0.13, [0,8191], constraints: x1<x2<x3,
-				default/ineffective 2048 */
-	uint16_t x3;	/** 4th kneepoint luma.
-				u0.13, [0,8191], constraints: x2<x3<x4,
-				default/ineffective 6144 */
-	uint16_t x4;	/** 5tn kneepoint luma.
-				u0.13, [0,8191], constraints: x3<x4<8191,
-				default/ineffective 7168 */
-};
-
-union ia_css_ctc_data {
-	uint16_t vamem_1[IA_CSS_VAMEM_1_CTC_TABLE_SIZE];
-	uint16_t vamem_2[IA_CSS_VAMEM_2_CTC_TABLE_SIZE];
-};
-
-/* CTC table, used for Chroma Tone Control.
- *
- *  ISP block: CTC1 (CTC by look-up table)
- *  ISP1: CTC1 is used.
- * (ISP2: CTC2 (CTC by polygonal line approximation) is used.)
- */
-struct ia_css_ctc_table {
-	enum ia_css_vamem_type vamem_type;
-	union ia_css_ctc_data data;
-};
-
-#endif /* __IA_CSS_CTC_TYPES_H */
-
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de.host.c
deleted file mode 100644
index fbab2f1c396c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de.host.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "sh_css_frac.h"
-#include "ia_css_de.host.h"
-
-const struct ia_css_de_config default_de_config = {
-	0,
-	0,
-	0
-};
-
-void
-ia_css_de_encode(
-	struct sh_css_isp_de_params *to,
-	const struct ia_css_de_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->pixelnoise =
-	    uDIGIT_FITTING(from->pixelnoise, 16, SH_CSS_BAYER_BITS);
-	to->c1_coring_threshold =
-	    uDIGIT_FITTING(from->c1_coring_threshold, 16,
-			   SH_CSS_BAYER_BITS);
-	to->c2_coring_threshold =
-	    uDIGIT_FITTING(from->c2_coring_threshold, 16,
-			   SH_CSS_BAYER_BITS);
-}
-
-void
-ia_css_de_dump(
-	const struct sh_css_isp_de_params *de,
-	unsigned level)
-{
-	if (!de) return;
-	ia_css_debug_dtrace(level, "Demosaic:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"de_pixelnoise", de->pixelnoise);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"de_c1_coring_threshold",
-			de->c1_coring_threshold);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"de_c2_coring_threshold",
-			de->c2_coring_threshold);
-}
-
-void
-ia_css_de_debug_dtrace(
-	const struct ia_css_de_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.pixelnoise=%d, "
-		"config.c1_coring_threshold=%d, config.c2_coring_threshold=%d\n",
-		config->pixelnoise,
-		config->c1_coring_threshold, config->c2_coring_threshold);
-}
-
-void
-ia_css_init_de_state(
-	void/*struct sh_css_isp_de_vmem_state*/ *state,
-	size_t size)
-{
-	memset(state, 0, size);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de.host.h
deleted file mode 100644
index 5dd6f06f2bf1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de.host.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DE_HOST_H
-#define __IA_CSS_DE_HOST_H
-
-#include "ia_css_de_types.h"
-#include "ia_css_de_param.h"
-
-extern const struct ia_css_de_config default_de_config;
-
-void
-ia_css_de_encode(
-	struct sh_css_isp_de_params *to,
-	const struct ia_css_de_config *from,
-	unsigned size);
-
-void
-ia_css_de_dump(
-	const struct sh_css_isp_de_params *de,
-	unsigned level);
-
-void
-ia_css_de_debug_dtrace(
-	const struct ia_css_de_config *config,
-	unsigned level);
-
-void
-ia_css_init_de_state(
-	void/*struct sh_css_isp_de_vmem_state*/ *state,
-	size_t size);
-
-#endif /* __IA_CSS_DE_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_param.h
deleted file mode 100644
index 833c80afc7a8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_param.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DE_PARAM_H
-#define __IA_CSS_DE_PARAM_H
-
-#include "type_support.h"
-
-/* DE (Demosaic) */
-struct sh_css_isp_de_params {
-	int32_t pixelnoise;
-	int32_t c1_coring_threshold;
-	int32_t c2_coring_threshold;
-};
-
-#endif /* __IA_CSS_DE_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_state.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_state.h
deleted file mode 100644
index d64511763436..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_state.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DE_STATE_H
-#define __IA_CSS_DE_STATE_H
-
-#include "type_support.h"
-#include "vmem.h"
-
-/* DE (Demosaic) */
-struct sh_css_isp_de_vmem_state {
-	VMEM_ARRAY(de_buf[4], MAX_VECTORS_PER_BUF_LINE*ISP_NWAY);
-};
-
-#endif /* __IA_CSS_DE_STATE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_types.h
deleted file mode 100644
index 803be68abc54..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_1.0/ia_css_de_types.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DE_TYPES_H
-#define __IA_CSS_DE_TYPES_H
-
-/* @file
-* CSS-API header file for Demosaic (bayer-to-YCgCo) parameters.
-*/
-
-/* Demosaic (bayer-to-YCgCo) configuration.
- *
- *  ISP block: DE1
- *  ISP1: DE1 is used.
- * (ISP2: DE2 is used.)
- */
-struct ia_css_de_config {
-	ia_css_u0_16 pixelnoise; /** Pixel noise used in moire elimination.
-				u0.16, [0,65535],
-				default 0, ineffective 0 */
-	ia_css_u0_16 c1_coring_threshold; /** Coring threshold for C1.
-				This is the same as nr_config.threshold_cb.
-				u0.16, [0,65535],
-				default 128(0.001953125), ineffective 0 */
-	ia_css_u0_16 c2_coring_threshold; /** Coring threshold for C2.
-				This is the same as nr_config.threshold_cr.
-				u0.16, [0,65535],
-				default 128(0.001953125), ineffective 0 */
-};
-
-#endif /* __IA_CSS_DE_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2.host.c
deleted file mode 100644
index a5247a57bafb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2.host.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-
-#include "ia_css_de2.host.h"
-
-const struct ia_css_ecd_config default_ecd_config = {
-	(1 << (ISP_VEC_ELEMBITS - 1)) * 2 / 3,	/* 2/3 */
-	(1 << (ISP_VEC_ELEMBITS - 1)) - 1,	/* 1.0 */
-	0,					/* 0.0 */
-};
-
-void
-ia_css_ecd_encode(
-	struct sh_css_isp_ecd_params *to,
-	const struct ia_css_ecd_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->zip_strength = from->zip_strength;
-	to->fc_strength  = from->fc_strength;
-	to->fc_debias    = from->fc_debias;
-}
-
-void
-ia_css_ecd_dump(
-	const struct sh_css_isp_ecd_params *ecd,
-	unsigned level);
-
-void
-ia_css_ecd_debug_dtrace(
-	const struct ia_css_ecd_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.zip_strength=%d, "
-		"config.fc_strength=%d, config.fc_debias=%d\n",
-		config->zip_strength,
-		config->fc_strength, config->fc_debias);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2.host.h
deleted file mode 100644
index f7cd8448cb30..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2.host.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DE2_HOST_H
-#define __IA_CSS_DE2_HOST_H
-
-#include "ia_css_de2_types.h"
-#include "ia_css_de2_param.h"
-
-extern const struct ia_css_ecd_config default_ecd_config;
-
-void
-ia_css_ecd_encode(
-	struct sh_css_isp_ecd_params *to,
-	const struct ia_css_ecd_config *from,
-	unsigned size);
-
-void
-ia_css_ecd_dump(
-	const struct sh_css_isp_ecd_params *ecd,
-	unsigned level);
-
-void
-ia_css_ecd_debug_dtrace(
-	const struct ia_css_ecd_config *config, unsigned level);
-
-#endif /* __IA_CSS_DE2_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2_param.h
deleted file mode 100644
index ea2da73a4927..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2_param.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DE2_PARAM_H
-#define __IA_CSS_DE2_PARAM_H
-
-#include "type_support.h"
-
-/* Reuse DE1 params and extend them */
-#include "../de_1.0/ia_css_de_param.h"
-
-/* DE (Demosaic) */
-struct sh_css_isp_ecd_params {
-	int32_t zip_strength;
-	int32_t fc_strength;
-	int32_t fc_debias;
-};
-
-#endif /* __IA_CSS_DE2_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2_types.h
deleted file mode 100644
index 50bdde419bb1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de2_types.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DE2_TYPES_H
-#define __IA_CSS_DE2_TYPES_H
-
-/* @file
-* CSS-API header file for Demosaicing parameters.
-*/
-
-/* Eigen Color Demosaicing configuration.
- *
- *  ISP block: DE2
- * (ISP1: DE1 is used.)
- *  ISP2: DE2 is used.
- */
-struct ia_css_ecd_config {
-	uint16_t zip_strength;	/** Strength of zipper reduction.
-				u0.13, [0,8191],
-				default 5489(0.67), ineffective 0 */
-	uint16_t fc_strength;	/** Strength of false color reduction.
-				u0.13, [0,8191],
-				default 8191(almost 1.0), ineffective 0 */
-	uint16_t fc_debias;	/** Prevent color change
-				     on noise or Gr/Gb imbalance.
-				u0.13, [0,8191],
-				default 0, ineffective 0 */
-};
-
-#endif /* __IA_CSS_DE2_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de_param.h
deleted file mode 100644
index 59af9523604d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de_param.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DEX_PARAM_H
-#define __IA_CSS_DEX_PARAM_H
-
-#include "ia_css_de2_param.h"
-
-#endif /* __IA_CSS_DEX_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de_state.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de_state.h
deleted file mode 100644
index f2c65ba58983..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/de/de_2/ia_css_de_state.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DE2_STATE_H
-#define __IA_CSS_DE2_STATE_H
-
-/* Reuse DE1 states */
-#include "../de_1.0/ia_css_de_state.h"
-
-#endif /* __IA_CSS_DE2_STATE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.c
deleted file mode 100644
index b1f9dc8d662d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "sh_css_frac.h"
-
-#include "ia_css_dp.host.h"
-
-#ifdef ISP2401
-/* We use a different set of DPC configuration parameters when
- * DPC is used before OBC and NORM. Currently these parameters
- * are used in usecases which selects both BDS and DPC.
- **/
-const struct ia_css_dp_config default_dp_10bpp_config = {
-	1024,
-	2048,
-	32768,
-	32768,
-	32768,
-	32768
-};
-#endif
-const struct ia_css_dp_config default_dp_config = {
-	8192,
-	2048,
-	32768,
-	32768,
-	32768,
-	32768
-};
-
-void
-ia_css_dp_encode(
-	struct sh_css_isp_dp_params *to,
-	const struct ia_css_dp_config *from,
-	unsigned size)
-{
-	int gain = from->gain;
-	int gr   = from->gr;
-	int r    = from->r;
-	int b    = from->b;
-	int gb   = from->gb;
-
-	(void)size;
-	to->threshold_single =
-	    SH_CSS_BAYER_MAXVAL;
-	to->threshold_2adjacent =
-	    uDIGIT_FITTING(from->threshold, 16, SH_CSS_BAYER_BITS);
-	to->gain =
-	    uDIGIT_FITTING(from->gain, 8, SH_CSS_DP_GAIN_SHIFT);
-
-	to->coef_rr_gr =
-	    uDIGIT_FITTING (gain * gr / r, 8, SH_CSS_DP_GAIN_SHIFT);
-	to->coef_rr_gb =
-	    uDIGIT_FITTING (gain * gb / r, 8, SH_CSS_DP_GAIN_SHIFT);
-	to->coef_bb_gb =
-	    uDIGIT_FITTING (gain * gb / b, 8, SH_CSS_DP_GAIN_SHIFT);
-	to->coef_bb_gr =
-	    uDIGIT_FITTING (gain * gr / b, 8, SH_CSS_DP_GAIN_SHIFT);
-	to->coef_gr_rr =
-	    uDIGIT_FITTING (gain * r / gr, 8, SH_CSS_DP_GAIN_SHIFT);
-	to->coef_gr_bb =
-	    uDIGIT_FITTING (gain * b / gr, 8, SH_CSS_DP_GAIN_SHIFT);
-	to->coef_gb_bb =
-	    uDIGIT_FITTING (gain * b / gb, 8, SH_CSS_DP_GAIN_SHIFT);
-	to->coef_gb_rr =
-	    uDIGIT_FITTING (gain * r / gb, 8, SH_CSS_DP_GAIN_SHIFT);
-}
-
-void
-ia_css_dp_dump(
-	const struct sh_css_isp_dp_params *dp,
-	unsigned level)
-{
-	if (!dp) return;
-	ia_css_debug_dtrace(level, "Defect Pixel Correction:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"dp_threshold_single_w_2adj_on",
-		dp->threshold_single);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"dp_threshold_2adj_w_2adj_on",
-		dp->threshold_2adjacent);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"dp_gain", dp->gain);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"dpc_coef_rr_gr", dp->coef_rr_gr);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"dpc_coef_rr_gb", dp->coef_rr_gb);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"dpc_coef_bb_gb", dp->coef_bb_gb);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"dpc_coef_bb_gr", dp->coef_bb_gr);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"dpc_coef_gr_rr", dp->coef_gr_rr);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"dpc_coef_gr_bb", dp->coef_gr_bb);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"dpc_coef_gb_bb", dp->coef_gb_bb);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"dpc_coef_gb_rr", dp->coef_gb_rr);
-}
-
-void
-ia_css_dp_debug_dtrace(
-	const struct ia_css_dp_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.threshold=%d, config.gain=%d\n",
-		config->threshold, config->gain);
-}
-
-void
-ia_css_init_dp_state(
-	void/*struct sh_css_isp_dp_vmem_state*/ *state,
-	size_t size)
-{
-	memset(state, 0, size);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.h
deleted file mode 100644
index db21814ad3db..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp.host.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DP_HOST_H
-#define __IA_CSS_DP_HOST_H
-
-#include "ia_css_dp_types.h"
-#include "ia_css_dp_param.h"
-
-extern const struct ia_css_dp_config default_dp_config;
-#ifdef ISP2401
-extern const struct ia_css_dp_config default_dp_10bpp_config;
-#endif
-
-void
-ia_css_dp_encode(
-	struct sh_css_isp_dp_params *to,
-	const struct ia_css_dp_config *from,
-	unsigned size);
-
-void
-ia_css_dp_dump(
-	const struct sh_css_isp_dp_params *dp,
-	unsigned level);
-
-void
-ia_css_dp_debug_dtrace(
-	const struct ia_css_dp_config *config,
-	unsigned level);
-
-void
-ia_css_init_dp_state(
-	void/*struct sh_css_isp_dp_vmem_state*/ *state,
-	size_t size);
-
-#endif /* __IA_CSS_DP_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp_param.h
deleted file mode 100644
index fc9035a98d92..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp_param.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DP_PARAM_H
-#define __IA_CSS_DP_PARAM_H
-
-#include "type_support.h"
-#include "bnr/bnr_1.0/ia_css_bnr_param.h"
-
-/* DP (Defect Pixel Correction) */
-struct sh_css_isp_dp_params {
-	int32_t threshold_single;
-	int32_t threshold_2adjacent;
-	int32_t gain;
-	int32_t coef_rr_gr;
-	int32_t coef_rr_gb;
-	int32_t coef_bb_gb;
-	int32_t coef_bb_gr;
-	int32_t coef_gr_rr;
-	int32_t coef_gr_bb;
-	int32_t coef_gb_bb;
-	int32_t coef_gb_rr;
-};
-
-#endif /* __IA_CSS_DP_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp_types.h
deleted file mode 100644
index 1bf6dcef7dc7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dp/dp_1.0/ia_css_dp_types.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DP_TYPES_H
-#define __IA_CSS_DP_TYPES_H
-
-/* @file
-* CSS-API header file for Defect Pixel Correction (DPC) parameters.
-*/
-
-
-/* Defect Pixel Correction configuration.
- *
- *  ISP block: DPC1 (DPC after WB)
- *             DPC2 (DPC before WB)
- *  ISP1: DPC1 is used.
- *  ISP2: DPC2 is used.
- */
-struct ia_css_dp_config {
-	ia_css_u0_16 threshold; /** The threshold of defect pixel correction,
-			      representing the permissible difference of
-			      intensity between one pixel and its
-			      surrounding pixels. Smaller values result
-				in more frequent pixel corrections.
-				u0.16, [0,65535],
-				default 8192, ineffective 65535 */
-	ia_css_u8_8 gain;	 /** The sensitivity of mis-correction. ISP will
-			      miss a lot of defects if the value is set
-				too large.
-				u8.8, [0,65535],
-				default 4096, ineffective 65535 */
-	uint32_t gr;	/* unsigned <integer_bits>.<16-integer_bits> */
-	uint32_t r;	/* unsigned <integer_bits>.<16-integer_bits> */
-	uint32_t b;	/* unsigned <integer_bits>.<16-integer_bits> */
-	uint32_t gb;	/* unsigned <integer_bits>.<16-integer_bits> */
-};
-
-#endif /* __IA_CSS_DP_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2.host.c
deleted file mode 100644
index bc14b85cf952..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2.host.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_dpc2.host.h"
-#include "assert_support.h"
-
-void
-ia_css_dpc2_encode(
-	struct ia_css_isp_dpc2_params *to,
-	const struct ia_css_dpc2_config *from,
-	size_t size)
-{
-	(void)size;
-
-	assert ((from->metric1 >= 0) && (from->metric1 <= METRIC1_ONE_FP));
-	assert ((from->metric3 >= 0) && (from->metric3 <= METRIC3_ONE_FP));
-	assert ((from->metric2 >= METRIC2_ONE_FP) &&
-			(from->metric2 < 256*METRIC2_ONE_FP));
-	assert ((from->wb_gain_gr > 0) && (from->wb_gain_gr < 16*WBGAIN_ONE_FP));
-	assert ((from->wb_gain_r  > 0) && (from->wb_gain_r  < 16*WBGAIN_ONE_FP));
-	assert ((from->wb_gain_b  > 0) && (from->wb_gain_b  < 16*WBGAIN_ONE_FP));
-	assert ((from->wb_gain_gb > 0) && (from->wb_gain_gb < 16*WBGAIN_ONE_FP));
-
-	to->metric1 = from->metric1;
-	to->metric2 = from->metric2;
-	to->metric3 = from->metric3;
-
-	to->wb_gain_gr = from->wb_gain_gr;
-	to->wb_gain_r  = from->wb_gain_r;
-	to->wb_gain_b  = from->wb_gain_b;
-	to->wb_gain_gb = from->wb_gain_gb;
-}
-
-/* TODO: AM: This needs a proper implementation. */
-void
-ia_css_init_dpc2_state(
-	void *state,
-	size_t size)
-{
-	(void)state;
-	(void)size;
-}
-
-#ifndef IA_CSS_NO_DEBUG
-/* TODO: AM: This needs a proper implementation. */
-void
-ia_css_dpc2_debug_dtrace(
-	const struct ia_css_dpc2_config *config,
-	unsigned level)
-{
-	(void)config;
-	(void)level;
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2.host.h
deleted file mode 100644
index 38d10a5237c6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2.host.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DPC2_HOST_H
-#define __IA_CSS_DPC2_HOST_H
-
-#include "ia_css_dpc2_types.h"
-#include "ia_css_dpc2_param.h"
-
-void
-ia_css_dpc2_encode(
-	struct ia_css_isp_dpc2_params *to,
-	const struct ia_css_dpc2_config *from,
-	size_t size);
-
-void
-ia_css_init_dpc2_state(
-	void *state,
-	size_t size);
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_dpc2_debug_dtrace(
-	const struct ia_css_dpc2_config *config,
-	unsigned level);
-#endif
-
-#endif /* __IA_CSS_DPC2_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2_param.h
deleted file mode 100644
index ef668d54fe16..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2_param.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DPC2_PARAM_H
-#define __IA_CSS_DPC2_PARAM_H
-
-#include "type_support.h"
-#include "vmem.h" /* for VMEM_ARRAY*/
-
-
-/* 4 planes : GR, R, B, GB */
-#define NUM_PLANES		4
-
-/* ToDo: Move this to testsetup */
-#define MAX_FRAME_SIMDWIDTH	30
-
-/* 3 lines state per color plane input_line_state */
-#define DPC2_STATE_INPUT_BUFFER_HEIGHT	(3 * NUM_PLANES)
-/* Each plane has width equal to half frame line */
-#define DPC2_STATE_INPUT_BUFFER_WIDTH	CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-/* 1 line state per color plane for local deviation state*/
-#define DPC2_STATE_LOCAL_DEVIATION_BUFFER_HEIGHT	(1 * NUM_PLANES)
-/* Each plane has width equal to half frame line */
-#define DPC2_STATE_LOCAL_DEVIATION_BUFFER_WIDTH		CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-/* MINMAX state buffer stores 1 full input line (GR-R color line) */
-#define DPC2_STATE_SECOND_MINMAX_BUFFER_HEIGHT	1
-#define DPC2_STATE_SECOND_MINMAX_BUFFER_WIDTH	MAX_FRAME_SIMDWIDTH
-
-
-struct ia_css_isp_dpc2_params {
-	int32_t metric1;
-	int32_t metric2;
-	int32_t metric3;
-	int32_t wb_gain_gr;
-	int32_t wb_gain_r;
-	int32_t wb_gain_b;
-	int32_t wb_gain_gb;
-};
-
-#endif /* __IA_CSS_DPC2_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2_types.h
deleted file mode 100644
index 6727682d287f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dpc2/ia_css_dpc2_types.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DPC2_TYPES_H
-#define __IA_CSS_DPC2_TYPES_H
-
-/* @file
-* CSS-API header file for Defect Pixel Correction 2 (DPC2) parameters.
-*/
-
-#include "type_support.h"
-
-/**@{*/
-/* Floating point constants for different metrics. */
-#define METRIC1_ONE_FP	(1<<12)
-#define METRIC2_ONE_FP	(1<<5)
-#define METRIC3_ONE_FP	(1<<12)
-#define WBGAIN_ONE_FP	(1<<9)
-/**@}*/
-
-/**@{*/
-/* Defect Pixel Correction 2 configuration.
- *
- * \brief DPC2 public parameters.
- * \details Struct with all parameters for the Defect Pixel Correction 2
- * kernel that can be set from the CSS API.
- *
- * ISP block: DPC1 (DPC after WB)
- *            DPC2 (DPC before WB)
- * ISP1: DPC1 is used.
- * ISP2: DPC2 is used.
- *
- */
-struct ia_css_dpc2_config {
-	/**@{*/
-	int32_t metric1;
-	int32_t metric2;
-	int32_t metric3;
-	int32_t wb_gain_gr;
-	int32_t wb_gain_r;
-	int32_t wb_gain_b;
-	int32_t wb_gain_gb;
-	/**@}*/
-};
-/**@}*/
-
-#endif /* __IA_CSS_DPC2_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c
deleted file mode 100644
index 955adc4d6ab0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_frame_public.h"
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-
-#include "ia_css_types.h"
-#include "ia_css_host_data.h"
-#include "sh_css_param_dvs.h"
-#include "sh_css_params.h"
-#include "ia_css_binary.h"
-#include "ia_css_debug.h"
-#include "memory_access.h"
-#include "assert_support.h"
-
-#include "ia_css_dvs.host.h"
-
-static const struct ia_css_dvs_configuration default_config = {
-	.info = (struct ia_css_frame_info *)NULL,
-};
-
-void
-ia_css_dvs_config(
-	struct sh_css_isp_dvs_isp_config *to,
-	const struct ia_css_dvs_configuration  *from,
-	unsigned size)
-{
-	(void)size;
-	to->num_horizontal_blocks =
-	    DVS_NUM_BLOCKS_X(from->info->res.width);
-	to->num_vertical_blocks =
-	    DVS_NUM_BLOCKS_Y(from->info->res.height);
-}
-
-void
-ia_css_dvs_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *info)
-{
-	struct ia_css_dvs_configuration config = default_config;
-
-	config.info = info;
-
-	ia_css_configure_dvs(binary, &config);
-}
-
-static void
-convert_coords_to_ispparams(
-	struct ia_css_host_data *gdc_warp_table,
-	const struct ia_css_dvs_6axis_config *config,
-	unsigned int i_stride,
-	unsigned int o_width,
-	unsigned int o_height,
-	unsigned int uv_flag)
-{
-	unsigned int i, j;
-#ifndef ISP2401
-	/* Coverity CID 298073 - initialize */
-#endif
-	gdc_warp_param_mem_t s = { 0 };
-	unsigned int x00, x01, x10, x11,
-		     y00, y01, y10, y11;
-
-	unsigned int xmin, ymin, xmax, ymax;
-	unsigned int topleft_x, topleft_y, bottom_x, bottom_y,
-		     topleft_x_frac, topleft_y_frac;
-	unsigned int dvs_interp_envelope = (DVS_GDC_INTERP_METHOD == HRT_GDC_BLI_MODE ?
-					   DVS_GDC_BLI_INTERP_ENVELOPE : DVS_GDC_BCI_INTERP_ENVELOPE);
-
-	/* number of blocks per height and width */
-	unsigned int num_blocks_y =  (uv_flag ? DVS_NUM_BLOCKS_Y_CHROMA(o_height) : DVS_NUM_BLOCKS_Y(o_height) );
-	unsigned int num_blocks_x =  (uv_flag ? DVS_NUM_BLOCKS_X_CHROMA(o_width)  : DVS_NUM_BLOCKS_X(o_width)  ); // round num_x up to blockdim_x, if it concerns the Y0Y1 block (uv_flag==0) round up to even
-
-
-	unsigned int in_stride = i_stride * DVS_INPUT_BYTES_PER_PIXEL;
-	unsigned width, height;
-	unsigned int *xbuff = NULL;
-	unsigned int *ybuff = NULL;
-	struct gdc_warp_param_mem_s *ptr;
-
-	assert(config != NULL);
-	assert(gdc_warp_table != NULL);
-	assert(gdc_warp_table->address != NULL);
-
-	ptr = (struct gdc_warp_param_mem_s *)gdc_warp_table->address;
-
-	ptr += (2 * uv_flag); /* format is Y0 Y1 UV, so UV starts at 3rd position */
-
-	if(uv_flag == 0)
-	{
-		xbuff = config->xcoords_y;
-		ybuff = config->ycoords_y;
-		width = config->width_y;
-		height = config->height_y;
-	}
-	else
-	{
-		xbuff = config->xcoords_uv;
-		ybuff = config->ycoords_uv;
-		width = config->width_uv;
-		height = config->height_uv;
-	}
-
-	IA_CSS_LOG("blockdim_x %d blockdim_y %d",
-		   DVS_BLOCKDIM_X, DVS_BLOCKDIM_Y_LUMA >> uv_flag);
-	IA_CSS_LOG("num_blocks_x %d num_blocks_y %d", num_blocks_x,num_blocks_y);
-	IA_CSS_LOG("width %d height %d", width, height);
-
-	assert(width == num_blocks_x + 1); // the width and height of the provided morphing table should be 1 more than the number of blocks
-	assert(height == num_blocks_y + 1);
-
-	for (j = 0; j < num_blocks_y; j++) {
-		for (i = 0; i < num_blocks_x; i++) {
-
-			x00 = xbuff[j * width + i];
-			x01 = xbuff[j * width + (i+1)];
-			x10 = xbuff[(j+1) * width + i];
-			x11 = xbuff[(j+1) * width + (i+1)];
-
-			y00 = ybuff[j * width + i];
-			y01 = ybuff[j * width + (i+1)];
-			y10 = ybuff[(j+1) * width + i];
-			y11 = ybuff[(j+1) * width + (i+1)];
-
-			xmin = min(x00, x10);
-			xmax = max(x01, x11);
-			ymin = min(y00, y01);
-			ymax = max(y10, y11);
-
-			/* Assert that right column's X is greater */
-			assert ( x01 >= xmin);
-			assert ( x11 >= xmin);
-			/* Assert that bottom row's Y is greater */
-			assert ( y10 >= ymin);
-			assert ( y11 >= ymin);
-
-			topleft_y = ymin >> DVS_COORD_FRAC_BITS;
-			topleft_x = ((xmin >> DVS_COORD_FRAC_BITS)
-					>> XMEM_ALIGN_LOG2)
-					<< (XMEM_ALIGN_LOG2);
-			s.in_addr_offset = topleft_y * in_stride + topleft_x;
-
-			/* similar to topleft_y calculation, but round up if ymax
-			 * has any fraction bits */
-			bottom_y = CEIL_DIV(ymax, 1 << DVS_COORD_FRAC_BITS);
-			s.in_block_height = bottom_y - topleft_y + dvs_interp_envelope;
-
-			bottom_x = CEIL_DIV(xmax, 1 << DVS_COORD_FRAC_BITS);
-			s.in_block_width = bottom_x - topleft_x + dvs_interp_envelope;
-
-			topleft_x_frac = topleft_x << (DVS_COORD_FRAC_BITS);
-			topleft_y_frac = topleft_y << (DVS_COORD_FRAC_BITS);
-
-			s.p0_x = x00 - topleft_x_frac;
-			s.p1_x = x01 - topleft_x_frac;
-			s.p2_x = x10 - topleft_x_frac;
-			s.p3_x = x11 - topleft_x_frac;
-
-			s.p0_y = y00 - topleft_y_frac;
-			s.p1_y = y01 - topleft_y_frac;
-			s.p2_y = y10 - topleft_y_frac;
-			s.p3_y = y11 - topleft_y_frac;
-
-			// block should fit within the boundingbox.
-			assert(s.p0_x < (s.in_block_width << DVS_COORD_FRAC_BITS));
-			assert(s.p1_x < (s.in_block_width << DVS_COORD_FRAC_BITS));
-			assert(s.p2_x < (s.in_block_width << DVS_COORD_FRAC_BITS));
-			assert(s.p3_x < (s.in_block_width << DVS_COORD_FRAC_BITS));
-			assert(s.p0_y < (s.in_block_height << DVS_COORD_FRAC_BITS));
-			assert(s.p1_y < (s.in_block_height << DVS_COORD_FRAC_BITS));
-			assert(s.p2_y < (s.in_block_height << DVS_COORD_FRAC_BITS));
-			assert(s.p3_y < (s.in_block_height << DVS_COORD_FRAC_BITS));
-
-			// block size should be greater than zero.
-			assert(s.p0_x < s.p1_x);
-			assert(s.p2_x < s.p3_x);
-			assert(s.p0_y < s.p2_y);
-			assert(s.p1_y < s.p3_y);
-
-#if 0
-			printf("j: %d\ti:%d\n", j, i);
-			printf("offset: %d\n", s.in_addr_offset);
-			printf("p0_x: %d\n", s.p0_x);
-			printf("p0_y: %d\n", s.p0_y);
-			printf("p1_x: %d\n", s.p1_x);
-			printf("p1_y: %d\n", s.p1_y);
-			printf("p2_x: %d\n", s.p2_x);
-			printf("p2_y: %d\n", s.p2_y);
-			printf("p3_x: %d\n", s.p3_x);
-			printf("p3_y: %d\n", s.p3_y);
-
-			printf("p0_x_nofrac[0]: %d\n", s.p0_x>>DVS_COORD_FRAC_BITS);
-			printf("p0_y_nofrac[1]: %d\n", s.p0_y>>DVS_COORD_FRAC_BITS);
-			printf("p1_x_nofrac[2]: %d\n", s.p1_x>>DVS_COORD_FRAC_BITS);
-			printf("p1_y_nofrac[3]: %d\n", s.p1_y>>DVS_COORD_FRAC_BITS);
-			printf("p2_x_nofrac[0]: %d\n", s.p2_x>>DVS_COORD_FRAC_BITS);
-			printf("p2_y_nofrac[1]: %d\n", s.p2_y>>DVS_COORD_FRAC_BITS);
-			printf("p3_x_nofrac[2]: %d\n", s.p3_x>>DVS_COORD_FRAC_BITS);
-			printf("p3_y_nofrac[3]: %d\n", s.p3_y>>DVS_COORD_FRAC_BITS);
-			printf("\n");
-#endif
-
-			*ptr = s;
-
-			// storage format:
-			// Y0 Y1 UV0 Y2 Y3 UV1
-			/* if uv_flag equals true increment with 2 incase x is odd, this to
-			skip the uv position. */
-			if (uv_flag)
-				ptr += 3;
-			else
-				ptr += (1 + (i&1));
-		}
-	}
-}
-
-struct ia_css_host_data *
-convert_allocate_dvs_6axis_config(
-	const struct ia_css_dvs_6axis_config *dvs_6axis_config,
-	const struct ia_css_binary *binary,
-	const struct ia_css_frame_info *dvs_in_frame_info)
-{
-	unsigned int i_stride;
-	unsigned int o_width;
-	unsigned int o_height;
-	struct ia_css_host_data *me;
-	struct gdc_warp_param_mem_s *isp_data_ptr;
-
-	assert(binary != NULL);
-	assert(dvs_6axis_config != NULL);
-	assert(dvs_in_frame_info != NULL);
-
-	me = ia_css_host_data_allocate((size_t)((DVS_6AXIS_BYTES(binary) / 2) * 3));
-
-	if (!me)
-		return NULL;
-
-	/*DVS only supports input frame of YUV420 or NV12. Fail for all other cases*/
-	assert((dvs_in_frame_info->format == IA_CSS_FRAME_FORMAT_NV12)
-		|| (dvs_in_frame_info->format == IA_CSS_FRAME_FORMAT_YUV420));
-
-	isp_data_ptr = (struct gdc_warp_param_mem_s *)me->address;
-
-	i_stride  = dvs_in_frame_info->padded_width;
-
-	o_width  = binary->out_frame_info[0].res.width;
-	o_height = binary->out_frame_info[0].res.height;
-
-	/* Y plane */
-	convert_coords_to_ispparams(me, dvs_6axis_config,
-				    i_stride, o_width, o_height, 0);
-
-	if (dvs_in_frame_info->format == IA_CSS_FRAME_FORMAT_YUV420) {
-		/*YUV420 has half the stride for U/V plane*/
-		i_stride /=2;
-	}
-
-	/* UV plane (packed inside the y plane) */
-	convert_coords_to_ispparams(me, dvs_6axis_config,
-				    i_stride, o_width/2, o_height/2, 1);
-
-	return me;
-}
-
-enum ia_css_err
-store_dvs_6axis_config(
-	const struct ia_css_dvs_6axis_config *dvs_6axis_config,
-	const struct ia_css_binary *binary,
-	const struct ia_css_frame_info *dvs_in_frame_info,
-	hrt_vaddress ddr_addr_y)
-{
-
-	struct ia_css_host_data *me;
-	assert(dvs_6axis_config != NULL);
-	assert(ddr_addr_y != mmgr_NULL);
-	assert(dvs_in_frame_info != NULL);
-
-	me = convert_allocate_dvs_6axis_config(dvs_6axis_config,
-				 binary,
-				 dvs_in_frame_info);
-
-	if (!me) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	}
-
-	ia_css_params_store_ia_css_host_data(
-				ddr_addr_y,
-				me);
-	ia_css_host_data_free(me);
-
-	return IA_CSS_SUCCESS;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h
deleted file mode 100644
index 2f513e29d88c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DVS_HOST_H
-#define __IA_CSS_DVS_HOST_H
-
-#include "ia_css_frame_public.h"
-#include "ia_css_binary.h"
-#include "sh_css_params.h"
-
-#include "ia_css_types.h"
-#include "ia_css_dvs_types.h"
-#include "ia_css_dvs_param.h"
-
-/* For bilinear interpolation, we need to add +1 to input block height calculation.
- * For bicubic interpolation, we will need to add +3 instaed */
-#define DVS_GDC_BLI_INTERP_ENVELOPE 1
-#define DVS_GDC_BCI_INTERP_ENVELOPE 3
-
-void
-ia_css_dvs_config(
-	struct sh_css_isp_dvs_isp_config      *to,
-	const struct ia_css_dvs_configuration *from,
-	unsigned size);
-
-void
-ia_css_dvs_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *from);
-
-void
-convert_dvs_6axis_config(
-	struct ia_css_isp_parameters *params,
-	const struct ia_css_binary *binary);
-
-struct ia_css_host_data *
-convert_allocate_dvs_6axis_config(
-	const struct ia_css_dvs_6axis_config *dvs_6axis_config,
-	const struct ia_css_binary *binary,
-	const struct ia_css_frame_info *dvs_in_frame_info);
-
-enum ia_css_err
-store_dvs_6axis_config(
-	const struct ia_css_dvs_6axis_config *dvs_6axis_config,
-	const struct ia_css_binary *binary,
-	const struct ia_css_frame_info *dvs_in_frame_info,
-	hrt_vaddress ddr_addr_y);
-
-#endif /* __IA_CSS_DVS_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h
deleted file mode 100644
index 66a7e58659c0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DVS_PARAM_H
-#define __IA_CSS_DVS_PARAM_H
-
-#include <type_support.h>
-#ifdef ISP2401
-
-#if !defined(ENABLE_TPROXY) && !defined(ENABLE_CRUN_FOR_TD) && !defined(PARAMBIN_GENERATION)
-#endif
-#include "dma.h"
-#ifdef ISP2401
-#endif /* !defined(ENABLE_TPROXY) && !defined(ENABLE_CRUN_FOR_TD) */
-
-#endif
-#include "uds/uds_1.0/ia_css_uds_param.h"
-
-#ifdef ISP2401
-
-#endif
-/* dvserence frame */
-struct sh_css_isp_dvs_isp_config {
-	uint32_t num_horizontal_blocks;
-	uint32_t num_vertical_blocks;
-};
-
-#endif /* __IA_CSS_DVS_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h
deleted file mode 100644
index 30772d217fb2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_DVS_TYPES_H
-#define __IA_CSS_DVS_TYPES_H
-
-/* DVS frame
- *
- *  ISP block: dvs frame
- */
-
-#include "ia_css_frame_public.h"
-
-struct ia_css_dvs_configuration {
-	const struct ia_css_frame_info *info;
-};
-
-#endif /* __IA_CSS_DVS_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8.host.c
deleted file mode 100644
index 47bb5042381b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8.host.c
+++ /dev/null
@@ -1,321 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef IA_CSS_NO_DEBUG
-#include "ia_css_debug.h"
-#endif
-
-#include "type_support.h"
-#include "assert_support.h"
-#include "math_support.h" /* for min and max */
-
-#include "ia_css_eed1_8.host.h"
-
-/* WARNING1: Number of inv points should be less or equal to 16,
- * due to implementation limitation. See kernel design document
- * for more details.
- * WARNING2: Do not modify the number of inv points without correcting
- * the EED1_8 kernel implementation assumptions.
- */
-#define NUMBER_OF_CHGRINV_POINTS 15
-#define NUMBER_OF_TCINV_POINTS 9
-#define NUMBER_OF_FCINV_POINTS 9
-
-static const int16_t chgrinv_x[NUMBER_OF_CHGRINV_POINTS] = {
-0, 16, 64, 144, 272, 448, 672, 976,
-1376, 1888, 2528, 3312, 4256, 5376, 6688};
-
-static const int16_t chgrinv_a[NUMBER_OF_CHGRINV_POINTS] = {
--7171, -256, -29, -3456, -1071, -475, -189, -102,
--48, -38, -10, -9, -7, -6, 0};
-
-static const int16_t chgrinv_b[NUMBER_OF_CHGRINV_POINTS] = {
-8191, 1021, 256, 114, 60, 37, 24, 17,
-12, 9, 6, 5, 4, 3, 2};
-
-static const int16_t chgrinv_c[NUMBER_OF_CHGRINV_POINTS] = {
-1, 1, 1, 0, 0, 0, 0, 0,
-0, 0, 0, 0, 0, 0, 0};
-
-static const int16_t tcinv_x[NUMBER_OF_TCINV_POINTS] = {
-0, 4, 11, 23, 42, 68, 102, 148, 205};
-
-static const int16_t tcinv_a[NUMBER_OF_TCINV_POINTS] = {
--6364, -631, -126, -34, -13, -6, -4452, -2156, 0};
-
-static const int16_t tcinv_b[NUMBER_OF_TCINV_POINTS] = {
-8191, 1828, 726, 352, 197, 121, 80, 55, 40};
-
-static const int16_t tcinv_c[NUMBER_OF_TCINV_POINTS] = {
-1, 1, 1, 1, 1, 1, 0, 0, 0};
-
-static const int16_t fcinv_x[NUMBER_OF_FCINV_POINTS] = {
-0, 80, 216, 456, 824, 1344, 2040, 2952, 4096};
-
-static const int16_t fcinv_a[NUMBER_OF_FCINV_POINTS] = {
--5244, -486, -86, -2849, -961, -400, -180, -86, 0};
-
-static const int16_t fcinv_b[NUMBER_OF_FCINV_POINTS] = {
-8191, 1637, 607, 287, 159, 98, 64, 44, 32};
-
-static const int16_t fcinv_c[NUMBER_OF_FCINV_POINTS] = {
-1, 1, 1, 0, 0, 0, 0, 0, 0};
-
-
-void
-ia_css_eed1_8_vmem_encode(
-	struct eed1_8_vmem_params *to,
-	const struct ia_css_eed1_8_config *from,
-	size_t size)
-{
-	unsigned i, j, base;
-	const unsigned total_blocks = 4;
-	const unsigned shuffle_block = 16;
-
-	(void)size;
-
-	/* Init */
-	for (i = 0; i < ISP_VEC_NELEMS; i++) {
-		to->e_dew_enh_x[0][i] = 0;
-		to->e_dew_enh_y[0][i] = 0;
-		to->e_dew_enh_a[0][i] = 0;
-		to->e_dew_enh_f[0][i] = 0;
-		to->chgrinv_x[0][i] = 0;
-		to->chgrinv_a[0][i] = 0;
-		to->chgrinv_b[0][i] = 0;
-		to->chgrinv_c[0][i] = 0;
-		to->tcinv_x[0][i] = 0;
-		to->tcinv_a[0][i] = 0;
-		to->tcinv_b[0][i] = 0;
-		to->tcinv_c[0][i] = 0;
-		to->fcinv_x[0][i] = 0;
-		to->fcinv_a[0][i] = 0;
-		to->fcinv_b[0][i] = 0;
-		to->fcinv_c[0][i] = 0;
-	}
-
-	/* Constraints on dew_enhance_seg_x and dew_enhance_seg_y:
-	 * - values should be greater or equal to 0.
-	 * - values should be ascending.
-	 * - value of index zero is equal to 0.
-	 */
-
-	/* Checking constraints: */
-	/* TODO: investigate if an assert is the right way to report that
-	 * the constraints are violated.
-	 */
-	for (j = 0; j < IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS; j++) {
-		assert(from->dew_enhance_seg_x[j] > -1);
-		assert(from->dew_enhance_seg_y[j] > -1);
-	}
-
-	for (j = 1; j < IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS; j++) {
-		assert(from->dew_enhance_seg_x[j] > from->dew_enhance_seg_x[j-1]);
-		assert(from->dew_enhance_seg_y[j] > from->dew_enhance_seg_y[j-1]);
-	}
-
-	assert(from->dew_enhance_seg_x[0] == 0);
-	assert(from->dew_enhance_seg_y[0] == 0);
-
-	/* Constraints on chgrinv_x, tcinv_x and fcinv_x:
-	 * - values should be greater or equal to 0.
-	 * - values should be ascending.
-	 * - value of index zero is equal to 0.
-	 */
-	assert(chgrinv_x[0] == 0);
-	assert(tcinv_x[0] == 0);
-	assert(fcinv_x[0] == 0);
-
-	for (j = 1; j < NUMBER_OF_CHGRINV_POINTS; j++) {
-		assert(chgrinv_x[j] > chgrinv_x[j-1]);
-	}
-
-	for (j = 1; j < NUMBER_OF_TCINV_POINTS; j++) {
-		assert(tcinv_x[j] > tcinv_x[j-1]);
-	}
-
-	for (j = 1; j < NUMBER_OF_FCINV_POINTS; j++) {
-		assert(fcinv_x[j] > fcinv_x[j-1]);
-	}
-
-	/* The implementation of the calulating 1/x is based on the availability
-	 * of the OP_vec_shuffle16 operation.
-	 * A 64 element vector is split up in 4 blocks of 16 element. Each array is copied to
-	 * a vector 4 times, (starting at 0, 16, 32 and 48). All array elements are copied or
-	 * initialised as described in the KFS. The remaining elements of a vector are set to 0.
-	 */
-	/* TODO: guard this code with above assumptions */
-	for(i = 0; i < total_blocks; i++) {
-		base = shuffle_block * i;
-
-		for (j = 0; j < IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS; j++) {
-			to->e_dew_enh_x[0][base + j] = min(max(from->dew_enhance_seg_x[j], 0), 8191);
-			to->e_dew_enh_y[0][base + j] = min(max(from->dew_enhance_seg_y[j], -8192), 8191);
-		}
-
-		for (j = 0; j < (IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS - 1); j++) {
-			to->e_dew_enh_a[0][base + j] = min(max(from->dew_enhance_seg_slope[j], -8192), 8191);
-			/* Convert dew_enhance_seg_exp to flag:
-			 * 0 -> 0
-			 * 1...13 -> 1
-			 */
-			to->e_dew_enh_f[0][base + j] = (min(max(from->dew_enhance_seg_exp[j], 0), 13) > 0);
-		}
-
-		/* Hard-coded to 0, in order to be able to handle out of
-		 * range input in the same way as the other segments.
-		 * See KFS for more details.
-		 */
-		to->e_dew_enh_a[0][base + (IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS - 1)] = 0;
-		to->e_dew_enh_f[0][base + (IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS - 1)] = 0;
-
-		for (j = 0; j < NUMBER_OF_CHGRINV_POINTS; j++) {
-			to->chgrinv_x[0][base + j] = chgrinv_x[j];
-			to->chgrinv_a[0][base + j] = chgrinv_a[j];
-			to->chgrinv_b[0][base + j] = chgrinv_b[j];
-			to->chgrinv_c[0][base + j] = chgrinv_c[j];
-		}
-
-		for (j = 0; j < NUMBER_OF_TCINV_POINTS; j++) {
-			to->tcinv_x[0][base + j] = tcinv_x[j];
-			to->tcinv_a[0][base + j] = tcinv_a[j];
-			to->tcinv_b[0][base + j] = tcinv_b[j];
-			to->tcinv_c[0][base + j] = tcinv_c[j];
-		}
-
-		for (j = 0; j < NUMBER_OF_FCINV_POINTS; j++) {
-			to->fcinv_x[0][base + j] = fcinv_x[j];
-			to->fcinv_a[0][base + j] = fcinv_a[j];
-			to->fcinv_b[0][base + j] = fcinv_b[j];
-			to->fcinv_c[0][base + j] = fcinv_c[j];
-		}
-	}
-}
-
-
-void
-ia_css_eed1_8_encode(
-	struct eed1_8_dmem_params *to,
-	const struct ia_css_eed1_8_config *from,
-	size_t size)
-{
-	int i;
-	int min_exp = 0;
-
-	(void)size;
-
-	to->rbzp_strength = from->rbzp_strength;
-
-	to->fcstrength = from->fcstrength;
-	to->fcthres_0 = from->fcthres_0;
-	to->fc_sat_coef = from->fc_sat_coef;
-	to->fc_coring_prm = from->fc_coring_prm;
-	to->fc_slope = from->fcthres_1 - from->fcthres_0;
-
-	to->aerel_thres0 = from->aerel_thres0;
-	to->aerel_gain0 = from->aerel_gain0;
-	to->aerel_thres_diff = from->aerel_thres1 - from->aerel_thres0;
-	to->aerel_gain_diff = from->aerel_gain1 - from->aerel_gain0;
-
-	to->derel_thres0 = from->derel_thres0;
-	to->derel_gain0 = from->derel_gain0;
-	to->derel_thres_diff = (from->derel_thres1 - from->derel_thres0);
-	to->derel_gain_diff = (from->derel_gain1 - from->derel_gain0);
-
-	to->coring_pos0 = from->coring_pos0;
-	to->coring_pos_diff = (from->coring_pos1 - from->coring_pos0);
-	to->coring_neg0 = from->coring_neg0;
-	to->coring_neg_diff = (from->coring_neg1 - from->coring_neg0);
-
-	/* Note: (ISP_VEC_ELEMBITS -1)
-	 * TODO: currently the testbench does not support to use
-	 * ISP_VEC_ELEMBITS. Investigate how to fix this
-	 */
-	to->gain_exp = (13 - from->gain_exp);
-	to->gain_pos0 = from->gain_pos0;
-	to->gain_pos_diff = (from->gain_pos1 - from->gain_pos0);
-	to->gain_neg0 = from->gain_neg0;
-	to->gain_neg_diff = (from->gain_neg1 - from->gain_neg0);
-
-	to->margin_pos0 = from->pos_margin0;
-	to->margin_pos_diff = (from->pos_margin1 - from->pos_margin0);
-	to->margin_neg0 = from->neg_margin0;
-	to->margin_neg_diff = (from->neg_margin1 - from->neg_margin0);
-
-	/* Encode DEWEnhance exp (e_dew_enh_asr) */
-	for (i = 0; i < (IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS - 1); i++) {
-		min_exp = max(min_exp, from->dew_enhance_seg_exp[i]);
-	}
-	to->e_dew_enh_asr = 13 - min(max(min_exp, 0), 13);
-
-	to->dedgew_max = from->dedgew_max;
-}
-
-
-void
-ia_css_init_eed1_8_state(
-	void *state,
-	size_t size)
-{
-	memset(state, 0, size);
-}
-
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_eed1_8_debug_dtrace(
-	const struct ia_css_eed1_8_config *eed,
-	unsigned level)
-{
-	if (!eed)
-		return;
-
-	ia_css_debug_dtrace(level, "Edge Enhancing Demosaic 1.8:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "rbzp_strength", eed->rbzp_strength);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "fcstrength", eed->fcstrength);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "fcthres_0", eed->fcthres_0);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "fcthres_1", eed->fcthres_1);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "fc_sat_coef", eed->fc_sat_coef);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "fc_coring_prm", eed->fc_coring_prm);
-
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "aerel_thres0", eed->aerel_thres0);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "aerel_gain0", eed->aerel_gain0);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "aerel_thres1", eed->aerel_thres1);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "aerel_gain1", eed->aerel_gain1);
-
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "derel_thres0", eed->derel_thres0);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "derel_gain0", eed->derel_gain0);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "derel_thres1", eed->derel_thres1);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "derel_gain1", eed->derel_gain1);
-
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "coring_pos0", eed->coring_pos0);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "coring_pos1", eed->coring_pos1);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "coring_neg0", eed->coring_neg0);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "coring_neg1", eed->coring_neg1);
-
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "gain_exp", eed->gain_exp);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "gain_pos0", eed->gain_pos0);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "gain_pos1", eed->gain_pos1);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "gain_neg0", eed->gain_neg0);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "gain_neg1", eed->gain_neg1);
-
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "pos_margin0", eed->pos_margin0);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "pos_margin1", eed->pos_margin1);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "neg_margin0", eed->neg_margin0);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "neg_margin1", eed->neg_margin1);
-
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n", "dedgew_max", eed->dedgew_max);
-}
-#endif
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8.host.h
deleted file mode 100644
index fff932c1364e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8.host.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_EED1_8_HOST_H
-#define __IA_CSS_EED1_8_HOST_H
-
-#include "ia_css_eed1_8_types.h"
-#include "ia_css_eed1_8_param.h"
-
-void
-ia_css_eed1_8_vmem_encode(
-	struct eed1_8_vmem_params *to,
-	const struct ia_css_eed1_8_config *from,
-	size_t size);
-
-void
-ia_css_eed1_8_encode(
-	struct eed1_8_dmem_params *to,
-	const struct ia_css_eed1_8_config *from,
-	size_t size);
-
-void
-ia_css_init_eed1_8_state(
-	void *state,
-	size_t size);
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_eed1_8_debug_dtrace(
-	const struct ia_css_eed1_8_config *config,
-	unsigned level);
-#endif
-
-#endif /* __IA_CSS_EED1_8_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8_param.h
deleted file mode 100644
index bc3a07fd07eb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8_param.h
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_EED1_8_PARAM_H
-#define __IA_CSS_EED1_8_PARAM_H
-
-#include "type_support.h"
-#include "vmem.h" /* needed for VMEM_ARRAY */
-
-#include "ia_css_eed1_8_types.h" /* IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS */
-
-
-/* Configuration parameters: */
-
-/* Enable median for false color correction
- * 0: Do not use median
- * 1: Use median
- * Default: 1
- */
-#define EED1_8_FC_ENABLE_MEDIAN		1
-
-/* Coring Threshold minima
- * Used in Tint color suppression.
- * Default: 1
- */
-#define EED1_8_CORINGTHMIN	1
-
-/* Define size of the state..... TODO: check if this is the correct place */
-/* 4 planes : GR, R, B, GB */
-#define NUM_PLANES	4
-
-/* 5 lines state per color plane input_line_state */
-#define EED1_8_STATE_INPUT_BUFFER_HEIGHT	(5 * NUM_PLANES)
-
-/* Each plane has width equal to half frame line */
-#define EED1_8_STATE_INPUT_BUFFER_WIDTH	CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-/* 1 line state per color plane LD_H state */
-#define EED1_8_STATE_LD_H_HEIGHT	(1 * NUM_PLANES)
-#define EED1_8_STATE_LD_H_WIDTH		CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-/* 1 line state per color plane LD_V state */
-#define EED1_8_STATE_LD_V_HEIGHT	(1 * NUM_PLANES)
-#define EED1_8_STATE_LD_V_WIDTH		CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-/* 1 line (single plane) state for D_Hr state */
-#define EED1_8_STATE_D_HR_HEIGHT	1
-#define EED1_8_STATE_D_HR_WIDTH		CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-/* 1 line (single plane) state for D_Hb state */
-#define EED1_8_STATE_D_HB_HEIGHT	1
-#define EED1_8_STATE_D_HB_WIDTH		CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-/* 2 lines (single plane) state for D_Vr state */
-#define EED1_8_STATE_D_VR_HEIGHT	2
-#define EED1_8_STATE_D_VR_WIDTH		CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-/* 2 line (single plane) state for D_Vb state */
-#define EED1_8_STATE_D_VB_HEIGHT	2
-#define EED1_8_STATE_D_VB_WIDTH		CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-/* 2 lines state for R and B (= 2 planes) rb_zipped_state */
-#define EED1_8_STATE_RB_ZIPPED_HEIGHT	(2 * 2)
-#define EED1_8_STATE_RB_ZIPPED_WIDTH	CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-#if EED1_8_FC_ENABLE_MEDIAN
-/* 1 full input line (GR-R color line) for Yc state */
-#define EED1_8_STATE_YC_HEIGHT	1
-#define EED1_8_STATE_YC_WIDTH	MAX_FRAME_SIMDWIDTH
-
-/* 1 line state per color plane Cg_state */
-#define EED1_8_STATE_CG_HEIGHT	(1 * NUM_PLANES)
-#define EED1_8_STATE_CG_WIDTH	CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-/* 1 line state per color plane Co_state */
-#define EED1_8_STATE_CO_HEIGHT	(1 * NUM_PLANES)
-#define EED1_8_STATE_CO_WIDTH	CEIL_DIV(MAX_FRAME_SIMDWIDTH, 2)
-
-/* 1 full input line (GR-R color line) for AbsK state */
-#define EED1_8_STATE_ABSK_HEIGHT	1
-#define EED1_8_STATE_ABSK_WIDTH		MAX_FRAME_SIMDWIDTH
-#endif
-
-struct eed1_8_vmem_params {
-	VMEM_ARRAY(e_dew_enh_x, ISP_VEC_NELEMS);
-	VMEM_ARRAY(e_dew_enh_y, ISP_VEC_NELEMS);
-	VMEM_ARRAY(e_dew_enh_a, ISP_VEC_NELEMS);
-	VMEM_ARRAY(e_dew_enh_f, ISP_VEC_NELEMS);
-	VMEM_ARRAY(chgrinv_x, ISP_VEC_NELEMS);
-	VMEM_ARRAY(chgrinv_a, ISP_VEC_NELEMS);
-	VMEM_ARRAY(chgrinv_b, ISP_VEC_NELEMS);
-	VMEM_ARRAY(chgrinv_c, ISP_VEC_NELEMS);
-	VMEM_ARRAY(fcinv_x, ISP_VEC_NELEMS);
-	VMEM_ARRAY(fcinv_a, ISP_VEC_NELEMS);
-	VMEM_ARRAY(fcinv_b, ISP_VEC_NELEMS);
-	VMEM_ARRAY(fcinv_c, ISP_VEC_NELEMS);
-	VMEM_ARRAY(tcinv_x, ISP_VEC_NELEMS);
-	VMEM_ARRAY(tcinv_a, ISP_VEC_NELEMS);
-	VMEM_ARRAY(tcinv_b, ISP_VEC_NELEMS);
-	VMEM_ARRAY(tcinv_c, ISP_VEC_NELEMS);
-};
-
-/* EED (Edge Enhancing Demosaic) ISP parameters */
-struct eed1_8_dmem_params {
-	int32_t rbzp_strength;
-
-	int32_t fcstrength;
-	int32_t fcthres_0;
-	int32_t fc_sat_coef;
-	int32_t fc_coring_prm;
-	int32_t fc_slope;
-
-	int32_t aerel_thres0;
-	int32_t aerel_gain0;
-	int32_t aerel_thres_diff;
-	int32_t aerel_gain_diff;
-
-	int32_t derel_thres0;
-	int32_t derel_gain0;
-	int32_t derel_thres_diff;
-	int32_t derel_gain_diff;
-
-	int32_t coring_pos0;
-	int32_t coring_pos_diff;
-	int32_t coring_neg0;
-	int32_t coring_neg_diff;
-
-	int32_t gain_exp;
-	int32_t gain_pos0;
-	int32_t gain_pos_diff;
-	int32_t gain_neg0;
-	int32_t gain_neg_diff;
-
-	int32_t margin_pos0;
-	int32_t margin_pos_diff;
-	int32_t margin_neg0;
-	int32_t margin_neg_diff;
-
-	int32_t e_dew_enh_asr;
-	int32_t dedgew_max;
-};
-
-#endif /* __IA_CSS_EED1_8_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8_types.h
deleted file mode 100644
index 32e91824a5e5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/eed1_8/ia_css_eed1_8_types.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_EED1_8_TYPES_H
-#define __IA_CSS_EED1_8_TYPES_H
-
-/* @file
-* CSS-API header file for Edge Enhanced Demosaic parameters.
-*/
-
-
-#include "type_support.h"
-
-/**
- * \brief EED1_8 public parameters.
- * \details Struct with all parameters for the EED1.8 kernel that can be set
- * from the CSS API.
- */
-
-/* parameter list is based on ISP261 CSS API public parameter list_all.xlsx from 28-01-2015 */
-
-/* Number of segments + 1 segment used in edge reliability enhancement
- * Ineffective: N/A
- * Default:	9
- */
-#define IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS	9
-
-/* Edge Enhanced Demosaic configuration
- *
- * ISP2.6.1: EED1_8 is used.
- */
-
-struct ia_css_eed1_8_config {
-	int32_t rbzp_strength;	/** Strength of zipper reduction. */
-
-	int32_t fcstrength;	/** Strength of false color reduction. */
-	int32_t fcthres_0;	/** Threshold to prevent chroma coring due to noise or green disparity in dark region. */
-	int32_t fcthres_1;	/** Threshold to prevent chroma coring due to noise or green disparity in bright region. */
-	int32_t fc_sat_coef;	/** How much color saturation to maintain in high color saturation region. */
-	int32_t fc_coring_prm;	/** Chroma coring coefficient for tint color suppression. */
-
-	int32_t aerel_thres0;	/** Threshold for Non-Directional Reliability at dark region. */
-	int32_t aerel_gain0;	/** Gain for Non-Directional Reliability at dark region. */
-	int32_t aerel_thres1;	/** Threshold for Non-Directional Reliability at bright region. */
-	int32_t aerel_gain1;	/** Gain for Non-Directional Reliability at bright region. */
-
-	int32_t derel_thres0;	/** Threshold for Directional Reliability at dark region. */
-	int32_t derel_gain0;	/** Gain for Directional Reliability at dark region. */
-	int32_t derel_thres1;	/** Threshold for Directional Reliability at bright region. */
-	int32_t derel_gain1;	/** Gain for Directional Reliability at bright region. */
-
-	int32_t coring_pos0;	/** Positive Edge Coring Threshold in dark region. */
-	int32_t coring_pos1;	/** Positive Edge Coring Threshold in bright region. */
-	int32_t coring_neg0;	/** Negative Edge Coring Threshold in dark region. */
-	int32_t coring_neg1;	/** Negative Edge Coring Threshold in bright region. */
-
-	int32_t gain_exp;	/** Common Exponent of Gain. */
-	int32_t gain_pos0;	/** Gain for Positive Edge in dark region. */
-	int32_t gain_pos1;	/** Gain for Positive Edge in bright region. */
-	int32_t gain_neg0;	/** Gain for Negative Edge in dark region. */
-	int32_t gain_neg1;	/** Gain for Negative Edge in bright region. */
-
-	int32_t pos_margin0;	/** Margin for Positive Edge in dark region. */
-	int32_t pos_margin1;	/** Margin for Positive Edge in bright region. */
-	int32_t neg_margin0;	/** Margin for Negative Edge in dark region. */
-	int32_t neg_margin1;	/** Margin for Negative Edge in bright region. */
-
-	int32_t dew_enhance_seg_x[IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS];		/** Segment data for directional edge weight: X. */
-	int32_t dew_enhance_seg_y[IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS];		/** Segment data for directional edge weight: Y. */
-	int32_t dew_enhance_seg_slope[(IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS - 1)];	/** Segment data for directional edge weight: Slope. */
-	int32_t dew_enhance_seg_exp[(IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS - 1)];	/** Segment data for directional edge weight: Exponent. */
-	int32_t dedgew_max;	/** Max Weight for Directional Edge. */
-};
-
-#endif /* __IA_CSS_EED1_8_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats.host.c
deleted file mode 100644
index 94631eee8614..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats.host.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_formats.host.h"
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-
-/*#include "sh_css_frac.h"*/
-#ifndef IA_CSS_NO_DEBUG
-/* FIXME: See BZ 4427 */
-#include "ia_css_debug.h"
-#endif
-
-const struct ia_css_formats_config default_formats_config = {
-	1
-};
-
-void
-ia_css_formats_encode(
-	struct sh_css_isp_formats_params *to,
-	const struct ia_css_formats_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->video_full_range_flag = from->video_full_range_flag;
-}
-#ifndef IA_CSS_NO_DEBUG
-/* FIXME: See BZ 4427 */
-void
-ia_css_formats_dump(
-	const struct sh_css_isp_formats_params *formats,
-	unsigned level)
-{
-	if (!formats) return;
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"video_full_range_flag", formats->video_full_range_flag);
-}
-#endif
-
-#ifndef IA_CSS_NO_DEBUG
-/* FIXME: See BZ 4427 */
-void
-ia_css_formats_debug_dtrace(
-	const struct ia_css_formats_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.video_full_range_flag=%d\n",
-		config->video_full_range_flag);
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats.host.h
deleted file mode 100644
index 8a90cd83b248..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats.host.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_FORMATS_HOST_H
-#define __IA_CSS_FORMATS_HOST_H
-
-#include "ia_css_formats_types.h"
-#include "ia_css_formats_param.h"
-
-extern const struct ia_css_formats_config default_formats_config;
-
-void
-ia_css_formats_encode(
-	struct sh_css_isp_formats_params *to,
-	const struct ia_css_formats_config *from,
-	unsigned size);
-#ifndef IA_CSS_NO_DEBUG
-/* FIXME: See BZ 4427 */
-void
-ia_css_formats_dump(
-	const struct sh_css_isp_formats_params *formats,
-	unsigned level);
-#endif
-
-#ifndef IA_CSS_NO_DEBUG
-/* FIXME: See BZ 4427 */
-void
-ia_css_formats_debug_dtrace(
-	const struct ia_css_formats_config *formats,
-	unsigned level);
-#endif /*IA_CSS_NO_DEBUG*/
-
-#endif /* __IA_CSS_FORMATS_HOST_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats_param.h
deleted file mode 100644
index 2eb6030b6081..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats_param.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_FORMATS_PARAM_H
-#define __IA_CSS_FORMATS_PARAM_H
-
-#include "type_support.h"
-
-/* FORMATS (Format conversion) */
-struct sh_css_isp_formats_params {
-	int32_t video_full_range_flag;
-};
-
-#endif /* __IA_CSS_FORMATS_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats_types.h
deleted file mode 100644
index 49479572b40d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fc/fc_1.0/ia_css_formats_types.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_FORMATS_TYPES_H
-#define __IA_CSS_FORMATS_TYPES_H
-
-/* @file
-* CSS-API header file for output format parameters.
-*/
-
-#include "type_support.h"
-
-/* Formats configuration.
- *
- *  ISP block: FORMATS
- *  ISP1: FORMATS is used.
- *  ISP2: FORMATS is used.
- */
-struct ia_css_formats_config {
-	uint32_t video_full_range_flag; /** selects the range of YUV output.
-				u8.0, [0,1],
-				default 1, ineffective n/a\n
-				1 - full range, luma 0-255, chroma 0-255\n
-				0 - reduced range, luma 16-235, chroma 16-240 */
-};
-
-#endif /* __IA_CSS_FORMATS_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h
deleted file mode 100644
index cc8dd1a7007f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_FIXEDBDS_PARAM_H
-#define __IA_CSS_FIXEDBDS_PARAM_H
-
-#include "type_support.h"
-
-#ifdef ISP2401
-#define BDS_UNIT 8
-#define FRAC_LOG 3
-#define FRAC_ACC (1<<FRAC_LOG)
-#if FRAC_ACC != BDS_UNIT
-#error "FRAC_ACC and BDS_UNIT need to be merged into one define"
-#endif
-
-#endif
-struct sh_css_isp_bds_params {
-	int baf_strength;
-};
-
-#endif /* __IA_CSS_FIXEDBDS_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h
deleted file mode 100644
index 5b59d9dec4d0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_FIXEDBDS_TYPES_H
-#define __IA_CSS_FIXEDBDS_TYPES_H
-
-
-struct sh_css_bds_factor {
-	unsigned numerator;
-	unsigned denominator;
-	unsigned int bds_factor;
-};
-
-
-#endif	/*__IA_CSS_FIXEDBDS_TYPES_H*/
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c
deleted file mode 100644
index 1fb9f27540f3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <assert_support.h>
-#include <ia_css_frame_public.h>
-#include <ia_css_frame.h>
-#include <ia_css_binary.h>
-#include <ia_css_types.h>
-#include <sh_css_defs.h>
-#include <ia_css_debug.h>
-
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-#include "isp.h"
-
-#include "ia_css_fpn.host.h"
-
-void
-ia_css_fpn_encode(
-	struct sh_css_isp_fpn_params *to,
-	const struct ia_css_fpn_table *from,
-	unsigned size)
-{
-	(void)size;
-	to->shift = from->shift;
-	to->enabled = from->data != NULL;
-}
-
-void
-ia_css_fpn_dump(
-	const struct sh_css_isp_fpn_params *fpn,
-	unsigned level)
-{
-	if (!fpn) return;
-	ia_css_debug_dtrace(level, "Fixed Pattern Noise Reduction:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"fpn_shift", fpn->shift);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"fpn_enabled", fpn->enabled);
-}
-
-void
-ia_css_fpn_config(
-	struct sh_css_isp_fpn_isp_config *to,
-	const struct ia_css_fpn_configuration *from,
-	unsigned size)
-{
-	unsigned elems_a = ISP_VEC_NELEMS;
-
-	(void)size;
-	ia_css_dma_configure_from_info(&to->port_b, from->info);
-	to->width_a_over_b = elems_a / to->port_b.elems;
-
-	/* Assume divisiblity here, may need to generalize to fixed point. */
-	assert (elems_a % to->port_b.elems == 0);
-}
-
-void
-ia_css_fpn_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *info)
-{
-	struct ia_css_frame_info my_info = IA_CSS_BINARY_DEFAULT_FRAME_INFO;
-	const struct ia_css_fpn_configuration config = {
-		&my_info
-	};
-
-	my_info.res.width       = CEIL_DIV(info->res.width, 2);		/* Packed by 2x */
-	my_info.res.height      = info->res.height;
-	my_info.padded_width    = CEIL_DIV(info->padded_width, 2);	/* Packed by 2x */
-	my_info.format          = info->format;
-	my_info.raw_bit_depth   = FPN_BITS_PER_PIXEL;
-	my_info.raw_bayer_order = info->raw_bayer_order;
-	my_info.crop_info       = info->crop_info;
-
-	ia_css_configure_fpn(binary, &config);
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h
deleted file mode 100644
index bb905c8db8c8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_FPN_HOST_H
-#define __IA_CSS_FPN_HOST_H
-
-#include "ia_css_binary.h"
-#include "ia_css_fpn_types.h"
-#include "ia_css_fpn_param.h"
-
-void
-ia_css_fpn_encode(
-	struct sh_css_isp_fpn_params *to,
-	const struct ia_css_fpn_table *from,
-	unsigned size);
-
-void
-ia_css_fpn_dump(
-	const struct sh_css_isp_fpn_params *fpn,
-	unsigned level);
-
-void
-ia_css_fpn_config(
-	struct sh_css_isp_fpn_isp_config      *to,
-	const struct ia_css_fpn_configuration *from,
-	unsigned size);
-
-void
-ia_css_fpn_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *from);
-
-#endif /* __IA_CSS_FPN_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h
deleted file mode 100644
index 68765c3f3bf7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_FPN_PARAM_H
-#define __IA_CSS_FPN_PARAM_H
-
-#include "type_support.h"
-
-#include "dma.h"
-
-#define FPN_BITS_PER_PIXEL	16
-
-/* FPNR (Fixed Pattern Noise Reduction) */
-struct sh_css_isp_fpn_params {
-	int32_t shift;
-	int32_t enabled;
-};
-
-struct sh_css_isp_fpn_isp_config {
-	uint32_t width_a_over_b;
-	struct dma_port_config port_b;
-};
-
-#endif /* __IA_CSS_FPN_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h
deleted file mode 100644
index ef287fa3c428..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_FPN_TYPES_H
-#define __IA_CSS_FPN_TYPES_H
-
-/* @file
-* CSS-API header file for Fixed Pattern Noise parameters.
-*/
-
-/* Fixed Pattern Noise table.
- *
- *  This contains the fixed patterns noise values
- *  obtained from a black frame capture.
- *
- *  "shift" should be set as the smallest value
- *  which satisfies the requirement the maximum data is less than 64.
- *
- *  ISP block: FPN1
- *  ISP1: FPN1 is used.
- *  ISP2: FPN1 is used.
- */
-
-struct ia_css_fpn_table {
-	int16_t *data;		/** Table content (fixed patterns noise).
-					u0.[13-shift], [0,63] */
-	uint32_t width;		/** Table width (in pixels).
-					This is the input frame width. */
-	uint32_t height;	/** Table height (in pixels).
-					This is the input frame height. */
-	uint32_t shift;		/** Common exponent of table content.
-					u8.0, [0,13] */
-	uint32_t enabled;	/** Fpn is enabled.
-					bool */
-};
-
-struct ia_css_fpn_configuration {
-	const struct ia_css_frame_info *info;
-};
-
-#endif /* __IA_CSS_FPN_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.c
deleted file mode 100644
index 0cfb5c94447f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.c
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#ifndef IA_CSS_NO_DEBUG
-/* FIXME: See BZ 4427 */
-#include "ia_css_debug.h"
-#endif
-#include "sh_css_frac.h"
-#include "vamem.h"
-
-#include "ia_css_gc.host.h"
-
-const struct ia_css_gc_config default_gc_config = {
-	0,
-	0
-};
-
-const struct ia_css_ce_config default_ce_config = {
-	0,
-	255
-};
-
-void
-ia_css_gc_encode(
-	struct sh_css_isp_gc_params *to,
-	const struct ia_css_gc_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->gain_k1 =
-	    uDIGIT_FITTING((int)from->gain_k1, 16,
-		IA_CSS_GAMMA_GAIN_K_SHIFT);
-	to->gain_k2 =
-	    uDIGIT_FITTING((int)from->gain_k2, 16,
-		IA_CSS_GAMMA_GAIN_K_SHIFT);
-}
-
-void
-ia_css_ce_encode(
-	struct sh_css_isp_ce_params *to,
-	const struct ia_css_ce_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->uv_level_min = from->uv_level_min;
-	to->uv_level_max = from->uv_level_max;
-}
-
-void
-ia_css_gc_vamem_encode(
-	struct sh_css_isp_gc_vamem_params *to,
-	const struct ia_css_gamma_table *from,
-	unsigned size)
-{
-	(void)size;
-	memcpy (&to->gc,  &from->data, sizeof(to->gc));
-}
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_gc_dump(
-	const struct sh_css_isp_gc_params *gc,
-	unsigned level)
-{
-	if (!gc) return;
-	ia_css_debug_dtrace(level, "Gamma Correction:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"gamma_gain_k1", gc->gain_k1);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"gamma_gain_k2", gc->gain_k2);
-}
-
-void
-ia_css_ce_dump(
-	const struct sh_css_isp_ce_params *ce,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level, "Chroma Enhancement:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ce_uv_level_min", ce->uv_level_min);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ce_uv_level_max", ce->uv_level_max);
-}
-
-void
-ia_css_gc_debug_dtrace(
-	const struct ia_css_gc_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.gain_k1=%d, config.gain_k2=%d\n",
-		config->gain_k1, config->gain_k2);
-}
-
-void
-ia_css_ce_debug_dtrace(
-	const struct ia_css_ce_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.uv_level_min=%d, config.uv_level_max=%d\n",
-		config->uv_level_min, config->uv_level_max);
-}
-#endif
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.h
deleted file mode 100644
index 06f08840563e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc.host.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_GC_HOST_H
-#define __IA_CSS_GC_HOST_H
-
-#include "ia_css_gc_param.h"
-#include "ia_css_gc_table.host.h"
-
-extern const struct ia_css_gc_config default_gc_config;
-extern const struct ia_css_ce_config default_ce_config;
-
-void
-ia_css_gc_encode(
-	struct sh_css_isp_gc_params *to,
-	const struct ia_css_gc_config *from,
-	unsigned size);
-
-void
-ia_css_gc_vamem_encode(
-	struct sh_css_isp_gc_vamem_params *to,
-	const struct ia_css_gamma_table *from,
-	unsigned size);
-
-void
-ia_css_ce_encode(
-	struct sh_css_isp_ce_params *to,
-	const struct ia_css_ce_config *from,
-	unsigned size);
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_gc_dump(
-	const struct sh_css_isp_gc_params *gc,
-	unsigned level);
-
-void
-ia_css_ce_dump(
-	const struct sh_css_isp_ce_params *ce,
-	unsigned level);
-
-void
-ia_css_gc_debug_dtrace(
-	const struct ia_css_gc_config *config,
-	unsigned level);
-
-void
-ia_css_ce_debug_dtrace(
-	const struct ia_css_ce_config *config,
-	unsigned level);
-
-#endif
-
-#endif /* __IA_CSS_GC_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_param.h
deleted file mode 100644
index 52972b1a07ff..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_param.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_GC_PARAM_H
-#define __IA_CSS_GC_PARAM_H
-
-#include "type_support.h"
-#ifndef PIPE_GENERATION
-#ifdef __ISP
-#define __INLINE_VAMEM__
-#endif
-#include "vamem.h"
-#include "ia_css_gc_types.h"
-
-#if defined(IS_VAMEM_VERSION_1)
-#define SH_CSS_ISP_GAMMA_TABLE_SIZE_LOG2 IA_CSS_VAMEM_1_GAMMA_TABLE_SIZE_LOG2
-#define SH_CSS_ISP_GC_TABLE_SIZE	 IA_CSS_VAMEM_1_GAMMA_TABLE_SIZE
-#elif defined(IS_VAMEM_VERSION_2)
-#define SH_CSS_ISP_GAMMA_TABLE_SIZE_LOG2 IA_CSS_VAMEM_2_GAMMA_TABLE_SIZE_LOG2
-#define SH_CSS_ISP_GC_TABLE_SIZE	 IA_CSS_VAMEM_2_GAMMA_TABLE_SIZE
-#else
-#error "Undefined vamem version"
-#endif
-
-#else
-/* For pipe generation, the size is not relevant */
-#define SH_CSS_ISP_GC_TABLE_SIZE 0
-#endif
-
-#define GAMMA_OUTPUT_BITS		8
-#define GAMMA_OUTPUT_MAX_VAL		((1<<GAMMA_OUTPUT_BITS)-1)
-
-/* GC (Gamma Correction) */
-struct sh_css_isp_gc_params {
-	int32_t gain_k1;
-	int32_t gain_k2;
-};
-
-/* CE (Chroma Enhancement) */
-struct sh_css_isp_ce_params {
-	int32_t uv_level_min;
-	int32_t uv_level_max;
-};
-
-/* This should be vamem_data_t, but that breaks the pipe generator */
-struct sh_css_isp_gc_vamem_params {
-	uint16_t gc[SH_CSS_ISP_GC_TABLE_SIZE];
-};
-
-#endif /* __IA_CSS_GC_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c
deleted file mode 100644
index 082db22a5329..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <type_support.h>
-#include <string_support.h> /* memcpy */
-#include "system_global.h"
-#include "vamem.h"
-#include "ia_css_types.h"
-#include "ia_css_gc_table.host.h"
-
-#if defined(HAS_VAMEM_VERSION_2)
-
-struct ia_css_gamma_table default_gamma_table;
-
-static const uint16_t
-default_gamma_table_data[IA_CSS_VAMEM_2_GAMMA_TABLE_SIZE] = {
-  0,   4,   8,  12,  17,  21,  27,  32,
- 38,  44,  49,  55,  61,  66,  71,  76,
- 80,  84,  88,  92,  95,  98, 102, 105,
-108, 110, 113, 116, 118, 121, 123, 126,
-128, 130, 132, 135, 137, 139, 141, 143,
-145, 146, 148, 150, 152, 153, 155, 156,
-158, 160, 161, 162, 164, 165, 166, 168,
-169, 170, 171, 172, 174, 175, 176, 177,
-178, 179, 180, 181, 182, 183, 184, 184,
-185, 186, 187, 188, 189, 189, 190, 191,
-192, 192, 193, 194, 195, 195, 196, 197,
-197, 198, 198, 199, 200, 200, 201, 201,
-202, 203, 203, 204, 204, 205, 205, 206,
-206, 207, 207, 208, 208, 209, 209, 210,
-210, 210, 211, 211, 212, 212, 213, 213,
-214, 214, 214, 215, 215, 216, 216, 216,
-217, 217, 218, 218, 218, 219, 219, 220,
-220, 220, 221, 221, 222, 222, 222, 223,
-223, 223, 224, 224, 225, 225, 225, 226,
-226, 226, 227, 227, 227, 228, 228, 228,
-229, 229, 229, 230, 230, 230, 231, 231,
-231, 232, 232, 232, 233, 233, 233, 234,
-234, 234, 234, 235, 235, 235, 236, 236,
-236, 237, 237, 237, 237, 238, 238, 238,
-239, 239, 239, 239, 240, 240, 240, 241,
-241, 241, 241, 242, 242, 242, 242, 243,
-243, 243, 243, 244, 244, 244, 245, 245,
-245, 245, 246, 246, 246, 246, 247, 247,
-247, 247, 248, 248, 248, 248, 249, 249,
-249, 249, 250, 250, 250, 250, 251, 251,
-251, 251, 252, 252, 252, 252, 253, 253,
-253, 253, 254, 254, 254, 254, 255, 255,
-255
-};
-
-#elif defined(HAS_VAMEM_VERSION_1)
-
-static const uint16_t
-default_gamma_table_data[IA_CSS_VAMEM_1_GAMMA_TABLE_SIZE] = {
-		0, 1, 2, 3, 4, 5, 6, 7,
-		8, 9, 10, 11, 12, 13, 14, 16,
-		17, 18, 19, 20, 21, 23, 24, 25,
-		27, 28, 29, 31, 32, 33, 35, 36,
-		38, 39, 41, 42, 44, 45, 47, 48,
-		49, 51, 52, 54, 55, 57, 58, 60,
-		61, 62, 64, 65, 66, 68, 69, 70,
-		71, 72, 74, 75, 76, 77, 78, 79,
-		80, 81, 82, 83, 84, 85, 86, 87,
-		88, 89, 90, 91, 92, 93, 93, 94,
-		95, 96, 97, 98, 98, 99, 100, 101,
-		102, 102, 103, 104, 105, 105, 106, 107,
-		108, 108, 109, 110, 110, 111, 112, 112,
-		113, 114, 114, 115, 116, 116, 117, 118,
-		118, 119, 120, 120, 121, 121, 122, 123,
-		123, 124, 125, 125, 126, 126, 127, 127,	/* 128 */
-		128, 129, 129, 130, 130, 131, 131, 132,
-		132, 133, 134, 134, 135, 135, 136, 136,
-		137, 137, 138, 138, 139, 139, 140, 140,
-		141, 141, 142, 142, 143, 143, 144, 144,
-		145, 145, 145, 146, 146, 147, 147, 148,
-		148, 149, 149, 150, 150, 150, 151, 151,
-		152, 152, 152, 153, 153, 154, 154, 155,
-		155, 155, 156, 156, 156, 157, 157, 158,
-		158, 158, 159, 159, 160, 160, 160, 161,
-		161, 161, 162, 162, 162, 163, 163, 163,
-		164, 164, 164, 165, 165, 165, 166, 166,
-		166, 167, 167, 167, 168, 168, 168, 169,
-		169, 169, 170, 170, 170, 170, 171, 171,
-		171, 172, 172, 172, 172, 173, 173, 173,
-		174, 174, 174, 174, 175, 175, 175, 176,
-		176, 176, 176, 177, 177, 177, 177, 178,	/* 256 */
-		178, 178, 178, 179, 179, 179, 179, 180,
-		180, 180, 180, 181, 181, 181, 181, 182,
-		182, 182, 182, 182, 183, 183, 183, 183,
-		184, 184, 184, 184, 184, 185, 185, 185,
-		185, 186, 186, 186, 186, 186, 187, 187,
-		187, 187, 187, 188, 188, 188, 188, 188,
-		189, 189, 189, 189, 189, 190, 190, 190,
-		190, 190, 191, 191, 191, 191, 191, 192,
-		192, 192, 192, 192, 192, 193, 193, 193,
-		193, 193, 194, 194, 194, 194, 194, 194,
-		195, 195, 195, 195, 195, 195, 196, 196,
-		196, 196, 196, 196, 197, 197, 197, 197,
-		197, 197, 198, 198, 198, 198, 198, 198,
-		198, 199, 199, 199, 199, 199, 199, 200,
-		200, 200, 200, 200, 200, 200, 201, 201,
-		201, 201, 201, 201, 201, 202, 202, 202,	/* 384 */
-		202, 202, 202, 202, 203, 203, 203, 203,
-		203, 203, 203, 204, 204, 204, 204, 204,
-		204, 204, 204, 205, 205, 205, 205, 205,
-		205, 205, 205, 206, 206, 206, 206, 206,
-		206, 206, 206, 207, 207, 207, 207, 207,
-		207, 207, 207, 208, 208, 208, 208, 208,
-		208, 208, 208, 209, 209, 209, 209, 209,
-		209, 209, 209, 209, 210, 210, 210, 210,
-		210, 210, 210, 210, 210, 211, 211, 211,
-		211, 211, 211, 211, 211, 211, 212, 212,
-		212, 212, 212, 212, 212, 212, 212, 213,
-		213, 213, 213, 213, 213, 213, 213, 213,
-		214, 214, 214, 214, 214, 214, 214, 214,
-		214, 214, 215, 215, 215, 215, 215, 215,
-		215, 215, 215, 216, 216, 216, 216, 216,
-		216, 216, 216, 216, 216, 217, 217, 217,	/* 512 */
-		217, 217, 217, 217, 217, 217, 217, 218,
-		218, 218, 218, 218, 218, 218, 218, 218,
-		218, 219, 219, 219, 219, 219, 219, 219,
-		219, 219, 219, 220, 220, 220, 220, 220,
-		220, 220, 220, 220, 220, 221, 221, 221,
-		221, 221, 221, 221, 221, 221, 221, 221,
-		222, 222, 222, 222, 222, 222, 222, 222,
-		222, 222, 223, 223, 223, 223, 223, 223,
-		223, 223, 223, 223, 223, 224, 224, 224,
-		224, 224, 224, 224, 224, 224, 224, 224,
-		225, 225, 225, 225, 225, 225, 225, 225,
-		225, 225, 225, 226, 226, 226, 226, 226,
-		226, 226, 226, 226, 226, 226, 226, 227,
-		227, 227, 227, 227, 227, 227, 227, 227,
-		227, 227, 228, 228, 228, 228, 228, 228,
-		228, 228, 228, 228, 228, 228, 229, 229,
-		229, 229, 229, 229, 229, 229, 229, 229,
-		229, 229, 230, 230, 230, 230, 230, 230,
-		230, 230, 230, 230, 230, 230, 231, 231,
-		231, 231, 231, 231, 231, 231, 231, 231,
-		231, 231, 231, 232, 232, 232, 232, 232,
-		232, 232, 232, 232, 232, 232, 232, 233,
-		233, 233, 233, 233, 233, 233, 233, 233,
-		233, 233, 233, 233, 234, 234, 234, 234,
-		234, 234, 234, 234, 234, 234, 234, 234,
-		234, 235, 235, 235, 235, 235, 235, 235,
-		235, 235, 235, 235, 235, 235, 236, 236,
-		236, 236, 236, 236, 236, 236, 236, 236,
-		236, 236, 236, 236, 237, 237, 237, 237,
-		237, 237, 237, 237, 237, 237, 237, 237,
-		237, 237, 238, 238, 238, 238, 238, 238,
-		238, 238, 238, 238, 238, 238, 238, 238,
-		239, 239, 239, 239, 239, 239, 239, 239,
-		239, 239, 239, 239, 239, 239, 240, 240,
-		240, 240, 240, 240, 240, 240, 240, 240,
-		240, 240, 240, 240, 241, 241, 241, 241,
-		241, 241, 241, 241, 241, 241, 241, 241,
-		241, 241, 241, 242, 242, 242, 242, 242,
-		242, 242, 242, 242, 242, 242, 242, 242,
-		242, 242, 243, 243, 243, 243, 243, 243,
-		243, 243, 243, 243, 243, 243, 243, 243,
-		243, 244, 244, 244, 244, 244, 244, 244,
-		244, 244, 244, 244, 244, 244, 244, 244,
-		245, 245, 245, 245, 245, 245, 245, 245,
-		245, 245, 245, 245, 245, 245, 245, 246,
-		246, 246, 246, 246, 246, 246, 246, 246,
-		246, 246, 246, 246, 246, 246, 246, 247,
-		247, 247, 247, 247, 247, 247, 247, 247,
-		247, 247, 247, 247, 247, 247, 247, 248,
-		248, 248, 248, 248, 248, 248, 248, 248,
-		248, 248, 248, 248, 248, 248, 248, 249,
-		249, 249, 249, 249, 249, 249, 249, 249,
-		249, 249, 249, 249, 249, 249, 249, 250,
-		250, 250, 250, 250, 250, 250, 250, 250,
-		250, 250, 250, 250, 250, 250, 250, 251,
-		251, 251, 251, 251, 251, 251, 251, 251,
-		251, 251, 251, 251, 251, 251, 251, 252,
-		252, 252, 252, 252, 252, 252, 252, 252,
-		252, 252, 252, 252, 252, 252, 252, 253,
-		253, 253, 253, 253, 253, 253, 253, 253,
-		253, 253, 253, 253, 253, 253, 253, 253,
-		254, 254, 254, 254, 254, 254, 254, 254,
-		254, 254, 254, 254, 254, 254, 254, 254,
-		255, 255, 255, 255, 255, 255, 255, 255
-};
-
-#else
-#error "VAMEM version must be one of {VAMEM_VERSION_1, VAMEM_VERSION_2}"
-#endif
-
-void
-ia_css_config_gamma_table(void)
-{
-#if defined(HAS_VAMEM_VERSION_2)
-	memcpy(default_gamma_table.data.vamem_2, default_gamma_table_data,
-	       sizeof(default_gamma_table_data));
-	default_gamma_table.vamem_type   = IA_CSS_VAMEM_TYPE_2;
-#else
-	memcpy(default_gamma_table.data.vamem_1, default_gamma_table_data,
-	       sizeof(default_gamma_table_data));
-	default_gamma_table.vamem_type   = IA_CSS_VAMEM_TYPE_1;
-#endif
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h
deleted file mode 100644
index 9686623d9cdd..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_GC_TABLE_HOST_H
-#define __IA_CSS_GC_TABLE_HOST_H
-
-#include "ia_css_gc_types.h"
-
-extern struct ia_css_gamma_table default_gamma_table;
-
-void ia_css_config_gamma_table(void);
-
-#endif /* __IA_CSS_GC_TABLE_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_types.h
deleted file mode 100644
index 594807fe2925..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_1.0/ia_css_gc_types.h
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_GC_TYPES_H
-#define __IA_CSS_GC_TYPES_H
-
-/* @file
-* CSS-API header file for Gamma Correction parameters.
-*/
-
-#include "isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h"  /* FIXME: Needed for ia_css_vamem_type */
-
-/* Fractional bits for GAMMA gain */
-#define IA_CSS_GAMMA_GAIN_K_SHIFT      13
-
-/* Number of elements in the gamma table. */
-#define IA_CSS_VAMEM_1_GAMMA_TABLE_SIZE_LOG2    10
-#define IA_CSS_VAMEM_1_GAMMA_TABLE_SIZE         (1U<<IA_CSS_VAMEM_1_GAMMA_TABLE_SIZE_LOG2)
-
-/* Number of elements in the gamma table. */
-#define IA_CSS_VAMEM_2_GAMMA_TABLE_SIZE_LOG2    8
-#define IA_CSS_VAMEM_2_GAMMA_TABLE_SIZE         ((1U<<IA_CSS_VAMEM_2_GAMMA_TABLE_SIZE_LOG2) + 1)
-
-/* Gamma table, used for Y(Luma) Gamma Correction.
- *
- *  ISP block: GC1 (YUV Gamma Correction)
- *  ISP1: GC1 is used.
- * (ISP2: GC2(sRGB Gamma Correction) is used.)
- */
-/** IA_CSS_VAMEM_TYPE_1(ISP2300) or
-     IA_CSS_VAMEM_TYPE_2(ISP2400) */
-union ia_css_gc_data {
-	uint16_t vamem_1[IA_CSS_VAMEM_1_GAMMA_TABLE_SIZE];
-	/** Y(Luma) Gamma table on vamem type 1. u0.8, [0,255] */
-	uint16_t vamem_2[IA_CSS_VAMEM_2_GAMMA_TABLE_SIZE];
-	/** Y(Luma) Gamma table on vamem type 2. u0.8, [0,255] */
-};
-
-struct ia_css_gamma_table {
-	enum ia_css_vamem_type vamem_type;
-	union ia_css_gc_data data;
-};
-
-/* Gamma Correction configuration (used only for YUV Gamma Correction).
- *
- *  ISP block: GC1 (YUV Gamma Correction)
- *  ISP1: GC1 is used.
- * (ISP2: GC2 (sRGB Gamma Correction) is used.)
-  */
-struct ia_css_gc_config {
-	uint16_t gain_k1; /** Gain to adjust U after YUV Gamma Correction.
-				u0.16, [0,65535],
-				default/ineffective 19000(0.29) */
-	uint16_t gain_k2; /** Gain to adjust V after YUV Gamma Correction.
-				u0.16, [0,65535],
-				default/ineffective 19000(0.29) */
-};
-
-/* Chroma Enhancement configuration.
- *
- *  This parameter specifies range of chroma output level.
- *  The standard range is [0,255] or [16,240].
- *
- *  ISP block: CE1
- *  ISP1: CE1 is used.
- * (ISP2: CE1 is not used.)
- */
-struct ia_css_ce_config {
-	uint8_t uv_level_min; /** Minimum of chroma output level.
-				u0.8, [0,255], default/ineffective 0 */
-	uint8_t uv_level_max; /** Maximum of chroma output level.
-				u0.8, [0,255], default/ineffective 255 */
-};
-
-/* Multi-Axes Color Correction (MACC) configuration.
- *
- *  ISP block: MACC2 (MACC by matrix and exponent(ia_css_macc_config))
- * (ISP1: MACC1 (MACC by only matrix) is used.)
- *  ISP2: MACC2 is used.
- */
-struct ia_css_macc_config {
-	uint8_t exp;	/** Common exponent of ia_css_macc_table.
-				u8.0, [0,13], default 1, ineffective 1 */
-};
-
-#endif /* __IA_CSS_GC_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.c
deleted file mode 100644
index 0fb1a919fccd..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.c
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#ifndef IA_CSS_NO_DEBUG
-/* FIXME: See BZ 4427 */
-#include "ia_css_debug.h"
-#endif
-#include "csc/csc_1.0/ia_css_csc.host.h"
-#include "vamem.h"
-
-#include "ia_css_gc2.host.h"
-
-const struct ia_css_cc_config default_yuv2rgb_cc_config = {
-	12,
-	{4096, -4096, 4096, 4096, 4096, 0, 4096, -4096, -4096}
-};
-
-const struct ia_css_cc_config default_rgb2yuv_cc_config = {
-	13,
-	{2449, 4809, 934, -1382, -2714, 4096, 4096, -3430, -666}
-};
-
-void
-ia_css_yuv2rgb_encode(
-	struct sh_css_isp_csc_params *to,
-	const struct ia_css_cc_config *from,
-	unsigned size)
-{
-	ia_css_encode_cc(to, from, size);
-}
-
-void
-ia_css_rgb2yuv_encode(
-	struct sh_css_isp_csc_params *to,
-	const struct ia_css_cc_config *from,
-	unsigned size)
-{
-	ia_css_encode_cc(to, from, size);
-}
-
-void
-ia_css_r_gamma_vamem_encode(
-	struct sh_css_isp_rgb_gamma_vamem_params *to,
-	const struct ia_css_rgb_gamma_table *from,
-	unsigned size)
-{
-	(void)size;
-	memcpy (&to->gc,  &from->data, sizeof(to->gc));
-}
-
-void
-ia_css_g_gamma_vamem_encode(
-	struct sh_css_isp_rgb_gamma_vamem_params *to,
-	const struct ia_css_rgb_gamma_table *from,
-	unsigned size)
-{
-	(void)size;
-	memcpy (&to->gc,  &from->data, sizeof(to->gc));
-}
-
-void
-ia_css_b_gamma_vamem_encode(
-	struct sh_css_isp_rgb_gamma_vamem_params *to,
-	const struct ia_css_rgb_gamma_table *from,
-	unsigned size)
-{
-	(void)size;
-	memcpy (&to->gc,  &from->data, sizeof(to->gc));
-}
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_yuv2rgb_dump(
-	const struct sh_css_isp_csc_params *yuv2rgb,
-	unsigned level)
-{
-	ia_css_cc_dump(yuv2rgb, level, "YUV to RGB Conversion");
-}
-
-void
-ia_css_rgb2yuv_dump(
-	const struct sh_css_isp_csc_params *rgb2yuv,
-	unsigned level)
-{
-	ia_css_cc_dump(rgb2yuv, level, "RGB to YUV Conversion");
-}
-
-void
-ia_css_rgb_gamma_table_debug_dtrace(
-	const struct ia_css_rgb_gamma_table *config,
-	unsigned level)
-{
-	(void)config;
-	(void)level;
-}
-#endif
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.h
deleted file mode 100644
index ba140eefd525..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2.host.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_GC2_HOST_H
-#define __IA_CSS_GC2_HOST_H
-
-#include "ia_css_gc2_types.h"
-#include "ia_css_gc2_param.h"
-#include "ia_css_gc2_table.host.h"
-
-extern const struct ia_css_cc_config default_yuv2rgb_cc_config;
-extern const struct ia_css_cc_config default_rgb2yuv_cc_config;
-
-void
-ia_css_yuv2rgb_encode(
-	struct sh_css_isp_csc_params *to,
-	const struct ia_css_cc_config *from,
-	unsigned size);
-
-void
-ia_css_rgb2yuv_encode(
-	struct sh_css_isp_csc_params *to,
-	const struct ia_css_cc_config *from,
-	unsigned size);
-
-void
-ia_css_r_gamma_vamem_encode(
-	struct sh_css_isp_rgb_gamma_vamem_params *to,
-	const struct ia_css_rgb_gamma_table *from,
-	unsigned size);
-
-void
-ia_css_g_gamma_vamem_encode(
-	struct sh_css_isp_rgb_gamma_vamem_params *to,
-	const struct ia_css_rgb_gamma_table *from,
-	unsigned size);
-
-void
-ia_css_b_gamma_vamem_encode(
-	struct sh_css_isp_rgb_gamma_vamem_params *to,
-	const struct ia_css_rgb_gamma_table *from,
-	unsigned size);
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_yuv2rgb_dump(
-	const struct sh_css_isp_csc_params *yuv2rgb,
-	unsigned level);
-
-void
-ia_css_rgb2yuv_dump(
-	const struct sh_css_isp_csc_params *rgb2yuv,
-	unsigned level);
-
-void
-ia_css_rgb_gamma_table_debug_dtrace(
-	const struct ia_css_rgb_gamma_table *config,
-	unsigned level);
-
-#define ia_css_yuv2rgb_debug_dtrace ia_css_cc_config_debug_dtrace
-#define ia_css_rgb2yuv_debug_dtrace ia_css_cc_config_debug_dtrace
-#define ia_css_r_gamma_debug_dtrace ia_css_rgb_gamma_table_debug_dtrace
-#define ia_css_g_gamma_debug_dtrace ia_css_rgb_gamma_table_debug_dtrace
-#define ia_css_b_gamma_debug_dtrace ia_css_rgb_gamma_table_debug_dtrace
-
-#endif
-
-#endif /* __IA_CSS_GC2_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_param.h
deleted file mode 100644
index d25239f4d86f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_param.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_GC2_PARAM_H
-#define __IA_CSS_GC2_PARAM_H
-
-#include "type_support.h"
-/* Extend GC1 */
-#include "ia_css_gc2_types.h"
-#include "gc/gc_1.0/ia_css_gc_param.h"
-#include "csc/csc_1.0/ia_css_csc_param.h"
-
-#ifndef PIPE_GENERATION
-#if defined(IS_VAMEM_VERSION_1)
-#define SH_CSS_ISP_RGB_GAMMA_TABLE_SIZE IA_CSS_VAMEM_1_RGB_GAMMA_TABLE_SIZE
-#elif defined(IS_VAMEM_VERSION_2)
-#define SH_CSS_ISP_RGB_GAMMA_TABLE_SIZE IA_CSS_VAMEM_2_RGB_GAMMA_TABLE_SIZE
-#else
-#error "Undefined vamem version"
-#endif
-
-#else
-/* For pipe generation, the size is not relevant */
-#define SH_CSS_ISP_RGB_GAMMA_TABLE_SIZE 0
-#endif
-
-/* This should be vamem_data_t, but that breaks the pipe generator */
-struct sh_css_isp_rgb_gamma_vamem_params {
-	uint16_t gc[SH_CSS_ISP_RGB_GAMMA_TABLE_SIZE];
-};
-
-#endif /* __IA_CSS_GC2_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c
deleted file mode 100644
index f14a66b78714..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <type_support.h>
-#include <string_support.h> /* memcpy */
-#include "system_global.h"
-#include "vamem.h"
-#include "ia_css_types.h"
-#include "ia_css_gc2_table.host.h"
-
-struct ia_css_rgb_gamma_table default_r_gamma_table;
-struct ia_css_rgb_gamma_table default_g_gamma_table;
-struct ia_css_rgb_gamma_table default_b_gamma_table;
-
-/* Identical default gamma table for R, G, and B. */
-
-#if defined(HAS_VAMEM_VERSION_2)
-
-static const uint16_t
-default_gamma_table_data[IA_CSS_VAMEM_2_RGB_GAMMA_TABLE_SIZE] = {
-   0,   72,  144,  216,  288,  360,  426,  486,
- 541,  592,  641,  687,  730,  772,  812,  850,
- 887,  923,  958,  991, 1024, 1055, 1086, 1117,
-1146, 1175, 1203, 1230, 1257, 1284, 1310, 1335,
-1360, 1385, 1409, 1433, 1457, 1480, 1502, 1525,
-1547, 1569, 1590, 1612, 1632, 1653, 1674, 1694,
-1714, 1734, 1753, 1772, 1792, 1811, 1829, 1848,
-1866, 1884, 1902, 1920, 1938, 1955, 1973, 1990,
-2007, 2024, 2040, 2057, 2074, 2090, 2106, 2122,
-2138, 2154, 2170, 2185, 2201, 2216, 2231, 2247,
-2262, 2277, 2291, 2306, 2321, 2335, 2350, 2364,
-2378, 2393, 2407, 2421, 2435, 2449, 2462, 2476,
-2490, 2503, 2517, 2530, 2543, 2557, 2570, 2583,
-2596, 2609, 2622, 2634, 2647, 2660, 2673, 2685,
-2698, 2710, 2722, 2735, 2747, 2759, 2771, 2783,
-2795, 2807, 2819, 2831, 2843, 2855, 2867, 2878,
-2890, 2901, 2913, 2924, 2936, 2947, 2958, 2970,
-2981, 2992, 3003, 3014, 3025, 3036, 3047, 3058,
-3069, 3080, 3091, 3102, 3112, 3123, 3134, 3144,
-3155, 3165, 3176, 3186, 3197, 3207, 3217, 3228,
-3238, 3248, 3258, 3268, 3279, 3289, 3299, 3309,
-3319, 3329, 3339, 3349, 3358, 3368, 3378, 3388,
-3398, 3407, 3417, 3427, 3436, 3446, 3455, 3465,
-3474, 3484, 3493, 3503, 3512, 3521, 3531, 3540,
-3549, 3559, 3568, 3577, 3586, 3595, 3605, 3614,
-3623, 3632, 3641, 3650, 3659, 3668, 3677, 3686,
-3694, 3703, 3712, 3721, 3730, 3739, 3747, 3756,
-3765, 3773, 3782, 3791, 3799, 3808, 3816, 3825,
-3833, 3842, 3850, 3859, 3867, 3876, 3884, 3893,
-3901, 3909, 3918, 3926, 3934, 3942, 3951, 3959,
-3967, 3975, 3984, 3992, 4000, 4008, 4016, 4024,
-4032, 4040, 4048, 4056, 4064, 4072, 4080, 4088,
-4095
-};
-#elif defined(HAS_VAMEM_VERSION_1)
-
-static const uint16_t
-default_gamma_table_data[IA_CSS_VAMEM_1_RGB_GAMMA_TABLE_SIZE] = {
-   0,   72,  144,  216,  288,  360,  426,  486,
- 541,  592,  641,  687,  730,  772,  812,  850,
- 887,  923,  958,  991, 1024, 1055, 1086, 1117,
-1146, 1175, 1203, 1230, 1257, 1284, 1310, 1335,
-1360, 1385, 1409, 1433, 1457, 1480, 1502, 1525,
-1547, 1569, 1590, 1612, 1632, 1653, 1674, 1694,
-1714, 1734, 1753, 1772, 1792, 1811, 1829, 1848,
-1866, 1884, 1902, 1920, 1938, 1955, 1973, 1990,
-2007, 2024, 2040, 2057, 2074, 2090, 2106, 2122,
-2138, 2154, 2170, 2185, 2201, 2216, 2231, 2247,
-2262, 2277, 2291, 2306, 2321, 2335, 2350, 2364,
-2378, 2393, 2407, 2421, 2435, 2449, 2462, 2476,
-2490, 2503, 2517, 2530, 2543, 2557, 2570, 2583,
-2596, 2609, 2622, 2634, 2647, 2660, 2673, 2685,
-2698, 2710, 2722, 2735, 2747, 2759, 2771, 2783,
-2795, 2807, 2819, 2831, 2843, 2855, 2867, 2878,
-2890, 2901, 2913, 2924, 2936, 2947, 2958, 2970,
-2981, 2992, 3003, 3014, 3025, 3036, 3047, 3058,
-3069, 3080, 3091, 3102, 3112, 3123, 3134, 3144,
-3155, 3165, 3176, 3186, 3197, 3207, 3217, 3228,
-3238, 3248, 3258, 3268, 3279, 3289, 3299, 3309,
-3319, 3329, 3339, 3349, 3358, 3368, 3378, 3388,
-3398, 3407, 3417, 3427, 3436, 3446, 3455, 3465,
-3474, 3484, 3493, 3503, 3512, 3521, 3531, 3540,
-3549, 3559, 3568, 3577, 3586, 3595, 3605, 3614,
-3623, 3632, 3641, 3650, 3659, 3668, 3677, 3686,
-3694, 3703, 3712, 3721, 3730, 3739, 3747, 3756,
-3765, 3773, 3782, 3791, 3799, 3808, 3816, 3825,
-3833, 3842, 3850, 3859, 3867, 3876, 3884, 3893,
-3901, 3909, 3918, 3926, 3934, 3942, 3951, 3959,
-3967, 3975, 3984, 3992, 4000, 4008, 4016, 4024,
-4032, 4040, 4048, 4056, 4064, 4072, 4080, 4088
-};
-#else
-#error "VAMEM version must be one of {VAMEM_VERSION_1, VAMEM_VERSION_2}"
-#endif
-
-void
-ia_css_config_rgb_gamma_tables(void)
-{
-#if defined(HAS_VAMEM_VERSION_2)
-	default_r_gamma_table.vamem_type   = IA_CSS_VAMEM_TYPE_2;
-	default_g_gamma_table.vamem_type   = IA_CSS_VAMEM_TYPE_2;
-	default_b_gamma_table.vamem_type   = IA_CSS_VAMEM_TYPE_2;
-	memcpy(default_r_gamma_table.data.vamem_2, default_gamma_table_data,
-	       sizeof(default_gamma_table_data));
-	memcpy(default_g_gamma_table.data.vamem_2, default_gamma_table_data,
-	       sizeof(default_gamma_table_data));
-	memcpy(default_b_gamma_table.data.vamem_2, default_gamma_table_data,
-	       sizeof(default_gamma_table_data));
-#else
-	memcpy(default_r_gamma_table.data.vamem_1, default_gamma_table_data,
-	       sizeof(default_gamma_table_data));
-	memcpy(default_g_gamma_table.data.vamem_1, default_gamma_table_data,
-	       sizeof(default_gamma_table_data));
-	memcpy(default_b_gamma_table.data.vamem_1, default_gamma_table_data,
-	       sizeof(default_gamma_table_data));
-	default_r_gamma_table.vamem_type   = IA_CSS_VAMEM_TYPE_1;
-	default_g_gamma_table.vamem_type   = IA_CSS_VAMEM_TYPE_1;
-	default_b_gamma_table.vamem_type   = IA_CSS_VAMEM_TYPE_1;
-#endif
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h
deleted file mode 100644
index 8686e6e3586c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_GC2_TABLE_HOST_H
-#define __IA_CSS_GC2_TABLE_HOST_H
-
-#include "ia_css_gc2_types.h"
-
-extern struct ia_css_rgb_gamma_table default_r_gamma_table;
-extern struct ia_css_rgb_gamma_table default_g_gamma_table;
-extern struct ia_css_rgb_gamma_table default_b_gamma_table;
-
-void ia_css_config_rgb_gamma_tables(void);
-
-#endif /* __IA_CSS_GC2_TABLE_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_types.h
deleted file mode 100644
index fab7467d30a5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/gc/gc_2/ia_css_gc2_types.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_GC2_TYPES_H
-#define __IA_CSS_GC2_TYPES_H
-
-#include "isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h"  /* FIXME: needed for ia_css_vamem_type */
-
-/* @file
-* CSS-API header file for Gamma Correction parameters.
-*/
-
-/* sRGB Gamma table, used for sRGB Gamma Correction.
- *
- *  ISP block: GC2 (sRGB Gamma Correction)
- * (ISP1: GC1(YUV Gamma Correction) is used.)
- *  ISP2: GC2 is used.
- */
-
-/* Number of elements in the sRGB gamma table. */
-#define IA_CSS_VAMEM_1_RGB_GAMMA_TABLE_SIZE_LOG2 8
-#define IA_CSS_VAMEM_1_RGB_GAMMA_TABLE_SIZE      (1U<<IA_CSS_VAMEM_1_RGB_GAMMA_TABLE_SIZE_LOG2)
-
-/* Number of elements in the sRGB gamma table. */
-#define IA_CSS_VAMEM_2_RGB_GAMMA_TABLE_SIZE_LOG2    8
-#define IA_CSS_VAMEM_2_RGB_GAMMA_TABLE_SIZE     ((1U<<IA_CSS_VAMEM_2_RGB_GAMMA_TABLE_SIZE_LOG2) + 1)
-
-/** IA_CSS_VAMEM_TYPE_1(ISP2300) or
-     IA_CSS_VAMEM_TYPE_2(ISP2400) */
-union ia_css_rgb_gamma_data {
-	uint16_t vamem_1[IA_CSS_VAMEM_1_RGB_GAMMA_TABLE_SIZE];
-	/** RGB Gamma table on vamem type1. This table is not used,
-		because sRGB Gamma Correction is not implemented for ISP2300. */
-	uint16_t vamem_2[IA_CSS_VAMEM_2_RGB_GAMMA_TABLE_SIZE];
-		/** RGB Gamma table on vamem type2. u0.12, [0,4095] */
-};
-
-struct ia_css_rgb_gamma_table {
-	enum ia_css_vamem_type vamem_type;
-	union ia_css_rgb_gamma_data data;
-};
-
-#endif /* __IA_CSS_GC2_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr.host.c
deleted file mode 100644
index 8215ae47d384..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr.host.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/* Release Version: irci_stable_candrpv_0415_20150521_0458 */
-/* Release Version: irci_ecr-master_20150911_0724 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_hdr.host.h"
-
-void
-ia_css_hdr_init_config(
-	struct sh_css_isp_hdr_params *to,
-	const struct ia_css_hdr_config *from,
-	unsigned size)
-{
-	int i;
-	(void)size;
-
-	for (i = 0; i < HDR_NUM_INPUT_FRAMES - 1; i++) {
-		to->irradiance.match_shift[i] = from->irradiance.match_shift[i];
-		to->irradiance.match_mul[i]   = from->irradiance.match_mul[i];
-		to->irradiance.thr_low[i]     = from->irradiance.thr_low[i];
-		to->irradiance.thr_high[i]    = from->irradiance.thr_high[i];
-		to->irradiance.thr_coeff[i]   = from->irradiance.thr_coeff[i];
-		to->irradiance.thr_shift[i]   = from->irradiance.thr_shift[i];
-	}
-	to->irradiance.test_irr    = from->irradiance.test_irr;
-	to->irradiance.weight_bpp  = from->irradiance.weight_bpp;
-
-	to->deghost.test_deg    = from->deghost.test_deg;
-	to->exclusion.test_excl = from->exclusion.test_excl;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr.host.h
deleted file mode 100644
index 8f89bc8f1ca2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr.host.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/* Release Version: irci_stable_candrpv_0415_20150521_0458 */
-/* Release Version: irci_ecr-master_20150911_0724 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_HDR_HOST_H
-#define __IA_CSS_HDR_HOST_H
-
-#include "ia_css_hdr_param.h"
-#include "ia_css_hdr_types.h"
-
-extern const struct ia_css_hdr_config default_hdr_config;
-
-void
-ia_css_hdr_init_config(
-	struct sh_css_isp_hdr_params *to,
-	const struct ia_css_hdr_config *from,
-	unsigned size);
-
-#endif /* __IA_CSS_HDR_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr_param.h
deleted file mode 100644
index 1c053af7d0d3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr_param.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/* Release Version: irci_stable_candrpv_0415_20150521_0458 */
-/* Release Version: irci_ecr-master_20150911_0724 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_HDR_PARAMS_H
-#define __IA_CSS_HDR_PARAMS_H
-
-#include "type_support.h"
-
-#define HDR_NUM_INPUT_FRAMES         (3)
-
-/* HDR irradiance map parameters on ISP. */
-struct sh_css_hdr_irradiance_params {
-	int32_t test_irr;
-	int32_t match_shift[HDR_NUM_INPUT_FRAMES - 1];  /* Histogram matching shift parameter */
-	int32_t match_mul[HDR_NUM_INPUT_FRAMES - 1];    /* Histogram matching multiplication parameter */
-	int32_t thr_low[HDR_NUM_INPUT_FRAMES - 1];      /* Weight map soft threshold low bound parameter */
-	int32_t thr_high[HDR_NUM_INPUT_FRAMES - 1];     /* Weight map soft threshold high bound parameter */
-	int32_t thr_coeff[HDR_NUM_INPUT_FRAMES - 1];    /* Soft threshold linear function coefficient */
-	int32_t thr_shift[HDR_NUM_INPUT_FRAMES - 1];    /* Soft threshold precision shift parameter */
-	int32_t weight_bpp;                             /* Weight map bits per pixel */
-};
-
-/* HDR deghosting parameters on ISP */
-struct sh_css_hdr_deghost_params {
-	int32_t test_deg;
-};
-
-/* HDR exclusion parameters on ISP */
-struct sh_css_hdr_exclusion_params {
-	int32_t test_excl;
-};
-
-/* HDR ISP parameters */
-struct sh_css_isp_hdr_params {
-	struct sh_css_hdr_irradiance_params irradiance;
-	struct sh_css_hdr_deghost_params    deghost;
-	struct sh_css_hdr_exclusion_params  exclusion;
-};
-
-#endif /* __IA_CSS_HDR_PARAMS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr_types.h
deleted file mode 100644
index 26464421b077..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/hdr/ia_css_hdr_types.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/* Release Version: irci_stable_candrpv_0415_20150521_0458 */
-/* Release Version: irci_ecr-master_20150911_0724 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_HDR_TYPES_H
-#define __IA_CSS_HDR_TYPES_H
-
-#define IA_CSS_HDR_MAX_NUM_INPUT_FRAMES         (3)
-
-/**
- * \brief HDR Irradiance Parameters
- * \detail Currently HDR paramters are used only for testing purposes
- */
-struct ia_css_hdr_irradiance_params {
-	int test_irr;                                          /** Test parameter */
-	int match_shift[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];  /** Histogram matching shift parameter */
-	int match_mul[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];    /** Histogram matching multiplication parameter */
-	int thr_low[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];      /** Weight map soft threshold low bound parameter */
-	int thr_high[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];     /** Weight map soft threshold high bound parameter */
-	int thr_coeff[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];    /** Soft threshold linear function coefficien */
-	int thr_shift[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];    /** Soft threshold precision shift parameter */
-	int weight_bpp;                                        /** Weight map bits per pixel */
-};
-
-/**
- * \brief HDR Deghosting Parameters
- * \detail Currently HDR paramters are used only for testing purposes
- */
-struct ia_css_hdr_deghost_params {
-	int test_deg; /** Test parameter */
-};
-
-/**
- * \brief HDR Exclusion Parameters
- * \detail Currently HDR paramters are used only for testing purposes
- */
-struct ia_css_hdr_exclusion_params {
-	int test_excl; /** Test parameter */
-};
-
-/**
- * \brief HDR public paramterers.
- * \details Struct with all paramters for HDR that can be seet from
- * the CSS API. Currenly, only test paramters are defined.
- */
-struct ia_css_hdr_config {
-	struct ia_css_hdr_irradiance_params irradiance; /** HDR irradiance paramaters */
-	struct ia_css_hdr_deghost_params    deghost;    /** HDR deghosting parameters */
-	struct ia_css_hdr_exclusion_params  exclusion; /** HDR exclusion parameters */
-};
-
-#endif /* __IA_CSS_HDR_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.c
deleted file mode 100644
index a31c9e828e22..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.c
+++ /dev/null
@@ -1,86 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_bayer_io.host.h"
-#include "dma.h"
-#include "math_support.h"
-#ifndef IA_CSS_NO_DEBUG
-#include "ia_css_debug.h"
-#endif
-#include "ia_css_isp_params.h"
-#include "ia_css_frame.h"
-
-void
-ia_css_bayer_io_config(
-	const struct ia_css_binary      *binary,
-	const struct sh_css_binary_args *args)
-{
-	const struct ia_css_frame *in_frame = args->in_frame;
-	const struct ia_css_frame **out_frames = (const struct ia_css_frame **)& args->out_frame;
-	const struct ia_css_frame_info *in_frame_info = (in_frame) ? &in_frame->info : &binary->in_frame_info;
-
-	const unsigned ddr_bits_per_element = sizeof(short) * 8;
-	const unsigned ddr_elems_per_word = ceil_div(HIVE_ISP_DDR_WORD_BITS, ddr_bits_per_element);
-	unsigned size_get = 0, size_put = 0;
-	unsigned offset = 0;
-
-	if (binary->info->mem_offsets.offsets.param) {
-		size_get = binary->info->mem_offsets.offsets.param->dmem.get.size;
-		offset = binary->info->mem_offsets.offsets.param->dmem.get.offset;
-	}
-
-	if (size_get) {
-		struct ia_css_common_io_config *to = (struct ia_css_common_io_config *)&binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-		struct dma_port_config config;
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_bayer_io_config() get part enter:\n");
-#endif
-
-		ia_css_dma_configure_from_info(&config, in_frame_info);
-		// The base_address of the input frame will be set in the ISP
-		to->width = in_frame_info->res.width;
-		to->height = in_frame_info->res.height;
-		to->stride = config.stride;
-		to->ddr_elems_per_word = ddr_elems_per_word;
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_bayer_io_config() get part leave:\n");
-#endif
-	}
-
-	if (binary->info->mem_offsets.offsets.param) {
-		size_put = binary->info->mem_offsets.offsets.param->dmem.put.size;
-		offset = binary->info->mem_offsets.offsets.param->dmem.put.offset;
-	}
-
-	if (size_put) {
-		struct ia_css_common_io_config *to = (struct ia_css_common_io_config *)&binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-		struct dma_port_config config;
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_bayer_io_config() put part enter:\n");
-#endif
-
-		ia_css_dma_configure_from_info(&config, &out_frames[0]->info);
-		to->base_address = out_frames[0]->data;
-		to->width = out_frames[0]->info.res.width;
-		to->height = out_frames[0]->info.res.height;
-		to->stride = config.stride;
-		to->ddr_elems_per_word = ddr_elems_per_word;
-
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_bayer_io_config() put part leave:\n");
-#endif
-	}
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.h
deleted file mode 100644
index 7e5d4cfe3454..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.h
+++ /dev/null
@@ -1,31 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __BAYER_IO_HOST_H
-#define __BAYER_IO_HOST_H
-
-#include "ia_css_bayer_io_param.h"
-#include "ia_css_bayer_io_types.h"
-#include "ia_css_binary.h"
-#include "sh_css_internal.h"
-
-
-void
-ia_css_bayer_io_config(
-	const struct ia_css_binary     *binary,
-	const struct sh_css_binary_args *args);
-
-#endif /*__BAYER_IO_HOST_H */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io_param.h
deleted file mode 100644
index 7b6f581c4a80..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io_param.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BAYER_IO_PARAM
-#define __IA_CSS_BAYER_IO_PARAM
-
-#include "../common/ia_css_common_io_param.h"
-
-#endif /* __IA_CSS_BAYER_IO_PARAM */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io_types.h
deleted file mode 100644
index 2291b01452f8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io_types.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_BAYER_IO_TYPES_H
-#define __IA_CSS_BAYER_IO_TYPES_H
-
-#include "../common/ia_css_common_io_types.h"
-
-#endif /* __IA_CSS_BAYER_IO_TYPES_H */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/common/ia_css_common_io_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/common/ia_css_common_io_param.h
deleted file mode 100644
index f1ce03aa7951..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/common/ia_css_common_io_param.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_COMMON_IO_PARAM
-#define __IA_CSS_COMMON_IO_PARAM
-
-#include "../common/ia_css_common_io_types.h"
-
-#endif /* __IA_CSS_COMMON_IO_PARAM */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/common/ia_css_common_io_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/common/ia_css_common_io_types.h
deleted file mode 100644
index 8a9a97063264..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/common/ia_css_common_io_types.h
+++ /dev/null
@@ -1,31 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_COMMON_IO_TYPES
-#define __IA_CSS_COMMON_IO_TYPES
-
-#define MAX_IO_DMA_CHANNELS 2
-
-struct ia_css_common_io_config {
-	unsigned base_address;
-	unsigned width;
-	unsigned height;
-	unsigned stride;
-	unsigned ddr_elems_per_word;
-	unsigned dma_channel[MAX_IO_DMA_CHANNELS];
-};
-
-#endif /* __IA_CSS_COMMON_IO_TYPES */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
deleted file mode 100644
index 91fb5168c357..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_YUV444_IO_PARAM
-#define __IA_CSS_YUV444_IO_PARAM
-
-#include "../common/ia_css_common_io_param.h"
-
-#endif
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
deleted file mode 100644
index dac440309394..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_YUV444_IO_TYPES
-#define __IA_CSS_YUV444_IO_TYPES
-
-#include "../common/ia_css_common_io_types.h"
-
-#endif
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c
deleted file mode 100644
index f80480cf9de2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c
+++ /dev/null
@@ -1,86 +0,0 @@
-#ifdef ISP2401
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#include "ia_css_bayer_io.host.h"
-#include "dma.h"
-#include "math_support.h"
-#ifndef IA_CSS_NO_DEBUG
-#include "ia_css_debug.h"
-#endif
-#include "ia_css_isp_params.h"
-#include "ia_css_frame.h"
-
-void
-ia_css_bayer_io_config(
-	const struct ia_css_binary      *binary,
-	const struct sh_css_binary_args *args)
-{
-	const struct ia_css_frame *in_frame = args->in_frame;
-	const struct ia_css_frame **out_frames = (const struct ia_css_frame **)& args->out_frame;
-	const struct ia_css_frame_info *in_frame_info = (in_frame) ? &in_frame->info : &binary->in_frame_info;
-
-	const unsigned ddr_bits_per_element = sizeof(short) * 8;
-	const unsigned ddr_elems_per_word = ceil_div(HIVE_ISP_DDR_WORD_BITS, ddr_bits_per_element);
-	unsigned size_get = 0, size_put = 0;
-	unsigned offset = 0;
-
-	if (binary->info->mem_offsets.offsets.param) {
-		size_get = binary->info->mem_offsets.offsets.param->dmem.get.size;
-		offset = binary->info->mem_offsets.offsets.param->dmem.get.offset;
-	}
-
-	if (size_get) {
-		struct ia_css_common_io_config *to = (struct ia_css_common_io_config *)&binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-		struct dma_port_config config;
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_bayer_io_config() get part enter:\n");
-#endif
-
-		ia_css_dma_configure_from_info(&config, in_frame_info);
-		// The base_address of the input frame will be set in the ISP
-		to->width = in_frame_info->res.width;
-		to->height = in_frame_info->res.height;
-		to->stride = config.stride;
-		to->ddr_elems_per_word = ddr_elems_per_word;
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_bayer_io_config() get part leave:\n");
-#endif
-	}
-
-	if (binary->info->mem_offsets.offsets.param) {
-		size_put = binary->info->mem_offsets.offsets.param->dmem.put.size;
-		offset = binary->info->mem_offsets.offsets.param->dmem.put.offset;
-	}
-
-	if (size_put) {
-		struct ia_css_common_io_config *to = (struct ia_css_common_io_config *)&binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-		struct dma_port_config config;
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_bayer_io_config() put part enter:\n");
-#endif
-
-		ia_css_dma_configure_from_info(&config, &out_frames[0]->info);
-		to->base_address = out_frames[0]->data;
-		to->width = out_frames[0]->info.res.width;
-		to->height = out_frames[0]->info.res.height;
-		to->stride = config.stride;
-		to->ddr_elems_per_word = ddr_elems_per_word;
-
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_bayer_io_config() put part leave:\n");
-#endif
-	}
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h
deleted file mode 100644
index ab9fa31bfc5e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h
+++ /dev/null
@@ -1,31 +0,0 @@
-#ifdef ISP2401
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#ifndef __BAYER_IO_HOST_H
-#define __BAYER_IO_HOST_H
-
-#include "ia_css_bayer_io_param.h"
-#include "ia_css_bayer_io_types.h"
-#include "ia_css_binary.h"
-#include "sh_css_internal.h"
-
-
-void
-ia_css_bayer_io_config(
-	const struct ia_css_binary     *binary,
-	const struct sh_css_binary_args *args);
-
-#endif /*__BAYER_IO_HOST_H */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h
deleted file mode 100644
index bf5a3eccb330..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifdef ISP2401
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#ifndef __IA_CSS_BAYER_IO_PARAM
-#define __IA_CSS_BAYER_IO_PARAM
-
-#include "../common/ia_css_common_io_param.h"
-
-#endif /* __IA_CSS_BAYER_IO_PARAM */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h
deleted file mode 100644
index 9e3c622db4d4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifdef ISP2401
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#ifndef __IA_CSS_BAYER_IO_TYPES_H
-#define __IA_CSS_BAYER_IO_TYPES_H
-
-#include "../common/ia_css_common_io_types.h"
-
-#endif /* __IA_CSS_BAYER_IO_TYPES_H */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h
deleted file mode 100644
index e5fdcfff0cf7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifdef ISP2401
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#ifndef __IA_CSS_COMMON_IO_PARAM
-#define __IA_CSS_COMMON_IO_PARAM
-
-#include "../common/ia_css_common_io_types.h"
-
-#endif /* __IA_CSS_COMMON_IO_PARAM */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h
deleted file mode 100644
index 0a19e2d1aff4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h
+++ /dev/null
@@ -1,31 +0,0 @@
-#ifdef ISP2401
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#ifndef __IA_CSS_COMMON_IO_TYPES
-#define __IA_CSS_COMMON_IO_TYPES
-
-#define MAX_IO_DMA_CHANNELS 3
-
-struct ia_css_common_io_config {
-	unsigned base_address;
-	unsigned width;
-	unsigned height;
-	unsigned stride;
-	unsigned ddr_elems_per_word;
-	unsigned dma_channel[MAX_IO_DMA_CHANNELS];
-};
-
-#endif /* __IA_CSS_COMMON_IO_TYPES */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c
deleted file mode 100644
index eb9e9439cc21..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c
+++ /dev/null
@@ -1,86 +0,0 @@
-#ifdef ISP2401
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#include "ia_css_yuv444_io.host.h"
-#include "dma.h"
-#include "math_support.h"
-#ifndef IA_CSS_NO_DEBUG
-#include "ia_css_debug.h"
-#endif
-#include "ia_css_isp_params.h"
-#include "ia_css_frame.h"
-
-void
-ia_css_yuv444_io_config(
-	const struct ia_css_binary      *binary,
-	const struct sh_css_binary_args *args)
-{
-	const struct ia_css_frame *in_frame = args->in_frame;
-	const struct ia_css_frame **out_frames = (const struct ia_css_frame **)& args->out_frame;
-	const struct ia_css_frame_info *in_frame_info = (in_frame) ? &in_frame->info : &binary->in_frame_info;
-
-	const unsigned ddr_bits_per_element = sizeof(short) * 8;
-	const unsigned ddr_elems_per_word = ceil_div(HIVE_ISP_DDR_WORD_BITS, ddr_bits_per_element);
-	unsigned size_get = 0, size_put = 0;
-	unsigned offset = 0;
-
-	if (binary->info->mem_offsets.offsets.param) {
-		size_get = binary->info->mem_offsets.offsets.param->dmem.get.size;
-		offset = binary->info->mem_offsets.offsets.param->dmem.get.offset;
-	}
-
-	if (size_get) {
-		struct ia_css_common_io_config *to = (struct ia_css_common_io_config *)&binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-		struct dma_port_config config;
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_yuv444_io_config() get part enter:\n");
-#endif
-
-		ia_css_dma_configure_from_info(&config, in_frame_info);
-		// The base_address of the input frame will be set in the ISP
-		to->width = in_frame_info->res.width;
-		to->height = in_frame_info->res.height;
-		to->stride = config.stride;
-		to->ddr_elems_per_word = ddr_elems_per_word;
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_yuv444_io_config() get part leave:\n");
-#endif
-	}
-
-	if (binary->info->mem_offsets.offsets.param) {
-		size_put = binary->info->mem_offsets.offsets.param->dmem.put.size;
-		offset = binary->info->mem_offsets.offsets.param->dmem.put.offset;
-	}
-
-	if (size_put) {
-		struct ia_css_common_io_config *to = (struct ia_css_common_io_config *)&binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset];
-		struct dma_port_config config;
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_yuv444_io_config() put part enter:\n");
-#endif
-
-		ia_css_dma_configure_from_info(&config, &out_frames[0]->info);
-		to->base_address = out_frames[0]->data;
-		to->width = out_frames[0]->info.res.width;
-		to->height = out_frames[0]->info.res.height;
-		to->stride = config.stride;
-		to->ddr_elems_per_word = ddr_elems_per_word;
-
-#ifndef IA_CSS_NO_DEBUG
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_yuv444_io_config() put part leave:\n");
-#endif
-	}
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h
deleted file mode 100644
index 480172d39aee..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h
+++ /dev/null
@@ -1,31 +0,0 @@
-#ifdef ISP2401
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#ifndef __YUV444_IO_HOST_H
-#define __YUV444_IO_HOST_H
-
-#include "ia_css_yuv444_io_param.h"
-#include "ia_css_yuv444_io_types.h"
-#include "ia_css_binary.h"
-#include "sh_css_internal.h"
-
-
-void
-ia_css_yuv444_io_config(
-	const struct ia_css_binary     *binary,
-	const struct sh_css_binary_args *args);
-
-#endif /*__YUV44_IO_HOST_H */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
deleted file mode 100644
index cc8eda19c6e8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifdef ISP2401
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#ifndef __IA_CSS_YUV444_IO_PARAM
-#define __IA_CSS_YUV444_IO_PARAM
-
-#include "../common/ia_css_common_io_param.h"
-
-#endif
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
deleted file mode 100644
index 343325a111e1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifdef ISP2401
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#ifndef __IA_CSS_YUV444_IO_TYPES
-#define __IA_CSS_YUV444_IO_TYPES
-
-#include "../common/ia_css_common_io_types.h"
-
-#endif
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c
deleted file mode 100644
index 9e41cc0a307f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_iterator.host.h"
-#include "ia_css_frame_public.h"
-#include "ia_css_binary.h"
-#include "ia_css_err.h"
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-
-static const struct ia_css_iterator_configuration default_config = {
-	.input_info = (struct ia_css_frame_info *)NULL,
-};
-
-void
-ia_css_iterator_config(
-	struct sh_css_isp_iterator_isp_config *to,
-	const struct ia_css_iterator_configuration *from,
-	unsigned size)
-{
-	(void)size;
-	ia_css_frame_info_to_frame_sp_info(&to->input_info,    from->input_info);
-	ia_css_frame_info_to_frame_sp_info(&to->internal_info, from->internal_info);
-	ia_css_frame_info_to_frame_sp_info(&to->output_info,   from->output_info);
-	ia_css_frame_info_to_frame_sp_info(&to->vf_info,       from->vf_info);
-	ia_css_resolution_to_sp_resolution(&to->dvs_envelope,  from->dvs_envelope);
-}
-
-enum ia_css_err
-ia_css_iterator_configure(
-	const struct ia_css_binary *binary,
-	const struct ia_css_frame_info *in_info)
-{
-	struct ia_css_frame_info my_info = IA_CSS_BINARY_DEFAULT_FRAME_INFO;
-	struct ia_css_iterator_configuration config = default_config;
-
-	config.input_info    = &binary->in_frame_info;
-	config.internal_info = &binary->internal_frame_info;
-	config.output_info   = &binary->out_frame_info[0];
-	config.vf_info       = &binary->vf_frame_info;
-	config.dvs_envelope  = &binary->dvs_envelope;
-
-	/* Use in_info iso binary->in_frame_info.
-	 * They can differ in padded width in case of scaling, e.g. for capture_pp.
-	 * Find out why.
-	*/
-	if (in_info)
-		config.input_info = in_info;
-	if (binary->out_frame_info[0].res.width == 0)
-		config.output_info = &binary->out_frame_info[1];
-	my_info = *config.output_info;
-	config.output_info = &my_info;
-	/* we do this only for preview pipe because in fill_binary_info function
-	 * we assign vf_out res to out res, but for ISP internal processing, we need
-	 * the original out res. for video pipe, it has two output pins --- out and
-	 * vf_out, so it can keep these two resolutions already. */
-	if (binary->info->sp.pipeline.mode == IA_CSS_BINARY_MODE_PREVIEW &&
-	    binary->vf_downscale_log2 > 0) {
-		/* TODO: Remove this after preview output decimation is fixed
-		 * by configuring out&vf info files properly */
-		my_info.padded_width <<= binary->vf_downscale_log2;
-		my_info.res.width    <<= binary->vf_downscale_log2;
-		my_info.res.height   <<= binary->vf_downscale_log2;
-	}
-
-	ia_css_configure_iterator(binary, &config);
-
-	return IA_CSS_SUCCESS;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h
deleted file mode 100644
index d8f249c5a53b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ITERATOR_HOST_H
-#define __IA_CSS_ITERATOR_HOST_H
-
-#include "ia_css_frame_public.h"
-#include "ia_css_binary.h"
-#include "ia_css_err.h"
-#include "ia_css_iterator_param.h"
-
-void
-ia_css_iterator_config(
-	struct sh_css_isp_iterator_isp_config *to,
-	const struct ia_css_iterator_configuration *from,
-	unsigned size);
-
-enum ia_css_err
-ia_css_iterator_configure(
-	const struct ia_css_binary *binary,
-	const struct ia_css_frame_info *in_info);
-
-#endif /* __IA_CSS_ITERATOR_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h
deleted file mode 100644
index d308126e41d3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_ITERATOR_PARAM_H
-#define __IA_CSS_ITERATOR_PARAM_H
-
-#include "ia_css_types.h" /* ia_css_resolution */
-#include "ia_css_frame_public.h" /* ia_css_frame_info */
-#include "ia_css_frame_comm.h" /* ia_css_frame_sp_info */
-
-struct ia_css_iterator_configuration {
-	const struct ia_css_frame_info *input_info;
-	const struct ia_css_frame_info *internal_info;
-	const struct ia_css_frame_info *output_info;
-	const struct ia_css_frame_info *vf_info;
-	const struct ia_css_resolution *dvs_envelope;
-};
-
-struct sh_css_isp_iterator_isp_config {
-	struct ia_css_frame_sp_info input_info;
-	struct ia_css_frame_sp_info internal_info;
-	struct ia_css_frame_sp_info output_info;
-	struct ia_css_frame_sp_info vf_info;
-	struct ia_css_sp_resolution dvs_envelope;
-};
-
-#endif /* __IA_CSS_ITERATOR_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c
deleted file mode 100644
index 5ddf61fc95fa..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-
-#ifndef IA_CSS_NO_DEBUG
-/* FIXME: See BZ 4427 */
-#include "ia_css_debug.h"
-#endif
-
-#include "ia_css_macc1_5.host.h"
-
-const struct ia_css_macc1_5_config default_macc1_5_config = {
-	1
-};
-
-void
-ia_css_macc1_5_encode(
-	struct sh_css_isp_macc1_5_params *to,
-	const struct ia_css_macc1_5_config *from,
-	unsigned int size)
-{
-	(void)size;
-	to->exp = from->exp;
-}
-
-void
-ia_css_macc1_5_vmem_encode(
-	struct sh_css_isp_macc1_5_vmem_params *params,
-	const struct ia_css_macc1_5_table *from,
-	unsigned int size)
-{
-	unsigned int i, j, k, idx;
-	unsigned int idx_map[] = {
-		0, 1, 3, 2, 6, 7, 5, 4, 12, 13, 15, 14, 10, 11, 9, 8};
-
-	(void)size;
-
-	for (k = 0; k < 4; k++)
-		for (i = 0; i < IA_CSS_MACC_NUM_AXES; i++) {
-			idx = idx_map[i] + (k * IA_CSS_MACC_NUM_AXES);
-			j   = 4 * i;
-
-			params->data[0][(idx)] = from->data[j];
-			params->data[1][(idx)] = from->data[j + 1];
-			params->data[2][(idx)] = from->data[j + 2];
-			params->data[3][(idx)] = from->data[j + 3];
-		}
-
-}
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_macc1_5_debug_dtrace(
-	const struct ia_css_macc1_5_config *config,
-	unsigned int level)
-{
-	ia_css_debug_dtrace(level,
-		"config.exp=%d\n",
-		config->exp);
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h
deleted file mode 100644
index 53ef18f7e912..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MACC1_5_HOST_H
-#define __IA_CSS_MACC1_5_HOST_H
-
-#include "ia_css_macc1_5_param.h"
-#include "ia_css_macc1_5_table.host.h"
-
-extern const struct ia_css_macc1_5_config default_macc1_5_config;
-
-void
-ia_css_macc1_5_encode(
-	struct sh_css_isp_macc1_5_params *to,
-	const struct ia_css_macc1_5_config *from,
-	unsigned int size);
-
-void
-ia_css_macc1_5_vmem_encode(
-	struct sh_css_isp_macc1_5_vmem_params *params,
-	const struct ia_css_macc1_5_table *from,
-	unsigned int size);
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_macc1_5_debug_dtrace(
-	const struct ia_css_macc1_5_config *config,
-	unsigned int level);
-#endif
-#endif /* __IA_CSS_MACC1_5_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h
deleted file mode 100644
index 41a2da460dcf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MACC1_5_PARAM_H
-#define __IA_CSS_MACC1_5_PARAM_H
-
-#include "type_support.h"
-#include "vmem.h"
-#include "ia_css_macc1_5_types.h"
-
-/* MACC */
-struct sh_css_isp_macc1_5_params {
-	int32_t exp;
-};
-
-struct sh_css_isp_macc1_5_vmem_params {
-	VMEM_ARRAY(data, IA_CSS_MACC_NUM_COEFS*ISP_NWAY);
-};
-
-#endif /* __IA_CSS_MACC1_5_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c
deleted file mode 100644
index 89714bf87b52..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "system_global.h"
-#include "ia_css_types.h"
-#include "ia_css_macc1_5_table.host.h"
-
-/* Multi-Axes Color Correction table for ISP2.
- *	64values = 2x2matrix for 16area, [s1.12]
- *	ineffective: 16 of "identity 2x2 matix" {4096,0,0,4096}
- */
-const struct ia_css_macc1_5_table default_macc1_5_table = {
-	      { 4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096 }
-};
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h
deleted file mode 100644
index 10a50aa82be8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MACC1_5_TABLE_HOST_H
-#define __IA_CSS_MACC1_5_TABLE_HOST_H
-
-#include "macc/macc1_5/ia_css_macc1_5_types.h"
-
-extern const struct ia_css_macc1_5_table default_macc1_5_table;
-
-#endif /* __IA_CSS_MACC1_5_TABLE_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h
deleted file mode 100644
index 9cd31c2c0253..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MACC1_5_TYPES_H
-#define __IA_CSS_MACC1_5_TYPES_H
-
-/* @file
-* CSS-API header file for Multi-Axis Color Conversion algorithm parameters.
-*/
-
-/* Multi-Axis Color Conversion configuration
- *
- * ISP2.6.1: MACC1_5 is used.
- */
-
-
-/* Number of axes in the MACC table. */
-#define IA_CSS_MACC_NUM_AXES           16
-/* Number of coefficients per MACC axes. */
-#define IA_CSS_MACC_NUM_COEFS          4
-
-/* Multi-Axes Color Correction (MACC) table.
- *
- *  ISP block: MACC (MACC by only matrix)
- *             MACC1_5 (MACC by matrix and exponent(ia_css_macc_config))
- *  ISP1: MACC is used.
- *  ISP2: MACC1_5 is used.
- *
- *  [MACC]
- *   OutU = (data00 * InU + data01 * InV) >> 13
- *   OutV = (data10 * InU + data11 * InV) >> 13
- *
- *   default/ineffective:
- *   OutU = (8192 * InU +    0 * InV) >> 13
- *   OutV = (   0 * InU + 8192 * InV) >> 13
- *
- *  [MACC1_5]
- *   OutU = (data00 * InU + data01 * InV) >> (13 - exp)
- *   OutV = (data10 * InU + data11 * InV) >> (13 - exp)
- *
- *   default/ineffective: (exp=1)
- *   OutU = (4096 * InU +    0 * InV) >> (13 - 1)
- *   OutV = (   0 * InU + 4096 * InV) >> (13 - 1)
- */
-struct ia_css_macc1_5_table {
-	int16_t data[IA_CSS_MACC_NUM_COEFS * IA_CSS_MACC_NUM_AXES];
-	/** 16 of 2x2 matix
-	  MACC1_5: s[macc_config.exp].[13-macc_config.exp], [-8192,8191]
-	    default/ineffective: (s1.12)
-		16 of "identity 2x2 matix" {4096,0,0,4096} */
-};
-
-/* Multi-Axes Color Correction (MACC) configuration.
- *
- *  ISP block: MACC1_5 (MACC by matrix and exponent(ia_css_macc_config))
- *  ISP2: MACC1_5 is used.
- */
-struct ia_css_macc1_5_config {
-	uint8_t exp;	/** Common exponent of ia_css_macc_table.
-				u8.0, [0,13], default 1, ineffective 1 */
-};
-
-#endif /* __IA_CSS_MACC1_5_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc.host.c
deleted file mode 100644
index 1f7e9e4eec3c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc.host.c
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "sh_css_frac.h"
-
-#include "ia_css_macc.host.h"
-
-const struct ia_css_macc_config default_macc_config = {
-	1,
-};
-
-void
-ia_css_macc_encode(
-	struct sh_css_isp_macc_params *to,
-	const struct ia_css_macc_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->exp = from->exp;
-}
-
-void
-ia_css_macc_dump(
-	const struct sh_css_isp_macc_params *macc,
-	unsigned level);
-
-void
-ia_css_macc_debug_dtrace(
-	const struct ia_css_macc_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.exp=%d\n",
-		config->exp);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc.host.h
deleted file mode 100644
index 044b01d38ad6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc.host.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MACC_HOST_H
-#define __IA_CSS_MACC_HOST_H
-
-#include "sh_css_params.h"
-
-#include "ia_css_macc_param.h"
-#include "ia_css_macc_table.host.h"
-
-extern const struct ia_css_macc_config default_macc_config;
-
-void
-ia_css_macc_encode(
-	struct sh_css_isp_macc_params *to,
-	const struct ia_css_macc_config *from,
-	unsigned size);
-	
-
-void
-ia_css_macc_dump(
-	const struct sh_css_isp_macc_params *macc,
-	unsigned level);
-
-void
-ia_css_macc_debug_dtrace(
-	const struct ia_css_macc_config *config,
-	unsigned level);
-
-#endif /* __IA_CSS_MACC_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_param.h
deleted file mode 100644
index 6a12b922c485..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_param.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MACC_PARAM_H
-#define __IA_CSS_MACC_PARAM_H
-
-#include "type_support.h"
-
-/* MACC */
-struct sh_css_isp_macc_params {
-	int32_t exp;
-};
-
-#endif /* __IA_CSS_MACC_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c
deleted file mode 100644
index 8a6c3cafabdc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "system_global.h"
-#include "ia_css_types.h"
-#include "ia_css_macc_table.host.h"
-
-/* Multi-Axes Color Correction table for ISP1.
- * 	64values = 2x2matrix for 16area, [s2.13]
- * 	ineffective: 16 of "identity 2x2 matix" {8192,0,0,8192}
- */
-const struct ia_css_macc_table default_macc_table = {
-		{ 8192, 0, 0, 8192, 8192, 0, 0, 8192,
-		8192, 0, 0, 8192, 8192, 0, 0, 8192,
-		8192, 0, 0, 8192, 8192, 0, 0, 8192,
-		8192, 0, 0, 8192, 8192, 0, 0, 8192,
-		8192, 0, 0, 8192, 8192, 0, 0, 8192,
-		8192, 0, 0, 8192, 8192, 0, 0, 8192,
-		8192, 0, 0, 8192, 8192, 0, 0, 8192,
-		8192, 0, 0, 8192, 8192, 0, 0, 8192 }
-};
-
-/* Multi-Axes Color Correction table for ISP2.
- * 	64values = 2x2matrix for 16area, [s1.12]
- * 	ineffective: 16 of "identity 2x2 matix" {4096,0,0,4096}
- */
-const struct ia_css_macc_table default_macc2_table = {
-	      { 4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096,
-		4096, 0, 0, 4096, 4096, 0, 0, 4096 }
-};
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h
deleted file mode 100644
index 96d62c9912b8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MACC_TABLE_HOST_H
-#define __IA_CSS_MACC_TABLE_HOST_H
-
-#include "ia_css_macc_types.h"
-
-extern const struct ia_css_macc_table default_macc_table;
-extern const struct ia_css_macc_table default_macc2_table;
-
-#endif /* __IA_CSS_MACC_TABLE_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_types.h
deleted file mode 100644
index 2c9e5a8ceb98..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/macc/macc_1.0/ia_css_macc_types.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_MACC_TYPES_H
-#define __IA_CSS_MACC_TYPES_H
-
-/* @file
-* CSS-API header file for Multi-Axis Color Correction (MACC) parameters.
-*/
-
-/* Number of axes in the MACC table. */
-#define IA_CSS_MACC_NUM_AXES           16
-/* Number of coefficients per MACC axes. */
-#define IA_CSS_MACC_NUM_COEFS          4
-/* The number of planes in the morphing table. */
-
-/* Multi-Axis Color Correction (MACC) table.
- *
- *  ISP block: MACC1 (MACC by only matrix)
- *             MACC2 (MACC by matrix and exponent(ia_css_macc_config))
- *  ISP1: MACC1 is used.
- *  ISP2: MACC2 is used.
- *
- *  [MACC1]
- *   OutU = (data00 * InU + data01 * InV) >> 13
- *   OutV = (data10 * InU + data11 * InV) >> 13
- *
- *   default/ineffective:
- *   OutU = (8192 * InU +    0 * InV) >> 13
- *   OutV = (   0 * InU + 8192 * InV) >> 13
- *
- *  [MACC2]
- *   OutU = (data00 * InU + data01 * InV) >> (13 - exp)
- *   OutV = (data10 * InU + data11 * InV) >> (13 - exp)
- *
- *   default/ineffective: (exp=1)
- *   OutU = (4096 * InU +    0 * InV) >> (13 - 1)
- *   OutV = (   0 * InU + 4096 * InV) >> (13 - 1)
- */
-
-struct ia_css_macc_table {
-	int16_t data[IA_CSS_MACC_NUM_COEFS * IA_CSS_MACC_NUM_AXES];
-	/** 16 of 2x2 matix
-	  MACC1: s2.13, [-65536,65535]
-	    default/ineffective:
-		16 of "identity 2x2 matix" {8192,0,0,8192}
-	  MACC2: s[macc_config.exp].[13-macc_config.exp], [-8192,8191]
-	    default/ineffective: (s1.12)
-		16 of "identity 2x2 matix" {4096,0,0,4096} */
-};
-
-#endif /* __IA_CSS_MACC_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm.host.c
deleted file mode 100644
index 2c2c5a5854a0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm.host.c
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_norm.host.h"
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm.host.h
deleted file mode 100644
index 42b5143ef78f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm.host.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_NORM_HOST_H
-#define __IA_CSS_NORM_HOST_H
-
-#include "ia_css_norm_param.h"
-
-#endif /* __IA_CSS_NORM_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm_param.h
deleted file mode 100644
index 85dc6fc0a56b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/norm/norm_1.0/ia_css_norm_param.h
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_NORM_PARAM_H
-#define __IA_CSS_NORM_PARAM_H
-
-
-#endif /* __IA_CSS_NORM_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2.host.c
deleted file mode 100644
index f77aff13f8e3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2.host.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "sh_css_frac.h"
-#ifndef IA_CSS_NO_DEBUG
-#include "ia_css_debug.h"
-#endif
-#include "isp.h"
-#include "ia_css_ob2.host.h"
-
-const struct ia_css_ob2_config default_ob2_config = {
-	0,
-	0,
-	0,
-	0
-};
-
-void
-ia_css_ob2_encode(
-	struct sh_css_isp_ob2_params *to,
-	const struct ia_css_ob2_config *from,
-	unsigned size)
-{
-	(void)size;
-
-	/* Blacklevels types are u0_16 */
-	to->blacklevel_gr = uDIGIT_FITTING(from->level_gr, 16, SH_CSS_BAYER_BITS);
-	to->blacklevel_r  = uDIGIT_FITTING(from->level_r,  16, SH_CSS_BAYER_BITS);
-	to->blacklevel_b  = uDIGIT_FITTING(from->level_b,  16, SH_CSS_BAYER_BITS);
-	to->blacklevel_gb = uDIGIT_FITTING(from->level_gb, 16, SH_CSS_BAYER_BITS);
-}
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_ob2_dump(
-	const struct sh_css_isp_ob2_params *ob2,
-	unsigned level)
-{
-	if (!ob2)
-		return;
-
-	ia_css_debug_dtrace(level, "Optical Black 2:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"ob2_blacklevel_gr", ob2->blacklevel_gr);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"ob2_blacklevel_r", ob2->blacklevel_r);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"ob2_blacklevel_b", ob2->blacklevel_b);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"ob2_blacklevel_gb", ob2->blacklevel_gb);
-
-}
-
-
-void
-ia_css_ob2_debug_dtrace(
-	const struct ia_css_ob2_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.level_gr=%d, config.level_r=%d, "
-		"config.level_b=%d,  config.level_gb=%d, ",
-		config->level_gr, config->level_r,
-		config->level_b, config->level_gb);
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2.host.h
deleted file mode 100644
index 06846502eca3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2.host.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_OB2_HOST_H
-#define __IA_CSS_OB2_HOST_H
-
-#include "ia_css_ob2_types.h"
-#include "ia_css_ob2_param.h"
-
-extern const struct ia_css_ob2_config default_ob2_config;
-
-void
-ia_css_ob2_encode(
-	struct sh_css_isp_ob2_params *to,
-	const struct ia_css_ob2_config *from,
-	unsigned size);
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_ob2_dump(
-	const struct sh_css_isp_ob2_params *ob2,
-	unsigned level);
-
-void
-ia_css_ob2_debug_dtrace(
-	const struct ia_css_ob2_config *config, unsigned level);
-#endif
-
-#endif /* __IA_CSS_OB2_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2_param.h
deleted file mode 100644
index 5c21d6a3911b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2_param.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_OB2_PARAM_H
-#define __IA_CSS_OB2_PARAM_H
-
-#include "type_support.h"
-
-
-/* OB2 (Optical Black) */
-struct sh_css_isp_ob2_params {
-	int32_t blacklevel_gr;
-	int32_t blacklevel_r;
-	int32_t blacklevel_b;
-	int32_t blacklevel_gb;
-};
-
-#endif /* __IA_CSS_OB2_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2_types.h
deleted file mode 100644
index d981394c1c11..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob2/ia_css_ob2_types.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_OB2_TYPES_H
-#define __IA_CSS_OB2_TYPES_H
-
-/* @file
-* CSS-API header file for Optical Black algorithm parameters.
-*/
-
-/* Optical Black configuration
- *
- * ISP2.6.1: OB2 is used.
- */
-
-#include "ia_css_frac.h"
-
-struct ia_css_ob2_config {
-	ia_css_u0_16 level_gr;    /** Black level for GR pixels.
-					u0.16, [0,65535],
-					default/ineffective 0 */
-	ia_css_u0_16  level_r;     /** Black level for R pixels.
-					u0.16, [0,65535],
-					default/ineffective 0 */
-	ia_css_u0_16  level_b;     /** Black level for B pixels.
-					u0.16, [0,65535],
-					default/ineffective 0 */
-	ia_css_u0_16  level_gb;    /** Black level for GB pixels.
-					u0.16, [0,65535],
-					default/ineffective 0 */
-};
-
-#endif /* __IA_CSS_OB2_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.c
deleted file mode 100644
index fd891ac092ed..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.c
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "isp.h"
-
-#include "ia_css_ob.host.h"
-
-const struct ia_css_ob_config default_ob_config = {
-	IA_CSS_OB_MODE_NONE,
-	0,
-	0,
-	0,
-	0,
-	0,
-	0
-};
-
-/* TODO: include ob.isp.h to get isp knowledge and
-   add assert on platform restrictions */
-
-void
-ia_css_ob_configure(
-	struct sh_css_isp_ob_stream_config *config,
-	unsigned int isp_pipe_version,
-	unsigned int raw_bit_depth)
-{
-	config->isp_pipe_version = isp_pipe_version;
-	config->raw_bit_depth    = raw_bit_depth;
-}
-
-void
-ia_css_ob_encode(
-	struct sh_css_isp_ob_params *to,
-	const struct ia_css_ob_config *from,
-	const struct sh_css_isp_ob_stream_config *config,
-	unsigned size)
-{
-	unsigned int ob_bit_depth
-		= config->isp_pipe_version == 2 ? SH_CSS_BAYER_BITS : config->raw_bit_depth;
-	unsigned int scale = 16 - ob_bit_depth;
-
-	(void)size;
-	switch (from->mode) {
-	case IA_CSS_OB_MODE_FIXED:
-		to->blacklevel_gr = from->level_gr >> scale;
-		to->blacklevel_r  = from->level_r  >> scale;
-		to->blacklevel_b  = from->level_b  >> scale;
-		to->blacklevel_gb = from->level_gb >> scale;
-		to->area_start_bq = 0;
-		to->area_length_bq = 0;
-		to->area_length_bq_inverse = 0;
-		break;
-	case IA_CSS_OB_MODE_RASTER:
-		to->blacklevel_gr = 0;
-		to->blacklevel_r = 0;
-		to->blacklevel_b = 0;
-		to->blacklevel_gb = 0;
-		to->area_start_bq = from->start_position;
-		to->area_length_bq =
-		    (from->end_position - from->start_position) + 1;
-		to->area_length_bq_inverse = AREA_LENGTH_UNIT / to->area_length_bq;
-		break;
-	default:
-		to->blacklevel_gr = 0;
-		to->blacklevel_r = 0;
-		to->blacklevel_b = 0;
-		to->blacklevel_gb = 0;
-		to->area_start_bq = 0;
-		to->area_length_bq = 0;
-		to->area_length_bq_inverse = 0;
-		break;
-	}
-}
-
-void
-ia_css_ob_vmem_encode(
-	struct sh_css_isp_ob_vmem_params *to,
-	const struct ia_css_ob_config *from,
-	const struct sh_css_isp_ob_stream_config *config,
-	unsigned size)
-{
-	struct sh_css_isp_ob_params tmp;
-	struct sh_css_isp_ob_params *ob = &tmp;
-
-	(void)size;
-	ia_css_ob_encode(&tmp, from, config, sizeof(tmp));
-
-	{
-		unsigned i;
-		unsigned sp_obarea_start_bq  = ob->area_start_bq;
-		unsigned sp_obarea_length_bq = ob->area_length_bq;
-		unsigned low = sp_obarea_start_bq;
-		unsigned high = low + sp_obarea_length_bq;
-		uint16_t all_ones = ~0;
-
-		for (i = 0; i < OBAREA_MASK_SIZE; i++) {
-			if (i >= low && i < high)
-				to->vmask[i/ISP_VEC_NELEMS][i%ISP_VEC_NELEMS] = all_ones;
-			else
-				to->vmask[i/ISP_VEC_NELEMS][i%ISP_VEC_NELEMS] = 0;
-		}
-	}
-}
-
-void
-ia_css_ob_dump(
-	const struct sh_css_isp_ob_params *ob,
-	unsigned level)
-{
-	if (!ob) return;
-	ia_css_debug_dtrace(level, "Optical Black:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"ob_blacklevel_gr", ob->blacklevel_gr);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"ob_blacklevel_r", ob->blacklevel_r);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"ob_blacklevel_b", ob->blacklevel_b);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"ob_blacklevel_gb", ob->blacklevel_gb);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"obarea_start_bq", ob->area_start_bq);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"obarea_length_bq", ob->area_length_bq);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-		"obarea_length_bq_inverse",
-		ob->area_length_bq_inverse);
-}
-
-
-void
-ia_css_ob_debug_dtrace(
-	const struct ia_css_ob_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.mode=%d, "
-		"config.level_gr=%d, config.level_r=%d, "
-		"config.level_b=%d,  config.level_gb=%d, "
-		"config.start_position=%d, config.end_position=%d\n",
-		config->mode,
-		config->level_gr, config->level_r,
-		config->level_b, config->level_gb,
-		config->start_position, config->end_position);
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.h
deleted file mode 100644
index 4af181470f8d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob.host.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_OB_HOST_H
-#define __IA_CSS_OB_HOST_H
-
-#include "ia_css_ob_types.h"
-#include "ia_css_ob_param.h"
-
-extern const struct ia_css_ob_config default_ob_config;
-
-void
-ia_css_ob_configure(
-	struct sh_css_isp_ob_stream_config *config,
-	unsigned int isp_pipe_version,
-	unsigned int raw_bit_depth);
-
-void
-ia_css_ob_encode(
-	struct sh_css_isp_ob_params *to,
-	const struct ia_css_ob_config *from,
-	const struct sh_css_isp_ob_stream_config *config,
-	unsigned size);
-
-void
-ia_css_ob_vmem_encode(
-	struct sh_css_isp_ob_vmem_params *to,
-	const struct ia_css_ob_config *from,
-	const struct sh_css_isp_ob_stream_config *config,
-	unsigned size);
-
-void
-ia_css_ob_dump(
-	const struct sh_css_isp_ob_params *ob,
-	unsigned level);
-
-void
-ia_css_ob_debug_dtrace(
-	const struct ia_css_ob_config *config, unsigned level)
-;
-
-#endif /* __IA_CSS_OB_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob_param.h
deleted file mode 100644
index a60a644bb4ff..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob_param.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_OB_PARAM_H
-#define __IA_CSS_OB_PARAM_H
-
-#include "type_support.h"
-#include "vmem.h"
-
-#define OBAREA_MASK_SIZE 64
-#define OBAREA_LENGTHBQ_INVERSE_SHIFT     12
-
-/* AREA_LENGTH_UNIT is dependent on NWAY, requires rewrite */
-#define AREA_LENGTH_UNIT (1<<12)
-
-
-/* OB (Optical Black) */
-struct sh_css_isp_ob_stream_config {
-	unsigned isp_pipe_version;
-	unsigned raw_bit_depth;
-};
-
-struct sh_css_isp_ob_params {
-	int32_t blacklevel_gr;
-	int32_t blacklevel_r;
-	int32_t blacklevel_b;
-	int32_t blacklevel_gb;
-	int32_t area_start_bq;
-	int32_t area_length_bq;
-	int32_t area_length_bq_inverse;
-};
-
-struct sh_css_isp_ob_vmem_params {
-	VMEM_ARRAY(vmask, OBAREA_MASK_SIZE);
-};
-
-#endif /* __IA_CSS_OB_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob_types.h
deleted file mode 100644
index a9717b8f44ac..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ob/ob_1.0/ia_css_ob_types.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_OB_TYPES_H
-#define __IA_CSS_OB_TYPES_H
-
-/* @file
-* CSS-API header file for Optical Black level parameters.
-*/
-
-#include "ia_css_frac.h"
-
-/* Optical black mode.
- */
-enum ia_css_ob_mode {
-	IA_CSS_OB_MODE_NONE,	/** OB has no effect. */
-	IA_CSS_OB_MODE_FIXED,	/** Fixed OB */
-	IA_CSS_OB_MODE_RASTER	/** Raster OB */
-};
-
-/* Optical Black level configuration.
- *
- *  ISP block: OB1
- *  ISP1: OB1 is used.
- *  ISP2: OB1 is used.
- */
-struct ia_css_ob_config {
-	enum ia_css_ob_mode mode; /** Mode (None / Fixed / Raster).
-					enum, [0,2],
-					default 1, ineffective 0 */
-	ia_css_u0_16 level_gr;    /** Black level for GR pixels
-					(used for Fixed Mode only).
-					u0.16, [0,65535],
-					default/ineffective 0 */
-	ia_css_u0_16 level_r;     /** Black level for R pixels
-					(used for Fixed Mode only).
-					u0.16, [0,65535],
-					default/ineffective 0 */
-	ia_css_u0_16 level_b;     /** Black level for B pixels
-					(used for Fixed Mode only).
-					u0.16, [0,65535],
-					default/ineffective 0 */
-	ia_css_u0_16 level_gb;    /** Black level for GB pixels
-					(used for Fixed Mode only).
-					u0.16, [0,65535],
-					default/ineffective 0 */
-	uint16_t start_position; /** Start position of OB area
-					(used for Raster Mode only).
-					u16.0, [0,63],
-					default/ineffective 0 */
-	uint16_t end_position;  /** End position of OB area
-					(used for Raster Mode only).
-					u16.0, [0,63],
-					default/ineffective 0 */
-};
-
-#endif /* __IA_CSS_OB_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output.host.c
deleted file mode 100644
index 9efe5e5e4e06..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output.host.c
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_frame.h"
-#include "ia_css_debug.h"
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-#include "ia_css_output.host.h"
-#include "isp.h"
-
-#include "assert_support.h"
-
-const struct ia_css_output_config default_output_config = {
-	0,
-	0
-};
-
-static const struct ia_css_output_configuration default_output_configuration = {
-	.info = (struct ia_css_frame_info *)NULL,
-};
-
-static const struct ia_css_output0_configuration default_output0_configuration = {
-	.info = (struct ia_css_frame_info *)NULL,
-};
-
-static const struct ia_css_output1_configuration default_output1_configuration = {
-	.info = (struct ia_css_frame_info *)NULL,
-};
-
-void
-ia_css_output_encode(
-	struct sh_css_isp_output_params *to,
-	const struct ia_css_output_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->enable_hflip = from->enable_hflip;
-	to->enable_vflip = from->enable_vflip;
-}
-
-void
-ia_css_output_config(
-	struct sh_css_isp_output_isp_config *to,
-	const struct ia_css_output_configuration  *from,
-	unsigned size)
-{
-	unsigned elems_a = ISP_VEC_NELEMS;
-
-	(void)size;
-	ia_css_dma_configure_from_info(&to->port_b, from->info);
-	to->width_a_over_b = elems_a / to->port_b.elems;
-	to->height = from->info ? from->info->res.height : 0;
-	to->enable = from->info != NULL;
-	ia_css_frame_info_to_frame_sp_info(&to->info, from->info);
-
-	/* Assume divisiblity here, may need to generalize to fixed point. */
-	assert (elems_a % to->port_b.elems == 0);
-}
-
-void
-ia_css_output0_config(
-	struct sh_css_isp_output_isp_config       *to,
-	const struct ia_css_output0_configuration *from,
-	unsigned size)
-{
-	ia_css_output_config (
-		to, (const struct ia_css_output_configuration *)from, size);
-}
-
-void
-ia_css_output1_config(
-	struct sh_css_isp_output_isp_config       *to,
-	const struct ia_css_output1_configuration *from,
-	unsigned size)
-{
-	ia_css_output_config (
-		to, (const struct ia_css_output_configuration *)from, size);
-}
-
-void
-ia_css_output_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *info)
-{
-	if (NULL != info) {
-		struct ia_css_output_configuration config =
-				default_output_configuration;
-
-		config.info = info;
-
-		ia_css_configure_output(binary, &config);
-	}
-}
-
-void
-ia_css_output0_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *info)
-{
-	if (NULL != info) {
-		struct ia_css_output0_configuration config =
-				default_output0_configuration;
-
-		config.info = info;
-
-		ia_css_configure_output0(binary, &config);
-	}
-}
-
-void
-ia_css_output1_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *info)
-{
-
-	if (NULL != info) {
-		struct ia_css_output1_configuration config =
-				default_output1_configuration;
-
-		config.info = info;
-
-		ia_css_configure_output1(binary, &config);
-	}
-}
-
-void
-ia_css_output_dump(
-	const struct sh_css_isp_output_params *output,
-	unsigned level)
-{
-	if (!output) return;
-	ia_css_debug_dtrace(level, "Horizontal Output Flip:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"enable", output->enable_hflip);
-	ia_css_debug_dtrace(level, "Vertical Output Flip:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"enable", output->enable_vflip);
-}
-
-void
-ia_css_output_debug_dtrace(
-	const struct ia_css_output_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.enable_hflip=%d",
-		config->enable_hflip);
-	ia_css_debug_dtrace(level,
-		"config.enable_vflip=%d",
-		config->enable_vflip);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output.host.h
deleted file mode 100644
index 530f934ce81e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output.host.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_OUTPUT_HOST_H
-#define __IA_CSS_OUTPUT_HOST_H
-
-#include "ia_css_frame_public.h"
-#include "ia_css_binary.h"
-
-#include "ia_css_output_types.h"
-#include "ia_css_output_param.h"
-
-extern const struct ia_css_output_config default_output_config;
-
-void
-ia_css_output_encode(
-	struct sh_css_isp_output_params *to,
-	const struct ia_css_output_config *from,
-	unsigned size);
-
-void
-ia_css_output_config(
-	struct sh_css_isp_output_isp_config      *to,
-	const struct ia_css_output_configuration *from,
-	unsigned size);
-
-void
-ia_css_output0_config(
-	struct sh_css_isp_output_isp_config       *to,
-	const struct ia_css_output0_configuration *from,
-	unsigned size);
-
-void
-ia_css_output1_config(
-	struct sh_css_isp_output_isp_config       *to,
-	const struct ia_css_output1_configuration *from,
-	unsigned size);
-
-void
-ia_css_output_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *from);
-
-void
-ia_css_output0_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *from);
-
-void
-ia_css_output1_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *from);
-
-void
-ia_css_output_dump(
-	const struct sh_css_isp_output_params *output,
-	unsigned level);
-
-void
-ia_css_output_debug_dtrace(
-	const struct ia_css_output_config *config,
-	unsigned level);
-
-#endif /* __IA_CSS_OUTPUT_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output_param.h
deleted file mode 100644
index eb7defa41145..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output_param.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_OUTPUT_PARAM_H
-#define __IA_CSS_OUTPUT_PARAM_H
-
-#include <type_support.h>
-#include "dma.h"
-#include "ia_css_frame_comm.h" /* ia_css_frame_sp_info */
-
-/* output frame */
-struct sh_css_isp_output_isp_config {
-	uint32_t width_a_over_b;
-	uint32_t height;
-	uint32_t enable;
-	struct ia_css_frame_sp_info info;
-	struct dma_port_config port_b;
-};
-
-struct sh_css_isp_output_params {
-	uint8_t enable_hflip;
-	uint8_t enable_vflip;
-};
-
-#endif /* __IA_CSS_OUTPUT_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output_types.h
deleted file mode 100644
index 9c7342fb8145..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/output/output_1.0/ia_css_output_types.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_OUTPUT_TYPES_H
-#define __IA_CSS_OUTPUT_TYPES_H
-
-/* @file
-* CSS-API header file for parameters of output frames.
-*/
-
-/* Output frame
- *
- *  ISP block: output frame
- */
-
-//#include "ia_css_frame_public.h"
-struct ia_css_frame_info;
-
-struct ia_css_output_configuration {
-	const struct ia_css_frame_info *info;
-};
-
-struct ia_css_output0_configuration {
-	const struct ia_css_frame_info *info;
-};
-
-struct ia_css_output1_configuration {
-	const struct ia_css_frame_info *info;
-};
-
-struct ia_css_output_config {
-	uint8_t enable_hflip;  /** enable horizontal output mirroring */
-	uint8_t enable_vflip;  /** enable vertical output mirroring */
-};
-
-#endif /* __IA_CSS_OUTPUT_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c
deleted file mode 100644
index d1fb4b116003..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_frame.h"
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-#include "isp.h"
-
-#include "ia_css_qplane.host.h"
-
-static const struct ia_css_qplane_configuration default_config = {
-	.pipe = (struct sh_css_sp_pipeline *)NULL,
-};
-
-void
-ia_css_qplane_config(
-	struct sh_css_isp_qplane_isp_config *to,
-	const struct ia_css_qplane_configuration  *from,
-	unsigned size)
-{
-	unsigned elems_a = ISP_VEC_NELEMS;
-
-	(void)size;
-	ia_css_dma_configure_from_info(&to->port_b, from->info);
-	to->width_a_over_b = elems_a / to->port_b.elems;
-
-	/* Assume divisiblity here, may need to generalize to fixed point. */
-	assert (elems_a % to->port_b.elems == 0);
-
-	to->inout_port_config = from->pipe->inout_port_config;
-	to->format = from->info->format;
-}
-
-void
-ia_css_qplane_configure(
-	const struct sh_css_sp_pipeline *pipe,
-	const struct ia_css_binary      *binary,
-	const struct ia_css_frame_info  *info)
-{
-	struct ia_css_qplane_configuration config = default_config;
-
-	config.pipe = pipe;
-	config.info = info;
-
-	ia_css_configure_qplane(binary, &config);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h
deleted file mode 100644
index c41e9e5e0fd7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_QPLANE_HOST_H
-#define __IA_CSS_QPLANE_HOST_H
-
-#include <ia_css_frame_public.h>
-#include <ia_css_binary.h>
-
-#if 0
-/* Cannot be included, since sh_css_internal.h is too generic
- * e.g. for FW generation.
-*/
-#include "sh_css_internal.h"	/* sh_css_sp_pipeline */
-#endif
-
-#include "ia_css_qplane_types.h"
-#include "ia_css_qplane_param.h"
-
-void
-ia_css_qplane_config(
-	struct sh_css_isp_qplane_isp_config      *to,
-	const struct ia_css_qplane_configuration *from,
-	unsigned size);
-
-void
-ia_css_qplane_configure(
-	const struct sh_css_sp_pipeline *pipe,
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *from);
-
-#endif /* __IA_CSS_QPLANE_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h
deleted file mode 100644
index 5885f621de88..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_QPLANE_PARAM_H
-#define __IA_CSS_QPLANE_PARAM_H
-
-#include <type_support.h>
-#include "dma.h"
-
-/* qplane channel */
-struct sh_css_isp_qplane_isp_config {
-	uint32_t width_a_over_b;
-	struct dma_port_config port_b;
-	uint32_t inout_port_config;
-	uint32_t input_needs_raw_binning;
-	uint32_t format; /* enum ia_css_frame_format */
-};
-
-#endif /* __IA_CSS_QPLANE_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h
deleted file mode 100644
index 62d371841619..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_QPLANE_TYPES_H
-#define __IA_CSS_QPLANE_TYPES_H
-
-#include <ia_css_frame_public.h>
-#include "sh_css_internal.h"
-
-/* qplane frame
- *
- *  ISP block: qplane frame
- */
-
-
-struct ia_css_qplane_configuration {
-	const struct sh_css_sp_pipeline *pipe;
-	const struct ia_css_frame_info  *info;
-};
-
-#endif /* __IA_CSS_QPLANE_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw.host.c
deleted file mode 100644
index fa9ce0fedf23..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw.host.c
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_frame.h"
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-#include "isp.h"
-#include "isp/modes/interface/isp_types.h"
-
-#include "ia_css_raw.host.h"
-
-
-static const struct ia_css_raw_configuration default_config = {
-	.pipe = (struct sh_css_sp_pipeline *)NULL,
-};
-
-static inline unsigned
-sh_css_elems_bytes_from_info (unsigned raw_bit_depth)
-{
-	return CEIL_DIV(raw_bit_depth,8);
-}
-
-/* MW: These areMIPI / ISYS properties, not camera function properties */
-static enum sh_stream_format
-css2isp_stream_format(enum atomisp_input_format from)
-{
-	switch (from) {
-	case ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY:
-		return sh_stream_format_yuv420_legacy;
-	case ATOMISP_INPUT_FORMAT_YUV420_8:
-	case ATOMISP_INPUT_FORMAT_YUV420_10:
-	case ATOMISP_INPUT_FORMAT_YUV420_16:
-		return sh_stream_format_yuv420;
-	case ATOMISP_INPUT_FORMAT_YUV422_8:
-	case ATOMISP_INPUT_FORMAT_YUV422_10:
-	case ATOMISP_INPUT_FORMAT_YUV422_16:
-		return sh_stream_format_yuv422;
-	case ATOMISP_INPUT_FORMAT_RGB_444:
-	case ATOMISP_INPUT_FORMAT_RGB_555:
-	case ATOMISP_INPUT_FORMAT_RGB_565:
-	case ATOMISP_INPUT_FORMAT_RGB_666:
-	case ATOMISP_INPUT_FORMAT_RGB_888:
-		return sh_stream_format_rgb;
-	case ATOMISP_INPUT_FORMAT_RAW_6:
-	case ATOMISP_INPUT_FORMAT_RAW_7:
-	case ATOMISP_INPUT_FORMAT_RAW_8:
-	case ATOMISP_INPUT_FORMAT_RAW_10:
-	case ATOMISP_INPUT_FORMAT_RAW_12:
-	case ATOMISP_INPUT_FORMAT_RAW_14:
-	case ATOMISP_INPUT_FORMAT_RAW_16:
-		return sh_stream_format_raw;
-	case ATOMISP_INPUT_FORMAT_BINARY_8:
-	default:
-		return sh_stream_format_raw;
-	}
-}
-
-void
-ia_css_raw_config(
-	struct sh_css_isp_raw_isp_config *to,
-	const struct ia_css_raw_configuration  *from,
-	unsigned size)
-{
-	unsigned elems_a = ISP_VEC_NELEMS;
-	const struct ia_css_frame_info *in_info = from->in_info;
-	const struct ia_css_frame_info *internal_info = from->internal_info;
-
-	(void)size;
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
-	/* 2401 input system uses input width width */
-	in_info = internal_info;
-#else
-	/*in some cases, in_info is NULL*/
-	if (in_info)
-		(void)internal_info;
-	else
-		in_info = internal_info;
-
-#endif
-	ia_css_dma_configure_from_info(&to->port_b, in_info);
-
-	/* Assume divisiblity here, may need to generalize to fixed point. */
-	assert((in_info->format == IA_CSS_FRAME_FORMAT_RAW_PACKED) ||
-		   (elems_a % to->port_b.elems == 0));
-
-	to->width_a_over_b      = elems_a / to->port_b.elems;
-	to->inout_port_config   = from->pipe->inout_port_config;
-	to->format              = in_info->format;
-	to->required_bds_factor = from->pipe->required_bds_factor;
-	to->two_ppc             = from->two_ppc;
-	to->stream_format       = css2isp_stream_format(from->stream_format);
-	to->deinterleaved       = from->deinterleaved;
-#if (defined(USE_INPUT_SYSTEM_VERSION_2401) || defined(CONFIG_CSI2_PLUS))
-	to->start_column        = in_info->crop_info.start_column;
-	to->start_line          = in_info->crop_info.start_line;
-	to->enable_left_padding = from->enable_left_padding;
-#endif
-}
-
-void
-ia_css_raw_configure(
-	const struct sh_css_sp_pipeline *pipe,
-	const struct ia_css_binary      *binary,
-	const struct ia_css_frame_info  *in_info,
-	const struct ia_css_frame_info  *internal_info,
-	bool two_ppc,
-	bool deinterleaved)
-{
-	uint8_t enable_left_padding = (uint8_t)((binary->left_padding) ? 1 : 0);
-	struct ia_css_raw_configuration config = default_config;
-
-	config.pipe                = pipe;
-	config.in_info             = in_info;
-	config.internal_info       = internal_info;
-	config.two_ppc             = two_ppc;
-	config.stream_format       = binary->input_format;
-	config.deinterleaved       = deinterleaved;
-	config.enable_left_padding = enable_left_padding;
-
-	ia_css_configure_raw(binary, &config);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw.host.h
deleted file mode 100644
index ac6b7f6b59c6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw.host.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_RAW_HOST_H
-#define __IA_CSS_RAW_HOST_H
-
-#include "ia_css_binary.h"
-
-#include "ia_css_raw_types.h"
-#include "ia_css_raw_param.h"
-
-void
-ia_css_raw_config(
-	struct sh_css_isp_raw_isp_config      *to,
-	const struct ia_css_raw_configuration *from,
-	unsigned size);
-
-void
-ia_css_raw_configure(
-	const struct sh_css_sp_pipeline *pipe,
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame_info *in_info,
-	const struct ia_css_frame_info *internal_info,
-	bool two_ppc,
-	bool deinterleaved);
-
-#endif /* __IA_CSS_RAW_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw_param.h
deleted file mode 100644
index 12168b2dec2d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw_param.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_RAW_PARAM_H
-#define __IA_CSS_RAW_PARAM_H
-
-#include "type_support.h"
-
-#include "dma.h"
-
-/* Raw channel */
-struct sh_css_isp_raw_isp_config {
-	uint32_t width_a_over_b;
-	struct dma_port_config port_b;
-	uint32_t inout_port_config;
-	uint32_t input_needs_raw_binning;
-	uint32_t format; /* enum ia_css_frame_format */
-	uint32_t required_bds_factor;
-	uint32_t two_ppc;
-	uint32_t stream_format; /* enum sh_stream_format */
-	uint32_t deinterleaved;
-	uint32_t start_column; /*left crop offset*/
-	uint32_t start_line; /*top crop offset*/
-	uint8_t enable_left_padding; /*need this for multiple binary case*/
-};
-
-#endif /* __IA_CSS_RAW_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw_types.h
deleted file mode 100644
index ae868eb5e10f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw/raw_1.0/ia_css_raw_types.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_RAW_TYPES_H
-#define __IA_CSS_RAW_TYPES_H
-
-#include <ia_css_frame_public.h>
-#include "sh_css_internal.h"
-
-/* Raw frame
- *
- *  ISP block: Raw frame
- */
-
-struct ia_css_raw_configuration {
-	const struct sh_css_sp_pipeline *pipe;
-	const struct ia_css_frame_info  *in_info;
-	const struct ia_css_frame_info  *internal_info;
-	bool two_ppc;
-	enum atomisp_input_format stream_format;
-	bool deinterleaved;
-	uint8_t enable_left_padding;
-};
-
-#endif /* __IA_CSS_RAW_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c
deleted file mode 100644
index 92168211683d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#if !defined(HAS_NO_HMEM)
-
-#include "memory_access.h"
-#include "ia_css_types.h"
-#include "sh_css_internal.h"
-#include "sh_css_frac.h"
-
-#include "ia_css_raa.host.h"
-
-void
-ia_css_raa_encode(
-	struct sh_css_isp_aa_params *to,
-	const struct ia_css_aa_config *from,
-	unsigned size)
-{
-	(void)size;
-	(void)to;
-	(void)from;
-}
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h
deleted file mode 100644
index b4f245c19f18..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_RAA_HOST_H
-#define __IA_CSS_RAA_HOST_H
-
-#include "aa/aa_2/ia_css_aa2_types.h"
-#include "aa/aa_2/ia_css_aa2_param.h"
-
-void
-ia_css_raa_encode(
-	struct sh_css_isp_aa_params *to,
-	const struct ia_css_aa_config *from,
-	unsigned size);
-
-#endif /* __IA_CSS_RAA_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref.host.c
deleted file mode 100644
index 4c0ed5d4d971..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref.host.c
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <assert_support.h>
-#include <ia_css_frame_public.h>
-#include <ia_css_frame.h>
-#include <ia_css_binary.h>
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-#include "isp.h"
-#include "ia_css_ref.host.h"
-
-void
-ia_css_ref_config(
-	struct sh_css_isp_ref_isp_config *to,
-	const struct ia_css_ref_configuration  *from,
-	unsigned size)
-{
-	unsigned elems_a = ISP_VEC_NELEMS, i;
-
-	(void)size;
-	ia_css_dma_configure_from_info(&to->port_b, &(from->ref_frames[0]->info));
-	to->width_a_over_b = elems_a / to->port_b.elems;
-	to->dvs_frame_delay = from->dvs_frame_delay;
-	for (i = 0; i < MAX_NUM_VIDEO_DELAY_FRAMES; i++) {
-		if (from->ref_frames[i]) {
-			to->ref_frame_addr_y[i] = from->ref_frames[i]->data + from->ref_frames[i]->planes.yuv.y.offset;
-			to->ref_frame_addr_c[i] = from->ref_frames[i]->data + from->ref_frames[i]->planes.yuv.u.offset;
-		} else {
-			to->ref_frame_addr_y[i] = 0;
-			to->ref_frame_addr_c[i] = 0;
-		}
-	}
-
-	/* Assume divisiblity here, may need to generalize to fixed point. */
-	assert (elems_a % to->port_b.elems == 0);
-}
-
-void
-ia_css_ref_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame **ref_frames,
-	const uint32_t dvs_frame_delay)
-{
-	struct ia_css_ref_configuration config;
-	unsigned i;
-
-	for (i = 0; i < MAX_NUM_VIDEO_DELAY_FRAMES; i++)
-		config.ref_frames[i] = ref_frames[i];
-	config.dvs_frame_delay = dvs_frame_delay;
-	ia_css_configure_ref(binary, &config);
-}
-
-void
-ia_css_init_ref_state(
-	struct sh_css_isp_ref_dmem_state *state,
-	unsigned size)
-{
-	(void)size;
-	assert(MAX_NUM_VIDEO_DELAY_FRAMES >= 2);
-	state->ref_in_buf_idx = 0;
-	state->ref_out_buf_idx = 1;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref.host.h
deleted file mode 100644
index 3c6d728d49ec..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref.host.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_REF_HOST_H
-#define __IA_CSS_REF_HOST_H
-
-#include <ia_css_frame_public.h>
-#include <ia_css_binary.h>
-
-#include "ia_css_ref_types.h"
-#include "ia_css_ref_param.h"
-#include "ia_css_ref_state.h"
-
-void
-ia_css_ref_config(
-	struct sh_css_isp_ref_isp_config      *to,
-	const struct ia_css_ref_configuration *from,
-	unsigned size);
-
-void
-ia_css_ref_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame **ref_frames,
-	const uint32_t dvs_frame_delay);
-
-void
-ia_css_init_ref_state(
-	struct sh_css_isp_ref_dmem_state *state,
-	unsigned size);
-#endif /* __IA_CSS_REF_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_param.h
deleted file mode 100644
index 026443b999a6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_param.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_REF_PARAM_H
-#define __IA_CSS_REF_PARAM_H
-
-#include <type_support.h>
-#include "sh_css_defs.h"
-#include "dma.h"
-
-/* Reference frame */
-struct ia_css_ref_configuration {
-	const struct ia_css_frame *ref_frames[MAX_NUM_VIDEO_DELAY_FRAMES];
-	uint32_t dvs_frame_delay;
-};
-
-struct sh_css_isp_ref_isp_config {
-	uint32_t width_a_over_b;
-	struct dma_port_config port_b;
-	hrt_vaddress ref_frame_addr_y[MAX_NUM_VIDEO_DELAY_FRAMES];
-	hrt_vaddress ref_frame_addr_c[MAX_NUM_VIDEO_DELAY_FRAMES];
-	uint32_t dvs_frame_delay;
-};
-
-#endif /* __IA_CSS_REF_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_state.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_state.h
deleted file mode 100644
index 7867be8a7958..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_state.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_REF_STATE_H
-#define __IA_CSS_REF_STATE_H
-
-#include "type_support.h"
-
-/* REF (temporal noise reduction) */
-struct sh_css_isp_ref_dmem_state {
-	int32_t ref_in_buf_idx;
-	int32_t ref_out_buf_idx;
-};
-
-#endif /* __IA_CSS_REF_STATE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_types.h
deleted file mode 100644
index 4750fba268b9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ref/ref_1.0/ia_css_ref_types.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_REF_TYPES_H
-#define __IA_CSS_REF_TYPES_H
-
-/* Reference frame
- *
- *  ISP block: reference frame
- */
-
-#include <ia_css_frame_public.h>
-
-
-
-#endif /* __IA_CSS_REF_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c
deleted file mode 100644
index aa733674f42b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c
+++ /dev/null
@@ -1,386 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#ifndef IA_CSS_NO_DEBUG
-#include "ia_css_debug.h"
-#endif
-#include "sh_css_frac.h"
-#include "assert_support.h"
-
-#include "bh/bh_2/ia_css_bh.host.h"
-#include "ia_css_s3a.host.h"
-
-const struct ia_css_3a_config default_3a_config = {
-	25559,
-	32768,
-	7209,
-	65535,
-	0,
-	65535,
-	{-3344, -6104, -19143, 19143, 6104, 3344, 0},
-	{1027, 0, -9219, 16384, -9219, 1027, 0}
-};
-
-static unsigned int s3a_raw_bit_depth;
-
-void
-ia_css_s3a_configure(unsigned int raw_bit_depth)
-{
-  s3a_raw_bit_depth = raw_bit_depth;
-}
-
-static void
-ia_css_ae_encode(
-	struct sh_css_isp_ae_params *to,
-	const struct ia_css_3a_config *from,
-	unsigned size)
-{
-	(void)size;
-	/* coefficients to calculate Y */
-	to->y_coef_r =
-	    uDIGIT_FITTING(from->ae_y_coef_r, 16, SH_CSS_AE_YCOEF_SHIFT);
-	to->y_coef_g =
-	    uDIGIT_FITTING(from->ae_y_coef_g, 16, SH_CSS_AE_YCOEF_SHIFT);
-	to->y_coef_b =
-	    uDIGIT_FITTING(from->ae_y_coef_b, 16, SH_CSS_AE_YCOEF_SHIFT);
-}
-
-static void
-ia_css_awb_encode(
-	struct sh_css_isp_awb_params *to,
-	const struct ia_css_3a_config *from,
-	unsigned size)
-{
-	(void)size;
-	/* AWB level gate */
-	to->lg_high_raw =
-		uDIGIT_FITTING(from->awb_lg_high_raw, 16, s3a_raw_bit_depth);
-	to->lg_low =
-		uDIGIT_FITTING(from->awb_lg_low, 16, SH_CSS_BAYER_BITS);
-	to->lg_high =
-		uDIGIT_FITTING(from->awb_lg_high, 16, SH_CSS_BAYER_BITS);
-}
-
-static void
-ia_css_af_encode(
-	struct sh_css_isp_af_params *to,
-	const struct ia_css_3a_config *from,
-	unsigned size)
-{
-	unsigned int i;
-	(void)size;
-
-	/* af fir coefficients */
-	for (i = 0; i < 7; ++i) {
-		to->fir1[i] =
-		  sDIGIT_FITTING(from->af_fir1_coef[i], 15,
-				 SH_CSS_AF_FIR_SHIFT);
-		to->fir2[i] =
-		  sDIGIT_FITTING(from->af_fir2_coef[i], 15,
-				 SH_CSS_AF_FIR_SHIFT);
-	}
-}
-
-void
-ia_css_s3a_encode(
-	struct sh_css_isp_s3a_params *to,
-	const struct ia_css_3a_config *from,
-	unsigned size)
-{
-	(void)size;
-
-	ia_css_ae_encode(&to->ae,   from, sizeof(to->ae));
-	ia_css_awb_encode(&to->awb, from, sizeof(to->awb));
-	ia_css_af_encode(&to->af,   from, sizeof(to->af));
-}
-
-#if 0
-void
-ia_css_process_s3a(
-	unsigned pipe_id,
-	const struct ia_css_pipeline_stage *stage,
-	struct ia_css_isp_parameters *params)
-{
-	short dmem_offset = stage->binary->info->mem_offsets->dmem.s3a;
-
-	assert(params != NULL);
-
-	if (dmem_offset >= 0) {
-		ia_css_s3a_encode((struct sh_css_isp_s3a_params *)
-				&stage->isp_mem_params[IA_CSS_ISP_DMEM0].address[dmem_offset],
-				&params->s3a_config);
-		ia_css_bh_encode((struct sh_css_isp_bh_params *)
-				&stage->isp_mem_params[IA_CSS_ISP_DMEM0].address[dmem_offset],
-				&params->s3a_config);
-		params->isp_params_changed = true;
-		params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM0] = true;
-	}
-
-	params->isp_params_changed = true;
-}
-#endif
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_ae_dump(
-	const struct sh_css_isp_ae_params *ae,
-	unsigned level)
-{
-	if (!ae) return;
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ae_y_coef_r", ae->y_coef_r);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ae_y_coef_g", ae->y_coef_g);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ae_y_coef_b", ae->y_coef_b);
-}
-
-void
-ia_css_awb_dump(
-	const struct sh_css_isp_awb_params *awb,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"awb_lg_high_raw", awb->lg_high_raw);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"awb_lg_low", awb->lg_low);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"awb_lg_high", awb->lg_high);
-}
-
-void
-ia_css_af_dump(
-	const struct sh_css_isp_af_params *af,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir1[0]", af->fir1[0]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir1[1]", af->fir1[1]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir1[2]", af->fir1[2]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir1[3]", af->fir1[3]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir1[4]", af->fir1[4]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir1[5]", af->fir1[5]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir1[6]", af->fir1[6]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir2[0]", af->fir2[0]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir2[1]", af->fir2[1]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir2[2]", af->fir2[2]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir2[3]", af->fir2[3]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir2[4]", af->fir2[4]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir2[5]", af->fir2[5]);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"af_fir2[6]", af->fir2[6]);
-}
-
-void
-ia_css_s3a_dump(
-	const struct sh_css_isp_s3a_params *s3a,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level, "S3A Support:\n");
-	ia_css_ae_dump  (&s3a->ae, level);
-	ia_css_awb_dump (&s3a->awb, level);
-	ia_css_af_dump  (&s3a->af, level);
-}
-
-void
-ia_css_s3a_debug_dtrace(
-	const struct ia_css_3a_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.ae_y_coef_r=%d, config.ae_y_coef_g=%d, "
-		"config.ae_y_coef_b=%d, config.awb_lg_high_raw=%d, "
-		"config.awb_lg_low=%d, config.awb_lg_high=%d\n",
-		config->ae_y_coef_r, config->ae_y_coef_g,
-		config->ae_y_coef_b, config->awb_lg_high_raw,
-		config->awb_lg_low, config->awb_lg_high);
-}
-#endif
-
-void
-ia_css_s3a_hmem_decode(
-	struct ia_css_3a_statistics *host_stats,
-	const struct ia_css_bh_table *hmem_buf)
-{
-#if defined(HAS_NO_HMEM)
-	(void)host_stats;
-	(void)hmem_buf;
-#else
-	struct ia_css_3a_rgby_output	*out_ptr;
-	int			i;
-
-	/* pixel counts(BQ) for 3A area */
-	int count_for_3a;
-	int sum_r, diff;
-
-	assert(host_stats != NULL);
-	assert(host_stats->rgby_data != NULL);
-	assert(hmem_buf != NULL);
-
-	count_for_3a = host_stats->grid.width * host_stats->grid.height
-	    * host_stats->grid.bqs_per_grid_cell
-	    * host_stats->grid.bqs_per_grid_cell;
-
-	out_ptr = host_stats->rgby_data;
-
-	ia_css_bh_hmem_decode(out_ptr, hmem_buf);
-
-	/* Calculate sum of histogram of R,
-	   which should not be less than count_for_3a */
-	sum_r = 0;
-	for (i = 0; i < HMEM_UNIT_SIZE; i++) {
-		sum_r += out_ptr[i].r;
-	}
-	if (sum_r < count_for_3a) {
-		/* histogram is invalid */
-		return;
-	}
-
-	/* Verify for sum of histogram of R/G/B/Y */
-#if 0
-	{
-		int sum_g = 0;
-		int sum_b = 0;
-		int sum_y = 0;
-		for (i = 0; i < HMEM_UNIT_SIZE; i++) {
-			sum_g += out_ptr[i].g;
-			sum_b += out_ptr[i].b;
-			sum_y += out_ptr[i].y;
-		}
-		if (sum_g != sum_r || sum_b != sum_r || sum_y != sum_r) {
-			/* histogram is invalid */
-			return;
-		}
-	}
-#endif
-
-	/*
-	 * Limit the histogram area only to 3A area.
-	 * In DSP, the histogram of 0 is incremented for pixels
-	 * which are outside of 3A area. That amount should be subtracted here.
-	 *   hist[0] = hist[0] - ((sum of all hist[]) - (pixel count for 3A area))
-	 */
-	diff = sum_r - count_for_3a;
-	out_ptr[0].r -= diff;
-	out_ptr[0].g -= diff;
-	out_ptr[0].b -= diff;
-	out_ptr[0].y -= diff;
-#endif
-}
-
-void
-ia_css_s3a_dmem_decode(
-	struct ia_css_3a_statistics *host_stats,
-	const struct ia_css_3a_output *isp_stats)
-{
-	int isp_width, host_width, height, i;
-	struct ia_css_3a_output *host_ptr;
-
-	assert(host_stats != NULL);
-	assert(host_stats->data != NULL);
-	assert(isp_stats != NULL);
-
-	isp_width  = host_stats->grid.aligned_width;
-	host_width = host_stats->grid.width;
-	height     = host_stats->grid.height;
-	host_ptr   = host_stats->data;
-
-	/* Getting 3A statistics from DMEM does not involve any
-	 * transformation (like the VMEM version), we just copy the data
-	 * using a different output width. */
-	for (i = 0; i < height; i++) {
-		memcpy(host_ptr, isp_stats, host_width * sizeof(*host_ptr));
-		isp_stats += isp_width;
-		host_ptr += host_width;
-	}
-}
-
-/* MW: this is an ISP function */
-static inline int
-merge_hi_lo_14(unsigned short hi, unsigned short lo)
-{
-	int val = (int) ((((unsigned int) hi << 14) & 0xfffc000) |
-			((unsigned int) lo & 0x3fff));
-	return val;
-}
-
-void
-ia_css_s3a_vmem_decode(
-	struct ia_css_3a_statistics *host_stats,
-	const uint16_t *isp_stats_hi,
-	const uint16_t *isp_stats_lo)
-{
-	int out_width, out_height, chunk, rest, kmax, y, x, k, elm_start, elm, ofs;
-	const uint16_t *hi, *lo;
-	struct ia_css_3a_output *output;
-
-	assert(host_stats!= NULL);
-	assert(host_stats->data != NULL);
-	assert(isp_stats_hi != NULL);
-	assert(isp_stats_lo != NULL);
-
-	output = host_stats->data;
-	out_width  = host_stats->grid.width;
-	out_height = host_stats->grid.height;
-	hi = isp_stats_hi;
-	lo = isp_stats_lo;
-
-	chunk = ISP_VEC_NELEMS >> host_stats->grid.deci_factor_log2;
-	chunk = max(chunk, 1);
-
-	for (y = 0; y < out_height; y++) {
-		elm_start = y * ISP_S3ATBL_HI_LO_STRIDE;
-		rest = out_width;
-		x = 0;
-		while (x < out_width) {
-			kmax = (rest > chunk) ? chunk : rest;
-			ofs = y * out_width + x;
-			elm = elm_start + x * sizeof(*output) / sizeof(int32_t);
-			for (k = 0; k < kmax; k++, elm++) {
-				output[ofs + k].ae_y    = merge_hi_lo_14(
-				    hi[elm + chunk * 0], lo[elm + chunk * 0]);
-				output[ofs + k].awb_cnt = merge_hi_lo_14(
-				    hi[elm + chunk * 1], lo[elm + chunk * 1]);
-				output[ofs + k].awb_gr  = merge_hi_lo_14(
-				    hi[elm + chunk * 2], lo[elm + chunk * 2]);
-				output[ofs + k].awb_r   = merge_hi_lo_14(
-				    hi[elm + chunk * 3], lo[elm + chunk * 3]);
-				output[ofs + k].awb_b   = merge_hi_lo_14(
-				    hi[elm + chunk * 4], lo[elm + chunk * 4]);
-				output[ofs + k].awb_gb  = merge_hi_lo_14(
-				    hi[elm + chunk * 5], lo[elm + chunk * 5]);
-				output[ofs + k].af_hpf1 = merge_hi_lo_14(
-				    hi[elm + chunk * 6], lo[elm + chunk * 6]);
-				output[ofs + k].af_hpf2 = merge_hi_lo_14(
-				    hi[elm + chunk * 7], lo[elm + chunk * 7]);
-			}
-			x += chunk;
-			rest -= chunk;
-		}
-	}
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h
deleted file mode 100644
index 4bc6c0bf478f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_S3A_HOST_H
-#define __IA_CSS_S3A_HOST_H
-
-#include "ia_css_s3a_types.h"
-#include "ia_css_s3a_param.h"
-#include "bh/bh_2/ia_css_bh.host.h"
-
-extern const struct ia_css_3a_config default_3a_config;
-
-void
-ia_css_s3a_configure(
-	unsigned int raw_bit_depth);
-
-void
-ia_css_s3a_encode(
-	struct sh_css_isp_s3a_params *to,
-	const struct ia_css_3a_config *from,
-	unsigned size);
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_ae_dump(
-	const struct sh_css_isp_ae_params *ae,
-	unsigned level);
-
-void
-ia_css_awb_dump(
-	const struct sh_css_isp_awb_params *awb,
-	unsigned level);
-
-void
-ia_css_af_dump(
-	const struct sh_css_isp_af_params *af,
-	unsigned level);
-
-void
-ia_css_s3a_dump(
-	const struct sh_css_isp_s3a_params *s3a,
-	unsigned level);
-
-void
-ia_css_s3a_debug_dtrace(
-	const struct ia_css_3a_config *config,
-	unsigned level);
-#endif
-
-void
-ia_css_s3a_hmem_decode(
-	struct ia_css_3a_statistics *host_stats,
-	const struct ia_css_bh_table *hmem_buf);
-
-void
-ia_css_s3a_dmem_decode(
-	struct ia_css_3a_statistics *host_stats,
-	const struct ia_css_3a_output *isp_stats);
-
-void
-ia_css_s3a_vmem_decode(
-	struct ia_css_3a_statistics *host_stats,
-	const uint16_t *isp_stats_hi,
-	const uint16_t *isp_stats_lo);
-
-#endif /* __IA_CSS_S3A_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h
deleted file mode 100644
index 35fb0a2c921a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_S3A_PARAM_H
-#define __IA_CSS_S3A_PARAM_H
-
-#include "type_support.h"
-
-/* AE (3A Support) */
-struct sh_css_isp_ae_params {
-	/* coefficients to calculate Y */
-	int32_t y_coef_r;
-	int32_t y_coef_g;
-	int32_t y_coef_b;
-};
-
-/* AWB (3A Support) */
-struct sh_css_isp_awb_params {
-	int32_t lg_high_raw;
-	int32_t lg_low;
-	int32_t lg_high;
-};
-
-/* AF (3A Support) */
-struct sh_css_isp_af_params {
-	int32_t fir1[7];
-	int32_t fir2[7];
-};
-
-/* S3A (3A Support) */
-struct sh_css_isp_s3a_params {
-	/* coefficients to calculate Y */
-	struct sh_css_isp_ae_params ae;
-	
-	/* AWB level gate */
-	struct sh_css_isp_awb_params awb;
-
-	/* af fir coefficients */
-	struct sh_css_isp_af_params af;
-};
-
-
-#endif /* __IA_CSS_S3A_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h
deleted file mode 100644
index 63e70669f085..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_S3A_TYPES_H
-#define __IA_CSS_S3A_TYPES_H
-
-/* @file
-* CSS-API header file for 3A statistics parameters.
-*/
-
-#include <ia_css_frac.h>
-
-#if (defined(SYSTEM_css_skycam_c0_system)) && (! defined(PIPE_GENERATION) )
-#include "../../../../components/stats_3a/src/stats_3a_public.h"
-#endif
-
-/* 3A configuration. This configures the 3A statistics collection
- *  module.
- */
-
-/* 3A statistics grid
- *
- *  ISP block: S3A1 (3A Support for 3A ver.1 (Histogram is not used for AE))
- *             S3A2 (3A Support for 3A ver.2 (Histogram is used for AE))
- *  ISP1: S3A1 is used.
- *  ISP2: S3A2 is used.
- */
-struct ia_css_3a_grid_info {
-
-#if defined(SYSTEM_css_skycam_c0_system)
-	uint32_t ae_enable;					/** ae enabled in binary,
-								   0:disabled, 1:enabled */
-	struct ae_public_config_grid_config	ae_grd_info;	/** see description in ae_public.h*/
-
-  	uint32_t awb_enable;					/** awb enabled in binary,
-								   0:disabled, 1:enabled */
-	struct awb_public_config_grid_config	awb_grd_info;	/** see description in awb_public.h*/
-
-  	uint32_t af_enable;					/** af enabled in binary,
-								   0:disabled, 1:enabled */
-	struct af_public_grid_config		af_grd_info;	/** see description in af_public.h*/
-
-  	uint32_t awb_fr_enable;					/** awb_fr enabled in binary,
-								   0:disabled, 1:enabled */
-	struct awb_fr_public_grid_config	awb_fr_grd_info;/** see description in awb_fr_public.h*/
-
-        uint32_t elem_bit_depth;    /** TODO:Taken from BYT  - need input from AIQ
-					if needed for SKC
-					Bit depth of element used
-					to calculate 3A statistics.
-					This is 13, which is the normalized
-					bayer bit depth in DSP. */
-
-#else
-	uint32_t enable;            /** 3A statistics enabled.
-					0:disabled, 1:enabled */
-	uint32_t use_dmem;          /** DMEM or VMEM determines layout.
-					0:3A statistics are stored to VMEM,
-					1:3A statistics are stored to DMEM */
-	uint32_t has_histogram;     /** Statistics include histogram.
-					0:no histogram, 1:has histogram */
-	uint32_t width;	    	    /** Width of 3A grid table.
-					(= Horizontal number of grid cells
-					in table, which cells have effective
-					statistics.) */
-	uint32_t height;	    /** Height of 3A grid table.
-					(= Vertical number of grid cells
-					in table, which cells have effective
-					statistics.) */
-	uint32_t aligned_width;     /** Horizontal stride (for alloc).
-					(= Horizontal number of grid cells
-					in table, which means
-					the allocated width.) */
-	uint32_t aligned_height;    /** Vertical stride (for alloc).
-					(= Vertical number of grid cells
-					in table, which means
-					the allocated height.) */
-	uint32_t bqs_per_grid_cell; /** Grid cell size in BQ(Bayer Quad) unit.
-					(1BQ means {Gr,R,B,Gb}(2x2 pixels).)
-					Valid values are 8,16,32,64. */
-	uint32_t deci_factor_log2;  /** log2 of bqs_per_grid_cell. */
-	uint32_t elem_bit_depth;    /** Bit depth of element used
-					to calculate 3A statistics.
-					This is 13, which is the normalized
-					bayer bit depth in DSP. */
-#endif
-};
-
-
-/* This struct should be split into 3, for AE, AWB and AF.
- * However, that will require driver/ 3A lib modifications.
- */
-
-/* 3A configuration. This configures the 3A statistics collection
- *  module.
- *
- *  ae_y_*: Coefficients to calculate luminance from bayer.
- *  awb_lg_*: Thresholds to check the saturated bayer pixels for AWB.
- *    Condition of effective pixel for AWB level gate check:
- *      bayer(sensor) <= awb_lg_high_raw &&
- *      bayer(when AWB statisitcs is calculated) >= awb_lg_low &&
- *      bayer(when AWB statisitcs is calculated) <= awb_lg_high
- *  af_fir*: Coefficients of high pass filter to calculate AF statistics.
- *
- *  ISP block: S3A1(ae_y_* for AE/AF, awb_lg_* for AWB)
- *             S3A2(ae_y_* for AF, awb_lg_* for AWB)
- *             SDVS1(ae_y_*)
- *             SDVS2(ae_y_*)
- *  ISP1: S3A1 and SDVS1 are used.
- *  ISP2: S3A2 and SDVS2 are used.
- */
-struct ia_css_3a_config {
-	ia_css_u0_16 ae_y_coef_r;	/** Weight of R for Y.
-						u0.16, [0,65535],
-						default/ineffective 25559 */
-	ia_css_u0_16 ae_y_coef_g;	/** Weight of G for Y.
-						u0.16, [0,65535],
-						default/ineffective 32768 */
-	ia_css_u0_16 ae_y_coef_b;	/** Weight of B for Y.
-						u0.16, [0,65535],
-						default/ineffective 7209 */
-	ia_css_u0_16 awb_lg_high_raw;	/** AWB level gate high for raw.
-						u0.16, [0,65535],
-						default 65472(=1023*64),
-						ineffective 65535 */
-	ia_css_u0_16 awb_lg_low;	/** AWB level gate low.
-						u0.16, [0,65535],
-						default 64(=1*64),
-						ineffective 0 */
-	ia_css_u0_16 awb_lg_high;	/** AWB level gate high.
-						u0.16, [0,65535],
-						default 65535,
-						ineffective 65535 */
-	ia_css_s0_15 af_fir1_coef[7];	/** AF FIR coefficients of fir1.
-						s0.15, [-32768,32767],
-				default/ineffective
-				-6689,-12207,-32768,32767,12207,6689,0 */
-	ia_css_s0_15 af_fir2_coef[7];	/** AF FIR coefficients of fir2.
-						s0.15, [-32768,32767],
-				default/ineffective
-				2053,0,-18437,32767,-18437,2053,0 */
-};
-
-/* 3A statistics. This structure describes the data stored
- *  in each 3A grid point.
- *
- *  ISP block: S3A1 (3A Support for 3A ver.1) (Histogram is not used for AE)
- *             S3A2 (3A Support for 3A ver.2) (Histogram is used for AE)
- *             - ae_y is used only for S3A1.
- *             - awb_* and af_* are used both for S3A1 and S3A2.
- *  ISP1: S3A1 is used.
- *  ISP2: S3A2 is used.
- */
-struct ia_css_3a_output {
-	int32_t ae_y;    /** Sum of Y in a statistics window, for AE.
-				(u19.13) */
-	int32_t awb_cnt; /** Number of effective pixels
-				in a statistics window.
-				Pixels passed by the AWB level gate check are
-				judged as "effective". (u32) */
-	int32_t awb_gr;  /** Sum of Gr in a statistics window, for AWB.
-				All Gr pixels (not only for effective pixels)
-				are summed. (u19.13) */
-	int32_t awb_r;   /** Sum of R in a statistics window, for AWB.
-				All R pixels (not only for effective pixels)
-				are summed. (u19.13) */
-	int32_t awb_b;   /** Sum of B in a statistics window, for AWB.
-				All B pixels (not only for effective pixels)
-				are summed. (u19.13) */
-	int32_t awb_gb;  /** Sum of Gb in a statistics window, for AWB.
-				All Gb pixels (not only for effective pixels)
-				are summed. (u19.13) */
-	int32_t af_hpf1; /** Sum of |Y| following high pass filter af_fir1
-				within a statistics window, for AF. (u19.13) */
-	int32_t af_hpf2; /** Sum of |Y| following high pass filter af_fir2
-				within a statistics window, for AF. (u19.13) */
-};
-
-
-/* 3A Statistics. This structure describes the statistics that are generated
- *  using the provided configuration (ia_css_3a_config).
- */
-struct ia_css_3a_statistics {
-	struct ia_css_3a_grid_info    grid;	/** grid info contains the dimensions of the 3A grid */
-	struct ia_css_3a_output      *data;	/** the pointer to 3a_output[grid.width * grid.height]
-						     containing the 3A statistics */
-	struct ia_css_3a_rgby_output *rgby_data;/** the pointer to 3a_rgby_output[256]
-						     containing the histogram */
-};
-
-/* Histogram (Statistics for AE).
- *
- *  4 histograms(r,g,b,y),
- *  256 bins for each histogram, unsigned 24bit value for each bin.
- *    struct ia_css_3a_rgby_output data[256];
-
- *  ISP block: HIST2
- * (ISP1: HIST2 is not used.)
- *  ISP2: HIST2 is used.
- */
-struct ia_css_3a_rgby_output {
-	uint32_t r;	/** Number of R of one bin of the histogram R. (u24) */
-	uint32_t g;	/** Number of G of one bin of the histogram G. (u24) */
-	uint32_t b;	/** Number of B of one bin of the histogram B. (u24) */
-	uint32_t y;	/** Number of Y of one bin of the histogram Y. (u24) */
-};
-
-#endif /* __IA_CSS_S3A_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.c
deleted file mode 100644
index 565ae45b7541..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-#ifdef ISP2401
-#include "math_support.h"	/* min() */
-
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-#endif
-
-#include "ia_css_sc.host.h"
-
-void
-ia_css_sc_encode(
-	struct sh_css_isp_sc_params *to,
-	struct ia_css_shading_table **from,
-	unsigned size)
-{
-	(void)size;
-	to->gain_shift = (*from)->fraction_bits;
-}
-
-void
-ia_css_sc_dump(
-	const struct sh_css_isp_sc_params *sc,
-	unsigned level)
-{
-	if (!sc) return;
-	ia_css_debug_dtrace(level, "Shading Correction:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"sc_gain_shift", sc->gain_shift);
-}
-
-#ifdef ISP2401
-void
-ia_css_sc_config(
-	struct sh_css_isp_sc_isp_config *to,
-	const struct ia_css_sc_configuration *from,
-	unsigned size)
-{
-	uint32_t internal_org_x_bqs = from->internal_frame_origin_x_bqs_on_sctbl;
-	uint32_t internal_org_y_bqs = from->internal_frame_origin_y_bqs_on_sctbl;
-	uint32_t slice, rest, i;
-
-	(void)size;
-
-	/* The internal_frame_origin_x_bqs_on_sctbl is separated to 8 times of slice_vec. */
-	rest = internal_org_x_bqs;
-	for (i = 0; i < SH_CSS_SC_INTERPED_GAIN_HOR_SLICE_TIMES; i++) {
-		slice = min(rest, ((uint32_t)ISP_SLICE_NELEMS));
-		rest = rest - slice;
-		to->interped_gain_hor_slice_bqs[i] = slice;
-	}
-
-	to->internal_frame_origin_y_bqs_on_sctbl = internal_org_y_bqs;
-}
-
-void
-ia_css_sc_configure(
-	const struct ia_css_binary *binary,
-	uint32_t internal_frame_origin_x_bqs_on_sctbl,
-	uint32_t internal_frame_origin_y_bqs_on_sctbl)
-{
-	const struct ia_css_sc_configuration config = {
-		internal_frame_origin_x_bqs_on_sctbl,
-		internal_frame_origin_y_bqs_on_sctbl };
-
-	ia_css_configure_sc(binary, &config);
-}
-
-#endif
-/* ------ deprecated(bz675) : from ------ */
-/* It looks like @parameter{} (in *.pipe) is used to generate the process/get/set functions,
-   for parameters which should be used in the isp kernels.
-   However, the ia_css_shading_settings structure has a parameter which is used only in the css,
-   and does not have a parameter which is used in the isp kernels.
-   Then, I did not use @parameter{} to generate the get/set function
-   for the ia_css_shading_settings structure. (michie) */
-void
-sh_css_get_shading_settings(const struct ia_css_isp_parameters *params,
-			struct ia_css_shading_settings *settings)
-{
-	if (settings == NULL)
-		return;
-	assert(params != NULL);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_get_shading_settings() enter: settings=%p\n", settings);
-
-	*settings = params->shading_settings;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_get_shading_settings() leave: settings.enable_shading_table_conversion=%d\n",
-		settings->enable_shading_table_conversion);
-}
-
-void
-sh_css_set_shading_settings(struct ia_css_isp_parameters *params,
-			const struct ia_css_shading_settings *settings)
-{
-	if (settings == NULL)
-		return;
-	assert(params != NULL);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_set_shading_settings() enter: settings.enable_shading_table_conversion=%d\n",
-		settings->enable_shading_table_conversion);
-
-	params->shading_settings = *settings;
-	params->shading_settings_changed = true;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_set_shading_settings() leave: return_void\n");
-}
-/* ------ deprecated(bz675) : to ------ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.h
deleted file mode 100644
index b35ac3e4009b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc.host.h
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_SC_HOST_H
-#define __IA_CSS_SC_HOST_H
-
-#include "sh_css_params.h"
-
-#include "ia_css_sc_types.h"
-#include "ia_css_sc_param.h"
-
-void
-ia_css_sc_encode(
-	struct sh_css_isp_sc_params *to,
-	struct ia_css_shading_table **from,
-	unsigned size);
-
-void
-ia_css_sc_dump(
-	const struct sh_css_isp_sc_params *sc,
-	unsigned level);
-
-#ifdef ISP2401
-/* @brief Configure the shading correction.
- * @param[out]	to	Parameters used in the shading correction kernel in the isp.
- * @param[in]	from	Parameters passed from the host.
- * @param[in]	size	Size of the sh_css_isp_sc_isp_config structure.
- *
- * This function passes the parameters for the shading correction from the host to the isp.
- */
-void
-ia_css_sc_config(
-	struct sh_css_isp_sc_isp_config *to,
-	const struct ia_css_sc_configuration *from,
-	unsigned size);
-
-/* @brief Configure the shading correction.
- * @param[in]	binary	The binary, which has the shading correction.
- * @param[in]	internal_frame_origin_x_bqs_on_sctbl
- *			X coordinate (in bqs) of the origin of the internal frame on the shading table.
- * @param[in]	internal_frame_origin_y_bqs_on_sctbl
- *			Y coordinate (in bqs) of the origin of the internal frame on the shading table.
- *
- * This function calls the ia_css_configure_sc() function.
- * (The ia_css_configure_sc() function is automatically generated in ia_css_isp.configs.c.)
- * The ia_css_configure_sc() function calls the ia_css_sc_config() function
- * to pass the parameters for the shading correction from the host to the isp.
- */
-void
-ia_css_sc_configure(
-	const struct ia_css_binary *binary,
-	uint32_t internal_frame_origin_x_bqs_on_sctbl,
-	uint32_t internal_frame_origin_y_bqs_on_sctbl);
-
-#endif
-/* ------ deprecated(bz675) : from ------ */
-void
-sh_css_get_shading_settings(const struct ia_css_isp_parameters *params,
-			struct ia_css_shading_settings *settings);
-
-void
-sh_css_set_shading_settings(struct ia_css_isp_parameters *params,
-			const struct ia_css_shading_settings *settings);
-/* ------ deprecated(bz675) : to ------ */
-
-#endif /* __IA_CSS_SC_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc_param.h
deleted file mode 100644
index d997d5137634..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc_param.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_SC_PARAM_H
-#define __IA_CSS_SC_PARAM_H
-
-#include "type_support.h"
-
-#ifdef ISP2401
-/* To position the shading center grid point on the center of output image,
- * one more grid cell is needed as margin. */
-#define SH_CSS_SCTBL_CENTERING_MARGIN	1
-
-/* The shading table width and height are the number of grids, not cells. The last grid should be counted. */
-#define SH_CSS_SCTBL_LAST_GRID_COUNT	1
-
-/* Number of horizontal grids per color in the shading table. */
-#define _ISP_SCTBL_WIDTH_PER_COLOR(input_width, deci_factor_log2) \
-	(ISP_BQ_GRID_WIDTH(input_width, deci_factor_log2) + \
-	SH_CSS_SCTBL_CENTERING_MARGIN + SH_CSS_SCTBL_LAST_GRID_COUNT)
-
-/* Number of vertical grids per color in the shading table. */
-#define _ISP_SCTBL_HEIGHT(input_height, deci_factor_log2) \
-	(ISP_BQ_GRID_HEIGHT(input_height, deci_factor_log2) + \
-	SH_CSS_SCTBL_CENTERING_MARGIN + SH_CSS_SCTBL_LAST_GRID_COUNT)
-
-/* Legacy API: Number of horizontal grids per color in the shading table. */
-#define _ISP_SCTBL_LEGACY_WIDTH_PER_COLOR(input_width, deci_factor_log2) \
-	(ISP_BQ_GRID_WIDTH(input_width, deci_factor_log2) + SH_CSS_SCTBL_LAST_GRID_COUNT)
-
-/* Legacy API: Number of vertical grids per color in the shading table. */
-#define _ISP_SCTBL_LEGACY_HEIGHT(input_height, deci_factor_log2) \
-	(ISP_BQ_GRID_HEIGHT(input_height, deci_factor_log2) + SH_CSS_SCTBL_LAST_GRID_COUNT)
-
-#endif
-/* SC (Shading Corrction) */
-struct sh_css_isp_sc_params {
-	int32_t gain_shift;
-};
-
-#ifdef ISP2401
-/* Number of horizontal slice times for interpolated gain:
- *
- * The start position of the internal frame does not match the start position of the shading table.
- * To get a vector of shading gains (interpolated horizontally and vertically)
- * which matches a vector on the internal frame,
- * vec_slice is used for 2 adjacent vectors of shading gains.
- * The number of shift times by vec_slice is 8.
- *     Max grid cell bqs to support the shading table centerting: N = 32
- *     CEIL_DIV(N-1, ISP_SLICE_NELEMS) = CEIL_DIV(31, 4) = 8
- */
-#define SH_CSS_SC_INTERPED_GAIN_HOR_SLICE_TIMES   8
-
-struct sh_css_isp_sc_isp_config {
-	uint32_t interped_gain_hor_slice_bqs[SH_CSS_SC_INTERPED_GAIN_HOR_SLICE_TIMES];
-	uint32_t internal_frame_origin_y_bqs_on_sctbl;
-};
-
-#endif
-#endif /* __IA_CSS_SC_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc_types.h
deleted file mode 100644
index 30ce499ac8cf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sc/sc_1.0/ia_css_sc_types.h
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_SC_TYPES_H
-#define __IA_CSS_SC_TYPES_H
-
-/* @file
-* CSS-API header file for Lens Shading Correction (SC) parameters.
-*/
-
-
-/* Number of color planes in the shading table. */
-#define IA_CSS_SC_NUM_COLORS           4
-
-/* The 4 colors that a shading table consists of.
- *  For each color we store a grid of values.
- */
-enum ia_css_sc_color {
-	IA_CSS_SC_COLOR_GR, /** Green on a green-red line */
-	IA_CSS_SC_COLOR_R,  /** Red */
-	IA_CSS_SC_COLOR_B,  /** Blue */
-	IA_CSS_SC_COLOR_GB  /** Green on a green-blue line */
-};
-
-/* Lens Shading Correction table.
- *
- *  This describes the color shading artefacts
- *  introduced by lens imperfections. To correct artefacts,
- *  bayer values should be multiplied by gains in this table.
- *
- *------------ deprecated(bz675) : from ---------------------------
- *  When shading_settings.enable_shading_table_conversion is set as 0,
- *  this shading table is directly sent to the isp. This table should contain
- *  the data based on the ia_css_shading_info information filled in the css.
- *  So, the driver needs to get the ia_css_shading_info information
- *  from the css, prior to generating the shading table.
- *
- *  When shading_settings.enable_shading_table_conversion is set as 1,
- *  this shading table is converted in the legacy way in the css
- *  before it is sent to the isp.
- *  The driver does not need to get the ia_css_shading_info information.
- *
- *  NOTE:
- *  The shading table conversion will be removed from the css in the near future,
- *  because it does not support the bayer scaling by sensor.
- *  Also, we had better generate the shading table only in one place(AIC).
- *  At the moment, to support the old driver which assumes the conversion is done in the css,
- *  shading_settings.enable_shading_table_conversion is set as 1 by default.
- *------------ deprecated(bz675) : to ---------------------------
- *
- *  ISP block: SC1
- *  ISP1: SC1 is used.
- *  ISP2: SC1 is used.
- */
-struct ia_css_shading_table {
-	uint32_t enable; /** Set to false for no shading correction.
-		          The data field can be NULL when enable == true */
-/* ------ deprecated(bz675) : from ------ */
-	uint32_t sensor_width;  /** Native sensor width in pixels. */
-	uint32_t sensor_height; /** Native sensor height in lines.
-		When shading_settings.enable_shading_table_conversion is set
-		as 0, sensor_width and sensor_height are NOT used.
-		These are used only in the legacy shading table conversion
-		in the css, when shading_settings.
-		enable_shading_table_conversion is set as 1. */
-/* ------ deprecated(bz675) : to ------ */
-	uint32_t width;  /** Number of data points per line per color.
-				u8.0, [0,81] */
-	uint32_t height; /** Number of lines of data points per color.
-				u8.0, [0,61] */
-	uint32_t fraction_bits; /** Bits of fractional part in the data
-				points.
-				u8.0, [0,13] */
-	uint16_t *data[IA_CSS_SC_NUM_COLORS];
-	/** Table data, one array for each color.
-	     Use ia_css_sc_color to index this array.
-	     u[13-fraction_bits].[fraction_bits], [0,8191] */
-};
-
-/* ------ deprecated(bz675) : from ------ */
-/* Shading Correction settings.
- *
- *  NOTE:
- *  This structure should be removed when the shading table conversion is
- *  removed from the css.
- */
-struct ia_css_shading_settings {
-	uint32_t enable_shading_table_conversion; /** Set to 0,
-		if the conversion of the shading table should be disabled
-		in the css. (default 1)
-		  0: The shading table is directly sent to the isp.
-		     The shading table should contain the data based on the
-		     ia_css_shading_info information filled in the css.
-		  1: The shading table is converted in the css, to be fitted
-		     to the shading table definition required in the isp.
-		NOTE:
-		Previously, the shading table was always converted in the css
-		before it was sent to the isp, and this config was not defined.
-		Currently, the driver is supposed to pass the shading table
-		which should be directly sent to the isp.
-		However, some drivers may still pass the shading table which
-		needs the conversion without setting this config as 1.
-		To support such an unexpected case for the time being,
-		enable_shading_table_conversion is set as 1 by default
-		in the css. */
-};
-/* ------ deprecated(bz675) : to ------ */
-
-#ifdef ISP2401
-
-/* Shading Correction configuration.
- *
- *  NOTE: The shading table size is larger than or equal to the internal frame size.
- */
-struct ia_css_sc_configuration {
-	uint32_t internal_frame_origin_x_bqs_on_sctbl; /** Origin X (in bqs) of internal frame on shading table. */
-	uint32_t internal_frame_origin_y_bqs_on_sctbl; /** Origin Y (in bqs) of internal frame on shading table. */
-						/** NOTE: bqs = size in BQ(Bayer Quad) unit.
-							1BQ means {Gr,R,B,Gb}(2x2 pixels).
-							Horizontal 1 bqs corresponds to horizontal 2 pixels.
-							Vertical 1 bqs corresponds to vertical 2 pixels. */
-};
-#endif
-
-#endif /* __IA_CSS_SC_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/common/ia_css_sdis_common.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/common/ia_css_sdis_common.host.h
deleted file mode 100644
index 4eb4910798fa..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/common/ia_css_sdis_common.host.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_SDIS_COMMON_HOST_H
-#define _IA_CSS_SDIS_COMMON_HOST_H
-
-#define ISP_MAX_SDIS_HOR_PROJ_NUM_ISP \
-	__ISP_SDIS_HOR_PROJ_NUM_ISP(ISP_MAX_INTERNAL_WIDTH, ISP_MAX_INTERNAL_HEIGHT, \
-		SH_CSS_DIS_DECI_FACTOR_LOG2, ISP_PIPE_VERSION)
-#define ISP_MAX_SDIS_VER_PROJ_NUM_ISP \
-	__ISP_SDIS_VER_PROJ_NUM_ISP(ISP_MAX_INTERNAL_WIDTH, \
-		SH_CSS_DIS_DECI_FACTOR_LOG2)
-
-#define _ISP_SDIS_HOR_COEF_NUM_VECS \
-	__ISP_SDIS_HOR_COEF_NUM_VECS(ISP_INTERNAL_WIDTH)
-#define ISP_MAX_SDIS_HOR_COEF_NUM_VECS \
-	__ISP_SDIS_HOR_COEF_NUM_VECS(ISP_MAX_INTERNAL_WIDTH)
-#define ISP_MAX_SDIS_VER_COEF_NUM_VECS \
-	__ISP_SDIS_VER_COEF_NUM_VECS(ISP_MAX_INTERNAL_HEIGHT)
-
-/* SDIS Coefficients: */
-/* The ISP uses vectors to store the coefficients, so we round
-   the number of coefficients up to vectors. */
-#define __ISP_SDIS_HOR_COEF_NUM_VECS(in_width)  _ISP_VECS(_ISP_BQS(in_width))
-#define __ISP_SDIS_VER_COEF_NUM_VECS(in_height) _ISP_VECS(_ISP_BQS(in_height))
-
-/* SDIS Projections:
- * SDIS1: Horizontal projections are calculated for each line.
- * Vertical projections are calculated for each column.
- * SDIS2: Projections are calculated for each grid cell.
- * Grid cells that do not fall completely within the image are not
- * valid. The host needs to use the bigger one for the stride but
- * should only return the valid ones to the 3A. */
-#define __ISP_SDIS_HOR_PROJ_NUM_ISP(in_width, in_height, deci_factor_log2, \
-	isp_pipe_version) \
-	((isp_pipe_version == 1) ? \
-		CEIL_SHIFT(_ISP_BQS(in_height), deci_factor_log2) : \
-		CEIL_SHIFT(_ISP_BQS(in_width), deci_factor_log2))
-
-#define __ISP_SDIS_VER_PROJ_NUM_ISP(in_width, deci_factor_log2) \
-	CEIL_SHIFT(_ISP_BQS(in_width), deci_factor_log2)
-
-#define SH_CSS_DIS_VER_NUM_COEF_TYPES(b) \
-  (((b)->info->sp.pipeline.isp_pipe_version == 2) ? \
-	IA_CSS_DVS2_NUM_COEF_TYPES : \
-	IA_CSS_DVS_NUM_COEF_TYPES)
-
-#ifndef PIPE_GENERATION
-#if defined(__ISP) || defined (MK_FIRMWARE)
-
-/* Array cannot be 2-dimensional, since driver ddr allocation does not know stride */
-struct sh_css_isp_sdis_hori_proj_tbl {
-  int32_t tbl[ISP_DVS_NUM_COEF_TYPES * ISP_MAX_SDIS_HOR_PROJ_NUM_ISP];
-#if DVS2_PROJ_MARGIN > 0
-  int32_t margin[DVS2_PROJ_MARGIN];
-#endif
-};
-
-struct sh_css_isp_sdis_vert_proj_tbl {
-  int32_t tbl[ISP_DVS_NUM_COEF_TYPES * ISP_MAX_SDIS_VER_PROJ_NUM_ISP];
-#if DVS2_PROJ_MARGIN > 0
-  int32_t margin[DVS2_PROJ_MARGIN];
-#endif
-};
-
-struct sh_css_isp_sdis_hori_coef_tbl {
-  VMEM_ARRAY(tbl[ISP_DVS_NUM_COEF_TYPES], ISP_MAX_SDIS_HOR_COEF_NUM_VECS*ISP_NWAY);
-};
-
-struct sh_css_isp_sdis_vert_coef_tbl {
-  VMEM_ARRAY(tbl[ISP_DVS_NUM_COEF_TYPES], ISP_MAX_SDIS_VER_COEF_NUM_VECS*ISP_NWAY);
-};
-
-#endif /* defined(__ISP) || defined (MK_FIRMWARE) */
-#endif /* PIPE_GENERATION */
-
-#ifndef PIPE_GENERATION
-struct s_sdis_config {
-  unsigned horicoef_vectors;
-  unsigned vertcoef_vectors;
-  unsigned horiproj_num;
-  unsigned vertproj_num;
-};
-
-extern struct s_sdis_config sdis_config;
-#endif
-
-#endif /* _IA_CSS_SDIS_COMMON_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/common/ia_css_sdis_common_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/common/ia_css_sdis_common_types.h
deleted file mode 100644
index 381e5730d405..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/common/ia_css_sdis_common_types.h
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_SDIS_COMMON_TYPES_H
-#define __IA_CSS_SDIS_COMMON_TYPES_H
-
-/* @file
-* CSS-API header file for DVS statistics parameters.
-*/
-
-#include <type_support.h>
-
-/* DVS statistics grid dimensions in number of cells.
- */
-
-struct ia_css_dvs_grid_dim {
-	uint32_t width;		/** Width of DVS grid table in cells */
-	uint32_t height;	/** Height of DVS grid table in cells */
-};
-
-/* DVS statistics dimensions in number of cells for
- * grid, coeffieicient and projection.
- */
-
-struct ia_css_sdis_info {
-	struct {
-		struct ia_css_dvs_grid_dim dim; /* Dimensions */
-		struct ia_css_dvs_grid_dim pad; /* Padded dimensions */
-	} grid, coef, proj;
-	uint32_t deci_factor_log2;
-};
-
-/* DVS statistics grid
- *
- *  ISP block: SDVS1 (DIS/DVS Support for DIS/DVS ver.1 (2-axes))
- *             SDVS2 (DVS Support for DVS ver.2 (6-axes))
- *  ISP1: SDVS1 is used.
- *  ISP2: SDVS2 is used.
- */
-struct ia_css_dvs_grid_res {
-	uint32_t width;	    	/** Width of DVS grid table.
-					(= Horizontal number of grid cells
-					in table, which cells have effective
-					statistics.)
-					For DVS1, this is equal to
-					 the number of vertical statistics. */
-	uint32_t aligned_width; /** Stride of each grid line.
-					(= Horizontal number of grid cells
-					in table, which means
-					the allocated width.) */
-	uint32_t height;	/** Height of DVS grid table.
-					(= Vertical number of grid cells
-					in table, which cells have effective
-					statistics.)
-					For DVS1, This is equal to
-					the number of horizontal statistics. */
-	uint32_t aligned_height;/** Stride of each grid column.
-					(= Vertical number of grid cells
-					in table, which means
-					the allocated height.) */
-};
-
-/* TODO: use ia_css_dvs_grid_res in here.
- * However, that implies driver I/F changes
- */
-struct ia_css_dvs_grid_info {
-	uint32_t enable;        /** DVS statistics enabled.
-					0:disabled, 1:enabled */
-	uint32_t width;	    	/** Width of DVS grid table.
-					(= Horizontal number of grid cells
-					in table, which cells have effective
-					statistics.)
-					For DVS1, this is equal to
-					 the number of vertical statistics. */
-	uint32_t aligned_width; /** Stride of each grid line.
-					(= Horizontal number of grid cells
-					in table, which means
-					the allocated width.) */
-	uint32_t height;	/** Height of DVS grid table.
-					(= Vertical number of grid cells
-					in table, which cells have effective
-					statistics.)
-					For DVS1, This is equal to
-					the number of horizontal statistics. */
-	uint32_t aligned_height;/** Stride of each grid column.
-					(= Vertical number of grid cells
-					in table, which means
-					the allocated height.) */
-	uint32_t bqs_per_grid_cell; /** Grid cell size in BQ(Bayer Quad) unit.
-					(1BQ means {Gr,R,B,Gb}(2x2 pixels).)
-					For DVS1, valid value is 64.
-					For DVS2, valid value is only 64,
-					currently. */
-	uint32_t num_hor_coefs;	/** Number of horizontal coefficients. */
-	uint32_t num_ver_coefs;	/** Number of vertical coefficients. */
-};
-
-/* Number of DVS statistics levels
- */
-#define IA_CSS_DVS_STAT_NUM_OF_LEVELS	3
-
-/* DVS statistics generated by accelerator global configuration
- */
-struct dvs_stat_public_dvs_global_cfg {
-	unsigned char kappa;
-	/** DVS statistics global configuration - kappa */
-	unsigned char match_shift;
-	/** DVS statistics global configuration - match_shift */
-	unsigned char ybin_mode;
-	/** DVS statistics global configuration - y binning mode */
-};
-
-/* DVS statistics generated by accelerator level grid
- *  configuration
- */
-struct dvs_stat_public_dvs_level_grid_cfg {
-	unsigned char grid_width;
-	/** DVS statistics grid width */
-	unsigned char grid_height;
-	/** DVS statistics grid height */
-	unsigned char block_width;
-	/** DVS statistics block width */
-	unsigned char block_height;
-	/** DVS statistics block  height */
-};
-
-/* DVS statistics generated by accelerator level grid start
- *  configuration
- */
-struct dvs_stat_public_dvs_level_grid_start {
-	unsigned short x_start;
-	/** DVS statistics level x start */
-	unsigned short y_start;
-	/** DVS statistics level y start */
-	unsigned char enable;
-	/** DVS statistics level enable */
-};
-
-/* DVS statistics generated by accelerator level grid end
- *  configuration
- */
-struct dvs_stat_public_dvs_level_grid_end {
-	unsigned short x_end;
-	/** DVS statistics level x end */
-	unsigned short y_end;
-	/** DVS statistics level y end */
-};
-
-/* DVS statistics generated by accelerator Feature Extraction
- *  Region Of Interest (FE-ROI) configuration
- */
-struct dvs_stat_public_dvs_level_fe_roi_cfg {
-	unsigned char x_start;
-	/** DVS statistics fe-roi level x start */
-	unsigned char y_start;
-	/** DVS statistics fe-roi level y start */
-	unsigned char x_end;
-	/** DVS statistics fe-roi level x end */
-	unsigned char y_end;
-	/** DVS statistics fe-roi level y end */
-};
-
-/* DVS statistics generated by accelerator public configuration
- */
-struct dvs_stat_public_dvs_grd_cfg {
-	struct dvs_stat_public_dvs_level_grid_cfg    grd_cfg;
-	/** DVS statistics level grid configuration */
-	struct dvs_stat_public_dvs_level_grid_start  grd_start;
-	/** DVS statistics level grid start configuration */
-	struct dvs_stat_public_dvs_level_grid_end    grd_end;
-	/** DVS statistics level grid end configuration */
-};
-
-/* DVS statistics grid generated by accelerator
- */
-struct ia_css_dvs_stat_grid_info {
-	struct dvs_stat_public_dvs_global_cfg       dvs_gbl_cfg;
-	/** DVS statistics global configuration (kappa, match, binning) */
-	struct dvs_stat_public_dvs_grd_cfg       grd_cfg[IA_CSS_DVS_STAT_NUM_OF_LEVELS];
-	/** DVS statistics grid configuration (blocks and grids) */
-	struct dvs_stat_public_dvs_level_fe_roi_cfg fe_roi_cfg[IA_CSS_DVS_STAT_NUM_OF_LEVELS];
-	/** DVS statistics FE ROI (region of interest) configuration */
-};
-
-/* DVS statistics generated by accelerator default grid info
- */
-#define DEFAULT_DVS_GRID_INFO \
-(union ia_css_dvs_grid_u) { \
-	.dvs_stat_grid_info = (struct ia_css_dvs_stat_grid_info) { \
-		.fe_roi_cfg = { \
-			[1] = (struct dvs_stat_public_dvs_level_fe_roi_cfg) { \
-				.x_start = 4 \
-			} \
-		} \
-	} \
-}
-
-/* Union that holds all types of DVS statistics grid info in
- *  CSS format
- * */
-union ia_css_dvs_grid_u {
-	struct ia_css_dvs_stat_grid_info dvs_stat_grid_info;
-	/** DVS statistics produced by accelerator grid info */
-	struct ia_css_dvs_grid_info dvs_grid_info;
-	/** DVS (DVS1/DVS2) grid info */
-};
-
-#endif /* __IA_CSS_SDIS_COMMON_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c
deleted file mode 100644
index 0fdd696bf654..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "memory_access.h"
-#include "assert_support.h"
-#include "ia_css_debug.h"
-#include "ia_css_sdis_types.h"
-#include "sdis/common/ia_css_sdis_common.host.h"
-#include "ia_css_sdis.host.h"
-
-const struct ia_css_dvs_coefficients default_sdis_config = {
-	.grid = { 0, 0, 0, 0, 0, 0, 0, 0 },
-	.hor_coefs = NULL,
-	.ver_coefs = NULL
-};
-
-static void
-fill_row(short *private, const short *public, unsigned width, unsigned padding)
-{
-	assert((int)width >= 0);
-	assert((int)padding >= 0);
-	memcpy (private, public, width*sizeof(short));
-	memset (&private[width], 0, padding*sizeof(short));
-}
-
-void ia_css_sdis_horicoef_vmem_encode (
-	struct sh_css_isp_sdis_hori_coef_tbl *to,
-	const struct ia_css_dvs_coefficients *from,
-	unsigned size)
-{
-	unsigned aligned_width = from->grid.aligned_width * from->grid.bqs_per_grid_cell;
-	unsigned width         = from->grid.num_hor_coefs;
-	int      padding       = aligned_width-width;
-	unsigned stride        = size/IA_CSS_DVS_NUM_COEF_TYPES/sizeof(short);
-	unsigned total_bytes   = aligned_width*IA_CSS_DVS_NUM_COEF_TYPES*sizeof(short);
-	short   *public        = from->hor_coefs;
-	short   *private       = (short*)to;
-	unsigned type;
-
-	/* Copy the table, add padding */
-	assert(padding >= 0);
-	assert(total_bytes <= size);
-	assert(size % (IA_CSS_DVS_NUM_COEF_TYPES*ISP_VEC_NELEMS*sizeof(short)) == 0);
-
-	for (type = 0; type < IA_CSS_DVS_NUM_COEF_TYPES; type++) {
-		fill_row(&private[type*stride], &public[type*width], width, padding);
-	}
-}
-
-void ia_css_sdis_vertcoef_vmem_encode (
-	struct sh_css_isp_sdis_vert_coef_tbl *to,
-	const struct ia_css_dvs_coefficients *from,
-	unsigned size)
-{
-	unsigned aligned_height = from->grid.aligned_height * from->grid.bqs_per_grid_cell;
-	unsigned height         = from->grid.num_ver_coefs;
-	int      padding        = aligned_height-height;
-	unsigned stride         = size/IA_CSS_DVS_NUM_COEF_TYPES/sizeof(short);
-	unsigned total_bytes    = aligned_height*IA_CSS_DVS_NUM_COEF_TYPES*sizeof(short);
-	short   *public         = from->ver_coefs;
-	short   *private        = (short*)to;
-	unsigned type;
-
-	/* Copy the table, add padding */
-	assert(padding >= 0);
-	assert(total_bytes <= size);
-	assert(size % (IA_CSS_DVS_NUM_COEF_TYPES*ISP_VEC_NELEMS*sizeof(short)) == 0);
-
-	for (type = 0; type < IA_CSS_DVS_NUM_COEF_TYPES; type++) {
-		fill_row(&private[type*stride], &public[type*height], height, padding);
-	}
-}
-
-void ia_css_sdis_horiproj_encode (
-	struct sh_css_isp_sdis_hori_proj_tbl *to,
-	const struct ia_css_dvs_coefficients *from,
-	unsigned size)
-{
-	(void)to;
-	(void)from;
-	(void)size;
-}
-
-void ia_css_sdis_vertproj_encode (
-	struct sh_css_isp_sdis_vert_proj_tbl *to,
-	const struct ia_css_dvs_coefficients *from,
-	unsigned size)
-{
-	(void)to;
-	(void)from;
-	(void)size;
-}
-
-void ia_css_get_isp_dis_coefficients(
-	struct ia_css_stream *stream,
-	short *horizontal_coefficients,
-	short *vertical_coefficients)
-{
-	struct ia_css_isp_parameters *params;
-	unsigned int hor_num_isp, ver_num_isp;
-	unsigned int hor_num_3a,  ver_num_3a;
-	int i;
-	struct ia_css_binary *dvs_binary;
-
-	IA_CSS_ENTER("void");
-
-	assert(horizontal_coefficients != NULL);
-	assert(vertical_coefficients != NULL);
-
-	params = stream->isp_params_configs;
-
-	/* Only video pipe supports DVS */
-	dvs_binary = ia_css_stream_get_dvs_binary(stream);
-	if (!dvs_binary)
-		return;
-
-	hor_num_isp = dvs_binary->dis.coef.pad.width;
-	ver_num_isp = dvs_binary->dis.coef.pad.height;
-	hor_num_3a  = dvs_binary->dis.coef.dim.width;
-	ver_num_3a  = dvs_binary->dis.coef.dim.height;
-
-	for (i = 0; i < IA_CSS_DVS_NUM_COEF_TYPES; i++) {
-		fill_row(&horizontal_coefficients[i*hor_num_isp],
-			 &params->dvs_coefs.hor_coefs[i*hor_num_3a], hor_num_3a, hor_num_isp-hor_num_3a);
-	}
-	for (i = 0; i < SH_CSS_DIS_VER_NUM_COEF_TYPES(dvs_binary); i++) {
-		fill_row(&vertical_coefficients[i*ver_num_isp],
-			 &params->dvs_coefs.ver_coefs[i*ver_num_3a], ver_num_3a, ver_num_isp-ver_num_3a);
-	}
-
-	IA_CSS_LEAVE("void");
-}
-
-size_t
-ia_css_sdis_hor_coef_tbl_bytes(
-	const struct ia_css_binary *binary)
-{
-	if (binary->info->sp.pipeline.isp_pipe_version == 1)
-		return sizeof(short) * IA_CSS_DVS_NUM_COEF_TYPES  * binary->dis.coef.pad.width;
-	else
-		return sizeof(short) * IA_CSS_DVS2_NUM_COEF_TYPES * binary->dis.coef.pad.width;
-}
-
-size_t
-ia_css_sdis_ver_coef_tbl_bytes(
-	const struct ia_css_binary *binary)
-{
-	return sizeof(short) * SH_CSS_DIS_VER_NUM_COEF_TYPES(binary) * binary->dis.coef.pad.height;
-}
-
-void
-ia_css_sdis_init_info(
-	struct ia_css_sdis_info *dis,
-	unsigned sc_3a_dis_width,
-	unsigned sc_3a_dis_padded_width,
-	unsigned sc_3a_dis_height,
-	unsigned isp_pipe_version,
-	unsigned enabled)
-{
-	if (!enabled) {
-		*dis = (struct ia_css_sdis_info) { };
-		return;
-	}
-
-	dis->deci_factor_log2 = SH_CSS_DIS_DECI_FACTOR_LOG2;
-
-	dis->grid.dim.width  =
-			_ISP_BQS(sc_3a_dis_width) >> SH_CSS_DIS_DECI_FACTOR_LOG2;
-	dis->grid.dim.height =
-			_ISP_BQS(sc_3a_dis_height) >> SH_CSS_DIS_DECI_FACTOR_LOG2;
-	dis->grid.pad.width  =
-			CEIL_SHIFT(_ISP_BQS(sc_3a_dis_padded_width), SH_CSS_DIS_DECI_FACTOR_LOG2);
-	dis->grid.pad.height =
-			CEIL_SHIFT(_ISP_BQS(sc_3a_dis_height), SH_CSS_DIS_DECI_FACTOR_LOG2);
-
-	dis->coef.dim.width  =
-			(_ISP_BQS(sc_3a_dis_width)  >> SH_CSS_DIS_DECI_FACTOR_LOG2) << SH_CSS_DIS_DECI_FACTOR_LOG2;
-	dis->coef.dim.height =
-			(_ISP_BQS(sc_3a_dis_height) >> SH_CSS_DIS_DECI_FACTOR_LOG2) << SH_CSS_DIS_DECI_FACTOR_LOG2;
-	dis->coef.pad.width  =
-			__ISP_SDIS_HOR_COEF_NUM_VECS(sc_3a_dis_padded_width) * ISP_VEC_NELEMS;
-	dis->coef.pad.height =
-			__ISP_SDIS_VER_COEF_NUM_VECS(sc_3a_dis_height) * ISP_VEC_NELEMS;
-	if (isp_pipe_version == 1) {
-		dis->proj.dim.width  =
-			_ISP_BQS(sc_3a_dis_height) >> SH_CSS_DIS_DECI_FACTOR_LOG2;
-		dis->proj.dim.height =
-			_ISP_BQS(sc_3a_dis_width)  >> SH_CSS_DIS_DECI_FACTOR_LOG2;
-	} else {
-		dis->proj.dim.width  =
-			(_ISP_BQS(sc_3a_dis_width)  >> SH_CSS_DIS_DECI_FACTOR_LOG2) *
-			(_ISP_BQS(sc_3a_dis_height) >> SH_CSS_DIS_DECI_FACTOR_LOG2);
-		dis->proj.dim.height =
-			(_ISP_BQS(sc_3a_dis_width)  >> SH_CSS_DIS_DECI_FACTOR_LOG2) *
-			(_ISP_BQS(sc_3a_dis_height) >> SH_CSS_DIS_DECI_FACTOR_LOG2);
-	}
-	dis->proj.pad.width  =
-			__ISP_SDIS_HOR_PROJ_NUM_ISP(sc_3a_dis_padded_width,
-				sc_3a_dis_height,
-				SH_CSS_DIS_DECI_FACTOR_LOG2,
-				isp_pipe_version);
-	dis->proj.pad.height =
-			__ISP_SDIS_VER_PROJ_NUM_ISP(sc_3a_dis_padded_width,
-				SH_CSS_DIS_DECI_FACTOR_LOG2);
-}
-
-void ia_css_sdis_clear_coefficients(
-	struct ia_css_dvs_coefficients *dvs_coefs)
-{
-	dvs_coefs->hor_coefs = NULL;
-	dvs_coefs->ver_coefs = NULL;
-}
-
-enum ia_css_err
-ia_css_get_dvs_statistics(
-	struct ia_css_dvs_statistics	       *host_stats,
-	const struct ia_css_isp_dvs_statistics *isp_stats)
-{
-	struct ia_css_isp_dvs_statistics_map *map;
-	enum ia_css_err ret = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER("host_stats=%p, isp_stats=%p", host_stats, isp_stats);
-
-	assert(host_stats != NULL);
-	assert(isp_stats != NULL);
-
-	map = ia_css_isp_dvs_statistics_map_allocate(isp_stats, NULL);
-	if (map) {
-		mmgr_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size);
-		ia_css_translate_dvs_statistics(host_stats, map);
-		ia_css_isp_dvs_statistics_map_free(map);
-	} else {
-		IA_CSS_ERROR("out of memory");
-		ret = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	}
-
-	IA_CSS_LEAVE_ERR(ret);
-	return ret;
-}
-
-void
-ia_css_translate_dvs_statistics(
-	struct ia_css_dvs_statistics               *host_stats,
-	const struct ia_css_isp_dvs_statistics_map *isp_stats)
-{
-	unsigned int hor_num_isp, ver_num_isp, hor_num_dvs, ver_num_dvs, i;
-	int32_t *hor_ptr_dvs, *ver_ptr_dvs, *hor_ptr_isp, *ver_ptr_isp;
-
-	assert(host_stats != NULL);
-	assert(host_stats->hor_proj != NULL);
-	assert(host_stats->ver_proj != NULL);
-	assert(isp_stats != NULL);
-	assert(isp_stats->hor_proj != NULL);
-	assert(isp_stats->ver_proj != NULL);
-
-	IA_CSS_ENTER("hproj=%p, vproj=%p, haddr=%p, vaddr=%p",
-		     host_stats->hor_proj, host_stats->ver_proj,
-		     isp_stats->hor_proj, isp_stats->ver_proj);
-
-	hor_num_isp = host_stats->grid.aligned_height;
-	ver_num_isp = host_stats->grid.aligned_width;
-	hor_ptr_isp = isp_stats->hor_proj;
-	ver_ptr_isp = isp_stats->ver_proj;
-	hor_num_dvs = host_stats->grid.height;
-	ver_num_dvs = host_stats->grid.width;
-	hor_ptr_dvs = host_stats->hor_proj;
-	ver_ptr_dvs = host_stats->ver_proj;
-
-	for (i = 0; i < IA_CSS_DVS_NUM_COEF_TYPES; i++) {
-		memcpy(hor_ptr_dvs, hor_ptr_isp, hor_num_dvs * sizeof(int32_t));
-		hor_ptr_isp += hor_num_isp;
-		hor_ptr_dvs += hor_num_dvs;
-
-		memcpy(ver_ptr_dvs, ver_ptr_isp, ver_num_dvs * sizeof(int32_t));
-		ver_ptr_isp += ver_num_isp;
-		ver_ptr_dvs += ver_num_dvs;
-	}
-
-	IA_CSS_LEAVE("void");
-}
-
-struct ia_css_isp_dvs_statistics *
-ia_css_isp_dvs_statistics_allocate(
-	const struct ia_css_dvs_grid_info *grid)
-{
-	struct ia_css_isp_dvs_statistics *me;
-	int hor_size, ver_size;
-
-	assert(grid != NULL);
-
-	IA_CSS_ENTER("grid=%p", grid);
-
-	if (!grid->enable)
-		return NULL;
-
-	me = sh_css_calloc(1,sizeof(*me));
-	if (!me)
-		goto err;
-
-	hor_size = CEIL_MUL(sizeof(int) * IA_CSS_DVS_NUM_COEF_TYPES * grid->aligned_height,
-			    HIVE_ISP_DDR_WORD_BYTES);
-	ver_size = CEIL_MUL(sizeof(int) * IA_CSS_DVS_NUM_COEF_TYPES * grid->aligned_width,
-			    HIVE_ISP_DDR_WORD_BYTES);
-
-
-	me->size = hor_size + ver_size;
-	me->data_ptr = mmgr_malloc(me->size);
-	if (me->data_ptr == mmgr_NULL)
-		goto err;
-	me->hor_size = hor_size;
-	me->hor_proj = me->data_ptr;
-	me->ver_size = ver_size;
-	me->ver_proj = me->data_ptr + hor_size;
-
-	IA_CSS_LEAVE("return=%p", me);
-
-	return me;
-err:
-	ia_css_isp_dvs_statistics_free(me);
-
-	IA_CSS_LEAVE("return=%p", NULL);
-
-	return NULL;
-}
-
-struct ia_css_isp_dvs_statistics_map *
-ia_css_isp_dvs_statistics_map_allocate(
-	const struct ia_css_isp_dvs_statistics *isp_stats,
-	void *data_ptr)
-{
-	struct ia_css_isp_dvs_statistics_map *me;
-	/* Windows compiler does not like adding sizes to a void *
-	 * so we use a local char * instead. */
-	char *base_ptr;
-
-	me = sh_css_malloc(sizeof(*me));
-	if (!me) {
-		IA_CSS_LOG("cannot allocate memory");
-		goto err;
-	}
-
-	me->data_ptr = data_ptr;
-	me->data_allocated = data_ptr == NULL;
-
-	if (!me->data_ptr) {
-		me->data_ptr = sh_css_malloc(isp_stats->size);
-		if (!me->data_ptr) {
-			IA_CSS_LOG("cannot allocate memory");
-			goto err;
-		}
-	}
-	base_ptr = me->data_ptr;
-
-	me->size = isp_stats->size;
-	/* GCC complains when we assign a char * to a void *, so these
-	 * casts are necessary unfortunately. */
-	me->hor_proj = (void*)base_ptr;
-	me->ver_proj = (void*)(base_ptr + isp_stats->hor_size);
-
-	return me;
-err:
-	if (me)
-		sh_css_free(me);
-	return NULL;
-}
-
-void
-ia_css_isp_dvs_statistics_map_free(struct ia_css_isp_dvs_statistics_map *me)
-{
-	if (me) {
-		if (me->data_allocated)
-			sh_css_free(me->data_ptr);
-		sh_css_free(me);
-	}
-}
-
-void
-ia_css_isp_dvs_statistics_free(struct ia_css_isp_dvs_statistics *me)
-{
-	if (me != NULL) {
-		hmm_free(me->data_ptr);
-		sh_css_free(me);
-	}
-}
-
-void ia_css_sdis_horicoef_debug_dtrace(
-	const struct ia_css_dvs_coefficients *config, unsigned level)
-{
-	(void)config;
-	(void)level;
-}
-
-void ia_css_sdis_vertcoef_debug_dtrace(
-	const struct ia_css_dvs_coefficients *config, unsigned level)
-{
-	(void)config;
-	(void)level;
-}
-
-void ia_css_sdis_horiproj_debug_dtrace(
-	const struct ia_css_dvs_coefficients *config, unsigned level)
-{
-	(void)config;
-	(void)level;
-}
-
-void ia_css_sdis_vertproj_debug_dtrace(
-	const struct ia_css_dvs_coefficients *config, unsigned level)
-{
-	(void)config;
-	(void)level;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h
deleted file mode 100644
index 95e2c61bbcba..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_SDIS_HOST_H
-#define __IA_CSS_SDIS_HOST_H
-
-#include "ia_css_sdis_types.h"
-#include "ia_css_binary.h"
-#include "ia_css_stream.h"
-#include "sh_css_params.h"
-
-extern const struct ia_css_dvs_coefficients default_sdis_config;
-
-/* Opaque here, since size is binary dependent. */
-struct sh_css_isp_sdis_hori_coef_tbl;
-struct sh_css_isp_sdis_vert_coef_tbl;
-struct sh_css_isp_sdis_hori_proj_tbl;
-struct sh_css_isp_sdis_vert_proj_tbl;
-
-void ia_css_sdis_horicoef_vmem_encode (
-	struct sh_css_isp_sdis_hori_coef_tbl *to,
-	const struct ia_css_dvs_coefficients *from,
-	unsigned size);
-
-void ia_css_sdis_vertcoef_vmem_encode (
-	struct sh_css_isp_sdis_vert_coef_tbl *to,
-	const struct ia_css_dvs_coefficients *from,
-	unsigned size);
-
-void ia_css_sdis_horiproj_encode (
-	struct sh_css_isp_sdis_hori_proj_tbl *to,
-	const struct ia_css_dvs_coefficients *from,
-	unsigned size);
-
-void ia_css_sdis_vertproj_encode (
-	struct sh_css_isp_sdis_vert_proj_tbl *to,
-	const struct ia_css_dvs_coefficients *from,
-	unsigned size);
-
-void ia_css_get_isp_dis_coefficients(
-	struct ia_css_stream *stream,
-	short *horizontal_coefficients,
-	short *vertical_coefficients);
-
-enum ia_css_err
-ia_css_get_dvs_statistics(
-	struct ia_css_dvs_statistics	       *host_stats,
-	const struct ia_css_isp_dvs_statistics *isp_stats);
-
-void
-ia_css_translate_dvs_statistics(
-		struct ia_css_dvs_statistics               *host_stats,
-		const struct ia_css_isp_dvs_statistics_map *isp_stats);
-
-struct ia_css_isp_dvs_statistics *
-ia_css_isp_dvs_statistics_allocate(
-	const struct ia_css_dvs_grid_info *grid);
-
-void
-ia_css_isp_dvs_statistics_free(
-	struct ia_css_isp_dvs_statistics *me);
-
-size_t ia_css_sdis_hor_coef_tbl_bytes(const struct ia_css_binary *binary);
-size_t ia_css_sdis_ver_coef_tbl_bytes(const struct ia_css_binary *binary);
-
-void
-ia_css_sdis_init_info(
-	struct ia_css_sdis_info *dis,
-	unsigned sc_3a_dis_width,
-	unsigned sc_3a_dis_padded_width,
-	unsigned sc_3a_dis_height,
-	unsigned isp_pipe_version,
-	unsigned enabled);
-
-void ia_css_sdis_clear_coefficients(
-	struct ia_css_dvs_coefficients *dvs_coefs);
-
-void ia_css_sdis_horicoef_debug_dtrace(
-	const struct ia_css_dvs_coefficients *config, unsigned level);
-
-void ia_css_sdis_vertcoef_debug_dtrace(
-	const struct ia_css_dvs_coefficients *config, unsigned level);
-
-void ia_css_sdis_horiproj_debug_dtrace(
-	const struct ia_css_dvs_coefficients *config, unsigned level);
-
-void ia_css_sdis_vertproj_debug_dtrace(
-	const struct ia_css_dvs_coefficients *config, unsigned level);
-
-#endif /* __IA_CSS_SDIS_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h
deleted file mode 100644
index d2ee57008fb6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_SDIS_TYPES_H
-#define __IA_CSS_SDIS_TYPES_H
-
-/* @file
-* CSS-API header file for DVS statistics parameters.
-*/
-
-/* Number of DVS coefficient types */
-#define IA_CSS_DVS_NUM_COEF_TYPES      6
-
-#ifndef PIPE_GENERATION
-#include "isp/kernels/sdis/common/ia_css_sdis_common_types.h"
-#endif
-
-/* DVS 1.0 Coefficients.
- *  This structure describes the coefficients that are needed for the dvs statistics.
- */
-
-struct ia_css_dvs_coefficients {
-	struct ia_css_dvs_grid_info grid;/** grid info contains the dimensions of the dvs grid */
-	int16_t *hor_coefs;	/** the pointer to int16_t[grid.num_hor_coefs * IA_CSS_DVS_NUM_COEF_TYPES]
-				     containing the horizontal coefficients */
-	int16_t *ver_coefs;	/** the pointer to int16_t[grid.num_ver_coefs * IA_CSS_DVS_NUM_COEF_TYPES]
-				     containing the vertical coefficients */
-};
-
-/* DVS 1.0 Statistics.
- *  This structure describes the statistics that are generated using the provided coefficients.
- */
-
-struct ia_css_dvs_statistics {
-	struct ia_css_dvs_grid_info grid;/** grid info contains the dimensions of the dvs grid */
-	int32_t *hor_proj;	/** the pointer to int16_t[grid.height * IA_CSS_DVS_NUM_COEF_TYPES]
-				     containing the horizontal projections */
-	int32_t *ver_proj;	/** the pointer to int16_t[grid.width * IA_CSS_DVS_NUM_COEF_TYPES]
-				     containing the vertical projections */
-};
-
-#endif /* __IA_CSS_SDIS_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c
deleted file mode 100644
index 9bccb6473154..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c
+++ /dev/null
@@ -1,338 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <assert_support.h>
-#include "memory_access.h"
-#include "ia_css_debug.h"
-#include "ia_css_sdis2.host.h"
-
-const struct ia_css_dvs2_coefficients default_sdis2_config = {
-	.grid = { 0, 0, 0, 0, 0, 0, 0, 0 },
-	.hor_coefs = { NULL, NULL, NULL, NULL },
-	.ver_coefs = { NULL, NULL, NULL, NULL },
-};
-
-static void
-fill_row(short *private, const short *public, unsigned width, unsigned padding)
-{
-	memcpy (private, public, width*sizeof(short));
-	memset (&private[width], 0, padding*sizeof(short));
-}
-
-void ia_css_sdis2_horicoef_vmem_encode (
-	struct sh_css_isp_sdis_hori_coef_tbl *to,
-	const struct ia_css_dvs2_coefficients *from,
-	unsigned size)
-{
-	unsigned aligned_width = from->grid.aligned_width * from->grid.bqs_per_grid_cell;
-	unsigned width         = from->grid.num_hor_coefs;
-	int      padding       = aligned_width-width;
-	unsigned stride        = size/IA_CSS_DVS2_NUM_COEF_TYPES/sizeof(short);
-	unsigned total_bytes   = aligned_width*IA_CSS_DVS2_NUM_COEF_TYPES*sizeof(short);
-	short   *private       = (short*)to;
-
-
-	/* Copy the table, add padding */
-	assert(padding >= 0);
-	assert(total_bytes <= size);
-	assert(size % (IA_CSS_DVS2_NUM_COEF_TYPES*ISP_VEC_NELEMS*sizeof(short)) == 0);
-	fill_row(&private[0*stride], from->hor_coefs.odd_real,  width, padding);
-	fill_row(&private[1*stride], from->hor_coefs.odd_imag,  width, padding);
-	fill_row(&private[2*stride], from->hor_coefs.even_real, width, padding);
-	fill_row(&private[3*stride], from->hor_coefs.even_imag, width, padding);
-}
-
-void ia_css_sdis2_vertcoef_vmem_encode (
-	struct sh_css_isp_sdis_vert_coef_tbl *to,
-	const struct ia_css_dvs2_coefficients *from,
-	unsigned size)
-{
-	unsigned aligned_height = from->grid.aligned_height * from->grid.bqs_per_grid_cell;
-	unsigned height         = from->grid.num_ver_coefs;
-	int      padding        = aligned_height-height;
-	unsigned stride         = size/IA_CSS_DVS2_NUM_COEF_TYPES/sizeof(short);
-	unsigned total_bytes    = aligned_height*IA_CSS_DVS2_NUM_COEF_TYPES*sizeof(short);
-	short   *private        = (short*)to;
-
-	/* Copy the table, add padding */
-	assert(padding >= 0);
-	assert(total_bytes <= size);
-	assert(size % (IA_CSS_DVS2_NUM_COEF_TYPES*ISP_VEC_NELEMS*sizeof(short)) == 0);
-	fill_row(&private[0*stride], from->ver_coefs.odd_real,  height, padding);
-	fill_row(&private[1*stride], from->ver_coefs.odd_imag,  height, padding);
-	fill_row(&private[2*stride], from->ver_coefs.even_real, height, padding);
-	fill_row(&private[3*stride], from->ver_coefs.even_imag, height, padding);
-}
-
-void ia_css_sdis2_horiproj_encode (
-	struct sh_css_isp_sdis_hori_proj_tbl *to,
-	const struct ia_css_dvs2_coefficients *from,
-	unsigned size)
-{
-	(void)to;
-	(void)from;
-	(void)size;
-}
-
-void ia_css_sdis2_vertproj_encode (
-	struct sh_css_isp_sdis_vert_proj_tbl *to,
-	const struct ia_css_dvs2_coefficients *from,
-	unsigned size)
-{
-	(void)to;
-	(void)from;
-	(void)size;
-}
-
-void ia_css_get_isp_dvs2_coefficients(
-	struct ia_css_stream *stream,
-	short *hor_coefs_odd_real,
-	short *hor_coefs_odd_imag,
-	short *hor_coefs_even_real,
-	short *hor_coefs_even_imag,
-	short *ver_coefs_odd_real,
-	short *ver_coefs_odd_imag,
-	short *ver_coefs_even_real,
-	short *ver_coefs_even_imag)
-{
-	struct ia_css_isp_parameters *params;
-	unsigned int hor_num_3a, ver_num_3a;
-	unsigned int hor_num_isp, ver_num_isp;
-	struct ia_css_binary *dvs_binary;
-
-	IA_CSS_ENTER("void");
-
-	assert(stream != NULL);
-	assert(hor_coefs_odd_real  != NULL);
-	assert(hor_coefs_odd_imag  != NULL);
-	assert(hor_coefs_even_real != NULL);
-	assert(hor_coefs_even_imag != NULL);
-	assert(ver_coefs_odd_real  != NULL);
-	assert(ver_coefs_odd_imag  != NULL);
-	assert(ver_coefs_even_real != NULL);
-	assert(ver_coefs_even_imag != NULL);
-
-	params = stream->isp_params_configs;
-
-	/* Only video pipe supports DVS */
-	dvs_binary = ia_css_stream_get_dvs_binary(stream);
-	if (!dvs_binary)
-		return;
-
-	hor_num_3a  = dvs_binary->dis.coef.dim.width;
-	ver_num_3a  = dvs_binary->dis.coef.dim.height;
-	hor_num_isp = dvs_binary->dis.coef.pad.width;
-	ver_num_isp = dvs_binary->dis.coef.pad.height;
-
-	memcpy (hor_coefs_odd_real,  params->dvs2_coefs.hor_coefs.odd_real,  hor_num_3a * sizeof(short));
-	memcpy (hor_coefs_odd_imag,  params->dvs2_coefs.hor_coefs.odd_imag,  hor_num_3a * sizeof(short));
-	memcpy (hor_coefs_even_real, params->dvs2_coefs.hor_coefs.even_real, hor_num_3a * sizeof(short));
-	memcpy (hor_coefs_even_imag, params->dvs2_coefs.hor_coefs.even_imag, hor_num_3a * sizeof(short));
-	memcpy (ver_coefs_odd_real,  params->dvs2_coefs.ver_coefs.odd_real,  ver_num_3a * sizeof(short));
-	memcpy (ver_coefs_odd_imag,  params->dvs2_coefs.ver_coefs.odd_imag,  ver_num_3a * sizeof(short));
-	memcpy (ver_coefs_even_real, params->dvs2_coefs.ver_coefs.even_real, ver_num_3a * sizeof(short));
-	memcpy (ver_coefs_even_imag, params->dvs2_coefs.ver_coefs.even_imag, ver_num_3a * sizeof(short));
-
-	IA_CSS_LEAVE("void");
-}
-
-void ia_css_sdis2_clear_coefficients(
-	struct ia_css_dvs2_coefficients *dvs2_coefs)
-{
-	dvs2_coefs->hor_coefs.odd_real  = NULL;
-	dvs2_coefs->hor_coefs.odd_imag  = NULL;
-	dvs2_coefs->hor_coefs.even_real = NULL;
-	dvs2_coefs->hor_coefs.even_imag = NULL;
-	dvs2_coefs->ver_coefs.odd_real  = NULL;
-	dvs2_coefs->ver_coefs.odd_imag  = NULL;
-	dvs2_coefs->ver_coefs.even_real = NULL;
-	dvs2_coefs->ver_coefs.even_imag = NULL;
-}
-
-enum ia_css_err
-ia_css_get_dvs2_statistics(
-	struct ia_css_dvs2_statistics          *host_stats,
-	const struct ia_css_isp_dvs_statistics *isp_stats)
-{
-	struct ia_css_isp_dvs_statistics_map *map;
-	enum ia_css_err ret = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER("host_stats=%p, isp_stats=%p", host_stats, isp_stats);
-
-	assert(host_stats != NULL);
-	assert(isp_stats != NULL);
-
-	map = ia_css_isp_dvs_statistics_map_allocate(isp_stats, NULL);
-	if (map) {
-		mmgr_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size);
-		ia_css_translate_dvs2_statistics(host_stats, map);
-		ia_css_isp_dvs_statistics_map_free(map);
-	} else {
-		IA_CSS_ERROR("out of memory");
-		ret = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	}
-
-	IA_CSS_LEAVE_ERR(ret);
-	return ret;
-}
-
-void
-ia_css_translate_dvs2_statistics(
-	struct ia_css_dvs2_statistics		   *host_stats,
-	const struct ia_css_isp_dvs_statistics_map *isp_stats)
-{
-	unsigned int size_bytes, table_width, table_size, height;
-	unsigned int src_offset = 0, dst_offset = 0;
-	int32_t *htemp_ptr, *vtemp_ptr;
-
-	assert(host_stats != NULL);
-	assert(host_stats->hor_prod.odd_real  != NULL);
-	assert(host_stats->hor_prod.odd_imag  != NULL);
-	assert(host_stats->hor_prod.even_real != NULL);
-	assert(host_stats->hor_prod.even_imag != NULL);
-	assert(host_stats->ver_prod.odd_real  != NULL);
-	assert(host_stats->ver_prod.odd_imag  != NULL);
-	assert(host_stats->ver_prod.even_real != NULL);
-	assert(host_stats->ver_prod.even_imag != NULL);
-	assert(isp_stats != NULL);
-	assert(isp_stats->hor_proj != NULL);
-	assert(isp_stats->ver_proj != NULL);
-
-	IA_CSS_ENTER("hor_coefs.odd_real=%p, hor_coefs.odd_imag=%p, "
-		     "hor_coefs.even_real=%p, hor_coefs.even_imag=%p, "
-		     "ver_coefs.odd_real=%p, ver_coefs.odd_imag=%p, "
-		     "ver_coefs.even_real=%p, ver_coefs.even_imag=%p, "
-		     "haddr=%p, vaddr=%p",
-		host_stats->hor_prod.odd_real, host_stats->hor_prod.odd_imag,
-		host_stats->hor_prod.even_real, host_stats->hor_prod.even_imag,
-		host_stats->ver_prod.odd_real, host_stats->ver_prod.odd_imag,
-		host_stats->ver_prod.even_real, host_stats->ver_prod.even_imag,
-		isp_stats->hor_proj, isp_stats->ver_proj);
-
-	/* Host side: reflecting the true width in bytes */
-	size_bytes = host_stats->grid.aligned_width * sizeof(*htemp_ptr);
-
-	/* DDR side: need to be aligned to the system bus width */
-	/* statistics table width in terms of 32-bit words*/
-	table_width = CEIL_MUL(size_bytes, HIVE_ISP_DDR_WORD_BYTES) / sizeof(*htemp_ptr);
-	table_size = table_width * host_stats->grid.aligned_height;
-
-	htemp_ptr = isp_stats->hor_proj; /* horizontal stats */
-	vtemp_ptr = isp_stats->ver_proj; /* vertical stats */
-	for (height = 0; height < host_stats->grid.aligned_height; height++) {
-		/* hor stats */
-		memcpy(host_stats->hor_prod.odd_real + dst_offset,
-			&htemp_ptr[0*table_size+src_offset], size_bytes);
-		memcpy(host_stats->hor_prod.odd_imag + dst_offset,
-			&htemp_ptr[1*table_size+src_offset], size_bytes);
-		memcpy(host_stats->hor_prod.even_real + dst_offset,
-			&htemp_ptr[2*table_size+src_offset], size_bytes);
-		memcpy(host_stats->hor_prod.even_imag + dst_offset,
-			&htemp_ptr[3*table_size+src_offset], size_bytes);
-
-		/* ver stats */
-		memcpy(host_stats->ver_prod.odd_real + dst_offset,
-			&vtemp_ptr[0*table_size+src_offset], size_bytes);
-		memcpy(host_stats->ver_prod.odd_imag + dst_offset,
-			&vtemp_ptr[1*table_size+src_offset], size_bytes);
-		memcpy(host_stats->ver_prod.even_real + dst_offset,
-			&vtemp_ptr[2*table_size+src_offset], size_bytes);
-		memcpy(host_stats->ver_prod.even_imag + dst_offset,
-			&vtemp_ptr[3*table_size+src_offset], size_bytes);
-
-		src_offset += table_width; /* aligned table width */
-		dst_offset += host_stats->grid.aligned_width;
-	}
-
-	IA_CSS_LEAVE("void");
-}
-
-struct ia_css_isp_dvs_statistics *
-ia_css_isp_dvs2_statistics_allocate(
-	const struct ia_css_dvs_grid_info *grid)
-{
-	struct ia_css_isp_dvs_statistics *me;
-	int size;
-
-	assert(grid != NULL);
-
-	IA_CSS_ENTER("grid=%p", grid);
-
-	if (!grid->enable)
-		return NULL;
-
-	me = sh_css_calloc(1,sizeof(*me));
-	if (!me)
-		goto err;
-
-	/* on ISP 2 SDIS DMA model, every row of projection table width must be
-	   aligned to HIVE_ISP_DDR_WORD_BYTES
-	*/
-	size = CEIL_MUL(sizeof(int) * grid->aligned_width, HIVE_ISP_DDR_WORD_BYTES)
-		* grid->aligned_height * IA_CSS_DVS2_NUM_COEF_TYPES;
-
-	me->size = 2*size;
-	me->data_ptr = mmgr_malloc(me->size);
-	if (me->data_ptr == mmgr_NULL)
-		goto err;
-	me->hor_proj = me->data_ptr;
-	me->hor_size = size;
-	me->ver_proj = me->data_ptr + size;
-	me->ver_size = size;
-
-	IA_CSS_LEAVE("return=%p", me);
-	return me;
-err:
-	ia_css_isp_dvs2_statistics_free(me);
-	IA_CSS_LEAVE("return=%p", NULL);
-
-	return NULL;
-}
-
-void
-ia_css_isp_dvs2_statistics_free(struct ia_css_isp_dvs_statistics *me)
-{
-	if (me != NULL) {
-		hmm_free(me->data_ptr);
-		sh_css_free(me);
-	}
-}
-
-void ia_css_sdis2_horicoef_debug_dtrace(
-	const struct ia_css_dvs2_coefficients *config, unsigned level)
-{
-	(void)config;
-	(void)level;
-}
-
-void ia_css_sdis2_vertcoef_debug_dtrace(
-	const struct ia_css_dvs2_coefficients *config, unsigned level)
-{
-	(void)config;
-	(void)level;
-}
-
-void ia_css_sdis2_horiproj_debug_dtrace(
-	const struct ia_css_dvs2_coefficients *config, unsigned level)
-{
-	(void)config;
-	(void)level;
-}
-
-void ia_css_sdis2_vertproj_debug_dtrace(
-	const struct ia_css_dvs2_coefficients *config, unsigned level)
-{
-	(void)config;
-	(void)level;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h
deleted file mode 100644
index 60198d4279b4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_SDIS2_HOST_H
-#define __IA_CSS_SDIS2_HOST_H
-
-#include "ia_css_sdis2_types.h"
-#include "ia_css_binary.h"
-#include "ia_css_stream.h"
-#include "sh_css_params.h"
-
-extern const struct ia_css_dvs2_coefficients default_sdis2_config;
-
-/* Opaque here, since size is binary dependent. */
-struct sh_css_isp_sdis_hori_coef_tbl;
-struct sh_css_isp_sdis_vert_coef_tbl;
-struct sh_css_isp_sdis_hori_proj_tbl;
-struct sh_css_isp_sdis_vert_proj_tbl;
-
-void ia_css_sdis2_horicoef_vmem_encode (
-	struct sh_css_isp_sdis_hori_coef_tbl *to,
-	const struct ia_css_dvs2_coefficients *from,
-	unsigned size);
-
-void ia_css_sdis2_vertcoef_vmem_encode (
-	struct sh_css_isp_sdis_vert_coef_tbl *to,
-	const struct ia_css_dvs2_coefficients *from,
-	unsigned size);
-
-void ia_css_sdis2_horiproj_encode (
-	struct sh_css_isp_sdis_hori_proj_tbl *to,
-	const struct ia_css_dvs2_coefficients *from,
-	unsigned size);
-
-void ia_css_sdis2_vertproj_encode (
-	struct sh_css_isp_sdis_vert_proj_tbl *to,
-	const struct ia_css_dvs2_coefficients *from,
-	unsigned size);
-
-void ia_css_get_isp_dvs2_coefficients(
-	struct ia_css_stream *stream,
-	short *hor_coefs_odd_real,
-	short *hor_coefs_odd_imag,
-	short *hor_coefs_even_real,
-	short *hor_coefs_even_imag,
-	short *ver_coefs_odd_real,
-	short *ver_coefs_odd_imag,
-	short *ver_coefs_even_real,
-	short *ver_coefs_even_imag);
-
-void ia_css_sdis2_clear_coefficients(
-	struct ia_css_dvs2_coefficients *dvs2_coefs);
-
-enum ia_css_err
-ia_css_get_dvs2_statistics(
-	struct ia_css_dvs2_statistics	       *host_stats,
-	const struct ia_css_isp_dvs_statistics *isp_stats);
-
-void
-ia_css_translate_dvs2_statistics(
-	struct ia_css_dvs2_statistics              *host_stats,
-	const struct ia_css_isp_dvs_statistics_map *isp_stats);
-
-struct ia_css_isp_dvs_statistics *
-ia_css_isp_dvs2_statistics_allocate(
-	const struct ia_css_dvs_grid_info *grid);
-
-void
-ia_css_isp_dvs2_statistics_free(
-	struct ia_css_isp_dvs_statistics *me);
-
-void ia_css_sdis2_horicoef_debug_dtrace(
-	const struct ia_css_dvs2_coefficients *config, unsigned level);
-
-void ia_css_sdis2_vertcoef_debug_dtrace(
-	const struct ia_css_dvs2_coefficients *config, unsigned level);
-
-void ia_css_sdis2_horiproj_debug_dtrace(
-	const struct ia_css_dvs2_coefficients *config, unsigned level);
-
-void ia_css_sdis2_vertproj_debug_dtrace(
-	const struct ia_css_dvs2_coefficients *config, unsigned level);
-
-#endif /* __IA_CSS_SDIS2_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h
deleted file mode 100644
index 2a0bc4031746..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_SDIS2_TYPES_H
-#define __IA_CSS_SDIS2_TYPES_H
-
-/* @file
-* CSS-API header file for DVS statistics parameters.
-*/
-
-/* Number of DVS coefficient types */
-#define IA_CSS_DVS2_NUM_COEF_TYPES     4
-
-#ifndef PIPE_GENERATION
-#include "isp/kernels/sdis/common/ia_css_sdis_common_types.h"
-#endif
-
-/* DVS 2.0 Coefficient types. This structure contains 4 pointers to
- *  arrays that contain the coeffients for each type.
- */
-struct ia_css_dvs2_coef_types {
-	int16_t *odd_real; /** real part of the odd coefficients*/
-	int16_t *odd_imag; /** imaginary part of the odd coefficients*/
-	int16_t *even_real;/** real part of the even coefficients*/
-	int16_t *even_imag;/** imaginary part of the even coefficients*/
-};
-
-/* DVS 2.0 Coefficients. This structure describes the coefficients that are needed for the dvs statistics.
- *  e.g. hor_coefs.odd_real is the pointer to int16_t[grid.num_hor_coefs] containing the horizontal odd real 
- *  coefficients.
- */
-struct ia_css_dvs2_coefficients {
-	struct ia_css_dvs_grid_info grid;        /** grid info contains the dimensions of the dvs grid */
-	struct ia_css_dvs2_coef_types hor_coefs; /** struct with pointers that contain the horizontal coefficients */
-	struct ia_css_dvs2_coef_types ver_coefs; /** struct with pointers that contain the vertical coefficients */
-};
-
-/* DVS 2.0 Statistic types. This structure contains 4 pointers to
- *  arrays that contain the statistics for each type.
- */
-struct ia_css_dvs2_stat_types {
-	int32_t *odd_real; /** real part of the odd statistics*/
-	int32_t *odd_imag; /** imaginary part of the odd statistics*/
-	int32_t *even_real;/** real part of the even statistics*/
-	int32_t *even_imag;/** imaginary part of the even statistics*/
-};
-
-/* DVS 2.0 Statistics. This structure describes the statistics that are generated using the provided coefficients.
- *  e.g. hor_prod.odd_real is the pointer to int16_t[grid.aligned_height][grid.aligned_width] containing 
- *  the horizontal odd real statistics. Valid statistics data area is int16_t[0..grid.height-1][0..grid.width-1]
- */
-struct ia_css_dvs2_statistics {
-	struct ia_css_dvs_grid_info grid;       /** grid info contains the dimensions of the dvs grid */
-	struct ia_css_dvs2_stat_types hor_prod; /** struct with pointers that contain the horizontal statistics */
-	struct ia_css_dvs2_stat_types ver_prod; /** struct with pointers that contain the vertical statistics */
-};
-
-#endif /* __IA_CSS_SDIS2_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c
deleted file mode 100644
index 78a113bfe8f1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_debug.h"
-#include "ia_css_tdf.host.h"
-
-static const int16_t g_pyramid[8][8] = {
-{128, 384, 640, 896, 896, 640, 384, 128},
-{384, 1152, 1920, 2688, 2688, 1920, 1152, 384},
-{640, 1920, 3200, 4480, 4480, 3200, 1920, 640},
-{896, 2688, 4480, 6272, 6272, 4480, 2688, 896},
-{896, 2688, 4480, 6272, 6272, 4480, 2688, 896},
-{640, 1920, 3200, 4480, 4480, 3200, 1920, 640},
-{384, 1152, 1920, 2688, 2688, 1920, 1152, 384},
-{128, 384, 640, 896, 896, 640, 384, 128}
-};
-
-void
-ia_css_tdf_vmem_encode(
-	struct ia_css_isp_tdf_vmem_params *to,
-	const struct ia_css_tdf_config *from,
-	size_t size)
-{
-	unsigned i;
-	(void)size;
-
-	for (i = 0; i < ISP_VEC_NELEMS; i++) {
-		to->pyramid[0][i]          = g_pyramid[i/8][i%8];
-		to->threshold_flat[0][i]   = from->thres_flat_table[i];
-		to->threshold_detail[0][i] = from->thres_detail_table[i];
-	}
-
-}
-
-void
-ia_css_tdf_encode(
-	struct ia_css_isp_tdf_dmem_params *to,
-	const struct ia_css_tdf_config *from,
-	size_t size)
-{
-	(void)size;
-	to->Epsilon_0        = from->epsilon_0;
-	to->Epsilon_1        = from->epsilon_1;
-	to->EpsScaleText     = from->eps_scale_text;
-	to->EpsScaleEdge     = from->eps_scale_edge;
-	to->Sepa_flat	     = from->sepa_flat;
-	to->Sepa_Edge	     = from->sepa_edge;
-	to->Blend_Flat	     = from->blend_flat;
-	to->Blend_Text	     = from->blend_text;
-	to->Blend_Edge	     = from->blend_edge;
-	to->Shading_Gain     = from->shading_gain;
-	to->Shading_baseGain = from->shading_base_gain;
-	to->LocalY_Gain      = from->local_y_gain;
-	to->LocalY_baseGain  = from->local_y_base_gain;
-}
-
-void
-ia_css_tdf_debug_dtrace(
-	const struct ia_css_tdf_config *config,
-	unsigned level)
-{
-	(void)config;
-	(void)level;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h
deleted file mode 100644
index bd628a18e839..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_TDF_HOST_H
-#define __IA_CSS_TDF_HOST_H
-
-#include "ia_css_tdf_types.h"
-#include "ia_css_tdf_param.h"
-
-void
-ia_css_tdf_vmem_encode(
-	struct ia_css_isp_tdf_vmem_params *to,
-	const struct ia_css_tdf_config *from,
-	size_t size);
-
-void
-ia_css_tdf_encode(
-	struct ia_css_isp_tdf_dmem_params *to,
-	const struct ia_css_tdf_config *from,
-	size_t size);
-
-void
-ia_css_tdf_debug_dtrace(
-	const struct ia_css_tdf_config *config, unsigned level)
-;
-
-#endif /* __IA_CSS_TDF_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h
deleted file mode 100644
index 9334f2e0698b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_TDF_PARAM_H
-#define __IA_CSS_TDF_PARAM_H
-
-#include "type_support.h"
-#include "vmem.h" /* needed for VMEM_ARRAY */
-
-struct ia_css_isp_tdf_vmem_params {
-	VMEM_ARRAY(pyramid, ISP_VEC_NELEMS);
-	VMEM_ARRAY(threshold_flat, ISP_VEC_NELEMS);
-	VMEM_ARRAY(threshold_detail, ISP_VEC_NELEMS);
-};
-
-struct ia_css_isp_tdf_dmem_params {
-	int32_t Epsilon_0;
-	int32_t Epsilon_1;
-	int32_t EpsScaleText;
-	int32_t EpsScaleEdge;
-	int32_t Sepa_flat;
-	int32_t Sepa_Edge;
-	int32_t Blend_Flat;
-	int32_t Blend_Text;
-	int32_t Blend_Edge;
-	int32_t Shading_Gain;
-	int32_t Shading_baseGain;
-	int32_t LocalY_Gain;
-	int32_t LocalY_baseGain;
-};
-
-#endif /* __IA_CSS_TDF_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h
deleted file mode 100644
index 91ea8dd4651d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_TDF_TYPES_H
-#define __IA_CSS_TDF_TYPES_H
-
-/* @file
-* CSS-API header file for Transform Domain Filter parameters.
-*/
-
-#include "type_support.h"
-
-/* Transform Domain Filter configuration
- *
- * \brief TDF public parameters.
- * \details Struct with all parameters for the TDF kernel that can be set
- * from the CSS API.
- *
- * ISP2.6.1: TDF is used.
- */
-struct ia_css_tdf_config {
-	int32_t thres_flat_table[64];	/** Final optimized strength table of NR for flat region. */
-	int32_t thres_detail_table[64];	/** Final optimized strength table of NR for detail region. */
-	int32_t epsilon_0;		/** Coefficient to control variance for dark area (for flat region). */
-	int32_t epsilon_1;		/** Coefficient to control variance for bright area (for flat region). */
-	int32_t eps_scale_text;		/** Epsilon scaling coefficient for texture region. */
-	int32_t eps_scale_edge;		/** Epsilon scaling coefficient for edge region. */
-	int32_t sepa_flat;		/** Threshold to judge flat (edge < m_Flat_thre). */
-	int32_t sepa_edge;		/** Threshold to judge edge (edge > m_Edge_thre). */
-	int32_t blend_flat;		/** Blending ratio at flat region. */
-	int32_t blend_text;		/** Blending ratio at texture region. */
-	int32_t blend_edge;		/** Blending ratio at edge region. */
-	int32_t shading_gain;		/** Gain of Shading control. */
-	int32_t shading_base_gain;	/** Base Gain of Shading control. */
-	int32_t local_y_gain;		/** Gain of local luminance control. */
-	int32_t local_y_base_gain;	/** Base gain of local luminance control. */
-	int32_t rad_x_origin;		/** Initial x coord. for radius computation. */
-	int32_t rad_y_origin;		/** Initial y coord. for radius computation. */
-};
-
-#endif /* __IA_CSS_TDF_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h
deleted file mode 100644
index 223423f8c40b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h
+++ /dev/null
@@ -1,61 +0,0 @@
-#ifdef ISP2401
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#ifndef _IA_CSS_TNR3_TYPES_H
-#define _IA_CSS_TNR3_TYPES_H
-
-/* @file
-* CSS-API header file for Temporal Noise Reduction v3 (TNR3) kernel
-*/
-
-/**
- * \brief Number of piecewise linear segments.
- * \details The parameters to TNR3 are specified as a piecewise linear segment.
- * The number of such segments is fixed at 3.
- */
-#define TNR3_NUM_SEGMENTS    3
-
-/* Temporal Noise Reduction v3 (TNR3) configuration.
- * The parameter to this kernel is fourfold
- * 1. Three piecewise linear graphs (one for each plane) with three segments
- * each. Each line graph has Luma values on the x axis and sigma values for
- * each plane on the y axis. The three linear segments may have a different
- * slope and the point of Luma value which where the slope may change is called
- * a "Knee" point. As there are three such segments, four points need to be
- * specified each on the Luma axis and the per plane Sigma axis. On the Luma
- * axis two points are fixed (namely 0 and maximum luma value - depending on
- * ISP bit depth). The other two points are the points where the slope may
- * change its value. These two points are called knee points. The four points on
- * the per plane sigma axis are also specified at the interface.
- * 2. One rounding adjustment parameter for each plane
- * 3. One maximum feedback threshold value for each plane
- * 4. Selection of the reference frame buffer to be used for noise reduction.
- */
-struct ia_css_tnr3_kernel_config {
-	unsigned int maxfb_y;                        /** Maximum Feedback Gain for Y */
-	unsigned int maxfb_u;                        /** Maximum Feedback Gain for U */
-	unsigned int maxfb_v;                        /** Maximum Feedback Gain for V */
-	unsigned int round_adj_y;                    /** Rounding Adjust for Y */
-	unsigned int round_adj_u;                    /** Rounding Adjust for U */
-	unsigned int round_adj_v;                    /** Rounding Adjust for V */
-	unsigned int knee_y[TNR3_NUM_SEGMENTS - 1];  /** Knee points */
-	unsigned int sigma_y[TNR3_NUM_SEGMENTS + 1]; /** Standard deviation for Y at points Y0, Y1, Y2, Y3 */
-	unsigned int sigma_u[TNR3_NUM_SEGMENTS + 1]; /** Standard deviation for U at points U0, U1, U2, U3 */
-	unsigned int sigma_v[TNR3_NUM_SEGMENTS + 1]; /** Standard deviation for V at points V0, V1, V2, V3 */
-	unsigned int ref_buf_select;                 /** Selection of the reference buffer */
-};
-
-#endif
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c
deleted file mode 100644
index 222a7bd7f176..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "ia_css_frame.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "sh_css_frac.h"
-#include "assert_support.h"
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-#include "isp.h"
-
-#include "ia_css_tnr.host.h"
-const struct ia_css_tnr_config default_tnr_config = {
-	32768,
-	32,
-	32,
-};
-
-void
-ia_css_tnr_encode(
-	struct sh_css_isp_tnr_params *to,
-	const struct ia_css_tnr_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->coef =
-	    uDIGIT_FITTING(from->gain, 16, SH_CSS_TNR_COEF_SHIFT);
-	to->threshold_Y =
-	    uDIGIT_FITTING(from->threshold_y, 16, SH_CSS_ISP_YUV_BITS);
-	to->threshold_C =
-	    uDIGIT_FITTING(from->threshold_uv, 16, SH_CSS_ISP_YUV_BITS);
-}
-
-void
-ia_css_tnr_dump(
-	const struct sh_css_isp_tnr_params *tnr,
-	unsigned level)
-{
-	if (!tnr) return;
-	ia_css_debug_dtrace(level, "Temporal Noise Reduction:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"tnr_coef", tnr->coef);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"tnr_threshold_Y", tnr->threshold_Y);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"tnr_threshold_C", tnr->threshold_C);
-}
-
-void
-ia_css_tnr_debug_dtrace(
-	const struct ia_css_tnr_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.gain=%d, "
-		"config.threshold_y=%d, config.threshold_uv=%d\n",
-		config->gain,
-		config->threshold_y, config->threshold_uv);
-}
-
-void
-ia_css_tnr_config(
-	struct sh_css_isp_tnr_isp_config *to,
-	const struct ia_css_tnr_configuration *from,
-	unsigned size)
-{
-	unsigned elems_a = ISP_VEC_NELEMS;
-	unsigned i;
-
-	(void)size;
-	ia_css_dma_configure_from_info(&to->port_b, &from->tnr_frames[0]->info);
-	to->width_a_over_b = elems_a / to->port_b.elems;
-	to->frame_height = from->tnr_frames[0]->info.res.height;
-#ifndef ISP2401
-	for (i = 0; i < NUM_VIDEO_TNR_FRAMES; i++) {
-#else
-	for (i = 0; i < NUM_TNR_FRAMES; i++) {
-#endif
-		to->tnr_frame_addr[i] = from->tnr_frames[i]->data + from->tnr_frames[i]->planes.yuyv.offset;
-	}
-
-	/* Assume divisiblity here, may need to generalize to fixed point. */
-	assert (elems_a % to->port_b.elems == 0);
-}
-
-void
-ia_css_tnr_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame **frames)
-{
-	struct ia_css_tnr_configuration config;
-	unsigned i;
-
-#ifndef ISP2401
-	for (i = 0; i < NUM_VIDEO_TNR_FRAMES; i++)
-#else
-	for (i = 0; i < NUM_TNR_FRAMES; i++)
-#endif
-		config.tnr_frames[i] = frames[i];
-
-	ia_css_configure_tnr(binary, &config);
-}
-
-void
-ia_css_init_tnr_state(
-	struct sh_css_isp_tnr_dmem_state *state,
-	size_t size)
-{
-	(void)size;
-
-#ifndef ISP2401
-	assert(NUM_VIDEO_TNR_FRAMES >= 2);
-#endif
-	assert(sizeof(*state) == size);
-	state->tnr_in_buf_idx = 0;
-	state->tnr_out_buf_idx = 1;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h
deleted file mode 100644
index 9290dfad574e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_TNR_HOST_H
-#define __IA_CSS_TNR_HOST_H
-
-#include "ia_css_binary.h"
-#include "ia_css_tnr_state.h"
-#include "ia_css_tnr_types.h"
-#include "ia_css_tnr_param.h"
-
-extern const struct ia_css_tnr_config default_tnr_config;
-
-void
-ia_css_tnr_encode(
-	struct sh_css_isp_tnr_params *to,
-	const struct ia_css_tnr_config *from,
-	unsigned size);
-
-void
-ia_css_tnr_dump(
-	const struct sh_css_isp_tnr_params *tnr,
-	unsigned level);
-
-void
-ia_css_tnr_debug_dtrace(
-	const struct ia_css_tnr_config *config,
-	unsigned level);
-
-void
-ia_css_tnr_config(
-	struct sh_css_isp_tnr_isp_config      *to,
-	const struct ia_css_tnr_configuration *from,
-	unsigned size);
-
-void
-ia_css_tnr_configure(
-	const struct ia_css_binary     *binary,
-	const struct ia_css_frame **frames);
-
-void
-ia_css_init_tnr_state(
-	struct sh_css_isp_tnr_dmem_state *state,
-	size_t size);
-#endif /* __IA_CSS_TNR_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h
deleted file mode 100644
index db4a7cced264..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_TNR_PARAM_H
-#define __IA_CSS_TNR_PARAM_H
-
-#include "type_support.h"
-#include "sh_css_defs.h"
-#include "dma.h"
-
-/* TNR (Temporal Noise Reduction) */
-struct sh_css_isp_tnr_params {
-	int32_t coef;
-	int32_t threshold_Y;
-	int32_t threshold_C;
-};
-
-struct ia_css_tnr_configuration {
-#ifndef ISP2401
-	const struct ia_css_frame *tnr_frames[NUM_VIDEO_TNR_FRAMES];
-#else
-	const struct ia_css_frame *tnr_frames[NUM_TNR_FRAMES];
-#endif
-};
-
-struct sh_css_isp_tnr_isp_config {
-	uint32_t width_a_over_b;
-	uint32_t frame_height;
-	struct dma_port_config port_b;
-#ifndef ISP2401
-	hrt_vaddress tnr_frame_addr[NUM_VIDEO_TNR_FRAMES];
-#else
-	hrt_vaddress tnr_frame_addr[NUM_TNR_FRAMES];
-#endif
-};
-
-#endif /* __IA_CSS_TNR_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h
deleted file mode 100644
index 8b1218f7235d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_TNR_STATE_H
-#define __IA_CSS_TNR_STATE_H
-
-#include "type_support.h"
-
-/* TNR (temporal noise reduction) */
-struct sh_css_isp_tnr_dmem_state {
-	uint32_t tnr_in_buf_idx;
-	uint32_t tnr_out_buf_idx;
-};
-
-#endif /* __IA_CSS_TNR_STATE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h
deleted file mode 100644
index 9bbc9ab2e6c0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_TNR_TYPES_H
-#define __IA_CSS_TNR_TYPES_H
-
-/* @file
-* CSS-API header file for Temporal Noise Reduction (TNR) parameters.
-*/
-
-/* Temporal Noise Reduction (TNR) configuration.
- *
- *  When difference between current frame and previous frame is less than or
- *  equal to threshold, TNR works and current frame is mixed
- *  with previous frame.
- *  When difference between current frame and previous frame is greater
- *  than threshold, we judge motion is detected. Then, TNR does not work and
- *  current frame is outputted as it is.
- *  Therefore, when threshold_y and threshold_uv are set as 0, TNR can be disabled.
- *
- *  ISP block: TNR1
- *  ISP1: TNR1 is used.
- *  ISP2: TNR1 is used.
- */
-
-
-struct ia_css_tnr_config {
-	ia_css_u0_16 gain; /** Interpolation ratio of current frame
-			        and previous frame.
-				gain=0.0 -> previous frame is outputted.
-				gain=1.0 -> current frame is outputted.
-				u0.16, [0,65535],
-			default 32768(0.5), ineffective 65535(almost 1.0) */
-	ia_css_u0_16 threshold_y; /** Threshold to enable interpolation of Y.
-				If difference between current frame and
-				previous frame is greater than threshold_y,
-				TNR for Y is disabled.
-				u0.16, [0,65535], default/ineffective 0 */
-	ia_css_u0_16 threshold_uv; /** Threshold to enable interpolation of
-				U/V.
-				If difference between current frame and
-				previous frame is greater than threshold_uv,
-				TNR for UV is disabled.
-				u0.16, [0,65535], default/ineffective 0 */
-};
-
-
-#endif /* __IA_CSS_TNR_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/uds/uds_1.0/ia_css_uds_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/uds/uds_1.0/ia_css_uds_param.h
deleted file mode 100644
index 26b7b5bc9391..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/uds/uds_1.0/ia_css_uds_param.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_UDS_PARAM_H
-#define __IA_CSS_UDS_PARAM_H
-
-#include "sh_css_uds.h"
-
-/* uds (Up and Down scaling) */
-struct ia_css_uds_config {
-	struct sh_css_crop_pos crop_pos;
-	struct sh_css_uds_info uds;
-};
-
-struct sh_css_sp_uds_params {
-	struct sh_css_crop_pos crop_pos;
-	struct sh_css_uds_info uds;
-};
-
-#endif /* __IA_CSS_UDS_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.c
deleted file mode 100644
index c2076e412410..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.c
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_vf.host.h"
-#include <assert_support.h>
-#include <ia_css_err.h>
-#include <ia_css_frame.h>
-#include <ia_css_frame_public.h>
-#include <ia_css_pipeline.h>
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-
-#include "isp.h"
-
-void
-ia_css_vf_config(
-	struct sh_css_isp_vf_isp_config      *to,
-	const struct ia_css_vf_configuration *from,
-	unsigned size)
-{
-	unsigned elems_a = ISP_VEC_NELEMS;
-
-	(void)size;
-	to->vf_downscale_bits = from->vf_downscale_bits;
-	to->enable = from->info != NULL;
-
-	if (from->info) {
-		ia_css_frame_info_to_frame_sp_info(&to->info, from->info);
-		ia_css_dma_configure_from_info(&to->dma.port_b, from->info);
-		to->dma.width_a_over_b = elems_a / to->dma.port_b.elems;
-
-		/* Assume divisiblity here, may need to generalize to fixed point. */
-		assert (elems_a % to->dma.port_b.elems == 0);
-	}
-}
-
-/* compute the log2 of the downscale factor needed to get closest
- * to the requested viewfinder resolution on the upper side. The output cannot
- * be smaller than the requested viewfinder resolution.
- */
-enum ia_css_err
-sh_css_vf_downscale_log2(
-	const struct ia_css_frame_info *out_info,
-	const struct ia_css_frame_info *vf_info,
-	unsigned int *downscale_log2)
-{
-       unsigned int ds_log2 = 0;
-       unsigned int out_width;
-
-       if ((out_info == NULL) | (vf_info == NULL))
-	       return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-       out_width = out_info->res.width;
-
-       if (out_width == 0)
-	       return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-       /* downscale until width smaller than the viewfinder width. We don't
-	* test for the height since the vmem buffers only put restrictions on
-	* the width of a line, not on the number of lines in a frame.
-	*/
-       while (out_width >= vf_info->res.width) {
-	       ds_log2++;
-	       out_width /= 2;
-       }
-       /* now width is smaller, so we go up one step */
-       if ((ds_log2 > 0) && (out_width < ia_css_binary_max_vf_width()))
-	       ds_log2--;
-       /* TODO: use actual max input resolution of vf_pp binary */
-       if ((out_info->res.width >> ds_log2) >= 2 * ia_css_binary_max_vf_width())
-	       return IA_CSS_ERR_INVALID_ARGUMENTS;
-       *downscale_log2 = ds_log2;
-       return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-configure_kernel(
-	const struct ia_css_binary_info *info,
-	const struct ia_css_frame_info *out_info,
-	const struct ia_css_frame_info *vf_info,
-	unsigned int *downscale_log2,
-	struct ia_css_vf_configuration *config)
-{
-       enum ia_css_err err;
-       unsigned vf_log_ds = 0;
-
-       /* First compute value */
-       if (vf_info) {
-	       err = sh_css_vf_downscale_log2(out_info, vf_info, &vf_log_ds);
-	       if (err != IA_CSS_SUCCESS)
-		       return err;
-       }
-       vf_log_ds = min(vf_log_ds, info->vf_dec.max_log_downscale);
-       *downscale_log2 = vf_log_ds;
-
-       /* Then store it in isp config section */
-       config->vf_downscale_bits = vf_log_ds;
-       return IA_CSS_SUCCESS;
-}
-
-static void
-configure_dma(
-	struct ia_css_vf_configuration *config,
-	const struct ia_css_frame_info *vf_info)
-{
-	config->info = vf_info;
-}
-
-enum ia_css_err
-ia_css_vf_configure(
-	const struct ia_css_binary *binary,
-	const struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info,
-	unsigned int *downscale_log2)
-{
-	enum ia_css_err err;
-	struct ia_css_vf_configuration config;
-	const struct ia_css_binary_info *info = &binary->info->sp;
-
-	err = configure_kernel(info, out_info, vf_info, downscale_log2, &config);
-	configure_dma(&config, vf_info);
-
-	if (vf_info)
-		vf_info->raw_bit_depth = info->dma.vfdec_bits_per_pixel;
-	ia_css_configure_vf (binary, &config);
-
-	return IA_CSS_SUCCESS;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.h
deleted file mode 100644
index c7c3625a9a96..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf.host.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_VF_HOST_H
-#define __IA_CSS_VF_HOST_H
-
-#include "ia_css_frame_public.h"
-#include "ia_css_binary.h"
-
-#include "ia_css_vf_types.h"
-#include "ia_css_vf_param.h"
-
-/* compute the log2 of the downscale factor needed to get closest
- * to the requested viewfinder resolution on the upper side. The output cannot
- * be smaller than the requested viewfinder resolution.
- */
-enum ia_css_err
-sh_css_vf_downscale_log2(
-	const struct ia_css_frame_info *out_info,
-	const struct ia_css_frame_info *vf_info,
-	unsigned int *downscale_log2);
-
-void
-ia_css_vf_config(
-	struct sh_css_isp_vf_isp_config *to,
-	const struct ia_css_vf_configuration *from,
-	unsigned size);
-
-enum ia_css_err
-ia_css_vf_configure(
-	const struct ia_css_binary *binary,
-	const struct ia_css_frame_info *out_info,
-	struct ia_css_frame_info *vf_info,
-	unsigned int *downscale_log2);
-
-#endif /* __IA_CSS_VF_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf_param.h
deleted file mode 100644
index 9df4e12f6c2c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf_param.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_VF_PARAM_H
-#define __IA_CSS_VF_PARAM_H
-
-#include "type_support.h"
-#include "dma.h"
-#include "gc/gc_1.0/ia_css_gc_param.h" /* GAMMA_OUTPUT_BITS */
-#include "ia_css_frame_comm.h" /* ia_css_frame_sp_info */
-#include "ia_css_vf_types.h"
-
-#define VFDEC_BITS_PER_PIXEL	GAMMA_OUTPUT_BITS
-
-/* Viewfinder decimation */
-struct sh_css_isp_vf_isp_config {
-	uint32_t vf_downscale_bits; /** Log VF downscale value */
-	uint32_t enable;
-	struct ia_css_frame_sp_info info;
-	struct {
-		uint32_t width_a_over_b;
-		struct dma_port_config port_b;
-	} dma;
-};
-
-#endif /* __IA_CSS_VF_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf_types.h
deleted file mode 100644
index e3efafa279ff..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/vf/vf_1.0/ia_css_vf_types.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_VF_TYPES_H
-#define __IA_CSS_VF_TYPES_H
-
-/* Viewfinder decimation
- *
- *  ISP block: vfeven_horizontal_downscale
- */
-
-#include <ia_css_frame_public.h>
-#include <type_support.h>
-
-struct ia_css_vf_configuration {
-	uint32_t vf_downscale_bits; /** Log VF downscale value */
-	const struct ia_css_frame_info *info;
-};
-
-#endif /* __IA_CSS_VF_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.c
deleted file mode 100644
index b43cb88c6ae4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#ifndef IA_CSS_NO_DEBUG
-#include "ia_css_debug.h"
-#endif
-#include "sh_css_frac.h"
-
-#include "ia_css_wb.host.h"
-
-const struct ia_css_wb_config default_wb_config = {
-	1,
-	32768,
-	32768,
-	32768,
-	32768
-};
-
-void
-ia_css_wb_encode(
-	struct sh_css_isp_wb_params *to,
-	const struct ia_css_wb_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->gain_shift =
-	    uISP_REG_BIT - from->integer_bits;
-	to->gain_gr =
-	    uDIGIT_FITTING(from->gr, 16 - from->integer_bits,
-			   to->gain_shift);
-	to->gain_r =
-	    uDIGIT_FITTING(from->r, 16 - from->integer_bits,
-			   to->gain_shift);
-	to->gain_b =
-	    uDIGIT_FITTING(from->b, 16 - from->integer_bits,
-			   to->gain_shift);
-	to->gain_gb =
-	    uDIGIT_FITTING(from->gb, 16 - from->integer_bits,
-			   to->gain_shift);
-}
-
-#ifndef IA_CSS_NO_DEBUG
-void
-ia_css_wb_dump(
-	const struct sh_css_isp_wb_params *wb,
-	unsigned level)
-{
-	if (!wb) return;
-	ia_css_debug_dtrace(level, "White Balance:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"wb_gain_shift", wb->gain_shift);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"wb_gain_gr", wb->gain_gr);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"wb_gain_r", wb->gain_r);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"wb_gain_b", wb->gain_b);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"wb_gain_gb", wb->gain_gb);
-}
-
-void
-ia_css_wb_debug_dtrace(
-	const struct ia_css_wb_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.integer_bits=%d, "
-		"config.gr=%d, config.r=%d, "
-		"config.b=%d, config.gb=%d\n",
-		config->integer_bits,
-		config->gr, config->r,
-		config->b, config->gb);
-}
-#endif
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.h
deleted file mode 100644
index 18666baf9f76..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb.host.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_WB_HOST_H
-#define __IA_CSS_WB_HOST_H
-
-#include "ia_css_wb_types.h"
-#include "ia_css_wb_param.h"
-
-extern const struct ia_css_wb_config default_wb_config;
-
-void
-ia_css_wb_encode(
-	struct sh_css_isp_wb_params *to,
-	const struct ia_css_wb_config *from,
-	unsigned size);
-
-void
-ia_css_wb_dump(
-	const struct sh_css_isp_wb_params *wb,
-	unsigned level);
-
-void
-ia_css_wb_debug_dtrace(
-	const struct ia_css_wb_config *wb,
-	unsigned level);
-
-#endif /* __IA_CSS_WB_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb_param.h
deleted file mode 100644
index c95c53a24067..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb_param.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_WB_PARAM_H
-#define __IA_CSS_WB_PARAM_H
-
-#include "type_support.h"
-
-/* WB (White Balance) */
-struct sh_css_isp_wb_params {
-	int32_t gain_shift;
-	int32_t gain_gr;
-	int32_t gain_r;
-	int32_t gain_b;
-	int32_t gain_gb;
-};
-
-#endif /* __IA_CSS_WB_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb_types.h
deleted file mode 100644
index bf98734d057e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/wb/wb_1.0/ia_css_wb_types.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_WB_TYPES_H
-#define __IA_CSS_WB_TYPES_H
-
-/* @file
-* CSS-API header file for White Balance parameters.
-*/
-
-
-/* White Balance configuration (Gain Adjust).
- *
- *  ISP block: WB1
- *  ISP1: WB1 is used.
- *  ISP2: WB1 is used.
- */
-struct ia_css_wb_config {
-	uint32_t integer_bits; /** Common exponent of gains.
-				u8.0, [0,3],
-				default 1, ineffective 1 */
-	uint32_t gr;	/** Significand of Gr gain.
-				u[integer_bits].[16-integer_bits], [0,65535],
-				default/ineffective 32768(u1.15, 1.0) */
-	uint32_t r;	/** Significand of R gain.
-				u[integer_bits].[16-integer_bits], [0,65535],
-				default/ineffective 32768(u1.15, 1.0) */
-	uint32_t b;	/** Significand of B gain.
-				u[integer_bits].[16-integer_bits], [0,65535],
-				default/ineffective 32768(u1.15, 1.0) */
-	uint32_t gb;	/** Significand of Gb gain.
-				u[integer_bits].[16-integer_bits], [0,65535],
-				default/ineffective 32768(u1.15, 1.0) */
-};
-
-#endif /* __IA_CSS_WB_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c
deleted file mode 100644
index abcb531f51cc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "sh_css_frac.h"
-
-#include "ia_css_xnr.host.h"
-
-const struct ia_css_xnr_config default_xnr_config = {
-	/* default threshold 6400 translates to 25 on ISP. */
-	6400
-};
-
-void
-ia_css_xnr_table_vamem_encode(
-	struct sh_css_isp_xnr_vamem_params *to,
-	const struct ia_css_xnr_table *from,
-	unsigned size)
-{
-	(void)size;
-	memcpy (&to->xnr,  &from->data, sizeof(to->xnr));
-}
-
-void
-ia_css_xnr_encode(
-	struct sh_css_isp_xnr_params *to,
-	const struct ia_css_xnr_config *from,
-	unsigned size)
-{
-	(void)size;
-
-	to->threshold =
-		(uint16_t)uDIGIT_FITTING(from->threshold, 16, SH_CSS_ISP_YUV_BITS);
-}
-
-void
-ia_css_xnr_table_debug_dtrace(
-	const struct ia_css_xnr_table *config,
-	unsigned level)
-{
-	(void)config;
-	(void)level;
-}
-
-void
-ia_css_xnr_debug_dtrace(
-	const struct ia_css_xnr_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.threshold=%d\n", config->threshold);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h
deleted file mode 100644
index eb3425eafbbe..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_XNR_HOST_H
-#define __IA_CSS_XNR_HOST_H
-
-#include "sh_css_params.h"
-
-#include "ia_css_xnr_param.h"
-#include "ia_css_xnr_table.host.h"
-
-extern const struct ia_css_xnr_config default_xnr_config;
-
-void
-ia_css_xnr_table_vamem_encode(
-	struct sh_css_isp_xnr_vamem_params *to,
-	const struct ia_css_xnr_table *from,
-	unsigned size);
-
-void
-ia_css_xnr_encode(
-	struct sh_css_isp_xnr_params *to,
-	const struct ia_css_xnr_config *from,
-	unsigned size);
-
-void
-ia_css_xnr_table_debug_dtrace(
-	const struct ia_css_xnr_table *s3a,
-	unsigned level);
-
-void
-ia_css_xnr_debug_dtrace(
-	const struct ia_css_xnr_config *config,
-	unsigned level);
-
-#endif /* __IA_CSS_XNR_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h
deleted file mode 100644
index a5caebbe2f84..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_XNR_PARAM_H
-#define __IA_CSS_XNR_PARAM_H
-
-#include "type_support.h"
-#include <system_global.h>
-
-#ifndef PIPE_GENERATION
-#if defined(HAS_VAMEM_VERSION_2)
-#define SH_CSS_ISP_XNR_TABLE_SIZE_LOG2       IA_CSS_VAMEM_2_XNR_TABLE_SIZE_LOG2
-#define SH_CSS_ISP_XNR_TABLE_SIZE            IA_CSS_VAMEM_2_XNR_TABLE_SIZE
-#elif defined(HAS_VAMEM_VERSION_1)
-#define SH_CSS_ISP_XNR_TABLE_SIZE_LOG2       IA_CSS_VAMEM_1_XNR_TABLE_SIZE_LOG2
-#define SH_CSS_ISP_XNR_TABLE_SIZE            IA_CSS_VAMEM_1_XNR_TABLE_SIZE
-#else
-#error "Unknown vamem type"
-#endif
-
-
-#else
-/* For pipe generation, the size is not relevant */
-#define SH_CSS_ISP_XNR_TABLE_SIZE 0
-#endif
-
-/* This should be vamem_data_t, but that breaks the pipe generator */
-struct sh_css_isp_xnr_vamem_params {
-	uint16_t xnr[SH_CSS_ISP_XNR_TABLE_SIZE];
-};
-
-struct sh_css_isp_xnr_params {
-	/* XNR threshold.
-	 * type:u0.16 but actual valid range is:[0,255]
-	 * valid range is dependent on SH_CSS_ISP_YUV_BITS (currently 8bits)
-	 * default: 25 */
-	uint16_t threshold;
-};
-
-#endif /* __IA_CSS_XNR_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c
deleted file mode 100644
index cd5fb72fce3f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <type_support.h>
-#include <string_support.h> /* memcpy */
-#include "system_global.h"
-#include "vamem.h"
-#include "ia_css_types.h"
-#include "ia_css_xnr_table.host.h"
-
-struct ia_css_xnr_table default_xnr_table;
-
-#if defined(HAS_VAMEM_VERSION_2)
-
-static const uint16_t
-default_xnr_table_data[IA_CSS_VAMEM_2_XNR_TABLE_SIZE] = {
-  /* 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 */
-  8191>>1, 4096>>1, 2730>>1, 2048>>1, 1638>>1, 1365>>1, 1170>>1, 1024>>1, 910>>1, 819>>1, 744>>1, 682>>1, 630>>1, 585>>1,
-    546>>1, 512>>1,
-
-  /* 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 */
-  481>>1, 455>>1, 431>>1, 409>>1, 390>>1, 372>>1, 356>>1, 341>>1, 327>>1, 315>>1, 303>>1, 292>>1, 282>>1, 273>>1, 264>>1,
-    256>>1,
-
-  /* 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 */
-  248>>1, 240>>1, 234>>1, 227>>1, 221>>1, 215>>1, 210>>1, 204>>1, 199>>1, 195>>1, 190>>1, 186>>1, 182>>1, 178>>1, 174>>1,
-    170>>1,
-
-  /* 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 */
-  167>>1, 163>>1, 160>>1, 157>>1, 154>>1, 151>>1, 148>>1, 146>>1, 143>>1, 141>>1, 138>>1, 136>>1, 134>>1, 132>>1, 130>>1, 128>>1
-};
-
-#elif defined(HAS_VAMEM_VERSION_1)
-
-static const uint16_t
-default_xnr_table_data[IA_CSS_VAMEM_1_XNR_TABLE_SIZE] = {
-  /* 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 */
-  8191>>1, 4096>>1, 2730>>1, 2048>>1, 1638>>1, 1365>>1, 1170>>1, 1024>>1, 910>>1, 819>>1, 744>>1, 682>>1, 630>>1, 585>>1,
-    546>>1, 512>>1,
-
-  /* 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 */
-  481>>1, 455>>1, 431>>1, 409>>1, 390>>1, 372>>1, 356>>1, 341>>1, 327>>1, 315>>1, 303>>1, 292>>1, 282>>1, 273>>1, 264>>1,
-    256>>1,
-
-  /* 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 */
-  248>>1, 240>>1, 234>>1, 227>>1, 221>>1, 215>>1, 210>>1, 204>>1, 199>>1, 195>>1, 190>>1, 186>>1, 182>>1, 178>>1, 174>>1,
-    170>>1,
-
-  /* 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 */
-  167>>1, 163>>1, 160>>1, 157>>1, 154>>1, 151>>1, 148>>1, 146>>1, 143>>1, 141>>1, 138>>1, 136>>1, 134>>1, 132>>1, 130>>1, 128>>1
-};
-
-#else
-#error "sh_css_params.c: VAMEM version must \
-	be one of {VAMEM_VERSION_1, VAMEM_VERSION_2}"
-#endif
-
-void
-ia_css_config_xnr_table(void)
-{
-#if defined(HAS_VAMEM_VERSION_2)
-	memcpy(default_xnr_table.data.vamem_2, default_xnr_table_data,
-	       sizeof(default_xnr_table_data));
-	default_xnr_table.vamem_type     = IA_CSS_VAMEM_TYPE_2;
-#else
-	memcpy(default_xnr_table.data.vamem_1, default_xnr_table_data,
-	       sizeof(default_xnr_table_data));
-	default_xnr_table.vamem_type     = IA_CSS_VAMEM_TYPE_1;
-#endif
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h
deleted file mode 100644
index 130086713a7f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_XNR_TABLE_HOST_H
-#define __IA_CSS_XNR_TABLE_HOST_H
-
-extern struct ia_css_xnr_table default_xnr_table;
-
-void ia_css_config_xnr_table(void);
-
-#endif /* __IA_CSS_XNR_TABLE_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h
deleted file mode 100644
index d2b634211a3f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_XNR_TYPES_H
-#define __IA_CSS_XNR_TYPES_H
-
-/* @file
-* CSS-API header file for Extra Noise Reduction (XNR) parameters.
-*/
-
-/* XNR table.
- *
- *  NOTE: The driver does not need to set this table,
- *        because the default values are set inside the css.
- *
- *  This table contains coefficients used for division in XNR.
- *
- *  	u0.12, [0,4095],
- *      {4095, 2048, 1365, .........., 65, 64}
- *      ({1/1, 1/2, 1/3, ............., 1/63, 1/64})
- *
- *  ISP block: XNR1
- *  ISP1: XNR1 is used.
- *  ISP2: XNR1 is used.
- *
- */
-
-/* Number of elements in the xnr table. */
-#define IA_CSS_VAMEM_1_XNR_TABLE_SIZE_LOG2      6
-/* Number of elements in the xnr table. */
-#define IA_CSS_VAMEM_1_XNR_TABLE_SIZE           (1U<<IA_CSS_VAMEM_1_XNR_TABLE_SIZE_LOG2)
-
-/* Number of elements in the xnr table. */
-#define IA_CSS_VAMEM_2_XNR_TABLE_SIZE_LOG2      6
-/* Number of elements in the xnr table. */
-#define IA_CSS_VAMEM_2_XNR_TABLE_SIZE	        (1U<<IA_CSS_VAMEM_2_XNR_TABLE_SIZE_LOG2)
-
-/** IA_CSS_VAMEM_TYPE_1(ISP2300) or
-     IA_CSS_VAMEM_TYPE_2(ISP2400) */
-union ia_css_xnr_data {
-	uint16_t vamem_1[IA_CSS_VAMEM_1_XNR_TABLE_SIZE];
-	/** Coefficients table on vamem type1. u0.12, [0,4095] */
-	uint16_t vamem_2[IA_CSS_VAMEM_2_XNR_TABLE_SIZE];
-	/** Coefficients table on vamem type2. u0.12, [0,4095] */
-};
-
-struct ia_css_xnr_table {
-	enum ia_css_vamem_type vamem_type;
-	union ia_css_xnr_data data;
-};
-
-struct ia_css_xnr_config {
-	/* XNR threshold.
-	 * type:u0.16 valid range:[0,65535]
-	 * default: 6400 */
-	uint16_t threshold;
-};
-
-#endif /* __IA_CSS_XNR_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c
deleted file mode 100644
index 955b6c8a8738..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "type_support.h"
-#include "math_support.h"
-#include "sh_css_defs.h"
-#include "ia_css_types.h"
-#ifdef ISP2401
-#include "assert_support.h"
-#endif
-#include "ia_css_xnr3.host.h"
-
-/* Maximum value for alpha on ISP interface */
-#define XNR_MAX_ALPHA  ((1 << (ISP_VEC_ELEMBITS - 1)) - 1)
-
-/* Minimum value for sigma on host interface. Lower values translate to
- * max_alpha.
- */
-#define XNR_MIN_SIGMA  (IA_CSS_XNR3_SIGMA_SCALE / 100)
-
-/*
-#ifdef ISP2401
- * division look-up table
- * Refers to XNR3.0.5
- */
-#define XNR3_LOOK_UP_TABLE_POINTS 16
-
-static const int16_t x[XNR3_LOOK_UP_TABLE_POINTS] = {
-1024, 1164, 1320, 1492, 1680, 1884, 2108, 2352,
-2616, 2900, 3208, 3540, 3896, 4276, 4684, 5120};
-
-static const int16_t a[XNR3_LOOK_UP_TABLE_POINTS] = {
--7213, -5580, -4371, -3421, -2722, -2159, -6950, -5585,
--4529, -3697, -3010, -2485, -2070, -1727, -1428, 0};
-
-static const int16_t b[XNR3_LOOK_UP_TABLE_POINTS] = {
-4096, 3603, 3178, 2811, 2497, 2226, 1990, 1783,
-1603, 1446, 1307, 1185, 1077, 981, 895, 819};
-
-static const int16_t c[XNR3_LOOK_UP_TABLE_POINTS] = {
-1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/*
-#endif
- * Default kernel parameters. In general, default is bypass mode or as close
- * to the ineffective values as possible. Due to the chroma down+upsampling,
- * perfect bypass mode is not possible for xnr3 filter itself. Instead, the
- * 'blending' parameter is used to create a bypass.
- */
-const struct ia_css_xnr3_config default_xnr3_config = {
-	/* sigma */
-	{ 0, 0, 0, 0, 0, 0 },
-	/* coring */
-	{ 0, 0, 0, 0 },
-	/* blending */
-	{ 0 }
-};
-
-/*
- * Compute an alpha value for the ISP kernel from sigma value on the host
- * parameter interface as: alpha_scale * 1/(sigma/sigma_scale)
- */
-static int32_t
-compute_alpha(int sigma)
-{
-	int32_t alpha;
-#if defined(XNR_ATE_ROUNDING_BUG)
-	int32_t alpha_unscaled;
-#else
-	int offset = sigma / 2;
-#endif
-	if (sigma < XNR_MIN_SIGMA) {
-		alpha = XNR_MAX_ALPHA;
-	} else {
-#if defined(XNR_ATE_ROUNDING_BUG)
-		/* The scale factor for alpha must be the same as on the ISP,
-		 * For sigma, it must match the public interface. The code
-		 * below mimics the rounding and unintended loss of precision
-		 * of the ATE reference code. It computes an unscaled alpha,
-		 * rounds down, and then scales it to get the required fixed
-		 * point representation. It would have been more precise to
-		 * round after scaling. */
-		alpha_unscaled = IA_CSS_XNR3_SIGMA_SCALE / sigma;
-		alpha = alpha_unscaled * XNR_ALPHA_SCALE_FACTOR;
-#else
-		alpha = ((IA_CSS_XNR3_SIGMA_SCALE * XNR_ALPHA_SCALE_FACTOR) + offset)/ sigma;
-#endif
-
-		if (alpha > XNR_MAX_ALPHA)
-			alpha = XNR_MAX_ALPHA;
-	}
-
-	return alpha;
-}
-
-/*
- * Compute the scaled coring value for the ISP kernel from the value on the
- * host parameter interface.
- */
-static int32_t
-compute_coring(int coring)
-{
-	int32_t isp_coring;
-	int32_t isp_scale = XNR_CORING_SCALE_FACTOR;
-	int32_t host_scale = IA_CSS_XNR3_CORING_SCALE;
-	int32_t offset = host_scale / 2; /* fixed-point 0.5 */
-
-	/* Convert from public host-side scale factor to isp-side scale
-	 * factor. Clip to [0, isp_scale-1).
-	 */
-	isp_coring = ((coring * isp_scale) + offset) / host_scale;
-	return min(max(isp_coring, 0), isp_scale - 1);
-}
-
-/*
- * Compute the scaled blending strength for the ISP kernel from the value on
- * the host parameter interface.
- */
-static int32_t
-compute_blending(int strength)
-{
-	int32_t isp_strength;
-	int32_t isp_scale = XNR_BLENDING_SCALE_FACTOR;
-	int32_t host_scale = IA_CSS_XNR3_BLENDING_SCALE;
-	int32_t offset = host_scale / 2; /* fixed-point 0.5 */
-
-	/* Convert from public host-side scale factor to isp-side scale
-	 * factor. The blending factor is positive on the host side, but
-	 * negative on the ISP side because +1.0 cannot be represented
-	 * exactly as s0.11 fixed point, but -1.0 can.
-	 */
-	isp_strength = -(((strength * isp_scale) + offset) / host_scale);
-	return max(min(isp_strength, 0), -XNR_BLENDING_SCALE_FACTOR);
-}
-
-void
-ia_css_xnr3_encode(
-	struct sh_css_isp_xnr3_params *to,
-	const struct ia_css_xnr3_config *from,
-	unsigned size)
-{
-	int kernel_size = XNR_FILTER_SIZE;
-	/* The adjust factor is the next power of 2
-	   w.r.t. the kernel size*/
-	int adjust_factor = ceil_pow2(kernel_size);
-	int32_t max_diff = (1 << (ISP_VEC_ELEMBITS - 1)) - 1;
-	int32_t min_diff = -(1 << (ISP_VEC_ELEMBITS - 1));
-
-	int32_t alpha_y0 = compute_alpha(from->sigma.y0);
-	int32_t alpha_y1 = compute_alpha(from->sigma.y1);
-	int32_t alpha_u0 = compute_alpha(from->sigma.u0);
-	int32_t alpha_u1 = compute_alpha(from->sigma.u1);
-	int32_t alpha_v0 = compute_alpha(from->sigma.v0);
-	int32_t alpha_v1 = compute_alpha(from->sigma.v1);
-	int32_t alpha_ydiff = (alpha_y1 - alpha_y0) * adjust_factor / kernel_size;
-	int32_t alpha_udiff = (alpha_u1 - alpha_u0) * adjust_factor / kernel_size;
-	int32_t alpha_vdiff = (alpha_v1 - alpha_v0) * adjust_factor / kernel_size;
-
-	int32_t coring_u0 = compute_coring(from->coring.u0);
-	int32_t coring_u1 = compute_coring(from->coring.u1);
-	int32_t coring_v0 = compute_coring(from->coring.v0);
-	int32_t coring_v1 = compute_coring(from->coring.v1);
-	int32_t coring_udiff = (coring_u1 - coring_u0) * adjust_factor / kernel_size;
-	int32_t coring_vdiff = (coring_v1 - coring_v0) * adjust_factor / kernel_size;
-
-	int32_t blending = compute_blending(from->blending.strength);
-
-	(void)size;
-
-	/* alpha's are represented in qN.5 format */
-	to->alpha.y0 = alpha_y0;
-	to->alpha.u0 = alpha_u0;
-	to->alpha.v0 = alpha_v0;
-	to->alpha.ydiff = min(max(alpha_ydiff, min_diff), max_diff);
-	to->alpha.udiff = min(max(alpha_udiff, min_diff), max_diff);
-	to->alpha.vdiff = min(max(alpha_vdiff, min_diff), max_diff);
-
-	/* coring parameters are expressed in q1.NN format */
-	to->coring.u0 = coring_u0;
-	to->coring.v0 = coring_v0;
-	to->coring.udiff = min(max(coring_udiff, min_diff), max_diff);
-	to->coring.vdiff = min(max(coring_vdiff, min_diff), max_diff);
-
-	/* blending strength is expressed in q1.NN format */
-	to->blending.strength = blending;
-}
-
-#ifdef ISP2401
-/* (void) = ia_css_xnr3_vmem_encode(*to, *from)
- * -----------------------------------------------
- * VMEM Encode Function to translate UV parameters from userspace into ISP space
-*/
-void
-ia_css_xnr3_vmem_encode(
-	struct sh_css_isp_xnr3_vmem_params *to,
-	const struct ia_css_xnr3_config *from,
-	unsigned size)
-{
-	unsigned i, j, base;
-	const unsigned total_blocks = 4;
-	const unsigned shuffle_block = 16;
-
-	(void)from;
-	(void)size;
-
-	/* Init */
-	for (i = 0; i < ISP_VEC_NELEMS; i++) {
-		to->x[0][i] = 0;
-		to->a[0][i] = 0;
-		to->b[0][i] = 0;
-		to->c[0][i] = 0;
-	}
-
-	/* Constraints on "x":
-	 * - values should be greater or equal to 0.
-	 * - values should be ascending.
-	 */
-	assert(x[0] >= 0);
-
-	for (j = 1; j < XNR3_LOOK_UP_TABLE_POINTS; j++) {
-		assert(x[j] >= 0);
-		assert(x[j] > x[j - 1]);
-
-	}
-
-	/* The implementation of the calulating 1/x is based on the availability
-	 * of the OP_vec_shuffle16 operation.
-	 * A 64 element vector is split up in 4 blocks of 16 element. Each array is copied to
-	 * a vector 4 times, (starting at 0, 16, 32 and 48). All array elements are copied or
-	 * initialised as described in the KFS. The remaining elements of a vector are set to 0.
-	 */
-	/* TODO: guard this code with above assumptions */
-	for (i = 0; i < total_blocks; i++) {
-		base = shuffle_block * i;
-
-		for (j = 0; j < XNR3_LOOK_UP_TABLE_POINTS; j++) {
-			to->x[0][base + j] = x[j];
-			to->a[0][base + j] = a[j];
-			to->b[0][base + j] = b[j];
-			to->c[0][base + j] = c[j];
-		}
-	}
-}
-
-#endif
-/* Dummy Function added as the tool expects it*/
-void
-ia_css_xnr3_debug_dtrace(
-	const struct ia_css_xnr3_config *config,
-	unsigned level)
-{
-	(void)config;
-	(void)level;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h
deleted file mode 100644
index 6a86924a71fe..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_XNR3_HOST_H
-#define __IA_CSS_XNR3_HOST_H
-
-#include "ia_css_xnr3_param.h"
-#include "ia_css_xnr3_types.h"
-
-extern const struct ia_css_xnr3_config default_xnr3_config;
-
-void
-ia_css_xnr3_encode(
-	struct sh_css_isp_xnr3_params *to,
-	const struct ia_css_xnr3_config *from,
-	unsigned size);
-
-#ifdef ISP2401
-void
-ia_css_xnr3_vmem_encode(
-	struct sh_css_isp_xnr3_vmem_params *to,
-	const struct ia_css_xnr3_config *from,
-	unsigned size);
-
-#endif
-void
-ia_css_xnr3_debug_dtrace(
-	const struct ia_css_xnr3_config *config,
-	unsigned level);
-
-#endif /* __IA_CSS_XNR3_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h
deleted file mode 100644
index 06c24e848234..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_XNR3_PARAM_H
-#define __IA_CSS_XNR3_PARAM_H
-
-#include "type_support.h"
-#ifdef ISP2401
-#include "vmem.h" /* needed for VMEM_ARRAY */
-
-#endif
-
-/* Scaling factor of the alpha values: which fixed-point value represents 1.0?
- * It must be chosen such that 1/min_sigma still fits in an ISP vector
- * element. */
-#define XNR_ALPHA_SCALE_LOG2        5
-#define XNR_ALPHA_SCALE_FACTOR      (1 << XNR_ALPHA_SCALE_LOG2)
-
-/* Scaling factor of the coring values on the ISP. */
-#define XNR_CORING_SCALE_LOG2       (ISP_VEC_ELEMBITS-1)
-#define XNR_CORING_SCALE_FACTOR     (1 << XNR_CORING_SCALE_LOG2)
-
-/* Scaling factor of the blending strength on the ISP. */
-#define XNR_BLENDING_SCALE_LOG2     (ISP_VEC_ELEMBITS-1)
-#define XNR_BLENDING_SCALE_FACTOR   (1 << XNR_BLENDING_SCALE_LOG2)
-
-/* XNR3 filter size. Must be 11x11, 9x9 or 5x5. */
-#ifdef FLT_KERNEL_9x9
-#define XNR_FILTER_SIZE             9
-#else
-#ifdef FLT_KERNEL_11x11
-#define XNR_FILTER_SIZE             11
-#else
-#define XNR_FILTER_SIZE             5
-#endif
-#endif
-
-/* XNR3 alpha (1/sigma) parameters on the ISP, expressed as a base (0) value
- * for dark areas, and a scaled diff towards the value for bright areas. */
-struct sh_css_xnr3_alpha_params {
-	int32_t y0;
-	int32_t u0;
-	int32_t v0;
-	int32_t ydiff;
-	int32_t udiff;
-	int32_t vdiff;
-};
-
-/* XNR3 coring parameters on the ISP, expressed as a base (0) value
- * for dark areas, and a scaled diff towards the value for bright areas. */
-struct sh_css_xnr3_coring_params {
-	int32_t u0;
-	int32_t v0;
-	int32_t udiff;
-	int32_t vdiff;
-};
-
-/* XNR3 blending strength on the ISP. */
-struct sh_css_xnr3_blending_params {
-	int32_t strength;
-};
-
-/* XNR3 ISP parameters */
-struct sh_css_isp_xnr3_params {
-	struct sh_css_xnr3_alpha_params    alpha;
-	struct sh_css_xnr3_coring_params   coring;
-	struct sh_css_xnr3_blending_params blending;
-};
-
-#ifdef ISP2401
-/*
- * STRUCT sh_css_isp_xnr3_vmem_params
- * -----------------------------------------------
- * ISP VMEM parameters
- */
-struct sh_css_isp_xnr3_vmem_params {
-	VMEM_ARRAY(x, ISP_VEC_NELEMS);
-	VMEM_ARRAY(a, ISP_VEC_NELEMS);
-	VMEM_ARRAY(b, ISP_VEC_NELEMS);
-	VMEM_ARRAY(c, ISP_VEC_NELEMS);
-};
-
-
-#endif
-#endif  /*__IA_CSS_XNR3_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h
deleted file mode 100644
index 669200caf72e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_XNR3_TYPES_H
-#define __IA_CSS_XNR3_TYPES_H
-
-/* @file
-* CSS-API header file for Extra Noise Reduction (XNR) parameters.
-*/
-
-/**
- * \brief Scale of the XNR sigma parameters.
- * \details The define specifies which fixed-point value represents 1.0.
- */
-#define IA_CSS_XNR3_SIGMA_SCALE  (1 << 10)
-
-/**
- * \brief Scale of the XNR coring parameters.
- * \details The define specifies which fixed-point value represents 1.0.
- */
-#define IA_CSS_XNR3_CORING_SCALE (1 << 15)
-
-/**
- * \brief Scale of the XNR blending parameter.
- * \details The define specifies which fixed-point value represents 1.0.
- */
-#define IA_CSS_XNR3_BLENDING_SCALE (1 << 11)
-
-
-/**
- * \brief XNR3 Sigma Parameters.
- * \details Sigma parameters define the strength of the XNR filter.
- * A higher number means stronger filtering. There are two values for each of
- * the three YUV planes: one for dark areas and one for bright areas. All
- * sigma parameters are fixed-point values between 0.0 and 1.0, scaled with
- * IA_CSS_XNR3_SIGMA_SCALE.
- */
-struct ia_css_xnr3_sigma_params {
-	int y0;     /** Sigma for Y range similarity in dark area */
-	int y1;     /** Sigma for Y range similarity in bright area */
-	int u0;     /** Sigma for U range similarity in dark area */
-	int u1;     /** Sigma for U range similarity in bright area */
-	int v0;     /** Sigma for V range similarity in dark area */
-	int v1;     /** Sigma for V range similarity in bright area */
-};
-
-/**
- * \brief XNR3 Coring Parameters
- * \details Coring parameters define the "coring" strength, which is a soft
- * thresholding technique to avoid false coloring. There are two values for
- * each of the two chroma planes: one for dark areas and one for bright areas.
- * All coring parameters are fixed-point values between 0.0 and 1.0, scaled
- * with IA_CSS_XNR3_CORING_SCALE. The ineffective value is 0.
- */
-struct ia_css_xnr3_coring_params {
-	int u0;     /** Coring threshold of U channel in dark area */
-	int u1;     /** Coring threshold of U channel in bright area */
-	int v0;     /** Coring threshold of V channel in dark area */
-	int v1;     /** Coring threshold of V channel in bright area */
-};
-
-/**
- * \brief XNR3 Blending Parameters
- * \details Blending parameters define the blending strength of filtered
- * output pixels with the original chroma pixels from before xnr3. The
- * blending strength is a fixed-point value between 0.0 and 1.0 (inclusive),
- * scaled with IA_CSS_XNR3_BLENDING_SCALE.
- * A higher number applies xnr filtering more strongly. A value of 1.0
- * disables the blending and returns the xnr3 filtered output, while a
- * value of 0.0 bypasses the entire xnr3 filter.
- */
-struct ia_css_xnr3_blending_params {
-	int strength;   /** Blending strength */
-};
-
-/**
- * \brief XNR3 public parameters.
- * \details Struct with all parameters for the XNR3 kernel that can be set
- * from the CSS API.
- */
-struct ia_css_xnr3_config {
-	struct ia_css_xnr3_sigma_params    sigma;    /** XNR3 sigma parameters */
-	struct ia_css_xnr3_coring_params   coring;   /** XNR3 coring parameters */
-	struct ia_css_xnr3_blending_params blending; /** XNR3 blending parameters */
-};
-
-#endif /* __IA_CSS_XNR3_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c
deleted file mode 100644
index d8dccce772a9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "sh_css_frac.h"
-
-#include "bnr/bnr_1.0/ia_css_bnr.host.h"
-#include "ia_css_ynr.host.h"
-
-const struct ia_css_nr_config default_nr_config = {
-	16384,
-	8192,
-	1280,
-	0,
-	0
-};
-
-const struct ia_css_ee_config default_ee_config = {
-	8192,
-	128,
-	2048
-};
-
-void
-ia_css_nr_encode(
-	struct sh_css_isp_ynr_params *to,
-	const struct ia_css_nr_config *from,
-	unsigned size)
-{
-	(void)size;
-	/* YNR (Y Noise Reduction) */
-	to->threshold =
-		uDIGIT_FITTING((unsigned)8192, 16, SH_CSS_BAYER_BITS);
-	to->gain_all =
-	    uDIGIT_FITTING(from->ynr_gain, 16, SH_CSS_YNR_GAIN_SHIFT);
-	to->gain_dir =
-	    uDIGIT_FITTING(from->ynr_gain, 16, SH_CSS_YNR_GAIN_SHIFT);
-	to->threshold_cb =
-	    uDIGIT_FITTING(from->threshold_cb, 16, SH_CSS_BAYER_BITS);
-	to->threshold_cr =
-	    uDIGIT_FITTING(from->threshold_cr, 16, SH_CSS_BAYER_BITS);
-}
-
-void
-ia_css_yee_encode(
-	struct sh_css_isp_yee_params *to,
-	const struct ia_css_yee_config *from,
-	unsigned size)
-{
-	int asiWk1 = (int) from->ee.gain;
-	int asiWk2 = asiWk1 / 8;
-	int asiWk3 = asiWk1 / 4;
-
-	(void)size;
-	/* YEE (Y Edge Enhancement) */
-	to->dirthreshold_s =
-	    min((uDIGIT_FITTING(from->nr.direction, 16, SH_CSS_BAYER_BITS)
-				    << 1),
-		SH_CSS_BAYER_MAXVAL);
-	to->dirthreshold_g =
-	    min((uDIGIT_FITTING(from->nr.direction, 16, SH_CSS_BAYER_BITS)
-				    << 4),
-		SH_CSS_BAYER_MAXVAL);
-	to->dirthreshold_width_log2 =
-	    uFRACTION_BITS_FITTING(8);
-	to->dirthreshold_width =
-	    1 << to->dirthreshold_width_log2;
-	to->detailgain =
-	    uDIGIT_FITTING(from->ee.detail_gain, 11,
-			   SH_CSS_YEE_DETAIL_GAIN_SHIFT);
-	to->coring_s =
-	    (uDIGIT_FITTING((unsigned)56, 16, SH_CSS_BAYER_BITS) *
-	     from->ee.threshold) >> 8;
-	to->coring_g =
-	    (uDIGIT_FITTING((unsigned)224, 16, SH_CSS_BAYER_BITS) *
-	     from->ee.threshold) >> 8;
-	/* 8; // *1.125 ->[s4.8] */
-	to->scale_plus_s =
-	    (asiWk1 + asiWk2) >> (11 - SH_CSS_YEE_SCALE_SHIFT);
-	/* 8; // ( * -.25)->[s4.8] */
-	to->scale_plus_g =
-	    (0 - asiWk3) >> (11 - SH_CSS_YEE_SCALE_SHIFT);
-	/* 8; // *0.875 ->[s4.8] */
-	to->scale_minus_s =
-	    (asiWk1 - asiWk2) >> (11 - SH_CSS_YEE_SCALE_SHIFT);
-	/* 8; // ( *.25 ) ->[s4.8] */
-	to->scale_minus_g =
-	    (asiWk3) >> (11 - SH_CSS_YEE_SCALE_SHIFT);
-	to->clip_plus_s =
-	    uDIGIT_FITTING((unsigned)32760, 16, SH_CSS_BAYER_BITS);
-	to->clip_plus_g = 0;
-	to->clip_minus_s =
-	    uDIGIT_FITTING((unsigned)504, 16, SH_CSS_BAYER_BITS);
-	to->clip_minus_g =
-	    uDIGIT_FITTING((unsigned)32256, 16, SH_CSS_BAYER_BITS);
-	to->Yclip = SH_CSS_BAYER_MAXVAL;
-}
-
-void
-ia_css_nr_dump(
-	const struct sh_css_isp_ynr_params *ynr,
-	unsigned level)
-{
-	if (!ynr) return;
-	ia_css_debug_dtrace(level,
-		"Y Noise Reduction:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ynr_threshold", ynr->threshold);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ynr_gain_all", ynr->gain_all);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ynr_gain_dir", ynr->gain_dir);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ynr_threshold_cb", ynr->threshold_cb);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ynr_threshold_cr", ynr->threshold_cr);
-}
-
-void
-ia_css_yee_dump(
-	const struct sh_css_isp_yee_params *yee,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"Y Edge Enhancement:\n");
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ynryee_dirthreshold_s",
-			yee->dirthreshold_s);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ynryee_dirthreshold_g",
-			yee->dirthreshold_g);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ynryee_dirthreshold_width_log2",
-			yee->dirthreshold_width_log2);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ynryee_dirthreshold_width",
-			yee->dirthreshold_width);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"yee_detailgain",
-			yee->detailgain);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"yee_coring_s",
-			yee->coring_s);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"yee_coring_g",
-			yee->coring_g);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"yee_scale_plus_s",
-			yee->scale_plus_s);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"yee_scale_plus_g",
-			yee->scale_plus_g);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"yee_scale_minus_s",
-			yee->scale_minus_s);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"yee_scale_minus_g",
-			yee->scale_minus_g);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"yee_clip_plus_s",
-			yee->clip_plus_s);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"yee_clip_plus_g",
-			yee->clip_plus_g);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"yee_clip_minus_s",
-			yee->clip_minus_s);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"yee_clip_minus_g",
-			yee->clip_minus_g);
-	ia_css_debug_dtrace(level, "\t%-32s = %d\n",
-			"ynryee_Yclip",
-			yee->Yclip);
-}
-
-void
-ia_css_nr_debug_dtrace(
-	const struct ia_css_nr_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.direction=%d, "
-		"config.bnr_gain=%d, config.ynr_gain=%d, "
-		"config.threshold_cb=%d, config.threshold_cr=%d\n",
-		config->direction,
-		config->bnr_gain, config->ynr_gain,
-		config->threshold_cb, config->threshold_cr);
-}
-
-void
-ia_css_ee_debug_dtrace(
-	const struct ia_css_ee_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.threshold=%d, config.gain=%d, config.detail_gain=%d\n",
-		config->threshold, config->gain, config->detail_gain);
-}
-
-void
-ia_css_init_ynr_state(
-	void/*struct sh_css_isp_ynr_vmem_state*/ *state,
-	size_t size)
-{
-	memset(state, 0, size);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h
deleted file mode 100644
index b5730df313ef..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_YNR_HOST_H
-#define __IA_CSS_YNR_HOST_H
-
-#include "ia_css_ynr_types.h"
-#include "ia_css_ynr_param.h"
-
-extern const struct ia_css_nr_config default_nr_config;
-extern const struct ia_css_ee_config default_ee_config;
-
-void
-ia_css_nr_encode(
-	struct sh_css_isp_ynr_params *to,
-	const struct ia_css_nr_config *from,
-	unsigned size);
-
-void
-ia_css_yee_encode(
-	struct sh_css_isp_yee_params *to,
-	const struct ia_css_yee_config *from,
-	unsigned size);
-
-void
-ia_css_nr_dump(
-	const struct sh_css_isp_ynr_params *ynr,
-	unsigned level);
-
-void
-ia_css_yee_dump(
-	const struct sh_css_isp_yee_params *yee,
-	unsigned level);
-
-void
-ia_css_nr_debug_dtrace(
-	const struct ia_css_nr_config *config,
-	unsigned level);
-
-void
-ia_css_ee_debug_dtrace(
-	const struct ia_css_ee_config *config,
-	unsigned level);
-
-void
-ia_css_init_ynr_state(
-	void/*struct sh_css_isp_ynr_vmem_state*/ *state,
-	size_t size);
-#endif /* __IA_CSS_YNR_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h
deleted file mode 100644
index ad61ec1211e8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_YNR_PARAM_H
-#define __IA_CSS_YNR_PARAM_H
-
-#include "type_support.h"
-
-/* YNR (Y Noise Reduction) */
-struct sh_css_isp_ynr_params {
-	int32_t threshold;
-	int32_t gain_all;
-	int32_t gain_dir;
-	int32_t threshold_cb;
-	int32_t threshold_cr;
-};
-
-/* YEE (Y Edge Enhancement) */
-struct sh_css_isp_yee_params {
-	int32_t dirthreshold_s;
-	int32_t dirthreshold_g;
-	int32_t dirthreshold_width_log2;
-	int32_t dirthreshold_width;
-	int32_t detailgain;
-	int32_t coring_s;
-	int32_t coring_g;
-	int32_t scale_plus_s;
-	int32_t scale_plus_g;
-	int32_t scale_minus_s;
-	int32_t scale_minus_g;
-	int32_t clip_plus_s;
-	int32_t clip_plus_g;
-	int32_t clip_minus_s;
-	int32_t clip_minus_g;
-	int32_t Yclip;
-};
-
-#endif /* __IA_CSS_YNR_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_state.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_state.h
deleted file mode 100644
index b2348b19c3cd..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_state.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_YNR_STATE_H
-#define __IA_CSS_YNR_STATE_H
-
-#include "type_support.h"
-#include "vmem.h"
-
-/* YNR (luminance noise reduction) */
-struct sh_css_isp_ynr_vmem_state {
-	VMEM_ARRAY(ynr_buf[4], MAX_VECTORS_PER_BUF_LINE*ISP_NWAY);
-};
-
-#endif /* __IA_CSS_YNR_STATE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h
deleted file mode 100644
index 3f8589a5a43a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_YNR_TYPES_H
-#define __IA_CSS_YNR_TYPES_H
-
-/* @file
-* CSS-API header file for Noise Reduction (BNR) and YCC Noise Reduction (YNR,CNR).
-*/
-
-/* Configuration used by Bayer Noise Reduction (BNR) and
- *  YCC Noise Reduction (YNR,CNR).
- *
- *  ISP block: BNR1, YNR1, CNR1
- *  ISP1: BNR1,YNR1,CNR1 are used.
- *  ISP2: BNR1,YNR1,CNR1 are used for Preview/Video.
- *        BNR1,YNR2,CNR2 are used for Still.
- */
-struct ia_css_nr_config {
-	ia_css_u0_16 bnr_gain;	   /** Strength of noise reduction (BNR).
-				u0.16, [0,65535],
-				default 14336(0.21875), ineffective 0 */
-	ia_css_u0_16 ynr_gain;	   /** Strength of noise reduction (YNR).
-				u0.16, [0,65535],
-				default 14336(0.21875), ineffective 0 */
-	ia_css_u0_16 direction;    /** Sensitivity of edge (BNR).
-				u0.16, [0,65535],
-				default 512(0.0078125), ineffective 0 */
-	ia_css_u0_16 threshold_cb; /** Coring threshold for Cb (CNR).
-				This is the same as
-				de_config.c1_coring_threshold.
-				u0.16, [0,65535],
-				default 0(0), ineffective 0 */
-	ia_css_u0_16 threshold_cr; /** Coring threshold for Cr (CNR).
-				This is the same as
-				de_config.c2_coring_threshold.
-				u0.16, [0,65535],
-				default 0(0), ineffective 0 */
-};
-
-/* Edge Enhancement (sharpen) configuration.
- *
- *  ISP block: YEE1
- *  ISP1: YEE1 is used.
- *  ISP2: YEE1 is used for Preview/Video.
- *       (YEE2 is used for Still.)
- */
-struct ia_css_ee_config {
-	ia_css_u5_11 gain;	  /** The strength of sharpness.
-					u5.11, [0,65535],
-					default 8192(4.0), ineffective 0 */
-	ia_css_u8_8 threshold;    /** The threshold that divides noises from
-					edge.
-					u8.8, [0,65535],
-					default 256(1.0), ineffective 65535 */
-	ia_css_u5_11 detail_gain; /** The strength of sharpness in pell-mell
-					area.
-					u5.11, [0,65535],
-					default 2048(1.0), ineffective 0 */
-};
-
-/* YNR and YEE (sharpen) configuration.
- */
-struct ia_css_yee_config {
-	struct ia_css_nr_config nr; /** The NR configuration. */
-	struct ia_css_ee_config ee; /** The EE configuration. */
-};
-
-#endif /* __IA_CSS_YNR_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c
deleted file mode 100644
index 44b005004238..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "assert_support.h"
-
-#include "ia_css_ynr2.host.h"
-
-const struct ia_css_ynr_config default_ynr_config = {
-	0,
-	0,
-	0,
-	0,
-};
-
-const struct ia_css_fc_config default_fc_config = {
-	1,
-	0,		/* 0 -> ineffective */
-	0,		/* 0 -> ineffective */
-	0,		/* 0 -> ineffective */
-	0,		/* 0 -> ineffective */
-	(1 << (ISP_VEC_ELEMBITS - 2)),		/* 0.5 */
-	(1 << (ISP_VEC_ELEMBITS - 2)),		/* 0.5 */
-	(1 << (ISP_VEC_ELEMBITS - 2)),		/* 0.5 */
-	(1 << (ISP_VEC_ELEMBITS - 2)),		/* 0.5 */
-	(1 << (ISP_VEC_ELEMBITS - 1)) - 1,	/* 1 */
-	(1 << (ISP_VEC_ELEMBITS - 1)) - 1,	/* 1 */
-	(int16_t)- (1 << (ISP_VEC_ELEMBITS - 1)),	/* -1 */
-	(int16_t)- (1 << (ISP_VEC_ELEMBITS - 1)),	/* -1 */
-};
-
-void
-ia_css_ynr_encode(
-	struct sh_css_isp_yee2_params *to,
-	const struct ia_css_ynr_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->edge_sense_gain_0   = from->edge_sense_gain_0;
-	to->edge_sense_gain_1   = from->edge_sense_gain_1;
-	to->corner_sense_gain_0 = from->corner_sense_gain_0;
-	to->corner_sense_gain_1 = from->corner_sense_gain_1;
-}
-
-void
-ia_css_fc_encode(
-	struct sh_css_isp_fc_params *to,
-	const struct ia_css_fc_config *from,
-	unsigned size)
-{
-	(void)size;
-	to->gain_exp   = from->gain_exp;
-
-	to->coring_pos_0 = from->coring_pos_0;
-	to->coring_pos_1 = from->coring_pos_1;
-	to->coring_neg_0 = from->coring_neg_0;
-	to->coring_neg_1 = from->coring_neg_1;
-
-	to->gain_pos_0 = from->gain_pos_0;
-	to->gain_pos_1 = from->gain_pos_1;
-	to->gain_neg_0 = from->gain_neg_0;
-	to->gain_neg_1 = from->gain_neg_1;
-
-	to->crop_pos_0 = from->crop_pos_0;
-	to->crop_pos_1 = from->crop_pos_1;
-	to->crop_neg_0 = from->crop_neg_0;
-	to->crop_neg_1 = from->crop_neg_1;
-}
-
-void
-ia_css_ynr_dump(
-	const struct sh_css_isp_yee2_params *yee2,
-	unsigned level);
-
-void
-ia_css_fc_dump(
-	const struct sh_css_isp_fc_params *fc,
-	unsigned level);
-
-void
-ia_css_fc_debug_dtrace(
-	const struct ia_css_fc_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.gain_exp=%d, "
-		"config.coring_pos_0=%d, config.coring_pos_1=%d, "
-		"config.coring_neg_0=%d, config.coring_neg_1=%d, "
-		"config.gain_pos_0=%d, config.gain_pos_1=%d, "
-		"config.gain_neg_0=%d, config.gain_neg_1=%d, "
-		"config.crop_pos_0=%d, config.crop_pos_1=%d, "
-		"config.crop_neg_0=%d, config.crop_neg_1=%d\n",
-		config->gain_exp,
-		config->coring_pos_0, config->coring_pos_1,
-		config->coring_neg_0, config->coring_neg_1,
-		config->gain_pos_0, config->gain_pos_1,
-		config->gain_neg_0, config->gain_neg_1,
-		config->crop_pos_0, config->crop_pos_1,
-		config->crop_neg_0, config->crop_neg_1);
-}
-
-void
-ia_css_ynr_debug_dtrace(
-	const struct ia_css_ynr_config *config,
-	unsigned level)
-{
-	ia_css_debug_dtrace(level,
-		"config.edge_sense_gain_0=%d, config.edge_sense_gain_1=%d, "
-		"config.corner_sense_gain_0=%d, config.corner_sense_gain_1=%d\n",
-		config->edge_sense_gain_0, config->edge_sense_gain_1,
-		config->corner_sense_gain_0, config->corner_sense_gain_1);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h
deleted file mode 100644
index 71e89c469e4c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_YNR2_HOST_H
-#define __IA_CSS_YNR2_HOST_H
-
-#include "ia_css_ynr2_types.h"
-#include "ia_css_ynr2_param.h"
-
-extern const struct ia_css_ynr_config default_ynr_config;
-extern const struct ia_css_fc_config  default_fc_config;
-
-void
-ia_css_ynr_encode(
-	struct sh_css_isp_yee2_params *to,
-	const struct ia_css_ynr_config *from,
-	unsigned size);
-
-void
-ia_css_fc_encode(
-	struct sh_css_isp_fc_params *to,
-	const struct ia_css_fc_config *from,
-	unsigned size);
-
-void
-ia_css_ynr_dump(
-	const struct sh_css_isp_yee2_params *yee2,
-	unsigned level);
-
-void
-ia_css_fc_dump(
-	const struct sh_css_isp_fc_params *fc,
-	unsigned level);
-
-void
-ia_css_fc_debug_dtrace(
-	const struct ia_css_fc_config *config,
-	unsigned level);
-
-void
-ia_css_ynr_debug_dtrace(
-	const struct ia_css_ynr_config *config,
-	unsigned level);
-
-#endif /* __IA_CSS_YNR2_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h
deleted file mode 100644
index e56b695bef27..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_YNR2_PARAM_H
-#define __IA_CSS_YNR2_PARAM_H
-
-#include "type_support.h"
-
-/* YNR (Y Noise Reduction), YEE (Y Edge Enhancement) */
-struct sh_css_isp_yee2_params {
-	int32_t edge_sense_gain_0;
-	int32_t edge_sense_gain_1;
-	int32_t corner_sense_gain_0;
-	int32_t corner_sense_gain_1;
-};
-
-/* Fringe Control */
-struct sh_css_isp_fc_params {
-	int32_t gain_exp;
-	uint16_t coring_pos_0;
-	uint16_t coring_pos_1;
-	uint16_t coring_neg_0;
-	uint16_t coring_neg_1;
-	int32_t gain_pos_0;
-	int32_t gain_pos_1;
-	int32_t gain_neg_0;
-	int32_t gain_neg_1;
-	int32_t crop_pos_0;
-	int32_t crop_pos_1;
-	int32_t crop_neg_0;
-	int32_t crop_neg_1;
-};
-
-#endif /* __IA_CSS_YNR2_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h
deleted file mode 100644
index 83161a24207d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_YNR2_TYPES_H
-#define __IA_CSS_YNR2_TYPES_H
-
-/* @file
-* CSS-API header file for Y(Luma) Noise Reduction.
-*/
-
-/* Y(Luma) Noise Reduction configuration.
- *
- *  ISP block: YNR2 & YEE2
- * (ISP1: YNR1 and YEE1 are used.)
- * (ISP2: YNR1 and YEE1 are used for Preview/Video.)
- *  ISP2: YNR2 and YEE2 are used for Still.
- */
-struct ia_css_ynr_config {
-	uint16_t edge_sense_gain_0;   /** Sensitivity of edge in dark area.
-					u13.0, [0,8191],
-					default 1000, ineffective 0 */
-	uint16_t edge_sense_gain_1;   /** Sensitivity of edge in bright area.
-					u13.0, [0,8191],
-					default 1000, ineffective 0 */
-	uint16_t corner_sense_gain_0; /** Sensitivity of corner in dark area.
-					u13.0, [0,8191],
-					default 1000, ineffective 0 */
-	uint16_t corner_sense_gain_1; /** Sensitivity of corner in bright area.
-					u13.0, [0,8191],
-					default 1000, ineffective 0 */
-};
-
-/* Fringe Control configuration.
- *
- *  ISP block: FC2 (FC2 is used with YNR2/YEE2.)
- * (ISP1: FC2 is not used.)
- * (ISP2: FC2 is not for Preview/Video.)
- *  ISP2: FC2 is used for Still.
- */
-struct ia_css_fc_config {
-	uint8_t  gain_exp;   /** Common exponent of gains.
-				u8.0, [0,13],
-				default 1, ineffective 0 */
-	uint16_t coring_pos_0; /** Coring threshold for positive edge in dark area.
-				u0.13, [0,8191],
-				default 0(0), ineffective 0 */
-	uint16_t coring_pos_1; /** Coring threshold for positive edge in bright area.
-				u0.13, [0,8191],
-				default 0(0), ineffective 0 */
-	uint16_t coring_neg_0; /** Coring threshold for negative edge in dark area.
-				u0.13, [0,8191],
-				default 0(0), ineffective 0 */
-	uint16_t coring_neg_1; /** Coring threshold for negative edge in bright area.
-				u0.13, [0,8191],
-				default 0(0), ineffective 0 */
-	uint16_t gain_pos_0; /** Gain for positive edge in dark area.
-				u0.13, [0,8191],
-				default 4096(0.5), ineffective 0 */
-	uint16_t gain_pos_1; /** Gain for positive edge in bright area.
-				u0.13, [0,8191],
-				default 4096(0.5), ineffective 0 */
-	uint16_t gain_neg_0; /** Gain for negative edge in dark area.
-				u0.13, [0,8191],
-				default 4096(0.5), ineffective 0 */
-	uint16_t gain_neg_1; /** Gain for negative edge in bright area.
-				u0.13, [0,8191],
-				default 4096(0.5), ineffective 0 */
-	uint16_t crop_pos_0; /** Limit for positive edge in dark area.
-				u0.13, [0,8191],
-				default/ineffective 8191(almost 1.0) */
-	uint16_t crop_pos_1; /** Limit for positive edge in bright area.
-				u0.13, [0,8191],
-				default/ineffective 8191(almost 1.0) */
-	int16_t  crop_neg_0; /** Limit for negative edge in dark area.
-				s0.13, [-8192,0],
-				default/ineffective -8192(-1.0) */
-	int16_t  crop_neg_1; /** Limit for negative edge in bright area.
-				s0.13, [-8192,0],
-				default/ineffective -8192(-1.0) */
-};
-
-#endif /* __IA_CSS_YNR2_TYPES_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr_param.h
deleted file mode 100644
index 48fb7d22d7c1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr_param.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_YNRX_PARAM_H
-#define __IA_CSS_YNRX_PARAM_H
-
-#include "ia_css_ynr2_param.h"
-
-#endif /* __IA_CSS_YNRX_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr_state.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr_state.h
deleted file mode 100644
index 2516dd3dc12b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/kernels/ynr/ynr_2/ia_css_ynr_state.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IA_CSS_YNR2_STATE_H
-#define __IA_CSS_YNR2_STATE_H
-
-/* Reuse YNR1 states */
-#include "../ynr_1.0/ia_css_ynr_state.h"
-
-#endif /* __IA_CSS_YNR2_STATE_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/input_buf.isp.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/input_buf.isp.h
deleted file mode 100644
index 32714d5870cf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/input_buf.isp.h
+++ /dev/null
@@ -1,73 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _INPUT_BUF_ISP_H_
-#define _INPUT_BUF_ISP_H_
-
-/* Temporary include, since IA_CSS_BINARY_MODE_COPY is still needed */
-#include "sh_css_defs.h"
-#include "isp_const.h" /* MAX_VECTORS_PER_INPUT_LINE */
-
-#define INPUT_BUF_HEIGHT	2 /* double buffer */
-#define INPUT_BUF_LINES		2
-
-#ifndef ENABLE_CONTINUOUS
-#define ENABLE_CONTINUOUS 0
-#endif
-
-/* In continuous mode, the input buffer must be a fixed size for all binaries
- * and at a fixed address since it will be used by the SP. */
-#define EXTRA_INPUT_VECTORS	2 /* For left padding */
-#define MAX_VECTORS_PER_INPUT_LINE_CONT (CEIL_DIV(SH_CSS_MAX_SENSOR_WIDTH, ISP_NWAY) + EXTRA_INPUT_VECTORS)
-
-/* The input buffer should be on a fixed address in vmem, for continuous capture */
-#define INPUT_BUF_ADDR 0x0
-#if (defined(__ISP) && (!defined(MODE) || MODE != IA_CSS_BINARY_MODE_COPY))
-
-#if ENABLE_CONTINUOUS
-typedef struct {
-  tmemvectoru  raw[INPUT_BUF_HEIGHT][INPUT_BUF_LINES][MAX_VECTORS_PER_INPUT_LINE_CONT]; /* 2 bayer lines */
-  /* Two more lines for SP raw copy efficiency */
-#ifndef ENABLE_REDUCED_INPUT_BUFFER
-  /* "Workaround" solution in the case that space needed vmem exceeds the size of the vmem. */
-  /* Since in theory this buffer is not needed for IPU 2.2/2.3,  */
-  /* the workaround solution will not be needed (and the whole buffer) after the code refactoring. */
-  tmemvectoru _raw[INPUT_BUF_HEIGHT][INPUT_BUF_LINES][MAX_VECTORS_PER_INPUT_LINE_CONT]; /* 2 bayer lines */
-#endif
-} input_line_type;
-#else /* ENABLE CONTINUOUS == 0 */
-typedef struct {
-  tmemvectoru  raw[INPUT_BUF_HEIGHT][INPUT_BUF_LINES][MAX_VECTORS_PER_INPUT_LINE]; /* 2 bayer lines */
-} input_line_type;
-#endif /* ENABLE_CONTINUOUS */
-
-#endif /*MODE*/
-
-#endif /* _INPUT_BUF_ISP_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/isp_const.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/isp_const.h
deleted file mode 100644
index 2f215dc2ac32..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/isp_const.h
+++ /dev/null
@@ -1,482 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _COMMON_ISP_CONST_H_
-#define _COMMON_ISP_CONST_H_
-
-/*#include "isp.h"*/	/* ISP_VEC_NELEMS */
-
-/* Binary independent constants */
-
-#ifndef NO_HOIST
-#  define		NO_HOIST 	HIVE_ATTRIBUTE (( no_hoist ))
-#endif
-
-#define NO_HOIST_CSE HIVE_ATTRIBUTE ((no_hoist, no_cse))
-
-#define UNION struct /* Union constructors not allowed in C++ */
-
-/* ISP binary identifiers.
-   These determine the order in which the binaries are looked up, do not change
-   this!
-   Also, the SP firmware uses this same order (isp_loader.hive.c).
-   Also, gen_firmware.c uses this order in its firmware_header.
-*/
-/* The binary id is used in pre-processor expressions so we cannot
- * use an enum here. */
- /* 24xx pipelines*/
-#define SH_CSS_BINARY_ID_COPY                      0
-#define SH_CSS_BINARY_ID_BAYER_DS                  1
-#define SH_CSS_BINARY_ID_VF_PP_FULL                2
-#define SH_CSS_BINARY_ID_VF_PP_OPT                 3
-#define SH_CSS_BINARY_ID_YUV_SCALE                 4
-#define SH_CSS_BINARY_ID_CAPTURE_PP                5
-#define SH_CSS_BINARY_ID_PRE_ISP                   6
-#define SH_CSS_BINARY_ID_PRE_ISP_ISP2              7
-#define SH_CSS_BINARY_ID_GDC                       8
-#define SH_CSS_BINARY_ID_POST_ISP                  9
-#define SH_CSS_BINARY_ID_POST_ISP_ISP2            10
-#define SH_CSS_BINARY_ID_ANR                      11
-#define SH_CSS_BINARY_ID_ANR_ISP2                 12
-#define SH_CSS_BINARY_ID_PREVIEW_CONT_DS          13
-#define SH_CSS_BINARY_ID_PREVIEW_DS               14
-#define SH_CSS_BINARY_ID_PREVIEW_DEC              15
-#define SH_CSS_BINARY_ID_PREVIEW_CONT_BDS125_ISP2 16
-#define SH_CSS_BINARY_ID_PREVIEW_CONT_DPC_BDS150_ISP2 17
-#define SH_CSS_BINARY_ID_PREVIEW_CONT_BDS150_ISP2 18
-#define SH_CSS_BINARY_ID_PREVIEW_CONT_DPC_BDS200_ISP2 19
-#define SH_CSS_BINARY_ID_PREVIEW_CONT_BDS200_ISP2 20
-#define SH_CSS_BINARY_ID_PREVIEW_DZ               21
-#define SH_CSS_BINARY_ID_PREVIEW_DZ_ISP2          22
-#define SH_CSS_BINARY_ID_PRIMARY_DS               23
-#define SH_CSS_BINARY_ID_PRIMARY_VAR              24
-#define SH_CSS_BINARY_ID_PRIMARY_VAR_ISP2         25
-#define SH_CSS_BINARY_ID_PRIMARY_SMALL            26
-#define SH_CSS_BINARY_ID_PRIMARY_STRIPED          27
-#define SH_CSS_BINARY_ID_PRIMARY_STRIPED_ISP2     28
-#define SH_CSS_BINARY_ID_PRIMARY_8MP              29
-#define SH_CSS_BINARY_ID_PRIMARY_14MP             30
-#define SH_CSS_BINARY_ID_PRIMARY_16MP             31
-#define SH_CSS_BINARY_ID_PRIMARY_REF              32
-#define SH_CSS_BINARY_ID_PRIMARY_ISP261_STAGE0        33
-#define SH_CSS_BINARY_ID_PRIMARY_ISP261_STAGE1        34
-#define SH_CSS_BINARY_ID_PRIMARY_ISP261_STAGE2        35
-#define SH_CSS_BINARY_ID_PRIMARY_ISP261_STAGE3        36
-#define SH_CSS_BINARY_ID_PRIMARY_ISP261_STAGE4        37
-#define SH_CSS_BINARY_ID_PRIMARY_ISP261_STAGE5        38
-#define SH_CSS_BINARY_ID_VIDEO_OFFLINE            39
-#define SH_CSS_BINARY_ID_VIDEO_DS                 40
-#define SH_CSS_BINARY_ID_VIDEO_YUV_DS             41
-#define SH_CSS_BINARY_ID_VIDEO_DZ                 42
-#define SH_CSS_BINARY_ID_VIDEO_DZ_2400_ONLY       43
-#define SH_CSS_BINARY_ID_VIDEO_HIGH               44
-#define SH_CSS_BINARY_ID_VIDEO_NODZ               45
-#define SH_CSS_BINARY_ID_VIDEO_CONT_MULTIBDS_ISP2_MIN 46
-#define SH_CSS_BINARY_ID_VIDEO_CONT_BDS_300_600_ISP2_MIN 47
-#define SH_CSS_BINARY_ID_VIDEO_CONT_DPC_BDS150_ISP2_MIN 48
-#define SH_CSS_BINARY_ID_VIDEO_CONT_BDS150_ISP2_MIN   49
-#define SH_CSS_BINARY_ID_VIDEO_CONT_DPC_BDS200_ISP2_MIN   50
-#define SH_CSS_BINARY_ID_VIDEO_CONT_BDS200_ISP2_MIN   51
-#define SH_CSS_BINARY_ID_VIDEO_CONT_NOBDS_ISP2_MIN    52
-#define SH_CSS_BINARY_ID_VIDEO_DZ_ISP2_MIN      53
-#define SH_CSS_BINARY_ID_VIDEO_DZ_ISP2          54
-#define SH_CSS_BINARY_ID_VIDEO_LP_ISP2          55
-#define SH_CSS_BINARY_ID_RESERVED1              56
-#define SH_CSS_BINARY_ID_ACCELERATION           57
-#define SH_CSS_BINARY_ID_PRE_DE_ISP2            58
-#define SH_CSS_BINARY_ID_KERNEL_TEST_LOAD_STORE 59
-#define SH_CSS_BINARY_ID_CAPTURE_PP_BLI         60
-#define SH_CSS_BINARY_ID_CAPTURE_PP_LDC         61
-#ifdef ISP2401
-#define SH_CSS_BINARY_ID_PRIMARY_STRIPED_ISP2_XNR      62
-#endif
-
-/* skycam kerneltest pipelines */
-#ifndef ISP2401
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_NORM              120
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_NORM_STRIPED      121
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_LIN               122
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_LIN_STRIPED       123
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_SHD           124
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_SHD_STRIPED   125
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_AWB           126
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_3A                127
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_3A_STRIPED        128
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_AF            129
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OBGRID            130
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_BAYER_DENOISE       131
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_BAYER_DENOISE_STRIPED 132
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_DEMOSAIC            133
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_YUVP1_C0            134
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_YUVP2               135
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_REF               136
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_REF_STRIPED       137
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_XNR_REF           138
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_DVS               139
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_XNR               140
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_XNR_STRIPED       141
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_XNR_BLENDING      142
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_TNR_BLOCK         143
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_AE            144
-#define SH_CSS_BINARY_ID_VIDEO_RAW                          145
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_AWB_FR        146
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_DM_RGBPP          147
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_DM_RGBPP_STRIPED  148
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_ANR                 149
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_IF                150
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_IF_STRIPED        151
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OUTPUT_SYSTEM     152
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_TNR_STRIPED       153
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_DVS_STRIPED       154
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OBGRID_STRIPED    155
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_COPY_YUV          156
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_COPY_YUV_BLOCK    157
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_COPY_YUV16_BLOCK  158
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_COPY_YUV16_STRIPED 159
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_COPY_BLOCK_STRIPED 160
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_INPUT_YUV         161
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OUTPUT_YUV        162
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OUTPUT_YUV_16     163
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OUTPUT_SPLIT      164
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OUTPUT_SYSTEM_STRIPED 165
-
-#else
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_NORM              121
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_NORM_STRIPED      122
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OBGRID            123
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OBGRID_STRIPED    124
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_LIN               125
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_LIN_STRIPED       126
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_SHD           127
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_SHD_STRIPED   128
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_AE            129
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_AWB           130
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_AF            131
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_ACC_AWB_FR        132
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_3A                133
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_3A_STRIPED        134
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_BAYER_DENOISE       135
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_BAYER_DENOISE_STRIPED 136
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_ANR                 137
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_ANR_STRIPED         138
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_DEMOSAIC            139
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_DM_RGBPP          140
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_DM_RGBPP_STRIPED  141
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_YUVP1_C0            142
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_YUVP2               143
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_YUVP2_STRIPED       144
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_XNR_REF           145
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_XNR               146
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_XNR_STRIPED       147
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_XNR_BLENDING      148
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_REF               149
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_REF_STRIPED       150
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_DVS               151
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_DVS_STRIPED       152
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_DVS_STAT_C0         153
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_TNR_BLOCK         154
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_TNR_STRIPED       155
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OUTPUT_SYSTEM     156
-#define SH_CSS_BINARY_ID_VIDEO_RAW                          157
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_COPY_YUV          158
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_COPY_YUV_BLOCK    159
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_COPY_YUV16_BLOCK  160
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_COPY_YUV16_STRIPED 161
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_COPY_BLOCK_STRIPED 162
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_INPUT_YUV         163
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OUTPUT_YUV        164
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OUTPUT_YUV_16     165
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OUTPUT_SPLIT      166
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_OUTPUT_SYSTEM_STRIPED 167
-#define SH_CSS_BINARY_ID_COPY_KERNELTEST_OUTPUT_SYSTEM      168
-#endif
-
-/* skycam partial test pipelines*/
-#ifndef ISP2401
-#define SH_CSS_BINARY_ID_IF_TO_DPC                          201
-#define SH_CSS_BINARY_ID_IF_TO_BDS                          202
-#else
-#define SH_CSS_BINARY_ID_IF_TO_BDS                          201
-#define SH_CSS_BINARY_ID_IF_TO_BDS_STRIPED                  202
-#endif
-#define SH_CSS_BINARY_ID_IF_TO_NORM                         203
-#ifndef ISP2401
-#define SH_CSS_BINARY_ID_IF_TO_OB                           204
-#define SH_CSS_BINARY_ID_IF_TO_LIN                          205
-#define SH_CSS_BINARY_ID_IF_TO_SHD                          206
-#define SH_CSS_BINARY_ID_IF_TO_BNR                          207
-#define SH_CSS_BINARY_ID_IF_TO_RGBPP_NV12_16                208
-#define SH_CSS_BINARY_ID_IF_TO_RGBPP                        210
-#define SH_CSS_BINARY_ID_IF_TO_YUVP1                        211
-#define SH_CSS_BINARY_ID_IF_TO_DM                           214
-#define SH_CSS_BINARY_ID_IF_TO_YUVP2_C0                     216
-#define SH_CSS_BINARY_ID_IF_TO_YUVP2_ANR_VIA_ISP            217
-#define SH_CSS_BINARY_ID_VIDEO_IF_TO_DVS                    218
-#define SH_CSS_BINARY_ID_VIDEO_IF_TO_TNR                    219
-#define SH_CSS_BINARY_ID_IF_TO_BDS_STRIPED                  224
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_ANR_STRIPED         225
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_YUVP2_STRIPED       227
-#define SH_CSS_BINARY_ID_IF_TO_BDS_RGBP_DVS_STAT_C0         228
-#define SH_CSS_BINARY_ID_IF_TO_BDS_RGBP_DVS_STAT_C0_STRIPED 229
-#define SH_CSS_BINARY_ID_IF_TO_REF                          236
-#define SH_CSS_BINARY_ID_IF_TO_DVS_STRIPED                  237
-#define SH_CSS_BINARY_ID_IF_TO_YUVP2_STRIPED                238
-#define SH_CSS_BINARY_ID_IF_TO_YUVP1_STRIPED                239
-#define SH_CSS_BINARY_ID_IF_TO_RGBPP_STRIPED                240
-#define SH_CSS_BINARY_ID_IF_TO_ANR_STRIPED                  241
-#define SH_CSS_BINARY_ID_IF_TO_BNR_STRIPED                  242
-#define SH_CSS_BINARY_ID_IF_TO_SHD_STRIPED                  243
-#define SH_CSS_BINARY_ID_IF_TO_LIN_STRIPED                  244
-#define SH_CSS_BINARY_ID_IF_TO_OB_STRIPED                   245
-#define SH_CSS_BINARY_ID_IF_TO_NORM_STRIPED                 248
-#define SH_CSS_BINARY_ID_COPY_KERNELTEST_OUTPUT_SYSTEM      253
-#define SH_CSS_BINARY_ID_IF_TO_XNR                          256
-#define SH_CSS_BINARY_ID_IF_TO_XNR_STRIPED                  257
-#define SH_CSS_BINARY_ID_IF_TO_REF_STRIPED                  258
-#define SH_CSS_BINARY_ID_VIDEO_IF_TO_OSYS                   259
-#define SH_CSS_BINARY_ID_IF_TO_YUVP1_C0                     262
-#define SH_CSS_BINARY_ID_IF_TO_XNR_PRIMARY                  263
-#define SH_CSS_BINARY_ID_IF_TO_XNR_PRIMARY_STRIPED          264
-#define SH_CSS_BINARY_ID_IF_TO_ANR                          265
-#define SH_CSS_BINARY_ID_VIDEO_TEST_ACC_DVS_STAT_C0         266
-#define SH_CSS_BINARY_ID_VIDEO_IF_TO_OSYS_STRIPED           270
-#define SH_CSS_BINARY_ID_IF_TO_OSYS_PRIMARY                 276
-#define SH_CSS_BINARY_ID_IF_TO_OSYS_PRIMARY_STRIPED         277
-#define SH_CSS_BINARY_ID_IF_TO_YUVP1_C0_STRIPED             278
-#else
-#define SH_CSS_BINARY_ID_IF_TO_NORM_STRIPED                 204
-#define SH_CSS_BINARY_ID_IF_TO_OB                           205
-#define SH_CSS_BINARY_ID_IF_TO_OB_STRIPED                   206
-#define SH_CSS_BINARY_ID_IF_TO_LIN                          207
-#define SH_CSS_BINARY_ID_IF_TO_LIN_STRIPED                  208
-#define SH_CSS_BINARY_ID_IF_TO_SHD                          209
-#define SH_CSS_BINARY_ID_IF_TO_SHD_STRIPED                  210
-#define SH_CSS_BINARY_ID_IF_TO_BNR                          211
-#define SH_CSS_BINARY_ID_IF_TO_BNR_STRIPED                  212
-#define SH_CSS_BINARY_ID_IF_TO_ANR                          213
-#define SH_CSS_BINARY_ID_IF_TO_ANR_STRIPED                  214
-#define SH_CSS_BINARY_ID_IF_TO_DM                           215
-#define SH_CSS_BINARY_ID_IF_TO_BDS_RGBP_DVS_STAT_C0         216
-#define SH_CSS_BINARY_ID_IF_TO_BDS_RGBP_DVS_STAT_C0_STRIPED 217
-#define SH_CSS_BINARY_ID_IF_TO_RGBPP                        218
-#define SH_CSS_BINARY_ID_IF_TO_RGBPP_NV12_16                219
-#define SH_CSS_BINARY_ID_IF_TO_RGBPP_STRIPED                220
-#define SH_CSS_BINARY_ID_IF_TO_YUVP1                        221
-#define SH_CSS_BINARY_ID_IF_TO_YUVP1_STRIPED                222
-#define SH_CSS_BINARY_ID_IF_TO_YUVP1_C0                     223
-#define SH_CSS_BINARY_ID_IF_TO_YUVP2_C0                     224
-#define SH_CSS_BINARY_ID_IF_TO_YUVP2_STRIPED                225
-#define SH_CSS_BINARY_ID_IF_TO_XNR                          226
-#define SH_CSS_BINARY_ID_IF_TO_XNR_STRIPED                  227
-#define SH_CSS_BINARY_ID_IF_TO_XNR_PRIMARY                  228
-#define SH_CSS_BINARY_ID_IF_TO_XNR_PRIMARY_STRIPED          229
-#define SH_CSS_BINARY_ID_IF_TO_REF                          230
-#define SH_CSS_BINARY_ID_IF_TO_REF_STRIPED                  231
-#define SH_CSS_BINARY_ID_VIDEO_IF_TO_DVS                    232
-#define SH_CSS_BINARY_ID_IF_TO_DVS_STRIPED                  233
-#define SH_CSS_BINARY_ID_VIDEO_IF_TO_TNR                    234
-#define SH_CSS_BINARY_ID_VIDEO_IF_TO_OSYS                   235
-#define SH_CSS_BINARY_ID_VIDEO_IF_TO_OSYS_STRIPED           236
-#define SH_CSS_BINARY_ID_IF_TO_OSYS_PRIMARY                 237
-#define SH_CSS_BINARY_ID_IF_TO_OSYS_PRIMARY_STRIPED         238
-#define SH_CSS_BINARY_ID_IF_TO_YUVP1_C0_STRIPED             239
-#define SH_CSS_BINARY_ID_VIDEO_YUVP1_TO_OSYS                240
-#define SH_CSS_BINARY_ID_IF_TO_OSYS_PREVIEW                 241
-#define SH_CSS_BINARY_ID_IF_TO_OSYS_PREVIEW_STRIPED         242
-#endif
-
-/* Skycam IR camera binaries */
-#ifndef ISP2401
-#define SH_CSS_BINARY_ID_IR_IF_TO_OSYS_NO_XNR               300
-#define SH_CSS_BINARY_ID_VIDEO_IR_IF_TO_OSYS_NO_DVS_NO_TNR_NO_XNR    301
-#define SH_CSS_BINARY_ID_IR_IF_TO_OSYS_NO_XNR_NO_DVS_PRIMARY         302
-#else
-#define SH_CSS_BINARY_ID_IR_IF_TO_OSYS                      300
-#define SH_CSS_BINARY_ID_IR_IF_TO_OSYS_NO_TNR3              301
-#define SH_CSS_BINARY_ID_IR_IF_TO_OSYS_PRIMARY              302
-
-/* Binaries under development */
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_TNR3              401
-#define SH_CSS_BINARY_ID_VIDEO_KERNELTEST_TNR3_STRIPED      402
-
-#endif
-
-#define XMEM_WIDTH_BITS              HIVE_ISP_DDR_WORD_BITS
-#define XMEM_SHORTS_PER_WORD         (HIVE_ISP_DDR_WORD_BITS/16)
-#define XMEM_INTS_PER_WORD           (HIVE_ISP_DDR_WORD_BITS/32)
-#define XMEM_POW2_BYTES_PER_WORD      HIVE_ISP_DDR_WORD_BYTES
-
-#define BITS8_ELEMENTS_PER_XMEM_ADDR    CEIL_DIV(XMEM_WIDTH_BITS, 8)
-#define BITS16_ELEMENTS_PER_XMEM_ADDR    CEIL_DIV(XMEM_WIDTH_BITS, 16)
-
-#if ISP_VEC_NELEMS == 64
-#define ISP_NWAY_LOG2  6
-#elif ISP_VEC_NELEMS == 32
-#define ISP_NWAY_LOG2  5
-#elif ISP_VEC_NELEMS == 16
-#define ISP_NWAY_LOG2  4
-#elif ISP_VEC_NELEMS == 8
-#define ISP_NWAY_LOG2  3
-#else
-#error "isp_const.h ISP_VEC_NELEMS must be one of {8, 16, 32, 64}"
-#endif
-
-/* *****************************
- * ISP input/output buffer sizes
- * ****************************/
-/* input image */
-#define INPUT_BUF_DMA_HEIGHT          2
-#define INPUT_BUF_HEIGHT              2 /* double buffer */
-#define OUTPUT_BUF_DMA_HEIGHT         2
-#define OUTPUT_BUF_HEIGHT             2 /* double buffer */
-#define OUTPUT_NUM_TRANSFERS	      4
-
-/* GDC accelerator: Up/Down Scaling */
-/* These should be moved to the gdc_defs.h in the device */
-#define UDS_SCALING_N                 HRT_GDC_N
-/* AB: This should cover the zooming up to 16MP */
-#define UDS_MAX_OXDIM                 5000
-/* We support maximally 2 planes with different parameters
-       - luma and chroma (YUV420) */
-#define UDS_MAX_PLANES                2
-#define UDS_BLI_BLOCK_HEIGHT          2
-#define UDS_BCI_BLOCK_HEIGHT          4
-#define UDS_BLI_INTERP_ENVELOPE       1
-#define UDS_BCI_INTERP_ENVELOPE       3
-#define UDS_MAX_ZOOM_FAC              64
-/* Make it always one FPGA vector.
-   Four FPGA vectors are required and
-   four of them fit in one ASIC vector.*/
-#define UDS_MAX_CHUNKS                16
-
-#define ISP_LEFT_PADDING	_ISP_LEFT_CROP_EXTRA(ISP_LEFT_CROPPING)
-#define ISP_LEFT_PADDING_VECS	CEIL_DIV(ISP_LEFT_PADDING, ISP_VEC_NELEMS)
-/* in case of continuous the croppong of the current binary doesn't matter for the buffer calculation, but the cropping of the sp copy should be used */
-#define ISP_LEFT_PADDING_CONT	_ISP_LEFT_CROP_EXTRA(SH_CSS_MAX_LEFT_CROPPING)
-#define ISP_LEFT_PADDING_VECS_CONT	CEIL_DIV(ISP_LEFT_PADDING_CONT, ISP_VEC_NELEMS)
-
-#define CEIL_ROUND_DIV_STRIPE(width, stripe, padding) \
-	CEIL_MUL(padding + CEIL_DIV(width - padding, stripe), ((ENABLE_RAW_BINNING || ENABLE_FIXED_BAYER_DS)?4:2))
-
-/* output (Y,U,V) image, 4:2:0 */
-#define MAX_VECTORS_PER_LINE \
-	CEIL_ROUND_DIV_STRIPE(CEIL_DIV(ISP_MAX_INTERNAL_WIDTH, ISP_VEC_NELEMS), \
-			      ISP_NUM_STRIPES, \
-			      ISP_LEFT_PADDING_VECS)
-
-/*
- * ITERATOR_VECTOR_INCREMENT' explanation:
- * when striping an even number of iterations, one of the stripes is
- * one iteration wider than the other to account for overlap
- * so the calc for the output buffer vmem size is:
- * ((width[vectors]/num_of_stripes) + 2[vectors])
- */
-#define MAX_VECTORS_PER_OUTPUT_LINE \
-	CEIL_DIV(CEIL_DIV(ISP_MAX_OUTPUT_WIDTH, ISP_NUM_STRIPES) + ISP_LEFT_PADDING, ISP_VEC_NELEMS)
-
-/* Must be even due to interlaced bayer input */
-#define MAX_VECTORS_PER_INPUT_LINE	CEIL_MUL((CEIL_DIV(ISP_MAX_INPUT_WIDTH, ISP_VEC_NELEMS) + ISP_LEFT_PADDING_VECS), 2)
-#define MAX_VECTORS_PER_INPUT_STRIPE	CEIL_ROUND_DIV_STRIPE(MAX_VECTORS_PER_INPUT_LINE, \
-							      ISP_NUM_STRIPES, \
-							      ISP_LEFT_PADDING_VECS)
-
-
-/* Add 2 for left croppping */
-#define MAX_SP_RAW_COPY_VECTORS_PER_INPUT_LINE	(CEIL_DIV(ISP_MAX_INPUT_WIDTH, ISP_VEC_NELEMS) + 2)
-
-#define MAX_VECTORS_PER_BUF_LINE \
-	(MAX_VECTORS_PER_LINE + DUMMY_BUF_VECTORS)
-#define MAX_VECTORS_PER_BUF_INPUT_LINE \
-	(MAX_VECTORS_PER_INPUT_STRIPE + DUMMY_BUF_VECTORS)
-#define MAX_OUTPUT_Y_FRAME_WIDTH \
-	(MAX_VECTORS_PER_LINE * ISP_VEC_NELEMS)
-#define MAX_OUTPUT_Y_FRAME_SIMDWIDTH \
-	MAX_VECTORS_PER_LINE
-#define MAX_OUTPUT_C_FRAME_WIDTH \
-	(MAX_OUTPUT_Y_FRAME_WIDTH / 2)
-#define MAX_OUTPUT_C_FRAME_SIMDWIDTH \
-	CEIL_DIV(MAX_OUTPUT_C_FRAME_WIDTH, ISP_VEC_NELEMS)
-
-/* should be even */
-#define NO_CHUNKING (OUTPUT_NUM_CHUNKS == 1)
-
-#define MAX_VECTORS_PER_CHUNK \
-	(NO_CHUNKING ? MAX_VECTORS_PER_LINE \
-				: 2*CEIL_DIV(MAX_VECTORS_PER_LINE, \
-					     2*OUTPUT_NUM_CHUNKS))
-
-#define MAX_C_VECTORS_PER_CHUNK \
-	(MAX_VECTORS_PER_CHUNK/2)
-
-/* should be even */
-#define MAX_VECTORS_PER_OUTPUT_CHUNK \
-	(NO_CHUNKING ? MAX_VECTORS_PER_OUTPUT_LINE \
-				: 2*CEIL_DIV(MAX_VECTORS_PER_OUTPUT_LINE, \
-					     2*OUTPUT_NUM_CHUNKS))
-
-#define MAX_C_VECTORS_PER_OUTPUT_CHUNK \
-	(MAX_VECTORS_PER_OUTPUT_CHUNK/2)
-
-
-
-/* should be even */
-#define MAX_VECTORS_PER_INPUT_CHUNK \
-	(INPUT_NUM_CHUNKS == 1 ? MAX_VECTORS_PER_INPUT_STRIPE \
-			       : 2*CEIL_DIV(MAX_VECTORS_PER_INPUT_STRIPE, \
-					    2*OUTPUT_NUM_CHUNKS))
-
-#define DEFAULT_C_SUBSAMPLING      2
-
-/****** DMA buffer properties */
-
-#define RAW_BUF_LINES ((ENABLE_RAW_BINNING || ENABLE_FIXED_BAYER_DS) ? 4 : 2)
-
-#define RAW_BUF_STRIDE \
-	(BINARY_ID == SH_CSS_BINARY_ID_POST_ISP ? MAX_VECTORS_PER_INPUT_CHUNK : \
-	 ISP_NUM_STRIPES > 1 ? MAX_VECTORS_PER_INPUT_STRIPE+_ISP_EXTRA_PADDING_VECS : \
-	 !ENABLE_CONTINUOUS ? MAX_VECTORS_PER_INPUT_LINE : \
-	 MAX_VECTORS_PER_INPUT_CHUNK)
-
-/* [isp vmem] table size[vectors] per line per color (GR,R,B,GB),
-   multiples of NWAY */
-#define SCTBL_VECTORS_PER_LINE_PER_COLOR \
-	CEIL_DIV(SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR, ISP_VEC_NELEMS)
-/* [isp vmem] table size[vectors] per line for 4colors (GR,R,B,GB),
-   multiples of NWAY */
-#define SCTBL_VECTORS_PER_LINE \
-	(SCTBL_VECTORS_PER_LINE_PER_COLOR * IA_CSS_SC_NUM_COLORS)
-
-/*************/
-
-/* Format for fixed primaries */
-
-#define ISP_FIXED_PRIMARY_FORMAT IA_CSS_FRAME_FORMAT_NV12
-
-#endif /* _COMMON_ISP_CONST_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/isp_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/isp_types.h
deleted file mode 100644
index 37a7d28f6d9f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/isp/modes/interface/isp_types.h
+++ /dev/null
@@ -1,128 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _ISP_TYPES_H_
-#define _ISP_TYPES_H_
-
-/* Workaround: hivecc complains about "tag "sh_css_3a_output" already declared"
-   without this extra decl. */
-struct ia_css_3a_output;
-
-#if defined(__ISP)
-struct isp_uds_config {
-	int      hive_dx;
-	int      hive_dy;
-	unsigned hive_woix;
-	unsigned hive_bpp; /* gdc_bits_per_pixel */
-	unsigned hive_bci;
-};
-
-struct s_isp_gdcac_config {
-	unsigned nbx;
-	unsigned nby;
-};
-
-/* output.hive.c request information */
-typedef enum {
-  output_y_channel,
-  output_c_channel,
-  OUTPUT_NUM_CHANNELS
-} output_channel_type;
-
-typedef struct s_output_dma_info {
-  unsigned            cond;		/* Condition for transfer */
-  output_channel_type channel_type;
-  dma_channel         channel;
-  unsigned            width_a;
-  unsigned            width_b;
-  unsigned            stride;
-  unsigned            v_delta;	        /* Offset for v address to do cropping */
-  char               *x_base;           /* X base address */
-} output_dma_info_type;
-#endif
-
-/* Input stream formats, these correspond to the MIPI formats and the way
- * the CSS receiver sends these to the input formatter.
- * The bit depth of each pixel element is stored in the global variable
- * isp_bits_per_pixel.
- * NOTE: for rgb565, we set isp_bits_per_pixel to 565, for all other rgb
- * formats it's the actual depth (4, for 444, 8 for 888 etc).
- */
-enum sh_stream_format {
-	sh_stream_format_yuv420_legacy,
-	sh_stream_format_yuv420,
-	sh_stream_format_yuv422,
-	sh_stream_format_rgb,
-	sh_stream_format_raw,
-	sh_stream_format_binary,	/* bytestream such as jpeg */
-};
-
-struct s_isp_frames {
-	/* global variables that are written to by either the SP or the host,
-	   every ISP binary needs these. */
-	/* output frame */
-	char *xmem_base_addr_y;
-	char *xmem_base_addr_uv;
-	char *xmem_base_addr_u;
-	char *xmem_base_addr_v;
-	/* 2nd output frame */
-	char *xmem_base_addr_second_out_y;
-	char *xmem_base_addr_second_out_u;
-	char *xmem_base_addr_second_out_v;
-	/* input yuv frame */
-	char *xmem_base_addr_y_in;
-	char *xmem_base_addr_u_in;
-	char *xmem_base_addr_v_in;
-	/* input raw frame */
-	char *xmem_base_addr_raw;
-	/* output raw frame */
-	char *xmem_base_addr_raw_out;
-	/* viewfinder output (vf_veceven) */
-	char *xmem_base_addr_vfout_y;
-	char *xmem_base_addr_vfout_u;
-	char *xmem_base_addr_vfout_v;
-	/* overlay frame (for vf_pp) */
-	char *xmem_base_addr_overlay_y;
-	char *xmem_base_addr_overlay_u;
-	char *xmem_base_addr_overlay_v;
-	/* pre-gdc output frame (gdc input) */
-	char *xmem_base_addr_qplane_r;
-	char *xmem_base_addr_qplane_ratb;
-	char *xmem_base_addr_qplane_gr;
-	char *xmem_base_addr_qplane_gb;
-	char *xmem_base_addr_qplane_b;
-	char *xmem_base_addr_qplane_batr;
-	/* YUV as input, used by postisp binary */
-	char *xmem_base_addr_yuv_16_y;
-	char *xmem_base_addr_yuv_16_u;
-	char *xmem_base_addr_yuv_16_v;
-};
-
-#endif /* _ISP_TYPES_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/memory_realloc.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/memory_realloc.c
deleted file mode 100644
index 6512a1ceb9d3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/memory_realloc.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#include "memory_realloc.h"
-#include "ia_css_debug.h"
-#include "ia_css_refcount.h"
-#include "memory_access.h"
-
-static bool realloc_isp_css_mm_buf(
-	hrt_vaddress *curr_buf,
-	size_t *curr_size,
-	size_t needed_size,
-	bool force,
-	enum ia_css_err *err,
-	uint16_t mmgr_attribute);
-
-
-bool reallocate_buffer(
-	hrt_vaddress *curr_buf,
-	size_t *curr_size,
-	size_t needed_size,
-	bool force,
-	enum ia_css_err *err)
-{
-	bool ret;
-	uint16_t	mmgr_attribute = MMGR_ATTRIBUTE_DEFAULT;
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	ret = realloc_isp_css_mm_buf(curr_buf,
-		curr_size, needed_size, force, err, mmgr_attribute);
-
-	IA_CSS_LEAVE_PRIVATE("ret=%d", ret);
-	return ret;
-}
-
-static bool realloc_isp_css_mm_buf(
-	hrt_vaddress *curr_buf,
-	size_t *curr_size,
-	size_t needed_size,
-	bool force,
-	enum ia_css_err *err,
-	uint16_t mmgr_attribute)
-{
-	int32_t id;
-
-	*err = IA_CSS_SUCCESS;
-	/* Possible optimization: add a function sh_css_isp_css_mm_realloc()
-	 * and implement on top of hmm. */
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	if (ia_css_refcount_is_single(*curr_buf) && !force && *curr_size >= needed_size) {
-		IA_CSS_LEAVE_PRIVATE("false");
-		return false;
-	}
-
-	id = IA_CSS_REFCOUNT_PARAM_BUFFER;
-	ia_css_refcount_decrement(id, *curr_buf);
-	*curr_buf = ia_css_refcount_increment(id, mmgr_alloc_attr(needed_size,
-							mmgr_attribute));
-
-	if (!*curr_buf) {
-		*err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		*curr_size = 0;
-	} else {
-		*curr_size = needed_size;
-	}
-	IA_CSS_LEAVE_PRIVATE("true");
-	return true;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/binary/interface/ia_css_binary.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/binary/interface/ia_css_binary.h
deleted file mode 100644
index b62c4d321a4e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/binary/interface/ia_css_binary.h
+++ /dev/null
@@ -1,257 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _IA_CSS_BINARY_H_
-#define _IA_CSS_BINARY_H_
-
-#include <type_support.h>
-#include "ia_css_types.h"
-#include "ia_css_err.h"
-#include "ia_css_stream_format.h"
-#include "ia_css_stream_public.h"
-#include "ia_css_frame_public.h"
-#include "sh_css_metrics.h"
-#include "isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h"
-
-/* The binary mode is used in pre-processor expressions so we cannot
- * use an enum here. */
-#define IA_CSS_BINARY_MODE_COPY       0
-#define IA_CSS_BINARY_MODE_PREVIEW    1
-#define IA_CSS_BINARY_MODE_PRIMARY    2
-#define IA_CSS_BINARY_MODE_VIDEO      3
-#define IA_CSS_BINARY_MODE_PRE_ISP    4
-#define IA_CSS_BINARY_MODE_GDC        5
-#define IA_CSS_BINARY_MODE_POST_ISP   6
-#define IA_CSS_BINARY_MODE_ANR        7
-#define IA_CSS_BINARY_MODE_CAPTURE_PP 8
-#define IA_CSS_BINARY_MODE_VF_PP      9
-#define IA_CSS_BINARY_MODE_PRE_DE     10
-#define IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE0    11
-#define IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE1    12
-#define IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE2    13
-#define IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE3    14
-#define IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE4    15
-#define IA_CSS_BINARY_MODE_PRIMARY_HQ_STAGE5    16
-#define IA_CSS_BINARY_NUM_MODES       17
-
-#define MAX_NUM_PRIMARY_STAGES 6
-#define NUM_PRIMARY_HQ_STAGES  6  /* number of primary stages for ISP2.6.1 high quality pipe */
-#define NUM_PRIMARY_STAGES     1  /* number of primary satges for ISP1/ISP2.2 pipe */
-
-/* Indicate where binaries can read input from */
-#define IA_CSS_BINARY_INPUT_SENSOR   0
-#define IA_CSS_BINARY_INPUT_MEMORY   1
-#define IA_CSS_BINARY_INPUT_VARIABLE 2
-
-/* Should be included without the path.
-   However, that requires adding the path to numerous makefiles
-   that have nothing to do with isp parameters.
- */
-#include "runtime/isp_param/interface/ia_css_isp_param_types.h"
-
-/* now these ports only include output ports but not vf output ports */
-enum {
-	IA_CSS_BINARY_OUTPUT_PORT_0 = 0,
-	IA_CSS_BINARY_OUTPUT_PORT_1 = 1,
-	IA_CSS_BINARY_MAX_OUTPUT_PORTS = 2
-};
-
-struct ia_css_cas_binary_descr {
-	unsigned int num_stage;
-	unsigned int num_output_stage;
-	struct ia_css_frame_info *in_info;
-	struct ia_css_frame_info *internal_out_info;
-	struct ia_css_frame_info *out_info;
-	struct ia_css_frame_info *vf_info;
-	bool *is_output_stage;
-};
-
-struct ia_css_binary_descr {
-	int mode;
-	bool online;
-	bool continuous;
-	bool striped;
-	bool two_ppc;
-	bool enable_yuv_ds;
-	bool enable_high_speed;
-	bool enable_dvs_6axis;
-	bool enable_reduced_pipe;
-	bool enable_dz;
-	bool enable_xnr;
-	bool enable_fractional_ds;
-	bool enable_dpc;
-#ifdef ISP2401
-	bool enable_luma_only;
-	bool enable_tnr;
-#endif
-	bool enable_capture_pp_bli;
-	struct ia_css_resolution dvs_env;
-	enum atomisp_input_format stream_format;
-	struct ia_css_frame_info *in_info;		/* the info of the input-frame with the
-							   ISP required resolution. */
-	struct ia_css_frame_info *bds_out_info;
-	struct ia_css_frame_info *out_info[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	struct ia_css_frame_info *vf_info;
-	unsigned int isp_pipe_version;
-	unsigned int required_bds_factor;
-	int stream_config_left_padding;
-};
-
-struct ia_css_binary {
-	const struct ia_css_binary_xinfo *info;
-	enum atomisp_input_format input_format;
-	struct ia_css_frame_info in_frame_info;
-	struct ia_css_frame_info internal_frame_info;
-	struct ia_css_frame_info out_frame_info[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	struct ia_css_resolution effective_in_frame_res;
-	struct ia_css_frame_info vf_frame_info;
-	int                      input_buf_vectors;
-	int                      deci_factor_log2;
-	int                      vf_downscale_log2;
-	int                      s3atbl_width;
-	int                      s3atbl_height;
-	int                      s3atbl_isp_width;
-	int                      s3atbl_isp_height;
-	unsigned int             morph_tbl_width;
-	unsigned int             morph_tbl_aligned_width;
-	unsigned int             morph_tbl_height;
-	int                      sctbl_width_per_color;
-	int                      sctbl_aligned_width_per_color;
-	int                      sctbl_height;
-#ifdef ISP2401
-	int                      sctbl_legacy_width_per_color;
-	int                      sctbl_legacy_height;
-#endif
-	struct ia_css_sdis_info	 dis;
-	struct ia_css_resolution dvs_envelope;
-	bool                     online;
-	unsigned int             uds_xc;
-	unsigned int             uds_yc;
-	unsigned int             left_padding;
-	struct sh_css_binary_metrics metrics;
-	struct ia_css_isp_param_host_segments mem_params;
-	struct ia_css_isp_param_css_segments  css_params;
-};
-
-#define IA_CSS_BINARY_DEFAULT_SETTINGS \
-(struct ia_css_binary) { \
-	.input_format		= ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY, \
-	.in_frame_info		= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
-	.internal_frame_info	= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
-	.out_frame_info		= {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \
-	.vf_frame_info		= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
-}
-
-enum ia_css_err
-ia_css_binary_init_infos(void);
-
-enum ia_css_err
-ia_css_binary_uninit(void);
-
-enum ia_css_err
-ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
-		 bool online,
-		 bool two_ppc,
-		 enum atomisp_input_format stream_format,
-		 const struct ia_css_frame_info *in_info,
-		 const struct ia_css_frame_info *bds_out_info,
-		 const struct ia_css_frame_info *out_info[],
-		 const struct ia_css_frame_info *vf_info,
-		 struct ia_css_binary *binary,
-		 struct ia_css_resolution *dvs_env,
-		 int stream_config_left_padding,
-		 bool accelerator);
-
-enum ia_css_err
-ia_css_binary_find(struct ia_css_binary_descr *descr,
-		   struct ia_css_binary *binary);
-
-/* @brief Get the shading information of the specified shading correction type.
- *
- * @param[in] binary: The isp binary which has the shading correction.
- * @param[in] type: The shading correction type.
- * @param[in] required_bds_factor: The bayer downscaling factor required in the pipe.
- * @param[in] stream_config: The stream configuration.
-#ifndef ISP2401
- * @param[out] info: The shading information.
-#else
- * @param[out] shading_info: The shading information.
- *		The shading information necessary as API is stored in the shading_info.
-#endif
- *		The driver needs to get this information to generate
-#ifndef ISP2401
- *		the shading table directly required in the isp.
-#else
- *		the shading table directly required from ISP.
- * @param[out] pipe_config: The pipe configuration.
- *		The shading information related to ISP (but, not necessary as API) is stored in the pipe_config.
-#endif
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
- */
-enum ia_css_err
-ia_css_binary_get_shading_info(const struct ia_css_binary *binary,
-			enum ia_css_shading_correction_type type,
-			unsigned int required_bds_factor,
-			const struct ia_css_stream_config *stream_config,
-#ifndef ISP2401
-			struct ia_css_shading_info *info);
-#else
-			struct ia_css_shading_info *shading_info,
-			struct ia_css_pipe_config *pipe_config);
-#endif
-
-enum ia_css_err
-ia_css_binary_3a_grid_info(const struct ia_css_binary *binary,
-			   struct ia_css_grid_info *info,
-			   struct ia_css_pipe *pipe);
-
-void
-ia_css_binary_dvs_grid_info(const struct ia_css_binary *binary,
-			    struct ia_css_grid_info *info,
-			    struct ia_css_pipe *pipe);
-
-void
-ia_css_binary_dvs_stat_grid_info(
-	const struct ia_css_binary *binary,
-	struct ia_css_grid_info *info,
-	struct ia_css_pipe *pipe);
-
-unsigned
-ia_css_binary_max_vf_width(void);
-
-void
-ia_css_binary_destroy_isp_parameters(struct ia_css_binary *binary);
-
-void
-ia_css_binary_get_isp_binaries(struct ia_css_binary_xinfo **binaries,
-	uint32_t *num_isp_binaries);
-
-#endif /* _IA_CSS_BINARY_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/binary/src/binary.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/binary/src/binary.c
deleted file mode 100644
index 0cd6e1da43cf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/binary/src/binary.c
+++ /dev/null
@@ -1,1838 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <math_support.h>
-#include <gdc_device.h>	/* HR_GDC_N */
-#include "isp.h"	/* ISP_VEC_NELEMS */
-
-#include "ia_css_binary.h"
-#include "ia_css_debug.h"
-#include "ia_css_util.h"
-#include "ia_css_isp_param.h"
-#include "sh_css_internal.h"
-#include "sh_css_sp.h"
-#include "sh_css_firmware.h"
-#include "sh_css_defs.h"
-#include "sh_css_legacy.h"
-
-#include "vf/vf_1.0/ia_css_vf.host.h"
-#ifdef ISP2401
-#include "sc/sc_1.0/ia_css_sc.host.h"
-#endif
-#include "sdis/sdis_1.0/ia_css_sdis.host.h"
-#ifdef ISP2401
-#include "fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h"	/* FRAC_ACC */
-#endif
-
-#include "camera/pipe/interface/ia_css_pipe_binarydesc.h"
-
-#include "memory_access.h"
-
-#include "assert_support.h"
-
-#define IMPLIES(a, b)           (!(a) || (b))   /* A => B */
-
-static struct ia_css_binary_xinfo *all_binaries; /* ISP binaries only (no SP) */
-static struct ia_css_binary_xinfo
-	*binary_infos[IA_CSS_BINARY_NUM_MODES] = { NULL, };
-
-static void
-ia_css_binary_dvs_env(const struct ia_css_binary_info *info,
-		      const struct ia_css_resolution *dvs_env,
-		      struct ia_css_resolution *binary_dvs_env)
-{
-	if (info->enable.dvs_envelope) {
-		assert(dvs_env != NULL);
-		binary_dvs_env->width  = max(dvs_env->width, SH_CSS_MIN_DVS_ENVELOPE);
-		binary_dvs_env->height = max(dvs_env->height, SH_CSS_MIN_DVS_ENVELOPE);
-	}
-}
-
-static void
-ia_css_binary_internal_res(const struct ia_css_frame_info *in_info,
-			   const struct ia_css_frame_info *bds_out_info,
-			   const struct ia_css_frame_info *out_info,
-			   const struct ia_css_resolution *dvs_env,
-			   const struct ia_css_binary_info *info,
-			   struct ia_css_resolution *internal_res)
-{
-	unsigned int isp_tmp_internal_width = 0,
-		     isp_tmp_internal_height = 0;
-	bool binary_supports_yuv_ds = info->enable.ds & 2;
-	struct ia_css_resolution binary_dvs_env;
-
-	binary_dvs_env.width = 0;
-	binary_dvs_env.height = 0;
-	ia_css_binary_dvs_env(info, dvs_env, &binary_dvs_env);
-
-	if (binary_supports_yuv_ds) {
-		if (in_info != NULL) {
-			isp_tmp_internal_width = in_info->res.width
-				+ info->pipeline.left_cropping + binary_dvs_env.width;
-			isp_tmp_internal_height = in_info->res.height
-				+ info->pipeline.top_cropping + binary_dvs_env.height;
-		}
-	} else if ((bds_out_info != NULL) && (out_info != NULL) &&
-				/* TODO: hack to make video_us case work. this should be reverted after
-				a nice solution in ISP */
-				(bds_out_info->res.width >= out_info->res.width)) {
-			isp_tmp_internal_width = bds_out_info->padded_width;
-			isp_tmp_internal_height = bds_out_info->res.height;
-	} else {
-		if (out_info != NULL) {
-			isp_tmp_internal_width = out_info->padded_width;
-			isp_tmp_internal_height = out_info->res.height;
-		}
-	}
-
-	/* We first calculate the resolutions used by the ISP. After that,
-	 * we use those resolutions to compute sizes for tables etc. */
-	internal_res->width = __ISP_INTERNAL_WIDTH(isp_tmp_internal_width,
-		(int)binary_dvs_env.width,
-		info->pipeline.left_cropping, info->pipeline.mode,
-		info->pipeline.c_subsampling,
-		info->output.num_chunks, info->pipeline.pipelining);
-	internal_res->height = __ISP_INTERNAL_HEIGHT(isp_tmp_internal_height,
-		info->pipeline.top_cropping,
-		binary_dvs_env.height);
-}
-
-#ifndef ISP2401
-/* Computation results of the origin coordinate of bayer on the shading table. */
-struct sh_css_shading_table_bayer_origin_compute_results {
-	uint32_t bayer_scale_hor_ratio_in;	/* Horizontal ratio (in) of bayer scaling. */
-	uint32_t bayer_scale_hor_ratio_out;	/* Horizontal ratio (out) of bayer scaling. */
-	uint32_t bayer_scale_ver_ratio_in;	/* Vertical ratio (in) of bayer scaling. */
-	uint32_t bayer_scale_ver_ratio_out;	/* Vertical ratio (out) of bayer scaling. */
-	uint32_t sc_bayer_origin_x_bqs_on_shading_table; /* X coordinate (in bqs) of bayer origin on shading table. */
-	uint32_t sc_bayer_origin_y_bqs_on_shading_table; /* Y coordinate (in bqs) of bayer origin on shading table. */
-#else
-/* Requirements for the shading correction. */
-struct sh_css_binary_sc_requirements {
-	/* Bayer scaling factor, for the scaling which is applied before shading correction. */
-	uint32_t bayer_scale_hor_ratio_in;  /* Horizontal ratio (in) of scaling applied BEFORE shading correction. */
-	uint32_t bayer_scale_hor_ratio_out; /* Horizontal ratio (out) of scaling applied BEFORE shading correction. */
-	uint32_t bayer_scale_ver_ratio_in;  /* Vertical ratio (in) of scaling applied BEFORE shading correction. */
-	uint32_t bayer_scale_ver_ratio_out; /* Vertical ratio (out) of scaling applied BEFORE shading correction. */
-
-	/* ISP internal frame is composed of the real sensor data and the padding data. */
-	uint32_t sensor_data_origin_x_bqs_on_internal; /* X origin (in bqs) of sensor data on internal frame
-								at shading correction. */
-	uint32_t sensor_data_origin_y_bqs_on_internal; /* Y origin (in bqs) of sensor data on internal frame
-								at shading correction. */
-#endif
-};
-
-/* Get the requirements for the shading correction. */
-static enum ia_css_err
-#ifndef ISP2401
-ia_css_binary_compute_shading_table_bayer_origin(
-	const struct ia_css_binary *binary,				/* [in] */
-	unsigned int required_bds_factor,				/* [in] */
-	const struct ia_css_stream_config *stream_config,		/* [in] */
-	struct sh_css_shading_table_bayer_origin_compute_results *res)	/* [out] */
-#else
-sh_css_binary_get_sc_requirements(
-	const struct ia_css_binary *binary,			/* [in] */
-	unsigned int required_bds_factor,			/* [in] */
-	const struct ia_css_stream_config *stream_config,	/* [in] */
-	struct sh_css_binary_sc_requirements *scr)		/* [out] */
-#endif
-{
-	enum ia_css_err err;
-
-#ifndef ISP2401
-	/* Numerator and denominator of the fixed bayer downscaling factor.
-	(numerator >= denominator) */
-#else
-	/* Numerator and denominator of the fixed bayer downscaling factor. (numerator >= denominator) */
-#endif
-	unsigned int bds_num, bds_den;
-
-#ifndef ISP2401
-	/* Horizontal/Vertical ratio of bayer scaling
-	between input area and output area. */
-	unsigned int bs_hor_ratio_in;
-	unsigned int bs_hor_ratio_out;
-	unsigned int bs_ver_ratio_in;
-	unsigned int bs_ver_ratio_out;
-#else
-	/* Horizontal/Vertical ratio of bayer scaling between input area and output area. */
-	unsigned int bs_hor_ratio_in, bs_hor_ratio_out, bs_ver_ratio_in, bs_ver_ratio_out;
-#endif
-
-	/* Left padding set by InputFormatter. */
-#ifndef ISP2401
-	unsigned int left_padding_bqs;			/* in bqs */
-#else
-	unsigned int left_padding_bqs;
-#endif
-
-#ifndef ISP2401
-	/* Flag for the NEED_BDS_FACTOR_2_00 macro defined in isp kernels. */
-	unsigned int need_bds_factor_2_00;
-
-	/* Left padding adjusted inside the isp. */
-	unsigned int left_padding_adjusted_bqs;		/* in bqs */
-
-	/* Bad pixels caused by filters.
-	NxN-filter (before/after bayer scaling) moves the image position
-	to right/bottom directions by a few pixels.
-	It causes bad pixels at left/top sides,
-	and effective bayer size decreases. */
-	unsigned int bad_bqs_on_left_before_bs;	/* in bqs */
-	unsigned int bad_bqs_on_left_after_bs;	/* in bqs */
-	unsigned int bad_bqs_on_top_before_bs;	/* in bqs */
-	unsigned int bad_bqs_on_top_after_bs;	/* in bqs */
-
-	/* Get the numerator and denominator of bayer downscaling factor. */
-	err = sh_css_bds_factor_get_numerator_denominator
-		(required_bds_factor, &bds_num, &bds_den);
-	if (err != IA_CSS_SUCCESS)
-#else
-	/* Flags corresponding to NEED_BDS_FACTOR_2_00/NEED_BDS_FACTOR_1_50/NEED_BDS_FACTOR_1_25 macros
-	 * defined in isp kernels. */
-	unsigned int need_bds_factor_2_00, need_bds_factor_1_50, need_bds_factor_1_25;
-
-	/* Left padding adjusted inside the isp kernels. */
-	unsigned int left_padding_adjusted_bqs;
-
-	/* Top padding padded inside the isp kernel for bayer downscaling binaries. */
-	unsigned int top_padding_bqs;
-
-	/* Bayer downscaling factor 1.0 by fixed-point. */
-	int bds_frac_acc = FRAC_ACC;	/* FRAC_ACC is defined in ia_css_fixedbds_param.h. */
-
-	/* Right/Down shift amount caused by filters applied BEFORE shading corrertion. */
-	unsigned int right_shift_bqs_before_bs; /* right shift before bayer scaling */
-	unsigned int right_shift_bqs_after_bs;  /* right shift after bayer scaling */
-	unsigned int down_shift_bqs_before_bs;  /* down shift before bayer scaling */
-	unsigned int down_shift_bqs_after_bs;   /* down shift after bayer scaling */
-
-	/* Origin of the real sensor data area on the internal frame at shading correction. */
-	unsigned int sensor_data_origin_x_bqs_on_internal;
-	unsigned int sensor_data_origin_y_bqs_on_internal;
-
-	IA_CSS_ENTER_PRIVATE("binary=%p, required_bds_factor=%d, stream_config=%p",
-		binary, required_bds_factor, stream_config);
-
-	/* Get the numerator and denominator of the required bayer downscaling factor. */
-	err = sh_css_bds_factor_get_numerator_denominator(required_bds_factor, &bds_num, &bds_den);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-#endif
-		return err;
-#ifdef ISP2401
-	}
-#endif
-
-#ifndef ISP2401
-	/* Set the horizontal/vertical ratio of bayer scaling
-	between input area and output area. */
-#else
-	IA_CSS_LOG("bds_num=%d, bds_den=%d", bds_num, bds_den);
-
-	/* Set the horizontal/vertical ratio of bayer scaling between input area and output area. */
-#endif
-	bs_hor_ratio_in  = bds_num;
-	bs_hor_ratio_out = bds_den;
-	bs_ver_ratio_in  = bds_num;
-	bs_ver_ratio_out = bds_den;
-
-#ifndef ISP2401
-	/* Set the left padding set by InputFormatter. (ifmtr.c) */
-#else
-	/* Set the left padding set by InputFormatter. (ia_css_ifmtr_configure() in ifmtr.c) */
-#endif
-	if (stream_config->left_padding == -1)
-		left_padding_bqs = _ISP_BQS(binary->left_padding);
-	else
-#ifndef ISP2401
-		left_padding_bqs = (unsigned int)((int)ISP_VEC_NELEMS
-			- _ISP_BQS(stream_config->left_padding));
-#else
-		left_padding_bqs = (unsigned int)((int)ISP_VEC_NELEMS - _ISP_BQS(stream_config->left_padding));
-#endif
-
-#ifndef ISP2401
-	/* Set the left padding adjusted inside the isp.
-	When bds_factor 2.00 is needed, some padding is added to left_padding
-	inside the isp, before bayer downscaling. (raw.isp.c)
-	(Hopefully, left_crop/left_padding/top_crop should be defined in css
-	appropriately, depending on bds_factor.)
-	*/
-#else
-	IA_CSS_LOG("stream.left_padding=%d, binary.left_padding=%d, left_padding_bqs=%d",
-		stream_config->left_padding, binary->left_padding, left_padding_bqs);
-
-	/* Set the left padding adjusted inside the isp kernels.
-	 * When the bds_factor isn't 1.00, the left padding size is adjusted inside the isp,
-	 * before bayer downscaling. (scaled_hor_plane_index(), raw_compute_hphase() in raw.isp.c)
-	 */
-#endif
-	need_bds_factor_2_00 = ((binary->info->sp.bds.supported_bds_factors &
-		(PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_00) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_50) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_3_00) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_4_00) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_4_50) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_5_00) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_6_00) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_8_00))) != 0);
-
-#ifndef ISP2401
-	if (need_bds_factor_2_00 && binary->info->sp.pipeline.left_cropping > 0)
-		left_padding_adjusted_bqs = left_padding_bqs + ISP_VEC_NELEMS;
-	else
-#else
-	need_bds_factor_1_50 = ((binary->info->sp.bds.supported_bds_factors &
-		(PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_1_50) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_25) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_3_00) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_4_50) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_6_00))) != 0);
-
-	need_bds_factor_1_25 = ((binary->info->sp.bds.supported_bds_factors &
-		(PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_1_25) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_50) |
-		 PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_5_00))) != 0);
-
-	if (binary->info->sp.pipeline.left_cropping > 0 &&
-	    (need_bds_factor_2_00 || need_bds_factor_1_50 || need_bds_factor_1_25)) {
-		/*
-		 * downscale 2.0  -> first_vec_adjusted_bqs = 128
-		 * downscale 1.5  -> first_vec_adjusted_bqs = 96
-		 * downscale 1.25 -> first_vec_adjusted_bqs = 80
-		 */
-		unsigned int first_vec_adjusted_bqs
-			= ISP_VEC_NELEMS * bs_hor_ratio_in / bs_hor_ratio_out;
-		left_padding_adjusted_bqs = first_vec_adjusted_bqs
-			- _ISP_BQS(binary->info->sp.pipeline.left_cropping);
-	} else
-#endif
-		left_padding_adjusted_bqs = left_padding_bqs;
-
-#ifndef ISP2401
-	/* Currently, the bad pixel caused by filters before bayer scaling
-	is NOT considered, because the bad pixel is subtle.
-	When some large filter is used in the future,
-	we need to consider the bad pixel.
-
-	Currently, when bds_factor isn't 1.00, 3x3 anti-alias filter is applied
-	to each color plane(Gr/R/B/Gb) before bayer downscaling.
-	This filter moves each color plane to right/bottom directions
-	by 1 pixel at the most, depending on downscaling factor.
-	*/
-	bad_bqs_on_left_before_bs = 0;
-	bad_bqs_on_top_before_bs = 0;
-#else
-	IA_CSS_LOG("supported_bds_factors=%d, need_bds_factor:2_00=%d, 1_50=%d, 1_25=%d",
-		binary->info->sp.bds.supported_bds_factors,
-		need_bds_factor_2_00, need_bds_factor_1_50, need_bds_factor_1_25);
-	IA_CSS_LOG("left_cropping=%d, left_padding_adjusted_bqs=%d",
-		binary->info->sp.pipeline.left_cropping, left_padding_adjusted_bqs);
-
-	/* Set the top padding padded inside the isp kernel for bayer downscaling binaries.
-	 * When the bds_factor isn't 1.00, the top padding is padded inside the isp
-	 * before bayer downscaling, because the top cropping size (input margin) is not enough.
-	 * (calculate_input_line(), raw_compute_vphase(), dma_read_raw() in raw.isp.c)
-	 * NOTE: In dma_read_raw(), the factor passed to raw_compute_vphase() is got by get_bds_factor_for_dma_read().
-	 *       This factor is BDS_FPVAL_100/BDS_FPVAL_125/BDS_FPVAL_150/BDS_FPVAL_200.
-	 */
-	top_padding_bqs = 0;
-	if (binary->info->sp.pipeline.top_cropping > 0 &&
-	    (required_bds_factor == SH_CSS_BDS_FACTOR_1_25 ||
-	     required_bds_factor == SH_CSS_BDS_FACTOR_1_50 ||
-	     required_bds_factor == SH_CSS_BDS_FACTOR_2_00)) {
-		/* Calculation from calculate_input_line() and raw_compute_vphase() in raw.isp.c. */
-		int top_cropping_bqs = _ISP_BQS(binary->info->sp.pipeline.top_cropping);
-								/* top cropping (in bqs) */
-		int factor = bds_num * bds_frac_acc / bds_den;	/* downscaling factor by fixed-point */
-		int top_padding_bqsxfrac_acc = (top_cropping_bqs * factor - top_cropping_bqs * bds_frac_acc)
-				+ (2 * bds_frac_acc - factor);	/* top padding by fixed-point (in bqs) */
-
-		top_padding_bqs = (unsigned int)((top_padding_bqsxfrac_acc + bds_frac_acc/2 - 1) / bds_frac_acc);
-	}
-
-	IA_CSS_LOG("top_cropping=%d, top_padding_bqs=%d", binary->info->sp.pipeline.top_cropping, top_padding_bqs);
-
-	/* Set the right/down shift amount caused by filters applied BEFORE bayer scaling,
-	 * which scaling is applied BEFORE shading corrertion.
-	 *
-	 * When the bds_factor isn't 1.00, 3x3 anti-alias filter is applied to each color plane(Gr/R/B/Gb)
-	 * before bayer downscaling.
-	 * This filter shifts each color plane (Gr/R/B/Gb) to right/down directions by 1 pixel.
-	 */
-	right_shift_bqs_before_bs = 0;
-	down_shift_bqs_before_bs = 0;
-#endif
-
-#ifndef ISP2401
-	/* Currently, the bad pixel caused by filters after bayer scaling
-	is NOT considered, because the bad pixel is subtle.
-	When some large filter is used in the future,
-	we need to consider the bad pixel.
-
-	Currently, when DPC&BNR is processed between bayer scaling and
-	shading correction, DPC&BNR moves each color plane to
-	right/bottom directions by 1 pixel.
-	*/
-	bad_bqs_on_left_after_bs = 0;
-	bad_bqs_on_top_after_bs = 0;
-#else
-	if (need_bds_factor_2_00 || need_bds_factor_1_50 || need_bds_factor_1_25) {
-		right_shift_bqs_before_bs = 1;
-		down_shift_bqs_before_bs = 1;
-	}
-
-	IA_CSS_LOG("right_shift_bqs_before_bs=%d, down_shift_bqs_before_bs=%d",
-		right_shift_bqs_before_bs, down_shift_bqs_before_bs);
-
-	/* Set the right/down shift amount caused by filters applied AFTER bayer scaling,
-	 * which scaling is applied BEFORE shading corrertion.
-	 *
-	 * When DPC&BNR is processed between bayer scaling and shading correction,
-	 * DPC&BNR moves each color plane (Gr/R/B/Gb) to right/down directions by 1 pixel.
-	 */
-	right_shift_bqs_after_bs = 0;
-	down_shift_bqs_after_bs = 0;
-#endif
-
-#ifndef ISP2401
-	/* Calculate the origin of bayer (real sensor data area)
-	located on the shading table during the shading correction. */
-	res->sc_bayer_origin_x_bqs_on_shading_table
-		= ((left_padding_adjusted_bqs + bad_bqs_on_left_before_bs)
-		* bs_hor_ratio_out + bs_hor_ratio_in/2) / bs_hor_ratio_in
-		+ bad_bqs_on_left_after_bs;
-			/* "+ bs_hor_ratio_in/2": rounding for division by bs_hor_ratio_in */
-	res->sc_bayer_origin_y_bqs_on_shading_table
-		= (bad_bqs_on_top_before_bs
-		* bs_ver_ratio_out + bs_ver_ratio_in/2) / bs_ver_ratio_in
-		+ bad_bqs_on_top_after_bs;
-			/* "+ bs_ver_ratio_in/2": rounding for division by bs_ver_ratio_in */
-
-	res->bayer_scale_hor_ratio_in  = (uint32_t)bs_hor_ratio_in;
-	res->bayer_scale_hor_ratio_out = (uint32_t)bs_hor_ratio_out;
-	res->bayer_scale_ver_ratio_in  = (uint32_t)bs_ver_ratio_in;
-	res->bayer_scale_ver_ratio_out = (uint32_t)bs_ver_ratio_out;
-#else
-	if (binary->info->mem_offsets.offsets.param->dmem.dp.size != 0) { /* if DPC&BNR is enabled in the binary */
-		right_shift_bqs_after_bs = 1;
-		down_shift_bqs_after_bs = 1;
-	}
-
-	IA_CSS_LOG("right_shift_bqs_after_bs=%d, down_shift_bqs_after_bs=%d",
-		right_shift_bqs_after_bs, down_shift_bqs_after_bs);
-
-	/* Set the origin of the sensor data area on the internal frame at shading correction. */
-	{
-		unsigned int bs_frac = bds_frac_acc;	/* scaling factor 1.0 in fixed point */
-		unsigned int bs_out, bs_in;		/* scaling ratio in fixed point */
-
-		bs_out = bs_hor_ratio_out * bs_frac;
-		bs_in = bs_hor_ratio_in * bs_frac;
-		sensor_data_origin_x_bqs_on_internal
-			= ((left_padding_adjusted_bqs + right_shift_bqs_before_bs) * bs_out + bs_in/2) / bs_in
-				+ right_shift_bqs_after_bs;	/* "+ bs_in/2": rounding */
-
-		bs_out = bs_ver_ratio_out * bs_frac;
-		bs_in = bs_ver_ratio_in * bs_frac;
-		sensor_data_origin_y_bqs_on_internal
-			= ((top_padding_bqs + down_shift_bqs_before_bs) * bs_out + bs_in/2) / bs_in
-				+ down_shift_bqs_after_bs;	/* "+ bs_in/2": rounding */
-	}
-
-	scr->bayer_scale_hor_ratio_in			= (uint32_t)bs_hor_ratio_in;
-	scr->bayer_scale_hor_ratio_out			= (uint32_t)bs_hor_ratio_out;
-	scr->bayer_scale_ver_ratio_in			= (uint32_t)bs_ver_ratio_in;
-	scr->bayer_scale_ver_ratio_out			= (uint32_t)bs_ver_ratio_out;
-	scr->sensor_data_origin_x_bqs_on_internal	= (uint32_t)sensor_data_origin_x_bqs_on_internal;
-	scr->sensor_data_origin_y_bqs_on_internal	= (uint32_t)sensor_data_origin_y_bqs_on_internal;
-
-	IA_CSS_LOG("sc_requirements: %d, %d, %d, %d, %d, %d",
-		scr->bayer_scale_hor_ratio_in, scr->bayer_scale_hor_ratio_out,
-		scr->bayer_scale_ver_ratio_in, scr->bayer_scale_ver_ratio_out,
-		scr->sensor_data_origin_x_bqs_on_internal, scr->sensor_data_origin_y_bqs_on_internal);
-#endif
-
-#ifdef ISP2401
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-#endif
-	return err;
-}
-
-/* Get the shading information of Shading Correction Type 1. */
-static enum ia_css_err
-ia_css_binary_get_shading_info_type_1(const struct ia_css_binary *binary,	/* [in] */
-			unsigned int required_bds_factor,			/* [in] */
-			const struct ia_css_stream_config *stream_config,	/* [in] */
-#ifndef ISP2401
-			struct ia_css_shading_info *info)			/* [out] */
-#else
-			struct ia_css_shading_info *shading_info,		/* [out] */
-			struct ia_css_pipe_config *pipe_config)			/* [out] */
-#endif
-{
-	enum ia_css_err err;
-#ifndef ISP2401
-	struct sh_css_shading_table_bayer_origin_compute_results res;
-#else
-	struct sh_css_binary_sc_requirements scr;
-#endif
-
-#ifndef ISP2401
-	assert(binary != NULL);
-	assert(info != NULL);
-#else
-	uint32_t in_width_bqs, in_height_bqs, internal_width_bqs, internal_height_bqs;
-	uint32_t num_hor_grids, num_ver_grids, bqs_per_grid_cell, tbl_width_bqs, tbl_height_bqs;
-	uint32_t sensor_org_x_bqs_on_internal, sensor_org_y_bqs_on_internal, sensor_width_bqs, sensor_height_bqs;
-	uint32_t sensor_center_x_bqs_on_internal, sensor_center_y_bqs_on_internal;
-	uint32_t left, right, upper, lower;
-	uint32_t adjust_left, adjust_right, adjust_upper, adjust_lower, adjust_width_bqs, adjust_height_bqs;
-	uint32_t internal_org_x_bqs_on_tbl, internal_org_y_bqs_on_tbl;
-	uint32_t sensor_org_x_bqs_on_tbl, sensor_org_y_bqs_on_tbl;
-#endif
-
-#ifndef ISP2401
-	info->type = IA_CSS_SHADING_CORRECTION_TYPE_1;
-#else
-	assert(binary != NULL);
-	assert(stream_config != NULL);
-	assert(shading_info != NULL);
-	assert(pipe_config != NULL);
-#endif
-
-#ifndef ISP2401
-	info->info.type_1.enable	    = binary->info->sp.enable.sc;
-	info->info.type_1.num_hor_grids	    = binary->sctbl_width_per_color;
-	info->info.type_1.num_ver_grids	    = binary->sctbl_height;
-	info->info.type_1.bqs_per_grid_cell = (1 << binary->deci_factor_log2);
-#else
-	IA_CSS_ENTER_PRIVATE("binary=%p, required_bds_factor=%d, stream_config=%p",
-		binary, required_bds_factor, stream_config);
-#endif
-
-	/* Initialize by default values. */
-#ifndef ISP2401
-	info->info.type_1.bayer_scale_hor_ratio_in	= 1;
-	info->info.type_1.bayer_scale_hor_ratio_out	= 1;
-	info->info.type_1.bayer_scale_ver_ratio_in	= 1;
-	info->info.type_1.bayer_scale_ver_ratio_out	= 1;
-	info->info.type_1.sc_bayer_origin_x_bqs_on_shading_table = 0;
-	info->info.type_1.sc_bayer_origin_y_bqs_on_shading_table = 0;
-
-	err = ia_css_binary_compute_shading_table_bayer_origin(
-		binary,
-		required_bds_factor,
-		stream_config,
-		&res);
-	if (err != IA_CSS_SUCCESS)
-#else
-	*shading_info = DEFAULT_SHADING_INFO_TYPE_1;
-
-	err = sh_css_binary_get_sc_requirements(binary, required_bds_factor, stream_config, &scr);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-#endif
-		return err;
-#ifdef ISP2401
-	}
-
-	IA_CSS_LOG("binary: id=%d, sctbl=%dx%d, deci=%d",
-		binary->info->sp.id, binary->sctbl_width_per_color, binary->sctbl_height, binary->deci_factor_log2);
-	IA_CSS_LOG("binary: in=%dx%d, in_padded_w=%d, int=%dx%d, int_padded_w=%d, out=%dx%d, out_padded_w=%d",
-		binary->in_frame_info.res.width, binary->in_frame_info.res.height, binary->in_frame_info.padded_width,
-		binary->internal_frame_info.res.width, binary->internal_frame_info.res.height,
-		binary->internal_frame_info.padded_width,
-		binary->out_frame_info[0].res.width, binary->out_frame_info[0].res.height,
-		binary->out_frame_info[0].padded_width);
-
-	/* Set the input size from sensor, which includes left/top crop size. */
-	in_width_bqs	    = _ISP_BQS(binary->in_frame_info.res.width);
-	in_height_bqs	    = _ISP_BQS(binary->in_frame_info.res.height);
-
-	/* Frame size internally used in ISP, including sensor data and padding.
-	 * This is the frame size, to which the shading correction is applied.
-	 */
-	internal_width_bqs  = _ISP_BQS(binary->internal_frame_info.res.width);
-	internal_height_bqs = _ISP_BQS(binary->internal_frame_info.res.height);
-
-	/* Shading table. */
-	num_hor_grids = binary->sctbl_width_per_color;
-	num_ver_grids = binary->sctbl_height;
-	bqs_per_grid_cell = (1 << binary->deci_factor_log2);
-	tbl_width_bqs  = (num_hor_grids - 1) * bqs_per_grid_cell;
-	tbl_height_bqs = (num_ver_grids - 1) * bqs_per_grid_cell;
-#endif
-
-#ifndef ISP2401
-	info->info.type_1.bayer_scale_hor_ratio_in	= res.bayer_scale_hor_ratio_in;
-	info->info.type_1.bayer_scale_hor_ratio_out	= res.bayer_scale_hor_ratio_out;
-	info->info.type_1.bayer_scale_ver_ratio_in	= res.bayer_scale_ver_ratio_in;
-	info->info.type_1.bayer_scale_ver_ratio_out	= res.bayer_scale_ver_ratio_out;
-	info->info.type_1.sc_bayer_origin_x_bqs_on_shading_table = res.sc_bayer_origin_x_bqs_on_shading_table;
-	info->info.type_1.sc_bayer_origin_y_bqs_on_shading_table = res.sc_bayer_origin_y_bqs_on_shading_table;
-#else
-	IA_CSS_LOG("tbl_width_bqs=%d, tbl_height_bqs=%d", tbl_width_bqs, tbl_height_bqs);
-#endif
-
-#ifdef ISP2401
-	/* Real sensor data area on the internal frame at shading correction.
-	 * Filters and scaling are applied to the internal frame before shading correction, depending on the binary.
-	 */
-	sensor_org_x_bqs_on_internal = scr.sensor_data_origin_x_bqs_on_internal;
-	sensor_org_y_bqs_on_internal = scr.sensor_data_origin_y_bqs_on_internal;
-	{
-		unsigned int bs_frac = 8;	/* scaling factor 1.0 in fixed point (8 == FRAC_ACC macro in ISP) */
-		unsigned int bs_out, bs_in;	/* scaling ratio in fixed point */
-
-		bs_out = scr.bayer_scale_hor_ratio_out * bs_frac;
-		bs_in = scr.bayer_scale_hor_ratio_in * bs_frac;
-		sensor_width_bqs  = (in_width_bqs * bs_out + bs_in/2) / bs_in; /* "+ bs_in/2": rounding */
-
-		bs_out = scr.bayer_scale_ver_ratio_out * bs_frac;
-		bs_in = scr.bayer_scale_ver_ratio_in * bs_frac;
-		sensor_height_bqs = (in_height_bqs * bs_out + bs_in/2) / bs_in; /* "+ bs_in/2": rounding */
-	}
-
-	/* Center of the sensor data on the internal frame at shading correction. */
-	sensor_center_x_bqs_on_internal = sensor_org_x_bqs_on_internal + sensor_width_bqs / 2;
-	sensor_center_y_bqs_on_internal = sensor_org_y_bqs_on_internal + sensor_height_bqs / 2;
-
-	/* Size of left/right/upper/lower sides of the sensor center on the internal frame. */
-	left  = sensor_center_x_bqs_on_internal;
-	right = internal_width_bqs - sensor_center_x_bqs_on_internal;
-	upper = sensor_center_y_bqs_on_internal;
-	lower = internal_height_bqs - sensor_center_y_bqs_on_internal;
-
-	/* Align the size of left/right/upper/lower sides to a multiple of the grid cell size. */
-	adjust_left  = CEIL_MUL(left,  bqs_per_grid_cell);
-	adjust_right = CEIL_MUL(right, bqs_per_grid_cell);
-	adjust_upper = CEIL_MUL(upper, bqs_per_grid_cell);
-	adjust_lower = CEIL_MUL(lower, bqs_per_grid_cell);
-
-	/* Shading table should cover the adjusted frame size. */
-	adjust_width_bqs  = adjust_left + adjust_right;
-	adjust_height_bqs = adjust_upper + adjust_lower;
-
-	IA_CSS_LOG("adjust_width_bqs=%d, adjust_height_bqs=%d", adjust_width_bqs, adjust_height_bqs);
-
-	if (adjust_width_bqs > tbl_width_bqs || adjust_height_bqs > tbl_height_bqs) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-	/* Origin of the internal frame on the shading table. */
-	internal_org_x_bqs_on_tbl = adjust_left - left;
-	internal_org_y_bqs_on_tbl = adjust_upper - upper;
-
-	/* Origin of the real sensor data area on the shading table. */
-	sensor_org_x_bqs_on_tbl = internal_org_x_bqs_on_tbl + sensor_org_x_bqs_on_internal;
-	sensor_org_y_bqs_on_tbl = internal_org_y_bqs_on_tbl + sensor_org_y_bqs_on_internal;
-
-	/* The shading information necessary as API is stored in the shading_info. */
-	shading_info->info.type_1.num_hor_grids	    = num_hor_grids;
-	shading_info->info.type_1.num_ver_grids	    = num_ver_grids;
-	shading_info->info.type_1.bqs_per_grid_cell = bqs_per_grid_cell;
-
-	shading_info->info.type_1.bayer_scale_hor_ratio_in  = scr.bayer_scale_hor_ratio_in;
-	shading_info->info.type_1.bayer_scale_hor_ratio_out = scr.bayer_scale_hor_ratio_out;
-	shading_info->info.type_1.bayer_scale_ver_ratio_in  = scr.bayer_scale_ver_ratio_in;
-	shading_info->info.type_1.bayer_scale_ver_ratio_out = scr.bayer_scale_ver_ratio_out;
-
-	shading_info->info.type_1.isp_input_sensor_data_res_bqs.width  = in_width_bqs;
-	shading_info->info.type_1.isp_input_sensor_data_res_bqs.height = in_height_bqs;
-
-	shading_info->info.type_1.sensor_data_res_bqs.width  = sensor_width_bqs;
-	shading_info->info.type_1.sensor_data_res_bqs.height = sensor_height_bqs;
-
-	shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.x = (int32_t)sensor_org_x_bqs_on_tbl;
-	shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.y = (int32_t)sensor_org_y_bqs_on_tbl;
-
-	/* The shading information related to ISP (but, not necessary as API) is stored in the pipe_config. */
-	pipe_config->internal_frame_origin_bqs_on_sctbl.x = (int32_t)internal_org_x_bqs_on_tbl;
-	pipe_config->internal_frame_origin_bqs_on_sctbl.y = (int32_t)internal_org_y_bqs_on_tbl;
-
-	IA_CSS_LOG("shading_info: grids=%dx%d, cell=%d, scale=%d,%d,%d,%d, input=%dx%d, data=%dx%d, origin=(%d,%d)",
-		shading_info->info.type_1.num_hor_grids,
-		shading_info->info.type_1.num_ver_grids,
-		shading_info->info.type_1.bqs_per_grid_cell,
-		shading_info->info.type_1.bayer_scale_hor_ratio_in,
-		shading_info->info.type_1.bayer_scale_hor_ratio_out,
-		shading_info->info.type_1.bayer_scale_ver_ratio_in,
-		shading_info->info.type_1.bayer_scale_ver_ratio_out,
-		shading_info->info.type_1.isp_input_sensor_data_res_bqs.width,
-		shading_info->info.type_1.isp_input_sensor_data_res_bqs.height,
-		shading_info->info.type_1.sensor_data_res_bqs.width,
-		shading_info->info.type_1.sensor_data_res_bqs.height,
-		shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.x,
-		shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.y);
-
-	IA_CSS_LOG("pipe_config: origin=(%d,%d)",
-		pipe_config->internal_frame_origin_bqs_on_sctbl.x,
-		pipe_config->internal_frame_origin_bqs_on_sctbl.y);
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-#endif
-	return err;
-}
-
-enum ia_css_err
-ia_css_binary_get_shading_info(const struct ia_css_binary *binary,			/* [in] */
-				enum ia_css_shading_correction_type type,		/* [in] */
-				unsigned int required_bds_factor,			/* [in] */
-				const struct ia_css_stream_config *stream_config,	/* [in] */
-#ifndef ISP2401
-				struct ia_css_shading_info *info)			/* [out] */
-#else
-				struct ia_css_shading_info *shading_info,		/* [out] */
-				struct ia_css_pipe_config *pipe_config)			/* [out] */
-#endif
-{
-	enum ia_css_err err;
-
-	assert(binary != NULL);
-#ifndef ISP2401
-	assert(info != NULL);
-#else
-	assert(shading_info != NULL);
-
-	IA_CSS_ENTER_PRIVATE("binary=%p, type=%d, required_bds_factor=%d, stream_config=%p",
-		binary, type, required_bds_factor, stream_config);
-#endif
-
-	if (type == IA_CSS_SHADING_CORRECTION_TYPE_1)
-#ifndef ISP2401
-		err = ia_css_binary_get_shading_info_type_1(binary, required_bds_factor, stream_config, info);
-#else
-		err = ia_css_binary_get_shading_info_type_1(binary, required_bds_factor, stream_config,
-								shading_info, pipe_config);
-#endif
-
-	/* Other function calls can be added here when other shading correction types will be added in the future. */
-
-	else
-		err = IA_CSS_ERR_NOT_SUPPORTED;
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static void sh_css_binary_common_grid_info(const struct ia_css_binary *binary,
-				struct ia_css_grid_info *info)
-{
-	assert(binary != NULL);
-	assert(info != NULL);
-
-	info->isp_in_width = binary->internal_frame_info.res.width;
-	info->isp_in_height = binary->internal_frame_info.res.height;
-
-	info->vamem_type = IA_CSS_VAMEM_TYPE_2;
-}
-
-void
-ia_css_binary_dvs_grid_info(const struct ia_css_binary *binary,
-			    struct ia_css_grid_info *info,
-			    struct ia_css_pipe *pipe)
-{
-	struct ia_css_dvs_grid_info *dvs_info;
-
-	(void)pipe;
-	assert(binary != NULL);
-	assert(info != NULL);
-
-	dvs_info = &info->dvs_grid.dvs_grid_info;
-
-	/* for DIS, we use a division instead of a ceil_div. If this is smaller
-	 * than the 3a grid size, it indicates that the outer values are not
-	 * valid for DIS.
-	 */
-	dvs_info->enable            = binary->info->sp.enable.dis;
-	dvs_info->width             = binary->dis.grid.dim.width;
-	dvs_info->height            = binary->dis.grid.dim.height;
-	dvs_info->aligned_width     = binary->dis.grid.pad.width;
-	dvs_info->aligned_height    = binary->dis.grid.pad.height;
-	dvs_info->bqs_per_grid_cell = 1 << binary->dis.deci_factor_log2;
-	dvs_info->num_hor_coefs     = binary->dis.coef.dim.width;
-	dvs_info->num_ver_coefs     = binary->dis.coef.dim.height;
-
-	sh_css_binary_common_grid_info(binary, info);
-}
-
-void
-ia_css_binary_dvs_stat_grid_info(
-	const struct ia_css_binary *binary,
-	struct ia_css_grid_info *info,
-	struct ia_css_pipe *pipe)
-{
-	(void)pipe;
-	sh_css_binary_common_grid_info(binary, info);
-	return;
-}
-
-enum ia_css_err
-ia_css_binary_3a_grid_info(const struct ia_css_binary *binary,
-			   struct ia_css_grid_info *info,
-			   struct ia_css_pipe *pipe)
-{
-	struct ia_css_3a_grid_info *s3a_info;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER_PRIVATE("binary=%p, info=%p, pipe=%p",
-			     binary, info, pipe);
-
-	assert(binary != NULL);
-	assert(info != NULL);
-	s3a_info = &info->s3a_grid;
-
-
-	/* 3A statistics grid */
-	s3a_info->enable            = binary->info->sp.enable.s3a;
-	s3a_info->width             = binary->s3atbl_width;
-	s3a_info->height            = binary->s3atbl_height;
-	s3a_info->aligned_width     = binary->s3atbl_isp_width;
-	s3a_info->aligned_height    = binary->s3atbl_isp_height;
-	s3a_info->bqs_per_grid_cell = (1 << binary->deci_factor_log2);
-	s3a_info->deci_factor_log2  = binary->deci_factor_log2;
-	s3a_info->elem_bit_depth    = SH_CSS_BAYER_BITS;
-	s3a_info->use_dmem          = binary->info->sp.s3a.s3atbl_use_dmem;
-#if defined(HAS_NO_HMEM)
-	s3a_info->has_histogram     = 1;
-#else
-	s3a_info->has_histogram     = 0;
-#endif
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static void
-binary_init_pc_histogram(struct sh_css_pc_histogram *histo)
-{
-	assert(histo != NULL);
-
-	histo->length = 0;
-	histo->run = NULL;
-	histo->stall = NULL;
-}
-
-static void
-binary_init_metrics(struct sh_css_binary_metrics *metrics,
-	     const struct ia_css_binary_info *info)
-{
-	assert(metrics != NULL);
-	assert(info != NULL);
-
-	metrics->mode = info->pipeline.mode;
-	metrics->id   = info->id;
-	metrics->next = NULL;
-	binary_init_pc_histogram(&metrics->isp_histogram);
-	binary_init_pc_histogram(&metrics->sp_histogram);
-}
-
-/* move to host part of output module */
-static bool
-binary_supports_output_format(const struct ia_css_binary_xinfo *info,
-		       enum ia_css_frame_format format)
-{
-	int i;
-
-	assert(info != NULL);
-
-	for (i = 0; i < info->num_output_formats; i++) {
-		if (info->output_formats[i] == format)
-			return true;
-	}
-	return false;
-}
-
-#ifdef ISP2401
-static bool
-binary_supports_input_format(const struct ia_css_binary_xinfo *info,
-			     enum atomisp_input_format format)
-{
-
-	assert(info != NULL);
-	(void)format;
-
-	return true;
-}
-#endif
-
-static bool
-binary_supports_vf_format(const struct ia_css_binary_xinfo *info,
-			  enum ia_css_frame_format format)
-{
-	int i;
-
-	assert(info != NULL);
-
-	for (i = 0; i < info->num_vf_formats; i++) {
-		if (info->vf_formats[i] == format)
-			return true;
-	}
-	return false;
-}
-
-/* move to host part of bds module */
-static bool
-supports_bds_factor(uint32_t supported_factors,
-		       uint32_t bds_factor)
-{
-	return ((supported_factors & PACK_BDS_FACTOR(bds_factor)) != 0);
-}
-
-static enum ia_css_err
-binary_init_info(struct ia_css_binary_xinfo *info, unsigned int i,
-		 bool *binary_found)
-{
-	const unsigned char *blob = sh_css_blob_info[i].blob;
-	unsigned size = sh_css_blob_info[i].header.blob.size;
-
-	if ((info == NULL) || (binary_found == NULL))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	*info = sh_css_blob_info[i].header.info.isp;
-	*binary_found = blob != NULL;
-	info->blob_index = i;
-	/* we don't have this binary, skip it */
-	if (!size)
-		return IA_CSS_SUCCESS;
-
-	info->xmem_addr = sh_css_load_blob(blob, size);
-	if (!info->xmem_addr)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	return IA_CSS_SUCCESS;
-}
-
-/* When binaries are put at the beginning, they will only
- * be selected if no other primary matches.
- */
-enum ia_css_err
-ia_css_binary_init_infos(void)
-{
-	unsigned int i;
-	unsigned int num_of_isp_binaries = sh_css_num_binaries - NUM_OF_SPS - NUM_OF_BLS;
-
-	if (num_of_isp_binaries == 0)
-		return IA_CSS_SUCCESS;
-
-	all_binaries = sh_css_malloc(num_of_isp_binaries *
-						sizeof(*all_binaries));
-	if (all_binaries == NULL)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-
-	for (i = 0; i < num_of_isp_binaries; i++) {
-		enum ia_css_err ret;
-		struct ia_css_binary_xinfo *binary = &all_binaries[i];
-		bool binary_found;
-
-		ret = binary_init_info(binary, i, &binary_found);
-		if (ret != IA_CSS_SUCCESS)
-			return ret;
-		if (!binary_found)
-			continue;
-		/* Prepend new binary information */
-		binary->next = binary_infos[binary->sp.pipeline.mode];
-		binary_infos[binary->sp.pipeline.mode] = binary;
-		binary->blob = &sh_css_blob_info[i];
-		binary->mem_offsets = sh_css_blob_info[i].mem_offsets;
-	}
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err
-ia_css_binary_uninit(void)
-{
-	unsigned int i;
-	struct ia_css_binary_xinfo *b;
-
-	for (i = 0; i < IA_CSS_BINARY_NUM_MODES; i++) {
-		for (b = binary_infos[i]; b; b = b->next) {
-			if (b->xmem_addr)
-				hmm_free(b->xmem_addr);
-			b->xmem_addr = mmgr_NULL;
-		}
-		binary_infos[i] = NULL;
-	}
-	sh_css_free(all_binaries);
-	return IA_CSS_SUCCESS;
-}
-
-/* @brief Compute decimation factor for 3A statistics and shading correction.
- *
- * @param[in]	width	Frame width in pixels.
- * @param[in]	height	Frame height in pixels.
- * @return	Log2 of decimation factor (= grid cell size) in bayer quads.
- */
-static int
-binary_grid_deci_factor_log2(int width, int height)
-{
-/* 3A/Shading decimation factor spcification (at August 2008)
- * ------------------------------------------------------------------
- * [Image Width (BQ)] [Decimation Factor (BQ)] [Resulting grid cells]
-#ifndef ISP2401
- * 1280 ?c             32                       40 ?c
- *  640 ?c 1279        16                       40 ?c 80
- *      ?c  639         8                          ?c 80
-#else
- * from 1280                   32                 from 40
- * from  640 to 1279           16                 from 40 to 80
- *           to  639            8                         to 80
-#endif
- * ------------------------------------------------------------------
- */
-/* Maximum and minimum decimation factor by the specification */
-#define MAX_SPEC_DECI_FACT_LOG2		5
-#define MIN_SPEC_DECI_FACT_LOG2		3
-/* the smallest frame width in bayer quads when decimation factor (log2) is 5 or 4, by the specification */
-#define DECI_FACT_LOG2_5_SMALLEST_FRAME_WIDTH_BQ	1280
-#define DECI_FACT_LOG2_4_SMALLEST_FRAME_WIDTH_BQ	640
-
-	int smallest_factor; /* the smallest factor (log2) where the number of cells does not exceed the limitation */
-	int spec_factor;     /* the factor (log2) which satisfies the specification */
-
-	/* Currently supported maximum width and height are 5120(=80*64) and 3840(=60*64). */
-	assert(ISP_BQ_GRID_WIDTH(width, MAX_SPEC_DECI_FACT_LOG2) <= SH_CSS_MAX_BQ_GRID_WIDTH);
-	assert(ISP_BQ_GRID_HEIGHT(height, MAX_SPEC_DECI_FACT_LOG2) <= SH_CSS_MAX_BQ_GRID_HEIGHT);
-
-	/* Compute the smallest factor. */
-	smallest_factor = MAX_SPEC_DECI_FACT_LOG2;
-	while (ISP_BQ_GRID_WIDTH(width, smallest_factor - 1) <= SH_CSS_MAX_BQ_GRID_WIDTH &&
-	       ISP_BQ_GRID_HEIGHT(height, smallest_factor - 1) <= SH_CSS_MAX_BQ_GRID_HEIGHT
-	       && smallest_factor > MIN_SPEC_DECI_FACT_LOG2)
-		smallest_factor--;
-
-	/* Get the factor by the specification. */
-	if (_ISP_BQS(width) >= DECI_FACT_LOG2_5_SMALLEST_FRAME_WIDTH_BQ)
-		spec_factor = 5;
-	else if (_ISP_BQS(width) >= DECI_FACT_LOG2_4_SMALLEST_FRAME_WIDTH_BQ)
-		spec_factor = 4;
-	else
-		spec_factor = 3;
-
-	/* If smallest_factor is smaller than or equal to spec_factor, choose spec_factor to follow the specification.
-	   If smallest_factor is larger than spec_factor, choose smallest_factor.
-
-		ex. width=2560, height=1920
-			smallest_factor=4, spec_factor=5
-			smallest_factor < spec_factor   ->   return spec_factor
-
-		ex. width=300, height=3000
-			smallest_factor=5, spec_factor=3
-			smallest_factor > spec_factor   ->   return smallest_factor
-	*/
-	return max(smallest_factor, spec_factor);
-
-#undef MAX_SPEC_DECI_FACT_LOG2
-#undef MIN_SPEC_DECI_FACT_LOG2
-#undef DECI_FACT_LOG2_5_SMALLEST_FRAME_WIDTH_BQ
-#undef DECI_FACT_LOG2_4_SMALLEST_FRAME_WIDTH_BQ
-}
-
-static int
-binary_in_frame_padded_width(int in_frame_width,
-			     int isp_internal_width,
-			     int dvs_env_width,
-			     int stream_config_left_padding,
-			     int left_cropping,
-			     bool need_scaling)
-{
-	int rval;
-	int nr_of_left_paddings;	/* number of paddings pixels on the left of an image line */
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-	/* the output image line of Input System 2401 does not have the left paddings  */
-	nr_of_left_paddings = 0;
-#else
-	/* in other cases, the left padding pixels are always 128 */
-	nr_of_left_paddings = 2*ISP_VEC_NELEMS;
-#endif
-	if (need_scaling) {
-		/* In SDV use-case, we need to match left-padding of
-		 * primary and the video binary. */
-		if (stream_config_left_padding != -1) {
-			/* Different than before, we do left&right padding. */
-			rval =
-				CEIL_MUL(in_frame_width + nr_of_left_paddings,
-					2*ISP_VEC_NELEMS);
-		} else {
-			/* Different than before, we do left&right padding. */
-			in_frame_width += dvs_env_width;
-			rval =
-				CEIL_MUL(in_frame_width +
-					(left_cropping ? nr_of_left_paddings : 0),
-					2*ISP_VEC_NELEMS);
-		}
-	} else {
-		rval = isp_internal_width;
-	}
-
-	return rval;
-}
-
-
-enum ia_css_err
-ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
-		 bool online,
-		 bool two_ppc,
-		 enum atomisp_input_format stream_format,
-		 const struct ia_css_frame_info *in_info, /* can be NULL */
-		 const struct ia_css_frame_info *bds_out_info, /* can be NULL */
-		 const struct ia_css_frame_info *out_info[], /* can be NULL */
-		 const struct ia_css_frame_info *vf_info, /* can be NULL */
-		 struct ia_css_binary *binary,
-		 struct ia_css_resolution *dvs_env,
-		 int stream_config_left_padding,
-		 bool accelerator)
-{
-	const struct ia_css_binary_info *info = &xinfo->sp;
-	unsigned int dvs_env_width = 0,
-		     dvs_env_height = 0,
-		     vf_log_ds = 0,
-		     s3a_log_deci = 0,
-		     bits_per_pixel = 0,
-		     /* Resolution at SC/3A/DIS kernel. */
-		     sc_3a_dis_width = 0,
-		     /* Resolution at SC/3A/DIS kernel. */
-		     sc_3a_dis_padded_width = 0,
-		     /* Resolution at SC/3A/DIS kernel. */
-		     sc_3a_dis_height = 0,
-		     isp_internal_width = 0,
-		     isp_internal_height = 0,
-		     s3a_isp_width = 0;
-
-	bool need_scaling = false;
-	struct ia_css_resolution binary_dvs_env, internal_res;
-	enum ia_css_err err;
-	unsigned int i;
-	const struct ia_css_frame_info *bin_out_info = NULL;
-
-	assert(info != NULL);
-	assert(binary != NULL);
-
-	binary->info = xinfo;
-	if (!accelerator) {
-		/* binary->css_params has been filled by accelerator itself. */
-		err = ia_css_isp_param_allocate_isp_parameters(
-			&binary->mem_params, &binary->css_params,
-			&info->mem_initializers);
-		if (err != IA_CSS_SUCCESS) {
-			return err;
-		}
-	}
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		if (out_info[i] && (out_info[i]->res.width != 0)) {
-			bin_out_info = out_info[i];
-			break;
-		}
-	}
-	if (in_info != NULL && bin_out_info != NULL) {
-		need_scaling = (in_info->res.width != bin_out_info->res.width) ||
-			(in_info->res.height != bin_out_info->res.height);
-	}
-
-
-	/* binary_dvs_env has to be equal or larger than SH_CSS_MIN_DVS_ENVELOPE */
-	binary_dvs_env.width = 0;
-	binary_dvs_env.height = 0;
-	ia_css_binary_dvs_env(info, dvs_env, &binary_dvs_env);
-	dvs_env_width = binary_dvs_env.width;
-	dvs_env_height = binary_dvs_env.height;
-	binary->dvs_envelope.width  = dvs_env_width;
-	binary->dvs_envelope.height = dvs_env_height;
-
-	/* internal resolution calculation */
-	internal_res.width = 0;
-	internal_res.height = 0;
-	ia_css_binary_internal_res(in_info, bds_out_info, bin_out_info, dvs_env,
-				   info, &internal_res);
-	isp_internal_width = internal_res.width;
-	isp_internal_height = internal_res.height;
-
-	/* internal frame info */
-	if (bin_out_info != NULL) /* { */
-		binary->internal_frame_info.format = bin_out_info->format;
-	/* } */
-	binary->internal_frame_info.res.width       = isp_internal_width;
-	binary->internal_frame_info.padded_width    = CEIL_MUL(isp_internal_width, 2*ISP_VEC_NELEMS);
-	binary->internal_frame_info.res.height      = isp_internal_height;
-	binary->internal_frame_info.raw_bit_depth   = bits_per_pixel;
-
-	if (in_info != NULL) {
-		binary->effective_in_frame_res.width = in_info->res.width;
-		binary->effective_in_frame_res.height = in_info->res.height;
-
-		bits_per_pixel = in_info->raw_bit_depth;
-
-		/* input info */
-		binary->in_frame_info.res.width = in_info->res.width + info->pipeline.left_cropping;
-		binary->in_frame_info.res.height = in_info->res.height + info->pipeline.top_cropping;
-
-		binary->in_frame_info.res.width += dvs_env_width;
-		binary->in_frame_info.res.height += dvs_env_height;
-
-		binary->in_frame_info.padded_width =
-			binary_in_frame_padded_width(in_info->res.width,
-						     isp_internal_width,
-						     dvs_env_width,
-						     stream_config_left_padding,
-						     info->pipeline.left_cropping,
-						     need_scaling);
-
-		binary->in_frame_info.format = in_info->format;
-		binary->in_frame_info.raw_bayer_order = in_info->raw_bayer_order;
-		binary->in_frame_info.crop_info = in_info->crop_info;
-	}
-
-	if (online) {
-		bits_per_pixel = ia_css_util_input_format_bpp(
-			stream_format, two_ppc);
-	}
-	binary->in_frame_info.raw_bit_depth = bits_per_pixel;
-
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		if (out_info[i] != NULL) {
-			binary->out_frame_info[i].res.width     = out_info[i]->res.width;
-			binary->out_frame_info[i].res.height    = out_info[i]->res.height;
-			binary->out_frame_info[i].padded_width  = out_info[i]->padded_width;
-			if (info->pipeline.mode == IA_CSS_BINARY_MODE_COPY) {
-				binary->out_frame_info[i].raw_bit_depth = bits_per_pixel;
-			} else {
-				/* Only relevant for RAW format.
-				 * At the moment, all outputs are raw, 16 bit per pixel, except for copy.
-				 * To do this cleanly, the binary should specify in its info
-				 * the bit depth per output channel.
-				 */
-				binary->out_frame_info[i].raw_bit_depth = 16;
-			}
-			binary->out_frame_info[i].format        = out_info[i]->format;
-		}
-	}
-
-	if (vf_info && (vf_info->res.width != 0)) {
-		err = ia_css_vf_configure(binary, bin_out_info, (struct ia_css_frame_info *)vf_info, &vf_log_ds);
-		if (err != IA_CSS_SUCCESS) {
-			if (!accelerator) {
-				ia_css_isp_param_destroy_isp_parameters(
-					&binary->mem_params,
-					&binary->css_params);
-			}
-			return err;
-		}
-	}
-	binary->vf_downscale_log2 = vf_log_ds;
-
-	binary->online            = online;
-	binary->input_format      = stream_format;
-
-	/* viewfinder output info */
-	if ((vf_info != NULL) && (vf_info->res.width != 0)) {
-		unsigned int vf_out_vecs, vf_out_width, vf_out_height;
-		binary->vf_frame_info.format = vf_info->format;
-		if (bin_out_info == NULL)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		vf_out_vecs = __ISP_VF_OUTPUT_WIDTH_VECS(bin_out_info->padded_width,
-			vf_log_ds);
-		vf_out_width = _ISP_VF_OUTPUT_WIDTH(vf_out_vecs);
-		vf_out_height = _ISP_VF_OUTPUT_HEIGHT(bin_out_info->res.height,
-			vf_log_ds);
-
-		/* For preview mode, output pin is used instead of vf. */
-		if (info->pipeline.mode == IA_CSS_BINARY_MODE_PREVIEW) {
-			binary->out_frame_info[0].res.width =
-				(bin_out_info->res.width >> vf_log_ds);
-			binary->out_frame_info[0].padded_width = vf_out_width;
-			binary->out_frame_info[0].res.height   = vf_out_height;
-
-			binary->vf_frame_info.res.width    = 0;
-			binary->vf_frame_info.padded_width = 0;
-			binary->vf_frame_info.res.height   = 0;
-		} else {
-			/* we also store the raw downscaled width. This is
-			 * used for digital zoom in preview to zoom only on
-			 * the width that we actually want to keep, not on
-			 * the aligned width. */
-			binary->vf_frame_info.res.width =
-				(bin_out_info->res.width >> vf_log_ds);
-			binary->vf_frame_info.padded_width = vf_out_width;
-			binary->vf_frame_info.res.height   = vf_out_height;
-		}
-	} else {
-		binary->vf_frame_info.res.width    = 0;
-		binary->vf_frame_info.padded_width = 0;
-		binary->vf_frame_info.res.height   = 0;
-	}
-
-	if (info->enable.ca_gdc) {
-		binary->morph_tbl_width =
-			_ISP_MORPH_TABLE_WIDTH(isp_internal_width);
-		binary->morph_tbl_aligned_width  =
-			_ISP_MORPH_TABLE_ALIGNED_WIDTH(isp_internal_width);
-		binary->morph_tbl_height =
-			_ISP_MORPH_TABLE_HEIGHT(isp_internal_height);
-	} else {
-		binary->morph_tbl_width  = 0;
-		binary->morph_tbl_aligned_width  = 0;
-		binary->morph_tbl_height = 0;
-	}
-
-	sc_3a_dis_width = binary->in_frame_info.res.width;
-	sc_3a_dis_padded_width = binary->in_frame_info.padded_width;
-	sc_3a_dis_height = binary->in_frame_info.res.height;
-	if (bds_out_info != NULL && in_info != NULL &&
-			bds_out_info->res.width != in_info->res.width) {
-		/* TODO: Next, "internal_frame_info" should be derived from
-		 * bds_out. So this part will change once it is in place! */
-		sc_3a_dis_width = bds_out_info->res.width + info->pipeline.left_cropping;
-		sc_3a_dis_padded_width = isp_internal_width;
-		sc_3a_dis_height = isp_internal_height;
-	}
-
-
-	s3a_isp_width = _ISP_S3A_ELEMS_ISP_WIDTH(sc_3a_dis_padded_width,
-		info->pipeline.left_cropping);
-	if (info->s3a.fixed_s3a_deci_log) {
-		s3a_log_deci = info->s3a.fixed_s3a_deci_log;
-	} else {
-		s3a_log_deci = binary_grid_deci_factor_log2(s3a_isp_width,
-							    sc_3a_dis_height);
-	}
-	binary->deci_factor_log2  = s3a_log_deci;
-
-	if (info->enable.s3a) {
-		binary->s3atbl_width  =
-			_ISP_S3ATBL_WIDTH(sc_3a_dis_width,
-				s3a_log_deci);
-		binary->s3atbl_height =
-			_ISP_S3ATBL_HEIGHT(sc_3a_dis_height,
-				s3a_log_deci);
-		binary->s3atbl_isp_width =
-			_ISP_S3ATBL_ISP_WIDTH(s3a_isp_width,
-					s3a_log_deci);
-		binary->s3atbl_isp_height =
-			_ISP_S3ATBL_ISP_HEIGHT(sc_3a_dis_height,
-				s3a_log_deci);
-	} else {
-		binary->s3atbl_width  = 0;
-		binary->s3atbl_height = 0;
-		binary->s3atbl_isp_width  = 0;
-		binary->s3atbl_isp_height = 0;
-	}
-
-	if (info->enable.sc) {
-		binary->sctbl_width_per_color  =
-#ifndef ISP2401
-			_ISP_SCTBL_WIDTH_PER_COLOR(sc_3a_dis_padded_width,
-				s3a_log_deci);
-#else
-			_ISP_SCTBL_WIDTH_PER_COLOR(isp_internal_width, s3a_log_deci);
-#endif
-		binary->sctbl_aligned_width_per_color =
-			SH_CSS_MAX_SCTBL_ALIGNED_WIDTH_PER_COLOR;
-		binary->sctbl_height =
-#ifndef ISP2401
-			_ISP_SCTBL_HEIGHT(sc_3a_dis_height, s3a_log_deci);
-#else
-			_ISP_SCTBL_HEIGHT(isp_internal_height, s3a_log_deci);
-		binary->sctbl_legacy_width_per_color  =
-			_ISP_SCTBL_LEGACY_WIDTH_PER_COLOR(sc_3a_dis_padded_width, s3a_log_deci);
-		binary->sctbl_legacy_height =
-			_ISP_SCTBL_LEGACY_HEIGHT(sc_3a_dis_height, s3a_log_deci);
-#endif
-	} else {
-		binary->sctbl_width_per_color         = 0;
-		binary->sctbl_aligned_width_per_color = 0;
-		binary->sctbl_height                  = 0;
-#ifdef ISP2401
-		binary->sctbl_legacy_width_per_color  = 0;
-		binary->sctbl_legacy_height	      = 0;
-#endif
-	}
-	ia_css_sdis_init_info(&binary->dis,
-				sc_3a_dis_width,
-				sc_3a_dis_padded_width,
-				sc_3a_dis_height,
-				info->pipeline.isp_pipe_version,
-				info->enable.dis);
-	if (info->pipeline.left_cropping)
-		binary->left_padding = 2 * ISP_VEC_NELEMS - info->pipeline.left_cropping;
-	else
-		binary->left_padding = 0;
-
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err
-ia_css_binary_find(struct ia_css_binary_descr *descr,
-		   struct ia_css_binary *binary)
-{
-	int mode;
-	bool online;
-	bool two_ppc;
-	enum atomisp_input_format stream_format;
-	const struct ia_css_frame_info *req_in_info,
-				       *req_bds_out_info,
-				       *req_out_info[IA_CSS_BINARY_MAX_OUTPUT_PORTS],
-				       *req_bin_out_info = NULL,
-				       *req_vf_info;
-
-	struct ia_css_binary_xinfo *xcandidate;
-#ifndef ISP2401
-	bool need_ds, need_dz, need_dvs, need_xnr, need_dpc;
-#else
-	bool need_ds, need_dz, need_dvs, need_xnr, need_dpc, need_tnr;
-#endif
-	bool striped;
-	bool enable_yuv_ds;
-	bool enable_high_speed;
-	bool enable_dvs_6axis;
-	bool enable_reduced_pipe;
-	bool enable_capture_pp_bli;
-#ifdef ISP2401
-	bool enable_luma_only;
-#endif
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
-	bool continuous;
-	unsigned int isp_pipe_version;
-	struct ia_css_resolution dvs_env, internal_res;
-	unsigned int i;
-
-	assert(descr != NULL);
-	/* MW: used after an error check, may accept NULL, but doubtfull */
-	assert(binary != NULL);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_binary_find() enter: descr=%p, (mode=%d), binary=%p\n",
-		descr, descr->mode,
-		binary);
-
-	mode = descr->mode;
-	online = descr->online;
-	two_ppc = descr->two_ppc;
-	stream_format = descr->stream_format;
-	req_in_info = descr->in_info;
-	req_bds_out_info = descr->bds_out_info;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		req_out_info[i] = descr->out_info[i];
-		if (req_out_info[i] && (req_out_info[i]->res.width != 0))
-			req_bin_out_info = req_out_info[i];
-	}
-	if (req_bin_out_info == NULL)
-		return IA_CSS_ERR_INTERNAL_ERROR;
-#ifndef ISP2401
-	req_vf_info = descr->vf_info;
-#else
-
-	if ((descr->vf_info != NULL) && (descr->vf_info->res.width == 0))
-		/* width==0 means that there is no vf pin (e.g. in SkyCam preview case) */
-		req_vf_info = NULL;
-	else
-		req_vf_info = descr->vf_info;
-#endif
-
-	need_xnr = descr->enable_xnr;
-	need_ds = descr->enable_fractional_ds;
-	need_dz = false;
-	need_dvs = false;
-	need_dpc = descr->enable_dpc;
-#ifdef ISP2401
-	need_tnr = descr->enable_tnr;
-#endif
-	enable_yuv_ds = descr->enable_yuv_ds;
-	enable_high_speed = descr->enable_high_speed;
-	enable_dvs_6axis  = descr->enable_dvs_6axis;
-	enable_reduced_pipe = descr->enable_reduced_pipe;
-	enable_capture_pp_bli = descr->enable_capture_pp_bli;
-#ifdef ISP2401
-	enable_luma_only = descr->enable_luma_only;
-#endif
-	continuous = descr->continuous;
-	striped = descr->striped;
-	isp_pipe_version = descr->isp_pipe_version;
-
-	dvs_env.width = 0;
-	dvs_env.height = 0;
-	internal_res.width = 0;
-	internal_res.height = 0;
-
-
-	if (mode == IA_CSS_BINARY_MODE_VIDEO) {
-		dvs_env = descr->dvs_env;
-		need_dz = descr->enable_dz;
-		/* Video is the only mode that has a nodz variant. */
-		need_dvs = dvs_env.width || dvs_env.height;
-	}
-
-	/* print a map of the binary file */
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,	"BINARY INFO:\n");
-	for (i = 0; i < IA_CSS_BINARY_NUM_MODES; i++) {
-		xcandidate = binary_infos[i];
-		if (xcandidate) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,	"%d:\n", i);
-			while (xcandidate) {
-				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, " Name:%s Type:%d Cont:%d\n",
-						xcandidate->blob->name, xcandidate->type,
-						xcandidate->sp.enable.continuous);
-				xcandidate = xcandidate->next;
-			}
-		}
-	}
-
-	/* printf("sh_css_binary_find: pipe version %d\n", isp_pipe_version); */
-	for (xcandidate = binary_infos[mode]; xcandidate;
-	     xcandidate = xcandidate->next) {
-		struct ia_css_binary_info *candidate = &xcandidate->sp;
-		/* printf("sh_css_binary_find: evaluating candidate:
-		 * %d\n",candidate->id); */
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			"ia_css_binary_find() candidate = %p, mode = %d ID = %d\n",
-			candidate, candidate->pipeline.mode, candidate->id);
-
-		/*
-		 * MW: Only a limited set of jointly configured binaries can
-		 * be used in a continuous preview/video mode unless it is
-		 * the copy mode and runs on SP.
-		*/
-		if (!candidate->enable.continuous &&
-		    continuous && (mode != IA_CSS_BINARY_MODE_COPY)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: !%d && %d && (%d != %d)\n",
-					__LINE__, candidate->enable.continuous,
-					continuous, mode,
-					IA_CSS_BINARY_MODE_COPY);
-			continue;
-		}
-		if (striped && candidate->iterator.num_stripes == 1) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: binary is not striped\n",
-					__LINE__);
-			continue;
-		}
-
-		if (candidate->pipeline.isp_pipe_version != isp_pipe_version &&
-		    (mode != IA_CSS_BINARY_MODE_COPY) &&
-		    (mode != IA_CSS_BINARY_MODE_CAPTURE_PP) &&
-		    (mode != IA_CSS_BINARY_MODE_VF_PP)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: (%d != %d)\n",
-				__LINE__,
-				candidate->pipeline.isp_pipe_version, isp_pipe_version);
-			continue;
-		}
-		if (!candidate->enable.reduced_pipe && enable_reduced_pipe) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: !%d && %d\n",
-				__LINE__,
-				candidate->enable.reduced_pipe,
-				enable_reduced_pipe);
-			continue;
-		}
-		if (!candidate->enable.dvs_6axis && enable_dvs_6axis) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: !%d && %d\n",
-				__LINE__,
-				candidate->enable.dvs_6axis,
-				enable_dvs_6axis);
-			continue;
-		}
-		if (candidate->enable.high_speed && !enable_high_speed) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: %d && !%d\n",
-				__LINE__,
-				candidate->enable.high_speed,
-				enable_high_speed);
-			continue;
-		}
-		if (!candidate->enable.xnr && need_xnr) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: %d && !%d\n",
-				__LINE__,
-				candidate->enable.xnr,
-				need_xnr);
-			continue;
-		}
-		if (!(candidate->enable.ds & 2) && enable_yuv_ds) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: !%d && %d\n",
-				__LINE__,
-				((candidate->enable.ds & 2) != 0),
-				enable_yuv_ds);
-			continue;
-		}
-		if ((candidate->enable.ds & 2) && !enable_yuv_ds) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: %d && !%d\n",
-				__LINE__,
-				((candidate->enable.ds & 2) != 0),
-				enable_yuv_ds);
-			continue;
-		}
-
-		if (mode == IA_CSS_BINARY_MODE_VIDEO &&
-			candidate->enable.ds && need_ds)
-			need_dz = false;
-
-		/* when we require vf output, we need to have vf_veceven */
-		if ((req_vf_info != NULL) && !(candidate->enable.vf_veceven ||
-				/* or variable vf vec even */
-				candidate->vf_dec.is_variable ||
-				/* or more than one output pin. */
-				xcandidate->num_output_pins > 1)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: (%p != NULL) && !(%d || %d || (%d >%d))\n",
-				__LINE__, req_vf_info,
-				candidate->enable.vf_veceven,
-				candidate->vf_dec.is_variable,
-				xcandidate->num_output_pins, 1);
-			continue;
-		}
-		if (!candidate->enable.dvs_envelope && need_dvs) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: !%d && %d\n",
-				__LINE__,
-				candidate->enable.dvs_envelope, (int)need_dvs);
-			continue;
-		}
-		/* internal_res check considers input, output, and dvs envelope sizes */
-		ia_css_binary_internal_res(req_in_info, req_bds_out_info,
-					   req_bin_out_info, &dvs_env, candidate, &internal_res);
-		if (internal_res.width > candidate->internal.max_width) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			"ia_css_binary_find() [%d] continue: (%d > %d)\n",
-			__LINE__, internal_res.width,
-			candidate->internal.max_width);
-			continue;
-		}
-		if (internal_res.height > candidate->internal.max_height) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			"ia_css_binary_find() [%d] continue: (%d > %d)\n",
-			__LINE__, internal_res.height,
-			candidate->internal.max_height);
-			continue;
-		}
-		if (!candidate->enable.ds && need_ds && !(xcandidate->num_output_pins > 1)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: !%d && %d\n",
-				__LINE__, candidate->enable.ds, (int)need_ds);
-			continue;
-		}
-		if (!candidate->enable.uds && !candidate->enable.dvs_6axis && need_dz) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: !%d && !%d && %d\n",
-				__LINE__, candidate->enable.uds,
-				candidate->enable.dvs_6axis, (int)need_dz);
-			continue;
-		}
-		if (online && candidate->input.source == IA_CSS_BINARY_INPUT_MEMORY) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: %d && (%d == %d)\n",
-				__LINE__, online, candidate->input.source,
-				IA_CSS_BINARY_INPUT_MEMORY);
-			continue;
-		}
-		if (!online && candidate->input.source == IA_CSS_BINARY_INPUT_SENSOR) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: !%d && (%d == %d)\n",
-				__LINE__, online, candidate->input.source,
-				IA_CSS_BINARY_INPUT_SENSOR);
-			continue;
-		}
-		if (req_bin_out_info->res.width < candidate->output.min_width ||
-		    req_bin_out_info->res.width > candidate->output.max_width) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: (%d > %d) || (%d < %d)\n",
-				__LINE__,
-				req_bin_out_info->padded_width,
-				candidate->output.min_width,
-				req_bin_out_info->padded_width,
-				candidate->output.max_width);
-			continue;
-		}
-		if (xcandidate->num_output_pins > 1 && /* in case we have a second output pin, */
-		     req_vf_info) { /* and we need vf output. */
-			if (req_vf_info->res.width > candidate->output.max_width) {
-				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-					"ia_css_binary_find() [%d] continue: (%d < %d)\n",
-					__LINE__,
-					req_vf_info->res.width,
-					candidate->output.max_width);
-				continue;
-			}
-		}
-		if (req_in_info->padded_width > candidate->input.max_width) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: (%d > %d)\n",
-				__LINE__, req_in_info->padded_width,
-				candidate->input.max_width);
-			continue;
-		}
-		if (!binary_supports_output_format(xcandidate, req_bin_out_info->format)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: !%d\n",
-				__LINE__,
-				binary_supports_output_format(xcandidate, req_bin_out_info->format));
-			continue;
-		}
-#ifdef ISP2401
-		if (!binary_supports_input_format(xcandidate, descr->stream_format)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-					    "ia_css_binary_find() [%d] continue: !%d\n",
-					    __LINE__,
-					    binary_supports_input_format(xcandidate, req_in_info->format));
-			continue;
-		}
-#endif
-		if (xcandidate->num_output_pins > 1 && /* in case we have a second output pin, */
-		    req_vf_info                   && /* and we need vf output. */
-						      /* check if the required vf format
-							 is supported. */
-		    !binary_supports_output_format(xcandidate, req_vf_info->format)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: (%d > %d) && (%p != NULL) && !%d\n",
-				__LINE__, xcandidate->num_output_pins, 1,
-				req_vf_info,
-				binary_supports_output_format(xcandidate, req_vf_info->format));
-			continue;
-		}
-
-		/* Check if vf_veceven supports the requested vf format */
-		if (xcandidate->num_output_pins == 1 &&
-		    req_vf_info && candidate->enable.vf_veceven &&
-		    !binary_supports_vf_format(xcandidate, req_vf_info->format)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: (%d == %d) && (%p != NULL) && %d && !%d\n",
-				__LINE__, xcandidate->num_output_pins, 1,
-				req_vf_info, candidate->enable.vf_veceven,
-				binary_supports_vf_format(xcandidate, req_vf_info->format));
-			continue;
-		}
-
-		/* Check if vf_veceven supports the requested vf width */
-		if (xcandidate->num_output_pins == 1 &&
-		    req_vf_info && candidate->enable.vf_veceven) { /* and we need vf output. */
-			if (req_vf_info->res.width > candidate->output.max_width) {
-				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-					"ia_css_binary_find() [%d] continue: (%d < %d)\n",
-					__LINE__,
-					req_vf_info->res.width,
-					candidate->output.max_width);
-				continue;
-			}
-		}
-
-		if (!supports_bds_factor(candidate->bds.supported_bds_factors,
-		    descr->required_bds_factor)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: 0x%x & 0x%x)\n",
-				__LINE__, candidate->bds.supported_bds_factors,
-				descr->required_bds_factor);
-			continue;
-		}
-
-		if (!candidate->enable.dpc && need_dpc) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: 0x%x & 0x%x)\n",
-				__LINE__, candidate->enable.dpc,
-				descr->enable_dpc);
-			continue;
-		}
-
-		if (candidate->uds.use_bci && enable_capture_pp_bli) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: 0x%x & 0x%x)\n",
-				__LINE__, candidate->uds.use_bci,
-				descr->enable_capture_pp_bli);
-			continue;
-		}
-
-#ifdef ISP2401
-		if (candidate->enable.luma_only != enable_luma_only) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: %d != %d\n",
-				__LINE__, candidate->enable.luma_only,
-				descr->enable_luma_only);
-			continue;
-		}
-
-		if(!candidate->enable.tnr && need_tnr) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_binary_find() [%d] continue: !%d && %d\n",
-				__LINE__, candidate->enable.tnr,
-				descr->enable_tnr);
-			continue;
-		}
-
-#endif
-		/* reconfigure any variable properties of the binary */
-		err = ia_css_binary_fill_info(xcandidate, online, two_ppc,
-				       stream_format, req_in_info,
-				       req_bds_out_info,
-				       req_out_info, req_vf_info,
-				       binary, &dvs_env,
-				       descr->stream_config_left_padding,
-				       false);
-
-		if (err != IA_CSS_SUCCESS)
-			break;
-		binary_init_metrics(&binary->metrics, &binary->info->sp);
-		break;
-	}
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_binary_find() selected = %p, mode = %d ID = %d\n",
-		xcandidate, xcandidate ? xcandidate->sp.pipeline.mode : 0, xcandidate ? xcandidate->sp.id : 0);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_binary_find() leave: return_err=%d\n", err);
-
-	return err;
-}
-
-unsigned
-ia_css_binary_max_vf_width(void)
-{
-	/* This is (should be) true for IPU1 and IPU2 */
-	/* For IPU3 (SkyCam) this pointer is guarenteed to be NULL simply because such a binary does not exist  */
-	if (binary_infos[IA_CSS_BINARY_MODE_VF_PP])
-		return binary_infos[IA_CSS_BINARY_MODE_VF_PP]->sp.output.max_width;
-	return 0;
-}
-
-void
-ia_css_binary_destroy_isp_parameters(struct ia_css_binary *binary)
-{
-	if (binary) {
-		ia_css_isp_param_destroy_isp_parameters(&binary->mem_params,
-							&binary->css_params);
-	}
-}
-
-void
-ia_css_binary_get_isp_binaries(struct ia_css_binary_xinfo **binaries,
-	uint32_t *num_isp_binaries)
-{
-	assert(binaries != NULL);
-
-	if (num_isp_binaries)
-		*num_isp_binaries = 0;
-
-	*binaries = all_binaries;
-	if (all_binaries && num_isp_binaries) {
-		/* -1 to account for sp binary which is not stored in all_binaries */
-		if (sh_css_num_binaries > 0)
-			*num_isp_binaries = sh_css_num_binaries - 1;
-	}
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/interface/ia_css_bufq.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/interface/ia_css_bufq.h
deleted file mode 100644
index 034ec15ec4a1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/interface/ia_css_bufq.h
+++ /dev/null
@@ -1,197 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _IA_CSS_BUFQ_H
-#define _IA_CSS_BUFQ_H
-
-#include <type_support.h>
-#include "ia_css_bufq_comm.h"
-#include "ia_css_buffer.h"
-#include "ia_css_err.h"
-#define BUFQ_EVENT_SIZE 4
-
-
-/**
- * @brief Query the internal frame ID.
- *
- * @param[in]	key	The query key.
- * @param[out]	val	The query value.
- *
- * @return
- *	true, if the query succeeds;
- *	false, if the query fails.
- */
-bool ia_css_query_internal_queue_id(
-	enum ia_css_buffer_type buf_type,
-	unsigned int thread_id,
-	enum sh_css_queue_id *val
-	);
-
-
-/**
- * @brief  Map buffer type to a internal queue id.
- *
- * @param[in] thread id		Thread in which the buffer type has to be mapped or unmapped
- * @param[in] buf_type		buffer type.
- * @param[in] map		boolean flag to specify map or unmap
- * @return none
- */
-void ia_css_queue_map(
-	unsigned int thread_id,
-	enum ia_css_buffer_type buf_type,
-	bool map
-	);
-
-
-/**
- * @brief  Initilize buffer type to a queue id mapping
- * @return none
- */
-void ia_css_queue_map_init(void);
-
-
-/**
- * @brief initializes bufq module
- * It create instances of
- * -host to SP buffer queue  which is a list with predefined size,
- *	MxN queues where M is the number threads and N is the number queues per thread
- *-SP to host buffer queue , is a list with N queues
- *-host to SP event communication queue
- * -SP to host event communication queue
- * -queue for tagger commands
- * @return none
- */
-void ia_css_bufq_init(void);
-
-
-/**
-* @brief Enqueues an item into host to SP buffer queue
- *
- * @param thread_index[in]	Thread in which the item to be enqueued
- *
- * @param queue_id[in]		Index of the queue in the specified thread
- * @param item[in]		Object to enqueue.
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
-*/
-enum ia_css_err ia_css_bufq_enqueue_buffer(
-	int thread_index,
-	int queue_id,
-	uint32_t item);
-
-/**
-* @brief Dequeues an item from SP to host buffer queue.
- *
- * @param queue_id[in]		Specifies  the index of the queue in the list where
- *				the item has to be read.
- * @paramitem [out]		Object to be dequeued into this item.
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
-*/
-enum  ia_css_err ia_css_bufq_dequeue_buffer(
-	int queue_id,
-	uint32_t *item);
-
-/**
-* @brief  Enqueue an event item into host to SP communication event queue.
- *
- * @param[in]	evt_id		      The event ID.
- * @param[in]	evt_payload_0	The event payload.
- * @param[in]	evt_payload_1	The event payload.
- * @param[in]	evt_payload_2	The event payload.
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
-*/
-enum ia_css_err ia_css_bufq_enqueue_psys_event(
-	uint8_t evt_id,
-	uint8_t evt_payload_0,
-	uint8_t evt_payload_1,
-	uint8_t evt_payload_2
-	);
-
-/**
- * @brief   Dequeue an item from  SP to host communication event queue.
- *
- * @param item	Object to be dequeued into this item.
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
-*/
-enum  ia_css_err ia_css_bufq_dequeue_psys_event(
-	uint8_t item[BUFQ_EVENT_SIZE]
-	);
-
-/**
- * @brief  Enqueue an event item into host to SP EOF event queue.
- *
- * @param[in]	evt_id		      The event ID.
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
- */
-enum ia_css_err ia_css_bufq_enqueue_isys_event(
-	uint8_t evt_id);
-
-/**
-* @brief   Dequeue an item from  SP to host communication EOF event queue.
-
- *
- * @param item	Object to be dequeued into this item.
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
- */
-enum  ia_css_err ia_css_bufq_dequeue_isys_event(
-	uint8_t item[BUFQ_EVENT_SIZE]);
-
-/**
-* @brief   Enqueue a tagger command item into tagger command queue..
- *
- * @param item	Object to be enqueue.
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
-*/
-enum ia_css_err ia_css_bufq_enqueue_tag_cmd(
-	uint32_t item);
-
-/**
-* @brief  Uninitializes bufq module.
- *
- * @return	IA_CSS_SUCCESS or error code upon error.
- *
-*/
-enum ia_css_err ia_css_bufq_deinit(void);
-
-/**
-* @brief  Dump queue states
- *
- * @return	None
- *
-*/
-void ia_css_bufq_dump_queue_info(void);
-
-#endif	/* _IA_CSS_BUFQ_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/interface/ia_css_bufq_comm.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/interface/ia_css_bufq_comm.h
deleted file mode 100644
index bb77080591b9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/interface/ia_css_bufq_comm.h
+++ /dev/null
@@ -1,66 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _IA_CSS_BUFQ_COMM_H
-#define _IA_CSS_BUFQ_COMM_H
-
-#include "system_global.h"
-
-enum sh_css_queue_id {
-	SH_CSS_INVALID_QUEUE_ID     = -1,
-	SH_CSS_QUEUE_A_ID = 0,
-	SH_CSS_QUEUE_B_ID,
-	SH_CSS_QUEUE_C_ID,
-	SH_CSS_QUEUE_D_ID,
-	SH_CSS_QUEUE_E_ID,
-	SH_CSS_QUEUE_F_ID,
-	SH_CSS_QUEUE_G_ID,
-#if defined(HAS_NO_INPUT_SYSTEM)
-	/* input frame queue for skycam */
-	SH_CSS_QUEUE_H_ID,
-#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	SH_CSS_QUEUE_H_ID, /* for metadata */
-#endif
-
-#if defined(HAS_NO_INPUT_SYSTEM) || defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-#define SH_CSS_MAX_NUM_QUEUES (SH_CSS_QUEUE_H_ID+1)
-#else
-#define SH_CSS_MAX_NUM_QUEUES (SH_CSS_QUEUE_G_ID+1)
-#endif
-
-};
-
-#define SH_CSS_MAX_DYNAMIC_BUFFERS_PER_THREAD SH_CSS_MAX_NUM_QUEUES
-/* for now we staticaly assign queue 0 & 1 to parameter sets */
-#define IA_CSS_PARAMETER_SET_QUEUE_ID SH_CSS_QUEUE_A_ID
-#define IA_CSS_PER_FRAME_PARAMETER_SET_QUEUE_ID SH_CSS_QUEUE_B_ID
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/src/bufq.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/src/bufq.c
deleted file mode 100644
index ffbcdd80d934..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/bufq/src/bufq.c
+++ /dev/null
@@ -1,589 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "assert_support.h"		/* assert */
-#include "ia_css_buffer.h"
-#include "sp.h"
-#include "ia_css_bufq.h"		/* Bufq API's */
-#include "ia_css_queue.h"		/* ia_css_queue_t */
-#include "sw_event_global.h"		/* Event IDs.*/
-#include "ia_css_eventq.h"		/* ia_css_eventq_recv()*/
-#include "ia_css_debug.h"		/* ia_css_debug_dtrace*/
-#include "sh_css_internal.h"		/* sh_css_queue_type */
-#include "sp_local.h"			/* sp_address_of */
-#include "ia_css_util.h"		/* ia_css_convert_errno()*/
-#include "sh_css_firmware.h"		/* sh_css_sp_fw*/
-
-#define BUFQ_DUMP_FILE_NAME_PREFIX_SIZE 256
-
-static char prefix[BUFQ_DUMP_FILE_NAME_PREFIX_SIZE] = {0};
-
-/*********************************************************/
-/* Global Queue objects used by CSS                      */
-/*********************************************************/
-
-#ifndef ISP2401
-
-struct sh_css_queues {
-	/* Host2SP buffer queue */
-	ia_css_queue_t host2sp_buffer_queue_handles
-		[SH_CSS_MAX_SP_THREADS][SH_CSS_MAX_NUM_QUEUES];
-	/* SP2Host buffer queue */
-	ia_css_queue_t sp2host_buffer_queue_handles
-		[SH_CSS_MAX_NUM_QUEUES];
-
-	/* Host2SP event queue */
-	ia_css_queue_t host2sp_psys_event_queue_handle;
-
-	/* SP2Host event queue */
-	ia_css_queue_t sp2host_psys_event_queue_handle;
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	/* Host2SP ISYS event queue */
-	ia_css_queue_t host2sp_isys_event_queue_handle;
-
-	/* SP2Host ISYS event queue */
-	ia_css_queue_t sp2host_isys_event_queue_handle;
-#endif
-	/* Tagger command queue */
-	ia_css_queue_t host2sp_tag_cmd_queue_handle;
-};
-
-#else
-
-struct sh_css_queues {
-	/* Host2SP buffer queue */
-	ia_css_queue_t host2sp_buffer_queue_handles
-		[SH_CSS_MAX_SP_THREADS][SH_CSS_MAX_NUM_QUEUES];
-	/* SP2Host buffer queue */
-	ia_css_queue_t sp2host_buffer_queue_handles
-		[SH_CSS_MAX_NUM_QUEUES];
-
-	/* Host2SP event queue */
-	ia_css_queue_t host2sp_psys_event_queue_handle;
-
-	/* SP2Host event queue */
-	ia_css_queue_t sp2host_psys_event_queue_handle;
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	/* Host2SP ISYS event queue */
-	ia_css_queue_t host2sp_isys_event_queue_handle;
-
-	/* SP2Host ISYS event queue */
-	ia_css_queue_t sp2host_isys_event_queue_handle;
-
-	/* Tagger command queue */
-	ia_css_queue_t host2sp_tag_cmd_queue_handle;
-#endif
-};
-
-#endif
-
-/*******************************************************
-*** Static variables
-********************************************************/
-static struct sh_css_queues css_queues;
-
-static int buffer_type_to_queue_id_map[SH_CSS_MAX_SP_THREADS][IA_CSS_NUM_DYNAMIC_BUFFER_TYPE];
-static bool queue_availability[SH_CSS_MAX_SP_THREADS][SH_CSS_MAX_NUM_QUEUES];
-
-/*******************************************************
-*** Static functions
-********************************************************/
-static void map_buffer_type_to_queue_id(
-	unsigned int thread_id,
-	enum ia_css_buffer_type buf_type
-	);
-static void unmap_buffer_type_to_queue_id(
-	unsigned int thread_id,
-	enum ia_css_buffer_type buf_type
-	);
-
-static ia_css_queue_t *bufq_get_qhandle(
-	enum sh_css_queue_type type,
-	enum sh_css_queue_id id,
-	int thread
-	);
-
-/*******************************************************
-*** Public functions
-********************************************************/
-void ia_css_queue_map_init(void)
-{
-	unsigned int i, j;
-
-	for (i = 0; i < SH_CSS_MAX_SP_THREADS; i++) {
-		for (j = 0; j < SH_CSS_MAX_NUM_QUEUES; j++)
-			queue_availability[i][j] = true;
-	}
-
-	for (i = 0; i < SH_CSS_MAX_SP_THREADS; i++) {
-		for (j = 0; j < IA_CSS_NUM_DYNAMIC_BUFFER_TYPE; j++)
-			buffer_type_to_queue_id_map[i][j] = SH_CSS_INVALID_QUEUE_ID;
-	}
-}
-
-void ia_css_queue_map(
-	unsigned int thread_id,
-	enum ia_css_buffer_type buf_type,
-	bool map)
-{
-	assert(buf_type < IA_CSS_NUM_DYNAMIC_BUFFER_TYPE);
-	assert(thread_id < SH_CSS_MAX_SP_THREADS);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_queue_map() enter: buf_type=%d, thread_id=%d\n", buf_type, thread_id);
-
-	if (map)
-		map_buffer_type_to_queue_id(thread_id, buf_type);
-	else
-		unmap_buffer_type_to_queue_id(thread_id, buf_type);
-}
-
-/*
- * @brief Query the internal queue ID.
- */
-bool ia_css_query_internal_queue_id(
-	enum ia_css_buffer_type buf_type,
-	unsigned int thread_id,
-	enum sh_css_queue_id *val)
-{
-	IA_CSS_ENTER("buf_type=%d, thread_id=%d, val = %p", buf_type, thread_id, val);
-
-	if ((val == NULL) || (thread_id >= SH_CSS_MAX_SP_THREADS) || (buf_type >= IA_CSS_NUM_DYNAMIC_BUFFER_TYPE)) {
-		IA_CSS_LEAVE("return_val = false");
-		return false;
-	}
-
-	*val = buffer_type_to_queue_id_map[thread_id][buf_type];
-	if ((*val == SH_CSS_INVALID_QUEUE_ID) || (*val >= SH_CSS_MAX_NUM_QUEUES)) {
-		IA_CSS_LOG("INVALID queue ID MAP = %d\n", *val);
-		IA_CSS_LEAVE("return_val = false");
-		return false;
-	}
-	IA_CSS_LEAVE("return_val = true");
-	return true;
-}
-
-/*******************************************************
-*** Static functions
-********************************************************/
-static void map_buffer_type_to_queue_id(
-	unsigned int thread_id,
-	enum ia_css_buffer_type buf_type)
-{
-	unsigned int i;
-
-	assert(thread_id < SH_CSS_MAX_SP_THREADS);
-	assert(buf_type < IA_CSS_NUM_DYNAMIC_BUFFER_TYPE);
-	assert(buffer_type_to_queue_id_map[thread_id][buf_type] == SH_CSS_INVALID_QUEUE_ID);
-
-	/* queue 0 is reserved for parameters because it doesn't depend on events */
-	if (buf_type == IA_CSS_BUFFER_TYPE_PARAMETER_SET) {
-		assert(queue_availability[thread_id][IA_CSS_PARAMETER_SET_QUEUE_ID]);
-		queue_availability[thread_id][IA_CSS_PARAMETER_SET_QUEUE_ID] = false;
-		buffer_type_to_queue_id_map[thread_id][buf_type] = IA_CSS_PARAMETER_SET_QUEUE_ID;
-		return;
-	}
-
-	/* queue 1 is reserved for per frame parameters because it doesn't depend on events */
-	if (buf_type == IA_CSS_BUFFER_TYPE_PER_FRAME_PARAMETER_SET) {
-		assert(queue_availability[thread_id][IA_CSS_PER_FRAME_PARAMETER_SET_QUEUE_ID]);
-		queue_availability[thread_id][IA_CSS_PER_FRAME_PARAMETER_SET_QUEUE_ID] = false;
-		buffer_type_to_queue_id_map[thread_id][buf_type] = IA_CSS_PER_FRAME_PARAMETER_SET_QUEUE_ID;
-		return;
-	}
-
-	for (i = SH_CSS_QUEUE_C_ID; i < SH_CSS_MAX_NUM_QUEUES; i++) {
-		if (queue_availability[thread_id][i]) {
-			queue_availability[thread_id][i] = false;
-			buffer_type_to_queue_id_map[thread_id][buf_type] = i;
-			break;
-		}
-	}
-
-	assert(i != SH_CSS_MAX_NUM_QUEUES);
-	return;
-}
-
-static void unmap_buffer_type_to_queue_id(
-	unsigned int thread_id,
-	enum ia_css_buffer_type buf_type)
-{
-	int queue_id;
-
-	assert(thread_id < SH_CSS_MAX_SP_THREADS);
-	assert(buf_type < IA_CSS_NUM_DYNAMIC_BUFFER_TYPE);
-	assert(buffer_type_to_queue_id_map[thread_id][buf_type] != SH_CSS_INVALID_QUEUE_ID);
-
-	queue_id = buffer_type_to_queue_id_map[thread_id][buf_type];
-	buffer_type_to_queue_id_map[thread_id][buf_type] = SH_CSS_INVALID_QUEUE_ID;
-	queue_availability[thread_id][queue_id] = true;
-}
-
-
-static ia_css_queue_t *bufq_get_qhandle(
-	enum sh_css_queue_type type,
-	enum sh_css_queue_id id,
-	int thread)
-{
-	ia_css_queue_t *q = NULL;
-
-	switch (type) {
-	case sh_css_host2sp_buffer_queue:
-		if ((thread >= SH_CSS_MAX_SP_THREADS) || (thread < 0) ||
-			(id == SH_CSS_INVALID_QUEUE_ID))
-			break;
-		q = &css_queues.host2sp_buffer_queue_handles[thread][id];
-		break;
-	case sh_css_sp2host_buffer_queue:
-		if (id == SH_CSS_INVALID_QUEUE_ID)
-			break;
-		q = &css_queues.sp2host_buffer_queue_handles[id];
-		break;
-	case sh_css_host2sp_psys_event_queue:
-		q = &css_queues.host2sp_psys_event_queue_handle;
-		break;
-	case sh_css_sp2host_psys_event_queue:
-		q = &css_queues.sp2host_psys_event_queue_handle;
-		break;
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	case sh_css_host2sp_isys_event_queue:
-		q = &css_queues.host2sp_isys_event_queue_handle;
-		break;
-	case sh_css_sp2host_isys_event_queue:
-		q = &css_queues.sp2host_isys_event_queue_handle;
-		break;
-#endif
-	case sh_css_host2sp_tag_cmd_queue:
-		q = &css_queues.host2sp_tag_cmd_queue_handle;
-		break;
-	default:
-		break;
-	}
-
-	return q;
-}
-
-/* Local function to initialize a buffer queue. This reduces
- * the chances of copy-paste errors or typos.
- */
-static inline void
-init_bufq(unsigned int desc_offset,
-	  unsigned int elems_offset,
-	  ia_css_queue_t *handle)
-{
-	const struct ia_css_fw_info *fw;
-	unsigned int q_base_addr;
-	ia_css_queue_remote_t remoteq;
-
-	fw = &sh_css_sp_fw;
-	q_base_addr = fw->info.sp.host_sp_queue;
-
-	/* Setup queue location as SP and proc id as SP0_ID */
-	remoteq.location = IA_CSS_QUEUE_LOC_SP;
-	remoteq.proc_id = SP0_ID;
-	remoteq.cb_desc_addr = q_base_addr + desc_offset;
-	remoteq.cb_elems_addr = q_base_addr + elems_offset;
-	/* Initialize the queue instance and obtain handle */
-	ia_css_queue_remote_init(handle, &remoteq);
-}
-
-void ia_css_bufq_init(void)
-{
-	int i, j;
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	/* Setup all the local queue descriptors for Host2SP Buffer Queues */
-	for (i = 0; i < SH_CSS_MAX_SP_THREADS; i++)
-		for (j = 0; j < SH_CSS_MAX_NUM_QUEUES; j++) {
-			init_bufq((uint32_t)offsetof(struct host_sp_queues, host2sp_buffer_queues_desc[i][j]),
-				  (uint32_t)offsetof(struct host_sp_queues, host2sp_buffer_queues_elems[i][j]),
-				  &css_queues.host2sp_buffer_queue_handles[i][j]);
-		}
-
-	/* Setup all the local queue descriptors for SP2Host Buffer Queues */
-	for (i = 0; i < SH_CSS_MAX_NUM_QUEUES; i++) {
-		init_bufq(offsetof(struct host_sp_queues, sp2host_buffer_queues_desc[i]),
-			  offsetof(struct host_sp_queues, sp2host_buffer_queues_elems[i]),
-			  &css_queues.sp2host_buffer_queue_handles[i]);
-	}
-
-	/* Host2SP event queue*/
-	init_bufq((uint32_t)offsetof(struct host_sp_queues, host2sp_psys_event_queue_desc),
-		  (uint32_t)offsetof(struct host_sp_queues, host2sp_psys_event_queue_elems),
-		  &css_queues.host2sp_psys_event_queue_handle);
-
-	/* SP2Host event queue */
-	init_bufq((uint32_t)offsetof(struct host_sp_queues, sp2host_psys_event_queue_desc),
-		  (uint32_t)offsetof(struct host_sp_queues, sp2host_psys_event_queue_elems),
-		  &css_queues.sp2host_psys_event_queue_handle);
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	/* Host2SP ISYS event queue */
-	init_bufq((uint32_t)offsetof(struct host_sp_queues, host2sp_isys_event_queue_desc),
-		  (uint32_t)offsetof(struct host_sp_queues, host2sp_isys_event_queue_elems),
-		  &css_queues.host2sp_isys_event_queue_handle);
-
-	/* SP2Host ISYS event queue*/
-	init_bufq((uint32_t)offsetof(struct host_sp_queues, sp2host_isys_event_queue_desc),
-		  (uint32_t)offsetof(struct host_sp_queues, sp2host_isys_event_queue_elems),
-		  &css_queues.sp2host_isys_event_queue_handle);
-
-	/* Host2SP tagger command queue */
-	init_bufq((uint32_t)offsetof(struct host_sp_queues, host2sp_tag_cmd_queue_desc),
-		  (uint32_t)offsetof(struct host_sp_queues, host2sp_tag_cmd_queue_elems),
-		  &css_queues.host2sp_tag_cmd_queue_handle);
-#endif
-
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-enum ia_css_err ia_css_bufq_enqueue_buffer(
-	int thread_index,
-	int queue_id,
-	uint32_t item)
-{
-	enum ia_css_err return_err = IA_CSS_SUCCESS;
-	ia_css_queue_t *q;
-	int error;
-
-	IA_CSS_ENTER_PRIVATE("queue_id=%d", queue_id);
-	if ((thread_index >= SH_CSS_MAX_SP_THREADS) || (thread_index < 0) ||
-			(queue_id == SH_CSS_INVALID_QUEUE_ID))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	/* Get the queue for communication */
-	q = bufq_get_qhandle(sh_css_host2sp_buffer_queue,
-		queue_id,
-		thread_index);
-	if (q != NULL) {
-		error = ia_css_queue_enqueue(q, item);
-		return_err = ia_css_convert_errno(error);
-	} else {
-		IA_CSS_ERROR("queue is not initialized");
-		return_err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-
-	IA_CSS_LEAVE_ERR_PRIVATE(return_err);
-	return return_err;
-}
-
-enum ia_css_err ia_css_bufq_dequeue_buffer(
-	int queue_id,
-	uint32_t *item)
-{
-	enum ia_css_err return_err;
-	int error = 0;
-	ia_css_queue_t *q;
-
-	IA_CSS_ENTER_PRIVATE("queue_id=%d", queue_id);
-	if ((item == NULL) ||
-	    (queue_id <= SH_CSS_INVALID_QUEUE_ID) ||
-	    (queue_id >= SH_CSS_MAX_NUM_QUEUES)
-	   )
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	q = bufq_get_qhandle(sh_css_sp2host_buffer_queue,
-		queue_id,
-		-1);
-	if (q != NULL) {
-		error = ia_css_queue_dequeue(q, item);
-		return_err = ia_css_convert_errno(error);
-	} else {
-		IA_CSS_ERROR("queue is not initialized");
-		return_err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-
-	IA_CSS_LEAVE_ERR_PRIVATE(return_err);
-	return return_err;
-}
-
-enum ia_css_err ia_css_bufq_enqueue_psys_event(
-	uint8_t evt_id,
-	uint8_t evt_payload_0,
-	uint8_t evt_payload_1,
-	uint8_t evt_payload_2)
-{
-	enum ia_css_err return_err;
-	int error = 0;
-	ia_css_queue_t *q;
-
-	IA_CSS_ENTER_PRIVATE("evt_id=%d", evt_id);
-	q = bufq_get_qhandle(sh_css_host2sp_psys_event_queue, -1, -1);
-	if (NULL == q) {
-		IA_CSS_ERROR("queue is not initialized");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-
-	error = ia_css_eventq_send(q,
-			evt_id, evt_payload_0, evt_payload_1, evt_payload_2);
-
-	return_err = ia_css_convert_errno(error);
-	IA_CSS_LEAVE_ERR_PRIVATE(return_err);
-	return return_err;
-}
-
-enum  ia_css_err ia_css_bufq_dequeue_psys_event(
-	uint8_t item[BUFQ_EVENT_SIZE])
-{
-	enum ia_css_err;
-	int error = 0;
-	ia_css_queue_t *q;
-
-	/* No ENTER/LEAVE in this function since this is polled
-	 * by some test apps. Enablign logging here floods the log
-	 * files which may cause timeouts. */
-	if (item == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	q = bufq_get_qhandle(sh_css_sp2host_psys_event_queue, -1, -1);
-	if (NULL == q) {
-		IA_CSS_ERROR("queue is not initialized");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-	error = ia_css_eventq_recv(q, item);
-
-	return ia_css_convert_errno(error);
-
-}
-
-enum  ia_css_err ia_css_bufq_dequeue_isys_event(
-	uint8_t item[BUFQ_EVENT_SIZE])
-{
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	enum ia_css_err;
-	int error = 0;
-	ia_css_queue_t *q;
-
-	/* No ENTER/LEAVE in this function since this is polled
-	 * by some test apps. Enablign logging here floods the log
-	 * files which may cause timeouts. */
-	if (item == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	q = bufq_get_qhandle(sh_css_sp2host_isys_event_queue, -1, -1);
-	if (q == NULL) {
-		IA_CSS_ERROR("queue is not initialized");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-	error = ia_css_eventq_recv(q, item);
-	return ia_css_convert_errno(error);
-#else
-	(void)item;
-	return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-#endif
-}
-
-enum ia_css_err ia_css_bufq_enqueue_isys_event(uint8_t evt_id)
-{
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	enum ia_css_err return_err;
-	int error = 0;
-	ia_css_queue_t *q;
-
-	IA_CSS_ENTER_PRIVATE("event_id=%d", evt_id);
-	q = bufq_get_qhandle(sh_css_host2sp_isys_event_queue, -1, -1);
-	if (q == NULL) {
-		IA_CSS_ERROR("queue is not initialized");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-
-	error = ia_css_eventq_send(q, evt_id, 0, 0, 0);
-	return_err = ia_css_convert_errno(error);
-	IA_CSS_LEAVE_ERR_PRIVATE(return_err);
-	return return_err;
-#else
-	(void)evt_id;
-	return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-#endif
-}
-
-enum ia_css_err ia_css_bufq_enqueue_tag_cmd(
-	uint32_t item)
-{
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	enum ia_css_err return_err;
-	int error = 0;
-	ia_css_queue_t *q;
-
-	IA_CSS_ENTER_PRIVATE("item=%d", item);
-	q = bufq_get_qhandle(sh_css_host2sp_tag_cmd_queue, -1, -1);
-	if (NULL == q) {
-		IA_CSS_ERROR("queue is not initialized");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-	error = ia_css_queue_enqueue(q, item);
-
-	return_err = ia_css_convert_errno(error);
-	IA_CSS_LEAVE_ERR_PRIVATE(return_err);
-	return return_err;
-#else
-	(void)item;
-	return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-#endif
-}
-
-enum ia_css_err ia_css_bufq_deinit(void)
-{
-	return IA_CSS_SUCCESS;
-}
-
-static void bufq_dump_queue_info(const char *prefix, ia_css_queue_t *qhandle)
-{
-	uint32_t free = 0, used = 0;
-	assert(prefix != NULL && qhandle != NULL);
-	ia_css_queue_get_used_space(qhandle, &used);
-	ia_css_queue_get_free_space(qhandle, &free);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%s: used=%u free=%u\n",
-		prefix, used, free);
-
-}
-
-void ia_css_bufq_dump_queue_info(void)
-{
-	int i, j;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "Queue Information:\n");
-
-	for (i = 0; i < SH_CSS_MAX_SP_THREADS; i++) {
-		for (j = 0; j < SH_CSS_MAX_NUM_QUEUES; j++) {
-			snprintf(prefix, BUFQ_DUMP_FILE_NAME_PREFIX_SIZE,
-				"host2sp_buffer_queue[%u][%u]", i, j);
-			bufq_dump_queue_info(prefix,
-				&css_queues.host2sp_buffer_queue_handles[i][j]);
-		}
-	}
-
-	for (i = 0; i < SH_CSS_MAX_NUM_QUEUES; i++) {
-		snprintf(prefix, BUFQ_DUMP_FILE_NAME_PREFIX_SIZE,
-			"sp2host_buffer_queue[%u]", i);
-		bufq_dump_queue_info(prefix,
-			&css_queues.sp2host_buffer_queue_handles[i]);
-	}
-	bufq_dump_queue_info("host2sp_psys_event",
-		&css_queues.host2sp_psys_event_queue_handle);
-	bufq_dump_queue_info("sp2host_psys_event",
-		&css_queues.sp2host_psys_event_queue_handle);
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	bufq_dump_queue_info("host2sp_isys_event",
-		&css_queues.host2sp_isys_event_queue_handle);
-	bufq_dump_queue_info("sp2host_isys_event",
-		&css_queues.sp2host_isys_event_queue_handle);
-	bufq_dump_queue_info("host2sp_tag_cmd",
-		&css_queues.host2sp_tag_cmd_queue_handle);
-#endif
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug.h
deleted file mode 100644
index 4b28b2a0863a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug.h
+++ /dev/null
@@ -1,509 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _IA_CSS_DEBUG_H_
-#define _IA_CSS_DEBUG_H_
-
-/*! \file */
-
-#include <type_support.h>
-#include <stdarg.h>
-#include "ia_css_types.h"
-#include "ia_css_binary.h"
-#include "ia_css_frame_public.h"
-#include "ia_css_pipe_public.h"
-#include "ia_css_stream_public.h"
-#include "ia_css_metadata.h"
-#include "sh_css_internal.h"
-#ifdef ISP2401
-#if defined(IS_ISP_2500_SYSTEM)
-#include "ia_css_pipe.h"
-#endif
-#endif
-
-/* available levels */
-/*! Level for tracing errors */
-#define IA_CSS_DEBUG_ERROR   1
-/*! Level for tracing warnings */
-#define IA_CSS_DEBUG_WARNING 3
-/*! Level for tracing debug messages */
-#define IA_CSS_DEBUG_VERBOSE   5
-/*! Level for tracing trace messages a.o. ia_css public function calls */
-#define IA_CSS_DEBUG_TRACE   6
-/*! Level for tracing trace messages a.o. ia_css private function calls */
-#define IA_CSS_DEBUG_TRACE_PRIVATE   7
-/*! Level for tracing parameter messages e.g. in and out params of functions */
-#define IA_CSS_DEBUG_PARAM   8
-/*! Level for tracing info messages */
-#define IA_CSS_DEBUG_INFO    9
-/* Global variable which controls the verbosity levels of the debug tracing */
-extern unsigned int ia_css_debug_trace_level;
-
-/*! @brief Enum defining the different isp parameters to dump.
- *  Values can be combined to dump a combination of sets.
- */
-enum ia_css_debug_enable_param_dump {
-	IA_CSS_DEBUG_DUMP_FPN = 1 << 0, /** FPN table */
-	IA_CSS_DEBUG_DUMP_OB = 1 << 1,  /** OB table */
-	IA_CSS_DEBUG_DUMP_SC = 1 << 2,  /** Shading table */
-	IA_CSS_DEBUG_DUMP_WB = 1 << 3,  /** White balance */
-	IA_CSS_DEBUG_DUMP_DP = 1 << 4,  /** Defect Pixel */
-	IA_CSS_DEBUG_DUMP_BNR = 1 << 5,  /** Bayer Noise Reductions */
-	IA_CSS_DEBUG_DUMP_S3A = 1 << 6,  /** 3A Statistics */
-	IA_CSS_DEBUG_DUMP_DE = 1 << 7,  /** De Mosaicing */
-	IA_CSS_DEBUG_DUMP_YNR = 1 << 8,  /** Luma Noise Reduction */
-	IA_CSS_DEBUG_DUMP_CSC = 1 << 9,  /** Color Space Conversion */
-	IA_CSS_DEBUG_DUMP_GC = 1 << 10,  /** Gamma Correction */
-	IA_CSS_DEBUG_DUMP_TNR = 1 << 11,  /** Temporal Noise Reduction */
-	IA_CSS_DEBUG_DUMP_ANR = 1 << 12,  /** Advanced Noise Reduction */
-	IA_CSS_DEBUG_DUMP_CE = 1 << 13,  /** Chroma Enhancement */
-	IA_CSS_DEBUG_DUMP_ALL = 1 << 14  /** Dump all device parameters */
-};
-
-#define IA_CSS_ERROR(fmt, ...) \
-	ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR, \
-		"%s() %d: error: " fmt "\n", __func__, __LINE__, ##__VA_ARGS__)
-
-#define IA_CSS_WARNING(fmt, ...) \
-	ia_css_debug_dtrace(IA_CSS_DEBUG_WARNING, \
-		"%s() %d: warning: " fmt "\n", __func__, __LINE__, ##__VA_ARGS__)
-
-/* Logging macros for public functions (API functions) */
-#define IA_CSS_ENTER(fmt, ...) \
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, \
-		"%s(): enter: " fmt "\n", __func__, ##__VA_ARGS__)
-
-/* Use this macro for small functions that do not call other functions. */
-#define IA_CSS_ENTER_LEAVE(fmt, ...) \
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, \
-		"%s(): enter: leave: " fmt "\n", __func__, ##__VA_ARGS__)
-
-#define IA_CSS_LEAVE(fmt, ...) \
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, \
-		"%s(): leave: " fmt "\n", __func__, ##__VA_ARGS__)
-
-/* Shorthand for returning an enum ia_css_err return value */
-#define IA_CSS_LEAVE_ERR(__err) \
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, \
-		"%s() %d: leave: return_err=%d\n", __func__, __LINE__, __err)
-
-/* Use this macro for logging other than enter/leave.
- * Note that this macro always uses the PRIVATE logging level.
- */
-#define IA_CSS_LOG(fmt, ...) \
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, \
-		"%s(): " fmt "\n", __func__, ##__VA_ARGS__)
-
-/* Logging macros for non-API functions. These have a lower trace level */
-#define IA_CSS_ENTER_PRIVATE(fmt, ...) \
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, \
-		"%s(): enter: " fmt "\n", __func__, ##__VA_ARGS__)
-
-#define IA_CSS_LEAVE_PRIVATE(fmt, ...) \
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, \
-		"%s(): leave: " fmt "\n", __func__, ##__VA_ARGS__)
-
-/* Shorthand for returning an enum ia_css_err return value */
-#define IA_CSS_LEAVE_ERR_PRIVATE(__err) \
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, \
-		"%s() %d: leave: return_err=%d\n", __func__, __LINE__, __err)
-
-/* Use this macro for small functions that do not call other functions. */
-#define IA_CSS_ENTER_LEAVE_PRIVATE(fmt, ...) \
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, \
-		"%s(): enter: leave: " fmt "\n", __func__, ##__VA_ARGS__)
-
-/*! @brief Function for tracing to the provided printf function in the
- *	environment.
- * @param[in]	level		Level of the message.
- * @param[in]	fmt		printf like format string
- * @param[in]	args		arguments for the format string
- */
-static inline void
-ia_css_debug_vdtrace(unsigned int level, const char *fmt, va_list args)
-{
-	if (ia_css_debug_trace_level >= level)
-		sh_css_vprint(fmt, args);
-}
-
-__printf(2, 3)
-extern void ia_css_debug_dtrace(unsigned int level, const char *fmt, ...);
-
-/*! @brief Dump sp thread's stack contents
- * SP thread's stack contents are set to 0xcafecafe. This function dumps the
- * stack to inspect if the stack's boundaries are compromised.
- * @return	None
- */
-void ia_css_debug_dump_sp_stack_info(void);
-
-/*! @brief Function to set the global dtrace verbosity level.
- * @param[in]	trace_level	Maximum level of the messages to be traced.
- * @return	None
- */
-void ia_css_debug_set_dtrace_level(
-	const unsigned int	trace_level);
-
-/*! @brief Function to get the global dtrace verbosity level.
- * @return	global dtrace verbosity level
- */
-unsigned int ia_css_debug_get_dtrace_level(void);
-
-/*! @brief Dump input formatter state.
- * Dumps the input formatter state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_if_state(void);
-
-/*! @brief Dump isp hardware state.
- * Dumps the isp hardware state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_isp_state(void);
-
-/*! @brief Dump sp hardware state.
- * Dumps the sp hardware state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_sp_state(void);
-
-#ifdef ISP2401
-/*! @brief Dump GAC hardware state.
- * Dumps the GAC ACB hardware registers. may be useful for
- * detecting a GAC which got hang.
- * @return	None
- */
-void ia_css_debug_dump_gac_state(void);
-
-#endif
-/*! @brief Dump dma controller state.
- * Dumps the dma controller state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_dma_state(void);
-
-/*! @brief Dump internal sp software state.
- * Dumps the sp software state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_sp_sw_debug_info(void);
-
-/*! @brief Dump all related hardware state to the trace output
- * @param[in]  context	String to identify context in output.
- * @return	None
- */
-void ia_css_debug_dump_debug_info(
-	const char	*context);
-
-#if SP_DEBUG != SP_DEBUG_NONE
-void ia_css_debug_print_sp_debug_state(
-	const struct sh_css_sp_debug_state *state);
-#endif
-
-/*! @brief Dump all related binary info data
- * @param[in]  bi	Binary info struct.
- * @return	None
- */
-void ia_css_debug_binary_print(
-	const struct ia_css_binary *bi);
-
-void ia_css_debug_sp_dump_mipi_fifo_high_water(void);
-
-/*! @brief Dump isp gdc fifo state to the trace output
- * Dumps the isp gdc fifo state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_isp_gdc_fifo_state(void);
-
-/*! @brief Dump dma isp fifo state
- * Dumps the dma isp fifo state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_dma_isp_fifo_state(void);
-
-/*! @brief Dump dma sp fifo state
- * Dumps the dma sp fifo state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_dma_sp_fifo_state(void);
-
-/*! \brief Dump pif A isp fifo state
- * Dumps the primary input formatter state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_pif_a_isp_fifo_state(void);
-
-/*! \brief Dump pif B isp fifo state
- * Dumps the primary input formatter state to tracing output.
- * \return	None
- */
-void ia_css_debug_dump_pif_b_isp_fifo_state(void);
-
-/*! @brief Dump stream-to-memory sp fifo state
- * Dumps the stream-to-memory block state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_str2mem_sp_fifo_state(void);
-
-/*! @brief Dump isp sp fifo state
- * Dumps the isp sp fifo state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_isp_sp_fifo_state(void);
-
-/*! @brief Dump all fifo state info to the output
- * Dumps all fifo state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_all_fifo_state(void);
-
-/*! @brief Dump the rx state to the output
- * Dumps the rx state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_rx_state(void);
-
-/*! @brief Dump the input system state to the output
- * Dumps the input system state to tracing output.
- * @return	None
- */
-void ia_css_debug_dump_isys_state(void);
-
-/*! @brief Dump the frame info to the trace output
- * Dumps the frame info to tracing output.
- * @param[in]	frame		pointer to struct ia_css_frame
- * @param[in]	descr		description output along with the frame info
- * @return	None
- */
-void ia_css_debug_frame_print(
-	const struct ia_css_frame	*frame,
-	const char	*descr);
-
-/*! @brief Function to enable sp sleep mode.
- * Function that enables sp sleep mode
- * @param[in]	mode		indicates when to put sp to sleep
- * @return	None
- */
-void ia_css_debug_enable_sp_sleep_mode(enum ia_css_sp_sleep_mode mode);
-
-/*! @brief Function to wake up sp when in sleep mode.
- * After sp has been put to sleep, use this function to let it continue
- * to run again.
- * @return	None
- */
-void ia_css_debug_wake_up_sp(void);
-
-/*! @brief Function to dump isp parameters.
- * Dump isp parameters to tracing output
- * @param[in]	stream		pointer to ia_css_stream struct
- * @param[in]	enable		flag indicating which parameters to dump.
- * @return	None
- */
-void ia_css_debug_dump_isp_params(struct ia_css_stream *stream, unsigned int enable);
-
-/*! @brief Function to dump some sp performance counters.
- * Dump sp performance counters, currently input system errors.
- * @return	None
- */
-void ia_css_debug_dump_perf_counters(void);
-
-#ifdef HAS_WATCHDOG_SP_THREAD_DEBUG
-void sh_css_dump_thread_wait_info(void);
-void sh_css_dump_pipe_stage_info(void);
-void sh_css_dump_pipe_stripe_info(void);
-#endif
-
-void ia_css_debug_dump_isp_binary(void);
-
-void sh_css_dump_sp_raw_copy_linecount(bool reduced);
-
-/*! @brief Dump the resolution info to the trace output
- * Dumps the resolution info to the trace output.
- * @param[in]	res	pointer to struct ia_css_resolution
- * @param[in]	label	description of resolution output
- * @return	None
- */
-void ia_css_debug_dump_resolution(
-	const struct ia_css_resolution *res,
-	const char *label);
-
-/*! @brief Dump the frame info to the trace output
- * Dumps the frame info to the trace output.
- * @param[in]	info	pointer to struct ia_css_frame_info
- * @param[in]	label	description of frame_info output
- * @return	None
- */
-void ia_css_debug_dump_frame_info(
-	const struct ia_css_frame_info *info,
-	const char *label);
-
-/*! @brief Dump the capture config info to the trace output
- * Dumps the capture config info to the trace output.
- * @param[in]	config	pointer to struct ia_css_capture_config
- * @return	None
- */
-void ia_css_debug_dump_capture_config(
-	const struct ia_css_capture_config *config);
-
-/*! @brief Dump the pipe extra config info to the trace output
- * Dumps the pipe extra config info to the trace output.
- * @param[in]	extra_config	pointer to struct ia_css_pipe_extra_config
- * @return	None
- */
-void ia_css_debug_dump_pipe_extra_config(
-	const struct ia_css_pipe_extra_config *extra_config);
-
-/*! @brief Dump the pipe config info to the trace output
- * Dumps the pipe config info to the trace output.
- * @param[in]	config	pointer to struct ia_css_pipe_config
- * @return	None
- */
-void ia_css_debug_dump_pipe_config(
-	const struct ia_css_pipe_config *config);
-
-
-/*! @brief Dump the stream config source info to the trace output
- * Dumps the stream config source info to the trace output.
- * @param[in]	config	pointer to struct ia_css_stream_config
- * @return	None
- */
-void ia_css_debug_dump_stream_config_source(
-	const struct ia_css_stream_config *config);
-
-/*! @brief Dump the mipi buffer config info to the trace output
- * Dumps the mipi buffer config info to the trace output.
- * @param[in]	config	pointer to struct ia_css_mipi_buffer_config
- * @return	None
- */
-void ia_css_debug_dump_mipi_buffer_config(
-	const struct ia_css_mipi_buffer_config *config);
-
-/*! @brief Dump the metadata config info to the trace output
- * Dumps the metadata config info to the trace output.
- * @param[in]	config	pointer to struct ia_css_metadata_config
- * @return	None
- */
-void ia_css_debug_dump_metadata_config(
-	const struct ia_css_metadata_config *config);
-
-/*! @brief Dump the stream config info to the trace output
- * Dumps the stream config info to the trace output.
- * @param[in]	config		pointer to struct ia_css_stream_config
- * @param[in]	num_pipes	number of pipes for the stream
- * @return	None
- */
-void ia_css_debug_dump_stream_config(
-	const struct ia_css_stream_config *config,
-	int num_pipes);
-
-/*! @brief Dump the state of the SP tagger
- * Dumps the internal state of the SP tagger
- * @return	None
- */
-void ia_css_debug_tagger_state(void);
-
-/**
- * @brief Initialize the debug mode.
- *
- * WARNING:
- * This API should be called ONLY once in the debug mode.
- *
- * @return
- *	- true, if it is successful.
- *	- false, otherwise.
- */
-bool ia_css_debug_mode_init(void);
-
-/**
- * @brief Disable the DMA channel.
- *
- * @param[in]	dma_ID		The ID of the target DMA.
- * @param[in]	channel_id	The ID of the target DMA channel.
- * @param[in]	request_type	The type of the DMA request.
- *				For example:
- *				- "0" indicates the writing request.
- *				- "1" indicates the reading request.
- *
- * This is part of the DMA API -> dma.h
- *
- * @return
- *	- true, if it is successful.
- *	- false, otherwise.
- */
-bool ia_css_debug_mode_disable_dma_channel(
-	int dma_ID,
-	int channel_id,
-	int request_type);
-/**
- * @brief Enable the DMA channel.
- *
- * @param[in]	dma_ID		The ID of the target DMA.
- * @param[in]	channel_id	The ID of the target DMA channel.
- * @param[in]	request_type	The type of the DMA request.
- *				For example:
- *				- "0" indicates the writing request.
- *				- "1" indicates the reading request.
- *
- * @return
- *	- true, if it is successful.
- *	- false, otherwise.
- */
-bool ia_css_debug_mode_enable_dma_channel(
-	int dma_ID,
-	int channel_id,
-	int request_type);
-
-/**
- * @brief Dump tracer data.
- * [Currently support is only for SKC]
- *
- * @return
- *	- none.
- */
-void ia_css_debug_dump_trace(void);
-
-#ifdef ISP2401
-/**
- * @brief Program counter dumping (in loop)
- *
- * @param[in]	id		The ID of the SP
- * @param[in]	num_of_dumps	The number of dumps
- *
- * @return
- *	- none
- */
-void ia_css_debug_pc_dump(sp_ID_t id, unsigned int num_of_dumps);
-
-#if defined(IS_ISP_2500_SYSTEM)
-/*! @brief Dump all states for ISP hang case.
- * Dumps the ISP previous and current configurations
- * GACs status, SP0/1 statuses.
- *
- * @param[in]	pipe	The current pipe
- *
- * @return	None
- */
-void ia_css_debug_dump_hang_status(
-	struct ia_css_pipe *pipe);
-
-/*! @brief External command handler
- * External command handler
- *
- * @return	None
- */
-void ia_css_debug_ext_command_handler(void);
-
-#endif
-#endif
-
-#endif /* _IA_CSS_DEBUG_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug_internal.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug_internal.h
deleted file mode 100644
index 88d025807201..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug_internal.h
+++ /dev/null
@@ -1,31 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-/* TO DO: Move debug related code from ia_css_internal.h in */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug_pipe.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug_pipe.h
deleted file mode 100644
index 72ac0e32ebf7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/interface/ia_css_debug_pipe.h
+++ /dev/null
@@ -1,84 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _IA_CSS_DEBUG_PIPE_H_
-#define _IA_CSS_DEBUG_PIPE_H_
-
-/*! \file */
-
-#include <ia_css_frame_public.h>
-#include <ia_css_stream_public.h>
-#include "ia_css_pipeline.h"
-
-/**
- * @brief Internal debug support for constructing a pipe graph.
- *
- * @return	None
- */
-extern void ia_css_debug_pipe_graph_dump_prologue(void);
-
-/**
- * @brief Internal debug support for constructing a pipe graph.
- *
- * @return	None
- */
-extern void ia_css_debug_pipe_graph_dump_epilogue(void);
-
-/**
- * @brief Internal debug support for constructing a pipe graph.
- * @param[in]	stage		Pipeline stage.
- * @param[in]	id		Pipe id.
- *
- * @return	None
- */
-extern void ia_css_debug_pipe_graph_dump_stage(
-		struct ia_css_pipeline_stage *stage,
-		enum ia_css_pipe_id id);
-
-/**
- * @brief Internal debug support for constructing a pipe graph.
- * @param[in]	out_frame	Output frame of SP raw copy.
- *
- * @return	None
- */
-extern void ia_css_debug_pipe_graph_dump_sp_raw_copy(
-		struct ia_css_frame *out_frame);
-
-
-/**
- * @brief Internal debug support for constructing a pipe graph.
- * @param[in]	stream_config	info about sensor and input formatter.
- *
- * @return	None
- */
-extern void ia_css_debug_pipe_graph_dump_stream_config(
-		const struct ia_css_stream_config *stream_config);
-
-#endif /* _IA_CSS_DEBUG_PIPE_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/src/ia_css_debug.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/src/ia_css_debug.c
deleted file mode 100644
index 4607a76dc78a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/debug/src/ia_css_debug.c
+++ /dev/null
@@ -1,3596 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "debug.h"
-#include "memory_access.h"
-
-#ifndef __INLINE_INPUT_SYSTEM__
-#define __INLINE_INPUT_SYSTEM__
-#endif
-#ifndef __INLINE_IBUF_CTRL__
-#define __INLINE_IBUF_CTRL__
-#endif
-#ifndef __INLINE_CSI_RX__
-#define __INLINE_CSI_RX__
-#endif
-#ifndef __INLINE_PIXELGEN__
-#define __INLINE_PIXELGEN__
-#endif
-#ifndef __INLINE_STREAM2MMIO__
-#define __INLINE_STREAM2MMIO__
-#endif
-
-#include "ia_css_debug.h"
-#include "ia_css_debug_pipe.h"
-#include "ia_css_irq.h"
-#include "ia_css_stream.h"
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_param.h"
-#include "sh_css_params.h"
-#include "ia_css_bufq.h"
-#ifdef ISP2401
-#include "ia_css_queue.h"
-#endif
-
-#include "ia_css_isp_params.h"
-
-#include "system_local.h"
-#include "assert_support.h"
-#include "print_support.h"
-#include "string_support.h"
-#ifdef ISP2401
-#include "ia_css_system_ctrl.h"
-#endif
-
-#include "fifo_monitor.h"
-
-#if !defined(HAS_NO_INPUT_FORMATTER)
-#include "input_formatter.h"
-#endif
-#include "dma.h"
-#include "irq.h"
-#include "gp_device.h"
-#include "sp.h"
-#include "isp.h"
-#include "type_support.h"
-#include "math_support.h" /* CEIL_DIV */
-#if defined(HAS_INPUT_FORMATTER_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-#include "input_system.h"	/* input_formatter_reg_load */
-#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-#include "ia_css_tagger_common.h"
-#endif
-
-#include "sh_css_internal.h"
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#include "ia_css_isys.h"
-#endif
-#include "sh_css_sp.h"		/* sh_css_sp_get_debug_state() */
-
-#include "css_trace.h"      /* tracer */
-
-#include "device_access.h"	/* for ia_css_device_load_uint32 */
-
-/* Include all kernel host interfaces for ISP1 */
-#include "anr/anr_1.0/ia_css_anr.host.h"
-#include "cnr/cnr_1.0/ia_css_cnr.host.h"
-#include "csc/csc_1.0/ia_css_csc.host.h"
-#include "de/de_1.0/ia_css_de.host.h"
-#include "dp/dp_1.0/ia_css_dp.host.h"
-#include "bnr/bnr_1.0/ia_css_bnr.host.h"
-#include "fpn/fpn_1.0/ia_css_fpn.host.h"
-#include "gc/gc_1.0/ia_css_gc.host.h"
-#include "ob/ob_1.0/ia_css_ob.host.h"
-#include "s3a/s3a_1.0/ia_css_s3a.host.h"
-#include "sc/sc_1.0/ia_css_sc.host.h"
-#include "tnr/tnr_1.0/ia_css_tnr.host.h"
-#include "uds/uds_1.0/ia_css_uds_param.h"
-#include "wb/wb_1.0/ia_css_wb.host.h"
-#include "ynr/ynr_1.0/ia_css_ynr.host.h"
-
-/* Include additional kernel host interfaces for ISP2 */
-#include "aa/aa_2/ia_css_aa2.host.h"
-#include "anr/anr_2/ia_css_anr2.host.h"
-#include "cnr/cnr_2/ia_css_cnr2.host.h"
-#include "de/de_2/ia_css_de2.host.h"
-#include "gc/gc_2/ia_css_gc2.host.h"
-#include "ynr/ynr_2/ia_css_ynr2.host.h"
-
-/* Global variable to store the dtrace verbosity level */
-unsigned int ia_css_debug_trace_level = IA_CSS_DEBUG_WARNING;
-
-#define DPG_START "ia_css_debug_pipe_graph_dump_start "
-#define DPG_END   " ia_css_debug_pipe_graph_dump_end\n"
-
-#define ENABLE_LINE_MAX_LENGTH (25)
-
-#ifdef ISP2401
-#define DBG_EXT_CMD_TRACE_PNTS_DUMP (1 << 8)
-#define DBG_EXT_CMD_PUB_CFG_DUMP (1 << 9)
-#define DBG_EXT_CMD_GAC_REG_DUMP (1 << 10)
-#define DBG_EXT_CMD_GAC_ACB_REG_DUMP (1 << 11)
-#define DBG_EXT_CMD_FIFO_DUMP (1 << 12)
-#define DBG_EXT_CMD_QUEUE_DUMP (1 << 13)
-#define DBG_EXT_CMD_DMA_DUMP (1 << 14)
-#define DBG_EXT_CMD_MASK 0xAB0000CD
-
-#endif
-/*
- * TODO:SH_CSS_MAX_SP_THREADS is not the max number of sp threads
- * future rework should fix this and remove the define MAX_THREAD_NUM
- */
-#define MAX_THREAD_NUM (SH_CSS_MAX_SP_THREADS + SH_CSS_MAX_SP_INTERNAL_THREADS)
-
-static struct pipe_graph_class {
-	bool do_init;
-	int height;
-	int width;
-	int eff_height;
-	int eff_width;
-	enum atomisp_input_format stream_format;
-} pg_inst = {true, 0, 0, 0, 0, N_ATOMISP_INPUT_FORMAT};
-
-static const char * const queue_id_to_str[] = {
-	/* [SH_CSS_QUEUE_A_ID]     =*/ "queue_A",
-	/* [SH_CSS_QUEUE_B_ID]     =*/ "queue_B",
-	/* [SH_CSS_QUEUE_C_ID]     =*/ "queue_C",
-	/* [SH_CSS_QUEUE_D_ID]     =*/ "queue_D",
-	/* [SH_CSS_QUEUE_E_ID]     =*/ "queue_E",
-	/* [SH_CSS_QUEUE_F_ID]     =*/ "queue_F",
-	/* [SH_CSS_QUEUE_G_ID]     =*/ "queue_G",
-	/* [SH_CSS_QUEUE_H_ID]     =*/ "queue_H"
-};
-
-static const char * const pipe_id_to_str[] = {
-	/* [IA_CSS_PIPE_ID_PREVIEW]   =*/ "preview",
-	/* [IA_CSS_PIPE_ID_COPY]      =*/ "copy",
-	/* [IA_CSS_PIPE_ID_VIDEO]     =*/ "video",
-	/* [IA_CSS_PIPE_ID_CAPTURE]   =*/ "capture",
-	/* [IA_CSS_PIPE_ID_YUVPP]     =*/ "yuvpp",
-	/* [IA_CSS_PIPE_ID_ACC]       =*/ "accelerator"
-};
-
-static char dot_id_input_bin[SH_CSS_MAX_BINARY_NAME+10];
-static char ring_buffer[200];
-
-void ia_css_debug_dtrace(unsigned int level, const char *fmt, ...)
-{
-	va_list ap;
-
-	va_start(ap, fmt);
-	ia_css_debug_vdtrace(level, fmt, ap);
-	va_end(ap);
-}
-
-static void debug_dump_long_array_formatted(
-	const sp_ID_t sp_id,
-	hrt_address stack_sp_addr,
-	unsigned stack_size)
-{
-	unsigned int i;
-	uint32_t val;
-	uint32_t addr = (uint32_t) stack_sp_addr;
-	uint32_t stack_size_words = CEIL_DIV(stack_size, sizeof(uint32_t));
-
-	/* When size is not multiple of four, last word is only relevant for
-	 * remaining bytes */
-	for (i = 0; i < stack_size_words; i++) {
-		val = sp_dmem_load_uint32(sp_id, (hrt_address)addr);
-		if ((i%8) == 0)
-			ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "\n");
-
-		ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "0x%08x ", val);
-		addr += sizeof(uint32_t);
-	}
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "\n");
-}
-
-static void debug_dump_sp_stack_info(
-	const sp_ID_t sp_id)
-{
-	const struct ia_css_fw_info *fw;
-	unsigned int HIVE_ADDR_sp_threads_stack;
-	unsigned int HIVE_ADDR_sp_threads_stack_size;
-	uint32_t stack_sizes[MAX_THREAD_NUM];
-	uint32_t stack_sp_addr[MAX_THREAD_NUM];
-	unsigned int i;
-
-	fw = &sh_css_sp_fw;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "sp_id(%u) stack info\n", sp_id);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-		"from objects stack_addr_offset:0x%x stack_size_offset:0x%x\n",
-		fw->info.sp.threads_stack,
-		fw->info.sp.threads_stack_size);
-
-	HIVE_ADDR_sp_threads_stack = fw->info.sp.threads_stack;
-	HIVE_ADDR_sp_threads_stack_size = fw->info.sp.threads_stack_size;
-
-	if (fw->info.sp.threads_stack == 0 ||
-		fw->info.sp.threads_stack_size == 0)
-		return;
-
-	(void) HIVE_ADDR_sp_threads_stack;
-	(void) HIVE_ADDR_sp_threads_stack_size;
-
-	sp_dmem_load(sp_id,
-		(unsigned int)sp_address_of(sp_threads_stack),
-		&stack_sp_addr, sizeof(stack_sp_addr));
-	sp_dmem_load(sp_id,
-		(unsigned int)sp_address_of(sp_threads_stack_size),
-		&stack_sizes, sizeof(stack_sizes));
-
-	for (i = 0 ; i < MAX_THREAD_NUM; i++) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			"thread: %u stack_addr: 0x%08x stack_size: %u\n",
-			i, stack_sp_addr[i], stack_sizes[i]);
-		debug_dump_long_array_formatted(sp_id, (hrt_address)stack_sp_addr[i],
-			stack_sizes[i]);
-	}
-}
-
-void ia_css_debug_dump_sp_stack_info(void)
-{
-	debug_dump_sp_stack_info(SP0_ID);
-}
-
-
-void ia_css_debug_set_dtrace_level(const unsigned int trace_level)
-{
-	ia_css_debug_trace_level = trace_level;
-	return;
-}
-
-unsigned int ia_css_debug_get_dtrace_level(void)
-{
-	return ia_css_debug_trace_level;
-}
-
-static const char *debug_stream_format2str(const enum atomisp_input_format stream_format)
-{
-	switch (stream_format) {
-	case ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY:
-		return "yuv420-8-legacy";
-	case ATOMISP_INPUT_FORMAT_YUV420_8:
-		return "yuv420-8";
-	case ATOMISP_INPUT_FORMAT_YUV420_10:
-		return "yuv420-10";
-	case ATOMISP_INPUT_FORMAT_YUV420_16:
-		return "yuv420-16";
-	case ATOMISP_INPUT_FORMAT_YUV422_8:
-		return "yuv422-8";
-	case ATOMISP_INPUT_FORMAT_YUV422_10:
-		return "yuv422-10";
-	case ATOMISP_INPUT_FORMAT_YUV422_16:
-		return "yuv422-16";
-	case ATOMISP_INPUT_FORMAT_RGB_444:
-		return "rgb444";
-	case ATOMISP_INPUT_FORMAT_RGB_555:
-		return "rgb555";
-	case ATOMISP_INPUT_FORMAT_RGB_565:
-		return "rgb565";
-	case ATOMISP_INPUT_FORMAT_RGB_666:
-		return "rgb666";
-	case ATOMISP_INPUT_FORMAT_RGB_888:
-		return "rgb888";
-	case ATOMISP_INPUT_FORMAT_RAW_6:
-		return "raw6";
-	case ATOMISP_INPUT_FORMAT_RAW_7:
-		return "raw7";
-	case ATOMISP_INPUT_FORMAT_RAW_8:
-		return "raw8";
-	case ATOMISP_INPUT_FORMAT_RAW_10:
-		return "raw10";
-	case ATOMISP_INPUT_FORMAT_RAW_12:
-		return "raw12";
-	case ATOMISP_INPUT_FORMAT_RAW_14:
-		return "raw14";
-	case ATOMISP_INPUT_FORMAT_RAW_16:
-		return "raw16";
-	case ATOMISP_INPUT_FORMAT_BINARY_8:
-		return "binary8";
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT1:
-		return "generic-short1";
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT2:
-		return "generic-short2";
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT3:
-		return "generic-short3";
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT4:
-		return "generic-short4";
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT5:
-		return "generic-short5";
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT6:
-		return "generic-short6";
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT7:
-		return "generic-short7";
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT8:
-		return "generic-short8";
-	case ATOMISP_INPUT_FORMAT_YUV420_8_SHIFT:
-		return "yuv420-8-shift";
-	case ATOMISP_INPUT_FORMAT_YUV420_10_SHIFT:
-		return "yuv420-10-shift";
-	case ATOMISP_INPUT_FORMAT_EMBEDDED:
-		return "embedded-8";
-	case ATOMISP_INPUT_FORMAT_USER_DEF1:
-		return "user-def-8-type-1";
-	case ATOMISP_INPUT_FORMAT_USER_DEF2:
-		return "user-def-8-type-2";
-	case ATOMISP_INPUT_FORMAT_USER_DEF3:
-		return "user-def-8-type-3";
-	case ATOMISP_INPUT_FORMAT_USER_DEF4:
-		return "user-def-8-type-4";
-	case ATOMISP_INPUT_FORMAT_USER_DEF5:
-		return "user-def-8-type-5";
-	case ATOMISP_INPUT_FORMAT_USER_DEF6:
-		return "user-def-8-type-6";
-	case ATOMISP_INPUT_FORMAT_USER_DEF7:
-		return "user-def-8-type-7";
-	case ATOMISP_INPUT_FORMAT_USER_DEF8:
-		return "user-def-8-type-8";
-
-	default:
-		assert(!"Unknown stream format");
-		return "unknown-stream-format";
-	}
-};
-
-static const char *debug_frame_format2str(const enum ia_css_frame_format frame_format)
-{
-	switch (frame_format) {
-
-	case IA_CSS_FRAME_FORMAT_NV11:
-		return "NV11";
-	case IA_CSS_FRAME_FORMAT_NV12:
-		return "NV12";
-	case IA_CSS_FRAME_FORMAT_NV12_16:
-		return "NV12_16";
-	case IA_CSS_FRAME_FORMAT_NV12_TILEY:
-		return "NV12_TILEY";
-	case IA_CSS_FRAME_FORMAT_NV16:
-		return "NV16";
-	case IA_CSS_FRAME_FORMAT_NV21:
-		return "NV21";
-	case IA_CSS_FRAME_FORMAT_NV61:
-		return "NV61";
-	case IA_CSS_FRAME_FORMAT_YV12:
-		return "YV12";
-	case IA_CSS_FRAME_FORMAT_YV16:
-		return "YV16";
-	case IA_CSS_FRAME_FORMAT_YUV420:
-		return "YUV420";
-	case IA_CSS_FRAME_FORMAT_YUV420_16:
-		return "YUV420_16";
-	case IA_CSS_FRAME_FORMAT_YUV422:
-		return "YUV422";
-	case IA_CSS_FRAME_FORMAT_YUV422_16:
-		return "YUV422_16";
-	case IA_CSS_FRAME_FORMAT_UYVY:
-		return "UYVY";
-	case IA_CSS_FRAME_FORMAT_YUYV:
-		return "YUYV";
-	case IA_CSS_FRAME_FORMAT_YUV444:
-		return "YUV444";
-	case IA_CSS_FRAME_FORMAT_YUV_LINE:
-		return "YUV_LINE";
-	case IA_CSS_FRAME_FORMAT_RAW:
-		return "RAW";
-	case IA_CSS_FRAME_FORMAT_RGB565:
-		return "RGB565";
-	case IA_CSS_FRAME_FORMAT_PLANAR_RGB888:
-		return "PLANAR_RGB888";
-	case IA_CSS_FRAME_FORMAT_RGBA888:
-		return "RGBA888";
-	case IA_CSS_FRAME_FORMAT_QPLANE6:
-		return "QPLANE6";
-	case IA_CSS_FRAME_FORMAT_BINARY_8:
-		return "BINARY_8";
-	case IA_CSS_FRAME_FORMAT_MIPI:
-		return "MIPI";
-	case IA_CSS_FRAME_FORMAT_RAW_PACKED:
-		return "RAW_PACKED";
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_8:
-		return "CSI_MIPI_YUV420_8";
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_LEGACY_YUV420_8:
-		return "CSI_MIPI_LEGACY_YUV420_8";
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_10:
-		return "CSI_MIPI_YUV420_10";
-
-	default:
-		assert(!"Unknown frame format");
-		return "unknown-frame-format";
-	}
-}
-
-static void debug_print_sp_state(const sp_state_t *state, const char *cell)
-{
-	assert(cell != NULL);
-	assert(state != NULL);
-
-	ia_css_debug_dtrace(2, "%s state:\n", cell);
-	ia_css_debug_dtrace(2, "\t%-32s: 0x%X\n", "PC", state->pc);
-	ia_css_debug_dtrace(2, "\t%-32s: 0x%X\n", "Status register",
-			    state->status_register);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "Is broken", state->is_broken);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "Is idle", state->is_idle);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "Is sleeping",
-			    state->is_sleeping);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "Is stalling",
-			    state->is_stalling);
-	return;
-}
-
-static void debug_print_isp_state(const isp_state_t *state, const char *cell)
-{
-	assert(state != NULL);
-	assert(cell != NULL);
-
-	ia_css_debug_dtrace(2, "%s state:\n", cell);
-	ia_css_debug_dtrace(2, "\t%-32s: 0x%X\n", "PC", state->pc);
-	ia_css_debug_dtrace(2, "\t%-32s: 0x%X\n", "Status register",
-			    state->status_register);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "Is broken", state->is_broken);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "Is idle", state->is_idle);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "Is sleeping",
-			    state->is_sleeping);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "Is stalling",
-			    state->is_stalling);
-	return;
-}
-
-void ia_css_debug_dump_isp_state(void)
-{
-	isp_state_t state;
-	isp_stall_t stall;
-
-	isp_get_state(ISP0_ID, &state, &stall);
-
-	debug_print_isp_state(&state, "ISP");
-
-	if (state.is_stalling) {
-#if !defined(HAS_NO_INPUT_FORMATTER)
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n",
-				    "[0] if_prim_a_FIFO stalled", stall.fifo0);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n",
-				    "[1] if_prim_b_FIFO stalled", stall.fifo1);
-#endif
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[2] dma_FIFO stalled",
-				    stall.fifo2);
-#if defined(HAS_ISP_2400_MAMOIADA) || defined(HAS_ISP_2401_MAMOIADA) || defined(IS_ISP_2500_SYSTEM)
-
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[3] gdc0_FIFO stalled",
-				    stall.fifo3);
-#if !defined(IS_ISP_2500_SYSTEM)
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[4] gdc1_FIFO stalled",
-				    stall.fifo4);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[5] gpio_FIFO stalled",
-				    stall.fifo5);
-#endif
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[6] sp_FIFO stalled",
-				    stall.fifo6);
-#else
-#error "ia_css_debug: ISP cell must be one of {2400_MAMOIADA,, 2401_MAMOIADA, 2500_SKYCAM}"
-#endif
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n",
-				    "status & control stalled",
-				    stall.stat_ctrl);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "dmem stalled",
-				    stall.dmem);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "vmem stalled",
-				    stall.vmem);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "vamem1 stalled",
-				    stall.vamem1);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "vamem2 stalled",
-				    stall.vamem2);
-#if defined(HAS_ISP_2400_MAMOIADA) || defined(HAS_ISP_2401_MAMOIADA)
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "vamem3 stalled",
-				    stall.vamem3);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "hmem stalled",
-				    stall.hmem);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "pmem stalled",
-				    stall.pmem);
-#endif
-	}
-	return;
-}
-
-void ia_css_debug_dump_sp_state(void)
-{
-	sp_state_t state;
-	sp_stall_t stall;
-	sp_get_state(SP0_ID, &state, &stall);
-	debug_print_sp_state(&state, "SP");
-	if (state.is_stalling) {
-#if defined(HAS_SP_2400) || defined(IS_ISP_2500_SYSTEM)
-#if !defined(HAS_NO_INPUT_SYSTEM)
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "isys_FIFO stalled",
-				    stall.fifo0);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "if_sec_FIFO stalled",
-				    stall.fifo1);
-#endif
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n",
-				    "str_to_mem_FIFO stalled", stall.fifo2);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "dma_FIFO stalled",
-				    stall.fifo3);
-#if !defined(HAS_NO_INPUT_FORMATTER)
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n",
-				    "if_prim_a_FIFO stalled", stall.fifo4);
-#endif
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "isp_FIFO stalled",
-				    stall.fifo5);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "gp_FIFO stalled",
-				    stall.fifo6);
-#if !defined(HAS_NO_INPUT_FORMATTER)
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n",
-				    "if_prim_b_FIFO stalled", stall.fifo7);
-#endif
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "gdc0_FIFO stalled",
-				    stall.fifo8);
-#if !defined(IS_ISP_2500_SYSTEM)
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "gdc1_FIFO stalled",
-				    stall.fifo9);
-#endif
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "irq FIFO stalled",
-				    stall.fifoa);
-#else
-#error "ia_css_debug: SP cell must be one of {SP2400, SP2500}"
-#endif
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "dmem stalled",
-				    stall.dmem);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n",
-				    "control master stalled",
-				    stall.control_master);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n",
-				    "i-cache master stalled",
-				    stall.icache_master);
-	}
-	ia_css_debug_dump_trace();
-	return;
-}
-
-static void debug_print_fifo_channel_state(const fifo_channel_state_t *state,
-					   const char *descr)
-{
-	assert(state != NULL);
-	assert(descr != NULL);
-
-	ia_css_debug_dtrace(2, "FIFO channel: %s\n", descr);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "source valid",
-			    state->src_valid);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "fifo accept",
-			    state->fifo_accept);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "fifo valid",
-			    state->fifo_valid);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "sink accept",
-			    state->sink_accept);
-	return;
-}
-
-#if !defined(HAS_NO_INPUT_FORMATTER) && defined(USE_INPUT_SYSTEM_VERSION_2)
-void ia_css_debug_dump_pif_a_isp_fifo_state(void)
-{
-	fifo_channel_state_t pif_to_isp, isp_to_pif;
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_IF0_TO_ISP0, &pif_to_isp);
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_ISP0_TO_IF0, &isp_to_pif);
-	debug_print_fifo_channel_state(&pif_to_isp, "Primary IF A to ISP");
-	debug_print_fifo_channel_state(&isp_to_pif, "ISP to Primary IF A");
-}
-
-void ia_css_debug_dump_pif_b_isp_fifo_state(void)
-{
-	fifo_channel_state_t pif_to_isp, isp_to_pif;
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_IF1_TO_ISP0, &pif_to_isp);
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_ISP0_TO_IF1, &isp_to_pif);
-	debug_print_fifo_channel_state(&pif_to_isp, "Primary IF B to ISP");
-	debug_print_fifo_channel_state(&isp_to_pif, "ISP to Primary IF B");
-}
-
-void ia_css_debug_dump_str2mem_sp_fifo_state(void)
-{
-	fifo_channel_state_t s2m_to_sp, sp_to_s2m;
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_STREAM2MEM0_TO_SP0, &s2m_to_sp);
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_SP0_TO_STREAM2MEM0, &sp_to_s2m);
-	debug_print_fifo_channel_state(&s2m_to_sp, "Stream-to-memory to SP");
-	debug_print_fifo_channel_state(&sp_to_s2m, "SP to stream-to-memory");
-}
-
-static void debug_print_if_state(input_formatter_state_t *state, const char *id)
-{
-	unsigned int val;
-
-#if defined(HAS_INPUT_FORMATTER_VERSION_1)
-	const char *st_reset = (state->reset ? "Active" : "Not active");
-#endif
-	const char *st_vsync_active_low =
-	    (state->vsync_active_low ? "low" : "high");
-	const char *st_hsync_active_low =
-	    (state->hsync_active_low ? "low" : "high");
-
-	const char *fsm_sync_status_str = "unknown";
-	const char *fsm_crop_status_str = "unknown";
-	const char *fsm_padding_status_str = "unknown";
-
-	int st_stline = state->start_line;
-	int st_stcol = state->start_column;
-	int st_crpht = state->cropped_height;
-	int st_crpwd = state->cropped_width;
-	int st_verdcm = state->ver_decimation;
-	int st_hordcm = state->hor_decimation;
-	int st_ver_deinterleaving = state->ver_deinterleaving;
-	int st_hor_deinterleaving = state->hor_deinterleaving;
-	int st_leftpd = state->left_padding;
-	int st_eoloff = state->eol_offset;
-	int st_vmstartaddr = state->vmem_start_address;
-	int st_vmendaddr = state->vmem_end_address;
-	int st_vmincr = state->vmem_increment;
-	int st_yuv420 = state->is_yuv420;
-	int st_allow_fifo_overflow = state->allow_fifo_overflow;
-	int st_block_fifo_when_no_req = state->block_fifo_when_no_req;
-
-	assert(state != NULL);
-	ia_css_debug_dtrace(2, "InputFormatter State (%s):\n", id);
-
-	ia_css_debug_dtrace(2, "\tConfiguration:\n");
-
-#if defined(HAS_INPUT_FORMATTER_VERSION_1)
-	ia_css_debug_dtrace(2, "\t\t%-32s: %s\n", "Software reset", st_reset);
-#endif
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Start line", st_stline);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Start column", st_stcol);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Cropped height", st_crpht);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Cropped width", st_crpwd);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Ver decimation", st_verdcm);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Hor decimation", st_hordcm);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Ver deinterleaving", st_ver_deinterleaving);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Hor deinterleaving", st_hor_deinterleaving);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Left padding", st_leftpd);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "EOL offset (bytes)", st_eoloff);
-	ia_css_debug_dtrace(2, "\t\t%-32s: 0x%06X\n",
-			    "VMEM start address", st_vmstartaddr);
-	ia_css_debug_dtrace(2, "\t\t%-32s: 0x%06X\n",
-			    "VMEM end address", st_vmendaddr);
-	ia_css_debug_dtrace(2, "\t\t%-32s: 0x%06X\n",
-			    "VMEM increment", st_vmincr);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "YUV 420 format", st_yuv420);
-	ia_css_debug_dtrace(2, "\t\t%-32s: Active %s\n",
-			    "Vsync", st_vsync_active_low);
-	ia_css_debug_dtrace(2, "\t\t%-32s: Active %s\n",
-			    "Hsync", st_hsync_active_low);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Allow FIFO overflow", st_allow_fifo_overflow);
-/* Flag that tells whether the IF gives backpressure on frames */
-/*
- * FYI, this is only on the frame request (indicate), when the IF has
- * synch'd on a frame it will always give back pressure
- */
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Block when no request", st_block_fifo_when_no_req);
-
-#if defined(HAS_INPUT_FORMATTER_VERSION_2)
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "IF_BLOCKED_FIFO_NO_REQ_ADDRESS",
-			    input_formatter_reg_load(INPUT_FORMATTER0_ID,
-			    HIVE_IF_BLOCK_FIFO_NO_REQ_ADDRESS)
-	    );
-
-	ia_css_debug_dtrace(2, "\t%-32s:\n", "InputSwitch State");
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "_REG_GP_IFMT_input_switch_lut_reg0",
-			    gp_device_reg_load(GP_DEVICE0_ID,
-			    _REG_GP_IFMT_input_switch_lut_reg0));
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "_REG_GP_IFMT_input_switch_lut_reg1",
-			    gp_device_reg_load(GP_DEVICE0_ID,
-				_REG_GP_IFMT_input_switch_lut_reg1));
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "_REG_GP_IFMT_input_switch_lut_reg2",
-			    gp_device_reg_load(GP_DEVICE0_ID,
-				_REG_GP_IFMT_input_switch_lut_reg2));
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "_REG_GP_IFMT_input_switch_lut_reg3",
-			    gp_device_reg_load(GP_DEVICE0_ID,
-				_REG_GP_IFMT_input_switch_lut_reg3));
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "_REG_GP_IFMT_input_switch_lut_reg4",
-			    gp_device_reg_load(GP_DEVICE0_ID,
-				_REG_GP_IFMT_input_switch_lut_reg4));
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "_REG_GP_IFMT_input_switch_lut_reg5",
-			    gp_device_reg_load(GP_DEVICE0_ID,
-				_REG_GP_IFMT_input_switch_lut_reg5));
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "_REG_GP_IFMT_input_switch_lut_reg6",
-			    gp_device_reg_load(GP_DEVICE0_ID,
-				_REG_GP_IFMT_input_switch_lut_reg6));
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "_REG_GP_IFMT_input_switch_lut_reg7",
-			    gp_device_reg_load(GP_DEVICE0_ID,
-				_REG_GP_IFMT_input_switch_lut_reg7));
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "_REG_GP_IFMT_input_switch_fsync_lut",
-			    gp_device_reg_load(GP_DEVICE0_ID,
-				_REG_GP_IFMT_input_switch_fsync_lut));
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "_REG_GP_IFMT_srst",
-			    gp_device_reg_load(GP_DEVICE0_ID,
-				_REG_GP_IFMT_srst));
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "_REG_GP_IFMT_slv_reg_srst",
-			    gp_device_reg_load(GP_DEVICE0_ID,
-				 _REG_GP_IFMT_slv_reg_srst));
-#endif
-
-	ia_css_debug_dtrace(2, "\tFSM Status:\n");
-
-	val = state->fsm_sync_status;
-
-	if (val > 7)
-		fsm_sync_status_str = "ERROR";
-
-	switch (val & 0x7) {
-	case 0:
-		fsm_sync_status_str = "idle";
-		break;
-	case 1:
-		fsm_sync_status_str = "request frame";
-		break;
-	case 2:
-		fsm_sync_status_str = "request lines";
-		break;
-	case 3:
-		fsm_sync_status_str = "request vectors";
-		break;
-	case 4:
-		fsm_sync_status_str = "send acknowledge";
-		break;
-	default:
-		fsm_sync_status_str = "unknown";
-		break;
-	}
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: (0x%X: %s)\n",
-			    "FSM Synchronization Status", val,
-			    fsm_sync_status_str);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "FSM Synchronization Counter",
-			    state->fsm_sync_counter);
-
-	val = state->fsm_crop_status;
-
-	if (val > 7)
-		fsm_crop_status_str = "ERROR";
-
-	switch (val & 0x7) {
-	case 0:
-		fsm_crop_status_str = "idle";
-		break;
-	case 1:
-		fsm_crop_status_str = "wait line";
-		break;
-	case 2:
-		fsm_crop_status_str = "crop line";
-		break;
-	case 3:
-		fsm_crop_status_str = "crop pixel";
-		break;
-	case 4:
-		fsm_crop_status_str = "pass pixel";
-		break;
-	case 5:
-		fsm_crop_status_str = "pass line";
-		break;
-	case 6:
-		fsm_crop_status_str = "lost line";
-		break;
-	default:
-		fsm_crop_status_str = "unknown";
-		break;
-	}
-	ia_css_debug_dtrace(2, "\t\t%-32s: (0x%X: %s)\n",
-			    "FSM Crop Status", val, fsm_crop_status_str);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "FSM Crop Line Counter",
-			    state->fsm_crop_line_counter);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "FSM Crop Pixel Counter",
-			    state->fsm_crop_pixel_counter);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "FSM Deinterleaving idx buffer",
-			    state->fsm_deinterleaving_index);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "FSM H decimation counter",
-			    state->fsm_dec_h_counter);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "FSM V decimation counter",
-			    state->fsm_dec_v_counter);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "FSM block V decimation counter",
-			    state->fsm_dec_block_v_counter);
-
-	val = state->fsm_padding_status;
-
-	if (val > 7)
-		fsm_padding_status_str = "ERROR";
-
-	switch (val & 0x7) {
-	case 0:
-		fsm_padding_status_str = "idle";
-		break;
-	case 1:
-		fsm_padding_status_str = "left pad";
-		break;
-	case 2:
-		fsm_padding_status_str = "write";
-		break;
-	case 3:
-		fsm_padding_status_str = "right pad";
-		break;
-	case 4:
-		fsm_padding_status_str = "send end of line";
-		break;
-	default:
-		fsm_padding_status_str = "unknown";
-		break;
-	}
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: (0x%X: %s)\n", "FSM Padding Status",
-			    val, fsm_padding_status_str);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "FSM Padding element idx counter",
-			    state->fsm_padding_elem_counter);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Vector support error",
-			    state->fsm_vector_support_error);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Vector support buf full",
-			    state->fsm_vector_buffer_full);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Vector support",
-			    state->vector_support);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Fifo sensor data lost",
-			    state->sensor_data_lost);
-	return;
-}
-
-static void debug_print_if_bin_state(input_formatter_bin_state_t *state)
-{
-	ia_css_debug_dtrace(2, "Stream-to-memory state:\n");
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "reset", state->reset);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "input endianness",
-			    state->input_endianness);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "output endianness",
-			    state->output_endianness);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "bitswap", state->bitswap);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "block_synch",
-			    state->block_synch);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "packet_synch",
-			    state->packet_synch);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "readpostwrite_sync",
-			    state->readpostwrite_synch);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "is_2ppc", state->is_2ppc);
-	ia_css_debug_dtrace(2, "\t%-32s: %d\n", "en_status_update",
-			    state->en_status_update);
-}
-
-void ia_css_debug_dump_if_state(void)
-{
-	input_formatter_state_t if_state;
-	input_formatter_bin_state_t if_bin_state;
-
-	input_formatter_get_state(INPUT_FORMATTER0_ID, &if_state);
-	debug_print_if_state(&if_state, "Primary IF A");
-	ia_css_debug_dump_pif_a_isp_fifo_state();
-
-	input_formatter_get_state(INPUT_FORMATTER1_ID, &if_state);
-	debug_print_if_state(&if_state, "Primary IF B");
-	ia_css_debug_dump_pif_b_isp_fifo_state();
-
-	input_formatter_bin_get_state(INPUT_FORMATTER3_ID, &if_bin_state);
-	debug_print_if_bin_state(&if_bin_state);
-	ia_css_debug_dump_str2mem_sp_fifo_state();
-}
-#endif
-
-void ia_css_debug_dump_dma_state(void)
-{
-	/* note: the var below is made static as it is quite large;
-	   if it is not static it ends up on the stack which could
-	   cause issues for drivers
-	*/
-	static dma_state_t state;
-	int i, ch_id;
-
-	const char *fsm_cmd_st_lbl = "FSM Command flag state";
-	const char *fsm_ctl_st_lbl = "FSM Control flag state";
-	const char *fsm_ctl_state = NULL;
-	const char *fsm_ctl_flag = NULL;
-	const char *fsm_pack_st = NULL;
-	const char *fsm_read_st = NULL;
-	const char *fsm_write_st = NULL;
-	char last_cmd_str[64];
-
-	dma_get_state(DMA0_ID, &state);
-	/* Print header for DMA dump status */
-	ia_css_debug_dtrace(2, "DMA dump status:\n");
-
-	/* Print FSM command flag state */
-	if (state.fsm_command_idle)
-		ia_css_debug_dtrace(2, "\t%-32s: %s\n", fsm_cmd_st_lbl, "IDLE");
-	if (state.fsm_command_run)
-		ia_css_debug_dtrace(2, "\t%-32s: %s\n", fsm_cmd_st_lbl, "RUN");
-	if (state.fsm_command_stalling)
-		ia_css_debug_dtrace(2, "\t%-32s: %s\n", fsm_cmd_st_lbl,
-				    "STALL");
-	if (state.fsm_command_error)
-		ia_css_debug_dtrace(2, "\t%-32s: %s\n", fsm_cmd_st_lbl,
-				    "ERROR");
-
-	/* Print last command along with the channel */
-	ch_id = state.last_command_channel;
-
-	switch (state.last_command) {
-	case DMA_COMMAND_READ:
-		snprintf(last_cmd_str, 64,
-			 "Read 2D Block [Channel: %d]", ch_id);
-		break;
-	case DMA_COMMAND_WRITE:
-		snprintf(last_cmd_str, 64,
-			 "Write 2D Block [Channel: %d]", ch_id);
-		break;
-	case DMA_COMMAND_SET_CHANNEL:
-		snprintf(last_cmd_str, 64, "Set Channel [Channel: %d]", ch_id);
-		break;
-	case DMA_COMMAND_SET_PARAM:
-		snprintf(last_cmd_str, 64,
-			 "Set Param: %d [Channel: %d]",
-			 state.last_command_param, ch_id);
-		break;
-	case DMA_COMMAND_READ_SPECIFIC:
-		snprintf(last_cmd_str, 64,
-			 "Read Specific 2D Block [Channel: %d]", ch_id);
-		break;
-	case DMA_COMMAND_WRITE_SPECIFIC:
-		snprintf(last_cmd_str, 64,
-			 "Write Specific 2D Block [Channel: %d]", ch_id);
-		break;
-	case DMA_COMMAND_INIT:
-		snprintf(last_cmd_str, 64,
-			 "Init 2D Block on Device A [Channel: %d]", ch_id);
-		break;
-	case DMA_COMMAND_INIT_SPECIFIC:
-		snprintf(last_cmd_str, 64,
-			 "Init Specific 2D Block [Channel: %d]", ch_id);
-		break;
-	case DMA_COMMAND_RST:
-		snprintf(last_cmd_str, 64, "DMA SW Reset");
-		break;
-	case N_DMA_COMMANDS:
-		snprintf(last_cmd_str, 64, "UNKNOWN");
-		break;
-	default:
-		snprintf(last_cmd_str, 64,
-		  "unknown [Channel: %d]", ch_id);
-		break;
-	}
-	ia_css_debug_dtrace(2, "\t%-32s: (0x%X : %s)\n",
-			    "last command received", state.last_command,
-			    last_cmd_str);
-
-	/* Print DMA registers */
-	ia_css_debug_dtrace(2, "\t%-32s\n",
-			    "DMA registers, connection group 0");
-	ia_css_debug_dtrace(2, "\t\t%-32s: 0x%X\n", "Cmd Fifo Command",
-			    state.current_command);
-	ia_css_debug_dtrace(2, "\t\t%-32s: 0x%X\n", "Cmd Fifo Address A",
-			    state.current_addr_a);
-	ia_css_debug_dtrace(2, "\t\t%-32s: 0x%X\n", "Cmd Fifo Address B",
-			    state.current_addr_b);
-
-	if (state.fsm_ctrl_idle)
-		fsm_ctl_flag = "IDLE";
-	else if (state.fsm_ctrl_run)
-		fsm_ctl_flag = "RUN";
-	else if (state.fsm_ctrl_stalling)
-		fsm_ctl_flag = "STAL";
-	else if (state.fsm_ctrl_error)
-		fsm_ctl_flag = "ERROR";
-	else
-		fsm_ctl_flag = "UNKNOWN";
-
-	switch (state.fsm_ctrl_state) {
-	case DMA_CTRL_STATE_IDLE:
-		fsm_ctl_state = "Idle state";
-		break;
-	case DMA_CTRL_STATE_REQ_RCV:
-		fsm_ctl_state = "Req Rcv state";
-		break;
-	case DMA_CTRL_STATE_RCV:
-		fsm_ctl_state = "Rcv state";
-		break;
-	case DMA_CTRL_STATE_RCV_REQ:
-		fsm_ctl_state = "Rcv Req state";
-		break;
-	case DMA_CTRL_STATE_INIT:
-		fsm_ctl_state = "Init state";
-		break;
-	case N_DMA_CTRL_STATES:
-		fsm_ctl_state = "Unknown";
-		break;
-	}
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %s -> %s\n", fsm_ctl_st_lbl,
-			    fsm_ctl_flag, fsm_ctl_state);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Ctrl source dev",
-			    state.fsm_ctrl_source_dev);
-	ia_css_debug_dtrace(2, "\t\t%-32s: 0x%X\n", "FSM Ctrl source addr",
-			    state.fsm_ctrl_source_addr);
-	ia_css_debug_dtrace(2, "\t\t%-32s: 0x%X\n", "FSM Ctrl source stride",
-			    state.fsm_ctrl_source_stride);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Ctrl source width",
-			    state.fsm_ctrl_source_width);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Ctrl source height",
-			    state.fsm_ctrl_source_height);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Ctrl pack source dev",
-			    state.fsm_ctrl_pack_source_dev);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Ctrl pack dest dev",
-			    state.fsm_ctrl_pack_dest_dev);
-	ia_css_debug_dtrace(2, "\t\t%-32s: 0x%X\n", "FSM Ctrl dest addr",
-			    state.fsm_ctrl_dest_addr);
-	ia_css_debug_dtrace(2, "\t\t%-32s: 0x%X\n", "FSM Ctrl dest stride",
-			    state.fsm_ctrl_dest_stride);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Ctrl pack source width",
-			    state.fsm_ctrl_pack_source_width);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Ctrl pack dest height",
-			    state.fsm_ctrl_pack_dest_height);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Ctrl pack dest width",
-			    state.fsm_ctrl_pack_dest_width);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Ctrl pack source elems",
-			    state.fsm_ctrl_pack_source_elems);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Ctrl pack dest elems",
-			    state.fsm_ctrl_pack_dest_elems);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Ctrl pack extension",
-			    state.fsm_ctrl_pack_extension);
-
-	if (state.pack_idle)
-		fsm_pack_st = "IDLE";
-	if (state.pack_run)
-		fsm_pack_st = "RUN";
-	if (state.pack_stalling)
-		fsm_pack_st = "STALL";
-	if (state.pack_error)
-		fsm_pack_st = "ERROR";
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %s\n", "FSM Pack flag state",
-			    fsm_pack_st);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Pack cnt height",
-			    state.pack_cnt_height);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Pack src cnt width",
-			    state.pack_src_cnt_width);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Pack dest cnt width",
-			    state.pack_dest_cnt_width);
-
-	if (state.read_state == DMA_RW_STATE_IDLE)
-		fsm_read_st = "Idle state";
-	if (state.read_state == DMA_RW_STATE_REQ)
-		fsm_read_st = "Req state";
-	if (state.read_state == DMA_RW_STATE_NEXT_LINE)
-		fsm_read_st = "Next line";
-	if (state.read_state == DMA_RW_STATE_UNLOCK_CHANNEL)
-		fsm_read_st = "Unlock channel";
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %s\n", "FSM Read state",
-			    fsm_read_st);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Read cnt height",
-			    state.read_cnt_height);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Read cnt width",
-			    state.read_cnt_width);
-
-	if (state.write_state == DMA_RW_STATE_IDLE)
-		fsm_write_st = "Idle state";
-	if (state.write_state == DMA_RW_STATE_REQ)
-		fsm_write_st = "Req state";
-	if (state.write_state == DMA_RW_STATE_NEXT_LINE)
-		fsm_write_st = "Next line";
-	if (state.write_state == DMA_RW_STATE_UNLOCK_CHANNEL)
-		fsm_write_st = "Unlock channel";
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %s\n", "FSM Write state",
-			    fsm_write_st);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Write height",
-			    state.write_height);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "FSM Write width",
-			    state.write_width);
-
-	for (i = 0; i < HIVE_ISP_NUM_DMA_CONNS; i++) {
-		dma_port_state_t *port = &(state.port_states[i]);
-		ia_css_debug_dtrace(2, "\tDMA device interface %d\n", i);
-		ia_css_debug_dtrace(2, "\t\tDMA internal side state\n");
-		ia_css_debug_dtrace(2,
-				    "\t\t\tCS:%d - We_n:%d - Run:%d - Ack:%d\n",
-				    port->req_cs, port->req_we_n, port->req_run,
-				    port->req_ack);
-		ia_css_debug_dtrace(2, "\t\tMaster Output side state\n");
-		ia_css_debug_dtrace(2,
-				    "\t\t\tCS:%d - We_n:%d - Run:%d - Ack:%d\n",
-				    port->send_cs, port->send_we_n,
-				    port->send_run, port->send_ack);
-		ia_css_debug_dtrace(2, "\t\tFifo state\n");
-		if (port->fifo_state == DMA_FIFO_STATE_WILL_BE_FULL)
-			ia_css_debug_dtrace(2, "\t\t\tFiFo will be full\n");
-		else if (port->fifo_state == DMA_FIFO_STATE_FULL)
-			ia_css_debug_dtrace(2, "\t\t\tFifo Full\n");
-		else if (port->fifo_state == DMA_FIFO_STATE_EMPTY)
-			ia_css_debug_dtrace(2, "\t\t\tFifo Empty\n");
-		else
-			ia_css_debug_dtrace(2, "\t\t\tFifo state unknown\n");
-
-		ia_css_debug_dtrace(2, "\t\tFifo counter %d\n\n",
-				    port->fifo_counter);
-	}
-
-	for (i = 0; i < HIVE_DMA_NUM_CHANNELS; i++) {
-		dma_channel_state_t *ch = &(state.channel_states[i]);
-		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "DMA channel register",
-				    i);
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Connection",
-				    ch->connection);
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Sign extend",
-				    ch->sign_extend);
-		ia_css_debug_dtrace(2, "\t\t%-32s: 0x%X\n", "Stride Dev A",
-				    ch->stride_a);
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Elems Dev A",
-				    ch->elems_a);
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Cropping Dev A",
-				    ch->cropping_a);
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Width Dev A",
-				    ch->width_a);
-		ia_css_debug_dtrace(2, "\t\t%-32s: 0x%X\n", "Stride Dev B",
-				    ch->stride_b);
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Elems Dev B",
-				    ch->elems_b);
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Cropping Dev B",
-				    ch->cropping_b);
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Width Dev B",
-				    ch->width_b);
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Height", ch->height);
-	}
-	ia_css_debug_dtrace(2, "\n");
-	return;
-}
-
-void ia_css_debug_dump_dma_sp_fifo_state(void)
-{
-	fifo_channel_state_t dma_to_sp, sp_to_dma;
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_DMA0_TO_SP0, &dma_to_sp);
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_SP0_TO_DMA0, &sp_to_dma);
-	debug_print_fifo_channel_state(&dma_to_sp, "DMA to SP");
-	debug_print_fifo_channel_state(&sp_to_dma, "SP to DMA");
-	return;
-}
-
-void ia_css_debug_dump_dma_isp_fifo_state(void)
-{
-	fifo_channel_state_t dma_to_isp, isp_to_dma;
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_DMA0_TO_ISP0, &dma_to_isp);
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_ISP0_TO_DMA0, &isp_to_dma);
-	debug_print_fifo_channel_state(&dma_to_isp, "DMA to ISP");
-	debug_print_fifo_channel_state(&isp_to_dma, "ISP to DMA");
-	return;
-}
-
-void ia_css_debug_dump_isp_sp_fifo_state(void)
-{
-	fifo_channel_state_t sp_to_isp, isp_to_sp;
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_SP0_TO_ISP0, &sp_to_isp);
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_ISP0_TO_SP0, &isp_to_sp);
-	debug_print_fifo_channel_state(&sp_to_isp, "SP to ISP");
-	debug_print_fifo_channel_state(&isp_to_sp, "ISP to SP");
-	return;
-}
-
-void ia_css_debug_dump_isp_gdc_fifo_state(void)
-{
-	fifo_channel_state_t gdc_to_isp, isp_to_gdc;
-
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_GDC0_TO_ISP0, &gdc_to_isp);
-	fifo_channel_get_state(FIFO_MONITOR0_ID,
-			       FIFO_CHANNEL_ISP0_TO_GDC0, &isp_to_gdc);
-	debug_print_fifo_channel_state(&gdc_to_isp, "GDC to ISP");
-	debug_print_fifo_channel_state(&isp_to_gdc, "ISP to GDC");
-	return;
-}
-
-void ia_css_debug_dump_all_fifo_state(void)
-{
-	int i;
-	fifo_monitor_state_t state;
-	fifo_monitor_get_state(FIFO_MONITOR0_ID, &state);
-
-	for (i = 0; i < N_FIFO_CHANNEL; i++)
-		debug_print_fifo_channel_state(&(state.fifo_channels[i]),
-					       "squepfstqkt");
-	return;
-}
-
-static void debug_binary_info_print(const struct ia_css_binary_xinfo *info)
-{
-	assert(info != NULL);
-	ia_css_debug_dtrace(2, "id = %d\n", info->sp.id);
-	ia_css_debug_dtrace(2, "mode = %d\n", info->sp.pipeline.mode);
-	ia_css_debug_dtrace(2, "max_input_width = %d\n", info->sp.input.max_width);
-	ia_css_debug_dtrace(2, "min_output_width = %d\n",
-			    info->sp.output.min_width);
-	ia_css_debug_dtrace(2, "max_output_width = %d\n",
-			    info->sp.output.max_width);
-	ia_css_debug_dtrace(2, "top_cropping = %d\n", info->sp.pipeline.top_cropping);
-	ia_css_debug_dtrace(2, "left_cropping = %d\n", info->sp.pipeline.left_cropping);
-	ia_css_debug_dtrace(2, "xmem_addr = %d\n", info->xmem_addr);
-	ia_css_debug_dtrace(2, "enable_vf_veceven = %d\n",
-			    info->sp.enable.vf_veceven);
-	ia_css_debug_dtrace(2, "enable_dis = %d\n", info->sp.enable.dis);
-	ia_css_debug_dtrace(2, "enable_uds = %d\n", info->sp.enable.uds);
-	ia_css_debug_dtrace(2, "enable ds = %d\n", info->sp.enable.ds);
-	ia_css_debug_dtrace(2, "s3atbl_use_dmem = %d\n", info->sp.s3a.s3atbl_use_dmem);
-	return;
-}
-
-void ia_css_debug_binary_print(const struct ia_css_binary *bi)
-{
-	unsigned int i;
-	debug_binary_info_print(bi->info);
-	ia_css_debug_dtrace(2,
-			    "input:  %dx%d, format = %d, padded width = %d\n",
-			    bi->in_frame_info.res.width,
-			    bi->in_frame_info.res.height,
-			    bi->in_frame_info.format,
-			    bi->in_frame_info.padded_width);
-	ia_css_debug_dtrace(2,
-			    "internal :%dx%d, format = %d, padded width = %d\n",
-			    bi->internal_frame_info.res.width,
-			    bi->internal_frame_info.res.height,
-			    bi->internal_frame_info.format,
-			    bi->internal_frame_info.padded_width);
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		if (bi->out_frame_info[i].res.width != 0) {
-			ia_css_debug_dtrace(2,
-				    "out%d:    %dx%d, format = %d, padded width = %d\n",
-					i,
-				    bi->out_frame_info[i].res.width,
-				    bi->out_frame_info[i].res.height,
-				    bi->out_frame_info[i].format,
-				    bi->out_frame_info[i].padded_width);
-		}
-	}
-	ia_css_debug_dtrace(2,
-			    "vf out: %dx%d, format = %d, padded width = %d\n",
-			    bi->vf_frame_info.res.width,
-			    bi->vf_frame_info.res.height,
-			    bi->vf_frame_info.format,
-			    bi->vf_frame_info.padded_width);
-	ia_css_debug_dtrace(2, "online = %d\n", bi->online);
-	ia_css_debug_dtrace(2, "input_buf_vectors = %d\n",
-			    bi->input_buf_vectors);
-	ia_css_debug_dtrace(2, "deci_factor_log2 = %d\n", bi->deci_factor_log2);
-	ia_css_debug_dtrace(2, "vf_downscale_log2 = %d\n",
-			    bi->vf_downscale_log2);
-	ia_css_debug_dtrace(2, "dis_deci_factor_log2 = %d\n",
-			    bi->dis.deci_factor_log2);
-	ia_css_debug_dtrace(2, "dis hor coef num = %d\n",
-			    bi->dis.coef.pad.width);
-	ia_css_debug_dtrace(2, "dis ver coef num = %d\n",
-			    bi->dis.coef.pad.height);
-	ia_css_debug_dtrace(2, "dis hor proj num = %d\n",
-			    bi->dis.proj.pad.height);
-	ia_css_debug_dtrace(2, "sctbl_width_per_color = %d\n",
-			    bi->sctbl_width_per_color);
-	ia_css_debug_dtrace(2, "s3atbl_width = %d\n", bi->s3atbl_width);
-	ia_css_debug_dtrace(2, "s3atbl_height = %d\n", bi->s3atbl_height);
-	return;
-}
-
-void ia_css_debug_frame_print(const struct ia_css_frame *frame,
-			      const char *descr)
-{
-	char *data = NULL;
-
-	assert(frame != NULL);
-	assert(descr != NULL);
-
-	data = (char *)HOST_ADDRESS(frame->data);
-	ia_css_debug_dtrace(2, "frame %s (%p):\n", descr, frame);
-	ia_css_debug_dtrace(2, "  resolution    = %dx%d\n",
-			    frame->info.res.width, frame->info.res.height);
-	ia_css_debug_dtrace(2, "  padded width  = %d\n",
-			    frame->info.padded_width);
-	ia_css_debug_dtrace(2, "  format        = %d\n", frame->info.format);
-	ia_css_debug_dtrace(2, "  is contiguous = %s\n",
-			    frame->contiguous ? "yes" : "no");
-	switch (frame->info.format) {
-	case IA_CSS_FRAME_FORMAT_NV12:
-	case IA_CSS_FRAME_FORMAT_NV16:
-	case IA_CSS_FRAME_FORMAT_NV21:
-	case IA_CSS_FRAME_FORMAT_NV61:
-		ia_css_debug_dtrace(2, "  Y = %p\n",
-				    data + frame->planes.nv.y.offset);
-		ia_css_debug_dtrace(2, "  UV = %p\n",
-				    data + frame->planes.nv.uv.offset);
-		break;
-	case IA_CSS_FRAME_FORMAT_YUYV:
-	case IA_CSS_FRAME_FORMAT_UYVY:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_8:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_LEGACY_YUV420_8:
-	case IA_CSS_FRAME_FORMAT_YUV_LINE:
-		ia_css_debug_dtrace(2, "  YUYV = %p\n",
-				    data + frame->planes.yuyv.offset);
-		break;
-	case IA_CSS_FRAME_FORMAT_YUV420:
-	case IA_CSS_FRAME_FORMAT_YUV422:
-	case IA_CSS_FRAME_FORMAT_YUV444:
-	case IA_CSS_FRAME_FORMAT_YV12:
-	case IA_CSS_FRAME_FORMAT_YV16:
-	case IA_CSS_FRAME_FORMAT_YUV420_16:
-	case IA_CSS_FRAME_FORMAT_YUV422_16:
-		ia_css_debug_dtrace(2, "  Y = %p\n",
-				    data + frame->planes.yuv.y.offset);
-		ia_css_debug_dtrace(2, "  U = %p\n",
-				    data + frame->planes.yuv.u.offset);
-		ia_css_debug_dtrace(2, "  V = %p\n",
-				    data + frame->planes.yuv.v.offset);
-		break;
-	case IA_CSS_FRAME_FORMAT_RAW_PACKED:
-		ia_css_debug_dtrace(2, "  RAW PACKED = %p\n",
-				    data + frame->planes.raw.offset);
-		break;
-	case IA_CSS_FRAME_FORMAT_RAW:
-		ia_css_debug_dtrace(2, "  RAW = %p\n",
-				    data + frame->planes.raw.offset);
-		break;
-	case IA_CSS_FRAME_FORMAT_RGBA888:
-	case IA_CSS_FRAME_FORMAT_RGB565:
-		ia_css_debug_dtrace(2, "  RGB = %p\n",
-				    data + frame->planes.rgb.offset);
-		break;
-	case IA_CSS_FRAME_FORMAT_QPLANE6:
-		ia_css_debug_dtrace(2, "  R    = %p\n",
-				    data + frame->planes.plane6.r.offset);
-		ia_css_debug_dtrace(2, "  RatB = %p\n",
-				    data + frame->planes.plane6.r_at_b.offset);
-		ia_css_debug_dtrace(2, "  Gr   = %p\n",
-				    data + frame->planes.plane6.gr.offset);
-		ia_css_debug_dtrace(2, "  Gb   = %p\n",
-				    data + frame->planes.plane6.gb.offset);
-		ia_css_debug_dtrace(2, "  B    = %p\n",
-				    data + frame->planes.plane6.b.offset);
-		ia_css_debug_dtrace(2, "  BatR = %p\n",
-				    data + frame->planes.plane6.b_at_r.offset);
-		break;
-	case IA_CSS_FRAME_FORMAT_BINARY_8:
-		ia_css_debug_dtrace(2, "  Binary data = %p\n",
-				    data + frame->planes.binary.data.offset);
-		break;
-	default:
-		ia_css_debug_dtrace(2, "  unknown frame type\n");
-		break;
-	}
-	return;
-}
-
-#if SP_DEBUG != SP_DEBUG_NONE
-
-void ia_css_debug_print_sp_debug_state(const struct sh_css_sp_debug_state
-				       *state)
-{
-
-#endif
-
-#if SP_DEBUG == SP_DEBUG_DUMP
-
-	assert(state != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "current SP software counter: %d\n",
-			    state->debug[0]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty output buffer queue head: 0x%x\n",
-			    state->debug[1]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty output buffer queue tail: 0x%x\n",
-			    state->debug[2]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty s3a buffer queue head: 0x%x\n",
-			    state->debug[3]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty s3a buffer queue tail: 0x%x\n",
-			    state->debug[4]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "full output buffer queue head: 0x%x\n",
-			    state->debug[5]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "full output buffer queue tail: 0x%x\n",
-			    state->debug[6]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "full s3a buffer queue head: 0x%x\n",
-			    state->debug[7]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "full s3a buffer queue tail: 0x%x\n",
-			    state->debug[8]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "event queue head: 0x%x\n",
-			    state->debug[9]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "event queue tail: 0x%x\n",
-			    state->debug[10]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "num of stages of current pipeline: 0x%x\n",
-			    state->debug[11]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "DDR address of stage 1: 0x%x\n",
-			    state->debug[12]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "DDR address of stage 2: 0x%x\n",
-			    state->debug[13]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "current stage out_vf buffer idx: 0x%x\n",
-			    state->debug[14]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "current stage output buffer idx: 0x%x\n",
-			    state->debug[15]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "current stage s3a buffer idx: 0x%x\n",
-			    state->debug[16]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "first char of current stage name: 0x%x\n",
-			    state->debug[17]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "current SP thread id: 0x%x\n",
-			    state->debug[18]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty output buffer address 1: 0x%x\n",
-			    state->debug[19]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty output buffer address 2: 0x%x\n",
-			    state->debug[20]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty out_vf buffer address 1: 0x%x\n",
-			    state->debug[21]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty out_vf buffer address 2: 0x%x\n",
-			    state->debug[22]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty s3a_hi buffer address 1: 0x%x\n",
-			    state->debug[23]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty s3a_hi buffer address 2: 0x%x\n",
-			    state->debug[24]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty s3a_lo buffer address 1: 0x%x\n",
-			    state->debug[25]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty s3a_lo buffer address 2: 0x%x\n",
-			    state->debug[26]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty dis_hor buffer address 1: 0x%x\n",
-			    state->debug[27]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty dis_hor buffer address 2: 0x%x\n",
-			    state->debug[28]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty dis_ver buffer address 1: 0x%x\n",
-			    state->debug[29]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty dis_ver buffer address 2: 0x%x\n",
-			    state->debug[30]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "empty param buffer address: 0x%x\n",
-			    state->debug[31]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "first incorrect frame address: 0x%x\n",
-			    state->debug[32]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "first incorrect frame container address: 0x%x\n",
-			    state->debug[33]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "first incorrect frame container payload: 0x%x\n",
-			    state->debug[34]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "first incorrect s3a_hi address: 0x%x\n",
-			    state->debug[35]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "first incorrect s3a_hi container address: 0x%x\n",
-			    state->debug[36]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "first incorrect s3a_hi container payload: 0x%x\n",
-			    state->debug[37]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "first incorrect s3a_lo address: 0x%x\n",
-			    state->debug[38]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "first incorrect s3a_lo container address: 0x%x\n",
-			    state->debug[39]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "first incorrect s3a_lo container payload: 0x%x\n",
-			    state->debug[40]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "number of calling flash start function: 0x%x\n",
-			    state->debug[41]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "number of calling flash close function: 0x%x\n",
-			    state->debug[42]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "number of flashed frame: 0x%x\n",
-			    state->debug[43]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "flash in use flag: 0x%x\n",
-			    state->debug[44]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "number of update frame flashed flag: 0x%x\n",
-			    state->debug[46]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			    "number of active threads: 0x%x\n",
-			    state->debug[45]);
-
-#elif SP_DEBUG == SP_DEBUG_COPY
-
-	/* Remember last_index because we only want to print new entries */
-	static int last_index;
-	int sp_index = state->index;
-	int n;
-
-	assert(state != NULL);
-	if (sp_index < last_index) {
-		/* SP has been reset */
-		last_index = 0;
-	}
-
-	if (last_index == 0) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-				    "copy-trace init: sp_dbg_if_start_line=%d, "
-				    "sp_dbg_if_start_column=%d, "
-				    "sp_dbg_if_cropped_height=%d, "
-				    "sp_debg_if_cropped_width=%d\n",
-				    state->if_start_line,
-				    state->if_start_column,
-				    state->if_cropped_height,
-				    state->if_cropped_width);
-	}
-
-	if ((last_index + SH_CSS_SP_DBG_TRACE_DEPTH) < sp_index) {
-		/* last index can be multiple rounds behind */
-		/* while trace size is only SH_CSS_SP_DBG_TRACE_DEPTH */
-		last_index = sp_index - SH_CSS_SP_DBG_TRACE_DEPTH;
-	}
-
-	for (n = last_index; n < sp_index; n++) {
-		int i = n % SH_CSS_SP_DBG_TRACE_DEPTH;
-		if (state->trace[i].frame != 0) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-					    "copy-trace: frame=%d, line=%d, "
-					    "pixel_distance=%d, "
-					    "mipi_used_dword=%d, "
-					    "sp_index=%d\n",
-					    state->trace[i].frame,
-					    state->trace[i].line,
-					    state->trace[i].pixel_distance,
-					    state->trace[i].mipi_used_dword,
-					    state->trace[i].sp_index);
-		}
-	}
-
-	last_index = sp_index;
-
-#elif SP_DEBUG == SP_DEBUG_TRACE
-
-/*
- * This is just an example how TRACE_FILE_ID (see ia_css_debug.sp.h) will
- * me mapped on the file name string.
- *
- * Adjust this to your trace case!
- */
-	static char const * const id2filename[8] = {
-		"param_buffer.sp.c | tagger.sp.c | pipe_data.sp.c",
-		"isp_init.sp.c",
-		"sp_raw_copy.hive.c",
-		"dma_configure.sp.c",
-		"sp.hive.c",
-		"event_proxy_sp.hive.c",
-		"circular_buffer.sp.c",
-		"frame_buffer.sp.c"
-	};
-
-#if 1
-	/* Example SH_CSS_SP_DBG_NR_OF_TRACES==1 */
-	/* Adjust this to your trace case */
-	static char const *trace_name[SH_CSS_SP_DBG_NR_OF_TRACES] = {
-		"default"
-	};
-#else
-	/* Example SH_CSS_SP_DBG_NR_OF_TRACES==4 */
-	/* Adjust this to your trace case */
-	static char const *trace_name[SH_CSS_SP_DBG_NR_OF_TRACES] = {
-		"copy", "preview/video", "capture", "acceleration"
-	};
-#endif
-
-	/* Remember host_index_last because we only want to print new entries */
-	static int host_index_last[SH_CSS_SP_DBG_NR_OF_TRACES] = { 0 };
-	int t, n;
-
-	assert(state != NULL);
-
-	for (t = 0; t < SH_CSS_SP_DBG_NR_OF_TRACES; t++) {
-		int sp_index_last = state->index_last[t];
-
-		if (sp_index_last < host_index_last[t]) {
-			/* SP has been reset */
-			host_index_last[t] = 0;
-		}
-
-		if ((host_index_last[t] + SH_CSS_SP_DBG_TRACE_DEPTH) <
-		    sp_index_last) {
-			/* last index can be multiple rounds behind */
-			/* while trace size is only SH_CSS_SP_DBG_TRACE_DEPTH */
-			ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-					    "Warning: trace %s has gap of %d "
-					    "traces\n",
-					    trace_name[t],
-					    (sp_index_last -
-					     (host_index_last[t] +
-					      SH_CSS_SP_DBG_TRACE_DEPTH)));
-
-			host_index_last[t] =
-			    sp_index_last - SH_CSS_SP_DBG_TRACE_DEPTH;
-		}
-
-		for (n = host_index_last[t]; n < sp_index_last; n++) {
-			int i = n % SH_CSS_SP_DBG_TRACE_DEPTH;
-			int l = state->trace[t][i].location &
-			    ((1 << SH_CSS_SP_DBG_TRACE_FILE_ID_BIT_POS) - 1);
-			int fid = state->trace[t][i].location >>
-			    SH_CSS_SP_DBG_TRACE_FILE_ID_BIT_POS;
-			int ts = state->trace[t][i].time_stamp;
-
-			if (ts) {
-				ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-						    "%05d trace=%s, file=%s:%d, "
-						    "data=0x%08x\n",
-						    ts,
-						    trace_name[t],
-						    id2filename[fid], l,
-						    state->trace[t][i].data);
-			}
-		}
-		host_index_last[t] = sp_index_last;
-	}
-
-#elif SP_DEBUG == SP_DEBUG_MINIMAL
-	int i;
-	int base = 0;
-	int limit = SH_CSS_NUM_SP_DEBUG;
-	int step = 1;
-
-	assert(state != NULL);
-
-	for (i = base; i < limit; i += step) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-				    "sp_dbg_trace[%d] = %d\n",
-				    i, state->debug[i]);
-	}
-#endif
-
-#if SP_DEBUG != SP_DEBUG_NONE
-
-	return;
-}
-#endif
-
-#if defined(HAS_INPUT_FORMATTER_VERSION_2) && !defined(HAS_NO_INPUT_FORMATTER)
-static void debug_print_rx_mipi_port_state(mipi_port_state_t *state)
-{
-	int i;
-	unsigned int bits, infos;
-
-	assert(state != NULL);
-
-	bits = state->irq_status;
-	infos = ia_css_isys_rx_translate_irq_infos(bits);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: (irq reg = 0x%X)\n",
-			    "receiver errors", bits);
-
-	if (infos & IA_CSS_RX_IRQ_INFO_BUFFER_OVERRUN)
-		ia_css_debug_dtrace(2, "\t\t\tbuffer overrun\n");
-	if (infos & IA_CSS_RX_IRQ_INFO_ERR_SOT)
-		ia_css_debug_dtrace(2, "\t\t\tstart-of-transmission error\n");
-	if (infos & IA_CSS_RX_IRQ_INFO_ERR_SOT_SYNC)
-		ia_css_debug_dtrace(2, "\t\t\tstart-of-transmission sync error\n");
-	if (infos & IA_CSS_RX_IRQ_INFO_ERR_CONTROL)
-		ia_css_debug_dtrace(2, "\t\t\tcontrol error\n");
-	if (infos & IA_CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE)
-		ia_css_debug_dtrace(2, "\t\t\t2 or more ECC errors\n");
-	if (infos & IA_CSS_RX_IRQ_INFO_ERR_CRC)
-		ia_css_debug_dtrace(2, "\t\t\tCRC mismatch\n");
-	if (infos & IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID)
-		ia_css_debug_dtrace(2, "\t\t\tunknown error\n");
-	if (infos & IA_CSS_RX_IRQ_INFO_ERR_FRAME_SYNC)
-		ia_css_debug_dtrace(2, "\t\t\tframe sync error\n");
-	if (infos & IA_CSS_RX_IRQ_INFO_ERR_FRAME_DATA)
-		ia_css_debug_dtrace(2, "\t\t\tframe data error\n");
-	if (infos & IA_CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT)
-		ia_css_debug_dtrace(2, "\t\t\tdata timeout\n");
-	if (infos & IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC)
-		ia_css_debug_dtrace(2, "\t\t\tunknown escape command entry\n");
-	if (infos & IA_CSS_RX_IRQ_INFO_ERR_LINE_SYNC)
-		ia_css_debug_dtrace(2, "\t\t\tline sync error\n");
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "device_ready", state->device_ready);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "irq_status", state->irq_status);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "irq_enable", state->irq_enable);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "timeout_count", state->timeout_count);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "init_count", state->init_count);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "raw16_18", state->raw16_18);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "sync_count", state->sync_count);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "rx_count", state->rx_count);
-
-	for (i = 0; i < MIPI_4LANE_CFG; i++) {
-		ia_css_debug_dtrace(2, "\t\t%-32s%d%-32s: %d\n",
-				    "lane_sync_count[", i, "]",
-				    state->lane_sync_count[i]);
-	}
-
-	for (i = 0; i < MIPI_4LANE_CFG; i++) {
-		ia_css_debug_dtrace(2, "\t\t%-32s%d%-32s: %d\n",
-				    "lane_rx_count[", i, "]",
-				    state->lane_rx_count[i]);
-	}
-
-	return;
-}
-
-static void debug_print_rx_channel_state(rx_channel_state_t *state)
-{
-	int i;
-
-	assert(state != NULL);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "compression_scheme0", state->comp_scheme0);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "compression_scheme1", state->comp_scheme1);
-
-	for (i = 0; i < N_MIPI_FORMAT_CUSTOM; i++) {
-		ia_css_debug_dtrace(2, "\t\t%-32s%d: %d\n",
-				    "MIPI Predictor ", i, state->pred[i]);
-	}
-
-	for (i = 0; i < N_MIPI_FORMAT_CUSTOM; i++) {
-		ia_css_debug_dtrace(2, "\t\t%-32s%d: %d\n",
-				    "MIPI Compressor ", i, state->comp[i]);
-	}
-
-	return;
-}
-
-static void debug_print_rx_state(receiver_state_t *state)
-{
-	int i;
-
-	assert(state != NULL);
-	ia_css_debug_dtrace(2, "CSI Receiver State:\n");
-
-	ia_css_debug_dtrace(2, "\tConfiguration:\n");
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "fs_to_ls_delay", state->fs_to_ls_delay);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "ls_to_data_delay", state->ls_to_data_delay);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "data_to_le_delay", state->data_to_le_delay);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "le_to_fe_delay", state->le_to_fe_delay);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "fe_to_fs_delay", state->fe_to_fs_delay);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "le_to_fs_delay", state->le_to_fs_delay);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "is_two_ppc", state->is_two_ppc);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "backend_rst", state->backend_rst);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "raw18", state->raw18);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "force_raw8", state->force_raw8);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "raw16", state->raw16);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "be_gsp_acc_ovl", state->be_gsp_acc_ovl);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "be_srst", state->be_srst);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "be_is_two_ppc", state->be_is_two_ppc);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "be_comp_format0", state->be_comp_format0);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "be_comp_format1", state->be_comp_format1);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "be_comp_format2", state->be_comp_format2);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "be_comp_format3", state->be_comp_format3);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "be_sel", state->be_sel);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "be_raw16_config", state->be_raw16_config);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "be_raw18_config", state->be_raw18_config);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "be_force_raw8", state->be_force_raw8);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "be_irq_status", state->be_irq_status);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "be_irq_clear", state->be_irq_clear);
-
-	/* mipi port state */
-	for (i = 0; i < N_MIPI_PORT_ID; i++) {
-		ia_css_debug_dtrace(2, "\tMIPI Port %d State:\n", i);
-
-		debug_print_rx_mipi_port_state(&state->mipi_port_state[i]);
-	}
-	/* end of mipi port state */
-
-	/* rx channel state */
-	for (i = 0; i < N_RX_CHANNEL_ID; i++) {
-		ia_css_debug_dtrace(2, "\tRX Channel %d State:\n", i);
-
-		debug_print_rx_channel_state(&state->rx_channel_state[i]);
-	}
-	/* end of rx channel state */
-
-	return;
-}
-#endif
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-void ia_css_debug_dump_rx_state(void)
-{
-#if defined(HAS_INPUT_FORMATTER_VERSION_2) && !defined(HAS_NO_INPUT_FORMATTER)
-	receiver_state_t state;
-
-	receiver_get_state(RX0_ID, &state);
-	debug_print_rx_state(&state);
-#endif
-}
-#endif
-
-void ia_css_debug_dump_sp_sw_debug_info(void)
-{
-#if SP_DEBUG != SP_DEBUG_NONE
-	struct sh_css_sp_debug_state state;
-
-	sh_css_sp_get_debug_state(&state);
-	ia_css_debug_print_sp_debug_state(&state);
-#endif
-	ia_css_bufq_dump_queue_info();
-	ia_css_pipeline_dump_thread_map_info();
-	return;
-}
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-static void debug_print_isys_capture_unit_state(capture_unit_state_t *state)
-{
-	assert(state != NULL);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Packet_Length", state->Packet_Length);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Received_Length", state->Received_Length);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Received_Short_Packets",
-			    state->Received_Short_Packets);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Received_Long_Packets",
-			    state->Received_Long_Packets);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Last_Command", state->Last_Command);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Next_Command", state->Next_Command);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Last_Acknowledge", state->Last_Acknowledge);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Next_Acknowledge", state->Next_Acknowledge);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "FSM_State_Info", state->FSM_State_Info);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "StartMode", state->StartMode);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Start_Addr", state->Start_Addr);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Mem_Region_Size", state->Mem_Region_Size);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Num_Mem_Regions", state->Num_Mem_Regions);
-	return;
-}
-
-static void debug_print_isys_acquisition_unit_state(
-				acquisition_unit_state_t *state)
-{
-	assert(state != NULL);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Received_Short_Packets",
-			    state->Received_Short_Packets);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Received_Long_Packets",
-			    state->Received_Long_Packets);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Last_Command", state->Last_Command);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Next_Command", state->Next_Command);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Last_Acknowledge", state->Last_Acknowledge);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Next_Acknowledge", state->Next_Acknowledge);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "FSM_State_Info", state->FSM_State_Info);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Int_Cntr_Info", state->Int_Cntr_Info);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Start_Addr", state->Start_Addr);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Mem_Region_Size", state->Mem_Region_Size);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "Num_Mem_Regions", state->Num_Mem_Regions);
-}
-
-static void debug_print_isys_ctrl_unit_state(ctrl_unit_state_t *state)
-{
-	assert(state != NULL);
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "last_cmd", state->last_cmd);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "next_cmd", state->next_cmd);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "last_ack", state->last_ack);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "next_ack", state->next_ack);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "top_fsm_state", state->top_fsm_state);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captA_fsm_state", state->captA_fsm_state);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captB_fsm_state", state->captB_fsm_state);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captC_fsm_state", state->captC_fsm_state);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "acq_fsm_state", state->acq_fsm_state);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captA_start_addr", state->captA_start_addr);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captB_start_addr", state->captB_start_addr);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captC_start_addr", state->captC_start_addr);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captA_mem_region_size",
-			    state->captA_mem_region_size);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captB_mem_region_size",
-			    state->captB_mem_region_size);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captC_mem_region_size",
-			    state->captC_mem_region_size);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captA_num_mem_regions",
-			    state->captA_num_mem_regions);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captB_num_mem_regions",
-			    state->captB_num_mem_regions);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "captC_num_mem_regions",
-			    state->captC_num_mem_regions);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "acq_start_addr", state->acq_start_addr);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "acq_mem_region_size", state->acq_mem_region_size);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "acq_num_mem_regions", state->acq_num_mem_regions);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "capt_reserve_one_mem_region",
-			    state->capt_reserve_one_mem_region);
-
-	return;
-}
-
-static void debug_print_isys_state(input_system_state_t *state)
-{
-	int i;
-
-	assert(state != NULL);
-	ia_css_debug_dtrace(2, "InputSystem State:\n");
-
-	/* configuration */
-	ia_css_debug_dtrace(2, "\tConfiguration:\n");
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "str_multiCastA_sel", state->str_multicastA_sel);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "str_multicastB_sel", state->str_multicastB_sel);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "str_multicastC_sel", state->str_multicastC_sel);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "str_mux_sel", state->str_mux_sel);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "str_mon_status", state->str_mon_status);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "str_mon_irq_cond", state->str_mon_irq_cond);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "str_mon_irq_en", state->str_mon_irq_en);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "isys_srst", state->isys_srst);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "isys_slv_reg_srst", state->isys_slv_reg_srst);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "str_deint_portA_cnt", state->str_deint_portA_cnt);
-
-	ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-			    "str_deint_portB_cnd", state->str_deint_portB_cnt);
-	/* end of configuration */
-
-	/* capture unit state */
-	for (i = 0; i < N_CAPTURE_UNIT_ID; i++) {
-		capture_unit_state_t *capture_unit_state;
-
-		ia_css_debug_dtrace(2, "\tCaptureUnit %d State:\n", i);
-
-		capture_unit_state = &state->capture_unit[i];
-		debug_print_isys_capture_unit_state(capture_unit_state);
-	}
-	/* end of capture unit state */
-
-	/* acquisition unit state */
-	for (i = 0; i < N_ACQUISITION_UNIT_ID; i++) {
-		acquisition_unit_state_t *acquisition_unit_state;
-
-		ia_css_debug_dtrace(2, "\tAcquisitionUnit %d State:\n", i);
-
-		acquisition_unit_state = &state->acquisition_unit[i];
-		debug_print_isys_acquisition_unit_state(acquisition_unit_state);
-	}
-	/* end of acquisition unit state */
-
-	/* control unit state */
-	for (i = 0; i < N_CTRL_UNIT_ID; i++) {
-		ia_css_debug_dtrace(2, "\tControlUnit %d State:\n", i);
-
-		debug_print_isys_ctrl_unit_state(&state->ctrl_unit_state[i]);
-	}
-	/* end of control unit state */
-}
-
-void ia_css_debug_dump_isys_state(void)
-{
-	input_system_state_t state;
-
-	input_system_get_state(INPUT_SYSTEM0_ID, &state);
-	debug_print_isys_state(&state);
-
-	return;
-}
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
-void ia_css_debug_dump_isys_state(void)
-{
-	/* Android compilation fails if made a local variable
-	stack size on android is limited to 2k and this structure
-	is around 3.5K, in place of static malloc can be done but
-	if this call is made too often it will lead to fragment memory
-	versus a fixed allocation */
-	static input_system_state_t state;
-
-	input_system_get_state(INPUT_SYSTEM0_ID, &state);
-	input_system_dump_state(INPUT_SYSTEM0_ID, &state);
-}
-#endif
-
-void ia_css_debug_dump_debug_info(const char *context)
-{
-	if (context == NULL)
-		context = "No Context provided";
-
-	ia_css_debug_dtrace(2, "CSS Debug Info dump [Context = %s]\n", context);
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-	ia_css_debug_dump_rx_state();
-#endif
-#if !defined(HAS_NO_INPUT_FORMATTER) && defined(USE_INPUT_SYSTEM_VERSION_2)
-	ia_css_debug_dump_if_state();
-#endif
-	ia_css_debug_dump_isp_state();
-	ia_css_debug_dump_isp_sp_fifo_state();
-	ia_css_debug_dump_isp_gdc_fifo_state();
-	ia_css_debug_dump_sp_state();
-	ia_css_debug_dump_perf_counters();
-
-#ifdef HAS_WATCHDOG_SP_THREAD_DEBUG
-	sh_css_dump_thread_wait_info();
-	sh_css_dump_pipe_stage_info();
-	sh_css_dump_pipe_stripe_info();
-#endif
-	ia_css_debug_dump_dma_isp_fifo_state();
-	ia_css_debug_dump_dma_sp_fifo_state();
-	ia_css_debug_dump_dma_state();
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-	ia_css_debug_dump_isys_state();
-
-	{
-		irq_controller_state_t state;
-		irq_controller_get_state(IRQ2_ID, &state);
-
-		ia_css_debug_dtrace(2, "\t%-32s:\n",
-				    "Input System IRQ Controller State");
-
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-				    "irq_edge", state.irq_edge);
-
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-				    "irq_mask", state.irq_mask);
-
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-				    "irq_status", state.irq_status);
-
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-				    "irq_enable", state.irq_enable);
-
-		ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
-				    "irq_level_not_pulse",
-				    state.irq_level_not_pulse);
-	}
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
-	ia_css_debug_dump_isys_state();
-#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	ia_css_debug_tagger_state();
-#endif
-	return;
-}
-
-/* this function is for debug use, it can make SP go to sleep
-  state after each frame, then user can dump the stable SP dmem.
-  this function can be called after ia_css_start_sp()
-  and before sh_css_init_buffer_queues()
-*/
-void ia_css_debug_enable_sp_sleep_mode(enum ia_css_sp_sleep_mode mode)
-{
-	const struct ia_css_fw_info *fw;
-	unsigned int HIVE_ADDR_sp_sleep_mode;
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_sp_sleep_mode = fw->info.sp.sleep_mode;
-
-	(void)HIVE_ADDR_sp_sleep_mode;	/* Suppres warnings in CRUN */
-
-	sp_dmem_store_uint32(SP0_ID,
-			     (unsigned int)sp_address_of(sp_sleep_mode),
-			     (uint32_t) mode);
-}
-
-void ia_css_debug_wake_up_sp(void)
-{
-	/*hrt_ctl_start(SP); */
-	sp_ctrl_setbit(SP0_ID, SP_SC_REG, SP_START_BIT);
-}
-
-#if !defined(IS_ISP_2500_SYSTEM)
-#define FIND_DMEM_PARAMS_TYPE(stream, kernel, type) \
-	(struct HRTCAT(HRTCAT(sh_css_isp_, type), _params) *) \
-	findf_dmem_params(stream, offsetof(struct ia_css_memory_offsets, dmem.kernel))
-
-#define FIND_DMEM_PARAMS(stream, kernel) FIND_DMEM_PARAMS_TYPE(stream, kernel, kernel)
-
-/* Find a stage that support the kernel and return the parameters for that kernel */
-static char *
-findf_dmem_params(struct ia_css_stream *stream, short idx)
-{
-	int i;
-	for (i = 0; i < stream->num_pipes; i++) {
-		struct ia_css_pipe *pipe = stream->pipes[i];
-		struct ia_css_pipeline *pipeline = ia_css_pipe_get_pipeline(pipe);
-		struct ia_css_pipeline_stage *stage;
-		for (stage = pipeline->stages; stage; stage = stage->next) {
-			struct ia_css_binary *binary = stage->binary;
-			short *offsets = (short *)&binary->info->mem_offsets.offsets.param->dmem;
-			short dmem_offset = offsets[idx];
-			const struct ia_css_host_data *isp_data =
-				ia_css_isp_param_get_mem_init(&binary->mem_params,
-							IA_CSS_PARAM_CLASS_PARAM, IA_CSS_ISP_DMEM0);
-			if (dmem_offset < 0)
-				continue;
-			return &isp_data->address[dmem_offset];
-		}
-	}
-	return NULL;
-}
-#endif
-
-void ia_css_debug_dump_isp_params(struct ia_css_stream *stream,
-				  unsigned int enable)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "ISP PARAMETERS:\n");
-#if defined(IS_ISP_2500_SYSTEM)
-	(void)enable;
-	(void)stream;
-#else
-
-	assert(stream != NULL);
-	if ((enable & IA_CSS_DEBUG_DUMP_FPN)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_fpn_dump(FIND_DMEM_PARAMS(stream, fpn), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_OB)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_ob_dump(FIND_DMEM_PARAMS(stream, ob), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_SC)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_sc_dump(FIND_DMEM_PARAMS(stream, sc), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_WB)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_wb_dump(FIND_DMEM_PARAMS(stream, wb), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_DP)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_dp_dump(FIND_DMEM_PARAMS(stream, dp), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_BNR)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_bnr_dump(FIND_DMEM_PARAMS(stream, bnr), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_S3A)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_s3a_dump(FIND_DMEM_PARAMS(stream, s3a), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_DE)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_de_dump(FIND_DMEM_PARAMS(stream, de), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_YNR)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_nr_dump(FIND_DMEM_PARAMS_TYPE(stream, nr, ynr),  IA_CSS_DEBUG_VERBOSE);
-		ia_css_yee_dump(FIND_DMEM_PARAMS(stream, yee), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_CSC)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_csc_dump(FIND_DMEM_PARAMS(stream, csc), IA_CSS_DEBUG_VERBOSE);
-		ia_css_yuv2rgb_dump(FIND_DMEM_PARAMS_TYPE(stream, yuv2rgb, csc), IA_CSS_DEBUG_VERBOSE);
-		ia_css_rgb2yuv_dump(FIND_DMEM_PARAMS_TYPE(stream, rgb2yuv, csc), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_GC)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_gc_dump(FIND_DMEM_PARAMS(stream, gc), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_TNR)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_tnr_dump(FIND_DMEM_PARAMS(stream, tnr), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_ANR)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_anr_dump(FIND_DMEM_PARAMS(stream, anr), IA_CSS_DEBUG_VERBOSE);
-	}
-	if ((enable & IA_CSS_DEBUG_DUMP_CE)
-	    || (enable & IA_CSS_DEBUG_DUMP_ALL)) {
-		ia_css_ce_dump(FIND_DMEM_PARAMS(stream, ce), IA_CSS_DEBUG_VERBOSE);
-	}
-#endif
-}
-
-void sh_css_dump_sp_raw_copy_linecount(bool reduced)
-{
-	const struct ia_css_fw_info *fw;
-	unsigned int HIVE_ADDR_raw_copy_line_count;
-	int32_t raw_copy_line_count;
-	static int32_t prev_raw_copy_line_count = -1;
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_raw_copy_line_count =
-			fw->info.sp.raw_copy_line_count;
-
-	(void)HIVE_ADDR_raw_copy_line_count;
-
-	sp_dmem_load(SP0_ID,
-		(unsigned int)sp_address_of(raw_copy_line_count),
-		     &raw_copy_line_count,
-		     sizeof(raw_copy_line_count));
-
-	/* only indicate if copy loop is active */
-	if (reduced)
-		raw_copy_line_count = (raw_copy_line_count < 0)?raw_copy_line_count:1;
-	/* do the handling */
-	if (prev_raw_copy_line_count != raw_copy_line_count) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			"sh_css_dump_sp_raw_copy_linecount() "
-			"line_count=%d\n",
-			raw_copy_line_count);
-		prev_raw_copy_line_count = raw_copy_line_count;
-	}
-}
-
-void ia_css_debug_dump_isp_binary(void)
-{
-	const struct ia_css_fw_info *fw;
-	unsigned int HIVE_ADDR_pipeline_sp_curr_binary_id;
-	uint32_t curr_binary_id;
-	static uint32_t prev_binary_id = 0xFFFFFFFF;
-	static uint32_t sample_count;
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_pipeline_sp_curr_binary_id = fw->info.sp.curr_binary_id;
-
-	(void)HIVE_ADDR_pipeline_sp_curr_binary_id;
-
-	sp_dmem_load(SP0_ID,
-		     (unsigned int)sp_address_of(pipeline_sp_curr_binary_id),
-		     &curr_binary_id,
-		     sizeof(curr_binary_id));
-
-	/* do the handling */
-	sample_count++;
-	if (prev_binary_id != curr_binary_id) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-				    "sh_css_dump_isp_binary() "
-				    "pipe_id=%d, binary_id=%d, sample_count=%d\n",
-				    (curr_binary_id >> 16),
-				    (curr_binary_id & 0x0ffff),
-				    sample_count);
-		sample_count = 0;
-		prev_binary_id = curr_binary_id;
-	}
-}
-
-void ia_css_debug_dump_perf_counters(void)
-{
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-	const struct ia_css_fw_info *fw;
-	int i;
-	unsigned int HIVE_ADDR_ia_css_isys_sp_error_cnt;
-	int32_t ia_css_sp_input_system_error_cnt[N_MIPI_PORT_ID + 1]; /* 3 Capture Units and 1 Acquire Unit. */
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "Input System Error Counters:\n");
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_ia_css_isys_sp_error_cnt = fw->info.sp.perf_counter_input_system_error;
-
-	(void)HIVE_ADDR_ia_css_isys_sp_error_cnt;
-
-	sp_dmem_load(SP0_ID,
-		     (unsigned int)sp_address_of(ia_css_isys_sp_error_cnt),
-		     &ia_css_sp_input_system_error_cnt,
-		     sizeof(ia_css_sp_input_system_error_cnt));
-
-	for (i = 0; i < N_MIPI_PORT_ID + 1; i++) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "\tport[%d] = %d\n",
-				i, ia_css_sp_input_system_error_cnt[i]);
-	}
-#endif
-}
-
-/*
-
-void sh_css_init_ddr_debug_queue(void)
-{
-	hrt_vaddress ddr_debug_queue_addr =
-			mmgr_malloc(sizeof(debug_data_ddr_t));
-	const struct ia_css_fw_info *fw;
-	unsigned int HIVE_ADDR_debug_buffer_ddr_address;
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_debug_buffer_ddr_address =
-			fw->info.sp.debug_buffer_ddr_address;
-
-	(void)HIVE_ADDR_debug_buffer_ddr_address;
-
-	debug_buffer_ddr_init(ddr_debug_queue_addr);
-
-	sp_dmem_store_uint32(SP0_ID,
-		(unsigned int)sp_address_of(debug_buffer_ddr_address),
-		(uint32_t)(ddr_debug_queue_addr));
-}
-
-void sh_css_load_ddr_debug_queue(void)
-{
-	debug_synch_queue_ddr();
-}
-
-void ia_css_debug_dump_ddr_debug_queue(void)
-{
-	int i;
-	sh_css_load_ddr_debug_queue();
-	for (i = 0; i < DEBUG_BUF_SIZE; i++) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			"ddr_debug_queue[%d] = 0x%x\n",
-			i, debug_data_ptr->buf[i]);
-	}
-}
-*/
-
-/*
- * @brief Initialize the debug mode.
- * Refer to "ia_css_debug.h" for more details.
- */
-bool ia_css_debug_mode_init(void)
-{
-	bool rc;
-	rc = sh_css_sp_init_dma_sw_reg(0);
-	return rc;
-}
-
-/*
- * @brief Disable the DMA channel.
- * Refer to "ia_css_debug.h" for more details.
- */
-bool
-ia_css_debug_mode_disable_dma_channel(int dma_id,
-				      int channel_id, int request_type)
-{
-	bool rc;
-
-	rc = sh_css_sp_set_dma_sw_reg(dma_id, channel_id, request_type, false);
-
-	return rc;
-}
-
-/*
- * @brief Enable the DMA channel.
- * Refer to "ia_css_debug.h" for more details.
- */
-bool
-ia_css_debug_mode_enable_dma_channel(int dma_id,
-				     int channel_id, int request_type)
-{
-	bool rc;
-
-	rc = sh_css_sp_set_dma_sw_reg(dma_id, channel_id, request_type, true);
-
-	return rc;
-}
-
-static
-void dtrace_dot(const char *fmt, ...)
-{
-	va_list ap;
-
-	assert(fmt != NULL);
-	va_start(ap, fmt);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_INFO, "%s", DPG_START);
-	ia_css_debug_vdtrace(IA_CSS_DEBUG_INFO, fmt, ap);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_INFO, "%s", DPG_END);
-	va_end(ap);
-}
-#ifdef HAS_WATCHDOG_SP_THREAD_DEBUG
-void sh_css_dump_thread_wait_info(void)
-{
-	const struct ia_css_fw_info *fw;
-	int i;
-	unsigned int HIVE_ADDR_sp_thread_wait;
-	int32_t sp_thread_wait[MAX_THREAD_NUM];
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "SEM WAITS:\n");
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_sp_thread_wait =
-			fw->info.sp.debug_wait;
-
-	(void)HIVE_ADDR_sp_thread_wait;
-
-	sp_dmem_load(SP0_ID,
-		(unsigned int)sp_address_of(sp_thread_wait),
-		     &sp_thread_wait,
-		     sizeof(sp_thread_wait));
-	for (i = 0; i < MAX_THREAD_NUM; i++) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			"\twait[%d] = 0x%X\n",
-			i, sp_thread_wait[i]);
-	}
-
-}
-
-void sh_css_dump_pipe_stage_info(void)
-{
-	const struct ia_css_fw_info *fw;
-	int i;
-	unsigned int HIVE_ADDR_sp_pipe_stage;
-	int32_t sp_pipe_stage[MAX_THREAD_NUM];
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "PIPE STAGE:\n");
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_sp_pipe_stage =
-			fw->info.sp.debug_stage;
-
-	(void)HIVE_ADDR_sp_pipe_stage;
-
-	sp_dmem_load(SP0_ID,
-		(unsigned int)sp_address_of(sp_pipe_stage),
-		     &sp_pipe_stage,
-		     sizeof(sp_pipe_stage));
-	for (i = 0; i < MAX_THREAD_NUM; i++) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			"\tstage[%d] = %d\n",
-			i, sp_pipe_stage[i]);
-	}
-
-}
-
-void sh_css_dump_pipe_stripe_info(void)
-{
-	const struct ia_css_fw_info *fw;
-	int i;
-	unsigned int HIVE_ADDR_sp_pipe_stripe;
-	int32_t sp_pipe_stripe[MAX_THREAD_NUM];
-	ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "PIPE STRIPE:\n");
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_sp_pipe_stripe =
-			fw->info.sp.debug_stripe;
-
-	(void)HIVE_ADDR_sp_pipe_stripe;
-
-	sp_dmem_load(SP0_ID,
-		(unsigned int)sp_address_of(sp_pipe_stripe),
-		     &sp_pipe_stripe,
-		     sizeof(sp_pipe_stripe));
-	for (i = 0; i < MAX_THREAD_NUM; i++) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
-			"\tstripe[%d] = %d\n",
-			i, sp_pipe_stripe[i]);
-	}
-
-}
-#endif
-
-static void
-ia_css_debug_pipe_graph_dump_frame(
-	struct ia_css_frame *frame,
-	enum ia_css_pipe_id id,
-	char const *blob_name,
-	char const *frame_name,
-	bool in_frame)
-{
-	char bufinfo[100];
-
-	if (frame->dynamic_queue_id == SH_CSS_INVALID_QUEUE_ID) {
-		snprintf(bufinfo, sizeof(bufinfo), "Internal");
-	} else {
-		snprintf(bufinfo, sizeof(bufinfo), "Queue: %s %s",
-			pipe_id_to_str[id],
-			queue_id_to_str[frame->dynamic_queue_id]);
-	}
-	dtrace_dot(
-		"node [shape = box, "
-		"fixedsize=true, width=2, height=0.7]; \"%p\" "
-		"[label = \"%s\\n%d(%d) x %d, %dbpp\\n%s\"];",
-		frame,
-		debug_frame_format2str(frame->info.format),
-		frame->info.res.width,
-		frame->info.padded_width,
-		frame->info.res.height,
-		frame->info.raw_bit_depth,
-		bufinfo);
-
-	if (in_frame) {
-		dtrace_dot(
-			"\"%p\"->\"%s(pipe%d)\" "
-			"[label = %s_frame];",
-			frame,
-			blob_name, id, frame_name);
-	} else {
-		dtrace_dot(
-			"\"%s(pipe%d)\"->\"%p\" "
-			"[label = %s_frame];",
-			blob_name, id,
-			frame,
-			frame_name);
-	}
-}
-
-void
-ia_css_debug_pipe_graph_dump_prologue(void)
-{
-	dtrace_dot("digraph sh_css_pipe_graph {");
-	dtrace_dot("rankdir=LR;");
-
-	dtrace_dot("fontsize=9;");
-	dtrace_dot("label = \"\\nEnable options: rp=reduced pipe, vfve=vf_veceven, "
-		"dvse=dvs_envelope, dvs6=dvs_6axis, bo=block_out, "
-		"fbds=fixed_bayer_ds, bf6=bayer_fir_6db, "
-		"rawb=raw_binning, cont=continuous, disc=dis_crop\\n"
-		"dp2a=dp_2adjacent, outp=output, outt=out_table, "
-		"reff=ref_frame, par=params, gam=gamma, "
-		"cagdc=ca_gdc, ispa=isp_addresses, inf=in_frame, "
-		"outf=out_frame, hs=high_speed, inpc=input_chunking\"");
-}
-
-void ia_css_debug_pipe_graph_dump_epilogue(void)
-{
-
-	if (strlen(ring_buffer) > 0) {
-		dtrace_dot(ring_buffer);
-	}
-
-
-	if (pg_inst.stream_format != N_ATOMISP_INPUT_FORMAT) {
-		/* An input stream format has been set so assume we have
-		 * an input system and sensor
-		 */
-
-
-		dtrace_dot(
-			"node [shape = doublecircle, "
-			"fixedsize=true, width=2.5]; \"input_system\" "
-			"[label = \"Input system\"];");
-
-		dtrace_dot(
-			"\"input_system\"->\"%s\" "
-			"[label = \"%s\"];",
-			dot_id_input_bin, debug_stream_format2str(pg_inst.stream_format));
-
-		dtrace_dot(
-			"node [shape = doublecircle, "
-			"fixedsize=true, width=2.5]; \"sensor\" "
-			"[label = \"Sensor\"];");
-
-		dtrace_dot(
-			"\"sensor\"->\"input_system\" "
-			"[label = \"%s\\n%d x %d\\n(%d x %d)\"];",
-			debug_stream_format2str(pg_inst.stream_format),
-			pg_inst.width, pg_inst.height,
-			pg_inst.eff_width, pg_inst.eff_height);
-	}
-
-	dtrace_dot("}");
-
-	/* Reset temp strings */
-	memset(dot_id_input_bin, 0, sizeof(dot_id_input_bin));
-	memset(ring_buffer, 0, sizeof(ring_buffer));
-
-	pg_inst.do_init = true;
-	pg_inst.width = 0;
-	pg_inst.height = 0;
-	pg_inst.eff_width = 0;
-	pg_inst.eff_height = 0;
-	pg_inst.stream_format = N_ATOMISP_INPUT_FORMAT;
-}
-
-void
-ia_css_debug_pipe_graph_dump_stage(
-	struct ia_css_pipeline_stage *stage,
-	enum ia_css_pipe_id id)
-{
-	char blob_name[SH_CSS_MAX_BINARY_NAME+10] = "<unknown type>";
-	char const *bin_type = "<unknown type>";
-	int i;
-
-	assert(stage != NULL);
-	if (stage->sp_func != IA_CSS_PIPELINE_NO_FUNC)
-		return;
-
-	if (pg_inst.do_init) {
-		ia_css_debug_pipe_graph_dump_prologue();
-		pg_inst.do_init = false;
-	}
-
-	if (stage->binary) {
-		bin_type = "binary";
-		if (stage->binary->info->blob)
-			snprintf(blob_name, sizeof(blob_name), "%s_stage%d",
-				stage->binary->info->blob->name, stage->stage_num);
-	} else if (stage->firmware) {
-		bin_type = "firmware";
-		strncpy_s(blob_name, sizeof(blob_name), IA_CSS_EXT_ISP_PROG_NAME(stage->firmware), sizeof(blob_name));
-	}
-
-	/* Guard in case of binaries that don't have any binary_info */
-	if (stage->binary_info != NULL) {
-		char enable_info1[100];
-		char enable_info2[100];
-		char enable_info3[100];
-		char enable_info[200];
-		struct ia_css_binary_info *bi = stage->binary_info;
-
-		/* Split it in 2 function-calls to keep the amount of
-		 * parameters per call "reasonable"
-		 */
-		snprintf(enable_info1, sizeof(enable_info1),
-			"%s%s%s%s%s%s%s%s%s%s%s%s%s%s",
-			bi->enable.reduced_pipe ?	"rp," : "",
-			bi->enable.vf_veceven ?		"vfve," : "",
-			bi->enable.dis ?		"dis," : "",
-			bi->enable.dvs_envelope ?	"dvse," : "",
-			bi->enable.uds ?		"uds," : "",
-			bi->enable.dvs_6axis ?		"dvs6," : "",
-			bi->enable.block_output ?	"bo," : "",
-			bi->enable.ds ?			"ds," : "",
-			bi->enable.bayer_fir_6db ?	"bf6," : "",
-			bi->enable.raw_binning ?	"rawb," : "",
-			bi->enable.continuous ?		"cont," : "",
-			bi->enable.s3a ?		"s3a," : "",
-			bi->enable.fpnr ?		"fpnr," : "",
-			bi->enable.sc ?			"sc," : ""
-			);
-
-		snprintf(enable_info2, sizeof(enable_info2),
-			"%s%s%s%s%s%s%s%s%s%s%s",
-			bi->enable.macc ?		"macc," : "",
-			bi->enable.output ?		"outp," : "",
-			bi->enable.ref_frame ?		"reff," : "",
-			bi->enable.tnr ?		"tnr," : "",
-			bi->enable.xnr ?		"xnr," : "",
-			bi->enable.params ?		"par," : "",
-			bi->enable.ca_gdc ?		"cagdc," : "",
-			bi->enable.isp_addresses ?	"ispa," : "",
-			bi->enable.in_frame ?		"inf," : "",
-			bi->enable.out_frame ?		"outf," : "",
-			bi->enable.high_speed ?		"hs," : ""
-			);
-
-		/* And merge them into one string */
-		snprintf(enable_info, sizeof(enable_info), "%s%s",
-						enable_info1, enable_info2);
-		{
-			int l, p;
-			char *ei = enable_info;
-
-			l = strlen(ei);
-
-			/* Replace last ',' with \0 if present */
-			if (l && enable_info[l-1] == ',')
-				enable_info[--l] = '\0';
-
-			if (l > ENABLE_LINE_MAX_LENGTH) {
-				/* Too big for one line, find last comma */
-				p = ENABLE_LINE_MAX_LENGTH;
-				while (ei[p] != ',')
-					p--;
-				/* Last comma found, copy till that comma */
-				strncpy_s(enable_info1,
-					sizeof(enable_info1),
-					ei, p);
-				enable_info1[p] = '\0';
-
-				ei += p+1;
-				l = strlen(ei);
-
-				if (l <= ENABLE_LINE_MAX_LENGTH) {
-					/* The 2nd line fits */
-					/* we cannot use ei as argument because
-					 * it is not guarenteed dword aligned
-					 */
-					strncpy_s(enable_info2,
-						sizeof(enable_info2),
-						ei, l);
-					enable_info2[l] = '\0';
-					snprintf(enable_info, sizeof(enable_info), "%s\\n%s",
-						enable_info1, enable_info2);
-
-				} else {
-					/* 2nd line is still too long */
-					p = ENABLE_LINE_MAX_LENGTH;
-					while (ei[p] != ',')
-						p--;
-					strncpy_s(enable_info2,
-						sizeof(enable_info2),
-						ei, p);
-					enable_info2[p] = '\0';
-					ei += p+1;
-					l = strlen(ei);
-
-					if (l <= ENABLE_LINE_MAX_LENGTH) {
-						/* The 3rd line fits */
-						/* we cannot use ei as argument because
-						* it is not guarenteed dword aligned
-						*/
-						strcpy_s(enable_info3,
-							sizeof(enable_info3), ei);
-						enable_info3[l] = '\0';
-						snprintf(enable_info, sizeof(enable_info),
-							"%s\\n%s\\n%s",
-							enable_info1, enable_info2,
-							enable_info3);
-					} else {
-						/* 3rd line is still too long */
-						p = ENABLE_LINE_MAX_LENGTH;
-						while (ei[p] != ',')
-							p--;
-						strncpy_s(enable_info3,
-							sizeof(enable_info3),
-							ei, p);
-						enable_info3[p] = '\0';
-						ei += p+1;
-						strcpy_s(enable_info3,
-							sizeof(enable_info3), ei);
-						snprintf(enable_info, sizeof(enable_info),
-							"%s\\n%s\\n%s",
-							enable_info1, enable_info2,
-							enable_info3);
-					}
-				}
-			}
-		}
-
-		dtrace_dot("node [shape = circle, fixedsize=true, width=2.5, "
-			"label=\"%s\\n%s\\n\\n%s\"]; \"%s(pipe%d)\"",
-			bin_type, blob_name, enable_info, blob_name, id);
-
-	}
-	else {
-		dtrace_dot("node [shape = circle, fixedsize=true, width=2.5, "
-			"label=\"%s\\n%s\\n\"]; \"%s(pipe%d)\"",
-			bin_type, blob_name, blob_name, id);
-	}
-
-	if (stage->stage_num == 0) {
-		/*
-		 * There are some implicite assumptions about which bin is the
-		 * input binary e.g. which one is connected to the input system
-		 * Priority:
-		 * 1) sp_raw_copy bin has highest priority
-		 * 2) First stage==0 binary of preview, video or capture
-		 */
-		if (strlen(dot_id_input_bin) == 0) {
-			snprintf(dot_id_input_bin, sizeof(dot_id_input_bin),
-				"%s(pipe%d)", blob_name, id);
-		}
-	}
-
-	if (stage->args.in_frame) {
-		ia_css_debug_pipe_graph_dump_frame(
-			stage->args.in_frame, id, blob_name,
-			"in", true);
-	}
-
-#ifndef ISP2401
-	for (i = 0; i < NUM_VIDEO_TNR_FRAMES; i++) {
-#else
-	for (i = 0; i < NUM_TNR_FRAMES; i++) {
-#endif
-		if (stage->args.tnr_frames[i]) {
-			ia_css_debug_pipe_graph_dump_frame(
-					stage->args.tnr_frames[i], id,
-					blob_name, "tnr_frame", true);
-		}
-	}
-
-	for (i = 0; i < MAX_NUM_VIDEO_DELAY_FRAMES; i++) {
-		if (stage->args.delay_frames[i]) {
-			ia_css_debug_pipe_graph_dump_frame(
-					stage->args.delay_frames[i], id,
-					blob_name, "delay_frame", true);
-		}
-	}
-
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		if (stage->args.out_frame[i]) {
-			ia_css_debug_pipe_graph_dump_frame(
-				stage->args.out_frame[i], id, blob_name,
-				"out", false);
-		}
-	}
-
-	if (stage->args.out_vf_frame) {
-		ia_css_debug_pipe_graph_dump_frame(
-			stage->args.out_vf_frame, id, blob_name,
-			"out_vf", false);
-	}
-}
-
-void
-ia_css_debug_pipe_graph_dump_sp_raw_copy(
-	struct ia_css_frame *out_frame)
-{
-	assert(out_frame != NULL);
-	if (pg_inst.do_init) {
-		ia_css_debug_pipe_graph_dump_prologue();
-		pg_inst.do_init = false;
-	}
-
-	dtrace_dot("node [shape = circle, fixedsize=true, width=2.5, "
-		"label=\"%s\\n%s\"]; \"%s(pipe%d)\"",
-		"sp-binary", "sp_raw_copy", "sp_raw_copy", 1);
-
-	snprintf(ring_buffer, sizeof(ring_buffer),
-		"node [shape = box, "
-		"fixedsize=true, width=2, height=0.7]; \"%p\" "
-		"[label = \"%s\\n%d(%d) x %d\\nRingbuffer\"];",
-		out_frame,
-		debug_frame_format2str(out_frame->info.format),
-		out_frame->info.res.width,
-		out_frame->info.padded_width,
-		out_frame->info.res.height);
-
-	dtrace_dot(ring_buffer);
-
-	dtrace_dot(
-		"\"%s(pipe%d)\"->\"%p\" "
-		"[label = out_frame];",
-		"sp_raw_copy", 1, out_frame);
-
-	snprintf(dot_id_input_bin, sizeof(dot_id_input_bin), "%s(pipe%d)", "sp_raw_copy", 1);
-}
-
-void
-ia_css_debug_pipe_graph_dump_stream_config(
-	const struct ia_css_stream_config *stream_config)
-{
-	pg_inst.width = stream_config->input_config.input_res.width;
-	pg_inst.height = stream_config->input_config.input_res.height;
-	pg_inst.eff_width = stream_config->input_config.effective_res.width;
-	pg_inst.eff_height = stream_config->input_config.effective_res.height;
-	pg_inst.stream_format = stream_config->input_config.format;
-}
-
-void
-ia_css_debug_dump_resolution(
-	const struct ia_css_resolution *res,
-	const char *label)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%s: =%d x =%d\n",
-			label, res->width, res->height);
-}
-
-void
-ia_css_debug_dump_frame_info(
-	const struct ia_css_frame_info *info,
-	const char *label)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%s\n", label);
-	ia_css_debug_dump_resolution(&info->res, "res");
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "padded_width: %d\n",
-			info->padded_width);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "format: %d\n", info->format);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "raw_bit_depth: %d\n",
-			info->raw_bit_depth);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "raw_bayer_order: %d\n",
-			info->raw_bayer_order);
-}
-
-void
-ia_css_debug_dump_capture_config(
-	const struct ia_css_capture_config *config)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%s\n", __func__);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "mode: %d\n", config->mode);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "enable_xnr:  %d\n",
-			config->enable_xnr);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "enable_raw_output: %d\n",
-			config->enable_raw_output);
-}
-
-void
-ia_css_debug_dump_pipe_extra_config(
-	const struct ia_css_pipe_extra_config *extra_config)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%s\n", __func__);
-	if (extra_config) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"enable_raw_binning: %d\n",
-				extra_config->enable_raw_binning);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "enable_yuv_ds: %d\n",
-				extra_config->enable_yuv_ds);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"enable_high_speed:  %d\n",
-				extra_config->enable_high_speed);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"enable_dvs_6axis: %d\n",
-				extra_config->enable_dvs_6axis);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"enable_reduced_pipe: %d\n",
-				extra_config->enable_reduced_pipe);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"enable_fractional_ds: %d\n",
-				extra_config->enable_fractional_ds);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "disable_vf_pp: %d\n",
-				extra_config->disable_vf_pp);
-	}
-}
-
-void
-ia_css_debug_dump_pipe_config(
-	const struct ia_css_pipe_config *config)
-{
-	unsigned int i;
-
-	IA_CSS_ENTER_PRIVATE("config = %p", config);
-	if (!config) {
-		IA_CSS_ERROR("NULL input parameter");
-		IA_CSS_LEAVE_PRIVATE("");
-		return;
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "mode: %d\n", config->mode);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "isp_pipe_version: %d\n",
-			config->isp_pipe_version);
-	ia_css_debug_dump_resolution(&config->bayer_ds_out_res,
-			"bayer_ds_out_res");
-	ia_css_debug_dump_resolution(&config->capt_pp_in_res,
-			"capt_pp_in_res");
-	ia_css_debug_dump_resolution(&config->vf_pp_in_res, "vf_pp_in_res");
-#ifdef ISP2401
-	ia_css_debug_dump_resolution(&config->output_system_in_res,
-				     "output_system_in_res");
-#endif
-	ia_css_debug_dump_resolution(&config->dvs_crop_out_res,
-			"dvs_crop_out_res");
-	for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-		ia_css_debug_dump_frame_info(&config->output_info[i], "output_info");
-		ia_css_debug_dump_frame_info(&config->vf_output_info[i],
-				"vf_output_info");
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "acc_extension: %p\n",
-			    config->acc_extension);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "num_acc_stages: %d\n",
-			config->num_acc_stages);
-	ia_css_debug_dump_capture_config(&config->default_capture_config);
-	ia_css_debug_dump_resolution(&config->dvs_envelope, "dvs_envelope");
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "dvs_frame_delay: %d\n",
-			config->dvs_frame_delay);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "acc_num_execs: %d\n",
-			config->acc_num_execs);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "enable_dz: %d\n",
-			config->enable_dz);
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void
-ia_css_debug_dump_stream_config_source(
-	const struct ia_css_stream_config *config)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%s()\n", __func__);
-	switch (config->mode) {
-	case IA_CSS_INPUT_MODE_SENSOR:
-	case IA_CSS_INPUT_MODE_BUFFERED_SENSOR:
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "source.port\n");
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "port: %d\n",
-				config->source.port.port);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "num_lanes: %d\n",
-				config->source.port.num_lanes);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "timeout: %d\n",
-				config->source.port.timeout);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "compression: %d\n",
-				config->source.port.compression.type);
-		break;
-	case IA_CSS_INPUT_MODE_TPG:
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "source.tpg\n");
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "id: %d\n",
-				config->source.tpg.id);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "mode: %d\n",
-				config->source.tpg.mode);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "x_mask: 0x%x\n",
-				config->source.tpg.x_mask);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "x_delta: %d\n",
-				config->source.tpg.x_delta);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "y_mask: 0x%x\n",
-				config->source.tpg.y_mask);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "y_delta: %d\n",
-				config->source.tpg.y_delta);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "xy_mask: 0x%x\n",
-				config->source.tpg.xy_mask);
-		break;
-	case IA_CSS_INPUT_MODE_PRBS:
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "source.prbs\n");
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "id: %d\n",
-				config->source.prbs.id);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "h_blank: %d\n",
-				config->source.prbs.h_blank);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "v_blank: %d\n",
-				config->source.prbs.v_blank);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "seed: 0x%x\n",
-				config->source.prbs.seed);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "seed1: 0x%x\n",
-				config->source.prbs.seed1);
-		break;
-	default:
-	case IA_CSS_INPUT_MODE_FIFO:
-	case IA_CSS_INPUT_MODE_MEMORY:
-		break;
-	}
-}
-
-void
-ia_css_debug_dump_mipi_buffer_config(
-	const struct ia_css_mipi_buffer_config *config)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%s()\n", __func__);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "size_mem_words: %d\n",
-			config->size_mem_words);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "nof_mipi_buffers: %d\n",
-			config->nof_mipi_buffers);
-}
-
-void
-ia_css_debug_dump_metadata_config(
-	const struct ia_css_metadata_config *config)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%s()\n", __func__);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "data_type: %d\n",
-			config->data_type);
-	ia_css_debug_dump_resolution(&config->resolution, "resolution");
-}
-
-void
-ia_css_debug_dump_stream_config(
-	const struct ia_css_stream_config *config,
-	int num_pipes)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%s()\n", __func__);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "num_pipes: %d\n", num_pipes);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "mode: %d\n", config->mode);
-	ia_css_debug_dump_stream_config_source(config);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "channel_id: %d\n",
-			config->channel_id);
-	ia_css_debug_dump_resolution(&config->input_config.input_res, "input_res");
-	ia_css_debug_dump_resolution(&config->input_config.effective_res, "effective_res");
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "format: %d\n",
-			config->input_config.format);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "bayer_order: %d\n",
-			config->input_config.bayer_order);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sensor_binning_factor: %d\n",
-			config->sensor_binning_factor);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "pixels_per_clock: %d\n",
-			config->pixels_per_clock);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "online: %d\n",
-			config->online);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "init_num_cont_raw_buf: %d\n",
-			config->init_num_cont_raw_buf);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			"target_num_cont_raw_buf: %d\n",
-			config->target_num_cont_raw_buf);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "pack_raw_pixels: %d\n",
-			config->pack_raw_pixels);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "continuous: %d\n",
-			config->continuous);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "flash_gpio_pin: %d\n",
-			config->flash_gpio_pin);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "left_padding: %d\n",
-			config->left_padding);
-	ia_css_debug_dump_mipi_buffer_config(&config->mipi_buffer_config);
-	ia_css_debug_dump_metadata_config(&config->metadata_config);
-}
-
-/*
-    Trace support.
-
-    This tracer is using a buffer to trace the flow of the FW and dump misc values (see below for details).
-    Currently, support is only for SKC.
-    To enable support for other platforms:
-     - Allocate a buffer for tracing in DMEM. The longer the better.
-     - Use the DBG_init routine in sp.hive.c to initiatilize the tracer with the address and size selected.
-     - Add trace points in the SP code wherever needed.
-     - Enable the dump below with the required address and required adjustments.
-	   Dump is called at the end of ia_css_debug_dump_sp_state().
-*/
-
-/*
- dump_trace() : dump the trace points from DMEM2.
- for every trace point, the following are printed: index, major:minor and the 16-bit attached value.
- The routine looks for the first 0, and then prints from it cyclically.
- Data forma in DMEM2:
-  first 4 DWORDS: header
-   DWORD 0: data description
-    byte 0: version
-    byte 1: number of threads (for future use)
-    byte 2+3: number ot TPs
-   DWORD 1: command byte + data (for future use)
-    byte 0: command
-    byte 1-3: command signature
-   DWORD 2-3: additional data (for future use)
-  Following data is 4-byte oriented:
-    byte 0:   major
-	byte 1:   minor
-	byte 2-3: data
-*/
-#if TRACE_ENABLE_SP0 || TRACE_ENABLE_SP1 || TRACE_ENABLE_ISP
-#ifndef ISP2401
-static void debug_dump_one_trace(TRACE_CORE_ID proc_id)
-#else
-static void debug_dump_one_trace(enum TRACE_CORE_ID proc_id)
-#endif
-{
-#if defined(HAS_TRACER_V2)
-	uint32_t start_addr;
-	uint32_t start_addr_data;
-	uint32_t item_size;
-#ifndef ISP2401
-	uint32_t tmp;
-#else
-	uint8_t tid_val;
-	enum TRACE_DUMP_FORMAT dump_format;
-#endif
-	int i, j, max_trace_points, point_num, limit = -1;
-	/* using a static buffer here as the driver has issues allocating memory */
-	static uint32_t trace_read_buf[TRACE_BUFF_SIZE] = {0};
-#ifdef ISP2401
-	static struct trace_header_t header;
-	uint8_t *header_arr;
-#endif
-
-	/* read the header and parse it */
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "~~~ Tracer ");
-	switch (proc_id)
-	{
-	case TRACE_SP0_ID:
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "SP0");
-		start_addr = TRACE_SP0_ADDR;
-		start_addr_data = TRACE_SP0_DATA_ADDR;
-		item_size = TRACE_SP0_ITEM_SIZE;
-		max_trace_points = TRACE_SP0_MAX_POINTS;
-		break;
-	case TRACE_SP1_ID:
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "SP1");
-		start_addr = TRACE_SP1_ADDR;
-		start_addr_data = TRACE_SP1_DATA_ADDR;
-		item_size = TRACE_SP1_ITEM_SIZE;
-		max_trace_points = TRACE_SP1_MAX_POINTS;
-		break;
-	case TRACE_ISP_ID:
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ISP");
-		start_addr = TRACE_ISP_ADDR;
-		start_addr_data = TRACE_ISP_DATA_ADDR;
-		item_size = TRACE_ISP_ITEM_SIZE;
-		max_trace_points = TRACE_ISP_MAX_POINTS;
-		break;
-	default:
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "\t\ttraces are not supported for this processor ID - exiting\n");
-		return;
-	}
-#ifndef ISP2401
-	tmp = ia_css_device_load_uint32(start_addr);
-	point_num = (tmp >> 16) & 0xFFFF;
-#endif
-
-#ifndef ISP2401
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, " ver %d %d points\n", tmp & 0xFF, point_num);
-	if ((tmp & 0xFF) != TRACER_VER) {
-#else
-	/* Loading byte-by-byte as using the master routine had issues */
-	header_arr = (uint8_t *)&header;
-	for (i = 0; i < (int)sizeof(struct trace_header_t); i++)
-		header_arr[i] = ia_css_device_load_uint8(start_addr + (i));
-
-	point_num = header.max_tracer_points;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, " ver %d %d points\n", header.version, point_num);
-	if ((header.version & 0xFF) != TRACER_VER) {
-#endif
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "\t\tUnknown version - exiting\n");
-		return;
-	}
-	if (point_num > max_trace_points) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "\t\tToo many points - exiting\n");
-		return;
-	}
-	/* copy the TPs and find the first 0 */
-	for (i = 0; i < point_num; i++) {
-		trace_read_buf[i] = ia_css_device_load_uint32(start_addr_data + (i * item_size));
-		if ((limit == (-1)) && (trace_read_buf[i] == 0))
-			limit = i;
-	}
-#ifdef ISP2401
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "Status:\n");
-	for (i = 0; i < SH_CSS_MAX_SP_THREADS; i++)
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "\tT%d: %3d (%02x)  %6d (%04x)  %10d (%08x)\n", i,
-				header.thr_status_byte[i], header.thr_status_byte[i],
-				header.thr_status_word[i], header.thr_status_word[i],
-				header.thr_status_dword[i], header.thr_status_dword[i]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "Scratch:\n");
-	for (i = 0; i < MAX_SCRATCH_DATA; i++)
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%10d (%08x)  ",
-			header.scratch_debug[i], header.scratch_debug[i]);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "\n");
-
-#endif
-	/* two 0s in the beginning: empty buffer */
-	if ((trace_read_buf[0] == 0) && (trace_read_buf[1] == 0)) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "\t\tEmpty tracer - exiting\n");
-		return;
-	}
-	/* no overrun: start from 0 */
-	if ((limit == point_num-1) ||         /* first 0 is at the end - border case */
-	    (trace_read_buf[limit+1] == 0))   /* did not make a full cycle after the memset */
-		limit = 0;
-	/* overrun: limit is the first non-zero after the first zero */
-	else
-		limit++;
-
-	/* print the TPs */
-	for (i = 0; i < point_num; i++) {
-		j = (limit + i) % point_num;
-		if (trace_read_buf[j])
-		{
-#ifndef ISP2401
-			TRACE_DUMP_FORMAT dump_format = FIELD_FORMAT_UNPACK(trace_read_buf[j]);
-#else
-
-			tid_val = FIELD_TID_UNPACK(trace_read_buf[j]);
-			dump_format = TRACE_DUMP_FORMAT_POINT;
-
-			/*
-			 * When tid value is 111b, the data will be interpreted differently:
-			 * tid val is ignored, major field contains 2 bits (msb) for format type
-			 */
-			if (tid_val == FIELD_TID_SEL_FORMAT_PAT) {
-				dump_format = FIELD_FORMAT_UNPACK(trace_read_buf[j]);
-			}
-#endif
-			switch (dump_format)
-			{
-			case TRACE_DUMP_FORMAT_POINT:
-				ia_css_debug_dtrace(
-#ifndef ISP2401
-						IA_CSS_DEBUG_TRACE,	"\t\t%d %d:%d value - %d\n",
-						j, FIELD_MAJOR_UNPACK(trace_read_buf[j]),
-#else
-						IA_CSS_DEBUG_TRACE,	"\t\t%d T%d %d:%d value - %x (%d)\n",
-						j,
-						tid_val,
-						FIELD_MAJOR_UNPACK(trace_read_buf[j]),
-#endif
-						FIELD_MINOR_UNPACK(trace_read_buf[j]),
-#ifdef ISP2401
-						FIELD_VALUE_UNPACK(trace_read_buf[j]),
-#endif
-						FIELD_VALUE_UNPACK(trace_read_buf[j]));
-				break;
-#ifndef ISP2401
-			case TRACE_DUMP_FORMAT_VALUE24_HEX:
-#else
-			case TRACE_DUMP_FORMAT_POINT_NO_TID:
-#endif
-				ia_css_debug_dtrace(
-#ifndef ISP2401
-						IA_CSS_DEBUG_TRACE,	"\t\t%d, %d, 24bit value %x H\n",
-#else
-						IA_CSS_DEBUG_TRACE,	"\t\t%d %d:%d value - %x (%d)\n",
-#endif
-						j,
-#ifndef ISP2401
-						FIELD_MAJOR_UNPACK(trace_read_buf[j]),
-						FIELD_VALUE_24_UNPACK(trace_read_buf[j]));
-#else
-						FIELD_MAJOR_W_FMT_UNPACK(trace_read_buf[j]),
-						FIELD_MINOR_UNPACK(trace_read_buf[j]),
-						FIELD_VALUE_UNPACK(trace_read_buf[j]),
-						FIELD_VALUE_UNPACK(trace_read_buf[j]));
-#endif
-				break;
-#ifndef ISP2401
-			case TRACE_DUMP_FORMAT_VALUE24_DEC:
-#else
-			case TRACE_DUMP_FORMAT_VALUE24:
-#endif
-				ia_css_debug_dtrace(
-#ifndef ISP2401
-						IA_CSS_DEBUG_TRACE,	"\t\t%d, %d, 24bit value %d D\n",
-#else
-						IA_CSS_DEBUG_TRACE,	"\t\t%d, %d, 24bit value %x (%d)\n",
-#endif
-						j,
-						FIELD_MAJOR_UNPACK(trace_read_buf[j]),
-#ifdef ISP2401
-						FIELD_MAJOR_W_FMT_UNPACK(trace_read_buf[j]),
-						FIELD_VALUE_24_UNPACK(trace_read_buf[j]),
-#endif
-						FIELD_VALUE_24_UNPACK(trace_read_buf[j]));
-				break;
-#ifdef ISP2401
-
-#endif
-			case TRACE_DUMP_FORMAT_VALUE24_TIMING:
-				ia_css_debug_dtrace(
-						IA_CSS_DEBUG_TRACE,	"\t\t%d, %d, timing %x\n",
-						j,
-#ifndef ISP2401
-						FIELD_MAJOR_UNPACK(trace_read_buf[j]),
-#else
-						FIELD_MAJOR_W_FMT_UNPACK(trace_read_buf[j]),
-#endif
-						FIELD_VALUE_24_UNPACK(trace_read_buf[j]));
-				break;
-			case TRACE_DUMP_FORMAT_VALUE24_TIMING_DELTA:
-				ia_css_debug_dtrace(
-						IA_CSS_DEBUG_TRACE,	"\t\t%d, %d, timing delta %x\n",
-						j,
-#ifndef ISP2401
-						FIELD_MAJOR_UNPACK(trace_read_buf[j]),
-#else
-						FIELD_MAJOR_W_FMT_UNPACK(trace_read_buf[j]),
-#endif
-						FIELD_VALUE_24_UNPACK(trace_read_buf[j]));
-				break;
-			default:
-				ia_css_debug_dtrace(
-						IA_CSS_DEBUG_TRACE,
-						"no such trace dump format %d",
-#ifndef ISP2401
-						FIELD_FORMAT_UNPACK(trace_read_buf[j]));
-#else
-						dump_format);
-#endif
-				break;
-			}
-		}
-	}
-#else
-	(void)proc_id;
-#endif /* HAS_TRACER_V2 */
-}
-#endif /* TRACE_ENABLE_SP0 || TRACE_ENABLE_SP1 || TRACE_ENABLE_ISP */
-
-void ia_css_debug_dump_trace(void)
-{
-#if TRACE_ENABLE_SP0
-	debug_dump_one_trace(TRACE_SP0_ID);
-#endif
-#if TRACE_ENABLE_SP1
-	debug_dump_one_trace(TRACE_SP1_ID);
-#endif
-#if TRACE_ENABLE_ISP
-	debug_dump_one_trace(TRACE_ISP_ID);
-#endif
-}
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-/* Tagger state dump function. The tagger is only available when the CSS
- * contains an input system (2400 or 2401). */
-void ia_css_debug_tagger_state(void)
-{
-	unsigned int i;
-	unsigned int HIVE_ADDR_tagger_frames;
-	ia_css_tagger_buf_sp_elem_t tbuf_frames[MAX_CB_ELEMS_FOR_TAGGER];
-
-	HIVE_ADDR_tagger_frames = sh_css_sp_fw.info.sp.tagger_frames_addr;
-
-	/* This variable is not used in crun */
-	(void)HIVE_ADDR_tagger_frames;
-
-	/* 2400 and 2401 only have 1 SP, so the tagger lives on SP0 */
-	sp_dmem_load(SP0_ID,
-		     (unsigned int)sp_address_of(tagger_frames),
-		     tbuf_frames,
-		     sizeof(tbuf_frames));
-
-	ia_css_debug_dtrace(2, "Tagger Info:\n");
-	for (i = 0; i < MAX_CB_ELEMS_FOR_TAGGER; i++) {
-		ia_css_debug_dtrace(2, "\t tagger frame[%d]: exp_id=%d, marked=%d, locked=%d\n",
-				i, tbuf_frames[i].exp_id, tbuf_frames[i].mark, tbuf_frames[i].lock);
-	}
-
-}
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401) */
-
-#ifdef ISP2401
-void ia_css_debug_pc_dump(sp_ID_t id, unsigned int num_of_dumps)
-{
-	unsigned int pc;
-	unsigned int i;
-	hrt_data sc = sp_ctrl_load(id, SP_SC_REG);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "SP%-1d Status reg: 0x%X\n", id, sc);
-	sc = sp_ctrl_load(id, SP_CTRL_SINK_REG);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "SP%-1d Stall reg: 0x%X\n", id, sc);
-	for (i = 0; i < num_of_dumps; i++) {
-		pc = sp_ctrl_load(id, SP_PC_REG);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "SP%-1d PC: 0x%X\n", id, pc);
-	}
-}
-#endif
-
-#if defined(HRT_SCHED) || defined(SH_CSS_DEBUG_SPMEM_DUMP_SUPPORT)
-#include "spmem_dump.c"
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/event/interface/ia_css_event.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/event/interface/ia_css_event.h
deleted file mode 100644
index ab1d9bed9fd8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/event/interface/ia_css_event.h
+++ /dev/null
@@ -1,46 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _IA_CSS_EVENT_H
-#define _IA_CSS_EVENT_H
-
-#include <type_support.h>
-#include "sw_event_global.h"    /*event macros.TODO : Change File Name..???*/
-
-bool ia_css_event_encode(
-	uint8_t	*in,
-	uint8_t	nr,
-	uint32_t	*out);
-
-void ia_css_event_decode(
-	uint32_t event,
-	uint8_t *payload);
-
-#endif /*_IA_CSS_EVENT_H*/
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/event/src/event.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/event/src/event.c
deleted file mode 100644
index 239c06730bf4..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/event/src/event.c
+++ /dev/null
@@ -1,126 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "sh_css_sp.h"
-
-#include "dma.h"	/* N_DMA_CHANNEL_ID */
-
-#include <type_support.h>
-#include "ia_css_binary.h"
-#include "sh_css_hrt.h"
-#include "sh_css_defs.h"
-#include "sh_css_internal.h"
-#include "ia_css_debug.h"
-#include "ia_css_debug_internal.h"
-#include "sh_css_legacy.h"
-
-#include "gdc_device.h"				/* HRT_GDC_N */
-
-/*#include "sp.h"*/	/* host2sp_enqueue_frame_data() */
-
-#include "memory_access.h"
-
-#include "assert_support.h"
-#include "platform_support.h"	/* hrt_sleep() */
-
-#include "ia_css_queue.h"	/* host_sp_enqueue_XXX */
-#include "ia_css_event.h"	/* ia_css_event_encode */
-/*
- * @brief Encode the information into the software-event.
- * Refer to "sw_event_public.h" for details.
- */
-bool ia_css_event_encode(
-	uint8_t	*in,
-	uint8_t	nr,
-	uint32_t	*out)
-{
-	bool ret;
-	uint32_t nr_of_bits;
-	uint32_t i;
-	assert(in != NULL);
-	assert(out != NULL);
-	OP___assert(nr > 0 && nr <= MAX_NR_OF_PAYLOADS_PER_SW_EVENT);
-
-	/* initialize the output */
-	*out = 0;
-
-	/* get the number of bits per information */
-	nr_of_bits = sizeof(uint32_t) * 8 / nr;
-
-	/* compress the all inputs into a signle output */
-	for (i = 0; i < nr; i++) {
-		*out <<= nr_of_bits;
-		*out |= in[i];
-	}
-
-	/* get the return value */
-	ret = (nr > 0 && nr <= MAX_NR_OF_PAYLOADS_PER_SW_EVENT);
-
-	return ret;
-}
-
-void ia_css_event_decode(
-	uint32_t event,
-	uint8_t *payload)
-{
-	assert(payload[1] == 0);
-	assert(payload[2] == 0);
-	assert(payload[3] == 0);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_event_decode() enter:\n");
-
-	/* First decode according to the common case
-	 * In case of a PORT_EOF event we overwrite with
-	 * the specific values
-	 * This is somewhat ugly but probably somewhat efficient
-	 * (and it avoids some code duplication)
-	 */
-	payload[0] = event & 0xff;  /*event_code */
-	payload[1] = (event >> 8) & 0xff;
-	payload[2] = (event >> 16) & 0xff;
-	payload[3] = 0;
-
-	switch (payload[0]) {
-	case SH_CSS_SP_EVENT_PORT_EOF:
-		payload[2] = 0;
-		payload[3] = (event >> 24) & 0xff;
-		break;
-
-	case SH_CSS_SP_EVENT_ACC_STAGE_COMPLETE:
-	case SH_CSS_SP_EVENT_TIMER:
-	case SH_CSS_SP_EVENT_FRAME_TAGGED:
-	case SH_CSS_SP_EVENT_FW_WARNING:
-	case SH_CSS_SP_EVENT_FW_ASSERT:
-		payload[3] = (event >> 24) & 0xff;
-		break;
-	default:
-		break;
-	}
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/eventq/interface/ia_css_eventq.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/eventq/interface/ia_css_eventq.h
deleted file mode 100644
index 67eb8fdb33c5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/eventq/interface/ia_css_eventq.h
+++ /dev/null
@@ -1,69 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _IA_CSS_EVENTQ_H
-#define _IA_CSS_EVENTQ_H
-
-#include "ia_css_queue.h"	/* queue APIs */
-
-/**
- * @brief HOST receives event from SP.
- *
- * @param[in]	eventq_handle	eventq_handle.
- * @param[in]	payload		The event payload.
- * @return	0		- Successfully dequeue.
- * @return	EINVAL		- Invalid argument.
- * @return	ENODATA		- Queue is empty.
- */
-int ia_css_eventq_recv(
-		ia_css_queue_t *eventq_handle,
-		uint8_t *payload);
-
-/**
- * @brief The Host sends the event to SP.
- * The caller of this API will be blocked until the event
- * is sent.
- *
- * @param[in]	eventq_handle   eventq_handle.
- * @param[in]	evt_id		The event ID.
- * @param[in]	evt_payload_0	The event payload.
- * @param[in]	evt_payload_1	The event payload.
- * @param[in]	evt_payload_2	The event payload.
- * @return	0		- Successfully enqueue.
- * @return	EINVAL		- Invalid argument.
- * @return	ENOBUFS		- Queue is full.
- */
-int ia_css_eventq_send(
-		ia_css_queue_t *eventq_handle,
-		uint8_t evt_id,
-		uint8_t evt_payload_0,
-		uint8_t evt_payload_1,
-		uint8_t evt_payload_2);
-#endif /* _IA_CSS_EVENTQ_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/eventq/src/eventq.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/eventq/src/eventq.c
deleted file mode 100644
index 913a4bf7a34f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/eventq/src/eventq.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_types.h"
-#include "assert_support.h"
-#include "ia_css_queue.h" /* sp2host_dequeue_irq_event() */
-#include "ia_css_eventq.h"
-#include "ia_css_event.h"	/* ia_css_event_encode()
-				ia_css_event_decode()
-				*/
-#include "platform_support.h" /* hrt_sleep() */
-
-int ia_css_eventq_recv(
-		ia_css_queue_t *eventq_handle,
-		uint8_t *payload)
-{
-	uint32_t sp_event;
-	int error;
-
-	/* dequeue the IRQ event */
-	error = ia_css_queue_dequeue(eventq_handle, &sp_event);
-
-	/* check whether the IRQ event is available or not */
-	if (!error)
-		ia_css_event_decode(sp_event, payload);
-	return error;
-}
-
-/*
- * @brief The Host sends the event to the SP.
- * Refer to "sh_css_sp.h" for details.
- */
-int ia_css_eventq_send(
-			ia_css_queue_t *eventq_handle,
-			uint8_t evt_id,
-			uint8_t evt_payload_0,
-			uint8_t evt_payload_1,
-			uint8_t evt_payload_2)
-{
-	uint8_t tmp[4];
-	uint32_t sw_event;
-	int error = ENOSYS;
-
-	/*
-	 * Encode the queue type, the thread ID and
-	 * the queue ID into the event.
-	 */
-	tmp[0] = evt_id;
-	tmp[1] = evt_payload_0;
-	tmp[2] = evt_payload_1;
-	tmp[3] = evt_payload_2;
-	ia_css_event_encode(tmp, 4, &sw_event);
-
-	/* queue the software event (busy-waiting) */
-	for ( ; ; ) {
-		error = ia_css_queue_enqueue(eventq_handle, sw_event);
-		if (ENOBUFS != error) {
-			/* We were able to successfully send the event
-			   or had a real failure. return the status*/
-			break;
-		}
-		/* Wait for the queue to be not full and try again*/
-		hrt_sleep();
-	}
-	return error;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/interface/ia_css_frame.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/interface/ia_css_frame.h
deleted file mode 100644
index 89ad8080ceb1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/interface/ia_css_frame.h
+++ /dev/null
@@ -1,180 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_FRAME_H__
-#define __IA_CSS_FRAME_H__
-
-#ifdef ISP2401
-#include <ia_css_types.h>
-#endif
-#include <ia_css_frame_format.h>
-#include <ia_css_frame_public.h>
-#include "dma.h"
-
-/*********************************************************************
-****	Frame INFO APIs
-**********************************************************************/
-/* @brief Sets the given width and alignment to the frame info
- *
- * @param
- * @param[in]	info        The info to which parameters would set
- * @param[in]	width       The width to be set to info
- * @param[in]	aligned     The aligned to be set to info
- * @return
- */
-void ia_css_frame_info_set_width(struct ia_css_frame_info *info,
-	unsigned int width,
-	unsigned int min_padded_width);
-
-/* @brief Sets the given format to the frame info
- *
- * @param
- * @param[in]	info        The info to which parameters would set
- * @param[in]	format      The format to be set to info
- * @return
- */
-void ia_css_frame_info_set_format(struct ia_css_frame_info *info,
-	enum ia_css_frame_format format);
-
-/* @brief Sets the frame info with the given parameters
- *
- * @param
- * @param[in]	info        The info to which parameters would set
- * @param[in]	width       The width to be set to info
- * @param[in]	height      The height to be set to info
- * @param[in]	format      The format to be set to info
- * @param[in]	aligned     The aligned to be set to info
- * @return
- */
-void ia_css_frame_info_init(struct ia_css_frame_info *info,
-	unsigned int width,
-	unsigned int height,
-	enum ia_css_frame_format format,
-	unsigned int aligned);
-
-/* @brief Checks whether 2 frame infos has the same resolution
- *
- * @param
- * @param[in]	frame_a         The first frame to be compared
- * @param[in]	frame_b         The second frame to be compared
- * @return      Returns true if the frames are equal
- */
-bool ia_css_frame_info_is_same_resolution(
-	const struct ia_css_frame_info *info_a,
-	const struct ia_css_frame_info *info_b);
-
-/* @brief Check the frame info is valid
- *
- * @param
- * @param[in]	info       The frame attributes to be initialized
- * @return	The error code.
- */
-enum ia_css_err ia_css_frame_check_info(const struct ia_css_frame_info *info);
-
-/*********************************************************************
-****	Frame APIs
-**********************************************************************/
-
-/* @brief Initialize the plane depending on the frame type
- *
- * @param
- * @param[in]	frame           The frame attributes to be initialized
- * @return	The error code.
- */
-enum ia_css_err ia_css_frame_init_planes(struct ia_css_frame *frame);
-
-/* @brief Free an array of frames
- *
- * @param
- * @param[in]	num_frames      The number of frames to be freed in the array
- * @param[in]   **frames_array  The array of frames to be removed
- * @return
- */
-void ia_css_frame_free_multiple(unsigned int num_frames,
-	struct ia_css_frame **frames_array);
-
-/* @brief Allocate a CSS frame structure of given size in bytes..
- *
- * @param	frame	The allocated frame.
- * @param[in]	size_bytes	The frame size in bytes.
- * @param[in]	contiguous	Allocate memory physically contiguously or not.
- * @return	The error code.
- *
- * Allocate a frame using the given size in bytes.
- * The frame structure is partially null initialized.
- */
-enum ia_css_err ia_css_frame_allocate_with_buffer_size(
-	struct ia_css_frame **frame,
-	const unsigned int size_bytes,
-	const bool contiguous);
-
-/* @brief Check whether 2 frames are same type
- *
- * @param
- * @param[in]	frame_a         The first frame to be compared
- * @param[in]	frame_b         The second frame to be compared
- * @return      Returns true if the frames are equal
- */
-bool ia_css_frame_is_same_type(
-	const struct ia_css_frame *frame_a,
-	const struct ia_css_frame *frame_b);
-
-/* @brief Configure a dma port from frame info
- *
- * @param
- * @param[in]	config         The DAM port configuration
- * @param[in]	info           The frame info
- * @return
- */
-void ia_css_dma_configure_from_info(
-	struct dma_port_config *config,
-	const struct ia_css_frame_info *info);
-
-#ifdef ISP2401
-/* @brief Finds the cropping resolution
- * This function finds the maximum cropping resolution in an input image keeping
- * the aspect ratio for the given output resolution.Calculates the coordinates
- * for cropping from the center and returns the starting pixel location of the
- * region in the input image. Also returns the dimension of the cropping
- * resolution.
- *
- * @param
- * @param[in]	in_res		Resolution of input image
- * @param[in]	out_res		Resolution of output image
- * @param[out]	crop_res	Crop resolution of input image
- * @return	Returns IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS on error
- */
-enum ia_css_err
-ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res,
-	const struct ia_css_resolution *out_res,
-	struct ia_css_resolution *crop_res);
-
-#endif
-#endif /* __IA_CSS_FRAME_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/interface/ia_css_frame_comm.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/interface/ia_css_frame_comm.h
deleted file mode 100644
index a469e0afb2b5..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/interface/ia_css_frame_comm.h
+++ /dev/null
@@ -1,132 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_FRAME_COMM_H__
-#define __IA_CSS_FRAME_COMM_H__
-
-#include "type_support.h"
-#include "platform_support.h"
-#include "runtime/bufq/interface/ia_css_bufq_comm.h"
-#include <system_types.h>	 /* hrt_vaddress */
-
-/*
- * These structs are derived from structs defined in ia_css_types.h
- * (just take out the "_sp" from the struct name to get the "original")
- * All the fields that are not needed by the SP are removed.
- */
-struct ia_css_frame_sp_plane {
-	unsigned int offset;	/* offset in bytes to start of frame data */
-				/* offset is wrt data in sh_css_sp_sp_frame */
-};
-
-struct ia_css_frame_sp_binary_plane {
-	unsigned int size;
-	struct ia_css_frame_sp_plane data;
-};
-
-struct ia_css_frame_sp_yuv_planes {
-	struct ia_css_frame_sp_plane y;
-	struct ia_css_frame_sp_plane u;
-	struct ia_css_frame_sp_plane v;
-};
-
-struct ia_css_frame_sp_nv_planes {
-	struct ia_css_frame_sp_plane y;
-	struct ia_css_frame_sp_plane uv;
-};
-
-struct ia_css_frame_sp_rgb_planes {
-	struct ia_css_frame_sp_plane r;
-	struct ia_css_frame_sp_plane g;
-	struct ia_css_frame_sp_plane b;
-};
-
-struct ia_css_frame_sp_plane6 {
-	struct ia_css_frame_sp_plane r;
-	struct ia_css_frame_sp_plane r_at_b;
-	struct ia_css_frame_sp_plane gr;
-	struct ia_css_frame_sp_plane gb;
-	struct ia_css_frame_sp_plane b;
-	struct ia_css_frame_sp_plane b_at_r;
-};
-
-struct ia_css_sp_resolution {
-	uint16_t width;		/* width of valid data in pixels */
-	uint16_t height;	/* Height of valid data in lines */
-};
-
-/*
- * Frame info struct. This describes the contents of an image frame buffer.
- */
-struct ia_css_frame_sp_info {
-	struct ia_css_sp_resolution res;
-	uint16_t padded_width;		/* stride of line in memory
-					(in pixels) */
-	unsigned char format;		/* format of the frame data */
-	unsigned char raw_bit_depth;	/* number of valid bits per pixel,
-					only valid for RAW bayer frames */
-	unsigned char raw_bayer_order;	/* bayer order, only valid
-					for RAW bayer frames */
-	unsigned char padding[3];	/* Extend to 32 bit multiple */
-};
-
-struct ia_css_buffer_sp {
-	union {
-		hrt_vaddress xmem_addr;
-		enum sh_css_queue_id queue_id;
-	} buf_src;
-	enum ia_css_buffer_type buf_type;
-};
-
-struct ia_css_frame_sp {
-	struct ia_css_frame_sp_info info;
-	struct ia_css_buffer_sp buf_attr;
-	union {
-		struct ia_css_frame_sp_plane raw;
-		struct ia_css_frame_sp_plane rgb;
-		struct ia_css_frame_sp_rgb_planes planar_rgb;
-		struct ia_css_frame_sp_plane yuyv;
-		struct ia_css_frame_sp_yuv_planes yuv;
-		struct ia_css_frame_sp_nv_planes nv;
-		struct ia_css_frame_sp_plane6 plane6;
-		struct ia_css_frame_sp_binary_plane binary;
-	} planes;
-};
-
-void ia_css_frame_info_to_frame_sp_info(
-	struct ia_css_frame_sp_info *sp_info,
-	const struct ia_css_frame_info *info);
-
-void ia_css_resolution_to_sp_resolution(
-	struct ia_css_sp_resolution *sp_info,
-	const struct ia_css_resolution *info);
-
-#endif /*__IA_CSS_FRAME_COMM_H__*/
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/src/frame.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/src/frame.c
deleted file mode 100644
index 7562beadabef..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/frame/src/frame.c
+++ /dev/null
@@ -1,1026 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "ia_css_frame.h"
-#include <math_support.h>
-#include "assert_support.h"
-#include "ia_css_debug.h"
-#include "isp.h"
-#include "sh_css_internal.h"
-#include "memory_access.h"
-
-
-#define NV12_TILEY_TILE_WIDTH  128
-#define NV12_TILEY_TILE_HEIGHT  32
-
-/**************************************************************************
-**	Static functions declarations
-**************************************************************************/
-static void frame_init_plane(struct ia_css_frame_plane *plane,
-	unsigned int width,
-	unsigned int stride,
-	unsigned int height,
-	unsigned int offset);
-
-static void frame_init_single_plane(struct ia_css_frame *frame,
-	struct ia_css_frame_plane *plane,
-	unsigned int height,
-	unsigned int subpixels_per_line,
-	unsigned int bytes_per_pixel);
-
-static void frame_init_raw_single_plane(
-	struct ia_css_frame *frame,
-	struct ia_css_frame_plane *plane,
-	unsigned int height,
-	unsigned int subpixels_per_line,
-	unsigned int bits_per_pixel);
-
-static void frame_init_mipi_plane(struct ia_css_frame *frame,
-	struct ia_css_frame_plane *plane,
-	unsigned int height,
-	unsigned int subpixels_per_line,
-	unsigned int bytes_per_pixel);
-
-static void frame_init_nv_planes(struct ia_css_frame *frame,
-				 unsigned int horizontal_decimation,
-				 unsigned int vertical_decimation,
-				 unsigned int bytes_per_element);
-
-static void frame_init_yuv_planes(struct ia_css_frame *frame,
-	unsigned int horizontal_decimation,
-	unsigned int vertical_decimation,
-	bool swap_uv,
-	unsigned int bytes_per_element);
-
-static void frame_init_rgb_planes(struct ia_css_frame *frame,
-	unsigned int bytes_per_element);
-
-static void frame_init_qplane6_planes(struct ia_css_frame *frame);
-
-static enum ia_css_err frame_allocate_buffer_data(struct ia_css_frame *frame);
-
-static enum ia_css_err frame_allocate_with_data(struct ia_css_frame **frame,
-	unsigned int width,
-	unsigned int height,
-	enum ia_css_frame_format format,
-	unsigned int padded_width,
-	unsigned int raw_bit_depth,
-	bool contiguous);
-
-static struct ia_css_frame *frame_create(unsigned int width,
-	unsigned int height,
-	enum ia_css_frame_format format,
-	unsigned int padded_width,
-	unsigned int raw_bit_depth,
-	bool contiguous,
-	bool valid);
-
-static unsigned
-ia_css_elems_bytes_from_info(
-	const struct ia_css_frame_info *info);
-
-/**************************************************************************
-**	CSS API functions, exposed by ia_css.h
-**************************************************************************/
-
-void ia_css_frame_zero(struct ia_css_frame *frame)
-{
-	assert(frame != NULL);
-	mmgr_clear(frame->data, frame->data_bytes);
-}
-
-enum ia_css_err ia_css_frame_allocate_from_info(struct ia_css_frame **frame,
-	const struct ia_css_frame_info *info)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	if (frame == NULL || info == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_frame_allocate_from_info() enter:\n");
-	err =
-	    ia_css_frame_allocate(frame, info->res.width, info->res.height,
-				  info->format, info->padded_width,
-				  info->raw_bit_depth);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_frame_allocate_from_info() leave:\n");
-	return err;
-}
-
-enum ia_css_err ia_css_frame_allocate(struct ia_css_frame **frame,
-	unsigned int width,
-	unsigned int height,
-	enum ia_css_frame_format format,
-	unsigned int padded_width,
-	unsigned int raw_bit_depth)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	if (frame == NULL || width == 0 || height == 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-#ifndef ISP2401
-	  "ia_css_frame_allocate() enter: width=%d, height=%d, format=%d\n",
-	  width, height, format);
-#else
-	  "ia_css_frame_allocate() enter: width=%d, height=%d, format=%d, padded_width=%d, raw_bit_depth=%d\n",
-	  width, height, format, padded_width, raw_bit_depth);
-#endif
-
-	err = frame_allocate_with_data(frame, width, height, format,
-				       padded_width, raw_bit_depth, false);
-
-#ifndef ISP2401
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_frame_allocate() leave: frame=%p\n", *frame);
-#else
-	if ((*frame != NULL) && err == IA_CSS_SUCCESS)
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_frame_allocate() leave: frame=%p, data(DDR address)=0x%x\n", *frame, (*frame)->data);
-	else
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_frame_allocate() leave: frame=%p, data(DDR address)=0x%x\n",
-		      (void *)-1, (unsigned int)-1);
-#endif
-
-	return err;
-}
-
-enum ia_css_err ia_css_frame_map(struct ia_css_frame **frame,
-	const struct ia_css_frame_info *info,
-	const void *data,
-	uint16_t attribute,
-	void *context)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_frame *me;
-	assert(frame != NULL);
-
-	/* Create the frame structure */
-	err = ia_css_frame_create_from_info(&me, info);
-
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-	if (err == IA_CSS_SUCCESS) {
-		/* use mmgr_mmap to map */
-		me->data = (ia_css_ptr) mmgr_mmap(data,
-						  me->data_bytes,
-						  attribute, context);
-		if (me->data == mmgr_NULL)
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if (err != IA_CSS_SUCCESS) {
-		sh_css_free(me);
-#ifndef ISP2401
-		return err;
-#else
-		me = NULL;
-#endif
-	}
-
-	*frame = me;
-
-	return err;
-}
-
-enum ia_css_err ia_css_frame_create_from_info(struct ia_css_frame **frame,
-	const struct ia_css_frame_info *info)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_frame *me;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_frame_create_from_info() enter:\n");
-	if (frame == NULL || info == NULL) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			"ia_css_frame_create_from_info() leave:"
-			" invalid arguments\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	me = frame_create(info->res.width,
-		info->res.height,
-		info->format,
-		info->padded_width,
-		info->raw_bit_depth,
-		false,
-		false);
-	if (me == NULL) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			"ia_css_frame_create_from_info() leave:"
-			" frame create failed\n");
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	}
-
-	err = ia_css_frame_init_planes(me);
-
-#ifndef ISP2401
-	if (err == IA_CSS_SUCCESS)
-		*frame = me;
-	else
-#else
-	if (err != IA_CSS_SUCCESS) {
-#endif
-		sh_css_free(me);
-#ifdef ISP2401
-		me = NULL;
-	}
-
-	*frame = me;
-#endif
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_frame_create_from_info() leave:\n");
-
-	return err;
-}
-
-enum ia_css_err ia_css_frame_set_data(struct ia_css_frame *frame,
-	const ia_css_ptr mapped_data,
-	size_t data_bytes)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_frame_set_data() enter:\n");
-	if (frame == NULL) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			"ia_css_frame_set_data() leave: NULL frame\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	/* If we are setting a valid data.
-	 * Make sure that there is enough
-	 * room for the expected frame format
-	 */
-	if ((mapped_data != mmgr_NULL) && (frame->data_bytes > data_bytes)) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			"ia_css_frame_set_data() leave: invalid arguments\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	frame->data = mapped_data;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_frame_set_data() leave:\n");
-
-	return err;
-}
-
-enum ia_css_err ia_css_frame_allocate_contiguous(struct ia_css_frame **frame,
-	unsigned int width,
-	unsigned int height,
-	enum ia_css_frame_format format,
-	unsigned int padded_width,
-	unsigned int raw_bit_depth)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_frame_allocate_contiguous() "
-#ifndef ISP2401
-		"enter: width=%d, height=%d, format=%d\n",
-		width, height, format);
-#else
-		"enter: width=%d, height=%d, format=%d, padded_width=%d, raw_bit_depth=%d\n",
-		width, height, format, padded_width, raw_bit_depth);
-#endif
-
-	err = frame_allocate_with_data(frame, width, height, format,
-			padded_width, raw_bit_depth, true);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_frame_allocate_contiguous() leave: frame=%p\n",
-		frame ? *frame : (void *)-1);
-
-	return err;
-}
-
-enum ia_css_err ia_css_frame_allocate_contiguous_from_info(
-	struct ia_css_frame **frame,
-	const struct ia_css_frame_info *info)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	assert(frame != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_frame_allocate_contiguous_from_info() enter:\n");
-	err = ia_css_frame_allocate_contiguous(frame,
-						info->res.width,
-						info->res.height,
-						info->format,
-						info->padded_width,
-						info->raw_bit_depth);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_frame_allocate_contiguous_from_info() leave:\n");
-	return err;
-}
-
-void ia_css_frame_free(struct ia_css_frame *frame)
-{
-	IA_CSS_ENTER_PRIVATE("frame = %p", frame);
-
-	if (frame != NULL) {
-		hmm_free(frame->data);
-		sh_css_free(frame);
-	}
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-/**************************************************************************
-**	Module public functions
-**************************************************************************/
-
-enum ia_css_err ia_css_frame_check_info(const struct ia_css_frame_info *info)
-{
-	assert(info != NULL);
-	if (info->res.width == 0 || info->res.height == 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err ia_css_frame_init_planes(struct ia_css_frame *frame)
-{
-	assert(frame != NULL);
-
-	switch (frame->info.format) {
-	case IA_CSS_FRAME_FORMAT_MIPI:
-		frame_init_mipi_plane(frame, &frame->planes.raw,
-			frame->info.res.height,
-			frame->info.padded_width,
-			frame->info.raw_bit_depth <= 8 ? 1 : 2);
-		break;
-	case IA_CSS_FRAME_FORMAT_RAW_PACKED:
-		frame_init_raw_single_plane(frame, &frame->planes.raw,
-			frame->info.res.height,
-			frame->info.padded_width,
-			frame->info.raw_bit_depth);
-		break;
-	case IA_CSS_FRAME_FORMAT_RAW:
-		frame_init_single_plane(frame, &frame->planes.raw,
-			frame->info.res.height,
-			frame->info.padded_width,
-			frame->info.raw_bit_depth <= 8 ? 1 : 2);
-		break;
-	case IA_CSS_FRAME_FORMAT_RGB565:
-		frame_init_single_plane(frame, &frame->planes.rgb,
-			frame->info.res.height,
-			frame->info.padded_width, 2);
-		break;
-	case IA_CSS_FRAME_FORMAT_RGBA888:
-		frame_init_single_plane(frame, &frame->planes.rgb,
-			frame->info.res.height,
-			frame->info.padded_width * 4, 1);
-		break;
-	case IA_CSS_FRAME_FORMAT_PLANAR_RGB888:
-		frame_init_rgb_planes(frame, 1);
-		break;
-		/* yuyv and uyvu have the same frame layout, only the data
-		 * positioning differs.
-		 */
-	case IA_CSS_FRAME_FORMAT_YUYV:
-	case IA_CSS_FRAME_FORMAT_UYVY:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_8:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_LEGACY_YUV420_8:
-		frame_init_single_plane(frame, &frame->planes.yuyv,
-			frame->info.res.height,
-			frame->info.padded_width * 2, 1);
-		break;
-	case IA_CSS_FRAME_FORMAT_YUV_LINE:
-		/* Needs 3 extra lines to allow vf_pp prefetching */
-		frame_init_single_plane(frame, &frame->planes.yuyv,
-			frame->info.res.height * 3 / 2 + 3,
-			frame->info.padded_width, 1);
-		break;
-	case IA_CSS_FRAME_FORMAT_NV11:
-	  frame_init_nv_planes(frame, 4, 1, 1);
-		break;
-		/* nv12 and nv21 have the same frame layout, only the data
-		 * positioning differs.
-		 */
-	case IA_CSS_FRAME_FORMAT_NV12:
-	case IA_CSS_FRAME_FORMAT_NV21:
-	case IA_CSS_FRAME_FORMAT_NV12_TILEY:
-		frame_init_nv_planes(frame, 2, 2, 1);
-		break;
-	case IA_CSS_FRAME_FORMAT_NV12_16:
-		frame_init_nv_planes(frame, 2, 2, 2);
-		break;
-		/* nv16 and nv61 have the same frame layout, only the data
-		 * positioning differs.
-		 */
-	case IA_CSS_FRAME_FORMAT_NV16:
-	case IA_CSS_FRAME_FORMAT_NV61:
-		frame_init_nv_planes(frame, 2, 1, 1);
-		break;
-	case IA_CSS_FRAME_FORMAT_YUV420:
-		frame_init_yuv_planes(frame, 2, 2, false, 1);
-		break;
-	case IA_CSS_FRAME_FORMAT_YUV422:
-		frame_init_yuv_planes(frame, 2, 1, false, 1);
-		break;
-	case IA_CSS_FRAME_FORMAT_YUV444:
-		frame_init_yuv_planes(frame, 1, 1, false, 1);
-		break;
-	case IA_CSS_FRAME_FORMAT_YUV420_16:
-		frame_init_yuv_planes(frame, 2, 2, false, 2);
-		break;
-	case IA_CSS_FRAME_FORMAT_YUV422_16:
-		frame_init_yuv_planes(frame, 2, 1, false, 2);
-		break;
-	case IA_CSS_FRAME_FORMAT_YV12:
-		frame_init_yuv_planes(frame, 2, 2, true, 1);
-		break;
-	case IA_CSS_FRAME_FORMAT_YV16:
-		frame_init_yuv_planes(frame, 2, 1, true, 1);
-		break;
-	case IA_CSS_FRAME_FORMAT_QPLANE6:
-		frame_init_qplane6_planes(frame);
-		break;
-	case IA_CSS_FRAME_FORMAT_BINARY_8:
-		frame_init_single_plane(frame, &frame->planes.binary.data,
-			frame->info.res.height,
-			frame->info.padded_width, 1);
-		frame->planes.binary.size = 0;
-		break;
-	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	return IA_CSS_SUCCESS;
-}
-
-void ia_css_frame_info_set_width(struct ia_css_frame_info *info,
-	unsigned int width,
-	unsigned int min_padded_width)
-{
-	unsigned int align;
-
-	IA_CSS_ENTER_PRIVATE("info = %p,width = %d, minimum padded width = %d",
-			     info, width, min_padded_width);
-	if (info == NULL) {
-		IA_CSS_ERROR("NULL input parameter");
-		IA_CSS_LEAVE_PRIVATE("");
-		return;
-	}
-	if (min_padded_width > width)
-		align = min_padded_width;
-	else
-		align = width;
-
-	info->res.width = width;
-	/* frames with a U and V plane of 8 bits per pixel need to have
-	   all planes aligned, this means double the alignment for the
-	   Y plane if the horizontal decimation is 2. */
-	if (info->format == IA_CSS_FRAME_FORMAT_YUV420 ||
-	    info->format == IA_CSS_FRAME_FORMAT_YV12 ||
-	    info->format == IA_CSS_FRAME_FORMAT_NV12 ||
-	    info->format == IA_CSS_FRAME_FORMAT_NV21 ||
-	    info->format == IA_CSS_FRAME_FORMAT_BINARY_8 ||
-	    info->format == IA_CSS_FRAME_FORMAT_YUV_LINE)
-		info->padded_width =
-		    CEIL_MUL(align, 2 * HIVE_ISP_DDR_WORD_BYTES);
-	else if (info->format == IA_CSS_FRAME_FORMAT_NV12_TILEY)
-		info->padded_width = CEIL_MUL(align, NV12_TILEY_TILE_WIDTH);
-	else if (info->format == IA_CSS_FRAME_FORMAT_RAW ||
-		 info->format == IA_CSS_FRAME_FORMAT_RAW_PACKED)
-		info->padded_width = CEIL_MUL(align, 2 * ISP_VEC_NELEMS);
-	else {
-		info->padded_width = CEIL_MUL(align, HIVE_ISP_DDR_WORD_BYTES);
-	}
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void ia_css_frame_info_set_format(struct ia_css_frame_info *info,
-	enum ia_css_frame_format format)
-{
-	assert(info != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_frame_info_set_format() enter:\n");
-	info->format = format;
-}
-
-void ia_css_frame_info_init(struct ia_css_frame_info *info,
-	unsigned int width,
-	unsigned int height,
-	enum ia_css_frame_format format,
-	unsigned int aligned)
-{
-	IA_CSS_ENTER_PRIVATE("info = %p, width = %d, height = %d, format = %d, aligned = %d",
-			     info, width, height, format, aligned);
-	if (info == NULL) {
-		IA_CSS_ERROR("NULL input parameter");
-		IA_CSS_LEAVE_PRIVATE("");
-		return;
-	}
-	info->res.height = height;
-	info->format     = format;
-	ia_css_frame_info_set_width(info, width, aligned);
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-void ia_css_frame_free_multiple(unsigned int num_frames,
-	struct ia_css_frame **frames_array)
-{
-	unsigned int i;
-	for (i = 0; i < num_frames; i++) {
-		if (frames_array[i]) {
-			ia_css_frame_free(frames_array[i]);
-			frames_array[i] = NULL;
-		}
-	}
-}
-
-enum ia_css_err ia_css_frame_allocate_with_buffer_size(
-	struct ia_css_frame **frame,
-	const unsigned int buffer_size_bytes,
-	const bool contiguous)
-{
-	/* AM: Body coppied from frame_allocate_with_data(). */
-	enum ia_css_err err;
-	struct ia_css_frame *me = frame_create(0, 0,
-		IA_CSS_FRAME_FORMAT_NUM,/* Not valid format yet */
-		0, 0, contiguous, false);
-
-	if (me == NULL)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-
-	/* Get the data size */
-	me->data_bytes = buffer_size_bytes;
-
-	err = frame_allocate_buffer_data(me);
-
-	if (err != IA_CSS_SUCCESS) {
-		sh_css_free(me);
-#ifndef ISP2401
-		return err;
-#else
-		me = NULL;
-#endif
-	}
-
-	*frame = me;
-
-	return err;
-}
-
-bool ia_css_frame_info_is_same_resolution(
-	const struct ia_css_frame_info *info_a,
-	const struct ia_css_frame_info *info_b)
-{
-	if (!info_a || !info_b)
-		return false;
-	return (info_a->res.width == info_b->res.width) &&
-	    (info_a->res.height == info_b->res.height);
-}
-
-bool ia_css_frame_is_same_type(const struct ia_css_frame *frame_a,
-	const struct ia_css_frame *frame_b)
-{
-	bool is_equal = false;
-	const struct ia_css_frame_info *info_a = &frame_a->info,
-	    *info_b = &frame_b->info;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_frame_is_same_type() enter:\n");
-
-	if (!info_a || !info_b)
-		return false;
-	if (info_a->format != info_b->format)
-		return false;
-	if (info_a->padded_width != info_b->padded_width)
-		return false;
-	is_equal = ia_css_frame_info_is_same_resolution(info_a, info_b);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_frame_is_same_type() leave:\n");
-
-	return is_equal;
-}
-
-void
-ia_css_dma_configure_from_info(
-	struct dma_port_config *config,
-	const struct ia_css_frame_info *info)
-{
-	unsigned is_raw_packed = info->format == IA_CSS_FRAME_FORMAT_RAW_PACKED;
-	unsigned bits_per_pixel = is_raw_packed ? info->raw_bit_depth : ia_css_elems_bytes_from_info(info)*8;
-	unsigned pix_per_ddrword = HIVE_ISP_DDR_WORD_BITS / bits_per_pixel;
-	unsigned words_per_line = CEIL_DIV(info->padded_width, pix_per_ddrword);
-	unsigned elems_b = pix_per_ddrword;
-
-	config->stride = HIVE_ISP_DDR_WORD_BYTES * words_per_line;
-	config->elems  = (uint8_t)elems_b;
-	config->width  = (uint16_t)info->res.width;
-	config->crop   = 0;
-	assert(config->width <= info->padded_width);
-}
-
-/**************************************************************************
-**	Static functions
-**************************************************************************/
-
-static void frame_init_plane(struct ia_css_frame_plane *plane,
-	unsigned int width,
-	unsigned int stride,
-	unsigned int height,
-	unsigned int offset)
-{
-	plane->height = height;
-	plane->width = width;
-	plane->stride = stride;
-	plane->offset = offset;
-}
-
-static void frame_init_single_plane(struct ia_css_frame *frame,
-	struct ia_css_frame_plane *plane,
-	unsigned int height,
-	unsigned int subpixels_per_line,
-	unsigned int bytes_per_pixel)
-{
-	unsigned int stride;
-
-	stride = subpixels_per_line * bytes_per_pixel;
-	/* Frame height needs to be even number - needed by hw ISYS2401
-	   In case of odd number, round up to even.
-	   Images won't be impacted by this round up,
-	   only needed by jpeg/embedded data.
-	   As long as buffer allocation and release are using data_bytes,
-	   there won't be memory leak. */
-	frame->data_bytes = stride * CEIL_MUL2(height, 2);
-	frame_init_plane(plane, subpixels_per_line, stride, height, 0);
-	return;
-}
-
-static void frame_init_raw_single_plane(
-	struct ia_css_frame *frame,
-	struct ia_css_frame_plane *plane,
-	unsigned int height,
-	unsigned int subpixels_per_line,
-	unsigned int bits_per_pixel)
-{
-	unsigned int stride;
-	assert(frame != NULL);
-
-	stride = HIVE_ISP_DDR_WORD_BYTES *
-			CEIL_DIV(subpixels_per_line,
-				HIVE_ISP_DDR_WORD_BITS / bits_per_pixel);
-	frame->data_bytes = stride * height;
-	frame_init_plane(plane, subpixels_per_line, stride, height, 0);
-	return;
-}
-
-static void frame_init_mipi_plane(struct ia_css_frame *frame,
-	struct ia_css_frame_plane *plane,
-	unsigned int height,
-	unsigned int subpixels_per_line,
-	unsigned int bytes_per_pixel)
-{
-	unsigned int stride;
-
-	stride = subpixels_per_line * bytes_per_pixel;
-	frame->data_bytes = 8388608; /* 8*1024*1024 */
-	frame->valid = false;
-	frame->contiguous = true;
-	frame_init_plane(plane, subpixels_per_line, stride, height, 0);
-	return;
-}
-
-static void frame_init_nv_planes(struct ia_css_frame *frame,
-				 unsigned int horizontal_decimation,
-				 unsigned int vertical_decimation,
-				 unsigned int bytes_per_element)
-{
-	unsigned int y_width = frame->info.padded_width;
-	unsigned int y_height = frame->info.res.height;
-	unsigned int uv_width;
-	unsigned int uv_height;
-	unsigned int y_bytes;
-	unsigned int uv_bytes;
-	unsigned int y_stride;
-	unsigned int uv_stride;
-
-	assert(horizontal_decimation != 0 && vertical_decimation != 0);
-
-	uv_width = 2 * (y_width / horizontal_decimation);
-	uv_height = y_height / vertical_decimation;
-
-	if (IA_CSS_FRAME_FORMAT_NV12_TILEY == frame->info.format) {
-		y_width   = CEIL_MUL(y_width,   NV12_TILEY_TILE_WIDTH);
-		uv_width  = CEIL_MUL(uv_width,  NV12_TILEY_TILE_WIDTH);
-		y_height  = CEIL_MUL(y_height,  NV12_TILEY_TILE_HEIGHT);
-		uv_height = CEIL_MUL(uv_height, NV12_TILEY_TILE_HEIGHT);
-	}
-
-	y_stride = y_width * bytes_per_element;
-	uv_stride = uv_width * bytes_per_element;
-	y_bytes = y_stride * y_height;
-	uv_bytes = uv_stride * uv_height;
-
-	frame->data_bytes = y_bytes + uv_bytes;
-	frame_init_plane(&frame->planes.nv.y, y_width, y_stride, y_height, 0);
-	frame_init_plane(&frame->planes.nv.uv, uv_width,
-			 uv_stride, uv_height, y_bytes);
-	return;
-}
-
-static void frame_init_yuv_planes(struct ia_css_frame *frame,
-	unsigned int horizontal_decimation,
-	unsigned int vertical_decimation,
-	bool swap_uv,
-	unsigned int bytes_per_element)
-{
-	unsigned int y_width = frame->info.padded_width,
-	    y_height = frame->info.res.height,
-	    uv_width = y_width / horizontal_decimation,
-	    uv_height = y_height / vertical_decimation,
-	    y_stride, y_bytes, uv_bytes, uv_stride;
-
-	y_stride = y_width * bytes_per_element;
-	uv_stride = uv_width * bytes_per_element;
-	y_bytes = y_stride * y_height;
-	uv_bytes = uv_stride * uv_height;
-
-	frame->data_bytes = y_bytes + 2 * uv_bytes;
-	frame_init_plane(&frame->planes.yuv.y, y_width, y_stride, y_height, 0);
-	if (swap_uv) {
-		frame_init_plane(&frame->planes.yuv.v, uv_width, uv_stride,
-				 uv_height, y_bytes);
-		frame_init_plane(&frame->planes.yuv.u, uv_width, uv_stride,
-				 uv_height, y_bytes + uv_bytes);
-	} else {
-		frame_init_plane(&frame->planes.yuv.u, uv_width, uv_stride,
-				 uv_height, y_bytes);
-		frame_init_plane(&frame->planes.yuv.v, uv_width, uv_stride,
-				 uv_height, y_bytes + uv_bytes);
-	}
-	return;
-}
-
-static void frame_init_rgb_planes(struct ia_css_frame *frame,
-	unsigned int bytes_per_element)
-{
-	unsigned int width = frame->info.res.width,
-	    height = frame->info.res.height, stride, bytes;
-
-	stride = width * bytes_per_element;
-	bytes = stride * height;
-	frame->data_bytes = 3 * bytes;
-	frame_init_plane(&frame->planes.planar_rgb.r, width, stride, height, 0);
-	frame_init_plane(&frame->planes.planar_rgb.g,
-			 width, stride, height, 1 * bytes);
-	frame_init_plane(&frame->planes.planar_rgb.b,
-			 width, stride, height, 2 * bytes);
-	return;
-}
-
-static void frame_init_qplane6_planes(struct ia_css_frame *frame)
-{
-	unsigned int width = frame->info.padded_width / 2,
-	    height = frame->info.res.height / 2, bytes, stride;
-
-	stride = width * 2;
-	bytes = stride * height;
-
-	frame->data_bytes = 6 * bytes;
-	frame_init_plane(&frame->planes.plane6.r,
-			 width, stride, height, 0 * bytes);
-	frame_init_plane(&frame->planes.plane6.r_at_b,
-			 width, stride, height, 1 * bytes);
-	frame_init_plane(&frame->planes.plane6.gr,
-			 width, stride, height, 2 * bytes);
-	frame_init_plane(&frame->planes.plane6.gb,
-			 width, stride, height, 3 * bytes);
-	frame_init_plane(&frame->planes.plane6.b,
-			 width, stride, height, 4 * bytes);
-	frame_init_plane(&frame->planes.plane6.b_at_r,
-			 width, stride, height, 5 * bytes);
-	return;
-}
-
-static enum ia_css_err frame_allocate_buffer_data(struct ia_css_frame *frame)
-{
-#ifdef ISP2401
-	IA_CSS_ENTER_LEAVE_PRIVATE("frame->data_bytes=%d\n", frame->data_bytes);
-#endif
-	frame->data = mmgr_alloc_attr(frame->data_bytes,
-				      frame->contiguous ?
-				      MMGR_ATTRIBUTE_CONTIGUOUS :
-				      MMGR_ATTRIBUTE_DEFAULT);
-
-	if (frame->data == mmgr_NULL)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err frame_allocate_with_data(struct ia_css_frame **frame,
-	unsigned int width,
-	unsigned int height,
-	enum ia_css_frame_format format,
-	unsigned int padded_width,
-	unsigned int raw_bit_depth,
-	bool contiguous)
-{
-	enum ia_css_err err;
-	struct ia_css_frame *me = frame_create(width,
-		height,
-		format,
-		padded_width,
-		raw_bit_depth,
-		contiguous,
-		true);
-
-	if (me == NULL)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-
-	err = ia_css_frame_init_planes(me);
-
-	if (err == IA_CSS_SUCCESS)
-		err = frame_allocate_buffer_data(me);
-
-	if (err != IA_CSS_SUCCESS) {
-		sh_css_free(me);
-#ifndef ISP2401
-		return err;
-#else
-		me = NULL;
-#endif
-	}
-
-	*frame = me;
-
-	return err;
-}
-
-static struct ia_css_frame *frame_create(unsigned int width,
-	unsigned int height,
-	enum ia_css_frame_format format,
-	unsigned int padded_width,
-	unsigned int raw_bit_depth,
-	bool contiguous,
-	bool valid)
-{
-	struct ia_css_frame *me = sh_css_malloc(sizeof(*me));
-
-	if (me == NULL)
-		return NULL;
-
-	memset(me, 0, sizeof(*me));
-	me->info.res.width = width;
-	me->info.res.height = height;
-	me->info.format = format;
-	me->info.padded_width = padded_width;
-	me->info.raw_bit_depth = raw_bit_depth;
-	me->contiguous = contiguous;
-	me->valid = valid;
-	me->data_bytes = 0;
-	me->data = mmgr_NULL;
-	/* To indicate it is not valid frame. */
-	me->dynamic_queue_id = (int)SH_CSS_INVALID_QUEUE_ID;
-	me->buf_type = IA_CSS_BUFFER_TYPE_INVALID;
-
-	return me;
-}
-
-static unsigned
-ia_css_elems_bytes_from_info(const struct ia_css_frame_info *info)
-{
-	if (info->format == IA_CSS_FRAME_FORMAT_RGB565)
-		return 2; /* bytes per pixel */
-	if (info->format == IA_CSS_FRAME_FORMAT_YUV420_16)
-		return 2; /* bytes per pixel */
-	if (info->format == IA_CSS_FRAME_FORMAT_YUV422_16)
-		return 2; /* bytes per pixel */
-	/* Note: Essentially NV12_16 is a 2 bytes per pixel format, this return value is used
-	 * to configure DMA for the output buffer,
-	 * At least in SKC this data is overwriten by isp_output_init.sp.c except for elements(elems),
-	 * which is configured from this return value,
-	 * NV12_16 is implemented by a double buffer of 8 bit elements hence elems should be configured as 8 */
-	if (info->format == IA_CSS_FRAME_FORMAT_NV12_16)
-		return 1; /* bytes per pixel */
-
-	if (info->format == IA_CSS_FRAME_FORMAT_RAW
-		|| (info->format == IA_CSS_FRAME_FORMAT_RAW_PACKED)) {
-		if (info->raw_bit_depth)
-			return CEIL_DIV(info->raw_bit_depth,8);
-		else
-			return 2; /* bytes per pixel */
-	}
-	if (info->format == IA_CSS_FRAME_FORMAT_PLANAR_RGB888)
-		return 3; /* bytes per pixel */
-	if (info->format == IA_CSS_FRAME_FORMAT_RGBA888)
-		return 4; /* bytes per pixel */
-	if (info->format == IA_CSS_FRAME_FORMAT_QPLANE6)
-		return 2; /* bytes per pixel */
-	return 1; /* Default is 1 byte per pixel */
-}
-
-void ia_css_frame_info_to_frame_sp_info(
-	struct ia_css_frame_sp_info *to,
-	const struct ia_css_frame_info *from)
-{
-	ia_css_resolution_to_sp_resolution(&to->res, &from->res);
-	to->padded_width = (uint16_t)from->padded_width;
-	to->format = (uint8_t)from->format;
-	to->raw_bit_depth = (uint8_t)from->raw_bit_depth;
-	to->raw_bayer_order = from->raw_bayer_order;
-}
-
-void ia_css_resolution_to_sp_resolution(
-	struct ia_css_sp_resolution *to,
-	const struct ia_css_resolution *from)
-{
-	to->width  = (uint16_t)from->width;
-	to->height = (uint16_t)from->height;
-}
-#ifdef ISP2401
-
-enum ia_css_err
-ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res,
-	const struct ia_css_resolution *out_res,
-	struct ia_css_resolution *crop_res)
-{
-	uint32_t wd_even_ceil, ht_even_ceil;
-	uint32_t in_ratio, out_ratio;
-
-	if ((in_res == NULL) || (out_res == NULL) || (crop_res == NULL))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	IA_CSS_ENTER_PRIVATE("in(%ux%u) -> out(%ux%u)", in_res->width,
-		in_res->height, out_res->width, out_res->height);
-
-	if ((in_res->width == 0)
-		|| (in_res->height == 0)
-		|| (out_res->width == 0)
-		|| (out_res->height == 0))
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	if ((out_res->width > in_res->width) ||
-		 (out_res->height > in_res->height))
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	/* If aspect ratio (width/height) of out_res is higher than the aspect
-	 * ratio of the in_res, then we crop vertically, otherwise we crop
-	 * horizontally.
-	 */
-	in_ratio = in_res->width * out_res->height;
-	out_ratio = out_res->width * in_res->height;
-
-	if (in_ratio == out_ratio) {
-		crop_res->width = in_res->width;
-		crop_res->height = in_res->height;
-	} else if (out_ratio > in_ratio) {
-		crop_res->width = in_res->width;
-		crop_res->height = ROUND_DIV(out_res->height * crop_res->width,
-			out_res->width);
-	} else {
-		crop_res->height = in_res->height;
-		crop_res->width = ROUND_DIV(out_res->width * crop_res->height,
-			out_res->height);
-	}
-
-	/* Round new (cropped) width and height to an even number.
-	 * binarydesc_calculate_bds_factor is such that we should consider as
-	 * much of the input as possible. This is different only when we end up
-	 * with an odd number in the last step. So, we take the next even number
-	 * if it falls within the input, otherwise take the previous even no.
-	 */
-	wd_even_ceil = EVEN_CEIL(crop_res->width);
-	ht_even_ceil = EVEN_CEIL(crop_res->height);
-	if ((wd_even_ceil > in_res->width) || (ht_even_ceil > in_res->height)) {
-		crop_res->width = EVEN_FLOOR(crop_res->width);
-		crop_res->height = EVEN_FLOOR(crop_res->height);
-	} else {
-		crop_res->width = wd_even_ceil;
-		crop_res->height = ht_even_ceil;
-	}
-
-	IA_CSS_LEAVE_PRIVATE("in(%ux%u) -> out(%ux%u)", crop_res->width,
-		crop_res->height, out_res->width, out_res->height);
-	return IA_CSS_SUCCESS;
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/ifmtr/interface/ia_css_ifmtr.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/ifmtr/interface/ia_css_ifmtr.h
deleted file mode 100644
index d02bff1bbf46..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/ifmtr/interface/ia_css_ifmtr.h
+++ /dev/null
@@ -1,49 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_IFMTR_H__
-#define __IA_CSS_IFMTR_H__
-
-#include <type_support.h>
-#include <ia_css_stream_public.h>
-#include <ia_css_binary.h>
-
-extern bool ifmtr_set_if_blocking_mode_reset;
-
-unsigned int ia_css_ifmtr_lines_needed_for_bayer_order(
-			const struct ia_css_stream_config *config);
-
-unsigned int ia_css_ifmtr_columns_needed_for_bayer_order(
-			const struct ia_css_stream_config *config);
-
-enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config,
-				       struct ia_css_binary *binary);
-
-#endif /* __IA_CSS_IFMTR_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/ifmtr/src/ifmtr.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/ifmtr/src/ifmtr.c
deleted file mode 100644
index 1bed027435fd..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/ifmtr/src/ifmtr.c
+++ /dev/null
@@ -1,569 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "system_global.h"
-#include <linux/kernel.h>
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2
-
-#include "ia_css_ifmtr.h"
-#include <math_support.h>
-#include "sh_css_internal.h"
-#include "input_formatter.h"
-#include "assert_support.h"
-#include "sh_css_sp.h"
-#include "isp/modes/interface/input_buf.isp.h"
-
-/************************************************************
- * Static functions declarations
- ************************************************************/
-static enum ia_css_err ifmtr_start_column(
-		const struct ia_css_stream_config *config,
-		unsigned int bin_in,
-		unsigned int *start_column);
-
-static enum ia_css_err ifmtr_input_start_line(
-		const struct ia_css_stream_config *config,
-		unsigned int bin_in,
-		unsigned int *start_line);
-
-static void ifmtr_set_if_blocking_mode(
-		const input_formatter_cfg_t * const config_a,
-		const input_formatter_cfg_t * const config_b);
-
-/************************************************************
- * Public functions
- ************************************************************/
-
-/* ISP expects GRBG bayer order, we skip one line and/or one row
- * to correct in case the input bayer order is different.
- */
-unsigned int ia_css_ifmtr_lines_needed_for_bayer_order(
-		const struct ia_css_stream_config *config)
-{
-	assert(config != NULL);
-	if ((IA_CSS_BAYER_ORDER_BGGR == config->input_config.bayer_order)
-	    || (IA_CSS_BAYER_ORDER_GBRG == config->input_config.bayer_order))
-		return 1;
-
-	return 0;
-}
-
-unsigned int ia_css_ifmtr_columns_needed_for_bayer_order(
-		const struct ia_css_stream_config *config)
-{
-	assert(config != NULL);
-	if ((IA_CSS_BAYER_ORDER_RGGB == config->input_config.bayer_order)
-	    || (IA_CSS_BAYER_ORDER_GBRG == config->input_config.bayer_order))
-		return 1;
-
-	return 0;
-}
-
-enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config,
-				       struct ia_css_binary *binary)
-{
-	unsigned int start_line, start_column = 0,
-	    cropped_height,
-	    cropped_width,
-	    num_vectors,
-	    buffer_height = 2,
-	    buffer_width,
-	    two_ppc,
-	    vmem_increment = 0,
-	    deinterleaving = 0,
-	    deinterleaving_b = 0,
-	    width_a = 0,
-	    width_b = 0,
-	    bits_per_pixel,
-	    vectors_per_buffer,
-	    vectors_per_line = 0,
-	    buffers_per_line = 0,
-	    buf_offset_a = 0,
-	    buf_offset_b = 0,
-	    line_width = 0,
-	    width_b_factor = 1, start_column_b,
-	    left_padding = 0;
-	input_formatter_cfg_t if_a_config, if_b_config;
-	enum atomisp_input_format input_format;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	uint8_t if_config_index;
-
-	/* Determine which input formatter config set is targeted. */
-	/* Index is equal to the CSI-2 port used. */
-	enum mipi_port_id port;
-
-	if (binary) {
-		cropped_height = binary->in_frame_info.res.height;
-		cropped_width = binary->in_frame_info.res.width;
-		/* This should correspond to the input buffer definition for
-		ISP binaries in input_buf.isp.h */
-		if (binary->info->sp.enable.continuous && binary->info->sp.pipeline.mode != IA_CSS_BINARY_MODE_COPY)
-			buffer_width = MAX_VECTORS_PER_INPUT_LINE_CONT * ISP_VEC_NELEMS;
-		else
-			buffer_width = binary->info->sp.input.max_width;
-		input_format = binary->input_format;
-	} else {
-		/* sp raw copy pipe (IA_CSS_PIPE_MODE_COPY): binary is NULL */
-		cropped_height = config->input_config.input_res.height;
-		cropped_width = config->input_config.input_res.width;
-		buffer_width = MAX_VECTORS_PER_INPUT_LINE_CONT * ISP_VEC_NELEMS;
-		input_format = config->input_config.format;
-	}
-	two_ppc = config->pixels_per_clock == 2;
-	if (config->mode == IA_CSS_INPUT_MODE_SENSOR
-	    || config->mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) {
-		port = config->source.port.port;
-		if_config_index = (uint8_t) (port - MIPI_PORT0_ID);
-	} else if (config->mode == IA_CSS_INPUT_MODE_MEMORY) {
-		if_config_index = SH_CSS_IF_CONFIG_NOT_NEEDED;
-	} else {
-		if_config_index = 0;
-	}
-
-	assert(if_config_index <= SH_CSS_MAX_IF_CONFIGS
-	       || if_config_index == SH_CSS_IF_CONFIG_NOT_NEEDED);
-
-	/* TODO: check to see if input is RAW and if current mode interprets
-	 * RAW data in any particular bayer order. copy binary with output
-	 * format other than raw should not result in dropping lines and/or
-	 * columns.
-	 */
-	err = ifmtr_input_start_line(config, cropped_height, &start_line);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	err = ifmtr_start_column(config, cropped_width, &start_column);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-	if (config->left_padding == -1)
-		if (!binary)
-			/* sp raw copy pipe: set left_padding value */
-			left_padding = 0;
-		else
-			left_padding = binary->left_padding;
-	else
-		left_padding = 2*ISP_VEC_NELEMS - config->left_padding;
-
-
-	if (left_padding) {
-		num_vectors = CEIL_DIV(cropped_width + left_padding,
-				       ISP_VEC_NELEMS);
-	} else {
-		num_vectors = CEIL_DIV(cropped_width, ISP_VEC_NELEMS);
-		num_vectors *= buffer_height;
-		/* todo: in case of left padding,
-		   num_vectors is vectors per line,
-		   otherwise vectors per line * buffer_height. */
-	}
-
-	start_column_b = start_column;
-
-	bits_per_pixel = input_formatter_get_alignment(INPUT_FORMATTER0_ID)
-	    * 8 / ISP_VEC_NELEMS;
-	switch (input_format) {
-	case ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY:
-		if (two_ppc) {
-			vmem_increment = 1;
-			deinterleaving = 1;
-			deinterleaving_b = 1;
-			/* half lines */
-			width_a = cropped_width * deinterleaving / 2;
-			width_b_factor = 2;
-			/* full lines */
-			width_b = width_a * width_b_factor;
-			buffer_width *= deinterleaving * 2;
-			/* Patch from bayer to yuv */
-			num_vectors *= deinterleaving;
-			buf_offset_b = buffer_width / 2 / ISP_VEC_NELEMS;
-			vectors_per_line = num_vectors / buffer_height;
-			/* Even lines are half size */
-			line_width = vectors_per_line *
-			    input_formatter_get_alignment(INPUT_FORMATTER0_ID) /
-			    2;
-			start_column /= 2;
-		} else {
-			vmem_increment = 1;
-			deinterleaving = 3;
-			width_a = cropped_width * deinterleaving / 2;
-			buffer_width = buffer_width * deinterleaving / 2;
-			/* Patch from bayer to yuv */
-			num_vectors = num_vectors / 2 * deinterleaving;
-			start_column = start_column * deinterleaving / 2;
-		}
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_8:
-	case ATOMISP_INPUT_FORMAT_YUV420_10:
-	case ATOMISP_INPUT_FORMAT_YUV420_16:
-		if (two_ppc) {
-			vmem_increment = 1;
-			deinterleaving = 1;
-			width_a = width_b = cropped_width * deinterleaving / 2;
-			buffer_width *= deinterleaving * 2;
-			num_vectors *= deinterleaving;
-			buf_offset_b = buffer_width / 2 / ISP_VEC_NELEMS;
-			vectors_per_line = num_vectors / buffer_height;
-			/* Even lines are half size */
-			line_width = vectors_per_line *
-			    input_formatter_get_alignment(INPUT_FORMATTER0_ID) /
-			    2;
-			start_column *= deinterleaving;
-			start_column /= 2;
-			start_column_b = start_column;
-		} else {
-			vmem_increment = 1;
-			deinterleaving = 1;
-			width_a = cropped_width * deinterleaving;
-			buffer_width *= deinterleaving * 2;
-			num_vectors *= deinterleaving;
-			start_column *= deinterleaving;
-		}
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV422_8:
-	case ATOMISP_INPUT_FORMAT_YUV422_10:
-	case ATOMISP_INPUT_FORMAT_YUV422_16:
-		if (two_ppc) {
-			vmem_increment = 1;
-			deinterleaving = 1;
-			width_a = width_b = cropped_width * deinterleaving;
-			buffer_width *= deinterleaving * 2;
-			num_vectors *= deinterleaving;
-			start_column *= deinterleaving;
-			buf_offset_b = buffer_width / 2 / ISP_VEC_NELEMS;
-			start_column_b = start_column;
-		} else {
-			vmem_increment = 1;
-			deinterleaving = 2;
-			width_a = cropped_width * deinterleaving;
-			buffer_width *= deinterleaving;
-			num_vectors *= deinterleaving;
-			start_column *= deinterleaving;
-		}
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_444:
-	case ATOMISP_INPUT_FORMAT_RGB_555:
-	case ATOMISP_INPUT_FORMAT_RGB_565:
-	case ATOMISP_INPUT_FORMAT_RGB_666:
-	case ATOMISP_INPUT_FORMAT_RGB_888:
-		num_vectors *= 2;
-		if (two_ppc) {
-			deinterleaving = 2;	/* BR in if_a, G in if_b */
-			deinterleaving_b = 1;	/* BR in if_a, G in if_b */
-			buffers_per_line = 4;
-			start_column_b = start_column;
-			start_column *= deinterleaving;
-			start_column_b *= deinterleaving_b;
-		} else {
-			deinterleaving = 3;	/* BGR */
-			buffers_per_line = 3;
-			start_column *= deinterleaving;
-		}
-		vmem_increment = 1;
-		width_a = cropped_width * deinterleaving;
-		width_b = cropped_width * deinterleaving_b;
-		buffer_width *= buffers_per_line;
-		/* Patch from bayer to rgb */
-		num_vectors = num_vectors / 2 * deinterleaving;
-		buf_offset_b = buffer_width / 2 / ISP_VEC_NELEMS;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_6:
-	case ATOMISP_INPUT_FORMAT_RAW_7:
-	case ATOMISP_INPUT_FORMAT_RAW_8:
-	case ATOMISP_INPUT_FORMAT_RAW_10:
-	case ATOMISP_INPUT_FORMAT_RAW_12:
-		if (two_ppc) {
-			int crop_col = (start_column % 2) == 1;
-			vmem_increment = 2;
-			deinterleaving = 1;
-			width_a = width_b = cropped_width / 2;
-
-			/* When two_ppc is enabled AND we need to crop one extra
-			 * column, if_a crops by one extra and we swap the
-			 * output offsets to interleave the bayer pattern in
-			 * the correct order.
-			 */
-			buf_offset_a   = crop_col ? 1 : 0;
-			buf_offset_b   = crop_col ? 0 : 1;
-			start_column_b = start_column / 2;
-			start_column   = start_column / 2 + crop_col;
-		} else {
-			vmem_increment = 1;
-			deinterleaving = 2;
-			if ((!binary) || (config->continuous && binary
-				&& binary->info->sp.pipeline.mode == IA_CSS_BINARY_MODE_COPY)) {
-				/* !binary -> sp raw copy pipe, no deinterleaving */
-				deinterleaving = 1;
-			}
-			width_a = cropped_width;
-			/* Must be multiple of deinterleaving */
-			num_vectors = CEIL_MUL(num_vectors, deinterleaving);
-		}
-		buffer_height *= 2;
-		if ((!binary) || config->continuous)
-			/* !binary -> sp raw copy pipe */
-			buffer_height *= 2;
-		vectors_per_line = CEIL_DIV(cropped_width, ISP_VEC_NELEMS);
-		vectors_per_line = CEIL_MUL(vectors_per_line, deinterleaving);
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_14:
-	case ATOMISP_INPUT_FORMAT_RAW_16:
-		if (two_ppc) {
-			num_vectors *= 2;
-			vmem_increment = 1;
-			deinterleaving = 2;
-			width_a = width_b = cropped_width;
-			/* B buffer is one line further */
-			buf_offset_b = buffer_width / ISP_VEC_NELEMS;
-			bits_per_pixel *= 2;
-		} else {
-			vmem_increment = 1;
-			deinterleaving = 2;
-			width_a = cropped_width;
-			start_column /= deinterleaving;
-		}
-		buffer_height *= 2;
-		break;
-	case ATOMISP_INPUT_FORMAT_BINARY_8:
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT1:
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT2:
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT3:
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT4:
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT5:
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT6:
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT7:
-	case ATOMISP_INPUT_FORMAT_GENERIC_SHORT8:
-	case ATOMISP_INPUT_FORMAT_YUV420_8_SHIFT:
-	case ATOMISP_INPUT_FORMAT_YUV420_10_SHIFT:
-	case ATOMISP_INPUT_FORMAT_EMBEDDED:
-	case ATOMISP_INPUT_FORMAT_USER_DEF1:
-	case ATOMISP_INPUT_FORMAT_USER_DEF2:
-	case ATOMISP_INPUT_FORMAT_USER_DEF3:
-	case ATOMISP_INPUT_FORMAT_USER_DEF4:
-	case ATOMISP_INPUT_FORMAT_USER_DEF5:
-	case ATOMISP_INPUT_FORMAT_USER_DEF6:
-	case ATOMISP_INPUT_FORMAT_USER_DEF7:
-	case ATOMISP_INPUT_FORMAT_USER_DEF8:
-		break;
-	}
-	if (width_a == 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	if (two_ppc)
-		left_padding /= 2;
-
-	/* Default values */
-	if (left_padding)
-		vectors_per_line = num_vectors;
-	if (!vectors_per_line) {
-		vectors_per_line = CEIL_MUL(num_vectors / buffer_height,
-					    deinterleaving);
-		line_width = 0;
-	}
-	if (!line_width)
-		line_width = vectors_per_line *
-		    input_formatter_get_alignment(INPUT_FORMATTER0_ID);
-	if (!buffers_per_line)
-		buffers_per_line = deinterleaving;
-	line_width = CEIL_MUL(line_width,
-			      input_formatter_get_alignment(INPUT_FORMATTER0_ID)
-			      * vmem_increment);
-
-	vectors_per_buffer = buffer_height * buffer_width / ISP_VEC_NELEMS;
-
-	if (config->mode == IA_CSS_INPUT_MODE_TPG &&
-	    ((binary && binary->info->sp.pipeline.mode == IA_CSS_BINARY_MODE_VIDEO) ||
-	    (!binary))) {
-		/* !binary -> sp raw copy pipe */
-		/* workaround for TPG in video mode */
-		start_line = 0;
-		start_column = 0;
-		cropped_height -= start_line;
-		width_a -= start_column;
-	}
-
-	if_a_config.start_line = start_line;
-	if_a_config.start_column = start_column;
-	if_a_config.left_padding = left_padding / deinterleaving;
-	if_a_config.cropped_height = cropped_height;
-	if_a_config.cropped_width = width_a;
-	if_a_config.deinterleaving = deinterleaving;
-	if_a_config.buf_vecs = vectors_per_buffer;
-	if_a_config.buf_start_index = buf_offset_a;
-	if_a_config.buf_increment = vmem_increment;
-	if_a_config.buf_eol_offset =
-	    buffer_width * bits_per_pixel / 8 - line_width;
-	if_a_config.is_yuv420_format =
-	    (input_format == ATOMISP_INPUT_FORMAT_YUV420_8)
-	    || (input_format == ATOMISP_INPUT_FORMAT_YUV420_10)
-	    || (input_format == ATOMISP_INPUT_FORMAT_YUV420_16);
-	if_a_config.block_no_reqs = (config->mode != IA_CSS_INPUT_MODE_SENSOR);
-
-	if (two_ppc) {
-		if (deinterleaving_b) {
-			deinterleaving = deinterleaving_b;
-			width_b = cropped_width * deinterleaving;
-			buffer_width *= deinterleaving;
-			/* Patch from bayer to rgb */
-			num_vectors = num_vectors / 2 *
-			    deinterleaving * width_b_factor;
-			vectors_per_line = num_vectors / buffer_height;
-			line_width = vectors_per_line *
-			    input_formatter_get_alignment(INPUT_FORMATTER0_ID);
-		}
-		if_b_config.start_line = start_line;
-		if_b_config.start_column = start_column_b;
-		if_b_config.left_padding = left_padding / deinterleaving;
-		if_b_config.cropped_height = cropped_height;
-		if_b_config.cropped_width = width_b;
-		if_b_config.deinterleaving = deinterleaving;
-		if_b_config.buf_vecs = vectors_per_buffer;
-		if_b_config.buf_start_index = buf_offset_b;
-		if_b_config.buf_increment = vmem_increment;
-		if_b_config.buf_eol_offset =
-		    buffer_width * bits_per_pixel / 8 - line_width;
-		if_b_config.is_yuv420_format =
-		    input_format == ATOMISP_INPUT_FORMAT_YUV420_8
-		    || input_format == ATOMISP_INPUT_FORMAT_YUV420_10
-		    || input_format == ATOMISP_INPUT_FORMAT_YUV420_16;
-		if_b_config.block_no_reqs =
-		    (config->mode != IA_CSS_INPUT_MODE_SENSOR);
-
-		if (SH_CSS_IF_CONFIG_NOT_NEEDED != if_config_index) {
-			assert(if_config_index <= SH_CSS_MAX_IF_CONFIGS);
-
-			ifmtr_set_if_blocking_mode(&if_a_config, &if_b_config);
-			/* Set the ifconfigs to SP group */
-			sh_css_sp_set_if_configs(&if_a_config, &if_b_config,
-						 if_config_index);
-		}
-	} else {
-		if (SH_CSS_IF_CONFIG_NOT_NEEDED != if_config_index) {
-			assert(if_config_index <= SH_CSS_MAX_IF_CONFIGS);
-
-			ifmtr_set_if_blocking_mode(&if_a_config, NULL);
-			/* Set the ifconfigs to SP group */
-			sh_css_sp_set_if_configs(&if_a_config, NULL,
-						 if_config_index);
-		}
-	}
-
-	return IA_CSS_SUCCESS;
-}
-
-bool ifmtr_set_if_blocking_mode_reset = true;
-
-/************************************************************
- * Static functions
- ************************************************************/
-static void ifmtr_set_if_blocking_mode(
-		const input_formatter_cfg_t * const config_a,
-		const input_formatter_cfg_t * const config_b)
-{
-	int i;
-	bool block[] = { false, false, false, false };
-	assert(N_INPUT_FORMATTER_ID <= (ARRAY_SIZE(block)));
-
-#if !defined(IS_ISP_2400_SYSTEM)
-#error "ifmtr_set_if_blocking_mode: ISP_SYSTEM must be one of {IS_ISP_2400_SYSTEM}"
-#endif
-
-	block[INPUT_FORMATTER0_ID] = (bool)config_a->block_no_reqs;
-	if (NULL != config_b)
-		block[INPUT_FORMATTER1_ID] = (bool)config_b->block_no_reqs;
-
-	/* TODO: next could cause issues when streams are started after
-	 * eachother. */
-	/*IF should not be reconfigured/reset from host */
-	if (ifmtr_set_if_blocking_mode_reset) {
-		ifmtr_set_if_blocking_mode_reset = false;
-		for (i = 0; i < N_INPUT_FORMATTER_ID; i++) {
-			input_formatter_ID_t id = (input_formatter_ID_t) i;
-			input_formatter_rst(id);
-			input_formatter_set_fifo_blocking_mode(id, block[id]);
-		}
-	}
-
-	return;
-}
-
-static enum ia_css_err ifmtr_start_column(
-		const struct ia_css_stream_config *config,
-		unsigned int bin_in,
-		unsigned int *start_column)
-{
-	unsigned int in = config->input_config.input_res.width, start,
-	    for_bayer = ia_css_ifmtr_columns_needed_for_bayer_order(config);
-
-	if (bin_in + 2 * for_bayer > in)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	/* On the hardware, we want to use the middle of the input, so we
-	 * divide the start column by 2. */
-	start = (in - bin_in) / 2;
-	/* in case the number of extra columns is 2 or odd, we round the start
-	 * column down */
-	start &= ~0x1;
-
-	/* now we add the one column (if needed) to correct for the bayer
-	 * order).
-	 */
-	start += for_bayer;
-	*start_column = start;
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err ifmtr_input_start_line(
-		const struct ia_css_stream_config *config,
-		unsigned int bin_in,
-		unsigned int *start_line)
-{
-	unsigned int in = config->input_config.input_res.height, start,
-	    for_bayer = ia_css_ifmtr_lines_needed_for_bayer_order(config);
-
-	if (bin_in + 2 * for_bayer > in)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	/* On the hardware, we want to use the middle of the input, so we
-	 * divide the start line by 2. On the simulator, we cannot handle extra
-	 * lines at the end of the frame.
-	 */
-	start = (in - bin_in) / 2;
-	/* in case the number of extra lines is 2 or odd, we round the start
-	 * line down.
-	 */
-	start &= ~0x1;
-
-	/* now we add the one line (if needed) to correct for the bayer order */
-	start += for_bayer;
-	*start_line = start;
-	return IA_CSS_SUCCESS;
-}
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/inputfifo/interface/ia_css_inputfifo.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/inputfifo/interface/ia_css_inputfifo.h
deleted file mode 100644
index 545f9e2da59e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/inputfifo/interface/ia_css_inputfifo.h
+++ /dev/null
@@ -1,69 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _IA_CSS_INPUTFIFO_H
-#define _IA_CSS_INPUTFIFO_H
-
-#include <sp.h>
-#include <isp.h>
-
-#include "ia_css_stream_format.h"
-
-/* SP access */
-void ia_css_inputfifo_send_input_frame(
-	const unsigned short	*data,
-	unsigned int	width,
-	unsigned int	height,
-	unsigned int	ch_id,
-	enum atomisp_input_format	input_format,
-	bool			two_ppc);
-
-void ia_css_inputfifo_start_frame(
-	unsigned int	ch_id,
-	enum atomisp_input_format	input_format,
-	bool			two_ppc);
-
-void ia_css_inputfifo_send_line(
-	unsigned int	ch_id,
-	const unsigned short	*data,
-	unsigned int	width,
-	const unsigned short	*data2,
-	unsigned int	width2);
-
-void ia_css_inputfifo_send_embedded_line(
-	unsigned int	ch_id,
-	enum atomisp_input_format	data_type,
-	const unsigned short	*data,
-	unsigned int	width);
-
-void ia_css_inputfifo_end_frame(
-	unsigned int	ch_id);
-
-#endif /* _IA_CSS_INPUTFIFO_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/inputfifo/src/inputfifo.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/inputfifo/src/inputfifo.c
deleted file mode 100644
index 24ca4aaf8df1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/inputfifo/src/inputfifo.c
+++ /dev/null
@@ -1,613 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "platform_support.h"
-
-#include "ia_css_inputfifo.h"
-
-#include "device_access.h"
-
-#define __INLINE_SP__
-#include "sp.h"
-#define __INLINE_ISP__
-#include "isp.h"
-#define __INLINE_IRQ__
-#include "irq.h"
-#define __INLINE_FIFO_MONITOR__
-#include "fifo_monitor.h"
-
-#define __INLINE_EVENT__
-#include "event_fifo.h"
-#define __INLINE_SP__
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#include "input_system.h"	/* MIPI_PREDICTOR_NONE,... */
-#endif
-
-#include "assert_support.h"
-
-/* System independent */
-#include "sh_css_internal.h"
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#include "ia_css_isys.h"
-#endif
-
-#define HBLANK_CYCLES (187)
-#define MARKER_CYCLES (6)
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#include <hive_isp_css_streaming_to_mipi_types_hrt.h>
-#endif
-
-/* The data type is used to send special cases:
- * yuv420: odd lines (1, 3 etc) are twice as wide as even
- *         lines (0, 2, 4 etc).
- * rgb: for two pixels per clock, the R and B values are sent
- *      to output_0 while only G is sent to output_1. This means
- *      that output_1 only gets half the number of values of output_0.
- *      WARNING: This type should also be used for Legacy YUV420.
- * regular: used for all other data types (RAW, YUV422, etc)
- */
-enum inputfifo_mipi_data_type {
-	inputfifo_mipi_data_type_regular,
-	inputfifo_mipi_data_type_yuv420,
-	inputfifo_mipi_data_type_yuv420_legacy,
-	inputfifo_mipi_data_type_rgb,
-};
-#if !defined(HAS_NO_INPUT_SYSTEM)
-static unsigned int inputfifo_curr_ch_id, inputfifo_curr_fmt_type;
-#endif
-struct inputfifo_instance {
-	unsigned int				ch_id;
-	enum atomisp_input_format	input_format;
-	bool						two_ppc;
-	bool						streaming;
-	unsigned int				hblank_cycles;
-	unsigned int				marker_cycles;
-	unsigned int				fmt_type;
-	enum inputfifo_mipi_data_type	type;
-};
-#if !defined(HAS_NO_INPUT_SYSTEM)
-/*
- * Maintain a basic streaming to Mipi administration with ch_id as index
- * ch_id maps on the "Mipi virtual channel ID" and can have value 0..3
- */
-#define INPUTFIFO_NR_OF_S2M_CHANNELS	(4)
-static struct inputfifo_instance
-	inputfifo_inst_admin[INPUTFIFO_NR_OF_S2M_CHANNELS];
-
-/* Streaming to MIPI */
-static unsigned inputfifo_wrap_marker(
-/* static inline unsigned inputfifo_wrap_marker( */
-	unsigned marker)
-{
-	return marker |
-	(inputfifo_curr_ch_id << HIVE_STR_TO_MIPI_CH_ID_LSB) |
-	(inputfifo_curr_fmt_type << _HIVE_STR_TO_MIPI_FMT_TYPE_LSB);
-}
-
-static inline void
-_sh_css_fifo_snd(unsigned token)
-{
-	while (!can_event_send_token(STR2MIPI_EVENT_ID))
-		hrt_sleep();
-	event_send_token(STR2MIPI_EVENT_ID, token);
-	return;
-}
-
-static void inputfifo_send_data_a(
-/* static inline void inputfifo_send_data_a( */
-unsigned int data)
-{
-	unsigned int token = (1 << HIVE_STR_TO_MIPI_VALID_A_BIT) |
-			     (data << HIVE_STR_TO_MIPI_DATA_A_LSB);
-	_sh_css_fifo_snd(token);
-	return;
-}
-
-
-
-static void inputfifo_send_data_b(
-/* static inline void inputfifo_send_data_b( */
-	unsigned int data)
-{
-	unsigned int token = (1 << HIVE_STR_TO_MIPI_VALID_B_BIT) |
-			     (data << _HIVE_STR_TO_MIPI_DATA_B_LSB);
-	_sh_css_fifo_snd(token);
-	return;
-}
-
-
-
-static void inputfifo_send_data(
-/* static inline void inputfifo_send_data( */
-	unsigned int a,
-	unsigned int b)
-{
-	unsigned int token = ((1 << HIVE_STR_TO_MIPI_VALID_A_BIT) |
-			      (1 << HIVE_STR_TO_MIPI_VALID_B_BIT) |
-			      (a << HIVE_STR_TO_MIPI_DATA_A_LSB) |
-			      (b << _HIVE_STR_TO_MIPI_DATA_B_LSB));
-	_sh_css_fifo_snd(token);
-	return;
-}
-
-
-
-static void inputfifo_send_sol(void)
-/* static inline void inputfifo_send_sol(void) */
-{
-	hrt_data	token = inputfifo_wrap_marker(
-		1 << HIVE_STR_TO_MIPI_SOL_BIT);
-
-	_sh_css_fifo_snd(token);
-	return;
-}
-
-
-
-static void inputfifo_send_eol(void)
-/* static inline void inputfifo_send_eol(void) */
-{
-	hrt_data	token = inputfifo_wrap_marker(
-		1 << HIVE_STR_TO_MIPI_EOL_BIT);
-	_sh_css_fifo_snd(token);
-	return;
-}
-
-
-
-static void inputfifo_send_sof(void)
-/* static inline void inputfifo_send_sof(void) */
-{
-	hrt_data	token = inputfifo_wrap_marker(
-		1 << HIVE_STR_TO_MIPI_SOF_BIT);
-
-	_sh_css_fifo_snd(token);
-	return;
-}
-
-
-
-static void inputfifo_send_eof(void)
-/* static inline void inputfifo_send_eof(void) */
-{
-	hrt_data	token = inputfifo_wrap_marker(
-		1 << HIVE_STR_TO_MIPI_EOF_BIT);
-	_sh_css_fifo_snd(token);
-	return;
-}
-
-
-
-#ifdef __ON__
-static void inputfifo_send_ch_id(
-/* static inline void inputfifo_send_ch_id( */
-	unsigned int ch_id)
-{
-	hrt_data	token;
-	inputfifo_curr_ch_id = ch_id & _HIVE_ISP_CH_ID_MASK;
-	/* we send an zero marker, this will wrap the ch_id and
-	 * fmt_type automatically.
-	 */
-	token = inputfifo_wrap_marker(0);
-	_sh_css_fifo_snd(token);
-	return;
-}
-
-static void inputfifo_send_fmt_type(
-/* static inline void inputfifo_send_fmt_type( */
-	unsigned int fmt_type)
-{
-	hrt_data	token;
-	inputfifo_curr_fmt_type = fmt_type & _HIVE_ISP_FMT_TYPE_MASK;
-	/* we send an zero marker, this will wrap the ch_id and
-	 * fmt_type automatically.
-	 */
-	token = inputfifo_wrap_marker(0);
-	_sh_css_fifo_snd(token);
-	return;
-}
-#endif /*  __ON__ */
-
-
-
-static void inputfifo_send_ch_id_and_fmt_type(
-/* static inline
-void inputfifo_send_ch_id_and_fmt_type( */
-	unsigned int ch_id,
-	unsigned int fmt_type)
-{
-	hrt_data	token;
-	inputfifo_curr_ch_id = ch_id & _HIVE_ISP_CH_ID_MASK;
-	inputfifo_curr_fmt_type = fmt_type & _HIVE_ISP_FMT_TYPE_MASK;
-	/* we send an zero marker, this will wrap the ch_id and
-	 * fmt_type automatically.
-	 */
-	token = inputfifo_wrap_marker(0);
-	_sh_css_fifo_snd(token);
-	return;
-}
-
-
-
-static void inputfifo_send_empty_token(void)
-/* static inline void inputfifo_send_empty_token(void) */
-{
-	hrt_data	token = inputfifo_wrap_marker(0);
-	_sh_css_fifo_snd(token);
-	return;
-}
-
-
-
-static void inputfifo_start_frame(
-/* static inline void inputfifo_start_frame( */
-	unsigned int ch_id,
-	unsigned int fmt_type)
-{
-	inputfifo_send_ch_id_and_fmt_type(ch_id, fmt_type);
-	inputfifo_send_sof();
-	return;
-}
-
-
-
-static void inputfifo_end_frame(
-	unsigned int marker_cycles)
-{
-	unsigned int i;
-	for (i = 0; i < marker_cycles; i++)
-		inputfifo_send_empty_token();
-	inputfifo_send_eof();
-	return;
-}
-
-
-
-static void inputfifo_send_line2(
-	const unsigned short *data,
-	unsigned int width,
-	const unsigned short *data2,
-	unsigned int width2,
-	unsigned int hblank_cycles,
-	unsigned int marker_cycles,
-	unsigned int two_ppc,
-	enum inputfifo_mipi_data_type type)
-{
-	unsigned int i, is_rgb = 0, is_legacy = 0;
-
-	assert(data != NULL);
-	assert((data2 != NULL) || (width2 == 0));
-	if (type == inputfifo_mipi_data_type_rgb)
-		is_rgb = 1;
-
-	if (type == inputfifo_mipi_data_type_yuv420_legacy)
-		is_legacy = 1;
-
-	for (i = 0; i < hblank_cycles; i++)
-		inputfifo_send_empty_token();
-	inputfifo_send_sol();
-	for (i = 0; i < marker_cycles; i++)
-		inputfifo_send_empty_token();
-	for (i = 0; i < width; i++, data++) {
-		/* for RGB in two_ppc, we only actually send 2 pixels per
-		 * clock in the even pixels (0, 2 etc). In the other cycles,
-		 * we only send 1 pixel, to data[0].
-		 */
-		unsigned int send_two_pixels = two_ppc;
-		if ((is_rgb || is_legacy) && (i % 3 == 2))
-			send_two_pixels = 0;
-		if (send_two_pixels) {
-			if (i + 1 == width) {
-				/* for jpg (binary) copy, this can occur
-				 * if the file contains an odd number of bytes.
-				 */
-				inputfifo_send_data(
-							data[0], 0);
-			} else {
-				inputfifo_send_data(
-							data[0], data[1]);
-			}
-			/* Additional increment because we send 2 pixels */
-			data++;
-			i++;
-		} else if (two_ppc && is_legacy) {
-			inputfifo_send_data_b(data[0]);
-		} else {
-			inputfifo_send_data_a(data[0]);
-		}
-	}
-
-	for (i = 0; i < width2; i++, data2++) {
-		/* for RGB in two_ppc, we only actually send 2 pixels per
-		 * clock in the even pixels (0, 2 etc). In the other cycles,
-		 * we only send 1 pixel, to data2[0].
-		 */
-		unsigned int send_two_pixels = two_ppc;
-		if ((is_rgb || is_legacy) && (i % 3 == 2))
-			send_two_pixels = 0;
-		if (send_two_pixels) {
-			if (i + 1 == width2) {
-				/* for jpg (binary) copy, this can occur
-				 * if the file contains an odd number of bytes.
-				 */
-				inputfifo_send_data(
-							data2[0], 0);
-			} else {
-				inputfifo_send_data(
-							data2[0], data2[1]);
-			}
-			/* Additional increment because we send 2 pixels */
-			data2++;
-			i++;
-		} else if (two_ppc && is_legacy) {
-			inputfifo_send_data_b(data2[0]);
-		} else {
-			inputfifo_send_data_a(data2[0]);
-		}
-	}
-	for (i = 0; i < hblank_cycles; i++)
-		inputfifo_send_empty_token();
-	inputfifo_send_eol();
-	return;
-}
-
-
-
-static void
-inputfifo_send_line(const unsigned short *data,
-			 unsigned int width,
-			 unsigned int hblank_cycles,
-			 unsigned int marker_cycles,
-			 unsigned int two_ppc,
-			 enum inputfifo_mipi_data_type type)
-{
-	assert(data != NULL);
-	inputfifo_send_line2(data, width, NULL, 0,
-					hblank_cycles,
-					marker_cycles,
-					two_ppc,
-					type);
-}
-
-
-/* Send a frame of data into the input network via the GP FIFO.
- *  Parameters:
- *   - data: array of 16 bit values that contains all data for the frame.
- *   - width: width of a line in number of subpixels, for yuv420 it is the
- *            number of Y components per line.
- *   - height: height of the frame in number of lines.
- *   - ch_id: channel ID.
- *   - fmt_type: format type.
- *   - hblank_cycles: length of horizontal blanking in cycles.
- *   - marker_cycles: number of empty cycles after start-of-line and before
- *                    end-of-frame.
- *   - two_ppc: boolean, describes whether to send one or two pixels per clock
- *              cycle. In this mode, we sent pixels N and N+1 in the same cycle,
- *              to IF_PRIM_A and IF_PRIM_B respectively. The caller must make
- *              sure the input data has been formatted correctly for this.
- *              For example, for RGB formats this means that unused values
- *              must be inserted.
- *   - yuv420: boolean, describes whether (non-legacy) yuv420 data is used. In
- *             this mode, the odd lines (1,3,5 etc) are half as long as the
- *             even lines (2,4,6 etc).
- *             Note that the first line is odd (1) and the second line is even
- *             (2).
- *
- * This function does not do any reordering of pixels, the caller must make
- * sure the data is in the righ format. Please refer to the CSS receiver
- * documentation for details on the data formats.
- */
-
-static void inputfifo_send_frame(
-	const unsigned short *data,
-	unsigned int width,
-	unsigned int height,
-	unsigned int ch_id,
-	unsigned int fmt_type,
-	unsigned int hblank_cycles,
-	unsigned int marker_cycles,
-	unsigned int two_ppc,
-	enum inputfifo_mipi_data_type type)
-{
-	unsigned int i;
-
-	assert(data != NULL);
-	inputfifo_start_frame(ch_id, fmt_type);
-
-	for (i = 0; i < height; i++) {
-		if ((type == inputfifo_mipi_data_type_yuv420) &&
-		    (i & 1) == 1) {
-			inputfifo_send_line(data, 2 * width,
-							   hblank_cycles,
-							   marker_cycles,
-							   two_ppc, type);
-			data += 2 * width;
-		} else {
-			inputfifo_send_line(data, width,
-							   hblank_cycles,
-							   marker_cycles,
-							   two_ppc, type);
-			data += width;
-		}
-	}
-	inputfifo_end_frame(marker_cycles);
-	return;
-}
-
-
-
-static enum inputfifo_mipi_data_type inputfifo_determine_type(
-	enum atomisp_input_format input_format)
-{
-	enum inputfifo_mipi_data_type type;
-
-	type = inputfifo_mipi_data_type_regular;
-	if (input_format == ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY) {
-		type =
-			inputfifo_mipi_data_type_yuv420_legacy;
-	} else if (input_format == ATOMISP_INPUT_FORMAT_YUV420_8  ||
-		   input_format == ATOMISP_INPUT_FORMAT_YUV420_10 ||
-		   input_format == ATOMISP_INPUT_FORMAT_YUV420_16) {
-		type =
-			inputfifo_mipi_data_type_yuv420;
-	} else if (input_format >= ATOMISP_INPUT_FORMAT_RGB_444 &&
-		   input_format <= ATOMISP_INPUT_FORMAT_RGB_888) {
-		type =
-			inputfifo_mipi_data_type_rgb;
-	}
-	return type;
-}
-
-
-
-static struct inputfifo_instance *inputfifo_get_inst(
-	unsigned int ch_id)
-{
-	return &inputfifo_inst_admin[ch_id];
-}
-
-void ia_css_inputfifo_send_input_frame(
-	const unsigned short *data,
-	unsigned int width,
-	unsigned int height,
-	unsigned int ch_id,
-	enum atomisp_input_format input_format,
-	bool two_ppc)
-{
-	unsigned int fmt_type, hblank_cycles, marker_cycles;
-	enum inputfifo_mipi_data_type type;
-
-	assert(data != NULL);
-	hblank_cycles = HBLANK_CYCLES;
-	marker_cycles = MARKER_CYCLES;
-	ia_css_isys_convert_stream_format_to_mipi_format(input_format,
-				 MIPI_PREDICTOR_NONE,
-				 &fmt_type);
-
-	type = inputfifo_determine_type(input_format);
-
-	inputfifo_send_frame(data, width, height,
-			ch_id, fmt_type, hblank_cycles, marker_cycles,
-			two_ppc, type);
-}
-
-
-
-void ia_css_inputfifo_start_frame(
-	unsigned int ch_id,
-	enum atomisp_input_format input_format,
-	bool two_ppc)
-{
-	struct inputfifo_instance *s2mi;
-	s2mi = inputfifo_get_inst(ch_id);
-
-	s2mi->ch_id = ch_id;
-	ia_css_isys_convert_stream_format_to_mipi_format(input_format,
-				MIPI_PREDICTOR_NONE,
-				&s2mi->fmt_type);
-	s2mi->two_ppc = two_ppc;
-	s2mi->type = inputfifo_determine_type(input_format);
-	s2mi->hblank_cycles = HBLANK_CYCLES;
-	s2mi->marker_cycles = MARKER_CYCLES;
-	s2mi->streaming = true;
-
-	inputfifo_start_frame(ch_id, s2mi->fmt_type);
-	return;
-}
-
-
-
-void ia_css_inputfifo_send_line(
-	unsigned int ch_id,
-	const unsigned short *data,
-	unsigned int width,
-	const unsigned short *data2,
-	unsigned int width2)
-{
-	struct inputfifo_instance *s2mi;
-
-	assert(data != NULL);
-	assert((data2 != NULL) || (width2 == 0));
-	s2mi = inputfifo_get_inst(ch_id);
-
-
-	/* Set global variables that indicate channel_id and format_type */
-	inputfifo_curr_ch_id = (s2mi->ch_id) & _HIVE_ISP_CH_ID_MASK;
-	inputfifo_curr_fmt_type = (s2mi->fmt_type) & _HIVE_ISP_FMT_TYPE_MASK;
-
-	inputfifo_send_line2(data, width, data2, width2,
-					s2mi->hblank_cycles,
-					s2mi->marker_cycles,
-					s2mi->two_ppc,
-					s2mi->type);
-}
-
-
-void ia_css_inputfifo_send_embedded_line(
-	unsigned int	ch_id,
-	enum atomisp_input_format	data_type,
-	const unsigned short	*data,
-	unsigned int	width)
-{
-	struct inputfifo_instance *s2mi;
-	unsigned int fmt_type;
-
-	assert(data != NULL);
-	s2mi = inputfifo_get_inst(ch_id);
-	ia_css_isys_convert_stream_format_to_mipi_format(data_type,
-			MIPI_PREDICTOR_NONE, &fmt_type);
-
-	/* Set format_type for metadata line. */
-	inputfifo_curr_fmt_type = fmt_type & _HIVE_ISP_FMT_TYPE_MASK;
-
-	inputfifo_send_line(data, width, s2mi->hblank_cycles, s2mi->marker_cycles,
-			s2mi->two_ppc, inputfifo_mipi_data_type_regular);
-}
-
-
-void ia_css_inputfifo_end_frame(
-	unsigned int	ch_id)
-{
-	struct inputfifo_instance *s2mi;
-	s2mi = inputfifo_get_inst(ch_id);
-
-	/* Set global variables that indicate channel_id and format_type */
-	inputfifo_curr_ch_id = (s2mi->ch_id) & _HIVE_ISP_CH_ID_MASK;
-	inputfifo_curr_fmt_type = (s2mi->fmt_type) & _HIVE_ISP_FMT_TYPE_MASK;
-
-	/* Call existing HRT function */
-	inputfifo_end_frame(s2mi->marker_cycles);
-
-	s2mi->streaming = false;
-	return;
-}
-#endif /* #if !defined(HAS_NO_INPUT_SYSTEM) */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/interface/ia_css_isp_param.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/interface/ia_css_isp_param.h
deleted file mode 100644
index 285749885105..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/interface/ia_css_isp_param.h
+++ /dev/null
@@ -1,118 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _IA_CSS_ISP_PARAM_H_
-#define _IA_CSS_ISP_PARAM_H_
-
-#include <ia_css_err.h>
-#include "ia_css_isp_param_types.h"
-
-/* Set functions for parameter memory descriptors */
-void
-ia_css_isp_param_set_mem_init(
-	struct ia_css_isp_param_host_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem,
-	char *address, size_t size);
-
-void
-ia_css_isp_param_set_css_mem_init(
-	struct ia_css_isp_param_css_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem,
-	hrt_vaddress address, size_t size);
-
-void
-ia_css_isp_param_set_isp_mem_init(
-	struct ia_css_isp_param_isp_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem,
-	uint32_t address, size_t size);
-
-/* Get functions for parameter memory descriptors */
-const struct ia_css_host_data*
-ia_css_isp_param_get_mem_init(
-	const struct ia_css_isp_param_host_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem);
-
-const struct ia_css_data*
-ia_css_isp_param_get_css_mem_init(
-	const struct ia_css_isp_param_css_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem);
-
-const struct ia_css_isp_data*
-ia_css_isp_param_get_isp_mem_init(
-	const struct ia_css_isp_param_isp_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem);
-
-/* Initialize the memory interface sizes and addresses */
-void
-ia_css_init_memory_interface(
-	struct ia_css_isp_param_css_segments *isp_mem_if,
-	const struct ia_css_isp_param_host_segments *mem_params,
-	const struct ia_css_isp_param_css_segments *css_params);
-
-/* Allocate memory parameters */
-enum ia_css_err
-ia_css_isp_param_allocate_isp_parameters(
-	struct ia_css_isp_param_host_segments *mem_params,
-	struct ia_css_isp_param_css_segments *css_params,
-	const struct ia_css_isp_param_isp_segments *mem_initializers);
-
-/* Destroy memory parameters */
-void
-ia_css_isp_param_destroy_isp_parameters(
-	struct ia_css_isp_param_host_segments *mem_params,
-	struct ia_css_isp_param_css_segments *css_params);
-
-/* Load fw parameters */
-void
-ia_css_isp_param_load_fw_params(
-	const char *fw,
-	union ia_css_all_memory_offsets *mem_offsets,
-	const struct ia_css_isp_param_memory_offsets *memory_offsets,
-	bool init);
-
-/* Copy host parameter images to ddr */
-enum ia_css_err
-ia_css_isp_param_copy_isp_mem_if_to_ddr(
-	struct ia_css_isp_param_css_segments *ddr,
-	const struct ia_css_isp_param_host_segments *host,
-	enum ia_css_param_class pclass);
-
-/* Enable a pipeline by setting the control field in the isp dmem parameters */
-void
-ia_css_isp_param_enable_pipeline(
-	const struct ia_css_isp_param_host_segments *mem_params);
-
-#endif /* _IA_CSS_ISP_PARAM_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/interface/ia_css_isp_param_types.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/interface/ia_css_isp_param_types.h
deleted file mode 100644
index 9d111793bb65..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/interface/ia_css_isp_param_types.h
+++ /dev/null
@@ -1,98 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _IA_CSS_ISP_PARAM_TYPES_H_
-#define _IA_CSS_ISP_PARAM_TYPES_H_
-
-#include "ia_css_types.h"
-#include <platform_support.h>
-#include <system_global.h>
-
-/* Short hands */
-#define IA_CSS_ISP_DMEM IA_CSS_ISP_DMEM0
-#define IA_CSS_ISP_VMEM IA_CSS_ISP_VMEM0
-
-/* The driver depends on this, to be removed later. */
-#define IA_CSS_NUM_ISP_MEMORIES IA_CSS_NUM_MEMORIES
-
-/* Explicit member numbering to avoid fish type checker bug */
-enum ia_css_param_class {
-	IA_CSS_PARAM_CLASS_PARAM  = 0,	/* Late binding parameters, like 3A */
-	IA_CSS_PARAM_CLASS_CONFIG = 1,	/* Pipe config time parameters, like resolution */
-	IA_CSS_PARAM_CLASS_STATE  = 2,  /* State parameters, like tnr buffer index */
-#if 0 /* Not yet implemented */
-	IA_CSS_PARAM_CLASS_FRAME  = 3,  /* Frame time parameters, like output buffer */
-#endif
-};
-#define IA_CSS_NUM_PARAM_CLASSES (IA_CSS_PARAM_CLASS_STATE + 1)
-
-/* ISP parameter descriptor */
-struct ia_css_isp_parameter {
-	uint32_t offset; /* Offset in isp_<mem>)parameters, etc. */
-	uint32_t size;   /* Disabled if 0 */
-};
-
-
-/* Address/size of each parameter class in each isp memory, host memory pointers */
-struct ia_css_isp_param_host_segments {
-	struct ia_css_host_data params[IA_CSS_NUM_PARAM_CLASSES][IA_CSS_NUM_MEMORIES];
-};
-
-/* Address/size of each parameter class in each isp memory, css memory pointers */
-struct ia_css_isp_param_css_segments {
-	struct ia_css_data      params[IA_CSS_NUM_PARAM_CLASSES][IA_CSS_NUM_MEMORIES];
-};
-
-/* Address/size of each parameter class in each isp memory, isp memory pointers */
-struct ia_css_isp_param_isp_segments {
-	struct ia_css_isp_data  params[IA_CSS_NUM_PARAM_CLASSES][IA_CSS_NUM_MEMORIES];
-};
-
-/* Memory offsets in binary info */
-struct ia_css_isp_param_memory_offsets {
-	uint32_t offsets[IA_CSS_NUM_PARAM_CLASSES];  /** offset wrt hdr in bytes */
-};
-
-/* Offsets for ISP kernel parameters per isp memory.
- * Only relevant for standard ISP binaries, not ACC or SP.
- */
-union ia_css_all_memory_offsets {
-	struct {
-		CSS_ALIGN(struct ia_css_memory_offsets	      *param, 8);
-		CSS_ALIGN(struct ia_css_config_memory_offsets *config, 8);
-		CSS_ALIGN(struct ia_css_state_memory_offsets  *state, 8);
-	} offsets;
-	struct {
-		CSS_ALIGN(void *ptr, 8);
-	} array[IA_CSS_NUM_PARAM_CLASSES];
-};
-
-#endif /* _IA_CSS_ISP_PARAM_TYPES_H_ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/src/isp_param.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/src/isp_param.c
deleted file mode 100644
index f793ce125f02..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isp_param/src/isp_param.c
+++ /dev/null
@@ -1,227 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "memory_access.h"
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_param.h"
-
-/* Set functions for parameter memory descriptors */
-
-void
-ia_css_isp_param_set_mem_init(
-	struct ia_css_isp_param_host_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem,
-	char *address, size_t size)
-{
-	mem_init->params[pclass][mem].address = address;
-	mem_init->params[pclass][mem].size = (uint32_t)size;
-}
-
-void
-ia_css_isp_param_set_css_mem_init(
-	struct ia_css_isp_param_css_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem,
-	hrt_vaddress address, size_t size)
-{
-	mem_init->params[pclass][mem].address = address;
-	mem_init->params[pclass][mem].size = (uint32_t)size;
-}
-
-void
-ia_css_isp_param_set_isp_mem_init(
-	struct ia_css_isp_param_isp_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem,
-	uint32_t address, size_t size)
-{
-	mem_init->params[pclass][mem].address = address;
-	mem_init->params[pclass][mem].size = (uint32_t)size;
-}
-
-/* Get functions for parameter memory descriptors */
-const struct ia_css_host_data*
-ia_css_isp_param_get_mem_init(
-	const struct ia_css_isp_param_host_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem)
-{
-	return &mem_init->params[pclass][mem];
-}
-
-const struct ia_css_data*
-ia_css_isp_param_get_css_mem_init(
-	const struct ia_css_isp_param_css_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem)
-{
-	return &mem_init->params[pclass][mem];
-}
-
-const struct ia_css_isp_data*
-ia_css_isp_param_get_isp_mem_init(
-	const struct ia_css_isp_param_isp_segments *mem_init,
-	enum ia_css_param_class pclass,
-	enum ia_css_isp_memories mem)
-{
-	return &mem_init->params[pclass][mem];
-}
-
-void
-ia_css_init_memory_interface(
-	struct ia_css_isp_param_css_segments *isp_mem_if,
-	const struct ia_css_isp_param_host_segments *mem_params,
-	const struct ia_css_isp_param_css_segments *css_params)
-{
-	unsigned pclass, mem;
-	for (pclass = 0; pclass < IA_CSS_NUM_PARAM_CLASSES; pclass++) {
-		memset(isp_mem_if->params[pclass], 0, sizeof(isp_mem_if->params[pclass]));
-		for (mem = 0; mem < IA_CSS_NUM_MEMORIES; mem++) {
-			if (!mem_params->params[pclass][mem].address)
-				continue;
-			isp_mem_if->params[pclass][mem].size = mem_params->params[pclass][mem].size;
-			if (pclass != IA_CSS_PARAM_CLASS_PARAM)
-				isp_mem_if->params[pclass][mem].address = css_params->params[pclass][mem].address;
-		}
-	}
-}
-
-enum ia_css_err
-ia_css_isp_param_allocate_isp_parameters(
-	struct ia_css_isp_param_host_segments *mem_params,
-	struct ia_css_isp_param_css_segments *css_params,
-	const struct ia_css_isp_param_isp_segments *mem_initializers)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	unsigned mem, pclass;
-
-	pclass = IA_CSS_PARAM_CLASS_PARAM;
-	for (mem = 0; mem < IA_CSS_NUM_MEMORIES; mem++) {
-		for (pclass = 0; pclass < IA_CSS_NUM_PARAM_CLASSES; pclass++) {
-			uint32_t size = 0;
-			if (mem_initializers)
-				size = mem_initializers->params[pclass][mem].size;
-			mem_params->params[pclass][mem].size = size;
-			mem_params->params[pclass][mem].address = NULL;
-			css_params->params[pclass][mem].size = size;
-			css_params->params[pclass][mem].address = 0x0;
-			if (size) {
-				mem_params->params[pclass][mem].address = sh_css_calloc(1, size);
-				if (!mem_params->params[pclass][mem].address) {
-					err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-					goto cleanup;
-				}
-				if (pclass != IA_CSS_PARAM_CLASS_PARAM) {
-					css_params->params[pclass][mem].address = mmgr_malloc(size);
-					if (!css_params->params[pclass][mem].address) {
-						err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-						goto cleanup;
-					}
-				}
-			}
-		}
-	}
-	return err;
-cleanup:
-	ia_css_isp_param_destroy_isp_parameters(mem_params, css_params);
-	return err;
-}
-
-void
-ia_css_isp_param_destroy_isp_parameters(
-	struct ia_css_isp_param_host_segments *mem_params,
-	struct ia_css_isp_param_css_segments *css_params)
-{
-	unsigned mem, pclass;
-
-	for (mem = 0; mem < IA_CSS_NUM_MEMORIES; mem++) {
-		for (pclass = 0; pclass < IA_CSS_NUM_PARAM_CLASSES; pclass++) {
-			if (mem_params->params[pclass][mem].address)
-				sh_css_free(mem_params->params[pclass][mem].address);
-			if (css_params->params[pclass][mem].address)
-				hmm_free(css_params->params[pclass][mem].address);
-			mem_params->params[pclass][mem].address = NULL;
-			css_params->params[pclass][mem].address = 0x0;
-		}
-	}
-}
-
-void
-ia_css_isp_param_load_fw_params(
-	const char *fw,
-	union ia_css_all_memory_offsets *mem_offsets,
-	const struct ia_css_isp_param_memory_offsets *memory_offsets,
-	bool init)
-{
-	unsigned pclass;
-	for (pclass = 0; pclass < IA_CSS_NUM_PARAM_CLASSES; pclass++) {
-		mem_offsets->array[pclass].ptr = NULL;
-		if (init)
-			mem_offsets->array[pclass].ptr = (void *)(fw + memory_offsets->offsets[pclass]);
-	}
-}
-
-enum ia_css_err
-ia_css_isp_param_copy_isp_mem_if_to_ddr(
-	struct ia_css_isp_param_css_segments *ddr,
-	const struct ia_css_isp_param_host_segments *host,
-	enum ia_css_param_class pclass)
-{
-	unsigned mem;
-
-	for (mem = 0; mem < N_IA_CSS_ISP_MEMORIES; mem++) {
-		size_t       size	  = host->params[pclass][mem].size;
-		hrt_vaddress ddr_mem_ptr  = ddr->params[pclass][mem].address;
-		char	    *host_mem_ptr = host->params[pclass][mem].address;
-		if (size != ddr->params[pclass][mem].size)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		if (!size)
-			continue;
-		mmgr_store(ddr_mem_ptr, host_mem_ptr, size);
-	}
-	return IA_CSS_SUCCESS;
-}
-
-void
-ia_css_isp_param_enable_pipeline(
-	const struct ia_css_isp_param_host_segments *mem_params)
-{
-	/* By protocol b0 of the mandatory uint32_t first field of the
-	   input parameter is a disable bit*/
-	short dmem_offset = 0;
-
-	if (mem_params->params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM0].size == 0)
-		return;
-
-	*(uint32_t *)&mem_params->params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM0].address[dmem_offset] = 0x0;
-}
-
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/interface/ia_css_isys.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/interface/ia_css_isys.h
deleted file mode 100644
index 8c005db9766e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/interface/ia_css_isys.h
+++ /dev/null
@@ -1,201 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_ISYS_H__
-#define __IA_CSS_ISYS_H__
-
-#include <type_support.h>
-#include <input_system.h>
-#include <ia_css_input_port.h>
-#include <ia_css_stream_format.h>
-#include <ia_css_stream_public.h>
-#include <system_global.h>
-#include "ia_css_isys_comm.h"
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-/**
- * Virtual Input System. (Input System 2401)
- */
-typedef input_system_cfg_t	ia_css_isys_descr_t;
-/* end of Virtual Input System */
-#endif
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-input_system_error_t ia_css_isys_init(void);
-void ia_css_isys_uninit(void);
-enum mipi_port_id ia_css_isys_port_to_mipi_port(
-	enum mipi_port_id api_port);
-#endif
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-
-/**
- * @brief Register one (virtual) stream. This is used to track when all
- * virtual streams are configured inside the input system. The CSI RX is
- * only started when all registered streams are configured.
- *
- * @param[in]	port		CSI port
- * @param[in]	isys_stream_id	Stream handle generated with ia_css_isys_generate_stream_id()
- *				Must be lower than SH_CSS_MAX_ISYS_CHANNEL_NODES
- * @return			IA_CSS_SUCCESS if successful, IA_CSS_ERR_INTERNAL_ERROR if
- *				there is already a stream registered with the same handle
- */
-enum ia_css_err ia_css_isys_csi_rx_register_stream(
-	enum mipi_port_id port,
-	uint32_t isys_stream_id);
-
-/**
- * @brief Unregister one (virtual) stream. This is used to track when all
- * virtual streams are configured inside the input system. The CSI RX is
- * only started when all registered streams are configured.
- *
- * @param[in]	port		CSI port
- * @param[in]	isys_stream_id	Stream handle generated with ia_css_isys_generate_stream_id()
- *				Must be lower than SH_CSS_MAX_ISYS_CHANNEL_NODES
- * @return			IA_CSS_SUCCESS if successful, IA_CSS_ERR_INTERNAL_ERROR if
- *				there is no stream registered with that handle
- */
-enum ia_css_err ia_css_isys_csi_rx_unregister_stream(
-	enum mipi_port_id port,
-	uint32_t isys_stream_id);
-
-enum ia_css_err ia_css_isys_convert_compressed_format(
-		struct ia_css_csi2_compression *comp,
-		struct input_system_cfg_s *cfg);
-unsigned int ia_css_csi2_calculate_input_system_alignment(
-	enum atomisp_input_format fmt_type);
-#endif
-
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
-/* CSS Receiver */
-void ia_css_isys_rx_configure(
-	const rx_cfg_t *config,
-	const enum ia_css_input_mode input_mode);
-
-void ia_css_isys_rx_disable(void);
-
-void ia_css_isys_rx_enable_all_interrupts(enum mipi_port_id port);
-
-unsigned int ia_css_isys_rx_get_interrupt_reg(enum mipi_port_id port);
-void ia_css_isys_rx_get_irq_info(enum mipi_port_id port,
-				 unsigned int *irq_infos);
-void ia_css_isys_rx_clear_irq_info(enum mipi_port_id port,
-				   unsigned int irq_infos);
-unsigned int ia_css_isys_rx_translate_irq_infos(unsigned int bits);
-
-#endif /* #if !defined(USE_INPUT_SYSTEM_VERSION_2401) */
-
-/* @brief Translate format and compression to format type.
- *
- * @param[in]	input_format	The input format.
- * @param[in]	compression	The compression scheme.
- * @param[out]	fmt_type	Pointer to the resulting format type.
- * @return			Error code.
- *
- * Translate an input format and mipi compression pair to the fmt_type.
- * This is normally done by the sensor, but when using the input fifo, this
- * format type must be sumitted correctly by the application.
- */
-enum ia_css_err ia_css_isys_convert_stream_format_to_mipi_format(
-		enum atomisp_input_format input_format,
-		mipi_predictor_t compression,
-		unsigned int *fmt_type);
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-/**
- * Virtual Input System. (Input System 2401)
- */
-extern ia_css_isys_error_t ia_css_isys_stream_create(
-		ia_css_isys_descr_t	*isys_stream_descr,
-		ia_css_isys_stream_h	isys_stream,
-		uint32_t isys_stream_id);
-
-extern void ia_css_isys_stream_destroy(
-		ia_css_isys_stream_h	isys_stream);
-
-extern ia_css_isys_error_t ia_css_isys_stream_calculate_cfg(
-		ia_css_isys_stream_h		isys_stream,
-		ia_css_isys_descr_t		*isys_stream_descr,
-		ia_css_isys_stream_cfg_t	*isys_stream_cfg);
-
-extern void ia_css_isys_csi_rx_lut_rmgr_init(void);
-
-extern void ia_css_isys_csi_rx_lut_rmgr_uninit(void);
-
-extern bool ia_css_isys_csi_rx_lut_rmgr_acquire(
-	csi_rx_backend_ID_t		backend,
-	csi_mipi_packet_type_t		packet_type,
-	csi_rx_backend_lut_entry_t	*entry);
-
-extern void ia_css_isys_csi_rx_lut_rmgr_release(
-	csi_rx_backend_ID_t		backend,
-	csi_mipi_packet_type_t		packet_type,
-	csi_rx_backend_lut_entry_t	*entry);
-
-
-extern void ia_css_isys_ibuf_rmgr_init(void);
-
-extern void ia_css_isys_ibuf_rmgr_uninit(void);
-
-extern bool ia_css_isys_ibuf_rmgr_acquire(
-	uint32_t	size,
-	uint32_t	*start_addr);
-
-extern void ia_css_isys_ibuf_rmgr_release(
-	uint32_t	*start_addr);
-
-extern void ia_css_isys_dma_channel_rmgr_init(void);
-
-extern void ia_css_isys_dma_channel_rmgr_uninit(void);
-
-extern bool ia_css_isys_dma_channel_rmgr_acquire(
-	isys2401_dma_ID_t	dma_id,
-	isys2401_dma_channel	*channel);
-
-extern void ia_css_isys_dma_channel_rmgr_release(
-	isys2401_dma_ID_t	dma_id,
-	isys2401_dma_channel	*channel);
-
-extern void ia_css_isys_stream2mmio_sid_rmgr_init(void);
-
-extern void ia_css_isys_stream2mmio_sid_rmgr_uninit(void);
-
-extern bool ia_css_isys_stream2mmio_sid_rmgr_acquire(
-	stream2mmio_ID_t	stream2mmio,
-	stream2mmio_sid_ID_t	*sid);
-
-extern void ia_css_isys_stream2mmio_sid_rmgr_release(
-	stream2mmio_ID_t	stream2mmio,
-	stream2mmio_sid_ID_t	*sid);
-
-/* end of Virtual Input System */
-#endif
-
-#endif				/* __IA_CSS_ISYS_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/interface/ia_css_isys_comm.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/interface/ia_css_isys_comm.h
deleted file mode 100644
index 0c3434ad0613..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/interface/ia_css_isys_comm.h
+++ /dev/null
@@ -1,69 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_ISYS_COMM_H
-#define __IA_CSS_ISYS_COMM_H
-
-#include <type_support.h>
-#include <input_system.h>
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-#include <platform_support.h>		/* inline */
-#include <input_system_global.h>
-#include <ia_css_stream_public.h>	/* IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH */
-
-#define SH_CSS_NODES_PER_THREAD		2
-#define SH_CSS_MAX_ISYS_CHANNEL_NODES	(SH_CSS_MAX_SP_THREADS * SH_CSS_NODES_PER_THREAD)
-
-/*
- * a) ia_css_isys_stream_h & ia_css_isys_stream_cfg_t come from host.
- *
- * b) Here it is better  to use actual structures for stream handle
- * instead of opaque handles. Otherwise, we need to have another
- * communication channel to interpret that opaque handle(this handle is
- * maintained by host and needs to be populated to sp for every stream open)
- * */
-typedef virtual_input_system_stream_t		*ia_css_isys_stream_h;
-typedef virtual_input_system_stream_cfg_t	ia_css_isys_stream_cfg_t;
-
-/*
- * error check for ISYS APIs.
- * */
-typedef bool ia_css_isys_error_t;
-
-static inline uint32_t ia_css_isys_generate_stream_id(
-	uint32_t	sp_thread_id,
-	uint32_t	stream_id)
-{
-	return sp_thread_id * IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH + stream_id;
-}
-
-#endif  /* USE_INPUT_SYSTEM_VERSION_2401*/
-#endif  /*_IA_CSS_ISYS_COMM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/csi_rx_rmgr.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/csi_rx_rmgr.c
deleted file mode 100644
index a914ce5532ec..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/csi_rx_rmgr.c
+++ /dev/null
@@ -1,179 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "system_global.h"
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-
-#include "assert_support.h"
-#include "platform_support.h"
-#include "ia_css_isys.h"
-#include "bitop_support.h"
-#include "ia_css_pipeline.h"	/* ia_css_pipeline_get_pipe_io_status() */
-#include "sh_css_internal.h"	/* sh_css_sp_pipeline_io_status
-				 * SH_CSS_MAX_SP_THREADS
-				 */
-#include "csi_rx_rmgr.h"
-
-static isys_csi_rx_rsrc_t  isys_csi_rx_rsrc[N_CSI_RX_BACKEND_ID];
-
-void ia_css_isys_csi_rx_lut_rmgr_init(void)
-{
-	memset(isys_csi_rx_rsrc, 0, sizeof(isys_csi_rx_rsrc));
-}
-
-void ia_css_isys_csi_rx_lut_rmgr_uninit(void)
-{
-	memset(isys_csi_rx_rsrc, 0, sizeof(isys_csi_rx_rsrc));
-}
-
-bool ia_css_isys_csi_rx_lut_rmgr_acquire(
-	csi_rx_backend_ID_t		backend,
-	csi_mipi_packet_type_t		packet_type,
-	csi_rx_backend_lut_entry_t	*entry)
-{
-	bool retval = false;
-	uint32_t max_num_packets_of_type;
-	uint32_t num_active_of_type;
-	isys_csi_rx_rsrc_t *cur_rsrc = NULL;
-	uint16_t i;
-
-	assert(backend < N_CSI_RX_BACKEND_ID);
-	assert((packet_type == CSI_MIPI_PACKET_TYPE_LONG) || (packet_type == CSI_MIPI_PACKET_TYPE_SHORT));
-	assert(entry != NULL);
-
-	if ((backend < N_CSI_RX_BACKEND_ID) && (entry != NULL)) {
-		cur_rsrc = &isys_csi_rx_rsrc[backend];
-		if (packet_type == CSI_MIPI_PACKET_TYPE_LONG) {
-			max_num_packets_of_type = N_LONG_PACKET_LUT_ENTRIES[backend];
-			num_active_of_type = cur_rsrc->num_long_packets;
-		} else {
-			max_num_packets_of_type = N_SHORT_PACKET_LUT_ENTRIES[backend];
-			num_active_of_type = cur_rsrc->num_short_packets;
-		}
-
-		if (num_active_of_type < max_num_packets_of_type) {
-			for (i = 0; i < max_num_packets_of_type; i++) {
-				if (bitop_getbit(cur_rsrc->active_table, i) == 0) {
-					bitop_setbit(cur_rsrc->active_table, i);
-
-					if (packet_type == CSI_MIPI_PACKET_TYPE_LONG) {
-						entry->long_packet_entry = i;
-						entry->short_packet_entry = 0;
-						cur_rsrc->num_long_packets++;
-					} else {
-						entry->long_packet_entry = 0;
-						entry->short_packet_entry = i;
-						cur_rsrc->num_short_packets++;
-					}
-					cur_rsrc->num_active++;
-					retval = true;
-					break;
-				}
-			}
-		}
-	}
-	return retval;
-}
-
-void ia_css_isys_csi_rx_lut_rmgr_release(
-	csi_rx_backend_ID_t		backend,
-	csi_mipi_packet_type_t		packet_type,
-	csi_rx_backend_lut_entry_t	*entry)
-{
-	uint32_t max_num_packets;
-	isys_csi_rx_rsrc_t *cur_rsrc = NULL;
-	uint32_t packet_entry = 0;
-
-	assert(backend < N_CSI_RX_BACKEND_ID);
-	assert(entry != NULL);
-	assert((packet_type >= CSI_MIPI_PACKET_TYPE_LONG) || (packet_type <= CSI_MIPI_PACKET_TYPE_SHORT));
-
-	if ((backend < N_CSI_RX_BACKEND_ID) && (entry != NULL)) {
-		if (packet_type == CSI_MIPI_PACKET_TYPE_LONG) {
-			max_num_packets = N_LONG_PACKET_LUT_ENTRIES[backend];
-			packet_entry = entry->long_packet_entry;
-		} else {
-			max_num_packets = N_SHORT_PACKET_LUT_ENTRIES[backend];
-			packet_entry = entry->short_packet_entry;
-		}
-
-		cur_rsrc = &isys_csi_rx_rsrc[backend];
-		if ((packet_entry < max_num_packets) && (cur_rsrc->num_active > 0)) {
-			if (bitop_getbit(cur_rsrc->active_table, packet_entry) == 1) {
-				bitop_clearbit(cur_rsrc->active_table, packet_entry);
-
-				if (packet_type == CSI_MIPI_PACKET_TYPE_LONG)
-					cur_rsrc->num_long_packets--;
-				else
-					cur_rsrc->num_short_packets--;
-				cur_rsrc->num_active--;
-			}
-		}
-	}
-}
-
-enum ia_css_err ia_css_isys_csi_rx_register_stream(
-	enum mipi_port_id port,
-	uint32_t isys_stream_id)
-{
-	enum ia_css_err retval = IA_CSS_ERR_INTERNAL_ERROR;
-
-	if ((port < N_INPUT_SYSTEM_CSI_PORT) &&
-	    (isys_stream_id < SH_CSS_MAX_ISYS_CHANNEL_NODES)) {
-		struct sh_css_sp_pipeline_io_status *pipe_io_status;
-		pipe_io_status = ia_css_pipeline_get_pipe_io_status();
-		if (bitop_getbit(pipe_io_status->active[port], isys_stream_id) == 0) {
-			bitop_setbit(pipe_io_status->active[port], isys_stream_id);
-			pipe_io_status->running[port] = 0;
-			retval = IA_CSS_SUCCESS;
-		}
-	}
-	return retval;
-}
-
-enum ia_css_err ia_css_isys_csi_rx_unregister_stream(
-	enum mipi_port_id port,
-	uint32_t isys_stream_id)
-{
-	enum ia_css_err retval = IA_CSS_ERR_INTERNAL_ERROR;
-
-	if ((port < N_INPUT_SYSTEM_CSI_PORT) &&
-	    (isys_stream_id < SH_CSS_MAX_ISYS_CHANNEL_NODES)) {
-		struct sh_css_sp_pipeline_io_status *pipe_io_status;
-		pipe_io_status = ia_css_pipeline_get_pipe_io_status();
-		if (bitop_getbit(pipe_io_status->active[port], isys_stream_id) == 1) {
-			bitop_clearbit(pipe_io_status->active[port], isys_stream_id);
-			retval = IA_CSS_SUCCESS;
-		}
-	}
-	return retval;
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/csi_rx_rmgr.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/csi_rx_rmgr.h
deleted file mode 100644
index c27b0ab83c93..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/csi_rx_rmgr.h
+++ /dev/null
@@ -1,43 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __CSI_RX_RMGR_H_INCLUDED__
-#define __CSI_RX_RMGR_H_INCLUDED__
-
-typedef struct isys_csi_rx_rsrc_s isys_csi_rx_rsrc_t;
-struct isys_csi_rx_rsrc_s {
-	uint32_t	active_table;
-	uint32_t        num_active;
-	uint16_t	num_long_packets;
-	uint16_t	num_short_packets;
-};
-
-#endif /* __CSI_RX_RMGR_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.c
deleted file mode 100644
index d8c3b75d7fac..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.c
+++ /dev/null
@@ -1,140 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010 - 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#endif
-
-#include "system_global.h"
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-
-#include "assert_support.h"
-#include "platform_support.h"
-#include "ia_css_isys.h"
-#include "ibuf_ctrl_rmgr.h"
-
-static ibuf_rsrc_t	ibuf_rsrc;
-
-static ibuf_handle_t *getHandle(uint16_t index)
-{
-	ibuf_handle_t *handle = NULL;
-
-	if (index < MAX_IBUF_HANDLES)
-		handle = &ibuf_rsrc.handles[index];
-	return handle;
-}
-
-void ia_css_isys_ibuf_rmgr_init(void)
-{
-	memset(&ibuf_rsrc, 0, sizeof(ibuf_rsrc));
-	ibuf_rsrc.free_size = MAX_INPUT_BUFFER_SIZE;
-}
-
-void ia_css_isys_ibuf_rmgr_uninit(void)
-{
-	memset(&ibuf_rsrc, 0, sizeof(ibuf_rsrc));
-	ibuf_rsrc.free_size = MAX_INPUT_BUFFER_SIZE;
-}
-
-bool ia_css_isys_ibuf_rmgr_acquire(
-	uint32_t	size,
-	uint32_t	*start_addr)
-{
-	bool retval = false;
-	bool input_buffer_found = false;
-	uint32_t aligned_size;
-	ibuf_handle_t *handle = NULL;
-	uint16_t i;
-
-	assert(start_addr != NULL);
-	assert(size > 0);
-
-	aligned_size = (size + (IBUF_ALIGN - 1)) & ~(IBUF_ALIGN - 1);
-
-	/* Check if there is an available un-used handle with the size
-	 * that will fulfill the request.
-	 */
-	if (ibuf_rsrc.num_active < ibuf_rsrc.num_allocated) {
-		for (i = 0; i < ibuf_rsrc.num_allocated; i++) {
-			handle = getHandle(i);
-			if (!handle->active) {
-				if (handle->size >= aligned_size) {
-					handle->active = true;
-					input_buffer_found = true;
-					ibuf_rsrc.num_active++;
-					break;
-				}
-			}
-		}
-	}
-
-	if (!input_buffer_found) {
-		/* There were no available handles that fulfilled the
-		 * request. Allocate a new handle with the requested size.
-		 */
-		if ((ibuf_rsrc.num_allocated < MAX_IBUF_HANDLES) &&
-		    (ibuf_rsrc.free_size >= aligned_size)) {
-			handle = getHandle(ibuf_rsrc.num_allocated);
-			handle->start_addr	= ibuf_rsrc.free_start_addr;
-			handle->size		= aligned_size;
-			handle->active		= true;
-
-			ibuf_rsrc.free_start_addr += aligned_size;
-			ibuf_rsrc.free_size -= aligned_size;
-			ibuf_rsrc.num_active++;
-			ibuf_rsrc.num_allocated++;
-
-			input_buffer_found = true;
-		}
-	}
-
-	if (input_buffer_found && handle) {
-		*start_addr = handle->start_addr;
-		retval = true;
-	}
-
-	return retval;
-}
-
-void ia_css_isys_ibuf_rmgr_release(
-	uint32_t	*start_addr)
-{
-	uint16_t i;
-	ibuf_handle_t *handle = NULL;
-
-	assert(start_addr != NULL);
-
-	for (i = 0; i < ibuf_rsrc.num_allocated; i++) {
-		handle = getHandle(i);
-		if (handle->active && handle->start_addr == *start_addr) {
-			handle->active = false;
-			ibuf_rsrc.num_active--;
-			break;
-		}
-	}
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.h
deleted file mode 100644
index 424cfe9f3b2a..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.h
+++ /dev/null
@@ -1,55 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IBUF_CTRL_RMGR_H_INCLUDED__
-#define __IBUF_CTRL_RMGR_H_INCLUDED__
-
-#define MAX_IBUF_HANDLES	24
-#define MAX_INPUT_BUFFER_SIZE	(64 * 1024)
-#define IBUF_ALIGN		8
-
-typedef struct ibuf_handle_s ibuf_handle_t;
-struct ibuf_handle_s {
-	uint32_t	start_addr;
-	uint32_t	size;
-	bool		active;
-};
-
-typedef struct ibuf_rsrc_s ibuf_rsrc_t;
-struct ibuf_rsrc_s {
-	uint32_t	free_start_addr;
-	uint32_t	free_size;
-	uint16_t	num_active;
-	uint16_t	num_allocated;
-	ibuf_handle_t	handles[MAX_IBUF_HANDLES];
-};
-
-#endif /* __IBUF_CTRL_RMGR_H_INCLUDED */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_dma_rmgr.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_dma_rmgr.c
deleted file mode 100644
index 4def4a542b7d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_dma_rmgr.c
+++ /dev/null
@@ -1,103 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "system_global.h"
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-
-#include "assert_support.h"
-#include "platform_support.h"
-#include "ia_css_isys.h"
-#include "bitop_support.h"
-#include "isys_dma_rmgr.h"
-
-static isys_dma_rsrc_t isys_dma_rsrc[N_ISYS2401_DMA_ID];
-
-void ia_css_isys_dma_channel_rmgr_init(void)
-{
-	memset(&isys_dma_rsrc, 0, sizeof(isys_dma_rsrc_t));
-}
-
-void ia_css_isys_dma_channel_rmgr_uninit(void)
-{
-	memset(&isys_dma_rsrc, 0, sizeof(isys_dma_rsrc_t));
-}
-
-bool ia_css_isys_dma_channel_rmgr_acquire(
-	isys2401_dma_ID_t	dma_id,
-	isys2401_dma_channel	*channel)
-{
-	bool retval = false;
-	isys2401_dma_channel	i;
-	isys2401_dma_channel	max_dma_channel;
-	isys_dma_rsrc_t		*cur_rsrc = NULL;
-
-	assert(dma_id < N_ISYS2401_DMA_ID);
-	assert(channel != NULL);
-
-	max_dma_channel = N_ISYS2401_DMA_CHANNEL_PROCS[dma_id];
-	cur_rsrc = &isys_dma_rsrc[dma_id];
-
-	if (cur_rsrc->num_active < max_dma_channel) {
-		for (i = ISYS2401_DMA_CHANNEL_0; i < N_ISYS2401_DMA_CHANNEL; i++) {
-			if (bitop_getbit(cur_rsrc->active_table, i) == 0) {
-				bitop_setbit(cur_rsrc->active_table, i);
-				*channel = i;
-				cur_rsrc->num_active++;
-				retval = true;
-				break;
-			}
-		}
-	}
-
-	return retval;
-}
-
-void ia_css_isys_dma_channel_rmgr_release(
-	isys2401_dma_ID_t	dma_id,
-	isys2401_dma_channel	*channel)
-{
-	isys2401_dma_channel	max_dma_channel;
-	isys_dma_rsrc_t		*cur_rsrc = NULL;
-
-	assert(dma_id < N_ISYS2401_DMA_ID);
-	assert(channel != NULL);
-
-	max_dma_channel = N_ISYS2401_DMA_CHANNEL_PROCS[dma_id];
-	cur_rsrc = &isys_dma_rsrc[dma_id];
-
-	if ((*channel < max_dma_channel) && (cur_rsrc->num_active > 0)) {
-		if (bitop_getbit(cur_rsrc->active_table, *channel) == 1) {
-			bitop_clearbit(cur_rsrc->active_table, *channel);
-			cur_rsrc->num_active--;
-		}
-	}
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_dma_rmgr.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_dma_rmgr.h
deleted file mode 100644
index b2c286537774..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_dma_rmgr.h
+++ /dev/null
@@ -1,41 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __ISYS_DMA_RMGR_H_INCLUDED__
-#define __ISYS_DMA_RMGR_H_INCLUDED__
-
-typedef struct isys_dma_rsrc_s	isys_dma_rsrc_t;
-struct isys_dma_rsrc_s {
-	uint32_t active_table;
-	uint16_t num_active;
-};
-
-#endif /* __ISYS_DMA_RMGR_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_init.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_init.c
deleted file mode 100644
index 2ae5e59d5e31..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_init.c
+++ /dev/null
@@ -1,139 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "input_system.h"
-
-#ifdef HAS_INPUT_SYSTEM_VERSION_2
-#include "ia_css_isys.h"
-#include "platform_support.h"
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-#include "isys_dma.h"		/* isys2401_dma_set_max_burst_size() */
-#include "isys_irq.h"
-#endif
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-input_system_error_t ia_css_isys_init(void)
-{
-	backend_channel_cfg_t backend_ch0;
-	backend_channel_cfg_t backend_ch1;
-	target_cfg2400_t targetB;
-	target_cfg2400_t targetC;
-	uint32_t acq_mem_region_size = 24;
-	uint32_t acq_nof_mem_regions = 2;
-	input_system_error_t error = INPUT_SYSTEM_ERR_NO_ERROR;
-
-	memset(&backend_ch0, 0, sizeof(backend_channel_cfg_t));
-	memset(&backend_ch1, 0, sizeof(backend_channel_cfg_t));
-	memset(&targetB, 0, sizeof(targetB));
-	memset(&targetC, 0, sizeof(targetC));
-
-	error = input_system_configuration_reset();
-	if (error != INPUT_SYSTEM_ERR_NO_ERROR)
-		return error;
-
-	error = input_system_csi_xmem_channel_cfg(
-			0,			/*ch_id                 */
-			INPUT_SYSTEM_PORT_A,	/*port                  */
-			backend_ch0,		/*backend_ch            */
-			32,			/*mem_region_size       */
-			6,			/*nof_mem_regions       */
-			acq_mem_region_size,	/*acq_mem_region_size   */
-			acq_nof_mem_regions,	/*acq_nof_mem_regions   */
-			targetB,		/*target                */
-			3);			/*nof_xmem_buffers      */
-	if (error != INPUT_SYSTEM_ERR_NO_ERROR)
-		return error;
-
-	error = input_system_csi_xmem_channel_cfg(
-			1,			/*ch_id                 */
-			INPUT_SYSTEM_PORT_B,	/*port                  */
-			backend_ch0,		/*backend_ch            */
-			16,			/*mem_region_size       */
-			3,			/*nof_mem_regions       */
-			acq_mem_region_size,	/*acq_mem_region_size   */
-			acq_nof_mem_regions,	/*acq_nof_mem_regions   */
-			targetB,		/*target                */
-			3);			/*nof_xmem_buffers      */
-	if (error != INPUT_SYSTEM_ERR_NO_ERROR)
-		return error;
-
-	error = input_system_csi_xmem_channel_cfg(
-			2,			/*ch_id                 */
-			INPUT_SYSTEM_PORT_C,	/*port                  */
-			backend_ch1,		/*backend_ch            */
-			32,			/*mem_region_size       */
-			3,			/*nof_mem_regions       */
-			acq_mem_region_size,	/*acq_mem_region_size   */
-			acq_nof_mem_regions,	/*acq_nof_mem_regions   */
-			targetC,		/*target                */
-			2);			/*nof_xmem_buffers      */
-	if (error != INPUT_SYSTEM_ERR_NO_ERROR)
-		return error;
-
-	error = input_system_configuration_commit();
-
-	return error;
-}
-#elif defined(USE_INPUT_SYSTEM_VERSION_2401)
-input_system_error_t ia_css_isys_init(void)
-{
-	ia_css_isys_csi_rx_lut_rmgr_init();
-	ia_css_isys_ibuf_rmgr_init();
-	ia_css_isys_dma_channel_rmgr_init();
-	ia_css_isys_stream2mmio_sid_rmgr_init();
-
-	isys2401_dma_set_max_burst_size(ISYS2401_DMA0_ID,
-		1 /* Non Burst DMA transactions */);
-
-	/* Enable 2401 input system IRQ status for driver to retrieve */
-	isys_irqc_status_enable(ISYS_IRQ0_ID);
-	isys_irqc_status_enable(ISYS_IRQ1_ID);
-	isys_irqc_status_enable(ISYS_IRQ2_ID);
-
-	return INPUT_SYSTEM_ERR_NO_ERROR;
-}
-#endif
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-void ia_css_isys_uninit(void)
-{
-}
-#elif defined(USE_INPUT_SYSTEM_VERSION_2401)
-void ia_css_isys_uninit(void)
-{
-	ia_css_isys_csi_rx_lut_rmgr_uninit();
-	ia_css_isys_ibuf_rmgr_uninit();
-	ia_css_isys_dma_channel_rmgr_uninit();
-	ia_css_isys_stream2mmio_sid_rmgr_uninit();
-}
-#endif
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_stream2mmio_rmgr.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_stream2mmio_rmgr.c
deleted file mode 100644
index 222b294c0ab0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_stream2mmio_rmgr.c
+++ /dev/null
@@ -1,105 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "system_global.h"
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-
-#include "assert_support.h"
-#include "platform_support.h"
-#include "ia_css_isys.h"
-#include "bitop_support.h"
-#include "isys_stream2mmio_rmgr.h"
-
-static isys_stream2mmio_rsrc_t	isys_stream2mmio_rsrc[N_STREAM2MMIO_ID];
-
-void ia_css_isys_stream2mmio_sid_rmgr_init(void)
-{
-	memset(isys_stream2mmio_rsrc, 0, sizeof(isys_stream2mmio_rsrc));
-}
-
-void ia_css_isys_stream2mmio_sid_rmgr_uninit(void)
-{
-	memset(isys_stream2mmio_rsrc, 0, sizeof(isys_stream2mmio_rsrc));
-}
-
-bool ia_css_isys_stream2mmio_sid_rmgr_acquire(
-	stream2mmio_ID_t	stream2mmio,
-	stream2mmio_sid_ID_t	*sid)
-{
-	bool retval = false;
-	stream2mmio_sid_ID_t max_sid;
-	isys_stream2mmio_rsrc_t *cur_rsrc = NULL;
-	stream2mmio_sid_ID_t	i;
-
-	assert(stream2mmio < N_STREAM2MMIO_ID);
-	assert(sid != NULL);
-
-	if ((stream2mmio < N_STREAM2MMIO_ID) && (sid != NULL)) {
-		max_sid = N_STREAM2MMIO_SID_PROCS[stream2mmio];
-		cur_rsrc = &isys_stream2mmio_rsrc[stream2mmio];
-
-		if (cur_rsrc->num_active < max_sid) {
-			for (i = STREAM2MMIO_SID0_ID; i < max_sid; i++) {
-				if (bitop_getbit(cur_rsrc->active_table, i) == 0) {
-					bitop_setbit(cur_rsrc->active_table, i);
-					*sid = i;
-					cur_rsrc->num_active++;
-					retval = true;
-					break;
-				}
-			}
-		}
-	}
-	return retval;
-}
-
-void ia_css_isys_stream2mmio_sid_rmgr_release(
-	stream2mmio_ID_t	stream2mmio,
-	stream2mmio_sid_ID_t	*sid)
-{
-	stream2mmio_sid_ID_t max_sid;
-	isys_stream2mmio_rsrc_t *cur_rsrc = NULL;
-
-	assert(stream2mmio < N_STREAM2MMIO_ID);
-	assert(sid != NULL);
-
-	if ((stream2mmio < N_STREAM2MMIO_ID) && (sid != NULL)) {
-		max_sid = N_STREAM2MMIO_SID_PROCS[stream2mmio];
-		cur_rsrc = &isys_stream2mmio_rsrc[stream2mmio];
-		if ((*sid < max_sid) && (cur_rsrc->num_active > 0)) {
-			if (bitop_getbit(cur_rsrc->active_table, *sid) == 1) {
-				bitop_clearbit(cur_rsrc->active_table, *sid);
-				cur_rsrc->num_active--;
-			}
-		}
-	}
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_stream2mmio_rmgr.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_stream2mmio_rmgr.h
deleted file mode 100644
index 4f63005b1071..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/isys_stream2mmio_rmgr.h
+++ /dev/null
@@ -1,41 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __ISYS_STREAM2MMIO_RMGR_H_INCLUDED__
-#define __ISYS_STREAM2MMIO_RMGR_H_INCLUDED__
-
-typedef struct isys_stream2mmio_rsrc_s isys_stream2mmio_rsrc_t;
-struct isys_stream2mmio_rsrc_s {
-	uint32_t	active_table;
-	uint16_t	num_active;
-};
-
-#endif /* __ISYS_STREAM2MMIO_RMGR_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/rx.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/rx.c
deleted file mode 100644
index 425bd3cc3f34..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/rx.c
+++ /dev/null
@@ -1,607 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#define __INLINE_INPUT_SYSTEM__
-#include "input_system.h"
-#include "assert_support.h"
-#include "ia_css_isys.h"
-#include "ia_css_irq.h"
-#include "sh_css_internal.h"
-
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
-void ia_css_isys_rx_enable_all_interrupts(enum mipi_port_id port)
-{
-	hrt_data bits = receiver_port_reg_load(RX0_ID,
-				port,
-				_HRT_CSS_RECEIVER_IRQ_ENABLE_REG_IDX);
-
-	bits |= (1U << _HRT_CSS_RECEIVER_IRQ_OVERRUN_BIT) |
-#if defined(HAS_RX_VERSION_2)
-	    (1U << _HRT_CSS_RECEIVER_IRQ_INIT_TIMEOUT_BIT) |
-#endif
-	    (1U << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_ENTRY_BIT) |
-	    (1U << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_EXIT_BIT) |
-	    (1U << _HRT_CSS_RECEIVER_IRQ_ERR_SOT_HS_BIT) |
-	    (1U << _HRT_CSS_RECEIVER_IRQ_ERR_SOT_SYNC_HS_BIT) |
-	    (1U << _HRT_CSS_RECEIVER_IRQ_ERR_CONTROL_BIT) |
-	    (1U << _HRT_CSS_RECEIVER_IRQ_ERR_ECC_DOUBLE_BIT) |
-	    (1U << _HRT_CSS_RECEIVER_IRQ_ERR_ECC_CORRECTED_BIT) |
-	    /*(1U << _HRT_CSS_RECEIVER_IRQ_ERR_ECC_NO_CORRECTION_BIT) | */
-	    (1U << _HRT_CSS_RECEIVER_IRQ_ERR_CRC_BIT) |
-	    (1U << _HRT_CSS_RECEIVER_IRQ_ERR_ID_BIT) |
-	    (1U << _HRT_CSS_RECEIVER_IRQ_ERR_FRAME_SYNC_BIT) |
-	    (1U << _HRT_CSS_RECEIVER_IRQ_ERR_FRAME_DATA_BIT) |
-	    (1U << _HRT_CSS_RECEIVER_IRQ_DATA_TIMEOUT_BIT) |
-	    (1U << _HRT_CSS_RECEIVER_IRQ_ERR_ESCAPE_BIT);
-	/*(1U << _HRT_CSS_RECEIVER_IRQ_ERR_LINE_SYNC_BIT); */
-
-	receiver_port_reg_store(RX0_ID,
-				port,
-				_HRT_CSS_RECEIVER_IRQ_ENABLE_REG_IDX, bits);
-
-	/*
-	 * The CSI is nested into the Iunit IRQ's
-	 */
-	ia_css_irq_enable(IA_CSS_IRQ_INFO_CSS_RECEIVER_ERROR, true);
-
-	return;
-}
-
-/* This function converts between the enum used on the CSS API and the
- * internal DLI enum type.
- * We do not use an array for this since we cannot use named array
- * initializers in Windows. Without that there is no easy way to guarantee
- * that the array values would be in the correct order.
- * */
-enum mipi_port_id ia_css_isys_port_to_mipi_port(enum mipi_port_id api_port)
-{
-	/* In this module the validity of the inptu variable should
-	 * have been checked already, so we do not check for erroneous
-	 * values. */
-	enum mipi_port_id port = MIPI_PORT0_ID;
-
-	if (api_port == MIPI_PORT1_ID)
-		port = MIPI_PORT1_ID;
-	else if (api_port == MIPI_PORT2_ID)
-		port = MIPI_PORT2_ID;
-
-	return port;
-}
-
-unsigned int ia_css_isys_rx_get_interrupt_reg(enum mipi_port_id port)
-{
-	return receiver_port_reg_load(RX0_ID,
-				      port,
-				      _HRT_CSS_RECEIVER_IRQ_STATUS_REG_IDX);
-}
-
-void ia_css_rx_get_irq_info(unsigned int *irq_infos)
-{
-	ia_css_rx_port_get_irq_info(MIPI_PORT1_ID, irq_infos);
-}
-
-void ia_css_rx_port_get_irq_info(enum mipi_port_id api_port,
-				 unsigned int *irq_infos)
-{
-	enum mipi_port_id port = ia_css_isys_port_to_mipi_port(api_port);
-	ia_css_isys_rx_get_irq_info(port, irq_infos);
-}
-
-void ia_css_isys_rx_get_irq_info(enum mipi_port_id port,
-				 unsigned int *irq_infos)
-{
-	unsigned int bits;
-
-	assert(irq_infos != NULL);
-	bits = ia_css_isys_rx_get_interrupt_reg(port);
-	*irq_infos = ia_css_isys_rx_translate_irq_infos(bits);
-}
-
-/* Translate register bits to CSS API enum mask */
-unsigned int ia_css_isys_rx_translate_irq_infos(unsigned int bits)
-{
-	unsigned int infos = 0;
-
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_OVERRUN_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_BUFFER_OVERRUN;
-#if defined(HAS_RX_VERSION_2)
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_INIT_TIMEOUT_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_INIT_TIMEOUT;
-#endif
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_ENTRY_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ENTER_SLEEP_MODE;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_EXIT_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_EXIT_SLEEP_MODE;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_ERR_ECC_CORRECTED_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ECC_CORRECTED;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_ERR_SOT_HS_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ERR_SOT;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_ERR_SOT_SYNC_HS_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ERR_SOT_SYNC;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_ERR_CONTROL_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ERR_CONTROL;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_ERR_ECC_DOUBLE_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_ERR_CRC_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ERR_CRC;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_ERR_ID_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_ERR_FRAME_SYNC_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ERR_FRAME_SYNC;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_ERR_FRAME_DATA_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ERR_FRAME_DATA;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_DATA_TIMEOUT_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_ERR_ESCAPE_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC;
-	if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_ERR_LINE_SYNC_BIT))
-		infos |= IA_CSS_RX_IRQ_INFO_ERR_LINE_SYNC;
-
-	return infos;
-}
-
-void ia_css_rx_clear_irq_info(unsigned int irq_infos)
-{
-	ia_css_rx_port_clear_irq_info(MIPI_PORT1_ID, irq_infos);
-}
-
-void ia_css_rx_port_clear_irq_info(enum mipi_port_id api_port, unsigned int irq_infos)
-{
-	enum mipi_port_id port = ia_css_isys_port_to_mipi_port(api_port);
-	ia_css_isys_rx_clear_irq_info(port, irq_infos);
-}
-
-void ia_css_isys_rx_clear_irq_info(enum mipi_port_id port, unsigned int irq_infos)
-{
-	hrt_data bits = receiver_port_reg_load(RX0_ID,
-				port,
-				_HRT_CSS_RECEIVER_IRQ_ENABLE_REG_IDX);
-
-	/* MW: Why do we remap the receiver bitmap */
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_BUFFER_OVERRUN)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_OVERRUN_BIT;
-#if defined(HAS_RX_VERSION_2)
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_INIT_TIMEOUT)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_INIT_TIMEOUT_BIT;
-#endif
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ENTER_SLEEP_MODE)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_ENTRY_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_EXIT_SLEEP_MODE)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_EXIT_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ECC_CORRECTED)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_ERR_ECC_CORRECTED_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ERR_SOT)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_ERR_SOT_HS_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ERR_SOT_SYNC)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_ERR_SOT_SYNC_HS_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ERR_CONTROL)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_ERR_CONTROL_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_ERR_ECC_DOUBLE_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ERR_CRC)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_ERR_CRC_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_ERR_ID_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ERR_FRAME_SYNC)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_ERR_FRAME_SYNC_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ERR_FRAME_DATA)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_ERR_FRAME_DATA_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_DATA_TIMEOUT_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_ERR_ESCAPE_BIT;
-	if (irq_infos & IA_CSS_RX_IRQ_INFO_ERR_LINE_SYNC)
-		bits |= 1U << _HRT_CSS_RECEIVER_IRQ_ERR_LINE_SYNC_BIT;
-
-	receiver_port_reg_store(RX0_ID,
-				port,
-				_HRT_CSS_RECEIVER_IRQ_ENABLE_REG_IDX, bits);
-
-	return;
-}
-#endif /* #if !defined(USE_INPUT_SYSTEM_VERSION_2401) */
-
-enum ia_css_err ia_css_isys_convert_stream_format_to_mipi_format(
-		enum atomisp_input_format input_format,
-		mipi_predictor_t compression,
-		unsigned int *fmt_type)
-{
-	assert(fmt_type != NULL);
-	/*
-	 * Custom (user defined) modes. Used for compressed
-	 * MIPI transfers
-	 *
-	 * Checkpatch thinks the indent before "if" is suspect
-	 * I think the only suspect part is the missing "else"
-	 * because of the return.
-	 */
-	if (compression != MIPI_PREDICTOR_NONE) {
-		switch (input_format) {
-		case ATOMISP_INPUT_FORMAT_RAW_6:
-			*fmt_type = 6;
-			break;
-		case ATOMISP_INPUT_FORMAT_RAW_7:
-			*fmt_type = 7;
-			break;
-		case ATOMISP_INPUT_FORMAT_RAW_8:
-			*fmt_type = 8;
-			break;
-		case ATOMISP_INPUT_FORMAT_RAW_10:
-			*fmt_type = 10;
-			break;
-		case ATOMISP_INPUT_FORMAT_RAW_12:
-			*fmt_type = 12;
-			break;
-		case ATOMISP_INPUT_FORMAT_RAW_14:
-			*fmt_type = 14;
-			break;
-		case ATOMISP_INPUT_FORMAT_RAW_16:
-			*fmt_type = 16;
-			break;
-		default:
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		}
-		return IA_CSS_SUCCESS;
-	}
-	/*
-	 * This mapping comes from the Arasan CSS function spec
-	 * (CSS_func_spec1.08_ahb_sep29_08.pdf).
-	 *
-	 * MW: For some reason the mapping is not 1-to-1
-	 */
-	switch (input_format) {
-	case ATOMISP_INPUT_FORMAT_RGB_888:
-		*fmt_type = MIPI_FORMAT_RGB888;
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_555:
-		*fmt_type = MIPI_FORMAT_RGB555;
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_444:
-		*fmt_type = MIPI_FORMAT_RGB444;
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_565:
-		*fmt_type = MIPI_FORMAT_RGB565;
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_666:
-		*fmt_type = MIPI_FORMAT_RGB666;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_8:
-		*fmt_type = MIPI_FORMAT_RAW8;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_10:
-		*fmt_type = MIPI_FORMAT_RAW10;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_6:
-		*fmt_type = MIPI_FORMAT_RAW6;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_7:
-		*fmt_type = MIPI_FORMAT_RAW7;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_12:
-		*fmt_type = MIPI_FORMAT_RAW12;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_14:
-		*fmt_type = MIPI_FORMAT_RAW14;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_8:
-		*fmt_type = MIPI_FORMAT_YUV420_8;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_10:
-		*fmt_type = MIPI_FORMAT_YUV420_10;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV422_8:
-		*fmt_type = MIPI_FORMAT_YUV422_8;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV422_10:
-		*fmt_type = MIPI_FORMAT_YUV422_10;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY:
-		*fmt_type = MIPI_FORMAT_YUV420_8_LEGACY;
-		break;
-	case ATOMISP_INPUT_FORMAT_EMBEDDED:
-		*fmt_type = MIPI_FORMAT_EMBEDDED;
-		break;
-#ifndef USE_INPUT_SYSTEM_VERSION_2401
-	case ATOMISP_INPUT_FORMAT_RAW_16:
-		/* This is not specified by Arasan, so we use
-		 * 17 for now.
-		 */
-		*fmt_type = MIPI_FORMAT_RAW16;
-		break;
-	case ATOMISP_INPUT_FORMAT_BINARY_8:
-		*fmt_type = MIPI_FORMAT_BINARY_8;
-		break;
-#else
-	case ATOMISP_INPUT_FORMAT_USER_DEF1:
-		*fmt_type = MIPI_FORMAT_CUSTOM0;
-		break;
-	case ATOMISP_INPUT_FORMAT_USER_DEF2:
-		*fmt_type = MIPI_FORMAT_CUSTOM1;
-		break;
-	case ATOMISP_INPUT_FORMAT_USER_DEF3:
-		*fmt_type = MIPI_FORMAT_CUSTOM2;
-		break;
-	case ATOMISP_INPUT_FORMAT_USER_DEF4:
-		*fmt_type = MIPI_FORMAT_CUSTOM3;
-		break;
-	case ATOMISP_INPUT_FORMAT_USER_DEF5:
-		*fmt_type = MIPI_FORMAT_CUSTOM4;
-		break;
-	case ATOMISP_INPUT_FORMAT_USER_DEF6:
-		*fmt_type = MIPI_FORMAT_CUSTOM5;
-		break;
-	case ATOMISP_INPUT_FORMAT_USER_DEF7:
-		*fmt_type = MIPI_FORMAT_CUSTOM6;
-		break;
-	case ATOMISP_INPUT_FORMAT_USER_DEF8:
-		*fmt_type = MIPI_FORMAT_CUSTOM7;
-		break;
-#endif
-
-	case ATOMISP_INPUT_FORMAT_YUV420_16:
-	case ATOMISP_INPUT_FORMAT_YUV422_16:
-	default:
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-	return IA_CSS_SUCCESS;
-}
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-static mipi_predictor_t sh_css_csi2_compression_type_2_mipi_predictor(enum ia_css_csi2_compression_type type)
-{
-	mipi_predictor_t predictor = MIPI_PREDICTOR_NONE;
-
-	switch (type) {
-	case IA_CSS_CSI2_COMPRESSION_TYPE_1:
-		predictor = MIPI_PREDICTOR_TYPE1-1;
-		break;
-	case IA_CSS_CSI2_COMPRESSION_TYPE_2:
-		predictor = MIPI_PREDICTOR_TYPE2-1;
-	default:
-		break;
-	}
-	return predictor;
-}
-enum ia_css_err ia_css_isys_convert_compressed_format(
-		struct ia_css_csi2_compression *comp,
-		struct input_system_cfg_s *cfg)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	assert(comp != NULL);
-	assert(cfg != NULL);
-
-	if (comp->type != IA_CSS_CSI2_COMPRESSION_TYPE_NONE) {
-		/* compression register bit slicing
-		4 bit for each user defined data type
-			3 bit indicate compression scheme
-				000 No compression
-				001 10-6-10
-				010 10-7-10
-				011 10-8-10
-				100 12-6-12
-				101 12-6-12
-				100 12-7-12
-				110 12-8-12
-			1 bit indicate predictor
-		*/
-		if (comp->uncompressed_bits_per_pixel == UNCOMPRESSED_BITS_PER_PIXEL_10) {
-			switch (comp->compressed_bits_per_pixel) {
-			case COMPRESSED_BITS_PER_PIXEL_6:
-				cfg->csi_port_attr.comp_scheme = MIPI_COMPRESSOR_10_6_10;
-				break;
-			case COMPRESSED_BITS_PER_PIXEL_7:
-				cfg->csi_port_attr.comp_scheme = MIPI_COMPRESSOR_10_7_10;
-				break;
-			case COMPRESSED_BITS_PER_PIXEL_8:
-				cfg->csi_port_attr.comp_scheme = MIPI_COMPRESSOR_10_8_10;
-				break;
-			default:
-				err = IA_CSS_ERR_INVALID_ARGUMENTS;
-			}
-		} else if (comp->uncompressed_bits_per_pixel == UNCOMPRESSED_BITS_PER_PIXEL_12) {
-			switch (comp->compressed_bits_per_pixel) {
-			case COMPRESSED_BITS_PER_PIXEL_6:
-				cfg->csi_port_attr.comp_scheme = MIPI_COMPRESSOR_12_6_12;
-				break;
-			case COMPRESSED_BITS_PER_PIXEL_7:
-				cfg->csi_port_attr.comp_scheme = MIPI_COMPRESSOR_12_7_12;
-				break;
-			case COMPRESSED_BITS_PER_PIXEL_8:
-				cfg->csi_port_attr.comp_scheme = MIPI_COMPRESSOR_12_8_12;
-				break;
-			default:
-				err = IA_CSS_ERR_INVALID_ARGUMENTS;
-			}
-		} else
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		cfg->csi_port_attr.comp_predictor = sh_css_csi2_compression_type_2_mipi_predictor(comp->type);
-		cfg->csi_port_attr.comp_enable = true;
-	} else /* No compression */
-		cfg->csi_port_attr.comp_enable = false;
-	return err;
-}
-
-unsigned int ia_css_csi2_calculate_input_system_alignment(
-	enum atomisp_input_format fmt_type)
-{
-	unsigned int memory_alignment_in_bytes = HIVE_ISP_DDR_WORD_BYTES;
-
-	switch (fmt_type) {
-	case ATOMISP_INPUT_FORMAT_RAW_6:
-	case ATOMISP_INPUT_FORMAT_RAW_7:
-	case ATOMISP_INPUT_FORMAT_RAW_8:
-	case ATOMISP_INPUT_FORMAT_RAW_10:
-	case ATOMISP_INPUT_FORMAT_RAW_12:
-	case ATOMISP_INPUT_FORMAT_RAW_14:
-		memory_alignment_in_bytes = 2 * ISP_VEC_NELEMS;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_8:
-	case ATOMISP_INPUT_FORMAT_YUV422_8:
-	case ATOMISP_INPUT_FORMAT_USER_DEF1:
-	case ATOMISP_INPUT_FORMAT_USER_DEF2:
-	case ATOMISP_INPUT_FORMAT_USER_DEF3:
-	case ATOMISP_INPUT_FORMAT_USER_DEF4:
-	case ATOMISP_INPUT_FORMAT_USER_DEF5:
-	case ATOMISP_INPUT_FORMAT_USER_DEF6:
-	case ATOMISP_INPUT_FORMAT_USER_DEF7:
-	case ATOMISP_INPUT_FORMAT_USER_DEF8:
-		/* Planar YUV formats need to have all planes aligned, this means
-		 * double the alignment for the Y plane if the horizontal decimation is 2. */
-		memory_alignment_in_bytes = 2 * HIVE_ISP_DDR_WORD_BYTES;
-		break;
-	case ATOMISP_INPUT_FORMAT_EMBEDDED:
-	default:
-		memory_alignment_in_bytes = HIVE_ISP_DDR_WORD_BYTES;
-		break;
-	}
-	return memory_alignment_in_bytes;
-}
-
-#endif
-
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
-void ia_css_isys_rx_configure(const rx_cfg_t *config,
-			      const enum ia_css_input_mode input_mode)
-{
-#if defined(HAS_RX_VERSION_2)
-	bool port_enabled[N_MIPI_PORT_ID];
-	bool any_port_enabled = false;
-	enum mipi_port_id port;
-
-	if ((config == NULL)
-		|| (config->mode >= N_RX_MODE)
-		|| (config->port >= N_MIPI_PORT_ID)) {
-		assert(0);
-		return;
-	}
-	for (port = (enum mipi_port_id) 0; port < N_MIPI_PORT_ID; port++) {
-		if (is_receiver_port_enabled(RX0_ID, port))
-			any_port_enabled = true;
-	}
-	/* AM: Check whether this is a problem with multiple
-	 * streams. MS: This is the case. */
-
-	port = config->port;
-	receiver_port_enable(RX0_ID, port, false);
-
-	port = config->port;
-
-	/* AM: Check whether this is a problem with multiple streams. */
-	if (MIPI_PORT_LANES[config->mode][port] != MIPI_0LANE_CFG) {
-		receiver_port_reg_store(RX0_ID, port,
-				_HRT_CSS_RECEIVER_FUNC_PROG_REG_IDX,
-				config->timeout);
-		receiver_port_reg_store(RX0_ID, port,
-				_HRT_CSS_RECEIVER_2400_INIT_COUNT_REG_IDX,
-				config->initcount);
-		receiver_port_reg_store(RX0_ID, port,
-				_HRT_CSS_RECEIVER_2400_SYNC_COUNT_REG_IDX,
-				config->synccount);
-		receiver_port_reg_store(RX0_ID, port,
-				_HRT_CSS_RECEIVER_2400_RX_COUNT_REG_IDX,
-				config->rxcount);
-
-		port_enabled[port] = true;
-
-		if (input_mode != IA_CSS_INPUT_MODE_BUFFERED_SENSOR) {
-
-			/* MW: A bit of a hack, straight wiring of the capture
-			 * units,assuming they are linearly enumerated. */
-			input_system_sub_system_reg_store(INPUT_SYSTEM0_ID,
-					GPREGS_UNIT0_ID,
-					HIVE_ISYS_GPREG_MULTICAST_A_IDX
-						+ (unsigned int)port,
-					INPUT_SYSTEM_CSI_BACKEND);
-			/* MW: Like the integration test example we overwite,
-			 * the GPREG_MUX register */
-			input_system_sub_system_reg_store(INPUT_SYSTEM0_ID,
-					GPREGS_UNIT0_ID,
-					HIVE_ISYS_GPREG_MUX_IDX,
-					(input_system_multiplex_t) port);
-		} else {
-			/*
-			 * AM: A bit of a hack, wiring the input system.
-			 */
-			input_system_sub_system_reg_store(INPUT_SYSTEM0_ID,
-					GPREGS_UNIT0_ID,
-					HIVE_ISYS_GPREG_MULTICAST_A_IDX
-						+ (unsigned int)port,
-					INPUT_SYSTEM_INPUT_BUFFER);
-			input_system_sub_system_reg_store(INPUT_SYSTEM0_ID,
-					GPREGS_UNIT0_ID,
-					HIVE_ISYS_GPREG_MUX_IDX,
-					INPUT_SYSTEM_ACQUISITION_UNIT);
-		}
-	}
-	/*
-	 * The 2ppc is shared for all ports, so we cannot
-	 * disable->configure->enable individual ports
-	 */
-	/* AM: Check whether this is a problem with multiple streams. */
-	/* MS: 2ppc should be a property per binary and should be
-	 * enabled/disabled per binary.
-	 * Currently it is implemented as a system wide setting due
-	 * to effort and risks. */
-	if (!any_port_enabled) {
-		receiver_reg_store(RX0_ID,
-				   _HRT_CSS_RECEIVER_TWO_PIXEL_EN_REG_IDX,
-				   config->is_two_ppc);
-		receiver_reg_store(RX0_ID, _HRT_CSS_RECEIVER_BE_TWO_PPC_REG_IDX,
-				   config->is_two_ppc);
-	}
-	receiver_port_enable(RX0_ID, port, true);
-	/* TODO: JB: need to add the beneath used define to mizuchi */
-	/* sh_css_sw_hive_isp_css_2400_system_20121224_0125\css
-	 *                      \hrt\input_system_defs.h
-	 * #define INPUT_SYSTEM_CSI_RECEIVER_SELECT_BACKENG 0X207
-	 */
-	/* TODO: need better name for define
-	 * input_system_reg_store(INPUT_SYSTEM0_ID,
-	 *                INPUT_SYSTEM_CSI_RECEIVER_SELECT_BACKENG, 1);
-	 */
-	input_system_reg_store(INPUT_SYSTEM0_ID, 0x207, 1);
-#else
-#error "rx.c: RX version must be one of {RX_VERSION_2}"
-#endif
-
-	return;
-}
-
-void ia_css_isys_rx_disable(void)
-{
-	enum mipi_port_id port;
-	for (port = (enum mipi_port_id) 0; port < N_MIPI_PORT_ID; port++) {
-		receiver_port_reg_store(RX0_ID, port,
-					_HRT_CSS_RECEIVER_DEVICE_READY_REG_IDX,
-					false);
-	}
-	return;
-}
-#endif /* if !defined(USE_INPUT_SYSTEM_VERSION_2401) */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/virtual_isys.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/virtual_isys.c
deleted file mode 100644
index 2484949453b7..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/virtual_isys.c
+++ /dev/null
@@ -1,898 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "system_global.h"
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-
-#include "ia_css_isys.h"
-#include "ia_css_debug.h"
-#include "math_support.h"
-#include "string_support.h"
-#include "virtual_isys.h"
-#include "isp.h"
-#include "sh_css_defs.h"
-
-/*************************************************
- *
- * Forwarded Declaration
- *
- *************************************************/
-#ifndef ISP2401
-
-#endif
-static bool create_input_system_channel(
-	input_system_cfg_t	*cfg,
-	bool			metadata,
-	input_system_channel_t	*channel);
-
-static void destroy_input_system_channel(
-	input_system_channel_t	*channel);
-
-static bool create_input_system_input_port(
-	input_system_cfg_t		*cfg,
-	input_system_input_port_t	*input_port);
-
-static void destroy_input_system_input_port(
-	input_system_input_port_t	*input_port);
-
-static bool calculate_input_system_channel_cfg(
-	input_system_channel_t		*channel,
-	input_system_input_port_t	*input_port,
-	input_system_cfg_t		*isys_cfg,
-	input_system_channel_cfg_t	*channel_cfg,
-	bool metadata);
-
-static bool calculate_input_system_input_port_cfg(
-	input_system_channel_t		*channel,
-	input_system_input_port_t	*input_port,
-	input_system_cfg_t		*isys_cfg,
-	input_system_input_port_cfg_t	*input_port_cfg);
-
-static bool acquire_sid(
-	stream2mmio_ID_t	stream2mmio,
-	stream2mmio_sid_ID_t	*sid);
-
-static void release_sid(
-	stream2mmio_ID_t	stream2mmio,
-	stream2mmio_sid_ID_t	*sid);
-
-static bool acquire_ib_buffer(
-	int32_t bits_per_pixel,
-	int32_t pixels_per_line,
-	int32_t lines_per_frame,
-	int32_t align_in_bytes,
-	bool online,
-	ib_buffer_t *buf);
-
-static void release_ib_buffer(
-	ib_buffer_t *buf);
-
-static bool acquire_dma_channel(
-	isys2401_dma_ID_t	dma_id,
-	isys2401_dma_channel	*channel);
-
-static void release_dma_channel(
-	isys2401_dma_ID_t	dma_id,
-	isys2401_dma_channel	*channel);
-
-static bool acquire_be_lut_entry(
-	csi_rx_backend_ID_t		backend,
-	csi_mipi_packet_type_t		packet_type,
-	csi_rx_backend_lut_entry_t	*entry);
-
-static void release_be_lut_entry(
-	csi_rx_backend_ID_t		backend,
-	csi_mipi_packet_type_t		packet_type,
-	csi_rx_backend_lut_entry_t	*entry);
-
-static bool calculate_tpg_cfg(
-	input_system_channel_t		*channel,
-	input_system_input_port_t	*input_port,
-	input_system_cfg_t		*isys_cfg,
-	pixelgen_tpg_cfg_t		*cfg);
-
-static bool calculate_prbs_cfg(
-	input_system_channel_t		*channel,
-	input_system_input_port_t	*input_port,
-	input_system_cfg_t		*isys_cfg,
-	pixelgen_prbs_cfg_t		*cfg);
-
-static bool calculate_fe_cfg(
-	const input_system_cfg_t	*isys_cfg,
-	csi_rx_frontend_cfg_t		*cfg);
-
-static bool calculate_be_cfg(
-	const input_system_input_port_t	*input_port,
-	const input_system_cfg_t	*isys_cfg,
-	bool				metadata,
-	csi_rx_backend_cfg_t		*cfg);
-
-static bool calculate_stream2mmio_cfg(
-	const input_system_cfg_t	*isys_cfg,
-	bool				metadata,
-	stream2mmio_cfg_t		*cfg);
-
-static bool calculate_ibuf_ctrl_cfg(
-	const input_system_channel_t	*channel,
-	const input_system_input_port_t	*input_port,
-	const input_system_cfg_t	*isys_cfg,
-	ibuf_ctrl_cfg_t			*cfg);
-
-static bool calculate_isys2401_dma_cfg(
-	const input_system_channel_t	*channel,
-	const input_system_cfg_t	*isys_cfg,
-	isys2401_dma_cfg_t		*cfg);
-
-static bool calculate_isys2401_dma_port_cfg(
-	const input_system_cfg_t	*isys_cfg,
-	bool				raw_packed,
-	bool				metadata,
-	isys2401_dma_port_cfg_t		*cfg);
-
-static csi_mipi_packet_type_t get_csi_mipi_packet_type(
-	int32_t data_type);
-
-static int32_t calculate_stride(
-	int32_t bits_per_pixel,
-	int32_t pixels_per_line,
-	bool	raw_packed,
-	int32_t	align_in_bytes);
-
-/* end of Forwarded Declaration */
-
-/**************************************************
- *
- * Public Methods
- *
- **************************************************/
-ia_css_isys_error_t ia_css_isys_stream_create(
-	ia_css_isys_descr_t	*isys_stream_descr,
-	ia_css_isys_stream_h	isys_stream,
-	uint32_t isys_stream_id)
-{
-	ia_css_isys_error_t rc;
-
-	if (isys_stream_descr == NULL || isys_stream == NULL ||
-		isys_stream_id >= SH_CSS_MAX_ISYS_CHANNEL_NODES)
-		return	false;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"ia_css_isys_stream_create() enter:\n");
-
-	/*Reset isys_stream to 0*/
-	memset(isys_stream, 0, sizeof(*isys_stream));
-	isys_stream->enable_metadata = isys_stream_descr->metadata.enable;
-	isys_stream->id = isys_stream_id;
-
-	isys_stream->linked_isys_stream_id = isys_stream_descr->linked_isys_stream_id;
-	rc = create_input_system_input_port(isys_stream_descr, &(isys_stream->input_port));
-	if (rc == false)
-		return false;
-
-	rc = create_input_system_channel(isys_stream_descr, false, &(isys_stream->channel));
-	if (rc == false) {
-		destroy_input_system_input_port(&isys_stream->input_port);
-		return false;
-	}
-
-#ifdef ISP2401
-	/*
-	 * Early polling is required for timestamp accuracy in certain cause.
-	 * The ISYS HW polling is started on
-	 * ia_css_isys_stream_capture_indication() instead of
-	 * ia_css_pipeline_sp_wait_for_isys_stream_N() as isp processing of
-	 * capture takes longer than getting an ISYS frame
-	 */
-	isys_stream->polling_mode = isys_stream_descr->polling_mode;
-
-#endif
-	/* create metadata channel */
-	if (isys_stream_descr->metadata.enable) {
-		rc = create_input_system_channel(isys_stream_descr, true, &isys_stream->md_channel);
-		if (rc == false) {
-			destroy_input_system_input_port(&isys_stream->input_port);
-			destroy_input_system_channel(&isys_stream->channel);
-			return false;
-		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"ia_css_isys_stream_create() leave:\n");
-
-	return true;
-}
-
-void ia_css_isys_stream_destroy(
-	ia_css_isys_stream_h	isys_stream)
-{
-	destroy_input_system_input_port(&isys_stream->input_port);
-	destroy_input_system_channel(&(isys_stream->channel));
-	if (isys_stream->enable_metadata) {
-		/* Destroy metadata channel only if its allocated*/
-		destroy_input_system_channel(&isys_stream->md_channel);
-	}
-}
-
-ia_css_isys_error_t ia_css_isys_stream_calculate_cfg(
-	ia_css_isys_stream_h		isys_stream,
-	ia_css_isys_descr_t		*isys_stream_descr,
-	ia_css_isys_stream_cfg_t	*isys_stream_cfg)
-{
-	ia_css_isys_error_t rc;
-
-	if (isys_stream_cfg == NULL		||
-		isys_stream_descr == NULL	||
-		isys_stream == NULL)
-		return false;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"ia_css_isys_stream_calculate_cfg() enter:\n");
-
-	rc  = calculate_input_system_channel_cfg(
-			&(isys_stream->channel),
-			&(isys_stream->input_port),
-			isys_stream_descr,
-			&(isys_stream_cfg->channel_cfg),
-			false);
-	if (rc == false)
-		return false;
-
-	/* configure metadata channel */
-	if (isys_stream_descr->metadata.enable) {
-		isys_stream_cfg->enable_metadata = true;
-		rc  = calculate_input_system_channel_cfg(
-				&isys_stream->md_channel,
-				&isys_stream->input_port,
-				isys_stream_descr,
-				&isys_stream_cfg->md_channel_cfg,
-				true);
-		if (rc == false)
-			return false;
-	}
-
-	rc = calculate_input_system_input_port_cfg(
-			&(isys_stream->channel),
-			&(isys_stream->input_port),
-			isys_stream_descr,
-			&(isys_stream_cfg->input_port_cfg));
-	if (rc == false)
-		return false;
-
-	isys_stream->valid = 1;
-	isys_stream_cfg->valid = 1;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"ia_css_isys_stream_calculate_cfg() leave:\n");
-	return rc;
-}
-
-/* end of Public Methods */
-
-/**************************************************
- *
- * Private Methods
- *
- **************************************************/
-static bool create_input_system_channel(
-	input_system_cfg_t	*cfg,
-	bool			metadata,
-	input_system_channel_t	*me)
-{
-	bool rc = true;
-
-	me->dma_id = ISYS2401_DMA0_ID;
-
-	switch (cfg->input_port_id) {
-	case INPUT_SYSTEM_CSI_PORT0_ID:
-	case INPUT_SYSTEM_PIXELGEN_PORT0_ID:
-		me->stream2mmio_id = STREAM2MMIO0_ID;
-		me->ibuf_ctrl_id = IBUF_CTRL0_ID;
-		break;
-
-	case INPUT_SYSTEM_CSI_PORT1_ID:
-	case INPUT_SYSTEM_PIXELGEN_PORT1_ID:
-		me->stream2mmio_id = STREAM2MMIO1_ID;
-		me->ibuf_ctrl_id = IBUF_CTRL1_ID;
-		break;
-
-	case INPUT_SYSTEM_CSI_PORT2_ID:
-	case INPUT_SYSTEM_PIXELGEN_PORT2_ID:
-		me->stream2mmio_id = STREAM2MMIO2_ID;
-		me->ibuf_ctrl_id = IBUF_CTRL2_ID;
-		break;
-	default:
-		rc = false;
-		break;
-	}
-
-	if (!rc)
-		return false;
-
-	if (!acquire_sid(me->stream2mmio_id, &(me->stream2mmio_sid_id))) {
-		return false;
-	}
-
-	if (!acquire_ib_buffer(
-			metadata ? cfg->metadata.bits_per_pixel : cfg->input_port_resolution.bits_per_pixel,
-			metadata ? cfg->metadata.pixels_per_line : cfg->input_port_resolution.pixels_per_line,
-			metadata ? cfg->metadata.lines_per_frame : cfg->input_port_resolution.lines_per_frame,
-			metadata ? cfg->metadata.align_req_in_bytes : cfg->input_port_resolution.align_req_in_bytes,
-			cfg->online,
-			&(me->ib_buffer))) {
-		release_sid(me->stream2mmio_id, &(me->stream2mmio_sid_id));
-		return false;
-	}
-
-	if (!acquire_dma_channel(me->dma_id, &(me->dma_channel))) {
-		release_sid(me->stream2mmio_id, &(me->stream2mmio_sid_id));
-		release_ib_buffer(&(me->ib_buffer));
-		return false;
-	}
-
-	return true;
-}
-
-static void destroy_input_system_channel(
-	input_system_channel_t	*me)
-{
-	release_sid(me->stream2mmio_id,
-		&(me->stream2mmio_sid_id));
-
-	release_ib_buffer(&(me->ib_buffer));
-
-	release_dma_channel(me->dma_id, &(me->dma_channel));
-}
-
-static bool create_input_system_input_port(
-	input_system_cfg_t		*cfg,
-	input_system_input_port_t	*me)
-{
-	csi_mipi_packet_type_t packet_type;
-	bool rc = true;
-
-	switch (cfg->input_port_id) {
-	case INPUT_SYSTEM_CSI_PORT0_ID:
-		me->csi_rx.frontend_id = CSI_RX_FRONTEND0_ID;
-		me->csi_rx.backend_id = CSI_RX_BACKEND0_ID;
-
-		packet_type = get_csi_mipi_packet_type(cfg->csi_port_attr.fmt_type);
-		me->csi_rx.packet_type = packet_type;
-
-		rc = acquire_be_lut_entry(
-				me->csi_rx.backend_id,
-				packet_type,
-				&(me->csi_rx.backend_lut_entry));
-		break;
-	case INPUT_SYSTEM_PIXELGEN_PORT0_ID:
-		me->pixelgen.pixelgen_id = PIXELGEN0_ID;
-		break;
-	case INPUT_SYSTEM_CSI_PORT1_ID:
-		me->csi_rx.frontend_id = CSI_RX_FRONTEND1_ID;
-		me->csi_rx.backend_id = CSI_RX_BACKEND1_ID;
-
-		packet_type = get_csi_mipi_packet_type(cfg->csi_port_attr.fmt_type);
-		me->csi_rx.packet_type = packet_type;
-
-		rc = acquire_be_lut_entry(
-				me->csi_rx.backend_id,
-				packet_type,
-				&(me->csi_rx.backend_lut_entry));
-		break;
-	case INPUT_SYSTEM_PIXELGEN_PORT1_ID:
-		me->pixelgen.pixelgen_id = PIXELGEN1_ID;
-
-		break;
-	case INPUT_SYSTEM_CSI_PORT2_ID:
-		me->csi_rx.frontend_id = CSI_RX_FRONTEND2_ID;
-		me->csi_rx.backend_id = CSI_RX_BACKEND2_ID;
-
-		packet_type = get_csi_mipi_packet_type(cfg->csi_port_attr.fmt_type);
-		me->csi_rx.packet_type = packet_type;
-
-		rc = acquire_be_lut_entry(
-				me->csi_rx.backend_id,
-				packet_type,
-				&(me->csi_rx.backend_lut_entry));
-		break;
-	case INPUT_SYSTEM_PIXELGEN_PORT2_ID:
-		me->pixelgen.pixelgen_id = PIXELGEN2_ID;
-		break;
-	default:
-		rc = false;
-		break;
-	}
-
-	me->source_type = cfg->mode;
-
-	/* for metadata */
-	me->metadata.packet_type = CSI_MIPI_PACKET_TYPE_UNDEFINED;
-	if (rc && cfg->metadata.enable) {
-		me->metadata.packet_type = get_csi_mipi_packet_type(
-				cfg->metadata.fmt_type);
-		rc = acquire_be_lut_entry(
-				me->csi_rx.backend_id,
-				me->metadata.packet_type,
-				&me->metadata.backend_lut_entry);
-	}
-
-	return rc;
-}
-
-static void destroy_input_system_input_port(
-	input_system_input_port_t	*me)
-{
-	if (me->source_type == INPUT_SYSTEM_SOURCE_TYPE_SENSOR) {
-		release_be_lut_entry(
-				me->csi_rx.backend_id,
-				me->csi_rx.packet_type,
-				&me->csi_rx.backend_lut_entry);
-	}
-
-	if (me->metadata.packet_type != CSI_MIPI_PACKET_TYPE_UNDEFINED) {
-		/*Free the backend lut allocated for metadata*/
-		release_be_lut_entry(
-				me->csi_rx.backend_id,
-				me->metadata.packet_type,
-				&me->metadata.backend_lut_entry);
-	}
-}
-
-static bool calculate_input_system_channel_cfg(
-	input_system_channel_t		*channel,
-	input_system_input_port_t	*input_port,
-	input_system_cfg_t		*isys_cfg,
-	input_system_channel_cfg_t	*channel_cfg,
-	bool metadata)
-{
-	bool rc;
-
-	rc = calculate_stream2mmio_cfg(isys_cfg, metadata,
-			&(channel_cfg->stream2mmio_cfg));
-	if (!rc)
-		return false;
-
-	rc = calculate_ibuf_ctrl_cfg(
-			channel,
-			input_port,
-			isys_cfg,
-			&(channel_cfg->ibuf_ctrl_cfg));
-	if (!rc)
-		return false;
-	if (metadata)
-		channel_cfg->ibuf_ctrl_cfg.stores_per_frame = isys_cfg->metadata.lines_per_frame;
-
-	rc = calculate_isys2401_dma_cfg(
-			channel,
-			isys_cfg,
-			&(channel_cfg->dma_cfg));
-	if (!rc)
-		return false;
-
-	rc = calculate_isys2401_dma_port_cfg(
-			isys_cfg,
-			false,
-			metadata,
-			&(channel_cfg->dma_src_port_cfg));
-	if (!rc)
-		return false;
-
-	rc = calculate_isys2401_dma_port_cfg(
-			isys_cfg,
-			isys_cfg->raw_packed,
-			metadata,
-			&(channel_cfg->dma_dest_port_cfg));
-	if (!rc)
-		return false;
-
-	return true;
-}
-
-static bool calculate_input_system_input_port_cfg(
-	input_system_channel_t		*channel,
-	input_system_input_port_t	*input_port,
-	input_system_cfg_t		*isys_cfg,
-	input_system_input_port_cfg_t	*input_port_cfg)
-{
-	bool rc;
-
-	switch (input_port->source_type) {
-	case INPUT_SYSTEM_SOURCE_TYPE_SENSOR:
-		rc  = calculate_fe_cfg(
-				isys_cfg,
-				&(input_port_cfg->csi_rx_cfg.frontend_cfg));
-
-		rc &= calculate_be_cfg(
-				input_port,
-				isys_cfg,
-				false,
-				&(input_port_cfg->csi_rx_cfg.backend_cfg));
-
-		if (rc && isys_cfg->metadata.enable)
-			rc &= calculate_be_cfg(input_port, isys_cfg, true,
-					&input_port_cfg->csi_rx_cfg.md_backend_cfg);
-		break;
-	case INPUT_SYSTEM_SOURCE_TYPE_TPG:
-		rc = calculate_tpg_cfg(
-				channel,
-				input_port,
-				isys_cfg,
-				&(input_port_cfg->pixelgen_cfg.tpg_cfg));
-		break;
-	case INPUT_SYSTEM_SOURCE_TYPE_PRBS:
-		rc = calculate_prbs_cfg(
-				channel,
-				input_port,
-				isys_cfg,
-				&(input_port_cfg->pixelgen_cfg.prbs_cfg));
-		break;
-	default:
-		rc = false;
-		break;
-	}
-
-	return rc;
-}
-
-static bool acquire_sid(
-	stream2mmio_ID_t	stream2mmio,
-	stream2mmio_sid_ID_t	*sid)
-{
-	return ia_css_isys_stream2mmio_sid_rmgr_acquire(stream2mmio, sid);
-}
-
-static void release_sid(
-	stream2mmio_ID_t	stream2mmio,
-	stream2mmio_sid_ID_t	*sid)
-{
-	ia_css_isys_stream2mmio_sid_rmgr_release(stream2mmio, sid);
-}
-
-/* See also: ia_css_dma_configure_from_info() */
-static int32_t calculate_stride(
-	int32_t bits_per_pixel,
-	int32_t pixels_per_line,
-	bool	raw_packed,
-	int32_t align_in_bytes)
-{
-	int32_t bytes_per_line;
-	int32_t pixels_per_word;
-	int32_t words_per_line;
-	int32_t pixels_per_line_padded;
-
-	pixels_per_line_padded = CEIL_MUL(pixels_per_line, align_in_bytes);
-
-	if (!raw_packed)
-		bits_per_pixel = CEIL_MUL(bits_per_pixel, 8);
-
-	pixels_per_word = HIVE_ISP_DDR_WORD_BITS / bits_per_pixel;
-	words_per_line  = ceil_div(pixels_per_line_padded, pixels_per_word);
-	bytes_per_line  = HIVE_ISP_DDR_WORD_BYTES * words_per_line;
-
-	return bytes_per_line;
-}
-
-static bool acquire_ib_buffer(
-	int32_t bits_per_pixel,
-	int32_t pixels_per_line,
-	int32_t lines_per_frame,
-	int32_t align_in_bytes,
-	bool online,
-	ib_buffer_t *buf)
-{
-	buf->stride = calculate_stride(bits_per_pixel, pixels_per_line, false, align_in_bytes);
-	if (online)
-		buf->lines = 4; /* use double buffering for online usecases */
-	else
-		buf->lines = 2;
-
-	(void)(lines_per_frame);
-	return ia_css_isys_ibuf_rmgr_acquire(buf->stride * buf->lines, &buf->start_addr);
-}
-
-static void release_ib_buffer(
-	ib_buffer_t *buf)
-{
-	ia_css_isys_ibuf_rmgr_release(&buf->start_addr);
-}
-
-static bool acquire_dma_channel(
-	isys2401_dma_ID_t	dma_id,
-	isys2401_dma_channel	*channel)
-{
-	return ia_css_isys_dma_channel_rmgr_acquire(dma_id, channel);
-}
-
-static void release_dma_channel(
-	isys2401_dma_ID_t	dma_id,
-	isys2401_dma_channel	*channel)
-{
-	ia_css_isys_dma_channel_rmgr_release(dma_id, channel);
-}
-
-static bool acquire_be_lut_entry(
-	csi_rx_backend_ID_t		backend,
-	csi_mipi_packet_type_t		packet_type,
-	csi_rx_backend_lut_entry_t	*entry)
-{
-	return ia_css_isys_csi_rx_lut_rmgr_acquire(backend, packet_type, entry);
-}
-
-static void release_be_lut_entry(
-	csi_rx_backend_ID_t		backend,
-	csi_mipi_packet_type_t		packet_type,
-	csi_rx_backend_lut_entry_t	*entry)
-{
-	ia_css_isys_csi_rx_lut_rmgr_release(backend, packet_type, entry);
-}
-
-static bool calculate_tpg_cfg(
-	input_system_channel_t		*channel,
-	input_system_input_port_t	*input_port,
-	input_system_cfg_t		*isys_cfg,
-	pixelgen_tpg_cfg_t		*cfg)
-{
-	(void)channel;
-	(void)input_port;
-
-	memcpy_s(
-		(void *)cfg,
-		sizeof(pixelgen_tpg_cfg_t),
-		(void *)(&(isys_cfg->tpg_port_attr)),
-		sizeof(pixelgen_tpg_cfg_t));
-	return true;
-}
-
-static bool calculate_prbs_cfg(
-	input_system_channel_t		*channel,
-	input_system_input_port_t	*input_port,
-	input_system_cfg_t		*isys_cfg,
-	pixelgen_prbs_cfg_t		*cfg)
-{
-	(void)channel;
-	(void)input_port;
-
-	memcpy_s(
-		(void *)cfg,
-		sizeof(pixelgen_prbs_cfg_t),
-		(void *)(&(isys_cfg->prbs_port_attr)),
-		sizeof(pixelgen_prbs_cfg_t));
-	return true;
-}
-
-static bool calculate_fe_cfg(
-	const input_system_cfg_t	*isys_cfg,
-	csi_rx_frontend_cfg_t		*cfg)
-{
-	cfg->active_lanes = isys_cfg->csi_port_attr.active_lanes;
-	return true;
-}
-
-static bool calculate_be_cfg(
-	const input_system_input_port_t	*input_port,
-	const input_system_cfg_t	*isys_cfg,
-	bool				metadata,
-	csi_rx_backend_cfg_t		*cfg)
-{
-
-	memcpy_s(
-		(void *)(&cfg->lut_entry),
-		sizeof(csi_rx_backend_lut_entry_t),
-		metadata ? (void *)(&input_port->metadata.backend_lut_entry) :
-			(void *)(&input_port->csi_rx.backend_lut_entry),
-		sizeof(csi_rx_backend_lut_entry_t));
-
-	cfg->csi_mipi_cfg.virtual_channel = isys_cfg->csi_port_attr.ch_id;
-	if (metadata) {
-		cfg->csi_mipi_packet_type = get_csi_mipi_packet_type(isys_cfg->metadata.fmt_type);
-		cfg->csi_mipi_cfg.comp_enable = false;
-		cfg->csi_mipi_cfg.data_type = isys_cfg->metadata.fmt_type;
-	}
-	else {
-		cfg->csi_mipi_packet_type = get_csi_mipi_packet_type(isys_cfg->csi_port_attr.fmt_type);
-		cfg->csi_mipi_cfg.data_type = isys_cfg->csi_port_attr.fmt_type;
-		cfg->csi_mipi_cfg.comp_enable = isys_cfg->csi_port_attr.comp_enable;
-		cfg->csi_mipi_cfg.comp_scheme = isys_cfg->csi_port_attr.comp_scheme;
-		cfg->csi_mipi_cfg.comp_predictor = isys_cfg->csi_port_attr.comp_predictor;
-		cfg->csi_mipi_cfg.comp_bit_idx = cfg->csi_mipi_cfg.data_type - MIPI_FORMAT_CUSTOM0;
-	}
-
-	return true;
-}
-
-static bool calculate_stream2mmio_cfg(
-	const input_system_cfg_t	*isys_cfg,
-	bool				metadata,
-	stream2mmio_cfg_t		*cfg
-)
-{
-	cfg->bits_per_pixel = metadata ? isys_cfg->metadata.bits_per_pixel :
-		isys_cfg->input_port_resolution.bits_per_pixel;
-
-	cfg->enable_blocking =
-		((isys_cfg->mode == INPUT_SYSTEM_SOURCE_TYPE_TPG) ||
-		 (isys_cfg->mode == INPUT_SYSTEM_SOURCE_TYPE_PRBS));
-
-	return true;
-}
-
-static bool calculate_ibuf_ctrl_cfg(
-	const input_system_channel_t	*channel,
-	const input_system_input_port_t	*input_port,
-	const input_system_cfg_t	*isys_cfg,
-	ibuf_ctrl_cfg_t			*cfg)
-{
-	const int32_t bits_per_byte = 8;
-	int32_t bits_per_pixel;
-	int32_t bytes_per_pixel;
-	int32_t left_padding;
-
-	(void)input_port;
-
-	bits_per_pixel = isys_cfg->input_port_resolution.bits_per_pixel;
-	bytes_per_pixel = ceil_div(bits_per_pixel, bits_per_byte);
-
-	left_padding = CEIL_MUL(isys_cfg->output_port_attr.left_padding, ISP_VEC_NELEMS)
-			* bytes_per_pixel;
-
-	cfg->online	= isys_cfg->online;
-
-	cfg->dma_cfg.channel	= channel->dma_channel;
-	cfg->dma_cfg.cmd	= _DMA_V2_MOVE_A2B_NO_SYNC_CHK_COMMAND;
-
-	cfg->dma_cfg.shift_returned_items	= 0;
-	cfg->dma_cfg.elems_per_word_in_ibuf	= 0;
-	cfg->dma_cfg.elems_per_word_in_dest	= 0;
-
-	cfg->ib_buffer.start_addr		= channel->ib_buffer.start_addr;
-	cfg->ib_buffer.stride			= channel->ib_buffer.stride;
-	cfg->ib_buffer.lines			= channel->ib_buffer.lines;
-
-	/*
-#ifndef ISP2401
-	 * zhengjie.lu@intel.com:
-#endif
-	 * "dest_buf_cfg" should be part of the input system output
-	 * port configuration.
-	 *
-	 * TODO: move "dest_buf_cfg" to the input system output
-	 * port configuration.
-	 */
-
-	/* input_buf addr only available in sched mode;
-	   this buffer is allocated in isp, crun mode addr
-	   can be passed by after ISP allocation */
-	if (cfg->online) {
-		cfg->dest_buf_cfg.start_addr	= ISP_INPUT_BUF_START_ADDR + left_padding;
-		cfg->dest_buf_cfg.stride	= bytes_per_pixel
-			* isys_cfg->output_port_attr.max_isp_input_width;
-		cfg->dest_buf_cfg.lines		= LINES_OF_ISP_INPUT_BUF;
-	} else if (isys_cfg->raw_packed) {
-		cfg->dest_buf_cfg.stride	= calculate_stride(bits_per_pixel,
-							isys_cfg->input_port_resolution.pixels_per_line,
-							isys_cfg->raw_packed,
-							isys_cfg->input_port_resolution.align_req_in_bytes);
-	} else {
-		cfg->dest_buf_cfg.stride	= channel->ib_buffer.stride;
-	}
-
-	/*
-#ifndef ISP2401
-	 * zhengjie.lu@intel.com:
-#endif
-	 * "items_per_store" is hard coded as "1", which is ONLY valid
-	 * when the CSI-MIPI long packet is transferred.
-	 *
-	 * TODO: After the 1st stage of MERR+,  make the proper solution to
-	 * configure "items_per_store" so that it can also handle the CSI-MIPI
-	 * short packet.
-	 */
-	cfg->items_per_store		= 1;
-
-	cfg->stores_per_frame		= isys_cfg->input_port_resolution.lines_per_frame;
-
-
-	cfg->stream2mmio_cfg.sync_cmd	= _STREAM2MMIO_CMD_TOKEN_SYNC_FRAME;
-
-	/* TODO: Define conditions as when to use store words vs store packets */
-	cfg->stream2mmio_cfg.store_cmd	= _STREAM2MMIO_CMD_TOKEN_STORE_PACKETS;
-
-	return true;
-}
-
-static bool calculate_isys2401_dma_cfg(
-	const input_system_channel_t	*channel,
-	const input_system_cfg_t	*isys_cfg,
-	isys2401_dma_cfg_t		*cfg)
-{
-	cfg->channel	= channel->dma_channel;
-
-	/* only online/sensor mode goto vmem
-	   offline/buffered_sensor, tpg and prbs will go to ddr */
-	if (isys_cfg->online)
-		cfg->connection = isys2401_dma_ibuf_to_vmem_connection;
-	else
-		cfg->connection = isys2401_dma_ibuf_to_ddr_connection;
-
-	cfg->extension	= isys2401_dma_zero_extension;
-	cfg->height	= 1;
-
-	return true;
-}
-
-/* See also: ia_css_dma_configure_from_info() */
-static bool calculate_isys2401_dma_port_cfg(
-	const input_system_cfg_t	*isys_cfg,
-	bool				raw_packed,
-	bool				metadata,
-	isys2401_dma_port_cfg_t		*cfg)
-{
-	int32_t bits_per_pixel;
-	int32_t pixels_per_line;
-	int32_t align_req_in_bytes;
-
-	/* TODO: Move metadata away from isys_cfg to application layer */
-	if (metadata) {
-		bits_per_pixel = isys_cfg->metadata.bits_per_pixel;
-		pixels_per_line = isys_cfg->metadata.pixels_per_line;
-		align_req_in_bytes = isys_cfg->metadata.align_req_in_bytes;
-	} else {
-		bits_per_pixel = isys_cfg->input_port_resolution.bits_per_pixel;
-		pixels_per_line = isys_cfg->input_port_resolution.pixels_per_line;
-		align_req_in_bytes = isys_cfg->input_port_resolution.align_req_in_bytes;
-	}
-
-	cfg->stride	= calculate_stride(bits_per_pixel, pixels_per_line, raw_packed, align_req_in_bytes);
-
-	if (!raw_packed)
-		bits_per_pixel = CEIL_MUL(bits_per_pixel, 8);
-
-	cfg->elements	= HIVE_ISP_DDR_WORD_BITS / bits_per_pixel;
-	cfg->cropping	= 0;
-	cfg->width	= CEIL_DIV(cfg->stride, HIVE_ISP_DDR_WORD_BYTES);
-
-	return true;
-}
-
-static csi_mipi_packet_type_t get_csi_mipi_packet_type(
-	int32_t data_type)
-{
-	csi_mipi_packet_type_t packet_type;
-
-	packet_type = CSI_MIPI_PACKET_TYPE_RESERVED;
-
-	if (data_type >= 0 && data_type <= MIPI_FORMAT_SHORT8)
-		packet_type = CSI_MIPI_PACKET_TYPE_SHORT;
-
-	if (data_type > MIPI_FORMAT_SHORT8 && data_type <= N_MIPI_FORMAT)
-		packet_type = CSI_MIPI_PACKET_TYPE_LONG;
-
-	return packet_type;
-}
-/* end of Private Methods */
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/virtual_isys.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/virtual_isys.h
deleted file mode 100644
index 66c7293c0a93..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/virtual_isys.h
+++ /dev/null
@@ -1,41 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __VIRTUAL_ISYS_H_INCLUDED__
-#define __VIRTUAL_ISYS_H_INCLUDED__
-
-/* cmd for storing a number of packets indicated by reg _STREAM2MMIO_NUM_ITEMS*/
-#define _STREAM2MMIO_CMD_TOKEN_STORE_PACKETS	1
-
-/* command for waiting for a frame start */
-#define _STREAM2MMIO_CMD_TOKEN_SYNC_FRAME	2
-
-#endif /* __VIRTUAL_ISYS_H_INCLUDED__ */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/interface/ia_css_pipeline.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/interface/ia_css_pipeline.h
deleted file mode 100644
index 85ed7db0af55..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/interface/ia_css_pipeline.h
+++ /dev/null
@@ -1,302 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_PIPELINE_H__
-#define __IA_CSS_PIPELINE_H__
-
-#include "sh_css_internal.h"
-#include "ia_css_pipe_public.h"
-#include "ia_css_pipeline_common.h"
-
-#define IA_CSS_PIPELINE_NUM_MAX		(20)
-
-
-/* Pipeline stage to be executed on SP/ISP */
-struct ia_css_pipeline_stage {
-	unsigned int stage_num;
-	struct ia_css_binary *binary;	/* built-in binary */
-	struct ia_css_binary_info *binary_info;
-	const struct ia_css_fw_info *firmware;	/* acceleration binary */
-	/* SP function for SP stage */
-	enum ia_css_pipeline_stage_sp_func sp_func;
-	unsigned max_input_width;	/* For SP raw copy */
-	struct sh_css_binary_args args;
-	int mode;
-	bool out_frame_allocated[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	bool vf_frame_allocated;
-	struct ia_css_pipeline_stage *next;
-	bool enable_zoom;
-};
-
-/* Pipeline of n stages to be executed on SP/ISP per stage */
-struct ia_css_pipeline {
-	enum ia_css_pipe_id pipe_id;
-	uint8_t pipe_num;
-	bool stop_requested;
-	struct ia_css_pipeline_stage *stages;
-	struct ia_css_pipeline_stage *current_stage;
-	unsigned num_stages;
-	struct ia_css_frame in_frame;
-	struct ia_css_frame out_frame[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	struct ia_css_frame vf_frame[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	unsigned int dvs_frame_delay;
-	unsigned inout_port_config;
-	int num_execs;
-	bool acquire_isp_each_stage;
-	uint32_t pipe_qos_config;
-};
-
-#define DEFAULT_PIPELINE \
-(struct ia_css_pipeline) { \
-	.pipe_id		= IA_CSS_PIPE_ID_PREVIEW, \
-	.in_frame		= DEFAULT_FRAME, \
-	.out_frame		= {DEFAULT_FRAME}, \
-	.vf_frame		= {DEFAULT_FRAME}, \
-	.dvs_frame_delay	= IA_CSS_FRAME_DELAY_1, \
-	.num_execs		= -1, \
-	.acquire_isp_each_stage	= true, \
-	.pipe_qos_config	= QOS_INVALID \
-}
-
-/* Stage descriptor used to create a new stage in the pipeline */
-struct ia_css_pipeline_stage_desc {
-	struct ia_css_binary *binary;
-	const struct ia_css_fw_info *firmware;
-	enum ia_css_pipeline_stage_sp_func sp_func;
-	unsigned max_input_width;
-	unsigned int mode;
-	struct ia_css_frame *in_frame;
-	struct ia_css_frame *out_frame[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	struct ia_css_frame *vf_frame;
-};
-
-/* @brief initialize the pipeline module
- *
- * @return    None
- *
- * Initializes the pipeline module. This API has to be called
- * before any operation on the pipeline module is done
- */
-void ia_css_pipeline_init(void);
-
-/* @brief initialize the pipeline structure with default values
- *
- * @param[out] pipeline  structure to be initialized with defaults
- * @param[in] pipe_id
- * @param[in] pipe_num Number that uniquely identifies a pipeline.
- * @return                     IA_CSS_SUCCESS or error code upon error.
- *
- * Initializes the pipeline structure with a set of default values.
- * This API is expected to be used when a pipeline structure is allocated
- * externally and needs sane defaults
- */
-enum ia_css_err ia_css_pipeline_create(
-	struct ia_css_pipeline *pipeline,
-	enum ia_css_pipe_id pipe_id,
-	unsigned int pipe_num,
-	unsigned int dvs_frame_delay);
-
-/* @brief destroy a pipeline
- *
- * @param[in] pipeline
- * @return    None
- *
- */
-void ia_css_pipeline_destroy(struct ia_css_pipeline *pipeline);
-
-
-/* @brief Starts a pipeline
- *
- * @param[in] pipe_id
- * @param[in] pipeline
- * @return    None
- *
- */
-void ia_css_pipeline_start(enum ia_css_pipe_id pipe_id,
-			   struct ia_css_pipeline *pipeline);
-
-/* @brief Request to stop a pipeline
- *
- * @param[in] pipeline
- * @return                     IA_CSS_SUCCESS or error code upon error.
- *
- */
-enum ia_css_err ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline);
-
-/* @brief Check whether pipeline has stopped
- *
- * @param[in] pipeline
- * @return    true if the pipeline has stopped
- *
- */
-bool ia_css_pipeline_has_stopped(struct ia_css_pipeline *pipe);
-
-/* @brief clean all the stages pipeline and make it as new
- *
- * @param[in] pipeline
- * @return    None
- *
- */
-void ia_css_pipeline_clean(struct ia_css_pipeline *pipeline);
-
-/* @brief Add a stage to pipeline.
- *
- * @param     pipeline               Pointer to the pipeline to be added to.
- * @param[in] stage_desc       The description of the stage
- * @param[out] stage            The successor of the stage.
- * @return                     IA_CSS_SUCCESS or error code upon error.
- *
- * Add a new stage to a non-NULL pipeline.
- * The stage consists of an ISP binary or firmware and input and output
- * arguments.
-*/
-enum ia_css_err ia_css_pipeline_create_and_add_stage(
-			struct ia_css_pipeline *pipeline,
-			struct ia_css_pipeline_stage_desc *stage_desc,
-			struct ia_css_pipeline_stage **stage);
-
-/* @brief Finalize the stages in a pipeline
- *
- * @param     pipeline               Pointer to the pipeline to be added to.
- * @return                     None
- *
- * This API is expected to be called after adding all stages
-*/
-void ia_css_pipeline_finalize_stages(struct ia_css_pipeline *pipeline,
-			bool continuous);
-
-/* @brief gets a stage from the pipeline
- *
- * @param[in] pipeline
- * @return                     IA_CSS_SUCCESS or error code upon error.
- *
- */
-enum ia_css_err ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline,
-			  int mode,
-			  struct ia_css_pipeline_stage **stage);
-
-/* @brief Gets a pipeline stage corresponding Firmware handle from the pipeline
- *
- * @param[in] pipeline
- * @param[in] fw_handle
- * @param[out] stage Pointer to Stage
- *
- * @return   IA_CSS_SUCCESS or error code upon error.
- *
- */
-enum ia_css_err ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline *pipeline,
-			  uint32_t fw_handle,
-			  struct ia_css_pipeline_stage **stage);
-
-/* @brief Gets the Firmware handle correponding the stage num from the pipeline
- *
- * @param[in] pipeline
- * @param[in] stage_num
- * @param[out] fw_handle
- *
- * @return   IA_CSS_SUCCESS or error code upon error.
- *
- */
-enum ia_css_err ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline *pipeline,
-			  uint32_t stage_num,
-			  uint32_t *fw_handle);
-
-/* @brief gets the output stage from the pipeline
- *
- * @param[in] pipeline
- * @return                     IA_CSS_SUCCESS or error code upon error.
- *
- */
-enum ia_css_err ia_css_pipeline_get_output_stage(
-			struct ia_css_pipeline *pipeline,
-			int mode,
-			struct ia_css_pipeline_stage **stage);
-
-/* @brief Checks whether the pipeline uses params
- *
- * @param[in] pipeline
- * @return    true if the pipeline uses params
- *
- */
-bool ia_css_pipeline_uses_params(struct ia_css_pipeline *pipeline);
-
-/**
- * @brief get the SP thread ID.
- *
- * @param[in]	key	The query key, typical use is pipe_num.
- * @param[out]	val	The query value.
- *
- * @return
- *	true, if the query succeeds;
- *	false, if the query fails.
- */
-bool ia_css_pipeline_get_sp_thread_id(unsigned int key, unsigned int *val);
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-/**
- * @brief Get the pipeline io status
- *
- * @param[in] None
- * @return
- *	Pointer to pipe_io_status
- */
-struct sh_css_sp_pipeline_io_status *ia_css_pipeline_get_pipe_io_status(void);
-#endif
-
-/**
- * @brief Map an SP thread to this pipeline
- *
- * @param[in]	pipe_num
- * @param[in]	map true for mapping and false for unmapping sp threads.
- *
- */
-void ia_css_pipeline_map(unsigned int pipe_num, bool map);
-
-/**
- * @brief Checks whether the pipeline is mapped to SP threads
- *
- * @param[in]	Query key, typical use is pipe_num
- *
- * return
- *	true, pipeline is mapped to SP threads
- *	false, pipeline is not mapped to SP threads
- */
-bool ia_css_pipeline_is_mapped(unsigned int key);
-
-/**
- * @brief Print pipeline thread mapping
- *
- * @param[in]	none
- *
- * return none
- */
-void ia_css_pipeline_dump_thread_map_info(void);
-
-#endif /*__IA_CSS_PIPELINE_H__*/
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/interface/ia_css_pipeline_common.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/interface/ia_css_pipeline_common.h
deleted file mode 100644
index a7e6edf41cdb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/interface/ia_css_pipeline_common.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_PIPELINE_COMMON_H__
-#define __IA_CSS_PIPELINE_COMMON_H__
-
-enum ia_css_pipeline_stage_sp_func {
-	IA_CSS_PIPELINE_RAW_COPY = 0,
-	IA_CSS_PIPELINE_BIN_COPY = 1,
-	IA_CSS_PIPELINE_ISYS_COPY = 2,
-	IA_CSS_PIPELINE_NO_FUNC = 3,
-};
-#define IA_CSS_PIPELINE_NUM_STAGE_FUNCS 3
-
-#endif /*__IA_CSS_PIPELINE_COMMON_H__*/
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/src/pipeline.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/src/pipeline.c
deleted file mode 100644
index 4746620ca212..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/pipeline/src/pipeline.c
+++ /dev/null
@@ -1,805 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "ia_css_debug.h"
-#include "sw_event_global.h"		/* encode_sw_event */
-#include "sp.h"			/* cnd_sp_irq_enable() */
-#include "assert_support.h"
-#include "memory_access.h"
-#include "sh_css_sp.h"
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_param.h"
-#include "ia_css_bufq.h"
-
-#define PIPELINE_NUM_UNMAPPED                   (~0U)
-#define PIPELINE_SP_THREAD_EMPTY_TOKEN          (0x0)
-#define PIPELINE_SP_THREAD_RESERVED_TOKEN       (0x1)
-
-
-/*******************************************************
-*** Static variables
-********************************************************/
-static unsigned int pipeline_num_to_sp_thread_map[IA_CSS_PIPELINE_NUM_MAX];
-static unsigned int pipeline_sp_thread_list[SH_CSS_MAX_SP_THREADS];
-
-/*******************************************************
-*** Static functions
-********************************************************/
-static void pipeline_init_sp_thread_map(void);
-static void pipeline_map_num_to_sp_thread(unsigned int pipe_num);
-static void pipeline_unmap_num_to_sp_thread(unsigned int pipe_num);
-static void pipeline_init_defaults(
-	struct ia_css_pipeline *pipeline,
-	enum ia_css_pipe_id pipe_id,
-	unsigned int pipe_num,
-	unsigned int dvs_frame_delay);
-
-static void pipeline_stage_destroy(struct ia_css_pipeline_stage *stage);
-static enum ia_css_err pipeline_stage_create(
-	struct ia_css_pipeline_stage_desc *stage_desc,
-	struct ia_css_pipeline_stage **new_stage);
-static void ia_css_pipeline_set_zoom_stage(struct ia_css_pipeline *pipeline);
-static void ia_css_pipeline_configure_inout_port(struct ia_css_pipeline *me,
-	bool continuous);
-
-/*******************************************************
-*** Public functions
-********************************************************/
-void ia_css_pipeline_init(void)
-{
-	pipeline_init_sp_thread_map();
-}
-
-enum ia_css_err ia_css_pipeline_create(
-	struct ia_css_pipeline *pipeline,
-	enum ia_css_pipe_id pipe_id,
-	unsigned int pipe_num,
-	unsigned int dvs_frame_delay)
-{
-	assert(pipeline != NULL);
-	IA_CSS_ENTER_PRIVATE("pipeline = %p, pipe_id = %d, pipe_num = %d, dvs_frame_delay = %d",
-		     pipeline, pipe_id, pipe_num, dvs_frame_delay);
-	if (pipeline == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	pipeline_init_defaults(pipeline, pipe_id, pipe_num, dvs_frame_delay);
-
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-void ia_css_pipeline_map(unsigned int pipe_num, bool map)
-{
-	assert(pipe_num < IA_CSS_PIPELINE_NUM_MAX);
-	IA_CSS_ENTER_PRIVATE("pipe_num = %d, map = %d", pipe_num, map);
-
-	if (pipe_num >= IA_CSS_PIPELINE_NUM_MAX) {
-		IA_CSS_ERROR("Invalid pipe number");
-		IA_CSS_LEAVE_PRIVATE("void");
-		return;
-	}
-	if (map)
-		pipeline_map_num_to_sp_thread(pipe_num);
-	else
-		pipeline_unmap_num_to_sp_thread(pipe_num);
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-/* @brief destroy a pipeline
- *
- * @param[in] pipeline
- * @return    None
- *
- */
-void ia_css_pipeline_destroy(struct ia_css_pipeline *pipeline)
-{
-	assert(pipeline != NULL);
-	IA_CSS_ENTER_PRIVATE("pipeline = %p", pipeline);
-
-	if (pipeline == NULL) {
-		IA_CSS_ERROR("NULL input parameter");
-		IA_CSS_LEAVE_PRIVATE("void");
-		return;
-	}
-
-	IA_CSS_LOG("pipe_num = %d", pipeline->pipe_num);
-
-	/* Free the pipeline number */
-	ia_css_pipeline_clean(pipeline);
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-/* Run a pipeline and wait till it completes. */
-void ia_css_pipeline_start(enum ia_css_pipe_id pipe_id,
-			   struct ia_css_pipeline *pipeline)
-{
-	uint8_t pipe_num = 0;
-	unsigned int thread_id;
-
-	assert(pipeline != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-	      "ia_css_pipeline_start() enter: pipe_id=%d, pipeline=%p\n",
-	      pipe_id, pipeline);
-	pipeline->pipe_id = pipe_id;
-	sh_css_sp_init_pipeline(pipeline, pipe_id, pipe_num,
-				false, false, false, true, SH_CSS_BDS_FACTOR_1_00,
-				SH_CSS_PIPE_CONFIG_OVRD_NO_OVRD,
-#ifndef ISP2401
-				IA_CSS_INPUT_MODE_MEMORY, NULL, NULL
-#else
-				IA_CSS_INPUT_MODE_MEMORY, NULL, NULL,
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#ifndef ISP2401
-				, (enum mipi_port_id) 0
-#else
-				(enum mipi_port_id) 0,
-#endif
-#endif
-#ifndef ISP2401
-				);
-#else
-				NULL, NULL);
-#endif
-	ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
-	if (!sh_css_sp_is_running()) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_pipeline_start() error,leaving\n");
-		/* queues are invalid*/
-		return;
-	}
-	ia_css_bufq_enqueue_psys_event(IA_CSS_PSYS_SW_EVENT_START_STREAM,
-				       (uint8_t)thread_id,
-				       0,
-				       0);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-	      "ia_css_pipeline_start() leave: return_void\n");
-}
-
-/*
- * @brief Query the SP thread ID.
- * Refer to "sh_css_internal.h" for details.
- */
-bool ia_css_pipeline_get_sp_thread_id(unsigned int key, unsigned int *val)
-{
-
-	IA_CSS_ENTER("key=%d, val=%p", key, val);
-
-	if ((val == NULL) || (key >= IA_CSS_PIPELINE_NUM_MAX) || (key >= IA_CSS_PIPE_ID_NUM)) {
-		IA_CSS_LEAVE("return value = false");
-		return false;
-	}
-
-	*val = pipeline_num_to_sp_thread_map[key];
-
-	if (*val == (unsigned)PIPELINE_NUM_UNMAPPED) {
-		IA_CSS_LOG("unmapped pipeline number");
-		IA_CSS_LEAVE("return value = false");
-		return false;
-	}
-	IA_CSS_LEAVE("return value = true");
-	return true;
-}
-
-void ia_css_pipeline_dump_thread_map_info(void)
-{
-	unsigned int i;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"pipeline_num_to_sp_thread_map:\n");
-	for (i = 0; i < IA_CSS_PIPELINE_NUM_MAX; i++) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			"pipe_num: %u, tid: 0x%x\n", i, pipeline_num_to_sp_thread_map[i]);
-	}
-}
-
-enum ia_css_err ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	unsigned int thread_id;
-
-	assert(pipeline != NULL);
-
-	if (pipeline == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_pipeline_request_stop() enter: pipeline=%p\n",
-		pipeline);
-	pipeline->stop_requested = true;
-
-	/* Send stop event to the sp*/
-	/* This needs improvement, stop on all the pipes available
-	 * in the stream*/
-	ia_css_pipeline_get_sp_thread_id(pipeline->pipe_num, &thread_id);
-	if (!sh_css_sp_is_running())
-	{
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_pipeline_request_stop() leaving\n");
-		/* queues are invalid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-	ia_css_bufq_enqueue_psys_event(IA_CSS_PSYS_SW_EVENT_STOP_STREAM,
-				       (uint8_t)thread_id,
-				       0,
-				       0);
-	sh_css_sp_uninit_pipeline(pipeline->pipe_num);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_pipeline_request_stop() leave: return_err=%d\n",
-		      err);
-	return err;
-}
-
-void ia_css_pipeline_clean(struct ia_css_pipeline *pipeline)
-{
-	struct ia_css_pipeline_stage *s;
-
-	assert(pipeline != NULL);
-	IA_CSS_ENTER_PRIVATE("pipeline = %p", pipeline);
-
-	if (pipeline == NULL) {
-		IA_CSS_ERROR("NULL input parameter");
-		IA_CSS_LEAVE_PRIVATE("void");
-		return;
-	}
-	s = pipeline->stages;
-
-	while (s) {
-		struct ia_css_pipeline_stage *next = s->next;
-		pipeline_stage_destroy(s);
-		s = next;
-	}
-	pipeline_init_defaults(pipeline, pipeline->pipe_id, pipeline->pipe_num, pipeline->dvs_frame_delay);
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-/* @brief Add a stage to pipeline.
- *
- * @param       pipeline      Pointer to the pipeline to be added to.
- * @param[in]   stage_desc    The description of the stage
- * @param[out]	stage         The successor of the stage.
- * @return      IA_CSS_SUCCESS or error code upon error.
- *
- * Add a new stage to a non-NULL pipeline.
- * The stage consists of an ISP binary or firmware and input and
- * output arguments.
-*/
-enum ia_css_err ia_css_pipeline_create_and_add_stage(
-		struct ia_css_pipeline *pipeline,
-		struct ia_css_pipeline_stage_desc *stage_desc,
-		struct ia_css_pipeline_stage **stage)
-{
-	struct ia_css_pipeline_stage *last, *new_stage = NULL;
-	enum ia_css_err err;
-
-	/* other arguments can be NULL */
-	assert(pipeline != NULL);
-	assert(stage_desc != NULL);
-	last = pipeline->stages;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_pipeline_create_and_add_stage() enter:\n");
-	if (!stage_desc->binary && !stage_desc->firmware
-	    && (stage_desc->sp_func == IA_CSS_PIPELINE_NO_FUNC)) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			      "ia_css_pipeline_create_and_add_stage() done:"
-			      " Invalid args\n");
-
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-	/* Find the last stage */
-	while (last && last->next)
-		last = last->next;
-
-	/* if in_frame is not set, we use the out_frame from the previous
-	 * stage, if no previous stage, it's an error.
-	 */
-	if ((stage_desc->sp_func == IA_CSS_PIPELINE_NO_FUNC)
-		&& (!stage_desc->in_frame)
-		&& (!stage_desc->firmware)
-		&& (!stage_desc->binary->online)) {
-
-		/* Do this only for ISP stages*/
-		if (last && last->args.out_frame[0])
-			stage_desc->in_frame = last->args.out_frame[0];
-
-		if (!stage_desc->in_frame)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-	/* Create the new stage */
-	err = pipeline_stage_create(stage_desc, &new_stage);
-	if (err != IA_CSS_SUCCESS) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			      "ia_css_pipeline_create_and_add_stage() done:"
-			      " stage_create_failed\n");
-		return err;
-	}
-
-	if (last)
-		last->next = new_stage;
-	else
-		pipeline->stages = new_stage;
-
-	/* Output the new stage */
-	if (stage)
-		*stage = new_stage;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_pipeline_create_and_add_stage() done:\n");
-	return IA_CSS_SUCCESS;
-}
-
-void ia_css_pipeline_finalize_stages(struct ia_css_pipeline *pipeline,
-			bool continuous)
-{
-	unsigned i = 0;
-	struct ia_css_pipeline_stage *stage;
-
-	assert(pipeline != NULL);
-	for (stage = pipeline->stages; stage; stage = stage->next) {
-		stage->stage_num = i;
-		i++;
-	}
-	pipeline->num_stages = i;
-
-	ia_css_pipeline_set_zoom_stage(pipeline);
-	ia_css_pipeline_configure_inout_port(pipeline, continuous);
-}
-
-enum ia_css_err ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline,
-					  int mode,
-					  struct ia_css_pipeline_stage **stage)
-{
-	struct ia_css_pipeline_stage *s;
-	assert(pipeline != NULL);
-	assert(stage != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_pipeline_get_stage() enter:\n");
-	for (s = pipeline->stages; s; s = s->next) {
-		if (s->mode == mode) {
-			*stage = s;
-			return IA_CSS_SUCCESS;
-		}
-	}
-	return IA_CSS_ERR_INTERNAL_ERROR;
-}
-
-enum ia_css_err ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline *pipeline,
-			  uint32_t fw_handle,
-			  struct ia_css_pipeline_stage **stage)
-{
-	struct ia_css_pipeline_stage *s;
-	assert(pipeline != NULL);
-	assert(stage != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,"%s() \n",__func__);
-	for (s = pipeline->stages; s; s = s->next) {
-		if ((s->firmware) && (s->firmware->handle == fw_handle)) {
-			*stage = s;
-			return IA_CSS_SUCCESS;
-		}
-	}
-	return IA_CSS_ERR_INTERNAL_ERROR;
-}
-
-enum ia_css_err ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline *pipeline,
-			  uint32_t stage_num,
-			  uint32_t *fw_handle)
-{
-	struct ia_css_pipeline_stage *s;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,"%s() \n",__func__);
-	if ((pipeline == NULL) || (fw_handle == NULL))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	for (s = pipeline->stages; s; s = s->next) {
-		if((s->stage_num == stage_num) && (s->firmware)) {
-			*fw_handle = s->firmware->handle;
-			return IA_CSS_SUCCESS;
-		}
-	}
-	return IA_CSS_ERR_INTERNAL_ERROR;
-}
-
-enum ia_css_err ia_css_pipeline_get_output_stage(
-		struct ia_css_pipeline *pipeline,
-		int mode,
-		struct ia_css_pipeline_stage **stage)
-{
-	struct ia_css_pipeline_stage *s;
-	assert(pipeline != NULL);
-	assert(stage != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		      "ia_css_pipeline_get_output_stage() enter:\n");
-
-	*stage = NULL;
-	/* First find acceleration firmware at end of pipe */
-	for (s = pipeline->stages; s; s = s->next) {
-		if (s->firmware && s->mode == mode &&
-		    s->firmware->info.isp.sp.enable.output)
-			*stage = s;
-	}
-	if (*stage)
-		return IA_CSS_SUCCESS;
-	/* If no firmware, find binary in pipe */
-	return ia_css_pipeline_get_stage(pipeline, mode, stage);
-}
-
-bool ia_css_pipeline_has_stopped(struct ia_css_pipeline *pipeline)
-{
-	/* Android compilation files if made an local variable
-	stack size on android is limited to 2k and this structure
-	is around 2.5K, in place of static malloc can be done but
-	if this call is made too often it will lead to fragment memory
-	versus a fixed allocation */
-	static struct sh_css_sp_group sp_group;
-	unsigned int thread_id;
-	const struct ia_css_fw_info *fw;
-	unsigned int HIVE_ADDR_sp_group;
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_sp_group = fw->info.sp.group;
-
-	ia_css_pipeline_get_sp_thread_id(pipeline->pipe_num, &thread_id);
-	sp_dmem_load(SP0_ID,
-		     (unsigned int)sp_address_of(sp_group),
-		     &sp_group, sizeof(struct sh_css_sp_group));
-	return sp_group.pipe[thread_id].num_stages == 0;
-}
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-struct sh_css_sp_pipeline_io_status *ia_css_pipeline_get_pipe_io_status(void)
-{
-	return(&sh_css_sp_group.pipe_io_status);
-}
-#endif
-
-bool ia_css_pipeline_is_mapped(unsigned int key)
-{
-	bool ret = false;
-
-	IA_CSS_ENTER_PRIVATE("key = %d", key);
-
-	if ((key >= IA_CSS_PIPELINE_NUM_MAX) || (key >= IA_CSS_PIPE_ID_NUM)) {
-		IA_CSS_ERROR("Invalid key!!");
-		IA_CSS_LEAVE_PRIVATE("return = %d", false);
-		return false;
-	}
-
-	ret = (bool)(pipeline_num_to_sp_thread_map[key] != (unsigned)PIPELINE_NUM_UNMAPPED);
-
-	IA_CSS_LEAVE_PRIVATE("return = %d", ret);
-	return ret;
-}
-
-/*******************************************************
-*** Static functions
-********************************************************/
-
-/* Pipeline:
- * To organize the several different binaries for each type of mode,
- * we use a pipeline. A pipeline contains a number of stages, each with
- * their own binary and frame pointers.
- * When stages are added to a pipeline, output frames that are not passed
- * from outside are automatically allocated.
- * When input frames are not passed from outside, each stage will use the
- * output frame of the previous stage as input (the full resolution output,
- * not the viewfinder output).
- * Pipelines must be cleaned and re-created when settings of the binaries
- * change.
- */
-static void pipeline_stage_destroy(struct ia_css_pipeline_stage *stage)
-{
-	unsigned int i;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		if (stage->out_frame_allocated[i]) {
-			ia_css_frame_free(stage->args.out_frame[i]);
-			stage->args.out_frame[i] = NULL;
-		}
-	}
-	if (stage->vf_frame_allocated) {
-		ia_css_frame_free(stage->args.out_vf_frame);
-		stage->args.out_vf_frame = NULL;
-	}
-	sh_css_free(stage);
-}
-
-static void pipeline_init_sp_thread_map(void)
-{
-	unsigned int i;
-
-	for (i = 1; i < SH_CSS_MAX_SP_THREADS; i++)
-		pipeline_sp_thread_list[i] = PIPELINE_SP_THREAD_EMPTY_TOKEN;
-
-	for (i = 0; i < IA_CSS_PIPELINE_NUM_MAX; i++)
-		pipeline_num_to_sp_thread_map[i] = PIPELINE_NUM_UNMAPPED;
-}
-
-static void pipeline_map_num_to_sp_thread(unsigned int pipe_num)
-{
-	unsigned int i;
-	bool found_sp_thread = false;
-
-	/* pipe is not mapped to any thread */
-	assert(pipeline_num_to_sp_thread_map[pipe_num]
-		== (unsigned)PIPELINE_NUM_UNMAPPED);
-
-	for (i = 0; i < SH_CSS_MAX_SP_THREADS; i++) {
-		if (pipeline_sp_thread_list[i] ==
-		    PIPELINE_SP_THREAD_EMPTY_TOKEN) {
-			pipeline_sp_thread_list[i] =
-			    PIPELINE_SP_THREAD_RESERVED_TOKEN;
-			pipeline_num_to_sp_thread_map[pipe_num] = i;
-			found_sp_thread = true;
-			break;
-		}
-	}
-
-	/* Make sure a mapping is found */
-	/* I could do:
-		assert(i < SH_CSS_MAX_SP_THREADS);
-
-		But the below is more descriptive.
-	*/
-	assert(found_sp_thread);
-}
-
-static void pipeline_unmap_num_to_sp_thread(unsigned int pipe_num)
-{
-	unsigned int thread_id;
-	assert(pipeline_num_to_sp_thread_map[pipe_num]
-		!= (unsigned)PIPELINE_NUM_UNMAPPED);
-
-	thread_id = pipeline_num_to_sp_thread_map[pipe_num];
-	pipeline_num_to_sp_thread_map[pipe_num] = PIPELINE_NUM_UNMAPPED;
-	pipeline_sp_thread_list[thread_id] = PIPELINE_SP_THREAD_EMPTY_TOKEN;
-}
-
-static enum ia_css_err pipeline_stage_create(
-	struct ia_css_pipeline_stage_desc *stage_desc,
-	struct ia_css_pipeline_stage **new_stage)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_pipeline_stage *stage = NULL;
-	struct ia_css_binary *binary;
-	struct ia_css_frame *vf_frame;
-	struct ia_css_frame *out_frame[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	const struct ia_css_fw_info *firmware;
-	unsigned int i;
-
-	/* Verify input parameters*/
-	if (!(stage_desc->in_frame) && !(stage_desc->firmware)
-	    && (stage_desc->binary) && !(stage_desc->binary->online)) {
-		err = IA_CSS_ERR_INTERNAL_ERROR;
-		goto ERR;
-	}
-
-	binary = stage_desc->binary;
-	firmware = stage_desc->firmware;
-	vf_frame = stage_desc->vf_frame;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		out_frame[i] = stage_desc->out_frame[i];
-	}
-
-	stage = sh_css_malloc(sizeof(*stage));
-	if (stage == NULL) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-	memset(stage, 0, sizeof(*stage));
-
-	if (firmware) {
-		stage->binary = NULL;
-		stage->binary_info =
-		    (struct ia_css_binary_info *)&firmware->info.isp;
-	} else {
-		stage->binary = binary;
-		if (binary)
-			stage->binary_info =
-			    (struct ia_css_binary_info *)binary->info;
-		else
-			stage->binary_info = NULL;
-	}
-
-	stage->firmware = firmware;
-	stage->sp_func = stage_desc->sp_func;
-	stage->max_input_width = stage_desc->max_input_width;
-	stage->mode = stage_desc->mode;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		stage->out_frame_allocated[i] = false;
-	stage->vf_frame_allocated = false;
-	stage->next = NULL;
-	sh_css_binary_args_reset(&stage->args);
-
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		if (!(out_frame[i]) && (binary)
-			&& (binary->out_frame_info[i].res.width)) {
-			err = ia_css_frame_allocate_from_info(&out_frame[i],
-							&binary->out_frame_info[i]);
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-			stage->out_frame_allocated[i] = true;
-		}
-	}
-	/* VF frame is not needed in case of need_pp
-	   However, the capture binary needs a vf frame to write to.
-	 */
-	if (!vf_frame) {
-		if ((binary && binary->vf_frame_info.res.width) ||
-		    (firmware && firmware->info.isp.sp.enable.vf_veceven)
-		    ) {
-			err = ia_css_frame_allocate_from_info(&vf_frame,
-							&binary->vf_frame_info);
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-			stage->vf_frame_allocated = true;
-		}
-	} else if (vf_frame && binary && binary->vf_frame_info.res.width
-		&& !firmware) {
-		/* only mark as allocated if buffer pointer available */
-		if (vf_frame->data != mmgr_NULL)
-			stage->vf_frame_allocated = true;
-	}
-
-	stage->args.in_frame = stage_desc->in_frame;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		stage->args.out_frame[i] = out_frame[i];
-	stage->args.out_vf_frame = vf_frame;
-	*new_stage = stage;
-	return err;
-ERR:
-	if (stage != NULL)
-		pipeline_stage_destroy(stage);
-	return err;
-}
-
-static void pipeline_init_defaults(
-	struct ia_css_pipeline *pipeline,
-	enum ia_css_pipe_id pipe_id,
-	unsigned int pipe_num,
-	unsigned int dvs_frame_delay)
-{
-	unsigned int i;
-
-	pipeline->pipe_id = pipe_id;
-	pipeline->stages = NULL;
-	pipeline->stop_requested = false;
-	pipeline->current_stage = NULL;
-	pipeline->in_frame = DEFAULT_FRAME;
-	for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-		pipeline->out_frame[i] = DEFAULT_FRAME;
-		pipeline->vf_frame[i] = DEFAULT_FRAME;
-	}
-	pipeline->num_execs = -1;
-	pipeline->acquire_isp_each_stage = true;
-	pipeline->pipe_num = (uint8_t)pipe_num;
-	pipeline->dvs_frame_delay = dvs_frame_delay;
-}
-
-static void ia_css_pipeline_set_zoom_stage(struct ia_css_pipeline *pipeline)
-{
-	struct ia_css_pipeline_stage *stage = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	assert(pipeline != NULL);
-	if (pipeline->pipe_id == IA_CSS_PIPE_ID_PREVIEW) {
-		/* in preview pipeline, vf_pp stage should do zoom */
-		err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_VF_PP, &stage);
-		if (err == IA_CSS_SUCCESS)
-			stage->enable_zoom = true;
-	} else if (pipeline->pipe_id == IA_CSS_PIPE_ID_CAPTURE) {
-		/* in capture pipeline, capture_pp stage should do zoom */
-		err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_CAPTURE_PP, &stage);
-		if (err == IA_CSS_SUCCESS)
-			stage->enable_zoom = true;
-	} else if (pipeline->pipe_id == IA_CSS_PIPE_ID_VIDEO) {
-		/* in video pipeline, video stage should do zoom */
-		err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_VIDEO, &stage);
-		if (err == IA_CSS_SUCCESS)
-			stage->enable_zoom = true;
-	} else if (pipeline->pipe_id == IA_CSS_PIPE_ID_YUVPP) {
-		/* in yuvpp pipeline, first yuv_scaler stage should do zoom */
-		err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_CAPTURE_PP, &stage);
-		if (err == IA_CSS_SUCCESS)
-			stage->enable_zoom = true;
-	}
-}
-
-static void
-ia_css_pipeline_configure_inout_port(struct ia_css_pipeline *me, bool continuous)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"ia_css_pipeline_configure_inout_port() enter: pipe_id(%d) continuous(%d)\n",
-			me->pipe_id, continuous);
-	switch (me->pipe_id) {
-		case IA_CSS_PIPE_ID_PREVIEW:
-		case IA_CSS_PIPE_ID_VIDEO:
-			SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_INPUT,
-						   (uint8_t)(continuous ? SH_CSS_COPYSINK_TYPE : SH_CSS_HOST_TYPE), 1);
-			SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_OUTPUT,
-						   (uint8_t)SH_CSS_HOST_TYPE, 1);
-			break;
-		case IA_CSS_PIPE_ID_COPY: /*Copy pipe ports configured to "offline" mode*/
-			SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_INPUT,
-						   (uint8_t)SH_CSS_HOST_TYPE, 1);
-			if (continuous) {
-				SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_OUTPUT,
-						   (uint8_t)SH_CSS_COPYSINK_TYPE, 1);
-				SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_OUTPUT,
-						   (uint8_t)SH_CSS_TAGGERSINK_TYPE, 1);
-			} else {
-				SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_OUTPUT,
-						   (uint8_t)SH_CSS_HOST_TYPE, 1);
-			}
-			break;
-		case IA_CSS_PIPE_ID_CAPTURE:
-			SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_INPUT,
-						   (uint8_t)(continuous ? SH_CSS_TAGGERSINK_TYPE : SH_CSS_HOST_TYPE),
-						   1);
-			SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_OUTPUT,
-						   (uint8_t)SH_CSS_HOST_TYPE, 1);
-			break;
-		case IA_CSS_PIPE_ID_YUVPP:
-			SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_INPUT,
-						   (uint8_t)(SH_CSS_HOST_TYPE), 1);
-			SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_OUTPUT,
-						   (uint8_t)SH_CSS_HOST_TYPE, 1);
-			break;
-		case IA_CSS_PIPE_ID_ACC:
-			SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_INPUT,
-						   (uint8_t)SH_CSS_HOST_TYPE, 1);
-			SH_CSS_PIPE_PORT_CONFIG_SET(me->inout_port_config,
-						   (uint8_t)SH_CSS_PORT_OUTPUT,
-						   (uint8_t)SH_CSS_HOST_TYPE, 1);
-			break;
-		default:
-			break;
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"ia_css_pipeline_configure_inout_port() leave: inout_port_config(%x)\n",
-		me->inout_port_config);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/interface/ia_css_queue.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/interface/ia_css_queue.h
deleted file mode 100644
index aaf2e247cafb..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/interface/ia_css_queue.h
+++ /dev/null
@@ -1,192 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_QUEUE_H
-#define __IA_CSS_QUEUE_H
-
-#include <platform_support.h>
-#include <type_support.h>
-
-#include "ia_css_queue_comm.h"
-#include "../src/queue_access.h"
-
-/* Local Queue object descriptor */
-struct ia_css_queue_local {
-	ia_css_circbuf_desc_t *cb_desc; /*Circbuf desc for local queues*/
-	ia_css_circbuf_elem_t *cb_elems; /*Circbuf elements*/
-};
-typedef struct ia_css_queue_local ia_css_queue_local_t;
-
-/* Handle for queue object*/
-typedef struct ia_css_queue ia_css_queue_t;
-
-
-/*****************************************************************************
- * Queue Public APIs
- *****************************************************************************/
-/* @brief Initialize a local queue instance.
- *
- * @param[out] qhandle. Handle to queue instance for use with API
- * @param[in]  desc.   Descriptor with queue properties filled-in
- * @return     0      - Successful init of local queue instance.
- * @return     EINVAL - Invalid argument.
- *
- */
-extern int ia_css_queue_local_init(
-			ia_css_queue_t *qhandle,
-			ia_css_queue_local_t *desc);
-
-/* @brief Initialize a remote queue instance
- *
- * @param[out] qhandle. Handle to queue instance for use with API
- * @param[in]  desc.   Descriptor with queue properties filled-in
- * @return     0      - Successful init of remote queue instance.
- * @return     EINVAL - Invalid argument.
- */
-extern int ia_css_queue_remote_init(
-			ia_css_queue_t *qhandle,
-			ia_css_queue_remote_t *desc);
-
-/* @brief Uninitialize a queue instance
- *
- * @param[in]  qhandle. Handle to queue instance
- * @return     0 - Successful uninit.
- *
- */
-extern int ia_css_queue_uninit(
-			ia_css_queue_t *qhandle);
-
-/* @brief Enqueue an item in the queue instance
- *
- * @param[in]  qhandle. Handle to queue instance
- * @param[in]  item.    Object to be enqueued.
- * @return     0       - Successful enqueue.
- * @return     EINVAL  - Invalid argument.
- * @return     ENOBUFS - Queue is full.
- *
- */
-extern int ia_css_queue_enqueue(
-			ia_css_queue_t *qhandle,
-			uint32_t item);
-
-/* @brief Dequeue an item from the queue instance
- *
- * @param[in]  qhandle. Handle to queue instance
- * @param[out] item.    Object to be dequeued into this item.
-
- * @return     0       - Successful dequeue.
- * @return     EINVAL  - Invalid argument.
- * @return     ENODATA - Queue is empty.
- *
- */
-extern int ia_css_queue_dequeue(
-			ia_css_queue_t *qhandle,
-			uint32_t *item);
-
-/* @brief Check if the queue is empty
- *
- * @param[in]  qhandle.  Handle to queue instance
- * @param[in]  is_empty  True if empty, False if not.
- * @return     0       - Successful access state.
- * @return     EINVAL  - Invalid argument.
- * @return     ENOSYS  - Function not implemented.
- *
- */
-extern int ia_css_queue_is_empty(
-			ia_css_queue_t *qhandle,
-			bool *is_empty);
-
-/* @brief Check if the queue is full
- *
- * @param[in]  qhandle.  Handle to queue instance
- * @param[in]  is_full   True if Full, False if not.
- * @return     0       - Successfully access state.
- * @return     EINVAL  - Invalid argument.
- * @return     ENOSYS  - Function not implemented.
- *
- */
-extern int ia_css_queue_is_full(
-			ia_css_queue_t *qhandle,
-			bool *is_full);
-
-/* @brief Get used space in the queue
- *
- * @param[in]  qhandle.  Handle to queue instance
- * @param[in]  size      Number of available elements in the queue
- * @return     0       - Successfully access state.
- * @return     EINVAL  - Invalid argument.
- *
- */
-extern int ia_css_queue_get_used_space(
-			ia_css_queue_t *qhandle,
-			uint32_t *size);
-
-/* @brief Get free space in the queue
- *
- * @param[in]  qhandle.  Handle to queue instance
- * @param[in]  size      Number of free elements in the queue
- * @return     0       - Successfully access state.
- * @return     EINVAL  - Invalid argument.
- *
- */
-extern int ia_css_queue_get_free_space(
-			ia_css_queue_t *qhandle,
-			uint32_t *size);
-
-/* @brief Peek at an element in the queue
- *
- * @param[in]  qhandle.  Handle to queue instance
- * @param[in]  offset   Offset of element to peek,
- * 			 starting from head of queue
- * @param[in]  element   Value of element returned
- * @return     0       - Successfully access state.
- * @return     EINVAL  - Invalid argument.
- *
- */
-extern int ia_css_queue_peek(
-		ia_css_queue_t *qhandle,
-		uint32_t offset,
-		uint32_t *element);
-
-/* @brief Get the usable size for the queue
- *
- * @param[in]  qhandle. Handle to queue instance
- * @param[out] size     Size value to be returned here.
- * @return     0       - Successful get size.
- * @return     EINVAL  - Invalid argument.
- * @return     ENOSYS  - Function not implemented.
- *
- */
-extern int ia_css_queue_get_size(
-		ia_css_queue_t *qhandle,
-		uint32_t *size);
-
-#endif /* __IA_CSS_QUEUE_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/interface/ia_css_queue_comm.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/interface/ia_css_queue_comm.h
deleted file mode 100644
index 4ebaeb0c1847..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/interface/ia_css_queue_comm.h
+++ /dev/null
@@ -1,69 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_QUEUE_COMM_H
-#define __IA_CSS_QUEUE_COMM_H
-
-#include "type_support.h"
-#include "ia_css_circbuf.h"
-/*****************************************************************************
- * Queue Public Data Structures
- *****************************************************************************/
-
-/* Queue location specifier */
-/* Avoiding enums to save space */
-#define IA_CSS_QUEUE_LOC_HOST 0
-#define IA_CSS_QUEUE_LOC_SP   1
-#define IA_CSS_QUEUE_LOC_ISP  2
-
-/* Queue type specifier */
-/* Avoiding enums to save space */
-#define IA_CSS_QUEUE_TYPE_LOCAL  0
-#define IA_CSS_QUEUE_TYPE_REMOTE 1
-
-/* for DDR Allocated queues,
-allocate minimum these many elements.
-DDR->SP' DMEM DMA transfer needs 32byte aligned address.
-Since each element size is 4 bytes, 8 elements need to be
-DMAed to access single element.*/
-#define IA_CSS_MIN_ELEM_COUNT    8
-#define IA_CSS_DMA_XFER_MASK (IA_CSS_MIN_ELEM_COUNT - 1)
-
-/* Remote Queue object descriptor */
-struct ia_css_queue_remote {
-	uint32_t cb_desc_addr; /*Circbuf desc address for remote queues*/
-	uint32_t cb_elems_addr; /*Circbuf elements addr for remote queue*/
-	uint8_t location;    /* Cell location for queue */
-	uint8_t proc_id;     /* Processor id for queue access */
-};
-typedef struct ia_css_queue_remote ia_css_queue_remote_t;
-
-
-#endif /* __IA_CSS_QUEUE_COMM_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue.c
deleted file mode 100644
index 606376fdf0ba..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue.c
+++ /dev/null
@@ -1,412 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_queue.h"
-#include <math_support.h>
-#include <ia_css_circbuf.h>
-#include <ia_css_circbuf_desc.h>
-#include "queue_access.h"
-
-/*****************************************************************************
- * Queue Public APIs
- *****************************************************************************/
-int ia_css_queue_local_init(
-			ia_css_queue_t *qhandle,
-			ia_css_queue_local_t *desc)
-{
-	if (NULL == qhandle || NULL == desc
-		|| NULL == desc->cb_elems || NULL == desc->cb_desc) {
-		/* Invalid parameters, return error*/
-		return EINVAL;
-	}
-
-	/* Mark the queue as Local */
-	qhandle->type = IA_CSS_QUEUE_TYPE_LOCAL;
-
-	/* Create a local circular buffer queue*/
-	ia_css_circbuf_create(&qhandle->desc.cb_local,
-	      desc->cb_elems,
-	      desc->cb_desc);
-
-	return 0;
-}
-
-int ia_css_queue_remote_init(
-			ia_css_queue_t *qhandle,
-			ia_css_queue_remote_t *desc)
-{
-	if (NULL == qhandle || NULL == desc) {
-		/* Invalid parameters, return error*/
-		return EINVAL;
-	}
-
-	/* Mark the queue as remote*/
-	qhandle->type = IA_CSS_QUEUE_TYPE_REMOTE;
-
-	/* Copy over the local queue descriptor*/
-	qhandle->location = desc->location;
-	qhandle->proc_id = desc->proc_id;
-	qhandle->desc.remote.cb_desc_addr = desc->cb_desc_addr;
-	qhandle->desc.remote.cb_elems_addr = desc->cb_elems_addr;
-
-	/* If queue is remote, we let the local processor
-	 * do its init, before using it. This is just to get us
-	 * started, we can remove this restriction as we go ahead
-	 */
-
-	return 0;
-}
-
-int ia_css_queue_uninit(
-			ia_css_queue_t *qhandle)
-{
-	if (!qhandle)
-		return EINVAL;
-
-	/* Load the required queue object */
-	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
-		/* Local queues are created. Destroy it*/
-		ia_css_circbuf_destroy(&qhandle->desc.cb_local);
-	}
-
-	return 0;
-}
-
-int ia_css_queue_enqueue(
-			ia_css_queue_t *qhandle,
-			uint32_t item)
-{
-	int error = 0;
-	if (NULL == qhandle)
-		return EINVAL;
-
-	/* 1. Load the required queue object */
-	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
-		/* Directly de-ref the object and
-		 * operate on the queue
-		 */
-		if (ia_css_circbuf_is_full(&qhandle->desc.cb_local)) {
-			/* Cannot push the element. Return*/
-			return ENOBUFS;
-		}
-
-		/* Push the element*/
-		ia_css_circbuf_push(&qhandle->desc.cb_local, item);
-	} else if (qhandle->type == IA_CSS_QUEUE_TYPE_REMOTE) {
-		ia_css_circbuf_desc_t cb_desc;
-		ia_css_circbuf_elem_t cb_elem;
-		uint32_t ignore_desc_flags = QUEUE_IGNORE_STEP_FLAG;
-
-		/* a. Load the queue cb_desc from remote */
-		QUEUE_CB_DESC_INIT(&cb_desc);
-		error = ia_css_queue_load(qhandle, &cb_desc, ignore_desc_flags);
-		if (error != 0)
-			return error;
-
-		/* b. Operate on the queue */
-		if (ia_css_circbuf_desc_is_full(&cb_desc))
-			return ENOBUFS;
-
-		cb_elem.val = item;
-
-		error = ia_css_queue_item_store(qhandle, cb_desc.end, &cb_elem);
-		if (error != 0)
-			return error;
-
-		cb_desc.end = (cb_desc.end + 1) % cb_desc.size;
-
-		/* c. Store the queue object */
-		/* Set only fields requiring update with
-		 * valid value. Avoids uncessary calls
-		 * to load/store functions
-		 */
-		ignore_desc_flags = QUEUE_IGNORE_SIZE_START_STEP_FLAGS;
-
-		error = ia_css_queue_store(qhandle, &cb_desc, ignore_desc_flags);
-		if (error != 0)
-			return error;
-	}
-
-	return 0;
-}
-
-int ia_css_queue_dequeue(
-			ia_css_queue_t *qhandle,
-			uint32_t *item)
-{
-	int error = 0;
-	if (qhandle == NULL || NULL == item)
-		return EINVAL;
-
-	/* 1. Load the required queue object */
-	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
-		/* Directly de-ref the object and
-		 * operate on the queue
-		 */
-		if (ia_css_circbuf_is_empty(&qhandle->desc.cb_local)) {
-			/* Nothing to pop. Return empty queue*/
-			return ENODATA;
-		}
-
-		*item = ia_css_circbuf_pop(&qhandle->desc.cb_local);
-	} else if (qhandle->type == IA_CSS_QUEUE_TYPE_REMOTE) {
-		/* a. Load the queue from remote */
-		ia_css_circbuf_desc_t cb_desc;
-		ia_css_circbuf_elem_t cb_elem;
-		uint32_t ignore_desc_flags = QUEUE_IGNORE_STEP_FLAG;
-
-		QUEUE_CB_DESC_INIT(&cb_desc);
-
-		error = ia_css_queue_load(qhandle, &cb_desc, ignore_desc_flags);
-		if (error != 0)
-			return error;
-
-		/* b. Operate on the queue */
-		if (ia_css_circbuf_desc_is_empty(&cb_desc))
-			return ENODATA;
-
-		error = ia_css_queue_item_load(qhandle, cb_desc.start, &cb_elem);
-		if (error != 0)
-			return error;
-
-		*item = cb_elem.val;
-
-		cb_desc.start = OP_std_modadd(cb_desc.start, 1, cb_desc.size);
-
-		/* c. Store the queue object */
-		/* Set only fields requiring update with
-		 * valid value. Avoids uncessary calls
-		 * to load/store functions
-		 */
-		ignore_desc_flags = QUEUE_IGNORE_SIZE_END_STEP_FLAGS;
-		error = ia_css_queue_store(qhandle, &cb_desc, ignore_desc_flags);
-		if (error != 0)
-			return error;
-	}
-	return 0;
-}
-
-int ia_css_queue_is_full(
-			ia_css_queue_t *qhandle,
-			bool *is_full)
-{
-	int error = 0;
-	if ((qhandle == NULL) || (is_full == NULL))
-		return EINVAL;
-
-	/* 1. Load the required queue object */
-	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
-		/* Directly de-ref the object and
-		 * operate on the queue
-		 */
-		*is_full = ia_css_circbuf_is_full(&qhandle->desc.cb_local);
-		return 0;
-	} else if (qhandle->type == IA_CSS_QUEUE_TYPE_REMOTE) {
-		/* a. Load the queue from remote */
-		ia_css_circbuf_desc_t cb_desc;
-		uint32_t ignore_desc_flags = QUEUE_IGNORE_STEP_FLAG;
-		QUEUE_CB_DESC_INIT(&cb_desc);
-		error = ia_css_queue_load(qhandle, &cb_desc, ignore_desc_flags);
-		if (error != 0)
-			return error;
-
-		/* b. Operate on the queue */
-		*is_full = ia_css_circbuf_desc_is_full(&cb_desc);
-		return 0;
-	}
-
-	return EINVAL;
-}
-
-int ia_css_queue_get_free_space(
-			ia_css_queue_t *qhandle,
-			uint32_t *size)
-{
-	int error = 0;
-	if ((qhandle == NULL) || (size == NULL))
-		return EINVAL;
-
-	/* 1. Load the required queue object */
-	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
-		/* Directly de-ref the object and
-		 * operate on the queue
-		 */
-		*size = ia_css_circbuf_get_free_elems(&qhandle->desc.cb_local);
-		return 0;
-	} else if (qhandle->type == IA_CSS_QUEUE_TYPE_REMOTE) {
-		/* a. Load the queue from remote */
-		ia_css_circbuf_desc_t cb_desc;
-		uint32_t ignore_desc_flags = QUEUE_IGNORE_STEP_FLAG;
-		QUEUE_CB_DESC_INIT(&cb_desc);
-		error = ia_css_queue_load(qhandle, &cb_desc, ignore_desc_flags);
-		if (error != 0)
-			return error;
-
-		/* b. Operate on the queue */
-		*size = ia_css_circbuf_desc_get_free_elems(&cb_desc);
-		return 0;
-	}
-
-	return EINVAL;
-}
-
-int ia_css_queue_get_used_space(
-			ia_css_queue_t *qhandle,
-			uint32_t *size)
-{
-	int error = 0;
-	if ((qhandle == NULL) || (size == NULL))
-		return EINVAL;
-
-	/* 1. Load the required queue object */
-	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
-		/* Directly de-ref the object and
-		 * operate on the queue
-		 */
-		*size = ia_css_circbuf_get_num_elems(&qhandle->desc.cb_local);
-		return 0;
-	} else if (qhandle->type == IA_CSS_QUEUE_TYPE_REMOTE) {
-		/* a. Load the queue from remote */
-		ia_css_circbuf_desc_t cb_desc;
-		uint32_t ignore_desc_flags = QUEUE_IGNORE_STEP_FLAG;
-		QUEUE_CB_DESC_INIT(&cb_desc);
-		error = ia_css_queue_load(qhandle, &cb_desc, ignore_desc_flags);
-		if (error != 0)
-			return error;
-
-		/* b. Operate on the queue */
-		*size = ia_css_circbuf_desc_get_num_elems(&cb_desc);
-		return 0;
-	}
-
-	return EINVAL;
-}
-
-int ia_css_queue_peek(
-		ia_css_queue_t *qhandle,
-		uint32_t offset,
-		uint32_t *element)
-{
-	uint32_t num_elems = 0;
-	int error = 0;
-
-	if ((qhandle == NULL) || (element == NULL))
-		return EINVAL;
-
-	/* 1. Load the required queue object */
-	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
-		/* Directly de-ref the object and
-		 * operate on the queue
-		 */
-		/* Check if offset is valid */
-		num_elems = ia_css_circbuf_get_num_elems(&qhandle->desc.cb_local);
-		if (offset > num_elems)
-			return EINVAL;
-
-		*element = ia_css_circbuf_peek_from_start(&qhandle->desc.cb_local, (int) offset);
-		return 0;
-	} else if (qhandle->type == IA_CSS_QUEUE_TYPE_REMOTE) {
-		/* a. Load the queue from remote */
-		ia_css_circbuf_desc_t cb_desc;
-		ia_css_circbuf_elem_t cb_elem;
-		uint32_t ignore_desc_flags = QUEUE_IGNORE_STEP_FLAG;
-
-		QUEUE_CB_DESC_INIT(&cb_desc);
-
-		error =  ia_css_queue_load(qhandle, &cb_desc, ignore_desc_flags);
-		if (error != 0)
-			return error;
-
-		/* Check if offset is valid */
-		num_elems = ia_css_circbuf_desc_get_num_elems(&cb_desc);
-		if (offset > num_elems)
-			return EINVAL;
-
-		offset = OP_std_modadd(cb_desc.start, offset, cb_desc.size);
-		error = ia_css_queue_item_load(qhandle, (uint8_t)offset, &cb_elem);
-		if (error != 0)
-			return error;
-
-		*element = cb_elem.val;
-		return 0;
-	}
-
-	return EINVAL;
-}
-
-int ia_css_queue_is_empty(
-			ia_css_queue_t *qhandle,
-			bool *is_empty)
-{
-	int error = 0;
-	if ((qhandle == NULL) || (is_empty == NULL))
-		return EINVAL;
-
-	/* 1. Load the required queue object */
-	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
-		/* Directly de-ref the object and
-		 * operate on the queue
-		 */
-		*is_empty = ia_css_circbuf_is_empty(&qhandle->desc.cb_local);
-		return 0;
-	} else if (qhandle->type == IA_CSS_QUEUE_TYPE_REMOTE) {
-		/* a. Load the queue from remote */
-		ia_css_circbuf_desc_t cb_desc;
-		uint32_t ignore_desc_flags = QUEUE_IGNORE_STEP_FLAG;
-
-		QUEUE_CB_DESC_INIT(&cb_desc);
-		error = ia_css_queue_load(qhandle, &cb_desc, ignore_desc_flags);
-		if (error != 0)
-			return error;
-
-		/* b. Operate on the queue */
-		*is_empty = ia_css_circbuf_desc_is_empty(&cb_desc);
-		return 0;
-	}
-
-	return EINVAL;
-}
-
-int ia_css_queue_get_size(
-			ia_css_queue_t *qhandle,
-			uint32_t *size)
-{
-	int error = 0;
-	if ((qhandle == NULL) || (size == NULL))
-		return EINVAL;
-
-	/* 1. Load the required queue object */
-	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
-		/* Directly de-ref the object and
-		 * operate on the queue
-		 */
-		/* Return maximum usable capacity */
-		*size = ia_css_circbuf_get_size(&qhandle->desc.cb_local);
-	} else if (qhandle->type == IA_CSS_QUEUE_TYPE_REMOTE) {
-		/* a. Load the queue from remote */
-		ia_css_circbuf_desc_t cb_desc;
-		uint32_t ignore_desc_flags = QUEUE_IGNORE_START_END_STEP_FLAGS;
-
-		QUEUE_CB_DESC_INIT(&cb_desc);
-
-		error = ia_css_queue_load(qhandle, &cb_desc, ignore_desc_flags);
-		if (error != 0)
-			return error;
-
-		/* Return maximum usable capacity */
-		*size = cb_desc.size;
-	}
-
-	return 0;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue_access.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue_access.c
deleted file mode 100644
index 7bb2b494836e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue_access.c
+++ /dev/null
@@ -1,192 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "type_support.h"
-#include "queue_access.h"
-#include "ia_css_circbuf.h"
-#include "sp.h"
-#include "memory_access.h"
-#include "assert_support.h"
-
-int ia_css_queue_load(
-		struct ia_css_queue *rdesc,
-		ia_css_circbuf_desc_t *cb_desc,
-		uint32_t ignore_desc_flags)
-{
-	if (rdesc == NULL || cb_desc == NULL)
-		return EINVAL;
-
-	if (rdesc->location == IA_CSS_QUEUE_LOC_SP) {
-		assert(ignore_desc_flags <= QUEUE_IGNORE_DESC_FLAGS_MAX);
-
-		if (0 == (ignore_desc_flags & QUEUE_IGNORE_SIZE_FLAG)) {
-			cb_desc->size = sp_dmem_load_uint8(rdesc->proc_id,
-				rdesc->desc.remote.cb_desc_addr
-				+ offsetof(ia_css_circbuf_desc_t, size));
-
-			if (0 == cb_desc->size) {
-				/* Adding back the workaround which was removed
-				   while refactoring queues. When reading size
-				   through sp_dmem_load_*, sometimes we get back
-				   the value as zero. This causes division by 0
-				   exception as the size is used in a modular
-				   division operation. */
-				return EDOM;
-			}
-		}
-
-		if (0 == (ignore_desc_flags & QUEUE_IGNORE_START_FLAG))
-			cb_desc->start = sp_dmem_load_uint8(rdesc->proc_id,
-				rdesc->desc.remote.cb_desc_addr
-				+ offsetof(ia_css_circbuf_desc_t, start));
-
-		if (0 == (ignore_desc_flags & QUEUE_IGNORE_END_FLAG))
-			cb_desc->end = sp_dmem_load_uint8(rdesc->proc_id,
-				rdesc->desc.remote.cb_desc_addr
-				+ offsetof(ia_css_circbuf_desc_t, end));
-
-		if (0 == (ignore_desc_flags & QUEUE_IGNORE_STEP_FLAG))
-			cb_desc->step = sp_dmem_load_uint8(rdesc->proc_id,
-				rdesc->desc.remote.cb_desc_addr
-				+ offsetof(ia_css_circbuf_desc_t, step));
-
-	} else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) {
-		/* doing DMA transfer of entire structure */
-		mmgr_load(rdesc->desc.remote.cb_desc_addr,
-			(void *)cb_desc,
-			sizeof(ia_css_circbuf_desc_t));
-	} else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) {
-		/* Not supported yet */
-		return ENOTSUP;
-	}
-
-	return 0;
-}
-
-int ia_css_queue_store(
-		struct ia_css_queue *rdesc,
-		ia_css_circbuf_desc_t *cb_desc,
-		uint32_t ignore_desc_flags)
-{
-	if (rdesc == NULL || cb_desc == NULL)
-		return EINVAL;
-
-	if (rdesc->location == IA_CSS_QUEUE_LOC_SP) {
-		assert(ignore_desc_flags <= QUEUE_IGNORE_DESC_FLAGS_MAX);
-
-		if (0 == (ignore_desc_flags & QUEUE_IGNORE_SIZE_FLAG))
-			sp_dmem_store_uint8(rdesc->proc_id,
-				rdesc->desc.remote.cb_desc_addr
-				+ offsetof(ia_css_circbuf_desc_t, size),
-				cb_desc->size);
-
-		if (0 == (ignore_desc_flags & QUEUE_IGNORE_START_FLAG))
-			sp_dmem_store_uint8(rdesc->proc_id,
-				rdesc->desc.remote.cb_desc_addr
-				+ offsetof(ia_css_circbuf_desc_t, start),
-				cb_desc->start);
-
-		if (0 == (ignore_desc_flags & QUEUE_IGNORE_END_FLAG))
-			sp_dmem_store_uint8(rdesc->proc_id,
-				rdesc->desc.remote.cb_desc_addr
-				+ offsetof(ia_css_circbuf_desc_t, end),
-				cb_desc->end);
-
-		if (0 == (ignore_desc_flags & QUEUE_IGNORE_STEP_FLAG))
-			sp_dmem_store_uint8(rdesc->proc_id,
-				rdesc->desc.remote.cb_desc_addr
-				+ offsetof(ia_css_circbuf_desc_t, step),
-				cb_desc->step);
-	} else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) {
-		/* doing DMA transfer of entire structure */
-		mmgr_store(rdesc->desc.remote.cb_desc_addr,
-			(void *)cb_desc,
-			sizeof(ia_css_circbuf_desc_t));
-	} else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) {
-		/* Not supported yet */
-		return ENOTSUP;
-	}
-
-	return 0;
-}
-
-int ia_css_queue_item_load(
-		struct ia_css_queue *rdesc,
-		uint8_t position,
-		ia_css_circbuf_elem_t *item)
-{
-	if (rdesc == NULL || item == NULL)
-		return EINVAL;
-
-	if (rdesc->location == IA_CSS_QUEUE_LOC_SP) {
-		sp_dmem_load(rdesc->proc_id,
-			rdesc->desc.remote.cb_elems_addr
-			+ position * sizeof(ia_css_circbuf_elem_t),
-			item,
-			sizeof(ia_css_circbuf_elem_t));
-	} else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) {
-		mmgr_load(rdesc->desc.remote.cb_elems_addr
-			+ position * sizeof(ia_css_circbuf_elem_t),
-			(void *)item,
-			sizeof(ia_css_circbuf_elem_t));
-	} else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) {
-		/* Not supported yet */
-		return ENOTSUP;
-	}
-
-	return 0;
-}
-
-int ia_css_queue_item_store(
-		struct ia_css_queue *rdesc,
-		uint8_t position,
-		ia_css_circbuf_elem_t *item)
-{
-	if (rdesc == NULL || item == NULL)
-		return EINVAL;
-
-	if (rdesc->location == IA_CSS_QUEUE_LOC_SP) {
-		sp_dmem_store(rdesc->proc_id,
-			rdesc->desc.remote.cb_elems_addr
-			+ position * sizeof(ia_css_circbuf_elem_t),
-			item,
-			sizeof(ia_css_circbuf_elem_t));
-	} else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) {
-		mmgr_store(rdesc->desc.remote.cb_elems_addr
-			+ position * sizeof(ia_css_circbuf_elem_t),
-			(void *)item,
-			sizeof(ia_css_circbuf_elem_t));
-	} else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) {
-		/* Not supported yet */
-		return ENOTSUP;
-	}
-
-	return 0;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue_access.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue_access.h
deleted file mode 100644
index 4775513f54cf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/queue/src/queue_access.h
+++ /dev/null
@@ -1,101 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __QUEUE_ACCESS_H
-#define __QUEUE_ACCESS_H
-
-#include <type_support.h>
-#include <ia_css_queue_comm.h>
-#include <ia_css_circbuf.h>
-#include <error_support.h>
-
-#define QUEUE_IGNORE_START_FLAG	0x0001
-#define QUEUE_IGNORE_END_FLAG	0x0002
-#define QUEUE_IGNORE_SIZE_FLAG	0x0004
-#define QUEUE_IGNORE_STEP_FLAG	0x0008
-#define QUEUE_IGNORE_DESC_FLAGS_MAX 0x000f
-
-#define QUEUE_IGNORE_SIZE_START_STEP_FLAGS \
-	(QUEUE_IGNORE_SIZE_FLAG | \
-	QUEUE_IGNORE_START_FLAG | \
-	QUEUE_IGNORE_STEP_FLAG)
-
-#define QUEUE_IGNORE_SIZE_END_STEP_FLAGS \
-	(QUEUE_IGNORE_SIZE_FLAG | \
-	QUEUE_IGNORE_END_FLAG   | \
-	QUEUE_IGNORE_STEP_FLAG)
-
-#define QUEUE_IGNORE_START_END_STEP_FLAGS \
-	(QUEUE_IGNORE_START_FLAG | \
-	QUEUE_IGNORE_END_FLAG	  | \
-	QUEUE_IGNORE_STEP_FLAG)
-
-#define QUEUE_CB_DESC_INIT(cb_desc)	\
-	do {				\
-		(cb_desc)->size  = 0;	\
-		(cb_desc)->step  = 0;	\
-		(cb_desc)->start = 0;	\
-		(cb_desc)->end   = 0;	\
-	} while(0)
-
-struct ia_css_queue {
-	uint8_t type;        /* Specify remote/local type of access */
-	uint8_t location;    /* Cell location for queue */
-	uint8_t proc_id;     /* Processor id for queue access */
-	union {
-		ia_css_circbuf_t cb_local;
-		struct {
-			uint32_t cb_desc_addr; /*Circbuf desc address for remote queues*/
-			uint32_t cb_elems_addr; /*Circbuf elements addr for remote queue*/
-		}	remote;
-	} desc;
-};
-
-extern int ia_css_queue_load(
-		struct ia_css_queue *rdesc,
-		ia_css_circbuf_desc_t *cb_desc,
-		uint32_t ignore_desc_flags);
-
-extern int ia_css_queue_store(
-		struct ia_css_queue *rdesc,
-		ia_css_circbuf_desc_t *cb_desc,
-		uint32_t ignore_desc_flags);
-
-extern int ia_css_queue_item_load(
-		struct ia_css_queue *rdesc,
-		uint8_t position,
-		ia_css_circbuf_elem_t *item);
-
-extern int ia_css_queue_item_store(
-		struct ia_css_queue *rdesc,
-		uint8_t position,
-		ia_css_circbuf_elem_t *item);
-
-#endif /* __QUEUE_ACCESS_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/interface/ia_css_rmgr.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/interface/ia_css_rmgr.h
deleted file mode 100644
index 9f78e709b3d0..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/interface/ia_css_rmgr.h
+++ /dev/null
@@ -1,88 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _IA_CSS_RMGR_H
-#define _IA_CSS_RMGR_H
-
-#include <ia_css_err.h>
-
-#ifndef __INLINE_RMGR__
-#define STORAGE_CLASS_RMGR_H extern
-#define STORAGE_CLASS_RMGR_C
-#else				/* __INLINE_RMGR__ */
-#define STORAGE_CLASS_RMGR_H static inline
-#define STORAGE_CLASS_RMGR_C static inline
-#endif				/* __INLINE_RMGR__ */
-
-/**
- * @brief Initialize resource manager (host/common)
- */
-enum ia_css_err ia_css_rmgr_init(void);
-
-/**
- * @brief Uninitialize resource manager (host/common)
- */
-void ia_css_rmgr_uninit(void);
-
-/*****************************************************************
- * Interface definition - resource type (host/common)
- *****************************************************************
- *
- * struct ia_css_rmgr_<type>_pool;
- * struct ia_css_rmgr_<type>_handle;
- *
- * STORAGE_CLASS_RMGR_H void ia_css_rmgr_init_<type>(
- *	struct ia_css_rmgr_<type>_pool *pool);
- *
- * STORAGE_CLASS_RMGR_H void ia_css_rmgr_uninit_<type>(
- *	struct ia_css_rmgr_<type>_pool *pool);
- *
- * STORAGE_CLASS_RMGR_H void ia_css_rmgr_acq_<type>(
- *	struct ia_css_rmgr_<type>_pool *pool,
- *	struct ia_css_rmgr_<type>_handle **handle);
- *
- * STORAGE_CLASS_RMGR_H void ia_css_rmgr_rel_<type>(
- *	struct ia_css_rmgr_<type>_pool *pool,
- *	struct ia_css_rmgr_<type>_handle **handle);
- *
- *****************************************************************
- * Interface definition - refcounting (host/common)
- *****************************************************************
- *
- * void ia_css_rmgr_refcount_retain_<type>(
- *	struct ia_css_rmgr_<type>_handle **handle);
- *
- * void ia_css_rmgr_refcount_release_<type>(
- *	struct ia_css_rmgr_<type>_handle **handle);
- */
-
-#include "ia_css_rmgr_vbuf.h"
-
-#endif	/* _IA_CSS_RMGR_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/interface/ia_css_rmgr_vbuf.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/interface/ia_css_rmgr_vbuf.h
deleted file mode 100644
index 90ac27cf02cf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/interface/ia_css_rmgr_vbuf.h
+++ /dev/null
@@ -1,115 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef _IA_CSS_RMGR_VBUF_H
-#define _IA_CSS_RMGR_VBUF_H
-
-#include "ia_css_rmgr.h"
-#include <type_support.h>
-#include <system_types.h>
-
-/**
- * @brief Data structure for the resource handle (host, vbuf)
- */
-struct ia_css_rmgr_vbuf_handle {
-	hrt_vaddress vptr;
-	uint8_t count;
-	uint32_t size;
-};
-
-/**
- * @brief Data structure for the resource pool (host, vbuf)
- */
-struct ia_css_rmgr_vbuf_pool {
-	uint8_t copy_on_write;
-	uint8_t recycle;
-	uint32_t size;
-	uint32_t index;
-	struct ia_css_rmgr_vbuf_handle **handles;
-};
-
-/**
- * @brief VBUF resource pools
- */
-extern struct ia_css_rmgr_vbuf_pool *vbuf_ref;
-extern struct ia_css_rmgr_vbuf_pool *vbuf_write;
-extern struct ia_css_rmgr_vbuf_pool *hmm_buffer_pool;
-
-/**
- * @brief Initialize the resource pool (host, vbuf)
- *
- * @param pool	The pointer to the pool
- */
-STORAGE_CLASS_RMGR_H enum ia_css_err ia_css_rmgr_init_vbuf(
-	struct ia_css_rmgr_vbuf_pool *pool);
-
-/**
- * @brief Uninitialize the resource pool (host, vbuf)
- *
- * @param pool	The pointer to the pool
- */
-STORAGE_CLASS_RMGR_H void ia_css_rmgr_uninit_vbuf(
-	struct ia_css_rmgr_vbuf_pool *pool);
-
-/**
- * @brief Acquire a handle from the pool (host, vbuf)
- *
- * @param pool		The pointer to the pool
- * @param handle	The pointer to the handle
- */
-STORAGE_CLASS_RMGR_H void ia_css_rmgr_acq_vbuf(
-	struct ia_css_rmgr_vbuf_pool *pool,
-	struct ia_css_rmgr_vbuf_handle **handle);
-
-/**
- * @brief Release a handle to the pool (host, vbuf)
- *
- * @param pool		The pointer to the pool
- * @param handle	The pointer to the handle
- */
-STORAGE_CLASS_RMGR_H void ia_css_rmgr_rel_vbuf(
-	struct ia_css_rmgr_vbuf_pool *pool,
-	struct ia_css_rmgr_vbuf_handle **handle);
-
-/**
- * @brief Retain the reference count for a handle (host, vbuf)
- *
- * @param handle	The pointer to the handle
- */
-void ia_css_rmgr_refcount_retain_vbuf(struct ia_css_rmgr_vbuf_handle **handle);
-
-/**
- * @brief Release the reference count for a handle (host, vbuf)
- *
- * @param handle	The pointer to the handle
- */
-void ia_css_rmgr_refcount_release_vbuf(struct ia_css_rmgr_vbuf_handle **handle);
-
-#endif	/* _IA_CSS_RMGR_VBUF_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/src/rmgr.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/src/rmgr.c
deleted file mode 100644
index 370ff3816dbe..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/src/rmgr.c
+++ /dev/null
@@ -1,55 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "ia_css_rmgr.h"
-
-enum ia_css_err ia_css_rmgr_init(void)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	err = ia_css_rmgr_init_vbuf(vbuf_ref);
-	if (err == IA_CSS_SUCCESS)
-		err = ia_css_rmgr_init_vbuf(vbuf_write);
-	if (err == IA_CSS_SUCCESS)
-		err = ia_css_rmgr_init_vbuf(hmm_buffer_pool);
-	if (err != IA_CSS_SUCCESS)
-		ia_css_rmgr_uninit();
-	return err;
-}
-
-/*
- * @brief Uninitialize resource pool (host)
- */
-void ia_css_rmgr_uninit(void)
-{
-	ia_css_rmgr_uninit_vbuf(hmm_buffer_pool);
-	ia_css_rmgr_uninit_vbuf(vbuf_write);
-	ia_css_rmgr_uninit_vbuf(vbuf_ref);
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/src/rmgr_vbuf.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/src/rmgr_vbuf.c
deleted file mode 100644
index a4d8a48f95ba..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/rmgr/src/rmgr_vbuf.c
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2010-2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_rmgr.h"
-
-#include <type_support.h>
-#include <assert_support.h>
-#include <platform_support.h> /* memset */
-#include <memory_access.h>    /* mmmgr_malloc, mhmm_free */
-#include <ia_css_debug.h>
-
-/*
- * @brief VBUF resource handles
- */
-#define NUM_HANDLES 1000
-static struct ia_css_rmgr_vbuf_handle handle_table[NUM_HANDLES];
-
-/*
- * @brief VBUF resource pool - refpool
- */
-static struct ia_css_rmgr_vbuf_pool refpool = {
-	false,			/* copy_on_write */
-	false,			/* recycle */
-	0,			/* size */
-	0,			/* index */
-	NULL,			/* handles */
-};
-
-/*
- * @brief VBUF resource pool - writepool
- */
-static struct ia_css_rmgr_vbuf_pool writepool = {
-	true,			/* copy_on_write */
-	false,			/* recycle */
-	0,			/* size */
-	0,			/* index */
-	NULL,			/* handles */
-};
-
-/*
- * @brief VBUF resource pool - hmmbufferpool
- */
-static struct ia_css_rmgr_vbuf_pool hmmbufferpool = {
-	true,			/* copy_on_write */
-	true,			/* recycle */
-	32,			/* size */
-	0,			/* index */
-	NULL,			/* handles */
-};
-
-struct ia_css_rmgr_vbuf_pool *vbuf_ref = &refpool;
-struct ia_css_rmgr_vbuf_pool *vbuf_write = &writepool;
-struct ia_css_rmgr_vbuf_pool *hmm_buffer_pool = &hmmbufferpool;
-
-/*
- * @brief Initialize the reference count (host, vbuf)
- */
-static void rmgr_refcount_init_vbuf(void)
-{
-	/* initialize the refcount table */
-	memset(&handle_table, 0, sizeof(handle_table));
-}
-
-/*
- * @brief Retain the reference count for a handle (host, vbuf)
- *
- * @param handle	The pointer to the handle
- */
-void ia_css_rmgr_refcount_retain_vbuf(struct ia_css_rmgr_vbuf_handle **handle)
-{
-	int i;
-	struct ia_css_rmgr_vbuf_handle *h;
-	if ((handle == NULL) || (*handle == NULL)) {
-		IA_CSS_LOG("Invalid inputs");
-		return;
-	}
-	/* new vbuf to count on */
-	if ((*handle)->count == 0) {
-		h = *handle;
-		*handle = NULL;
-		for (i = 0; i < NUM_HANDLES; i++) {
-			if (handle_table[i].count == 0) {
-				*handle = &handle_table[i];
-				break;
-			}
-		}
-		/* if the loop dus not break and *handle == NULL
-		   this is an error handle and report it.
-		 */
-		if (*handle == NULL) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-				"ia_css_i_host_refcount_retain_vbuf() failed to find empty slot!\n");
-			return;
-		}
-		(*handle)->vptr = h->vptr;
-		(*handle)->size = h->size;
-	}
-	(*handle)->count++;
-}
-
-/*
- * @brief Release the reference count for a handle (host, vbuf)
- *
- * @param handle	The pointer to the handle
- */
-void ia_css_rmgr_refcount_release_vbuf(struct ia_css_rmgr_vbuf_handle **handle)
-{
-	if ((handle == NULL) || ((*handle) == NULL) || (((*handle)->count) == 0)) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-				    "ia_css_rmgr_refcount_release_vbuf() invalid arguments!\n");
-		return;
-	}
-	/* decrease reference count */
-	(*handle)->count--;
-	/* remove from admin */
-	if ((*handle)->count == 0) {
-		(*handle)->vptr = 0x0;
-		(*handle)->size = 0;
-		*handle = NULL;
-	}
-}
-
-/*
- * @brief Initialize the resource pool (host, vbuf)
- *
- * @param pool	The pointer to the pool
- */
-enum ia_css_err ia_css_rmgr_init_vbuf(struct ia_css_rmgr_vbuf_pool *pool)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	size_t bytes_needed;
-	rmgr_refcount_init_vbuf();
-	assert(pool != NULL);
-	if (pool == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	/* initialize the recycle pool if used */
-	if (pool->recycle && pool->size) {
-		/* allocate memory for storing the handles */
-		bytes_needed =
-		    sizeof(void *) *
-		    pool->size;
-		pool->handles = sh_css_malloc(bytes_needed);
-		if (pool->handles != NULL)
-			memset(pool->handles, 0, bytes_needed);
-		else
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	} else {
-		/* just in case, set the size to 0 */
-		pool->size = 0;
-		pool->handles = NULL;
-	}
-	return err;
-}
-
-/*
- * @brief Uninitialize the resource pool (host, vbuf)
- *
- * @param pool	The pointer to the pool
- */
-void ia_css_rmgr_uninit_vbuf(struct ia_css_rmgr_vbuf_pool *pool)
-{
-	uint32_t i;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_rmgr_uninit_vbuf()\n");
-	if (pool == NULL) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR, "ia_css_rmgr_uninit_vbuf(): NULL argument\n");
-		return;
-	}
-	if (pool->handles != NULL) {
-		/* free the hmm buffers */
-		for (i = 0; i < pool->size; i++) {
-			if (pool->handles[i] != NULL) {
-				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				      "   freeing/releasing %x (count=%d)\n",
-				      pool->handles[i]->vptr,
-				      pool->handles[i]->count);
-				/* free memory */
-				hmm_free(pool->handles[i]->vptr);
-				/* remove from refcount admin */
-				ia_css_rmgr_refcount_release_vbuf(
-					&pool->handles[i]);
-			}
-		}
-		/* now free the pool handles list */
-		sh_css_free(pool->handles);
-		pool->handles = NULL;
-	}
-}
-
-/*
- * @brief Push a handle to the pool
- *
- * @param pool		The pointer to the pool
- * @param handle	The pointer to the handle
- */
-static
-void rmgr_push_handle(struct ia_css_rmgr_vbuf_pool *pool,
-		      struct ia_css_rmgr_vbuf_handle **handle)
-{
-	uint32_t i;
-	bool succes = false;
-	assert(pool != NULL);
-	assert(pool->recycle);
-	assert(pool->handles != NULL);
-	assert(handle != NULL);
-	for (i = 0; i < pool->size; i++) {
-		if (pool->handles[i] == NULL) {
-			ia_css_rmgr_refcount_retain_vbuf(handle);
-			pool->handles[i] = *handle;
-			succes = true;
-			break;
-		}
-	}
-	assert(succes);
-}
-
-/*
- * @brief Pop a handle from the pool
- *
- * @param pool		The pointer to the pool
- * @param handle	The pointer to the handle
- */
-static
-void rmgr_pop_handle(struct ia_css_rmgr_vbuf_pool *pool,
-		     struct ia_css_rmgr_vbuf_handle **handle)
-{
-	uint32_t i;
-	bool succes = false;
-	assert(pool != NULL);
-	assert(pool->recycle);
-	assert(pool->handles != NULL);
-	assert(handle != NULL);
-	assert(*handle != NULL);
-	for (i = 0; i < pool->size; i++) {
-		if ((pool->handles[i] != NULL) &&
-		    (pool->handles[i]->size == (*handle)->size)) {
-			*handle = pool->handles[i];
-			pool->handles[i] = NULL;
-			/* dont release, we are returning it...
-			   ia_css_rmgr_refcount_release_vbuf(handle); */
-			succes = true;
-			break;
-		}
-	}
-}
-
-/*
- * @brief Acquire a handle from the pool (host, vbuf)
- *
- * @param pool		The pointer to the pool
- * @param handle	The pointer to the handle
- */
-void ia_css_rmgr_acq_vbuf(struct ia_css_rmgr_vbuf_pool *pool,
-			  struct ia_css_rmgr_vbuf_handle **handle)
-{
-	struct ia_css_rmgr_vbuf_handle h;
-
-	if ((pool == NULL) || (handle == NULL) || (*handle == NULL)) {
-		IA_CSS_LOG("Invalid inputs");
-		return;
-	}
-
-	if (pool->copy_on_write) {
-		/* only one reference, reuse (no new retain) */
-		if ((*handle)->count == 1)
-			return;
-		/* more than one reference, release current buffer */
-		if ((*handle)->count > 1) {
-			/* store current values */
-			h.vptr = 0x0;
-			h.size = (*handle)->size;
-			/* release ref to current buffer */
-			ia_css_rmgr_refcount_release_vbuf(handle);
-			*handle = &h;
-		}
-		/* get new buffer for needed size */
-		if ((*handle)->vptr == 0x0) {
-			if (pool->recycle) {
-				/* try and pop from pool */
-				rmgr_pop_handle(pool, handle);
-			}
-			if ((*handle)->vptr == 0x0) {
-				/* we need to allocate */
-				(*handle)->vptr = mmgr_malloc((*handle)->size);
-			} else {
-				/* we popped a buffer */
-				return;
-			}
-		}
-	}
-	/* Note that handle will change to an internally maintained one */
-	ia_css_rmgr_refcount_retain_vbuf(handle);
-}
-
-/*
- * @brief Release a handle to the pool (host, vbuf)
- *
- * @param pool		The pointer to the pool
- * @param handle	The pointer to the handle
- */
-void ia_css_rmgr_rel_vbuf(struct ia_css_rmgr_vbuf_pool *pool,
-			  struct ia_css_rmgr_vbuf_handle **handle)
-{
-	if ((pool == NULL) || (handle == NULL) || (*handle == NULL)) {
-		IA_CSS_LOG("Invalid inputs");
-		return;
-	}
-	/* release the handle */
-	if ((*handle)->count == 1) {
-		if (!pool->recycle) {
-			/* non recycling pool, free mem */
-			hmm_free((*handle)->vptr);
-		} else {
-			/* recycle to pool */
-			rmgr_push_handle(pool, handle);
-		}
-	}
-	ia_css_rmgr_refcount_release_vbuf(handle);
-	*handle = NULL;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/interface/ia_css_spctrl.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/interface/ia_css_spctrl.h
deleted file mode 100644
index bc4b1723369e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/interface/ia_css_spctrl.h
+++ /dev/null
@@ -1,87 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_SPCTRL_H__
-#define __IA_CSS_SPCTRL_H__
-
-#include <system_global.h>
-#include <ia_css_err.h>
-#include "ia_css_spctrl_comm.h"
-
-
-typedef struct {
-	uint32_t        ddr_data_offset;       /**  posistion of data in DDR */
-	uint32_t        dmem_data_addr;        /** data segment address in dmem */
-	uint32_t        dmem_bss_addr;         /** bss segment address in dmem  */
-	uint32_t        data_size;             /** data segment size            */
-	uint32_t        bss_size;              /** bss segment size             */
-	uint32_t        spctrl_config_dmem_addr; /* <location of dmem_cfg  in SP dmem */
-	uint32_t        spctrl_state_dmem_addr;  /* < location of state  in SP dmem */
-	unsigned int    sp_entry;                /* < entry function ptr on SP */
-	const void      *code;                   /** location of firmware */
-	uint32_t         code_size;
-	char      *program_name;    /** not used on hardware, only for simulation */
-} ia_css_spctrl_cfg;
-
-/* Get the code addr in DDR of SP */
-hrt_vaddress get_sp_code_addr(sp_ID_t  sp_id);
-
-/* ! Load firmware on to specfied SP
-*/
-enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id,
-			ia_css_spctrl_cfg *spctrl_cfg);
-
-#ifdef ISP2401
-/*! Setup registers for reloading FW */
-void sh_css_spctrl_reload_fw(sp_ID_t sp_id);
-
-#endif
-/*!  Unload/release any memory allocated to hold the firmware
-*/
-enum ia_css_err ia_css_spctrl_unload_fw(sp_ID_t sp_id);
-
-
-/*! Intilaize dmem_cfg in SP dmem  and  start SP program
-*/
-enum ia_css_err ia_css_spctrl_start(sp_ID_t sp_id);
-
-/*! stop spctrl
-*/
-enum ia_css_err ia_css_spctrl_stop(sp_ID_t sp_id);
-
-/*! Query the state of SP
-*/
-ia_css_spctrl_sp_sw_state ia_css_spctrl_get_state(sp_ID_t sp_id);
-
-/*! Check if SP is idle/ready
-*/
-int ia_css_spctrl_is_idle(sp_ID_t sp_id);
-
-#endif /* __IA_CSS_SPCTRL_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/interface/ia_css_spctrl_comm.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/interface/ia_css_spctrl_comm.h
deleted file mode 100644
index 2620d7514f79..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/interface/ia_css_spctrl_comm.h
+++ /dev/null
@@ -1,61 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_SPCTRL_COMM_H__
-#define __IA_CSS_SPCTRL_COMM_H__
-
-#include <type_support.h>
-
-/* state of SP */
-typedef enum {
-	IA_CSS_SP_SW_TERMINATED = 0,
-	IA_CSS_SP_SW_INITIALIZED,
-	IA_CSS_SP_SW_CONNECTED,
-	IA_CSS_SP_SW_RUNNING
-} ia_css_spctrl_sp_sw_state;
-
-/* Structure to encapsulate required arguments for
- * initialization of SP DMEM using the SP itself
- */
-struct ia_css_sp_init_dmem_cfg {
-	ia_css_ptr      ddr_data_addr;  /** data segment address in ddr  */
-	uint32_t        dmem_data_addr; /** data segment address in dmem */
-	uint32_t        dmem_bss_addr;  /** bss segment address in dmem  */
-	uint32_t        data_size;      /** data segment size            */
-	uint32_t        bss_size;       /** bss segment size             */
-	sp_ID_t         sp_id;          /* <sp Id */
-};
-
-#define SIZE_OF_IA_CSS_SP_INIT_DMEM_CFG_STRUCT	\
-	(1 * SIZE_OF_IA_CSS_PTR) +		\
-	(4 * sizeof(uint32_t)) +		\
-	(1 * sizeof(sp_ID_t))
-
-#endif /* __IA_CSS_SPCTRL_COMM_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/src/spctrl.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/src/spctrl.c
deleted file mode 100644
index 844e4d536cec..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/spctrl/src/spctrl.c
+++ /dev/null
@@ -1,193 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include "ia_css_types.h"
-#define __INLINE_SP__
-#include "sp.h"
-
-#include "memory_access.h"
-#include "assert_support.h"
-#include "ia_css_spctrl.h"
-#include "ia_css_debug.h"
-
-struct spctrl_context_info {
-	struct ia_css_sp_init_dmem_cfg dmem_config;
-	uint32_t        spctrl_config_dmem_addr; /* location of dmem_cfg  in SP dmem */
-	uint32_t        spctrl_state_dmem_addr;
-	unsigned int    sp_entry;           /* entry function ptr on SP */
-	hrt_vaddress    code_addr;          /* sp firmware location in host mem-DDR*/
-	uint32_t        code_size;
-	char           *program_name;       /* used in case of PLATFORM_SIM */
-};
-
-static struct spctrl_context_info spctrl_cofig_info[N_SP_ID];
-static bool spctrl_loaded[N_SP_ID] = {0};
-
-/* Load firmware */
-enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id,
-				ia_css_spctrl_cfg *spctrl_cfg)
-{
-	hrt_vaddress code_addr = mmgr_NULL;
-	struct ia_css_sp_init_dmem_cfg *init_dmem_cfg;
-
-	if ((sp_id >= N_SP_ID) || (spctrl_cfg == NULL))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	spctrl_cofig_info[sp_id].code_addr = mmgr_NULL;
-
-	init_dmem_cfg = &spctrl_cofig_info[sp_id].dmem_config;
-	init_dmem_cfg->dmem_data_addr = spctrl_cfg->dmem_data_addr;
-	init_dmem_cfg->dmem_bss_addr  = spctrl_cfg->dmem_bss_addr;
-	init_dmem_cfg->data_size      = spctrl_cfg->data_size;
-	init_dmem_cfg->bss_size       = spctrl_cfg->bss_size;
-	init_dmem_cfg->sp_id          = sp_id;
-
-	spctrl_cofig_info[sp_id].spctrl_config_dmem_addr = spctrl_cfg->spctrl_config_dmem_addr;
-	spctrl_cofig_info[sp_id].spctrl_state_dmem_addr = spctrl_cfg->spctrl_state_dmem_addr;
-
-	/* store code (text + icache) and data to DDR
-	 *
-	 * Data used to be stored separately, because of access alignment constraints,
-	 * fix the FW generation instead
-	 */
-	code_addr = mmgr_malloc(spctrl_cfg->code_size);
-	if (code_addr == mmgr_NULL)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	mmgr_store(code_addr, spctrl_cfg->code, spctrl_cfg->code_size);
-
-	if (sizeof(hrt_vaddress) > sizeof(hrt_data)) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-				    "size of hrt_vaddress can not be greater than hrt_data\n");
-		hmm_free(code_addr);
-		code_addr = mmgr_NULL;
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-	init_dmem_cfg->ddr_data_addr  = code_addr + spctrl_cfg->ddr_data_offset;
-	if ((init_dmem_cfg->ddr_data_addr % HIVE_ISP_DDR_WORD_BYTES) != 0) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-				    "DDR address pointer is not properly aligned for DMA transfer\n");
-		hmm_free(code_addr);
-		code_addr = mmgr_NULL;
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-	spctrl_cofig_info[sp_id].sp_entry = spctrl_cfg->sp_entry;
-	spctrl_cofig_info[sp_id].code_addr = code_addr;
-	spctrl_cofig_info[sp_id].program_name = spctrl_cfg->program_name;
-
-	/* now we program the base address into the icache and
-	 * invalidate the cache.
-	 */
-	sp_ctrl_store(sp_id, SP_ICACHE_ADDR_REG, (hrt_data)spctrl_cofig_info[sp_id].code_addr);
-	sp_ctrl_setbit(sp_id, SP_ICACHE_INV_REG, SP_ICACHE_INV_BIT);
-	spctrl_loaded[sp_id] = true;
-	return IA_CSS_SUCCESS;
-}
-
-#ifdef ISP2401
-/* reload pre-loaded FW */
-void sh_css_spctrl_reload_fw(sp_ID_t sp_id)
-{
-	/* now we program the base address into the icache and
-	* invalidate the cache.
-	*/
-	sp_ctrl_store(sp_id, SP_ICACHE_ADDR_REG, (hrt_data)spctrl_cofig_info[sp_id].code_addr);
-	sp_ctrl_setbit(sp_id, SP_ICACHE_INV_REG, SP_ICACHE_INV_BIT);
-	spctrl_loaded[sp_id] = true;
-}
-#endif
-
-hrt_vaddress get_sp_code_addr(sp_ID_t  sp_id)
-{
-	return spctrl_cofig_info[sp_id].code_addr;
-}
-
-enum ia_css_err ia_css_spctrl_unload_fw(sp_ID_t sp_id)
-{
-	if ((sp_id >= N_SP_ID) || ((sp_id < N_SP_ID) && (!spctrl_loaded[sp_id])))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	/*  freeup the resource */
-	if (spctrl_cofig_info[sp_id].code_addr)
-		hmm_free(spctrl_cofig_info[sp_id].code_addr);
-	spctrl_loaded[sp_id] = false;
-	return IA_CSS_SUCCESS;
-}
-
-/* Initialize dmem_cfg in SP dmem  and  start SP program*/
-enum ia_css_err ia_css_spctrl_start(sp_ID_t sp_id)
-{
-	if ((sp_id >= N_SP_ID) || ((sp_id < N_SP_ID) && (!spctrl_loaded[sp_id])))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	/* Set descr in the SP to initialize the SP DMEM */
-	/*
-	 * The FW stores user-space pointers to the FW, the ISP pointer
-	 * is only available here
-	 *
-	 */
-	assert(sizeof(unsigned int) <= sizeof(hrt_data));
-
-	sp_dmem_store(sp_id,
-		spctrl_cofig_info[sp_id].spctrl_config_dmem_addr,
-		&spctrl_cofig_info[sp_id].dmem_config,
-		sizeof(spctrl_cofig_info[sp_id].dmem_config));
-	/* set the start address */
-	sp_ctrl_store(sp_id, SP_START_ADDR_REG, (hrt_data)spctrl_cofig_info[sp_id].sp_entry);
-	sp_ctrl_setbit(sp_id, SP_SC_REG, SP_RUN_BIT);
-	sp_ctrl_setbit(sp_id, SP_SC_REG, SP_START_BIT);
-	return IA_CSS_SUCCESS;
-}
-
-/* Query the state of SP1 */
-ia_css_spctrl_sp_sw_state ia_css_spctrl_get_state(sp_ID_t sp_id)
-{
-	ia_css_spctrl_sp_sw_state state = 0;
-	unsigned int HIVE_ADDR_sp_sw_state;
-	if (sp_id >= N_SP_ID)
-		return IA_CSS_SP_SW_TERMINATED;
-
-	HIVE_ADDR_sp_sw_state = spctrl_cofig_info[sp_id].spctrl_state_dmem_addr;
-	(void)HIVE_ADDR_sp_sw_state; /* Suppres warnings in CRUN */
-	if (sp_id == SP0_ID)
-		state = sp_dmem_load_uint32(sp_id, (unsigned)sp_address_of(sp_sw_state));
-	return state;
-}
-
-int ia_css_spctrl_is_idle(sp_ID_t sp_id)
-{
-	int state = 0;
-	assert (sp_id < N_SP_ID);
-
-	state = sp_ctrl_getbit(sp_id, SP_SC_REG, SP_IDLE_BIT);
-	return state;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/tagger/interface/ia_css_tagger_common.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/tagger/interface/ia_css_tagger_common.h
deleted file mode 100644
index d0d74957358b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/tagger/interface/ia_css_tagger_common.h
+++ /dev/null
@@ -1,59 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#ifndef __IA_CSS_TAGGER_COMMON_H__
-#define __IA_CSS_TAGGER_COMMON_H__
-
-#include <system_local.h>
-#include <type_support.h>
-
-/**
- * @brief The tagger's circular buffer.
- *
- * Should be one less than NUM_CONTINUOUS_FRAMES in sh_css_internal.h
- */
-#if defined(HAS_SP_2400)
-#define MAX_CB_ELEMS_FOR_TAGGER 14
-#else
-#define MAX_CB_ELEMS_FOR_TAGGER 9
-#endif
-
-/**
- * @brief Data structure for the tagger buffer element.
- */
-typedef struct {
-	uint32_t frame;	/* the frame value stored in the element */
-	uint32_t param;	/* the param value stored in the element */
-	uint8_t mark;	/* the mark on the element */
-	uint8_t lock;	/* the lock on the element */
-	uint8_t exp_id; /* exp_id of frame, for debugging only */
-} ia_css_tagger_buf_sp_elem_t;
-
-#endif /* __IA_CSS_TAGGER_COMMON_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/timer/src/timer.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/timer/src/timer.c
deleted file mode 100644
index b7dd18492a91..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/timer/src/timer.c
+++ /dev/null
@@ -1,48 +0,0 @@
-#ifndef ISP2401
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#else
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#endif
-
-#include <type_support.h>		/* for uint32_t */
-#include "ia_css_timer.h" /*struct ia_css_clock_tick */
-#include "sh_css_legacy.h" /* IA_CSS_PIPE_ID_NUM*/
-#include "gp_timer.h" /*gp_timer_read()*/
-#include "assert_support.h"
-
-enum ia_css_err
-ia_css_timer_get_current_tick(
-	struct ia_css_clock_tick *curr_ts) {
-
-	assert(curr_ts !=  NULL);
-	if (curr_ts == NULL) {
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	curr_ts->ticks = (clock_value_t)gp_timer_read(GP_TIMER_SEL);
-	return IA_CSS_SUCCESS;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css.c
deleted file mode 100644
index c771e4b910f3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css.c
+++ /dev/null
@@ -1,11094 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/*! \file */
-#include <linux/mm.h>
-#include <linux/slab.h>
-#include <linux/vmalloc.h>
-
-#include "ia_css.h"
-#include "sh_css_hrt.h"		/* only for file 2 MIPI */
-#include "ia_css_buffer.h"
-#include "ia_css_binary.h"
-#include "sh_css_internal.h"
-#include "sh_css_mipi.h"
-#include "sh_css_sp.h"		/* sh_css_sp_group */
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#include "ia_css_isys.h"
-#endif
-#include "ia_css_frame.h"
-#include "sh_css_defs.h"
-#include "sh_css_firmware.h"
-#include "sh_css_params.h"
-#include "sh_css_params_internal.h"
-#include "sh_css_param_shading.h"
-#include "ia_css_refcount.h"
-#include "ia_css_rmgr.h"
-#include "ia_css_debug.h"
-#include "ia_css_debug_pipe.h"
-#include "ia_css_device_access.h"
-#include "device_access.h"
-#include "sh_css_legacy.h"
-#include "ia_css_pipeline.h"
-#include "ia_css_stream.h"
-#include "sh_css_stream_format.h"
-#include "ia_css_pipe.h"
-#include "ia_css_util.h"
-#include "ia_css_pipe_util.h"
-#include "ia_css_pipe_binarydesc.h"
-#include "ia_css_pipe_stagedesc.h"
-#ifdef USE_INPUT_SYSTEM_VERSION_2
-#include "ia_css_isys.h"
-#endif
-
-#include "memory_access.h"
-#include "tag.h"
-#include "assert_support.h"
-#include "math_support.h"
-#include "sw_event_global.h"			/* Event IDs.*/
-#if !defined(HAS_NO_INPUT_FORMATTER)
-#include "ia_css_ifmtr.h"
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#include "input_system.h"
-#endif
-#include "mmu_device.h"		/* mmu_set_page_table_base_index(), ... */
-#include "ia_css_mmu_private.h" /* sh_css_mmu_set_page_table_base_index() */
-#include "gdc_device.h"		/* HRT_GDC_N */
-#include "dma.h"		/* dma_set_max_burst_size() */
-#include "irq.h"			/* virq */
-#include "sp.h"				/* cnd_sp_irq_enable() */
-#include "isp.h"			/* cnd_isp_irq_enable, ISP_VEC_NELEMS */
-#include "gp_device.h"		/* gp_device_reg_store() */
-#define __INLINE_GPIO__
-#include "gpio.h"
-#include "timed_ctrl.h"
-#include "platform_support.h" /* hrt_sleep(), inline */
-#include "ia_css_inputfifo.h"
-#define WITH_PC_MONITORING  0
-
-#define SH_CSS_VIDEO_BUFFER_ALIGNMENT 0
-
-#if WITH_PC_MONITORING
-#define MULTIPLE_SAMPLES 1
-#define NOF_SAMPLES      60
-#include "linux/kthread.h"
-#include "linux/sched.h"
-#include "linux/delay.h"
-#include "sh_css_metrics.h"
-static int thread_alive;
-#endif /* WITH_PC_MONITORING */
-
-#include "ia_css_spctrl.h"
-#include "ia_css_version_data.h"
-#include "sh_css_struct.h"
-#include "ia_css_bufq.h"
-#include "ia_css_timer.h" /* clock_value_t */
-
-#include "isp/modes/interface/input_buf.isp.h"
-
-/* Name of the sp program: should not be built-in */
-#define SP_PROG_NAME "sp"
-/* Size of Refcount List */
-#define REFCOUNT_SIZE 1000
-
-/* for JPEG, we don't know the length of the image upfront,
- * but since we support sensor upto 16MP, we take this as
- * upper limit.
- */
-#define JPEG_BYTES (16 * 1024 * 1024)
-
-#define STATS_ENABLED(stage) (stage && stage->binary && stage->binary->info && \
-        (stage->binary->info->sp.enable.s3a || stage->binary->info->sp.enable.dis))
-
-struct sh_css my_css;
-
-int (*sh_css_printf) (const char *fmt, va_list args) = NULL;
-
-/* modes of work: stream_create and stream_destroy will update the save/restore data
-   only when in working mode, not suspend/resume
-*/
-enum ia_sh_css_modes {
-	sh_css_mode_none = 0,
-	sh_css_mode_working,
-	sh_css_mode_suspend,
-	sh_css_mode_resume
-};
-
-/* a stream seed, to save and restore the stream data.
-   the stream seed contains all the data required to "grow" the seed again after it was closed.
-*/
-struct sh_css_stream_seed {
-	struct ia_css_stream		**orig_stream;                /* pointer to restore the original handle */
-	struct ia_css_stream		*stream;                      /* handle, used as ID too.*/
-	struct ia_css_stream_config	stream_config;				/* stream config struct */
-	int				num_pipes;
-	struct ia_css_pipe		*pipes[IA_CSS_PIPE_ID_NUM];			/* pipe handles */
-	struct ia_css_pipe		**orig_pipes[IA_CSS_PIPE_ID_NUM];	/* pointer to restore original handle */
-	struct ia_css_pipe_config	pipe_config[IA_CSS_PIPE_ID_NUM];	/* pipe config structs */
-};
-
-#define MAX_ACTIVE_STREAMS	5
-/* A global struct for save/restore to hold all the data that should sustain power-down:
-   MMU base, IRQ type, env for routines, binary loaded FW and the stream seeds.
-*/
-struct sh_css_save {
-	enum ia_sh_css_modes		mode;
-	uint32_t		       mmu_base;				/* the last mmu_base */
-	enum ia_css_irq_type           irq_type;
-	struct sh_css_stream_seed      stream_seeds[MAX_ACTIVE_STREAMS];
-	struct ia_css_fw	       *loaded_fw;				/* fw struct previously loaded */
-	struct ia_css_env	       driver_env;				/* driver-supplied env copy */
-};
-
-static bool my_css_save_initialized;	/* if my_css_save was initialized */
-static struct sh_css_save my_css_save;
-
-/* pqiao NOTICE: this is for css internal buffer recycling when stopping pipeline,
-   this array is temporary and will be replaced by resource manager*/
-/* Taking the biggest Size for number of Elements */
-#define MAX_HMM_BUFFER_NUM	\
-	(SH_CSS_MAX_NUM_QUEUES * (IA_CSS_NUM_ELEMS_SP2HOST_BUFFER_QUEUE + 2))
-
-struct sh_css_hmm_buffer_record {
-	bool in_use;
-	enum ia_css_buffer_type type;
-	struct ia_css_rmgr_vbuf_handle *h_vbuf;
-	hrt_address kernel_ptr;
-};
-
-static struct sh_css_hmm_buffer_record hmm_buffer_record[MAX_HMM_BUFFER_NUM];
-
-#define GPIO_FLASH_PIN_MASK (1 << HIVE_GPIO_STROBE_TRIGGER_PIN)
-
-static bool fw_explicitly_loaded = false;
-
-/*
- * Local prototypes
- */
-
-static enum ia_css_err
-allocate_delay_frames(struct ia_css_pipe *pipe);
-
-static enum ia_css_err
-sh_css_pipe_start(struct ia_css_stream *stream);
-
-#ifdef ISP2401
-/*
- * @brief Stop all "ia_css_pipe" instances in the target
- * "ia_css_stream" instance.
- *
- * @param[in] stream	Point to the target "ia_css_stream" instance.
- *
- * @return
- * - IA_CSS_SUCCESS, if the "stop" requests have been successfully sent out.
- * - CSS error code, otherwise.
- *
- *
- * NOTE
- * This API sends the "stop" requests to the "ia_css_pipe"
- * instances in the same "ia_css_stream" instance. It will
- * return without waiting for all "ia_css_pipe" instatnces
- * being stopped.
- */
-static enum ia_css_err
-sh_css_pipes_stop(struct ia_css_stream *stream);
-
-/*
- * @brief Check if all "ia_css_pipe" instances in the target
- * "ia_css_stream" instance have stopped.
- *
- * @param[in] stream	Point to the target "ia_css_stream" instance.
- *
- * @return
- * - true, if all "ia_css_pipe" instances in the target "ia_css_stream"
- *   instance have ben stopped.
- * - false, otherwise.
- */
-static bool
-sh_css_pipes_have_stopped(struct ia_css_stream *stream);
-
-static enum ia_css_err
-ia_css_pipe_check_format(struct ia_css_pipe *pipe, enum ia_css_frame_format format);
-
-static enum ia_css_err
-check_pipe_resolutions(const struct ia_css_pipe *pipe);
-
-#endif
-
-static enum ia_css_err
-ia_css_pipe_load_extension(struct ia_css_pipe *pipe,
-		struct ia_css_fw_info *firmware);
-
-static void
-ia_css_pipe_unload_extension(struct ia_css_pipe *pipe,
-		struct ia_css_fw_info *firmware);
-static void
-ia_css_reset_defaults(struct sh_css* css);
-
-static void
-sh_css_init_host_sp_control_vars(void);
-
-static enum ia_css_err set_num_primary_stages(unsigned int *num, enum ia_css_pipe_version version);
-
-static bool
-need_capture_pp(const struct ia_css_pipe *pipe);
-
-static bool
-need_yuv_scaler_stage(const struct ia_css_pipe *pipe);
-
-static enum ia_css_err ia_css_pipe_create_cas_scaler_desc_single_output(
-	struct ia_css_frame_info *cas_scaler_in_info,
-	struct ia_css_frame_info *cas_scaler_out_info,
-	struct ia_css_frame_info *cas_scaler_vf_info,
-	struct ia_css_cas_binary_descr *descr);
-
-static void ia_css_pipe_destroy_cas_scaler_desc(struct ia_css_cas_binary_descr *descr);
-
-static bool
-need_downscaling(const struct ia_css_resolution in_res,
-		const struct ia_css_resolution out_res);
-
-static bool need_capt_ldc(const struct ia_css_pipe *pipe);
-
-static enum ia_css_err
-sh_css_pipe_load_binaries(struct ia_css_pipe *pipe);
-
-static
-enum ia_css_err sh_css_pipe_get_viewfinder_frame_info(
-	struct ia_css_pipe *pipe,
-	struct ia_css_frame_info *info,
-	unsigned int idx);
-
-static enum ia_css_err
-sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe,
-				  struct ia_css_frame_info *info,
-				  unsigned int idx);
-
-static enum ia_css_err
-capture_start(struct ia_css_pipe *pipe);
-
-static enum ia_css_err
-video_start(struct ia_css_pipe *pipe);
-
-static enum ia_css_err
-preview_start(struct ia_css_pipe *pipe);
-
-static enum ia_css_err
-yuvpp_start(struct ia_css_pipe *pipe);
-
-static bool copy_on_sp(struct ia_css_pipe *pipe);
-
-static enum ia_css_err
-init_vf_frameinfo_defaults(struct ia_css_pipe *pipe,
-	struct ia_css_frame *vf_frame, unsigned int idx);
-
-static enum ia_css_err
-init_in_frameinfo_memory_defaults(struct ia_css_pipe *pipe,
-	struct ia_css_frame *frame, enum ia_css_frame_format format);
-
-static enum ia_css_err
-init_out_frameinfo_defaults(struct ia_css_pipe *pipe,
-	struct ia_css_frame *out_frame, unsigned int idx);
-
-static enum ia_css_err
-sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline,
-			      const void *acc_fw);
-
-static enum ia_css_err
-alloc_continuous_frames(
-	struct ia_css_pipe *pipe, bool init_time);
-
-static void
-pipe_global_init(void);
-
-static enum ia_css_err
-pipe_generate_pipe_num(const struct ia_css_pipe *pipe, unsigned int *pipe_number);
-
-static void
-pipe_release_pipe_num(unsigned int pipe_num);
-
-static enum ia_css_err
-create_host_pipeline_structure(struct ia_css_stream *stream);
-
-static enum ia_css_err
-create_host_pipeline(struct ia_css_stream *stream);
-
-static enum ia_css_err
-create_host_preview_pipeline(struct ia_css_pipe *pipe);
-
-static enum ia_css_err
-create_host_video_pipeline(struct ia_css_pipe *pipe);
-
-static enum ia_css_err
-create_host_copy_pipeline(struct ia_css_pipe *pipe,
-    unsigned max_input_width,
-    struct ia_css_frame *out_frame);
-
-static enum ia_css_err
-create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe);
-
-static enum ia_css_err
-create_host_capture_pipeline(struct ia_css_pipe *pipe);
-
-static enum ia_css_err
-create_host_yuvpp_pipeline(struct ia_css_pipe *pipe);
-
-static enum ia_css_err
-create_host_acc_pipeline(struct ia_css_pipe *pipe);
-
-static unsigned int
-sh_css_get_sw_interrupt_value(unsigned int irq);
-
-static struct ia_css_binary *ia_css_pipe_get_shading_correction_binary(const struct ia_css_pipe *pipe);
-
-static struct ia_css_binary *
-ia_css_pipe_get_s3a_binary(const struct ia_css_pipe *pipe);
-
-static struct ia_css_binary *
-ia_css_pipe_get_sdis_binary(const struct ia_css_pipe *pipe);
-
-static void
-sh_css_hmm_buffer_record_init(void);
-
-static void
-sh_css_hmm_buffer_record_uninit(void);
-
-static void
-sh_css_hmm_buffer_record_reset(struct sh_css_hmm_buffer_record *buffer_record);
-
-static struct sh_css_hmm_buffer_record
-*sh_css_hmm_buffer_record_acquire(struct ia_css_rmgr_vbuf_handle *h_vbuf,
-			enum ia_css_buffer_type type,
-			hrt_address kernel_ptr);
-
-static struct sh_css_hmm_buffer_record
-*sh_css_hmm_buffer_record_validate(hrt_vaddress ddr_buffer_addr,
-		enum ia_css_buffer_type type);
-
-void
-ia_css_get_acc_configs(
-	struct ia_css_pipe *pipe,
-	struct ia_css_isp_config *config);
-
-
-#if CONFIG_ON_FRAME_ENQUEUE()
-static enum ia_css_err set_config_on_frame_enqueue(struct ia_css_frame_info *info, struct frame_data_wrapper *frame);
-#endif
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-static unsigned int get_crop_lines_for_bayer_order(const struct ia_css_stream_config *config);
-static unsigned int get_crop_columns_for_bayer_order(const struct ia_css_stream_config *config);
-static void get_pipe_extra_pixel(struct ia_css_pipe *pipe,
-		unsigned int *extra_row, unsigned int *extra_column);
-#endif
-
-#ifdef ISP2401
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-static enum ia_css_err
-aspect_ratio_crop_init(struct ia_css_stream *curr_stream,
-		struct ia_css_pipe *pipes[],
-		bool *do_crop_status);
-
-static bool
-aspect_ratio_crop_check(bool enabled, struct ia_css_pipe *curr_pipe);
-
-static enum ia_css_err
-aspect_ratio_crop(struct ia_css_pipe *curr_pipe,
-		struct ia_css_resolution *effective_res);
-#endif
-
-#endif
-static void
-sh_css_pipe_free_shading_table(struct ia_css_pipe *pipe)
-{
-	assert(pipe != NULL);
-	if (pipe == NULL) {
-		IA_CSS_ERROR("NULL input parameter");
-		return;
-	}
-
-	if (pipe->shading_table)
-		ia_css_shading_table_free(pipe->shading_table);
-	pipe->shading_table = NULL;
-}
-
-static enum ia_css_frame_format yuv420_copy_formats[] = {
-	IA_CSS_FRAME_FORMAT_NV12,
-	IA_CSS_FRAME_FORMAT_NV21,
-	IA_CSS_FRAME_FORMAT_YV12,
-	IA_CSS_FRAME_FORMAT_YUV420,
-	IA_CSS_FRAME_FORMAT_YUV420_16,
-	IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_8,
-	IA_CSS_FRAME_FORMAT_CSI_MIPI_LEGACY_YUV420_8
-};
-
-static enum ia_css_frame_format yuv422_copy_formats[] = {
-	IA_CSS_FRAME_FORMAT_NV12,
-	IA_CSS_FRAME_FORMAT_NV16,
-	IA_CSS_FRAME_FORMAT_NV21,
-	IA_CSS_FRAME_FORMAT_NV61,
-	IA_CSS_FRAME_FORMAT_YV12,
-	IA_CSS_FRAME_FORMAT_YV16,
-	IA_CSS_FRAME_FORMAT_YUV420,
-	IA_CSS_FRAME_FORMAT_YUV420_16,
-	IA_CSS_FRAME_FORMAT_YUV422,
-	IA_CSS_FRAME_FORMAT_YUV422_16,
-	IA_CSS_FRAME_FORMAT_UYVY,
-	IA_CSS_FRAME_FORMAT_YUYV
-};
-
-/* Verify whether the selected output format is can be produced
- * by the copy binary given the stream format.
- * */
-static enum ia_css_err
-verify_copy_out_frame_format(struct ia_css_pipe *pipe)
-{
-	enum ia_css_frame_format out_fmt = pipe->output_info[0].format;
-	unsigned int i, found = 0;
-
-	assert(pipe != NULL);
-	assert(pipe->stream != NULL);
-
-	switch (pipe->stream->config.input_config.format) {
-	case ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY:
-	case ATOMISP_INPUT_FORMAT_YUV420_8:
-		for (i=0; i<ARRAY_SIZE(yuv420_copy_formats) && !found; i++)
-			found = (out_fmt == yuv420_copy_formats[i]);
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_10:
-	case ATOMISP_INPUT_FORMAT_YUV420_16:
-		found = (out_fmt == IA_CSS_FRAME_FORMAT_YUV420_16);
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV422_8:
-		for (i=0; i<ARRAY_SIZE(yuv422_copy_formats) && !found; i++)
-			found = (out_fmt == yuv422_copy_formats[i]);
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV422_10:
-	case ATOMISP_INPUT_FORMAT_YUV422_16:
-		found = (out_fmt == IA_CSS_FRAME_FORMAT_YUV422_16 ||
-			 out_fmt == IA_CSS_FRAME_FORMAT_YUV420_16);
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_444:
-	case ATOMISP_INPUT_FORMAT_RGB_555:
-	case ATOMISP_INPUT_FORMAT_RGB_565:
-		found = (out_fmt == IA_CSS_FRAME_FORMAT_RGBA888 ||
-			 out_fmt == IA_CSS_FRAME_FORMAT_RGB565);
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_666:
-	case ATOMISP_INPUT_FORMAT_RGB_888:
-		found = (out_fmt == IA_CSS_FRAME_FORMAT_RGBA888 ||
-			 out_fmt == IA_CSS_FRAME_FORMAT_YUV420);
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_6:
-	case ATOMISP_INPUT_FORMAT_RAW_7:
-	case ATOMISP_INPUT_FORMAT_RAW_8:
-	case ATOMISP_INPUT_FORMAT_RAW_10:
-	case ATOMISP_INPUT_FORMAT_RAW_12:
-	case ATOMISP_INPUT_FORMAT_RAW_14:
-	case ATOMISP_INPUT_FORMAT_RAW_16:
-		found = (out_fmt == IA_CSS_FRAME_FORMAT_RAW) ||
-			(out_fmt == IA_CSS_FRAME_FORMAT_RAW_PACKED);
-		break;
-	case ATOMISP_INPUT_FORMAT_BINARY_8:
-		found = (out_fmt == IA_CSS_FRAME_FORMAT_BINARY_8);
-		break;
-	default:
-		break;
-	}
-	if (!found)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	return IA_CSS_SUCCESS;
-}
-
-unsigned int
-ia_css_stream_input_format_bits_per_pixel(struct ia_css_stream *stream)
-{
-	int bpp = 0;
-
-	if (stream != NULL)
-		bpp = ia_css_util_input_format_bpp(stream->config.input_config.format,
-						stream->config.pixels_per_clock == 2);
-
-	return bpp;
-}
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-static enum ia_css_err
-sh_css_config_input_network(struct ia_css_stream *stream)
-{
-	unsigned int fmt_type;
-	struct ia_css_pipe *pipe = stream->last_pipe;
-	struct ia_css_binary *binary = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	assert(stream != NULL);
-	assert(pipe != NULL);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"sh_css_config_input_network() enter:\n");
-
-	if (pipe->pipeline.stages)
-		binary = pipe->pipeline.stages->binary;
-
-	err = ia_css_isys_convert_stream_format_to_mipi_format(
-				stream->config.input_config.format,
-				stream->csi_rx_config.comp,
-				&fmt_type);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	sh_css_sp_program_input_circuit(fmt_type,
-					stream->config.channel_id,
-					stream->config.mode);
-
-	if ((binary && (binary->online || stream->config.continuous)) ||
-			pipe->config.mode == IA_CSS_PIPE_MODE_COPY) {
-		err = ia_css_ifmtr_configure(&stream->config,
-			binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-
-	if (stream->config.mode == IA_CSS_INPUT_MODE_TPG ||
-	    stream->config.mode == IA_CSS_INPUT_MODE_PRBS) {
-		unsigned int hblank_cycles = 100,
-			     vblank_lines = 6,
-			     width,
-			     height,
-			     vblank_cycles;
-		width  = (stream->config.input_config.input_res.width) / (1 + (stream->config.pixels_per_clock == 2));
-		height = stream->config.input_config.input_res.height;
-		vblank_cycles = vblank_lines * (width + hblank_cycles);
-		sh_css_sp_configure_sync_gen(width, height, hblank_cycles,
-					     vblank_cycles);
-#if defined(IS_ISP_2400_SYSTEM)
-		if (pipe->stream->config.mode == IA_CSS_INPUT_MODE_TPG) {
-			/* TODO: move define to proper file in tools */
-			#define GP_ISEL_TPG_MODE 0x90058
-			ia_css_device_store_uint32(GP_ISEL_TPG_MODE, 0);
-		}
-#endif
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"sh_css_config_input_network() leave:\n");
-	return IA_CSS_SUCCESS;
-}
-#elif !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
-static unsigned int csi2_protocol_calculate_max_subpixels_per_line(
-		enum atomisp_input_format	format,
-		unsigned int			pixels_per_line)
-{
-	unsigned int rval;
-
-	switch (format) {
-	case ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY:
-		/*
-		 * The frame format layout is shown below.
-		 *
-		 *		Line	0:	UYY0 UYY0 ... UYY0
-		 *		Line	1:	VYY0 VYY0 ... VYY0
-		 *		Line	2:	UYY0 UYY0 ... UYY0
-		 *		Line	3:	VYY0 VYY0 ... VYY0
-		 *		...
-		 *		Line (n-2):	UYY0 UYY0 ... UYY0
-		 *		Line (n-1):	VYY0 VYY0 ... VYY0
-		 *
-		 *	In this frame format, the even-line is
-		 *	as wide as the odd-line.
-		 *	The 0 is introduced by the input system
-		 *	(mipi backend).
-		 */
-		rval = pixels_per_line * 2;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_8:
-	case ATOMISP_INPUT_FORMAT_YUV420_10:
-	case ATOMISP_INPUT_FORMAT_YUV420_16:
-		/*
-		 * The frame format layout is shown below.
-		 *
-		 *		Line	0:	YYYY YYYY ... YYYY
-		 *		Line	1:	UYVY UYVY ... UYVY UYVY
-		 *		Line	2:	YYYY YYYY ... YYYY
-		 *		Line	3:	UYVY UYVY ... UYVY UYVY
-		 *		...
-		 *		Line (n-2):	YYYY YYYY ... YYYY
-		 *		Line (n-1):	UYVY UYVY ... UYVY UYVY
-		 *
-		 * In this frame format, the odd-line is twice
-		 * wider than the even-line.
-		 */
-		rval = pixels_per_line * 2;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV422_8:
-	case ATOMISP_INPUT_FORMAT_YUV422_10:
-	case ATOMISP_INPUT_FORMAT_YUV422_16:
-		/*
-		 * The frame format layout is shown below.
-		 *
-		 *		Line	0:	UYVY UYVY ... UYVY
-		 *		Line	1:	UYVY UYVY ... UYVY
-		 *		Line	2:	UYVY UYVY ... UYVY
-		 *		Line	3:	UYVY UYVY ... UYVY
-		 *		...
-		 *		Line (n-2):	UYVY UYVY ... UYVY
-		 *		Line (n-1):	UYVY UYVY ... UYVY
-		 *
-		 * In this frame format, the even-line is
-		 * as wide as the odd-line.
-		 */
-		rval = pixels_per_line * 2;
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_444:
-	case ATOMISP_INPUT_FORMAT_RGB_555:
-	case ATOMISP_INPUT_FORMAT_RGB_565:
-	case ATOMISP_INPUT_FORMAT_RGB_666:
-	case ATOMISP_INPUT_FORMAT_RGB_888:
-		/*
-		 * The frame format layout is shown below.
-		 *
-		 *		Line	0:	ABGR ABGR ... ABGR
-		 *		Line	1:	ABGR ABGR ... ABGR
-		 *		Line	2:	ABGR ABGR ... ABGR
-		 *		Line	3:	ABGR ABGR ... ABGR
-		 *		...
-		 *		Line (n-2):	ABGR ABGR ... ABGR
-		 *		Line (n-1):	ABGR ABGR ... ABGR
-		 *
-		 * In this frame format, the even-line is
-		 * as wide as the odd-line.
-		 */
-		rval = pixels_per_line * 4;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_6:
-	case ATOMISP_INPUT_FORMAT_RAW_7:
-	case ATOMISP_INPUT_FORMAT_RAW_8:
-	case ATOMISP_INPUT_FORMAT_RAW_10:
-	case ATOMISP_INPUT_FORMAT_RAW_12:
-	case ATOMISP_INPUT_FORMAT_RAW_14:
-	case ATOMISP_INPUT_FORMAT_RAW_16:
-	case ATOMISP_INPUT_FORMAT_BINARY_8:
-	case ATOMISP_INPUT_FORMAT_USER_DEF1:
-	case ATOMISP_INPUT_FORMAT_USER_DEF2:
-	case ATOMISP_INPUT_FORMAT_USER_DEF3:
-	case ATOMISP_INPUT_FORMAT_USER_DEF4:
-	case ATOMISP_INPUT_FORMAT_USER_DEF5:
-	case ATOMISP_INPUT_FORMAT_USER_DEF6:
-	case ATOMISP_INPUT_FORMAT_USER_DEF7:
-	case ATOMISP_INPUT_FORMAT_USER_DEF8:
-		/*
-		 * The frame format layout is shown below.
-		 *
-		 *		Line	0:	Pixel Pixel ... Pixel
-		 *		Line	1:	Pixel Pixel ... Pixel
-		 *		Line	2:	Pixel Pixel ... Pixel
-		 *		Line	3:	Pixel Pixel ... Pixel
-		 *		...
-		 *		Line (n-2):	Pixel Pixel ... Pixel
-		 *		Line (n-1):	Pixel Pixel ... Pixel
-		 *
-		 * In this frame format, the even-line is
-		 * as wide as the odd-line.
-		 */
-		rval = pixels_per_line;
-		break;
-	default:
-		rval = 0;
-		break;
-	}
-
-	return rval;
-}
-
-static bool sh_css_translate_stream_cfg_to_input_system_input_port_id(
-		struct ia_css_stream_config *stream_cfg,
-		ia_css_isys_descr_t	*isys_stream_descr)
-{
-	bool rc;
-
-	rc = true;
-	switch (stream_cfg->mode) {
-	case IA_CSS_INPUT_MODE_TPG:
-
-		if (stream_cfg->source.tpg.id == IA_CSS_TPG_ID0) {
-			isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT0_ID;
-		} else if (stream_cfg->source.tpg.id == IA_CSS_TPG_ID1) {
-			isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT1_ID;
-		} else if (stream_cfg->source.tpg.id == IA_CSS_TPG_ID2) {
-			isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT2_ID;
-		}
-
-		break;
-	case IA_CSS_INPUT_MODE_PRBS:
-
-		if (stream_cfg->source.prbs.id == IA_CSS_PRBS_ID0) {
-			isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT0_ID;
-		} else if (stream_cfg->source.prbs.id == IA_CSS_PRBS_ID1) {
-			isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT1_ID;
-		} else if (stream_cfg->source.prbs.id == IA_CSS_PRBS_ID2) {
-			isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT2_ID;
-		}
-
-		break;
-	case IA_CSS_INPUT_MODE_BUFFERED_SENSOR:
-
-		if (stream_cfg->source.port.port == MIPI_PORT0_ID) {
-			isys_stream_descr->input_port_id = INPUT_SYSTEM_CSI_PORT0_ID;
-		} else if (stream_cfg->source.port.port == MIPI_PORT1_ID) {
-			isys_stream_descr->input_port_id = INPUT_SYSTEM_CSI_PORT1_ID;
-		} else if (stream_cfg->source.port.port == MIPI_PORT2_ID) {
-			isys_stream_descr->input_port_id = INPUT_SYSTEM_CSI_PORT2_ID;
-		}
-
-		break;
-	default:
-		rc = false;
-		break;
-	}
-
-	return rc;
-}
-
-static bool sh_css_translate_stream_cfg_to_input_system_input_port_type(
-		struct ia_css_stream_config *stream_cfg,
-		ia_css_isys_descr_t	*isys_stream_descr)
-{
-	bool rc;
-
-	rc = true;
-	switch (stream_cfg->mode) {
-	case IA_CSS_INPUT_MODE_TPG:
-
-		isys_stream_descr->mode = INPUT_SYSTEM_SOURCE_TYPE_TPG;
-
-		break;
-	case IA_CSS_INPUT_MODE_PRBS:
-
-		isys_stream_descr->mode = INPUT_SYSTEM_SOURCE_TYPE_PRBS;
-
-		break;
-	case IA_CSS_INPUT_MODE_SENSOR:
-	case IA_CSS_INPUT_MODE_BUFFERED_SENSOR:
-
-		isys_stream_descr->mode = INPUT_SYSTEM_SOURCE_TYPE_SENSOR;
-		break;
-
-	default:
-		rc = false;
-		break;
-	}
-
-	return rc;
-}
-
-static bool sh_css_translate_stream_cfg_to_input_system_input_port_attr(
-		struct ia_css_stream_config *stream_cfg,
-		ia_css_isys_descr_t	*isys_stream_descr,
-		int isys_stream_idx)
-{
-	bool rc;
-
-	rc = true;
-	switch (stream_cfg->mode) {
-	case IA_CSS_INPUT_MODE_TPG:
-		if (stream_cfg->source.tpg.mode == IA_CSS_TPG_MODE_RAMP) {
-			isys_stream_descr->tpg_port_attr.mode = PIXELGEN_TPG_MODE_RAMP;
-		} else if (stream_cfg->source.tpg.mode == IA_CSS_TPG_MODE_CHECKERBOARD) {
-			isys_stream_descr->tpg_port_attr.mode = PIXELGEN_TPG_MODE_CHBO;
-		} else if (stream_cfg->source.tpg.mode == IA_CSS_TPG_MODE_MONO) {
-			isys_stream_descr->tpg_port_attr.mode = PIXELGEN_TPG_MODE_MONO;
-		} else {
-			rc = false;
-		}
-
-		/*
-		 * TODO
-		 * - Make "color_cfg" as part of "ia_css_tpg_config".
-		 */
-		isys_stream_descr->tpg_port_attr.color_cfg.R1 = 51;
-		isys_stream_descr->tpg_port_attr.color_cfg.G1 = 102;
-		isys_stream_descr->tpg_port_attr.color_cfg.B1 = 255;
-		isys_stream_descr->tpg_port_attr.color_cfg.R2 = 0;
-		isys_stream_descr->tpg_port_attr.color_cfg.G2 = 100;
-		isys_stream_descr->tpg_port_attr.color_cfg.B2 = 160;
-
-		isys_stream_descr->tpg_port_attr.mask_cfg.h_mask = stream_cfg->source.tpg.x_mask;
-		isys_stream_descr->tpg_port_attr.mask_cfg.v_mask = stream_cfg->source.tpg.y_mask;
-		isys_stream_descr->tpg_port_attr.mask_cfg.hv_mask = stream_cfg->source.tpg.xy_mask;
-
-		isys_stream_descr->tpg_port_attr.delta_cfg.h_delta = stream_cfg->source.tpg.x_delta;
-		isys_stream_descr->tpg_port_attr.delta_cfg.v_delta = stream_cfg->source.tpg.y_delta;
-
-		/*
-		 * TODO
-		 * - Make "sync_gen_cfg" as part of "ia_css_tpg_config".
-		 */
-		isys_stream_descr->tpg_port_attr.sync_gen_cfg.hblank_cycles = 100;
-		isys_stream_descr->tpg_port_attr.sync_gen_cfg.vblank_cycles = 100;
-		isys_stream_descr->tpg_port_attr.sync_gen_cfg.pixels_per_clock = stream_cfg->pixels_per_clock;
-		isys_stream_descr->tpg_port_attr.sync_gen_cfg.nr_of_frames = (uint32_t) ~(0x0);
-		isys_stream_descr->tpg_port_attr.sync_gen_cfg.pixels_per_line = stream_cfg->isys_config[IA_CSS_STREAM_DEFAULT_ISYS_STREAM_IDX].input_res.width;
-		isys_stream_descr->tpg_port_attr.sync_gen_cfg.lines_per_frame = stream_cfg->isys_config[IA_CSS_STREAM_DEFAULT_ISYS_STREAM_IDX].input_res.height;
-
-		break;
-	case IA_CSS_INPUT_MODE_PRBS:
-
-		isys_stream_descr->prbs_port_attr.seed0 = stream_cfg->source.prbs.seed;
-		isys_stream_descr->prbs_port_attr.seed1 = stream_cfg->source.prbs.seed1;
-
-		/*
-		 * TODO
-		 * - Make "sync_gen_cfg" as part of "ia_css_prbs_config".
-		 */
-		isys_stream_descr->prbs_port_attr.sync_gen_cfg.hblank_cycles = 100;
-		isys_stream_descr->prbs_port_attr.sync_gen_cfg.vblank_cycles = 100;
-		isys_stream_descr->prbs_port_attr.sync_gen_cfg.pixels_per_clock = stream_cfg->pixels_per_clock;
-		isys_stream_descr->prbs_port_attr.sync_gen_cfg.nr_of_frames = (uint32_t) ~(0x0);
-		isys_stream_descr->prbs_port_attr.sync_gen_cfg.pixels_per_line = stream_cfg->isys_config[IA_CSS_STREAM_DEFAULT_ISYS_STREAM_IDX].input_res.width;
-		isys_stream_descr->prbs_port_attr.sync_gen_cfg.lines_per_frame = stream_cfg->isys_config[IA_CSS_STREAM_DEFAULT_ISYS_STREAM_IDX].input_res.height;
-
-		break;
-	case IA_CSS_INPUT_MODE_BUFFERED_SENSOR:
-	{
-		enum ia_css_err err;
-		unsigned int fmt_type;
-
-		err = ia_css_isys_convert_stream_format_to_mipi_format(
-			stream_cfg->isys_config[isys_stream_idx].format,
-			MIPI_PREDICTOR_NONE,
-			&fmt_type);
-		if (err != IA_CSS_SUCCESS)
-			rc = false;
-
-		isys_stream_descr->csi_port_attr.active_lanes = stream_cfg->source.port.num_lanes;
-		isys_stream_descr->csi_port_attr.fmt_type = fmt_type;
-		isys_stream_descr->csi_port_attr.ch_id = stream_cfg->channel_id;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-		isys_stream_descr->online = stream_cfg->online;
-#endif
-		err |= ia_css_isys_convert_compressed_format(
-				&stream_cfg->source.port.compression,
-				isys_stream_descr);
-		if (err != IA_CSS_SUCCESS)
-			rc = false;
-
-		/* metadata */
-		isys_stream_descr->metadata.enable = false;
-		if (stream_cfg->metadata_config.resolution.height > 0) {
-			err = ia_css_isys_convert_stream_format_to_mipi_format(
-				stream_cfg->metadata_config.data_type,
-				MIPI_PREDICTOR_NONE,
-					&fmt_type);
-			if (err != IA_CSS_SUCCESS)
-				rc = false;
-			isys_stream_descr->metadata.fmt_type = fmt_type;
-			isys_stream_descr->metadata.bits_per_pixel =
-				ia_css_util_input_format_bpp(stream_cfg->metadata_config.data_type, true);
-			isys_stream_descr->metadata.pixels_per_line = stream_cfg->metadata_config.resolution.width;
-			isys_stream_descr->metadata.lines_per_frame = stream_cfg->metadata_config.resolution.height;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-			/* For new input system, number of str2mmio requests must be even.
-			 * So we round up number of metadata lines to be even. */
-			if (isys_stream_descr->metadata.lines_per_frame > 0)
-				isys_stream_descr->metadata.lines_per_frame +=
-					(isys_stream_descr->metadata.lines_per_frame & 1);
-#endif
-			isys_stream_descr->metadata.align_req_in_bytes =
-				ia_css_csi2_calculate_input_system_alignment(stream_cfg->metadata_config.data_type);
-			isys_stream_descr->metadata.enable = true;
-		}
-
-		break;
-	}
-	default:
-		rc = false;
-		break;
-	}
-
-	return rc;
-}
-
-static bool sh_css_translate_stream_cfg_to_input_system_input_port_resolution(
-		struct ia_css_stream_config *stream_cfg,
-		ia_css_isys_descr_t	*isys_stream_descr,
-		int isys_stream_idx)
-{
-	unsigned int bits_per_subpixel;
-	unsigned int max_subpixels_per_line;
-	unsigned int lines_per_frame;
-	unsigned int align_req_in_bytes;
-	enum atomisp_input_format fmt_type;
-
-	fmt_type = stream_cfg->isys_config[isys_stream_idx].format;
-	if ((stream_cfg->mode == IA_CSS_INPUT_MODE_SENSOR ||
-			stream_cfg->mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) &&
-		stream_cfg->source.port.compression.type != IA_CSS_CSI2_COMPRESSION_TYPE_NONE) {
-
-		if (stream_cfg->source.port.compression.uncompressed_bits_per_pixel ==
-			UNCOMPRESSED_BITS_PER_PIXEL_10) {
-				fmt_type = ATOMISP_INPUT_FORMAT_RAW_10;
-		}
-		else if (stream_cfg->source.port.compression.uncompressed_bits_per_pixel ==
-			UNCOMPRESSED_BITS_PER_PIXEL_12) {
-				fmt_type = ATOMISP_INPUT_FORMAT_RAW_12;
-		}
-		else
-			return false;
-	}
-
-	bits_per_subpixel =
-		sh_css_stream_format_2_bits_per_subpixel(fmt_type);
-	if (bits_per_subpixel == 0)
-		return false;
-
-	max_subpixels_per_line =
-		csi2_protocol_calculate_max_subpixels_per_line(fmt_type,
-			stream_cfg->isys_config[isys_stream_idx].input_res.width);
-	if (max_subpixels_per_line == 0)
-		return false;
-
-	lines_per_frame = stream_cfg->isys_config[isys_stream_idx].input_res.height;
-	if (lines_per_frame == 0)
-		return false;
-
-	align_req_in_bytes = ia_css_csi2_calculate_input_system_alignment(fmt_type);
-
-	/* HW needs subpixel info for their settings */
-	isys_stream_descr->input_port_resolution.bits_per_pixel = bits_per_subpixel;
-	isys_stream_descr->input_port_resolution.pixels_per_line = max_subpixels_per_line;
-	isys_stream_descr->input_port_resolution.lines_per_frame = lines_per_frame;
-	isys_stream_descr->input_port_resolution.align_req_in_bytes = align_req_in_bytes;
-
-	return true;
-}
-
-static bool sh_css_translate_stream_cfg_to_isys_stream_descr(
-		struct ia_css_stream_config *stream_cfg,
-		bool early_polling,
-		ia_css_isys_descr_t	*isys_stream_descr,
-		int isys_stream_idx)
-{
-	bool rc;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"sh_css_translate_stream_cfg_to_isys_stream_descr() enter:\n");
-	rc  = sh_css_translate_stream_cfg_to_input_system_input_port_id(stream_cfg, isys_stream_descr);
-	rc &= sh_css_translate_stream_cfg_to_input_system_input_port_type(stream_cfg, isys_stream_descr);
-	rc &= sh_css_translate_stream_cfg_to_input_system_input_port_attr(stream_cfg, isys_stream_descr, isys_stream_idx);
-	rc &= sh_css_translate_stream_cfg_to_input_system_input_port_resolution(stream_cfg, isys_stream_descr, isys_stream_idx);
-
-	isys_stream_descr->raw_packed = stream_cfg->pack_raw_pixels;
-	isys_stream_descr->linked_isys_stream_id = (int8_t) stream_cfg->isys_config[isys_stream_idx].linked_isys_stream_id;
-	/*
-	 * Early polling is required for timestamp accuracy in certain case.
-	 * The ISYS HW polling is started on
-	 * ia_css_isys_stream_capture_indication() instead of
-	 * ia_css_pipeline_sp_wait_for_isys_stream_N() as isp processing of
-	 * capture takes longer than getting an ISYS frame
-	 *
-	 * Only 2401 relevant ??
-	 */
-	isys_stream_descr->polling_mode
-		= early_polling ? INPUT_SYSTEM_POLL_ON_CAPTURE_REQUEST
-			: INPUT_SYSTEM_POLL_ON_WAIT_FOR_FRAME;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"sh_css_translate_stream_cfg_to_isys_stream_descr() leave:\n");
-
-	return rc;
-}
-
-static bool sh_css_translate_binary_info_to_input_system_output_port_attr(
-		struct ia_css_binary *binary,
-                ia_css_isys_descr_t     *isys_stream_descr)
-{
-	if (!binary)
-		return false;
-
-	isys_stream_descr->output_port_attr.left_padding = binary->left_padding;
-	isys_stream_descr->output_port_attr.max_isp_input_width = binary->info->sp.input.max_width;
-
-	return true;
-}
-
-static enum ia_css_err
-sh_css_config_input_network(struct ia_css_stream *stream)
-{
-	bool					rc;
-	ia_css_isys_descr_t			isys_stream_descr;
-	unsigned int				sp_thread_id;
-	struct sh_css_sp_pipeline_terminal	*sp_pipeline_input_terminal;
-	struct ia_css_pipe *pipe = NULL;
-	struct ia_css_binary *binary = NULL;
-	int i;
-	uint32_t isys_stream_id;
-	bool early_polling = false;
-
-	assert(stream != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"sh_css_config_input_network() enter 0x%p:\n", stream);
-
-	if (stream->config.continuous == true) {
-		if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_CAPTURE) {
-			pipe = stream->last_pipe;
-		} else if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_YUVPP) {
-			pipe = stream->last_pipe;
-		} else if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_PREVIEW) {
-			pipe = stream->last_pipe->pipe_settings.preview.copy_pipe;
-		} else if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_VIDEO) {
-			pipe = stream->last_pipe->pipe_settings.video.copy_pipe;
-		}
-	} else {
-		pipe = stream->last_pipe;
-		if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_CAPTURE) {
-			/*
-			 * We need to poll the ISYS HW in capture_indication itself
-			 * for "non-continuous" capture usecase for getting accurate
-			 * isys frame capture timestamps.
-			 * This is because the capturepipe propcessing takes longer
-			 * to execute than the input system frame capture.
-			 * 2401 specific
-			 */
-			early_polling = true;
-		}
-	}
-
-	assert(pipe != NULL);
-	if (pipe == NULL)
-		return IA_CSS_ERR_INTERNAL_ERROR;
-
-	if (pipe->pipeline.stages != NULL)
-		if (pipe->pipeline.stages->binary != NULL)
-			binary = pipe->pipeline.stages->binary;
-
-
-
-	if (binary) {
-		/* this was being done in ifmtr in 2400.
-		 * online and cont bypass the init_in_frameinfo_memory_defaults
-		 * so need to do it here
-		 */
-		ia_css_get_crop_offsets(pipe, &binary->in_frame_info);
-	}
-
-	/* get the SP thread id */
-	rc = ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &sp_thread_id);
-	if (!rc)
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	/* get the target input terminal */
-	sp_pipeline_input_terminal = &(sh_css_sp_group.pipe_io[sp_thread_id].input);
-
-	for (i = 0; i < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH; i++) {
-		/* initialization */
-		memset((void*)(&isys_stream_descr), 0, sizeof(ia_css_isys_descr_t));
-		sp_pipeline_input_terminal->context.virtual_input_system_stream[i].valid = 0;
-		sp_pipeline_input_terminal->ctrl.virtual_input_system_stream_cfg[i].valid = 0;
-
-		if (!stream->config.isys_config[i].valid)
-			continue;
-
-		/* translate the stream configuration to the Input System (2401) configuration */
-		rc = sh_css_translate_stream_cfg_to_isys_stream_descr(
-				&(stream->config),
-				early_polling,
-				&(isys_stream_descr), i);
-
-		if (stream->config.online) {
-			rc &= sh_css_translate_binary_info_to_input_system_output_port_attr(
-					binary,
-					&(isys_stream_descr));
-		}
-
-		if (!rc)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-
-		isys_stream_id = ia_css_isys_generate_stream_id(sp_thread_id, i);
-
-		/* create the virtual Input System (2401) */
-		rc =  ia_css_isys_stream_create(
-				&(isys_stream_descr),
-				&(sp_pipeline_input_terminal->context.virtual_input_system_stream[i]),
-				isys_stream_id);
-		if (!rc)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-
-		/* calculate the configuration of the virtual Input System (2401) */
-		rc = ia_css_isys_stream_calculate_cfg(
-				&(sp_pipeline_input_terminal->context.virtual_input_system_stream[i]),
-				&(isys_stream_descr),
-				&(sp_pipeline_input_terminal->ctrl.virtual_input_system_stream_cfg[i]));
-		if (!rc) {
-			ia_css_isys_stream_destroy(&(sp_pipeline_input_terminal->context.virtual_input_system_stream[i]));
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		}
-	}
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"sh_css_config_input_network() leave:\n");
-
-	return IA_CSS_SUCCESS;
-}
-
-static inline struct ia_css_pipe *stream_get_last_pipe(
-		struct ia_css_stream *stream)
-{
-	struct ia_css_pipe *last_pipe = NULL;
-	if (stream != NULL)
-		last_pipe = stream->last_pipe;
-
-	return last_pipe;
-}
-
-static inline struct ia_css_pipe *stream_get_copy_pipe(
-		struct ia_css_stream *stream)
-{
-	struct ia_css_pipe *copy_pipe = NULL;
-	struct ia_css_pipe *last_pipe = NULL;
-	enum ia_css_pipe_id pipe_id;
-
-	last_pipe = stream_get_last_pipe(stream);
-
-	if ((stream != NULL) &&
-	    (last_pipe != NULL) &&
-	    (stream->config.continuous)) {
-
-		pipe_id = last_pipe->mode;
-		switch (pipe_id) {
-			case IA_CSS_PIPE_ID_PREVIEW:
-				copy_pipe = last_pipe->pipe_settings.preview.copy_pipe;
-				break;
-			case IA_CSS_PIPE_ID_VIDEO:
-				copy_pipe = last_pipe->pipe_settings.video.copy_pipe;
-				break;
-			default:
-				copy_pipe = NULL;
-				break;
-		}
-	}
-
-	return copy_pipe;
-}
-
-static inline struct ia_css_pipe *stream_get_target_pipe(
-		struct ia_css_stream *stream)
-{
-	struct ia_css_pipe *target_pipe;
-
-	/* get the pipe that consumes the stream */
-	if (stream->config.continuous) {
-		target_pipe = stream_get_copy_pipe(stream);
-	} else {
-		target_pipe = stream_get_last_pipe(stream);
-	}
-
-	return target_pipe;
-}
-
-static enum ia_css_err stream_csi_rx_helper(
-	struct ia_css_stream *stream,
-	enum ia_css_err (*func)(enum mipi_port_id, uint32_t))
-{
-	enum ia_css_err retval = IA_CSS_ERR_INTERNAL_ERROR;
-	uint32_t sp_thread_id, stream_id;
-	bool rc;
-	struct ia_css_pipe *target_pipe = NULL;
-
-	if ((stream == NULL) || (stream->config.mode != IA_CSS_INPUT_MODE_BUFFERED_SENSOR))
-		goto exit;
-
-	target_pipe = stream_get_target_pipe(stream);
-
-	if (target_pipe == NULL)
-		goto exit;
-
-	rc = ia_css_pipeline_get_sp_thread_id(
-		ia_css_pipe_get_pipe_num(target_pipe),
-		&sp_thread_id);
-
-	if (!rc)
-		goto exit;
-
-	/* (un)register all valid "virtual isys streams" within the ia_css_stream */
-	stream_id = 0;
-	do {
-		if (stream->config.isys_config[stream_id].valid) {
-			uint32_t isys_stream_id = ia_css_isys_generate_stream_id(sp_thread_id, stream_id);
-			retval = func(stream->config.source.port.port, isys_stream_id);
-		}
-		stream_id++;
-	} while ((retval == IA_CSS_SUCCESS) &&
-		 (stream_id < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH));
-
-exit:
-	return retval;
-}
-
-static inline enum ia_css_err stream_register_with_csi_rx(
-	struct ia_css_stream *stream)
-{
-	return stream_csi_rx_helper(stream, ia_css_isys_csi_rx_register_stream);
-}
-
-static inline enum ia_css_err stream_unregister_with_csi_rx(
-	struct ia_css_stream *stream)
-{
-	return stream_csi_rx_helper(stream, ia_css_isys_csi_rx_unregister_stream);
-}
-#endif
-
-#if WITH_PC_MONITORING
-static struct task_struct *my_kthread;    /* Handle for the monitoring thread */
-static int sh_binary_running;         /* Enable sampling in the thread */
-
-static void print_pc_histo(char *core_name, struct sh_css_pc_histogram *hist)
-{
-	unsigned i;
-	unsigned cnt_run = 0;
-	unsigned cnt_stall = 0;
-
-	if (hist == NULL)
-		return;
-
-	sh_css_print("%s histogram length = %d\n", core_name, hist->length);
-	sh_css_print("%s PC\trun\tstall\n", core_name);
-
-	for (i = 0; i < hist->length; i++) {
-		if ((hist->run[i] == 0) && (hist->run[i] == hist->stall[i]))
-			continue;
-		sh_css_print("%s %d\t%d\t%d\n",
-				core_name, i, hist->run[i], hist->stall[i]);
-		cnt_run += hist->run[i];
-		cnt_stall += hist->stall[i];
-	}
-
-	sh_css_print(" Statistics for %s, cnt_run = %d, cnt_stall = %d, "
-	       "hist->length = %d\n",
-			core_name, cnt_run, cnt_stall, hist->length);
-}
-
-static void print_pc_histogram(void)
-{
-	struct ia_css_binary_metrics *metrics;
-
-	for (metrics = sh_css_metrics.binary_metrics;
-	     metrics;
-	     metrics = metrics->next) {
-		if (metrics->mode == IA_CSS_BINARY_MODE_PREVIEW ||
-		    metrics->mode == IA_CSS_BINARY_MODE_VF_PP) {
-			sh_css_print("pc_histogram for binary %d is SKIPPED\n",
-				metrics->id);
-			continue;
-		}
-
-		sh_css_print(" pc_histogram for binary %d\n", metrics->id);
-		print_pc_histo("  ISP", &metrics->isp_histogram);
-		print_pc_histo("  SP",   &metrics->sp_histogram);
-		sh_css_print("print_pc_histogram() done for binay->id = %d, "
-			     "done.\n", metrics->id);
-	}
-
-	sh_css_print("PC_MONITORING:print_pc_histogram() -- DONE\n");
-}
-
-static int pc_monitoring(void *data)
-{
-	int i = 0;
-
-	(void)data;
-	while (true) {
-		if (sh_binary_running) {
-			sh_css_metrics_sample_pcs();
-#if MULTIPLE_SAMPLES
-			for (i = 0; i < NOF_SAMPLES; i++)
-				sh_css_metrics_sample_pcs();
-#endif
-		}
-		usleep_range(10, 50);
-	}
-	return 0;
-}
-
-static void spying_thread_create(void)
-{
-	my_kthread = kthread_run(pc_monitoring, NULL, "sh_pc_monitor");
-	sh_css_metrics_enable_pc_histogram(1);
-}
-
-static void input_frame_info(struct ia_css_frame_info frame_info)
-{
-	sh_css_print("SH_CSS:input_frame_info() -- frame->info.res.width = %d, "
-	       "frame->info.res.height = %d, format = %d\n",
-			frame_info.res.width, frame_info.res.height, frame_info.format);
-}
-#endif /* WITH_PC_MONITORING */
-
-static void
-start_binary(struct ia_css_pipe *pipe,
-	     struct ia_css_binary *binary)
-{
-	struct ia_css_stream *stream;
-
-	assert(pipe != NULL);
-	/* Acceleration uses firmware, the binary thus can be NULL */
-	/* assert(binary != NULL); */
-
-	(void)binary;
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	stream = pipe->stream;
-#else
-	(void)pipe;
-	(void)stream;
-#endif
-
-	if (binary)
-		sh_css_metrics_start_binary(&binary->metrics);
-
-#if WITH_PC_MONITORING
-	sh_css_print("PC_MONITORING: %s() -- binary id = %d , "
-		     "enable_dvs_envelope = %d\n",
-		     __func__, binary->info->sp.id,
-		     binary->info->sp.enable.dvs_envelope);
-	input_frame_info(binary->in_frame_info);
-
-	if (binary && binary->info->sp.pipeline.mode == IA_CSS_BINARY_MODE_VIDEO)
-		sh_binary_running = true;
-#endif
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
-	if (stream->reconfigure_css_rx) {
-		ia_css_isys_rx_configure(&pipe->stream->csi_rx_config,
-					 pipe->stream->config.mode);
-		stream->reconfigure_css_rx = false;
-	}
-#endif
-}
-
-/* start the copy function on the SP */
-static enum ia_css_err
-start_copy_on_sp(struct ia_css_pipe *pipe,
-		 struct ia_css_frame *out_frame)
-{
-
-	(void)out_frame;
-	assert(pipe != NULL);
-	assert(pipe->stream != NULL);
-
-	if ((pipe == NULL) || (pipe->stream == NULL))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
-	if (pipe->stream->reconfigure_css_rx)
-		ia_css_isys_rx_disable();
-#endif
-
-	if (pipe->stream->config.input_config.format != ATOMISP_INPUT_FORMAT_BINARY_8)
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	sh_css_sp_start_binary_copy(ia_css_pipe_get_pipe_num(pipe), out_frame, pipe->stream->config.pixels_per_clock == 2);
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
-	if (pipe->stream->reconfigure_css_rx) {
-		ia_css_isys_rx_configure(&pipe->stream->csi_rx_config, pipe->stream->config.mode);
-		pipe->stream->reconfigure_css_rx = false;
-	}
-#endif
-
-	return IA_CSS_SUCCESS;
-}
-
-void sh_css_binary_args_reset(struct sh_css_binary_args *args)
-{
-	unsigned int i;
-
-#ifndef ISP2401
-	for (i = 0; i < NUM_VIDEO_TNR_FRAMES; i++)
-#else
-	for (i = 0; i < NUM_TNR_FRAMES; i++)
-#endif
-		args->tnr_frames[i] = NULL;
-	for (i = 0; i < MAX_NUM_VIDEO_DELAY_FRAMES; i++)
-		args->delay_frames[i] = NULL;
-	args->in_frame      = NULL;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-		args->out_frame[i] = NULL;
-	args->out_vf_frame  = NULL;
-	args->copy_vf       = false;
-	args->copy_output   = true;
-	args->vf_downscale_log2 = 0;
-}
-
-static void start_pipe(
-	struct ia_css_pipe *me,
-	enum sh_css_pipe_config_override copy_ovrd,
-	enum ia_css_input_mode input_mode)
-{
-#if defined(HAS_NO_INPUT_SYSTEM)
-	(void)input_mode;
-#endif
-
-	IA_CSS_ENTER_PRIVATE("me = %p, copy_ovrd = %d, input_mode = %d",
-			     me, copy_ovrd, input_mode);
-
-	assert(me != NULL); /* all callers are in this file and call with non null argument */
-
-	sh_css_sp_init_pipeline(&me->pipeline,
-				me->mode,
-				(uint8_t)ia_css_pipe_get_pipe_num(me),
-				me->config.default_capture_config.enable_xnr != 0,
-				me->stream->config.pixels_per_clock == 2,
-				me->stream->config.continuous,
-				false,
-				me->required_bds_factor,
-				copy_ovrd,
-				input_mode,
-				&me->stream->config.metadata_config,
-				&me->stream->info.metadata_info
-#if !defined(HAS_NO_INPUT_SYSTEM)
-				,(input_mode==IA_CSS_INPUT_MODE_MEMORY) ?
-					(enum mipi_port_id)0 :
-					me->stream->config.source.port.port
-#endif
-#ifdef ISP2401
-				,&me->config.internal_frame_origin_bqs_on_sctbl,
-				me->stream->isp_params_configs
-#endif
-			);
-
-	if (me->config.mode != IA_CSS_PIPE_MODE_COPY) {
-		struct ia_css_pipeline_stage *stage;
-		stage = me->pipeline.stages;
-		if (stage) {
-			me->pipeline.current_stage = stage;
-			start_binary(me, stage->binary);
-		}
-	}
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-void
-sh_css_invalidate_shading_tables(struct ia_css_stream *stream)
-{
-	int i;
-	assert(stream != NULL);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"sh_css_invalidate_shading_tables() enter:\n");
-
-	for (i=0; i<stream->num_pipes; i++) {
-		assert(stream->pipes[i] != NULL);
-		sh_css_pipe_free_shading_table(stream->pipes[i]);
-	}
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"sh_css_invalidate_shading_tables() leave: return_void\n");
-}
-
-#ifndef ISP2401
-static void
-enable_interrupts(enum ia_css_irq_type irq_type)
-{
-#ifdef USE_INPUT_SYSTEM_VERSION_2
-	enum mipi_port_id port;
-#endif
-	bool enable_pulse = irq_type != IA_CSS_IRQ_TYPE_EDGE;
-	IA_CSS_ENTER_PRIVATE("");
-	/* Enable IRQ on the SP which signals that SP goes to idle
-	 * (aka ready state) */
-	cnd_sp_irq_enable(SP0_ID, true);
-	/* Set the IRQ device 0 to either level or pulse */
-	irq_enable_pulse(IRQ0_ID, enable_pulse);
-
-	cnd_virq_enable_channel(virq_sp, true);
-
-	/* Enable SW interrupt 0, this is used to signal ISYS events */
-	cnd_virq_enable_channel(
-			(virq_id_t)(IRQ_SW_CHANNEL0_ID + IRQ_SW_CHANNEL_OFFSET),
-			true);
-	/* Enable SW interrupt 1, this is used to signal PSYS events */
-	cnd_virq_enable_channel(
-			(virq_id_t)(IRQ_SW_CHANNEL1_ID + IRQ_SW_CHANNEL_OFFSET),
-			true);
-#if !defined(HAS_IRQ_MAP_VERSION_2)
-	/* IRQ_SW_CHANNEL2_ID does not exist on 240x systems */
-	cnd_virq_enable_channel(
-			(virq_id_t)(IRQ_SW_CHANNEL2_ID + IRQ_SW_CHANNEL_OFFSET),
-			true);
-	virq_clear_all();
-#endif
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2
-	for (port = 0; port < N_MIPI_PORT_ID; port++)
-		ia_css_isys_rx_enable_all_interrupts(port);
-#endif
-
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-#endif
-
-static bool sh_css_setup_spctrl_config(const struct ia_css_fw_info *fw,
-							const char * program,
-							ia_css_spctrl_cfg  *spctrl_cfg)
-{
-	if((fw == NULL)||(spctrl_cfg == NULL))
-		return false;
-	spctrl_cfg->sp_entry = 0;
-	spctrl_cfg->program_name = (char *)(program);
-
-	spctrl_cfg->ddr_data_offset =  fw->blob.data_source;
-	spctrl_cfg->dmem_data_addr = fw->blob.data_target;
-	spctrl_cfg->dmem_bss_addr = fw->blob.bss_target;
-	spctrl_cfg->data_size = fw->blob.data_size ;
-	spctrl_cfg->bss_size = fw->blob.bss_size;
-
-	spctrl_cfg->spctrl_config_dmem_addr = fw->info.sp.init_dmem_data;
-	spctrl_cfg->spctrl_state_dmem_addr = fw->info.sp.sw_state;
-
-	spctrl_cfg->code_size = fw->blob.size;
-	spctrl_cfg->code      = fw->blob.code;
-	spctrl_cfg->sp_entry  = fw->info.sp.sp_entry; /* entry function ptr on SP */
-
-	return true;
-}
-void
-ia_css_unload_firmware(void)
-{
-	if (sh_css_num_binaries)
-	{
-		/* we have already loaded before so get rid of the old stuff */
-		ia_css_binary_uninit();
-		sh_css_unload_firmware();
-	}
-	fw_explicitly_loaded = false;
-}
-
-static void
-ia_css_reset_defaults(struct sh_css* css)
-{
-	struct sh_css default_css;
-
-	/* Reset everything to zero */
-	memset(&default_css, 0, sizeof(default_css));
-
-	/* Initialize the non zero values*/
-	default_css.check_system_idle = true;
-	default_css.num_cont_raw_frames = NUM_CONTINUOUS_FRAMES;
-
-	/* All should be 0: but memset does it already.
-	 * default_css.num_mipi_frames[N_CSI_PORTS] = 0;
-	 */
-
-	default_css.irq_type = IA_CSS_IRQ_TYPE_EDGE;
-
-	/*Set the defaults to the output */
-	*css = default_css;
-}
-
-bool
-ia_css_check_firmware_version(const struct ia_css_fw  *fw)
-{
-	bool retval = false;
-
-	if (fw != NULL) {
-		retval = sh_css_check_firmware_version(fw->data);
-	}
-	return retval;
-}
-
-enum ia_css_err
-ia_css_load_firmware(const struct ia_css_env *env,
-	    const struct ia_css_fw  *fw)
-{
-	enum ia_css_err err;
-
-	if (env == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	if (fw == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_load_firmware() enter\n");
-
-	/* make sure we initialize my_css */
-	if (my_css.flush != env->cpu_mem_env.flush) {
-		ia_css_reset_defaults(&my_css);
-		my_css.flush = env->cpu_mem_env.flush;
-	}
-
-	ia_css_unload_firmware(); /* in case we are called twice */
-	err = sh_css_load_firmware(fw->data, fw->bytes);
-	if (err == IA_CSS_SUCCESS) {
-		err = ia_css_binary_init_infos();
-		if (err == IA_CSS_SUCCESS)
-			fw_explicitly_loaded = true;
-	}
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_load_firmware() leave \n");
-	return err;
-}
-
-enum ia_css_err
-ia_css_init(const struct ia_css_env *env,
-	    const struct ia_css_fw  *fw,
-	    uint32_t                 mmu_l1_base,
-	    enum ia_css_irq_type     irq_type)
-{
-	enum ia_css_err err;
-	ia_css_spctrl_cfg spctrl_cfg;
-
-	void (*flush_func)(struct ia_css_acc_fw *fw);
-	hrt_data select, enable;
-
-	/*
-	 * The C99 standard does not specify the exact object representation of structs;
-	 * the representation is compiler dependent.
-	 *
-	 * The structs that are communicated between host and SP/ISP should have the
-	 * exact same object representation. The compiler that is used to compile the
-	 * firmware is hivecc.
-	 *
-	 * To check if a different compiler, used to compile a host application, uses
-	 * another object representation, macros are defined specifying the size of
-	 * the structs as expected by the firmware.
-	 *
-	 * A host application shall verify that a sizeof( ) of the struct is equal to
-	 * the SIZE_OF_XXX macro of the corresponding struct. If they are not
-	 * equal, functionality will break.
-	 */
-	/* Check struct sh_css_ddr_address_map */
-	COMPILATION_ERROR_IF( sizeof(struct sh_css_ddr_address_map)		!= SIZE_OF_SH_CSS_DDR_ADDRESS_MAP_STRUCT	);
-	/* Check struct host_sp_queues */
-	COMPILATION_ERROR_IF( sizeof(struct host_sp_queues)			!= SIZE_OF_HOST_SP_QUEUES_STRUCT		);
-	COMPILATION_ERROR_IF( sizeof(struct ia_css_circbuf_desc_s)		!= SIZE_OF_IA_CSS_CIRCBUF_DESC_S_STRUCT		);
-	COMPILATION_ERROR_IF( sizeof(struct ia_css_circbuf_elem_s)		!= SIZE_OF_IA_CSS_CIRCBUF_ELEM_S_STRUCT		);
-
-	/* Check struct host_sp_communication */
-	COMPILATION_ERROR_IF( sizeof(struct host_sp_communication)		!= SIZE_OF_HOST_SP_COMMUNICATION_STRUCT		);
-	COMPILATION_ERROR_IF( sizeof(struct sh_css_event_irq_mask)		!= SIZE_OF_SH_CSS_EVENT_IRQ_MASK_STRUCT		);
-
-	/* Check struct sh_css_hmm_buffer */
-	COMPILATION_ERROR_IF( sizeof(struct sh_css_hmm_buffer)			!= SIZE_OF_SH_CSS_HMM_BUFFER_STRUCT		);
-	COMPILATION_ERROR_IF( sizeof(struct ia_css_isp_3a_statistics)		!= SIZE_OF_IA_CSS_ISP_3A_STATISTICS_STRUCT	);
-	COMPILATION_ERROR_IF( sizeof(struct ia_css_isp_dvs_statistics)		!= SIZE_OF_IA_CSS_ISP_DVS_STATISTICS_STRUCT	);
-	COMPILATION_ERROR_IF( sizeof(struct ia_css_metadata)			!= SIZE_OF_IA_CSS_METADATA_STRUCT		);
-
-	/* Check struct ia_css_init_dmem_cfg */
-	COMPILATION_ERROR_IF( sizeof(struct ia_css_sp_init_dmem_cfg)		!= SIZE_OF_IA_CSS_SP_INIT_DMEM_CFG_STRUCT	);
-
-	if (fw == NULL && !fw_explicitly_loaded)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	if (env == NULL)
-	    return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	sh_css_printf = env->print_env.debug_print;
-
-	IA_CSS_ENTER("void");
-
-	flush_func     = env->cpu_mem_env.flush;
-
-	pipe_global_init();
-	ia_css_pipeline_init();
-	ia_css_queue_map_init();
-
-	ia_css_device_access_init(&env->hw_access_env);
-
-	select = gpio_reg_load(GPIO0_ID, _gpio_block_reg_do_select)
-						& (~GPIO_FLASH_PIN_MASK);
-	enable = gpio_reg_load(GPIO0_ID, _gpio_block_reg_do_e)
-							| GPIO_FLASH_PIN_MASK;
-	sh_css_mmu_set_page_table_base_index(mmu_l1_base);
-#ifndef ISP2401
-	my_css_save.mmu_base = mmu_l1_base;
-#else
-	ia_css_save_mmu_base_addr(mmu_l1_base);
-#endif
-
-	ia_css_reset_defaults(&my_css);
-
-	my_css_save.driver_env = *env;
-	my_css.flush     = flush_func;
-
-	err = ia_css_rmgr_init();
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-
-#ifndef ISP2401
-	IA_CSS_LOG("init: %d", my_css_save_initialized);
-#else
-	ia_css_save_restore_data_init();
-#endif
-
-#ifndef ISP2401
-	if (!my_css_save_initialized)
-	{
-		my_css_save_initialized = true;
-		my_css_save.mode = sh_css_mode_working;
-		memset(my_css_save.stream_seeds, 0, sizeof(struct sh_css_stream_seed) * MAX_ACTIVE_STREAMS);
-		IA_CSS_LOG("init: %d mode=%d", my_css_save_initialized, my_css_save.mode);
-	}
-#endif
-	mipi_init();
-
-#ifndef ISP2401
-	/* In case this has been programmed already, update internal
-	   data structure ... DEPRECATED */
-	my_css.page_table_base_index = mmu_get_page_table_base_index(MMU0_ID);
-
-#endif
-	my_css.irq_type = irq_type;
-#ifndef ISP2401
-	my_css_save.irq_type = irq_type;
-#else
-	ia_css_save_irq_type(irq_type);
-#endif
-	enable_interrupts(my_css.irq_type);
-
-	/* configure GPIO to output mode */
-	gpio_reg_store(GPIO0_ID, _gpio_block_reg_do_select, select);
-	gpio_reg_store(GPIO0_ID, _gpio_block_reg_do_e, enable);
-	gpio_reg_store(GPIO0_ID, _gpio_block_reg_do_0, 0);
-
-	err = ia_css_refcount_init(REFCOUNT_SIZE);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-	err = sh_css_params_init();
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-	if (fw)
-	{
-		ia_css_unload_firmware(); /* in case we already had firmware loaded */
-		err = sh_css_load_firmware(fw->data, fw->bytes);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR(err);
-			return err;
-		}
-		err = ia_css_binary_init_infos();
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR(err);
-			return err;
-		}
-		fw_explicitly_loaded = false;
-#ifndef ISP2401
-		my_css_save.loaded_fw = (struct ia_css_fw *)fw;
-#endif
-	}
-	if(!sh_css_setup_spctrl_config(&sh_css_sp_fw,SP_PROG_NAME,&spctrl_cfg))
-		return IA_CSS_ERR_INTERNAL_ERROR;
-
-	err = ia_css_spctrl_load_fw(SP0_ID, &spctrl_cfg);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-
-#if WITH_PC_MONITORING
-	if (!thread_alive) {
-		thread_alive++;
-		sh_css_print("PC_MONITORING: %s() -- create thread DISABLED\n",
-			     __func__);
-		spying_thread_create();
-	}
-#endif
-	if (!sh_css_hrt_system_is_idle()) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_SYSTEM_NOT_IDLE);
-		return IA_CSS_ERR_SYSTEM_NOT_IDLE;
-	}
-	/* can be called here, queuing works, but:
-	   - when sp is started later, it will wipe queued items
-	   so for now we leave it for later and make sure
-	   updates are not called to frequently.
-	sh_css_init_buffer_queues();
-	*/
-
-#if defined(HAS_INPUT_SYSTEM_VERSION_2) && defined(HAS_INPUT_SYSTEM_VERSION_2401)
-#if	defined(USE_INPUT_SYSTEM_VERSION_2)
-	gp_device_reg_store(GP_DEVICE0_ID, _REG_GP_SWITCH_ISYS2401_ADDR, 0);
-#elif defined (USE_INPUT_SYSTEM_VERSION_2401)
-	gp_device_reg_store(GP_DEVICE0_ID, _REG_GP_SWITCH_ISYS2401_ADDR, 1);
-#endif
-#endif
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	dma_set_max_burst_size(DMA0_ID, HIVE_DMA_BUS_DDR_CONN,
-			       ISP_DMA_MAX_BURST_LENGTH);
-
-	if(ia_css_isys_init() != INPUT_SYSTEM_ERR_NO_ERROR)
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-#endif
-
-	sh_css_params_map_and_store_default_gdc_lut();
-
-	IA_CSS_LEAVE_ERR(err);
-	return err;
-}
-
-enum ia_css_err ia_css_suspend(void)
-{
-	int i;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_suspend() enter\n");
-	my_css_save.mode = sh_css_mode_suspend;
-	for(i=0;i<MAX_ACTIVE_STREAMS;i++)
-		if (my_css_save.stream_seeds[i].stream != NULL)
-		{
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "==*> unloading seed %d (%p)\n", i, my_css_save.stream_seeds[i].stream);
-			ia_css_stream_unload(my_css_save.stream_seeds[i].stream);
-		}
-	my_css_save.mode = sh_css_mode_working;
-	ia_css_stop_sp();
-	ia_css_uninit();
-	for(i=0;i<MAX_ACTIVE_STREAMS;i++)
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "==*> after 1: seed %d (%p)\n", i, my_css_save.stream_seeds[i].stream);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_suspend() leave\n");
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err
-ia_css_resume(void)
-{
-	int i, j;
-	enum ia_css_err err;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_resume() enter: void\n");
-
-	err = ia_css_init(&(my_css_save.driver_env), my_css_save.loaded_fw, my_css_save.mmu_base, my_css_save.irq_type);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	err = ia_css_start_sp();
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	my_css_save.mode = sh_css_mode_resume;
-	for(i=0;i<MAX_ACTIVE_STREAMS;i++)
-	{
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "==*> seed stream %p\n", my_css_save.stream_seeds[i].stream);
-		if (my_css_save.stream_seeds[i].stream != NULL)
-		{
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "==*> loading seed %d\n", i);
-			err = ia_css_stream_load(my_css_save.stream_seeds[i].stream);
-			if (err != IA_CSS_SUCCESS)
-			{
-				if (i)
-					for(j=0;j<i;j++)
-						ia_css_stream_unload(my_css_save.stream_seeds[j].stream);
-				return err;
-			}
-			err = ia_css_stream_start(my_css_save.stream_seeds[i].stream);
-			if (err != IA_CSS_SUCCESS)
-			{
-				for(j=0;j<=i;j++)
-				{
-					ia_css_stream_stop(my_css_save.stream_seeds[j].stream);
-					ia_css_stream_unload(my_css_save.stream_seeds[j].stream);
-				}
-				return err;
-			}
-			*my_css_save.stream_seeds[i].orig_stream = my_css_save.stream_seeds[i].stream;
-			for(j=0;j<my_css_save.stream_seeds[i].num_pipes;j++)
-				*(my_css_save.stream_seeds[i].orig_pipes[j]) = my_css_save.stream_seeds[i].pipes[j];
-		}
-	}
-	my_css_save.mode = sh_css_mode_working;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_resume() leave: return_void\n");
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err
-ia_css_enable_isys_event_queue(bool enable)
-{
-	if (sh_css_sp_is_running())
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	sh_css_sp_enable_isys_event_queue(enable);
-	return IA_CSS_SUCCESS;
-}
-
-void *sh_css_malloc(size_t size)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_malloc() enter: size=%zu\n",size);
-	/* FIXME: This first test can probably go away */
-	if (size == 0)
-		return NULL;
-	if (size > PAGE_SIZE)
-		return vmalloc(size);
-	return kmalloc(size, GFP_KERNEL);
-}
-
-void *sh_css_calloc(size_t N, size_t size)
-{
-	void *p;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_calloc() enter: N=%zu, size=%zu\n",N,size);
-
-	/* FIXME: this test can probably go away */
-	if (size > 0) {
-		p = sh_css_malloc(N*size);
-		if (p)
-			memset(p, 0, size);
-		return p;
-	}
-	return NULL;
-}
-
-void sh_css_free(void *ptr)
-{
-	if (is_vmalloc_addr(ptr))
-		vfree(ptr);
-	else
-		kfree(ptr);
-}
-
-/* For Acceleration API: Flush FW (shared buffer pointer) arguments */
-void
-sh_css_flush(struct ia_css_acc_fw *fw)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_flush() enter:\n");
-	if ((fw != NULL) && (my_css.flush != NULL))
-		my_css.flush(fw);
-}
-
-/* Mapping sp threads. Currently, this is done when a stream is created and
- * pipelines are ready to be converted to sp pipelines. Be careful if you are
- * doing it from stream_create since we could run out of sp threads due to
- * allocation on inactive pipelines. */
-static enum ia_css_err
-map_sp_threads(struct ia_css_stream *stream, bool map)
-{
-	struct ia_css_pipe *main_pipe = NULL;
-	struct ia_css_pipe *copy_pipe = NULL;
-	struct ia_css_pipe *capture_pipe = NULL;
-	struct ia_css_pipe *acc_pipe = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	enum ia_css_pipe_id pipe_id;
-
-	assert(stream != NULL);
-	IA_CSS_ENTER_PRIVATE("stream = %p, map = %s",
-			     stream, map ? "true" : "false");
-
-	if (stream == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	main_pipe = stream->last_pipe;
-	pipe_id	= main_pipe->mode;
-
-	ia_css_pipeline_map(main_pipe->pipe_num, map);
-
-	switch (pipe_id) {
-	case IA_CSS_PIPE_ID_PREVIEW:
-		copy_pipe    = main_pipe->pipe_settings.preview.copy_pipe;
-		capture_pipe = main_pipe->pipe_settings.preview.capture_pipe;
-		acc_pipe     = main_pipe->pipe_settings.preview.acc_pipe;
-		break;
-
-	case IA_CSS_PIPE_ID_VIDEO:
-		copy_pipe    = main_pipe->pipe_settings.video.copy_pipe;
-		capture_pipe = main_pipe->pipe_settings.video.capture_pipe;
-		break;
-
-	case IA_CSS_PIPE_ID_CAPTURE:
-	case IA_CSS_PIPE_ID_ACC:
-	default:
-		break;
-	}
-
-	if (acc_pipe) {
-		ia_css_pipeline_map(acc_pipe->pipe_num, map);
-	}
-
-	if(capture_pipe) {
-		ia_css_pipeline_map(capture_pipe->pipe_num, map);
-	}
-
-	/* Firmware expects copy pipe to be the last pipe mapped. (if needed) */
-	if(copy_pipe) {
-		ia_css_pipeline_map(copy_pipe->pipe_num, map);
-	}
-	/* DH regular multi pipe - not continuous mode: map the next pipes too */
-	if (!stream->config.continuous) {
-		int i;
-		for (i = 1; i < stream->num_pipes; i++)
-			ia_css_pipeline_map(stream->pipes[i]->pipe_num, map);
-	}
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-/* creates a host pipeline skeleton for all pipes in a stream. Called during
- * stream_create. */
-static enum ia_css_err
-create_host_pipeline_structure(struct ia_css_stream *stream)
-{
-	struct ia_css_pipe *copy_pipe = NULL, *capture_pipe = NULL;
-	struct ia_css_pipe *acc_pipe = NULL;
-	enum ia_css_pipe_id pipe_id;
-	struct ia_css_pipe *main_pipe = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	unsigned int copy_pipe_delay = 0,
-		     capture_pipe_delay = 0;
-
-	assert(stream != NULL);
-	IA_CSS_ENTER_PRIVATE("stream = %p", stream);
-
-	if (stream == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	main_pipe	= stream->last_pipe;
-	assert(main_pipe != NULL);
-	if (main_pipe == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	pipe_id	= main_pipe->mode;
-
-	switch (pipe_id) {
-	case IA_CSS_PIPE_ID_PREVIEW:
-		copy_pipe    = main_pipe->pipe_settings.preview.copy_pipe;
-		copy_pipe_delay = main_pipe->dvs_frame_delay;
-		capture_pipe = main_pipe->pipe_settings.preview.capture_pipe;
-		capture_pipe_delay = IA_CSS_FRAME_DELAY_0;
-		acc_pipe     = main_pipe->pipe_settings.preview.acc_pipe;
-		err = ia_css_pipeline_create(&main_pipe->pipeline, main_pipe->mode, main_pipe->pipe_num, main_pipe->dvs_frame_delay);
-		break;
-
-	case IA_CSS_PIPE_ID_VIDEO:
-		copy_pipe    = main_pipe->pipe_settings.video.copy_pipe;
-		copy_pipe_delay = main_pipe->dvs_frame_delay;
-		capture_pipe = main_pipe->pipe_settings.video.capture_pipe;
-		capture_pipe_delay = IA_CSS_FRAME_DELAY_0;
-		err = ia_css_pipeline_create(&main_pipe->pipeline, main_pipe->mode, main_pipe->pipe_num, main_pipe->dvs_frame_delay);
-		break;
-
-	case IA_CSS_PIPE_ID_CAPTURE:
-		capture_pipe = main_pipe;
-		capture_pipe_delay = main_pipe->dvs_frame_delay;
-		break;
-
-	case IA_CSS_PIPE_ID_YUVPP:
-		err = ia_css_pipeline_create(&main_pipe->pipeline, main_pipe->mode,
-						main_pipe->pipe_num, main_pipe->dvs_frame_delay);
-		break;
-
-	case IA_CSS_PIPE_ID_ACC:
-		err = ia_css_pipeline_create(&main_pipe->pipeline, main_pipe->mode, main_pipe->pipe_num, main_pipe->dvs_frame_delay);
-		break;
-
-	default:
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if ((IA_CSS_SUCCESS == err) && copy_pipe) {
-		err = ia_css_pipeline_create(&copy_pipe->pipeline,
-								copy_pipe->mode,
-								copy_pipe->pipe_num,
-								copy_pipe_delay);
-	}
-
-	if ((IA_CSS_SUCCESS == err) && capture_pipe) {
-		err = ia_css_pipeline_create(&capture_pipe->pipeline,
-								capture_pipe->mode,
-								capture_pipe->pipe_num,
-								capture_pipe_delay);
-	}
-
-	if ((IA_CSS_SUCCESS == err) && acc_pipe) {
-		err = ia_css_pipeline_create(&acc_pipe->pipeline, acc_pipe->mode, acc_pipe->pipe_num, main_pipe->dvs_frame_delay);
-	}
-
-	/* DH regular multi pipe - not continuous mode: create the next pipelines too */
- 	if (!stream->config.continuous) {
-		int i;
-		for (i = 1; i < stream->num_pipes && IA_CSS_SUCCESS == err; i++) {
-			main_pipe = stream->pipes[i];
-			err = ia_css_pipeline_create(&main_pipe->pipeline,
-							main_pipe->mode,
-							main_pipe->pipe_num,
-							main_pipe->dvs_frame_delay);
-		}
-	}
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-/* creates a host pipeline for all pipes in a stream. Called during
- * stream_start. */
-static enum ia_css_err
-create_host_pipeline(struct ia_css_stream *stream)
-{
-	struct ia_css_pipe *copy_pipe = NULL, *capture_pipe = NULL;
-	struct ia_css_pipe *acc_pipe = NULL;
-	enum ia_css_pipe_id pipe_id;
-	struct ia_css_pipe *main_pipe = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	unsigned max_input_width = 0;
-
-	IA_CSS_ENTER_PRIVATE("stream = %p", stream);
-	if (stream == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	main_pipe	= stream->last_pipe;
-	pipe_id	= main_pipe->mode;
-
-	/* No continuous frame allocation for capture pipe. It uses the
-	 * "main" pipe's frames. */
-	if ((pipe_id == IA_CSS_PIPE_ID_PREVIEW) ||
-	   (pipe_id == IA_CSS_PIPE_ID_VIDEO)) {
-		/* About pipe_id == IA_CSS_PIPE_ID_PREVIEW && stream->config.mode != IA_CSS_INPUT_MODE_MEMORY:
-		 * The original condition pipe_id == IA_CSS_PIPE_ID_PREVIEW is too strong. E.g. in SkyCam (with memory
-		 * based input frames) there is no continuous mode and thus no need for allocated continuous frames
-		 * This is not only for SkyCam but for all preview cases that use DDR based input frames. For this
-		 * reason the stream->config.mode != IA_CSS_INPUT_MODE_MEMORY has beed added.
-		 */
-		if (stream->config.continuous ||
-			(pipe_id == IA_CSS_PIPE_ID_PREVIEW && stream->config.mode != IA_CSS_INPUT_MODE_MEMORY)) {
-			err = alloc_continuous_frames(main_pipe, true);
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-		}
-
-	}
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-	/* old isys: need to allocate_mipi_frames() even in IA_CSS_PIPE_MODE_COPY */
-	if (pipe_id != IA_CSS_PIPE_ID_ACC) {
-		err = allocate_mipi_frames(main_pipe, &stream->info);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-#elif defined(USE_INPUT_SYSTEM_VERSION_2401)
-	if ((pipe_id != IA_CSS_PIPE_ID_ACC) &&
-		(main_pipe->config.mode != IA_CSS_PIPE_MODE_COPY)) {
-		err = allocate_mipi_frames(main_pipe, &stream->info);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-#endif
-
-	switch (pipe_id) {
-	case IA_CSS_PIPE_ID_PREVIEW:
-		copy_pipe    = main_pipe->pipe_settings.preview.copy_pipe;
-		capture_pipe = main_pipe->pipe_settings.preview.capture_pipe;
-		acc_pipe     = main_pipe->pipe_settings.preview.acc_pipe;
-		max_input_width =
-			main_pipe->pipe_settings.preview.preview_binary.info->sp.input.max_width;
-
-		err = create_host_preview_pipeline(main_pipe);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-
-		break;
-
-	case IA_CSS_PIPE_ID_VIDEO:
-		copy_pipe    = main_pipe->pipe_settings.video.copy_pipe;
-		capture_pipe = main_pipe->pipe_settings.video.capture_pipe;
-		max_input_width =
-			main_pipe->pipe_settings.video.video_binary.info->sp.input.max_width;
-
-		err = create_host_video_pipeline(main_pipe);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-
-		break;
-
-	case IA_CSS_PIPE_ID_CAPTURE:
-		capture_pipe = main_pipe;
-
-		break;
-
-	case IA_CSS_PIPE_ID_YUVPP:
-		err = create_host_yuvpp_pipeline(main_pipe);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-
-		break;
-
-	case IA_CSS_PIPE_ID_ACC:
-		err = create_host_acc_pipeline(main_pipe);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-
-		break;
-	default:
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	if (err != IA_CSS_SUCCESS)
-		goto ERR;
-
-	if(copy_pipe) {
-		err = create_host_copy_pipeline(copy_pipe, max_input_width,
-					  main_pipe->continuous_frames[0]);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-
-	if(capture_pipe) {
-		err = create_host_capture_pipeline(capture_pipe);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-
-	if (acc_pipe) {
-		err = create_host_acc_pipeline(acc_pipe);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-
-	/* DH regular multi pipe - not continuous mode: create the next pipelines too */
-	if (!stream->config.continuous) {
-		int i;
-		for (i = 1; i < stream->num_pipes && IA_CSS_SUCCESS == err; i++) {
-			switch (stream->pipes[i]->mode) {
-			case IA_CSS_PIPE_ID_PREVIEW:
-				err = create_host_preview_pipeline(stream->pipes[i]);
-				break;
-			case IA_CSS_PIPE_ID_VIDEO:
-				err = create_host_video_pipeline(stream->pipes[i]);
-				break;
-			case IA_CSS_PIPE_ID_CAPTURE:
-				err = create_host_capture_pipeline(stream->pipes[i]);
-				break;
-			case IA_CSS_PIPE_ID_YUVPP:
-				err = create_host_yuvpp_pipeline(stream->pipes[i]);
-				break;
-			case IA_CSS_PIPE_ID_ACC:
-				err = create_host_acc_pipeline(stream->pipes[i]);
-				break;
-			default:
-				err = IA_CSS_ERR_INVALID_ARGUMENTS;
-			}
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-		}
-	}
-
-ERR:
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static enum ia_css_err
-init_pipe_defaults(enum ia_css_pipe_mode mode,
-	       struct ia_css_pipe *pipe,
-	       bool copy_pipe)
-{
-	if (pipe == NULL) {
-		IA_CSS_ERROR("NULL pipe parameter");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	/* Initialize pipe to pre-defined defaults */
-	*pipe = IA_CSS_DEFAULT_PIPE;
-
-	/* TODO: JB should not be needed, but temporary backward reference */
-	switch (mode) {
-	case IA_CSS_PIPE_MODE_PREVIEW:
-		pipe->mode = IA_CSS_PIPE_ID_PREVIEW;
-		pipe->pipe_settings.preview = IA_CSS_DEFAULT_PREVIEW_SETTINGS;
-		break;
-	case IA_CSS_PIPE_MODE_CAPTURE:
-		if (copy_pipe) {
-			pipe->mode = IA_CSS_PIPE_ID_COPY;
-		} else {
-			pipe->mode = IA_CSS_PIPE_ID_CAPTURE;
-		}
-		pipe->pipe_settings.capture = IA_CSS_DEFAULT_CAPTURE_SETTINGS;
-		break;
-	case IA_CSS_PIPE_MODE_VIDEO:
-		pipe->mode = IA_CSS_PIPE_ID_VIDEO;
-		pipe->pipe_settings.video = IA_CSS_DEFAULT_VIDEO_SETTINGS;
-		break;
-	case IA_CSS_PIPE_MODE_ACC:
-		pipe->mode = IA_CSS_PIPE_ID_ACC;
-		break;
-	case IA_CSS_PIPE_MODE_COPY:
-		pipe->mode = IA_CSS_PIPE_ID_CAPTURE;
-		break;
-	case IA_CSS_PIPE_MODE_YUVPP:
-		pipe->mode = IA_CSS_PIPE_ID_YUVPP;
-		pipe->pipe_settings.yuvpp = IA_CSS_DEFAULT_YUVPP_SETTINGS;
-		break;
-	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	return IA_CSS_SUCCESS;
-}
-
-static void
-pipe_global_init(void)
-{
-	uint8_t i;
-
-	my_css.pipe_counter = 0;
-	for (i = 0; i < IA_CSS_PIPELINE_NUM_MAX; i++) {
-		my_css.all_pipes[i] = NULL;
-	}
-}
-
-static enum ia_css_err
-pipe_generate_pipe_num(const struct ia_css_pipe *pipe, unsigned int *pipe_number)
-{
-	const uint8_t INVALID_PIPE_NUM = (uint8_t)~(0);
-	uint8_t pipe_num = INVALID_PIPE_NUM;
-	uint8_t i;
-
-	if (pipe == NULL) {
-		IA_CSS_ERROR("NULL pipe parameter");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	/* Assign a new pipe_num .... search for empty place */
-	for (i = 0; i < IA_CSS_PIPELINE_NUM_MAX; i++) {
-		if (my_css.all_pipes[i] == NULL) {
-			/*position is reserved */
-			my_css.all_pipes[i] = (struct ia_css_pipe *)pipe;
-			pipe_num = i;
-			break;
-		}
-	}
-	if (pipe_num == INVALID_PIPE_NUM) {
-		/* Max number of pipes already allocated */
-		IA_CSS_ERROR("Max number of pipes already created");
-		return IA_CSS_ERR_RESOURCE_EXHAUSTED;
-	}
-
-	my_css.pipe_counter++;
-
-	IA_CSS_LOG("pipe_num (%d)", pipe_num);
-
-	*pipe_number = pipe_num;
-	return IA_CSS_SUCCESS;
-}
-
-static void
-pipe_release_pipe_num(unsigned int pipe_num)
-{
-	my_css.all_pipes[pipe_num] = NULL;
-	my_css.pipe_counter--;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"pipe_release_pipe_num (%d)\n", pipe_num);
-}
-
-static enum ia_css_err
-create_pipe(enum ia_css_pipe_mode mode,
-	    struct ia_css_pipe **pipe,
-	    bool copy_pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_pipe *me;
-
-	if (pipe == NULL) {
-		IA_CSS_ERROR("NULL pipe parameter");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	me = kmalloc(sizeof(*me), GFP_KERNEL);
-	if (!me)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-
-	err = init_pipe_defaults(mode, me, copy_pipe);
-	if (err != IA_CSS_SUCCESS) {
-		kfree(me);
-		return err;
-	}
-
-	err = pipe_generate_pipe_num(me, &(me->pipe_num));
-	if (err != IA_CSS_SUCCESS) {
-		kfree(me);
-		return err;
-	}
-
-	*pipe = me;
-	return IA_CSS_SUCCESS;
-}
-
-struct ia_css_pipe *
-find_pipe_by_num(uint32_t pipe_num)
-{
-	unsigned int i;
-	for (i = 0; i < IA_CSS_PIPELINE_NUM_MAX; i++){
-		if (my_css.all_pipes[i] &&
-				ia_css_pipe_get_pipe_num(my_css.all_pipes[i]) == pipe_num) {
-			return my_css.all_pipes[i];
-		}
-	}
-	return NULL;
-}
-
-static void sh_css_pipe_free_acc_binaries (
-    struct ia_css_pipe *pipe)
-{
-	struct ia_css_pipeline *pipeline;
-	struct ia_css_pipeline_stage *stage;
-
-	assert(pipe != NULL);
-	if (pipe == NULL) {
-		IA_CSS_ERROR("NULL input pointer");
-		return;
-	}
-	pipeline = &pipe->pipeline;
-
-	/* loop through the stages and unload them */
-	for (stage = pipeline->stages; stage; stage = stage->next) {
-		struct ia_css_fw_info *firmware = (struct ia_css_fw_info *)
-						stage->firmware;
-		if (firmware)
-			ia_css_pipe_unload_extension(pipe, firmware);
-	}
-}
-
-enum ia_css_err
-ia_css_pipe_destroy(struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	IA_CSS_ENTER("pipe = %p", pipe);
-
-	if (pipe == NULL) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if (pipe->stream != NULL) {
-		IA_CSS_LOG("ia_css_stream_destroy not called!");
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	switch (pipe->config.mode) {
-	case IA_CSS_PIPE_MODE_PREVIEW:
-		/* need to take into account that this function is also called
-		   on the internal copy pipe */
-		if (pipe->mode == IA_CSS_PIPE_ID_PREVIEW) {
-			ia_css_frame_free_multiple(NUM_CONTINUOUS_FRAMES,
-					pipe->continuous_frames);
-			ia_css_metadata_free_multiple(NUM_CONTINUOUS_FRAMES,
-					pipe->cont_md_buffers);
-			if (pipe->pipe_settings.preview.copy_pipe) {
-				err = ia_css_pipe_destroy(pipe->pipe_settings.preview.copy_pipe);
-				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipe_destroy(): "
-					"destroyed internal copy pipe err=%d\n", err);
-			}
-		}
-		break;
-	case IA_CSS_PIPE_MODE_VIDEO:
-		if (pipe->mode == IA_CSS_PIPE_ID_VIDEO) {
-			ia_css_frame_free_multiple(NUM_CONTINUOUS_FRAMES,
-				pipe->continuous_frames);
-			ia_css_metadata_free_multiple(NUM_CONTINUOUS_FRAMES,
-					pipe->cont_md_buffers);
-			if (pipe->pipe_settings.video.copy_pipe) {
-				err = ia_css_pipe_destroy(pipe->pipe_settings.video.copy_pipe);
-				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipe_destroy(): "
-					"destroyed internal copy pipe err=%d\n", err);
-			}
-		}
-#ifndef ISP2401
-		ia_css_frame_free_multiple(NUM_VIDEO_TNR_FRAMES, pipe->pipe_settings.video.tnr_frames);
-#else
-		ia_css_frame_free_multiple(NUM_TNR_FRAMES, pipe->pipe_settings.video.tnr_frames);
-#endif
-		ia_css_frame_free_multiple(MAX_NUM_VIDEO_DELAY_FRAMES, pipe->pipe_settings.video.delay_frames);
-		break;
-	case IA_CSS_PIPE_MODE_CAPTURE:
-		ia_css_frame_free_multiple(MAX_NUM_VIDEO_DELAY_FRAMES, pipe->pipe_settings.capture.delay_frames);
-		break;
-	case IA_CSS_PIPE_MODE_ACC:
-		sh_css_pipe_free_acc_binaries(pipe);
-		break;
-	case IA_CSS_PIPE_MODE_COPY:
-		break;
-	case IA_CSS_PIPE_MODE_YUVPP:
-		break;
-	}
-
-	sh_css_params_free_gdc_lut(pipe->scaler_pp_lut);
-	pipe->scaler_pp_lut = mmgr_NULL;
-
-	my_css.active_pipes[ia_css_pipe_get_pipe_num(pipe)] = NULL;
-	sh_css_pipe_free_shading_table(pipe);
-
-	ia_css_pipeline_destroy(&pipe->pipeline);
-	pipe_release_pipe_num(ia_css_pipe_get_pipe_num(pipe));
-
-	/* Temporarily, not every sh_css_pipe has an acc_extension. */
-	if (pipe->config.acc_extension) {
-		ia_css_pipe_unload_extension(pipe, pipe->config.acc_extension);
-	}
-	kfree(pipe);
-	IA_CSS_LEAVE("err = %d", err);
-	return err;
-}
-
-void
-ia_css_uninit(void)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_uninit() enter: void\n");
-#if WITH_PC_MONITORING
-	sh_css_print("PC_MONITORING: %s() -- started\n", __func__);
-	print_pc_histogram();
-#endif
-
-	sh_css_params_free_default_gdc_lut();
-
-
-	/* TODO: JB: implement decent check and handling of freeing mipi frames */
-	//assert(ref_count_mipi_allocation == 0); //mipi frames are not freed
-	/* cleanup generic data */
-	sh_css_params_uninit();
-	ia_css_refcount_uninit();
-
-	ia_css_rmgr_uninit();
-
-#if !defined(HAS_NO_INPUT_FORMATTER)
-	/* needed for reprogramming the inputformatter after power cycle of css */
-	ifmtr_set_if_blocking_mode_reset = true;
-#endif
-
-	if (!fw_explicitly_loaded) {
-		ia_css_unload_firmware();
-	}
-	ia_css_spctrl_unload_fw(SP0_ID);
-	sh_css_sp_set_sp_running(false);
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	/* check and free any remaining mipi frames */
-	free_mipi_frames(NULL);
-#endif
-
-	sh_css_sp_reset_global_vars();
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	ia_css_isys_uninit();
-#endif
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_uninit() leave: return_void\n");
-}
-
-#if defined(HAS_IRQ_MAP_VERSION_2)
-enum ia_css_err ia_css_irq_translate(
-	unsigned int *irq_infos)
-{
-	virq_id_t	irq;
-	enum hrt_isp_css_irq_status status = hrt_isp_css_irq_status_more_irqs;
-	unsigned int infos = 0;
-
-/* irq_infos can be NULL, but that would make the function useless */
-/* assert(irq_infos != NULL); */
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_irq_translate() enter: irq_infos=%p\n",irq_infos);
-
-	while (status == hrt_isp_css_irq_status_more_irqs) {
-		status = virq_get_channel_id(&irq);
-		if (status == hrt_isp_css_irq_status_error)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-
-#if WITH_PC_MONITORING
-		sh_css_print("PC_MONITORING: %s() irq = %d, "
-			     "sh_binary_running set to 0\n", __func__, irq);
-		sh_binary_running = 0 ;
-#endif
-
-		switch (irq) {
-		case virq_sp:
-			/* When SP goes to idle, info is available in the
-			 * event queue. */
-			infos |= IA_CSS_IRQ_INFO_EVENTS_READY;
-			break;
-		case virq_isp:
-			break;
-#if !defined(HAS_NO_INPUT_SYSTEM)
-		case virq_isys_sof:
-			infos |= IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF;
-			break;
-		case virq_isys_eof:
-			infos |= IA_CSS_IRQ_INFO_CSS_RECEIVER_EOF;
-			break;
-		case virq_isys_csi:
-			infos |= IA_CSS_IRQ_INFO_INPUT_SYSTEM_ERROR;
-			break;
-#endif
-#if !defined(HAS_NO_INPUT_FORMATTER)
-		case virq_ifmt0_id:
-			infos |= IA_CSS_IRQ_INFO_IF_ERROR;
-			break;
-#endif
-		case virq_dma:
-			infos |= IA_CSS_IRQ_INFO_DMA_ERROR;
-			break;
-		case virq_sw_pin_0:
-			infos |= sh_css_get_sw_interrupt_value(0);
-			break;
-		case virq_sw_pin_1:
-			infos |= sh_css_get_sw_interrupt_value(1);
-			/* pqiao TODO: also assumption here */
-			break;
-		default:
-			break;
-		}
-	}
-
-	if (irq_infos)
-		*irq_infos = infos;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_irq_translate() "
-		"leave: irq_infos=%u\n", infos);
-
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err ia_css_irq_enable(
-	enum ia_css_irq_info info,
-	bool enable)
-{
-	virq_id_t	irq = N_virq_id;
-	IA_CSS_ENTER("info=%d, enable=%d", info, enable);
-
-	switch (info) {
-#if !defined(HAS_NO_INPUT_FORMATTER)
-	case IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF:
-		irq = virq_isys_sof;
-		break;
-	case IA_CSS_IRQ_INFO_CSS_RECEIVER_EOF:
-		irq = virq_isys_eof;
-		break;
-	case IA_CSS_IRQ_INFO_INPUT_SYSTEM_ERROR:
-		irq = virq_isys_csi;
-		break;
-#endif
-#if !defined(HAS_NO_INPUT_FORMATTER)
-	case IA_CSS_IRQ_INFO_IF_ERROR:
-		irq = virq_ifmt0_id;
-		break;
-#endif
-	case IA_CSS_IRQ_INFO_DMA_ERROR:
-		irq = virq_dma;
-		break;
-	case IA_CSS_IRQ_INFO_SW_0:
-		irq = virq_sw_pin_0;
-		break;
-	case IA_CSS_IRQ_INFO_SW_1:
-		irq = virq_sw_pin_1;
-		break;
-	default:
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	cnd_virq_enable_channel(irq, enable);
-
-	IA_CSS_LEAVE_ERR(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-#else
-#error "sh_css.c: IRQ MAP must be one of \
-	{IRQ_MAP_VERSION_2}"
-#endif
-
-static unsigned int
-sh_css_get_sw_interrupt_value(unsigned int irq)
-{
-	unsigned int irq_value;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_get_sw_interrupt_value() enter: irq=%d\n",irq);
-	irq_value = sh_css_sp_get_sw_interrupt_value(irq);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_get_sw_interrupt_value() leave: irq_value=%d\n",irq_value);
-	return irq_value;
-}
-
-/* configure and load the copy binary, the next binary is used to
-   determine whether the copy binary needs to do left padding. */
-static enum ia_css_err load_copy_binary(
-	struct ia_css_pipe *pipe,
-	struct ia_css_binary *copy_binary,
-	struct ia_css_binary *next_binary)
-{
-	struct ia_css_frame_info copy_out_info, copy_in_info, copy_vf_info;
-	unsigned int left_padding;
-	enum ia_css_err err;
-	struct ia_css_binary_descr copy_descr;
-
-	/* next_binary can be NULL */
-	assert(pipe != NULL);
-	assert(copy_binary != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"load_copy_binary() enter:\n");
-
-	if (next_binary != NULL) {
-		copy_out_info = next_binary->in_frame_info;
-		left_padding = next_binary->left_padding;
-	} else {
-		copy_out_info = pipe->output_info[0];
-		copy_vf_info = pipe->vf_output_info[0];
-		ia_css_frame_info_set_format(&copy_vf_info, IA_CSS_FRAME_FORMAT_YUV_LINE);
-		left_padding = 0;
-	}
-
-	ia_css_pipe_get_copy_binarydesc(pipe, &copy_descr,
-		&copy_in_info, &copy_out_info, (next_binary != NULL) ? NULL : NULL/*TODO: &copy_vf_info*/);
-	err = ia_css_binary_find(&copy_descr, copy_binary);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	copy_binary->left_padding = left_padding;
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-alloc_continuous_frames(
-	struct ia_css_pipe *pipe, bool init_time)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_frame_info ref_info;
-	enum ia_css_pipe_id pipe_id;
-	bool continuous;
-	unsigned int i, idx;
-	unsigned int num_frames;
-	struct ia_css_pipe *capture_pipe = NULL;
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p, init_time = %d", pipe, init_time);
-
-	if ((pipe == NULL) || (pipe->stream == NULL)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	pipe_id = pipe->mode;
-	continuous = pipe->stream->config.continuous;
-
-	if (continuous) {
-		if (init_time) {
-			num_frames = pipe->stream->config.init_num_cont_raw_buf;
-			pipe->stream->continuous_pipe = pipe;
-		} else
-			num_frames = pipe->stream->config.target_num_cont_raw_buf;
-	} else {
-	    num_frames = NUM_ONLINE_INIT_CONTINUOUS_FRAMES;
-	}
-
-	if (pipe_id == IA_CSS_PIPE_ID_PREVIEW) {
-		ref_info = pipe->pipe_settings.preview.preview_binary.in_frame_info;
-	} else if (pipe_id == IA_CSS_PIPE_ID_VIDEO) {
-		ref_info = pipe->pipe_settings.video.video_binary.in_frame_info;
-	}
-	else {
-		/* should not happen */
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-	/* For CSI2+, the continuous frame will hold the full input frame */
-	ref_info.res.width = pipe->stream->config.input_config.input_res.width;
-	ref_info.res.height = pipe->stream->config.input_config.input_res.height;
-
-	/* Ensure padded width is aligned for 2401 */
-	ref_info.padded_width = CEIL_MUL(ref_info.res.width, 2 * ISP_VEC_NELEMS);
-#endif
-
-#if !defined(HAS_NO_PACKED_RAW_PIXELS)
-	if (pipe->stream->config.pack_raw_pixels) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-			"alloc_continuous_frames() IA_CSS_FRAME_FORMAT_RAW_PACKED\n");
-		ref_info.format = IA_CSS_FRAME_FORMAT_RAW_PACKED;
-	} else
-#endif
-	{
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-			"alloc_continuous_frames() IA_CSS_FRAME_FORMAT_RAW\n");
-		ref_info.format = IA_CSS_FRAME_FORMAT_RAW;
-	}
-
-	/* Write format back to binary */
-	if (pipe_id == IA_CSS_PIPE_ID_PREVIEW) {
-		pipe->pipe_settings.preview.preview_binary.in_frame_info.format = ref_info.format;
-		capture_pipe = pipe->pipe_settings.preview.capture_pipe;
-	} else if (pipe_id == IA_CSS_PIPE_ID_VIDEO) {
-		pipe->pipe_settings.video.video_binary.in_frame_info.format = ref_info.format;
-		capture_pipe = pipe->pipe_settings.video.capture_pipe;
-	} else {
-		/* should not happen */
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-	if (init_time)
-		idx = 0;
-	else
-		idx = pipe->stream->config.init_num_cont_raw_buf;
-
-	for (i = idx; i < NUM_CONTINUOUS_FRAMES; i++) {
-		/* free previous frame */
-		if (pipe->continuous_frames[i]) {
-			ia_css_frame_free(pipe->continuous_frames[i]);
-			pipe->continuous_frames[i] = NULL;
-		}
-		/* free previous metadata buffer */
-		ia_css_metadata_free(pipe->cont_md_buffers[i]);
-		pipe->cont_md_buffers[i] = NULL;
-
-		/* check if new frame needed */
-		if (i < num_frames) {
-			/* allocate new frame */
-			err = ia_css_frame_allocate_from_info(
-				&pipe->continuous_frames[i],
-				&ref_info);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-			/* allocate metadata buffer */
-			pipe->cont_md_buffers[i] = ia_css_metadata_allocate(
-					&pipe->stream->info.metadata_info);
-		}
-	}
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err
-ia_css_alloc_continuous_frame_remain(struct ia_css_stream *stream)
-{
-	if (stream == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	return alloc_continuous_frames(stream->continuous_pipe, false);
-}
-
-static enum ia_css_err
-load_preview_binaries(struct ia_css_pipe *pipe)
-{
-	struct ia_css_frame_info prev_in_info,
-				 prev_bds_out_info,
-				 prev_out_info,
-				 prev_vf_info;
-	struct ia_css_binary_descr preview_descr;
-	bool online;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	bool continuous, need_vf_pp = false;
-	bool need_isp_copy_binary = false;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	bool sensor = false;
-#endif
-	/* preview only have 1 output pin now */
-	struct ia_css_frame_info *pipe_out_info = &pipe->output_info[0];
-	struct ia_css_preview_settings *mycs  = &pipe->pipe_settings.preview;
-
-	IA_CSS_ENTER_PRIVATE("");
-	assert(pipe != NULL);
-	assert(pipe->stream != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_PREVIEW);
-
-	online = pipe->stream->config.online;
-	continuous = pipe->stream->config.continuous;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	sensor = pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR;
-#endif
-
-	if (mycs->preview_binary.info)
-		return IA_CSS_SUCCESS;
-
-	err = ia_css_util_check_input(&pipe->stream->config, false, false);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	err = ia_css_frame_check_info(pipe_out_info);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-	/* Note: the current selection of vf_pp binary and
-	 * parameterization of the preview binary contains a few pieces
-	 * of hardcoded knowledge. This needs to be cleaned up such that
-	 * the binary selection becomes more generic.
-	 * The vf_pp binary is needed if one or more of the following features
-	 * are required:
-	 * 1. YUV downscaling.
-	 * 2. Digital zoom.
-	 * 3. An output format that is not supported by the preview binary.
-	 *    In practice this means something other than yuv_line or nv12.
-	 * The decision if the vf_pp binary is needed for YUV downscaling is
-	 * made after the preview binary selection, since some preview binaries
-	 * can perform the requested YUV downscaling.
-	 * */
-	need_vf_pp = pipe->config.enable_dz;
-	need_vf_pp |= pipe_out_info->format != IA_CSS_FRAME_FORMAT_YUV_LINE &&
-		      !(pipe_out_info->format == IA_CSS_FRAME_FORMAT_NV12 ||
-			pipe_out_info->format == IA_CSS_FRAME_FORMAT_NV12_16 ||
-			pipe_out_info->format == IA_CSS_FRAME_FORMAT_NV12_TILEY);
-
-	/* Preview step 1 */
-	if (pipe->vf_yuv_ds_input_info.res.width)
-		prev_vf_info = pipe->vf_yuv_ds_input_info;
-	else
-		prev_vf_info = *pipe_out_info;
-	/* If vf_pp is needed, then preview must output yuv_line.
-	 * The exception is when vf_pp is manually disabled, that is only
-	 * used in combination with a pipeline extension that requires
-	 * yuv_line as input.
-	 * */
-	if (need_vf_pp)
-		ia_css_frame_info_set_format(&prev_vf_info,
-					     IA_CSS_FRAME_FORMAT_YUV_LINE);
-
-	err = ia_css_pipe_get_preview_binarydesc(
-			pipe,
-			&preview_descr,
-			&prev_in_info,
-			&prev_bds_out_info,
-			&prev_out_info,
-			&prev_vf_info);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	err = ia_css_binary_find(&preview_descr, &mycs->preview_binary);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-#ifdef ISP2401
-	/* The delay latency determines the number of invalid frames after
-	 * a stream is started. */
-	pipe->num_invalid_frames = pipe->dvs_frame_delay;
-	pipe->info.num_invalid_frames = pipe->num_invalid_frames;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"load_preview_binaries() num_invalid_frames=%d dvs_frame_delay=%d\n",
-		pipe->num_invalid_frames, pipe->dvs_frame_delay);
-
-#endif
-	/* The vf_pp binary is needed when (further) YUV downscaling is required */
-	need_vf_pp |= mycs->preview_binary.out_frame_info[0].res.width != pipe_out_info->res.width;
-	need_vf_pp |= mycs->preview_binary.out_frame_info[0].res.height != pipe_out_info->res.height;
-
-	/* When vf_pp is needed, then the output format of the selected
-	 * preview binary must be yuv_line. If this is not the case,
-	 * then the preview binary selection is done again.
-	 */
-	if (need_vf_pp &&
-		(mycs->preview_binary.out_frame_info[0].format != IA_CSS_FRAME_FORMAT_YUV_LINE)) {
-
-		/* Preview step 2 */
-		if (pipe->vf_yuv_ds_input_info.res.width)
-			prev_vf_info = pipe->vf_yuv_ds_input_info;
-		else
-			prev_vf_info = *pipe_out_info;
-
-		ia_css_frame_info_set_format(&prev_vf_info,
-			IA_CSS_FRAME_FORMAT_YUV_LINE);
-
-		err = ia_css_pipe_get_preview_binarydesc(
-				pipe,
-				&preview_descr,
-				&prev_in_info,
-				&prev_bds_out_info,
-				&prev_out_info,
-				&prev_vf_info);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-		err = ia_css_binary_find(&preview_descr,
-				&mycs->preview_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-
-	if (need_vf_pp) {
-		struct ia_css_binary_descr vf_pp_descr;
-
-		/* Viewfinder post-processing */
-		ia_css_pipe_get_vfpp_binarydesc(pipe, &vf_pp_descr,
-			&mycs->preview_binary.out_frame_info[0],
-			pipe_out_info);
-		err = ia_css_binary_find(&vf_pp_descr,
-				 &mycs->vf_pp_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	/* When the input system is 2401, only the Direct Sensor Mode
-	 * Offline Preview uses the ISP copy binary.
-	 */
-	need_isp_copy_binary = !online && sensor;
-#else
-#ifndef ISP2401
-	need_isp_copy_binary = !online && !continuous;
-#else
-	/* About pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY:
-	 * This is typical the case with SkyCam (which has no input system) but it also applies to all cases
-	 * where the driver chooses for memory based input frames. In these cases, a copy binary (which typical
-	 * copies sensor data to DDR) does not have much use.
-	 */
-	need_isp_copy_binary = !online && !continuous && !(pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY);
-#endif
-#endif
-
-	/* Copy */
-	if (need_isp_copy_binary) {
-		err = load_copy_binary(pipe,
-				       &mycs->copy_binary,
-				       &mycs->preview_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-
-	if (pipe->shading_table) {
-		ia_css_shading_table_free(pipe->shading_table);
-		pipe->shading_table = NULL;
-	}
-
-	return IA_CSS_SUCCESS;
-}
-
-static void
-ia_css_binary_unload(struct ia_css_binary *binary)
-{
-	ia_css_binary_destroy_isp_parameters(binary);
-}
-
-static enum ia_css_err
-unload_preview_binaries(struct ia_css_pipe *pipe)
-{
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-
-	if ((pipe == NULL) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	ia_css_binary_unload(&pipe->pipe_settings.preview.copy_binary);
-	ia_css_binary_unload(&pipe->pipe_settings.preview.preview_binary);
-	ia_css_binary_unload(&pipe->pipe_settings.preview.vf_pp_binary);
-
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-static const struct ia_css_fw_info *last_output_firmware(
-	const struct ia_css_fw_info *fw)
-{
-	const struct ia_css_fw_info *last_fw = NULL;
-/* fw can be NULL */
-	IA_CSS_ENTER_LEAVE_PRIVATE("");
-
-	for (; fw; fw = fw->next) {
-		const struct ia_css_fw_info *info = fw;
-		if (info->info.isp.sp.enable.output)
-			last_fw = fw;
-	}
-	return last_fw;
-}
-
-static enum ia_css_err add_firmwares(
-	struct ia_css_pipeline *me,
-	struct ia_css_binary *binary,
-	const struct ia_css_fw_info *fw,
-	const struct ia_css_fw_info *last_fw,
-	unsigned int binary_mode,
-	struct ia_css_frame *in_frame,
-	struct ia_css_frame *out_frame,
-	struct ia_css_frame *vf_frame,
-	struct ia_css_pipeline_stage **my_stage,
-	struct ia_css_pipeline_stage **vf_stage)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_pipeline_stage *extra_stage = NULL;
-	struct ia_css_pipeline_stage_desc stage_desc;
-
-/* all args can be NULL ??? */
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"add_firmwares() enter:\n");
-
-	for (; fw; fw = fw->next) {
-		struct ia_css_frame *out[IA_CSS_BINARY_MAX_OUTPUT_PORTS] = {NULL};
-		struct ia_css_frame *in = NULL;
-		struct ia_css_frame *vf = NULL;
-		if ((fw == last_fw) && (fw->info.isp.sp.enable.out_frame  != 0)) {
-			out[0] = out_frame;
-		}
-		if (fw->info.isp.sp.enable.in_frame != 0) {
-			in = in_frame;
-		}
-		if (fw->info.isp.sp.enable.out_frame != 0) {
-			vf = vf_frame;
-		}
-		ia_css_pipe_get_firmwares_stage_desc(&stage_desc, binary,
-			out, in, vf, fw, binary_mode);
-		err = ia_css_pipeline_create_and_add_stage(me,
-				&stage_desc,
-				&extra_stage);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-		if (fw->info.isp.sp.enable.output != 0)
-			in_frame = extra_stage->args.out_frame[0];
-		if (my_stage && !*my_stage && extra_stage)
-			*my_stage = extra_stage;
-		if (vf_stage && !*vf_stage && extra_stage &&
-		    fw->info.isp.sp.enable.vf_veceven)
-			*vf_stage = extra_stage;
-	}
-	return err;
-}
-
-static enum ia_css_err add_vf_pp_stage(
-	struct ia_css_pipe *pipe,
-	struct ia_css_frame *in_frame,
-	struct ia_css_frame *out_frame,
-	struct ia_css_binary *vf_pp_binary,
-	struct ia_css_pipeline_stage **vf_pp_stage)
-{
-
-	struct ia_css_pipeline *me = NULL;
-	const struct ia_css_fw_info *last_fw = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	struct ia_css_pipeline_stage_desc stage_desc;
-
-/* out_frame can be NULL ??? */
-
-	if (pipe == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	if (in_frame == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	if (vf_pp_binary == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	if (vf_pp_stage == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	ia_css_pipe_util_create_output_frames(out_frames);
-	me = &pipe->pipeline;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-	"add_vf_pp_stage() enter:\n");
-
-	*vf_pp_stage = NULL;
-
-	last_fw = last_output_firmware(pipe->vf_stage);
-	if (!pipe->extra_config.disable_vf_pp) {
-		if (last_fw) {
-			ia_css_pipe_util_set_output_frames(out_frames, 0, NULL);
-			ia_css_pipe_get_generic_stage_desc(&stage_desc, vf_pp_binary,
-				out_frames, in_frame, NULL);
-		} else{
-			ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame);
-			ia_css_pipe_get_generic_stage_desc(&stage_desc, vf_pp_binary,
-				out_frames, in_frame, NULL);
-		}
-		err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, vf_pp_stage);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-		in_frame = (*vf_pp_stage)->args.out_frame[0];
-	}
-	err = add_firmwares(me, vf_pp_binary, pipe->vf_stage, last_fw,
-			    IA_CSS_BINARY_MODE_VF_PP,
-			    in_frame, out_frame, NULL,
-			    vf_pp_stage, NULL);
-	return err;
-}
-
-static enum ia_css_err add_yuv_scaler_stage(
-	struct ia_css_pipe *pipe,
-	struct ia_css_pipeline *me,
-	struct ia_css_frame *in_frame,
-	struct ia_css_frame *out_frame,
-	struct ia_css_frame *internal_out_frame,
-	struct ia_css_binary *yuv_scaler_binary,
-	struct ia_css_pipeline_stage **pre_vf_pp_stage)
-{
-	const struct ia_css_fw_info *last_fw;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_frame *vf_frame = NULL;
-	struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	struct ia_css_pipeline_stage_desc stage_desc;
-
-	/* out_frame can be NULL ??? */
-	assert(in_frame != NULL);
-	assert(pipe != NULL);
-	assert(me != NULL);
-	assert(yuv_scaler_binary != NULL);
-	assert(pre_vf_pp_stage != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"add_yuv_scaler_stage() enter:\n");
-
-	*pre_vf_pp_stage = NULL;
-	ia_css_pipe_util_create_output_frames(out_frames);
-
-	last_fw = last_output_firmware(pipe->output_stage);
-
-	if(last_fw) {
-		ia_css_pipe_util_set_output_frames(out_frames, 0, NULL);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc,
-			yuv_scaler_binary, out_frames, in_frame, vf_frame);
-	} else {
-		ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame);
-		ia_css_pipe_util_set_output_frames(out_frames, 1, internal_out_frame);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc,
-			yuv_scaler_binary, out_frames, in_frame, vf_frame);
-	}
-	err = ia_css_pipeline_create_and_add_stage(me,
-		&stage_desc,
-		pre_vf_pp_stage);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	in_frame = (*pre_vf_pp_stage)->args.out_frame[0];
-
-	err = add_firmwares(me, yuv_scaler_binary, pipe->output_stage, last_fw,
-			    IA_CSS_BINARY_MODE_CAPTURE_PP,
-			    in_frame, out_frame, vf_frame,
-			    NULL, pre_vf_pp_stage);
-	/* If a firmware produce vf_pp output, we set that as vf_pp input */
-	(*pre_vf_pp_stage)->args.vf_downscale_log2 = yuv_scaler_binary->vf_downscale_log2;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"add_yuv_scaler_stage() leave:\n");
-	return err;
-}
-
-static enum ia_css_err add_capture_pp_stage(
-	struct ia_css_pipe *pipe,
-	struct ia_css_pipeline *me,
-	struct ia_css_frame *in_frame,
-	struct ia_css_frame *out_frame,
-	struct ia_css_binary *capture_pp_binary,
-	struct ia_css_pipeline_stage **capture_pp_stage)
-{
-	const struct ia_css_fw_info *last_fw = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_frame *vf_frame = NULL;
-	struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	struct ia_css_pipeline_stage_desc stage_desc;
-
-	/* out_frame can be NULL ??? */
-	assert(in_frame != NULL);
-	assert(pipe != NULL);
-	assert(me != NULL);
-	assert(capture_pp_binary != NULL);
-	assert(capture_pp_stage != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"add_capture_pp_stage() enter:\n");
-
-	*capture_pp_stage = NULL;
-	ia_css_pipe_util_create_output_frames(out_frames);
-
-	last_fw = last_output_firmware(pipe->output_stage);
-	err = ia_css_frame_allocate_from_info(&vf_frame,
-				    &capture_pp_binary->vf_frame_info);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	if(last_fw)	{
-		ia_css_pipe_util_set_output_frames(out_frames, 0, NULL);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc,
-			capture_pp_binary, out_frames, NULL, vf_frame);
-	} else {
-		ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc,
-			capture_pp_binary, out_frames, NULL, vf_frame);
-	}
-	err = ia_css_pipeline_create_and_add_stage(me,
-		&stage_desc,
-		capture_pp_stage);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	err = add_firmwares(me, capture_pp_binary, pipe->output_stage, last_fw,
-			    IA_CSS_BINARY_MODE_CAPTURE_PP,
-			    in_frame, out_frame, vf_frame,
-			    NULL, capture_pp_stage);
-	/* If a firmware produce vf_pp output, we set that as vf_pp input */
-	if (*capture_pp_stage) {
-		(*capture_pp_stage)->args.vf_downscale_log2 =
-		  capture_pp_binary->vf_downscale_log2;
-	}
-	return err;
-}
-
-static void sh_css_setup_queues(void)
-{
-	const struct ia_css_fw_info *fw;
-	unsigned int HIVE_ADDR_host_sp_queues_initialized;
-
-	sh_css_hmm_buffer_record_init();
-
-	sh_css_event_init_irq_mask();
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_host_sp_queues_initialized =
-		fw->info.sp.host_sp_queues_initialized;
-
-	ia_css_bufq_init();
-
-	/* set "host_sp_queues_initialized" to "true" */
-	sp_dmem_store_uint32(SP0_ID,
-		(unsigned int)sp_address_of(host_sp_queues_initialized),
-		(uint32_t)(1));
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_setup_queues() leave:\n");
-}
-
-static enum ia_css_err
-init_vf_frameinfo_defaults(struct ia_css_pipe *pipe,
-	struct ia_css_frame *vf_frame, unsigned int idx)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	unsigned int thread_id;
-	enum sh_css_queue_id queue_id;
-
-	assert(vf_frame != NULL);
-
-	sh_css_pipe_get_viewfinder_frame_info(pipe, &vf_frame->info, idx);
-	vf_frame->contiguous = false;
-	vf_frame->flash_state = IA_CSS_FRAME_FLASH_STATE_NONE;
-	ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-	ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME + idx, thread_id, &queue_id);
-	vf_frame->dynamic_queue_id = queue_id;
-	vf_frame->buf_type = IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME + idx;
-
-	err = ia_css_frame_init_planes(vf_frame);
-	return err;
-}
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-static unsigned int
-get_crop_lines_for_bayer_order (
-		const struct ia_css_stream_config *config)
-{
-	assert(config != NULL);
-	if ((IA_CSS_BAYER_ORDER_BGGR == config->input_config.bayer_order)
-	    || (IA_CSS_BAYER_ORDER_GBRG == config->input_config.bayer_order))
-		return 1;
-
-	return 0;
-}
-
-static unsigned int
-get_crop_columns_for_bayer_order (
-		const struct ia_css_stream_config *config)
-{
-	assert(config != NULL);
-	if ((IA_CSS_BAYER_ORDER_RGGB == config->input_config.bayer_order)
-	    || (IA_CSS_BAYER_ORDER_GBRG == config->input_config.bayer_order))
-		return 1;
-
-	return 0;
-}
-
-/* This function is to get the sum of all extra pixels in addition to the effective
- * input, it includes dvs envelop and filter run-in */
-static void get_pipe_extra_pixel(struct ia_css_pipe *pipe,
-		unsigned int *extra_row, unsigned int *extra_column)
-{
-	enum ia_css_pipe_id pipe_id = pipe->mode;
-	unsigned int left_cropping = 0, top_cropping = 0;
-	unsigned int i;
-	struct ia_css_resolution dvs_env = pipe->config.dvs_envelope;
-
-	/* The dvs envelope info may not be correctly sent down via pipe config
-	 * The check is made and the correct value is populated in the binary info
-	 * Use this value when computing crop, else excess lines may get trimmed
-	 */
-	switch (pipe_id) {
-	case IA_CSS_PIPE_ID_PREVIEW:
-		if (pipe->pipe_settings.preview.preview_binary.info) {
-			left_cropping = pipe->pipe_settings.preview.preview_binary.info->sp.pipeline.left_cropping;
-			top_cropping = pipe->pipe_settings.preview.preview_binary.info->sp.pipeline.top_cropping;
-		}
-		dvs_env = pipe->pipe_settings.preview.preview_binary.dvs_envelope;
-		break;
-	case IA_CSS_PIPE_ID_VIDEO:
-		if (pipe->pipe_settings.video.video_binary.info) {
-			left_cropping = pipe->pipe_settings.video.video_binary.info->sp.pipeline.left_cropping;
-			top_cropping = pipe->pipe_settings.video.video_binary.info->sp.pipeline.top_cropping;
-		}
-		dvs_env = pipe->pipe_settings.video.video_binary.dvs_envelope;
-		break;
-	case IA_CSS_PIPE_ID_CAPTURE:
-		for (i = 0; i < pipe->pipe_settings.capture.num_primary_stage; i++) {
-			if (pipe->pipe_settings.capture.primary_binary[i].info) {
-				left_cropping += pipe->pipe_settings.capture.primary_binary[i].info->sp.pipeline.left_cropping;
-				top_cropping += pipe->pipe_settings.capture.primary_binary[i].info->sp.pipeline.top_cropping;
-			}
-			dvs_env.width += pipe->pipe_settings.capture.primary_binary[i].dvs_envelope.width;
-			dvs_env.height += pipe->pipe_settings.capture.primary_binary[i].dvs_envelope.height;
-		}
-		break;
-	default:
-		break;
-	}
-
-	*extra_row = top_cropping + dvs_env.height;
-	*extra_column = left_cropping + dvs_env.width;
-}
-
-void
-ia_css_get_crop_offsets (
-    struct ia_css_pipe *pipe,
-    struct ia_css_frame_info *in_frame)
-{
-	unsigned int row = 0;
-	unsigned int column = 0;
-	struct ia_css_resolution *input_res;
-	struct ia_css_resolution *effective_res;
-	unsigned int extra_row = 0, extra_col = 0;
-	unsigned int min_reqd_height, min_reqd_width;
-
-	assert(pipe != NULL);
-	assert(pipe->stream != NULL);
-	assert(in_frame != NULL);
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p effective_wd = %u effective_ht = %u",
-		pipe, pipe->config.input_effective_res.width,
-		pipe->config.input_effective_res.height);
-
-	input_res = &pipe->stream->config.input_config.input_res;
-#ifndef ISP2401
-	effective_res = &pipe->stream->config.input_config.effective_res;
-#else
-	effective_res = &pipe->config.input_effective_res;
-#endif
-
-	get_pipe_extra_pixel(pipe, &extra_row, &extra_col);
-
-	in_frame->raw_bayer_order = pipe->stream->config.input_config.bayer_order;
-
-	min_reqd_height = effective_res->height + extra_row;
-	min_reqd_width = effective_res->width + extra_col;
-
-	if (input_res->height > min_reqd_height) {
-		row = (input_res->height - min_reqd_height) / 2;
-		row &= ~0x1;
-	}
-	if (input_res->width > min_reqd_width) {
-		column = (input_res->width - min_reqd_width) / 2;
-		column &= ~0x1;
-	}
-
-	/*
-	 * TODO:
-	 * 1. Require the special support for RAW10 packed mode.
-	 * 2. Require the special support for the online use cases.
-	 */
-
-	/* ISP expects GRBG bayer order, we skip one line and/or one row
-	 * to correct in case the input bayer order is different.
-	 */
-	column += get_crop_columns_for_bayer_order(&pipe->stream->config);
-	row += get_crop_lines_for_bayer_order(&pipe->stream->config);
-
-	in_frame->crop_info.start_column = column;
-	in_frame->crop_info.start_line = row;
-
-	IA_CSS_LEAVE_PRIVATE("void start_col: %u start_row: %u", column, row);
-
-	return;
-}
-#endif
-
-static enum ia_css_err
-init_in_frameinfo_memory_defaults(struct ia_css_pipe *pipe,
-	struct ia_css_frame *frame, enum ia_css_frame_format format)
-{
-	struct ia_css_frame *in_frame;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	unsigned int thread_id;
-	enum sh_css_queue_id queue_id;
-
-	assert(frame != NULL);
-	in_frame = frame;
-
-	in_frame->info.format = format;
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	if (format == IA_CSS_FRAME_FORMAT_RAW)
-		in_frame->info.format = (pipe->stream->config.pack_raw_pixels) ?
-					IA_CSS_FRAME_FORMAT_RAW_PACKED : IA_CSS_FRAME_FORMAT_RAW;
-#endif
-
-
-	in_frame->info.res.width = pipe->stream->config.input_config.input_res.width;
-	in_frame->info.res.height = pipe->stream->config.input_config.input_res.height;
-	in_frame->info.raw_bit_depth =
-		ia_css_pipe_util_pipe_input_format_bpp(pipe);
-	ia_css_frame_info_set_width(&in_frame->info, pipe->stream->config.input_config.input_res.width, 0);
-	in_frame->contiguous = false;
-	in_frame->flash_state = IA_CSS_FRAME_FLASH_STATE_NONE;
-	ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-	ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_INPUT_FRAME, thread_id, &queue_id);
-	in_frame->dynamic_queue_id = queue_id;
-	in_frame->buf_type = IA_CSS_BUFFER_TYPE_INPUT_FRAME;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	ia_css_get_crop_offsets(pipe, &in_frame->info);
-#endif
-	err = ia_css_frame_init_planes(in_frame);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"init_in_frameinfo_memory_defaults() bayer_order = %d:\n", in_frame->info.raw_bayer_order);
-
-	return err;
-}
-
-static enum ia_css_err
-init_out_frameinfo_defaults(struct ia_css_pipe *pipe,
-	struct ia_css_frame *out_frame, unsigned int idx)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	unsigned int thread_id;
-	enum sh_css_queue_id queue_id;
-
-	assert(out_frame != NULL);
-
-	sh_css_pipe_get_output_frame_info(pipe, &out_frame->info, idx);
-	out_frame->contiguous = false;
-	out_frame->flash_state = IA_CSS_FRAME_FLASH_STATE_NONE;
-	ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-	ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_OUTPUT_FRAME + idx, thread_id, &queue_id);
-	out_frame->dynamic_queue_id = queue_id;
-	out_frame->buf_type = IA_CSS_BUFFER_TYPE_OUTPUT_FRAME + idx;
-	err = ia_css_frame_init_planes(out_frame);
-
-	return err;
-}
-
-/* Create stages for video pipe */
-static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe)
-{
-	struct ia_css_pipeline_stage_desc stage_desc;
-	struct ia_css_binary *copy_binary, *video_binary,
-			     *yuv_scaler_binary, *vf_pp_binary;
-	struct ia_css_pipeline_stage *copy_stage  = NULL;
-	struct ia_css_pipeline_stage *video_stage = NULL;
-	struct ia_css_pipeline_stage *yuv_scaler_stage  = NULL;
-	struct ia_css_pipeline_stage *vf_pp_stage = NULL;
-	struct ia_css_pipeline *me;
-	struct ia_css_frame *in_frame = NULL;
-	struct ia_css_frame *out_frame;
-	struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	struct ia_css_frame *vf_frame = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	bool need_copy   = false;
-	bool need_vf_pp  = false;
-	bool need_yuv_pp = false;
-	unsigned num_output_pins;
-	bool need_in_frameinfo_memory = false;
-
-	unsigned int i, num_yuv_scaler;
-	bool *is_output_stage = NULL;
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-	if ((pipe == NULL) || (pipe->stream == NULL) || (pipe->mode != IA_CSS_PIPE_ID_VIDEO)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	ia_css_pipe_util_create_output_frames(out_frames);
-	out_frame = &pipe->out_frame_struct;
-
-	/* pipeline already created as part of create_host_pipeline_structure */
-	me = &pipe->pipeline;
-	ia_css_pipeline_clean(me);
-
-	me->dvs_frame_delay = pipe->dvs_frame_delay;
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	/* When the input system is 2401, always enable 'in_frameinfo_memory'
-	 * except for the following: online or continuous
-	 */
-	need_in_frameinfo_memory = !(pipe->stream->config.online || pipe->stream->config.continuous);
-#else
-	/* Construct in_frame info (only in case we have dynamic input */
-	need_in_frameinfo_memory = pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY;
-#endif
-
-	/* Construct in_frame info (only in case we have dynamic input */
-	if (need_in_frameinfo_memory) {
-		in_frame = &pipe->in_frame_struct;
-		err = init_in_frameinfo_memory_defaults(pipe, in_frame, IA_CSS_FRAME_FORMAT_RAW);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-
-	out_frame->data = 0;
-	err = init_out_frameinfo_defaults(pipe, out_frame, 0);
-	if (err != IA_CSS_SUCCESS)
-		goto ERR;
-
-	if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) {
-		vf_frame = &pipe->vf_frame_struct;
-		vf_frame->data = 0;
-		err = init_vf_frameinfo_defaults(pipe, vf_frame, 0);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-
-	copy_binary  = &pipe->pipe_settings.video.copy_binary;
-	video_binary = &pipe->pipe_settings.video.video_binary;
-	vf_pp_binary = &pipe->pipe_settings.video.vf_pp_binary;
-	num_output_pins = video_binary->info->num_output_pins;
-
-	yuv_scaler_binary = pipe->pipe_settings.video.yuv_scaler_binary;
-	num_yuv_scaler  = pipe->pipe_settings.video.num_yuv_scaler;
-	is_output_stage = pipe->pipe_settings.video.is_output_stage;
-
-	need_copy   = (copy_binary != NULL && copy_binary->info != NULL);
-	need_vf_pp  = (vf_pp_binary != NULL && vf_pp_binary->info != NULL);
-	need_yuv_pp = (yuv_scaler_binary != NULL && yuv_scaler_binary->info != NULL);
-
-	if (need_copy) {
-		ia_css_pipe_util_set_output_frames(out_frames, 0, NULL);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary,
-			out_frames, NULL, NULL);
-		err = ia_css_pipeline_create_and_add_stage(me,
-			&stage_desc,
-			&copy_stage);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-		in_frame = me->stages->args.out_frame[0];
-	} else if (pipe->stream->config.continuous) {
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-		/* When continuous is enabled, configure in_frame with the
-		 * last pipe, which is the copy pipe.
-		 */
-		in_frame = pipe->stream->last_pipe->continuous_frames[0];
-#else
-		in_frame = pipe->continuous_frames[0];
-#endif
-	}
-
-	ia_css_pipe_util_set_output_frames(out_frames, 0, need_yuv_pp ? NULL : out_frame);
-
-	/* when the video binary supports a second output pin,
-	   it can directly produce the vf_frame.  */
-	if(need_vf_pp) {
-		ia_css_pipe_get_generic_stage_desc(&stage_desc, video_binary,
-			out_frames, in_frame, NULL);
-	} else {
-		ia_css_pipe_get_generic_stage_desc(&stage_desc, video_binary,
-			out_frames, in_frame, vf_frame);
-	}
-	err = ia_css_pipeline_create_and_add_stage(me,
-			&stage_desc,
-			&video_stage);
-	if (err != IA_CSS_SUCCESS)
-		goto ERR;
-
-	/* If we use copy iso video, the input must be yuv iso raw */
-	if(video_stage) {
-		video_stage->args.copy_vf =
-			video_binary->info->sp.pipeline.mode == IA_CSS_BINARY_MODE_COPY;
-		video_stage->args.copy_output = video_stage->args.copy_vf;
-	}
-
-	/* when the video binary supports only 1 output pin, vf_pp is needed to
-	produce the vf_frame.*/
-	if (need_vf_pp && video_stage) {
-		in_frame = video_stage->args.out_vf_frame;
-		err = add_vf_pp_stage(pipe, in_frame, vf_frame, vf_pp_binary,
-				      &vf_pp_stage);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-	if (video_stage) {
-		int frm;
-#ifndef ISP2401
-		for (frm = 0; frm < NUM_VIDEO_TNR_FRAMES; frm++) {
-#else
-		for (frm = 0; frm < NUM_TNR_FRAMES; frm++) {
-#endif
-			video_stage->args.tnr_frames[frm] =
-				pipe->pipe_settings.video.tnr_frames[frm];
-		}
-		for (frm = 0; frm < MAX_NUM_VIDEO_DELAY_FRAMES; frm++) {
-			video_stage->args.delay_frames[frm] =
-				pipe->pipe_settings.video.delay_frames[frm];
-		}
-	}
-
-	/* Append Extension on Video out, if enabled */
-	if (!need_vf_pp && video_stage && pipe->config.acc_extension &&
-		 (pipe->config.acc_extension->info.isp.type == IA_CSS_ACC_OUTPUT))
-	{
-		struct ia_css_frame *out = NULL;
-		struct ia_css_frame *in = NULL;
-
-		if ((pipe->config.acc_extension->info.isp.sp.enable.output) &&
-		    (pipe->config.acc_extension->info.isp.sp.enable.in_frame) &&
-		    (pipe->config.acc_extension->info.isp.sp.enable.out_frame)) {
-
-			/* In/Out Frame mapping to support output frame extension.*/
-			out = video_stage->args.out_frame[0];
-			err = ia_css_frame_allocate_from_info(&in, &(pipe->output_info[0]));
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-			video_stage->args.out_frame[0] = in;
-		}
-
-		err = add_firmwares( me, video_binary, pipe->output_stage,
-					last_output_firmware(pipe->output_stage),
-					IA_CSS_BINARY_MODE_VIDEO,
-					in, out, NULL, &video_stage, NULL);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-
-	if (need_yuv_pp && video_stage) {
-		struct ia_css_frame *tmp_in_frame = video_stage->args.out_frame[0];
-		struct ia_css_frame *tmp_out_frame = NULL;
-
-		for (i = 0; i < num_yuv_scaler; i++) {
-			if (is_output_stage[i] == true) {
-				tmp_out_frame = out_frame;
-			} else {
-				tmp_out_frame = NULL;
-			}
-			err = add_yuv_scaler_stage(pipe, me, tmp_in_frame, tmp_out_frame,
-						   NULL,
-						   &yuv_scaler_binary[i],
-						   &yuv_scaler_stage);
-
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-			/* we use output port 1 as internal output port */
-			if (yuv_scaler_stage)
-				tmp_in_frame = yuv_scaler_stage->args.out_frame[1];
-		}
-	}
-
-	pipe->pipeline.acquire_isp_each_stage = false;
-	ia_css_pipeline_finalize_stages(&pipe->pipeline, pipe->stream->config.continuous);
-
-ERR:
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static enum ia_css_err
-create_host_acc_pipeline(struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	const struct ia_css_fw_info *fw;
-	unsigned int i;
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-	if ((pipe == NULL) || (pipe->stream == NULL)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	pipe->pipeline.num_execs = pipe->config.acc_num_execs;
-	/* Reset pipe_qos_config to default disable all QOS extension stages */
-	if (pipe->config.acc_extension)
-	   pipe->pipeline.pipe_qos_config = 0;
-
-	fw = pipe->vf_stage;
-	for (i = 0; fw; fw = fw->next){
-		err = sh_css_pipeline_add_acc_stage(&pipe->pipeline, fw);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-
-	for (i=0; i<pipe->config.num_acc_stages; i++) {
-		struct ia_css_fw_info *fw = pipe->config.acc_stages[i];
-		err = sh_css_pipeline_add_acc_stage(&pipe->pipeline, fw);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-
-	ia_css_pipeline_finalize_stages(&pipe->pipeline, pipe->stream->config.continuous);
-
-ERR:
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-/* Create stages for preview */
-static enum ia_css_err
-create_host_preview_pipeline(struct ia_css_pipe *pipe)
-{
-	struct ia_css_pipeline_stage *copy_stage = NULL;
-	struct ia_css_pipeline_stage *preview_stage = NULL;
-	struct ia_css_pipeline_stage *vf_pp_stage = NULL;
-	struct ia_css_pipeline_stage_desc stage_desc;
-	struct ia_css_pipeline *me = NULL;
-	struct ia_css_binary *copy_binary, *preview_binary, *vf_pp_binary = NULL;
-	struct ia_css_frame *in_frame = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_frame *out_frame;
-	struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	bool need_in_frameinfo_memory = false;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	bool sensor = false;
-	bool buffered_sensor = false;
-	bool online = false;
-	bool continuous = false;
-#endif
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-	if ((pipe == NULL) || (pipe->stream == NULL) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-
-	ia_css_pipe_util_create_output_frames(out_frames);
-	/* pipeline already created as part of create_host_pipeline_structure */
-	me = &pipe->pipeline;
-	ia_css_pipeline_clean(me);
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	/* When the input system is 2401, always enable 'in_frameinfo_memory'
-	 * except for the following:
-	 * - Direct Sensor Mode Online Preview
-	 * - Buffered Sensor Mode Online Preview
-	 * - Direct Sensor Mode Continuous Preview
-	 * - Buffered Sensor Mode Continuous Preview
-	 */
-	sensor = (pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR);
-	buffered_sensor = (pipe->stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR);
-	online = pipe->stream->config.online;
-	continuous = pipe->stream->config.continuous;
-	need_in_frameinfo_memory =
-		!((sensor && (online || continuous)) || (buffered_sensor && (online || continuous)));
-#else
-	/* Construct in_frame info (only in case we have dynamic input */
-	need_in_frameinfo_memory = pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY;
-#endif
-	if (need_in_frameinfo_memory) {
-		err = init_in_frameinfo_memory_defaults(pipe, &me->in_frame, IA_CSS_FRAME_FORMAT_RAW);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-
-		in_frame = &me->in_frame;
-	} else {
-		in_frame = NULL;
-	}
-
-	err = init_out_frameinfo_defaults(pipe, &me->out_frame[0], 0);
-	if (err != IA_CSS_SUCCESS)
-		goto ERR;
-	out_frame = &me->out_frame[0];
-
-	copy_binary    = &pipe->pipe_settings.preview.copy_binary;
-	preview_binary = &pipe->pipe_settings.preview.preview_binary;
-	if (pipe->pipe_settings.preview.vf_pp_binary.info)
-		vf_pp_binary = &pipe->pipe_settings.preview.vf_pp_binary;
-
-	if (pipe->pipe_settings.preview.copy_binary.info) {
-		ia_css_pipe_util_set_output_frames(out_frames, 0, NULL);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary,
-			out_frames, NULL, NULL);
-		err = ia_css_pipeline_create_and_add_stage(me,
-				&stage_desc,
-				&copy_stage);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-		in_frame = me->stages->args.out_frame[0];
-#ifndef ISP2401
-	} else {
-#else
-	} else if (pipe->stream->config.continuous) {
-#endif
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-		/* When continuous is enabled, configure in_frame with the
-		 * last pipe, which is the copy pipe.
-		 */
-		if (continuous || !online){
-			in_frame = pipe->stream->last_pipe->continuous_frames[0];
-		}
-#else
-		in_frame = pipe->continuous_frames[0];
-#endif
-	}
-
-	if (vf_pp_binary) {
-		ia_css_pipe_util_set_output_frames(out_frames, 0, NULL);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc, preview_binary,
-			out_frames, in_frame, NULL);
-	} else {
-		ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc, preview_binary,
-			out_frames, in_frame, NULL);
-	}
-	err = ia_css_pipeline_create_and_add_stage(me,
-		&stage_desc,
-		&preview_stage);
-	if (err != IA_CSS_SUCCESS)
-		goto ERR;
-	/* If we use copy iso preview, the input must be yuv iso raw */
-	preview_stage->args.copy_vf =
-		preview_binary->info->sp.pipeline.mode == IA_CSS_BINARY_MODE_COPY;
-	preview_stage->args.copy_output = !preview_stage->args.copy_vf;
-	if (preview_stage->args.copy_vf && !preview_stage->args.out_vf_frame) {
-		/* in case of copy, use the vf frame as output frame */
-		preview_stage->args.out_vf_frame =
-			preview_stage->args.out_frame[0];
-	}
-	if (vf_pp_binary) {
-		if (preview_binary->info->sp.pipeline.mode == IA_CSS_BINARY_MODE_COPY)
-			in_frame = preview_stage->args.out_vf_frame;
-		else
-			in_frame = preview_stage->args.out_frame[0];
-		err = add_vf_pp_stage(pipe, in_frame, out_frame, vf_pp_binary,
-				&vf_pp_stage);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	}
-
-	pipe->pipeline.acquire_isp_each_stage = false;
-	ia_css_pipeline_finalize_stages(&pipe->pipeline, pipe->stream->config.continuous);
-
-ERR:
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static void send_raw_frames(struct ia_css_pipe *pipe)
-{
-	if (pipe->stream->config.continuous) {
-		unsigned int i;
-
-		sh_css_update_host2sp_cont_num_raw_frames
-			(pipe->stream->config.init_num_cont_raw_buf, true);
-		sh_css_update_host2sp_cont_num_raw_frames
-			(pipe->stream->config.target_num_cont_raw_buf, false);
-
-		/* Hand-over all the SP-internal buffers */
-		for (i = 0; i < pipe->stream->config.init_num_cont_raw_buf; i++) {
-			sh_css_update_host2sp_offline_frame(i,
-				pipe->continuous_frames[i], pipe->cont_md_buffers[i]);
-		}
-	}
-
-	return;
-}
-
-static enum ia_css_err
-preview_start(struct ia_css_pipe *pipe)
-{
-	struct ia_css_pipeline *me ;
-	struct ia_css_binary *copy_binary, *preview_binary, *vf_pp_binary = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_pipe *copy_pipe, *capture_pipe;
-	struct ia_css_pipe *acc_pipe;
-	enum sh_css_pipe_config_override copy_ovrd;
-	enum ia_css_input_mode preview_pipe_input_mode;
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-	if ((pipe == NULL) || (pipe->stream == NULL) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	me = &pipe->pipeline;
-
-	preview_pipe_input_mode = pipe->stream->config.mode;
-
-	copy_pipe    = pipe->pipe_settings.preview.copy_pipe;
-	capture_pipe = pipe->pipe_settings.preview.capture_pipe;
-	acc_pipe     = pipe->pipe_settings.preview.acc_pipe;
-
-	copy_binary    = &pipe->pipe_settings.preview.copy_binary;
-	preview_binary = &pipe->pipe_settings.preview.preview_binary;
-	if (pipe->pipe_settings.preview.vf_pp_binary.info)
-		vf_pp_binary = &pipe->pipe_settings.preview.vf_pp_binary;
-
-	sh_css_metrics_start_frame();
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	/* multi stream video needs mipi buffers */
-	err = send_mipi_frames(pipe);
-	if (err != IA_CSS_SUCCESS)
-		goto ERR;
-#endif
-	send_raw_frames(pipe);
-
-	{
-		unsigned int thread_id;
-
-		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-		copy_ovrd = 1 << thread_id;
-
-		if (pipe->stream->cont_capt) {
-			ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(capture_pipe), &thread_id);
-			copy_ovrd |= 1 << thread_id;
-		}
-	}
-
-	/* Construct and load the copy pipe */
-	if (pipe->stream->config.continuous) {
-		sh_css_sp_init_pipeline(&copy_pipe->pipeline,
-			IA_CSS_PIPE_ID_COPY,
-			(uint8_t)ia_css_pipe_get_pipe_num(copy_pipe),
-			false,
-			pipe->stream->config.pixels_per_clock == 2, false,
-			false, pipe->required_bds_factor,
-			copy_ovrd,
-			pipe->stream->config.mode,
-			&pipe->stream->config.metadata_config,
-#ifndef ISP2401
-			&pipe->stream->info.metadata_info
-#else
-			&pipe->stream->info.metadata_info,
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#ifndef ISP2401
-			, pipe->stream->config.source.port.port
-#else
-			pipe->stream->config.source.port.port,
-#endif
-#endif
-#ifndef ISP2401
-			);
-#else
-			&pipe->config.internal_frame_origin_bqs_on_sctbl,
-			pipe->stream->isp_params_configs);
-#endif
-
-		/* make the preview pipe start with mem mode input, copy handles
-		   the actual mode */
-		preview_pipe_input_mode = IA_CSS_INPUT_MODE_MEMORY;
-	}
-
-	/* Construct and load the capture pipe */
-	if (pipe->stream->cont_capt) {
-		sh_css_sp_init_pipeline(&capture_pipe->pipeline,
-			IA_CSS_PIPE_ID_CAPTURE,
-			(uint8_t)ia_css_pipe_get_pipe_num(capture_pipe),
-			capture_pipe->config.default_capture_config.enable_xnr != 0,
-			capture_pipe->stream->config.pixels_per_clock == 2,
-			true, /* continuous */
-			false, /* offline */
-			capture_pipe->required_bds_factor,
-			0,
-			IA_CSS_INPUT_MODE_MEMORY,
-			&pipe->stream->config.metadata_config,
-#ifndef ISP2401
-			&pipe->stream->info.metadata_info
-#else
-			&pipe->stream->info.metadata_info,
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#ifndef ISP2401
-			, (enum mipi_port_id)0
-#else
-			(enum mipi_port_id)0,
-#endif
-#endif
-#ifndef ISP2401
-			);
-#else
-			&capture_pipe->config.internal_frame_origin_bqs_on_sctbl,
-			capture_pipe->stream->isp_params_configs);
-#endif
-	}
-
-	if (acc_pipe) {
-		sh_css_sp_init_pipeline(&acc_pipe->pipeline,
-			IA_CSS_PIPE_ID_ACC,
-			(uint8_t) ia_css_pipe_get_pipe_num(acc_pipe),
-			false,
-			pipe->stream->config.pixels_per_clock == 2,
-			false, /* continuous */
-			false, /* offline */
-			pipe->required_bds_factor,
-			0,
-			IA_CSS_INPUT_MODE_MEMORY,
-			NULL,
-#ifndef ISP2401
-			NULL
-#else
-			NULL,
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#ifndef ISP2401
-			, (enum mipi_port_id) 0
-#else
-			(enum mipi_port_id) 0,
-#endif
-#endif
-#ifndef ISP2401
-			);
-#else
-			&pipe->config.internal_frame_origin_bqs_on_sctbl,
-			pipe->stream->isp_params_configs);
-#endif
-	}
-
-	start_pipe(pipe, copy_ovrd, preview_pipe_input_mode);
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-ERR:
-#endif
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-enum ia_css_err
-ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
-			   const struct ia_css_buffer *buffer)
-{
-	enum ia_css_err return_err = IA_CSS_SUCCESS;
-	unsigned int thread_id;
-	enum sh_css_queue_id queue_id;
-	struct ia_css_pipeline *pipeline;
-	struct ia_css_pipeline_stage *stage;
-	struct ia_css_rmgr_vbuf_handle p_vbuf;
-	struct ia_css_rmgr_vbuf_handle *h_vbuf;
-	struct sh_css_hmm_buffer ddr_buffer;
-	enum ia_css_buffer_type buf_type;
-	enum ia_css_pipe_id pipe_id;
-	bool ret_err;
-
-	IA_CSS_ENTER("pipe=%p, buffer=%p", pipe, buffer);
-
-	if ((pipe == NULL) || (buffer == NULL)) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	buf_type = buffer->type;
-	/* following code will be enabled when IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME
-	   is removed */
-#if 0
-	if (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) {
-		bool found_pipe = false;
-		for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-			if ((buffer->data.frame->info.res.width == pipe->output_info[i].res.width) &&
-				(buffer->data.frame->info.res.height == pipe->output_info[i].res.height)) {
-				buf_type += i;
-				found_pipe = true;
-				break;
-			}
-		}
-		if (!found_pipe)
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	if (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) {
-		bool found_pipe = false;
-		for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-			if ((buffer->data.frame->info.res.width == pipe->vf_output_info[i].res.width) &&
-				(buffer->data.frame->info.res.height == pipe->vf_output_info[i].res.height)) {
-				buf_type += i;
-				found_pipe = true;
-				break;
-			}
-		}
-		if (!found_pipe)
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-#endif
-	pipe_id = pipe->mode;
-
-	IA_CSS_LOG("pipe_id=%d, buf_type=%d", pipe_id, buf_type);
-
-
-	assert(pipe_id < IA_CSS_PIPE_ID_NUM);
-	assert(buf_type < IA_CSS_NUM_DYNAMIC_BUFFER_TYPE);
-	if ((buf_type == IA_CSS_BUFFER_TYPE_INVALID) ||
-	    (buf_type >= IA_CSS_NUM_DYNAMIC_BUFFER_TYPE) ||
-	    (pipe_id >= IA_CSS_PIPE_ID_NUM)) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR);
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-	ret_err = ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-	if (!ret_err) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	ret_err = ia_css_query_internal_queue_id(buf_type, thread_id, &queue_id);
-	if (!ret_err) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if ((queue_id <= SH_CSS_INVALID_QUEUE_ID) || (queue_id >= SH_CSS_MAX_NUM_QUEUES)) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if (!sh_css_sp_is_running()) {
-		IA_CSS_LOG("SP is not running!");
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
-		/* SP is not running. The queues are not valid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-
-
-	pipeline = &pipe->pipeline;
-
-	assert(pipeline != NULL ||
-	       pipe_id == IA_CSS_PIPE_ID_COPY ||
-	       pipe_id == IA_CSS_PIPE_ID_ACC);
-
-	assert(sizeof(NULL) <= sizeof(ddr_buffer.kernel_ptr));
-	ddr_buffer.kernel_ptr = HOST_ADDRESS(NULL);
-	ddr_buffer.cookie_ptr = buffer->driver_cookie;
-	ddr_buffer.timing_data = buffer->timing_data;
-
-	if (buf_type == IA_CSS_BUFFER_TYPE_3A_STATISTICS) {
-		if (buffer->data.stats_3a == NULL) {
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-		}
-		ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.stats_3a);
-		ddr_buffer.payload.s3a = *buffer->data.stats_3a;
-	} else if (buf_type == IA_CSS_BUFFER_TYPE_DIS_STATISTICS) {
-		if (buffer->data.stats_dvs == NULL) {
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-		}
-		ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.stats_dvs);
-		ddr_buffer.payload.dis = *buffer->data.stats_dvs;
-	} else if (buf_type == IA_CSS_BUFFER_TYPE_METADATA) {
-		if (buffer->data.metadata == NULL) {
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-		}
-		ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.metadata);
-		ddr_buffer.payload.metadata = *buffer->data.metadata;
-	} else if ((buf_type == IA_CSS_BUFFER_TYPE_INPUT_FRAME)
-		|| (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME)
-		|| (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME)
-		|| (buf_type == IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME)
-		|| (buf_type == IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME)) {
-		if (buffer->data.frame == NULL) {
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-		}
-		ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.frame);
-		ddr_buffer.payload.frame.frame_data = buffer->data.frame->data;
-		ddr_buffer.payload.frame.flashed = 0;
-
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			"ia_css_pipe_enqueue_buffer() buf_type=%d, data(DDR address)=0x%x\n",
-			buf_type, buffer->data.frame->data);
-
-
-#if CONFIG_ON_FRAME_ENQUEUE()
-		return_err = set_config_on_frame_enqueue(
-				&buffer->data.frame->info,
-				&ddr_buffer.payload.frame);
-		if (IA_CSS_SUCCESS != return_err) {
-			IA_CSS_LEAVE_ERR(return_err);
-			return return_err;
-		}
-#endif
-	}
-
-	/* start of test for using rmgr for acq/rel memory */
-	p_vbuf.vptr = 0;
-	p_vbuf.count = 0;
-	p_vbuf.size = sizeof(struct sh_css_hmm_buffer);
-	h_vbuf = &p_vbuf;
-	/* TODO: change next to correct pool for optimization */
-	ia_css_rmgr_acq_vbuf(hmm_buffer_pool, &h_vbuf);
-
-	assert(h_vbuf != NULL);
-	assert(h_vbuf->vptr != 0x0);
-
-	if ((h_vbuf == NULL) || (h_vbuf->vptr == 0x0)) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR);
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-	mmgr_store(h_vbuf->vptr,
-				(void *)(&ddr_buffer),
-				sizeof(struct sh_css_hmm_buffer));
-	if ((buf_type == IA_CSS_BUFFER_TYPE_3A_STATISTICS)
-		|| (buf_type == IA_CSS_BUFFER_TYPE_DIS_STATISTICS)
-		|| (buf_type == IA_CSS_BUFFER_TYPE_LACE_STATISTICS)) {
-		if (pipeline == NULL) {
-			ia_css_rmgr_rel_vbuf(hmm_buffer_pool, &h_vbuf);
-			IA_CSS_LOG("pipeline is empty!");
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		}
-
-		for (stage = pipeline->stages; stage; stage = stage->next) {
-			/* The SP will read the params
-				after it got empty 3a and dis */
-			if (STATS_ENABLED(stage)) {
-				/* there is a stage that needs it */
-				return_err = ia_css_bufq_enqueue_buffer(thread_id,
-								queue_id,
-								(uint32_t)h_vbuf->vptr);
-			}
-		}
-	} else if ((buf_type == IA_CSS_BUFFER_TYPE_INPUT_FRAME)
-		   || (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME)
-		   || (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME)
-		   || (buf_type == IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME)
-		   || (buf_type == IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME)
-		   || (buf_type == IA_CSS_BUFFER_TYPE_METADATA)) {
-
-		return_err = ia_css_bufq_enqueue_buffer(thread_id,
-							queue_id,
-							(uint32_t)h_vbuf->vptr);
-#if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-		if ((return_err == IA_CSS_SUCCESS) && (IA_CSS_BUFFER_TYPE_OUTPUT_FRAME == buf_type)) {
-			IA_CSS_LOG("pfp: enqueued OF %d to q %d thread %d",
-				ddr_buffer.payload.frame.frame_data,
-				queue_id, thread_id);
-		}
-#endif
-
-	}
-
-	if (return_err == IA_CSS_SUCCESS) {
-		if (sh_css_hmm_buffer_record_acquire(
-				h_vbuf, buf_type,
-				HOST_ADDRESS(ddr_buffer.kernel_ptr))) {
-			IA_CSS_LOG("send vbuf=%p", h_vbuf);
-		} else {
-			return_err = IA_CSS_ERR_INTERNAL_ERROR;
-			IA_CSS_ERROR("hmm_buffer_record[]: no available slots\n");
-		}
-	}
-
-	/*
-	 * Tell the SP which queues are not empty,
-	 * by sending the software event.
-	 */
-	if (return_err == IA_CSS_SUCCESS) {
-		if (!sh_css_sp_is_running()) {
-			/* SP is not running. The queues are not valid */
-			IA_CSS_LOG("SP is not running!");
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
-			return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-		}
-		return_err = ia_css_bufq_enqueue_psys_event(
-				IA_CSS_PSYS_SW_EVENT_BUFFER_ENQUEUED,
-				(uint8_t)thread_id,
-				queue_id,
-				0);
-	} else {
-		ia_css_rmgr_rel_vbuf(hmm_buffer_pool, &h_vbuf);
-		IA_CSS_ERROR("buffer not enqueued");
-	}
-
-	IA_CSS_LEAVE("return value = %d", return_err);
-
-	return return_err;
-}
-
-/*
- * TODO: Free up the hmm memory space.
-	 */
-enum ia_css_err
-ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
-			   struct ia_css_buffer *buffer)
-{
-	enum ia_css_err return_err;
-	enum sh_css_queue_id queue_id;
-	hrt_vaddress ddr_buffer_addr = (hrt_vaddress)0;
-	struct sh_css_hmm_buffer ddr_buffer;
-	enum ia_css_buffer_type buf_type;
-	enum ia_css_pipe_id pipe_id;
-	unsigned int thread_id;
-	hrt_address kernel_ptr = 0;
-	bool ret_err;
-
-	IA_CSS_ENTER("pipe=%p, buffer=%p", pipe, buffer);
-
-	if ((pipe == NULL) || (buffer == NULL)) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	pipe_id = pipe->mode;
-
-	buf_type = buffer->type;
-
-	IA_CSS_LOG("pipe_id=%d, buf_type=%d", pipe_id, buf_type);
-
-	ddr_buffer.kernel_ptr = 0;
-
-	ret_err = ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-	if (!ret_err) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	ret_err = ia_css_query_internal_queue_id(buf_type, thread_id, &queue_id);
-	if (!ret_err) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if ((queue_id <= SH_CSS_INVALID_QUEUE_ID) || (queue_id >= SH_CSS_MAX_NUM_QUEUES)) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if (!sh_css_sp_is_running()) {
-		IA_CSS_LOG("SP is not running!");
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
-		/* SP is not running. The queues are not valid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-
-	return_err = ia_css_bufq_dequeue_buffer(queue_id,
-		(uint32_t *)&ddr_buffer_addr);
-
-	if (return_err == IA_CSS_SUCCESS) {
-		struct ia_css_frame *frame;
-		struct sh_css_hmm_buffer_record *hmm_buffer_record = NULL;
-
-		IA_CSS_LOG("receive vbuf=%x", (int)ddr_buffer_addr);
-
-		/* Validate the ddr_buffer_addr and buf_type */
-		hmm_buffer_record = sh_css_hmm_buffer_record_validate(
-				ddr_buffer_addr, buf_type);
-		if (hmm_buffer_record != NULL) {
-			/* valid hmm_buffer_record found. Save the kernel_ptr
-			 * for validation after performing mmgr_load.  The
-			 * vbuf handle and buffer_record can be released.
-			 */
-			kernel_ptr = hmm_buffer_record->kernel_ptr;
-			ia_css_rmgr_rel_vbuf(hmm_buffer_pool, &hmm_buffer_record->h_vbuf);
-			sh_css_hmm_buffer_record_reset(hmm_buffer_record);
-		} else {
-			IA_CSS_ERROR("hmm_buffer_record not found (0x%u) buf_type(%d)",
-				 ddr_buffer_addr, buf_type);
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		}
-
-		mmgr_load(ddr_buffer_addr,
-				&ddr_buffer,
-				sizeof(struct sh_css_hmm_buffer));
-
-		/* if the kernel_ptr is 0 or an invalid, return an error.
-		 * do not access the buffer via the kernal_ptr.
-		 */
-		if ((ddr_buffer.kernel_ptr == 0) ||
-		    (kernel_ptr != HOST_ADDRESS(ddr_buffer.kernel_ptr))) {
-			IA_CSS_ERROR("kernel_ptr invalid");
-			IA_CSS_ERROR("expected: (0x%llx)", (u64)kernel_ptr);
-			IA_CSS_ERROR("actual: (0x%llx)", (u64)HOST_ADDRESS(ddr_buffer.kernel_ptr));
-			IA_CSS_ERROR("buf_type: %d\n", buf_type);
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		}
-
-		if (ddr_buffer.kernel_ptr != 0) {
-			/* buffer->exp_id : all instances to be removed later once the driver change
-			 * is completed. See patch #5758 for reference */
-			buffer->exp_id = 0;
-			buffer->driver_cookie = ddr_buffer.cookie_ptr;
-			buffer->timing_data = ddr_buffer.timing_data;
-
-			if ((buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) ||
-				(buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME)) {
-				buffer->isys_eof_clock_tick.ticks = ddr_buffer.isys_eof_clock_tick;
-			}
-
-			switch (buf_type) {
-			case IA_CSS_BUFFER_TYPE_INPUT_FRAME:
-			case IA_CSS_BUFFER_TYPE_OUTPUT_FRAME:
-			case IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
-				if ((pipe) && (pipe->stop_requested == true))
-				{
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-					/* free mipi frames only for old input system
-					 * for 2401 it is done in ia_css_stream_destroy call
-					 */
-					return_err = free_mipi_frames(pipe);
-					if (return_err != IA_CSS_SUCCESS) {
-						IA_CSS_LOG("free_mipi_frames() failed");
-						IA_CSS_LEAVE_ERR(return_err);
-						return return_err;
-					}
-#endif
-					pipe->stop_requested = false;
-				}
-			case IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME:
-			case IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME:
-				frame = (struct ia_css_frame*)HOST_ADDRESS(ddr_buffer.kernel_ptr);
-				buffer->data.frame = frame;
-				buffer->exp_id = ddr_buffer.payload.frame.exp_id;
-				frame->exp_id = ddr_buffer.payload.frame.exp_id;
-				frame->isp_config_id = ddr_buffer.payload.frame.isp_parameters_id;
-				if (ddr_buffer.payload.frame.flashed == 1)
-					frame->flash_state =
-						IA_CSS_FRAME_FLASH_STATE_PARTIAL;
-				if (ddr_buffer.payload.frame.flashed == 2)
-					frame->flash_state =
-						IA_CSS_FRAME_FLASH_STATE_FULL;
-				frame->valid = pipe->num_invalid_frames == 0;
-				if (!frame->valid)
-					pipe->num_invalid_frames--;
-
-				if (frame->info.format == IA_CSS_FRAME_FORMAT_BINARY_8) {
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-					frame->planes.binary.size = frame->data_bytes;
-#else
-					frame->planes.binary.size =
-					    sh_css_sp_get_binary_copy_size();
-#endif
-				}
-#if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-				if (IA_CSS_BUFFER_TYPE_OUTPUT_FRAME == buf_type) {
-					IA_CSS_LOG("pfp: dequeued OF %d with config id %d thread %d",
-						frame->data, frame->isp_config_id, thread_id);
-				}
-#endif
-
-				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-					"ia_css_pipe_dequeue_buffer() buf_type=%d, data(DDR address)=0x%x\n",
-					buf_type, buffer->data.frame->data);
-
-				break;
-			case IA_CSS_BUFFER_TYPE_3A_STATISTICS:
-				buffer->data.stats_3a =
-					(struct ia_css_isp_3a_statistics*)HOST_ADDRESS(ddr_buffer.kernel_ptr);
-				buffer->exp_id = ddr_buffer.payload.s3a.exp_id;
-				buffer->data.stats_3a->exp_id = ddr_buffer.payload.s3a.exp_id;
-				buffer->data.stats_3a->isp_config_id = ddr_buffer.payload.s3a.isp_config_id;
-				break;
-			case IA_CSS_BUFFER_TYPE_DIS_STATISTICS:
-				buffer->data.stats_dvs =
-					(struct ia_css_isp_dvs_statistics*)
-						HOST_ADDRESS(ddr_buffer.kernel_ptr);
-				buffer->exp_id = ddr_buffer.payload.dis.exp_id;
-				buffer->data.stats_dvs->exp_id = ddr_buffer.payload.dis.exp_id;
-				break;
-			case IA_CSS_BUFFER_TYPE_LACE_STATISTICS:
-				break;
-			case IA_CSS_BUFFER_TYPE_METADATA:
-				buffer->data.metadata =
-					(struct ia_css_metadata*)HOST_ADDRESS(ddr_buffer.kernel_ptr);
-				buffer->exp_id = ddr_buffer.payload.metadata.exp_id;
-				buffer->data.metadata->exp_id = ddr_buffer.payload.metadata.exp_id;
-				break;
-			default:
-				return_err = IA_CSS_ERR_INTERNAL_ERROR;
-				break;
-			}
-		}
-	}
-
-	/*
-	 * Tell the SP which queues are not full,
-	 * by sending the software event.
-	 */
-	if (return_err == IA_CSS_SUCCESS){
-		if (!sh_css_sp_is_running()) {
-			IA_CSS_LOG("SP is not running!");
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
-			/* SP is not running. The queues are not valid */
-			return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-		}
-		ia_css_bufq_enqueue_psys_event(
-					IA_CSS_PSYS_SW_EVENT_BUFFER_DEQUEUED,
-					0,
-					queue_id,
-					0);
-	}
-	IA_CSS_LEAVE("buffer=%p", buffer);
-
-	return return_err;
-}
-
-/*
- * Cannot Move this to event module as it is of ia_css_event_type which is declared in ia_css.h
- * TODO: modify and move it if possible.
- *
- * !!!IMPORTANT!!! KEEP THE FOLLOWING IN SYNC:
- * 1) "enum ia_css_event_type"					(ia_css_event_public.h)
- * 2) "enum sh_css_sp_event_type"				(sh_css_internal.h)
- * 3) "enum ia_css_event_type event_id_2_event_mask"		(event_handler.sp.c)
- * 4) "enum ia_css_event_type convert_event_sp_to_host_domain"	(sh_css.c)
- */
-static enum ia_css_event_type convert_event_sp_to_host_domain[] = {
-	IA_CSS_EVENT_TYPE_OUTPUT_FRAME_DONE,	/** Output frame ready. */
-	IA_CSS_EVENT_TYPE_SECOND_OUTPUT_FRAME_DONE,	/** Second output frame ready. */
-	IA_CSS_EVENT_TYPE_VF_OUTPUT_FRAME_DONE,	/** Viewfinder Output frame ready. */
-	IA_CSS_EVENT_TYPE_SECOND_VF_OUTPUT_FRAME_DONE,	/** Second viewfinder Output frame ready. */
-	IA_CSS_EVENT_TYPE_3A_STATISTICS_DONE,	/** Indication that 3A statistics are available. */
-	IA_CSS_EVENT_TYPE_DIS_STATISTICS_DONE,	/** Indication that DIS statistics are available. */
-	IA_CSS_EVENT_TYPE_PIPELINE_DONE,	/** Pipeline Done event, sent after last pipeline stage. */
-	IA_CSS_EVENT_TYPE_FRAME_TAGGED,		/** Frame tagged. */
-	IA_CSS_EVENT_TYPE_INPUT_FRAME_DONE,	/** Input frame ready. */
-	IA_CSS_EVENT_TYPE_METADATA_DONE,	/** Metadata ready. */
-	IA_CSS_EVENT_TYPE_LACE_STATISTICS_DONE,	/** Indication that LACE statistics are available. */
-	IA_CSS_EVENT_TYPE_ACC_STAGE_COMPLETE,	/** Extension stage executed. */
-	IA_CSS_EVENT_TYPE_TIMER,		/** Timing measurement data. */
-	IA_CSS_EVENT_TYPE_PORT_EOF,		/** End Of Frame event, sent when in buffered sensor mode. */
-	IA_CSS_EVENT_TYPE_FW_WARNING,		/** Performance warning encountered by FW */
-	IA_CSS_EVENT_TYPE_FW_ASSERT,		/** Assertion hit by FW */
-	0,					/* error if sp passes  SH_CSS_SP_EVENT_NR_OF_TYPES as a valid event. */
-};
-
-enum ia_css_err
-ia_css_dequeue_event(struct ia_css_event *event)
-{
-	return ia_css_dequeue_psys_event(event);
-}
-
-enum ia_css_err
-ia_css_dequeue_psys_event(struct ia_css_event *event)
-{
-	enum ia_css_pipe_id pipe_id = 0;
-	uint8_t payload[4] = {0,0,0,0};
-	enum ia_css_err ret_err;
-
-	/*TODO:
-	 * a) use generic decoding function , same as the one used by sp.
-	 * b) group decode and dequeue into eventQueue module
-	 *
-	 * We skip the IA_CSS_ENTER logging call
-	 * to avoid flooding the logs when the host application
-	 * uses polling. */
-	if (event == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	if (!sh_css_sp_is_running()) {
-		/* SP is not running. The queues are not valid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-
-	/* dequeue the event (if any) from the psys event queue */
-	ret_err = ia_css_bufq_dequeue_psys_event(payload);
-	if (ret_err != IA_CSS_SUCCESS)
-		return ret_err;
-
-	IA_CSS_LOG("event dequeued from psys event queue");
-
-	/* Tell the SP that we dequeued an event from the event queue. */
-	ia_css_bufq_enqueue_psys_event(
-			IA_CSS_PSYS_SW_EVENT_EVENT_DEQUEUED, 0, 0, 0);
-
-	/* Events are decoded into 4 bytes of payload, the first byte
-	 * contains the sp event type. This is converted to a host enum.
-	 * TODO: can this enum conversion be eliminated */
-	event->type = convert_event_sp_to_host_domain[payload[0]];
-	/* Some sane default values since not all events use all fields. */
-	event->pipe = NULL;
-	event->port = MIPI_PORT0_ID;
-	event->exp_id = 0;
-	event->fw_warning = IA_CSS_FW_WARNING_NONE;
-	event->fw_handle = 0;
-	event->timer_data = 0;
-	event->timer_code = 0;
-	event->timer_subcode = 0;
-
-	if (event->type == IA_CSS_EVENT_TYPE_TIMER) {
-		/* timer event ??? get the 2nd event and decode the data into the event struct */
-		uint32_t tmp_data;
-		/* 1st event: LSB 16-bit timer data and code */
-		event->timer_data = ((payload[1] & 0xFF) | ((payload[3] & 0xFF) << 8));
-		event->timer_code = payload[2];
-		payload[0] = payload[1] = payload[2] = payload[3] = 0;
-		ret_err = ia_css_bufq_dequeue_psys_event(payload);
-		if (ret_err != IA_CSS_SUCCESS) {
-			/* no 2nd event ??? an error */
-			/* Putting IA_CSS_ERROR is resulting in failures in
-			 * Merrifield smoke testing  */
-			IA_CSS_WARNING("Timer: Error de-queuing the 2nd TIMER event!!!\n");
-			return ret_err;
-		}
-		ia_css_bufq_enqueue_psys_event(
-			IA_CSS_PSYS_SW_EVENT_EVENT_DEQUEUED, 0, 0, 0);
-		event->type = convert_event_sp_to_host_domain[payload[0]];
-		 /* It's a timer */
-		if (event->type == IA_CSS_EVENT_TYPE_TIMER) {
-			/* 2nd event data: MSB 16-bit timer and subcode */
-			tmp_data = ((payload[1] & 0xFF) | ((payload[3] & 0xFF) << 8));
-			event->timer_data |= (tmp_data << 16);
-			event->timer_subcode = payload[2];
-		}
-		/* It's a non timer event. So clear first half of the timer event data.
-		* If the second part of the TIMER event is not received, we discard
-		* the first half of the timer data and process the non timer event without
-		* affecting the flow. So the non timer event falls through
-		* the code. */
-		else {
-			event->timer_data = 0;
-			event->timer_code = 0;
-			event->timer_subcode = 0;
-			IA_CSS_ERROR("Missing 2nd timer event. Timer event discarded");
-		}
-	}
-	if (event->type == IA_CSS_EVENT_TYPE_PORT_EOF) {
-		event->port = (enum mipi_port_id)payload[1];
-		event->exp_id = payload[3];
-	} else if (event->type == IA_CSS_EVENT_TYPE_FW_WARNING) {
-		event->fw_warning = (enum ia_css_fw_warning)payload[1];
-		/* exp_id is only available in these warning types */
-		if (event->fw_warning == IA_CSS_FW_WARNING_EXP_ID_LOCKED ||
-		    event->fw_warning == IA_CSS_FW_WARNING_TAG_EXP_ID_FAILED)
-			event->exp_id = payload[3];
-	} else if (event->type == IA_CSS_EVENT_TYPE_FW_ASSERT) {
-		event->fw_assert_module_id = payload[1]; /* module */
-		event->fw_assert_line_no = (payload[2] << 8) + payload[3];
-		/* payload[2] is line_no>>8, payload[3] is line_no&0xff */
-	} else if (event->type != IA_CSS_EVENT_TYPE_TIMER) {
-		/* pipe related events.
-		 * payload[1] contains the pipe_num,
-		 * payload[2] contains the pipe_id. These are different. */
-		event->pipe = find_pipe_by_num(payload[1]);
-		pipe_id = (enum ia_css_pipe_id)payload[2];
-		/* Check to see if pipe still exists */
-		if (!event->pipe)
-			return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-
-		if (event->type == IA_CSS_EVENT_TYPE_FRAME_TAGGED) {
-			/* find the capture pipe that goes with this */
-			int i, n;
-			n = event->pipe->stream->num_pipes;
-			for (i = 0; i < n; i++) {
-				struct ia_css_pipe *p =
-					event->pipe->stream->pipes[i];
-				if (p->config.mode == IA_CSS_PIPE_MODE_CAPTURE) {
-					event->pipe = p;
-					break;
-				}
-			}
-			event->exp_id = payload[3];
-		}
-		if (event->type == IA_CSS_EVENT_TYPE_ACC_STAGE_COMPLETE) {
-			/* payload[3] contains the acc fw handle. */
-			uint32_t stage_num = (uint32_t)payload[3];
-			ret_err = ia_css_pipeline_get_fw_from_stage(
-					&(event->pipe->pipeline),
-					stage_num,
-					&(event->fw_handle));
-			if (ret_err != IA_CSS_SUCCESS) {
-				IA_CSS_ERROR("Invalid stage num received for ACC event. stage_num:%u",
-					     stage_num);
-				return ret_err;
-			}
-		}
-	}
-
-	if (event->pipe)
-		IA_CSS_LEAVE("event_id=%d, pipe_id=%d", event->type, pipe_id);
-	else
-		IA_CSS_LEAVE("event_id=%d", event->type);
-
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err
-ia_css_dequeue_isys_event(struct ia_css_event *event)
-{
-	uint8_t payload[4] = {0, 0, 0, 0};
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	/* We skip the IA_CSS_ENTER logging call
-	 * to avoid flooding the logs when the host application
-	 * uses polling. */
-	if (event == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	if (!sh_css_sp_is_running()) {
-		/* SP is not running. The queues are not valid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-
-	err = ia_css_bufq_dequeue_isys_event(payload);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-	IA_CSS_LOG("event dequeued from isys event queue");
-
-	/* Update SP state to indicate that element was dequeued. */
-	ia_css_bufq_enqueue_isys_event(IA_CSS_ISYS_SW_EVENT_EVENT_DEQUEUED);
-
-	/* Fill return struct with appropriate info */
-	event->type = IA_CSS_EVENT_TYPE_PORT_EOF;
-	/* EOF events are associated with a CSI port, not with a pipe */
-	event->pipe = NULL;
-	event->port = payload[1];
-	event->exp_id = payload[3];
-
-	IA_CSS_LEAVE_ERR(err);
-	return err;
-}
-
-static void
-acc_start(struct ia_css_pipe *pipe)
-{
-	assert(pipe != NULL);
-	assert(pipe->stream != NULL);
-
-	start_pipe(pipe, SH_CSS_PIPE_CONFIG_OVRD_NO_OVRD,
-			pipe->stream->config.mode);
-}
-
-static enum ia_css_err
-sh_css_pipe_start(struct ia_css_stream *stream)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	struct ia_css_pipe *pipe;
-	enum ia_css_pipe_id pipe_id;
-	unsigned int thread_id;
-
-	IA_CSS_ENTER_PRIVATE("stream = %p", stream);
-
-	if (stream == NULL) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	pipe = stream->last_pipe;
-	if (pipe == NULL) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	pipe_id = pipe->mode;
-
-	if(stream->started == true) {
-		IA_CSS_WARNING("Cannot start stream that is already started");
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-
-	pipe->stop_requested = false;
-
-	switch (pipe_id) {
-	case IA_CSS_PIPE_ID_PREVIEW:
-		err = preview_start(pipe);
-		break;
-	case IA_CSS_PIPE_ID_VIDEO:
-		err = video_start(pipe);
-		break;
-	case IA_CSS_PIPE_ID_CAPTURE:
-		err = capture_start(pipe);
-		break;
-	case IA_CSS_PIPE_ID_YUVPP:
-		err = yuvpp_start(pipe);
-		break;
-	case IA_CSS_PIPE_ID_ACC:
-		acc_start(pipe);
-		break;
-	default:
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	/* DH regular multi pipe - not continuous mode: start the next pipes too */
-	if (!stream->config.continuous) {
-		int i;
-		for (i = 1; i < stream->num_pipes && IA_CSS_SUCCESS == err ; i++) {
-			switch (stream->pipes[i]->mode) {
-			case IA_CSS_PIPE_ID_PREVIEW:
-				stream->pipes[i]->stop_requested = false;
-				err = preview_start(stream->pipes[i]);
-				break;
-			case IA_CSS_PIPE_ID_VIDEO:
-				stream->pipes[i]->stop_requested = false;
-				err = video_start(stream->pipes[i]);
-				break;
-			case IA_CSS_PIPE_ID_CAPTURE:
-				stream->pipes[i]->stop_requested = false;
-				err = capture_start(stream->pipes[i]);
-				break;
-			case IA_CSS_PIPE_ID_YUVPP:
-				stream->pipes[i]->stop_requested = false;
-				err = yuvpp_start(stream->pipes[i]);
-				break;
-			case IA_CSS_PIPE_ID_ACC:
-				stream->pipes[i]->stop_requested = false;
-				acc_start(stream->pipes[i]);
-				break;
-			default:
-				err = IA_CSS_ERR_INVALID_ARGUMENTS;
-			}
-		}
-	}
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-
-	/* Force ISP parameter calculation after a mode change
-	 * Acceleration API examples pass NULL for stream but they
-	 * don't use ISP parameters anyway. So this should be okay.
-	 * The SP binary (jpeg) copy does not use any parameters.
-	 */
-	if (!copy_on_sp(pipe)) {
-		sh_css_invalidate_params(stream);
-		err = sh_css_param_update_isp_params(pipe,
-							stream->isp_params_configs, true, NULL);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-
-	ia_css_debug_pipe_graph_dump_epilogue();
-
-	ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-
-	if (!sh_css_sp_is_running()) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
-		/* SP is not running. The queues are not valid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-	ia_css_bufq_enqueue_psys_event(IA_CSS_PSYS_SW_EVENT_START_STREAM,
-				       (uint8_t)thread_id, 0, 0);
-
-	/* DH regular multi pipe - not continuous mode: enqueue event to the next pipes too */
-	if (!stream->config.continuous) {
-		int i;
-		for (i = 1; i < stream->num_pipes; i++) {
-			ia_css_pipeline_get_sp_thread_id(
-							ia_css_pipe_get_pipe_num(stream->pipes[i]),
-							&thread_id);
-			ia_css_bufq_enqueue_psys_event(
-				IA_CSS_PSYS_SW_EVENT_START_STREAM,
-				(uint8_t)thread_id, 0, 0);
-		}
-	}
-
-	/* in case of continuous capture mode, we also start capture thread and copy thread*/
-	if (pipe->stream->config.continuous) {
-		struct ia_css_pipe *copy_pipe = NULL;
-
-		if (pipe_id == IA_CSS_PIPE_ID_PREVIEW)
-			copy_pipe = pipe->pipe_settings.preview.copy_pipe;
-		else if (pipe_id == IA_CSS_PIPE_ID_VIDEO)
-			copy_pipe = pipe->pipe_settings.video.copy_pipe;
-
-		if (copy_pipe == NULL) {
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		}
-		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(copy_pipe), &thread_id);
-		 /* by the time we reach here q is initialized and handle is available.*/
-		ia_css_bufq_enqueue_psys_event(
-				IA_CSS_PSYS_SW_EVENT_START_STREAM,
-				(uint8_t)thread_id, 0,  0);
-	}
-	if (pipe->stream->cont_capt) {
-		struct ia_css_pipe *capture_pipe = NULL;
-		if (pipe_id == IA_CSS_PIPE_ID_PREVIEW)
-			capture_pipe = pipe->pipe_settings.preview.capture_pipe;
-		else if (pipe_id == IA_CSS_PIPE_ID_VIDEO)
-			capture_pipe = pipe->pipe_settings.video.capture_pipe;
-
-		if (capture_pipe == NULL) {
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		}
-		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(capture_pipe), &thread_id);
-		 /* by the time we reach here q is initialized and handle is available.*/
-		ia_css_bufq_enqueue_psys_event(
-				IA_CSS_PSYS_SW_EVENT_START_STREAM,
-				(uint8_t)thread_id, 0,  0);
-	}
-
-	/* in case of PREVIEW mode, check whether QOS acc_pipe is available, then start the qos pipe */
-	if (pipe_id == IA_CSS_PIPE_ID_PREVIEW) {
-		struct ia_css_pipe *acc_pipe = NULL;
-		acc_pipe = pipe->pipe_settings.preview.acc_pipe;
-
-		if (acc_pipe){
-			ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(acc_pipe), &thread_id);
-			/* by the time we reach here q is initialized and handle is available.*/
-			ia_css_bufq_enqueue_psys_event(
-				IA_CSS_PSYS_SW_EVENT_START_STREAM,
-				(uint8_t) thread_id, 0, 0);
-		}
-	}
-
-	stream->started = true;
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-#ifndef ISP2401
-void
-sh_css_enable_cont_capt(bool enable, bool stop_copy_preview)
-{
-       ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-              "sh_css_enable_cont_capt() enter: enable=%d\n", enable);
-//my_css.cont_capt = enable;
-       my_css.stop_copy_preview = stop_copy_preview;
-}
-
-bool
-sh_css_continuous_is_enabled(uint8_t pipe_num)
-#else
-/*
- * @brief Stop all "ia_css_pipe" instances in the target
- * "ia_css_stream" instance.
- *
- * Refer to "Local prototypes" for more info.
- */
-static enum ia_css_err
-sh_css_pipes_stop(struct ia_css_stream *stream)
-#endif
-{
-#ifndef ISP2401
-	struct ia_css_pipe *pipe;
-	bool continuous;
-#else
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_pipe *main_pipe;
-	enum ia_css_pipe_id main_pipe_id;
-	int i;
-#endif
-
-#ifndef ISP2401
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_continuous_is_enabled() enter: pipe_num=%d\n", pipe_num);
-#else
-	assert(stream != NULL);
-	if (stream == NULL) {
-		IA_CSS_LOG("stream does NOT exist!");
-		err = IA_CSS_ERR_INTERNAL_ERROR;
-		goto ERR;
-	}
-#endif
-
-#ifndef ISP2401
-	pipe = find_pipe_by_num(pipe_num);
-	continuous = pipe && pipe->stream->config.continuous;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"sh_css_continuous_is_enabled() leave: enable=%d\n",
-		continuous);
-	return continuous;
-}
-#else
-	main_pipe = stream->last_pipe;
-	assert(main_pipe != NULL);
-	if (main_pipe == NULL) {
-		IA_CSS_LOG("main_pipe does NOT exist!");
-		err = IA_CSS_ERR_INTERNAL_ERROR;
-		goto ERR;
-	}
-#endif
-
-#ifndef ISP2401
-enum ia_css_err
-ia_css_stream_get_max_buffer_depth(struct ia_css_stream *stream, int *buffer_depth)
-{
-	if (buffer_depth == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_max_buffer_depth() enter: void\n");
-	(void)stream;
-	*buffer_depth = NUM_CONTINUOUS_FRAMES;
-	return IA_CSS_SUCCESS;
-}
-#else
-	main_pipe_id = main_pipe->mode;
-	IA_CSS_ENTER_PRIVATE("main_pipe_id=%d", main_pipe_id);
-#endif
-
-#ifndef ISP2401
-enum ia_css_err
-ia_css_stream_set_buffer_depth(struct ia_css_stream *stream, int buffer_depth)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_set_buffer_depth() enter: num_frames=%d\n",buffer_depth);
-	(void)stream;
-	if (buffer_depth > NUM_CONTINUOUS_FRAMES || buffer_depth < 1)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	/* ok, value allowed */
-	stream->config.target_num_cont_raw_buf = buffer_depth;
-	/* TODO: check what to regarding initialization */
-	return IA_CSS_SUCCESS;
-}
-#else
-	/*
-	 * Stop all "ia_css_pipe" instances in this target
-	 * "ia_css_stream" instance.
-	 */
-	for (i = 0; i < stream->num_pipes; i++) {
-		/* send the "stop" request to the "ia_css_pipe" instance */
-		IA_CSS_LOG("Send the stop-request to the pipe: pipe_id=%d",
-				stream->pipes[i]->pipeline.pipe_id);
-		err = ia_css_pipeline_request_stop(&stream->pipes[i]->pipeline);
-#endif
-
-#ifndef ISP2401
-enum ia_css_err
-ia_css_stream_get_buffer_depth(struct ia_css_stream *stream, int *buffer_depth)
-{
-	if (buffer_depth == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_buffer_depth() enter: void\n");
-#else
-		/*
-		 * Exit this loop if "ia_css_pipeline_request_stop()"
-		 * returns the error code.
-		 *
-		 * The error code would be generated in the following
-		 * two cases:
-		 * (1) The Scalar Processor has already been stopped.
-		 * (2) The "Host->SP" event queue is full.
-		 *
-		 * As the convention of using CSS API 2.0/2.1, such CSS
-		 * error code would be propogated from the CSS-internal
-		 * API returned value to the CSS API returned value. Then
-		 * the CSS driver should capture these error code and
-		 * handle it in the driver exception handling mechanism.
-		 */
-		if (err != IA_CSS_SUCCESS) {
-			goto ERR;
-		}
-	}
-
-	/*
-	 * In the CSS firmware use scenario "Continuous Preview"
-	 * as well as "Continuous Video", the "ia_css_pipe" instance
-	 * "Copy Pipe" is activated. This "Copy Pipe" is private to
-	 * the CSS firmware so that it is not listed in the target
-	 * "ia_css_stream" instance.
-	 *
-	 * We need to stop this "Copy Pipe", as well.
-	 */
-	if (main_pipe->stream->config.continuous) {
-		struct ia_css_pipe *copy_pipe = NULL;
-
-		/* get the reference to "Copy Pipe" */
-		if (main_pipe_id == IA_CSS_PIPE_ID_PREVIEW)
-			copy_pipe = main_pipe->pipe_settings.preview.copy_pipe;
-		else if (main_pipe_id == IA_CSS_PIPE_ID_VIDEO)
-			copy_pipe = main_pipe->pipe_settings.video.copy_pipe;
-
-		/* return the error code if "Copy Pipe" does NOT exist */
-		assert(copy_pipe != NULL);
-		if (copy_pipe == NULL) {
-			IA_CSS_LOG("Copy Pipe does NOT exist!");
-			err = IA_CSS_ERR_INTERNAL_ERROR;
-			goto ERR;
-		}
-
-		/* send the "stop" request to "Copy Pipe" */
-		IA_CSS_LOG("Send the stop-request to the pipe: pipe_id=%d",
-				copy_pipe->pipeline.pipe_id);
-		err = ia_css_pipeline_request_stop(&copy_pipe->pipeline);
-	}
-
-ERR:
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-/*
- * @brief Check if all "ia_css_pipe" instances in the target
- * "ia_css_stream" instance have stopped.
- *
- * Refer to "Local prototypes" for more info.
- */
-static bool
-sh_css_pipes_have_stopped(struct ia_css_stream *stream)
-{
-	bool rval = true;
-
-	struct ia_css_pipe *main_pipe;
-	enum ia_css_pipe_id main_pipe_id;
-
-	int i;
-
-	assert(stream != NULL);
-	if (stream == NULL) {
-		IA_CSS_LOG("stream does NOT exist!");
-		rval = false;
-		goto RET;
-	}
-
-	main_pipe = stream->last_pipe;
-	assert(main_pipe != NULL);
-
-	if (main_pipe == NULL) {
-		IA_CSS_LOG("main_pipe does NOT exist!");
-		rval = false;
-		goto RET;
-	}
-
-	main_pipe_id = main_pipe->mode;
-	IA_CSS_ENTER_PRIVATE("main_pipe_id=%d", main_pipe_id);
-
-	/*
-	 * Check if every "ia_css_pipe" instance in this target
-	 * "ia_css_stream" instance has stopped.
-	 */
-	for (i = 0; i < stream->num_pipes; i++) {
-		rval = rval && ia_css_pipeline_has_stopped(&stream->pipes[i]->pipeline);
-		IA_CSS_LOG("Pipe has stopped: pipe_id=%d, stopped=%d",
-				stream->pipes[i]->pipeline.pipe_id,
-				rval);
-	}
-
-	/*
-	 * In the CSS firmware use scenario "Continuous Preview"
-	 * as well as "Continuous Video", the "ia_css_pipe" instance
-	 * "Copy Pipe" is activated. This "Copy Pipe" is private to
-	 * the CSS firmware so that it is not listed in the target
-	 * "ia_css_stream" instance.
-	 *
-	 * We need to check if this "Copy Pipe" has stopped, as well.
-	 */
-	if (main_pipe->stream->config.continuous) {
-		struct ia_css_pipe *copy_pipe = NULL;
-
-		/* get the reference to "Copy Pipe" */
-		if (main_pipe_id == IA_CSS_PIPE_ID_PREVIEW)
-			copy_pipe = main_pipe->pipe_settings.preview.copy_pipe;
-		else if (main_pipe_id == IA_CSS_PIPE_ID_VIDEO)
-			copy_pipe = main_pipe->pipe_settings.video.copy_pipe;
-
-		/* return if "Copy Pipe" does NOT exist */
-		assert(copy_pipe != NULL);
-		if (copy_pipe == NULL) {
-			IA_CSS_LOG("Copy Pipe does NOT exist!");
-
-			rval = false;
-			goto RET;
-		}
-
-		/* check if "Copy Pipe" has stopped or not */
-		rval = rval && ia_css_pipeline_has_stopped(&copy_pipe->pipeline);
-		IA_CSS_LOG("Pipe has stopped: pipe_id=%d, stopped=%d",
-				copy_pipe->pipeline.pipe_id,
-				rval);
-	}
-
-RET:
-	IA_CSS_LEAVE_PRIVATE("rval=%d", rval);
-	return rval;
-}
-
-bool
-sh_css_continuous_is_enabled(uint8_t pipe_num)
-{
-	struct ia_css_pipe *pipe;
-	bool continuous;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_continuous_is_enabled() enter: pipe_num=%d\n", pipe_num);
-
-	pipe = find_pipe_by_num(pipe_num);
-	continuous = pipe && pipe->stream->config.continuous;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"sh_css_continuous_is_enabled() leave: enable=%d\n",
-		continuous);
-	return continuous;
-}
-
-enum ia_css_err
-ia_css_stream_get_max_buffer_depth(struct ia_css_stream *stream, int *buffer_depth)
-{
-	if (buffer_depth == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_max_buffer_depth() enter: void\n");
-	(void)stream;
-	*buffer_depth = NUM_CONTINUOUS_FRAMES;
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err
-ia_css_stream_set_buffer_depth(struct ia_css_stream *stream, int buffer_depth)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_set_buffer_depth() enter: num_frames=%d\n",buffer_depth);
-	(void)stream;
-	if (buffer_depth > NUM_CONTINUOUS_FRAMES || buffer_depth < 1)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	/* ok, value allowed */
-	stream->config.target_num_cont_raw_buf = buffer_depth;
-	/* TODO: check what to regarding initialization */
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err
-ia_css_stream_get_buffer_depth(struct ia_css_stream *stream, int *buffer_depth)
-{
-	if (buffer_depth == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_buffer_depth() enter: void\n");
-#endif
-	(void)stream;
-	*buffer_depth = stream->config.target_num_cont_raw_buf;
-	return IA_CSS_SUCCESS;
-}
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-unsigned int
-sh_css_get_mipi_sizes_for_check(const unsigned int port, const unsigned int idx)
-{
-	OP___assert(port < N_CSI_PORTS);
-	OP___assert(idx  < IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "sh_css_get_mipi_sizes_for_check(port %d, idx %d): %d\n",
-		port, idx, my_css.mipi_sizes_for_check[port][idx]);
-	return my_css.mipi_sizes_for_check[port][idx];
-}
-#endif
-
-static enum ia_css_err sh_css_pipe_configure_output(
-	struct ia_css_pipe *pipe,
-	unsigned int width,
-	unsigned int height,
-	unsigned int padded_width,
-	enum ia_css_frame_format format,
-	unsigned int idx)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p, width = %d, height = %d, paddaed width = %d, format = %d, idx = %d",
-			     pipe, width, height, padded_width, format, idx);
-	if (pipe == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	err = ia_css_util_check_res(width, height);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-	if (pipe->output_info[idx].res.width != width ||
-	    pipe->output_info[idx].res.height != height ||
-	    pipe->output_info[idx].format != format)
-	{
-		ia_css_frame_info_init(
-				&pipe->output_info[idx],
-				width,
-				height,
-				format,
-				padded_width);
-	}
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-sh_css_pipe_get_shading_info(struct ia_css_pipe *pipe,
-#ifndef ISP2401
-			     struct ia_css_shading_info *info)
-#else
-			     struct ia_css_shading_info *shading_info,
-			     struct ia_css_pipe_config *pipe_config)
-#endif
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_binary *binary = NULL;
-
-	assert(pipe != NULL);
-#ifndef ISP2401
-	assert(info != NULL);
-#else
-	assert(shading_info != NULL);
-	assert(pipe_config != NULL);
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"sh_css_pipe_get_shading_info() enter:\n");
-
-	binary = ia_css_pipe_get_shading_correction_binary(pipe);
-
-	if (binary) {
-		err = ia_css_binary_get_shading_info(binary,
-			IA_CSS_SHADING_CORRECTION_TYPE_1,
-			pipe->required_bds_factor,
-			(const struct ia_css_stream_config *)&pipe->stream->config,
-#ifndef ISP2401
-			info);
-#else
-			shading_info, pipe_config);
-#endif
-		/* Other function calls can be added here when other shading correction types will be added
-		 * in the future.
-		 */
-	} else {
-		/* When the pipe does not have a binary which has the shading
-		 * correction, this function does not need to fill the shading
-		 * information. It is not a error case, and then
-		 * this function should return IA_CSS_SUCCESS.
-		 */
-#ifndef ISP2401
-		memset(info, 0, sizeof(*info));
-#else
-		memset(shading_info, 0, sizeof(*shading_info));
-#endif
-	}
-	return err;
-}
-
-static enum ia_css_err
-sh_css_pipe_get_grid_info(struct ia_css_pipe *pipe,
-			  struct ia_css_grid_info *info)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_binary *binary = NULL;
-
-	assert(pipe != NULL);
-	assert(info != NULL);
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	binary = ia_css_pipe_get_s3a_binary(pipe);
-
-	if (binary) {
-		err = ia_css_binary_3a_grid_info(binary, info, pipe);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-	} else
-		memset(&info->s3a_grid, 0, sizeof(info->s3a_grid));
-
-	binary = ia_css_pipe_get_sdis_binary(pipe);
-
-	if (binary) {
-		ia_css_binary_dvs_grid_info(binary, info, pipe);
-		ia_css_binary_dvs_stat_grid_info(binary, info, pipe);
-	} else {
-		memset(&info->dvs_grid.dvs_grid_info, 0,
-			   sizeof(info->dvs_grid.dvs_grid_info));
-		memset(&info->dvs_grid.dvs_stat_grid_info, 0,
-			   sizeof(info->dvs_grid.dvs_stat_grid_info));
-	}
-
-	if (binary != NULL) {
-		/* copy pipe does not have ISP binary*/
-		info->isp_in_width = binary->internal_frame_info.res.width;
-		info->isp_in_height = binary->internal_frame_info.res.height;
-	}
-
-#if defined(HAS_VAMEM_VERSION_2)
-	info->vamem_type = IA_CSS_VAMEM_TYPE_2;
-#elif defined(HAS_VAMEM_VERSION_1)
-	info->vamem_type = IA_CSS_VAMEM_TYPE_1;
-#else
-#error "Unknown VAMEM version"
-#endif
-
-ERR:
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-#ifdef ISP2401
-/*
- * @brief Check if a format is supported by the pipe.
- *
- */
-static enum ia_css_err
-ia_css_pipe_check_format(struct ia_css_pipe *pipe, enum ia_css_frame_format format)
-{
-	const enum ia_css_frame_format *supported_formats;
-	int number_of_formats;
-	int found = 0;
-	int i;
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	if (NULL == pipe || NULL == pipe->pipe_settings.video.video_binary.info) {
-		IA_CSS_ERROR("Pipe or binary info is not set");
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	supported_formats = pipe->pipe_settings.video.video_binary.info->output_formats;
-	number_of_formats = sizeof(pipe->pipe_settings.video.video_binary.info->output_formats)/sizeof(enum ia_css_frame_format);
-
-	for (i = 0; i < number_of_formats && !found; i++) {
-		if (supported_formats[i] == format) {
-			found = 1;
-			break;
-		}
-	}
-	if (!found) {
-		IA_CSS_ERROR("Requested format is not supported by binary");
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	} else {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-		return IA_CSS_SUCCESS;
-	}
-}
-#endif
-
-static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
-{
-	struct ia_css_frame_info video_in_info, tnr_info,
-				 *video_vf_info, video_bds_out_info, *pipe_out_info, *pipe_vf_out_info;
-	bool online;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	bool continuous = pipe->stream->config.continuous;
-	unsigned int i;
-	unsigned num_output_pins;
-	struct ia_css_frame_info video_bin_out_info;
-	bool need_scaler = false;
-	bool vf_res_different_than_output = false;
-	bool need_vf_pp = false;
-	int vf_ds_log2;
-	struct ia_css_video_settings *mycs  = &pipe->pipe_settings.video;
-
-	IA_CSS_ENTER_PRIVATE("");
-	assert(pipe != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_VIDEO);
-	/* we only test the video_binary because offline video doesn't need a
-	 * vf_pp binary and online does not (always use) the copy_binary.
-	 * All are always reset at the same time anyway.
-	 */
-	if (mycs->video_binary.info)
-		return IA_CSS_SUCCESS;
-
-	online = pipe->stream->config.online;
-	pipe_out_info = &pipe->output_info[0];
-	pipe_vf_out_info = &pipe->vf_output_info[0];
-
-	assert(pipe_out_info != NULL);
-
-	/*
-	 * There is no explicit input format requirement for raw or yuv
-	 * What matters is that there is a binary that supports the stream format.
-	 * This is checked in the binary_find(), so no need to check it here
-	 */
-	err = ia_css_util_check_input(&pipe->stream->config, false, false);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	/* cannot have online video and input_mode memory */
-	if (online && pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) {
-		err = ia_css_util_check_vf_out_info(pipe_out_info,
-					pipe_vf_out_info);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	} else {
-		err = ia_css_frame_check_info(pipe_out_info);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-
-	if (pipe->out_yuv_ds_input_info.res.width)
-		video_bin_out_info = pipe->out_yuv_ds_input_info;
-	else
-		video_bin_out_info = *pipe_out_info;
-
-	/* Video */
-	if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]){
-		video_vf_info = pipe_vf_out_info;
-		vf_res_different_than_output = (video_vf_info->res.width != video_bin_out_info.res.width) ||
-						(video_vf_info->res.height != video_bin_out_info.res.height);
-	}
-	else {
-		video_vf_info = NULL;
-	}
-
-	need_scaler = need_downscaling(video_bin_out_info.res, pipe_out_info->res);
-
-	/* we build up the pipeline starting at the end */
-	/* YUV post-processing if needed */
-	if (need_scaler) {
-		struct ia_css_cas_binary_descr cas_scaler_descr = { };
-
-		/* NV12 is the common format that is supported by both */
-		/* yuv_scaler and the video_xx_isp2_min binaries. */
-		video_bin_out_info.format = IA_CSS_FRAME_FORMAT_NV12;
-
-		err = ia_css_pipe_create_cas_scaler_desc_single_output(
-			&video_bin_out_info,
-			pipe_out_info,
-			NULL,
-			&cas_scaler_descr);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-		mycs->num_yuv_scaler = cas_scaler_descr.num_stage;
-		mycs->yuv_scaler_binary = kzalloc(cas_scaler_descr.num_stage *
-			sizeof(struct ia_css_binary), GFP_KERNEL);
-		if (!mycs->yuv_scaler_binary) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-			return err;
-		}
-		mycs->is_output_stage = kzalloc(cas_scaler_descr.num_stage
-					* sizeof(bool), GFP_KERNEL);
-		if (!mycs->is_output_stage) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-			return err;
-		}
-		for (i = 0; i < cas_scaler_descr.num_stage; i++) {
-			struct ia_css_binary_descr yuv_scaler_descr;
-			mycs->is_output_stage[i] = cas_scaler_descr.is_output_stage[i];
-			ia_css_pipe_get_yuvscaler_binarydesc(pipe,
-				&yuv_scaler_descr, &cas_scaler_descr.in_info[i],
-				&cas_scaler_descr.out_info[i],
-				&cas_scaler_descr.internal_out_info[i],
-				&cas_scaler_descr.vf_info[i]);
-			err = ia_css_binary_find(&yuv_scaler_descr,
-						&mycs->yuv_scaler_binary[i]);
-			if (err != IA_CSS_SUCCESS) {
-				kfree(mycs->is_output_stage);
-				mycs->is_output_stage = NULL;
-				return err;
-			}
-		}
-		ia_css_pipe_destroy_cas_scaler_desc(&cas_scaler_descr);
-	}
-
-
-	{
-		struct ia_css_binary_descr video_descr;
-		enum ia_css_frame_format vf_info_format;
-
-		err = ia_css_pipe_get_video_binarydesc(pipe,
-			&video_descr, &video_in_info, &video_bds_out_info, &video_bin_out_info, video_vf_info,
-			pipe->stream->config.left_padding);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-
-		/* In the case where video_vf_info is not NULL, this allows
-		 * us to find a potential video library with desired vf format.
-		 * If success, no vf_pp binary is needed.
-		 * If failed, we will look up video binary with YUV_LINE vf format
-		 */
-		err = ia_css_binary_find(&video_descr,
-					 &mycs->video_binary);
-
-		if (err != IA_CSS_SUCCESS) {
-			if (video_vf_info) {
-				/* This will do another video binary lookup later for YUV_LINE format*/
-				need_vf_pp = true;
-			} else
-				return err;
-		} else if (video_vf_info) {
-			/* The first video binary lookup is successful, but we may
-			 * still need vf_pp binary based on additiona check */
-			num_output_pins = mycs->video_binary.info->num_output_pins;
-			vf_ds_log2 = mycs->video_binary.vf_downscale_log2;
-
-			/* If the binary has dual output pins, we need vf_pp if the resolution
-			* is different. */
-			need_vf_pp |= ((num_output_pins == 2) && vf_res_different_than_output);
-
-			/* If the binary has single output pin, we need vf_pp if additional
-			* scaling is needed for vf */
-			need_vf_pp |= ((num_output_pins == 1) &&
-				((video_vf_info->res.width << vf_ds_log2 != pipe_out_info->res.width) ||
-				(video_vf_info->res.height << vf_ds_log2 != pipe_out_info->res.height)));
-		}
-
-		if (need_vf_pp) {
-			/* save the current vf_info format for restoration later */
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"load_video_binaries() need_vf_pp; find video binary with YUV_LINE again\n");
-
-			vf_info_format = video_vf_info->format;
-
-			if (!pipe->config.enable_vfpp_bci)
-				ia_css_frame_info_set_format(video_vf_info,
-					IA_CSS_FRAME_FORMAT_YUV_LINE);
-
-			ia_css_binary_destroy_isp_parameters(&mycs->video_binary);
-
-			err = ia_css_binary_find(&video_descr,
-						&mycs->video_binary);
-
-			/* restore original vf_info format */
-			ia_css_frame_info_set_format(video_vf_info,
-					vf_info_format);
-			if (err != IA_CSS_SUCCESS)
-				return err;
-		}
-	}
-
-	/* If a video binary does not use a ref_frame, we set the frame delay
-	 * to 0. This is the case for the 1-stage low-power video binary. */
-	if (!mycs->video_binary.info->sp.enable.ref_frame)
-		pipe->dvs_frame_delay = 0;
-
-	/* The delay latency determines the number of invalid frames after
-	 * a stream is started. */
-	pipe->num_invalid_frames = pipe->dvs_frame_delay;
-	pipe->info.num_invalid_frames = pipe->num_invalid_frames;
-
-	/* Viewfinder frames also decrement num_invalid_frames. If the pipe
-	 * outputs a viewfinder output, then we need double the number of
-	 * invalid frames */
-	if (video_vf_info)
-		pipe->num_invalid_frames *= 2;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"load_video_binaries() num_invalid_frames=%d dvs_frame_delay=%d\n",
-		pipe->num_invalid_frames, pipe->dvs_frame_delay);
-
-/* pqiao TODO: temp hack for PO, should be removed after offline YUVPP is enabled */
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
-	/* Copy */
-	if (!online && !continuous) {
-		/* TODO: what exactly needs doing, prepend the copy binary to
-		 *	 video base this only on !online?
-		 */
-		err = load_copy_binary(pipe,
-				       &mycs->copy_binary,
-				       &mycs->video_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-#else
-	(void)continuous;
-#endif
-
-#if !defined(HAS_OUTPUT_SYSTEM)
-	if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0] && need_vf_pp) {
-		struct ia_css_binary_descr vf_pp_descr;
-
-		if (mycs->video_binary.vf_frame_info.format
-				== IA_CSS_FRAME_FORMAT_YUV_LINE) {
-			ia_css_pipe_get_vfpp_binarydesc(pipe, &vf_pp_descr,
-				&mycs->video_binary.vf_frame_info,
-				pipe_vf_out_info);
-		} else {
-			/* output from main binary is not yuv line. currently this is
-			 * possible only when bci is enabled on vfpp output */
-			assert(pipe->config.enable_vfpp_bci == true);
-			ia_css_pipe_get_yuvscaler_binarydesc(pipe, &vf_pp_descr,
-				&mycs->video_binary.vf_frame_info,
-				pipe_vf_out_info, NULL, NULL);
-		}
-
-		err = ia_css_binary_find(&vf_pp_descr,
-				&mycs->vf_pp_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-#endif
-
-	err = allocate_delay_frames(pipe);
-
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-	if (mycs->video_binary.info->sp.enable.block_output) {
-#ifdef ISP2401
-		unsigned int tnr_width;
-		unsigned int tnr_height;
-#endif
-		tnr_info = mycs->video_binary.out_frame_info[0];
-#ifdef ISP2401
-
-		/* Select resolution for TNR. If
-		 * output_system_in_resolution(GDC_out_resolution) is
-		 * being used, then select that as it will also be in resolution for
-		 * TNR. At present, it only make sense for Skycam */
-		if (pipe->config.output_system_in_res.width && pipe->config.output_system_in_res.height) {
-			tnr_width = pipe->config.output_system_in_res.width;
-			tnr_height = pipe->config.output_system_in_res.height;
-		} else {
-			tnr_width = tnr_info.res.width;
-			tnr_height = tnr_info.res.height;
-		}
-
-		/* Make tnr reference buffers output block width(in pix) align */
-		tnr_info.res.width  =
-			CEIL_MUL(tnr_width,
-			 (mycs->video_binary.info->sp.block.block_width * ISP_NWAY));
-		tnr_info.padded_width = tnr_info.res.width;
-
-#endif
-		/* Make tnr reference buffers output block height align */
-#ifndef ISP2401
-		tnr_info.res.height =
-			CEIL_MUL(tnr_info.res.height,
-				 mycs->video_binary.info->sp.block.output_block_height);
-#else
-		tnr_info.res.height =
-			CEIL_MUL(tnr_height,
-				 mycs->video_binary.info->sp.block.output_block_height);
-#endif
-	} else {
-		tnr_info = mycs->video_binary.internal_frame_info;
-	}
-	tnr_info.format = IA_CSS_FRAME_FORMAT_YUV_LINE;
-	tnr_info.raw_bit_depth = SH_CSS_TNR_BIT_DEPTH;
-
-#ifndef ISP2401
-	for (i = 0; i < NUM_VIDEO_TNR_FRAMES; i++) {
-#else
-	for (i = 0; i < NUM_TNR_FRAMES; i++) {
-#endif
-		if (mycs->tnr_frames[i]) {
-			ia_css_frame_free(mycs->tnr_frames[i]);
-			mycs->tnr_frames[i] = NULL;
-		}
-		err = ia_css_frame_allocate_from_info(
-				&mycs->tnr_frames[i],
-				&tnr_info);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-	IA_CSS_LEAVE_PRIVATE("");
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-unload_video_binaries(struct ia_css_pipe *pipe)
-{
-	unsigned int i;
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-
-	if ((pipe == NULL) || (pipe->mode != IA_CSS_PIPE_ID_VIDEO)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	ia_css_binary_unload(&pipe->pipe_settings.video.copy_binary);
-	ia_css_binary_unload(&pipe->pipe_settings.video.video_binary);
-	ia_css_binary_unload(&pipe->pipe_settings.video.vf_pp_binary);
-#ifndef ISP2401
-	ia_css_binary_unload(&pipe->pipe_settings.video.vf_pp_binary);
-#endif
-
-	for (i = 0; i < pipe->pipe_settings.video.num_yuv_scaler; i++)
-		ia_css_binary_unload(&pipe->pipe_settings.video.yuv_scaler_binary[i]);
-
-	kfree(pipe->pipe_settings.video.is_output_stage);
-	pipe->pipe_settings.video.is_output_stage = NULL;
-	kfree(pipe->pipe_settings.video.yuv_scaler_binary);
-	pipe->pipe_settings.video.yuv_scaler_binary = NULL;
-
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err video_start(struct ia_css_pipe *pipe)
-{
-	struct ia_css_binary *copy_binary;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_pipe *copy_pipe, *capture_pipe;
-	enum sh_css_pipe_config_override copy_ovrd;
-	enum ia_css_input_mode video_pipe_input_mode;
-
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-	if ((pipe == NULL) || (pipe->mode != IA_CSS_PIPE_ID_VIDEO)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	video_pipe_input_mode = pipe->stream->config.mode;
-
-	copy_pipe    = pipe->pipe_settings.video.copy_pipe;
-	capture_pipe = pipe->pipe_settings.video.capture_pipe;
-
-	copy_binary  = &pipe->pipe_settings.video.copy_binary;
-
-	sh_css_metrics_start_frame();
-
-	/* multi stream video needs mipi buffers */
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	err = send_mipi_frames(pipe);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-#endif
-
-	send_raw_frames(pipe);
-	{
-		unsigned int thread_id;
-
-		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-		copy_ovrd = 1 << thread_id;
-
-		if (pipe->stream->cont_capt) {
-			ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(capture_pipe), &thread_id);
-			copy_ovrd |= 1 << thread_id;
-		}
-	}
-
-	/* Construct and load the copy pipe */
-	if (pipe->stream->config.continuous) {
-		sh_css_sp_init_pipeline(&copy_pipe->pipeline,
-			IA_CSS_PIPE_ID_COPY,
-			(uint8_t)ia_css_pipe_get_pipe_num(copy_pipe),
-			false,
-			pipe->stream->config.pixels_per_clock == 2, false,
-			false, pipe->required_bds_factor,
-			copy_ovrd,
-			pipe->stream->config.mode,
-			&pipe->stream->config.metadata_config,
-#ifndef ISP2401
-			&pipe->stream->info.metadata_info
-#else
-			&pipe->stream->info.metadata_info,
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#ifndef ISP2401
-			, pipe->stream->config.source.port.port
-#else
-			pipe->stream->config.source.port.port,
-#endif
-#endif
-#ifndef ISP2401
-			);
-#else
-			&copy_pipe->config.internal_frame_origin_bqs_on_sctbl,
-			copy_pipe->stream->isp_params_configs);
-#endif
-
-		/* make the video pipe start with mem mode input, copy handles
-		   the actual mode */
-		video_pipe_input_mode = IA_CSS_INPUT_MODE_MEMORY;
-	}
-
-	/* Construct and load the capture pipe */
-	if (pipe->stream->cont_capt) {
-		sh_css_sp_init_pipeline(&capture_pipe->pipeline,
-			IA_CSS_PIPE_ID_CAPTURE,
-			(uint8_t)ia_css_pipe_get_pipe_num(capture_pipe),
-			capture_pipe->config.default_capture_config.enable_xnr != 0,
-			capture_pipe->stream->config.pixels_per_clock == 2,
-			true, /* continuous */
-			false, /* offline */
-			capture_pipe->required_bds_factor,
-			0,
-			IA_CSS_INPUT_MODE_MEMORY,
-			&pipe->stream->config.metadata_config,
-#ifndef ISP2401
-			&pipe->stream->info.metadata_info
-#else
-			&pipe->stream->info.metadata_info,
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#ifndef ISP2401
-			, (enum mipi_port_id)0
-#else
-			(enum mipi_port_id)0,
-#endif
-#endif
-#ifndef ISP2401
-			);
-#else
-			&capture_pipe->config.internal_frame_origin_bqs_on_sctbl,
-			capture_pipe->stream->isp_params_configs);
-#endif
-	}
-
-	start_pipe(pipe, copy_ovrd, video_pipe_input_mode);
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static
-enum ia_css_err sh_css_pipe_get_viewfinder_frame_info(
-	struct ia_css_pipe *pipe,
-	struct ia_css_frame_info *info,
-	unsigned int idx)
-{
-	assert(pipe != NULL);
-	assert(info != NULL);
-
-/* We could print the pointer as input arg, and the values as output */
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "sh_css_pipe_get_viewfinder_frame_info() enter: void\n");
-
-	if ( pipe->mode == IA_CSS_PIPE_ID_CAPTURE &&
-	    (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_RAW ||
-	     pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER))
-		return IA_CSS_ERR_MODE_HAS_NO_VIEWFINDER;
-	/* offline video does not generate viewfinder output */
-	*info = pipe->vf_output_info[idx];
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"sh_css_pipe_get_viewfinder_frame_info() leave: \
-		info.res.width=%d, info.res.height=%d, \
-		info.padded_width=%d, info.format=%d, \
-		info.raw_bit_depth=%d, info.raw_bayer_order=%d\n",
-		info->res.width,info->res.height,
-		info->padded_width,info->format,
-		info->raw_bit_depth,info->raw_bayer_order);
-
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-sh_css_pipe_configure_viewfinder(struct ia_css_pipe *pipe, unsigned int width,
-				 unsigned int height, unsigned int min_width,
-				 enum ia_css_frame_format format,
-				 unsigned int idx)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p, width = %d, height = %d, min_width = %d, format = %d, idx = %d\n",
-			     pipe, width, height, min_width, format, idx);
-
-	if (pipe == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-
-	err = ia_css_util_check_res(width, height);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-	if (pipe->vf_output_info[idx].res.width != width ||
-	    pipe->vf_output_info[idx].res.height != height ||
-	    pipe->vf_output_info[idx].format != format) {
-		ia_css_frame_info_init(&pipe->vf_output_info[idx], width, height,
-				       format, min_width);
-	}
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err load_copy_binaries(struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	assert(pipe != NULL);
-	IA_CSS_ENTER_PRIVATE("");
-
-	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || pipe->mode == IA_CSS_PIPE_ID_COPY);
-	if (pipe->pipe_settings.capture.copy_binary.info)
-		return IA_CSS_SUCCESS;
-
-	err = ia_css_frame_check_info(&pipe->output_info[0]);
-	if (err != IA_CSS_SUCCESS)
-		goto ERR;
-
-	err = verify_copy_out_frame_format(pipe);
-	if (err != IA_CSS_SUCCESS)
-		goto ERR;
-
-	err = load_copy_binary(pipe,
-				&pipe->pipe_settings.capture.copy_binary,
-				NULL);
-
-ERR:
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static bool need_capture_pp(
-	const struct ia_css_pipe *pipe)
-{
-	const struct ia_css_frame_info *out_info = &pipe->output_info[0];
-	IA_CSS_ENTER_LEAVE_PRIVATE("");
-	assert(pipe != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE);
-#ifdef ISP2401
-
-	/* ldc and capture_pp are not supported in the same pipeline */
-	if (need_capt_ldc(pipe) == true)
-		return false;
-#endif
-	/* determine whether we need to use the capture_pp binary.
-	 * This is needed for:
-	 *   1. XNR or
-	 *   2. Digital Zoom or
-	 *   3. YUV downscaling
-	 */
-	if (pipe->out_yuv_ds_input_info.res.width &&
-	    ((pipe->out_yuv_ds_input_info.res.width != out_info->res.width) ||
-	     (pipe->out_yuv_ds_input_info.res.height != out_info->res.height)))
-		return true;
-
-	if (pipe->config.default_capture_config.enable_xnr != 0)
-		return true;
-
-	if ((pipe->stream->isp_params_configs->dz_config.dx < HRT_GDC_N) ||
-	    (pipe->stream->isp_params_configs->dz_config.dy < HRT_GDC_N) ||
-	    pipe->config.enable_dz)
-		return true;
-
-	return false;
-}
-
-static bool need_capt_ldc(
-	const struct ia_css_pipe *pipe)
-{
-	IA_CSS_ENTER_LEAVE_PRIVATE("");
-	assert(pipe != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE);
-	return (pipe->extra_config.enable_dvs_6axis) ? true:false;
-}
-
-static enum ia_css_err set_num_primary_stages(unsigned int *num, enum ia_css_pipe_version version)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	if (num == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	switch (version) {
-	case IA_CSS_PIPE_VERSION_2_6_1:
-		*num = NUM_PRIMARY_HQ_STAGES;
-		break;
-	case IA_CSS_PIPE_VERSION_2_2:
-	case IA_CSS_PIPE_VERSION_1:
-		*num = NUM_PRIMARY_STAGES;
-		break;
-	default:
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		break;
-	}
-
-	return err;
-}
-
-static enum ia_css_err load_primary_binaries(
-	struct ia_css_pipe *pipe)
-{
-	bool online = false;
-	bool memory = false;
-	bool continuous = false;
-	bool need_pp = false;
-	bool need_isp_copy_binary = false;
-	bool need_ldc = false;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	bool sensor = false;
-#endif
-	struct ia_css_frame_info prim_in_info,
-				 prim_out_info,
-				 capt_pp_out_info, vf_info,
-				 *vf_pp_in_info, *pipe_out_info,
-#ifndef ISP2401
-				 *pipe_vf_out_info, *capt_pp_in_info,
-				 capt_ldc_out_info;
-#else
-				 *pipe_vf_out_info;
-#endif
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_capture_settings *mycs;
-	unsigned int i;
-	bool need_extra_yuv_scaler = false;
-
-	IA_CSS_ENTER_PRIVATE("");
-	assert(pipe != NULL);
-	assert(pipe->stream != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || pipe->mode == IA_CSS_PIPE_ID_COPY);
-
-	online = pipe->stream->config.online;
-	memory = pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY;
-	continuous = pipe->stream->config.continuous;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	sensor = (pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR);
-#endif
-
-	mycs = &pipe->pipe_settings.capture;
-	pipe_out_info = &pipe->output_info[0];
-	pipe_vf_out_info = &pipe->vf_output_info[0];
-
-	if (mycs->primary_binary[0].info)
-		return IA_CSS_SUCCESS;
-
-	err = set_num_primary_stages(&mycs->num_primary_stage, pipe->config.isp_pipe_version);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-
-	if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) {
-		err = ia_css_util_check_vf_out_info(pipe_out_info, pipe_vf_out_info);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-	else{
-		err = ia_css_frame_check_info(pipe_out_info);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-	need_pp = need_capture_pp(pipe);
-
-	/* we use the vf output info to get the primary/capture_pp binary
-	   configured for vf_veceven. It will select the closest downscaling
-	   factor. */
-	vf_info = *pipe_vf_out_info;
-
-/*
- * WARNING: The #if def flag has been added below as a
- * temporary solution to solve the problem of enabling the
- * view finder in a single binary in a capture flow. The
- * vf-pp stage has been removed for Skycam in the solution
- * provided. The vf-pp stage should be re-introduced when
- * required. This should not be considered as a clean solution.
- * Proper investigation should be done to come up with the clean
- * solution.
- * */
-	ia_css_frame_info_set_format(&vf_info, IA_CSS_FRAME_FORMAT_YUV_LINE);
-
-	/* TODO: All this yuv_scaler and capturepp calculation logic
-	 * can be shared later. Capture_pp is also a yuv_scale binary
-	 * with extra XNR funcionality. Therefore, it can be made as the
-	 * first step of the cascade. */
-	capt_pp_out_info = pipe->out_yuv_ds_input_info;
-	capt_pp_out_info.format = IA_CSS_FRAME_FORMAT_YUV420;
-	capt_pp_out_info.res.width  /= MAX_PREFERRED_YUV_DS_PER_STEP;
-	capt_pp_out_info.res.height /= MAX_PREFERRED_YUV_DS_PER_STEP;
-	ia_css_frame_info_set_width(&capt_pp_out_info, capt_pp_out_info.res.width, 0);
-
-/*
- * WARNING: The #if def flag has been added below as a
- * temporary solution to solve the problem of enabling the
- * view finder in a single binary in a capture flow. The
- * vf-pp stage has been removed for Skycam in the solution
- * provided. The vf-pp stage should be re-introduced when
- * required. This should not be considered as a clean solution.
- * Proper investigation should be done to come up with the clean
- * solution.
- * */
-	need_extra_yuv_scaler = need_downscaling(capt_pp_out_info.res,
-						pipe_out_info->res);
-
-	if (need_extra_yuv_scaler) {
-		struct ia_css_cas_binary_descr cas_scaler_descr = { };
-
-		err = ia_css_pipe_create_cas_scaler_desc_single_output(
-			&capt_pp_out_info,
-			pipe_out_info,
-			NULL,
-			&cas_scaler_descr);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-		mycs->num_yuv_scaler = cas_scaler_descr.num_stage;
-		mycs->yuv_scaler_binary = kzalloc(cas_scaler_descr.num_stage *
-			sizeof(struct ia_css_binary), GFP_KERNEL);
-		if (!mycs->yuv_scaler_binary) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-		mycs->is_output_stage = kzalloc(cas_scaler_descr.num_stage *
-			sizeof(bool), GFP_KERNEL);
-		if (!mycs->is_output_stage) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-		for (i = 0; i < cas_scaler_descr.num_stage; i++) {
-			struct ia_css_binary_descr yuv_scaler_descr;
-			mycs->is_output_stage[i] = cas_scaler_descr.is_output_stage[i];
-			ia_css_pipe_get_yuvscaler_binarydesc(pipe,
-				&yuv_scaler_descr, &cas_scaler_descr.in_info[i],
-				&cas_scaler_descr.out_info[i],
-				&cas_scaler_descr.internal_out_info[i],
-				&cas_scaler_descr.vf_info[i]);
-			err = ia_css_binary_find(&yuv_scaler_descr,
-						&mycs->yuv_scaler_binary[i]);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-		}
-		ia_css_pipe_destroy_cas_scaler_desc(&cas_scaler_descr);
-
-	} else {
-		capt_pp_out_info = pipe->output_info[0];
-	}
-
-	/* TODO Do we disable ldc for skycam */
-	need_ldc = need_capt_ldc(pipe);
-#ifdef ISP2401
-	/* ldc and capt_pp are not supported in the same pipeline */
-	if (need_ldc) {
-		struct ia_css_binary_descr capt_ldc_descr;
-		ia_css_pipe_get_ldc_binarydesc(pipe,
-			&capt_ldc_descr, &prim_out_info,
-			&capt_pp_out_info);
-#endif
-
-#ifdef ISP2401
-		err = ia_css_binary_find(&capt_ldc_descr,
-					&mycs->capture_ldc_binary);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	} else if (need_pp) {
-#endif
-	/* we build up the pipeline starting at the end */
-	/* Capture post-processing */
-#ifndef ISP2401
-	if (need_pp) {
-#endif
-		struct ia_css_binary_descr capture_pp_descr;
-#ifndef ISP2401
-		capt_pp_in_info = need_ldc ? &capt_ldc_out_info : &prim_out_info;
-#endif
-
-		ia_css_pipe_get_capturepp_binarydesc(pipe,
-#ifndef ISP2401
-			&capture_pp_descr, capt_pp_in_info,
-#else
-			&capture_pp_descr, &prim_out_info,
-#endif
-			&capt_pp_out_info, &vf_info);
-		err = ia_css_binary_find(&capture_pp_descr,
-					&mycs->capture_pp_binary);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-#ifndef ISP2401
-
-		if(need_ldc) {
-			struct ia_css_binary_descr capt_ldc_descr;
-			ia_css_pipe_get_ldc_binarydesc(pipe,
-				&capt_ldc_descr, &prim_out_info,
-				&capt_ldc_out_info);
-
-			err = ia_css_binary_find(&capt_ldc_descr,
-						&mycs->capture_ldc_binary);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-		}
-#endif
-	} else {
-		prim_out_info = *pipe_out_info;
-	}
-
-	/* Primary */
-	{
-		struct ia_css_binary_descr prim_descr[MAX_NUM_PRIMARY_STAGES];
-
-		for (i = 0; i < mycs->num_primary_stage; i++) {
-			struct ia_css_frame_info *local_vf_info = NULL;
-			if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0] && (i == mycs->num_primary_stage - 1))
-				local_vf_info = &vf_info;
-			ia_css_pipe_get_primary_binarydesc(pipe, &prim_descr[i], &prim_in_info, &prim_out_info, local_vf_info, i);
-			err = ia_css_binary_find(&prim_descr[i], &mycs->primary_binary[i]);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-		}
-	}
-
-	/* Viewfinder post-processing */
-	if (need_pp) {
-		vf_pp_in_info =
-		    &mycs->capture_pp_binary.vf_frame_info;
-	} else {
-		vf_pp_in_info =
-		    &mycs->primary_binary[mycs->num_primary_stage - 1].vf_frame_info;
-	}
-
-/*
- * WARNING: The #if def flag has been added below as a
- * temporary solution to solve the problem of enabling the
- * view finder in a single binary in a capture flow. The
- * vf-pp stage has been removed for Skycam in the solution
- * provided. The vf-pp stage should be re-introduced when
- * required. Thisshould not be considered as a clean solution.
- * Proper  * investigation should be done to come up with the clean
- * solution.
- * */
-	if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0])
-	{
-		struct ia_css_binary_descr vf_pp_descr;
-
-		ia_css_pipe_get_vfpp_binarydesc(pipe,
-				&vf_pp_descr, vf_pp_in_info, pipe_vf_out_info);
-		err = ia_css_binary_find(&vf_pp_descr, &mycs->vf_pp_binary);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-	err = allocate_delay_frames(pipe);
-
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	/* When the input system is 2401, only the Direct Sensor Mode
-	 * Offline Capture uses the ISP copy binary.
-	 */
-	need_isp_copy_binary = !online && sensor;
-#else
-	need_isp_copy_binary = !online && !continuous && !memory;
-#endif
-
-	/* ISP Copy */
-	if (need_isp_copy_binary) {
-		err = load_copy_binary(pipe,
-				&mycs->copy_binary,
-				&mycs->primary_binary[0]);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-allocate_delay_frames(struct ia_css_pipe *pipe)
-{
-	unsigned int num_delay_frames = 0, i = 0;
-	unsigned int dvs_frame_delay = 0;
-	struct ia_css_frame_info ref_info;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	enum ia_css_pipe_id mode = IA_CSS_PIPE_ID_VIDEO;
-	struct ia_css_frame **delay_frames = NULL;
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	if (pipe == NULL) {
-		IA_CSS_ERROR("Invalid args - pipe %p", pipe);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	mode = pipe->mode;
-	dvs_frame_delay = pipe->dvs_frame_delay;
-
-	if (dvs_frame_delay > 0)
-		num_delay_frames = dvs_frame_delay + 1;
-
-	switch (mode) {
-		case IA_CSS_PIPE_ID_CAPTURE:
-		{
-			struct ia_css_capture_settings *mycs_capture = &pipe->pipe_settings.capture;
-			(void)mycs_capture;
-			return err;
-		}
-		break;
-		case IA_CSS_PIPE_ID_VIDEO:
-		{
-			struct ia_css_video_settings *mycs_video = &pipe->pipe_settings.video;
-			ref_info = mycs_video->video_binary.internal_frame_info;
-			/*The ref frame expects
-			 * 	1. Y plane
-			 * 	2. UV plane with line interleaving, like below
-			 * 		UUUUUU(width/2 times) VVVVVVVV..(width/2 times)
-			 *
-			 *	This format is not YUV420(which has Y, U and V planes).
-			 *	Its closer to NV12, except that the UV plane has UV
-			 *	interleaving, like UVUVUVUVUVUVUVUVU...
-			 *
-			 *	TODO: make this ref_frame format as a separate frame format
-			 */
-			ref_info.format        = IA_CSS_FRAME_FORMAT_NV12;
-			delay_frames = mycs_video->delay_frames;
-		}
-		break;
-		case IA_CSS_PIPE_ID_PREVIEW:
-		{
-			struct ia_css_preview_settings *mycs_preview = &pipe->pipe_settings.preview;
-			ref_info = mycs_preview->preview_binary.internal_frame_info;
-			/*The ref frame expects
-			 *	1. Y plane
-			 *	2. UV plane with line interleaving, like below
-			 *		UUUUUU(width/2 times) VVVVVVVV..(width/2 times)
-			 *
-			 *	This format is not YUV420(which has Y, U and V planes).
-			 *	Its closer to NV12, except that the UV plane has UV
-			 *	interleaving, like UVUVUVUVUVUVUVUVU...
-			 *
-			 *	TODO: make this ref_frame format as a separate frame format
-			 */
-			ref_info.format        = IA_CSS_FRAME_FORMAT_NV12;
-			delay_frames = mycs_preview->delay_frames;
-		}
-		break;
-		default:
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	}
-
-	ref_info.raw_bit_depth = SH_CSS_REF_BIT_DEPTH;
-
-	assert(num_delay_frames <= MAX_NUM_VIDEO_DELAY_FRAMES);
-	for (i = 0; i < num_delay_frames; i++) {
-		err = ia_css_frame_allocate_from_info(&delay_frames[i],	&ref_info);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-	IA_CSS_LEAVE_PRIVATE("");
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err load_advanced_binaries(
-	struct ia_css_pipe *pipe)
-{
-	struct ia_css_frame_info pre_in_info, gdc_in_info,
-				 post_in_info, post_out_info,
-				 vf_info, *vf_pp_in_info, *pipe_out_info,
-				 *pipe_vf_out_info;
-	bool need_pp;
-	bool need_isp_copy = true;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	assert(pipe != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || pipe->mode == IA_CSS_PIPE_ID_COPY);
-	if (pipe->pipe_settings.capture.pre_isp_binary.info)
-		return IA_CSS_SUCCESS;
-	pipe_out_info = &pipe->output_info[0];
-	pipe_vf_out_info = &pipe->vf_output_info[0];
-
-	vf_info = *pipe_vf_out_info;
-	err = ia_css_util_check_vf_out_info(pipe_out_info, &vf_info);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	need_pp = need_capture_pp(pipe);
-
-	ia_css_frame_info_set_format(&vf_info,
-				     IA_CSS_FRAME_FORMAT_YUV_LINE);
-
-	/* we build up the pipeline starting at the end */
-	/* Capture post-processing */
-	if (need_pp) {
-		struct ia_css_binary_descr capture_pp_descr;
-
-		ia_css_pipe_get_capturepp_binarydesc(pipe,
-			&capture_pp_descr, &post_out_info, pipe_out_info, &vf_info);
-		err = ia_css_binary_find(&capture_pp_descr,
-				&pipe->pipe_settings.capture.capture_pp_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	} else {
-		post_out_info = *pipe_out_info;
-	}
-
-	/* Post-gdc */
-	{
-		struct ia_css_binary_descr post_gdc_descr;
-
-		ia_css_pipe_get_post_gdc_binarydesc(pipe,
-			&post_gdc_descr, &post_in_info, &post_out_info, &vf_info);
-		err = ia_css_binary_find(&post_gdc_descr,
-					 &pipe->pipe_settings.capture.post_isp_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-
-	/* Gdc */
-	{
-		struct ia_css_binary_descr gdc_descr;
-
-		ia_css_pipe_get_gdc_binarydesc(pipe, &gdc_descr, &gdc_in_info,
-			       &pipe->pipe_settings.capture.post_isp_binary.in_frame_info);
-		err = ia_css_binary_find(&gdc_descr,
-					 &pipe->pipe_settings.capture.anr_gdc_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-	pipe->pipe_settings.capture.anr_gdc_binary.left_padding =
-		pipe->pipe_settings.capture.post_isp_binary.left_padding;
-
-	/* Pre-gdc */
-	{
-		struct ia_css_binary_descr pre_gdc_descr;
-
-		ia_css_pipe_get_pre_gdc_binarydesc(pipe, &pre_gdc_descr, &pre_in_info,
-				   &pipe->pipe_settings.capture.anr_gdc_binary.in_frame_info);
-		err = ia_css_binary_find(&pre_gdc_descr,
-					 &pipe->pipe_settings.capture.pre_isp_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-	pipe->pipe_settings.capture.pre_isp_binary.left_padding =
-		pipe->pipe_settings.capture.anr_gdc_binary.left_padding;
-
-	/* Viewfinder post-processing */
-	if (need_pp) {
-		vf_pp_in_info =
-		    &pipe->pipe_settings.capture.capture_pp_binary.vf_frame_info;
-	} else {
-		vf_pp_in_info =
-		    &pipe->pipe_settings.capture.post_isp_binary.vf_frame_info;
-	}
-
-	{
-		struct ia_css_binary_descr vf_pp_descr;
-
-		ia_css_pipe_get_vfpp_binarydesc(pipe,
-			&vf_pp_descr, vf_pp_in_info, pipe_vf_out_info);
-		err = ia_css_binary_find(&vf_pp_descr,
-					 &pipe->pipe_settings.capture.vf_pp_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-
-	/* Copy */
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	/* For CSI2+, only the direct sensor mode/online requires ISP copy */
-	need_isp_copy = pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR;
-#endif
-	if (need_isp_copy)
-		load_copy_binary(pipe,
-			       &pipe->pipe_settings.capture.copy_binary,
-			       &pipe->pipe_settings.capture.pre_isp_binary);
-
-	return err;
-}
-
-static enum ia_css_err load_bayer_isp_binaries(
-	struct ia_css_pipe *pipe)
-{
-	struct ia_css_frame_info pre_isp_in_info, *pipe_out_info;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_binary_descr pre_de_descr;
-
-	IA_CSS_ENTER_PRIVATE("");
-	assert(pipe != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || pipe->mode == IA_CSS_PIPE_ID_COPY);
-	pipe_out_info = &pipe->output_info[0];
-
-	if (pipe->pipe_settings.capture.pre_isp_binary.info)
-		return IA_CSS_SUCCESS;
-
-	err = ia_css_frame_check_info(pipe_out_info);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-	ia_css_pipe_get_pre_de_binarydesc(pipe, &pre_de_descr,
-				&pre_isp_in_info,
-				pipe_out_info);
-
-	err = ia_css_binary_find(&pre_de_descr,
-				 &pipe->pipe_settings.capture.pre_isp_binary);
-
-	return err;
-}
-
-static enum ia_css_err load_low_light_binaries(
-	struct ia_css_pipe *pipe)
-{
-	struct ia_css_frame_info pre_in_info, anr_in_info,
-				 post_in_info, post_out_info,
-				 vf_info, *pipe_vf_out_info, *pipe_out_info,
-				 *vf_pp_in_info;
-	bool need_pp;
-	bool need_isp_copy = true;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER_PRIVATE("");
-	assert(pipe != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || pipe->mode == IA_CSS_PIPE_ID_COPY);
-
-	if (pipe->pipe_settings.capture.pre_isp_binary.info)
-		return IA_CSS_SUCCESS;
-	pipe_vf_out_info = &pipe->vf_output_info[0];
-	pipe_out_info = &pipe->output_info[0];
-
-	vf_info = *pipe_vf_out_info;
-	err = ia_css_util_check_vf_out_info(pipe_out_info,
-				&vf_info);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	need_pp = need_capture_pp(pipe);
-
-	ia_css_frame_info_set_format(&vf_info,
-				     IA_CSS_FRAME_FORMAT_YUV_LINE);
-
-	/* we build up the pipeline starting at the end */
-	/* Capture post-processing */
-	if (need_pp) {
-		struct ia_css_binary_descr capture_pp_descr;
-
-		ia_css_pipe_get_capturepp_binarydesc(pipe,
-			&capture_pp_descr, &post_out_info, pipe_out_info, &vf_info);
-		err = ia_css_binary_find(&capture_pp_descr,
-				&pipe->pipe_settings.capture.capture_pp_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	} else {
-		post_out_info = *pipe_out_info;
-	}
-
-	/* Post-anr */
-	{
-		struct ia_css_binary_descr post_anr_descr;
-
-		ia_css_pipe_get_post_anr_binarydesc(pipe,
-			&post_anr_descr, &post_in_info, &post_out_info, &vf_info);
-		err = ia_css_binary_find(&post_anr_descr,
-					 &pipe->pipe_settings.capture.post_isp_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-
-	/* Anr */
-	{
-		struct ia_css_binary_descr anr_descr;
-
-		ia_css_pipe_get_anr_binarydesc(pipe, &anr_descr, &anr_in_info,
-				&pipe->pipe_settings.capture.post_isp_binary.in_frame_info);
-		err = ia_css_binary_find(&anr_descr,
-					 &pipe->pipe_settings.capture.anr_gdc_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-	pipe->pipe_settings.capture.anr_gdc_binary.left_padding =
-		pipe->pipe_settings.capture.post_isp_binary.left_padding;
-
-	/* Pre-anr */
-	{
-		struct ia_css_binary_descr pre_anr_descr;
-
-		ia_css_pipe_get_pre_anr_binarydesc(pipe, &pre_anr_descr, &pre_in_info,
-				   &pipe->pipe_settings.capture.anr_gdc_binary.in_frame_info);
-		err = ia_css_binary_find(&pre_anr_descr,
-				&pipe->pipe_settings.capture.pre_isp_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-	pipe->pipe_settings.capture.pre_isp_binary.left_padding =
-		pipe->pipe_settings.capture.anr_gdc_binary.left_padding;
-
-	/* Viewfinder post-processing */
-	if (need_pp) {
-		vf_pp_in_info =
-		    &pipe->pipe_settings.capture.capture_pp_binary.vf_frame_info;
-	} else {
-		vf_pp_in_info =
-		    &pipe->pipe_settings.capture.post_isp_binary.vf_frame_info;
-	}
-
-	{
-		struct ia_css_binary_descr vf_pp_descr;
-
-		ia_css_pipe_get_vfpp_binarydesc(pipe,
-			&vf_pp_descr, vf_pp_in_info, pipe_vf_out_info);
-		err = ia_css_binary_find(&vf_pp_descr,
-					 &pipe->pipe_settings.capture.vf_pp_binary);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-
-	/* Copy */
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	/* For CSI2+, only the direct sensor mode/online requires ISP copy */
-	need_isp_copy = pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR;
-#endif
-	if (need_isp_copy)
-		err = load_copy_binary(pipe,
-			       &pipe->pipe_settings.capture.copy_binary,
-			       &pipe->pipe_settings.capture.pre_isp_binary);
-
-	return err;
-}
-
-static bool copy_on_sp(struct ia_css_pipe *pipe)
-{
-	bool rval;
-
-	assert(pipe != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "copy_on_sp() enter:\n");
-
-	rval = true;
-
-	rval &=	(pipe->mode == IA_CSS_PIPE_ID_CAPTURE);
-
-	rval &= (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_RAW);
-
-	rval &= ((pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8) ||
-		(pipe->config.mode == IA_CSS_PIPE_MODE_COPY));
-
-	return rval;
-}
-
-static enum ia_css_err load_capture_binaries(
-	struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	bool must_be_raw;
-
-	IA_CSS_ENTER_PRIVATE("");
-	assert(pipe != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || pipe->mode == IA_CSS_PIPE_ID_COPY);
-
-	if (pipe->pipe_settings.capture.primary_binary[0].info) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-		return IA_CSS_SUCCESS;
-	}
-
-	/* in primary, advanced,low light or bayer,
-						the input format must be raw */
-	must_be_raw =
-		pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_ADVANCED ||
-		pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER ||
-		pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_LOW_LIGHT;
-	err = ia_css_util_check_input(&pipe->stream->config, must_be_raw, false);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-	if (copy_on_sp(pipe) &&
-	    pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8) {
-		ia_css_frame_info_init(
-			&pipe->output_info[0],
-			JPEG_BYTES,
-			1,
-			IA_CSS_FRAME_FORMAT_BINARY_8,
-			0);
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-		return IA_CSS_SUCCESS;
-	}
-
-	switch (pipe->config.default_capture_config.mode) {
-	case IA_CSS_CAPTURE_MODE_RAW:
-		err = load_copy_binaries(pipe);
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
-	  if (err == IA_CSS_SUCCESS)
-		  pipe->pipe_settings.capture.copy_binary.online = pipe->stream->config.online;
-#endif
-		break;
-	case IA_CSS_CAPTURE_MODE_BAYER:
-		err = load_bayer_isp_binaries(pipe);
-		break;
-	case IA_CSS_CAPTURE_MODE_PRIMARY:
-		err = load_primary_binaries(pipe);
-		break;
-	case IA_CSS_CAPTURE_MODE_ADVANCED:
-		err = load_advanced_binaries(pipe);
-		break;
-	case IA_CSS_CAPTURE_MODE_LOW_LIGHT:
-		err = load_low_light_binaries(pipe);
-		break;
-	}
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static enum ia_css_err
-unload_capture_binaries(struct ia_css_pipe *pipe)
-{
-	unsigned int i;
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-
-	if ((pipe == NULL) || ((pipe->mode != IA_CSS_PIPE_ID_CAPTURE) && (pipe->mode != IA_CSS_PIPE_ID_COPY))) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	ia_css_binary_unload(&pipe->pipe_settings.capture.copy_binary);
-	for (i = 0; i < MAX_NUM_PRIMARY_STAGES; i++)
-		ia_css_binary_unload(&pipe->pipe_settings.capture.primary_binary[i]);
-	ia_css_binary_unload(&pipe->pipe_settings.capture.pre_isp_binary);
-	ia_css_binary_unload(&pipe->pipe_settings.capture.anr_gdc_binary);
-	ia_css_binary_unload(&pipe->pipe_settings.capture.post_isp_binary);
-	ia_css_binary_unload(&pipe->pipe_settings.capture.capture_pp_binary);
-	ia_css_binary_unload(&pipe->pipe_settings.capture.capture_ldc_binary);
-	ia_css_binary_unload(&pipe->pipe_settings.capture.vf_pp_binary);
-
-	for (i = 0; i < pipe->pipe_settings.capture.num_yuv_scaler; i++)
-		ia_css_binary_unload(&pipe->pipe_settings.capture.yuv_scaler_binary[i]);
-
-	kfree(pipe->pipe_settings.capture.is_output_stage);
-	pipe->pipe_settings.capture.is_output_stage = NULL;
-	kfree(pipe->pipe_settings.capture.yuv_scaler_binary);
-	pipe->pipe_settings.capture.yuv_scaler_binary = NULL;
-
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-static bool
-need_downscaling(const struct ia_css_resolution in_res,
-		const struct ia_css_resolution out_res)
-{
-
-	if (in_res.width > out_res.width || in_res.height > out_res.height)
-		return true;
-
-	return false;
-}
-
-static bool
-need_yuv_scaler_stage(const struct ia_css_pipe *pipe)
-{
-	unsigned int i;
-	struct ia_css_resolution in_res, out_res;
-
-	bool need_format_conversion = false;
-
-	IA_CSS_ENTER_PRIVATE("");
-	assert(pipe != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_YUVPP);
-
-	/* TODO: make generic function */
-	need_format_conversion =
-		((pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY) &&
-		(pipe->output_info[0].format != IA_CSS_FRAME_FORMAT_CSI_MIPI_LEGACY_YUV420_8));
-
-	in_res = pipe->config.input_effective_res;
-
-	if (pipe->config.enable_dz)
-		return true;
-
-	if ((pipe->output_info[0].res.width != 0) && need_format_conversion)
-		return true;
-
-	for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-		out_res = pipe->output_info[i].res;
-
-		/* A non-zero width means it is a valid output port */
-		if ((out_res.width != 0) && need_downscaling(in_res, out_res))
-			return true;
-	}
-
-	return false;
-}
-
-/* TODO: it is temporarily created from ia_css_pipe_create_cas_scaler_desc */
-/* which has some hard-coded knowledge which prevents reuse of the function. */
-/* Later, merge this with ia_css_pipe_create_cas_scaler_desc */
-static enum ia_css_err ia_css_pipe_create_cas_scaler_desc_single_output(
-	struct ia_css_frame_info *cas_scaler_in_info,
-	struct ia_css_frame_info *cas_scaler_out_info,
-	struct ia_css_frame_info *cas_scaler_vf_info,
-	struct ia_css_cas_binary_descr *descr)
-{
-	unsigned int i;
-	unsigned int hor_ds_factor = 0, ver_ds_factor = 0;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_frame_info tmp_in_info;
-
-	unsigned max_scale_factor_per_stage = MAX_PREFERRED_YUV_DS_PER_STEP;
-
-	assert(cas_scaler_in_info != NULL);
-	assert(cas_scaler_out_info != NULL);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_pipe_create_cas_scaler_desc() enter:\n");
-
-	/* We assume that this function is used only for single output port case. */
-	descr->num_output_stage = 1;
-
-	hor_ds_factor = CEIL_DIV(cas_scaler_in_info->res.width , cas_scaler_out_info->res.width);
-	ver_ds_factor = CEIL_DIV(cas_scaler_in_info->res.height, cas_scaler_out_info->res.height);
-	/* use the same horizontal and vertical downscaling factor for simplicity */
-	assert(hor_ds_factor == ver_ds_factor);
-
-	i = 1;
-	while (i < hor_ds_factor) {
-		descr->num_stage++;
-		i *= max_scale_factor_per_stage;
-	}
-
-	descr->in_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL);
-	if (!descr->in_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-	descr->internal_out_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL);
-	if (!descr->internal_out_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-	descr->out_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL);
-	if (!descr->out_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-	descr->vf_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL);
-	if (!descr->vf_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-	descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool), GFP_KERNEL);
-	if (!descr->is_output_stage) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-
-	tmp_in_info = *cas_scaler_in_info;
-	for (i = 0; i < descr->num_stage; i++) {
-
-		descr->in_info[i] = tmp_in_info;
-		if ((tmp_in_info.res.width / max_scale_factor_per_stage) <= cas_scaler_out_info->res.width) {
-			descr->is_output_stage[i] = true;
-			if ((descr->num_output_stage > 1) && (i != (descr->num_stage - 1))) {
-				descr->internal_out_info[i].res.width = cas_scaler_out_info->res.width;
-				descr->internal_out_info[i].res.height = cas_scaler_out_info->res.height;
-				descr->internal_out_info[i].padded_width = cas_scaler_out_info->padded_width;
-				descr->internal_out_info[i].format = IA_CSS_FRAME_FORMAT_YUV420;
-			} else {
-				assert(i == (descr->num_stage - 1));
-				descr->internal_out_info[i].res.width = 0;
-				descr->internal_out_info[i].res.height = 0;
-			}
-			descr->out_info[i].res.width = cas_scaler_out_info->res.width;
-			descr->out_info[i].res.height = cas_scaler_out_info->res.height;
-			descr->out_info[i].padded_width = cas_scaler_out_info->padded_width;
-			descr->out_info[i].format = cas_scaler_out_info->format;
-			if (cas_scaler_vf_info != NULL) {
-				descr->vf_info[i].res.width = cas_scaler_vf_info->res.width;
-				descr->vf_info[i].res.height = cas_scaler_vf_info->res.height;
-				descr->vf_info[i].padded_width = cas_scaler_vf_info->padded_width;
-				ia_css_frame_info_set_format(&descr->vf_info[i], IA_CSS_FRAME_FORMAT_YUV_LINE);
-			} else {
-				descr->vf_info[i].res.width = 0;
-				descr->vf_info[i].res.height = 0;
-				descr->vf_info[i].padded_width = 0;
-			}
-		} else {
-			descr->is_output_stage[i] = false;
-			descr->internal_out_info[i].res.width = tmp_in_info.res.width / max_scale_factor_per_stage;
-			descr->internal_out_info[i].res.height = tmp_in_info.res.height / max_scale_factor_per_stage;
-			descr->internal_out_info[i].format = IA_CSS_FRAME_FORMAT_YUV420;
-			ia_css_frame_info_init(&descr->internal_out_info[i],
-					tmp_in_info.res.width / max_scale_factor_per_stage,
-					tmp_in_info.res.height / max_scale_factor_per_stage,
-					IA_CSS_FRAME_FORMAT_YUV420, 0);
-			descr->out_info[i].res.width = 0;
-			descr->out_info[i].res.height = 0;
-			descr->vf_info[i].res.width = 0;
-			descr->vf_info[i].res.height = 0;
-		}
-		tmp_in_info = descr->internal_out_info[i];
-	}
-ERR:
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_pipe_create_cas_scaler_desc() leave, err=%d\n",
-			err);
-	return err;
-}
-
-/* FIXME: merge most of this and single output version */
-static enum ia_css_err ia_css_pipe_create_cas_scaler_desc(struct ia_css_pipe *pipe,
-	struct ia_css_cas_binary_descr *descr)
-{
-	struct ia_css_frame_info in_info = IA_CSS_BINARY_DEFAULT_FRAME_INFO;
-	struct ia_css_frame_info *out_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	struct ia_css_frame_info *vf_out_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	struct ia_css_frame_info tmp_in_info = IA_CSS_BINARY_DEFAULT_FRAME_INFO;
-	unsigned int i, j;
-	unsigned int hor_scale_factor[IA_CSS_PIPE_MAX_OUTPUT_STAGE],
-				ver_scale_factor[IA_CSS_PIPE_MAX_OUTPUT_STAGE],
-				scale_factor = 0;
-	unsigned int num_stages = 0;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	unsigned max_scale_factor_per_stage = MAX_PREFERRED_YUV_DS_PER_STEP;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_pipe_create_cas_scaler_desc() enter:\n");
-
-	for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-		out_info[i] = NULL;
-		vf_out_info[i] = NULL;
-		hor_scale_factor[i] = 0;
-		ver_scale_factor[i] = 0;
-	}
-
-	in_info.res = pipe->config.input_effective_res;
-	in_info.padded_width = in_info.res.width;
-	descr->num_output_stage = 0;
-	/* Find out how much scaling we need for each output */
-	for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-		if (pipe->output_info[i].res.width != 0) {
-			out_info[i] = &pipe->output_info[i];
-			if (pipe->vf_output_info[i].res.width != 0)
-				vf_out_info[i] = &pipe->vf_output_info[i];
-			descr->num_output_stage += 1;
-		}
-
-		if (out_info[i] != NULL) {
-			hor_scale_factor[i] = CEIL_DIV(in_info.res.width, out_info[i]->res.width);
-			ver_scale_factor[i] = CEIL_DIV(in_info.res.height, out_info[i]->res.height);
-			/* use the same horizontal and vertical scaling factor for simplicity */
-			assert(hor_scale_factor[i] == ver_scale_factor[i]);
-			scale_factor = 1;
-			do {
-				num_stages++;
-				scale_factor *= max_scale_factor_per_stage;
-			} while (scale_factor < hor_scale_factor[i]);
-
-			in_info.res = out_info[i]->res;
-		}
-	}
-
-	if (need_yuv_scaler_stage(pipe) && (num_stages == 0))
-		num_stages = 1;
-
-	descr->num_stage = num_stages;
-
-	descr->in_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL);
-	if (!descr->in_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-	descr->internal_out_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL);
-	if (!descr->internal_out_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-	descr->out_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL);
-	if (!descr->out_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-	descr->vf_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL);
-	if (!descr->vf_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-	descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool), GFP_KERNEL);
-	if (descr->is_output_stage == NULL) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-
-	for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-		if (out_info[i]) {
-			if (i > 0) {
-				assert((out_info[i-1]->res.width >= out_info[i]->res.width) &&
-						(out_info[i-1]->res.height >= out_info[i]->res.height));
-			}
-		}
-	}
-
-	tmp_in_info.res = pipe->config.input_effective_res;
-	tmp_in_info.format = IA_CSS_FRAME_FORMAT_YUV420;
-	for (i = 0, j = 0; i < descr->num_stage; i++) {
-		assert(j < 2);
-		assert(out_info[j] != NULL);
-
-		descr->in_info[i] = tmp_in_info;
-		if ((tmp_in_info.res.width / max_scale_factor_per_stage) <= out_info[j]->res.width) {
-			descr->is_output_stage[i] = true;
-			if ((descr->num_output_stage > 1) && (i != (descr->num_stage - 1))) {
-				descr->internal_out_info[i].res.width = out_info[j]->res.width;
-				descr->internal_out_info[i].res.height = out_info[j]->res.height;
-				descr->internal_out_info[i].padded_width = out_info[j]->padded_width;
-				descr->internal_out_info[i].format = IA_CSS_FRAME_FORMAT_YUV420;
-			} else {
-				assert(i == (descr->num_stage - 1));
-				descr->internal_out_info[i].res.width = 0;
-				descr->internal_out_info[i].res.height = 0;
-			}
-			descr->out_info[i].res.width = out_info[j]->res.width;
-			descr->out_info[i].res.height = out_info[j]->res.height;
-			descr->out_info[i].padded_width = out_info[j]->padded_width;
-			descr->out_info[i].format = out_info[j]->format;
-			if (vf_out_info[j] != NULL) {
-				descr->vf_info[i].res.width = vf_out_info[j]->res.width;
-				descr->vf_info[i].res.height = vf_out_info[j]->res.height;
-				descr->vf_info[i].padded_width = vf_out_info[j]->padded_width;
-				ia_css_frame_info_set_format(&descr->vf_info[i], IA_CSS_FRAME_FORMAT_YUV_LINE);
-			} else {
-				descr->vf_info[i].res.width = 0;
-				descr->vf_info[i].res.height = 0;
-				descr->vf_info[i].padded_width = 0;
-			}
-			j++;
-		} else {
-			descr->is_output_stage[i] = false;
-			descr->internal_out_info[i].res.width = tmp_in_info.res.width / max_scale_factor_per_stage;
-			descr->internal_out_info[i].res.height = tmp_in_info.res.height / max_scale_factor_per_stage;
-			descr->internal_out_info[i].format = IA_CSS_FRAME_FORMAT_YUV420;
-			ia_css_frame_info_init(&descr->internal_out_info[i],
-					tmp_in_info.res.width / max_scale_factor_per_stage,
-					tmp_in_info.res.height / max_scale_factor_per_stage,
-					IA_CSS_FRAME_FORMAT_YUV420, 0);
-			descr->out_info[i].res.width = 0;
-			descr->out_info[i].res.height = 0;
-			descr->vf_info[i].res.width = 0;
-			descr->vf_info[i].res.height = 0;
-		}
-		tmp_in_info = descr->internal_out_info[i];
-	}
-ERR:
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_pipe_create_cas_scaler_desc() leave, err=%d\n",
-			err);
-	return err;
-}
-
-static void ia_css_pipe_destroy_cas_scaler_desc(struct ia_css_cas_binary_descr *descr)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_pipe_destroy_cas_scaler_desc() enter:\n");
-	kfree(descr->in_info);
-	descr->in_info = NULL;
-	kfree(descr->internal_out_info);
-	descr->internal_out_info = NULL;
-	kfree(descr->out_info);
-	descr->out_info = NULL;
-	kfree(descr->vf_info);
-	descr->vf_info = NULL;
-	kfree(descr->is_output_stage);
-	descr->is_output_stage = NULL;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_pipe_destroy_cas_scaler_desc() leave\n");
-}
-
-static enum ia_css_err
-load_yuvpp_binaries(struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	bool need_scaler = false;
-	struct ia_css_frame_info *vf_pp_in_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	struct ia_css_yuvpp_settings *mycs;
-	struct ia_css_binary *next_binary;
-	struct ia_css_cas_binary_descr cas_scaler_descr = { };
-	unsigned int i, j;
-	bool need_isp_copy_binary = false;
-
-	IA_CSS_ENTER_PRIVATE("");
-	assert(pipe != NULL);
-	assert(pipe->stream != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_YUVPP);
-
-	if (pipe->pipe_settings.yuvpp.copy_binary.info)
-		goto ERR;
-
-        /* Set both must_be_raw and must_be_yuv to false then yuvpp can take rgb inputs */
-	err = ia_css_util_check_input(&pipe->stream->config, false, false);
-	if (err != IA_CSS_SUCCESS)
-		goto ERR;
-
-	mycs = &pipe->pipe_settings.yuvpp;
-
-	for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-		if (pipe->vf_output_info[i].res.width != 0) {
-			err = ia_css_util_check_vf_out_info(&pipe->output_info[i],
-					&pipe->vf_output_info[i]);
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-		}
-		vf_pp_in_info[i] = NULL;
-	}
-
-	need_scaler = need_yuv_scaler_stage(pipe);
-
-	/* we build up the pipeline starting at the end */
-	/* Capture post-processing */
-	if (need_scaler) {
-		struct ia_css_binary_descr yuv_scaler_descr;
-
-		err = ia_css_pipe_create_cas_scaler_desc(pipe,
-			&cas_scaler_descr);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-		mycs->num_output = cas_scaler_descr.num_output_stage;
-		mycs->num_yuv_scaler = cas_scaler_descr.num_stage;
-		mycs->yuv_scaler_binary = kzalloc(cas_scaler_descr.num_stage *
-			sizeof(struct ia_css_binary), GFP_KERNEL);
-		if (!mycs->yuv_scaler_binary) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-			goto ERR;
-		}
-		mycs->is_output_stage = kzalloc(cas_scaler_descr.num_stage *
-			sizeof(bool), GFP_KERNEL);
-		if (!mycs->is_output_stage) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-			goto ERR;
-		}
-		for (i = 0; i < cas_scaler_descr.num_stage; i++) {
-			mycs->is_output_stage[i] = cas_scaler_descr.is_output_stage[i];
-			ia_css_pipe_get_yuvscaler_binarydesc(pipe,
-				&yuv_scaler_descr, &cas_scaler_descr.in_info[i],
-				&cas_scaler_descr.out_info[i],
-				&cas_scaler_descr.internal_out_info[i],
-				&cas_scaler_descr.vf_info[i]);
-			err = ia_css_binary_find(&yuv_scaler_descr,
-						&mycs->yuv_scaler_binary[i]);
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-		}
-		ia_css_pipe_destroy_cas_scaler_desc(&cas_scaler_descr);
-	} else {
-		mycs->num_output = 1;
-	}
-
-	if (need_scaler) {
-		next_binary = &mycs->yuv_scaler_binary[0];
-	} else {
-		next_binary = NULL;
-	}
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-	/*
-	 * NOTES
-	 * - Why does the "yuvpp" pipe needs "isp_copy_binary" (i.e. ISP Copy) when
-	 *   its input is "ATOMISP_INPUT_FORMAT_YUV422_8"?
-	 *
-	 *   In most use cases, the first stage in the "yuvpp" pipe is the "yuv_scale_
-	 *   binary". However, the "yuv_scale_binary" does NOT support the input-frame
-	 *   format as "IA_CSS_STREAM _FORMAT_YUV422_8".
-	 *
-	 *   Hence, the "isp_copy_binary" is required to be present in front of the "yuv
-	 *   _scale_binary". It would translate the input-frame to the frame formats that
-	 *   are supported by the "yuv_scale_binary".
-	 *
-	 *   Please refer to "FrameWork/css/isp/pipes/capture_pp/capture_pp_1.0/capture_
-	 *   pp_defs.h" for the list of input-frame formats that are supported by the
-	 *   "yuv_scale_binary".
-	 */
-	need_isp_copy_binary =
-		(pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_YUV422_8);
-#else  /* !USE_INPUT_SYSTEM_VERSION_2401 */
-	need_isp_copy_binary = true;
-#endif /*  USE_INPUT_SYSTEM_VERSION_2401 */
-
-	if (need_isp_copy_binary) {
-		err = load_copy_binary(pipe,
-				       &mycs->copy_binary,
-				       next_binary);
-
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-
-		/*
-		 * NOTES
-		 * - Why is "pipe->pipe_settings.capture.copy_binary.online" specified?
-		 *
-		 *   In some use cases, the first stage in the "yuvpp" pipe is the
-		 *   "isp_copy_binary". The "isp_copy_binary" is designed to process
-		 *   the input from either the system DDR or from the IPU internal VMEM.
-		 *   So it provides the flag "online" to specify where its input is from,
-		 *   i.e.:
-		 *
-		 *      (1) "online <= true", the input is from the IPU internal VMEM.
-		 *      (2) "online <= false", the input is from the system DDR.
-		 *
-		 *   In other use cases, the first stage in the "yuvpp" pipe is the
-		 *   "yuv_scale_binary". "The "yuv_scale_binary" is designed to process the
-		 *   input ONLY from the system DDR. So it does not provide the flag "online"
-		 *   to specify where its input is from.
-		 */
-		pipe->pipe_settings.capture.copy_binary.online = pipe->stream->config.online;
-	}
-
-	/* Viewfinder post-processing */
-	if (need_scaler) {
-		for (i = 0, j = 0; i < mycs->num_yuv_scaler; i++) {
-			if (mycs->is_output_stage[i]) {
-				assert(j < 2);
-				vf_pp_in_info[j] =
-					&mycs->yuv_scaler_binary[i].vf_frame_info;
-				j++;
-			}
-		}
-		mycs->num_vf_pp = j;
-	} else {
-		vf_pp_in_info[0] =
-		    &mycs->copy_binary.vf_frame_info;
-		for (i = 1; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-			vf_pp_in_info[i] = NULL;
-		}
-		mycs->num_vf_pp = 1;
-	}
-	mycs->vf_pp_binary = kzalloc(mycs->num_vf_pp * sizeof(struct ia_css_binary),
-						GFP_KERNEL);
-	if (!mycs->vf_pp_binary) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		goto ERR;
-	}
-
-	{
-		struct ia_css_binary_descr vf_pp_descr;
-
-		for (i = 0; i < mycs->num_vf_pp; i++) {
-			if (pipe->vf_output_info[i].res.width != 0) {
-				ia_css_pipe_get_vfpp_binarydesc(pipe,
-					&vf_pp_descr, vf_pp_in_info[i], &pipe->vf_output_info[i]);
-				err = ia_css_binary_find(&vf_pp_descr, &mycs->vf_pp_binary[i]);
-				if (err != IA_CSS_SUCCESS)
-					goto ERR;
-			}
-		}
-	}
-
-	if (err != IA_CSS_SUCCESS)
-		goto ERR;
-
-ERR:
-	if (need_scaler) {
-		ia_css_pipe_destroy_cas_scaler_desc(&cas_scaler_descr);
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "load_yuvpp_binaries() leave, err=%d\n",
-			err);
-	return err;
-}
-
-static enum ia_css_err
-unload_yuvpp_binaries(struct ia_css_pipe *pipe)
-{
-	unsigned int i;
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-
-	if ((pipe == NULL) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	ia_css_binary_unload(&pipe->pipe_settings.yuvpp.copy_binary);
-	for (i = 0; i < pipe->pipe_settings.yuvpp.num_yuv_scaler; i++) {
-		ia_css_binary_unload(&pipe->pipe_settings.yuvpp.yuv_scaler_binary[i]);
-	}
-	for (i = 0; i < pipe->pipe_settings.yuvpp.num_vf_pp; i++) {
-		ia_css_binary_unload(&pipe->pipe_settings.yuvpp.vf_pp_binary[i]);
-	}
-	kfree(pipe->pipe_settings.yuvpp.is_output_stage);
-	pipe->pipe_settings.yuvpp.is_output_stage = NULL;
-	kfree(pipe->pipe_settings.yuvpp.yuv_scaler_binary);
-	pipe->pipe_settings.yuvpp.yuv_scaler_binary = NULL;
-	kfree(pipe->pipe_settings.yuvpp.vf_pp_binary);
-	pipe->pipe_settings.yuvpp.vf_pp_binary = NULL;
-
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err yuvpp_start(struct ia_css_pipe *pipe)
-{
-	struct ia_css_binary *copy_binary;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	enum sh_css_pipe_config_override copy_ovrd;
-	enum ia_css_input_mode yuvpp_pipe_input_mode;
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-	if ((pipe == NULL) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	yuvpp_pipe_input_mode = pipe->stream->config.mode;
-
-	copy_binary  = &pipe->pipe_settings.yuvpp.copy_binary;
-
-	sh_css_metrics_start_frame();
-
-	/* multi stream video needs mipi buffers */
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && ( defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401) )
-	err = send_mipi_frames(pipe);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-#endif
-
-	{
-		unsigned int thread_id;
-
-		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-		copy_ovrd = 1 << thread_id;
-	}
-
-	start_pipe(pipe, copy_ovrd, yuvpp_pipe_input_mode);
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static enum ia_css_err
-sh_css_pipe_unload_binaries(struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-
-	if (pipe == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	/* PIPE_MODE_COPY has no binaries, but has output frames to outside*/
-	if (pipe->config.mode == IA_CSS_PIPE_MODE_COPY) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-		return IA_CSS_SUCCESS;
-	}
-
-	switch (pipe->mode) {
-	case IA_CSS_PIPE_ID_PREVIEW:
-		err = unload_preview_binaries(pipe);
-		break;
-	case IA_CSS_PIPE_ID_VIDEO:
-		err = unload_video_binaries(pipe);
-		break;
-	case IA_CSS_PIPE_ID_CAPTURE:
-		err = unload_capture_binaries(pipe);
-		break;
-	case IA_CSS_PIPE_ID_YUVPP:
-		err = unload_yuvpp_binaries(pipe);
-		break;
-	default:
-		break;
-	}
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static enum ia_css_err
-sh_css_pipe_load_binaries(struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	assert(pipe != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "sh_css_pipe_load_binaries() enter:\n");
-
-	/* PIPE_MODE_COPY has no binaries, but has output frames to outside*/
-	if (pipe->config.mode == IA_CSS_PIPE_MODE_COPY)
-		return err;
-
-	switch (pipe->mode) {
-	case IA_CSS_PIPE_ID_PREVIEW:
-		err = load_preview_binaries(pipe);
-		break;
-	case IA_CSS_PIPE_ID_VIDEO:
-		err = load_video_binaries(pipe);
-		break;
-	case IA_CSS_PIPE_ID_CAPTURE:
-		err = load_capture_binaries(pipe);
-		break;
-	case IA_CSS_PIPE_ID_YUVPP:
-		err = load_yuvpp_binaries(pipe);
-		break;
-	case IA_CSS_PIPE_ID_ACC:
-		break;
-	default:
-		err = IA_CSS_ERR_INTERNAL_ERROR;
-		break;
-	}
-	if (err != IA_CSS_SUCCESS) {
-		if (sh_css_pipe_unload_binaries(pipe) != IA_CSS_SUCCESS) {
-			/* currently css does not support multiple error returns in a single function,
-			 * using IA_CSS_ERR_INTERNAL_ERROR in this case */
-			err = IA_CSS_ERR_INTERNAL_ERROR;
-		}
-	}
-	return err;
-}
-
-static enum ia_css_err
-create_host_yuvpp_pipeline(struct ia_css_pipe *pipe)
-{
-	struct ia_css_pipeline *me;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_pipeline_stage *vf_pp_stage = NULL,
-				     *copy_stage = NULL,
-				     *yuv_scaler_stage = NULL;
-	struct ia_css_binary *copy_binary,
-			     *vf_pp_binary,
-			     *yuv_scaler_binary;
-	bool need_scaler = false;
-	unsigned int num_stage, num_vf_pp_stage, num_output_stage;
-	unsigned int i, j;
-
-	struct ia_css_frame *in_frame = NULL;
-	struct ia_css_frame *out_frame[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	struct ia_css_frame *bin_out_frame[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	struct ia_css_frame *vf_frame[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
-	struct ia_css_pipeline_stage_desc stage_desc;
-	bool need_in_frameinfo_memory = false;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	bool sensor = false;
-	bool buffered_sensor = false;
-	bool online = false;
-	bool continuous = false;
-#endif
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-	if ((pipe == NULL) || (pipe->stream == NULL) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	me = &pipe->pipeline;
-	ia_css_pipeline_clean(me);
-	for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-		out_frame[i] = NULL;
-		vf_frame[i] = NULL;
-	}
-	ia_css_pipe_util_create_output_frames(bin_out_frame);
-	num_stage  = pipe->pipe_settings.yuvpp.num_yuv_scaler;
-	num_vf_pp_stage   = pipe->pipe_settings.yuvpp.num_vf_pp;
-	num_output_stage   = pipe->pipe_settings.yuvpp.num_output;
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	/* When the input system is 2401, always enable 'in_frameinfo_memory'
-	 * except for the following:
-	 * - Direct Sensor Mode Online Capture
-	 * - Direct Sensor Mode Continuous Capture
-	 * - Buffered Sensor Mode Continuous Capture
-	 */
-	sensor = pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR;
-	buffered_sensor = pipe->stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR;
-	online = pipe->stream->config.online;
-	continuous = pipe->stream->config.continuous;
-	need_in_frameinfo_memory =
-		!((sensor && (online || continuous)) || (buffered_sensor && continuous));
-#else
-	/* Construct in_frame info (only in case we have dynamic input */
-	need_in_frameinfo_memory = pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY;
-#endif
-	/* the input frame can come from:
-	 *  a) memory: connect yuvscaler to me->in_frame
-	 *  b) sensor, via copy binary: connect yuvscaler to copy binary later on */
-	if (need_in_frameinfo_memory) {
-		/* TODO: improve for different input formats. */
-
-		/*
-		 * "pipe->stream->config.input_config.format" represents the sensor output
-		 * frame format, e.g. YUV422 8-bit.
-		 *
-		 * "in_frame_format" represents the imaging pipe's input frame format, e.g.
-		 * Bayer-Quad RAW.
-		 */
-		int in_frame_format;
-		if (pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY) {
-			in_frame_format = IA_CSS_FRAME_FORMAT_CSI_MIPI_LEGACY_YUV420_8;
-		} else if (pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_YUV422_8) {
-			/*
-			 * When the sensor output frame format is "ATOMISP_INPUT_FORMAT_YUV422_8",
-			 * the "isp_copy_var" binary is selected as the first stage in the yuvpp
-			 * pipe.
-			 *
-			 * For the "isp_copy_var" binary, it reads the YUV422-8 pixels from
-			 * the frame buffer (at DDR) to the frame-line buffer (at VMEM).
-			 *
-			 * By now, the "isp_copy_var" binary does NOT provide a separated
-			 * frame-line buffer to store the YUV422-8 pixels. Instead, it stores
-			 * the YUV422-8 pixels in the frame-line buffer which is designed to
-			 * store the Bayer-Quad RAW pixels.
-			 *
-			 * To direct the "isp_copy_var" binary reading from the RAW frame-line
-			 * buffer, its input frame format must be specified as "IA_CSS_FRAME_
-			 * FORMAT_RAW".
-			 */
-			in_frame_format = IA_CSS_FRAME_FORMAT_RAW;
-		} else {
-			in_frame_format = IA_CSS_FRAME_FORMAT_NV12;
-		}
-
-		err = init_in_frameinfo_memory_defaults(pipe,
-			&me->in_frame,
-			in_frame_format);
-
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-
-		in_frame = &me->in_frame;
-	} else {
-		in_frame = NULL;
-	}
-
-	for (i = 0; i < num_output_stage; i++) {
-		assert(i < IA_CSS_PIPE_MAX_OUTPUT_STAGE);
-		if (pipe->output_info[i].res.width != 0) {
-			err = init_out_frameinfo_defaults(pipe, &me->out_frame[i], i);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-			out_frame[i] = &me->out_frame[i];
-		}
-
-		/* Construct vf_frame info (only in case we have VF) */
-		if (pipe->vf_output_info[i].res.width != 0) {
-			err = init_vf_frameinfo_defaults(pipe, &me->vf_frame[i], i);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-			vf_frame[i] = &me->vf_frame[i];
-		}
-	}
-
-	copy_binary       = &pipe->pipe_settings.yuvpp.copy_binary;
-	vf_pp_binary      = pipe->pipe_settings.yuvpp.vf_pp_binary;
-	yuv_scaler_binary = pipe->pipe_settings.yuvpp.yuv_scaler_binary;
-	need_scaler = need_yuv_scaler_stage(pipe);
-
-	if (pipe->pipe_settings.yuvpp.copy_binary.info) {
-
-		struct ia_css_frame *in_frame_local = NULL;
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-		/* After isp copy is enabled in_frame needs to be passed. */
-		if (!online)
-			in_frame_local = in_frame;
-#endif
-
-		if (need_scaler) {
-			ia_css_pipe_util_set_output_frames(bin_out_frame, 0, NULL);
-			ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary,
-				bin_out_frame, in_frame_local, NULL);
-		} else {
-			ia_css_pipe_util_set_output_frames(bin_out_frame, 0, out_frame[0]);
-			ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary,
-				bin_out_frame, in_frame_local, NULL);
-		}
-
-		err = ia_css_pipeline_create_and_add_stage(me,
-			&stage_desc,
-			&copy_stage);
-
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-
-		if (copy_stage) {
-			/* if we use yuv scaler binary, vf output should be from there */
-			copy_stage->args.copy_vf = !need_scaler;
-			/* for yuvpp pipe, it should always be enabled */
-			copy_stage->args.copy_output = true;
-			/* connect output of copy binary to input of yuv scaler */
-			in_frame = copy_stage->args.out_frame[0];
-		}
-	}
-
-	if (need_scaler) {
-		struct ia_css_frame *tmp_out_frame = NULL;
-		struct ia_css_frame *tmp_vf_frame = NULL;
-		struct ia_css_frame *tmp_in_frame = in_frame;
-
-		for (i = 0, j = 0; i < num_stage; i++) {
-			assert(j < num_output_stage);
-			if (pipe->pipe_settings.yuvpp.is_output_stage[i]) {
-				tmp_out_frame = out_frame[j];
-				tmp_vf_frame = vf_frame[j];
-			} else {
-				tmp_out_frame = NULL;
-				tmp_vf_frame = NULL;
-			}
-
-			err = add_yuv_scaler_stage(pipe, me, tmp_in_frame, tmp_out_frame,
-						   NULL,
-						   &yuv_scaler_binary[i],
-						   &yuv_scaler_stage);
-
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-			/* we use output port 1 as internal output port */
-			tmp_in_frame = yuv_scaler_stage->args.out_frame[1];
-			if (pipe->pipe_settings.yuvpp.is_output_stage[i]) {
-				if (tmp_vf_frame && (tmp_vf_frame->info.res.width != 0)) {
-					in_frame = yuv_scaler_stage->args.out_vf_frame;
-					err = add_vf_pp_stage(pipe, in_frame, tmp_vf_frame, &vf_pp_binary[j],
-						      &vf_pp_stage);
-
-					if (err != IA_CSS_SUCCESS) {
-						IA_CSS_LEAVE_ERR_PRIVATE(err);
-						return err;
-					}
-				}
-				j++;
-			}
-		}
-	} else if (copy_stage != NULL) {
-		if (vf_frame[0] != NULL && vf_frame[0]->info.res.width != 0) {
-			in_frame = copy_stage->args.out_vf_frame;
-			err = add_vf_pp_stage(pipe, in_frame, vf_frame[0], &vf_pp_binary[0],
-				      &vf_pp_stage);
-		}
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-
-	ia_css_pipeline_finalize_stages(&pipe->pipeline, pipe->stream->config.continuous);
-
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-create_host_copy_pipeline(struct ia_css_pipe *pipe,
-    unsigned max_input_width,
-    struct ia_css_frame *out_frame)
-{
-	struct ia_css_pipeline *me;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_pipeline_stage_desc stage_desc;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"create_host_copy_pipeline() enter:\n");
-
-	/* pipeline already created as part of create_host_pipeline_structure */
-	me = &pipe->pipeline;
-	ia_css_pipeline_clean(me);
-
-	/* Construct out_frame info */
-	out_frame->contiguous = false;
-	out_frame->flash_state = IA_CSS_FRAME_FLASH_STATE_NONE;
-
-	if (copy_on_sp(pipe) &&
-	    pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8) {
-		ia_css_frame_info_init(
-			&out_frame->info,
-			JPEG_BYTES,
-			1,
-			IA_CSS_FRAME_FORMAT_BINARY_8,
-			0);
-	} else if (out_frame->info.format == IA_CSS_FRAME_FORMAT_RAW) {
-		out_frame->info.raw_bit_depth =
-			ia_css_pipe_util_pipe_input_format_bpp(pipe);
-	}
-
-	me->num_stages = 1;
-	me->pipe_id = IA_CSS_PIPE_ID_COPY;
-	pipe->mode  = IA_CSS_PIPE_ID_COPY;
-
-	ia_css_pipe_get_sp_func_stage_desc(&stage_desc, out_frame,
-		IA_CSS_PIPELINE_RAW_COPY, max_input_width);
-	err = ia_css_pipeline_create_and_add_stage(me,
-		&stage_desc,
-		NULL);
-
-	ia_css_pipeline_finalize_stages(&pipe->pipeline, pipe->stream->config.continuous);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"create_host_copy_pipeline() leave:\n");
-
-	return err;
-}
-
-static enum ia_css_err
-create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe)
-{
-	struct ia_css_pipeline *me = &pipe->pipeline;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_pipeline_stage_desc stage_desc;
-	struct ia_css_frame *out_frame = &me->out_frame[0];
-	struct ia_css_pipeline_stage *out_stage = NULL;
-	unsigned int thread_id;
-	enum sh_css_queue_id queue_id;
-	unsigned int max_input_width = MAX_VECTORS_PER_INPUT_LINE_CONT * ISP_VEC_NELEMS;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"create_host_isyscopy_capture_pipeline() enter:\n");
-	ia_css_pipeline_clean(me);
-
-	/* Construct out_frame info */
-	err = sh_css_pipe_get_output_frame_info(pipe, &out_frame->info, 0);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	out_frame->contiguous = false;
-	out_frame->flash_state = IA_CSS_FRAME_FLASH_STATE_NONE;
-	ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-	ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_OUTPUT_FRAME, thread_id, &queue_id);
-	out_frame->dynamic_queue_id = queue_id;
-	out_frame->buf_type = IA_CSS_BUFFER_TYPE_OUTPUT_FRAME;
-
-	me->num_stages = 1;
-	me->pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-	pipe->mode  = IA_CSS_PIPE_ID_CAPTURE;
-	ia_css_pipe_get_sp_func_stage_desc(&stage_desc, out_frame,
-		IA_CSS_PIPELINE_ISYS_COPY, max_input_width);
-	err = ia_css_pipeline_create_and_add_stage(me,
-		&stage_desc, &out_stage);
-	if(err != IA_CSS_SUCCESS)
-		return err;
-
-	ia_css_pipeline_finalize_stages(me, pipe->stream->config.continuous);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"create_host_isyscopy_capture_pipeline() leave:\n");
-
-	return err;
-}
-
-static enum ia_css_err
-create_host_regular_capture_pipeline(struct ia_css_pipe *pipe)
-{
-	struct ia_css_pipeline *me;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	enum ia_css_capture_mode mode;
-	struct ia_css_pipeline_stage *current_stage = NULL;
-	struct ia_css_pipeline_stage *yuv_scaler_stage = NULL;
-	struct ia_css_binary *copy_binary,
-			     *primary_binary[MAX_NUM_PRIMARY_STAGES],
-			     *vf_pp_binary,
-			     *pre_isp_binary,
-			     *anr_gdc_binary,
-			     *post_isp_binary,
-			     *yuv_scaler_binary,
-			     *capture_pp_binary,
-			     *capture_ldc_binary;
-	bool need_pp = false;
-	bool raw;
-
-	struct ia_css_frame *in_frame;
-	struct ia_css_frame *out_frame;
-	struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	struct ia_css_frame *vf_frame;
-	struct ia_css_pipeline_stage_desc stage_desc;
-	bool need_in_frameinfo_memory = false;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	bool sensor = false;
-	bool buffered_sensor = false;
-	bool online = false;
-	bool continuous = false;
-#endif
-	unsigned int i, num_yuv_scaler, num_primary_stage;
-	bool need_yuv_pp = false;
-	bool *is_output_stage = NULL;
-	bool need_ldc = false;
-
-	IA_CSS_ENTER_PRIVATE("");
-	assert(pipe != NULL);
-	assert(pipe->stream != NULL);
-	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || pipe->mode == IA_CSS_PIPE_ID_COPY);
-
-	me = &pipe->pipeline;
-	mode = pipe->config.default_capture_config.mode;
-	raw = (mode == IA_CSS_CAPTURE_MODE_RAW);
-	ia_css_pipeline_clean(me);
-	ia_css_pipe_util_create_output_frames(out_frames);
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	/* When the input system is 2401, always enable 'in_frameinfo_memory'
-	 * except for the following:
-	 * - Direct Sensor Mode Online Capture
-	 * - Direct Sensor Mode Online Capture
-	 * - Direct Sensor Mode Continuous Capture
-	 * - Buffered Sensor Mode Continuous Capture
-	 */
-	sensor = (pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR);
-	buffered_sensor = (pipe->stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR);
-	online = pipe->stream->config.online;
-	continuous = pipe->stream->config.continuous;
-	need_in_frameinfo_memory =
-		!((sensor && (online || continuous)) || (buffered_sensor && (online || continuous)));
-#else
-	/* Construct in_frame info (only in case we have dynamic input */
-	need_in_frameinfo_memory = pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY;
-#endif
-	if (need_in_frameinfo_memory) {
-		err = init_in_frameinfo_memory_defaults(pipe, &me->in_frame, IA_CSS_FRAME_FORMAT_RAW);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-
-		in_frame = &me->in_frame;
-	} else {
-		in_frame = NULL;
-	}
-
-	err = init_out_frameinfo_defaults(pipe, &me->out_frame[0], 0);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-	out_frame = &me->out_frame[0];
-
-	/* Construct vf_frame info (only in case we have VF) */
-	if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) {
-		if (mode == IA_CSS_CAPTURE_MODE_RAW || mode == IA_CSS_CAPTURE_MODE_BAYER) {
-			/* These modes don't support viewfinder output */
-			vf_frame = NULL;
-		} else {
-			init_vf_frameinfo_defaults(pipe, &me->vf_frame[0], 0);
-			vf_frame = &me->vf_frame[0];
-		}
-	} else {
-		vf_frame = NULL;
-	}
-
-	copy_binary       = &pipe->pipe_settings.capture.copy_binary;
-	num_primary_stage = pipe->pipe_settings.capture.num_primary_stage;
-	if ((num_primary_stage == 0) && (mode == IA_CSS_CAPTURE_MODE_PRIMARY)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-	for (i = 0; i < num_primary_stage; i++) {
-		primary_binary[i] = &pipe->pipe_settings.capture.primary_binary[i];
-	}
-	vf_pp_binary      = &pipe->pipe_settings.capture.vf_pp_binary;
-	pre_isp_binary    = &pipe->pipe_settings.capture.pre_isp_binary;
-	anr_gdc_binary    = &pipe->pipe_settings.capture.anr_gdc_binary;
-	post_isp_binary   = &pipe->pipe_settings.capture.post_isp_binary;
-	capture_pp_binary = &pipe->pipe_settings.capture.capture_pp_binary;
-	yuv_scaler_binary = pipe->pipe_settings.capture.yuv_scaler_binary;
-	num_yuv_scaler	  = pipe->pipe_settings.capture.num_yuv_scaler;
-	is_output_stage   = pipe->pipe_settings.capture.is_output_stage;
-	capture_ldc_binary = &pipe->pipe_settings.capture.capture_ldc_binary;
-
-	need_pp = (need_capture_pp(pipe) || pipe->output_stage) &&
-		  mode != IA_CSS_CAPTURE_MODE_RAW &&
-		  mode != IA_CSS_CAPTURE_MODE_BAYER;
-	need_yuv_pp = (yuv_scaler_binary != NULL && yuv_scaler_binary->info != NULL);
-	need_ldc = (capture_ldc_binary != NULL && capture_ldc_binary->info != NULL);
-
-	if (pipe->pipe_settings.capture.copy_binary.info) {
-		if (raw) {
-			ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame);
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
-			if (!continuous) {
-				ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary,
-					out_frames, in_frame, NULL);
-			} else {
-				in_frame = pipe->stream->last_pipe->continuous_frames[0];
-				ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary,
-					out_frames, in_frame, NULL);
-			}
-#else
-			ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary,
-				out_frames, NULL, NULL);
-#endif
-		} else {
-			ia_css_pipe_util_set_output_frames(out_frames, 0, in_frame);
-			ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary,
-				out_frames, NULL, NULL);
-		}
-
-		err = ia_css_pipeline_create_and_add_stage(me,
-			&stage_desc,
-			&current_stage);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	} else if (pipe->stream->config.continuous) {
-		in_frame = pipe->stream->last_pipe->continuous_frames[0];
-	}
-
-	if (mode == IA_CSS_CAPTURE_MODE_PRIMARY) {
-		struct ia_css_frame *local_in_frame = NULL;
-		struct ia_css_frame *local_out_frame = NULL;
-
-		for (i = 0; i < num_primary_stage; i++) {
-			if (i == 0)
-				local_in_frame = in_frame;
-			else
-				local_in_frame = NULL;
-#ifndef ISP2401
-			if (!need_pp && (i == num_primary_stage - 1))
-#else
-			if (!need_pp && (i == num_primary_stage - 1) && !need_ldc)
-#endif
-				local_out_frame = out_frame;
-			else
-				local_out_frame = NULL;
-			ia_css_pipe_util_set_output_frames(out_frames, 0, local_out_frame);
-/*
- * WARNING: The #if def flag has been added below as a
- * temporary solution to solve the problem of enabling the
- * view finder in a single binary in a capture flow. The
- * vf-pp stage has been removed from Skycam in the solution
- * provided. The vf-pp stage should be re-introduced when
- * required. This  * should not be considered as a clean solution.
- * Proper investigation should be done to come up with the clean
- * solution.
- * */
-			ia_css_pipe_get_generic_stage_desc(&stage_desc, primary_binary[i],
-				out_frames, local_in_frame, NULL);
-			err = ia_css_pipeline_create_and_add_stage(me,
-				&stage_desc,
-				&current_stage);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-		}
-		/* If we use copy iso primary,
-		   the input must be yuv iso raw */
-		current_stage->args.copy_vf =
-			primary_binary[0]->info->sp.pipeline.mode ==
-			IA_CSS_BINARY_MODE_COPY;
-		current_stage->args.copy_output = current_stage->args.copy_vf;
-	} else if (mode == IA_CSS_CAPTURE_MODE_ADVANCED ||
-	           mode == IA_CSS_CAPTURE_MODE_LOW_LIGHT) {
-		ia_css_pipe_util_set_output_frames(out_frames, 0, NULL);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc, pre_isp_binary,
-			out_frames, in_frame, NULL);
-		err = ia_css_pipeline_create_and_add_stage(me,
-				&stage_desc, NULL);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-		ia_css_pipe_util_set_output_frames(out_frames, 0, NULL);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc, anr_gdc_binary,
-			out_frames, NULL, NULL);
-		err = ia_css_pipeline_create_and_add_stage(me,
-				&stage_desc, NULL);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-
-		if(need_pp) {
-			ia_css_pipe_util_set_output_frames(out_frames, 0, NULL);
-			ia_css_pipe_get_generic_stage_desc(&stage_desc, post_isp_binary,
-				out_frames, NULL, NULL);
-		} else {
-			ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame);
-			ia_css_pipe_get_generic_stage_desc(&stage_desc, post_isp_binary,
-				out_frames, NULL, NULL);
-		}
-
-		err = ia_css_pipeline_create_and_add_stage(me,
-				&stage_desc, &current_stage);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	} else if (mode == IA_CSS_CAPTURE_MODE_BAYER) {
-		ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc, pre_isp_binary,
-			out_frames, in_frame, NULL);
-		err = ia_css_pipeline_create_and_add_stage(me,
-			&stage_desc,
-			NULL);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-
-#ifndef ISP2401
-	if (need_pp && current_stage) {
-		struct ia_css_frame *local_in_frame = NULL;
-		local_in_frame = current_stage->args.out_frame[0];
-
-		if(need_ldc) {
-			ia_css_pipe_util_set_output_frames(out_frames, 0, NULL);
-			ia_css_pipe_get_generic_stage_desc(&stage_desc, capture_ldc_binary,
-				out_frames, local_in_frame, NULL);
-			err = ia_css_pipeline_create_and_add_stage(me,
-				&stage_desc,
-				&current_stage);
-			local_in_frame = current_stage->args.out_frame[0];
-		}
-		err = add_capture_pp_stage(pipe, me, local_in_frame, need_yuv_pp ? NULL : out_frame,
-#else
-	/* ldc and capture_pp not supported in same pipeline */
-	if (need_ldc && current_stage) {
-		in_frame = current_stage->args.out_frame[0];
-		ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame);
-		ia_css_pipe_get_generic_stage_desc(&stage_desc, capture_ldc_binary,
-			out_frames, in_frame, NULL);
-		err = ia_css_pipeline_create_and_add_stage(me,
-			&stage_desc,
-			NULL);
-	} else if (need_pp && current_stage) {
-		in_frame = current_stage->args.out_frame[0];
-		err = add_capture_pp_stage(pipe, me, in_frame, need_yuv_pp ? NULL : out_frame,
-#endif
-					   capture_pp_binary,
-					   &current_stage);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-
-	if (need_yuv_pp && current_stage) {
-		struct ia_css_frame *tmp_in_frame = current_stage->args.out_frame[0];
-		struct ia_css_frame *tmp_out_frame = NULL;
-
-		for (i = 0; i < num_yuv_scaler; i++) {
-			if (is_output_stage[i] == true)
-				tmp_out_frame = out_frame;
-			else
-				tmp_out_frame = NULL;
-
-			err = add_yuv_scaler_stage(pipe, me, tmp_in_frame, tmp_out_frame,
-						   NULL,
-						   &yuv_scaler_binary[i],
-						   &yuv_scaler_stage);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-			/* we use output port 1 as internal output port */
-			tmp_in_frame = yuv_scaler_stage->args.out_frame[1];
-		}
-	}
-
-/*
- * WARNING: The #if def flag has been added below as a
- * temporary solution to solve the problem of enabling the
- * view finder in a single binary in a capture flow. The vf-pp
- * stage has been removed from Skycam in the solution provided.
- * The vf-pp stage should be re-introduced when required. This
- * should not be considered as a clean solution. Proper
- * investigation should be done to come up with the clean solution.
- * */
-	if (mode != IA_CSS_CAPTURE_MODE_RAW && mode != IA_CSS_CAPTURE_MODE_BAYER && current_stage && vf_frame) {
-		in_frame = current_stage->args.out_vf_frame;
-		err = add_vf_pp_stage(pipe, in_frame, vf_frame, vf_pp_binary,
-				      &current_stage);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-	ia_css_pipeline_finalize_stages(&pipe->pipeline, pipe->stream->config.continuous);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"create_host_regular_capture_pipeline() leave:\n");
-
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-create_host_capture_pipeline(struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-
-	if (pipe->config.mode == IA_CSS_PIPE_MODE_COPY)
-		err = create_host_isyscopy_capture_pipeline(pipe);
-	else
-		err = create_host_regular_capture_pipeline(pipe);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-
-	return err;
-}
-
-static enum ia_css_err capture_start(
-	struct ia_css_pipe *pipe)
-{
-	struct ia_css_pipeline *me;
-
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	enum sh_css_pipe_config_override copy_ovrd;
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
-	if (pipe == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	me = &pipe->pipeline;
-
-	if ((pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_RAW   ||
-	     pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER   ) &&
-		(pipe->config.mode != IA_CSS_PIPE_MODE_COPY)) {
-		if (copy_on_sp(pipe)) {
-			err = start_copy_on_sp(pipe, &me->out_frame[0]);
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-	/* old isys: need to send_mipi_frames() in all pipe modes */
-	err = send_mipi_frames(pipe);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-#elif defined(USE_INPUT_SYSTEM_VERSION_2401)
-	if (pipe->config.mode != IA_CSS_PIPE_MODE_COPY) {
-		err = send_mipi_frames(pipe);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-
-#endif
-
-	{
-		unsigned int thread_id;
-
-		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-		copy_ovrd = 1 << thread_id;
-
-	}
-	start_pipe(pipe, copy_ovrd, pipe->stream->config.mode);
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
-	/*
-	 * old isys: for IA_CSS_PIPE_MODE_COPY pipe, isys rx has to be configured,
-	 * which is currently done in start_binary(); but COPY pipe contains no binary,
-	 * and does not call start_binary(); so we need to configure the rx here.
-	 */
-	if (pipe->config.mode == IA_CSS_PIPE_MODE_COPY && pipe->stream->reconfigure_css_rx) {
-		ia_css_isys_rx_configure(&pipe->stream->csi_rx_config, pipe->stream->config.mode);
-		pipe->stream->reconfigure_css_rx = false;
-	}
-#endif
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-
-}
-
-static enum ia_css_err
-sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe,
-				  struct ia_css_frame_info *info,
-				  unsigned int idx)
-{
-	assert(pipe != NULL);
-	assert(info != NULL);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-						"sh_css_pipe_get_output_frame_info() enter:\n");
-
-	*info = pipe->output_info[idx];
-	if (copy_on_sp(pipe) &&
-	    pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8) {
-		ia_css_frame_info_init(
-			info,
-			JPEG_BYTES,
-			1,
-			IA_CSS_FRAME_FORMAT_BINARY_8,
-			0);
-	} else if (info->format == IA_CSS_FRAME_FORMAT_RAW ||
-		   info->format == IA_CSS_FRAME_FORMAT_RAW_PACKED) {
-        info->raw_bit_depth =
-            ia_css_pipe_util_pipe_input_format_bpp(pipe);
-
-	}
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-						"sh_css_pipe_get_output_frame_info() leave:\n");
-	return IA_CSS_SUCCESS;
-}
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-void
-ia_css_stream_send_input_frame(const struct ia_css_stream *stream,
-			       const unsigned short *data,
-			       unsigned int width,
-			       unsigned int height)
-{
-	assert(stream != NULL);
-
-	ia_css_inputfifo_send_input_frame(
-			data, width, height,
-			stream->config.channel_id,
-			stream->config.input_config.format,
-			stream->config.pixels_per_clock == 2);
-}
-
-void
-ia_css_stream_start_input_frame(const struct ia_css_stream *stream)
-{
-	assert(stream != NULL);
-
-	ia_css_inputfifo_start_frame(
-			stream->config.channel_id,
-			stream->config.input_config.format,
-			stream->config.pixels_per_clock == 2);
-}
-
-void
-ia_css_stream_send_input_line(const struct ia_css_stream *stream,
-			      const unsigned short *data,
-			      unsigned int width,
-			      const unsigned short *data2,
-			      unsigned int width2)
-{
-	assert(stream != NULL);
-
-	ia_css_inputfifo_send_line(stream->config.channel_id,
-					       data, width, data2, width2);
-}
-
-void
-ia_css_stream_send_input_embedded_line(const struct ia_css_stream *stream,
-		enum atomisp_input_format format,
-		const unsigned short *data,
-		unsigned int width)
-{
-	assert(stream != NULL);
-	if (data == NULL || width == 0)
-		return;
-	ia_css_inputfifo_send_embedded_line(stream->config.channel_id,
-			format, data, width);
-}
-
-void
-ia_css_stream_end_input_frame(const struct ia_css_stream *stream)
-{
-	assert(stream != NULL);
-
-	ia_css_inputfifo_end_frame(stream->config.channel_id);
-}
-#endif
-
-static void
-append_firmware(struct ia_css_fw_info **l, struct ia_css_fw_info *firmware)
-{
-	IA_CSS_ENTER_PRIVATE("l = %p, firmware = %p", l , firmware);
-	if (l == NULL) {
-		IA_CSS_ERROR("NULL fw_info");
-		IA_CSS_LEAVE_PRIVATE("");
-		return;
-	}
-	while (*l)
-		l = &(*l)->next;
-	*l = firmware;
-	/*firmware->next = NULL;*/ /* when multiple acc extensions are loaded, 'next' can be not NULL */
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-static void
-remove_firmware(struct ia_css_fw_info **l, struct ia_css_fw_info *firmware)
-{
-	assert(*l);
-	assert(firmware);
-	(void)l;
-	(void)firmware;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "remove_firmware() enter:\n");
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "remove_firmware() leave:\n");
-	return; /* removing single and multiple firmware is handled in acc_unload_extension() */
-}
-
-static enum ia_css_err upload_isp_code(struct ia_css_fw_info *firmware)
-{
-	hrt_vaddress binary;
-
-	if (firmware == NULL) {
-		IA_CSS_ERROR("NULL input parameter");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	binary = firmware->info.isp.xmem_addr;
-
-	if (!binary) {
-		unsigned size = firmware->blob.size;
-		const unsigned char *blob;
-		const unsigned char *binary_name;
-		binary_name =
-			(const unsigned char *)(IA_CSS_EXT_ISP_PROG_NAME(
-						firmware));
-		blob = binary_name +
-			strlen((const char *)binary_name) +
-			1;
-		binary = sh_css_load_blob(blob, size);
-		firmware->info.isp.xmem_addr = binary;
-	}
-
-	if (!binary)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-acc_load_extension(struct ia_css_fw_info *firmware)
-{
-	enum ia_css_err err;
-	struct ia_css_fw_info *hd = firmware;
-	while (hd){
-		err = upload_isp_code(hd);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-		hd = hd->next;
-	}
-
-	if (firmware == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	firmware->loaded = true;
-	return IA_CSS_SUCCESS;
-}
-
-static void
-acc_unload_extension(struct ia_css_fw_info *firmware)
-{
-	struct ia_css_fw_info *hd = firmware;
-	struct ia_css_fw_info *hdn = NULL;
-
-	if (firmware == NULL) /* should not happen */
-		return;
-	/* unload and remove multiple firmwares */
-	while (hd){
-		hdn = (hd->next) ? &(*hd->next) : NULL;
-		if (hd->info.isp.xmem_addr) {
-			hmm_free(hd->info.isp.xmem_addr);
-			hd->info.isp.xmem_addr = mmgr_NULL;
-		}
-		hd->isp_code = NULL;
-		hd->next = NULL;
-		hd = hdn;
-	}
-
-	firmware->loaded = false;
-}
-/* Load firmware for extension */
-static enum ia_css_err
-ia_css_pipe_load_extension(struct ia_css_pipe *pipe,
-			   struct ia_css_fw_info *firmware)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER_PRIVATE("fw = %p pipe = %p", firmware, pipe);
-
-	if ((firmware == NULL) || (pipe == NULL)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if (firmware->info.isp.type == IA_CSS_ACC_OUTPUT) {
-		if (&pipe->output_stage != NULL)
-			append_firmware(&pipe->output_stage, firmware);
-		else {
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		}
-	}
-	else if (firmware->info.isp.type == IA_CSS_ACC_VIEWFINDER) {
-		if (&pipe->vf_stage != NULL)
-			append_firmware(&pipe->vf_stage, firmware);
-		else {
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		}
-	}
-	err = acc_load_extension(firmware);
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-/* Unload firmware for extension */
-static void
-ia_css_pipe_unload_extension(struct ia_css_pipe *pipe,
-			     struct ia_css_fw_info *firmware)
-{
-	IA_CSS_ENTER_PRIVATE("fw = %p pipe = %p", firmware, pipe);
-
-	if ((firmware == NULL) || (pipe == NULL)) {
-		IA_CSS_ERROR("NULL input parameters");
-		IA_CSS_LEAVE_PRIVATE("");
-		return;
-	}
-
-	if (firmware->info.isp.type == IA_CSS_ACC_OUTPUT)
-		remove_firmware(&pipe->output_stage, firmware);
-	else if (firmware->info.isp.type == IA_CSS_ACC_VIEWFINDER)
-		remove_firmware(&pipe->vf_stage, firmware);
-	acc_unload_extension(firmware);
-
-	IA_CSS_LEAVE_PRIVATE("");
-}
-
-bool
-ia_css_pipeline_uses_params(struct ia_css_pipeline *me)
-{
-	struct ia_css_pipeline_stage *stage;
-
-	assert(me != NULL);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_pipeline_uses_params() enter: me=%p\n", me);
-
-	for (stage = me->stages; stage; stage = stage->next)
-		if (stage->binary_info && stage->binary_info->enable.params) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_pipeline_uses_params() leave: "
-				"return_bool=true\n");
-			return true;
-		}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_pipeline_uses_params() leave: return_bool=false\n");
-	return false;
-}
-
-static enum ia_css_err
-sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline,
-			      const void *acc_fw)
-{
-	struct ia_css_fw_info *fw = (struct ia_css_fw_info *)acc_fw;
-	/* In QoS case, load_extension already called, so skipping */
-	enum ia_css_err	err = IA_CSS_SUCCESS;
-	if (fw->loaded == false)
-		err = acc_load_extension(fw);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"sh_css_pipeline_add_acc_stage() enter: pipeline=%p,"
-		" acc_fw=%p\n", pipeline, acc_fw);
-
-	if (err == IA_CSS_SUCCESS) {
-		struct ia_css_pipeline_stage_desc stage_desc;
-		ia_css_pipe_get_acc_stage_desc(&stage_desc, NULL, fw);
-		err = ia_css_pipeline_create_and_add_stage(pipeline,
-			&stage_desc,
-			NULL);
-	}
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"sh_css_pipeline_add_acc_stage() leave: return_err=%d\n",err);
-	return err;
-}
-
-/*
- * @brief Tag a specific frame in continuous capture.
- * Refer to "sh_css_internal.h" for details.
- */
-enum ia_css_err ia_css_stream_capture_frame(struct ia_css_stream *stream,
-				unsigned int exp_id)
-{
-	struct sh_css_tag_descr tag_descr;
-	uint32_t encoded_tag_descr;
-	enum ia_css_err err;
-
-	assert(stream != NULL);
-	IA_CSS_ENTER("exp_id=%d", exp_id);
-
-	/* Only continuous streams have a tagger */
-	if (exp_id == 0 || !stream->config.continuous) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if (!sh_css_sp_is_running()) {
-		/* SP is not running. The queues are not valid */
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-
-	/* Create the tag descriptor from the parameters */
-	sh_css_create_tag_descr(0, 0, 0, exp_id, &tag_descr);
-	/* Encode the tag descriptor into a 32-bit value */
-	encoded_tag_descr = sh_css_encode_tag_descr(&tag_descr);
-	/* Enqueue the encoded tag to the host2sp queue.
-	 * Note: The pipe and stage IDs for tag_cmd queue are hard-coded to 0
-	 * on both host and the SP side.
-	 * It is mainly because it is enough to have only one tag_cmd queue */
-	err= ia_css_bufq_enqueue_tag_cmd(encoded_tag_descr);
-
-	IA_CSS_LEAVE_ERR(err);
-	return err;
-}
-
-/*
- * @brief Configure the continuous capture.
- * Refer to "sh_css_internal.h" for details.
- */
-enum ia_css_err ia_css_stream_capture(
-	struct ia_css_stream *stream,
-	int num_captures,
-	unsigned int skip,
-	int offset)
-{
-	struct sh_css_tag_descr tag_descr;
-	unsigned int encoded_tag_descr;
-	enum ia_css_err return_err;
-
-	if (stream == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_stream_capture() enter: num_captures=%d,"
-		" skip=%d, offset=%d\n", num_captures, skip,offset);
-
-	/* Check if the tag descriptor is valid */
-	if (num_captures < SH_CSS_MINIMUM_TAG_ID) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				    "ia_css_stream_capture() leave: return_err=%d\n",
-				    IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	/* Create the tag descriptor from the parameters */
-	sh_css_create_tag_descr(num_captures, skip, offset, 0, &tag_descr);
-
-
-	/* Encode the tag descriptor into a 32-bit value */
-	encoded_tag_descr = sh_css_encode_tag_descr(&tag_descr);
-
-	if (!sh_css_sp_is_running()) {
-		/* SP is not running. The queues are not valid */
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			"ia_css_stream_capture() leaving:"
-			"queues unavailable\n");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	}
-
-	/* Enqueue the encoded tag to the host2sp queue.
-	 * Note: The pipe and stage IDs for tag_cmd queue are hard-coded to 0
-	 * on both host and the SP side.
-	 * It is mainly because it is enough to have only one tag_cmd queue */
-	return_err = ia_css_bufq_enqueue_tag_cmd((uint32_t)encoded_tag_descr);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_stream_capture() leave: return_err=%d\n",
-		return_err);
-
-	return return_err;
-}
-
-void ia_css_stream_request_flash(struct ia_css_stream *stream)
-{
-	(void)stream;
-
-	assert(stream != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_request_flash() enter: void\n");
-
-#ifndef ISP2401
-	sh_css_write_host2sp_command(host2sp_cmd_start_flash);
-#else
-	if (sh_css_sp_is_running()) {
-		if (!sh_css_write_host2sp_command(host2sp_cmd_start_flash)) {
-			IA_CSS_ERROR("Call to 'sh-css_write_host2sp_command()' failed");
-			ia_css_debug_dump_sp_sw_debug_info();
-			ia_css_debug_dump_debug_info(NULL);
-		}
-	} else
-		IA_CSS_LOG("SP is not running!");
-
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_stream_request_flash() leave: return_void\n");
-}
-
-static void
-sh_css_init_host_sp_control_vars(void)
-{
-	const struct ia_css_fw_info *fw;
-	unsigned int HIVE_ADDR_ia_css_ispctrl_sp_isp_started;
-
-	unsigned int HIVE_ADDR_host_sp_queues_initialized;
-	unsigned int HIVE_ADDR_sp_sleep_mode;
-	unsigned int HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb;
-#ifndef ISP2401
-	unsigned int HIVE_ADDR_sp_stop_copy_preview;
-#endif
-	unsigned int HIVE_ADDR_host_sp_com;
-	unsigned int o = offsetof(struct host_sp_communication, host2sp_command)
-				/ sizeof(int);
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	unsigned int i;
-#endif
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"sh_css_init_host_sp_control_vars() enter: void\n");
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_ia_css_ispctrl_sp_isp_started = fw->info.sp.isp_started;
-
-	HIVE_ADDR_host_sp_queues_initialized =
-		fw->info.sp.host_sp_queues_initialized;
-	HIVE_ADDR_sp_sleep_mode = fw->info.sp.sleep_mode;
-	HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb = fw->info.sp.invalidate_tlb;
-#ifndef ISP2401
-	HIVE_ADDR_sp_stop_copy_preview = fw->info.sp.stop_copy_preview;
-#endif
-	HIVE_ADDR_host_sp_com = fw->info.sp.host_sp_com;
-
-	(void)HIVE_ADDR_ia_css_ispctrl_sp_isp_started; /* Suppres warnings in CRUN */
-
-	(void)HIVE_ADDR_sp_sleep_mode;
-	(void)HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb;
-#ifndef ISP2401
-	(void)HIVE_ADDR_sp_stop_copy_preview;
-#endif
-	(void)HIVE_ADDR_host_sp_com;
-
-	sp_dmem_store_uint32(SP0_ID,
-		(unsigned int)sp_address_of(ia_css_ispctrl_sp_isp_started),
-		(uint32_t)(0));
-
-	sp_dmem_store_uint32(SP0_ID,
-		(unsigned int)sp_address_of(host_sp_queues_initialized),
-		(uint32_t)(0));
-	sp_dmem_store_uint32(SP0_ID,
-		(unsigned int)sp_address_of(sp_sleep_mode),
-		(uint32_t)(0));
-	sp_dmem_store_uint32(SP0_ID,
-		(unsigned int)sp_address_of(ia_css_dmaproxy_sp_invalidate_tlb),
-		(uint32_t)(false));
-#ifndef ISP2401
-	sp_dmem_store_uint32(SP0_ID,
-		(unsigned int)sp_address_of(sp_stop_copy_preview),
-		my_css.stop_copy_preview?(uint32_t)(1):(uint32_t)(0));
-#endif
-	store_sp_array_uint(host_sp_com, o, host2sp_cmd_ready);
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	for (i = 0; i < N_CSI_PORTS; i++) {
-		sh_css_update_host2sp_num_mipi_frames
-			(my_css.num_mipi_frames[i]);
-	}
-#endif
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"sh_css_init_host_sp_control_vars() leave: return_void\n");
-}
-
-/*
- * create the internal structures and fill in the configuration data
- */
-void ia_css_pipe_config_defaults(struct ia_css_pipe_config *pipe_config)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipe_config_defaults()\n");
-	*pipe_config = DEFAULT_PIPE_CONFIG;
-}
-
-void
-ia_css_pipe_extra_config_defaults(struct ia_css_pipe_extra_config *extra_config)
-{
-	if (extra_config == NULL) {
-		IA_CSS_ERROR("NULL input parameter");
-		return;
-	}
-
-	extra_config->enable_raw_binning = false;
-	extra_config->enable_yuv_ds = false;
-	extra_config->enable_high_speed = false;
-	extra_config->enable_dvs_6axis = false;
-	extra_config->enable_reduced_pipe = false;
-	extra_config->disable_vf_pp = false;
-	extra_config->enable_fractional_ds = false;
-}
-
-void ia_css_stream_config_defaults(struct ia_css_stream_config *stream_config)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_config_defaults()\n");
-	assert(stream_config != NULL);
-	memset(stream_config, 0, sizeof(*stream_config));
-	stream_config->online = true;
-	stream_config->left_padding = -1;
-	stream_config->pixels_per_clock = 1;
-	/* temporary default value for backwards compatibility.
-	 * This field used to be hardcoded within CSS but this has now
-	 * been moved to the stream_config struct. */
-	stream_config->source.port.rxcount = 0x04040404;
-}
-
-static enum ia_css_err
-ia_css_acc_pipe_create(struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	if (pipe == NULL) {
-		IA_CSS_ERROR("NULL input parameter");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	/* There is not meaning for num_execs = 0 semantically. Run atleast once. */
-	if (pipe->config.acc_num_execs == 0)
-		pipe->config.acc_num_execs = 1;
-
-	if (pipe->config.acc_extension) {
-		err = ia_css_pipe_load_extension(pipe, pipe->config.acc_extension);
-	}
-
-	return err;
-}
-
-enum ia_css_err
-ia_css_pipe_create(const struct ia_css_pipe_config *config,
-		   struct ia_css_pipe **pipe)
-{
-#ifndef ISP2401
-	if (config == NULL)
-#else
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	IA_CSS_ENTER_PRIVATE("config = %p, pipe = %p", config, pipe);
-
-	if (config == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-#endif
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-#ifndef ISP2401
-	if (pipe == NULL)
-#else
-	}
-	if (pipe == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-#endif
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-#ifndef ISP2401
-	return ia_css_pipe_create_extra(config, NULL, pipe);
-#else
-	}
-
-	err = ia_css_pipe_create_extra(config, NULL, pipe);
-
-	if(err == IA_CSS_SUCCESS) {
-		IA_CSS_LOG("pipe created successfully = %p", *pipe);
-	}
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-
-	return err;
-#endif
-}
-
-enum ia_css_err
-ia_css_pipe_create_extra(const struct ia_css_pipe_config *config,
-			 const struct ia_css_pipe_extra_config *extra_config,
-			 struct ia_css_pipe **pipe)
-{
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
-	struct ia_css_pipe *internal_pipe = NULL;
-	unsigned int i;
-
-	IA_CSS_ENTER_PRIVATE("config = %p, extra_config = %p and pipe = %p", config, extra_config, pipe);
-
-	/* do not allow to create more than the maximum limit */
-	if (my_css.pipe_counter >= IA_CSS_PIPELINE_NUM_MAX) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_RESOURCE_EXHAUSTED);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if ((pipe == NULL) || (config == NULL)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	ia_css_debug_dump_pipe_config(config);
-	ia_css_debug_dump_pipe_extra_config(extra_config);
-
-	err = create_pipe(config->mode, &internal_pipe, false);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-
-	/* now we have a pipe structure to fill */
-	internal_pipe->config = *config;
-	if (extra_config)
-		internal_pipe->extra_config = *extra_config;
-	else
-		ia_css_pipe_extra_config_defaults(&internal_pipe->extra_config);
-
-	if (config->mode == IA_CSS_PIPE_MODE_ACC) {
-		/* Temporary hack to migrate acceleration to CSS 2.0.
-		 * In the future the code for all pipe types should be
-		 * unified. */
-		*pipe = internal_pipe;
-		if (!internal_pipe->config.acc_extension &&
-			internal_pipe->config.num_acc_stages == 0){ /* if no acc binary and no standalone stage */
-			*pipe = NULL;
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-			return IA_CSS_SUCCESS;
-		}
-		return ia_css_acc_pipe_create(internal_pipe);
-	}
-
-	/* Use config value when dvs_frame_delay setting equal to 2, otherwise always 1 by default */
-	if (internal_pipe->config.dvs_frame_delay == IA_CSS_FRAME_DELAY_2)
-		internal_pipe->dvs_frame_delay = 2;
-	else
-		internal_pipe->dvs_frame_delay = 1;
-
-
-	/* we still keep enable_raw_binning for backward compatibility, for any new
-	   fractional bayer downscaling, we should use bayer_ds_out_res. if both are
-	   specified, bayer_ds_out_res will take precedence.if none is specified, we
-	   set bayer_ds_out_res equal to IF output resolution(IF may do cropping on
-	   sensor output) or use default decimation factor 1. */
-	if (internal_pipe->extra_config.enable_raw_binning &&
-		 internal_pipe->config.bayer_ds_out_res.width) {
-		/* fill some code here, if no code is needed, please remove it during integration */
-	}
-
-	/* YUV downscaling */
-	if ((internal_pipe->config.vf_pp_in_res.width ||
-		 internal_pipe->config.capt_pp_in_res.width)) {
-		enum ia_css_frame_format format;
-		if (internal_pipe->config.vf_pp_in_res.width) {
-			format = IA_CSS_FRAME_FORMAT_YUV_LINE;
-			ia_css_frame_info_init(
-				&internal_pipe->vf_yuv_ds_input_info,
-				internal_pipe->config.vf_pp_in_res.width,
-				internal_pipe->config.vf_pp_in_res.height,
-				format, 0);
-		}
-		if (internal_pipe->config.capt_pp_in_res.width) {
-			format = IA_CSS_FRAME_FORMAT_YUV420;
-			ia_css_frame_info_init(
-				&internal_pipe->out_yuv_ds_input_info,
-				internal_pipe->config.capt_pp_in_res.width,
-				internal_pipe->config.capt_pp_in_res.height,
-				format, 0);
-		}
-	}
-	if (internal_pipe->config.vf_pp_in_res.width &&
-	    internal_pipe->config.mode == IA_CSS_PIPE_MODE_PREVIEW) {
-		ia_css_frame_info_init(
-				&internal_pipe->vf_yuv_ds_input_info,
-				internal_pipe->config.vf_pp_in_res.width,
-				internal_pipe->config.vf_pp_in_res.height,
-				IA_CSS_FRAME_FORMAT_YUV_LINE, 0);
-	}
-	/* handle bayer downscaling output info */
-	if (internal_pipe->config.bayer_ds_out_res.width) {
-			ia_css_frame_info_init(
-				&internal_pipe->bds_output_info,
-				internal_pipe->config.bayer_ds_out_res.width,
-				internal_pipe->config.bayer_ds_out_res.height,
-				IA_CSS_FRAME_FORMAT_RAW, 0);
-	}
-
-	/* handle output info, assume always needed */
-	for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-		if (internal_pipe->config.output_info[i].res.width) {
-			err = sh_css_pipe_configure_output(
-					internal_pipe,
-					internal_pipe->config.output_info[i].res.width,
-					internal_pipe->config.output_info[i].res.height,
-					internal_pipe->config.output_info[i].padded_width,
-					internal_pipe->config.output_info[i].format,
-					i);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				sh_css_free(internal_pipe);
-				internal_pipe = NULL;
-				return err;
-			}
-		}
-
-		/* handle vf output info, when configured */
-		internal_pipe->enable_viewfinder[i] = (internal_pipe->config.vf_output_info[i].res.width != 0);
-		if (internal_pipe->config.vf_output_info[i].res.width) {
-			err = sh_css_pipe_configure_viewfinder(
-					internal_pipe,
-					internal_pipe->config.vf_output_info[i].res.width,
-					internal_pipe->config.vf_output_info[i].res.height,
-					internal_pipe->config.vf_output_info[i].padded_width,
-					internal_pipe->config.vf_output_info[i].format,
-					i);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				sh_css_free(internal_pipe);
-				internal_pipe = NULL;
-				return err;
-			}
-		}
-	}
-	if (internal_pipe->config.acc_extension) {
-		err = ia_css_pipe_load_extension(internal_pipe,
-			internal_pipe->config.acc_extension);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			sh_css_free(internal_pipe);
-			return err;
-		}
-	}
-	/* set all info to zeroes first */
-	memset(&internal_pipe->info, 0, sizeof(internal_pipe->info));
-
-	/* all went well, return the pipe */
-	*pipe = internal_pipe;
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-
-enum ia_css_err
-ia_css_pipe_get_info(const struct ia_css_pipe *pipe,
-		     struct ia_css_pipe_info *pipe_info)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-		"ia_css_pipe_get_info()\n");
-	assert(pipe_info != NULL);
-	if (pipe_info == NULL) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-			"ia_css_pipe_get_info: pipe_info cannot be NULL\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	if (pipe == NULL || pipe->stream == NULL) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-			"ia_css_pipe_get_info: ia_css_stream_create needs to"
-			" be called before ia_css_[stream/pipe]_get_info\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	/* we succeeded return the info */
-	*pipe_info = pipe->info;
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipe_get_info() leave\n");
-	return IA_CSS_SUCCESS;
-}
-
-bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info)
-{
-	unsigned int i;
-
-	if (pipe_info != NULL) {
-		for (i = 0; i < IA_CSS_DVS_STAT_NUM_OF_LEVELS; i++) {
-			if (pipe_info->grid_info.dvs_grid.dvs_stat_grid_info.grd_cfg[i].grd_start.enable)
-				return true;
-		}
-	}
-
-	return false;
-}
-
-#ifdef ISP2401
-enum ia_css_err
-ia_css_pipe_override_frame_format(struct ia_css_pipe *pipe,
-				int pin_index,
-				enum ia_css_frame_format new_format)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER_PRIVATE("pipe = %p, pin_index = %d, new_formats = %d", pipe, pin_index, new_format);
-
-	if (NULL == pipe) {
-		IA_CSS_ERROR("pipe is not set");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-	if (0 != pin_index && 1 != pin_index) {
-		IA_CSS_ERROR("pin index is not valid");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-	if (IA_CSS_FRAME_FORMAT_NV12_TILEY != new_format) {
-		IA_CSS_ERROR("new format is not valid");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	} else {
-		err = ia_css_pipe_check_format(pipe, new_format);
-		if (IA_CSS_SUCCESS == err) {
-			if (pin_index == 0) {
-				pipe->output_info[0].format = new_format;
-			} else {
-				pipe->vf_output_info[0].format = new_format;
-			}
-		}
-	}
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-/* Configuration of INPUT_SYSTEM_VERSION_2401 is done on SP */
-static enum ia_css_err
-ia_css_stream_configure_rx(struct ia_css_stream *stream)
-{
-	struct ia_css_input_port *config;
-	assert(stream != NULL);
-
-	config = &stream->config.source.port;
-/* AM: this code is not reliable, especially for 2400 */
-	if (config->num_lanes == 1)
-		stream->csi_rx_config.mode = MONO_1L_1L_0L;
-	else if (config->num_lanes == 2)
-		stream->csi_rx_config.mode = MONO_2L_1L_0L;
-	else if (config->num_lanes == 3)
-		stream->csi_rx_config.mode = MONO_3L_1L_0L;
-	else if (config->num_lanes == 4)
-		stream->csi_rx_config.mode = MONO_4L_1L_0L;
-	else if (config->num_lanes != 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	if (config->port > MIPI_PORT2_ID)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	stream->csi_rx_config.port =
-		ia_css_isys_port_to_mipi_port(config->port);
-	stream->csi_rx_config.timeout    = config->timeout;
-	stream->csi_rx_config.initcount  = 0;
-	stream->csi_rx_config.synccount  = 0x28282828;
-	stream->csi_rx_config.rxcount    = config->rxcount;
-	if (config->compression.type == IA_CSS_CSI2_COMPRESSION_TYPE_NONE)
-		stream->csi_rx_config.comp = MIPI_PREDICTOR_NONE;
-	else {
-		/* not implemented yet, requires extension of the rx_cfg_t
-		 * struct */
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	stream->csi_rx_config.is_two_ppc = (stream->config.pixels_per_clock == 2);
-	stream->reconfigure_css_rx = true;
-	return IA_CSS_SUCCESS;
-}
-#endif
-
-static struct ia_css_pipe *
-find_pipe(struct ia_css_pipe *pipes[],
-		unsigned int num_pipes,
-		enum ia_css_pipe_mode mode,
-		bool copy_pipe)
-{
-	unsigned i;
-	assert(pipes != NULL);
-	for (i = 0; i < num_pipes; i++) {
-		assert(pipes[i] != NULL);
-		if (pipes[i]->config.mode != mode)
-			continue;
-		if (copy_pipe && pipes[i]->mode != IA_CSS_PIPE_ID_COPY)
-			continue;
-		return pipes[i];
-	}
-	return NULL;
-}
-
-static enum ia_css_err
-ia_css_acc_stream_create(struct ia_css_stream *stream)
-{
-	int i;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	assert(stream != NULL);
-	IA_CSS_ENTER_PRIVATE("stream = %p", stream);
-
-	if (stream == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	for (i = 0;  i < stream->num_pipes; i++) {
-		struct ia_css_pipe *pipe = stream->pipes[i];
-		assert(pipe != NULL);
-		if (pipe == NULL) {
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-		}
-
-		pipe->stream = stream;
-	}
-
-	/* Map SP threads before doing anything. */
-	err = map_sp_threads(stream, true);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-
-	for (i = 0;  i < stream->num_pipes; i++) {
-		struct ia_css_pipe *pipe = stream->pipes[i];
-		assert(pipe != NULL);
-		ia_css_pipe_map_queue(pipe, true);
-	}
-
-	err = create_host_pipeline_structure(stream);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-
-	stream->started = false;
-
-
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-metadata_info_init(const struct ia_css_metadata_config *mdc,
-		   struct ia_css_metadata_info *md)
-{
-	/* Either both width and height should be set or neither */
-	if ((mdc->resolution.height > 0) ^ (mdc->resolution.width > 0))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	md->resolution = mdc->resolution;
-        /* We round up the stride to a multiple of the width
-         * of the port going to DDR, this is a HW requirements (DMA). */
-	md->stride = CEIL_MUL(mdc->resolution.width, HIVE_ISP_DDR_WORD_BYTES);
-	md->size = mdc->resolution.height * md->stride;
-	return IA_CSS_SUCCESS;
-}
-
-#ifdef ISP2401
-static enum ia_css_err check_pipe_resolutions(const struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	if (!pipe || !pipe->stream) {
-		IA_CSS_ERROR("null arguments");
-		err = IA_CSS_ERR_INTERNAL_ERROR;
-		goto EXIT;
-	}
-
-	if (ia_css_util_check_res(pipe->config.input_effective_res.width,
-				pipe->config.input_effective_res.height) != IA_CSS_SUCCESS) {
-		IA_CSS_ERROR("effective resolution not supported");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		goto EXIT;
-	}
-	if (!ia_css_util_resolution_is_zero(pipe->stream->config.input_config.input_res)) {
-		if (!ia_css_util_res_leq(pipe->config.input_effective_res,
-						pipe->stream->config.input_config.input_res)) {
-			IA_CSS_ERROR("effective resolution is larger than input resolution");
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
-			goto EXIT;
-		}
-	}
-	if (!ia_css_util_resolution_is_even(pipe->config.output_info[0].res)) {
-		IA_CSS_ERROR("output resolution must be even");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		goto EXIT;
-	}
-	if (!ia_css_util_resolution_is_even(pipe->config.vf_output_info[0].res)) {
-		IA_CSS_ERROR("VF resolution must be even");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		goto EXIT;
-	}
-EXIT:
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-#endif
-
-enum ia_css_err
-ia_css_stream_create(const struct ia_css_stream_config *stream_config,
-					 int num_pipes,
-					 struct ia_css_pipe *pipes[],
-					 struct ia_css_stream **stream)
-{
-	struct ia_css_pipe *curr_pipe;
-	struct ia_css_stream *curr_stream = NULL;
-	bool spcopyonly;
-	bool sensor_binning_changed;
-	int i, j;
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
-	struct ia_css_metadata_info md_info;
-#ifndef ISP2401
-	struct ia_css_resolution effective_res;
-#else
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	bool aspect_ratio_crop_enabled = false;
-#endif
-#endif
-
-	IA_CSS_ENTER("num_pipes=%d", num_pipes);
-	ia_css_debug_dump_stream_config(stream_config, num_pipes);
-
-	/* some checks */
-	if (num_pipes == 0 ||
-		stream == NULL ||
-		pipes == NULL) {
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-	/* We don't support metadata for JPEG stream, since they both use str2mem */
-	if (stream_config->input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8 &&
-	    stream_config->metadata_config.resolution.height > 0) {
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-#endif
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	if (stream_config->online && stream_config->pack_raw_pixels) {
-		IA_CSS_LOG("online and pack raw is invalid on input system 2401");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-#endif
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	ia_css_debug_pipe_graph_dump_stream_config(stream_config);
-
-	/* check if mipi size specified */
-	if (stream_config->mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	if (!stream_config->online)
-#endif
-	{
-		unsigned int port = (unsigned int) stream_config->source.port.port;
-		if (port >= N_MIPI_PORT_ID) {
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
-			IA_CSS_LEAVE_ERR(err);
-			return err;
-		}
-
-		if (my_css.size_mem_words != 0){
-			my_css.mipi_frame_size[port] = my_css.size_mem_words;
-		} else if (stream_config->mipi_buffer_config.size_mem_words != 0) {
-			my_css.mipi_frame_size[port] = stream_config->mipi_buffer_config.size_mem_words;
-		} else {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_stream_create() exit: error, need to set mipi frame size.\n");
-			assert(stream_config->mipi_buffer_config.size_mem_words != 0);
-			err = IA_CSS_ERR_INTERNAL_ERROR;
-			IA_CSS_LEAVE_ERR(err);
-			return err;
-		}
-
-		if (my_css.size_mem_words != 0) {
-			my_css.num_mipi_frames[port] = 2; /* Temp change: Default for backwards compatibility. */
-		} else if (stream_config->mipi_buffer_config.nof_mipi_buffers != 0) {
-			my_css.num_mipi_frames[port] = stream_config->mipi_buffer_config.nof_mipi_buffers;
-		} else {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-				"ia_css_stream_create() exit: error, need to set number of mipi frames.\n");
-			assert(stream_config->mipi_buffer_config.nof_mipi_buffers != 0);
-			err = IA_CSS_ERR_INTERNAL_ERROR;
-			IA_CSS_LEAVE_ERR(err);
-			return err;
-		}
-
-	}
-#endif
-
-	/* Currently we only supported metadata up to a certain size. */
-	err = metadata_info_init(&stream_config->metadata_config, &md_info);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-
-	/* allocate the stream instance */
-	curr_stream = kmalloc(sizeof(struct ia_css_stream), GFP_KERNEL);
-	if (!curr_stream) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-	/* default all to 0 */
-	memset(curr_stream, 0, sizeof(struct ia_css_stream));
-	curr_stream->info.metadata_info = md_info;
-
-	/* allocate pipes */
-	curr_stream->num_pipes = num_pipes;
-	curr_stream->pipes = kzalloc(num_pipes * sizeof(struct ia_css_pipe *), GFP_KERNEL);
-	if (!curr_stream->pipes) {
-		curr_stream->num_pipes = 0;
-		kfree(curr_stream);
-		curr_stream = NULL;
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-	/* store pipes */
-	spcopyonly = (num_pipes == 1) && (pipes[0]->config.mode == IA_CSS_PIPE_MODE_COPY);
-	for (i = 0; i < num_pipes; i++)
-		curr_stream->pipes [i] = pipes[i];
-	curr_stream->last_pipe = curr_stream->pipes[0];
-	/* take over stream config */
-	curr_stream->config = *stream_config;
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401) && defined(CSI2P_DISABLE_ISYS2401_ONLINE_MODE)
-	if (stream_config->mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR &&
-		stream_config->online)
-		curr_stream->config.online = false;
-#endif
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	if (curr_stream->config.online) {
-		curr_stream->config.source.port.num_lanes = stream_config->source.port.num_lanes;
-		curr_stream->config.mode =  IA_CSS_INPUT_MODE_BUFFERED_SENSOR;
-	}
-#endif
-	/* in case driver doesn't configure init number of raw buffers, configure it here */
-	if (curr_stream->config.target_num_cont_raw_buf == 0)
-		curr_stream->config.target_num_cont_raw_buf = NUM_CONTINUOUS_FRAMES;
-	if (curr_stream->config.init_num_cont_raw_buf == 0)
-		curr_stream->config.init_num_cont_raw_buf = curr_stream->config.target_num_cont_raw_buf;
-
-	/* Enable locking & unlocking of buffers in RAW buffer pool */
-	if (curr_stream->config.ia_css_enable_raw_buffer_locking)
-		sh_css_sp_configure_enable_raw_pool_locking(
-					curr_stream->config.lock_all);
-
-	/* copy mode specific stuff */
-	switch (curr_stream->config.mode) {
-		case IA_CSS_INPUT_MODE_SENSOR:
-		case IA_CSS_INPUT_MODE_BUFFERED_SENSOR:
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-		ia_css_stream_configure_rx(curr_stream);
-#endif
-		break;
-	case IA_CSS_INPUT_MODE_TPG:
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-		IA_CSS_LOG("tpg_configuration: x_mask=%d, y_mask=%d, x_delta=%d, y_delta=%d, xy_mask=%d",
-			curr_stream->config.source.tpg.x_mask,
-			curr_stream->config.source.tpg.y_mask,
-			curr_stream->config.source.tpg.x_delta,
-			curr_stream->config.source.tpg.y_delta,
-			curr_stream->config.source.tpg.xy_mask);
-
-		sh_css_sp_configure_tpg(
-			curr_stream->config.source.tpg.x_mask,
-			curr_stream->config.source.tpg.y_mask,
-			curr_stream->config.source.tpg.x_delta,
-			curr_stream->config.source.tpg.y_delta,
-			curr_stream->config.source.tpg.xy_mask);
-#endif
-		break;
-	case IA_CSS_INPUT_MODE_PRBS:
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-		IA_CSS_LOG("mode prbs");
-		sh_css_sp_configure_prbs(curr_stream->config.source.prbs.seed);
-#endif
-		break;
-	case IA_CSS_INPUT_MODE_MEMORY:
-		IA_CSS_LOG("mode memory");
-		curr_stream->reconfigure_css_rx = false;
-		break;
-	default:
-		IA_CSS_LOG("mode sensor/default");
-	}
-
-#ifdef ISP2401
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	err = aspect_ratio_crop_init(curr_stream,
-				pipes,
-				&aspect_ratio_crop_enabled);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-#endif
-
-#endif
-	for (i = 0; i < num_pipes; i++) {
-#ifdef ISP2401
-		struct ia_css_resolution effective_res;
-#endif
-		curr_pipe = pipes[i];
-		/* set current stream */
-		curr_pipe->stream = curr_stream;
-		/* take over effective info */
-
-		effective_res = curr_pipe->config.input_effective_res;
-		if (effective_res.height == 0 || effective_res.width == 0) {
-			effective_res = curr_pipe->stream->config.input_config.effective_res;
-#ifdef ISP2401
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-			/* The aspect ratio cropping is currently only
-			 * supported on the new input system. */
-			if (aspect_ratio_crop_check(aspect_ratio_crop_enabled, curr_pipe)) {
-
-				struct ia_css_resolution crop_res;
-
-				err = aspect_ratio_crop(curr_pipe, &crop_res);
-				if (err == IA_CSS_SUCCESS) {
-					effective_res = crop_res;
-				} else {
-					/* in case of error fallback to default
-					 * effective resolution from driver. */
-					IA_CSS_LOG("aspect_ratio_crop() failed with err(%d)", err);
-				}
-			}
-#endif
-#endif
-			curr_pipe->config.input_effective_res = effective_res;
-		}
-		IA_CSS_LOG("effective_res=%dx%d",
-					effective_res.width,
-					effective_res.height);
-	}
-
-#ifdef ISP2401
-	for (i = 0; i < num_pipes; i++) {
-		if (pipes[i]->config.mode != IA_CSS_PIPE_MODE_ACC &&
-			pipes[i]->config.mode != IA_CSS_PIPE_MODE_COPY) {
-			err = check_pipe_resolutions(pipes[i]);
-			if (err != IA_CSS_SUCCESS) {
-				goto ERR;
-			}
-		}
-	}
-
-#endif
-	err = ia_css_stream_isp_parameters_init(curr_stream);
-	if (err != IA_CSS_SUCCESS)
-		goto ERR;
-	IA_CSS_LOG("isp_params_configs: %p", curr_stream->isp_params_configs);
-
-	if (num_pipes == 1 && pipes[0]->config.mode == IA_CSS_PIPE_MODE_ACC) {
-		*stream = curr_stream;
-		err = ia_css_acc_stream_create(curr_stream);
-		goto ERR;
-	}
-	/* sensor binning */
-	if (!spcopyonly){
-		sensor_binning_changed =
-			sh_css_params_set_binning_factor(curr_stream, curr_stream->config.sensor_binning_factor);
-	} else {
-		sensor_binning_changed = false;
-	}
-
-	IA_CSS_LOG("sensor_binning=%d, changed=%d",
-		curr_stream->config.sensor_binning_factor, sensor_binning_changed);
-	/* loop over pipes */
-	IA_CSS_LOG("num_pipes=%d", num_pipes);
-	curr_stream->cont_capt = false;
-	/* Temporary hack: we give the preview pipe a reference to the capture
-	 * pipe in continuous capture mode. */
-	if (curr_stream->config.continuous) {
-		/* Search for the preview pipe and create the copy pipe */
-		struct ia_css_pipe *preview_pipe;
-		struct ia_css_pipe *video_pipe;
-		struct ia_css_pipe *acc_pipe;
-		struct ia_css_pipe *capture_pipe = NULL;
-		struct ia_css_pipe *copy_pipe = NULL;
-
-		if (num_pipes >= 2) {
-			curr_stream->cont_capt = true;
-			curr_stream->disable_cont_vf = curr_stream->config.disable_cont_viewfinder;
-#ifndef ISP2401
-			curr_stream->stop_copy_preview = my_css.stop_copy_preview;
-#endif
-		}
-
-		/* Create copy pipe here, since it may not be exposed to the driver */
-		preview_pipe = find_pipe(pipes, num_pipes,
-						IA_CSS_PIPE_MODE_PREVIEW, false);
-		video_pipe = find_pipe(pipes, num_pipes,
-						IA_CSS_PIPE_MODE_VIDEO, false);
-		acc_pipe = find_pipe(pipes, num_pipes,
-						IA_CSS_PIPE_MODE_ACC, false);
-		if (acc_pipe && num_pipes == 2 && curr_stream->cont_capt == true)
-			curr_stream->cont_capt = false; /* preview + QoS case will not need cont_capt switch */
-		if (curr_stream->cont_capt == true) {
-			capture_pipe = find_pipe(pipes, num_pipes,
-						IA_CSS_PIPE_MODE_CAPTURE, false);
-			if (capture_pipe == NULL) {
-				err = IA_CSS_ERR_INTERNAL_ERROR;
-				goto ERR;
-			}
-		}
-		/* We do not support preview and video pipe at the same time */
-		if (preview_pipe && video_pipe) {
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
-			goto ERR;
-		}
-
-		if (preview_pipe && !preview_pipe->pipe_settings.preview.copy_pipe) {
-			err = create_pipe(IA_CSS_PIPE_MODE_CAPTURE, &copy_pipe, true);
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-			ia_css_pipe_config_defaults(&copy_pipe->config);
-			preview_pipe->pipe_settings.preview.copy_pipe = copy_pipe;
-			copy_pipe->stream = curr_stream;
-		}
-		if (preview_pipe && (curr_stream->cont_capt == true)) {
-			preview_pipe->pipe_settings.preview.capture_pipe = capture_pipe;
-		}
-		if (video_pipe && !video_pipe->pipe_settings.video.copy_pipe) {
-			err = create_pipe(IA_CSS_PIPE_MODE_CAPTURE, &copy_pipe, true);
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-			ia_css_pipe_config_defaults(&copy_pipe->config);
-			video_pipe->pipe_settings.video.copy_pipe = copy_pipe;
-			copy_pipe->stream = curr_stream;
-		}
-		if (video_pipe && (curr_stream->cont_capt == true)) {
-			video_pipe->pipe_settings.video.capture_pipe = capture_pipe;
-		}
-		if (preview_pipe && acc_pipe) {
-			preview_pipe->pipe_settings.preview.acc_pipe = acc_pipe;
-		}
-	}
-	for (i = 0; i < num_pipes; i++) {
-		curr_pipe = pipes[i];
-		/* set current stream */
-		curr_pipe->stream = curr_stream;
-#ifndef ISP2401
-		/* take over effective info */
-
-		effective_res = curr_pipe->config.input_effective_res;
-		err = ia_css_util_check_res(
-					effective_res.width,
-					effective_res.height);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-#endif
-		/* sensor binning per pipe */
-		if (sensor_binning_changed)
-			sh_css_pipe_free_shading_table(curr_pipe);
-	}
-
-	/* now pipes have been configured, info should be available */
-	for (i = 0; i < num_pipes; i++) {
-		struct ia_css_pipe_info *pipe_info = NULL;
-		curr_pipe = pipes[i];
-
-		err = sh_css_pipe_load_binaries(curr_pipe);
-		if (err != IA_CSS_SUCCESS)
-			goto ERR;
-
-		/* handle each pipe */
-		pipe_info = &curr_pipe->info;
-		for (j = 0; j < IA_CSS_PIPE_MAX_OUTPUT_STAGE; j++) {
-			err = sh_css_pipe_get_output_frame_info(curr_pipe,
-					&pipe_info->output_info[j], j);
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-		}
-#ifdef ISP2401
-		pipe_info->output_system_in_res_info = curr_pipe->config.output_system_in_res;
-#endif
-		if (!spcopyonly){
-			err = sh_css_pipe_get_shading_info(curr_pipe,
-#ifndef ISP2401
-						&pipe_info->shading_info);
-#else
-					&pipe_info->shading_info, &curr_pipe->config);
-#endif
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-			err = sh_css_pipe_get_grid_info(curr_pipe,
-						&pipe_info->grid_info);
-			if (err != IA_CSS_SUCCESS)
-				goto ERR;
-			for (j = 0; j < IA_CSS_PIPE_MAX_OUTPUT_STAGE; j++) {
-				sh_css_pipe_get_viewfinder_frame_info(curr_pipe,
-						&pipe_info->vf_output_info[j], j);
-				if (err != IA_CSS_SUCCESS)
-					goto ERR;
-			}
-		}
-
-		my_css.active_pipes[ia_css_pipe_get_pipe_num(curr_pipe)] = curr_pipe;
-	}
-
-	curr_stream->started = false;
-
-	/* Map SP threads before doing anything. */
-	err = map_sp_threads(curr_stream, true);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LOG("map_sp_threads: return_err=%d", err);
-		goto ERR;
-	}
-
-	for (i = 0; i < num_pipes; i++) {
-		curr_pipe = pipes[i];
-		ia_css_pipe_map_queue(curr_pipe, true);
-	}
-
-	/* Create host side pipeline objects without stages */
-	err = create_host_pipeline_structure(curr_stream);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LOG("create_host_pipeline_structure: return_err=%d", err);
-		goto ERR;
-	}
-
-	/* assign curr_stream */
-	*stream = curr_stream;
-
-ERR:
-#ifndef ISP2401
-	if (err == IA_CSS_SUCCESS)
-	{
-		/* working mode: enter into the seed list */
-		if (my_css_save.mode == sh_css_mode_working) {
-			for (i = 0; i < MAX_ACTIVE_STREAMS; i++)
-				if (!my_css_save.stream_seeds[i].stream) {
-					IA_CSS_LOG("entered stream into loc=%d", i);
-					my_css_save.stream_seeds[i].orig_stream = stream;
-					my_css_save.stream_seeds[i].stream = curr_stream;
-					my_css_save.stream_seeds[i].num_pipes = num_pipes;
-					my_css_save.stream_seeds[i].stream_config = *stream_config;
-					for (j = 0; j < num_pipes; j++) {
-						my_css_save.stream_seeds[i].pipe_config[j] = pipes[j]->config;
-						my_css_save.stream_seeds[i].pipes[j] = pipes[j];
-						my_css_save.stream_seeds[i].orig_pipes[j] = &pipes[j];
-					}
-					break;
-			}
-		}
-#else
-	if (err == IA_CSS_SUCCESS) {
-		err = ia_css_save_stream(curr_stream);
-#endif
-	} else {
-		ia_css_stream_destroy(curr_stream);
-	}
-#ifndef ISP2401
-	IA_CSS_LEAVE("return_err=%d mode=%d", err, my_css_save.mode);
-#else
-	IA_CSS_LEAVE("return_err=%d", err);
-#endif
-	return err;
-}
-
-enum ia_css_err
-ia_css_stream_destroy(struct ia_css_stream *stream)
-{
-	int i;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-#ifdef ISP2401
-	enum ia_css_err err1 = IA_CSS_SUCCESS;
-	enum ia_css_err err2 = IA_CSS_SUCCESS;
-#endif
-
-	IA_CSS_ENTER_PRIVATE("stream = %p", stream);
-	if (stream == NULL) {
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-
-	ia_css_stream_isp_parameters_uninit(stream);
-
-	if ((stream->last_pipe != NULL) &&
-		ia_css_pipeline_is_mapped(stream->last_pipe->pipe_num)) {
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-		for (i = 0; i < stream->num_pipes; i++) {
-			struct ia_css_pipe *entry = stream->pipes[i];
-			unsigned int sp_thread_id;
-			struct sh_css_sp_pipeline_terminal *sp_pipeline_input_terminal;
-
-			assert(entry != NULL);
-			if (entry != NULL) {
-				/* get the SP thread id */
-				if (ia_css_pipeline_get_sp_thread_id(
-					ia_css_pipe_get_pipe_num(entry), &sp_thread_id) != true)
-					return IA_CSS_ERR_INTERNAL_ERROR;
-				/* get the target input terminal */
-				sp_pipeline_input_terminal =
-					&(sh_css_sp_group.pipe_io[sp_thread_id].input);
-
-				for (i = 0; i < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH; i++) {
-					ia_css_isys_stream_h isys_stream =
-						&(sp_pipeline_input_terminal->context.virtual_input_system_stream[i]);
-					if (stream->config.isys_config[i].valid && isys_stream->valid)
-						ia_css_isys_stream_destroy(isys_stream);
-				}
-			}
-		}
-#ifndef ISP2401
-		if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) {
-#else
-		if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR ||
-			stream->config.mode == IA_CSS_INPUT_MODE_TPG ||
-			stream->config.mode == IA_CSS_INPUT_MODE_PRBS) {
-#endif
-			for (i = 0; i < stream->num_pipes; i++) {
-				struct ia_css_pipe *entry = stream->pipes[i];
-				/* free any mipi frames that are remaining:
-				 * some test stream create-destroy cycles do not generate output frames
-				 * and the mipi buffer is not freed in the deque function
-				 */
-				if (entry != NULL)
-					free_mipi_frames(entry);
-			}
-		}
-		stream_unregister_with_csi_rx(stream);
-#endif
-
-		for (i = 0; i < stream->num_pipes; i++) {
-			struct ia_css_pipe *curr_pipe = stream->pipes[i];
-			assert(curr_pipe != NULL);
-			ia_css_pipe_map_queue(curr_pipe, false);
-		}
-
-		err = map_sp_threads(stream, false);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-	}
-
-	/* remove references from pipes to stream */
-	for (i = 0; i < stream->num_pipes; i++) {
-		struct ia_css_pipe *entry = stream->pipes[i];
-		assert(entry != NULL);
-		if (entry != NULL) {
-			/* clear reference to stream */
-			entry->stream = NULL;
-			/* check internal copy pipe */
-			if (entry->mode == IA_CSS_PIPE_ID_PREVIEW &&
-			    entry->pipe_settings.preview.copy_pipe) {
-				IA_CSS_LOG("clearing stream on internal preview copy pipe");
-				entry->pipe_settings.preview.copy_pipe->stream = NULL;
-			}
-			if (entry->mode == IA_CSS_PIPE_ID_VIDEO &&
-				entry->pipe_settings.video.copy_pipe) {
-				IA_CSS_LOG("clearing stream on internal video copy pipe");
-				entry->pipe_settings.video.copy_pipe->stream = NULL;
-			}
-			err = sh_css_pipe_unload_binaries(entry);
-		}
-	}
-	/* free associated memory of stream struct */
-	kfree(stream->pipes);
-	stream->pipes = NULL;
-	stream->num_pipes = 0;
-#ifndef ISP2401
-	/* working mode: take out of the seed list */
-	if (my_css_save.mode == sh_css_mode_working)
-		for(i=0;i<MAX_ACTIVE_STREAMS;i++)
-			if (my_css_save.stream_seeds[i].stream == stream)
-			{
-				IA_CSS_LOG("took out stream %d", i);
-				my_css_save.stream_seeds[i].stream = NULL;
-				break;
-			}
-#else
-	err2 = ia_css_save_restore_remove_stream(stream);
-
-	err1 = (err != IA_CSS_SUCCESS) ? err : err2;
-#endif
-	kfree(stream);
-#ifndef ISP2401
-	IA_CSS_LEAVE_ERR(err);
-#else
-	IA_CSS_LEAVE_ERR(err1);
-#endif
-
-#ifndef ISP2401
-	return err;
-#else
-	return err1;
-#endif
-}
-
-enum ia_css_err
-ia_css_stream_get_info(const struct ia_css_stream *stream,
-		       struct ia_css_stream_info *stream_info)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_info: enter/exit\n");
-	assert(stream != NULL);
-	assert(stream_info != NULL);
-
-	*stream_info = stream->info;
-	return IA_CSS_SUCCESS;
-}
-
-/*
- * Rebuild a stream, including allocating structs, setting configuration and
- * building the required pipes.
- * The data is taken from the css_save struct updated upon stream creation.
- * The stream handle is used to identify the correct entry in the css_save struct
- */
-enum ia_css_err
-ia_css_stream_load(struct ia_css_stream *stream)
-{
-#ifndef ISP2401
-	int i;
-	enum ia_css_err err;
-	assert(stream != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,	"ia_css_stream_load() enter, \n");
-	for (i = 0; i < MAX_ACTIVE_STREAMS; i++) {
-		if (my_css_save.stream_seeds[i].stream == stream) {
-			int j;
-			for ( j = 0; j < my_css_save.stream_seeds[i].num_pipes; j++) {
-				if ((err = ia_css_pipe_create(&(my_css_save.stream_seeds[i].pipe_config[j]), &my_css_save.stream_seeds[i].pipes[j])) != IA_CSS_SUCCESS) {
-					if (j) {
-						int k;
-						for(k=0;k<j;k++)
-							ia_css_pipe_destroy(my_css_save.stream_seeds[i].pipes[k]);
-					}
-					return err;
-				}
-			}
-			err = ia_css_stream_create(&(my_css_save.stream_seeds[i].stream_config),
-						   my_css_save.stream_seeds[i].num_pipes,
-						   my_css_save.stream_seeds[i].pipes,
-						   &(my_css_save.stream_seeds[i].stream));
-			if (err != IA_CSS_SUCCESS) {
-				ia_css_stream_destroy(stream);
-				for (j = 0; j < my_css_save.stream_seeds[i].num_pipes; j++)
-					ia_css_pipe_destroy(my_css_save.stream_seeds[i].pipes[j]);
-				return err;
-			}
-			break;
-		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,	"ia_css_stream_load() exit, \n");
-	return IA_CSS_SUCCESS;
-#else
-	/* TODO remove function - DEPRECATED */
-	(void)stream;
-	return IA_CSS_ERR_NOT_SUPPORTED;
-#endif
-}
-
-enum ia_css_err
-ia_css_stream_start(struct ia_css_stream *stream)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	IA_CSS_ENTER("stream = %p", stream);
-	if ((stream == NULL) || (stream->last_pipe == NULL)) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	IA_CSS_LOG("starting %d", stream->last_pipe->mode);
-
-	sh_css_sp_set_disable_continuous_viewfinder(stream->disable_cont_vf);
-
-	/* Create host side pipeline. */
-	err = create_host_pipeline(stream);
-	if (err != IA_CSS_SUCCESS) {
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-	if((stream->config.mode == IA_CSS_INPUT_MODE_SENSOR) ||
-	   (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR))
-		stream_register_with_csi_rx(stream);
-#endif
-#endif
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-	/* Initialize mipi size checks */
-	if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
-	{
-		unsigned int idx;
-		unsigned int port = (unsigned int) (stream->config.source.port.port) ;
-
-		for (idx = 0; idx < IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT; idx++) {
-			sh_css_sp_group.config.mipi_sizes_for_check[port][idx] =  sh_css_get_mipi_sizes_for_check(port, idx);
-		}
-	}
-#endif
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	if (stream->config.mode != IA_CSS_INPUT_MODE_MEMORY) {
-		err = sh_css_config_input_network(stream);
-		if (err != IA_CSS_SUCCESS)
-			return err;
-	}
-#endif /* !HAS_NO_INPUT_SYSTEM */
-
-	err = sh_css_pipe_start(stream);
-	IA_CSS_LEAVE_ERR(err);
-	return err;
-}
-
-enum ia_css_err
-ia_css_stream_stop(struct ia_css_stream *stream)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_stop() enter/exit\n");
-	assert(stream != NULL);
-	assert(stream->last_pipe != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_stop: stopping %d\n",
-		stream->last_pipe->mode);
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-	/* De-initialize mipi size checks */
-	if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
-	{
-		unsigned int idx;
-		unsigned int port = (unsigned int) (stream->config.source.port.port) ;
-
-		for (idx = 0; idx < IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT; idx++) {
-			sh_css_sp_group.config.mipi_sizes_for_check[port][idx] = 0;
-		}
-	}
-#endif
-#ifndef ISP2401
-	err = ia_css_pipeline_request_stop(&stream->last_pipe->pipeline);
-#else
-
-	err = sh_css_pipes_stop(stream);
-#endif
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-	/* Ideally, unmapping should happen after pipeline_stop, but current
-	 * semantics do not allow that. */
-	/* err = map_sp_threads(stream, false); */
-
-	return err;
-}
-
-bool
-ia_css_stream_has_stopped(struct ia_css_stream *stream)
-{
-	bool stopped;
-	assert(stream != NULL);
-
-#ifndef ISP2401
-	stopped = ia_css_pipeline_has_stopped(&stream->last_pipe->pipeline);
-#else
-	stopped = sh_css_pipes_have_stopped(stream);
-#endif
-
-	return stopped;
-}
-
-#ifndef ISP2401
-/*
- * Destroy the stream and all the pipes related to it.
- * The stream handle is used to identify the correct entry in the css_save struct
- */
-enum ia_css_err
-ia_css_stream_unload(struct ia_css_stream *stream)
-{
-	int i;
-	assert(stream != NULL);
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,	"ia_css_stream_unload() enter, \n");
-	/* some checks */
-	assert (stream != NULL);
-	for(i=0;i<MAX_ACTIVE_STREAMS;i++)
-		if (my_css_save.stream_seeds[i].stream == stream)
-		{
-			int j;
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,	"ia_css_stream_unload(): unloading %d (%p)\n", i, my_css_save.stream_seeds[i].stream);
-			ia_css_stream_destroy(stream);
-			for(j=0;j<my_css_save.stream_seeds[i].num_pipes;j++)
-				ia_css_pipe_destroy(my_css_save.stream_seeds[i].pipes[j]);
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,	"ia_css_stream_unload(): after unloading %d (%p)\n", i, my_css_save.stream_seeds[i].stream);
-			break;
-		}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,	"ia_css_stream_unload() exit, \n");
-	return IA_CSS_SUCCESS;
-}
-
-#endif
-enum ia_css_err
-ia_css_temp_pipe_to_pipe_id(const struct ia_css_pipe *pipe, enum ia_css_pipe_id *pipe_id)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_temp_pipe_to_pipe_id() enter/exit\n");
-	if (pipe != NULL)
-		*pipe_id = pipe->mode;
-	else
-		*pipe_id = IA_CSS_PIPE_ID_COPY;
-
-	return IA_CSS_SUCCESS;
-}
-
-enum atomisp_input_format
-ia_css_stream_get_format(const struct ia_css_stream *stream)
-{
-	return stream->config.input_config.format;
-}
-
-bool
-ia_css_stream_get_two_pixels_per_clock(const struct ia_css_stream *stream)
-{
-	return (stream->config.pixels_per_clock == 2);
-}
-
-struct ia_css_binary *
-ia_css_stream_get_shading_correction_binary(const struct ia_css_stream *stream)
-{
-	struct ia_css_pipe *pipe;
-
-	assert(stream != NULL);
-
-	pipe = stream->pipes[0];
-
-	if (stream->num_pipes == 2) {
-		assert(stream->pipes[1] != NULL);
-		if (stream->pipes[1]->config.mode == IA_CSS_PIPE_MODE_VIDEO ||
-		    stream->pipes[1]->config.mode == IA_CSS_PIPE_MODE_PREVIEW)
-			pipe = stream->pipes[1];
-	}
-
-	return ia_css_pipe_get_shading_correction_binary(pipe);
-}
-
-struct ia_css_binary *
-ia_css_stream_get_dvs_binary(const struct ia_css_stream *stream)
-{
-	int i;
-	struct ia_css_pipe *video_pipe = NULL;
-
-	/* First we find the video pipe */
-	for (i=0; i<stream->num_pipes; i++) {
-		struct ia_css_pipe *pipe = stream->pipes[i];
-		if (pipe->config.mode == IA_CSS_PIPE_MODE_VIDEO) {
-			video_pipe = pipe;
-			break;
-		}
-	}
-	if (video_pipe)
-		return &video_pipe->pipe_settings.video.video_binary;
-	return NULL;
-}
-
-struct ia_css_binary *
-ia_css_stream_get_3a_binary(const struct ia_css_stream *stream)
-{
-	struct ia_css_pipe *pipe;
-	struct ia_css_binary *s3a_binary = NULL;
-
-	assert(stream != NULL);
-
-	pipe = stream->pipes[0];
-
-	if (stream->num_pipes == 2) {
-		assert(stream->pipes[1] != NULL);
-		if (stream->pipes[1]->config.mode == IA_CSS_PIPE_MODE_VIDEO ||
-		    stream->pipes[1]->config.mode == IA_CSS_PIPE_MODE_PREVIEW)
-			pipe = stream->pipes[1];
-	}
-
-	s3a_binary = ia_css_pipe_get_s3a_binary(pipe);
-
-	return s3a_binary;
-}
-
-
-enum ia_css_err
-ia_css_stream_set_output_padded_width(struct ia_css_stream *stream, unsigned int output_padded_width)
-{
-	struct ia_css_pipe *pipe;
-
-	assert(stream != NULL);
-
-	pipe = stream->last_pipe;
-
-	assert(pipe != NULL);
-
-	/* set the config also just in case (redundant info? why do we save config in pipe?) */
-	pipe->config.output_info[IA_CSS_PIPE_OUTPUT_STAGE_0].padded_width = output_padded_width;
-	pipe->output_info[IA_CSS_PIPE_OUTPUT_STAGE_0].padded_width = output_padded_width;
-
-	return IA_CSS_SUCCESS;
-}
-
-static struct ia_css_binary *
-ia_css_pipe_get_shading_correction_binary(const struct ia_css_pipe *pipe)
-{
-	struct ia_css_binary *binary = NULL;
-
-	assert(pipe != NULL);
-
-	switch (pipe->config.mode) {
-	case IA_CSS_PIPE_MODE_PREVIEW:
-		binary = (struct ia_css_binary *)&pipe->pipe_settings.preview.preview_binary;
-		break;
-	case IA_CSS_PIPE_MODE_VIDEO:
-		binary = (struct ia_css_binary *)&pipe->pipe_settings.video.video_binary;
-		break;
-	case IA_CSS_PIPE_MODE_CAPTURE:
-		if (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_PRIMARY) {
-			unsigned int i;
-
-			for (i = 0; i < pipe->pipe_settings.capture.num_primary_stage; i++) {
-				if (pipe->pipe_settings.capture.primary_binary[i].info->sp.enable.sc) {
-					binary = (struct ia_css_binary *)&pipe->pipe_settings.capture.primary_binary[i];
-					break;
-				}
-			}
-		}
-		else if (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER)
-			binary = (struct ia_css_binary *)&pipe->pipe_settings.capture.pre_isp_binary;
-		else if (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_ADVANCED ||
-			 pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_LOW_LIGHT) {
-			if (pipe->config.isp_pipe_version == IA_CSS_PIPE_VERSION_1)
-				binary = (struct ia_css_binary *)&pipe->pipe_settings.capture.pre_isp_binary;
-			else if (pipe->config.isp_pipe_version == IA_CSS_PIPE_VERSION_2_2)
-				binary = (struct ia_css_binary *)&pipe->pipe_settings.capture.post_isp_binary;
-		}
-		break;
-	default:
-		break;
-	}
-
-	if (binary && binary->info->sp.enable.sc)
-		return binary;
-
-	return NULL;
-}
-
-static struct ia_css_binary *
-ia_css_pipe_get_s3a_binary(const struct ia_css_pipe *pipe)
-{
-	struct ia_css_binary *binary = NULL;
-
-	assert(pipe != NULL);
-
-	switch (pipe->config.mode) {
-		case IA_CSS_PIPE_MODE_PREVIEW:
-			binary = (struct ia_css_binary*)&pipe->pipe_settings.preview.preview_binary;
-			break;
-		case IA_CSS_PIPE_MODE_VIDEO:
-			binary = (struct ia_css_binary*)&pipe->pipe_settings.video.video_binary;
-			break;
-		case IA_CSS_PIPE_MODE_CAPTURE:
-			if (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_PRIMARY) {
-				unsigned int i;
-				for (i = 0; i < pipe->pipe_settings.capture.num_primary_stage; i++) {
-					if (pipe->pipe_settings.capture.primary_binary[i].info->sp.enable.s3a) {
-						binary = (struct ia_css_binary *)&pipe->pipe_settings.capture.primary_binary[i];
-						break;
-					}
-				}
-			}
-			else if (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER)
-				binary = (struct ia_css_binary *)&pipe->pipe_settings.capture.pre_isp_binary;
-			else if (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_ADVANCED ||
-				 pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_LOW_LIGHT) {
-				if (pipe->config.isp_pipe_version == IA_CSS_PIPE_VERSION_1)
-					binary = (struct ia_css_binary *)&pipe->pipe_settings.capture.pre_isp_binary;
-				else if (pipe->config.isp_pipe_version == IA_CSS_PIPE_VERSION_2_2)
-					binary = (struct ia_css_binary *)&pipe->pipe_settings.capture.post_isp_binary;
-				else
-					assert(0);
-			}
-			break;
-		default:
-			break;
-	}
-
-	if (binary && !binary->info->sp.enable.s3a)
-		binary = NULL;
-
-        return binary;
-}
-
-static struct ia_css_binary *
-ia_css_pipe_get_sdis_binary(const struct ia_css_pipe *pipe)
-{
-	struct ia_css_binary *binary = NULL;
-
-	assert(pipe != NULL);
-
-	switch (pipe->config.mode) {
-		case IA_CSS_PIPE_MODE_VIDEO:
-			binary = (struct ia_css_binary*)&pipe->pipe_settings.video.video_binary;
-			break;
-		default:
-			break;
-	}
-
-	if (binary && !binary->info->sp.enable.dis)
-		binary = NULL;
-
-	return binary;
-}
-
-struct ia_css_pipeline *
-ia_css_pipe_get_pipeline(const struct ia_css_pipe *pipe)
-{
-	assert(pipe != NULL);
-
-	return (struct ia_css_pipeline*)&pipe->pipeline;
-}
-
-unsigned int
-ia_css_pipe_get_pipe_num(const struct ia_css_pipe *pipe)
-{
-	assert(pipe != NULL);
-
-	/* KW was not sure this function was not returning a value
-	   that was out of range; so added an assert, and, for the
-	   case when asserts are not enabled, clip to the largest
-	   value; pipe_num is unsigned so the value cannot be too small
-	*/
-	assert(pipe->pipe_num < IA_CSS_PIPELINE_NUM_MAX);
-
-	if (pipe->pipe_num >= IA_CSS_PIPELINE_NUM_MAX)
-		return (IA_CSS_PIPELINE_NUM_MAX - 1);
-
-	return pipe->pipe_num;
-}
-
-
-unsigned int
-ia_css_pipe_get_isp_pipe_version(const struct ia_css_pipe *pipe)
-{
-	assert(pipe != NULL);
-
-	return (unsigned int)pipe->config.isp_pipe_version;
-}
-
-#define SP_START_TIMEOUT_US 30000000
-
-enum ia_css_err
-ia_css_start_sp(void)
-{
-	unsigned long timeout;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER("");
-	sh_css_sp_start_isp();
-
-	/* waiting for the SP is completely started */
-	timeout = SP_START_TIMEOUT_US;
-	while((ia_css_spctrl_get_state(SP0_ID) != IA_CSS_SP_SW_INITIALIZED) && timeout) {
-		timeout--;
-		hrt_sleep();
-	}
-	if (timeout == 0) {
-		IA_CSS_ERROR("timeout during SP initialization");
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-	/* Workaround, in order to run two streams in parallel. See TASK 4271*/
-	/* TODO: Fix this. */
-
-	sh_css_init_host_sp_control_vars();
-
-	/* buffers should be initialized only when sp is started */
-	/* AM: At the moment it will be done only when there is no stream active. */
-
-	sh_css_setup_queues();
-	ia_css_bufq_dump_queue_info();
-
-#ifdef ISP2401
-	if (ia_css_is_system_mode_suspend_or_resume() == false) { /* skip in suspend/resume flow */
-		ia_css_set_system_mode(IA_CSS_SYS_MODE_WORKING);
-	}
-#endif
-	IA_CSS_LEAVE_ERR(err);
-	return err;
-}
-
-/*
- *	Time to wait SP for termincate. Only condition when this can happen
- *	is a fatal hw failure, but we must be able to detect this and emit
- *	a proper error trace.
- */
-#define SP_SHUTDOWN_TIMEOUT_US 200000
-
-enum ia_css_err
-ia_css_stop_sp(void)
-{
-	unsigned long timeout;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER("void");
-
-	if (!sh_css_sp_is_running()) {
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE("SP already stopped : return_err=%d", err);
-
-		/* Return an error - stop SP should not have been called by driver */
-		return err;
-	}
-
-	/* For now, stop whole SP */
-#ifndef ISP2401
-	sh_css_write_host2sp_command(host2sp_cmd_terminate);
-#else
-	if (!sh_css_write_host2sp_command(host2sp_cmd_terminate)) {
-		IA_CSS_ERROR("Call to 'sh-css_write_host2sp_command()' failed");
-		ia_css_debug_dump_sp_sw_debug_info();
-		ia_css_debug_dump_debug_info(NULL);
-	}
-#endif
-	sh_css_sp_set_sp_running(false);
-
-	timeout = SP_SHUTDOWN_TIMEOUT_US;
-	while (!ia_css_spctrl_is_idle(SP0_ID) && timeout) {
-		timeout--;
-		hrt_sleep();
-	}
-	if ((ia_css_spctrl_get_state(SP0_ID) != IA_CSS_SP_SW_TERMINATED))
-		IA_CSS_WARNING("SP has not terminated (SW)");
-
-	if (timeout == 0) {
-		IA_CSS_WARNING("SP is not idle");
-		ia_css_debug_dump_sp_sw_debug_info();
-	}
-	timeout = SP_SHUTDOWN_TIMEOUT_US;
-	while (!isp_ctrl_getbit(ISP0_ID, ISP_SC_REG, ISP_IDLE_BIT) && timeout) {
-		timeout--;
-		hrt_sleep();
-	}
-	if (timeout == 0) {
-		IA_CSS_WARNING("ISP is not idle");
-		ia_css_debug_dump_sp_sw_debug_info();
-	}
-
-	sh_css_hmm_buffer_record_uninit();
-
-#ifndef ISP2401
-	/* clear pending param sets from refcount */
-	sh_css_param_clear_param_sets();
-#else
-	if (ia_css_is_system_mode_suspend_or_resume() == false) { /* skip in suspend/resume flow */
-		/* clear pending param sets from refcount */
-		sh_css_param_clear_param_sets();
-		ia_css_set_system_mode(IA_CSS_SYS_MODE_INIT);  /* System is initialized but not 'running' */
-	}
-#endif
-
-	IA_CSS_LEAVE_ERR(err);
-	return err;
-}
-
-enum ia_css_err
-ia_css_update_continuous_frames(struct ia_css_stream *stream)
-{
-	struct ia_css_pipe *pipe;
-	unsigned int i;
-
-	ia_css_debug_dtrace(
-	    IA_CSS_DEBUG_TRACE,
-	    "sh_css_update_continuous_frames() enter:\n");
-
-	if (stream == NULL) {
-		ia_css_debug_dtrace(
-			IA_CSS_DEBUG_TRACE,
-			"sh_css_update_continuous_frames() leave: invalid stream, return_void\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	pipe = stream->continuous_pipe;
-
-	for (i = stream->config.init_num_cont_raw_buf;
-				i < stream->config.target_num_cont_raw_buf; i++) {
-		sh_css_update_host2sp_offline_frame(i,
-				pipe->continuous_frames[i], pipe->cont_md_buffers[i]);
-	}
-	sh_css_update_host2sp_cont_num_raw_frames
-			(stream->config.target_num_cont_raw_buf, true);
-	ia_css_debug_dtrace(
-	    IA_CSS_DEBUG_TRACE,
-	    "sh_css_update_continuous_frames() leave: return_void\n");
-
-	return IA_CSS_SUCCESS;
-}
-
-void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map)
-{
-	unsigned int thread_id;
-	enum ia_css_pipe_id pipe_id;
-	unsigned int pipe_num;
-	bool need_input_queue;
-
-	IA_CSS_ENTER("");
-	assert(pipe != NULL);
-
-	pipe_id = pipe->mode;
-	pipe_num = pipe->pipe_num;
-
-	ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
-
-#if defined(HAS_NO_INPUT_SYSTEM) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	need_input_queue = true;
-#else
-	need_input_queue = pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY;
-#endif
-
-	/* map required buffer queues to resources */
-	/* TODO: to be improved */
-	if (pipe->mode == IA_CSS_PIPE_ID_PREVIEW) {
-		if (need_input_queue)
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_INPUT_FRAME, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_OUTPUT_FRAME, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_PARAMETER_SET, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_PER_FRAME_PARAMETER_SET, map);
-#if defined SH_CSS_ENABLE_METADATA
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_METADATA, map);
-#endif
-		if (pipe->pipe_settings.preview.preview_binary.info &&
-			pipe->pipe_settings.preview.preview_binary.info->sp.enable.s3a)
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_3A_STATISTICS, map);
-	} else if (pipe->mode == IA_CSS_PIPE_ID_CAPTURE) {
-		unsigned int i;
-
-		if (need_input_queue)
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_INPUT_FRAME, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_OUTPUT_FRAME, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_PARAMETER_SET, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_PER_FRAME_PARAMETER_SET, map);
-#if defined SH_CSS_ENABLE_METADATA
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_METADATA, map);
-#endif
-		if (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_PRIMARY) {
-			for (i = 0; i < pipe->pipe_settings.capture.num_primary_stage; i++) {
-				if (pipe->pipe_settings.capture.primary_binary[i].info &&
-					pipe->pipe_settings.capture.primary_binary[i].info->sp.enable.s3a) {
-					ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_3A_STATISTICS, map);
-					break;
-				}
-			}
-		} else if (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_ADVANCED ||
-				 pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_LOW_LIGHT ||
-				 pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER) {
-			if (pipe->pipe_settings.capture.pre_isp_binary.info &&
-				pipe->pipe_settings.capture.pre_isp_binary.info->sp.enable.s3a)
-				ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_3A_STATISTICS, map);
-		}
-	} else if (pipe->mode == IA_CSS_PIPE_ID_VIDEO) {
-		if (need_input_queue)
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_INPUT_FRAME, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_OUTPUT_FRAME, map);
-		if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0])
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_PARAMETER_SET, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_PER_FRAME_PARAMETER_SET, map);
-#if defined SH_CSS_ENABLE_METADATA
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_METADATA, map);
-#endif
-		if (pipe->pipe_settings.video.video_binary.info &&
-			pipe->pipe_settings.video.video_binary.info->sp.enable.s3a)
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_3A_STATISTICS, map);
-		if (pipe->pipe_settings.video.video_binary.info &&
-			(pipe->pipe_settings.video.video_binary.info->sp.enable.dis
-			))
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_DIS_STATISTICS, map);
-	} else if (pipe->mode == IA_CSS_PIPE_ID_COPY) {
-		if (need_input_queue)
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_INPUT_FRAME, map);
-		if (!pipe->stream->config.continuous)
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_OUTPUT_FRAME, map);
-#if defined SH_CSS_ENABLE_METADATA
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_METADATA, map);
-#endif
-	} else if (pipe->mode == IA_CSS_PIPE_ID_ACC) {
-		if (need_input_queue)
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_INPUT_FRAME, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_OUTPUT_FRAME, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_PARAMETER_SET, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_PER_FRAME_PARAMETER_SET, map);
-#if defined SH_CSS_ENABLE_METADATA
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_METADATA, map);
-#endif
-	} else if (pipe->mode == IA_CSS_PIPE_ID_YUVPP) {
-		unsigned int idx;
-		for (idx = 0; idx < IA_CSS_PIPE_MAX_OUTPUT_STAGE; idx++) {
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_OUTPUT_FRAME + idx, map);
-			if (pipe->enable_viewfinder[idx])
-				ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME + idx, map);
-		}
-		if (need_input_queue)
-			ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_INPUT_FRAME, map);
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_PARAMETER_SET, map);
-#if defined SH_CSS_ENABLE_METADATA
-		ia_css_queue_map(thread_id, IA_CSS_BUFFER_TYPE_METADATA, map);
-#endif
-	}
-	IA_CSS_LEAVE("");
-}
-
-#if CONFIG_ON_FRAME_ENQUEUE()
-static enum ia_css_err set_config_on_frame_enqueue(struct ia_css_frame_info *info, struct frame_data_wrapper *frame)
-{
-	frame->config_on_frame_enqueue.padded_width = 0;
-
-	/* currently we support configuration on frame enqueue only on YUV formats */
-	/* on other formats the padded_width is zeroed for no configuration override */
-	switch (info->format) {
-	case IA_CSS_FRAME_FORMAT_YUV420:
-	case IA_CSS_FRAME_FORMAT_NV12:
-		if (info->padded_width > info->res.width)
-		{
-			frame->config_on_frame_enqueue.padded_width = info->padded_width;
-		}
-		else if ((info->padded_width < info->res.width) && (info->padded_width > 0))
-		{
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-		}
-		/* nothing to do if width == padded width or padded width is zeroed (the same) */
-		break;
-	default:
-		break;
-	}
-
-	return IA_CSS_SUCCESS;
-}
-#endif
-
-enum ia_css_err
-ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id)
-{
-	enum ia_css_err ret;
-
-	IA_CSS_ENTER("");
-
-	/* Only continuous streams have a tagger to which we can send the
-	 * unlock message. */
-	if (stream == NULL || !stream->config.continuous) {
-		IA_CSS_ERROR("invalid stream pointer");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if (exp_id > IA_CSS_ISYS_MAX_EXPOSURE_ID ||
-	    exp_id < IA_CSS_ISYS_MIN_EXPOSURE_ID) {
-		IA_CSS_ERROR("invalid expsure ID: %d\n", exp_id);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	/* Send the event. Since we verified that the exp_id is valid,
-	 * we can safely assign it to an 8-bit argument here. */
-	ret = ia_css_bufq_enqueue_psys_event(
-			IA_CSS_PSYS_SW_EVENT_UNLOCK_RAW_BUFFER, exp_id, 0, 0);
-
-	IA_CSS_LEAVE_ERR(ret);
-	return ret;
-}
-
-/* @brief	Set the state (Enable or Disable) of the Extension stage in the
- * 		given pipe.
- */
-enum ia_css_err
-ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, bool enable)
-{
-	unsigned int thread_id;
-	struct ia_css_pipeline_stage *stage;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER("");
-
-	/* Parameter Check */
-	if (pipe == NULL || pipe->stream == NULL) {
-		IA_CSS_ERROR("Invalid Pipe.");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-	} else if (!(pipe->config.acc_extension)) {
-		IA_CSS_ERROR("Invalid Pipe(No Extension Firmware)");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-	} else if (!sh_css_sp_is_running()) {
-		IA_CSS_ERROR("Leaving: queue unavailable.");
-		err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	} else {
-		/* Query the threadid and stage_num for the Extension firmware*/
-		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-		err = ia_css_pipeline_get_stage_from_fw(&(pipe->pipeline), fw_handle, &stage);
-		if (err == IA_CSS_SUCCESS) {
-			/* Set the Extension State;. TODO: Add check for stage firmware.type (QOS)*/
-			err = ia_css_bufq_enqueue_psys_event(
-				(uint8_t) IA_CSS_PSYS_SW_EVENT_STAGE_ENABLE_DISABLE,
-				(uint8_t) thread_id,
-				(uint8_t) stage->stage_num,
-				enable ? 1 : 0);
-			if (err == IA_CSS_SUCCESS) {
-				if(enable)
-					SH_CSS_QOS_STAGE_ENABLE(&(sh_css_sp_group.pipe[thread_id]),stage->stage_num);
-				else
-					SH_CSS_QOS_STAGE_DISABLE(&(sh_css_sp_group.pipe[thread_id]),stage->stage_num);
-			}
-		}
-	}
-	IA_CSS_LEAVE("err:%d handle:%u enable:%d", err, fw_handle, enable);
-	return err;
-}
-
-/*	@brief	Get the state (Enable or Disable) of the Extension stage in the
- *	given pipe.
- */
-enum ia_css_err
-ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, bool *enable)
-{
-	struct ia_css_pipeline_stage *stage;
-	unsigned int thread_id;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER("");
-
-	/* Parameter Check */
-	if (pipe == NULL || pipe->stream == NULL) {
-		IA_CSS_ERROR("Invalid Pipe.");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-	} else if (!(pipe->config.acc_extension)) {
-		IA_CSS_ERROR("Invalid Pipe (No Extension Firmware).");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-	} else if (!sh_css_sp_is_running()) {
-		IA_CSS_ERROR("Leaving: queue unavailable.");
-		err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	} else {
-		/* Query the threadid and stage_num corresponding to the Extension firmware*/
-		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-		err = ia_css_pipeline_get_stage_from_fw(&pipe->pipeline, fw_handle, &stage);
-
-		if (err == IA_CSS_SUCCESS) {
-			/* Get the Extension State */
-			*enable = (SH_CSS_QOS_STAGE_IS_ENABLED(&(sh_css_sp_group.pipe[thread_id]),stage->stage_num)) ? true : false;
-		}
-	}
-	IA_CSS_LEAVE("err:%d handle:%u enable:%d", err, fw_handle, *enable);
-	return err;
-}
-
-#ifdef ISP2401
-enum ia_css_err
-ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, uint32_t fw_handle,
-	struct ia_css_isp_param_css_segments *css_seg, struct ia_css_isp_param_isp_segments *isp_seg)
-{
-	unsigned int HIVE_ADDR_sp_group;
-	static struct sh_css_sp_group sp_group;
-	static struct sh_css_sp_stage sp_stage;
-	static struct sh_css_isp_stage isp_stage;
-	const struct ia_css_fw_info *fw;
-	unsigned int thread_id;
-	struct ia_css_pipeline_stage *stage;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	int stage_num = 0;
-	enum ia_css_isp_memories mem;
-	bool enabled;
-
-	IA_CSS_ENTER("");
-
-	fw = &sh_css_sp_fw;
-
-	/* Parameter Check */
-	if (pipe == NULL || pipe->stream == NULL) {
-		IA_CSS_ERROR("Invalid Pipe.");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-	} else if (!(pipe->config.acc_extension)) {
-		IA_CSS_ERROR("Invalid Pipe (No Extension Firmware).");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-	} else if (!sh_css_sp_is_running()) {
-		IA_CSS_ERROR("Leaving: queue unavailable.");
-		err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-	} else {
-		/* Query the thread_id and stage_num corresponding to the Extension firmware */
-		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
-		err = ia_css_pipeline_get_stage_from_fw(&(pipe->pipeline), fw_handle, &stage);
-		if (err == IA_CSS_SUCCESS) {
-			/* Get the Extension State */
-			enabled = (SH_CSS_QOS_STAGE_IS_ENABLED(&(sh_css_sp_group.pipe[thread_id]), stage->stage_num)) ? true : false;
-			/* Update mapped arg only when extension stage is not enabled */
-			if (enabled) {
-				IA_CSS_ERROR("Leaving: cannot update when stage is enabled.");
-				err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-			} else {
-				stage_num = stage->stage_num;
-
-				HIVE_ADDR_sp_group = fw->info.sp.group;
-				sp_dmem_load(SP0_ID,
-					(unsigned int)sp_address_of(sp_group),
-					&sp_group, sizeof(struct sh_css_sp_group));
-				mmgr_load(sp_group.pipe[thread_id].sp_stage_addr[stage_num],
-					&sp_stage, sizeof(struct sh_css_sp_stage));
-
-				mmgr_load(sp_stage.isp_stage_addr,
-					&isp_stage, sizeof(struct sh_css_isp_stage));
-
-				for (mem = 0; mem < N_IA_CSS_ISP_MEMORIES; mem++) {
-					isp_stage.mem_initializers.params[IA_CSS_PARAM_CLASS_PARAM][mem].address =
-						css_seg->params[IA_CSS_PARAM_CLASS_PARAM][mem].address;
-					isp_stage.mem_initializers.params[IA_CSS_PARAM_CLASS_PARAM][mem].size =
-						css_seg->params[IA_CSS_PARAM_CLASS_PARAM][mem].size;
-					isp_stage.binary_info.mem_initializers.params[IA_CSS_PARAM_CLASS_PARAM][mem].address =
-						isp_seg->params[IA_CSS_PARAM_CLASS_PARAM][mem].address;
-					isp_stage.binary_info.mem_initializers.params[IA_CSS_PARAM_CLASS_PARAM][mem].size =
-						isp_seg->params[IA_CSS_PARAM_CLASS_PARAM][mem].size;
-				}
-
-				mmgr_store(sp_stage.isp_stage_addr,
-					&isp_stage, sizeof(struct sh_css_isp_stage));
-			}
-		}
-	}
-	IA_CSS_LEAVE("err:%d handle:%u", err, fw_handle);
-	return err;
-}
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-static enum ia_css_err
-aspect_ratio_crop_init(struct ia_css_stream *curr_stream,
-		struct ia_css_pipe *pipes[],
-		bool *do_crop_status)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	int i;
-	struct ia_css_pipe *curr_pipe;
-	uint32_t pipe_mask = 0;
-
-	if ((curr_stream == NULL) ||
-	    (curr_stream->num_pipes == 0) ||
-	    (pipes == NULL) ||
-	    (do_crop_status == NULL)) {
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-
-	for (i = 0; i < curr_stream->num_pipes; i++) {
-		curr_pipe = pipes[i];
-		pipe_mask |= (1 << curr_pipe->config.mode);
-	}
-
-	*do_crop_status =
-		(((pipe_mask & (1 << IA_CSS_PIPE_MODE_PREVIEW)) ||
-		  (pipe_mask & (1 << IA_CSS_PIPE_MODE_VIDEO))) &&
-		  (pipe_mask & (1 << IA_CSS_PIPE_MODE_CAPTURE)) &&
-		  curr_stream->config.continuous);
-	return IA_CSS_SUCCESS;
-}
-
-static bool
-aspect_ratio_crop_check(bool enabled, struct ia_css_pipe *curr_pipe)
-{
-	bool status = false;
-
-	if ((curr_pipe != NULL) && enabled) {
-		if ((curr_pipe->config.mode == IA_CSS_PIPE_MODE_PREVIEW) ||
-		    (curr_pipe->config.mode == IA_CSS_PIPE_MODE_VIDEO) ||
-		    (curr_pipe->config.mode == IA_CSS_PIPE_MODE_CAPTURE))
-			status = true;
-	}
-
-	return status;
-}
-
-static enum ia_css_err
-aspect_ratio_crop(struct ia_css_pipe *curr_pipe,
-		struct ia_css_resolution *effective_res)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_resolution crop_res;
-	struct ia_css_resolution *in_res = NULL;
-	struct ia_css_resolution *out_res = NULL;
-	bool use_bds_output_info = false;
-	bool use_vf_pp_in_res = false;
-	bool use_capt_pp_in_res = false;
-
-	if ((curr_pipe == NULL) ||
-	    (effective_res == NULL)) {
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-
-	if ((curr_pipe->config.mode != IA_CSS_PIPE_MODE_PREVIEW) &&
-	    (curr_pipe->config.mode != IA_CSS_PIPE_MODE_VIDEO) &&
-	    (curr_pipe->config.mode != IA_CSS_PIPE_MODE_CAPTURE)) {
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE_ERR(err);
-		return err;
-	}
-
-	use_bds_output_info =
-		((curr_pipe->bds_output_info.res.width != 0) &&
-		 (curr_pipe->bds_output_info.res.height != 0));
-
-	use_vf_pp_in_res =
-		((curr_pipe->config.vf_pp_in_res.width != 0) &&
-		 (curr_pipe->config.vf_pp_in_res.height != 0));
-
-	use_capt_pp_in_res =
-		((curr_pipe->config.capt_pp_in_res.width != 0) &&
-		 (curr_pipe->config.capt_pp_in_res.height != 0));
-
-	in_res = &curr_pipe->stream->config.input_config.effective_res;
-	out_res = &curr_pipe->output_info[0].res;
-
-	switch (curr_pipe->config.mode) {
-	case IA_CSS_PIPE_MODE_PREVIEW:
-		if (use_bds_output_info)
-			out_res = &curr_pipe->bds_output_info.res;
-		else if (use_vf_pp_in_res)
-			out_res = &curr_pipe->config.vf_pp_in_res;
-		break;
-	case IA_CSS_PIPE_MODE_VIDEO:
-		if (use_bds_output_info)
-			out_res = &curr_pipe->bds_output_info.res;
-		break;
-	case IA_CSS_PIPE_MODE_CAPTURE:
-		if (use_capt_pp_in_res)
-			out_res = &curr_pipe->config.capt_pp_in_res;
-		break;
-	case IA_CSS_PIPE_MODE_ACC:
-	case IA_CSS_PIPE_MODE_COPY:
-	case IA_CSS_PIPE_MODE_YUVPP:
-	default:
-		IA_CSS_ERROR("aspect ratio cropping invalid args: mode[%d]\n",
-			curr_pipe->config.mode);
-		assert(0);
-		break;
-	}
-
-	err = ia_css_frame_find_crop_resolution(in_res, out_res, &crop_res);
-	if (err == IA_CSS_SUCCESS) {
-		*effective_res = crop_res;
-	} else {
-		/* in case of error fallback to default
-		 * effective resolution from driver. */
-		IA_CSS_LOG("ia_css_frame_find_crop_resolution() failed with err(%d)", err);
-	}
-	return err;
-}
-#endif
-
-#endif
-static void
-sh_css_hmm_buffer_record_init(void)
-{
-	int i;
-
-#ifndef ISP2401
-	for (i = 0; i < MAX_HMM_BUFFER_NUM; i++) {
-		sh_css_hmm_buffer_record_reset(&hmm_buffer_record[i]);
-#else
-	if (ia_css_is_system_mode_suspend_or_resume() == false) { /* skip in suspend/resume flow */
-		for (i = 0; i < MAX_HMM_BUFFER_NUM; i++) {
-			sh_css_hmm_buffer_record_reset(&hmm_buffer_record[i]);
-		}
-#endif
-	}
-}
-
-static void
-sh_css_hmm_buffer_record_uninit(void)
-{
-	int i;
-	struct sh_css_hmm_buffer_record *buffer_record = NULL;
-
-#ifndef ISP2401
-	buffer_record = &hmm_buffer_record[0];
-	for (i = 0; i < MAX_HMM_BUFFER_NUM; i++) {
-		if (buffer_record->in_use) {
-			if (buffer_record->h_vbuf != NULL)
-				ia_css_rmgr_rel_vbuf(hmm_buffer_pool, &buffer_record->h_vbuf);
-			sh_css_hmm_buffer_record_reset(buffer_record);
-#else
-	if (ia_css_is_system_mode_suspend_or_resume() == false) { /* skip in suspend/resume flow */
-		buffer_record = &hmm_buffer_record[0];
-		for (i = 0; i < MAX_HMM_BUFFER_NUM; i++) {
-			if (buffer_record->in_use) {
-				if (buffer_record->h_vbuf != NULL)
-					ia_css_rmgr_rel_vbuf(hmm_buffer_pool, &buffer_record->h_vbuf);
-				sh_css_hmm_buffer_record_reset(buffer_record);
-			}
-			buffer_record++;
-#endif
-		}
-#ifndef ISP2401
-		buffer_record++;
-#endif
-	}
-}
-
-static void
-sh_css_hmm_buffer_record_reset(struct sh_css_hmm_buffer_record *buffer_record)
-{
-	assert(buffer_record != NULL);
-	buffer_record->in_use = false;
-	buffer_record->type = IA_CSS_BUFFER_TYPE_INVALID;
-	buffer_record->h_vbuf = NULL;
-	buffer_record->kernel_ptr = 0;
-}
-
-static struct sh_css_hmm_buffer_record
-*sh_css_hmm_buffer_record_acquire(struct ia_css_rmgr_vbuf_handle *h_vbuf,
-			enum ia_css_buffer_type type,
-			hrt_address kernel_ptr)
-{
-	int i;
-	struct sh_css_hmm_buffer_record *buffer_record = NULL;
-	struct sh_css_hmm_buffer_record *out_buffer_record = NULL;
-
-	assert(h_vbuf != NULL);
-	assert((type > IA_CSS_BUFFER_TYPE_INVALID) && (type < IA_CSS_NUM_DYNAMIC_BUFFER_TYPE));
-	assert(kernel_ptr != 0);
-
-	buffer_record = &hmm_buffer_record[0];
-	for (i = 0; i < MAX_HMM_BUFFER_NUM; i++) {
-		if (!buffer_record->in_use) {
-			buffer_record->in_use = true;
-			buffer_record->type = type;
-			buffer_record->h_vbuf = h_vbuf;
-			buffer_record->kernel_ptr = kernel_ptr;
-			out_buffer_record = buffer_record;
-			break;
-		}
-		buffer_record++;
-	}
-
-	return out_buffer_record;
-}
-
-static struct sh_css_hmm_buffer_record
-*sh_css_hmm_buffer_record_validate(hrt_vaddress ddr_buffer_addr,
-		enum ia_css_buffer_type type)
-{
-	int i;
-	struct sh_css_hmm_buffer_record *buffer_record = NULL;
-	bool found_record = false;
-
-	buffer_record = &hmm_buffer_record[0];
-	for (i = 0; i < MAX_HMM_BUFFER_NUM; i++) {
-		if ((buffer_record->in_use) &&
-		    (buffer_record->type == type) &&
-		    (buffer_record->h_vbuf != NULL) &&
-		    (buffer_record->h_vbuf->vptr == ddr_buffer_addr)) {
-			found_record = true;
-			break;
-		}
-		buffer_record++;
-	}
-
-	if (found_record)
-		return buffer_record;
-	else
-		return NULL;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_defs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_defs.h
deleted file mode 100644
index 4072c564f911..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_defs.h
+++ /dev/null
@@ -1,410 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _SH_CSS_DEFS_H_
-#define _SH_CSS_DEFS_H_
-
-#include "isp.h"
-
-/*#include "vamem.h"*/ /* Cannot include for VAMEM properties this file is visible on ISP -> pipeline generator */
-
-#include "math_support.h"	/* max(), min, etc etc */
-
-/* ID's for refcount */
-#define IA_CSS_REFCOUNT_PARAM_SET_POOL  0xCAFE0001
-#define IA_CSS_REFCOUNT_PARAM_BUFFER    0xCAFE0002
-
-/* Digital Image Stabilization */
-#define SH_CSS_DIS_DECI_FACTOR_LOG2       6
-
-/* UV offset: 1:uv=-128...127, 0:uv=0...255 */
-#define SH_CSS_UV_OFFSET_IS_0             0
-
-/* Bits of bayer is adjusted as 13 in ISP */
-#define SH_CSS_BAYER_BITS                 13
-
-/* Max value of bayer data (unsigned 13bit in ISP) */
-#define SH_CSS_BAYER_MAXVAL               ((1U << SH_CSS_BAYER_BITS) - 1)
-
-/* Bits of yuv in ISP */
-#define SH_CSS_ISP_YUV_BITS               8
-
-#define SH_CSS_DP_GAIN_SHIFT              5
-#define SH_CSS_BNR_GAIN_SHIFT             13
-#define SH_CSS_YNR_GAIN_SHIFT             13
-#define SH_CSS_AE_YCOEF_SHIFT             13
-#define SH_CSS_AF_FIR_SHIFT               13
-#define SH_CSS_YEE_DETAIL_GAIN_SHIFT      8  /* [u5.8] */
-#define SH_CSS_YEE_SCALE_SHIFT            8
-#define SH_CSS_TNR_COEF_SHIFT             13
-#define SH_CSS_MACC_COEF_SHIFT            11 /* [s2.11] for ISP1 */
-#define SH_CSS_MACC2_COEF_SHIFT           13 /* [s[exp].[13-exp]] for ISP2 */
-#define SH_CSS_DIS_COEF_SHIFT             13
-
-/* enumeration of the bayer downscale factors. When a binary supports multiple
- * factors, the OR of these defines is used to build the mask of supported
- * factors. The BDS factor is used in pre-processor expressions so we cannot
- * use an enum here. */
-#define SH_CSS_BDS_FACTOR_1_00	(0)
-#define SH_CSS_BDS_FACTOR_1_25	(1)
-#define SH_CSS_BDS_FACTOR_1_50	(2)
-#define SH_CSS_BDS_FACTOR_2_00	(3)
-#define SH_CSS_BDS_FACTOR_2_25	(4)
-#define SH_CSS_BDS_FACTOR_2_50	(5)
-#define SH_CSS_BDS_FACTOR_3_00	(6)
-#define SH_CSS_BDS_FACTOR_4_00	(7)
-#define SH_CSS_BDS_FACTOR_4_50	(8)
-#define SH_CSS_BDS_FACTOR_5_00	(9)
-#define SH_CSS_BDS_FACTOR_6_00	(10)
-#define SH_CSS_BDS_FACTOR_8_00	(11)
-#define NUM_BDS_FACTORS	        (12)
-
-#define PACK_BDS_FACTOR(factor)	(1<<(factor))
-
-/* Following macros should match with the type enum ia_css_pipe_version in
- * ia_css_pipe_public.h. The reason to add these macros is that enum type
- * will be evaluted to 0 in preprocessing time. */
-#define SH_CSS_ISP_PIPE_VERSION_1	1
-#define SH_CSS_ISP_PIPE_VERSION_2_2	2
-#define SH_CSS_ISP_PIPE_VERSION_2_6_1	3
-#define SH_CSS_ISP_PIPE_VERSION_2_7	4
-
-/*--------------- sRGB Gamma -----------------
-CCM        : YCgCo[0,8191] -> RGB[0,4095]
-sRGB Gamma : RGB  [0,4095] -> RGB[0,8191]
-CSC        : RGB  [0,8191] -> YUV[0,8191]
-
-CCM:
-Y[0,8191],CgCo[-4096,4095],coef[-8192,8191] -> RGB[0,4095]
-
-sRGB Gamma:
-RGB[0,4095] -(interpolation step16)-> RGB[0,255] -(LUT 12bit)-> RGB[0,4095] -> RGB[0,8191]
-
-CSC:
-RGB[0,8191],coef[-8192,8191] -> RGB[0,8191]
---------------------------------------------*/
-/* Bits of input/output of sRGB Gamma */
-#define SH_CSS_RGB_GAMMA_INPUT_BITS       12 /* [0,4095] */
-#define SH_CSS_RGB_GAMMA_OUTPUT_BITS      13 /* [0,8191] */
-
-/* Bits of fractional part of interpolation in vamem, [0,4095]->[0,255] */
-#define SH_CSS_RGB_GAMMA_FRAC_BITS        \
-	(SH_CSS_RGB_GAMMA_INPUT_BITS - SH_CSS_ISP_RGB_GAMMA_TABLE_SIZE_LOG2)
-#define SH_CSS_RGB_GAMMA_ONE              (1 << SH_CSS_RGB_GAMMA_FRAC_BITS)
-
-/* Bits of input of CCM,  = 13, Y[0,8191],CgCo[-4096,4095] */
-#define SH_CSS_YUV2RGB_CCM_INPUT_BITS     SH_CSS_BAYER_BITS
-
-/* Bits of output of CCM,  = 12, RGB[0,4095] */
-#define SH_CSS_YUV2RGB_CCM_OUTPUT_BITS    SH_CSS_RGB_GAMMA_INPUT_BITS
-
-/* Maximum value of output of CCM */
-#define SH_CSS_YUV2RGB_CCM_MAX_OUTPUT     \
-	((1 << SH_CSS_YUV2RGB_CCM_OUTPUT_BITS) - 1)
-
-#define SH_CSS_NUM_INPUT_BUF_LINES        4
-
-/* Left cropping only applicable for sufficiently large nway */
-#if ISP_VEC_NELEMS == 16
-#define SH_CSS_MAX_LEFT_CROPPING          0
-#define SH_CSS_MAX_TOP_CROPPING           0
-#else
-#define SH_CSS_MAX_LEFT_CROPPING          12
-#define SH_CSS_MAX_TOP_CROPPING           12
-#endif
-
-#define	SH_CSS_SP_MAX_WIDTH               1280
-
-/* This is the maximum grid we can handle in the ISP binaries.
- * The host code makes sure no bigger grid is ever selected. */
-#define SH_CSS_MAX_BQ_GRID_WIDTH          80
-#define SH_CSS_MAX_BQ_GRID_HEIGHT         60
-
-/* The minimum dvs envelope is 12x12(for IPU2) to make sure the 
- * invalid rows/columns that result from filter initialization are skipped. */
-#define SH_CSS_MIN_DVS_ENVELOPE           12U
-
-/* The FPGA system (vec_nelems == 16) only supports upto 5MP */
-#if ISP_VEC_NELEMS == 16
-#define SH_CSS_MAX_SENSOR_WIDTH           2560
-#define SH_CSS_MAX_SENSOR_HEIGHT          1920
-#else
-#define SH_CSS_MAX_SENSOR_WIDTH           4608
-#define SH_CSS_MAX_SENSOR_HEIGHT          3450
-#endif
-
-/* Limited to reduce vmem pressure */
-#if ISP_VMEM_DEPTH >= 3072
-#define SH_CSS_MAX_CONTINUOUS_SENSOR_WIDTH  SH_CSS_MAX_SENSOR_WIDTH
-#define SH_CSS_MAX_CONTINUOUS_SENSOR_HEIGHT SH_CSS_MAX_SENSOR_HEIGHT
-#else
-#define SH_CSS_MAX_CONTINUOUS_SENSOR_WIDTH  3264
-#define SH_CSS_MAX_CONTINUOUS_SENSOR_HEIGHT 2448
-#endif
-/* When using bayer decimation */
-/*
-#define SH_CSS_MAX_CONTINUOUS_SENSOR_WIDTH_DEC  4224
-#define SH_CSS_MAX_CONTINUOUS_SENSOR_HEIGHT_DEC 3168
-*/
-#define SH_CSS_MAX_CONTINUOUS_SENSOR_WIDTH_DEC  SH_CSS_MAX_SENSOR_WIDTH
-#define SH_CSS_MAX_CONTINUOUS_SENSOR_HEIGHT_DEC SH_CSS_MAX_SENSOR_HEIGHT
-
-#define SH_CSS_MIN_SENSOR_WIDTH           2
-#define SH_CSS_MIN_SENSOR_HEIGHT          2
-
-#if defined(IS_ISP_2400_SYSTEM)
-/* MAX width and height set to the same to allow for rotated
- * resolutions. */
-#define SH_CSS_MAX_VF_WIDTH               1920
-#define SH_CSS_MAX_VF_HEIGHT              1920
-#else
-#define SH_CSS_MAX_VF_WIDTH               1280
-#define SH_CSS_MAX_VF_HEIGHT              960
-#endif
-/*
-#define SH_CSS_MAX_VF_WIDTH_DEC               1920
-#define SH_CSS_MAX_VF_HEIGHT_DEC              1080
-*/
-#define SH_CSS_MAX_VF_WIDTH_DEC               SH_CSS_MAX_VF_WIDTH
-#define SH_CSS_MAX_VF_HEIGHT_DEC              SH_CSS_MAX_VF_HEIGHT
-
-/* We use 16 bits per coordinate component, including integer
-   and fractional bits */
-#define SH_CSS_MORPH_TABLE_GRID               ISP_VEC_NELEMS
-#define SH_CSS_MORPH_TABLE_ELEM_BYTES         2
-#define SH_CSS_MORPH_TABLE_ELEMS_PER_DDR_WORD \
-	(HIVE_ISP_DDR_WORD_BYTES/SH_CSS_MORPH_TABLE_ELEM_BYTES)
-
-#ifndef ISP2401
-#define SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR   (SH_CSS_MAX_BQ_GRID_WIDTH + 1)
-#define SH_CSS_MAX_SCTBL_HEIGHT_PER_COLOR   (SH_CSS_MAX_BQ_GRID_HEIGHT + 1)
-#else
-/* TODO: I will move macros of "*_SCTBL_*" to SC kernel.
-   "+ 2" should be "+ SH_CSS_SCTBL_CENTERING_MARGIN + SH_CSS_SCTBL_LAST_GRID_COUNT". (michie, Sep/23/2014) */
-#define SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR   (SH_CSS_MAX_BQ_GRID_WIDTH + 2)
-#define SH_CSS_MAX_SCTBL_HEIGHT_PER_COLOR   (SH_CSS_MAX_BQ_GRID_HEIGHT + 2)
-#endif
-#define SH_CSS_MAX_SCTBL_ALIGNED_WIDTH_PER_COLOR \
-	CEIL_MUL(SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR, ISP_VEC_NELEMS)
-
-/* Each line of this table is aligned to the maximum line width. */
-#define SH_CSS_MAX_S3ATBL_WIDTH              SH_CSS_MAX_BQ_GRID_WIDTH
-
-#ifndef ISP2401
-/* The video binary supports a delay of 1 or 2 */
-#define MAX_DVS_FRAME_DELAY		2
-/* We always need one additional frame because the video binary
- * reads the previous and writes the current frame concurrently */
-#define MAX_NUM_VIDEO_DELAY_FRAMES	(MAX_DVS_FRAME_DELAY + 1)
-#define NUM_VIDEO_TNR_FRAMES		2
-
-#define NUM_TNR_FRAMES			2	/* FIXME */
-
-
-#define MAX_NUM_DELAY_FRAMES		MAX_NUM_VIDEO_DELAY_FRAMES
-
-#else
-/* Video mode specific DVS define */
-/* The video binary supports a delay of 1 or 2 frames */
-#define VIDEO_FRAME_DELAY		2
-/* +1 because DVS reads the previous and writes the current frame concurrently */
-#define MAX_NUM_VIDEO_DELAY_FRAMES	(VIDEO_FRAME_DELAY + 1)
-
-/* Preview mode specific DVS define. */
-/* In preview we only need GDC functionality (and not the DVS functionality) */
-/* The minimum number of DVS frames you need is 2, one were GDC reads from and another where GDC writes into */
-#define NUM_PREVIEW_DVS_FRAMES		(2)
-
-/* TNR is no longer exclusive to video, SkyCam preview has TNR too (same kernel as video).
- * All uses the generic define NUM_TNR_FRAMES. The define NUM_VIDEO_TNR_FRAMES has been deprecated.
- *
- * Notes
- * 1) The value depends on the used TNR kernel and is not something that depends on the mode
- *    and it is not something you just could choice.
- * 2) For the luma only pipeline a version that supports two different sets of TNR reference frames
- * is being used.
- *.
- */
-#define NUM_VALID_TNR_REF_FRAMES		(1) /* At least one valid TNR reference frame is required */
-#define NUM_TNR_FRAMES_PER_REF_BUF_SET		(2)
-
-/* In luma-only mode alternate illuminated frames are supported, that requires two double buffers */
-#ifdef ENABLE_LUMA_ONLY
-#define NUM_TNR_REF_BUF_SETS	(2)
-#else
-#define NUM_TNR_REF_BUF_SETS	(1)
-#endif
-
-#define NUM_TNR_FRAMES		(NUM_TNR_FRAMES_PER_REF_BUF_SET * NUM_TNR_REF_BUF_SETS)
-
-#define MAX_NUM_DELAY_FRAMES	MAX(MAX_NUM_VIDEO_DELAY_FRAMES, NUM_PREVIEW_DVS_FRAMES)
-
-#endif
-
-/* Note that this is the define used to configure all data structures common for all modes */
-/* It should be equal or bigger to the max number of DVS frames for all possible modes */
-/* Rules: these implement logic shared between the host code and ISP firmware.
-   The ISP firmware needs these rules to be applied at pre-processor time,
-   that's why these are macros, not functions. */
-#define _ISP_BQS(num)  ((num)/2)
-#define _ISP_VECS(width) CEIL_DIV(width, ISP_VEC_NELEMS)
-
-#define ISP_BQ_GRID_WIDTH(elements_per_line, deci_factor_log2) \
-	CEIL_SHIFT(elements_per_line/2,  deci_factor_log2)
-#define ISP_BQ_GRID_HEIGHT(lines_per_frame, deci_factor_log2) \
-	CEIL_SHIFT(lines_per_frame/2,  deci_factor_log2)
-#define ISP_C_VECTORS_PER_LINE(elements_per_line) \
-	_ISP_VECS(elements_per_line/2)
-
-/* The morphing table is similar to the shading table in the sense that we
-   have 1 more value than we have cells in the grid. */
-#define _ISP_MORPH_TABLE_WIDTH(int_width) \
-	(CEIL_DIV(int_width, SH_CSS_MORPH_TABLE_GRID) + 1)
-#define _ISP_MORPH_TABLE_HEIGHT(int_height) \
-	(CEIL_DIV(int_height, SH_CSS_MORPH_TABLE_GRID) + 1)
-#define _ISP_MORPH_TABLE_ALIGNED_WIDTH(width) \
-	CEIL_MUL(_ISP_MORPH_TABLE_WIDTH(width), \
-		 SH_CSS_MORPH_TABLE_ELEMS_PER_DDR_WORD)
-
-#ifndef ISP2401
-#define _ISP_SCTBL_WIDTH_PER_COLOR(input_width, deci_factor_log2) \
-	(ISP_BQ_GRID_WIDTH(input_width, deci_factor_log2) + 1)
-#define _ISP_SCTBL_HEIGHT(input_height, deci_factor_log2) \
-	(ISP_BQ_GRID_HEIGHT(input_height, deci_factor_log2) + 1)
-#define _ISP_SCTBL_ALIGNED_WIDTH_PER_COLOR(input_width, deci_factor_log2) \
-	CEIL_MUL(_ISP_SCTBL_WIDTH_PER_COLOR(input_width, deci_factor_log2), \
-		 ISP_VEC_NELEMS)
-
-#endif
-/* *****************************************************************
- * Statistics for 3A (Auto Focus, Auto White Balance, Auto Exposure)
- * *****************************************************************/
-/* if left cropping is used, 3A statistics are also cropped by 2 vectors. */
-#define _ISP_S3ATBL_WIDTH(in_width, deci_factor_log2) \
-	(_ISP_BQS(in_width) >> deci_factor_log2)
-#define _ISP_S3ATBL_HEIGHT(in_height, deci_factor_log2) \
-	(_ISP_BQS(in_height) >> deci_factor_log2)
-#define _ISP_S3A_ELEMS_ISP_WIDTH(width, left_crop) \
-	(width - ((left_crop) ? 2 * ISP_VEC_NELEMS : 0))
-
-#define _ISP_S3ATBL_ISP_WIDTH(in_width, deci_factor_log2) \
-	CEIL_SHIFT(_ISP_BQS(in_width), deci_factor_log2)
-#define _ISP_S3ATBL_ISP_HEIGHT(in_height, deci_factor_log2) \
-	CEIL_SHIFT(_ISP_BQS(in_height), deci_factor_log2)
-#define ISP_S3ATBL_VECTORS \
-	_ISP_VECS(SH_CSS_MAX_S3ATBL_WIDTH * \
-		  (sizeof(struct ia_css_3a_output)/sizeof(int32_t)))
-#define ISP_S3ATBL_HI_LO_STRIDE \
-	(ISP_S3ATBL_VECTORS * ISP_VEC_NELEMS)
-#define ISP_S3ATBL_HI_LO_STRIDE_BYTES \
-	(sizeof(unsigned short) * ISP_S3ATBL_HI_LO_STRIDE)
-
-/* Viewfinder support */
-#define __ISP_MAX_VF_OUTPUT_WIDTH(width, left_crop) \
-	(width - 2*ISP_VEC_NELEMS + ((left_crop) ? 2 * ISP_VEC_NELEMS : 0))
-
-#define __ISP_VF_OUTPUT_WIDTH_VECS(out_width, vf_log_downscale) \
-	(_ISP_VECS((out_width) >> (vf_log_downscale)))
-
-#define _ISP_VF_OUTPUT_WIDTH(vf_out_vecs) ((vf_out_vecs) * ISP_VEC_NELEMS)
-#define _ISP_VF_OUTPUT_HEIGHT(out_height, vf_log_ds) \
-	((out_height) >> (vf_log_ds))
-
-#define _ISP_LOG_VECTOR_STEP(mode) \
-	((mode) == IA_CSS_BINARY_MODE_CAPTURE_PP ? 2 : 1)
-
-/* It is preferred to have not more than 2x scaling at one step
- * in GDC (assumption is for capture_pp and yuv_scale stages) */
-#define MAX_PREFERRED_YUV_DS_PER_STEP	2
-
-/* Rules for computing the internal width. This is extremely complicated
- * and definitely needs to be commented and explained. */
-#define _ISP_LEFT_CROP_EXTRA(left_crop) ((left_crop) > 0 ? 2*ISP_VEC_NELEMS : 0)
-
-#define __ISP_MIN_INTERNAL_WIDTH(num_chunks, pipelining, mode) \
-	((num_chunks) * (pipelining) * (1<<_ISP_LOG_VECTOR_STEP(mode)) * \
-	 ISP_VEC_NELEMS)
-
-#define __ISP_PADDED_OUTPUT_WIDTH(out_width, dvs_env_width, left_crop) \
-	((out_width) + MAX(dvs_env_width, _ISP_LEFT_CROP_EXTRA(left_crop)))
-
-#define __ISP_CHUNK_STRIDE_ISP(mode) \
-	((1<<_ISP_LOG_VECTOR_STEP(mode)) * ISP_VEC_NELEMS)
-
-#define __ISP_CHUNK_STRIDE_DDR(c_subsampling, num_chunks) \
-	((c_subsampling) * (num_chunks) * HIVE_ISP_DDR_WORD_BYTES)
-#define __ISP_INTERNAL_WIDTH(out_width, \
-			     dvs_env_width, \
-			     left_crop, \
-			     mode, \
-			     c_subsampling, \
-			     num_chunks, \
-			     pipelining) \
-	CEIL_MUL2(CEIL_MUL2(MAX(__ISP_PADDED_OUTPUT_WIDTH(out_width, \
-							    dvs_env_width, \
-							    left_crop), \
-				  __ISP_MIN_INTERNAL_WIDTH(num_chunks, \
-							   pipelining, \
-							   mode) \
-				 ), \
-			  __ISP_CHUNK_STRIDE_ISP(mode) \
-			 ), \
-		 __ISP_CHUNK_STRIDE_DDR(c_subsampling, num_chunks) \
-		)
-
-#define __ISP_INTERNAL_HEIGHT(out_height, dvs_env_height, top_crop) \
-	((out_height) + (dvs_env_height) + top_crop)
-
-/* @GC: Input can be up to sensor resolution when either bayer downscaling
- *	or raw binning is enabled.
- *	Also, during continuous mode, we need to align to 4*NWAY since input
- *	should support binning */
-#define _ISP_MAX_INPUT_WIDTH(max_internal_width, enable_ds, enable_fixed_bayer_ds, enable_raw_bin, \
-				enable_continuous) \
-	((enable_ds) ? \
-	   SH_CSS_MAX_SENSOR_WIDTH :\
-	 (enable_fixed_bayer_ds) ? \
-	   CEIL_MUL(SH_CSS_MAX_CONTINUOUS_SENSOR_WIDTH_DEC, 4*ISP_VEC_NELEMS) : \
-	 (enable_raw_bin) ? \
-	   CEIL_MUL(SH_CSS_MAX_CONTINUOUS_SENSOR_WIDTH, 4*ISP_VEC_NELEMS) : \
-	 (enable_continuous) ? \
-	   SH_CSS_MAX_CONTINUOUS_SENSOR_WIDTH \
-	   : max_internal_width)
-
-#define _ISP_INPUT_WIDTH(internal_width, ds_input_width, enable_ds) \
-	((enable_ds) ? (ds_input_width) : (internal_width))
-
-#define _ISP_MAX_INPUT_HEIGHT(max_internal_height, enable_ds, enable_fixed_bayer_ds, enable_raw_bin, \
-				enable_continuous) \
-	((enable_ds) ? \
-	   SH_CSS_MAX_SENSOR_HEIGHT :\
-	 (enable_fixed_bayer_ds) ? \
-	   SH_CSS_MAX_CONTINUOUS_SENSOR_HEIGHT_DEC : \
-	 (enable_raw_bin || enable_continuous) ? \
-	   SH_CSS_MAX_CONTINUOUS_SENSOR_HEIGHT \
-	   : max_internal_height)
-
-#define _ISP_INPUT_HEIGHT(internal_height, ds_input_height, enable_ds) \
-	((enable_ds) ? (ds_input_height) : (internal_height))
-
-#define SH_CSS_MAX_STAGES 8 /* primary_stage[1-6], capture_pp, vf_pp */
-
-/* For CSI2+ input system, it requires extra paddinga from vmem */
-#ifdef CONFIG_CSI2_PLUS
-#define _ISP_EXTRA_PADDING_VECS 2
-#else
-#define _ISP_EXTRA_PADDING_VECS 0
-#endif /* CONFIG_CSI2_PLUS */
-
-#endif /* _SH_CSS_DEFS_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_dvs_info.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_dvs_info.h
deleted file mode 100644
index 23044aad654f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_dvs_info.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-
-#ifndef __SH_CSS_DVS_INFO_H__
-#define __SH_CSS_DVS_INFO_H__
-
-#include <math_support.h>
-
-/* horizontal 64x64 blocks round up to DVS_BLOCKDIM_X, make even */
-#define DVS_NUM_BLOCKS_X(X)		(CEIL_MUL(CEIL_DIV((X), DVS_BLOCKDIM_X), 2))
-
-/* vertical   64x64 blocks round up to DVS_BLOCKDIM_Y */
-#define DVS_NUM_BLOCKS_Y(X)		(CEIL_DIV((X), DVS_BLOCKDIM_Y_LUMA))
-
-/* Bilinear interpolation (HRT_GDC_BLI_MODE) is the supported method currently.
- * Bicubic interpolation (HRT_GDC_BCI_MODE) is not supported yet */
-#define DVS_GDC_INTERP_METHOD HRT_GDC_BLI_MODE
-
-#define DVS_INPUT_BYTES_PER_PIXEL (1)
-
-#define DVS_NUM_BLOCKS_X_CHROMA(X)	(CEIL_DIV((X), DVS_BLOCKDIM_X))
-
-#define DVS_NUM_BLOCKS_Y_CHROMA(X)	(CEIL_DIV((X), DVS_BLOCKDIM_Y_CHROMA))
-
-#endif /* __SH_CSS_DVS_INFO_H__ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.c
deleted file mode 100644
index 8158ea40d069..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.c
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <linux/slab.h>
-#include <linux/vmalloc.h>
-
-#include <math_support.h>
-#include "platform_support.h"
-#include "sh_css_firmware.h"
-
-#include "sh_css_defs.h"
-#include "ia_css_debug.h"
-#include "sh_css_internal.h"
-#include "ia_css_isp_param.h"
-
-#include "memory_access.h"
-#include "assert_support.h"
-#include "string_support.h"
-
-#include "isp.h"				/* PMEM_WIDTH_LOG2 */
-
-#include "ia_css_isp_params.h"
-#include "ia_css_isp_configs.h"
-#include "ia_css_isp_states.h"
-
-#define _STR(x) #x
-#define STR(x) _STR(x)
-
-struct firmware_header {
-	struct sh_css_fw_bi_file_h file_header;
-	struct ia_css_fw_info      binary_header;
-};
-
-struct fw_param {
-	const char *name;
-	const void *buffer;
-};
-
-/* Warning: same order as SH_CSS_BINARY_ID_* */
-static struct firmware_header *firmware_header;
-
-/* The string STR is a place holder
- * which will be replaced with the actual RELEASE_VERSION
- * during package generation. Please do not modify  */
-#ifndef ISP2401
-static const char *release_version = STR(irci_stable_candrpv_0415_20150521_0458);
-#else
-static const char *release_version = STR(irci_ecr-master_20150911_0724);
-#endif
-
-#define MAX_FW_REL_VER_NAME	300
-static char FW_rel_ver_name[MAX_FW_REL_VER_NAME] = "---";
-
-struct ia_css_fw_info	  sh_css_sp_fw;
-struct ia_css_blob_descr *sh_css_blob_info; /* Only ISP blob info (no SP) */
-unsigned		  sh_css_num_binaries; /* This includes 1 SP binary */
-
-static struct fw_param *fw_minibuffer;
-
-
-char *sh_css_get_fw_version(void)
-{
-	return FW_rel_ver_name;
-}
-
-
-/*
- * Split the loaded firmware into blobs
- */
-
-/* Setup sp/sp1 binary */
-static enum ia_css_err
-setup_binary(struct ia_css_fw_info *fw, const char *fw_data, struct ia_css_fw_info *sh_css_fw, unsigned binary_id)
-{
-	const char *blob_data;
-
-	if ((fw == NULL) || (fw_data == NULL))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	blob_data = fw_data + fw->blob.offset;
-
-	*sh_css_fw = *fw;
-
-	sh_css_fw->blob.code = vmalloc(fw->blob.size);
-	if (sh_css_fw->blob.code == NULL)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-
-	memcpy((void *)sh_css_fw->blob.code, blob_data, fw->blob.size);
-	sh_css_fw->blob.data = (char *)sh_css_fw->blob.code + fw->blob.data_source;
-	fw_minibuffer[binary_id].buffer = sh_css_fw->blob.code;
-
-	return IA_CSS_SUCCESS;
-}
-enum ia_css_err
-sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi, struct ia_css_blob_descr *bd, unsigned index)
-{
-	const char *name;
-	const unsigned char *blob;
-
-	if ((fw == NULL) || (bd == NULL))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	/* Special case: only one binary in fw */
-	if (bi == NULL) bi = (const struct ia_css_fw_info *)fw;
-
-	name = fw + bi->blob.prog_name_offset;
-	blob = (const unsigned char *)fw + bi->blob.offset;
-
-	/* sanity check */
-	if (bi->blob.size != bi->blob.text_size + bi->blob.icache_size + bi->blob.data_size + bi->blob.padding_size) {
-		/* sanity check, note the padding bytes added for section to DDR alignment */
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	if ((bi->blob.offset % (1UL<<(ISP_PMEM_WIDTH_LOG2-3))) != 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	bd->blob = blob;
-	bd->header = *bi;
-
-	if (bi->type == ia_css_isp_firmware || bi->type == ia_css_sp_firmware) {
-		char *namebuffer;
-
-		namebuffer = kstrdup(name, GFP_KERNEL);
-		if (!namebuffer)
-			return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		bd->name = fw_minibuffer[index].name = namebuffer;
-	} else {
-		bd->name = name;
-	}
-
-	if (bi->type == ia_css_isp_firmware) {
-		size_t paramstruct_size = sizeof(struct ia_css_memory_offsets);
-		size_t configstruct_size = sizeof(struct ia_css_config_memory_offsets);
-		size_t statestruct_size = sizeof(struct ia_css_state_memory_offsets);
-
-		char *parambuf = kmalloc(paramstruct_size + configstruct_size + statestruct_size,
-					 GFP_KERNEL);
-		if (!parambuf)
-			return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-
-		bd->mem_offsets.array[IA_CSS_PARAM_CLASS_PARAM].ptr = NULL;
-		bd->mem_offsets.array[IA_CSS_PARAM_CLASS_CONFIG].ptr = NULL;
-		bd->mem_offsets.array[IA_CSS_PARAM_CLASS_STATE].ptr = NULL;
-
-		fw_minibuffer[index].buffer = parambuf;
-
-		/* copy ia_css_memory_offsets */
-		memcpy(parambuf, (void *)(fw + bi->blob.memory_offsets.offsets[IA_CSS_PARAM_CLASS_PARAM]),
-			paramstruct_size);
-		bd->mem_offsets.array[IA_CSS_PARAM_CLASS_PARAM].ptr = parambuf;
-
-		/* copy ia_css_config_memory_offsets */
-		memcpy(parambuf + paramstruct_size,
-				(void *)(fw + bi->blob.memory_offsets.offsets[IA_CSS_PARAM_CLASS_CONFIG]),
-				configstruct_size);
-		bd->mem_offsets.array[IA_CSS_PARAM_CLASS_CONFIG].ptr = parambuf + paramstruct_size;
-
-		/* copy ia_css_state_memory_offsets */
-		memcpy(parambuf + paramstruct_size + configstruct_size,
-				(void *)(fw + bi->blob.memory_offsets.offsets[IA_CSS_PARAM_CLASS_STATE]),
-				statestruct_size);
-		bd->mem_offsets.array[IA_CSS_PARAM_CLASS_STATE].ptr = parambuf + paramstruct_size + configstruct_size;
-	}
-	return IA_CSS_SUCCESS;
-}
-
-bool
-sh_css_check_firmware_version(const char *fw_data)
-{
-	struct sh_css_fw_bi_file_h *file_header;
-
-	firmware_header = (struct firmware_header *)fw_data;
-	file_header = &firmware_header->file_header;
-
-	if (strcmp(file_header->version, release_version) != 0) {
-		return false;
-	} else {
-		/* firmware version matches */
-		return true;
-	}
-}
-
-enum ia_css_err
-sh_css_load_firmware(const char *fw_data,
-		     unsigned int fw_size)
-{
-	unsigned i;
-	struct ia_css_fw_info *binaries;
-	struct sh_css_fw_bi_file_h *file_header;
-	bool valid_firmware = false;
-
-	firmware_header = (struct firmware_header *)fw_data;
-	file_header = &firmware_header->file_header;
-	binaries = &firmware_header->binary_header;
-	strncpy(FW_rel_ver_name, file_header->version, min(sizeof(FW_rel_ver_name), sizeof(file_header->version)) - 1);
-	valid_firmware = sh_css_check_firmware_version(fw_data);
-	if (!valid_firmware) {
-#if !defined(HRT_RTL)
-		IA_CSS_ERROR("CSS code version (%s) and firmware version (%s) mismatch!",
-				file_header->version, release_version);
-		return IA_CSS_ERR_VERSION_MISMATCH;
-#endif
-	} else {
-		IA_CSS_LOG("successfully load firmware version %s", release_version);
-	}
-
-	/* some sanity checks */
-	if (!fw_data || fw_size < sizeof(struct sh_css_fw_bi_file_h))
-		return IA_CSS_ERR_INTERNAL_ERROR;
-
-	if (file_header->h_size != sizeof(struct sh_css_fw_bi_file_h))
-		return IA_CSS_ERR_INTERNAL_ERROR;
-
-	sh_css_num_binaries = file_header->binary_nr;
-	/* Only allocate memory for ISP blob info */
-	if (sh_css_num_binaries > NUM_OF_SPS) {
-		sh_css_blob_info = kmalloc(
-					(sh_css_num_binaries - NUM_OF_SPS) *
-					sizeof(*sh_css_blob_info), GFP_KERNEL);
-		if (!sh_css_blob_info)
-			return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	} else {
-		sh_css_blob_info = NULL;
-	}
-
-	fw_minibuffer = kcalloc(sh_css_num_binaries, sizeof(struct fw_param),
-				GFP_KERNEL);
-	if (!fw_minibuffer)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-
-	for (i = 0; i < sh_css_num_binaries; i++) {
-		struct ia_css_fw_info *bi = &binaries[i];
-		/* note: the var below is made static as it is quite large;
-		   if it is not static it ends up on the stack which could
-		   cause issues for drivers
-		*/
-		static struct ia_css_blob_descr bd;
-		enum ia_css_err err;
-
-		err = sh_css_load_blob_info(fw_data, bi, &bd, i);
-
-		if (err != IA_CSS_SUCCESS)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-
-		if (bi->blob.offset + bi->blob.size > fw_size)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-
-		if (bi->type == ia_css_sp_firmware) {
-			if (i != SP_FIRMWARE)
-				return IA_CSS_ERR_INTERNAL_ERROR;
-			err = setup_binary(bi, fw_data, &sh_css_sp_fw, i);
-			if (err != IA_CSS_SUCCESS)
-				return err;
-		} else {
-			/* All subsequent binaries (including bootloaders) (i>NUM_OF_SPS) are ISP firmware */
-			if (i < NUM_OF_SPS)
-				return IA_CSS_ERR_INTERNAL_ERROR;
-
-			if (bi->type != ia_css_isp_firmware)
-				return IA_CSS_ERR_INTERNAL_ERROR;
-			if (sh_css_blob_info == NULL) /* cannot happen but KW does not see this */
-				return IA_CSS_ERR_INTERNAL_ERROR;
-			sh_css_blob_info[i - NUM_OF_SPS] = bd;
-		}
-	}
-
-	return IA_CSS_SUCCESS;
-}
-
-void sh_css_unload_firmware(void)
-{
-
-	/* release firmware minibuffer */
-	if (fw_minibuffer) {
-		unsigned int i = 0;
-		for (i = 0; i < sh_css_num_binaries; i++) {
-			if (fw_minibuffer[i].name)
-				kfree((void *)fw_minibuffer[i].name);
-			if (fw_minibuffer[i].buffer)
-				vfree((void *)fw_minibuffer[i].buffer);
-		}
-		kfree(fw_minibuffer);
-		fw_minibuffer = NULL;
-	}
-
-	memset(&sh_css_sp_fw, 0, sizeof(sh_css_sp_fw));
-	kfree(sh_css_blob_info);
-	sh_css_blob_info = NULL;
-	sh_css_num_binaries = 0;
-}
-
-hrt_vaddress
-sh_css_load_blob(const unsigned char *blob, unsigned size)
-{
-	hrt_vaddress target_addr = mmgr_malloc(size);
-	/* this will allocate memory aligned to a DDR word boundary which
-	   is required for the CSS DMA to read the instructions. */
-
-	assert(blob != NULL);
-	if (target_addr) 
-		mmgr_store(target_addr, blob, size);
-	return target_addr;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.h
deleted file mode 100644
index 588aabde8a86..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _SH_CSS_FIRMWARE_H_
-#define _SH_CSS_FIRMWARE_H_
-
-#include <system_types.h>
-
-#include <ia_css_err.h>
-#include <ia_css_acc_types.h>
-
-/* This is for the firmware loaded from user space */
-struct  sh_css_fw_bi_file_h {
-	char version[64];		/* branch tag + week day + time */
-	int binary_nr;			/* Number of binaries */
-	unsigned int h_size;		/* sizeof(struct sh_css_fw_bi_file_h) */
-};
-
-extern struct ia_css_fw_info     sh_css_sp_fw;
-#if defined(HAS_BL)
-extern struct ia_css_fw_info     sh_css_bl_fw;
-#endif /* HAS_BL */
-extern struct ia_css_blob_descr *sh_css_blob_info;
-extern unsigned			 sh_css_num_binaries;
-
-char
-*sh_css_get_fw_version(void);
-
-bool
-sh_css_check_firmware_version(const char *fw_data);
-
-enum ia_css_err
-sh_css_load_firmware(const char *fw_data,
-		     unsigned int fw_size);
-
-void sh_css_unload_firmware(void);
-
-hrt_vaddress sh_css_load_blob(const unsigned char *blob, unsigned size);
-
-enum ia_css_err
-sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi, struct ia_css_blob_descr *bd, unsigned int i);
-
-#endif /* _SH_CSS_FIRMWARE_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_frac.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_frac.h
deleted file mode 100644
index 1d1771d71f3c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_frac.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SH_CSS_FRAC_H
-#define __SH_CSS_FRAC_H
-
-#include <math_support.h>
-
-#define sISP_REG_BIT		      ISP_VEC_ELEMBITS
-#define uISP_REG_BIT		      ((unsigned)(sISP_REG_BIT-1))
-#define sSHIFT				    (16-sISP_REG_BIT)
-#define uSHIFT				    ((unsigned)(16-uISP_REG_BIT))
-#define sFRACTION_BITS_FITTING(a) (a-sSHIFT)
-#define uFRACTION_BITS_FITTING(a) ((unsigned)(a-uSHIFT))
-#define sISP_VAL_MIN		      (-(1<<uISP_REG_BIT))
-#define sISP_VAL_MAX		      ((1<<uISP_REG_BIT)-1)
-#define uISP_VAL_MIN		      ((unsigned)0)
-#define uISP_VAL_MAX		      ((unsigned)((1<<uISP_REG_BIT)-1))
-
-/* a:fraction bits for 16bit precision, b:fraction bits for ISP precision */
-#define sDIGIT_FITTING(v, a, b) \
-	min(max((((v)>>sSHIFT) >> max(sFRACTION_BITS_FITTING(a)-(b), 0)), \
-	  sISP_VAL_MIN), sISP_VAL_MAX)
-#define uDIGIT_FITTING(v, a, b) \
-	min((unsigned)max((unsigned)(((v)>>uSHIFT) \
-	>> max((int)(uFRACTION_BITS_FITTING(a)-(b)), 0)), \
-	  uISP_VAL_MIN), uISP_VAL_MAX)
-
-#endif /* __SH_CSS_FRAC_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_host_data.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_host_data.c
deleted file mode 100644
index 348183a221a8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_host_data.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <linux/slab.h>
-#include <ia_css_host_data.h>
-#include <sh_css_internal.h>
-
-struct ia_css_host_data *ia_css_host_data_allocate(size_t size)
-{
-	struct ia_css_host_data *me;
-
-	me =  kmalloc(sizeof(struct ia_css_host_data), GFP_KERNEL);
-	if (!me)
-		return NULL;
-	me->size = (uint32_t)size;
-	me->address = sh_css_malloc(size);
-	if (!me->address) {
-		kfree(me);
-		return NULL;
-	}
-	return me;
-}
-
-void ia_css_host_data_free(struct ia_css_host_data *me)
-{
-	if (me) {
-		sh_css_free(me->address);
-		me->address = NULL;
-		kfree(me);
-	}
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_hrt.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_hrt.c
deleted file mode 100644
index 716d808d56db..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_hrt.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "platform_support.h"
-
-#include "sh_css_hrt.h"
-#include "ia_css_debug.h"
-
-#include "device_access.h"
-
-#define __INLINE_EVENT__
-#include "event_fifo.h"
-#define __INLINE_SP__
-#include "sp.h"
-#define __INLINE_ISP__
-#include "isp.h"
-#define __INLINE_IRQ__
-#include "irq.h"
-#define __INLINE_FIFO_MONITOR__
-#include "fifo_monitor.h"
-
-/* System independent */
-#include "sh_css_internal.h"
-
-bool sh_css_hrt_system_is_idle(void)
-{
-	bool not_idle = false, idle;
-	fifo_channel_t ch;
-
-	idle = sp_ctrl_getbit(SP0_ID, SP_SC_REG, SP_IDLE_BIT);
-	not_idle |= !idle;
-	if (!idle)
-		IA_CSS_WARNING("SP not idle");
-
-	idle = isp_ctrl_getbit(ISP0_ID, ISP_SC_REG, ISP_IDLE_BIT);
-	not_idle |= !idle;
-	if (!idle)
-		IA_CSS_WARNING("ISP not idle");
-
-	for (ch=0; ch<N_FIFO_CHANNEL; ch++) {
-		fifo_channel_state_t state;
-		fifo_channel_get_state(FIFO_MONITOR0_ID, ch, &state);
-		if (state.fifo_valid) {
-			IA_CSS_WARNING("FIFO channel %d is not empty", ch);
-			not_idle = true;
-		}
-	}
-
-	return !not_idle;
-}
-
-enum ia_css_err sh_css_hrt_sp_wait(void)
-{
-#if defined(HAS_IRQ_MAP_VERSION_2)
-	irq_sw_channel_id_t	irq_id = IRQ_SW_CHANNEL0_ID;
-#else
-	irq_sw_channel_id_t	irq_id = IRQ_SW_CHANNEL2_ID;
-#endif
-	/*
-	 * Wait till SP is idle or till there is a SW2 interrupt
-	 * The SW2 interrupt will be used when frameloop runs on SP
-	 * and signals an event with similar meaning as SP idle
-	 * (e.g. frame_done)
-	 */
-	while (!sp_ctrl_getbit(SP0_ID, SP_SC_REG, SP_IDLE_BIT) &&
-		((irq_reg_load(IRQ0_ID,
-			_HRT_IRQ_CONTROLLER_STATUS_REG_IDX) &
-			(1U<<(irq_id + IRQ_SW_CHANNEL_OFFSET))) == 0)) {
-		hrt_sleep();
-	}
-
-	return IA_CSS_SUCCESS;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_hrt.h
deleted file mode 100644
index fd23ed1848ec..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_hrt.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _SH_CSS_HRT_H_
-#define _SH_CSS_HRT_H_
-
-#include <sp.h>
-#include <isp.h>
-
-#include <ia_css_err.h>
-
-/* SP access */
-void sh_css_hrt_sp_start_si(void);
-
-void sh_css_hrt_sp_start_copy_frame(void);
-
-void sh_css_hrt_sp_start_isp(void);
-
-enum ia_css_err sh_css_hrt_sp_wait(void);
-
-bool sh_css_hrt_system_is_idle(void);
-
-#endif /* _SH_CSS_HRT_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_internal.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_internal.h
deleted file mode 100644
index 161122e1bcbc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_internal.h
+++ /dev/null
@@ -1,1089 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _SH_CSS_INTERNAL_H_
-#define _SH_CSS_INTERNAL_H_
-
-#include <system_global.h>
-#include <math_support.h>
-#include <type_support.h>
-#include <platform_support.h>
-#include <stdarg.h>
-
-#if !defined(HAS_NO_INPUT_FORMATTER)
-#include "input_formatter.h"
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#include "input_system.h"
-#endif
-
-#include "ia_css_types.h"
-#include "ia_css_acc_types.h"
-#include "ia_css_buffer.h"
-
-#include "ia_css_binary.h"
-#if !defined(__ISP)
-#include "sh_css_firmware.h" /* not needed/desired on SP/ISP */
-#endif
-#include "sh_css_legacy.h"
-#include "sh_css_defs.h"
-#include "sh_css_uds.h"
-#include "dma.h"	/* N_DMA_CHANNEL_ID */
-#include "ia_css_circbuf_comm.h" /* Circular buffer */
-#include "ia_css_frame_comm.h"
-#include "ia_css_3a.h"
-#include "ia_css_dvs.h"
-#include "ia_css_metadata.h"
-#include "runtime/bufq/interface/ia_css_bufq.h"
-#include "ia_css_timer.h"
-
-/* TODO: Move to a more suitable place when sp pipeline design is done. */
-#define IA_CSS_NUM_CB_SEM_READ_RESOURCE	2
-#define IA_CSS_NUM_CB_SEM_WRITE_RESOURCE	1
-#define IA_CSS_NUM_CBS						2
-#define IA_CSS_CB_MAX_ELEMS					2
-
-/* Use case specific. index limited to IA_CSS_NUM_CB_SEM_READ_RESOURCE or
- * IA_CSS_NUM_CB_SEM_WRITE_RESOURCE for read and write respectively.
- * TODO: Enforce the limitation above.
-*/
-#define IA_CSS_COPYSINK_SEM_INDEX	0
-#define IA_CSS_TAGGER_SEM_INDEX	1
-
-/* Force generation of output event. Used by acceleration pipe. */
-#define IA_CSS_POST_OUT_EVENT_FORCE		2
-
-#define SH_CSS_MAX_BINARY_NAME	64
-
-#define SP_DEBUG_NONE	(0)
-#define SP_DEBUG_DUMP	(1)
-#define SP_DEBUG_COPY	(2)
-#define SP_DEBUG_TRACE	(3)
-#define SP_DEBUG_MINIMAL (4)
-
-#define SP_DEBUG SP_DEBUG_NONE
-#define SP_DEBUG_MINIMAL_OVERWRITE 1
-
-#define SH_CSS_TNR_BIT_DEPTH 8
-#define SH_CSS_REF_BIT_DEPTH 8
-
-/* keep next up to date with the definition for MAX_CB_ELEMS_FOR_TAGGER in tagger.sp.c */
-#if defined(HAS_SP_2400)
-#define NUM_CONTINUOUS_FRAMES	15
-#else
-#define NUM_CONTINUOUS_FRAMES	10
-#endif
-#define NUM_MIPI_FRAMES_PER_STREAM		2
-
-#define NUM_ONLINE_INIT_CONTINUOUS_FRAMES      2
-
-#define NR_OF_PIPELINES			IA_CSS_PIPE_ID_NUM /* Must match with IA_CSS_PIPE_ID_NUM */
-
-#define SH_CSS_MAX_IF_CONFIGS	3 /* Must match with IA_CSS_NR_OF_CONFIGS (not defined yet).*/
-#define SH_CSS_IF_CONFIG_NOT_NEEDED	0xFF
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-#define SH_CSS_ENABLE_METADATA
-#endif
-
-#if defined(SH_CSS_ENABLE_METADATA) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
-#define SH_CSS_ENABLE_METADATA_THREAD
-#endif
-
-
- /*
-  * SH_CSS_MAX_SP_THREADS:
-  *	 sp threads visible to host with connected communication queues
-  *	 these threads are capable of running an image pipe
-  * SH_CSS_MAX_SP_INTERNAL_THREADS:
-  *	 internal sp service threads, no communication queues to host
-  *	 these threads can't be used as image pipe
-  */
-
-#if defined(SH_CSS_ENABLE_METADATA_THREAD)
-#define SH_CSS_SP_INTERNAL_METADATA_THREAD	1
-#else
-#define SH_CSS_SP_INTERNAL_METADATA_THREAD	0
-#endif
-
-#define SH_CSS_SP_INTERNAL_SERVICE_THREAD		1
-
-#ifdef __DISABLE_UNUSED_THREAD__
-	#define SH_CSS_MAX_SP_THREADS			0
-#else
-	#define SH_CSS_MAX_SP_THREADS		5
-#endif
-
-#define SH_CSS_MAX_SP_INTERNAL_THREADS	(\
-	 SH_CSS_SP_INTERNAL_SERVICE_THREAD +\
-	 SH_CSS_SP_INTERNAL_METADATA_THREAD)
-
-#define SH_CSS_MAX_PIPELINES	SH_CSS_MAX_SP_THREADS
-
-/**
- * The C99 standard does not specify the exact object representation of structs;
- * the representation is compiler dependent.
- *
- * The structs that are communicated between host and SP/ISP should have the
- * exact same object representation. The compiler that is used to compile the
- * firmware is hivecc.
- *
- * To check if a different compiler, used to compile a host application, uses
- * another object representation, macros are defined specifying the size of
- * the structs as expected by the firmware.
- *
- * A host application shall verify that a sizeof( ) of the struct is equal to
- * the SIZE_OF_XXX macro of the corresponding struct. If they are not
- * equal, functionality will break.
- */
-#define CALC_ALIGNMENT_MEMBER(x, y)	(CEIL_MUL(x, y) - x)
-#define SIZE_OF_HRT_VADDRESS		sizeof(hive_uint32)
-#define SIZE_OF_IA_CSS_PTR		sizeof(uint32_t)
-
-/* Number of SP's */
-#define NUM_OF_SPS 1
-
-#define NUM_OF_BLS 0
-
-/* Enum for order of Binaries */
-enum sh_css_order_binaries {
-	SP_FIRMWARE = 0,
-	ISP_FIRMWARE
-};
-
- /*
- * JB: keep next enum in sync with thread id's
- * and pipe id's
- */
-enum sh_css_pipe_config_override {
-	SH_CSS_PIPE_CONFIG_OVRD_NONE     = 0,
-	SH_CSS_PIPE_CONFIG_OVRD_NO_OVRD  = 0xffff
-};
-
-enum host2sp_commands {
-	host2sp_cmd_error = 0,
-	/*
-	 * The host2sp_cmd_ready command is the only command written by the SP
-	 * It acknowledges that is previous command has been received.
-	 * (this does not mean that the command has been executed)
-	 * It also indicates that a new command can be send (it is a queue
-	 * with depth 1).
-	 */
-	host2sp_cmd_ready = 1,
-	/* Command written by the Host */
-	host2sp_cmd_dummy,		/* No action, can be used as watchdog */
-	host2sp_cmd_start_flash,	/* Request SP to start the flash */
-	host2sp_cmd_terminate,		/* SP should terminate itself */
-	N_host2sp_cmd
-};
-
-/* Enumeration used to indicate the events that are produced by
- *  the SP and consumed by the Host.
- *
- * !!!IMPORTANT!!! KEEP THE FOLLOWING IN SYNC:
- * 1) "enum ia_css_event_type"					(ia_css_event_public.h)
- * 2) "enum sh_css_sp_event_type"				(sh_css_internal.h)
- * 3) "enum ia_css_event_type event_id_2_event_mask"		(event_handler.sp.c)
- * 4) "enum ia_css_event_type convert_event_sp_to_host_domain"	(sh_css.c)
- */
-enum sh_css_sp_event_type {
-	SH_CSS_SP_EVENT_OUTPUT_FRAME_DONE,
-	SH_CSS_SP_EVENT_SECOND_OUTPUT_FRAME_DONE,
-	SH_CSS_SP_EVENT_VF_OUTPUT_FRAME_DONE,
-	SH_CSS_SP_EVENT_SECOND_VF_OUTPUT_FRAME_DONE,
-	SH_CSS_SP_EVENT_3A_STATISTICS_DONE,
-	SH_CSS_SP_EVENT_DIS_STATISTICS_DONE,
-	SH_CSS_SP_EVENT_PIPELINE_DONE,
-	SH_CSS_SP_EVENT_FRAME_TAGGED,
-	SH_CSS_SP_EVENT_INPUT_FRAME_DONE,
-	SH_CSS_SP_EVENT_METADATA_DONE,
-	SH_CSS_SP_EVENT_LACE_STATISTICS_DONE,
-	SH_CSS_SP_EVENT_ACC_STAGE_COMPLETE,
-	SH_CSS_SP_EVENT_TIMER,
-	SH_CSS_SP_EVENT_PORT_EOF,
-	SH_CSS_SP_EVENT_FW_WARNING,
-	SH_CSS_SP_EVENT_FW_ASSERT,
-	SH_CSS_SP_EVENT_NR_OF_TYPES		/* must be last */
-};
-
-/* xmem address map allocation per pipeline, css pointers */
-/* Note that the struct below should only consist of hrt_vaddress-es
-   Otherwise this will cause a fail in the function ref_sh_css_ddr_address_map
- */
-struct sh_css_ddr_address_map {
-	hrt_vaddress isp_param;
-	hrt_vaddress isp_mem_param[SH_CSS_MAX_STAGES][IA_CSS_NUM_MEMORIES];
-	hrt_vaddress macc_tbl;
-	hrt_vaddress fpn_tbl;
-	hrt_vaddress sc_tbl;
-	hrt_vaddress tetra_r_x;
-	hrt_vaddress tetra_r_y;
-	hrt_vaddress tetra_gr_x;
-	hrt_vaddress tetra_gr_y;
-	hrt_vaddress tetra_gb_x;
-	hrt_vaddress tetra_gb_y;
-	hrt_vaddress tetra_b_x;
-	hrt_vaddress tetra_b_y;
-	hrt_vaddress tetra_ratb_x;
-	hrt_vaddress tetra_ratb_y;
-	hrt_vaddress tetra_batr_x;
-	hrt_vaddress tetra_batr_y;
-	hrt_vaddress dvs_6axis_params_y;
-};
-#define SIZE_OF_SH_CSS_DDR_ADDRESS_MAP_STRUCT					\
-	(SIZE_OF_HRT_VADDRESS +							\
-	(SH_CSS_MAX_STAGES * IA_CSS_NUM_MEMORIES * SIZE_OF_HRT_VADDRESS) +	\
-	(16 * SIZE_OF_HRT_VADDRESS))
-
-/* xmem address map allocation per pipeline */
-struct sh_css_ddr_address_map_size {
-	size_t isp_param;
-	size_t isp_mem_param[SH_CSS_MAX_STAGES][IA_CSS_NUM_MEMORIES];
-	size_t macc_tbl;
-	size_t fpn_tbl;
-	size_t sc_tbl;
-	size_t tetra_r_x;
-	size_t tetra_r_y;
-	size_t tetra_gr_x;
-	size_t tetra_gr_y;
-	size_t tetra_gb_x;
-	size_t tetra_gb_y;
-	size_t tetra_b_x;
-	size_t tetra_b_y;
-	size_t tetra_ratb_x;
-	size_t tetra_ratb_y;
-	size_t tetra_batr_x;
-	size_t tetra_batr_y;
-	size_t dvs_6axis_params_y;
-};
-
-struct sh_css_ddr_address_map_compound {
-	struct sh_css_ddr_address_map		map;
-	struct sh_css_ddr_address_map_size	size;
-};
-
-struct ia_css_isp_parameter_set_info {
-	struct sh_css_ddr_address_map  mem_map;/** pointers to Parameters in ISP format IMPT:
-						    This should be first member of this struct */
-	uint32_t                       isp_parameters_id;/** Unique ID to track which config was actually applied to a particular frame */
-	ia_css_ptr                     output_frame_ptr;/** Output frame to which this config has to be applied (optional) */
-};
-
-/* this struct contains all arguments that can be passed to
-   a binary. It depends on the binary which ones are used. */
-struct sh_css_binary_args {
-	struct ia_css_frame *in_frame;	     /* input frame */
-	struct ia_css_frame *delay_frames[MAX_NUM_VIDEO_DELAY_FRAMES];   /* reference input frame */
-#ifndef ISP2401
-	struct ia_css_frame *tnr_frames[NUM_VIDEO_TNR_FRAMES];   /* tnr frames */
-#else
-	struct ia_css_frame *tnr_frames[NUM_TNR_FRAMES];   /* tnr frames */
-#endif
-	struct ia_css_frame *out_frame[IA_CSS_BINARY_MAX_OUTPUT_PORTS];      /* output frame */
-	struct ia_css_frame *out_vf_frame;   /* viewfinder output frame */
-	bool                 copy_vf;
-	bool                 copy_output;
-	unsigned             vf_downscale_log2;
-};
-
-#if SP_DEBUG == SP_DEBUG_DUMP
-
-#define SH_CSS_NUM_SP_DEBUG 48
-
-struct sh_css_sp_debug_state {
-	unsigned int error;
-	unsigned int debug[SH_CSS_NUM_SP_DEBUG];
-};
-
-#elif SP_DEBUG == SP_DEBUG_COPY
-
-#define SH_CSS_SP_DBG_TRACE_DEPTH	(40)
-
-struct sh_css_sp_debug_trace {
-	uint16_t frame;
-	uint16_t line;
-	uint16_t pixel_distance;
-	uint16_t mipi_used_dword;
-	uint16_t sp_index;
-};
-
-struct sh_css_sp_debug_state {
-	uint16_t if_start_line;
-	uint16_t if_start_column;
-	uint16_t if_cropped_height;
-	uint16_t if_cropped_width;
-	unsigned int index;
-	struct sh_css_sp_debug_trace
-		trace[SH_CSS_SP_DBG_TRACE_DEPTH];
-};
-
-#elif SP_DEBUG == SP_DEBUG_TRACE
-
-#if 1
-/* Example of just one global trace */
-#define SH_CSS_SP_DBG_NR_OF_TRACES	(1)
-#define SH_CSS_SP_DBG_TRACE_DEPTH	(40)
-#else
-/* E.g. if you like seperate traces for 4 threads */
-#define SH_CSS_SP_DBG_NR_OF_TRACES	(4)
-#define SH_CSS_SP_DBG_TRACE_DEPTH	(10)
-#endif
-
-#define SH_CSS_SP_DBG_TRACE_FILE_ID_BIT_POS (13)
-
-struct sh_css_sp_debug_trace {
-	uint16_t time_stamp;
-	uint16_t location;	/* bit 15..13 = file_id, 12..0 = line nr. */
-	uint32_t data;
-};
-
-struct sh_css_sp_debug_state {
-	struct sh_css_sp_debug_trace
-		trace[SH_CSS_SP_DBG_NR_OF_TRACES][SH_CSS_SP_DBG_TRACE_DEPTH];
-	uint16_t index_last[SH_CSS_SP_DBG_NR_OF_TRACES];
-	uint8_t index[SH_CSS_SP_DBG_NR_OF_TRACES];
-};
-
-#elif SP_DEBUG == SP_DEBUG_MINIMAL
-
-#define SH_CSS_NUM_SP_DEBUG 128
-
-struct sh_css_sp_debug_state {
-	unsigned int error;
-	unsigned int debug[SH_CSS_NUM_SP_DEBUG];
-};
-
-#endif
-
-
-struct sh_css_sp_debug_command {
-	/*
-	 * The DMA software-mask,
-	 *	Bit 31...24: unused.
-	 *	Bit 23...16: unused.
-	 *	Bit 15...08: reading-request enabling bits for DMA channel 7..0
-	 *	Bit 07...00: writing-reqeust enabling bits for DMA channel 7..0
-	 *
-	 * For example, "0...0 0...0 11111011 11111101" indicates that the
-	 * writing request through DMA Channel 1 and the reading request
-	 * through DMA channel 2 are both disabled. The others are enabled.
-	 */
-	uint32_t dma_sw_reg;
-};
-
-#if !defined(HAS_NO_INPUT_FORMATTER)
-/* SP input formatter configuration.*/
-struct sh_css_sp_input_formatter_set {
-	uint32_t				stream_format;
-	input_formatter_cfg_t	config_a;
-	input_formatter_cfg_t	config_b;
-};
-#endif
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#define IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT (3)
-#endif
-
-/* SP configuration information */
-struct sh_css_sp_config {
-	uint8_t			no_isp_sync; /* Signal host immediately after start */
-	uint8_t			enable_raw_pool_locking; /** Enable Raw Buffer Locking for HALv3 Support */
-	uint8_t			lock_all;
-	/** If raw buffer locking is enabled, this flag indicates whether raw
-	     frames are locked when their EOF event is successfully sent to the
-	     host (true) or when they are passed to the preview/video pipe
-	     (false). */
-#if !defined(HAS_NO_INPUT_FORMATTER)
-	struct {
-		uint8_t					a_changed;
-		uint8_t					b_changed;
-		uint8_t					isp_2ppc;
-		struct sh_css_sp_input_formatter_set	set[SH_CSS_MAX_IF_CONFIGS]; /* CSI-2 port is used as index. */
-	} input_formatter;
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-	sync_generator_cfg_t	sync_gen;
-	tpg_cfg_t		tpg;
-	prbs_cfg_t		prbs;
-	input_system_cfg_t	input_circuit;
-	uint8_t			input_circuit_cfg_changed;
-	uint32_t		mipi_sizes_for_check[N_CSI_PORTS][IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT];
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	uint8_t                 enable_isys_event_queue;
-#endif
-	uint8_t			disable_cont_vf;
-};
-
-enum sh_css_stage_type {
-	SH_CSS_SP_STAGE_TYPE  = 0,
-	SH_CSS_ISP_STAGE_TYPE = 1
-};
-#define SH_CSS_NUM_STAGE_TYPES 2
-
-#define SH_CSS_PIPE_CONFIG_SAMPLE_PARAMS	(1 << 0)
-#define SH_CSS_PIPE_CONFIG_SAMPLE_PARAMS_MASK \
-	((SH_CSS_PIPE_CONFIG_SAMPLE_PARAMS << SH_CSS_MAX_SP_THREADS)-1)
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
-struct sh_css_sp_pipeline_terminal {
-	union {
-		/* Input System 2401 */
-		virtual_input_system_stream_t virtual_input_system_stream[IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH];
-	} context;
-	/*
-	 * TODO
-	 * - Remove "virtual_input_system_cfg" when the ISYS2401 DLI is ready.
-	 */
-	union {
-		/* Input System 2401 */
-		virtual_input_system_stream_cfg_t virtual_input_system_stream_cfg[IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH];
-	} ctrl;
-};
-
-struct sh_css_sp_pipeline_io {
-	struct sh_css_sp_pipeline_terminal	input;
-	/* pqiao: comment out temporarily to save dmem */
-	/*struct sh_css_sp_pipeline_terminal	output;*/
-};
-
-/* This struct tracks how many streams are registered per CSI port.
- * This is used to track which streams have already been configured.
- * Only when all streams are configured, the CSI RX is started for that port.
- */
-struct sh_css_sp_pipeline_io_status {
-	uint32_t	active[N_INPUT_SYSTEM_CSI_PORT];	/** registered streams */
-	uint32_t	running[N_INPUT_SYSTEM_CSI_PORT];	/** configured streams */
-};
-
-#endif
-enum sh_css_port_dir {
-	SH_CSS_PORT_INPUT  = 0,
-	SH_CSS_PORT_OUTPUT  = 1
-};
-
-enum sh_css_port_type {
-	SH_CSS_HOST_TYPE  = 0,
-	SH_CSS_COPYSINK_TYPE  = 1,
-	SH_CSS_TAGGERSINK_TYPE  = 2
-};
-
-/* Pipe inout settings: output port on 7-4bits, input port on 3-0bits */
-#define SH_CSS_PORT_FLD_WIDTH_IN_BITS (4)
-#define SH_CSS_PORT_TYPE_BIT_FLD(pt) (0x1 << (pt))
-#define SH_CSS_PORT_FLD(pd) ((pd) ? SH_CSS_PORT_FLD_WIDTH_IN_BITS : 0)
-#define SH_CSS_PIPE_PORT_CONFIG_ON(p, pd, pt) ((p) |= (SH_CSS_PORT_TYPE_BIT_FLD(pt) << SH_CSS_PORT_FLD(pd)))
-#define SH_CSS_PIPE_PORT_CONFIG_OFF(p, pd, pt) ((p) &= ~(SH_CSS_PORT_TYPE_BIT_FLD(pt) << SH_CSS_PORT_FLD(pd)))
-#define SH_CSS_PIPE_PORT_CONFIG_SET(p, pd, pt, val) ((val) ? \
-		SH_CSS_PIPE_PORT_CONFIG_ON(p, pd, pt) : SH_CSS_PIPE_PORT_CONFIG_OFF(p, pd, pt))
-#define SH_CSS_PIPE_PORT_CONFIG_GET(p, pd, pt) ((p) & (SH_CSS_PORT_TYPE_BIT_FLD(pt) << SH_CSS_PORT_FLD(pd)))
-#define SH_CSS_PIPE_PORT_CONFIG_IS_CONTINUOUS(p) \
-	(!(SH_CSS_PIPE_PORT_CONFIG_GET(p, SH_CSS_PORT_INPUT, SH_CSS_HOST_TYPE) && \
-	   SH_CSS_PIPE_PORT_CONFIG_GET(p, SH_CSS_PORT_OUTPUT, SH_CSS_HOST_TYPE)))
-
-#define IA_CSS_ACQUIRE_ISP_POS	31
-
-/* Flags for metadata processing */
-#define SH_CSS_METADATA_ENABLED        0x01
-#define SH_CSS_METADATA_PROCESSED      0x02
-#define SH_CSS_METADATA_OFFLINE_MODE   0x04
-#define SH_CSS_METADATA_WAIT_INPUT     0x08
-
-/* @brief Free an array of metadata buffers.
- *
- * @param[in]	num_bufs	Number of metadata buffers to be freed.
- * @param[in]	bufs		Pointer of array of metadata buffers.
- *
- * This function frees an array of metadata buffers.
- */
-void
-ia_css_metadata_free_multiple(unsigned int num_bufs, struct ia_css_metadata **bufs);
-
-/* Macro for handling pipe_qos_config */
-#define QOS_INVALID                  (~0U)
-#define QOS_ALL_STAGES_DISABLED      (0U)
-#define QOS_STAGE_MASK(num)          (0x00000001 << num)
-#define SH_CSS_IS_QOS_PIPE(pipe)               ((pipe)->pipe_qos_config != QOS_INVALID)
-#define SH_CSS_QOS_STAGE_ENABLE(pipe, num)     ((pipe)->pipe_qos_config |= QOS_STAGE_MASK(num))
-#define SH_CSS_QOS_STAGE_DISABLE(pipe, num)    ((pipe)->pipe_qos_config &= ~QOS_STAGE_MASK(num))
-#define SH_CSS_QOS_STAGE_IS_ENABLED(pipe, num) ((pipe)->pipe_qos_config & QOS_STAGE_MASK(num))
-#define SH_CSS_QOS_STAGE_IS_ALL_DISABLED(pipe) ((pipe)->pipe_qos_config == QOS_ALL_STAGES_DISABLED)
-#define SH_CSS_QOS_MODE_PIPE_ADD(mode, pipe)    ((mode) |= (0x1 << (pipe)->pipe_id))
-#define SH_CSS_QOS_MODE_PIPE_REMOVE(mode, pipe) ((mode) &= ~(0x1 << (pipe)->pipe_id))
-#define SH_CSS_IS_QOS_ONLY_MODE(mode)           ((mode) == (0x1 << IA_CSS_PIPE_ID_ACC))
-
-/* Information for a pipeline */
-struct sh_css_sp_pipeline {
-	uint32_t	pipe_id;	/* the pipe ID */
-	uint32_t	pipe_num;	/* the dynamic pipe number */
-	uint32_t	thread_id;	/* the sp thread ID */
-	uint32_t	pipe_config;	/* the pipe config */
-	uint32_t	pipe_qos_config;	/* Bitmap of multiple QOS extension fw state.
-						(0xFFFFFFFF) indicates non QOS pipe.*/
-	uint32_t	inout_port_config;
-	uint32_t	required_bds_factor;
-	uint32_t	dvs_frame_delay;
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	uint32_t	input_system_mode;	/* enum ia_css_input_mode */
-	uint32_t	port_id;	/* port_id for input system */
-#endif
-	uint32_t	num_stages;		/* the pipe config */
-	uint32_t	running;	/* needed for pipe termination */
-	hrt_vaddress	sp_stage_addr[SH_CSS_MAX_STAGES];
-	hrt_vaddress	scaler_pp_lut; /* Early bound LUT */
-	uint32_t	dummy; /* stage ptr is only used on sp but lives in
-				  this struct; needs cleanup */
-	int32_t num_execs; /* number of times to run if this is
-			      an acceleration pipe. */
-#if defined(SH_CSS_ENABLE_METADATA)
-	struct {
-		uint32_t        format;   /* Metadata format in hrt format */
-		uint32_t        width;    /* Width of a line */
-		uint32_t        height;   /* Number of lines */
-		uint32_t        stride;   /* Stride (in bytes) per line */
-		uint32_t        size;     /* Total size (in bytes) */
-		hrt_vaddress    cont_buf; /* Address of continuous buffer */
-	} metadata;
-#endif
-#if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-	uint32_t	output_frame_queue_id;
-#endif
-	union {
-		struct {
-			uint32_t	bytes_available;
-		} bin;
-		struct {
-			uint32_t	height;
-			uint32_t	width;
-			uint32_t	padded_width;
-			uint32_t	max_input_width;
-			uint32_t	raw_bit_depth;
-		} raw;
-	} copy;
-#ifdef ISP2401
-
-	/* Parameters passed to Shading Correction kernel. */
-	struct {
-		uint32_t internal_frame_origin_x_bqs_on_sctbl; /* Origin X (bqs) of internal frame on shading table */
-		uint32_t internal_frame_origin_y_bqs_on_sctbl; /* Origin Y (bqs) of internal frame on shading table */
-	} shading;
-#endif
-};
-
-/*
- * The first frames (with comment Dynamic) can be dynamic or static
- * The other frames (ref_in and below) can only be static
- * Static means that the data addres will not change during the life time
- * of the associated pipe. Dynamic means that the data address can
- * change with every (frame) iteration of the associated pipe
- *
- * s3a and dis are now also dynamic but (stil) handled seperately
- */
-#define SH_CSS_NUM_DYNAMIC_FRAME_IDS (3)
-
-struct ia_css_frames_sp {
-	struct ia_css_frame_sp	in;
-	struct ia_css_frame_sp	out[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	struct ia_css_resolution effective_in_res;
-	struct ia_css_frame_sp	out_vf;
-	struct ia_css_frame_sp_info internal_frame_info;
-	struct ia_css_buffer_sp s3a_buf;
-	struct ia_css_buffer_sp dvs_buf;
-#if defined SH_CSS_ENABLE_METADATA
-	struct ia_css_buffer_sp metadata_buf;
-#endif
-};
-
-/* Information for a single pipeline stage for an ISP */
-struct sh_css_isp_stage {
-	/*
-	 * For compatability and portabilty, only types
-	 * from "stdint.h" are allowed
-	 *
-	 * Use of "enum" and "bool" is prohibited
-	 * Multiple boolean flags can be stored in an
-	 * integer
-	 */
-	struct ia_css_blob_info	  blob_info;
-	struct ia_css_binary_info binary_info;
-	char			  binary_name[SH_CSS_MAX_BINARY_NAME];
-	struct ia_css_isp_param_css_segments mem_initializers;
-};
-
-/* Information for a single pipeline stage */
-struct sh_css_sp_stage {
-	/*
-	 * For compatability and portabilty, only types
-	 * from "stdint.h" are allowed
-	 *
-	 * Use of "enum" and "bool" is prohibited
-	 * Multiple boolean flags can be stored in an
-	 * integer
-	 */
-	uint8_t			num; /* Stage number */
-	uint8_t			isp_online;
-	uint8_t			isp_copy_vf;
-	uint8_t			isp_copy_output;
-	uint8_t			sp_enable_xnr;
-	uint8_t			isp_deci_log_factor;
-	uint8_t			isp_vf_downscale_bits;
-	uint8_t			deinterleaved;
-/*
- * NOTE: Programming the input circuit can only be done at the
- * start of a session. It is illegal to program it during execution
- * The input circuit defines the connectivity
- */
-	uint8_t			program_input_circuit;
-/* enum ia_css_pipeline_stage_sp_func	func; */
-	uint8_t			func;
-	/* The type of the pipe-stage */
-	/* enum sh_css_stage_type	stage_type; */
-	uint8_t			stage_type;
-	uint8_t			num_stripes;
-	uint8_t			isp_pipe_version;
-	struct {
-		uint8_t		vf_output;
-		uint8_t		s3a;
-		uint8_t		sdis;
-		uint8_t		dvs_stats;
-		uint8_t		lace_stats;
-	} enable;
-	/* Add padding to come to a word boundary */
-	/* unsigned char			padding[0]; */
-
-	struct sh_css_crop_pos		sp_out_crop_pos;
-	struct ia_css_frames_sp		frames;
-	struct ia_css_resolution	dvs_envelope;
-	struct sh_css_uds_info		uds;
-	hrt_vaddress			isp_stage_addr;
-	hrt_vaddress			xmem_bin_addr;
-	hrt_vaddress			xmem_map_addr;
-
-	uint16_t		top_cropping;
-	uint16_t		row_stripes_height;
-	uint16_t		row_stripes_overlap_lines;
-	uint8_t			if_config_index; /* Which should be applied by this stage. */
-};
-
-/*
- * Time: 2012-07-19, 17:40.
- * Note: Add a new data memeber "debug" in "sh_css_sp_group". This
- * data member is used to pass the debugging command from the
- * Host to the SP.
- *
- * Time: Before 2012-07-19.
- * Note:
- * Group all host initialized SP variables into this struct.
- * This is initialized every stage through dma.
- * The stage part itself is transfered through sh_css_sp_stage.
-*/
-struct sh_css_sp_group {
-	struct sh_css_sp_config		config;
-	struct sh_css_sp_pipeline	pipe[SH_CSS_MAX_SP_THREADS];
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
-	struct sh_css_sp_pipeline_io	pipe_io[SH_CSS_MAX_SP_THREADS];
-	struct sh_css_sp_pipeline_io_status	pipe_io_status;
-#endif
-	struct sh_css_sp_debug_command	debug;
-};
-
-/* Data in SP dmem that is set from the host every stage. */
-struct sh_css_sp_per_frame_data {
-	/* ddr address of sp_group and sp_stage */
-	hrt_vaddress			sp_group_addr;
-};
-
-#define SH_CSS_NUM_SDW_IRQS 3
-
-/* Output data from SP to css */
-struct sh_css_sp_output {
-	unsigned int			bin_copy_bytes_copied;
-#if SP_DEBUG != SP_DEBUG_NONE
-	struct sh_css_sp_debug_state	debug;
-#endif
-	unsigned int		sw_interrupt_value[SH_CSS_NUM_SDW_IRQS];
-};
-
-#define CONFIG_ON_FRAME_ENQUEUE() 0
-
-/**
- * @brief Data structure for the circular buffer.
- * The circular buffer is empty if "start == end". The
- * circular buffer is full if "(end + 1) % size == start".
- */
-/* Variable Sized Buffer Queue Elements */
-
-#define  IA_CSS_NUM_ELEMS_HOST2SP_BUFFER_QUEUE    6
-#define  IA_CSS_NUM_ELEMS_HOST2SP_PARAM_QUEUE    3
-#define  IA_CSS_NUM_ELEMS_HOST2SP_TAG_CMD_QUEUE  6
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-/* sp-to-host queue is expected to be emptied in ISR since
- * it is used instead of HW interrupts (due to HW design issue).
- * We need one queue element per CSI port. */
-#define  IA_CSS_NUM_ELEMS_SP2HOST_ISYS_EVENT_QUEUE (2 * N_CSI_PORTS)
-/* The host-to-sp queue needs to allow for some delay
- * in the emptying of this queue in the SP since there is no
- * separate SP thread for this. */
-#define  IA_CSS_NUM_ELEMS_HOST2SP_ISYS_EVENT_QUEUE (2 * N_CSI_PORTS)
-#else
-#define  IA_CSS_NUM_ELEMS_SP2HOST_ISYS_EVENT_QUEUE 0
-#define  IA_CSS_NUM_ELEMS_HOST2SP_ISYS_EVENT_QUEUE 0
-#define  IA_CSS_NUM_ELEMS_HOST2SP_TAG_CMD_QUEUE  0
-#endif
-
-#if defined(HAS_SP_2400)
-#define  IA_CSS_NUM_ELEMS_HOST2SP_PSYS_EVENT_QUEUE    13
-#define  IA_CSS_NUM_ELEMS_SP2HOST_BUFFER_QUEUE        19
-#define  IA_CSS_NUM_ELEMS_SP2HOST_PSYS_EVENT_QUEUE    26 /* holds events for all type of buffers, hence deeper */
-#else
-#define  IA_CSS_NUM_ELEMS_HOST2SP_PSYS_EVENT_QUEUE    6
-#define  IA_CSS_NUM_ELEMS_SP2HOST_BUFFER_QUEUE        6
-#define  IA_CSS_NUM_ELEMS_SP2HOST_PSYS_EVENT_QUEUE    6
-#endif
-
-struct sh_css_hmm_buffer {
-	union {
-		struct ia_css_isp_3a_statistics  s3a;
-		struct ia_css_isp_dvs_statistics dis;
-		hrt_vaddress skc_dvs_statistics;
-		hrt_vaddress lace_stat;
-		struct ia_css_metadata	metadata;
-		struct frame_data_wrapper {
-			hrt_vaddress	frame_data;
-			uint32_t	flashed;
-			uint32_t	exp_id;
-			uint32_t	isp_parameters_id; /** Unique ID to track which config was
-								actually applied to a particular frame */
-#if CONFIG_ON_FRAME_ENQUEUE()
-			struct sh_css_config_on_frame_enqueue config_on_frame_enqueue;
-#endif
-		} frame;
-		hrt_vaddress ddr_ptrs;
-	} payload;
-	/*
-	 * kernel_ptr is present for host administration purposes only.
-	 * type is uint64_t in order to be 64-bit host compatible.
-	 * uint64_t does not exist on SP/ISP.
-	 * Size of the struct is checked by sp.hive.c.
-	 */
-#if !defined(__ISP)
-	CSS_ALIGN(uint64_t cookie_ptr, 8); /* TODO: check if this alignment is needed */
-	uint64_t kernel_ptr;
-#else
-	CSS_ALIGN(struct { uint32_t a[2]; } cookie_ptr, 8); /* TODO: check if this alignment is needed */
-	struct { uint32_t a[2]; } kernel_ptr;
-#endif
-	struct ia_css_time_meas timing_data;
-	clock_value_t isys_eof_clock_tick;
-};
-#if CONFIG_ON_FRAME_ENQUEUE()
-#define SIZE_OF_FRAME_STRUCT						\
-	(SIZE_OF_HRT_VADDRESS +						\
-	(3 * sizeof(uint32_t)) +					\
-	sizeof(uint32_t))
-#else
-#define SIZE_OF_FRAME_STRUCT						\
-	(SIZE_OF_HRT_VADDRESS +						\
-	(3 * sizeof(uint32_t)))
-#endif
-
-#define SIZE_OF_PAYLOAD_UNION						\
-	(MAX(MAX(MAX(MAX(						\
-	SIZE_OF_IA_CSS_ISP_3A_STATISTICS_STRUCT,			\
-	SIZE_OF_IA_CSS_ISP_DVS_STATISTICS_STRUCT),			\
-	SIZE_OF_IA_CSS_METADATA_STRUCT),				\
-	SIZE_OF_FRAME_STRUCT),						\
-	SIZE_OF_HRT_VADDRESS))
-
-/* Do not use sizeof(uint64_t) since that does not exist of SP */
-#define SIZE_OF_SH_CSS_HMM_BUFFER_STRUCT				\
-	(SIZE_OF_PAYLOAD_UNION +					\
-	CALC_ALIGNMENT_MEMBER(SIZE_OF_PAYLOAD_UNION, 8) +		\
-	8 +						\
-	8 +						\
-	SIZE_OF_IA_CSS_TIME_MEAS_STRUCT +				\
-	SIZE_OF_IA_CSS_CLOCK_TICK_STRUCT +			\
-	CALC_ALIGNMENT_MEMBER(SIZE_OF_IA_CSS_CLOCK_TICK_STRUCT, 8))
-
-enum sh_css_queue_type {
-	sh_css_invalid_queue_type = -1,
-	sh_css_host2sp_buffer_queue,
-	sh_css_sp2host_buffer_queue,
-	sh_css_host2sp_psys_event_queue,
-	sh_css_sp2host_psys_event_queue,
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	sh_css_sp2host_isys_event_queue,
-	sh_css_host2sp_isys_event_queue,
-	sh_css_host2sp_tag_cmd_queue,
-#endif
-};
-
-struct sh_css_event_irq_mask {
-	uint16_t or_mask;
-	uint16_t and_mask;
-};
-#define SIZE_OF_SH_CSS_EVENT_IRQ_MASK_STRUCT				\
-	(2 * sizeof(uint16_t))
-
-struct host_sp_communication {
-	/*
-	 * Don't use enum host2sp_commands, because the sizeof an enum is
-	 * compiler dependant and thus non-portable
-	 */
-	uint32_t host2sp_command;
-
-	/*
-	 * The frame buffers that are reused by the
-	 * copy pipe in the offline preview mode.
-	 *
-	 * host2sp_offline_frames[0]: the input frame of the preview pipe.
-	 * host2sp_offline_frames[1]: the output frame of the copy pipe.
-	 *
-	 * TODO:
-	 *   Remove it when the Host and the SP is decoupled.
-	 */
-	hrt_vaddress host2sp_offline_frames[NUM_CONTINUOUS_FRAMES];
-	hrt_vaddress host2sp_offline_metadata[NUM_CONTINUOUS_FRAMES];
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	hrt_vaddress host2sp_mipi_frames[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM];
-	hrt_vaddress host2sp_mipi_metadata[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM];
-	uint32_t host2sp_num_mipi_frames[N_CSI_PORTS];
-#endif
-	uint32_t host2sp_cont_avail_num_raw_frames;
-	uint32_t host2sp_cont_extra_num_raw_frames;
-	uint32_t host2sp_cont_target_num_raw_frames;
-	struct sh_css_event_irq_mask host2sp_event_irq_mask[NR_OF_PIPELINES];
-
-};
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-#define SIZE_OF_HOST_SP_COMMUNICATION_STRUCT				\
-	(sizeof(uint32_t) +						\
-	(NUM_CONTINUOUS_FRAMES * SIZE_OF_HRT_VADDRESS * 2) +		\
-	(N_CSI_PORTS * NUM_MIPI_FRAMES_PER_STREAM * SIZE_OF_HRT_VADDRESS * 2) +			\
-	((3 + N_CSI_PORTS) * sizeof(uint32_t)) +						\
-	(NR_OF_PIPELINES * SIZE_OF_SH_CSS_EVENT_IRQ_MASK_STRUCT))
-#else
-#define SIZE_OF_HOST_SP_COMMUNICATION_STRUCT				\
-	(sizeof(uint32_t) +						\
-	(NUM_CONTINUOUS_FRAMES * SIZE_OF_HRT_VADDRESS * 2) +		\
-	(3 * sizeof(uint32_t)) +						\
-	(NR_OF_PIPELINES * SIZE_OF_SH_CSS_EVENT_IRQ_MASK_STRUCT))
-#endif
-
-struct host_sp_queues {
-	/*
-	 * Queues for the dynamic frame information,
-	 * i.e. the "in_frame" buffer, the "out_frame"
-	 * buffer and the "vf_out_frame" buffer.
-	 */
-	ia_css_circbuf_desc_t host2sp_buffer_queues_desc
-		[SH_CSS_MAX_SP_THREADS][SH_CSS_MAX_NUM_QUEUES];
-	ia_css_circbuf_elem_t host2sp_buffer_queues_elems
-		[SH_CSS_MAX_SP_THREADS][SH_CSS_MAX_NUM_QUEUES]
-		[IA_CSS_NUM_ELEMS_HOST2SP_BUFFER_QUEUE];
-	ia_css_circbuf_desc_t sp2host_buffer_queues_desc
-		[SH_CSS_MAX_NUM_QUEUES];
-	ia_css_circbuf_elem_t sp2host_buffer_queues_elems
-		[SH_CSS_MAX_NUM_QUEUES][IA_CSS_NUM_ELEMS_SP2HOST_BUFFER_QUEUE];
-
-	/*
-	 * The queues for the events.
-	 */
-	ia_css_circbuf_desc_t host2sp_psys_event_queue_desc;
-	ia_css_circbuf_elem_t host2sp_psys_event_queue_elems
-		[IA_CSS_NUM_ELEMS_HOST2SP_PSYS_EVENT_QUEUE];
-	ia_css_circbuf_desc_t sp2host_psys_event_queue_desc;
-	ia_css_circbuf_elem_t sp2host_psys_event_queue_elems
-		[IA_CSS_NUM_ELEMS_SP2HOST_PSYS_EVENT_QUEUE];
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	/*
-	 * The queues for the ISYS events.
-	 */
-	ia_css_circbuf_desc_t host2sp_isys_event_queue_desc;
-	ia_css_circbuf_elem_t host2sp_isys_event_queue_elems
-		[IA_CSS_NUM_ELEMS_HOST2SP_ISYS_EVENT_QUEUE];
-	ia_css_circbuf_desc_t sp2host_isys_event_queue_desc;
-	ia_css_circbuf_elem_t sp2host_isys_event_queue_elems
-		[IA_CSS_NUM_ELEMS_SP2HOST_ISYS_EVENT_QUEUE];
-	/*
-	 * The queue for the tagger commands.
-	 * CHECK: are these last two present on the 2401 ?
-	 */
-	ia_css_circbuf_desc_t host2sp_tag_cmd_queue_desc;
-	ia_css_circbuf_elem_t host2sp_tag_cmd_queue_elems
-		[IA_CSS_NUM_ELEMS_HOST2SP_TAG_CMD_QUEUE];
-#endif
-};
-
-#define SIZE_OF_QUEUES_ELEMS							\
-	(SIZE_OF_IA_CSS_CIRCBUF_ELEM_S_STRUCT *				\
-	((SH_CSS_MAX_SP_THREADS * SH_CSS_MAX_NUM_QUEUES * IA_CSS_NUM_ELEMS_HOST2SP_BUFFER_QUEUE) + \
-	(SH_CSS_MAX_NUM_QUEUES * IA_CSS_NUM_ELEMS_SP2HOST_BUFFER_QUEUE) +	\
-	(IA_CSS_NUM_ELEMS_HOST2SP_PSYS_EVENT_QUEUE) +				\
-	(IA_CSS_NUM_ELEMS_SP2HOST_PSYS_EVENT_QUEUE) +				\
-	(IA_CSS_NUM_ELEMS_HOST2SP_ISYS_EVENT_QUEUE) +				\
-	(IA_CSS_NUM_ELEMS_SP2HOST_ISYS_EVENT_QUEUE) +				\
-	(IA_CSS_NUM_ELEMS_HOST2SP_TAG_CMD_QUEUE)))
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#define IA_CSS_NUM_CIRCBUF_DESCS 5
-#else
-#ifndef ISP2401
-#define IA_CSS_NUM_CIRCBUF_DESCS 3
-#else
-#define IA_CSS_NUM_CIRCBUF_DESCS 2
-#endif
-#endif
-
-#define SIZE_OF_QUEUES_DESC \
-	((SH_CSS_MAX_SP_THREADS * SH_CSS_MAX_NUM_QUEUES * \
-	  SIZE_OF_IA_CSS_CIRCBUF_DESC_S_STRUCT) + \
-	 (SH_CSS_MAX_NUM_QUEUES * SIZE_OF_IA_CSS_CIRCBUF_DESC_S_STRUCT) + \
-	 (IA_CSS_NUM_CIRCBUF_DESCS * SIZE_OF_IA_CSS_CIRCBUF_DESC_S_STRUCT))
-
-#define SIZE_OF_HOST_SP_QUEUES_STRUCT		\
-	(SIZE_OF_QUEUES_ELEMS + SIZE_OF_QUEUES_DESC)
-
-extern int (*sh_css_printf)(const char *fmt, va_list args);
-
-static inline void
-sh_css_print(const char *fmt, ...)
-{
-	va_list ap;
-
-	if (sh_css_printf) {
-		va_start(ap, fmt);
-		sh_css_printf(fmt, ap);
-		va_end(ap);
-	}
-}
-
-static inline void
-sh_css_vprint(const char *fmt, va_list args)
-{
-	if (sh_css_printf)
-		sh_css_printf(fmt, args);
-}
-
-/* The following #if is there because this header file is also included
-   by SP and ISP code but they do not need this data and HIVECC has alignment
-   issue with the firmware struct/union's.
-   More permanent solution will be to refactor this include.
-*/
-#if !defined(__ISP)
-hrt_vaddress
-sh_css_params_ddr_address_map(void);
-
-enum ia_css_err
-sh_css_params_init(void);
-
-void
-sh_css_params_uninit(void);
-
-void *sh_css_malloc(size_t size);
-
-void *sh_css_calloc(size_t N, size_t size);
-
-void sh_css_free(void *ptr);
-
-/* For Acceleration API: Flush FW (shared buffer pointer) arguments */
-void sh_css_flush(struct ia_css_acc_fw *fw);
-
-
-void
-sh_css_binary_args_reset(struct sh_css_binary_args *args);
-
-/* Check two frames for equality (format, resolution, bits per element) */
-bool
-sh_css_frame_equal_types(const struct ia_css_frame *frame_a,
-			 const struct ia_css_frame *frame_b);
-
-bool
-sh_css_frame_info_equal_resolution(const struct ia_css_frame_info *info_a,
-				   const struct ia_css_frame_info *info_b);
-
-void
-sh_css_capture_enable_bayer_downscaling(bool enable);
-
-void
-sh_css_binary_print(const struct ia_css_binary *binary);
-
-/* aligned argument of sh_css_frame_info_set_width can be used for an extra alignment requirement.
-  When 0, no extra alignment is done. */
-void
-sh_css_frame_info_set_width(struct ia_css_frame_info *info,
-			    unsigned int width,
-			    unsigned int aligned);
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-
-unsigned int
-sh_css_get_mipi_sizes_for_check(const unsigned int port, const unsigned int idx);
-
-#endif
-
-hrt_vaddress
-sh_css_store_sp_group_to_ddr(void);
-
-hrt_vaddress
-sh_css_store_sp_stage_to_ddr(unsigned pipe, unsigned stage);
-
-hrt_vaddress
-sh_css_store_isp_stage_to_ddr(unsigned pipe, unsigned stage);
-
-
-void
-sh_css_update_uds_and_crop_info(
-		const struct ia_css_binary_info *info,
-		const struct ia_css_frame_info *in_frame_info,
-		const struct ia_css_frame_info *out_frame_info,
-		const struct ia_css_resolution *dvs_env,
-		const struct ia_css_dz_config *zoom,
-		const struct ia_css_vector *motion_vector,
-		struct sh_css_uds_info *uds,		/* out */
-		struct sh_css_crop_pos *sp_out_crop_pos,	/* out */
-		bool enable_zoom
-		);
-
-void
-sh_css_invalidate_shading_tables(struct ia_css_stream *stream);
-
-struct ia_css_pipeline *
-ia_css_pipe_get_pipeline(const struct ia_css_pipe *pipe);
-
-unsigned int
-ia_css_pipe_get_pipe_num(const struct ia_css_pipe *pipe);
-
-unsigned int
-ia_css_pipe_get_isp_pipe_version(const struct ia_css_pipe *pipe);
-
-bool
-sh_css_continuous_is_enabled(uint8_t pipe_num);
-
-struct ia_css_pipe *
-find_pipe_by_num(uint32_t pipe_num);
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-void
-ia_css_get_crop_offsets(
-		struct ia_css_pipe *pipe,
-		struct ia_css_frame_info *in_frame);
-#endif
-#endif /* !defined(__ISP) */
-
-#endif /* _SH_CSS_INTERNAL_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_legacy.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_legacy.h
deleted file mode 100644
index 4fd25ba2cd0d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_legacy.h
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _SH_CSS_LEGACY_H_
-#define _SH_CSS_LEGACY_H_
-
-#include <type_support.h>
-#include <ia_css_err.h>
-#include <ia_css_types.h>
-#include <ia_css_frame_public.h>
-#include <ia_css_pipe_public.h>
-#include <ia_css_stream_public.h>
-
-/* The pipe id type, distinguishes the kind of pipes that
- *  can be run in parallel.
- */
-enum ia_css_pipe_id {
-	IA_CSS_PIPE_ID_PREVIEW,
-	IA_CSS_PIPE_ID_COPY,
-	IA_CSS_PIPE_ID_VIDEO,
-	IA_CSS_PIPE_ID_CAPTURE,
-	IA_CSS_PIPE_ID_YUVPP,
-#ifndef ISP2401
-	IA_CSS_PIPE_ID_ACC,
-	IA_CSS_PIPE_ID_NUM
-#else
-	IA_CSS_PIPE_ID_ACC
-#endif
-};
-#ifdef ISP2401
-#define IA_CSS_PIPE_ID_NUM (IA_CSS_PIPE_ID_ACC+1)
-#endif
-
-struct ia_css_pipe_extra_config {
-	bool enable_raw_binning;
-	bool enable_yuv_ds;
-	bool enable_high_speed;
-	bool enable_dvs_6axis;
-	bool enable_reduced_pipe;
-	bool enable_fractional_ds;
-	bool disable_vf_pp;
-};
-
-enum ia_css_err
-ia_css_pipe_create_extra(const struct ia_css_pipe_config *config,
-			 const struct ia_css_pipe_extra_config *extra_config,
-			 struct ia_css_pipe **pipe);
-
-void
-ia_css_pipe_extra_config_defaults(struct ia_css_pipe_extra_config *extra_config);
-
-enum ia_css_err
-ia_css_temp_pipe_to_pipe_id(const struct ia_css_pipe *pipe,
-			    enum ia_css_pipe_id *pipe_id);
-
-/* DEPRECATED. FPN is not supported. */
-enum ia_css_err
-sh_css_set_black_frame(struct ia_css_stream *stream,
-			const struct ia_css_frame *raw_black_frame);
-
-#ifndef ISP2401
-void
-sh_css_enable_cont_capt(bool enable, bool stop_copy_preview);
-
-#endif
-#endif /* _SH_CSS_LEGACY_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metadata.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metadata.c
deleted file mode 100644
index ebdf84d4a138..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metadata.c
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* This file will contain the code to implement the functions declared in ia_css_metadata.h
-   and associated helper functions */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metrics.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metrics.c
deleted file mode 100644
index 48e5542b3a43..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metrics.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "assert_support.h"
-#include "sh_css_metrics.h"
-
-#include "sp.h"
-#include "isp.h"
-
-#include "sh_css_internal.h"
-
-#define MULTIPLE_PCS 0
-#define SUSPEND      0
-#define NOF_PCS      1
-#define RESUME_MASK  0x8
-#define STOP_MASK    0x0
-
-static bool pc_histogram_enabled;
-static struct sh_css_pc_histogram *isp_histogram;
-static struct sh_css_pc_histogram *sp_histogram;
-
-struct sh_css_metrics sh_css_metrics;
-
-void
-sh_css_metrics_start_frame(void)
-{
-	sh_css_metrics.frame_metrics.num_frames++;
-}
-
-static void
-clear_histogram(struct sh_css_pc_histogram *histogram)
-{
-	unsigned i;
-
-	assert(histogram != NULL);
-
-	for (i = 0; i < histogram->length; i++) {
-		histogram->run[i] = 0;
-		histogram->stall[i] = 0;
-		histogram->msink[i] = 0xFFFF;
-	}
-}
-
-void
-sh_css_metrics_enable_pc_histogram(bool enable)
-{
-	pc_histogram_enabled = enable;
-}
-
-static void
-make_histogram(struct sh_css_pc_histogram *histogram, unsigned length)
-{
-	assert(histogram != NULL);
-
-	if (histogram->length)
-		return;
-	if (histogram->run)
-		return;
-	histogram->run = sh_css_malloc(length * sizeof(*histogram->run));
-	if (!histogram->run)
-		return;
-	histogram->stall = sh_css_malloc(length * sizeof(*histogram->stall));
-	if (!histogram->stall)
-		return;
-	histogram->msink = sh_css_malloc(length * sizeof(*histogram->msink));
-	if (!histogram->msink)
-		return;
-
-	histogram->length = length;
-	clear_histogram(histogram);
-}
-
-static void
-insert_binary_metrics(struct sh_css_binary_metrics **l,
-			struct sh_css_binary_metrics *metrics)
-{
-	assert(l != NULL);
-	assert(*l != NULL);
-	assert(metrics != NULL);
-
-	for (; *l; l = &(*l)->next)
-		if (*l == metrics)
-			return;
-
-	*l = metrics;
-	metrics->next = NULL;
-}
-
-void
-sh_css_metrics_start_binary(struct sh_css_binary_metrics *metrics)
-{
-	assert(metrics != NULL);
-
-	if (!pc_histogram_enabled)
-		return;
-
-	isp_histogram = &metrics->isp_histogram;
-	sp_histogram = &metrics->sp_histogram;
-	make_histogram(isp_histogram, ISP_PMEM_DEPTH);
-	make_histogram(sp_histogram, SP_PMEM_DEPTH);
-	insert_binary_metrics(&sh_css_metrics.binary_metrics, metrics);
-}
-
-void
-sh_css_metrics_sample_pcs(void)
-{
-	bool stall;
-	unsigned int pc;
-	unsigned int msink;
-
-#if SUSPEND
-	unsigned int sc = 0;
-	unsigned int stopped_sc = 0;
-	unsigned int resume_sc = 0;
-#endif
-
-
-#if MULTIPLE_PCS
-	int i;
-	unsigned int pc_tab[NOF_PCS];
-
-	for (i = 0; i < NOF_PCS; i++)
-		pc_tab[i] = 0;
-#endif
-
-	if (!pc_histogram_enabled)
-		return;
-
-	if (isp_histogram) {
-#if SUSPEND
-		/* STOP the ISP */
-		isp_ctrl_store(ISP0_ID, ISP_SC_REG, STOP_MASK);
-#endif
-		msink = isp_ctrl_load(ISP0_ID, ISP_CTRL_SINK_REG);
-#if MULTIPLE_PCS
-		for (i = 0; i < NOF_PCS; i++)
-			pc_tab[i] = isp_ctrl_load(ISP0_ID, ISP_PC_REG);
-#else
-		pc = isp_ctrl_load(ISP0_ID, ISP_PC_REG);
-#endif
-
-#if SUSPEND
-		/* RESUME the ISP */
-		isp_ctrl_store(ISP0_ID, ISP_SC_REG, RESUME_MASK);
-#endif
-		isp_histogram->msink[pc] &= msink;
-		stall = (msink != 0x7FF);
-
-		if (stall)
-			isp_histogram->stall[pc]++;
-		else
-			isp_histogram->run[pc]++;
-	}
-
-	if (sp_histogram && 0) {
-		msink = sp_ctrl_load(SP0_ID, SP_CTRL_SINK_REG);
-		pc = sp_ctrl_load(SP0_ID, SP_PC_REG);
-		sp_histogram->msink[pc] &= msink;
-		stall = (msink != 0x7FF);
-		if (stall)
-			sp_histogram->stall[pc]++;
-		else
-			sp_histogram->run[pc]++;
-	}
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metrics.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metrics.h
deleted file mode 100644
index 2ef9238d95ad..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_metrics.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _SH_CSS_METRICS_H_
-#define _SH_CSS_METRICS_H_
-
-#include <type_support.h>
-
-struct sh_css_pc_histogram {
-	unsigned length;
-	unsigned *run;
-	unsigned *stall;
-	unsigned *msink;
-};
-
-struct sh_css_binary_metrics {
-	unsigned mode;
-	unsigned id;
-	struct sh_css_pc_histogram isp_histogram;
-	struct sh_css_pc_histogram sp_histogram;
-	struct sh_css_binary_metrics *next;
-};
-
-struct ia_css_frame_metrics {
-	unsigned num_frames;
-};
-
-struct sh_css_metrics {
-	struct sh_css_binary_metrics *binary_metrics;
-	struct ia_css_frame_metrics   frame_metrics;
-};
-
-extern struct sh_css_metrics sh_css_metrics;
-
-/* includes ia_css_binary.h, which depends on sh_css_metrics.h */
-#include "ia_css_types.h"
-
-/* Sample ISP and SP pc and add to histogram */
-void sh_css_metrics_enable_pc_histogram(bool enable);
-void sh_css_metrics_start_frame(void);
-void sh_css_metrics_start_binary(struct sh_css_binary_metrics *metrics);
-void sh_css_metrics_sample_pcs(void);
-
-#endif /* _SH_CSS_METRICS_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mipi.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mipi.c
deleted file mode 100644
index a6a00024bae8..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mipi.c
+++ /dev/null
@@ -1,749 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_mipi.h"
-#include "sh_css_mipi.h"
-#include <type_support.h>
-#include "system_global.h"
-#include "ia_css_err.h"
-#include "ia_css_pipe.h"
-#include "ia_css_stream_format.h"
-#include "sh_css_stream_format.h"
-#include "ia_css_stream_public.h"
-#include "ia_css_frame_public.h"
-#include "ia_css_input_port.h"
-#include "ia_css_debug.h"
-#include "sh_css_struct.h"
-#include "sh_css_defs.h"
-#include "sh_css_sp.h" /* sh_css_update_host2sp_mipi_frame sh_css_update_host2sp_num_mipi_frames ... */
-#include "sw_event_global.h" /* IA_CSS_PSYS_SW_EVENT_MIPI_BUFFERS_READY */
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-static uint32_t ref_count_mipi_allocation[N_CSI_PORTS]; /* Initialized in mipi_init */
-#endif
-
-enum ia_css_err
-ia_css_mipi_frame_specify(const unsigned int size_mem_words,
-				const bool contiguous)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	my_css.size_mem_words = size_mem_words;
-	(void)contiguous;
-
-	return err;
-}
-
-#ifdef ISP2401
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-/*
- * Check if a source port or TPG/PRBS ID is valid
- */
-static bool ia_css_mipi_is_source_port_valid(struct ia_css_pipe *pipe,
-						unsigned int *pport)
-{
-	bool ret = true;
-	unsigned int port = 0;
-	unsigned int max_ports = 0;
-
-	switch (pipe->stream->config.mode) {
-	case IA_CSS_INPUT_MODE_BUFFERED_SENSOR:
-		port = (unsigned int) pipe->stream->config.source.port.port;
-		max_ports = N_CSI_PORTS;
-		break;
-	case IA_CSS_INPUT_MODE_TPG:
-		port = (unsigned int) pipe->stream->config.source.tpg.id;
-		max_ports = N_CSS_TPG_IDS;
-		break;
-	case IA_CSS_INPUT_MODE_PRBS:
-		port = (unsigned int) pipe->stream->config.source.prbs.id;
-		max_ports = N_CSS_PRBS_IDS;
-		break;
-	default:
-		assert(false);
-		ret = false;
-		break;
-	}
-
-	if (ret) {
-		assert(port < max_ports);
-
-		if (port >= max_ports)
-			ret = false;
-	}
-
-	*pport = port;
-
-	return ret;
-}
-#endif
-
-#endif
-/* Assumptions:
- *	- A line is multiple of 4 bytes = 1 word.
- *	- Each frame has SOF and EOF (each 1 word).
- *	- Each line has format header and optionally SOL and EOL (each 1 word).
- *	- Odd and even lines of YUV420 format are different in bites per pixel size.
- *	- Custom size of embedded data.
- *  -- Interleaved frames are not taken into account.
- *  -- Lines are multiples of 8B, and not necessary of (custom 3B, or 7B
- *  etc.).
- * Result is given in DDR mem words, 32B or 256 bits
- */
-enum ia_css_err
-ia_css_mipi_frame_calculate_size(const unsigned int width,
-				const unsigned int height,
-				const enum atomisp_input_format format,
-				const bool hasSOLandEOL,
-				const unsigned int embedded_data_size_words,
-				unsigned int *size_mem_words)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	unsigned int bits_per_pixel = 0;
-	unsigned int even_line_bytes = 0;
-	unsigned int odd_line_bytes = 0;
-	unsigned int words_per_odd_line = 0;
-	unsigned int words_for_first_line = 0;
-	unsigned int words_per_even_line = 0;
-	unsigned int mem_words_per_even_line = 0;
-	unsigned int mem_words_per_odd_line = 0;
-	unsigned int mem_words_for_first_line = 0;
-	unsigned int mem_words_for_EOF = 0;
-	unsigned int mem_words = 0;
-	unsigned int width_padded = width;
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-	/* The changes will be reverted as soon as RAW
-	 * Buffers are deployed by the 2401 Input System
-	 * in the non-continuous use scenario.
-	 */
-	width_padded += (2 * ISP_VEC_NELEMS);
-#endif
-
-	IA_CSS_ENTER("padded_width=%d, height=%d, format=%d, hasSOLandEOL=%d, embedded_data_size_words=%d\n",
-		     width_padded, height, format, hasSOLandEOL, embedded_data_size_words);
-
-	switch (format) {
-	case ATOMISP_INPUT_FORMAT_RAW_6:		/* 4p, 3B, 24bits */
-		bits_per_pixel = 6;	break;
-	case ATOMISP_INPUT_FORMAT_RAW_7:		/* 8p, 7B, 56bits */
-		bits_per_pixel = 7;		break;
-	case ATOMISP_INPUT_FORMAT_RAW_8:		/* 1p, 1B, 8bits */
-	case ATOMISP_INPUT_FORMAT_BINARY_8:		/*  8bits, TODO: check. */
-	case ATOMISP_INPUT_FORMAT_YUV420_8:		/* odd 2p, 2B, 16bits, even 2p, 4B, 32bits */
-		bits_per_pixel = 8;		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_10:		/* odd 4p, 5B, 40bits, even 4p, 10B, 80bits */
-	case ATOMISP_INPUT_FORMAT_RAW_10:		/* 4p, 5B, 40bits */
-#if !defined(HAS_NO_PACKED_RAW_PIXELS)
-		/* The changes will be reverted as soon as RAW
-		 * Buffers are deployed by the 2401 Input System
-		 * in the non-continuous use scenario.
-		 */
-		bits_per_pixel = 10;
-#else
-		bits_per_pixel = 16;
-#endif
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY:	/* 2p, 3B, 24bits */
-	case ATOMISP_INPUT_FORMAT_RAW_12:		/* 2p, 3B, 24bits */
-		bits_per_pixel = 12;	break;
-	case ATOMISP_INPUT_FORMAT_RAW_14:		/* 4p, 7B, 56bits */
-		bits_per_pixel = 14;	break;
-	case ATOMISP_INPUT_FORMAT_RGB_444:		/* 1p, 2B, 16bits */
-	case ATOMISP_INPUT_FORMAT_RGB_555:		/* 1p, 2B, 16bits */
-	case ATOMISP_INPUT_FORMAT_RGB_565:		/* 1p, 2B, 16bits */
-	case ATOMISP_INPUT_FORMAT_YUV422_8:		/* 2p, 4B, 32bits */
-		bits_per_pixel = 16;	break;
-	case ATOMISP_INPUT_FORMAT_RGB_666:		/* 4p, 9B, 72bits */
-		bits_per_pixel = 18;	break;
-	case ATOMISP_INPUT_FORMAT_YUV422_10:		/* 2p, 5B, 40bits */
-		bits_per_pixel = 20;	break;
-	case ATOMISP_INPUT_FORMAT_RGB_888:		/* 1p, 3B, 24bits */
-		bits_per_pixel = 24;	break;
-
-	case ATOMISP_INPUT_FORMAT_YUV420_16:		/* Not supported */
-	case ATOMISP_INPUT_FORMAT_YUV422_16:		/* Not supported */
-	case ATOMISP_INPUT_FORMAT_RAW_16:		/* TODO: not specified in MIPI SPEC, check */
-	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	odd_line_bytes = (width_padded * bits_per_pixel + 7) >> 3; /* ceil ( bits per line / 8) */
-
-	/* Even lines for YUV420 formats are double in bits_per_pixel. */
-	if (format == ATOMISP_INPUT_FORMAT_YUV420_8
-			|| format == ATOMISP_INPUT_FORMAT_YUV420_10
-			|| format == ATOMISP_INPUT_FORMAT_YUV420_16) {
-		even_line_bytes = (width_padded * 2 * bits_per_pixel + 7) >> 3; /* ceil ( bits per line / 8) */
-	} else {
-		even_line_bytes = odd_line_bytes;
-	}
-
-   /*  a frame represented in memory:  ()- optional; data - payload words.
-	*  addr		0	1	2	3	4	5	6	7:
-	*  first	SOF	(SOL)	PACK_H	data	data	data	data	data
-	*		data	data	data	data	data	data	data	data
-	*		...
-	*		data	data	0	0	0	0	0	0
-	*  second	(EOL)	(SOL)	PACK_H	data	data	data	data	data
-	*		data	data	data	data	data	data	data	data
-	*		...
-	*		data	data	0	0	0	0	0	0
-	*  ...
-	*  last		(EOL)	EOF	0	0	0	0	0	0
-	*
-	*  Embedded lines are regular lines stored before the first and after
-	*  payload lines.
-	*/
-
-	words_per_odd_line = (odd_line_bytes + 3) >> 2;
-		/* ceil(odd_line_bytes/4); word = 4 bytes */
-	words_per_even_line  = (even_line_bytes  + 3) >> 2;
-	words_for_first_line = words_per_odd_line + 2 + (hasSOLandEOL ? 1 : 0);
-		/* + SOF +packet header + optionally (SOL), but (EOL) is not in the first line */
-	words_per_odd_line	+= (1 + (hasSOLandEOL ? 2 : 0));
-		/* each non-first line has format header, and optionally (SOL) and (EOL). */
-	words_per_even_line += (1 + (hasSOLandEOL ? 2 : 0));
-
-	mem_words_per_odd_line	 = (words_per_odd_line + 7) >> 3;
-		/* ceil(words_per_odd_line/8); mem_word = 32 bytes, 8 words */
-	mem_words_for_first_line = (words_for_first_line + 7) >> 3;
-	mem_words_per_even_line  = (words_per_even_line + 7) >> 3;
-	mem_words_for_EOF        = 1; /* last line consisit of the optional (EOL) and EOF */
-
-	mem_words = ((embedded_data_size_words + 7) >> 3) +
-		mem_words_for_first_line +
-				(((height + 1) >> 1) - 1) * mem_words_per_odd_line +
-				/* ceil (height/2) - 1 (first line is calculated separatelly) */
-				  (height      >> 1) * mem_words_per_even_line + /* floor(height/2) */
-				mem_words_for_EOF;
-
-	*size_mem_words = mem_words; /* ceil(words/8); mem word is 32B = 8words. */
-	/* Check if the above is still needed. */
-
-	IA_CSS_LEAVE_ERR(err);
-	return err;
-}
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-enum ia_css_err
-ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port,
-				const unsigned int	size_mem_words)
-{
-	uint32_t idx;
-
-	enum ia_css_err err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-
-	OP___assert(port < N_CSI_PORTS);
-	OP___assert(size_mem_words != 0);
-
-	for (idx = 0; idx < IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT &&
-		my_css.mipi_sizes_for_check[port][idx] != 0;
-		idx++) { /* do nothing */
-	}
-	if (idx < IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT) {
-		my_css.mipi_sizes_for_check[port][idx] = size_mem_words;
-		err = IA_CSS_SUCCESS;
-	}
-
-	return err;
-}
-#endif
-
-void
-mipi_init(void)
-{
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	unsigned int i;
-
-	for (i = 0; i < N_CSI_PORTS; i++)
-		ref_count_mipi_allocation[i] = 0;
-#endif
-}
-
-enum ia_css_err
-calculate_mipi_buff_size(
-		struct ia_css_stream_config *stream_cfg,
-		unsigned int *size_mem_words)
-{
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
-	(void)stream_cfg;
-	(void)size_mem_words;
-#else
-	unsigned int width;
-	unsigned int height;
-	enum atomisp_input_format format;
-	bool pack_raw_pixels;
-
-	unsigned int width_padded;
-	unsigned int bits_per_pixel = 0;
-
-	unsigned int even_line_bytes = 0;
-	unsigned int odd_line_bytes = 0;
-
-	unsigned int words_per_odd_line = 0;
-	unsigned int words_per_even_line = 0;
-
-	unsigned int mem_words_per_even_line = 0;
-	unsigned int mem_words_per_odd_line = 0;
-
-	unsigned int mem_words_per_buff_line = 0;
-	unsigned int mem_words_per_buff = 0;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	/**
-#ifndef ISP2401
-	 * zhengjie.lu@intel.com
-	 *
-#endif
-	 * NOTE
-	 * - In the struct "ia_css_stream_config", there
-	 *   are two members: "input_config" and "isys_config".
-	 *   Both of them provide the same information, e.g.
-	 *   input_res and format.
-	 *
-	 *   Question here is that: which one shall be used?
-	 */
-	width = stream_cfg->input_config.input_res.width;
-	height = stream_cfg->input_config.input_res.height;
-	format = stream_cfg->input_config.format;
-	pack_raw_pixels = stream_cfg->pack_raw_pixels;
-	/* end of NOTE */
-
-	/**
-#ifndef ISP2401
-	 * zhengjie.lu@intel.com
-	 *
-#endif
-	 * NOTE
-	 * - The following code is derived from the
-	 *   existing code "ia_css_mipi_frame_calculate_size()".
-	 *
-	 *   Question here is: why adding "2 * ISP_VEC_NELEMS"
-	 *   to "width_padded", but not making "width_padded"
-	 *   aligned with "2 * ISP_VEC_NELEMS"?
-	 */
-	/* The changes will be reverted as soon as RAW
-	 * Buffers are deployed by the 2401 Input System
-	 * in the non-continuous use scenario.
-	 */
-	width_padded = width + (2 * ISP_VEC_NELEMS);
-	/* end of NOTE */
-
-	IA_CSS_ENTER("padded_width=%d, height=%d, format=%d\n",
-		     width_padded, height, format);
-
-	bits_per_pixel = sh_css_stream_format_2_bits_per_subpixel(format);
-	bits_per_pixel =
-		(format == ATOMISP_INPUT_FORMAT_RAW_10 && pack_raw_pixels) ? bits_per_pixel : 16;
-	if (bits_per_pixel == 0)
-		return IA_CSS_ERR_INTERNAL_ERROR;
-
-	odd_line_bytes = (width_padded * bits_per_pixel + 7) >> 3; /* ceil ( bits per line / 8) */
-
-	/* Even lines for YUV420 formats are double in bits_per_pixel. */
-	if (format == ATOMISP_INPUT_FORMAT_YUV420_8
-		|| format == ATOMISP_INPUT_FORMAT_YUV420_10) {
-		even_line_bytes = (width_padded * 2 * bits_per_pixel + 7) >> 3; /* ceil ( bits per line / 8) */
-	} else {
-		even_line_bytes = odd_line_bytes;
-	}
-
-	words_per_odd_line	 = (odd_line_bytes   + 3) >> 2;
-		/* ceil(odd_line_bytes/4); word = 4 bytes */
-	words_per_even_line  = (even_line_bytes  + 3) >> 2;
-
-	mem_words_per_odd_line	 = (words_per_odd_line + 7) >> 3;
-		/* ceil(words_per_odd_line/8); mem_word = 32 bytes, 8 words */
-	mem_words_per_even_line  = (words_per_even_line + 7) >> 3;
-
-	mem_words_per_buff_line =
-		(mem_words_per_odd_line > mem_words_per_even_line) ? mem_words_per_odd_line : mem_words_per_even_line;
-	mem_words_per_buff = mem_words_per_buff_line * height;
-
-	*size_mem_words = mem_words_per_buff;
-
-	IA_CSS_LEAVE_ERR(err);
-#endif
-	return err;
-}
-
-enum ia_css_err
-allocate_mipi_frames(struct ia_css_pipe *pipe, struct ia_css_stream_info *info)
-{
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
-#ifndef ISP2401
-	unsigned int port;
-#else
-	unsigned int port = 0;
-#endif
-	struct ia_css_frame_info mipi_intermediate_info;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"allocate_mipi_frames(%p) enter:\n", pipe);
-
-	assert(pipe != NULL);
-	assert(pipe->stream != NULL);
-	if ((pipe == NULL) || (pipe->stream == NULL)) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-			"allocate_mipi_frames(%p) exit: pipe or stream is null.\n",
-			pipe);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	if (pipe->stream->config.online) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-			"allocate_mipi_frames(%p) exit: no buffers needed for 2401 pipe mode.\n",
-			pipe);
-		return IA_CSS_SUCCESS;
-	}
-
-#endif
-#ifndef ISP2401
-	if (pipe->stream->config.mode != IA_CSS_INPUT_MODE_BUFFERED_SENSOR) {
-#else
-	if (!(pipe->stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR ||
-		pipe->stream->config.mode == IA_CSS_INPUT_MODE_TPG ||
-		pipe->stream->config.mode == IA_CSS_INPUT_MODE_PRBS)) {
-#endif
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-			"allocate_mipi_frames(%p) exit: no buffers needed for pipe mode.\n",
-			pipe);
-		return IA_CSS_SUCCESS; /* AM TODO: Check  */
-	}
-
-#ifndef ISP2401
-	port = (unsigned int) pipe->stream->config.source.port.port;
-	assert(port < N_CSI_PORTS);
-	if (port >= N_CSI_PORTS) {
-#else
-	if (!ia_css_mipi_is_source_port_valid(pipe, &port)) {
-#endif
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-			"allocate_mipi_frames(%p) exit: error: port is not correct (port=%d).\n",
-			pipe, port);
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-	err = calculate_mipi_buff_size(
-			&(pipe->stream->config),
-			&(my_css.mipi_frame_size[port]));
-#endif
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-	if (ref_count_mipi_allocation[port] != 0) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-			"allocate_mipi_frames(%p) exit: already allocated for this port (port=%d).\n",
-			pipe, port);
-		return IA_CSS_SUCCESS;
-	}
-#else
-	/* 2401 system allows multiple streams to use same physical port. This is not
-	 * true for 2400 system. Currently 2401 uses MIPI buffers as a temporary solution.
-	 * TODO AM: Once that is changed (removed) this code should be removed as well.
-	 * In that case only 2400 related code should remain.
-	 */
-	if (ref_count_mipi_allocation[port] != 0) {
-		ref_count_mipi_allocation[port]++;
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-			"allocate_mipi_frames(%p) leave: nothing to do, already allocated for this port (port=%d).\n",
-			pipe, port);
-		return IA_CSS_SUCCESS;
-	}
-#endif
-
-	ref_count_mipi_allocation[port]++;
-
-	/* TODO: Cleaning needed. */
-	/* This code needs to modified to allocate the MIPI frames in the correct normal way
-	  with an allocate from info, by justin */
-	mipi_intermediate_info = pipe->pipe_settings.video.video_binary.internal_frame_info;
-	mipi_intermediate_info.res.width = 0;
-	mipi_intermediate_info.res.height = 0;
-	/* To indicate it is not (yet) valid format. */
-	mipi_intermediate_info.format = IA_CSS_FRAME_FORMAT_NUM;
-	mipi_intermediate_info.padded_width = 0;
-	mipi_intermediate_info.raw_bit_depth = 0;
-
-	/* AM TODO: mipi frames number should come from stream struct. */
-	my_css.num_mipi_frames[port] = NUM_MIPI_FRAMES_PER_STREAM;
-
-	/* Incremental allocation (per stream), not for all streams at once. */
-	{ /* limit the scope of i,j */
-		unsigned i, j;
-		for (i = 0; i < my_css.num_mipi_frames[port]; i++) {
-			/* free previous frame */
-			if (my_css.mipi_frames[port][i]) {
-				ia_css_frame_free(my_css.mipi_frames[port][i]);
-				my_css.mipi_frames[port][i] = NULL;
-			}
-			/* check if new frame is needed */
-			if (i < my_css.num_mipi_frames[port]) {
-				/* allocate new frame */
-				err = ia_css_frame_allocate_with_buffer_size(
-					&my_css.mipi_frames[port][i],
-					my_css.mipi_frame_size[port] * HIVE_ISP_DDR_WORD_BYTES,
-					false);
-				if (err != IA_CSS_SUCCESS) {
-					for (j = 0; j < i; j++) {
-						if (my_css.mipi_frames[port][j]) {
-							ia_css_frame_free(my_css.mipi_frames[port][j]);
-							my_css.mipi_frames[port][j] = NULL;
-						}
-					}
-					ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-						"allocate_mipi_frames(%p, %d) exit: error: allocation failed.\n",
-						pipe, port);
-					return err;
-				}
-			}
-			if (info->metadata_info.size > 0) {
-				/* free previous metadata buffer */
-				if (my_css.mipi_metadata[port][i] != NULL) {
-					ia_css_metadata_free(my_css.mipi_metadata[port][i]);
-					my_css.mipi_metadata[port][i] = NULL;
-				}
-				/* check if need to allocate a new metadata buffer */
-				if (i < my_css.num_mipi_frames[port]) {
-					/* allocate new metadata buffer */
-					my_css.mipi_metadata[port][i] = ia_css_metadata_allocate(&info->metadata_info);
-					if (my_css.mipi_metadata[port][i] == NULL) {
-						ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-							"allocate_mipi_metadata(%p, %d) failed.\n",
-							pipe, port);
-						return err;
-					}
-				}
-			}
-		}
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"allocate_mipi_frames(%p) exit:\n", pipe);
-
-	return err;
-#else
-	(void)pipe;
-	(void)info;
-	return IA_CSS_SUCCESS;
-#endif
-}
-
-enum ia_css_err
-free_mipi_frames(struct ia_css_pipe *pipe)
-{
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
-#ifndef ISP2401
-	unsigned int port;
-#else
-	unsigned int port = 0;
-#endif
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"free_mipi_frames(%p) enter:\n", pipe);
-
-	/* assert(pipe != NULL); TEMP: TODO: Should be assert only. */
-	if (pipe != NULL) {
-		assert(pipe->stream != NULL);
-		if ((pipe == NULL) || (pipe->stream == NULL)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-				"free_mipi_frames(%p) exit: error: pipe or stream is null.\n",
-				pipe);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
-		}
-
-#ifndef ISP2401
-		if (pipe->stream->config.mode != IA_CSS_INPUT_MODE_BUFFERED_SENSOR) {
-#else
-		if (!(pipe->stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR ||
-			pipe->stream->config.mode == IA_CSS_INPUT_MODE_TPG ||
-			pipe->stream->config.mode == IA_CSS_INPUT_MODE_PRBS)) {
-#endif
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-				"free_mipi_frames(%p) exit: error: wrong mode.\n",
-				pipe);
-			return err;
-		}
-
-#ifndef ISP2401
-		port = (unsigned int) pipe->stream->config.source.port.port;
-		assert(port < N_CSI_PORTS);
-		if (port >= N_CSI_PORTS) {
-#else
-		if (!ia_css_mipi_is_source_port_valid(pipe, &port)) {
-#endif
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-#ifndef ISP2401
-				"free_mipi_frames(%p, %d) exit: error: pipe port is not correct.\n",
-#else
-				"free_mipi_frames(%p) exit: error: pipe port is not correct (port=%d).\n",
-#endif
-				pipe, port);
-			return err;
-		}
-#ifdef ISP2401
-
-#endif
-		if (ref_count_mipi_allocation[port] > 0) {
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-			assert(ref_count_mipi_allocation[port] == 1);
-			if (ref_count_mipi_allocation[port] != 1) {
-				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-					"free_mipi_frames(%p) exit: error: wrong ref_count (ref_count=%d).\n",
-					pipe, ref_count_mipi_allocation[port]);
-				return err;
-			}
-#endif
-
-			ref_count_mipi_allocation[port]--;
-
-			if (ref_count_mipi_allocation[port] == 0) {
-				/* no streams are using this buffer, so free it */
-				unsigned int i;
-				for (i = 0; i < my_css.num_mipi_frames[port]; i++) {
-					if (my_css.mipi_frames[port][i] != NULL) {
-						ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-							"free_mipi_frames(port=%d, num=%d).\n", port, i);
-						ia_css_frame_free(my_css.mipi_frames[port][i]);
-						my_css.mipi_frames[port][i] = NULL;
-					}
-					if (my_css.mipi_metadata[port][i] != NULL) {
-						ia_css_metadata_free(my_css.mipi_metadata[port][i]);
-						my_css.mipi_metadata[port][i] = NULL;
-					}
-				}
-
-				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-					"free_mipi_frames(%p) exit (deallocated).\n", pipe);
-			}
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-			else {
-				/* 2401 system allows multiple streams to use same physical port. This is not
-				 * true for 2400 system. Currently 2401 uses MIPI buffers as a temporary solution.
-				 * TODO AM: Once that is changed (removed) this code should be removed as well.
-				 * In that case only 2400 related code should remain.
-				 */
-				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-					"free_mipi_frames(%p) leave: nothing to do, other streams still use this port (port=%d).\n",
-					pipe, port);
-			}
-#endif
-		}
-	} else { /* pipe ==NULL */
-		/* AM TEMP: free-ing all mipi buffers just like a legacy code. */
-		for (port = CSI_PORT0_ID; port < N_CSI_PORTS; port++) {
-			unsigned int i;
-			for (i = 0; i < my_css.num_mipi_frames[port]; i++) {
-				if (my_css.mipi_frames[port][i] != NULL) {
-					ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-						"free_mipi_frames(port=%d, num=%d).\n", port, i);
-					ia_css_frame_free(my_css.mipi_frames[port][i]);
-					my_css.mipi_frames[port][i] = NULL;
-				}
-				if (my_css.mipi_metadata[port][i] != NULL) {
-					ia_css_metadata_free(my_css.mipi_metadata[port][i]);
-					my_css.mipi_metadata[port][i] = NULL;
-				}
-			}
-			ref_count_mipi_allocation[port] = 0;
-		}
-	}
-#else
-	(void)pipe;
-#endif
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err
-send_mipi_frames(struct ia_css_pipe *pipe)
-{
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
-	unsigned int i;
-#ifndef ISP2401
-	unsigned int port;
-#else
-	unsigned int port = 0;
-#endif
-
-	IA_CSS_ENTER_PRIVATE("pipe=%p", pipe);
-
-	assert(pipe != NULL);
-	assert(pipe->stream != NULL);
-	if (pipe == NULL || pipe->stream == NULL) {
-		IA_CSS_ERROR("pipe or stream is null");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	/* multi stream video needs mipi buffers */
-	/* nothing to be done in other cases. */
-#ifndef ISP2401
-	if (pipe->stream->config.mode != IA_CSS_INPUT_MODE_BUFFERED_SENSOR) {
-#else
-	if (!(pipe->stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR ||
-		pipe->stream->config.mode == IA_CSS_INPUT_MODE_TPG ||
-		pipe->stream->config.mode == IA_CSS_INPUT_MODE_PRBS)) {
-#endif
-		IA_CSS_LOG("nothing to be done for this mode");
-		return IA_CSS_SUCCESS;
-		/* TODO: AM: maybe this should be returning an error. */
-	}
-
-#ifndef ISP2401
-	port = (unsigned int) pipe->stream->config.source.port.port;
-	assert(port < N_CSI_PORTS);
-	if (port >= N_CSI_PORTS) {
-		IA_CSS_ERROR("invalid port specified (%d)", port);
-#else
-	if (!ia_css_mipi_is_source_port_valid(pipe, &port)) {
-		IA_CSS_ERROR("send_mipi_frames(%p) exit: invalid port specified (port=%d).\n", pipe, port);
-#endif
-		return err;
-	}
-
-	/* Hand-over the SP-internal mipi buffers */
-	for (i = 0; i < my_css.num_mipi_frames[port]; i++) {
-		/* Need to include the ofset for port. */
-		sh_css_update_host2sp_mipi_frame(port * NUM_MIPI_FRAMES_PER_STREAM + i,
-			my_css.mipi_frames[port][i]);
-		sh_css_update_host2sp_mipi_metadata(port * NUM_MIPI_FRAMES_PER_STREAM + i,
-			my_css.mipi_metadata[port][i]);
-	}
-	sh_css_update_host2sp_num_mipi_frames(my_css.num_mipi_frames[port]);
-
-	/**********************************
-	 * Send an event to inform the SP
-	 * that all MIPI frames are passed.
-	 **********************************/
-	if (!sh_css_sp_is_running()) {
-		/* SP is not running. The queues are not valid */
-		IA_CSS_ERROR("sp is not running");
-		return err;
-	}
-
-	ia_css_bufq_enqueue_psys_event(
-			IA_CSS_PSYS_SW_EVENT_MIPI_BUFFERS_READY,
-			(uint8_t)port,
-			(uint8_t)my_css.num_mipi_frames[port],
-			0 /* not used */);
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-#else
-	(void)pipe;
-#endif
-	return IA_CSS_SUCCESS;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mipi.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mipi.h
deleted file mode 100644
index 990f678422fd..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mipi.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SH_CSS_MIPI_H
-#define __SH_CSS_MIPI_H
-
-#include <ia_css_err.h>		  /* ia_css_err */
-#include <ia_css_types.h>	  /* ia_css_pipe */
-#include <ia_css_stream_public.h> /* ia_css_stream_config */
-
-void
-mipi_init(void);
-
-enum ia_css_err
-allocate_mipi_frames(struct ia_css_pipe *pipe, struct ia_css_stream_info *info);
-
-enum ia_css_err
-free_mipi_frames(struct ia_css_pipe *pipe);
-
-enum ia_css_err
-send_mipi_frames(struct ia_css_pipe *pipe);
-
-/**
- * @brief Calculate the required MIPI buffer sizes.
- * Based on the stream configuration, calculate the
- * required MIPI buffer sizes (in DDR words).
- *
- * @param[in]	stream_cfg		Point to the target stream configuration
- * @param[out]	size_mem_words	MIPI buffer size in DDR words.
- *
- * @return
- */
-enum ia_css_err
-calculate_mipi_buff_size(
-		struct ia_css_stream_config *stream_cfg,
-		unsigned int *size_mem_words);
-
-#endif /* __SH_CSS_MIPI_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mmu.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mmu.c
deleted file mode 100644
index 237e38b2f0c1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_mmu.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_mmu.h"
-#include "ia_css_mmu_private.h"
-#include <ia_css_debug.h>
-#include "sh_css_sp.h"
-#include "sh_css_firmware.h"
-#include "sp.h"
-#include "mmu_device.h"
-
-void
-ia_css_mmu_invalidate_cache(void)
-{
-	const struct ia_css_fw_info *fw = &sh_css_sp_fw;
-	unsigned int HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_mmu_invalidate_cache() enter\n");
-
-	/* if the SP is not running we should not access its dmem */
-	if (sh_css_sp_is_running()) {
-		HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb = fw->info.sp.invalidate_tlb;
-
-		(void)HIVE_ADDR_ia_css_dmaproxy_sp_invalidate_tlb; /* Suppres warnings in CRUN */
-
-		sp_dmem_store_uint32(SP0_ID,
-			(unsigned int)sp_address_of(ia_css_dmaproxy_sp_invalidate_tlb),
-			true);
-	}
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_mmu_invalidate_cache() leave\n");
-}
-
-/* Deprecated, this is an HRT backend function (memory_access.h) */
-void
-sh_css_mmu_set_page_table_base_index(hrt_data base_index)
-{
-	int i;
-	IA_CSS_ENTER_PRIVATE("base_index=0x%08x\n", base_index);
-	for (i = 0; i < N_MMU_ID; i++) {
-		mmu_ID_t mmu_id = i;
-		mmu_set_page_table_base_index(mmu_id, base_index);
-		mmu_invalidate_cache(mmu_id);
-	}
-	IA_CSS_LEAVE_PRIVATE("");
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_morph.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_morph.c
deleted file mode 100644
index 1f4fa25b1e79..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_morph.c
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* This file will contain the code to implement the functions declared in ia_css_morph.h
-   and associated helper functions */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_dvs.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_dvs.c
deleted file mode 100644
index 57dd5e7988c9..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_dvs.c
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "sh_css_param_dvs.h"
-#include <assert_support.h>
-#include <type_support.h>
-#include <ia_css_err.h>
-#include <ia_css_types.h>
-#include "ia_css_debug.h"
-#include "memory_access.h"
-
-static struct ia_css_dvs_6axis_config *
-alloc_dvs_6axis_table(const struct ia_css_resolution *frame_res, struct ia_css_dvs_6axis_config  *dvs_config_src)
-{
-	unsigned int width_y = 0;
-	unsigned int height_y = 0;
-	unsigned int width_uv = 0;
-	unsigned int height_uv = 0;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_dvs_6axis_config  *dvs_config = NULL;
-
-	dvs_config = (struct ia_css_dvs_6axis_config *)sh_css_malloc(sizeof(struct ia_css_dvs_6axis_config));
-	if (dvs_config == NULL)	{
-		IA_CSS_ERROR("out of memory");
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	}
-	else
-	{	/*Initialize new struct with latest config settings*/
-		if (NULL != dvs_config_src) {
-			dvs_config->width_y = width_y = dvs_config_src->width_y;
-			dvs_config->height_y = height_y = dvs_config_src->height_y;
-			dvs_config->width_uv = width_uv = dvs_config_src->width_uv;
-			dvs_config->height_uv = height_uv = dvs_config_src->height_uv;
-			IA_CSS_LOG("alloc_dvs_6axis_table Y: W %d H %d", width_y, height_y);
-		}
-		else if (NULL != frame_res) {
-			dvs_config->width_y = width_y = DVS_TABLE_IN_BLOCKDIM_X_LUMA(frame_res->width);
-			dvs_config->height_y = height_y = DVS_TABLE_IN_BLOCKDIM_Y_LUMA(frame_res->height);
-			dvs_config->width_uv = width_uv = DVS_TABLE_IN_BLOCKDIM_X_CHROMA(frame_res->width / 2); /* UV = Y/2, depens on colour format YUV 4.2.0*/
-			dvs_config->height_uv = height_uv = DVS_TABLE_IN_BLOCKDIM_Y_CHROMA(frame_res->height / 2);/* UV = Y/2, depens on colour format YUV 4.2.0*/
-			IA_CSS_LOG("alloc_dvs_6axis_table Y: W %d H %d", width_y, height_y);
-		}
-
-		/* Generate Y buffers  */
-		dvs_config->xcoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof(uint32_t));
-		if (dvs_config->xcoords_y == NULL) {
-			IA_CSS_ERROR("out of memory");
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-			goto exit;
-		}
-
-		dvs_config->ycoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof(uint32_t));
-		if (dvs_config->ycoords_y == NULL) {
-			IA_CSS_ERROR("out of memory");
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-			goto exit;
-		}
-
-		/* Generate UV buffers  */
-		IA_CSS_LOG("UV W %d H %d", width_uv, height_uv);
-
-		dvs_config->xcoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv * sizeof(uint32_t));
-		if (dvs_config->xcoords_uv == NULL) {
-			IA_CSS_ERROR("out of memory");
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-			goto exit;
-		}
-
-		dvs_config->ycoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv * sizeof(uint32_t));
-		if (dvs_config->ycoords_uv == NULL) {
-			IA_CSS_ERROR("out of memory");
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		}
-exit:
-		if (err != IA_CSS_SUCCESS) {
-			free_dvs_6axis_table(&dvs_config); /* we might have allocated some memory, release this */
-			dvs_config = NULL;
-		}
-	}
-
-	IA_CSS_LEAVE("dvs_config=%p", dvs_config);
-	return dvs_config;
-}
-
-static void
-init_dvs_6axis_table_from_default(struct ia_css_dvs_6axis_config *dvs_config, const struct ia_css_resolution *dvs_offset)
-{
-	unsigned int x, y;
-	unsigned int width_y = dvs_config->width_y;
-	unsigned int height_y = dvs_config->height_y;
-	unsigned int width_uv = dvs_config->width_uv;
-	unsigned int height_uv = dvs_config->height_uv;
-
-	IA_CSS_LOG("Env_X=%d, Env_Y=%d, width_y=%d, height_y=%d",
-			   dvs_offset->width, dvs_offset->height, width_y, height_y);
-	for (y = 0; y < height_y; y++) {
-		for (x = 0; x < width_y; x++) {
-			dvs_config->xcoords_y[y*width_y + x] =  (dvs_offset->width + x*DVS_BLOCKDIM_X) << DVS_COORD_FRAC_BITS;
-		}
-	}
-
-	for (y = 0; y < height_y; y++) {
-		for (x = 0; x < width_y; x++) {
-			dvs_config->ycoords_y[y*width_y + x] =  (dvs_offset->height + y*DVS_BLOCKDIM_Y_LUMA) << DVS_COORD_FRAC_BITS;
-		}
-	}
-
-	for (y = 0; y < height_uv; y++) {
-		for (x = 0; x < width_uv; x++) { /* Envelope dimensions set in Ypixels hence offset UV = offset Y/2 */
-			dvs_config->xcoords_uv[y*width_uv + x] =  ((dvs_offset->width / 2) + x*DVS_BLOCKDIM_X) << DVS_COORD_FRAC_BITS;
-		}
-	}
-
-	for (y = 0; y < height_uv; y++) {
-		for (x = 0; x < width_uv; x++) { /* Envelope dimensions set in Ypixels hence offset UV = offset Y/2 */
-			dvs_config->ycoords_uv[y*width_uv + x] =  ((dvs_offset->height / 2) + y*DVS_BLOCKDIM_Y_CHROMA) << DVS_COORD_FRAC_BITS;
-		}
-	}
-
-}
-
-static void
-init_dvs_6axis_table_from_config(struct ia_css_dvs_6axis_config *dvs_config, struct ia_css_dvs_6axis_config  *dvs_config_src)
-{
-	unsigned int width_y = dvs_config->width_y;
-	unsigned int height_y = dvs_config->height_y;
-	unsigned int width_uv = dvs_config->width_uv;
-	unsigned int height_uv = dvs_config->height_uv;
-
-	memcpy(dvs_config->xcoords_y, dvs_config_src->xcoords_y, (width_y * height_y * sizeof(uint32_t)));
-	memcpy(dvs_config->ycoords_y, dvs_config_src->ycoords_y, (width_y * height_y * sizeof(uint32_t)));
-	memcpy(dvs_config->xcoords_uv, dvs_config_src->xcoords_uv, (width_uv * height_uv * sizeof(uint32_t)));
-	memcpy(dvs_config->ycoords_uv, dvs_config_src->ycoords_uv, (width_uv * height_uv * sizeof(uint32_t)));
-}
-
-struct ia_css_dvs_6axis_config *
-generate_dvs_6axis_table(const struct ia_css_resolution *frame_res, const struct ia_css_resolution *dvs_offset)
-{
-	struct ia_css_dvs_6axis_config *dvs_6axis_table;
-
-	assert(frame_res != NULL);
-	assert(dvs_offset != NULL);
-
-	dvs_6axis_table = alloc_dvs_6axis_table(frame_res, NULL);
-	if (dvs_6axis_table) {
-		init_dvs_6axis_table_from_default(dvs_6axis_table, dvs_offset);
-		return dvs_6axis_table;
-	}
-	return NULL;
-}
-
-struct ia_css_dvs_6axis_config *
-generate_dvs_6axis_table_from_config(struct ia_css_dvs_6axis_config  *dvs_config_src)
-{
-	struct ia_css_dvs_6axis_config *dvs_6axis_table;
-
-	assert(NULL != dvs_config_src);
-
-	dvs_6axis_table = alloc_dvs_6axis_table(NULL, dvs_config_src);
-	if (dvs_6axis_table) {
-		init_dvs_6axis_table_from_config(dvs_6axis_table, dvs_config_src);
-		return dvs_6axis_table;
-	}
-	return NULL;
-}
-
-void
-free_dvs_6axis_table(struct ia_css_dvs_6axis_config  **dvs_6axis_config)
-{
-	assert(dvs_6axis_config != NULL);
-	assert(*dvs_6axis_config != NULL);
-
-	if ((dvs_6axis_config != NULL) && (*dvs_6axis_config != NULL))
-	{
-		IA_CSS_ENTER_PRIVATE("dvs_6axis_config %p", (*dvs_6axis_config));
-		if ((*dvs_6axis_config)->xcoords_y != NULL)
-		{
-			sh_css_free((*dvs_6axis_config)->xcoords_y);
-			(*dvs_6axis_config)->xcoords_y = NULL;
-		}
-
-		if ((*dvs_6axis_config)->ycoords_y != NULL)
-		{
-			sh_css_free((*dvs_6axis_config)->ycoords_y);
-			(*dvs_6axis_config)->ycoords_y = NULL;
-		}
-
-		/* Free up UV buffers */
-		if ((*dvs_6axis_config)->xcoords_uv != NULL)
-		{
-			sh_css_free((*dvs_6axis_config)->xcoords_uv);
-			(*dvs_6axis_config)->xcoords_uv = NULL;
-		}
-
-		if ((*dvs_6axis_config)->ycoords_uv != NULL)
-		{
-			sh_css_free((*dvs_6axis_config)->ycoords_uv);
-			(*dvs_6axis_config)->ycoords_uv = NULL;
-		}
-
-		IA_CSS_LEAVE_PRIVATE("dvs_6axis_config %p", (*dvs_6axis_config));
-		sh_css_free(*dvs_6axis_config);
-		*dvs_6axis_config = NULL;
-	}
-}
-
-void copy_dvs_6axis_table(struct ia_css_dvs_6axis_config *dvs_config_dst,
-			const struct ia_css_dvs_6axis_config *dvs_config_src)
-{
-	unsigned int width_y;
-	unsigned int height_y;
-	unsigned int width_uv;
-	unsigned int height_uv;
-
-	assert(dvs_config_src != NULL);
-	assert(dvs_config_dst != NULL);
-	assert(dvs_config_src->xcoords_y != NULL);
-	assert(dvs_config_src->xcoords_uv != NULL);
-	assert(dvs_config_src->ycoords_y != NULL);
-	assert(dvs_config_src->ycoords_uv != NULL);
-	assert(dvs_config_src->width_y == dvs_config_dst->width_y);
-	assert(dvs_config_src->width_uv == dvs_config_dst->width_uv);
-	assert(dvs_config_src->height_y == dvs_config_dst->height_y);
-	assert(dvs_config_src->height_uv == dvs_config_dst->height_uv);
-
-	width_y = dvs_config_src->width_y;
-	height_y = dvs_config_src->height_y;
-	width_uv = dvs_config_src->width_uv; /* = Y/2, depens on colour format YUV 4.2.0*/
-	height_uv = dvs_config_src->height_uv;
-
-	memcpy(dvs_config_dst->xcoords_y, dvs_config_src->xcoords_y, (width_y * height_y * sizeof(uint32_t)));
-	memcpy(dvs_config_dst->ycoords_y, dvs_config_src->ycoords_y, (width_y * height_y * sizeof(uint32_t)));
-
-	memcpy(dvs_config_dst->xcoords_uv, dvs_config_src->xcoords_uv, (width_uv * height_uv * sizeof(uint32_t)));
-	memcpy(dvs_config_dst->ycoords_uv, dvs_config_src->ycoords_uv, (width_uv * height_uv * sizeof(uint32_t)));
-
-}
-
-void
-ia_css_dvs_statistics_get(enum dvs_statistics_type type,
-			  union ia_css_dvs_statistics_host  *host_stats,
-			  const union ia_css_dvs_statistics_isp *isp_stats)
-{
-
-	if (DVS_STATISTICS == type)
-	{
-		ia_css_get_dvs_statistics(host_stats->p_dvs_statistics_host,
-			isp_stats->p_dvs_statistics_isp);
-	} else if (DVS2_STATISTICS == type)
-	{
-		ia_css_get_dvs2_statistics(host_stats->p_dvs2_statistics_host,
-			isp_stats->p_dvs_statistics_isp);
-	}
-	return;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_dvs.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_dvs.h
deleted file mode 100644
index 79b563dc78ee..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_dvs.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _SH_CSS_PARAMS_DVS_H_
-#define _SH_CSS_PARAMS_DVS_H_
-
-#include <math_support.h>
-#include <ia_css_types.h>
-#ifdef ISP2401
-#include <sh_css_dvs_info.h>
-#endif
-#include "gdc_global.h" /* gdc_warp_param_mem_t */
-
-#define DVS_ENV_MIN_X (12)
-#define DVS_ENV_MIN_Y (12)
-
-#define DVS_BLOCKDIM_X (64)        /* X block height*/
-#define DVS_BLOCKDIM_Y_LUMA (64)   /* Y block height*/
-#define DVS_BLOCKDIM_Y_CHROMA (32) /* UV height block size is half the Y block height*/
-
-#ifndef ISP2401
-/* horizontal 64x64 blocks round up to DVS_BLOCKDIM_X, make even */
-#define DVS_NUM_BLOCKS_X(X)		(CEIL_MUL(CEIL_DIV((X), DVS_BLOCKDIM_X), 2))
-
-/* vertical   64x64 blocks round up to DVS_BLOCKDIM_Y */
-#define DVS_NUM_BLOCKS_Y(X)		(CEIL_DIV((X), DVS_BLOCKDIM_Y_LUMA))
-#define DVS_NUM_BLOCKS_X_CHROMA(X)	(CEIL_DIV((X), DVS_BLOCKDIM_X))
-#define DVS_NUM_BLOCKS_Y_CHROMA(X)	(CEIL_DIV((X), DVS_BLOCKDIM_Y_CHROMA))
-
-
-#endif
-#define DVS_TABLE_IN_BLOCKDIM_X_LUMA(X)	(DVS_NUM_BLOCKS_X(X) + 1)  /* N blocks have N + 1 set of coords */
-#define DVS_TABLE_IN_BLOCKDIM_X_CHROMA(X)   (DVS_NUM_BLOCKS_X_CHROMA(X) + 1)
-#define DVS_TABLE_IN_BLOCKDIM_Y_LUMA(X)		(DVS_NUM_BLOCKS_Y(X) + 1)
-#define DVS_TABLE_IN_BLOCKDIM_Y_CHROMA(X)	(DVS_NUM_BLOCKS_Y_CHROMA(X) + 1)
-
-#define DVS_ENVELOPE_X(X) (((X) == 0) ? (DVS_ENV_MIN_X) : (X))
-#define DVS_ENVELOPE_Y(X) (((X) == 0) ? (DVS_ENV_MIN_Y) : (X))
-
-#define DVS_COORD_FRAC_BITS (10)
-#ifndef ISP2401
-#define DVS_INPUT_BYTES_PER_PIXEL (1)
-#endif
-#define XMEM_ALIGN_LOG2 (5)
-
-#define DVS_6AXIS_COORDS_ELEMS CEIL_MUL(sizeof(gdc_warp_param_mem_t) \
-					, HIVE_ISP_DDR_WORD_BYTES)
-
-/* currently we only support two output with the same resolution, output 0 is th default one. */
-#define DVS_6AXIS_BYTES(binary) \
-	(DVS_6AXIS_COORDS_ELEMS \
-	* DVS_NUM_BLOCKS_X((binary)->out_frame_info[0].res.width) \
-	* DVS_NUM_BLOCKS_Y((binary)->out_frame_info[0].res.height))
-
-#ifndef ISP2401
-/* Bilinear interpolation (HRT_GDC_BLI_MODE) is the supported method currently.
- * Bicubic interpolation (HRT_GDC_BCI_MODE) is not supported yet */
-#define DVS_GDC_INTERP_METHOD HRT_GDC_BLI_MODE
-
-#endif
-struct ia_css_dvs_6axis_config *
-generate_dvs_6axis_table(const struct ia_css_resolution	*frame_res, const struct ia_css_resolution *dvs_offset);
-
-struct ia_css_dvs_6axis_config *
-generate_dvs_6axis_table_from_config(struct ia_css_dvs_6axis_config  *dvs_config_src);
-
-void
-free_dvs_6axis_table(struct ia_css_dvs_6axis_config  **dvs_6axis_config);
-
-void
-copy_dvs_6axis_table(struct ia_css_dvs_6axis_config *dvs_config_dst,
-			 const struct ia_css_dvs_6axis_config *dvs_config_src);
-
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_shading.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_shading.c
deleted file mode 100644
index e6ebd1b08f0d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_shading.c
+++ /dev/null
@@ -1,417 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <linux/slab.h>
-
-#include <math_support.h>
-#include "sh_css_param_shading.h"
-#include "ia_css_shading.h"
-#include "assert_support.h"
-#include "sh_css_defs.h"
-#include "sh_css_internal.h"
-#include "ia_css_debug.h"
-#include "ia_css_pipe_binarydesc.h"
-
-#include "sh_css_hrt.h"
-
-#include "platform_support.h"
-
-/* Bilinear interpolation on shading tables:
- * For each target point T, we calculate the 4 surrounding source points:
- * ul (upper left), ur (upper right), ll (lower left) and lr (lower right).
- * We then calculate the distances from the T to the source points: x0, x1,
- * y0 and y1.
- * We then calculate the value of T:
- *   dx0*dy0*Slr + dx0*dy1*Sur + dx1*dy0*Sll + dx1*dy1*Sul.
- * We choose a grid size of 1x1 which means:
- *   dx1 = 1-dx0
- *   dy1 = 1-dy0
- *
- *   Sul dx0         dx1      Sur
- *    .<----->|<------------->.
- *    ^
- * dy0|
- *    v        T
- *    -        .
- *    ^
- *    |
- * dy1|
- *    v
- *    .                        .
- *   Sll                      Slr
- *
- * Padding:
- * The area that the ISP operates on can include padding both on the left
- * and the right. We need to padd the shading table such that the shading
- * values end up on the correct pixel values. This means we must padd the
- * shading table to match the ISP padding.
- * We can have 5 cases:
- * 1. All 4 points fall in the left padding.
- * 2. The left 2 points fall in the left padding.
- * 3. All 4 points fall in the cropped (target) region.
- * 4. The right 2 points fall in the right padding.
- * 5. All 4 points fall in the right padding.
- * Cases 1 and 5 are easy to handle: we simply use the
- * value 1 in the shading table.
- * Cases 2 and 4 require interpolation that takes into
- * account how far into the padding area the pixels
- * fall. We extrapolate the shading table into the
- * padded area and then interpolate.
- */
-static void
-crop_and_interpolate(unsigned int cropped_width,
-		     unsigned int cropped_height,
-		     unsigned int left_padding,
-		     int right_padding,
-		     int top_padding,
-		     const struct ia_css_shading_table *in_table,
-		     struct ia_css_shading_table *out_table,
-		     enum ia_css_sc_color color)
-{
-	unsigned int i, j,
-		     sensor_width,
-		     sensor_height,
-		     table_width,
-		     table_height,
-		     table_cell_h,
-		     out_cell_size,
-		     in_cell_size,
-		     out_start_row,
-		     padded_width;
-	int out_start_col, /* can be negative to indicate padded space */
-	    table_cell_w;
-	unsigned short *in_ptr,
-		       *out_ptr;
-
-	assert(in_table != NULL);
-	assert(out_table != NULL);
-
-	sensor_width  = in_table->sensor_width;
-	sensor_height = in_table->sensor_height;
-	table_width   = in_table->width;
-	table_height  = in_table->height;
-	in_ptr = in_table->data[color];
-	out_ptr = out_table->data[color];
-
-	padded_width = cropped_width + left_padding + right_padding;
-	out_cell_size = CEIL_DIV(padded_width, out_table->width - 1);
-	in_cell_size  = CEIL_DIV(sensor_width, table_width - 1);
-
-	out_start_col = ((int)sensor_width - (int)cropped_width)/2 - left_padding;
-	out_start_row = ((int)sensor_height - (int)cropped_height)/2 - top_padding;
-	table_cell_w = (int)((table_width-1) * in_cell_size);
-	table_cell_h = (table_height-1) * in_cell_size;
-
-	for (i = 0; i < out_table->height; i++) {
-		int ty, src_y0, src_y1;
-		unsigned int sy0, sy1, dy0, dy1, divy;
-
-		/* calculate target point and make sure it falls within
-		   the table */
-		ty = out_start_row + i * out_cell_size;
-
-		/* calculate closest source points in shading table and
-		   make sure they fall within the table */
-		src_y0 = ty / (int)in_cell_size;
-		if (in_cell_size < out_cell_size)
-			src_y1 = (ty + out_cell_size) / in_cell_size;
-		else
-			src_y1 = src_y0 + 1;
-		src_y0 = clamp(src_y0, 0, (int)table_height-1);
-		src_y1 = clamp(src_y1, 0, (int)table_height-1);
-		ty = min(clamp(ty, 0, (int)sensor_height-1),
-				 (int)table_cell_h);
-
-		/* calculate closest source points for distance computation */
-		sy0 = min(src_y0 * in_cell_size, sensor_height-1);
-		sy1 = min(src_y1 * in_cell_size, sensor_height-1);
-		/* calculate distance between source and target pixels */
-		dy0 = ty - sy0;
-		dy1 = sy1 - ty;
-		divy = sy1 - sy0;
-		if (divy == 0) {
-			dy0 = 1;
-			divy = 1;
-		}
-
-		for (j = 0; j < out_table->width; j++, out_ptr++) {
-			int tx, src_x0, src_x1;
-			unsigned int sx0, sx1, dx0, dx1, divx;
-			unsigned short s_ul, s_ur, s_ll, s_lr;
-
-			/* calculate target point */
-			tx = out_start_col + j * out_cell_size;
-			/* calculate closest source points. */
-			src_x0 = tx / (int)in_cell_size;
-			if (in_cell_size < out_cell_size) {
-				src_x1 = (tx + out_cell_size) /
-					 (int)in_cell_size;
-			} else {
-				src_x1 = src_x0 + 1;
-			}
-			/* if src points fall in padding, select closest ones.*/
-			src_x0 = clamp(src_x0, 0, (int)table_width-1);
-			src_x1 = clamp(src_x1, 0, (int)table_width-1);
-			tx = min(clamp(tx, 0, (int)sensor_width-1),
-				 (int)table_cell_w);
-			/* calculate closest source points for distance
-			   computation */
-			sx0 = min(src_x0 * in_cell_size, sensor_width-1);
-			sx1 = min(src_x1 * in_cell_size, sensor_width-1);
-			/* calculate distances between source and target
-			   pixels */
-			dx0 = tx - sx0;
-			dx1 = sx1 - tx;
-			divx = sx1 - sx0;
-			/* if we're at the edge, we just use the closest
-			   point still in the grid. We make up for the divider
-			   in this case by setting the distance to
-			   out_cell_size, since it's actually 0. */
-			if (divx == 0) {
-				dx0 = 1;
-				divx = 1;
-			}
-
-			/* get source pixel values */
-			s_ul = in_ptr[(table_width*src_y0)+src_x0];
-			s_ur = in_ptr[(table_width*src_y0)+src_x1];
-			s_ll = in_ptr[(table_width*src_y1)+src_x0];
-			s_lr = in_ptr[(table_width*src_y1)+src_x1];
-
-			*out_ptr = (unsigned short) ((dx0*dy0*s_lr + dx0*dy1*s_ur + dx1*dy0*s_ll + dx1*dy1*s_ul) /
-					(divx*divy));
-		}
-	}
-}
-
-void
-sh_css_params_shading_id_table_generate(
-	struct ia_css_shading_table **target_table,
-#ifndef ISP2401
-	const struct ia_css_binary *binary)
-#else
-	unsigned int table_width,
-	unsigned int table_height)
-#endif
-{
-	/* initialize table with ones, shift becomes zero */
-#ifndef ISP2401
-	unsigned int i, j, table_width, table_height;
-#else
-	unsigned int i, j;
-#endif
-	struct ia_css_shading_table *result;
-
-	assert(target_table != NULL);
-#ifndef ISP2401
-	assert(binary != NULL);
-#endif
-
-#ifndef ISP2401
-	table_width  = binary->sctbl_width_per_color;
-	table_height = binary->sctbl_height;
-#endif
-	result = ia_css_shading_table_alloc(table_width, table_height);
-	if (result == NULL) {
-		*target_table = NULL;
-		return;
-	}
-
-	for (i = 0; i < IA_CSS_SC_NUM_COLORS; i++) {
-		for (j = 0; j < table_height * table_width; j++)
-			result->data[i][j] = 1;
-	}
-	result->fraction_bits = 0;
-	*target_table = result;
-}
-
-void
-prepare_shading_table(const struct ia_css_shading_table *in_table,
-		      unsigned int sensor_binning,
-		      struct ia_css_shading_table **target_table,
-		      const struct ia_css_binary *binary,
-		      unsigned int bds_factor)
-{
-	unsigned int input_width,
-		     input_height,
-		     table_width,
-		     table_height,
-		     left_padding,
-		     top_padding,
-		     padded_width,
-		     left_cropping,
-		     i;
-	unsigned int bds_numerator, bds_denominator;
-	int right_padding;
-
-	struct ia_css_shading_table *result;
-
-	assert(target_table != NULL);
-	assert(binary != NULL);
-
-	if (!in_table) {
-#ifndef ISP2401
-		sh_css_params_shading_id_table_generate(target_table, binary);
-#else
-		sh_css_params_shading_id_table_generate(target_table,
-			binary->sctbl_legacy_width_per_color, binary->sctbl_legacy_height);
-#endif
-		return;
-	}
-
-	padded_width = binary->in_frame_info.padded_width;
-	/* We use the ISP input resolution for the shading table because
-	   shading correction is performed in the bayer domain (before bayer
-	   down scaling). */
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-	padded_width = CEIL_MUL(binary->effective_in_frame_res.width + 2*ISP_VEC_NELEMS,
-					2*ISP_VEC_NELEMS);
-#endif
-	input_height  = binary->in_frame_info.res.height;
-	input_width   = binary->in_frame_info.res.width;
-	left_padding  = binary->left_padding;
-	left_cropping = (binary->info->sp.pipeline.left_cropping == 0) ?
-			binary->dvs_envelope.width : 2*ISP_VEC_NELEMS;
-
-	sh_css_bds_factor_get_numerator_denominator
-		(bds_factor, &bds_numerator, &bds_denominator);
-
-	left_padding  = (left_padding + binary->info->sp.pipeline.left_cropping) * bds_numerator / bds_denominator - binary->info->sp.pipeline.left_cropping;
-	right_padding = (binary->internal_frame_info.res.width - binary->effective_in_frame_res.width * bds_denominator / bds_numerator - left_cropping) * bds_numerator / bds_denominator;
-	top_padding = binary->info->sp.pipeline.top_cropping * bds_numerator / bds_denominator - binary->info->sp.pipeline.top_cropping;
-
-#if !defined(USE_WINDOWS_BINNING_FACTOR)
-	/* @deprecated{This part of the code will be replaced by the code
-	 * in the #else section below to make the calculation same across
-	 * all platforms.
-	 * Android and Windows platforms interpret the binning_factor parameter
-	 * differently. In Android, the binning factor is expressed in the form
-	 * 2^N * 2^N, whereas in Windows platform, the binning factor is N*N}
-	 */
-
-	/* We take into account the binning done by the sensor. We do this
-	   by cropping the non-binned part of the shading table and then
-	   increasing the size of a grid cell with this same binning factor. */
-	input_width  <<= sensor_binning;
-	input_height <<= sensor_binning;
-	/* We also scale the padding by the same binning factor. This will
-	   make it much easier later on to calculate the padding of the
-	   shading table. */
-	left_padding  <<= sensor_binning;
-	right_padding <<= sensor_binning;
-	top_padding   <<= sensor_binning;
-#else
-	input_width   *= sensor_binning;
-	input_height  *= sensor_binning;
-	left_padding  *= sensor_binning;
-	right_padding *= sensor_binning;
-	top_padding   *= sensor_binning;
-#endif /*USE_WINDOWS_BINNING_FACTOR*/
-
-	/* during simulation, the used resolution can exceed the sensor
-	   resolution, so we clip it. */
-	input_width  = min(input_width,  in_table->sensor_width);
-	input_height = min(input_height, in_table->sensor_height);
-
-#ifndef ISP2401
-	table_width  = binary->sctbl_width_per_color;
-	table_height = binary->sctbl_height;
-#else
-	/* This prepare_shading_table() function is called only in legacy API (not in new API).
-	   Then, the legacy shading table width and height should be used. */
-	table_width  = binary->sctbl_legacy_width_per_color;
-	table_height = binary->sctbl_legacy_height;
-#endif
-
-	result = ia_css_shading_table_alloc(table_width, table_height);
-	if (result == NULL) {
-		*target_table = NULL;
-		return;
-	}
-	result->sensor_width  = in_table->sensor_width;
-	result->sensor_height = in_table->sensor_height;
-	result->fraction_bits = in_table->fraction_bits;
-
-	/* now we crop the original shading table and then interpolate to the
-	   requested resolution and decimation factor. */
-	for (i = 0; i < IA_CSS_SC_NUM_COLORS; i++) {
-		crop_and_interpolate(input_width, input_height,
-				     left_padding, right_padding, top_padding,
-				     in_table,
-				     result, i);
-	}
-	*target_table = result;
-}
-
-struct ia_css_shading_table *
-ia_css_shading_table_alloc(
-	unsigned int width,
-	unsigned int height)
-{
-	unsigned int i;
-	struct ia_css_shading_table *me;
-
-	IA_CSS_ENTER("");
-
-	me = kmalloc(sizeof(*me), GFP_KERNEL);
-	if (!me)
-		return me;
-
-	me->width         = width;
-	me->height        = height;
-	me->sensor_width  = 0;
-	me->sensor_height = 0;
-	me->fraction_bits = 0;
-	for (i = 0; i < IA_CSS_SC_NUM_COLORS; i++) {
-		me->data[i] =
-		    sh_css_malloc(width * height * sizeof(*me->data[0]));
-		if (me->data[i] == NULL) {
-			unsigned int j;
-			for (j = 0; j < i; j++) {
-				sh_css_free(me->data[j]);
-				me->data[j] = NULL;
-			}
-			kfree(me);
-			return NULL;
-		}
-	}
-
-	IA_CSS_LEAVE("");
-	return me;
-}
-
-void
-ia_css_shading_table_free(struct ia_css_shading_table *table)
-{
-	unsigned int i;
-
-	if (table == NULL)
-		return;
-
-	/* We only output logging when the table is not NULL, otherwise
-	 * logs will give the impression that a table was freed.
-	 * */
-	IA_CSS_ENTER("");
-
-	for (i = 0; i < IA_CSS_SC_NUM_COLORS; i++) {
-		if (table->data[i]) {
-			sh_css_free(table->data[i]);
-			table->data[i] = NULL;
-		}
-	}
-	kfree(table);
-
-	IA_CSS_LEAVE("");
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_shading.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_shading.h
deleted file mode 100644
index e87863b7c8cc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_param_shading.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SH_CSS_PARAMS_SHADING_H
-#define __SH_CSS_PARAMS_SHADING_H
-
-#include <ia_css_types.h>
-#include <ia_css_binary.h>
-
-void
-sh_css_params_shading_id_table_generate(
-	struct ia_css_shading_table **target_table,
-#ifndef ISP2401
-	const struct ia_css_binary *binary);
-#else
-	unsigned int table_width,
-	unsigned int table_height);
-#endif
-
-void
-prepare_shading_table(const struct ia_css_shading_table *in_table,
-		      unsigned int sensor_binning,
-		      struct ia_css_shading_table **target_table,
-		      const struct ia_css_binary *binary,
-		      unsigned int bds_factor);
-
-#endif /* __SH_CSS_PARAMS_SHADING_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params.c
deleted file mode 100644
index 43529b1605c3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params.c
+++ /dev/null
@@ -1,5253 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "gdc_device.h"		/* gdc_lut_store(), ... */
-#include "isp.h"			/* ISP_VEC_ELEMBITS */
-#include "vamem.h"
-#if !defined(HAS_NO_HMEM)
-#ifndef __INLINE_HMEM__
-#define __INLINE_HMEM__
-#endif
-#include "hmem.h"
-#endif /* !defined(HAS_NO_HMEM) */
-#define IA_CSS_INCLUDE_PARAMETERS
-#define IA_CSS_INCLUDE_ACC_PARAMETERS
-
-#include "sh_css_params.h"
-#include "ia_css_queue.h"
-#include "sw_event_global.h"		/* Event IDs */
-
-#include "platform_support.h"
-#include "assert_support.h"
-#include "misc_support.h"	/* NOT_USED */
-#include "math_support.h"	/* max(), min()  EVEN_FLOOR()*/
-
-#include "ia_css_stream.h"
-#include "sh_css_params_internal.h"
-#include "sh_css_param_shading.h"
-#include "sh_css_param_dvs.h"
-#include "ia_css_refcount.h"
-#include "sh_css_internal.h"
-#include "ia_css_control.h"
-#include "ia_css_shading.h"
-#include "sh_css_defs.h"
-#include "sh_css_sp.h"
-#include "ia_css_pipeline.h"
-#include "ia_css_debug.h"
-#include "memory_access.h"
-#if 0   /* FIXME */
-#include "memory_realloc.h"
-#endif
-#include "ia_css_isp_param.h"
-#include "ia_css_isp_params.h"
-#include "ia_css_mipi.h"
-#include "ia_css_morph.h"
-#include "ia_css_host_data.h"
-#include "ia_css_pipe.h"
-#include "ia_css_pipe_binarydesc.h"
-#if 0
-#include "ia_css_system_ctrl.h"
-#endif
-
-/* Include all kernel host interfaces for ISP1 */
-
-#include "anr/anr_1.0/ia_css_anr.host.h"
-#include "cnr/cnr_1.0/ia_css_cnr.host.h"
-#include "csc/csc_1.0/ia_css_csc.host.h"
-#include "de/de_1.0/ia_css_de.host.h"
-#include "dp/dp_1.0/ia_css_dp.host.h"
-#include "bnr/bnr_1.0/ia_css_bnr.host.h"
-#include "dvs/dvs_1.0/ia_css_dvs.host.h"
-#include "fpn/fpn_1.0/ia_css_fpn.host.h"
-#include "gc/gc_1.0/ia_css_gc.host.h"
-#include "macc/macc_1.0/ia_css_macc.host.h"
-#include "ctc/ctc_1.0/ia_css_ctc.host.h"
-#include "ob/ob_1.0/ia_css_ob.host.h"
-#include "raw/raw_1.0/ia_css_raw.host.h"
-#include "fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h"
-#include "s3a/s3a_1.0/ia_css_s3a.host.h"
-#include "sc/sc_1.0/ia_css_sc.host.h"
-#include "sdis/sdis_1.0/ia_css_sdis.host.h"
-#include "tnr/tnr_1.0/ia_css_tnr.host.h"
-#include "uds/uds_1.0/ia_css_uds_param.h"
-#include "wb/wb_1.0/ia_css_wb.host.h"
-#include "ynr/ynr_1.0/ia_css_ynr.host.h"
-#include "xnr/xnr_1.0/ia_css_xnr.host.h"
-
-/* Include additional kernel host interfaces for ISP2 */
-
-#include "aa/aa_2/ia_css_aa2.host.h"
-#include "anr/anr_2/ia_css_anr2.host.h"
-#include "bh/bh_2/ia_css_bh.host.h"
-#include "cnr/cnr_2/ia_css_cnr2.host.h"
-#include "ctc/ctc1_5/ia_css_ctc1_5.host.h"
-#include "de/de_2/ia_css_de2.host.h"
-#include "gc/gc_2/ia_css_gc2.host.h"
-#include "sdis/sdis_2/ia_css_sdis2.host.h"
-#include "ynr/ynr_2/ia_css_ynr2.host.h"
-#include "fc/fc_1.0/ia_css_formats.host.h"
-
-#include "xnr/xnr_3.0/ia_css_xnr3.host.h"
-
-#if defined(HAS_OUTPUT_SYSTEM)
-#include <components/output_system/sc_output_system_1.0/host/output_system.host.h>
-#endif
-
-#include "sh_css_frac.h"
-#include "ia_css_bufq.h"
-
-#define FPNTBL_BYTES(binary) \
-	(sizeof(char) * (binary)->in_frame_info.res.height * \
-	 (binary)->in_frame_info.padded_width)
-
-#ifndef ISP2401
-
-#define SCTBL_BYTES(binary) \
-	(sizeof(unsigned short) * (binary)->sctbl_height * \
-	 (binary)->sctbl_aligned_width_per_color * IA_CSS_SC_NUM_COLORS)
-
-#else
-
-#define SCTBL_BYTES(binary) \
-	(sizeof(unsigned short) * max((binary)->sctbl_height, (binary)->sctbl_legacy_height) * \
-			/* height should be the larger height between new api and legacy api */ \
-	 (binary)->sctbl_aligned_width_per_color * IA_CSS_SC_NUM_COLORS)
-
-#endif
-
-#define MORPH_PLANE_BYTES(binary) \
-	(SH_CSS_MORPH_TABLE_ELEM_BYTES * (binary)->morph_tbl_aligned_width * \
-	 (binary)->morph_tbl_height)
-
-/* We keep a second copy of the ptr struct for the SP to access.
-   Again, this would not be necessary on the chip. */
-static hrt_vaddress sp_ddr_ptrs;
-
-/* sp group address on DDR */
-static hrt_vaddress xmem_sp_group_ptrs;
-
-static hrt_vaddress xmem_sp_stage_ptrs[IA_CSS_PIPE_ID_NUM]
-						[SH_CSS_MAX_STAGES];
-static hrt_vaddress xmem_isp_stage_ptrs[IA_CSS_PIPE_ID_NUM]
-						[SH_CSS_MAX_STAGES];
-
-static hrt_vaddress default_gdc_lut;
-static int interleaved_lut_temp[4][HRT_GDC_N];
-
-/* END DO NOT MOVE INTO VIMALS_WORLD */
-
-/* Digital Zoom lookup table. See documentation for more details about the
- * contents of this table.
- */
-#if defined(HAS_GDC_VERSION_2)
-#if defined(CONFIG_CSI2_PLUS)
-/*
- * Coefficients from
- * Css_Mizuchi/regressions/20140424_0930/all/applications/common/gdc_v2_common/lut.h
- */
-
-static const int zoom_table[4][HRT_GDC_N] = {
-	{	   0,    0,    0,    0,    0,    0,    0,    0,
-		   0,    0,    0,    0,    0,    0,    0,    0,
-		   0,    0,    0,    0,    0,    0,    0,   -1,
-		  -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
-		  -1,   -2,   -2,   -2,   -2,   -2,   -2,   -2,
-		  -3,   -3,   -3,   -3,   -3,   -3,   -3,   -4,
-		  -4,   -4,   -4,   -4,   -5,   -5,   -5,   -5,
-		  -5,   -5,   -6,   -6,   -6,   -6,   -7,   -7,
-		  -7,   -7,   -7,   -8,   -8,   -8,   -8,   -9,
-		  -9,   -9,   -9,  -10,  -10,  -10,  -10,  -11,
-		 -11,  -11,  -12,  -12,  -12,  -12,  -13,  -13,
-		 -13,  -14,  -14,  -14,  -15,  -15,  -15,  -15,
-		 -16,  -16,  -16,  -17,  -17,  -17,  -18,  -18,
-		 -18,  -19,  -19,  -20,  -20,  -20,  -21,  -21,
-		 -21,  -22,  -22,  -22,  -23,  -23,  -24,  -24,
-		 -24,  -25,  -25,  -25,  -26,  -26,  -27,  -27,
-		 -28,  -28,  -28,  -29,  -29,  -30,  -30,  -30,
-		 -31,  -31,  -32,  -32,  -33,  -33,  -33,  -34,
-		 -34,  -35,  -35,  -36,  -36,  -37,  -37,  -37,
-		 -38,  -38,  -39,  -39,  -40,  -40,  -41,  -41,
-		 -42,  -42,  -43,  -43,  -44,  -44,  -45,  -45,
-		 -46,  -46,  -47,  -47,  -48,  -48,  -49,  -49,
-		 -50,  -50,  -51,  -51,  -52,  -52,  -53,  -53,
-		 -54,  -54,  -55,  -55,  -56,  -56,  -57,  -57,
-		 -58,  -59,  -59,  -60,  -60,  -61,  -61,  -62,
-		 -62,  -63,  -63,  -64,  -65,  -65,  -66,  -66,
-		 -67,  -67,  -68,  -69,  -69,  -70,  -70,  -71,
-		 -71,  -72,  -73,  -73,  -74,  -74,  -75,  -75,
-		 -76,  -77,  -77,  -78,  -78,  -79,  -80,  -80,
-		 -81,  -81,  -82,  -83,  -83,  -84,  -84,  -85,
-		 -86,  -86,  -87,  -87,  -88,  -89,  -89,  -90,
-		 -91,  -91,  -92,  -92,  -93,  -94,  -94,  -95,
-		 -96,  -96,  -97,  -97,  -98,  -99,  -99, -100,
-		-101, -101, -102, -102, -103, -104, -104, -105,
-		-106, -106, -107, -108, -108, -109, -109, -110,
-		-111, -111, -112, -113, -113, -114, -115, -115,
-		-116, -117, -117, -118, -119, -119, -120, -121,
-		-121, -122, -122, -123, -124, -124, -125, -126,
-		-126, -127, -128, -128, -129, -130, -130, -131,
-		-132, -132, -133, -134, -134, -135, -136, -136,
-		-137, -138, -138, -139, -140, -140, -141, -142,
-		-142, -143, -144, -144, -145, -146, -146, -147,
-		-148, -148, -149, -150, -150, -151, -152, -152,
-		-153, -154, -154, -155, -156, -156, -157, -158,
-		-158, -159, -160, -160, -161, -162, -162, -163,
-		-164, -164, -165, -166, -166, -167, -168, -168,
-		-169, -170, -170, -171, -172, -172, -173, -174,
-		-174, -175, -176, -176, -177, -178, -178, -179,
-		-180, -180, -181, -181, -182, -183, -183, -184,
-		-185, -185, -186, -187, -187, -188, -189, -189,
-		-190, -191, -191, -192, -193, -193, -194, -194,
-		-195, -196, -196, -197, -198, -198, -199, -200,
-		-200, -201, -201, -202, -203, -203, -204, -205,
-		-205, -206, -206, -207, -208, -208, -209, -210,
-		-210, -211, -211, -212, -213, -213, -214, -215,
-		-215, -216, -216, -217, -218, -218, -219, -219,
-		-220, -221, -221, -222, -222, -223, -224, -224,
-		-225, -225, -226, -227, -227, -228, -228, -229,
-		-229, -230, -231, -231, -232, -232, -233, -233,
-		-234, -235, -235, -236, -236, -237, -237, -238,
-		-239, -239, -240, -240, -241, -241, -242, -242,
-		-243, -244, -244, -245, -245, -246, -246, -247,
-		-247, -248, -248, -249, -249, -250, -250, -251,
-		-251, -252, -252, -253, -253, -254, -254, -255,
-		-256, -256, -256, -257, -257, -258, -258, -259,
-		-259, -260, -260, -261, -261, -262, -262, -263,
-		-263, -264, -264, -265, -265, -266, -266, -266,
-		-267, -267, -268, -268, -269, -269, -270, -270,
-		-270, -271, -271, -272, -272, -273, -273, -273,
-		-274, -274, -275, -275, -275, -276, -276, -277,
-		-277, -277, -278, -278, -279, -279, -279, -280,
-		-280, -280, -281, -281, -282, -282, -282, -283,
-		-283, -283, -284, -284, -284, -285, -285, -285,
-		-286, -286, -286, -287, -287, -287, -288, -288,
-		-288, -289, -289, -289, -289, -290, -290, -290,
-		-291, -291, -291, -291, -292, -292, -292, -293,
-		-293, -293, -293, -294, -294, -294, -294, -295,
-		-295, -295, -295, -295, -296, -296, -296, -296,
-		-297, -297, -297, -297, -297, -298, -298, -298,
-		-298, -298, -299, -299, -299, -299, -299, -299,
-		-300, -300, -300, -300, -300, -300, -300, -301,
-		-301, -301, -301, -301, -301, -301, -301, -301,
-		-302, -302, -302, -302, -302, -302, -302, -302,
-		-302, -302, -302, -302, -302, -303, -303, -303,
-		-303, -303, -303, -303, -303, -303, -303, -303,
-		-303, -303, -303, -303, -303, -303, -303, -303,
-		-303, -303, -303, -303, -303, -303, -303, -303,
-		-303, -303, -302, -302, -302, -302, -302, -302,
-		-302, -302, -302, -302, -302, -302, -301, -301,
-		-301, -301, -301, -301, -301, -301, -300, -300,
-		-300, -300, -300, -300, -299, -299, -299, -299,
-		-299, -299, -298, -298, -298, -298, -298, -297,
-		-297, -297, -297, -296, -296, -296, -296, -295,
-		-295, -295, -295, -294, -294, -294, -293, -293,
-		-293, -293, -292, -292, -292, -291, -291, -291,
-		-290, -290, -290, -289, -289, -289, -288, -288,
-		-288, -287, -287, -286, -286, -286, -285, -285,
-		-284, -284, -284, -283, -283, -282, -282, -281,
-		-281, -280, -280, -279, -279, -279, -278, -278,
-		-277, -277, -276, -276, -275, -275, -274, -273,
-		-273, -272, -272, -271, -271, -270, -270, -269,
-		-268, -268, -267, -267, -266, -266, -265, -264,
-		-264, -263, -262, -262, -261, -260, -260, -259,
-		-259, -258, -257, -256, -256, -255, -254, -254,
-		-253, -252, -252, -251, -250, -249, -249, -248,
-		-247, -246, -246, -245, -244, -243, -242, -242,
-		-241, -240, -239, -238, -238, -237, -236, -235,
-		-234, -233, -233, -232, -231, -230, -229, -228,
-		-227, -226, -226, -225, -224, -223, -222, -221,
-		-220, -219, -218, -217, -216, -215, -214, -213,
-		-212, -211, -210, -209, -208, -207, -206, -205,
-		-204, -203, -202, -201, -200, -199, -198, -197,
-		-196, -194, -193, -192, -191, -190, -189, -188,
-		-187, -185, -184, -183, -182, -181, -180, -178,
-		-177, -176, -175, -174, -172, -171, -170, -169,
-		-167, -166, -165, -164, -162, -161, -160, -158,
-		-157, -156, -155, -153, -152, -151, -149, -148,
-		-147, -145, -144, -142, -141, -140, -138, -137,
-		-135, -134, -133, -131, -130, -128, -127, -125,
-		-124, -122, -121, -120, -118, -117, -115, -114,
-		-112, -110, -109, -107, -106, -104, -103, -101,
-		-100,  -98,  -96,  -95,  -93,  -92,  -90,  -88,
-		 -87,  -85,  -83,  -82,  -80,  -78,  -77,  -75,
-		 -73,  -72,  -70,  -68,  -67,  -65,  -63,  -61,
-		 -60,  -58,  -56,  -54,  -52,  -51,  -49,  -47,
-		 -45,  -43,  -42,  -40,  -38,  -36,  -34,  -32,
-		 -31,  -29,  -27,  -25,  -23,  -21,  -19,  -17,
-		 -15,  -13,  -11,   -9,   -7,   -5,   -3,   -1
-	},
-	{	   0,    2,    4,    6,    8,   10,   12,   14,
-		  16,   18,   20,   22,   25,   27,   29,   31,
-		  33,   36,   38,   40,   43,   45,   47,   50,
-		  52,   54,   57,   59,   61,   64,   66,   69,
-		  71,   74,   76,   79,   81,   84,   86,   89,
-		  92,   94,   97,   99,  102,  105,  107,  110,
-		 113,  116,  118,  121,  124,  127,  129,  132,
-		 135,  138,  141,  144,  146,  149,  152,  155,
-		 158,  161,  164,  167,  170,  173,  176,  179,
-		 182,  185,  188,  191,  194,  197,  200,  203,
-		 207,  210,  213,  216,  219,  222,  226,  229,
-		 232,  235,  239,  242,  245,  248,  252,  255,
-		 258,  262,  265,  269,  272,  275,  279,  282,
-		 286,  289,  292,  296,  299,  303,  306,  310,
-		 313,  317,  321,  324,  328,  331,  335,  338,
-		 342,  346,  349,  353,  357,  360,  364,  368,
-		 372,  375,  379,  383,  386,  390,  394,  398,
-		 402,  405,  409,  413,  417,  421,  425,  429,
-		 432,  436,  440,  444,  448,  452,  456,  460,
-		 464,  468,  472,  476,  480,  484,  488,  492,
-		 496,  500,  504,  508,  512,  516,  521,  525,
-		 529,  533,  537,  541,  546,  550,  554,  558,
-		 562,  567,  571,  575,  579,  584,  588,  592,
-		 596,  601,  605,  609,  614,  618,  622,  627,
-		 631,  635,  640,  644,  649,  653,  657,  662,
-		 666,  671,  675,  680,  684,  689,  693,  698,
-		 702,  707,  711,  716,  720,  725,  729,  734,
-		 738,  743,  747,  752,  757,  761,  766,  771,
-		 775,  780,  784,  789,  794,  798,  803,  808,
-		 813,  817,  822,  827,  831,  836,  841,  846,
-		 850,  855,  860,  865,  870,  874,  879,  884,
-		 889,  894,  898,  903,  908,  913,  918,  923,
-		 928,  932,  937,  942,  947,  952,  957,  962,
-		 967,  972,  977,  982,  986,  991,  996, 1001,
-		1006, 1011, 1016, 1021, 1026, 1031, 1036, 1041,
-		1046, 1051, 1056, 1062, 1067, 1072, 1077, 1082,
-		1087, 1092, 1097, 1102, 1107, 1112, 1117, 1122,
-		1128, 1133, 1138, 1143, 1148, 1153, 1158, 1164,
-		1169, 1174, 1179, 1184, 1189, 1195, 1200, 1205,
-		1210, 1215, 1221, 1226, 1231, 1236, 1242, 1247,
-		1252, 1257, 1262, 1268, 1273, 1278, 1284, 1289,
-		1294, 1299, 1305, 1310, 1315, 1321, 1326, 1331,
-		1336, 1342, 1347, 1352, 1358, 1363, 1368, 1374,
-		1379, 1384, 1390, 1395, 1400, 1406, 1411, 1417,
-		1422, 1427, 1433, 1438, 1443, 1449, 1454, 1460,
-		1465, 1470, 1476, 1481, 1487, 1492, 1497, 1503,
-		1508, 1514, 1519, 1525, 1530, 1535, 1541, 1546,
-		1552, 1557, 1563, 1568, 1574, 1579, 1585, 1590,
-		1596, 1601, 1606, 1612, 1617, 1623, 1628, 1634,
-		1639, 1645, 1650, 1656, 1661, 1667, 1672, 1678,
-		1683, 1689, 1694, 1700, 1705, 1711, 1716, 1722,
-		1727, 1733, 1738, 1744, 1749, 1755, 1761, 1766,
-		1772, 1777, 1783, 1788, 1794, 1799, 1805, 1810,
-		1816, 1821, 1827, 1832, 1838, 1844, 1849, 1855,
-		1860, 1866, 1871, 1877, 1882, 1888, 1893, 1899,
-		1905, 1910, 1916, 1921, 1927, 1932, 1938, 1943,
-		1949, 1955, 1960, 1966, 1971, 1977, 1982, 1988,
-		1993, 1999, 2005, 2010, 2016, 2021, 2027, 2032,
-		2038, 2043, 2049, 2055, 2060, 2066, 2071, 2077,
-		2082, 2088, 2093, 2099, 2105, 2110, 2116, 2121,
-		2127, 2132, 2138, 2143, 2149, 2154, 2160, 2165,
-		2171, 2177, 2182, 2188, 2193, 2199, 2204, 2210,
-		2215, 2221, 2226, 2232, 2237, 2243, 2248, 2254,
-		2259, 2265, 2270, 2276, 2281, 2287, 2292, 2298,
-		2304, 2309, 2314, 2320, 2325, 2331, 2336, 2342,
-		2347, 2353, 2358, 2364, 2369, 2375, 2380, 2386,
-		2391, 2397, 2402, 2408, 2413, 2419, 2424, 2429,
-		2435, 2440, 2446, 2451, 2457, 2462, 2467, 2473,
-		2478, 2484, 2489, 2495, 2500, 2505, 2511, 2516,
-		2522, 2527, 2532, 2538, 2543, 2549, 2554, 2559,
-		2565, 2570, 2575, 2581, 2586, 2591, 2597, 2602,
-		2607, 2613, 2618, 2623, 2629, 2634, 2639, 2645,
-		2650, 2655, 2661, 2666, 2671, 2676, 2682, 2687,
-		2692, 2698, 2703, 2708, 2713, 2719, 2724, 2729,
-		2734, 2740, 2745, 2750, 2755, 2760, 2766, 2771,
-		2776, 2781, 2786, 2792, 2797, 2802, 2807, 2812,
-		2817, 2823, 2828, 2833, 2838, 2843, 2848, 2853,
-		2859, 2864, 2869, 2874, 2879, 2884, 2889, 2894,
-		2899, 2904, 2909, 2914, 2919, 2924, 2930, 2935,
-		2940, 2945, 2950, 2955, 2960, 2965, 2970, 2975,
-		2980, 2984, 2989, 2994, 2999, 3004, 3009, 3014,
-		3019, 3024, 3029, 3034, 3039, 3044, 3048, 3053,
-		3058, 3063, 3068, 3073, 3078, 3082, 3087, 3092,
-		3097, 3102, 3106, 3111, 3116, 3121, 3126, 3130,
-		3135, 3140, 3145, 3149, 3154, 3159, 3163, 3168,
-		3173, 3177, 3182, 3187, 3191, 3196, 3201, 3205,
-		3210, 3215, 3219, 3224, 3228, 3233, 3238, 3242,
-		3247, 3251, 3256, 3260, 3265, 3269, 3274, 3279,
-		3283, 3287, 3292, 3296, 3301, 3305, 3310, 3314,
-		3319, 3323, 3327, 3332, 3336, 3341, 3345, 3349,
-		3354, 3358, 3362, 3367, 3371, 3375, 3380, 3384,
-		3388, 3393, 3397, 3401, 3405, 3410, 3414, 3418,
-		3422, 3426, 3431, 3435, 3439, 3443, 3447, 3451,
-		3455, 3460, 3464, 3468, 3472, 3476, 3480, 3484,
-		3488, 3492, 3496, 3500, 3504, 3508, 3512, 3516,
-		3520, 3524, 3528, 3532, 3536, 3540, 3544, 3548,
-		3552, 3555, 3559, 3563, 3567, 3571, 3575, 3578,
-		3582, 3586, 3590, 3593, 3597, 3601, 3605, 3608,
-		3612, 3616, 3619, 3623, 3627, 3630, 3634, 3638,
-		3641, 3645, 3649, 3652, 3656, 3659, 3663, 3666,
-		3670, 3673, 3677, 3680, 3684, 3687, 3691, 3694,
-		3698, 3701, 3704, 3708, 3711, 3714, 3718, 3721,
-		3724, 3728, 3731, 3734, 3738, 3741, 3744, 3747,
-		3751, 3754, 3757, 3760, 3763, 3767, 3770, 3773,
-		3776, 3779, 3782, 3785, 3788, 3791, 3794, 3798,
-		3801, 3804, 3807, 3809, 3812, 3815, 3818, 3821,
-		3824, 3827, 3830, 3833, 3836, 3839, 3841, 3844,
-		3847, 3850, 3853, 3855, 3858, 3861, 3864, 3866,
-		3869, 3872, 3874, 3877, 3880, 3882, 3885, 3887,
-		3890, 3893, 3895, 3898, 3900, 3903, 3905, 3908,
-		3910, 3913, 3915, 3917, 3920, 3922, 3925, 3927,
-		3929, 3932, 3934, 3936, 3939, 3941, 3943, 3945,
-		3948, 3950, 3952, 3954, 3956, 3958, 3961, 3963,
-		3965, 3967, 3969, 3971, 3973, 3975, 3977, 3979,
-		3981, 3983, 3985, 3987, 3989, 3991, 3993, 3994,
-		3996, 3998, 4000, 4002, 4004, 4005, 4007, 4009,
-		4011, 4012, 4014, 4016, 4017, 4019, 4021, 4022,
-		4024, 4025, 4027, 4028, 4030, 4031, 4033, 4034,
-		4036, 4037, 4039, 4040, 4042, 4043, 4044, 4046,
-		4047, 4048, 4050, 4051, 4052, 4053, 4055, 4056,
-		4057, 4058, 4059, 4060, 4062, 4063, 4064, 4065,
-		4066, 4067, 4068, 4069, 4070, 4071, 4072, 4073,
-		4074, 4075, 4075, 4076, 4077, 4078, 4079, 4079,
-		4080, 4081, 4082, 4082, 4083, 4084, 4084, 4085,
-		4086, 4086, 4087, 4087, 4088, 4088, 4089, 4089,
-		4090, 4090, 4091, 4091, 4092, 4092, 4092, 4093,
-		4093, 4093, 4094, 4094, 4094, 4094, 4095, 4095,
-		4095, 4095, 4095, 4095, 4095, 4095, 4095, 4095
-	},
-	{	4096, 4095, 4095, 4095, 4095, 4095, 4095, 4095,
-		4095, 4095, 4095, 4094, 4094, 4094, 4094, 4093,
-		4093, 4093, 4092, 4092, 4092, 4091, 4091, 4090,
-		4090, 4089, 4089, 4088, 4088, 4087, 4087, 4086,
-		4086, 4085, 4084, 4084, 4083, 4082, 4082, 4081,
-		4080, 4079, 4079, 4078, 4077, 4076, 4075, 4075,
-		4074, 4073, 4072, 4071, 4070, 4069, 4068, 4067,
-		4066, 4065, 4064, 4063, 4062, 4060, 4059, 4058,
-		4057, 4056, 4055, 4053, 4052, 4051, 4050, 4048,
-		4047, 4046, 4044, 4043, 4042, 4040, 4039, 4037,
-		4036, 4034, 4033, 4031, 4030, 4028, 4027, 4025,
-		4024, 4022, 4021, 4019, 4017, 4016, 4014, 4012,
-		4011, 4009, 4007, 4005, 4004, 4002, 4000, 3998,
-		3996, 3994, 3993, 3991, 3989, 3987, 3985, 3983,
-		3981, 3979, 3977, 3975, 3973, 3971, 3969, 3967,
-		3965, 3963, 3961, 3958, 3956, 3954, 3952, 3950,
-		3948, 3945, 3943, 3941, 3939, 3936, 3934, 3932,
-		3929, 3927, 3925, 3922, 3920, 3917, 3915, 3913,
-		3910, 3908, 3905, 3903, 3900, 3898, 3895, 3893,
-		3890, 3887, 3885, 3882, 3880, 3877, 3874, 3872,
-		3869, 3866, 3864, 3861, 3858, 3855, 3853, 3850,
-		3847, 3844, 3841, 3839, 3836, 3833, 3830, 3827,
-		3824, 3821, 3818, 3815, 3812, 3809, 3807, 3804,
-		3801, 3798, 3794, 3791, 3788, 3785, 3782, 3779,
-		3776, 3773, 3770, 3767, 3763, 3760, 3757, 3754,
-		3751, 3747, 3744, 3741, 3738, 3734, 3731, 3728,
-		3724, 3721, 3718, 3714, 3711, 3708, 3704, 3701,
-		3698, 3694, 3691, 3687, 3684, 3680, 3677, 3673,
-		3670, 3666, 3663, 3659, 3656, 3652, 3649, 3645,
-		3641, 3638, 3634, 3630, 3627, 3623, 3619, 3616,
-		3612, 3608, 3605, 3601, 3597, 3593, 3590, 3586,
-		3582, 3578, 3575, 3571, 3567, 3563, 3559, 3555,
-		3552, 3548, 3544, 3540, 3536, 3532, 3528, 3524,
-		3520, 3516, 3512, 3508, 3504, 3500, 3496, 3492,
-		3488, 3484, 3480, 3476, 3472, 3468, 3464, 3460,
-		3455, 3451, 3447, 3443, 3439, 3435, 3431, 3426,
-		3422, 3418, 3414, 3410, 3405, 3401, 3397, 3393,
-		3388, 3384, 3380, 3375, 3371, 3367, 3362, 3358,
-		3354, 3349, 3345, 3341, 3336, 3332, 3327, 3323,
-		3319, 3314, 3310, 3305, 3301, 3296, 3292, 3287,
-		3283, 3279, 3274, 3269, 3265, 3260, 3256, 3251,
-		3247, 3242, 3238, 3233, 3228, 3224, 3219, 3215,
-		3210, 3205, 3201, 3196, 3191, 3187, 3182, 3177,
-		3173, 3168, 3163, 3159, 3154, 3149, 3145, 3140,
-		3135, 3130, 3126, 3121, 3116, 3111, 3106, 3102,
-		3097, 3092, 3087, 3082, 3078, 3073, 3068, 3063,
-		3058, 3053, 3048, 3044, 3039, 3034, 3029, 3024,
-		3019, 3014, 3009, 3004, 2999, 2994, 2989, 2984,
-		2980, 2975, 2970, 2965, 2960, 2955, 2950, 2945,
-		2940, 2935, 2930, 2924, 2919, 2914, 2909, 2904,
-		2899, 2894, 2889, 2884, 2879, 2874, 2869, 2864,
-		2859, 2853, 2848, 2843, 2838, 2833, 2828, 2823,
-		2817, 2812, 2807, 2802, 2797, 2792, 2786, 2781,
-		2776, 2771, 2766, 2760, 2755, 2750, 2745, 2740,
-		2734, 2729, 2724, 2719, 2713, 2708, 2703, 2698,
-		2692, 2687, 2682, 2676, 2671, 2666, 2661, 2655,
-		2650, 2645, 2639, 2634, 2629, 2623, 2618, 2613,
-		2607, 2602, 2597, 2591, 2586, 2581, 2575, 2570,
-		2565, 2559, 2554, 2549, 2543, 2538, 2532, 2527,
-		2522, 2516, 2511, 2505, 2500, 2495, 2489, 2484,
-		2478, 2473, 2467, 2462, 2457, 2451, 2446, 2440,
-		2435, 2429, 2424, 2419, 2413, 2408, 2402, 2397,
-		2391, 2386, 2380, 2375, 2369, 2364, 2358, 2353,
-		2347, 2342, 2336, 2331, 2325, 2320, 2314, 2309,
-		2304, 2298, 2292, 2287, 2281, 2276, 2270, 2265,
-		2259, 2254, 2248, 2243, 2237, 2232, 2226, 2221,
-		2215, 2210, 2204, 2199, 2193, 2188, 2182, 2177,
-		2171, 2165, 2160, 2154, 2149, 2143, 2138, 2132,
-		2127, 2121, 2116, 2110, 2105, 2099, 2093, 2088,
-		2082, 2077, 2071, 2066, 2060, 2055, 2049, 2043,
-		2038, 2032, 2027, 2021, 2016, 2010, 2005, 1999,
-		1993, 1988, 1982, 1977, 1971, 1966, 1960, 1955,
-		1949, 1943, 1938, 1932, 1927, 1921, 1916, 1910,
-		1905, 1899, 1893, 1888, 1882, 1877, 1871, 1866,
-		1860, 1855, 1849, 1844, 1838, 1832, 1827, 1821,
-		1816, 1810, 1805, 1799, 1794, 1788, 1783, 1777,
-		1772, 1766, 1761, 1755, 1749, 1744, 1738, 1733,
-		1727, 1722, 1716, 1711, 1705, 1700, 1694, 1689,
-		1683, 1678, 1672, 1667, 1661, 1656, 1650, 1645,
-		1639, 1634, 1628, 1623, 1617, 1612, 1606, 1601,
-		1596, 1590, 1585, 1579, 1574, 1568, 1563, 1557,
-		1552, 1546, 1541, 1535, 1530, 1525, 1519, 1514,
-		1508, 1503, 1497, 1492, 1487, 1481, 1476, 1470,
-		1465, 1460, 1454, 1449, 1443, 1438, 1433, 1427,
-		1422, 1417, 1411, 1406, 1400, 1395, 1390, 1384,
-		1379, 1374, 1368, 1363, 1358, 1352, 1347, 1342,
-		1336, 1331, 1326, 1321, 1315, 1310, 1305, 1299,
-		1294, 1289, 1284, 1278, 1273, 1268, 1262, 1257,
-		1252, 1247, 1242, 1236, 1231, 1226, 1221, 1215,
-		1210, 1205, 1200, 1195, 1189, 1184, 1179, 1174,
-		1169, 1164, 1158, 1153, 1148, 1143, 1138, 1133,
-		1128, 1122, 1117, 1112, 1107, 1102, 1097, 1092,
-		1087, 1082, 1077, 1072, 1067, 1062, 1056, 1051,
-		1046, 1041, 1036, 1031, 1026, 1021, 1016, 1011,
-		1006, 1001,  996,  991,  986,  982,  977,  972,
-		 967,  962,  957,  952,  947,  942,  937,  932,
-		 928,  923,  918,  913,  908,  903,  898,  894,
-		 889,  884,  879,  874,  870,  865,  860,  855,
-		 850,  846,  841,  836,  831,  827,  822,  817,
-		 813,  808,  803,  798,  794,  789,  784,  780,
-		 775,  771,  766,  761,  757,  752,  747,  743,
-		 738,  734,  729,  725,  720,  716,  711,  707,
-		 702,  698,  693,  689,  684,  680,  675,  671,
-		 666,  662,  657,  653,  649,  644,  640,  635,
-		 631,  627,  622,  618,  614,  609,  605,  601,
-		 596,  592,  588,  584,  579,  575,  571,  567,
-		 562,  558,  554,  550,  546,  541,  537,  533,
-		 529,  525,  521,  516,  512,  508,  504,  500,
-		 496,  492,  488,  484,  480,  476,  472,  468,
-		 464,  460,  456,  452,  448,  444,  440,  436,
-		 432,  429,  425,  421,  417,  413,  409,  405,
-		 402,  398,  394,  390,  386,  383,  379,  375,
-		 372,  368,  364,  360,  357,  353,  349,  346,
-		 342,  338,  335,  331,  328,  324,  321,  317,
-		 313,  310,  306,  303,  299,  296,  292,  289,
-		 286,  282,  279,  275,  272,  269,  265,  262,
-		 258,  255,  252,  248,  245,  242,  239,  235,
-		 232,  229,  226,  222,  219,  216,  213,  210,
-		 207,  203,  200,  197,  194,  191,  188,  185,
-		 182,  179,  176,  173,  170,  167,  164,  161,
-		 158,  155,  152,  149,  146,  144,  141,  138,
-		 135,  132,  129,  127,  124,  121,  118,  116,
-		 113,  110,  107,  105,  102,   99,   97,   94,
-		  92,   89,   86,   84,   81,   79,   76,   74,
-		  71,   69,   66,   64,   61,   59,   57,   54,
-		  52,   50,   47,   45,   43,   40,   38,   36,
-		  33,   31,   29,   27,   25,   22,   20,   18,
-		  16,   14,   12,   10,    8,    6,    4,    2
-	},
-	{	   0,   -1,   -3,   -5,   -7,   -9,  -11,  -13,
-		 -15,  -17,  -19,  -20,  -23,  -25,  -27,  -28,
-		 -30,  -33,  -34,  -36,  -39,  -40,  -42,  -43,
-		 -45,  -46,  -49,  -50,  -52,  -54,  -56,  -58,
-		 -60,  -61,  -62,  -65,  -66,  -68,  -70,  -72,
-		 -73,  -74,  -77,  -78,  -80,  -82,  -83,  -85,
-		 -87,  -89,  -90,  -92,  -93,  -95,  -96,  -98,
-		-100, -102, -103, -105, -106, -107, -108, -110,
-		-112, -114, -116, -116, -118, -120, -122, -122,
-		-124, -126, -127, -128, -130, -131, -133, -133,
-		-136, -137, -138, -139, -141, -142, -144, -145,
-		-147, -147, -150, -151, -151, -153, -155, -156,
-		-157, -159, -160, -161, -163, -164, -165, -166,
-		-168, -168, -170, -171, -172, -174, -174, -176,
-		-177, -178, -180, -181, -182, -183, -184, -185,
-		-187, -188, -189, -190, -191, -192, -193, -195,
-		-196, -196, -198, -199, -200, -200, -202, -204,
-		-204, -205, -206, -207, -208, -209, -211, -212,
-		-212, -213, -214, -215, -216, -217, -218, -220,
-		-220, -221, -222, -223, -224, -225, -225, -227,
-		-227, -228, -229, -230, -230, -231, -233, -234,
-		-234, -235, -235, -237, -238, -239, -239, -240,
-		-240, -242, -242, -243, -243, -245, -246, -247,
-		-247, -249, -248, -249, -250, -251, -251, -253,
-		-253, -253, -255, -255, -256, -256, -257, -258,
-		-259, -259, -260, -261, -261, -262, -262, -264,
-		-263, -265, -265, -265, -266, -267, -267, -268,
-		-269, -269, -269, -270, -271, -271, -272, -273,
-		-273, -273, -274, -274, -276, -275, -276, -277,
-		-277, -278, -278, -278, -279, -279, -280, -281,
-		-280, -281, -282, -283, -283, -282, -284, -284,
-		-284, -285, -285, -286, -286, -286, -287, -287,
-		-288, -288, -288, -289, -289, -289, -290, -290,
-		-290, -291, -291, -292, -291, -291, -292, -292,
-		-292, -293, -293, -293, -294, -294, -295, -295,
-		-294, -295, -295, -296, -297, -297, -297, -297,
-		-297, -297, -298, -298, -297, -298, -298, -298,
-		-299, -299, -300, -299, -299, -300, -299, -300,
-		-301, -300, -300, -301, -300, -301, -301, -301,
-		-301, -301, -302, -301, -302, -301, -302, -302,
-		-302, -302, -302, -302, -302, -302, -303, -302,
-		-303, -302, -303, -303, -302, -303, -303, -303,
-		-302, -303, -303, -302, -303, -303, -302, -303,
-		-303, -302, -303, -303, -302, -303, -303, -303,
-		-303, -302, -303, -303, -302, -302, -302, -303,
-		-302, -302, -302, -301, -303, -302, -301, -302,
-		-301, -301, -301, -302, -301, -301, -301, -300,
-		-301, -300, -300, -300, -300, -299, -300, -299,
-		-300, -300, -299, -300, -299, -299, -299, -299,
-		-298, -299, -298, -297, -297, -297, -296, -297,
-		-296, -296, -296, -296, -295, -296, -295, -296,
-		-295, -294, -294, -294, -293, -294, -294, -293,
-		-293, -292, -293, -292, -292, -292, -291, -290,
-		-291, -290, -291, -289, -289, -290, -289, -289,
-		-288, -288, -288, -288, -286, -287, -286, -286,
-		-286, -285, -286, -284, -284, -284, -284, -283,
-		-283, -283, -282, -282, -282, -281, -280, -281,
-		-279, -280, -280, -278, -279, -278, -278, -277,
-		-278, -276, -276, -277, -275, -276, -274, -275,
-		-274, -273, -273, -272, -273, -272, -272, -271,
-		-270, -270, -269, -269, -269, -268, -268, -267,
-		-267, -266, -266, -266, -265, -265, -264, -264,
-		-263, -263, -262, -262, -261, -261, -260, -260,
-		-259, -259, -258, -258, -257, -257, -256, -256,
-		-256, -255, -254, -254, -253, -253, -252, -252,
-		-251, -251, -250, -250, -249, -249, -248, -248,
-		-247, -247, -246, -246, -245, -245, -244, -244,
-		-243, -242, -242, -241, -241, -240, -239, -239,
-		-239, -238, -238, -237, -237, -235, -235, -235,
-		-234, -234, -232, -233, -232, -232, -231, -229,
-		-230, -229, -228, -228, -227, -226, -227, -225,
-		-224, -225, -223, -223, -222, -222, -221, -221,
-		-220, -219, -219, -218, -218, -216, -217, -216,
-		-215, -215, -214, -213, -212, -213, -211, -211,
-		-210, -210, -209, -209, -208, -206, -207, -206,
-		-205, -204, -204, -204, -203, -202, -202, -200,
-		-200, -200, -200, -198, -197, -197, -196, -195,
-		-195, -195, -194, -194, -192, -192, -191, -191,
-		-189, -189, -188, -188, -187, -186, -186, -186,
-		-185, -185, -183, -183, -182, -182, -181, -181,
-		-180, -178, -178, -177, -177, -176, -176, -174,
-		-174, -173, -173, -172, -172, -172, -170, -170,
-		-168, -168, -167, -167, -167, -165, -165, -164,
-		-164, -164, -162, -162, -161, -160, -160, -158,
-		-158, -158, -157, -156, -155, -155, -154, -153,
-		-153, -152, -151, -151, -150, -149, -149, -148,
-		-147, -147, -146, -146, -144, -144, -144, -142,
-		-142, -141, -142, -140, -140, -139, -138, -138,
-		-137, -136, -136, -134, -134, -133, -134, -132,
-		-132, -131, -130, -130, -128, -128, -128, -127,
-		-127, -126, -124, -124, -124, -123, -123, -122,
-		-121, -120, -120, -119, -118, -118, -117, -117,
-		-116, -115, -115, -115, -114, -113, -111, -111,
-		-110, -110, -109, -109, -108, -107, -107, -106,
-		-105, -104, -104, -103, -102, -103, -102, -101,
-		-101, -100,  -99,  -99,  -98,  -97,  -97,  -96,
-		 -96,  -95,  -94,  -94,  -93,  -92,  -92,  -91,
-		 -91,  -90,  -89,  -88,  -88,  -88,  -87,  -86,
-		 -85,  -86,  -84,  -84,  -83,  -82,  -82,  -81,
-		 -81,  -80,  -80,  -78,  -79,  -77,  -77,  -77,
-		 -76,  -76,  -75,  -74,  -74,  -73,  -72,  -72,
-		 -72,  -71,  -70,  -70,  -69,  -68,  -68,  -68,
-		 -66,  -67,  -66,  -65,  -65,  -65,  -63,  -63,
-		 -62,  -62,  -61,  -61,  -60,  -60,  -60,  -58,
-		 -58,  -58,  -56,  -56,  -56,  -55,  -54,  -55,
-		 -54,  -54,  -53,  -52,  -51,  -51,  -51,  -50,
-		 -49,  -49,  -49,  -49,  -48,  -47,  -46,  -46,
-		 -46,  -46,  -45,  -43,  -43,  -43,  -43,  -42,
-		 -42,  -42,  -40,  -40,  -40,  -39,  -39,  -38,
-		 -38,  -38,  -37,  -37,  -36,  -36,  -35,  -35,
-		 -34,  -35,  -34,  -33,  -33,  -32,  -32,  -31,
-		 -31,  -31,  -30,  -29,  -29,  -29,  -28,  -27,
-		 -28,  -28,  -27,  -26,  -26,  -25,  -25,  -25,
-		 -24,  -24,  -24,  -23,  -23,  -22,  -22,  -22,
-		 -21,  -21,  -20,  -20,  -20,  -20,  -19,  -18,
-		 -19,  -18,  -18,  -17,  -18,  -17,  -16,  -17,
-		 -16,  -15,  -15,  -15,  -14,  -14,  -15,  -13,
-		 -13,  -13,  -13,  -12,  -12,  -11,  -12,  -11,
-		 -12,  -10,  -10,  -10,  -10,  -10,   -9,  -10,
-		  -9,   -9,   -9,   -8,   -8,   -7,   -8,   -7,
-		  -7,   -7,   -6,   -6,   -6,   -7,   -6,   -6,
-		  -5,   -5,   -5,   -5,   -5,   -4,   -4,   -5,
-		  -4,   -4,   -3,   -3,   -3,   -3,   -3,   -2,
-		  -3,   -2,   -2,   -2,   -1,   -2,   -1,   -2,
-		  -1,   -1,   -1,   -1,   -1,    0,   -1,    0,
-		  -1,   -1,    0,    0,   -1,    0,    0,   -1,
-		   1,    1,    0,    0,    0,    1,    0,    0,
-		   0,    0,    0,    0,    0,    0,    0,    0
-	}
-};
-#else   /* defined(CONFIG_CSI2_PLUS) */
-static const int zoom_table[4][HRT_GDC_N] = {
-	{	  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		 -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,
-		 -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,
-		 -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,
-		 -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,
-		 -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,
-		 -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,
-		 -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,
-		 -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,
-		 -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,
-		 -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,
-		 -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,
-		 -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,
-		 -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,
-		 -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		-10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4,
-		-10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4,
-		-11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4,
-		-11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4,
-		-11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4,
-		-11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4,
-		-13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4,
-		-13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4,
-		-13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4,
-		-13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4,
-		-11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,
-		 -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,
-		 -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,
-		 -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,  -7<<4,
-		 -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,
-		 -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4
-	},
-	{	  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4,
-		  2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4,
-		  4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,
-		  4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,
-		  7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,
-		  7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,
-		  9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,
-		  9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,
-		  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,
-		  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,
-		  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,
-		  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,
-		  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,
-		  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,
-		  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,
-		  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,
-		  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,
-		  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,
-		  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,
-		  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,
-		  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,
-		  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,
-		  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,
-		  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,
-		  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,
-		  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,
-		  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,
-		  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,
-		  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,
-		  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,
-		  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,
-		  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,
-		  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,
-		  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,
-		  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,
-		  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,
-		  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,
-		  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,
-		  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,
-		  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,
-		  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,
-		  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,
-		  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,
-		  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,
-		  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,
-		  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,
-		  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,
-		  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,
-		 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4,
-		 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4,
-		 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4,
-		 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4,
-		 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4,
-		 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4,
-		 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4,
-		 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4,
-		 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4,
-		 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4,
-		 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4,
-		 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4,
-		 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4,
-		 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4,
-		 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4,
-		 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4,
-		 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4,
-		 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4,
-		 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4,
-		 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4,
-		 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4,
-		 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4,
-		 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4,
-		 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4,
-		 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4,
-		 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4,
-		 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4,
-		 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4,
-		 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4,
-		 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4,
-		 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4,
-		 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4,
-		 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4,
-		 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4,
-		 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4,
-		 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4,
-		 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4,
-		 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4,
-		 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4,
-		 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4,
-		 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4,
-		 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4,
-		 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4,
-		 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4,
-		 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4,
-		 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4,
-		 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4,
-		 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4,
-		 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4,
-		 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4,
-		 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4,
-		 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4,
-		 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4,
-		 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4,
-		 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4,
-		 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4,
-		 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4,
-		 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4,
-		 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4,
-		 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4,
-		 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4,
-		 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4,
-		 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4,
-		 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4,
-		 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4,
-		 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4,
-		 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4,
-		 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4,
-		 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4,
-		 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4,
-		 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4,
-		 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4,
-		 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4,
-		 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4,
-		 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4,
-		 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4,
-		 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4,
-		 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4
-	},
-	{	 256<<4, 256<<4, 256<<4, 256<<4, 256<<4, 256<<4, 256<<4, 256<<4,
-		 256<<4, 256<<4, 256<<4, 256<<4, 256<<4, 256<<4, 256<<4, 256<<4,
-		 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4,
-		 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4,
-		 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4,
-		 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4, 255<<4,
-		 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4,
-		 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4, 254<<4,
-		 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4,
-		 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4, 253<<4,
-		 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4,
-		 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4, 252<<4,
-		 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4,
-		 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4, 250<<4,
-		 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4,
-		 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4, 248<<4,
-		 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4,
-		 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4, 246<<4,
-		 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4,
-		 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4, 244<<4,
-		 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4,
-		 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4, 241<<4,
-		 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4,
-		 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4, 239<<4,
-		 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4,
-		 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4, 236<<4,
-		 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4,
-		 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4, 232<<4,
-		 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4,
-		 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4, 229<<4,
-		 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4,
-		 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4, 225<<4,
-		 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4,
-		 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4, 222<<4,
-		 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4,
-		 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4, 218<<4,
-		 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4,
-		 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4, 213<<4,
-		 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4,
-		 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4, 209<<4,
-		 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4,
-		 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4, 205<<4,
-		 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4,
-		 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4, 200<<4,
-		 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4,
-		 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4, 195<<4,
-		 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4,
-		 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4, 191<<4,
-		 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4,
-		 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4, 186<<4,
-		 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4,
-		 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4, 181<<4,
-		 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4,
-		 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4, 176<<4,
-		 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4,
-		 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4, 170<<4,
-		 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4,
-		 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4, 165<<4,
-		 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4,
-		 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4, 160<<4,
-		 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4,
-		 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4, 154<<4,
-		 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4,
-		 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4, 149<<4,
-		 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4,
-		 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4, 144<<4,
-		 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4,
-		 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4, 138<<4,
-		 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4,
-		 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4, 132<<4,
-		 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4,
-		 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4, 127<<4,
-		 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4,
-		 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4, 121<<4,
-		 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4,
-		 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4, 116<<4,
-		 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4,
-		 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4, 110<<4,
-		 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4,
-		 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4, 105<<4,
-		  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,
-		  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,  99<<4,
-		  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,
-		  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,  94<<4,
-		  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,
-		  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,  88<<4,
-		  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,
-		  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,  83<<4,
-		  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,
-		  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,  78<<4,
-		  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,
-		  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,  73<<4,
-		  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,
-		  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,  67<<4,
-		  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,
-		  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,  62<<4,
-		  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,
-		  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,  58<<4,
-		  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,
-		  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,  53<<4,
-		  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,
-		  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,  48<<4,
-		  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,
-		  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,  43<<4,
-		  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,
-		  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,  39<<4,
-		  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,
-		  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,  35<<4,
-		  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,
-		  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,  31<<4,
-		  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,
-		  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,  27<<4,
-		  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,
-		  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,  23<<4,
-		  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,
-		  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,  19<<4,
-		  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,
-		  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,  16<<4,
-		  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,
-		  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,  12<<4,
-		  9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,
-		  9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,   9<<4,
-		  7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,
-		  7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,   7<<4,
-		  4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,
-		  4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,   4<<4,
-		  2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4,
-		  2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4,   2<<4
-	},
-	{	  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,
-		 -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,
-		 -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		-10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4,
-		-10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4, -10<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4,
-		-13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4,
-		-19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4,
-		-19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4,
-		-19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4,
-		-19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4,
-		-19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4, -19<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4, -18<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4, -17<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4, -16<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4, -15<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4, -14<<4,
-		-13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4,
-		-13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4, -13<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4, -12<<4,
-		-11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4,
-		-11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4,
-		-11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4,
-		-11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4, -11<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,  -9<<4,
-		 -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,
-		 -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,  -8<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,  -6<<4,
-		 -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,
-		 -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,  -5<<4,
-		 -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,
-		 -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,
-		 -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,  -4<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,  -3<<4,
-		 -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,
-		 -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,
-		 -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,
-		 -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,  -2<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		 -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,  -1<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  1<<4,   1<<4,   1<<4,   1<<4,   1<<4,   1<<4,   1<<4,   1<<4,
-		  1<<4,   1<<4,   1<<4,   1<<4,   1<<4,   1<<4,   1<<4,   1<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,
-		  0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4,   0<<4
-	}
-};
-#endif
-#else
-#error "sh_css_params.c: GDC version must be \
-	one of {GDC_VERSION_2}"
-#endif
-
-static const struct ia_css_dz_config default_dz_config = {
-	HRT_GDC_N,
-	HRT_GDC_N,
-	{ \
-		{0, 0}, \
-		{0, 0}, \
-	}
-};
-
-static const struct ia_css_vector default_motion_config = {
-	0,
-	0
-};
-
-/* ------ deprecated(bz675) : from ------ */
-static const struct ia_css_shading_settings default_shading_settings = {
-	1	/* enable shading table conversion in the css
-		(This matches the legacy way.) */
-};
-/* ------ deprecated(bz675) : to ------ */
-
-struct ia_css_isp_skc_dvs_statistics {
-	ia_css_ptr p_data;
-};
-
-static enum ia_css_err
-ref_sh_css_ddr_address_map(
-		struct sh_css_ddr_address_map *map,
-		struct sh_css_ddr_address_map *out);
-
-static enum ia_css_err
-write_ia_css_isp_parameter_set_info_to_ddr(
-	struct ia_css_isp_parameter_set_info *me,
-	hrt_vaddress *out);
-
-static enum ia_css_err
-free_ia_css_isp_parameter_set_info(hrt_vaddress ptr);
-
-static enum ia_css_err
-sh_css_params_write_to_ddr_internal(
-		struct ia_css_pipe *pipe,
-		unsigned pipe_id,
-		struct ia_css_isp_parameters *params,
-		const struct ia_css_pipeline_stage *stage,
-		struct sh_css_ddr_address_map *ddr_map,
-		struct sh_css_ddr_address_map_size *ddr_map_size);
-
-static enum ia_css_err
-sh_css_create_isp_params(struct ia_css_stream *stream,
-			 struct ia_css_isp_parameters **isp_params_out);
-
-static bool
-sh_css_init_isp_params_from_global(struct ia_css_stream *stream,
-		struct ia_css_isp_parameters *params,
-		bool use_default_config,
-		struct ia_css_pipe *pipe_in);
-
-static enum ia_css_err
-sh_css_init_isp_params_from_config(struct ia_css_pipe *pipe,
-		struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config,
-		struct ia_css_pipe *pipe_in);
-
-static enum ia_css_err
-sh_css_set_global_isp_config_on_pipe(
-	struct ia_css_pipe *curr_pipe,
-	const struct ia_css_isp_config *config,
-	struct ia_css_pipe *pipe);
-
-#if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-static enum ia_css_err
-sh_css_set_per_frame_isp_config_on_pipe(
-	struct ia_css_stream *stream,
-	const struct ia_css_isp_config *config,
-	struct ia_css_pipe *pipe);
-#endif
-
-static enum ia_css_err
-sh_css_update_uds_and_crop_info_based_on_zoom_region(
-	const struct ia_css_binary_info *info,
-	const struct ia_css_frame_info *in_frame_info,
-	const struct ia_css_frame_info *out_frame_info,
-	const struct ia_css_resolution *dvs_env,
-	const struct ia_css_dz_config *zoom,
-	const struct ia_css_vector *motion_vector,
-	struct sh_css_uds_info *uds,		/* out */
-	struct sh_css_crop_pos *sp_out_crop_pos,	/* out */
-	struct ia_css_resolution pipe_in_res,
-	bool enable_zoom);
-
-hrt_vaddress
-sh_css_params_ddr_address_map(void)
-{
-	return sp_ddr_ptrs;
-}
-
-/* ****************************************************
- * Each coefficient is stored as 7bits to fit 2 of them into one
- * ISP vector element, so we will store 4 coefficents on every
- * memory word (32bits)
- *
- * 0: Coefficient 0 used bits
- * 1: Coefficient 1 used bits
- * 2: Coefficient 2 used bits
- * 3: Coefficient 3 used bits
- * x: not used
- *
- * xx33333332222222 | xx11111110000000
- *
- * ***************************************************
- */
-static struct ia_css_host_data *
-convert_allocate_fpntbl(struct ia_css_isp_parameters *params)
-{
-	unsigned int i, j;
-	short *data_ptr;
-	struct ia_css_host_data *me;
-	unsigned int isp_format_data_size;
-	uint32_t *isp_format_data_ptr;
-
-	assert(params != NULL);
-
-	data_ptr = params->fpn_config.data;
-	isp_format_data_size = params->fpn_config.height * params->fpn_config.width * sizeof(uint32_t);
-
-	me = ia_css_host_data_allocate(isp_format_data_size);
-
-	if (!me)
-		return NULL;
-
-	isp_format_data_ptr = (uint32_t *)me->address;
-
-	for (i = 0; i < params->fpn_config.height; i++) {
-		for (j = 0;
-		     j < params->fpn_config.width;
-		     j += 4, data_ptr += 4, isp_format_data_ptr++) {
-			int data = data_ptr[0] << 0 |
-				   data_ptr[1] << 7 |
-				   data_ptr[2] << 16 |
-				   data_ptr[3] << 23;
-			*isp_format_data_ptr = data;
-		}
-	}
-	return me;
-}
-
-static enum ia_css_err
-store_fpntbl(struct ia_css_isp_parameters *params, hrt_vaddress ptr)
-{
-	struct ia_css_host_data *isp_data;
-
-	assert(params != NULL);
-	assert(ptr != mmgr_NULL);
-
-	isp_data = convert_allocate_fpntbl(params);
-	if (!isp_data) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	}
-	ia_css_params_store_ia_css_host_data(ptr, isp_data);
-
-	ia_css_host_data_free(isp_data);
-	return IA_CSS_SUCCESS;
-}
-
-static void
-convert_raw_to_fpn(struct ia_css_isp_parameters *params)
-{
-	int maxval = 0;
-	unsigned int i;
-
-	assert(params != NULL);
-
-	/* Find the maximum value in the table */
-	for (i = 0; i < params->fpn_config.height * params->fpn_config.width; i++) {
-		int val = params->fpn_config.data[i];
-		/* Make sure FPN value can be represented in 13-bit unsigned
-		 * number (ISP precision - 1), but note that actual input range
-		 * depends on precision of input frame data.
-		 */
-		if (val < 0) {
-/* Checkpatch patch */
-			val = 0;
-		} else if (val >= (1 << 13)) {
-/* Checkpatch patch */
-/* MW: BUG, is "13" a system or application property */
-			val = (1 << 13) - 1;
-		}
-		maxval = max(maxval, val);
-	}
-	/* Find the lowest shift value to remap the values in the range
-	 * 0..maxval to 0..2^shiftval*63.
-	 */
-	params->fpn_config.shift = 0;
-	while (maxval > 63) {
-/* MW: BUG, is "63" a system or application property */
-		maxval >>= 1;
-		params->fpn_config.shift++;
-	}
-	/* Adjust the values in the table for the shift value */
-	for (i = 0; i < params->fpn_config.height * params->fpn_config.width; i++)
-		((unsigned short *) params->fpn_config.data)[i] >>= params->fpn_config.shift;
-}
-
-static void
-ia_css_process_kernel(struct ia_css_stream *stream,
-		      struct ia_css_isp_parameters *params,
-		      void (*process)(unsigned pipe_id,
-				      const struct ia_css_pipeline_stage *stage,
-				      struct ia_css_isp_parameters *params))
-{
-	int i;
-	for (i = 0; i < stream->num_pipes; i++) {
-		struct ia_css_pipe *pipe = stream->pipes[i];
-		struct ia_css_pipeline *pipeline = ia_css_pipe_get_pipeline(pipe);
-		struct ia_css_pipeline_stage *stage;
-
-		/* update the other buffers to the pipe specific copies */
-		for (stage = pipeline->stages; stage; stage = stage->next) {
-			if (!stage || !stage->binary) continue;
-			process(pipeline->pipe_id, stage, params);
-		}
-	}
-}
-
-static enum ia_css_err
-sh_css_select_dp_10bpp_config(const struct ia_css_pipe *pipe, bool *is_dp_10bpp) {
-
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	/* Currently we check if 10bpp DPC configuration is required based
-	 * on the use case,i.e. if BDS and DPC is both enabled. The more cleaner
-	 * design choice would be to expose the type of DPC (either 10bpp or 13bpp)
-	 * using the binary info, but the current control flow does not allow this
-	 * implementation. (This is because the configuration is set before a
-	 * binary is selected, and the binary info is not available)
-	 */
-	if((pipe == NULL) || (is_dp_10bpp == NULL)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-		err = IA_CSS_ERR_INTERNAL_ERROR;
-	} else {
-		*is_dp_10bpp = false;
-
-		/* check if DPC is enabled from the host */
-		if (pipe->config.enable_dpc) {
-			/*check if BDS is enabled*/
-			unsigned int required_bds_factor = SH_CSS_BDS_FACTOR_1_00;
-			if ((pipe->config.bayer_ds_out_res.width != 0) &&
-			  (pipe->config.bayer_ds_out_res.height != 0)) {
-				if (IA_CSS_SUCCESS == binarydesc_calculate_bds_factor(
-					pipe->config.input_effective_res,
-					pipe->config.bayer_ds_out_res,
-					&required_bds_factor)) {
-					if (SH_CSS_BDS_FACTOR_1_00 != required_bds_factor) {
-						/*we use 10bpp BDS configuration*/
-						*is_dp_10bpp = true;
-					}
-				}
-			}
-		}
-	}
-
-	return err;
-}
-
-enum ia_css_err
-sh_css_set_black_frame(struct ia_css_stream *stream,
-	const struct ia_css_frame *raw_black_frame)
-{
-	struct ia_css_isp_parameters *params;
-	/* this function desperately needs to be moved to the ISP or SP such
-	 * that it can use the DMA.
-	 */
-	unsigned int height, width, y, x, k, data;
-	hrt_vaddress ptr;
-
-	assert(stream != NULL);
-	assert(raw_black_frame != NULL);
-
-	params = stream->isp_params_configs;
-	height = raw_black_frame->info.res.height;
-	width = raw_black_frame->info.padded_width,
-
-	ptr = raw_black_frame->data
-		+ raw_black_frame->planes.raw.offset;
-
-	IA_CSS_ENTER_PRIVATE("black_frame=%p", raw_black_frame);
-
-	if (params->fpn_config.data &&
-	    (params->fpn_config.width != width || params->fpn_config.height != height)) {
-		sh_css_free(params->fpn_config.data);
-		params->fpn_config.data = NULL;
-	}
-	if (params->fpn_config.data == NULL) {
-		params->fpn_config.data = sh_css_malloc(height * width * sizeof(short));
-		if (!params->fpn_config.data) {
-			IA_CSS_ERROR("out of memory");
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-			return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		}
-		params->fpn_config.width = width;
-		params->fpn_config.height = height;
-		params->fpn_config.shift = 0;
-	}
-
-	/* store raw to fpntbl */
-	for (y = 0; y < height; y++) {
-		for (x = 0; x < width; x += (ISP_VEC_NELEMS * 2)) {
-			int ofs = y * width + x;
-			for (k = 0; k < ISP_VEC_NELEMS; k += 2) {
-				mmgr_load(ptr, (void *)(&data), sizeof(int));
-				params->fpn_config.data[ofs + 2 * k] =
-				    (short) (data & 0xFFFF);
-				params->fpn_config.data[ofs + 2 * k + 2] =
-				    (short) ((data >> 16) & 0xFFFF);
-				ptr += sizeof(int);	/* byte system address */
-			}
-			for (k = 0; k < ISP_VEC_NELEMS; k += 2) {
-				mmgr_load(ptr, (void *)(&data), sizeof(int));
-				params->fpn_config.data[ofs + 2 * k + 1] =
-				    (short) (data & 0xFFFF);
-				params->fpn_config.data[ofs + 2 * k + 3] =
-				    (short) ((data >> 16) & 0xFFFF);
-				ptr += sizeof(int);	/* byte system address */
-			}
-		}
-	}
-
-	/* raw -> fpn */
-	convert_raw_to_fpn(params);
-
-	/* overwrite isp parameter */
-	ia_css_process_kernel(stream, params, ia_css_kernel_process_param[IA_CSS_FPN_ID]);
-
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-
-	return IA_CSS_SUCCESS;
-}
-
-bool
-sh_css_params_set_binning_factor(struct ia_css_stream *stream, unsigned int binning_fact)
-{
-	struct ia_css_isp_parameters *params;
-
-	IA_CSS_ENTER_PRIVATE("void");
-	assert(stream != NULL);
-
-	params = stream->isp_params_configs;
-
-	if (params->sensor_binning != binning_fact) {
-		params->sensor_binning = binning_fact;
-		params->sc_table_changed = true;
-	}
-
-	IA_CSS_LEAVE_PRIVATE("void");
-
-	return params->sc_table_changed;
-}
-
-static void
-sh_css_update_shading_table_status(struct ia_css_pipe *pipe,
-			struct ia_css_isp_parameters *params)
-{
-	if (params && pipe && (pipe->pipe_num != params->sc_table_last_pipe_num)) {
-		params->sc_table_dirty = true;
-		params->sc_table_last_pipe_num = pipe->pipe_num;
-	}
-}
-
-static void
-sh_css_set_shading_table(struct ia_css_stream *stream,
-			 struct ia_css_isp_parameters *params,
-			 const struct ia_css_shading_table *table)
-{
-	IA_CSS_ENTER_PRIVATE("");
-	if (table == NULL)
-		return;
-	assert(stream != NULL);
-
-	if (!table->enable)
-		table = NULL;
-
-	if ((table != params->sc_table) || params->sc_table_dirty) {
-		params->sc_table = table;
-		params->sc_table_changed = true;
-		params->sc_table_dirty = false;
-		/* Not very clean, this goes to sh_css.c to invalidate the
-		 * shading table for all pipes. Should replaced by a loop
-		 * and a pipe-specific call.
-		 */
-		if (!params->output_frame)
-			sh_css_invalidate_shading_tables(stream);
-	}
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-void
-ia_css_params_store_ia_css_host_data(
-	hrt_vaddress ddr_addr,
-	struct ia_css_host_data *data)
-{
-	assert(data != NULL);
-	assert(data->address != NULL);
-	assert(ddr_addr != mmgr_NULL);
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	mmgr_store(ddr_addr,
-	   (void *)(data->address),
-	   (size_t)data->size);
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-struct ia_css_host_data *
-ia_css_params_alloc_convert_sctbl(
-	    const struct ia_css_pipeline_stage *stage,
-	    const struct ia_css_shading_table *shading_table)
-{
-	const struct ia_css_binary *binary = stage->binary;
-	struct ia_css_host_data    *sctbl;
-	unsigned int i, j, aligned_width, row_padding;
-	unsigned int sctbl_size;
-	short int    *ptr;
-
-	assert(binary != NULL);
-	assert(shading_table != NULL);
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	if (shading_table == NULL) {
-		IA_CSS_LEAVE_PRIVATE("void");
-		return NULL;
-	}
-
-	aligned_width = binary->sctbl_aligned_width_per_color;
-	row_padding = aligned_width - shading_table->width;
-	sctbl_size = shading_table->height * IA_CSS_SC_NUM_COLORS * aligned_width * sizeof(short);
-
-	sctbl = ia_css_host_data_allocate((size_t)sctbl_size);
-
-	if (!sctbl)
-		return NULL;
-	ptr = (short int*)sctbl->address;
-	memset(ptr,
-		0,
-		sctbl_size);
-
-	for (i = 0; i < shading_table->height; i++) {
-		for (j = 0; j < IA_CSS_SC_NUM_COLORS; j++) {
-			memcpy(ptr,
-				   &shading_table->data[j]
-					[i*shading_table->width],
-				   shading_table->width * sizeof(short));
-			ptr += aligned_width;
-		}
-	}
-
-	IA_CSS_LEAVE_PRIVATE("void");
-	return sctbl;
-}
-
-enum ia_css_err ia_css_params_store_sctbl(
-	const struct ia_css_pipeline_stage *stage,
-	hrt_vaddress sc_tbl,
-	const struct ia_css_shading_table  *sc_config)
-{
-	struct ia_css_host_data *isp_sc_tbl;
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	if (sc_config == NULL) {
-		IA_CSS_LEAVE_PRIVATE("void");
-		return IA_CSS_SUCCESS;
-	}
-
-	isp_sc_tbl = ia_css_params_alloc_convert_sctbl(stage, sc_config);
-	if (!isp_sc_tbl) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	}
-	/* store the shading table to ddr */
-	ia_css_params_store_ia_css_host_data(sc_tbl, isp_sc_tbl);
-	ia_css_host_data_free(isp_sc_tbl);
-
-	IA_CSS_LEAVE_PRIVATE("void");
-
-	return IA_CSS_SUCCESS;
-}
-
-static void
-sh_css_enable_pipeline(const struct ia_css_binary *binary)
-{
-	if (!binary)
-		return;
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	ia_css_isp_param_enable_pipeline(&binary->mem_params);
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static enum ia_css_err
-ia_css_process_zoom_and_motion(
-	struct ia_css_isp_parameters *params,
-	const struct ia_css_pipeline_stage *first_stage)
-{
-	/* first_stage can be  NULL */
-	const struct ia_css_pipeline_stage *stage;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_resolution pipe_in_res;
-	pipe_in_res.width = 0;
-	pipe_in_res.height = 0;
-
-	assert(params != NULL);
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	/* Go through all stages to udate uds and cropping */
-	for (stage = first_stage; stage; stage = stage->next) {
-
-		struct ia_css_binary *binary;
-		/* note: the var below is made static as it is quite large;
-		   if it is not static it ends up on the stack which could
-		   cause issues for drivers
-		*/
-		static struct ia_css_binary tmp_binary;
-
-		const struct ia_css_binary_xinfo *info = NULL;
-
-		binary = stage->binary;
-		if (binary) {
-			info = binary->info;
-		} else {
-			const struct sh_css_binary_args *args = &stage->args;
-			const struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS] = {NULL};
-			if (args->out_frame[0])
-				out_infos[0] = &args->out_frame[0]->info;
-			info = &stage->firmware->info.isp;
-			ia_css_binary_fill_info(info, false, false,
-				ATOMISP_INPUT_FORMAT_RAW_10,
-				args->in_frame  ? &args->in_frame->info  : NULL,
-				NULL,
-				out_infos,
-				args->out_vf_frame ? &args->out_vf_frame->info
-									: NULL,
-				&tmp_binary,
-				NULL,
-				-1, true);
-			binary = &tmp_binary;
-			binary->info = info;
-		}
-
-		if (stage == first_stage) {
-			/* we will use pipe_in_res to scale the zoom crop region if needed */
-			pipe_in_res = binary->effective_in_frame_res;
-		}
-
-		assert(stage->stage_num < SH_CSS_MAX_STAGES);
-		if (params->dz_config.zoom_region.resolution.width == 0 &&
-		    params->dz_config.zoom_region.resolution.height == 0) {
-			sh_css_update_uds_and_crop_info(
-				&info->sp,
-				&binary->in_frame_info,
-				&binary->out_frame_info[0],
-				&binary->dvs_envelope,
-				&params->dz_config,
-				&params->motion_config,
-				&params->uds[stage->stage_num].uds,
-				&params->uds[stage->stage_num].crop_pos,
-				stage->enable_zoom);
-		} else {
-			err = sh_css_update_uds_and_crop_info_based_on_zoom_region(
-				&info->sp,
-				&binary->in_frame_info,
-				&binary->out_frame_info[0],
-				&binary->dvs_envelope,
-				&params->dz_config,
-				&params->motion_config,
-				&params->uds[stage->stage_num].uds,
-				&params->uds[stage->stage_num].crop_pos,
-				pipe_in_res,
-				stage->enable_zoom);
-			if (err != IA_CSS_SUCCESS)
-			    return err;
-		}
-	}
-	params->isp_params_changed = true;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-	return err;
-}
-
-static void
-sh_css_set_gamma_table(struct ia_css_isp_parameters *params,
-			const struct ia_css_gamma_table *table)
-{
-	if (table == NULL)
-		return;
-	IA_CSS_ENTER_PRIVATE("table=%p", table);
-
-	assert(params != NULL);
-	params->gc_table = *table;
-	params->config_changed[IA_CSS_GC_ID] = true;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_get_gamma_table(const struct ia_css_isp_parameters *params,
-			struct ia_css_gamma_table *table)
-{
-	if (table == NULL)
-		return;
-	IA_CSS_ENTER_PRIVATE("table=%p", table);
-
-	assert(params != NULL);
-	*table = params->gc_table;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_set_ctc_table(struct ia_css_isp_parameters *params,
-			const struct ia_css_ctc_table *table)
-{
-	if (table == NULL)
-		return;
-
-	IA_CSS_ENTER_PRIVATE("table=%p", table);
-
-	assert(params != NULL);
-	params->ctc_table = *table;
-	params->config_changed[IA_CSS_CTC_ID] = true;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_get_ctc_table(const struct ia_css_isp_parameters *params,
-			struct ia_css_ctc_table *table)
-{
-	if (table == NULL)
-		return;
-
-	IA_CSS_ENTER_PRIVATE("table=%p", table);
-
-	assert(params != NULL);
-	*table = params->ctc_table;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_set_macc_table(struct ia_css_isp_parameters *params,
-			const struct ia_css_macc_table *table)
-{
-	if (table == NULL)
-		return;
-
-	IA_CSS_ENTER_PRIVATE("table=%p", table);
-
-	assert(params != NULL);
-	params->macc_table = *table;
-	params->config_changed[IA_CSS_MACC_ID] = true;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_get_macc_table(const struct ia_css_isp_parameters *params,
-			struct ia_css_macc_table *table)
-{
-	if (table == NULL)
-		return;
-
-	IA_CSS_ENTER_PRIVATE("table=%p", table);
-
-	assert(params != NULL);
-	*table = params->macc_table;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-void ia_css_morph_table_free(
-	struct ia_css_morph_table *me)
-{
-
-	unsigned int i;
-
-	if (me == NULL)
-		return;
-
-	IA_CSS_ENTER("");
-
-
-
-	for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
-		if (me->coordinates_x[i]) {
-			sh_css_free(me->coordinates_x[i]);
-			me->coordinates_x[i] = NULL;
-		}
-		if (me->coordinates_y[i]) {
-			sh_css_free(me->coordinates_y[i]);
-			me->coordinates_y[i] = NULL;
-		}
-	}
-
-	sh_css_free(me);
-	IA_CSS_LEAVE("void");
-
-}
-
-
-struct ia_css_morph_table *ia_css_morph_table_allocate(
-	unsigned int width,
-	unsigned int height)
-{
-
-	unsigned int i;
-	struct ia_css_morph_table *me;
-
-	IA_CSS_ENTER("");
-
-	me = sh_css_malloc(sizeof(*me));
-	if (me == NULL) {
-		IA_CSS_ERROR("out of memory");
-		return me;
-	}
-
-	for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
-		me->coordinates_x[i] = NULL;
-		me->coordinates_y[i] = NULL;
-	}
-
-	for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
-		me->coordinates_x[i] =
-		    sh_css_malloc(height * width *
-				  sizeof(*me->coordinates_x[i]));
-		me->coordinates_y[i] =
-		    sh_css_malloc(height * width *
-				  sizeof(*me->coordinates_y[i]));
-
-		if ((me->coordinates_x[i] == NULL) ||
-			(me->coordinates_y[i] == NULL)) {
-			ia_css_morph_table_free(me);
-			me = NULL;
-			return me;
-		}
-	}
-	me->width = width;
-	me->height = height;
-	IA_CSS_LEAVE("");
-	return me;
-
-}
-
-
-static enum ia_css_err sh_css_params_default_morph_table(
-	struct ia_css_morph_table **table,
-	const struct ia_css_binary *binary)
-{
-	/* MW 2400 advanced requires different scaling */
-	unsigned int i, j, k, step, width, height;
-	short start_x[IA_CSS_MORPH_TABLE_NUM_PLANES] = { -8, 0, -8, 0, 0, -8 },
-	      start_y[IA_CSS_MORPH_TABLE_NUM_PLANES] = { 0, 0, -8, -8, -8, 0 };
-	struct ia_css_morph_table *tab;
-
-	assert(table != NULL);
-	assert(binary != NULL);
-
-	IA_CSS_ENTER_PRIVATE("");
-
-	step = (ISP_VEC_NELEMS / 16) * 128,
-	width = binary->morph_tbl_width,
-	height = binary->morph_tbl_height;
-
-	tab = ia_css_morph_table_allocate(width, height);
-	if (tab == NULL)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-
-	for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
-		short val_y = start_y[i];
-		for (j = 0; j < height; j++) {
-			short val_x = start_x[i];
-			unsigned short *x_ptr, *y_ptr;
-
-			x_ptr = &tab->coordinates_x[i][j * width];
-			y_ptr = &tab->coordinates_y[i][j * width];
-			for (k = 0; k < width;
-			     k++, x_ptr++, y_ptr++, val_x += (short)step) {
-				if (k == 0)
-					*x_ptr = 0;
-				else if (k == width - 1)
-					*x_ptr = val_x + 2 * start_x[i];
-				else
-					*x_ptr = val_x;
-				if (j == 0)
-					*y_ptr = 0;
-				else
-					*y_ptr = val_y;
-			}
-			val_y += (short)step;
-		}
-	}
-	*table = tab;
-
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-
-	return IA_CSS_SUCCESS;
-}
-
-static void
-sh_css_set_morph_table(struct ia_css_isp_parameters *params,
-			const struct ia_css_morph_table *table)
-{
-	if (table == NULL)
-		return;
-
-	IA_CSS_ENTER_PRIVATE("table=%p", table);
-
-	assert(params != NULL);
-	if (table->enable == false)
-		table = NULL;
-	params->morph_table = table;
-	params->morph_table_changed = true;
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-void
-ia_css_translate_3a_statistics(
-		struct ia_css_3a_statistics               *host_stats,
-		const struct ia_css_isp_3a_statistics_map *isp_stats)
-{
-	IA_CSS_ENTER("");
-	if (host_stats->grid.use_dmem) {
-		IA_CSS_LOG("3A: DMEM");
-		ia_css_s3a_dmem_decode(host_stats, isp_stats->dmem_stats);
-	} else {
-		IA_CSS_LOG("3A: VMEM");
-		ia_css_s3a_vmem_decode(host_stats, isp_stats->vmem_stats_hi,
-				       isp_stats->vmem_stats_lo);
-	}
-#if !defined(HAS_NO_HMEM)
-	IA_CSS_LOG("3A: HMEM");
-	ia_css_s3a_hmem_decode(host_stats, isp_stats->hmem_stats);
-#endif
-
-	IA_CSS_LEAVE("void");
-}
-
-void
-ia_css_isp_3a_statistics_map_free(struct ia_css_isp_3a_statistics_map *me)
-{
-	if (me) {
-		if (me->data_allocated) {
-			sh_css_free(me->data_ptr);
-			me->data_ptr = NULL;
-			me->data_allocated = false;
-		}
-		sh_css_free(me);
-	}
-}
-
-struct ia_css_isp_3a_statistics_map *
-ia_css_isp_3a_statistics_map_allocate(
-	const struct ia_css_isp_3a_statistics *isp_stats,
-	void *data_ptr)
-{
-	struct ia_css_isp_3a_statistics_map *me;
-	/* Windows compiler does not like adding sizes to a void *
-	 * so we use a local char * instead. */
-	char *base_ptr;
-
-	me = sh_css_malloc(sizeof(*me));
-	if (!me) {
-		IA_CSS_LEAVE("cannot allocate memory");
-		goto err;
-	}
-
-	me->data_ptr = data_ptr;
-	me->data_allocated = data_ptr == NULL;
-	if (!data_ptr) {
-		me->data_ptr = sh_css_malloc(isp_stats->size);
-		if (!me->data_ptr) {
-			IA_CSS_LEAVE("cannot allocate memory");
-			goto err;
-		}
-	}
-	base_ptr = me->data_ptr;
-
-	me->size = isp_stats->size;
-	/* GCC complains when we assign a char * to a void *, so these
-	 * casts are necessary unfortunately. */
-	me->dmem_stats    = (void *)base_ptr;
-	me->vmem_stats_hi = (void *)(base_ptr + isp_stats->dmem_size);
-	me->vmem_stats_lo = (void *)(base_ptr + isp_stats->dmem_size +
-				    isp_stats->vmem_size);
-	me->hmem_stats    = (void *)(base_ptr + isp_stats->dmem_size +
-				    2 * isp_stats->vmem_size);
-
-	IA_CSS_LEAVE("map=%p", me);
-	return me;
-
-err:
-	if (me)
-		sh_css_free(me);
-	return NULL;
-
-}
-
-enum ia_css_err
-ia_css_get_3a_statistics(struct ia_css_3a_statistics           *host_stats,
-			 const struct ia_css_isp_3a_statistics *isp_stats)
-{
-	struct ia_css_isp_3a_statistics_map *map;
-	enum ia_css_err ret = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER("host_stats=%p, isp_stats=%p", host_stats, isp_stats);
-
-	assert(host_stats != NULL);
-	assert(isp_stats != NULL);
-
-	map = ia_css_isp_3a_statistics_map_allocate(isp_stats, NULL);
-	if (map) {
-		mmgr_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size);
-		ia_css_translate_3a_statistics(host_stats, map);
-		ia_css_isp_3a_statistics_map_free(map);
-	} else {
-		IA_CSS_ERROR("out of memory");
-		ret = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	}
-
-	IA_CSS_LEAVE_ERR(ret);
-	return ret;
-}
-
-/* Parameter encoding is not yet orthogonal.
-   This function hnadles some of the exceptions.
-*/
-static void
-ia_css_set_param_exceptions(const struct ia_css_pipe *pipe,
-				struct ia_css_isp_parameters *params)
-{
-	assert(params != NULL);
-
-	/* Copy also to DP. Should be done by the driver. */
-	params->dp_config.gr = params->wb_config.gr;
-	params->dp_config.r  = params->wb_config.r;
-	params->dp_config.b  = params->wb_config.b;
-	params->dp_config.gb = params->wb_config.gb;
-#ifdef ISP2401
-	assert(pipe != NULL);
-	assert(pipe->mode < IA_CSS_PIPE_ID_NUM);
-
-	if (pipe->mode < IA_CSS_PIPE_ID_NUM) {
-		params->pipe_dp_config[pipe->mode].gr = params->wb_config.gr;
-		params->pipe_dp_config[pipe->mode].r  = params->wb_config.r;
-		params->pipe_dp_config[pipe->mode].b  = params->wb_config.b;
-		params->pipe_dp_config[pipe->mode].gb = params->wb_config.gb;
-	}
-#endif
-}
-
-#ifdef ISP2401
-static void
-sh_css_set_dp_config(const struct ia_css_pipe *pipe,
-			struct ia_css_isp_parameters *params,
-			const struct ia_css_dp_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	assert(pipe != NULL);
-	assert(pipe->mode < IA_CSS_PIPE_ID_NUM);
-
-	IA_CSS_ENTER_PRIVATE("config=%p", config);
-	ia_css_dp_debug_dtrace(config, IA_CSS_DEBUG_TRACE_PRIVATE);
-	if (pipe->mode < IA_CSS_PIPE_ID_NUM) {
-		params->pipe_dp_config[pipe->mode] = *config;
-		params->pipe_dpc_config_changed[pipe->mode] = true;
-	}
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-#endif
-
-static void
-sh_css_get_dp_config(const struct ia_css_pipe *pipe,
-			const struct ia_css_isp_parameters *params,
-			struct ia_css_dp_config *config)
-{
-	if (config == NULL)
-		return;
-
-	assert(params != NULL);
-	assert(pipe != NULL);
-	IA_CSS_ENTER_PRIVATE("config=%p", config);
-
-	*config = params->pipe_dp_config[pipe->mode];
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_set_nr_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_nr_config *config)
-{
-	if (config == NULL)
-		return;
-	assert(params != NULL);
-
-	IA_CSS_ENTER_PRIVATE("config=%p", config);
-
-	ia_css_nr_debug_dtrace(config, IA_CSS_DEBUG_TRACE_PRIVATE);
-	params->nr_config = *config;
-	params->yee_config.nr = *config;
-	params->config_changed[IA_CSS_NR_ID]  = true;
-	params->config_changed[IA_CSS_YEE_ID] = true;
-	params->config_changed[IA_CSS_BNR_ID] = true;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_set_ee_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_ee_config *config)
-{
-	if (config == NULL)
-		return;
-	assert(params != NULL);
-
-	IA_CSS_ENTER_PRIVATE("config=%p", config);
-	ia_css_ee_debug_dtrace(config, IA_CSS_DEBUG_TRACE_PRIVATE);
-
-	params->ee_config = *config;
-	params->yee_config.ee = *config;
-	params->config_changed[IA_CSS_YEE_ID] = true;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_get_ee_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_ee_config *config)
-{
-	if (config == NULL)
-		return;
-
-	IA_CSS_ENTER_PRIVATE("config=%p", config);
-
-	assert(params != NULL);
-	*config = params->ee_config;
-
-	ia_css_ee_debug_dtrace(config, IA_CSS_DEBUG_TRACE_PRIVATE);
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_set_pipe_dvs_6axis_config(const struct ia_css_pipe *pipe,
-							struct ia_css_isp_parameters *params,
-							const struct ia_css_dvs_6axis_config  *dvs_config)
-{
-	if (dvs_config == NULL)
-		return;
-	assert(params != NULL);
-	assert(pipe != NULL);
-	assert(dvs_config->height_y == dvs_config->height_uv);
-	assert((dvs_config->width_y - 1) == 2 * (dvs_config->width_uv - 1));
-	assert(pipe->mode < IA_CSS_PIPE_ID_NUM);
-
-	IA_CSS_ENTER_PRIVATE("dvs_config=%p", dvs_config);
-
-	copy_dvs_6axis_table(params->pipe_dvs_6axis_config[pipe->mode], dvs_config);
-
-#if !defined(HAS_NO_DVS_6AXIS_CONFIG_UPDATE)
-	params->pipe_dvs_6axis_config_changed[pipe->mode] = true;
-#endif
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_get_pipe_dvs_6axis_config(const struct ia_css_pipe *pipe,
-				const struct ia_css_isp_parameters *params,
-				struct ia_css_dvs_6axis_config *dvs_config)
-{
-	if (dvs_config == NULL)
-		return;
-	assert(params != NULL);
-	assert(pipe != NULL);
-	assert(dvs_config->height_y == dvs_config->height_uv);
-	assert((dvs_config->width_y - 1) == 2 * dvs_config->width_uv - 1);
-
-	IA_CSS_ENTER_PRIVATE("dvs_config=%p", dvs_config);
-
-	if ((pipe->mode < IA_CSS_PIPE_ID_NUM) &&
-	    (dvs_config->width_y == params->pipe_dvs_6axis_config[pipe->mode]->width_y) &&
-	    (dvs_config->height_y == params->pipe_dvs_6axis_config[pipe->mode]->height_y) &&
-	    (dvs_config->width_uv == params->pipe_dvs_6axis_config[pipe->mode]->width_uv) &&
-	    (dvs_config->height_uv == params->pipe_dvs_6axis_config[pipe->mode]->height_uv) &&
-	     dvs_config->xcoords_y &&
-	     dvs_config->ycoords_y &&
-	     dvs_config->xcoords_uv &&
-	     dvs_config->ycoords_uv)
-	{
-		copy_dvs_6axis_table(dvs_config, params->pipe_dvs_6axis_config[pipe->mode]);
-	}
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_set_baa_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_aa_config *config)
-{
-	if (config == NULL)
-		return;
-	assert(params != NULL);
-
-	IA_CSS_ENTER_PRIVATE("config=%p", config);
-
-	params->bds_config = *config;
-	params->config_changed[IA_CSS_BDS_ID] = true;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_get_baa_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_aa_config *config)
-{
-	if (config == NULL)
-		return;
-	assert(params != NULL);
-
-	IA_CSS_ENTER_PRIVATE("config=%p", config);
-
-	*config = params->bds_config;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_set_dz_config(struct ia_css_isp_parameters *params,
-			const struct ia_css_dz_config *config)
-{
-	if (config == NULL)
-		return;
-	assert(params != NULL);
-
-	IA_CSS_ENTER_PRIVATE("dx=%d, dy=%d", config->dx, config->dy);
-
-	assert(config->dx <= HRT_GDC_N);
-	assert(config->dy <= HRT_GDC_N);
-
-	params->dz_config = *config;
-	params->dz_config_changed = true;
-	/* JK: Why isp params changed?? */
-	params->isp_params_changed = true;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_get_dz_config(const struct ia_css_isp_parameters *params,
-			struct ia_css_dz_config *config)
-{
-	if (config == NULL)
-		return;
-	assert(params != NULL);
-
-	IA_CSS_ENTER_PRIVATE("config=%p", config);
-
-	*config = params->dz_config;
-
-	IA_CSS_LEAVE_PRIVATE("dx=%d, dy=%d", config->dx, config->dy);
-}
-
-static void
-sh_css_set_motion_vector(struct ia_css_isp_parameters *params,
-			const struct ia_css_vector *motion)
-{
-	if (motion == NULL)
-		return;
-	assert(params != NULL);
-
-	IA_CSS_ENTER_PRIVATE("x=%d, y=%d", motion->x, motion->y);
-
-	params->motion_config = *motion;
-	/* JK: Why do isp params change? */
-	params->motion_config_changed = true;
-	params->isp_params_changed = true;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-sh_css_get_motion_vector(const struct ia_css_isp_parameters *params,
-			struct ia_css_vector *motion)
-{
-	if (motion == NULL)
-		return;
-	assert(params != NULL);
-
-	IA_CSS_ENTER_PRIVATE("motion=%p", motion);
-
-	*motion = params->motion_config;
-
-	IA_CSS_LEAVE_PRIVATE("x=%d, y=%d", motion->x, motion->y);
-}
-
-struct ia_css_isp_config *
-sh_css_pipe_isp_config_get(struct ia_css_pipe *pipe)
-{
-	if (pipe == NULL)
-	{
-		IA_CSS_ERROR("pipe=%p", NULL);
-		return NULL;
-	}
-	return pipe->config.p_isp_config;
-}
-
-enum ia_css_err
-ia_css_stream_set_isp_config(
-	struct ia_css_stream *stream,
-	const struct ia_css_isp_config *config)
-{
-	return ia_css_stream_set_isp_config_on_pipe(stream, config, NULL);
-}
-
-enum ia_css_err
-ia_css_stream_set_isp_config_on_pipe(
-	struct ia_css_stream *stream,
-	const struct ia_css_isp_config *config,
-	struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	if ((stream == NULL) || (config == NULL))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	IA_CSS_ENTER("stream=%p, config=%p, pipe=%p", stream, config, pipe);
-
-#if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-	if (config->output_frame)
-		err = sh_css_set_per_frame_isp_config_on_pipe(stream, config, pipe);
-	else
-#endif
-		err = sh_css_set_global_isp_config_on_pipe(stream->pipes[0], config, pipe);
-
-	IA_CSS_LEAVE_ERR(err);
-	return err;
-}
-
-enum ia_css_err
-ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe,
-	struct ia_css_isp_config *config)
-{
-	struct ia_css_pipe *pipe_in = pipe;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER("pipe=%p", pipe);
-
-	if ((pipe == NULL) || (pipe->stream == NULL))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "config=%p\n", config);
-
-#if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-	if (config->output_frame)
-		err = sh_css_set_per_frame_isp_config_on_pipe(pipe->stream, config, pipe);
-	else
-#endif
-		err = sh_css_set_global_isp_config_on_pipe(pipe, config, pipe_in);
-	IA_CSS_LEAVE_ERR(err);
-	return err;
-}
-
-static enum ia_css_err
-sh_css_set_global_isp_config_on_pipe(
-	struct ia_css_pipe *curr_pipe,
-	const struct ia_css_isp_config *config,
-	struct ia_css_pipe *pipe)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	enum ia_css_err err1 = IA_CSS_SUCCESS;
-	enum ia_css_err err2 = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER_PRIVATE("stream=%p, config=%p, pipe=%p", curr_pipe, config, pipe);
-
-	err1 = sh_css_init_isp_params_from_config(curr_pipe, curr_pipe->stream->isp_params_configs, config, pipe);
-
-	/* Now commit all changes to the SP */
-	err2 = sh_css_param_update_isp_params(curr_pipe, curr_pipe->stream->isp_params_configs, sh_css_sp_is_running(), pipe);
-
-	/* The following code is intentional. The sh_css_init_isp_params_from_config interface
-	 * throws an error when both DPC and BDS is enabled. The CSS API must pass this error
-	 * information to the caller, ie. the host. We do not return this error immediately,
-	 * but instead continue with updating the ISP params to enable testing of features
-	 * which are currently in TR phase. */
-
-	err = (err1 != IA_CSS_SUCCESS ) ? err1 : ((err2 != IA_CSS_SUCCESS) ? err2 : err);
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-#if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-static enum ia_css_err
-sh_css_set_per_frame_isp_config_on_pipe(
-	struct ia_css_stream *stream,
-	const struct ia_css_isp_config *config,
-	struct ia_css_pipe *pipe)
-{
-	unsigned i;
-	bool per_frame_config_created = false;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	enum ia_css_err err1 = IA_CSS_SUCCESS;
-	enum ia_css_err err2 = IA_CSS_SUCCESS;
-	enum ia_css_err err3 = IA_CSS_SUCCESS;
-
-	struct sh_css_ddr_address_map *ddr_ptrs;
-	struct sh_css_ddr_address_map_size *ddr_ptrs_size;
-	struct ia_css_isp_parameters *params;
-
-	IA_CSS_ENTER_PRIVATE("stream=%p, config=%p, pipe=%p", stream, config, pipe);
-
-	if (!pipe) {
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		goto exit;
-	}
-
-	/* create per-frame ISP params object with default values
-	 * from stream->isp_params_configs if one doesn't already exist
-	*/
-	if (!stream->per_frame_isp_params_configs)
-	{
-		err = sh_css_create_isp_params(stream,
-					       &stream->per_frame_isp_params_configs);
-		if(err != IA_CSS_SUCCESS)
-			goto exit;
-		per_frame_config_created = true;
-	}
-
-	params = stream->per_frame_isp_params_configs;
-
-	/* update new ISP params object with the new config */
-	if (!sh_css_init_isp_params_from_global(stream, params, false, pipe)) {
-		err1 = IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	err2 = sh_css_init_isp_params_from_config(stream->pipes[0], params, config, pipe);
-
-	if (per_frame_config_created)
-	{
-		ddr_ptrs = &params->ddr_ptrs;
-		ddr_ptrs_size = &params->ddr_ptrs_size;
-		/* create per pipe reference to general ddr_ptrs */
-		for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) {
-			ref_sh_css_ddr_address_map(ddr_ptrs, &params->pipe_ddr_ptrs[i]);
-			params->pipe_ddr_ptrs_size[i] = *ddr_ptrs_size;
-		}
-	}
-
-	/* now commit to ddr */
-	err3 = sh_css_param_update_isp_params(stream->pipes[0], params, sh_css_sp_is_running(), pipe);
-
-	/* The following code is intentional. The sh_css_init_sp_params_from_config and
-	 * sh_css_init_isp_params_from_config throws an error when both DPC and BDS is enabled.
-	 * The CSS API must pass this error information to the caller, ie. the host.
-	 * We do not return this error immediately, but instead continue with updating the ISP params
-	 *  to enable testing of features which are currently in TR phase. */
-	err = (err1 != IA_CSS_SUCCESS) ? err1 :
-		(err2 != IA_CSS_SUCCESS) ? err2 :
-		(err3 != IA_CSS_SUCCESS) ? err3 : err;
-exit:
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-#endif
-
-static enum ia_css_err
-sh_css_init_isp_params_from_config(struct ia_css_pipe *pipe,
-		struct ia_css_isp_parameters *params,
-		const struct ia_css_isp_config *config,
-		struct ia_css_pipe *pipe_in)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	bool is_dp_10bpp = true;
-	assert(pipe != NULL);
-
-	IA_CSS_ENTER_PRIVATE("pipe=%p, config=%p, params=%p", pipe, config, params);
-
-	ia_css_set_configs(params, config);
-
-
-	sh_css_set_nr_config(params, config->nr_config);
-	sh_css_set_ee_config(params, config->ee_config);
-	sh_css_set_baa_config(params, config->baa_config);
-	if ((pipe->mode < IA_CSS_PIPE_ID_NUM) &&
-			(params->pipe_dvs_6axis_config[pipe->mode]))
-			sh_css_set_pipe_dvs_6axis_config(pipe, params, config->dvs_6axis_config);
-	sh_css_set_dz_config(params, config->dz_config);
-	sh_css_set_motion_vector(params, config->motion_vector);
-	sh_css_update_shading_table_status(pipe_in, params);
-	sh_css_set_shading_table(pipe->stream, params, config->shading_table);
-	sh_css_set_morph_table(params, config->morph_table);
-	sh_css_set_macc_table(params, config->macc_table);
-	sh_css_set_gamma_table(params, config->gamma_table);
-	sh_css_set_ctc_table(params, config->ctc_table);
-/* ------ deprecated(bz675) : from ------ */
-	sh_css_set_shading_settings(params, config->shading_settings);
-/* ------ deprecated(bz675) : to ------ */
-
-	params->dis_coef_table_changed = (config->dvs_coefs != NULL);
-	params->dvs2_coef_table_changed = (config->dvs2_coefs != NULL);
-
-	params->output_frame = config->output_frame;
-	params->isp_parameters_id = config->isp_config_id;
-#ifdef ISP2401
-	/* Currently we do not offer CSS interface to set different
-	 * configurations for DPC, i.e. depending on DPC being enabled
-	 * before (NORM+OBC) or after. The folllowing code to set the
-	 * DPC configuration should be updated when this interface is made
-	 * available */
-	sh_css_set_dp_config(pipe, params, config->dp_config);
-	ia_css_set_param_exceptions(pipe, params);
-#endif
-
-	if (IA_CSS_SUCCESS ==
-		sh_css_select_dp_10bpp_config(pipe, &is_dp_10bpp)) {
-		/* return an error when both DPC and BDS is enabled by the
-		 * user. */
-		/* we do not exit from this point immediately to allow internal
-		 * firmware feature testing. */
-		if(is_dp_10bpp) {
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		}
-	} else {
-		err = IA_CSS_ERR_INTERNAL_ERROR;
-		goto exit;
-	}
-
-#ifndef ISP2401
-	ia_css_set_param_exceptions(pipe, params);
-#endif
-exit:
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-void
-ia_css_stream_get_isp_config(
-	const struct ia_css_stream *stream,
-	struct ia_css_isp_config *config)
-{
-	IA_CSS_ENTER("void");
-	ia_css_pipe_get_isp_config(stream->pipes[0], config);
-	IA_CSS_LEAVE("void");
-}
-
-void
-ia_css_pipe_get_isp_config(struct ia_css_pipe *pipe,
-						   struct ia_css_isp_config *config)
-{
-	struct ia_css_isp_parameters *params = NULL;
-
-	assert(config != NULL);
-
-	IA_CSS_ENTER("config=%p", config);
-
-	params = pipe->stream->isp_params_configs;
-	assert(params != NULL);
-
-	ia_css_get_configs(params, config);
-
-	sh_css_get_ee_config(params, config->ee_config);
-	sh_css_get_baa_config(params, config->baa_config);
-	sh_css_get_pipe_dvs_6axis_config(pipe, params, config->dvs_6axis_config);
-	sh_css_get_dp_config(pipe, params, config->dp_config);
-	sh_css_get_macc_table(params, config->macc_table);
-	sh_css_get_gamma_table(params, config->gamma_table);
-	sh_css_get_ctc_table(params, config->ctc_table);
-	sh_css_get_dz_config(params, config->dz_config);
-	sh_css_get_motion_vector(params, config->motion_vector);
-/* ------ deprecated(bz675) : from ------ */
-	sh_css_get_shading_settings(params, config->shading_settings);
-/* ------ deprecated(bz675) : to ------ */
-
-	config->output_frame = params->output_frame;
-	config->isp_config_id = params->isp_parameters_id;
-
-	IA_CSS_LEAVE("void");
-}
-
-#ifndef ISP2401
-/*
- * coding style says the return of "mmgr_NULL" is the error signal
- *
- * Deprecated: Implement mmgr_realloc()
- */
-static bool realloc_isp_css_mm_buf(
-	hrt_vaddress *curr_buf,
-	size_t *curr_size,
-	size_t needed_size,
-	bool force,
-	enum ia_css_err *err,
-	uint16_t mmgr_attribute)
-{
-	int32_t id;
-
-	*err = IA_CSS_SUCCESS;
-	/* Possible optimization: add a function sh_css_isp_css_mm_realloc()
-	 * and implement on top of hmm. */
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	if (!force && *curr_size >= needed_size) {
-		IA_CSS_LEAVE_PRIVATE("false");
-		return false;
-	}
-	/* don't reallocate if single ref to buffer and same size */
-	if (*curr_size == needed_size && ia_css_refcount_is_single(*curr_buf)) {
-		IA_CSS_LEAVE_PRIVATE("false");
-		return false;
-	}
-
-	id = IA_CSS_REFCOUNT_PARAM_BUFFER;
-	ia_css_refcount_decrement(id, *curr_buf);
-	*curr_buf = ia_css_refcount_increment(id, mmgr_alloc_attr(needed_size,
-							mmgr_attribute));
-
-	if (!*curr_buf) {
-		*err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		*curr_size = 0;
-	} else {
-		*curr_size = needed_size;
-	}
-	IA_CSS_LEAVE_PRIVATE("true");
-	return true;
-}
-
-static bool reallocate_buffer(
-	hrt_vaddress *curr_buf,
-	size_t *curr_size,
-	size_t needed_size,
-	bool force,
-	enum ia_css_err *err)
-{
-	bool ret;
-	uint16_t	mmgr_attribute = MMGR_ATTRIBUTE_DEFAULT;
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	ret = realloc_isp_css_mm_buf(curr_buf,
-		curr_size, needed_size, force, err, mmgr_attribute);
-
-	IA_CSS_LEAVE_PRIVATE("ret=%d", ret);
-	return ret;
-}
-
-#endif
-
-struct ia_css_isp_3a_statistics *
-ia_css_isp_3a_statistics_allocate(const struct ia_css_3a_grid_info *grid)
-{
-	struct ia_css_isp_3a_statistics *me;
-
-	IA_CSS_ENTER("grid=%p", grid);
-
-	assert(grid != NULL);
-
-	/* MW: Does "grid->enable" also control the histogram output ?? */
-	if (!grid->enable)
-		return NULL;
-
-	me = sh_css_calloc(1, sizeof(*me));
-	if (!me)
-		goto err;
-
-	if (grid->use_dmem) {
-		me->dmem_size = sizeof(struct ia_css_3a_output) *
-				       grid->aligned_width *
-				       grid->aligned_height;
-	} else {
-		me->vmem_size = ISP_S3ATBL_HI_LO_STRIDE_BYTES *
-				grid->aligned_height;
-	}
-#if !defined(HAS_NO_HMEM)
-	me->hmem_size = sizeof_hmem(HMEM0_ID);
-#endif
-
-	/* All subsections need to be aligned to the system bus width */
-	me->dmem_size = CEIL_MUL(me->dmem_size, HIVE_ISP_DDR_WORD_BYTES);
-	me->vmem_size = CEIL_MUL(me->vmem_size, HIVE_ISP_DDR_WORD_BYTES);
-	me->hmem_size = CEIL_MUL(me->hmem_size, HIVE_ISP_DDR_WORD_BYTES);
-
-	me->size = me->dmem_size + me->vmem_size * 2 + me->hmem_size;
-	me->data_ptr = mmgr_malloc(me->size);
-	if (me->data_ptr == mmgr_NULL) {
-		sh_css_free(me);
-		me = NULL;
-		goto err;
-	}
-	if (me->dmem_size)
-		me->data.dmem.s3a_tbl = me->data_ptr;
-	if (me->vmem_size) {
-		me->data.vmem.s3a_tbl_hi = me->data_ptr + me->dmem_size;
-		me->data.vmem.s3a_tbl_lo = me->data_ptr + me->dmem_size + me->vmem_size;
-	}
-	if (me->hmem_size)
-		me->data_hmem.rgby_tbl = me->data_ptr + me->dmem_size + 2 * me->vmem_size;
-
-
-err:
-	IA_CSS_LEAVE("return=%p", me);
-	return me;
-}
-
-void
-ia_css_isp_3a_statistics_free(struct ia_css_isp_3a_statistics *me)
-{
-	if (me != NULL) {
-		hmm_free(me->data_ptr);
-		sh_css_free(me);
-	}
-}
-
-struct ia_css_isp_skc_dvs_statistics *ia_css_skc_dvs_statistics_allocate(void)
-{
-	return NULL;
-}
-
-struct ia_css_metadata *
-ia_css_metadata_allocate(const struct ia_css_metadata_info *metadata_info)
-{
-	struct ia_css_metadata *md = NULL;
-
-	IA_CSS_ENTER("");
-
-	if (metadata_info->size == 0)
-		return NULL;
-
-	md = sh_css_malloc(sizeof(*md));
-	if (md == NULL)
-		goto error;
-
-	md->info = *metadata_info;
-	md->exp_id = 0;
-	md->address = mmgr_malloc(metadata_info->size);
-	if (md->address == mmgr_NULL)
-		goto error;
-
-	IA_CSS_LEAVE("return=%p", md);
-	return md;
-
-error:
-	ia_css_metadata_free(md);
-	IA_CSS_LEAVE("return=%p", NULL);
-	return NULL;
-}
-
-void
-ia_css_metadata_free(struct ia_css_metadata *me)
-{
-	if (me != NULL) {
-		/* The enter and leave macros are placed inside
-		 * the condition to avoid false logging of metadata
-		 * free events when metadata is disabled.
-		 * We found this to be confusing during development
-		 * and debugging. */
-		IA_CSS_ENTER("me=%p", me);
-		hmm_free(me->address);
-		sh_css_free(me);
-		IA_CSS_LEAVE("void");
-	}
-}
-
-void
-ia_css_metadata_free_multiple(unsigned int num_bufs, struct ia_css_metadata **bufs)
-{
-	unsigned int i;
-
-	if (bufs != NULL) {
-		for (i = 0; i < num_bufs; i++)
-			ia_css_metadata_free(bufs[i]);
-	}
-}
-
-static unsigned g_param_buffer_dequeue_count = 0;
-static unsigned g_param_buffer_enqueue_count = 0;
-
-enum ia_css_err
-ia_css_stream_isp_parameters_init(struct ia_css_stream *stream)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	unsigned i;
-	struct sh_css_ddr_address_map *ddr_ptrs;
-	struct sh_css_ddr_address_map_size *ddr_ptrs_size;
-	struct ia_css_isp_parameters *params;
-
-	assert(stream != NULL);
-	IA_CSS_ENTER_PRIVATE("void");
-
-	if (stream == NULL) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	/* TMP: tracking of paramsets */
-	g_param_buffer_dequeue_count = 0;
-	g_param_buffer_enqueue_count = 0;
-
-	stream->per_frame_isp_params_configs = NULL;
-	err = sh_css_create_isp_params(stream,
-				       &stream->isp_params_configs);
-	if(err != IA_CSS_SUCCESS)
-		 goto ERR;
-
-	params = stream->isp_params_configs;
-	if (!sh_css_init_isp_params_from_global(stream, params, true, NULL)) {
-		/* we do not return the error immediately to enable internal
-		 * firmware feature testing */
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	ddr_ptrs = &params->ddr_ptrs;
-	ddr_ptrs_size = &params->ddr_ptrs_size;
-
-	/* create per pipe reference to general ddr_ptrs */
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) {
-		ref_sh_css_ddr_address_map(ddr_ptrs, &params->pipe_ddr_ptrs[i]);
-		params->pipe_ddr_ptrs_size[i] = *ddr_ptrs_size;
-	}
-
-ERR:
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static void
-ia_css_set_sdis_config(
-	struct ia_css_isp_parameters *params,
-	const struct ia_css_dvs_coefficients *dvs_coefs)
-{
-	ia_css_set_sdis_horicoef_config(params, dvs_coefs);
-	ia_css_set_sdis_vertcoef_config(params, dvs_coefs);
-	ia_css_set_sdis_horiproj_config(params, dvs_coefs);
-	ia_css_set_sdis_vertproj_config(params, dvs_coefs);
-}
-
-static void
-ia_css_set_sdis2_config(
-	struct ia_css_isp_parameters *params,
-	const struct ia_css_dvs2_coefficients *dvs2_coefs)
-{
-	ia_css_set_sdis2_horicoef_config(params, dvs2_coefs);
-	ia_css_set_sdis2_vertcoef_config(params, dvs2_coefs);
-	ia_css_set_sdis2_horiproj_config(params, dvs2_coefs);
-	ia_css_set_sdis2_vertproj_config(params, dvs2_coefs);
-}
-
-static enum ia_css_err
-sh_css_create_isp_params(struct ia_css_stream *stream,
-			 struct ia_css_isp_parameters **isp_params_out)
-{
-	bool succ = true;
-	unsigned i;
-	struct sh_css_ddr_address_map *ddr_ptrs;
-	struct sh_css_ddr_address_map_size *ddr_ptrs_size;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	size_t params_size;
-	struct ia_css_isp_parameters *params =
-				sh_css_malloc(sizeof(struct ia_css_isp_parameters));
-
-	if (!params)
-	{
-		*isp_params_out = NULL;
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		IA_CSS_ERROR("%s:%d error: cannot allocate memory", __FILE__, __LINE__);
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	} else {
-		memset(params, 0, sizeof(struct ia_css_isp_parameters));
-	}
-
-	ddr_ptrs = &params->ddr_ptrs;
-	ddr_ptrs_size = &params->ddr_ptrs_size;
-
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) {
-		memset(&params->pipe_ddr_ptrs[i], 0,
-			sizeof(params->pipe_ddr_ptrs[i]));
-		memset(&params->pipe_ddr_ptrs_size[i], 0,
-			sizeof(params->pipe_ddr_ptrs_size[i]));
-	}
-
-	memset(ddr_ptrs, 0, sizeof(*ddr_ptrs));
-	memset(ddr_ptrs_size, 0, sizeof(*ddr_ptrs_size));
-
-	params_size = sizeof(params->uds);
-	ddr_ptrs_size->isp_param = params_size;
-	ddr_ptrs->isp_param =
-				ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_BUFFER,
-					mmgr_malloc(params_size));
-	succ &= (ddr_ptrs->isp_param != mmgr_NULL);
-
-	ddr_ptrs_size->macc_tbl = sizeof(struct ia_css_macc_table);
-	ddr_ptrs->macc_tbl =
-				ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_BUFFER,
-					mmgr_malloc(sizeof(struct ia_css_macc_table)));
-	succ &= (ddr_ptrs->macc_tbl != mmgr_NULL);
-
-	*isp_params_out = params;
-	return err;
-}
-
-static bool
-sh_css_init_isp_params_from_global(struct ia_css_stream *stream,
-		struct ia_css_isp_parameters *params,
-		bool use_default_config,
-		struct ia_css_pipe *pipe_in)
-{
-	bool retval = true;
-	int i = 0;
-	bool is_dp_10bpp = true;
-	unsigned isp_pipe_version = ia_css_pipe_get_isp_pipe_version(stream->pipes[0]);
-	struct ia_css_isp_parameters *stream_params = stream->isp_params_configs;
-
-	if (!use_default_config && !stream_params) {
-		retval = false;
-		goto exit;
-	}
-
-	params->output_frame = NULL;
-	params->isp_parameters_id = 0;
-
-	if (use_default_config)
-	{
-		ia_css_set_xnr3_config(params, &default_xnr3_config);
-
-		sh_css_set_nr_config(params, &default_nr_config);
-		sh_css_set_ee_config(params, &default_ee_config);
-		if (isp_pipe_version == SH_CSS_ISP_PIPE_VERSION_1)
-			sh_css_set_macc_table(params, &default_macc_table);
-		else if (isp_pipe_version == SH_CSS_ISP_PIPE_VERSION_2_2)
-			sh_css_set_macc_table(params, &default_macc2_table);
-		sh_css_set_gamma_table(params, &default_gamma_table);
-		sh_css_set_ctc_table(params, &default_ctc_table);
-		sh_css_set_baa_config(params, &default_baa_config);
-		sh_css_set_dz_config(params, &default_dz_config);
-/* ------ deprecated(bz675) : from ------ */
-		sh_css_set_shading_settings(params, &default_shading_settings);
-/* ------ deprecated(bz675) : to ------ */
-
-		ia_css_set_s3a_config(params, &default_3a_config);
-		ia_css_set_wb_config(params, &default_wb_config);
-		ia_css_set_csc_config(params, &default_cc_config);
-		ia_css_set_tnr_config(params, &default_tnr_config);
-		ia_css_set_ob_config(params, &default_ob_config);
-		ia_css_set_dp_config(params, &default_dp_config);
-#ifndef ISP2401
-		ia_css_set_param_exceptions(pipe_in, params);
-#else
-
-		for (i = 0; i < stream->num_pipes; i++) {
-			if (IA_CSS_SUCCESS == sh_css_select_dp_10bpp_config(stream->pipes[i], &is_dp_10bpp)) {
-				/* set the return value as false if both DPC and
-				 * BDS is enabled by the user. But we do not return
-				 * the value immediately to enable internal firmware
-				 * feature testing. */
-				if(is_dp_10bpp) {
-					sh_css_set_dp_config(stream->pipes[i], params, &default_dp_10bpp_config);
-				} else {
-					sh_css_set_dp_config(stream->pipes[i], params, &default_dp_config);
-				}
-			} else {
-				retval = false;
-				goto exit;
-			}
-
-			ia_css_set_param_exceptions(stream->pipes[i], params);
-		}
-
-#endif
-		ia_css_set_de_config(params, &default_de_config);
-		ia_css_set_gc_config(params, &default_gc_config);
-		ia_css_set_anr_config(params, &default_anr_config);
-		ia_css_set_anr2_config(params, &default_anr_thres);
-		ia_css_set_ce_config(params, &default_ce_config);
-		ia_css_set_xnr_table_config(params, &default_xnr_table);
-		ia_css_set_ecd_config(params, &default_ecd_config);
-		ia_css_set_ynr_config(params, &default_ynr_config);
-		ia_css_set_fc_config(params, &default_fc_config);
-		ia_css_set_cnr_config(params, &default_cnr_config);
-		ia_css_set_macc_config(params, &default_macc_config);
-		ia_css_set_ctc_config(params, &default_ctc_config);
-		ia_css_set_aa_config(params, &default_aa_config);
-		ia_css_set_r_gamma_config(params, &default_r_gamma_table);
-		ia_css_set_g_gamma_config(params, &default_g_gamma_table);
-		ia_css_set_b_gamma_config(params, &default_b_gamma_table);
-		ia_css_set_yuv2rgb_config(params, &default_yuv2rgb_cc_config);
-		ia_css_set_rgb2yuv_config(params, &default_rgb2yuv_cc_config);
-		ia_css_set_xnr_config(params, &default_xnr_config);
-		ia_css_set_sdis_config(params, &default_sdis_config);
-		ia_css_set_sdis2_config(params, &default_sdis2_config);
-		ia_css_set_formats_config(params, &default_formats_config);
-
-		params->fpn_config.data = NULL;
-		params->config_changed[IA_CSS_FPN_ID] = true;
-		params->fpn_config.enabled = 0;
-
-		params->motion_config = default_motion_config;
-		params->motion_config_changed = true;
-
-		params->morph_table = NULL;
-		params->morph_table_changed = true;
-
-		params->sc_table = NULL;
-		params->sc_table_changed = true;
-		params->sc_table_dirty = false;
-		params->sc_table_last_pipe_num = 0;
-
-		ia_css_sdis2_clear_coefficients(&params->dvs2_coefs);
-		params->dvs2_coef_table_changed = true;
-
-		ia_css_sdis_clear_coefficients(&params->dvs_coefs);
-		params->dis_coef_table_changed = true;
-#ifdef ISP2401
-		ia_css_tnr3_set_default_config(&params->tnr3_config);
-#endif
-	}
-	else
-	{
-		ia_css_set_xnr3_config(params, &stream_params->xnr3_config);
-
-		sh_css_set_nr_config(params, &stream_params->nr_config);
-		sh_css_set_ee_config(params, &stream_params->ee_config);
-		if (isp_pipe_version == SH_CSS_ISP_PIPE_VERSION_1)
-			sh_css_set_macc_table(params, &stream_params->macc_table);
-		else if (isp_pipe_version == SH_CSS_ISP_PIPE_VERSION_2_2)
-			sh_css_set_macc_table(params, &stream_params->macc_table);
-		sh_css_set_gamma_table(params, &stream_params->gc_table);
-		sh_css_set_ctc_table(params, &stream_params->ctc_table);
-		sh_css_set_baa_config(params, &stream_params->bds_config);
-		sh_css_set_dz_config(params, &stream_params->dz_config);
-/* ------ deprecated(bz675) : from ------ */
-		sh_css_set_shading_settings(params, &stream_params->shading_settings);
-/* ------ deprecated(bz675) : to ------ */
-
-		ia_css_set_s3a_config(params, &stream_params->s3a_config);
-		ia_css_set_wb_config(params, &stream_params->wb_config);
-		ia_css_set_csc_config(params, &stream_params->cc_config);
-		ia_css_set_tnr_config(params, &stream_params->tnr_config);
-		ia_css_set_ob_config(params, &stream_params->ob_config);
-		ia_css_set_dp_config(params, &stream_params->dp_config);
-		ia_css_set_de_config(params, &stream_params->de_config);
-		ia_css_set_gc_config(params, &stream_params->gc_config);
-		ia_css_set_anr_config(params, &stream_params->anr_config);
-		ia_css_set_anr2_config(params, &stream_params->anr_thres);
-		ia_css_set_ce_config(params, &stream_params->ce_config);
-		ia_css_set_xnr_table_config(params, &stream_params->xnr_table);
-		ia_css_set_ecd_config(params, &stream_params->ecd_config);
-		ia_css_set_ynr_config(params, &stream_params->ynr_config);
-		ia_css_set_fc_config(params, &stream_params->fc_config);
-		ia_css_set_cnr_config(params, &stream_params->cnr_config);
-		ia_css_set_macc_config(params, &stream_params->macc_config);
-		ia_css_set_ctc_config(params, &stream_params->ctc_config);
-		ia_css_set_aa_config(params, &stream_params->aa_config);
-		ia_css_set_r_gamma_config(params, &stream_params->r_gamma_table);
-		ia_css_set_g_gamma_config(params, &stream_params->g_gamma_table);
-		ia_css_set_b_gamma_config(params, &stream_params->b_gamma_table);
-		ia_css_set_yuv2rgb_config(params, &stream_params->yuv2rgb_cc_config);
-		ia_css_set_rgb2yuv_config(params, &stream_params->rgb2yuv_cc_config);
-		ia_css_set_xnr_config(params, &stream_params->xnr_config);
-		ia_css_set_formats_config(params, &stream_params->formats_config);
-
-		for (i = 0; i < stream->num_pipes; i++) {
-			if (IA_CSS_SUCCESS ==
-				sh_css_select_dp_10bpp_config(stream->pipes[i], &is_dp_10bpp)) {
-				/* set the return value as false if both DPC and
-				 * BDS is enabled by the user. But we do not return
-				 * the value immediately to enable internal firmware
-				 * feature testing. */
-#ifndef ISP2401
-				retval = !is_dp_10bpp;
-#else
-				if (is_dp_10bpp) {
-					retval = false;
-				}
-			} else {
-				retval = false;
-				goto exit;
-			}
-			if (stream->pipes[i]->mode < IA_CSS_PIPE_ID_NUM) {
-				sh_css_set_dp_config(stream->pipes[i], params,
-					&stream_params->pipe_dp_config[stream->pipes[i]->mode]);
-				ia_css_set_param_exceptions(stream->pipes[i], params);
-#endif
-			} else {
-				retval = false;
-				goto exit;
-			}
-		}
-
-#ifndef ISP2401
-		ia_css_set_param_exceptions(pipe_in, params);
-
-#endif
-		params->fpn_config.data = stream_params->fpn_config.data;
-		params->config_changed[IA_CSS_FPN_ID] = stream_params->config_changed[IA_CSS_FPN_ID];
-		params->fpn_config.enabled = stream_params->fpn_config.enabled;
-
-		sh_css_set_motion_vector(params, &stream_params->motion_config);
-		sh_css_set_morph_table(params, stream_params->morph_table);
-
-		if (stream_params->sc_table) {
-			sh_css_update_shading_table_status(pipe_in, params);
-			sh_css_set_shading_table(stream, params, stream_params->sc_table);
-		}
-		else {
-			params->sc_table = NULL;
-			params->sc_table_changed = true;
-			params->sc_table_dirty = false;
-			params->sc_table_last_pipe_num = 0;
-		}
-
-		/* Only IA_CSS_PIPE_ID_VIDEO & IA_CSS_PIPE_ID_CAPTURE will support dvs_6axis_config*/
-		for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) {
-			if (stream_params->pipe_dvs_6axis_config[i]) {
-				if (params->pipe_dvs_6axis_config[i]) {
-					copy_dvs_6axis_table(params->pipe_dvs_6axis_config[i],
-								stream_params->pipe_dvs_6axis_config[i]);
-				} else {
-					params->pipe_dvs_6axis_config[i] =
-						generate_dvs_6axis_table_from_config(stream_params->pipe_dvs_6axis_config[i]);
-				}
-			}
-		}
-		ia_css_set_sdis_config(params, &stream_params->dvs_coefs);
-		params->dis_coef_table_changed = stream_params->dis_coef_table_changed;
-
-		ia_css_set_sdis2_config(params, &stream_params->dvs2_coefs);
-		params->dvs2_coef_table_changed = stream_params->dvs2_coef_table_changed;
-		params->sensor_binning = stream_params->sensor_binning;
-	}
-
-exit:
-	return retval;
-}
-
-enum ia_css_err
-sh_css_params_init(void)
-{
-	int i, p;
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	/* TMP: tracking of paramsets */
-	g_param_buffer_dequeue_count = 0;
-	g_param_buffer_enqueue_count = 0;
-
-	for (p = 0; p < IA_CSS_PIPE_ID_NUM; p++) {
-		for (i = 0; i < SH_CSS_MAX_STAGES; i++) {
-			xmem_sp_stage_ptrs[p][i] =
-					ia_css_refcount_increment(-1,
-					    mmgr_calloc(1,
-					    sizeof(struct sh_css_sp_stage)));
-			xmem_isp_stage_ptrs[p][i] =
-					ia_css_refcount_increment(-1,
-					    mmgr_calloc(1,
-					    sizeof(struct sh_css_isp_stage)));
-
-			if ((xmem_sp_stage_ptrs[p][i] == mmgr_NULL) ||
-			    (xmem_isp_stage_ptrs[p][i] == mmgr_NULL)) {
-				sh_css_params_uninit();
-				IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-				return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-			}
-		}
-	}
-
-	ia_css_config_gamma_table();
-	ia_css_config_ctc_table();
-	ia_css_config_rgb_gamma_tables();
-	ia_css_config_xnr_table();
-
-	sp_ddr_ptrs = ia_css_refcount_increment(-1, mmgr_calloc(1,
-		CEIL_MUL(sizeof(struct sh_css_ddr_address_map),
-			 HIVE_ISP_DDR_WORD_BYTES)));
-	xmem_sp_group_ptrs = ia_css_refcount_increment(-1, mmgr_calloc(1,
-		sizeof(struct sh_css_sp_group)));
-
-	if ((sp_ddr_ptrs == mmgr_NULL) ||
-	    (xmem_sp_group_ptrs == mmgr_NULL)) {
-		ia_css_uninit();
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	}
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-static void host_lut_store(const void *lut)
-{
-	unsigned i;
-
-	for (i = 0; i < N_GDC_ID; i++)
-		gdc_lut_store((gdc_ID_t)i, (const int (*)[HRT_GDC_N]) lut);
-}
-
-/* Note that allocation is in ipu address space. */
-inline hrt_vaddress sh_css_params_alloc_gdc_lut(void)
-{
-	return mmgr_malloc(sizeof(zoom_table));
-}
-
-inline void sh_css_params_free_gdc_lut(hrt_vaddress addr)
-{
-	if (addr != mmgr_NULL)
-		hmm_free(addr);
-}
-
-enum ia_css_err ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe,
-	const void *lut)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-#ifndef ISP2401
-	bool store = true;
-#else
-	bool stream_started = false;
-#endif
-	IA_CSS_ENTER("pipe=%p lut=%p", pipe, lut);
-
-	if (lut == NULL || pipe == NULL) {
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE("err=%d", err);
-		return err;
-	}
-
-	/* If the pipe belongs to a stream and the stream has started, it is not
-	 * safe to store lut to gdc HW. If pipe->stream is NULL, then no stream is
-	 * created with this pipe, so it is safe to do this operation as long as
-	 * ia_css_init() has been called. */
-	if (pipe->stream && pipe->stream->started) {
-		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-			"unable to set scaler lut since stream has started\n");
-#ifndef ISP2401
-		store = false;
-#else
-		stream_started = true;
-#endif
-		err = IA_CSS_ERR_NOT_SUPPORTED;
-	}
-
-	/* Free any existing tables. */
-	sh_css_params_free_gdc_lut(pipe->scaler_pp_lut);
-	pipe->scaler_pp_lut = mmgr_NULL;
-
-#ifndef ISP2401
-	if (store) {
-		pipe->scaler_pp_lut = mmgr_malloc(sizeof(zoom_table));
-#else
-	if (!stream_started) {
-		pipe->scaler_pp_lut = sh_css_params_alloc_gdc_lut();
-#endif
-		if (pipe->scaler_pp_lut == mmgr_NULL) {
-#ifndef ISP2401
-			IA_CSS_LEAVE("lut(%u) err=%d", pipe->scaler_pp_lut, err);
-			return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-#else
-			ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-				"unable to allocate scaler_pp_lut\n");
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		} else {
-			gdc_lut_convert_to_isp_format((const int(*)[HRT_GDC_N])lut,
-				interleaved_lut_temp);
-			mmgr_store(pipe->scaler_pp_lut,
-				(int *)interleaved_lut_temp,
-				sizeof(zoom_table));
-#endif
-		}
-#ifndef ISP2401
-
-		gdc_lut_convert_to_isp_format((const int(*)[HRT_GDC_N])lut, interleaved_lut_temp);
-		mmgr_store(pipe->scaler_pp_lut, (int *)interleaved_lut_temp,
-			sizeof(zoom_table));
-#endif
-	}
-
-	IA_CSS_LEAVE("lut(%u) err=%d", pipe->scaler_pp_lut, err);
-	return err;
-}
-
-/* if pipe is NULL, returns default lut addr. */
-hrt_vaddress sh_css_pipe_get_pp_gdc_lut(const struct ia_css_pipe *pipe)
-{
-	assert(pipe != NULL);
-
-	if (pipe->scaler_pp_lut != mmgr_NULL)
-		return pipe->scaler_pp_lut;
-	else
-		return sh_css_params_get_default_gdc_lut();
-}
-
-enum ia_css_err sh_css_params_map_and_store_default_gdc_lut(void)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	/* Is table already mapped? Nothing to do if it is mapped. */
-	if (default_gdc_lut != mmgr_NULL)
-		return err;
-
-	host_lut_store((void *)zoom_table);
-
-#ifndef ISP2401
-	default_gdc_lut = mmgr_malloc(sizeof(zoom_table));
-#else
-	default_gdc_lut = sh_css_params_alloc_gdc_lut();
-#endif
-	if (default_gdc_lut == mmgr_NULL)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-
-	gdc_lut_convert_to_isp_format((const int(*)[HRT_GDC_N])zoom_table,
-		interleaved_lut_temp);
-	mmgr_store(default_gdc_lut, (int *)interleaved_lut_temp,
-		sizeof(zoom_table));
-
-	IA_CSS_LEAVE_PRIVATE("lut(%u) err=%d", default_gdc_lut, err);
-	return err;
-}
-
-void sh_css_params_free_default_gdc_lut(void)
-{
-	IA_CSS_ENTER_PRIVATE("void");
-
-	sh_css_params_free_gdc_lut(default_gdc_lut);
-	default_gdc_lut = mmgr_NULL;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-
-}
-
-hrt_vaddress sh_css_params_get_default_gdc_lut(void)
-{
-	return default_gdc_lut;
-}
-
-static void free_param_set_callback(
-	hrt_vaddress ptr)
-{
-	IA_CSS_ENTER_PRIVATE("void");
-
-	free_ia_css_isp_parameter_set_info(ptr);
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void free_buffer_callback(
-	hrt_vaddress ptr)
-{
-	IA_CSS_ENTER_PRIVATE("void");
-
-	hmm_free(ptr);
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-void
-sh_css_param_clear_param_sets(void)
-{
-	IA_CSS_ENTER_PRIVATE("void");
-
-	ia_css_refcount_clear(IA_CSS_REFCOUNT_PARAM_SET_POOL, &free_param_set_callback);
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-/*
- * MW: we can define hmm_free() to return a NULL
- * then you can write ptr = hmm_free(ptr);
- */
-#define safe_free(id, x)      \
-	do {                  \
-		ia_css_refcount_decrement(id, x);     \
-		(x) = mmgr_NULL;  \
-	} while(0)
-
-static void free_map(struct sh_css_ddr_address_map *map)
-{
-	unsigned int i;
-
-	hrt_vaddress *addrs = (hrt_vaddress *)map;
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	/* free buffers */
-	for (i = 0; i < (sizeof(struct sh_css_ddr_address_map_size)/
-						sizeof(size_t)); i++) {
-		if (addrs[i] == mmgr_NULL)
-			continue;
-		safe_free(IA_CSS_REFCOUNT_PARAM_BUFFER, addrs[i]);
-	}
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-void
-ia_css_stream_isp_parameters_uninit(struct ia_css_stream *stream)
-{
-	int i;
-	struct ia_css_isp_parameters *params = stream->isp_params_configs;
-	struct ia_css_isp_parameters *per_frame_params =
-						stream->per_frame_isp_params_configs;
-
-	IA_CSS_ENTER_PRIVATE("void");
-	if (params == NULL) {
-		IA_CSS_LEAVE_PRIVATE("isp_param_configs is NULL");
-		return;
-	}
-
-	/* free existing ddr_ptr maps */
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
-	{
-		free_map(&params->pipe_ddr_ptrs[i]);
-		if (per_frame_params)
-			free_map(&per_frame_params->pipe_ddr_ptrs[i]);
-		/* Free up theDVS table memory blocks before recomputing new table */
-		if (params->pipe_dvs_6axis_config[i])
-			free_dvs_6axis_table(&(params->pipe_dvs_6axis_config[i]));
-		if (per_frame_params && per_frame_params->pipe_dvs_6axis_config[i])
-			free_dvs_6axis_table(&(per_frame_params->pipe_dvs_6axis_config[i]));
-	}
-	free_map(&params->ddr_ptrs);
-	if (per_frame_params)
-		free_map(&per_frame_params->ddr_ptrs);
-
-	if (params->fpn_config.data) {
-		sh_css_free(params->fpn_config.data);
-		params->fpn_config.data = NULL;
-	}
-
-	/* Free up sc_config (temporal shading table) if it is allocated. */
-	if (params->sc_config) {
-		ia_css_shading_table_free(params->sc_config);
-		params->sc_config = NULL;
-	}
-	if (per_frame_params) {
-		if (per_frame_params->sc_config) {
-			ia_css_shading_table_free(per_frame_params->sc_config);
-			per_frame_params->sc_config = NULL;
-		}
-	}
-
-	sh_css_free(params);
-	if (per_frame_params)
-		sh_css_free(per_frame_params);
-	stream->isp_params_configs = NULL;
-	stream->per_frame_isp_params_configs = NULL;
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-void
-sh_css_params_uninit(void)
-{
-	unsigned p, i;
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	ia_css_refcount_decrement(-1, sp_ddr_ptrs);
-	sp_ddr_ptrs = mmgr_NULL;
-	ia_css_refcount_decrement(-1, xmem_sp_group_ptrs);
-	xmem_sp_group_ptrs = mmgr_NULL;
-
-	for (p = 0; p < IA_CSS_PIPE_ID_NUM; p++)
-		for (i = 0; i < SH_CSS_MAX_STAGES; i++) {
-			ia_css_refcount_decrement(-1, xmem_sp_stage_ptrs[p][i]);
-			xmem_sp_stage_ptrs[p][i] = mmgr_NULL;
-			ia_css_refcount_decrement(-1, xmem_isp_stage_ptrs[p][i]);
-			xmem_isp_stage_ptrs[p][i] = mmgr_NULL;
-		}
-
-	/* go through the pools to clear references */
-	ia_css_refcount_clear(IA_CSS_REFCOUNT_PARAM_SET_POOL, &free_param_set_callback);
-	ia_css_refcount_clear(IA_CSS_REFCOUNT_PARAM_BUFFER, &free_buffer_callback);
-	ia_css_refcount_clear(-1, &free_buffer_callback);
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static struct ia_css_host_data *
-convert_allocate_morph_plane(
-	unsigned short *data,
-	unsigned int width,
-	unsigned int height,
-	unsigned int aligned_width)
-{
-	unsigned int i, j, padding, w;
-	struct ia_css_host_data *me;
-	unsigned int isp_data_size;
-	uint16_t *isp_data_ptr;
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	/* currently we don't have morph table interpolation yet,
-	 * so we allow a wider table to be used. This will be removed
-	 * in the future. */
-	if (width > aligned_width) {
-		padding = 0;
-		w = aligned_width;
-	} else {
-		padding = aligned_width - width;
-		w = width;
-	}
-	isp_data_size = height * (w + padding) * sizeof(uint16_t);
-
-	me = ia_css_host_data_allocate((size_t) isp_data_size);
-
-	if (!me) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-		return NULL;
-	}
-
-	isp_data_ptr = (uint16_t *)me->address;
-
-	memset(isp_data_ptr, 0, (size_t)isp_data_size);
-
-	for (i = 0; i < height; i++) {
-		for (j = 0; j < w; j++)
-			*isp_data_ptr++ = (uint16_t)data[j];
-		isp_data_ptr += padding;
-		data += width;
-	}
-
-	IA_CSS_LEAVE_PRIVATE("void");
-	return me;
-}
-
-static enum ia_css_err
-store_morph_plane(
-	unsigned short *data,
-	unsigned int width,
-	unsigned int height,
-	hrt_vaddress dest,
-	unsigned int aligned_width)
-{
-	struct ia_css_host_data *isp_data;
-
-	assert(dest != mmgr_NULL);
-
-	isp_data = convert_allocate_morph_plane(data, width, height, aligned_width);
-	if (!isp_data) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	}
-	ia_css_params_store_ia_css_host_data(dest, isp_data);
-
-	ia_css_host_data_free(isp_data);
-	return IA_CSS_SUCCESS;
-}
-
-static void sh_css_update_isp_params_to_ddr(
-	struct ia_css_isp_parameters *params,
-	hrt_vaddress ddr_ptr)
-{
-	size_t size = sizeof(params->uds);
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	assert(params != NULL);
-
-	mmgr_store(ddr_ptr, &(params->uds), size);
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void sh_css_update_isp_mem_params_to_ddr(
-	const struct ia_css_binary *binary,
-	hrt_vaddress ddr_mem_ptr,
-	size_t size,
-	enum ia_css_isp_memories mem)
-{
-	const struct ia_css_host_data *params;
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	params = ia_css_isp_param_get_mem_init(&binary->mem_params, IA_CSS_PARAM_CLASS_PARAM, mem);
-	mmgr_store(ddr_mem_ptr, params->address, size);
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-void ia_css_dequeue_param_buffers(/*unsigned int pipe_num*/ void)
-{
-	unsigned int i;
-	hrt_vaddress cpy;
-	enum sh_css_queue_id param_queue_ids[3] = {	IA_CSS_PARAMETER_SET_QUEUE_ID,
-							IA_CSS_PER_FRAME_PARAMETER_SET_QUEUE_ID,
-							SH_CSS_INVALID_QUEUE_ID};
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	if (!sh_css_sp_is_running()) {
-		IA_CSS_LEAVE_PRIVATE("sp is not running");
-		/* SP is not running. The queues are not valid */
-		return;
-	}
-
-	for (i = 0; SH_CSS_INVALID_QUEUE_ID != param_queue_ids[i]; i++) {
-		cpy = (hrt_vaddress)0;
-		/* clean-up old copy */
-		while (IA_CSS_SUCCESS == ia_css_bufq_dequeue_buffer(param_queue_ids[i], (uint32_t *)&cpy)) {
-			/* TMP: keep track of dequeued param set count
-			 */
-			g_param_buffer_dequeue_count++;
-			ia_css_bufq_enqueue_psys_event(
-					IA_CSS_PSYS_SW_EVENT_BUFFER_DEQUEUED,
-					0,
-					param_queue_ids[i],
-					0);
-
-			IA_CSS_LOG("dequeued param set %x from %d, release ref", cpy, 0);
-			free_ia_css_isp_parameter_set_info(cpy);
-			cpy = (hrt_vaddress)0;
-		}
-	}
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static void
-process_kernel_parameters(unsigned int pipe_id,
-			  struct ia_css_pipeline_stage *stage,
-			  struct ia_css_isp_parameters *params,
-			  unsigned int isp_pipe_version,
-			  unsigned int raw_bit_depth)
-{
-	unsigned param_id;
-
-	(void)isp_pipe_version;
-	(void)raw_bit_depth;
-
-	sh_css_enable_pipeline(stage->binary);
-
-	if (params->config_changed[IA_CSS_OB_ID]) {
-		ia_css_ob_configure(&params->stream_configs.ob,
-			    isp_pipe_version, raw_bit_depth);
-	}
-	if (params->config_changed[IA_CSS_S3A_ID]) {
-		ia_css_s3a_configure(raw_bit_depth);
-	}
-	/* Copy stage uds parameters to config, since they can differ per stage.
-	 */
-	params->crop_config.crop_pos = params->uds[stage->stage_num].crop_pos;
-	params->uds_config.crop_pos  = params->uds[stage->stage_num].crop_pos;
-	params->uds_config.uds       = params->uds[stage->stage_num].uds;
-	/* Call parameter process functions for all kernels */
-	/* Skip SC, since that is called on a temp sc table */
-	for (param_id = 0; param_id < IA_CSS_NUM_PARAMETER_IDS; param_id++) {
-		if (param_id == IA_CSS_SC_ID) continue;
-		if (params->config_changed[param_id])
-			ia_css_kernel_process_param[param_id](pipe_id, stage, params);
-	}
-}
-
-enum ia_css_err
-sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
-	struct ia_css_isp_parameters *params,
-	bool commit,
-	struct ia_css_pipe *pipe_in)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	hrt_vaddress cpy;
-	int i;
-	unsigned int raw_bit_depth = 10;
-	unsigned int isp_pipe_version = SH_CSS_ISP_PIPE_VERSION_1;
-	bool acc_cluster_params_changed = false;
-	unsigned int thread_id, pipe_num;
-
-	(void)acc_cluster_params_changed;
-
-	assert(curr_pipe != NULL);
-
-	IA_CSS_ENTER_PRIVATE("pipe=%p, isp_parameters_id=%d", pipe_in, params->isp_parameters_id);
-	raw_bit_depth = ia_css_stream_input_format_bits_per_pixel(curr_pipe->stream);
-
-	/* now make the map available to the sp */
-	if (!commit) {
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-	/* enqueue a copies of the mem_map to
-	   the designated pipelines */
-	for (i = 0; i < curr_pipe->stream->num_pipes; i++) {
-		struct ia_css_pipe *pipe;
-		struct sh_css_ddr_address_map *cur_map;
-		struct sh_css_ddr_address_map_size *cur_map_size;
-		struct ia_css_isp_parameter_set_info isp_params_info;
-		struct ia_css_pipeline *pipeline;
-		struct ia_css_pipeline_stage *stage;
-
-		enum sh_css_queue_id queue_id;
-
-		pipe = curr_pipe->stream->pipes[i];
-		pipeline = ia_css_pipe_get_pipeline(pipe);
-		pipe_num = ia_css_pipe_get_pipe_num(pipe);
-		isp_pipe_version = ia_css_pipe_get_isp_pipe_version(pipe);
-		ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
-
-#if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-		ia_css_query_internal_queue_id(params->output_frame
-						? IA_CSS_BUFFER_TYPE_PER_FRAME_PARAMETER_SET
-						: IA_CSS_BUFFER_TYPE_PARAMETER_SET,
-						thread_id, &queue_id);
-#else
-		ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_PARAMETER_SET, thread_id, &queue_id);
-#endif
-		if (!sh_css_sp_is_running()) {
-			/* SP is not running. The queues are not valid */
-			err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-			break;
-		}
-		cur_map = &params->pipe_ddr_ptrs[pipeline->pipe_id];
-		cur_map_size = &params->pipe_ddr_ptrs_size[pipeline->pipe_id];
-
-		/* TODO: Normally, zoom and motion parameters shouldn't
-		 * be part of "isp_params" as it is resolution/pipe dependant
-		 * Therefore, move the zoom config elsewhere (e.g. shading
-		 * table can be taken as an example! @GC
-		 * */
-		{
-			/* we have to do this per pipeline because */
-			/* the processing is a.o. resolution dependent */
-			err = ia_css_process_zoom_and_motion(params,
-					pipeline->stages);
-			if (err != IA_CSS_SUCCESS)
-			    return err;
-		}
-		/* check if to actually update the parameters for this pipe */
-		/* When API change is implemented making good distinction between
-		* stream config and pipe config this skipping code can be moved out of the #ifdef */
-		if (pipe_in && (pipe != pipe_in)) {
-			IA_CSS_LOG("skipping pipe %p", pipe);
-			continue;
-		}
-
-		/* BZ 125915, should be moved till after "update other buff" */
-		/* update the other buffers to the pipe specific copies */
-		for (stage = pipeline->stages; stage; stage = stage->next) {
-			unsigned mem;
-
-			if (!stage || !stage->binary)
-				continue;
-
-			process_kernel_parameters(pipeline->pipe_id,
-					stage, params,
-					isp_pipe_version, raw_bit_depth);
-
-			err = sh_css_params_write_to_ddr_internal(
-					pipe,
-					pipeline->pipe_id,
-					params,
-					stage,
-					cur_map,
-					cur_map_size);
-
-			if (err != IA_CSS_SUCCESS)
-				break;
-			for (mem = 0; mem < IA_CSS_NUM_MEMORIES; mem++) {
-				params->isp_mem_params_changed
-				[pipeline->pipe_id][stage->stage_num][mem] = false;
-			}
-		} /* for */
-		if (err != IA_CSS_SUCCESS)
-			break;
-		/* update isp_params to pipe specific copies */
-		if (params->isp_params_changed) {
-			reallocate_buffer(&cur_map->isp_param,
-					&cur_map_size->isp_param,
-					cur_map_size->isp_param,
-					true,
-					&err);
-			if (err != IA_CSS_SUCCESS)
-				break;
-			sh_css_update_isp_params_to_ddr(params, cur_map->isp_param);
-		}
-
-		/* last make referenced copy */
-		err = ref_sh_css_ddr_address_map(
-				cur_map,
-				&isp_params_info.mem_map);
-		if (err != IA_CSS_SUCCESS)
-			break;
-
-		/* Update Parameters ID */
-		isp_params_info.isp_parameters_id = params->isp_parameters_id;
-
-		/* Update output frame pointer */
-		isp_params_info.output_frame_ptr =
-				(params->output_frame) ? params->output_frame->data : mmgr_NULL;
-
-		/* now write the copy to ddr */
-		err = write_ia_css_isp_parameter_set_info_to_ddr(&isp_params_info, &cpy);
-		if (err != IA_CSS_SUCCESS)
-			break;
-
-		/* enqueue the set to sp */
-		IA_CSS_LOG("queue param set %x to %d", cpy, thread_id);
-
-		err = ia_css_bufq_enqueue_buffer(thread_id, queue_id, (uint32_t)cpy);
-		if (IA_CSS_SUCCESS != err) {
-			free_ia_css_isp_parameter_set_info(cpy);
-#if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-			IA_CSS_LOG("pfp: FAILED to add config id %d for OF %d to q %d on thread %d",
-				isp_params_info.isp_parameters_id,
-				isp_params_info.output_frame_ptr,
-				queue_id, thread_id);
-#endif
-			break;
-		}
-		else {
-			/* TMP: check discrepancy between nr of enqueued
-			 * parameter sets and dequeued sets
-			 */
-			g_param_buffer_enqueue_count++;
-			assert(g_param_buffer_enqueue_count < g_param_buffer_dequeue_count+50);
-#ifdef ISP2401
-			ia_css_save_latest_paramset_ptr(pipe, cpy);
-#endif
-			/*
-			 * Tell the SP which queues are not empty,
-			 * by sending the software event.
-			 */
-			if (!sh_css_sp_is_running()) {
-				/* SP is not running. The queues are not valid */
-				IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
-				return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
-			}
-			ia_css_bufq_enqueue_psys_event(
-					IA_CSS_PSYS_SW_EVENT_BUFFER_ENQUEUED,
-					(uint8_t)thread_id,
-					(uint8_t)queue_id,
-					0);
-#if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-			IA_CSS_LOG("pfp: added config id %d for OF %d to q %d on thread %d",
-				isp_params_info.isp_parameters_id,
-				isp_params_info.output_frame_ptr,
-				queue_id, thread_id);
-#endif
-		}
-		/* clean-up old copy */
-		ia_css_dequeue_param_buffers(/*pipe_num*/);
-		params->pipe_dvs_6axis_config_changed[pipeline->pipe_id] = false;
-	} /* end for each 'active' pipeline */
-	/* clear the changed flags after all params
-	for all pipelines have been updated */
-	params->isp_params_changed = false;
-	params->sc_table_changed = false;
-	params->dis_coef_table_changed = false;
-	params->dvs2_coef_table_changed = false;
-	params->morph_table_changed = false;
-	params->dz_config_changed = false;
-	params->motion_config_changed = false;
-/* ------ deprecated(bz675) : from ------ */
-	params->shading_settings_changed = false;
-/* ------ deprecated(bz675) : to ------ */
-
-	memset(&params->config_changed[0], 0, sizeof(params->config_changed));
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static enum ia_css_err
-sh_css_params_write_to_ddr_internal(
-	struct ia_css_pipe *pipe,
-	unsigned pipe_id,
-	struct ia_css_isp_parameters *params,
-	const struct ia_css_pipeline_stage *stage,
-	struct sh_css_ddr_address_map *ddr_map,
-	struct sh_css_ddr_address_map_size *ddr_map_size)
-{
-	enum ia_css_err err;
-	const struct ia_css_binary *binary;
-
-	unsigned stage_num;
-	unsigned mem;
-	bool buff_realloced;
-
-	/* struct is > 128 bytes so it should not be on stack (see checkpatch) */
-	static struct ia_css_macc_table converted_macc_table;
-
-	IA_CSS_ENTER_PRIVATE("void");
-	assert(params != NULL);
-	assert(ddr_map != NULL);
-	assert(ddr_map_size != NULL);
-	assert(stage != NULL);
-
-	binary = stage->binary;
-	assert(binary != NULL);
-
-
-	stage_num = stage->stage_num;
-
-	if (binary->info->sp.enable.fpnr) {
-		buff_realloced = reallocate_buffer(&ddr_map->fpn_tbl,
-			&ddr_map_size->fpn_tbl,
-			(size_t)(FPNTBL_BYTES(binary)),
-			params->config_changed[IA_CSS_FPN_ID],
-			&err);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-		if (params->config_changed[IA_CSS_FPN_ID] || buff_realloced) {
-			if (params->fpn_config.enabled) {
-				err = store_fpntbl(params, ddr_map->fpn_tbl);
-				if (err != IA_CSS_SUCCESS) {
-					IA_CSS_LEAVE_ERR_PRIVATE(err);
-					return err;
-				}
-			}
-		}
-	}
-
-	if (binary->info->sp.enable.sc) {
-		uint32_t enable_conv = params->
-			shading_settings.enable_shading_table_conversion;
-
-		buff_realloced = reallocate_buffer(&ddr_map->sc_tbl,
-			&ddr_map_size->sc_tbl,
-			(size_t)(SCTBL_BYTES(binary)),
-			params->sc_table_changed,
-			&err);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-
-		if (params->shading_settings_changed ||
-		    params->sc_table_changed || buff_realloced) {
-			if (enable_conv == 0) {
-				if (params->sc_table) {
-					/* store the shading table to ddr */
-					err = ia_css_params_store_sctbl(stage, ddr_map->sc_tbl, params->sc_table);
-					if (err != IA_CSS_SUCCESS) {
-						IA_CSS_LEAVE_ERR_PRIVATE(err);
-						return err;
-					}
-					/* set sc_config to isp */
-					params->sc_config = (struct ia_css_shading_table *)params->sc_table;
-					ia_css_kernel_process_param[IA_CSS_SC_ID](pipe_id, stage, params);
-					params->sc_config = NULL;
-				} else {
-					/* generate the identical shading table */
-					if (params->sc_config) {
-						ia_css_shading_table_free(params->sc_config);
-						params->sc_config = NULL;
-					}
-#ifndef ISP2401
-					sh_css_params_shading_id_table_generate(&params->sc_config, binary);
-#else
-					sh_css_params_shading_id_table_generate(&params->sc_config,
-						binary->sctbl_width_per_color, binary->sctbl_height);
-#endif
-					if (params->sc_config == NULL) {
-						IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-						return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-					}
-
-					/* store the shading table to ddr */
-					err = ia_css_params_store_sctbl(stage, ddr_map->sc_tbl, params->sc_config);
-					if (err != IA_CSS_SUCCESS) {
-						IA_CSS_LEAVE_ERR_PRIVATE(err);
-						return err;
-					}
-
-					/* set sc_config to isp */
-					ia_css_kernel_process_param[IA_CSS_SC_ID](pipe_id, stage, params);
-
-					/* free the shading table */
-					ia_css_shading_table_free(params->sc_config);
-					params->sc_config = NULL;
-				}
-			} else { /* legacy */
-/* ------ deprecated(bz675) : from ------ */
-				/* shading table is full resolution, reduce */
-				if (params->sc_config) {
-					ia_css_shading_table_free(params->sc_config);
-					params->sc_config = NULL;
-				}
-				prepare_shading_table(
-					(const struct ia_css_shading_table *)params->sc_table,
-					params->sensor_binning,
-					&params->sc_config,
-					binary, pipe->required_bds_factor);
-				if (params->sc_config == NULL) {
-					IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-					return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-				}
-
-				/* store the shading table to ddr */
-				err = ia_css_params_store_sctbl(stage, ddr_map->sc_tbl, params->sc_config);
-				if (err != IA_CSS_SUCCESS) {
-					IA_CSS_LEAVE_ERR_PRIVATE(err);
-					return err;
-				}
-
-				/* set sc_config to isp */
-				ia_css_kernel_process_param[IA_CSS_SC_ID](pipe_id, stage, params);
-
-				/* free the shading table */
-				ia_css_shading_table_free(params->sc_config);
-				params->sc_config = NULL;
-/* ------ deprecated(bz675) : to ------ */
-			}
-		}
-	}
-#ifdef ISP2401
-	/* DPC configuration is made pipe specific to allow flexibility in positioning of the
-	 * DPC kernel. The code below sets the pipe specific configuration to
-	 * individual binaries. */
-	if (params->pipe_dpc_config_changed[pipe_id] && binary->info->sp.enable.dpc) {
-		unsigned size   = stage->binary->info->mem_offsets.offsets.param->dmem.dp.size;
-
-		unsigned offset = stage->binary->info->mem_offsets.offsets.param->dmem.dp.offset;
-		if (size) {
-			ia_css_dp_encode((struct sh_css_isp_dp_params *)
-				&binary->mem_params.params[IA_CSS_PARAM_CLASS_PARAM][IA_CSS_ISP_DMEM].address[offset],
-				&params->pipe_dp_config[pipe_id], size);
-#endif
-
-#ifdef ISP2401
-			params->isp_params_changed = true;
-			params->isp_mem_params_changed[pipe_id][stage->stage_num][IA_CSS_ISP_DMEM] = true;
-		}
-	}
-#endif
-	if (params->config_changed[IA_CSS_MACC_ID] && binary->info->sp.enable.macc) {
-		unsigned int i, j, idx;
-		unsigned int idx_map[] = {
-			0, 1, 3, 2, 6, 7, 5, 4, 12, 13, 15, 14, 10, 11, 9, 8};
-
-		for (i = 0; i < IA_CSS_MACC_NUM_AXES; i++) {
-			idx = 4*idx_map[i];
-			j   = 4*i;
-
-			if (binary->info->sp.pipeline.isp_pipe_version == SH_CSS_ISP_PIPE_VERSION_1) {
-				converted_macc_table.data[idx] =
-				  (int16_t)sDIGIT_FITTING(params->macc_table.data[j],
-				  13, SH_CSS_MACC_COEF_SHIFT);
-				converted_macc_table.data[idx+1] =
-				  (int16_t)sDIGIT_FITTING(params->macc_table.data[j+1],
-				  13, SH_CSS_MACC_COEF_SHIFT);
-				converted_macc_table.data[idx+2] =
-				  (int16_t)sDIGIT_FITTING(params->macc_table.data[j+2],
-				  13, SH_CSS_MACC_COEF_SHIFT);
-				converted_macc_table.data[idx+3] =
-				  (int16_t)sDIGIT_FITTING(params->macc_table.data[j+3],
-				  13, SH_CSS_MACC_COEF_SHIFT);
-			} else if (binary->info->sp.pipeline.isp_pipe_version == SH_CSS_ISP_PIPE_VERSION_2_2) {
-				converted_macc_table.data[idx] =
-					params->macc_table.data[j];
-				converted_macc_table.data[idx+1] =
-					params->macc_table.data[j+1];
-				converted_macc_table.data[idx+2] =
-					params->macc_table.data[j+2];
-				converted_macc_table.data[idx+3] =
-					params->macc_table.data[j+3];
-			}
-		}
-		reallocate_buffer(&ddr_map->macc_tbl,
-				  &ddr_map_size->macc_tbl,
-				  ddr_map_size->macc_tbl,
-				  true,
-				  &err);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-		mmgr_store(ddr_map->macc_tbl,
-				     converted_macc_table.data,
-				     sizeof(converted_macc_table.data));
-	}
-
-	if (binary->info->sp.enable.dvs_6axis) {
-		/* because UV is packed into the Y plane, calc total
-		 * YYU size = /2 gives size of UV-only,
-		 * total YYU size = UV-only * 3.
-		 */
-		buff_realloced = reallocate_buffer(
-				&ddr_map->dvs_6axis_params_y,
-				&ddr_map_size->dvs_6axis_params_y,
-				(size_t)((DVS_6AXIS_BYTES(binary) / 2) * 3),
-				params->pipe_dvs_6axis_config_changed[pipe_id],
-				&err);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-
-		if (params->pipe_dvs_6axis_config_changed[pipe_id] || buff_realloced) {
-			const struct ia_css_frame_info *dvs_in_frame_info;
-
-			if ( stage->args.delay_frames[0] ) {
-				/*When delay frames are present(as in case of video),
-				they are used for dvs. Configure DVS using those params*/
-				dvs_in_frame_info = &stage->args.delay_frames[0]->info;
-			} else {
-				/*Otherwise, use input frame to configure DVS*/
-				dvs_in_frame_info = &stage->args.in_frame->info;
-			}
-
-			/* Generate default DVS unity table on start up*/
-			if (params->pipe_dvs_6axis_config[pipe_id] == NULL) {
-
-#ifndef ISP2401
-				struct ia_css_resolution dvs_offset;
-				dvs_offset.width  =
-#else
-				struct ia_css_resolution dvs_offset = {0, 0};
-				if (binary->dvs_envelope.width || binary->dvs_envelope.height) {
-					dvs_offset.width  =
-#endif
-						(PIX_SHIFT_FILTER_RUN_IN_X + binary->dvs_envelope.width) / 2;
-#ifndef ISP2401
-				dvs_offset.height =
-#else
-					dvs_offset.height =
-#endif
-						(PIX_SHIFT_FILTER_RUN_IN_Y + binary->dvs_envelope.height) / 2;
-#ifdef ISP2401
-				}
-#endif
-
-				params->pipe_dvs_6axis_config[pipe_id] =
-						generate_dvs_6axis_table(&binary->out_frame_info[0].res, &dvs_offset);
-				if (params->pipe_dvs_6axis_config[pipe_id] == NULL) {
-					IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-					return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-				}
-				params->pipe_dvs_6axis_config_changed[pipe_id] = true;
-			}
-
-			store_dvs_6axis_config(params->pipe_dvs_6axis_config[pipe_id],
-				binary,
-				dvs_in_frame_info,
-				ddr_map->dvs_6axis_params_y);
-			params->isp_params_changed = true;
-		}
-	}
-
-	if (binary->info->sp.enable.ca_gdc) {
-		unsigned int i;
-		hrt_vaddress *virt_addr_tetra_x[
-			IA_CSS_MORPH_TABLE_NUM_PLANES];
-		size_t *virt_size_tetra_x[
-			IA_CSS_MORPH_TABLE_NUM_PLANES];
-		hrt_vaddress *virt_addr_tetra_y[
-			IA_CSS_MORPH_TABLE_NUM_PLANES];
-		size_t *virt_size_tetra_y[
-			IA_CSS_MORPH_TABLE_NUM_PLANES];
-
-		virt_addr_tetra_x[0] = &ddr_map->tetra_r_x;
-		virt_addr_tetra_x[1] = &ddr_map->tetra_gr_x;
-		virt_addr_tetra_x[2] = &ddr_map->tetra_gb_x;
-		virt_addr_tetra_x[3] = &ddr_map->tetra_b_x;
-		virt_addr_tetra_x[4] = &ddr_map->tetra_ratb_x;
-		virt_addr_tetra_x[5] = &ddr_map->tetra_batr_x;
-
-		virt_size_tetra_x[0] = &ddr_map_size->tetra_r_x;
-		virt_size_tetra_x[1] = &ddr_map_size->tetra_gr_x;
-		virt_size_tetra_x[2] = &ddr_map_size->tetra_gb_x;
-		virt_size_tetra_x[3] = &ddr_map_size->tetra_b_x;
-		virt_size_tetra_x[4] = &ddr_map_size->tetra_ratb_x;
-		virt_size_tetra_x[5] = &ddr_map_size->tetra_batr_x;
-
-		virt_addr_tetra_y[0] = &ddr_map->tetra_r_y;
-		virt_addr_tetra_y[1] = &ddr_map->tetra_gr_y;
-		virt_addr_tetra_y[2] = &ddr_map->tetra_gb_y;
-		virt_addr_tetra_y[3] = &ddr_map->tetra_b_y;
-		virt_addr_tetra_y[4] = &ddr_map->tetra_ratb_y;
-		virt_addr_tetra_y[5] = &ddr_map->tetra_batr_y;
-
-		virt_size_tetra_y[0] = &ddr_map_size->tetra_r_y;
-		virt_size_tetra_y[1] = &ddr_map_size->tetra_gr_y;
-		virt_size_tetra_y[2] = &ddr_map_size->tetra_gb_y;
-		virt_size_tetra_y[3] = &ddr_map_size->tetra_b_y;
-		virt_size_tetra_y[4] = &ddr_map_size->tetra_ratb_y;
-		virt_size_tetra_y[5] = &ddr_map_size->tetra_batr_y;
-
-		buff_realloced = false;
-		for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
-			buff_realloced |=
-					reallocate_buffer(virt_addr_tetra_x[i],
-						virt_size_tetra_x[i],
-						(size_t)
-						  (MORPH_PLANE_BYTES(binary)),
-						params->morph_table_changed,
-						&err);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-			buff_realloced |=
-					reallocate_buffer(virt_addr_tetra_y[i],
-						virt_size_tetra_y[i],
-						(size_t)
-						  (MORPH_PLANE_BYTES(binary)),
-						params->morph_table_changed,
-						&err);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-		}
-		if (params->morph_table_changed || buff_realloced) {
-			const struct ia_css_morph_table *table = params->morph_table;
-			struct ia_css_morph_table *id_table = NULL;
-
-			if ((table != NULL) &&
-			    (table->width < binary->morph_tbl_width ||
-			     table->height < binary->morph_tbl_height)) {
-				table = NULL;
-			}
-			if (table == NULL) {
-				err = sh_css_params_default_morph_table(&id_table,
-								  binary);
-				if (err != IA_CSS_SUCCESS) {
-					IA_CSS_LEAVE_ERR_PRIVATE(err);
-					return err;
-				}
-				table = id_table;
-			}
-
-			for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
-				store_morph_plane(table->coordinates_x[i],
-					table->width,
-					table->height,
-					*virt_addr_tetra_x[i],
-					binary->morph_tbl_aligned_width);
-				store_morph_plane(table->coordinates_y[i],
-					table->width,
-					table->height,
-					*virt_addr_tetra_y[i],
-					binary->morph_tbl_aligned_width);
-			}
-			if (id_table != NULL)
-				ia_css_morph_table_free(id_table);
-		}
-	}
-
-	/* After special cases like SC, FPN since they may change parameters */
-	for (mem = 0; mem < N_IA_CSS_MEMORIES; mem++) {
-		const struct ia_css_isp_data *isp_data =
-			ia_css_isp_param_get_isp_mem_init(&binary->info->sp.mem_initializers, IA_CSS_PARAM_CLASS_PARAM, mem);
-		size_t size = isp_data->size;
-		if (!size) continue;
-		buff_realloced = reallocate_buffer(&ddr_map->isp_mem_param[stage_num][mem],
-			&ddr_map_size->isp_mem_param[stage_num][mem],
-			size,
-			params->isp_mem_params_changed[pipe_id][stage_num][mem],
-			&err);
-		if (err != IA_CSS_SUCCESS) {
-			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			return err;
-		}
-		if (params->isp_mem_params_changed[pipe_id][stage_num][mem] || buff_realloced) {
-			sh_css_update_isp_mem_params_to_ddr(binary,
-				ddr_map->isp_mem_param[stage_num][mem],
-				ddr_map_size->isp_mem_param[stage_num][mem], mem);
-		}
-	}
-
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
-}
-
-const struct ia_css_fpn_table *ia_css_get_fpn_table(struct ia_css_stream *stream)
-{
-	struct ia_css_isp_parameters *params;
-
-	IA_CSS_ENTER_LEAVE("void");
-	assert(stream != NULL);
-
-	params = stream->isp_params_configs;
-
-	return &(params->fpn_config);
-}
-
-struct ia_css_shading_table *ia_css_get_shading_table(struct ia_css_stream *stream)
-{
-	struct ia_css_shading_table *table = NULL;
-	struct ia_css_isp_parameters *params;
-
-	IA_CSS_ENTER("void");
-
-	assert(stream != NULL);
-
-	params = stream->isp_params_configs;
-	if (!params)
-		return NULL;
-
-	if (params->shading_settings.enable_shading_table_conversion == 0) {
-		if (params->sc_table) {
-			table = (struct ia_css_shading_table *)params->sc_table;
-		} else {
-			const struct ia_css_binary *binary
-				= ia_css_stream_get_shading_correction_binary(stream);
-			if (binary) {
-				/* generate the identical shading table */
-				if (params->sc_config) {
-					ia_css_shading_table_free(params->sc_config);
-					params->sc_config = NULL;
-				}
-#ifndef ISP2401
-				sh_css_params_shading_id_table_generate(&params->sc_config, binary);
-
-#else
-				sh_css_params_shading_id_table_generate(&params->sc_config,
-					binary->sctbl_width_per_color, binary->sctbl_height);
-#endif
-				table = params->sc_config;
-				/* The sc_config will be freed in the
-				 * ia_css_stream_isp_parameters_uninit function. */
-			}
-		}
-	} else {
-/* ------ deprecated(bz675) : from ------ */
-		const struct ia_css_binary *binary
-			= ia_css_stream_get_shading_correction_binary(stream);
-		struct ia_css_pipe *pipe;
-
-		/**********************************************************************/
-		/* following code is copied from function ia_css_stream_get_shading_correction_binary()
-		 * to match with the binary */
-		pipe = stream->pipes[0];
-
-		if (stream->num_pipes == 2) {
-			assert(stream->pipes[1] != NULL);
-			if (stream->pipes[1]->config.mode == IA_CSS_PIPE_MODE_VIDEO ||
-			    stream->pipes[1]->config.mode == IA_CSS_PIPE_MODE_PREVIEW)
-				pipe = stream->pipes[1];
-		}
-		/**********************************************************************/
-		if (binary) {
-			if (params->sc_config) {
-				ia_css_shading_table_free(params->sc_config);
-				params->sc_config = NULL;
-			}
-			prepare_shading_table(
-				(const struct ia_css_shading_table *)params->sc_table,
-				params->sensor_binning,
-				&params->sc_config,
-				binary, pipe->required_bds_factor);
-
-			table = params->sc_config;
-			/* The sc_config will be freed in the
-			 * ia_css_stream_isp_parameters_uninit function. */
-		}
-/* ------ deprecated(bz675) : to ------ */
-	}
-
-	IA_CSS_LEAVE("table=%p", table);
-
-	return table;
-}
-
-
-hrt_vaddress sh_css_store_sp_group_to_ddr(void)
-{
-	IA_CSS_ENTER_LEAVE_PRIVATE("void");
-	mmgr_store(xmem_sp_group_ptrs,
-			     &sh_css_sp_group,
-			     sizeof(struct sh_css_sp_group));
-	return xmem_sp_group_ptrs;
-}
-
-hrt_vaddress sh_css_store_sp_stage_to_ddr(
-	unsigned pipe,
-	unsigned stage)
-{
-	IA_CSS_ENTER_LEAVE_PRIVATE("void");
-	mmgr_store(xmem_sp_stage_ptrs[pipe][stage],
-			     &sh_css_sp_stage,
-			     sizeof(struct sh_css_sp_stage));
-	return xmem_sp_stage_ptrs[pipe][stage];
-}
-
-hrt_vaddress sh_css_store_isp_stage_to_ddr(
-	unsigned pipe,
-	unsigned stage)
-{
-	IA_CSS_ENTER_LEAVE_PRIVATE("void");
-	mmgr_store(xmem_isp_stage_ptrs[pipe][stage],
-			     &sh_css_isp_stage,
-			     sizeof(struct sh_css_isp_stage));
-	return xmem_isp_stage_ptrs[pipe][stage];
-}
-
-static enum ia_css_err ref_sh_css_ddr_address_map(
-	struct sh_css_ddr_address_map *map,
-	struct sh_css_ddr_address_map *out)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	unsigned int i;
-
-	/* we will use a union to copy things; overlaying an array
-	   with the struct; that way adding fields in the struct
-	   will keep things working, and we will not get type errors.
-	*/
-	union {
-		struct sh_css_ddr_address_map *map;
-		hrt_vaddress *addrs;
-	} in_addrs, to_addrs;
-
-	IA_CSS_ENTER_PRIVATE("void");
-	assert(map != NULL);
-	assert(out != NULL);
-
-	in_addrs.map = map;
-	to_addrs.map = out;
-
-	assert(sizeof(struct sh_css_ddr_address_map_size)/sizeof(size_t) ==
-	       sizeof(struct sh_css_ddr_address_map)/sizeof(hrt_vaddress));
-
-	/* copy map using size info */
-	for (i = 0; i < (sizeof(struct sh_css_ddr_address_map_size)/
-						sizeof(size_t)); i++) {
-		if (in_addrs.addrs[i] == mmgr_NULL)
-			to_addrs.addrs[i] = mmgr_NULL;
-		else
-			to_addrs.addrs[i] = ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_BUFFER, in_addrs.addrs[i]);
-	}
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static enum ia_css_err write_ia_css_isp_parameter_set_info_to_ddr(
-	struct ia_css_isp_parameter_set_info *me,
-	hrt_vaddress *out)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	bool succ;
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	assert(me != NULL);
-	assert(out != NULL);
-
-	*out = ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_SET_POOL, mmgr_malloc(
-				sizeof(struct ia_css_isp_parameter_set_info)));
-	succ = (*out != mmgr_NULL);
-	if (succ)
-		mmgr_store(*out,
-			me, sizeof(struct ia_css_isp_parameter_set_info));
-	else
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-static enum ia_css_err
-free_ia_css_isp_parameter_set_info(
-	hrt_vaddress ptr)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	struct ia_css_isp_parameter_set_info isp_params_info;
-	unsigned int i;
-	hrt_vaddress *addrs = (hrt_vaddress *)&isp_params_info.mem_map;
-
-	IA_CSS_ENTER_PRIVATE("ptr = %u", ptr);
-
-	/* sanity check - ptr must be valid */
-	if (!ia_css_refcount_is_valid(ptr)) {
-		IA_CSS_ERROR("%s: IA_CSS_REFCOUNT_PARAM_SET_POOL(0x%x) invalid arg", __func__, ptr);
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		IA_CSS_LEAVE_ERR_PRIVATE(err);
-		return err;
-	}
-
-	mmgr_load(ptr, &isp_params_info.mem_map, sizeof(struct sh_css_ddr_address_map));
-	/* copy map using size info */
-	for (i = 0; i < (sizeof(struct sh_css_ddr_address_map_size)/
-						sizeof(size_t)); i++) {
-		if (addrs[i] == mmgr_NULL)
-			continue;
-
-		/* sanity check - ptr must be valid */
-#ifndef ISP2401
-		if (!ia_css_refcount_is_valid(addrs[i])) {
-#else
-		if (ia_css_refcount_is_valid(addrs[i])) {
-			ia_css_refcount_decrement(IA_CSS_REFCOUNT_PARAM_BUFFER, addrs[i]);
-		} else {
-#endif
-			IA_CSS_ERROR("%s: IA_CSS_REFCOUNT_PARAM_BUFFER(0x%x) invalid arg", __func__, ptr);
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
-			continue;
-		}
-#ifndef ISP2401
-
-		ia_css_refcount_decrement(IA_CSS_REFCOUNT_PARAM_BUFFER, addrs[i]);
-#endif
-	}
-	ia_css_refcount_decrement(IA_CSS_REFCOUNT_PARAM_SET_POOL, ptr);
-
-	IA_CSS_LEAVE_ERR_PRIVATE(err);
-	return err;
-}
-
-/* Mark all parameters as changed to force recomputing the derived ISP parameters */
-void
-sh_css_invalidate_params(struct ia_css_stream *stream)
-{
-	struct	ia_css_isp_parameters *params;
-	unsigned i, j, mem;
-
-	IA_CSS_ENTER_PRIVATE("void");
-	assert(stream != NULL);
-
-	params = stream->isp_params_configs;
-	params->isp_params_changed = true;
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) {
-		for (j = 0; j < SH_CSS_MAX_STAGES; j++) {
-			for (mem = 0; mem < N_IA_CSS_MEMORIES; mem++) {
-				params->isp_mem_params_changed[i][j][mem] = true;
-			}
-		}
-	}
-
-	memset(&params->config_changed[0], 1, sizeof(params->config_changed));
-	params->dis_coef_table_changed = true;
-	params->dvs2_coef_table_changed = true;
-	params->morph_table_changed = true;
-	params->sc_table_changed = true;
-	params->dz_config_changed = true;
-	params->motion_config_changed = true;
-
-	/*Free up theDVS table memory blocks before recomputing new table  */
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) {
-		if (params->pipe_dvs_6axis_config[i]) {
-			free_dvs_6axis_table(&(params->pipe_dvs_6axis_config[i]));
-			params->pipe_dvs_6axis_config_changed[i] = true;
-		}
-	}
-
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-void
-sh_css_update_uds_and_crop_info(
-	const struct ia_css_binary_info *info,
-	const struct ia_css_frame_info *in_frame_info,
-	const struct ia_css_frame_info *out_frame_info,
-	const struct ia_css_resolution *dvs_env,
-	const struct ia_css_dz_config *zoom,
-	const struct ia_css_vector *motion_vector,
-	struct sh_css_uds_info *uds,		/* out */
-	struct sh_css_crop_pos *sp_out_crop_pos,	/* out */
-	bool enable_zoom)
-{
-	IA_CSS_ENTER_PRIVATE("void");
-
-	assert(info != NULL);
-	assert(in_frame_info != NULL);
-	assert(out_frame_info != NULL);
-	assert(dvs_env != NULL);
-	assert(zoom != NULL);
-	assert(motion_vector != NULL);
-	assert(uds != NULL);
-	assert(sp_out_crop_pos != NULL);
-
-	uds->curr_dx   = enable_zoom ? (uint16_t)zoom->dx : HRT_GDC_N;
-	uds->curr_dy   = enable_zoom ? (uint16_t)zoom->dy : HRT_GDC_N;
-
-	if (info->enable.dvs_envelope) {
-		unsigned int crop_x = 0,
-			     crop_y = 0,
-			     uds_xc = 0,
-			     uds_yc = 0,
-			     env_width, env_height;
-		int half_env_x, half_env_y;
-		int motion_x = motion_vector->x;
-		int motion_y = motion_vector->y;
-		bool upscale_x = in_frame_info->res.width < out_frame_info->res.width;
-		bool upscale_y = in_frame_info->res.height < out_frame_info->res.height;
-
-		if (info->enable.uds && !info->enable.ds) {
-			/**
-			 * we calculate with the envelope that we can actually
-			 * use, the min dvs envelope is for the filter
-			 * initialization.
-			 */
-			env_width  = dvs_env->width -
-					SH_CSS_MIN_DVS_ENVELOPE;
-			env_height = dvs_env->height -
-					SH_CSS_MIN_DVS_ENVELOPE;
-			half_env_x = env_width / 2;
-			half_env_y = env_height / 2;
-			/**
-			 * for digital zoom, we use the dvs envelope and make
-			 * sure that we don't include the 8 leftmost pixels or
-			 * 8 topmost rows.
-			 */
-			if (upscale_x) {
-				uds_xc = (in_frame_info->res.width
-					+ env_width
-					+ SH_CSS_MIN_DVS_ENVELOPE) / 2;
-			} else {
-				uds_xc = (out_frame_info->res.width
-							+ env_width) / 2
-					+ SH_CSS_MIN_DVS_ENVELOPE;
-			}
-			if (upscale_y) {
-				uds_yc = (in_frame_info->res.height
-					+ env_height
-					+ SH_CSS_MIN_DVS_ENVELOPE) / 2;
-			} else {
-				uds_yc = (out_frame_info->res.height
-							+ env_height) / 2
-					+ SH_CSS_MIN_DVS_ENVELOPE;
-			}
-			/* clip the motion vector to +/- half the envelope */
-			motion_x = clamp(motion_x, -half_env_x, half_env_x);
-			motion_y = clamp(motion_y, -half_env_y, half_env_y);
-			uds_xc += motion_x;
-			uds_yc += motion_y;
-			/* uds can be pipelined, remove top lines */
-			crop_y = 2;
-		} else if (info->enable.ds) {
-			env_width  = dvs_env->width;
-			env_height = dvs_env->height;
-			half_env_x = env_width / 2;
-			half_env_y = env_height / 2;
-			/* clip the motion vector to +/- half the envelope */
-			motion_x = clamp(motion_x, -half_env_x, half_env_x);
-			motion_y = clamp(motion_y, -half_env_y, half_env_y);
-			/* for video with downscaling, the envelope is included
-			    in the input resolution. */
-			uds_xc = in_frame_info->res.width/2 + motion_x;
-			uds_yc = in_frame_info->res.height/2 + motion_y;
-			crop_x = info->pipeline.left_cropping;
-			/* ds == 2 (yuv_ds) can be pipelined, remove top
-			   lines */
-			if (info->enable.ds & 1)
-				crop_y = info->pipeline.top_cropping;
-			else
-				crop_y = 2;
-		} else {
-			/* video nodz: here we can only crop. We make sure we
-			   crop at least the first 8x8 pixels away. */
-			env_width  = dvs_env->width -
-					SH_CSS_MIN_DVS_ENVELOPE;
-			env_height = dvs_env->height -
-					SH_CSS_MIN_DVS_ENVELOPE;
-			half_env_x = env_width / 2;
-			half_env_y = env_height / 2;
-			motion_x = clamp(motion_x, -half_env_x, half_env_x);
-			motion_y = clamp(motion_y, -half_env_y, half_env_y);
-			crop_x = SH_CSS_MIN_DVS_ENVELOPE
-						+ half_env_x + motion_x;
-			crop_y = SH_CSS_MIN_DVS_ENVELOPE
-						+ half_env_y + motion_y;
-		}
-
-		/* Must enforce that the crop position is even */
-		crop_x = EVEN_FLOOR(crop_x);
-		crop_y = EVEN_FLOOR(crop_y);
-		uds_xc = EVEN_FLOOR(uds_xc);
-		uds_yc = EVEN_FLOOR(uds_yc);
-
-		uds->xc = (uint16_t)uds_xc;
-		uds->yc = (uint16_t)uds_yc;
-		sp_out_crop_pos->x = (uint16_t)crop_x;
-		sp_out_crop_pos->y = (uint16_t)crop_y;
-	}
-	else {
-		/* for down scaling, we always use the center of the image */
-		uds->xc = (uint16_t)in_frame_info->res.width / 2;
-		uds->yc = (uint16_t)in_frame_info->res.height / 2;
-		sp_out_crop_pos->x = (uint16_t)info->pipeline.left_cropping;
-		sp_out_crop_pos->y = (uint16_t)info->pipeline.top_cropping;
-	}
-	IA_CSS_LEAVE_PRIVATE("void");
-}
-
-static enum ia_css_err
-sh_css_update_uds_and_crop_info_based_on_zoom_region(
-	const struct ia_css_binary_info *info,
-	const struct ia_css_frame_info *in_frame_info,
-	const struct ia_css_frame_info *out_frame_info,
-	const struct ia_css_resolution *dvs_env,
-	const struct ia_css_dz_config *zoom,
-	const struct ia_css_vector *motion_vector,
-	struct sh_css_uds_info *uds,		/* out */
-	struct sh_css_crop_pos *sp_out_crop_pos,	/* out */
-	struct ia_css_resolution pipe_in_res,
-	bool enable_zoom)
-{
-	unsigned int x0 = 0, y0 = 0, x1 = 0, y1 = 0;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	/* Note:
-	* Filter_Envelope = 0 for NND/LUT
-	* Filter_Envelope = 1 for BCI
-	* Filter_Envelope = 3 for BLI
-	* Currently, not considering this filter envelope because, In uds.sp.c is recalculating
-	* the dx/dy based on filter envelope and other information (ia_css_uds_sp_scale_params)
-	* Ideally, That should be done on host side not on sp side.
-	*/
-	unsigned int filter_envelope = 0;
-	IA_CSS_ENTER_PRIVATE("void");
-
-	assert(info != NULL);
-	assert(in_frame_info != NULL);
-	assert(out_frame_info != NULL);
-	assert(dvs_env != NULL);
-	assert(zoom != NULL);
-	assert(motion_vector != NULL);
-	assert(uds != NULL);
-	assert(sp_out_crop_pos != NULL);
-	x0 = zoom->zoom_region.origin.x;
-	y0 = zoom->zoom_region.origin.y;
-	x1 = zoom->zoom_region.resolution.width + x0;
-	y1 = zoom->zoom_region.resolution.height + y0;
-
-	if ((x0 > x1) || (y0 > y1) || (x1 > pipe_in_res.width) || (y1 > pipe_in_res.height))
-	    return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	if (!enable_zoom) {
-	    uds->curr_dx = HRT_GDC_N;
-	    uds->curr_dy = HRT_GDC_N;
-	}
-
-	if (info->enable.dvs_envelope) {
-		/* Zoom region is only supported by the UDS module on ISP
-		 * 2 and higher. It is not supported in video mode on ISP 1 */
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	} else {
-		if (enable_zoom) {
-			/* A. Calculate dx/dy based on crop region using in_frame_info
-			* Scale the crop region if in_frame_info to the stage is not same as
-			* actual effective input of the pipeline
-			*/
-			if (in_frame_info->res.width != pipe_in_res.width ||
-			    in_frame_info->res.height != pipe_in_res.height) {
-				x0 = (x0 * in_frame_info->res.width) / (pipe_in_res.width);
-				y0 = (y0 * in_frame_info->res.height) / (pipe_in_res.height);
-				x1 = (x1 * in_frame_info->res.width) / (pipe_in_res.width);
-				y1 = (y1 * in_frame_info->res.height) / (pipe_in_res.height);
-			}
-			uds->curr_dx =
-				((x1 - x0 - filter_envelope) * HRT_GDC_N) / in_frame_info->res.width;
-			uds->curr_dy =
-				((y1 - y0 - filter_envelope) * HRT_GDC_N) / in_frame_info->res.height;
-
-			/* B. Calculate xc/yc based on crop region */
-			uds->xc = (uint16_t) x0 + (((x1)-(x0)) / 2);
-			uds->yc = (uint16_t) y0 + (((y1)-(y0)) / 2);
-		} else {
-			uds->xc = (uint16_t)in_frame_info->res.width / 2;
-			uds->yc = (uint16_t)in_frame_info->res.height / 2;
-		}
-
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "uds->curr_dx=%d, uds->xc=%d, uds->yc=%d\n",
-				uds->curr_dx, uds->xc, uds->yc);
-		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "x0=%d, y0=%d, x1=%d, y1=%d\n",
-				x0, y0, x1, y1);
-		sp_out_crop_pos->x = (uint16_t)info->pipeline.left_cropping;
-		sp_out_crop_pos->y = (uint16_t)info->pipeline.top_cropping;
-	}
-	IA_CSS_LEAVE_PRIVATE("void");
-	return err;
-}
-
-struct ia_css_3a_statistics *
-ia_css_3a_statistics_allocate(const struct ia_css_3a_grid_info *grid)
-{
-	struct ia_css_3a_statistics *me;
-	int grid_size;
-
-	IA_CSS_ENTER("grid=%p", grid);
-
-	assert(grid != NULL);
-
-	me = sh_css_calloc(1, sizeof(*me));
-	if (!me)
-		goto err;
-
-	me->grid = *grid;
-	grid_size = grid->width * grid->height;
-	me->data = sh_css_malloc(grid_size * sizeof(*me->data));
-	if (!me->data)
-		goto err;
-#if !defined(HAS_NO_HMEM)
-	/* No weighted histogram, no structure, treat the histogram data as a byte dump in a byte array */
-	me->rgby_data = (struct ia_css_3a_rgby_output *)sh_css_malloc(sizeof_hmem(HMEM0_ID));
-#else
-	me->rgby_data = NULL;
-#endif
-
-	IA_CSS_LEAVE("return=%p", me);
-	return me;
-err:
-	ia_css_3a_statistics_free(me);
-
-	IA_CSS_LEAVE("return=%p", NULL);
-	return NULL;
-}
-
-void
-ia_css_3a_statistics_free(struct ia_css_3a_statistics *me)
-{
-	if (me) {
-		sh_css_free(me->rgby_data);
-		sh_css_free(me->data);
-		memset(me, 0, sizeof(struct ia_css_3a_statistics));
-		sh_css_free(me);
-	}
-}
-
-struct ia_css_dvs_statistics *
-ia_css_dvs_statistics_allocate(const struct ia_css_dvs_grid_info *grid)
-{
-	struct ia_css_dvs_statistics *me;
-
-	assert(grid != NULL);
-
-	me = sh_css_calloc(1, sizeof(*me));
-	if (!me)
-		goto err;
-
-	me->grid = *grid;
-	me->hor_proj = sh_css_malloc(grid->height * IA_CSS_DVS_NUM_COEF_TYPES *
-					sizeof(*me->hor_proj));
-	if (!me->hor_proj)
-		goto err;
-
-	me->ver_proj = sh_css_malloc(grid->width * IA_CSS_DVS_NUM_COEF_TYPES *
-					sizeof(*me->ver_proj));
-	if (!me->ver_proj)
-		goto err;
-
-	return me;
-err:
-	ia_css_dvs_statistics_free(me);
-	return NULL;
-
-}
-
-void
-ia_css_dvs_statistics_free(struct ia_css_dvs_statistics *me)
-{
-	if (me) {
-		sh_css_free(me->hor_proj);
-		sh_css_free(me->ver_proj);
-		memset(me, 0, sizeof(struct ia_css_dvs_statistics));
-		sh_css_free(me);
-	}
-}
-
-struct ia_css_dvs_coefficients *
-ia_css_dvs_coefficients_allocate(const struct ia_css_dvs_grid_info *grid)
-{
-	struct ia_css_dvs_coefficients *me;
-
-	assert(grid != NULL);
-
-	me = sh_css_calloc(1, sizeof(*me));
-	if (!me)
-		goto err;
-
-	me->grid = *grid;
-
-	me->hor_coefs = sh_css_malloc(grid->num_hor_coefs *
-				IA_CSS_DVS_NUM_COEF_TYPES *
-				sizeof(*me->hor_coefs));
-	if (!me->hor_coefs)
-		goto err;
-
-	me->ver_coefs = sh_css_malloc(grid->num_ver_coefs *
-				IA_CSS_DVS_NUM_COEF_TYPES *
-				sizeof(*me->ver_coefs));
-	if (!me->ver_coefs)
-		goto err;
-
-	return me;
-err:
-	ia_css_dvs_coefficients_free(me);
-	return NULL;
-}
-
-void
-ia_css_dvs_coefficients_free(struct ia_css_dvs_coefficients *me)
-{
-	if (me) {
-		sh_css_free(me->hor_coefs);
-		sh_css_free(me->ver_coefs);
-		memset(me, 0, sizeof(struct ia_css_dvs_coefficients));
-		sh_css_free(me);
-	}
-}
-
-struct ia_css_dvs2_statistics *
-ia_css_dvs2_statistics_allocate(const struct ia_css_dvs_grid_info *grid)
-{
-	struct ia_css_dvs2_statistics *me;
-
-	assert(grid != NULL);
-
-	me = sh_css_calloc(1, sizeof(*me));
-	if (!me)
-		goto err;
-
-	me->grid = *grid;
-
-	me->hor_prod.odd_real = sh_css_malloc(grid->aligned_width *
-		grid->aligned_height * sizeof(*me->hor_prod.odd_real));
-	if (!me->hor_prod.odd_real)
-		goto err;
-
-	me->hor_prod.odd_imag = sh_css_malloc(grid->aligned_width *
-		grid->aligned_height * sizeof(*me->hor_prod.odd_imag));
-	if (!me->hor_prod.odd_imag)
-		goto err;
-
-	me->hor_prod.even_real = sh_css_malloc(grid->aligned_width *
-		grid->aligned_height * sizeof(*me->hor_prod.even_real));
-	if (!me->hor_prod.even_real)
-		goto err;
-
-	me->hor_prod.even_imag = sh_css_malloc(grid->aligned_width *
-		grid->aligned_height * sizeof(*me->hor_prod.even_imag));
-	if (!me->hor_prod.even_imag)
-		goto err;
-
-	me->ver_prod.odd_real = sh_css_malloc(grid->aligned_width *
-		grid->aligned_height * sizeof(*me->ver_prod.odd_real));
-	if (!me->ver_prod.odd_real)
-		goto err;
-
-	me->ver_prod.odd_imag = sh_css_malloc(grid->aligned_width *
-		grid->aligned_height * sizeof(*me->ver_prod.odd_imag));
-	if (!me->ver_prod.odd_imag)
-		goto err;
-
-	me->ver_prod.even_real = sh_css_malloc(grid->aligned_width *
-		grid->aligned_height * sizeof(*me->ver_prod.even_real));
-	if (!me->ver_prod.even_real)
-		goto err;
-
-	me->ver_prod.even_imag = sh_css_malloc(grid->aligned_width *
-		grid->aligned_height * sizeof(*me->ver_prod.even_imag));
-	if (!me->ver_prod.even_imag)
-		goto err;
-
-	return me;
-err:
-	ia_css_dvs2_statistics_free(me);
-	return NULL;
-
-}
-
-void
-ia_css_dvs2_statistics_free(struct ia_css_dvs2_statistics *me)
-{
-	if (me) {
-		sh_css_free(me->hor_prod.odd_real);
-		sh_css_free(me->hor_prod.odd_imag);
-		sh_css_free(me->hor_prod.even_real);
-		sh_css_free(me->hor_prod.even_imag);
-		sh_css_free(me->ver_prod.odd_real);
-		sh_css_free(me->ver_prod.odd_imag);
-		sh_css_free(me->ver_prod.even_real);
-		sh_css_free(me->ver_prod.even_imag);
-		memset(me, 0, sizeof(struct ia_css_dvs2_statistics));
-		sh_css_free(me);
-	}
-}
-
-
-struct ia_css_dvs2_coefficients *
-ia_css_dvs2_coefficients_allocate(const struct ia_css_dvs_grid_info *grid)
-{
-	struct ia_css_dvs2_coefficients *me;
-
-	assert(grid != NULL);
-
-	me = sh_css_calloc(1, sizeof(*me));
-	if (!me)
-		goto err;
-
-	me->grid = *grid;
-
-	me->hor_coefs.odd_real = sh_css_malloc(grid->num_hor_coefs *
-		sizeof(*me->hor_coefs.odd_real));
-	if (!me->hor_coefs.odd_real)
-		goto err;
-
-	me->hor_coefs.odd_imag = sh_css_malloc(grid->num_hor_coefs *
-		sizeof(*me->hor_coefs.odd_imag));
-	if (!me->hor_coefs.odd_imag)
-		goto err;
-
-	me->hor_coefs.even_real = sh_css_malloc(grid->num_hor_coefs *
-		sizeof(*me->hor_coefs.even_real));
-	if (!me->hor_coefs.even_real)
-		goto err;
-
-	me->hor_coefs.even_imag = sh_css_malloc(grid->num_hor_coefs *
-		sizeof(*me->hor_coefs.even_imag));
-	if (!me->hor_coefs.even_imag)
-		goto err;
-
-	me->ver_coefs.odd_real = sh_css_malloc(grid->num_ver_coefs *
-		sizeof(*me->ver_coefs.odd_real));
-	if (!me->ver_coefs.odd_real)
-		goto err;
-
-	me->ver_coefs.odd_imag = sh_css_malloc(grid->num_ver_coefs *
-		sizeof(*me->ver_coefs.odd_imag));
-	if (!me->ver_coefs.odd_imag)
-		goto err;
-
-	me->ver_coefs.even_real = sh_css_malloc(grid->num_ver_coefs *
-		sizeof(*me->ver_coefs.even_real));
-	if (!me->ver_coefs.even_real)
-		goto err;
-
-	me->ver_coefs.even_imag = sh_css_malloc(grid->num_ver_coefs *
-		sizeof(*me->ver_coefs.even_imag));
-	if (!me->ver_coefs.even_imag)
-		goto err;
-
-	return me;
-err:
-	ia_css_dvs2_coefficients_free(me);
-	return NULL;
-}
-
-void
-ia_css_dvs2_coefficients_free(struct ia_css_dvs2_coefficients *me)
-{
-	if (me) {
-		sh_css_free(me->hor_coefs.odd_real);
-		sh_css_free(me->hor_coefs.odd_imag);
-		sh_css_free(me->hor_coefs.even_real);
-		sh_css_free(me->hor_coefs.even_imag);
-		sh_css_free(me->ver_coefs.odd_real);
-		sh_css_free(me->ver_coefs.odd_imag);
-		sh_css_free(me->ver_coefs.even_real);
-		sh_css_free(me->ver_coefs.even_imag);
-		memset(me, 0, sizeof(struct ia_css_dvs2_coefficients));
-		sh_css_free(me);
-	}
-}
-
-struct ia_css_dvs_6axis_config *
-ia_css_dvs2_6axis_config_allocate(const struct ia_css_stream *stream)
-{
-	struct ia_css_dvs_6axis_config *dvs_config = NULL;
-	struct ia_css_isp_parameters *params = NULL;
-	unsigned int width_y;
-	unsigned int height_y;
-	unsigned int width_uv;
-	unsigned int height_uv;
-
-	assert(stream != NULL);
-	params = stream->isp_params_configs;
-
-	/* Backward compatibility by default consider pipe as Video*/
-	if (!params || (params && !params->pipe_dvs_6axis_config[IA_CSS_PIPE_ID_VIDEO])) {
-		goto err;
-	}
-
-	dvs_config = (struct ia_css_dvs_6axis_config *)sh_css_calloc(1, sizeof(struct ia_css_dvs_6axis_config));
-	if (!dvs_config)
-		goto err;
-
-	dvs_config->width_y = width_y = params->pipe_dvs_6axis_config[IA_CSS_PIPE_ID_VIDEO]->width_y;
-	dvs_config->height_y = height_y = params->pipe_dvs_6axis_config[IA_CSS_PIPE_ID_VIDEO]->height_y;
-	dvs_config->width_uv = width_uv = params->pipe_dvs_6axis_config[IA_CSS_PIPE_ID_VIDEO]->width_uv;
-	dvs_config->height_uv = height_uv = params->pipe_dvs_6axis_config[IA_CSS_PIPE_ID_VIDEO]->height_uv;
-	IA_CSS_LOG("table Y: W %d H %d", width_y, height_y);
-	IA_CSS_LOG("table UV: W %d H %d", width_uv, height_uv);
-	dvs_config->xcoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof(uint32_t));
-	if (!dvs_config->xcoords_y)
-		goto err;
-
-	dvs_config->ycoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof(uint32_t));
-	if (!dvs_config->ycoords_y)
-		goto err;
-
-	dvs_config->xcoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv * sizeof(uint32_t));
-	if (!dvs_config->xcoords_uv)
-		goto err;
-
-	dvs_config->ycoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv * sizeof(uint32_t));
-	if (!dvs_config->ycoords_uv)
-		goto err;
-
-	return dvs_config;
-err:
-	ia_css_dvs2_6axis_config_free(dvs_config);
-	return NULL;
-}
-
-void
-ia_css_dvs2_6axis_config_free(struct ia_css_dvs_6axis_config *dvs_6axis_config)
-{
-	if (dvs_6axis_config) {
-		sh_css_free(dvs_6axis_config->xcoords_y);
-		sh_css_free(dvs_6axis_config->ycoords_y);
-		sh_css_free(dvs_6axis_config->xcoords_uv);
-		sh_css_free(dvs_6axis_config->ycoords_uv);
-		memset(dvs_6axis_config, 0, sizeof(struct ia_css_dvs_6axis_config));
-		sh_css_free(dvs_6axis_config);
-	}
-}
-
-void
-ia_css_en_dz_capt_pipe(struct ia_css_stream *stream, bool enable)
-{
-	struct ia_css_pipe *pipe;
-	struct ia_css_pipeline *pipeline;
-	struct ia_css_pipeline_stage *stage;
-	enum ia_css_pipe_id pipe_id;
-	enum ia_css_err err;
-	int i;
-
-	if (stream == NULL)
-		return;
-
-	for (i = 0; i < stream->num_pipes; i++) {
-		pipe = stream->pipes[i];
-		pipeline = ia_css_pipe_get_pipeline(pipe);
-		pipe_id = pipeline->pipe_id;
-
-		if (pipe_id == IA_CSS_PIPE_ID_CAPTURE) {
-			err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_CAPTURE_PP, &stage);
-			if (err == IA_CSS_SUCCESS)
-				stage->enable_zoom = enable;
-			break;
-		}
-	}
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params.h
deleted file mode 100644
index 270ec2b60a3e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params.h
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _SH_CSS_PARAMS_H_
-#define _SH_CSS_PARAMS_H_
-
-/*! \file */
-
-/* Forward declaration to break mutual dependency */
-struct ia_css_isp_parameters;
-
-#include <type_support.h>
-#include "ia_css_types.h"
-#include "ia_css_binary.h"
-#include "sh_css_legacy.h"
-
-#include "sh_css_defs.h"	/* SH_CSS_MAX_STAGES */
-#include "ia_css_pipeline.h"
-#include "ia_css_isp_params.h"
-#include "uds/uds_1.0/ia_css_uds_param.h"
-#include "crop/crop_1.0/ia_css_crop_types.h"
-
-
-#define PIX_SHIFT_FILTER_RUN_IN_X 12
-#define PIX_SHIFT_FILTER_RUN_IN_Y 12
-
-#include "ob/ob_1.0/ia_css_ob_param.h"
-/* Isp configurations per stream */
-struct sh_css_isp_param_configs {
-	/* OB (Optical Black) */
-	struct sh_css_isp_ob_stream_config ob;
-};
-
-
-/* Isp parameters per stream */
-struct ia_css_isp_parameters {
-	/* UDS */
-	struct sh_css_sp_uds_params uds[SH_CSS_MAX_STAGES];
-	struct sh_css_isp_param_configs stream_configs;
-	struct ia_css_fpn_table     fpn_config;
-	struct ia_css_vector	    motion_config;
-	const struct ia_css_morph_table   *morph_table;
-	const struct ia_css_shading_table *sc_table;
-	struct ia_css_shading_table *sc_config;
-	struct ia_css_macc_table    macc_table;
-	struct ia_css_gamma_table   gc_table;
-	struct ia_css_ctc_table     ctc_table;
-	struct ia_css_xnr_table     xnr_table;
-
-	struct ia_css_dz_config     dz_config;
-	struct ia_css_3a_config     s3a_config;
-	struct ia_css_wb_config     wb_config;
-	struct ia_css_cc_config     cc_config;
-	struct ia_css_cc_config     yuv2rgb_cc_config;
-	struct ia_css_cc_config     rgb2yuv_cc_config;
-	struct ia_css_tnr_config    tnr_config;
-	struct ia_css_ob_config     ob_config;
-	/*----- DPC configuration -----*/
-	/* The default DPC configuration is retained and currently set
-	 * using the stream configuration. The code generated from genparams
-	 * uses this configuration to set the DPC parameters per stage but this
-	 * will be overwritten by the per pipe configuration */
-	struct ia_css_dp_config     dp_config;
-	/* ------ pipe specific DPC configuration ------ */
-	/* Please note that this implementation is a temporary solution and
-	 * should be replaced by CSS per pipe configuration when the support
-	 * is ready (HSD 1303967698)*/
-	struct ia_css_dp_config     pipe_dp_config[IA_CSS_PIPE_ID_NUM];
-	struct ia_css_nr_config     nr_config;
-	struct ia_css_ee_config     ee_config;
-	struct ia_css_de_config     de_config;
-	struct ia_css_gc_config     gc_config;
-	struct ia_css_anr_config    anr_config;
-	struct ia_css_ce_config     ce_config;
-	struct ia_css_formats_config     formats_config;
-/* ---- deprecated: replaced with pipe_dvs_6axis_config---- */
-	struct ia_css_dvs_6axis_config  *dvs_6axis_config;
-	struct ia_css_ecd_config    ecd_config;
-	struct ia_css_ynr_config    ynr_config;
-	struct ia_css_yee_config    yee_config;
-	struct ia_css_fc_config     fc_config;
-	struct ia_css_cnr_config    cnr_config;
-	struct ia_css_macc_config   macc_config;
-	struct ia_css_ctc_config    ctc_config;
-	struct ia_css_aa_config     aa_config;
-	struct ia_css_aa_config     bds_config;
-	struct ia_css_aa_config     raa_config;
-	struct ia_css_rgb_gamma_table     r_gamma_table;
-	struct ia_css_rgb_gamma_table     g_gamma_table;
-	struct ia_css_rgb_gamma_table     b_gamma_table;
-	struct ia_css_anr_thres     anr_thres;
-	struct ia_css_xnr_config    xnr_config;
-	struct ia_css_xnr3_config   xnr3_config;
-	struct ia_css_uds_config    uds_config;
-	struct ia_css_crop_config   crop_config;
-	struct ia_css_output_config output_config;
-	struct ia_css_dvs_6axis_config  *pipe_dvs_6axis_config[IA_CSS_PIPE_ID_NUM];
-/* ------ deprecated(bz675) : from ------ */
-	struct ia_css_shading_settings shading_settings;
-/* ------ deprecated(bz675) : to ------ */
-	struct ia_css_dvs_coefficients  dvs_coefs;
-	struct ia_css_dvs2_coefficients dvs2_coefs;
-
-	bool isp_params_changed;
-	bool isp_mem_params_changed
-		[IA_CSS_PIPE_ID_NUM][SH_CSS_MAX_STAGES][IA_CSS_NUM_MEMORIES];
-	bool dz_config_changed;
-	bool motion_config_changed;
-	bool dis_coef_table_changed;
-	bool dvs2_coef_table_changed;
-	bool morph_table_changed;
-	bool sc_table_changed;
-	bool sc_table_dirty;
-	unsigned int sc_table_last_pipe_num;
-	bool anr_thres_changed;
-/* ---- deprecated: replaced with pipe_dvs_6axis_config_changed ---- */
-	bool dvs_6axis_config_changed;
-	/* ------ pipe specific DPC configuration ------ */
-	/* Please note that this implementation is a temporary solution and
-	 * should be replaced by CSS per pipe configuration when the support
-	 * is ready (HSD 1303967698) */
-	bool pipe_dpc_config_changed[IA_CSS_PIPE_ID_NUM];
-/* ------ deprecated(bz675) : from ------ */
-	bool shading_settings_changed;
-/* ------ deprecated(bz675) : to ------ */
-	bool pipe_dvs_6axis_config_changed[IA_CSS_PIPE_ID_NUM];
-
-	bool config_changed[IA_CSS_NUM_PARAMETER_IDS];
-
-	unsigned int sensor_binning;
-	/* local buffers, used to re-order the 3a statistics in vmem-format */
-	struct sh_css_ddr_address_map pipe_ddr_ptrs[IA_CSS_PIPE_ID_NUM];
-	struct sh_css_ddr_address_map_size pipe_ddr_ptrs_size[IA_CSS_PIPE_ID_NUM];
-	struct sh_css_ddr_address_map ddr_ptrs;
-	struct sh_css_ddr_address_map_size ddr_ptrs_size;
-	struct ia_css_frame *output_frame; /** Output frame the config is to be applied to (optional) */
-	uint32_t isp_parameters_id; /** Unique ID to track which config was actually applied to a particular frame */
-};
-
-void
-ia_css_params_store_ia_css_host_data(
-	hrt_vaddress ddr_addr,
-	struct ia_css_host_data *data);
-
-enum ia_css_err
-ia_css_params_store_sctbl(
-	    const struct ia_css_pipeline_stage *stage,
-	    hrt_vaddress ddr_addr,
-	    const struct ia_css_shading_table *shading_table);
-
-struct ia_css_host_data *
-ia_css_params_alloc_convert_sctbl(
-	    const struct ia_css_pipeline_stage *stage,
-	    const struct ia_css_shading_table *shading_table);
-
-struct ia_css_isp_config *
-sh_css_pipe_isp_config_get(struct ia_css_pipe *pipe);
-
-/* ipu address allocation/free for gdc lut */
-hrt_vaddress
-sh_css_params_alloc_gdc_lut(void);
-void
-sh_css_params_free_gdc_lut(hrt_vaddress addr);
-
-enum ia_css_err
-sh_css_params_map_and_store_default_gdc_lut(void);
-
-void
-sh_css_params_free_default_gdc_lut(void);
-
-hrt_vaddress
-sh_css_params_get_default_gdc_lut(void);
-
-hrt_vaddress
-sh_css_pipe_get_pp_gdc_lut(const struct ia_css_pipe *pipe);
-
-#endif /* _SH_CSS_PARAMS_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params_internal.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params_internal.h
deleted file mode 100644
index baca24532f9f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_params_internal.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _SH_CSS_PARAMS_INTERNAL_H_
-#define _SH_CSS_PARAMS_INTERNAL_H_
-
-void
-sh_css_param_clear_param_sets(void);
-
-#endif /* _SH_CSS_PARAMS_INTERNAL_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_pipe.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_pipe.c
deleted file mode 100644
index 1f57ffad8921..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_pipe.c
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* This file will contain the code to implement the functions declared in ia_css_pipe.h and ia_css_pipe_public.h
-   and associated helper functions */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_properties.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_properties.c
deleted file mode 100644
index ad46996cfbd3..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_properties.c
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_properties.h"
-#include <assert_support.h>
-#include "ia_css_types.h"
-#include "gdc_device.h"
-
-void
-ia_css_get_properties(struct ia_css_properties *properties)
-{
-	assert(properties != NULL);
-#if defined(HAS_GDC_VERSION_2) || defined(HAS_GDC_VERSION_3)
-/*
- * MW: We don't want to store the coordinates
- * full range in memory: Truncate
- */
-	properties->gdc_coord_one = gdc_get_unity(GDC0_ID)/HRT_GDC_COORD_SCALE;
-#else
-#error "Unknown GDC version"
-#endif
-
-	properties->l1_base_is_index = true;
-
-#if defined(HAS_VAMEM_VERSION_1)
-	properties->vamem_type = IA_CSS_VAMEM_TYPE_1;
-#elif defined(HAS_VAMEM_VERSION_2)
-	properties->vamem_type = IA_CSS_VAMEM_TYPE_2;
-#else
-#error "Unknown VAMEM version"
-#endif
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_shading.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_shading.c
deleted file mode 100644
index 2a2d0f4db44b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_shading.c
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* This file will contain the code to implement the functions declared in ia_css_shading.h
-   and associated helper functions */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_sp.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_sp.c
deleted file mode 100644
index 85263725540d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_sp.c
+++ /dev/null
@@ -1,1803 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "sh_css_sp.h"
-
-#if !defined(HAS_NO_INPUT_FORMATTER)
-#include "input_formatter.h"
-#endif
-
-#include "dma.h"	/* N_DMA_CHANNEL_ID */
-
-#include "ia_css_buffer.h"
-#include "ia_css_binary.h"
-#include "sh_css_hrt.h"
-#include "sh_css_defs.h"
-#include "sh_css_internal.h"
-#include "ia_css_control.h"
-#include "ia_css_debug.h"
-#include "ia_css_debug_pipe.h"
-#include "ia_css_event_public.h"
-#include "ia_css_mmu.h"
-#include "ia_css_stream.h"
-#include "ia_css_isp_param.h"
-#include "sh_css_params.h"
-#include "sh_css_legacy.h"
-#include "ia_css_frame_comm.h"
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#include "ia_css_isys.h"
-#endif
-
-#include "gdc_device.h"				/* HRT_GDC_N */
-
-/*#include "sp.h"*/	/* host2sp_enqueue_frame_data() */
-
-#include "memory_access.h"
-
-#include "assert_support.h"
-#include "platform_support.h"	/* hrt_sleep() */
-
-#include "sw_event_global.h"			/* Event IDs.*/
-#include "ia_css_event.h"
-#include "mmu_device.h"
-#include "ia_css_spctrl.h"
-
-#ifndef offsetof
-#define offsetof(T, x) ((unsigned)&(((T *)0)->x))
-#endif
-
-#define IA_CSS_INCLUDE_CONFIGURATIONS
-#include "ia_css_isp_configs.h"
-#define IA_CSS_INCLUDE_STATES
-#include "ia_css_isp_states.h"
-
-#ifndef ISP2401
-#include "isp/kernels/io_ls/bayer_io_ls/ia_css_bayer_io.host.h"
-#else
-#include "isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h"
-#endif
-
-struct sh_css_sp_group		sh_css_sp_group;
-struct sh_css_sp_stage		sh_css_sp_stage;
-struct sh_css_isp_stage		sh_css_isp_stage;
-static struct sh_css_sp_output		sh_css_sp_output;
-static struct sh_css_sp_per_frame_data per_frame_data;
-
-/* true if SP supports frame loop and host2sp_commands */
-/* For the moment there is only code that sets this bool to true */
-/* TODO: add code that sets this bool to false */
-static bool sp_running;
-
-static enum ia_css_err
-set_output_frame_buffer(const struct ia_css_frame *frame,
-			unsigned idx);
-
-static void
-sh_css_copy_buffer_attr_to_spbuffer(struct ia_css_buffer_sp *dest_buf,
-				const enum sh_css_queue_id queue_id,
-				const hrt_vaddress xmem_addr,
-				const enum ia_css_buffer_type buf_type);
-
-static void
-initialize_frame_buffer_attribute(struct ia_css_buffer_sp *buf_attr);
-
-static void
-initialize_stage_frames(struct ia_css_frames_sp *frames);
-
-/* This data is stored every frame */
-void
-store_sp_group_data(void)
-{
-	per_frame_data.sp_group_addr = sh_css_store_sp_group_to_ddr();
-}
-
-static void
-copy_isp_stage_to_sp_stage(void)
-{
-	/* [WW07.5]type casting will cause potential issues */
-	sh_css_sp_stage.num_stripes = (uint8_t) sh_css_isp_stage.binary_info.iterator.num_stripes;
-	sh_css_sp_stage.row_stripes_height = (uint16_t) sh_css_isp_stage.binary_info.iterator.row_stripes_height;
-	sh_css_sp_stage.row_stripes_overlap_lines = (uint16_t) sh_css_isp_stage.binary_info.iterator.row_stripes_overlap_lines;
-	sh_css_sp_stage.top_cropping = (uint16_t) sh_css_isp_stage.binary_info.pipeline.top_cropping;
-	/* moved to sh_css_sp_init_stage
-	   sh_css_sp_stage.enable.vf_output =
-	   sh_css_isp_stage.binary_info.enable.vf_veceven ||
-	   sh_css_isp_stage.binary_info.num_output_pins > 1;
-	*/
-	sh_css_sp_stage.enable.sdis = sh_css_isp_stage.binary_info.enable.dis;
-	sh_css_sp_stage.enable.s3a = sh_css_isp_stage.binary_info.enable.s3a;
-#ifdef ISP2401
-	sh_css_sp_stage.enable.lace_stats = sh_css_isp_stage.binary_info.enable.lace_stats;
-#endif
-}
-
-void
-store_sp_stage_data(enum ia_css_pipe_id id, unsigned int pipe_num, unsigned stage)
-{
-	unsigned int thread_id;
-	ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
-	copy_isp_stage_to_sp_stage();
-	if (id != IA_CSS_PIPE_ID_COPY)
-		sh_css_sp_stage.isp_stage_addr =
-			sh_css_store_isp_stage_to_ddr(pipe_num, stage);
-	sh_css_sp_group.pipe[thread_id].sp_stage_addr[stage] =
-		sh_css_store_sp_stage_to_ddr(pipe_num, stage);
-
-	/* Clear for next frame */
-	sh_css_sp_stage.program_input_circuit = false;
-}
-
-static void
-store_sp_per_frame_data(const struct ia_css_fw_info *fw)
-{
-	unsigned int HIVE_ADDR_sp_per_frame_data = 0;
-
-	assert(fw != NULL);
-
-	switch (fw->type) {
-	case ia_css_sp_firmware:
-		HIVE_ADDR_sp_per_frame_data = fw->info.sp.per_frame_data;
-		break;
-	case ia_css_acc_firmware:
-		HIVE_ADDR_sp_per_frame_data = fw->info.acc.per_frame_data;
-		break;
-	case ia_css_isp_firmware:
-		return;
-	}
-
-	sp_dmem_store(SP0_ID,
-		(unsigned int)sp_address_of(sp_per_frame_data),
-		&per_frame_data,
-			sizeof(per_frame_data));
-}
-
-static void
-sh_css_store_sp_per_frame_data(enum ia_css_pipe_id pipe_id,
-				   unsigned int pipe_num,
-			       const struct ia_css_fw_info *sp_fw)
-{
-	if (!sp_fw)
-		sp_fw = &sh_css_sp_fw;
-
-	store_sp_stage_data(pipe_id, pipe_num, 0);
-	store_sp_group_data();
-	store_sp_per_frame_data(sp_fw);
-}
-
-#if SP_DEBUG != SP_DEBUG_NONE
-
-void
-sh_css_sp_get_debug_state(struct sh_css_sp_debug_state *state)
-{
-	const struct ia_css_fw_info *fw = &sh_css_sp_fw;
-	unsigned int HIVE_ADDR_sp_output = fw->info.sp.output;
-	unsigned i;
-	unsigned offset = (unsigned int)offsetof(struct sh_css_sp_output, debug)/sizeof(int);
-
-	assert(state != NULL);
-
-	(void)HIVE_ADDR_sp_output; /* To get rid of warning in CRUN */
-	for (i = 0; i < sizeof(*state)/sizeof(int); i++)
-		((unsigned *)state)[i] = load_sp_array_uint(sp_output, i+offset);
-}
-
-#endif
-
-void
-sh_css_sp_start_binary_copy(unsigned int pipe_num, struct ia_css_frame *out_frame,
-			    unsigned two_ppc)
-{
-	enum ia_css_pipe_id pipe_id;
-	unsigned int thread_id;
-	struct sh_css_sp_pipeline *pipe;
-	uint8_t stage_num = 0;
-
-	assert(out_frame != NULL);
-	pipe_id = IA_CSS_PIPE_ID_CAPTURE;
-	ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
-	pipe = &sh_css_sp_group.pipe[thread_id];
-
-	pipe->copy.bin.bytes_available = out_frame->data_bytes;
-	pipe->num_stages = 1;
-	pipe->pipe_id = pipe_id;
-	pipe->pipe_num = pipe_num;
-	pipe->thread_id = thread_id;
-	pipe->pipe_config = 0x0; /* No parameters */
-	pipe->pipe_qos_config = QOS_INVALID;
-
-	if (pipe->inout_port_config == 0) {
-		SH_CSS_PIPE_PORT_CONFIG_SET(pipe->inout_port_config,
-						(uint8_t)SH_CSS_PORT_INPUT,
-						(uint8_t)SH_CSS_HOST_TYPE, 1);
-		SH_CSS_PIPE_PORT_CONFIG_SET(pipe->inout_port_config,
-						(uint8_t)SH_CSS_PORT_OUTPUT,
-						(uint8_t)SH_CSS_HOST_TYPE, 1);
-	}
-	IA_CSS_LOG("pipe_id %d port_config %08x",
-		   pipe->pipe_id, pipe->inout_port_config);
-
-#if !defined(HAS_NO_INPUT_FORMATTER)
-	sh_css_sp_group.config.input_formatter.isp_2ppc = (uint8_t)two_ppc;
-#else
-	(void)two_ppc;
-#endif
-
-	sh_css_sp_stage.num = stage_num;
-	sh_css_sp_stage.stage_type = SH_CSS_SP_STAGE_TYPE;
-	sh_css_sp_stage.func =
-		(unsigned int)IA_CSS_PIPELINE_BIN_COPY;
-
-	set_output_frame_buffer(out_frame, 0);
-
-	/* sp_bin_copy_init on the SP does not deal with dynamica/static yet */
-	/* For now always update the dynamic data from out frames. */
-	sh_css_store_sp_per_frame_data(pipe_id, pipe_num, &sh_css_sp_fw);
-}
-
-static void
-sh_css_sp_start_raw_copy(struct ia_css_frame *out_frame,
-			 unsigned pipe_num,
-			 unsigned two_ppc,
-			 unsigned max_input_width,
-			 enum sh_css_pipe_config_override pipe_conf_override,
-			 unsigned int if_config_index)
-{
-	enum ia_css_pipe_id pipe_id;
-	unsigned int thread_id;
-	uint8_t stage_num = 0;
-	struct sh_css_sp_pipeline *pipe;
-
-	assert(out_frame != NULL);
-
-	{
-		/*
-		 * Clear sh_css_sp_stage for easy debugging.
-		 * program_input_circuit must be saved as it is set outside
-		 * this function.
-		 */
-		uint8_t program_input_circuit;
-		program_input_circuit = sh_css_sp_stage.program_input_circuit;
-		memset(&sh_css_sp_stage, 0, sizeof(sh_css_sp_stage));
-		sh_css_sp_stage.program_input_circuit = program_input_circuit;
-	}
-
-	pipe_id = IA_CSS_PIPE_ID_COPY;
-	ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
-	pipe = &sh_css_sp_group.pipe[thread_id];
-
-	pipe->copy.raw.height	    = out_frame->info.res.height;
-	pipe->copy.raw.width	    = out_frame->info.res.width;
-	pipe->copy.raw.padded_width  = out_frame->info.padded_width;
-	pipe->copy.raw.raw_bit_depth = out_frame->info.raw_bit_depth;
-	pipe->copy.raw.max_input_width = max_input_width;
-	pipe->num_stages = 1;
-	pipe->pipe_id = pipe_id;
-	/* TODO: next indicates from which queues parameters need to be
-		 sampled, needs checking/improvement */
-	if (pipe_conf_override == SH_CSS_PIPE_CONFIG_OVRD_NO_OVRD)
-		pipe->pipe_config =
-			(SH_CSS_PIPE_CONFIG_SAMPLE_PARAMS << thread_id);
-	else
-		pipe->pipe_config = pipe_conf_override;
-
-	pipe->pipe_qos_config = QOS_INVALID;
-
-	if (pipe->inout_port_config == 0) {
-		SH_CSS_PIPE_PORT_CONFIG_SET(pipe->inout_port_config,
-						(uint8_t)SH_CSS_PORT_INPUT,
-						(uint8_t)SH_CSS_HOST_TYPE, 1);
-		SH_CSS_PIPE_PORT_CONFIG_SET(pipe->inout_port_config,
-						(uint8_t)SH_CSS_PORT_OUTPUT,
-						(uint8_t)SH_CSS_HOST_TYPE, 1);
-	}
-	IA_CSS_LOG("pipe_id %d port_config %08x",
-		   pipe->pipe_id, pipe->inout_port_config);
-
-#if !defined(HAS_NO_INPUT_FORMATTER)
-	sh_css_sp_group.config.input_formatter.isp_2ppc = (uint8_t)two_ppc;
-#else
-	(void)two_ppc;
-#endif
-
-	sh_css_sp_stage.num = stage_num;
-	sh_css_sp_stage.xmem_bin_addr = 0x0;
-	sh_css_sp_stage.stage_type = SH_CSS_SP_STAGE_TYPE;
-	sh_css_sp_stage.func = (unsigned int)IA_CSS_PIPELINE_RAW_COPY;
-	sh_css_sp_stage.if_config_index = (uint8_t) if_config_index;
-	set_output_frame_buffer(out_frame, 0);
-
-	ia_css_debug_pipe_graph_dump_sp_raw_copy(out_frame);
-}
-
-static void
-sh_css_sp_start_isys_copy(struct ia_css_frame *out_frame,
-	unsigned pipe_num, unsigned max_input_width, unsigned int if_config_index)
-{
-	enum ia_css_pipe_id pipe_id;
-	unsigned int thread_id;
-	uint8_t stage_num = 0;
-	struct sh_css_sp_pipeline *pipe;
-#if defined SH_CSS_ENABLE_METADATA
-	enum sh_css_queue_id queue_id;
-#endif
-
-	assert(out_frame != NULL);
-
-	{
-		/*
-		 * Clear sh_css_sp_stage for easy debugging.
-		 * program_input_circuit must be saved as it is set outside
-		 * this function.
-		 */
-		uint8_t program_input_circuit;
-		program_input_circuit = sh_css_sp_stage.program_input_circuit;
-		memset(&sh_css_sp_stage, 0, sizeof(sh_css_sp_stage));
-		sh_css_sp_stage.program_input_circuit = program_input_circuit;
-	}
-
-	pipe_id = IA_CSS_PIPE_ID_COPY;
-	ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
-	pipe = &sh_css_sp_group.pipe[thread_id];
-
-	pipe->copy.raw.height		= out_frame->info.res.height;
-	pipe->copy.raw.width		= out_frame->info.res.width;
-	pipe->copy.raw.padded_width	= out_frame->info.padded_width;
-	pipe->copy.raw.raw_bit_depth	= out_frame->info.raw_bit_depth;
-	pipe->copy.raw.max_input_width	= max_input_width;
-	pipe->num_stages		= 1;
-	pipe->pipe_id			= pipe_id;
-	pipe->pipe_config		= 0x0;	/* No parameters */
-	pipe->pipe_qos_config		= QOS_INVALID;
-
-	initialize_stage_frames(&sh_css_sp_stage.frames);
-	sh_css_sp_stage.num = stage_num;
-	sh_css_sp_stage.xmem_bin_addr = 0x0;
-	sh_css_sp_stage.stage_type = SH_CSS_SP_STAGE_TYPE;
-	sh_css_sp_stage.func = (unsigned int)IA_CSS_PIPELINE_ISYS_COPY;
-	sh_css_sp_stage.if_config_index = (uint8_t) if_config_index;
-
-	set_output_frame_buffer(out_frame, 0);
-
-#if defined SH_CSS_ENABLE_METADATA
-	if (pipe->metadata.height > 0) {
-		ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_METADATA, thread_id, &queue_id);
-		sh_css_copy_buffer_attr_to_spbuffer(&sh_css_sp_stage.frames.metadata_buf, queue_id, mmgr_EXCEPTION, IA_CSS_BUFFER_TYPE_METADATA);
-	}
-#endif
-
-	ia_css_debug_pipe_graph_dump_sp_raw_copy(out_frame);
-}
-
-unsigned int
-sh_css_sp_get_binary_copy_size(void)
-{
-	const struct ia_css_fw_info *fw = &sh_css_sp_fw;
-	unsigned int HIVE_ADDR_sp_output = fw->info.sp.output;
-	unsigned int offset = (unsigned int)offsetof(struct sh_css_sp_output,
-				bin_copy_bytes_copied) / sizeof(int);
-	(void)HIVE_ADDR_sp_output; /* To get rid of warning in CRUN */
-	return load_sp_array_uint(sp_output, offset);
-}
-
-unsigned int
-sh_css_sp_get_sw_interrupt_value(unsigned int irq)
-{
-	const struct ia_css_fw_info *fw = &sh_css_sp_fw;
-	unsigned int HIVE_ADDR_sp_output = fw->info.sp.output;
-	unsigned int offset = (unsigned int)offsetof(struct sh_css_sp_output, sw_interrupt_value)
-				/ sizeof(int);
-	(void)HIVE_ADDR_sp_output; /* To get rid of warning in CRUN */
-	return load_sp_array_uint(sp_output, offset+irq);
-}
-
-static void
-sh_css_copy_buffer_attr_to_spbuffer(struct ia_css_buffer_sp *dest_buf,
-				const enum sh_css_queue_id queue_id,
-				const hrt_vaddress xmem_addr,
-				const enum ia_css_buffer_type buf_type)
-{
-	assert(buf_type < IA_CSS_NUM_BUFFER_TYPE);
-	if (queue_id > SH_CSS_INVALID_QUEUE_ID) {
-		/*
-		 * value >=0 indicates that function init_frame_pointers()
-		 * should use the dynamic data address
-		 */
-		assert(queue_id < SH_CSS_MAX_NUM_QUEUES);
-
-		/* Klocwork assumes assert can be disabled;
-		   Since we can get there with any type, and it does not
-		   know that frame_in->dynamic_data_index can only be set
-		   for one of the types in the assert) it has to assume we
-		   can get here for any type. however this could lead to an
-		   out of bounds reference when indexing buf_type about 10
-		   lines below. In order to satisfy KW an additional if
-		   has been added. This one will always yield true.
-		 */
-		if ((queue_id < SH_CSS_MAX_NUM_QUEUES))
-		{
-			dest_buf->buf_src.queue_id = queue_id;
-		}
-	} else {
-		assert(xmem_addr != mmgr_EXCEPTION);
-		dest_buf->buf_src.xmem_addr = xmem_addr;
-	}
-	dest_buf->buf_type = buf_type;
-}
-
-static void
-sh_css_copy_frame_to_spframe(struct ia_css_frame_sp *sp_frame_out,
-				const struct ia_css_frame *frame_in)
-{
-	assert(frame_in != NULL);
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
-		"sh_css_copy_frame_to_spframe():\n");
-
-
-	sh_css_copy_buffer_attr_to_spbuffer(&sp_frame_out->buf_attr,
-					frame_in->dynamic_queue_id,
-					frame_in->data,
-					frame_in->buf_type);
-
-	ia_css_frame_info_to_frame_sp_info(&sp_frame_out->info, &frame_in->info);
-
-	switch (frame_in->info.format) {
-	case IA_CSS_FRAME_FORMAT_RAW_PACKED:
-	case IA_CSS_FRAME_FORMAT_RAW:
-		sp_frame_out->planes.raw.offset = frame_in->planes.raw.offset;
-		break;
-	case IA_CSS_FRAME_FORMAT_RGB565:
-	case IA_CSS_FRAME_FORMAT_RGBA888:
-		sp_frame_out->planes.rgb.offset = frame_in->planes.rgb.offset;
-		break;
-	case IA_CSS_FRAME_FORMAT_PLANAR_RGB888:
-		sp_frame_out->planes.planar_rgb.r.offset =
-			frame_in->planes.planar_rgb.r.offset;
-		sp_frame_out->planes.planar_rgb.g.offset =
-			frame_in->planes.planar_rgb.g.offset;
-		sp_frame_out->planes.planar_rgb.b.offset =
-			frame_in->planes.planar_rgb.b.offset;
-		break;
-	case IA_CSS_FRAME_FORMAT_YUYV:
-	case IA_CSS_FRAME_FORMAT_UYVY:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_8:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_LEGACY_YUV420_8:
-	case IA_CSS_FRAME_FORMAT_YUV_LINE:
-		sp_frame_out->planes.yuyv.offset = frame_in->planes.yuyv.offset;
-		break;
-	case IA_CSS_FRAME_FORMAT_NV11:
-	case IA_CSS_FRAME_FORMAT_NV12:
-	case IA_CSS_FRAME_FORMAT_NV12_16:
-	case IA_CSS_FRAME_FORMAT_NV12_TILEY:
-	case IA_CSS_FRAME_FORMAT_NV21:
-	case IA_CSS_FRAME_FORMAT_NV16:
-	case IA_CSS_FRAME_FORMAT_NV61:
-		sp_frame_out->planes.nv.y.offset =
-			frame_in->planes.nv.y.offset;
-		sp_frame_out->planes.nv.uv.offset =
-			frame_in->planes.nv.uv.offset;
-		break;
-	case IA_CSS_FRAME_FORMAT_YUV420:
-	case IA_CSS_FRAME_FORMAT_YUV422:
-	case IA_CSS_FRAME_FORMAT_YUV444:
-	case IA_CSS_FRAME_FORMAT_YUV420_16:
-	case IA_CSS_FRAME_FORMAT_YUV422_16:
-	case IA_CSS_FRAME_FORMAT_YV12:
-	case IA_CSS_FRAME_FORMAT_YV16:
-		sp_frame_out->planes.yuv.y.offset =
-			frame_in->planes.yuv.y.offset;
-		sp_frame_out->planes.yuv.u.offset =
-			frame_in->planes.yuv.u.offset;
-		sp_frame_out->planes.yuv.v.offset =
-			frame_in->planes.yuv.v.offset;
-		break;
-	case IA_CSS_FRAME_FORMAT_QPLANE6:
-		sp_frame_out->planes.plane6.r.offset =
-			frame_in->planes.plane6.r.offset;
-		sp_frame_out->planes.plane6.r_at_b.offset =
-			frame_in->planes.plane6.r_at_b.offset;
-		sp_frame_out->planes.plane6.gr.offset =
-			frame_in->planes.plane6.gr.offset;
-		sp_frame_out->planes.plane6.gb.offset =
-			frame_in->planes.plane6.gb.offset;
-		sp_frame_out->planes.plane6.b.offset =
-			frame_in->planes.plane6.b.offset;
-		sp_frame_out->planes.plane6.b_at_r.offset =
-			frame_in->planes.plane6.b_at_r.offset;
-		break;
-	case IA_CSS_FRAME_FORMAT_BINARY_8:
-		sp_frame_out->planes.binary.data.offset =
-			frame_in->planes.binary.data.offset;
-		break;
-	default:
-		/* This should not happen, but in case it does,
-		 * nullify the planes
-		 */
-		memset(&sp_frame_out->planes, 0, sizeof(sp_frame_out->planes));
-		break;
-	}
-
-}
-
-static enum ia_css_err
-set_input_frame_buffer(const struct ia_css_frame *frame)
-{
-	if (frame == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	switch (frame->info.format) {
-	case IA_CSS_FRAME_FORMAT_QPLANE6:
-	case IA_CSS_FRAME_FORMAT_YUV420_16:
-	case IA_CSS_FRAME_FORMAT_RAW_PACKED:
-	case IA_CSS_FRAME_FORMAT_RAW:
-	case IA_CSS_FRAME_FORMAT_YUV420:
-	case IA_CSS_FRAME_FORMAT_YUYV:
-	case IA_CSS_FRAME_FORMAT_YUV_LINE:
-	case IA_CSS_FRAME_FORMAT_NV12:
-	case IA_CSS_FRAME_FORMAT_NV12_16:
-	case IA_CSS_FRAME_FORMAT_NV12_TILEY:
-	case IA_CSS_FRAME_FORMAT_NV21:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_8:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_LEGACY_YUV420_8:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_10:
-		break;
-	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	sh_css_copy_frame_to_spframe(&sh_css_sp_stage.frames.in, frame);
-
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-set_output_frame_buffer(const struct ia_css_frame *frame,
-			unsigned idx)
-{
-	if (frame == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	switch (frame->info.format) {
-	case IA_CSS_FRAME_FORMAT_YUV420:
-	case IA_CSS_FRAME_FORMAT_YUV422:
-	case IA_CSS_FRAME_FORMAT_YUV444:
-	case IA_CSS_FRAME_FORMAT_YV12:
-	case IA_CSS_FRAME_FORMAT_YV16:
-	case IA_CSS_FRAME_FORMAT_YUV420_16:
-	case IA_CSS_FRAME_FORMAT_YUV422_16:
-	case IA_CSS_FRAME_FORMAT_NV11:
-	case IA_CSS_FRAME_FORMAT_NV12:
-	case IA_CSS_FRAME_FORMAT_NV12_16:
-	case IA_CSS_FRAME_FORMAT_NV12_TILEY:
-	case IA_CSS_FRAME_FORMAT_NV16:
-	case IA_CSS_FRAME_FORMAT_NV21:
-	case IA_CSS_FRAME_FORMAT_NV61:
-	case IA_CSS_FRAME_FORMAT_YUYV:
-	case IA_CSS_FRAME_FORMAT_UYVY:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_8:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_LEGACY_YUV420_8:
-	case IA_CSS_FRAME_FORMAT_YUV_LINE:
-	case IA_CSS_FRAME_FORMAT_RGB565:
-	case IA_CSS_FRAME_FORMAT_RGBA888:
-	case IA_CSS_FRAME_FORMAT_PLANAR_RGB888:
-	case IA_CSS_FRAME_FORMAT_RAW:
-	case IA_CSS_FRAME_FORMAT_RAW_PACKED:
-	case IA_CSS_FRAME_FORMAT_QPLANE6:
-	case IA_CSS_FRAME_FORMAT_BINARY_8:
-		break;
-	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-	sh_css_copy_frame_to_spframe(&sh_css_sp_stage.frames.out[idx], frame);
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-set_view_finder_buffer(const struct ia_css_frame *frame)
-{
-	if (frame == NULL)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-
-	switch (frame->info.format) {
-	/* the dual output pin */
-	case IA_CSS_FRAME_FORMAT_NV12:
-	case IA_CSS_FRAME_FORMAT_NV12_16:
-	case IA_CSS_FRAME_FORMAT_NV21:
-	case IA_CSS_FRAME_FORMAT_YUYV:
-	case IA_CSS_FRAME_FORMAT_UYVY:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_8:
-	case IA_CSS_FRAME_FORMAT_CSI_MIPI_LEGACY_YUV420_8:
-	case IA_CSS_FRAME_FORMAT_YUV420:
-	case IA_CSS_FRAME_FORMAT_YV12:
-	case IA_CSS_FRAME_FORMAT_NV12_TILEY:
-
-	/* for vf_veceven */
-	case IA_CSS_FRAME_FORMAT_YUV_LINE:
-		break;
-	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	}
-
-	sh_css_copy_frame_to_spframe(&sh_css_sp_stage.frames.out_vf, frame);
-	return IA_CSS_SUCCESS;
-}
-
-#if !defined(HAS_NO_INPUT_FORMATTER)
-void sh_css_sp_set_if_configs(
-	const input_formatter_cfg_t	*config_a,
-	const input_formatter_cfg_t	*config_b,
-	const uint8_t		if_config_index
-	)
-{
-	assert(if_config_index < SH_CSS_MAX_IF_CONFIGS);
-	assert(config_a != NULL);
-
-	sh_css_sp_group.config.input_formatter.set[if_config_index].config_a = *config_a;
-	sh_css_sp_group.config.input_formatter.a_changed = true;
-
-	if (config_b != NULL) {
-		sh_css_sp_group.config.input_formatter.set[if_config_index].config_b = *config_b;
-		sh_css_sp_group.config.input_formatter.b_changed = true;
-	}
-
-	return;
-}
-#endif
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-void
-sh_css_sp_program_input_circuit(int fmt_type,
-				int ch_id,
-				enum ia_css_input_mode input_mode)
-{
-	sh_css_sp_group.config.input_circuit.no_side_band = false;
-	sh_css_sp_group.config.input_circuit.fmt_type     = fmt_type;
-	sh_css_sp_group.config.input_circuit.ch_id	      = ch_id;
-	sh_css_sp_group.config.input_circuit.input_mode   = input_mode;
-/*
- * The SP group is only loaded at SP boot time and is read once
- * change flags as "input_circuit_cfg_changed" must be reset on the SP
- */
-	sh_css_sp_group.config.input_circuit_cfg_changed = true;
-	sh_css_sp_stage.program_input_circuit = true;
-}
-#endif
-
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-void
-sh_css_sp_configure_sync_gen(int width, int height,
-			     int hblank_cycles,
-			     int vblank_cycles)
-{
-	sh_css_sp_group.config.sync_gen.width	       = width;
-	sh_css_sp_group.config.sync_gen.height	       = height;
-	sh_css_sp_group.config.sync_gen.hblank_cycles = hblank_cycles;
-	sh_css_sp_group.config.sync_gen.vblank_cycles = vblank_cycles;
-}
-
-void
-sh_css_sp_configure_tpg(int x_mask,
-			int y_mask,
-			int x_delta,
-			int y_delta,
-			int xy_mask)
-{
-	sh_css_sp_group.config.tpg.x_mask  = x_mask;
-	sh_css_sp_group.config.tpg.y_mask  = y_mask;
-	sh_css_sp_group.config.tpg.x_delta = x_delta;
-	sh_css_sp_group.config.tpg.y_delta = y_delta;
-	sh_css_sp_group.config.tpg.xy_mask = xy_mask;
-}
-
-void
-sh_css_sp_configure_prbs(int seed)
-{
-	sh_css_sp_group.config.prbs.seed = seed;
-}
-#endif
-
-void
-sh_css_sp_configure_enable_raw_pool_locking(bool lock_all)
-{
-	sh_css_sp_group.config.enable_raw_pool_locking = true;
-	sh_css_sp_group.config.lock_all = lock_all;
-}
-
-void
-sh_css_sp_enable_isys_event_queue(bool enable)
-{
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	sh_css_sp_group.config.enable_isys_event_queue = enable;
-#else
-	(void)enable;
-#endif
-}
-
-void
-sh_css_sp_set_disable_continuous_viewfinder(bool flag)
-{
-	sh_css_sp_group.config.disable_cont_vf = flag;
-}
-
-static enum ia_css_err
-sh_css_sp_write_frame_pointers(const struct sh_css_binary_args *args)
-{
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	int i;
-
-	assert(args != NULL);
-
-	if (args->in_frame)
-		err = set_input_frame_buffer(args->in_frame);
-	if (err == IA_CSS_SUCCESS && args->out_vf_frame)
-		err = set_view_finder_buffer(args->out_vf_frame);
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		if (err == IA_CSS_SUCCESS && args->out_frame[i])
-			err = set_output_frame_buffer(args->out_frame[i], i);
-	}
-
-	/* we don't pass this error back to the upper layer, so we add a assert here
-	   because we actually hit the error here but it still works by accident... */
-	if (err != IA_CSS_SUCCESS) assert(false);
-	return err;
-}
-
-static void
-sh_css_sp_init_group(bool two_ppc,
-		     enum atomisp_input_format input_format,
-		     bool no_isp_sync,
-		     uint8_t if_config_index)
-{
-#if !defined(HAS_NO_INPUT_FORMATTER)
-	sh_css_sp_group.config.input_formatter.isp_2ppc = two_ppc;
-#else
-	(void)two_ppc;
-#endif
-
-	sh_css_sp_group.config.no_isp_sync = (uint8_t)no_isp_sync;
-	/* decide whether the frame is processed online or offline */
-	if (if_config_index == SH_CSS_IF_CONFIG_NOT_NEEDED) return;
-#if !defined(HAS_NO_INPUT_FORMATTER)
-	assert(if_config_index < SH_CSS_MAX_IF_CONFIGS);
-	sh_css_sp_group.config.input_formatter.set[if_config_index].stream_format = input_format;
-#else
-	(void)input_format;
-#endif
-}
-
-void
-sh_css_stage_write_binary_info(struct ia_css_binary_info *info)
-{
-	assert(info != NULL);
-	sh_css_isp_stage.binary_info = *info;
-}
-
-static enum ia_css_err
-copy_isp_mem_if_to_ddr(struct ia_css_binary *binary)
-{
-	enum ia_css_err err;
-
-	err = ia_css_isp_param_copy_isp_mem_if_to_ddr(
-		&binary->css_params,
-		&binary->mem_params,
-		IA_CSS_PARAM_CLASS_CONFIG);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	err = ia_css_isp_param_copy_isp_mem_if_to_ddr(
-		&binary->css_params,
-		&binary->mem_params,
-		IA_CSS_PARAM_CLASS_STATE);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-	return IA_CSS_SUCCESS;
-}
-
-static bool
-is_sp_stage(struct ia_css_pipeline_stage *stage)
-{
-	assert(stage != NULL);
-	return stage->sp_func != IA_CSS_PIPELINE_NO_FUNC;
-}
-
-static enum ia_css_err
-configure_isp_from_args(
-	const struct sh_css_sp_pipeline *pipeline,
-	const struct ia_css_binary      *binary,
-	const struct sh_css_binary_args *args,
-	bool two_ppc,
-	bool deinterleaved)
-{
-#ifdef ISP2401
-	struct ia_css_pipe *pipe = find_pipe_by_num(pipeline->pipe_num);
-	const struct ia_css_resolution *res;
-
-#endif
-	ia_css_fpn_configure(binary,  &binary->in_frame_info);
-	ia_css_crop_configure(binary, &args->delay_frames[0]->info);
-	ia_css_qplane_configure(pipeline, binary, &binary->in_frame_info);
-	ia_css_output0_configure(binary, &args->out_frame[0]->info);
-	ia_css_output1_configure(binary, &args->out_vf_frame->info);
-	ia_css_copy_output_configure(binary, args->copy_output);
-	ia_css_output0_configure(binary, &args->out_frame[0]->info);
-#ifdef ISP2401
-	ia_css_sc_configure(binary, pipeline->shading.internal_frame_origin_x_bqs_on_sctbl,
-				    pipeline->shading.internal_frame_origin_y_bqs_on_sctbl);
-#endif
-	ia_css_iterator_configure(binary, &args->in_frame->info);
-	ia_css_dvs_configure(binary, &args->out_frame[0]->info);
-	ia_css_output_configure(binary, &args->out_frame[0]->info);
-	ia_css_raw_configure(pipeline, binary, &args->in_frame->info, &binary->in_frame_info, two_ppc, deinterleaved);
-	ia_css_ref_configure(binary, (const struct ia_css_frame **)args->delay_frames, pipeline->dvs_frame_delay);
-	ia_css_tnr_configure(binary, (const struct ia_css_frame **)args->tnr_frames);
-	ia_css_bayer_io_config(binary, args);
-	return IA_CSS_SUCCESS;
-}
-
-static void
-initialize_isp_states(const struct ia_css_binary *binary)
-{
-	unsigned int i;
-
-	if (!binary->info->mem_offsets.offsets.state)
-		return;
-	for (i = 0; i < IA_CSS_NUM_STATE_IDS; i++) {
-		ia_css_kernel_init_state[i](binary);
-	}
-}
-
-static void
-initialize_frame_buffer_attribute(struct ia_css_buffer_sp *buf_attr)
-{
-	buf_attr->buf_src.queue_id = SH_CSS_INVALID_QUEUE_ID;
-	buf_attr->buf_type = IA_CSS_BUFFER_TYPE_INVALID;
-}
-
-static void
-initialize_stage_frames(struct ia_css_frames_sp *frames)
-{
-	unsigned int i;
-
-	initialize_frame_buffer_attribute(&frames->in.buf_attr);
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		initialize_frame_buffer_attribute(&frames->out[i].buf_attr);
-	}
-	initialize_frame_buffer_attribute(&frames->out_vf.buf_attr);
-	initialize_frame_buffer_attribute(&frames->s3a_buf);
-	initialize_frame_buffer_attribute(&frames->dvs_buf);
-#if defined SH_CSS_ENABLE_METADATA
-	initialize_frame_buffer_attribute(&frames->metadata_buf);
-#endif
-}
-
-static enum ia_css_err
-sh_css_sp_init_stage(struct ia_css_binary *binary,
-		    const char *binary_name,
-		    const struct ia_css_blob_info *blob_info,
-		    const struct sh_css_binary_args *args,
-		    unsigned int pipe_num,
-		    unsigned stage,
-		    bool xnr,
-		    const struct ia_css_isp_param_css_segments *isp_mem_if,
-		    unsigned int if_config_index,
-		    bool two_ppc)
-{
-	const struct ia_css_binary_xinfo *xinfo;
-	const struct ia_css_binary_info  *info;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	int i;
-	struct ia_css_pipe *pipe = NULL;
-	unsigned int thread_id;
-	enum sh_css_queue_id queue_id;
-	bool continuous = sh_css_continuous_is_enabled((uint8_t)pipe_num);
-
-	assert(binary != NULL);
-	assert(blob_info != NULL);
-	assert(args != NULL);
-	assert(isp_mem_if != NULL);
-
-	xinfo = binary->info;
-	info  = &xinfo->sp;
-	{
-		/*
-		 * Clear sh_css_sp_stage for easy debugging.
-		 * program_input_circuit must be saved as it is set outside
-		 * this function.
-		 */
-		uint8_t program_input_circuit;
-		program_input_circuit = sh_css_sp_stage.program_input_circuit;
-		memset(&sh_css_sp_stage, 0, sizeof(sh_css_sp_stage));
-		sh_css_sp_stage.program_input_circuit = (uint8_t)program_input_circuit;
-	}
-
-	ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
-
-	if (info == NULL) {
-		sh_css_sp_group.pipe[thread_id].sp_stage_addr[stage] = mmgr_NULL;
-		return IA_CSS_SUCCESS;
-	}
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
-	(void)continuous;
-	sh_css_sp_stage.deinterleaved = 0;
-#else
-	sh_css_sp_stage.deinterleaved = ((stage == 0) && continuous);
-#endif
-
-	initialize_stage_frames(&sh_css_sp_stage.frames);
-	/*
-	 * TODO: Make the Host dynamically determine
-	 * the stage type.
-	 */
-	sh_css_sp_stage.stage_type = SH_CSS_ISP_STAGE_TYPE;
-	sh_css_sp_stage.num		= (uint8_t)stage;
-	sh_css_sp_stage.isp_online	= (uint8_t)binary->online;
-	sh_css_sp_stage.isp_copy_vf     = (uint8_t)args->copy_vf;
-	sh_css_sp_stage.isp_copy_output = (uint8_t)args->copy_output;
-	sh_css_sp_stage.enable.vf_output = (args->out_vf_frame != NULL);
-
-	/* Copy the frame infos first, to be overwritten by the frames,
-	   if these are present.
-	*/
-	sh_css_sp_stage.frames.effective_in_res.width = binary->effective_in_frame_res.width;
-	sh_css_sp_stage.frames.effective_in_res.height = binary->effective_in_frame_res.height;
-
-	ia_css_frame_info_to_frame_sp_info(&sh_css_sp_stage.frames.in.info,
-				&binary->in_frame_info);
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
-		ia_css_frame_info_to_frame_sp_info(&sh_css_sp_stage.frames.out[i].info,
-					&binary->out_frame_info[i]);
-	}
-	ia_css_frame_info_to_frame_sp_info(&sh_css_sp_stage.frames.internal_frame_info,
-				&binary->internal_frame_info);
-	sh_css_sp_stage.dvs_envelope.width    = binary->dvs_envelope.width;
-	sh_css_sp_stage.dvs_envelope.height   = binary->dvs_envelope.height;
-	sh_css_sp_stage.isp_pipe_version      = (uint8_t)info->pipeline.isp_pipe_version;
-	sh_css_sp_stage.isp_deci_log_factor   = (uint8_t)binary->deci_factor_log2;
-	sh_css_sp_stage.isp_vf_downscale_bits = (uint8_t)binary->vf_downscale_log2;
-
-	sh_css_sp_stage.if_config_index = (uint8_t) if_config_index;
-
-	sh_css_sp_stage.sp_enable_xnr = (uint8_t)xnr;
-	sh_css_sp_stage.xmem_bin_addr = xinfo->xmem_addr;
-	sh_css_sp_stage.xmem_map_addr = sh_css_params_ddr_address_map();
-	sh_css_isp_stage.blob_info = *blob_info;
-	sh_css_stage_write_binary_info((struct ia_css_binary_info *)info);
-
-	/* Make sure binary name is smaller than allowed string size */
-	assert(strlen(binary_name) < SH_CSS_MAX_BINARY_NAME-1);
-	strncpy(sh_css_isp_stage.binary_name, binary_name, SH_CSS_MAX_BINARY_NAME-1);
-	sh_css_isp_stage.binary_name[SH_CSS_MAX_BINARY_NAME - 1] = 0;
-	sh_css_isp_stage.mem_initializers = *isp_mem_if;
-
-	/*
-	 * Even when a stage does not need uds and does not params,
-	 * ia_css_uds_sp_scale_params() seems to be called (needs
-	 * further investigation). This function can not deal with
-	 * dx, dy = {0, 0}
-	 */
-
-	err = sh_css_sp_write_frame_pointers(args);
-	/* TODO: move it to a better place */
-	if (binary->info->sp.enable.s3a) {
-		ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_3A_STATISTICS, thread_id, &queue_id);
-		sh_css_copy_buffer_attr_to_spbuffer(&sh_css_sp_stage.frames.s3a_buf, queue_id, mmgr_EXCEPTION, IA_CSS_BUFFER_TYPE_3A_STATISTICS);
-	}
-	if (binary->info->sp.enable.dis) {
-		ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_DIS_STATISTICS, thread_id, &queue_id);
-		sh_css_copy_buffer_attr_to_spbuffer(&sh_css_sp_stage.frames.dvs_buf, queue_id, mmgr_EXCEPTION, IA_CSS_BUFFER_TYPE_DIS_STATISTICS);
-	}
-#if defined SH_CSS_ENABLE_METADATA
-	ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_METADATA, thread_id, &queue_id);
-	sh_css_copy_buffer_attr_to_spbuffer(&sh_css_sp_stage.frames.metadata_buf, queue_id, mmgr_EXCEPTION, IA_CSS_BUFFER_TYPE_METADATA);
-#endif
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-#ifndef ISP2401
-	if (args->in_frame) {
-		pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num);
-		if (pipe == NULL)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		ia_css_get_crop_offsets(pipe, &args->in_frame->info);
-	} else if (&binary->in_frame_info) {
-		pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num);
-		if (pipe == NULL)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		ia_css_get_crop_offsets(pipe, &binary->in_frame_info);
-#else
-	if (stage == 0) {
-		if (args->in_frame) {
-			pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num);
-			if (pipe == NULL)
-				return IA_CSS_ERR_INTERNAL_ERROR;
-			ia_css_get_crop_offsets(pipe, &args->in_frame->info);
-		} else if (&binary->in_frame_info) {
-			pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num);
-			if (pipe == NULL)
-				return IA_CSS_ERR_INTERNAL_ERROR;
-			ia_css_get_crop_offsets(pipe, &binary->in_frame_info);
-		}
-#endif
-	}
-#else
-	(void)pipe; /*avoid build warning*/
-#endif
-
-	err = configure_isp_from_args(&sh_css_sp_group.pipe[thread_id],
-			binary, args, two_ppc, sh_css_sp_stage.deinterleaved);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-	initialize_isp_states(binary);
-
-	/* we do this only for preview pipe because in fill_binary_info function
-	 * we assign vf_out res to out res, but for ISP internal processing, we need
-	 * the original out res. for video pipe, it has two output pins --- out and
-	 * vf_out, so it can keep these two resolutions already. */
-	if (binary->info->sp.pipeline.mode == IA_CSS_BINARY_MODE_PREVIEW &&
-		(binary->vf_downscale_log2 > 0)) {
-		/* TODO: Remove this after preview output decimation is fixed
-		 * by configuring out&vf info fiels properly */
-		sh_css_sp_stage.frames.out[0].info.padded_width
-			<<= binary->vf_downscale_log2;
-		sh_css_sp_stage.frames.out[0].info.res.width
-			<<= binary->vf_downscale_log2;
-		sh_css_sp_stage.frames.out[0].info.res.height
-			<<= binary->vf_downscale_log2;
-	}
-	err = copy_isp_mem_if_to_ddr(binary);
-	if (err != IA_CSS_SUCCESS)
-		return err;
-
-	return IA_CSS_SUCCESS;
-}
-
-static enum ia_css_err
-sp_init_stage(struct ia_css_pipeline_stage *stage,
-	      unsigned int pipe_num,
-	      bool xnr,
-	      unsigned int if_config_index,
-	      bool two_ppc)
-{
-	struct ia_css_binary *binary;
-	const struct ia_css_fw_info *firmware;
-	const struct sh_css_binary_args *args;
-	unsigned stage_num;
-/*
- * Initialiser required because of the "else" path below.
- * Is this a valid path ?
- */
-	const char *binary_name = "";
-	const struct ia_css_binary_xinfo *info = NULL;
-	/* note: the var below is made static as it is quite large;
-	   if it is not static it ends up on the stack which could
-	   cause issues for drivers
-	*/
-	static struct ia_css_binary tmp_binary;
-	const struct ia_css_blob_info *blob_info = NULL;
-	struct ia_css_isp_param_css_segments isp_mem_if;
-	/* LA: should be ia_css_data, should not contain host pointer.
-	   However, CSS/DDR pointer is not available yet.
-	   Hack is to store it in params->ddr_ptrs and then copy it late in the SP just before vmem init.
-	   TODO: Call this after CSS/DDR allocation and store that pointer.
-	   Best is to allocate it at stage creation time together with host pointer.
-	   Remove vmem from params.
-	*/
-	struct ia_css_isp_param_css_segments *mem_if = &isp_mem_if;
-
-	enum ia_css_err err = IA_CSS_SUCCESS;
-
-	assert(stage != NULL);
-
-	binary = stage->binary;
-	firmware = stage->firmware;
-	args = &stage->args;
-	stage_num = stage->stage_num;
-
-
-	if (binary) {
-		info = binary->info;
-		binary_name = (const char *)(info->blob->name);
-		blob_info = &info->blob->header.blob;
-		ia_css_init_memory_interface(mem_if, &binary->mem_params, &binary->css_params);
-	} else if (firmware) {
-		const struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS] = {NULL};
-		if (args->out_frame[0])
-			out_infos[0] = &args->out_frame[0]->info;
-		info = &firmware->info.isp;
-		ia_css_binary_fill_info(info, false, false,
-			    ATOMISP_INPUT_FORMAT_RAW_10,
-			    args->in_frame  ? &args->in_frame->info  : NULL,
-			    NULL,
-				out_infos,
-			    args->out_vf_frame ? &args->out_vf_frame->info
-						: NULL,
-			    &tmp_binary,
-			    NULL,
-			    -1, true);
-		binary = &tmp_binary;
-		binary->info = info;
-		binary_name = IA_CSS_EXT_ISP_PROG_NAME(firmware);
-		blob_info = &firmware->blob;
-		mem_if = (struct ia_css_isp_param_css_segments *)&firmware->mem_initializers;
-	} else {
-	    /* SP stage */
-	    assert(stage->sp_func != IA_CSS_PIPELINE_NO_FUNC);
-		/* binary and blob_info are now NULL.
-		   These will be passed to sh_css_sp_init_stage
-		   and dereferenced there, so passing a NULL
-		   pointer is no good. return an error */
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	}
-
-	err = sh_css_sp_init_stage(binary,
-			     (const char *)binary_name,
-			     blob_info,
-			     args,
-			     pipe_num,
-			     stage_num,
-			     xnr,
-			     mem_if,
-			     if_config_index,
-			     two_ppc);
-	return err;
-}
-
-static void
-sp_init_sp_stage(struct ia_css_pipeline_stage *stage,
-		 unsigned pipe_num,
-		 bool two_ppc,
-		 enum sh_css_pipe_config_override copy_ovrd,
-		 unsigned int if_config_index)
-{
-	const struct sh_css_binary_args *args = &stage->args;
-
-	assert(stage != NULL);
-	switch (stage->sp_func) {
-	case IA_CSS_PIPELINE_RAW_COPY:
-		sh_css_sp_start_raw_copy(args->out_frame[0],
-				pipe_num, two_ppc,
-				stage->max_input_width,
-				copy_ovrd, if_config_index);
-		break;
-	case IA_CSS_PIPELINE_BIN_COPY:
-		assert(false); /* TBI */
-	case IA_CSS_PIPELINE_ISYS_COPY:
-		sh_css_sp_start_isys_copy(args->out_frame[0],
-				pipe_num, stage->max_input_width, if_config_index);
-		break;
-	case IA_CSS_PIPELINE_NO_FUNC:
-		assert(false);
-	}
-}
-
-void
-sh_css_sp_init_pipeline(struct ia_css_pipeline *me,
-			enum ia_css_pipe_id id,
-			uint8_t pipe_num,
-			bool xnr,
-			bool two_ppc,
-			bool continuous,
-			bool offline,
-			unsigned int required_bds_factor,
-			enum sh_css_pipe_config_override copy_ovrd,
-			enum ia_css_input_mode input_mode,
-			const struct ia_css_metadata_config *md_config,
-			const struct ia_css_metadata_info *md_info,
-#if !defined(HAS_NO_INPUT_SYSTEM)
-			const enum mipi_port_id port_id
-#endif
-#ifdef ISP2401
-			,
-			const struct ia_css_coordinate *internal_frame_origin_bqs_on_sctbl, /* Origin of internal frame
-							positioned on shading table at shading correction in ISP. */
-			const struct ia_css_isp_parameters *params
-#endif
-	)
-{
-	/* Get first stage */
-	struct ia_css_pipeline_stage *stage        = NULL;
-	struct ia_css_binary	     *first_binary = NULL;
-	struct ia_css_pipe *pipe = NULL;
-	unsigned num;
-
-	enum ia_css_pipe_id pipe_id = id;
-	unsigned int thread_id;
-	uint8_t if_config_index, tmp_if_config_index;
-
-	assert(me != NULL);
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	assert(me->stages != NULL);
-
-	first_binary = me->stages->binary;
-
-	if (input_mode == IA_CSS_INPUT_MODE_SENSOR ||
-	    input_mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) {
-		assert(port_id < N_MIPI_PORT_ID);
-		if (port_id >= N_MIPI_PORT_ID) /* should not happen but KW does not know */
-			return; /* we should be able to return an error */
-		if_config_index  = (uint8_t) (port_id - MIPI_PORT0_ID);
-	} else if (input_mode == IA_CSS_INPUT_MODE_MEMORY) {
-		if_config_index = SH_CSS_IF_CONFIG_NOT_NEEDED;
-	} else {
-		if_config_index = 0x0;
-	}
-#else
-	(void)input_mode;
-	if_config_index = SH_CSS_IF_CONFIG_NOT_NEEDED;
-#endif
-
-	ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
-	memset(&sh_css_sp_group.pipe[thread_id], 0, sizeof(struct sh_css_sp_pipeline));
-
-	/* Count stages */
-	for (stage = me->stages, num = 0; stage; stage = stage->next, num++) {
-		stage->stage_num = num;
-		ia_css_debug_pipe_graph_dump_stage(stage, id);
-	}
-	me->num_stages = num;
-
-	if (first_binary != NULL) {
-		/* Init pipeline data */
-		sh_css_sp_init_group(two_ppc, first_binary->input_format,
-				     offline, if_config_index);
-	} /* if (first_binary != NULL) */
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2401) || defined(USE_INPUT_SYSTEM_VERSION_2)
-	/* Signal the host immediately after start for SP_ISYS_COPY only */
-	if ((me->num_stages == 1) && me->stages &&
-	    (me->stages->sp_func == IA_CSS_PIPELINE_ISYS_COPY))
-		sh_css_sp_group.config.no_isp_sync = true;
-#endif
-
-	/* Init stage data */
-	sh_css_init_host2sp_frame_data();
-
-	sh_css_sp_group.pipe[thread_id].num_stages = 0;
-	sh_css_sp_group.pipe[thread_id].pipe_id = pipe_id;
-	sh_css_sp_group.pipe[thread_id].thread_id = thread_id;
-	sh_css_sp_group.pipe[thread_id].pipe_num = pipe_num;
-	sh_css_sp_group.pipe[thread_id].num_execs = me->num_execs;
-	sh_css_sp_group.pipe[thread_id].pipe_qos_config = me->pipe_qos_config;
-	sh_css_sp_group.pipe[thread_id].required_bds_factor = required_bds_factor;
-#if !defined(HAS_NO_INPUT_SYSTEM)
-	sh_css_sp_group.pipe[thread_id].input_system_mode
-						= (uint32_t)input_mode;
-	sh_css_sp_group.pipe[thread_id].port_id = port_id;
-#endif
-	sh_css_sp_group.pipe[thread_id].dvs_frame_delay = (uint32_t)me->dvs_frame_delay;
-
-	/* TODO: next indicates from which queues parameters need to be
-		 sampled, needs checking/improvement */
-	if (ia_css_pipeline_uses_params(me)) {
-		sh_css_sp_group.pipe[thread_id].pipe_config =
-			SH_CSS_PIPE_CONFIG_SAMPLE_PARAMS << thread_id;
-	}
-
-	/* For continuous use-cases, SP copy is responsible for sampling the
-	 * parameters */
-	if (continuous)
-		sh_css_sp_group.pipe[thread_id].pipe_config = 0;
-
-	sh_css_sp_group.pipe[thread_id].inout_port_config = me->inout_port_config;
-
-	pipe = find_pipe_by_num(pipe_num);
-	assert(pipe != NULL);
-	if (pipe == NULL) {
-		return;
-	}
-	sh_css_sp_group.pipe[thread_id].scaler_pp_lut = sh_css_pipe_get_pp_gdc_lut(pipe);
-
-#if defined(SH_CSS_ENABLE_METADATA)
-	if (md_info != NULL && md_info->size > 0) {
-		sh_css_sp_group.pipe[thread_id].metadata.width  = md_info->resolution.width;
-		sh_css_sp_group.pipe[thread_id].metadata.height = md_info->resolution.height;
-		sh_css_sp_group.pipe[thread_id].metadata.stride = md_info->stride;
-		sh_css_sp_group.pipe[thread_id].metadata.size   = md_info->size;
-		ia_css_isys_convert_stream_format_to_mipi_format(
-				md_config->data_type, MIPI_PREDICTOR_NONE,
-				&sh_css_sp_group.pipe[thread_id].metadata.format);
-	}
-#else
-	(void)md_config;
-	(void)md_info;
-#endif
-
-#if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-	sh_css_sp_group.pipe[thread_id].output_frame_queue_id = (uint32_t)SH_CSS_INVALID_QUEUE_ID;
-	if (IA_CSS_PIPE_ID_COPY != pipe_id) {
-		ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_OUTPUT_FRAME, thread_id, (enum sh_css_queue_id *)(&sh_css_sp_group.pipe[thread_id].output_frame_queue_id));
-	}
-#endif
-
-#ifdef ISP2401
-	/* For the shading correction type 1 (the legacy shading table conversion in css is not used),
-	 * the parameters are passed to the isp for the shading table centering.
-	 */
-	if (internal_frame_origin_bqs_on_sctbl != NULL &&
-			params != NULL && params->shading_settings.enable_shading_table_conversion == 0) {
-		sh_css_sp_group.pipe[thread_id].shading.internal_frame_origin_x_bqs_on_sctbl
-								= (uint32_t)internal_frame_origin_bqs_on_sctbl->x;
-		sh_css_sp_group.pipe[thread_id].shading.internal_frame_origin_y_bqs_on_sctbl
-								= (uint32_t)internal_frame_origin_bqs_on_sctbl->y;
-	} else {
-		sh_css_sp_group.pipe[thread_id].shading.internal_frame_origin_x_bqs_on_sctbl = 0;
-		sh_css_sp_group.pipe[thread_id].shading.internal_frame_origin_y_bqs_on_sctbl = 0;
-	}
-
-#endif
-	IA_CSS_LOG("pipe_id %d port_config %08x",
-		   pipe_id, sh_css_sp_group.pipe[thread_id].inout_port_config);
-
-	for (stage = me->stages, num = 0; stage; stage = stage->next, num++) {
-		sh_css_sp_group.pipe[thread_id].num_stages++;
-		if (is_sp_stage(stage)) {
-			sp_init_sp_stage(stage, pipe_num, two_ppc,
-				copy_ovrd, if_config_index);
-		} else {
-			if ((stage->stage_num != 0) || SH_CSS_PIPE_PORT_CONFIG_IS_CONTINUOUS(me->inout_port_config))
-				tmp_if_config_index = SH_CSS_IF_CONFIG_NOT_NEEDED;
-			else
-				tmp_if_config_index = if_config_index;
-			sp_init_stage(stage, pipe_num,
-				      xnr, tmp_if_config_index, two_ppc);
-		}
-
-		store_sp_stage_data(pipe_id, pipe_num, num);
-	}
-	sh_css_sp_group.pipe[thread_id].pipe_config |= (uint32_t)
-		(me->acquire_isp_each_stage << IA_CSS_ACQUIRE_ISP_POS);
-	store_sp_group_data();
-
-}
-
-void
-sh_css_sp_uninit_pipeline(unsigned int pipe_num)
-{
-	unsigned int thread_id;
-	ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
-	/*memset(&sh_css_sp_group.pipe[thread_id], 0, sizeof(struct sh_css_sp_pipeline));*/
-	sh_css_sp_group.pipe[thread_id].num_stages = 0;
-}
-
-bool sh_css_write_host2sp_command(enum host2sp_commands host2sp_command)
-{
-	unsigned int HIVE_ADDR_host_sp_com = sh_css_sp_fw.info.sp.host_sp_com;
-	unsigned int offset = (unsigned int)offsetof(struct host_sp_communication, host2sp_command)
-				/ sizeof(int);
-	enum host2sp_commands last_cmd = host2sp_cmd_error;
-	(void)HIVE_ADDR_host_sp_com; /* Suppres warnings in CRUN */
-
-	/* Previous command must be handled by SP (by design) */
-	last_cmd = load_sp_array_uint(host_sp_com, offset);
-	if (last_cmd != host2sp_cmd_ready)
-		IA_CSS_ERROR("last host command not handled by SP(%d)", last_cmd);
-
-	store_sp_array_uint(host_sp_com, offset, host2sp_command);
-
-	return (last_cmd == host2sp_cmd_ready);
-}
-
-enum host2sp_commands
-sh_css_read_host2sp_command(void)
-{
-	unsigned int HIVE_ADDR_host_sp_com = sh_css_sp_fw.info.sp.host_sp_com;
-	unsigned int offset = (unsigned int)offsetof(struct host_sp_communication, host2sp_command)
-				/ sizeof(int);
-	(void)HIVE_ADDR_host_sp_com; /* Suppres warnings in CRUN */
-	return (enum host2sp_commands)load_sp_array_uint(host_sp_com, offset);
-}
-
-
-/*
- * Frame data is no longer part of the sp_stage structure but part of a
- * seperate structure. The aim is to make the sp_data struct static
- * (it defines a pipeline) and that the dynamic (per frame) data is stored
- * separetly.
- *
- * This function must be called first every where were you start constructing
- * a new pipeline by defining one or more stages with use of variable
- * sh_css_sp_stage. Even the special cases like accelerator and copy_frame
- * These have a pipeline of just 1 stage.
- */
-void
-sh_css_init_host2sp_frame_data(void)
-{
-	/* Clean table */
-	unsigned int HIVE_ADDR_host_sp_com = sh_css_sp_fw.info.sp.host_sp_com;
-
-	(void)HIVE_ADDR_host_sp_com; /* Suppres warnings in CRUN */
-	/*
-	 * rvanimme: don't clean it to save static frame info line ref_in
-	 * ref_out, and tnr_frames. Once this static data is in a
-	 * seperate data struct, this may be enable (but still, there is
-	 * no need for it)
-	 */
-}
-
-
-/*
- * @brief Update the offline frame information in host_sp_communication.
- * Refer to "sh_css_sp.h" for more details.
- */
-void
-sh_css_update_host2sp_offline_frame(
-				unsigned frame_num,
-				struct ia_css_frame *frame,
-				struct ia_css_metadata *metadata)
-{
-	unsigned int HIVE_ADDR_host_sp_com;
-	unsigned int offset;
-
-	assert(frame_num < NUM_CONTINUOUS_FRAMES);
-
-	/* Write new frame data into SP DMEM */
-	HIVE_ADDR_host_sp_com = sh_css_sp_fw.info.sp.host_sp_com;
-	offset = (unsigned int)offsetof(struct host_sp_communication, host2sp_offline_frames)
-		/ sizeof(int);
-	offset += frame_num;
-	store_sp_array_uint(host_sp_com, offset, frame ? frame->data : 0);
-
-	/* Write metadata buffer into SP DMEM */
-	offset = (unsigned int)offsetof(struct host_sp_communication, host2sp_offline_metadata)
-		/ sizeof(int);
-	offset += frame_num;
-	store_sp_array_uint(host_sp_com, offset, metadata ? metadata->address : 0);
-}
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-/*
- * @brief Update the mipi frame information in host_sp_communication.
- * Refer to "sh_css_sp.h" for more details.
- */
-void
-sh_css_update_host2sp_mipi_frame(
-				unsigned frame_num,
-				struct ia_css_frame *frame)
-{
-	unsigned int HIVE_ADDR_host_sp_com;
-	unsigned int offset;
-
-	/* MIPI buffers are dedicated to port, so now there are more of them. */
-	assert(frame_num < (N_CSI_PORTS * NUM_MIPI_FRAMES_PER_STREAM));
-
-	/* Write new frame data into SP DMEM */
-	HIVE_ADDR_host_sp_com = sh_css_sp_fw.info.sp.host_sp_com;
-	offset = (unsigned int)offsetof(struct host_sp_communication, host2sp_mipi_frames)
-		/ sizeof(int);
-	offset += frame_num;
-
-	store_sp_array_uint(host_sp_com, offset,
-				frame ? frame->data : 0);
-}
-
-/*
- * @brief Update the mipi metadata information in host_sp_communication.
- * Refer to "sh_css_sp.h" for more details.
- */
-void
-sh_css_update_host2sp_mipi_metadata(
-				unsigned frame_num,
-				struct ia_css_metadata *metadata)
-{
-	unsigned int HIVE_ADDR_host_sp_com;
-	unsigned int o;
-
-	/* MIPI buffers are dedicated to port, so now there are more of them. */
-	assert(frame_num < (N_CSI_PORTS * NUM_MIPI_FRAMES_PER_STREAM));
-
-	/* Write new frame data into SP DMEM */
-	HIVE_ADDR_host_sp_com = sh_css_sp_fw.info.sp.host_sp_com;
-	o = offsetof(struct host_sp_communication, host2sp_mipi_metadata)
-		/ sizeof(int);
-	o += frame_num;
-	store_sp_array_uint(host_sp_com, o,
-				metadata ? metadata->address : 0);
-}
-
-void
-sh_css_update_host2sp_num_mipi_frames(unsigned num_frames)
-{
-	unsigned int HIVE_ADDR_host_sp_com;
-	unsigned int offset;
-
-	/* Write new frame data into SP DMEM */
-	HIVE_ADDR_host_sp_com = sh_css_sp_fw.info.sp.host_sp_com;
-	offset = (unsigned int)offsetof(struct host_sp_communication, host2sp_num_mipi_frames)
-		/ sizeof(int);
-
-	store_sp_array_uint(host_sp_com, offset, num_frames);
-}
-#endif
-
-void
-sh_css_update_host2sp_cont_num_raw_frames(unsigned num_frames, bool set_avail)
-{
-	const struct ia_css_fw_info *fw;
-	unsigned int HIVE_ADDR_host_sp_com;
-	unsigned int extra_num_frames, avail_num_frames;
-	unsigned int offset, offset_extra;
-
-	/* Write new frame data into SP DMEM */
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_host_sp_com = fw->info.sp.host_sp_com;
-	if (set_avail) {
-		offset = (unsigned int)offsetof(struct host_sp_communication, host2sp_cont_avail_num_raw_frames)
-			/ sizeof(int);
-		avail_num_frames = load_sp_array_uint(host_sp_com, offset);
-		extra_num_frames = num_frames - avail_num_frames;
-		offset_extra = (unsigned int)offsetof(struct host_sp_communication, host2sp_cont_extra_num_raw_frames)
-			/ sizeof(int);
-		store_sp_array_uint(host_sp_com, offset_extra, extra_num_frames);
-	} else
-		offset = (unsigned int)offsetof(struct host_sp_communication, host2sp_cont_target_num_raw_frames)
-			/ sizeof(int);
-
-	store_sp_array_uint(host_sp_com, offset, num_frames);
-}
-
-void
-sh_css_event_init_irq_mask(void)
-{
-	int i;
-	unsigned int HIVE_ADDR_host_sp_com = sh_css_sp_fw.info.sp.host_sp_com;
-	unsigned int offset;
-	struct sh_css_event_irq_mask event_irq_mask_init;
-
-	event_irq_mask_init.or_mask  = IA_CSS_EVENT_TYPE_ALL;
-	event_irq_mask_init.and_mask = IA_CSS_EVENT_TYPE_NONE;
-	(void)HIVE_ADDR_host_sp_com; /* Suppress warnings in CRUN */
-
-	assert(sizeof(event_irq_mask_init) % HRT_BUS_BYTES == 0);
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) {
-		offset = (unsigned int)offsetof(struct host_sp_communication,
-						host2sp_event_irq_mask[i]);
-		assert(offset % HRT_BUS_BYTES == 0);
-		sp_dmem_store(SP0_ID,
-			(unsigned int)sp_address_of(host_sp_com) + offset,
-			&event_irq_mask_init, sizeof(event_irq_mask_init));
-	}
-
-}
-
-enum ia_css_err
-ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe,
-			 unsigned int or_mask,
-			 unsigned int and_mask)
-{
-	unsigned int HIVE_ADDR_host_sp_com = sh_css_sp_fw.info.sp.host_sp_com;
-	unsigned int offset;
-	struct sh_css_event_irq_mask event_irq_mask;
-	unsigned int pipe_num;
-
-	assert(pipe != NULL);
-
-	assert(IA_CSS_PIPE_ID_NUM == NR_OF_PIPELINES);
-	/* Linux kernel does not have UINT16_MAX
-	 * Therefore decided to comment out these 2 asserts for Linux
-	 * Alternatives that were not chosen:
-	 * - add a conditional #define for UINT16_MAX
-	 * - compare with (uint16_t)~0 or 0xffff
-	 * - different assert for Linux and Windows
-	 */
-#ifndef __KERNEL__
-	assert(or_mask <= UINT16_MAX);
-	assert(and_mask <= UINT16_MAX);
-#endif
-
-	(void)HIVE_ADDR_host_sp_com; /* Suppres warnings in CRUN */
-
-	IA_CSS_LOG("or_mask=%x, and_mask=%x", or_mask, and_mask);
-	event_irq_mask.or_mask  = (uint16_t)or_mask;
-	event_irq_mask.and_mask = (uint16_t)and_mask;
-
-	pipe_num = ia_css_pipe_get_pipe_num(pipe);
-	if (pipe_num >= IA_CSS_PIPE_ID_NUM)
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	offset = (unsigned int)offsetof(struct host_sp_communication,
-					host2sp_event_irq_mask[pipe_num]);
-	assert(offset % HRT_BUS_BYTES == 0);
-	sp_dmem_store(SP0_ID,
-		(unsigned int)sp_address_of(host_sp_com) + offset,
-		&event_irq_mask, sizeof(event_irq_mask));
-
-	return IA_CSS_SUCCESS;
-}
-
-enum ia_css_err
-ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe,
-			  unsigned int *or_mask,
-			  unsigned int *and_mask)
-{
-	unsigned int HIVE_ADDR_host_sp_com = sh_css_sp_fw.info.sp.host_sp_com;
-	unsigned int offset;
-	struct sh_css_event_irq_mask event_irq_mask;
-	unsigned int pipe_num;
-
-	(void)HIVE_ADDR_host_sp_com; /* Suppres warnings in CRUN */
-
-	IA_CSS_ENTER_LEAVE("");
-
-	assert(pipe != NULL);
-	assert(IA_CSS_PIPE_ID_NUM == NR_OF_PIPELINES);
-
-	pipe_num = ia_css_pipe_get_pipe_num(pipe);
-	if (pipe_num >= IA_CSS_PIPE_ID_NUM)
-		return IA_CSS_ERR_INTERNAL_ERROR;
-	offset = (unsigned int)offsetof(struct host_sp_communication,
-					host2sp_event_irq_mask[pipe_num]);
-	assert(offset % HRT_BUS_BYTES == 0);
-	sp_dmem_load(SP0_ID,
-		(unsigned int)sp_address_of(host_sp_com) + offset,
-		&event_irq_mask, sizeof(event_irq_mask));
-
-	if (or_mask)
-		*or_mask = event_irq_mask.or_mask;
-
-	if (and_mask)
-		*and_mask = event_irq_mask.and_mask;
-
-	return IA_CSS_SUCCESS;
-}
-
-void
-sh_css_sp_set_sp_running(bool flag)
-{
-	sp_running = flag;
-}
-
-bool
-sh_css_sp_is_running(void)
-{
-	return sp_running;
-}
-
-void
-sh_css_sp_start_isp(void)
-{
-	const struct ia_css_fw_info *fw;
-	unsigned int HIVE_ADDR_sp_sw_state;
-
-	fw = &sh_css_sp_fw;
-	HIVE_ADDR_sp_sw_state = fw->info.sp.sw_state;
-
-
-	if (sp_running)
-		return;
-
-	(void)HIVE_ADDR_sp_sw_state; /* Suppres warnings in CRUN */
-
-	/* no longer here, sp started immediately */
-	/*ia_css_debug_pipe_graph_dump_epilogue();*/
-
-	store_sp_group_data();
-	store_sp_per_frame_data(fw);
-
-	sp_dmem_store_uint32(SP0_ID,
-		(unsigned int)sp_address_of(sp_sw_state),
-		(uint32_t)(IA_CSS_SP_SW_TERMINATED));
-
-
-	/* Note 1: The sp_start_isp function contains a wait till
-	 * the input network is configured by the SP.
-	 * Note 2: Not all SP binaries supports host2sp_commands.
-	 * In case a binary does support it, the host2sp_command
-	 * will have status cmd_ready after return of the function
-	 * sh_css_hrt_sp_start_isp. There is no race-condition here
-	 * because only after the process_frame command has been
-	 * received, the SP starts configuring the input network.
-	 */
-
-	/* we need to set sp_running before we call ia_css_mmu_invalidate_cache
-	 * as ia_css_mmu_invalidate_cache checks on sp_running to
-	 * avoid that it accesses dmem while the SP is not powered
-	 */
-	sp_running = true;
-	ia_css_mmu_invalidate_cache();
-	/* Invalidate all MMU caches */
-	mmu_invalidate_cache_all();
-
-	ia_css_spctrl_start(SP0_ID);
-
-}
-
-bool
-ia_css_isp_has_started(void)
-{
-	const struct ia_css_fw_info *fw = &sh_css_sp_fw;
-	unsigned int HIVE_ADDR_ia_css_ispctrl_sp_isp_started = fw->info.sp.isp_started;
-	(void)HIVE_ADDR_ia_css_ispctrl_sp_isp_started; /* Suppres warnings in CRUN */
-
-	return (bool)load_sp_uint(ia_css_ispctrl_sp_isp_started);
-}
-
-
-/*
- * @brief Initialize the DMA software-mask in the debug mode.
- * Refer to "sh_css_sp.h" for more details.
- */
-bool
-sh_css_sp_init_dma_sw_reg(int dma_id)
-{
-	int i;
-
-	/* enable all the DMA channels */
-	for (i = 0; i < N_DMA_CHANNEL_ID; i++) {
-		/* enable the writing request */
-		sh_css_sp_set_dma_sw_reg(dma_id,
-				i,
-				0,
-				true);
-		/* enable the reading request */
-		sh_css_sp_set_dma_sw_reg(dma_id,
-				i,
-				1,
-				true);
-	}
-
-	return true;
-}
-
-/*
- * @brief Set the DMA software-mask in the debug mode.
- * Refer to "sh_css_sp.h" for more details.
- */
-bool
-sh_css_sp_set_dma_sw_reg(int dma_id,
-		int channel_id,
-		int request_type,
-		bool enable)
-{
-	uint32_t sw_reg;
-	uint32_t bit_val;
-	uint32_t bit_offset;
-	uint32_t bit_mask;
-
-	(void)dma_id;
-
-	assert(channel_id >= 0 && channel_id < N_DMA_CHANNEL_ID);
-	assert(request_type >= 0);
-
-	/* get the software-mask */
-	sw_reg =
-		sh_css_sp_group.debug.dma_sw_reg;
-
-	/* get the offest of the target bit */
-	bit_offset = (8 * request_type) + channel_id;
-
-	/* clear the value of the target bit */
-	bit_mask = ~(1 << bit_offset);
-	sw_reg &= bit_mask;
-
-	/* set the value of the bit for the DMA channel */
-	bit_val = enable ? 1 : 0;
-	bit_val <<= bit_offset;
-	sw_reg |= bit_val;
-
-	/* update the software status of DMA channels */
-	sh_css_sp_group.debug.dma_sw_reg = sw_reg;
-
-	return true;
-}
-
-void
-sh_css_sp_reset_global_vars(void)
-{
-	memset(&sh_css_sp_group, 0, sizeof(struct sh_css_sp_group));
-	memset(&sh_css_sp_stage, 0, sizeof(struct sh_css_sp_stage));
-	memset(&sh_css_isp_stage, 0, sizeof(struct sh_css_isp_stage));
-	memset(&sh_css_sp_output, 0, sizeof(struct sh_css_sp_output));
-	memset(&per_frame_data, 0, sizeof(struct sh_css_sp_per_frame_data));
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_sp.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_sp.h
deleted file mode 100644
index 3c41e997de79..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_sp.h
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _SH_CSS_SP_H_
-#define _SH_CSS_SP_H_
-
-#include <system_global.h>
-#include <type_support.h>
-#if !defined(HAS_NO_INPUT_FORMATTER)
-#include "input_formatter.h"
-#endif
-
-#include "ia_css_binary.h"
-#include "ia_css_types.h"
-#include "ia_css_pipeline.h"
-
-/* Function to initialize the data and bss section descr of the binary */
-void
-sh_css_sp_store_init_dmem(const struct ia_css_fw_info *fw);
-
-void
-store_sp_stage_data(enum ia_css_pipe_id id, unsigned int pipe_num, unsigned stage);
-
-void
-sh_css_stage_write_binary_info(struct ia_css_binary_info *info);
-
-void
-store_sp_group_data(void);
-
-/* Start binary (jpeg) copy on the SP */
-void
-sh_css_sp_start_binary_copy(unsigned int pipe_num, struct ia_css_frame *out_frame,
-			    unsigned two_ppc);
-
-unsigned int
-sh_css_sp_get_binary_copy_size(void);
-
-/* Return the value of a SW interrupt */
-unsigned int
-sh_css_sp_get_sw_interrupt_value(unsigned int irq);
-
-void
-sh_css_sp_init_pipeline(struct ia_css_pipeline *me,
-			enum ia_css_pipe_id id,
-			uint8_t pipe_num,
-			bool xnr,
-			bool two_ppc,
-			bool continuous,
-			bool offline,
-			unsigned int required_bds_factor,
-			enum sh_css_pipe_config_override copy_ovrd,
-			enum ia_css_input_mode input_mode,
-			const struct ia_css_metadata_config *md_config,
-			const struct ia_css_metadata_info *md_info,
-#if !defined(HAS_NO_INPUT_SYSTEM)
-			const enum mipi_port_id port_id
-#endif
-#ifdef ISP2401
-			,
-			const struct ia_css_coordinate *internal_frame_origin_bqs_on_sctbl, /* Origin of internal frame
-							positioned on shading table at shading correction in ISP. */
-			const struct ia_css_isp_parameters *params
-#endif
-			);
-
-void
-sh_css_sp_uninit_pipeline(unsigned int pipe_num);
-
-bool sh_css_write_host2sp_command(enum host2sp_commands host2sp_command);
-
-enum host2sp_commands
-sh_css_read_host2sp_command(void);
-
-void
-sh_css_init_host2sp_frame_data(void);
-
-/**
- * @brief Update the offline frame information in host_sp_communication.
- *
- * @param[in] frame_num The offline frame number.
- * @param[in] frame The pointer to the offline frame.
- */
-void
-sh_css_update_host2sp_offline_frame(
-				unsigned frame_num,
-				struct ia_css_frame *frame,
-				struct ia_css_metadata *metadata);
-
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-/**
- * @brief Update the mipi frame information in host_sp_communication.
- *
- * @param[in] frame_num The mipi frame number.
- * @param[in] frame The pointer to the mipi frame.
- */
-void
-sh_css_update_host2sp_mipi_frame(
-				unsigned frame_num,
-				struct ia_css_frame *frame);
-
-/**
- * @brief Update the mipi metadata information in host_sp_communication.
- *
- * @param[in] frame_num The mipi frame number.
- * @param[in] metadata The pointer to the mipi metadata.
- */
-void
-sh_css_update_host2sp_mipi_metadata(
-				unsigned frame_num,
-				struct ia_css_metadata *metadata);
-
-/**
- * @brief Update the nr of mipi frames to use in host_sp_communication.
- *
- * @param[in] num_frames The number of mipi frames to use.
- */
-void
-sh_css_update_host2sp_num_mipi_frames(unsigned num_frames);
-#endif
-
-/**
- * @brief Update the nr of offline frames to use in host_sp_communication.
- *
- * @param[in] num_frames The number of raw frames to use.
- */
-void
-sh_css_update_host2sp_cont_num_raw_frames(unsigned num_frames, bool set_avail);
-
-void
-sh_css_event_init_irq_mask(void);
-
-void
-sh_css_sp_start_isp(void);
-
-void
-sh_css_sp_set_sp_running(bool flag);
-
-bool
-sh_css_sp_is_running(void);
-
-#if SP_DEBUG != SP_DEBUG_NONE
-
-void
-sh_css_sp_get_debug_state(struct sh_css_sp_debug_state *state);
-
-#endif
-
-#if !defined(HAS_NO_INPUT_FORMATTER)
-void
-sh_css_sp_set_if_configs(
-	const input_formatter_cfg_t	*config_a,
-	const input_formatter_cfg_t	*config_b,
-	const uint8_t		if_config_index);
-#endif
-
-void
-sh_css_sp_program_input_circuit(int fmt_type,
-				int ch_id,
-				enum ia_css_input_mode input_mode);
-
-void
-sh_css_sp_configure_sync_gen(int width,
-			     int height,
-			     int hblank_cycles,
-			     int vblank_cycles);
-
-void
-sh_css_sp_configure_tpg(int x_mask,
-			int y_mask,
-			int x_delta,
-			int y_delta,
-			int xy_mask);
-
-void
-sh_css_sp_configure_prbs(int seed);
-
-void
-sh_css_sp_configure_enable_raw_pool_locking(bool lock_all);
-
-void
-sh_css_sp_enable_isys_event_queue(bool enable);
-
-void
-sh_css_sp_set_disable_continuous_viewfinder(bool flag);
-
-void
-sh_css_sp_reset_global_vars(void);
-
-/**
- * @brief Initialize the DMA software-mask in the debug mode.
- * This API should be ONLY called in the debugging mode.
- * And it should be always called before the first call of
- * "sh_css_set_dma_sw_reg(...)".
- *
- * @param[in]	dma_id		The ID of the target DMA.
- *
- * @return
- *	- true, if it is successful.
- *	- false, otherwise.
- */
-bool
-sh_css_sp_init_dma_sw_reg(int dma_id);
-
-/**
- * @brief Set the DMA software-mask in the debug mode.
- * This API should be ONLYL called in the debugging mode. Must
- * call "sh_css_set_dma_sw_reg(...)" before this
- * API is called for the first time.
- *
- * @param[in]	dma_id		The ID of the target DMA.
- * @param[in]	channel_id	The ID of the target DMA channel.
- * @param[in]	request_type	The type of the DMA request.
- *				For example:
- *				- "0" indicates the writing request.
- *				- "1" indicates the reading request.
- *
- * @param[in]	enable		If it is "true", the target DMA
- *				channel is enabled in the software.
- *				Otherwise, the target DMA channel
- *				is disabled in the software.
- *
- * @return
- *	- true, if it is successful.
- *	- false, otherwise.
- */
-bool
-sh_css_sp_set_dma_sw_reg(int dma_id,
-		int channel_id,
-		int request_type,
-		bool enable);
-
-
-extern struct sh_css_sp_group sh_css_sp_group;
-extern struct sh_css_sp_stage sh_css_sp_stage;
-extern struct sh_css_isp_stage sh_css_isp_stage;
-
-#endif /* _SH_CSS_SP_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream.c
deleted file mode 100644
index 60bddbb3d4c6..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream.c
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* This file will contain the code to implement the functions declared in ia_css_stream.h
-   and associated helper functions */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream_format.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream_format.c
deleted file mode 100644
index 77f135e7dc3c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream_format.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "sh_css_stream_format.h"
-#include <ia_css_stream_format.h>
-
-unsigned int sh_css_stream_format_2_bits_per_subpixel(
-		enum atomisp_input_format format)
-{
-	unsigned int rval;
-
-	switch (format) {
-	case ATOMISP_INPUT_FORMAT_RGB_444:
-		rval = 4;
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_555:
-		rval = 5;
-		break;
-	case ATOMISP_INPUT_FORMAT_RGB_565:
-	case ATOMISP_INPUT_FORMAT_RGB_666:
-	case ATOMISP_INPUT_FORMAT_RAW_6:
-		rval = 6;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_7:
-		rval = 7;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY:
-	case ATOMISP_INPUT_FORMAT_YUV420_8:
-	case ATOMISP_INPUT_FORMAT_YUV422_8:
-	case ATOMISP_INPUT_FORMAT_RGB_888:
-	case ATOMISP_INPUT_FORMAT_RAW_8:
-	case ATOMISP_INPUT_FORMAT_BINARY_8:
-	case ATOMISP_INPUT_FORMAT_USER_DEF1:
-	case ATOMISP_INPUT_FORMAT_USER_DEF2:
-	case ATOMISP_INPUT_FORMAT_USER_DEF3:
-	case ATOMISP_INPUT_FORMAT_USER_DEF4:
-	case ATOMISP_INPUT_FORMAT_USER_DEF5:
-	case ATOMISP_INPUT_FORMAT_USER_DEF6:
-	case ATOMISP_INPUT_FORMAT_USER_DEF7:
-	case ATOMISP_INPUT_FORMAT_USER_DEF8:
-		rval = 8;
-		break;
-	case ATOMISP_INPUT_FORMAT_YUV420_10:
-	case ATOMISP_INPUT_FORMAT_YUV422_10:
-	case ATOMISP_INPUT_FORMAT_RAW_10:
-		rval = 10;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_12:
-		rval = 12;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_14:
-		rval = 14;
-		break;
-	case ATOMISP_INPUT_FORMAT_RAW_16:
-	case ATOMISP_INPUT_FORMAT_YUV420_16:
-	case ATOMISP_INPUT_FORMAT_YUV422_16:
-		rval = 16;
-		break;
-	default:
-		rval = 0;
-		break;
-	}
-
-	return rval;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream_format.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream_format.h
deleted file mode 100644
index b699f538e0dd..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_stream_format.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SH_CSS_STREAM_FORMAT_H
-#define __SH_CSS_STREAM_FORMAT_H
-
-#include <ia_css_stream_format.h>
-
-unsigned int sh_css_stream_format_2_bits_per_subpixel(
-		enum atomisp_input_format format);
-
-#endif /* __SH_CSS_STREAM_FORMAT_H */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_struct.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_struct.h
deleted file mode 100644
index 0b8e3d872069..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_struct.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __SH_CSS_STRUCT_H
-#define __SH_CSS_STRUCT_H
-
-/* This header files contains the definition of the
-   sh_css struct and friends; locigally the file would
-   probably be called sh_css.h after the pattern
-   <type>.h but sh_css.h is the predecesssor of ia_css.h
-   so this could cause confusion; hence the _struct
-   in the filename
-*/
-
-#include <type_support.h>
-#include <system_types.h>
-#include "ia_css_pipeline.h"
-#include "ia_css_pipe_public.h"
-#include "ia_css_frame_public.h"
-#include "ia_css_queue.h"
-#include "ia_css_irq.h"
-
-struct sh_css {
-	struct ia_css_pipe            *active_pipes[IA_CSS_PIPELINE_NUM_MAX];
-	/* All of the pipes created at any point of time. At this moment there can
-	 * be no more than MAX_SP_THREADS of them because pipe_num is reused as SP
-	 * thread_id to which a pipe's pipeline is associated. At a later point, if
-	 * we support more pipe objects, we should add test code to test that
-	 * possibility. Also, active_pipes[] should be able to hold only
-	 * SH_CSS_MAX_SP_THREADS objects. Anything else is misleading. */
-	struct ia_css_pipe            *all_pipes[IA_CSS_PIPELINE_NUM_MAX];
-	void * (*malloc)(size_t bytes, bool zero_mem);
-	void (*free)(void *ptr);
-#ifdef ISP2401
-	void * (*malloc_ex)(size_t bytes, bool zero_mem, const char *caller_func, int caller_line);
-	void (*free_ex)(void *ptr, const char *caller_func, int caller_line);
-#endif
-	void (*flush)(struct ia_css_acc_fw *fw);
-	bool                           check_system_idle;
-#ifndef ISP2401
-	bool stop_copy_preview;
-#endif
-	unsigned int                   num_cont_raw_frames;
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	unsigned int                   num_mipi_frames[N_CSI_PORTS];
-	struct ia_css_frame           *mipi_frames[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM];
-	struct ia_css_metadata        *mipi_metadata[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM];
-	unsigned int                   mipi_sizes_for_check[N_CSI_PORTS][IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT];
-	unsigned int                   mipi_frame_size[N_CSI_PORTS];
-#endif
-	hrt_vaddress                   sp_bin_addr;
-	hrt_data                       page_table_base_index;
-	unsigned int                   size_mem_words; /* \deprecated{Use ia_css_mipi_buffer_config instead.}*/
-	enum ia_css_irq_type           irq_type;
-	unsigned int                   pipe_counter;
-	
-	unsigned int		type;	/* 2400 or 2401 for now */
-};
-
-#define IPU_2400		1
-#define IPU_2401		2
-
-#define IS_2400()		(my_css.type == IPU_2400)
-#define IS_2401()		(my_css.type == IPU_2401)
-
-extern struct sh_css my_css;
-
-#endif /* __SH_CSS_STRUCT_H */
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_uds.h b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_uds.h
deleted file mode 100644
index 5ded3a1437bf..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_uds.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _SH_CSS_UDS_H_
-#define _SH_CSS_UDS_H_
-
-#include <type_support.h>
-
-#define SIZE_OF_SH_CSS_UDS_INFO_IN_BITS (4 * 16)
-#define SIZE_OF_SH_CSS_CROP_POS_IN_BITS (2 * 16)
-
-/* Uds types, used in pipeline_global.h and sh_css_internal.h */
-
-struct sh_css_uds_info {
-	uint16_t curr_dx;
-	uint16_t curr_dy;
-	uint16_t xc;
-	uint16_t yc;
-};
-
-struct sh_css_crop_pos {
-	uint16_t x;
-	uint16_t y;
-};
-
-#endif /* _SH_CSS_UDS_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_version.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_version.c
deleted file mode 100644
index 6e0c5e7f8620..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_version.c
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include "ia_css_version.h"
-#include "ia_css_version_data.h"
-#include "ia_css_err.h"
-#include "sh_css_firmware.h"
-
-enum ia_css_err
-ia_css_get_version(char *version, int max_size)
-{
-	if (max_size <= (int)strlen(CSS_VERSION_STRING) + (int)strlen(sh_css_get_fw_version()) + 5)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	strcpy(version, CSS_VERSION_STRING);
-	strcat(version, "FW:");
-	strcat(version, sh_css_get_fw_version());
-	strcat(version, "; ");
-	return IA_CSS_SUCCESS;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm.c b/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm.c
deleted file mode 100644
index 4338b8a1309f..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm.c
+++ /dev/null
@@ -1,727 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010-2017 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-/*
- * This file contains entry functions for memory management of ISP driver
- */
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/mm.h>
-#include <linux/highmem.h>	/* for kmap */
-#include <linux/io.h>		/* for page_to_phys */
-#include <linux/sysfs.h>
-
-#include "hmm/hmm.h"
-#include "hmm/hmm_pool.h"
-#include "hmm/hmm_bo.h"
-
-#include "atomisp_internal.h"
-#include "asm/cacheflush.h"
-#include "mmu/isp_mmu.h"
-#include "mmu/sh_mmu_mrfld.h"
-
-struct hmm_bo_device bo_device;
-struct hmm_pool	dynamic_pool;
-struct hmm_pool	reserved_pool;
-static ia_css_ptr dummy_ptr;
-static bool hmm_initialized;
-struct _hmm_mem_stat hmm_mem_stat;
-
-/*
- * p: private
- * s: shared
- * u: user
- * i: ion
- */
-static const char hmm_bo_type_string[] = "psui";
-
-static ssize_t bo_show(struct device *dev, struct device_attribute *attr,
-		       char *buf, struct list_head *bo_list, bool active)
-{
-	ssize_t ret = 0;
-	struct hmm_buffer_object *bo;
-	unsigned long flags;
-	int i;
-	long total[HMM_BO_LAST] = { 0 };
-	long count[HMM_BO_LAST] = { 0 };
-	int index1 = 0;
-	int index2 = 0;
-
-	ret = scnprintf(buf, PAGE_SIZE, "type pgnr\n");
-	if (ret <= 0)
-		return 0;
-
-	index1 += ret;
-
-	spin_lock_irqsave(&bo_device.list_lock, flags);
-	list_for_each_entry(bo, bo_list, list) {
-		if ((active && (bo->status & HMM_BO_ALLOCED)) ||
-		    (!active && !(bo->status & HMM_BO_ALLOCED))) {
-			ret = scnprintf(buf + index1, PAGE_SIZE - index1,
-					"%c %d\n",
-					hmm_bo_type_string[bo->type], bo->pgnr);
-
-			total[bo->type] += bo->pgnr;
-			count[bo->type]++;
-			if (ret > 0)
-				index1 += ret;
-		}
-	}
-	spin_unlock_irqrestore(&bo_device.list_lock, flags);
-
-	for (i = 0; i < HMM_BO_LAST; i++) {
-		if (count[i]) {
-			ret = scnprintf(buf + index1 + index2,
-					PAGE_SIZE - index1 - index2,
-					"%ld %c buffer objects: %ld KB\n",
-					count[i], hmm_bo_type_string[i],
-					total[i] * 4);
-			if (ret > 0)
-				index2 += ret;
-		}
-	}
-
-	/* Add trailing zero, not included by scnprintf */
-	return index1 + index2 + 1;
-}
-
-static ssize_t active_bo_show(struct device *dev, struct device_attribute *attr,
-			      char *buf)
-{
-	return bo_show(dev, attr, buf, &bo_device.entire_bo_list, true);
-}
-
-static ssize_t free_bo_show(struct device *dev, struct device_attribute *attr,
-			    char *buf)
-{
-	return bo_show(dev, attr, buf, &bo_device.entire_bo_list, false);
-}
-
-static ssize_t reserved_pool_show(struct device *dev,
-				  struct device_attribute *attr,
-				  char *buf)
-{
-	ssize_t ret = 0;
-
-	struct hmm_reserved_pool_info *pinfo = reserved_pool.pool_info;
-	unsigned long flags;
-
-	if (!pinfo || !pinfo->initialized)
-		return 0;
-
-	spin_lock_irqsave(&pinfo->list_lock, flags);
-	ret = scnprintf(buf, PAGE_SIZE, "%d out of %d pages available\n",
-			pinfo->index, pinfo->pgnr);
-	spin_unlock_irqrestore(&pinfo->list_lock, flags);
-
-	if (ret > 0)
-		ret++; /* Add trailing zero, not included by scnprintf */
-
-	return ret;
-};
-
-static ssize_t dynamic_pool_show(struct device *dev,
-				 struct device_attribute *attr,
-				 char *buf)
-{
-	ssize_t ret = 0;
-
-	struct hmm_dynamic_pool_info *pinfo = dynamic_pool.pool_info;
-	unsigned long flags;
-
-	if (!pinfo || !pinfo->initialized)
-		return 0;
-
-	spin_lock_irqsave(&pinfo->list_lock, flags);
-	ret = scnprintf(buf, PAGE_SIZE, "%d (max %d) pages available\n",
-			pinfo->pgnr, pinfo->pool_size);
-	spin_unlock_irqrestore(&pinfo->list_lock, flags);
-
-	if (ret > 0)
-		ret++; /* Add trailing zero, not included by scnprintf */
-
-	return ret;
-};
-
-static DEVICE_ATTR_RO(active_bo);
-static DEVICE_ATTR_RO(free_bo);
-static DEVICE_ATTR_RO(reserved_pool);
-static DEVICE_ATTR_RO(dynamic_pool);
-
-static struct attribute *sysfs_attrs_ctrl[] = {
-	&dev_attr_active_bo.attr,
-	&dev_attr_free_bo.attr,
-	&dev_attr_reserved_pool.attr,
-	&dev_attr_dynamic_pool.attr,
-	NULL
-};
-
-static struct attribute_group atomisp_attribute_group[] = {
-	{.attrs = sysfs_attrs_ctrl },
-};
-
-int hmm_init(void)
-{
-	int ret;
-
-	ret = hmm_bo_device_init(&bo_device, &sh_mmu_mrfld,
-				 ISP_VM_START, ISP_VM_SIZE);
-	if (ret)
-		dev_err(atomisp_dev, "hmm_bo_device_init failed.\n");
-
-	hmm_initialized = true;
-
-	/*
-	 * As hmm use NULL to indicate invalid ISP virtual address,
-	 * and ISP_VM_START is defined to 0 too, so we allocate
-	 * one piece of dummy memory, which should return value 0,
-	 * at the beginning, to avoid hmm_alloc return 0 in the
-	 * further allocation.
-	 */
-	dummy_ptr = hmm_alloc(1, HMM_BO_PRIVATE, 0, NULL, HMM_UNCACHED);
-
-	if (!ret) {
-		ret = sysfs_create_group(&atomisp_dev->kobj,
-					 atomisp_attribute_group);
-		if (ret)
-			dev_err(atomisp_dev,
-				"%s Failed to create sysfs\n", __func__);
-	}
-
-	return ret;
-}
-
-void hmm_cleanup(void)
-{
-	sysfs_remove_group(&atomisp_dev->kobj, atomisp_attribute_group);
-
-	/* free dummy memory first */
-	hmm_free(dummy_ptr);
-	dummy_ptr = 0;
-
-	hmm_bo_device_exit(&bo_device);
-	hmm_initialized = false;
-}
-
-ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type,
-		     int from_highmem, void *userptr, bool cached)
-{
-	unsigned int pgnr;
-	struct hmm_buffer_object *bo;
-	int ret;
-
-	/*
-	 * Check if we are initialized. In the ideal world we wouldn't need
-	 * this but we can tackle it once the driver is a lot cleaner
-	 */
-
-	if (!hmm_initialized)
-		hmm_init();
-	/* Get page number from size */
-	pgnr = size_to_pgnr_ceil(bytes);
-
-	/* Buffer object structure init */
-	bo = hmm_bo_alloc(&bo_device, pgnr);
-	if (!bo) {
-		dev_err(atomisp_dev, "hmm_bo_create failed.\n");
-		goto create_bo_err;
-	}
-
-	/* Allocate pages for memory */
-	ret = hmm_bo_alloc_pages(bo, type, from_highmem, userptr, cached);
-	if (ret) {
-		dev_err(atomisp_dev, "hmm_bo_alloc_pages failed.\n");
-		goto alloc_page_err;
-	}
-
-	/* Combind the virtual address and pages togather */
-	ret = hmm_bo_bind(bo);
-	if (ret) {
-		dev_err(atomisp_dev, "hmm_bo_bind failed.\n");
-		goto bind_err;
-	}
-
-	hmm_mem_stat.tol_cnt += pgnr;
-
-	return bo->start;
-
-bind_err:
-	hmm_bo_free_pages(bo);
-alloc_page_err:
-	hmm_bo_unref(bo);
-create_bo_err:
-	return 0;
-}
-
-void hmm_free(ia_css_ptr virt)
-{
-	struct hmm_buffer_object *bo;
-
-	WARN_ON(!virt);
-
-	bo = hmm_bo_device_search_start(&bo_device, (unsigned int)virt);
-
-	if (!bo) {
-		dev_err(atomisp_dev,
-			"can not find buffer object start with address 0x%x\n",
-			(unsigned int)virt);
-		return;
-	}
-
-	hmm_mem_stat.tol_cnt -= bo->pgnr;
-
-	hmm_bo_unbind(bo);
-	hmm_bo_free_pages(bo);
-	hmm_bo_unref(bo);
-}
-
-static inline int hmm_check_bo(struct hmm_buffer_object *bo, unsigned int ptr)
-{
-	if (!bo) {
-		dev_err(atomisp_dev,
-			"can not find buffer object contains address 0x%x\n",
-			ptr);
-		return -EINVAL;
-	}
-
-	if (!hmm_bo_page_allocated(bo)) {
-		dev_err(atomisp_dev,
-			"buffer object has no page allocated.\n");
-		return -EINVAL;
-	}
-
-	if (!hmm_bo_allocated(bo)) {
-		dev_err(atomisp_dev,
-			"buffer object has no virtual address space allocated.\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/* Read function in ISP memory management */
-static int load_and_flush_by_kmap(ia_css_ptr virt, void *data,
-				  unsigned int bytes)
-{
-	struct hmm_buffer_object *bo;
-	unsigned int idx, offset, len;
-	char *src, *des;
-	int ret;
-
-	bo = hmm_bo_device_search_in_range(&bo_device, virt);
-	ret = hmm_check_bo(bo, virt);
-	if (ret)
-		return ret;
-
-	des = (char *)data;
-	while (bytes) {
-		idx = (virt - bo->start) >> PAGE_SHIFT;
-		offset = (virt - bo->start) - (idx << PAGE_SHIFT);
-
-		src = (char *)kmap(bo->page_obj[idx].page) + offset;
-
-		if ((bytes + offset) >= PAGE_SIZE) {
-			len = PAGE_SIZE - offset;
-			bytes -= len;
-		} else {
-			len = bytes;
-			bytes = 0;
-		}
-
-		virt += len;	/* update virt for next loop */
-
-		if (des) {
-			memcpy(des, src, len);
-			des += len;
-		}
-
-		clflush_cache_range(src, len);
-
-		kunmap(bo->page_obj[idx].page);
-	}
-
-	return 0;
-}
-
-/* Read function in ISP memory management */
-static int load_and_flush(ia_css_ptr virt, void *data, unsigned int bytes)
-{
-	struct hmm_buffer_object *bo;
-	int ret;
-
-	bo = hmm_bo_device_search_in_range(&bo_device, virt);
-	ret = hmm_check_bo(bo, virt);
-	if (ret)
-		return ret;
-
-	if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
-		void *src = bo->vmap_addr;
-
-		src += (virt - bo->start);
-		memcpy(data, src, bytes);
-		if (bo->status & HMM_BO_VMAPED_CACHED)
-			clflush_cache_range(src, bytes);
-	} else {
-		void *vptr;
-
-		vptr = hmm_bo_vmap(bo, true);
-		if (!vptr)
-			return load_and_flush_by_kmap(virt, data, bytes);
-		else
-			vptr = vptr + (virt - bo->start);
-
-		memcpy(data, vptr, bytes);
-		clflush_cache_range(vptr, bytes);
-		hmm_bo_vunmap(bo);
-	}
-
-	return 0;
-}
-
-/* Read function in ISP memory management */
-int hmm_load(ia_css_ptr virt, void *data, unsigned int bytes)
-{
-	if (!data) {
-		dev_err(atomisp_dev,
-			"hmm_load NULL argument\n");
-		return -EINVAL;
-	}
-	return load_and_flush(virt, data, bytes);
-}
-
-/* Flush hmm data from the data cache */
-int hmm_flush(ia_css_ptr virt, unsigned int bytes)
-{
-	return load_and_flush(virt, NULL, bytes);
-}
-
-/* Write function in ISP memory management */
-int hmm_store(ia_css_ptr virt, const void *data, unsigned int bytes)
-{
-	struct hmm_buffer_object *bo;
-	unsigned int idx, offset, len;
-	char *src, *des;
-	int ret;
-
-	bo = hmm_bo_device_search_in_range(&bo_device, virt);
-	ret = hmm_check_bo(bo, virt);
-	if (ret)
-		return ret;
-
-	if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
-		void *dst = bo->vmap_addr;
-
-		dst += (virt - bo->start);
-		memcpy(dst, data, bytes);
-		if (bo->status & HMM_BO_VMAPED_CACHED)
-			clflush_cache_range(dst, bytes);
-	} else {
-		void *vptr;
-
-		vptr = hmm_bo_vmap(bo, true);
-		if (vptr) {
-			vptr = vptr + (virt - bo->start);
-
-			memcpy(vptr, data, bytes);
-			clflush_cache_range(vptr, bytes);
-			hmm_bo_vunmap(bo);
-			return 0;
-		}
-	}
-
-	src = (char *)data;
-	while (bytes) {
-		idx = (virt - bo->start) >> PAGE_SHIFT;
-		offset = (virt - bo->start) - (idx << PAGE_SHIFT);
-
-		if (in_atomic())
-			des = (char *)kmap_atomic(bo->page_obj[idx].page);
-		else
-			des = (char *)kmap(bo->page_obj[idx].page);
-
-		if (!des) {
-			dev_err(atomisp_dev,
-				"kmap buffer object page failed: pg_idx = %d\n",
-				idx);
-			return -EINVAL;
-		}
-
-		des += offset;
-
-		if ((bytes + offset) >= PAGE_SIZE) {
-			len = PAGE_SIZE - offset;
-			bytes -= len;
-		} else {
-			len = bytes;
-			bytes = 0;
-		}
-
-		virt += len;
-
-		memcpy(des, src, len);
-
-		src += len;
-
-		clflush_cache_range(des, len);
-
-		if (in_atomic())
-			/*
-			 * Note: kunmap_atomic requires return addr from
-			 * kmap_atomic, not the page. See linux/highmem.h
-			 */
-			kunmap_atomic(des - offset);
-		else
-			kunmap(bo->page_obj[idx].page);
-	}
-
-	return 0;
-}
-
-/* memset function in ISP memory management */
-int hmm_set(ia_css_ptr virt, int c, unsigned int bytes)
-{
-	struct hmm_buffer_object *bo;
-	unsigned int idx, offset, len;
-	char *des;
-	int ret;
-
-	bo = hmm_bo_device_search_in_range(&bo_device, virt);
-	ret = hmm_check_bo(bo, virt);
-	if (ret)
-		return ret;
-
-	if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
-		void *dst = bo->vmap_addr;
-
-		dst += (virt - bo->start);
-		memset(dst, c, bytes);
-
-		if (bo->status & HMM_BO_VMAPED_CACHED)
-			clflush_cache_range(dst, bytes);
-	} else {
-		void *vptr;
-
-		vptr = hmm_bo_vmap(bo, true);
-		if (vptr) {
-			vptr = vptr + (virt - bo->start);
-			memset(vptr, c, bytes);
-			clflush_cache_range(vptr, bytes);
-			hmm_bo_vunmap(bo);
-			return 0;
-		}
-	}
-
-	while (bytes) {
-		idx = (virt - bo->start) >> PAGE_SHIFT;
-		offset = (virt - bo->start) - (idx << PAGE_SHIFT);
-
-		des = (char *)kmap(bo->page_obj[idx].page) + offset;
-
-		if ((bytes + offset) >= PAGE_SIZE) {
-			len = PAGE_SIZE - offset;
-			bytes -= len;
-		} else {
-			len = bytes;
-			bytes = 0;
-		}
-
-		virt += len;
-
-		memset(des, c, len);
-
-		clflush_cache_range(des, len);
-
-		kunmap(bo->page_obj[idx].page);
-	}
-
-	return 0;
-}
-
-/* Virtual address to physical address convert */
-phys_addr_t hmm_virt_to_phys(ia_css_ptr virt)
-{
-	unsigned int idx, offset;
-	struct hmm_buffer_object *bo;
-
-	bo = hmm_bo_device_search_in_range(&bo_device, virt);
-	if (!bo) {
-		dev_err(atomisp_dev,
-			"can not find buffer object contains address 0x%x\n",
-			virt);
-		return -1;
-	}
-
-	idx = (virt - bo->start) >> PAGE_SHIFT;
-	offset = (virt - bo->start) - (idx << PAGE_SHIFT);
-
-	return page_to_phys(bo->page_obj[idx].page) + offset;
-}
-
-int hmm_mmap(struct vm_area_struct *vma, ia_css_ptr virt)
-{
-	struct hmm_buffer_object *bo;
-
-	bo = hmm_bo_device_search_start(&bo_device, virt);
-	if (!bo) {
-		dev_err(atomisp_dev,
-			"can not find buffer object start with address 0x%x\n",
-			virt);
-		return -EINVAL;
-	}
-
-	return hmm_bo_mmap(vma, bo);
-}
-
-/* Map ISP virtual address into IA virtual address */
-void *hmm_vmap(ia_css_ptr virt, bool cached)
-{
-	struct hmm_buffer_object *bo;
-	void *ptr;
-
-	bo = hmm_bo_device_search_in_range(&bo_device, virt);
-	if (!bo) {
-		dev_err(atomisp_dev,
-			"can not find buffer object contains address 0x%x\n",
-			virt);
-		return NULL;
-	}
-
-	ptr = hmm_bo_vmap(bo, cached);
-	if (ptr)
-		return ptr + (virt - bo->start);
-	else
-		return NULL;
-}
-
-/* Flush the memory which is mapped as cached memory through hmm_vmap */
-void hmm_flush_vmap(ia_css_ptr virt)
-{
-	struct hmm_buffer_object *bo;
-
-	bo = hmm_bo_device_search_in_range(&bo_device, virt);
-	if (!bo) {
-		dev_warn(atomisp_dev,
-			 "can not find buffer object contains address 0x%x\n",
-			 virt);
-		return;
-	}
-
-	hmm_bo_flush_vmap(bo);
-}
-
-void hmm_vunmap(ia_css_ptr virt)
-{
-	struct hmm_buffer_object *bo;
-
-	bo = hmm_bo_device_search_in_range(&bo_device, virt);
-	if (!bo) {
-		dev_warn(atomisp_dev,
-			 "can not find buffer object contains address 0x%x\n",
-			 virt);
-		return;
-	}
-
-	hmm_bo_vunmap(bo);
-}
-
-int hmm_pool_register(unsigned int pool_size, enum hmm_pool_type pool_type)
-{
-	switch (pool_type) {
-	case HMM_POOL_TYPE_RESERVED:
-		reserved_pool.pops = &reserved_pops;
-		return reserved_pool.pops->pool_init(&reserved_pool.pool_info,
-						     pool_size);
-	case HMM_POOL_TYPE_DYNAMIC:
-		dynamic_pool.pops = &dynamic_pops;
-		return dynamic_pool.pops->pool_init(&dynamic_pool.pool_info,
-						    pool_size);
-	default:
-		dev_err(atomisp_dev, "invalid pool type.\n");
-		return -EINVAL;
-	}
-}
-
-void hmm_pool_unregister(enum hmm_pool_type pool_type)
-{
-	switch (pool_type) {
-	case HMM_POOL_TYPE_RESERVED:
-		if (reserved_pool.pops && reserved_pool.pops->pool_exit)
-			reserved_pool.pops->pool_exit(&reserved_pool.pool_info);
-		break;
-	case HMM_POOL_TYPE_DYNAMIC:
-		if (dynamic_pool.pops && dynamic_pool.pops->pool_exit)
-			dynamic_pool.pops->pool_exit(&dynamic_pool.pool_info);
-		break;
-	default:
-		dev_err(atomisp_dev, "invalid pool type.\n");
-		break;
-	}
-
-	return;
-}
-
-void *hmm_isp_vaddr_to_host_vaddr(ia_css_ptr ptr, bool cached)
-{
-	return hmm_vmap(ptr, cached);
-	/* vmunmap will be done in hmm_bo_release() */
-}
-
-ia_css_ptr hmm_host_vaddr_to_hrt_vaddr(const void *ptr)
-{
-	struct hmm_buffer_object *bo;
-
-	bo = hmm_bo_device_search_vmap_start(&bo_device, ptr);
-	if (bo)
-		return bo->start;
-
-	dev_err(atomisp_dev,
-		"can not find buffer object whose kernel virtual address is %p\n",
-		ptr);
-	return 0;
-}
-
-void hmm_show_mem_stat(const char *func, const int line)
-{
-	trace_printk("tol_cnt=%d usr_size=%d res_size=%d res_cnt=%d sys_size=%d  dyc_thr=%d dyc_size=%d.\n",
-		     hmm_mem_stat.tol_cnt,
-		     hmm_mem_stat.usr_size, hmm_mem_stat.res_size,
-		     hmm_mem_stat.res_cnt, hmm_mem_stat.sys_size,
-		     hmm_mem_stat.dyc_thr, hmm_mem_stat.dyc_size);
-}
-
-void hmm_init_mem_stat(int res_pgnr, int dyc_en, int dyc_pgnr)
-{
-	hmm_mem_stat.res_size = res_pgnr;
-	/* If reserved mem pool is not enabled, set its "mem stat" values as -1. */
-	if (0 == hmm_mem_stat.res_size) {
-		hmm_mem_stat.res_size = -1;
-		hmm_mem_stat.res_cnt = -1;
-	}
-
-	/* If dynamic memory pool is not enabled, set its "mem stat" values as -1. */
-	if (!dyc_en) {
-		hmm_mem_stat.dyc_size = -1;
-		hmm_mem_stat.dyc_thr = -1;
-	} else {
-		hmm_mem_stat.dyc_size = 0;
-		hmm_mem_stat.dyc_thr = dyc_pgnr;
-	}
-	hmm_mem_stat.usr_size = 0;
-	hmm_mem_stat.sys_size = 0;
-	hmm_mem_stat.tol_cnt = 0;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c b/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c
deleted file mode 100644
index 79bd540d7882..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c
+++ /dev/null
@@ -1,1528 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-/*
- * This file contains functions for buffer object structure management
- */
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/gfp.h>		/* for GFP_ATOMIC */
-#include <linux/mm.h>
-#include <linux/mm_types.h>
-#include <linux/hugetlb.h>
-#include <linux/highmem.h>
-#include <linux/slab.h>		/* for kmalloc */
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/string.h>
-#include <linux/list.h>
-#include <linux/errno.h>
-#include <linux/io.h>
-#include <asm/current.h>
-#include <linux/sched/signal.h>
-#include <linux/file.h>
-
-#include <asm/set_memory.h>
-
-#include "atomisp_internal.h"
-#include "hmm/hmm_common.h"
-#include "hmm/hmm_pool.h"
-#include "hmm/hmm_bo.h"
-
-static unsigned int order_to_nr(unsigned int order)
-{
-	return 1U << order;
-}
-
-static unsigned int nr_to_order_bottom(unsigned int nr)
-{
-	return fls(nr) - 1;
-}
-
-static struct hmm_buffer_object *__bo_alloc(struct kmem_cache *bo_cache)
-{
-	struct hmm_buffer_object *bo;
-
-	bo = kmem_cache_alloc(bo_cache, GFP_KERNEL);
-	if (!bo)
-		dev_err(atomisp_dev, "%s: failed!\n", __func__);
-
-	return bo;
-}
-
-static int __bo_init(struct hmm_bo_device *bdev, struct hmm_buffer_object *bo,
-					unsigned int pgnr)
-{
-	check_bodev_null_return(bdev, -EINVAL);
-	var_equal_return(hmm_bo_device_inited(bdev), 0, -EINVAL,
-			"hmm_bo_device not inited yet.\n");
-	/* prevent zero size buffer object */
-	if (pgnr == 0) {
-		dev_err(atomisp_dev, "0 size buffer is not allowed.\n");
-		return -EINVAL;
-	}
-
-	memset(bo, 0, sizeof(*bo));
-	mutex_init(&bo->mutex);
-
-	/* init the bo->list HEAD as an element of entire_bo_list */
-	INIT_LIST_HEAD(&bo->list);
-
-	bo->bdev = bdev;
-	bo->vmap_addr = NULL;
-	bo->status = HMM_BO_FREE;
-	bo->start = bdev->start;
-	bo->pgnr = pgnr;
-	bo->end = bo->start + pgnr_to_size(pgnr);
-	bo->prev = NULL;
-	bo->next = NULL;
-
-	return 0;
-}
-
-static struct hmm_buffer_object *__bo_search_and_remove_from_free_rbtree(
-				struct rb_node *node, unsigned int pgnr)
-{
-	struct hmm_buffer_object *this, *ret_bo, *temp_bo;
-
-	this = rb_entry(node, struct hmm_buffer_object, node);
-	if (this->pgnr == pgnr ||
-		(this->pgnr > pgnr && this->node.rb_left == NULL)) {
-		goto remove_bo_and_return;
-	} else {
-		if (this->pgnr < pgnr) {
-			if (!this->node.rb_right)
-				return NULL;
-			ret_bo = __bo_search_and_remove_from_free_rbtree(
-				this->node.rb_right, pgnr);
-		} else {
-			ret_bo = __bo_search_and_remove_from_free_rbtree(
-				this->node.rb_left, pgnr);
-		}
-		if (!ret_bo) {
-			if (this->pgnr > pgnr)
-				goto remove_bo_and_return;
-			else
-				return NULL;
-		}
-		return ret_bo;
-	}
-
-remove_bo_and_return:
-	/* NOTE: All nodes on free rbtree have a 'prev' that points to NULL.
-	 * 1. check if 'this->next' is NULL:
-	 *	yes: erase 'this' node and rebalance rbtree, return 'this'.
-	 */
-	if (this->next == NULL) {
-		rb_erase(&this->node, &this->bdev->free_rbtree);
-		return this;
-	}
-	/* NOTE: if 'this->next' is not NULL, always return 'this->next' bo.
-	 * 2. check if 'this->next->next' is NULL:
-	 *	yes: change the related 'next/prev' pointer,
-	 *		return 'this->next' but the rbtree stays unchanged.
-	 */
-	temp_bo = this->next;
-	this->next = temp_bo->next;
-	if (temp_bo->next)
-		temp_bo->next->prev = this;
-	temp_bo->next = NULL;
-	temp_bo->prev = NULL;
-	return temp_bo;
-}
-
-static struct hmm_buffer_object *__bo_search_by_addr(struct rb_root *root,
-							ia_css_ptr start)
-{
-	struct rb_node *n = root->rb_node;
-	struct hmm_buffer_object *bo;
-
-	do {
-		bo = rb_entry(n, struct hmm_buffer_object, node);
-
-		if (bo->start > start) {
-			if (n->rb_left == NULL)
-				return NULL;
-			n = n->rb_left;
-		} else if (bo->start < start) {
-			if (n->rb_right == NULL)
-				return NULL;
-			n = n->rb_right;
-		} else {
-			return bo;
-		}
-	} while (n);
-
-	return NULL;
-}
-
-static struct hmm_buffer_object *__bo_search_by_addr_in_range(
-		struct rb_root *root, unsigned int start)
-{
-	struct rb_node *n = root->rb_node;
-	struct hmm_buffer_object *bo;
-
-	do {
-		bo = rb_entry(n, struct hmm_buffer_object, node);
-
-		if (bo->start > start) {
-			if (n->rb_left == NULL)
-				return NULL;
-			n = n->rb_left;
-		} else {
-			if (bo->end > start)
-				return bo;
-			if (n->rb_right == NULL)
-				return NULL;
-			n = n->rb_right;
-		}
-	} while (n);
-
-	return NULL;
-}
-
-static void __bo_insert_to_free_rbtree(struct rb_root *root,
-					struct hmm_buffer_object *bo)
-{
-	struct rb_node **new = &(root->rb_node);
-	struct rb_node *parent = NULL;
-	struct hmm_buffer_object *this;
-	unsigned int pgnr = bo->pgnr;
-
-	while (*new) {
-		parent = *new;
-		this = container_of(*new, struct hmm_buffer_object, node);
-
-		if (pgnr < this->pgnr) {
-			new = &((*new)->rb_left);
-		} else if (pgnr > this->pgnr) {
-			new = &((*new)->rb_right);
-		} else {
-			bo->prev = this;
-			bo->next = this->next;
-			if (this->next)
-				this->next->prev = bo;
-			this->next = bo;
-			bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_FREE;
-			return;
-		}
-	}
-
-	bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_FREE;
-
-	rb_link_node(&bo->node, parent, new);
-	rb_insert_color(&bo->node, root);
-}
-
-static void __bo_insert_to_alloc_rbtree(struct rb_root *root,
-					struct hmm_buffer_object *bo)
-{
-	struct rb_node **new = &(root->rb_node);
-	struct rb_node *parent = NULL;
-	struct hmm_buffer_object *this;
-	unsigned int start = bo->start;
-
-	while (*new) {
-		parent = *new;
-		this = container_of(*new, struct hmm_buffer_object, node);
-
-		if (start < this->start)
-			new = &((*new)->rb_left);
-		else
-			new = &((*new)->rb_right);
-	}
-
-	kref_init(&bo->kref);
-	bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_ALLOCED;
-
-	rb_link_node(&bo->node, parent, new);
-	rb_insert_color(&bo->node, root);
-}
-
-static struct hmm_buffer_object *__bo_break_up(struct hmm_bo_device *bdev,
-					struct hmm_buffer_object *bo,
-					unsigned int pgnr)
-{
-	struct hmm_buffer_object *new_bo;
-	unsigned long flags;
-	int ret;
-
-	new_bo = __bo_alloc(bdev->bo_cache);
-	if (!new_bo) {
-		dev_err(atomisp_dev, "%s: __bo_alloc failed!\n", __func__);
-		return NULL;
-	}
-	ret = __bo_init(bdev, new_bo, pgnr);
-	if (ret) {
-		dev_err(atomisp_dev, "%s: __bo_init failed!\n", __func__);
-		kmem_cache_free(bdev->bo_cache, new_bo);
-		return NULL;
-	}
-
-	new_bo->start = bo->start;
-	new_bo->end = new_bo->start + pgnr_to_size(pgnr);
-	bo->start = new_bo->end;
-	bo->pgnr = bo->pgnr - pgnr;
-
-	spin_lock_irqsave(&bdev->list_lock, flags);
-	list_add_tail(&new_bo->list, &bo->list);
-	spin_unlock_irqrestore(&bdev->list_lock, flags);
-
-	return new_bo;
-}
-
-static void __bo_take_off_handling(struct hmm_buffer_object *bo)
-{
-	struct hmm_bo_device *bdev = bo->bdev;
-	/* There are 4 situations when we take off a known bo from free rbtree:
-	 * 1. if bo->next && bo->prev == NULL, bo is a rbtree node
-	 *	and does not have a linked list after bo, to take off this bo,
-	 *	we just need erase bo directly and rebalance the free rbtree
-	 */
-	if (bo->prev == NULL && bo->next == NULL) {
-		rb_erase(&bo->node, &bdev->free_rbtree);
-	/* 2. when bo->next != NULL && bo->prev == NULL, bo is a rbtree node,
-	 *	and has a linked list,to take off this bo we need erase bo
-	 *	first, then, insert bo->next into free rbtree and rebalance
-	 *	the free rbtree
-	 */
-	} else if (bo->prev == NULL && bo->next != NULL) {
-		bo->next->prev = NULL;
-		rb_erase(&bo->node, &bdev->free_rbtree);
-		__bo_insert_to_free_rbtree(&bdev->free_rbtree, bo->next);
-		bo->next = NULL;
-	/* 3. when bo->prev != NULL && bo->next == NULL, bo is not a rbtree
-	 *	node, bo is the last element of the linked list after rbtree
-	 *	node, to take off this bo, we just need set the "prev/next"
-	 *	pointers to NULL, the free rbtree stays unchaged
-	 */
-	} else if (bo->prev != NULL && bo->next == NULL) {
-		bo->prev->next = NULL;
-		bo->prev = NULL;
-	/* 4. when bo->prev != NULL && bo->next != NULL ,bo is not a rbtree
-	 *	node, bo is in the middle of the linked list after rbtree node,
-	 *	to take off this bo, we just set take the "prev/next" pointers
-	 *	to NULL, the free rbtree stays unchaged
-	 */
-	} else {
-		bo->next->prev = bo->prev;
-		bo->prev->next = bo->next;
-		bo->next = NULL;
-		bo->prev = NULL;
-	}
-}
-
-static struct hmm_buffer_object *__bo_merge(struct hmm_buffer_object *bo,
-					struct hmm_buffer_object *next_bo)
-{
-	struct hmm_bo_device *bdev;
-	unsigned long flags;
-
-	bdev = bo->bdev;
-	next_bo->start = bo->start;
-	next_bo->pgnr = next_bo->pgnr + bo->pgnr;
-
-	spin_lock_irqsave(&bdev->list_lock, flags);
-	list_del(&bo->list);
-	spin_unlock_irqrestore(&bdev->list_lock, flags);
-
-	kmem_cache_free(bo->bdev->bo_cache, bo);
-
-	return next_bo;
-}
-
-/*
- * hmm_bo_device functions.
- */
-int hmm_bo_device_init(struct hmm_bo_device *bdev,
-				struct isp_mmu_client *mmu_driver,
-				unsigned int vaddr_start,
-				unsigned int size)
-{
-	struct hmm_buffer_object *bo;
-	unsigned long flags;
-	int ret;
-
-	check_bodev_null_return(bdev, -EINVAL);
-
-	ret = isp_mmu_init(&bdev->mmu, mmu_driver);
-	if (ret) {
-		dev_err(atomisp_dev, "isp_mmu_init failed.\n");
-		return ret;
-	}
-
-	bdev->start = vaddr_start;
-	bdev->pgnr = size_to_pgnr_ceil(size);
-	bdev->size = pgnr_to_size(bdev->pgnr);
-
-	spin_lock_init(&bdev->list_lock);
-	mutex_init(&bdev->rbtree_mutex);
-
-	bdev->flag = HMM_BO_DEVICE_INITED;
-
-	INIT_LIST_HEAD(&bdev->entire_bo_list);
-	bdev->allocated_rbtree = RB_ROOT;
-	bdev->free_rbtree = RB_ROOT;
-
-	bdev->bo_cache = kmem_cache_create("bo_cache",
-				sizeof(struct hmm_buffer_object), 0, 0, NULL);
-	if (!bdev->bo_cache) {
-		dev_err(atomisp_dev, "%s: create cache failed!\n", __func__);
-		isp_mmu_exit(&bdev->mmu);
-		return -ENOMEM;
-	}
-
-	bo = __bo_alloc(bdev->bo_cache);
-	if (!bo) {
-		dev_err(atomisp_dev, "%s: __bo_alloc failed!\n", __func__);
-		isp_mmu_exit(&bdev->mmu);
-		return -ENOMEM;
-	}
-
-	ret = __bo_init(bdev, bo, bdev->pgnr);
-	if (ret) {
-		dev_err(atomisp_dev, "%s: __bo_init failed!\n", __func__);
-		kmem_cache_free(bdev->bo_cache, bo);
-		isp_mmu_exit(&bdev->mmu);
-		return -EINVAL;
-	}
-
-	spin_lock_irqsave(&bdev->list_lock, flags);
-	list_add_tail(&bo->list, &bdev->entire_bo_list);
-	spin_unlock_irqrestore(&bdev->list_lock, flags);
-
-	__bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
-
-	return 0;
-}
-
-struct hmm_buffer_object *hmm_bo_alloc(struct hmm_bo_device *bdev,
-					unsigned int pgnr)
-{
-	struct hmm_buffer_object *bo, *new_bo;
-	struct rb_root *root = &bdev->free_rbtree;
-
-	check_bodev_null_return(bdev, NULL);
-	var_equal_return(hmm_bo_device_inited(bdev), 0, NULL,
-			"hmm_bo_device not inited yet.\n");
-
-	if (pgnr == 0) {
-		dev_err(atomisp_dev, "0 size buffer is not allowed.\n");
-		return NULL;
-	}
-
-	mutex_lock(&bdev->rbtree_mutex);
-	bo = __bo_search_and_remove_from_free_rbtree(root->rb_node, pgnr);
-	if (!bo) {
-		mutex_unlock(&bdev->rbtree_mutex);
-		dev_err(atomisp_dev, "%s: Out of Memory! hmm_bo_alloc failed",
-			__func__);
-		return NULL;
-	}
-
-	if (bo->pgnr > pgnr) {
-		new_bo = __bo_break_up(bdev, bo, pgnr);
-		if (!new_bo) {
-			mutex_unlock(&bdev->rbtree_mutex);
-			dev_err(atomisp_dev, "%s: __bo_break_up failed!\n",
-				__func__);
-			return NULL;
-		}
-
-		__bo_insert_to_alloc_rbtree(&bdev->allocated_rbtree, new_bo);
-		__bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
-
-		mutex_unlock(&bdev->rbtree_mutex);
-		return new_bo;
-	}
-
-	__bo_insert_to_alloc_rbtree(&bdev->allocated_rbtree, bo);
-
-	mutex_unlock(&bdev->rbtree_mutex);
-	return bo;
-}
-
-void hmm_bo_release(struct hmm_buffer_object *bo)
-{
-	struct hmm_bo_device *bdev = bo->bdev;
-	struct hmm_buffer_object *next_bo, *prev_bo;
-
-	mutex_lock(&bdev->rbtree_mutex);
-
-	/*
-	 * FIX ME:
-	 *
-	 * how to destroy the bo when it is stilled MMAPED?
-	 *
-	 * ideally, this will not happened as hmm_bo_release
-	 * will only be called when kref reaches 0, and in mmap
-	 * operation the hmm_bo_ref will eventually be called.
-	 * so, if this happened, something goes wrong.
-	 */
-	if (bo->status & HMM_BO_MMAPED) {
-		mutex_unlock(&bdev->rbtree_mutex);
-		dev_dbg(atomisp_dev, "destroy bo which is MMAPED, do nothing\n");
-		return;
-	}
-
-	if (bo->status & HMM_BO_BINDED) {
-		dev_warn(atomisp_dev, "the bo is still binded, unbind it first...\n");
-		hmm_bo_unbind(bo);
-	}
-
-	if (bo->status & HMM_BO_PAGE_ALLOCED) {
-		dev_warn(atomisp_dev, "the pages is not freed, free pages first\n");
-		hmm_bo_free_pages(bo);
-	}
-	if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
-		dev_warn(atomisp_dev, "the vunmap is not done, do it...\n");
-		hmm_bo_vunmap(bo);
-	}
-
-	rb_erase(&bo->node, &bdev->allocated_rbtree);
-
-	prev_bo = list_entry(bo->list.prev, struct hmm_buffer_object, list);
-	next_bo = list_entry(bo->list.next, struct hmm_buffer_object, list);
-
-	if (bo->list.prev != &bdev->entire_bo_list &&
-		prev_bo->end == bo->start &&
-		(prev_bo->status & HMM_BO_MASK) == HMM_BO_FREE) {
-		__bo_take_off_handling(prev_bo);
-		bo = __bo_merge(prev_bo, bo);
-	}
-
-	if (bo->list.next != &bdev->entire_bo_list &&
-		next_bo->start == bo->end &&
-		(next_bo->status & HMM_BO_MASK) == HMM_BO_FREE) {
-		__bo_take_off_handling(next_bo);
-		bo = __bo_merge(bo, next_bo);
-	}
-
-	__bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
-
-	mutex_unlock(&bdev->rbtree_mutex);
-	return;
-}
-
-void hmm_bo_device_exit(struct hmm_bo_device *bdev)
-{
-	struct hmm_buffer_object *bo;
-	unsigned long flags;
-
-	dev_dbg(atomisp_dev, "%s: entering!\n", __func__);
-
-	check_bodev_null_return_void(bdev);
-
-	/*
-	 * release all allocated bos even they a in use
-	 * and all bos will be merged into a big bo
-	 */
-	while (!RB_EMPTY_ROOT(&bdev->allocated_rbtree))
-		hmm_bo_release(
-			rbtree_node_to_hmm_bo(bdev->allocated_rbtree.rb_node));
-
-	dev_dbg(atomisp_dev, "%s: finished releasing all allocated bos!\n",
-		__func__);
-
-	/* free all bos to release all ISP virtual memory */
-	while (!list_empty(&bdev->entire_bo_list)) {
-		bo = list_to_hmm_bo(bdev->entire_bo_list.next);
-
-		spin_lock_irqsave(&bdev->list_lock, flags);
-		list_del(&bo->list);
-		spin_unlock_irqrestore(&bdev->list_lock, flags);
-
-		kmem_cache_free(bdev->bo_cache, bo);
-	}
-
-	dev_dbg(atomisp_dev, "%s: finished to free all bos!\n", __func__);
-
-	kmem_cache_destroy(bdev->bo_cache);
-
-	isp_mmu_exit(&bdev->mmu);
-}
-
-int hmm_bo_device_inited(struct hmm_bo_device *bdev)
-{
-	check_bodev_null_return(bdev, -EINVAL);
-
-	return bdev->flag == HMM_BO_DEVICE_INITED;
-}
-
-int hmm_bo_allocated(struct hmm_buffer_object *bo)
-{
-	check_bo_null_return(bo, 0);
-
-	return bo->status & HMM_BO_ALLOCED;
-}
-
-struct hmm_buffer_object *hmm_bo_device_search_start(
-	struct hmm_bo_device *bdev, ia_css_ptr vaddr)
-{
-	struct hmm_buffer_object *bo;
-
-	check_bodev_null_return(bdev, NULL);
-
-	mutex_lock(&bdev->rbtree_mutex);
-	bo = __bo_search_by_addr(&bdev->allocated_rbtree, vaddr);
-	if (!bo) {
-		mutex_unlock(&bdev->rbtree_mutex);
-		dev_err(atomisp_dev, "%s can not find bo with addr: 0x%x\n",
-			__func__, vaddr);
-		return NULL;
-	}
-	mutex_unlock(&bdev->rbtree_mutex);
-
-	return bo;
-}
-
-struct hmm_buffer_object *hmm_bo_device_search_in_range(
-	struct hmm_bo_device *bdev, unsigned int vaddr)
-{
-	struct hmm_buffer_object *bo;
-
-	check_bodev_null_return(bdev, NULL);
-
-	mutex_lock(&bdev->rbtree_mutex);
-	bo = __bo_search_by_addr_in_range(&bdev->allocated_rbtree, vaddr);
-	if (!bo) {
-		mutex_unlock(&bdev->rbtree_mutex);
-		dev_err(atomisp_dev, "%s can not find bo contain addr: 0x%x\n",
-			__func__, vaddr);
-		return NULL;
-	}
-	mutex_unlock(&bdev->rbtree_mutex);
-
-	return bo;
-}
-
-struct hmm_buffer_object *hmm_bo_device_search_vmap_start(
-	struct hmm_bo_device *bdev, const void *vaddr)
-{
-	struct list_head *pos;
-	struct hmm_buffer_object *bo;
-	unsigned long flags;
-
-	check_bodev_null_return(bdev, NULL);
-
-	spin_lock_irqsave(&bdev->list_lock, flags);
-	list_for_each(pos, &bdev->entire_bo_list) {
-		bo = list_to_hmm_bo(pos);
-		/* pass bo which has no vm_node allocated */
-		if ((bo->status & HMM_BO_MASK) == HMM_BO_FREE)
-			continue;
-		if (bo->vmap_addr == vaddr)
-			goto found;
-	}
-	spin_unlock_irqrestore(&bdev->list_lock, flags);
-	return NULL;
-found:
-	spin_unlock_irqrestore(&bdev->list_lock, flags);
-	return bo;
-
-}
-
-
-static void free_private_bo_pages(struct hmm_buffer_object *bo,
-				struct hmm_pool *dypool,
-				struct hmm_pool *repool,
-				int free_pgnr)
-{
-	int i, ret;
-
-	for (i = 0; i < free_pgnr; i++) {
-		switch (bo->page_obj[i].type) {
-		case HMM_PAGE_TYPE_RESERVED:
-			if (repool->pops
-			    && repool->pops->pool_free_pages) {
-				repool->pops->pool_free_pages(repool->pool_info,
-							&bo->page_obj[i]);
-				hmm_mem_stat.res_cnt--;
-			}
-			break;
-		/*
-		 * HMM_PAGE_TYPE_GENERAL indicates that pages are from system
-		 * memory, so when free them, they should be put into dynamic
-		 * pool.
-		 */
-		case HMM_PAGE_TYPE_DYNAMIC:
-		case HMM_PAGE_TYPE_GENERAL:
-			if (dypool->pops
-			    && dypool->pops->pool_inited
-			    && dypool->pops->pool_inited(dypool->pool_info)) {
-				if (dypool->pops->pool_free_pages)
-					dypool->pops->pool_free_pages(
-							      dypool->pool_info,
-							      &bo->page_obj[i]);
-				break;
-			}
-
-			/*
-			 * if dynamic memory pool doesn't exist, need to free
-			 * pages to system directly.
-			 */
-		default:
-			ret = set_pages_wb(bo->page_obj[i].page, 1);
-			if (ret)
-				dev_err(atomisp_dev,
-						"set page to WB err ...ret = %d\n",
-							ret);
-			/*
-			W/A: set_pages_wb seldom return value = -EFAULT
-			indicate that address of page is not in valid
-			range(0xffff880000000000~0xffffc7ffffffffff)
-			then, _free_pages would panic; Do not know why page
-			address be valid,it maybe memory corruption by lowmemory
-			*/
-			if (!ret) {
-				__free_pages(bo->page_obj[i].page, 0);
-				hmm_mem_stat.sys_size--;
-			}
-			break;
-		}
-	}
-
-	return;
-}
-
-/*Allocate pages which will be used only by ISP*/
-static int alloc_private_pages(struct hmm_buffer_object *bo,
-				int from_highmem,
-				bool cached,
-				struct hmm_pool *dypool,
-				struct hmm_pool *repool)
-{
-	int ret;
-	unsigned int pgnr, order, blk_pgnr, alloc_pgnr;
-	struct page *pages;
-	gfp_t gfp = GFP_NOWAIT | __GFP_NOWARN; /* REVISIT: need __GFP_FS too? */
-	int i, j;
-	int failure_number = 0;
-	bool reduce_order = false;
-	bool lack_mem = true;
-
-	if (from_highmem)
-		gfp |= __GFP_HIGHMEM;
-
-	pgnr = bo->pgnr;
-
-	bo->page_obj = kmalloc_array(pgnr, sizeof(struct hmm_page_object),
-				GFP_KERNEL);
-	if (unlikely(!bo->page_obj))
-		return -ENOMEM;
-
-	i = 0;
-	alloc_pgnr = 0;
-
-	/*
-	 * get physical pages from dynamic pages pool.
-	 */
-	if (dypool->pops && dypool->pops->pool_alloc_pages) {
-		alloc_pgnr = dypool->pops->pool_alloc_pages(dypool->pool_info,
-							bo->page_obj, pgnr,
-							cached);
-		hmm_mem_stat.dyc_size -= alloc_pgnr;
-
-		if (alloc_pgnr == pgnr)
-			return 0;
-	}
-
-	pgnr -= alloc_pgnr;
-	i += alloc_pgnr;
-
-	/*
-	 * get physical pages from reserved pages pool for atomisp.
-	 */
-	if (repool->pops && repool->pops->pool_alloc_pages) {
-		alloc_pgnr = repool->pops->pool_alloc_pages(repool->pool_info,
-							&bo->page_obj[i], pgnr,
-							cached);
-		hmm_mem_stat.res_cnt += alloc_pgnr;
-		if (alloc_pgnr == pgnr)
-			return 0;
-	}
-
-	pgnr -= alloc_pgnr;
-	i += alloc_pgnr;
-
-	while (pgnr) {
-		order = nr_to_order_bottom(pgnr);
-		/*
-		 * if be short of memory, we will set order to 0
-		 * everytime.
-		 */
-		if (lack_mem)
-			order = HMM_MIN_ORDER;
-		else if (order > HMM_MAX_ORDER)
-			order = HMM_MAX_ORDER;
-retry:
-		/*
-		 * When order > HMM_MIN_ORDER, for performance reasons we don't
-		 * want alloc_pages() to sleep. In case it fails and fallbacks
-		 * to HMM_MIN_ORDER or in case the requested order is originally
-		 * the minimum value, we can allow alloc_pages() to sleep for
-		 * robustness purpose.
-		 *
-		 * REVISIT: why __GFP_FS is necessary?
-		 */
-		if (order == HMM_MIN_ORDER) {
-			gfp &= ~GFP_NOWAIT;
-			gfp |= __GFP_RECLAIM | __GFP_FS;
-		}
-
-		pages = alloc_pages(gfp, order);
-		if (unlikely(!pages)) {
-			/*
-			 * in low memory case, if allocation page fails,
-			 * we turn to try if order=0 allocation could
-			 * succeed. if order=0 fails too, that means there is
-			 * no memory left.
-			 */
-			if (order == HMM_MIN_ORDER) {
-				dev_err(atomisp_dev,
-					"%s: cannot allocate pages\n",
-					 __func__);
-				goto cleanup;
-			}
-			order = HMM_MIN_ORDER;
-			failure_number++;
-			reduce_order = true;
-			/*
-			 * if fail two times continuously, we think be short
-			 * of memory now.
-			 */
-			if (failure_number == 2) {
-				lack_mem = true;
-				failure_number = 0;
-			}
-			goto retry;
-		} else {
-			blk_pgnr = order_to_nr(order);
-
-			if (!cached) {
-				/*
-				 * set memory to uncacheable -- UC_MINUS
-				 */
-				ret = set_pages_uc(pages, blk_pgnr);
-				if (ret) {
-					dev_err(atomisp_dev,
-						     "set page uncacheable"
-							"failed.\n");
-
-					__free_pages(pages, order);
-
-					goto cleanup;
-				}
-			}
-
-			for (j = 0; j < blk_pgnr; j++) {
-				bo->page_obj[i].page = pages + j;
-				bo->page_obj[i++].type = HMM_PAGE_TYPE_GENERAL;
-			}
-
-			pgnr -= blk_pgnr;
-			hmm_mem_stat.sys_size += blk_pgnr;
-
-			/*
-			 * if order is not reduced this time, clear
-			 * failure_number.
-			 */
-			if (reduce_order)
-				reduce_order = false;
-			else
-				failure_number = 0;
-		}
-	}
-
-	return 0;
-cleanup:
-	alloc_pgnr = i;
-	free_private_bo_pages(bo, dypool, repool, alloc_pgnr);
-
-	kfree(bo->page_obj);
-
-	return -ENOMEM;
-}
-
-static void free_private_pages(struct hmm_buffer_object *bo,
-				struct hmm_pool *dypool,
-				struct hmm_pool *repool)
-{
-	free_private_bo_pages(bo, dypool, repool, bo->pgnr);
-
-	kfree(bo->page_obj);
-}
-
-/*
- * Hacked from kernel function __get_user_pages in mm/memory.c
- *
- * Handle buffers allocated by other kernel space driver and mmaped into user
- * space, function Ignore the VM_PFNMAP and VM_IO flag in VMA structure
- *
- * Get physical pages from user space virtual address and update into page list
- */
-static int __get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm,
-			      unsigned long start, int nr_pages,
-			      unsigned int gup_flags, struct page **pages,
-			      struct vm_area_struct **vmas)
-{
-	int i, ret;
-	unsigned long vm_flags;
-
-	if (nr_pages <= 0)
-		return 0;
-
-	VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
-
-	/*
-	 * Require read or write permissions.
-	 * If FOLL_FORCE is set, we only require the "MAY" flags.
-	 */
-	vm_flags  = (gup_flags & FOLL_WRITE) ?
-			(VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
-	vm_flags &= (gup_flags & FOLL_FORCE) ?
-			(VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
-	i = 0;
-
-	do {
-		struct vm_area_struct *vma;
-
-		vma = find_vma(mm, start);
-		if (!vma) {
-			dev_err(atomisp_dev, "find_vma failed\n");
-			return i ? : -EFAULT;
-		}
-
-		if (is_vm_hugetlb_page(vma)) {
-			/*
-			i = follow_hugetlb_page(mm, vma, pages, vmas,
-					&start, &nr_pages, i, gup_flags);
-			*/
-			continue;
-		}
-
-		do {
-			struct page *page;
-			unsigned long pfn;
-
-			/*
-			 * If we have a pending SIGKILL, don't keep faulting
-			 * pages and potentially allocating memory.
-			 */
-			if (unlikely(fatal_signal_pending(current))) {
-				dev_err(atomisp_dev,
-					"fatal_signal_pending in %s\n",
-					__func__);
-				return i ? i : -ERESTARTSYS;
-			}
-
-			ret = follow_pfn(vma, start, &pfn);
-			if (ret) {
-				dev_err(atomisp_dev, "follow_pfn() failed\n");
-				return i ? : -EFAULT;
-			}
-
-			page = pfn_to_page(pfn);
-			if (IS_ERR(page))
-				return i ? i : PTR_ERR(page);
-			if (pages) {
-				pages[i] = page;
-				get_page(page);
-				flush_anon_page(vma, page, start);
-				flush_dcache_page(page);
-			}
-			if (vmas)
-				vmas[i] = vma;
-			i++;
-			start += PAGE_SIZE;
-			nr_pages--;
-		} while (nr_pages && start < vma->vm_end);
-	} while (nr_pages);
-
-	return i;
-}
-
-static int get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm,
-		     unsigned long start, int nr_pages, int write, int force,
-		     struct page **pages, struct vm_area_struct **vmas)
-{
-	int flags = FOLL_TOUCH;
-
-	if (pages)
-		flags |= FOLL_GET;
-	if (write)
-		flags |= FOLL_WRITE;
-	if (force)
-		flags |= FOLL_FORCE;
-
-	return __get_pfnmap_pages(tsk, mm, start, nr_pages, flags, pages, vmas);
-}
-
-/*
- * Convert user space virtual address into pages list
- */
-static int alloc_user_pages(struct hmm_buffer_object *bo,
-			      void *userptr, bool cached)
-{
-	int page_nr;
-	int i;
-	struct vm_area_struct *vma;
-	struct page **pages;
-
-	pages = kmalloc_array(bo->pgnr, sizeof(struct page *), GFP_KERNEL);
-	if (unlikely(!pages))
-		return -ENOMEM;
-
-	bo->page_obj = kmalloc_array(bo->pgnr, sizeof(struct hmm_page_object),
-		GFP_KERNEL);
-	if (unlikely(!bo->page_obj)) {
-		kfree(pages);
-		return -ENOMEM;
-	}
-
-	mutex_unlock(&bo->mutex);
-	down_read(&current->mm->mmap_sem);
-	vma = find_vma(current->mm, (unsigned long)userptr);
-	up_read(&current->mm->mmap_sem);
-	if (vma == NULL) {
-		dev_err(atomisp_dev, "find_vma failed\n");
-		kfree(bo->page_obj);
-		kfree(pages);
-		mutex_lock(&bo->mutex);
-		return -EFAULT;
-	}
-	mutex_lock(&bo->mutex);
-	/*
-	 * Handle frame buffer allocated in other kerenl space driver
-	 * and map to user space
-	 */
-	if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
-		page_nr = get_pfnmap_pages(current, current->mm,
-					   (unsigned long)userptr,
-					   (int)(bo->pgnr), 1, 0,
-					   pages, NULL);
-		bo->mem_type = HMM_BO_MEM_TYPE_PFN;
-	} else {
-		/*Handle frame buffer allocated in user space*/
-		mutex_unlock(&bo->mutex);
-		page_nr = get_user_pages_fast((unsigned long)userptr,
-					 (int)(bo->pgnr), 1, pages);
-		mutex_lock(&bo->mutex);
-		bo->mem_type = HMM_BO_MEM_TYPE_USER;
-	}
-
-	/* can be written by caller, not forced */
-	if (page_nr != bo->pgnr) {
-		dev_err(atomisp_dev,
-				"get_user_pages err: bo->pgnr = %d, "
-				"pgnr actually pinned = %d.\n",
-				bo->pgnr, page_nr);
-		goto out_of_mem;
-	}
-
-	for (i = 0; i < bo->pgnr; i++) {
-		bo->page_obj[i].page = pages[i];
-		bo->page_obj[i].type = HMM_PAGE_TYPE_GENERAL;
-	}
-	hmm_mem_stat.usr_size += bo->pgnr;
-	kfree(pages);
-
-	return 0;
-
-out_of_mem:
-	for (i = 0; i < page_nr; i++)
-		put_page(pages[i]);
-	kfree(pages);
-	kfree(bo->page_obj);
-
-	return -ENOMEM;
-}
-
-static void free_user_pages(struct hmm_buffer_object *bo)
-{
-	int i;
-
-	for (i = 0; i < bo->pgnr; i++)
-		put_page(bo->page_obj[i].page);
-	hmm_mem_stat.usr_size -= bo->pgnr;
-
-	kfree(bo->page_obj);
-}
-
-/*
- * allocate/free physical pages for the bo.
- *
- * type indicate where are the pages from. currently we have 3 types
- * of memory: HMM_BO_PRIVATE, HMM_BO_USER, HMM_BO_SHARE.
- *
- * from_highmem is only valid when type is HMM_BO_PRIVATE, it will
- * try to alloc memory from highmem if from_highmem is set.
- *
- * userptr is only valid when type is HMM_BO_USER, it indicates
- * the start address from user space task.
- *
- * from_highmem and userptr will both be ignored when type is
- * HMM_BO_SHARE.
- */
-int hmm_bo_alloc_pages(struct hmm_buffer_object *bo,
-		       enum hmm_bo_type type, int from_highmem,
-		       void *userptr, bool cached)
-{
-	int ret = -EINVAL;
-
-	check_bo_null_return(bo, -EINVAL);
-
-	mutex_lock(&bo->mutex);
-	check_bo_status_no_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
-
-	/*
-	 * TO DO:
-	 * add HMM_BO_USER type
-	 */
-	if (type == HMM_BO_PRIVATE) {
-		ret = alloc_private_pages(bo, from_highmem,
-				cached, &dynamic_pool, &reserved_pool);
-	} else if (type == HMM_BO_USER) {
-		ret = alloc_user_pages(bo, userptr, cached);
-	} else {
-		dev_err(atomisp_dev, "invalid buffer type.\n");
-		ret = -EINVAL;
-	}
-	if (ret)
-		goto alloc_err;
-
-	bo->type = type;
-
-	bo->status |= HMM_BO_PAGE_ALLOCED;
-
-	mutex_unlock(&bo->mutex);
-
-	return 0;
-
-alloc_err:
-	mutex_unlock(&bo->mutex);
-	dev_err(atomisp_dev, "alloc pages err...\n");
-	return ret;
-status_err:
-	mutex_unlock(&bo->mutex);
-	dev_err(atomisp_dev,
-			"buffer object has already page allocated.\n");
-	return -EINVAL;
-}
-
-/*
- * free physical pages of the bo.
- */
-void hmm_bo_free_pages(struct hmm_buffer_object *bo)
-{
-	check_bo_null_return_void(bo);
-
-	mutex_lock(&bo->mutex);
-
-	check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err2);
-
-	/* clear the flag anyway. */
-	bo->status &= (~HMM_BO_PAGE_ALLOCED);
-
-	if (bo->type == HMM_BO_PRIVATE)
-		free_private_pages(bo, &dynamic_pool, &reserved_pool);
-	else if (bo->type == HMM_BO_USER)
-		free_user_pages(bo);
-	else
-		dev_err(atomisp_dev, "invalid buffer type.\n");
-	mutex_unlock(&bo->mutex);
-
-	return;
-
-status_err2:
-	mutex_unlock(&bo->mutex);
-	dev_err(atomisp_dev,
-			"buffer object not page allocated yet.\n");
-}
-
-int hmm_bo_page_allocated(struct hmm_buffer_object *bo)
-{
-	check_bo_null_return(bo, 0);
-
-	return bo->status & HMM_BO_PAGE_ALLOCED;
-}
-
-/*
- * get physical page info of the bo.
- */
-int hmm_bo_get_page_info(struct hmm_buffer_object *bo,
-			 struct hmm_page_object **page_obj, int *pgnr)
-{
-	check_bo_null_return(bo, -EINVAL);
-
-	mutex_lock(&bo->mutex);
-
-	check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
-
-	*page_obj = bo->page_obj;
-	*pgnr = bo->pgnr;
-
-	mutex_unlock(&bo->mutex);
-
-	return 0;
-
-status_err:
-	dev_err(atomisp_dev,
-			"buffer object not page allocated yet.\n");
-	mutex_unlock(&bo->mutex);
-	return -EINVAL;
-}
-
-/*
- * bind the physical pages to a virtual address space.
- */
-int hmm_bo_bind(struct hmm_buffer_object *bo)
-{
-	int ret;
-	unsigned int virt;
-	struct hmm_bo_device *bdev;
-	unsigned int i;
-
-	check_bo_null_return(bo, -EINVAL);
-
-	mutex_lock(&bo->mutex);
-
-	check_bo_status_yes_goto(bo,
-				   HMM_BO_PAGE_ALLOCED | HMM_BO_ALLOCED,
-				   status_err1);
-
-	check_bo_status_no_goto(bo, HMM_BO_BINDED, status_err2);
-
-	bdev = bo->bdev;
-
-	virt = bo->start;
-
-	for (i = 0; i < bo->pgnr; i++) {
-		ret =
-		    isp_mmu_map(&bdev->mmu, virt,
-				page_to_phys(bo->page_obj[i].page), 1);
-		if (ret)
-			goto map_err;
-		virt += (1 << PAGE_SHIFT);
-	}
-
-	/*
-	 * flush TBL here.
-	 *
-	 * theoretically, we donot need to flush TLB as we didnot change
-	 * any existed address mappings, but for Silicon Hive's MMU, its
-	 * really a bug here. I guess when fetching PTEs (page table entity)
-	 * to TLB, its MMU will fetch additional INVALID PTEs automatically
-	 * for performance issue. EX, we only set up 1 page address mapping,
-	 * meaning updating 1 PTE, but the MMU fetches 4 PTE at one time,
-	 * so the additional 3 PTEs are invalid.
-	 */
-	if (bo->start != 0x0)
-		isp_mmu_flush_tlb_range(&bdev->mmu, bo->start,
-						(bo->pgnr << PAGE_SHIFT));
-
-	bo->status |= HMM_BO_BINDED;
-
-	mutex_unlock(&bo->mutex);
-
-	return 0;
-
-map_err:
-	/* unbind the physical pages with related virtual address space */
-	virt = bo->start;
-	for ( ; i > 0; i--) {
-		isp_mmu_unmap(&bdev->mmu, virt, 1);
-		virt += pgnr_to_size(1);
-	}
-
-	mutex_unlock(&bo->mutex);
-	dev_err(atomisp_dev,
-			"setup MMU address mapping failed.\n");
-	return ret;
-
-status_err2:
-	mutex_unlock(&bo->mutex);
-	dev_err(atomisp_dev, "buffer object already binded.\n");
-	return -EINVAL;
-status_err1:
-	mutex_unlock(&bo->mutex);
-	dev_err(atomisp_dev,
-		     "buffer object vm_node or page not allocated.\n");
-	return -EINVAL;
-}
-
-/*
- * unbind the physical pages with related virtual address space.
- */
-void hmm_bo_unbind(struct hmm_buffer_object *bo)
-{
-	unsigned int virt;
-	struct hmm_bo_device *bdev;
-	unsigned int i;
-
-	check_bo_null_return_void(bo);
-
-	mutex_lock(&bo->mutex);
-
-	check_bo_status_yes_goto(bo,
-				   HMM_BO_PAGE_ALLOCED |
-				   HMM_BO_ALLOCED |
-				   HMM_BO_BINDED, status_err);
-
-	bdev = bo->bdev;
-
-	virt = bo->start;
-
-	for (i = 0; i < bo->pgnr; i++) {
-		isp_mmu_unmap(&bdev->mmu, virt, 1);
-		virt += pgnr_to_size(1);
-	}
-
-	/*
-	 * flush TLB as the address mapping has been removed and
-	 * related TLBs should be invalidated.
-	 */
-	isp_mmu_flush_tlb_range(&bdev->mmu, bo->start,
-				(bo->pgnr << PAGE_SHIFT));
-
-	bo->status &= (~HMM_BO_BINDED);
-
-	mutex_unlock(&bo->mutex);
-
-	return;
-
-status_err:
-	mutex_unlock(&bo->mutex);
-	dev_err(atomisp_dev,
-		     "buffer vm or page not allocated or not binded yet.\n");
-}
-
-int hmm_bo_binded(struct hmm_buffer_object *bo)
-{
-	int ret;
-
-	check_bo_null_return(bo, 0);
-
-	mutex_lock(&bo->mutex);
-
-	ret = bo->status & HMM_BO_BINDED;
-
-	mutex_unlock(&bo->mutex);
-
-	return ret;
-}
-
-void *hmm_bo_vmap(struct hmm_buffer_object *bo, bool cached)
-{
-	struct page **pages;
-	int i;
-
-	check_bo_null_return(bo, NULL);
-
-	mutex_lock(&bo->mutex);
-	if (((bo->status & HMM_BO_VMAPED) && !cached) ||
-	    ((bo->status & HMM_BO_VMAPED_CACHED) && cached)) {
-		mutex_unlock(&bo->mutex);
-		return bo->vmap_addr;
-	}
-
-	/* cached status need to be changed, so vunmap first */
-	if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
-		vunmap(bo->vmap_addr);
-		bo->vmap_addr = NULL;
-		bo->status &= ~(HMM_BO_VMAPED | HMM_BO_VMAPED_CACHED);
-	}
-
-	pages = kmalloc_array(bo->pgnr, sizeof(*pages), GFP_KERNEL);
-	if (unlikely(!pages)) {
-		mutex_unlock(&bo->mutex);
-		return NULL;
-	}
-
-	for (i = 0; i < bo->pgnr; i++)
-		pages[i] = bo->page_obj[i].page;
-
-	bo->vmap_addr = vmap(pages, bo->pgnr, VM_MAP,
-		cached ? PAGE_KERNEL : PAGE_KERNEL_NOCACHE);
-	if (unlikely(!bo->vmap_addr)) {
-		kfree(pages);
-		mutex_unlock(&bo->mutex);
-		dev_err(atomisp_dev, "vmap failed...\n");
-		return NULL;
-	}
-	bo->status |= (cached ? HMM_BO_VMAPED_CACHED : HMM_BO_VMAPED);
-
-	kfree(pages);
-
-	mutex_unlock(&bo->mutex);
-	return bo->vmap_addr;
-}
-
-void hmm_bo_flush_vmap(struct hmm_buffer_object *bo)
-{
-	check_bo_null_return_void(bo);
-
-	mutex_lock(&bo->mutex);
-	if (!(bo->status & HMM_BO_VMAPED_CACHED) || !bo->vmap_addr) {
-		mutex_unlock(&bo->mutex);
-		return;
-	}
-
-	clflush_cache_range(bo->vmap_addr, bo->pgnr * PAGE_SIZE);
-	mutex_unlock(&bo->mutex);
-}
-
-void hmm_bo_vunmap(struct hmm_buffer_object *bo)
-{
-	check_bo_null_return_void(bo);
-
-	mutex_lock(&bo->mutex);
-	if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
-		vunmap(bo->vmap_addr);
-		bo->vmap_addr = NULL;
-		bo->status &= ~(HMM_BO_VMAPED | HMM_BO_VMAPED_CACHED);
-	}
-
-	mutex_unlock(&bo->mutex);
-	return;
-}
-
-void hmm_bo_ref(struct hmm_buffer_object *bo)
-{
-	check_bo_null_return_void(bo);
-
-	kref_get(&bo->kref);
-}
-
-static void kref_hmm_bo_release(struct kref *kref)
-{
-	if (!kref)
-		return;
-
-	hmm_bo_release(kref_to_hmm_bo(kref));
-}
-
-void hmm_bo_unref(struct hmm_buffer_object *bo)
-{
-	check_bo_null_return_void(bo);
-
-	kref_put(&bo->kref, kref_hmm_bo_release);
-}
-
-static void hmm_bo_vm_open(struct vm_area_struct *vma)
-{
-	struct hmm_buffer_object *bo =
-	    (struct hmm_buffer_object *)vma->vm_private_data;
-
-	check_bo_null_return_void(bo);
-
-	hmm_bo_ref(bo);
-
-	mutex_lock(&bo->mutex);
-
-	bo->status |= HMM_BO_MMAPED;
-
-	bo->mmap_count++;
-
-	mutex_unlock(&bo->mutex);
-}
-
-static void hmm_bo_vm_close(struct vm_area_struct *vma)
-{
-	struct hmm_buffer_object *bo =
-	    (struct hmm_buffer_object *)vma->vm_private_data;
-
-	check_bo_null_return_void(bo);
-
-	hmm_bo_unref(bo);
-
-	mutex_lock(&bo->mutex);
-
-	bo->mmap_count--;
-
-	if (!bo->mmap_count) {
-		bo->status &= (~HMM_BO_MMAPED);
-		vma->vm_private_data = NULL;
-	}
-
-	mutex_unlock(&bo->mutex);
-}
-
-static const struct vm_operations_struct hmm_bo_vm_ops = {
-	.open = hmm_bo_vm_open,
-	.close = hmm_bo_vm_close,
-};
-
-/*
- * mmap the bo to user space.
- */
-int hmm_bo_mmap(struct vm_area_struct *vma, struct hmm_buffer_object *bo)
-{
-	unsigned int start, end;
-	unsigned int virt;
-	unsigned int pgnr, i;
-	unsigned int pfn;
-
-	check_bo_null_return(bo, -EINVAL);
-
-	check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
-
-	pgnr = bo->pgnr;
-	start = vma->vm_start;
-	end = vma->vm_end;
-
-	/*
-	 * check vma's virtual address space size and buffer object's size.
-	 * must be the same.
-	 */
-	if ((start + pgnr_to_size(pgnr)) != end) {
-		dev_warn(atomisp_dev,
-			     "vma's address space size not equal"
-			     " to buffer object's size");
-		return -EINVAL;
-	}
-
-	virt = vma->vm_start;
-	for (i = 0; i < pgnr; i++) {
-		pfn = page_to_pfn(bo->page_obj[i].page);
-		if (remap_pfn_range(vma, virt, pfn, PAGE_SIZE, PAGE_SHARED)) {
-			dev_warn(atomisp_dev,
-					"remap_pfn_range failed:"
-					" virt = 0x%x, pfn = 0x%x,"
-					" mapped_pgnr = %d\n", virt, pfn, 1);
-			return -EINVAL;
-		}
-		virt += PAGE_SIZE;
-	}
-
-	vma->vm_private_data = bo;
-
-	vma->vm_ops = &hmm_bo_vm_ops;
-	vma->vm_flags |= VM_IO|VM_DONTEXPAND|VM_DONTDUMP;
-
-	/*
-	 * call hmm_bo_vm_open explictly.
-	 */
-	hmm_bo_vm_open(vma);
-
-	return 0;
-
-status_err:
-	dev_err(atomisp_dev, "buffer page not allocated yet.\n");
-	return -EINVAL;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_dynamic_pool.c b/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_dynamic_pool.c
deleted file mode 100644
index f59fd9908257..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_dynamic_pool.c
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-/*
- * This file contains functions for dynamic memory pool management
- */
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/mm.h>
-
-#include <asm/set_memory.h>
-
-#include "atomisp_internal.h"
-
-#include "hmm/hmm_pool.h"
-
-/*
- * dynamic memory pool ops.
- */
-static unsigned int get_pages_from_dynamic_pool(void *pool,
-					struct hmm_page_object *page_obj,
-					unsigned int size, bool cached)
-{
-	struct hmm_page *hmm_page;
-	unsigned long flags;
-	unsigned int i = 0;
-	struct hmm_dynamic_pool_info *dypool_info = pool;
-
-	if (!dypool_info)
-		return 0;
-
-	spin_lock_irqsave(&dypool_info->list_lock, flags);
-	if (dypool_info->initialized) {
-		while (!list_empty(&dypool_info->pages_list)) {
-			hmm_page = list_entry(dypool_info->pages_list.next,
-						struct hmm_page, list);
-
-			list_del(&hmm_page->list);
-			dypool_info->pgnr--;
-			spin_unlock_irqrestore(&dypool_info->list_lock, flags);
-
-			page_obj[i].page = hmm_page->page;
-			page_obj[i++].type = HMM_PAGE_TYPE_DYNAMIC;
-			kmem_cache_free(dypool_info->pgptr_cache, hmm_page);
-
-			if (i == size)
-				return i;
-
-			spin_lock_irqsave(&dypool_info->list_lock, flags);
-		}
-	}
-	spin_unlock_irqrestore(&dypool_info->list_lock, flags);
-
-	return i;
-}
-
-static void free_pages_to_dynamic_pool(void *pool,
-					struct hmm_page_object *page_obj)
-{
-	struct hmm_page *hmm_page;
-	unsigned long flags;
-	int ret;
-	struct hmm_dynamic_pool_info *dypool_info = pool;
-
-	if (!dypool_info)
-		return;
-
-	spin_lock_irqsave(&dypool_info->list_lock, flags);
-	if (!dypool_info->initialized) {
-		spin_unlock_irqrestore(&dypool_info->list_lock, flags);
-		return;
-	}
-	spin_unlock_irqrestore(&dypool_info->list_lock, flags);
-
-	if (page_obj->type == HMM_PAGE_TYPE_RESERVED)
-		return;
-
-	if (dypool_info->pgnr >= dypool_info->pool_size) {
-		/* free page directly back to system */
-		ret = set_pages_wb(page_obj->page, 1);
-		if (ret)
-			dev_err(atomisp_dev,
-				"set page to WB err ...ret=%d\n", ret);
-		/*
-		W/A: set_pages_wb seldom return value = -EFAULT
-		indicate that address of page is not in valid
-		range(0xffff880000000000~0xffffc7ffffffffff)
-		then, _free_pages would panic; Do not know why page
-		address be valid, it maybe memory corruption by lowmemory
-		*/
-		if (!ret) {
-			__free_pages(page_obj->page, 0);
-			hmm_mem_stat.sys_size--;
-		}
-		return;
-	}
-	hmm_page = kmem_cache_zalloc(dypool_info->pgptr_cache,
-						GFP_KERNEL);
-	if (!hmm_page) {
-		/* free page directly */
-		ret = set_pages_wb(page_obj->page, 1);
-		if (ret)
-			dev_err(atomisp_dev,
-				"set page to WB err ...ret=%d\n", ret);
-		if (!ret) {
-			__free_pages(page_obj->page, 0);
-			hmm_mem_stat.sys_size--;
-		}
-		return;
-	}
-
-	hmm_page->page = page_obj->page;
-
-	/*
-	 * add to pages_list of pages_pool
-	 */
-	spin_lock_irqsave(&dypool_info->list_lock, flags);
-	list_add_tail(&hmm_page->list, &dypool_info->pages_list);
-	dypool_info->pgnr++;
-	spin_unlock_irqrestore(&dypool_info->list_lock, flags);
-	hmm_mem_stat.dyc_size++;
-}
-
-static int hmm_dynamic_pool_init(void **pool, unsigned int pool_size)
-{
-	struct hmm_dynamic_pool_info *dypool_info;
-
-	if (pool_size == 0)
-		return 0;
-
-	dypool_info = kmalloc(sizeof(struct hmm_dynamic_pool_info),
-		GFP_KERNEL);
-	if (unlikely(!dypool_info))
-		return -ENOMEM;
-
-	dypool_info->pgptr_cache = kmem_cache_create("pgptr_cache",
-						sizeof(struct hmm_page), 0,
-						SLAB_HWCACHE_ALIGN, NULL);
-	if (!dypool_info->pgptr_cache) {
-		kfree(dypool_info);
-		return -ENOMEM;
-	}
-
-	INIT_LIST_HEAD(&dypool_info->pages_list);
-	spin_lock_init(&dypool_info->list_lock);
-	dypool_info->initialized = true;
-	dypool_info->pool_size = pool_size;
-	dypool_info->pgnr = 0;
-
-	*pool = dypool_info;
-
-	return 0;
-}
-
-static void hmm_dynamic_pool_exit(void **pool)
-{
-	struct hmm_dynamic_pool_info *dypool_info = *pool;
-	struct hmm_page *hmm_page;
-	unsigned long flags;
-	int ret;
-
-	if (!dypool_info)
-		return;
-
-	spin_lock_irqsave(&dypool_info->list_lock, flags);
-	if (!dypool_info->initialized) {
-		spin_unlock_irqrestore(&dypool_info->list_lock, flags);
-		return;
-	}
-	dypool_info->initialized = false;
-
-	while (!list_empty(&dypool_info->pages_list)) {
-		hmm_page = list_entry(dypool_info->pages_list.next,
-					struct hmm_page, list);
-
-		list_del(&hmm_page->list);
-		spin_unlock_irqrestore(&dypool_info->list_lock, flags);
-
-		/* can cause thread sleep, so cannot be put into spin_lock */
-		ret = set_pages_wb(hmm_page->page, 1);
-		if (ret)
-			dev_err(atomisp_dev,
-				"set page to WB err...ret=%d\n", ret);
-		if (!ret) {
-			__free_pages(hmm_page->page, 0);
-			hmm_mem_stat.dyc_size--;
-			hmm_mem_stat.sys_size--;
-		}
-		kmem_cache_free(dypool_info->pgptr_cache, hmm_page);
-		spin_lock_irqsave(&dypool_info->list_lock, flags);
-	}
-
-	spin_unlock_irqrestore(&dypool_info->list_lock, flags);
-
-	kmem_cache_destroy(dypool_info->pgptr_cache);
-
-	kfree(dypool_info);
-
-	*pool = NULL;
-}
-
-static int hmm_dynamic_pool_inited(void *pool)
-{
-	struct hmm_dynamic_pool_info *dypool_info = pool;
-
-	if (!dypool_info)
-		return 0;
-
-	return dypool_info->initialized;
-}
-
-struct hmm_pool_ops dynamic_pops = {
-	.pool_init		= hmm_dynamic_pool_init,
-	.pool_exit		= hmm_dynamic_pool_exit,
-	.pool_alloc_pages	= get_pages_from_dynamic_pool,
-	.pool_free_pages	= free_pages_to_dynamic_pool,
-	.pool_inited		= hmm_dynamic_pool_inited,
-};
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_reserved_pool.c b/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_reserved_pool.c
deleted file mode 100644
index f300e7547997..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_reserved_pool.c
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-/*
- * This file contains functions for reserved memory pool management
- */
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/mm.h>
-
-#include <asm/set_memory.h>
-
-#include "atomisp_internal.h"
-#include "hmm/hmm_pool.h"
-
-/*
- * reserved memory pool ops.
- */
-static unsigned int get_pages_from_reserved_pool(void *pool,
-					struct hmm_page_object *page_obj,
-					unsigned int size, bool cached)
-{
-	unsigned long flags;
-	unsigned int i = 0;
-	unsigned int repool_pgnr;
-	int j;
-	struct hmm_reserved_pool_info *repool_info = pool;
-
-	if (!repool_info)
-		return 0;
-
-	spin_lock_irqsave(&repool_info->list_lock, flags);
-	if (repool_info->initialized) {
-		repool_pgnr = repool_info->index;
-
-		for (j = repool_pgnr-1; j >= 0; j--) {
-			page_obj[i].page = repool_info->pages[j];
-			page_obj[i].type = HMM_PAGE_TYPE_RESERVED;
-			i++;
-			repool_info->index--;
-			if (i == size)
-				break;
-		}
-	}
-	spin_unlock_irqrestore(&repool_info->list_lock, flags);
-	return i;
-}
-
-static void free_pages_to_reserved_pool(void *pool,
-					struct hmm_page_object *page_obj)
-{
-	unsigned long flags;
-	struct hmm_reserved_pool_info *repool_info = pool;
-
-	if (!repool_info)
-		return;
-
-	spin_lock_irqsave(&repool_info->list_lock, flags);
-
-	if (repool_info->initialized &&
-	    repool_info->index < repool_info->pgnr &&
-	    page_obj->type == HMM_PAGE_TYPE_RESERVED) {
-		repool_info->pages[repool_info->index++] = page_obj->page;
-	}
-
-	spin_unlock_irqrestore(&repool_info->list_lock, flags);
-}
-
-static int hmm_reserved_pool_setup(struct hmm_reserved_pool_info **repool_info,
-					unsigned int pool_size)
-{
-	struct hmm_reserved_pool_info *pool_info;
-
-	pool_info = kmalloc(sizeof(struct hmm_reserved_pool_info),
-				GFP_KERNEL);
-	if (unlikely(!pool_info))
-		return -ENOMEM;
-
-	pool_info->pages = kmalloc(sizeof(struct page *) * pool_size,
-			GFP_KERNEL);
-	if (unlikely(!pool_info->pages)) {
-		kfree(pool_info);
-		return -ENOMEM;
-	}
-
-	pool_info->index = 0;
-	pool_info->pgnr = 0;
-	spin_lock_init(&pool_info->list_lock);
-	pool_info->initialized = true;
-
-	*repool_info = pool_info;
-
-	return 0;
-}
-
-static int hmm_reserved_pool_init(void **pool, unsigned int pool_size)
-{
-	int ret;
-	unsigned int blk_pgnr;
-	unsigned int pgnr = pool_size;
-	unsigned int order = 0;
-	unsigned int i = 0;
-	int fail_number = 0;
-	struct page *pages;
-	int j;
-	struct hmm_reserved_pool_info *repool_info;
-	if (pool_size == 0)
-		return 0;
-
-	ret = hmm_reserved_pool_setup(&repool_info, pool_size);
-	if (ret) {
-		dev_err(atomisp_dev, "hmm_reserved_pool_setup failed.\n");
-		return ret;
-	}
-
-	pgnr = pool_size;
-
-	i = 0;
-	order = MAX_ORDER;
-
-	while (pgnr) {
-		blk_pgnr = 1U << order;
-		while (blk_pgnr > pgnr) {
-			order--;
-			blk_pgnr >>= 1U;
-		}
-		BUG_ON(order > MAX_ORDER);
-
-		pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN, order);
-		if (unlikely(!pages)) {
-			if (order == 0) {
-				fail_number++;
-				dev_err(atomisp_dev, "%s: alloc_pages failed: %d\n",
-						__func__, fail_number);
-				/* if fail five times, will goto end */
-
-				/* FIXME: whether is the mechanism is ok? */
-				if (fail_number == ALLOC_PAGE_FAIL_NUM)
-					goto end;
-			} else {
-				order--;
-			}
-		} else {
-			blk_pgnr = 1U << order;
-
-			ret = set_pages_uc(pages, blk_pgnr);
-			if (ret) {
-				dev_err(atomisp_dev,
-						"set pages uncached failed\n");
-				__free_pages(pages, order);
-				goto end;
-			}
-
-			for (j = 0; j < blk_pgnr; j++)
-				repool_info->pages[i++] = pages + j;
-
-			repool_info->index += blk_pgnr;
-			repool_info->pgnr += blk_pgnr;
-
-			pgnr -= blk_pgnr;
-
-			fail_number = 0;
-		}
-	}
-
-end:
-	repool_info->initialized = true;
-
-	*pool = repool_info;
-
-	dev_info(atomisp_dev,
-			"hmm_reserved_pool init successfully,"
-			"hmm_reserved_pool is with %d pages.\n",
-			repool_info->pgnr);
-	return 0;
-}
-
-static void hmm_reserved_pool_exit(void **pool)
-{
-	unsigned long flags;
-	int i, ret;
-	unsigned int pgnr;
-	struct hmm_reserved_pool_info *repool_info = *pool;
-
-	if (!repool_info)
-		return;
-
-	spin_lock_irqsave(&repool_info->list_lock, flags);
-	if (!repool_info->initialized) {
-		spin_unlock_irqrestore(&repool_info->list_lock, flags);
-		return;
-	}
-	pgnr = repool_info->pgnr;
-	repool_info->index = 0;
-	repool_info->pgnr = 0;
-	repool_info->initialized = false;
-	spin_unlock_irqrestore(&repool_info->list_lock, flags);
-
-	for (i = 0; i < pgnr; i++) {
-		ret = set_pages_wb(repool_info->pages[i], 1);
-		if (ret)
-			dev_err(atomisp_dev,
-				"set page to WB err...ret=%d\n", ret);
-		/*
-		W/A: set_pages_wb seldom return value = -EFAULT
-		indicate that address of page is not in valid
-		range(0xffff880000000000~0xffffc7ffffffffff)
-		then, _free_pages would panic; Do not know why
-		page address be valid, it maybe memory corruption by lowmemory
-		*/
-		if (!ret)
-			__free_pages(repool_info->pages[i], 0);
-	}
-
-	kfree(repool_info->pages);
-	kfree(repool_info);
-
-	*pool = NULL;
-}
-
-static int hmm_reserved_pool_inited(void *pool)
-{
-	struct hmm_reserved_pool_info *repool_info = pool;
-
-	if (!repool_info)
-		return 0;
-
-	return repool_info->initialized;
-}
-
-struct hmm_pool_ops reserved_pops = {
-	.pool_init		= hmm_reserved_pool_init,
-	.pool_exit		= hmm_reserved_pool_exit,
-	.pool_alloc_pages	= get_pages_from_reserved_pool,
-	.pool_free_pages	= free_pages_to_reserved_pool,
-	.pool_inited		= hmm_reserved_pool_inited,
-};
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_vm.c b/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_vm.c
deleted file mode 100644
index 0df96e661983..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_vm.c
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-/*
- * This file contains function for ISP virtual address management in ISP driver
- */
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/mm.h>
-#include <linux/slab.h>
-#include <asm/page.h>
-
-#include "atomisp_internal.h"
-#include "mmu/isp_mmu.h"
-#include "hmm/hmm_vm.h"
-#include "hmm/hmm_common.h"
-
-static unsigned int vm_node_end(unsigned int start, unsigned int pgnr)
-{
-	return start + pgnr_to_size(pgnr);
-}
-
-static int addr_in_vm_node(unsigned int addr,
-		struct hmm_vm_node *node)
-{
-	return (addr >= node->start) && (addr < (node->start + node->size));
-}
-
-int hmm_vm_init(struct hmm_vm *vm, unsigned int start,
-		unsigned int size)
-{
-	if (!vm)
-		return -1;
-
-	vm->start = start;
-	vm->pgnr = size_to_pgnr_ceil(size);
-	vm->size = pgnr_to_size(vm->pgnr);
-
-	INIT_LIST_HEAD(&vm->vm_node_list);
-	spin_lock_init(&vm->lock);
-	vm->cache = kmem_cache_create("atomisp_vm", sizeof(struct hmm_vm_node),
-				      0, 0, NULL);
-
-	return vm->cache != NULL ? 0 : -ENOMEM;
-}
-
-void hmm_vm_clean(struct hmm_vm *vm)
-{
-	struct hmm_vm_node *node, *tmp;
-	struct list_head new_head;
-
-	if (!vm)
-		return;
-
-	spin_lock(&vm->lock);
-	list_replace_init(&vm->vm_node_list, &new_head);
-	spin_unlock(&vm->lock);
-
-	list_for_each_entry_safe(node, tmp, &new_head, list) {
-		list_del(&node->list);
-		kmem_cache_free(vm->cache, node);
-	}
-
-	kmem_cache_destroy(vm->cache);
-}
-
-static struct hmm_vm_node *alloc_hmm_vm_node(unsigned int pgnr,
-					     struct hmm_vm *vm)
-{
-	struct hmm_vm_node *node;
-
-	node = kmem_cache_alloc(vm->cache, GFP_KERNEL);
-	if (!node)
-		return NULL;
-
-	INIT_LIST_HEAD(&node->list);
-	node->pgnr = pgnr;
-	node->size = pgnr_to_size(pgnr);
-	node->vm = vm;
-
-	return node;
-}
-
-struct hmm_vm_node *hmm_vm_alloc_node(struct hmm_vm *vm, unsigned int pgnr)
-{
-	struct list_head *head;
-	struct hmm_vm_node *node, *cur, *next;
-	unsigned int vm_start, vm_end;
-	unsigned int addr;
-	unsigned int size;
-
-	if (!vm)
-		return NULL;
-
-	vm_start = vm->start;
-	vm_end = vm_node_end(vm->start, vm->pgnr);
-	size = pgnr_to_size(pgnr);
-
-	addr = vm_start;
-	head = &vm->vm_node_list;
-
-	node = alloc_hmm_vm_node(pgnr, vm);
-	if (!node) {
-		dev_err(atomisp_dev, "no memory to allocate hmm vm node.\n");
-		return NULL;
-	}
-
-	spin_lock(&vm->lock);
-	/*
-	 * if list is empty, the loop code will not be executed.
-	 */
-	list_for_each_entry(cur, head, list) {
-		/* Add gap between vm areas as helper to not hide overflow */
-		addr = PAGE_ALIGN(vm_node_end(cur->start, cur->pgnr) + 1);
-
-		if (list_is_last(&cur->list, head)) {
-			if (addr + size > vm_end) {
-				/* vm area does not have space anymore */
-				spin_unlock(&vm->lock);
-				kmem_cache_free(vm->cache, node);
-				dev_err(atomisp_dev,
-					  "no enough virtual address space.\n");
-				return NULL;
-			}
-
-			/* We still have vm space to add new node to tail */
-			break;
-		}
-
-		next = list_entry(cur->list.next, struct hmm_vm_node, list);
-		if ((next->start - addr) > size)
-			break;
-	}
-	node->start = addr;
-	node->vm = vm;
-	list_add(&node->list, &cur->list);
-	spin_unlock(&vm->lock);
-
-	return node;
-}
-
-void hmm_vm_free_node(struct hmm_vm_node *node)
-{
-	struct hmm_vm *vm;
-
-	if (!node)
-		return;
-
-	vm = node->vm;
-
-	spin_lock(&vm->lock);
-	list_del(&node->list);
-	spin_unlock(&vm->lock);
-
-	kmem_cache_free(vm->cache, node);
-}
-
-struct hmm_vm_node *hmm_vm_find_node_start(struct hmm_vm *vm, unsigned int addr)
-{
-	struct hmm_vm_node *node;
-
-	if (!vm)
-		return NULL;
-
-	spin_lock(&vm->lock);
-
-	list_for_each_entry(node, &vm->vm_node_list, list) {
-		if (node->start == addr) {
-			spin_unlock(&vm->lock);
-			return node;
-		}
-	}
-
-	spin_unlock(&vm->lock);
-	return NULL;
-}
-
-struct hmm_vm_node *hmm_vm_find_node_in_range(struct hmm_vm *vm,
-					      unsigned int addr)
-{
-	struct hmm_vm_node *node;
-
-	if (!vm)
-		return NULL;
-
-	spin_lock(&vm->lock);
-
-	list_for_each_entry(node, &vm->vm_node_list, list) {
-		if (addr_in_vm_node(addr, node)) {
-			spin_unlock(&vm->lock);
-			return node;
-		}
-	}
-
-	spin_unlock(&vm->lock);
-	return NULL;
-}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_custom_host_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_custom_host_hrt.h
deleted file mode 100644
index fb38fc540b81..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_custom_host_hrt.h
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef _hive_isp_css_custom_host_hrt_h_
-#define _hive_isp_css_custom_host_hrt_h_
-
-#include <linux/delay.h>
-#include "atomisp_helper.h"
-
-/*
- * _hrt_master_port_store/load/uload -macros using __force attributed
- * cast to intentional dereferencing __iomem attributed (noderef)
- * pointer from atomisp_get_io_virt_addr
- */
-#define _hrt_master_port_store_8(a, d) \
-	(*((s8 __force *)atomisp_get_io_virt_addr(a)) = (d))
-
-#define _hrt_master_port_store_16(a, d) \
-	(*((s16 __force *)atomisp_get_io_virt_addr(a)) = (d))
-
-#define _hrt_master_port_store_32(a, d) \
-	(*((s32 __force *)atomisp_get_io_virt_addr(a)) = (d))
-
-#define _hrt_master_port_load_8(a) \
-	(*(s8 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_load_16(a) \
-	(*(s16 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_load_32(a) \
-	(*(s32 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_uload_8(a) \
-	(*(u8 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_uload_16(a) \
-	(*(u16 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_uload_32(a) \
-	(*(u32 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_store_8_volatile(a, d)  _hrt_master_port_store_8(a, d)
-#define _hrt_master_port_store_16_volatile(a, d) _hrt_master_port_store_16(a, d)
-#define _hrt_master_port_store_32_volatile(a, d) _hrt_master_port_store_32(a, d)
-
-#define _hrt_master_port_load_8_volatile(a)      _hrt_master_port_load_8(a)
-#define _hrt_master_port_load_16_volatile(a)     _hrt_master_port_load_16(a)
-#define _hrt_master_port_load_32_volatile(a)     _hrt_master_port_load_32(a)
-
-#define _hrt_master_port_uload_8_volatile(a)     _hrt_master_port_uload_8(a)
-#define _hrt_master_port_uload_16_volatile(a)    _hrt_master_port_uload_16(a)
-#define _hrt_master_port_uload_32_volatile(a)    _hrt_master_port_uload_32(a)
-
-static inline void hrt_sleep(void)
-{
-	udelay(1);
-}
-
-static inline uint32_t _hrt_mem_store(uint32_t to, const void *from, size_t n)
-{
-	unsigned i;
-	uint32_t _to = to;
-	const char *_from = (const char *)from;
-	for (i = 0; i < n; i++, _to++, _from++)
-		_hrt_master_port_store_8(_to, *_from);
-	return _to;
-}
-
-static inline void *_hrt_mem_load(uint32_t from, void *to, size_t n)
-{
-	unsigned i;
-	char *_to = (char *)to;
-	uint32_t _from = from;
-	for (i = 0; i < n; i++, _to++, _from++)
-		*_to = _hrt_master_port_load_8(_from);
-	return _to;
-}
-
-static inline uint32_t _hrt_mem_set(uint32_t to, int c, size_t n)
-{
-	unsigned i;
-	uint32_t _to = to;
-	for (i = 0; i < n; i++, _to++)
-		_hrt_master_port_store_8(_to, c);
-	return _to;
-}
-
-#endif /* _hive_isp_css_custom_host_hrt_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.c b/drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.c
deleted file mode 100644
index a94958bde718..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.c
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#include "atomisp_internal.h"
-
-#include "hive_isp_css_mm_hrt.h"
-#include "hmm/hmm.h"
-
-#define __page_align(size)	(((size) + (PAGE_SIZE-1)) & (~(PAGE_SIZE-1)))
-
-static void *my_userptr;
-static unsigned my_num_pages;
-static enum hrt_userptr_type my_usr_type;
-
-void hrt_isp_css_mm_set_user_ptr(void *userptr,
-				 unsigned int num_pages,
-				 enum hrt_userptr_type type)
-{
-	my_userptr = userptr;
-	my_num_pages = num_pages;
-	my_usr_type = type;
-}
-
-static ia_css_ptr __hrt_isp_css_mm_alloc(size_t bytes, void *userptr,
-				    unsigned int num_pages,
-				    enum hrt_userptr_type type,
-				    bool cached)
-{
-#ifdef CONFIG_ION
-	if (type == HRT_USR_ION)
-		return hmm_alloc(bytes, HMM_BO_ION, 0,
-					 userptr, cached);
-
-#endif
-	if (type == HRT_USR_PTR) {
-		if (userptr == NULL)
-			return hmm_alloc(bytes, HMM_BO_PRIVATE, 0,
-						 NULL, cached);
-		else {
-			if (num_pages < ((__page_align(bytes)) >> PAGE_SHIFT))
-				dev_err(atomisp_dev,
-					 "user space memory size is less"
-					 " than the expected size..\n");
-			else if (num_pages > ((__page_align(bytes))
-					      >> PAGE_SHIFT))
-				dev_err(atomisp_dev,
-					 "user space memory size is"
-					 " large than the expected size..\n");
-
-			return hmm_alloc(bytes, HMM_BO_USER, 0,
-						 userptr, cached);
-		}
-	} else {
-		dev_err(atomisp_dev, "user ptr type is incorrect.\n");
-		return 0;
-	}
-}
-
-ia_css_ptr hrt_isp_css_mm_alloc(size_t bytes)
-{
-	return __hrt_isp_css_mm_alloc(bytes, my_userptr,
-				      my_num_pages, my_usr_type, false);
-}
-
-ia_css_ptr hrt_isp_css_mm_alloc_user_ptr(size_t bytes, void *userptr,
-				    unsigned int num_pages,
-				    enum hrt_userptr_type type,
-				    bool cached)
-{
-	return __hrt_isp_css_mm_alloc(bytes, userptr, num_pages,
-				      type, cached);
-}
-
-ia_css_ptr hrt_isp_css_mm_alloc_cached(size_t bytes)
-{
-	if (my_userptr == NULL)
-		return hmm_alloc(bytes, HMM_BO_PRIVATE, 0, NULL,
-						HMM_CACHED);
-	else {
-		if (my_num_pages < ((__page_align(bytes)) >> PAGE_SHIFT))
-			dev_err(atomisp_dev,
-					"user space memory size is less"
-					" than the expected size..\n");
-		else if (my_num_pages > ((__page_align(bytes)) >> PAGE_SHIFT))
-			dev_err(atomisp_dev,
-					"user space memory size is"
-					" large than the expected size..\n");
-
-		return hmm_alloc(bytes, HMM_BO_USER, 0,
-						my_userptr, HMM_CACHED);
-	}
-}
-
-ia_css_ptr hrt_isp_css_mm_calloc(size_t bytes)
-{
-	ia_css_ptr ptr = hrt_isp_css_mm_alloc(bytes);
-	if (ptr)
-		hmm_set(ptr, 0, bytes);
-	return ptr;
-}
-
-ia_css_ptr hrt_isp_css_mm_calloc_cached(size_t bytes)
-{
-	ia_css_ptr ptr = hrt_isp_css_mm_alloc_cached(bytes);
-	if (ptr)
-		hmm_set(ptr, 0, bytes);
-	return ptr;
-}
-
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.h b/drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.h
deleted file mode 100644
index 15c2dfb6794e..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/hrt/hive_isp_css_mm_hrt.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Support for Medfield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef _hive_isp_css_mm_hrt_h_
-#define _hive_isp_css_mm_hrt_h_
-
-#include <hmm/hmm.h>
-#include <hrt/hive_isp_css_custom_host_hrt.h>
-
-#define HRT_BUF_FLAG_CACHED (1 << 0)
-
-enum hrt_userptr_type {
-	HRT_USR_PTR = 0,
-#ifdef CONFIG_ION
-	HRT_USR_ION,
-#endif
-};
-
-struct hrt_userbuffer_attr {
-	enum hrt_userptr_type	type;
-	unsigned int		pgnr;
-};
-
-void hrt_isp_css_mm_set_user_ptr(void *userptr,
-				unsigned int num_pages, enum hrt_userptr_type);
-
-/* Allocate memory, returns a virtual address */
-ia_css_ptr hrt_isp_css_mm_alloc(size_t bytes);
-ia_css_ptr hrt_isp_css_mm_alloc_user_ptr(size_t bytes, void *userptr,
-				    unsigned int num_pages,
-				    enum hrt_userptr_type,
-				    bool cached);
-ia_css_ptr hrt_isp_css_mm_alloc_cached(size_t bytes);
-
-/* allocate memory and initialize with zeros,
-   returns a virtual address */
-ia_css_ptr hrt_isp_css_mm_calloc(size_t bytes);
-ia_css_ptr hrt_isp_css_mm_calloc_cached(size_t bytes);
-
-#endif /* _hive_isp_css_mm_hrt_h_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm.h b/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm.h
deleted file mode 100644
index 1e135c7c6d9b..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm.h
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef	__HMM_H__
-#define	__HMM_H__
-
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/slab.h>
-#include <linux/mm.h>
-
-#include "hmm/hmm_pool.h"
-#include "ia_css_types.h"
-
-#define HMM_CACHED true
-#define HMM_UNCACHED false
-
-int hmm_pool_register(unsigned int pool_size, enum hmm_pool_type pool_type);
-void hmm_pool_unregister(enum hmm_pool_type pool_type);
-
-int hmm_init(void);
-void hmm_cleanup(void);
-
-ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type,
-		int from_highmem, void *userptr, bool cached);
-void hmm_free(ia_css_ptr ptr);
-int hmm_load(ia_css_ptr virt, void *data, unsigned int bytes);
-int hmm_store(ia_css_ptr virt, const void *data, unsigned int bytes);
-int hmm_set(ia_css_ptr virt, int c, unsigned int bytes);
-int hmm_flush(ia_css_ptr virt, unsigned int bytes);
-
-/*
- * get kernel memory physical address from ISP virtual address.
- */
-phys_addr_t hmm_virt_to_phys(ia_css_ptr virt);
-
-/*
- * map ISP memory starts with virt to kernel virtual address
- * by using vmap. return NULL if failed.
- *
- * virt must be the start address of ISP memory (return by hmm_alloc),
- * do not pass any other address.
- */
-void *hmm_vmap(ia_css_ptr virt, bool cached);
-void hmm_vunmap(ia_css_ptr virt);
-
-/*
- * flush the cache for the vmapped buffer.
- * if the buffer has not been vmapped, return directly.
- */
-void hmm_flush_vmap(ia_css_ptr virt);
-
-/*
- * Address translation from ISP shared memory address to kernel virtual address
- * if the memory is not vmmaped,  then do it.
- */
-void *hmm_isp_vaddr_to_host_vaddr(ia_css_ptr ptr, bool cached);
-
-/*
- * Address translation from kernel virtual address to ISP shared memory address
- */
-ia_css_ptr hmm_host_vaddr_to_hrt_vaddr(const void *ptr);
-
-/*
- * map ISP memory starts with virt to specific vma.
- *
- * used for mmap operation.
- *
- * virt must be the start address of ISP memory (return by hmm_alloc),
- * do not pass any other address.
- */
-int hmm_mmap(struct vm_area_struct *vma, ia_css_ptr virt);
-
-/* show memory statistic
- */
-void hmm_show_mem_stat(const char *func, const int line);
-
-/* init memory statistic
- */
-void hmm_init_mem_stat(int res_pgnr, int dyc_en, int dyc_pgnr);
-
-extern bool dypool_enable;
-extern unsigned int dypool_pgnr;
-extern struct hmm_bo_device bo_device;
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_bo.h b/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_bo.h
deleted file mode 100644
index bd44ebbc427c..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_bo.h
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef	__HMM_BO_H__
-#define	__HMM_BO_H__
-
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/list.h>
-#include <linux/spinlock.h>
-#include <linux/mutex.h>
-#include "mmu/isp_mmu.h"
-#include "hmm/hmm_common.h"
-#include "ia_css_types.h"
-
-#define	check_bodev_null_return(bdev, exp)	\
-		check_null_return(bdev, exp, \
-			"NULL hmm_bo_device.\n")
-
-#define	check_bodev_null_return_void(bdev)	\
-		check_null_return_void(bdev, \
-			"NULL hmm_bo_device.\n")
-
-#define	check_bo_status_yes_goto(bo, _status, label) \
-	var_not_equal_goto((bo->status & (_status)), (_status), \
-			label, \
-			"HMM buffer status not contain %s.\n", \
-			#_status)
-
-#define	check_bo_status_no_goto(bo, _status, label) \
-	var_equal_goto((bo->status & (_status)), (_status), \
-			label, \
-			"HMM buffer status contains %s.\n", \
-			#_status)
-
-#define rbtree_node_to_hmm_bo(root_node)	\
-	container_of((root_node), struct hmm_buffer_object, node)
-
-#define	list_to_hmm_bo(list_ptr)	\
-	list_entry((list_ptr), struct hmm_buffer_object, list)
-
-#define	kref_to_hmm_bo(kref_ptr)	\
-	list_entry((kref_ptr), struct hmm_buffer_object, kref)
-
-#define	check_bo_null_return(bo, exp)	\
-	check_null_return(bo, exp, "NULL hmm buffer object.\n")
-
-#define	check_bo_null_return_void(bo)	\
-	check_null_return_void(bo, "NULL hmm buffer object.\n")
-
-#define	HMM_MAX_ORDER		3
-#define	HMM_MIN_ORDER		0
-
-#define	ISP_VM_START	0x0
-#define	ISP_VM_SIZE	(0x7FFFFFFF)	/* 2G address space */
-#define	ISP_PTR_NULL	NULL
-
-#define	HMM_BO_DEVICE_INITED	0x1
-
-enum hmm_bo_type {
-	HMM_BO_PRIVATE,
-	HMM_BO_SHARE,
-	HMM_BO_USER,
-#ifdef CONFIG_ION
-	HMM_BO_ION,
-#endif
-	HMM_BO_LAST,
-};
-
-enum hmm_page_type {
-	HMM_PAGE_TYPE_RESERVED,
-	HMM_PAGE_TYPE_DYNAMIC,
-	HMM_PAGE_TYPE_GENERAL,
-};
-
-#define	HMM_BO_MASK		0x1
-#define	HMM_BO_FREE		0x0
-#define	HMM_BO_ALLOCED	0x1
-#define	HMM_BO_PAGE_ALLOCED	0x2
-#define	HMM_BO_BINDED		0x4
-#define	HMM_BO_MMAPED		0x8
-#define	HMM_BO_VMAPED		0x10
-#define	HMM_BO_VMAPED_CACHED	0x20
-#define	HMM_BO_ACTIVE		0x1000
-#define	HMM_BO_MEM_TYPE_USER     0x1
-#define	HMM_BO_MEM_TYPE_PFN      0x2
-
-struct hmm_bo_device {
-	struct isp_mmu		mmu;
-
-	/* start/pgnr/size is used to record the virtual memory of this bo */
-	unsigned int start;
-	unsigned int pgnr;
-	unsigned int size;
-
-	/* list lock is used to protect the entire_bo_list */
-	spinlock_t	list_lock;
-#ifdef CONFIG_ION
-	struct ion_client	*iclient;
-#endif
-	int flag;
-
-	/* linked list for entire buffer object */
-	struct list_head entire_bo_list;
-	/* rbtree for maintain entire allocated vm */
-	struct rb_root allocated_rbtree;
-	/* rbtree for maintain entire free vm */
-	struct rb_root free_rbtree;
-	struct mutex rbtree_mutex;
-	struct kmem_cache *bo_cache;
-};
-
-struct hmm_page_object {
-	struct page		*page;
-	enum hmm_page_type	type;
-};
-
-struct hmm_buffer_object {
-	struct hmm_bo_device	*bdev;
-	struct list_head	list;
-	struct kref	kref;
-
-	/* mutex protecting this BO */
-	struct mutex		mutex;
-	enum hmm_bo_type	type;
-	struct hmm_page_object	*page_obj;	/* physical pages */
-	int		from_highmem;
-	int		mmap_count;
-#ifdef CONFIG_ION
-	struct ion_handle	*ihandle;
-#endif
-	int		status;
-	int		mem_type;
-	void		*vmap_addr; /* kernel virtual address by vmap */
-
-	struct rb_node	node;
-	unsigned int	start;
-	unsigned int	end;
-	unsigned int	pgnr;
-	/*
-	 * When insert a bo which has the same pgnr with an existed
-	 * bo node in the free_rbtree, using "prev & next" pointer
-	 * to maintain a bo linked list instead of insert this bo
-	 * into free_rbtree directly, it will make sure each node
-	 * in free_rbtree has different pgnr.
-	 * "prev & next" default is NULL.
-	 */
-	struct hmm_buffer_object	*prev;
-	struct hmm_buffer_object	*next;
-};
-
-struct hmm_buffer_object *hmm_bo_alloc(struct hmm_bo_device *bdev,
-				unsigned int pgnr);
-
-void hmm_bo_release(struct hmm_buffer_object *bo);
-
-int hmm_bo_device_init(struct hmm_bo_device *bdev,
-				struct isp_mmu_client *mmu_driver,
-				unsigned int vaddr_start, unsigned int size);
-
-/*
- * clean up all hmm_bo_device related things.
- */
-void hmm_bo_device_exit(struct hmm_bo_device *bdev);
-
-/*
- * whether the bo device is inited or not.
- */
-int hmm_bo_device_inited(struct hmm_bo_device *bdev);
-
-/*
- * increse buffer object reference.
- */
-void hmm_bo_ref(struct hmm_buffer_object *bo);
-
-/*
- * decrese buffer object reference. if reference reaches 0,
- * release function of the buffer object will be called.
- *
- * this call is also used to release hmm_buffer_object or its
- * upper level object with it embedded in. you need to call
- * this function when it is no longer used.
- *
- * Note:
- *
- * user dont need to care about internal resource release of
- * the buffer object in the release callback, it will be
- * handled internally.
- *
- * this call will only release internal resource of the buffer
- * object but will not free the buffer object itself, as the
- * buffer object can be both pre-allocated statically or
- * dynamically allocated. so user need to deal with the release
- * of the buffer object itself manually. below example shows
- * the normal case of using the buffer object.
- *
- *	struct hmm_buffer_object *bo = hmm_bo_create(bdev, pgnr);
- *	......
- *	hmm_bo_unref(bo);
- *
- * or:
- *
- *	struct hmm_buffer_object bo;
- *
- *	hmm_bo_init(bdev, &bo, pgnr, NULL);
- *	...
- *	hmm_bo_unref(&bo);
- */
-void hmm_bo_unref(struct hmm_buffer_object *bo);
-
-
-/*
- * allocate/free physical pages for the bo. will try to alloc mem
- * from highmem if from_highmem is set, and type indicate that the
- * pages will be allocated by using video driver (for share buffer)
- * or by ISP driver itself.
- */
-
-
-int hmm_bo_allocated(struct hmm_buffer_object *bo);
-
-
-/*
- * allocate/free physical pages for the bo. will try to alloc mem
- * from highmem if from_highmem is set, and type indicate that the
- * pages will be allocated by using video driver (for share buffer)
- * or by ISP driver itself.
- */
-int hmm_bo_alloc_pages(struct hmm_buffer_object *bo,
-		enum hmm_bo_type type, int from_highmem,
-		void *userptr, bool cached);
-void hmm_bo_free_pages(struct hmm_buffer_object *bo);
-int hmm_bo_page_allocated(struct hmm_buffer_object *bo);
-
-/*
- * get physical page info of the bo.
- */
-int hmm_bo_get_page_info(struct hmm_buffer_object *bo,
-		struct hmm_page_object **page_obj, int *pgnr);
-
-/*
- * bind/unbind the physical pages to a virtual address space.
- */
-int hmm_bo_bind(struct hmm_buffer_object *bo);
-void hmm_bo_unbind(struct hmm_buffer_object *bo);
-int hmm_bo_binded(struct hmm_buffer_object *bo);
-
-/*
- * vmap buffer object's pages to contiguous kernel virtual address.
- * if the buffer has been vmaped, return the virtual address directly.
- */
-void *hmm_bo_vmap(struct hmm_buffer_object *bo, bool cached);
-
-/*
- * flush the cache for the vmapped buffer object's pages,
- * if the buffer has not been vmapped, return directly.
- */
-void hmm_bo_flush_vmap(struct hmm_buffer_object *bo);
-
-/*
- * vunmap buffer object's kernel virtual address.
- */
-void hmm_bo_vunmap(struct hmm_buffer_object *bo);
-
-/*
- * mmap the bo's physical pages to specific vma.
- *
- * vma's address space size must be the same as bo's size,
- * otherwise it will return -EINVAL.
- *
- * vma->vm_flags will be set to (VM_RESERVED | VM_IO).
- */
-int hmm_bo_mmap(struct vm_area_struct *vma,
-		struct hmm_buffer_object *bo);
-
-extern struct hmm_pool	dynamic_pool;
-extern struct hmm_pool	reserved_pool;
-
-/*
- * find the buffer object by its virtual address vaddr.
- * return NULL if no such buffer object found.
- */
-struct hmm_buffer_object *hmm_bo_device_search_start(
-		struct hmm_bo_device *bdev, ia_css_ptr vaddr);
-
-/*
- * find the buffer object by its virtual address.
- * it does not need to be the start address of one bo,
- * it can be an address within the range of one bo.
- * return NULL if no such buffer object found.
- */
-struct hmm_buffer_object *hmm_bo_device_search_in_range(
-		struct hmm_bo_device *bdev, ia_css_ptr vaddr);
-
-/*
- * find the buffer object with kernel virtual address vaddr.
- * return NULL if no such buffer object found.
- */
-struct hmm_buffer_object *hmm_bo_device_search_vmap_start(
-		struct hmm_bo_device *bdev, const void *vaddr);
-
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_common.h b/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_common.h
deleted file mode 100644
index 00885203fb14..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_common.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef	__HMM_BO_COMMON_H__
-#define	__HMM_BO_COMMON_H__
-
-#define	HMM_BO_NAME	"HMM"
-
-/*
- * some common use micros
- */
-#define	var_equal_return(var1, var2, exp, fmt, arg ...)	\
-	do { \
-		if ((var1) == (var2)) { \
-			dev_err(atomisp_dev, \
-			fmt, ## arg); \
-			return exp;\
-		} \
-	} while (0)
-
-#define	var_equal_return_void(var1, var2, fmt, arg ...)	\
-	do { \
-		if ((var1) == (var2)) { \
-			dev_err(atomisp_dev, \
-			fmt, ## arg); \
-			return;\
-		} \
-	} while (0)
-
-#define	var_equal_goto(var1, var2, label, fmt, arg ...)	\
-	do { \
-		if ((var1) == (var2)) { \
-			dev_err(atomisp_dev, \
-			fmt, ## arg); \
-			goto label;\
-		} \
-	} while (0)
-
-#define	var_not_equal_goto(var1, var2, label, fmt, arg ...)	\
-	do { \
-		if ((var1) != (var2)) { \
-			dev_err(atomisp_dev, \
-			fmt, ## arg); \
-			goto label;\
-		} \
-	} while (0)
-
-#define	check_null_return(ptr, exp, fmt, arg ...)	\
-		var_equal_return(ptr, NULL, exp, fmt, ## arg)
-
-#define	check_null_return_void(ptr, fmt, arg ...)	\
-		var_equal_return_void(ptr, NULL, fmt, ## arg)
-
-/* hmm_mem_stat is used to trace the hmm mem used by ISP pipe. The unit is page
- * number.
- *
- * res_size:  reserved mem pool size, being allocated from system at system boot time.
- *		res_size >= res_cnt.
- * sys_size:  system mem pool size, being allocated from system at camera running time.
- *		dyc_size:  dynamic mem pool size.
- *		dyc_thr:   dynamic mem pool high watermark.
- *		dyc_size <= dyc_thr.
- * usr_size:  user ptr mem size.
- *
- * res_cnt:   track the mem allocated from reserved pool at camera running time.
- * tol_cnt:   track the total mem used by ISP pipe at camera running time.
- */
-struct _hmm_mem_stat {
-	int res_size;
-	int sys_size;
-	int dyc_size;
-	int dyc_thr;
-	int usr_size;
-	int res_cnt;
-	int tol_cnt;
-};
-
-extern struct _hmm_mem_stat hmm_mem_stat;
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_pool.h b/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_pool.h
deleted file mode 100644
index bf24e44462bc..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_pool.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef __HMM_POOL_H__
-#define __HMM_POOL_H__
-
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/list.h>
-#include <linux/spinlock.h>
-#include <linux/mutex.h>
-#include <linux/kref.h>
-#include "hmm_common.h"
-#include "hmm/hmm_bo.h"
-
-#define ALLOC_PAGE_FAIL_NUM		5
-
-enum hmm_pool_type {
-	HMM_POOL_TYPE_RESERVED,
-	HMM_POOL_TYPE_DYNAMIC,
-};
-
-/**
- * struct hmm_pool_ops  -  memory pool callbacks.
- *
- * @pool_init:		   initialize the memory pool.
- * @pool_exit:		   uninitialize the memory pool.
- * @pool_alloc_pages:	   allocate pages from memory pool.
- * @pool_free_pages:	   free pages to memory pool.
- * @pool_inited:	   check whether memory pool is initialized.
- */
-struct hmm_pool_ops {
-	int (*pool_init)(void **pool, unsigned int pool_size);
-	void (*pool_exit)(void **pool);
-	unsigned int (*pool_alloc_pages)(void *pool,
-					struct hmm_page_object *page_obj,
-					unsigned int size, bool cached);
-	void (*pool_free_pages)(void *pool,
-				struct hmm_page_object *page_obj);
-	int (*pool_inited)(void *pool);
-};
-
-struct hmm_pool {
-	struct hmm_pool_ops	*pops;
-
-	void			*pool_info;
-};
-
-/**
- * struct hmm_reserved_pool_info  - represents reserved pool private data.
- * @pages:			    a array that store physical pages.
- *				    The array is as reserved memory pool.
- * @index:			    to indicate the first blank page number
- *				    in reserved memory pool(pages array).
- * @pgnr:			    the valid page amount in reserved memory
- *				    pool.
- * @list_lock:			    list lock is used to protect the operation
- *				    to reserved memory pool.
- * @flag:			    reserved memory pool state flag.
- */
-struct hmm_reserved_pool_info {
-	struct page		**pages;
-
-	unsigned int		index;
-	unsigned int		pgnr;
-	spinlock_t		list_lock;
-	bool			initialized;
-};
-
-/**
- * struct hmm_dynamic_pool_info  -  represents dynamic pool private data.
- * @pages_list:			    a list that store physical pages.
- *				    The pages list is as dynamic memory pool.
- * @list_lock:			    list lock is used to protect the operation
- *				    to dynamic memory pool.
- * @flag:			    dynamic memory pool state flag.
- * @pgptr_cache:		    struct kmem_cache, manages a cache.
- */
-struct hmm_dynamic_pool_info {
-	struct list_head	pages_list;
-
-	/* list lock is used to protect the free pages block lists */
-	spinlock_t		list_lock;
-
-	struct kmem_cache	*pgptr_cache;
-	bool			initialized;
-
-	unsigned int		pool_size;
-	unsigned int		pgnr;
-};
-
-struct hmm_page {
-	struct page		*page;
-	struct list_head	list;
-};
-
-extern struct hmm_pool_ops	reserved_pops;
-extern struct hmm_pool_ops	dynamic_pops;
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_vm.h b/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_vm.h
deleted file mode 100644
index 52098161082d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/include/hmm/hmm_vm.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef	__HMM_VM_H__
-#define	__HMM_VM_H__
-
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/mutex.h>
-#include <linux/list.h>
-
-struct hmm_vm {
-	unsigned int start;
-	unsigned int pgnr;
-	unsigned int size;
-	struct list_head vm_node_list;
-	spinlock_t lock;
-	struct kmem_cache *cache;
-};
-
-struct hmm_vm_node {
-	struct list_head list;
-	unsigned int start;
-	unsigned int pgnr;
-	unsigned int size;
-	struct hmm_vm *vm;
-};
-#define	ISP_VM_START	0x0
-#define	ISP_VM_SIZE	(0x7FFFFFFF)	/* 2G address space */
-#define	ISP_PTR_NULL	NULL
-
-int hmm_vm_init(struct hmm_vm *vm, unsigned int start,
-		unsigned int size);
-
-void hmm_vm_clean(struct hmm_vm *vm);
-
-struct hmm_vm_node *hmm_vm_alloc_node(struct hmm_vm *vm,
-		unsigned int pgnr);
-
-void hmm_vm_free_node(struct hmm_vm_node *node);
-
-struct hmm_vm_node *hmm_vm_find_node_start(struct hmm_vm *vm,
-		unsigned int addr);
-
-struct hmm_vm_node *hmm_vm_find_node_in_range(struct hmm_vm *vm,
-		unsigned int addr);
-
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/include/mmu/isp_mmu.h b/drivers/staging/media/atomisp/pci/atomisp2/include/mmu/isp_mmu.h
deleted file mode 100644
index 4b2d94a37ea1..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/include/mmu/isp_mmu.h
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-/*
- * ISP MMU driver for classic two-level page tables
- */
-#ifndef	__ISP_MMU_H__
-#define	__ISP_MMU_H__
-
-#include <linux/types.h>
-#include <linux/mutex.h>
-#include <linux/slab.h>
-
-/*
- * do not change these values, the page size for ISP must be the
- * same as kernel's page size.
- */
-#define	ISP_PAGE_OFFSET		12
-#define	ISP_PAGE_SIZE		(1U << ISP_PAGE_OFFSET)
-#define	ISP_PAGE_MASK		(~(phys_addr_t)(ISP_PAGE_SIZE - 1))
-
-#define	ISP_L1PT_OFFSET		22
-#define	ISP_L1PT_MASK		(~((1U << ISP_L1PT_OFFSET) - 1))
-
-#define	ISP_L2PT_OFFSET		12
-#define	ISP_L2PT_MASK		(~(ISP_L1PT_MASK|(~(ISP_PAGE_MASK))))
-
-#define	ISP_L1PT_PTES		1024
-#define	ISP_L2PT_PTES		1024
-
-#define	ISP_PTR_TO_L1_IDX(x)	((((x) & ISP_L1PT_MASK)) \
-					>> ISP_L1PT_OFFSET)
-
-#define	ISP_PTR_TO_L2_IDX(x)	((((x) & ISP_L2PT_MASK)) \
-					>> ISP_L2PT_OFFSET)
-
-#define	ISP_PAGE_ALIGN(x)	(((x) + (ISP_PAGE_SIZE-1)) \
-					& ISP_PAGE_MASK)
-
-#define	ISP_PT_TO_VIRT(l1_idx, l2_idx, offset) do {\
-		((l1_idx) << ISP_L1PT_OFFSET) | \
-		((l2_idx) << ISP_L2PT_OFFSET) | \
-		(offset)\
-} while (0)
-
-#define	pgnr_to_size(pgnr)	((pgnr) << ISP_PAGE_OFFSET)
-#define	size_to_pgnr_ceil(size)	(((size) + (1 << ISP_PAGE_OFFSET) - 1)\
-						>> ISP_PAGE_OFFSET)
-#define	size_to_pgnr_bottom(size)	((size) >> ISP_PAGE_OFFSET)
-
-struct isp_mmu;
-
-struct isp_mmu_client {
-	/*
-	 * const value
-	 *
-	 * @name:
-	 *      driver name
-	 * @pte_valid_mask:
-	 *      should be 1 bit valid data, meaning the value should
-	 *      be power of 2.
-	 */
-	char *name;
-	unsigned int pte_valid_mask;
-	unsigned int null_pte;
-
-	/*
-	 * get page directory base address (physical address).
-	 *
-	 * must be provided.
-	 */
-	unsigned int (*get_pd_base) (struct isp_mmu *mmu, phys_addr_t pd_base);
-	/*
-	 * callback to flush tlb.
-	 *
-	 * tlb_flush_range will at least flush TLBs containing
-	 * address mapping from addr to addr + size.
-	 *
-	 * tlb_flush_all will flush all TLBs.
-	 *
-	 * tlb_flush_all is must be provided. if tlb_flush_range is
-	 * not valid, it will set to tlb_flush_all by default.
-	 */
-	void (*tlb_flush_range) (struct isp_mmu *mmu,
-				 unsigned int addr, unsigned int size);
-	void (*tlb_flush_all) (struct isp_mmu *mmu);
-	unsigned int (*phys_to_pte) (struct isp_mmu *mmu,
-				     phys_addr_t phys);
-	phys_addr_t (*pte_to_phys) (struct isp_mmu *mmu,
-				    unsigned int pte);
-
-};
-
-struct isp_mmu {
-	struct isp_mmu_client *driver;
-	unsigned int l1_pte;
-	int l2_pgt_refcount[ISP_L1PT_PTES];
-	phys_addr_t base_address;
-
-	struct mutex pt_mutex;
-	struct kmem_cache *tbl_cache;
-};
-
-/* flags for PDE and PTE */
-#define	ISP_PTE_VALID_MASK(mmu)	\
-	((mmu)->driver->pte_valid_mask)
-
-#define	ISP_PTE_VALID(mmu, pte)	\
-	((pte) & ISP_PTE_VALID_MASK(mmu))
-
-#define	NULL_PAGE	((phys_addr_t)(-1) & ISP_PAGE_MASK)
-#define	PAGE_VALID(page)	((page) != NULL_PAGE)
-
-/*
- * init mmu with specific mmu driver.
- */
-int isp_mmu_init(struct isp_mmu *mmu, struct isp_mmu_client *driver);
-/*
- * cleanup all mmu related things.
- */
-void isp_mmu_exit(struct isp_mmu *mmu);
-
-/*
- * setup/remove address mapping for pgnr continous physical pages
- * and isp_virt.
- *
- * map/unmap is mutex lock protected, and caller does not have
- * to do lock/unlock operation.
- *
- * map/unmap will not flush tlb, and caller needs to deal with
- * this itself.
- */
-int isp_mmu_map(struct isp_mmu *mmu, unsigned int isp_virt,
-		phys_addr_t phys, unsigned int pgnr);
-
-void isp_mmu_unmap(struct isp_mmu *mmu, unsigned int isp_virt,
-		   unsigned int pgnr);
-
-static inline void isp_mmu_flush_tlb_all(struct isp_mmu *mmu)
-{
-	if (mmu->driver && mmu->driver->tlb_flush_all)
-		mmu->driver->tlb_flush_all(mmu);
-}
-
-#define isp_mmu_flush_tlb isp_mmu_flush_tlb_all
-
-static inline void isp_mmu_flush_tlb_range(struct isp_mmu *mmu,
-		unsigned int start, unsigned int size)
-{
-	if (mmu->driver && mmu->driver->tlb_flush_range)
-		mmu->driver->tlb_flush_range(mmu, start, size);
-}
-
-#endif /* ISP_MMU_H_ */
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/include/mmu/sh_mmu_mrfld.h b/drivers/staging/media/atomisp/pci/atomisp2/include/mmu/sh_mmu_mrfld.h
deleted file mode 100644
index 662e98f41da2..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/include/mmu/sh_mmu_mrfld.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Support for Merrifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef	__SH_MMU_MRFLD_H__
-#define	__SH_MMU_MRFLD_H__
-
-extern struct isp_mmu_client sh_mmu_mrfld;
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/mmu/isp_mmu.c b/drivers/staging/media/atomisp/pci/atomisp2/mmu/isp_mmu.c
deleted file mode 100644
index 198f29f4a324..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp2/mmu/isp_mmu.c
+++ /dev/null
@@ -1,584 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-/*
- * ISP MMU management wrap code
- */
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/gfp.h>
-#include <linux/mm.h>		/* for GFP_ATOMIC */
-#include <linux/slab.h>		/* for kmalloc */
-#include <linux/list.h>
-#include <linux/io.h>
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/string.h>
-#include <linux/errno.h>
-#include <linux/sizes.h>
-
-#ifdef CONFIG_X86
-#include <asm/set_memory.h>
-#endif
-
-#include "atomisp_internal.h"
-#include "mmu/isp_mmu.h"
-
-/*
- * 64-bit x86 processor physical address layout:
- * 0		- 0x7fffffff		DDR RAM	(2GB)
- * 0x80000000	- 0xffffffff		MMIO	(2GB)
- * 0x100000000	- 0x3fffffffffff	DDR RAM	(64TB)
- * So if the system has more than 2GB DDR memory, the lower 2GB occupies the
- * physical address 0 - 0x7fffffff and the rest will start from 0x100000000.
- * We have to make sure memory is allocated from the lower 2GB for devices
- * that are only 32-bit capable(e.g. the ISP MMU).
- *
- * For any confusion, contact bin.gao@intel.com.
- */
-#define NR_PAGES_2GB	(SZ_2G / PAGE_SIZE)
-
-static void free_mmu_map(struct isp_mmu *mmu, unsigned int start_isp_virt,
-				unsigned int end_isp_virt);
-
-static unsigned int atomisp_get_pte(phys_addr_t pt, unsigned int idx)
-{
-	unsigned int *pt_virt = phys_to_virt(pt);
-	return *(pt_virt + idx);
-}
-
-static void atomisp_set_pte(phys_addr_t pt,
-			    unsigned int idx, unsigned int pte)
-{
-	unsigned int *pt_virt = phys_to_virt(pt);
-	*(pt_virt + idx) = pte;
-}
-
-static void *isp_pt_phys_to_virt(phys_addr_t phys)
-{
-	return phys_to_virt(phys);
-}
-
-static phys_addr_t isp_pte_to_pgaddr(struct isp_mmu *mmu,
-				     unsigned int pte)
-{
-	return mmu->driver->pte_to_phys(mmu, pte);
-}
-
-static unsigned int isp_pgaddr_to_pte_valid(struct isp_mmu *mmu,
-					    phys_addr_t phys)
-{
-	unsigned int pte = mmu->driver->phys_to_pte(mmu, phys);
-	return (unsigned int) (pte | ISP_PTE_VALID_MASK(mmu));
-}
-
-/*
- * allocate a uncacheable page table.
- * return physical address.
- */
-static phys_addr_t alloc_page_table(struct isp_mmu *mmu)
-{
-	int i;
-	phys_addr_t page;
-	void *virt;
-
-	/*page table lock may needed here*/
-	/*
-	 * The slab allocator(kmem_cache and kmalloc family) doesn't handle
-	 * GFP_DMA32 flag, so we have to use buddy allocator.
-	 */
-	if (totalram_pages > (unsigned long)NR_PAGES_2GB)
-		virt = (void *)__get_free_page(GFP_KERNEL | GFP_DMA32);
-	else
-		virt = kmem_cache_zalloc(mmu->tbl_cache, GFP_KERNEL);
-	if (!virt)
-		return (phys_addr_t)NULL_PAGE;
-
-	/*
-	 * we need a uncacheable page table.
-	 */
-#ifdef	CONFIG_X86
-	set_memory_uc((unsigned long)virt, 1);
-#endif
-
-	page = virt_to_phys(virt);
-
-	for (i = 0; i < 1024; i++) {
-		/* NEED CHECK */
-		atomisp_set_pte(page, i, mmu->driver->null_pte);
-	}
-
-	return page;
-}
-
-static void free_page_table(struct isp_mmu *mmu, phys_addr_t page)
-{
-	void *virt;
-	page &= ISP_PAGE_MASK;
-	/*
-	 * reset the page to write back before free
-	 */
-	virt = phys_to_virt(page);
-
-#ifdef	CONFIG_X86
-	set_memory_wb((unsigned long)virt, 1);
-#endif
-
-	kmem_cache_free(mmu->tbl_cache, virt);
-}
-
-static void mmu_remap_error(struct isp_mmu *mmu,
-			    phys_addr_t l1_pt, unsigned int l1_idx,
-			    phys_addr_t l2_pt, unsigned int l2_idx,
-			    unsigned int isp_virt, phys_addr_t old_phys,
-			    phys_addr_t new_phys)
-{
-	dev_err(atomisp_dev, "address remap:\n\n"
-		     "\tL1 PT: virt = %p, phys = 0x%llx, "
-		     "idx = %d\n"
-		     "\tL2 PT: virt = %p, phys = 0x%llx, "
-		     "idx = %d\n"
-		     "\told: isp_virt = 0x%x, phys = 0x%llx\n"
-		     "\tnew: isp_virt = 0x%x, phys = 0x%llx\n",
-		     isp_pt_phys_to_virt(l1_pt),
-		     (u64)l1_pt, l1_idx,
-		     isp_pt_phys_to_virt(l2_pt),
-		     (u64)l2_pt, l2_idx, isp_virt,
-		     (u64)old_phys, isp_virt,
-		     (u64)new_phys);
-}
-
-static void mmu_unmap_l2_pte_error(struct isp_mmu *mmu,
-				   phys_addr_t l1_pt, unsigned int l1_idx,
-				   phys_addr_t l2_pt, unsigned int l2_idx,
-				   unsigned int isp_virt, unsigned int pte)
-{
-	dev_err(atomisp_dev, "unmap unvalid L2 pte:\n\n"
-		     "\tL1 PT: virt = %p, phys = 0x%llx, "
-		     "idx = %d\n"
-		     "\tL2 PT: virt = %p, phys = 0x%llx, "
-		     "idx = %d\n"
-		     "\tisp_virt = 0x%x, pte(page phys) = 0x%x\n",
-		     isp_pt_phys_to_virt(l1_pt),
-		     (u64)l1_pt, l1_idx,
-		     isp_pt_phys_to_virt(l2_pt),
-		     (u64)l2_pt, l2_idx, isp_virt,
-		     pte);
-}
-
-static void mmu_unmap_l1_pte_error(struct isp_mmu *mmu,
-				   phys_addr_t l1_pt, unsigned int l1_idx,
-				   unsigned int isp_virt, unsigned int pte)
-{
-	dev_err(atomisp_dev, "unmap unvalid L1 pte (L2 PT):\n\n"
-		     "\tL1 PT: virt = %p, phys = 0x%llx, "
-		     "idx = %d\n"
-		     "\tisp_virt = 0x%x, l1_pte(L2 PT) = 0x%x\n",
-		     isp_pt_phys_to_virt(l1_pt),
-		     (u64)l1_pt, l1_idx, (unsigned int)isp_virt,
-		     pte);
-}
-
-static void mmu_unmap_l1_pt_error(struct isp_mmu *mmu, unsigned int pte)
-{
-	dev_err(atomisp_dev, "unmap unvalid L1PT:\n\n"
-		     "L1PT = 0x%x\n", (unsigned int)pte);
-}
-
-/*
- * Update L2 page table according to isp virtual address and page physical
- * address
- */
-static int mmu_l2_map(struct isp_mmu *mmu, phys_addr_t l1_pt,
-		      unsigned int l1_idx, phys_addr_t l2_pt,
-		      unsigned int start, unsigned int end, phys_addr_t phys)
-{
-	unsigned int ptr;
-	unsigned int idx;
-	unsigned int pte;
-
-	l2_pt &= ISP_PAGE_MASK;
-
-	start = start & ISP_PAGE_MASK;
-	end = ISP_PAGE_ALIGN(end);
-	phys &= ISP_PAGE_MASK;
-
-	ptr = start;
-	do {
-		idx = ISP_PTR_TO_L2_IDX(ptr);
-
-		pte = atomisp_get_pte(l2_pt, idx);
-
-		if (ISP_PTE_VALID(mmu, pte)) {
-			mmu_remap_error(mmu, l1_pt, l1_idx,
-					  l2_pt, idx, ptr, pte, phys);
-
-			/* free all mapped pages */
-			free_mmu_map(mmu, start, ptr);
-
-			return -EINVAL;
-		}
-
-		pte = isp_pgaddr_to_pte_valid(mmu, phys);
-
-		atomisp_set_pte(l2_pt, idx, pte);
-		mmu->l2_pgt_refcount[l1_idx]++;
-		ptr += (1U << ISP_L2PT_OFFSET);
-		phys += (1U << ISP_L2PT_OFFSET);
-	} while (ptr < end && idx < ISP_L2PT_PTES - 1);
-
-	return 0;
-}
-
-/*
- * Update L1 page table according to isp virtual address and page physical
- * address
- */
-static int mmu_l1_map(struct isp_mmu *mmu, phys_addr_t l1_pt,
-		      unsigned int start, unsigned int end,
-		      phys_addr_t phys)
-{
-	phys_addr_t l2_pt;
-	unsigned int ptr, l1_aligned;
-	unsigned int idx;
-	unsigned int l2_pte;
-	int ret;
-
-	l1_pt &= ISP_PAGE_MASK;
-
-	start = start & ISP_PAGE_MASK;
-	end = ISP_PAGE_ALIGN(end);
-	phys &= ISP_PAGE_MASK;
-
-	ptr = start;
-	do {
-		idx = ISP_PTR_TO_L1_IDX(ptr);
-
-		l2_pte = atomisp_get_pte(l1_pt, idx);
-
-		if (!ISP_PTE_VALID(mmu, l2_pte)) {
-			l2_pt = alloc_page_table(mmu);
-			if (l2_pt == NULL_PAGE) {
-				dev_err(atomisp_dev,
-					     "alloc page table fail.\n");
-
-				/* free all mapped pages */
-				free_mmu_map(mmu, start, ptr);
-
-				return -ENOMEM;
-			}
-
-			l2_pte = isp_pgaddr_to_pte_valid(mmu, l2_pt);
-
-			atomisp_set_pte(l1_pt, idx, l2_pte);
-			mmu->l2_pgt_refcount[idx] = 0;
-		}
-
-		l2_pt = isp_pte_to_pgaddr(mmu, l2_pte);
-
-		l1_aligned = (ptr & ISP_PAGE_MASK) + (1U << ISP_L1PT_OFFSET);
-
-		if (l1_aligned < end) {
-			ret = mmu_l2_map(mmu, l1_pt, idx,
-					   l2_pt, ptr, l1_aligned, phys);
-			phys += (l1_aligned - ptr);
-			ptr = l1_aligned;
-		} else {
-			ret = mmu_l2_map(mmu, l1_pt, idx,
-					   l2_pt, ptr, end, phys);
-			phys += (end - ptr);
-			ptr = end;
-		}
-
-		if (ret) {
-			dev_err(atomisp_dev, "setup mapping in L2PT fail.\n");
-
-			/* free all mapped pages */
-			free_mmu_map(mmu, start, ptr);
-
-			return -EINVAL;
-		}
-	} while (ptr < end && idx < ISP_L1PT_PTES);
-
-	return 0;
-}
-
-/*
- * Update page table according to isp virtual address and page physical
- * address
- */
-static int mmu_map(struct isp_mmu *mmu, unsigned int isp_virt,
-		   phys_addr_t phys, unsigned int pgnr)
-{
-	unsigned int start, end;
-	phys_addr_t l1_pt;
-	int ret;
-
-	mutex_lock(&mmu->pt_mutex);
-	if (!ISP_PTE_VALID(mmu, mmu->l1_pte)) {
-		/*
-		 * allocate 1 new page for L1 page table
-		 */
-		l1_pt = alloc_page_table(mmu);
-		if (l1_pt == NULL_PAGE) {
-			dev_err(atomisp_dev, "alloc page table fail.\n");
-			mutex_unlock(&mmu->pt_mutex);
-			return -ENOMEM;
-		}
-
-		/*
-		 * setup L1 page table physical addr to MMU
-		 */
-		mmu->base_address = l1_pt;
-		mmu->l1_pte = isp_pgaddr_to_pte_valid(mmu, l1_pt);
-		memset(mmu->l2_pgt_refcount, 0, sizeof(int) * ISP_L1PT_PTES);
-	}
-
-	l1_pt = isp_pte_to_pgaddr(mmu, mmu->l1_pte);
-
-	start = (isp_virt) & ISP_PAGE_MASK;
-	end = start + (pgnr << ISP_PAGE_OFFSET);
-	phys &= ISP_PAGE_MASK;
-
-	ret = mmu_l1_map(mmu, l1_pt, start, end, phys);
-
-	if (ret)
-		dev_err(atomisp_dev, "setup mapping in 